From f28a6bf46afd73de42842e7ca6732b27ac58c25d Mon Sep 17 00:00:00 2001 From: Alex Date: Mon, 17 Aug 2026 18:57:51 -0400 Subject: [PATCH 01/10] Gate PRISM Postgres test containers on the real server (#134) The scale fixture step failed on main with: psql: error: connection to server on socket "/var/run/postgresql/.s.PGSQL.5432" failed: No such file or directory The Postgres image initialises a fresh data directory by running a temporary server that listens on the unix socket only, then shuts it down and starts the real one. `pg_isready` over that socket reports "accepting connections" during initialisation, so the harness proceeded against the throwaway server and the very next call raced its shutdown -- hitting `database "qbit" does not exist`, `the database system is shutting down`, or the missing socket once the file was removed. The window is roughly a second, which is why the scale step was the one to break: the preceding ledger step had already pulled the image, so its container reached the temporary-server phase inside the wait loop's first probe. Gate on the TCP listener instead -- absent until the real server is up, since the temporary one runs with listen_addresses='' -- and confirm with a real query over the socket the tests actually use. The probe now lives in test/prism-postgres-lib.sh and replaces the identical racy loop in all six PRISM test scripts. Measured against postgres:16-alpine, 30 psql calls issued immediately after the gate: 19-21 failures per 30 with the old socket probe, 0 per 30 with this one. --- test/prism-postgres-lib.sh | 43 +++++++++++++++++++++++ test/test-prism-combined-regtest.sh | 12 ++----- test/test-prism-postgres-ledger.sh | 13 +++---- test/test-prism-postgres-native-ledger.sh | 13 +++---- test/test-prism-postgres-scale.sh | 13 +++---- test/test-prism-postgres-throughput.sh | 13 +++---- test/test-prism-stratum-regtest-live.sh | 12 ++----- 7 files changed, 65 insertions(+), 54 deletions(-) create mode 100644 test/prism-postgres-lib.sh diff --git a/test/prism-postgres-lib.sh b/test/prism-postgres-lib.sh new file mode 100644 index 000000000..0493cc034 --- /dev/null +++ b/test/prism-postgres-lib.sh @@ -0,0 +1,43 @@ +#!/usr/bin/env bash + +set -euo pipefail + +# Readiness gate for the throwaway Postgres containers the PRISM tests spin up. +# +# The official Postgres image initialises a fresh data directory by running a +# temporary server that listens on the unix socket only (listen_addresses=''), +# then shuts it down and starts the real one. `pg_isready` over that socket +# therefore reports "accepting connections" during initialisation, and the very +# next client call can hit `FATAL: database "qbit" does not exist`, `FATAL: the +# database system is shutting down`, or -- once the socket file is gone -- +# `connection to server on socket "/var/run/postgresql/.s.PGSQL.5432" failed: +# No such file or directory`. The window is short, so the race only bites when +# the image is already cached and the first probe lands inside it. +# +# Gate on the TCP listener instead (absent until the real server is up) and +# confirm with a real query over the socket the tests actually use. +wait_for_prism_postgres_container() { + local container="$1" + local user="${2:-qbit}" + local database="${3:-qbit}" + local timeout_seconds="${4:-120}" + local deadline=$((SECONDS + timeout_seconds)) + + while :; do + if docker exec "${container}" \ + pg_isready -h 127.0.0.1 -U "${user}" -d "${database}" >/dev/null 2>&1 && + echo 'SELECT 1;' | docker exec -i "${container}" \ + psql --no-psqlrc --set ON_ERROR_STOP=1 -U "${user}" -d "${database}" \ + >/dev/null 2>&1; then + return 0 + fi + + if [[ "${SECONDS}" -ge "${deadline}" ]]; then + echo "timed out waiting for PRISM Postgres container: ${container}" >&2 + docker logs "${container}" >&2 || true + return 1 + fi + + sleep 1 + done +} diff --git a/test/test-prism-combined-regtest.sh b/test/test-prism-combined-regtest.sh index 981e9fc4e..c392ea4ad 100644 --- a/test/test-prism-combined-regtest.sh +++ b/test/test-prism-combined-regtest.sh @@ -3,6 +3,8 @@ set -euo pipefail ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" source "${ROOT_DIR}/test/test-lib.sh" +# shellcheck source=test/prism-postgres-lib.sh +source "${ROOT_DIR}/test/prism-postgres-lib.sh" DATADIR="${DATADIR:-$(mktemp -d -t qbit-prism-combined.XXXXXX)}" RPC_USER="${RPC_USER:-qbitrpc}" @@ -79,15 +81,7 @@ else postgres_started=1 PSQL_COMMAND="docker exec -i ${POSTGRES_CONTAINER} psql -U qbit -d qbit" - deadline=$((SECONDS + 60)) - until docker exec "${POSTGRES_CONTAINER}" pg_isready -U qbit -d qbit >/dev/null 2>&1; do - if [[ "${SECONDS}" -ge "${deadline}" ]]; then - echo "timed out waiting for PRISM Postgres container" >&2 - docker logs "${POSTGRES_CONTAINER}" >&2 || true - exit 1 - fi - sleep 1 - done + wait_for_prism_postgres_container "${POSTGRES_CONTAINER}" fi "${QBITD_BIN}" \ diff --git a/test/test-prism-postgres-ledger.sh b/test/test-prism-postgres-ledger.sh index ae26e1ceb..403c84175 100644 --- a/test/test-prism-postgres-ledger.sh +++ b/test/test-prism-postgres-ledger.sh @@ -2,6 +2,9 @@ set -euo pipefail ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +# shellcheck source=test/prism-postgres-lib.sh +source "${ROOT_DIR}/test/prism-postgres-lib.sh" + POSTGRES_IMAGE="${QBIT_PRISM_POSTGRES_IMAGE:-postgres:16-alpine}" POSTGRES_CONTAINER="${QBIT_PRISM_POSTGRES_CONTAINER:-qbit-prism-ledger-pg-$$}" @@ -47,15 +50,7 @@ else -e POSTGRES_DB=qbit \ "${POSTGRES_IMAGE}" >/dev/null - deadline=$((SECONDS + 60)) - until docker exec "${POSTGRES_CONTAINER}" pg_isready -U qbit -d qbit >/dev/null 2>&1; do - if [[ "${SECONDS}" -ge "${deadline}" ]]; then - echo "timed out waiting for PRISM Postgres container" >&2 - docker logs "${POSTGRES_CONTAINER}" >&2 || true - exit 1 - fi - sleep 1 - done + wait_for_prism_postgres_container "${POSTGRES_CONTAINER}" PSQL_COMMAND="docker exec -i ${POSTGRES_CONTAINER} psql -U qbit -d qbit" fi diff --git a/test/test-prism-postgres-native-ledger.sh b/test/test-prism-postgres-native-ledger.sh index ee42dc065..8bdac55bf 100755 --- a/test/test-prism-postgres-native-ledger.sh +++ b/test/test-prism-postgres-native-ledger.sh @@ -9,6 +9,9 @@ set -euo pipefail ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +# shellcheck source=test/prism-postgres-lib.sh +source "${ROOT_DIR}/test/prism-postgres-lib.sh" + POSTGRES_IMAGE="${QBIT_PRISM_POSTGRES_IMAGE:-postgres:16-alpine}" POSTGRES_CONTAINER="${QBIT_PRISM_POSTGRES_CONTAINER:-qbit-prism-native-ledger-pg-$$}" @@ -43,15 +46,7 @@ docker run \ -e POSTGRES_DB=qbit \ "${POSTGRES_IMAGE}" >/dev/null -deadline=$((SECONDS + 60)) -until docker exec "${POSTGRES_CONTAINER}" pg_isready -U qbit -d qbit >/dev/null 2>&1; do - if [[ "${SECONDS}" -ge "${deadline}" ]]; then - echo "timed out waiting for PRISM Postgres container" >&2 - docker logs "${POSTGRES_CONTAINER}" >&2 || true - exit 1 - fi - sleep 1 -done +wait_for_prism_postgres_container "${POSTGRES_CONTAINER}" HOST_PORT="$(docker port "${POSTGRES_CONTAINER}" 5432/tcp | head -n 1 | awk -F: '{print $NF}')" if [[ -z "${HOST_PORT}" ]]; then diff --git a/test/test-prism-postgres-scale.sh b/test/test-prism-postgres-scale.sh index 44f4fee95..46a8812a0 100755 --- a/test/test-prism-postgres-scale.sh +++ b/test/test-prism-postgres-scale.sh @@ -23,6 +23,9 @@ set -euo pipefail # QBIT_PRISM_SCALE_WINDOW_READ_BUDGET_MS warm budget (default 2000) ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +# shellcheck source=test/prism-postgres-lib.sh +source "${ROOT_DIR}/test/prism-postgres-lib.sh" + POSTGRES_IMAGE="${QBIT_PRISM_POSTGRES_IMAGE:-postgres:16-alpine}" POSTGRES_CONTAINER="${QBIT_PRISM_POSTGRES_CONTAINER:-qbit-prism-scale-pg-$$}" @@ -66,15 +69,7 @@ else -e POSTGRES_DB=qbit \ "${POSTGRES_IMAGE}" >/dev/null - deadline=$((SECONDS + 60)) - until docker exec "${POSTGRES_CONTAINER}" pg_isready -U qbit -d qbit >/dev/null 2>&1; do - if [[ "${SECONDS}" -ge "${deadline}" ]]; then - echo "timed out waiting for PRISM Postgres container" >&2 - docker logs "${POSTGRES_CONTAINER}" >&2 || true - exit 1 - fi - sleep 1 - done + wait_for_prism_postgres_container "${POSTGRES_CONTAINER}" PSQL_COMMAND="docker exec -i ${POSTGRES_CONTAINER} psql -U qbit -d qbit" fi diff --git a/test/test-prism-postgres-throughput.sh b/test/test-prism-postgres-throughput.sh index 6e25c501f..3d06819f5 100644 --- a/test/test-prism-postgres-throughput.sh +++ b/test/test-prism-postgres-throughput.sh @@ -2,6 +2,9 @@ set -euo pipefail ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +# shellcheck source=test/prism-postgres-lib.sh +source "${ROOT_DIR}/test/prism-postgres-lib.sh" + POSTGRES_IMAGE="${QBIT_PRISM_POSTGRES_IMAGE:-postgres:16-alpine}" POSTGRES_CONTAINER="${QBIT_PRISM_POSTGRES_CONTAINER:-qbit-prism-throughput-pg-$$}" SHARE_COUNT="${QBIT_PRISM_THROUGHPUT_SHARES:-20000}" @@ -33,15 +36,7 @@ docker run \ -e POSTGRES_DB=qbit \ "${POSTGRES_IMAGE}" >/dev/null -deadline=$((SECONDS + 60)) -until docker exec "${POSTGRES_CONTAINER}" pg_isready -U qbit -d qbit >/dev/null 2>&1; do - if [[ "${SECONDS}" -ge "${deadline}" ]]; then - echo "timed out waiting for PRISM Postgres container" >&2 - docker logs "${POSTGRES_CONTAINER}" >&2 || true - exit 1 - fi - sleep 1 -done +wait_for_prism_postgres_container "${POSTGRES_CONTAINER}" mkdir -p "$(dirname "${REPORT_PATH}")" diff --git a/test/test-prism-stratum-regtest-live.sh b/test/test-prism-stratum-regtest-live.sh index cb34af1e4..4fffce84d 100755 --- a/test/test-prism-stratum-regtest-live.sh +++ b/test/test-prism-stratum-regtest-live.sh @@ -3,6 +3,8 @@ set -euo pipefail ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" source "${ROOT_DIR}/test/test-lib.sh" +# shellcheck source=test/prism-postgres-lib.sh +source "${ROOT_DIR}/test/prism-postgres-lib.sh" DATADIR="${DATADIR:-$(mktemp -d -t qbit-prism-live.XXXXXX)}" RPC_USER="${RPC_USER:-qbitrpc}" RPC_PASSWORD="${RPC_PASSWORD:-change-this}" @@ -103,15 +105,7 @@ if [[ "${POSTGRES_ENABLED}" == "1" ]]; then "${POSTGRES_IMAGE}" >/dev/null postgres_started=1 PSQL_COMMAND="docker exec -i ${POSTGRES_CONTAINER} psql -U qbit -d qbit" - deadline=$((SECONDS + 60)) - until docker exec "${POSTGRES_CONTAINER}" pg_isready -U qbit -d qbit >/dev/null 2>&1; do - if [[ "${SECONDS}" -ge "${deadline}" ]]; then - echo "timed out waiting for PRISM Postgres container" >&2 - docker logs "${POSTGRES_CONTAINER}" >&2 || true - exit 1 - fi - sleep 1 - done + wait_for_prism_postgres_container "${POSTGRES_CONTAINER}" fi fi From 3e233d22abc6f0466473d9c816449c4d8432685c Mon Sep 17 00:00:00 2001 From: Alex Date: Fri, 11 Sep 2026 12:54:37 -0400 Subject: [PATCH 02/10] v2.0.0 chore: consolidate release for main (#246) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Shard PRISM coordinator tests by domain Re-cut of the original test sharding against current 1.x.x (36267d6): the two monolithic coordinator test files split into nine domain shard files plus two shared no-environment support modules, with the current test bodies as the source of truth. All 609 sharded test methods are conserved exactly once, and dependent test files change import paths only. Co-Authored-By: Claude Fable 5 * Extract PRISM coordinator core owners Re-cut of the original core-owner extraction against current 1.x.x (36267d6). Fifteen owner modules leave prism_coordinator.py at 15,436 lines (from 31,041): coordinator_config (immutable snapshot construction, all env reads), rpc (JsonRpc + transport retry policy), bounded_executor, ctv_runtime, progress_health, template_artifacts, bundle_compiler, job_bundle, job_delivery, tip_refresh, stratum_session, share_writer, payout_state, background_services (incl. the ledger lease heartbeat service), coordinator_shutdown. The old extraction commit served as the placement and wiring map; every moved body is the current coordinator implementation, so all upstream fixes through #120 are carried, including the landing-class deadlines, degraded accepted-parent previews, append/anchor fencing, tip-refresh epoch waves and lease fencing. Coordinator compatibility facades, private-name aliases and descriptor- routed legacy fields preserve every existing import path and test seam. share_ledger gains typed exact/conflict share replay consumed by the share writer; the Postgres ledger shell test asserts the new replay semantics. Nine owner-boundary test files are added (116 tests) and support modules gain typed-recovery fakes; no existing behavioral assertion is weakened. Co-Authored-By: Claude Fable 5 * Extract PRISM audit artifact ownership Re-cut of the original audit-artifact extraction against the rebuilt PR 74 layer. lab/prism/audit_artifacts.py owns every audit path and filesystem mutation under one identity-checked root: candidate-artifact lifecycle with compiler inode-transfer adoption, ordered publication (thread + lock-file process fencing) keyed by a durable audit publication ordinal (SQL sequence/column/backfill migration + ledger confirm/reactivate/floor, with an in-memory pool-block state machine for the compatibility ledger), legacy-evidence upgrade, and the #88 gap-recovery flow with the ledger range backfill threaded in as a per-call callback. The coordinator and PsqlShareLedger keep one-line delegating shims; verify_bundle honors the configured bundle build timeout. Bodies for pre-existing behavior come from the current tree, so all upstream fixes through #120 are preserved; the Postgres ledger shell gate gains publication-ordinal assertions plus three A1 gates (transition parity, migration, cross-process fencing), and the canonicalizer round-trip contract is pinned by new crate regression tests. Co-Authored-By: Claude Fable 5 * Extract PRISM block candidate submission Re-cut of the original B1 extraction against the rebuilt PR 75 layer. lab/prism/block_candidates.py owns the durable candidate codec and the full submit-before-accounting lane from the current tree: node-submission workers with hard RPC deadlines, same-hash disposition leases and terminal outcomes, the accounting queue/thread handoff, startup and outbox replay with quarantine, acceptance evidence and chain probes, retry merge/pacing/backoff, block-work heartbeats and the DB/RPC call workers including the landing-class deadline split with per-block escalation and the startup enumeration-owed gate. The coordinator keeps the node submission finalization side, thin delegates, and descriptor-routed compatibility state; replay adoption registers credit floors through the share-writer owner before preview registration so one bad row cannot abort the rest. Co-Authored-By: Claude Fable 5 * Extract PRISM share submission Re-cut of the original share-submission extraction against the rebuilt PR 76 layer. lab/prism/share_submission.py decomposes the submit hot path into pure parse/validate/classify functions plus ShareSubmissionService, with bodies from the current tree: the cross-connection retained-submit authority (same-tip-only guard, retained version mask, original extranonce), stale-grace block-solve drop accounting ahead of low-difficulty classification, and the durable-outbox fast paths in both block routes wired to the block-candidate owner's facades. The recent- share duplicate window becomes RecentShareIndex with oldest-insertion eviction, and the service owns share-ack histogram state behind the existing coordinator routing seam. The coordinator keeps a one-line decorated handle_submit, the rejection-constant re-exports, and descriptor-routed compatibility state. Co-Authored-By: Claude Fable 5 * Extract PRISM vardiff service and block finalization Re-cut of the original vardiff/finalization extraction against the rebuilt PR 77 layer. lab/prism/block_finalization.py owns the post-offer B3 lane from the current tree — admission with acceptance-evidence short-circuits, landing under the append-epoch fences and window-proof decision, the CTV/share-credit tail, ordered audit publication through the PR 75 store, and accepted-block accounting — packaged in the six bounded finalization phases with candidate-interarrival metrics. The block-candidate owner's ports are untouched: its targets remain the coordinator facades, which now delegate to the service, and landing round-trips through the coordinator seam so existing monkeypatches intercept. lab/prism/vardiff_service.py owns vardiff configuration, share windows, the idle retarget engine and its executor state behind twelve adopted compatibility fields, keeping the difficulty-direction clean-jobs rule. The B3 decision document is refreshed for the node-first accounting-actor architecture. Co-Authored-By: Claude Fable 5 * Extract PRISM observability and audit HTTP Re-cut of the original observability extraction against the rebuilt PR 78 layer. lab/prism/observability.py owns mining-delivery and semantic- currency policy, the cached base-health snapshot with live progress overlay, the starting-state 503 and warm-up loop, and a new cached complete-metrics snapshot pipeline with bounded collector diagnostics. lab/prism/audit_http.py owns the audit listener lifecycle and route handler behind a typed port, with genuinely bounded shutdown: a stop racing startup before the serve loop commits now cancels the startup window and returns within its deadline, pinned by new socket-lifecycle regressions. The coordinator keeps delegating shims, the health-attribute compatibility properties, the renamed complete-metrics renderer, and the listener-binds-before-warm-up startup ordering with its phase stamps; the compose environment gains the metrics refresh interval alongside every existing pass-through. Co-Authored-By: Claude Fable 5 * Complete PRISM coordinator ownership cleanup Re-cut of the original terminal cleanup layer against the rebuilt PR 79 stack. lab/prism/metrics.py owns the complete Prometheus payload and the extracted formatter bodies behind copied owner snapshots, proven byte-identical to the previous renderer by a frozen-reference parity test. lab/prism/reorg_reconciler.py owns reconcile flight admission, the per-tip trusted memo, lookup counters and the bounded proving prefetch executor around the current core state machine, while the coordinator keeps the exact serialized writer/payout/landed-preview adapter and compatibility descriptors. The watchdog moves into a WatchdogService with the coordination-blocked streak, classification, bounded lease release and unconditional hard exit behind the dynamic coordinator seam that the lease heartbeat service also fires. BlockFinalizationService drops its implicit attribute forwarding for an explicit typed port with mechanical coverage, the interim health compatibility properties and forty-plus compatibility re-exports are deleted with every caller repointed, and white-box tests now target the owning services. Co-Authored-By: Claude Fable 5 * Document PRISM refactor review stack Re-derived documentation layer: the review-stack guide, ownership map, invariants, roadmap, A1 contract and validation evidence, updated for the rebuilt stack against current 1.x.x — the node-first finalization split, the re-measured structural audit, and validation results actually produced on this tree (Docker-backed Postgres/A1 gates, Rust workspace, container compile and image build, compose validation; live Stratum targets recorded unavailable for the missing qbitd prerequisite). The ledger operations guide keeps its current landing/replay text and gains the known-outcome replay-resumption contract; the audit publication ordinal comment in the finalization owner is refreshed for reactivation reuse. Co-Authored-By: Claude Fable 5 * Add a deterministic concurrency harness for PRISM, proven against the writer-lease lifecycle (#139) * Add a deterministic concurrency harness and prove it against #123 The concurrency defects this line keeps shipping share a shape: two owners interleave around a PostgreSQL row, and the bad schedule is too rare to reproduce by running the coordinator and hoping. Every one was found in production or in review, never by a test. The nine-slice refactor removed the structural reason for that, so this adds the missing tool: a scheduler that makes an interleaving an input rather than an accident. The harness has three parts. A baton scheduler runs real threads one at a time, stopping only at named checkpoints placed at PostgreSQL statement boundaries, so the controlling test decides who runs next and the resulting schedule is order-stable rather than merely likely. A virtual clock backs every interval the lease lifecycle measures about itself, so nothing sleeps in wall-clock time. An in-memory PostgreSQL model covers the semantics the lease lifecycle actually depends on: tuple locks held for a transaction's lifetime, FOR NO KEY UPDATE SKIP LOCKED declining to queue, per-statement READ COMMITTED snapshots, lock and statement timeouts, session-scoped advisory locks, and pg_stat_activity locker attribution against a snapshot-visible xmax. Substitution runs through ports rather than monkeypatching. LedgerSqlPort and LeaseGuardPort name the two seams PsqlShareLedger already wrote through, and the constructor now takes sql_backend_factory, lease_guard_factory and monotonic alongside the existing lease_retry_sleep. No bound method is reassigned and no clock is patched, so the code under test stays the shipped code: the real retry loop, the real CAS SQL, the real adoption arithmetic, the real fail-closed branches. FakePostgres answers statements by recognising them against fragments lifted from the production SQL, and raises on anything else. Statements belonging to the state machines deferred to follow-up PRs are recognised only well enough to name the follow-up. A fake that silently answers an unrecognised statement drifts from production without anyone noticing; this one fails instead. Issue #123 is the proving ground. Its trigger sequence now runs without a live Postgres: a coordinator vanishes in the gap between a deadline-scoped statement completing and its COMMIT, leaving an open transaction holding the lease tuple lock, and a failover twin parks on that lock inside PsqlShareLedger.__init__ with no deadline anywhere in the system that can release it. The schedule is identical across repeated runs. One test in this commit fails, deliberately. OrphanedLeaseLockFailoverBoundTests asserts the bound #123's fix must establish, and #123 is not fixed on this line: the startup lease upsert runs with no lock_timeout, so nothing bounds the successor's wait. That assertion is written against the fixed behaviour rather than against today's, because weakening it to green would delete the only executable statement of what #123 asks for. It turns green when the fix lands. Refs #128, #123 Co-Authored-By: Claude Opus 5 (1M context) * Cover the remaining writer-lease interleavings, and two model fidelity fixes Completes the writer-lease lifecycle as the harness's proving ground. Three scenario modules, each driving shipped code through the harness rather than re-describing it: Renewal against a competing expiry claim. The heartbeat renews uncontended; an expired row whose skipped lock belongs to a different-identity claimant fails closed, and the control proves it is the combination that raises rather than either half; the documented attribution misfire — this coordinator's own fenced write committing between the verification statement's MVCC snapshot and its live pg_stat_activity probe — is driven at exactly that checkpoint and resolved by exactly one recheck; a claim that commits fences the session. Heartbeat during a long fenced statement. The verification crosses an in-flight fenced write without ever queueing, skipping renewal and resuming it the moment that write commits. The control runs the pre-fix locking spelling into the same lock and watches it die on the guard's statement timeout, which is what makes renewed_count == 0 mean "declined a lock that was really there" rather than "nothing was attempted". An own fenced write outlasting the TTL defers instead of self-fencing, and the same expired row locked by a foreign application_name still fails closed. Takeover and epoch fencing. The advisory guard keeps a same-identity successor out of the lease upsert entirely; adoption silence is measured from the successor's own guard acquisition even when the row is already far staler than the interval; the CAS rewrites the session token and fences the predecessor; a lost CAS re-observes after a fresh full interval instead of refusing the token permanently; and a different epoch refuses an unexpired lease while taking an expired one through the ordinary expiry CAS. Two fidelity fixes to the model, both found while reviewing the above. A cancelled lock wait now reports which bound fired. A caller deadline arrives as SET LOCAL lock_timeout, while the guard session's bound is a connection-level statement_timeout; PostgreSQL reports them with different messages, and the whole point of the SKIP LOCKED heartbeat is that the guard's bound is the fatal one, so blurring them would let a scenario assert the wrong mechanism. Vanishing a coordinator no longer fabricates an orphan on a path that cannot produce one. An explicit transaction's COMMIT is a separate client message, so a vanished client leaves it open forever; an autocommit statement has no such message and PostgreSQL finishes and commits it. #123 is explicit that the orphan shape did not exist before the deadline plumbing arrived, and a model that offered it on the autocommit path would argue for a fix to a defect that path cannot reach. Determinism: eight assert_deterministic tests at 25 repeats each, every one asserting an exact checkpoint schedule rather than merely that runs agree. Twenty consecutive full runs of the five modules produce an identical result. Refs #128 Co-Authored-By: Claude Opus 5 (1M context) * Correct the PostgreSQL snapshot model and the #123 bound An adversarial review pass verified the model's PostgreSQL claims against a scratch PostgreSQL 16.15 cluster and found two defects that mattered. The model re-read committed state after a tuple-lock wait. PostgreSQL takes one snapshot per command in READ COMMITTED: a statement that blocks re-reads only the conflicting tuple, through EvalPlanQual, to evaluate its write predicate, while every other scan in the statement keeps the original snapshot. So a queued acquire reports the pre-wait row and a lease_age_seconds that has silently absorbed the entire wait, where the model reported a fresh row and a small age. That age is the row edge of _writer_lease_adoption_wait_seconds, so the model was hiding the very input that makes the adoption CAS's updated_at predicate load-bearing: in the model the observation was always fresh, and the CAS would have matched even with that predicate deleted. Evaluation now takes the snapshot before any lock wait, resolves write predicates against the freshly locked version, and reports results from the snapshot. Statements that evaluate to SQL NULL return None and go through production's own parse_single_json_value, rather than being smoothed over. That surfaced a real production behaviour the model had been concealing. When two coordinators race the first ever acquire, the loser's DO UPDATE matches nothing and its COALESCE fallback reads a snapshot in which no row exists yet, so the whole statement returns NULL and PsqlShareLedger.__init__ dies with "postgres query returned no JSON" — an error naming neither the lease nor the coordinator holding it. The harness's own contention self-test was constructing exactly that interleaving and turning it into a clean refusal. It now asserts the real error, and a dedicated test pins the contrast against the ordinary "lease is held by" refusal. The deliberately failing #123 test asserted a bound the fix it names would not have satisfied. It advanced the clock once and drained once, so a fix built from lock_timeout plus a bounded retry — which is what #123 proposes — would have left the successor parked on the first of several short deadlines with the test still red, and the person landing that fix looking at a red test and reaching for the weakening this test exists to refuse. It now drives the successor through every deadline it reaches. Both candidate fixes were simulated: lock_timeout alone and lock_timeout with a bounded retry each turn it green, and it stays red on this line. Four smaller model corrections. A non-matching UPDATE no longer queues on the tuple lock, because PostgreSQL evaluates the predicate against the snapshot version and never locks a row it excludes; latent today, but it would have misrepresented any fencing argument, where "the predecessor is blocked" and "the predecessor already knows it lost" are different states. Tuple-lock waiters are granted in arrival order rather than whichever the controller steps first. Pool exhaustion honours the caller's deadline instead of waiting forever. The guard session's serialization is now pinned against _NativePostgresLeaseGuard._query_lock itself, since deleting that lock left every assertion in the harness-level test green. The two new protocols had no executable contract: the repository runs no mypy, they were not runtime-checkable, and the signature test compared parameter names only. They are now @runtime_checkable, asserted against both the production classes and the fakes, and checked with full signatures across every method. LedgerSqlPort.pool_size is dropped, being surface nothing reads. Two production tidies from the same pass. The lease clock is a declared class attribute overridden per instance, rather than a getattr fallback that would silently revert to wall-clock time under a rename. _pool_application_name is injectable, removing the last uncontrolled nondeterminism inside the code under test — lease_locked_by_this_process turns on that value, so a scenario that surfaced it would have failed the determinism check for reasons nothing in the harness explained. Full suite: 2023 tests, failures limited to the five pre-existing macOS os.splice tests and the one intentional #123 assertion. Twenty consecutive runs of the five harness modules are identical. Refs #128, #123 Co-Authored-By: Claude Opus 5 (1M context) * Clean up the harness under a standard Python linter The repository configures no Python linter — CI runs shellcheck, bash -n and compileall — so this is a quality pass on the files this branch adds rather than a fix for a failing check. Run with ruff in Docker. Real findings, fixed: a stale noqa suppressing BLE001 on an `except BaseException` the rule does not apply to; Callable imported from typing rather than collections.abc; an unsorted __all__; re.S in place of re.DOTALL; a three-line if/return that is just its own condition; __enter__ annotated with the concrete class rather than Self, imported under TYPE_CHECKING so no runtime version floor is introduced. The guard serialization test caught a blind `except Exception: pass`, which was hiding what the test actually proves. It now catches the AttributeError the statement itself raises once past the query slot, and asserts it fired: reaching the statement is the evidence, and any other outcome means run_json did not get where the test claims. Two findings are deliberately left. Every test module in this repository carries a shebang without the executable bit, so EXE001 is the house convention and departing from it in six files would be the anomaly. The `noqa: E402` on the deferred share_ledger import is reported unused only because E402 is not enabled in this ad-hoc invocation; the import genuinely does follow a sys.path statement, so the suppression is correct and documents why. Also corrects a comment the review pass flagged as overstated: the baton timeout cannot change what a run does, but it is not free of consequence, and claiming it "cannot make a passing run flaky" overstated a real if remote failure mode. Refs #128 * Mark the #123 bound as an expected failure The assertion is unchanged. What changes is who gets told about it, and when. Left as a plain failure, it made every CI run on this branch red, which is a poor way to communicate one known-open issue: a permanently red check is indistinguishable from a broken one at a glance, and it teaches people to stop reading CI. Three separate readers took the red "Lint, compile, and test" badge for a lint failure, which is the failure mode in miniature. unittest reports an unexpected success as a suite failure, so the marker is self-removing: the day someone fixes #123 the suite goes red and hands them this file, which is exactly when learning about it is useful. Verified rather than assumed — a marked test that starts passing does set wasSuccessful False. The marker is only defensible because the assertion genuinely goes green under a real fix, which it did not before the drive-to-quiescence correction. Both candidate fixes named in #123 were simulated again against this commit: lock_timeout alone, and lock_timeout with a bounded retry, each turns it green and each leaves 2.x.x red. An expectedFailure on a test that would stay red after the fix would be strictly worse than a plain failure, and that is the state this branch was in two commits ago. Full suite: 2023 tests, one expected failure, and the five pre-existing macOS os.splice failures that pass on Linux. Ten consecutive runs of the five harness modules report OK with one expected failure every time. Refs #128, #123 --------- Co-authored-by: Claude Opus 5 (1M context) * Preserve the audit envelope when two found blocks land in one confirm-publish window (#136) * Preserve the audit envelope on interleaved found-block landings Two distinct found blocks can land inside one confirm->publish window. The synchronous miner-connection-thread tail confirms block A under the payout-balance lock and the publication order guard, then releases both before it publishes; the accounting-actor tail lands block B in that gap, allocates the next ordinal and publishes first. The per-hash disposition lease does not serialise two distinct hashes, so nothing prevents the interleaving. Block A then reached publish_success already superseded and returned without writing anything: its live audit envelope -- the block's only published evidence pointer -- was lost permanently and silently. The Postgres audit row, canonical bundle, body, accounting, share credit and settlement all survived; only the public evidence trail was dropped. The old code conflated two invariants behind one floor check. The evidence path is a single mutable pointer that must advance monotonically in sequence, and floor gating is correct for it. The live envelope is a per-block immutable artifact on its own height+hash path, and a superseded ordinal's envelope competes with nothing. Being behind the floor means you may not become current; it does not mean your own envelope is not yours to write. Sequence-versus-floor alone cannot authorise that write, though. A process whose writer lease has been taken over is also behind the floor, and it must stay fenced out of the shared audit root entirely: its in-memory report may be stale, and publishing a stale report as a block's public evidence is worse than leaving the envelope missing. Only the caller knows which it is. The write is therefore opt-in through a new restore_superseded_envelope keyword that defaults to false, and only _publish_finalization_evidence -- the live writer finalizing a block it confirmed itself in this landing -- passes true. A deposed writer replaying stale state never reaches that call site, so the A1 process gate's C3 fencing invariants are preserved exactly. When the write is authorised it routes through _write_superseded_live_envelope_locked, which writes the envelope only when it is absent and logs the event. The publication order guard and the process flock are held throughout, and the write is rolled back if any post-write identity assertion fails. A present envelope is returned untouched, so a same-hash stale replay keeps its existing behaviour exactly. The current evidence reference, the cached identity and retention pruning are not touched, and the publication still reports published=False. The diagnostic goes to stderr: this store is hosted inside worker processes that reserve stdout for a strict JSON protocol stream, and the A1 process gate drives this exact superseded path. Add two regression tests, both failing before the fix at the envelope-existence assertion: the distinct-hash interleaving across the synchronous and actor tails, and the convergence path where the peer is confirmed but not yet published so the earlier block is retried behind the floor. Because the write is opt-in, no existing assertion changes -- the test diff is purely additive. Fixes #133 Co-Authored-By: Claude Opus 5 * Lease-gate the superseded-envelope restore and pin its wiring An adversarial review of the interleaved-landing fix for issue #133 returned four findings; this lands all four. The restore's opt-in rested on "a deposed writer never reaches the call site", which is false for a writer deposed inside its own confirm->publish gap: it confirms with a valid lease, loses it during the gap, and everything from there to the publish call is lease-unfenced. Content divergence was independently impossible -- the report and persistence digests bind to the durably confirmed ordinal -- so this was a coherence defect against the A1 process gate's C3 immobility invariant rather than a corruption path, but that invariant is cheap to keep literally true. _publish_finalization_evidence now asks the coordinator's bounded, non-blocking exact-session fence (require_fresh_lease_for_external_side_effect) for restore authority before entering the publication order guard, holding no store lock across the verification. The asymmetry is deliberate: a clean proof grants restore_superseded_envelope=True, while a lease failure (WriterLeaseRenewalDeferred, ShutdownInProgress) withholds the restore, logs the withholding with the component and exception type, and lets the publication proceed unchanged. The restore is a repair of an evidence pointer, not the publication itself; degrading to False is exactly the previously shipped behaviour, whereas failing the found-block path on a lease hiccup would be a live regression. The fence stays a no-op for ledgers without the writer lease guard, never waits on the lease row, and its internal hard-exit arming for a genuinely dead lease is left intact. The production opt-in was also untested: flipping the call site back to False passed all 254 tests. A new coordinator-level regression test drives the real two-tail topology -- handle_submit's synchronous landing paused inside its confirm->publish gap while a second distinct-hash candidate rides the submitter->accounting-actor handoff, lands, and publishes -- and asserts both blocks' live envelopes exist afterwards. It fails at the envelope-existence assertion with the call site reverted to False. A sibling test pins the degrade path: a fence that raises withholds the restore without failing either landing. The present-file early return in _write_superseded_live_envelope_locked accepted any bytes and fsynced nothing, so a truncated or corrupt envelope was silently accepted and could never be repaired. It now compares the existing envelope against the one this identity would build, ignoring created_at exactly as the main publish path does -- the comparison is extracted into a helper shared with that path -- raises the same invalid/conflicts errors, and fsyncs the parent directory on a match before returning. The superseded-write diagnostic used to print while holding both the store lock and the publication-order flock; an undrained stderr would have blocked the found-block path with locks held. The store now records the write on the returned publication (AuditPublication.superseded_envelope_written, defaulted so every existing construction site is unchanged) and _publish_finalization_evidence emits the message -- still on stderr, because worker stdout is a strict JSON-lines protocol -- after the balance lock and the order guard have been released. Verified: the three unittest suites (256 tests), the Postgres ledger gate and the four A1 gates including C3-B-wins-late-A, and ruff E4/E7/E9/F on the changed files. Co-Authored-By: Claude Fable 5 --------- Co-authored-by: Claude Opus 5 * Persist per-session vardiff difficulty across miner reconnects (#138) * feat(prism): persist per-session vardiff difficulty across reconnects A coordinator restart or miner reconnect previously reset every session to its lane's start difficulty; at 4x per 300s retarget a large miner needed 15-25 minutes to re-climb while flooding the share path, feeding the 2026-07-16 loop: coordinator slows -> watchdog exits -> every miner reconnects -> every session resets to lane start -> share rate spikes -> coordinator slows further. Remove the amplifier by retaining each worker's last converged difficulty in a bounded, TTL'd in-process store keyed by (listener name, exact Stratum username). The value is recorded at every committed retarget and on disconnect, and adopted on the first mining.authorize of a new connection, before password parsing, so an explicit d= still outranks it and an md=-only password clamps the resumed value. Adoption is capped at min(lane max difficulty, lane start * resume factor): a stale or absurd retained entry is clamped instead of adopted, while a value below the lane start resumes as-is (vardiff targets a share rate, so a converged session is on target by definition). New tunables, documented in .env.example: - PRISM_STRATUM_VARDIFF_RESUME (default 1): master switch; 0 restores lane-start reconnect behavior exactly. - PRISM_STRATUM_VARDIFF_RESUME_TTL_SECONDS (default 900): how long a converged difficulty stays adoptable; keep short where the idle step-down sweep is disabled. - PRISM_STRATUM_VARDIFF_RESUME_MAX_ENTRIES (default 8192): LRU bound on retained state (2x the deployed connection cap). - PRISM_STRATUM_VARDIFF_RESUME_MAX_START_FACTOR (default 1024, must be >= 1): plausibility ceiling as a multiple of the lane start. Also adds convergence observability -- sessions sitting at their vardiff ceiling, per-lane accepted share counts and rates, resume outcomes by bounded label, and a retained-sessions gauge -- mirrored into the frozen metrics reference implementation. * fix(prism): age retained vardiff difficulty from real convergence SessionDifficultyStore.record re-stamped recorded_monotonic on every write, and the disconnect seam records on every disconnect with no share evidence. A session that resumed a retained difficulty, submitted nothing and disconnected therefore refreshed that value's clock, so reconnect cycles shorter than the TTL kept it adoptable indefinitely (reproduced: ttl=900 survives past 4000s on 800s cycles). That defeats the short TTL exactly where it matters: with the idle step-down sweep disabled as deployed, a session resumed above its true hashrate produces no accepted shares and cannot step itself down. record now takes an explicit share_backed flag and refreshes the timestamp only on a first record, on a value that actually moved, or on an unchanged value backed by accepted shares; otherwise the entry keeps ageing from its original convergence. The stored difficulty is always updated and LRU recency still moves on every record, so recency and TTL stay independent. ClientState gains vardiff_accepted_any, set on the first accepted share of a connection and never reset within it; the retarget commit point passes its own window's evidence (accepted_shares > 0), and the disconnect seam passes the connection's accumulated flag. An idle step-down carries no shares but lowers the value, so it refreshes on the changed-value branch as before. Also from the same review: - Disconnect retains client.share_difficulty only. The pending value is advertised with a future job, so a disconnect racing an in-flight retarget was retaining a difficulty the miner never received; a retarget whose paired send completed still records itself. - qbit_prism_vardiff_resume_total gains outcome="overridden", counted when an adopted resume is superseded by an explicit difficulty request in the same authorize, so resumed + clamped - overridden is the number that stuck. Routed through a coordinator seam like the adjacent ones. - Document shared-username retention as last-writer-wins next to PRISM_STRATUM_VARDIFF_RESUME, sharpen the per-lane rate gauge HELP to say it is a since-start average and point at rate() over the counter, and record the at-ceiling census cost at convergence_snapshot. Tests cover the reconnect-cycle repro and its share-backed counterpart, the undelivered-pending disconnect, the retarget commit-point recording, the env -> config -> coordinator -> store sizing chain, and an authorize whose username reservation fails. The frozen metrics reference mirrors both rendered-byte changes. * Cap the escalated landing budget below the watchdog tolerance (#135) * Cap the escalated landing budget below the watchdog tolerance A landing-class ledger step cost two independent budgets of heartbeat silence, and the escalated budget could exceed what the watchdog would wait for. Both halves are fixed here (issue #125). Half A -- the ledger admission wait is no longer heartbeat-silent. PsqlShareLedger._operation_gate waited for the writer lock or read semaphore in a single blocking acquire sized by _remaining_operation_timeout, which under statement_timeout is the full constant budget. That wait is local admission, not database work: no statement has been sent, so no server-side deadline bounds it and the ledger reports nothing until the gate opens. A new operation_progress thread-local scope -- shaped exactly like its operation_timeout and statement_timeout siblings -- installs a callback plus a slice; when one is installed the acquire is served in slices and the callback fires between them. Without a hook the path is unchanged: one blocking acquire and the same LedgerOperationTimeout message, so non-block-work callers pay nothing. The caller's deadline stays authoritative, the loop always attempts the acquire at least once so an expired budget still fails where it always did, the callback never runs while this thread holds the gate, and an exception from it propagates rather than being swallowed inside a lock wait. Both statement-timeout scopes in BlockCandidateService run on the watchdog-monitored block-work owner thread, so they now install that hook (duck-typed, like the statement_timeout/operation_timeout probes beside them) with _record_block_submitter_wait as the callback and the already-watchdog-clamped _block_work_wait_slice as the slice. Ledgers predating the hook keep working unchanged. Half B -- the landing cap is derived from the configured watchdog. DEFAULT_BLOCK_LANDING_DB_TIMEOUT_MAX_SECONDS (120s) and the watchdog default (120s) were independent literals, so an escalated landing budget could legally outrun the tolerance. _block_landing_db_timeout now clamps both the base and the escalation cap to watchdog_timeout_seconds * BLOCK_LANDING_DB_TIMEOUT_WATCHDOG_FRACTION (0.5), read from the configured coordinator rather than a second hardcoded number, and DEFAULT_PRISM_WATCHDOG_TIMEOUT_SECONDS replaces the literal in the env default so the two cannot drift apart. Arithmetic. Before: admission wait (up to 120s) plus a fresh server-side statement (120s) gave 240s of silence against a 120s watchdog; the watchdog hard-exited mid-landing, the in-memory escalation counts died with the process, and the restart replayed the same doomed attempt from the base budget. After: the silent span of one landing step is a single statement, and at the 120s default the ceiling is 60s with escalation running 30s -> 60s -> 60s. At the 300s production tolerance the ceiling is 150s, so the reviewed 120s cap is granted in full and deployments using that override see no behavior change. Tests. New coverage pins the escalated budget inside half the configured watchdog (and pins the 300s override leaving the 120s cap intact), the sliced admission wait stamping progress while blocked and still raising at the caller's deadline, the unhooked gate path staying a single acquire, hook validation/nesting/propagation, and both scopes installing the hook while a ledger without operation_progress still works. Two existing tests that assert budgets above half the default watchdog now set watchdog_timeout_seconds explicitly rather than weakening their assertions. Refs #125 Co-Authored-By: Claude Opus 5 (1M context) * Make the landing scope's one-statement heartbeat invariant true The escalated landing budget is clamped to half the watchdog tolerance on the grounds that the heartbeat-silent span of one landing step is a single server-side statement. That fraction is only sound if the claim holds, and in three places it did not. Ledger operations that hold a gate across two statements. Three _operation_gate bodies issue a second statement without releasing the lock: confirm_accepted_block and reactivate_pool_block read the publication ordinal their mutating statement's command snapshot could not see, and carry_forward_integrity_report reads its audit head under the same lock so the two reads observe one another. No admission slice runs between the statements, so a landing caller's monitor was asked to sit through two statement budgets of silence -- at the 120s default that is the whole tolerance with no margin at all. PsqlShareLedger._note_operation_progress reports the completed first round trip, following the same reasoning as verify_writer_lease_guard_session's on_statement_progress callback. That callback stays as it is: its callers pass it explicitly per call rather than through the thread-local scope, and its statements never run under _operation_gate. Unstamped stretches between the current-tip RPC and the audit build. The reorg reconciliation, the prior-balance checks and the tip-height RPC each run inline on the block-work thread and stamped nothing. All three now bracket with the file's phase-pair idiom. Bracketing the reorg walk alone would not have been enough: one pass spends a getblockhash plus up to two ledger statements per watched pool block, so its silent span grows with the reorg window. The reconcile state machine takes a record_progress port -- wired to the coordinator's block-work phase recorder, defaulted to a sink -- and stamps per row. The unfenced-append drain. Its condition wait was untimed and unstamped. It stays unbounded, because bounding it could let a landing arm its epoch fences ahead of an append that must drain first; instead it waits in watchdog-sized slices from the existing _block_work_wait_slice helper and stamps each slice that still finds the predicate true. The while loop's re-check makes the timed wait semantically transparent, and the common path never enters the loop. Every stamper here no-ops off the registered block-work owner thread, so client-thread dispositions and background reconciliation are unchanged. The fraction's comment now states what is actually bracketed and calls the result a margin rather than a global invariant: a network stall mid-statement has no client-side socket-read deadline on the native psycopg path and can outlast any budget, which is a genuinely wedged process where a watchdog exit is the right outcome. Co-Authored-By: Claude Opus 5 * Fix the progress-hook race and the landing clamp's blind spots Four follow-ups from the review of the landing-budget clamp. The progress thread-local was built lazily. operation_progress did a getattr-then-assign that is not atomic, so two block-work threads entering their first-ever scope on a fresh ledger could each build a local() and have one assignment win. The loser's hook then lived on an object the ledger no longer referenced and its admission wait silently reverted to the heartbeat-silent path -- exactly the failure the hook exists to remove. PsqlShareLedger.__init__ now creates it alongside its two siblings; the scope keeps its lazy fallback for ledgers built through __new__ in focused tests. The clamp was silent. When the watchdog-derived ceiling lands below the configured base or cap the operator gets less than they asked for -- at the 120s default tolerance a 120s cap becomes 60s -- and "escalation exhausted at my cap" becomes indistinguishable from "escalation never reached my cap". _block_landing_db_timeout runs on every landing attempt, so the notice names the configured values, the granted values, the ceiling and the watchdog it came from, and prints exactly once; the flag flips under the coordinator lock so concurrent first landings cannot both print, and the line itself is emitted outside that lock. The clamp ignored watchdog_enabled. A deployment running with the watchdog off has no hard-exit hazard to stay under, so halving its cap bought nothing. The ceiling is now infinite in that case, which also keeps the clamp notice silent (an infinite ceiling never compares below a finite configured value) and cannot poison the escalation arithmetic, whose inputs stay finite. The attribute is read defensively and defaults to enabled, since clamping is the safe answer; a test pins the name on LifecycleConfig so the defensive read cannot outlive the setting. Nesting semantics did not match the siblings. operation_timeout and statement_timeout merge by minimum, so nesting only ever tightens; operation_progress replaced outright, letting an inner scope with a larger slice widen heartbeat gaps its enclosing caller had already sized against its own monitor. Slices now merge by minimum. The callback still replaces: the innermost caller is the one whose liveness the wait belongs to. Co-Authored-By: Claude Opus 5 --------- Co-authored-by: Claude Opus 5 (1M context) Co-authored-by: Alex * Bound PRISM writer-lease acquisition and guard coordinator Postgres sessions (#137) * Bound writer-lease acquisition and guard Postgres sessions A coordinator client that vanishes without an RST (partition, SIGSTOP, VM pause) leaves its backend idle-in-transaction, still holding the qbit_ledger_writer_lease row lock taken by the landing CTE. The successor's startup lease upsert ran with no statement or lock timeout, so it blocked inside PsqlShareLedger.__init__ - before the watchdog arms - until TCP keepalive teardown, a multi-hour full-pool outage (issue #123). Settlement correctness was never at risk; the outbox row stays pending and replays. Two independent halves: - Every coordinator Postgres session (pooled native client, dedicated lease-guard session, psql subprocess backend) now carries session guards: idle_in_transaction_session_timeout reaps the orphaned transaction, and the server-side tcp_keepalives_* GUCs bound dead-socket detection to 30 + 3x10 = 60s. DSN/PGOPTIONS values the operator set are merged, with the coordinator fragments last so they win. - The startup lease upsert and adoption CAS run through _run_lease_acquisition_json: each attempt under the existing statement_timeout machinery (SET LOCAL statement_timeout/lock_timeout, or PGOPTIONS plus subprocess timeout), a diagnostic per timed-out attempt, and a RuntimeError naming the locked lease row after the retry budget. 5 attempts x 5s outlasts the 15s idle-in-transaction timeout, so the orphan case self-heals; only a genuinely stuck lock exits the process for the supervisor to restart. The outer _ensure_writer_lease TTL wait is unchanged - waiting out a live holder is intended failover. Six tunables (PRISM_POSTGRES_IDLE_IN_TRANSACTION_TIMEOUT_SECONDS, PRISM_POSTGRES_TCP_KEEPALIVES_{IDLE_SECONDS,INTERVAL_SECONDS,COUNT}, PRISM_LEDGER_LEASE_ACQUIRE_{LOCK_TIMEOUT_SECONDS,ATTEMPTS}) flow through LedgerConfig, the coordinator wiring, .env.example, and compose.yaml, with fail-closed validation in both load_coordinator_config and PsqlShareLedger.__init__ for the direct-construction daemons. Regression tests cover the guard options on all three connection paths, conninfo option merging, the bounded startup deadline, the bounded retry with visible failure, recovery on a later attempt, and config defaults/validation; each was observed failing against the unmodified tree. Co-Authored-By: Claude Fable 5 * Report lease-acquisition timeouts honestly and pin the bounds The fatal path in _run_lease_acquisition_json diagnosed every LedgerOperationTimeout as a lock conflict on the qbit_ledger_writer_lease row. A connect timeout, an exhausted pool slot, a psql subprocess timeout, and a healthy-but-overloaded server all raise the same exception, so an unreachable Postgres was reported to the operator as a stuck transaction. The RuntimeError was also raised without `from`, discarding the one exception that said what actually failed. Both the per-attempt line and the fatal error now name the lock conflict as the most common cause without asserting it, quote the underlying exception text, and the fatal error chains the originating timeout. Two mutations of the acquisition bound survived the suite: - Reverting _try_adopt_writer_lease to a bare _run_json left all eight tests green. Two new tests drive the real _ensure_writer_lease adoption branch — fast-adoption capable, a heartbeat-prefixed predecessor silent past both the row and guard edges — and assert the CAS runs under a bounded deadline and that a persistently blocked CAS stops after DEFAULT_LEASE_ACQUIRE_ATTEMPTS with a chained RuntimeError. - Widening `except LedgerOperationTimeout` also left the suite green. A new test asserts a non-timeout failure (the RuntimeError run_json raises for a wrapped OperationalError, and an IdleInTransactionSessionTimeout — sqlstate 25P03, an InternalError rather than an OperationalError) propagates unchanged on the first attempt, with no retry and no diagnostic printed. Co-Authored-By: Claude Opus 5 * Match the orphan-lock comments and docs to what the guards deliver Several comments from the bounded-acquisition change claimed more than the code provides. The PostgresSessionGuards docstring said no session the coordinator opens can become an unreapable lock holder; tcp_keepalives_* are silently ignored on Unix-socket connections and on platforms that do not implement them, and session settings reach only this coordinator's own sessions — a foreign holder is bounded solely by the acquisition deadline. The attempt-budget comments said the ~25s budget "exceeds" the 15s idle_in_transaction_session_timeout so the orphan is reaped mid-budget; that timer runs on the blocking backend's clock from when its transaction went idle, not from when the successor starts retrying, so the budget is sized to outlast a typical reap rather than guaranteed to contain one. Also documents the deployment caveats in the ops guide: native connections now always set the libpq `options` parameter, so a pooler that rejects startup options fails visibly at connect (the default compose topology connects directly and is unaffected); the psql backend carries the guards in PGOPTIONS, which a wrapper command that drops its environment silently discards; and Unix-socket DSNs ignore the keepalive GUCs, leaving the idle-in-transaction timeout as the guard that still applies. Co-Authored-By: Claude Opus 5 * Reconcile the concurrency harness with the bounded acquisition path The harness suites encode #123 as an unfixed defect; this branch fixes it, so fourteen of their assertions had to be answered rather than muted. The orphan-lock bound is now proven. OrphanedLeaseLockFailoverBoundTests was authored against the fixed behaviour with an expectedFailure marker and a docstring demanding its removal by the fixing commit; it passes, so the marker is gone and the assertion is untouched. Its two siblings encoded "the bug is present": the successor now parks under the configured acquisition deadline instead of forever, so the wait is asserted as a positive bound tied to the shipped tunable, and the "does not end" test now pins how it ends -- the exact retry budget, the elapsed time that budget implies, and a RuntimeError chaining the underlying LedgerOperationTimeout. The orphan and the successor's park on it are kept as evidence throughout. FakeSqlBackend needed a fidelity fix first, and it is the substantive one. Both shipped LedgerSqlPort implementations translate a cancellation caused by the caller's own deadline into LedgerOperationTimeout, and only when that deadline was armed; the model did this for pool-slot exhaustion but not for the tuple-lock wait, because before this branch no deadline ever reached a pooled lease statement. Without the translation the model raised a bare LockTimeout straight through the retry loop, so the suites appeared to exercise a bounded retry that in fact ran once. Arming the acquisition deadline also makes the acquire transaction-scoped, which is a real widening: a coordinator that vanishes mid-acquire can now orphan the lease lock on a path that previously could not produce one. The model-fidelity test that used the acquire as its example of an autocommit statement drives an unscoped renewal instead, the new shape is pinned by a test of its own, and docs/prism-ledger-ops.md records the trade -- the new orphan is bounded by idle_in_transaction_session_timeout, which is why that guard is not optional. This harness models no reaper, so that test pins only what the model supports and says where the guard itself is pinned. Remaining changes are exact-trace updates: the acquire and adoption CAS now commit explicitly, so each offers a precommit stop. Traces were replaced with the observed ones rather than loosened to prefix or subset matches, since exact order is what they exist to prove. The lost-adoption-CAS driver needed one extra step to reach the same interleaving, the successor's observation no longer committing itself. Co-Authored-By: Claude Opus 5 --------- Co-authored-by: Claude Fable 5 Co-authored-by: Alex * Apply stability-review fixups from PRs 134-139 (PRISM core + infra/vardiff) (#141) * Land PRISM core fixups from the PRs-134-139 stability review F1: _acquire_operation_gate's slicing path now reads the injected self._monotonic clock at all three deadline sites, so a virtual test clock can drive admission-wait slicing deterministically; covered by a new virtual-clock test. F2: load_coordinator_config rejects PRISM_WATCHDOG_TIMEOUT_SECONDS below 1.0 at startup, keeping the derived landing-budget ceiling (half the tolerance) meaningfully below the watchdog tolerance. F3: the sliced-admission liveness test drops its per-gap 0.3s wall clock bound, which false-reds on loaded CI hosts; the determinism intent (stamp count, total elapsed, raised timeout) is unchanged and cadence is pinned by the new virtual-clock test instead. F4: the concurrency harness baton timeout is overridable via PRISM_HARNESS_BATON_TIMEOUT_SECONDS (default unchanged at 20.0); it remains a false-red failure detector and never influences ordering. F5: startup warns on stderr when the idle-in-transaction timeout exceeds the writer-lease acquire budget (attempts x per-attempt lock deadline), where an orphaned backend holding the lease row lock can no longer be reaped mid-budget and startup crash-loops instead of self-healing. F6: the superseded-envelope restore authority proof now runs after the payout balance lock and the publication order guard are acquired, so a writer deposed during the unbounded lock wait no longer restores on stale authority; the withheld-authority diagnostic is deferred until the locks are released. A new test deposes the writer inside the publication guard and fails against the old check-then-act shape. Co-Authored-By: Claude Fable 5 * Replace the racy socket-only prism-postgres healthcheck with a TCP-gated probe plus SELECT 1 confirm The postgres image's initdb temporary server accepts unix-socket connections and then shuts down, so a socket-only pg_isready can release dependents against the throwaway server (the race PR #134 fixed in the test scripts). Gate on the real TCP listener first, then confirm with a real query. Co-Authored-By: Claude Fable 5 * Wire the #137 lease/session-guard env knobs into the ctv broadcaster and backfill scripts Both operator-run scripts construct PsqlShareLedger directly and got the class defaults for lease_acquire_lock_timeout_seconds, lease_acquire_attempts, and the four Postgres session-guard knobs, so operator tuning of PRISM_LEDGER_LEASE_ACQUIRE_* / PRISM_POSTGRES_* never reached them. Mirror the coordinator: read the same six env vars with the coordinator_config helpers and defaults and pass them through. The backfill script exposes them as CLI flags with env defaults, matching its existing --lease-ttl-seconds pattern, with construction extracted into ledger_from_args so the path is testable. Co-Authored-By: Claude Fable 5 * Document that vardiff retention is process-local and can strand a stuck-high resume The SessionDifficultyStore is in-memory: a coordinator restart, including the watchdog's terminal exit_process(1), empties it, so the post-restart reconnect wave is not smoothed by retention. And with the idle sweep off (the deployed default) a rig that converged high and reconnects within TTL on weaker hardware can resume too high with no share signal to bring it down, bounded by the resume TTL. Honest-disclosure comments and .env.example notes only; no behavior change. Co-Authored-By: Claude Fable 5 --------- Co-authored-by: Claude Fable 5 * Paginate startup candidate enumeration and sweep stranded prepared blocks (#142) * Paginate startup pending-candidate enumeration Startup replay must observe every durable pending candidate before child job builds unblock, and it proved that by re-querying with a doubling window until a batch came back short. Past MAX_BLOCK_REPLAY_ENUMERATION_ROWS the window stops doubling and the pass fails closed, so a backlog larger than the cap can never produce a complete enumeration: every pass reports "still truncated", enumeration stays owed, and _await_pending_parent_payout_preview raises TemplateRefreshBlocked for every job build until the outbox drains under the cap on its own. Testnet4 hit this on 2026-08-18 with ~5,000 pending rows and refused job builds for ~25 minutes. Replace the widening window with a keyset walk. pending_block_candidate_rows gains an after_cursor keyword on both ledger backends and returns an opaque per-row cursor built from the two ordering columns: the microsecond-precision UTC created_at text plus block_hash in Postgres, (created_monotonic, block_hash) in memory. The block-hash tiebreak is what makes the order total -- creation stamps collide, and a cursor on a colliding stamp alone would replay or skip its peers. In Postgres the predicate and the ORDER BY both stay on (created_at, block_hash), so each page is one bounded range scan of qbit_block_candidate_outbox_pending_idx no matter how deep into the backlog it starts. replay_pending walks pages of MAX_BLOCK_REPLAY_ENUMERATION_ROWS while enumeration is owed and ends only on a page proven short. Row adoption moves to _adopt_durable_block_candidate_rows so both enumeration shapes restore rows through the same decode, dedupe, and poison path. Ledgers without cursor support keep today's exact windowed semantics, fail-closed truncation included: signature introspection detects them without spending a ledger worker, and a keyword probe that raises TypeError falls back the same way. A page that cannot advance its cursor also fails closed rather than declare an unproven enumeration complete. Each page carries its ordinal and cursor in the bounded-call dedupe key. A timed-out outbox call stays registered for the next paced retry to reuse, and without that key page N+1 would be answered by page N's in-flight call, silently dropping a page from an enumeration that then declared itself complete. Steady-state polling is unchanged: one small fetch, the same early-outs, no pagination. Co-Authored-By: Claude Opus 5 * Sweep stranded prepared pool blocks in the reorg reconciler A prepared pool-block row is resolved by the live submit/replay path that holds its outbox entry: block_finalization rejects it when the block turns out to be orphaned. When that outbox entry is gone -- quarantined, or completed by a process that died before confirming -- nothing re-examines the ledger row again, because reorg_watch_blocks selects only chain_state IN ('confirmed', 'inactive'). Testnet4 row 0000000000000190d487ff4d5bdcc920d7b5c7a978c2f82d29c2b1f2bb6d08e9 (height 30822, found 2026-07-08) sat prepared/immature for six weeks that way: the canonical block at 30822 is a different hash, so the pool block was orphaned, and its payout entries, carry forward, and CTV fanout artifacts stayed pinned immature with it. Add stranded_prepared_blocks to both ledger backends -- prepared, immature, and buried at least min_depth below the active tip, ordered by height and bounded by limit. The Postgres predicate leads with maturity_state and block_height so it rides qbit_pool_blocks_maturity_idx. The reconciler sweeps that read in the same guarded not-untrusted section as the watch loop, feature-detected like every other step, and stamps progress per row because each one costs a getblockhash plus possibly a fenced ledger mutation. A row whose height no longer resolves to its own hash is provably orphaned and goes through the existing fenced reject_prepared_block, which cascades the payout entries, carry forward, and fanout artifacts and returns 0 if the row already moved. A row that is still canonical is never mutated -- the confirm path owns audit publication sequencing -- and is reported loudly for an operator instead. STRANDED_PREPARED_REJECT_MIN_DEPTH is 100: deep enough that no legitimate in-flight finalization, ancestor replay included, can still be racing, and shallow enough to heal within roughly two hours of block time. Rejections count into the pass totals like inactive_blocks_total (error path included, so a partial pass stays accounted), surface as summary stranded_prepared_rejected/stranded_prepared_canonical, and drive the same republication and cross-tip memo eviction as any other row mutation. Co-Authored-By: Claude Opus 5 --------- Co-authored-by: Claude Opus 5 * Distinguish an idempotent confirm replay from a fresh block confirmation (#146) qbit_confirm_pool_block collapsed two different outcomes into a return of 1: the fresh flip, where the UPDATE matched a 'prepared' row and allocated an audit publication ordinal through nextval, and the idempotent arm, where the UPDATE matched nothing but a confirmed non-reversed row already exists at that (hash, height) and no ordinal is allocated. A caller could not tell them apart, and a genuinely new confirmation must publish for payout correctness, so every replay of a candidate whose block is still the best tip republished identical payout state -- bumping the payout generation, wiping the job-bundle cache and aborting in-flight refreshes until a successor block arrived. The audit publication ordinal already draws this line durably: only the flip allocates one. Report that fact instead of discarding it. Return contract, was 1 / 0 / -1, now 1 / 2 / 0 / -1: 1 fresh flip out of 'prepared'; allocates the publication ordinal 2 idempotent replay of an already-confirmed row; allocates none, and returns the ordinal its original flip assigned (new) 0 no row, or a live row this confirmation does not match -1 terminally disposed (reorg quarantine, rejection, reversal) Threaded through every consumer: * the SQL function itself * the memory ledger's parity implementation, which now splits on whether the row was already 'confirmed' * the psql wrapper, which reads the ordinal on 1 and 2 (a covered replay still needs it to address its audit publication) and stays a single statement for the non-confirming dispositions * the coordinator confirm branch, where 2 completes the candidate exactly like a fresh flip instead of falling into the confirmation-failure arm * the shared test ledger fake, so unit coverage models the split * the A1 gate disposition matrix and the Postgres ledger suite An at-tip replay now completes as a covered replay: it reserves no payout source, bumps no generation, wipes no job-bundle cache and schedules no refresh churn. The fence and pending-source checks that decide publication are unchanged and unweakened -- a source left unpublished by a prior attempt, or a leaked delivery fence, still publishes exactly as before. Publication-ordering interaction with the audit ordinal work: the confirmation dict rides in the published evidence, which publish_success compares in full on an exact replay. A restart replay reporting 2 where the flip published 1 would have been rejected as a payload conflict, so the disposition now joins the share counters as an observational annotation reused from the durable publication. The ordinal it rides with is block identity and is still compared unchanged, so a replay returning a different ordinal still conflicts. The ordinal is unaffected either way: the idempotent arm burns no sequence, so the durable publication floor cannot move on a replay. Refs #61 Co-authored-by: Claude Opus 5 (1M context) * Extend the deterministic concurrency harness to block candidate landing (#148) * Extend the deterministic concurrency harness to block candidate landing Issue #128 ranks block candidate landing first by incident history, and the harness landed by #139 covered only the writer-lease lifecycle. This teaches it the landing state machine, which is the one owner where two threads of the same process reach the same durable rows. The PostgreSQL model gains the rows landing turns on: qbit_pool_blocks, qbit_block_candidate_outbox, the qbit_confirm_pool_block publication ordinal allocator and the durable floor read. The allocator is modelled on the shipped PL/pgSQL rather than on an approximation of it: the ordinal comes from a sequence, so an exact replay of an already-confirmed row burns nothing, a terminally disposed row reports the superseded disposition, and an aborted confirmation leaves a gap the MAX-based floor tolerates. That distinction is the whole of the two-tail defect -- a later ordinal can publish first and leave an earlier one behind the floor. Landing writes are staged per transaction for the same reason lease writes already were: a deadline-scoped statement sends COMMIT as a separate message, and the interesting window is inside it. Statement classification is anchored on fragments lifted verbatim from the production SQL, as the lease signatures already are, so a change to that SQL breaks classification loudly instead of quietly changing what the fake models. Where a modelled statement is a large CTE, the model reproduces the effects a landing reads back and says so. Three new pieces support the topology. Scheduler-aware locks, conditions, semaphores and events make a contended process-local lock an interleaving point rather than a wedged baton -- the lease scenarios never needed them, because they interleave two processes around one row, while the two landing tails share the payout-balance serializer, the publication order guard and the ledger's admission gates. Phase breakpoints let a scenario stop a tail at a boundary the landing already names for its own watchdog, because landing's interesting windows are the gaps between statements and no statement runs at the instant that matters. And HarnessBase now holds what every state machine's harness owns, so run_scenario and assert_deterministic are generic over it rather than typed to one harness; the two owners still owed to #128 plug in without touching the determinism check again. The acceptance criterion is #133's two-distinct-hash interleaving, driven end to end: block A confirms on the synchronous client-thread tail, block B lands on the accounting actor inside A's confirm-to-publish window and publishes the next ordinal, and A then publishes behind the raised floor. It is already fixed here, so the scenario passes; the evidence is that the interleaving can be expressed at all, which nothing could do before. The suite also pins the load-bearing claim underneath it -- the per-hash disposition lease does not serialise distinct hashes -- and the fresh lease proof that gates the superseded-envelope restore. Every scenario is proved order-stable over 25 runs of the full checkpoint trace. Refs #128, #133. * Drive the #125 escalated landing budget against the watchdog Issue #125: a landing-class ledger step is two waits, not one. The caller queues for the ledger writer lock -- local, unbounded by any statement deadline -- and only then runs the statement it queued for, under its own full budget. Both are spent on the block-work thread the watchdog monitors, and before 49f04c3 both were heartbeat-silent, so an escalated landing at the reviewed 120s cap could hold the heartbeat still for ~240s against a 120s tolerance. The watchdog hard-exited mid-landing and the in-memory escalation counter died with the process, so the restart came back at the 30s base to repeat the cycle. Add landing-harness scenarios for both halves of the fix: - a real accounting tail queued behind a held writer lock stamps wait-ledger-admission:landing every slice instead of going quiet, and an A/B against the same gate at the same budget with no progress hook shows slicing changes nothing about when the wait ends; - the escalation ladder is climbed by genuinely timed-out landings and every rung's admission-plus-statement pair fits the watchdog tolerance, across a clamped cap, a clamped base and a configuration needing no clamp at all, with the clamp reported exactly once; - both composed: a landing that queues for all but one slice of its escalated budget still lands, inside the tolerance, with no silence longer than one slice. Every scenario is proved order-stable with assert_deterministic. Co-Authored-By: Claude Opus 5 * Pin the landing half of the concurrency harness with self-tests The lease half of the deterministic harness is defended by tests/test_prism_concurrency_harness.py; the landing half added in cefdbfa was not. A fake that answers an unrecognised statement plausibly drifts away from production silently, and every scenario built on it inherits the drift, so a landing scenario that failed could not distinguish "the code under test is wrong" from "the model is wrong". tests/test_prism_landing_harness_model.py pins the model's own contract in four parts: - The publication-ordinal allocator against qbit_confirm_pool_block: a fresh confirmation takes the next sequence value, an exact replay returns 1 without burning one (proved by the next block's ordinal), every terminal disposition reports -1 rather than 0, a height mismatch and a missing row both report 0, and the durable floor — a MAX over the table — stays put across the gap an aborted confirmation leaves behind. - Transaction scope: a deadline-scoped landing write is invisible to another session until COMMIT and visible to its own before it, and a rolled-back landing leaves no row while keeping its ordinal burned. - Classification: every LandingOp is driven through the shipped PsqlShareLedger method that emits it, a landing table mention that matches no signature raises naming _LANDING_SIGNATURES, the deferred markers still name their follow-up state machine, and a deposed writer fails the lease CTE in the two shapes the SQL dictates — the outbox statements report it, the PL/pgSQL functions raise. - The synchronisation primitives: mutual exclusion and FIFO grants, non-blocking and virtual-clock-timed acquires, the non-reentrant re-acquire that a real mutex would deadlock on, RLock depth, the condition's release/re-acquire and restored depth, semaphore exhaustion and over-release, and the event's parked and timed waits. One landing scenario is proved order-stable with assert_deterministic against LandingHarness. Nothing sleeps in wall-clock time. Co-Authored-By: Claude Opus 5 * Pin the vardiff resume TTL-boundary test to an exact clock base The retention test captured time.monotonic() and asserted that a silent reconnect cycle at exactly +TTL still resumes, relying on the store's strictly-greater expiry comparison. That boundary arithmetic is not exact for an arbitrary float base: (start + 900.0) - start rounds above 900.0 for roughly one in five thousand captured clock values, which flips the boundary cycle to expired and fails the test. CI hit this on the push run for the map commit while the pull_request run of the same commit passed. Every record and lookup in the test runs under frozen_clock, so the base never needs to be a real monotonic reading. Pin it to an exactly-representable constant, making the boundary arithmetic exact. Refs #145. --------- Co-authored-by: Claude Opus 5 Co-authored-by: Alex * Serve content-addressed artifacts as their exact canonical bytes (#154) * Serve content-addressed artifacts as their exact canonical bytes GET /public/v1/artifacts/{sha256} re-serialized every artifact through the generic write_json path (json.dumps sort_keys=True, default separators, plus a trailing newline), while manifest and manifest-set hashes are computed over canonical_json_text (compact separators). sha256(response body) therefore never matched the advertised hash, so no external verifier could confirm any artifact byte-for-byte. The ledger already persists the exact hashed text (manifest_json / manifest_set_json) next to the JSONB copies. Serve it: - share_ledger: dashboard_public_artifact_canonical_json on both backends returns the persisted canonical text for CTV manifests and manifest sets, None for audit bundles (their Rust struct-order canonical bytes are not stored). - public_api.artifact: wrap the canonical text as RawJsonBody after verifying sha256(bytes) == the requested hash; fall back to the legacy re-serialized response on any mismatch so a store-integrity fault can never serve bytes that contradict the content address. - audit_http.write_json: write RawJsonBody bytes verbatim with Content-Type: application/json; no re-serialization, no trailing newline. No hash definition changes; historical hashes stay valid. Audit bundles keep the legacy response until their canonical bytes are persisted at hash time; the expectedFailure regression marker documents that remaining gap. Co-Authored-By: Claude Fable 5 * Fold the canonical text into the artifact document read model Review follow-up: the standalone canonical-json lookup duplicated the legacy artifact query's predicates and added a second sequential read per uncached request — and since the response cache stores only 2xx results, arbitrary 404 probes against the unauthenticated endpoint paid both statements (four unindexed table probes) every time. Replace both reads with one dashboard_public_artifact_document() read model returning {payload, canonical_json} from a single statement: the audit/set/artifact lookups become shared CTEs selecting the JSONB copy and its persisted canonical text together, so every request — hit, miss, or audit bundle — touches each artifact table at most once, exactly like the pre-change query. dashboard_public_artifact() delegates to it, and public_api.artifact() consumes the document with the same sha256 self-verification and legacy fallback semantics. Co-Authored-By: Claude Fable 5 --------- Co-authored-by: Claude Fable 5 * Map the PRISM public/operator endpoint split before extracting the read tier (#147) * Map the PRISM public/operator endpoint split before extracting the read tier Issue #145 moves the public dashboard and audit read tier out of the coordinator process. That needs an explicit endpoint map first: some endpoints that look public are operator surfaces that merely share the listener today. Classify all 29 request paths the HTTP surface serves as public read, operator, or payout-affecting, with the evidence behind each call. The result is narrower than "move the read traffic": exactly the 13 /public/v1 routes are extracted, and everything else stays. Three findings drove the narrowing. The boundary is already written down and it excludes /audit/*. PRISM.md lists every /audit/* route, /healthz, /metrics and /owed-balances as the private audit/ops listener and says operators may expose only /public/v1. docs/public-dashboard-api/README.md repeats it as a contract and test_public_dashboard_api_contract.py enforces it. The published evidence content is public and already reaches callers content-addressed through /public/v1/artifacts/{sha256}, which resolves bodies from body_uri on disk; the /audit/* routes are internal tooling over the same data, in raw sats, unpaginated, carrying broadcast material. So the PRISM_PUBLIC_ARTIFACT_CACHE_* knobs move with the extracted service without widening the public surface. Two nominally public routes take the writer lock today. /public/v1/miners/{recipient_id}, the most-polled route on the surface, resolves an owed balance via current_owed_balances(), which returns every recipient's balance under the writer lock and filters in Python. /public/v1/blocks/{block_hash}/settlement-artifacts falls through to audit_bundle() under the writer lock for direct-coinbase blocks. Extracting the process without severing these would move the GIL contention but keep the lock contention, so both are recorded as extraction blockers. More routes are misfiled than the issue predicted. Beyond the drift count and the health probes: /owed, /owed-balances, the /payouts/{recipient_id}/status alias, and the whole /audit/* family. /miners/{recipient_id}/status is not in PRISM.md's inventory and has no consumer outside tests. Land the map as docs/prism-public-operator-endpoint-split.md and the classification as lab/prism/endpoint_registry.py, so a reviewer can disagree with one row without re-reading the HTTP layer. The registry is declarative and changes no request handling; AuditHttpFacade still owns dispatch. tests/test_prism_endpoint_registry.py pins the registry to the code in both directions: every classified path is really routed (driven through a bound server for the facade routes and through public_api.dispatch for the public ones), and every route literal in the HTTP layer is classified, so a new endpoint cannot be added without a classification. It also asserts the extracted set stays exactly /public/v1, that no /audit/ path is ever extracted, and that a writer-lock route cannot be marked for extraction without recording the blocker. Refs #145. Co-Authored-By: Claude Opus 5 (1M context) * Pin the vardiff resume TTL-boundary test to an exact clock base The retention test captured time.monotonic() and asserted that a silent reconnect cycle at exactly +TTL still resumes, relying on the store's strictly-greater expiry comparison. That boundary arithmetic is not exact for an arbitrary float base: (start + 900.0) - start rounds above 900.0 for roughly one in five thousand captured clock values, which flips the boundary cycle to expired and fails the test. CI hit this on the push run for the map commit while the pull_request run of the same commit passed. Every record and lookup in the test runs under frozen_clock, so the base never needs to be a real monotonic reading. Pin it to an exactly-representable constant, making the boundary arithmetic exact. Refs #145. * Harden the endpoint registry coverage guard with exact literal snapshots The substring-based unclassified-route guard was prefix-permissive: a new route extending an existing literal (e.g. a new /public/v1/blocks/... sub-route) passed unclassified, which is exactly the drift the guard exists to catch ahead of the read-tier extraction. Replace it with frozen exact-set snapshots of the do_GET and dispatch route literals, pin the 13-extracted/31-total path counts, assert every registry path is matched by a literal in its owning dispatcher, and pin the GET-only assumption so a future verb method cannot bypass the registry. --------- Co-authored-by: Claude Opus 5 (1M context) * Remove the meaningless shares-per-second gauge and give the coordinator lock a contention denominator (#161) * Remove the meaningless shares-per-second gauge and give the coordinator lock a denominator qbit_prism_shares_per_second divided the durable ledger's lifetime accepted count by this process's uptime. The two are different domains, so the value tracked 1/uptime rather than any share rate -- enormous right after a restart and decaying as the process aged. The published 3855.57 back-solves to about 4 h 23 min of uptime, not to the pool's throughput. Removed rather than redefined. A process-local numerator would be a different metric wearing the same name, and a series that silently changes meaning is worse for an operator than one that disappears. Nothing in the tree consumed it. qbit_prism_accepted_shares_total is unchanged and is the trustworthy counter; rate(qbit_prism_accepted_shares_total[5m]) is what the gauge was reached for. The per-lane gauge already publishes the process-local since-start average and its lanes sum to it. The vardiff lane gauge's help claimed it was "kept on the same elapsed formula as qbit_prism_shares_per_second so the two stay comparable". They never were comparable -- the lane numerator is process-local and the removed gauge's was the ledger lifetime total. The help and the owning docstring now say what the numerator actually is. _ObservedRLock recorded contended acquisitions but no total, so a contended percentage could not be derived from a live process: 30-82% contended during a reconnect herd and 0 in ~48k during steady state are indistinguishable from the counter alone. It now also counts granted acquisitions, published as qbit_prism_coordinator_lock_acquisitions_total. The class does zero shared-metrics work on uncontended acquisitions by design, so that observing it cannot recreate the convoy it diagnoses, and a counter under _metrics_lock would have put a second lock acquire on the fast path. Instead the increment happens while the underlying RLock is already held, inside a critical section the lock has already serialised. It adds no mutex acquisition and no cross-thread ordering the lock did not already impose. This deliberately does not rely on the increment being atomic under the GIL: the lock is what makes it safe, so it stays correct on a free-threaded build. The one unsynchronised access is the read in contention_snapshot, which assumes an attribute read yields a whole value and may be one increment stale. Both facts are stated in the class docstring so #150's GIL-assumption audit finds them. Only granted acquisitions count. A timed-out wait records contention without an acquisition, so the derived fraction is an upper bound where a caller passes a timeout; the help text and a test both say so. Hold duration is not instrumented -- a re-entrant lock needs per-thread depth tracking to avoid double counting nested holds, and a clock read on every acquire and release is the fast-path cost this class exists to avoid. Refs #143, #150. Co-Authored-By: Claude Opus 5 (1M context) * Say what the counters actually count Two documentation corrections from the review of the previous commit. No behaviour change. The _ObservedRLock summary line still claimed "zero shared-metrics work on uncontended acquisitions", which the acquisition counter makes literally false -- it runs on every acquisition, contended or not. The property that still holds, and the one that matters, is that no shared mutex is taken on an uncontended acquisition. The docstring now says that, and says the counter preserves it by where it runs rather than by how often. The vardiff lane gauge's new help said its numerator counts "shares this process accepted". It is narrower than that. VardiffService.note_accepted runs only where the share writer sees a newly inserted ledger row, and the startup recovery-journal replay commits recovered shares without passing through it at all, so an exact replay of an already-durable share and a recovered share are both acked accepted while never being lane-counted. An operator reconciling the lanes against an accepted-ack count would find a shortfall the old wording said could not exist. The help and the owning docstring now name both exclusions and say the lanes sum to this coordinator's commit rate. Refs #143. Co-Authored-By: Claude Opus 5 (1M context) --------- Co-authored-by: Claude Opus 5 (1M context) * Extract the PRISM public read tier into its own process (#152) * Extract the PRISM public read tier into its own process (#145) The coordinator served its public dashboard API on the same in-process listener as its operator endpoints, under the same GIL that acknowledges shares and lands blocks, against the same primary Postgres the lease-holding writer commits through. Read traffic scales with public interest rather than hashrate, so it was the one workload whose volume the pool does not control -- and it sat on the money path. Exactly the 13 /public/v1 routes classified Disposition.EXTRACT move. /healthz, /metrics, /audit/*, /owed* and the miner/payout status aliases stay on the coordinator, per docs/prism-public-operator-endpoint-split.md. Sever the two writer-lock reads on public routes first; without this the move would relocate GIL contention and keep lock contention: - PsqlShareLedger.dashboard_miner_owed_balance_bits filters one recipient in SQL through _run_read_json, replacing owed_balance_for_recipient's scan of every recipient's balance under the writer lock. That made /public/v1/miners/{recipient_id}, the most-polled route on the surface, a fenced read. - PsqlShareLedger.dashboard_direct_coinbase_settlement takes the direct-coinbase settlement branch off audit_bundle(), resolving the body through the existing _resolve_audit_bundle_row sha256 verification. public_api prefers both through the same duck-typed pattern as every other read model, keeping the old paths as in-memory-ledger fallbacks. Both were verified against real Postgres to return exactly what they replace, including multi-program recipients, negative balances, and quoted recipient ids. The new service is read-only in three independent ways: PsqlShareLedger gains read_only=True, which skips writer-lease acquisition and swaps the writer lock for a gate that refuses to be acquired at all; the audit artifact store gains read_only=True so it holds no publication lock and works against the :ro volume mount; and the container receives no writer, signing, or block-submission configuration. It refuses to start without PRISM_PUBLIC_STRATUM_URL (the fallback would advertise 127.0.0.1:3340 to miners) or on the in-memory ledger (whose missing dashboard_* read models would send public_api's fallbacks back onto writer-lock reads). Staleness becomes a contract, following observability.py's cached /metrics precedent: Endpoint gains max_staleness_seconds, derived as max(3 * (cache_ttl + underlying_cache), 15) rather than hand-picked, and every response publishes X-Prism-Staleness-Budget-Seconds beside Age and returns 503 rather than serve past it. /public/v1/artifacts/{sha256} is exempt via an explicit immutable_content flag. AuditHttpFacade's lifecycle machinery is untouched; audit_http no longer imports public_api at all, and an unknown /public/v1 request reaches its ordinary unknown-endpoint 404. Co-Authored-By: Claude Opus 5 * Apply the PR #152 review fixes: deployment seam, stratum env, doc claims REQUEST_CHANGES findings from the Fable review of the public read tier extraction, applied on top of the extraction rebased onto the hardened endpoint-registry base: - Blocker 1: make up-prism-pool and every mainnet runbook bring-up path (config/pull/up service lists and the lane table) started only the coordinator, so a pool brought up through the documented seam had no public API at all. prism-public-api is now part of each of them. - Blocker 2: PRISM_STRATUM_PORT and PRISM_STRATUM_HIGHDIFF_PORT never reached the prism-public-api container, so the rendered /public/v1/mining-configuration silently dropped the High-diff endpoint the coordinator was actually listening on. Both now pass through the compose environment; the compose-profile suite pins the pass-through and a contract-equality case pins the High-diff endpoint in the served body against public_api.dispatch with the env set. - Should-fix 3: the dashboard README told operators to raise a route's staleness budget alongside its cache TTL, but no budget knob exists -- the budgets are constants in lab/prism/endpoint_registry.py. The paragraph now names the cache TTL knobs that do exist and says the registry constant is what changes with them. - Should-fix 4: the extraction moved the process, not the read-replica back-end, so the read-service test docstring no longer equates the move with all of issue #145; the replica remains a follow-up and the service still reads the primary Postgres. Co-Authored-By: Claude Fable 5 * Carry #154's content-addressed artifact bytes into the extracted read tier Rebasing the read-tier extraction onto 2.x.x put it on top of #154, which landed while this PR was in review. Git merged the two textually, but the result was semantically broken in both directions: - audit_http kept #154's `public_api.RawJsonBody` branch in write_json while the extraction removed the public-API import, so every audit response raised NameError. GET /public/v1/artifacts/{sha256} moved out of this listener, so no route reaching it can carry pre-serialized bytes: the branch is dead here and is removed, restoring the module's import isolation. - public_read_service.write_json re-serialized every payload with sorted keys and a trailing newline. The artifacts route moved into this process, so #154's fix silently regressed: sha256(response body) would no longer match the advertised artifact hash. The RawJsonBody branch moves here with the route it serves. The two byte-exactness tests #154 added now exercise the extracted handler rather than the coordinator's audit handler, which is what pins the regression: they fail without the branch above. --------- Co-authored-by: Alex Co-authored-by: Claude Opus 5 Co-authored-by: Alex * Stop serializing the full block on every accepted share (#162) * Stop serializing the full block on every accepted share assemble_submission built block_hex for every accepted share, and serialize_block is O(block bytes): it bytes.fromhex-decodes the coinbase plus every transaction (each through validate_hex, which decodes once more and discards the result), concatenates, and the caller re-encodes the whole block to hex. Nothing on the share-accept path reads it -- block_hex is consumed only by the block-candidate landing path. Issue #143's findings comment measured the accept path against a 0-transaction template and named this as the largest open number in the report. Measured across the consensus range, the term is linear at 2.43 us of interpreter CPU per KB of block plus 0.25 us per transaction: an accepted share cost 2,447 us at a 1 MB template and 4,889 us at the 2,000,000-byte ceiling, of which 98.8% and 99.9% was serialization. Single-core share capacity collapsed from ~86,000/s at 0 transactions to 408/s at 1 MB, and the accept path alone crossed one full core at the deployed ~273 shares/s design point at ~1.45 MB -- inside the consensus range. Serialize only for a hash that solved the block. block_pass is the exact precondition, not a heuristic: classify_submit_work requires it for every route that constructs a PrismBlockCandidate, so gating on it computes a superset of what the landing path can ever ask for. The bytes are still built eagerly, on the client thread, in the same call that validates the coinbase -- before a candidate exists and long before its durable intent is written. Validation ordering does not move. Per-template transaction validation is already hoisted to job build and amortized across every share of that job: transaction_hexes_from_template validates each tx["data"], merkle_branch_for_coinbase structurally parses each transaction, and template_artifacts decodes them again when storing artifacts. make_job_from_builder_manifest is the only DirectQbitStratumJob constructor in lab/, the dataclass is frozen, and the per-client stamp in job_delivery never replaces transaction_hexes. The one per-share-varying input, the assembled coinbase, is still parsed unconditionally by strip_witness_transaction at assembly. The removed work was revalidating immutable, already-validated data -- measured at 55% of the term. block_candidate_intent now refuses to persist an empty block_hex for a real DirectQbitSubmission, failing on the client thread rather than resurrecting an unsubmittable candidate after every restart. Embedders that retain no block bytes keep the historical empty-string behaviour, and the attribute still exists on every submission so the landing path's hasattr guard keeps its meaning. After the fix the per-share composite is flat at 8.3-16.5 us across a 12,600x range of block size -- the residual is the merkle-branch fold, not serialization -- and single-core capacity is template-size independent at 52k-91k shares/s. The profiling driver lands under tests/ as a test instrument: standard library only, no database, network or coordinator, and not collected by unittest discover, so it costs no CI time. Co-Authored-By: Claude Opus 5 (1M context) * Exempt a durable replay from the empty-block guard Review of the previous commit found an asymmetry in _candidate_block_hex. The guard tolerates an embedder that retains no block bytes at write time, but block_candidate_from_intent rebuilds every replayed candidate as a real DirectQbitSubmission -- so on restart that tolerated empty value comes back wearing the strict type. The only caller that re-encodes a replayed candidate is the credit-on-accept append at block_finalization, which runs after the block has landed: raising there would fail a credit for a block already in the chain, strictly worse than re-persisting the same empty value the durable row already holds. Re-persisting an existing intent is not the write the guard exists to stop, so key it on the candidate's durable_replay flag. The guard still fires on every fresh candidate, which is where it protects something. Two tests cover the round trip: an embedder intent replays and re-encodes without raising, and a real serialized block survives the same path intact. Not reachable in this tree -- every durable intent writer goes through assemble_submission, which always carried non-empty block bytes for a block-worthy hash -- but the previous commit's message claimed embedders keep the historical empty-string behaviour, and across a restart they did not. Also fix the shipped instrument's term columns, which the same review found meaningless on the fixed tree: with the share path no longer serializing the block, the composite does not contain the term, so "term % of composite" printed ratios in the thousands of percent. The driver now asks the assembled submission whether the composite includes the term, renders both derived columns as n/a when it does not, explains why in a footnote, and emits null rather than NaN so the --json output stays strict JSON. Co-Authored-By: Claude Opus 5 (1M context) --------- Co-authored-by: Claude Opus 5 (1M context) * Serve the PRISM public read tier from a Postgres hot standby (#157) PR #152 extracted the public dashboard and audit read routes into their own process but left them reading the coordinator's primary, deferring the replica. That deferral is the remaining half of issue #145: the extraction takes public read volume off the coordinator's GIL, and this takes it off the database the lease-holding writer commits through. Deployment shape. prism-postgres now runs with a mounted hba_file so it can accept physical replication; config/prism-postgres/pg_hba.conf reproduces the stock image rules -- including the local trust rule the image's initdb bootstrap and the existing healthcheck depend on -- and adds two replication rules. Compose cannot interpolate that file, so those rules name the default superuser literally. Postgres 16's wal_level/sender/slot defaults already allow one standby, so no other server settings change. prism-postgres-replica is a streaming standby whose data directory is cloned by pg_basebackup rather than initdb: the entrypoint takes the base backup (creating, or reusing, the prism_public_replica slot) on a bounded retry loop so a cold `docker compose up` works, then hands off to the stock image entrypoint. It refuses to clear a directory holding a complete cluster, so a production bind mount aimed by mistake at a primary's data directory stops the script instead of being destroyed; pg_controldata is the discriminator because PG_VERSION is also present in the debris of an interrupted backup, which the script must still be free to clear. Its healthcheck gates on the TCP listener, a real query, and pg_is_in_recovery(), so an out-of-band promotion reports unhealthy rather than quietly becoming a second writable cluster. The contract. PsqlShareLedger grows read_replica_status(), a bounded read-slot probe returning recovery state, walreceiver heartbeat age, replay lag and apply backlog. The service polls it on the readiness thread it already runs -- one round trip answers both questions -- and refuses replica-backed routes that fail it: warm-up, a backing server not in recovery, and a replication stream silent past PRISM_PUBLIC_REPLICA_MAX_LAG_SECONDS. The enforced number is the heartbeat age, not replay lag. Heartbeats flow every wal_receiver_status_interval whether or not the primary has anything to send, whereas replay lag grows with wall clock on an idle primary -- bounding that would 503 a healthy standby every time the pool went quiet. Replay lag and apply backlog are published instead, on X-Prism-Replica-Lag-Seconds, /healthz and the qbit_prism_public_replica_* gauges. Freshness ages locally, which is what makes the gate fail closed: the bound is checked against the heartbeat age the last successful probe reported plus how long ago that probe ran, so a probe that starts failing ages its own last good answer out of the bound rather than pinning it there. The check runs before the response cache for the same reason -- an outage does not expire cache entries, so a check after the cache would keep answering 200 for as long as traffic kept one warm. Which routes are gated comes from endpoint_registry rather than a hand-kept list: those reading a read slot, excluding content-addressed ones, whose bodies either hash to the requested digest or do not. mining-configuration is assembled from environment and keeps serving through a replica outage. PRISM_PUBLIC_REPLICA_MODE defaults to off in code and require in compose, so merging this does not 503 a deployment that has not provisioned a standby while anyone bringing the stack up from this repository gets the enforced contract without opting in. An unrecognised value refuses startup rather than guessing, because the two modes disagree about whether this process may read the primary. Refusals use a new PUBLIC_ERROR_CODES entry mapping to the already-documented upstream_unavailable wire code; an undocumented code would have been silently rewritten to internal_error. Verified: the full unit suite (2300 tests; only the five pre-existing os.splice failures remain, identical on the base commit), the compose profile and mainnet contract suites, dockerized shellcheck and compileall, and a live container gate -- primary plus streaming standby, replicated data served through a moved endpoint, the staleness bound refusing, the writable-primary refusal, and mining-configuration still serving underneath both. * Make schema applies atomic and repair a poisoned carry-forward summary seed (#124) (#155) * Make schema applies atomic and repair a poisoned carry-forward summary seed The psql schema-apply path ran every statement of crates/qbit-prism/sql/001_share_ledger.sql in its own autocommit transaction, so the carry-forward summary sync triggers could commit before the one-time seeding block ran. An interruption in that gap, or a live writer confirming or reversing a block in it, left the summary holding only post-trigger deltas; the seed's NOT EXISTS guard then treated that partial summary as seeded and skipped permanently, on that apply and every later one, silently under-reporting every miner's balance by their full pre-upgrade carry (#124). - The schema script now wraps itself in one BEGIN/COMMIT transaction, so the apply is atomic under every client: the coordinator's psql backend, the native psycopg client, and a manual operator apply with plain autocommit psql. The old explicit BEGIN/COMMIT around only the audit-publication migration phase remains folded into that wrapper; the transaction-scoped advisory lock is now held for the whole apply. - _run_sql additionally passes --single-transaction, so every psql invocation (query batches included) is one transaction. psql tolerates the script's own wrapper with a warning; the combination was checked against ON_ERROR_ROLLBACK (never set: --no-psqlrc), \connect (absent), and explicit transaction control in runtime SQL (absent). - The seed guard compares the summary's active row-count total against the active carry history (one statement, so concurrent commits cannot manufacture a mismatch) and also checks summary drift, rebuilding from history on disagreement. A deployment already poisoned by the old guard is repaired by the next apply instead of locked in. - The psycopg ImportError fallback to the psql backend is no longer silent; it prints a warning line. - docs/prism-ledger-ops.md now documents the single-transaction apply requirement with the writer stopped. * Add a poison-and-repair regression test for the carry-forward seed A Docker-gated Postgres test (test/test-prism-postgres-seed-guard.sh, wired as make test-prism-postgres-seed-guard and run in the CI postgres-native-client job, which installs psycopg) reproduces #124's mechanism end to end: - The schema is applied non-atomically the way the pre-fix ops doc instructed (the script's own BEGIN wrapper is stripped and the head -- tables, functions, and the carry-forward summary sync triggers, but not the seed -- is applied with per-statement autocommit psql). - Pre-upgrade carry history is inserted with the summary triggers suppressed, then a live writer confirms a prepared block in the trigger/seed gap, leaving the summary holding only that post-trigger delta. - The old emptiness-based seed guard (embedded verbatim) locks the poison in: miner-a's pre-upgrade carry is silently absent. - A re-apply through the psql backend, and a re-poisoned re-apply through the native psycopg backend, both repair the summary to match the carry history with zero drift. - A second scenario injects a failure inside the apply's transaction on both backends and asserts nothing committed, proving the atomicity half of the fix. The container readiness gate follows the #134 pattern: the real TCP listener plus a confirming query, never the racy socket-only probe. Test harness built by a delegated remote worker; the split-marker, wrapper-stripping, and in-transaction failure injection were adapted to the landed self-wrapping schema script and reviewed here. * Address review nits: setval comment accuracy, ops-doc apply semantics, drift-only guard test - The setval comment no longer claims rollback undoes the allocator mutation on a pre-existing sequence (it does not); it now states the real residue semantics -- a failed re-apply leaves ordinal gaps only -- and warns future edits not to move validation after the setval. - The ops doc's writer-stopped rationale now describes the actual behavior under the atomic apply (concurrent mutations block on the apply's table locks and fire the recreated triggers afterwards, inviting lock waits and deadlocks) instead of the stale dropped-mutations mechanism, and documents the two harmless psql warnings the single-transaction + self-wrapper combination emits. - The regression test gains the guard's drift-only branch: a summary with matching row-count totals but corrupted balances is repaired by the next apply. * Scope the PRISM append-invalidation epoch per anchor (#126) (#151) * Extend the deterministic concurrency harness to block candidate landing Issue #128 ranks block candidate landing first by incident history, and the harness landed by #139 covered only the writer-lease lifecycle. This teaches it the landing state machine, which is the one owner where two threads of the same process reach the same durable rows. The PostgreSQL model gains the rows landing turns on: qbit_pool_blocks, qbit_block_candidate_outbox, the qbit_confirm_pool_block publication ordinal allocator and the durable floor read. The allocator is modelled on the shipped PL/pgSQL rather than on an approximation of it: the ordinal comes from a sequence, so an exact replay of an already-confirmed row burns nothing, a terminally disposed row reports the superseded disposition, and an aborted confirmation leaves a gap the MAX-based floor tolerates. That distinction is the whole of the two-tail defect -- a later ordinal can publish first and leave an earlier one behind the floor. Landing writes are staged per transaction for the same reason lease writes already were: a deadline-scoped statement sends COMMIT as a separate message, and the interesting window is inside it. Statement classification is anchored on fragments lifted verbatim from the production SQL, as the lease signatures already are, so a change to that SQL breaks classification loudly instead of quietly changing what the fake models. Where a modelled statement is a large CTE, the model reproduces the effects a landing reads back and says so. Three new pieces support the topology. Scheduler-aware locks, conditions, semaphores and events make a contended process-local lock an interleaving point rather than a wedged baton -- the lease scenarios never needed them, because they interleave two processes around one row, while the two landing tails share the payout-balance serializer, the publication order guard and the ledger's admission gates. Phase breakpoints let a scenario stop a tail at a boundary the landing already names for its own watchdog, because landing's interesting windows are the gaps between statements and no statement runs at the instant that matters. And HarnessBase now holds what every state machine's harness owns, so run_scenario and assert_deterministic are generic over it rather than typed to one harness; the two owners still owed to #128 plug in without touching the determinism check again. The acceptance criterion is #133's two-distinct-hash interleaving, driven end to end: block A confirms on the synchronous client-thread tail, block B lands on the accounting actor inside A's confirm-to-publish window and publishes the next ordinal, and A then publishes behind the raised floor. It is already fixed here, so the scenario passes; the evidence is that the interleaving can be expressed at all, which nothing could do before. The suite also pins the load-bearing claim underneath it -- the per-hash disposition lease does not serialise distinct hashes -- and the fresh lease proof that gates the superseded-envelope restore. Every scenario is proved order-stable over 25 runs of the full checkpoint trace. Refs #128, #133. * Drive the #125 escalated landing budget against the watchdog Issue #125: a landing-class ledger step is two waits, not one. The caller queues for the ledger writer lock -- local, unbounded by any statement deadline -- and only then runs the statement it queued for, under its own full budget. Both are spent on the block-work thread the watchdog monitors, and before 49f04c3 both were heartbeat-silent, so an escalated landing at the reviewed 120s cap could hold the heartbeat still for ~240s against a 120s tolerance. The watchdog hard-exited mid-landing and the in-memory escalation counter died with the process, so the restart came back at the 30s base to repeat the cycle. Add landing-harness scenarios for both halves of the fix: - a real accounting tail queued behind a held writer lock stamps wait-ledger-admission:landing every slice instead of going quiet, and an A/B against the same gate at the same budget with no progress hook shows slicing changes nothing about when the wait ends; - the escalation ladder is climbed by genuinely timed-out landings and every rung's admission-plus-statement pair fits the watchdog tolerance, across a clamped cap, a clamped base and a configuration needing no clamp at all, with the clamp reported exactly once; - both composed: a landing that queues for all but one slice of its escalated budget still lands, inside the tolerance, with no silence longer than one slice. Every scenario is proved order-stable with assert_deterministic. Co-Authored-By: Claude Opus 5 * Pin the landing half of the concurrency harness with self-tests The lease half of the deterministic harness is defended by tests/test_prism_concurrency_harness.py; the landing half added in cefdbfa was not. A fake that answers an unrecognised statement plausibly drifts away from production silently, and every scenario built on it inherits the drift, so a landing scenario that failed could not distinguish "the code under test is wrong" from "the model is wrong". tests/test_prism_landing_harness_model.py pins the model's own contract in four parts: - The publication-ordinal allocator against qbit_confirm_pool_block: a fresh confirmation takes the next sequence value, an exact replay returns 1 without burning one (proved by the next block's ordinal), every terminal disposition reports -1 rather than 0, a height mismatch and a missing row both report 0, and the durable floor — a MAX over the table — stays put across the gap an aborted confirmation leaves behind. - Transaction scope: a deadline-scoped landing write is invisible to another session until COMMIT and visible to its own before it, and a rolled-back landing leaves no row while keeping its ordinal burned. - Classification: every LandingOp is driven through the shipped PsqlShareLedger method that emits it, a landing table mention that matches no signature raises naming _LANDING_SIGNATURES, the deferred markers still name their follow-up state machine, and a deposed writer fails the lease CTE in the two shapes the SQL dictates — the outbox statements report it, the PL/pgSQL functions raise. - The synchronisation primitives: mutual exclusion and FIFO grants, non-blocking and virtual-clock-timed acquires, the non-reentrant re-acquire that a real mutex would deadlock on, RLock depth, the condition's release/re-acquire and restored depth, semaphore exhaustion and over-release, and the event's parked and timed waits. One landing scenario is proved order-stable with assert_deterministic against LandingHarness. Nothing sleeps in wall-clock time. Co-Authored-By: Claude Opus 5 * Scope the PRISM append-invalidation epoch per anchor (#126) The append-invalidation epoch is one global counter, so a landing that compares its own baseline against the live value learns only that some payout window was invalidated. A share stamped later than this candidate's declared anchor -- but early enough to predate a newer exposed window -- bumped that counter without touching this candidate's window, and the pre-offer fence read the inequality and terminally abandoned the candidate as append_epoch_stale. On the fallback submit lane that happens before qbitd ever sees the block, so a valid block's reward is forfeited. Record, with each bump, the stamp of the row that caused it (max(job_issued_at_ms, accepted_at_ms) -- exactly what _pending_share_predates_anchor compares), keep the most recent 256 in a bounded map alongside the oldest epoch still retained, and let a landing ask whether any bump in (baseline, live] was caused by a row that predates its own declared anchor. Both landing fences ask that instead of comparing counters. The predicate fails closed on everything it cannot prove: an unknown declared anchor, a baseline older than the retained history, or a gap in the scanned range. The post-offer branch is untouched -- it already logs rather than abandons -- and collection_only candidates stay exempt. The relaxation rests on the existing ordering: a landing exposes its declared anchor and drains unfenced in-flight appends before its fences arm, so every append that could predate the anchor has already bumped or bumps behind the fence. tests/test_prism_landing_append_epoch.py drives all of it through the landing harness: the over-abandon (now lands, confirms, publishes), the fail-closed pairs at and inside the anchor, both fences (the bump placed between the advisory read and the fence-held one via phase:tip-height-rpc), a pruned history, and a candidate with no declared anchor. Every scenario is proved order-stable with assert_deterministic. Co-Authored-By: Claude Opus 5 * Pin the vardiff resume TTL-boundary test to an exact clock base The retention test captured time.monotonic() and asserted that a silent reconnect cycle at exactly +TTL still resumes, relying on the store's strictly-greater expiry comparison. That boundary arithmetic is not exact for an arbitrary float base: (start + 900.0) - start rounds above 900.0 for roughly one in five thousand captured clock values, which flips the boundary cycle to expired and fails the test. CI hit this on the push run for the map commit while the pull_request run of the same commit passed. Every record and lookup in the test runs under frozen_clock, so the base never needs to be a real monotonic reading. Pin it to an exactly-representable constant, making the boundary arithmetic exact. Refs #145. * Close the disarm gap in the anchor-scoped append epoch (#126) The anchor-scoped predicate reads recorded bumps, so it is only as sound as the guarantee that every append predating a landable window's anchor actually bumps. That guarantee had a hole: a seeded window's anchor is exposed to the append-side predates() checks only by the armed payout-ledger artifact, and the bump that consults that anchor also disarms the artifact. The live anchor set could then go empty while jobs from earlier windows were still landable, and a replay row genuinely predating such a candidate's declared anchor committed silently -- _record_late_visible_payout_append returned None, no bump, no stamp -- so the landing's fences read only harmless history and submitted an invalidated window. Demonstrated with a deterministic interleaving: the pre-#126-fix tree failed closed on it; the anchor-scoped tree submitted. Route every write to the armed-artifact slot through _replace_payout_ledger_artifact_locked, which folds the outgoing artifact's snapshot anchor into the monotone, never-retired published-job-window watermark before the slot moves. The fold runs under _job_cache_lock -- the same lock every predates() read takes -- so no append can observe the anchor gone with the watermark not yet raised. The invariant that now actually holds, and that the fence comments now state: the anchor-set maximum never decreases while any job is landable, so every append that predates a live or seeded anchor bumps and leaves a stamp for the fences to read. The watermark's permanence is what the per-anchor predicate makes affordable: the extra bumps a retired anchor forces come only from replay-shaped rows (the pending-commit floor keeps ordinary commits above every job anchor) and are forgiven per anchor rather than over-abandoning. Reword the authoritative-fence comment in block_finalization.py: it claimed the recorded stamps were "the whole history the block's window can still see", which held for bumps but not for predating appends until the fold above made every predating append bump. Also pin the recorded stamp's max(job_issued_at_ms, accepted_at_ms) choice, which every equal-stamp scenario left indistinguishable from min() or either stamp alone: a straddling row -- issued at or before the candidate's anchor, accepted after -- does not predate the window, and recording anything but the max would abandon a valid block, resurrecting the #126 over-abandon. tests/test_prism_landing_append_epoch.py gains the disarm-gap scenario (premise: the bump disarmed the artifact; verdict: nothing submitted) and the straddling pair (lands, and abandons once both stamps predate), all proved order-stable with assert_deterministic; the landing harness's append_late_visible_share learns an additive accepted_at_ms. Co-Authored-By: Claude Fable 5 --------- Co-authored-by: Claude Opus 5 Co-authored-by: Alex * Land PRISM concurrency instruments and fix harness classification (#170) * Discriminate the outbox record from the share append in the harness classifier (#156) `OUTBOX_RECORD` was identified by two fragments that `_append_batch_with_replay_outcomes` also emits: it opens the same `INSERT INTO qbit_block_candidate_outbox (` and raises the same `'block candidate payload mismatch'` as `persist_block_candidate_intent`. A captured share submission therefore matched the outbox signature first and was answered by the outbox evaluator, so the harness reported the statement against the wrong state machine — arguing for an outbox defect that did not exist while hiding the deferral that did. Add the conflict action as a third required fragment. The intent writer inserts a candidate with no share attached and leaves a colliding row alone (`ON CONFLICT (block_hash) DO NOTHING`); the share append claims that row (`ON CONFLICT (block_hash) DO UPDATE`). That difference is the SQL-level statement of which caller owns the row, so it is a load-bearing identity rather than an incidental token. At this baseline the fragment occurs in exactly one ledger method. Self-tests pin all three directions: the shipped `append_batch`, captured at the port, now reaches `_DEFERRED_SIGNATURES` and names the share-submission machine; `persist_block_candidate_intent` still classifies as `OUTBOX_RECORD`; and the discriminator is asserted unique to the intent writer, so a second emitter fails here rather than at the next confusing scenario failure. Unknown SQL stays loud. Co-Authored-By: Claude Opus 5 * Land the on-demand PRISM performance instruments (#160) Two on-demand drivers under tests/perf/, neither collected by test discovery: their filenames do not begin with `test_` and nothing imports them. window_pipeline_gil_scaling measures how much of the payout-window materialization pipeline is actually GIL-held rather than assumed to be. This is recovered work, restored byte-for-byte from the artifact that was already written rather than re-derived: cores-used via process_time/perf_counter, positive and negative controls, a control-floor abort, a wall-clock floor, alternating repetitions, load average, a memory guard, JSON capture and render replay. The one deviation from the recovered bytes is a `# noqa: F821` with its reasoning: `inputs` is bound at the top of the loop body and released by `del inputs` at the end of it, which is enough for Ruff to read the closure as unbound, and measure_cores joins every thread before returning so no invocation outlives the binding. stratum_session_concurrency_load is rebuilt, because the original was lost with its workspace and no reachable object contained it. It drives the shipped StratumSessionService, VardiffService and ShareSubmissionService through a real PrismCoordinator with real threads, real sockets and a real wall clock -- deliberately not the baton harness, which replaces the scheduler, clock and lock that this question is about. Paced, saturating and herd modes; herd reports a per-session re-establishment census including the exact count that never returned, because a storm that settles at high throughput while a third of the pool gets no work back is a failure a throughput-only report would call a success. Metrics: cores-used, shares/s, CPU us/share, ack p50/p95/p99/max, lock acquisitions/contentions with hold attribution by call site, watchdog misses, rejections, and a fixed-work GIL-wait probe that charges its own CPU so it cannot inflate the coordinator's. Hold attribution lives in the driver, not in production: _ObservedRLock is right that a clock read per acquire/release is the fast-path cost it exists to avoid, so the wrapper is paid for only while measuring and --no-lock-attribution quantifies it. No production code changes. Ledger mode is selectable and named in every output -- in-memory to isolate interpreter cost, or a real PsqlShareLedger through the repository's real-server gate, since a shares/s figure means something different with and without a durable write. Difficulty is not permanently pinned: resume and climb both exercise the real retarget path. The health abort refuses to report any settle, load, herd or lock number when the host cannot demonstrate parallelism, and --selftest asserts that the abort report contains none of them. Co-Authored-By: Claude Opus 5 * Key the harness lease-acquire model to the shipped statement shape (#140) The acquire model hardcoded the pre-#140 result: when a first-acquire race loser could see neither its own upsert nor a committed holder, the evaluator returned SQL NULL unconditionally, and test_first_acquire_race_loser_gets_an_unhelpful_error pinned the generic "postgres query returned no JSON" diagnostic that produces. Once the ledger PR adds an explicit not-acquired/retry arm that outcome is no longer reachable, and a model that kept producing it would let a scenario assert something the server can no longer do. Read the shape off the statement instead. `acquire_result_arms` counts the result arms of the final SELECT COALESCE and `acquire_terminal_arm_keys` names the keys of the last one, both computed from the SQL PsqlShareLedger actually emitted. Two arms is today's shape and still evaluates to NULL; three or more carries a terminal arm and cannot, so the evaluator answers it. Counting arms rather than matching a chosen literal is deliberate: the arm has to exist for the fix to work, whatever its author names its keys. Unknown SQL stays loud in both new directions. A statement without the recognised COALESCE, or with fewer than two arms, raises UnsupportedStatement naming the shape; and the terminal arm's value policy is narrow on purpose -- `acquired` is false, holder and lease columns are null because nothing is visible, a retry flag is true, and any other key raises rather than being guessed at. The tests switch with the statement rather than being rewritten, and are not vacuous. The forward branch is exercised now, against a terminal-arm statement built from the shipped one, so a mistake in it surfaces here rather than as a harness failure blamed on the ledger PR. Both directions are pinned: the baseline is asserted to have exactly two arms and the old diagnostic, and the new branch is asserted to retire it. Mutating the arm counter to a constant, forcing either branch, or letting an unteachable key through each fail this suite. No production files are touched, and this stays green on the current baseline. Co-Authored-By: Claude Opus 5 * Model the shipped lease-acquire retry arm's literal values (#140) The signed ledger branch renders a terminal arm with four keys: `acquired` false, `lease_snapshot_retry` true, `lease` naming the subject, and `retry_reason` carrying a sentence. The narrow policy landed earlier answered two of them wrongly. It refused `lease` outright, because the key was on neither of its lists. Worse, it answered `retry_reason` with `True`, because the policy keyed on the key's *name* and the name contains "retry" -- so it got `lease_snapshot_retry` right by luck and would have reported a boolean where production emits an explanation. Answer what the statement says instead of what the key is called. `acquire_terminal_arm_fields` pairs the terminal `json_build_object` arguments and returns each value's raw SQL text, and the value reader resolves `true`/`false`/`null` and single-quoted literals from it. The subject and the reason therefore come out of the shipped SQL verbatim and cannot drift from the wording the ledger PR chose. Splitting the arguments respects parenthesised sub-expressions and quoted literals, because the holder arm nests `GREATEST(0, EXTRACT(...))` and a reason sentence must survive as one argument. Unknown keys stay loud. The key set is still an allowlist, so a key this model has never seen is refused even when its value is a perfectly readable literal, and a literal key given a non-literal value is refused rather than guessed at. The new integration self-test reads `share_ledger.py` out of the signed ledger commit read-only, renders its final COALESCE with that commit's own retry constants, splices it onto the statement production emits here, and requires the exact terminal object -- not NULL and not a substitute. It matters that this reads the real arm: the synthetic fixtures the first attempt was written against were chosen to match its policy, which is exactly why both defects survived them. The expected object is written out independently rather than derived from the same parse, so the assertion cannot agree with itself. Mutations checked: name-based values, a fixed guess for string literals, a quote-blind splitter, a paren-blind splitter, a silently accepted unknown key, a three-arm shape still answered as NULL, and the baseline wrongly answered with the terminal object all fail this suite. Dropping the doubled-quote branch does not, and is an equivalent mutant -- toggling the quote state twice over adjacent quotes yields identical output -- rather than an untested path. No production files are touched, and the current baseline stays green. Co-Authored-By: Claude Opus 5 * Make the retry-arm coverage self-contained (#140) The integration self-test in a2ec459 read `share_ledger.py` out of a foreign commit object and fetched it when absent. That is the wrong trade for a test suite. It makes a result depend on the network and on objects this repository does not own, so it fails on a shallow clone, a pruned object or an offline runner for reasons that say nothing about the tree under test -- and its skip-when-missing fallback was worse still, because coverage that quietly disappears reads as coverage. Remove it, along with the machinery it needed: the subprocess and pathlib imports, `_git`, `_share_ledger_source_at`, `_acquire_coalesce_tail`, and the `LEDGER_RETRY_COMMIT` constant. No test path invokes git or opens a socket now, checked statically and by running the suite with subprocess and socket replaced by tripwires that raise: 34 tests, zero failures, zero skips. What that test was protecting is kept. The terminal arm stays as the captured fixture, spliced onto the statement production emits here, and the exact expected object stays written out independently of it, so the assertion still cannot agree with itself. The value-type assertions move across with it -- a boolean in `retry_reason` is the precise defect the first policy had, and dict equality alone would read as a typo check. The provenance of the fixture bytes (PR #168, commit 869bb7e) is now a comment documenting where to re-capture from, and nothing reaches for it. The literal-valued parser policy and the punctuation/quote regression are untouched; `prism_concurrency_harness.py` does not change in this commit. Coverage did not regress with the removal: name-based values, a fixed guess for string literals, a quote-blind splitter, a paren-blind splitter, a silently accepted unknown key, a three-arm shape answered as NULL, and the baseline answered with the terminal object all still fail this suite. Co-Authored-By: Claude Opus 5 --------- Co-authored-by: Claude Opus 5 * Name writer-lease races and enforce read-only sessions (#168) * Make the first writer-lease acquisition race diagnostic (#140) Two coordinators racing the first-ever acquisition against an empty qbit_ledger_writer_lease left the loser dying in PsqlShareLedger.__init__ with "postgres query returned no JSON" -- an error naming neither the lease nor the coordinator that holds it, at pool bootstrap, which is exactly where a misleading error costs the most. Both arms of the acquisition statement's COALESCE can be empty at once. Under READ COMMITTED the loser's INSERT ... ON CONFLICT DO UPDATE waits on the singleton unique index, then re-reads only the conflicting tuple through EvalPlanQual: the winner's identity differs and its lease is unexpired, so the upsert affects zero rows. The holder arm is a plain SELECT, which still reads the statement snapshot taken before the winner committed, so it finds no row either. COALESCE(NULL, NULL) is NULL, and parse_single_json_value turns that into the driver-level error. Make the statement total instead. A third COALESCE arm is a constant json_build_object naming the qbit ledger writer lease and marking the result not acquired and retryable, so the expression cannot evaluate to SQL NULL however the first two arms resolve. The retry is a signal local to this one statement -- the row is committed, and only a fresh statement snapshot can see it -- so _try_acquire_writer_lease consumes the sentinel itself and re-runs, converging to the ordinary acquired or held-by arm. _ensure_writer_lease never sees it, which matters because the sentinel carries no holder to wait on or adopt. The retry is bounded at three fresh snapshots. Two suffice under READ COMMITTED; the third covers a re-raced row and bounds a caller whose transaction pinned one snapshot for its whole life, which no retry can advance. Exhausting the bound raises an error that still names the lease, so no raw parser error can escape the constructor for this race. parse_single_json_value is unchanged: only the lease statement became total, and any other statement returning SQL NULL still fails loudly rather than being reinterpreted as a retry. tests/test_prism_share_ledger.py drives the loser's side through canned statement results: the sentinel converging to the named holder refusal, the sentinel converging to an ordinary acquisition, the sentinel being rejected by both the wait and adopt predicates it must never reach, an unconverging retry failing while still naming the lease, and an unrelated NULL-returning statement still raising the generic parser error through the native client. Ordinary acquisition, renewal, live-holder refusal and expiry takeover keep their existing coverage. Refs #140. Co-Authored-By: Claude Opus 5 * Enforce read-only PostgreSQL sessions for read_only ledgers (#163) PsqlShareLedger(read_only=True) refused the writer gate but never told PostgreSQL the session was read-only. The gate covers every path that takes it, which is most of them, but any statement issued outside it -- the lease upsert, release_writer_lease_fresh_connection's one-shot psql UPDATE -- reached SQL exactly as it would on a read-write ledger. Nothing bad happened, for a reason outside this class: since #157 the public tier reads a hot standby, and a standby refuses writes itself. So the guarantee was supplied by the deployment topology rather than by the object that advertises it, and it lapses in two supported cases -- PRISM_PUBLIC_REPLICA_MODE=off, the code default, which aims the same read-only ledger at the primary, and any future caller constructing read_only=True against a read-write database. Fail closed at connection time instead. PostgresSessionGuards, the libpq options carrier #137 established for the orphan-reaping GUCs, gains a read_only flag that appends default_transaction_read_only=on. That is one carrier, not a second GUC transport, and it is why the coverage is total: all three connection paths already share this object, so pool slots, read slots, the dedicated lease-guard session and the psql subprocess all inherit the setting, and a connect-time GUC covers autocommit statements that no per-transaction spelling would reach. Operator-supplied options still survive, and the coordinator fragment still goes last, so a DSN that sets default_transaction_read_only=off cannot turn the session writable again. The refusing writer gate is unchanged and remains defense in depth. The essential test target is a writable primary: a standby would refuse these writes whoever asked, so it can never show that read_only=True is what refused them. Both PostgreSQL integration scripts now build a read-only ledger against the primary they have spent the whole run writing to, and assert that a real write method taking no writer gate is refused by the server -- release_writer_lease_fresh_connection on the psql path, and the writer-lease upsert over the pooled psycopg session on the native path, so both kinds of connection one instance can create are covered. Each is contrasted against a writable ledger of the same identity running the same statement, and each read-only ledger is shown still serving reads. Both scripts reach PostgreSQL through "docker exec", which does not forward the caller's environment, so PGOPTIONS -- the only session-GUC transport on the psql path -- was being dropped before it reached the server. The read-only assertions run through a command that asks docker to pass the variable through; an external psql inherits the environment already. Unit coverage pins the conninfo and PGOPTIONS composition for the pooled client, the lease-guard session and the subprocess backend, the precedence of the coordinator fragment over an operator's DSN options, that a writable ledger's session is untouched, and that the writer gate still refuses. Refs #163. Co-Authored-By: Claude Opus 5 --------- Co-authored-by: Claude Opus 5 * Serve bounded-stale public reads during database outages (#167) * Remove the PRISM first-job admission lock convoy (#171) * Remove the PRISM first-job admission lock convoy (#159) * Align concurrency harness with lease retry arm * Add a differential parity oracle for the payout-window pipeline (#165) * Add a differential parity oracle for the payout-window pipeline The window pipeline (accepted-share records in; folded window, canonical digest, and audit-builder spool fragments out) is slated to gain a second implementation that must be proven byte-identical to the shipped Python one before it can take traffic. This adds the proof apparatus ahead of that implementation, composing the two existing precedents: the seeded-corpus discipline of the incremental payout-window golden tests (whose record factory it reuses) and the frozen byte-exact reference discipline of the metrics render-parity fixture. tests/window_pipeline_parity.py carries the corpus, the adapter seam, and the frozen-reference plumbing. The corpus is derived from the fold's own invariants: the retained crossing row paired with an exact-fit sibling, independent job/accepted eligibility exclusions with an on-anchor boundary row, unsorted input, credit_policy present and absent (null-vs-absent in the compact fragments), identity deduplication, non-ASCII/escaping including a non-BMP surrogate pair, page boundaries at 511/512/513 of the default page size, a multi-page interior cutoff, an empty and a single-record window, incremental advances that must land on full-rebuild bytes, and a seeded bulk case with a 700-record advance. Each case pins three byte outputs: the ordered per-record canonical encodings, the framed canonical document bytes plus their SHA-256 digest exactly as the digest streams page fragments, and the spool payload tail written by the real acquire_spooled_tail path. Small cases pin literal bytes for diagnosability; large ones pin length plus SHA-256. The frozen reference regenerates reproducibly with `python3 -m tests.window_pipeline_parity regenerate`, a no-op on an unchanged tree and enforced as such by test. The runner drives any registered backend over the same corpus through a plain adapter seam (inputs in, three byte outputs out), defaulting to the shipped Python pipeline; a future backend registers an adapter and is selected via QBIT_WINDOW_PIPELINE_PARITY_ADAPTER. No production switch is added anywhere. Divergence-detection tests drive deliberately perturbed backends (field reordering, missing page separator in the digest framing, explicit-null credit_policy, a dropped crossing row, spool whitespace) and prove the harness fails exactly where it should and passes where the perturbation is invisible, which is the evidence that single-page or crossing-free corpora alone would prove nothing. Co-Authored-By: Claude Fable 5 * Close the parity oracle's integer-width and rejection holes An adversarial review of the window-pipeline parity oracle found that its corpus and seam let wrong second implementations through: every integer the corpus fed the pipeline sat below 2^31, so nine perturbed backends carrying integers in u32/i32/u64/i64/u128 (wrapping or saturating) and an f64 round-trip all passed 14/14; and no input triggered any of the fold's eleven rejections while run() could only return outputs, so a fold with every check removed -- or one that silently performs the coordinator's full-rescan recovery and returns its bytes -- also passed 14/14. Both holes were already blessing a registered backend. This closes them and the four should-fix items, as a new commit on top of the oracle. Corpus. wide-integers carries every integer field above 2^32 at a realistic epoch-millisecond anchor, share_difficulty across 2^53+1, 2^63, 2^64+1 and 2^127+3, network_difficulty up to 10^77, template_height and ntime to 2^62 and 2^63-1 plus a legal zero pair, share_seq to 2^63-1, and a window_weight of 2^127+2^64+2^63+4 arranged so the three newest rows fall short of the weight only because of bits above 2^64 -- a 64-bit accumulator stops one row early, so the retained count diverges, not just the digits. The decision the review left open is made explicitly and documented in the module docstring: the corpus also carries values at and above 2^128 (difficulty-beyond-u128: share_difficulty 2^128+1 as the retained crossing row, network_difficulty at 2^128-1, 2^128 and numeric(78,0)'s maximum, window_weight above 2^128). The ledger schema is the only written width contract and the mandated wide-integers values already hold a backend to it rather than to any existing narrower type, so the oracle refuses a 128-bit difficulty accumulator -- saturating or not -- instead of blessing it silently; narrowing that contract belongs in the schema plus a visible edit to those cases. A structural assertion pins share_difficulty.bit_length() > 64 (and > 128 in the second case) so neither can be quietly shrunk. Seam. run() now returns WindowPipelineOutputs | WindowPipelineRejection, where the rejection carries one of eleven implementation-neutral categories (five for from_full_snapshot, six for advance), never an exception class or message; the Python adapter classifies the shipped fold's exceptions by exact message and re-raises anything it does not know. The fixture freezes {"rejected": ""} for such cases, and diverging_outputs compares rejected-vs-rejected by category and reports rejected-vs-outputs in either direction. One case per condition is added (both eligibility exclusions separately; the repeat case re-delivers a share still inside the window; the three DB-unreachable full-snapshot shapes are pinned anyway because the in-memory ledger and the daemon protocol are not the database). The docstring states that the oracle pins the pipeline's rejection, not the coordinator's recovery from it. Advance path. advance-exact-fit-expiry separates >= from > in the advance loops; advance-delta-exceeds-window makes expiry run into the freshly appended pages (the shape a drop-prefix/append-suffix protocol must get right); advance-partial-expiry-in-appended-page lands partial expiry inside a fresh page; and one-record cases cover an advance from an empty window, an empty delta at a later anchor, and an equal anchor (the check is <). The per-advance IncrementalWindowAdvanceStats triple is brought into the seam as advance_stats and frozen per step, with touched_pages' definition spelled out on the seam type: a page appended into and then expired wholesale still counts, a freshly allocated page never does. Escapes and regeneration. non-ascii-strings gains a record carrying DEL, solidus, CR/BS/FF, U+001F, U+2028/U+2029, space and uppercase in share_id and the identity fields, with the corresponding needles (and the absence of a raw DEL or an escaped solidus) asserted structurally. The corpus owns its record factory instead of importing the sibling golden test's, and literal-pinned cases freeze the input document itself, so an input-only change shows in the fixture diff as such instead of as a wall of moved hashes. The reference schema is bumped to v2 for the new entry shape and checked on load. Every closed hole is demonstrated, not merely green: throwaway perturbed backends modelling each divergence (the nine integer widths plus a difficulty-only saturating u128, a fold that never refuses, a misfiled category, strict advance expiry, expiry confined to pre-existing pages, an equal-anchor rejection, a trailing separator on an empty delta, a last-page unwrap on an empty window, a raw DEL, an escaped solidus, zeroed and survivor-derived advance stats) pass 0/14 against the previous corpus and fail against this one; the in-repo divergence-detection suite keeps a representative of each class and asserts it fails exactly where it should and passes where the perturbation is invisible. Co-Authored-By: Claude Fable 5 --------- Co-authored-by: Claude Fable 5 Co-authored-by: Alex * Route the block-candidate landing tail through the named submit port (#127) (#172) The block-candidate service decided which landing tail to run by reaching through the runtime seam for `_is_production_block_submit`, an identity comparison against `PrismCoordinator.submit_block_candidate`. That put a fact about the coordinator's own entrypoint inside a leaf owner, and split one landing decision across two modules. Broaden the existing named `BlockCandidatePorts.submit_candidate` seam to carry the whole landing call shape -- an already-created `node_submission` when the caller made one, and whether the caller already owns the same-hash disposition -- and let the coordinator resolve the entrypoint per call in `_land_block_candidate_submission`. Both landing consumers now state what they have and call the port; neither inspects the runtime. Late binding is preserved: the port lambda resolves `self.submit_block_candidate` at call time, so a replacement installed after `BlockCandidateService` is constructed is still observed. The synchronous path's two-way routing and the writer path's three-way routing, including its middle `node_submission is None` arm, are reproduced exactly behind the port, as are node submission, accounting, disposition ownership, serialization, and ordering. This is structural hygiene. Tests that install submit doubles still do not execute `_submit_block_candidate_serialized`; a new regression pins the other side, that ordinary production operation still reaches that serialized tail. Co-authored-by: Claude Opus 5 * Move the payout-window fold, digest, and advance() behind the --serve daemon (#166) * Move the payout-window fold, digest, and advance() behind the --serve daemon First slice of the #131 window-pipeline migration, stacked on the #165 parity oracle. The --serve audit builder gains a prepare_window request (protocol 2) with full and advance modes: the daemon owns the payout-window fold, the canonical digest (which is the window cache key), and the incremental advance, byte-identical to the shipped Python pipeline and proven by the existing 14-case parity corpus through a registered rust-daemon adapter. Prepared windows unify with the build window cache, so builds whose share_snapshot_sha256 matches a prepared window hit it with no window upload; prepared state is epoch-tagged and dropped by any request carrying a newer append-invalidation epoch. The coordinator holds the folded window as an opaque, digest-verified byte mirror of the canonical items stream (full stream on full mode; prefix-drop plus appended suffix on advance), from which shares_json parses lazily: the routine build path never materializes dicts or compact payloads, while the rare consumers that need real dicts (the durable candidate intent, the found-block canonical audit build, the one-shot builder fallback) parse the in-memory bytes at their own call sites with no database re-read and no daemon dependency on the landing path. Switched by PRISM_WINDOW_PIPELINE_RUST, default Python, strictly boolean (startup refuses anything else), and inert while PRISM_BUILDER_SERVE is off. needs_full and fallback are ordinary control flow: a daemon that lost its state is re-seeded through a full preparation, and an advance invariant failure takes exactly the IncrementalWindowFallback path (cache cleared, full rescan reasoned incremental_invariant_failed) without retiring the daemon. Any daemon anomaly degrades to the in-process pipeline for that materialization; the Python window pipeline stays fully intact. Co-Authored-By: Claude Fable 5 * Fix three degraded-mode defects in the window-pipeline daemon slice A prepare that cannot get the daemon promptly now degrades instead of retiring it. prepare_payout_window stamped its deadline before an untimed blocking acquire, so a preparation that queued behind a slow build arrived inside the lock with its whole budget already spent: the first write read as a daemon timeout, and the anomaly handler recorded a live termination, SIGKILLed a healthy daemon, counted a fallback, and sent the caller into a cache-clearing full DB walk, full fold and full window upload. The wait is now bounded by the preparation's own budget and the exchange deadline is stamped after the acquire, so a build's runtime is never charged against the daemon's health. Losing the race answers a new "busy" outcome -- ordinary control flow, counted as window_prepare_contended, surfaced as a window_daemon_busy full-rescan reason -- and bypass_build_interval materializations (the accepted-block preview, which takes the prepare lock non-blocking precisely so publication never queues) refuse to queue at all. A declared record count is reconciled with the mirror's own bytes eagerly. The construction-time digest pinned the bytes and said nothing about the count beside them, so a miscount could only surface at whichever consumer first parsed the stream -- possibly a serve request that had already written its prefix, leaving the daemon's stdin holding an unterminated line for the next request to concatenate onto. The count is now walked out of the bytes next to the digest check: fully for a full mirror, and from the dropped prefix plus appended suffix for an advance, so the hot path stays proportional to the surgery. Separator placement at both seams is checked too. Any DaemonWindowMirrorDivergence that can still escape now invalidates the cached window and counts a window_mirror_divergence event, and a request that escapes half-written retires the daemon on any exception rather than only the enumerated anomalies. The daemon's epoch tag no longer evicts prepared windows. The coordinator's own late-append policy is finer -- it clears the incremental cache only when the late row predates the cached anchor, and otherwise retags the window so the next delta build advances it in place -- and the daemon never chooses a window: entries are addressed by content digest, validity is decided by the install fence and the job-bundle epoch checks, and an advance base is the coordinator's own digest-verified mirror. A tag mismatch could catch nothing the coordinator had not already decided; it could only turn every retag into a full rescan under exactly the replay bursts that produce the bumps. The tag now rides along for diagnostics only, echoed back on advance beside the requesting epoch. An unheld digest remains the only needs_full. Co-Authored-By: Claude Opus 5 * Wire the Rust adapter to the rejection seam and settle the integer-width contract Running the strengthened 34-case parity corpus through the rust-daemon adapter left 22 cases not passing, for three causes, none of them in the fold itself. The adapter raised on every non-ok envelope, so the daemon's correct fold_invalid/fallback refusals (12 cases) read as harness errors; it never relayed the daemon's added_rows/expired_rows/touched_pages into the seam's advance_stats (8 advancing cases); and the two wide-integer cases died in serde with "number out of range", which the daemon answered as a malformed request -- the one shape the coordinator treats as an anomaly, so a window holding such a value retired a healthy daemon and respawned it on the next materialization only to kill it again. Rejections now cross the wire as structure, not prose. Both window error enums gain rejection_category(), the parity oracle's own vocabulary (five full-snapshot conditions, six advance conditions; a Rust unit test pins the eleven strings), and the daemon's fold_invalid and fallback envelopes carry it as `rejection` beside the human-readable `error`, which is diagnostic and free to drift. The adapter maps on that field alone -- an unknown or missing category is a harness error, never a guess -- and takes each advance's stats triple from its envelope. PreparedWindowOutcome carries the category too. Protocol 2 is already this branch's bump; its doc now lists these fields. The integer-width contract is narrowed to the Rust widths, deliberately and in writing: share_seq u64, share_difficulty and network_difficulty u128, template_height u64, ntime u32, the millisecond stamps and anchors i64, window_weight u128, with the fold's difficulty accumulators u128 as well. The reasoning sits in the parity docstring and beside the Rust types: a share is (pow_limit_target*1e6)//target, about 2^98 at a mainnet-scale target, so a 16x window totals about 2^102 -- a factor of roughly 6e7 short of 2^128 and unreachable on any chain this code will see; numeric(78,0) is a generic wide column, not a statement that difficulties exceed 2^128; and arbitrary precision would put bignum arithmetic on the path the migration exists to speed up. No bignum crate. The narrower domain is modelled as a domain, not a divergence. The seam gains a third outcome, WindowPipelineUnsupported(reason), and each adapter declares a WindowPipelineIntegerDomain (unbounded for Python, the table above for rust-daemon; an adapter declaring nothing claims everything). The frozen reference keeps Python's bytes for every case; diverging_outputs accepts Unsupported only where the case really lies outside the backend's declared domain, still fails wrong bytes there (a saturating u128 accumulator declaring the Rust domain is held to the frozen digits), and fails Unsupported for a case inside the declared domain. The divergence suite demonstrates each of those with perturbed backends. The daemon declares it cleanly. A line the strict parse refuses is re-parsed once, with exact digits, and a prepare_window request whose integer lies outside its declared width is answered as out_of_range -- naming the field, the literal and the width -- while a corrupt line stays a malformed request. The accumulators are checked rather than saturating: a retained or advanced total that leaves u128 is declined the same way (field total_difficulty) instead of silently changing later expiry. The coordinator treats out_of_range as ordinary control flow: a full preparation so declined falls to the in-process fold, an advance so declined rebuilds through the full path (reason window_value_out_of_range), the daemon keeps its state and is never retired, and the declines are counted as window_prepare_out_of_range. The fake serve builder mirrors the declared widths and rejection categories, and regression tests prove a daemon that receives an out-of-range window survives two materializations with one spawn and no termination, at both the snapshot and the delta. The width cases are split so the original protection survives. wide-integers now carries every integer far above 2^31/2^53/2^64 but inside every declared width (share_seq to u64::MAX, network_difficulty to u128::MAX, ntime to u32::MAX, share_difficulty still 2^127+3 with the 2^64-dependent cutoff), so both backends produce bytes and full parity is enforced there; its ntime 2^63-1 moves to a new ntime-beyond-u32 case beside difficulty-beyond-u128, both frozen with Python's bytes and answered Unsupported by the daemon. Re-running the backends that motivated the case -- u32-wrapping, u64-wrapping and a u64-saturating accumulator -- each still fails the retuned wide-integers (record_count 2 and 3 against the frozen 4 for the two that touch the accumulator) while passing the sub-2^31 corpus, and that check is kept in the divergence suite. With the 35-case corpus, both adapters pass every case with no case reported as an error; the fixture regenerates as a no-op. Co-Authored-By: Claude Fable 5 --------- Co-authored-by: Claude Fable 5 Co-authored-by: Alex * Gate payout-window parity on real Rust daemon (#178) * Persist byte-exact canonical audit bundles (#173) * Serve persisted canonical audit bundle bytes * Backfill verified canonical audit bundles for existing rows Rows written before the immutable canonical artifact API advertise an audit_bundle_sha256 with nothing canonical behind it. Add a standalone, idempotent, resumable backfill that walks qbit_pool_audit_bundles in primary-key order and recovers the exact advertised bytes for each row: * a pre-v2 literal external body is promoted verbatim, never parsed and re-emitted, because those bytes are the serialization the digest was computed over; * a compact/v2 external body is reconstructed through the artifact store, with an absent on-disk share segment reloaded from the append-only ledger, canonicalized with the store's segment encoder, and checked against the segment digest the bundle embeds; * an inline JSONB body is canonicalized with the repository's existing canonicalizer. Every candidate is verified against the advertised digest before anything reaches the filesystem, and the ledger's pre-publication preflight (writer-lease refresh plus row identity confirmation) runs immediately before each write. A row that fails verification is reported on stderr with its block, digest, and reason, left untouched, and turned into a nonzero exit status once the requested range has been processed; the advertised digest is never rewritten. The phase-2 canonical artifact API and the ledger preflight are bound by name in two small adapters that report what they resolved in the run summary, so reconciling a final signature difference is a local edit. Co-Authored-By: Claude Opus 5 * Persist the canonical audit-bundle bytes at publication time The public digest of an audit publication is the SHA-256 of the canonical JSON bundle, but only a compact pointer body reached the disk. Answering for that digest meant reconstructing the bundle from share segments and trusting the reconstruction, and a segment that had since been pruned made the identity unanswerable. Persist the exact bytes instead. `prepare_external_audit_body` already holds the canonical bytes it just digest-checked, so it publishes them as a first-class artifact kind before deriving the compact body -- reusing the bytes rather than reserializing, and running after the writer-lease preflight so the copy stays behind the same fence as every other audit side effect. The store gains the artifact kind and its APIs: a content-addressed path, a writer that verifies the uncompressed digest before publication and again after reopening the published file, and a reader that separates a clean miss from damage so a caller can log the loss before falling back. Compression is stdlib gzip level 9 at mtime=0, which is reproducible, so republishing identical bytes is a durable no-op and a conflicting artifact fails loudly. For the historical backfill, the resolvers now accept a missing-range callback and a helper reconstructs a bundle from an already published body. Recovered ranges are canonicalized and checked against the digest the body itself commits to, then the assembled bundle against the advertised digest, so database fallback bytes are never trusted on their own. Pre-v2 bodies that already are the canonical bundle are taken verbatim. A ledger preflight refreshes the writer lease and confirms the block/digest row identity immediately before each backfill publication. Canonical bundles are classified as their own kind, counted in metrics, and never touched by retention. * Bind canonical bundle backfill to storage APIs * Fix canonical artifact test annotation * Document canonical bundle backfill rollout * Keep canonical backfill dry runs read only * Fix canonical bundle republication across compressors * Address canonical audit bundle review findings --------- Co-authored-by: Claude Opus 5 * Make the interpreter thread switch interval tunable (#179) * Make the interpreter thread switch interval tunable PRISM is one CPython process and the GIL is its throughput ceiling, so how long a CPU-bound thread may hold the GIL before the interpreter forces a switch is a real tuning parameter. The tree never set it, so CPython's 5ms default has been in force on every deployment to date. #143's gate evidence measured it as a lever rather than a micro-optimisation: a sweep moved throughput 15% and CPU-per-share 13%, and the 5ms default had the worst ack tail of the three settings tested. That finding has sat unused because there was no way to act on it short of a code change. Adds PRISM_PYTHON_SWITCH_INTERVAL_SECONDS, applied in main() before the coordinator is constructed -- the interval is process-global, so applying it after threads exist would have running threads observe the change at an arbitrary point in their own scheduling. Unset means sys.setswitchinterval() is not called at all, rather than called with 5ms. The two differ if a future interpreter changes its own default, and this knob should not pin a value the tree never chose. Empty reads as unset too, which is what lets compose pass the variable through unconditionally. Values above 1 second are rejected at startup. A misplaced decimal point ("5" meaning 5ms) would let a thread hold the GIL for whole seconds, and that failure presents as unexplained ack latency and lock convoys rather than as a configuration error -- the hardest class of fault to read back from a running pool. No default value ships. The right interval trades ack tail latency against interpreter overhead and the balance is host-specific, so it must be measured per host with tests/perf/stratum_session_concurrency_load.py rather than copied between machines. Co-Authored-By: Claude Opus 5 (1M context) * Reject switch intervals the interpreter would truncate to zero CPython stores the thread switch interval in whole microseconds, so a positive value below one truncates to zero rather than to something small: sys.setswitchinterval(9e-07) leaves sys.getswitchinterval() reporting 0.0. Verified on python:3.12-slim, the deployment runtime. A zero interval asks the interpreter for a switch check at every opportunity -- the opposite of what someone lowering the value is reaching for, and worst in exactly this process, which is thread-heavy by design. The validation accepted it because it only tested for positive. Adds a lower bound of 1e-06, symmetrical to the existing upper bound and for the same reason: a value that passes validation but degrades the pool is harder to diagnose than one that refuses startup. The bound is the interpreter's own resolution rather than a number chosen here, so a test pins it to the interpreter -- the floor must apply intact and anything below it must still truncate to zero. If a future CPython changes its resolution that test fails, instead of the constant silently drifting out of agreement with the runtime it is meant to describe. The same truncation applies harmlessly above the bound: a requested 1.5us applies as 1us, so a tuned value may read back rounded down. Documented rather than rejected, since losing sub-microsecond precision is benign where losing the whole interval is not. Reported by Codex review on the pull request. Co-Authored-By: Claude Opus 5 (1M context) --------- Co-authored-by: Claude Opus 5 (1M context) * Keep PRISM telemetry useful under overload (#191) * Serve stale PRISM metrics as HTTP 200 with explicit freshness metadata (#184) /metrics answered 503 whenever the complete snapshot exceeded its staleness budget. Prometheus discards the body of a non-200 response, so the last complete document -- and the diagnostic gauges that explain why it was the last one -- were thrown away at exactly the moment they were the only thing left to look at. A stale complete snapshot is now served as 200 with the prior payload unchanged plus the current stale/age gauges. Only warm-up, where nothing complete has ever been published, still refuses with 503 and a diagnostic-only body. Freshness is stated out of band so a consumer never parses Prometheus text to find it, following the #164 header style: a bounded X-Prism-Metrics-State of fresh/stale/unavailable, an integer Age on served payloads, and the registered Warning 110 warn-code only on a stale payload that was served. The cached read now returns a MetricsSnapshotResponse carrying those facts, which still destructures as the (status, body) pair its callers unpacked. Nothing is rendered or collected on the scrape thread in any case, so a collector wedged on the GIL or on the renderer lock can neither delay nor re-enter this path. MetricsRenderer is unchanged. Co-Authored-By: Claude Opus 5 * Export the exact unresolved accepted-parent depth and its cap (#188) qbit_prism_accepted_parent_unresolved_transitions was rendered from len(accepted_parent_unresolved_ages_seconds()), which only sees landed transitions that carry a monotonic landing stamp. The admission fence uses _accepted_parent_unresolved_depth(), which counts every landed transition. Two reachable paths arm landed without a stamp -- the degraded preview-publication fallback and the durable candidate replay both rebuild the transition from the dataclass default -- so the gauge could read zero while the fence already refused admission at depth one. Render the gauge from the fence's own depth, and export the fence's configured limit as qbit_prism_accepted_parent_unresolved_depth_max so an alert can compare the depth against the cap the runtime actually enforces. The oldest-age gauge keeps deriving from known timestamps: a transition with no stamp is counted in the depth without inventing an age for it. Co-Authored-By: Claude Opus 5 * Specify PRISM overload and backlog alerts (#188) Add docs/prism-overload-alerts.md, the in-repository alert specification for issue #188's overload and backlog classes, based on the 2026-08-20 testnet4 capture and the completed signal inventory. The document is a specification only: this repository holds no Prometheus, Alertmanager, or Grafana configuration and does not own deployment monitoring, so nothing here is live until the separate deployment repository applies it. Ordering is stated for external rule deployment: #184 must be present in the target PRISM build before the rules are applied there, even though the combined delivery integrates #184 ahead of this change. The document states the before/after explicitly. Before #184, a stale complete snapshot was served as 503 and Prometheus discarded the body, so a degraded endpoint produced no firing alerts and looked like a healthy pool. After #184, a stale complete snapshot is served as 200 with freshness metadata and is ingested, so the stale and age conditions are the runtime detectors; only warm-up with no complete snapshot still returns 503. Seven alert classes are specified with exact conditions, pending windows, severities, and operator first actions: preview wait timeout increments, unresolved parent depth at and near the exported cap, unresolved oldest age past the 5s preview budget, durable candidate count and oldest age, semantic current-work coverage loss, refresh pending past the 15s health deadline, and stale/unavailable snapshots alongside up == 0. The snapshot-unavailable rule is described as startup and no-complete-snapshot coverage rather than as a refresh-failure detector: after warm-up the gauge is part of the last served complete payload and stays 1 while the refresh is stale, and it may not be ingested during the 503 warm-up window it describes. Thresholds are split by provenance. Configuration- and contract-derived values (preview budget, depth cap ratio, refresh deadline, staleness floor) transfer across cadences. Candidate counts and the short count-based timeout threshold are testnet-cadence-dependent, are labelled as such, and carry a re-baselining checklist rather than being presented as mainnet constants. The inventory records that the parallel #188 gauge task both adds the depth cap gauge and corrects the unresolved-transitions count to the exact admission-fence depth, including landed transitions carrying no timestamp, so the at-cap alert coincides with the refusal. Oldest age stays limited to transitions with a known landing timestamp and invents no age for the rest; the resulting population difference between the depth and age signals is documented at both rules. Cautions cover the zero-client ratio reading 1.0 rather than 0, the -1 unavailable sentinels, counter-reset handling via increase(), and the need for an independent up == 0 alert because body metrics cannot report their own absence. Co-Authored-By: Claude Opus 5 * Stabilize PRISM alert specification references Replace line-number citations that became stale when the #184 and #188 changes were combined with file-and-symbol references. Describe the gauge addition as an accompanying change so the specification matches the single delivery. --------- Co-authored-by: Claude Opus 5 * Add PRISM candidate storm characterization instrument (#192) * Probe each ancestor height once when selecting a payout transition (#193) _accepted_block_payout_transition_for_parent falls through to an ancestor scan when a parent has no exact transition, and that scan issued one getblockhash per preview barrier with no per-height dedupe. Every offered or replay-adopted block candidate arms a preview barrier, and siblings of one solved block all share a height, so the loop asks the same question once per sibling. The scan runs on the path every child job build takes before it can wait for a preview, and it runs twice per built bundle. The 2026-08-20 testnet4 load test reached 3,120 barriers at a single height: 3,120 identical RPCs for one answer, twice per build, decaying one-for-one as the per-row abandonments drained. Memoize the probe by height within the scan. Deliberately not across scans: the active block at a height changes under reorgs, and a longer-lived cache would answer with a chain that no longer exists. Within one scan a single view is what the comparison already assumes. Behaviour is otherwise unchanged -- same probes, same comparison, same fail-closed path when an RPC raises. Two tests, and they do different jobs. The dedupe test fails without this change (60 barriers across 2 heights probe 60 times rather than 2). The scoping test passes either way and is not cover for this change: it guards against a future memo that outlives the scan, which is the wrong fix for the same problem. Found by the #181 design spike; the mechanism is recorded in that issue's spike-result comment. Co-authored-by: Claude Opus 5 (1M context) * Type accepted-parent preview backpressure as coordination-blocked (#182) (#190) The bounded wait for an accepted parent's payout preview raised a plain TemplateRefreshBlocked when it timed out. That type also wraps genuine failures, so a parent whose preview was merely late armed the ordinary template-refresh failure budget and counted against job_build_failure_count -- benign backpressure taking the budgeted restart path. Add AcceptedParentPayoutPreviewPending, following the existing TemplateRefreshBlocked subclass precedent. It subclasses PayoutStatePublicationBlocked, so it lands in the coordination-blocked family already understood by the refresh poller: the retry coalesces with the scheduled tip refresh, it never arms the ordinary failure budget, and sustained pendency still exits through the coordination-blocked streak and its publication-progress watchdog rather than retrying silently forever. Callers catching either parent are unaffected. The typed raise covers exactly one condition: the bounded wait expiring on an armed, un-withdrawn transition. Withdrawn previews, unresolved-depth caps, owed replay enumeration, and RPC/build/trust failures keep their existing types and their budgeted path. Route it through the three job_delivery TemplateRefreshBlocked boundaries: the initial-job loop retries it on its bounded backoff, the pre-build reorg guard propagates it unrelabelled, and the client build boundary schedules the coalescing retry and surfaces the typed exception instead of the generic JobBuildFailed wrapper. Each carries an explicit retry reason on a rate-limited log, separate from the build-failure log. The existing preview-wait timeout counter stays meaningful and rides on the exception. Tests cover the exact typed raise and its parent compatibility, the four sibling conditions that must keep their existing types, all three delivery boundaries, exclusion from job_build_failure_count and the refresh budget, the watchdog exit for a sustained streak, and negative controls proving a plain TemplateRefreshBlocked and an ordinary build failure still use the generic accounting. Co-authored-by: Claude Opus 5 * Add per-row oracle for candidate storms (#194) * Generalise the candidate-storm instrument to a decided height Extend the reusable PRISM candidate-storm rig so a maintenance selector can be evaluated against the real storm population instead of a hand-built corpus, without the instrument carrying a selector of its own: - decide_height() makes one storm candidate the active block at the storm height through a deterministic DecidedHeightRpc (best tip, getblockhash, active header, inconclusive further submits), with per-method call counting so drain cost is measurable. - ownership() reads every seeded candidate's ownership and node-offer evidence facts from the shipped coordinator and ledger state (queue/replay membership, outstanding, replay-inflight, preview barrier, disposition flight, retry holders, retained acceptance, tip observation, accounted, terminal outcome, pool-block row, outbox state and attempt count). The snapshot gains an offer_evidence_marked gauge that excludes the two process-ownership markers by design. - drain_per_row() drives the shipped per-row disposition path the way the submitter loop does (dequeue, then outbox recovery enumeration) and reports rounds, enumerations, terminal row counts, RPC calls by method, and ledger attempt marks. Pinned in discovery for a 100-candidate decided storm: no sibling carries offer evidence while all are outstanding (live) or replay-inflight (restart); six distinct evidence kinds each register exactly once; the per-row drain abandons 99 siblings in both views at 99 submitblock plus 297 getbestblockhash plus 297 getblockheader calls and 99 attempt marks (three recovery enumerations live, none after restart); and one accepted-parent transition lookup issues one getblockhash per adopted barrier. * Drive the candidate-storm oracle through deferred accounting * Fix candidate storm oracle reporting Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01TcrV4a2Z2k8ERM1qHtyX2G * Clarify candidate selector evidence * Exclude the disposition lease from attested node-offer evidence `submit_next_block_candidate` claims the disposition lease on the dequeued hash *before* it consults the terminal outcome, before the accounted / capacity close, and before `_reserve_block_fast_lane_slot`. Every one of those paths releases the lease again without reaching submitblock, and the deferred split hands the still-held lease to accounting even when the node submission is `attempted=False`. The registry the rig reads is weaker still: `_claim_block_candidate_disposition` registers a user on the flight before it takes the flight's lock, so the key is present for a blocked waiter as well as for the holder. `disposition_held=True` therefore does not attest that qbitd was offered the hash -- it says only "some pass owns this row's eventual disposition", which is the same category as `outstanding` and `replay_inflight`. Folding it into `node_offer_evidence` credited the rig with node calls that never happened, and #183's safety comparison is published against exactly that split. Move it to the conservative widening, where a held lease belongs for the same reason a parked retry wakeup does: it may equally span an offer already in flight, and a row whose lease a selector cannot claim is not one it could dispose of anyway. `selector_evidence` is unchanged as a set -- verified identical across all 288 field combinations -- so the production must-not-abandon contract is untouched; only the published split moves. - `node_offer_evidence` drops `disposition_held`. - New `lease_held_without_offer`, symmetric with `retry_held_without_offer`, plus a `unoffered_lease_marked` snapshot gauge so the split still accounts for the whole union. - New `test_lease_retention_alone_is_not_node_offer_evidence` proves `perturb("lease_held")` performs no RPC, leaves no fact downstream of an offer, leaves the durable row pending and unattempted, and still lands in `selector_evidence` -- and that releasing the lease drops it out again. Test-only; no production code changed. * Exclude terminal outcomes from attested evidence; restore withheld wakeups Two review findings against the candidate-storm instrument, both test-only. 1. A terminal outcome is not proof of a node offer. `submit_next_block_candidate` builds a `_BlockCandidateNodeSubmission( attempted=False)` and hands it straight to the accounting queue whenever pool capacity is already closed -- `accepted_block_count` has reached `max_blocks`, or `stop_after_block` is set and one block landed -- and accounting then terminalizes the durable outbox row without calling qbitd once. The already-accounted and finalize-only replay paths do the same. Folding the terminal registry into `node_offer_evidence` therefore credited the rig with node calls that never happened, and #183's safety comparison is published against exactly that split. `attempt_count` is no substitute either, and this was checked rather than assumed: `submit_writer` calls `_mark_block_candidate_attempted` *before* the offer branch, so the durable row records an attempt on that same zero-RPC path. The new regression pins `attempt_count == 1` alongside zero submitblock to keep that trap documented. Excluding the terminal registry costs the attested gauge no attestation it legitimately had: finalization adds an accepted hash to `_accounted_accepted_block_hashes` and persists its pool-block row before the terminal outcome is recorded, so a candidate that really was offered and accepted still attests it in its own right. `selector_evidence` is unchanged as a set -- verified identical across all 1,152 field combinations -- because the terminal outcome moves from one disjunct of the union to another. The production must-not-abandon contract is untouched; only the published split moves. `node_offer_evidence` differs on exactly the 32 combinations where a terminal outcome stands alone. - `node_offer_evidence` drops `terminal_outcome`; `selector_evidence` gains it explicitly as a third widening. - New `terminal_without_offer`, symmetric with `retry_held_without_offer` and `lease_held_without_offer`, plus a `unoffered_terminal_marked` snapshot gauge so the split still accounts for the whole union. - New `test_capacity_closed_terminalization_is_not_node_offer_evidence` closes capacity through the shipped `max_blocks` tunable, drives one row of the shipped deferred split, and proves from the rig's own RPC call log that the row was terminalized with no submitblock at all -- then reopens capacity and shows the drain's exact-set oracle still partitions the sibling set. 2. `drain_per_row` must not eat the wakeups it withholds. `_withhold_head` discarded the wakeup it lifted out of a lane. For a replay-lane candidate that is fatal: replay adoption keeps the hash in `_block_replay_inflight_hashes` until a terminal outcome is recorded, and `_enqueue_replayed_block_candidate` drops any re-adoption of an in-flight hash as a duplicate, so the row was left pending with no wakeup and no enumeration able to rebuild one. No later unprotected drain could ever replay-adopt it. The same discard hit lease-blocked and deferred rows, and a live-lane row that a mid-drain enumeration had already re-adopted. Withholding is now genuinely scoped to the invocation: every lifted wakeup is recorded with its lane and put back in a `finally`, so exhaustion, `max_rounds`, `stop_before_hash` and an exception all restore it. Queue lanes are restored by pushing onto the front of the backing deque -- the wakeup was the head when lifted and everything ahead of it has since been consumed, so that is its exact position, and because `get_nowait` never touched `unfinished_tasks`, putting it back in place leaves that counter matching the item's original `put`. Restoring newest-lift-first keeps the original lane order and prefers the object the shipped markers now describe; a superseded same-hash object releases its credit floor holder through the shipped `_release_dropped_duplicate_candidate_floor`, exactly as production does with a duplicate it drops. - New `test_preserved_wakeups_survive_the_drain_that_withheld_them` runs both the live and restart views through repeated bounded drains and an unbounded one, all preserving the same hash, checks the row keeps exactly one wakeup with its durable row and evidence untouched, and then proves a later unprotected drain reaches it -- one offer, one accounting task, one durable attempt -- with every call partitioning the sibling set. Verified: all 14 candidate-storm tests pass, both new regressions fail against the pre-fix rig (the second in both views), the neighbouring `tests.test_prism_block_candidates` suite is green at 190 tests, and both CLI report shapes still render. Test-only; no production code changed. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01RbP8jydZKpenfLutEuP6tv * Count only actual node offers toward the drain's max_rounds budget drain_per_row documented rounds/max_rounds as node offers but incremented them on submit_next_block_candidate(defer_accounting=True)'s boolean. That boolean is also True for paths that never reach qbitd: a disposition lease held elsewhere parks the wakeup in _block_disposition_waiting_retries, a same-hash duplicate whose disposition already landed is dropped, a declined fast-lane reservation retains the candidate, and the capacity-closed split hands accounting an attempted=False submission that terminalizes the durable row with no submitblock at all. Counting those made the report disagree with its own rpc_calls/accounting_tasks/ledger_attempt_marks and could end a bounded drain before it performed a single promised offer. The rounds gauge now advances by the rig's authoritative submitblock delta, measured across each submit_next (which waits for its own bounded RPC worker, so the delta is complete). Boundedness for parked, terminally skipped and zero-RPC rows is unchanged: it comes from the existing withholding and stall logic, not from the budget. The RPC counter is now required rather than silently defaulted, so a missing counter fails loudly instead of leaving max_rounds unbounded. Tests: a held-lease head under max_rounds=1 now proves the parked row costs no round and the drain still performs one real offer on another row, with every cost field agreeing with the terminal sets; the capacity-closed zero-RPC regression now expects rounds == 0 and steps its single row by withholding the rest, keeping its RPC, accounting and exact-set proof intact. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01B43oqha7d4DccqyShjFVru --------- Co-authored-by: Alex Co-authored-by: Claude Opus 5 * Add a fenced bulk block-candidate abandonment to the share ledger (#195) Superseded siblings are abandoned one durable row at a time today: a statement per hash, each carrying its own writer-lease fence. The 2026-08-20 testnet4 capture reached 3,120 pending outbox rows behind a single decided height, which is where that shape becomes the cost rather than the work. Add mark_block_candidates_abandoned(block_hashes=..., error=...) to both backends. It transitions exactly the rows whose current state is 'pending' and writes the same terminal column set the single-hash path does -- last_error, updated_at, completed_at, candidate = NULL. In Postgres that is one statement for a whole page, fenced on the same writer id / epoch / session token, with block_hash = ANY(...) as the row predicate instead of N single-row calls. The memory ledger decides a page under one lock acquisition, so both backends resolve a page as one atomic decision, and an inactive writer lease fails exactly as the existing terminal operations do. The return value is the set of hashes the write actually transitioned, fed by RETURNING rather than a count. That distinction is the point: rows that were already terminal, that do not exist, or that another writer won are absent from it, so a caller cleans up only what this fenced write won. Input is normalized once for both backends -- lowercased, de-duplicated, sorted -- so a page naming one row twice in two cases asks for it once and the generated statement is a function of the set rather than of the caller's iteration order. An empty page performs no query at all: ANY(ARRAY[]) has no inferable element type and would not parse. A candidate that reached persist_accepted_block owns a qbit_pool_blocks row, and such a candidate was offered and landed -- it is not a superseded sibling. Two changes keep that fact both cheap and authoritative: - pending_block_candidate_rows now carries pool_block_exists per row, from an EXISTS probe placed outside the LIMITed subquery so it runs at most once per returned row rather than once per pending row behind it. Asking per hash would be one round trip per row on precisely the path that exists for large backlogs. A page whose rows do not carry the field raises rather than reading a missing key as false, so a failed or truncated read cannot present itself as "no pool block". - The bulk UPDATE re-asks the same question under the writer fence with NOT EXISTS. A candidate can land between the page read and the write, so the veto belongs in the statement that performs the transition; a row that acquired a pool block is simply absent from the returned set. No schema change. qbit_block_candidate_outbox already admits the terminal row shape, and block_hash is its primary key, so the set predicate is an existing index scan. This commit adds the primitive only -- no selector, no chain probes, no in-memory cleanup. Those consume it separately. Mechanism recorded in the #183 spike-result comment. Co-authored-by: Claude Opus 5 * Collapse superseded block candidates before replay adoption (#196) * Generalise the candidate-storm instrument to a decided height Extend the reusable PRISM candidate-storm rig so a maintenance selector can be evaluated against the real storm population instead of a hand-built corpus, without the instrument carrying a selector of its own: - decide_height() makes one storm candidate the active block at the storm height through a deterministic DecidedHeightRpc (best tip, getblockhash, active header, inconclusive further submits), with per-method call counting so drain cost is measurable. - ownership() reads every seeded candidate's ownership and node-offer evidence facts from the shipped coordinator and ledger state (queue/replay membership, outstanding, replay-inflight, preview barrier, disposition flight, retry holders, retained acceptance, tip observation, accounted, terminal outcome, pool-block row, outbox state and attempt count). The snapshot gains an offer_evidence_marked gauge that excludes the two process-ownership markers by design. - drain_per_row() drives the shipped per-row disposition path the way the submitter loop does (dequeue, then outbox recovery enumeration) and reports rounds, enumerations, terminal row counts, RPC calls by method, and ledger attempt marks. Pinned in discovery for a 100-candidate decided storm: no sibling carries offer evidence while all are outstanding (live) or replay-inflight (restart); six distinct evidence kinds each register exactly once; the per-row drain abandons 99 siblings in both views at 99 submitblock plus 297 getbestblockhash plus 297 getblockheader calls and 99 attempt marks (three recovery enumerations live, none after restart); and one accepted-parent transition lookup issues one getblockhash per adopted barrier. * Drive the candidate-storm oracle through deferred accounting * Add fenced bulk candidate abandonment to the share ledger * Collapse decided block candidates before replay adoption Select only safe decided-height siblings, revalidate them under nonblocking disposition leases, abandon them through one fenced batch write, and clean up exactly the returned set before replay adoption. Add bounded metrics and oracle-backed coverage for both pagination paths, races, failure modes, and storm cardinalities. * Count candidate collapse write races * Fix candidate storm oracle reporting Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01TcrV4a2Z2k8ERM1qHtyX2G * Harden collapse timeout and cleanup accounting * Clarify candidate selector evidence * Exclude the disposition lease from attested node-offer evidence `submit_next_block_candidate` claims the disposition lease on the dequeued hash *before* it consults the terminal outcome, before the accounted / capacity close, and before `_reserve_block_fast_lane_slot`. Every one of those paths releases the lease again without reaching submitblock, and the deferred split hands the still-held lease to accounting even when the node submission is `attempted=False`. The registry the rig reads is weaker still: `_claim_block_candidate_disposition` registers a user on the flight before it takes the flight's lock, so the key is present for a blocked waiter as well as for the holder. `disposition_held=True` therefore does not attest that qbitd was offered the hash -- it says only "some pass owns this row's eventual disposition", which is the same category as `outstanding` and `replay_inflight`. Folding it into `node_offer_evidence` credited the rig with node calls that never happened, and #183's safety comparison is published against exactly that split. Move it to the conservative widening, where a held lease belongs for the same reason a parked retry wakeup does: it may equally span an offer already in flight, and a row whose lease a selector cannot claim is not one it could dispose of anyway. `selector_evidence` is unchanged as a set -- verified identical across all 288 field combinations -- so the production must-not-abandon contract is untouched; only the published split moves. - `node_offer_evidence` drops `disposition_held`. - New `lease_held_without_offer`, symmetric with `retry_held_without_offer`, plus a `unoffered_lease_marked` snapshot gauge so the split still accounts for the whole union. - New `test_lease_retention_alone_is_not_node_offer_evidence` proves `perturb("lease_held")` performs no RPC, leaves no fact downstream of an offer, leaves the durable row pending and unattempted, and still lands in `selector_evidence` -- and that releasing the lease drops it out again. Test-only; no production code changed. * Retry partial collapse cleanup for terminal candidates After the fenced batch abandonment write, a won row's durable outbox entry is terminal: the collapse partitions it out of the replay page and no later enumeration can hand it back. A cleanup step that failed was only counted, so whatever it did not tear down -- a payout preview or its tombstone, a pending-share floor holder, an outstanding-hash marker, the terminal-outcome record -- stayed installed for the process lifetime with no replay source to retry it. Keep the still-owed steps per terminal hash in a cleanup-retry registry and drain it from the accounting lane's idle branch, beside the quarantine drain. The cleanup is split into a step runner that runs only the steps a caller lists and reports what each hash still owes, so a retry replays exactly the remainder and never repeats a completed step. The registry carries the resolved pending-share floor holders because the floor keys them by object identity and the queues they were indexed from drain before a retry runs; an aborted apply, which never indexed them, defers every step and re-runs the scan instead. The pass is bounded by the affected terminal hashes: one hash per call, only its owed steps, and only past its own doubled-and-capped backoff, which mirrors next_retry_delay. It touches no durable row and enqueues no candidate, so it can only finish tearing state down -- a terminal row is never re-adopted or re-offered. The exact-returned-set partition is unchanged, and so is cleanup_failed: it still counts affected hashes once per apply. Two fixed labels join the closed outcome set, cleanup_recovered (per recovered hash) and cleanup_retry_failed (per retry pass that still left work), so a stuck cleanup is distinguishable from a recovered one without any unbounded label. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01WCYRPZUDoZeV9cY8zXduxA * Exclude terminal outcomes from attested evidence; restore withheld wakeups Two review findings against the candidate-storm instrument, both test-only. 1. A terminal outcome is not proof of a node offer. `submit_next_block_candidate` builds a `_BlockCandidateNodeSubmission( attempted=False)` and hands it straight to the accounting queue whenever pool capacity is already closed -- `accepted_block_count` has reached `max_blocks`, or `stop_after_block` is set and one block landed -- and accounting then terminalizes the durable outbox row without calling qbitd once. The already-accounted and finalize-only replay paths do the same. Folding the terminal registry into `node_offer_evidence` therefore credited the rig with node calls that never happened, and #183's safety comparison is published against exactly that split. `attempt_count` is no substitute either, and this was checked rather than assumed: `submit_writer` calls `_mark_block_candidate_attempted` *before* the offer branch, so the durable row records an attempt on that same zero-RPC path. The new regression pins `attempt_count == 1` alongside zero submitblock to keep that trap documented. Excluding the terminal registry costs the attested gauge no attestation it legitimately had: finalization adds an accepted hash to `_accounted_accepted_block_hashes` and persists its pool-block row before the terminal outcome is recorded, so a candidate that really was offered and accepted still attests it in its own right. `selector_evidence` is unchanged as a set -- verified identical across all 1,152 field combinations -- because the terminal outcome moves from one disjunct of the union to another. The production must-not-abandon contract is untouched; only the published split moves. `node_offer_evidence` differs on exactly the 32 combinations where a terminal outcome stands alone. - `node_offer_evidence` drops `terminal_outcome`; `selector_evidence` gains it explicitly as a third widening. - New `terminal_without_offer`, symmetric with `retry_held_without_offer` and `lease_held_without_offer`, plus a `unoffered_terminal_marked` snapshot gauge so the split still accounts for the whole union. - New `test_capacity_closed_terminalization_is_not_node_offer_evidence` closes capacity through the shipped `max_blocks` tunable, drives one row of the shipped deferred split, and proves from the rig's own RPC call log that the row was terminalized with no submitblock at all -- then reopens capacity and shows the drain's exact-set oracle still partitions the sibling set. 2. `drain_per_row` must not eat the wakeups it withholds. `_withhold_head` discarded the wakeup it lifted out of a lane. For a replay-lane candidate that is fatal: replay adoption keeps the hash in `_block_replay_inflight_hashes` until a terminal outcome is recorded, and `_enqueue_replayed_block_candidate` drops any re-adoption of an in-flight hash as a duplicate, so the row was left pending with no wakeup and no enumeration able to rebuild one. No later unprotected drain could ever replay-adopt it. The same discard hit lease-blocked and deferred rows, and a live-lane row that a mid-drain enumeration had already re-adopted. Withholding is now genuinely scoped to the invocation: every lifted wakeup is recorded with its lane and put back in a `finally`, so exhaustion, `max_rounds`, `stop_before_hash` and an exception all restore it. Queue lanes are restored by pushing onto the front of the backing deque -- the wakeup was the head when lifted and everything ahead of it has since been consumed, so that is its exact position, and because `get_nowait` never touched `unfinished_tasks`, putting it back in place leaves that counter matching the item's original `put`. Restoring newest-lift-first keeps the original lane order and prefers the object the shipped markers now describe; a superseded same-hash object releases its credit floor holder through the shipped `_release_dropped_duplicate_candidate_floor`, exactly as production does with a duplicate it drops. - New `test_preserved_wakeups_survive_the_drain_that_withheld_them` runs both the live and restart views through repeated bounded drains and an unbounded one, all preserving the same hash, checks the row keeps exactly one wakeup with its durable row and evidence untouched, and then proves a later unprotected drain reaches it -- one offer, one accounting task, one durable attempt -- with every call partitioning the sibling set. Verified: all 14 candidate-storm tests pass, both new regressions fail against the pre-fix rig (the second in both views), the neighbouring `tests.test_prism_block_candidates` suite is green at 190 tests, and both CLI report shapes still render. Test-only; no production code changed. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01RbP8jydZKpenfLutEuP6tv * Fence collapsed candidates from node submission at the write The fenced batch write is the moment a block candidate row becomes durably terminal, but the in-memory fence the rest of the submitter honours -- the recorded terminal outcome -- was installed only by the `terminal-outcome` cleanup step that runs afterwards. That step can fail on its own, and a page-level abort can stop the cleanup pass before ever reaching it. Either way the hash lands in the cleanup-retry registry while a same-hash candidate may still be sitting in the live, replay, retry, or waiting lane. `submit_next` did not consult that registry, so once the apply released its disposition lease the candidate was dequeued, found no terminal outcome, and offered qbitd a block whose durable row was already abandoned -- before the accounting lane got anywhere near the retry. Publish the false terminal outcome for the exact returned hash set as soon as the write wins, while every won hash's disposition lease is still held and before any cleanup runs. No same-hash lane can be inside its own guarded region while that happens, and none can enter one afterwards without seeing the fence, so the invariant holds across the apply, the lease release, the queue dequeue, direct and inlined accounting, and every retry outcome. The entry is never pruned, so a cleanup that keeps failing cannot bring the fence back down, and a cleanup that recovers simply leaves the ordinary terminal fence behind. This reuses the fence the submitter already honours rather than adding a parallel one: the publication is a lock-guarded assignment into disposition state that already exists, with no durable read or write and no queue admission, so it can neither create a durable row nor re-adopt or re-offer one. The full `terminal-outcome` step still runs in cleanup for its other side effects -- the fast-lane reservation, the replay-inflight marker, and the parked same-hash retry's floor holder -- and stays owed, and retried, whenever it fails. The bounded per-hash retry/backoff, the cleanup accounting, the fixed metric labels, and the exact returned-set partition are all unchanged. Regressions cover a failed `terminal-outcome` step and a page-level abort before it, each with a same-hash candidate still queued and the shipped submit path driven before the cleanup retry; the live, replay, and waiting-retry lanes; the direct writer and synchronous seams; and the fence surviving repeated retry failures and its discharge after recovery. With the publication disabled, seven of the nine fail and the primary one records a real submitblock for a durably terminal row. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01AhKZG4TGvaW3PpQQi9SauE * Count only actual node offers toward the drain's max_rounds budget drain_per_row documented rounds/max_rounds as node offers but incremented them on submit_next_block_candidate(defer_accounting=True)'s boolean. That boolean is also True for paths that never reach qbitd: a disposition lease held elsewhere parks the wakeup in _block_disposition_waiting_retries, a same-hash duplicate whose disposition already landed is dropped, a declined fast-lane reservation retains the candidate, and the capacity-closed split hands accounting an attempted=False submission that terminalizes the durable row with no submitblock at all. Counting those made the report disagree with its own rpc_calls/accounting_tasks/ledger_attempt_marks and could end a bounded drain before it performed a single promised offer. The rounds gauge now advances by the rig's authoritative submitblock delta, measured across each submit_next (which waits for its own bounded RPC worker, so the delta is complete). Boundedness for parked, terminally skipped and zero-RPC rows is unchanged: it comes from the existing withholding and stall logic, not from the budget. The RPC counter is now required rather than silently defaulted, so a missing counter fails loudly instead of leaving max_rounds unbounded. Tests: a held-lease head under max_rounds=1 now proves the parked row costs no round and the drain still performs one real offer on another row, with every cost field agreeing with the terminal sets; the capacity-closed zero-RPC regression now expects rounds == 0 and steps its single row by withholding the rest, keeping its RPC, accounting and exact-set proof intact. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01B43oqha7d4DccqyShjFVru * Compare candidate collapse work in one difficulty unit The decided-height collapse selector compared two numbers a million apart. A candidate's durable found_block.network_difficulty is stamped with template_artifacts.scaled_network_difficulty(template["bits"]) -- the qbit powLimit target times QBIT_DIFFICULTY_SCALE (1e6) over the template target -- while _BlockCandidateChainView.difficulty_of returned the occupying block's raw getblockheader.difficulty float. Clause 4b/5 read an equal-work active sibling as a millionth of the candidate's work, took it for a lower-work occupant that is "not a decision", and preserved every row: a decided height never collapsed and the storm the selector exists for kept draining one node offer, one lease and one durable write per row. difficulty_of now re-derives the occupant's work from the header's compact bits with the same integer formula the coordinator used to stamp the candidate, so both sides of the comparison are the identical scaled integer for identical work. The formula is restated here from direct_stratum rather than imported from template_artifacts or public_api: both sit above this module in the import graph, and B1 must not grow an edge to the template/payout layer for two constants. A test pins the restatement equal to the production helper. Bits are also the only work fact read. There is no raw-difficulty fallback, so a header missing bits, or carrying a malformed or zero-target one, fails its page closed exactly as an unusable difficulty did before, in selection and in the pre-write revalidation alike. Keeping the value integral matters at the equality boundary the clause turns on: above 2**53 widening the stored integer to float rounds it, often upwards, so _collapse_difficulty now preserves an integral spelling as an int instead of forcing every stored value through binary floating point. The collapse fixtures state work as compact bits and report the raw difficulty float alongside it, so a selector that read the float again fails the equal-work tests. Coverage added for the realistic scaled-candidate/equal-work-occupant collapse at three difficulties, a candidate one scaled unit heavier being preserved, an equal-work changed occupant surviving revalidation, a changed occupant without bits failing the pre-write read closed, and the malformed/missing bits matrix. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01TL1VtRUhjurX2AiBpsERsX * Interleave collapse cleanup retries with accounting lane work The accounting lane drove _run_one_collapsed_block_candidate_cleanup_retry only from its idle branch, and only after the invalid-candidate quarantine drain reported nothing. A collapsed row whose post-write cleanup failed has no durable replay source left, so that made the retry hostage to the lane going idle: sustained ordinary accounting traffic, or a continuously replenished quarantine queue, starved a due terminal cleanup for as long as the traffic lasted. The loop now also offers one bounded retry after at most DEFAULT_BLOCK_ACCOUNTING_CLEANUP_RETRY_WORK_ITEMS completed work items, where a work item is one finished accounting task from either handoff queue or one finished quarantine item. The cadence resets on every offer -- even one that finds nothing due -- so a cleanup backlog spends at most one attempt per cadence and cannot starve accounting or quarantine in return. The truly idle path still runs the retry immediately. Every existing bound is untouched: one hash per call, only that hash's still-owed steps, the per-record not_before_monotonic deadline, round-robin registry order, and the fixed-label collapse metrics. The cadence offers are deliberately uncontained and outside the accounting task's try/finally -- the retry pops its record before attempting it and only its own failure paths re-register it, so an outer catch there could swallow the last reference to a terminal hash's owed steps. Tests cover both starvation directions within the documented bound, the reverse starvation, a not-yet-due record offered but not run early, the unchanged immediate idle path, and a regression proof that neutralising the cadence puts the starvation back. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01K4Nr2qqEzz4w32vYaNK8Ph * Assert real qbitd answers getblockheader.bits in collapse's shape Block-candidate collapse converts an occupying block's getblockheader.bits into PRISM-scaled difficulty and fails its whole page closed when that field is not an eight-character compact hex string with a positive target. The real-qbitd runtime smoke stood up a regtest node and checked its lifecycle, but never read a header, so a production node that stopped answering in that shape would have shipped green. Read the running regtest container's tip header over the same getbestblockhash/getblockheader pair collapse itself calls, require bits to be exactly eight hexadecimal characters, and derive its scaled difficulty by invoking the production helpers -- both block_candidates._collapse_scaled_difficulty and template_artifacts.scaled_network_difficulty -- rather than restating their arithmetic, requiring the two to agree on a positive value. Both import closures are standard-library only, so the smoke job needs nothing beyond the checked out tree and a python3, which it now sets up explicitly. The derived value is never fed to shell arithmetic: mainnet-scale difficulties exceed 2**63 and would wrap negative in bash's signed integers, so positivity is decided in the helper and re-checked here by digit shape. tests/test_qbit_image_runtime_smoke.py drives the whole script against a deterministic stand-in for the docker CLI, covering the passing case, short/long/non-hex/absent/non-string bits, a zero-target compact, a mainnet-scale value past 2**63, and the pre-existing lifecycle checks, so neither the assertion nor the surrounding smoke can disappear silently. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_016XFfk8c2MuBZPSAuKJZPxg * Retire finished block submitter ledger calls A call whose coordinator-side waiter timed out stayed in _block_submitter_ledger_calls forever once its bounded worker finished: only a later invocation of the exact same key popped it, and that key may never recur -- ("collapse-superseded", hashes) is keyed by the very page of block hashes the late write itself removes from the outbox. Every such timeout leaked a completed call object and its hash tuple for the life of the process. The registry now holds in-flight calls and nothing else. The worker identity-removes its own entry as soon as it publishes done, and the waiter path removes through the same seam, so the registry returns to empty whether or not anybody ever comes back for that key. Only a finished call is ever removed and only under its own identity, so a same-key caller arriving while the worker is still out still joins that one call, and a fresh call that has taken the key is never dropped by a stale one. Waiters parked on the call hold it directly and still receive its result or error; a caller arriving after the removal replays the idempotent operation on a fresh call. The slot goes back before the removal so that caller is never told the bounded pool is exhausted. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01Ct5ExHoqUtkEhhafSdBkao * Answer collapse evidence by probe and bound terminal outcomes _block_candidate_collapse_evidence built a set from every registry it consults, including _block_candidate_terminal_outcomes -- process-lifetime disposition history that reached 312,000 entries in one testnet4 storm. Each decided-height collapse poll copied all of it under coordinator.lock (~124 ms at that size, linear in history) to answer one membership question per page row, with every unrelated share ack queued behind the same lock. The selector now passes the hashes it is actually deciding -- the durable page during selection, the leased subset during revalidation -- and the evidence read probes each registry for those hashes instead of walking any of them. Page cost is O(page * registries) and no longer moves with history. Every evidence source is unchanged: retry holders, waiting/finalize/retained registries, tip-observed and accounted acceptances, terminal outcomes, disposition flights, and ignore_leases all still speak for their hash. The terminal-outcome registry is separately bounded at 8,192 entries with oldest-first eviction. A forgotten outcome reads exactly like one that never happened, so eviction pins every lane that can still hold a same-hash copy: the outstanding, replay-inflight, quarantine, fast-lane, waiting-retry, finalize-retry, retained-submission and collapse-cleanup registries, the single-slot retry holders, the candidate objects still sitting in the live and replay queues, and the hashes with a claimed disposition flight. The queues matter because a won row's cleanup clears the ownership markers for the hash it terminalized, leaving a queued duplicate named by no registry -- exactly the escape the collapse terminal fence exists to close. Queue and flight locks are only ever tried, never waited on under the global lock; a lane that cannot be read skips the pass rather than evicting against an incomplete pin set, and a bulk fence never evicts its own publication. Tests: a registry that counts probes and refuses to be walked pins the page-bounded shape deterministically (identical probe counts at 1,000 and 200,000 entries, one probe per decided hash, zero walks), plus bound, FIFO, pin-per-lane, no-stall, skip-rather-than-wait, oversized-fence, and an end-to-end check that a queued duplicate stays fenced under eviction pressure while an unheld hash ages out. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_011nkJJjw7DUC9EVj9bjhn8q * Pin a dequeued candidate's hash across the disposition claim The terminal-outcome registry is bounded, and eviction skips every hash some in-memory copy still needs: the lane registries, the single-slot retry holders, the candidate objects queued on the live and replay lanes, and the hashes with a claimed disposition flight. submit_next left one instant uncovered. It pulls a candidate out of the retry holder, a queue, or the waiting registry -- clearing that lane -- and only then calls _claim_block_candidate_disposition, which is what installs the flight. Between the two the candidate object is the method's local variable and nothing names its hash. A terminal duplicate's fence is exactly the kind of old entry at the front of the FIFO, so an eviction in that window drops it, the pass reads no outcome, and it offers a durably terminal block to the node -- the escape the fence exists to close, reopened by the bound meant to keep it cheap. The dequeue now transitions the hash into a dequeued-hash pin in the same coordinator.lock critical section that empties the lane. For the queues the non-blocking get itself moves under that hold, taking the queue mutex beneath coordinator.lock -- the order the eviction's queue read already establishes -- so the entry leaves the queue and enters the pin with no gap. The pin stays while the claim runs and is handed on under coordinator.lock again: to the installed flight on a lease, to the waiting registry on a miss (same critical section as the insert), or to nothing when the claim raises and the candidate object leaves with the exception. The eviction reads the pin as one more lane. It is a count per hash rather than a set so two dequeued same-hash objects cannot unpin each other, and it carries no TTL: the pin ends only where the hash is named elsewhere. The two ordinary retry paths that released the lease before retaining the candidate now retain first, as the exception paths already did, so the retry holder (and the outstanding registry retain_for_retry writes) names the hash before the flight stops doing so. The accounting lane had the same instant in its unexpected-failure path. _run_block_accounting_task releases the task's lease in its finally and block_accounting_loop retained the candidate only afterwards, in its own catch, leaving the hash unnamed in between. The task method now retains in its own exception path while it still holds the disposition -- an accounting-owner retain lands in the deferred holder -- and the finally releases the lease and merges that holder into the retry slot under coordinator.lock, as it always did for a writer-side retain. The task carries a marker the lane sets once it has retained, so the loop's catch does not retain (and count) the candidate twice, yet still retains it when a failure fired before the lane could -- a delegate raising ahead of it. The page-bounded evidence read is unchanged. Tests: a deterministic barrier parks a pass inside the old gap for each of the four dequeue sources, applies real eviction pressure from the other side, and proves the fence survives, the pin names the hash while it is held, the pass drops the duplicate without a node offer, and no pin outlives it; the lane-by-lane pin test gains the dequeued pin; a claim that misses, raises, or finds an un-due retry leaves no pin behind; a second barrier parks the accounting lane the instant its lease release returns and proves the fence survives eviction there too, with the candidate retried exactly once and its floor holder intact; and a failure ahead of the lane is still retained, once. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_012udMjxtAxSfijazaxRLQTn * Retire a counted abandonment with the fence that proved it The abandonment dedup set kept one key for every candidate the process had ever abandoned. It exists because a terminal disposition can run more than once for one hash -- a collapse whose cleanup failed retries every owed step, and a finalize failure freezes a false finalize-only disposition that replays -- and abandonment metrics count candidates, not attempts. Nothing retired the keys, so a collapsed storm added 3,120 of them per page and the set grew for the life of the process: the same unbounded history the terminal-outcome registry was already bounded away from. Retire the key inside the eviction pass, in the same coordinator-lock critical section that has already proved the hash unpinned by every live and cleanup-owing lane, and only for a hash whose fence that pass actually drops. No independent cap and no second sweep: a key may only go when no in-memory copy is left to re-enter the accounting path, and that is exactly the proof the eviction already performs. Both writer orderings are covered by the one rule. The direct finalize counts before it publishes its terminal outcome, so its key is retired by a later pass over the outcome it goes on to publish; an ambiguous false finalize failure has no outcome to evict at all while finalize_retries pins it. The collapse publishes the outcome first and counts during cleanup, holding the hash's disposition lease -- an in-flight pin -- across both. Tests: repeated storms leave the dedup set tracking the fences rather than the history, with every candidate still counted once; the shipped collapse apply retires its own page when the fences age out; the two orderings converge on identical state; a repeated disposition is still counted once; a false finalize failure stays counted with no fence to retire it by; and eviction pressure cannot forget a hash pinned by the live queue, the replay queue, the cleanup-retry backlog, finalize retries, a disposition flight, the outstanding registry, or the retry holder and let its owed accounting count the candidate twice. * Bound one collapse walk's chain-height reads (#196) Decided-height collapse reads the chain once per distinct candidate height. For the same-height storm it exists for that is a handful of round trips however wide the page is, but a replay page holds up to MAX_BLOCK_REPLAY_ENUMERATION_ROWS rows: a page whose rows sit at 1,024 different heights cost one getblockhash plus one getblockheader each, sequentially, on the sole block-submitter thread. At a production RPC deadline that is minutes to hours in which no live solve reaches submitblock. MAX_BLOCK_CANDIDATE_COLLAPSE_HEIGHT_PROBES caps the distinct heights one enumeration walk reads, so the walk's chain cost is a constant whether the backlog arrives as one page or fifty. A row at an unprobed height is never selected: it stays durable with all of its evidence and reaches the per-row disposition path that disposed of it this way before collapse existed, and a later walk starts with a fresh budget. Nothing is terminalized on an unread height, and the short-page completeness proof is untouched. Between pages -- never inside one page's fenced collapse, write, or adoption -- the walk hands the submitter thread back when its budget is spent and a live candidate or retry is waiting: the pages it would fetch next can no longer collapse anything, so the yield costs no efficacy, and it leaves the enumeration owed rather than claiming an unproven one. The storm never spends the budget, so it still collapses inside a single complete enumeration. The bounded pre-write revalidation is budgeted for exactly the heights selection already probed, so it can never be starved by the walk. Deferred rows are counted on the new fixed-label height_deferred outcome. Tests: a 1,024-row page at 1,024 heights makes exactly the bounded number of probes; deferred rows are retained durably and successive passes converge on all of them; a diverse two-page backlog yields mid-walk and the waiting live solve is offered on the very next submitter turn; the same-height storm walk is unchanged with a live solve waiting throughout. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01NTqbTf1q4RKeitDUpqXbxg * Share one collapse walk's height reads across its pages (#196) The chain-height probe budget is created once per enumeration walk and shared by that walk's pages, but the reads it paid for were not: every page built a fresh _BlockCandidateChainView with its own _active and _difficulty caches, so a backlog sitting at one decided height charged the walk a probe -- and repeated the getblockhash and getblockheader round trips -- on every single page. That defeats the shape collapse exists for. After MAX_BLOCK_CANDIDATE_COLLAPSE_HEIGHT_PROBES full pages the budget is spent, and from there the exhausted-budget early return preserves each remaining page whole and the walk replay-adopts it (or, with a live candidate waiting, the walk yields and never fetches it). A same-height storm therefore collapsed at most 8,192 rows per walk: at the 312,000-row scale this path is sized for, collapse handled under 3% of the backlog and the rest fell back to the per-row path it was built to replace. The caches now live on _CollapseHeightProbeBudget, beside the counter, for the same reason the counter does: a walk that has paid for a height must not pay for it again on its next page. Each page's view reads and writes the walk's dictionaries, so page two answers page one's height for free and the whole storm collapses inside one complete enumeration off a single probe. difficulty is keyed by block hash, whose work is immutable, so sharing it can only remove round trips. The pre-write revalidation is untouched and stays authoritative: it already builds its own _CollapseHeightProbeBudget sized to the leased heights, and a budget of its own is now a cache of its own as well, so every leased height is genuinely re-read from the chain under the held leases before the fenced write. Page-scope reads stay page-scope: each page's selection still opens on a freshly read best tip and tip height. The budget's value, the yield condition, and the deferral accounting are unchanged. Tests: a nine-page storm at one decided height -- one page past the eight-page boundary, with a live solve waiting on the last page -- collapses in full inside one enumeration that neither defers a row nor yields, adopting only the live solve, while the height is read once for selection and once per page for revalidation and the occupant header is read exactly once for the walk. The 3,120-candidate storm's chain-read bound tightens from 2*pages height reads and one header per page to 1+pages and one header per walk. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01TsXBRGe95RgbPQK8LT1EEt * Reuse a collapse walk's cached heights past its probe bound (#196) The distinct-height probe budget and its two caches are shared by every page of one replay walk, but the page pass returned the whole page the moment the counter hit zero. A first page spread across all eight affordable heights therefore disarmed collapse for the remainder of the walk, including later siblings at exactly those heights -- rows whose occupant was already in `probe_budget.active` and could have been decided without one new round trip. They were preserved and replay-adopted instead, which is the same defeat the shared cache was added to prevent, one page later. The spent-budget early return now fires only when no row on the page sits at a height this walk already read. A cached height stays answerable for free however late it is asked, so those pages run selection as usual; rows at heights the walk can no longer afford are still counted height_deferred and preserved, and no new distinct height is ever probed past the bound. The eight-height bound is unchanged, fresh per-page tip reads and the independent pre-write revalidation are untouched. `_collapse_row_height` is the single place that knows where a fetched row records its height, so the cheap pre-selection peek and the decode in `_superseded_candidate_row` cannot drift apart on it. Regression: a nine-height, three-page walk whose first page spends the budget exactly and whose later pages hold cached-height siblings plus a never-probed negative control. Before the fix it collapsed 1,024 rows and replay-adopted 1,031; after it collapses 2,047, defers exactly the 8 control rows, probes 8 distinct heights and reads 8 headers in both cases, and adopts only the deferred controls. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_012Q366teeGhYaPxr6KT4LuK --------- Co-authored-by: Alex Co-authored-by: Claude Opus 5 * Measure acceptance-to-preview-publication latency (#181) Issue #181 item 3. The 2026-08-21 testnet4 run had #190/#191/#196 deployed and demonstrably active -- bulk-collapse failures zero, accepted-parent depth below its cap, the durable queue repeatedly draining -- and the accepted-parent tail still dominated: 25 preview wait timeouts, the first two at only 0.32 PH/s, semantic current-work coverage repeatedly at 0.0, and about 1.26 CPU cores at ~8.5 PH/s. Items 1 and 2 cannot be shown to have worked without a percentile, so this adds the measurement first. The interval. Start: the instant definitive node acceptance becomes known -- submitblock returning None on the submitter thread, discriminated by all three clauses (attempted, error is None, result is None), because the submitblock error path also builds result=None and a two-clause test would stamp a failed offer as an acceptance. Deliberately not AcceptedBlockPayoutTransition.landed_monotonic, which is armed before the RPC and re-armed by every retry. End: the preview becoming visible to waiting child work in _publish_accepted_block_payout_preview. Both of its exits publish and are labelled apart -- "published" for the atomic generation publication, "degraded" for issue #188's fenced local-retention branch, which installs the compact preview and notifies waiters without a generation. Only the first publication of a hash is measured; a republication or a second definitive offer does not restart or duplicate the interval. qbit_prism_accepted_block_preview_publication_seconds is a labelled histogram wired through the same seams as qbit_prism_block_submit_seconds: histogram state and observer on the B1 owner, coordinator delegate, copied snapshot surfaced through block_submitter_snapshot, both state-field registries, and the renderer. Its buckets are a dedicated tuple rather than a widening of the shared job-build one: the bar is a p95 below the 5 s child wait budget, so 5.0 has to be an exact boundary with the approach to it resolved and the range extending past it. The regression is parameterised on accepted-block cadence, not hashrate, as the validation comment requires. It releases one definitively accepted winner per tip at consecutive heights with same-height siblings behind it, and drives the real bounded FIFO candidate queue, the real submitter split, the shipped accounting handoff and spillover, block_accounting_loop on its own thread, and the shipped publication boundary. Node round-trip latency is modelled and reported, because a zero-cost fake makes every dispatch order look identical. Measured on this base: at 40 tips/min p95 is 0.25-0.5 s; at 150 tips/min p95 is 10.0 s with a mean of 4.7 s, against a 5 s budget. The same 150 tips/min sweep with no stale siblings publishes at p95 0.5 s, which locates the cost in dispatch order rather than in the accepted tail's own work. The high-cadence assertion is written against both worlds: without a dedicated definitive-acceptance lane it asserts the tail is genuinely exercised, and it becomes the 5 s bar as soon as that lane exists. DEFAULT_ACCEPTED_BLOCK_PAYOUT_PREVIEW_WAIT_SECONDS is not touched, and neither is #193's active_hash_by_height or #196's collapse machinery. * Give definitive node acceptances their own accounting lane (#181) The 2026-08-21 testnet4 run left an accepted-parent tail behind: 25 payout preview wait timeouts, the first two at only 0.32 PH/s, with the coordinator at ~1.26 CPU cores and Postgres at ~12%. It is an ordering problem, not a saturation one. A task that carries the decided block already knows it is decided, and the accounting lane was serving stale same-height replay ahead of it. The trap this had to get past is that a priority *key* on the primary handoff queue is inert exactly when it matters. `_enqueue_block_accounting_task` spills into an unbounded overflow queue once the primary is full, and from then on every later handoff joins the spill; `block_accounting_loop` drains the bounded primary in full before it looks at the overflow at all. Under a burst the accepted task is therefore *put* behind up to eight stale entries, each costing ~6 RPCs, two ledger writes and a payout-balance mutation. Priority only ever orders one heap. So membership changes rather than the rule: - `_is_definitive_node_acceptance` states the discriminator once -- attempted, no error, no result. The `error is None` clause is load-bearing: `_submit_block_candidate_to_node`'s transport-failure path builds `attempted=True, error=exc`, which leaves `result` at its `None` default, so a two-clause test would read a failed offer as an acceptance. `_stash_retained_block_candidate_node_submission` now calls the same helper it used to spell inline, so the two readings cannot drift. - `_block_accounting_accepted_queue` is a third, unbounded lane keyed `(height, sequence)`. Unbounded for the reason the overflow queue is: the offer has already happened and must never be converted back into a raw-submit retry, and a `queue.Full` here would need a spill target, which is the inversion being removed. Max-block admission bounds the number of unresolved real offers. - The spillover put rule and the primary-then-overflow dequeue discipline are unchanged, and still govern exactly the population they were written for. Their anti-starvation property is now asserted by a test: with the primary drained and room available, a newer non-accepted handoff still joins the spill behind the older entry. - Fairness for the stale lanes is an explicit bound instead of an emergent property. `BLOCK_ACCOUNTING_ACCEPTED_DISPATCH_QUOTA` (4, overridable via `block_accounting_accepted_dispatch_quota`) means a non-accepted task at the head of its lane waits behind at most four accepted services, whatever the accepted arrival rate. Four rather than one because an accepted tip's own tail can carry more than one accepted-evidence task, and interleaving a stale task's RPCs into that tail is the thing being fixed. Starving stale replay would not be a fix at all: it would leave the outbox row pending, the payout-preview barrier armed and the disposition lease held. Regressions cover the issue's own counterexample, the burst case a priority key cannot reach, the quota bound at Q=1 and Q=4 including a real durable row reaching terminal abandonment while delayed, the discriminator's three near-miss shapes, and the unchanged spillover property. The storm rig's lane drain and the collapse suite's lane wait follow the shipped three-lane discipline. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01LcLiM6K5GPGVuBvY2r6riw * Skip provably stale block candidates at dequeue (#181) Issue #181, item 2. The 2026-08-21 testnet4 validation had #196 deployed and demonstrably active, and the replay-adopted candidate population was gone -- yet coordinator logs still carried bursts of individual "block candidate abandoned reason=stale-job: tip moved before submit" for live/fast-path siblings. Those rows are created and disposed per-row while accepted tips keep arriving, and each one costs a submitblock, ~6 chain reads and two ledger writes, plus an accounting task, a fast-lane reservation and an accepted-block payout-preview barrier armed and withdrawn. A candidate whose parent is no longer the best tip, whose height the chain has decided in favour of a block carrying at least its work, and which holds no offer evidence is provably stale before the offer. submit_next now says so at dequeue and terminalizes the row instead of offering it. The predicate is #196's, unchanged: same evidence set E, same clauses (including 4b's work comparison), the same non-blocking-lease/revalidate/ fenced-write/exact-returned-set apply, and the same cleanup M. This is a second caller of that machinery, not a second notion of staleness. Placement and shape - Inside `if pending_finalize is None:`, after the disposition lease is claimed and the terminal-outcome fence has passed, and before the fast-lane reservation: a sibling on its way to abandonment must not consume max-block capacity. - It terminalizes rather than drops (D4). Dropping would leak the dequeued object's pending-share floor holder, which is keyed by object identity and cannot be reached by the apply's queue scan, and would leave the hash in `_block_candidate_disposition_flights` where #196's clause 1 rejects it forever. The floor is released through the existing `_release_dropped_duplicate_candidate_floor` seam. - The one deliberate divergence is an explicit self-exemption: the disposition flight this pass itself holds the lock of is not offer evidence. Exactly one hash is ever exempted, no other holder can hide behind it (the lock has a single owner, and the terminal fence is published before the lease is released), and every other member of E still rejects. Batching: one fenced write per skipped candidate. A dequeue is one candidate, and a batch accumulator on the submitter thread would hold leases, floor holders and preview barriers for an unbounded window on the only thread that reaches submitblock. The write was never the dominant cost; the submitblock was, and it is gone either way. The chain reads are amortized instead, by sharing one `_CollapseHeightProbeBudget` across the burst -- the mechanism the replay walk already uses across pages -- retired the moment the best tip changes, which is exactly as long as its cached heights are valid. Clause 3 is read from the chain, never from the coordinator's published tip: a blocked refresh is #181's symptom, and during the incident the published tip sits at the very parent every stale sibling names. That also satisfies D5 for replay-adopted rows, which are never marked outstanding. Two additive parameters on #196's functions, both defaulting to today's exact behaviour: `_select_superseded_block_candidates(..., ignore_leases=)` (mirroring the one `_block_candidate_collapse_evidence` already has for the pre-write revalidation) and `_apply_superseded_block_candidate_collapse(..., held_leases=)`. `_BlockCandidateChainView` now memoizes its best-tip and tip-height reads per view; both were already made at most once per view. Three fixed metric labels -- dequeue_considered, dequeue_skipped, dequeue_preserved -- partition this caller's population. The shared outcomes stay shared because they describe the same machinery. Instrument fidelity: `CadenceChainRpc.getblockheader` now reports compact `bits`, as qbitd does. Without it clause 4b had no work value, every evaluation failed closed, and the latency instrument reported a moved p95 produced entirely by the cost of failing closed. `drain_per_row` suppresses the new skip by default so it stays #194's per-row oracle, and takes `dequeue_skip=True` to drive the skip instead. Measured - Decided 100-candidate storm, live and restart views: 99 of 99 stale siblings skipped, winner never; `skip_set - oracle_abandoned` empty at 100 and at 3,120 in both views. Cost against the per-row oracle for the same population: 99 -> 0 submitblock, 693 -> 398 chain reads, 99 -> 0 durable attempt marks, 99 -> 0 accounting tasks. - Acceptance-to-preview publication (tests/test_prism_accepted_preview_ publication_latency.py, this host, 3 alternating runs per side). 150 tips/min: p50 0.5 -> 0.25 s, p95 2.0 -> 0.5 s, mean 0.65 -> 0.20 s, durable rows left pending 70 -> 32, submitblock 100 -> 20, chain reads 403 -> 353. 40 tips/min: p50 unchanged at 0.25 s, p95 0.25 -> 0.5 s, mean ~0.17 -> ~0.19 s, pending 4 -> 2-3. The uncontended cadence pays one clause-3 read it has nothing to gain from; both cadences stay far inside the 5 s child wait budget, which is not touched. Verified: full discovery 2,784 tests OK (2 skipped) against base 2,747 OK (2 skipped), no failures on either; the 135-test collapse suite, the storm suite, payout state, metrics and the new 37-test suite green; Docker Python 3.12 compileall clean; Ruff F/E9 clean on every touched file; bash -n and ShellCheck clean; PostgreSQL ledger and production-scale fixture suites pass against a real PostgreSQL 16 cluster. * Re-drive stuck accepted-parent payout transitions in-process (#205) * Add in-process targeted ancestor re-drive for stuck payout transitions (#190) A found block whose finalization defers on an unresolved accepted-ancestor payout transition could only re-check that transition: the retained-retry loop re-abandoned with ledger-confirmation-failed every cycle while the durable-replay path -- the one path that provably resolves the ancestor -- was short-circuited by the very retry candidate it needed to unblock (observed 2026-08-24: 37 cycles / 906s until the publication-progress watchdog restarted the process, and startup replay then resolved the same state in ~19s). The pending-parent deferral fence now tracks a per-ancestor streak of consecutive deferrals. On crossing a configured threshold (default 3) the submitter arms a targeted re-drive: the next replay_pending pass bypasses the retry/live/replay-queue short-circuits and runs the same durable-outbox enumeration startup replay runs, so a pending ancestor row is re-adopted and finalized in-process. When the completed enumeration proves no pending row exists for the ancestor, the stale armed transition is cleared only under the exact predicate the landing's already-confirmed branch uses (durable pool-block row confirmed and not reversed, and no in-process lane still owns the hash); anything short of that proof is left for the watchdog, which remains the backstop. Re-drives are capped per ancestor (default 3); on exhaustion deferrals fall back to the pre-#190 behavior. The submitter loop also stops skipping the replay entrypoint while a re-drive is owed, so sustained live traffic can no longer starve the resolving path. New qbit_prism_accepted_parent_redrive_{attempts,resolved, exhausted}_total counters and per-event log lines make the mechanism observable; coordination_blocked_budget and the watchdog are unchanged. New env knobs: PRISM_ACCEPTED_PARENT_REDRIVE_DEFER_THRESHOLD, PRISM_ACCEPTED_PARENT_REDRIVE_ATTEMPT_MAX (0 disables the mechanism). Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01FHyzBJpX88B5Y8gKvkXZjo * Test the #190 ancestor re-drive: wedge regression, cap, and sweep guards The wedge regression test reconstructs the incident shape -- a retained child whose ancestor transition is armed and landed with the preview never published -- and asserts both halves of the finding: below the threshold the deferral repeats while replay_pending stays short-circuited by the retained retry candidate (the pre-fix starvation), and on crossing it the forced pass re-adopts the ancestor's pending outbox row through the same durable-replay adoption startup uses, the submitter finalizes it while the child's retry is parked, the transition clears, and the child's next retry passes the fence and completes. The cap test drives repeated threshold crossings and asserts exactly attempt_max re-drives arm, exhaustion is counted once, and every later cycle falls back to plain deferral with nothing armed. The sweep tests pin the no-pending-row guards: a durably confirmed, unowned transition is cleared; an unconfirmed or reversed pool block leaves the transition for the watchdog; and an in-process-owned hash is never even consulted against the ledger. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01FHyzBJpX88B5Y8gKvkXZjo * Pass the #190 ancestor re-drive knobs through the compose environment PRISM_ACCEPTED_PARENT_REDRIVE_DEFER_THRESHOLD and PRISM_ACCEPTED_PARENT_REDRIVE_ATTEMPT_MAX reach the coordinator container alongside the other accepted-parent knobs, defaulted to the code defaults. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01FHyzBJpX88B5Y8gKvkXZjo * Harden the #190 ancestor re-drive against review findings A multi-pass review of the re-drive surfaced one gap in the sweep's safety proof, three bookkeeping defects, budget/pagination drift in the forced pass, and two untested seams. All are fixed here; the mechanism and its knobs are unchanged. The stale-transition sweep now demands the same node-side proof the landing's already-confirmed branch rests on: a fresh chain probe must place the block's own hash on the active chain at its height before the in-memory transition is cleared. The durable row alone is not proof -- reconciliation fails closed while a landed transition exists, so a block qbitd has already reorged out can still read confirmed in the row, and clearing on the row alone would have unfenced descendants onto the orphaned block's carry. Anything short of a definite probe verdict stands down to the watchdog. The forced pass now genuinely runs the code startup replay runs: one forced-enumeration flag drives the short-circuit bypasses, the keyset pagination, and the landing-class query budget, so a backlog beyond the legacy widening window no longer burns every capped attempt without reaching the target, and a slow forced pass surfaces on the landing series the alerts watch. Its trailing disposition (truncation stand-down, adopted check, sweep dispatch) moved into _conclude_ancestor_redrive_pass. The per-ancestor bookkeeping collapsed into one record (streak, attempts, consumed, armed, exhausted) plus the child-keyed blocking index, each registry evicting strictly against itself. That closes two eviction defects: a hash that is both child and ancestor could lose its armed request to the eviction of a stale child entry, and an exhausted ancestor's attempts entry (refreshed only at arm time) aged to the eviction front where a pathological ancestor stream would discard the cap latch. The streak is now frozen while a request is armed, so every attempt needs a fresh threshold of deferrals instead of attempts 2..N arming on single deferrals right after consumption. Setting the attempt cap to zero now disables the mechanism outright instead of firing the alert-worthy exhausted counter, and the resolved counter counts only transitions a consumed pass actually ran for, so it can no longer overstate the mechanism's efficacy against ordinary-path resolutions. Replay adoption now skips a hash the single-slot retry holders still carry: the re-drive bypass let a forced enumeration reach the retained wedge child's still-pending row and adopt a second in-process copy under the holder. Only the holders qualify -- the wider lane registries name every admitted hash, and replay adoption is the queue-overflow recovery path for those. The in-process ownership check and the terminal-outcome eviction now share one pin-set collector so the two liveness proofs cannot drift. The suite grows from 8 to 18 tests: the ownership guard is now actually reached (retry-held ancestor, mutation-checked), the run-loop fall-through is covered in both directions (mutation-checked), and the new semantics -- chain-probe gating, fresh-streak-per-attempt, cap-zero disable, unconsumed-resolution accounting, eviction key-space isolation and wedged-ancestor survival under an ancestor storm, forced-pass pagination on the landing budget, and no-adoption-under-a-holder -- are each pinned. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01MpyKY3UFqeXZu3MUNH25Gz * Hold the re-drive streak while the ancestor is already in the replay lane Review of #205 flagged that re-arms against an ancestor a forced pass had already re-adopted burn capped attempts on passes that no-op on the adopted check. The starvation premise behind the finding is narrower than claimed -- the run loop's wakeup gate reads the replay queue too, so an adopted ancestor is served whenever the live queue empties or the retained retry is parked in backoff -- but the attempt burn was real: deferrals while a replay-lane copy of the ancestor is inflight prove nothing a new pass could act on, and banking them could exhaust the cap against slow-but-working finalization, standing the mechanism down for an ancestor it had already reached. The streak now holds while the ancestor is inflight and resumes the moment the flight ends. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01MpyKY3UFqeXZu3MUNH25Gz --------- Co-authored-by: Dan Hepworth Co-authored-by: Claude Fable 5 * Hold the payout snapshot weight in a band instead of rescanning per block (#206) * Hold the payout snapshot weight in a band instead of rescanning per block qbit retargets every block (ASERT), so the snapshot window weight -- the 2x margin times the 8 x difficulty reward window -- drifts a fraction of a percent with every tip change. The incremental window cache invalidated on any weight delta, which forced the full oracle rescan (a multi-second snapshot_at_job_issue read under the postgres writer lock) on effectively every block: ~211 full rescans in 6h on union-mainnet, and exactly the writer-lock pressure that starves the block submitter's landing-window reads and delays accepted-block preview publication (mainnet alert #236, 2026-08-25). The superset stays payout-correct while it still covers the live reward window: every consumer re-derives the exact 8 x difficulty cutoff from its own live network_difficulty (_prism_window_shares, the fold's requested_window_weight), and the artifact stamps the live difficulty regardless of the cached weight. Replace weight equality with a band -- rescan only when the cached weight falls below 5/8 of the live snapshot weight (reward window plus 25% headroom out of the 2x margin) or exceeds twice it. The periodic runtime-check now reads its oracle at the cached weight so ordinary drift cannot masquerade as self_check_mismatch, and weight re-centering rides the forced rescans (landing reconcile invalidation, cold start, invariant failures) plus the band breach itself. Full local suite: 2404 tests, the only failures are the five spool/splice cases that fail identically on a pristine origin/2.x.x checkout on this platform. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01GzhZmYKEHVyVYWp52KmKNr * Address deep-review findings on the snapshot-weight band Findings from the multi-lane review (code-review medium, thermo-nuclear, adversarial Fable, adversarial Codex): - The band floor's payout safety silently depended on PRISM_SNAPSHOT_WINDOW_MARGIN == 2 with nothing coupling them, and the fold has no error path for an under-covering superset -- a margin retune would have truncated the reward-window tail and misdistributed the coinbase without a single failure. The relationship now fails at import (an explicit raise, so python -O cannot strip it). - The periodic self-check adopted its oracle window at the cached weight, so the one recurring full-oracle read never re-centered the cache: a falling difficulty could pin an up-to-2x oversized ledger scan under the writer lock until the next forced rescan. The self-check now reads once at the larger of the cached and live weights, builds the comparison window at the cached weight (in-band drift is not a pipeline mismatch), and adopts the oracle window at the live weight so every runtime-check re-centers. - The share-window serialization cache was keyed on the live-difficulty window weight, so ASERT rotated the key -- and re-spooled an identical multi-megabyte payload -- every block even though the payload is a pure function of the shares. The weight component is dropped from the key. - full_rescan_reason "network_difficulty_changed" now fires only on band breaches, so the name lied during exactly the incidents it exists for; renamed to "snapshot_window_weight_out_of_band". - SnapshotWindowWeightBandTests sat below the __main__ guard and never ran on direct execution; moved above it. A new test drives an overdue self-check under in-band drift and asserts both the like-for-like match verdict and the re-centered weight -- reverting either half of the self-check change fails it. Full local sweep: 2405 tests, failures limited to the five spool/splice cases that fail identically on a pristine origin/2.x.x checkout on this platform. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01GzhZmYKEHVyVYWp52KmKNr * Re-center the self-check weight only when well oversized Bugbot: adopting the runtime-check's oracle window at the live snapshot weight installed a DaemonShareWindowMirror digest the daemon never prepared, so the next advance answered needs_full and re-paid a full oracle read under the writer lock at every self-check once ASERT had drifted in-band -- the exact cost class this branch removes. Re-centering is now one-sided and thresholded: ordinary drift adopts the oracle window at the cached weight, keeping the daemon-prepared digest warm, and only a cached weight more than 25% above the live snapshot weight re-centers (the falling-difficulty oversize direction nothing else bounds; undersize is safety-bounded by the band floor and re-centered by every forced rescan). The oversize case makes the cached weight the larger one, so the single oracle read at the cached weight covers both exact cutoffs. Tests split accordingly: small drift keeps the cached weight with a match verdict, a well-oversized weight re-centers, and both stay at one oracle read. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01GzhZmYKEHVyVYWp52KmKNr --------- Co-authored-by: Claude Fable 5 * Fix PRISM difficulty convergence and credited hashrate visibility (#204) * Converge PRISM difficulty and expose credited rates * Fix durable vardiff ordering and prune races * Fix vardiff prune scheduling and AuxPoW cap validation * Move the coordinator images and CI to Python 3.14 (#201) PRISM is interpreter-bound on the paths that matter. #162 established that directly: once block serialization left the accept path, what remained was 16.5 us of pure interpreter work, flat across a 12,600x range of template size. Interpreter version is therefore a first-order lever on this workload, and the tree was two feature releases behind. Eight pins move: three Dockerfiles, four CI jobs, and the documented compileall validation command. No code changes. The Debian release is now pinned explicitly rather than inherited. The unsuffixed -slim tags track whatever Debian is current, so today's agreement between 3.12-slim and 3.14-slim is luck rather than contract; without the suffix a future rebuild silently becomes a distro bump that moves the apt Rust toolchain under cargo build --locked. Verified both bases resolve identically today: trixie, rustc 1.85.0+dfsg3-1, postgresql-client 17+278. Compatibility was checked rather than assumed. The coordinator image has one third-party dependency and psycopg-binary ships cp314 wheels, so the 3.2.* pin does not move -- confirmed by installing and importing psycopg 3.2.13 on 3.14.7 in the new base. Nothing in the tree imports a stdlib module removed in 3.13 or 3.14. Behaviour is unchanged on the suite. Full discovery run in both containers under identical conditions: 2,675 tests, 7 failures and 36 errors on each, and the failing test-name sets diff clean. Those 43 are environmental -- the bare container has no make or git, so the Makefile-lifecycle, source-staging and production-gate suites fail the same way on both interpreters. One consequence to record: this invalidates the absolute numbers in #143, #162 and #131, which were all measured on 3.12. The direction of every conclusion survives; the figures quoted in those issue comments should be re-baselined rather than carried forward. Closes #169. Co-authored-by: Claude Opus 5 (1M context) * Validate fresh block confirmations against the current-balance summary (#210) A fresh confirmation previously invoked the height-bounded historical balance reader, whose O(history) carry-table scan can hold payout serialization long enough for a following block to exceed the fixed parent-preview wait and raise an alert (issue #209). A fresh confirmation lands as the ledger's newest confirmed pool block under the balance serializer, with every younger landing deferring on its unresolved payout transition, so the transactionally maintained current-balance summary is exactly the active-chain view as of that block; read that O(recipients) summary instead. Only an exact replay of an already-confirmed ancestor can run after a later pool block changed global balances, so replays keep the height-bounded reader. Regression coverage: the suite now fails if a fresh confirmation touches the historical reader, and the confirmed-ancestor replay test asserts its height-bounded read still happens. The landing statement classification drives the as-of read explicitly, since the happy-path landing no longer emits it. Fixes #209 Co-authored-by: Claude Fable 5 * Keep block-candidate replay reads out of the writer convoy (#213) * Keep read-only candidate enumeration out of the writer convoy (#211) `PsqlShareLedger.pending_block_candidate_rows()` is one read-only, single-snapshot statement, but it took `_operation_gate(self._lock, "writer lock")` before sending it. That put a read in the same coordinator-local convoy as accepted-block accounting: on union-mainnet the outer call reported `replay-outbox-query exceeded 5s` while the inner PostgreSQL deadline had only ~0.4-1.3s consumed, PostgreSQL showed zero blocked backends and the host was ~96% idle. Accepted candidates converged in 79-186s and `qbit_prism_accepted_parent_unresolved_oldest_seconds` reached ~101s, firing PrismUnresolvedParentAgeOverBudget. The page now takes the existing bounded read slot instead. Same statement, same retry-safe execution, no new connection and no new thread: the read semaphore admits `read_concurrency` callers and the pooled client holds `read_concurrency + 1` connections, so the gate/pool invariant is unchanged. Nothing downstream weakens, because nothing downstream trusted the snapshot. `mark_block_candidates_abandoned` re-asks `qbit_pool_blocks` inside its fenced UPDATE under the writer-id/epoch/session-token lease predicate and reports the exact set it transitioned; `mark_block_candidate_attempted` and `_finish_block_candidate` carry the same fence plus `state = 'pending'`; the dequeue-time offer path re-reads `pool_block_state` and the chain rather than reusing a page fact. Pagination, `pool_block_exists`, single-flight and commit-before-wakeup ordering are untouched. Attribution is the other half. Read-slot operations now record local admission apart from server execution, exported per operation as `qbit_prism_ledger_read_gate_wait_seconds_total`/`_max`, `qbit_prism_ledger_read_gate_timeouts_total`, `qbit_prism_ledger_read_execute_seconds_total`/`_max`, `qbit_prism_ledger_read_execute_timeouts_total` and `qbit_prism_ledger_read_calls_total`, so the next budget exhaustion names the half it went to instead of reporting one opaque duration. Coverage: - tests/test_prism_replay_enumeration_gate.py drives the interleaving on the #128 deterministic harness: a fenced write holds the writer gate past the fast-call budget while the enumeration completes through the read path, with a same-budget writer-admission control proving the hold was impassable; and, in the landing topology, a page read across an in-flight accepted-block write comes back stale about the landing candidate and the fenced batch write refuses it anyway. - tests/prism_concurrency_harness.py models the pending-page read and the fenced batch abandonment (both pinned to production SQL fragments); tests/test_prism_landing_harness_model.py drives them through real calls. - tests/test_prism_share_ledger.py pins the gate choice against a refusing writer gate and pins the gate/execute attribution split, including which timeout counter each expiry lands on. - test/test-prism-postgres-native-ledger.sh covers the production call shape over the pooled psycopg client against a real server, with pagination exactness across the same held gate; test/test-prism-postgres-ledger.sh adds the strictly stale page ordering against real SQL. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01E8ZVLRWXd19URK3TtT7ZjR * Isolate stale-page Postgres fixture * Attribute pre-statement read expiry locally * Delay psql execution timing until spawn * Preserve legacy psql execution seam * Attribute native reads only after connection setup --------- Co-authored-by: Claude Opus 5 * Make writer-lease heartbeat tolerate verification tail latency (#214) * Size the PRISM writer-lease heartbeat for verification tail latency (#212) union-mainnet accumulated 69 coordinator restarts during one rapid-block burst while PostgreSQL, qbitd, storage, CPU and memory were healthy and the coordinator still held its writer advisory guard. The exits were genuine protective exits: the heartbeat's server-proven envelope had elapsed. It had elapsed because the envelope was derived from the adoption silence alone (silence/4, silence*0.75) and so covered a heartbeat's idle interval but not the tail of a lawful verification. Representative exits reported activity ages of 0.54-0.78s against a 0.75s budget and server-proven ages of 0.76-0.91s against a 0.80s cap. Four changes, none of which weakens fencing. State the timing as one inequality. lab/prism/writer_lease_timing.py owns the adoption silence, heartbeat interval, failure budget, monitor interval and exit margin as a single WriterLeaseHeartbeatPolicy, derived from measurable phase budgets: the guard session's own 0.5s statement_timeout, the 0.25s heartbeat interval, and a 0.5s process scheduler-slack allowance. The largest gap a healthy coordinator can leave between completed guard round trips is their sum, 1.25s, which is both the failure budget and the floor for the staleness cap. The exit envelope that must sit between that cap and the adoption silence reserves the monitor thread's own lateness, not merely its poll granularity: exit_margin + 2 x monitor_interval + scheduler_slack = 0.70s. The monitor is a Python thread in the same process, behind the same GIL, as the heartbeat whose stalls this policy explicitly budgets for, so it cannot be assumed to wake on time when the heartbeat does not. Reserving only the poll intervals leaves a real split-brain window: at the shipped budgets a maximally late monitor observes the stale edge and finishes its exit 2.449s after the last provable instant, while a successor becomes adoption-eligible at 2.0s. With the scheduler slack reserved the same worst case completes at 1.95s, one monitor poll clear of the adoption edge, and the second monitor_interval term is what makes that strict rather than exactly equal. The adoption silence must therefore exceed 1.25 + 0.70 = 1.95s, and ships at 2.0s - unchanged, because it was already large enough for the corrected envelope; the staleness cap tightens from 1.80s to 1.30s and the stability surplus to 0.05s. Failover costs 2.0s of silence rather than 1.0s - half the temporary downstream 4.0s pin, and derived rather than dialled in. coordinator_config reads these values from the policy instead of re-deriving them, the process scheduler slack is resolved through the same port pattern as every other term so a scaled deployment or test policy validates what it actually runs, and the policy validates itself at import. Reject unsafe policies at startup. A combination that cannot guarantee hard exit before replacement adoption now hard-exits with the failing term named, before either thread is armed, instead of printing a warning and running with no exit-before-adoption guarantee. A combination that is safe but leaves the cap below the healthy-gap bound - which cannot produce two writers, only avoidable restarts - still warns and starts. Separate frequent ownership proof from TTL renewal. prove_writer_lease_guard_session asks only the ownership question, on one non-blocking statement with no row lock and no pg_stat_activity scan: the same two conditions the full verification raises on, so the exact-session guarantee is unchanged. Renewal is preserved by escalation, not by dropping it: the proof reports the committed row's TTL, and the heartbeat runs the full renewing verification on the first beat and on any beat whose proof reports the row inside the own-write authority margin (>= half the TTL), escalating within that same beat. Renewal is therefore never later than before, and throughout a long own fenced write - when renewals are actually being skipped - every beat runs the same fail-closed verification as before. Adoption safety is unaffected because the successor's silence is gated by the later of the row edge and its own guard-acquisition edge, and the guard edge is a full silence from acquisition regardless of how stale the row looks. Attribute the phases. Every attempt records guard-slot wait, guard SQL across its round trips, scheduler delay, statement count, mode and outcome; the monitor also records its own worst wake delay. Hard exits now name which bound tripped, the last and worst attempts' phase breakdowns, the monitor's lateness, and every term of the policy. The same data is exported with fixed label vocabularies (28 series, closed sets for modes, outcomes, phases and policy terms). Server-proven stamps also move to the conservative send edge of each round trip rather than its receipt time, so scheduler delay after a response can no longer make the session look fresher than PostgreSQL proved. Testing extends the #128 deterministic harness rather than adding a framework: it models the read-only proof statement, per-statement server time bounded by the guard's statement_timeout, and the per-round-trip callback. The real heartbeat and monitor loops run as harness actors on the virtual clock, so a production-shaped burst - near-bound guard statements, repeated scheduler stalls, a fenced write holding the lease tuple, and fence callers competing for the guard slot - is deterministic and instant. It reaches a peak server-proven age above the pre-#212 cap with no exit, while injected guard loss, a wedged proof and a cancelled guarded statement each end this coordinator's authority before any successor can take the guard. Two further scenarios drive the combined worst case the envelope is sized for - authority gone, the heartbeat thread descheduled entirely, and the monitor waking a full monitor_interval + scheduler_slack late on a poll phase chosen so the bound elapses immediately after a poll - and assert the exit completes, exit margin included, before the successor's measured guard-acquisition edge plus the silence. Both fail with the unreserved envelope at exactly the 2.449s vs 2.0s breach above. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01GX1dKh4fGhRhqf5jyuHRWE * Expose every lease timing policy term --------- Co-authored-by: Claude Opus 5 * Public dashboard API: reorg visibility and network hashrate (#217) * Public dashboard API: reorg visibility and network hashrate Add a chain_state filter (active|all|reversed) to GET /public/v1/blocks. The default stays byte-compatible prism.dashboard.blocks.v1; the non-default filters return prism.dashboard.blocks.v2 rows carrying chain_state and disconnected_at, with pagination totals describing the filtered set and the origin cache partitioning on the new param. Surface pool.blocks_reversed_total and pool.blocks_inactive_total on pool-summary (reusing the audit summary's per-state count SQL), plus a nullable, non-fatal network.hashrate_ths from qbitd's getnetworkhashps restricted to the permissionless lane the pool's shares are credited against. blocks_found_total semantics and the bits-loss 503 behavior are unchanged. Update the OpenAPI contract, README, and fixtures (including a reversed block fixture), and cover the new paths in the contract, dashboard API, share-ledger, and real-Postgres ledger gate suites. Co-Authored-By: Claude Fable 5 * Serialize disconnected_at only for reversed public block rows The reject flip stamps disconnected_at too (its maturity_state flip requires the column), so a rejected block in the chain_state=all view read as a reorg casualty. Mask the public field to reversed chain states, matching the contract's definition of a reorg disconnect time that is null for every non-reversed row. The real-Postgres ledger gate now surfaces a rejected block through chain_state=all and requires its disconnected_at to be null, and the SQL-shape unit test pins the guard. Co-Authored-By: Claude Fable 5 --------- Co-authored-by: Claude Fable 5 * Harden public hashrate-series reads: statement deadline, bucket vocabulary, in-process stale-while-revalidate (#218) * Harden public hashrate-series reads Three independent hardenings on the public read tier, each standing alone ahead of the aggregate-rollup work. Statement deadline on public origin computations. A request the fronting proxy abandoned at its ~15s timeout kept its Postgres scan running to completion, holding one of the four bounded read slots against every other public read and competing with the share writer's I/O. Database-backed routes now run dispatch under the ledger's statement_timeout scope (PRISM_PUBLIC_READ_STATEMENT_TIMEOUT_SECONDS, default 20s, 0 disables), duck-typed exactly like the coordinator's block-submitter scope so scope-less ledgers keep working. A ledger-reported deadline becomes a 503 with the new read_timeout error code, no-store and never cached, so one expensive miss cannot poison the response cache with a refusal. On 2.x.x the read service is the only public dispatch site; the coordinator's listener refuses /public/v1 outright, so no second call site exists to wrap. Static bucket-per-range vocabulary on the hashrate series. range=all&bucket=5m was a legal unauthenticated full-table scan whose response outgrows the cache size cap, so every such request re-scanned the ledger with no cache in front of it. Each range now admits a fixed bucket set (1w: 5m/1h/1d; 1m: 1h/1d; 6m and all: 1d), rejected with a 400 naming the allowed buckets. bucket=auto always resolves inside the allowed sets, and the deployed frontend only ever sends auto, so no real client breaks. The vocabulary and its check are importable for the block-marker series to reuse. In-process stale-while-revalidate. The cache policy already carried stale_while_revalidate_seconds but only advertised it to CDNs; an expired in-process entry was dropped and the next requester blocked for the full recompute. An expired entry inside the window is now served immediately with its honest Age while at most one background refresh per key recomputes through the ordinary status/size gates; concurrent stale hits share that refresh, and a failed refresh keeps the stale entry servable and frees the slot for a retry. The in-process window is clamped to the route's staleness budget minus its TTL, so a stale serve can never trip the unconditional budget refusal that the blocking path used to satisfy by recomputing, and it is withdrawn entirely while the database is unreachable so the outage contract is unchanged. The read service counts the new STALE state in its cache metrics. No response body schema changes: only the new 400/503 error paths and cache/header behavior. The dashboard README and OpenAPI document the vocabulary, the deadline knob, and the in-process revalidation window. Co-Authored-By: Claude Fable 5 * Widen the read deadline to artifact lookups and spare stale-servable cache entries Review follow-ups on the public read hardening. The statement deadline armed on path_reads_database(), whose immutable-content exemption answers a staleness question, not a cost one: a cold /public/v1/artifacts/{sha256} occupies a bounded Postgres read slot exactly like any other cold read, for as long as the lookup takes. A separate path_occupies_read_slot() predicate -- registry READ_SLOT access, immutable or not -- now decides the deadline, while the replica and outage gates keep their exemption. Capacity reaping removed every entry past its TTL, including ones still inside the stale-while-revalidate window the lookup path had just promised to serve. Entries record their own stale_until at store time and reaping removes only entries past it, so a full cache sheds dead weight without turning a promised STALE answer into a blocking recompute. The statement-timeout tests cleared the whole process environment, which also dropped NO_PROXY: on a proxy-configured machine the localhost request then went through the proxy and failed. They now patch only the deadline knob itself. Co-Authored-By: Claude Fable 5 * Give the whole public dispatch one deadline and survive refresh-thread failure Second-round review follow-ups on the public read hardening. The deadline armed the ledger's statement_timeout scope, which hands every admission and SQL statement a fresh budget: a route that performs several sequential ledger reads (the miner detail page does) could run many multiples of the configured limit after the proxy hung up. The scope is operation_timeout now -- one budget counting down across the entire dispatch, with every statement still cancelled server-side on whatever remains -- and a wiring test pins that a multi-read dispatch enters the scope exactly once. The stale-serve path registered its background-refresh slot before Thread.start(), so a thread-creation failure under resource pressure stranded the slot: later stale hits never retried the refresh and, past the window, blocking requests waited on an event nobody could set. Startup failure now frees the slot and wakes any coalesced waiter; the stale answer is still served and the next stale hit retries. Co-Authored-By: Claude Fable 5 --------- Co-authored-by: Claude Fable 5 * Serve the public hashrate series from incremental rollups (#219) * Serve the public hashrate series from incremental rollups The dashboard hashrate-series endpoint re-aggregated raw share-ledger rows on every cache miss, so long ranges (1m/6m/all) cost O(shares in range) and timed out behind the proxy limit as pool history grew. Maintain per-bucket rollups instead -- pool-wide and per-miner, at the 5m/1h/1d grains, keyed by the exact bucketing expression the serving query uses so rollup and raw bucketing can never disagree -- advanced incrementally past a single durable watermark. Share rows are immutable and share_seq is append-only, so watermark batching folds every share into its bucket exactly once with no re-aggregation window; rejected rows advance the watermark without counting. The watermark advance is a guarded upsert and both rollup upserts are gated on it winning, so a lost race writes nothing at all and the caller fails loudly instead of double-counting -- two live maintenance passes would mean the single-writer invariant is already broken. Serving merges exact-grain rollup buckets with a live tail of shares above the watermark inside one statement (one snapshot, so a concurrent advance cannot double-count), and re-aggregates only the bucket straddling the range lower bound, which raw scans emit partially. The bar is byte-identical responses in every state: mid-backfill, an absent progress row, and a database without the rollup tables all degrade to exactly the raw scan, which also keeps pre-migration databases and half-deployed states working with no operator action. Postgres gate scenarios pin the parity matrix (subjects x ranges x grains across every serving state, the partial boundary bucket included), accumulation across adjacent passes, the watermark guard under a real row-lock race, and both fallbacks. Co-Authored-By: Claude Fable 5 * Run hashrate rollup maintenance as a coordinator background service The rollups only stay cheap to serve while something keeps folding new shares past the watermark. Register a coordinator background loop for it -- the coordinator already holds the single writer role, and the public read tier constructs its ledger without a writer lease and must never run maintenance. Each pass folds one bounded batch and keeps looping while the ledger reports more history behind the watermark, which is also how a cold database with weeks of history backfills itself through the very statement live shares take: there is no separate migration to operate. Once caught up the loop sleeps the configured interval; a failed pass logs and retries on the next interval, and the transactional watermark makes retries exact. The service registers only for ledgers exposing the rollup advance (the Postgres backend). Unlike an enable flag, that capability cannot flip after construction -- a memory-backed ledger never grows the method at runtime -- so registering an unstartable service would only mislead the registry snapshot. The enable flag and cadence stay live serve-time attributes (PRISM_HASHRATE_ROLLUP_ENABLED, _INTERVAL_SECONDS, _BATCH_SHARES) so tests and embedders can retune a constructed coordinator, and lightweight coordinators built without the full constructor read the absent flag as disabled. Co-Authored-By: Claude Fable 5 * Document the hashrate rollup knobs and serving path The three PRISM_HASHRATE_ROLLUP_* variables join the operational knob table with a short section on how the rollups, the backfill loop, and the raw-scan fallback fit together, and the public dashboard contract notes that hashrate-series is now served from incremental rollups -- a serving-side change only, with the response schemas and values unchanged. Co-Authored-By: Claude Fable 5 * Fence rollup maintenance on the writer lease and unconstrain bucket totals Review follow-ups on the incremental hashrate rollups. The maintenance statement ran under the process-local writer lock alone. Every other mutation of ledger-derived state renews and verifies the exact (writer_id, writer_epoch, writer_session_token) lease tuple inside its own statement, and the rollup advance now does the same: the watermark upsert applies only when that renewal matched, the rollup upserts were already gated on the advance winning, and a fenced-out coordinator therefore writes nothing and raises instead of continuing to mutate rollups after a takeover. The watermark CAS already made concurrent passes safe against double-counting; the lease gate closes the fencing-discipline gap, not a counting race. The aggregate difficulty columns mirrored the source column's numeric(78, 0). Individually valid 78-digit share difficulties can sum past that in one bucket, and a constrained aggregate would make the upsert overflow on every retry and wedge the watermark permanently. Bucket totals are unconstrained numeric now -- matching the raw scan's sum(numeric) -- with an idempotent catalog-only widen for databases that applied the earlier definition. Gate scenarios pin both: a 79-digit bucket total folds and accumulates exactly, and a usurped lease leaves the watermark and both rollup tables untouched while the pass fails loudly, then folds the held-back share once the lease is restored. Co-Authored-By: Claude Fable 5 * Re-aggregate the current bucket instead of serving its rollup row Second-round review follow-up on the incremental hashrate rollups. Maintenance folds by sequence, so a coordinator clock running ahead of the database clock can put a share into the current bucket's rollup row that the raw scan's accepted_at <= ended_at would exclude until the clocks catch up -- and the rolled branch served that row whole, breaking byte parity for exactly the window of the skew. The bucket containing the statement's upper bound is now treated like the partial lower-bound bucket: the rolled branch stops strictly below it and a single boundary re-aggregation covers both partial ends from the already-rolled-up sequence range, bounded by one bucket span of shares each, with the raw scan's upper-bound predicate applied. The gate's parity scenario now seeds a future-dated share and holds every serving state to excluding it exactly as the raw scan does. Co-Authored-By: Claude Fable 5 * Run each rollup pass inside the writer-operation admission Third-round review follow-up on the incremental hashrate rollups. The maintenance loop only consulted the stop event, so a pass beginning after the stop check could race an orderly shutdown: quiescence observed zero active writers, released the lease -- which expires the tuple but keeps its identity -- and the delayed pass then renewed that exact identity and mutated rollups after shutdown reported the writer quiesced, potentially delaying a successor's takeover. Each ledger advance now runs inside _writer_operation("hashrate_rollup_maintenance"), like every other mutation of ledger-derived state: shutdown counts an in-flight pass and refuses a new one, and the loop treats the refusal as the orderly stop rather than a failure. Unit tests pin both sides: a pass executes with the admission held and releases it, and a closed admission exits the loop without touching the ledger or logging an error. Co-Authored-By: Claude Fable 5 --------- Co-authored-by: Claude Fable 5 * Serve found-block markers at /public/v1/block-markers (#220) * Add a block-marker read model to both share-ledger backends dashboard_block_markers aggregates qbit_pool_blocks onto the hashrate chart's bucket grid: floor-aligned bucket epochs, a per-bucket count, and at most the three most recent blocks per bucket (found_at DESC, block_height DESC), excluding reversed blocks with the same predicate dashboard_blocks applies. Bounded ranges anchor their lower bound on a caller-provided epoch through hashrate_series_min_epoch, keeping only fully covered buckets, so marker buckets and hashrate buckets share one grid even when the database clock disagrees with the application clock; the all range is unbounded. The Postgres implementation is one self-contained statement on the bounded read path (_run_read_json); the table is ~10^4 rows, so a grouped scan is cheap and no new index is needed. The in-memory backend records no found_at for its pool blocks, so it returns an empty series the way its other dashboard read models degrade. The Postgres A1 gate grows a scenario seeding blocks across buckets -- including a found_at tie, a block one second below the range bound, and a reversed block -- and asserts counts, top-3 ordering, range filtering, and the unbounded range. Co-Authored-By: Claude Fable 5 * Serve found-block markers at /public/v1/block-markers The dashboard builds chart markers by paging /public/v1/blocks and can only afford them on its shortest view; at hundreds of found blocks a day the 1m/6m/all timeframes would need hundreds of pages per chart render. Serve the markers pre-bucketed instead: prism.dashboard.block-markers.v1 reports the total non-reversed blocks in range and, for each non-empty bucket, the block count, the three most recent blocks, and a truncation flag. Range and bucket vocabulary and auto resolution are identical to hashrate-series, and each range is clamped to the buckets its chart renders (1w: 5m/1h/1d, 1m: 1h/1d, 6m and all: 1d) so a stranger cannot request an unbounded-length public series. The route is classified in the endpoint registry as an extracted read-slot route with the plain row-read TTL and the derived 15s staleness budget -- the query is a cheap grouped scan of the small pool-blocks table, not one of the expensive pool-wide aggregates -- and the registry-derived read-service gates (replica freshness, outage refusal, staleness budgets) pick it up from the classification. Co-Authored-By: Claude Fable 5 * Document the block-markers endpoint contract OpenAPI path and component schemas in the surface's existing style (additionalProperties: false throughout), a README section explaining why markers are served pre-bucketed and how they align with the hashrate series buckets, the block-markers staleness budget row, and a block-markers.json fixture wired into the contract suite. The new contract tests pin the fixture to the derivations the endpoint applies: bucket-grid alignment and ascending order, blocks confined to their bucket's span, top-3 listing with the truncated flag derived from block_count, total_blocks equal to the summed bucket counts, and agreement with the blocks.json fixture's latest block so the two fixtures describe one pool. Co-Authored-By: Claude Fable 5 * Fold the marker bucket vocabulary into the shared clamp helper The block-markers endpoint carried its own copy of the per-range bucket table because it was written against a base that predated the hashrate series' shared vocabulary. Markers land on top of the hashrate chart, so the chart's own allowed_hashrate_bucket() is the markers' clamp too; the duplicate table and its bespoke rejection message are gone, and the contract test reads the one shared table. Co-Authored-By: Claude Fable 5 * Cap the marker scan at the statement clock Review follow-up on the block-markers endpoint. The scan had only a lower bound, so a block whose found_at sits in the future -- a database clock jump later corrected, or a bad import -- would render beyond the chart's current endpoint and inflate total_blocks until wall time caught up. The same found_at <= clock_timestamp() cutoff the hashrate series applies now bounds the markers, and the gate scenario seeds a future-dated block and holds every range to excluding it. Co-Authored-By: Claude Fable 5 --------- Co-authored-by: Claude Fable 5 * Use semantic coverage for mining delivery health (#221) * Use semantic coverage for delivery health Time the initial-delivery-stalled predicate on semantic current-work coverage instead of exact current-tip coverage (#216). Exact-tip churn alone no longer fails /healthz: the 2026-08-31 fleet held 4 of 33 miners on the exact observed tip with all 33 semantically current, no genuinely pending initial jobs, and continuing deliveries, and now stays ready past the initial-job deadline instead of flapping to 503. Genuine first-job starvation and sustained loss of semantic current work still fail closed after the deadline. The semantic gap has its own timer and a new semantic_current_work_gap_age_seconds field; the exact-tip coverage ratio and current_tip_coverage_gap_age_seconds are unchanged telemetry. overload_now, reject_storm, and the cap-saturated reason stay on exact-tip coverage, each pinned by a test. Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_01BEZjdXmzYQtgC24D2s784d * Keep semantic coverage session-current --------- Co-authored-by: Claude Fable 5.1 * Bound collapsed-candidate cleanup retry backlog (#222) * Bound collapse cleanup retry backlog Issue #198. A collapsed row whose post-write cleanup failed keeps one in-memory retry record, the only replay source it has left, so under a systemic cleanup fault the registry grew by one record per row the collapse won. Bound admission instead of cleanup authority: once the backlog holds PRISM_BLOCK_CANDIDATE_CLEANUP_RETRY_BACKLOG_MAX records (default 4096, validated at startup, capped at the 8192 terminal-outcome registry bound) the replay-page walk and the dequeue-time skip stop handing rows to the fenced bulk terminalization; declined rows stay durable and pending for the per-row path, and a page with partial headroom admits exactly that many selected rows. Records, holders and fences for already-terminal rows are never shed, and #196's selection predicate and fencing are unchanged. Export fixed-cardinality gauges for backlog depth, the configured bound, oldest owed age from first deferral, retained pending-share holders, terminal-outcome pins, a backpressure gauge and engagement counter, plus a closed backlog_deferred collapse outcome, and pin the new family in the frozen metrics reference. Warn once per 60 s with count-only fields when the bound engages. Extend the candidate-storm rig with fault injection for every cleanup dependency and a measured recovery proof; derive the default from the 3,120-row measurement (about 968 B per record with its holder, tens of thousands of records drained per second on an idle lane, half the fence registry as the pin budget). Add deterministic sustained-failure and recovery tests at the shipped bound, storm corroboration for every fault, startup validation tests, and document the gauges, knob, measurement basis, threshold policy, alert rule and recovery contract. Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_01SwbrD98QdEfAHAZ1qQXy4T * Harden cleanup backlog bounds and recovery guidance * Bound backpressured replay enumeration --------- Co-authored-by: Claude Fable 5.1 * Bound accepted-preview publication latency during dense accepted-tip cadence (#230) * Freeze the accepted-preview latency attribution contract Issue #224, Wave 0. Union mainnet's accepted-preview publication tail crosses the 4 s warning and 5 s child wait budget without anything on /metrics naming the landing stretch that owns a sample. Three lanes will instrument the core landing path, landing attribution and ledger attribution in parallel; this commit is the shared surface they record into, behavior-neutral and mergeable on its own. lab/prism/accepted_preview_telemetry.py freezes the closed vocabularies (landing sub-phases; reconcile callers and steps, spelled to join the existing lookup family; payout-window full-rescan reasons mirrored from payout_state's literals plus an ``other`` fold, and the daemon versus in-process pipeline path; the ledger read operation names), a thread-safe sum/count/max owner with context-manager timers, a bounded per-publication diagnostic record and ring where a hash or height may appear, ``fold_ledger_read_stats`` for the existing operation label, and ``ensure_accepted_preview_telemetry`` so every lane and the renderer share one owner per coordinator without touching prism_coordinator.py. The module docstring records the verified invariant: reconciliation mutates pool-block and payout balance state but never qbit_share_ledger, which the later reread-instead-of-rescan optimization relies on and this wave does not implement. The renderer appends four summary families -- landing phase, reconcile pass, reconcile step, payout-window full rescan -- from an empty owner, so every cell renders at zero from process start, and folds out-of- contract ledger read operations into ``other``. Every pre-existing line of the metrics document is unchanged: the base render is an exact prefix of the new one. The frozen reference, parity seed and needles cover the new families and the fold; a direct test module pins the vocabularies, parses payout_state for reason drift, and checks the recorder, ring, owner sharing and rendered labels. Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_01GFRhW6iq62pKU8pGZSJskw * Reread prior balances instead of rescanning the window after reconciliation Issue #224: a reconcile pass that matured payouts, marked a pool block inactive, or reactivated one answered with force_full_window_rescan, so the accepted-preview landing paid an O(window) oracle read under the writer lock for a window that had not changed. Reconciliation mutates pool-block, payout and carry state but never qbit_share_ledger (the invariant recorded in accepted_preview_telemetry), so at a fixed anchor the accepted-share window stays exact and only the prior balances need a live read. Core fix (Option A): - Add a distinct force_prior_balances_read intent through _prepared_payout_state_candidate and _build_payout_ledger_artifact. It disables published-balance reuse and reads the carry aggregate live while the window goes through the normal debounce/delta path. It is never a full rescan and never records one. - The reconciler's confirmed-mutation path now requests that reread (ReorgReconcilerService._candidate_intent); the intent keyword is only forwarded when set so pre-existing preparation and build seams keep their exact signatures. - force_full_window_rescan keeps its whole meaning everywhere else: cold and cache invalidation, late-visible append, anchor/weight regression, daemon and incremental fallbacks, the periodic oracle, accepted-preview withdrawal, and the error-after-possible-commit path, which still republishes against a forced full rescan (and therefore fresh balances). Attribution (fixed telemetry contract only): - ReorgPorts gains an optional accepted_preview_telemetry owner; the coordinator wires ensure_accepted_preview_telemetry(self). Ports without one get a private owner. - reconcile records qbit_prism_reorg_reconcile_pass_seconds{caller} on every exit and the step family (admission_wait, watch_query, chain_probe, mutations, candidate_prepare, publish) around the calls themselves. - Callers are stamped per thread by the entry points -- landing (the flight-bypassing ensure_tip), post_confirm (forced publication), job_build (ensure_current), tip_refresh (the prefetch lane) and other -- so the reconcile_with_admission / reconcile_serialized / ensure_tip seams keep their kwargs. Optional _caller keywords are accepted by the coordinator entry points for explicit attribution. - payout_state records qbit_prism_payout_window_full_rescan_seconds {reason,path} only where the whole-window oracle actually runs: the full materialization (daemon vs in_process by which pipeline folded) and the periodic self-check. Tests: tests/test_prism_issue_224_core.py covers the intent split, the fail-closed paths, pass/step/caller observations on success and failure exits, rescan reason/path recording, seam compatibility, and the end-to-end coordinator path (maturation reuses the window, rereads and publishes fresh balances, records no reconcile_invalidation rescan). The two reconciler assertions that pinned the #112 forced rescan on the confirmed-mutation path now assert the reread intent. Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_01MgytztiBedDJPHGxnWnAqg * Attribute accepted-landing latency to its owning phase (#224) The publication histogram proves an acceptance-to-preview tail exists, but nothing on /metrics said which stretch of the landing owned a sample above the 4 s warning or the 5 s child wait budget. This instruments the existing definitive-acceptance-to-publication path against the Wave 0 contract, changing no payout, finalization, candidate, lock, retry, timeout or publication behaviour. The seven fixed phases are recorded where they actually run: lane_wait from the acceptance stamp to the lane starting the landing task, balance_lock_wait around the serializer acquisition (the plain `with` becomes an ExitStack so only the acquisition is timed and the release point is unchanged), reconcile around both inline ensure_reorg_reconciled_for_tip stretches, prior_balances_check around the bracketed reread and the live current_prior_balances comparison, chain_probe around pool-block-state, tip-height-rpc and _block_candidate_chain_probe, preview_prepare around both preview materializations, and preview_publish from entering the publish call to the instant the preview becomes visible. Per-hash state lives beside B1's acceptance stamps, under the same leaf lock and bounded by the same cap with the same oldest-first eviction, and is discarded on terminal abandonment and collapse withdrawal. It accumulates only while a hash's interval is open, so a stretch that ran before definitive acceptance and a retry's stretch after the interval closed reach the process-wide family without ever being claimed by a landing's diagnostic. The first publication -- published or degraded -- is the single point that closes the interval, so it pops the record, builds one AcceptedPreviewPublicationDiagnostic with reconcile caller=landing, and emits one bounded JSON lifecycle line in the payout-artifact style, both outside the leaf lock. Rescan and ledger totals stay at the contract's neutral values: no per-landing causal evidence for either is available here without a process-wide delta read that would claim another lane's work as this block's. Hash and height appear in that diagnostic and log only; every metric label is a fixed phase name. Checks: tests.test_prism_block_finalization, tests.test_prism_block_candidates, tests.test_prism_accepted_preview_telemetry, tests.test_prism_metrics, tests.test_prism_issue_224_landing, the full `unittest discover -s tests` suite (3107 tests, 2 pre-existing skips), compileall, and diff --check. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01EgdzHvd8NL26qjYnmJncAn * Attribute the payout-window and prior-balances reads (#224) The accepted-preview latency alert fired while the coordinator stayed healthy, and nothing on /metrics said which stretch of the landing owned the sample. Four ledger reads sit in that stretch, and each can be slow for either of two unrelated reasons: coordinator-local admission behind an accounting write, or PostgreSQL execution. One duration covering both cannot tell an operator which -- the same ambiguity that misdiagnosed #211 on the enumeration path before it was split. Route those four through the attributed read helper the enumeration already uses, under the operation names Wave 0 froze: snapshot_at_job_issue -> payout_window_snapshot snapshot_between_job_issues -> payout_window_delta current_prior_balances -> current_prior_balances prior_balances_after_pool_block -> prior_balances_after_pool_block The helper was hard-wired to the read semaphore, so it is generalized over the gate the caller already takes. That is the invariant most at risk here: the prior-balances reread stays on the *writer lock*, where it must observe the fenced state a payout mutation leaves behind, and the other three stay on the bounded read slot. gate_name is the string _operation_gate would have been given, so an expired admission still names the primitive it was waiting for. No SQL text, result parsing, thread, connection, retry, lock nesting or label value changes. The legacy qbit_prism_prior_balances_* aggregate is left exactly as it was -- one duration over admission and execution, recorded only on success. It duplicates part of the new per-operation sample on purpose: an operator comparing that existing series across this upgrade must not find its meaning quietly redefined. Operation names stay bare string literals at each call site because tests.test_prism_accepted_preview_telemetry harvests them from this file's source to fail on a name outside the closed vocabulary; imported constants would leave that check with nothing to read. tests/test_prism_issue_224_ledger.py drives the new module against real Lock/BoundedSemaphore primitives behind a recording wrapper and a scripted clock, so recorded seconds are exact and a released gate is a fact about a primitive rather than a counter. It pins the fixed operation per read, the gate each read keeps (proved from both sides, including a refusing writer gate the read-slot reads never touch), gate wait separated from execution, admission expiry as a gate timeout with no execution sample, statement expiry as an execution timeout, release-and-preserve on failure exits, a differential against the pre-change admission body for byte-identical SQL and results, and the fold that keeps an out-of-contract name from growing a series. The two existing tests that call the helper directly gained the now explicit gate arguments; their scenarios are unchanged. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01GQRKR6tm627cizwNTZ81h2 * Prove the #224 reread over a dense reconcile cadence and in PostgreSQL Wave 1 answers a confirmed payout mutation with a live prior-balances reread over the still-exact accepted-share window instead of the O(window) oracle rescan. Its tests pin one mutation at a time; the alert hour was a dense accepted-tip cadence. tests/test_prism_issue_224_cadence.py drives consecutive trusted passes through the real coordinator adapters over one frozen-anchor window while the carry aggregate moves through a maturation, an inactive marking and a reactivation, interleaved with quiet passes and with the post-confirm and landing entry points. Per pass it records the preparation intent, the ledger artifact, the published balances and the ledger reads, and pins them against each other and against the fixed-cardinality telemetry: mutating passes forward force_prior_balances_read=True with force_full_window_rescan=False, reuse the very armed window object and publish the live balances after exactly one attributed current_prior_balances read; quiet passes prepare, read and publish nothing; the rendered families carry only closed-vocabulary labels. A lost maturation response mid-cadence still fails closed to the reconcile_invalidation full rescan. The Wave 1 fixture gains an optional ledger parameter so the cadence module reuses its setup. test/test-prism-postgres-ledger.sh captures the exact identity of qbit_share_ledger (every row in sequence order with ctid and xmin, plus the sequence allocator) before block 66's reconciliation-owned mutations and requires it back after each inactive quarantine, maturation sweep and reactivation, while the owed balance proves each transition landed. Its PASS summary names the invariant. Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_01SSg1LisJpnfCju56WjZrpM * Add the #224 operator validation contract for the accepted-preview boundaries Issue #224, Wave 3 closeout. The integrated Wave 0/1/2 stack was reviewed end to end from origin/2.x.x: confirmed reconciliation mutations request force_prior_balances_read=True with force_full_window_rescan=False and reuse the armed window object; the lost-response path still forces the reconcile_invalidation full rescan; withdrawal, late-visible append, anchor/weight regression, daemon loss and the periodic self-check keep their reasons; landing attribution opens only at definitive acceptance, is popped by the first published/degraded publication under the same lock that tombstones the stamp, and a retry or republication after that reaches the process-wide family only; every rendered label is drawn from a closed vocabulary; the attributed ledger helper keeps each read on the gate it already took with identical SQL, retry and release ordering; the flight, fencing and replay paths are wrapped, not changed; and the dense cadence and PostgreSQL identity proofs assert the successful and ambiguous paths against distinct balance values and tuple identity. No production or test defect was demonstrated, so no code changes. docs/prism-overload-alerts.md gains the operator contract the rollout needs to judge the 4 s warning and 5 s child-wait boundaries from the shipped metrics: the verified signal inventory (histogram le="4"/le="5" labels, the four attribution families, the ledger operation split, the artifact event counter, the legacy carry aggregate and the backlog gauges), the two bounded structured log lines that carry block identity, bucket-derived rolling-hour breach counts with a subquery form for whole soaks and the reason percentiles cannot prove a count, phase / caller / step / rescan-reason / admission-versus-execution breakdowns, the log-correlation recipe, the pre-deploy baseline and post-deploy testnet cadence and Union mainnet soak checklist, the #224 pass/fail criteria, and a fail-closed rollback/escalation decision that never raises the alert, extends the child wait, weakens exactness or disables the conservative rescan. It distinguishes the deliberate reconcile_invalidation fallback from a successful balance-only reread and notes open #207's separate window_daemon_state_lost scope. docs/prism-ledger-ops.md now lists the four #224 read operations and the writer-lock gate of current_prior_balances beside the existing read-slot enumeration paragraph, and points at the new section. Checks: the two focused unit groups (135 and 595 tests), compileall, bash -n on the PostgreSQL gate, git diff --check, and the real Docker PostgreSQL ledger gate (all five gates PASS, including share-ledger-identity=inactive+reactivate+mature). Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_014FMYTWWK1au9ptcu2sB4fq * Keep the #224 landing attribution honest under replay and writer stalls Codex review of #230 found two ways the attribution reports something the landing did not do. The publish span allocated an in-flight record for every publication, including the startup replays and chain-proven candidates that publish without an in-process acceptance stamp. The observer returns before it can pop anything for those, and the span's exit only cleared its token, so each one stayed in the bounded map as a publication permanently in flight -- and a large enough replay evicted live landings from the cap before their diagnostics were ever emitted. The record is now allocated only for an open interval, and an interval that closed under the publication because its stamp aged out of the shared cap drops it. The phase family still takes its sample either way. lane_wait closed after _mark_block_candidate_attempted, so the landing's own PostgreSQL write was charged to queue time, and a marker that timed out kept charging its retry backoff there too: the one stretch that exonerates the landing grew precisely when the writer was degraded. Both queued paths now close the lane before the marker write, leaving that write in the unattributed remainder the operator guide already describes. * Close the landing lane on both sides of the node offer The previous commit moved lane_wait's close above the attempt marker, but in doing so moved it above _node_submission_for_candidate_or_retained as well -- and that call is where a fresh candidate gets its definitive acceptance stamp, inside _submit_block_candidate_to_node. The close then found no open interval and returned, so every first-pass queued landing published a diagnostic with no lane_wait sample at all. A zero sample and a missing sample are indistinguishable in the per-landing diagnostic, so the loss would only have shown up as the family count trailing the accepted block count. Both queued paths now close the lane on either side of the offer. _close_accepted_landing_lane_wait is already once-per-open-interval, so the second close is a no-op when the first recorded: a candidate carrying a retained acceptance keeps the earlier, tighter sample that excludes this landing's own work, and a fresh candidate reports the zero queue time it genuinely had. Both closes stay above the attempt marker, so that write is still outside lane_wait. The ordering guard now also requires a close after each path's last node offer, which is what fails against the previous commit. --------- Co-authored-by: Claude Fable 5.1 * Stop forcing zero stale grace on mainnet so prior-tip shares are credited, not rejected (#225) * Stop forcing zero stale grace on mainnet so prior-tip shares are credited, not rejected The production gate pinned PRISM_STRATUM_STALE_GRACE_SECONDS=0 on mainnet (coordinator startup, scripts/check-env.sh, scripts/prism-self-check.py). With a zero window every in-flight share on the just-replaced tip is rejected with Stratum error 21 (stale-job / unknown-job) at each block, and miner firmware counts any pool-side error as a reject: "stale" on the miner side is only ever derived client-side, so the only way the pool avoids a reject is to accept the share. That is exactly what the stale-grace credit policy does (ack, ledger with credit_policy=stale-grace, never block-submit), and the classification logic itself was verified against the existing tip-flip tests. Mainnet now follows the same bounded-grace rule as every other production chain (default 3 s; an explicit 0 remains valid). The pin was added in #19 citing audit-bundle v1.1 verifier compatibility; this release's verifier accepts v1.1, and the docs now say to keep external mirrors and verifiers on a release that does too. No hot-path code changes. On this release line the coordinator-side gate lives in lab/prism/coordinator_config.py rather than lab/prism/prism_coordinator.py, and its regression test lives in tests/test_prism_coordinator_config.py. Co-Authored-By: Claude Fable 5.1 * check-env: validate the exported stale-grace value instead of dropping the check Restore the ENV_ preservation for PRISM_STRATUM_STALE_GRACE_SECONDS so the doctor validates the value Compose will hand the coordinator rather than the .env.example default, and replace the removed mainnet-only equality check with a lane-level non-negative numeric check that runs in every mode. Without this, an exported value such as "not-a-number" passed the doctor and then crashed the coordinator at startup in env_nonnegative_float(). Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_01K8uGJEY2npTQuJPwS9Qg6M * prism-self-check: parse stale grace with the coordinator's float semantics The coordinator accepts any finite non-negative float for PRISM_STRATUM_STALE_GRACE_SECONDS and check-env now validates the same shape, but the self-check still parsed the value with int(), so a valid fractional window such as 0.5 failed `make prism-self-check`. Parse a finite non-negative decimal instead. Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_01K8uGJEY2npTQuJPwS9Qg6M * Validate stale grace with the coordinator's float conversion in the doctor and self-check A hand-written regex cannot match env_nonnegative_float(): it rejected valid runtime syntax such as 1e-1, .5 and 3., and accepted an integer large enough to overflow to infinity at startup. Both check-env.sh and prism-self-check.py now convert with float() and require a finite, non-negative result, exactly as the coordinator does. Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_01K8uGJEY2npTQuJPwS9Qg6M * Address the rebase review: doc contradiction, doctor prerequisite, mainnet coverage Four follow-ups from reviewing the rebase against this release line. Docs: the ledger-ops share-accounting paragraph still told auditors to upgrade "before operators enable stale-grace crediting", which contradicts the deployment paragraph in the same document now that the window is on by default. It also, along with two other lines, named the v1.1 bundle schema without saying it is the logical schema; the on-disk body is the v2 container carrying logical_audit_bundle_schema once share segmentation is on. Both are now stated precisely. check-env: validating stale grace with python3 made the interpreter a hard prerequisite for every prism-lane doctor run. Production already required it for the difficulty validation, but lab bring-up did not, and README lists Python 3 only for the unit-test helpers. Keep the hard failure in production and skip the check with a doctor notice otherwise, with a test covering both branches over a python3-free PATH. Tests: the production environment fixtures still pinned the retired zero, so every full-environment production test proved the old contract, and the mainnet positive test asserted only that the old message was gone while its non-zero exit came from unrelated missing inputs. The fixtures now carry the bounded default, and the mainnet test runs a complete production environment and asserts a clean exit for 3, 0.5, and an explicit 0. --------- Co-authored-by: Alex Co-authored-by: Claude Fable 5.1 Co-authored-by: Robert Clarke * Export always-on heap and component-cardinality telemetry (#226 part 1) (#232) Issue #226 recorded a coordinator growing about 390 MB/h of RSS across 45 h with a memory map shaped like glibc per-thread arenas over 64 threads, and nothing on /metrics could say whether that was live Python objects, pymalloc fragmentation, or glibc arena fragmentation. This lands the cheap, fixed cardinality instrument for that question, and the component gauge list #185 asked for, ahead of the census and allocator work in the second part. lab/prism/process_telemetry.py (new) collects sys.getallocatedblocks(), gc.get_count() and gc.get_stats() per generation, threading.active_count(), and glibc mallinfo2 arena/in-use/free/mmapped bytes through ctypes. The mallinfo2 binding is resolved once per collector and an absent or broken symbol is remembered without logging; it renders as an availability gauge at 0 with every byte gauge at -1, never as zero bytes, so musl, macOS, and pre-2.33 glibc read as unavailable rather than empty. The older int-typed mallinfo is deliberately not used as a fallback: its 32-bit fields wrap on a multi-gigabyte heap, which is exactly this issue's heap. The collector never walks the heap: no gc.get_objects(), no tracemalloc. lab/prism/metrics.py renders eleven process families (only the GC families carry a label, the closed three-value generation set) and two component families, qbit_prism_component_entries and qbit_prism_component_bytes, over a closed component label set covering the payout window and its serialization cache, job contexts and the bundle cache, the evicted-job graveyard, the candidate lanes and every hash-keyed candidate registry, reconcile flights, the pending-share floor, and the daemon mirror. Component reads are direct attribute loads on the shipped owners so a renamed field fails loudly. There is no hashrate-rollup gauge: the rollup folds in PostgreSQL and retains no in-process buffer. The cached payout window is accounted exactly once so the byte family can be summed for attribution. Its two backings are mutually exclusive, so a mirror-backed window (the Rust path, which is what mainnet runs) reports only under daemon_window_mirror_*, and a page-backed window only under payout_window_*. Which pair is non-zero is how an operator reads the active backing. share_window_serialization_spool is labelled in HELP and in the docs as an on-disk temporary file rather than resident bytes, so it is not mistaken for RSS. lab/prism/coordinator_config.py adds PRISM_MALLOC_TELEMETRY (default on, strict boolean validated at startup through LifecycleConfig) gating only the mallinfo2 call, which walks glibc free lists under each arena lock and is the one reading that is not a plain counter load. The renderer resolves it once at construction and falls back to the shipped default if the value has become invalid since startup: a telemetry switch must not be able to take the coordinator down from a metrics thread. compose.yaml passes it through and .env.example documents it, so the compose stack can turn it off with a restart. tests/test_prism_metrics.py pins the exact family sequences, the unavailable rendering, the shipped-owner reads, the closed label set under growth, the no-double-count property stated as a sum, and the no-heap-walk contract, and extends the frozen parity reference in the same commit. tests/test_prism_process_telemetry.py pins the collector and the switch. docs/prism-capacity-readiness.md documents the families, the glibc-only statement, the switch and how to set it in each deployment, the single accounting of the window, and what to read first when RSS climbs. Co-authored-by: Claude Fable 5.1 * Attribute in-process lease-monitor stalls and decide the lateness response (#227) (#233) `PrismWriterLeaseMonitorWakeDelayHigh` fired on mainnet with a 0.648 s monitor wake delay against a 0.50 s scheduler slack, on a host that was 93% idle with PSI near zero. The stall was in-process, and none of the existing telemetry could say so: `guard_sql` is measured client-side, so a GIL stall between PostgreSQL answering and the heartbeat thread resuming was booked as database time (0.714 s, above the 0.50 s server-side statement timeout), while `scheduler_delay` never exceeded 22 microseconds in 645k attempts. Attribution that survives a GIL stall. Both guard statements now return a server-side `clock_timestamp()` delta, so `guard_sql` splits into server execution and client resume and the residual is charged to the scheduler. The split is clamped with `min(statement_span, server_seconds)` so a clock skew cannot manufacture negative client time. The attribution recheck re-runs the same SQL, so a two-statement attempt reports both. A distribution instead of a lifetime high-water mark. The old gauge never reset, so every stall below the record was invisible and the alert could not fire again. Added a fixed-bucket histogram of monitor wake delay, counters of wakes at or above 0.5, 0.8 and 1.0 times the slack, a rolling-window maximum, and the age of the current record. The original gauge is kept for dashboard continuity. The counters are what let the alert be re-expressed on a counter rather than a never-resetting gauge. GC and stall instrumentation. `gc.callbacks` records pause duration per generation as a histogram plus a last-pause gauge, and a rate-limited stall probe samples running threads when a wake exceeds half the slack. The lateness response, and why. Lateness beyond the slack is accepted rather than converted into a tightened server-proven cap for that beat. Tightening would hard-exit healthy coordinators under a transient stall — issue #212 — without moving the exit any earlier on the beat that actually breached, and every effect that could escape during the uncovered interval is already fenced at the row or behind a synchronous exact-session verification. What changes is that the breach is no longer invisible: it is counted, its overrun past `max_guaranteed_monitor_lateness` is kept, one rate-limited structured warning names it, and a stack sample says what held the interpreter. The reasoning is recorded in the `writer_lease_timing` module docstring, which previously presented `cap + monitor_interval + scheduler_slack + exit_margin = 1.95 < 2.00` as if the slack term bounded reality; production exceeded it. The instrument stays off the exit path. Accounting one wake is O(1) arithmetic on the monitor thread; everything that walks frames, formats text or writes a log line is returned as a closure that `monitor_loop` runs only after the exit decision, and drops entirely when the decision is to exit. A measurement that extended the envelope it reports on would consume the very budget it exists to protect. The stack sample walks frame objects directly and never touches `linecache`: `traceback.extract_stack` would stat every file and read every source line for text this sample does not print, and even `StackSummary.extract(lookup_lines=False)` still calls `linecache.checkcache` per file. Safety constraints from #212 are preserved: the heartbeat is not disabled, no failure budget was raised on its own, and PostgreSQL durability is untouched. Co-authored-by: Claude Fable 5.1 * Add a bounded heap census, allocator control, and the RSS soak bound (#226 part 2) (#234) Part 1 landed the always-on telemetry that discriminates retention from fragmentation. This adds the operator-triggered instrument that names a retention site when there is one, puts the glibc allocator under deliberate control, and turns "RSS is bounded" into a check rather than an observation. The census. SIGUSR1 produces a bounded report: a type histogram over gc.get_objects(), optionally tracemalloc top-N by file, written to a file and never reachable from a request path. It is off unless PRISM_HEAP_CENSUS is set at startup, because tracemalloc has a standing cost and a full heap walk on a 17.7 GB process must not be reachable by accident. The walk is bounded by top-N, by an output byte cap with payload halving, and by a wall-clock cap that stops mid-walk and says so. The signal handler only writes to a self-pipe; the walk runs on the census worker thread. The allocator. MALLOC_ARENA_MAX defaults to 2 in the image, argued in the Dockerfile, and is passed through compose along with the census and trim switches so an operator can change it without rebuilding. malloc_trim is exposed on SIGRTMIN+1, off by default, rate-limited, and reports its arena and RSS deltas through part 1's gauges. Where the platform has no SIGRTMIN the trim signal is simply not armed. The bound. An RSS soak check reads a sample series and fails when RSS exceeds a stated multiple of baseline, with the operator command exiting 0 on pass, 1 on fail and 2 on bad input, so a soak asserts the bound instead of a human reading a graph. What the census found. Locally it names nothing retained: sys.getallocatedblocks() returns to its pre-storm count once the storm's objects are released (169,729 before, 169,653 after), and the type histogram is the coordinator's own bookkeeping. The evidence points at glibc per-thread arena fragmentation instead, and the storm instrument reproduces the mainnet shape exactly: a 64-thread allocation probe takes the process from 5 to 64 arenas and from 9 to 72-73 anonymous regions in the 4-64 MiB band -- the same band the mainnet capture reported 658 of 943 regions in -- while malloc_in_use_bytes stays flat at 10 MB. That is the "RSS rises over a flat in-use" signature, so no retention fix is invented; the allocator settings are the fix. Whether retention also sits underneath the fragmentation on the mainnet process can only be settled by a census on that process, which is what this ships. Test-portability and correctness fixes from review: SIGRTMIN is Linux-only and was evaluated at class scope, making the module unimportable elsewhere; the rss-sample test asserted the Linux /proc outcome unconditionally; census file retention sorted on (mtime_ns, name) with an unpadded sequence, so within one mtime tick the newest report could be deleted -- the sequence is now zero-padded and the ordering no longer depends on filesystem timestamp granularity; the malloc_info bin parser counted fastbin rows twice; and the type histogram overwrote rather than merged same-named types, which would have hidden the very retention the census hunts. Co-authored-by: Claude Fable 5.1 * Bound payout-window full rescans and keep the daemon prepared after a re-center (#228, #207) (#235) Half of mainnet payout-window builds were `reconcile_invalidation` full rescans of about 60k shares taking 3-13 s on the publication path, holding tip-refresh waves for 5-15 s and putting post-block work in front of miners late at roughly every other block. 489 of the 503 full rescans in 48 h carried that reason, against 8 reorgs and 55 accepted-block confirmations in the same window. The trigger. The sampled 10.2 s build reported `delta_rows` 0 and `touched_pages` 0: a forced full rescan that found nothing to change. No statement in the tree writes `qbit_share_ledger` -- it takes INSERT only -- so the accepted-share window at a fixed anchor is a pure function of append-only rows, and a payout mutation moves balances but can never move the window. PR which asks for a prior-balances reread rather than the oracle; this change adopts that vocabulary rather than adding a second spelling of it. What remains, and is fixed here, is the sticky invalidation cause. The `(generation, tip, cause, invalidated)` tuple is captured without being cleared, so every preparation at one source generation re-read a withdrawal's cause and forced the oracle again. The cause cannot simply be consumed: it is a persisted uncertain-state marker that `block_finalization` reads later to decide whether to republish. So the cause stays readable and the *entitlement to act on it* moves into the reserved-source object the reserver holds, where no pruning rule can take it away; later captures inherit the tuple for identity only. Publication cost is bounded. A tip-refresh wave no longer waits on a full rescan when a valid recent window exists: the routine path takes the prepare lock non-blocking and reuses the armed window inside a stated staleness bound, following the shape found-block publication already used for the same reason. `window_build_seconds` is now attributed by phase -- ledger read, record conversion, daemon prepare or upload, and lock waits -- over closed phase and outcome label sets, observed in a `finally` so an aborted or failed build reports its phases too. A build slow enough to lose its generation race is exactly the one an operator needs the split for. oracle without preparing the daemon, so the next advance answered `needs_full` and rescanned again; and that path reported `window_daemon_state_lost`, making a deliberate re-center indistinguishable from a genuine eviction in production telemetry. The daemon is now prepared for the adopted digest, and the deliberate re-center has its own reason. Real eviction still reports `window_daemon_state_lost` and follows the existing recovery path. Byte-exact payout-window parity and the real-daemon gate stay green, as do the 5/8 floor, the 2x ceiling and the >1.25x oversized re-center threshold. Co-authored-by: Claude Fable 5.1 * 2.x.x: replace PRISM builder pipe polling with readiness waits (#237) * Replace fixed builder pipe polling sleeps with bounded readiness waits (#236) The --serve daemon transport and the one-shot writer drive the builder's pipes non-blocking so cancellation, supersession and the absolute deadline are checked between syscalls, and they slept a fixed 20 ms after every EAGAIN. On Linux that is one sleep per drained 64 KiB pipe capacity, so a multi-megabyte window exchange paid orders of magnitude more in idle sleeps than in copying: measured here, the 90 MiB request write of a cold 210k-share prepare_window spent 26.9-27.6 s in 1,327-1,362 sleeps, and the raw helpers moved 3-6 MiB/s. A small readiness waiter now takes the sleep's place. One instance serves one transport operation, registers the descriptor lazily (an operation satisfied from buffered bytes or by syscalls that never block costs no selector) and closes the registration on every exit path. Each wait is bounded by BOTH the remaining absolute deadline and the retired sleep's 20 ms cancellation-check cadence, so a waiting build wakes the moment the pipe can progress and still re-checks its control signals on the old schedule when nothing happens. EINTR is retried after the control checks without waiting; readiness the selector reports may be spurious and is resolved by retrying the syscall; a descriptor the selector refuses degrades to the old bounded sleep. Nothing else about the loops changes: exact raw-payload lengths, bytes buffered after an envelope newline, short reads and writes, EOF and broken-pipe classification, exactly-once crash accounting, timeouts that leave the failure to the one-shot fallback, supersession before and after a fault, Linux splice with the portable fallback, and spool poisoning without ever replaying a partially sent prefix. The line reader also stops rescanning the whole buffer for the newline after each chunk. Measured with tests/perf/window_builder_pipe_latency.py against the baseline module extracted from e59bda3 and the real release daemon: the raw helpers reach 30-215 MiB/s with byte-exact digests; a cold 210k prepare_window drops from 39.8-43.8 s to 12.2-13.8 s with the request write at 0.13-0.15 s and identical digests and canonical bytes. The residual is elapsed daemon-side time plus the whole-request encoding PR 2 targets, neither attributed here; see the report for what the numbers do and do not establish. Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_012HydbXko45T99GQ4yi6yMB * Make the transport tests' blocked-wait assertions deterministic (#236) Full discovery failed once in the new one-shot writer test: a slowly paced child does not guarantee EAGAIN while json.dump emits small fragments, so "the writer blocked at least once" was a scheduling assumption, and the same assumption sat under several real-pipe tests. Every remaining wait-count assertion now holds by construction. Real-pipe and one-shot peers are gated: a peer thread waits on an event, a child process polls for a marker file, and the recording waiter releases them on the transport's first blocked wait, so at that moment the pipe is empty (reads) or full (writes) whatever the scheduler did. The scripted timeout tests drive the module's clock through a hand-advanced fake instead of letting 20 ms of real time elapse, and assert the exact wait count and remaining slack. Where no gate is possible -- a real deadline on an idle pipe, the fake daemon's parse time -- the wait count is no longer asserted and the byte, framing, classification and cleanup assertions remain. Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_012HydbXko45T99GQ4yi6yMB * Fall back to the bounded sleep when the selector cannot poll a registered pipe (#236) A selector can accept a builder pipe's registration and still refuse to poll it -- SelectSelector over a Windows pipe handle is the known case, a descriptor torn down under the registration another. The readiness waiter only guarded registration, so a select-time failure propagated out of the transport loop instead of reaching the bounded-sleep fallback the waiter documents (PR #237 review). A failed poll now releases the selector exactly once, marks the descriptor unwatchable for the rest of the operation so no selector is rebuilt on later waits, and finishes the same wait with the sleep over whatever budget the failed poll left: the remainder of the cancellation slice, and never past the absolute deadline, so a wait that already spent its budget inside the failed poll returns at once. EINTR from the poll stays what it was, a plain wakeup that keeps the selector and hands control back to the loop's cancellation and deadline checks. Regression coverage uses a deterministic selector fake that registers and then raises at select, with a hand-advanced clock: single release and no recreation across waits, the remainder-of-slice and deadline bounds, a deadline spent inside the failed poll, EINTR kept distinct, and a line read completing over the fallback through the real helper. Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_012HydbXko45T99GQ4yi6yMB * Drive the poll-failure detach test from the fake clock (#236) The detach-once test asserted two positive sleeps against the real clock, so a >20 ms deschedule during the failed poll would have made the waiter's correct no-sleep return look like a failure. It now runs on the existing fake-clock helper and asserts the two full-slice sleeps exactly, the same way its sibling budget tests already do. Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_012HydbXko45T99GQ4yi6yMB --------- Co-authored-by: Alex Co-authored-by: Claude Fable 5.1 * Bound share-window serialization and lazy parsing on every path (#236) (#239) Every whole-window JSON call on the payout-window paths -- the daemon prepare_window request, the audit-builder compact tail (spool, in-memory fallback and one-shot), the digest of a freshly read share list, the daemon mirror's lazy parse, the found-block candidate identity digest and the one-shot canonical payload -- is replaced by batched encoding or record-at-a-time parsing that produces byte-identical output while bounding any one C call to 512 records or one 64 KiB decode chunk, so the writer-lease monitor thread can run between batches instead of waiting behind a whole-window json.dumps/json.loads. - lab/prism/share_json_stream.py: new leaf helper (batched array/object encoders, streamed canonical digest, ShareArrayJsonSequence view, bounded list release). - share_ledger.py: page encoding through the helper; _walk_canonical_items (incremental UTF-8 decode + raw_decode per record, shared key strings, strict layout checks); DaemonShareJsonSequence parses record by record and publishes only after the count check; block_candidate_identity_sha256 streams shares_json. - bundle_compiler.py: compact tail as bounded chunks (compact_tail_chunks) written identically by spool, daemon in-memory fallback and one-shot; prepare_payout_window streams the request and retires the daemon on a half-written line; the canonical one-shot payload streams its arrays. - payout_state.py / job_bundle.py: share-list digests go through the streaming view; transient per-share lists released in bounded slices. - metrics.py: compact-json byte accounting via compact_json_bytes. - tests: byte-parity, malformed/chunk-boundary parse, fault injection (unencodable record mid-request, daemon death mid-request), candidate digest parity, spool/in-memory equality. - tests/perf: --latency-probe mode with a lease-monitor-cadence lateness thread, GC attribution and the real daemon round trip; report section 9 records the shared-host measurements and the Rust paging finding. Claude-Session: https://claude.ai/code/session_01AFvDqz6xwHRWxCWwJPiXvT Co-authored-by: Claude Fable 5.1 * 2.x.x: bound PRISM payout-window database decoding (#238) * Bound payout-window database decoding to per-row batches on both backends (#236) The coordinator's payout-window snapshot and delta returned one json_agg value the size of the whole window, and decoding it was a single C call that never released the GIL: at the incident's ~210k shares the lease monitor thread went unscheduled for the whole decode. Both reads now project one JSON object per row from the unchanged selection CTEs, and each backend converts the rows in 512-record batches, rechecking the caller's absolute operation deadline and stamping liveness between batches. - LedgerSqlPort gains run_json_rows; _NativePostgresClient implements it over fetchmany with run_json's connection borrowing, SET LOCAL deadline transaction (committed before decoding starts), OperationalError translation and once-only retry that discards partial rows and re-runs the complete SELECT inside the original deadline. - The psql backend spools stdout to an unnamed temporary file, validates exit status and stderr exactly as _run_sql does, kills and reaps a timed out child, and parses the spool one line per row. _psql_invocation is shared by both paths so flags, session guards and deadlines cannot drift. - snapshot_at_job_issue and snapshot_between_job_issues keep their SQL selection, ordering, weighted crossing row, eligibility predicates, returned record-list API, read-slot admission and #224 attribution (payout_window_snapshot / payout_window_delta); only the outer projection and the decode path changed. Mutations and lease statements keep run_json unchanged. - Legacy subclass seams (_run_json and _run_sql overrides) and the concurrency-harness fake port are updated explicitly so no backend bypasses the row path by accident. - Every decode path checks the statement and operation deadlines before decoding starts, between batches, and before publishing, so a trailing partial batch or a result shorter than one batch cannot return after the budget. Tests: 25 new unit tests (real fake-psql child process for the subprocess path; fake driver for the native client; virtual-clock deadline and progress checks; SQL shape); the issue-224 differential control gains a row-result twin; both PostgreSQL gates compare every window variant against the pre-change json_agg oracle and the native gate proves a share appended mid-decode is absent from the in-flight snapshot. tests/perf/window_ledger_decode_latency.py measures baseline vs patch at 210k and 400k shares with a wake-lateness monitor beside the read; results in tests/perf/window_ledger_decode_latency.md (shared-host figures: worst monitor wake lateness during a read 750-950 ms -> 176-260 ms at 210k and 1.4-1.6 s -> 342-484 ms at 400k on both backends, no wake-up over 500 ms with the patch, server time and peak RSS down; the residual stalls are gen-2 GC collections over the returned records, attributed with a probe and left as a coordinator-level follow-up). Refs #236 (does not close it: PRs 2 and 3 of the plan and the dedicated environment release gates remain). Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_01WkTwwTmMgLDe9WDY3rnrmh * Bound share-window serialization and lazy parsing on every path (#236) Every whole-window JSON call on the payout-window paths -- the daemon prepare_window request, the audit-builder compact tail (spool, in-memory fallback and one-shot), the digest of a freshly read share list, the daemon mirror's lazy parse, the found-block candidate identity digest and the one-shot canonical payload -- is replaced by batched encoding or record-at-a-time parsing that produces byte-identical output while bounding any one C call to 512 records or one 64 KiB decode chunk, so the writer-lease monitor thread can run between batches instead of waiting behind a whole-window json.dumps/json.loads. - lab/prism/share_json_stream.py: new leaf helper (batched array/object encoders, streamed canonical digest, ShareArrayJsonSequence view, bounded list release). - share_ledger.py: page encoding through the helper; _walk_canonical_items (incremental UTF-8 decode + raw_decode per record, shared key strings, strict layout checks); DaemonShareJsonSequence parses record by record and publishes only after the count check; block_candidate_identity_sha256 streams shares_json. - bundle_compiler.py: compact tail as bounded chunks (compact_tail_chunks) written identically by spool, daemon in-memory fallback and one-shot; prepare_payout_window streams the request and retires the daemon on a half-written line; the canonical one-shot payload streams its arrays. - payout_state.py / job_bundle.py: share-list digests go through the streaming view; transient per-share lists released in bounded slices. - metrics.py: compact-json byte accounting via compact_json_bytes. - tests: byte-parity, malformed/chunk-boundary parse, fault injection (unencodable record mid-request, daemon death mid-request), candidate digest parity, spool/in-memory equality. - tests/perf: --latency-probe mode with a lease-monitor-cadence lateness thread, GC attribution and the real daemon round trip; report section 9 records the shared-host measurements and the Rust paging finding. Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_01AFvDqz6xwHRWxCWwJPiXvT --------- Co-authored-by: Claude Fable 5.1 * 2.x.x: bound Rust payout-window paging memory (#241) * 2.x.x: bound PRISM payout-window paging allocation (#240) `PayoutWindow::from_full_snapshot` paged the retained records by repeatedly `split_off`-ing the tail from one owning Vec and storing the head as each page. `Vec::split_off` keeps the original allocation, so page k retained backing capacity for every record still unpaged when it was cut (N, N-P, N-2P, ...) and the tail was copied again on every iteration: O(N^2/P) retained memory and copying -- 9.6 GB of resident builder memory and a 12 s response at the ~210k-share mainnet window of #236, and a SIGKILL at 400k. Page by moving every retained record exactly once out of the snapshot's own buffer (now filtered in place with `retain`) into a vector allocated for exactly the records that page holds, so aggregate page capacity is the retained count and the work is linear in it. The `advance` delta loop had the same `split_off` shape and is paged the same way, and the merged boundary page is allocated for its final length. Eligibility on both timestamps, ascending share_seq order, the whole-share crossing-row cutoff, duplicate and zero-difficulty rejection, u128 overflow handling, canonical fragments and digests, the advance byte surgery and immutable page sharing across generations are unchanged. Structural regression tests assert the bound directly -- no page vector's capacity above page_size, aggregate capacity at most one page's worth per page -- across page sizes 1, 3, 7 and 512 at empty input, one record, one short of a page, exact single and multiple boundaries, one past a boundary and long partial tails; plus the weight cutoff, unsorted input, page-size and weight validation, page-size-independent items/digest/payout totals, Rc page sharing and a full-rebuild digest match across an advance with a multi-page delta. Appended to the previous implementation, four of them fail at the capacity assertion; the other 17 window tests pass on both. The daemon's prepare_window success envelopes gain an additive `metrics` object (request parse, fold/advance and canonical serialization seconds) so a measurement can separate Rust processing from pipe transport; the coordinator's parser ignores it, and the rust-daemon parity gate passes against the rebuilt binary. tests/perf/window_paging_allocation.py is the bounded measurement harness: old and new binaries through the real --serve protocol over blocking pipes (full, small advance, multi-page advance, re-center, advance) with RLIMIT_AS on the daemon and a MemAvailable guard, VmHWM peaks, per-phase timing split into transport and the daemon's own metrics, digest self-checks and the shipped Python fold as an oracle. Its report follows in tests/perf/window_paging_allocation.md. Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_01Vb3bGmWxUvZuRReN3Sfkpy * 2.x.x: harden window-paging benchmarks and document measured bounds Add daemon deadlines and cleanup coverage, plus host and CPython 3.14 image measurements for issue #240. Keep raw local evidence outside tracked history. * Format Rust paging additions * Clarify paging memory units and incomplete baseline evidence * 2.x.x: isolate the #240 advance-paging case and label reconstructed peak RSS window.rs: a focused test folds one exactly full page (4 records at page size 4, within bound even under the previous paging) and advances it by a 23-record delta at a weight that expires nothing, asserting every record and fragment page stays within page_size, the full base page is shared rather than rebuilt, the previous generation is unchanged, and the result matches a full rebuild with exact byte surgery. Against the base implementation it fails on its own at the advance loop ("page 1 of 7 retains capacity 23 for 4 records") while the snapshot page passes, so the delta split_off defect is pinned independently of the full-snapshot one; the broader sharing/expiry test is retained. The paging comment now cites the prior synthetic 9,613 MiB measurement instead of an approximate figure in a different unit. window_paging_allocation.py: the daemon outcome records where peak_rss_mb came from -- the kernel VmHWM read live at close (vmhwm), the watchdog's read before a kill (vmhwm_before_kill), a lower bound reconstructed from earlier per-phase VmHWM reads and the sampler's running maxima after the daemon exited on its own (lower_bound), or unavailable. Lower bounds render as ">= N" beside a peak-source column; the summary header no longer calls every figure VmHWM. Numeric data and existing JSON keys are unchanged; peak_rss_source is added. Every published measurement exited cleanly and stays a kernel VmHWM. Harness tests add deterministic cases for resolve_peak_rss provenance and exact-vs-lower-bound rendering, a live-at-close VmHWM read, and a fake daemon that exits after its handshake (never labelled VmHWM); the stall and reaping tests are retained. Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_01Vb3bGmWxUvZuRReN3Sfkpy * Preserve benchmark handshake timeout outcomes * Label live daemon memory observations as lower bounds * Check memory headroom before benchmark fixture and oracle work --------- Co-authored-by: Claude Fable 5.1 * ci: shard lint, compile, and test checks (#243) * chore: prepare 2.0.0 release (#245) Align the workspace package versions and add release notes covering the complete 2.x.x delta since v1.1.0, including API cutover and ledger migration requirements. * fix: validate PRISM startup and forward replica retries Require the advertised Stratum URL before starting the pool and public API, document the quickstart requirement, and honor replica bootstrap retry overrides. Cover missing and blank URL rejection plus rendered retry defaults and overrides. * fix: isolate public database commands and forward PRISM controls * 2.x.x: release stall-probe frame references after sampling (#248) * fix: release stall-probe frame references after sampling * test: isolate stall-probe formatting failure injection * fix: forward PRISM rollup, deadline, and cleanup controls * fix: derive replica credentials and validate storage before startup * fix(prism): release completed work and avoid consumer traceback cycles (#250) * fix(prism): release failed cache and address-validation flights (#252) * fix(prism): release cancelled job-build traceback state (#253) * fix(prism): allow one-candidate startup replay pages (#256) * fix(prism): allow one-candidate startup replay pages Add an opt-in cursor page size for startup and forced ancestor replay while preserving complete enumeration, retry deduplication, and accounting gates. Document the mitigation and its limits while issue #255 remains under investigation. * test: isolate environment checks from runner Docker Use a controlled Docker fixture for policy tests so runner daemon startup cannot exceed the subprocess deadline. Preserve explicit success fixtures and assert that valid configurations reach the expected Docker boundary. * feat(prism): add offline recovery for accepted pending blocks (#259) * Fix CKPool version-mask resolution after node readiness (#257) Resolve dynamic masks in the supervised child and fail startup when no live template can be obtained. Validate settings before wallet setup, bound retry configuration, and preserve explicit zero, missing-field compatibility and static mode. Update prelaunch guidance and regression coverage. * fix(prism): scope pending-fanout broadcast attempts to their own fanout (#292) The correlated broadcast_attempts subquery in the Postgres dashboard_pending_fanout_rows compared attempt.fanout_txid with an unqualified fanout_txid, which resolves to the attempt row itself. Every pending row on /public/v1/fanouts/pending therefore carried every broadcast attempt in the table. Qualify the outer reference with page_rows so each row aggregates only its own attempts. The postgres ledger gate now seeds two pending fanouts on one block with distinct attempts and asserts each row's attempts; the SQL-shape unit test pins the qualified correlation. Claude-Session: https://claude.ai/code/session_01LCd3ok6DUBZRt6UCp52XQA Co-authored-by: Claude Opus 5 * ci: run push builds only on main and the release branches (#295) Every commit to a branch with an open pull request ran the whole workflow twice: once for push (the branch tip) and once for pull_request (the merge result). Keep pull_request for PR branches and limit push to main, 1.x.x, 2.x.x and 3.x.x, so direct commits to the long-lived branches are still tested. Co-authored-by: Claude Opus 5 (1M context) * v2.0.2 fix: bound candidate replay and audit finalization (#258) * fix: bound replay membership and qualify candidate recovery * test: distinguish heartbeat wakes from proof latency * fix(prism): enumerate the audit root through a fresh directory description The audit artifact store listed its root through the long-lived pinned directory descriptor. Some kernels and overlay filesystems serve a stale or truncated enumeration through a description that was read once and mutated afterwards: on an aarch64 Docker host a conflict snapshot written moments earlier was absent from the next scan, so quarantine deduplication wrote a second copy and metrics and retention observed an empty root. The 2.x.x baseline reproduced this as test_psql_segment_gap_without_ledger_rows_quarantines_existing_slot and six audit-artifact test failures on that host. Every scan now reopens "." relative to the pinned descriptor, verifies the inode identity before and after enumerating, and closes the fresh description. Authority still flows only from the pinned descriptor, no pathname is trusted, and the quarantine, metrics and retention semantics are unchanged. The new tests simulate a stale pinned enumeration and a swapped root, and pin that every scan opens and closes its own description. Refs #255 Co-Authored-By: Claude Fable 5.1 * feat(prism): bounded canonical audit views and streamed body publication A found block's canonical audit bundle carries two window-sized arrays: the accepted shares and the reward manifest's counted shares. The audit store published its external body by reading the candidate whole, decoding it with json.loads, comparing it against the logical bundle as one dictionary graph, cutting segments from a materialized share list, encoding the compact body with one json.dumps and reconstructing the whole window again to verify it, so every accepted block held several window-sized structures in the lease-bearing process. audit_bundle_view adds an immutable, bounded view over a canonical artifact: small members are parsed eagerly, the share arrays and the transaction leaf arrays are replayable lazy sequences decoded through bounded read windows, equality with dictionaries streams record by record, and a streaming encoder reproduces the compact json.dumps bytes in bounded chunks. One JSON value remains the decoder's unit; values above a soft limit are counted in the scan statistics, never rejected. The audit store accepts a view or a dictionary everywhere the logical bundle flows: the canonical source is digested, compared (by identity for a view of that inode, member by member otherwise) and gzip-published through chunks; share segments are cut from slices bounded by the segment size and written through chunked immutable and mutable writers; the compact body is encoded chunk by chunk; the reconstruction check scans the body and each segment lazily and matches records against the logical window; and the byte length comes from the descriptor. Exact canonical bytes, digests, wrapper identity binding, proof validation, immutable publication and idempotent retries are unchanged, and the sorted v1 candidate-identity encoder is never substituted for the typed canonical bytes. Accepted-block persistence consumes the view: externalized rows carry the body pointer as before, the leaf arrays are materialized only for the statement (bounded by the block's transaction count), and the legacy inline lane without a body store splices the verified canonical bytes into the statement literal instead of re-encoding a decoded graph. Refs #255 Co-Authored-By: Claude Fable 5.1 * feat(prism): stream share windows into canonical builds and consume the artifact view The bundle compiler copied a non-list share window into a list before a canonical build and decoded the builder's complete output with json.load, which materialized the whole window twice on the found-block path. The one-shot builder input now streams any replayable share sequence record by record (each json.dumps covers at most one batch), a refuted daemon mirror is routed exactly as before, and a canonical build with an output artifact returns the bounded view of that artifact, scanned once through bounded windows with cancellation checked per read; a scan that decoded an oversized value reports it. Embedders without an artifact still receive the decoded document. Finalization treats the bundle as a mapping, reads only header members after landing, and closes the view on every exit of the ordered phases so a retired landing never pins the candidate file. Exact inode transfer, adopter failure cleanup and first-failure ownership are unchanged. The end-to-end tests drive the real Rust builder and verifier when the tools are available: streamed sequences are never copied, canonical bytes and digests are exact, the compact body reconstructs through the Rust canonicalizer, retries are durable no-ops, tampered or foreign artifacts are rejected, an oversized single field publishes through the bounded writers, generations release their descriptors without cyclic garbage collection, cancellation during the output scan cleans the candidate, and accepted-block persistence consumes the view on both the externalized and the inline lane. Refs #255 Co-Authored-By: Claude Fable 5.1 * test: track proof failures within each performance phase * feat(prism): keep artifact checkpoints on disk and skip oversized values within bounded windows The lazy record sequences of a canonical artifact view kept their byte-offset checkpoints in an in-memory array, which grows with the window, and a record larger than a read window was decoded whole during the initial scan to find its end. Checkpoints are now fixed-width entries appended to an unlinked temporary index file shared by the view's lazy arrays and read back by position, so the index scales with the window on disk and retires with the view (an in-memory array remains the fallback when no temporary file can be created, and the scan statistics say so). The scan skips values structurally window by window: a record that straddles a window, a nested value or an oversized string is walked with at most about two read chunks of text held, while structure and termination are still validated. A consumer that needs such a record still decodes it whole; that record-sized decode is the remaining per-value maximum, reported through the scan statistics and the compiler's diagnostic line rather than enforced by refusing valid work. Refs #255 Co-Authored-By: Claude Fable 5.1 * fix: reject invalid NUL values before candidate membership COPY * test: cover streaming deadlines and restart fixture configuration * test: exercise payout replay through the bounded membership port * fix: apply native COPY backpressure during candidate replay * test: separate audit codec and object retirement measurements * test: isolate recovery migration from public ledger functions * test: gate candidate replay through native PostgreSQL and psql * chore: prepare candidate recovery release 2.0.2 * test: verify candidate body durability across abrupt restart * feat(prism): close the remaining coordinator-sized JSON boundaries of canonical finalization Review of the first audit stack found four places where a window- or recipient-scaled structure could still form in the coordinator: The checkpoint index fell back to an in-memory array exactly when the scratch file could not be created. The fallback is removed; scratch pressure (index, record spool, helper) raises ArtifactResourcePressure, a plain Exception that is neither a decode error nor a corruption classification, so publication fails retry-safe instead of degrading. The artifact view kept balances, payout accounts, entitlements, settlement recipients and fanout manifests eager on the argument that recipient cardinality bounds them; it does not, since recipients can number as many as shares. Every such member is now a lazy on-disk sequence, the compact body's part index and header members are parsed lazily as well, and the share-part index is spooled to disk as it is produced and its digest hashed incrementally, so a small configured segment size cannot turn the index into a window-scaled list. A record above the in-process decode limit is now normalized by an isolated helper process that streams the record in and its compact encodings (original order, and sorted keys for equality) out to a scratch file; the coordinator keeps only the small members and never calls json.loads on the record. Consumers compare and encode such records through their streamed encodings. No valid input cap on share identifiers exists on the coordinator side (the Stratum username is read without a length limit), so isolation rather than a byte bound is the honest treatment. Accepted-block persistence and the CTV recovery statement build their one fenced literal from bounded chunks: lazy accounts, leaf arrays and fanout manifests stream from the view, the inline lane splices the canonical artifact verbatim once, the dollar-quote tag is checked by streaming, and the manifest-set text and per-manifest artifacts are derived on demand. The payout preview seam accepts a mapping with a lazy accounts sequence and walks it row by row. The one-shot builder input splices a daemon mirror's or page-backed window's pre-encoded canonical items instead of parsing records (never touching the parsed cache) and streams balances and leaves in batches. The flat-record skip regex is possessive so a window cut inside a long string fails without a backtracking stack proportional to the window. Refs #255 Co-Authored-By: Claude Fable 5.1 * feat(prism): chunked immutable candidate bodies and metadata-first replay (#255) Replace the whole-jsonb block-candidate intent with a bounded codec and chunked immutable storage: - candidate_codec.py: streaming encoder whose json.dumps calls cover at most 256 records / ~256 KiB, fixed 256 KiB chunks with per-chunk digests, a scalar manifest with per-field spans and page rows (index kept on disk on replay), jsonb-compatibility rejection (NaN/Infinity, U+0000, unpaired surrogates) at preparation, and a PreparedCandidateIntent mapping whose shares_json is the job's immutable sequence rather than a list copy. The body bytes are the historical v1 identity JSON, so candidate_sha256 is unchanged. - candidate_store.py: staging/seal/index/page/chunk statements, bounded hydration to a spool + index pair, the isolated legacy (v1 jsonb) helper process, the bounded header page with explicit exhaustion, and the split janitor (fenced orphan retirement, lease-free chunk reaping). - 002_candidate_bodies.sql: additive manifest/chunk/span/page tables, DB triggers enforcing sealed-body immutability, the dual-format outbox CHECK that refuses a pre-#255 writer's terminal writes, the bounded header SQL projection and a schema capability row. - share_ledger.py: both durable write routes stage outside the writer gate and publish under the fenced statement with a compare-and-swap on the observed outbox version; the exact content comparator reads the stored bytes page by page with a jsonb-equivalence fallback; metadata-first header pages; bounded hydration; janitor; schema-capability refusal; storage version 1 retained as the compatibility/rollback mode. - block_candidates.py: intent preparation without copies, header-page enumeration honouring the configured replay page size, bounded descriptor window with per-hash obligations, dequeue-time hydration, janitor step. - share_writer/share_submission/coordinator/config/compose wiring and the fake-Postgres harness model for every new statement. * fix(prism): recover chunked candidates and add codec regressions (#255) - recover_pending_blocks: read the outbox row's storage version and body reference; a version-2 row is hydrated through the ledger's bounded chunk reader (spool + on-disk index) instead of being misclassified as a missing payload; an unsealed body or an unknown version fails closed. - recovery tests: cover the version-2 decode route and its refusals; the native fixture uses a private search_path only (001's DROP FUNCTION must not touch the shared public schema) and applies 002 after 001. - coordinator: pass lease_retry_sleep through the split ledger constructor; the boot-time schema capability check reports a transport failure instead of swallowing it. - candidate codec: a hydrated intent whose large array fields are spool adapters re-stages verbatim; new tests/test_prism_candidate_codec.py pins oracle byte identity, jsonb-compatibility rejection, bounded json.dumps calls and chunk sizes, daemon/page-backed verbatim copies, spool round trips, hint-page correction, and manifest validation. * fix(prism): complete number tokens at window boundaries and validate skipped records at scan time An independent review found that a number cut by a read window decoded as its prefix: raw_decode returns 1 for a window ending in "1e+" or "1." without reaching the window end, so the scanner accepted the prefix and then reported a missing separator (reproduced with tiny windows and with the default 256 KiB window when the token straddled the boundary). A decoded number is now complete only when the character after it cannot continue a number or the file has ended; otherwise the window grows, and a continuation character with the whole file in view is reported as an invalid number, as the standard decoder does. The structural walker that skipped records without decoding them is replaced by a streaming token validator: a flat record that fits the window is validated and skipped by a strict-grammar regex, and anything else -- straddling, nested, string or scalar of any size -- is walked by a state machine that checks token syntax, escapes, control characters, separators, container balance and termination window by window. Every rejection the standard decoder makes for the document is now made at scan time, with the same bounded windows and the same isolated handling of oversized records. Tests sweep window sizes from one byte upward and the default-window alignment across decimal, exponent and sign boundaries, exercise lazy slice access over scalar arrays, and pin the scan-time rejections. Refs #255 Co-Authored-By: Claude Fable 5.1 * fix(prism): bounded janitor steps and DB-enforced publish/retire race (#255) - reap_retired_chunks_sql: every step is bounded (at most max_chunks rows of one part table) and every probe is an existence test; chunks drain first, then page rows, then span rows, and the manifest goes on a later step whose snapshot finds every child table empty. The result reports what remains so the submitter loop keeps stepping until the body is gone. - Publication statements lock the referenced body FOR SHARE while checking it is sealed; orphan retirement locks it FOR UPDATE; the manifest trigger refuses to retire a body any outbox row references (fresh snapshot after the lock wait), so the losing side of the race fails closed. The fake Postgres model mirrors the invariant. - Partial index on outbox.body_id WHERE body_id IS NOT NULL so reference probes never scan the terminal history. * test(prism): real-PostgreSQL gate for chunked candidate bodies (#255) tests/prism_postgres_candidate_gate.py drives the shipped ledger against a disposable PostgreSQL through the psql backend (and the native client when psycopg is present): idempotent migration, standalone and share-append publication with exact replay and mismatch refusal, the metadata-first page with explicit exhaustion, hydration of a 3,000-share body through bounded chunk pages, trigger-enforced immutability of sealed bodies, refusal to retire a referenced body, terminal detachment, bounded janitor draining and the schema-level refusal of a pre-#255 terminal write. test/test-prism-postgres-candidate-storage.sh provisions the container. Two statement fixes the gate surfaced: the staging manifest statement reports 'staging' for the row it just inserted (invisible to its own final SELECT), and the header page's COALESCE parenthesization. * fix(prism): retire owned artifact descriptors when the post-open authority check fails The owned artifact source and the owned artifact scan re-validate the directory authority after opening. When that check raised, the new source or view was left to its finalizer, which a retained traceback can postpone. Both now close the descriptors (artifact and checkpoint index) explicitly before re-raising. The test retains the exception with its traceback under disabled cyclic collection and asserts the descriptors are already closed. Refs #255 Co-Authored-By: Claude Fable 5.1 * chore(prism): drop the unused digest import from block finalization The CTV manifest digest now streams through the artifact view's sorted encoding, so the whole-object helper import is no longer used. Refs #255 Co-Authored-By: Claude Fable 5.1 * fix: bound candidate encoding and supervise helper backpressure * fix: stream finalization transport and preserve bounded replay ownership * fix: close candidate sealing races and preserve helper session guards * feat(prism): bounded lazy views for hydrated candidate spool bodies (#255) Hydration decoded every spanned value whole and guarded those calls with size ceilings, so an oversized share record made its page "not valid JSON" above 16 MiB, the record walker rejected records above 256 MiB as corruption, and oversized top-level strings and objects were joined or decoded in one call. The historical layout bounds none of those fields. Add lab/prism/candidate_spool_view.py: one decode policy for every JSON value in a body. A value within SPOOL_VIEW_DECODE_BYTES (1 MiB) is one bounded json.loads; a larger value is a lazy view over its own byte range (SpoolObjectView, SpoolArrayView, SpoolStringView, SpoolRawValue) whose members and items follow the same policy, so the reader's largest C call is bounded at any depth for a body of any size and nothing is rejected for size. Views expose their stored encoding verbatim (iter_byte_chunks, iter_encoded_chunks) and a streamed string its text (iter_text_chunks) in pieces that never split an escape, a surrogate pair or a UTF-8 sequence; equality streams against plain values; str() of a streamed string is a TypeError by design. Structure comes from a stateful byte scanner (quoted and nested },{ traps, split escapes and tokens across reads); item indices stay in memory up to a cap and spill to an unlinked scratch file, raising ArtifactResourcePressure (retryable) when scratch is unavailable. Views hold the body and form no cycles, close their scratch on build errors before the exception leaves, and fail closed after PreparedCandidateIntent.release(). Rewire the codec's decoding side onto it: SpoolJsonArraySequence splits an oversized page by the scanner instead of raising, ignores legacy hint offsets (sequential walk that records exact batch starts for later random access) and streams oversized pages verbatim when re-staging; the record walker has no per-record ceiling; decode_spool_small_fields applies the policy to spanned fields with an explicit decoded-metadata boundary (SPOOL_DECODED_METADATA_FIELDS: coinbase, hashes, schema, extranonces hydrate as str in slices; block_hex stays lazy above the threshold); prepared_intent_from_spool shallow-copies a lazy pending_share so the stamp restores and closes its index if that copy fails. Encoding and its constants are unchanged. Supervise the isolated record helper in audit_bundle_view: admission slots (HelperAdmission, shared by helper kinds), one deadline over slot wait, stdin feed, header/encoding reads and diagnostics drain, optional cancellation threaded from CanonicalAuditBundleView.scan through every lazy read, bounded stderr retention, kill/reap/close on every exit, no preexec_fn; exit 2 stays malformed, everything else is retryable pressure. iter_json_chunks and materialize_json accept any streamed scalar exposing iter_encoded_chunks/iter_text_chunks. tests/test_prism_candidate_spool_view.py instruments every json.loads and read against the threshold, and covers small/large/Unicode round trips at every slice cut, tiny-read scanning of raw UTF-8 and split numbers, random access and slicing through the disk index, restaging byte identity and per-field digests, the >16 MiB share_id case, legacy hinted pages, corrupt/truncated/missing/closed classification, ownership under retained exceptions with gc disabled, and the helper's deadline, cancellation, admission and exit-status handling. Refs #255 Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_01KU629eYst1jKoYgBkErwnR * fix: bound replay header keys and document schema prerequisites * fix: bound replay overflow and complete candidate spool qualification * fix: require the candidate-body migration and hydrate oversized replay headers - Refuse a database without 002_candidate_bodies.sql for every storage version, at coordinator boot and in the offline recovery plan, through one shared rule. Replay headers and the terminal outbox statements use the migrated columns even at storage version 1, so a legacy-query fallback would only move the failure. - Accept None header placeholders only when the replay header is oversized, and treat the hydrated body's pending share as authoritative: adopt it on the floor before releasing the header stand-in. A long worker name empties share_id as well as username. - Document that 002 must be applied before starting this release. Co-Authored-By: Claude Opus 5 (1M context) * fix: accept streamed block hex after replay and absent legacy shares - Pass a hydrated SpoolStringView block_hex through to the codec when a replayed credit-on-accept candidate is re-encoded; str() refuses it and the codec writes its stored encoding byte-identically. - Read the offline recovery header prefix through the text-chunk helper rather than str(). - Omit shares_json from the legacy storage-version-1 document when a prepared intent carries none, and decode streamed views there, since that route writes plain JSON. Co-Authored-By: Claude Opus 5 (1M context) * fix: decode a streamed username when rebuilding a replayed candidate block_candidate_from_intent passed username through str(). username is not fixed metadata, so above the 1 MiB view threshold a hydrated body keeps it as a SpoolStringView, which refuses str(). The digest-valid candidate was then quarantined as malformed. The Stratum reader sets no line-length cap, so an authorized username of that size is possible. Join the streamed text explicitly for the worker identity, as the view contract asks, and keep str() for every other value. Co-Authored-By: Claude Opus 5 (1M context) --------- Co-authored-by: Claude Fable 5.1 * fix: address public service and replica review findings * fix: bound public cache work and correct release CI contracts * fix: validate high-diff URLs and isolate loopback test traffic --------- Co-authored-by: Claude Fable 5 Co-authored-by: Alex Co-authored-by: Dan Hepworth Co-authored-by: Alex Co-authored-by: Dan Hepworth Co-authored-by: Robert Clarke Co-authored-by: Robert Clarke --- .env.example | 239 +- .github/scripts/test-qbit-image-runtime.sh | 103 + .github/workflows/ci.yml | 148 +- Cargo.lock | 4 +- Makefile | 22 +- PRISM.md | 62 +- README.md | 60 +- VERSION | 2 +- ckpool/README.md | 45 +- compose.production.yaml | 29 + compose.yaml | 274 +- config/prism-postgres/pg_hba.conf | 38 + config/prism-postgres/replica-entrypoint.sh | 133 + crates/qbit-pool-builder/Cargo.toml | 2 +- crates/qbit-prism/Cargo.toml | 2 +- crates/qbit-prism/README.md | 13 + crates/qbit-prism/sql/001_share_ledger.sql | 870 +- ...dger_revert_audit_publication_sequence.sql | 367 + .../qbit-prism/sql/002_candidate_bodies.sql | 511 + .../src/bin/qbit-prism-build-audit-bundle.rs | 531 +- crates/qbit-prism/src/lib.rs | 15 + crates/qbit-prism/src/window.rs | 1603 + crates/qbit-prism/tests/audit_cli.rs | 453 +- doc/mining.md | 4 +- doc/release-notes-2.0.0.md | 332 + docker/ckpool/ckpool-version-mask.py | 192 +- docker/ckpool/start-ckpool.sh | 17 +- docs/README.md | 13 + docs/mainnet-deployment.md | 206 +- docs/prism-candidate-storage.md | 391 + docs/prism-capacity-readiness.md | 483 + docs/prism-coordinator-refactor/README.md | 46 + .../a1-audit-artifacts.md | 78 + .../prism-coordinator-refactor/b3-decision.md | 107 + docs/prism-coordinator-refactor/invariants.md | 112 + docs/prism-coordinator-refactor/roadmap.md | 49 + .../prism-coordinator-refactor/stacked-prs.md | 46 + docs/prism-coordinator-refactor/structure.md | 80 + docs/prism-coordinator-refactor/validation.md | 135 + docs/prism-ledger-ops.md | 429 +- docs/prism-overload-alerts.md | 1022 + docs/prism-payout-artifact-measurement.md | 171 + docs/prism-postgres-replica.md | 250 + docs/prism-public-operator-endpoint-split.md | 310 + docs/prism-rejections.md | 2 +- docs/public-dashboard-api-v1.openapi.yaml | 458 +- docs/public-dashboard-api/README.md | 333 +- .../fixtures/block-markers.json | 73 + .../fixtures/blocks-chain-states.json | 63 + .../fixtures/hashrate-series-dual-rate.json | 75 + .../fixtures/pool-summary.json | 3 + docs/releasing.md | 66 + docs/router-integration-notes.md | 13 +- lab/auxpow/Dockerfile | 2 +- lab/auxpow/auxpow_coordinator.py | 8 +- lab/auxpow/vardiff.py | 102 +- lab/miner-sim/Dockerfile | 2 +- lab/prism/Dockerfile | 27 +- lab/prism/accepted_preview_telemetry.py | 723 + lab/prism/audit_artifacts.py | 6154 +++ lab/prism/audit_bundle_view.py | 2366 ++ lab/prism/audit_http.py | 706 + lab/prism/backfill_audit_bundle_canonical.py | 1117 + lab/prism/backfill_ctv_fanouts.py | 84 +- lab/prism/background_services.py | 2285 ++ lab/prism/block_candidates.py | 8967 ++++ lab/prism/block_finalization.py | 2365 ++ lab/prism/bounded_executor.py | 166 + lab/prism/bundle_compiler.py | 2639 ++ lab/prism/candidate_codec.py | 2238 + lab/prism/candidate_spool_view.py | 1029 + lab/prism/candidate_store.py | 1687 + lab/prism/candidate_window.py | 266 + lab/prism/coordinator_config.py | 2252 + lab/prism/coordinator_shutdown.py | 300 + lab/prism/ctv_runtime.py | 411 + lab/prism/direct_stratum.py | 19 +- lab/prism/endpoint_registry.py | 805 + lab/prism/future_callbacks.py | 31 + lab/prism/job_build_benchmark.py | 26 +- lab/prism/job_bundle.py | 3628 ++ lab/prism/job_delivery.py | 3042 ++ lab/prism/metrics.py | 2031 + lab/prism/observability.py | 780 + lab/prism/payout_state.py | 5944 +++ lab/prism/prism_coordinator.py | 33921 +++------------- lab/prism/process_telemetry.py | 1622 + lab/prism/progress_health.py | 698 + lab/prism/public_api.py | 670 +- lab/prism/public_read_service.py | 1535 + lab/prism/recover_pending_blocks.py | 476 + lab/prism/recovery_json.py | 38 + lab/prism/reorg_reconciler.py | 1366 + lab/prism/rpc.py | 241 + lab/prism/run_ctv_broadcaster_daemon.py | 43 +- lab/prism/share_json_stream.py | 363 + lab/prism/share_ledger.py | 7646 +++- lab/prism/share_submission.py | 799 + lab/prism/share_writer.py | 831 + lab/prism/statement_spool.py | 105 + lab/prism/stratum_session.py | 1583 + lab/prism/template_artifacts.py | 605 + lab/prism/tip_refresh.py | 4092 ++ lab/prism/vardiff_service.py | 2538 ++ lab/prism/worker_difficulty_store.py | 664 + lab/prism/writer_lease_timing.py | 859 + scripts/check-env.sh | 85 +- scripts/prism-self-check.py | 25 +- scripts/run_python_test_shard.py | 57 + test/test-prism-postgres-candidate-storage.sh | 65 + test/test-prism-postgres-ledger.sh | 840 +- test/test-prism-postgres-native-ledger.sh | 338 +- test/test-prism-postgres-scale.sh | 5 +- test/test-prism-postgres-seed-guard.sh | 606 + test/test-prism-public-read-replica.sh | 89 + test/test-prism-stratum-regtest-live.sh | 170 +- tests/fixtures/fake_serve_builder.py | 298 +- tests/fixtures/mainnet-compose.env | 7 +- .../window_pipeline_parity/reference.json | 2666 ++ tests/perf/candidate_storage.py | 162 + tests/perf/candidate_window_membership.py | 223 + tests/perf/canonical_audit_retirement.py | 172 + tests/perf/serialize_block_share_cost.md | 493 + tests/perf/serialize_block_share_cost.py | 1338 + .../perf/stratum_session_concurrency_load.md | 245 + .../perf/stratum_session_concurrency_load.py | 2028 + tests/perf/window_builder_pipe_latency.md | 270 + tests/perf/window_builder_pipe_latency.py | 951 + tests/perf/window_ledger_decode_latency.md | 219 + tests/perf/window_ledger_decode_latency.py | 593 + tests/perf/window_paging_allocation.md | 460 + tests/perf/window_paging_allocation.py | 1236 + tests/perf/window_pipeline_gil_scaling.md | 704 + tests/perf/window_pipeline_gil_scaling.py | 2390 ++ tests/prism_candidate_storm.py | 2508 ++ tests/prism_concurrency_harness.py | 4175 ++ tests/prism_coordinator_test_support.py | 500 + tests/prism_landing_harness.py | 951 + tests/prism_postgres_a1_gate.py | 2306 ++ tests/prism_postgres_a1_migration_gate.py | 1782 + tests/prism_postgres_a1_process_gate.py | 2624 ++ tests/prism_postgres_a1_revert_gate.py | 548 + tests/prism_postgres_candidate_gate.py | 281 + tests/prism_public_read_replica_gate.py | 288 + tests/prism_regtest_crash_coordinator.py | 73 + tests/prism_regtest_share_miner.py | 72 + tests/prism_vardiff_test_support.py | 1053 + tests/prism_window_rows_reference.py | 97 + tests/prism_worker_difficulty_store_gate.py | 427 + tests/test_auxpow_vardiff.py | 170 + tests/test_auxpow_vardiff_coordinator.py | 12 + tests/test_check_env_production_gate.py | 260 +- tests/test_ckpool_startup.py | 216 +- tests/test_ckpool_version_mask.py | 387 + tests/test_ctv_broadcaster.py | 1 - tests/test_ctv_broadcaster_daemon.py | 66 +- tests/test_mainnet_compose_contract.py | 52 + tests/test_makefile_lifecycle.py | 45 + tests/test_mining_compose_profiles.py | 273 +- tests/test_prism_accepted_block_accounting.py | 656 + ...sm_accepted_preview_publication_latency.py | 880 + .../test_prism_accepted_preview_telemetry.py | 654 + tests/test_prism_admission_lock.py | 651 + tests/test_prism_allocator_experiment.py | 256 + tests/test_prism_ancestor_redrive.py | 809 + tests/test_prism_audit_api.py | 103 +- tests/test_prism_audit_artifacts.py | 4564 +++ tests/test_prism_audit_bundle_view.py | 925 + tests/test_prism_audit_http.py | 555 + tests/test_prism_audit_root_scan.py | 144 + ...t_prism_backfill_audit_bundle_canonical.py | 1215 + tests/test_prism_background_services.py | 1022 + tests/test_prism_block_candidate_collapse.py | 4851 +++ ...diff.py => test_prism_block_candidates.py} | 25207 +++++------- tests/test_prism_block_finalization.py | 271 + ...st_prism_bounded_canonical_finalization.py | 1304 + tests/test_prism_bounded_executor.py | 181 + tests/test_prism_builder_pipe_transport.py | 1771 + tests/test_prism_candidate_codec.py | 457 + tests/test_prism_candidate_helper.py | 75 + tests/test_prism_candidate_replay_bounds.py | 119 + tests/test_prism_candidate_schema_refusal.py | 82 + .../test_prism_candidate_spool_integration.py | 80 + tests/test_prism_candidate_spool_view.py | 845 + tests/test_prism_candidate_storage_races.py | 252 + tests/test_prism_candidate_storm.py | 1150 + tests/test_prism_candidate_window.py | 411 + tests/test_prism_completion_callbacks.py | 238 + tests/test_prism_compose_profile.py | 302 +- tests/test_prism_concurrency_harness.py | 992 + tests/test_prism_coordinator_config.py | 2616 ++ .../test_prism_coordinator_config_loading.py | 352 + tests/test_prism_coordinator_job_cache.py | 10696 +---- tests/test_prism_coordinator_metrics.py | 1198 + tests/test_prism_coordinator_shutdown.py | 294 +- tests/test_prism_ctv_runtime.py | 547 + tests/test_prism_dequeue_stale_skip.py | 928 + tests/test_prism_direct_stratum.py | 187 +- tests/test_prism_endpoint_registry.py | 672 + tests/test_prism_first_job_latency.py | 22 +- tests/test_prism_heap_census.py | 1155 + tests/test_prism_hot_path.py | 8 +- .../test_prism_immutable_refresh_artifacts.py | 2 +- tests/test_prism_incremental_payout_window.py | 229 + tests/test_prism_initial_job_delivery.py | 78 +- tests/test_prism_issue_224_cadence.py | 596 + tests/test_prism_issue_224_core.py | 1040 + tests/test_prism_issue_224_landing.py | 1094 + tests/test_prism_issue_224_ledger.py | 759 + ...est_prism_job_build_exception_retention.py | 604 + tests/test_prism_job_build_promises.py | 3 +- tests/test_prism_job_build_retention.py | 144 + tests/test_prism_job_builder.py | 4810 +++ tests/test_prism_job_delivery.py | 207 + tests/test_prism_landing_append_epoch.py | 515 + tests/test_prism_landing_budget_watchdog.py | 710 + tests/test_prism_landing_harness_model.py | 1388 + tests/test_prism_landing_interleavings.py | 206 + ...test_prism_lease_heartbeat_tail_latency.py | 978 + ...rism_lease_heartbeat_under_fenced_write.py | 631 + tests/test_prism_lease_monitor_stalls.py | 1019 + tests/test_prism_lease_orphaned_lock.py | 384 + ...est_prism_lease_renewal_vs_expiry_claim.py | 564 + tests/test_prism_lease_takeover.py | 746 + tests/test_prism_metrics.py | 3480 ++ tests/test_prism_observability.py | 1119 + tests/test_prism_payout_state.py | 6043 +++ ...est_prism_payout_window_daemon_recenter.py | 501 + tests/test_prism_pending_block_recovery.py | 520 + tests/test_prism_process_telemetry.py | 309 + tests/test_prism_progress_health.py | 155 +- tests/test_prism_public_dashboard_api.py | 1560 +- tests/test_prism_public_http_limits.py | 151 + tests/test_prism_public_read_service.py | 2571 ++ tests/test_prism_public_refresh_limits.py | 76 + tests/test_prism_reconnect_backpressure.py | 34 +- tests/test_prism_refresh_retry_pacing.py | 2 +- tests/test_prism_reorg_reconciler.py | 633 + tests/test_prism_replay_enumeration_gate.py | 526 + .../test_prism_representative_independence.py | 5 +- tests/test_prism_retained_jobs.py | 1248 + tests/test_prism_self_check.py | 57 +- tests/test_prism_share_ledger.py | 4689 ++- tests/test_prism_share_submission.py | 374 + tests/test_prism_share_writer.py | 839 + tests/test_prism_share_writer_service.py | 654 + ..._prism_singleflight_exception_retention.py | 571 + tests/test_prism_statement_spool.py | 106 + tests/test_prism_stratum_session.py | 507 + ...test_prism_streamed_block_hex_consumers.py | 120 + tests/test_prism_streamed_submitblock.py | 85 + tests/test_prism_tip_publication_boundary.py | 2 +- tests/test_prism_tip_refresh.py | 185 + tests/test_prism_tip_refresh_delivery.py | 3180 ++ tests/test_prism_tip_refresh_epoch.py | 10 +- tests/test_prism_tip_refresh_validation.py | 6 +- tests/test_prism_vardiff.py | 1416 + tests/test_prism_vardiff_durable_resume.py | 1045 + tests/test_prism_vardiff_fast_arrival.py | 451 + tests/test_prism_vardiff_resume.py | 889 + tests/test_prism_vardiff_service.py | 108 + tests/test_prism_window_pipeline_rust.py | 1472 + tests/test_prism_worker_difficulty_store.py | 700 + tests/test_prism_writer_lease_policy.py | 660 + tests/test_public_dashboard_api_contract.py | 336 +- tests/test_qbit_image_runtime_smoke.py | 444 + tests/test_run_python_test_shard.py | 110 + .../test_window_paging_allocation_harness.py | 337 + tests/test_window_pipeline_parity.py | 1155 + tests/test_window_pipeline_parity_gate.py | 84 + tests/window_pipeline_parity.py | 2147 + tests/window_pipeline_parity_gate.py | 79 + 272 files changed, 233071 insertions(+), 54968 deletions(-) create mode 100644 config/prism-postgres/pg_hba.conf create mode 100755 config/prism-postgres/replica-entrypoint.sh create mode 100644 crates/qbit-prism/sql/001_share_ledger_revert_audit_publication_sequence.sql create mode 100644 crates/qbit-prism/sql/002_candidate_bodies.sql create mode 100644 crates/qbit-prism/src/window.rs create mode 100644 doc/release-notes-2.0.0.md create mode 100644 docs/prism-candidate-storage.md create mode 100644 docs/prism-coordinator-refactor/README.md create mode 100644 docs/prism-coordinator-refactor/a1-audit-artifacts.md create mode 100644 docs/prism-coordinator-refactor/b3-decision.md create mode 100644 docs/prism-coordinator-refactor/invariants.md create mode 100644 docs/prism-coordinator-refactor/roadmap.md create mode 100644 docs/prism-coordinator-refactor/stacked-prs.md create mode 100644 docs/prism-coordinator-refactor/structure.md create mode 100644 docs/prism-coordinator-refactor/validation.md create mode 100644 docs/prism-overload-alerts.md create mode 100644 docs/prism-postgres-replica.md create mode 100644 docs/prism-public-operator-endpoint-split.md create mode 100644 docs/public-dashboard-api/fixtures/block-markers.json create mode 100644 docs/public-dashboard-api/fixtures/blocks-chain-states.json create mode 100644 docs/public-dashboard-api/fixtures/hashrate-series-dual-rate.json create mode 100644 docs/releasing.md create mode 100644 lab/prism/accepted_preview_telemetry.py create mode 100644 lab/prism/audit_artifacts.py create mode 100644 lab/prism/audit_bundle_view.py create mode 100644 lab/prism/audit_http.py create mode 100644 lab/prism/backfill_audit_bundle_canonical.py create mode 100644 lab/prism/background_services.py create mode 100644 lab/prism/block_candidates.py create mode 100644 lab/prism/block_finalization.py create mode 100644 lab/prism/bounded_executor.py create mode 100644 lab/prism/bundle_compiler.py create mode 100644 lab/prism/candidate_codec.py create mode 100644 lab/prism/candidate_spool_view.py create mode 100644 lab/prism/candidate_store.py create mode 100644 lab/prism/candidate_window.py create mode 100644 lab/prism/coordinator_config.py create mode 100644 lab/prism/coordinator_shutdown.py create mode 100644 lab/prism/ctv_runtime.py create mode 100644 lab/prism/endpoint_registry.py create mode 100644 lab/prism/future_callbacks.py create mode 100644 lab/prism/job_bundle.py create mode 100644 lab/prism/job_delivery.py create mode 100644 lab/prism/metrics.py create mode 100644 lab/prism/observability.py create mode 100644 lab/prism/payout_state.py create mode 100644 lab/prism/process_telemetry.py create mode 100644 lab/prism/progress_health.py create mode 100644 lab/prism/public_read_service.py create mode 100644 lab/prism/recover_pending_blocks.py create mode 100644 lab/prism/recovery_json.py create mode 100644 lab/prism/reorg_reconciler.py create mode 100644 lab/prism/rpc.py create mode 100644 lab/prism/share_json_stream.py create mode 100644 lab/prism/share_submission.py create mode 100644 lab/prism/share_writer.py create mode 100644 lab/prism/statement_spool.py create mode 100644 lab/prism/stratum_session.py create mode 100644 lab/prism/template_artifacts.py create mode 100644 lab/prism/tip_refresh.py create mode 100644 lab/prism/vardiff_service.py create mode 100644 lab/prism/worker_difficulty_store.py create mode 100644 lab/prism/writer_lease_timing.py create mode 100644 scripts/run_python_test_shard.py create mode 100755 test/test-prism-postgres-candidate-storage.sh create mode 100755 test/test-prism-postgres-seed-guard.sh create mode 100755 test/test-prism-public-read-replica.sh create mode 100644 tests/fixtures/window_pipeline_parity/reference.json create mode 100644 tests/perf/candidate_storage.py create mode 100644 tests/perf/candidate_window_membership.py create mode 100644 tests/perf/canonical_audit_retirement.py create mode 100644 tests/perf/serialize_block_share_cost.md create mode 100644 tests/perf/serialize_block_share_cost.py create mode 100644 tests/perf/stratum_session_concurrency_load.md create mode 100644 tests/perf/stratum_session_concurrency_load.py create mode 100644 tests/perf/window_builder_pipe_latency.md create mode 100644 tests/perf/window_builder_pipe_latency.py create mode 100644 tests/perf/window_ledger_decode_latency.md create mode 100644 tests/perf/window_ledger_decode_latency.py create mode 100644 tests/perf/window_paging_allocation.md create mode 100644 tests/perf/window_paging_allocation.py create mode 100644 tests/perf/window_pipeline_gil_scaling.md create mode 100644 tests/perf/window_pipeline_gil_scaling.py create mode 100644 tests/prism_candidate_storm.py create mode 100644 tests/prism_concurrency_harness.py create mode 100644 tests/prism_coordinator_test_support.py create mode 100644 tests/prism_landing_harness.py create mode 100644 tests/prism_postgres_a1_gate.py create mode 100644 tests/prism_postgres_a1_migration_gate.py create mode 100644 tests/prism_postgres_a1_process_gate.py create mode 100644 tests/prism_postgres_a1_revert_gate.py create mode 100644 tests/prism_postgres_candidate_gate.py create mode 100644 tests/prism_public_read_replica_gate.py create mode 100644 tests/prism_regtest_crash_coordinator.py create mode 100644 tests/prism_regtest_share_miner.py create mode 100644 tests/prism_vardiff_test_support.py create mode 100644 tests/prism_window_rows_reference.py create mode 100644 tests/prism_worker_difficulty_store_gate.py create mode 100644 tests/test_prism_accepted_block_accounting.py create mode 100644 tests/test_prism_accepted_preview_publication_latency.py create mode 100644 tests/test_prism_accepted_preview_telemetry.py create mode 100644 tests/test_prism_admission_lock.py create mode 100644 tests/test_prism_allocator_experiment.py create mode 100644 tests/test_prism_ancestor_redrive.py create mode 100644 tests/test_prism_audit_artifacts.py create mode 100644 tests/test_prism_audit_bundle_view.py create mode 100644 tests/test_prism_audit_http.py create mode 100644 tests/test_prism_audit_root_scan.py create mode 100644 tests/test_prism_backfill_audit_bundle_canonical.py create mode 100644 tests/test_prism_background_services.py create mode 100644 tests/test_prism_block_candidate_collapse.py rename tests/{test_prism_coordinator_vardiff.py => test_prism_block_candidates.py} (51%) create mode 100644 tests/test_prism_block_finalization.py create mode 100644 tests/test_prism_bounded_canonical_finalization.py create mode 100644 tests/test_prism_bounded_executor.py create mode 100644 tests/test_prism_builder_pipe_transport.py create mode 100644 tests/test_prism_candidate_codec.py create mode 100644 tests/test_prism_candidate_helper.py create mode 100644 tests/test_prism_candidate_replay_bounds.py create mode 100644 tests/test_prism_candidate_schema_refusal.py create mode 100644 tests/test_prism_candidate_spool_integration.py create mode 100644 tests/test_prism_candidate_spool_view.py create mode 100644 tests/test_prism_candidate_storage_races.py create mode 100644 tests/test_prism_candidate_storm.py create mode 100644 tests/test_prism_candidate_window.py create mode 100644 tests/test_prism_completion_callbacks.py create mode 100644 tests/test_prism_concurrency_harness.py create mode 100644 tests/test_prism_coordinator_config.py create mode 100644 tests/test_prism_coordinator_config_loading.py create mode 100644 tests/test_prism_coordinator_metrics.py create mode 100644 tests/test_prism_ctv_runtime.py create mode 100644 tests/test_prism_dequeue_stale_skip.py create mode 100644 tests/test_prism_endpoint_registry.py create mode 100644 tests/test_prism_heap_census.py create mode 100644 tests/test_prism_issue_224_cadence.py create mode 100644 tests/test_prism_issue_224_core.py create mode 100644 tests/test_prism_issue_224_landing.py create mode 100644 tests/test_prism_issue_224_ledger.py create mode 100644 tests/test_prism_job_build_exception_retention.py create mode 100644 tests/test_prism_job_build_retention.py create mode 100644 tests/test_prism_job_builder.py create mode 100644 tests/test_prism_job_delivery.py create mode 100644 tests/test_prism_landing_append_epoch.py create mode 100644 tests/test_prism_landing_budget_watchdog.py create mode 100644 tests/test_prism_landing_harness_model.py create mode 100644 tests/test_prism_landing_interleavings.py create mode 100644 tests/test_prism_lease_heartbeat_tail_latency.py create mode 100644 tests/test_prism_lease_heartbeat_under_fenced_write.py create mode 100644 tests/test_prism_lease_monitor_stalls.py create mode 100644 tests/test_prism_lease_orphaned_lock.py create mode 100644 tests/test_prism_lease_renewal_vs_expiry_claim.py create mode 100644 tests/test_prism_lease_takeover.py create mode 100644 tests/test_prism_metrics.py create mode 100644 tests/test_prism_observability.py create mode 100644 tests/test_prism_payout_state.py create mode 100644 tests/test_prism_payout_window_daemon_recenter.py create mode 100644 tests/test_prism_pending_block_recovery.py create mode 100644 tests/test_prism_process_telemetry.py create mode 100644 tests/test_prism_public_http_limits.py create mode 100644 tests/test_prism_public_read_service.py create mode 100644 tests/test_prism_public_refresh_limits.py create mode 100644 tests/test_prism_reorg_reconciler.py create mode 100644 tests/test_prism_replay_enumeration_gate.py create mode 100644 tests/test_prism_retained_jobs.py create mode 100644 tests/test_prism_share_submission.py create mode 100644 tests/test_prism_share_writer.py create mode 100644 tests/test_prism_share_writer_service.py create mode 100644 tests/test_prism_singleflight_exception_retention.py create mode 100644 tests/test_prism_statement_spool.py create mode 100644 tests/test_prism_stratum_session.py create mode 100644 tests/test_prism_streamed_block_hex_consumers.py create mode 100644 tests/test_prism_streamed_submitblock.py create mode 100644 tests/test_prism_tip_refresh.py create mode 100644 tests/test_prism_tip_refresh_delivery.py create mode 100644 tests/test_prism_vardiff.py create mode 100644 tests/test_prism_vardiff_durable_resume.py create mode 100644 tests/test_prism_vardiff_fast_arrival.py create mode 100644 tests/test_prism_vardiff_resume.py create mode 100644 tests/test_prism_vardiff_service.py create mode 100644 tests/test_prism_window_pipeline_rust.py create mode 100644 tests/test_prism_worker_difficulty_store.py create mode 100644 tests/test_prism_writer_lease_policy.py create mode 100644 tests/test_qbit_image_runtime_smoke.py create mode 100644 tests/test_run_python_test_shard.py create mode 100644 tests/test_window_paging_allocation_harness.py create mode 100644 tests/test_window_pipeline_parity.py create mode 100644 tests/test_window_pipeline_parity_gate.py create mode 100644 tests/window_pipeline_parity.py create mode 100644 tests/window_pipeline_parity_gate.py diff --git a/.env.example b/.env.example index 88739ecfa..a5cc28273 100644 --- a/.env.example +++ b/.env.example @@ -35,6 +35,7 @@ QBIT_DATA_SOURCE=qbit-data BITCOIN_DATA_SOURCE=bitcoin-data PRISM_POSTGRES_DATA_SOURCE=prism-postgres-data PRISM_POSTGRES_WAL_SOURCE=prism-postgres-wal +PRISM_POSTGRES_REPLICA_DATA_SOURCE=prism-postgres-replica-data PRISM_AUDIT_DATA_SOURCE=prism-audit-data QBIT_CHAIN=regtest @@ -88,8 +89,13 @@ QBIT_EXPECTED_GENESIS_HASH= QBIT_EXPECTED_MAX_BLOCK_WEIGHT=2000000 QBIT_EXPECTED_WITNESS_SCALE_FACTOR=1 QBIT_EXPECTED_COINBASE_MATURITY=1000 -# dynamic makes ckpool use qbit getblocktemplate.versionrollingmask when present. -# Older qbitd builds without that field fall back to CKPOOL_VERSION_MASK. +# dynamic makes ckpool use qbit getblocktemplate.versionrollingmask. The probe +# runs after the startup readiness gate passes, and fails closed if no template +# can be retrieved, so ckpool never freezes a mask picked during a cold start. +# Older qbitd builds that answer without that field fall back to +# CKPOOL_VERSION_MASK. Set static to skip the probe and use CKPOOL_VERSION_MASK +# as configured; that is the explicit escape when getblocktemplate is +# deliberately unavailable, such as authorized mainnet prelaunch. # Direct qbit Stratum services use their own configured fallback masks: # CKPOOL_VERSION_MASK for ckpool, PRISM_VERSION_ROLLING_MASK for PRISM, and # AUXPOW_STRATUM_VERSION_MASK for parent-chain AuxPoW Stratum. @@ -98,6 +104,9 @@ QBIT_EXPECTED_COINBASE_MATURITY=1000 CKPOOL_VERSION_MASK_MODE=dynamic CKPOOL_VERSION_MASK=1fffe000 CKPOOL_VERSION_MASK_RPC_TIMEOUT_SECONDS=5 +# Bounded retry for transient getblocktemplate failures during dynamic resolution. +CKPOOL_VERSION_MASK_PROBE_ATTEMPTS=3 +CKPOOL_VERSION_MASK_PROBE_RETRY_SECONDS=2 BITCOIN_CHAIN=regtest BITCOIN_CHAIN_FLAG=-regtest @@ -169,6 +178,28 @@ PRISM_AUDIT_CANDIDATE_RETENTION_SECONDS=86400 PRISM_CTV_BROADCAST_ATTEMPT_DETAIL_LIMIT=20 PRISM_CTV_BROADCAST_RETRY_BACKOFF_SECONDS=300 PRISM_EVIDENCE_PATH=/var/lib/qbit-prism/audit/prism-live-evidence.json +# Allocator telemetry on /metrics (issue #226). 1 reads glibc mallinfo2 once +# per scrape; 0 renders the allocator gauges unavailable and skips the call. +# The interpreter and component gauges are unconditional either way. +PRISM_MALLOC_TELEMETRY=1 +# glibc malloc arena cap for the coordinator process (issue #226). The image +# defaults to 2 and compose passes this value through to override it; read +# docs/mainnet-deployment.md "PRISM Allocator Settings" before changing it, +# and never leave it empty (glibc reads "" as 0). +MALLOC_ARENA_MAX=2 +# Operator-triggered heap census (SIGUSR1) and malloc_trim (SIGRTMIN+1), +# issue #226 part 2. All off by default; read docs/prism-capacity-readiness.md +# "Heap Census, Allocator Control, and the Resident-Set Bound" before enabling +# any of them. Reports land under PRISM_AUDIT_DIR/heap-census unless +# PRISM_HEAP_CENSUS_DIR is set. +PRISM_HEAP_CENSUS=0 +PRISM_HEAP_CENSUS_TOP_N=40 +PRISM_HEAP_CENSUS_MAX_BYTES=262144 +PRISM_HEAP_CENSUS_MAX_SECONDS=10 +PRISM_HEAP_CENSUS_MIN_INTERVAL_SECONDS=60 +PRISM_HEAP_CENSUS_TRACEMALLOC=0 +PRISM_MALLOC_TRIM_SIGNAL=0 +PRISM_MALLOC_TRIM_INTERVAL_SECONDS=0 PRISM_STOP_AFTER_BLOCK=0 PRISM_MAX_BLOCKS=2147483647 PRISM_BLOCKPOLL_SECONDS=2 @@ -179,6 +210,10 @@ PRISM_BLOCK_SUBMIT_RPC_TIMEOUT_SECONDS=1 # Fresh deadline for each submitter Postgres statement and local DB gate # (cheap outbox polls and fast-lane-adjacent calls only). PRISM_BLOCK_SUBMIT_DB_TIMEOUT_SECONDS=1 +# Startup and ancestor re-drive payloads per cursor page (1..1024). +# Set to 1 as a temporary mitigation for large candidate JSON; see #255. +# Smaller pages add DB/RPC round trips and do not bound one candidate's bytes. +PRISM_BLOCK_REPLAY_PAGE_SIZE=1024 # Landing-class deadline for the post-acceptance accounting tail (persist, # prior balances, confirm, prepared-state rejection). Must stay well above # the poll budget; see issue #188. @@ -187,6 +222,9 @@ PRISM_BLOCK_LANDING_DB_TIMEOUT_SECONDS=30 PRISM_BLOCK_LANDING_DB_TIMEOUT_MAX_SECONDS=120 # Fail-closed bound on landed-but-unresolved accepted-parent transitions. PRISM_ACCEPTED_PARENT_UNRESOLVED_DEPTH_MAX=8 +# Bound candidate cleanup retry records and their retained terminal fences. +# Must be positive and no greater than 8192; lower to constrain overload memory. +PRISM_BLOCK_CANDIDATE_CLEANUP_RETRY_BACKLOG_MAX=4096 # Emit the contended lock and current submitter phase at this cadence. PRISM_BLOCK_SUBMIT_LOCK_WAIT_LOG_SECONDS=5 # Exit nonzero when a timeout-ignoring RPC/DB worker pool stays exhausted this long. @@ -194,8 +232,55 @@ PRISM_BLOCK_SUBMIT_STUCK_CALL_EXIT_SECONDS=30 PRISM_REORG_RECONCILER_ENABLED=1 PRISM_VERSION_ROLLING_MASK=1fffe000 PRISM_COINBASE_TAG=/PRISM/ +# The /public/v1 read surface runs in its own process (the prism-public-api +# compose service, `python3 -m lab.prism.public_read_service`). Every +# PRISM_PUBLIC_* knob below is read by that service, not by prism-coordinator. +# PRISM_PUBLIC_REWARD_WINDOW_CACHE_SECONDS (further down) is the one exception: +# it is read by the ledger, and both processes construct one. +PRISM_PUBLIC_API_BIND=0.0.0.0 +PRISM_PUBLIC_API_PORT=3342 +PRISM_PUBLIC_API_PORT_HOST=3342 +# Bound accepted public HTTP connections; excess connections are closed. +PRISM_PUBLIC_HTTP_MAX_CONNECTIONS=64 +# Total request-header deadline and socket I/O idle timeout. One response per connection. +PRISM_PUBLIC_HTTP_TIMEOUT_SECONDS=10 +# How often the public read service re-checks that it can reach Postgres for +# its own /healthz. Its answer goes stale after max(3 * this, 15) seconds. +# The same probe carries the replication check below, so it is also how often +# the replica contract is re-evaluated. +PRISM_PUBLIC_READINESS_PROBE_INTERVAL_SECONDS=5 +# Per-request database deadline across all origin reads. Set 0 to disable, +# or choose a shorter deadline to finish before the fronting proxy times out. +PRISM_PUBLIC_READ_STATEMENT_TIMEOUT_SECONDS=20 +# Read-only DSN for the standby the public tier serves from, using the +# PRISM_POSTGRES_* credentials the base backup cloned from the primary. This is +# deliberately not PRISM_DATABASE_URL: that one names the primary, and the +# coordinator needs it to. +# Leave empty for Compose to derive the replica URL from PRISM_POSTGRES_USER, +# PRISM_POSTGRES_PASSWORD, and PRISM_POSTGRES_DB. Set an explicit URL for a +# separately provisioned standby or different read credentials. +PRISM_PUBLIC_DATABASE_URL= +# Optional public-read psql command. Empty derives it from the public DSN +# above. This is separate from PRISM_POSTGRES_PSQL_COMMAND for the primary; +# use a replica command here when PRISM_POSTGRES_NATIVE_CLIENT=psql needs +# a custom wrapper or connection arguments. +PRISM_PUBLIC_PSQL_COMMAND= +# require = refuse replica-backed routes unless the backing server is in +# recovery and its replication stream is live within the bound below; off = +# serve whatever the DSN points at, which is the pre-replica behaviour. Leave +# this off until a standby is actually provisioned; see +# docs/prism-postgres-replica.md. +PRISM_PUBLIC_REPLICA_MODE=require +# How long the standby's replication stream may go silent before the read tier +# stops treating its answers as current. This bounds the *stream*, not replay +# lag: heartbeats keep flowing on an idle primary, replay lag does not. +PRISM_PUBLIC_REPLICA_MAX_LAG_SECONDS=60 PRISM_PUBLIC_POOL_NAME=PRISM PRISM_PUBLIC_STRATUM_HOST= +# REQUIRED for make up-prism-pool and prism-public-api. The API runs no +# Stratum listener, so it cannot infer the pool's endpoint. Left empty it +# refuses to start rather than +# advertise 127.0.0.1 to miners as the pool's Stratum URL. PRISM_PUBLIC_STRATUM_URL= # Optional external URL for the high-diff endpoint in /public/v1/mining-configuration. # Used only when PRISM_STRATUM_HIGHDIFF_PORT is set; empty derives from @@ -249,11 +334,85 @@ PRISM_STRATUM_VARDIFF_MAX_STEP_DOWN=4 PRISM_STRATUM_VARDIFF_START_DIFF=0.000000001 PRISM_STRATUM_VARDIFF_EWMA_ALPHA=0.4 PRISM_STRATUM_VARDIFF_RETARGET_TOLERANCE=0.25 +# Fast-arrival initial convergence. Credit is target-based: a session is paid +# for the difficulty its job was stamped at, so every retarget interval spent +# climbing toward the difficulty it is really working at is credit the miner +# never receives. A rental router that filters submissions at its own, much +# higher difficulty makes this acute -- the pool sees only the survivors, so +# the ordinary MAX_STEP_UP ladder needs several full RETARGET_SECONDS windows +# to catch up. These let ONE early move per connection, before the ordinary +# interval and with a wider step bound, close that gap. Set to 0 to restore +# the previous behavior exactly. +PRISM_STRATUM_VARDIFF_INITIAL_CONVERGENCE=1 +# Step bound for that one initial move; must be >= PRISM_STRATUM_VARDIFF_MAX_STEP_UP, +# which every later retarget goes back to. 64 is what lets a 16384 start cross +# the 500000 a NiceHash-style router filters at in a single move. +PRISM_STRATUM_VARDIFF_INITIAL_MAX_STEP_UP=64 +# Accepted shares of evidence required before the early move is allowed. +PRISM_STRATUM_VARDIFF_INITIAL_MIN_SHARES=8 +# Minimum observed/current difficulty gap required. This is the noise guard: +# at 8 shares an exactly-on-target miner still has a ~35% spread on its +# observed rate, so demanding a 4x gap keeps an ordinary miner from being +# pushed up merely because this path exists, and makes the early move +# upward-only. Lower it only with evidence. +PRISM_STRATUM_VARDIFF_INITIAL_MIN_STEP_UP=4 +# Minimum window length before an early observation counts, so a burst +# delivered in a near-zero interval cannot divide real work by an almost-zero +# elapsed time and explode the observed rate. +PRISM_STRATUM_VARDIFF_INITIAL_MIN_SECONDS=1 +# Observation only: the difficulty at which qbit_prism_vardiff_high_diff_arrival_* +# records how long, and how many accepted shares, a session needed to arrive. +# Never gates or clamps a retarget. Defaults to the high-diff listener floor. +# Set it inside the listener's MAX_DIFF or the arrival histograms stay empty. +PRISM_STRATUM_VARDIFF_HIGH_DIFF_ARRIVAL_DIFF=500000 # Set to 0 to disable idle zero-share vardiff step-down sweeps. PRISM_STRATUM_VARDIFF_IDLE_SWEEP_SECONDS=15 +# Reconnect difficulty resume: a reconnecting worker (same listener + exact +# username) starts at its last converged difficulty instead of the lane +# start, so a mass reconnect does not re-flood the share path while every +# session re-climbs. 0 restores lane-start behavior exactly. +# LIMITATION: retention is keyed by (listener, exact Stratum username) and is +# last-writer-wins, so heterogeneous rigs that deliberately share one username +# share one retained value: a small rig reconnecting after a big one can adopt +# the big rig's converged difficulty (and vice versa) until its own shares +# retarget it. The adopted value is still bounded by the lane clamp below. +# Give each rig its own worker name (address.rig) to keep the values separate; +# PRISM_STRATUM_MAX_CONNECTIONS_PER_USERNAME bounds how many connections can +# share one identity in the first place. +# With the PostgreSQL ledger, accepted-share evidence is also retained in +# qbit_worker_difficulty and preloaded before listeners accept, so recent +# converged values survive coordinator restarts. Non-PostgreSQL backends keep +# only the process-local cache and therefore start empty after a restart. +# The TTL is measured from accepted-share evidence recorded at a committed +# retarget or orderly disconnect. A long-running session that no longer +# retargets can therefore age out before an abrupt process exit even while it +# remains connected; this is bounded reconnect assistance, not an active- +# session lease or a guarantee that every restart resumes every worker. +PRISM_STRATUM_VARDIFF_RESUME=1 +# How long a retained difficulty stays adoptable. WARNING: when +# PRISM_STRATUM_VARDIFF_IDLE_SWEEP_SECONDS=0 (as deployed) there is no idle +# step-down sweep, so a session resumed above its true hashrate can only be +# corrected by an accepted share; keep this TTL short enough that retained +# values still reflect recent hashrate. The TTL always measures the age of the +# converged value itself: reconnecting and disconnecting without submitting a +# share does not extend it. +PRISM_STRATUM_VARDIFF_RESUME_TTL_SECONDS=900 +# Bound on retained per-worker entries (2x the deployed 4096 connection cap). +PRISM_STRATUM_VARDIFF_RESUME_MAX_ENTRIES=8192 +# Plausibility ceiling for a resumed value, as a multiple of the lane's own +# start difficulty (1024 = five 4x retarget steps). A retained value above +# min(lane max difficulty, start * factor) is clamped before adoption. +# WARNING: with PRISM_STRATUM_VARDIFF_IDLE_SWEEP_SECONDS=0 (as deployed), a +# rig that converged high and reconnects within the TTL on weaker hardware +# can resume too high, emit no shares, and give the retargeter no signal to +# come down — stuck for that connection's life (bounded by the TTL above; +# only that miner is harmed). On hardware where converged-high rigs stall +# after resume, lower this factor. +PRISM_STRATUM_VARDIFF_RESUME_MAX_START_FACTOR=1024 # Seconds a share computed against the just-replaced tip is still credited -# (never block-submitted). Mainnet requires 0; other chains may keep a small -# window that covers miner job-switch latency. +# (never block-submitted). Keep a small window that covers miner job-switch +# latency: at 0 every in-flight prior-tip share is rejected at each block, +# which miners read as pool failure. PRISM_STRATUM_STALE_GRACE_SECONDS=3 # Clean same-tip jobs remain valid against their original context for this # bounded window. Set the TTL to 0 to disable; positive TTLs require positive @@ -280,7 +439,22 @@ PRISM_PAYOUT_ARTIFACT_REANCHOR_SECONDS=60 PRISM_PAYOUT_ARTIFACT_MIN_BUILD_INTERVAL_SECONDS=60 PRISM_PAYOUT_ARTIFACT_FULL_RESCAN_SECONDS=3600 PRISM_PAYOUT_ARTIFACT_MAX_ANCHOR_AGE_SECONDS=300 +# Route the payout-window fold, canonical digest, and incremental advance +# through the persistent --serve audit builder instead of in-process Python. +# Strictly boolean (anything else refuses startup); inert while the daemon +# transport is disabled, and any daemon anomaly still degrades to the +# in-process pipeline for that materialization without flipping this. The +# daemon carries integers in fixed widths (share_difficulty/network_difficulty +# u128, share_seq/template_height u64, ntime u32, millisecond stamps i64); a +# window holding a value beyond them is declined as out_of_range and folded +# in-process, with the daemon kept alive. +PRISM_WINDOW_PIPELINE_RUST=0 PRISM_WORKER_METRICS_LIMIT=100 +# Coordinator maintenance of public hashrate rollups. Set 0 for raw share scans. +PRISM_HASHRATE_ROLLUP_ENABLED=1 +# Pause between caught-up passes; initial backfill runs batches back to back. +PRISM_HASHRATE_ROLLUP_INTERVAL_SECONDS=15 +PRISM_HASHRATE_ROLLUP_BATCH_SHARES=50000 # Global thread-per-connection capacity across every PRISM listener. The # production-facing default is above the normal 200-250 connection population; # set it explicitly to 0 only for focused local/regtest unlimited operation. @@ -299,6 +473,21 @@ PRISM_INITIAL_JOB_MAX_WORKERS=4 # concurrent build flights, so values above 2 are rejected; lower to 1 to # serialize builds on GIL-constrained single-core hosts. PRISM_JOB_BUILD_EXECUTOR_WORKERS=2 +# Interpreter thread switch interval, in seconds. Unset leaves CPython's own +# default (5ms) in force, which is what every deployment has run to date; +# setting it calls sys.setswitchinterval() before any thread starts. +# This trades share-ack tail latency against interpreter overhead, and the +# balance is host-specific -- #143's measurements moved throughput 15% and +# CPU-per-share 13% across a sweep, with the 5ms default showing the worst +# ack tail of the three settings tried. Re-measure per host with +# tests/perf/stratum_session_concurrency_load.py rather than copying a value +# between machines; a host with slower cores or a different core count will +# not share an optimum. Values above 1 are rejected at startup as a misplaced +# decimal point, and values below 1e-06 because CPython stores the interval in +# whole microseconds and would truncate them to zero -- a switch check at every +# opportunity. For the same reason a tuned value may read back rounded down to +# the nearest microsecond. +# PRISM_PYTHON_SWITCH_INTERVAL_SECONDS=0.002 # Suppress mining-delivery health failures during bounded startup preparation. PRISM_MINING_HEALTH_STARTUP_GRACE_SECONDS=30 # Keep listener threads alive during transient process/system descriptor @@ -371,13 +560,47 @@ PRISM_POSTGRES_IMAGE=postgres:16-alpine PRISM_POSTGRES_DB=qbit PRISM_POSTGRES_USER=qbit PRISM_POSTGRES_PASSWORD=change-this +# Replica bootstrap retry budget and delay, forwarded to the standby container. +PRISM_POSTGRES_REPLICA_BASEBACKUP_ATTEMPTS=60 +PRISM_POSTGRES_REPLICA_BASEBACKUP_RETRY_SECONDS=5 PRISM_DATABASE_URL=postgresql://qbit:change-this@prism-postgres:5432/qbit PRISM_POSTGRES_PSQL_COMMAND= +# Initialization applies 001_share_ledger.sql and 002_candidate_bodies.sql, +# including durable vardiff state and candidate-body storage. If disabled, +# apply both files from crates/qbit-prism/sql in that order before startup. +# The coordinator and offline recovery command refuse an unmigrated database. PRISM_POSTGRES_INIT_SCHEMA=1 PRISM_POSTGRES_READ_CONCURRENCY=4 +# Issue #255: block-candidate bodies are written as bounded 256 KiB chunks +# (storage version 2). Set to 1 only for the compatibility/rollback release +# that must keep writing the legacy whole-jsonb candidate; the reader handles +# both versions. Once any version-2 row exists, a pre-#255 coordinator must +# not be started against the database. +PRISM_CANDIDATE_STORAGE_VERSION=2 +# Local spool directory for hydrated replay bodies (empty: the system temp +# directory) and the byte reservation that bounds concurrent spool usage. +PRISM_CANDIDATE_SPOOL_DIR= +PRISM_CANDIDATE_SPOOL_RESERVATION_BYTES=4294967296 +# How long the server lets a coordinator session sit idle inside an open +# transaction before aborting it and releasing its locks. Bounds how long a +# vanished client (partition, SIGSTOP, VM pause) can strand the writer-lease +# row lock and block a successor's startup. +PRISM_POSTGRES_IDLE_IN_TRANSACTION_TIMEOUT_SECONDS=15 +# Server-side TCP keepalives toward coordinator sessions, the backstop for +# backends the idle-in-transaction timer cannot cover. On a TCP connection +# whose platform implements them, the defaults bound dead-socket detection at +# idle + count x interval = 30 + 3x10 = 60s instead of the OS default of +# roughly two hours; Unix-socket DSNs ignore them. +PRISM_POSTGRES_TCP_KEEPALIVES_IDLE_SECONDS=30 +PRISM_POSTGRES_TCP_KEEPALIVES_INTERVAL_SECONDS=10 +PRISM_POSTGRES_TCP_KEEPALIVES_COUNT=3 # How long the coordinator reuses one pool-wide PRISM reward-window aggregate # across public miner-page requests before re-running the recursive window # scan; 0 recomputes on every request. +# Read by the ledger, so prism-coordinator and prism-public-api both set it. +# It is also the one cache stacked under another on the public surface, which +# is why /public/v1/miners/{recipient_id} carries a larger staleness budget +# (105s) than the other plain row reads (15s). PRISM_PUBLIC_REWARD_WINDOW_CACHE_SECONDS=30 PRISM_SHARE_COMMIT_BATCH_SIZE=64 PRISM_SHARE_COMMIT_LINGER_MILLISECONDS=5 @@ -395,6 +618,14 @@ PRISM_LEDGER_WRITER_EPOCH=1 PRISM_LEDGER_WRITER_SESSION_TOKEN= PRISM_LEDGER_WRITER_PUBLIC_KEY_HEX= PRISM_LEDGER_LEASE_TTL_SECONDS=60 +# Per-attempt lock/statement deadline and attempt budget for the startup +# writer-lease acquisition. Attempts x deadline (5x5s = 25s) is sized to +# outlast PRISM_POSTGRES_IDLE_IN_TRANSACTION_TIMEOUT_SECONDS, so an orphaned +# lock is normally reaped mid-budget and startup self-heals; an acquisition +# that never completes within the budget exits the coordinator visibly +# instead of blocking for hours. +PRISM_LEDGER_LEASE_ACQUIRE_LOCK_TIMEOUT_SECONDS=5 +PRISM_LEDGER_LEASE_ACQUIRE_ATTEMPTS=5 # Bounds durable writer flushing before shutdown leaves the lease to its TTL. PRISM_WRITER_QUIESCENCE_TIMEOUT_SECONDS=15 PRISM_MANIFEST_SIGNING_SEED_HEX= diff --git a/.github/scripts/test-qbit-image-runtime.sh b/.github/scripts/test-qbit-image-runtime.sh index 7607a5fe9..9e11ea25e 100644 --- a/.github/scripts/test-qbit-image-runtime.sh +++ b/.github/scripts/test-qbit-image-runtime.sh @@ -2,6 +2,9 @@ set -euo pipefail QBIT_IMAGE="${QBIT_IMAGE:-${QBIT_CI_IMAGE:-}}" +# Resolved from this script's own location so the PRISM difficulty helpers +# are importable regardless of the caller's working directory. +REPO_ROOT="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")/../.." && pwd)" TIP_AGE_SECONDS=123456789 CONTAINER_PREFIX="qbit-runtime-smoke-${GITHUB_RUN_ID:-local}-${GITHUB_RUN_ATTEMPT:-0}-$$" REGTEST_CONTAINER="${CONTAINER_PREFIX}-regtest" @@ -129,6 +132,105 @@ assert_no_tip_age_arg() { done <<< "${argv}" } +scaled_difficulty_from_bits() { + # Derive one compact-bits value's PRISM-scaled difficulty by calling the + # production helpers themselves rather than restating their arithmetic. + # Collapse converts an occupying block's ``getblockheader.bits`` with + # ``block_candidates._collapse_scaled_difficulty`` and compares the result + # against a candidate row stamped by + # ``template_artifacts.scaled_network_difficulty``; a smoke that carried its + # own copy of that formula could keep passing while either side drifted, so + # both are invoked here and required to agree. Both import closures are + # standard-library only, so this needs nothing but the checked out tree and + # the runner's python3. + local bits="$1" + + python3 - "${REPO_ROOT}" "${bits}" <<'PYTHON' +import sys + +repo_root, bits = sys.argv[1], sys.argv[2] +sys.path.insert(0, repo_root) + +try: + from lab.prism.block_candidates import _collapse_scaled_difficulty + from lab.prism.template_artifacts import scaled_network_difficulty +except Exception as exc: + raise SystemExit(f"PRISM difficulty helpers are unimportable: {exc!r}") + +collapse = _collapse_scaled_difficulty(bits) +if collapse is None: + raise SystemExit( + f"collapse rejected the compact bits the node reported: {bits!r}" + ) +try: + template = scaled_network_difficulty(bits) +except Exception as exc: + raise SystemExit( + "template scaling rejected the compact bits the node reported: " + f"{bits!r} ({exc!r})" + ) +if collapse != template: + raise SystemExit( + "collapse and template scaled difficulty disagree for " + f"{bits!r}: {collapse} != {template}" + ) +if collapse <= 0: + raise SystemExit(f"scaled difficulty for {bits!r} is not positive: {collapse}") +print(collapse) +PYTHON +} + +assert_header_bits_contract() { + # Collapse reads an occupying block's work out of ``getblockheader.bits`` and + # fails its entire page closed unless that field is an eight-character + # compact hex string with a positive target. Nothing else in this smoke + # proves a production qbitd still answers in that shape, so assert it here + # against a real header from the running node, on the same + # getbestblockhash/getblockheader pair collapse itself calls. + local container="$1" + local chain_arg="$2" + local block_hash + local header + local bits + local scaled + + block_hash="$( + docker exec "${container}" \ + qbit-cli "${chain_arg}" -datadir=/var/lib/qbit getbestblockhash + )" || fail "${container} did not answer getbestblockhash" + block_hash="$(printf '%s' "${block_hash}" | tr -d '[:space:]')" + [[ "${block_hash}" =~ ^[0-9a-fA-F]{64}$ ]] || + fail "${container} getbestblockhash is not a block hash: ${block_hash}" + + header="$( + docker exec "${container}" \ + qbit-cli "${chain_arg}" -datadir=/var/lib/qbit getblockheader "${block_hash}" + )" || fail "${container} did not answer getblockheader for ${block_hash}" + + bits="$( + printf '%s\n' "${header}" | + sed -n 's/.*"bits"[[:space:]]*:[[:space:]]*"\([^"]*\)".*/\1/p' + )" + bits="${bits%%$'\n'*}" + [[ "${bits}" =~ ^[0-9a-fA-F]{8}$ ]] || + fail "$( + printf '%s getblockheader.bits for %s is not eight hex characters: %s\nheader: %s' \ + "${container}" "${block_hash}" "${bits:-}" "${header}" + )" + + # Deliberately no shell arithmetic on the derived value: mainnet-scale + # difficulties exceed 2**63 and would wrap to a negative in bash's signed + # 64-bit integers. The helper already refuses a non-positive result, and the + # digit-shape check below stays exact at any magnitude. + scaled="$(scaled_difficulty_from_bits "${bits}")" || + fail "${container} header bits ${bits} did not derive PRISM's scaled difficulty" + [[ "${scaled}" =~ ^[1-9][0-9]*$ ]] || + fail "${container} header bits ${bits} derived a non-positive scaled difficulty: ${scaled}" + + printf 'qbit runtime image smoke: %s header %s bits %s scaled difficulty %s\n' \ + "${container}" "${block_hash}" "${bits}" "${scaled}" +} + start_mainnet_container() { local container="$1" local readiness_gate="$2" @@ -194,6 +296,7 @@ wait_for_rpc "${REGTEST_CONTAINER}" -regtest regtest processes="$(docker top "${REGTEST_CONTAINER}" -eo pid,comm)" [[ "${processes}" == *"tini"* && "${processes}" == *"qbitd"* ]] || fail "runtime process chain does not contain both tini and qbitd: ${processes}" +assert_header_bits_contract "${REGTEST_CONTAINER}" -regtest require_clean_exit "${REGTEST_CONTAINER}" start_mainnet_container "${PRELAUNCH_CONTAINER}" 0 0 diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index df3653b8e..49ffbc1e3 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -2,15 +2,23 @@ name: CI on: pull_request: + # Pull requests are tested as their merge result; a push to a PR branch + # would run the same jobs a second time. Push runs cover direct commits + # to the long-lived branches only. push: + branches: + - main + - 1.x.x + - 2.x.x + - 3.x.x workflow_dispatch: permissions: contents: read jobs: - checks: - name: Lint, compile, and test + lint-compile-compose: + name: Lint, compile, and Compose validation runs-on: blacksmith-2vcpu-ubuntu-2404 steps: @@ -20,10 +28,7 @@ jobs: - name: Set up Python uses: actions/setup-python@v6 with: - python-version: "3.12" - - - name: Set up Rust - uses: dtolnay/rust-toolchain@stable + python-version: "3.14" - name: Install lint tools run: | @@ -31,7 +36,7 @@ jobs: sudo apt-get install -y --no-install-recommends shellcheck - name: Check shell syntax - run: git ls-files '*.sh' | xargs bash -n + run: git ls-files -z '*.sh' | xargs -0 -n 1 bash -n - name: ShellCheck shell scripts run: | @@ -41,21 +46,9 @@ jobs: - name: Compile Python sources run: python -m compileall -q docker lab tests examples scripts - - name: Run Python unit tests - run: python -m unittest discover -s tests -p 'test_*.py' - - - name: Run Postgres ledger integration tests - run: make test-prism-postgres-ledger - - - name: Run Postgres production-scale fixture tests - run: make test-prism-postgres-scale - - name: Validate mainnet Compose contract run: python -m unittest tests.test_mainnet_compose_contract - - name: Run Rust tests - run: cargo test --locked --workspace --all-targets - - name: Validate Docker Compose files run: | export QBIT_SRC_DIR="$PWD" @@ -70,6 +63,85 @@ jobs: config --quiet done + python-tests: + name: Python unit tests (shard ${{ matrix.shard }}) + runs-on: blacksmith-2vcpu-ubuntu-2404 + strategy: + fail-fast: false + matrix: + shard: [0, 1, 2, 3] + + steps: + - name: Check out repository + uses: actions/checkout@v7 + + - name: Set up Python + uses: actions/setup-python@v6 + with: + python-version: "3.14" + + - name: Set up Rust + uses: dtolnay/rust-toolchain@stable + + - name: Run Python unit test shard + env: + SHARD_INDEX: ${{ matrix.shard }} + SHARD_COUNT: ${{ strategy.job-total }} + run: GIT_CONFIG_GLOBAL=/dev/null python scripts/run_python_test_shard.py --shard-index "$SHARD_INDEX" --shard-count "$SHARD_COUNT" + + postgres-tests: + name: Postgres tests (${{ matrix.suite }}) + runs-on: blacksmith-2vcpu-ubuntu-2404 + strategy: + fail-fast: false + matrix: + suite: [ledger, scale, native-ledger, candidate-storage] + + steps: + - name: Check out repository + uses: actions/checkout@v7 + + - name: Set up Python + uses: actions/setup-python@v6 + with: + python-version: "3.14" + + - name: Run Postgres test suite + env: + SUITE: ${{ matrix.suite }} + run: | + python -m pip install 'psycopg[binary]==3.2.*' + make "test-prism-postgres-$SUITE" + + rust-tests: + name: Rust tests + runs-on: blacksmith-2vcpu-ubuntu-2404 + + steps: + - name: Check out repository + uses: actions/checkout@v7 + + - name: Set up Rust + uses: dtolnay/rust-toolchain@stable + + - name: Run Rust tests + run: cargo test --locked --workspace --all-targets + + checks: + # Preserve the existing check name for branch protection and consumers. + name: Lint, compile, and test + if: ${{ always() }} + needs: [lint-compile-compose, python-tests, postgres-tests, rust-tests] + runs-on: blacksmith-2vcpu-ubuntu-2404 + + steps: + - name: Require every check to pass + env: + RESULTS: ${{ toJSON(needs) }} + run: | + echo "$RESULTS" + echo "$RESULTS" | jq -e 'all(.[]; .result == "success")' + ckpool-rejects-observability: name: ckpool rejects observability live suite runs-on: blacksmith-2vcpu-ubuntu-2404 @@ -82,7 +154,7 @@ jobs: - name: Set up Python uses: actions/setup-python@v6 with: - python-version: "3.12" + python-version: "3.14" - name: Install ckpool build dependencies run: | @@ -106,7 +178,7 @@ jobs: - name: Set up Python uses: actions/setup-python@v6 with: - python-version: "3.12" + python-version: "3.14" - name: Install production PostgreSQL client dependency run: python -m pip install "psycopg[binary]==3.2.*" @@ -117,6 +189,9 @@ jobs: - name: Run native PostgreSQL ledger integration tests run: make test-prism-postgres-native-ledger + - name: Run carry-forward seed guard regression tests + run: make test-prism-postgres-seed-guard + docker-builds: name: Build lightweight Docker images runs-on: blacksmith-2vcpu-ubuntu-2404 @@ -147,9 +222,32 @@ jobs: docker buildx build \ --build-context qbit-src=/tmp/qbit-functional-fixture \ --load \ + --tag qbit-prism:ci \ -f lab/prism/Dockerfile \ . + - name: Run payout-window parity with Python adapter + run: | + docker run --rm \ + --volume "$PWD:/workspace:ro" \ + --workdir /workspace \ + --env PYTHONDONTWRITEBYTECODE=1 \ + --env PYTHONPATH=/workspace \ + --env QBIT_WINDOW_PIPELINE_PARITY_ADAPTER=python \ + qbit-prism:ci \ + python3 -m tests.window_pipeline_parity_gate + + - name: Run payout-window parity with real Rust daemon + run: | + docker run --rm \ + --volume "$PWD:/workspace:ro" \ + --workdir /workspace \ + --env PYTHONDONTWRITEBYTECODE=1 \ + --env PYTHONPATH=/workspace \ + --env QBIT_WINDOW_PIPELINE_PARITY_ADAPTER=rust-daemon \ + qbit-prism:ci \ + python3 -m tests.window_pipeline_parity_gate + qbit-runtime-image: name: Build and smoke real qbitd image if: github.event_name != 'push' || github.ref == 'refs/heads/main' @@ -166,6 +264,14 @@ jobs: - name: Set up Blacksmith Docker builder uses: useblacksmith/setup-docker-builder@v1 + # The runtime smoke derives its header-bits assertion from PRISM's own + # difficulty helpers, so this job needs the same interpreter the unit + # tests run under rather than whatever the runner image happens to ship. + - name: Set up Python + uses: actions/setup-python@v6 + with: + python-version: "3.14" + - name: Stage pinned qbit source run: | set -euo pipefail diff --git a/Cargo.lock b/Cargo.lock index 85feb0ec5..41aaed6d2 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -144,7 +144,7 @@ dependencies = [ [[package]] name = "qbit-pool-builder" -version = "1.1.0" +version = "2.0.0" dependencies = [ "ed25519-dalek", "hex", @@ -156,7 +156,7 @@ dependencies = [ [[package]] name = "qbit-prism" -version = "1.1.0" +version = "2.0.0" dependencies = [ "hex", "qbit-pool-builder", diff --git a/Makefile b/Makefile index cb70518be..23b41aa76 100644 --- a/Makefile +++ b/Makefile @@ -59,7 +59,7 @@ compose_env_value() { \ }; endef -.PHONY: doctor prism-self-check require-lab-mode test-builder test-builder-regtest test-prism-regtest test-prism-postgres-ledger test-prism-postgres-scale test-prism-postgres-native-ledger test-prism-postgres-throughput test-prism-stratum-regtest-live test-prism-stratum-postgres-regtest-live test-prism-combined-regtest test-compose-prism-config up up-permissionless up-permissionless-pool test-permissionless test-permissionless-p2mr test-ckpool-bip310 up-real-miner up-permissionless-real test-real-miner up-auxpow up-auxpow-bridge up-auxpow-pool up-prism up-prism-pool up-dual-pools test-auxpow test-auxpow-stratum test-auxpow-stratum-bip310 test-auxpow-stratum-age smoke-all down purge-local-volumes +.PHONY: doctor prism-self-check require-lab-mode test-builder test-builder-regtest test-prism-regtest test-prism-postgres-ledger test-prism-postgres-scale test-prism-postgres-native-ledger test-prism-postgres-seed-guard test-prism-postgres-throughput test-prism-public-read-replica test-prism-stratum-regtest-live test-prism-stratum-postgres-regtest-live test-prism-combined-regtest test-compose-prism-config up up-permissionless up-permissionless-pool test-permissionless test-permissionless-p2mr test-ckpool-bip310 up-real-miner up-permissionless-real test-real-miner up-auxpow up-auxpow-bridge up-auxpow-pool up-prism up-prism-pool up-dual-pools test-auxpow test-auxpow-stratum test-auxpow-stratum-bip310 test-auxpow-stratum-age smoke-all down purge-local-volumes require-lab-mode: @bash scripts/check-env.sh --require-lab @@ -99,9 +99,18 @@ test-prism-postgres-scale: test-prism-postgres-native-ledger: bash test/test-prism-postgres-native-ledger.sh +test-prism-postgres-candidate-storage: + bash test/test-prism-postgres-candidate-storage.sh + +test-prism-postgres-seed-guard: + bash test/test-prism-postgres-seed-guard.sh + test-prism-postgres-throughput: bash test/test-prism-postgres-throughput.sh +test-prism-public-read-replica: + bash test/test-prism-public-read-replica.sh + test-prism-stratum-regtest-live: @$(WITH_RESOLVED_QBIT) \ bash test/test-prism-stratum-regtest-live.sh @@ -269,14 +278,15 @@ up-prism-pool: @$(WITH_RESOLVED_QBIT) \ $(COMPOSE_ENV_HELPERS) \ missing=0; \ - for name in PRISM_MANIFEST_SIGNING_SEED_HEX PRISM_LEDGER_ATTESTATION_SIGNING_SEED_HEX PRISM_LEDGER_WRITER_PUBLIC_KEY_HEX; do \ - if [ -z "$$(compose_env_value "$$name")" ]; then \ + for name in PRISM_MANIFEST_SIGNING_SEED_HEX PRISM_LEDGER_ATTESTATION_SIGNING_SEED_HEX PRISM_LEDGER_WRITER_PUBLIC_KEY_HEX PRISM_PUBLIC_STRATUM_URL; do \ + value="$$(compose_env_value "$$name")"; \ + if [[ -z "$${value//[[:space:]]/}" ]]; then \ printf 'prism operator env: %s is required\n' "$$name" >&2; \ missing=1; \ fi; \ done; \ if [ "$${missing}" -ne 0 ]; then \ - printf 'prism operator env: set real PRISM signing keys in .env before running make up-prism-pool\n' >&2; \ + printf 'prism operator env: set PRISM signing keys and PRISM_PUBLIC_STRATUM_URL in .env or DEPLOY_ENV_FILE before running make up-prism-pool\n' >&2; \ printf 'prism operator env: keep PRISM_ALLOW_TEST_SIGNING_SEEDS=0 and PRISM_ALLOW_BUNDLE_EMBEDDED_LEDGER_KEY=0 for deploys\n' >&2; \ exit 1; \ fi; \ @@ -292,9 +302,9 @@ up-prism-pool: fi; \ printf 'audit HTTP stays inside the coordinator namespace at %s:%s\n' "$$(compose_env_value PRISM_AUDIT_BIND 127.0.0.1)" "$$(compose_env_value PRISM_AUDIT_PORT 3341)"; \ if [[ "$$(operator_build_mode)" == no-build ]]; then \ - $(PRODUCTION_COMPOSE) --profile prism up -d --no-build --pull never qbitd prism-postgres prism-coordinator; \ + $(PRODUCTION_COMPOSE) --profile prism up -d --no-build --pull never qbitd prism-postgres prism-coordinator prism-public-api; \ else \ - $(COMPOSE) --profile prism up --build qbitd prism-postgres prism-coordinator; \ + $(COMPOSE) --profile prism up --build qbitd prism-postgres prism-coordinator prism-public-api; \ fi up-dual-pools: export MINING_LANES=ckpool,auxpow diff --git a/PRISM.md b/PRISM.md index 3fc1c4b29..488a6400f 100644 --- a/PRISM.md +++ b/PRISM.md @@ -231,6 +231,9 @@ Operational knobs shared by the PRISM listeners: | `PRISM_STRATUM_STALE_GRACE_SECONDS` | `3` | after a tip flip, credits same-connection prior-tip shares until this long after that connection receives new-tip work (shares stay creditable while delivery is still pending); set `0` to reject all prior-tip shares | | `PRISM_STRATUM_VARDIFF_IDLE_SWEEP_SECONDS` | `15` | cadence for checking zero-submitted, zero-accepted vardiff windows so over-diffed idle miners can step down; set `0` to disable | | `PRISM_WORKER_METRICS_LIMIT` | `100` | maximum distinct worker labels in private metrics before new workers aggregate into `_other` | +| `PRISM_HASHRATE_ROLLUP_ENABLED` | `1` | coordinator background maintenance of the incremental hashrate rollups the public hashrate-series endpoint serves from; set `0` to fall back to raw share scans | +| `PRISM_HASHRATE_ROLLUP_INTERVAL_SECONDS` | `15` | sleep between hashrate rollup maintenance passes once caught up | +| `PRISM_HASHRATE_ROLLUP_BATCH_SHARES` | `50000` | shares folded into the hashrate rollups per pass; catch-up (including the initial backfill of a grown ledger) loops passes back to back | `/healthz` remains healthy across arbitrarily long periods without a new block when the observed template and payout generation are unchanged. It returns HTTP @@ -260,8 +263,9 @@ are still counted under the existing rejection reason IDs. The one exception is a hash that solves a block while missing the share target (possible while the listener floor sits above network difficulty): its share credit lands only when qbitd accepts the block, as before. Audit bundles containing any -`credit_policy` row use `qbit.prism.audit-bundle.v1.1`; upgrade mirrors and -verifiers before enabling a non-zero stale-grace window in production. +`credit_policy` row use `qbit.prism.audit-bundle.v1.1`; the stale-grace window +is non-zero by default on every chain, so keep mirrors and verifiers on a +release that accepts that schema. ## How Reward Accounting Works @@ -519,10 +523,22 @@ API readers. ## HTTP Surfaces -The coordinator exposes a private audit/ops listener and a dashboard-safe public -API from the same process. +PRISM serves two HTTP surfaces from two separate processes. -Private/internal endpoints include: +The **coordinator** (`prism-coordinator`) exposes only the private audit/ops +listener. The **public read tier** (`prism-public-api`, `python3 -m +lab.prism.public_read_service`) serves the dashboard-safe `/public/v1` API and +nothing else. They were split because public read traffic scales with public +interest rather than with hashrate: served in-process it put an uncontrolled +workload on the same GIL that acknowledges shares and lands blocks, and on the +same primary Postgres the lease-holding writer commits through. + +The public tier reads through bounded read slots only, never acquires a writer +lease, and depends on Postgres rather than on the coordinator — so it keeps +serving across coordinator restarts, and a surge of dashboard polling cannot +slow block landing. + +Private/internal endpoints, served by the coordinator, include: - `/healthz` - `/metrics` @@ -539,13 +555,40 @@ Private/internal endpoints include: Do not expose `/audit/*`, `/metrics`, `/healthz`, Postgres, qbit RPC, or Docker volumes directly to the internet. -Dashboard-safe endpoints live under `/public/v1`. The public API contract is: +Dashboard-safe endpoints live under `/public/v1`, served by `prism-public-api` +on `PRISM_PUBLIC_API_PORT` (default `3342`). The public API contract is: - [docs/public-dashboard-api/README.md](docs/public-dashboard-api/README.md) - [docs/public-dashboard-api-v1.openapi.yaml](docs/public-dashboard-api-v1.openapi.yaml) Operators can expose only `/public/v1` through a reverse proxy or dashboard -frontend. +frontend. Because that surface now has its own process, the reverse proxy points +at `prism-public-api` rather than at the coordinator's audit port, and the +coordinator's listener need not be reachable from the proxy at all. + +The public tier also serves its own `/healthz` and `/metrics` describing that +process (readiness, request counts, cache hit/miss, staleness refusals). These +are operator surfaces like the coordinator's own — do not expose them publicly. + +`prism-public-api` refuses to start unless `PRISM_PUBLIC_STRATUM_URL` is set: it +runs no Stratum listener, so it cannot infer the pool's endpoint, and the +fallback would advertise `127.0.0.1` to miners. It also refuses to run on the +in-memory ledger. + +### Hashrate rollups + +`/public/v1/hashrate-series` is served from incremental per-bucket rollups +(`qbit_hashrate_rollup_pool` / `qbit_hashrate_rollup_miner`) rather than by +re-aggregating raw share rows, so its cost tracks buckets in range instead of +shares in range and long ranges (`1m`, `6m`, `all`) stay cheap as the ledger +grows. The coordinator maintains the rollups in a background loop (the +`PRISM_HASHRATE_ROLLUP_*` knobs above) that folds shares past a single durable +watermark; a cold or pre-existing database backfills itself through the same +loop with no separate migration, and the serving query merges a live tail of +not-yet-rolled-up shares — mid-backfill included — so responses stay exactly +equal to the raw aggregation. A database missing the rollup tables or +watermark entirely is served from the unchanged raw scan. The public read +tier only reads the rollups; it never maintains them. ## Run PRISM Locally @@ -597,6 +640,11 @@ profile and requires `minimum <= start <= maximum`. Capacity qualification is optional and external to startup; see [docs/prism-capacity-readiness.md](docs/prism-capacity-readiness.md). +Set `PRISM_PUBLIC_STRATUM_URL` in the same environment file to your pool's +miner-facing endpoint, for example `stratum+tcp://pool.example:3340` with your +actual hostname and port. The public API advertises this URL, and +`make up-prism-pool` rejects a missing or blank value before starting services. + Start the pool: ```bash diff --git a/README.md b/README.md index 189c8ed56..e99e9239a 100644 --- a/README.md +++ b/README.md @@ -89,9 +89,9 @@ configuration. It exits non-zero on hard failures. ### Run PRISM Pool -Direct PRISM Stratum requires Postgres and three key values before -`make up-prism-pool` starts. Generate unique deployment seeds and derive the -trusted ledger public key from the ledger attestation seed: +Direct PRISM Stratum requires Postgres, three key values, and an advertised +Stratum URL before `make up-prism-pool` starts. Generate unique deployment +seeds and derive the trusted ledger public key from the ledger attestation seed: ```bash PRISM_MANIFEST_SIGNING_SEED_HEX="$(openssl rand -hex 32)" @@ -112,7 +112,15 @@ rejected, and the vardiff bounds must satisfy `minimum <= start <= maximum`. The public ledger key must be distributed to verifiers out of band; do not ask verifiers to trust a key copied from the audit bundle they are checking. -Then start and validate the pool: +Also set `PRISM_PUBLIC_STRATUM_URL` in the same environment file to the endpoint +miners can reach. Replace the example hostname and port with your pool's: + +```dotenv +PRISM_PUBLIC_STRATUM_URL=stratum+tcp://pool.example:3340 +``` + +The target rejects a missing or blank URL before starting services. Then start +and validate the pool: ```bash make up-prism-pool @@ -120,8 +128,8 @@ make prism-self-check ``` `make up-prism-pool` prints the Stratum URL. Miners authorize with -`[.]` usernames. The coordinator serves `/public/v1` -from its audit HTTP listener for dashboard-safe read models. Keep `/audit/*`, +`[.]` usernames. The separate `prism-public-api` +service serves `/public/v1` on port `3342` by default. Keep `/audit/*`, `/metrics`, `/healthz`, Postgres, qbit RPC, ckpool command sockets, and Docker volumes private unless you intentionally proxy them with access controls. See [`docs/prism-storage-sizing.md`](docs/prism-storage-sizing.md) for PRISM @@ -201,8 +209,8 @@ shape, and receives explicit `CKPOOL_MINDIFF` and `CKPOOL_STARTDIFF` values. Regtest keeps the lab-only `1/256` difficulty floor. The one launch-only exception is explicitly authorized mainnet prelaunch. It -requires the five authorization values below plus the explicit mainnet daemon -selector; missing, invalid, or mismatched values fail closed: +requires the authorization values, explicit mainnet daemon selector and mask +policy below; missing, invalid, or mismatched values fail closed: ```bash QBIT_CHAIN=mainnet @@ -211,13 +219,18 @@ QBIT_PRODUCTION=1 QBIT_TOOLS_PRODUCTION=1 CKPOOL_NON_TEST_READINESS_GATE=0 QBIT_MAINNET_LAUNCH_READINESS_CHECKS_ENABLED=0 +CKPOOL_VERSION_MASK_MODE=static +CKPOOL_VERSION_MASK=1fffe000 ``` Preflight still checks the RPC chain, the mandatory mainnet genesis hash, static qbit assumptions, difficulty policy, and P2MR payout address, but defers launch-only IBD, peer, live-template, freshness, and active-tip checks. CKPool remains running with its Stratum listener bound and retries GBT until qbitd can serve -work. At launch, set both readiness flags to `1` and restart or redeploy CKPool; +work. This requires the explicit static mask above while templates are +unavailable; dynamic mode aborts startup without a successful template. At +launch, set both readiness flags to `1`, restore +`CKPOOL_VERSION_MASK_MODE=dynamic`, and restart or redeploy CKPool; the new supervisor then checks IBD, peer count, live-template freshness, and active-tip agreement continuously. Runtime environment changes are not hot-reloaded by an already-running supervisor. @@ -270,7 +283,7 @@ exceeds `AUXPOW_TEMPLATE_MAX_AGE_SECONDS` (default 120), or after sits below qbit's default 60 minute `-auxpowtemplateexpiry`, so both parent work and the cached qbit candidate receive bounded replacement. -The AuxPoW Stratum bridge advertises parent-chain work, enables per-miner vardiff by default, and uses `AUXPOW_STRATUM_VERSION_MASK=1fffe000` for BIP310 version rolling. qbit child candidates still come from `createauxblock`; permissionless qbit, ckpool, and direct PRISM Stratum prefer the connected qbit node's `getblocktemplate.versionrollingmask` when present. Older or unavailable GBT probes fall back to each service's configured mask: `CKPOOL_VERSION_MASK` for ckpool and `PRISM_VERSION_ROLLING_MASK` for direct PRISM. For byte-order investigations, set `AUXPOW_STRATUM_DIAG_VARIANTS=1` and optionally `AUXPOW_STRATUM_DIAG_JSONL=1`; normal operation uses `AUXPOW_STRATUM_HEADER_VARIANT=canonical`. +The AuxPoW Stratum bridge advertises parent-chain work, enables per-miner vardiff by default, and uses `AUXPOW_STRATUM_VERSION_MASK=1fffe000` for BIP310 version rolling. qbit child candidates still come from `createauxblock`; permissionless qbit, ckpool, and direct PRISM Stratum prefer the connected qbit node's `getblocktemplate.versionrollingmask` when present. Nodes that answer but do not advertise the field fall back to each service's configured mask: `CKPOOL_VERSION_MASK` for ckpool and `PRISM_VERSION_ROLLING_MASK` for direct PRISM. ckpool's dynamic mode fails closed when `getblocktemplate` cannot be retrieved at all. For byte-order investigations, set `AUXPOW_STRATUM_DIAG_VARIANTS=1` and optionally `AUXPOW_STRATUM_DIAG_JSONL=1`; normal operation uses `AUXPOW_STRATUM_HEADER_VARIANT=canonical`. Vardiff defaults target one accepted share every 5 seconds, start miners at difficulty 8192, clamp the minimum advertised difficulty to 1024, and retarget every 120 seconds using share-weighted work with EWMA smoothing. By default retargets send `mining.set_difficulty` and defer the new share target until the next natural job refresh (`AUXPOW_STRATUM_VARDIFF_APPLY_MODE=next_job`), which avoids sending clean replacement jobs with identical header space. Set `AUXPOW_STRATUM_VARDIFF_APPLY_MODE=clean_job` only for miners that require immediate clean-job difficulty enforcement. @@ -334,13 +347,26 @@ make up-permissionless-pool That starts `qbitd + ckpool` and prints the local Stratum endpoint. Point any compatible SHA256d Stratum miner at that port and use a qbit payout address as the username. If `QBIT_MINER_ADDRESS=auto`, the compose helpers derive the chain-default payout address from the node first. On public chains that resolves to P2MR. By default, ckpool starts with `CKPOOL_VERSION_MASK_MODE=dynamic`: it asks qbitd -for `getblocktemplate` and uses the advertised `versionrollingmask` when the -connected node exposes it. Older qbitd builds that do not expose the field fall back to -the configured `CKPOOL_VERSION_MASK`; the sample env uses `1fffe000` because -current qbitd permissionless templates advertise that mask. The selected mask is -logged and rendered into `/etc/ckpool/ckpool.conf`. If qbitd advertises +for `getblocktemplate` and uses the advertised `versionrollingmask`. The probe +runs only after the startup supervisor's readiness gate and live-template checks +have passed, so the mask reflects a node that is actually serving templates +rather than one that is still starting up. Dynamic mode fails closed: if +`getblocktemplate` cannot be retrieved after +`CKPOOL_VERSION_MASK_PROBE_ATTEMPTS` tries (spaced by +`CKPOOL_VERSION_MASK_PROBE_RETRY_SECONDS`), startup aborts before any config is +written instead of freezing a fallback mask for the life of the process. + +A node that answers successfully but omits `versionrollingmask` is an older +qbitd build that does not advertise one; ckpool then uses the configured +`CKPOOL_VERSION_MASK`. The sample env uses `1fffe000` because current qbitd +permissionless templates advertise that mask. If qbitd advertises `versionrollingmask=00000000`, ckpool disables BIP310 version rolling for that -node. +node. The selected mask is logged and rendered into `/etc/ckpool/ckpool.conf`. + +Set `CKPOOL_VERSION_MASK_MODE=static` to skip the probe entirely and use +`CKPOOL_VERSION_MASK` as configured. That is the explicit escape for deployments +that must start while `getblocktemplate` is unavailable, including authorized +mainnet prelaunch. CKPool also exposes its private command socket below `CKPOOL_SOCK_DIR` (`/tmp/qbitlab` by default). Override that path only to share the Unix socket @@ -445,6 +471,8 @@ qbit has its own JSON-RPC defaults. The qbit-specific `8355` value is the mainne ## Operator Notes +- Release preparation, publication, and development-branch synchronization are + documented in [Publishing a release](docs/releasing.md). - Public qbit networks use P2MR-only wallet output types. Use default wallet-generated payout addresses, and keep payout transactions inside qbit's allowed output policy. - Direct PRISM Stratum accepts `[.]` usernames. On qbit test chains, an invalid username payout falls back to `tq1zlsq9dpxz8mennhdpr9nf9s0f2tjtq6gxs9m84k6xglhkfp92q2zszzu4m3` unless `PRISM_USERNAME_FALLBACK_ADDRESS` is set. - Direct PRISM Stratum tags its coinbase scriptSig with `PRISM_COINBASE_TAG` before the Stratum extranonce. The default is `/PRISM/`; set `PRISM_COINBASE_TAG=` to disable it or set another short printable ASCII tag. diff --git a/VERSION b/VERSION index 9084fa2f7..227cea215 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -1.1.0 +2.0.0 diff --git a/ckpool/README.md b/ckpool/README.md index dcf53b0b2..7b21ce64b 100644 --- a/ckpool/README.md +++ b/ckpool/README.md @@ -17,7 +17,8 @@ ckpool is the cleanest open-source fit for qbit's permissionless mining path bec - Witness scale factor: `1` - Max block weight: `2000000` - Permissionless version mask: use `getblocktemplate.versionrollingmask` when - qbitd advertises it; fall back to `CKPOOL_VERSION_MASK=1fffe000` + qbitd advertises it; fall back to `CKPOOL_VERSION_MASK=1fffe000` only when a + reachable node omits the field ## Likely friction points @@ -33,16 +34,38 @@ Do not leave maturity at Bitcoin defaults. qbit rewards stay immature for `1000` Keep miner-controlled version rolling inside the mask advertised by qbitd. The container defaults `CKPOOL_VERSION_MASK_MODE=dynamic`, so ckpool probes -`getblocktemplate` at startup and writes the selected `version_mask` into -`/etc/ckpool/ckpool.conf`. Older qbitd builds that do not return -`versionrollingmask` fall back to `CKPOOL_VERSION_MASK=1fffe000`. If qbitd -returns `versionrollingmask=00000000`, ckpool disables BIP310 version rolling. -For normal production qbit, the node-advertised mask is the source of truth. -During explicitly authorized mainnet prelaunch, the mask probe uses the -configured fallback and preflight defers only its live GBT shape check. CKPool's -generator then keeps retrying GBT every five seconds rather than exiting. Its -Stratum listener remains bound, but it does not serve mining work until qbitd -returns a valid template. +`getblocktemplate` and writes the selected `version_mask` into +`/etc/ckpool/ckpool.conf`. + +The probe runs inside the supervised child, after the startup supervisor's +readiness gate and live-template checks have passed. That ordering matters on a +cold start: a probe issued while qbitd is still in initial block download cannot +learn the node's mask, and the value chosen there would stay frozen for the life +of the process. Dynamic mode therefore fails closed. Transient RPC failures are +retried `CKPOOL_VERSION_MASK_PROBE_ATTEMPTS` times, spaced by +`CKPOOL_VERSION_MASK_PROBE_RETRY_SECONDS`; if none succeeds, startup aborts +before CKPool config or state is written. Configuration faults - an unparseable +`CKPOOL_VERSION_MASK_MODE` or `CKPOOL_VERSION_MASK`, missing RPC credentials, or +a malformed advertised mask - fail immediately without retrying, and the mode and +configured mask and dynamic probe settings are validated up front so a typo does not cost the readiness +wait. Probe attempts must be between 1 and 10, the RPC timeout must be finite +and positive, and the retry delay finite and nonnegative. Configured timeouts +and delays across all attempts must total at most 60 seconds. These startup +retries are separate from the supervisor's steady-state failure grace period. + +A reachable node that omits `versionrollingmask` is an older qbitd build, so the +configured `CKPOOL_VERSION_MASK=1fffe000` applies. If qbitd returns +`versionrollingmask=00000000`, ckpool disables BIP310 version rolling. For normal +production qbit, the node-advertised mask is the source of truth. + +Explicitly authorized mainnet prelaunch keeps qbitd in initial block download on +purpose, so `getblocktemplate` is unavailable and dynamic mode fails closed there +too. Deployments that must start in that window set +`CKPOOL_VERSION_MASK_MODE=static`, which uses `CKPOOL_VERSION_MASK` without +probing. That keeps the choice to run on an unverified mask explicit and +recorded in the deployment's configuration. CKPool's generator then keeps +retrying GBT every five seconds rather than exiting. Its Stratum listener remains +bound, but it does not serve mining work until qbitd returns a valid template. ### Preflight interfaces diff --git a/compose.production.yaml b/compose.production.yaml index 7add2e6fb..d90c92bb7 100644 --- a/compose.production.yaml +++ b/compose.production.yaml @@ -66,6 +66,35 @@ services: bind: create_host_path: false + # The standby the public read tier serves from. Its data directory is cloned + # by pg_basebackup rather than initdb, so it takes no WAL bind of its own -- + # the base backup lays the cluster down under this one source. Provisioning, + # slot management and the WAL-retention risk: docs/prism-postgres-replica.md. + prism-postgres-replica: + image: ${PRISM_POSTGRES_IMAGE:-invalid.invalid/qbit-mining-bootstrap/postgres:production-image-not-configured} + pull_policy: always + restart: unless-stopped + volumes: + - type: bind + source: ${PRISM_POSTGRES_REPLICA_DATA_SOURCE:-/production-source-not-configured/postgres-replica/data} + target: /var/lib/postgresql/data + bind: + create_host_path: false + + # Same image as the coordinator, overlaid the same way; the audit root is + # bound read-only here because this tier only publishes evidence it reads. + prism-public-api: + image: ${PRISM_COORDINATOR_IMAGE:-invalid.invalid/qbit-mining-bootstrap/prism:production-image-not-configured} + pull_policy: always + restart: unless-stopped + volumes: + - type: bind + source: ${PRISM_AUDIT_DATA_SOURCE:-/production-source-not-configured/prism/audit} + target: ${PRISM_AUDIT_DIR:-/var/lib/qbit-prism/audit} + read_only: true + bind: + create_host_path: false + permissionless-miner: profiles: !override ["production-disabled"] diff --git a/compose.yaml b/compose.yaml index c99a67c0c..62156deda 100644 --- a/compose.yaml +++ b/compose.yaml @@ -118,6 +118,8 @@ services: CKPOOL_VERSION_MASK: ${CKPOOL_VERSION_MASK:-1fffe000} CKPOOL_VERSION_MASK_MODE: ${CKPOOL_VERSION_MASK_MODE:-dynamic} CKPOOL_VERSION_MASK_RPC_TIMEOUT_SECONDS: ${CKPOOL_VERSION_MASK_RPC_TIMEOUT_SECONDS:-5} + CKPOOL_VERSION_MASK_PROBE_ATTEMPTS: ${CKPOOL_VERSION_MASK_PROBE_ATTEMPTS:-3} + CKPOOL_VERSION_MASK_PROBE_RETRY_SECONDS: ${CKPOOL_VERSION_MASK_PROBE_RETRY_SECONDS:-2} CKPOOL_STRATUM_PORT: 3333 ports: - "${CKPOOL_STRATUM_PORT_HOST:-3333}:3333" @@ -417,6 +419,12 @@ services: profiles: ["prism"] image: ${PRISM_POSTGRES_IMAGE:-postgres:16-alpine} restart: unless-stopped + # The image's own command is `postgres`; repeating it keeps the entrypoint's + # first-boot initdb path intact and only adds the hba file that carries the + # replication rules prism-postgres-replica connects through. Postgres 16 + # already defaults to wal_level=replica with 10 senders and 10 slots, which + # is enough for one standby, so no other server flags are needed. + command: ["postgres", "-c", "hba_file=/etc/postgresql/pg_hba.conf"] environment: POSTGRES_DB: ${PRISM_POSTGRES_DB:-qbit} POSTGRES_USER: ${PRISM_POSTGRES_USER:-qbit} @@ -425,12 +433,79 @@ services: volumes: - ${PRISM_POSTGRES_DATA_SOURCE:-prism-postgres-data}:/var/lib/postgresql/data - ${PRISM_POSTGRES_WAL_SOURCE:-prism-postgres-wal}:/var/lib/postgresql/wal + - ./config/prism-postgres/pg_hba.conf:/etc/postgresql/pg_hba.conf:ro healthcheck: + # A socket-only pg_isready is racy: the image's initdb-time temporary + # server accepts unix-socket connections (with listen_addresses='') + # and then shuts down, so a dependent can be released against the + # throwaway server. Gate on the real server's TCP listener first, then + # confirm with a real query. $$ keeps the variables for the container + # shell (the image sets POSTGRES_USER/POSTGRES_DB/POSTGRES_PASSWORD). test: - CMD-SHELL - - pg_isready -U ${PRISM_POSTGRES_USER:-qbit} -d ${PRISM_POSTGRES_DB:-qbit} + - >- + pg_isready -h 127.0.0.1 -p 5432 + && PGPASSWORD="$$POSTGRES_PASSWORD" + psql -h 127.0.0.1 -p 5432 -U "$$POSTGRES_USER" -d "$$POSTGRES_DB" + -c 'SELECT 1' >/dev/null + interval: 5s + timeout: 5s + retries: 30 + + prism-postgres-replica: + profiles: ["prism"] + image: ${PRISM_POSTGRES_IMAGE:-postgres:16-alpine} + restart: unless-stopped + depends_on: + prism-postgres: + condition: service_healthy + # Bootstrap by pg_basebackup instead of initdb, then hand off to the stock + # image entrypoint; see config/prism-postgres/replica-entrypoint.sh and + # docs/prism-postgres-replica.md. + entrypoint: ["/usr/local/bin/prism-replica-entrypoint.sh"] + environment: + # The data directory is cloned from the primary, so none of these drive + # initdb. They give the bootstrap its replication connection and the + # healthcheck its psql credentials, using the primary's defaults. + POSTGRES_DB: ${PRISM_POSTGRES_DB:-qbit} + POSTGRES_USER: ${PRISM_POSTGRES_USER:-qbit} + POSTGRES_PASSWORD: ${PRISM_POSTGRES_PASSWORD:-change-this} + PGPASSWORD: ${PRISM_POSTGRES_PASSWORD:-change-this} + PRISM_POSTGRES_DB: ${PRISM_POSTGRES_DB:-qbit} + PRISM_POSTGRES_USER: ${PRISM_POSTGRES_USER:-qbit} + PRISM_POSTGRES_PASSWORD: ${PRISM_POSTGRES_PASSWORD:-change-this} + PRISM_POSTGRES_REPLICATION_SLOT: ${PRISM_POSTGRES_REPLICATION_SLOT:-prism_public_replica} + PRISM_POSTGRES_REPLICA_BASEBACKUP_ATTEMPTS: ${PRISM_POSTGRES_REPLICA_BASEBACKUP_ATTEMPTS:-60} + PRISM_POSTGRES_REPLICA_BASEBACKUP_RETRY_SECONDS: ${PRISM_POSTGRES_REPLICA_BASEBACKUP_RETRY_SECONDS:-5} + volumes: + - ${PRISM_POSTGRES_REPLICA_DATA_SOURCE:-prism-postgres-replica-data}:/var/lib/postgresql/data + - ./config/prism-postgres/replica-entrypoint.sh:/usr/local/bin/prism-replica-entrypoint.sh:ro + healthcheck: + # Same socket-race lesson as the primary: gate on the real server's TCP + # listener, then confirm with a real query. The third clause additionally + # requires the server to still be in recovery, so an out-of-band promotion + # cannot leave the public read tier pointed at a second writable cluster + # while reporting healthy. $$ keeps the variables for the container shell. + # + # pg_isready names the role explicitly, unlike the primary's check: the + # standby's pg_hba.conf is the one initdb wrote on the primary, which + # trusts loopback, so an unauthenticated probe reaches the role lookup and + # logs `FATAL: role "root" does not exist` on every interval. + test: + - CMD-SHELL + - >- + pg_isready -h 127.0.0.1 -p 5432 -U "$$POSTGRES_USER" -d "$$POSTGRES_DB" + && PGPASSWORD="$$POSTGRES_PASSWORD" + psql -h 127.0.0.1 -p 5432 -U "$$POSTGRES_USER" -d "$$POSTGRES_DB" + -c 'SELECT 1' >/dev/null + && [ "$$(PGPASSWORD="$$POSTGRES_PASSWORD" + psql -h 127.0.0.1 -p 5432 -U "$$POSTGRES_USER" -d "$$POSTGRES_DB" + -tAc 'SELECT pg_is_in_recovery()')" = 't' ] interval: 5s timeout: 5s + # The first start streams a full base backup before the server exists at + # all, so failures inside the grace window must not burn the retry budget. + start_period: 60s retries: 30 prism-coordinator: @@ -480,7 +555,14 @@ services: PRISM_POSTGRES_PSQL_COMMAND: ${PRISM_POSTGRES_PSQL_COMMAND:-} PRISM_POSTGRES_INIT_SCHEMA: ${PRISM_POSTGRES_INIT_SCHEMA:-1} PRISM_POSTGRES_READ_CONCURRENCY: ${PRISM_POSTGRES_READ_CONCURRENCY:-4} + PRISM_POSTGRES_IDLE_IN_TRANSACTION_TIMEOUT_SECONDS: ${PRISM_POSTGRES_IDLE_IN_TRANSACTION_TIMEOUT_SECONDS:-15} + PRISM_POSTGRES_TCP_KEEPALIVES_IDLE_SECONDS: ${PRISM_POSTGRES_TCP_KEEPALIVES_IDLE_SECONDS:-30} + PRISM_POSTGRES_TCP_KEEPALIVES_INTERVAL_SECONDS: ${PRISM_POSTGRES_TCP_KEEPALIVES_INTERVAL_SECONDS:-10} + PRISM_POSTGRES_TCP_KEEPALIVES_COUNT: ${PRISM_POSTGRES_TCP_KEEPALIVES_COUNT:-3} PRISM_POSTGRES_NATIVE_CLIENT: ${PRISM_POSTGRES_NATIVE_CLIENT:-auto} + PRISM_CANDIDATE_STORAGE_VERSION: ${PRISM_CANDIDATE_STORAGE_VERSION:-2} + PRISM_CANDIDATE_SPOOL_DIR: ${PRISM_CANDIDATE_SPOOL_DIR:-} + PRISM_CANDIDATE_SPOOL_RESERVATION_BYTES: ${PRISM_CANDIDATE_SPOOL_RESERVATION_BYTES:-4294967296} PRISM_ACCEPTED_STATS_CACHE_SECONDS: ${PRISM_ACCEPTED_STATS_CACHE_SECONDS:-60} PRISM_PUBLIC_REWARD_WINDOW_CACHE_SECONDS: ${PRISM_PUBLIC_REWARD_WINDOW_CACHE_SECONDS:-30} PRISM_SUBMIT_TIP_MAX_AGE_SECONDS: ${PRISM_SUBMIT_TIP_MAX_AGE_SECONDS:-10} @@ -493,6 +575,8 @@ services: PRISM_LEDGER_WRITER_SESSION_TOKEN: ${PRISM_LEDGER_WRITER_SESSION_TOKEN:-} PRISM_LEDGER_WRITER_PUBLIC_KEY_HEX: ${PRISM_LEDGER_WRITER_PUBLIC_KEY_HEX:-} PRISM_LEDGER_LEASE_TTL_SECONDS: ${PRISM_LEDGER_LEASE_TTL_SECONDS:-60} + PRISM_LEDGER_LEASE_ACQUIRE_LOCK_TIMEOUT_SECONDS: ${PRISM_LEDGER_LEASE_ACQUIRE_LOCK_TIMEOUT_SECONDS:-5} + PRISM_LEDGER_LEASE_ACQUIRE_ATTEMPTS: ${PRISM_LEDGER_LEASE_ACQUIRE_ATTEMPTS:-5} PRISM_WRITER_QUIESCENCE_TIMEOUT_SECONDS: ${PRISM_WRITER_QUIESCENCE_TIMEOUT_SECONDS:-15} PRISM_MANIFEST_SIGNING_SEED_HEX: ${PRISM_MANIFEST_SIGNING_SEED_HEX:-} PRISM_LEDGER_ATTESTATION_SIGNING_SEED_HEX: ${PRISM_LEDGER_ATTESTATION_SIGNING_SEED_HEX:-} @@ -507,13 +591,39 @@ services: PRISM_STRATUM_PORT: ${PRISM_STRATUM_PORT:-3340} PRISM_STRATUM_PORT_HOST: ${PRISM_STRATUM_PORT_HOST:-3340} PRISM_BLOCKPOLL_SECONDS: ${PRISM_BLOCKPOLL_SECONDS:-2} + PRISM_MALLOC_TELEMETRY: ${PRISM_MALLOC_TELEMETRY:-1} + # Issue #226 part 2. glibc, not the coordinator, reads MALLOC_ARENA_MAX, + # but a deploy dotenv value still reaches the container process only + # through this passthrough; without it the image's ENV silently wins. + # The default here must equal the image's ARG default in + # lab/prism/Dockerfile (tests/test_prism_heap_census.py pins both), and + # it must never be empty: glibc parses "" as 0. + MALLOC_ARENA_MAX: ${MALLOC_ARENA_MAX:-2} + # The operator-triggered heap census and malloc_trim, all off by + # default; docs/prism-capacity-readiness.md before enabling any. The + # defaults mirror lab/prism/coordinator_config.py and are pinned there. + PRISM_HEAP_CENSUS: ${PRISM_HEAP_CENSUS:-0} + # No interpolation default on purpose: empty reaches the coordinator as + # unset and the reports land under PRISM_AUDIT_DIR/heap-census. + PRISM_HEAP_CENSUS_DIR: ${PRISM_HEAP_CENSUS_DIR:-} + PRISM_HEAP_CENSUS_TOP_N: ${PRISM_HEAP_CENSUS_TOP_N:-40} + PRISM_HEAP_CENSUS_MAX_BYTES: ${PRISM_HEAP_CENSUS_MAX_BYTES:-262144} + PRISM_HEAP_CENSUS_MAX_SECONDS: ${PRISM_HEAP_CENSUS_MAX_SECONDS:-10} + PRISM_HEAP_CENSUS_MIN_INTERVAL_SECONDS: ${PRISM_HEAP_CENSUS_MIN_INTERVAL_SECONDS:-60} + PRISM_HEAP_CENSUS_TRACEMALLOC: ${PRISM_HEAP_CENSUS_TRACEMALLOC:-0} + PRISM_MALLOC_TRIM_SIGNAL: ${PRISM_MALLOC_TRIM_SIGNAL:-0} + PRISM_MALLOC_TRIM_INTERVAL_SECONDS: ${PRISM_MALLOC_TRIM_INTERVAL_SECONDS:-0} PRISM_BLOCKWAIT_ENABLED: ${PRISM_BLOCKWAIT_ENABLED:-1} PRISM_BLOCKWAIT_TIMEOUT_SECONDS: ${PRISM_BLOCKWAIT_TIMEOUT_SECONDS:-5} PRISM_BLOCK_SUBMIT_RPC_TIMEOUT_SECONDS: ${PRISM_BLOCK_SUBMIT_RPC_TIMEOUT_SECONDS:-1} PRISM_BLOCK_SUBMIT_DB_TIMEOUT_SECONDS: ${PRISM_BLOCK_SUBMIT_DB_TIMEOUT_SECONDS:-1} + PRISM_BLOCK_REPLAY_PAGE_SIZE: ${PRISM_BLOCK_REPLAY_PAGE_SIZE:-1024} PRISM_BLOCK_LANDING_DB_TIMEOUT_SECONDS: ${PRISM_BLOCK_LANDING_DB_TIMEOUT_SECONDS:-30} PRISM_BLOCK_LANDING_DB_TIMEOUT_MAX_SECONDS: ${PRISM_BLOCK_LANDING_DB_TIMEOUT_MAX_SECONDS:-120} PRISM_ACCEPTED_PARENT_UNRESOLVED_DEPTH_MAX: ${PRISM_ACCEPTED_PARENT_UNRESOLVED_DEPTH_MAX:-8} + PRISM_ACCEPTED_PARENT_REDRIVE_DEFER_THRESHOLD: ${PRISM_ACCEPTED_PARENT_REDRIVE_DEFER_THRESHOLD:-3} + PRISM_ACCEPTED_PARENT_REDRIVE_ATTEMPT_MAX: ${PRISM_ACCEPTED_PARENT_REDRIVE_ATTEMPT_MAX:-3} + PRISM_BLOCK_CANDIDATE_CLEANUP_RETRY_BACKLOG_MAX: ${PRISM_BLOCK_CANDIDATE_CLEANUP_RETRY_BACKLOG_MAX:-4096} PRISM_BLOCK_SUBMIT_LOCK_WAIT_LOG_SECONDS: ${PRISM_BLOCK_SUBMIT_LOCK_WAIT_LOG_SECONDS:-5} PRISM_BLOCK_SUBMIT_STUCK_CALL_EXIT_SECONDS: ${PRISM_BLOCK_SUBMIT_STUCK_CALL_EXIT_SECONDS:-30} PRISM_OBSERVED_TIP_ACCEPT_WINDOW_SECONDS: ${PRISM_OBSERVED_TIP_ACCEPT_WINDOW_SECONDS:-300} @@ -528,11 +638,16 @@ services: PRISM_PAYOUT_ARTIFACT_MIN_BUILD_INTERVAL_SECONDS: ${PRISM_PAYOUT_ARTIFACT_MIN_BUILD_INTERVAL_SECONDS:-${PRISM_PAYOUT_ARTIFACT_REANCHOR_SECONDS:-60}} PRISM_PAYOUT_ARTIFACT_FULL_RESCAN_SECONDS: ${PRISM_PAYOUT_ARTIFACT_FULL_RESCAN_SECONDS:-3600} PRISM_PAYOUT_ARTIFACT_MAX_ANCHOR_AGE_SECONDS: ${PRISM_PAYOUT_ARTIFACT_MAX_ANCHOR_AGE_SECONDS:-300} + PRISM_WINDOW_PIPELINE_RUST: ${PRISM_WINDOW_PIPELINE_RUST:-0} PRISM_TEMPLATE_CACHE_SECONDS: ${PRISM_TEMPLATE_CACHE_SECONDS:-2} PRISM_TEMPLATE_REFRESH_FAILURE_EXIT_SECONDS: ${PRISM_TEMPLATE_REFRESH_FAILURE_EXIT_SECONDS:-120} PRISM_COORDINATION_BLOCKED_EXIT_SECONDS: ${PRISM_COORDINATION_BLOCKED_EXIT_SECONDS:-900} PRISM_REORG_RECONCILE_CACHE_SECONDS: ${PRISM_REORG_RECONCILE_CACHE_SECONDS:-5} PRISM_HEALTH_REFRESH_SECONDS: ${PRISM_HEALTH_REFRESH_SECONDS:-5} + PRISM_METRICS_REFRESH_SECONDS: ${PRISM_METRICS_REFRESH_SECONDS:-5} + PRISM_HASHRATE_ROLLUP_ENABLED: ${PRISM_HASHRATE_ROLLUP_ENABLED:-1} + PRISM_HASHRATE_ROLLUP_INTERVAL_SECONDS: ${PRISM_HASHRATE_ROLLUP_INTERVAL_SECONDS:-15} + PRISM_HASHRATE_ROLLUP_BATCH_SHARES: ${PRISM_HASHRATE_ROLLUP_BATCH_SHARES:-50000} PRISM_MINING_HEALTH_STARTUP_GRACE_SECONDS: ${PRISM_MINING_HEALTH_STARTUP_GRACE_SECONDS:-30} PRISM_HEALTH_PENDING_REFRESH_MAX_AGE_SECONDS: ${PRISM_HEALTH_PENDING_REFRESH_MAX_AGE_SECONDS:-15} PRISM_HEALTH_TIP_POLL_MAX_AGE_SECONDS: ${PRISM_HEALTH_TIP_POLL_MAX_AGE_SECONDS:-15} @@ -546,6 +661,10 @@ services: # backward-compatible implicit cap of min(4, pending bound). PRISM_INITIAL_JOB_MAX_WORKERS: ${PRISM_INITIAL_JOB_MAX_WORKERS:-} PRISM_JOB_BUILD_EXECUTOR_WORKERS: ${PRISM_JOB_BUILD_EXECUTOR_WORKERS:-2} + # No interpolation default on purpose: empty must reach the coordinator + # as unset so it leaves sys.setswitchinterval() uncalled, rather than + # pinning CPython's current 5ms default as though the tree had chosen it. + PRISM_PYTHON_SWITCH_INTERVAL_SECONDS: ${PRISM_PYTHON_SWITCH_INTERVAL_SECONDS:-} PRISM_STRATUM_ACCEPT_RESOURCE_EXHAUSTION_BACKOFF_SECONDS: ${PRISM_STRATUM_ACCEPT_RESOURCE_EXHAUSTION_BACKOFF_SECONDS:-1} PRISM_PAYOUT_ADDRESS_CACHE_MAX_ENTRIES: ${PRISM_PAYOUT_ADDRESS_CACHE_MAX_ENTRIES:-4096} PRISM_PAYOUT_ADDRESS_CACHE_TTL_SECONDS: ${PRISM_PAYOUT_ADDRESS_CACHE_TTL_SECONDS:-3600} @@ -553,30 +672,6 @@ services: PRISM_WATCHDOG_ENABLED: ${PRISM_WATCHDOG_ENABLED:-1} PRISM_WATCHDOG_TIMEOUT_SECONDS: ${PRISM_WATCHDOG_TIMEOUT_SECONDS:-120} PRISM_WATCHDOG_INTERVAL_SECONDS: ${PRISM_WATCHDOG_INTERVAL_SECONDS:-15} - PRISM_PUBLIC_POOL_NAME: ${PRISM_PUBLIC_POOL_NAME:-PRISM} - PRISM_PUBLIC_STRATUM_HOST: ${PRISM_PUBLIC_STRATUM_HOST:-} - PRISM_PUBLIC_STRATUM_URL: ${PRISM_PUBLIC_STRATUM_URL:-} - PRISM_PUBLIC_STRATUM_HIGHDIFF_URL: ${PRISM_PUBLIC_STRATUM_HIGHDIFF_URL:-} - PRISM_PUBLIC_POOL_FEE_BPS: ${PRISM_PUBLIC_POOL_FEE_BPS:-0} - PRISM_PUBLIC_CONFIGURATION_LABEL: ${PRISM_PUBLIC_CONFIGURATION_LABEL:-PRISM default} - PRISM_PUBLIC_CONFIGURATION_DESCRIPTION: ${PRISM_PUBLIC_CONFIGURATION_DESCRIPTION:-} - PRISM_PUBLIC_BLOCK_TEMPLATE_POLICY: ${PRISM_PUBLIC_BLOCK_TEMPLATE_POLICY:-pool-selected qbit block template with PRISM payout settlement} - PRISM_PUBLIC_MINIMUM_PAYOUT_BITS: ${PRISM_PUBLIC_MINIMUM_PAYOUT_BITS:-} - PRISM_PUBLIC_HASHRATE_SMOOTHING_SECONDS: ${PRISM_PUBLIC_HASHRATE_SMOOTHING_SECONDS:-1800} - PRISM_PUBLIC_EXPLORER_TX_URL_PREFIX: ${PRISM_PUBLIC_EXPLORER_TX_URL_PREFIX:-} - PRISM_PUBLIC_EXPLORER_BLOCK_URL_PREFIX: ${PRISM_PUBLIC_EXPLORER_BLOCK_URL_PREFIX:-} - PRISM_PUBLIC_CACHE_ENABLED: ${PRISM_PUBLIC_CACHE_ENABLED:-1} - PRISM_PUBLIC_CACHE_TTL_SECONDS: ${PRISM_PUBLIC_CACHE_TTL_SECONDS:-5} - PRISM_PUBLIC_CACHE_STALE_WHILE_REVALIDATE_SECONDS: ${PRISM_PUBLIC_CACHE_STALE_WHILE_REVALIDATE_SECONDS:-30} - PRISM_PUBLIC_AGGREGATE_CACHE_TTL_SECONDS: ${PRISM_PUBLIC_AGGREGATE_CACHE_TTL_SECONDS:-30} - PRISM_PUBLIC_AGGREGATE_CACHE_STALE_WHILE_REVALIDATE_SECONDS: ${PRISM_PUBLIC_AGGREGATE_CACHE_STALE_WHILE_REVALIDATE_SECONDS:-30} - PRISM_PUBLIC_CONFIG_CACHE_TTL_SECONDS: ${PRISM_PUBLIC_CONFIG_CACHE_TTL_SECONDS:-300} - PRISM_PUBLIC_CONFIG_CACHE_STALE_WHILE_REVALIDATE_SECONDS: ${PRISM_PUBLIC_CONFIG_CACHE_STALE_WHILE_REVALIDATE_SECONDS:-3600} - PRISM_PUBLIC_ARTIFACT_CACHE_TTL_SECONDS: ${PRISM_PUBLIC_ARTIFACT_CACHE_TTL_SECONDS:-86400} - PRISM_PUBLIC_ARTIFACT_CACHE_STALE_WHILE_REVALIDATE_SECONDS: ${PRISM_PUBLIC_ARTIFACT_CACHE_STALE_WHILE_REVALIDATE_SECONDS:-86400} - PRISM_PUBLIC_CACHE_MAX_ENTRIES: ${PRISM_PUBLIC_CACHE_MAX_ENTRIES:-512} - PRISM_PUBLIC_CACHE_MAX_RESPONSE_BYTES: ${PRISM_PUBLIC_CACHE_MAX_RESPONSE_BYTES:-1048576} - PRISM_PUBLIC_CACHE_DEBUG_HEADERS: ${PRISM_PUBLIC_CACHE_DEBUG_HEADERS:-0} PRISM_STRATUM_EXTRANONCE2_SIZE: ${PRISM_STRATUM_EXTRANONCE2_SIZE:-8} PRISM_COINBASE_TAG: ${PRISM_COINBASE_TAG-/PRISM/} PRISM_STRATUM_SHARE_DIFF: ${PRISM_STRATUM_SHARE_DIFF:-0.000000001} @@ -623,7 +718,17 @@ services: PRISM_STRATUM_VARDIFF_START_DIFF: ${PRISM_STRATUM_VARDIFF_START_DIFF:-0.000000001} PRISM_STRATUM_VARDIFF_EWMA_ALPHA: ${PRISM_STRATUM_VARDIFF_EWMA_ALPHA:-0.4} PRISM_STRATUM_VARDIFF_RETARGET_TOLERANCE: ${PRISM_STRATUM_VARDIFF_RETARGET_TOLERANCE:-0.25} + PRISM_STRATUM_VARDIFF_INITIAL_CONVERGENCE: ${PRISM_STRATUM_VARDIFF_INITIAL_CONVERGENCE:-1} + PRISM_STRATUM_VARDIFF_INITIAL_MAX_STEP_UP: ${PRISM_STRATUM_VARDIFF_INITIAL_MAX_STEP_UP:-64} + PRISM_STRATUM_VARDIFF_INITIAL_MIN_SHARES: ${PRISM_STRATUM_VARDIFF_INITIAL_MIN_SHARES:-8} + PRISM_STRATUM_VARDIFF_INITIAL_MIN_STEP_UP: ${PRISM_STRATUM_VARDIFF_INITIAL_MIN_STEP_UP:-4} + PRISM_STRATUM_VARDIFF_INITIAL_MIN_SECONDS: ${PRISM_STRATUM_VARDIFF_INITIAL_MIN_SECONDS:-1} + PRISM_STRATUM_VARDIFF_HIGH_DIFF_ARRIVAL_DIFF: ${PRISM_STRATUM_VARDIFF_HIGH_DIFF_ARRIVAL_DIFF:-500000} PRISM_STRATUM_VARDIFF_IDLE_SWEEP_SECONDS: ${PRISM_STRATUM_VARDIFF_IDLE_SWEEP_SECONDS:-15} + PRISM_STRATUM_VARDIFF_RESUME: ${PRISM_STRATUM_VARDIFF_RESUME:-1} + PRISM_STRATUM_VARDIFF_RESUME_TTL_SECONDS: ${PRISM_STRATUM_VARDIFF_RESUME_TTL_SECONDS:-900} + PRISM_STRATUM_VARDIFF_RESUME_MAX_ENTRIES: ${PRISM_STRATUM_VARDIFF_RESUME_MAX_ENTRIES:-8192} + PRISM_STRATUM_VARDIFF_RESUME_MAX_START_FACTOR: ${PRISM_STRATUM_VARDIFF_RESUME_MAX_START_FACTOR:-1024} PRISM_STRATUM_STALE_GRACE_SECONDS: ${PRISM_STRATUM_STALE_GRACE_SECONDS:-3} PRISM_STRATUM_SAME_TIP_JOB_RETENTION_SECONDS: ${PRISM_STRATUM_SAME_TIP_JOB_RETENTION_SECONDS:-30} PRISM_STRATUM_SAME_TIP_JOB_RETENTION_PER_CONNECTION: ${PRISM_STRATUM_SAME_TIP_JOB_RETENTION_PER_CONNECTION:-64} @@ -661,6 +766,124 @@ services: start_period: 15s retries: 3 + # The public /public/v1 read surface, served by its own process. It is a + # separate service rather than another listener in prism-coordinator because + # public read traffic scales with public interest instead of with hashrate: + # keeping it in-process put an uncontrolled workload on the same GIL that + # acknowledges shares and lands blocks, and on the same primary Postgres the + # lease-holding writer commits through. + prism-public-api: + profiles: ["prism"] + image: ${PRISM_COORDINATOR_IMAGE:-qbit-lab-prism-coordinator:local} + restart: on-failure + build: + context: . + dockerfile: lab/prism/Dockerfile + command: ["python3", "-m", "lab.prism.public_read_service"] + # Deliberately NOT depends_on prism-coordinator. The public tier must + # survive coordinator restarts -- that independence is a large part of why + # the surface was extracted -- so it waits only on the database it reads, + # which is the standby, not the primary the coordinator commits through. + depends_on: + prism-postgres-replica: + condition: service_healthy + volumes: + # Read-only: published evidence is served from the artifact files, and + # this process must never be able to write to the audit root. + - ${PRISM_AUDIT_DATA_SOURCE:-prism-audit-data}:${PRISM_AUDIT_DIR:-/var/lib/qbit-prism/audit}:ro + environment: + QBIT_PRODUCTION: ${QBIT_PRODUCTION:-0} + QBIT_CHAIN: ${QBIT_CHAIN:-regtest} + QBIT_RPC_HOST: qbitd + QBIT_RPC_PORT: ${QBIT_RPC_PORT:-18452} + QBIT_RPC_USER: ${QBIT_RPC_USER:-qbitrpc} + QBIT_RPC_PASSWORD: ${QBIT_RPC_PASSWORD:-change-this} + PGPASSWORD: ${PRISM_POSTGRES_PASSWORD:-change-this} + PRISM_POSTGRES_DB: ${PRISM_POSTGRES_DB:-qbit} + PRISM_POSTGRES_USER: ${PRISM_POSTGRES_USER:-qbit} + PRISM_POSTGRES_PASSWORD: ${PRISM_POSTGRES_PASSWORD:-change-this} + # The standby, under its own operator knob. Reusing PRISM_DATABASE_URL + # here would let a .env that names the primary -- which the coordinator's + # entry legitimately does -- silently point the public tier back at it, + # which is exactly what PRISM_PUBLIC_REPLICA_MODE=require then refuses. + # Encode the individual credentials at runtime when no explicit URL is set. + PRISM_DATABASE_URL: ${PRISM_PUBLIC_DATABASE_URL:-} + PRISM_PUBLIC_DATABASE_HOST: prism-postgres-replica + # Enforce the replica contract: refuse to serve replica-backed routes + # from a writable primary, or past a silent replication stream. `off` + # is the code default so an existing deployment that has not provisioned + # a standby keeps the merged primary-backed behaviour until it cuts over. + PRISM_PUBLIC_REPLICA_MODE: ${PRISM_PUBLIC_REPLICA_MODE:-require} + PRISM_PUBLIC_REPLICA_MAX_LAG_SECONDS: ${PRISM_PUBLIC_REPLICA_MAX_LAG_SECONDS:-60} + # Keep the primary's psql override out of the read tier too. Empty uses + # the public database URL above; wrappers need their own replica command. + PRISM_POSTGRES_PSQL_COMMAND: ${PRISM_PUBLIC_PSQL_COMMAND:-} + PRISM_POSTGRES_READ_CONCURRENCY: ${PRISM_POSTGRES_READ_CONCURRENCY:-4} + PRISM_POSTGRES_IDLE_IN_TRANSACTION_TIMEOUT_SECONDS: ${PRISM_POSTGRES_IDLE_IN_TRANSACTION_TIMEOUT_SECONDS:-15} + PRISM_POSTGRES_TCP_KEEPALIVES_IDLE_SECONDS: ${PRISM_POSTGRES_TCP_KEEPALIVES_IDLE_SECONDS:-30} + PRISM_POSTGRES_TCP_KEEPALIVES_INTERVAL_SECONDS: ${PRISM_POSTGRES_TCP_KEEPALIVES_INTERVAL_SECONDS:-10} + PRISM_POSTGRES_TCP_KEEPALIVES_COUNT: ${PRISM_POSTGRES_TCP_KEEPALIVES_COUNT:-3} + PRISM_POSTGRES_NATIVE_CLIENT: ${PRISM_POSTGRES_NATIVE_CLIENT:-auto} + PRISM_AUDIT_DIR: ${PRISM_AUDIT_DIR:-/var/lib/qbit-prism/audit} + # Refuse to start on the in-memory ledger: it has no dashboard_* read + # models, so public_api's fallbacks would reintroduce the writer-lock + # reads this tier exists to avoid. + PRISM_ALLOW_MEMORY_LEDGER: 0 + # Read by the ledger, so both processes construct one. + PRISM_PUBLIC_REWARD_WINDOW_CACHE_SECONDS: ${PRISM_PUBLIC_REWARD_WINDOW_CACHE_SECONDS:-30} + PRISM_PUBLIC_API_BIND: ${PRISM_PUBLIC_API_BIND:-0.0.0.0} + PRISM_PUBLIC_API_PORT: ${PRISM_PUBLIC_API_PORT:-3342} + PRISM_PUBLIC_HTTP_MAX_CONNECTIONS: ${PRISM_PUBLIC_HTTP_MAX_CONNECTIONS:-64} + PRISM_PUBLIC_HTTP_TIMEOUT_SECONDS: ${PRISM_PUBLIC_HTTP_TIMEOUT_SECONDS:-10} + PRISM_PUBLIC_READINESS_PROBE_INTERVAL_SECONDS: ${PRISM_PUBLIC_READINESS_PROBE_INTERVAL_SECONDS:-5} + PRISM_PUBLIC_READ_STATEMENT_TIMEOUT_SECONDS: ${PRISM_PUBLIC_READ_STATEMENT_TIMEOUT_SECONDS:-20} + PRISM_PUBLIC_POOL_NAME: ${PRISM_PUBLIC_POOL_NAME:-PRISM} + # This process runs no Stratum listener, but /public/v1/mining-configuration + # still renders from these: PRISM_STRATUM_PORT is the primary endpoint's + # fallback port, and the High-diff endpoint appears only when + # PRISM_STRATUM_HIGHDIFF_PORT is set. Without the pass-through the rendered + # body silently drops the high-diff listener the coordinator is running. + PRISM_STRATUM_PORT: ${PRISM_STRATUM_PORT:-3340} + PRISM_STRATUM_HIGHDIFF_PORT: ${PRISM_STRATUM_HIGHDIFF_PORT:-} + PRISM_PUBLIC_STRATUM_HOST: ${PRISM_PUBLIC_STRATUM_HOST:-} + # Required: this process runs no Stratum listener, so it cannot infer the + # pool's endpoint and refuses to start rather than advertise 127.0.0.1. + PRISM_PUBLIC_STRATUM_URL: ${PRISM_PUBLIC_STRATUM_URL:-} + PRISM_PUBLIC_STRATUM_HIGHDIFF_URL: ${PRISM_PUBLIC_STRATUM_HIGHDIFF_URL:-} + PRISM_PUBLIC_POOL_FEE_BPS: ${PRISM_PUBLIC_POOL_FEE_BPS:-0} + PRISM_PUBLIC_CONFIGURATION_LABEL: ${PRISM_PUBLIC_CONFIGURATION_LABEL:-PRISM default} + PRISM_PUBLIC_CONFIGURATION_DESCRIPTION: ${PRISM_PUBLIC_CONFIGURATION_DESCRIPTION:-} + PRISM_PUBLIC_BLOCK_TEMPLATE_POLICY: ${PRISM_PUBLIC_BLOCK_TEMPLATE_POLICY:-pool-selected qbit block template with PRISM payout settlement} + PRISM_PUBLIC_MINIMUM_PAYOUT_BITS: ${PRISM_PUBLIC_MINIMUM_PAYOUT_BITS:-} + PRISM_PAYOUT_MIN_OUTPUT_BITS: ${PRISM_PAYOUT_MIN_OUTPUT_BITS:-} + PRISM_PAYOUT_MIN_OUTPUT_SATS: ${PRISM_PAYOUT_MIN_OUTPUT_SATS:-} + PRISM_PUBLIC_HASHRATE_SMOOTHING_SECONDS: ${PRISM_PUBLIC_HASHRATE_SMOOTHING_SECONDS:-1800} + PRISM_PUBLIC_EXPLORER_TX_URL_PREFIX: ${PRISM_PUBLIC_EXPLORER_TX_URL_PREFIX:-} + PRISM_PUBLIC_EXPLORER_BLOCK_URL_PREFIX: ${PRISM_PUBLIC_EXPLORER_BLOCK_URL_PREFIX:-} + PRISM_PUBLIC_CACHE_ENABLED: ${PRISM_PUBLIC_CACHE_ENABLED:-1} + PRISM_PUBLIC_CACHE_TTL_SECONDS: ${PRISM_PUBLIC_CACHE_TTL_SECONDS:-5} + PRISM_PUBLIC_CACHE_STALE_WHILE_REVALIDATE_SECONDS: ${PRISM_PUBLIC_CACHE_STALE_WHILE_REVALIDATE_SECONDS:-30} + PRISM_PUBLIC_AGGREGATE_CACHE_TTL_SECONDS: ${PRISM_PUBLIC_AGGREGATE_CACHE_TTL_SECONDS:-30} + PRISM_PUBLIC_AGGREGATE_CACHE_STALE_WHILE_REVALIDATE_SECONDS: ${PRISM_PUBLIC_AGGREGATE_CACHE_STALE_WHILE_REVALIDATE_SECONDS:-30} + PRISM_PUBLIC_CONFIG_CACHE_TTL_SECONDS: ${PRISM_PUBLIC_CONFIG_CACHE_TTL_SECONDS:-300} + PRISM_PUBLIC_CONFIG_CACHE_STALE_WHILE_REVALIDATE_SECONDS: ${PRISM_PUBLIC_CONFIG_CACHE_STALE_WHILE_REVALIDATE_SECONDS:-3600} + PRISM_PUBLIC_ARTIFACT_CACHE_TTL_SECONDS: ${PRISM_PUBLIC_ARTIFACT_CACHE_TTL_SECONDS:-86400} + PRISM_PUBLIC_ARTIFACT_CACHE_STALE_WHILE_REVALIDATE_SECONDS: ${PRISM_PUBLIC_ARTIFACT_CACHE_STALE_WHILE_REVALIDATE_SECONDS:-86400} + PRISM_PUBLIC_CACHE_MAX_ENTRIES: ${PRISM_PUBLIC_CACHE_MAX_ENTRIES:-512} + PRISM_PUBLIC_CACHE_MAX_RESPONSE_BYTES: ${PRISM_PUBLIC_CACHE_MAX_RESPONSE_BYTES:-1048576} + PRISM_PUBLIC_CACHE_DEBUG_HEADERS: ${PRISM_PUBLIC_CACHE_DEBUG_HEADERS:-0} + ports: + - "${PRISM_PUBLIC_API_PORT_HOST:-3342}:${PRISM_PUBLIC_API_PORT:-3342}" + healthcheck: + test: + - CMD-SHELL + - >- + python3 -c "import os, urllib.request; p=int(os.environ.get('PRISM_PUBLIC_API_PORT','3342')); assert urllib.request.urlopen(f'http://127.0.0.1:{p}/healthz', timeout=2).status == 200" + interval: 5s + timeout: 5s + start_period: 15s + retries: 3 + volumes: bitcoin-data: qbit-data: @@ -668,3 +891,4 @@ volumes: prism-audit-data: prism-postgres-data: prism-postgres-wal: + prism-postgres-replica-data: diff --git a/config/prism-postgres/pg_hba.conf b/config/prism-postgres/pg_hba.conf new file mode 100644 index 000000000..903129daf --- /dev/null +++ b/config/prism-postgres/pg_hba.conf @@ -0,0 +1,38 @@ +# PRISM primary client authentication (issue #145). +# +# Mounted read-only at /etc/postgresql/pg_hba.conf and selected through +# `postgres -c hba_file=...`, so it replaces the copy initdb leaves in PGDATA. +# It exists only to add the `replication` rules the prism-postgres-replica +# standby needs; every other rule reproduces the stock postgres image default +# so that nothing about the primary's reachability changes. +# +# Compose does not interpolate this file. The replication rules below name the +# default superuser `qbit` literally: a deployment that overrides +# PRISM_POSTGRES_USER must edit those two lines to match, or pg_basebackup from +# the replica fails with "no pg_hba.conf entry for replication connection". +# See docs/prism-postgres-replica.md. +# +# The first matching line wins, so order matters. + +# TYPE DATABASE USER ADDRESS METHOD + +# Unix socket. `trust` is what initdb writes by default; it keeps the image's +# initdb bootstrap, the compose healthcheck's pg_isready, and interactive +# `docker exec ... psql` working exactly as they did before this file existed. +local all all trust + +# Loopback TCP. The compose healthcheck connects to 127.0.0.1 with PGPASSWORD. +host all all 127.0.0.1/32 scram-sha-256 +host all all ::1/128 scram-sha-256 + +# Ordinary client access from the compose network, i.e. the same reachability +# as the image's default `host all all all scram-sha-256`. +host all all 0.0.0.0/0 scram-sha-256 +host all all ::/0 scram-sha-256 + +# Physical replication for prism-postgres-replica (pg_basebackup and the +# standby's walreceiver). Edit the USER column when PRISM_POSTGRES_USER is not +# the default; production should use a dedicated REPLICATION role instead of +# the superuser (docs/prism-postgres-replica.md). +host replication qbit 0.0.0.0/0 scram-sha-256 +host replication qbit ::/0 scram-sha-256 diff --git a/config/prism-postgres/replica-entrypoint.sh b/config/prism-postgres/replica-entrypoint.sh new file mode 100755 index 000000000..85e536bdf --- /dev/null +++ b/config/prism-postgres/replica-entrypoint.sh @@ -0,0 +1,133 @@ +#!/usr/bin/env bash +# PRISM public-read standby bootstrap (issue #145). +# +# The prism-postgres-replica data directory is never initdb'd: it is cloned +# from prism-postgres with pg_basebackup and then handed to the stock postgres +# image entrypoint, which sees a populated PGDATA, skips initialization, and +# starts the server. Recovery is driven by the standby.signal and +# primary_conninfo that `pg_basebackup -R` writes. +# +# See docs/prism-postgres-replica.md. +set -euo pipefail + +PGDATA="${PGDATA:-/var/lib/postgresql/data}" +PRIMARY_HOST="${PRISM_POSTGRES_PRIMARY_HOST:-prism-postgres}" +PRIMARY_PORT="${PRISM_POSTGRES_PRIMARY_PORT:-5432}" +REPLICATION_USER="${PRISM_POSTGRES_USER:-${POSTGRES_USER:-qbit}}" +REPLICATION_DB="${PRISM_POSTGRES_DB:-${POSTGRES_DB:-postgres}}" +REPLICATION_SLOT="${PRISM_POSTGRES_REPLICATION_SLOT:-prism_public_replica}" +ATTEMPTS="${PRISM_POSTGRES_REPLICA_BASEBACKUP_ATTEMPTS:-60}" +RETRY_SECONDS="${PRISM_POSTGRES_REPLICA_BASEBACKUP_RETRY_SECONDS:-5}" + +export PGPASSWORD="${PGPASSWORD:-${PRISM_POSTGRES_PASSWORD:-${POSTGRES_PASSWORD:-}}}" + +log() { + printf 'prism-postgres-replica: %s\n' "$*" >&2 +} + +# postgres refuses to run as root and pg_basebackup would leave root-owned +# files the server cannot read, so drop to the image's postgres account for +# both. gosu and su-exec both ship in the official images; PGPASSWORD is +# exported, and both tools preserve the environment. +as_postgres() { + if [ "$(id -u)" = '0' ]; then + if command -v gosu >/dev/null 2>&1; then + gosu postgres "$@" + elif command -v su-exec >/dev/null 2>&1; then + su-exec postgres "$@" + else + log "no gosu or su-exec in this image; cannot drop privileges" + return 1 + fi + else + "$@" + fi +} + +slot_exists() { + local found + found="$(as_postgres psql \ + --host="$PRIMARY_HOST" \ + --port="$PRIMARY_PORT" \ + --username="$REPLICATION_USER" \ + --dbname="$REPLICATION_DB" \ + --no-align --tuples-only \ + --command="SELECT 1 FROM pg_replication_slots WHERE slot_name = '${REPLICATION_SLOT}'" \ + 2>/dev/null)" || return 1 + [ "$found" = "1" ] +} + +# standby.signal is the completion marker, not PG_VERSION: pg_basebackup writes +# PG_VERSION early in the stream but standby.signal only once the backup has +# finished, so an interrupted bootstrap is distinguishable from a usable +# standby. This volume only ever holds a standby produced by this script, and +# the replica is never promoted, so leftovers without standby.signal are debris +# and are cleared before retrying. +if [ ! -f "$PGDATA/standby.signal" ]; then + # Refuse to clear a directory that holds a complete cluster. The compose + # volume only ever holds standbys made here, but + # PRISM_POSTGRES_REPLICA_DATA_SOURCE is a production bind mount: aimed by + # mistake at the primary's data directory, the clear below would destroy it. + # + # pg_controldata is the discriminator rather than PG_VERSION, because + # PG_VERSION is also present in the debris of an interrupted base backup, + # which this script must still be free to clear. A readable pg_control means + # a cluster that finished being built; anything less is debris. No marker + # file can serve here -- pg_basebackup refuses a target directory that is + # not empty, dot files included. + if as_postgres pg_controldata "$PGDATA" >/dev/null 2>&1; then + log "refusing to clear ${PGDATA}: it holds a complete Postgres cluster that is not a standby" + log "point PRISM_POSTGRES_REPLICA_DATA_SOURCE at a dedicated volume, or empty it deliberately" + exit 1 + fi + attempt=1 + while :; do + if [ -n "$(ls -A "$PGDATA" 2>/dev/null)" ]; then + log "clearing incomplete data directory ${PGDATA} before base backup" + find "$PGDATA" -mindepth 1 -maxdepth 1 -exec rm -rf {} + + fi + + if ! pg_isready --host="$PRIMARY_HOST" --port="$PRIMARY_PORT" --quiet; then + log "primary ${PRIMARY_HOST}:${PRIMARY_PORT} not accepting connections (attempt ${attempt}/${ATTEMPTS})" + else + basebackup=(pg_basebackup + --host="$PRIMARY_HOST" + --port="$PRIMARY_PORT" + --username="$REPLICATION_USER" + --pgdata="$PGDATA" + --format=plain + --wal-method=stream + --progress + --write-recovery-conf + --slot="$REPLICATION_SLOT") + # -C creates the physical slot as part of the backup. A slot left behind + # by an earlier replica (the primary keeps it while the standby is gone) + # would make -C fail, so reuse it instead of recreating it. + if slot_exists; then + log "reusing existing replication slot ${REPLICATION_SLOT}" + else + basebackup+=(--create-slot) + fi + + log "base backup from ${PRIMARY_HOST}:${PRIMARY_PORT} (attempt ${attempt}/${ATTEMPTS})" + if as_postgres "${basebackup[@]}"; then + log "base backup complete; starting standby" + break + fi + log "base backup failed (attempt ${attempt}/${ATTEMPTS})" + fi + + if [ "$attempt" -ge "$ATTEMPTS" ]; then + log "giving up after ${ATTEMPTS} attempts" + exit 1 + fi + attempt=$((attempt + 1)) + sleep "$RETRY_SECONDS" + done +fi + +# The stock entrypoint chowns/chmods PGDATA, drops to the postgres user, sees a +# populated data directory, and skips initialization. It also picks up the +# pg_hba.conf pg_basebackup copied from the primary's data directory, which +# keeps the loopback healthcheck connections working. +exec docker-entrypoint.sh postgres diff --git a/crates/qbit-pool-builder/Cargo.toml b/crates/qbit-pool-builder/Cargo.toml index de21ea993..68335e351 100644 --- a/crates/qbit-pool-builder/Cargo.toml +++ b/crates/qbit-pool-builder/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "qbit-pool-builder" -version = "1.1.0" +version = "2.0.0" edition = "2021" license = "MIT" description = "Deterministic qbit P2MR coinbase and payout manifest builder" diff --git a/crates/qbit-prism/Cargo.toml b/crates/qbit-prism/Cargo.toml index 3f3f55144..4055f94fe 100644 --- a/crates/qbit-prism/Cargo.toml +++ b/crates/qbit-prism/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "qbit-prism" -version = "1.1.0" +version = "2.0.0" edition = "2021" license = "MIT" description = "Payouts, Rewards, and Integrity Settlement Manifest pool accounting for qbit" diff --git a/crates/qbit-prism/README.md b/crates/qbit-prism/README.md index f7929e387..be0b8a302 100644 --- a/crates/qbit-prism/README.md +++ b/crates/qbit-prism/README.md @@ -19,13 +19,26 @@ defines: - `qbit_share_ledger`: append-only canonical share log ordered by `share_seq` - `qbit_ledger_writer_lease`: one-row coordination table for a single logical writer/failover epoch +- `qbit_pool_blocks.audit_publication_sequence`: durable confirmation and + reactivation order for current audit evidence, independent of block height - `qbit_prism_window(...)`: deterministic newest-backward window query with partial oldest-share weighting +The schema file is cumulative and idempotent. Existing deployments must rerun +it before upgrading when automatic schema initialization is disabled; see the +[ledger operations contract](../../docs/prism-ledger-ops.md#audit-publication-ordering-migration). + Stratum frontends should enqueue share submissions outside this table. Only the active ledger writer inserts rows into `qbit_share_ledger`; that is what keeps all miners in the same reward universe. +[`sql/001_share_ledger_revert_audit_publication_sequence.sql`](sql/001_share_ledger_revert_audit_publication_sequence.sql) +returns `qbit_pool_blocks` to its pre-publication-ordinal shape for operators +rolling back past that migration; it discards assigned ordinals, so read +"Publication Ordinal Rollback And Schema Revert" in +[`docs/prism-ledger-ops.md`](../../docs/prism-ledger-ops.md) before applying +it. + ## Reward Rule For a found block, PRISM's TIDES-style reward window uses shares whose job was diff --git a/crates/qbit-prism/sql/001_share_ledger.sql b/crates/qbit-prism/sql/001_share_ledger.sql index 67ec6dc3d..02fb55488 100644 --- a/crates/qbit-prism/sql/001_share_ledger.sql +++ b/crates/qbit-prism/sql/001_share_ledger.sql @@ -3,6 +3,20 @@ -- Invariant: only one logical writer inserts into qbit_share_ledger. Stratum -- frontends may scale horizontally, but they must feed that writer through a -- queue instead of inserting shares independently. +-- +-- The whole file applies inside exactly one transaction, enforced by the +-- BEGIN/COMMIT wrapper below rather than by any caller's flags. This is what +-- makes the apply atomic everywhere it runs: the coordinator's psql backend +-- (which additionally passes --single-transaction), the native psycopg client +-- (whose single script execution wraps the string in the simple-query +-- protocol), and a manual operator apply with plain autocommit psql. Without +-- it, a failure or interruption mid-file commits some statements -- the +-- carry-forward summary sync triggers -- while later statements, including +-- the summary seed near the end of the file, never run; a live writer +-- mutating carry state in that gap leaves a permanently partial summary +-- (#124). A per-statement autocommit apply of this file is a bug regardless +-- of who performs it. +BEGIN; CREATE TABLE IF NOT EXISTS qbit_ledger_writer_lease ( singleton boolean PRIMARY KEY DEFAULT true CHECK (singleton), @@ -137,6 +151,7 @@ ALTER TABLE qbit_payout_carry_forward CREATE TABLE IF NOT EXISTS qbit_pool_blocks ( block_hash text PRIMARY KEY, + audit_publication_sequence bigint, block_height bigint NOT NULL CHECK (block_height >= 0), parent_hash text NOT NULL, coinbase_txid text NOT NULL, @@ -170,6 +185,628 @@ ALTER TABLE qbit_pool_blocks ADD CONSTRAINT qbit_pool_blocks_chain_state_check CHECK (chain_state IN ('prepared', 'confirmed', 'inactive', 'rejected', 'reversed')); +-- Artifact order is allocated at the durable prepared -> confirmed boundary. +-- Exact confirmed replay and a later inactive -> confirmed transition reuse it, +-- independent of block height. Upgrade existing confirmed and inactive rows +-- deterministically and advance the sequence beyond any value already installed +-- by a partial migration. +-- +-- This serialized phase relies on the whole-file BEGIN/COMMIT wrapper: the +-- transaction-scoped advisory lock is held from here until the end of the +-- entire apply, serializing concurrent appliers of this migration. +SELECT pg_advisory_xact_lock( + hashtext('qbit_audit_publication_sequence_migration') +); +DO $$ +DECLARE + table_namespace text := current_schema(); + table_oid oid; +BEGIN + SELECT pool_blocks.oid + INTO table_oid + FROM pg_class pool_blocks + JOIN pg_namespace namespace ON namespace.oid = pool_blocks.relnamespace + WHERE namespace.nspname = table_namespace + AND pool_blocks.relname = 'qbit_pool_blocks' + AND pool_blocks.relkind = 'r'; + IF table_oid IS NULL THEN + RAISE EXCEPTION 'missing qbit_pool_blocks in current schema'; + END IF; + IF NOT EXISTS ( + SELECT 1 + FROM pg_class relation + JOIN pg_namespace namespace ON namespace.oid = relation.relnamespace + WHERE namespace.nspname = table_namespace + AND relation.relname = 'qbit_audit_publication_sequence_seq' + ) THEN + EXECUTE format( + 'CREATE SEQUENCE %I.qbit_audit_publication_sequence_seq', + table_namespace + ); + END IF; +END; +$$; + +DO $$ +DECLARE + sequence_definition record; +BEGIN + SELECT + sequence.seqtypid, + sequence.seqstart, + sequence.seqincrement, + sequence.seqmax, + sequence.seqmin, + sequence.seqcache, + sequence.seqcycle, + relation.relnamespace AS sequence_namespace, + relation.relpersistence AS sequence_persistence, + relation.relowner AS sequence_owner, + pool_blocks.relnamespace AS table_namespace, + pool_blocks.relowner AS table_owner, + EXISTS ( + SELECT 1 + FROM pg_depend dependency + WHERE dependency.classid = 'pg_class'::regclass + AND dependency.objid = relation.oid + AND dependency.refclassid = 'pg_class'::regclass + AND dependency.refobjsubid > 0 + AND dependency.deptype IN ('a', 'i') + ) AS owned_by_column + INTO sequence_definition + FROM pg_class pool_blocks + JOIN pg_namespace table_namespace + ON table_namespace.oid = pool_blocks.relnamespace + JOIN pg_class relation + ON relation.relnamespace = pool_blocks.relnamespace + AND relation.relname = 'qbit_audit_publication_sequence_seq' + JOIN pg_sequence sequence ON sequence.seqrelid = relation.oid + WHERE relation.relkind = 'S' + AND table_namespace.nspname = current_schema() + AND pool_blocks.relname = 'qbit_pool_blocks' + AND pool_blocks.relkind = 'r'; + IF NOT FOUND + OR sequence_definition.seqtypid <> 'bigint'::regtype + OR sequence_definition.seqstart <> 1 + OR sequence_definition.seqincrement <> 1 + OR sequence_definition.seqmax <> 9223372036854775807 + OR sequence_definition.seqmin <> 1 + OR sequence_definition.seqcache <> 1 + OR sequence_definition.seqcycle + OR sequence_definition.sequence_namespace <> + sequence_definition.table_namespace + OR sequence_definition.sequence_persistence <> 'p' + OR sequence_definition.sequence_owner <> sequence_definition.table_owner + OR sequence_definition.owned_by_column THEN + RAISE EXCEPTION 'invalid audit publication sequence definition'; + END IF; +END; +$$; + +DO $$ +DECLARE + table_namespace text := current_schema(); +BEGIN + IF NOT EXISTS ( + SELECT 1 + FROM pg_class pool_blocks + JOIN pg_namespace namespace + ON namespace.oid = pool_blocks.relnamespace + WHERE namespace.nspname = table_namespace + AND pool_blocks.relname = 'qbit_pool_blocks' + AND pool_blocks.relkind = 'r' + ) THEN + RAISE EXCEPTION 'missing qbit_pool_blocks in current schema'; + END IF; + EXECUTE format( + 'ALTER TABLE %I.qbit_pool_blocks ' + 'ADD COLUMN IF NOT EXISTS audit_publication_sequence bigint', + table_namespace + ); +END; +$$; + +DO $$ +DECLARE + table_oid oid; +BEGIN + SELECT pool_blocks.oid + INTO table_oid + FROM pg_class pool_blocks + JOIN pg_namespace namespace + ON namespace.oid = pool_blocks.relnamespace + WHERE namespace.nspname = current_schema() + AND pool_blocks.relname = 'qbit_pool_blocks' + AND pool_blocks.relkind = 'r'; + IF NOT EXISTS ( + SELECT 1 + FROM pg_attribute + WHERE attrelid = table_oid + AND attname = 'audit_publication_sequence' + AND attnum > 0 + AND NOT attisdropped + AND atttypid = 'bigint'::regtype + AND NOT attnotnull + AND NOT atthasdef + AND attidentity = '' + AND attgenerated = '' + AND attcollation = 0 + ) THEN + RAISE EXCEPTION 'invalid audit publication sequence column definition'; + END IF; +END; +$$; + +DO $$ +DECLARE + pending record; + assigned_sequence numeric; + assignment_start numeric; + invalid_sequence boolean; + duplicate_sequence boolean; + maximum_sequence bigint; + pending_count bigint; + raw_next_sequence numeric; + sequence_last bigint; + sequence_called boolean; + sequence_relation regclass; + table_namespace text := current_schema(); + table_oid oid; +BEGIN + -- Serialize partial/concurrent schema initialization and exclude live + -- confirmation updates while setval/backfill establish the ordinal floor. + SELECT pool_blocks.oid + INTO table_oid + FROM pg_class pool_blocks + JOIN pg_namespace namespace + ON namespace.oid = pool_blocks.relnamespace + WHERE namespace.nspname = table_namespace + AND pool_blocks.relname = 'qbit_pool_blocks' + AND pool_blocks.relkind = 'r'; + IF table_oid IS NULL THEN + RAISE EXCEPTION 'missing qbit_pool_blocks in current schema'; + END IF; + EXECUTE format( + 'LOCK TABLE %I.qbit_pool_blocks IN SHARE ROW EXCLUSIVE MODE', + table_namespace + ); + EXECUTE format( + 'SELECT EXISTS (' + 'SELECT 1 FROM %I.qbit_pool_blocks ' + 'WHERE audit_publication_sequence IS NOT NULL ' + 'AND audit_publication_sequence <= 0)', + table_namespace + ) INTO invalid_sequence; + IF invalid_sequence THEN + RAISE EXCEPTION 'invalid non-positive audit publication sequence'; + END IF; + EXECUTE format( + 'SELECT EXISTS (' + 'SELECT audit_publication_sequence ' + 'FROM %I.qbit_pool_blocks ' + 'WHERE audit_publication_sequence IS NOT NULL ' + 'GROUP BY audit_publication_sequence HAVING count(*) > 1)', + table_namespace + ) INTO duplicate_sequence; + IF duplicate_sequence THEN + RAISE EXCEPTION 'duplicate audit publication sequence'; + END IF; + EXECUTE format( + 'SELECT COALESCE(MAX(audit_publication_sequence), 0) ' + 'FROM %I.qbit_pool_blocks', + table_namespace + ) INTO maximum_sequence; + SELECT sequence.oid::regclass + INTO sequence_relation + FROM pg_class sequence + WHERE sequence.relnamespace = ( + SELECT oid FROM pg_namespace + WHERE nspname = table_namespace + ) + AND sequence.relname = 'qbit_audit_publication_sequence_seq' + AND sequence.relkind = 'S'; + EXECUTE format( + 'SELECT last_value, is_called ' + 'FROM %I.qbit_audit_publication_sequence_seq', + table_namespace + ) INTO sequence_last, sequence_called; + raw_next_sequence := sequence_last::numeric + + CASE WHEN sequence_called THEN 1 ELSE 0 END; + EXECUTE format( + 'SELECT count(*) FROM %I.qbit_pool_blocks ' + 'WHERE chain_state IN (''confirmed'', ''inactive'') ' + 'AND audit_publication_sequence IS NULL', + table_namespace + ) INTO pending_count; + assignment_start := GREATEST( + raw_next_sequence, + maximum_sequence::numeric + 1 + ); + IF pending_count > 0 + AND ( + assignment_start < 1 + OR assignment_start + pending_count::numeric - 1 + > 9223372036854775807::numeric + ) THEN + RAISE EXCEPTION 'audit publication sequence exhausted'; + END IF; + assigned_sequence := assignment_start; + FOR pending IN EXECUTE format( + 'SELECT block_hash FROM %I.qbit_pool_blocks ' + 'WHERE chain_state IN (''confirmed'', ''inactive'') ' + 'AND audit_publication_sequence IS NULL ' + 'ORDER BY found_at, block_hash', + table_namespace + ) + LOOP + EXECUTE format( + 'UPDATE %I.qbit_pool_blocks ' + 'SET audit_publication_sequence = $1 ' + 'WHERE block_hash = $2 ' + 'AND audit_publication_sequence IS NULL', + table_namespace + ) USING assigned_sequence::bigint, pending.block_hash; + assigned_sequence := assigned_sequence + 1; + END LOOP; +END; +$$; + +DO $$ +DECLARE + index_name constant text := + 'qbit_pool_blocks_audit_publication_sequence_idx'; + canonical_index_oid oid; + table_namespace text := current_schema(); + table_oid oid; +BEGIN + SELECT pool_blocks.oid + INTO table_oid + FROM pg_class pool_blocks + JOIN pg_namespace namespace + ON namespace.oid = pool_blocks.relnamespace + WHERE namespace.nspname = table_namespace + AND pool_blocks.relname = 'qbit_pool_blocks' + AND pool_blocks.relkind = 'r'; + IF table_oid IS NULL THEN + RAISE EXCEPTION 'missing qbit_pool_blocks in current schema'; + END IF; + SELECT index_relation.oid + INTO canonical_index_oid + FROM pg_class index_relation + WHERE index_relation.relnamespace = ( + SELECT oid FROM pg_namespace + WHERE nspname = table_namespace + ) + AND index_relation.relname = index_name; + IF canonical_index_oid IS NULL THEN + EXECUTE format( + 'CREATE UNIQUE INDEX %I ' + 'ON %I.qbit_pool_blocks (audit_publication_sequence)', + index_name, + table_namespace + ); + SELECT index_relation.oid + INTO canonical_index_oid + FROM pg_class index_relation + WHERE index_relation.relnamespace = ( + SELECT oid FROM pg_namespace + WHERE nspname = table_namespace + ) + AND index_relation.relname = index_name; + END IF; + IF NOT EXISTS ( + SELECT 1 + FROM pg_index index_definition + JOIN pg_class index_relation + ON index_relation.oid = index_definition.indexrelid + JOIN pg_am access_method + ON access_method.oid = index_relation.relam + JOIN pg_attribute ordinal_attribute + ON ordinal_attribute.attrelid = index_definition.indrelid + AND ordinal_attribute.attname = 'audit_publication_sequence' + AND NOT ordinal_attribute.attisdropped + JOIN pg_opclass operator_class + ON index_definition.indclass::text = operator_class.oid::text + WHERE index_definition.indexrelid = canonical_index_oid + AND index_definition.indrelid = table_oid + AND access_method.amname = 'btree' + AND index_relation.relkind = 'i' + AND index_relation.relnamespace = ( + SELECT relnamespace + FROM pg_class + WHERE oid = index_definition.indrelid + ) + AND index_relation.relpersistence = 'p' + AND index_relation.relowner = ( + SELECT relowner + FROM pg_class + WHERE oid = index_definition.indrelid + ) + AND index_definition.indisunique + AND index_definition.indisvalid + AND index_definition.indisready + AND index_definition.indislive + AND index_definition.indimmediate + AND NOT index_definition.indisprimary + AND NOT index_definition.indisexclusion + AND NOT index_definition.indisclustered + AND NOT index_definition.indisreplident + AND NOT index_definition.indnullsnotdistinct + AND index_definition.indnkeyatts = 1 + AND index_definition.indnatts = 1 + AND index_definition.indkey::text = ordinal_attribute.attnum::text + AND index_definition.indcollation::text = '0' + AND index_definition.indoption::text = '0' + AND operator_class.opcname = 'int8_ops' + AND operator_class.opcmethod = index_relation.relam + AND operator_class.opcnamespace = 'pg_catalog'::regnamespace + AND operator_class.opcintype = 'bigint'::regtype + AND operator_class.opcdefault + AND index_definition.indexprs IS NULL + AND index_definition.indpred IS NULL + ) THEN + RAISE EXCEPTION 'invalid audit publication sequence index definition'; + END IF; + IF EXISTS ( + SELECT 1 + FROM pg_index index_definition + JOIN pg_class index_relation + ON index_relation.oid = index_definition.indexrelid + JOIN pg_am access_method + ON access_method.oid = index_relation.relam + JOIN pg_attribute ordinal_attribute + ON ordinal_attribute.attrelid = index_definition.indrelid + AND ordinal_attribute.attname = 'audit_publication_sequence' + AND NOT ordinal_attribute.attisdropped + JOIN pg_opclass operator_class + ON index_definition.indclass::text = operator_class.oid::text + WHERE index_definition.indrelid = table_oid + AND index_definition.indexrelid <> canonical_index_oid + AND access_method.amname = 'btree' + AND index_relation.relkind = 'i' + AND index_relation.relnamespace = ( + SELECT relnamespace + FROM pg_class + WHERE oid = index_definition.indrelid + ) + AND index_relation.relpersistence = 'p' + AND index_relation.relowner = ( + SELECT relowner + FROM pg_class + WHERE oid = index_definition.indrelid + ) + AND index_definition.indisunique + AND index_definition.indisvalid + AND index_definition.indisready + AND index_definition.indnkeyatts = 1 + AND index_definition.indnatts = 1 + AND index_definition.indkey::text = ordinal_attribute.attnum::text + AND index_definition.indcollation::text = '0' + AND index_definition.indoption::text = '0' + AND operator_class.opcname = 'int8_ops' + AND operator_class.opcmethod = index_relation.relam + AND operator_class.opcnamespace = 'pg_catalog'::regnamespace + AND operator_class.opcintype = 'bigint'::regtype + AND operator_class.opcdefault + AND index_definition.indexprs IS NULL + AND index_definition.indpred IS NULL + ) THEN + RAISE EXCEPTION 'duplicate audit publication sequence index definition'; + END IF; +END; +$$; + +DO $$ +DECLARE + constraint_definition text; + constraint_validated boolean; + table_namespace text := current_schema(); + table_oid oid; +BEGIN + SELECT pool_blocks.oid + INTO table_oid + FROM pg_class pool_blocks + JOIN pg_namespace namespace + ON namespace.oid = pool_blocks.relnamespace + WHERE namespace.nspname = table_namespace + AND pool_blocks.relname = 'qbit_pool_blocks' + AND pool_blocks.relkind = 'r'; + IF table_oid IS NULL THEN + RAISE EXCEPTION 'missing qbit_pool_blocks in current schema'; + END IF; + IF NOT EXISTS ( + SELECT 1 + FROM pg_constraint + WHERE conrelid = table_oid + AND conname = 'qbit_pool_blocks_audit_publication_sequence_check' + ) THEN + EXECUTE format( + 'ALTER TABLE %I.qbit_pool_blocks ' + 'ADD CONSTRAINT ' + 'qbit_pool_blocks_audit_publication_sequence_check ' + 'CHECK ((audit_publication_sequence IS NULL ' + 'OR audit_publication_sequence > 0) ' + 'AND (chain_state <> ''confirmed'' ' + 'OR audit_publication_sequence IS NOT NULL))', + table_namespace + ); + END IF; + SELECT + regexp_replace( + regexp_replace( + pg_get_constraintdef(oid, true), + '[[:space:]]+', + ' ', + 'g' + ), + ' NOT VALID$', + '' + ), + convalidated + INTO constraint_definition, constraint_validated + FROM pg_constraint + WHERE conrelid = table_oid + AND conname = 'qbit_pool_blocks_audit_publication_sequence_check' + AND contype = 'c' + AND NOT condeferrable + AND NOT condeferred + AND NOT connoinherit + AND conislocal + AND coninhcount = 0 + AND cardinality(conkey) = 2 + AND conkey @> ARRAY[ + ( + SELECT attnum::smallint + FROM pg_attribute + WHERE attrelid = table_oid + AND attname = 'audit_publication_sequence' + AND NOT attisdropped + ), + ( + SELECT attnum::smallint + FROM pg_attribute + WHERE attrelid = table_oid + AND attname = 'chain_state' + AND NOT attisdropped + ) + ]::smallint[]; + IF constraint_definition IS NULL + OR constraint_definition <> + 'CHECK ((audit_publication_sequence IS NULL OR audit_publication_sequence > 0) AND (chain_state <> ''confirmed''::text OR audit_publication_sequence IS NOT NULL))' THEN + RAISE EXCEPTION 'invalid audit publication sequence constraint definition'; + END IF; + IF EXISTS ( + SELECT 1 + FROM pg_constraint + WHERE conrelid = table_oid + AND conname <> 'qbit_pool_blocks_audit_publication_sequence_check' + AND contype = 'c' + AND regexp_replace( + regexp_replace( + pg_get_constraintdef(oid, true), + '[[:space:]]+', + ' ', + 'g' + ), + ' NOT VALID$', + '' + ) = constraint_definition + ) THEN + RAISE EXCEPTION 'duplicate audit publication sequence constraint definition'; + END IF; + IF NOT constraint_validated THEN + EXECUTE format( + 'ALTER TABLE %I.qbit_pool_blocks ' + 'VALIDATE CONSTRAINT ' + 'qbit_pool_blocks_audit_publication_sequence_check', + table_namespace + ); + END IF; +END; +$$; + +-- The validated CHECK above requires every confirmed row to carry a +-- publication ordinal, but a pre-ordinal writer confirms with a plain +-- chain_state UPDATE and a column DEFAULT only fires on INSERT. Assign the +-- ordinal here for any confirming write that omits it so the constraint +-- stays satisfiable under a code-only rollback; ordinal-aware writers set +-- the value explicitly in the same UPDATE, making this a no-op for them. +CREATE OR REPLACE FUNCTION qbit_pool_blocks_assign_publication_ordinal() +RETURNS trigger +LANGUAGE plpgsql +AS $$ +DECLARE + publication_sequence pg_catalog.regclass; +BEGIN + IF NEW.chain_state = 'confirmed' + AND NEW.audit_publication_sequence IS NULL + THEN + SELECT sequence.oid::pg_catalog.regclass + INTO publication_sequence + FROM pg_catalog.pg_class pool_blocks + JOIN pg_catalog.pg_class sequence + ON sequence.relnamespace = pool_blocks.relnamespace + AND sequence.relname = 'qbit_audit_publication_sequence_seq' + AND sequence.relkind = 'S' + WHERE pool_blocks.oid = TG_RELID; + IF publication_sequence IS NULL THEN + RAISE EXCEPTION 'missing audit publication sequence'; + END IF; + NEW.audit_publication_sequence := + pg_catalog.nextval(publication_sequence); + END IF; + RETURN NEW; +END; +$$; + +DO $$ +DECLARE + table_namespace text := current_schema(); +BEGIN + EXECUTE format( + 'DROP TRIGGER IF EXISTS ' + 'qbit_pool_blocks_assign_publication_ordinal ' + 'ON %I.qbit_pool_blocks', + table_namespace + ); + EXECUTE format( + 'CREATE TRIGGER qbit_pool_blocks_assign_publication_ordinal ' + 'BEFORE INSERT OR UPDATE ON %I.qbit_pool_blocks ' + 'FOR EACH ROW ' + 'EXECUTE FUNCTION %I.qbit_pool_blocks_assign_publication_ordinal()', + table_namespace, + table_namespace + ); +END; +$$; + +-- Sequence operations are nontransactional in PostgreSQL: a setval against a +-- pre-existing sequence is NOT undone by a rollback of this transaction (only +-- a sequence created inside it disappears with it). The sole allocator +-- mutation therefore still runs last, after every row and catalog validation, +-- so a rejected apply leaves an existing sequence's exact state untouched; +-- if a later statement of the apply fails, the row assignments roll back but +-- the setval persists. That residue is benign: this block only ever advances +-- the sequence, so the next apply assigns higher ordinals, leaving gaps that +-- no validation rejects. Do not move validation after the setval. +DO $$ +DECLARE + maximum_sequence bigint; + raw_next_sequence numeric; + sequence_called boolean; + sequence_last bigint; + sequence_relation regclass; + table_namespace text := current_schema(); +BEGIN + SELECT sequence.oid::regclass + INTO sequence_relation + FROM pg_class sequence + JOIN pg_namespace namespace + ON namespace.oid = sequence.relnamespace + WHERE namespace.nspname = table_namespace + AND sequence.relname = 'qbit_audit_publication_sequence_seq' + AND sequence.relkind = 'S'; + IF sequence_relation IS NULL THEN + RAISE EXCEPTION 'missing audit publication sequence'; + END IF; + EXECUTE format( + 'SELECT COALESCE(MAX(audit_publication_sequence), 0) ' + 'FROM %I.qbit_pool_blocks', + table_namespace + ) INTO maximum_sequence; + EXECUTE format( + 'SELECT last_value, is_called ' + 'FROM %I.qbit_audit_publication_sequence_seq', + table_namespace + ) INTO sequence_last, sequence_called; + raw_next_sequence := sequence_last::numeric + + CASE WHEN sequence_called THEN 1 ELSE 0 END; + IF maximum_sequence::numeric >= raw_next_sequence THEN + PERFORM setval(sequence_relation, maximum_sequence, true); + END IF; +END; +$$; + CREATE TABLE IF NOT EXISTS qbit_pool_audit_bundles ( block_hash text PRIMARY KEY REFERENCES qbit_pool_blocks(block_hash), audit_bundle jsonb NOT NULL, @@ -876,27 +1513,6 @@ CREATE TRIGGER qbit_pool_blocks_carry_forward_current_truncate_sync FOR EACH STATEMENT EXECUTE FUNCTION qbit_carry_forward_current_truncate_sync(); --- Seed the summary exactly once when this schema is applied to a database --- that already holds active carry history (the summary can only be empty --- while active rows exist before the triggers above have ever run). -DO $$ -BEGIN - IF NOT EXISTS (SELECT 1 FROM qbit_payout_carry_forward_current) - AND EXISTS ( - SELECT 1 - FROM qbit_payout_carry_forward carry - JOIN qbit_pool_blocks block - ON block.block_hash = carry.block_hash - WHERE carry.maturity_state <> 'reversed' - AND block.chain_state = 'confirmed' - AND block.maturity_state <> 'reversed' - ) - THEN - PERFORM qbit_rebuild_carry_forward_current_balances(); - END IF; -END -$$; - CREATE OR REPLACE FUNCTION qbit_current_carry_forward_balances() RETURNS TABLE ( miner_id text, @@ -1003,6 +1619,47 @@ AS $$ ORDER BY 1; $$; +-- Seed or repair the summary from carry history whenever the summary +-- disagrees with what it summarizes. This must run after the drift function +-- above and after the sync triggers earlier in the file. A non-atomic apply +-- (a per-statement autocommit psql run) can commit those triggers before +-- this block runs; a live writer mutating carry state in that gap leaves a +-- partial summary holding only post-trigger deltas. The previous guard +-- seeded only when the summary was empty, so it treated that partial +-- summary as already seeded and locked the damage in permanently on this +-- apply and every later one, silently under-reporting every miner's balance +-- by their pre-upgrade carry. Comparing the summary's active row-count +-- total against the active carry history -- plus the drift check for +-- balance-only divergence -- makes this a repair instead: an +-- already-poisoned deployment is rebuilt from history by the next apply. +-- Both counts come from one statement, so a concurrent commit cannot +-- manufacture a false mismatch between them; a rebuild it triggers is +-- correct by construction (it is the same recomputation the truncate +-- triggers perform). +DO $$ +DECLARE + summary_active_rows bigint; + ledger_active_rows bigint; +BEGIN + SELECT + (SELECT COALESCE(SUM(active_row_count), 0) + FROM qbit_payout_carry_forward_current), + (SELECT COUNT(*) + FROM qbit_payout_carry_forward carry + JOIN qbit_pool_blocks block + ON block.block_hash = carry.block_hash + WHERE carry.maturity_state <> 'reversed' + AND block.chain_state = 'confirmed' + AND block.maturity_state <> 'reversed') + INTO summary_active_rows, ledger_active_rows; + IF summary_active_rows <> ledger_active_rows + OR EXISTS (SELECT 1 FROM qbit_carry_forward_current_drift()) + THEN + PERFORM qbit_rebuild_carry_forward_current_balances(); + END IF; +END +$$; + CREATE OR REPLACE FUNCTION qbit_current_owed_balances() RETURNS TABLE ( miner_id text, @@ -1518,7 +2175,20 @@ AS $$ DECLARE lease_count integer; confirmed_count integer; + publication_sequence pg_catalog.regclass; BEGIN + SELECT sequence.oid::pg_catalog.regclass + INTO publication_sequence + FROM pg_catalog.pg_class pool_blocks + JOIN pg_catalog.pg_class sequence + ON sequence.relnamespace = pool_blocks.relnamespace + AND sequence.relname = 'qbit_audit_publication_sequence_seq' + AND sequence.relkind = 'S' + WHERE pool_blocks.oid = 'qbit_pool_blocks'::pg_catalog.regclass + AND pool_blocks.relkind = 'r'; + IF publication_sequence IS NULL THEN + RAISE EXCEPTION 'missing audit publication sequence'; + END IF; UPDATE qbit_ledger_writer_lease SET lease_expires_at = clock_timestamp() + lease_duration, updated_at = clock_timestamp() @@ -1533,13 +2203,27 @@ BEGIN END IF; UPDATE qbit_pool_blocks - SET chain_state = 'confirmed' + SET chain_state = 'confirmed', + audit_publication_sequence = pg_catalog.nextval(publication_sequence) WHERE block_hash = confirmed_block_hash AND block_height = active_tip_height AND chain_state = 'prepared' AND maturity_state = 'immature'; GET DIAGNOSTICS confirmed_count = ROW_COUNT; + -- The row is already confirmed at this (hash, height): this call changed + -- nothing. Report the distinct idempotent disposition (2) rather than 1. + -- 1 means exactly "this call flipped a prepared row", and only a flip + -- allocates an audit publication ordinal through the nextval above; the + -- replay below returns with the ordinal its original flip assigned and no + -- sequence burned. A caller must be able to tell the two apart: a + -- genuinely new confirmation has to publish payout state, while a replay + -- is already covered by the publication its flip produced, and + -- republishing it bumps the payout generation, wipes the job-bundle + -- cache, and aborts in-flight refreshes for identical state (issue #61). + -- This is the durable form of that distinction -- it is recorded in the + -- row, not inferred from process state -- so it survives reorg corners + -- that no in-memory discriminator can cover. IF confirmed_count = 0 AND EXISTS ( SELECT 1 @@ -1549,7 +2233,26 @@ BEGIN AND chain_state = 'confirmed' AND maturity_state <> 'reversed' ) THEN - RETURN 1; + RETURN 2; + END IF; + + -- The candidate row was terminally disposed before this confirmation + -- arrived (reorg quarantine, rejection, or reversal). That is a routine + -- race, not corruption: report it as the distinct superseded disposition + -- (-1) so the caller can abandon the candidate without treating the + -- ledger as unexplained. A plain 0 keeps meaning: no row, or a live row + -- this confirmation does not match. + IF confirmed_count = 0 + AND EXISTS ( + SELECT 1 + FROM qbit_pool_blocks + WHERE block_hash = confirmed_block_hash + AND ( + chain_state IN ('inactive', 'rejected', 'reversed') + OR maturity_state = 'reversed' + ) + ) THEN + RETURN -1; END IF; RETURN confirmed_count; @@ -1649,6 +2352,34 @@ BEGIN END; $$; +-- PL/pgSQL plans relation references on first execution. Pin confirmation +-- ordinal allocation and reactivation to their installation schema, and list +-- pg_temp last so a caller +-- cannot redirect the lease, pool-block, or sequence names through its own +-- search path or a temporary relation. +DO $$ +DECLARE + installation_schema pg_catalog.text := pg_catalog.current_schema(); +BEGIN + EXECUTE pg_catalog.format( + 'ALTER FUNCTION %I.qbit_confirm_pool_block(' + 'pg_catalog.text, pg_catalog.int8, pg_catalog.text, ' + 'pg_catalog.int8, pg_catalog.text, pg_catalog.interval) ' + 'SET search_path TO pg_catalog, %I, pg_temp', + installation_schema, + installation_schema + ); + EXECUTE pg_catalog.format( + 'ALTER FUNCTION %I.qbit_reactivate_pool_block(' + 'pg_catalog.text, pg_catalog.int8, pg_catalog.text, ' + 'pg_catalog.int8, pg_catalog.text, pg_catalog.interval) ' + 'SET search_path TO pg_catalog, %I, pg_temp', + installation_schema, + installation_schema + ); +END; +$$; + DROP FUNCTION IF EXISTS qbit_reject_prepared_pool_block(text, bigint, text, bigint, text); CREATE OR REPLACE FUNCTION qbit_reject_prepared_pool_block( @@ -1796,3 +2527,96 @@ BEGIN RETURN block_count + payout_count + carry_count + fanout_count; END; $$; + +-- Durable retention of the last delivered, safe per-worker vardiff wire +-- difficulty, keyed by listener lane plus exact Stratum username. Rows are +-- operational preload hints for reconnecting sessions only: they never join +-- share accounting or payout artifacts, and pruning them loses no canonical +-- state. Exact usernames are public identities, not authentication secrets. +-- +-- difficulty is an unconstrained numeric so the decimal wire value survives +-- without lossy integer conversion; the CHECK pins it to a positive finite +-- value (numeric NaN compares greater than every value including zero, and +-- only the < 'Infinity' comparison excludes it, so both guards are +-- load-bearing). evidence_at records when share-backed evidence last +-- validated the value — the writer's upsert keeps it monotonic per key — +-- while updated_at records the last write of any kind. The single index +-- serves both the bounded newest-first preload (its column order matches the +-- preload ORDER BY exactly) and the evidence_at range scan the prune uses. +-- The key columns collate as "C": preload/prune tie-breaks are byte order, +-- identical across deployments and to the in-memory store's code-point +-- ordering, instead of drifting with the database's locale collation. +-- This DDL block must stay byte-identical to WORKER_DIFFICULTY_SCHEMA_SQL in +-- lab/prism/worker_difficulty_store.py; a contract test pins the match. +CREATE TABLE IF NOT EXISTS qbit_worker_difficulty ( + listener text COLLATE "C" NOT NULL CHECK (listener <> ''), + worker_username text COLLATE "C" NOT NULL CHECK (worker_username <> ''), + difficulty numeric NOT NULL CHECK ( + difficulty > 0 AND difficulty < 'Infinity'::numeric + ), + evidence_at timestamptz NOT NULL, + updated_at timestamptz NOT NULL DEFAULT clock_timestamp(), + PRIMARY KEY (listener, worker_username) +); + +CREATE INDEX IF NOT EXISTS qbit_worker_difficulty_evidence_idx + ON qbit_worker_difficulty (evidence_at DESC, listener, worker_username); + +-- Incremental hashrate rollups for the public dashboard hashrate-series +-- endpoint. qbit_share_ledger rows are immutable after insert and share_seq +-- is append-only, so a single monotonically advancing watermark +-- (qbit_hashrate_rollup_progress.last_share_seq) makes per-bucket +-- accumulation exact: every share is folded into its (grain, bucket) rows +-- exactly once, regardless of accepted_at ordering. Buckets are keyed by +-- floor(extract(epoch FROM accepted_at) / grain)::bigint * grain -- the +-- exact expression the serving query uses -- so rollup and raw bucketing +-- can never disagree. Only the lease-holding coordinator maintains these +-- tables; the public read tier only reads them. accepted_share_difficulty +-- is unconstrained numeric: source rows are numeric(78, 0), but a bucket +-- total sums many of them and may need more digits than any single share -- +-- a constrained aggregate column would make the maintenance upsert overflow +-- on every retry and wedge the watermark. sum(numeric) in the raw scan is +-- likewise unconstrained, so text renderings stay byte-identical. +CREATE TABLE IF NOT EXISTS qbit_hashrate_rollup_pool ( + grain_seconds integer NOT NULL CHECK (grain_seconds IN (300, 3600, 86400)), + bucket_epoch bigint NOT NULL CHECK (bucket_epoch >= 0), + accepted_share_count bigint NOT NULL CHECK (accepted_share_count >= 0), + accepted_share_difficulty numeric NOT NULL CHECK (accepted_share_difficulty >= 0), + PRIMARY KEY (grain_seconds, bucket_epoch) +); + +CREATE TABLE IF NOT EXISTS qbit_hashrate_rollup_miner ( + grain_seconds integer NOT NULL CHECK (grain_seconds IN (300, 3600, 86400)), + bucket_epoch bigint NOT NULL CHECK (bucket_epoch >= 0), + miner_id text NOT NULL, + accepted_share_count bigint NOT NULL CHECK (accepted_share_count >= 0), + accepted_share_difficulty numeric NOT NULL CHECK (accepted_share_difficulty >= 0), + PRIMARY KEY (grain_seconds, bucket_epoch, miner_id) +); + +-- Databases that applied the earlier constrained definition are widened in +-- place; from numeric(78, 0) to unconstrained numeric this is a +-- catalog-only change, and it is a no-op once the column is unconstrained. +ALTER TABLE qbit_hashrate_rollup_pool + ALTER COLUMN accepted_share_difficulty TYPE numeric; +ALTER TABLE qbit_hashrate_rollup_miner + ALTER COLUMN accepted_share_difficulty TYPE numeric; + +CREATE INDEX IF NOT EXISTS qbit_hashrate_rollup_miner_series_idx + ON qbit_hashrate_rollup_miner (miner_id, grain_seconds, bucket_epoch); + +-- Single-row watermark. A missing row means the rollups have never run: +-- the serving query then degrades to the raw ledger scan, so a +-- pre-migration database or a half-deployed writer stays correct. The +-- first maintenance pass seeds the row itself and starts from sequence 0, +-- which is also how a grown ledger backfills -- there is no separate +-- migration path. +CREATE TABLE IF NOT EXISTS qbit_hashrate_rollup_progress ( + singleton boolean PRIMARY KEY DEFAULT true CHECK (singleton), + last_share_seq bigint NOT NULL CHECK (last_share_seq >= 0), + updated_at timestamptz NOT NULL DEFAULT clock_timestamp() +); + +-- Pairs with the BEGIN at the top of the file: the apply is one transaction +-- no matter which client performs it. +COMMIT; diff --git a/crates/qbit-prism/sql/001_share_ledger_revert_audit_publication_sequence.sql b/crates/qbit-prism/sql/001_share_ledger_revert_audit_publication_sequence.sql new file mode 100644 index 000000000..1d07deebd --- /dev/null +++ b/crates/qbit-prism/sql/001_share_ledger_revert_audit_publication_sequence.sql @@ -0,0 +1,367 @@ +-- Schema revert for the audit publication ordinal objects installed by +-- 001_share_ledger.sql: the qbit_pool_blocks.audit_publication_sequence +-- column, its validated CHECK constraint and unique index, the +-- qbit_pool_blocks_assign_publication_ordinal trigger and function, and the +-- qbit_audit_publication_sequence_seq allocator. +-- +-- This file is the second, belt-and-braces rollback path. The primary +-- rollback path needs no schema change at all: the assignment trigger keeps +-- a pre-ordinal writer's plain chain_state UPDATE satisfying the validated +-- CHECK constraint, so rolling back one release of coordinator code runs +-- correctly against the migrated schema. Apply this file only when the +-- schema itself must return to its pre-ordinal shape: rolling back more +-- than one release, or aligning a live ledger with a restored pre-migration +-- base backup. +-- +-- DATA LOSS: dropping the column discards every assigned publication +-- ordinal. Read "Publication Ordinal Rollback And Schema Revert" in +-- docs/prism-ledger-ops.md first; it states what is lost and the required +-- procedure (stop the coordinator, take a backup). +-- +-- The revert is one transaction serialized behind the same advisory lock as +-- the forward migration. Any failing step aborts the whole file, so the +-- schema is never left half-reverted. Each step is guarded in the style of +-- the forward migration: re-running this file after a resolved failure, on +-- a partially migrated schema, on a never-migrated schema, or on an +-- already-reverted schema is safe, and the last two are no-ops. + +BEGIN; +SELECT pg_advisory_xact_lock( + hashtext('qbit_audit_publication_sequence_migration') +); + +-- Fail before touching anything if the pool-block table is missing, then +-- take the table lock every later step relies on. ACCESS EXCLUSIVE is what +-- DROP COLUMN acquires anyway; taking it first excludes every reader and +-- writer for the whole revert instead of escalating mid-way, and stalls +-- here -- loudly, under any configured lock_timeout -- behind a coordinator +-- that was not stopped, instead of breaking it mid-transaction. +DO $$ +DECLARE + table_namespace text := current_schema(); + table_oid oid; + discarded_count bigint; +BEGIN + SELECT pool_blocks.oid + INTO table_oid + FROM pg_class pool_blocks + JOIN pg_namespace namespace + ON namespace.oid = pool_blocks.relnamespace + WHERE namespace.nspname = table_namespace + AND pool_blocks.relname = 'qbit_pool_blocks' + AND pool_blocks.relkind = 'r'; + IF table_oid IS NULL THEN + RAISE EXCEPTION 'missing qbit_pool_blocks in current schema'; + END IF; + EXECUTE format( + 'LOCK TABLE %I.qbit_pool_blocks IN ACCESS EXCLUSIVE MODE', + table_namespace + ); + IF EXISTS ( + SELECT 1 + FROM pg_attribute + WHERE attrelid = table_oid + AND attname = 'audit_publication_sequence' + AND attnum > 0 + AND NOT attisdropped + ) THEN + EXECUTE format( + 'SELECT count(*) FROM %I.qbit_pool_blocks ' + 'WHERE audit_publication_sequence IS NOT NULL', + table_namespace + ) INTO discarded_count; + RAISE NOTICE + 'reverting audit publication ordinal schema: discarding % assigned ordinal(s)', + discarded_count; + END IF; +END; +$$; + +-- 1/5: the CHECK constraint. Dropped first so no step of this file -- even +-- replayed statement by statement outside the transaction -- ever leaves a +-- state that requires confirmed rows to carry an ordinal after anything +-- that assigns one is gone. +DO $$ +BEGIN + EXECUTE format( + 'ALTER TABLE %I.qbit_pool_blocks DROP CONSTRAINT IF EXISTS ' + 'qbit_pool_blocks_audit_publication_sequence_check', + current_schema() + ); +END; +$$; + +-- 2/5: the assignment trigger, then its function. The trigger must go +-- before the function it executes; the function drop deliberately omits +-- CASCADE so an unexpected surviving dependency fails the revert instead of +-- being silently destroyed. +DO $$ +DECLARE + table_namespace text := current_schema(); +BEGIN + EXECUTE format( + 'DROP TRIGGER IF EXISTS ' + 'qbit_pool_blocks_assign_publication_ordinal ' + 'ON %I.qbit_pool_blocks', + table_namespace + ); + EXECUTE format( + 'DROP FUNCTION IF EXISTS ' + '%I.qbit_pool_blocks_assign_publication_ordinal()', + table_namespace + ); +END; +$$; + +-- 3/5: the unique index. Dropping the column would remove it implicitly; +-- dropping it explicitly first lets step 4 insist that nothing else still +-- hangs off the column. Refuse a same-named relation that is not an index +-- on qbit_pool_blocks rather than dropping an impostor. +DO $$ +DECLARE + table_namespace text := current_schema(); + table_oid oid; + index_oid oid; +BEGIN + SELECT pool_blocks.oid + INTO table_oid + FROM pg_class pool_blocks + JOIN pg_namespace namespace + ON namespace.oid = pool_blocks.relnamespace + WHERE namespace.nspname = table_namespace + AND pool_blocks.relname = 'qbit_pool_blocks' + AND pool_blocks.relkind = 'r'; + SELECT index_relation.oid + INTO index_oid + FROM pg_class index_relation + JOIN pg_namespace namespace + ON namespace.oid = index_relation.relnamespace + WHERE namespace.nspname = table_namespace + AND index_relation.relname = + 'qbit_pool_blocks_audit_publication_sequence_idx'; + IF index_oid IS NULL THEN + RETURN; + END IF; + IF NOT EXISTS ( + SELECT 1 + FROM pg_index index_definition + WHERE index_definition.indexrelid = index_oid + AND index_definition.indrelid = table_oid + ) THEN + RAISE EXCEPTION + 'refusing to drop %.qbit_pool_blocks_audit_publication_sequence_idx: not an index on qbit_pool_blocks', + table_namespace; + END IF; + EXECUTE format( + 'DROP INDEX %I.qbit_pool_blocks_audit_publication_sequence_idx', + table_namespace + ); +END; +$$; + +-- 4/5: the column itself. This is the destructive step: every assigned +-- ordinal is discarded with it. The migration's own constraint and index +-- were dropped above and the forward migration refuses duplicates of +-- either, so anything still attached to the column here was created by +-- someone else; enumerate and refuse instead of letting DROP COLUMN +-- silently take auto-dependent indexes or constraints down with it. +-- Dependencies outside the table -- a view over the column, for example -- +-- still fail the plain (non-CASCADE) DROP COLUMN itself. +DO $$ +DECLARE + table_namespace text := current_schema(); + table_oid oid; + ordinal_attnum smallint; + leftover_dependents text; +BEGIN + SELECT pool_blocks.oid + INTO table_oid + FROM pg_class pool_blocks + JOIN pg_namespace namespace + ON namespace.oid = pool_blocks.relnamespace + WHERE namespace.nspname = table_namespace + AND pool_blocks.relname = 'qbit_pool_blocks' + AND pool_blocks.relkind = 'r'; + SELECT attnum + INTO ordinal_attnum + FROM pg_attribute + WHERE attrelid = table_oid + AND attname = 'audit_publication_sequence' + AND attnum > 0 + AND NOT attisdropped; + IF ordinal_attnum IS NULL THEN + RETURN; + END IF; + SELECT string_agg(dependent_name, ', ' ORDER BY dependent_name) + INTO leftover_dependents + FROM ( + SELECT index_relation.relname AS dependent_name + FROM pg_index index_definition + JOIN pg_class index_relation + ON index_relation.oid = index_definition.indexrelid + WHERE index_definition.indrelid = table_oid + AND ( + ordinal_attnum = ANY (index_definition.indkey) + OR EXISTS ( + SELECT 1 + FROM pg_depend dependency + WHERE dependency.classid = 'pg_class'::regclass + AND dependency.objid = index_definition.indexrelid + AND dependency.refclassid = 'pg_class'::regclass + AND dependency.refobjid = table_oid + AND dependency.refobjsubid = ordinal_attnum + ) + ) + UNION + SELECT table_constraint.conname + FROM pg_constraint table_constraint + WHERE table_constraint.conrelid = table_oid + AND table_constraint.conkey @> ARRAY[ordinal_attnum] + ) dependents; + IF leftover_dependents IS NOT NULL THEN + RAISE EXCEPTION + 'refusing to drop qbit_pool_blocks.audit_publication_sequence: still referenced by %', + leftover_dependents; + END IF; + EXECUTE format( + 'ALTER TABLE %I.qbit_pool_blocks ' + 'DROP COLUMN audit_publication_sequence', + table_namespace + ); +END; +$$; + +-- 5/5: the allocator sequence, once nothing assigns from it. Refuse a +-- same-named relation that is not a free-standing sequence: a column-owned +-- sequence here belongs to some other schema arrangement, not to this +-- migration. The plain (non-CASCADE) drop still fails loudly on any other +-- surviving dependency, such as a column DEFAULT. +DO $$ +DECLARE + table_namespace text := current_schema(); + sequence_oid oid; +BEGIN + SELECT sequence.oid + INTO sequence_oid + FROM pg_class sequence + JOIN pg_namespace namespace + ON namespace.oid = sequence.relnamespace + WHERE namespace.nspname = table_namespace + AND sequence.relname = 'qbit_audit_publication_sequence_seq'; + IF sequence_oid IS NULL THEN + RETURN; + END IF; + IF NOT EXISTS ( + SELECT 1 + FROM pg_class + WHERE oid = sequence_oid + AND relkind = 'S' + ) THEN + RAISE EXCEPTION + 'refusing to drop %.qbit_audit_publication_sequence_seq: not a sequence', + table_namespace; + END IF; + IF EXISTS ( + SELECT 1 + FROM pg_depend dependency + WHERE dependency.classid = 'pg_class'::regclass + AND dependency.objid = sequence_oid + AND dependency.refclassid = 'pg_class'::regclass + AND dependency.refobjsubid > 0 + AND dependency.deptype IN ('a', 'i') + ) THEN + RAISE EXCEPTION + 'refusing to drop column-owned %.qbit_audit_publication_sequence_seq', + table_namespace; + END IF; + EXECUTE format( + 'DROP SEQUENCE %I.qbit_audit_publication_sequence_seq', + table_namespace + ); +END; +$$; + +-- Restore the pre-ordinal confirmation function. The migrated body assigns +-- nextval() on the sequence dropped above and raises if that sequence is +-- missing, so leaving it installed would break every confirmation for a +-- deployment that applies this revert and then starts pre-ordinal +-- coordinators with PRISM_POSTGRES_INIT_SCHEMA=0. This is the exact +-- pre-migration definition; CREATE OR REPLACE also clears the migration's +-- pinned search_path, which the pre-migration function did not have. +CREATE OR REPLACE FUNCTION qbit_confirm_pool_block( + confirmed_block_hash text, + active_tip_height bigint, + active_writer_id text, + active_writer_epoch bigint, + active_writer_session_token text, + lease_duration interval DEFAULT interval '5 minutes' +) +RETURNS integer +LANGUAGE plpgsql +AS $$ +DECLARE + lease_count integer; + confirmed_count integer; +BEGIN + UPDATE qbit_ledger_writer_lease + SET lease_expires_at = clock_timestamp() + lease_duration, + updated_at = clock_timestamp() + WHERE singleton + AND writer_id = active_writer_id + AND writer_epoch = active_writer_epoch + AND writer_session_token = active_writer_session_token; + GET DIAGNOSTICS lease_count = ROW_COUNT; + + IF lease_count = 0 THEN + RAISE EXCEPTION 'writer lease is not active'; + END IF; + + UPDATE qbit_pool_blocks + SET chain_state = 'confirmed' + WHERE block_hash = confirmed_block_hash + AND block_height = active_tip_height + AND chain_state = 'prepared' + AND maturity_state = 'immature'; + GET DIAGNOSTICS confirmed_count = ROW_COUNT; + + IF confirmed_count = 0 + AND EXISTS ( + SELECT 1 + FROM qbit_pool_blocks + WHERE block_hash = confirmed_block_hash + AND block_height = active_tip_height + AND chain_state = 'confirmed' + AND maturity_state <> 'reversed' + ) THEN + RETURN 1; + END IF; + + RETURN confirmed_count; +END; +$$; + +-- The migration also pinned qbit_reactivate_pool_block's search_path; its +-- body never changed. Clear the pin, when the function exists, to finish +-- restoring the pre-migration shape. +DO $$ +DECLARE + installation_schema text := current_schema(); +BEGIN + IF to_regprocedure( + format( + '%I.qbit_reactivate_pool_block(' + 'text, bigint, text, bigint, text, interval)', + installation_schema + ) + ) IS NOT NULL THEN + EXECUTE format( + 'ALTER FUNCTION %I.qbit_reactivate_pool_block(' + 'pg_catalog.text, pg_catalog.int8, pg_catalog.text, ' + 'pg_catalog.int8, pg_catalog.text, pg_catalog.interval) ' + 'RESET search_path', + installation_schema + ); + END IF; +END; +$$; +COMMIT; diff --git a/crates/qbit-prism/sql/002_candidate_bodies.sql b/crates/qbit-prism/sql/002_candidate_bodies.sql new file mode 100644 index 000000000..33ab99ce7 --- /dev/null +++ b/crates/qbit-prism/sql/002_candidate_bodies.sql @@ -0,0 +1,511 @@ +-- Issue #255: chunked immutable block-candidate bodies. +-- +-- Additive migration. Every statement is idempotent so the ledger can apply +-- it on each start after 001_share_ledger.sql, and nothing here rewrites or +-- drops the legacy ``candidate`` jsonb column: a v1 row stays readable by +-- the compatibility reader for as long as it is pending. +-- +-- Layout: +-- qbit_block_candidate_body one scalar manifest per body (staging/ +-- sealed/retired), owned by the exact +-- producer session while staging; +-- qbit_block_candidate_body_chunk fixed-offset byte chunks, at most +-- 262144 bytes each, with a digest the +-- seal statement re-verifies server-side; +-- qbit_block_candidate_body_span one row per large top-level field of +-- the body (byte span, kind, counts); +-- qbit_block_candidate_body_page one row per bounded page of a large +-- array field (body offset, record index); +-- qbit_block_candidate_outbox gains storage_version, body_id, a small +-- typed replay header and retired_body_id; +-- qbit_prism_schema_capabilities what this schema declares, so a process +-- can refuse a database newer than itself. +-- +-- Immutability is enforced here, not by caller discipline: the part tables +-- accept inserts only while their manifest is ``staging``, never accept +-- updates, and accept deletes only once the manifest is ``retired``; the +-- manifest itself may only move staging -> sealed, staging -> retired and +-- sealed -> retired, with every scalar frozen. The seal statement locks the +-- manifest row, so an upload racing a seal serializes behind it and is then +-- refused. +-- +-- Rollback floor: once a storage_version = 2 outbox row exists, a writer +-- that predates this migration must not run. The terminal-state CHECK below +-- is the schema-level refusal for its terminal writes (a legacy writer never +-- detaches body_id, so it cannot abandon or submit a v2 row); it cannot stop +-- such a process from acquiring the lease and offering blocks before it +-- fails, which is why the compatibility reader ships before v2 writers and +-- why the capability row below lets every later process refuse a database +-- newer than itself at startup. + +CREATE TABLE IF NOT EXISTS qbit_prism_schema_capabilities ( + capability text PRIMARY KEY, + capability_value integer NOT NULL, + updated_at timestamptz NOT NULL DEFAULT clock_timestamp() +); + +CREATE TABLE IF NOT EXISTS qbit_block_candidate_body ( + body_id text PRIMARY KEY CHECK (body_id ~ '^[0-9a-f]{32}$'), + storage_version integer NOT NULL CHECK (storage_version = 2), + block_hash text NOT NULL, + candidate_sha256 text NOT NULL CHECK (candidate_sha256 ~ '^[0-9a-f]{64}$'), + byte_count bigint NOT NULL CHECK (byte_count >= 0), + chunk_count integer NOT NULL CHECK (chunk_count >= 0), + chunk_bytes integer NOT NULL CHECK (chunk_bytes > 0 AND chunk_bytes <= 262144), + share_count bigint NOT NULL CHECK (share_count >= 0), + shares_offset bigint NOT NULL CHECK (shares_offset >= 0), + shares_end bigint NOT NULL CHECK (shares_end >= shares_offset), + span_count integer NOT NULL DEFAULT 0 CHECK (span_count >= 0), + page_count bigint NOT NULL DEFAULT 0 CHECK (page_count >= 0), + state text NOT NULL DEFAULT 'staging' + CHECK (state IN ('staging', 'sealed', 'retired')), + staging_writer_id text NOT NULL, + staging_writer_epoch bigint NOT NULL CHECK (staging_writer_epoch >= 0), + staging_session_token text NOT NULL, + created_at timestamptz NOT NULL DEFAULT clock_timestamp(), + sealed_at timestamptz, + retired_at timestamptz, + CHECK ( + (state = 'staging' AND sealed_at IS NULL AND retired_at IS NULL) + OR (state = 'sealed' AND sealed_at IS NOT NULL AND retired_at IS NULL) + OR (state = 'retired' AND retired_at IS NOT NULL) + ) +); + +CREATE INDEX IF NOT EXISTS qbit_block_candidate_body_orphan_idx + ON qbit_block_candidate_body (created_at, body_id) + WHERE state IN ('staging', 'sealed'); + +CREATE INDEX IF NOT EXISTS qbit_block_candidate_body_retired_idx + ON qbit_block_candidate_body (retired_at, body_id) + WHERE state = 'retired'; + +CREATE TABLE IF NOT EXISTS qbit_block_candidate_body_chunk ( + body_id text NOT NULL REFERENCES qbit_block_candidate_body(body_id), + ordinal integer NOT NULL CHECK (ordinal >= 0), + chunk bytea NOT NULL + CHECK (octet_length(chunk) > 0 AND octet_length(chunk) <= 262144), + chunk_sha256 text NOT NULL CHECK (chunk_sha256 ~ '^[0-9a-f]{64}$'), + PRIMARY KEY (body_id, ordinal) +); + +CREATE TABLE IF NOT EXISTS qbit_block_candidate_body_span ( + body_id text NOT NULL REFERENCES qbit_block_candidate_body(body_id), + field text NOT NULL CHECK (octet_length(field) <= 256), + kind text NOT NULL CHECK (kind IN ('array', 'string', 'value')), + start_offset bigint NOT NULL CHECK (start_offset >= 0), + end_offset bigint NOT NULL CHECK (end_offset >= start_offset), + item_count bigint NOT NULL CHECK (item_count >= 0), + page_count bigint NOT NULL CHECK (page_count >= 0), + pages_exact boolean NOT NULL DEFAULT true, + PRIMARY KEY (body_id, field) +); + +CREATE TABLE IF NOT EXISTS qbit_block_candidate_body_page ( + body_id text NOT NULL REFERENCES qbit_block_candidate_body(body_id), + field text NOT NULL CHECK (octet_length(field) <= 256), + page_ordinal bigint NOT NULL CHECK (page_ordinal >= 0), + body_offset bigint NOT NULL CHECK (body_offset >= 0), + record_index bigint NOT NULL CHECK (record_index >= 0), + PRIMARY KEY (body_id, field, page_ordinal) +); + +-- Part tables (chunks, spans, pages): insert only while staging, no +-- updates, delete only once retired. Touch the staging manifest for every +-- insert. Besides serializing with a seal, the new row version makes a sealer +-- using a repeatable-read snapshot fail serialization after a concurrent +-- upload; row locks alone would not invalidate that older snapshot. +CREATE OR REPLACE FUNCTION qbit_prism_candidate_body_part_guard() RETURNS trigger AS $$ +DECLARE + body_state text; +BEGIN + IF TG_OP = 'UPDATE' THEN + RAISE EXCEPTION 'candidate body parts are immutable (%)', TG_TABLE_NAME + USING ERRCODE = 'integrity_constraint_violation'; + END IF; + IF TG_OP = 'INSERT' THEN + UPDATE qbit_block_candidate_body SET state = state + WHERE body_id = NEW.body_id AND state = 'staging' + RETURNING state INTO body_state; + IF body_state IS DISTINCT FROM 'staging' THEN + RAISE EXCEPTION 'candidate body % is % and accepts no parts', NEW.body_id, COALESCE(body_state, 'missing') + USING ERRCODE = 'integrity_constraint_violation'; + END IF; + RETURN NEW; + END IF; + SELECT state INTO body_state + FROM qbit_block_candidate_body + WHERE body_id = OLD.body_id; + IF body_state IS DISTINCT FROM 'retired' OR EXISTS ( + SELECT 1 FROM qbit_block_candidate_outbox WHERE body_id = OLD.body_id + ) THEN + RAISE EXCEPTION 'candidate body % is % and its parts cannot be deleted', OLD.body_id, COALESCE(body_state, 'missing') + USING ERRCODE = 'integrity_constraint_violation'; + END IF; + RETURN OLD; +END +$$ LANGUAGE plpgsql; + +-- Manifest: staging -> sealed, staging -> retired, sealed -> retired only; +-- every scalar is frozen; a retired manifest can only be deleted. +CREATE OR REPLACE FUNCTION qbit_prism_candidate_body_guard() RETURNS trigger AS $$ +DECLARE + parts_complete boolean; +BEGIN + IF TG_OP = 'DELETE' THEN + IF OLD.state <> 'retired' THEN + RAISE EXCEPTION 'candidate body % is % and cannot be deleted', OLD.body_id, OLD.state + USING ERRCODE = 'integrity_constraint_violation'; + END IF; + RETURN OLD; + END IF; + IF OLD.state = 'retired' THEN + RAISE EXCEPTION 'candidate body % is retired', OLD.body_id + USING ERRCODE = 'integrity_constraint_violation'; + END IF; + -- Publication touches the sealed manifest to invalidate older MVCC + -- snapshots. Its content remains identical; all actual changes still + -- pass through the immutable-state checks below. + IF OLD.state = 'sealed' AND NEW IS NOT DISTINCT FROM OLD THEN + RETURN NEW; + END IF; + IF OLD.state = 'sealed' AND NEW.state <> 'retired' THEN + RAISE EXCEPTION 'candidate body % is sealed', OLD.body_id + USING ERRCODE = 'integrity_constraint_violation'; + END IF; + -- A body an outbox row references cannot be retired by anything but + -- the fenced terminal write that detaches it first. Evaluated here, + -- after any row-lock wait, with a fresh snapshot: a publication that + -- attached the body while this retirement waited for its FOR UPDATE + -- lock is visible now, and the retirement fails closed. + IF NEW.state = 'retired' AND EXISTS ( + SELECT 1 FROM qbit_block_candidate_outbox outbox + WHERE outbox.body_id = OLD.body_id + ) THEN + RAISE EXCEPTION 'candidate body % is referenced by a pending outbox row', OLD.body_id + USING ERRCODE = 'integrity_constraint_violation'; + END IF; + IF NEW.state = 'sealed' AND OLD.state <> 'staging' THEN + RAISE EXCEPTION 'candidate body % cannot be sealed from %', OLD.body_id, OLD.state + USING ERRCODE = 'integrity_constraint_violation'; + END IF; + IF NEW.state = 'sealed' THEN + -- The caller's statement snapshot can predate a concurrent uploader + -- whose manifest lock this UPDATE waited behind. Recheck from this + -- volatile trigger, after acquiring the manifest lock, so that upload + -- is included. The outer seal CTE remains an early completeness check. + SELECT count(*) = NEW.chunk_count + AND COALESCE(sum(octet_length(chunk)), 0) = NEW.byte_count + AND COALESCE(min(ordinal), 0) = 0 + AND COALESCE(max(ordinal), -1) = NEW.chunk_count - 1 + AND COALESCE(bool_and(sha256(chunk) = decode(chunk_sha256, 'hex')), true) + AND COALESCE(bool_and( + octet_length(chunk) = NEW.chunk_bytes + OR ordinal = NEW.chunk_count - 1 + ), true) + INTO parts_complete + FROM qbit_block_candidate_body_chunk + WHERE body_id = NEW.body_id; + IF NOT parts_complete + OR (SELECT count(*) FROM qbit_block_candidate_body_span + WHERE body_id = NEW.body_id) <> NEW.span_count + OR (SELECT count(*) FROM qbit_block_candidate_body_page + WHERE body_id = NEW.body_id) <> NEW.page_count THEN + RAISE EXCEPTION 'candidate body % is incomplete', NEW.body_id + USING ERRCODE = 'integrity_constraint_violation'; + END IF; + END IF; + IF NEW.body_id <> OLD.body_id + OR NEW.storage_version <> OLD.storage_version + OR NEW.block_hash <> OLD.block_hash + OR NEW.candidate_sha256 <> OLD.candidate_sha256 + OR NEW.byte_count <> OLD.byte_count + OR NEW.chunk_count <> OLD.chunk_count + OR NEW.chunk_bytes <> OLD.chunk_bytes + OR NEW.share_count <> OLD.share_count + OR NEW.shares_offset <> OLD.shares_offset + OR NEW.shares_end <> OLD.shares_end + OR NEW.span_count <> OLD.span_count + OR NEW.page_count <> OLD.page_count + OR NEW.staging_writer_id <> OLD.staging_writer_id + OR NEW.staging_writer_epoch <> OLD.staging_writer_epoch + OR NEW.staging_session_token <> OLD.staging_session_token + OR NEW.created_at <> OLD.created_at + THEN + RAISE EXCEPTION 'candidate body % manifest is immutable', OLD.body_id + USING ERRCODE = 'integrity_constraint_violation'; + END IF; + RETURN NEW; +END +$$ LANGUAGE plpgsql; + +DROP TRIGGER IF EXISTS qbit_block_candidate_body_chunk_guard ON qbit_block_candidate_body_chunk; +CREATE TRIGGER qbit_block_candidate_body_chunk_guard + BEFORE INSERT OR UPDATE OR DELETE ON qbit_block_candidate_body_chunk + FOR EACH ROW EXECUTE FUNCTION qbit_prism_candidate_body_part_guard(); + +DROP TRIGGER IF EXISTS qbit_block_candidate_body_span_guard ON qbit_block_candidate_body_span; +CREATE TRIGGER qbit_block_candidate_body_span_guard + BEFORE INSERT OR UPDATE OR DELETE ON qbit_block_candidate_body_span + FOR EACH ROW EXECUTE FUNCTION qbit_prism_candidate_body_part_guard(); + +DROP TRIGGER IF EXISTS qbit_block_candidate_body_page_guard ON qbit_block_candidate_body_page; +CREATE TRIGGER qbit_block_candidate_body_page_guard + BEFORE INSERT OR UPDATE OR DELETE ON qbit_block_candidate_body_page + FOR EACH ROW EXECUTE FUNCTION qbit_prism_candidate_body_part_guard(); + +DROP TRIGGER IF EXISTS qbit_block_candidate_body_manifest_guard ON qbit_block_candidate_body; +CREATE TRIGGER qbit_block_candidate_body_manifest_guard + BEFORE UPDATE OR DELETE ON qbit_block_candidate_body + FOR EACH ROW EXECUTE FUNCTION qbit_prism_candidate_body_guard(); + +CREATE OR REPLACE FUNCTION qbit_prism_bounded_fact(value jsonb, expected text, byte_limit integer) RETURNS jsonb AS $$ + SELECT CASE + WHEN value IS NULL THEN NULL + WHEN jsonb_typeof(value) = expected AND octet_length(value::text) <= byte_limit THEN value + ELSE NULL + END; +$$ LANGUAGE sql IMMUTABLE; + +CREATE OR REPLACE FUNCTION qbit_prism_fact_oversized(value jsonb, expected text, byte_limit integer) RETURNS boolean AS $$ + SELECT value IS NOT NULL + AND jsonb_typeof(value) <> 'null' + AND NOT (jsonb_typeof(value) = expected AND octet_length(value::text) <= byte_limit); +$$ LANGUAGE sql IMMUTABLE; + +-- Bounded replay header: an explicit projection of typed facts, each one +-- within a byte bound or replaced by null with ``oversized`` set. Mirrors +-- ``replay_header_from_fields`` in lab/prism/candidate_codec.py; a page of +-- these has a provable maximum size regardless of what the body holds. +CREATE OR REPLACE FUNCTION qbit_prism_bounded_replay_header(header jsonb) RETURNS jsonb AS $$ +DECLARE + text_limit constant integer := 256; + number_limit constant integer := 128; + hash_limit constant integer := 66; + oversized boolean := false; + pending jsonb := NULL; + bounded_pending jsonb := NULL; + entry record; + template jsonb; + found jsonb; +BEGIN + IF header IS NULL OR jsonb_typeof(header) <> 'object' THEN + RETURN jsonb_build_object('oversized', true); + END IF; + pending := header->'pending_share'; + IF pending IS NOT NULL AND jsonb_typeof(pending) = 'object' THEN + bounded_pending := '{}'::jsonb; + FOR entry IN SELECT key, value FROM jsonb_each(pending) LOOP + IF octet_length(entry.key) > 64 THEN + oversized := true; + CONTINUE; + END IF; + IF jsonb_typeof(entry.value) = 'string' THEN + IF octet_length(entry.value::text) <= text_limit THEN + bounded_pending := bounded_pending || jsonb_build_object(entry.key, entry.value); + ELSE + oversized := true; + bounded_pending := bounded_pending || jsonb_build_object(entry.key, NULL); + END IF; + ELSIF jsonb_typeof(entry.value) = 'number' THEN + IF octet_length(entry.value::text) <= number_limit THEN + bounded_pending := bounded_pending || jsonb_build_object(entry.key, entry.value); + ELSE + oversized := true; + bounded_pending := bounded_pending || jsonb_build_object(entry.key, NULL); + END IF; + ELSIF jsonb_typeof(entry.value) IN ('null', 'boolean') THEN + bounded_pending := bounded_pending || jsonb_build_object(entry.key, entry.value); + ELSE + oversized := true; + bounded_pending := bounded_pending || jsonb_build_object(entry.key, NULL); + END IF; + IF (SELECT count(*) FROM jsonb_object_keys(bounded_pending)) > 32 THEN + oversized := true; + EXIT; + END IF; + END LOOP; + ELSIF pending IS NOT NULL AND jsonb_typeof(pending) <> 'null' THEN + oversized := true; + END IF; + template := header->'template'; + found := header->'found_block'; + RETURN jsonb_build_object( + 'schema', qbit_prism_bounded_fact(header->'schema', 'string', text_limit), + 'block_hash_hex', qbit_prism_bounded_fact(header->'block_hash_hex', 'string', hash_limit), + 'parent_hash', qbit_prism_bounded_fact(header->'parent_hash', 'string', hash_limit), + 'expected_height', qbit_prism_bounded_fact(header->'expected_height', 'number', number_limit), + 'template', CASE + WHEN template IS NOT NULL AND jsonb_typeof(template) = 'object' THEN jsonb_build_object( + 'previousblockhash', qbit_prism_bounded_fact(template->'previousblockhash', 'string', hash_limit), + 'height', qbit_prism_bounded_fact(template->'height', 'number', number_limit), + 'coinbasevalue', qbit_prism_bounded_fact(template->'coinbasevalue', 'number', number_limit) + ) + ELSE NULL + END, + 'found_block', CASE + WHEN found IS NOT NULL AND jsonb_typeof(found) = 'object' THEN jsonb_build_object( + 'network_difficulty', qbit_prism_bounded_fact(found->'network_difficulty', 'number', number_limit) + ) + ELSE NULL + END, + 'pending_share', bounded_pending, + 'credit_share_on_accept', CASE + WHEN jsonb_typeof(header->'credit_share_on_accept') = 'boolean' THEN header->'credit_share_on_accept' + ELSE NULL + END, + 'collection_only', CASE + WHEN jsonb_typeof(header->'collection_only') = 'boolean' THEN header->'collection_only' + ELSE NULL + END, + 'username', qbit_prism_bounded_fact(header->'username', 'string', text_limit), + 'accepted_at_present', COALESCE( + CASE WHEN jsonb_typeof(header->'accepted_at_present') = 'boolean' THEN (header->>'accepted_at_present')::boolean END, + pending IS NOT NULL AND jsonb_typeof(pending) = 'object' AND pending ? 'accepted_at_ms' + ), + 'oversized', oversized + OR qbit_prism_fact_oversized(header->'schema', 'string', text_limit) + OR qbit_prism_fact_oversized(header->'block_hash_hex', 'string', hash_limit) + OR qbit_prism_fact_oversized(header->'parent_hash', 'string', hash_limit) + OR qbit_prism_fact_oversized(header->'expected_height', 'number', number_limit) + OR qbit_prism_fact_oversized(header->'username', 'string', text_limit) + OR (template IS NOT NULL AND jsonb_typeof(template) = 'object' AND ( + qbit_prism_fact_oversized(template->'previousblockhash', 'string', hash_limit) + OR qbit_prism_fact_oversized(template->'height', 'number', number_limit) + OR qbit_prism_fact_oversized(template->'coinbasevalue', 'number', number_limit))) + OR (found IS NOT NULL AND jsonb_typeof(found) = 'object' AND + qbit_prism_fact_oversized(found->'network_difficulty', 'number', number_limit)) + ); +END +$$ LANGUAGE plpgsql IMMUTABLE; + +ALTER TABLE qbit_block_candidate_outbox + ADD COLUMN IF NOT EXISTS storage_version integer NOT NULL DEFAULT 1; +ALTER TABLE qbit_block_candidate_outbox + ADD COLUMN IF NOT EXISTS body_id text REFERENCES qbit_block_candidate_body(body_id); +-- Only pending v2 rows carry a body reference; the manifest trigger, the +-- orphan retirement and the manifest reads all ask "does any outbox row +-- reference this body", and none of them may scan the terminal history. +CREATE INDEX IF NOT EXISTS qbit_block_candidate_outbox_body_idx + ON qbit_block_candidate_outbox (body_id) + WHERE body_id IS NOT NULL; +ALTER TABLE qbit_block_candidate_outbox + ADD COLUMN IF NOT EXISTS retired_body_id text; +-- Small typed replay header: explicit projection written by the v2 writer. +ALTER TABLE qbit_block_candidate_outbox + ADD COLUMN IF NOT EXISTS replay_header jsonb; +ALTER TABLE qbit_block_candidate_outbox + ADD COLUMN IF NOT EXISTS parent_hash text; +ALTER TABLE qbit_block_candidate_outbox + ADD COLUMN IF NOT EXISTS expected_height bigint; + +CREATE OR REPLACE FUNCTION qbit_prism_candidate_publication_guard() RETURNS trigger AS $$ +DECLARE + published_body text; +BEGIN + IF NEW.body_id IS NULL THEN + RETURN NEW; + END IF; + IF TG_OP = 'UPDATE' AND NEW.body_id IS NOT DISTINCT FROM OLD.body_id + AND NEW.block_hash IS NOT DISTINCT FROM OLD.block_hash + AND NEW.candidate_sha256 IS NOT DISTINCT FROM OLD.candidate_sha256 THEN + RETURN NEW; + END IF; + -- The no-op UPDATE also creates a manifest version. A repeatable-read + -- retirement that began before this publication then fails serialization + -- rather than using its old outbox snapshot to retire the referenced body. + UPDATE qbit_block_candidate_body SET state = state + WHERE body_id = NEW.body_id AND state = 'sealed' + AND block_hash = NEW.block_hash AND candidate_sha256 = NEW.candidate_sha256 + RETURNING body_id INTO published_body; + IF published_body IS NULL THEN + RAISE EXCEPTION 'candidate body % is not sealed with matching identity', NEW.body_id + USING ERRCODE = 'integrity_constraint_violation'; + END IF; + RETURN NEW; +END +$$ LANGUAGE plpgsql; + +DROP TRIGGER IF EXISTS qbit_block_candidate_publication_guard ON qbit_block_candidate_outbox; +CREATE TRIGGER qbit_block_candidate_publication_guard + BEFORE INSERT OR UPDATE ON qbit_block_candidate_outbox + FOR EACH ROW EXECUTE FUNCTION qbit_prism_candidate_publication_guard(); + +DO $$ +BEGIN + IF NOT EXISTS ( + SELECT 1 + FROM pg_constraint + WHERE conrelid = 'qbit_block_candidate_outbox'::regclass + AND conname = 'qbit_block_candidate_outbox_storage_version_check' + ) THEN + ALTER TABLE qbit_block_candidate_outbox + ADD CONSTRAINT qbit_block_candidate_outbox_storage_version_check + CHECK (storage_version IN (1, 2)); + END IF; +END +$$; + +-- The pending/terminal CHECK from 001 requires a pending row to carry the +-- legacy jsonb. Replace it with the dual-format rule: a pending v1 row +-- carries jsonb, a pending v2 row carries a body reference, and a terminal +-- row carries neither. Detaching the body at terminalization is what makes +-- the legacy writer's terminal UPDATE fail on a v2 row (it never clears +-- body_id), which is the version refusal the rollback floor relies on. +DO $$ +DECLARE + legacy_check text; +BEGIN + SELECT conname INTO legacy_check + FROM pg_constraint + WHERE conrelid = 'qbit_block_candidate_outbox'::regclass + AND contype = 'c' + AND conname NOT IN ( + 'qbit_block_candidate_outbox_storage_version_check', + 'qbit_block_candidate_outbox_candidate_sha256_check', + 'qbit_block_candidate_outbox_state_check', + 'qbit_block_candidate_outbox_attempt_count_check', + 'qbit_block_candidate_outbox_dual_format_check' + ) + AND pg_get_constraintdef(oid) LIKE '%candidate IS NOT NULL%' + LIMIT 1; + IF legacy_check IS NOT NULL THEN + EXECUTE format( + 'ALTER TABLE qbit_block_candidate_outbox DROP CONSTRAINT %I', + legacy_check + ); + END IF; + IF NOT EXISTS ( + SELECT 1 + FROM pg_constraint + WHERE conrelid = 'qbit_block_candidate_outbox'::regclass + AND conname = 'qbit_block_candidate_outbox_dual_format_check' + ) THEN + ALTER TABLE qbit_block_candidate_outbox + ADD CONSTRAINT qbit_block_candidate_outbox_dual_format_check + CHECK ( + ( + state = 'pending' + AND completed_at IS NULL + AND ( + (storage_version = 1 AND candidate IS NOT NULL AND body_id IS NULL) + OR (storage_version = 2 AND candidate IS NULL AND body_id IS NOT NULL) + ) + ) + OR ( + state IN ('submitted', 'abandoned') + AND completed_at IS NOT NULL + AND candidate IS NULL + AND body_id IS NULL + ) + ); + END IF; +END +$$; + +-- Declare the capability last, once every object above exists. +INSERT INTO qbit_prism_schema_capabilities (capability, capability_value) +VALUES ('candidate_storage_version', 2) +ON CONFLICT (capability) DO UPDATE +SET capability_value = GREATEST(qbit_prism_schema_capabilities.capability_value, EXCLUDED.capability_value), + updated_at = clock_timestamp(); diff --git a/crates/qbit-prism/src/bin/qbit-prism-build-audit-bundle.rs b/crates/qbit-prism/src/bin/qbit-prism-build-audit-bundle.rs index 9d075e9b6..078db22b0 100644 --- a/crates/qbit-prism/src/bin/qbit-prism-build-audit-bundle.rs +++ b/crates/qbit-prism/src/bin/qbit-prism-build-audit-bundle.rs @@ -1,4 +1,8 @@ use qbit_pool_builder::{ManifestSigningKey, SignedPayoutManifest}; +use qbit_prism::window::{ + prepare_window_out_of_range, OutOfRangeField, PayoutWindow, DEFAULT_WINDOW_PAGE_SIZE, + DIFFICULTY_OVERFLOW_MESSAGE, +}; use qbit_prism::{ build_audit_bundle_with_coinbase_options, build_audit_bundle_with_ctv_settlement_options, profile_audit_build, AcceptedShare, AuditBundle, CarryForwardBalance, FanoutFeeRatePolicy, @@ -13,8 +17,17 @@ use std::{env, error::Error, fs, process}; const PHASE_METRICS_PREFIX: &str = "qbit-prism-build-phase-metrics "; /// Version of the --serve JSONL protocol announced in the startup handshake. /// The coordinator refuses to speak to a daemon announcing a different -/// version and falls back to one-shot builds instead. -const SERVE_PROTOCOL_VERSION: u64 = 1; +/// version and falls back to one-shot builds instead. Version 2 adds the +/// prepare_window request (payout-window fold, canonical digest, and +/// incremental advance daemon-side) and unifies its window state with the +/// build cache. Its non-success prepare_window envelopes carry structure, +/// never just prose: `fold_invalid` and `fallback` name the refused +/// condition in a stable `rejection` category (the parity oracle's +/// vocabulary) beside the human-readable `error`, and `out_of_range` -- +/// distinct from a malformed request -- declares an integer outside the +/// daemon's declared widths (`field`, `width`, `error`) so the coordinator +/// folds that window in-process and keeps the daemon. +const SERVE_PROTOCOL_VERSION: u64 = 2; /// Parsed share windows retained by the --serve daemon. Windows rotate with /// payout/artifact generations, so two entries cover the current generation /// plus the previous one still finishing in-flight builds. @@ -60,16 +73,25 @@ struct ServeWindowKey { share_snapshot_sha256: String, } +/// One serve-mode request line, build and prepare_window shapes combined so +/// dispatch needs a single parse of megabyte-scale lines. `request` is absent +/// on build requests (the historical shape) and `"prepare_window"` on window +/// preparations; required fields are validated per shape after dispatch so a +/// malformed request answers with an error instead of killing the daemon. #[derive(Debug, Deserialize)] struct ServeRequest { + #[serde(default)] + request: Option, /// Identity of the parsed share window this build wants. Requests may - /// omit the inline window once a prior request uploaded it. - window_key: ServeWindowKey, + /// omit the inline window once a prior request uploaded or prepared it. + #[serde(default)] + window_key: Option, #[serde(default)] compact_share_identities: Vec, #[serde(default)] compact_shares: Vec, - found_block: FoundBlock, + #[serde(default)] + found_block: Option, #[serde(default)] prior_balances: Vec, #[serde(default)] @@ -80,6 +102,46 @@ struct ServeRequest { witness_merkle_leaves_hex: Vec, #[serde(default)] ctv_settlement: Option, + /// Coordinator append-invalidation epoch. Recorded against prepared + /// window state and echoed back on advance for diagnostics; it never + /// decides which window this daemon will serve. See `WindowState` for + /// why the coordinator's own policy is the only one that can be right. + #[serde(default)] + append_invalidation_epoch: Option, + // prepare_window fields. + #[serde(default)] + mode: Option, + #[serde(default)] + records: Vec, + #[serde(default)] + base_digest: Option, + #[serde(default)] + anchor_job_issued_at_ms: Option, + #[serde(default)] + window_weight: Option, + #[serde(default)] + page_size: Option, +} + +/// One unified window-cache entry: either a raw upload from the build path +/// (compact records, loaned to builds exactly as before) or a prepared +/// window that owns the fold state needed to advance incrementally. Prepared +/// windows are never loaned: builds clone their records so the advance +/// lineage survives every build outcome. +/// +/// The recorded `epoch` is diagnostic only. This daemon never chooses a +/// window: every entry is addressed by the content digest the coordinator +/// asks for, builds are validated by the install fence and the job-bundle +/// epoch checks, and an `advance` base is the coordinator's own +/// digest-verified mirror -- which the coordinator keeps only while its own +/// (finer) append-invalidation policy says it may. Refusing an older-tagged +/// base could therefore never catch anything the coordinator had not +/// already decided; it could only turn the coordinator's deliberate retag, +/// the one that preserves an unaffected window across a late append, into a +/// full DB walk, fold and upload on every replay burst. +enum WindowState { + Uploaded(Vec), + Prepared { window: PayoutWindow, epoch: u64 }, } #[derive(Serialize)] @@ -352,7 +414,7 @@ fn serve_requests( out.flush()?; } - let mut window_cache: Vec<(String, Vec)> = Vec::new(); + let mut window_cache: Vec<(String, WindowState)> = Vec::new(); let mut cache_hits: u64 = 0; let mut cache_misses: u64 = 0; @@ -366,7 +428,23 @@ fn serve_requests( let request: ServeRequest = match serde_json::from_str(&line) { Ok(request) => request, Err(error) => { - respond_error(&stdout, &format!("malformed serve request: {error}"), false)?; + // serde refuses an integer above its declared width with + // the same parse error a corrupt line gets, but the two are + // different outcomes: a prepare_window request carrying + // such a value is ordinary control flow (the daemon + // declines the window, the coordinator folds it + // in-process), never a daemon anomaly. Only the failure + // path pays for the second, exact-digit parse. + if let Some(out_of_range) = prepare_window_out_of_range_line(&line) { + respond_prepare_out_of_range( + &stdout, + &out_of_range.field, + out_of_range.width.name(), + &out_of_range.to_string(), + )?; + } else { + respond_error(&stdout, &format!("malformed serve request: {error}"), false)?; + } continue; } }; @@ -374,7 +452,40 @@ fn serve_requests( // expansion, exactly where one-shot mode samples the same metric, so // transport comparisons measure identical work. let input_deserialization_seconds = input_started.elapsed().as_secs_f64(); - let window_sha = request.window_key.share_snapshot_sha256.clone(); + // No epoch-driven eviction here: see `WindowState`. A window the + // coordinator no longer considers valid is one it never asks for + // again, and the entry it does ask for is one it has already + // cleared under its own policy. Evicting on the tag would only + // discard windows the coordinator still wants. + match request.request.as_deref() { + None => {} + Some("prepare_window") => { + serve_prepare_window( + &stdout, + request, + &mut window_cache, + input_deserialization_seconds, + )?; + continue; + } + Some(other) => { + respond_error( + &stdout, + &format!("unsupported serve request type: {other}"), + false, + )?; + continue; + } + } + let Some(window_key) = request.window_key else { + respond_error(&stdout, "build request carries no window_key", false)?; + continue; + }; + let Some(found_block) = request.found_block else { + respond_error(&stdout, "build request carries no found_block", false)?; + continue; + }; + let window_sha = window_key.share_snapshot_sha256; let uploaded_window = !request.compact_shares.is_empty(); if !uploaded_window && !request.compact_share_identities.is_empty() { respond_error( @@ -384,29 +495,57 @@ fn serve_requests( )?; continue; } - // The window vector is loaned to the build rather than cloned: both + // Uploaded windows are loaned to the build rather than cloned: both // audit build entry points only borrow the shares for derivation and - // move the vector unmodified into AuditBundle.shares, so it is - // reclaimed from the finished bundle below. A failed build drops the - // loaned window; the coordinator's needs_window bounce re-uploads it. + // move the vector unmodified into AuditBundle.shares, so a loan is + // reclaimed from the finished bundle below. Prepared windows are + // never loaned -- the build gets a clone and the entry stays intact, + // because the fold state must survive a failed build to keep + // serving advances. A failed build drops a loaned upload; the + // coordinator's needs_window bounce re-uploads it. + let cached_position = window_cache.iter().position(|(key, _)| key == &window_sha); + let prepared_hit = matches!( + cached_position.map(|position| &window_cache[position].1), + Some(WindowState::Prepared { .. }) + ); let shares: Vec = if uploaded_window { - match expand_compact_shares(&request.compact_share_identities, request.compact_shares) { - Ok(expanded) => { - cache_misses += 1; - window_cache.retain(|(key, _)| key != &window_sha); - expanded + cache_misses += 1; + if prepared_hit { + // The prepared state is the same content-addressed window at + // full fidelity; keep it (and its advance lineage) and serve + // the build from it, ignoring the redundant upload bytes. + let position = cached_position.expect("prepared_hit implies a position"); + match &window_cache[position].1 { + WindowState::Prepared { window, .. } => window.shares_for_build(), + WindowState::Uploaded(_) => unreachable!("prepared_hit checked the variant"), } - Err(error) => { - respond_error(&stdout, &format!("invalid window upload: {error}"), false)?; - continue; + } else { + match expand_compact_shares( + &request.compact_share_identities, + request.compact_shares, + ) { + Ok(expanded) => { + window_cache.retain(|(key, _)| key != &window_sha); + expanded + } + Err(error) => { + respond_error(&stdout, &format!("invalid window upload: {error}"), false)?; + continue; + } } } - } else if let Some(position) = window_cache - .iter() - .position(|(key, _)| key == &window_sha) - { + } else if let Some(position) = cached_position { cache_hits += 1; - window_cache.remove(position).1 + match &window_cache[position].1 { + WindowState::Prepared { window, .. } => window.shares_for_build(), + WindowState::Uploaded(_) => { + let (_, state) = window_cache.remove(position); + match state { + WindowState::Uploaded(shares) => shares, + WindowState::Prepared { .. } => unreachable!("variant checked above"), + } + } + } } else { // Not counted as a miss: the coordinator's follow-up upload of // this same window is the miss that gets counted. @@ -425,7 +564,7 @@ fn serve_requests( .unwrap_or_else(PayoutPolicy::day_one_default); let (bundle_result, phases_seconds) = run_profiled_build( shares, - request.found_block, + found_block, request.prior_balances, payout_policy, request.ctv_settlement, @@ -453,10 +592,15 @@ fn serve_requests( })?; let output_serialization_seconds = output_started.elapsed().as_secs_f64(); let summary_raw = serde_json::value::RawValue::from_string(summary_json)?; - // Reclaim the loaned window from the finished bundle before - // reporting cache occupancy: no bytes were copied on the way in or - // out, and the entry keeps most-recent position. - window_cache.insert(0, (window_sha, bundle.shares)); + // Reclaim a loaned upload from the finished bundle (no bytes were + // copied on the way in or out); a prepared entry only moves to + // most-recent position, its clone in bundle.shares is dropped. + if let Some(position) = window_cache.iter().position(|(key, _)| key == &window_sha) { + let entry = window_cache.remove(position); + window_cache.insert(0, entry); + } else { + window_cache.insert(0, (window_sha, WindowState::Uploaded(bundle.shares))); + } window_cache.truncate(SERVE_WINDOW_CACHE_MAX_ENTRIES); let response = ServeResponse { ok: true, @@ -481,6 +625,333 @@ fn serve_requests( Ok(()) } +/// Classify a line the strict parse refused: a syntactically valid +/// prepare_window request whose first out-of-width integer is returned, or +/// None when the line is malformed (or is not a prepare_window request, whose +/// width problems the build path does not declare). +fn prepare_window_out_of_range_line(line: &str) -> Option { + let document: serde_json::Value = serde_json::from_str(line).ok()?; + if document.get("request").and_then(serde_json::Value::as_str) != Some("prepare_window") { + return None; + } + prepare_window_out_of_range(&document) +} + +/// Handle one prepare_window request against the unified window cache. +/// +/// Success responses append a raw canonical-items section after the JSON +/// envelope line: the full items stream for `full` mode, or the appended +/// suffix for `advance` mode (whose envelope also carries the byte count to +/// drop from the front of the previous stream). The distinguishable +/// non-success outcomes -- `needs_full` (state not held), `fallback` (an +/// advance invariant failed), `fold_invalid` (the snapshot violates the +/// fold's invariants) and `out_of_range` (an integer, or the difficulty +/// total, outside the declared widths) -- are ordinary control flow for the +/// coordinator, never daemon anomalies. The two rejection shapes carry the +/// parity oracle's category in `rejection` so clients never match on the +/// message text. +/// +/// Success envelopes carry `metrics`: the request's JSON parse time +/// (`input_deserialization_seconds`, sampled where the build path samples +/// it), the fold or advance itself (`fold_seconds`), and the canonical +/// digest and items serialization (`output_serialization_seconds`). They +/// let a measurement separate daemon processing from pipe transport (#240); +/// the coordinator ignores them. +fn serve_prepare_window( + stdout: &io::Stdout, + request: ServeRequest, + window_cache: &mut Vec<(String, WindowState)>, + input_deserialization_seconds: f64, +) -> Result<(), Box> { + let Some(request_epoch) = request.append_invalidation_epoch else { + respond_error( + stdout, + "prepare_window carries no append_invalidation_epoch", + false, + )?; + return Ok(()); + }; + let Some(anchor_job_issued_at_ms) = request.anchor_job_issued_at_ms else { + respond_error( + stdout, + "prepare_window carries no anchor_job_issued_at_ms", + false, + )?; + return Ok(()); + }; + match request.mode.as_deref() { + Some("full") => { + let Some(window_weight) = request.window_weight else { + respond_error(stdout, "prepare_window full carries no window_weight", false)?; + return Ok(()); + }; + let page_size = request.page_size.unwrap_or(DEFAULT_WINDOW_PAGE_SIZE); + let fold_started = Instant::now(); + let window = match PayoutWindow::from_full_snapshot( + request.records, + anchor_job_issued_at_ms, + window_weight, + page_size, + ) { + Ok(window) => window, + Err(error) => { + let Some(rejection) = error.rejection_category() else { + // Valid snapshot, but its retained difficulty total + // leaves the declared u128 domain: decline it so + // the coordinator folds in-process, exactly as for + // an out-of-width input. + respond_prepare_out_of_range( + stdout, + "total_difficulty", + "u128", + DIFFICULTY_OVERFLOW_MESSAGE, + )?; + return Ok(()); + }; + // The snapshot itself violates the fold's invariants + // (duplicate share_seq/share_id, non-positive + // difficulty). The coordinator's in-process oracle + // raises the same rejection, so answer with a + // distinguishable outcome instead of a daemon anomaly. + let mut out = stdout.lock(); + serde_json::to_writer( + &mut out, + &serde_json::json!({ + "ok": false, + "request": "prepare_window", + "fold_invalid": true, + "rejection": rejection, + "error": error.to_string(), + }), + )?; + writeln!(out)?; + out.flush()?; + return Ok(()); + } + }; + let fold_seconds = fold_started.elapsed().as_secs_f64(); + let output_started = Instant::now(); + let digest = window.canonical_digest_hex(); + let items = window.canonical_items_bytes(); + let output_serialization_seconds = output_started.elapsed().as_secs_f64(); + let record_count = window.record_count(); + window_cache.retain(|(key, _)| key != &digest); + window_cache.insert( + 0, + ( + digest.clone(), + WindowState::Prepared { + window, + epoch: request_epoch, + }, + ), + ); + window_cache.truncate(SERVE_WINDOW_CACHE_MAX_ENTRIES); + let mut out = stdout.lock(); + serde_json::to_writer( + &mut out, + &serde_json::json!({ + "ok": true, + "request": "prepare_window", + "share_snapshot_sha256": digest, + "record_count": record_count, + "added_rows": 0, + "expired_rows": 0, + "touched_pages": 0, + "window_items_len": items.len(), + "metrics": { + "input_deserialization_seconds": input_deserialization_seconds, + "fold_seconds": fold_seconds, + "output_serialization_seconds": output_serialization_seconds, + }, + }), + )?; + writeln!(out)?; + out.write_all(&items)?; + writeln!(out)?; + out.flush()?; + } + Some("advance") => { + let Some(base_digest) = request.base_digest else { + respond_error(stdout, "prepare_window advance carries no base_digest", false)?; + return Ok(()); + }; + let position = window_cache.iter().position(|(key, state)| { + key == &base_digest && matches!(state, WindowState::Prepared { .. }) + }); + let Some(position) = position else { + // Respawn or eviction dropped the base window; the + // coordinator re-sends a full preparation. + let mut out = stdout.lock(); + serde_json::to_writer( + &mut out, + &serde_json::json!({ + "ok": false, + "request": "prepare_window", + "needs_full": true, + "error": format!("prepared window {base_digest} is not held"), + }), + )?; + writeln!(out)?; + out.flush()?; + return Ok(()); + }; + let fold_started = Instant::now(); + let (advance_result, base_epoch) = match &window_cache[position].1 { + WindowState::Prepared { window, epoch } => ( + window.advance(request.records, anchor_job_issued_at_ms), + *epoch, + ), + WindowState::Uploaded(_) => { + // Only prepared entries can match the predicate above, + // so this arm is dead -- answered rather than panicked + // because the position came from request-derived data. + respond_error( + stdout, + "prepare_window advance matched a non-prepared window", + false, + )?; + return Ok(()); + } + }; + let (advanced, stats, byte_delta) = match advance_result { + Ok(result) => result, + Err(error) => { + let Some(rejection) = error.rejection_category() else { + // The base stays held and untouched; the + // coordinator rebuilds this window in-process. + respond_prepare_out_of_range( + stdout, + "total_difficulty", + "u128", + DIFFICULTY_OVERFLOW_MESSAGE, + )?; + return Ok(()); + }; + // An advance invariant failed: the coordinator must take + // exactly its IncrementalWindowFallback path (clear the + // cache, full-rescan) rather than retiring the daemon. + let mut out = stdout.lock(); + serde_json::to_writer( + &mut out, + &serde_json::json!({ + "ok": false, + "request": "prepare_window", + "fallback": true, + "rejection": rejection, + "error": error.to_string(), + }), + )?; + writeln!(out)?; + out.flush()?; + return Ok(()); + } + }; + let fold_seconds = fold_started.elapsed().as_secs_f64(); + let output_started = Instant::now(); + let digest = advanced.canonical_digest_hex(); + let output_serialization_seconds = output_started.elapsed().as_secs_f64(); + let record_count = advanced.record_count(); + if digest == base_digest { + // Anchor-only advance: replace the entry in place so the + // stored anchor tracks the coordinator's, and promote it. + let (key, _) = window_cache.remove(position); + window_cache.insert( + 0, + ( + key, + WindowState::Prepared { + window: advanced, + epoch: request_epoch, + }, + ), + ); + } else { + // The base entry stays for in-flight builds of the previous + // generation; the bounded cache evicts it naturally. + window_cache.retain(|(key, _)| key != &digest); + window_cache.insert( + 0, + ( + digest.clone(), + WindowState::Prepared { + window: advanced, + epoch: request_epoch, + }, + ), + ); + window_cache.truncate(SERVE_WINDOW_CACHE_MAX_ENTRIES); + } + let mut out = stdout.lock(); + serde_json::to_writer( + &mut out, + &serde_json::json!({ + "ok": true, + "request": "prepare_window", + "share_snapshot_sha256": digest, + "record_count": record_count, + "added_rows": stats.added_rows, + "expired_rows": stats.expired_rows, + "touched_pages": stats.touched_pages, + "retained_drop_bytes": byte_delta.retained_drop_bytes, + "appended_items_len": byte_delta.appended_items.len(), + // Diagnostics: which epoch the base was prepared under, + // next to the epoch this request carried. Never acted on. + "base_append_invalidation_epoch": base_epoch, + "append_invalidation_epoch": request_epoch, + "metrics": { + "input_deserialization_seconds": input_deserialization_seconds, + "fold_seconds": fold_seconds, + "output_serialization_seconds": output_serialization_seconds, + }, + }), + )?; + writeln!(out)?; + out.write_all(&byte_delta.appended_items)?; + writeln!(out)?; + out.flush()?; + } + Some(other) => { + respond_error( + stdout, + &format!("unsupported prepare_window mode: {other}"), + false, + )?; + } + None => { + respond_error(stdout, "prepare_window carries no mode", false)?; + } + } + Ok(()) +} + +/// Decline a prepare_window request whose integers (or difficulty total) +/// lie outside the declared widths. Shaped as a prepare_window outcome, not +/// a malformed-request error, because the coordinator retires the daemon on +/// the latter and merely folds in-process on the former. +fn respond_prepare_out_of_range( + stdout: &io::Stdout, + field: &str, + width: &str, + error: &str, +) -> Result<(), Box> { + let mut out = stdout.lock(); + serde_json::to_writer( + &mut out, + &serde_json::json!({ + "ok": false, + "request": "prepare_window", + "out_of_range": true, + "field": field, + "width": width, + "error": error, + }), + )?; + writeln!(out)?; + out.flush()?; + Ok(()) +} + fn respond_error( stdout: &io::Stdout, error: &str, diff --git a/crates/qbit-prism/src/lib.rs b/crates/qbit-prism/src/lib.rs index 94c542f7a..595c9674c 100644 --- a/crates/qbit-prism/src/lib.rs +++ b/crates/qbit-prism/src/lib.rs @@ -81,6 +81,10 @@ mod audit_body_ref; pub use audit_body_ref::*; mod settlement; + +/// Payout-window fold, canonical digest, and incremental advance for the +/// --serve daemon's prepare_window request (#131 window-pipeline slice). +pub mod window; pub use settlement::*; pub const PRISM_WINDOW_MULTIPLIER: u128 = 8; @@ -165,6 +169,17 @@ pub enum PrismError { SettlementModeSelection { reason: String }, } +/// One accepted share as the payout pipeline sees it. +/// +/// The integer field types are the declared integer widths of the Rust +/// window pipeline -- `share_seq` u64, `share_difficulty` u128, +/// `network_difficulty` u128, `template_height` u64, `ntime` u32, +/// `job_issued_at_ms`/`accepted_at_ms` i64 -- and are a deliberately +/// narrower domain than the ledger's `numeric(78,0)`/`bigint` columns and +/// the coordinator's unbounded Python integers. A value outside these widths +/// is declined by the --serve daemon as `out_of_range` and folded +/// in-process instead; see the `window` module docs for the reasoning and +/// the wire classifier. #[derive(Clone, Debug, Deserialize, Serialize, PartialEq, Eq)] pub struct AcceptedShare { pub share_seq: u64, diff --git a/crates/qbit-prism/src/window.rs b/crates/qbit-prism/src/window.rs new file mode 100644 index 000000000..1f65aed11 --- /dev/null +++ b/crates/qbit-prism/src/window.rs @@ -0,0 +1,1603 @@ +//! Payout-window fold, canonical digest, and incremental advance. +//! +//! This is the Rust side of the #131 window-pipeline migration: a +//! byte-identical reimplementation of the coordinator's +//! `IncrementalShareWindow` (`lab/prism/share_ledger.py`) proven against the +//! differential parity oracle in `tests/window_pipeline_parity.py`. Every +//! rule here mirrors a named invariant of the Python fold: eligibility on +//! both timestamps, ascending `share_seq` with duplicate detection, duplicate +//! `share_id` detection, non-positive difficulty rejection, the exact +//! crossing-row retention rule (the final whole share crossing +//! `window_weight` is deliberately retained), 512-record paging, and the +//! digest framing (`[`, non-empty page fragments joined by `,`, `]`). +//! +//! The canonical per-record encoding reproduces CPython's +//! `json.dumps(record, sort_keys=True, separators=(",", ":"))` byte-for-byte +//! (`ensure_ascii` escapes included, non-BMP as surrogate pairs) because the +//! coordinator's cache keys, audit digests, and spool payloads are all +//! derived from those exact bytes. +//! +//! # Declared integer widths +//! +//! Python carries every integer exactly; this fold carries them in fixed +//! machine widths, and that is a deliberate, narrower domain rather than a +//! divergence. The widths, which are the `AcceptedShare` field types: +//! +//! | field | width | +//! |------------------------------------|--------| +//! | `share_seq` | `u64` | +//! | `share_difficulty` | `u128` | +//! | `network_difficulty` | `u128` | +//! | `template_height` | `u64` | +//! | `ntime` | `u32` | +//! | `job_issued_at_ms`, `accepted_at_ms`, and every anchor | `i64` | +//! | `window_weight` | `u128` | +//! | `page_size` | `usize` (`u64` on the daemon's targets) | +//! +//! and the fold's difficulty accumulators are `u128` as well: a window whose +//! retained total would not fit is declined, never saturated. Why `u128` is +//! the real contract even though the ledger column is `numeric(78,0)`: +//! `share_difficulty` is `(pow_limit_target * 10^6) // target`, so at a +//! Bitcoin-mainnet-scale target one share is roughly 2^98 and a 16x window +//! totals roughly 2^102 -- a factor of about 6 * 10^7 short of 2^128, +//! unreachable on any chain this code will see. `numeric(78,0)` is a generic +//! wide-numeric column choice, not a considered statement that difficulties +//! exceed 2^128, and widening this fold to arbitrary precision would put +//! bignum arithmetic on the exact path the migration exists to make faster. +//! +//! A prepare_window request carrying an integer outside these widths is +//! answered by the daemon as `out_of_range` -- distinct from a malformed +//! request -- and the coordinator folds that window in-process, where +//! Python's unbounded integers produce the right answer; the daemon keeps +//! its state and is never retired for it. [`prepare_window_out_of_range`] +//! is the classifier, and the parity oracle (`tests/window_pipeline_parity.py`) +//! mirrors the same table as the `rust-daemon` adapter's declared domain. + +use crate::AcceptedShare; +use sha2::{Digest, Sha256}; +use std::fmt; +use std::rc::Rc; + +/// Mirrors `DEFAULT_INCREMENTAL_SHARE_WINDOW_PAGE_SIZE`. +pub const DEFAULT_WINDOW_PAGE_SIZE: usize = 512; + +/// Why a full-snapshot fold did not produce a window. +/// +/// The first five variants are rejections and mirror the `ValueError`s +/// raised by `IncrementalShareWindow.from_full_snapshot`; each carries a +/// stable machine-readable category ([`WindowFoldError::rejection_category`]) +/// that the daemon puts on the wire beside the human-readable message, so a +/// client never has to match on the message text. `DifficultyOverflow` is +/// not a rejection: the snapshot is valid, but its retained difficulty total +/// leaves the declared `u128` domain (see the module docs), which the daemon +/// reports as `out_of_range`. +#[derive(Debug, PartialEq, Eq)] +pub enum WindowFoldError { + NonPositiveWindowWeight, + NonPositivePageSize, + DuplicateShareSeq, + DuplicateShareId, + NonPositiveDifficulty, + DifficultyOverflow, +} + +impl WindowFoldError { + /// The implementation-neutral rejection category, from the parity + /// oracle's `FULL_SNAPSHOT_REJECTIONS` vocabulary; `None` for the + /// out-of-domain outcome, which is not a rejection. + pub fn rejection_category(&self) -> Option<&'static str> { + Some(match self { + WindowFoldError::NonPositiveWindowWeight => "non_positive_window_weight", + WindowFoldError::NonPositivePageSize => "non_positive_page_size", + WindowFoldError::DuplicateShareSeq => "duplicate_share_seq", + WindowFoldError::DuplicateShareId => "duplicate_share_id", + WindowFoldError::NonPositiveDifficulty => "non_positive_difficulty", + WindowFoldError::DifficultyOverflow => return None, + }) + } +} + +impl fmt::Display for WindowFoldError { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + let message = match self { + WindowFoldError::NonPositiveWindowWeight => "window_weight must be positive", + WindowFoldError::NonPositivePageSize => "page_size must be positive", + WindowFoldError::DuplicateShareSeq => { + "full payout window contains duplicate share_seq" + } + WindowFoldError::DuplicateShareId => "full payout window contains duplicate share_id", + WindowFoldError::NonPositiveDifficulty => { + "full payout window contains non-positive difficulty" + } + WindowFoldError::DifficultyOverflow => DIFFICULTY_OVERFLOW_MESSAGE, + }; + formatter.write_str(message) + } +} + +/// Why an advance did not produce a window. +/// +/// The first six variants are rejections and mirror `IncrementalWindowFallback`, +/// whose message strings the coordinator logs when it falls back to a full +/// rescan; each carries a stable category ([`WindowAdvanceError::rejection_category`]). +/// `DifficultyOverflow` is the out-of-domain outcome: the delta is valid but +/// the accumulated difficulty leaves `u128`. +#[derive(Debug, PartialEq, Eq)] +pub enum WindowAdvanceError { + AnchorMovedBackwards, + NonPositiveDifficulty, + IneligibleAtNewAnchor, + RepeatsPreviouslyEligible, + NotAnAppend, + DeltaOrderNotIncreasing, + DifficultyOverflow, +} + +impl WindowAdvanceError { + /// The implementation-neutral rejection category, from the parity + /// oracle's `ADVANCE_REJECTIONS` vocabulary; `None` for the + /// out-of-domain outcome, which is not a rejection. + pub fn rejection_category(&self) -> Option<&'static str> { + Some(match self { + WindowAdvanceError::AnchorMovedBackwards => "anchor_regression", + WindowAdvanceError::NonPositiveDifficulty => "delta_non_positive_difficulty", + WindowAdvanceError::IneligibleAtNewAnchor => "delta_ineligible_at_anchor", + WindowAdvanceError::RepeatsPreviouslyEligible => "delta_repeats_eligible_share", + WindowAdvanceError::NotAnAppend => "delta_not_append", + WindowAdvanceError::DeltaOrderNotIncreasing => "delta_order_not_increasing", + WindowAdvanceError::DifficultyOverflow => return None, + }) + } +} + +impl fmt::Display for WindowAdvanceError { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + let message = match self { + WindowAdvanceError::AnchorMovedBackwards => "snapshot anchor moved backwards", + WindowAdvanceError::NonPositiveDifficulty => { + "delta contains non-positive share difficulty" + } + WindowAdvanceError::IneligibleAtNewAnchor => { + "delta contains a share ineligible at the new anchor" + } + WindowAdvanceError::RepeatsPreviouslyEligible => { + "delta repeats a share eligible at the previous anchor" + } + WindowAdvanceError::NotAnAppend => "newly eligible share is not an append", + WindowAdvanceError::DeltaOrderNotIncreasing => { + "delta share_seq order is not increasing" + } + WindowAdvanceError::DifficultyOverflow => DIFFICULTY_OVERFLOW_MESSAGE, + }; + formatter.write_str(message) + } +} + +/// Human-readable text for the accumulator leaving the declared domain; the +/// daemon reports it under `out_of_range` with `field` `"total_difficulty"`. +pub const DIFFICULTY_OVERFLOW_MESSAGE: &str = "window difficulty total exceeds u128"; + +/// The fixed widths this pipeline declares for its integer inputs (see the +/// module docs). Wire literals are checked against these exactly, digit by +/// digit, so a value one above a width is classified as out of range and a +/// value at the width is admitted. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum DeclaredWidth { + U32, + U64, + U128, + I64, +} + +impl DeclaredWidth { + pub fn name(self) -> &'static str { + match self { + DeclaredWidth::U32 => "u32", + DeclaredWidth::U64 => "u64", + DeclaredWidth::U128 => "u128", + DeclaredWidth::I64 => "i64", + } + } + + /// Whether an integer literal (optional leading `-`, then digits) is + /// representable at this width. + pub fn admits(self, literal: &str) -> bool { + match self { + DeclaredWidth::U32 => literal.parse::().is_ok(), + DeclaredWidth::U64 => literal.parse::().is_ok(), + DeclaredWidth::U128 => literal.parse::().is_ok(), + DeclaredWidth::I64 => literal.parse::().is_ok(), + } + } +} + +/// Declared width of every integer an `AcceptedShare` carries, by its wire +/// (serde) field name; the same table, in the same order, as the parity +/// oracle's record fields. +pub const ACCEPTED_SHARE_WIDTHS: [(&str, DeclaredWidth); 7] = [ + ("share_seq", DeclaredWidth::U64), + ("share_difficulty", DeclaredWidth::U128), + ("network_difficulty", DeclaredWidth::U128), + ("template_height", DeclaredWidth::U64), + ("job_issued_at_ms", DeclaredWidth::I64), + ("accepted_at_ms", DeclaredWidth::I64), + ("ntime", DeclaredWidth::U32), +]; + +/// Declared width of every top-level integer a prepare_window request +/// carries. `page_size` is `usize` in the fold; the daemon's targets are +/// 64-bit, so the wire contract states `u64`. +pub const PREPARE_WINDOW_REQUEST_WIDTHS: [(&str, DeclaredWidth); 4] = [ + ("window_weight", DeclaredWidth::U128), + ("page_size", DeclaredWidth::U64), + ("anchor_job_issued_at_ms", DeclaredWidth::I64), + ("append_invalidation_epoch", DeclaredWidth::U64), +]; + +/// One prepare_window integer outside its declared width: the JSON path of +/// the field, the literal exactly as it arrived, and the width it missed. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct OutOfRangeField { + pub field: String, + pub literal: String, + pub width: DeclaredWidth, +} + +impl fmt::Display for OutOfRangeField { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + write!( + formatter, + "prepare_window {} {} is outside the declared {}", + self.field, + self.literal, + self.width.name() + ) + } +} + +fn is_integer_literal(literal: &str) -> bool { + let digits = literal.strip_prefix('-').unwrap_or(literal); + !digits.is_empty() && digits.bytes().all(|byte| byte.is_ascii_digit()) +} + +fn out_of_range_field( + value: Option<&serde_json::Value>, + field: impl FnOnce() -> String, + width: DeclaredWidth, +) -> Option { + let serde_json::Value::Number(number) = value? else { + return None; + }; + // `arbitrary_precision` keeps the literal's digits verbatim, so the + // check is exact at every magnitude; a non-integer literal is not a + // width question and is left for the strict parse to call malformed. + let literal = number.as_str(); + if is_integer_literal(literal) && !width.admits(literal) { + return Some(OutOfRangeField { + field: field(), + literal: literal.to_string(), + width, + }); + } + None +} + +/// Locate the first integer in a (syntactically valid) prepare_window +/// request document that lies outside its declared width, walking the +/// top-level fields in [`PREPARE_WINDOW_REQUEST_WIDTHS`] order and then each +/// record's fields in [`ACCEPTED_SHARE_WIDTHS`] order. `None` means every +/// integer present is representable -- so a strict parse failure on this +/// document is a genuinely malformed request, not a width problem. Only +/// inspects the request; it never folds anything. +pub fn prepare_window_out_of_range(request: &serde_json::Value) -> Option { + for (field, width) in PREPARE_WINDOW_REQUEST_WIDTHS { + if let Some(found) = out_of_range_field(request.get(field), || field.to_string(), width) { + return Some(found); + } + } + let records = request + .get("records") + .and_then(serde_json::Value::as_array)?; + for (index, record) in records.iter().enumerate() { + for (field, width) in ACCEPTED_SHARE_WIDTHS { + if let Some(found) = out_of_range_field( + record.get(field), + || format!("records[{index}].{field}"), + width, + ) { + return Some(found); + } + } + } + None +} + +/// Mirrors `IncrementalWindowAdvanceStats`, `touched_pages` definition +/// included: pre-existing retained boundary pages whose records were +/// inspected or rewritten; newly allocated append pages and pages expired +/// wholesale do not count. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct WindowAdvanceStats { + pub added_rows: usize, + pub expired_rows: usize, + pub touched_pages: usize, +} + +/// Byte-level surgery for one advance, letting the coordinator maintain its +/// opaque mirror of the canonical items stream without re-encoding anything: +/// drop `retained_drop_bytes` from the front of the previous stream, then +/// append `appended_items` (leading `,` included exactly when the retained +/// part is non-empty). +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct WindowAdvanceByteDelta { + pub retained_drop_bytes: usize, + pub appended_items: Vec, +} + +fn push_python_json_escaped(out: &mut Vec, value: &str) { + out.push(b'"'); + for character in value.chars() { + let code_point = character as u32; + match character { + '"' => out.extend_from_slice(b"\\\""), + '\\' => out.extend_from_slice(b"\\\\"), + '\u{08}' => out.extend_from_slice(b"\\b"), + '\t' => out.extend_from_slice(b"\\t"), + '\n' => out.extend_from_slice(b"\\n"), + '\u{0c}' => out.extend_from_slice(b"\\f"), + '\r' => out.extend_from_slice(b"\\r"), + _ if (0x20..0x7f).contains(&code_point) => out.push(code_point as u8), + _ if code_point < 0x10000 => { + push_u16_escape(out, code_point as u16); + } + _ => { + // Non-BMP code points escape as a UTF-16 surrogate pair, + // exactly like CPython's ensure_ascii encoder. + let mut units = [0u16; 2]; + for unit in character.encode_utf16(&mut units) { + push_u16_escape(out, *unit); + } + } + } + } + out.push(b'"'); +} + +fn push_u16_escape(out: &mut Vec, unit: u16) { + const HEX: &[u8; 16] = b"0123456789abcdef"; + out.extend_from_slice(b"\\u"); + out.push(HEX[usize::from((unit >> 12) & 0xf)]); + out.push(HEX[usize::from((unit >> 8) & 0xf)]); + out.push(HEX[usize::from((unit >> 4) & 0xf)]); + out.push(HEX[usize::from(unit & 0xf)]); +} + +/// One record's canonical JSON encoding: byte-identical to +/// `json.dumps(record.to_prism_json(), sort_keys=True, separators=(",", ":"))`. +/// Keys are emitted in sorted order with `credit_policy` omitted when absent, +/// matching `AcceptedShareRecord.to_prism_json`. +pub fn canonical_share_fragment(share: &AcceptedShare) -> Vec { + let mut out = Vec::with_capacity(256 + share.share_id.len() + share.miner_id.len()); + out.extend_from_slice(b"{\"accepted_at_ms\":"); + out.extend_from_slice(share.accepted_at_ms.to_string().as_bytes()); + if let Some(credit_policy) = &share.credit_policy { + out.extend_from_slice(b",\"credit_policy\":"); + push_python_json_escaped(&mut out, credit_policy); + } + out.extend_from_slice(b",\"job_id\":"); + push_python_json_escaped(&mut out, &share.job_id); + out.extend_from_slice(b",\"job_issued_at_ms\":"); + out.extend_from_slice(share.job_issued_at_ms.to_string().as_bytes()); + out.extend_from_slice(b",\"miner_id\":"); + push_python_json_escaped(&mut out, &share.miner_id); + out.extend_from_slice(b",\"network_difficulty\":"); + out.extend_from_slice(share.network_difficulty.to_string().as_bytes()); + out.extend_from_slice(b",\"ntime\":"); + out.extend_from_slice(share.ntime.to_string().as_bytes()); + out.extend_from_slice(b",\"order_key\":"); + push_python_json_escaped(&mut out, &share.order_key); + out.extend_from_slice(b",\"p2mr_program_hex\":"); + push_python_json_escaped(&mut out, &share.p2mr_program_hex); + out.extend_from_slice(b",\"share_difficulty\":"); + out.extend_from_slice(share.share_difficulty.to_string().as_bytes()); + out.extend_from_slice(b",\"share_id\":"); + push_python_json_escaped(&mut out, &share.share_id); + out.extend_from_slice(b",\"share_seq\":"); + out.extend_from_slice(share.share_seq.to_string().as_bytes()); + out.extend_from_slice(b",\"template_height\":"); + out.extend_from_slice(share.template_height.to_string().as_bytes()); + out.push(b'}'); + out +} + +/// Mirrors `_IncrementalShareWindowPage`: records plus their pre-encoded +/// canonical fragments and the page's total difficulty. +#[derive(Debug)] +pub struct WindowPage { + pub records: Vec, + pub fragments: Vec>, + pub total_difficulty: u128, +} + +impl WindowPage { + // Page totals sum a subset of a window total that `from_full_snapshot` + // and `advance` have already checked against u128, so the saturating + // fold below never actually saturates; it only keeps the constructors + // infallible. + fn from_records(records: Vec) -> Self { + let fragments: Vec> = records.iter().map(canonical_share_fragment).collect(); + let total_difficulty = records + .iter() + .map(|record| record.share_difficulty) + .fold(0u128, u128::saturating_add); + WindowPage { + records, + fragments, + total_difficulty, + } + } + + fn from_retained(records: Vec, fragments: Vec>) -> Self { + let total_difficulty = records + .iter() + .map(|record| record.share_difficulty) + .fold(0u128, u128::saturating_add); + WindowPage { + records, + fragments, + total_difficulty, + } + } +} + +/// Mirrors `IncrementalShareWindow`: the immutable paged cache of the exact +/// whole-share payout-window superset. Pages are reference-counted so an +/// advanced window shares its retained interior with the previous generation +/// still serving in-flight builds. +#[derive(Debug)] +pub struct PayoutWindow { + pub anchor_job_issued_at_ms: i64, + pub window_weight: u128, + pub page_size: usize, + pub pages: Vec>, + pub total_difficulty: u128, +} + +impl PayoutWindow { + pub fn from_full_snapshot( + records: Vec, + anchor_job_issued_at_ms: i64, + window_weight: u128, + page_size: usize, + ) -> Result { + if window_weight == 0 { + return Err(WindowFoldError::NonPositiveWindowWeight); + } + if page_size == 0 { + return Err(WindowFoldError::NonPositivePageSize); + } + + // Filter in place: the snapshot's own buffer is the only record + // storage this fold ever holds besides the pages it returns. + let mut eligible = records; + eligible.retain(|record| { + record.job_issued_at_ms <= anchor_job_issued_at_ms + && record.accepted_at_ms <= anchor_job_issued_at_ms + }); + eligible.sort_by_key(|record| record.share_seq); + + let mut prior_seq: Option = None; + let mut share_ids: std::collections::HashSet<&str> = + std::collections::HashSet::with_capacity(eligible.len()); + for record in &eligible { + if let Some(prior) = prior_seq { + if record.share_seq <= prior { + return Err(WindowFoldError::DuplicateShareSeq); + } + } + if !share_ids.insert(record.share_id.as_str()) { + return Err(WindowFoldError::DuplicateShareId); + } + if record.share_difficulty == 0 { + return Err(WindowFoldError::NonPositiveDifficulty); + } + prior_seq = Some(record.share_seq); + } + drop(share_ids); + + // Walk newest-to-oldest accumulating weight; the final whole share + // crossing window_weight is deliberately retained. The accumulator + // is checked, not saturating: a total that leaves u128 is outside + // the declared domain and must be declined, because a saturated + // total would silently change later expiry decisions. + let mut start = eligible.len(); + let mut retained_weight: u128 = 0; + for index in (0..eligible.len()).rev() { + if retained_weight >= window_weight { + break; + } + retained_weight = retained_weight + .checked_add(eligible[index].share_difficulty) + .ok_or(WindowFoldError::DifficultyOverflow)?; + start = index; + } + // Page by moving every retained record exactly once, out of the + // snapshot's buffer and into a vector allocated for precisely the + // records that page holds, so aggregate page capacity is the + // retained count and the work is linear in it (#240). The previous + // loop stored the head of a repeatedly `split_off` vector as each + // page: `Vec::split_off` keeps the original allocation, so page k + // retained capacity for every record still unpaged when it was cut + // (N, N-P, N-2P, ...) and copied the tail again on every iteration, + // O(N^2/P) in both retained memory and copying. A prior synthetic + // reproduction of the #236 window measured 9,613 MiB of resident + // builder memory for one 210k-share preparation + // (tests/perf/window_pipeline_gil_scaling.md section 9, cited by #240). + let retained_count = eligible.len() - start; + let mut pages: Vec> = Vec::with_capacity(retained_count.div_ceil(page_size)); + let mut retained = eligible.into_iter().skip(start); + loop { + let page_len = retained.len().min(page_size); + if page_len == 0 { + break; + } + let mut page_records: Vec = Vec::with_capacity(page_len); + page_records.extend(retained.by_ref().take(page_len)); + debug_assert_eq!(page_records.len(), page_len); + pages.push(Rc::new(WindowPage::from_records(page_records))); + } + debug_assert_eq!(pages.len(), retained_count.div_ceil(page_size)); + Ok(PayoutWindow { + anchor_job_issued_at_ms, + window_weight, + page_size, + pages, + total_difficulty: retained_weight, + }) + } + + pub fn record_count(&self) -> usize { + self.pages.iter().map(|page| page.records.len()).sum() + } + + /// Every retained record's canonical fragment lengths, oldest first, + /// flattened across pages. + fn fragment_lengths(&self) -> Vec { + self.pages + .iter() + .flat_map(|page| page.fragments.iter().map(Vec::len)) + .collect() + } + + /// The canonical items stream: every fragment joined with `,`, no + /// enclosing brackets. Identical to the concatenation the digest hashes, + /// because the page framing skips empty pages and joins with the same + /// separator, making page layout invisible on the wire. + pub fn canonical_items_bytes(&self) -> Vec { + let total: usize = self + .pages + .iter() + .flat_map(|page| page.fragments.iter().map(Vec::len)) + .sum(); + let count = self.record_count(); + let mut out = Vec::with_capacity(total + count.saturating_sub(1)); + let mut needs_separator = false; + for page in &self.pages { + for fragment in &page.fragments { + if needs_separator { + out.push(b','); + } + out.extend_from_slice(fragment); + needs_separator = true; + } + } + out + } + + /// Mirrors `IncrementalShareJsonSequence.canonical_json_sha256`: `[`, + /// non-empty page fragments joined with `,`, `]`. + pub fn canonical_digest_hex(&self) -> String { + let mut digest = Sha256::new(); + digest.update(b"["); + let mut needs_separator = false; + for page in &self.pages { + if page.fragments.is_empty() { + continue; + } + let mut first_in_page = true; + if needs_separator { + digest.update(b","); + } + for fragment in &page.fragments { + if !first_in_page { + digest.update(b","); + } + digest.update(fragment); + first_in_page = false; + } + needs_separator = true; + } + digest.update(b"]"); + let bytes = digest.finalize(); + let mut out = String::with_capacity(64); + for byte in bytes { + use fmt::Write; + let _ = write!(out, "{byte:02x}"); + } + out + } + + /// Flattened clone of the retained records for one audit build. Prepared + /// windows are never loaned out: builds get their own copy so the window + /// state (and its advance lineage) survives every build outcome. + pub fn shares_for_build(&self) -> Vec { + let mut out = Vec::with_capacity(self.record_count()); + for page in &self.pages { + out.extend(page.records.iter().cloned()); + } + out + } + + /// Mirrors `IncrementalShareWindow.advance` exactly, including the + /// validation order, the partial-page expiry at the old edge, and the + /// `touched_pages` accounting. Also returns the byte-level surgery the + /// coordinator applies to its opaque canonical-items mirror. + pub fn advance( + &self, + delta: Vec, + anchor_job_issued_at_ms: i64, + ) -> Result<(PayoutWindow, WindowAdvanceStats, WindowAdvanceByteDelta), WindowAdvanceError> + { + if anchor_job_issued_at_ms < self.anchor_job_issued_at_ms { + return Err(WindowAdvanceError::AnchorMovedBackwards); + } + let prior_seq: Option = self + .pages + .last() + .and_then(|page| page.records.last()) + .map(|record| record.share_seq); + let mut prior_delta_seq: Option = None; + for record in &delta { + if record.share_difficulty == 0 { + return Err(WindowAdvanceError::NonPositiveDifficulty); + } + if record.job_issued_at_ms > anchor_job_issued_at_ms + || record.accepted_at_ms > anchor_job_issued_at_ms + { + return Err(WindowAdvanceError::IneligibleAtNewAnchor); + } + if record.job_issued_at_ms <= self.anchor_job_issued_at_ms + && record.accepted_at_ms <= self.anchor_job_issued_at_ms + { + return Err(WindowAdvanceError::RepeatsPreviouslyEligible); + } + if let Some(prior) = prior_seq { + if record.share_seq <= prior { + return Err(WindowAdvanceError::NotAnAppend); + } + } + if let Some(prior) = prior_delta_seq { + if record.share_seq <= prior { + return Err(WindowAdvanceError::DeltaOrderNotIncreasing); + } + } + prior_delta_seq = Some(record.share_seq); + } + + // Both totals are checked before any page is built: a delta or a + // combined total that leaves u128 is outside the declared domain, + // and every page total below is a subset of the combined total. + let delta_difficulty = delta + .iter() + .map(|record| record.share_difficulty) + .try_fold(0u128, u128::checked_add) + .ok_or(WindowAdvanceError::DifficultyOverflow)?; + let mut total_difficulty = self + .total_difficulty + .checked_add(delta_difficulty) + .ok_or(WindowAdvanceError::DifficultyOverflow)?; + + let old_fragment_lengths = self.fragment_lengths(); + let old_count = old_fragment_lengths.len(); + let added_rows = delta.len(); + let delta_fragments: Vec> = delta.iter().map(canonical_share_fragment).collect(); + + // Copy page references, never their retained contents. `touched` is + // pre-existing retained boundary pages only, exactly as in Python. + let mut pages: Vec> = self.pages.clone(); + let mut touched_existing_pages: std::collections::BTreeSet = + std::collections::BTreeSet::new(); + // The delta is paged the same way the full fold pages its retained + // records (#240): each record and fragment moves exactly once into a + // vector sized for the page it lands in. The merged boundary page is + // allocated for its final length as well, so no appended page + // retains capacity beyond `page_size`. + let mut delta_records = delta.into_iter(); + let mut delta_encoded = delta_fragments.into_iter(); + if delta_records.len() > 0 { + if let Some(last) = pages.last() { + if last.records.len() < self.page_size { + let available = self.page_size - last.records.len(); + let take = available.min(delta_records.len()); + if take > 0 { + let mut merged_records: Vec = + Vec::with_capacity(last.records.len() + take); + merged_records.extend(last.records.iter().cloned()); + merged_records.extend(delta_records.by_ref().take(take)); + let mut merged_fragments: Vec> = + Vec::with_capacity(last.fragments.len() + take); + merged_fragments.extend(last.fragments.iter().cloned()); + merged_fragments.extend(delta_encoded.by_ref().take(take)); + let last_index = pages.len() - 1; + pages[last_index] = + Rc::new(WindowPage::from_retained(merged_records, merged_fragments)); + touched_existing_pages.insert(self.pages.len() - 1); + } + } + } + } + loop { + let take = self.page_size.min(delta_records.len()); + if take == 0 { + break; + } + let mut page_records: Vec = Vec::with_capacity(take); + page_records.extend(delta_records.by_ref().take(take)); + let mut page_fragments: Vec> = Vec::with_capacity(take); + page_fragments.extend(delta_encoded.by_ref().take(take)); + pages.push(Rc::new(WindowPage::from_retained( + page_records, + page_fragments, + ))); + } + debug_assert_eq!(delta_encoded.len(), 0); + + let mut expired_rows: usize = 0; + let mut first_retained_page: usize = 0; + while first_retained_page < pages.len() + && total_difficulty - pages[first_retained_page].total_difficulty + >= self.window_weight + { + let page = &pages[first_retained_page]; + total_difficulty -= page.total_difficulty; + expired_rows += page.records.len(); + first_retained_page += 1; + } + let original_page_offset = first_retained_page; + if first_retained_page > 0 { + pages.drain(..first_retained_page); + } + + if let Some(first_page) = pages.first().cloned() { + let mut partial_expired: usize = 0; + while partial_expired < first_page.records.len() + && total_difficulty - first_page.records[partial_expired].share_difficulty + >= self.window_weight + { + total_difficulty -= first_page.records[partial_expired].share_difficulty; + partial_expired += 1; + } + if partial_expired > 0 { + pages[0] = Rc::new(WindowPage::from_retained( + first_page.records[partial_expired..].to_vec(), + first_page.fragments[partial_expired..].to_vec(), + )); + expired_rows += partial_expired; + if original_page_offset < self.pages.len() { + touched_existing_pages.insert(original_page_offset); + } + } + } + + let advanced = PayoutWindow { + anchor_job_issued_at_ms, + window_weight: self.window_weight, + page_size: self.page_size, + pages, + total_difficulty, + }; + let stats = WindowAdvanceStats { + added_rows, + expired_rows, + touched_pages: touched_existing_pages.len(), + }; + let byte_delta = advance_byte_delta( + &old_fragment_lengths, + &advanced, + old_count, + expired_rows, + added_rows, + ); + Ok((advanced, stats, byte_delta)) + } +} + +/// Byte surgery from the old canonical items stream to the advanced one. +/// Expiry always drops a prefix of the combined old-then-appended record +/// order, so the mirror update is a prefix drop plus a suffix append. +fn advance_byte_delta( + old_fragment_lengths: &[usize], + advanced: &PayoutWindow, + old_count: usize, + expired_rows: usize, + added_rows: usize, +) -> WindowAdvanceByteDelta { + let dropped_old = expired_rows.min(old_count); + let retained_old = old_count - dropped_old; + let retained_drop_bytes = if dropped_old == 0 { + 0 + } else { + let dropped_bytes: usize = old_fragment_lengths[..dropped_old].iter().sum(); + if retained_old > 0 { + // Each dropped fragment takes the comma that followed it. + dropped_bytes + dropped_old + } else { + // The whole old stream goes, separators included. + dropped_bytes + old_count.saturating_sub(1) + } + }; + // Fragments past the retained old records are the appended survivors: + // the delta records that were folded in and not immediately expired. + let surviving_new = advanced.record_count() - retained_old; + debug_assert!(surviving_new <= added_rows); + let mut appended_items = Vec::new(); + if surviving_new > 0 { + let mut skip = retained_old; + let mut first = true; + for page in &advanced.pages { + for fragment in &page.fragments { + if skip > 0 { + skip -= 1; + continue; + } + if !first || retained_old > 0 { + appended_items.push(b','); + } + appended_items.extend_from_slice(fragment); + first = false; + } + } + } + WindowAdvanceByteDelta { + retained_drop_bytes, + appended_items, + } +} + +#[cfg(test)] +mod tests { + use super::*; + + fn share(seq: u64, difficulty: u128, job_ms: i64, accepted_ms: i64) -> AcceptedShare { + AcceptedShare { + share_seq: seq, + share_id: format!("share-{seq}"), + miner_id: format!("miner-{}", seq % 3), + order_key: format!("{:02}:miner", seq % 3), + p2mr_program_hex: format!("aa{seq:02x}"), + share_difficulty: difficulty, + network_difficulty: 1000, + template_height: 100, + job_id: format!("job-{seq}"), + job_issued_at_ms: job_ms, + accepted_at_ms: accepted_ms, + ntime: 1_700_000_000, + credit_policy: if seq % 4 == 0 { + Some("stale-grace".to_string()) + } else { + None + }, + } + } + + #[test] + fn fragment_escapes_match_python_json_dumps() { + let mut record = share(1, 5, 10, 11); + record.share_id = "min\u{00e9}r \"q\" \\b \u{0001}\t\u{1f680}\u{007f}".to_string(); + record.credit_policy = None; + let fragment = canonical_share_fragment(&record); + let text = String::from_utf8(fragment).expect("fragment must be ASCII"); + assert!(text.contains( + "\"share_id\":\"min\\u00e9r \\\"q\\\" \\\\b \\u0001\\t\\ud83d\\ude80\\u007f\"" + )); + // Sorted key order, credit_policy omitted when absent. + assert!(text.starts_with("{\"accepted_at_ms\":11,\"job_id\":")); + assert!(text.ends_with(",\"template_height\":100}")); + } + + #[test] + fn crossing_row_is_retained() { + // Difficulties (2, 3, 8) at weight 10: the final whole share crossing + // window_weight (seq 2) is retained, total 11 > 10. + let records = vec![ + share(1, 2, 6, 7), + share(2, 3, 8, 9), + share(3, 8, 10, 10), + ]; + let window = PayoutWindow::from_full_snapshot(records, 10, 10, 512).expect("fold"); + assert_eq!(window.record_count(), 2); + assert_eq!(window.total_difficulty, 11); + let exact = PayoutWindow::from_full_snapshot( + vec![share(1, 2, 6, 7), share(2, 3, 8, 9), share(3, 7, 10, 10)], + 10, + 10, + 512, + ) + .expect("fold"); + assert_eq!(exact.record_count(), 2); + assert_eq!(exact.total_difficulty, 10); + } + + #[test] + fn eligibility_uses_both_timestamps() { + let records = vec![ + share(1, 4, 9, 9), + share(2, 5, 11, 9), + share(3, 6, 9, 11), + share(4, 8, 10, 10), + ]; + let window = PayoutWindow::from_full_snapshot(records, 10, 100, 512).expect("fold"); + assert_eq!(window.record_count(), 2); + } + + #[test] + fn duplicate_and_zero_difficulty_are_rejected() { + let duplicate_seq = vec![share(1, 2, 5, 5), share(1, 3, 6, 6)]; + assert_eq!( + PayoutWindow::from_full_snapshot(duplicate_seq, 10, 10, 512).unwrap_err(), + WindowFoldError::DuplicateShareSeq + ); + let mut duplicate_id = vec![share(1, 2, 5, 5), share(2, 3, 6, 6)]; + duplicate_id[1].share_id = "share-1".to_string(); + assert_eq!( + PayoutWindow::from_full_snapshot(duplicate_id, 10, 10, 512).unwrap_err(), + WindowFoldError::DuplicateShareId + ); + let zero = vec![share(1, 0, 5, 5)]; + assert_eq!( + PayoutWindow::from_full_snapshot(zero, 10, 10, 512).unwrap_err(), + WindowFoldError::NonPositiveDifficulty + ); + } + + #[test] + fn advance_matches_full_rebuild_and_reports_byte_surgery() { + let snapshot: Vec = (1..=7) + .map(|seq| share(seq, [5, 1, 1, 2, 1, 3, 1][usize::try_from(seq).unwrap() - 1], 100 - i64::try_from(seq).unwrap(), 100 - i64::try_from(seq).unwrap())) + .collect(); + let window = + PayoutWindow::from_full_snapshot(snapshot.clone(), 100, 9, 3).expect("fold"); + let old_items = window.canonical_items_bytes(); + let delta = vec![share(9, 2, 101, 99), share(10, 4, 90, 101)]; + let (advanced, stats, byte_delta) = + window.advance(delta.clone(), 101).expect("advance"); + + let mut union = snapshot; + union.extend(delta); + let rebuilt = PayoutWindow::from_full_snapshot(union, 101, 9, 3).expect("rebuild"); + assert_eq!( + advanced.canonical_digest_hex(), + rebuilt.canonical_digest_hex() + ); + assert_eq!(stats.added_rows, 2); + assert!(stats.expired_rows > 0); + + // The reported surgery must transform the old items stream into the + // advanced one. + let mut mirrored = old_items[byte_delta.retained_drop_bytes..].to_vec(); + mirrored.extend_from_slice(&byte_delta.appended_items); + assert_eq!(mirrored, advanced.canonical_items_bytes()); + } + + #[test] + fn advance_rejections_mirror_python() { + let window = PayoutWindow::from_full_snapshot( + vec![share(1, 2, 5, 5), share(2, 3, 6, 6)], + 10, + 100, + 512, + ) + .expect("fold"); + assert_eq!( + window.advance(vec![], 9).unwrap_err(), + WindowAdvanceError::AnchorMovedBackwards + ); + assert_eq!( + window.advance(vec![share(3, 0, 11, 11)], 11).unwrap_err(), + WindowAdvanceError::NonPositiveDifficulty + ); + assert_eq!( + window.advance(vec![share(3, 1, 12, 11)], 11).unwrap_err(), + WindowAdvanceError::IneligibleAtNewAnchor + ); + assert_eq!( + window.advance(vec![share(3, 1, 9, 9)], 11).unwrap_err(), + WindowAdvanceError::RepeatsPreviouslyEligible + ); + assert_eq!( + window.advance(vec![share(2, 1, 11, 11)], 11).unwrap_err(), + WindowAdvanceError::NotAnAppend + ); + assert_eq!( + window + .advance(vec![share(4, 1, 11, 11), share(3, 1, 11, 10)], 11) + .unwrap_err(), + WindowAdvanceError::DeltaOrderNotIncreasing + ); + } + + #[test] + fn empty_delta_advances_anchor_only() { + let window = PayoutWindow::from_full_snapshot( + vec![share(1, 2, 5, 5), share(2, 3, 6, 6)], + 10, + 100, + 512, + ) + .expect("fold"); + let digest = window.canonical_digest_hex(); + let (advanced, stats, byte_delta) = window.advance(vec![], 20).expect("advance"); + assert_eq!(advanced.anchor_job_issued_at_ms, 20); + assert_eq!(advanced.canonical_digest_hex(), digest); + assert_eq!( + stats, + WindowAdvanceStats { + added_rows: 0, + expired_rows: 0, + touched_pages: 0 + } + ); + assert_eq!(byte_delta.retained_drop_bytes, 0); + assert!(byte_delta.appended_items.is_empty()); + } + + #[test] + fn rejection_categories_mirror_the_parity_vocabulary() { + // The eleven categories the parity oracle freezes, in its order: + // five full-snapshot conditions, then six advance conditions. Both + // sides must agree on every one, and the out-of-domain outcome is + // deliberately not a category. + let fold = [ + WindowFoldError::DuplicateShareSeq, + WindowFoldError::DuplicateShareId, + WindowFoldError::NonPositiveDifficulty, + WindowFoldError::NonPositiveWindowWeight, + WindowFoldError::NonPositivePageSize, + ]; + let advance = [ + WindowAdvanceError::AnchorMovedBackwards, + WindowAdvanceError::NonPositiveDifficulty, + WindowAdvanceError::IneligibleAtNewAnchor, + WindowAdvanceError::RepeatsPreviouslyEligible, + WindowAdvanceError::NotAnAppend, + WindowAdvanceError::DeltaOrderNotIncreasing, + ]; + let categories: Vec<&str> = fold + .iter() + .map(|error| error.rejection_category().expect("rejection")) + .chain( + advance + .iter() + .map(|error| error.rejection_category().expect("rejection")), + ) + .collect(); + assert_eq!( + categories, + [ + "duplicate_share_seq", + "duplicate_share_id", + "non_positive_difficulty", + "non_positive_window_weight", + "non_positive_page_size", + "anchor_regression", + "delta_non_positive_difficulty", + "delta_ineligible_at_anchor", + "delta_repeats_eligible_share", + "delta_not_append", + "delta_order_not_increasing", + ] + ); + assert_eq!( + WindowFoldError::DifficultyOverflow.rejection_category(), + None + ); + assert_eq!( + WindowAdvanceError::DifficultyOverflow.rejection_category(), + None + ); + } + + #[test] + fn declared_widths_admit_the_edge_and_refuse_one_past_it() { + assert!(DeclaredWidth::U32.admits("4294967295")); + assert!(!DeclaredWidth::U32.admits("4294967296")); + assert!(DeclaredWidth::U64.admits("18446744073709551615")); + assert!(!DeclaredWidth::U64.admits("18446744073709551616")); + assert!(DeclaredWidth::U128.admits("340282366920938463463374607431768211455")); + assert!(!DeclaredWidth::U128.admits("340282366920938463463374607431768211456")); + assert!(DeclaredWidth::I64.admits("-9223372036854775808")); + assert!(!DeclaredWidth::I64.admits("9223372036854775808")); + assert!(!DeclaredWidth::U128.admits("-1")); + } + + #[test] + fn out_of_range_probe_names_the_first_offending_field_exactly() { + let request: serde_json::Value = serde_json::from_str( + r#"{"request":"prepare_window","mode":"full","window_weight":340282366920938463463374607431768211455, + "anchor_job_issued_at_ms":1755000000000, + "records":[ + {"share_seq":18446744073709551615,"share_difficulty":170141183460469231731687303715884105731,"ntime":4294967295}, + {"share_seq":1,"share_difficulty":340282366920938463463374607431768211457,"ntime":4294967296} + ]}"#, + ) + .expect("valid json"); + let found = prepare_window_out_of_range(&request).expect("out of range"); + assert_eq!(found.field, "records[1].share_difficulty"); + assert_eq!(found.literal, "340282366920938463463374607431768211457"); + assert_eq!(found.width, DeclaredWidth::U128); + assert_eq!( + found.to_string(), + "prepare_window records[1].share_difficulty 340282366920938463463374607431768211457 is outside the declared u128" + ); + + // Top-level fields are checked before records, and a window_weight one + // above u128 is caught there. + let request: serde_json::Value = serde_json::from_str( + r#"{"request":"prepare_window","window_weight":340282366920938463463374607431768211456,"records":[{"ntime":4294967296}]}"#, + ) + .expect("valid json"); + let found = prepare_window_out_of_range(&request).expect("out of range"); + assert_eq!(found.field, "window_weight"); + + // Everything at its width, a negative i64, and a non-integer literal + // (the strict parse's business, not a width question) are all None. + let request: serde_json::Value = serde_json::from_str( + r#"{"request":"prepare_window","window_weight":340282366920938463463374607431768211455, + "anchor_job_issued_at_ms":-9223372036854775808,"page_size":18446744073709551615, + "records":[{"share_seq":18446744073709551615,"share_difficulty":340282366920938463463374607431768211455, + "network_difficulty":340282366920938463463374607431768211455,"template_height":18446744073709551615, + "job_issued_at_ms":9223372036854775807,"accepted_at_ms":-1,"ntime":4294967295,"share_id":"x"}, + {"share_difficulty":1.5}]}"#, + ) + .expect("valid json"); + assert_eq!(prepare_window_out_of_range(&request), None); + } + + #[test] + fn difficulty_totals_that_leave_u128_are_declined_not_saturated() { + // Every input fits u128, but the retained total (the crossing row on + // top of a near-maximal weight) does not. + let records = vec![ + share(1, u128::MAX / 2 + 1, 5, 5), + share(2, u128::MAX / 2 + 1, 6, 6), + ]; + assert_eq!( + PayoutWindow::from_full_snapshot(records, 10, u128::MAX, 512).unwrap_err(), + WindowFoldError::DifficultyOverflow + ); + // The advance side: the held total plus the delta's leaves u128. + let window = PayoutWindow::from_full_snapshot( + vec![share(1, u128::MAX / 2 + 1, 5, 5)], + 10, + u128::MAX, + 512, + ) + .expect("fold"); + assert_eq!( + window + .advance(vec![share(2, u128::MAX / 2 + 1, 11, 11)], 11) + .unwrap_err(), + WindowAdvanceError::DifficultyOverflow + ); + assert_eq!( + window + .advance( + vec![ + share(2, u128::MAX / 2 + 1, 11, 11), + share(3, u128::MAX / 2 + 1, 11, 11) + ], + 11 + ) + .unwrap_err(), + WindowAdvanceError::DifficultyOverflow + ); + } + + #[test] + fn expiry_that_consumes_the_whole_old_window_still_reports_exact_surgery() { + let window = PayoutWindow::from_full_snapshot( + vec![share(1, 2, 5, 5), share(2, 3, 6, 6)], + 10, + 6, + 512, + ) + .expect("fold"); + let old_items = window.canonical_items_bytes(); + // A heavy delta expires every old record. + let (advanced, stats, byte_delta) = window + .advance(vec![share(3, 6, 11, 11), share(4, 1, 11, 11)], 11) + .expect("advance"); + assert_eq!(stats.expired_rows, 2); + let mut mirrored = old_items[byte_delta.retained_drop_bytes..].to_vec(); + mirrored.extend_from_slice(&byte_delta.appended_items); + assert_eq!(mirrored, advanced.canonical_items_bytes()); + } + // ----- #240: linear paging allocation ------------------------------- + + /// `count` eligible records of difficulty `difficulty` each, ascending + /// share_seq from 1, all eligible at anchor 10. + fn uniform_shares(count: usize, difficulty: u128) -> Vec { + (1..=count) + .map(|seq| share(u64::try_from(seq).unwrap(), difficulty, 10, 10)) + .collect() + } + + /// Aggregate backing capacity of every page's record and fragment + /// vectors: the number the #240 defect made quadratic. + fn aggregate_page_capacity(window: &PayoutWindow) -> (usize, usize) { + window + .pages + .iter() + .fold((0, 0), |(records, fragments), page| { + ( + records + page.records.capacity(), + fragments + page.fragments.capacity(), + ) + }) + } + + /// The structural bound every paged window must satisfy: no page vector + /// keeps capacity beyond `page_size`, so aggregate capacity is at most + /// one page's worth per page -- linear in the retained count -- with + /// the records in ascending share_seq order across pages and every + /// page's fragments and total consistent with its records. Deliberately + /// a capacity bound rather than an RSS or wall-clock threshold (#160). + fn assert_capacity_bounded(window: &PayoutWindow, expected_seqs: &[u64], page_size: usize) { + let count = expected_seqs.len(); + assert_eq!(window.record_count(), count); + let mut seen: Vec = Vec::with_capacity(count); + let mut page_total: u128 = 0; + for (index, page) in window.pages.iter().enumerate() { + assert!(!page.records.is_empty(), "page {index} is empty"); + assert!(page.records.len() <= page_size, "page {index} overflows"); + assert_eq!( + page.fragments.len(), + page.records.len(), + "page {index} fragments" + ); + assert!( + page.records.capacity() <= page_size, + "page {index} of {} retains capacity {} for {} records (page_size {page_size})", + window.pages.len(), + page.records.capacity(), + page.records.len() + ); + assert!( + page.fragments.capacity() <= page_size, + "page {index} fragments retain capacity {}", + page.fragments.capacity() + ); + assert_eq!( + page.total_difficulty, + page.records + .iter() + .map(|record| record.share_difficulty) + .sum::() + ); + page_total += page.total_difficulty; + for (record, fragment) in page.records.iter().zip(&page.fragments) { + assert_eq!(fragment, &canonical_share_fragment(record)); + seen.push(record.share_seq); + } + } + assert_eq!(seen, expected_seqs); + assert_eq!(page_total, window.total_difficulty); + let (records_capacity, fragments_capacity) = aggregate_page_capacity(window); + assert!( + records_capacity <= window.pages.len() * page_size, + "aggregate record capacity {records_capacity} exceeds {} pages x {page_size}", + window.pages.len() + ); + assert!(fragments_capacity <= window.pages.len() * page_size); + } + + /// A freshly folded window's layout on top of the capacity bound: the + /// retained records cut at exact page boundaries, so every page but the + /// last holds `page_size` records. (An advanced window may start with + /// a shorter, partially expired page, so it checks the bound alone.) + fn assert_linear_paging(window: &PayoutWindow, expected_seqs: &[u64], page_size: usize) { + assert_capacity_bounded(window, expected_seqs, page_size); + let count = expected_seqs.len(); + assert_eq!(window.pages.len(), count.div_ceil(page_size)); + for (index, page) in window.pages.iter().enumerate() { + let expected_len = if index + 1 < window.pages.len() { + page_size + } else { + count - index * page_size + }; + assert_eq!(page.records.len(), expected_len, "page {index} length"); + } + } + + #[test] + fn full_snapshot_paging_keeps_aggregate_capacity_linear_across_sizes() { + // Every shape the paging loop can see, at several page sizes: empty, + // a single record, one short of a page, exact single and multiple + // page boundaries, one past a boundary, and a long partial tail. + for page_size in [1usize, 3, 7, 512] { + let counts = [ + 0, + 1, + page_size.saturating_sub(1), + page_size, + page_size + 1, + 2 * page_size, + 2 * page_size + 1, + 5 * page_size + 3, + 17 * page_size, + ]; + for count in counts { + let window = PayoutWindow::from_full_snapshot( + uniform_shares(count, 1), + 10, + u128::try_from(count.max(1)).unwrap() * 10, + page_size, + ) + .expect("fold"); + let expected: Vec = (1..=u64::try_from(count).unwrap()).collect(); + assert_linear_paging(&window, &expected, page_size); + assert_eq!(window.total_difficulty, u128::try_from(count).unwrap()); + } + } + } + + #[test] + fn full_snapshot_paging_after_the_cutoff_allocates_only_retained_records() { + // 1,000 eligible records of difficulty 3 plus two ineligible ones. + // Weight 700 retains the newest 234 (233 x 3 = 699 < 700, so the + // whole share crossing the weight is kept: 702 total), paged at 100 + // into two full pages and a partial third, none sized for the 766 + // expired records or the ineligible ones. + let mut records = uniform_shares(1_000, 3); + records.push(share(1_001, 3, 11, 10)); + records.push(share(1_002, 3, 10, 11)); + let window = PayoutWindow::from_full_snapshot(records, 10, 700, 100).expect("fold"); + let expected: Vec = (767..=1_000).collect(); + assert_linear_paging(&window, &expected, 100); + assert_eq!(window.total_difficulty, 702); + assert_eq!(window.pages.len(), 3); + assert_eq!(window.pages[2].records.len(), 34); + + // An exact-fit weight retains exactly the crossing share and pages + // to an exact boundary. + let window = + PayoutWindow::from_full_snapshot(uniform_shares(1_000, 3), 10, 600, 100).expect("fold"); + let expected: Vec = (801..=1_000).collect(); + assert_linear_paging(&window, &expected, 100); + assert_eq!(window.total_difficulty, 600); + + // Unsorted input pages in ascending share_seq order regardless. + let mut shuffled = uniform_shares(1_000, 1); + shuffled.reverse(); + shuffled.swap(3, 700); + let window = PayoutWindow::from_full_snapshot(shuffled, 10, 350, 64).expect("fold"); + let expected: Vec = (651..=1_000).collect(); + assert_linear_paging(&window, &expected, 64); + } + + #[test] + fn empty_snapshots_and_page_size_validation() { + let window = PayoutWindow::from_full_snapshot(vec![], 10, 10, 512).expect("fold"); + assert!(window.pages.is_empty()); + assert_eq!(window.record_count(), 0); + assert_eq!(window.total_difficulty, 0); + assert!(window.canonical_items_bytes().is_empty()); + let mut empty_digest = Sha256::new(); + empty_digest.update(b"[]"); + assert_eq!( + window.canonical_digest_hex(), + hex::encode(empty_digest.finalize()) + ); + + // Nothing eligible is the same empty window. + let window = PayoutWindow::from_full_snapshot( + vec![share(1, 2, 11, 10), share(2, 2, 10, 11)], + 10, + 10, + 512, + ) + .expect("fold"); + assert!(window.pages.is_empty()); + assert_eq!(window.record_count(), 0); + + // Validation order and categories are unchanged by the paging + // rewrite: weight first, then page size, both before any record. + assert_eq!( + PayoutWindow::from_full_snapshot(vec![share(1, 1, 5, 5)], 10, 10, 0).unwrap_err(), + WindowFoldError::NonPositivePageSize + ); + assert_eq!( + PayoutWindow::from_full_snapshot(vec![], 10, 10, 0).unwrap_err(), + WindowFoldError::NonPositivePageSize + ); + assert_eq!( + PayoutWindow::from_full_snapshot(vec![share(1, 0, 5, 5)], 10, 0, 0).unwrap_err(), + WindowFoldError::NonPositiveWindowWeight + ); + assert_eq!( + PayoutWindow::from_full_snapshot(vec![share(1, 1, 5, 5)], 10, 1, 1) + .expect("fold") + .record_count(), + 1 + ); + } + + #[test] + fn digest_items_and_payout_totals_are_independent_of_page_size() { + // Page layout is invisible on the wire: the same snapshot folded at + // every page size yields identical canonical items, digest, count + // and retained difficulty. + let records: Vec = (1..=1_000u64) + .map(|seq| share(seq, u128::from(seq % 5 + 1), 10, 10)) + .collect(); + let reference = + PayoutWindow::from_full_snapshot(records.clone(), 10, 1_500, 4_096).expect("fold"); + assert_eq!(reference.pages.len(), 1); + for page_size in [1usize, 7, 100, 512, 513] { + let window = PayoutWindow::from_full_snapshot(records.clone(), 10, 1_500, page_size) + .expect("fold"); + assert_eq!( + window.canonical_items_bytes(), + reference.canonical_items_bytes() + ); + assert_eq!( + window.canonical_digest_hex(), + reference.canonical_digest_hex() + ); + assert_eq!(window.record_count(), reference.record_count()); + assert_eq!(window.total_difficulty, reference.total_difficulty); + assert_eq!(window.shares_for_build(), reference.shares_for_build()); + let (records_capacity, _) = aggregate_page_capacity(&window); + assert!(records_capacity <= window.pages.len() * page_size); + } + } + + #[test] + fn advance_pages_a_multi_page_delta_onto_a_full_page_within_bound() { + // One exactly full page (4 records at page size 4) was within bound + // even under the previous paging, so this case isolates the advance + // delta loop from the full-snapshot loop: 23 appended records must + // land in pages sized for their contents (five full pages and one + // of three), never in a vector that kept the whole delta's capacity. + let page_size = 4; + let window = PayoutWindow::from_full_snapshot(uniform_shares(4, 1), 10, 1_000, page_size) + .expect("fold"); + assert_eq!(window.pages.len(), 1); + assert!(window.pages[0].records.capacity() <= page_size); + assert!(window.pages[0].fragments.capacity() <= page_size); + let old_digest = window.canonical_digest_hex(); + let old_items = window.canonical_items_bytes(); + + let delta: Vec = (5..=27u64).map(|seq| share(seq, 1, 11, 11)).collect(); + let (advanced, stats, byte_delta) = window.advance(delta.clone(), 11).expect("advance"); + let expected: Vec = (1..=27).collect(); + assert_linear_paging(&advanced, &expected, page_size); + assert_eq!(advanced.pages.len(), 7); + assert_eq!( + stats, + WindowAdvanceStats { + added_rows: 23, + expired_rows: 0, + touched_pages: 0 + } + ); + + // Nothing merged into the full base page, so it is shared, not + // rebuilt, and the previous generation reads exactly as before. + assert!(Rc::ptr_eq(&window.pages[0], &advanced.pages[0])); + assert_eq!(Rc::strong_count(&window.pages[0]), 2); + for page in &advanced.pages[1..] { + assert_eq!(Rc::strong_count(page), 1); + } + assert_eq!(window.record_count(), 4); + assert_eq!(window.canonical_digest_hex(), old_digest); + assert_eq!(window.canonical_items_bytes(), old_items); + + let mut union = uniform_shares(4, 1); + union.extend(delta); + let rebuilt = PayoutWindow::from_full_snapshot(union, 11, 1_000, page_size).expect("rebuild"); + assert_eq!(advanced.canonical_digest_hex(), rebuilt.canonical_digest_hex()); + let mut mirrored = old_items[byte_delta.retained_drop_bytes..].to_vec(); + mirrored.extend_from_slice(&byte_delta.appended_items); + assert_eq!(mirrored, advanced.canonical_items_bytes()); + } + + #[test] + fn advance_shares_untouched_pages_and_pages_large_deltas_linearly() { + // 38 records at page size 4: nine full pages and a partial tenth. + let page_size = 4; + let window = PayoutWindow::from_full_snapshot(uniform_shares(38, 1), 10, 1_000, page_size) + .expect("fold"); + assert_eq!(window.pages.len(), 10); + let old_digest = window.canonical_digest_hex(); + let old_items = window.canonical_items_bytes(); + + // A delta of 23 records, far larger than one page: two complete the + // partial page, the remaining 21 become five full pages and one of + // a single record. + let delta: Vec = (39..=61u64).map(|seq| share(seq, 1, 11, 11)).collect(); + let (advanced, stats, byte_delta) = window.advance(delta.clone(), 11).expect("advance"); + let expected: Vec = (1..=61).collect(); + assert_linear_paging(&advanced, &expected, page_size); + assert_eq!(advanced.pages.len(), 16); + assert_eq!( + stats, + WindowAdvanceStats { + added_rows: 23, + expired_rows: 0, + touched_pages: 1 + } + ); + + // Every untouched retained page is the same allocation in both + // generations; only the boundary page was rebuilt, and the previous + // generation still reads exactly as before. + for index in 0..9 { + assert!( + Rc::ptr_eq(&window.pages[index], &advanced.pages[index]), + "page {index} was copied instead of shared" + ); + assert_eq!(Rc::strong_count(&window.pages[index]), 2); + } + assert!(!Rc::ptr_eq(&window.pages[9], &advanced.pages[9])); + assert_eq!(Rc::strong_count(&window.pages[9]), 1); + assert_eq!(window.record_count(), 38); + assert_eq!(window.canonical_digest_hex(), old_digest); + assert_eq!(window.canonical_items_bytes(), old_items); + + // The advanced generation matches a full rebuild of the union, and + // the reported byte surgery reproduces its items stream. + let mut union = uniform_shares(38, 1); + union.extend(delta); + let rebuilt = + PayoutWindow::from_full_snapshot(union, 11, 1_000, page_size).expect("rebuild"); + assert_eq!( + advanced.canonical_digest_hex(), + rebuilt.canonical_digest_hex() + ); + let mut mirrored = old_items[byte_delta.retained_drop_bytes..].to_vec(); + mirrored.extend_from_slice(&byte_delta.appended_items); + assert_eq!(mirrored, advanced.canonical_items_bytes()); + + // Expiry keeps sharing the surviving interior pages at their new + // positions, and a rebuilt partial first page stays within bound. + let window = PayoutWindow::from_full_snapshot(uniform_shares(38, 1), 10, 30, page_size) + .expect("fold"); + assert_eq!(window.record_count(), 30); + assert_eq!(window.pages.len(), 8); + let delta: Vec = (39..=47u64).map(|seq| share(seq, 1, 11, 11)).collect(); + let (advanced, stats, _) = window.advance(delta, 11).expect("advance"); + let expected: Vec = (18..=47).collect(); + assert_capacity_bounded(&advanced, &expected, page_size); + let lengths: Vec = advanced + .pages + .iter() + .map(|page| page.records.len()) + .collect(); + assert_eq!(lengths, [3, 4, 4, 4, 4, 4, 4, 3]); + assert_eq!(stats.added_rows, 9); + assert_eq!(stats.expired_rows, 9); + // Old pages [9..13), [13..17) expired whole; [17..21) lost one + // record and was rebuilt; [21..25) .. [33..37) are shared; the old + // partial page [37..39) merged with the first two delta records. + assert!(!Rc::ptr_eq(&window.pages[2], &advanced.pages[0])); + for (old_index, new_index) in (3..7).zip(1..5) { + assert!( + Rc::ptr_eq(&window.pages[old_index], &advanced.pages[new_index]), + "old page {old_index} should be new page {new_index}" + ); + } + assert!(!Rc::ptr_eq(&window.pages[7], &advanced.pages[5])); + assert_eq!(stats.touched_pages, 2); + } +} diff --git a/crates/qbit-prism/tests/audit_cli.rs b/crates/qbit-prism/tests/audit_cli.rs index 8998771c3..92238ee98 100644 --- a/crates/qbit-prism/tests/audit_cli.rs +++ b/crates/qbit-prism/tests/audit_cli.rs @@ -85,6 +85,27 @@ fn canonical_share_segment_bytes( .into_bytes() } +fn canonicalize_cli_value(value: &serde_json::Value, label: &str) -> Vec { + let bundle_path = std::env::temp_dir().join(format!( + "qbit-prism-audit-canonicalize-{label}-{}.json", + std::process::id() + )); + fs::write(&bundle_path, serde_json::to_vec_pretty(value).unwrap()).unwrap(); + let output = Command::new(env!("CARGO_BIN_EXE_qbit-prism-audit-canonicalize")) + .arg("--input") + .arg(&bundle_path) + .output() + .unwrap(); + let _ = fs::remove_file(&bundle_path); + assert!( + output.status.success(), + "stdout: {}\nstderr: {}", + String::from_utf8_lossy(&output.stdout), + String::from_utf8_lossy(&output.stderr) + ); + output.stdout +} + #[test] fn verifier_cli_accepts_exported_power_law_bundle() { let fixture: Fixture = serde_json::from_str(include_str!( @@ -153,6 +174,42 @@ fn verifier_cli_accepts_exported_power_law_bundle() { ); } +#[test] +fn canonicalizer_restores_typed_bytes_for_reordered_unicode_and_optional_input() { + let fixture: Fixture = serde_json::from_str(include_str!( + "../fixtures/power-law-accrual.prism-fixture.json" + )) + .unwrap(); + let bundle = build_audit_bundle( + fixture.shares, + fixture.found_block, + power_law_prior_balances(), + PayoutPolicy::day_one_default(), + &manifest_signing_key(), + &ledger_signing_key(), + ) + .unwrap(); + let canonical = canonical_audit_bundle_bytes(&bundle).unwrap(); + + // Value serialization uses map order rather than AuditBundle's typed field + // order. The adapter must restore the typed canonical representation. + let mut reordered = serde_json::to_value(&bundle).unwrap(); + assert_eq!(canonicalize_cli_value(&reordered, "reordered"), canonical); + + reordered["shares"][0]["credit_policy"] = serde_json::Value::Null; + assert_eq!( + canonicalize_cli_value(&reordered, "explicit-optional-null"), + canonical + ); + + reordered["shares"][0]["miner_id"] = serde_json::Value::String("miner-é".into()); + let unicode_bundle: AuditBundle = serde_json::from_value(reordered.clone()).unwrap(); + let unicode_canonical = canonical_audit_bundle_bytes(&unicode_bundle).unwrap(); + let unicode_output = canonicalize_cli_value(&reordered, "unicode"); + assert_eq!(unicode_output, unicode_canonical); + assert!(unicode_output.windows("miner-é".len()).any(|window| window == "miner-é".as_bytes())); +} + #[test] fn verifier_cli_accepts_live_testnet_scale_legacy_bundle() { let bundle = live_testnet_scale_bundle(); @@ -173,8 +230,6 @@ fn verifier_cli_accepts_live_testnet_scale_legacy_bundle() { .arg(report.coinbase_value_sats.to_string()) .output() .unwrap(); - let _ = fs::remove_file(&bundle_path); - assert!( output.status.success(), "stdout: {}\nstderr: {}", @@ -184,6 +239,18 @@ fn verifier_cli_accepts_live_testnet_scale_legacy_bundle() { assert!( String::from_utf8_lossy(&output.stdout).contains("qbit.prism.audit-verification-report.v1") ); + let canonical_output = Command::new(env!("CARGO_BIN_EXE_qbit-prism-audit-canonicalize")) + .arg("--input") + .arg(&bundle_path) + .output() + .unwrap(); + let _ = fs::remove_file(&bundle_path); + assert!(canonical_output.status.success()); + assert_eq!( + canonical_output.stdout, + canonical_audit_bundle_bytes(&bundle).unwrap() + ); + assert!(bundle.found_block.network_difficulty > u64::MAX as u128); } #[test] @@ -1188,7 +1255,8 @@ fn build_audit_bundle_serve_mode_caches_parsed_windows() { stdout.read_line(&mut line).unwrap(); let handshake: serde_json::Value = serde_json::from_str(&line).unwrap(); assert_eq!(handshake["event"], "handshake"); - assert_eq!(handshake["protocol"], 1); + // Protocol 2: prepare_window plus the append-invalidation epoch tag. + assert_eq!(handshake["protocol"], 2); assert_eq!(handshake["tool"], "qbit-prism-build-audit-bundle"); let mut request_with_window = build_fields.clone(); @@ -1282,3 +1350,382 @@ fn build_audit_bundle_serve_mode_caches_parsed_windows() { let status = daemon.wait().unwrap(); assert!(status.success()); } + +#[test] +fn prepare_window_declares_out_of_range_values_and_survives_them() { + // serde refuses an integer above its declared width with the same parse + // error a corrupt line gets. Answered as "malformed serve request", the + // coordinator read it as an anomaly and SIGKILLed a healthy daemon once + // per materialization for as long as the window held the value. The + // daemon now classifies such a request as out_of_range -- naming the + // field, its literal and the width -- keeps its state, and goes on + // serving; a genuinely malformed line is still malformed, and the two + // rejection shapes carry their stable category beside the message. + use std::io::{BufRead, BufReader as StdBufReader, Read as IoRead, Write as IoWrite}; + use std::process::Stdio; + + let mut daemon = Command::new(env!("CARGO_BIN_EXE_qbit-prism-build-audit-bundle")) + .arg("--serve") + .arg("--signing-key-seed-hex") + .arg("42".repeat(32)) + .arg("--ledger-signing-key-seed-hex") + .arg("43".repeat(32)) + .stdin(Stdio::piped()) + .stdout(Stdio::piped()) + .stderr(Stdio::piped()) + .spawn() + .unwrap(); + let mut stdin = daemon.stdin.take().unwrap(); + let mut stdout = StdBufReader::new(daemon.stdout.take().unwrap()); + let mut line = String::new(); + stdout.read_line(&mut line).unwrap(); + assert_eq!( + serde_json::from_str::(&line).unwrap()["event"], + "handshake" + ); + + // Literals above u128/u32 cannot be built with json!, so the request is + // written as text -- exactly as the coordinator's json.dumps sends it. + fn share_text(seq: u64, difficulty: &str, ntime: &str, ms: i64) -> String { + format!( + concat!( + "{{\"share_seq\":{seq},\"share_id\":\"share-{seq}\",\"miner_id\":\"miner-{m}\",", + "\"order_key\":\"{m:02}:miner\",\"p2mr_program_hex\":\"{hex}\",", + "\"share_difficulty\":{difficulty},\"network_difficulty\":1000,", + "\"template_height\":100,\"job_id\":\"job-{seq}\",", + "\"job_issued_at_ms\":{ms},\"accepted_at_ms\":{ms},\"ntime\":{ntime}}}" + ), + seq = seq, + m = seq % 3, + hex = format!("{:02x}", seq % 256).repeat(32), + difficulty = difficulty, + ms = ms, + ntime = ntime, + ) + } + let prepare = |records: &[String], window_weight: &str| { + format!( + "{{\"request\":\"prepare_window\",\"mode\":\"full\",\"append_invalidation_epoch\":1,\ + \"anchor_job_issued_at_ms\":1000003,\"window_weight\":{window_weight},\"records\":[{}]}}", + records.join(",") + ) + }; + fn answer( + stdin: &mut std::process::ChildStdin, + stdout: &mut StdBufReader, + request: &str, + ) -> serde_json::Value { + writeln!(stdin, "{request}").unwrap(); + let mut line = String::new(); + stdout.read_line(&mut line).unwrap(); + serde_json::from_str(&line).unwrap_or_else(|error| panic!("{error}: {line}")) + } + + // share_difficulty one above u128 on the second record. + let beyond_u128 = answer( + &mut stdin, + &mut stdout, + &prepare( + &[ + share_text(1, "1", "1700000000", 1_000_001), + share_text( + 2, + "340282366920938463463374607431768211456", + "1700000000", + 1_000_001, + ), + ], + "1000", + ), + ); + assert_eq!(beyond_u128["ok"], false, "{beyond_u128}"); + assert_eq!(beyond_u128["request"], "prepare_window"); + assert_eq!(beyond_u128["out_of_range"], true); + assert_eq!(beyond_u128["field"], "records[1].share_difficulty"); + assert_eq!(beyond_u128["width"], "u128"); + assert!(beyond_u128.get("needs_window").is_none()); + assert!(beyond_u128["error"] + .as_str() + .unwrap() + .contains("340282366920938463463374607431768211456")); + + // ntime one above u32, and a window_weight one above u128. + let beyond_u32 = answer( + &mut stdin, + &mut stdout, + &prepare(&[share_text(1, "1", "4294967296", 1_000_001)], "1000"), + ); + assert_eq!(beyond_u32["out_of_range"], true, "{beyond_u32}"); + assert_eq!(beyond_u32["field"], "records[0].ntime"); + assert_eq!(beyond_u32["width"], "u32"); + let beyond_weight = answer( + &mut stdin, + &mut stdout, + &prepare( + &[share_text(1, "1", "1700000000", 1_000_001)], + "340282366920938463463374607431768211456", + ), + ); + assert_eq!(beyond_weight["out_of_range"], true, "{beyond_weight}"); + assert_eq!(beyond_weight["field"], "window_weight"); + + // Every input at its width, but the retained total leaves u128: the + // accumulator is declined, never saturated. + let overflow = answer( + &mut stdin, + &mut stdout, + &prepare( + &[ + share_text( + 1, + "170141183460469231731687303715884105728", + "1700000000", + 1_000_001, + ), + share_text( + 2, + "170141183460469231731687303715884105728", + "1700000000", + 1_000_001, + ), + ], + "340282366920938463463374607431768211455", + ), + ); + assert_eq!(overflow["out_of_range"], true, "{overflow}"); + assert_eq!(overflow["field"], "total_difficulty"); + assert_eq!(overflow["width"], "u128"); + + // A genuinely malformed line is still a malformed request. + let malformed = answer(&mut stdin, &mut stdout, "{\"request\":\"prepare_window\",\"mode\":\"full\",\"records\":[{\"share_difficulty\":1.5}]"); + assert_eq!(malformed["ok"], false); + assert!(malformed["error"] + .as_str() + .unwrap() + .starts_with("malformed serve request")); + assert!(malformed.get("out_of_range").is_none()); + + // Rejections name their condition in a stable category. + let duplicate = answer( + &mut stdin, + &mut stdout, + &prepare( + &[ + share_text(1, "1", "1700000000", 1_000_001), + share_text(1, "2", "1700000000", 1_000_001), + ], + "1000", + ), + ); + assert_eq!(duplicate["fold_invalid"], true, "{duplicate}"); + assert_eq!(duplicate["rejection"], "duplicate_share_seq"); + assert_eq!( + duplicate["error"], + "full payout window contains duplicate share_seq" + ); + + // The same process, afterwards, prepares and advances a window at the + // declared edges exactly: u64::MAX share_seq, u128::MAX window_weight, + // u32::MAX ntime, and difficulties whose retained total is exactly + // u128::MAX. + let edge = answer( + &mut stdin, + &mut stdout, + &prepare( + &[ + share_text( + 1, + "340282366920938463463374607431768211454", + "4294967295", + 1_000_001, + ), + share_text(18446744073709551615, "1", "4294967295", 1_000_001), + ], + "340282366920938463463374607431768211455", + ), + ); + assert_eq!(edge["ok"], true, "{edge}"); + assert_eq!(edge["record_count"], 2); + let base_digest = edge["share_snapshot_sha256"].as_str().unwrap().to_string(); + let items_len = edge["window_items_len"].as_u64().unwrap() as usize; + let mut raw = vec![0u8; items_len + 1]; + stdout.read_exact(&mut raw).unwrap(); + assert_eq!(raw[items_len], b'\n'); + let items = String::from_utf8(raw[..items_len].to_vec()).unwrap(); + assert!(items.contains("\"share_difficulty\":340282366920938463463374607431768211454")); + assert!(items.contains("\"share_seq\":18446744073709551615")); + assert!(items.contains("\"ntime\":4294967295")); + + let advance = answer( + &mut stdin, + &mut stdout, + &format!( + "{{\"request\":\"prepare_window\",\"mode\":\"advance\",\"append_invalidation_epoch\":1,\ + \"anchor_job_issued_at_ms\":1000004,\"base_digest\":\"{base_digest}\",\"records\":[]}}" + ), + ); + assert_eq!(advance["ok"], true, "{advance}"); + assert_eq!(advance["record_count"], 2); + let appended_len = advance["appended_items_len"].as_u64().unwrap() as usize; + let mut raw = vec![0u8; appended_len + 1]; + stdout.read_exact(&mut raw).unwrap(); + + let not_append = answer( + &mut stdin, + &mut stdout, + &format!( + "{{\"request\":\"prepare_window\",\"mode\":\"advance\",\"append_invalidation_epoch\":1,\ + \"anchor_job_issued_at_ms\":1000005,\"base_digest\":\"{base_digest}\",\"records\":[{}]}}", + share_text(3, "1", "1700000000", 1_000_005) + ), + ); + assert_eq!(not_append["fallback"], true, "{not_append}"); + assert_eq!(not_append["rejection"], "delta_not_append"); + + drop(stdin); + let status = daemon.wait().unwrap(); + assert!(status.success()); +} + +#[test] +fn prepare_window_advance_ignores_the_append_invalidation_epoch_tag() { + // The coordinator's late-append policy is finer than any tag this daemon + // could apply: it clears the incremental cache only when the late row + // predates the cached anchor, and otherwise RETAGS the window so the next + // delta build advances it in place. A daemon that demanded an exact tag + // match turned every such retag into a needs_full -- a full DB walk, fold + // and upload -- while catching nothing the coordinator had not already + // decided. The tag rides along for diagnostics and nothing else. + use std::io::{BufRead, BufReader as StdBufReader, Read as IoRead, Write as IoWrite}; + use std::process::Stdio; + + fn share(seq: u64) -> serde_json::Value { + serde_json::json!({ + "share_seq": seq, + "share_id": format!("share-{seq}"), + "miner_id": format!("miner-{}", seq % 3), + "order_key": format!("{:02}:miner", seq % 3), + "p2mr_program_hex": format!("{:02x}", seq).repeat(32), + "share_difficulty": 1, + "network_difficulty": 1000, + "template_height": 100, + "job_id": format!("job-{seq}"), + "job_issued_at_ms": 1_000_000 + seq as i64, + "accepted_at_ms": 1_000_000 + seq as i64, + "ntime": 1_700_000_000u32, + }) + } + + let mut daemon = Command::new(env!("CARGO_BIN_EXE_qbit-prism-build-audit-bundle")) + .arg("--serve") + .arg("--signing-key-seed-hex") + .arg("42".repeat(32)) + .arg("--ledger-signing-key-seed-hex") + .arg("43".repeat(32)) + .stdin(Stdio::piped()) + .stdout(Stdio::piped()) + .stderr(Stdio::piped()) + .spawn() + .unwrap(); + let mut stdin = daemon.stdin.take().unwrap(); + let mut stdout = StdBufReader::new(daemon.stdout.take().unwrap()); + let mut line = String::new(); + stdout.read_line(&mut line).unwrap(); + assert_eq!( + serde_json::from_str::(&line).unwrap()["event"], + "handshake" + ); + + // A section reader that consumes the raw canonical-items payload plus its + // terminating newline, so the next envelope line starts clean. + let drain_section = |stdout: &mut StdBufReader, len: usize| { + let mut raw = vec![0u8; len + 1]; + stdout.read_exact(&mut raw).unwrap(); + assert_eq!(raw[len], b'\n'); + raw.truncate(len); + raw + }; + + let prepared = serde_json::json!({ + "request": "prepare_window", + "mode": "full", + "append_invalidation_epoch": 7, + "anchor_job_issued_at_ms": 1_000_003i64, + "window_weight": 1_000, + "records": [share(1), share(2), share(3)], + }); + writeln!(stdin, "{}", serde_json::to_string(&prepared).unwrap()).unwrap(); + line.clear(); + stdout.read_line(&mut line).unwrap(); + let full: serde_json::Value = serde_json::from_str(&line).unwrap(); + assert_eq!(full["ok"], true, "full response: {line}"); + assert_eq!(full["record_count"], 3); + let base_digest = full["share_snapshot_sha256"].as_str().unwrap().to_string(); + drain_section(&mut stdout, full["window_items_len"].as_u64().unwrap() as usize); + + // A build at a newer epoch must not evict the prepared window either. + let build = serde_json::json!({ + "window_key": {"share_snapshot_sha256": base_digest}, + "append_invalidation_epoch": 8, + "found_block": { + "block_height": 101, + "coinbase_value_sats": 5_000_000_000u64, + "network_difficulty": 1000, + "anchor_job_issued_at_ms": 1_000_003i64, + }, + "prior_balances": [], + }); + writeln!(stdin, "{}", serde_json::to_string(&build).unwrap()).unwrap(); + line.clear(); + stdout.read_line(&mut line).unwrap(); + let served: serde_json::Value = serde_json::from_str(&line).unwrap(); + assert_eq!(served["ok"], true, "build response: {line}"); + assert_eq!(served["window_cache"]["hit"], true); + + // The retag case: the base was prepared at epoch 7, the advance arrives + // at epoch 9, and it advances in place instead of answering needs_full. + let advance = serde_json::json!({ + "request": "prepare_window", + "mode": "advance", + "append_invalidation_epoch": 9, + "anchor_job_issued_at_ms": 1_000_004i64, + "base_digest": base_digest, + "records": [share(4)], + }); + writeln!(stdin, "{}", serde_json::to_string(&advance).unwrap()).unwrap(); + line.clear(); + stdout.read_line(&mut line).unwrap(); + let advanced: serde_json::Value = serde_json::from_str(&line).unwrap(); + assert_eq!(advanced["ok"], true, "advance response: {line}"); + assert!(advanced.get("needs_full").is_none()); + assert_eq!(advanced["record_count"], 4); + assert_eq!(advanced["added_rows"], 1); + // The tags are reported, never acted on. + assert_eq!(advanced["base_append_invalidation_epoch"], 7); + assert_eq!(advanced["append_invalidation_epoch"], 9); + let appended = drain_section( + &mut stdout, + advanced["appended_items_len"].as_u64().unwrap() as usize, + ); + assert!(!appended.is_empty()); + + // An unheld digest is still the one thing that answers needs_full. + let unknown = serde_json::json!({ + "request": "prepare_window", + "mode": "advance", + "append_invalidation_epoch": 9, + "anchor_job_issued_at_ms": 1_000_005i64, + "base_digest": "0".repeat(64), + "records": [], + }); + writeln!(stdin, "{}", serde_json::to_string(&unknown).unwrap()).unwrap(); + line.clear(); + stdout.read_line(&mut line).unwrap(); + let missing: serde_json::Value = serde_json::from_str(&line).unwrap(); + assert_eq!(missing["ok"], false); + assert_eq!(missing["needs_full"], true); + + drop(stdin); + let status = daemon.wait().unwrap(); + assert!(status.success()); +} diff --git a/doc/mining.md b/doc/mining.md index 9c058ccb1..bd8e9cecb 100644 --- a/doc/mining.md +++ b/doc/mining.md @@ -280,7 +280,7 @@ AuxPoW Stratum vardiff is enabled by default in compose so mixed miner hardware The bridge reconstructs one canonical parent header from each Stratum submit. Diagnostic header variants remain available with `AUXPOW_STRATUM_DIAG_VARIANTS=1`, but they are not accepted by default. Use `AUXPOW_STRATUM_HEADER_VARIANT=` or `AUXPOW_STRATUM_ACCEPT_DIAGNOSTIC_VARIANT=1` only while debugging a miner compatibility issue. -BIP310 version rolling defaults to `AUXPOW_STRATUM_VERSION_MASK=1fffe000`, and the bridge grants miners the intersection of that parent-chain mask and the requested mask. Permissionless qbit, ckpool, and direct PRISM Stratum use the connected qbit node's `getblocktemplate.versionrollingmask` as the server mask, falling back to `1fffe000` only when GBT is unavailable or does not expose the field. AuxPoW Stratum mode still serves parent-chain headers and submits child work through `createauxblock` / `submitauxblock`; when the connected qbit node exposes `versionrollingmask`, the bridge logs it at startup as compatibility context. +BIP310 version rolling defaults to `AUXPOW_STRATUM_VERSION_MASK=1fffe000`, and the bridge grants miners the intersection of that parent-chain mask and the requested mask. Permissionless ckpool and direct PRISM Stratum use the connected qbit node's `getblocktemplate.versionrollingmask` as the server mask. In dynamic mode, ckpool resolves its mask after the startup readiness gate and fails closed when GBT is unavailable; it falls back to `1fffe000` only when a successful template omits the field. Explicit static mode uses the configured ckpool mask. Direct PRISM retains its configured fallback when its startup probe is unavailable or the field is missing. AuxPoW Stratum mode still serves parent-chain headers and submits child work through `createauxblock` / `submitauxblock`; when the connected qbit node exposes `versionrollingmask`, the bridge logs it at startup as compatibility context. Useful diagnostics: @@ -324,7 +324,7 @@ Important behavior: - Direct PRISM Stratum uses the leading username segment as the payout address. On qbit test chains, invalid username payouts fall back to `tq1zlsq9dpxz8mennhdpr9nf9s0f2tjtq6gxs9m84k6xglhkfp92q2zszzu4m3`; set `PRISM_USERNAME_FALLBACK_ADDRESS` to override that address. - Direct PRISM Stratum does not emit miner outputs below the payout floor. The default floor is `14,720` bits from `3,680` P2MR spend bytes, `1` bit/byte, and a `4x` safety multiplier. Set `PRISM_PAYOUT_MIN_OUTPUT_BITS` for a fixed absolute floor, or tune `PRISM_PAYOUT_P2MR_SPEND_INPUT_BYTES`, `PRISM_PAYOUT_TARGET_FEERATE_BITS_PER_BYTE`, and `PRISM_PAYOUT_SAFETY_MULTIPLIER`. The old `_SATS` payout env names are still accepted as legacy aliases. - Until DATUM-style per-hardware templates are supported, use conservative non-DATUM settlement caps for one stock-firmware-safe Stratum template. The qbit-prism launch defaults are `12` direct recipient coinbase outputs, `16` total settlement coinbase outputs, and `1,000` recipients per CTV fanout chunk. These are hardware policy defaults, not qbit consensus limits. -- Direct PRISM Stratum refreshes jobs from both `waitfornewblock` and the fallback `PRISM_BLOCKPOLL_SECONDS` poller. After a tip flip, a short `PRISM_STRATUM_STALE_GRACE_SECONDS` window credits same-connection prior-tip shares until that long after the connection receives new-tip work (in-flight shares stay creditable while a slow or failed refresh pass has not reached the connection), but PRISM never submits those old-tip headers as blocks. Credited rows carry `credit_policy=stale-grace` for audit; bundles containing them use `qbit.prism.audit-bundle.v1.1`, so upgrade verifiers before enabling stale grace in production. +- Direct PRISM Stratum refreshes jobs from both `waitfornewblock` and the fallback `PRISM_BLOCKPOLL_SECONDS` poller. After a tip flip, a short `PRISM_STRATUM_STALE_GRACE_SECONDS` window credits same-connection prior-tip shares until that long after the connection receives new-tip work (in-flight shares stay creditable while a slow or failed refresh pass has not reached the connection), but PRISM never submits those old-tip headers as blocks. Credited rows carry `credit_policy=stale-grace` for audit; bundles containing them use `qbit.prism.audit-bundle.v1.1`, and the window is non-zero by default on every chain, so keep verifiers on a release that accepts that schema. - Monitor `/audit/carry-forward-integrity` for carry-forward replay mismatches and mirror the published `audit_head_sha256`. Launch policy is no compaction of carry-forward rows until signed checkpoints cover the old row range and replacement opening balances. - Expect more frequent UTXO creation than on Bitcoin if you pay out on a block-count schedule instead of a balance threshold. diff --git a/doc/release-notes-2.0.0.md b/doc/release-notes-2.0.0.md new file mode 100644 index 000000000..8612214f7 --- /dev/null +++ b/doc/release-notes-2.0.0.md @@ -0,0 +1,332 @@ +# qbit-mining-bootstrap 2.0.0 Release Notes + +This is the first 2.x release. Changes temporarily labelled 2.0.1 and 2.0.2 +during development are included here; those versions were not tagged or +published as GitHub releases. + +## Highlights + +- Separates the PRISM public dashboard API from the mining coordinator into + the `prism-public-api` service, with a Postgres hot standby for public reads, + bounded stale responses during outages, and independent readiness checks. +- Completes the PRISM coordinator decomposition into owners for share ingest, + payout state, job delivery, block landing, audit artifacts, vardiff, and + observability, backed by deterministic concurrency and landing harnesses. +- Hardens accepted-block accounting under dense block cadence: definitive node + acceptances have a dedicated accounting lane, obsolete candidates collapse + before replay, stuck accepted-parent transitions are retried in-process, + and accepted-preview publication and cleanup backlogs are bounded. +- Bounds payout-window work and memory across database decoding, serialization, + builder transport, and Rust paging. An opt-in Rust daemon pipeline handles + window folding, canonical digests, and incremental advances, with Python + fallback and a differential parity gate against the real daemon. +- Preserves converged worker difficulty across reconnects and, with Postgres, + coordinator restarts; accelerates initial vardiff convergence and restores + the configured prior-tip share grace window on mainnet. +- Adds public block markers, reorg visibility, network hashrate, dual raw and + smoothed credited-hashrate series, and incremental historical rollups. +- Persists canonical audit-bundle bytes and durable audit publication order, + improves writer-lease behavior under contention, and adds heap, allocator, + component-cardinality, and lease-monitor diagnostics. +- Bounds candidate persistence, startup replay, and canonical audit finalization + with chunked storage and file-backed views; adds offline recovery of + accepted pending blocks. +- Resolves CKPool version masks after node readiness and refuses dynamic-mode + startup when no live block template can be obtained. +- Moves the Python service images and CI to Python 3.14 and splits CI into + independent lint, Python, Postgres, and Rust checks. + +## Mainnet Compatibility + +This release continues to pair with qbit `v1.0.0` at +`7ebcddb622d6e639041f005a189b048ec2a221fe`. The checked-in qbit source pin, +Bitcoin Core 30.2 release pins, and AuxPoW chain IDs are unchanged: + +- Mainnet: `47` +- Public testnet4: `31430` + +The qbit mainnet genesis hash remains: + +`0000000000004d60aa5d46013991d0a0e2995d89ee98e53068ae196d763e79f2` + +The 2.0.0 version identifies this bootstrap and pool software release. The +public API remains under `/public/v1`; individual opt-in response schemas +have their own version tags. + +## Upgrade Notes + +### Public API service and replica cutover + +- Route `/public/v1` traffic to `prism-public-api`, on port `3342` by default. + The coordinator's audit HTTP listener now returns 404 for these routes. + Keep each process's `/healthz` and `/metrics`, and the coordinator's audit + and operator endpoints, private. +- The shipped Compose profile adds `prism-postgres-replica` and reads through + `PRISM_PUBLIC_DATABASE_URL`. Compose defaults + `PRISM_PUBLIC_REPLICA_MODE=require`; provision the standby before cutover. + Leave the public DSN empty to derive the built-in standby URL from the + configured Postgres credentials, or set an explicit URL for a separate + standby. The coordinator keeps its primary database DSN. Direct launches + outside Compose default replica enforcement to `off`. +- The public psql backend also uses the public DSN. If a wrapper or custom + connection arguments are needed, set `PRISM_PUBLIC_PSQL_COMMAND` separately; + the coordinator's `PRISM_POSTGRES_PSQL_COMMAND` is not inherited by the + public service. +- Production needs an explicit `PRISM_POSTGRES_REPLICA_DATA_SOURCE` directory + and read-only access to the shared audit files from the public API service. + Release-provenance checks require the replica path to be absolute and + non-overlapping with other enabled state sources before contacting Docker, + including normalized paths and filesystem aliases. + Use a dedicated replication role and monitor the physical replication slot + and retained WAL. The public standby is a read tier, not an automated + failover target. Follow + [the replica runbook](../docs/prism-postgres-replica.md) for provisioning, + the non-default database-user caveat, lag monitoring, and slot retirement. +- Replica freshness enforcement bounds replication-stream heartbeat age, + with a default limit of 60 seconds; replay lag is reported separately. + Outage responses may serve eligible cached data only within the documented + staleness bounds, after which the service refuses the read with 503. +- The public HTTP server defaults to 64 active connections, a 10-second total + request-header deadline and a 10-second socket I/O idle timeout. Excess + connections close immediately; each response closes its connection. + `PRISM_PUBLIC_HTTP_MAX_CONNECTIONS` and `PRISM_PUBLIC_HTTP_TIMEOUT_SECONDS` + control these bounds. Review proxy upstream connection settings. +- The advertised Stratum URL must use `stratum+tcp` or `stratum+ssl`, include a + valid hostname and explicit port, and contain no credentials, path, query, + fragment or whitespace. Invalid endpoints now refuse startup. + +### Ledger migration and audit history + +- Back up the ledger and stop the old coordinator before applying + `crates/qbit-prism/sql/001_share_ledger.sql`, followed by + `crates/qbit-prism/sql/002_candidate_bodies.sql`. Both are required, even + when configured to write legacy candidate bodies. The coordinator and + offline recovery command refuse a database without the second migration. + The first migration runs atomically, repairs partially seeded carry-forward + summaries, adds durable worker-difficulty state, and assigns + `audit_publication_sequence` to historical confirmed and inactive blocks. +- The migration takes exclusive table locks and builds a unique index + non-concurrently. Schedule a maintenance window that allows interruption + of reads as well as writes. With `PRISM_POSTGRES_INIT_SCHEMA=1`, the new + coordinator applies the schema before opening listeners. For explicit + application, follow the fail-fast transaction requirements in + [the ledger operations guide](../docs/prism-ledger-ops.md) and apply the + candidate-body migration next, as described in + [the candidate storage guide](../docs/prism-candidate-storage.md). +- New audit bundles are stored and served as their exact canonical bytes. + Historical bundles retain the legacy reconstructed response until verified + backfill publishes the canonical artifact. Run + `python3 -m lab.prism.backfill_audit_bundle_canonical --dry-run` to inspect + history; stop the coordinator and release its writer lease before publish + mode with `--no-init-schema`. Review failure rows as well as checkpoints. + See [settlement artifacts](../docs/public-dashboard-api/README.md#settlement-artifacts) + for the complete backfill and replica-visibility procedure. +- Use the documented ledger rollback procedure if reverting the software. + The targeted audit-publication revert SQL is for a coordinated schema + rollback with all new-version writers stopped. Candidate-format rollback + must also follow the compatibility restrictions below. + +### Candidate storage and offline recovery + +- `PRISM_CANDIDATE_STORAGE_VERSION` defaults to `2`. Candidate bodies use + immutable 256 KiB chunks and bounded replay headers; existing version 1 + JSONB candidates remain readable. Candidate identity, timestamp normalization, + atomic share/outbox publication, and writer-session fencing are preserved. +- Canonical audit finalization uses replayable file-backed views and streamed + publication. Replay membership uses an exact PostgreSQL subset check through + bounded COPY writes. Spools, temporary tables, indexes, and candidate body + references have explicit retirement paths. +- `PRISM_BLOCK_REPLAY_PAGE_SIZE` accepts 1–1,024 descriptors. Smaller pages + increase database/RPC round trips; a row-count limit alone does not bound + the bytes in one candidate. Candidate spool reservations default to 4 GiB + per ledger instance; an empty `PRISM_CANDIDATE_SPOOL_DIR` selects the system + temporary directory. Provision and monitor that filesystem separately. +- Use `python3 -m lab.prism.recover_pending_blocks --help` for the offline + accepted-pending recovery command. Follow + [the ledger operations guide](../docs/prism-ledger-ops.md) for stopping the + writer, acquiring its lease, and reviewing recovery results. +- Qualify a compatible reader before enabling new-format writes. Keep the + additive schema during rollback. Once a new-format pending candidate exists, + an older reader that cannot hydrate it is unsafe. Use a compatible reader + with `PRISM_CANDIDATE_STORAGE_VERSION=1` to stop new-format writes, or a + separately verified drain/conversion procedure. Setting `1` does not remove + the migration requirement. Follow + [the candidate storage guide](../docs/prism-candidate-storage.md) for the + complete schema, capacity, rollout, and rollback procedure. + +### Configuration and runtime behavior + +- No existing assigned `.env.example` key was removed, renamed, or given a + different value since 1.1.0. There are new service, replica, vardiff, + database-session, and diagnostic settings; deployment behavior still + changes as described above. Review your deployment environment alongside + the new example and [mainnet runbook](../docs/mainnet-deployment.md). +- Mainnet no longer forces `PRISM_STRATUM_STALE_GRACE_SECONDS=0`. The sample + and code default is 3 seconds: qualifying prior-tip shares are credited + during this window but never submitted as block candidates. An explicit + zero in an existing deployment remains zero until the operator changes it. +- Difficulty resume is enabled by default, retains up to 8,192 worker entries, + and has a 900-second TTL. It is keyed by listener and exact Stratum username; + give heterogeneous rigs distinct worker names. Postgres-backed deployments + can preload recent converged values after restart. Resume is bounded + assistance, not a guarantee that every active worker survives an abrupt + exit with a retained difficulty. +- `PRISM_WINDOW_PIPELINE_RUST` defaults to `0`. When enabled with the daemon + transport, unsupported integer ranges and daemon anomalies fall back to the + Python pipeline for that materialization. Qualify a deployment before + changing this switch; the real-daemon parity gate is part of CI. +- CKPool dynamic version-mask resolution runs after node readiness, retries + transient template failures, and aborts startup if no template is available. + A successful template without the field retains the configured fallback; + an explicit zero mask remains zero. `CKPOOL_VERSION_MASK_MODE=static` is the + explicit option for authorized prelaunch without template availability. +- Rebuild service images for Python 3.14. The PRISM image sets + `MALLOC_ARENA_MAX=2`; allocator telemetry defaults on, while heap census, + tracemalloc, and malloc-trim controls default off. The optional + `PRISM_PYTHON_SWITCH_INTERVAL_SECONDS` override remains unset by default. + Follow the capacity and allocator runbooks before tuning these controls. +- Update consumers of removed `qbit_prism_shares_per_second` + telemetry to rates derived from the monotonic share counters. Review + coordinator lock, accepted-preview, lease-monitor, and memory metrics with + the [overload alert specification](../docs/prism-overload-alerts.md). + That document is a specification; this repository does not install live + alert rules. +- Dashboard clients can opt into dual-rate series with `view=both`, reorg + block history with `chain_state=all` or `reversed`, and the new + `/public/v1/block-markers` endpoint. Unsupported range/bucket combinations + now return 400; use the allowed vocabulary in the + [public API contract](../docs/public-dashboard-api/README.md). + +## Changes Since v1.1.0 + +This release includes the `2.x.x` changes after `v1.1.0` through +`504846c` (#258), along with release-promotion fixes in #246. + +### Coordinator ownership and concurrency validation + +- Decompose coordinator core owners, audit artifacts, block candidate + submission, share submission, vardiff, block finalization, and observability; + shard tests by domain and document the completed refactor and review stack. +- Add deterministic writer-lease and block-landing concurrency harnesses + (#139, #148), concurrency instruments and classification fixes (#170), + candidate-storm characterization (#192), and a per-row storm oracle (#194). +- Apply the integrated PRISM core, infrastructure, and vardiff stability + review fixes (#141). + +### Block landing, accounting, and writer leases + +- Preserve the audit envelope when two found blocks land within one + confirmation/publication window (#136), distinguish confirmation replay + from fresh confirmation (#146), and validate fresh confirmations against + the current-balance summary (#210). +- Bound startup candidate enumeration and sweep stranded prepared blocks + (#142); add fenced bulk abandonment (#195), collapse superseded candidates + before replay (#196), and bound collapsed-candidate cleanup retries (#222). +- Give definitive node acceptances their own accounting lane, skip provably + stale candidates at dequeue, and measure acceptance-to-preview latency + (the three follow-up commits referencing #181). Bound preview publication + latency during dense accepted-tip cadence (#230). +- Re-drive stuck accepted-parent payout transitions in-process (#205), classify + their preview backpressure as coordination-blocked (#190), and probe each + ancestor height once when selecting a transition (#193). +- Keep replay reads out of the writer convoy (#213), scope append invalidation + per anchor (#151), and route the landing tail through the named submit port + (#172). +- Bound writer-lease acquisition and guard Postgres sessions (#137), enforce + read-only sessions and name lease races (#168), tolerate heartbeat + verification tail latency (#214), and attribute lease-monitor stalls with + explicit lateness responses (#233). +- Cap escalated landing deadlines below the watchdog tolerance (#135). +- Allow one-candidate startup replay pages without truncating enumeration + (#256), and add offline recovery for accepted pending blocks (#259). +- Scope pending-fanout broadcast attempts to their own fanout (#292). + +### Payout windows, audit artifacts, and memory + +- Make schema applies atomic and repair a partially seeded carry-forward + summary (#155). +- Serve content-addressed artifacts as their exact canonical bytes (#154) + and persist byte-exact canonical audit bundles with verified historical + backfill tooling (#173). +- Stop serializing a full block for every accepted share (#162), add a + differential window parity oracle (#165), move folding, digests, and + incremental advance into the persistent builder (#166), and gate parity + against the real Rust daemon (#178). +- Hold payout snapshot weight within a band to avoid per-block rescans + (#206); bound full rescans and retain daemon preparation after re-centering + (#235). +- Replace builder pipe polling with readiness waits (#237), bound share + serialization and lazy parsing across all paths (#239), bound database + decoding (#238), and bound Rust window paging memory (#241). +- Release completed executor work and callback captures, avoiding traceback + cycles that retain finished job builds and delivery requests (#250). +- Release failed public-cache and address-validation requests (#252), and + clear finished cancellation frames while keeping each job-build waiter's + traceback separate (#253), without waiting for cyclic garbage collection. +- Bound candidate encoding, persistence, replay, payout membership checks, + and canonical audit finalization; add the candidate-body migration, spool + ownership, and crash/restart recovery gates (#258). + +### Stratum, vardiff, and delivery health + +- Persist per-session vardiff difficulty across reconnects (#138), improve + initial difficulty convergence and credited-hashrate visibility (#204), + and stop overriding mainnet stale-share grace to zero (#225). +- Remove the first-job admission lock convoy (#171) and evaluate delivery + health by semantic work coverage (#221). +- Resolve CKPool version masks after node readiness, preserve static mode and + explicit zero masks, and validate retry settings before wallet setup (#257). + +### Public dashboard API + +- Document the public/operator endpoint split (#147), extract the public read + process (#152), and serve its database reads from a hot standby (#157). +- Serve bounded stale public reads during database outages (#167), enforce + hashrate-series statement deadlines and bucket vocabulary with in-process + stale-while-revalidate (#218), and serve historical series from incremental + rollups (#219). +- Add reorg visibility and network hashrate (#217), dual-rate credited-hashrate + series (#204), and found-block markers (#220). + +### Observability, runtime, and CI + +- Replace the shares-per-second gauge and add a coordinator-lock contention + denominator (#161); keep telemetry useful under overload (#191). +- Add always-on heap and component-cardinality telemetry (#232), bounded heap + census and allocator controls, and a documented resident-memory soak bound + (#234). +- Release sampled thread-frame references after each stall-probe sample, + including sampling caps and formatting errors, so completed workers' locals + are not retained until cyclic garbage collection (#248). +- Make the interpreter switch interval tunable (#179), move service images + and CI to Python 3.14 (#201), and shard lint, compile, Python, Postgres, and + Rust CI checks while preserving the aggregate required check name (#243). +- Run push CI only for long-lived branches; PR branches are tested through + their pull-request merge result (#295). + +## Operator Verification + +- Run `make doctor` with the deployment environment before starting the stack. +- After migration and startup, run `make prism-self-check`, inspect coordinator + and public API readiness separately, and verify public requests reach the + new service and its intended database. +- Verify replica health, WAL retention, canonical artifact downloads, accepted + share accounting, and worker difficulty after reconnects before widening + traffic. Keep production images digest-qualified. +- Additional validation targets include `make test-prism-postgres-seed-guard` + and `make test-prism-public-read-replica`, plus + `make test-prism-postgres-candidate-storage` and + `make test-prism-postgres-native-ledger` for both candidate storage formats. + Capacity qualification and memory soak procedures are in + [the capacity readiness guide](../docs/prism-capacity-readiness.md). +- Candidate codec/storage experiments and actual mined-block crash/restart + gates are separate evidence. ARM64 experiments do not establish production + x86_64 performance or prove the cause of the exits reported in #255. + PostgreSQL and exceptional helpers still need capacity proportional to their + inputs within the documented limits. Coordinate production acceptance with + #254: at least two continuous hours in one process epoch, the specified risky + workloads, stable ownership after drain, and unchanged raw lease-failure + counters. This release does not raise lease thresholds, suppress alerts, or + change mining and payout rules. + +Historical release notes remain in `doc/`. diff --git a/docker/ckpool/ckpool-version-mask.py b/docker/ckpool/ckpool-version-mask.py index c45e9c774..a484ae8e7 100644 --- a/docker/ckpool/ckpool-version-mask.py +++ b/docker/ckpool/ckpool-version-mask.py @@ -3,11 +3,15 @@ from __future__ import annotations +import argparse import base64 +import http.client import json +import math import os import re import sys +import time from dataclasses import dataclass from typing import Any from urllib import error, request @@ -18,6 +22,19 @@ STATIC_MODES = {"0", "false", "no", "off", "static", "configured", "manual"} QBIT_VERSION_ROLLING_MASK = 0x1FFFE000 QBIT_VERSION_ROLLING_MASK_HEX = f"{QBIT_VERSION_ROLLING_MASK:08x}" +DEFAULT_PROBE_ATTEMPTS = 3 +DEFAULT_PROBE_RETRY_SECONDS = 2.0 +MAX_PROBE_ATTEMPTS = 10 +MAX_CONFIGURED_PROBE_SECONDS = 60 + + +class ProbeError(RuntimeError): + """Raised when the live getblocktemplate probe did not produce a template. + + Dynamic mode fails closed on this: the configured fallback mask describes + an operator intent, not what the node actually accepts, so a template that + could not be retrieved must never be papered over with it. + """ @dataclass(frozen=True) @@ -55,6 +72,10 @@ def mode_is_dynamic(mode: str) -> bool: ) +def chain_name() -> str: + return os.environ.get("QBIT_CHAIN", "regtest").strip().lower() or "regtest" + + def gbt_rules(chain: str) -> list[str]: rules = ["segwit"] if chain.strip().lower() == "signet": @@ -62,13 +83,21 @@ def gbt_rules(chain: str) -> list[str]: return rules -def select_version_mask(template: dict[str, Any], fallback_mask: str) -> ResolveResult: +def configured_mask(fallback_mask: str) -> str: try: - fallback = normalize_mask(fallback_mask, field="CKPOOL_VERSION_MASK") + return normalize_mask(fallback_mask, field="CKPOOL_VERSION_MASK") except ValueError as exc: raise ValueError(f"invalid fallback CKPOOL_VERSION_MASK: {exc}") from exc + +def select_version_mask(template: dict[str, Any], fallback_mask: str) -> ResolveResult: + fallback = configured_mask(fallback_mask) + if "versionrollingmask" not in template: + # A template that a ready node produced without the field describes a + # node that does not advertise one. That is a node-version + # compatibility case, not a failed probe, so the configured mask still + # applies. return ResolveResult(fallback, "fallback", "missing_versionrollingmask") advertised = template.get("versionrollingmask") @@ -78,6 +107,8 @@ def select_version_mask(template: dict[str, Any], fallback_mask: str) -> Resolve raise ValueError(f"invalid getblocktemplate.versionrollingmask: {exc}") from exc if selected == "00000000": + # Zero is a valid answer: the node is telling the pool that version + # rolling is disabled. Honour it rather than substituting a fallback. return ResolveResult(selected, "qbit_getblocktemplate", "disabled_by_zero_mask", str(advertised)) return ResolveResult(selected, "qbit_getblocktemplate", "advertised", str(advertised)) @@ -101,9 +132,27 @@ def rpc_getblocktemplate(*, host: str, port: str, user: str, password: str, chai "Content-Type": "application/json", }, ) - with request.urlopen(req, timeout=timeout) as resp: - body = json.load(resp) - + try: + with request.urlopen(req, timeout=timeout) as resp: + body = json.load(resp) + except error.HTTPError as exc: + if exc.code in (401, 403): + exc.close() + raise ValueError(f"RPC authentication refused (HTTP {exc.code}); check RPC credentials") from exc + # qbitd reports JSON-RPC errors with an HTTP 500 and the reason in the + # body. Surface that reason so a fail-closed startup says why. + try: + body = json.load(exc) + except (OSError, ValueError): + raise RuntimeError(f"HTTP {exc.code}: {exc.reason}") from exc + finally: + exc.close() + if isinstance(body, dict) and body.get("error"): + raise RuntimeError(body["error"]) from exc + raise RuntimeError(f"HTTP {exc.code}: {exc.reason}") from exc + + if not isinstance(body, dict): + raise RuntimeError("getblocktemplate response was not an object") if body.get("error"): raise RuntimeError(body["error"]) result = body.get("result") @@ -112,38 +161,127 @@ def rpc_getblocktemplate(*, host: str, port: str, user: str, password: str, chai return result -def resolve_from_env() -> ResolveResult: +def probe_settings() -> tuple[float, int, float]: + def number(name: str, default: int | float, parse: Any) -> Any: + raw = os.environ.get(name, "") or str(default) + try: + return parse(raw) + except ValueError as exc: + raise ValueError(f"{name} has an invalid numeric value: {raw!r}") from exc + + timeout = number("CKPOOL_VERSION_MASK_RPC_TIMEOUT_SECONDS", 5, float) + attempts = number("CKPOOL_VERSION_MASK_PROBE_ATTEMPTS", DEFAULT_PROBE_ATTEMPTS, int) + retry_seconds = number("CKPOOL_VERSION_MASK_PROBE_RETRY_SECONDS", DEFAULT_PROBE_RETRY_SECONDS, float) + if not math.isfinite(timeout) or timeout <= 0: + raise ValueError("CKPOOL_VERSION_MASK_RPC_TIMEOUT_SECONDS must be finite and positive") + if not 1 <= attempts <= MAX_PROBE_ATTEMPTS: + raise ValueError(f"CKPOOL_VERSION_MASK_PROBE_ATTEMPTS must be between 1 and {MAX_PROBE_ATTEMPTS}") + if not math.isfinite(retry_seconds) or retry_seconds < 0: + raise ValueError("CKPOOL_VERSION_MASK_PROBE_RETRY_SECONDS must be finite and nonnegative") + if attempts * timeout + (attempts - 1) * retry_seconds > MAX_CONFIGURED_PROBE_SECONDS: + raise ValueError(f"configured version-mask probe timeouts and delays must total at most {MAX_CONFIGURED_PROBE_SECONDS} seconds") + + return timeout, attempts, retry_seconds + + +def probe_template(*, sleep: Any = time.sleep) -> dict[str, Any]: + """Fetch a live template, retrying a bounded number of transient failures. + + Missing RPC credentials are a configuration fault and are not retried. + """ + timeout, attempts, retry_seconds = probe_settings() + try: + user = os.environ["QBIT_RPC_USER"] + password = os.environ["QBIT_RPC_PASSWORD"] + except KeyError as exc: + raise ValueError(f"{exc.args[0]} is required to resolve the version mask") from exc + + last_error: Exception | None = None + for attempt in range(1, attempts + 1): + try: + return rpc_getblocktemplate( + host=os.environ.get("QBIT_RPC_HOST", "qbitd"), + port=os.environ.get("QBIT_RPC_PORT", "18452"), + user=user, + password=password, + chain=chain_name(), + timeout=timeout, + ) + except (OSError, RuntimeError, json.JSONDecodeError, error.URLError, http.client.HTTPException) as exc: + last_error = exc + if attempt >= attempts: + break + print( + "ckpool version mask: getblocktemplate probe failed " + f"(attempt {attempt}/{attempts}): {exc}", + file=sys.stderr, + ) + if retry_seconds: + sleep(retry_seconds) + + raise ProbeError( + f"getblocktemplate probe failed after {attempts} attempt(s): {last_error}. " + "Dynamic mode requires a live template. If templates are intentionally " + "unavailable during prelaunch, explicitly set CKPOOL_VERSION_MASK_MODE=static." + ) + + +def validate_config() -> str: + """Validate mask configuration without contacting the node. + + Lets startup reject a malformed mode or configured mask before it spends + the readiness wait, without resolving a mask that early. + """ fallback = os.environ.get("CKPOOL_VERSION_MASK", QBIT_VERSION_ROLLING_MASK_HEX) mode = os.environ.get("CKPOOL_VERSION_MASK_MODE", "dynamic") - if not mode_is_dynamic(mode): - selected = normalize_mask(fallback, field="CKPOOL_VERSION_MASK") - return ResolveResult(selected, "fallback", "static_mode") + dynamic = mode_is_dynamic(mode) + configured = configured_mask(fallback) + if dynamic: + probe_settings() + return f"mode={'dynamic' if dynamic else 'static'} configured={configured}" - try: - timeout = float(os.environ.get("CKPOOL_VERSION_MASK_RPC_TIMEOUT_SECONDS", "5")) - except ValueError: - timeout = 5.0 - try: - template = rpc_getblocktemplate( - host=os.environ.get("QBIT_RPC_HOST", "qbitd"), - port=os.environ.get("QBIT_RPC_PORT", "18452"), - user=os.environ["QBIT_RPC_USER"], - password=os.environ["QBIT_RPC_PASSWORD"], - chain=os.environ.get("QBIT_CHAIN", "regtest"), - timeout=timeout, - ) - except (KeyError, OSError, RuntimeError, json.JSONDecodeError, error.URLError) as exc: - selected = normalize_mask(fallback, field="CKPOOL_VERSION_MASK") - return ResolveResult(selected, "fallback", f"probe_error:{exc}") +def resolve_from_env(*, sleep: Any = time.sleep) -> ResolveResult: + validate_config() + fallback = os.environ.get("CKPOOL_VERSION_MASK", QBIT_VERSION_ROLLING_MASK_HEX) + mode = os.environ.get("CKPOOL_VERSION_MASK_MODE", "dynamic") + if not mode_is_dynamic(mode): + return ResolveResult(configured_mask(fallback), "fallback", "static_mode") + template = probe_template(sleep=sleep) return select_version_mask(template, fallback) -def main() -> int: +def build_parser() -> argparse.ArgumentParser: + parser = argparse.ArgumentParser( + description="Resolve the ckpool BIP310 version rolling mask for qbit." + ) + parser.add_argument( + "--validate-config", + action="store_true", + help=( + "validate mask mode, configured mask and dynamic probe settings, " + "without contacting qbit; prints nothing on stdout" + ), + ) + return parser + + +def main(argv: list[str] | None = None) -> int: + args = build_parser().parse_args(argv) + + if args.validate_config: + try: + summary = validate_config() + except ValueError as exc: + print(f"ckpool version mask: error={exc}", file=sys.stderr) + return 1 + print(f"ckpool version mask: config ok {summary}", file=sys.stderr) + return 0 + try: result = resolve_from_env() - except ValueError as exc: + except (ValueError, ProbeError) as exc: print(f"ckpool version mask: error={exc}", file=sys.stderr) return 1 diff --git a/docker/ckpool/start-ckpool.sh b/docker/ckpool/start-ckpool.sh index c1dcd5e6a..cac48ead8 100644 --- a/docker/ckpool/start-ckpool.sh +++ b/docker/ckpool/start-ckpool.sh @@ -26,6 +26,8 @@ fi : "${CKPOOL_VERSION_MASK:=1fffe000}" : "${CKPOOL_VERSION_MASK_MODE:=dynamic}" : "${CKPOOL_VERSION_MASK_RPC_TIMEOUT_SECONDS:=5}" +: "${CKPOOL_VERSION_MASK_PROBE_ATTEMPTS:=3}" +: "${CKPOOL_VERSION_MASK_PROBE_RETRY_SECONDS:=2}" : "${QBIT_MINER_WALLET_NAME:=ckpool}" : "${CKPOOL_BIN:=/usr/local/bin/ckpool}" : "${CKPOOL_CONFIG_FILE:=/etc/ckpool/ckpool.conf}" @@ -61,6 +63,7 @@ fi export QBIT_RPC_USER QBIT_RPC_PASSWORD QBIT_RPC_HOST QBIT_RPC_PORT QBIT_CHAIN QBIT_MINER_WALLET_NAME export QBIT_PRODUCTION QBIT_TOOLS_PRODUCTION CKPOOL_STRATUM_PORT export CKPOOL_VERSION_MASK CKPOOL_VERSION_MASK_MODE CKPOOL_VERSION_MASK_RPC_TIMEOUT_SECONDS +export CKPOOL_VERSION_MASK_PROBE_ATTEMPTS CKPOOL_VERSION_MASK_PROBE_RETRY_SECONDS export CKPOOL_PUBLIC_DIFF_POLICY CKPOOL_NON_TEST_READINESS_GATE CKPOOL_MIN_PEERS export CKPOOL_PREFLIGHT_RPC_TIMEOUT_SECONDS CKPOOL_PREFLIGHT_READINESS_TIMEOUT_SECONDS export CKPOOL_TEMPLATE_MAX_AGE_SECONDS CKPOOL_TEMPLATE_MAX_FUTURE_SECONDS @@ -255,8 +258,10 @@ if [[ "${SUPERVISED_CHILD}" == "0" ]]; then # state, or configuration creation can hide how values were supplied. qbit-ckpool-preflight --production-gate-only - CKPOOL_VERSION_MASK="$(ckpool-version-mask)" - export CKPOOL_VERSION_MASK + # Reject a malformed mask mode or configured mask now, without an RPC probe. + # The mask itself is resolved in the supervised child so that it reflects a + # template from a node the readiness gate has already accepted. + ckpool-version-mask --validate-config if [[ "${QBIT_MINER_ADDRESS_FILE_WAIT}" == "1" ]]; then for _ in $(seq 1 30); do @@ -273,6 +278,14 @@ if [[ "${SUPERVISED_CHILD}" == "0" ]]; then fi : "${QBIT_MINER_ADDRESS:?QBIT_MINER_ADDRESS is required in supervised child}" + +# The supervisor only starts this child once its readiness gate and live +# template checks have passed, so a dynamic probe here reads the mask the node +# actually advertises instead of freezing a fallback chosen during cold start. +# Dynamic mode fails closed, which aborts startup before any config is written. +CKPOOL_VERSION_MASK="$(ckpool-version-mask)" +export CKPOOL_VERSION_MASK + mkdir -p "$(dirname "${CKPOOL_CONFIG_FILE}")" "${CKPOOL_LOG_DIR}" "${CKPOOL_SOCK_DIR}" "${CKPOOL_STATE_DIR}" mkdir -p "$(dirname "${QBIT_MINER_ADDRESS_FILE}")" printf '%s\n' "${QBIT_MINER_ADDRESS}" > "${QBIT_MINER_ADDRESS_FILE}" diff --git a/docs/README.md b/docs/README.md index 30b28c94a..b71f33a90 100644 --- a/docs/README.md +++ b/docs/README.md @@ -16,6 +16,12 @@ references. For public-facing navigation, prefer the short path below. API ownership boundary and endpoint conventions. - [public-dashboard-api-v1.openapi.yaml](public-dashboard-api-v1.openapi.yaml): OpenAPI contract for `/public/v1`. +- [prism-public-operator-endpoint-split.md](prism-public-operator-endpoint-split.md): + every HTTP endpoint classified as public read, operator, or payout-affecting, + with the evidence behind each call. +- [prism-postgres-replica.md](prism-postgres-replica.md): provisioning and + operating the hot standby the public read tier answers from, including + replication slot management and the staleness bound it enforces. - [prism-storage-sizing.md](prism-storage-sizing.md): storage, VM sizing, artifact retention, and monitoring guidance. - [mainnet-deployment.md](mainnet-deployment.md): fail-closed production @@ -33,8 +39,15 @@ This is useful for reviewers and operators who need implementation detail: - [prism-ledger-ops.md](prism-ledger-ops.md): formal ledger invariants, writer-lease behavior, compaction contract, and readiness probes. +- [prism-overload-alerts.md](prism-overload-alerts.md): specification for + the PRISM overload and backlog alert rules (issue #188), including signal + inventory, thresholds and their provenance, and the ordering dependency on + #184. Not live: this repository holds no monitoring configuration. - [prism-capacity-readiness.md](prism-capacity-readiness.md): optional Stratum-to-Postgres qualification artifact and standalone validator contract. +- [prism-coordinator-refactor/README.md](prism-coordinator-refactor/README.md): + completed coordinator decomposition, ownership map, validation evidence, and + stacked-PR publication plan. ## Public-Site Guidance diff --git a/docs/mainnet-deployment.md b/docs/mainnet-deployment.md index 20e52b23d..adb971a8c 100644 --- a/docs/mainnet-deployment.md +++ b/docs/mainnet-deployment.md @@ -40,10 +40,7 @@ parent-node release pins. Validation runs in two tiers. The behavioral production checks (non-default credentials, explicit payout addresses, strict difficulty and readiness policies, commit-pinned sources) apply whenever `QBIT_PRODUCTION=1`, -`QBIT_TOOLS_PRODUCTION=1`, or `QBIT_CHAIN=mainnet`; zero stale grace -(`PRISM_STRATUM_STALE_GRACE_SECONDS=0`) is pinned only on mainnet, so a public -test-chain pool may credit shares that raced a block within a bounded grace -window. The +`QBIT_TOOLS_PRODUCTION=1`, or `QBIT_CHAIN=mainnet`. The release-provenance checks (digest-qualified `*_IMAGE` references and absolute `*_DATA_SOURCE` host paths) are enforced unconditionally on mainnet — no environment variable can disable them there — and on other chains only with @@ -74,6 +71,7 @@ QBIT_DATA_SOURCE=/srv/qbit-mining-bootstrap/mainnet/qbit BITCOIN_DATA_SOURCE=/srv/qbit-mining-bootstrap/mainnet/bitcoin PRISM_POSTGRES_DATA_SOURCE=/srv/qbit-mining-bootstrap/mainnet/postgres/data PRISM_POSTGRES_WAL_SOURCE=/srv/qbit-mining-bootstrap/mainnet/postgres/wal +PRISM_POSTGRES_REPLICA_DATA_SOURCE=/srv/qbit-mining-bootstrap/mainnet/postgres-replica/data PRISM_AUDIT_DATA_SOURCE=/srv/qbit-mining-bootstrap/mainnet/prism/audit ``` @@ -81,6 +79,11 @@ Create each directory before rendering Compose, then grant only the correspondin runtime the required access. Compose refuses to create missing production bind paths. +For the PRISM lane, release-provenance validation requires an absolute replica +data path distinct from the primary data, WAL, audit, and other enabled state +sources before contacting Docker. See [the replica runbook](prism-postgres-replica.md) +for provisioning the public read standby. + `PRISM_POSTGRES_WAL_SOURCE` is the primary's live WAL directory. Separating it from `PGDATA` permits an independent capacity and I/O boundary only when the two paths are backed by different mounts or devices. Two directories on one @@ -137,7 +140,7 @@ docker compose \ --profile permissionless \ --profile auxpow \ --profile prism \ - config --quiet qbitd ckpool bitcoind auxpow-stratum prism-postgres prism-coordinator + config --quiet qbitd ckpool bitcoind auxpow-stratum prism-postgres prism-coordinator prism-public-api ``` Pull reviewed artifacts before stopping the prior release. Start operator @@ -151,7 +154,7 @@ docker compose \ --env-file "$DEPLOY_ENV_FILE" \ -f compose.yaml \ -f compose.production.yaml \ - pull qbitd ckpool bitcoind auxpow-stratum prism-postgres prism-coordinator + pull qbitd ckpool bitcoind auxpow-stratum prism-postgres prism-coordinator prism-public-api docker compose \ --project-name "$COMPOSE_PROJECT_NAME" \ @@ -186,7 +189,7 @@ docker compose \ --env-file "$DEPLOY_ENV_FILE" \ -f compose.yaml \ -f compose.production.yaml \ - up -d --no-build --pull never ckpool auxpow-stratum prism-coordinator + up -d --no-build --pull never ckpool auxpow-stratum prism-coordinator prism-public-api ``` Remove services for lanes that are not enabled. The `make @@ -234,7 +237,7 @@ Compose contract: | Lane | Operator services | | --- | --- | | CKPool solo | `qbitd ckpool` | -| PRISM | `qbitd prism-postgres prism-coordinator` | +| PRISM | `qbitd prism-postgres prism-coordinator prism-public-api` | | AuxPoW Stratum | `qbitd bitcoind auxpow-stratum` | `permissionless-miner`, `real-miner`, `auxpow-real-miner`, and the one-shot @@ -269,7 +272,7 @@ seconds, and the default maximum future template time is 30 seconds. so CKPool cannot publish one job beyond that freshness bound. The only relaxed mode is explicitly authorized mainnet prelaunch. It requires -all five values below; any missing, invalid, or mismatched value fails closed: +the authorization values and mask policy below; any missing, invalid, or mismatched value fails closed: ```bash QBIT_CHAIN=mainnet @@ -277,12 +280,16 @@ QBIT_PRODUCTION=1 QBIT_TOOLS_PRODUCTION=1 CKPOOL_NON_TEST_READINESS_GATE=0 QBIT_MAINNET_LAUNCH_READINESS_CHECKS_ENABLED=0 +CKPOOL_VERSION_MASK_MODE=static +CKPOOL_VERSION_MASK=1fffe000 ``` Prelaunch still checks static policy, chain and mandatory genesis identity, and the explicit payout address. It defers IBD, peer, GBT, freshness, and active-tip -checks so CKPool can bind its listener and retry GBT while qbitd starts. At -launch, set both readiness flags to `1` and restart or redeploy CKPool. A running +checks so CKPool can bind its listener and retry GBT while qbitd starts, using +the explicitly configured static mask. Dynamic mode requires a successful +template even with these prelaunch flags. At launch, set both readiness flags +to `1`, restore `CKPOOL_VERSION_MASK_MODE=dynamic`, and restart or redeploy CKPool. A running supervisor does not hot-reload environment changes. ## AuxPoW Gate @@ -333,9 +340,15 @@ Keep Postgres `fsync`, `full_page_writes`, and `synchronous_commit` enabled. Measure acknowledgment latency under representative accepted-share load rather than weakening database durability. -Production requires `PRISM_STRATUM_STALE_GRACE_SECONDS=0` until every published -audit consumer has demonstrated compatibility with stale-grace receipts. -`PRISM_TEMPLATE_REFRESH_FAILURE_EXIT_SECONDS` similarly bounds a persistent +Keep `PRISM_STRATUM_STALE_GRACE_SECONDS` at its bounded default (3 seconds) +on every chain, mainnet included. At zero, every in-flight prior-tip share is +rejected with Stratum error 21 at each block, which miners read as pool +failure rather than as stale work. Reward windows that contain a credited +prior-tip share publish the logical `qbit.prism.audit-bundle.v1.1` schema; +this release's verifier accepts it, so keep external mirrors and verifiers on +a release that does too. + +`PRISM_TEMPLATE_REFRESH_FAILURE_EXIT_SECONDS` bounds a persistent PRISM template-refresh outage; it must be shorter than the operator alert and response window. Coordination-only refresh deferrals remain outside that ordinary failure budget, but only for @@ -356,12 +369,20 @@ client (`PRISM_POSTGRES_NATIVE_CLIENT=auto` or `1`). Each coordinator holds a dedicated PostgreSQL advisory guard for its writer ID and epoch and periodically proves that isolated session live with a non-blocking check (the session answers, still holds the advisory lock, and the committed lease row -still names it). The heartbeat never waits on the lease tuple's row lock: -fenced writes hold it for entire transactions, and accepted-block persistence -can legitimately exceed the guard's statement timeout. It renews the lease TTL -only while that tuple is uncontended (`SKIP LOCKED`), so an idle coordinator — -no fenced writes and the CTV broadcaster disabled — still keeps its lease from -expiring under a different writer identity's expiry claim. If the committed row +still names it). Most beats run only that cheap read-only ownership proof: one +statement, no row lock and no `pg_stat_activity` scan. The full renewing +verification runs on the first beat and thereafter on any beat whose proof +reports the committed lease row inside the own-write authority margin (at or +above half the lease TTL), escalating on that same beat. So renewal is never +later than it was before the split, and for the whole of a long own fenced +write — when renewals are actually being skipped — every beat runs the same +fail-closed verification as before. The heartbeat never waits on the lease +tuple's row lock: fenced writes hold it for entire transactions, and +accepted-block persistence can legitimately exceed the guard's statement +timeout. It renews the lease TTL only while that tuple is uncontended +(`SKIP LOCKED`), so an idle coordinator — no fenced writes and the CTV +broadcaster disabled — still keeps its lease from expiring under a different +writer identity's expiry claim. If the committed row is already expired and the renewal was lock-blocked, verification fails closed: the skipped lock may be an in-flight expiry claim, and a stale committed token read is not proof of liveness. The one exemption is a lock `pg_stat_activity` @@ -385,6 +406,77 @@ guard, logs that fast adoption is disabled, and conservatively falls back to the configured lease TTL. Keep generated session tokens in production; fixed tokens remain a local test-only facility. +#### Heartbeat timing is one validated policy + +The adoption silence, heartbeat interval, failure budget, monitor interval and +exit margin are terms of a single inequality — the old coordinator must be gone +before a replacement may compare-and-swap the lease row — owned by +`lab/prism/writer_lease_timing.py`. The defaults are derived from measurable +phase budgets rather than tuned individually: + +| term | value | where it comes from | +| --- | --- | --- | +| guard statement timeout | 0.50s | the guard session's server-side `statement_timeout` | +| heartbeat interval | 0.25s | idle gap between ownership proofs | +| scheduler slack | 0.50s | process-side scheduling delay, applied to both the heartbeat's stamping path and the monitor's own poll | +| failure budget | 1.25s | interval + statement timeout + slack | +| monitor interval | 0.05s | staleness poll | +| exit margin | 0.10s | hard-exit budget | +| exit envelope | 0.70s | exit margin + 2 x monitor interval + scheduler slack | +| adoption silence | 2.00s | must exceed 1.25 + 0.70 = 1.95s | +| server-proven cap | 1.30s | adoption silence - exit envelope | + +The exit envelope reserves the monitor's *own* lateness, not just its poll +granularity. The monitor is a Python thread in the same process, behind the +same GIL, as the heartbeat whose stalls the policy explicitly budgets for, so +it cannot be assumed to wake on time when the heartbeat does not. At the +shipped numbers, with a maximally late monitor: + +``` +1.30 (cap) + 0.05 (poll) + 0.50 (monitor lateness) + 0.10 (exit) = 1.95 < 2.00 +``` + +leaving one monitor poll of strict reserve before a successor may +compare-and-swap. + +Genuine writer failover therefore costs 2.0 seconds of adoption silence rather +than 1.0. That is the price of the guard session's own statement timeout: the +monitor can only trust completed round trips, so the envelope has to contain +one whole statement plus one idle interval plus process scheduling — and then, +separately, room for the monitor itself to be that late in noticing. It +replaces the temporary downstream 4.0-second pin. + +A combination that breaks the safety inequality is now **refused at startup**: +the coordinator hard-exits with the failing term named instead of printing a +warning and running with no exit-before-adoption guarantee. A combination that +is safe but leaves the staleness cap below the largest gap a healthy +coordinator can produce logs a `no tail-latency headroom` warning and starts — +that state cannot produce two writers, it only produces avoidable restarts. + +#### Reading a heartbeat exit + +Every heartbeat hard exit names which bound tripped (`Tripped: activity ... >= +failure budget` or `... >= adoption envelope cap`), the phase breakdown of the +last and worst verification attempts (`slot_wait` / `guard_sql` / `scheduler` +/ `total`), the monitor's own worst wake delay, and every term of the policy +that produced the decision. Guard-slot queueing, database round trips and +Python scheduling stalls are attributed separately, so an exit says whether to +look at PostgreSQL or at the coordinator process. + +The same attribution is exported with fixed metric cardinality: +`qbit_prism_lease_heartbeat_attempts_total{mode}` (`proof`, `renew`, `fence`), +`qbit_prism_lease_heartbeat_outcomes_total{outcome}`, +`qbit_prism_lease_heartbeat_phase_seconds{phase}` and its +`worst_phase_seconds` companion, +`qbit_prism_lease_heartbeat_activity_age_seconds`, +`qbit_prism_lease_heartbeat_server_proven_age_seconds`, +`qbit_prism_lease_heartbeat_monitor_wake_delay_seconds`, and +`qbit_prism_lease_heartbeat_policy_seconds{term}`. Alert on +`qbit_prism_lease_heartbeat_server_proven_age_seconds` approaching +`qbit_prism_lease_heartbeat_policy_seconds{term="server_proven_cap"}`, and on +`qbit_prism_lease_heartbeat_policy_seconds{term="stability_surplus"}` falling +to or below zero. + Before each mutating qbitd or wallet RPC, the coordinator performs the same bounded non-blocking exact-session verification on that advisory-guard connection, so the fence never contends with an in-flight accepted-block @@ -402,6 +494,80 @@ especially during a PostgreSQL restart or network partition. Strict elimination of that residual window requires qbitd/wallet RPCs to validate a fencing generation supplied by the coordinator. +### PRISM Allocator Settings + +The PRISM image sets `MALLOC_ARENA_MAX=2` (issue #226). This is a deliberate +allocator decision, recorded here so that a future change to it is a decision +too; `tests/test_prism_heap_census.py` pins the image's allocator environment. + +Why: on `union-mainnet` at pin `9c41894` the coordinator's resident set grew by +roughly 390 MB per hour across 45 hours of uptime with no restart, while the +process ran about 64 threads and its memory map was 943 anonymous regions, 658 +of them between 4 and 64 MiB. That band is the glibc per-thread malloc arena +signature (each arena grows in 64 MiB heaps), and glibc's default arena limit is +eight times the core count, 256 on the 32-core host, so every thread that ever +allocates keeps its own arena and its own retained free lists. Two arenas bound +that multiplication. The storm instrument reproduces the mechanism on any host +(`docs/prism-capacity-readiness.md`, "Allocator settings"): at the default a +64-thread probe creates 64 arenas and the 4-64 MiB region count climbs to the +mainnet shape; at `MALLOC_ARENA_MAX=2` the arena count stays at 2, and +`malloc_trim` can return the freed memory, which with 64 arenas it cannot, +because a per-thread heap's top is not something `malloc_trim` shrinks. + +Why it is safe before the testnet soak decides between 2 and 4: under the GIL +at most one Python thread allocates at a time, so arena-lock contention can only +come from native code that runs with the GIL released, which in this process is +libpq I/O on a handful of pooled connections; the audit-builder daemon the +coordinator spawns inherits the variable but is single-threaded; and the +failure mode of too few arenas is a measurable wait on share-ack latency +(`qbit_prism_share_ack_seconds`), whereas the failure mode of too many is the +17.7 GB resident set. The local storm drain ran at the same wall-clock under +the default, 2, and 4 on the development host. + +Override it per deployment rather than editing the image: + +```dotenv +MALLOC_ARENA_MAX=4 +``` + +How that value reaches the process: the `prism-coordinator` service in +`compose.yaml` passes it through as `MALLOC_ARENA_MAX: ${MALLOC_ARENA_MAX:-2}`. +Compose has no `env_file:` for this service, so a dotenv value reaches the +container only through that line; the image's `ENV` is what applies when the +dotenv leaves the variable unset, and the passthrough's default is pinned equal +to the image default by `tests/test_prism_heap_census.py` so the two cannot +drift. A deployment that sets the coordinator's environment by some other +means must set the variable there. The public read service +(`prism-public-api`) runs from the same image and keeps the image default; it +is not part of the experiment. At build time `--build-arg MALLOC_ARENA_MAX=4` +changes the image default. Do not set it to `1`: that forces every thread onto +the main arena, whose `brk` heap shrinks only from its end. Confirm the +setting actually applied before reading any arena gauge: a census report's +`process.malloc_arena_count` cannot exceed the cap, and +`docker exec sh -c 'echo $MALLOC_ARENA_MAX'` +shows what the process started with. + +Do not set any other `MALLOC_*` variable, `GLIBC_TUNABLES`, or `PYTHONMALLOC` +in production without a soak. In particular never set an allocator variable to +an empty value: glibc reads `""` as `0`, and `MALLOC_MMAP_THRESHOLD_=0` would +route every allocation through `mmap`. `PYTHONMALLOC` stays unset so pymalloc +keeps objects of 512 bytes and under out of glibc; routing them through glibc +would enlarge the arenas this setting bounds. + +The heap census (`PRISM_HEAP_CENSUS`), its `tracemalloc` tracing +(`PRISM_HEAP_CENSUS_TRACEMALLOC`), the `malloc_trim` signal +(`PRISM_MALLOC_TRIM_SIGNAL`) and the periodic trim +(`PRISM_MALLOC_TRIM_INTERVAL_SECONDS`) are all off in production by default and +stay off unless an operator turns one on for a diagnosis window. Each is a +strict boolean or a floored interval validated at startup, each passes through +`compose.yaml` to the coordinator service (so a dotenv value reaches the +process), and `.env.example` lists them; the census writes under +`PRISM_AUDIT_DIR/heap-census` by default. Read +`docs/prism-capacity-readiness.md`, "Heap census, allocator control, and the +resident-set bound", before enabling any of them: the census holds the GIL for +the duration of one `gc.get_objects()` call, and `tracemalloc` costs every +allocation for as long as it traces. + ### PRISM Watchdog Exit Forensics The PRISM image has no entrypoint or init wrapper: its exec-form Dockerfile diff --git a/docs/prism-candidate-storage.md b/docs/prism-candidate-storage.md new file mode 100644 index 000000000..cb2abbad4 --- /dev/null +++ b/docs/prism-candidate-storage.md @@ -0,0 +1,391 @@ +# Bounded candidate persistence and replay + +This guide covers the 2.x.x work for [#255](https://github.com/Qbit-Org/qbit-mining-bootstrap/issues/255) +and its release dependency on [#254](https://github.com/Qbit-Org/qbit-mining-bootstrap/issues/254). +It includes the complete cursor enumeration introduced by PR #256 and the +offline accepted-pending recovery command introduced by PR #259. + +## Storage and resource settings + +Apply `001_share_ledger.sql` followed by `002_candidate_bodies.sql` before +starting this release, whichever storage version it writes. +`PRISM_POSTGRES_INIT_SCHEMA=1` performs this at startup; the migrations are +idempotent. Replay headers and terminal outbox statements use the migrated +columns for both storage versions, so the coordinator and the offline recovery +command refuse a database without `002`; `PRISM_CANDIDATE_STORAGE_VERSION=1` +stops version 2 writes but does not make an unmigrated database usable. +Existing version 1 JSONB outbox rows remain intact +and readable. New version 2 rows reference a sealed body and carry a small replay +header; the old `candidate` column is null for those rows. + +| Setting or boundary | Default / limit | +| --- | --- | +| `PRISM_CANDIDATE_STORAGE_VERSION` | `2`; set `1` for compatible-reader rollout or rollback | +| `PRISM_CANDIDATE_SPOOL_DIR` | Empty selects the system temporary directory; choose a persistent-capacity mount for large workloads | +| `PRISM_CANDIDATE_SPOOL_RESERVATION_BYTES` | 4 GiB of admitted candidate body and index files per ledger instance | +| `PRISM_BLOCK_REPLAY_PAGE_SIZE` | 1,024 descriptors; supported range 1–1,024 | +| Replay header page | 256 KiB and the configured row cap | +| Stored chunk / read page | 256 KiB per chunk; at most three chunks per read | +| Index transfer | 256 page entries or 64 field spans per statement | +| Shared helper admission | One legacy, record-normalization or finalization helper at a time per process; deadline/cancellation also apply while awaiting admission | +| Legacy conversion / JSONB comparison helper | 4 GiB address-space limit, 600-second deadline, 64 KiB retained diagnostics | +| Oversized audit record helper | 4 GiB address-space limit, 300-second deadline, 1 MiB metadata response ceiling | +| Finalization SQL transport | 4 GiB statement spool, one helper under the writer gate, 4 GiB helper address space, 600 seconds or the shorter operation deadline | + +Replay retains at most 1,024 queued descriptors. When admission fills, enumeration +pauses before registering further credit floors or payout previews. The existing +enumeration-owed gate keeps job builds blocked until the queue drains and the +remaining durable rows are handled, including when the last SQL page was exhausted. + +Hydration limits ordinary native JSON decodes and the aggregate metadata skeleton +to 1 MiB. Larger arrays, objects and strings remain spool-backed views with disk +indexes. String decoding carries incomplete escapes and surrogate pairs across +slices; block submission streams its encoded body with an exact content length. +Consensus-bounded metadata needed by existing consumers is decoded in slices and +joined explicitly. This is a bound on coordinator work units, not a claim that +arbitrary metadata or PostgreSQL/helper memory is independent of input size. + +The codec preserves the historical identity bytes, including neutralizing only +the pending share's acknowledgment timestamp. Append retries compare the digest +and the actual body, using isolated JSONB comparison when different encodings +may be equivalent. Standalone intent persistence retains its historical +digest-only retry rule. Staging and sealing grant no share credit; publication +and credit remain one synchronous, fenced commit. + +Candidate spool reservations include the disk index and are released explicitly +on close and failed hydration. Disk exhaustion and helper capacity/deadline +failures leave valid durable candidates pending for retry. They must not be +classified as corrupt bodies. Monitor the spool filesystem and container memory +separately: reservations do not bound filesystem page cache. + +Finalization retains the existing atomic SQL. Its text streams to a temporary +file; an isolated `psql` process owns the complete statement buffer. PostgreSQL +still parses that complete statement, so server memory and temporary-file +capacity remain size-dependent. This transport preserves the native ledger's +exact DSN, private schema options, writer-session guards and operation deadlines. +The fixed helper limits are implementation constants, not environment settings. + +## Problem and evidence + +A streamed candidate identity hash did not bound the surrounding pipeline. +[Candidate construction](https://github.com/Qbit-Org/qbit-mining-bootstrap/blob/d39280070517a1bd66e09d5cfe8cd330a2be1f2f/lab/prism/block_candidates.py#L1145) +materialized the share sequence and validated the entire intent with `json.dumps`; +[append](https://github.com/Qbit-Org/qbit-mining-bootstrap/blob/d39280070517a1bd66e09d5cfe8cd330a2be1f2f/lab/prism/share_ledger.py#L4223) +subsequently encoded the nested intent through `_jsonb_literal`; +[startup replay](https://github.com/Qbit-Org/qbit-mining-bootstrap/blob/d39280070517a1bd66e09d5cfe8cd330a2be1f2f/lab/prism/share_ledger.py#L4580) +decoded full intents inside a single `json_agg`. +Limiting candidate count, or returning one row per candidate, still permits one +large native JSON decoder call. The downstream reproducibility check also read +an aggregate audit window and built a Python ID set, and canonical audit +finalization reparsed complete bundles. + +The original portable experiment used CPython 3.14.7, 375,000 synthetic shares +and approximately 232 MB per candidate. It reproduced long observer pauses with +cyclic GC disabled. This establishes the codec mechanism; it is neither a valid +mined-block fixture nor proof of the instruction responsible for a production +exit. Production runs x86_64. Recorded one-candidate replay and retirement remain +plausible contributors to the observed exits, which had little recorded GC time. + +The membership experiment below separately measures the downstream omission +check against PostgreSQL 16.15, psycopg 3.2.13 and CPython 3.14.7 in ARM64 Docker. +The fixture contains 375,000 actual ledger rows and runs a real writer-session +proof actor alongside the operation. Runs were serial; each result is one run, +not a distribution or a production capacity estimate. +The first runs used default allocator settings. Repeats with the coordinator's +`MALLOC_ARENA_MAX=2` policy are listed separately. + +| Phase | GC | Wall time | Largest observer delay | Process high-water RSS | +| --- | --- | ---: | ---: | ---: | +| Bounded membership | disabled | 0.449 s | 5.8 ms | 48 MiB | +| Prior aggregate read | disabled | 1.432 s | 309.9 ms | 903 MiB | +| Bounded membership | enabled | 0.465 s | 7.5 ms | 48 MiB | +| Prior aggregate read | enabled | 1.554 s | 352.5 ms | 903 MiB | +| Bounded membership, 1,024 rows | disabled | 0.009 s | 1.6 ms | 48 MiB | +| Bounded membership, two arenas | disabled | 0.453 s | 7.3 ms | 48 MiB | +| Prior aggregate, two arenas | disabled | 1.487 s | 318.7 ms | 903 MiB | +| Bounded membership, two arenas | enabled | 0.482 s | 13.5 ms | 48 MiB | +| Prior aggregate, two arenas | enabled | 1.483 s | 350.3 ms | 903 MiB | + +After adding native COPY backpressure, fresh two-arena runs measured 0.453 s +(GC disabled) and 0.508 s (enabled), with maximum observer delays of 15.0 ms and +70.9 ms respectively and the same 48 MiB high-water RSS. Their aggregate +controls measured 320.7 ms and 323.9 ms observer delays and 903 MiB high-water +RSS. Maximum bounded-phase heartbeat wake lateness was 31.4/54.1 ms, with +102.9/76.2 ms proof calls. These single-run variations are recorded rather than +attributed to a production cause; they are not a worst-case latency guarantee. + +There were no proof errors. Maximum proof duration was 8.4 ms for the large +GC-disabled bounded phase versus 320.8 ms for the aggregate phase. GC-enabled +proof maxima were 23.8 ms and 357.7 ms. The enabled aggregate run also recorded +a 28.7 ms generation-0 collection. Destruction of its returned row graph took +25.8 ms separately. These measurements must not be added together to infer a +production pause, and RSS includes the interpreter and native client. +The improved harness also separates heartbeat wake lateness from proof-call +duration: in the two-arena GC-enabled run, the aggregate phase delayed a wake +by 298.4 ms even though its slowest proof call took only 6.0 ms. Bounded +membership had 9.9 ms wake lateness and a 42.6 ms proof-call maximum. A short +database round trip therefore does not by itself establish timely scheduling. + +A separate audit experiment writes a 231,202,664-byte JSON artifact containing +375,000 synthetic shares and scans, walks, streams, and retires its actual +`CanonicalAuditBundleView`. Three artifacts run sequentially; the baseline +performs one decode of their combined array, matching the aggregate boundary. +It uses CPython 3.14.7 ARM64 Docker and `MALLOC_ARENA_MAX=2`. + +| Audit phase (three candidates) | GC | Largest observer delay | High-water RSS | +| --- | --- | ---: | ---: | +| Streamed scan/walk/encode/retirement | disabled | 8.5 ms | 32 MiB | +| Whole aggregate decode | disabled | 1,449 ms | 2.15 GiB | +| Whole-graph reference-count retirement | disabled | 142 ms | — | +| Streamed scan/walk/encode/retirement | enabled | 21.3 ms | 32 MiB | +| Whole aggregate decode | enabled | 1,473 ms | 2.15 GiB | +| Whole-graph reference-count retirement | enabled | 139 ms | — | + +The GC-enabled aggregate recorded one 117 ms collection. Explicit view retirement +took less than 0.14 ms in each run and released both weakly observed owners before +cyclic GC. A separate traced single-candidate run peaked at 1.35 MB for scan, +1.33 MB for lazy iteration and 2.14 MB for encoding; traced wall times are excluded +from the timing comparison. A 1,024-share control also passes. This probe has an +observer thread, not a native lease actor; use the membership and full pipeline +qualification for heartbeat behavior. + +Independent PostgreSQL backpressure qualification also froze a disposable server +for six seconds during COPY. The default Linux `LibpqWriter` accepted 60 MiB while +frozen and grew client RSS by 49 MiB. Per-chunk flushing kept RSS flat and blocked +until the server resumed; both preserved all 3,532,045 rows. A trigger-throttled +consumer confirmed statement-timeout rollback, connection reuse and producer +cancellation. Unfrozen throughput was comparable (4.66–4.88 s default versus +4.81–4.87 s flushing). This measures native transport allocation separately from +Python JSON allocation. The committed slow-consumer regression fails with the +prior writer and passes with explicit flushing. + +## Native candidate pipeline qualification + +`tests/perf/candidate_storage.py` creates a private disposable PostgreSQL schema +and exercises the production prepare, append, exact retry, header enumeration, +hydration, lazy walk, close, terminalization and janitor paths. A real +writer-session proof actor runs beside each phase. The fixture supplies 375,000 +synthetic shares and 232,787,365 canonical bytes per candidate, under CPython +3.14.7, PostgreSQL 16, psycopg 3.2.13 and ARM64 Docker with `MALLOC_ARENA_MAX=2`. +These measurements follow the final spool-view integration and are separate +from the real mined-block gate. + +| Candidates | GC | Largest observer delay | High-water RSS | Largest proof call | Largest heartbeat wake lateness | +| --- | --- | ---: | ---: | ---: | ---: | +| 3 | disabled | 28.0 ms | 103 MiB | 5.3 ms | 23.8 ms | +| 3 | enabled | 5.9 ms | 101 MiB | 3.9 ms | 7.6 ms | +| 1 | disabled | 4.6 ms | 101 MiB | 3.9 ms | 6.7 ms | +| 3 × 20 generations, 1,024 shares | disabled | 11.1 ms | 59 MiB | 7.4 ms | 7.9 ms | + +All four runs passed without proof errors. Each explicit close returned spool +reservations to zero and released the weakly observed body owner. The larger +delay in the GC-disabled run is retained as observed; these serial single runs +are not a worst-case guarantee or production-equivalent timings. + +```sh +python tests/perf/candidate_storage.py \ + --database-url "$TEST_DATABASE_URL" --shares 375000 --candidates 3 --gc off +``` + +Repeat with `--gc on`, with `--candidates 1`, and with +`--shares 1024 --candidates 3 --generations 20 --gc off` for repeated ownership +retirement without cyclic GC. This harness creates and drops its own private +schema; supply a disposable database and run timing measurements serially. + +## Reproducibility check + +A replayed candidate is reproducible only when its recorded share IDs cover +every ID in the durable window at its declared anchor and difficulty. Order, +duplicate IDs, and extra recorded IDs do not change this historical subset +rule. Missing durable IDs still reject replay. The check does not substitute a +new ledger window for the recorded candidate window or grant share credit. + +`candidate_window.py` streams IDs through PostgreSQL COPY into a private +transactional temporary table, then returns one boolean from an exact text +anti-join. COPY writes are at most 16 KiB and flush the native output buffer before the +producer advances, with cancellation/deadline checks at +most 256 records apart. Oversized individual strings are escaped and encoded in +4,096-character pieces. PostgreSQL join memory can spill under an 8 MiB +`work_mem`; the operation uses a read slot and never takes the writer-lease +lock. Failure rolls back the temporary table, and the candidate's existing retry +starts from its replayable immutable source. The psql fallback spools its input +and caps captured diagnostic text at 64 KiB. In-memory ledger compatibility uses +an on-disk SQLite ID index with a 2 MiB page cache. Its digest index narrows +lookups, then compares the actual ID bytes from file spans; a digest collision +cannot satisfy membership. + +## Running qualification + +Use disposable databases only. The membership harness deliberately replaces +share rows in the supplied database. Initialize the ledger schema first, then +run in the same Python image as the coordinator: + +```sh +python tests/perf/candidate_window_membership.py \ + --database-url "$TEST_DATABASE_URL" --shares 375000 --gc off \ + --prepare-disposable-fixture +``` + +For the standalone audit experiment: + +```sh +python tests/perf/canonical_audit_retirement.py \ + --shares 375000 --candidates 3 --gc off --baseline +``` + +Repeat with `--gc on` and with `--shares 1024`. Use `--trace` in a separate +single-candidate run to measure allocations without conflating tracing overhead +with ordinary timing. Run large measurements serially. +Record image/source IDs, architecture, Python and native-client versions, raw +output, bytes per phase, observer delays, proof latency/errors, GC events, +allocation/ownership counts and retained RSS after drain. Separate lazy decode, +reference-count retirement and forced-GC controls; force GC only in the isolated +qualification process. + +The live Stratum gate also has a primed, wallet-free fixture that first submits +three ordinary valid shares, then solves a real block. It uses the pinned qbit +functional helpers and binary. Historical node mocktime keeps ASERT difficulty +above the ordinary-share target. Only this disposable fixture permits the old +template timestamp; coordinator/database clocks and all lease budgets stay +unchanged. + +```sh +QBIT_PRISM_LIVE_POSTGRES=1 \ +QBIT_PRISM_LIVE_NATIVE_POSTGRES=1 \ +QBIT_PRISM_LIVE_PRIMED_WINDOW=1 \ +QBIT_PRISM_LIVE_AUDIT_API=1 \ +bash test/test-prism-stratum-regtest-live.sh +``` + +Run the gate in Docker with access to its Docker daemon, the pinned qbit +binaries and functional helper tree, Rust tools, PostgreSQL client, and psycopg. +Use a unique `QBIT_PRISM_POSTGRES_CONTAINER` when several workspaces run gates. +For abrupt restart qualification, also set `QBIT_PRISM_LIVE_RESTART_REPLAY=1`. +That test entry point verifies the share/outbox commit through an independent +connection, exits before the first node offer, stops every miner, and starts the +normal coordinator. The successor must recover and finalize the exact persisted +hash/digest/share once. No production fault-injection switch is added. + +## Design choices + +```mermaid +flowchart LR + Live[Live immutable candidate view] --> Codec[Stream validation and identity hash] + Codec --> Stage[Stage immutable body chunks] + Stage --> Commit[Atomic fenced share and outbox publication] + Commit --> Headers[Bounded replay headers] + Headers --> Hydrate[Bounded reads into owned spool] + Legacy[Legacy pending JSON] --> Helper[Isolated compatibility helper] + Helper --> Hydrate + Hydrate --> Land[Verify and finalize with lazy audit views] + Land --> Terminal[Fenced terminal state and body detachment] + Terminal --> Reap[Bounded physical cleanup] +``` + + +Small candidate-count pages are useful admission controls, but a byte bound must +also cover each native decoder call. Returning one complete JSON row per +candidate leaves the single-candidate failure mode intact. Moving work to a +Python thread does not isolate a C codec that holds the interpreter lock, and +disabling cyclic GC does not bound reference-count destruction. + +The implementation separates durable candidate identity from its transport and +retained representation. Immutable body chunks and small replay metadata permit +bounded reads and retries without retaining nested share dictionaries. An +additive format also preserves the old pending rows for compatible recovery. +Staging a body must not credit its share: the existing atomic, fenced share/outbox +publication remains the visibility boundary. Cleanup first detaches terminal +work, then reclaims unreachable body data in bounded operations. + +An alternative normalized child table with one SQL row per share would simplify +some range queries, but duplicates the canonical intent model, makes exact legacy +serialization harder to preserve, and still needs a policy for oversized records +and non-share collections. Chunked canonical bytes retain the identity contract +and separate transport sizing from mining rules, at the cost of chunk/index +integrity checks, spool capacity and a compatible-reader rollback floor. + +Pure-Python JSON is useful in the standalone recovery tool, but its cooperative +scheduling does not bound its object graph or retirement cost. Isolated helpers +are appropriate for legacy or oversized materialization that cannot be streamed +through the supported codec. They require explicit deadlines, capacity limits, +shutdown ownership and recoverable resource-pressure behavior. The ordinary path +must not fall back to whole-window decoding in the coordinator when such a +helper or its spool is unavailable. + +Dual-writing the entire old JSON body would ease binary rollback but retain the +original encoding and allocation boundary. A larger lease threshold or suppressed +alert would leave that boundary unchanged and is not part of the fix. + +## Validation contract + +| Area | Required assertion | +| --- | --- | +| Identity | Old and new canonical hashes are byte-identical across ordinary, Unicode, escaped, empty and large windows; pending timestamps normalize identically. | +| Exact retry | Equivalent JSONB numeric/key-order representations retain historical retry behavior; different content is refused even under an injected digest collision. | +| Malformed input | Invalid field types, forbidden JSONB text, malformed encodings, missing chunks, bad offsets and digest/count mismatches fail closed without publishing partial work. Resource pressure remains distinguishable from corrupt data. | +| Atomicity | Upload/seal alone credits no share. Share credit and the outbox reference appear in the same fenced commit. A lost acknowledgment replays exactly once. | +| Fencing | A stale writer/session cannot publish, mutate sealed content or terminalize another session's work. Recheck after hydration/comparison, including takeover races. | +| Replay | Cursor enumeration remains complete across equal timestamps, byte-truncated pages, more candidates than one page, one huge candidate and several huge candidates. Oversized work cannot silently disappear. | +| Cancellation and retries | Interrupt encoding, upload, hydration, helper work, finalization and cleanup; preserve durable pending work, release private resources and allow a fresh retry. | +| Shutdown and ownership | Join workers/helpers, close owned file/index handles and release retired representations before cyclic GC, including exceptions and repeated generations. Do not close a source still owned by legitimate work. | +| Stale cleanup | Preserve active-chain accepted candidates and accepted-parent accounting rules. Detach terminal references under the fence, then reclaim unreachable data without a long writer-lease lock. | +| Mining and payouts | Real Stratum shares and a mined block pass normal verification/accounting, including abrupt process exit before the first node offer; Python/Rust parity and the ordinary lease gates remain green. | + +Performance qualification records peak bytes per codec/transport call, file-index +buffering, native output buffering, admitted spool bytes, helper count and memory, +retained representations after drain, observer scheduling and real lease-proof +behavior separately. Exercise at least 375,000 shares per candidate and both one- +and three-candidate replay, GC enabled/disabled and small-window controls. Stress +many recipients, small audit segments and an oversized individual record as well +as ordinary large windows. A fixed number of candidates or records is not a byte +bound. Allocation tracing belongs in separate runs from ordinary timing. + +## Coordination and production acceptance + +#254 owns daemon sequence retention, matched self-check adoption, fallback +window construction, exception ownership, and their process metrics. Candidate +changes must consume its replayable byte-backed sequence interface and preserve +its payout-generation, balance-check and accepted-parent fences. Review the +combined ownership graph after both branches land; passing the candidate tests +alone does not qualify #254's independent paths. + +Before enabling new storage in production, rehearse schema upgrade, legacy +pending replay, partial staging cleanup, exact retries, stale-session refusal, +shutdown, and rollback against a representative backup on x86_64. Capacity-plan +spool space and helper memory from measured candidate sizes and solve/retirement +rates. A spool reservation is a disk admission bound, not a bound on filesystem +page cache; include helper RSS, container file memory and file-descriptor counts +in the capacity review. Resource pressure must leave valid pending work recoverable. + +Use an additive rollout: deploy and qualify the compatible reader before +enabling chunked writes. Record pending counts, candidate identity hashes, +credited-share state and body integrity before and after the migration rehearsal. +An interrupted upload must expose neither partial candidate work nor credited +shares without their outbox entry. Lost acknowledgments must be safe to retry +under the original exact comparison and writer-session fences. + +Once a chunked outbox body exists, the rollback floor is the compatible reader +with new-format writes disabled. Repinning directly to the investigated +production revision is unsafe: it cannot replay the new representation. A +further rollback requires either draining the pending work or a separately +verified conversion with the writer stopped, followed by identity and durability +checks. Keep the additive body tables during rollback; dual-writing the full +legacy JSON would restore the encoding boundary this change removes. + +Production acceptance requires the #254 minimum **two-hour continuous +observation** in one process epoch after any approved restart. Include at least +two naturally scheduled self-checks, daemon fallback and recovery, live solves, +real pending-candidate replay, accepted-parent accounting, both public Stratum +lanes and a scheduled backup. Extend the observation when a required workload +has not occurred. + +Require no unexplained restart, no new 0.8/1.0 late-wake or exit-guarantee-breach +increments, no hidden durable hard exit, no hash/payout divergence, no lost or +duplicated credit, complete replay, and stable ownership/spool/backlog levels +after drain. Record 0.5 diagnostics, GC distributions and Python/native/Rust +memory separately. Alert silence is not evidence that raw counters remained +unchanged. Any recurrence or stale-session write fails acceptance. + +Deployment and production restarts require the normal operator-approved pin +workflow. This implementation and its Docker experiments do not deploy anything, +change production configuration, or establish a proven production root cause. diff --git a/docs/prism-capacity-readiness.md b/docs/prism-capacity-readiness.md index 1ca58c2ee..c1f430d6f 100644 --- a/docs/prism-capacity-readiness.md +++ b/docs/prism-capacity-readiness.md @@ -131,3 +131,486 @@ orchestration should call it explicitly before opening public routes and archive the artifact and raw results. Private canaries and routine restarts do not require it. Re-run only when the qualified image, bound load configuration, database/hardware profile, forecast, or ACK objective changes. + +## Heap and Component-Cardinality Telemetry + +Issue #226 recorded a coordinator whose resident set grew by roughly 390 MB per +hour over 45 hours of uptime with no restart, no OOM kill, and a memory map made +of hundreds of 4-64 MiB anonymous regions -- the shape of glibc per-thread +malloc arenas across the process's 64 threads. Nothing on `/metrics` could say +whether that growth was live Python objects, pymalloc fragmentation, or glibc +arena fragmentation. The families below are the always-on instrument for that +question. They are cheap (every reading is a counter load, a `len()`, or one +`mallinfo2` call; never a heap walk, never `gc.get_objects()`, never +`tracemalloc`) and fixed-cardinality (no label carries a job id, tip hash, +generation, worker name, or any other value that grows with traffic). They are +not the census, the allocator experiment, or the fix; those are #226's second +part. + +### Interpreter and allocator families + +Rendered by `lab/prism/metrics.py` from `lab/prism/process_telemetry.py`, in +this order, after the progress-health block: + +| Family | Type | Meaning | +| --- | --- | --- | +| `qbit_prism_process_allocated_blocks` | gauge | `sys.getallocatedblocks()`: memory blocks the CPython allocator currently holds. A live-object proxy: it rises with retention and stays flat under pure allocator fragmentation. | +| `qbit_prism_process_gc_trigger_count{generation}` | gauge | `gc.get_count()`: the collector's per-generation trigger counters, compared against `gc.get_threshold()` to decide when to collect. Generation 0 is allocations minus deallocations. **Not** a count of retained objects. CPython 3.13 made the collector incremental, so the third entry is unused and reads `0`. | +| `qbit_prism_process_gc_collections_total{generation}` | counter | `gc.get_stats()`: collector passes per generation since start. | +| `qbit_prism_process_gc_collected_objects_total{generation}` | counter | Objects the collector freed per generation since start. | +| `qbit_prism_process_gc_uncollectable_objects_total{generation}` | counter | Objects found unreachable but not freeable. Growth here is a leak the collector already knows about. | +| `qbit_prism_process_threads` | gauge | `threading.active_count()`: each live thread is a candidate glibc arena. | +| `qbit_prism_process_malloc_info_available` | gauge | 1 when glibc `mallinfo2` is bound and `PRISM_MALLOC_TELEMETRY` is on; 0 otherwise. | +| `qbit_prism_process_malloc_arena_bytes` | gauge | `mallinfo2.arena`: heap space glibc obtained from the kernel for its arenas, main heap plus every per-thread arena heap, excluding mmapped chunks. | +| `qbit_prism_process_malloc_in_use_bytes` | gauge | `mallinfo2.uordblks`: bytes in allocated chunks across every arena. | +| `qbit_prism_process_malloc_free_bytes` | gauge | `mallinfo2.fordblks`: bytes in free chunks glibc retains rather than returning. | +| `qbit_prism_process_malloc_mmapped_bytes` | gauge | `mallinfo2.hblkhd`: bytes glibc served directly through `mmap`. | + +The `generation` label is the closed set `0`, `1`, `2`. An interpreter that +reports fewer generations renders `-1` for the missing one rather than +shrinking the series set. + +**`mallinfo2` is glibc-only.** The symbol exists in glibc 2.33 and later. It is +absent on musl, on macOS, and on older glibc; the older int-typed `mallinfo` is +deliberately not used as a fallback because its 32-bit fields wrap on a +multi-gigabyte heap. On a platform without it the collector binds once, remembers +the absence, never raises and never logs on a scrape, and renders +`qbit_prism_process_malloc_info_available 0` with all four byte gauges at `-1`. +It never renders zero bytes: a zero arena on a 17 GB process is a lie an operator +would act on. The family set is identical on every platform, so a dashboard built +against one reads the same series on another; only the availability gauge and +the sentinel values differ. + +`PRISM_MALLOC_TELEMETRY` (default `1`, strict boolean, validated at startup by the +coordinator config loader) switches the `mallinfo2` call off. The call walks +glibc's free lists under each arena lock; that is negligible on a healthy heap, +but it is the one reading here that is not a plain counter load, so an operator +can stop it without a redeploy. Off renders exactly like an absent symbol. The +interpreter families have no switch: they are pure counter loads and are always +on. + +The switch is passed through in `compose.yaml` and documented in `.env.example`, +so a restart with a changed `.env` is enough on the compose stack. The mainnet +deployment sets the coordinator's environment from its own configuration rather +than from this file, so turning the switch off there needs the same variable +added on that side. The renderer resolves the switch once, at construction, and +falls back to the shipped default if the value has become invalid since startup +-- a telemetry knob must not be able to take the coordinator down from a metrics +thread. + +### Component-cardinality families + +Two families with a closed `component` label set, rendered after the heap +block: + +| Family | Type | Meaning | +| --- | --- | --- | +| `qbit_prism_component_entries{component}` | gauge | Entries retained by each in-process component: one `len()` or `Queue.qsize()` over the owning structure. | +| `qbit_prism_component_bytes{component}` | gauge | Bytes retained by each byte-sized payload: the cached payout window's canonical JSON, the daemon window mirror, and the share-window serialization compact strings. Every value is resident bytes except `share_window_serialization_spool`, which is an on-disk temporary file and is *not* part of RSS. | + +The `component` values are fixed by `PRISM_COMPONENT_ENTRY_KINDS` and +`PRISM_COMPONENT_BYTE_KINDS` in `lab/prism/metrics.py` and pinned by the family +test. They cover the payout window (pages, records, the armed artifact's shares, +the preview and invalidation maps, the append-invalidation stamps, the in-flight +scan anchors), the single-slot share-window serialization cache, job contexts +and the bundle cache, the evicted-job graveyard and its same-tip index, the +candidate lanes and every hash-keyed candidate registry (outstanding, +tip-observed, counted abandonments, preview stamps, ancestor re-drives, terminal +outcomes, disposition flights, waiting retries, dequeued hashes, accounted +blocks), reconcile flights and the trusted-tip memo, the pending-share commit +floor, and the daemon mirror (records in a mirror-backed window and the +compiler's mirror of the daemon's uploaded-window LRU). + +The cached payout window is accounted exactly once, so the byte family can be +summed for attribution. Its two backings are mutually exclusive: on the Python +path the window reports under `payout_window_pages`, `payout_window_records`, +and `payout_window_canonical_json` with the daemon-mirror components at zero; +on the Rust path (`PRISM_WINDOW_PIPELINE_RUST=1`, which is what mainnet runs) +the mirror *is* the cached window, so it reports under +`daemon_window_mirror_records` and `daemon_window_mirror_canonical_items` with +all three `payout_window_*` components at zero. Which pair is non-zero is how +you read the active backing off `/metrics`. `daemon_uploaded_windows` is a +separate structure -- the compiler's LRU of windows already uploaded to the +daemon -- and is never part of that either/or. + +There is no hashrate-rollup gauge because there is no in-process rollup buffer: +the rollup maintenance loop folds shares in PostgreSQL through +`advance_hashrate_rollups` and retains nothing between passes. + +### What to look at first when RSS climbs + +1. **Is it retention or fragmentation?** Plot + `qbit_prism_process_allocated_blocks` and + `qbit_prism_process_malloc_in_use_bytes` against + `qbit_prism_process_resident_memory_bytes`. All three rising together is + retention: something is holding objects. RSS rising while allocated blocks + and in-use bytes stay flat is fragmentation *or* an mmapped retention -- see + step 2 before deciding. +2. **If fragmentation, whose?** `qbit_prism_process_malloc_arena_bytes minus + qbit_prism_process_malloc_in_use_bytes` (equivalently the free-bytes gauge) + growing with `qbit_prism_process_threads` high is glibc arena fragmentation, + the shape #226 captured. Arena flat while RSS rises points at pymalloc or at + the Rust daemon child, which this process's gauges do not see. + + **Check `qbit_prism_process_malloc_mmapped_bytes` before concluding + fragmentation.** glibc serves allocations above its mmap threshold with + `mmap` and records them in `hblkhd`, which this gauge exports — they are + *not* in `uordblks`/`..._in_use_bytes`. So a large retained native buffer + makes both step 1's in-use gauge and this step's arena figure look flat while + RSS climbs, which reads exactly like fragmentation and is in fact retention. + Rising mmapped bytes means step 1's answer was retention after all. +3. **If retention, which component?** Read `qbit_prism_component_entries` and + `qbit_prism_component_bytes` on one panel. The component whose slope matches + RSS is the one to census. A registry keyed by block hash that only ever grows + (terminal outcomes, preview stamps, accounted blocks) is the first suspect + after a candidate storm; the payout window and serialization bytes are the + first suspects after a run of full rescans. +4. **Is the collector keeping up?** A rising + `qbit_prism_process_gc_uncollectable_objects_total` is a reference cycle with + a finalizer the collector cannot break — that is retention the collector + cannot fix on its own, and it is the one GC series that counts objects. + `qbit_prism_process_gc_collections_total` flattening while allocation + continues means collections have stopped happening at all. Do **not** read + `qbit_prism_process_gc_trigger_count` as a backlog of retained objects: it is + the counter CPython compares against its thresholds, it resets on every + collection, and its third entry is unused on the shipped interpreter. + +Every gauge here is a scrape-time snapshot of the same process; read the +`X-Prism-Metrics-State` header before trusting a stale document. + +### Stack-sample frame ownership + +The stall probe retains formatted text only. `sys._current_frames()` includes +the sampler's own frame, so leaving its dictionary in a local variable creates +a cycle that can retain every sampled thread's locals until cyclic GC. This +includes completed payout-window workers and their share data; limiting the +number of printed threads or frames does not bound that retained heap (#247). +`StallProbe.capture_stacks()` releases the snapshot and traversal references in +`finally`, including when formatting fails or a sampling limit is reached. It +does not clear another thread's live execution frame. + +`StallProbeFrameOwnershipTests` in `tests/test_prism_lease_monitor_stalls.py` +checks this with real worker threads and weak references while cyclic GC is +disabled for the test. Payloads must disappear after workers finish, even while +the probe's last sample or a formatting-error traceback remains alive. GC stays +enabled according to the application's existing policy in production. + +Passing that ownership regression does not establish that the remaining #236 +stalls are resolved. After deployment, compare the native-ledger/window/job +pipeline at 350k–400k shares under concurrent work and a sustained soak. Record +coordinator RSS, per-generation GC-pause histogram deltas, late-wake counter +deltas, restart count, and mining-work availability; measure the Rust builder's +memory separately. Lifetime maximum pauses and a stack captured after the +monitor resumes cannot attribute an individual late wake to a collection. + +## Heap Census, Allocator Control, and the Resident-Set Bound + +Issue #226's second part. The always-on families above say *whether* the +resident set is growing through retention or through fragmentation; this +section is the instrument that says *what* is retained and *which allocator +setting* bounds the fragmentation, and the automated check that a fix has to +pass. Everything here is off by default, operator-triggered, bounded, and +unreachable from any request path: no HTTP route, no Stratum message, and no +`/metrics` scrape can start a heap walk or a trim. `tests/test_prism_heap_census.py` +pins each of those properties. + +### Running a census + +The census is `lab/prism/process_telemetry.py`'s `HeapCensus`. It is armed only +when the coordinator starts with `PRISM_HEAP_CENSUS=1`; without that the signal +is not registered, no worker thread exists, and the census entry point is inert +even if called. Armed, it responds to `SIGUSR1`: + +```sh +# Compose stack +docker kill --signal=SIGUSR1 +# Bare process +kill -USR1 +``` + +The signal handler writes one byte to a pipe and returns; a dedicated thread +(`prism-heap-census`) does the walk. That is the shape `SIGUSR2`'s +`faulthandler` dump follows, with the work moved off the handler so a second +signal cannot re-enter a lock the handler holds. Signals that arrive within +`PRISM_HEAP_CENSUS_MIN_INTERVAL_SECONDS` (default `60`) of the last census are +logged as suppressed, so a repeated `kill` cannot stack walks. + +What the walk costs, and why it is never automatic: it is one +`gc.get_objects()` call followed by one pass over every tracked object taking +its type and `sys.getsizeof()`. The `gc.get_objects()` call allocates one +pointer per tracked object and holds the GIL until it returns; nothing can +interrupt it, and on a heap of tens of millions of objects it is a stall of +seconds on every thread. The pass that follows is what the walk cap bounds. +Take a census in a low-load window, and expect a share-ack latency blip in +`qbit_prism_share_ack_seconds` when you do. + +The switches and bounds, all validated at startup (a value outside its range +refuses startup rather than promising a bound it does not keep). Each passes +through `compose.yaml` to the `prism-coordinator` service and is listed in +`.env.example`, so a value in the deploy dotenv reaches the process; compose +has no `env_file:` for the service, and a variable without a passthrough line +would silently never arrive: + +| Variable | Default | Meaning | +| --- | --- | --- | +| `PRISM_HEAP_CENSUS` | `0` | Arm `SIGUSR1`. Strict boolean. | +| `PRISM_HEAP_CENSUS_DIR` | `$PRISM_AUDIT_DIR/heap-census` | Where reports are written. Created on first census. | +| `PRISM_HEAP_CENSUS_TOP_N` | `40` (max `200`) | Entries in each of the four lists: types by count, types by shallow bytes, `tracemalloc` sites by line, sites by file. | +| `PRISM_HEAP_CENSUS_MAX_BYTES` | `262144` (`4096` to `4194304`) | Output ceiling per report. Met by halving top-N until the JSON fits, never by cutting the document. | +| `PRISM_HEAP_CENSUS_MAX_SECONDS` | `10` (max `120`) | Walk cap for the per-object pass; the report says when it was hit. | +| `PRISM_HEAP_CENSUS_MIN_INTERVAL_SECONDS` | `60` | Signals closer together than this coalesce. | +| `PRISM_HEAP_CENSUS_TRACEMALLOC` | `0` | Start `tracemalloc` (one frame) at startup so the census can name allocation sites. Requires `PRISM_HEAP_CENSUS=1`. | + +`tracemalloc` is the second, separate opt-in because its cost is standing, not +per census: every allocation pays for the trace for as long as it runs, and the +trace itself is resident memory proportional to live blocks. Use it for a +diagnosis window, not steady state. Without it the census still names the +retained *types*; with it the census also names the *sites*. + +Reports land as `heap-census--.json`, written to a temporary +name and renamed into place. At most 32 are kept per directory (oldest removed +first), so with the default byte ceiling the census can occupy at most 8 MiB +of the audit volume. + +### Reading a census + +A report is one JSON document: + +- `process`: the same readings `/metrics` exports (`resident_memory_bytes`, + `allocated_blocks`, `threads`, the GC counters, the `mallinfo2` bytes) plus + what the scrape cannot afford: `malloc_arena_count` and the split of glibc's + free bytes into `malloc_free_bin_bytes` (interior free chunks) and + `malloc_free_top_bytes` (the free space at the end of each arena's heaps), + and `anonymous_map`, the private-anonymous region histogram in the same + 4-64 MiB bands the mainnet capture was reported in. +- `walk`: `tracked_objects`, `objects_walked`, `distinct_types`, `truncated`, + and the seconds spent in `gc.get_objects()` and in the whole pass. +- `types_by_count` and `types_by_bytes`: top-N `{type, count, shallow_bytes}`. + Shallow bytes are `sys.getsizeof()`; a `dict` of 60k share records is one + large dict plus 60k small dicts plus their strings, so read the two lists + together. +- `tracemalloc`: `{"tracing": false}` or the top-N sites by line and by file + with bytes and block counts, plus the traced total and peak. + +Read it in this order: + +1. **Retention or fragmentation?** Compare `process.resident_memory_bytes` + with `process.malloc_in_use_bytes` and `process.allocated_blocks` against + an earlier census (or against the `/metrics` history). In-use bytes and + allocated blocks rising with RSS is retention; RSS rising over a flat + in-use is fragmentation. On the mainnet capture the map alone said + fragmentation, but only a census on that process can prove there is no + retention underneath it. +2. **If retention, of what?** `types_by_count` names the type; the first + census after startup is the baseline and the type whose count grows across + censuses is the leak. With tracing on, `tracemalloc.by_line` names the + allocation site. A site inside the payout-window materialization is an + issue #228 finding; a registry keyed by block hash is a candidate-lane + finding. +3. **If fragmentation, can a trim reach it?** `malloc_free_bin_bytes` is what + `malloc_trim` returns to the kernel, in every arena. `malloc_free_top_bytes` + across `malloc_arena_count` arenas is what it cannot: glibc trims a + per-thread heap's top only when the owning thread frees a chunk of 64 KiB + or more while the top exceeds `M_TRIM_THRESHOLD`, and that threshold + floats up to 64 MiB as large chunks are freed. Free bytes parked in the + tops of many arenas are the case `MALLOC_ARENA_MAX` fixes and a trim does + not; free bytes in bins are the case a trim fixes. +4. **Was the walk complete?** `walk.truncated` true means the type lists are + over a prefix of the heap. Raise `PRISM_HEAP_CENSUS_MAX_SECONDS` and take + another census in a quieter window rather than reading a partial census + as a whole one. + +### `malloc_trim` + +`MallocTrimmer` calls glibc `malloc_trim(0)` and reports the readings around +it through the same collector `/metrics` uses: RSS, arena, in-use and free +bytes before and after, the deltas, glibc's own released flag, and the call's +duration, on one log line (`prism malloc_trim ...`). A trim that works shows +`in_use_delta=0` (it frees nothing the program holds), a negative +`resident_delta`, and `released=True`. + +Two ways to run it, both off by default: + +| Variable | Default | Meaning | +| --- | --- | --- | +| `PRISM_MALLOC_TRIM_SIGNAL` | `0` | Arm `SIGRTMIN+1` (`docker kill --signal=RTMIN+1`, `kill -RTMIN+1`). Strict boolean. | +| `PRISM_MALLOC_TRIM_INTERVAL_SECONDS` | `0` (off; minimum `60`) | Periodic trim on the census thread, paced by this interval. | + +Both go through the same worker thread and the same +`PRISM_HEAP_CENSUS_MIN_INTERVAL_SECONDS` gate, so signals cannot stack trims +either. The design call is to offer both: the signal is the instrument (does +RSS return when asked?), and the paced periodic action is the candidate fix +for interior fragmentation once the soak shows it returns memory. The floor +exists because of what the call does: `malloc_trim` takes every arena's lock +in turn while it consolidates and walks that arena's free lists, and a thread +allocating in that arena at that moment waits. On a fragmented multi-gigabyte +heap that is a stall of tens to hundreds of milliseconds across whichever of +the 64 threads happen to allocate, not a wedge; it is not something to run +every second on a latency-sensitive process. Run the signal form in a +low-load window first and read the logged duration before choosing an +interval. + +What it cannot do: shrink the tops of per-thread heaps (see "Reading a +census"). With many arenas, `malloc_trim` consolidates each arena's free +chunks *into* its top and then cannot return that top, so RSS does not move; +with `MALLOC_ARENA_MAX=2` the same free chunks stay interleaved with other +threads' live chunks in two arenas, and the trim returns them. That is +measured, not inferred: the storm instrument below shows it on any glibc host. + +### Allocator settings and the storm instrument + +The image sets `MALLOC_ARENA_MAX=2`; `docs/mainnet-deployment.md` records the +decision, the rationale, and how to override it per deployment. The +experiment behind it is `tests/prism_candidate_storm.py` with three additions: + +- `--heap-report` reads the resident set, `allocated_blocks`, thread count, + the `mallinfo2` bytes, the arena count, the free-bytes split (tops versus + bins), and the anonymous-map histogram at every phase boundary of the + storm, then after the storm's objects are released and collected, then + after one `malloc_trim`. The release-and-collect reading is #185's + post-storm drain measurement, re-runnable on any host. +- `--arena-probe-threads N` (with `--arena-probe-rounds`) runs `N` threads that + each allocate medium buffers (4-120 KiB: over pymalloc's 512-byte ceiling, + under glibc's initial 128 KiB `mmap` threshold) and meet at a barrier before + exiting, which is how glibc gives each of them an arena. It reproduces the + coordinator's long-lived-thread shape; the storm rig itself is single- + threaded. +- `--allocator-experiment default,2,4` runs the instrument once per + `MALLOC_ARENA_MAX` setting in a child process (glibc reads the variable once + at start) and prints every child's heap section side by side; + `--allocator-mmap-threshold BYTES` pins `MALLOC_MMAP_THRESHOLD_` in the + children as an extra axis. + +```sh +python3 tests/prism_candidate_storm.py --decide --drain-per-row \ + --arena-probe-threads 64 --allocator-experiment default,2,4 > arena-experiment.json +``` + +Read the mechanism, not the absolutes: memory absolutes vary by up to an order +of magnitude across hosts, and the instrument prints `python`, `libc`, +`machine` and `cpu_count` beside every number for that reason. What should +transfer: `malloc_arena_count` after the probe reaches the thread count at +the default and stops at the cap otherwise; `anonymous_regions_4mib_to_64mib` +climbs into the mainnet shape at the default; `allocated_blocks` at +`after_release_gc` returns to `start` (no Python retention from the storm); +and `malloc_trim` returns memory at the cap and not at the default. The drain's +`wall_seconds` and the `phase_seconds` are the latency comparison across +settings. + +### The resident-set bound + +The stated bound: **after a one-hour warm-up, the resident set must stay within +2.0x the warm-up peak for the rest of a 24 h soak at ordinary load.** The +warm-up peak is the baseline because the first hour materializes the payout +window and runs the first full rescan; the #226 slope (390 MB/h from a 125-145 +MiB start) breaches this bound in its third hour, and a post-storm excursion +like #185's 613 MiB drains back under it. The check is automated, not a graph +someone reads: + +```sh +python3 -m lab.prism.process_telemetry rss-bound --samples soak-rss.csv \ + --warmup-seconds 3600 --multiple 2.0 --min-span-seconds 82800 +``` + +The input is one `seconds,rss_bytes` line per sample (absolute epoch seconds +are fine; `#` comments are skipped; `-1` samples are ignored). The command +prints the verdict as JSON (baseline, bound, peak and its time, the first +breach time, and the least-squares slope over the post-warm-up samples) and +exits `0` on pass, `1` on fail, `2` on unusable input. `--max-slope-bytes-per-hour` +adds a slope guard for a leak slow enough to stay under the multiple inside 24 +hours; `--min-span-seconds` refuses a series shorter than the soak. + +When it fails: the first breach time says whether the growth is the steady +slope (breach hours in) or an excursion (breach right after a storm or a +rescan burst). Take a census at the breach and read it as above; if the +census names a retained type, that type's owner is the fix; if it names free +bytes parked in arena tops, the arena cap is the fix and the soak is rerun at +the other setting. Do not raise the multiple to make a soak pass. + +### Testnet 24 h soak runbook (deferred to the operator) + +This soak has not been run. Nothing here was verified against a live host; +the instrument, the thresholds and the procedure are what this package +delivers, and the numbers are what the soak produces. + +1. **Build** the release image (which now carries `MALLOC_ARENA_MAX=2`) and + record its digest. +2. **Run three soaks in sequence**, each a fresh coordinator process for at + least 24 h at ordinary testnet load with the same miner population, with + these values in the deploy dotenv (each passes through `compose.yaml` to + the `prism-coordinator` service; a deployment that sets the coordinator's + environment another way sets them there): + - `MALLOC_ARENA_MAX=2` (the image default), + - `MALLOC_ARENA_MAX=4`, + - `MALLOC_ARENA_MAX=2` with `PRISM_MALLOC_TRIM_INTERVAL_SECONDS=900`. + + Set `PRISM_HEAP_CENSUS=1` and `PRISM_MALLOC_TRIM_SIGNAL=1` on all three; + leave `PRISM_HEAP_CENSUS_TRACEMALLOC` off unless a census names growth + without naming a site. Before trusting a run, confirm the setting reached + the process and the census is armed: + + ```sh + c= + docker exec "$c" sh -c 'echo MALLOC_ARENA_MAX=$MALLOC_ARENA_MAX PRISM_HEAP_CENSUS=$PRISM_HEAP_CENSUS' + docker logs "$c" 2>&1 | grep 'prism heap census armed' + ``` + + Record the exact environment beside each run. +3. **Capture every 5 minutes** for the whole soak. The audit port is bound to + the container's loopback and compose does not publish it, so both reads + run inside the container, where the coordinator is PID 1 and the `lab` + package is on the path (`3341` is the default `PRISM_AUDIT_PORT`; + substitute the deployment's value): + + ```sh + c= + while true; do + docker exec "$c" python3 -m lab.prism.process_telemetry rss-sample --pid 1 >> soak-rss.csv + docker exec "$c" python3 -c 'import sys, urllib.request; sys.stdout.write(urllib.request.urlopen("http://127.0.0.1:3341/metrics", timeout=5).read().decode())' \ + | grep -E '^qbit_prism_process_(resident_memory_bytes|allocated_blocks|threads|malloc_(arena|in_use|free)_bytes) ' \ + | sed "s/^/$(date +%s) /" >> soak-metrics.log + sleep 300 + done + ``` + + Also keep the share-ack histogram (`qbit_prism_share_ack_seconds`) at + hours 1, 12 and 24 for the latency comparison between settings. +4. **Census** at hour 1 (the baseline) and hour 24 with + `docker kill --signal=SIGUSR1 "$c"`, and once more at any breach. The + coordinator logs `prism heap census written path=...`; the report lands on + the audit volume inside the container, so copy it out with + `docker cp "$c":/var/lib/qbit-prism/audit/heap-census ./heap-census-` + (substitute `PRISM_AUDIT_DIR` if it was changed). Docker accepts both + `SIGUSR1` and `RTMIN+1` as `--signal` values and delivers them; both were + checked against a container on the development host. +5. **Trim** once at hour 23 with `docker kill --signal=RTMIN+1 "$c"` on the + runs without the periodic trim, and record the logged `prism malloc_trim` + line (`docker logs "$c" 2>&1 | grep 'prism malloc_trim'`): + `resident_delta`, `free_delta`, `seconds`. +6. **Judge** each run: + + ```sh + python3 -m lab.prism.process_telemetry rss-bound --samples soak-rss.csv \ + --warmup-seconds 3600 --multiple 2.0 --min-span-seconds 82800 + ``` + + Pass: exit `0` on every run you intend to ship, and share-ack p99 at + hour 24 within the alert threshold already configured for the deployment, + with no `prism malloc_trim` duration above 500 ms. Fail: any exit `1`, or a + latency regression between the arena settings that the operator would + alert on. Between 2 and 4, ship the one that passes with the larger margin + under the bound; if both pass and the latency is indistinguishable, keep + the image default of 2. +7. **Record** on issue #226: the three verdict JSON documents, the census + reports, the trim lines, the share-ack histograms, the image digest, and + the glibc version inside the image (`ldd --version`), because the trim + behaviour above was measured on glibc 2.39 and the image carries a newer + one. +8. **Re-run #185's drain** on the fixed build: + + ```sh + python3 tests/prism_candidate_storm.py --decide --drain-per-row --heap-report + ``` + + and record `start` against `after_release_gc` and `after_malloc_trim`, + with the platform line, on #185. diff --git a/docs/prism-coordinator-refactor/README.md b/docs/prism-coordinator-refactor/README.md new file mode 100644 index 000000000..a75de1e09 --- /dev/null +++ b/docs/prism-coordinator-refactor/README.md @@ -0,0 +1,46 @@ +# PRISM Coordinator Refactor + +Status: **complete** and organized as a nine-PR review stack. No required +roadmap item remains. + +The completed tree is integrated with `origin/1.x.x` at `36267d6` (#120), +carrying all thirty-three upstream commits #87-#120. It preserves +the base branch's public hashrate, refresh/livelock, initial/reconnect delivery, +queue-reclamation, and latest-tip priority fixes in the extracted owners. It +carries the upstream node-first block submission and accounting actor (#113), +tip-refresh epoch waves (#101), incremental payout windows and append/anchor +fencing (#112), the found-block landing livelock fix with landing-class +deadlines and degradable previews (#120), and the lease heartbeat/fencing +work (#111/#114/#116/#118) inside their extracted owners. The former exact-hash, literal-authorization, +mandatory-reviewer, and per-slice full-suite workflow is retired. + +## Result + +`lab/prism/prism_coordinator.py` is now the construction, startup, signal, +stable-facade, and top-level shutdown root. Domain state machines, background +loops, queues, locks, cached observability, persistence, HTTP, and mining work +live in dedicated owners. See [Structure](structure.md) for the boundary and +the documented size exception. + +B3 exists as the post-offer finalization owner (`block_finalization.py`) +behind upstream's #113 node-first split: the block-candidate owner makes the +node offer and schedules durable accounting, and B3 finalizes after the offer +in one lane. There is still no second transport path or duplicate durable +handoff. See the [decision record](b3-decision.md). + +The final runnable validation matrix passes, including the Docker-backed +PostgreSQL ledger and A1 publication gates, container compile and image +build, and the Rust workspace. Both live Stratum targets are `UNAVAILABLE` +in the current environment because `qbitd` is absent from the host; that is +missing-environment evidence, not a pass or a product failure. Exact results +are in [Validation](validation.md). + +## Reference documents + +- [Invariants](invariants.md): release behavior that must remain true. +- [Roadmap](roadmap.md): completed slices and decisions. +- [Structure](structure.md): final ownership map and structural audit. +- [Validation](validation.md): risk-based cadence and final evidence. +- [Stacked PRs](stacked-prs.md): publication order and reconstruction rules. +- [A1 audit artifacts](a1-audit-artifacts.md): durable storage contract. +- [B3 decision](b3-decision.md): finalization concurrency evidence. diff --git a/docs/prism-coordinator-refactor/a1-audit-artifacts.md b/docs/prism-coordinator-refactor/a1-audit-artifacts.md new file mode 100644 index 000000000..5323f9e20 --- /dev/null +++ b/docs/prism-coordinator-refactor/a1-audit-artifacts.md @@ -0,0 +1,78 @@ +# A1 Audit Artifact Contract + +A1 is complete. `lab/prism/audit_artifacts.py` owns audit +filesystem authority; `bundle_compiler.py` owns compiler/subprocess work; the +ledger owns database authorization and accepted-block transactions; the +coordinator only composes them and retains narrow compatibility calls. + +## Directory and file authority + +- Pin audit root and evidence-parent directory descriptors with no-follow + semantics. Perform owned reads, creates, replacement, fsync, scans, and + removals relative to those descriptors. +- A path replacement revokes the store until the original inode is restored or + `reconfigure()` adopts a new authority pair atomically. +- Parse only exact owned names. Malformed lookalikes, symlinks, nonregular + entries, traversal, and out-of-root URIs never grant read, pin, repair, or + deletion authority. +- Track candidate inode identity. Cleanup removes only the exact file created + by the current attempt, including builder-create-then-raise and swapped-path + failures. + +## Compiler and verifier boundary + +J1 receives a duplicated A1 directory descriptor and reserved candidate name, +creates the canonical output relative to it, and transfers the still-open exact +inode to A1. Compatibility builders use canonicalization fallback and are not +reported as exact compiler output. + +Verification uses an unlinked read-only descriptor snapshot, a bounded process +group, timeout, and independent stdout/stderr limits. Trust source, writer key, +literal digest/bytes, and normalized report form the verification identity. A +retry never reuses a prior attempt's success. + +## Publication and replay + +- PostgreSQL assigns a unique durable `audit_publication_sequence` at the + serialized confirmation boundary. Exact confirmation replay and later + inactive/reactivation transitions reuse it; reactivation does not create an + unpublished ordinal. +- Sequence, not height, hash ordering, process order, or mtime, chooses current + evidence. Hash, height, coinbase, digest, and verification identity remain + integrity fields. +- Allocation/publication lock order is payout balance mutation then the A1 + publication guard. The guard uses one pinned internal lock inode with an + in-process reentrant lock plus cross-process `flock`. +- Reload disk evidence while guarded before replay, repair, publish, or prune. + An already-valid exact pair may replay behind a later floor; damaged state + repairs only at the fresh durable-row floor. +- Legacy evidence grants no ordering or pin authority after restart until it is + re-proved against exact confirmed ledger state and adopted at its durable + sequence. + +Mutable publication installs the envelope first and evidence second, fsyncing +file and required parent boundaries. Only then does in-memory current evidence +advance. A failure preserves the previous valid pair. Exact replay is +byte/inode stable except for documented global observational counters. + +## Bodies, segments, and retention + +Inline and external audit bodies expose the same canonical bytes, digest, and +response metadata in memory and PostgreSQL modes. Share-slot merges are +serialized, lossless for disjoint updates, idempotent for identical overlap, +and preserving on conflict. Audit bodies and share segments are durable and not +reference-blind garbage-collected. + +Retention runs after successful publication, is best effort, and revalidates +both directory authorities and current/reserved identities at each removal. +Retention 0/1, tied mtimes, concurrent stores/processes, and prune failure may +not delete or regress current evidence. The internal lock file is never exposed +as an artifact. + +## Completion evidence + +A1 was reconciled with the integrated owners and validated through its direct +artifact/API/ledger/candidate/metrics suites, PostgreSQL parity/migration/process +helpers, Rust audit CLI tests, Docker compile/lint, and diff/temporary-artifact +hygiene. Exact hashes, literal authorization, and same-reviewer verdicts were +intentionally not part of completion. diff --git a/docs/prism-coordinator-refactor/b3-decision.md b/docs/prism-coordinator-refactor/b3-decision.md new file mode 100644 index 000000000..84fa95443 --- /dev/null +++ b/docs/prism-coordinator-refactor/b3-decision.md @@ -0,0 +1,107 @@ +# B3 Finalization Concurrency Decision + +Decision: **RETAIN** the single post-offer finalization lane behind the #113 +node-first split. Do not add a second landing/finalization actor. + +The original version of this decision predates the #113 rework and framed the +question as "one lane or two" for the whole submit path. #113 answered part of +that question upstream: the node offer now runs on a lock/DB-free fast lane +before writer admission, and a separate bounded accounting actor receives the +offer evidence and drives the durable tail. What remains single-lane — and what +this document decides to keep single-lane — is the post-offer finalization +policy itself: one candidate at a time holds the same-hash disposition, lands, +verifies, publishes, and accounts. `BlockFinalizationService` +(`lab/prism/block_finalization.py`) owns that policy and its bounded phase and +candidate-interarrival metrics, so production evidence can reopen this decision +without adding speculative durability machinery now. + +## Current shape (post-#113) + +```text +B1 dequeue → disposition lease → node offer (fast lane, pre-admission) +→ accounting handoff → B1 writer admission +→ B3 post-offer finalization (this owner) +→ B1 durable outbox terminalization/retry → disposition release +``` + +- B1 (`lab/prism/block_candidates.py`) owns the node offer, the same-hash + disposition guard, the accounting queue/thread, replay/quarantine, retry + pacing, and outbox terminalization. Accounting saturation cannot convoy new + node offers: the offer happens before the handoff. +- B3 accepts B1's prepared `node_submission` and never opens a second + transport path. Its only fallback `submitblock` (a candidate that was never + offered) runs under the append-landing fence and lease verification at the + RPC boundary, through B1's node-offer seam. +- P1 (`lab/prism/payout_state.py`) owns the payout previews, the balance + serializer, append/anchor fences, and the durable replay-window proof; B3 + consumes them through coordinator facades and decides only whether a failed + proof rejects an *unoffered* reconstructed candidate. Already-offered replay + keeps its as-issued snapshot. + +## Ordered phases + +The directly tested phases run in this order inside +`_submit_block_candidate_serialized` (the seam B1's accounting actor calls +with the disposition already held; the decorated public +`submit_block_candidate` remains a compatibility entrypoint over the same +runner): + +1. admission — post-offer tip/active classification, pool-closure override on + a provable chain hit, and terminal short-circuits for already-recorded + acceptances; +2. land and durable confirmation — anchor exposure, the unoffered-replay + window proof, epoch/lease fences, `submitblock` fallback, audit + build/verify with the balance serializer released, persist and confirm; +3. CTV persistence and conditional share credit (with the urgent payout + artifact fold); +4. evidence construction from bounded aggregate counters; +5. ordered audit publication through the PR 75 store (balance-mutation lock, + then the publication-order guard, with a fresh durable floor read); +6. process-local accounting and stop-or-refresh signaling. + +The admission phase also hosts the replay-window *decision context* capture; +the proof itself and the landing execute inside phase 2 so the anchor token's +exposure/retire pair brackets the landing exactly as the pre-extraction +coordinator body did. + +## Evidence + +The extraction-time fixture evidence from the original decision (interarrival +0.003005 s; land/confirm max 0.001473 s; audit publication max 0.001426 s; all +other phases ≤ 0.000060 s) remains representative: no phase after the node +offer sits on the miner's share-ack path, and the second same-height candidate +was admitted promptly and rejected as stale. The re-cut re-validated the moved +bodies against the full current suite (all accepted-block, replay, lost-ACK, +saturation, lease-deferral, and append-epoch regression tests pass unchanged). +Live `qbitd` validation remains **UNAVAILABLE** in this execution environment +(no installed `qbitd`), so the default rule applies again: missing live +evidence is not proof to add complexity. + +## Durability and ordering + +- Candidate intent is durable before its bounded in-memory wakeup. The ledger + outbox, not the wakeup queue, is the replay authority after restart or queue + coalescing; B3 records the accepted-block result and B1 terminalizes the + outbox row. +- The #113 accounting actor already gives the offer path its own lane. A + second *finalization* actor would need another durable post-offer handoff + and an exact recovery state for the boundary between landing and + confirmation, duplicating what the outbox and same-hash disposition provide. +- A child payout base depends on its parent's verified and durable transition. + The parent barrier (`_defer_for_pending_parent_payout_transition`) + deliberately serializes that dependency; concurrent later phases would not + remove the required ordering. +- Coordinator shutdown admits finalization as a ledger writer and waits for + writer quiescence. A second actor would add a new drain/join boundary and a + new ordering choice between parent completion and lease release. +- Phase metrics retain only count, sum, and maximum per phase plus aggregate + interarrival count, sum, and minimum. They add no candidate history or + unbounded backlog. + +Revisit this decision only if production metrics show a subsequent valid +candidate or required tip response consistently arriving before the preceding +finalization completes, with material delay attributable to the post-offer +phases (the offer itself is already unconvoyed). Any future implementation +must retain the durable outbox, bound every new queue, define the post-submit +restart state, preserve the offer-before-accounting invariant and +parent/accounting order, and extend shutdown quiescence tests. diff --git a/docs/prism-coordinator-refactor/invariants.md b/docs/prism-coordinator-refactor/invariants.md new file mode 100644 index 000000000..7ab6c4c62 --- /dev/null +++ b/docs/prism-coordinator-refactor/invariants.md @@ -0,0 +1,112 @@ +# Refactor Invariants + +These are release constraints, not process gates. If a proposed boundary +conflicts with one, change the boundary. + +## Architecture + +- `prism_coordinator.py` is the construction, signal, startup, and top-level + shutdown root. Leaf services must not import it. +- Each service owns its mutable state, lock, bounded queue/executor, metrics, + and worker lifecycle. Do not replace the coordinator with a broad context + object. +- Cross-service work that can be superseded carries immutable identity: + connection generation, tip/template generation, payout generation, worker, + difficulty, or durable candidate identity as applicable. +- No socket, RPC, PostgreSQL, subprocess, filesystem, or blocking queue work + runs while a shared coordinator/session lock is held. +- When both are required, a client's vardiff lock is acquired before the + coordinator control lock. Normal share submission takes one immutable + coordinator control snapshot; deduplication and share accounting use their + dedicated owner locks rather than reacquiring the coordinator lock. +- Compatibility delegates may remain only while an in-repository caller needs + them. The implementation body and mutable state remain in one owner. + +## Mining work and delivery + +- Detected tip, published tip, template artifacts, and payout state remain + distinct. Older work cannot overwrite newer observation authority. +- Publication-critical latest-tip work preempts or defers routine/initial work + without allowing concurrent heavy builders. +- `mining.set_difficulty` and its matching `mining.notify` commit as one client + update after final connection, authorization, tip, template, payout, worker, + and difficulty validation. +- Slow or failed sends do not hold global locks or block unrelated clients. +- Clean-job retirement, same-tip retention, stale grace, and initial/reconnect + queue bounds retain their current behavior. +- Vardiff changes become active only with a job stamped at that difficulty; + every failure or supersession restores speculative state. +- Failed tip refreshes use bounded, jittered owner-held pacing tied to the + observed tip. A successful refresh or a newer tip clears the holdoff, and a + same-tip poll may reuse only a still-fresh coherent template snapshot. + +## Shares, candidates, and payout + +- A normal share is acknowledged only after its durable ledger transaction. + Accepted counters and vardiff accounting advance at the same boundary. +- A share and required block-candidate intent commit atomically. The durable + outbox is authoritative; the in-memory queue is only a bounded wakeup. +- Duplicate identity uses the immutable job worker and header. Retry or + reauthorization cannot bypass it. +- Candidate replay is idempotent, keeps parents ahead of dependent children, + distinguishes retryable from terminal outcomes, and cannot double-account a + block or regress published evidence. +- Candidate attempt time is durable before processing. Intentional retry waits + wake in bounded heartbeat slices, expose pending/unattempted age and backoff + state, and keep blocked processing phases eligible for watchdog detection. +- A candidate whose network outcome is known but whose durable outbox + finalization failed resumes finalization with bounded pacing. It does not + resubmit the block, recount acceptance, rebuild audit evidence, or re-adopt + the released share-writer floor. +- Payout generations are monotonic and a stale prepared source cannot publish. + Accepted-block preview, final coinbase, confirmed balances, and delivered + jobs must agree. +- Post-accept and blockwait paths only notify the single refresh owner; they do + not enter the heavy refresh lane themselves. + +## Audit artifacts + +- Owned paths are descriptor-relative, no-follow, strictly parsed, and confined + to pinned directory identities. +- Canonical bytes are verified before durable publication. Writes and mutable + pair replacement are atomic and fsync their required parent boundaries. +- Ledger-assigned audit publication sequence, not height or process order, + decides current evidence across replay, restart, same-height replacement, and + reorg. +- Retention is post-publication best effort and cannot remove a current, + reserved, malformed, symlinked, or unowned entry. +- Audit bodies and share segments remain digest-checked and reconstructable in + memory and PostgreSQL modes. + +## Health, HTTP, and metrics + +- Health deadlines use monotonic time. Pending age starts at the first + unresolved change and does not slide on churn. +- First-job starvation and current-tip coverage loss have separate grace + clocks. Previously delivered clients do not masquerade as first-job + starvation, and restored coverage resets the coverage-loss clock. +- Publication and successful socket delivery are separate proofs. Cached base + health never masks a newer progress failure. +- Publication divergence is cleared only by a coherent completed publication, + including a no-op poll that proves existing work is current. +- Public routes, status codes, schemas, cache headers, metric names/labels, and + environment defaults remain compatible unless a roadmap item explicitly + authorizes a bounded behavior change. +- `/healthz` and `/metrics` perform no backend, ledger, artifact, or RPC work + after cached observability is complete. +- Metrics copy share counters under the dedicated accounting lock and expose + coordinator-lock contention plus durable candidate pending/backoff state + through narrow snapshots owned by their source services. +- The coordinator-owned HTTP listener reads complete cached metrics only. The + externally managed compatibility handler may render metrics synchronously + while its cache is still uninitialized because it owns no refresher thread. +- An audit HTTP serve-loop exit closes and retires its owned listener before a + replacement listener can start, including unexpected post-readiness exits. + +## Shutdown and liveness + +Shutdown remains: close admission and signal stop; cancel refresh work; drain +admitted writers; release or deliberately withhold the exact writer lease; +then drain non-writer threads, sockets, and executors. Publication-progress +watchdog protection remains active even when ordinary heartbeat checking is +disabled. diff --git a/docs/prism-coordinator-refactor/roadmap.md b/docs/prism-coordinator-refactor/roadmap.md new file mode 100644 index 000000000..af582a82d --- /dev/null +++ b/docs/prism-coordinator-refactor/roadmap.md @@ -0,0 +1,49 @@ +# Completed Roadmap + +There is no active work order. Completed commits are implementation history, +not gates that must be reconstructed. + +| Area | Result | +| --- | --- | +| foundations, configuration, RPC, shutdown, executors | extracted | +| progress, background services, CTV | extracted | +| payout, templates, bundles, refresh scheduler | extracted | +| sessions, delivery, share writer/recovery | extracted | +| `origin/1.x.x` through `36267d6` | complete, including node-first submission and the accounting actor (`#113`), tip-refresh epoch waves (`#101`), incremental payout windows (`#112`), and the found-block landing livelock fix (`#120`) | +| A1 audit artifact ownership | complete | +| B1 candidate submission | complete | +| S4 share classification | complete | +| D1 bounded duplicate index | complete | +| V1 vardiff and idle retarget | complete | +| B2 accepted-block finalization phases | complete | +| B3 post-offer finalization owner | complete behind the `#113` node-first split; no second transport lane ([decision](b3-decision.md)) | +| O2 cached health | complete | +| O3 cached metrics | complete | +| H1 audit/public HTTP | complete | +| X1 reorg, metrics, watchdog, and seam cleanup | complete | + +X1 removed unused re-exports and drifting state mirrors, moved the remaining +reorg, metrics, and watchdog domain bodies to owners, and replaced magic +attribute forwarding with explicit ports and test hooks. Compatibility aliases +or delegates remain only where in-repository callers demonstrate the stable +facade; they do not own duplicated mutable state. + +The upstream retry work is owned by `template_artifacts`, `tip_refresh`, and +`block_candidates`; the upstream health correction is owned by `job_delivery`, +`observability`, and `metrics`. The upstream hot-path correction is owned by +`stratum_session`, `job_delivery`, `share_submission`, `vardiff_service`, +`block_candidates`, `share_ledger`, and `metrics`: client vardiff state has a +per-client lock, share accounting has a dedicated lock, normal submission uses +one coordinator control snapshot, and candidate retry state remains observable +without making intentional waits look healthy. Coordinator code only wires +those ports and forwards stable compatibility calls. + +The three cumulative milestones—durability/submission, +concurrency/finalization, and observability/cleanup—are complete. The final +tree satisfies the [invariants](invariants.md), the structural result is +recorded in [Structure](structure.md), and the full evidence is recorded in +[Validation](validation.md). + +The implementation is organized into the review stack in +[Stacked PRs](stacked-prs.md). This document intentionally avoids commit IDs so +restacking cannot make the completion record stale. diff --git a/docs/prism-coordinator-refactor/stacked-prs.md b/docs/prism-coordinator-refactor/stacked-prs.md new file mode 100644 index 000000000..a7e0a1499 --- /dev/null +++ b/docs/prism-coordinator-refactor/stacked-prs.md @@ -0,0 +1,46 @@ +# Stacked PR Review Plan + +The refactor is published as nine bottom-up PRs. PR 1 targets `2.x.x` — the +v2 release line, which tracks `1.x.x` — while `1.x.x` remains the v1 +maintenance line; each later PR targets the branch immediately above it so +reviewers see only that slice. + +The stack was originally derived from `36267d6` and rebased onto `2.x.x` at +`56fb362` (the `v1.1.0` release commit) on 2026-08-17. That rebase replayed +all nine slices with no conflicts: the release commit touches only `VERSION`, +the two crate manifests, `Cargo.lock`, and the release notes, none of which +the stack modifies. When `2.x.x` advances again, rebase the whole stack the +same way rather than merging the new tip into individual slices. + +Stack commits are GPG-signed. An earlier waiver (owner decision, 2026-08-17) +applied while the publishing environment held no signing key; the `2.x.x` +rebase was performed where a key was available, so every slice now carries a +signature. No branch rule requires them. + +| PR | Branch | Scope | +| --- | --- | --- | +| [#73](https://github.com/Qbit-Org/qbit-mining-bootstrap/pull/73) | `prism-test-shards` | mechanical test sharding and shared fixtures; no production behavior | +| [#74](https://github.com/Qbit-Org/qbit-mining-bootstrap/pull/74) | `prism-coordinator-core-owners` | configuration/RPC, lifecycle, payout, templates, bundles, refresh, sessions, delivery, share writing, same-tip template reuse, retry pacing, observed coordinator locking, and per-client vardiff lock wiring | +| [#75](https://github.com/Qbit-Org/qbit-mining-bootstrap/pull/75) | `prism-audit-artifact-owner` | PostgreSQL publication order, audit filesystem authority, compilation/verification, replay, retention, and migration/process gates | +| [#76](https://github.com/Qbit-Org/qbit-mining-bootstrap/pull/76) | `prism-block-candidate-submission` | durable candidate replay, attempt marking, bounded retry heartbeats/backoff state, finalize-only replay pacing, terminalization, and coordinator ports | +| [#77](https://github.com/Qbit-Org/qbit-mining-bootstrap/pull/77) | `prism-share-submission` | single-snapshot share classification, bounded duplicate tracking, and dedicated share-accounting synchronization | +| [#78](https://github.com/Qbit-Org/qbit-mining-bootstrap/pull/78) | `prism-vardiff-finalization` | per-client vardiff/idle-retarget synchronization, named accepted-block finalization phases, and the documented B3 decision | +| [#79](https://github.com/Qbit-Org/qbit-mining-bootstrap/pull/79) | `prism-observability-http` | cached health, separate first-job/coverage-loss grace, complete cached metrics, and audit/public HTTP facade | +| [#80](https://github.com/Qbit-Org/qbit-mining-bootstrap/pull/80) | `prism-final-ownership-cleanup` | reorg reconciler, synchronized metrics snapshots including hot-lock/candidate gauges, watchdog owner, compatibility cleanup, and temporary-seam removal | +| [#81](https://github.com/Qbit-Org/qbit-mining-bootstrap/pull/81) | `prism-refactor-documentation` | final roadmap, ownership audit, validation evidence, stack guide, and ledger migration operations | + +Each PR targets the branch immediately above it; PR 1 targets `2.x.x`. Put the +stack position, base branch, dependency, focused validation, and any operator +impact in every PR body. PR 3 must call out the required existing-database +migration and maintenance-window lock; PR 9 carries the durable operator guide. + +Address review feedback from the bottom of the stack upward. Amend each fix +into its owning signed commit, then rebase and sign every descendant so each PR +continues to contain one intentional slice. Push rewritten branches together +with explicit force-with-lease expectations; stop if any remote tip changed. + +Validate focused behavior per PR. Run cumulative PRISM discovery after PR 2, +PostgreSQL/Rust after PR 3, the image build after PR 6, and the complete final +matrix on PR 9. Merge bottom-up. If the repository squash-merges a lower PR, +rebase the remaining branches onto the new `2.x.x` tip and force-push only with +lease. diff --git a/docs/prism-coordinator-refactor/structure.md b/docs/prism-coordinator-refactor/structure.md new file mode 100644 index 000000000..fc2163ca0 --- /dev/null +++ b/docs/prism-coordinator-refactor/structure.md @@ -0,0 +1,80 @@ +# Final Structure + +## Coordinator boundary + +`PrismCoordinator` constructs owners, wires explicit ports, exposes the stable +facade used by existing tests/tools, binds listeners, handles signals, and +orders top-level shutdown. It does not own a background worker loop or a large +domain state machine. + +| Concern | Owner modules | +| --- | --- | +| configuration and RPC | `coordinator_config`, `rpc` | +| lifecycle and bounded execution | `background_services`, `bounded_executor`, `coordinator_shutdown` | +| payout, reorg, and CTV | `payout_state`, `reorg_reconciler`, `ctv_runtime` | +| templates, bundles, refresh, delivery | `template_artifacts`, `bundle_compiler`, `job_bundle`, `tip_refresh`, `job_delivery` | +| sessions, shares, candidates, finalization | `stratum_session`, `share_submission`, `share_writer`, `block_candidates`, `block_finalization` | +| vardiff | `vardiff_service` | +| audit, health, metrics, HTTP | `audit_artifacts`, `observability`, `progress_health`, `metrics`, `audit_http` | + +Owners hold their own mutable state, locks, bounded queues/executors, counters, +and lifecycle. Cross-owner calls use explicit immutable inputs or narrow ports; +there is no broad replacement context object. + +## Structural audit + +- The coordinator fell from 31,041 lines on the integrated base (`36267d6`) + to 9,615 lines, a 69% reduction. +- No coordinator method at or over 200 lines is a domain state machine. The + only such method is the 511-line constructor (configuration projection and + eager owner construction); the seven other methods at or over 100 lines are + listener startup, the ledger factory, four owner-construction factories, + and idle-bundle cache validation. +- Loop-shaped coordinator facade methods are narrow delegates. The only local + loop is bounded listener-startup retry, not a background worker. +- No leaf service imports the coordinator at module load. + `block_candidates.py` never references `prism_coordinator`: post-offer + finalization stays coordinator-owned at this layer and is reached through + the injected runtime reference on the service's ports. + `job_build_benchmark.py` is an executable composition-level benchmark that + intentionally constructs it. +- No PRISM module defines magic `__getattr__` or `__setattr__` forwarding. +- Docker Ruff (`E4,E7,E9,F`) reports no unused imports. Its remaining 54 + findings are all F821 on lazily evaluated type annotations that + deliberately reference coordinator-owned types without importing them (no + runtime evaluation under `from __future__ import annotations`; importing + would recreate the reverse dependencies the extraction removed). Remaining + compatibility exports and descriptors have direct in-repository callers + and delegate to one owner. + +The earlier approximate 3,000-line aspiration is not a release requirement. +The remaining size is an explicit compatibility/composition exception: 636 +mostly narrow facade and wiring methods (including 41 lazy `_ensure_*` +factories) preserve existing private integration fixtures and public +behavior. Moving those methods to mixins would only hide +the composition surface, while deleting demonstrated callers would be a +breaking API/test migration rather than ownership extraction. Future changes +should remove a facade only when its callers can migrate without weakening the +compatibility contract. + +## Optional further decomposition + +There is still useful follow-up work, but it is separate from domain ownership: + +- About 900 lines of `_Coordinator*` port adapters can move to a + `coordinator_adapters` module. This is a low-risk file-size improvement, not a + new ownership boundary. +- The constructor plus 41 lazy `_ensure_*` factories can become a typed + component builder. This would clarify the dependency graph, but it must + preserve lazy startup, shutdown order, and partial test construction. +- Remaining cohesive orchestration can move to existing owners: live chain and + template validation, payout/CTV policy construction, synchronous candidate + credit, idle-bundle cache validation, and progress eligibility assembly. +- The largest reduction requires migrating the many test/tool construction + sites and then deleting their private compatibility delegates and + descriptors. Do this as an explicit compatibility cleanup, not with mixins or + magic forwarding. + +Those changes are optional because current domain state already has one owner. +They should be proposed after the review stack lands so they do not obscure the +behavior-preserving extraction. diff --git a/docs/prism-coordinator-refactor/validation.md b/docs/prism-coordinator-refactor/validation.md new file mode 100644 index 000000000..2f77d0525 --- /dev/null +++ b/docs/prism-coordinator-refactor/validation.md @@ -0,0 +1,135 @@ +# Validation Strategy + +Validation follows risk and invalidation scope. Focused checks run per slice; +broad checks run at cumulative milestones and once more on the final tree. + +## Per slice + +Run: + +```sh +python3 -m py_compile +python3 -m unittest +git diff --check +``` + +Inspect the complete slice diff and confirm no leaf module imports +`lab.prism.prism_coordinator`, including lazily inside a function. A leaf that +needs coordinator-owned behaviour reaches it through an injected seam that +carries the full call shape, rather than asking the coordinator for a fact and +re-deriving the decision itself. `BlockCandidatePorts.submit_candidate`, whose +production implementation is +`PrismCoordinator._land_block_candidate_submission`, is the pattern to copy: +the leaf states what it has (an already-created node submission, an +already-held disposition) and the coordinator resolves the entrypoint per +call. + +The invariant covers runtime modules under `lab/prism/`. `job_build_benchmark.py` +is a standalone benchmark entrypoint rather than a leaf owner and still imports +the coordinator directly; it is the one accepted exception. + +Add PostgreSQL, Rust, or process tests immediately when the slice changes those +boundaries; do not defer a directly relevant failure to a milestone. + +## Cumulative PRISM regression + +Use the current shard names discovered in `tests/`. At minimum include the +candidate, vardiff, retained-job, share-writer, payout, job-builder, metrics, +tip-refresh, publication-boundary, initial/reconnect, hot-path, shutdown, CTV, +progress-health, audit, and public-API suites. The exact module list may grow as +new direct owner suites are added. + +## Full Python discovery + +Full discovery includes fixtures that create temporary Git repositories, so +scope the global-config override to this command only: + +```sh +GIT_CONFIG_GLOBAL=/dev/null \ + python3 -m unittest discover -s tests -p 'test_*.py' +``` + +Never export `GIT_CONFIG_GLOBAL` and never use it for repository commits. + +## PostgreSQL, Rust, and live validation + +Run when a milestone touches persistence, payout, shares, audit, candidates, +vardiff, or finalization: + +```sh +make test-prism-postgres-ledger +cargo test -p qbit-prism +``` + +At the concurrency/finalization and final milestones, run: + +```sh +make test-prism-stratum-regtest-live +make test-prism-stratum-postgres-regtest-live +``` + +If qbitd, qbit-cli, PostgreSQL, Docker, Rust, credentials, or another real +prerequisite is unavailable, record `UNAVAILABLE`, the missing prerequisite, +and the exact unrun command. Do not call it a pass. + +## Docker compile, lint, and build + +Lint only in Docker: + +```sh +docker run --rm \ + -e PYTHONPYCACHEPREFIX=/tmp/pycache \ + -v "$PWD:/work:ro" -w /work \ + python:3.14-slim-trixie \ + python -m compileall -q docker lab tests examples scripts + +docker run --rm \ + -v "$PWD:/work" -w /work \ + ghcr.io/astral-sh/ruff:0.12.5 \ + check --select E4,E7,E9,F lab/prism tests + +docker build -f lab/prism/Dockerfile -t qbit-prism-refactor-check . +``` + +Focused slices may lint only their changed Python paths. Milestones lint the +complete PRISM paths. The image build is required after B2 and on the final +tree. + +## Final structural and diff audit + +Before declaring completion: + +- no leaf service imports the coordinator; +- mutable domain state has one owner and no drifting scalar mirror; +- coordinator worker loops and large domain state machines have moved; +- compatibility re-exports/delegates have a demonstrated caller or are gone; +- no blocking I/O occurs under coordinator/session locks; +- queues, executors, cancellation, and shutdown joins are bounded; +- public schemas, metrics, environment defaults, and wire payloads are stable; +- `git diff --check`, status, target-branch diff, and generated-artifact hygiene + are clean; +- cumulative regression, full discovery, PostgreSQL/Rust, Docker lint/build, + and live-or-unavailable evidence all reflect the final tree. + +## Final evidence + +Recorded on 2026-08-16 on the re-derived stack tip, integrated through +`origin/1.x.x` at `36267d6` (#120). The Docker daemon on this host requires +root; Docker-backed checks ran via `sudo`. + +| Check | Result | +| --- | --- | +| PRISM Python discovery (`test_prism_*.py`) | 1,601 passed | +| full Python discovery | 1,956 passed, 2 skipped (ckpool binary absent); `GIT_CONFIG_GLOBAL=/dev/null` was scoped only to this command | +| PostgreSQL ledger integration | passed via `sudo bash test/test-prism-postgres-ledger.sh`, including the typed share-replay assertions, the #120 carry-forward equivalence/drift checks, and the three A1 publication gates (transition parity, migration M0-M11, cross-process fencing C1-C3) | +| Rust workspace | 222 passed (`cargo test --locked --workspace --all-targets`), including the audit CLI canonicalizer round-trip regressions | +| Docker Python compile | passed | +| Docker Ruff (`E4,E7,E9,F`) | 54 findings, all F821 on lazily evaluated type annotations that deliberately reference coordinator-owned types without importing them (never evaluated at runtime under `from __future__ import annotations`); every removable finding was cleaned in the final ownership layer | +| PRISM image | built and tagged (`qbit-prism-refactor-check`) | +| Compose | 34 configuration-validation tests passed (mainnet contract, prism profile, mining profiles), including the metrics-refresh and #120 environment pass-throughs | +| structural and target diff audit | passed; see [Structure](structure.md) | + +`make test-prism-stratum-regtest-live` and +`make test-prism-stratum-postgres-regtest-live` are `UNAVAILABLE`: their +doctor stops at `Required executable not found: qbitd` before either live +test can run. `qbitd` and `qbit-cli` are absent from the host path. diff --git a/docs/prism-ledger-ops.md b/docs/prism-ledger-ops.md index d824ee1d2..21c50d1f1 100644 --- a/docs/prism-ledger-ops.md +++ b/docs/prism-ledger-ops.md @@ -86,6 +86,183 @@ path through `qbit_prism_shutdowns_total`, `qbit_prism_shutdown_non_writer_drain_seconds`, and `qbit_prism_shutdown_release_withheld_total`. +### Heartbeat Monitor Lateness and In-Process Stalls + +The writer-lease heartbeat's timing policy (`lab/prism/writer_lease_timing.py`) +guarantees that a coordinator whose guard session has died hard-exits before a +replacement may adopt the lease, *provided* the monitor thread that takes that +decision wakes no later than `scheduler_slack + monitor_interval` after its +poll is due (0.55 s at the shipped numbers; the module docstring derives this +as inequality (4)). That proviso is an assumption about the process, not a +property of the policy. Issue #227 recorded a 0.648 s monitor wake delay on +`union-mainnet` with the host idle: the stall was in-process, an interpreter +held by a multi-second payout-window rescan. + +Two consequences are exported and one is decided. + +**Attribution that survives GIL stalls.** Every guard statement returns its own +server-side execution time, so `guard_sql` in +`qbit_prism_lease_heartbeat_phase_seconds{phase}` and its `worst_` twin is now +server execution, the new `guard_client_resume` phase is the part of the round +trip during which PostgreSQL had already answered and this process had not +resumed, and `scheduler_delay` is the whole process-side residual +(`guard_slot_wait + guard_sql + scheduler_delay = total`; `guard_client_resume` +is the in-round-trip share of `scheduler_delay`). `guard_sql` is now bounded by the +server rather than by the client: each statement can run for at most the 0.5 s +server-enforced statement timeout, and a renewing verification may lawfully run +two statements inside one guard slot (the attribution recheck), so `guard_sql` +sums both and can legitimately reach about 1.0 s on a two-statement attempt +(`WRITER_LEASE_VERIFICATION_MAX_STATEMENTS` × 0.5 s) and cannot exceed that; the +`(N stmt)` count in the attempt summary says which case applies. Read a large +`guard_client_resume` as "this process, not PostgreSQL". + +**The monitor's lateness as re-armable signals.** +`qbit_prism_lease_heartbeat_monitor_wake_delay_seconds` stays the lifetime +high-water mark for dashboard continuity. Alert on the new families instead, +all fixed-cardinality: + +- `qbit_prism_lease_heartbeat_monitor_wake_lateness_seconds` (histogram; every + poll's lateness, closed bucket set); +- `qbit_prism_lease_heartbeat_monitor_late_wakes_total{slack_fraction}` with + `slack_fraction` in `0.5`, `0.8`, `1.0` (wakes at least that fraction of + `scheduler_slack` late; a counter, so `increase(...[5m]) > 0` on the `1.0` + series re-expresses the wake-delay alert without a never-resetting gauge); +- `qbit_prism_lease_heartbeat_monitor_wake_delay_window_max_seconds` (the worst + lateness in the trailing 300 s; falls back to zero once a stall ages out); +- `qbit_prism_lease_heartbeat_monitor_wake_delay_record_age_seconds` (how old + the lifetime record is, so an exit message's figure can be dated); +- `qbit_prism_lease_heartbeat_monitor_exit_guarantee_breaches_total` and + `qbit_prism_lease_heartbeat_monitor_worst_exit_guarantee_overrun_seconds` + (wakes later than `max_guaranteed_monitor_lateness`, and the worst amount + by which the exit could have landed after the adoption edge); +- `qbit_prism_lease_heartbeat_policy_seconds{term="max_guaranteed_monitor_lateness"}` + (the bound itself, for alert expressions); +- `qbit_prism_lease_heartbeat_stall_probe_samples_total` and + `..._stall_probe_suppressed_total` (stack samples the stall probe took, and + triggers it refused under its rate limit); +- `qbit_prism_process_gc_pause_seconds{generation}` (histogram), + `qbit_prism_process_gc_last_pause_seconds{generation}` and + `qbit_prism_process_gc_max_pause_seconds{generation}`: cyclic-collector + pause *durations* from `gc.callbacks`. These complement, and do not replace, + the `qbit_prism_process_gc_*` count families from issue #226, which report + how many passes ran and what they freed but not how long any pass held the + interpreter. + +**What a breach means and the decided response.** A wake later than +`scheduler_slack` (0.50 s) is a *lateness-beyond-slack breach*: the assumption +the policy budgets was exceeded. A wake later than +`max_guaranteed_monitor_lateness` (0.55 s) is an *exit-guarantee breach*: on +that beat, had the guard session died, this process could have outlived the +successor's adoption edge by the overrun. The monitor does **not** tighten its +cap or hard-exit on a breach (that would restart healthy coordinators under a +transient stall, issue #212's failure mode, without moving the exit earlier on +the beat that breached). It accepts the residual because every effect that +could escape is fenced: every ledger write joins the exact-session lease CTE +and matches nothing once adoption rewrites the session token, and every +external effect runs behind a synchronous exact-session verification that +hard-exits the moment the row names another session. A breach is therefore +counted, its overrun kept, and one structured warning is logged +(`prism coordinator: WARNING ledger lease heartbeat monitor wake delay ... exceeded +the ... scheduler slack`), rate-limited to at most 3 per 60 s and never from a +blocking write. A wake at least half a slack late additionally takes a stack +sample of the running threads, under the same 3-per-60 s limit +(`LEASE_MONITOR_STALL_PROBE_MAX_SAMPLES_PER_WINDOW` / +`LEASE_MONITOR_STALL_PROBE_WINDOW_SECONDS` in `writer_lease_timing.py`), so the +instrument cannot turn a stall into a stall storm. + +**Operator action on a lateness-beyond-slack warning.** + +1. Read the warning: it states whether exit-before-adoption still held, the + overrun if not, the phase attribution of the last and worst attempts, the + worst GC pause and its generation, and the sampled thread stacks. The + stacks name the frame that held the interpreter; a payout-window rescan + (`reconcile_invalidation`) or a large collection are the known causes. +2. Confirm it was in-process rather than host contention: PSI, steal, and + cgroup throttling on the host should be quiet. If they are not, the host is + the problem and the policy's slack is not the lever. +3. Check `qbit_prism_lease_heartbeat_monitor_exit_guarantee_breaches_total`. + An isolated breach with no coincident guard loss changed nothing; the + fences carried it. Repeated breaches mean the workload has outgrown the + interpreter's ability to schedule the monitor, and the durable fix is the + out-of-process watchdog tracked in issue #130 — not a larger slack, which + only lengthens failover, and not a shorter cap, which manufactures issue + #212. +4. Reduce the stall source: if the rescan attribution + (`qbit_prism_payout_window_full_rescan_seconds`) shows multi-second full + rescans, that is the workload to bound. If the worst GC pause is the stall, + the heap growth tracked by the issue #226 families is the lead. +5. Do not disable the heartbeat, do not raise only the failure budget, and do + not weaken PostgreSQL durability. + +### Orphaned Locks and Bounded Startup Acquisition + +A coordinator that vanishes without closing its sockets — network partition, +`SIGSTOP`, VM pause — can leave a Postgres backend idle in transaction, still +holding the `qbit_ledger_writer_lease` row lock its landing CTE took. Without +countermeasures the successor's startup lease upsert queues behind that lock, +inside ledger construction and before the watchdog arms, until kernel TCP +keepalive teardown (hours at OS defaults): a full-pool availability outage. +Settlement correctness is unaffected — the outbox row stays pending and +replays — only availability is at stake. + +Two independent layers bound this. First, every Postgres session the +coordinator opens (the pooled native client, the dedicated lease-guard +session, and the psql subprocess backend) carries session guards: +`idle_in_transaction_session_timeout` +(`PRISM_POSTGRES_IDLE_IN_TRANSACTION_TIMEOUT_SECONDS`, default 15) makes the +server abort an orphaned transaction and release its locks, and the +server-side keepalive GUCs (`PRISM_POSTGRES_TCP_KEEPALIVES_IDLE_SECONDS`, +`PRISM_POSTGRES_TCP_KEEPALIVES_INTERVAL_SECONDS`, +`PRISM_POSTGRES_TCP_KEEPALIVES_COUNT`, defaults 30/10/3) bound the server's +teardown of a socket toward a vanished client at 30 + 3x10 = 60 seconds, the +backstop where the idle-in-transaction timer does not apply. + +Second, the startup lease upsert and adoption CAS each run under a bounded +lock deadline (`PRISM_LEDGER_LEASE_ACQUIRE_LOCK_TIMEOUT_SECONDS`, default 5) +with a bounded retry (`PRISM_LEDGER_LEASE_ACQUIRE_ATTEMPTS`, default 5). Each +timed-out attempt is logged with its attempt number and the underlying error. +The retry budget (5x5s = 25s) is sized to outlast the idle-in-transaction +timeout — which runs on the blocking backend's own clock, from when its +transaction went idle rather than from when the successor started retrying — +so an orphaned lock is normally reaped mid-budget and startup self-heals +without operator action. An acquisition that never completes within the budget +fails construction with a `RuntimeError`, exiting the process visibly for the +supervisor to restart. That error names the lock conflict as the likeliest +cause but does not assert it: a connect timeout, an exhausted connection-pool +slot, and a server that is merely overloaded all expire the same deadline, so +read the chained cause it quotes before assuming a stuck transaction. Waiting +for a *live* holder's lease TTL to expire is unchanged — that outer wait is +intended failover behaviour; only the per-statement lock wait is bounded. + +Arming that deadline widens the orphan shape slightly, and the trade is +deliberate. A deadline is what makes the native client wrap a statement in an +explicit transaction whose `COMMIT` is a separate client message, so the +startup acquire is now transaction-scoped where it used to be plain +autocommit. A coordinator that vanishes mid-acquire can therefore orphan the +lease-row lock itself — on a path that previously could not produce one, +because autocommit leaves no transaction for the server to hold open. The two +orphans are not comparable in cost: the one this fix removes was bounded only +by TCP keepalive teardown, hours at OS defaults, while the one it introduces +is bounded by `PRISM_POSTGRES_IDLE_IN_TRANSACTION_TIMEOUT_SECONDS` (default +15) and delays a successor by at most that. It is also why the session guard +is not optional and why disarming it is rejected at construction: without it +the acquisition deadline would trade an unbounded wait for an unbounded wait +of its own making. + +The session guards carry three deployment caveats. The guards travel as +libpq startup options, so native connections now always set the `options` +connect parameter: a deployment routing `PRISM_DATABASE_URL` through a +connection pooler that rejects startup options (older PgBouncer builds) will +fail at connect rather than silently drop them, visibly at coordinator +startup. The default compose topology connects directly to `prism-postgres` +and is unaffected. The psql-subprocess backend delivers the same guards +through `PGOPTIONS`, so a wrapper script standing in for `psql` +(`PRISM_POSTGRES_PSQL_COMMAND`) that does not forward its environment drops +them without any error. On a Unix-socket DSN the `tcp_keepalives_*` GUCs are +ignored — accepted and inert — leaving +`PRISM_POSTGRES_IDLE_IN_TRANSACTION_TIMEOUT_SECONDS` as the guard that still +applies; it is the one that covers the orphaned lease-row lock in any case. + ## Block Candidate Outbox A block-worthy share transaction also inserts an immutable intent into @@ -136,6 +313,100 @@ Landing-path observability lives on `/metrics`: `qbit_prism_accepted_parent_preview_wait_timeouts_total`, `qbit_prism_prior_balances_reads_total` / `_read_last_seconds` / `_read_max_seconds`, and `qbit_prism_startup_phase_seconds{phase=...}`. + +Read-slot ledger operations additionally split local admission from server +execution, labelled by `operation`: +`qbit_prism_ledger_read_calls_total`, +`qbit_prism_ledger_read_gate_wait_seconds_total` / `_max` and +`qbit_prism_ledger_read_gate_timeouts_total` (coordinator-local admission — +waiting for `PRISM_POSTGRES_READ_CONCURRENCY`; a gate timeout means no +statement was ever sent), against +`qbit_prism_ledger_read_execute_seconds_total` / `_max` and +`qbit_prism_ledger_read_execute_timeouts_total` (PostgreSQL, including the +tail a cancelled statement spends returning). One duration covering both is +what made the #211 exhaustion unattributable from a scrape: the replay +enumeration reported `exceeded 5s` while its statement deadline was barely +touched and the database showed no blocked backends. Read the two halves +before widening any budget — a rising gate series is contention in this +process, a rising execute series is the database. + +Since #224 the same split covers the payout-window and prior-balances reads +under fixed operation names: `payout_window_snapshot`, +`payout_window_delta` and `prior_balances_after_pool_block` on the read +slot, and `current_prior_balances` on the **writer lock**, where the +landing's prior-balances check has always taken it. The `operation` label +is closed (`PRISM_LEDGER_READ_OPERATIONS` in +`lab/prism/accepted_preview_telemetry.py`); any other name folds into +`other`. The operator validation contract for these series, the landing +phase / reconcile caller / full-rescan families, and the 4 s and 5 s +accepted-preview boundaries is in `docs/prism-overload-alerts.md`, +"Issue #224". + +Pending block-candidate enumeration +(`pending_block_candidate_rows`, the `replay-outbox-query` phase) is a +read-only single-snapshot statement and takes the bounded read slot, not the +writer lock (#211). It is therefore unaffected by an accounting write holding +the writer gate. The page it returns is advisory: a candidate can land or be +terminalized between the snapshot and the caller's decision, so every terminal +transition re-checks under the writer lease — `mark_block_candidates_abandoned` +re-asks `qbit_pool_blocks` inside its fenced `UPDATE`, and the single-hash +terminal updates additionally require `state = 'pending'`. + +`PRISM_BLOCK_REPLAY_PAGE_SIZE` controls the number of candidate payloads in +each cursor page during startup and forced ancestor re-drive (default 1024, +allowed range 1–1024). Set it to `1` to reduce combined JSON decoding pauses +when pending candidates contain large payout windows (issue #255). The walk +still visits every page and requires a short page, including an empty final +page when necessary, before declaring enumeration complete. Timeouts, missing +cursors, and cleanup backpressure still leave the job-build gate closed. + +This is an opt-in mitigation, not a byte or memory bound: one candidate can +still exceed the lease timing envelope, and restored candidates remain queued +for accounting. Smaller pages add database and chain-RPC round trips. Ordinary +steady-state polls and the legacy cursorless fallback retain their existing +batch limits. Neither lease protection nor accounting rules change. After +deploying with the setting, check enumeration logs for `limit=1`, completion +of pending accepted-block accounting, and full subscribe/authorize/notify +health on both Stratum listeners. Restore `1024` to undo the tuning. + +For already accepted blocks that still cannot complete during normal replay, +`python3 -m lab.prism.recover_pending_blocks` provides a temporary offline +recovery command. Run it from an image containing the command, using the +managed coordinator's environment, signing keys, network and audit volumes. +PostgreSQL and qbitd must remain available. + +1. Inspect the exact hashes with a metadata-only plan: + ```sh + python3 -m lab.prism.recover_pending_blocks \ + --block-hash "$PARENT_HASH" --block-hash "$CHILD_HASH" + ``` + This takes no writer lease and reads no candidate payloads. Every hash must + be on the active chain and have a pending outbox row, or already have + submitted/confirmed accounting and persisted audit evidence. Include any + pending parent; the command sorts the selection by height. +2. Stop the normal coordinator and its restart supervisor. Launch a fresh + container with the same managed configuration, replacing its normal + coordinator command with the command above plus `--apply`. Recovery refuses + a live writer lease, acquires a fresh guarded session, and runs the normal + lease heartbeat and accepted-block accounting. It loads one selected + candidate at a time, verifies its stored identity and chain header, and + completes the outbox only after the normal finalizer succeeds. It starts + no Stratum listeners and makes no `submitblock` calls. +3. Check for a final `success` event and exit status zero. Failure stops the + run; repeat the same allowlist to resume after investigating. Completed + blocks are verified and skipped. The processing and cleanup deadline is + 600 seconds after construction (`--timeout-seconds`, range 1–3600). +4. Restart the managed coordinator, verify the selected blocks' accounting + and audit evidence, and check full subscribe/authorize/notify health on + both Stratum listeners. + +Recovery uses Python's cooperative JSON codecs only in its fresh process to +reduce long interpreter stalls while the heartbeat runs. This costs throughput +and still holds one complete candidate in memory; it does not replace the +bounded-payload work in #255. Lease timings, signing identity, and payout rules +remain unchanged. Logs include JSON progress events with schema +`qbit.prism.pending-block-recovery.v1` alongside normal coordinator logs. + Alert before the landing deadline is exhausted, not after: page when `qbit_prism_prior_balances_read_max_seconds` exceeds ~20% of the landing budget or the poll budget, when any @@ -198,6 +469,11 @@ if cleanup fails, the candidate remains pending and can still converge to submitted on later chain evidence. Replay carries the database row's block hash separately from candidate JSON, so malformed payloads can be quarantined using the authoritative outbox key instead of replaying forever. +If qbit has already returned the candidate outcome but durable outbox +finalization fails, replay resumes only that finalization step with the same +bounded pacing. It does not call `submitblock` again, recount an accepted block, +rebuild or republish audit evidence, or reacquire a share-writer floor already +released after the known outcome. The block submitter heartbeat carries its current phase, including replay query, node RPC, lock admission, audit, persistence, and finalization. A stale @@ -237,13 +513,77 @@ empty; `stale-grace` marks a prior-tip share credited by the coordinator's short stale-grace policy. Reward-window queries still count these rows because they are accepted shares, while audits can distinguish them from normal current-tip shares. Audit bundles containing a credited row use -`qbit.prism.audit-bundle.v1.1`; external auditors must upgrade before operators -enable stale-grace crediting. +the logical `qbit.prism.audit-bundle.v1.1` schema, and the window is enabled by +default on every chain, so external auditors must run a release that accepts it. Deployments that run with `PRISM_POSTGRES_INIT_SCHEMA=0` must apply -`crates/qbit-prism/sql/001_share_ledger.sql` before starting upgraded -coordinators. Otherwise share inserts will fail because the `credit_policy` -column and updated window function signatures are missing. +`crates/qbit-prism/sql/001_share_ledger.sql` before starting any upgraded +coordinator. The file is the cumulative, idempotent schema initializer and +migration path, despite its `001` name. Skipping it can break share inserts, +reward-window calls, pool-block confirmation/reactivation, and audit evidence +publication because required columns, functions, and the durable publication +ordinal will be missing. + +Apply the file in a single transaction and stop the PRISM share writer +first. The script enforces this itself with a `BEGIN`/`COMMIT` wrapper, so +even a plain autocommit `psql -f` runs as one transaction — but pass +`--single-transaction` (or `-1`) anyway, together with `ON_ERROR_STOP`, so +the intent is explicit and a failure cannot strand an open transaction: + +```sh +psql "$PRISM_DATABASE_URL" --single-transaction -v ON_ERROR_STOP=1 \ + -f crates/qbit-prism/sql/001_share_ledger.sql +``` + +Before that wrapper existed, a per-statement autocommit apply could commit +the carry-forward summary triggers before the summary seeding block later in +the file ran. If the apply was interrupted in that gap, or a still-running +writer confirmed or reversed a block in it, the summary received only those +post-trigger deltas: every miner's balance was silently under-reported by +their full pre-upgrade carry, and the seed's emptiness guard locked that +partial state in permanently. The schema's seed guard now compares the +summary against the carry history it summarizes and repairs a partial +summary on the next apply. The writer must still be stopped during the +apply: concurrent mutations block on the apply's table locks and then fire +the freshly (re)created triggers after it commits, which invites long +lock waits and deadlocks even though the apply itself is atomic. Note that +combining `--single-transaction` with the script's own `BEGIN`/`COMMIT` +wrapper makes psql print two harmless warnings ("there is already a +transaction in progress" / "there is no transaction in progress"); the apply +is still exactly one transaction. + +### Audit publication ordering migration + +Existing databases must receive the new `audit_publication_sequence` migration. +It creates and validates a bigint sequence, adds the nullable pool-block column, +deterministically backfills confirmed and inactive rows by `found_at` and +`block_hash`, adds a unique index and state constraint, advances the allocator +beyond every retained ordinal, and replaces confirmation/reactivation functions +so each new durable confirmation receives an ordinal. Exact confirmation replay +preserves its prior ordinal. Historical inactive rows are backfilled so later +reactivation can retain that already-published ordinal without allocating a new +audit publication. + +The migration includes `ALTER TABLE` operations that require PostgreSQL's +`ACCESS EXCLUSIVE` table lock. They wait for existing readers and writers and +can interrupt new reads as well as writes while held. A serialized phase takes +a transaction-scoped advisory lock and a `SHARE ROW EXCLUSIVE` lock on +`qbit_pool_blocks`, and the unique index is built non-concurrently. Treat the +whole migration as read-impacting: stop the old coordinator or use a reviewed +maintenance window, take the normal database backup, and apply the file with +`ON_ERROR_STOP` inside a single transaction (see the apply requirements above), +using the same database role and schema search path as PRISM. For example: + +```sh +psql "$PRISM_DATABASE_URL" --single-transaction -v ON_ERROR_STOP=1 \ + -f crates/qbit-prism/sql/001_share_ledger.sql +``` + +With `PRISM_POSTGRES_INIT_SCHEMA=1`, coordinator construction applies the same +script before listeners open. The migration is rerunnable and tested across +fresh, legacy, partial, concurrent, malformed, and bigint-boundary states; it +fails closed instead of accepting a conflicting sequence, column, index, or +constraint definition. `qbit_shares_since_template_height(min_template_height)` supports operational replay and frontend recovery. It returns accepted shares at or above the @@ -270,6 +610,78 @@ supplied active tip, must not be reversed by that path. qbit coinbase maturity is 1000 blocks, so operators must not mark pool payouts mature before `block_height + 1000`. +## Publication Ordinal Rollback And Schema Revert + +Schema initialization adds a durable publication ordinal to accepted pool +blocks: `qbit_pool_blocks.audit_publication_sequence`, allocated from +`qbit_audit_publication_sequence_seq` at the durable prepared -> confirmed +boundary, unique across the table, and required by a validated CHECK +constraint on every confirmed row. Historical confirmed and inactive rows are +backfilled deterministically in `(found_at, block_hash)` order the first time +the migration runs. + +Rolling back coordinator code does not require reverting this schema. The +`qbit_pool_blocks_assign_publication_ordinal` BEFORE trigger assigns the next +ordinal to any confirming write that omits it, so a pre-ordinal coordinator +that confirms with a plain `chain_state` UPDATE keeps satisfying the +constraint. That trigger is the primary rollback path; prefer it. + +`crates/qbit-prism/sql/001_share_ledger_revert_audit_publication_sequence.sql` +is the second path, for the cases the trigger cannot cover: rolling back more +than one release, or aligning a live ledger with a restored pre-migration +base backup. It returns `qbit_pool_blocks` to its pre-ordinal shape by +dropping, in order, the CHECK constraint, the assignment trigger and its +function, the unique index, the column, and the sequence, then restores the +pre-ordinal `qbit_confirm_pool_block` body. The file is one transaction +serialized behind the same advisory lock as the forward migration: any +failure aborts the whole revert, and re-running it after the failure is +resolved -- or on an already-reverted or never-migrated schema -- is safe. An +object the migration did not create, such as an operator view over the +ordinal column, fails the revert loudly instead of being dropped. + +Before applying the revert: + +1. Stop the coordinator and anything else writing to the ledger. The revert + takes an ACCESS EXCLUSIVE lock on `qbit_pool_blocks`, so it stalls behind + a live writer. Worse, a post-migration coordinator left running (or + restarted afterwards) confirms blocks by assigning the ordinal explicitly: + every such confirmation fails with `column "audit_publication_sequence" + does not exist` the moment the revert commits, and a post-migration + coordinator restarted with schema initialization enabled immediately + re-applies the forward migration, silently undoing the revert. +2. Take a base backup, or verify the continuous WAL archive covers this + point. The revert discards every assigned publication ordinal; nothing can + recover them afterwards except that backup. + +What is permanently lost: the recorded confirmed-publication order. The +ordinal is allocated when a block durably reaches `confirmed` and is reused +across exact replay and inactive -> confirmed reactivation, so it is the only +record of the order in which blocks were actually published. After the +revert, publication currency falls back to what pre-ordinal code used to +select current evidence -- `found_at`/`block_height` read order -- which is +not that durable publication order. Re-applying `001_share_ledger.sql` later +re-adds the column and backfills deterministically by +`(found_at, block_hash)`, but that is a fresh assignment: it does not +reproduce the ordinals observed before the revert, and external consumers +that recorded pre-revert ordinals will see the sequence renumbered. + +Apply the revert with the ledger role whose `search_path` selects the PRISM +schema, inside a single transaction, and stop on the first error (the revert +script wraps itself in one `BEGIN`/`COMMIT` and relies on it, matching the +forward script): + +```sh +psql "$PRISM_DATABASE_URL" \ + --single-transaction \ + --set ON_ERROR_STOP=1 \ + -f crates/qbit-prism/sql/001_share_ledger_revert_audit_publication_sequence.sql +``` + +The full round trip -- migrate, revert, confirm pre-ordinal-style, re-apply, +re-backfill without double assignment -- is exercised by +`tests/prism_postgres_a1_revert_gate.py` inside +`test/test-prism-postgres-ledger.sh`. + ## CTV Fanout Artifact Repair Schema initialization is also the idempotent repair path for deployed PRISM @@ -437,9 +849,10 @@ against the pin. Production builds using the git source provider must set `QBIT_GIT_COMMIT` to a full 40-character object ID. The environment doctor verifies that the resolved checkout is at that exact commit instead of trusting a mutable branch or tag. -Production also requires `PRISM_STRATUM_STALE_GRACE_SECONDS=0`; stale-credit -grace should be enabled only after the deployed verifier and accounting release -have an explicit compatibility proof for it. +Stale-credit grace (`PRISM_STRATUM_STALE_GRACE_SECONDS`, default 3) stays +enabled in production, mainnet included; reward windows that contain a credited +prior-tip share publish the logical `qbit.prism.audit-bundle.v1.1` schema, so +keep the deployed verifier and accounting release on a version that accepts it. The parent-chain selector is checked independently: `BITCOIN_CHAIN` and `BITCOIN_CHAIN_FLAG` must be an exact pair, including `mainnet` with diff --git a/docs/prism-overload-alerts.md b/docs/prism-overload-alerts.md new file mode 100644 index 000000000..d1e602a47 --- /dev/null +++ b/docs/prism-overload-alerts.md @@ -0,0 +1,1022 @@ +# PRISM Overload and Backlog Alert Specification + +Issue #188. This document specifies the alerting rules for PRISM's +landing-path, delivery, and backlog overload classes. It is a +specification only. + +## Status: not live + +**These alerts are not live.** This repository contains no Prometheus, +Alertmanager, or Grafana configuration and does not own deployment +monitoring. Nothing here fires until the separate deployment/Prometheus +repository transcribes these rules into its own rule files and applies +them. Treat this document as the source of truth for *what* to alert on +and *why*; that repository remains the source of truth for *how* the +rules are deployed, routed, and silenced. + +## Ordering: #184 must land before these rules are deployed + +**#184 must be present in the deployed PRISM build before the external +rule repository applies these alerts.** In the combined delivery, #184 is +integrated ahead of this specification; the requirement below governs +external rule deployment against any given running build. + +Every body-metric rule depends on a *successful* scrape, and the scrape +status of a degraded `/metrics` differs across #184: + +- **Before #184:** a stale complete snapshot was served as **HTTP 503**. + Prometheus records a failed scrape for a 503 and discards the response + body, so every series in this document went *absent* in precisely the + incident where the endpoint was degraded. +- **After #184:** a stale complete snapshot is served as **HTTP 200 with + freshness metadata**, so `qbit_prism_metrics_snapshot_stale`, + `..._age_seconds`, and the whole body are ingested and can be alerted + on. Only **warm-up, when no complete snapshot exists yet**, still + returns 503. + +Deploying these rules against a pre-#184 build would therefore invert +their meaning: an endpoint failing its own health contract would produce +no firing alerts at all, and that silence would be indistinguishable from +a healthy pool. Confirm #184 is in the target build, then apply these +rules, and always apply `PrismScrapeDown` (below) alongside them — body +metrics cannot report their own total absence, and the warm-up 503 window +remains unscrapable by construction. + +## Signal inventory + +All names and export sites below were verified against the source. Every +series is a single time series per target; none carries labels beyond the +target's own `job`/`instance`. Two rows include the accompanying #188 gauge +change: the new cap gauge and the corrected semantics of the +unresolved-transitions count. + +| Metric | Type | Exported at | Status | +| --- | --- | --- | --- | +| `qbit_prism_accepted_parent_preview_wait_timeouts_total` | counter | `lab/prism/metrics.py` | already present | +| `qbit_prism_accepted_parent_unresolved_transitions` | gauge, exact admission-fence depth | `lab/prism/metrics.py` | present, **semantics corrected by the accompanying #188 gauge change** | +| `qbit_prism_accepted_parent_unresolved_depth_max` | gauge | `lab/prism/metrics.py` | **added by the accompanying #188 gauge change** | +| `qbit_prism_accepted_parent_unresolved_oldest_seconds` | gauge, oldest over timestamped transitions, `-1` when none | `lab/prism/metrics.py` | already present | +| `qbit_prism_block_candidates_pending` | gauge, `-1` if unavailable | `lab/prism/metrics.py` | already present | +| `qbit_prism_block_candidate_oldest_pending_seconds` | gauge, `-1` if unavailable | `lab/prism/metrics.py` | already present | +| `qbit_prism_stratum_semantic_current_work_ratio` | gauge, `1.0` when no clients | `lab/prism/metrics.py` | already present | +| `qbit_prism_stratum_authorized_connections` | gauge | `lab/prism/metrics.py` | already present | +| `qbit_prism_refresh_pending` | gauge, 0/1 | `lab/prism/progress_health.py` | already present | +| `qbit_prism_refresh_pending_age_seconds` | gauge, `0` when not pending | `lab/prism/progress_health.py` | already present | +| `qbit_prism_metrics_snapshot_available` | gauge, 0/1; startup coverage only — stays `1` after warm-up even when the refresh is stale | `lab/prism/observability.py` | already present | +| `qbit_prism_metrics_snapshot_stale` | gauge, 0/1 | `lab/prism/observability.py` | already present | +| `qbit_prism_metrics_snapshot_age_seconds` | gauge, `-1` before first success | `lab/prism/observability.py` | already present | +| `up` | gauge, 0/1 | synthesised by Prometheus | not a PRISM series | + +What the gauge task adds and corrects: + +**Adds the cap.** Before it, the configured unresolved-depth cap +(`PRISM_ACCEPTED_PARENT_UNRESOLVED_DEPTH_MAX`, default 8, in +`lab/prism/coordinator_config.py`) was readable only from process +configuration, so no rule could express "at or near the cap" without +hard-coding a deployment-specific constant. Exporting the cap as +`qbit_prism_accepted_parent_unresolved_depth_max` makes the +`PrismUnresolvedParentDepthAtCap` rule a *ratio against the safety +contract itself*, which stays correct when an operator retunes the cap. + +**Corrects the count.** `qbit_prism_accepted_parent_unresolved_transitions` +now reports the **exact depth used by the admission fence** — every landed +transition whose durable bookkeeping is unresolved, including landed +transitions that carry no timestamp. It was previously derived from the +timestamped-ages list and so could under-report relative to the depth that +actually gates job issuance (`_await_pending_parent_payout_preview` in +`lab/prism/payout_state.py`). With the correction, rule 2 compares like +with like: the gauge and the fence agree exactly, so the at-cap alert fires +precisely when issuance stops. + +`qbit_prism_accepted_parent_unresolved_oldest_seconds` is deliberately +**not** changed to match that population. It remains the maximum age over +transitions with a known landing timestamp, and reports `-1` when there +are none. An untimestamped transition therefore contributes to the depth +gauge but not to the age gauge — which is correct, since no age is known +for it and none is invented. The practical consequence for alerting: the +depth signal (rule 2) is the complete one, and the age signal (rule 3) can +read `-1` or a low value while depth is nonzero. Use rule 2, not rule 3, +to judge whether the fence is close to blocking. + +Explicitly **not** used: current-tip coverage +(`current_tip_job_coverage`) and the initial-job queue series +(`pending_initial_jobs`, `pending_initial_job_capacity`). They are +similarly named but measure different populations; the semantic ratio is +the signal that accounts for template fingerprint, payout generation, and +client-session currency. + +## Threshold provenance + +Thresholds fall into two honestly distinct classes. + +**Configuration- and contract-derived (transfers to mainnet).** These are +read from the code's own safety contract and stay valid at any cadence: + +- the accepted-parent preview wait budget, 5.0 s + (`DEFAULT_ACCEPTED_BLOCK_PAYOUT_PREVIEW_WAIT_SECONDS` in + `lab/prism/payout_state.py`); +- the unresolved-depth cap, default 8 + (`DEFAULT_ACCEPTED_PARENT_UNRESOLVED_DEPTH_MAX` in + `lab/prism/coordinator_config.py`), compared as a ratio via the exported + gauge rather than as a literal; +- the pending-refresh health deadline, 15.0 s + (`DEFAULT_PRISM_HEALTH_PENDING_REFRESH_MAX_AGE_SECONDS` in + `lab/prism/coordinator_config.py`); +- the degraded-coverage line, 0.95, borrowed from the in-process health + evaluator, which treats a sustained loss of five percent of *current-tip* + job coverage as degraded in `lab/prism/observability.py`; the + semantic ratio is the stricter sibling of that coverage ratio and is not + itself thresholded in-process, so 0.95 is applied here by analogy rather + than read from an existing rule; +- the snapshot staleness floor, 15.0 s (`MINIMUM_HEALTH_STALE_SECONDS` in + `lab/prism/observability.py`); the effective budget is + `max(3 × refresh interval, 15 s)`. + +**Capture-derived (testnet-cadence-dependent, must be re-baselined).** +The 2026-08-20 testnet4 capture ramped from 0.124 to 32.24 accepted +shares/s over five minutes — a ~260× increase — and under that load +reached 3,120 durable pending candidates, 160 s oldest pending age, and a +semantic coverage floor of 0.077. Absolute candidate counts, candidate +rates, and any short-window count-based timeout thresholds derived from +that run are functions of testnet4 block cadence and of that specific +ramp. They are starting points, **not mainnet constants**. Each such +threshold is marked *cadence-dependent: yes* below and must be +re-baselined against a mainnet-cadence capture before it is trusted to +page anyone. + +Where a class could be expressed either as a count or as an age, this +specification prefers the age or the configuration ratio, because those +transfer across cadences and the counts do not. + +## Alert rules + +Severity convention: `warning` means investigate within the working day; +`critical` means page. `for` is the pending window. + +### 1. `PrismAcceptedParentPreviewWaitTimeouts` + +- **Purpose:** child job builds are timing out waiting for an + accepted-parent payout preview, so job issuance is being blocked behind + unfinished parent bookkeeping. +- **Condition (warning):** + `increase(qbit_prism_accepted_parent_preview_wait_timeouts_total[15m]) > 0` +- **Condition (critical):** + `increase(qbit_prism_accepted_parent_preview_wait_timeouts_total[15m]) > 10` +- **`for`:** `5m` (warning), `10m` (critical) +- **Interpretation / first action:** any increment means at least one + child build waited the full 5 s preview budget and failed closed. Check + `qbit_prism_accepted_parent_unresolved_oldest_seconds` and the + landing-class ledger timings + (`qbit_prism_block_ledger_call_timeouts_total{call_class="landing"}`) + to find which stage of the landing tail is not completing. +- **Cadence-dependent:** the *warning* (any increment) is not — it is a + fail-closed event and is meaningful at any cadence. The *critical* count + of 10 per 15 min **is** cadence-dependent: it counts events whose + natural rate scales with block and job-rebuild frequency. Re-baseline + it. + +### 2. `PrismUnresolvedParentDepthNonzero` / `PrismUnresolvedParentDepthAtCap` + +- **Purpose:** landed accepted-block transitions whose durable bookkeeping + is unresolved stack prospective balance chains on unfinished work. At + the cap, job issuance stops entirely + (`_await_pending_parent_payout_preview` in + `lab/prism/payout_state.py`). +- **Condition (warning, nonzero and sustained):** + `qbit_prism_accepted_parent_unresolved_transitions > 0` +- **Condition (warning, near cap):** + ``` + qbit_prism_accepted_parent_unresolved_transitions + >= on(job, instance) + (qbit_prism_accepted_parent_unresolved_depth_max - 1) + ``` +- **Condition (critical, at cap):** + ``` + qbit_prism_accepted_parent_unresolved_transitions + >= on(job, instance) qbit_prism_accepted_parent_unresolved_depth_max + ``` +- **`for`:** `10m` (nonzero), `2m` (near cap), `1m` (at cap) +- **Interpretation / first action:** a transient nonzero depth is normal + right after a block lands; a depth that does not drain within ten + minutes means the accounting lane is not resolving. At the cap, new + child jobs are already being refused with `TemplateRefreshBlocked` — + this is a mining outage, not a warning sign. Look at the landing-class + ledger call durations and at whether the accounting lane is wedged. With + the #188 gauge task's correction, this gauge is the exact depth the + admission fence uses, so the at-cap condition and the refusal coincide + rather than the alert trailing it. +- **Cadence-dependent:** no. Both are expressed against the exported cap, + which is the configured safety contract. + +### 3. `PrismUnresolvedParentAgeOverBudget` + +- **Purpose:** the oldest unresolved accepted-parent transition has + outlived the preview wait budget, so any child build selecting that + parent will time out. +- **Condition (warning):** + `qbit_prism_accepted_parent_unresolved_oldest_seconds > 5` +- **Condition (critical):** + `qbit_prism_accepted_parent_unresolved_oldest_seconds > 60` +- **`for`:** `2m` (warning), `5m` (critical) +- **Interpretation / first action:** 5 s is exactly the preview wait + budget, so this alert fires at the moment the backlog starts causing + build timeouts rather than after. Expect rule 1 to follow if this is not + resolved. The `-1` sentinel (no timestamped unresolved transitions) is + naturally excluded by a `>` comparison against a positive threshold. + Note the population difference from rule 2: this gauge covers only + transitions with a known landing timestamp, so it can read `-1` or a low + value while the depth gauge is nonzero. That is not a bug and no age is + invented for an untimestamped transition — but it means rule 3 must not + be used to rule out a depth problem. Rule 2 is the complete signal. +- **Cadence-dependent:** no. Both thresholds are anchored to the + configured 5 s budget. + +### 4. `PrismBlockCandidateBacklog` / `PrismBlockCandidateOldestPending` + +- **Purpose:** durable block candidates are accumulating without reaching + a terminal outcome. +- **Condition (warning, count):** `qbit_prism_block_candidates_pending > 250` +- **Condition (critical, count):** `qbit_prism_block_candidates_pending > 1000` +- **Condition (warning, age):** + `qbit_prism_block_candidate_oldest_pending_seconds > 60` +- **Condition (critical, age):** + `qbit_prism_block_candidate_oldest_pending_seconds > 300` +- **Condition (warning, unavailable):** + `qbit_prism_block_candidates_pending == -1` +- **`for`:** `10m` (counts), `5m` (ages), `10m` (unavailable) +- **Interpretation / first action:** prefer the **age** alerts as the + primary signal; a large but fast-draining queue is healthy, and a small + queue whose head is five minutes old is not. On the age alert, check the + submitter's retry backoff series + (`qbit_prism_block_submitter_retry_backoff_active`) to distinguish an + intentional interruptible wait from a wedged lane. +- **Cadence-dependent:** the **count** thresholds are — yes. They are + scaled from the capture's 3,120-candidate peak (warning at roughly 8% of + it, critical at roughly a third) under a testnet4 share rate of + 32.24/s, and mean nothing at a different cadence. The **age** + thresholds transfer far better and should be the ones that page; the + capture's 160 s oldest age sits between them. Re-baseline the counts. + +### 5. `PrismSemanticWorkCoverageLoss` + +- **Purpose:** authorized clients are mining work that no longer matches + the current template fingerprint, payout generation, or client session. +- **Condition (warning):** + ``` + qbit_prism_stratum_semantic_current_work_ratio < 0.95 + and on(job, instance) qbit_prism_stratum_authorized_connections > 0 + ``` +- **Condition (critical):** + ``` + qbit_prism_stratum_semantic_current_work_ratio < 0.5 + and on(job, instance) qbit_prism_stratum_authorized_connections > 0 + ``` +- **`for`:** `3m` (warning), `5m` (critical) +- **Interpretation / first action:** sustained coverage loss means miners + are producing shares against superseded work; those shares are wasted + hashrate and will show up as stale/unknown rejections. Check whether + job delivery or template refresh is the stalled side — rule 6 covers + the refresh side directly. +- **Cadence-dependent:** no. The ratio is normalised by client count, and + 0.95 is carried over from the in-process degraded-coverage line for the + sibling current-tip ratio (see *Threshold provenance*). The capture's + 0.077 floor is context for how far this can fall, not the threshold. + +### 6. `PrismRefreshPendingPastDeadline` + +- **Purpose:** current template or payout work still requires publication + or delivery, past the documented health deadline. +- **Condition (warning):** + ``` + qbit_prism_refresh_pending == 1 + and on(job, instance) qbit_prism_refresh_pending_age_seconds > 15 + ``` +- **Condition (critical):** + ``` + qbit_prism_refresh_pending == 1 + and on(job, instance) qbit_prism_refresh_pending_age_seconds > 60 + ``` +- **`for`:** `1m` (warning), `5m` (critical) +- **Interpretation / first action:** 15 s is the default + `PRISM_HEALTH_PENDING_REFRESH_MAX_AGE_SECONDS`, i.e. the point at which + the coordinator's own health evaluator considers the refresh overdue. + Firing here means the process is already reporting itself degraded on + this axis. If the deployment overrides that environment variable, the + literal `15` must be changed to match, since the deadline is not + currently exported as a series. +- **Cadence-dependent:** no. Both thresholds derive from the documented + default deadline. + +### 7. `PrismMetricsSnapshotStale` / `PrismMetricsSnapshotUnavailable` / `PrismScrapeDown` + +- **Purpose:** detect that the observability path itself has failed, so + that the silence of every rule above is never read as health. +- **Condition (warning, stale):** `qbit_prism_metrics_snapshot_stale == 1` +- **Condition (critical, no complete snapshot):** + `qbit_prism_metrics_snapshot_available == 0` +- **Condition (warning, age):** + `qbit_prism_metrics_snapshot_age_seconds > 45` +- **Condition (critical, scrape loss):** `up{job="prism"} == 0` +- **`for`:** `2m` (stale, no complete snapshot, age), `2m` (scrape loss) +- **Interpretation / first action:** on `up == 0`, treat every other + PRISM alert in this document as *unknown*, not *clear*, and check the + coordinator process and the `/metrics` status code directly. Post-#184, + a 503 means warm-up — no complete snapshot has been published yet — + which is a distinct failure from the process being down and from a + refresh that has stalled after warm-up. **The stale and age conditions + are the ones that detect a stalled refresh at runtime**, because + post-#184 that state is served as 200 and is therefore ingested. + `qbit_prism_metrics_snapshot_available` is **startup coverage only**: + once warm-up has succeeded, it is part of the last served complete + payload and stays `1` even while the refresh is stale, so it does not + detect refresh failure. It also may not be ingested at all during the + warm-up window it describes, since that window is exactly when the + endpoint answers 503 — keep it for the case where a partial payload is + still served, and rely on `up == 0` for warm-up that never completes. +- **Cadence-dependent:** no. 45 s is 3 × the 15 s staleness floor; if the + deployment sets a metrics refresh interval above 15 s, the effective + budget becomes `3 × refresh` and this literal should be raised to match. + +## False-positive and silence cautions + +**Scrape status and the 503 window.** Prometheus does not ingest the body +of a failed scrape, so any state served as 503 is invisible to body-metric +rules. Before #184 this swallowed the whole stale case: +`qbit_prism_metrics_snapshot_stale == 1` could essentially never be +*observed*, because the condition that set it to 1 was the same condition +that stopped the series being ingested. After #184 a stale complete +snapshot is served as 200 with freshness metadata, so the stale and age +conditions in rule 7 become genuinely observable and are the intended +detectors for a stalled refresh. What remains unscrapable is the warm-up +window with no complete snapshot, which still answers 503 — meaning +`qbit_prism_metrics_snapshot_available == 0` may itself go uningested in +exactly the situation it describes. `up == 0` is therefore the detector +that actually fires in the total-failure and never-warmed-up cases. +**Never deploy the body-metric rules without `PrismScrapeDown`.** + +**No metric can report its own absence.** Every rule in sections 1–6 is +silent when the target is gone. Route `PrismScrapeDown` to the same +destination as the critical rules, and consider an Alertmanager inhibition +so that scrape loss suppresses the (now meaningless) body-metric alerts +rather than the reverse. + +**`-1` sentinels are values, not gaps.** Four gauges in the #188 rules use +`-1`: `qbit_prism_accepted_parent_unresolved_oldest_seconds` (no unresolved +transitions *carrying a landing timestamp* — see rule 3, which can read +`-1` while the depth gauge is nonzero), +`qbit_prism_block_candidates_pending` and +`qbit_prism_block_candidate_oldest_pending_seconds` (state unavailable), +and `qbit_prism_metrics_snapshot_age_seconds` (before first success); issue +#198's `qbit_prism_block_candidate_cleanup_retry_oldest_seconds` (nothing +owed) is a fifth, covered in its own section below. All +threshold comparisons above are `>` against positive numbers, so `-1` +never triggers them — but any rule added later using `<`, `avg_over_time`, +`min_over_time`, or a rate must exclude `-1` explicitly, or it will read +"unavailable" as "excellent". The dedicated +`qbit_prism_block_candidates_pending == -1` rule exists precisely because +the healthy-looking `-1` would otherwise hide a lost candidate store. + +**The zero-client ratio reads 1.0, not 0.** +`qbit_prism_stratum_semantic_current_work_ratio` is computed as +`semantic_current / authorized if authorized else 1.0` +(`_mining_delivery_snapshot_serialized` in +`lab/prism/observability.py`). With no authorized connections it +reports *perfect* coverage. The +`qbit_prism_stratum_authorized_connections > 0` guard therefore does not +prevent a false positive — the ratio cannot false-positive at zero +clients — it makes explicit that this alert measures coverage *among +connected miners* and is not a pool-liveness signal. A pool that has lost +every miner is invisible here and needs a separate alert on +`qbit_prism_stratum_authorized_connections` itself, which this document +does not specify. + +**Counter resets.** `qbit_prism_accepted_parent_preview_wait_timeouts_total` +is an in-process counter that resets to 0 on restart. Rule 1 uses +`increase()`, which handles resets correctly; a naive +`... > N` on the raw counter would both latch forever after one incident +and silently clear on every deploy. Do not "simplify" it to a raw +comparison. Note also that `increase()` over a short window on a +low-frequency counter is extrapolated and can report fractional values — +the `> 0` form is robust to this, the `> 10` critical form less so, which +is a further reason to re-baseline it. + +**Restart flapping.** After a coordinator restart, unresolved depth, +pending candidates, and refresh age all start from a replay state rather +than a steady state. The `for` windows above are chosen to ride out a +normal restart; if a deployment's restarts routinely exceed them, add a +startup-grace inhibition rather than lengthening every window. + +## Re-baselining checklist + +Before these rules are trusted on mainnet: + +1. Capture a mainnet-cadence load run equivalent to the 2026-08-20 + testnet4 capture. +2. Recompute the *cadence-dependent* thresholds only — the critical + timeout count in rule 1 and both candidate-count thresholds in rule 4. +3. Leave the configuration-derived thresholds alone unless the underlying + defaults change; if `PRISM_ACCEPTED_PARENT_UNRESOLVED_DEPTH_MAX`, + `PRISM_HEALTH_PENDING_REFRESH_MAX_AGE_SECONDS`, or the metrics refresh + interval is overridden in deployment, update rules 2, 6, and 7 + respectively. +4. Confirm the target build contains #184 — that a stale complete snapshot + is served as 200 with freshness metadata rather than 503 — before + relying on rule 7's stale and age conditions, which are the runtime + detectors. Remember that the remaining 503 case, warm-up with no + complete snapshot, is covered by `up == 0` rather than by + `qbit_prism_metrics_snapshot_available`. + +## Issue #198: the bounded collapse cleanup-retry backlog + +Issue #198. The decided-height collapse (#183, #196, #181 item 2) wins rows +with one fenced batch write and then tears down each won row's in-memory +state — its payout preview or tombstone, its pending-share floor holder, +its retry, outstanding and tip-observation markers, its terminal-outcome +record and its abandonment accounting. A won row is durably terminal the +moment the write returns, so a cleanup step that fails has no durable +replay source left: the coordinator keeps one in-memory retry record per +such row (`_CollapsedCandidateCleanup` in `lab/prism/block_candidates.py`) +and the accounting lane retries one hash per pass. Under a *systemic* +cleanup fault that registry grew by one record per row the collapse won, +with no bound at all, each record retaining its exact pending-share holder +objects and pinning its terminal-outcome fence against eviction. + +The bound this section documents is applied to **admission, never to +cleanup authority**. Once the registry holds the configured number of +records, the collapse stops handing rows to its fenced bulk +terminalization; every row it declines stays durable and pending and takes +the ordinary per-row path. Nothing already terminal ever loses its record, +its holders or its fence, and #196's selection predicate and fencing are +untouched: the same rows are selected, only fewer of them are admitted per +pass. If a forced replay walk fills a page after backpressure engages, it +adopts that one bounded page and pauses pagination until the replay queue has +drained. The startup enumeration remains owed, so job builds stay blocked and +the walk resumes from the durable outbox instead of materializing every later +page in the unbounded replay queue. + +### Signal inventory + +All names were verified against `lab/prism/metrics.py` +(`block_candidate_cleanup_backlog_metrics_lines`). Every series is a single +unlabelled time series per target; none can grow with the candidate +population or carry a hash. The backlog snapshot they render is copied +under the coordinator lock in one O(backlog) walk, which the bound keeps at +a few thousand entries at most. + +| Metric | Type | Meaning | +| --- | --- | --- | +| `qbit_prism_block_candidate_cleanup_retry_backlog` | gauge | Durably terminal collapsed rows whose in-memory cleanup is still owed (registered or actively retrying authority depth). | +| `qbit_prism_block_candidate_cleanup_retry_backlog_max` | gauge | The configured admission bound, exported so rules compare against the running contract rather than a literal. | +| `qbit_prism_block_candidate_cleanup_retry_oldest_seconds` | gauge, `-1` when nothing is owed | Age of the oldest owed cleanup measured from its *first* deferral (a failed retry re-registers the record without resetting this). | +| `qbit_prism_block_candidate_cleanup_retry_pending_share_holders` | gauge | Pending-share floor holders the backlog retains, by exact object identity. | +| `qbit_prism_block_candidate_cleanup_retry_terminal_outcome_pins` | gauge | Terminal-outcome fences the backlog pins against eviction. Equals the depth whenever the apply's fence publication succeeded; a smaller value names records whose fence is still owed to the `terminal-outcome` step. | +| `qbit_prism_block_candidate_cleanup_backpressure_active` | gauge, 0/1 | Whether the backlog is at its bound and bulk terminalization is refusing new rows. | +| `qbit_prism_block_candidate_cleanup_backpressure_total` | counter | Occasions on which the bound preserved at least one row. | +| `qbit_prism_block_candidate_collapse_total{outcome="backlog_deferred"}` | counter, closed label set | Rows preserved by the bound (the existing collapse-outcome family gained this one fixed outcome). | + +The accompanying warning is rate limited to one line per 60 s +(`BLOCK_CANDIDATE_CLEANUP_BACKPRESSURE_LOG_SECONDS`) and every field is a +count or a closed-vocabulary name; the `backlog_deferred` counter still +moves per row while the line is suppressed: + +```text +prism coordinator: collapsed block candidate cleanup backpressure engaged caller=replay-page rows_preserved=1024 admitted=0 backlog=4096 backlog_max=4096 oldest_seconds=12.345; admitting them to bulk terminalization would take the cleanup-retry backlog past its bound, so they stay durable and pending for the per-row path until it drains +``` + +`caller` is `replay-page` (the replay-adoption page walk) or `dequeue` (the +dequeue-time stale sibling skip); `admitted` is how many rows the same pass +still handed on, so `admitted=2 backlog=0 backlog_max=2 rows_preserved=4` +reads as "two filled an empty backlog, four were declined". + +### Configuration knob + +`PRISM_BLOCK_CANDIDATE_CLEANUP_RETRY_BACKLOG_MAX` (default +`DEFAULT_BLOCK_CANDIDATE_CLEANUP_RETRY_BACKLOG_MAX = 4096`, loaded into +`BlockConfig.candidate_cleanup_retry_backlog_max` in +`lab/prism/coordinator_config.py`). Startup refuses a non-integer, a +non-positive value, and any value above +`MAX_BLOCK_CANDIDATE_CLEANUP_RETRY_BACKLOG_MAX = 8192`, which restates the +terminal-outcome registry bound (`MAX_BLOCK_CANDIDATE_TERMINAL_OUTCOMES`; a +test pins the two equal): every retry record pins one fence, and a backlog +allowed past the registry would leave the fence eviction with nothing it +may drop. At runtime the service reads the coordinator attribute +`block_candidate_cleanup_retry_backlog_max` first (the seam tests and +embedders pin), then the loaded block config, then the default. Runtime values +are clamped to the same 1..8192 safety range: a non-positive override degrades +into "admit one row at a time from an empty backlog", while an over-limit +override cannot outgrow the terminal-fence registry. + +### Measurement basis + +Measured with `tests/prism_candidate_storm.py --cleanup-fault all +--credit-shares` (CPython 3.12.3, Linux x86_64, in-memory ledger, one +thread) at the observed 3,120-candidate cardinality, restart view unless +noted, one fresh storm per run. `CleanupFaultInjector` breaks exactly one +cleanup dependency at its shipped seam — the seven cleanup steps plus the +page-scope holder index whose failure aborts the apply — while counting +every seam per hash; the walk, the fence, the registry and the retry pass +are the shipped code. Two independent memory readings are reported: a +`sys.getsizeof` walk of the shipped registry (records, step sets, hash +keys, and the exact holder objects retained, each charged once), and the +change in live `tracemalloc` bytes attributed to the block-candidate owner +module across the faulted walk, taken as a marginal figure against a clean +walk of the same storm. + +| Fault (restart view, 3,120 rows) | Records | Holders | Pins | Registry bytes | Bytes / record (deep) | Owner-traced marginal bytes / record | Recovery passes | Recovery s | Records / s | +| --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | +| payout-preview-withdrawal | 3,119 | 3,119 | 3,119 | 3,018,422 | 967.8 | 560.3 | 3,119 | 0.118 | 26,335 | +| finalize-retry | 3,119 | 3,119 | 3,119 | 3,018,411 | 967.8 | 559.7 | 3,119 | 0.055 | 57,066 | +| retry-state | 3,119 | 3,119 | 3,119 | 3,018,408 | 967.7 | 559.7 | 3,119 | 0.057 | 54,914 | +| outstanding-and-tip-observation | 3,119 | 3,119 | 3,119 | 3,018,428 | 967.8 | 559.7 | 3,119 | 0.065 | 47,792 | +| terminal-outcome | 3,119 | 3,119 | 3,119 | 3,018,413 | 967.8 | 559.7 | 3,119 | 0.068 | 46,173 | +| abandonment-accounting | 3,119 | 3,119 | 3,119 | 3,018,419 | 967.8 | 412.6 | 3,119 | 0.060 | 51,846 | +| pending-share-floor | 3,119 | 3,119 | 3,119 | 3,018,416 | 967.8 | 562.3 | 3,119 | 0.062 | 50,125 | +| floor-index (apply abort) | 3,119 | 0 | 3,119 | 3,351,703 | 1,074.6 | 774.2 | 3,119 | 8.577 | 364 | + +Of the 967.8 deep bytes per record, 357.3 are the retained +`PendingShare` holder itself; a record that retains none costs 610.5 bytes +(the live view, where only the 32 queued wakeups own a holder, measures +566.9 bytes per record over 3,119 records and 1,004 bytes per record when +only the 32 holder-owing records exist). The abort-shaped record is larger +(1,074.6 bytes: it owes all seven steps and must re-index its holders) and +drains far slower (364 records/s) because each retry re-scans the whole +replay queue for its holders — an O(n²) cost at storm size that is inherent +to the abort contract, not to the bound. + +Every fault also sustained 64 consecutive failing retry passes with depth, +holders and pins unchanged, then recovered with zero double cleanups (each +seam ran exactly once more per hash), zero cleanup calls after a record was +discharged, zero terminal rows replay-adopted, zero node offers, and the +decided winner's floor holder retained while the other 3,119 were released +once each by identity (floor 3,120 → 1). The walk itself took 0.68–1.17 s +per 3,120 rows. + +With the bound pinned below the storm (`--backlog-max 1024`): the first +1,024-row page filled the backlog, the remaining three pages (2,096 rows, +the winner among them) were preserved in three engagements, sustained +failure left the 1,024 records intact, recovery drained them in 0.022 s, +and the next walk collapsed the 2,095 preserved siblings in 0.25 s with the +same clean recovery proof. The optional 312,000-candidate run was +attempted (`--candidates 312000 --cleanup-fault retry-state +--credit-shares --no-tracemalloc`) and stopped after 12 min 47 s of CPU +time at about 1.7 GB resident without completing; no figure from it is +used, and none is needed for the derivation. + +### Threshold policy: how 4,096 was derived + +Two anchors bound the default and two checks confirm it: + +1. **Upper anchor — the fence registry.** Every record pins its terminal + outcome. `MAX_BLOCK_CANDIDATE_TERMINAL_OUTCOMES` is 8,192, so the + backlog may pin at most half of it and still leave the eviction the + other half to work with: `≤ 8192 / 2 = 4096`. The knob's hard limit is + the full 8,192. +2. **Lower anchor — absorption.** The bound must not engage on the storm + this machinery was built for, nor inside a single page: `≥ 3119` + (the observed storm's siblings) and `≥ 1024` + (`MAX_BLOCK_REPLAY_ENUMERATION_ROWS`). +3. **Memory check.** `4096 × 967.8 B ≈ 3.96 MB` with every holder retained + (`≈ 4.4 MB` for abort-shaped records; `≈ 8.8 MB` at the 8,192 limit). + The unbounded 312,000-record case the issue describes would have been + `≈ 302–335 MB` of retained records plus their pinned fences. +4. **Drain check.** On an idle accounting lane `4096 / 26,335 ≈ 0.16 s` + for a step fault and `4096 / 364 ≈ 11 s` for the abort shape; on a busy + lane the cadence (`DEFAULT_BLOCK_ACCOUNTING_CLEANUP_RETRY_WORK_ITEMS`, + 8) bounds the drain at `4096 × 8 = 32,768` completed work items, which + is the shipped cadence contract rather than a property of this bound. + +`4096` is the only power of two that satisfies both anchors, and both +checks hold at it. It is configuration-derived, not cadence-dependent: a +mainnet storm of a different size changes how quickly the bound is +*reached*, not whether it is correct. + +### Alert rules + +#### 8. `PrismCollapseCleanupBacklog` / `PrismCollapseCleanupBackpressure` + +- **Purpose:** collapsed rows are owing in-memory cleanup that the + accounting lane is not finishing, and at the bound the collapse has + stopped bulk-terminalizing new rows. +- **Condition (warning, owed and not draining):** + `qbit_prism_block_candidate_cleanup_retry_backlog > 0` +- **Condition (warning, oldest owed cleanup):** + `qbit_prism_block_candidate_cleanup_retry_oldest_seconds > 300` +- **Condition (warning, near the bound):** + ``` + qbit_prism_block_candidate_cleanup_retry_backlog + >= on(job, instance) + (qbit_prism_block_candidate_cleanup_retry_backlog_max * 0.5) + ``` +- **Condition (critical, at the bound):** + `qbit_prism_block_candidate_cleanup_backpressure_active == 1` +- **Condition (warning, engaged recently):** + `increase(qbit_prism_block_candidate_cleanup_backpressure_total[15m]) > 0` +- **`for`:** `10m` (owed), `5m` (oldest), `5m` (near), `2m` (at bound), + `0m` (engaged) +- **Interpretation / first action:** a transient nonzero depth right after + a storm collapses is normal and drains within seconds (see the measured + throughput). A depth that holds for ten minutes, or an oldest age past + 300 s — roughly seventeen consecutive failed attempts under the shipped + 0.25 s-doubling-to-30 s backoff — means one cleanup dependency is + persistently failing; `qbit_prism_block_candidate_collapse_total{outcome="cleanup_retry_failed"}` + climbs in step and the coordinator log names the failing step + (`collapsed block candidate cleanup failed step=… hash=…`, detailed for + the first three per pass). At the bound the process is protecting its + own memory: preserved siblings take the per-row path, so expect the #181 + symptoms to return for them — one `submitblock`, ~6 chain reads and two + ledger writes per preserved row, visible as `backlog_deferred` climbing + and `qbit_prism_block_candidates_pending` draining slowly rather than in + page-sized steps. That is the intended degradation, not a second fault. +- **Cadence-dependent:** no. Every threshold is against the exported bound + or the in-process retry pacing. + +### Operator symptoms and the recovery contract + +Symptoms while the bound is engaged, in the order they appear: + +1. `cleanup_failed` and then `cleanup_retry_failed` outcomes climb; the + backlog, holders and pins gauges rise together. +2. `qbit_prism_block_candidate_cleanup_backpressure_active` flips to 1 and + one `cleanup backpressure engaged` line prints (then at most one per + 60 s); `backlog_deferred` and `..._backpressure_total` climb per pass. +3. Preserved rows stay in `qbit_prism_block_candidates_pending` and are + disposed of by the per-row path at per-row cost. Nothing is abandoned + on an unknown, nothing is offered twice, and no terminal row is + re-adopted: the fence published at the write still holds for every + backlog entry. + +The recovery contract, proven by `tests/test_prism_block_candidates.py` +(deterministically, including the active-retry window and at the shipped +bound) and corroborated at 3,120 rows by +`tests/test_prism_candidate_storm.py`: once the failing dependency heals, +the accounting lane drains one hash per retry pass — immediately while +idle, otherwise once per `DEFAULT_BLOCK_ACCOUNTING_CLEANUP_RETRY_WORK_ITEMS` +completed work items — and each hash runs exactly the steps it still owes, +once. Fences stay published throughout; holders are released exactly once +by object identity and only for the collapsed row's own object; records, +holders and pins are never shed while the fault persists. The moment the +depth drops below the bound admission resumes, and the next enumeration +walk collapses the preserved rows in bulk again. No operator action is needed +beyond fixing the dependency, and no restart is required for durable recovery. +A controlled restart is nevertheless a valid short-term mitigation: the +backlog, retained holders, and terminal-outcome pins are process-local and are +cleared, while the durable terminal rows are not selected by pending-row replay +and therefore are neither re-adopted nor re-offered. Restarting does not repair +the cleanup dependency; new collapses can rebuild the backlog and re-engage +backpressure until the underlying fault is fixed. + +Re-baselining: rerun `python3 tests/prism_candidate_storm.py --cleanup-fault +all --credit-shares` (add `--cleanup-view live`, `--backlog-max N`) and +re-derive step 3 and step 4 above from its `registry_bytes_per_record` and +`retry_records_per_second`; the two anchors are configuration constants +and do not move with cadence. + +## Issue #224: accepted-preview latency attribution and rollout validation + +Issue #224. `PrismAcceptedPreviewPublicationLatencyHigh` (the external rule +on `qbit_prism_accepted_block_preview_publication_seconds`, warning at 4 s) +fired on Union mainnet while the coordinator stayed healthy, and nothing on +`/metrics` said which stretch of the landing owned a sample above the 4 s +warning or the 5 s accepted-parent child wait budget +(`DEFAULT_ACCEPTED_BLOCK_PAYOUT_PREVIEW_WAIT_SECONDS` in +`lab/prism/payout_state.py`). The #224 build does three things: + +- A reconcile pass that **confirmed** a payout mutation (maturation, + inactive marking, reactivation, stranded-prepared rejection) now asks + candidate preparation for a live prior-balances reread over the + still-exact accepted-share window (`force_prior_balances_read=True`, + `force_full_window_rescan=False`) instead of the O(window) oracle rescan. + This rests on the verified invariant that reconciliation never writes + `qbit_share_ledger` (proved in `test/test-prism-postgres-ledger.sh`, + `share-ledger-identity=inactive+reactivate+mature`). Every other full + rescan trigger is unchanged, including the fail-closed one below. +- The landing, the reconciler and the payout-window oracle record + fixed-cardinality attribution families (below). Every label is a closed + vocabulary from `lab/prism/accepted_preview_telemetry.py`; hashes, + heights, miner identities and exception text never become labels. +- The four payout-window and prior-balances ledger reads are attributed by + operation with admission split from PostgreSQL execution, on the same + gate each read always took. + +This section is the operator contract for judging the 4 s and 5 s +boundaries from the shipped metrics. Like the rest of this document it is +a specification: the rules and queries are transcribed by the external +rule repository, and the ordering requirement (#184 first) applies. + +### Signal inventory + +All names were verified against `lab/prism/metrics.py` +(`_accepted_block_preview_publication_lines`, +`_ledger_read_gate_metric_lines`, +`accepted_preview_attribution_metrics_lines`), `lab/prism/tip_refresh.py` +and `lab/prism/progress_health.py`. + +| Metric | Type | Labels | Meaning | +| --- | --- | --- | --- | +| `qbit_prism_accepted_block_preview_publication_seconds` | histogram | `result` ∈ `published`, `degraded`; `le` | Definitive node acceptance to the preview becoming visible to waiting children. The bucket boundaries include exactly `4` and `5`; the renderer prints `le` with `%g`, so the label values are `le="4"` and `le="5"`, not `"4.0"`. Only the **first** publication of a hash is observed. | +| `qbit_prism_accepted_block_landing_phase_seconds` | summary (`_sum`, `_count`, `_max`) | `phase` ∈ `lane_wait`, `balance_lock_wait`, `reconcile`, `prior_balances_check`, `chain_probe`, `preview_prepare`, `preview_publish` | Wall time of each landing stretch. `prior_balances_check` is a prior-balances **reread**, never a window rescan. Every cell renders from zero at process start. | +| `qbit_prism_reorg_reconcile_pass_seconds` | summary | `caller` ∈ `landing`, `post_confirm`, `tip_refresh`, `job_build`, `other` | One observation per reconcile pass, on every exit (success, untrusted skip, superseded, error). `_count` is the pass count by caller. `tip_refresh`/`job_build` are spelled as in `qbit_prism_reorg_reconcile_lookups_total{path}` so the two families join. | +| `qbit_prism_reorg_reconcile_step_seconds` | summary | `caller`, `step` ∈ `admission_wait`, `watch_query`, `chain_probe`, `mutations`, `candidate_prepare`, `publish` | Where a pass spent its time. `admission_wait` (writer admission plus the payout-balance mutation lock) is measured **before** the pass timer starts, so it is not part of the pass total. `candidate_prepare` is where a `reconcile_invalidation` rescan executes when one is forced. | +| `qbit_prism_payout_window_full_rescan_seconds` | summary | `reason` (18 fixed values, see below), `path` ∈ `daemon`, `in_process` | **True** whole-window oracle rescans only. A prior-balances reread is never an observation here. | +| `qbit_prism_ledger_read_calls_total`, `..._gate_wait_seconds_total`, `..._gate_wait_seconds_max`, `..._gate_timeouts_total`, `..._execute_seconds_total`, `..._execute_seconds_max`, `..._execute_timeouts_total` | counters and gauges | `operation` ∈ `pending_block_candidate_rows`, `payout_window_snapshot`, `payout_window_delta`, `current_prior_balances`, `prior_balances_after_pool_block`, `other` | Coordinator-local admission (`gate_wait`) against PostgreSQL execution (`execute`) per read. `current_prior_balances` waits on the **writer lock**; the other four wait on the bounded read slot (`PRISM_POSTGRES_READ_CONCURRENCY`). Any out-of-contract name folds into `other`. A series appears only after that operation's first call. | +| `qbit_prism_payout_artifact_events_total` | counter | `event` (`built`, `debounced`, `incremental`, `full_rescan`, …) | Artifact lifecycle counts. `full_rescan` counts builds whose window came from the oracle, without a reason; use the rescan family for the reason. | +| `qbit_prism_prior_balances_reads_total` / `_read_last_seconds` / `_read_max_seconds` | counter / gauges | none | Legacy carry-read aggregate, deliberately unchanged: one duration over admission plus execution, recorded on success only. | +| `qbit_prism_accepted_parent_preview_wait_timeouts_total` | counter | none | Child builds that waited the full 5 s budget and failed closed (rule 1 above). | +| `qbit_prism_reorg_reconcile_errors_total` | counter | none | Reconcile passes that errored. One that had already attempted a payout mutation forces the fail-closed `reconcile_invalidation` rescan below; a read-only failure does not. | +| `qbit_prism_stratum_semantic_current_work_ratio`, `qbit_prism_block_candidates_pending`, `qbit_prism_block_candidate_oldest_pending_seconds`, `qbit_prism_refresh_pending`, `qbit_prism_refresh_pending_age_seconds` | gauges | none | Current-work coverage and candidate/refresh backlog (rules 4–6 above, same `-1` cautions). | + +The rescan `reason` vocabulary, in render order: `reconcile_invalidation`, +`cold_start`, `cache_invalidated`, `late_visible_append`, +`anchor_regression`, `snapshot_window_weight_out_of_band`, +`window_pipeline_mode_changed`, `window_mirror_divergence`, +`window_daemon_unavailable`, `window_daemon_busy`, +`window_daemon_state_lost`, `window_value_out_of_range`, +`delta_api_unavailable`, `incremental_invariant_failed`, +`periodic_self_check`, `periodic_self_check_failed`, +`periodic_self_check_balance_check_failed`, `other`. + +Two shapes matter for the queries. The `_max` cell of every summary family +is the maximum **since process start**, so it answers "has this phase ever +exceeded X on this process" and cannot be windowed; use `_sum` and +`_count` under `increase()` for a rolling hour. And the histogram counts +publications, so a landing that never published (abandoned, collapsed) +is absent from it by design and shows up instead in +`qbit_prism_block_candidates_pending` and the collapse outcomes. + +### Structured log lines + +Block identity lives in two bounded log lines, never in a label: + +- `prism coordinator: {"event": "accepted_block_preview_landing", ...}` + (`lab/prism/block_candidates.py`), emitted **once** per closed + acceptance-to-publication interval, from the first `published` or + `degraded` publication, with `block_hash`, `block_height`, `result`, + `acceptance_to_publication_seconds`, `reconcile_caller` (always + `landing`), one `phase__seconds` per phase, and the neutral + `full_rescan_*` / `ledger_*` fields (always `null`/`0` in this build: + no per-landing causal evidence for those exists without claiming another + lane's work, so use the rescan and ledger families instead). The same + record is retained in an in-process ring of 64 entries + (`PRISM_ACCEPTED_PREVIEW_DIAGNOSTICS_CAPACITY`); that ring is not served + on any endpoint in this build, so the log line is the operator's copy. +- `prism coordinator: {"event": "payout_artifact_built" | "payout_artifact_build_debounced", ...}` + (`lab/prism/payout_state.py`) with `payout_state_generation`, + `window_build_mode`, `prior_balances_source` (`published` or `ledger`), + `prior_balances_read_forced` (`true` exactly when the #224 reread intent + drove the build) and, only when the oracle ran, `full_rescan_reason`. + +### Queries + +**Breach counts over a rolling hour, from the buckets.** A bucket counts +samples `<= le`, so "above 4 s" is the count minus the `le="4"` bucket: + +``` +# published previews above 4 s in the last hour +increase(qbit_prism_accepted_block_preview_publication_seconds_count{result="published"}[1h]) + - ignoring(le) +increase(qbit_prism_accepted_block_preview_publication_seconds_bucket{result="published",le="4"}[1h]) + +# published previews above 5 s in the last hour +increase(qbit_prism_accepted_block_preview_publication_seconds_count{result="published"}[1h]) + - ignoring(le) +increase(qbit_prism_accepted_block_preview_publication_seconds_bucket{result="published",le="5"}[1h]) +``` + +To judge **every** rolling hour of a soak at once, wrap either expression +in a subquery and take the maximum (a recording rule of the inner +expression plus `max_over_time` is equivalent where subqueries are not +available): + +``` +max_over_time( + ( + increase(qbit_prism_accepted_block_preview_publication_seconds_count{result="published"}[1h]) + - ignoring(le) + increase(qbit_prism_accepted_block_preview_publication_seconds_bucket{result="published",le="4"}[1h]) + )[24h:1m] +) < 1.5 +``` + +`increase()` extrapolates to the window edges and can return a fraction; +with a 15 s scrape interval the distortion is under one percent, so +compare against `1.5` for "fewer than two" and `0.5` for "none". Do not +`ceil()` the value and do not substitute a percentile: `histogram_quantile` +interpolates inside a bucket and cannot prove a breach count. The same +shape gives the other two hard counts: + +``` +# degraded publications in the last hour (must stay 0) +increase(qbit_prism_accepted_block_preview_publication_seconds_count{result="degraded"}[1h]) + +# child preview-wait timeouts in the last hour (must not rise over baseline) +increase(qbit_prism_accepted_parent_preview_wait_timeouts_total[1h]) +``` + +**Which landing phase owned the hour.** Total and mean seconds per phase: + +``` +sum by (phase) (increase(qbit_prism_accepted_block_landing_phase_seconds_sum[1h])) + +increase(qbit_prism_accepted_block_landing_phase_seconds_sum[1h]) + / increase(qbit_prism_accepted_block_landing_phase_seconds_count[1h]) +``` + +`qbit_prism_accepted_block_landing_phase_seconds_max > 4` names any phase +that has, on its own, exceeded the warning at least once since the +process started; read it during a soak that began at the deploy. + +**Reconcile callers and steps.** Pass counts by caller, step seconds by +caller and step, and the remainder a pass spent outside every step: + +``` +sum by (caller) (increase(qbit_prism_reorg_reconcile_pass_seconds_count[1h])) + +sum by (caller, step) (increase(qbit_prism_reorg_reconcile_step_seconds_sum[1h])) + +increase(qbit_prism_reorg_reconcile_pass_seconds_sum[1h]) + - on (job, instance, caller) +sum by (job, instance, caller) ( + increase(qbit_prism_reorg_reconcile_step_seconds_sum{step!="admission_wait"}[1h]) +) +``` + +For `caller="landing"`, `admission_wait` plus the pass is the landing's +`reconcile` phase. A large `admission_wait` under `tip_refresh` or +`job_build` is time spent behind a landing that holds the payout-balance +mutation lock, not reconcile work. + +**True full rescans against balance-only rereads.** Rescans by reason: + +``` +sum by (reason, path) (increase(qbit_prism_payout_window_full_rescan_seconds_count[1h])) +``` + +During normal forward progress expect `cold_start` once per process +start, one of the `periodic_self_check*` reasons at most about once per +`PRISM_PAYOUT_ARTIFACT_FULL_RESCAN_SECONDS` (default 3600 s), and every +other cell, `reconcile_invalidation` included, at zero. A successful #224 +reread has **no family of its own**: it is one +`qbit_prism_ledger_read_calls_total{operation="current_prior_balances"}` +increment (that operation also counts the landing's prior-balances checks +and cold builds) together with a `payout_artifact_build_debounced` or +`payout_artifact_built` line carrying `"prior_balances_read_forced": true`, +`"prior_balances_source": "ledger"` and no `full_rescan_reason`, while +`qbit_prism_payout_artifact_events_total{event="full_rescan"}` and the +`reconcile_invalidation` cell do not move. + +**Ledger admission against execution.** Per operation, which half of a +slow read grew: + +``` +sum by (operation) (increase(qbit_prism_ledger_read_gate_wait_seconds_total[1h])) +sum by (operation) (increase(qbit_prism_ledger_read_execute_seconds_total[1h])) +sum by (operation) (increase(qbit_prism_ledger_read_gate_timeouts_total[1h])) +sum by (operation) (increase(qbit_prism_ledger_read_execute_timeouts_total[1h])) +``` + +A rising `gate_wait` for `current_prior_balances` is contention on the +writer lock inside this process (an accounting write or another landing); +a rising `gate_wait` for `payout_window_snapshot` / `payout_window_delta` +is read-slot exhaustion; a rising `execute` is PostgreSQL. A missing +series means the operation has not run since the process started. + +**Coverage and backlog over the soak.** + +``` +min_over_time(qbit_prism_stratum_semantic_current_work_ratio[24h]) +max_over_time(qbit_prism_block_candidates_pending[24h]) +max_over_time(qbit_prism_block_candidate_oldest_pending_seconds[24h]) +max_over_time(qbit_prism_refresh_pending_age_seconds[24h]) +``` + +### Correlating a slow publication with the log + +Take the slow publications from the landing line, never from a label: + +```sh +rg '"event": "accepted_block_preview_landing"' "$PRISM_COORDINATOR_LOG" \ + | sed -E 's/^.*prism coordinator: //' \ + | jq -c 'select(.acceptance_to_publication_seconds > 4) + | {block_hash, block_height, result, + total: .acceptance_to_publication_seconds, + lane_wait: .phase_lane_wait_seconds, + balance_lock_wait: .phase_balance_lock_wait_seconds, + reconcile: .phase_reconcile_seconds, + prior_balances_check: .phase_prior_balances_check_seconds, + chain_probe: .phase_chain_probe_seconds, + preview_prepare: .phase_preview_prepare_seconds, + preview_publish: .phase_preview_publish_seconds}' +``` + +The phases sum to at most the total; the remainder is queue and +disposition time that no phase claims (for the fallback landing that +submits inside the balance serializer, `lane_wait` is legitimately zero, +because acceptance is stamped after that lane started). The candidate +attempt-marker write is deliberately part of that remainder rather than +part of `lane_wait`: it is the landing's own PostgreSQL write, so a +degraded writer must not inflate the one stretch that exonerates the +landing. A large remainder alongside small phases is therefore queue, +disposition or marker-write time, and no phase in this family will name +it. Then read the +`payout_artifact_*` and `reorg-reconcile` lines adjacent in the +timestamped log for the same window: they carry `payout_state_generation`, +`window_build_mode`, `full_rescan_reason` and `prior_balances_read_forced`, +so a `reconcile` phase that owned the sample can be tied to a forced +rescan (a reason is present) or to a reread (no reason, +`prior_balances_read_forced: true`) without either the hash or the height +ever entering Prometheus. Keep the extracted lines with the rollout +evidence exactly as `docs/prism-payout-artifact-measurement.md` §3 keeps +the artifact events. + +### Validation checklist + +**Pre-deploy baseline (current build, 24 h, same windows as +`docs/prism-payout-artifact-measurement.md`).** Record the commit and image +digest, the UTC window, the accepted-block count +(`increase(qbit_prism_accepted_block_preview_publication_seconds_count{result="published"}[24h])`) +and, per rolling hour via the subqueries above, the maximum count above +4 s and above 5 s, the degraded count and the preview-wait timeouts. +Record `increase(qbit_prism_payout_artifact_events_total{event="full_rescan"}[24h])`, +`increase(qbit_prism_prior_balances_reads_total[24h])`, the pages-CTE +call count from the measurement runbook §1, and the four coverage/backlog +extremes. Confirm the baseline build exports **no** +`qbit_prism_accepted_block_landing_phase_seconds` series, so the +post-deploy presence check below is meaningful. + +**Post-deploy, testnet cadence.** + +1. From the first scrape after the restart, confirm every cell of the four + attribution families renders at zero (for example + `qbit_prism_accepted_block_landing_phase_seconds_count{phase="lane_wait"} == 0`) + and that both `result` values carry `le="4"` and `le="5"` bucket lines. +2. Drive an accepted-tip cadence at least as dense as the alert hour (35 + accepted blocks in one hour) that includes payout maturations and, + where the chain allows it, an inactive/reactivate transition. +3. Through the cadence, `reconcile_invalidation` stays at zero, the + `current_prior_balances` call count rises with confirmed mutations, the + artifact lines show `prior_balances_read_forced: true` with no + `full_rescan_reason`, and `full_rescan` events are limited to the cold + start and the periodic self-check. +4. Evaluate the pass/fail criteria on every rolling hour of the cadence. + +**Post-deploy, Union mainnet soak (24 h minimum).** Repeat the baseline +captures on the new build, evaluate the criteria on every rolling hour, +and attach the phase, caller/step, rescan and ledger breakdowns for every +hour that carried a sample above 4 s, with the correlated landing lines. + +### Pass/fail criteria + +The build passes when all of the following hold over the whole soak: + +- fewer than two `published` previews above 4 s in every modeled rolling + hour (`max_over_time(...[soak:1m]) < 1.5` on the 4 s expression); +- no `published` preview above 5 s during normal forward progress + (`< 0.5` on the 5 s expression), where normal forward progress means an + hour with no increase in `qbit_prism_reorg_reconcile_errors_total`, no + `reconcile_invalidation` or `window_daemon_*` rescan, and no restart; a + sample above 5 s inside such an hour is a fail, and one outside it is + reported with its attribution; +- the `degraded` count stays at zero; +- `qbit_prism_accepted_parent_preview_wait_timeouts_total` does not + increase over the baseline rate; +- no regression against the baseline in the coverage floor or the + candidate/refresh backlog extremes. + +### Fail-closed rollback and escalation + +A failing soak never moves a safety boundary. Do not raise the 4 s alert +threshold, do not extend the 5 s child wait, do not lengthen or disable +the periodic self-check or the landing's prior-balances check, and do not +disable or bypass the conservative `reconcile_invalidation` full rescan: +the reread optimisation is only sound because that fallback still runs +whenever a mutation outcome is unknown. The decision is: + +1. **Regression against baseline** (any `degraded` publication, a rising + timeout rate, a coverage or backlog regression, or + `reconcile_invalidation` rescans or reconcile errors during normal + forward progress): stop the rollout and return to the prior reviewed + artifact under `docs/mainnet-deployment.md` "Stop And Rollback", + attaching the captures. The prior build fails closed in every place + this one does, so rolling back is always safe. +2. **No regression, criteria still unmet** (breach counts unchanged or + improved but not below the bar): the build is safe to keep, since it + changes no fail-closed path, but it does not close #224. Escalate on + #224 with the per-hour phase, caller/step, rescan-reason and + admission/execution breakdowns and the correlated landing lines; that + evidence is what the next change must be designed from. + +### Scope notes + +- `reconcile_invalidation` is **deliberate**, not a regression of the + reread. It is recorded only where `force_full_window_rescan=True` still + runs: the reconciler's error-after-possible-commit path (a mutator whose + response was lost may have committed, so the pass republishes against a + forced full rescan that necessarily rereads the balances) and the + accepted-preview withdrawal path. A successful balance-only reread never + reaches the rescan family; see the reread markers above. +- `window_daemon_state_lost` is the open #207 scope (the Rust window + daemon losing its state), not #224. A rescan under that reason during + the soak is a #207 finding to report separately; it still counts toward + the breach criteria if it lands inside a publication interval, which is + why the 5 s criterion excludes such hours from "normal forward + progress". diff --git a/docs/prism-payout-artifact-measurement.md b/docs/prism-payout-artifact-measurement.md index 488baf798..2698d892a 100644 --- a/docs/prism-payout-artifact-measurement.md +++ b/docs/prism-payout-artifact-measurement.md @@ -312,3 +312,174 @@ sidecar failures, zero normal-operation reconcile-prefetch timeouts, and Do not claim the production win from local simulation. Until both captures are complete, report these as expected effects and attach the raw before/after evidence to the rollout record. + +## 6. Full-rescan attribution (issue #228) + +Every `payout_artifact_built` and `payout_artifact_build_debounced` line now +says why the build was requested. Read these fields together: + +| Field | Meaning | +|---|---| +| `invalidation_cause` | The payout-state source cause the candidate was captured with, verbatim (`external_tip`, `accepted_block_preview`, `accepted_block_preview_withdrawn`, `direct_block_uncertain`, `payout_only`, `startup`); `null` for a background preparation, which carries no candidate. | +| `force_origin` | What marked the build as a payout mutation: `caller_flag` (the caller passed `force_full_window_rescan`), `reserved_cause` (the captured cause was `accepted_block_preview_withdrawn` and this preparation consumed it), `caller_flag+reserved_cause`, or `none`. | +| `cause_inherited` | `true` when the capture found a cause another preparation had already acted on. The source tuple is deliberately sticky (candidate supersession and the finalizer's pending-cause read depend on it); before this change every such capture re-forced the full oracle. An inherited cause never forces anything now, so a `true` here with `force_origin: "none"` is the expected shape. | +| `payout_mutated` | Balances were re-read from the ledger because pool-block, payout-entry or carry rows moved (a reorg, a maturity, a withdrawal), by either flag below. | +| `prior_balances_read_forced` | The caller passed `force_prior_balances_read` (the reorg reconciler after a confirmed mutation, issue #224): the accepted-share ledger is append-only, so the window advances from its delta and only the carry aggregate is reread. Never a full rescan. | +| `window_oracle_forced` | The caller passed `force_full_window_rescan`: the fail-closed full oracle, which also rereads balances. In production that is the preview-withdrawal path and the reconciler's error path after an uncertain mutation. | +| `self_check_deferred` | An overdue periodic self-check was left for the next build off the tip-refresh path because this build's cached window was inside the accepted staleness bound (below). | +| `self_check_recentered` | The periodic self-check re-centered an oversized window at the live weight (issue #207). | +| `phase_seconds` | The phase split of this build, see section 7. | + +Which builds run the full oracle (`window_build_mode: "full_rescan"`), by +`full_rescan_reason`: + +| Reason | Trigger | Expected rate | +|---|---|---| +| `cold_start` | No cached window (process start, or reuse re-enabled). | Once per process. | +| `late_visible_append` | The append-invalidation epoch moved: a share committed below an already-walked anchor. The epoch check is the guard that keeps a failed reorg check from resuming append-only state; it is unchanged. | Rare; alert if sustained. | +| `snapshot_window_weight_out_of_band`, `anchor_regression`, `window_pipeline_mode_changed`, `incremental_invariant_failed`, `window_mirror_divergence`, `window_value_out_of_range`, `delta_api_unavailable` | Genuine epoch breaks detected inside materialization. | Rare. | +| `window_daemon_state_lost` | The Rust daemon no longer holds the base digest after a respawn or LRU eviction. Recovery is one oracle read that re-prepares the daemon. | Tracks daemon restarts. | +| `window_daemon_busy`, `window_daemon_unavailable`, `window_value_out_of_range`, `window_mirror_divergence` (after a `self_check_recentered: true` build) | A periodic self-check adopted a window the daemon could not be prepared for at the time, and the reason names what declined the preparation; the next advance rebuilt through the oracle. A deliberate coordinator adoption, never reported as daemon state loss (issue #207). | Rare. | +| `reconcile_invalidation` | A caller passed `force_full_window_rescan`: the preview-withdrawal path and the reconciler's error path after an uncertain mutation. Since PR #230, reorg reconciliation and payout maturity ask for a prior-balances reread instead (`prior_balances_read_forced: true`, an incremental build), so this reason no longer tracks block cadence. | Withdrawals and reconciler errors only. | + +Self-check builds (`self_check_match` / `self_check_mismatch`) report +`periodic_self_check`, with `self_check_recentered: true` when the cached +weight sat more than 25% above the live snapshot weight and the window was +re-centered at the live weight. With the Rust pipeline on, the re-center +prepares the daemon for the adopted digest in the same build, so the next +build advances incrementally; a second full ledger snapshot on the following +build is the regression issue #207 fixed. The rescan reason vocabulary is +closed (`lab/prism/accepted_preview_telemetry.py`, exported as +`qbit_prism_payout_window_full_rescan_seconds{reason,path}`), so the +re-center is a flag on the build rather than a reason of its own. + +`payout_artifact_tip_refresh_cached` marks a routine candidate that found the +preparation lock busy and reused the armed window instead of queueing (see +section 8). It counts under `qbit_prism_payout_artifact_events_total{event="tip_refresh_cached"}` +once the tip-refresh exporter lists the event; until then count it from logs. + +Per-hour full-rescan count from the extract in section 3: + +```sh +jq -r ' + select(.event == "payout_artifact_built" and .window_build_mode == "full_rescan") + | "\(.full_rescan_reason) \(.force_origin) \(.cause_inherited)" +' "$PRISM_ARTIFACT_EVENTS" | sort | uniq -c +``` + +## 7. Build phase attribution (issue #228) + +`qbit_prism_payout_window_build_phase_seconds{phase=...,outcome=...}` is a +histogram over the same buckets as the other payout timings, observed once +per build for each of a closed phase set, under a closed `outcome`: +`completed`, `debounced`, `aborted` (a lost generation race, an anchor past +the audit ceiling, an empty window) or `failed` (the build died with the +ledger). Every exit observes, so a build slow enough to lose its generation +race keeps the time it measured instead of vanishing from the family; read +the `aborted` and `failed` cells first during an incident. No per-tip, +per-generation or per-reason label exists in this family. + +| `phase` | What it measures | +|---|---| +| `ledger_read` | Every PostgreSQL read the build issued: the full oracle, the delta query, the periodic self-check's oracle read, and the carry-balance aggregate when balances were not reused. | +| `record_conversion` | In-process work on records: folding into pages, canonical JSON and digests, mirror verification. | +| `daemon_prepare` | Round trips to the Rust daemon (`prepare_window` full or advance), including any window upload. | +| `lock_wait` | Time queued for the payout-state preparation lock before the build could start. | + +The same split is on each build line as `phase_seconds`. The phases account +for the build's `duration_seconds` up to artifact construction and the cheap +accepted-share count. A build whose `ledger_read` dominates is a database +problem; one whose `lock_wait` dominates is queued behind another build; one +whose `daemon_prepare` dominates is the daemon or its upload. + +## 8. Accepted staleness on the tip-refresh path (issue #228) + +`PRISM_TIP_REFRESH_WINDOW_STALENESS_SECONDS` (60 s, in `lab/prism/payout_state.py`) +is the staleness the coordinator accepts for a routine candidate build, the +one a tip refresh publishes for a new tip. Two rules key on it: + +- If the preparation lock is busy and an armed window exists whose anchor is + no older than the bound (and whose payout generation, append epoch and + balances are current, and whose difficulty is inside the snapshot-weight + band), the candidate reuses that window instead of queueing behind the + lock holder. This is the found-block degrade of issue #188 applied to the + tip-refresh path. +- If a periodic self-check is overdue when a routine candidate builds, and + the cached window's anchor is within the bound of the anchor being built, + the check is deferred, but never for more than one staleness bound past + its due time. The grace is measured from the last self-check attempt, not + from the last build, so candidates arriving faster than the bound cannot + push it out. A deferral also marks the check owed: the next payout + publication requests a background preparation, which normally runs the + check off the critical path well inside the grace. If nothing else runs + it, the first routine build after the grace runs it on the critical path. + +Why 60 s: it is one background re-anchor interval. In steady state the +re-anchor keeps the armed window younger than this, so a window inside the +bound is one the delivery path is already serving to new jobs as fresh, and +a fanout that reuses it declares an anchor no older than reuse itself +declares. Past the bound the wave pays the (incremental) build: the +re-anchor has already flagged that window for replacement, and carrying it +into a new payout generation would let a stalled background walker pin a +stale anchor for every following generation. It sits well inside the 300 s +audit ceiling that bounds how far any served window may trail the live +ledger. + +What this does not bound: the reorg reconciler acquires the preparation lock +before it captures its source, so a wave still waits for a background build +that holds the lock through a periodic self-check. That wait is one oracle +read at the self-check cadence (two in 48 h on mainnet), not one per +invalidation. + +The bookkeeping behind `cause_inherited` is one integer, the highest source +generation any candidate preparation has acted on, plus a one-shot +entitlement carried by the tuple a reservation hands back to its caller. A +capture acts only above that mark; the reserver's own preparation acts once +by presenting its reserved tuple, whatever happened at other generations in +between. Nothing accumulates, so there is nothing to prune or to watch. + +## 9. Deliberate re-center versus daemon state loss (issue #207) + +| Signal | Deliberate re-center | Genuine daemon eviction or restart | +|---|---|---| +| Self-check build line | `full_rescan_reason: "periodic_self_check"`, `self_check_recentered: true` | not applicable | +| Next build, daemon prepared | `window_build_mode: "incremental"`, no reason | `full_rescan_reason: "window_daemon_state_lost"` | +| Next build, daemon could not be prepared at adoption | the daemon's outcome at adoption: `window_daemon_busy`, `window_daemon_unavailable`, `window_value_out_of_range` or `window_mirror_divergence` | `window_daemon_state_lost` | +| Daemon counters | one extra `window_prepares` at the self-check build | `spawns` increments on a restart | + +Both recoveries take the existing path: one full oracle read that +re-prepares the daemon, after which the cache is a daemon mirror again. The +byte-exact parity of the window, the 5/8 band floor, the 2x ceiling and the +>1.25x re-center threshold are unchanged. + +## 10. Testnet soak thresholds (issue #228) + +Operator-run; the deterministic equivalents are in +`tests/test_prism_payout_state.py` (`PayoutWindowRescanBoundTests`) and +`tests/test_prism_payout_window_daemon_recenter.py`. Run a 24 h soak on +testnet with `PRISM_WINDOW_PIPELINE_RUST=1` and extract events as in section +3, then: + +```sh +jq -s ' + [.[] | select(.event == "payout_artifact_built")] as $built + | { + full_rescans: ($built | map(select(.window_build_mode == "full_rescan")) | length), + reconcile_invalidation: ($built | map(select(.full_rescan_reason == "reconcile_invalidation")) | length), + daemon_state_lost: ($built | map(select(.full_rescan_reason == "window_daemon_state_lost")) | length), + inherited_forces: ($built | map(select(.cause_inherited == true and .force_origin != "none")) | length), + p95_build_seconds: ($built | map(.duration_seconds) | sort | .[((length * 0.95 | ceil) - 1)]) + } +' "$PRISM_ARTIFACT_EVENTS" +``` + +Pass criteria for the 24 h window, before any mainnet rollout: + +| Measure | Threshold | +|---|---| +| Full rescans per hour (`full_rescans / 24`) | at most 2 | +| `reconcile_invalidation` full rescans | 0 | +| `window_daemon_state_lost` full rescans | at most the daemon `spawns` count over the window | +| `inherited_forces` | 0 | +| p95 `duration_seconds` of `payout_artifact_built` | at most 3 s | +| Tip-refresh waves with elapsed above 3 s whose outcome is `fanout_superseded` | none attributable to a payout build (`phase_seconds.lock_wait` under 1 s on every build during the wave) | diff --git a/docs/prism-postgres-replica.md b/docs/prism-postgres-replica.md new file mode 100644 index 000000000..50c2c7b6a --- /dev/null +++ b/docs/prism-postgres-replica.md @@ -0,0 +1,250 @@ +# PRISM Postgres read replica + +`prism-public-api` — the public dashboard and audit read tier extracted from +the coordinator in issue #145 — answers from a hot standby, not from the +coordinator's primary. This document covers how that standby is provisioned in +compose, what a production deployment has to do by hand instead, and how to +operate the replication slot it depends on. + +Scope note: everything below describes what is in this repository. The +production topology beyond the compose files (placement, storage classes, +backup schedules, monitoring stack) is not modelled here and is not guessed at. + +## What the compose wiring does + +`compose.yaml` defines two Postgres services under the `prism` profile: + +- `prism-postgres` — the primary the coordinator writes to. +- `prism-postgres-replica` — a streaming standby, read-only, that + `prism-public-api` reads through `PRISM_PUBLIC_DATABASE_URL`. + +The primary runs `postgres -c hba_file=/etc/postgresql/pg_hba.conf` with +`config/prism-postgres/pg_hba.conf` bind-mounted read-only. That file +reproduces the stock image's authentication rules and adds the two +`host replication ...` rules the standby connects through. Postgres 16 already +defaults to `wal_level=replica`, `max_wal_senders=10` and +`max_replication_slots=10`, which is enough for one standby, so no other server +settings are overridden. + +The standby's data directory is never `initdb`'d. +`config/prism-postgres/replica-entrypoint.sh` runs first and, when the +directory does not already hold a completed standby, streams one from the +primary: + +``` +pg_basebackup --host=prism-postgres --port=5432 --username="$PRISM_POSTGRES_USER" \ + --pgdata=/var/lib/postgresql/data --format=plain --wal-method=stream \ + --progress --write-recovery-conf --slot=prism_public_replica --create-slot +``` + +`--create-slot` creates the physical slot on the primary; +`--write-recovery-conf` writes `standby.signal` and a `primary_conninfo` +(including `primary_slot_name`) into `postgresql.auto.conf`. The script then +`exec`s the stock image entrypoint, which sees a populated data directory, +skips initialization, and starts the server in standby mode. + +Operational details worth knowing: + +- **Bootstrap retries.** The base backup is retried on a bounded loop (60 + attempts, 5s apart, tunable through + `PRISM_POSTGRES_REPLICA_BASEBACKUP_ATTEMPTS` and + `PRISM_POSTGRES_REPLICA_BASEBACKUP_RETRY_SECONDS`) so a cold + `docker compose up` succeeds even before the primary is accepting + connections. Exhausting the budget exits the container non-zero rather than + looping forever. +- **Slot reuse.** The primary keeps the slot while the standby is gone, so + `--create-slot` would fail on a rebuild. The script checks + `pg_replication_slots` first and reuses an existing slot instead of + recreating it. +- **The replica volume is disposable.** `standby.signal`, not `PG_VERSION`, is + the completion marker: `pg_basebackup` writes `PG_VERSION` early in the + stream but `standby.signal` only at the end, so an interrupted bootstrap is + distinguishable from a usable standby. Anything in the data directory without + `standby.signal` is treated as debris from an interrupted bootstrap and is + cleared before retrying. Never point this service at a data directory holding + a cluster you care about. +- **Authentication rules on the standby.** The base backup copies the + *primary's data directory* `pg_hba.conf` — the one `initdb` wrote, which + trusts loopback — not the mounted `hba_file`. That is what the standby then + runs with, and it is why the standby's healthcheck names the role explicitly: + an unauthenticated `pg_isready` would reach the role lookup on a trusted + connection and log `FATAL: role "root" does not exist` on every interval. +- **Healthcheck.** TCP listener, then a real `SELECT 1`, then + `pg_is_in_recovery()` returning `t`. The last clause means an out-of-band + promotion makes the service report unhealthy instead of quietly leaving the + read tier pointed at a second writable cluster. + +### Non-default superuser names + +`config/prism-postgres/pg_hba.conf` is a plain file; compose does not +interpolate it. The replication rules name the default superuser `qbit` +literally: + +``` +host replication qbit 0.0.0.0/0 scram-sha-256 +host replication qbit ::/0 scram-sha-256 +``` + +A deployment that overrides `PRISM_POSTGRES_USER` **must edit those two lines** +to match, or the base backup fails with `no pg_hba.conf entry for replication +connection from host ...`. + +### Settings + +| Variable | Default | Effect | +| --- | --- | --- | +| `PRISM_POSTGRES_REPLICATION_SLOT` | `prism_public_replica` | Physical slot the standby creates and streams through. | +| `PRISM_POSTGRES_REPLICA_DATA_SOURCE` | `prism-postgres-replica-data` | Volume or host path backing the standby's data directory. Release-provenance validation requires an absolute host path that neither overlaps nor aliases any other enabled state source (including ancestors, descendants and symlinks). Pre-create it before production startup. | +| `PRISM_PUBLIC_DATABASE_URL` | Empty (derived by the public service) | Read-only DSN `prism-public-api` uses. Empty builds a URL for `prism-postgres-replica:5432` using `PRISM_POSTGRES_USER`, `PRISM_POSTGRES_PASSWORD`, and `PRISM_POSTGRES_DB`. An explicit URL overrides that default. Compose passes an explicit URL as `PRISM_DATABASE_URL`; otherwise the public service percent-encodes the separate credentials when constructing its URL. The coordinator keeps its separate primary DSN. | +| `PRISM_PUBLIC_PSQL_COMMAND` | Empty | Optional command for the public psql backend. Compose passes it as `PRISM_POSTGRES_PSQL_COMMAND` only to the public service; empty derives `psql` from `PRISM_PUBLIC_DATABASE_URL`. The coordinator keeps its own `PRISM_POSTGRES_PSQL_COMMAND`. | +| `PRISM_PUBLIC_REPLICA_MODE` | `require` in compose, `off` in code | `require` refuses replica-backed routes unless the backing server is in recovery with a live replication stream. `off` serves whatever the DSN names — the behaviour before a standby existed. | +| `PRISM_PUBLIC_REPLICA_MAX_LAG_SECONDS` | `60` | How long the standby's replication stream may be silent before its answers stop counting as current. | + +Leave `PRISM_PUBLIC_DATABASE_URL` empty for the built-in standby so the public +connection follows the configured Postgres credentials. For an external standby +or separate read credentials, set the full URL explicitly and update it when +those credentials change. + +With `PRISM_POSTGRES_NATIVE_CLIENT=psql`, the public service must still connect +to the standby. Leave `PRISM_PUBLIC_PSQL_COMMAND` empty to use the public DSN, +or set a separate command that reaches the replica if a wrapper or extra psql +arguments are required. The coordinator's primary command is never inherited +by the public service. + +`PRISM_PUBLIC_REPLICA_MODE` defaults to `off` in the code and `require` in the +shipped compose. That split is deliberate: merging the replica work must not +503 a deployment that has not provisioned a standby yet, but anyone who brings +the stack up from this repository's compose gets the enforced contract without +having to opt in. Cut over by provisioning the standby, pointing +`PRISM_PUBLIC_DATABASE_URL` at it, and then setting `require`. + +### What the bound actually bounds + +The enforced number is the **walreceiver heartbeat age**, not replay lag. +Heartbeats flow every `wal_receiver_status_interval` (10s by default) whether +or not the primary has anything to send; replay lag grows with wall clock on an +idle primary. Bounding replay lag would therefore 503 a perfectly healthy +standby every time the pool went quiet — exactly when the dashboard is least +likely to be wrong. + +Replay lag and apply backlog are still measured and published, because they are +the numbers an operator wants: + +- `X-Prism-Replica-Lag-Seconds` on every replica-backed response; +- the `replica` block in `prism-public-api`'s `/healthz`; +- `qbit_prism_public_replica_*` gauges on its `/metrics`, including + `qbit_prism_public_replica_refusals_total`. + +Freshness ages locally: the bound is checked against the heartbeat age the last +successful probe reported **plus how long ago that probe ran**. A probe that +starts failing ages its own last good answer out of the bound rather than +pinning it there, so a standby that stops answering entirely ends in a 503 +rather than an indefinite 200. + +Routes that read no replica state are not gated — today that is +`/public/v1/mining-configuration`, assembled from environment, and the +content-addressed `/public/v1/artifacts/{sha256}`, whose body either hashes to +the requested digest or does not. The classification comes from +`lab/prism/endpoint_registry.py`, not a hand-kept list. +| `PRISM_POSTGRES_REPLICA_BASEBACKUP_ATTEMPTS` | `60` | Bootstrap retry budget. | +| `PRISM_POSTGRES_REPLICA_BASEBACKUP_RETRY_SECONDS` | `5` | Delay between bootstrap attempts. | + +## Production provisioning checklist + +The compose wiring is a lab convenience: it bootstraps as the superuser over a +network it trusts. A production standby should be provisioned deliberately. +This is a checklist, not automation — nothing in this repository performs these +steps. + +1. **Create a dedicated replication role on the primary.** Do not replicate as + the superuser. + + ```sql + CREATE ROLE prism_replication REPLICATION LOGIN PASSWORD ''; + ``` + +2. **Authorize it in `pg_hba.conf`**, scoped to the standby's address rather + than the whole internet: + + ``` + host replication prism_replication scram-sha-256 + ``` + + Reload the primary (`SELECT pg_reload_conf()`) and confirm the rule is live + in `pg_hba_file_rules`. + +3. **Take the base backup with an explicit slot**, from the standby host: + + ``` + pg_basebackup --host= --port=5432 --username=prism_replication \ + --pgdata= --format=plain --wal-method=stream \ + --progress --write-recovery-conf --create-slot --slot=prism_public_replica + ``` + +4. **Put the standby's data and WAL on storage separate from the primary's.** A + standby that shares a device with its primary protects against neither a + full disk nor a device failure, and its read traffic competes with the + primary's writes. + +5. **Verify before pointing traffic at it**: `pg_is_in_recovery()` returns `t`, + the slot shows `active = true` on the primary, and a row written on the + primary is visible on the standby. + +## Replication slot management + +The physical slot is what makes the standby reliable and what makes it +dangerous to forget. While the slot exists, the primary refuses to recycle WAL +the standby has not consumed — **including while the standby is down**. An +abandoned slot fills the primary's WAL volume and eventually stops the primary. + +Monitor it on the primary: + +```sql +SELECT slot_name, active, restart_lsn, wal_status, + pg_size_pretty(pg_wal_lsn_diff(pg_current_wal_lsn(), restart_lsn)) AS retained + FROM pg_replication_slots; +``` + +- `active` — `false` for anything longer than a deliberate standby restart + means WAL is piling up for a consumer that is not reading it. +- `restart_lsn` — the oldest LSN the primary must keep. A `restart_lsn` that + stops advancing is the leading indicator of a stuck standby. +- `wal_status` — `reserved` is normal, `extended` means retention has grown + past `max_wal_size` and is the point to act on. `unreserved` means the + required WAL is at risk of removal, and `lost` means it is gone and the + standby can no longer catch up. **Alert when `wal_status` leaves + `reserved`/`extended`.** + +Setting `max_slot_wal_keep_size` on the primary caps that retention: the slot +is invalidated (`wal_status = 'lost'`) instead of the primary filling its disk. +That trades an unusable standby for a healthy primary, which is normally the +right trade for a read replica. Nothing in this repository sets it. + +**Decommissioning.** When the replica is retired, drop the slot on the primary +in the same change — the primary keeps retaining WAL for it otherwise: + +```sql +SELECT pg_drop_replication_slot('prism_public_replica'); +``` + +Dropping the slot cannot be undone: the standby cannot resume streaming, and +bringing one back requires a **fresh base backup**. That is also the recovery +path for a slot invalidated with `wal_status = 'lost'`. + +## Failover semantics + +The replica is a **read scaling tier, not a failover target**. + +- It is read-only. Writes fail with `cannot execute INSERT in a read-only + transaction`. +- There is **no automated promotion anywhere in this repository** — no + `pg_promote` call, no `promote_trigger_file`, no cluster manager. A standby + only becomes writable if an operator promotes it by hand. +- Promotion is not a supported operation for this standby. The coordinator's + writer lease, ledger attestation, and audit invariants are all defined + against the primary; promoting the read replica does not produce a valid + PRISM primary. Losing the primary is a primary-restore problem, and + `docs/prism-ledger-ops.md` is the relevant runbook. +- Because promotion is out of scope, the standby's healthcheck fails when + `pg_is_in_recovery()` stops returning `t`. A promoted standby is a + misconfiguration, and it is meant to be loud. diff --git a/docs/prism-public-operator-endpoint-split.md b/docs/prism-public-operator-endpoint-split.md new file mode 100644 index 000000000..1d83a623a --- /dev/null +++ b/docs/prism-public-operator-endpoint-split.md @@ -0,0 +1,310 @@ +# PRISM Public / Operator Endpoint Split + +This document maps every endpoint the PRISM HTTP surface serves and classifies +each one as **public read surface**, **operator surface**, or +**payout-affecting**. It is the prerequisite for issue #145, which moves the +public read tier out of the coordinator process. + +It exists so a reviewer can disagree with one classification without re-reading +the HTTP layer. Each row carries the evidence its classification rests on, and +the classification itself is declared in +[`lab/prism/endpoint_registry.py`](../lab/prism/endpoint_registry.py) and pinned +against real dispatch by `tests/test_prism_endpoint_registry.py`. Nothing here +changes request handling; this is the boundary written down. + +## Result + +**Exactly the 13 `/public/v1/*` routes are extracted. Everything else stays.** + +That is a narrower answer than "move the read traffic", and the narrowing is the +substance of this document. Three findings drove it. + +### 1. The boundary is already written down, and it excludes `/audit/*` + +`PRISM.md` states that the coordinator "exposes a private audit/ops listener and +a dashboard-safe public API from the same process", lists `/healthz`, +`/metrics`, `/owed-balances` and every `/audit/*` route as private/internal, +instructs operators not to expose `/audit/*` to the internet, and concludes: +"operators can expose only `/public/v1` through a reverse proxy or dashboard +frontend." + +`docs/public-dashboard-api/README.md` repeats it as a contract — "the public API +must not expose `/audit/*`, `/metrics`, `/healthz`, operator controls, raw +private sockets, credentials, or unrestricted internal manifests" — and +`tests/test_public_dashboard_api_contract.py` enforces it by asserting the +OpenAPI document contains no `/audit/` path. + +The issue asks that published audit evidence be served from the artifact files +rather than through the coordinator. It already is, and through a public route: +`/public/v1/artifacts/{sha256}` resolves bodies from `body_uri` on disk when the +inline column is NULL, verified against the requested sha256, and +`/public/v1/blocks/{block_hash}/settlement-artifacts` links to it. **The +evidence content is public; the `/audit/*` routes are not.** They are internal +tooling over the same data: raw `sats` units, no pagination, full +broadcast-attempt histories, `parent_coinbase_tx_hex` and `manifest_set_json` +broadcast material, and in one case a caller-controlled unbounded window. + +So the `PRISM_PUBLIC_ARTIFACT_CACHE_*` knobs move with the extracted service +because `/public/v1/artifacts/{sha256}` moves — which is exactly what the issue +predicted, and it lands without widening the public surface. + +### 2. Two nominally public routes take the writer lock today + +This is the finding that most changes the shape of the work. + +`/public/v1/miners/{recipient_id}` — the most-polled route on the surface — +resolves an owed balance by calling `current_owed_balances()`, which returns +*every* recipient's balance under the writer lock and then filters in Python. +`/public/v1/blocks/{block_hash}/settlement-artifacts` falls through to +`audit_bundle()` under the writer lock for direct-coinbase blocks. + +Both must be severed before the extraction is worth anything. Moving the process +without fixing these would move the GIL contention but keep the lock contention. + +### 3. Some routes that look public are operator surfaces + +The issue predicted the drift count and the health probes. The sweep found more: +`/owed` and `/owed-balances`, the `/payouts/{recipient_id}/status` alias, and +the entire `/audit/*` family. `/miners/{recipient_id}/status` is not even listed +in `PRISM.md`'s endpoint inventory and has no consumer outside tests — it is +legacy, superseded by `/public/v1/miners/{recipient_id}`. + +## The three axes + +### Audience + +- **Public read** — the documented, versioned dashboard contract under + `/public/v1`. +- **Operator** — process state, or the private audit/ops tooling. +- **Payout-affecting** — live payout obligations: the state the settlement + machinery itself acts on. Out of scope for extraction by the terms of the + issue. + +### Ledger access + +How the response reaches durable state. This decides what may move to a replica, +and it is the axis that matters for the money path. + +| Tier | Mechanism | Consequence | +| --- | --- | --- | +| `WRITER_LOCK` | `_operation_gate(self._lock, "writer lock")` in `share_ledger.py` | Serializes against the lease-holding writer. Public volume here is share-ingest backpressure. | +| `READ_SLOT` | `_run_read_json` → `_operation_gate(self._read_semaphore, "read slot")`, bounded by `PRISM_POSTGRES_READ_CONCURRENCY` (default 4) | Primary Postgres, but not the writer lock. Replica-eligible as written. | +| `ARTIFACT_FILE` | The audit artifact directory (`PRISM_AUDIT_DIR`) | Already the durable public record. No database involved. | +| `PROCESS_STATE` | In-memory coordinator counters, gauges, snapshots | Cannot leave the process at all. | +| `QBIT_RPC` | `coordinator.rpc.call(...)` | A node read, not coordinator state. The extracted service gets its own client. | + +The writer-lock reads reached from HTTP, verified against the `PsqlShareLedger` +in `lab/prism/share_ledger.py` (`def` line, then the line holding the gate): + +| Ledger method | Definition | Gate | Reached by | +| --- | --- | --- | --- | +| `current_owed_balances` | `:4161` | `:4172` | `/owed`, `/owed-balances`, `/miners/{id}/status`, **`/public/v1/miners/{id}`** | +| `carry_forward_integrity_report` | `:4259` | `:4261` | `/audit/carry-forward-integrity` | +| `audit_share_window` | `:4327` | `:4353` | `/audit/share-window` | +| `audit_block_payouts` | `:4365` | `:4383` | `/audit/blocks/{h}/payouts`, `/audit/block/{h}` | +| `recipient_payout_history` | `:4389` | `:4418` | `/miners/{id}/status` | +| `audit_bundle` | `:4906` | `:4927` | `/audit/blocks/{h}/bundle`, **`/public/v1/blocks/{h}/settlement-artifacts`** | +| `audit_bundle_by_commitment` | `:4931` | `:4958` | `/audit/commitments/{leaf}/bundle` | + +The SQL these run is read-only — the stored functions they call +(`qbit_current_owed_balances`, `qbit_prism_window`, `qbit_audit_block_fanouts`, +`qbit_fanout_status`, `qbit_audit_share_window`) are declared `STABLE` in +`crates/qbit-prism/sql/001_share_ledger.sql`. The cost is the in-process gate, +not a database lock, which is what makes these replica-eligible once the read is +routed off the writer. No HTTP route on any path performs an `INSERT`, `UPDATE`, +`DELETE` or advisory lock. + +`public_api.py` also carries pre-read-model fallbacks that reach +`recipient_payout_history` and `ledger.all_shares()` when a `dashboard_*` method +is absent. On the production `PsqlShareLedger` every read model exists, so those +branches are unreachable; they matter only to the in-memory ledger used when +`PRISM_ALLOW_MEMORY_LEDGER=1`. The extracted service must keep that true, or the +fallbacks would reintroduce writer-lock reads on public routes. + +## Operator surface — retained + +| Path(s) | Ledger access | Why operator | +| --- | --- | --- | +| `/healthz` | `PROCESS_STATE` | Liveness of this instance: job-build progress, tip refresh, writer-lease state. Answered from a background-refreshed in-memory snapshot (`PRISM_HEALTH_REFRESH_SECONDS`, default 5s), overlaid with live progress on every request so a cached `ok=true` cannot mask a failed refresh. Wired to the compose healthcheck (`compose.yaml`, pinned by `tests/test_prism_compose_profile.py`) and to `make prism-self-check`. | +| `/metrics` | `PROCESS_STATE` | Prometheus exposition of this process's counters. Serves only the complete cached snapshot (`PRISM_METRICS_REFRESH_SECONDS`, default 5s). | +| `/audit/latest` | `ARTIFACT_FILE` | The operator's view of the live evidence envelope. Public consumers reach the same evidence content-addressed via `/public/v1/artifacts/{sha256}`. | +| `/audit/share-window` | `WRITER_LOCK` | Raw per-share rows for a caller-supplied anchor. The window is caller-controlled and unbounded. The public equivalent is the aggregated reward leaderboard. | +| `/audit/blocks/{block_hash}/payouts`, `/audit/block/{block_hash}` | `WRITER_LOCK` | Per-recipient payout rows in raw sats with `maturity_state`/`chain_state`. The short form is a legacy alias for the same handler. Public per-miner slice: `/public/v1/miners/{id}/payouts`. | +| `/audit/blocks/{block_hash}/bundle` | `WRITER_LOCK` + `ARTIFACT_FILE` | Full inline bundle plus `coinbase_tx_hex`. The bundle body itself is public, content-addressed. | +| `/audit/commitments/{commitment_leaf_hex}/bundle` | `WRITER_LOCK` + `ARTIFACT_FILE` | The same bundle by commitment leaf. No public commitment-leaf lookup exists. | +| `/audit/blocks/{block_hash}/ctv-fanout-manifest-set` | `READ_SLOT` | Full CTV recovery payload including `parent_coinbase_tx_hex` and `manifest_set_json` — broadcast material. | +| `/audit/blocks/{block_hash}/ctv-fanouts` | `READ_SLOT` | Raw artifact dicts with `fanout_tx_hex` and sats fields. | +| `/audit/fanouts/pending` | `READ_SLOT` | The broadcaster's work queue with full attempt histories. | +| `/audit/fanouts/{fanout_txid}/status` | `READ_SLOT` | Raw status including `submit_result` and attempt detail. | + +## Payout-affecting — retained, out of scope + +| Path(s) | Ledger access | Why | +| --- | --- | --- | +| `/audit/carry-forward-integrity`, `/audit/ledger-integrity` | `WRITER_LOCK` | The drift count (`current_drift_count`), mismatch list, and the `audit_head_sha256` operators are told to mirror after payout-affecting blocks. Reports whether the payout ledger disagrees with itself. Runs two statements under one held writer gate — the worst route on the surface to expose to uncontrolled volume. | +| `/owed`, `/owed-balances` | `WRITER_LOCK` | Every recipient's outstanding balance: the obligations the next settlement will pay. O(recipients) under the writer lock. There is no public all-recipients dump. | +| `/miners/{recipient_id}/status`, `/payouts/{recipient_id}/status` | `WRITER_LOCK` | The pre-dashboard payout-operations view in raw sats. Both aliases reach one handler. Superseded for public use by `/public/v1/miners/{recipient_id}`, which reports bits. | + +## Public read surface — extracted + +| Path | Ledger access | Extraction blocker | +| --- | --- | --- | +| `/public/v1/pool-summary` | `READ_SLOT` + `QBIT_RPC` | None. Needs its own qbit RPC client. | +| `/public/v1/blocks` | `READ_SLOT` | None. | +| `/public/v1/leaderboard` | `READ_SLOT` + `QBIT_RPC` | None. `window=reward` needs network difficulty from RPC. | +| `/public/v1/hashrate-series` | `READ_SLOT` | None. The widest ledger scan on the surface, and the strongest single argument for moving this traffic off the primary. | +| `/public/v1/mining-configuration` | none | Falls back to `getattr(coordinator, "port")` / `getattr(coordinator, "bind")` when the public URL env vars are unset. `PRISM_PUBLIC_STRATUM_URL` / `PRISM_STRATUM_PORT` must be supplied explicitly. | +| `/public/v1/miners/{recipient_id}` | `WRITER_LOCK` + `READ_SLOT` + `QBIT_RPC` | `owed_balance_for_recipient()` calls `current_owed_balances()`, a writer-lock read returning every recipient's balance, filtered in Python. Needs a per-recipient owed-balance read that does not fence. | +| `/public/v1/miners/{recipient_id}/earnings` | `READ_SLOT` | None. | +| `/public/v1/miners/{recipient_id}/payouts` | `READ_SLOT` | None. | +| `/public/v1/miners/{recipient_id}/workers` | `READ_SLOT` | None. | +| `/public/v1/blocks/{block_hash}/settlement-artifacts` | `WRITER_LOCK` + `READ_SLOT` + `ARTIFACT_FILE` | The CTV path is read-slot, but a direct-coinbase block falls through to `direct_coinbase_settlement_payload()` → `audit_bundle()` under the writer lock. Take that branch from the artifact file. | +| `/public/v1/fanouts/pending` | `READ_SLOT` | None. | +| `/public/v1/fanouts/{fanout_txid}` | `READ_SLOT` | None. | +| `/public/v1/artifacts/{sha256}` | `READ_SLOT` + `ARTIFACT_FILE` | None. Already resolves the body from `body_uri` on disk when the inline column is NULL. Carries the `PRISM_PUBLIC_ARTIFACT_CACHE_*` knobs. | + +## What makes this an extraction rather than a rewrite + +`public_api.py` reaches the coordinator through a small, complete surface: + +- `coordinator.ledger` — 24 references, 10 direct methods plus 7 duck-typed + `getattr` read models +- `coordinator.rpc` — 3 references: `getblockchaininfo`, `getblocktemplate`, + `getnetworkinfo` +- `getattr(coordinator, "port")` and `getattr(coordinator, "bind")`, both with + environment fallbacks already in place + +There is no other coordinator state on the public path. The extracted service +supplies its own object with `.ledger` and `.rpc`, and the dashboard code is +unchanged. + +## Completeness + +The repository contains exactly one HTTP-serving surface outside tests: the +`AuditHttpFacade` listener in `lab/prism/audit_http.py`, fed by +`_CoordinatorAuditHttp` and `public_api.dispatch`. `make_audit_handler` in +`prism_coordinator.py` is a compatibility factory over the same facade, not a +second surface. The coordinator binds only the Stratum listener(s) and the audit +HTTP port. No Rust crate serves HTTP. Every route string in `tests/`, `test/`, +`docs/` and `doc/` falls inside the union of the routes classified here; the +only exception is a test-only `/control` route in the fake qbit RPC server. + +## Staleness is a contract + +The precedent is in `lab/prism/observability.py`: the cached `/metrics` endpoint +returns 503 rather than serving an unbounded-stale snapshot. + +- `MINIMUM_HEALTH_STALE_SECONDS = 15.0` +- `stale_after = max(3 * refresh_seconds, MINIMUM_HEALTH_STALE_SECONDS)` +- `/healthz` returns 503 with `snapshot_age_seconds` once over budget +- `/metrics` returns `503 if stale else 200` and always emits + `qbit_prism_metrics_snapshot_age_seconds`, `qbit_prism_metrics_snapshot_stale` + and `qbit_prism_metrics_snapshot_available` + +**Landed.** Each extracted endpoint states the staleness it tolerates, surfaces +the observed age to callers, and refuses rather than serving beyond its budget. +The budgets live in the `max_staleness_seconds` field of `Endpoint` in +`lab/prism/endpoint_registry.py`, derived rather than hand-picked: + +``` +budget = max(3 * (cache_ttl_seconds + underlying_cache_seconds), + MINIMUM_PUBLIC_STALE_SECONDS) +``` + +with `MINIMUM_PUBLIC_STALE_SECONDS = 15`, matching the precedent's floor. The +derivation is documented in that module's docstring and pinned by +`tests/test_prism_endpoint_registry.py`, which asserts the budgets against the +formula and against `public_cache_policy` itself, so a changed cache default +cannot silently leave a budget behind. + +Three sources of staleness compound and must all be counted: + +1. The shared in-process response cache (`PublicResponseCache`), whose TTL is + already per-route via `public_cache_policy`. Counted. +2. The pool reward-window aggregate cache in the ledger + (`_pool_reward_window_aggregate`, `PRISM_PUBLIC_REWARD_WINDOW_CACHE_SECONDS`, + default 30s), which sits underneath every miner page. Counted, and it is the + only route where two caches sit in series: `/public/v1/miners/{recipient_id}` + at 105s versus 15s for the other plain row reads. +3. Replica lag, once a replica exists. Contributes 0 today because no replica + exists; when one lands it becomes another `underlying_cache_seconds` term + rather than a new mechanism. + +On the wire, every extracted response carries +`X-Prism-Staleness-Budget-Seconds` beside the existing `Age` header. Past the +budget the service returns 503 with `Cache-Control: no-store` and an ordinary +`prism.dashboard.error.v1` body naming both the budget and the observed age — +error code `upstream_unavailable`, already in the documented enum, rather than a +new error schema. + +At the documented cache defaults the budgets are 15s for the plain row reads +(`/blocks`, `/leaderboard`, `/miners/{id}/earnings`, `/miners/{id}/payouts`, +`/blocks/{h}/settlement-artifacts`, `/fanouts/pending`, `/fanouts/{txid}`), 90s +for the pool-wide aggregates (`/pool-summary`, `/hashrate-series`, +`/miners/{id}/workers`), 105s for `/miners/{recipient_id}`, and 900s for +`/mining-configuration`. + +`/public/v1/artifacts/{sha256}` is the exception, and says so explicitly through +the registry's `immutable_content` flag rather than by omitting a budget: content +is immutable and content-addressed, so a body that hashes to the requested +sha256 is correct at any age. It reports `X-Prism-Staleness-Budget-Seconds: +unbounded` and never refuses for staleness. The only observable staleness is the +404→200 transition for an artifact published moments ago, which no budget would +fix. + +## Health is a contract + +The coordinator's `/healthz` answers one question: is this instance failing to +deliver work to the miners it has admitted? It fails closed (HTTP 503, with +`initial-delivery-stalled` in `unhealthy_reasons`) for two conditions, both +measured against `PRISM_STRATUM_INITIAL_JOB_TIMEOUT_SECONDS`: + +- **Genuine first-job starvation.** An authorized miner has waited the full + deadline without any delivered work + (`oldest_genuinely_pending_initial_job_age_seconds`). +- **Sustained semantic current-work loss.** Fewer than 95% of authorized miners + hold work matching the current template fingerprint, payout generation, and + their current connection, authorization, and difficulty generations + (`semantic_current_work_ratio`) for longer than the deadline + (`semantic_current_work_gap_age_seconds`), with no exact-tip work covering + the gap either. + +Exact-tip churn by itself stays diagnostic. When the observed tip advances +faster than fanout, `current_tip_job_coverage` drops and +`current_tip_coverage_gap_age_seconds` grows while miners still hold the latest +published work; the 2026-08-31 incident read 4 of 33 miners on the exact tip +with all 33 semantically current. Those fields keep reporting the churn, and +exact coverage still corroborates `overload` and the +`connection-capacity-saturated` and `stale-unknown-rejection-storm` reasons, +each of which needs an independent pressure signal. It no longer fails health +on its own (issue #216). + +Router-facing mining readiness, whether a load balancer should send new miners +to this instance, is a distinct signal to be added by issue #186. It is not +derivable from `/healthz`, and this document does not classify it. + +## Out of scope + +Anything payout-affecting stays on the coordinator: the writer lease, share +acknowledgement, block landing, settlement, and the surfaces listed above. The +extracted service is read-only — it opens no write path to the ledger and holds +no lease. That is now structural rather than aspirational: it is constructed +without any `PRISM_LEDGER_WRITER_*` input, receives none of the signing or +block-submission configuration in compose, mounts the audit artifact volume +`:ro`, and refuses to start on the in-memory ledger (whose missing `dashboard_*` +read models would send `public_api`'s fallbacks back onto writer-lock reads). +`tests/test_prism_public_read_service.py` drives every extracted route against a +ledger that raises if any writer-lock read is reached. + +The two blockers named above are removed. `dashboard_miner_owed_balance_bits` +and `dashboard_direct_coinbase_settlement` on `PsqlShareLedger` answer both +formerly-fenced public reads through `_run_read_json`, and `public_api` prefers +them through the same duck-typed pattern used by every other read model, keeping +the old paths as in-memory-ledger fallbacks. + +Replica wiring is tracked separately. It needs `wal_level`, a replication slot, +and a standby service in the Postgres setup; today `prism-postgres` has a +dedicated WAL volume (`POSTGRES_INITDB_WALDIR`) but no replication +configuration. Until that lands, the extracted service reads the primary — a +separate process with its own connection pool, which already removes the GIL +contention, but not yet a separate database. diff --git a/docs/prism-rejections.md b/docs/prism-rejections.md index 19299f9c6..20b343ada 100644 --- a/docs/prism-rejections.md +++ b/docs/prism-rejections.md @@ -80,7 +80,7 @@ hash against the tip for which the refresh path published coherent work, not against a per-share `getbestblockhash` RPC. This removes the submit-races-ahead-of-the-refresh failure mode: a submit-path RPC could observe a new tip seconds before jobs refreshed, and with -`PRISM_STRATUM_STALE_GRACE_SECONDS=0` (mainnet-forced) that rejected every +`PRISM_STRATUM_STALE_GRACE_SECONDS=0` (then pinned on mainnet) that rejected every in-flight share on the old tip. The observed tip is also the anchor the stale-grace window and evicted-job classification already use, so all three now agree. diff --git a/docs/public-dashboard-api-v1.openapi.yaml b/docs/public-dashboard-api-v1.openapi.yaml index c8d9220a6..3dd0a086e 100644 --- a/docs/public-dashboard-api-v1.openapi.yaml +++ b/docs/public-dashboard-api-v1.openapi.yaml @@ -40,7 +40,14 @@ paths: get: tags: [dashboard] operationId: getHashrateSeries - summary: Get chart-ready pool or miner hashrate time series. + summary: Get chart-ready pool or miner credited hashrate time series. + description: > + Every rate is credited hashrate, derived from the accepted share + difficulty the pool credited to the subject in each bucket. It is not + physical router telemetry: while the assigned share target and the + router's own share filtering disagree, neither the raw nor the smoothed + series reproduces what the router reports. Buckets with no credited + shares are omitted; consumers synthesize gaps. parameters: - name: subject in: query @@ -60,13 +67,37 @@ paths: - name: bucket in: query required: false + description: > + Each range admits a fixed set of buckets: 1w allows 5m, 1h, and + 1d; 1m allows 1h and 1d; 6m and all allow only 1d. `auto` always + resolves inside the allowed set. A combination outside that + vocabulary is rejected with `400 bad_request` naming the allowed + buckets. schema: allOf: - $ref: "#/components/schemas/HashrateBucket" default: auto + - name: view + in: query + required: false + schema: + type: string + enum: [both] + description: > + Omit for the `prism.dashboard.hashrate-series.v1` response, whose + `hashrate_ths` is the smoothed credited rate. `both` returns + `prism.dashboard.hashrate-series.v2` carrying the raw and the + smoothed credited rate for every bucket. Any other non-empty value + is rejected with `400 bad_request`. The view is part of the cache + key, so the two responses never share a cache entry. `bucket=auto` + resolves to 1h for 1w/1m and 1d for 6m/all; with the default 30m + smoothing window those coarse series have `method=none`. Request + `bucket=5m` to compare raw buckets with the 30m trailing rate. responses: "200": - description: Hashrate series. + description: > + Hashrate series. `prism.dashboard.hashrate-series.v1` without + `view`; `prism.dashboard.hashrate-series.v2` for `view=both`. headers: Cache-Control: $ref: "#/components/headers/BrowserCacheControl" @@ -79,7 +110,59 @@ paths: content: application/json: schema: - $ref: "#/components/schemas/HashrateSeriesResponse" + oneOf: + - $ref: "#/components/schemas/HashrateSeriesResponse" + - $ref: "#/components/schemas/HashrateSeriesDualRateResponse" + "400": + $ref: "#/components/responses/BadRequest" + "503": + $ref: "#/components/responses/UpstreamUnavailable" + /block-markers: + get: + tags: [dashboard] + operationId: getBlockMarkers + summary: Get found-block markers pre-bucketed onto the hashrate chart grid. + description: > + Serves the found-block markers the dashboard draws over the hashrate + series. Bucket epochs use the identical floor arithmetic the hashrate + buckets use, and bounded ranges keep only fully covered buckets, so + markers land in the same buckets as the hashrate points on every + timeframe. Buckets with no found blocks are omitted. Each bucket lists + at most its 3 most recent blocks and reports the full count, so long + ranges stay bounded without paging `/blocks`. Finer buckets than a + range's chart renders are rejected with `400 bad_request`: 1w allows + 5m/1h/1d, 1m allows 1h/1d, 6m and all allow 1d only. + parameters: + - name: range + in: query + required: false + schema: + allOf: + - $ref: "#/components/schemas/HashrateRange" + default: 1m + - name: bucket + in: query + required: false + schema: + allOf: + - $ref: "#/components/schemas/HashrateBucket" + default: auto + responses: + "200": + description: Found-block marker series. + headers: + Cache-Control: + $ref: "#/components/headers/BrowserCacheControl" + CDN-Cache-Control: + $ref: "#/components/headers/SharedCacheControl" + Vercel-CDN-Cache-Control: + $ref: "#/components/headers/SharedCacheControl" + Age: + $ref: "#/components/headers/Age" + content: + application/json: + schema: + $ref: "#/components/schemas/BlockMarkersResponse" "400": $ref: "#/components/responses/BadRequest" "503": @@ -149,9 +232,31 @@ paths: parameters: - $ref: "#/components/parameters/Page" - $ref: "#/components/parameters/Limit" + - name: chain_state + in: query + required: false + schema: + type: string + enum: [active, all, reversed] + default: active + description: > + Omit or pass `active` for the `prism.dashboard.blocks.v1` response, + which hides reorged-out blocks exactly as before this parameter + existed. `all` includes every recorded pool block and `reversed` + returns only blocks the pool once landed that a chain + reorganization later disconnected; both return + `prism.dashboard.blocks.v2` rows carrying `chain_state` and + `disconnected_at`. Pagination `total_count`/`total_pages` always + describe the filtered set. Any other value is rejected with + `400 bad_request`. The filter is part of the cache key, so the + views never share a cache entry. responses: "200": - description: Recent block rows. + description: > + Recent block rows. `prism.dashboard.blocks.v1` without + `chain_state` (or with `chain_state=active`); + `prism.dashboard.blocks.v2` for `chain_state=all` and + `chain_state=reversed`. headers: Cache-Control: $ref: "#/components/headers/BrowserCacheControl" @@ -164,7 +269,9 @@ paths: content: application/json: schema: - $ref: "#/components/schemas/BlocksResponse" + oneOf: + - $ref: "#/components/schemas/BlocksResponse" + - $ref: "#/components/schemas/BlocksChainStateResponse" "400": $ref: "#/components/responses/BadRequest" "503": @@ -278,8 +385,13 @@ paths: Serves exact canonical public artifacts such as audit bundles, CTV manifest sets, per-fanout manifests, or raw fanout transaction data. These artifacts are verified by their path SHA-256 and may use their own - `qbit.prism.*` schemas. Dashboard-facing wrapper responses link here - rather than inlining large canonical artifacts. + `qbit.prism.*` schemas. For audit bundles, CTV fanout manifests, and + manifest sets the response body is the exact canonical byte sequence + the hash was computed over, so `sha256(response body)` equals the path + SHA-256. Audit bundles written before canonical-byte persistence retain + the legacy reconstructed response until verified backfill. + Dashboard-facing wrapper responses link here rather than inlining large + canonical artifacts. parameters: - name: sha256 in: path @@ -548,7 +660,9 @@ components: schema: $ref: "#/components/schemas/ErrorResponse" UpstreamUnavailable: - description: Required pool or node data source is unavailable. + description: > + Required pool or node data source is unavailable, or the read exceeded + its statement deadline (error code `read_timeout`). content: application/json: schema: @@ -619,6 +733,7 @@ components: - tip_hash - bits - network_difficulty + - hashrate_ths - initial_block_download - peers properties: @@ -633,6 +748,19 @@ components: $ref: "#/components/schemas/CompactBits" network_difficulty: $ref: "#/components/schemas/DecimalString" + hashrate_ths: + oneOf: + - $ref: "#/components/schemas/DecimalString" + - type: "null" + description: > + Estimated network hashrate in TH/s for the permissionless lane the + pool mines, from the node's own recent-blocks estimate. Additive + field on `prism.dashboard.pool-summary.v1` (2.x). Divide a pool + `hashrate_ths` window (`h1`/`h3`) by this value for an approximate + pool share-of-network: pool hashrate is credited work from + accepted shares while this is a chain-derived estimate, so the + ratio is approximate by construction. `null` when the node cannot + answer; that failure never turns into a 503. initial_block_download: type: boolean peers: @@ -705,6 +833,8 @@ components: - participants_3h - blocks_found_total - prism_blocks_total + - blocks_reversed_total + - blocks_inactive_total - total_mined_bits - expected_time_to_block_seconds - latest_block @@ -720,9 +850,28 @@ components: blocks_found_total: type: integer minimum: 0 + description: > + Every pool block not reversed by a chain reorganization. Excludes + `reversed` blocks by design; it is not the sum of the per-state + counters below. prism_blocks_total: type: integer minimum: 0 + blocks_reversed_total: + type: integer + minimum: 0 + description: > + Pool blocks a chain reorganization disconnected from the active + chain. Additive field on `prism.dashboard.pool-summary.v1` (2.x); + list the affected blocks with + `GET /public/v1/blocks?chain_state=reversed`. + blocks_inactive_total: + type: integer + minimum: 0 + description: > + Pool blocks currently off the active chain but not yet resolved as + reversed. Additive field on `prism.dashboard.pool-summary.v1` + (2.x). total_mined_bits: $ref: "#/components/schemas/Bits" expected_time_to_block_seconds: @@ -783,7 +932,11 @@ components: timestamp: $ref: "#/components/schemas/IsoTimestamp" hashrate_ths: - $ref: "#/components/schemas/DecimalString" + allOf: + - $ref: "#/components/schemas/DecimalString" + description: > + Trailing-smoothed credited hashrate estimated from accepted share + difficulty. This is not router-side physical telemetry. accepted_share_count: type: integer minimum: 0 @@ -812,6 +965,179 @@ components: type: array items: $ref: "#/components/schemas/HashratePoint" + HashrateSmoothing: + type: object + additionalProperties: false + required: [method, window_seconds] + description: > + How `smoothed_hashrate_ths` was produced. `trailing` averages credited + difficulty over the trailing `window_seconds` ending at each bucket, + with buckets missing from the series counting as zero. `none` means + smoothing is disabled or configured shorter than two buckets: the + window is then the bucket itself and the smoothed rate equals the raw + rate. + properties: + method: + type: string + enum: [trailing, none] + window_seconds: + type: integer + minimum: 1 + DualRateHashratePoint: + type: object + additionalProperties: false + required: + - timestamp + - raw_hashrate_ths + - smoothed_hashrate_ths + - accepted_share_count + - accepted_share_difficulty + - complete + properties: + timestamp: + $ref: "#/components/schemas/IsoTimestamp" + raw_hashrate_ths: + allOf: + - $ref: "#/components/schemas/DecimalString" + description: > + Credited difficulty in this bucket divided by the full + `bucket_seconds`, even when the bucket is not yet complete. This + is a credited-work estimate, not router telemetry; sparse + high-difficulty shares can make an individual bucket over-read the + delivered physical rate. + smoothed_hashrate_ths: + allOf: + - $ref: "#/components/schemas/DecimalString" + description: > + Trailing-window credited rate, the same estimate v1 serves as + `hashrate_ths`. Equals `raw_hashrate_ths` when the smoothing + method is `none`. + accepted_share_count: + type: integer + minimum: 0 + accepted_share_difficulty: + $ref: "#/components/schemas/DecimalString" + complete: + type: boolean + description: > + False when `generated_at` falls before the bucket's end; the + bucket is still accumulating credit and its raw rate under-reads. + HashrateSeriesDualRateResponse: + type: object + additionalProperties: false + required: + - schema + - generated_at + - subject + - range + - bucket + - bucket_seconds + - unit + - rate_basis + - smoothing + - points + properties: + schema: + type: string + const: prism.dashboard.hashrate-series.v2 + generated_at: + $ref: "#/components/schemas/IsoTimestamp" + subject: + $ref: "#/components/schemas/HashrateSubject" + range: + $ref: "#/components/schemas/HashrateRange" + bucket: + $ref: "#/components/schemas/HashrateBucket" + bucket_seconds: + type: integer + enum: [300, 3600, 86400] + description: Duration of every bucket, and the divisor behind each raw rate. + unit: + type: string + const: ths + rate_basis: + type: string + const: accepted_share_difficulty + smoothing: + $ref: "#/components/schemas/HashrateSmoothing" + points: + type: array + items: + $ref: "#/components/schemas/DualRateHashratePoint" + BlockMarkerBlock: + type: object + additionalProperties: false + required: [height, hash, found_at] + properties: + height: + type: integer + minimum: 0 + hash: + $ref: "#/components/schemas/HexHash" + found_at: + $ref: "#/components/schemas/IsoTimestamp" + BlockMarkerPoint: + type: object + additionalProperties: false + required: [timestamp, block_count, blocks, truncated] + properties: + timestamp: + allOf: + - $ref: "#/components/schemas/IsoTimestamp" + description: > + Bucket start, on the same floor-aligned grid as the hashrate + series buckets for the identical range and bucket. + block_count: + type: integer + minimum: 1 + description: Every non-reversed block found in the bucket, listed or not. + blocks: + type: array + maxItems: 3 + items: + $ref: "#/components/schemas/BlockMarkerBlock" + description: > + At most the 3 most recent blocks in the bucket, ordered by + found_at descending with height breaking ties. + truncated: + type: boolean + description: True when block_count exceeds the 3 blocks listed. + BlockMarkersResponse: + type: object + additionalProperties: false + required: + - schema + - generated_at + - range + - bucket + - bucket_seconds + - total_blocks + - points + properties: + schema: + type: string + const: prism.dashboard.block-markers.v1 + generated_at: + $ref: "#/components/schemas/IsoTimestamp" + range: + $ref: "#/components/schemas/HashrateRange" + bucket: + $ref: "#/components/schemas/HashrateBucket" + bucket_seconds: + type: integer + enum: [300, 3600, 86400] + description: Duration of every bucket, matching the hashrate series. + total_blocks: + type: integer + minimum: 0 + description: All non-reversed found blocks inside the range. + points: + type: array + items: + $ref: "#/components/schemas/BlockMarkerPoint" + description: > + Only buckets containing found blocks, ascending by timestamp; + consumers synthesize gaps. WindowInfo: type: object additionalProperties: false @@ -833,7 +1159,12 @@ components: - participant_count properties: pool_hashrate_ths: - $ref: "#/components/schemas/DecimalString" + allOf: + - $ref: "#/components/schemas/DecimalString" + description: > + Pool credited hashrate over the fixed 3h leaderboard window, + estimated from accepted share difficulty rather than router + telemetry. pool_accepted_share_difficulty: $ref: "#/components/schemas/DecimalString" participant_count: @@ -860,7 +1191,12 @@ components: display_name: $ref: "#/components/schemas/NullableString" hashrate_ths_3h: - $ref: "#/components/schemas/DecimalString" + allOf: + - $ref: "#/components/schemas/DecimalString" + description: > + Recipient credited hashrate over the fixed 3h leaderboard window, + estimated from accepted share difficulty rather than router + telemetry. share_percent: oneOf: - $ref: "#/components/schemas/DecimalString" @@ -1092,6 +1428,105 @@ components: type: array items: $ref: "#/components/schemas/BlockRow" + ChainState: + type: string + enum: + - prepared + - confirmed + - inactive + - rejected + - reversed + description: > + The pool's ledger view of where a found block stands against the + active chain. `prepared` blocks are still being landed, `confirmed` + blocks are on the active chain, `inactive` blocks are temporarily off + it, `rejected` blocks never activated, and `reversed` blocks were + disconnected by a chain reorganization after the pool landed them. + ChainStateBlockRow: + type: object + additionalProperties: false + required: + - height + - hash + - found_at + - network_difficulty + - bits + - solver_recipient_id + - solver_worker_name + - solver_share_difficulty + - reward_window_weight + - coinbase_value_bits + - audit_bundle_sha256 + - payout_manifest_sha256 + - explorer_url + - chain_state + - disconnected_at + properties: + height: + type: integer + minimum: 0 + hash: + $ref: "#/components/schemas/HexHash" + found_at: + $ref: "#/components/schemas/IsoTimestamp" + network_difficulty: + $ref: "#/components/schemas/DecimalString" + bits: + $ref: "#/components/schemas/CompactBits" + solver_recipient_id: + type: string + solver_worker_name: + $ref: "#/components/schemas/NullableString" + solver_share_difficulty: + oneOf: + - $ref: "#/components/schemas/DecimalString" + - type: "null" + reward_window_weight: + oneOf: + - $ref: "#/components/schemas/DecimalString" + - type: "null" + coinbase_value_bits: + $ref: "#/components/schemas/Bits" + audit_bundle_sha256: + oneOf: + - $ref: "#/components/schemas/HexHash" + - type: "null" + payout_manifest_sha256: + oneOf: + - $ref: "#/components/schemas/HexHash" + - type: "null" + explorer_url: + oneOf: + - type: string + format: uri + - type: "null" + chain_state: + $ref: "#/components/schemas/ChainState" + disconnected_at: + allOf: + - $ref: "#/components/schemas/NullableTimestamp" + description: > + When a chain reorganization disconnected this block from the + active chain; `null` for blocks that have not been reversed. + BlocksChainStateResponse: + type: object + additionalProperties: false + required: [schema, generated_at, pagination, rows] + description: > + The `chain_state=all` / `chain_state=reversed` view of the block + history: the v1 rows plus reorg visibility fields. + properties: + schema: + type: string + const: prism.dashboard.blocks.v2 + generated_at: + $ref: "#/components/schemas/IsoTimestamp" + pagination: + $ref: "#/components/schemas/Pagination" + rows: + type: array + items: + $ref: "#/components/schemas/ChainStateBlockRow" SettlementStatus: type: string enum: @@ -1664,6 +2099,7 @@ components: - rate_limited - internal_error - upstream_unavailable + - read_timeout message: type: string request_id: diff --git a/docs/public-dashboard-api/README.md b/docs/public-dashboard-api/README.md index 4814b4adc..3d81361ef 100644 --- a/docs/public-dashboard-api/README.md +++ b/docs/public-dashboard-api/README.md @@ -27,18 +27,59 @@ The dashboard app must not query Postgres, qbit RPC, private command sockets, or internal audit endpoints directly. Its only stable data dependency should be the sanitized `/public/v1` API described by this contract. -In deployment, the `prism-coordinator` audit HTTP listener serves `/public/v1`. +In deployment, `/public/v1` is served by its own process — the +`prism-public-api` service (`python3 -m lab.prism.public_read_service`), on +`PRISM_PUBLIC_API_PORT` (default `3342`). It is no longer served by the +`prism-coordinator` audit HTTP listener, which now answers only `/audit/*`, +`/healthz`, `/metrics`, `/owed*`, and the operator miner/payout status routes; a +`/public/v1` request to the coordinator returns its ordinary +`{"error": "unknown endpoint"}` 404. + +The split exists because public read traffic scales with public interest rather +than with hashrate. Served in-process it shared the GIL that acknowledges shares +and lands blocks, and the primary Postgres connection the lease-holding writer +commits through. The extracted tier reads through bounded read slots only, never +acquires a writer lease, and depends on Postgres rather than on the coordinator, +so it keeps serving across coordinator restarts. + Operators can expose only that path from the pool service, or place a dashboard/web proxy in front of it. The ownership boundary stays the same: pool read models live here; dashboard rendering lives outside the pool process. +`prism-public-api` also serves its own `/healthz` and `/metrics` for that +process; those are operator surfaces and must not be exposed publicly. + +The HTTP server admits up to `PRISM_PUBLIC_HTTP_MAX_CONNECTIONS` connections +(default 64, range 1–1,024); it immediately closes excess connections without +queuing workers. `PRISM_PUBLIC_HTTP_TIMEOUT_SECONDS` (default 10, finite and +positive) bounds the complete request-line/header phase and socket I/O idle +time. Each response closes its connection, including requests asking for +HTTP/1.1 keep-alive. Size proxy upstream connections accordingly. Database and +public dispatch deadlines remain separate from this network timeout. + +Startup requires `PRISM_PUBLIC_STRATUM_URL` to contain a `stratum+tcp` or +`stratum+ssl` scheme, a valid hostname (or bracketed IPv6 address), and an +explicit port from 1 to 65,535. Credentials, paths, queries, fragments and +whitespace are refused before opening the database or HTTP listener. The same +validation applies to an explicit `PRISM_PUBLIC_STRATUM_HIGHDIFF_URL` when +`PRISM_STRATUM_HIGHDIFF_PORT` enables that advertised endpoint; an empty override +continues to derive its URL from the primary endpoint. ## Caching -Successful `GET /public/v1` responses are safe to cache briefly. The coordinator +Successful `GET /public/v1` responses are safe to cache briefly. The service emits conservative browser caching (`Cache-Control: public, max-age=0, must-revalidate`) plus shared-cache headers for CDNs such as Vercel. Dynamic dashboard read models default to a 5-second shared-cache TTL with 30 seconds of -`stale-while-revalidate`. The pool-wide aggregate read models — +`stale-while-revalidate` before applying the route limit. Both shared-cache TTL +and stale-while-revalidate are clamped to the route's +published staleness budget. For example, a default plain read uses a 5-second +TTL and at most 10 additional stale seconds within its 15-second budget. +Background refreshes share four worker slots across all response caches in the +process. When they are busy, eligible stale responses remain available within +their existing window; additional refresh work is not queued. A later request +can retry after capacity returns. + +The pool-wide aggregate read models — `GET /public/v1/pool-summary`, `GET /public/v1/hashrate-series`, and `GET /public/v1/miners/{recipient_id}/workers` — are expensive to recompute and default to a 30-second shared-cache TTL instead. @@ -61,14 +102,99 @@ Operators can tune the defaults with: - `PRISM_PUBLIC_CACHE_MAX_RESPONSE_BYTES` - `PRISM_PUBLIC_CACHE_DEBUG_HEADERS` -The coordinator also keeps a small in-process origin cache keyed by normalized -path and query string, and coalesces concurrent misses for the same key. Error -responses use `Cache-Control: no-store` and are not cached by that origin cache. +The v2 dual-rate series is roughly 1.5 times the v1 payload at the same point +count because it carries two decimal rate strings. Operators serving long 5m +series with `view=both` should size `PRISM_PUBLIC_CACHE_MAX_RESPONSE_BYTES` +accordingly or the origin cache will deliberately skip oversized responses. + +The public read service also keeps a small in-process origin cache keyed by +normalized path and query string, and coalesces concurrent misses for the same +key. The `stale-while-revalidate` window applies in-process too, not only at +CDNs: an entry that has expired inside the window is served immediately with +its honest `Age` while a single background refresh recomputes it. Concurrent +stale hits share that one refresh, and a failed refresh keeps the stale entry +servable until the window ends. The in-process window is clamped so a +stale-served `Age` never exceeds the route's staleness budget (below); past +the window the next request blocks and recomputes as before. Error responses +use `Cache-Control: no-store` and are not cached by that origin cache. Miner pages additionally share one briefly cached pool-wide reward-window aggregate (`PRISM_PUBLIC_REWARD_WINDOW_CACHE_SECONDS`, default 30 seconds, 0 disables), so requests for different miners reuse a single recursive reward-window scan instead of each re-running it. +Every origin computation that takes a ledger read slot — the immutable +artifact route included, on a cold request — runs under one per-request +deadline (`PRISM_PUBLIC_READ_STATEMENT_TIMEOUT_SECONDS`, default 20 seconds, +`0` disables) that counts down across every ledger read the route performs. +A read that exceeds the remaining budget is cancelled server-side, +frees its bounded read slot instead of running to completion for a client +that has long since hung up, and returns `503` with error code `read_timeout` +and `Cache-Control: no-store`. The refusal is never cached, so the next +request — or a stale-while-revalidate refresh — retries the origin. + +## Staleness + +Every `/public/v1` response states how old an answer that route is willing to +serve, and refuses rather than serve past it. A silently stale dashboard is +worse than an honest one. + +- `X-Prism-Staleness-Budget-Seconds` — the budget for this route, in seconds. + `/public/v1/artifacts/{sha256}` reports `unbounded`: its content is + content-addressed and immutable, so a body that hashes to the requested + sha256 is correct at any age and this route never refuses for staleness. +- `Age` — the observed age of the response actually served, as before. + +When the observed age exceeds the budget, the service returns **503** with +`Cache-Control: no-store` and an ordinary `prism.dashboard.error.v1` body whose +message names both the budget and the observed age. Clients should treat this +as "the data behind this route is too old to answer with", not as a new error +schema — the error code is `upstream_unavailable`, already in the documented +enum. + +Budgets are derived from the caches that sit under each route rather than +hand-picked: + +``` +budget = max(3 * (cache_ttl_seconds + underlying_cache_seconds), 15) +``` + +`cache_ttl_seconds` is the route's own shared-response TTL and +`underlying_cache_seconds` is any second cache stacked beneath it — today only +the pool reward-window aggregate (`PRISM_PUBLIC_REWARD_WINDOW_CACHE_SECONDS`, +default 30s) under `/public/v1/miners/{recipient_id}`. The factor of three and +the 15-second floor match the existing precedent for PRISM's cached `/metrics` +endpoint. At the documented defaults this yields: + +| Route | Budget | +| --- | --- | +| `/public/v1/blocks` | 15s | +| `/public/v1/block-markers` | 15s | +| `/public/v1/leaderboard` | 15s | +| `/public/v1/miners/{recipient_id}/earnings` | 15s | +| `/public/v1/miners/{recipient_id}/payouts` | 15s | +| `/public/v1/blocks/{block_hash}/settlement-artifacts` | 15s | +| `/public/v1/fanouts/pending` | 15s | +| `/public/v1/fanouts/{fanout_txid}` | 15s | +| `/public/v1/pool-summary` | 90s | +| `/public/v1/hashrate-series` | 90s | +| `/public/v1/miners/{recipient_id}/workers` | 90s | +| `/public/v1/miners/{recipient_id}` | 105s | +| `/public/v1/mining-configuration` | 900s | +| `/public/v1/artifacts/{sha256}` | unbounded | + +The budgets are constants derived from the documented cache defaults — the +`max_staleness_seconds` values in `lab/prism/endpoint_registry.py` — and there +is no environment knob that raises a budget. An operator who raises one of the +cache TTL knobs that do exist (`PRISM_PUBLIC_CACHE_TTL_SECONDS`, +`PRISM_PUBLIC_AGGREGATE_CACHE_TTL_SECONDS`, +`PRISM_PUBLIC_CONFIG_CACHE_TTL_SECONDS`, or +`PRISM_PUBLIC_REWARD_WINDOW_CACHE_SECONDS`) above its route's budget will see +that route begin refusing with 503 rather than quietly serving older data. +Raising a TTL past its budget therefore also requires changing the registry +constant in the same change; otherwise leave the defaults alone. The +`PRISM_PUBLIC_ARTIFACT_CACHE_*` TTLs are exempt: the artifact route is +content-addressed and never refuses for staleness. + ## Conventions - Base path: `/public/v1`. @@ -84,6 +210,155 @@ reward-window scan instead of each re-running it. - Optional fields are present as `null` when unavailable, so dashboard layout can remain stable. +## Hashrate Series + +`GET /public/v1/hashrate-series` reports **credited hashrate**: each bucket's +rate is derived from the difficulty of the accepted shares the pool credited to +the subject in that bucket, not from anything a miner or router reports about +itself. Neither series below is physical router telemetry. While the share +target the pool assigned and the router's own share filtering disagree, the +credited rate and the rate the router shows will differ. + +Without `view`, the endpoint returns `prism.dashboard.hashrate-series.v1` +unchanged: `hashrate_ths` is the credited rate smoothed over a trailing window +(`PRISM_PUBLIC_HASHRATE_SMOOTHING_SECONDS`, default 1800 seconds, `0` +disables), while `accepted_share_count` and `accepted_share_difficulty` stay +raw per bucket. + +`view=both` returns `prism.dashboard.hashrate-series.v2`. It keeps +`generated_at`, `subject`, `range`, `bucket`, `unit`, and the bucket cadence, +and adds: + +- `bucket_seconds` — the bucket duration every raw rate is computed over. +- `rate_basis` — always `accepted_share_difficulty`. +- `smoothing` — `{ "method": "trailing" | "none", "window_seconds": n }`. + `trailing` averages credited difficulty over the trailing `window_seconds` + ending at each bucket, with buckets missing from the series counting as + zero. When smoothing is disabled or configured shorter than two buckets the + method is `none`, `window_seconds` equals `bucket_seconds`, and the smoothed + rate equals the raw rate. + +Each v2 point carries `timestamp`, `raw_hashrate_ths`, `smoothed_hashrate_ths`, +`accepted_share_count`, `accepted_share_difficulty`, and `complete`: + +- `raw_hashrate_ths` is the bucket's credited difficulty divided by the full + bucket duration, so a bucket that is still accumulating shares under-reads. + It is a credited-work estimate, not physical router telemetry: with sparse + high-difficulty shares, assigning a whole share to its arrival bucket can + temporarily over-read the miner's delivered rate (the 539 TH/s raw versus + 118 TH/s trailing incident is an example). +- `smoothed_hashrate_ths` is the same trailing-window estimate v1 serves as + `hashrate_ths`. +- `complete` is `false` when `generated_at` falls before the bucket's end. + Render such a point provisionally; it changes on the next request. + +Buckets with no credited shares are omitted from both views; consumers +synthesize gaps rather than expecting zero-valued points. Both views fetch one +smoothing window of pre-range history so the first in-range trailing windows +average over real data, then trim those context buckets from the response. Any +other non-empty `view` value is rejected with `400 bad_request`. `view` is part +of the origin cache key, so the two views never share a cached response. + +The series is served from incremental per-bucket rollups the coordinator +maintains in the ledger database, merged with a live tail of shares the +rollups have not folded in yet, so every range — `all` included — costs +buckets rather than raw shares. This is a serving-side optimization only: the +response schemas, values, and both views are unchanged from the raw +aggregation they replace. +`fixtures/hashrate-series.json` mocks the v1 response and +`fixtures/hashrate-series-dual-rate.json` mocks the v2 response. + +`bucket=auto` resolves to 1h for 1w/1m and 1d for 6m/all. With the default 30m +smoothing window those coarse buckets report `smoothing.method=none`, so raw +and smoothed are identical. Dashboards that need to expose short bursts and +their 30m trailing context should explicitly request `bucket=5m`. + +Each range admits a fixed set of buckets: `1w` allows `5m`, `1h`, and `1d`; +`1m` allows `1h` and `1d`; `6m` and `all` allow only `1d`. `bucket=auto` +always resolves inside the allowed set. A combination outside that vocabulary +— an unbounded-cost request such as `range=all&bucket=5m`, whose response +would outgrow the cache size cap and re-scan the ledger on every request — is +rejected with `400 bad_request` naming the allowed buckets. + +## Blocks and Reorg Visibility + +Chain reorganizations happen, and hiding them entirely made the pool's block +history look cleaner than the chain it mines. `GET /public/v1/blocks` therefore +takes a `chain_state` filter: + +- Omitted or `chain_state=active` — exactly the pre-filter behavior and the + `prism.dashboard.blocks.v1` schema tag: every recorded pool block except + those a reorganization reversed. Existing consumers see a byte-compatible + response. +- `chain_state=all` — every recorded pool block, including reversed ones. +- `chain_state=reversed` — only blocks the pool once landed that a chain + reorganization later disconnected. + +Both non-default filters return `prism.dashboard.blocks.v2`, whose rows carry +the v1 fields plus `chain_state` (`prepared`, `confirmed`, `inactive`, +`rejected`, or `reversed`) and `disconnected_at`, the reorg disconnect time +that is `null` for every non-reversed block. Pagination `total_count` and +`total_pages` always describe the filtered set. Any other `chain_state` value +is rejected with `400 bad_request`. The filter is part of the origin cache +key, so the three views never share a cache entry, and the route keeps the +5-second dynamic-read TTL class. `fixtures/blocks.json` mocks the default +response and `fixtures/blocks-chain-states.json` mocks the `chain_state=all` +response with a reversed block. + +`GET /public/v1/pool-summary` surfaces the same information in aggregate: +`pool.blocks_reversed_total` and `pool.blocks_inactive_total` count the +reversed and currently-inactive pool blocks. `blocks_found_total` is +unchanged and keeps excluding reversed blocks, so it is not the sum of the +per-state counters. These are **additive fields on +`prism.dashboard.pool-summary.v1`** (added in 2.x): pool-summary takes no +request parameter, so the repo's param-gated versioning precedent does not +apply cleanly; the schema tag stays `v1` and validators that pin the vendored +contract pick the fields up when they bump their pin. + +## Network Hashrate + +`network.hashrate_ths` in `GET /public/v1/pool-summary` is the node's own +estimate of the network hashrate in TH/s — qbit's `getnetworkhashps` over its +default 120-block window, restricted to the permissionless lane the pool's +shares are credited against. Divide a pool `hashrate_ths` window (`h1`/`h3`) +by it to render "pool is X% of network"; no precomputed percentage is served. +The ratio is approximate by construction: pool hashrate is credited work from +accepted shares while the network figure is a chain-derived estimate, so the +two can disagree over short windows. + +Like the reorg counters, this is an additive field on +`prism.dashboard.pool-summary.v1` (added in 2.x). It is nullable and +non-fatal: if the node cannot answer, the response carries +`"hashrate_ths": null` and never turns the failure into a 503. Pool-summary +still returns 503 only when no valid compact bits are available, exactly as +before. + +## Block Markers + +`GET /public/v1/block-markers` serves the found-block markers the dashboard +draws over the hashrate chart, pre-bucketed server-side. Paging +`GET /public/v1/blocks` cannot place markers on long timeframes — at hundreds +of found blocks a day the 1m/6m/all views would need hundreds of pages per +chart render — so this endpoint aggregates the pool's block history into the +chart's own buckets in one response. + +`range` and `bucket` use the hashrate-series vocabulary, and `bucket=auto` +resolves identically (1h for 1w/1m, 1d for 6m/all). Bucket epochs use the +identical floor arithmetic the hashrate buckets use, and bounded ranges keep +only fully covered buckets from the same range anchor, so markers land in the +same buckets as the hashrate points on every timeframe. Finer buckets than a +range's chart renders are rejected with `400 bad_request`: 1w allows 5m/1h/1d, +1m allows 1h/1d, 6m and all allow 1d only. + +The response (`prism.dashboard.block-markers.v1`) reports `total_blocks` — all +non-reversed found blocks in range — and `points` containing only buckets with +at least one found block, ascending by timestamp. Each point carries the +bucket's full `block_count`, at most its 3 most recent blocks (`found_at` +descending, height breaking ties) with `height`, `hash`, and `found_at`, and +`truncated: true` when more blocks exist than are listed. Reversed blocks are +excluded throughout, matching `/public/v1/blocks`. +`fixtures/block-markers.json` mocks the response. + ## Reward Leaderboard `GET /public/v1/leaderboard?window=reward` returns @@ -140,6 +415,52 @@ as PRISM audit bundles and CTV manifest JSON, are linked by URL and SHA-256 so they can be mirrored or downloaded without making dashboard clients depend on internal audit routes. +`GET /public/v1/artifacts/{sha256}` is content-addressed: for audit bundles, +CTV fanout manifests, and manifest sets the response body is the exact canonical +byte sequence the artifact hash was computed over, so +`sha256(response body) == {sha256}` verifies the download with no +re-serialization step. Audit bundles written before canonical-byte persistence +was introduced retain the legacy reconstructed response until the verified +backfill publishes their canonical artifact. + +Operators can verify the historical range while the coordinator is live: + +```sh +python3 -m lab.prism.backfill_audit_bundle_canonical --dry-run +``` + +Before publishing, stop the coordinator and confirm it no longer owns the +PostgreSQL writer lease. Publish mode requires exclusive lease ownership and +fails instead of waiting behind a live same-identity coordinator. Schema repair +is unrelated to this filesystem-only rollout, so the recommended publish +command disables it explicitly: + +```sh +python3 -m lab.prism.backfill_audit_bundle_canonical --no-init-schema +``` + +The command releases its exact writer lease and closes its database and +artifact-store resources on success or failure, so the coordinator can restart +without waiting for the backfill lease TTL. It pages in stable block-hash order +and reports `last_checkpoint`; pass that value to `--start-after` to resume a +bounded rollout. The checkpoint advances over failed rows. Their block hashes, +advertised digests, failure kinds, and reasons remain in the JSON +`failed_rows` array, but resuming after the checkpoint will skip them. After +correcting a failure, retry from before its block hash or rerun from the +beginning; already published rows are idempotent no-ops. + +On a read-replica deployment, the ledger row remains the visibility authority. +If the row has replayed but the canonical file is absent, the service uses the +legacy reconstructed response with `Cache-Control: no-store` and +`X-Prism-Artifact-Canonical-State: missing`; it is never admitted to the origin +cache or an immutable CDN cache. A corrupt canonical file fails closed rather +than being disguised as legacy history. If shared storage receives the file +before the replica replays its row, the artifact returns a non-cacheable `404` +until replay catches up; the file alone never exposes an uncommitted artifact. +This immutable route remains exempt from the ordinary freshness refusal. +Replica readiness is bounded by the existing WAL-receiver heartbeat contract, +not by replay-lag position. + Direct-coinbase blocks return the same settlement-artifacts wrapper with `settlement_mode: direct_coinbase` and `fanouts: []`. A `404` means no public settlement artifact index is known for that block, not an implied direct diff --git a/docs/public-dashboard-api/fixtures/block-markers.json b/docs/public-dashboard-api/fixtures/block-markers.json new file mode 100644 index 000000000..675ab6feb --- /dev/null +++ b/docs/public-dashboard-api/fixtures/block-markers.json @@ -0,0 +1,73 @@ +{ + "schema": "prism.dashboard.block-markers.v1", + "generated_at": "2026-06-26T20:45:00Z", + "range": "1m", + "bucket": "1h", + "bucket_seconds": 3600, + "total_blocks": 8, + "points": [ + { + "timestamp": "2026-06-26T14:00:00Z", + "block_count": 2, + "blocks": [ + { + "height": 123392, + "hash": "0000000000000000000abc1230000000000000000000000000000000000000cc", + "found_at": "2026-06-26T14:40:12Z" + }, + { + "height": 123391, + "hash": "0000000000000000000abc1230000000000000000000000000000000000000cd", + "found_at": "2026-06-26T14:07:55Z" + } + ], + "truncated": false + }, + { + "timestamp": "2026-06-26T15:00:00Z", + "block_count": 1, + "blocks": [ + { + "height": 123401, + "hash": "0000000000000000000abc1230000000000000000000000000000000000000bb", + "found_at": "2026-06-26T15:20:00Z" + } + ], + "truncated": false + }, + { + "timestamp": "2026-06-26T18:00:00Z", + "block_count": 4, + "blocks": [ + { + "height": 123447, + "hash": "0000000000000000000abc1230000000000000000000000000000000000000de", + "found_at": "2026-06-26T18:55:01Z" + }, + { + "height": 123446, + "hash": "0000000000000000000abc1230000000000000000000000000000000000000df", + "found_at": "2026-06-26T18:31:47Z" + }, + { + "height": 123445, + "hash": "0000000000000000000abc1230000000000000000000000000000000000000e0", + "found_at": "2026-06-26T18:12:09Z" + } + ], + "truncated": true + }, + { + "timestamp": "2026-06-26T19:00:00Z", + "block_count": 1, + "blocks": [ + { + "height": 123450, + "hash": "0000000000000000000abc1230000000000000000000000000000000000000aa", + "found_at": "2026-06-26T19:55:00Z" + } + ], + "truncated": false + } + ] +} diff --git a/docs/public-dashboard-api/fixtures/blocks-chain-states.json b/docs/public-dashboard-api/fixtures/blocks-chain-states.json new file mode 100644 index 000000000..97b85ffad --- /dev/null +++ b/docs/public-dashboard-api/fixtures/blocks-chain-states.json @@ -0,0 +1,63 @@ +{ + "schema": "prism.dashboard.blocks.v2", + "generated_at": "2026-06-26T20:45:00Z", + "pagination": { + "page": 1, + "limit": 15, + "total_count": 130, + "total_pages": 9 + }, + "rows": [ + { + "height": 123450, + "hash": "0000000000000000000abc1230000000000000000000000000000000000000aa", + "found_at": "2026-06-26T19:55:00Z", + "network_difficulty": "2147516160496101", + "bits": "1d00ffff", + "solver_recipient_id": "qbrt1prismminer000000000000000000000000000000000000001", + "solver_worker_name": "rig-a", + "solver_share_difficulty": "2200000000000000", + "reward_window_weight": "17180129283968808", + "coinbase_value_bits": 250000000, + "audit_bundle_sha256": "1111111111111111111111111111111111111111111111111111111111111111", + "payout_manifest_sha256": "2222222222222222222222222222222222222222222222222222222222222222", + "explorer_url": "https://explorer.example/block/0000000000000000000abc1230000000000000000000000000000000000000aa", + "chain_state": "confirmed", + "disconnected_at": null + }, + { + "height": 123430, + "hash": "0000000000000000000abc1230000000000000000000000000000000000000cc", + "found_at": "2026-06-26T16:58:00Z", + "network_difficulty": "2147516160496101", + "bits": "1d00ffff", + "solver_recipient_id": "qbrt1prismminer000000000000000000000000000000000000003", + "solver_worker_name": "rig-b", + "solver_share_difficulty": "2150000000000000", + "reward_window_weight": "17180129283968808", + "coinbase_value_bits": 250000000, + "audit_bundle_sha256": "3333333333333333333333333333333333333333333333333333333333333333", + "payout_manifest_sha256": null, + "explorer_url": null, + "chain_state": "reversed", + "disconnected_at": "2026-06-26T17:05:00Z" + }, + { + "height": 123401, + "hash": "0000000000000000000abc1230000000000000000000000000000000000000bb", + "found_at": "2026-06-26T15:20:00Z", + "network_difficulty": "2147516160496101", + "bits": "1d00ffff", + "solver_recipient_id": "qbrt1prismminer000000000000000000000000000000000000002", + "solver_worker_name": null, + "solver_share_difficulty": null, + "reward_window_weight": "17180129283968808", + "coinbase_value_bits": 250000000, + "audit_bundle_sha256": null, + "payout_manifest_sha256": null, + "explorer_url": null, + "chain_state": "confirmed", + "disconnected_at": null + } + ] +} diff --git a/docs/public-dashboard-api/fixtures/hashrate-series-dual-rate.json b/docs/public-dashboard-api/fixtures/hashrate-series-dual-rate.json new file mode 100644 index 000000000..8d09a59eb --- /dev/null +++ b/docs/public-dashboard-api/fixtures/hashrate-series-dual-rate.json @@ -0,0 +1,75 @@ +{ + "schema": "prism.dashboard.hashrate-series.v2", + "generated_at": "2026-06-26T20:45:00Z", + "subject": { + "type": "pool", + "id": null + }, + "range": "1m", + "bucket": "5m", + "unit": "ths", + "bucket_seconds": 300, + "rate_basis": "accepted_share_difficulty", + "smoothing": { + "method": "trailing", + "window_seconds": 1800 + }, + "points": [ + { + "timestamp": "2026-06-26T19:50:00Z", + "raw_hashrate_ths": "600.00", + "smoothed_hashrate_ths": "100.00", + "accepted_share_count": 184, + "accepted_share_difficulty": "41700000", + "complete": true + }, + { + "timestamp": "2026-06-26T19:55:00Z", + "raw_hashrate_ths": "600.00", + "smoothed_hashrate_ths": "200.00", + "accepted_share_count": 191, + "accepted_share_difficulty": "42800000", + "complete": true + }, + { + "timestamp": "2026-06-26T20:25:00Z", + "raw_hashrate_ths": "169.02", + "smoothed_hashrate_ths": "28.17", + "accepted_share_count": 52, + "accepted_share_difficulty": "11800000", + "complete": true + }, + { + "timestamp": "2026-06-26T20:30:00Z", + "raw_hashrate_ths": "539.40", + "smoothed_hashrate_ths": "118.07", + "accepted_share_count": 167, + "accepted_share_difficulty": "37700000", + "complete": true + }, + { + "timestamp": "2026-06-26T20:35:00Z", + "raw_hashrate_ths": "604.08", + "smoothed_hashrate_ths": "218.75", + "accepted_share_count": 188, + "accepted_share_difficulty": "42200000", + "complete": true + }, + { + "timestamp": "2026-06-26T20:40:00Z", + "raw_hashrate_ths": "618.36", + "smoothed_hashrate_ths": "321.81", + "accepted_share_count": 193, + "accepted_share_difficulty": "43200000", + "complete": true + }, + { + "timestamp": "2026-06-26T20:45:00Z", + "raw_hashrate_ths": "199.98", + "smoothed_hashrate_ths": "355.14", + "accepted_share_count": 61, + "accepted_share_difficulty": "14600000", + "complete": false + } + ] +} diff --git a/docs/public-dashboard-api/fixtures/pool-summary.json b/docs/public-dashboard-api/fixtures/pool-summary.json index af6816ebd..a9239d320 100644 --- a/docs/public-dashboard-api/fixtures/pool-summary.json +++ b/docs/public-dashboard-api/fixtures/pool-summary.json @@ -7,6 +7,7 @@ "tip_hash": "0000000000000000000abc123000000000000000000000000000000000000001", "bits": "181b32da", "network_difficulty": "86812541144089706339356132", + "hashrate_ths": "7716669.17", "initial_block_download": false, "peers": 8 }, @@ -20,6 +21,8 @@ "participants_3h": 42, "blocks_found_total": 128, "prism_blocks_total": 128, + "blocks_reversed_total": 2, + "blocks_inactive_total": 1, "total_mined_bits": 32000000000, "expected_time_to_block_seconds": 75, "latest_block": { diff --git a/docs/releasing.md b/docs/releasing.md new file mode 100644 index 000000000..f86dc51f4 --- /dev/null +++ b/docs/releasing.md @@ -0,0 +1,66 @@ +# Publishing a release + +`main` is the release promotion branch; `2.x.x` is the development branch for +the 2.x series. A version is published only when its tag and GitHub release +exist. A version bump or merged preparation PR alone does not publish it. + +## Prepare one release candidate + +Keep one pending release version across `VERSION`, the `qbit-prism` and +`qbit-pool-builder` package versions in their `Cargo.toml` files and `Cargo.lock`, +and `doc/release-notes-.md`. Ordinary development PRs contribute to +that pending release's notes; they do not each create a patch release. +Choose the next version after reconciling the last published release and +any identifiers already distributed to operators. + +For the first 2.x release, PR #246 is the single **2.0.0** promotion into +`main`. It includes the changes temporarily labelled 2.0.1 and 2.0.2 on +`2.x.x`; neither identifier was tagged or published as a GitHub release. +Their changes and migration requirements are consolidated in +[the 2.0.0 notes](../doc/release-notes-2.0.0.md). Do not create retrospective +2.0.1 or 2.0.2 tags for those preparation commits. + +Merge the intended development cutoff into the promotion branch and retain +fixes already made during promotion review. Check package versions together, +run Docker lint and the relevant test gates, and review upgrade and rollback +instructions. Leave the release date out while publication is unscheduled; +set the actual release date in a reviewed commit before the final merge. + +## Publish the reviewed main commit + +After the promotion PR passes required checks and is merged, identify the +exact resulting commit on `main`. A squash or rebase merge produces a different +commit from the promotion branch head. Verify the resulting commit's checks, +version metadata and dated release notes before tagging it. + +For 2.0.0, after confirming that `v2.0.0` does not already exist, use the +reviewed main commit in place of ``: + +```sh +git fetch origin main --tags +git tag -s v2.0.0 -m 'qbit-mining-bootstrap 2.0.0' +git verify-tag v2.0.0 +git push origin refs/tags/v2.0.0 +git show v2.0.0:doc/release-notes-2.0.0.md > /tmp/qbit-release-2.0.0-notes.md +gh release create v2.0.0 --verify-tag --title 'qbit-mining-bootstrap 2.0.0' \ + --notes-file /tmp/qbit-release-2.0.0-notes.md +``` + +Run publication commands only when authorized to publish. If a tag or release +already exists, inspect its target and contents instead of replacing it. +Use notes from the tagged tree, then verify the GitHub release points to that +same tag. Keep the upstream qbit binary pin separate from this repository's +release version. + +## Reconcile the development branch + +After promotion, open a synchronization PR from a branch that merges the +released `main` commit into the latest `2.x.x`. Preserve subsequent development +and resolve metadata and notes deliberately; do not reset or force-push the +shared branch. Run its checks before merging. + +For 2.0.0 this step is required: PR #246 contains promotion fixes and the +consolidated metadata that were not yet on `2.x.x`. Until this synchronization +lands, the old 2.0.2 metadata on that branch is not a published release. +Bring those fixes and the 2.0.0 release record back before selecting the next +version. Start the next pending notes separately and retain published notes. diff --git a/docs/router-integration-notes.md b/docs/router-integration-notes.md index 0147aaab2..12685dd3a 100644 --- a/docs/router-integration-notes.md +++ b/docs/router-integration-notes.md @@ -15,11 +15,14 @@ aggregator in front of the qbit permissionless ckpool path. ## Version Rolling -The bootstrap ckpool image starts with `CKPOOL_VERSION_MASK_MODE=dynamic`. At -startup it asks qbitd for `getblocktemplate` and uses -`versionrollingmask` when the connected node exposes it. Older qbitd builds -fall back to the configured `CKPOOL_VERSION_MASK`; the public sample env uses -`1fffe000` to match current qbitd permissionless templates. +The bootstrap ckpool image starts with `CKPOOL_VERSION_MASK_MODE=dynamic`. Once +its startup readiness gate passes, it asks qbitd for `getblocktemplate` and uses +the advertised `versionrollingmask`. Older qbitd builds that answer without the +field fall back to the configured `CKPOOL_VERSION_MASK`; the public sample env +uses `1fffe000` to match current qbitd permissionless templates. A node that +cannot serve a template at all is a startup failure, not a fallback, so a running +ckpool in dynamic mode has confirmed its mask against a live node. Explicit +static mode uses the configured mask without that probe. Routers that negotiate BIP310 should only pass miner-controlled version bits inside the mask granted by ckpool. If a miner requests a mask, the effective diff --git a/lab/auxpow/Dockerfile b/lab/auxpow/Dockerfile index 5368b64b0..0ff3eeef2 100644 --- a/lab/auxpow/Dockerfile +++ b/lab/auxpow/Dockerfile @@ -1,6 +1,6 @@ # syntax=docker/dockerfile:1.7 -FROM python:3.12-slim +FROM python:3.14-slim-trixie WORKDIR /app diff --git a/lab/auxpow/auxpow_coordinator.py b/lab/auxpow/auxpow_coordinator.py index 39a8e7b2b..9a770ca18 100644 --- a/lab/auxpow/auxpow_coordinator.py +++ b/lab/auxpow/auxpow_coordinator.py @@ -174,6 +174,7 @@ def load_vardiff_config() -> vardiff.VardiffConfig: target_shares_per_second = env_optional_decimal("AUXPOW_STRATUM_VARDIFF_TARGET_SHARES_PER_SECOND") if target_shares_per_second is not None: target_interval = Decimal(1) / target_shares_per_second + max_step_factor = env_decimal("AUXPOW_STRATUM_VARDIFF_MAX_STEP_FACTOR", "4") try: return vardiff.VardiffConfig( enabled=env_bool("AUXPOW_STRATUM_VARDIFF_ENABLED", True), @@ -181,7 +182,7 @@ def load_vardiff_config() -> vardiff.VardiffConfig: min_difficulty=env_decimal("AUXPOW_STRATUM_VARDIFF_MIN_DIFF", "1024"), max_difficulty=env_decimal("AUXPOW_STRATUM_VARDIFF_MAX_DIFF", "4294967296"), retarget_interval_seconds=env_decimal("AUXPOW_STRATUM_VARDIFF_RETARGET_SECONDS", "120"), - max_step_factor=env_decimal("AUXPOW_STRATUM_VARDIFF_MAX_STEP_FACTOR", "4"), + max_step_factor=max_step_factor, startup_difficulty=env_decimal_default_on_empty( "AUXPOW_STRATUM_VARDIFF_STARTUP_DIFF", "8192", @@ -189,6 +190,11 @@ def load_vardiff_config() -> vardiff.VardiffConfig: max_step_down_factor=env_decimal("AUXPOW_STRATUM_VARDIFF_MAX_STEP_DOWN_FACTOR", "2"), ewma_alpha=env_decimal("AUXPOW_STRATUM_VARDIFF_EWMA_ALPHA", "0.4"), retarget_tolerance=env_nonnegative_decimal("AUXPOW_STRATUM_VARDIFF_RETARGET_TOLERANCE", "0.25"), + # Fast initial convergence is a PRISM policy. Keep the shared + # math type's cap aligned with AuxPoW's ordinary cap so legacy + # values above PRISM's default of 64 remain valid. + initial_convergence_enabled=False, + initial_max_step_factor=max_step_factor, ) except ValueError as exc: raise SystemExit(f"AUXPOW_STRATUM_VARDIFF_* configuration is invalid: {exc}") from exc diff --git a/lab/auxpow/vardiff.py b/lab/auxpow/vardiff.py index ee1ec4f89..4d1178c39 100644 --- a/lab/auxpow/vardiff.py +++ b/lab/auxpow/vardiff.py @@ -18,6 +18,28 @@ class VardiffConfig: max_step_down_factor: Decimal = Decimal("2") ewma_alpha: Decimal = Decimal("0.4") retarget_tolerance: Decimal = Decimal("0.25") + # Fast-arrival initial convergence. A session whose real accepted-share + # rate sits far above its lane start difficulty -- a rental router that + # filters submissions at its own, much higher difficulty, so the pool + # only ever sees the survivors -- would otherwise need several ordinary + # max_step_factor retargets to reach the difficulty it is really working + # at. Because credit is target-based, every retarget interval spent + # climbing is credit the miner never receives. These bound ONE early, + # high-confidence move per connection: initial_min_accepted_shares of + # evidence over at least initial_min_elapsed_seconds, showing an observed + # difficulty at least initial_min_step_factor above the current one, + # buys a single step capped at initial_max_step_factor instead of + # max_step_factor. Set initial_convergence_enabled=False to restore the + # pre-#132 behavior exactly. + initial_convergence_enabled: bool = True + initial_max_step_factor: Decimal = Decimal("64") + initial_min_accepted_shares: int = 8 + initial_min_step_factor: Decimal = Decimal("4") + initial_min_elapsed_seconds: Decimal = Decimal("1") + # Observation-only difficulty at which a session counts as having reached + # rental-router scale; matches the high-diff listener's default floor. + # Metrics only -- it never gates or clamps a retarget. + high_diff_arrival_threshold: Decimal = Decimal("500000") def __post_init__(self) -> None: positive_fields = { @@ -33,6 +55,14 @@ def __post_init__(self) -> None: for name, value in positive_fields.items(): if not value.is_finite() or value <= 0: raise ValueError(f"{name} must be a positive finite decimal") + for name, value in { + "initial_max_step_factor": self.initial_max_step_factor, + "initial_min_step_factor": self.initial_min_step_factor, + "initial_min_elapsed_seconds": self.initial_min_elapsed_seconds, + "high_diff_arrival_threshold": self.high_diff_arrival_threshold, + }.items(): + if not value.is_finite() or value <= 0: + raise ValueError(f"{name} must be a positive finite decimal") if self.min_difficulty > self.max_difficulty: raise ValueError("min_difficulty must be <= max_difficulty") if self.max_step_factor < 1: @@ -43,6 +73,15 @@ def __post_init__(self) -> None: raise ValueError("ewma_alpha must be <= 1") if not self.retarget_tolerance.is_finite() or self.retarget_tolerance < 0: raise ValueError("retarget_tolerance must be a non-negative finite decimal") + # The initial cap is a relaxation of the ordinary bound, never a + # tightening of it: a configuration that inverts them would silently + # make the "fast" path slower than the one it replaces. + if self.initial_max_step_factor < self.max_step_factor: + raise ValueError("initial_max_step_factor must be >= max_step_factor") + if self.initial_min_step_factor < 1: + raise ValueError("initial_min_step_factor must be >= 1") + if self.initial_min_accepted_shares < 1: + raise ValueError("initial_min_accepted_shares must be >= 1") def clamp(value: Decimal, minimum: Decimal, maximum: Decimal) -> Decimal: @@ -96,6 +135,55 @@ def should_retarget(current_difficulty: Decimal, next_difficulty: Decimal, toler return ratio >= tolerance +def initial_convergence_ready( + *, + current_difficulty: Decimal, + accepted_shares: int, + accepted_difficulty: Decimal, + elapsed_seconds: Decimal, + config: VardiffConfig, +) -> bool: + """Whether this window is decisive enough for one early initial retarget. + + Three independent guards, all of which must hold: + + * at least ``initial_min_accepted_shares`` of evidence when the caller + evaluates this one-shot gate for the first time the share-count and + elapsed-time floors are both met, so a single lucky share cannot move + the session and repeatedly testing the same window cannot compound the + false-positive probability; + * ``initial_min_elapsed_seconds`` of window, so a burst delivered in a + near-zero interval cannot divide a real work total by an + almost-zero elapsed time and explode the observed rate; + * an observed difficulty at least ``initial_min_step_factor`` above the + current one, so only evidence far outside the Poisson noise of a + small sample qualifies. At the 8-share default a miner exactly on + target has a ~35% relative spread on its observed rate; requiring 4x + keeps an on-cadence miner from being pushed up merely because this + path exists, and makes the early move upward-only. + + The comparison uses this window's raw observed difficulty rather than + the caller's smoothed estimate: the question here is what THIS window + proves, and the smoothing that follows is free to temper the answer. + """ + if not config.enabled or not config.initial_convergence_enabled: + return False + if accepted_shares < config.initial_min_accepted_shares: + return False + if not current_difficulty.is_finite() or current_difficulty <= 0: + return False + if not elapsed_seconds.is_finite() or elapsed_seconds < config.initial_min_elapsed_seconds: + return False + observed = observed_difficulty( + accepted_difficulty=accepted_difficulty, + elapsed_seconds=elapsed_seconds, + target_share_interval_seconds=config.target_share_interval_seconds, + ) + if observed is None: + return False + return observed >= current_difficulty * config.initial_min_step_factor + + def calculate_next_difficulty( *, current_difficulty: Decimal, @@ -104,12 +192,19 @@ def calculate_next_difficulty( config: VardiffConfig, accepted_difficulty: Decimal | None = None, difficulty_estimate: Decimal | None = None, + initial_convergence: bool = False, ) -> Decimal: """Return the next difficulty for an observed share window. Difficulty moves toward a share-weighted observed difficulty estimate, then clamps to the configured per-retarget step and absolute bounds. A zero-share window steps down by the configured down factor. + + ``initial_convergence`` raises only the UPWARD step bound, to + ``initial_max_step_factor``, for a session still in its one qualifying + fast-arrival phase (see :func:`initial_convergence_ready`). The + step-down bound, the EWMA estimate, and the absolute lane min/max are + untouched, so a larger initial step can still never leave the lane. """ if not current_difficulty.is_finite() or current_difficulty <= 0: @@ -139,5 +234,10 @@ def calculate_next_difficulty( share_ratio = target_difficulty / current_difficulty min_step = Decimal(1) / config.max_step_down_factor - step = clamp(share_ratio, min_step, config.max_step_factor) + max_step = ( + config.initial_max_step_factor + if initial_convergence and config.initial_convergence_enabled + else config.max_step_factor + ) + step = clamp(share_ratio, min_step, max_step) return clamp(current_difficulty * step, config.min_difficulty, config.max_difficulty) diff --git a/lab/miner-sim/Dockerfile b/lab/miner-sim/Dockerfile index 5d2e2942b..38ce40ba7 100644 --- a/lab/miner-sim/Dockerfile +++ b/lab/miner-sim/Dockerfile @@ -1,4 +1,4 @@ -FROM python:3.12-slim +FROM python:3.14-slim-trixie WORKDIR /app diff --git a/lab/prism/Dockerfile b/lab/prism/Dockerfile index 6bffe500e..63a63aec9 100644 --- a/lab/prism/Dockerfile +++ b/lab/prism/Dockerfile @@ -1,6 +1,6 @@ # syntax=docker/dockerfile:1.7 -FROM python:3.12-slim +FROM python:3.14-slim-trixie WORKDIR /app @@ -30,6 +30,31 @@ RUN cargo build --locked --release -p qbit-prism --bins ENV PRISM_TOOL_BIN_DIR=/app/target/release +# Issue #226: glibc malloc arena policy, set deliberately rather than left to +# glibc. The mainnet coordinator runs about 64 threads and its memory map is +# hundreds of 4-64 MiB anonymous regions -- the per-thread arena signature -- +# on a host where glibc's default arena limit is 8 x cores = 256, so every +# thread that ever mallocs owns an arena and that arena's retained free lists. +# Two arenas is the default because it is safe before the soak decides +# between 2 and 4: under the GIL at most one Python thread allocates at a +# time, so arena-lock contention can only come from GIL-released native code +# (libpq I/O on a handful of pooled connections), and the audit-builder +# daemon this process spawns inherits the setting but is single-threaded. The +# failure mode of too few arenas is measurable lock wait on /metrics share-ack +# latency; the failure mode of too many is the 17.7 GB resident set. Override +# at build time with --build-arg MALLOC_ARENA_MAX=4 or at run time with +# -e MALLOC_ARENA_MAX=4; docs/mainnet-deployment.md records the decision and +# docs/prism-capacity-readiness.md the soak that revisits it. +# +# PYTHONMALLOC is deliberately unset: pymalloc keeps objects of 512 bytes and +# under out of glibc entirely, and routing them through glibc would enlarge +# the very arenas this bounds. No other MALLOC_* variable is set here. Never +# set one to an empty value: glibc parses "" as 0, and MALLOC_MMAP_THRESHOLD_=0 +# would send every allocation through mmap. tests/test_prism_heap_census.py +# pins exactly this allocator environment. +ARG MALLOC_ARENA_MAX=2 +ENV MALLOC_ARENA_MAX=${MALLOC_ARENA_MAX} + # Keep Python as PID 1 so watchdog os._exit(1) reaches Docker unchanged. Any # future wrapper must exec the coordinator or explicitly propagate its status. CMD ["python3", "-m", "lab.prism.prism_coordinator"] diff --git a/lab/prism/accepted_preview_telemetry.py b/lab/prism/accepted_preview_telemetry.py new file mode 100644 index 000000000..129078e8b --- /dev/null +++ b/lab/prism/accepted_preview_telemetry.py @@ -0,0 +1,723 @@ +"""Shared telemetry contract for accepted-preview latency attribution. + +Why this module exists +---------------------- +Issue #224: ``PrismAcceptedPreviewPublicationLatencyHigh`` fired on Union +mainnet while the coordinator stayed healthy. The publication histogram +(``qbit_prism_accepted_block_preview_publication_seconds``) proves that a +tail exists between definitive node acceptance and the visible payout +preview, but nothing on ``/metrics`` says *which stretch of the landing* +owns a sample above the 4 s warning or the 5 s child wait budget. Logs +around the alert showed reconcile-driven full payout-window rescans and +publication-blocked retries, which is correlated evidence, not +attribution. + +Three implementation lanes fix that in parallel: the core landing path, +landing attribution, and ledger attribution. They must agree on metric +names, label vocabularies, snapshot shapes, and neutral record types +before any of them instruments a call site, or the families they export +drift and the operator gets three partial answers. This module is that +agreement. It is deliberately behavior-neutral: nothing here is called on +a hot path yet, nothing here changes payout, reconciliation, +finalization, candidate, ledger, queue, lock, timeout, publication, or +retry behavior, and the renderer exports every family from an empty +owner so the series exist from process start. + +Contract at a glance +-------------------- +Every vocabulary below is a closed tuple. A metric label may only ever be +one of its members: the recorder raises on an unknown programmer-chosen +label (phase, caller, step, path) exactly as +:meth:`ReorgReconcilerService.record_lookup` does, and *normalizes* the two +values that arrive as runtime strings (a payout-window rescan reason, a +ledger read operation name) onto a fixed ``other`` member rather than +growing a series. Block hashes, heights, miner IDs, exception text and +caller-supplied operation strings never become labels; the per-publication +diagnostic record is where a hash or height may appear, bounded by a +small ring. + +``PRISM_ACCEPTED_LANDING_PHASES`` + Sub-phases of one accepted-block landing, in the order the landing + runs them (``BlockFinalizationService._land_and_confirm_block_candidate`` + and the accounting lane around it): + + ``lane_wait`` + definitive node acceptance stamped on the submitter thread + (``_note_accepted_block_preview_acceptance``) to the accounting + lane starting the landing task. + ``balance_lock_wait`` + waiting for ``_payout_balance_mutation_lock`` under + ``_block_submitter_lock(..., "payout-balance-mutation")``. + ``reconcile`` + the inline ``ensure_reorg_reconciled_for_tip`` stretch, the + ``reorg-reconcile`` heartbeat phase. + ``prior_balances_check`` + ``_stamped_prior_balances_match_current`` and the live + ``current_prior_balances`` comparison before the verified preview. + This is a prior-balances *reread*; it is never a payout-window + rescan (see the rescan family below). + ``chain_probe`` + the active-chain reads: ``pool-block-state``, ``tip-height-rpc`` + and ``_block_candidate_chain_probe``. + ``preview_prepare`` + materializing the compact preview from the issued job summary or + deriving the verified preview from the audit bundle. + ``preview_publish`` + ``_publish_accepted_block_payout_preview`` itself, both the + generation publication and the degraded fenced branch. + + Family: ``qbit_prism_accepted_block_landing_phase_seconds{phase}`` + as ``_sum`` / ``_count`` / ``_max``, the same shape as + ``qbit_prism_block_finalization_phase_seconds``. + +``PRISM_REORG_RECONCILE_CALLERS`` / ``PRISM_REORG_RECONCILE_STEPS`` + Which path asked for a reconcile pass and where the pass spent its + wall-clock. Callers: ``landing`` (the in-lock + ``ensure_reorg_reconciled_for_tip(..., _coalesce_same_tip=False)``), + ``post_confirm`` (the forced-publish + ``reconcile_prism_pool_blocks_once(_force_publish=True)`` after a + durable confirmation), ``tip_refresh``, ``job_build`` (spelled exactly + as ``PRISM_REORG_RECONCILE_LOOKUP_PATHS`` spells them, so the two + families join), and ``other`` for tools, benchmarks and periodic + passes. Steps follow ``ReorgReconcilerService.reconcile``: + ``admission_wait`` (the serialized adapter's writer admission plus the + payout-balance mutation lock, added because that wait is the one + stretch a tip-refresh caller spends behind a landing and it is not a + step of the pass itself), ``watch_query`` (``reorg_watch_blocks``), + ``chain_probe`` (``getblockcount`` / ``getblockhash`` per watched or + stranded row), ``mutations`` (``mark_pool_block_inactive``, + ``reactivate_pool_block``, ``reject_prepared_block``), + ``candidate_prepare`` (``prepared_candidate``, which is where a + ``reconcile_invalidation`` full rescan actually executes) and + ``publish`` (``publish_candidate``). + + Families: ``qbit_prism_reorg_reconcile_pass_seconds{caller}`` and + ``qbit_prism_reorg_reconcile_step_seconds{caller,step}``, each as + ``_sum`` / ``_count`` / ``_max``. The pass count by caller is the + pass family's ``_count``; pass minus the sum of its steps is the + unattributed remainder. + +``PRISM_PAYOUT_WINDOW_FULL_RESCAN_REASONS`` / ``..._PATHS`` + Full payout-window oracle rescans by the reason that forced them and + the window pipeline that executed the fold. The reason vocabulary is + the set of literals ``lab.prism.payout_state`` hands to + ``_full_payout_window_materialization(reason=...)`` or records as the + periodic self-check reason; it lives here only because that module + keeps them as scattered literals rather than a tuple, and + ``tests.test_prism_accepted_preview_telemetry`` parses that module to + fail the moment a literal appears that this tuple does not carry. + ``normalize_payout_window_full_rescan_reason`` maps anything else to + ``other``. Paths: ``daemon`` (the Rust ``--serve`` window pipeline) and + ``in_process`` (the Python fold, including the legacy whole-snapshot + read for ledgers without the incremental record contract). + + Family: ``qbit_prism_payout_window_full_rescan_seconds{reason,path}`` + as ``_sum`` / ``_count`` / ``_max``. A prior-balances reread is *not* + an observation of this family: it is a ledger read operation + (``current_prior_balances``) and, on the landing, the + ``prior_balances_check`` phase. Wave 1 must never encode the two as + one event. + +``PRISM_LEDGER_READ_OPERATIONS`` + The closed operation vocabulary for the existing + ``qbit_prism_ledger_read_*`` families, which already split + coordinator-local admission (``gate_wait``) from PostgreSQL execution + (``execute``) per operation. ``pending_block_candidate_rows`` is the + one operation shipped today; ``payout_window_snapshot`` + (``snapshot_at_job_issue``), ``payout_window_delta`` + (``snapshot_between_job_issues``), ``current_prior_balances`` and + ``prior_balances_after_pool_block`` are the names the ledger lane must + use when it routes those statements through the attributed read path. + ``fold_ledger_read_stats`` merges any other operation name into + ``other`` before rendering so the label stays bounded even if a call + site drifts. + +``AcceptedPreviewPublicationDiagnostic`` + One bounded per-publication record for structured logging or the + small in-process ring (``PRISM_ACCEPTED_PREVIEW_DIAGNOSTICS_CAPACITY`` + entries, oldest evicted). It carries the block hash and height that + the metrics must not, the publication result, the acceptance-to- + publication interval, the per-phase seconds, the reconcile caller, the + rescan reason/path/seconds and the landing's ledger gate/execute + totals. Every field is a fixed-vocabulary string, a number, or None. + +Verified invariant: reconciliation and the share ledger +------------------------------------------------------- +A reconcile pass may change pool-block and payout balance state. Its +three mutators -- ``mark_pool_block_inactive``, ``reactivate_pool_block`` +and ``reject_prepared_block`` -- call ``qbit_mark_pool_block_inactive``, +``qbit_reactivate_pool_block`` and ``qbit_reject_prepared_pool_block``, +which update ``qbit_pool_blocks``, ``qbit_pool_payout_entries``, +``qbit_payout_carry_forward``, ``qbit_ctv_fanout_artifacts`` and the +writer-lease row. Neither those functions, the ``qbit_confirm_pool_block`` +and ``qbit_reverse_immature_pool_block`` helpers they call, nor any other +statement in ``crates/qbit-prism/sql/001_share_ledger.sql`` updates or +deletes from ``qbit_share_ledger``; the only writers of that table are the +share-append inserts in ``lab.prism.share_ledger``. So a reconcile pass +can move the prior-balances base the next preview must observe, but it +cannot change the contents of an accepted-share window at a fixed anchor. + +The optimization that follows from this -- answering a +``reconcile_invalidation`` with a prior-balances reread instead of a full +window rescan -- relies on exactly that invariant. It is **not** +implemented in this wave. This module only makes sure the two events are +distinguishable once it is. + +Handoff +------- +Every lane obtains the shared owner with +``ensure_accepted_preview_telemetry(runtime)`` where ``runtime`` is the +coordinator (the object every service already holds as ``self.runtime`` / +``self._runtime`` / ``self._coordinator``); the renderer obtains it from +its port the same way, so no coordinator attribute or ``_ensure_*`` method +needs adding and the lanes do not collide in ``prism_coordinator.py``. +""" + +from __future__ import annotations + +from collections import deque +from contextlib import contextmanager +from dataclasses import dataclass, field, replace as dataclass_replace +import math +import threading +import time +from types import MappingProxyType +from typing import Any, Callable, Iterator, Mapping + + +# --- closed vocabularies --------------------------------------------------- + +LANDING_PHASE_LANE_WAIT = "lane_wait" +LANDING_PHASE_BALANCE_LOCK_WAIT = "balance_lock_wait" +LANDING_PHASE_RECONCILE = "reconcile" +LANDING_PHASE_PRIOR_BALANCES_CHECK = "prior_balances_check" +LANDING_PHASE_CHAIN_PROBE = "chain_probe" +LANDING_PHASE_PREVIEW_PREPARE = "preview_prepare" +LANDING_PHASE_PREVIEW_PUBLISH = "preview_publish" +PRISM_ACCEPTED_LANDING_PHASES = ( + LANDING_PHASE_LANE_WAIT, + LANDING_PHASE_BALANCE_LOCK_WAIT, + LANDING_PHASE_RECONCILE, + LANDING_PHASE_PRIOR_BALANCES_CHECK, + LANDING_PHASE_CHAIN_PROBE, + LANDING_PHASE_PREVIEW_PREPARE, + LANDING_PHASE_PREVIEW_PUBLISH, +) + +RECONCILE_CALLER_LANDING = "landing" +RECONCILE_CALLER_POST_CONFIRM = "post_confirm" +RECONCILE_CALLER_TIP_REFRESH = "tip_refresh" +RECONCILE_CALLER_JOB_BUILD = "job_build" +RECONCILE_CALLER_OTHER = "other" +PRISM_REORG_RECONCILE_CALLERS = ( + RECONCILE_CALLER_LANDING, + RECONCILE_CALLER_POST_CONFIRM, + RECONCILE_CALLER_TIP_REFRESH, + RECONCILE_CALLER_JOB_BUILD, + RECONCILE_CALLER_OTHER, +) + +RECONCILE_STEP_ADMISSION_WAIT = "admission_wait" +RECONCILE_STEP_WATCH_QUERY = "watch_query" +RECONCILE_STEP_CHAIN_PROBE = "chain_probe" +RECONCILE_STEP_MUTATIONS = "mutations" +RECONCILE_STEP_CANDIDATE_PREPARE = "candidate_prepare" +RECONCILE_STEP_PUBLISH = "publish" +PRISM_REORG_RECONCILE_STEPS = ( + RECONCILE_STEP_ADMISSION_WAIT, + RECONCILE_STEP_WATCH_QUERY, + RECONCILE_STEP_CHAIN_PROBE, + RECONCILE_STEP_MUTATIONS, + RECONCILE_STEP_CANDIDATE_PREPARE, + RECONCILE_STEP_PUBLISH, +) + +FULL_RESCAN_REASON_OTHER = "other" +# Mirrors the literals in lab.prism.payout_state (see the module docstring +# and the AST drift test). Order is render order: the landing-forced reason +# first, then cache-lifecycle reasons, daemon reasons, incremental fallbacks, +# the periodic self-check family, and the fold-in bucket last. +PRISM_PAYOUT_WINDOW_FULL_RESCAN_REASONS = ( + "reconcile_invalidation", + "cold_start", + "cache_invalidated", + "late_visible_append", + "anchor_regression", + "snapshot_window_weight_out_of_band", + "window_pipeline_mode_changed", + "window_mirror_divergence", + "window_daemon_unavailable", + "window_daemon_busy", + "window_daemon_state_lost", + "window_value_out_of_range", + "delta_api_unavailable", + "incremental_invariant_failed", + "periodic_self_check", + "periodic_self_check_failed", + "periodic_self_check_balance_check_failed", + FULL_RESCAN_REASON_OTHER, +) + +FULL_RESCAN_PATH_DAEMON = "daemon" +FULL_RESCAN_PATH_IN_PROCESS = "in_process" +PRISM_PAYOUT_WINDOW_FULL_RESCAN_PATHS = ( + FULL_RESCAN_PATH_DAEMON, + FULL_RESCAN_PATH_IN_PROCESS, +) + +LEDGER_READ_OPERATION_PENDING_BLOCK_CANDIDATE_ROWS = "pending_block_candidate_rows" +LEDGER_READ_OPERATION_PAYOUT_WINDOW_SNAPSHOT = "payout_window_snapshot" +LEDGER_READ_OPERATION_PAYOUT_WINDOW_DELTA = "payout_window_delta" +LEDGER_READ_OPERATION_CURRENT_PRIOR_BALANCES = "current_prior_balances" +LEDGER_READ_OPERATION_PRIOR_BALANCES_AFTER_POOL_BLOCK = ( + "prior_balances_after_pool_block" +) +LEDGER_READ_OPERATION_OTHER = "other" +PRISM_LEDGER_READ_OPERATIONS = ( + "observe_block_candidate_outbox_row", + "pending_block_candidate_headers", + "read_block_candidate_body_manifest", + "read_block_candidate_body_page", + "read_block_candidate_body_pages", + "read_block_candidate_body_spans", + LEDGER_READ_OPERATION_PENDING_BLOCK_CANDIDATE_ROWS, + LEDGER_READ_OPERATION_PAYOUT_WINDOW_SNAPSHOT, + LEDGER_READ_OPERATION_PAYOUT_WINDOW_DELTA, + LEDGER_READ_OPERATION_CURRENT_PRIOR_BALANCES, + LEDGER_READ_OPERATION_PRIOR_BALANCES_AFTER_POOL_BLOCK, + LEDGER_READ_OPERATION_OTHER, +) + +# How many per-publication diagnostic records the in-process ring keeps. +# Sized for an operator reading the last hour of a dense accepted-tip +# cadence (the #224 alert hour held 35 accepted blocks), not as a log. +PRISM_ACCEPTED_PREVIEW_DIAGNOSTICS_CAPACITY = 64 + + +def normalize_payout_window_full_rescan_reason(reason: object) -> str: + """Bound a payout_state reason string to the closed vocabulary.""" + if isinstance(reason, str) and reason in PRISM_PAYOUT_WINDOW_FULL_RESCAN_REASONS: + return reason + return FULL_RESCAN_REASON_OTHER + + +def normalize_ledger_read_operation(operation: object) -> str: + """Bound a ledger read operation name to the closed vocabulary.""" + if isinstance(operation, str) and operation in PRISM_LEDGER_READ_OPERATIONS: + return operation + return LEDGER_READ_OPERATION_OTHER + + +def fold_ledger_read_stats( + stats_by_operation: Mapping[str, Mapping[str, float | int]], +) -> dict[str, dict[str, float | int]]: + """Merge out-of-contract operation names into ``other`` for rendering. + + Operations in :data:`PRISM_LEDGER_READ_OPERATIONS` pass through as + copies, so the shipped ``pending_block_candidate_rows`` series render + exactly as before. Anything else is summed into ``other`` -- ``*_max`` + fields take the maximum, every other field adds -- so an operation + name the ledger lane did not take from this contract still cannot open + a new series. + """ + folded: dict[str, dict[str, float | int]] = {} + for operation, stats in stats_by_operation.items(): + key = normalize_ledger_read_operation(operation) + target = folded.get(key) + if target is None: + folded[key] = dict(stats) + continue + for name, value in stats.items(): + if name.endswith("_max"): + target[name] = max(float(target.get(name, 0.0)), float(value)) + else: + target[name] = target.get(name, 0) + value + return folded + + +def empty_duration_stats() -> dict[str, float | int]: + """The sum/count/max cell every family here accumulates into.""" + return {"count": 0, "sum": 0.0, "max": 0.0} + + +def _accumulate(stats: dict[str, float | int], seconds: float) -> None: + elapsed = float(seconds) + if not math.isfinite(elapsed) or elapsed < 0.0: + elapsed = 0.0 + stats["count"] = int(stats["count"]) + 1 + stats["sum"] = float(stats["sum"]) + elapsed + stats["max"] = max(float(stats["max"]), elapsed) + + +def _publication_results() -> tuple[str, ...]: + # Imported lazily: block_candidates is the B1 owner module and will + # import this contract from its landing paths; a module-level import + # here would make that a cycle. + from lab.prism.block_candidates import ( + PRISM_ACCEPTED_BLOCK_PREVIEW_PUBLICATION_RESULTS, + ) + + return PRISM_ACCEPTED_BLOCK_PREVIEW_PUBLICATION_RESULTS + + +def _bounded_seconds(value: object) -> float: + seconds = float(value) # type: ignore[arg-type] + if not math.isfinite(seconds) or seconds < 0.0: + return 0.0 + return seconds + + +# --- neutral data types ---------------------------------------------------- + + +@dataclass(frozen=True, slots=True) +class AcceptedPreviewPublicationDiagnostic: + """One bounded record of an accepted block's preview publication. + + This is the only place in the contract where a block hash or height + may appear. It is a value for a structured log line or the diagnostics + ring, never a metric label. Construction validates every vocabulary + field and clamps every duration, so a record that exists is already + safe to log. + """ + + block_hash: str + block_height: int | None + result: str + acceptance_to_publication_seconds: float + phase_seconds: Mapping[str, float] = field(default_factory=dict) + reconcile_caller: str | None = None + full_rescan_reason: str | None = None + full_rescan_path: str | None = None + full_rescan_seconds: float = 0.0 + ledger_gate_wait_seconds: float = 0.0 + ledger_execute_seconds: float = 0.0 + recorded_monotonic: float | None = None + + def __post_init__(self) -> None: + block_hash = str(self.block_hash).strip().lower() + if not block_hash: + raise ValueError("diagnostic block_hash must be non-empty") + object.__setattr__(self, "block_hash", block_hash) + if self.block_height is not None: + object.__setattr__(self, "block_height", int(self.block_height)) + if self.result not in _publication_results(): + raise ValueError( + f"unknown accepted preview publication result: {self.result!r}" + ) + phases: dict[str, float] = { + phase: 0.0 for phase in PRISM_ACCEPTED_LANDING_PHASES + } + for phase, seconds in dict(self.phase_seconds).items(): + if phase not in phases: + raise ValueError(f"unknown accepted landing phase: {phase!r}") + phases[phase] = _bounded_seconds(seconds) + object.__setattr__(self, "phase_seconds", MappingProxyType(phases)) + if ( + self.reconcile_caller is not None + and self.reconcile_caller not in PRISM_REORG_RECONCILE_CALLERS + ): + raise ValueError( + f"unknown reorg reconcile caller: {self.reconcile_caller!r}" + ) + if self.full_rescan_reason is not None: + object.__setattr__( + self, + "full_rescan_reason", + normalize_payout_window_full_rescan_reason(self.full_rescan_reason), + ) + if ( + self.full_rescan_path is not None + and self.full_rescan_path not in PRISM_PAYOUT_WINDOW_FULL_RESCAN_PATHS + ): + raise ValueError( + f"unknown payout window full rescan path: {self.full_rescan_path!r}" + ) + for name in ( + "acceptance_to_publication_seconds", + "full_rescan_seconds", + "ledger_gate_wait_seconds", + "ledger_execute_seconds", + ): + object.__setattr__(self, name, _bounded_seconds(getattr(self, name))) + if self.recorded_monotonic is not None: + object.__setattr__( + self, "recorded_monotonic", float(self.recorded_monotonic) + ) + + def log_fields(self) -> dict[str, object]: + """Flat JSON-serializable fields for ``_payout_artifact_log`` style lines.""" + fields: dict[str, object] = { + "block_hash": self.block_hash, + "block_height": self.block_height, + "result": self.result, + "acceptance_to_publication_seconds": round( + self.acceptance_to_publication_seconds, 6 + ), + "reconcile_caller": self.reconcile_caller, + "full_rescan_reason": self.full_rescan_reason, + "full_rescan_path": self.full_rescan_path, + "full_rescan_seconds": round(self.full_rescan_seconds, 6), + "ledger_gate_wait_seconds": round(self.ledger_gate_wait_seconds, 6), + "ledger_execute_seconds": round(self.ledger_execute_seconds, 6), + } + for phase in PRISM_ACCEPTED_LANDING_PHASES: + fields[f"phase_{phase}_seconds"] = round(self.phase_seconds[phase], 6) + return fields + + def summary(self) -> str: + """One line naming the owner of the interval, for plain prints.""" + phases = " ".join( + f"{phase}={self.phase_seconds[phase]:.3f}s" + for phase in PRISM_ACCEPTED_LANDING_PHASES + ) + rescan = ( + f" rescan={self.full_rescan_reason}/{self.full_rescan_path}" + f"={self.full_rescan_seconds:.3f}s" + if self.full_rescan_reason is not None + else "" + ) + return ( + f"hash={self.block_hash} height={self.block_height} " + f"result={self.result} " + f"total={self.acceptance_to_publication_seconds:.3f}s {phases}" + f"{rescan} ledger_gate_wait={self.ledger_gate_wait_seconds:.3f}s " + f"ledger_execute={self.ledger_execute_seconds:.3f}s" + ) + + +# --- the shared owner ------------------------------------------------------ + + +class AcceptedPreviewTelemetry: + """Fixed-cardinality accumulators plus the bounded diagnostics ring. + + One lock guards every cell, the same model as the B1 owner's + publication histogram and the finalization service's phase metrics: + recorders take it for one dictionary update, ``snapshot`` copies every + cell under it and returns plain dicts the renderer may read without + it. Every cell of every closed product exists from construction, so an + empty owner renders every series at zero and a scrape never sees a + family appear mid-life. + """ + + def __init__( + self, + *, + diagnostics_capacity: int = PRISM_ACCEPTED_PREVIEW_DIAGNOSTICS_CAPACITY, + monotonic: Callable[[], float] | None = None, + ) -> None: + capacity = int(diagnostics_capacity) + if capacity <= 0: + raise ValueError("diagnostics_capacity must be positive") + self._monotonic = monotonic or time.monotonic + self._lock = threading.Lock() + self._landing_phases = { + phase: empty_duration_stats() for phase in PRISM_ACCEPTED_LANDING_PHASES + } + self._reconcile_passes = { + caller: empty_duration_stats() for caller in PRISM_REORG_RECONCILE_CALLERS + } + self._reconcile_steps = { + (caller, step): empty_duration_stats() + for caller in PRISM_REORG_RECONCILE_CALLERS + for step in PRISM_REORG_RECONCILE_STEPS + } + self._full_rescans = { + (reason, path): empty_duration_stats() + for reason in PRISM_PAYOUT_WINDOW_FULL_RESCAN_REASONS + for path in PRISM_PAYOUT_WINDOW_FULL_RESCAN_PATHS + } + self._diagnostics: deque[AcceptedPreviewPublicationDiagnostic] = deque( + maxlen=capacity + ) + self._diagnostics_recorded = 0 + + # -- recorders --------------------------------------------------------- + + def observe_landing_phase(self, phase: str, seconds: float) -> None: + if phase not in self._landing_phases: + raise ValueError(f"unknown accepted landing phase: {phase!r}") + with self._lock: + _accumulate(self._landing_phases[phase], seconds) + + @contextmanager + def landing_phase(self, phase: str) -> Iterator[None]: + """Time one landing sub-phase; records on every exit, raising or not.""" + if phase not in self._landing_phases: + raise ValueError(f"unknown accepted landing phase: {phase!r}") + started = self._monotonic() + try: + yield + finally: + self.observe_landing_phase(phase, self._monotonic() - started) + + def observe_reconcile_pass(self, caller: str, seconds: float) -> None: + if caller not in self._reconcile_passes: + raise ValueError(f"unknown reorg reconcile caller: {caller!r}") + with self._lock: + _accumulate(self._reconcile_passes[caller], seconds) + + def observe_reconcile_step(self, caller: str, step: str, seconds: float) -> None: + key = (caller, step) + if key not in self._reconcile_steps: + if caller not in PRISM_REORG_RECONCILE_CALLERS: + raise ValueError(f"unknown reorg reconcile caller: {caller!r}") + raise ValueError(f"unknown reorg reconcile step: {step!r}") + with self._lock: + _accumulate(self._reconcile_steps[key], seconds) + + @contextmanager + def reconcile_step(self, caller: str, step: str) -> Iterator[None]: + """Time one reconcile step for one caller; records on every exit.""" + if (caller, step) not in self._reconcile_steps: + if caller not in PRISM_REORG_RECONCILE_CALLERS: + raise ValueError(f"unknown reorg reconcile caller: {caller!r}") + raise ValueError(f"unknown reorg reconcile step: {step!r}") + started = self._monotonic() + try: + yield + finally: + self.observe_reconcile_step(caller, step, self._monotonic() - started) + + def observe_payout_window_full_rescan( + self, + reason: object, + path: str, + seconds: float, + ) -> None: + """Record one executed full oracle rescan. + + ``reason`` is the payout_state string and is normalized; ``path`` + is chosen by the recording code and must be a vocabulary member. + A prior-balances reread must not be recorded here. + """ + if path not in PRISM_PAYOUT_WINDOW_FULL_RESCAN_PATHS: + raise ValueError(f"unknown payout window full rescan path: {path!r}") + key = (normalize_payout_window_full_rescan_reason(reason), path) + with self._lock: + _accumulate(self._full_rescans[key], seconds) + + def record_publication_diagnostic( + self, + diagnostic: AcceptedPreviewPublicationDiagnostic, + ) -> AcceptedPreviewPublicationDiagnostic: + """Retain one publication record, stamping it if the caller did not.""" + if not isinstance(diagnostic, AcceptedPreviewPublicationDiagnostic): + raise TypeError("expected an AcceptedPreviewPublicationDiagnostic") + if diagnostic.recorded_monotonic is None: + diagnostic = dataclass_replace( + diagnostic, recorded_monotonic=self._monotonic() + ) + with self._lock: + self._diagnostics.append(diagnostic) + self._diagnostics_recorded += 1 + return diagnostic + + # -- readers ----------------------------------------------------------- + + def snapshot(self) -> dict[str, Any]: + """Copied cells for one render; keys are the closed products.""" + with self._lock: + return { + "landing_phases": { + phase: dict(stats) + for phase, stats in self._landing_phases.items() + }, + "reconcile_passes": { + caller: dict(stats) + for caller, stats in self._reconcile_passes.items() + }, + "reconcile_steps": { + key: dict(stats) for key, stats in self._reconcile_steps.items() + }, + "full_rescans": { + key: dict(stats) for key, stats in self._full_rescans.items() + }, + "diagnostics_retained": len(self._diagnostics), + "diagnostics_recorded": self._diagnostics_recorded, + } + + def diagnostics_snapshot( + self, + ) -> tuple[AcceptedPreviewPublicationDiagnostic, ...]: + """Retained publication records, oldest first.""" + with self._lock: + return tuple(self._diagnostics) + + +_OWNER_ATTRIBUTE = "_accepted_preview_telemetry" +_OWNER_BOOTSTRAP_LOCK = threading.Lock() + + +def ensure_accepted_preview_telemetry(owner: object) -> AcceptedPreviewTelemetry: + """The one telemetry owner attached to ``owner`` (normally the coordinator). + + Idempotent and race-free: the first caller from any thread creates + the instance under a module bootstrap lock, every later caller gets + that same instance. State lives in the owner's instance ``__dict__`` + exactly as the coordinator's lazily built renderer does, so no + coordinator attribute, descriptor or ``_ensure_*`` method has to be + added for the lanes or the renderer to share it. + """ + try: + state = vars(owner) + except TypeError as exc: + raise TypeError( + "accepted preview telemetry needs an owner with an instance " + f"__dict__, got {type(owner).__name__}" + ) from exc + telemetry = state.get(_OWNER_ATTRIBUTE) + if telemetry is not None: + return telemetry + with _OWNER_BOOTSTRAP_LOCK: + telemetry = state.get(_OWNER_ATTRIBUTE) + if telemetry is None: + telemetry = AcceptedPreviewTelemetry() + state[_OWNER_ATTRIBUTE] = telemetry + return telemetry + + +__all__ = [ + "AcceptedPreviewPublicationDiagnostic", + "AcceptedPreviewTelemetry", + "FULL_RESCAN_PATH_DAEMON", + "FULL_RESCAN_PATH_IN_PROCESS", + "FULL_RESCAN_REASON_OTHER", + "LANDING_PHASE_BALANCE_LOCK_WAIT", + "LANDING_PHASE_CHAIN_PROBE", + "LANDING_PHASE_LANE_WAIT", + "LANDING_PHASE_PREVIEW_PREPARE", + "LANDING_PHASE_PREVIEW_PUBLISH", + "LANDING_PHASE_PRIOR_BALANCES_CHECK", + "LANDING_PHASE_RECONCILE", + "LEDGER_READ_OPERATION_CURRENT_PRIOR_BALANCES", + "LEDGER_READ_OPERATION_OTHER", + "LEDGER_READ_OPERATION_PAYOUT_WINDOW_DELTA", + "LEDGER_READ_OPERATION_PAYOUT_WINDOW_SNAPSHOT", + "LEDGER_READ_OPERATION_PENDING_BLOCK_CANDIDATE_ROWS", + "LEDGER_READ_OPERATION_PRIOR_BALANCES_AFTER_POOL_BLOCK", + "PRISM_ACCEPTED_LANDING_PHASES", + "PRISM_ACCEPTED_PREVIEW_DIAGNOSTICS_CAPACITY", + "PRISM_LEDGER_READ_OPERATIONS", + "PRISM_PAYOUT_WINDOW_FULL_RESCAN_PATHS", + "PRISM_PAYOUT_WINDOW_FULL_RESCAN_REASONS", + "PRISM_REORG_RECONCILE_CALLERS", + "PRISM_REORG_RECONCILE_STEPS", + "RECONCILE_CALLER_JOB_BUILD", + "RECONCILE_CALLER_LANDING", + "RECONCILE_CALLER_OTHER", + "RECONCILE_CALLER_POST_CONFIRM", + "RECONCILE_CALLER_TIP_REFRESH", + "RECONCILE_STEP_ADMISSION_WAIT", + "RECONCILE_STEP_CANDIDATE_PREPARE", + "RECONCILE_STEP_CHAIN_PROBE", + "RECONCILE_STEP_MUTATIONS", + "RECONCILE_STEP_PUBLISH", + "RECONCILE_STEP_WATCH_QUERY", + "empty_duration_stats", + "ensure_accepted_preview_telemetry", + "fold_ledger_read_stats", + "normalize_ledger_read_operation", + "normalize_payout_window_full_rescan_reason", +] diff --git a/lab/prism/audit_artifacts.py b/lab/prism/audit_artifacts.py new file mode 100644 index 000000000..f28ce7bf6 --- /dev/null +++ b/lab/prism/audit_artifacts.py @@ -0,0 +1,6154 @@ +#!/usr/bin/env python3 +"""Filesystem ownership, verification, and publication for PRISM audit data. + +This module deliberately has no coordinator dependency. The coordinator owns +block-finalization sequencing and the ledger owns database authorization; this +store is the sole authority for paths and filesystem mutation below the audit +root. +""" + +from __future__ import annotations + +import copy +from contextlib import contextmanager, nullcontext +import fcntl +import gzip +import hashlib +import hmac +import io +import json +import os +import re +import selectors +import signal +import stat +import subprocess +import threading +import time +import uuid +import zlib +from dataclasses import dataclass +from pathlib import Path +from collections.abc import Sequence +from typing import Any, Callable, Iterable, Iterator, Mapping + +from lab.prism.audit_bundle_view import ( + ArtifactSource, + CanonicalArtifactError, + CanonicalAuditBundleView, + RawJsonRecord, + iter_json_byte_chunks, + json_chunks_sha256_and_size, + materialize_json, + spool_json_document, +) +from lab.prism.prism_tools import prism_tool_command + + +AUDIT_BODY_REF_SCHEMA = "qbit.prism.audit-body-ref.v1" +AUDIT_BUNDLE_V2_SCHEMA = "qbit.prism.audit-bundle.v2" +AUDIT_SHARE_SEGMENT_SCHEMA = "qbit.prism.audit-share-segment.v1" +AUDIT_WINDOW_COMPLETENESS_PROOF_SCHEMA = ( + "qbit.prism.window-completeness-proof.v1" +) +LIVE_ENVELOPE_SCHEMA = "qbit.prism.live-audit-bundle-envelope.v1" +LIVE_EVIDENCE_SCHEMA = "qbit.prism.live-stratum-evidence.v1" +DEFAULT_AUDIT_SHARE_SEGMENT_SIZE = 10_000 +DEFAULT_VERIFIER_TIMEOUT_SECONDS = 60.0 +MAX_VERIFIER_OUTPUT_BYTES = 1024 * 1024 +VERIFICATION_REPORT_SCHEMA = "qbit.prism.audit-verification-report.v1" +VERIFICATION_IDENTITY_SCHEMA = "qbit.prism.audit-verification-identity.v1" +LEGACY_VERIFICATION_UNAVAILABLE_SCHEMA = ( + "qbit.prism.legacy-verification-unavailable.v1" +) +# The canonical bundle is stored compressed, but its public identity stays +# the SHA-256 of the *uncompressed* canonical JSON. Level 9 with mtime=0 +# keeps stdlib gzip reproducible for a given compressor; republication checks +# the uncompressed payload so a compressor change cannot redefine identity. +CANONICAL_BUNDLE_GZIP_LEVEL = 9 + +_HEX_64 = r"[0-9a-f]{64}" +_TOKEN_32 = r"[0-9a-f]{32}" +_BODY_RE = re.compile( + rf"\Aprism-audit-bundle-body-(?P{_HEX_64})-(?P{_HEX_64})\.json\Z" +) +_SHARE_CONTENT_RE = re.compile( + rf"\Aprism-audit-share-segment-(?P[1-9][0-9]*)-" + rf"(?P[1-9][0-9]*)-(?P{_HEX_64})\.json\Z" +) +_SHARE_SLOT_RE = re.compile( + r"\Aprism-audit-share-segment-slot-(?P[1-9][0-9]*)-" + r"(?P[1-9][0-9]*)\.json\Z" +) +_CANONICAL_BUNDLE_RE = re.compile( + rf"\Aprism-audit-bundle-canonical-(?P{_HEX_64})-" + rf"(?P{_HEX_64})\.json\.gz\Z" +) +_LIVE_RE = re.compile( + rf"\Aprism-live-audit-bundle-(?P0|[1-9][0-9]*)-" + rf"(?P{_HEX_64})\.json\Z" +) +_CANDIDATE_RE = re.compile( + rf"\A\.prism-live-audit-bundle-candidate-(?P{_HEX_64})-" + rf"(?P{_TOKEN_32})\.json\.tmp\Z" +) +_LEGACY_CANDIDATE_RE = re.compile( + rf"\Aprism-live-audit-bundle-candidate-(?P{_HEX_64})\.json\Z" +) +_LEGACY_HIDDEN_CANDIDATE_RE = re.compile( + rf"\A\.prism-live-audit-bundle-candidate-(?P{_HEX_64})\.json\.tmp\Z" +) +_PUBLICATION_LOCK_NAME = ".prism-audit-publication.lock" +_PUBLICATION_THREAD_LOCKS_GUARD = threading.Lock() +_PUBLICATION_THREAD_LOCKS: dict[tuple[int, int], threading.RLock] = {} +_PUBLICATION_LOCAL_OWNERS: dict[tuple[int, int], tuple[int, int]] = {} + + +def _publication_thread_lock(identity: tuple[int, int]) -> threading.RLock: + """Share a local lock across store instances opened on the same inode.""" + + with _PUBLICATION_THREAD_LOCKS_GUARD: + lock = _PUBLICATION_THREAD_LOCKS.get(identity) + if lock is None: + lock = threading.RLock() + _PUBLICATION_THREAD_LOCKS[identity] = lock + return lock + + +def _reject_cross_store_local_owner( + identity: tuple[int, int], + *, + thread_id: int, + store_id: int, +) -> None: + """Reject same-thread nesting through a second store before it can block.""" + + with _PUBLICATION_THREAD_LOCKS_GUARD: + local_owner = _PUBLICATION_LOCAL_OWNERS.get(identity) + if local_owner is not None: + owner_thread, owner_store = local_owner + if owner_thread == thread_id and owner_store != store_id: + raise RuntimeError( + "audit publication guard is already held by another store " + "on this thread" + ) + + +def _canonical_hex(value: object, *, name: str, expected_bytes: int = 32) -> str: + text = str(value) + if len(text) != expected_bytes * 2: + raise ValueError(f"{name} must be {expected_bytes} bytes of hex") + try: + bytes.fromhex(text) + except ValueError as exc: + raise ValueError(f"{name} must be hexadecimal") from exc + return text.lower() + + +def _sha256_bytes(payload: bytes) -> str: + return hashlib.sha256(payload).hexdigest() + + +def _json_bytes(payload: Mapping[str, Any], *, sort_keys: bool = False) -> bytes: + return json.dumps( + payload, + separators=(",", ":"), + sort_keys=sort_keys, + ).encode("utf-8") + + +def canonical_audit_bundle_bytes( + final_bundle: dict[str, Any], + canonicalizer: Callable[[dict[str, Any]], bytes] | None = None, +) -> bytes: + if canonicalizer is None: + raise RuntimeError("J1 canonical bundle capability is required") + canonical = canonicalizer(final_bundle) + return canonical.encode() if isinstance(canonical, str) else bytes(canonical) + + +def gzip_canonical_bundle_bytes(payload: bytes) -> bytes: + """Compress canonical bundle bytes reproducibly. + + A BytesIO sink carries no name, so gzip emits no FNAME field; with mtime=0 + the header is constant and equal inputs are reproducible for a given + compressor implementation. + """ + + sink = io.BytesIO() + with gzip.GzipFile( + fileobj=sink, + mode="wb", + compresslevel=CANONICAL_BUNDLE_GZIP_LEVEL, + mtime=0, + ) as handle: + handle.write(payload) + return sink.getvalue() + + +def gunzip_canonical_bundle_bytes(payload: bytes) -> bytes: + return gzip.decompress(payload) + + +class CanonicalAuditBundleCorrupt(RuntimeError): + """A canonical bundle artifact is present but unusable. + + Distinct from a clean miss so public readers can report the damage and fail + closed instead of silently serving a non-byte-exact reconstruction. + """ + + +class _AuditShareSegmentConflict(RuntimeError): + """A stable-slot merge saw two different payloads for one share_seq.""" + + +@dataclass(frozen=True) +class AuditArtifactConfig: + root: Path + evidence_path: Path + live_bundle_retention: int = 5 + candidate_retention_seconds: int = 24 * 60 * 60 + share_segment_size: int = 0 + verifier_timeout_seconds: float = DEFAULT_VERIFIER_TIMEOUT_SECONDS + + +@dataclass(frozen=True) +class AuditPublicationIdentity: + """Ordering token assigned while P1's balance serializer is held.""" + + sequence: int + block_height: int + block_hash: str + + def __post_init__(self) -> None: + if isinstance(self.sequence, bool) or not isinstance(self.sequence, int): + raise ValueError("publication sequence must be an integer") + if self.sequence < 0: + raise ValueError("publication sequence must be non-negative") + if isinstance(self.block_height, bool) or not isinstance( + self.block_height, + int, + ): + raise ValueError("publication block height must be an integer") + if self.block_height < 0: + raise ValueError("publication block height must be non-negative") + canonical_block_hash = _canonical_hex( + self.block_hash, + name="publication block hash", + ) + if self.block_hash != canonical_block_hash: + raise ValueError("publication block hash must be canonical") + + def to_json(self) -> dict[str, object]: + return { + "sequence": self.sequence, + "block_height": self.block_height, + "block_hash": self.block_hash, + } + + +@dataclass(frozen=True) +class OwnedCandidateArtifact: + path: Path + block_hash: str + token: str + store_token: str + + +@dataclass(frozen=True) +class VerifiedAuditBundle: + candidate: OwnedCandidateArtifact + report: Mapping[str, Any] + literal_sha256: str + byte_length: int + device: int + inode: int + mtime_ns: int + canonical_copy_eligible: bool + verification_identity: Mapping[str, Any] + + +@dataclass(frozen=True) +class PublishedAuditBodyRef: + path: Path + body_uri: str + audit_bundle_sha256: str + + +@dataclass(frozen=True) +class AuditPublication: + identity: AuditPublicationIdentity + envelope_path: Path + evidence: Mapping[str, Any] + published: bool + # True when this superseded publication restored its own missing live + # envelope. Recorded instead of logging inside the store's locked region; + # the caller emits the diagnostic after every lock scope has exited. + superseded_envelope_written: bool = False + + +@dataclass(frozen=True) +class RetentionResult: + live_removed: int = 0 + candidate_removed: int = 0 + errors: int = 0 + + +@dataclass(frozen=True) +class _FileIdentity: + device: int + inode: int + mode: int + size: int + mtime_ns: int + + @classmethod + def from_stat(cls, value: os.stat_result) -> _FileIdentity: + return cls( + value.st_dev, + value.st_ino, + value.st_mode, + value.st_size, + value.st_mtime_ns, + ) + + def matches(self, value: os.stat_result) -> bool: + return ( + self.device == value.st_dev + and self.inode == value.st_ino + and self.mode == value.st_mode + and self.size == value.st_size + and self.mtime_ns == value.st_mtime_ns + and stat.S_ISREG(value.st_mode) + ) + + +@dataclass(frozen=True) +class _LegacyProofToken: + identity: AuditPublicationIdentity + evidence_file: _FileIdentity + evidence_sha256: str + envelope_file: _FileIdentity + envelope_sha256: str + + +class AuditArtifactStore: + """Own every audit artifact path and filesystem mutation for one root.""" + + def __init__( + self, + config: AuditArtifactConfig, + *, + canonicalizer: Callable[[dict[str, Any]], bytes] | None = None, + verifier: Callable[..., dict[str, Any]] | None = None, + wall_time: Callable[[], float] = time.time, + read_only: bool = False, + ) -> None: + """Own one audit artifact root. + + ``read_only`` opens the root for reading published bodies and nothing + else. The publication lock is an ordinary file opened O_RDWR and + created on demand, so acquiring it needs a writable filesystem; the + extracted public read tier mounts this volume ``:ro`` precisely so it + cannot write, and would fail to construct a store at all. A read-only + store therefore skips the lock, and every publication path refuses + rather than running unserialized. + """ + live_bundle_retention = int(config.live_bundle_retention) + candidate_retention_seconds = int(config.candidate_retention_seconds) + share_segment_size = int(config.share_segment_size) + verifier_timeout_seconds = float(config.verifier_timeout_seconds) + if share_segment_size < 0: + raise ValueError("audit share segment size must be non-negative") + if verifier_timeout_seconds <= 0: + raise ValueError("verifier timeout must be positive") + configured_root = Path(config.root).expanduser().absolute() + if not read_only: + configured_root.mkdir(parents=True, exist_ok=True) + root_stat = configured_root.lstat() + if not stat.S_ISDIR(root_stat.st_mode) or stat.S_ISLNK(root_stat.st_mode): + raise RuntimeError("audit artifact root must be a non-symlink directory") + root = configured_root.resolve(strict=True) + configured_evidence = Path(config.evidence_path).expanduser().absolute() + if not read_only: + configured_evidence.parent.mkdir(parents=True, exist_ok=True) + evidence_path = configured_evidence.parent.resolve(strict=True) / configured_evidence.name + self._read_only = bool(read_only) + self._root = root + self._evidence_path = evidence_path + self._root_fd, self._root_identity = self._open_directory_authority(root) + try: + if read_only: + # No lock file, and no fd that could ever authorize a write. + self._publication_lock_fd = -1 + self._publication_lock_identity = None + self._publication_lifecycle_lock = threading.RLock() + self._publication_inode_thread_lock = threading.RLock() + else: + ( + self._publication_lock_fd, + self._publication_lock_identity, + ) = self._open_publication_lock_authority( + root, + self._root_fd, + self._root_identity, + ) + self._publication_lifecycle_lock = threading.RLock() + self._publication_inode_thread_lock = _publication_thread_lock( + ( + self._publication_lock_identity.device, + self._publication_lock_identity.inode, + ) + ) + self._publication_guard_owner: int | None = None + self._publication_guard_depth = 0 + ( + self._evidence_parent_fd, + self._evidence_parent_identity, + ) = self._open_directory_authority(evidence_path.parent) + except BaseException: + publication_lock_fd = getattr(self, "_publication_lock_fd", -1) + if publication_lock_fd >= 0: + os.close(publication_lock_fd) + self._publication_lock_fd = -1 + os.close(self._root_fd) + self._root_fd = -1 + raise + self._live_bundle_retention = live_bundle_retention + self._candidate_retention_seconds = candidate_retention_seconds + self._share_segment_size = share_segment_size + self._verifier_timeout_seconds = verifier_timeout_seconds + self._canonicalizer = canonicalizer + self._verifier = verifier + self._wall_time = wall_time + self._lock = threading.RLock() + self._closed = False + self._store_token = uuid.uuid4().hex + self._active_candidates: dict[str, tuple[Path, _FileIdentity | None]] = {} + self._latest_evidence: dict[str, Any] | None = None + self._current_envelope: Path | None = None + self._current_identity: AuditPublicationIdentity | None = None + self._evidence_state = "absent" + self._compatibility_evidence_override = False + self._invalidated_legacy_identity: AuditPublicationIdentity | None = None + self._legacy_proof_token: _LegacyProofToken | None = None + self._load_current_evidence() + + @property + def root(self) -> Path: + return self._root + + @property + def evidence_path(self) -> Path: + return self._evidence_path + + @property + def share_segment_size(self) -> int: + return self._share_segment_size + + @property + def live_bundle_retention(self) -> int: + return self._live_bundle_retention + + @property + def candidate_retention_seconds(self) -> int: + return self._candidate_retention_seconds + + @property + def verifier_timeout_seconds(self) -> float: + return self._verifier_timeout_seconds + + @staticmethod + def _directory_identity(path: Path) -> tuple[int, int]: + value = path.lstat() + if not stat.S_ISDIR(value.st_mode) or stat.S_ISLNK(value.st_mode): + raise RuntimeError("audit artifact parent must be a non-symlink directory") + return value.st_dev, value.st_ino + + @staticmethod + def _open_directory_authority(path: Path) -> tuple[int, tuple[int, int]]: + fd = os.open( + path, + os.O_RDONLY + | getattr(os, "O_DIRECTORY", 0) + | getattr(os, "O_NOFOLLOW", 0), + ) + try: + value = os.fstat(fd) + if not stat.S_ISDIR(value.st_mode): + raise RuntimeError("audit artifact parent is not a directory") + identity = (value.st_dev, value.st_ino) + if AuditArtifactStore._directory_identity(path) != identity: + raise RuntimeError("audit artifact parent identity changed") + return fd, identity + except RuntimeError: + os.close(fd) + raise + except OSError as exc: + os.close(fd) + raise RuntimeError( + "audit artifact parent authority is invalid" + ) from exc + except BaseException: + os.close(fd) + raise + + @staticmethod + def _open_publication_lock_authority( + root: Path, + root_fd: int, + root_identity: tuple[int, int], + ) -> tuple[int, _FileIdentity]: + flags = os.O_RDWR | getattr(os, "O_NOFOLLOW", 0) + created = False + try: + try: + fd = os.open( + _PUBLICATION_LOCK_NAME, + flags | os.O_CREAT | os.O_EXCL, + 0o600, + dir_fd=root_fd, + ) + created = True + except FileExistsError: + fd = os.open(_PUBLICATION_LOCK_NAME, flags, dir_fd=root_fd) + except OSError as exc: + raise RuntimeError( + "audit publication lock cannot be opened safely" + ) from exc + try: + value = os.fstat(fd) + identity = _FileIdentity.from_stat(value) + if not stat.S_ISREG(value.st_mode): + raise RuntimeError("audit publication lock is not a regular file") + linked = os.stat( + _PUBLICATION_LOCK_NAME, + dir_fd=root_fd, + follow_symlinks=False, + ) + if not identity.matches(linked): + raise RuntimeError("audit publication lock identity changed") + root_value = os.fstat(root_fd) + if (root_value.st_dev, root_value.st_ino) != root_identity: + raise RuntimeError("audit artifact root authority is invalid") + if AuditArtifactStore._directory_identity(root) != root_identity: + raise RuntimeError("audit artifact root identity changed") + if created: + os.fsync(fd) + os.fsync(root_fd) + return fd, identity + except RuntimeError: + os.close(fd) + raise + except OSError as exc: + os.close(fd) + raise RuntimeError( + "audit publication lock authority is invalid" + ) from exc + except BaseException: + os.close(fd) + raise + + def close(self) -> None: + lock = getattr(self, "_lock", None) + publication_lock = getattr(self, "_publication_lifecycle_lock", None) + if lock is None: + self._close_unlocked() + return + if publication_lock is None: + with lock: + self._close_unlocked() + return + with publication_lock: + if ( + self._publication_guard_owner == threading.get_ident() + and self._publication_guard_depth > 0 + ): + raise RuntimeError( + "cannot close audit artifact store inside publication guard" + ) + with lock: + self._close_unlocked() + + def _close_unlocked(self) -> None: + if getattr(self, "_closed", False): + return + self._closed = True + for field in ( + "_publication_lock_fd", + "_root_fd", + "_evidence_parent_fd", + ): + fd = getattr(self, field, -1) + if isinstance(fd, int) and fd >= 0: + try: + os.close(fd) + except OSError: + pass + setattr(self, field, -1) + + def __del__(self) -> None: + try: + self.close() + except Exception: + pass + + def _validate_publication_lock_identity( + self, + *, + root: Path | None = None, + root_fd: int | None = None, + root_identity: tuple[int, int] | None = None, + lock_fd: int | None = None, + lock_identity: _FileIdentity | None = None, + ) -> None: + if getattr(self, "_read_only", False): + raise RuntimeError( + "audit artifact store is read-only: it holds no publication lock" + ) + root = self._root if root is None else root + root_fd = self._root_fd if root_fd is None else root_fd + root_identity = ( + self._root_identity if root_identity is None else root_identity + ) + lock_fd = self._publication_lock_fd if lock_fd is None else lock_fd + lock_identity = ( + self._publication_lock_identity + if lock_identity is None + else lock_identity + ) + if getattr(self, "_closed", False) or root_fd < 0 or lock_fd < 0: + raise RuntimeError("audit artifact store is closed") + try: + if self._directory_identity(root) != root_identity: + raise RuntimeError("audit artifact root identity changed") + root_value = os.fstat(root_fd) + if (root_value.st_dev, root_value.st_ino) != root_identity: + raise RuntimeError("audit artifact root authority is invalid") + value = os.fstat(lock_fd) + if not lock_identity.matches(value): + raise RuntimeError("audit publication lock authority is invalid") + linked = os.stat( + _PUBLICATION_LOCK_NAME, + dir_fd=root_fd, + follow_symlinks=False, + ) + except OSError as exc: + raise RuntimeError("audit publication lock identity changed") from exc + if not lock_identity.matches(linked): + raise RuntimeError("audit publication lock identity changed") + + @contextmanager + def publication_order_guard(self) -> Iterator[None]: + """Serialize ordinal allocation/publication across threads/processes.""" + + if getattr(self, "_read_only", False): + raise RuntimeError( + "audit artifact store is read-only: publication is not available" + ) + with self._publication_order_guard(validate_on_exit=True): + yield + + @contextmanager + def _publication_order_guard( + self, + *, + validate_on_exit: bool, + ) -> Iterator[None]: + thread_id = threading.get_ident() + initial_identity = self._publication_lock_identity + _reject_cross_store_local_owner( + (initial_identity.device, initial_identity.inode), + thread_id=thread_id, + store_id=id(self), + ) + with self._publication_lifecycle_lock: + stable_identity = self._publication_lock_identity + _reject_cross_store_local_owner( + (stable_identity.device, stable_identity.inode), + thread_id=thread_id, + store_id=id(self), + ) + process_lock = self._publication_inode_thread_lock + with process_lock: + with self._publication_order_guard_locked( + validate_on_exit=validate_on_exit + ): + yield + + @contextmanager + def _publication_order_guard_locked( + self, + *, + validate_on_exit: bool, + ) -> Iterator[None]: + """Guard body with lifecycle and current inode thread locks held.""" + + thread_id = threading.get_ident() + if self._publication_guard_owner == thread_id: + self._publication_guard_depth += 1 + try: + self._validate_publication_lock_identity() + yield + if validate_on_exit: + self._validate_publication_lock_identity() + finally: + self._publication_guard_depth -= 1 + return + if self._publication_guard_owner is not None: + raise RuntimeError("audit publication guard ownership is inconsistent") + self._validate_publication_lock_identity() + lock_fd = self._publication_lock_fd + root = self._root + root_fd = self._root_fd + root_identity = self._root_identity + lock_identity = self._publication_lock_identity + local_key = (lock_identity.device, lock_identity.inode) + with _PUBLICATION_THREAD_LOCKS_GUARD: + local_owner = _PUBLICATION_LOCAL_OWNERS.get(local_key) + if local_owner is not None: + owner_thread, owner_store = local_owner + if owner_thread == thread_id and owner_store != id(self): + raise RuntimeError( + "audit publication guard is already held by another store " + "on this thread" + ) + raise RuntimeError("audit publication guard local ownership is inconsistent") + fcntl.flock(lock_fd, fcntl.LOCK_EX) + registered = False + try: + with _PUBLICATION_THREAD_LOCKS_GUARD: + local_owner = _PUBLICATION_LOCAL_OWNERS.get(local_key) + if local_owner is not None: + raise RuntimeError( + "audit publication guard local ownership is inconsistent" + ) + _PUBLICATION_LOCAL_OWNERS[local_key] = (thread_id, id(self)) + registered = True + self._publication_guard_owner = thread_id + self._publication_guard_depth = 1 + self._validate_publication_lock_identity( + root=root, + root_fd=root_fd, + root_identity=root_identity, + lock_fd=lock_fd, + lock_identity=lock_identity, + ) + yield + if validate_on_exit: + self._validate_publication_lock_identity( + root=root, + root_fd=root_fd, + root_identity=root_identity, + lock_fd=lock_fd, + lock_identity=lock_identity, + ) + finally: + self._publication_guard_depth = 0 + self._publication_guard_owner = None + if registered: + with _PUBLICATION_THREAD_LOCKS_GUARD: + if _PUBLICATION_LOCAL_OWNERS.get(local_key) == ( + thread_id, + id(self), + ): + del _PUBLICATION_LOCAL_OWNERS[local_key] + fcntl.flock(lock_fd, fcntl.LOCK_UN) + + def _require_publication_order_guard(self) -> None: + if ( + self._publication_guard_owner != threading.get_ident() + or self._publication_guard_depth <= 0 + ): + raise RuntimeError("audit publication order guard is required") + self._validate_publication_lock_identity() + + @contextmanager + def _prepared_publication_order_guard( + self, + *, + root: Path, + root_fd: int, + root_identity: tuple[int, int], + lock_fd: int, + lock_identity: _FileIdentity, + ) -> Iterator[None]: + current_identity = self._publication_lock_identity + if ( + lock_identity.device == current_identity.device + and lock_identity.inode == current_identity.inode + ): + self._validate_publication_lock_identity( + root=root, + root_fd=root_fd, + root_identity=root_identity, + lock_fd=lock_fd, + lock_identity=lock_identity, + ) + yield + self._validate_publication_lock_identity( + root=root, + root_fd=root_fd, + root_identity=root_identity, + lock_fd=lock_fd, + lock_identity=lock_identity, + ) + return + if root_identity == self._root_identity: + raise RuntimeError( + "audit publication lock changed within the current root" + ) + process_lock = _publication_thread_lock( + (lock_identity.device, lock_identity.inode) + ) + if not process_lock.acquire(blocking=False): + raise RuntimeError("new audit publication guard is busy") + flocked = False + try: + try: + fcntl.flock(lock_fd, fcntl.LOCK_EX | fcntl.LOCK_NB) + except BlockingIOError as exc: + raise RuntimeError("new audit publication guard is busy") from exc + flocked = True + self._validate_publication_lock_identity( + root=root, + root_fd=root_fd, + root_identity=root_identity, + lock_fd=lock_fd, + lock_identity=lock_identity, + ) + yield + self._validate_publication_lock_identity( + root=root, + root_fd=root_fd, + root_identity=root_identity, + lock_fd=lock_fd, + lock_identity=lock_identity, + ) + finally: + if flocked: + fcntl.flock(lock_fd, fcntl.LOCK_UN) + process_lock.release() + + def _validate_root_identity(self) -> None: + if getattr(self, "_closed", False) or self._root_fd < 0: + raise RuntimeError("audit artifact store is closed") + try: + current = self._directory_identity(self._root) + except (FileNotFoundError, OSError) as exc: + raise RuntimeError("audit artifact root identity changed") from exc + if current != self._root_identity: + raise RuntimeError("audit artifact root identity changed") + value = os.fstat(self._root_fd) + if (value.st_dev, value.st_ino) != self._root_identity: + raise RuntimeError("audit artifact root authority is invalid") + + def _validate_evidence_parent_identity(self) -> None: + if getattr(self, "_closed", False) or self._evidence_parent_fd < 0: + raise RuntimeError("audit artifact store is closed") + try: + current = self._directory_identity(self._evidence_path.parent) + except (FileNotFoundError, OSError) as exc: + raise RuntimeError("audit evidence parent identity changed") from exc + if current != self._evidence_parent_identity: + raise RuntimeError("audit evidence parent identity changed") + value = os.fstat(self._evidence_parent_fd) + if (value.st_dev, value.st_ino) != self._evidence_parent_identity: + raise RuntimeError("audit evidence parent authority is invalid") + + def _owned_parent_fd(self, path: Path) -> int | None: + path = Path(path).absolute() + try: + parent = path.parent.resolve(strict=True) + except OSError: + parent = path.parent + if parent == self._root: + return self._root_fd + if parent == self._evidence_path.parent: + return self._evidence_parent_fd + return None + + def duplicate_root_directory_fd(self) -> int: + with self._lock: + self._validate_root_identity() + return os.dup(self._root_fd) + + def _scan_root_names(self) -> list[str]: + """Enumerate the audit root through a fresh directory description. + + ``os.listdir(self._root_fd)`` reads through the long-lived pinned + descriptor. Some kernels and overlay filesystems serve a stale or + truncated enumeration through a directory description that has + already been read once and mutated since: on an aarch64 Docker host + a conflict snapshot written moments earlier was absent from the next + scan, so quarantine deduplication wrote a second copy and metrics + and retention saw an empty root. Reopening ``.`` relative to the + pinned descriptor yields a fresh description of the very same inode + -- authority still flows from ``_root_fd``, never from a pathname -- + and the identity is re-verified before and after the enumeration. + """ + self._validate_root_identity() + fd = os.open( + ".", + os.O_RDONLY | getattr(os, "O_DIRECTORY", 0), + dir_fd=self._root_fd, + ) + try: + value = os.fstat(fd) + if (value.st_dev, value.st_ino) != self._root_identity: + raise RuntimeError("audit artifact root authority is invalid") + names = os.listdir(fd) + finally: + os.close(fd) + self._validate_root_identity() + return names + + def _owned_lstat(self, path: Path) -> os.stat_result: + fd = self._owned_parent_fd(path) + if fd is None: + raise RuntimeError("audit stat target has no directory authority") + return os.stat(Path(path).name, dir_fd=fd, follow_symlinks=False) + + def _owned_open(self, path: Path, flags: int, mode: int = 0o777) -> int: + fd = self._owned_parent_fd(path) + if fd is None: + mutation_flags = ( + os.O_WRONLY + | os.O_RDWR + | os.O_CREAT + | os.O_EXCL + | getattr(os, "O_TRUNC", 0) + ) + if flags & mutation_flags: + raise RuntimeError("audit mutation target has no directory authority") + raise RuntimeError("audit read target has no directory authority") + return os.open(Path(path).name, flags, mode, dir_fd=fd) + + def _owned_unlink(self, path: Path) -> None: + fd = self._owned_parent_fd(path) + if fd is None: + raise RuntimeError("audit unlink target has no directory authority") + os.unlink(Path(path).name, dir_fd=fd) + + def _owned_replace(self, source: Path, target: Path) -> None: + source_fd = self._owned_parent_fd(source) + target_fd = self._owned_parent_fd(target) + if source_fd is None or target_fd is None: + raise RuntimeError("audit replace target has no directory authority") + os.replace( + Path(source).name, + Path(target).name, + src_dir_fd=source_fd, + dst_dir_fd=target_fd, + ) + + def _owned_link(self, source: Path, target: Path) -> None: + source_fd = self._owned_parent_fd(source) + target_fd = self._owned_parent_fd(target) + if source_fd is None or target_fd is None: + raise RuntimeError("audit link target has no directory authority") + os.link( + Path(source).name, + Path(target).name, + src_dir_fd=source_fd, + dst_dir_fd=target_fd, + follow_symlinks=False, + ) + + def _read_owned_regular_bytes( + self, + path: Path, + ) -> tuple[bytes, os.stat_result]: + parent_fd = self._owned_parent_fd(path) + if parent_fd is not None: + self._validate_owned_parent(path) + fd = self._owned_open( + path, + os.O_RDONLY | getattr(os, "O_NOFOLLOW", 0), + ) + result = self._read_regular_bytes_fd(fd) + if parent_fd is not None: + self._validate_owned_parent(path) + return result + + def _validate_owned_parent(self, path: Path) -> None: + path = Path(path).absolute() + try: + parent = path.parent.resolve(strict=True) + except OSError: + parent = path.parent + if parent == self._root: + self._validate_root_identity() + elif parent == self._evidence_path.parent: + self._validate_evidence_parent_identity() + else: + raise RuntimeError("audit target has no directory authority") + + def reconfigure( + self, + *, + root: Path | None = None, + evidence_path: Path | None = None, + live_bundle_retention: int | None = None, + candidate_retention_seconds: int | None = None, + share_segment_size: int | None = None, + verifier_timeout_seconds: float | None = None, + canonicalizer: Callable[[dict[str, Any]], bytes] | None = None, + ) -> None: + if ( + self._publication_guard_owner == threading.get_ident() + and self._publication_guard_depth > 0 + ): + raise RuntimeError( + "cannot reconfigure audit artifact store inside publication guard" + ) + retired_root_fd: int | None = None + retired_publication_lock_fd: int | None = None + with self._publication_order_guard(validate_on_exit=False): + ( + retired_root_fd, + retired_publication_lock_fd, + ) = self._reconfigure_under_publication_guard( + root=root, + evidence_path=evidence_path, + live_bundle_retention=live_bundle_retention, + candidate_retention_seconds=candidate_retention_seconds, + share_segment_size=share_segment_size, + verifier_timeout_seconds=verifier_timeout_seconds, + canonicalizer=canonicalizer, + ) + for retired_fd in (retired_publication_lock_fd, retired_root_fd): + if retired_fd is not None: + try: + os.close(retired_fd) + except OSError: + pass + + def _reconfigure_under_publication_guard( + self, + *, + root: Path | None, + evidence_path: Path | None, + live_bundle_retention: int | None, + candidate_retention_seconds: int | None, + share_segment_size: int | None, + verifier_timeout_seconds: float | None, + canonicalizer: Callable[[dict[str, Any]], bytes] | None, + ) -> tuple[int | None, int | None]: + self._require_publication_order_guard() + with self._lock: + if self._closed: + raise RuntimeError("audit artifact store is closed") + if root is not None and self._active_candidates: + raise RuntimeError( + "cannot reconfigure audit root while candidates are active" + ) + size = self._share_segment_size + if share_segment_size is not None: + size = int(share_segment_size) + if size < 0: + raise ValueError( + "audit share segment size must be non-negative" + ) + verifier_timeout = self._verifier_timeout_seconds + if verifier_timeout_seconds is not None: + verifier_timeout = float(verifier_timeout_seconds) + if verifier_timeout <= 0: + raise ValueError("verifier timeout must be positive") + new_root = self._root + new_root_fd: int | None = None + new_root_identity = self._root_identity + new_publication_lock_fd: int | None = None + new_publication_lock_identity = self._publication_lock_identity + new_evidence_path = self._evidence_path + new_evidence_parent_fd: int | None = None + new_evidence_parent_identity = self._evidence_parent_identity + try: + if root is not None: + candidate_root = Path(root).expanduser().absolute() + candidate_root.mkdir(parents=True, exist_ok=True) + new_root = candidate_root.resolve(strict=True) + new_root_fd, new_root_identity = ( + self._open_directory_authority(new_root) + ) + ( + new_publication_lock_fd, + new_publication_lock_identity, + ) = self._open_publication_lock_authority( + new_root, + new_root_fd, + new_root_identity, + ) + if evidence_path is not None: + candidate_evidence = Path(evidence_path).expanduser().absolute() + candidate_evidence.parent.mkdir(parents=True, exist_ok=True) + new_evidence_path = ( + candidate_evidence.parent.resolve(strict=True) + / candidate_evidence.name + ) + ( + new_evidence_parent_fd, + new_evidence_parent_identity, + ) = self._open_directory_authority( + new_evidence_path.parent + ) + except BaseException: + for prepared_fd in ( + new_publication_lock_fd, + new_root_fd, + new_evidence_parent_fd, + ): + if prepared_fd is not None: + os.close(prepared_fd) + raise + new_live_bundle_retention = ( + self._live_bundle_retention + if live_bundle_retention is None + else int(live_bundle_retention) + ) + new_candidate_retention_seconds = ( + self._candidate_retention_seconds + if candidate_retention_seconds is None + else int(candidate_retention_seconds) + ) + prepared_guard = ( + self._prepared_publication_order_guard( + root=new_root, + root_fd=new_root_fd, + root_identity=new_root_identity, + lock_fd=new_publication_lock_fd, + lock_identity=new_publication_lock_identity, + ) + if new_root_fd is not None + and new_publication_lock_fd is not None + else nullcontext() + ) + old_root = self._root + old_root_fd = self._root_fd + old_root_identity = self._root_identity + old_publication_lock_fd = self._publication_lock_fd + old_publication_lock_identity = self._publication_lock_identity + old_publication_inode_thread_lock = ( + self._publication_inode_thread_lock + ) + old_evidence_path = self._evidence_path + old_evidence_parent_fd = self._evidence_parent_fd + old_evidence_parent_identity = self._evidence_parent_identity + old_latest_evidence = self._latest_evidence + old_current_envelope = self._current_envelope + old_current_identity = self._current_identity + old_evidence_state = self._evidence_state + old_compatibility_override = self._compatibility_evidence_override + old_invalidated_legacy_identity = self._invalidated_legacy_identity + old_legacy_proof_token = self._legacy_proof_token + old_live_retention = self._live_bundle_retention + old_candidate_retention = self._candidate_retention_seconds + old_share_segment_size = self._share_segment_size + old_verifier_timeout = self._verifier_timeout_seconds + old_canonicalizer = self._canonicalizer + try: + with prepared_guard: + # Final old-authority validation is the linearization + # boundary. After this succeeds, only the prepared new + # authority may influence commit or rollback. + self._validate_publication_lock_identity( + root=old_root, + root_fd=old_root_fd, + root_identity=old_root_identity, + lock_fd=old_publication_lock_fd, + lock_identity=old_publication_lock_identity, + ) + if new_root_fd is not None: + self._root = new_root + self._root_fd = new_root_fd + self._root_identity = new_root_identity + assert new_publication_lock_fd is not None + self._publication_lock_fd = new_publication_lock_fd + self._publication_lock_identity = ( + new_publication_lock_identity + ) + self._publication_inode_thread_lock = _publication_thread_lock( + ( + new_publication_lock_identity.device, + new_publication_lock_identity.inode, + ) + ) + if new_evidence_parent_fd is not None: + self._evidence_path = new_evidence_path + self._evidence_parent_fd = new_evidence_parent_fd + self._evidence_parent_identity = ( + new_evidence_parent_identity + ) + if root is not None or evidence_path is not None: + self._latest_evidence = None + self._current_envelope = None + self._current_identity = None + self._evidence_state = "absent" + self._compatibility_evidence_override = False + self._invalidated_legacy_identity = None + self._legacy_proof_token = None + self._live_bundle_retention = new_live_bundle_retention + self._candidate_retention_seconds = ( + new_candidate_retention_seconds + ) + self._share_segment_size = size + self._verifier_timeout_seconds = verifier_timeout + if canonicalizer is not None: + self._canonicalizer = canonicalizer + if root is not None or evidence_path is not None: + # Readers must see either the complete old authority/cache + # or the complete new authority/cache. Both old and new + # process guards remain held across switch and reload. + self._reload_current_evidence_locked() + if new_evidence_parent_fd is not None: + try: + os.close(old_evidence_parent_fd) + except OSError: + pass + return ( + old_root_fd if new_root_fd is not None else None, + ( + old_publication_lock_fd + if new_publication_lock_fd is not None + else None + ), + ) + except BaseException: + self._root = old_root + self._root_fd = old_root_fd + self._root_identity = old_root_identity + self._publication_lock_fd = old_publication_lock_fd + self._publication_lock_identity = old_publication_lock_identity + self._publication_inode_thread_lock = ( + old_publication_inode_thread_lock + ) + self._evidence_path = old_evidence_path + self._evidence_parent_fd = old_evidence_parent_fd + self._evidence_parent_identity = old_evidence_parent_identity + self._latest_evidence = old_latest_evidence + self._current_envelope = old_current_envelope + self._current_identity = old_current_identity + self._evidence_state = old_evidence_state + self._compatibility_evidence_override = ( + old_compatibility_override + ) + self._invalidated_legacy_identity = ( + old_invalidated_legacy_identity + ) + self._legacy_proof_token = old_legacy_proof_token + self._live_bundle_retention = old_live_retention + self._candidate_retention_seconds = old_candidate_retention + self._share_segment_size = old_share_segment_size + self._verifier_timeout_seconds = old_verifier_timeout + self._canonicalizer = old_canonicalizer + for prepared_fd in ( + new_publication_lock_fd, + new_root_fd, + new_evidence_parent_fd, + ): + if prepared_fd is not None: + try: + os.close(prepared_fd) + except OSError: + pass + raise + + def publication_sequence_floor(self) -> int: + with self._lock: + if not self._directory_authority_is_current(): + return 0 + return ( + self._current_identity.sequence + if self._current_identity is not None + and self._evidence_state in {"valid", "legacy_proven"} + else 0 + ) + + def legacy_evidence_identity(self) -> AuditPublicationIdentity | None: + with self._lock: + if not self._directory_authority_is_current(): + return None + if ( + self._publication_guard_owner == threading.get_ident() + and self._publication_guard_depth > 0 + and not self._compatibility_evidence_override + ): + self._reload_current_evidence_locked() + if self._evidence_state not in { + "legacy", + "legacy_unproven", + } or self._current_identity is None: + return None + return self._current_identity + + def _directory_authority_is_current(self) -> bool: + try: + self._validate_root_identity() + self._validate_evidence_parent_identity() + return True + except RuntimeError: + # A transient pathname replacement revokes authority while it is + # present, but the pinned fd/cache can safely recover if the exact + # original directory inode is restored. Malformed evidence uses + # the separate sticky `invalid` state. + return False + + @staticmethod + def artifact_kind(name: str) -> str: + if _BODY_RE.fullmatch(name): + return "body" + segment = _SHARE_CONTENT_RE.fullmatch(name) or _SHARE_SLOT_RE.fullmatch(name) + if segment: + return ( + "share_segment" + if int(segment.group("first")) <= int(segment.group("last")) + else "other" + ) + if _CANONICAL_BUNDLE_RE.fullmatch(name): + # Classified as its own kind so retention never treats a canonical + # bundle as a live envelope or an expendable candidate. + return "canonical_bundle" + if ( + _CANDIDATE_RE.fullmatch(name) + or _LEGACY_CANDIDATE_RE.fullmatch(name) + or _LEGACY_HIDDEN_CANDIDATE_RE.fullmatch(name) + ): + return "candidate" + if _LIVE_RE.fullmatch(name): + return "live_bundle" + return "other" + + def metrics_snapshot(self) -> dict[str, dict[str, int] | int]: + metrics: dict[str, dict[str, int] | int] = { + kind: {"files": 0, "bytes": 0} + for kind in ( + "body", + "share_segment", + "canonical_bundle", + "live_bundle", + "candidate", + "other", + ) + } + metrics["scan_error"] = 0 + try: + self._validate_root_identity() + paths = [self._root / name for name in self._scan_root_names()] + except (OSError, RuntimeError): + metrics["scan_error"] = 1 + return metrics + for path in paths: + if path.name == _PUBLICATION_LOCK_NAME: + continue + try: + value = self._owned_lstat(path) + if not stat.S_ISREG(value.st_mode): + continue + except RuntimeError: + metrics["scan_error"] = 1 + break + except OSError: + metrics["scan_error"] = 1 + continue + kind = self.artifact_kind(path.name) + bucket = metrics[kind] + assert isinstance(bucket, dict) + bucket["files"] += 1 + bucket["bytes"] += value.st_size + try: + self._validate_root_identity() + except RuntimeError: + metrics["scan_error"] = 1 + return metrics + + def issue_candidate(self, *, block_hash: str) -> OwnedCandidateArtifact: + block_hash = _canonical_hex(block_hash, name="block_hash") + with self._lock: + self._validate_root_identity() + for _attempt in range(16): + token = uuid.uuid4().hex + path = self._root / ( + f".prism-live-audit-bundle-candidate-{block_hash}-{token}.json.tmp" + ) + try: + self._owned_lstat(path) + except FileNotFoundError: + candidate = OwnedCandidateArtifact( + path=path, + block_hash=block_hash, + token=token, + store_token=self._store_token, + ) + self._active_candidates[token] = (path, None) + return candidate + raise RuntimeError("could not allocate an absent audit candidate path") + + def _require_candidate( + self, + candidate: OwnedCandidateArtifact, + ) -> tuple[Path, _FileIdentity | None]: + self._validate_root_identity() + if candidate.store_token != self._store_token: + raise RuntimeError("candidate does not belong to this audit store") + if not _CANDIDATE_RE.fullmatch(candidate.path.name): + raise RuntimeError("candidate has an invalid owned filename") + if candidate.path.parent != self._root: + raise RuntimeError("candidate escapes the audit artifact root") + current = self._active_candidates.get(candidate.token) + if current is None or current[0] != candidate.path: + raise RuntimeError("candidate is no longer active") + return current + + def adopt_created_candidate( + self, + candidate: OwnedCandidateArtifact, + ) -> None: + raise RuntimeError( + "pathname adoption is forbidden; transfer the exact open compiler inode" + ) + + def adopt_compiler_candidate( + self, + candidate: OwnedCandidateArtifact, + *, + path: Path, + value: os.stat_result, + ) -> None: + """Transfer the exact still-open inode created exclusively by J1.""" + + with self._lock: + expected_path, identity = self._require_candidate(candidate) + if identity is not None or Path(path) != expected_path: + raise RuntimeError("compiler candidate transfer is invalid") + transferred = _FileIdentity.from_stat(value) + current = self._owned_lstat(expected_path) + if not transferred.matches(current): + raise RuntimeError("compiler candidate identity changed before transfer") + self._active_candidates[candidate.token] = (expected_path, transferred) + + def write_compatibility_candidate( + self, + candidate: OwnedCandidateArtifact, + bundle: Mapping[str, Any], + ) -> Path: + payload = _json_bytes(bundle) + with self._lock: + path, identity = self._require_candidate(candidate) + if identity is not None: + raise RuntimeError("candidate already exists") + created = False + try: + fd = self._owned_open( + path, + os.O_WRONLY | os.O_CREAT | os.O_EXCL | getattr(os, "O_NOFOLLOW", 0), + 0o600, + ) + created = True + opened = os.fstat(fd) + if not stat.S_ISREG(opened.st_mode): + raise RuntimeError("candidate is not a regular file") + candidate_identity = _FileIdentity.from_stat(opened) + with self._lock: + self._require_candidate(candidate) + self._active_candidates[candidate.token] = ( + path, + candidate_identity, + ) + with os.fdopen(fd, "wb") as handle: + try: + handle.write(payload) + handle.flush() + os.fsync(handle.fileno()) + finally: + finalized_identity = _FileIdentity.from_stat( + os.fstat(handle.fileno()) + ) + with self._lock: + current = self._active_candidates.get(candidate.token) + if current == (path, candidate_identity): + self._active_candidates[candidate.token] = ( + path, + finalized_identity, + ) + self._validate_root_identity() + return path + except BaseException: + if created: + self._unlink_candidate_if_same(candidate, allow_unadopted=True) + else: + self.release_candidate(candidate) + raise + + def release_candidate(self, candidate: OwnedCandidateArtifact) -> None: + """Release an uncreated reservation without acquiring delete authority.""" + + with self._lock: + self._require_candidate(candidate) + self._active_candidates.pop(candidate.token, None) + + def _unlink_candidate_if_same( + self, + candidate: OwnedCandidateArtifact, + *, + allow_unadopted: bool = False, + ) -> None: + with self._lock: + self._validate_root_identity() + current = self._active_candidates.get(candidate.token) + if current is None or current[0] != candidate.path: + return + expected = current[1] + try: + value = self._owned_lstat(candidate.path) + except FileNotFoundError: + self._active_candidates.pop(candidate.token, None) + return + if not stat.S_ISREG(value.st_mode): + self._active_candidates.pop(candidate.token, None) + return + if expected is not None and not expected.matches(value): + self._active_candidates.pop(candidate.token, None) + return + if expected is None and not allow_unadopted: + return + self._remove_identity_safe( + candidate.path, + _FileIdentity.from_stat(value), + ) + self._active_candidates.pop(candidate.token, None) + + def discard_candidate(self, candidate: OwnedCandidateArtifact) -> None: + with self._lock: + current = self._active_candidates.get(candidate.token) + if current is not None and current[1] is None: + # Reservation alone never grants deletion authority. J1 must + # return and transfer the exact created inode first. + self._active_candidates.pop(candidate.token, None) + return + self._unlink_candidate_if_same(candidate) + + @staticmethod + def verified_canonical_bundle_path( + candidate_bundle_path: Path, + report: Mapping[str, Any], + ) -> Path | None: + expected = str(report["audit_bundle_sha256_hex"]).lower() + payload, _value = AuditArtifactStore.read_regular_bytes( + Path(candidate_bundle_path) + ) + return ( + Path(candidate_bundle_path) + if _sha256_bytes(payload) == expected + else None + ) + + def verify_candidate( + self, + candidate: OwnedCandidateArtifact, + *, + coinbase_tx_hex: str, + expected_coinbase_value_sats: int, + trusted_writer_public_key_hex: str, + trust_source: str = "configured", + expected_block_height: int | None = None, + verifier: Callable[..., dict[str, Any]] | None = None, + ) -> VerifiedAuditBundle: + key = _canonical_hex( + trusted_writer_public_key_hex, + name="ledger writer public key", + ) + coinbase_tx_hex = _canonical_hex_bytes( + coinbase_tx_hex, + name="coinbase_tx_hex", + ) + with self._lock: + path, identity = self._require_candidate(candidate) + if identity is None: + raise RuntimeError( + "candidate compiler inode was not transferred before verification" + ) + before = self._owned_lstat(path) + if not identity.matches(before): + raise RuntimeError("candidate identity changed before verification") + before_bytes, before_descriptor_stat = self._read_owned_regular_bytes(path) + if not identity.matches(before_descriptor_stat): + raise RuntimeError("candidate identity changed before verification") + literal_before = _sha256_bytes(before_bytes) + verify = verifier or self._verifier or self.verify_bundle + snapshot_fd, verification_path = self._open_verification_snapshot( + before_bytes + ) + try: + report = verify( + verification_path, + coinbase_tx_hex, + key, + expected_coinbase_value_sats=expected_coinbase_value_sats, + expected_block_height=expected_block_height, + ) + finally: + os.close(snapshot_fd) + after_bytes, after = self._read_owned_regular_bytes(path) + literal_after = _sha256_bytes(after_bytes) + if ( + not identity.matches(after) + or before.st_size != after.st_size + or before.st_mtime_ns != after.st_mtime_ns + or not hmac.compare_digest(literal_before, literal_after) + ): + raise RuntimeError("candidate changed during verification") + normalized = self._validate_verifier_report( + report, + coinbase_tx_hex=coinbase_tx_hex, + expected_coinbase_value_sats=expected_coinbase_value_sats, + expected_block_height=expected_block_height, + ) + expected_digest = str(normalized["audit_bundle_sha256_hex"]) + verification_identity = self.build_verification_identity( + trust_source=trust_source, + trusted_writer_public_key_hex=key, + literal_sha256=literal_after, + literal_byte_len=after.st_size, + report=normalized, + ) + verified = VerifiedAuditBundle( + candidate=candidate, + report=copy.deepcopy(normalized), + literal_sha256=literal_after, + byte_length=after.st_size, + device=after.st_dev, + inode=after.st_ino, + mtime_ns=after.st_mtime_ns, + canonical_copy_eligible=hmac.compare_digest( + literal_after, + expected_digest, + ), + verification_identity=verification_identity, + ) + self._validate_root_identity() + return verified + + def require_current_verified_candidate( + self, + verified: VerifiedAuditBundle, + candidate: OwnedCandidateArtifact, + ) -> None: + """Reject reuse of a successful result across ephemeral candidates.""" + + with self._lock: + path, identity = self._require_candidate(candidate) + if verified.candidate != candidate or identity is None: + raise RuntimeError("verified audit result belongs to another candidate") + if ( + identity.device != verified.device + or identity.inode != verified.inode + or path != candidate.path + ): + raise RuntimeError("verified audit result candidate identity changed") + + @staticmethod + def build_verification_identity( + *, + trust_source: str, + trusted_writer_public_key_hex: str, + literal_sha256: str, + literal_byte_len: int, + report: Mapping[str, Any], + ) -> dict[str, Any]: + if trust_source not in {"configured", "embedded_test_only"}: + raise RuntimeError("audit verifier trust source is invalid") + normalized_report = AuditArtifactStore._normalize_report_identity(report) + byte_len = literal_byte_len + if isinstance(byte_len, bool) or not isinstance(byte_len, int) or byte_len < 0: + raise RuntimeError("audit verifier literal byte length is invalid") + base = { + "schema": VERIFICATION_IDENTITY_SCHEMA, + "trust_source": trust_source, + "ledger_writer_public_key_hex": _canonical_hex( + trusted_writer_public_key_hex, + name="ledger writer public key", + ), + "literal_sha256_hex": _canonical_hex( + literal_sha256, + name="verified literal sha256", + ), + "literal_byte_len": byte_len, + "report": normalized_report, + } + return { + **base, + "identity_sha256_hex": _sha256_bytes( + _json_bytes(base, sort_keys=True) + ), + } + + @staticmethod + def _normalize_verification_identity( + value: object, + *, + report: Mapping[str, Any], + ) -> dict[str, Any]: + if not isinstance(value, dict): + raise RuntimeError("audit verification identity is required") + normalized = AuditArtifactStore.build_verification_identity( + trust_source=str(value.get("trust_source") or ""), + trusted_writer_public_key_hex=str( + value.get("ledger_writer_public_key_hex") or "" + ), + literal_sha256=str(value.get("literal_sha256_hex") or ""), + literal_byte_len=value.get("literal_byte_len"), + report=report, + ) + if value != normalized: + raise RuntimeError("audit verification identity mismatch") + return normalized + + @staticmethod + def _legacy_verification_marker( + *, + identity: AuditPublicationIdentity, + report: Mapping[str, Any], + ) -> dict[str, Any]: + normalized_report = AuditArtifactStore._normalize_report_identity(report) + return { + "schema": LEGACY_VERIFICATION_UNAVAILABLE_SCHEMA, + "reason": "evidence-predates-deterministic-verification-identity", + "block_hash": identity.block_hash, + "block_height": identity.block_height, + "audit_publication_sequence": identity.sequence, + "report_identity_sha256_hex": _sha256_bytes( + _json_bytes(normalized_report, sort_keys=True) + ), + } + + @staticmethod + def _normalize_legacy_verification_marker( + value: object, + *, + identity: AuditPublicationIdentity, + report: Mapping[str, Any], + ) -> dict[str, Any]: + if not isinstance(value, dict): + raise RuntimeError("legacy audit verification marker is required") + expected = AuditArtifactStore._legacy_verification_marker( + identity=identity, + report=report, + ) + if value != expected: + raise RuntimeError("legacy audit verification marker mismatch") + return expected + + @staticmethod + def _loaded_evidence_state( + payload: Mapping[str, Any], + identity: AuditPublicationIdentity, + ) -> str: + if identity.sequence == 0: + return "legacy" + verification = payload.get("audit_verification_identity") + if ( + isinstance(verification, dict) + and verification.get("schema") + == LEGACY_VERIFICATION_UNAVAILABLE_SCHEMA + ): + # The on-disk ledger proof is a hint only. The coordinator must + # revalidate exact active ledger state on every process start. + return "legacy_unproven" + return "valid" + + def _open_verification_snapshot(self, payload: bytes) -> tuple[int, Path]: + path = self._root / f".prism-audit-verification-{uuid.uuid4().hex}.tmp" + identity: _FileIdentity | None = None + read_fd: int | None = None + try: + fd = self._owned_open( + path, + os.O_WRONLY | os.O_CREAT | os.O_EXCL | getattr(os, "O_NOFOLLOW", 0), + 0o600, + ) + identity = _FileIdentity.from_stat(os.fstat(fd)) + with os.fdopen(fd, "wb") as handle: + try: + handle.write(payload) + handle.flush() + os.fsync(handle.fileno()) + finally: + identity = _FileIdentity.from_stat(os.fstat(handle.fileno())) + read_fd = self._owned_open( + path, + os.O_RDONLY | getattr(os, "O_NOFOLLOW", 0), + ) + if not identity.matches(os.fstat(read_fd)): + raise RuntimeError("verification snapshot identity changed") + if not self._remove_identity_safe(path, identity): + raise RuntimeError("verification snapshot path was replaced") + self._fsync_directory(self._root) + return read_fd, Path(f"/dev/fd/{read_fd}") + except BaseException: + if read_fd is not None: + os.close(read_fd) + if identity is not None: + self._remove_identity_safe(path, identity) + raise + + @staticmethod + def _validate_verifier_report( + report: object, + *, + coinbase_tx_hex: str, + expected_coinbase_value_sats: int, + expected_block_height: int | None, + ) -> dict[str, Any]: + if not isinstance(report, dict): + raise RuntimeError("audit verifier returned a non-object report") + result = AuditArtifactStore._normalize_report_identity(report) + if not hmac.compare_digest(result["coinbase_tx_hex"], coinbase_tx_hex): + raise RuntimeError("audit verifier coinbase does not match submitted coinbase") + if int(result["coinbase_value_sats"]) != int(expected_coinbase_value_sats): + raise RuntimeError("audit verifier coinbase value does not match expected value") + if ( + expected_block_height is not None + and result["block_height"] != int(expected_block_height) + ): + raise RuntimeError("audit verifier block height does not match expected height") + return result + + @staticmethod + def _normalize_report_identity(report: Mapping[str, Any]) -> dict[str, Any]: + result = dict(report) + if result.get("schema") != VERIFICATION_REPORT_SCHEMA: + raise RuntimeError("audit verifier report schema is invalid") + block_height = result.get("block_height") + if isinstance(block_height, bool) or not isinstance(block_height, int): + raise RuntimeError("audit verifier block height is invalid") + if block_height < 0: + raise RuntimeError("audit verifier block height is invalid") + result["audit_bundle_sha256_hex"] = _canonical_hex( + result.get("audit_bundle_sha256_hex"), + name="audit bundle sha256", + ) + result["coinbase_tx_hex"] = _canonical_hex_bytes( + result.get("coinbase_tx_hex"), + name="report coinbase_tx_hex", + ) + report_value = result.get("coinbase_value_sats") + if isinstance(report_value, bool) or not isinstance(report_value, int): + raise RuntimeError("audit verifier coinbase value is required") + if result["coinbase_value_sats"] < 0: + raise RuntimeError("audit verifier coinbase value is invalid") + for key in ( + "reward_manifest_sha256_hex", + "payout_policy_manifest_sha256_hex", + "prism_audit_commitment_leaf_hex", + "audit_commitment_root_hex", + "coinbase_txid", + "coinbase_wtxid", + "coinbase_manifest_sha256_hex", + ): + result[key] = _canonical_hex(result.get(key), name=key) + for key in ( + "min_output_sats", + "onchain_output_count", + "accrued_account_count", + ): + value = result.get(key) + if isinstance(value, bool) or not isinstance(value, int) or value < 0: + raise RuntimeError(f"audit verifier {key} is invalid") + return result + + def verify_bundle( + self, + bundle_path: Path, + coinbase_tx_hex: str, + ledger_writer_public_key_hex: str, + *, + expected_coinbase_value_sats: int, + expected_block_height: int | None = None, + ) -> dict[str, Any]: + del expected_block_height + inherited_fds: tuple[int, ...] = () + match = re.fullmatch(r"/dev/fd/(?P[0-9]+)", str(bundle_path)) + if match is not None: + inherited_fds = (int(match.group("fd")),) + process = subprocess.Popen( + prism_tool_command("qbit-prism-audit-verify") + + [ + str(bundle_path), + "--coinbase-tx-hex", + coinbase_tx_hex, + "--ledger-writer-public-key-hex", + ledger_writer_public_key_hex, + "--expected-coinbase-value-sats", + str(expected_coinbase_value_sats), + ], + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + pass_fds=inherited_fds, + start_new_session=True, + ) + outputs = {"stdout": bytearray(), "stderr": bytearray()} + selector = selectors.DefaultSelector() + assert process.stdout is not None + assert process.stderr is not None + for name, stream in ( + ("stdout", process.stdout), + ("stderr", process.stderr), + ): + os.set_blocking(stream.fileno(), False) + selector.register(stream, selectors.EVENT_READ, name) + deadline = time.monotonic() + self._verifier_timeout_seconds + try: + while selector.get_map() or process.poll() is None: + remaining_seconds = deadline - time.monotonic() + if remaining_seconds <= 0: + self._kill_verifier_group(process) + process.wait() + raise RuntimeError("qbit-prism-audit-verify timed out") + events = selector.select(min(0.05, remaining_seconds)) + for key, _mask in events: + try: + chunk = os.read(key.fileobj.fileno(), 64 * 1024) + except (BlockingIOError, InterruptedError): + continue + if not chunk: + selector.unregister(key.fileobj) + continue + output = outputs[str(key.data)] + remaining = MAX_VERIFIER_OUTPUT_BYTES + 1 - len(output) + if remaining > 0: + output.extend(chunk[:remaining]) + if len(output) > MAX_VERIFIER_OUTPUT_BYTES: + self._kill_verifier_group(process) + process.wait() + raise RuntimeError( + "qbit-prism-audit-verify output exceeded limit" + ) + process.wait() + except BaseException: + if process.poll() is None: + self._kill_verifier_group(process) + process.wait() + raise + finally: + selector.close() + process.stdout.close() + process.stderr.close() + stdout = bytes(outputs["stdout"]) + stderr = bytes(outputs["stderr"]) + if process.returncode != 0: + raise RuntimeError( + "qbit-prism-audit-verify failed: " + + stderr.decode(errors="replace").strip() + ) + try: + report = json.loads(stdout) + except (UnicodeDecodeError, json.JSONDecodeError) as exc: + raise RuntimeError("qbit-prism-audit-verify returned invalid JSON") from exc + if not isinstance(report, dict): + raise RuntimeError("qbit-prism-audit-verify returned a non-object report") + return report + + @staticmethod + def _kill_verifier_group(process: subprocess.Popen[bytes]) -> None: + try: + os.killpg(process.pid, signal.SIGKILL) + except (ProcessLookupError, PermissionError, AttributeError): + try: + process.kill() + except (ProcessLookupError, PermissionError): + pass + + @staticmethod + def trusted_writer_key( + configured_key: str | None, + bundle: Mapping[str, Any], + *, + allow_embedded_test_key: bool, + ) -> str: + if configured_key is not None: + return _canonical_hex( + configured_key, + name="configured ledger writer public key", + ) + if not allow_embedded_test_key: + raise RuntimeError("configured ledger writer public key is required") + try: + embedded = bundle["ledger_window_attestation"]["signature"][ + "public_key_hex" + ] + except (KeyError, TypeError) as exc: + raise RuntimeError("audit bundle has no embedded test writer key") from exc + return _canonical_hex(embedded, name="bundle ledger public key") + + def live_envelope_path(self, *, block_height: int, block_hash: str) -> Path: + block_hash = _canonical_hex(block_hash, name="block_hash") + if isinstance(block_height, bool) or not isinstance(block_height, int): + raise ValueError("block_height must be an integer") + if block_height < 0: + raise ValueError("block_height must be non-negative") + return self._root / ( + f"prism-live-audit-bundle-{block_height}-{block_hash}.json" + ) + + def body_path(self, block_hash: str, audit_bundle_sha256: str) -> Path: + block_hash = _canonical_hex(block_hash, name="block_hash") + digest = _canonical_hex( + audit_bundle_sha256, + name="audit_bundle_sha256", + ) + return self._root / f"prism-audit-bundle-body-{block_hash}-{digest}.json" + + def canonical_bundle_path( + self, + block_hash: str, + audit_bundle_sha256: str, + ) -> Path: + """Owned path for the compressed canonical bundle of one publication. + + The digest in the name is the public SHA-256 over the *uncompressed* + canonical JSON, so the artifact is content-addressed by the identity + readers already advertise. + """ + + block_hash = _canonical_hex(block_hash, name="block_hash") + digest = _canonical_hex( + audit_bundle_sha256, + name="audit_bundle_sha256", + ) + self._validate_root_identity() + return self._root / ( + f"prism-audit-bundle-canonical-{block_hash}-{digest}.json.gz" + ) + + def resolve_owned_path(self, body_uri: object) -> Path: + self._validate_root_identity() + raw = Path(str(body_uri)).expanduser() + if not raw.is_absolute() and ( + len(raw.parts) != 1 or raw.name in {".", ".."} + ): + raise RuntimeError( + f"audit bundle body path escapes audit body store: {body_uri}" + ) + path = raw if raw.is_absolute() else self._root / raw + path = path.absolute() + try: + path = path.parent.resolve(strict=True) / path.name + except OSError as exc: + raise RuntimeError( + f"audit bundle body path is not resolvable: {body_uri}" + ) from exc + if path.parent != self._root: + raise RuntimeError( + f"audit bundle body path escapes audit body store: {body_uri}" + ) + if self.artifact_kind(path.name) not in {"body", "share_segment"}: + raise RuntimeError(f"audit bundle body path is not owned: {body_uri}") + self._validate_root_identity() + return path + + def publish_success( + self, + *, + identity: AuditPublicationIdentity, + publication_floor_sequence: int, + report: Mapping[str, Any], + persistence: Mapping[str, Any], + evidence: Mapping[str, Any], + verification_identity: Mapping[str, Any], + created_at: str, + restore_superseded_envelope: bool = False, + ) -> AuditPublication: + self._require_publication_order_guard() + block_hash = _canonical_hex(identity.block_hash, name="block_hash") + if block_hash != identity.block_hash: + raise ValueError("publication identity block_hash must be canonical") + if identity.sequence <= 0 or identity.block_height < 0: + raise ValueError("publication identity is invalid") + if ( + isinstance(publication_floor_sequence, bool) + or not isinstance(publication_floor_sequence, int) + or publication_floor_sequence < 0 + ): + raise ValueError("publication floor sequence must be a non-negative integer") + if identity.sequence > publication_floor_sequence: + raise RuntimeError( + "audit publication sequence exceeds the durable ledger floor" + ) + envelope_path = self.live_envelope_path( + block_height=identity.block_height, + block_hash=block_hash, + ) + with self._lock: + self._validate_root_identity() + self._validate_evidence_parent_identity() + if not self._compatibility_evidence_override: + invalidated_legacy = self._invalidated_legacy_identity + self._reload_current_evidence_locked() + if ( + invalidated_legacy is not None + and self._evidence_state in {"legacy", "legacy_unproven"} + and self._current_identity == invalidated_legacy + ): + # Explicit invalidation remains repairable only while disk + # still names the same unproven legacy publication. A peer + # valid publication always wins this reconciliation. + self._latest_evidence = None + self._current_envelope = None + self._current_identity = None + self._evidence_state = "invalid" + else: + self._invalidated_legacy_identity = None + current = self._current_identity + if self._evidence_state in {"legacy", "legacy_unproven"}: + raise RuntimeError( + "current audit evidence lacks a validated publication identity" + ) + if current is not None: + exact = current == identity + if ( + self._evidence_state == "legacy_proven" + and identity.sequence <= current.sequence + ): + raise RuntimeError( + "legacy audit evidence is never exact-replay-equivalent" + ) + if identity.sequence == current.sequence and not exact: + raise RuntimeError("audit publication identity conflict") + if identity.sequence < current.sequence: + # A superseded ordinal never becomes the current evidence + # reference, but the live envelope is this block's only + # published evidence pointer and lives on its own + # height+hash path. Two distinct blocks landing inside one + # confirm->publish window arrive here in ordinal order, so + # skipping the write erased the earlier block's public + # audit trail permanently and silently. + # + # Only the caller knows whether it is the live writer for + # this landing. A process whose writer lease was taken over + # must stay fenced out of the shared audit root entirely: + # its in-memory report may be stale, and publishing a stale + # report as a block's public evidence is worse than leaving + # the envelope missing. Such callers never opt in. + if not restore_superseded_envelope: + return AuditPublication( + identity=identity, + envelope_path=envelope_path, + evidence=copy.deepcopy(dict(evidence)), + published=False, + ) + return self._write_superseded_live_envelope_locked( + identity=identity, + envelope_path=envelope_path, + report=report, + persistence=persistence, + evidence=evidence, + created_at=created_at, + ) + if exact and self._latest_evidence is not None: + expected_envelope = self._build_live_envelope( + identity=identity, + report=report, + persistence=persistence, + created_at=created_at, + ) + replay_annotations = copy.deepcopy(dict(evidence)) + # These global counters are observational annotations, not + # immutable block identity. Reuse the originally durable + # values so an outbox replay after more shares arrive does + # not strand an otherwise exact publication. + for annotation in ( + "accepted_share_count", + "distinct_miner_count", + ): + if annotation in self._latest_evidence: + replay_annotations[annotation] = ( + self._latest_evidence[annotation] + ) + replay_confirmation = replay_annotations.get("confirmation") + current_confirmation = self._latest_evidence.get("confirmation") + if ( + isinstance(replay_confirmation, dict) + and isinstance(current_confirmation, dict) + and "confirmed_count" in current_confirmation + and "confirmed_count" in replay_confirmation + ): + # The confirm disposition says how *this* call landed, + # not what the block is. The flip that first published + # this evidence reported a fresh confirmation (1); + # every later replay of the same durable row reports + # the idempotent disposition (2) -- which is the point + # of that split (issue #61). Reuse the originally + # durable value so an exact replay is not stranded as + # a payload conflict. The publication ordinal inside + # the same confirmation is block identity and is + # compared unchanged: a replay that returned a + # different ordinal still conflicts. + replay_confirmation["confirmed_count"] = ( + current_confirmation["confirmed_count"] + ) + replay_persistence = copy.deepcopy(dict(persistence)) + current_persistence = self._latest_evidence.get( + "persistence" + ) + if isinstance(current_persistence, dict) and ( + "share_count" in current_persistence + ): + # persist_accepted_block reports the current global + # accepted-share count. It is observational and may + # advance between a committed block and outbox replay. + replay_persistence["share_count"] = ( + current_persistence["share_count"] + ) + replay_evidence = self._normalized_durable_evidence( + identity=identity, + envelope_path=envelope_path, + report=report, + persistence=replay_persistence, + evidence=replay_annotations, + verification_identity=verification_identity, + ) + if self._latest_evidence != replay_evidence: + raise RuntimeError( + "exact audit publication replay payload conflict" + ) + envelope_present = True + evidence_present = True + try: + disk_envelope_bytes, _value = ( + self._read_owned_regular_bytes(envelope_path) + ) + except FileNotFoundError: + envelope_present = False + else: + self._match_existing_live_envelope( + disk_envelope_bytes, + expected_envelope, + ) + try: + disk_evidence_bytes, _value = ( + self._read_owned_regular_bytes(self._evidence_path) + ) + except FileNotFoundError: + evidence_present = False + else: + try: + disk_evidence = json.loads(disk_evidence_bytes) + except (UnicodeDecodeError, json.JSONDecodeError): + # The evidence path is mutable publication state, + # not operator-owned preservation authority. A + # ledger-proven exact replay may repair malformed + # bytes while retaining the exact immutable + # envelope checked above. + evidence_present = False + else: + if disk_evidence != replay_evidence: + try: + self._validate_evidence(disk_evidence) + except ( + OSError, + RuntimeError, + TypeError, + ValueError, + ): + evidence_present = False + else: + raise RuntimeError( + "existing live audit evidence conflicts" + ) + if envelope_present and evidence_present: + self._fsync_directory(envelope_path.parent) + self._fsync_directory(self._evidence_path.parent) + return AuditPublication( + identity=identity, + envelope_path=envelope_path, + evidence=copy.deepcopy(self._latest_evidence), + published=False, + ) + if identity.sequence != publication_floor_sequence: + raise RuntimeError( + "audit publication repair is behind the durable ledger floor" + ) + # A missing exact entry is repaired by the same atomic + # primitives as a first publication. Preserve the durable + # observational annotations while falling through. + persistence = replay_persistence + evidence = replay_annotations + if identity.sequence != publication_floor_sequence: + raise RuntimeError( + "audit publication is behind the durable ledger floor" + ) + envelope = self._build_live_envelope( + identity=identity, + report=report, + persistence=persistence, + created_at=created_at, + ) + try: + old_envelope, _old_envelope_stat = self._read_owned_regular_bytes( + envelope_path + ) + except FileNotFoundError: + old_envelope = None + reuse_envelope = False + if old_envelope is not None: + self._match_existing_live_envelope(old_envelope, envelope) + reuse_envelope = True + durable_evidence = self._normalized_durable_evidence( + identity=identity, + envelope_path=envelope_path, + report=report, + persistence=persistence, + evidence=evidence, + verification_identity=verification_identity, + ) + try: + old_evidence, _old_evidence_stat = ( + self._read_owned_regular_bytes(self._evidence_path) + ) + except FileNotFoundError: + old_evidence = None + reuse_evidence = False + if old_evidence is not None: + try: + existing_evidence = json.loads(old_evidence) + except (UnicodeDecodeError, json.JSONDecodeError): + # Invalid bytes at the mutable evidence path may be + # repaired by this ledger-proven publication. + pass + else: + if existing_evidence == durable_evidence: + reuse_evidence = True + else: + try: + _payload, _path, existing_identity = ( + self._validate_evidence(existing_evidence) + ) + except (OSError, RuntimeError, TypeError, ValueError): + # Canonical JSON can still be structurally or + # cryptographically invalid. It has the same + # repair policy as malformed bytes. + pass + else: + if identity.sequence <= existing_identity.sequence: + raise RuntimeError( + "existing live audit evidence conflicts" + ) + if not reuse_envelope: + published_envelope_identity = self._write_mutable_json( + envelope_path, + envelope, + sort_keys=True, + ) + else: + published_envelope_identity = None + self._fsync_directory(envelope_path.parent) + published_evidence_identity: _FileIdentity | None = None + try: + if reuse_evidence: + self._fsync_directory(self._evidence_path.parent) + else: + published_evidence_identity = self._write_mutable_json( + self._evidence_path, + durable_evidence, + sort_keys=True, + ) + self._validate_root_identity() + self._validate_evidence_parent_identity() + self._require_publication_order_guard() + except BaseException: + if published_evidence_identity is not None: + try: + self._rollback_mutable_replace( + self._evidence_path, + old_evidence, + published_evidence_identity, + allow_detached_authority=True, + ) + except BaseException: + pass + if reuse_envelope: + pass + elif old_envelope is None: + assert published_envelope_identity is not None + self._remove_identity_safe( + envelope_path, + published_envelope_identity, + allow_detached_authority=True, + ) + self._fsync_pinned_directory(envelope_path.parent) + else: + assert published_envelope_identity is not None + self._rollback_mutable_replace( + envelope_path, + old_envelope, + published_envelope_identity, + allow_detached_authority=True, + ) + raise + self._require_publication_order_guard() + self._latest_evidence = copy.deepcopy(durable_evidence) + self._current_envelope = envelope_path + self._current_identity = identity + self._evidence_state = "valid" + self._compatibility_evidence_override = False + self._invalidated_legacy_identity = None + self._legacy_proof_token = None + try: + self.prune_best_effort() + except Exception: + # Retention is maintenance after the evidence reference is durable. + pass + return AuditPublication( + identity=identity, + envelope_path=envelope_path, + evidence=copy.deepcopy(durable_evidence), + published=True, + ) + + def _write_superseded_live_envelope_locked( + self, + *, + identity: AuditPublicationIdentity, + envelope_path: Path, + report: Mapping[str, Any], + persistence: Mapping[str, Any], + evidence: Mapping[str, Any], + created_at: str, + ) -> AuditPublication: + """Persist the live envelope for a ledger-proven superseded ordinal. + + ``publish_success`` has already fenced ``identity.sequence`` at or below + the durable publication floor, so this ordinal was durably allocated by + the ledger and is unique to this block. The envelope path is keyed by + height and hash, so writing it cannot disturb the current evidence + reference, which must keep naming the highest published sequence. + """ + self._require_publication_order_guard() + envelope = self._build_live_envelope( + identity=identity, + report=report, + persistence=persistence, + created_at=created_at, + ) + try: + existing_envelope_bytes, _existing_stat = ( + self._read_owned_regular_bytes(envelope_path) + ) + except FileNotFoundError: + pass + else: + # A same-hash stale replay always finds its own envelope already + # present. This path only restores missing evidence -- but + # accepting arbitrary present bytes would silently retain a + # truncated or corrupt envelope forever, so present bytes are held + # to the main publish path's immutable-content standard, and a + # match is made durable exactly like a reused envelope there. + self._match_existing_live_envelope(existing_envelope_bytes, envelope) + self._fsync_directory(envelope_path.parent) + return AuditPublication( + identity=identity, + envelope_path=envelope_path, + evidence=copy.deepcopy(dict(evidence)), + published=False, + ) + written_identity = self._write_mutable_json( + envelope_path, + envelope, + sort_keys=True, + ) + try: + self._validate_root_identity() + self._validate_evidence_parent_identity() + self._require_publication_order_guard() + except BaseException: + try: + self._remove_identity_safe( + envelope_path, + written_identity, + allow_detached_authority=True, + ) + except BaseException: + pass + try: + self._fsync_pinned_directory(envelope_path.parent) + except BaseException: + pass + raise + return AuditPublication( + identity=identity, + envelope_path=envelope_path, + evidence=copy.deepcopy(dict(evidence)), + published=False, + superseded_envelope_written=True, + ) + + def _match_existing_live_envelope( + self, + existing_bytes: bytes, + envelope: Mapping[str, Any], + ) -> None: + """Hold present envelope bytes to the immutable-content standard. + + ``created_at`` is the only field a lawful republication may vary; any + other divergence, and unparseable or non-object bytes, are conflicts + because a live envelope is immutable once published. + """ + try: + existing_envelope = json.loads(existing_bytes) + except (UnicodeDecodeError, json.JSONDecodeError) as exc: + raise RuntimeError("existing live audit envelope is invalid") from exc + if not isinstance(existing_envelope, dict): + raise RuntimeError("existing live audit envelope is invalid") + immutable_existing = { + key: value + for key, value in existing_envelope.items() + if key != "created_at" + } + immutable_new = { + key: value for key, value in envelope.items() if key != "created_at" + } + if immutable_existing != immutable_new: + raise RuntimeError("existing live audit envelope conflicts") + + def _build_live_envelope( + self, + *, + identity: AuditPublicationIdentity, + report: Mapping[str, Any], + persistence: Mapping[str, Any], + created_at: str, + ) -> dict[str, Any]: + normalized_report = self._normalize_report_identity(report) + if normalized_report["block_height"] != identity.block_height: + raise RuntimeError("audit report block height does not match publication") + report_digest = _canonical_hex( + normalized_report.get("audit_bundle_sha256_hex"), + name="audit report digest", + ) + persistence_digest = _canonical_hex( + persistence.get("audit_bundle_sha256"), + name="persistence digest", + ) + if not hmac.compare_digest(report_digest, persistence_digest): + raise RuntimeError("audit report and persistence digests differ") + body_uri = str(persistence.get("body_uri") or "") + if body_uri: + body_path = self.resolve_owned_path(body_uri) + expected_body_path = self.body_path(identity.block_hash, persistence_digest) + if body_path != expected_body_path: + raise RuntimeError( + "persistence body URI does not match publication identity" + ) + body_uri = str(body_path) + return { + "schema": LIVE_ENVELOPE_SCHEMA, + "block_hash": identity.block_hash, + "block_height": identity.block_height, + "audit_bundle_sha256": persistence_digest, + "body_uri": body_uri, + "body_filename": Path(body_uri).name if body_uri else None, + "coinbase_txid": normalized_report.get("coinbase_txid"), + "coinbase_manifest_sha256": normalized_report.get( + "coinbase_manifest_sha256_hex" + ), + "coinbase_tx_hex": normalized_report.get("coinbase_tx_hex"), + "coinbase_value_sats": normalized_report.get("coinbase_value_sats"), + "created_at": created_at, + } + + def _normalized_durable_evidence( + self, + *, + identity: AuditPublicationIdentity, + envelope_path: Path, + report: Mapping[str, Any], + persistence: Mapping[str, Any], + evidence: Mapping[str, Any], + verification_identity: Mapping[str, Any], + ) -> dict[str, Any]: + normalized_report = self._normalize_report_identity(report) + normalized_verification = self._normalize_verification_identity( + dict(verification_identity), + report=normalized_report, + ) + normalized_persistence = copy.deepcopy(dict(persistence)) + normalized_persistence["audit_bundle_sha256"] = _canonical_hex( + normalized_persistence.get("audit_bundle_sha256"), + name="persistence digest", + ) + body_uri = str(normalized_persistence.get("body_uri") or "") + if body_uri: + normalized_persistence["body_uri"] = str(self.resolve_owned_path(body_uri)) + durable = copy.deepcopy(dict(evidence)) + durable.update( + { + "schema": LIVE_EVIDENCE_SCHEMA, + "block_hash": identity.block_hash, + "block_height": identity.block_height, + "audit_bundle_path": str(envelope_path), + "audit_publication_identity": identity.to_json(), + "audit_report": normalized_report, + "audit_verification_identity": normalized_verification, + "persistence": normalized_persistence, + "coinbase_txid": normalized_report["coinbase_txid"], + "coinbase_manifest_sha256_hex": normalized_report[ + "coinbase_manifest_sha256_hex" + ], + "coinbase_tx_hex": normalized_report["coinbase_tx_hex"], + "coinbase_value_sats": normalized_report["coinbase_value_sats"], + } + ) + return durable + + def latest_evidence(self) -> dict[str, Any] | None: + with self._lock: + if not self._directory_authority_is_current(): + return None + if self._compatibility_evidence_override: + return copy.deepcopy(self._latest_evidence) + if self._evidence_state == "legacy_unproven": + return None + if self._latest_evidence is not None: + return copy.deepcopy(self._latest_evidence) + if self._evidence_state == "invalid": + return None + self._load_current_evidence() + with self._lock: + if not self._directory_authority_is_current(): + return None + return copy.deepcopy(self._latest_evidence) + + def set_latest_evidence_for_compatibility( + self, + payload: Mapping[str, Any] | None, + ) -> None: + with self._lock: + self._compatibility_evidence_override = True + self._invalidated_legacy_identity = None + self._legacy_proof_token = None + if payload is None: + self._latest_evidence = None + self._current_envelope = None + self._current_identity = None + self._evidence_state = "absent" + return + copied = copy.deepcopy(dict(payload)) + try: + latest, envelope, identity = self._validate_evidence(copied) + except (OSError, RuntimeError, TypeError, ValueError): + self._latest_evidence = copied + self._current_envelope = None + self._current_identity = None + self._evidence_state = "invalid" + return + self._latest_evidence = latest + self._current_envelope = envelope + self._current_identity = identity + self._evidence_state = self._loaded_evidence_state(latest, identity) + + def _load_current_evidence(self) -> None: + with self._lock: + self._load_current_evidence_locked() + + def _load_current_evidence_locked(self) -> None: + try: + self._validate_evidence_parent_identity() + evidence_bytes, _value = self._read_owned_regular_bytes(self._evidence_path) + except FileNotFoundError: + self._latest_evidence = None + self._current_envelope = None + self._current_identity = None + self._evidence_state = "absent" + return + except (OSError, RuntimeError): + self._latest_evidence = None + self._current_envelope = None + self._current_identity = None + self._evidence_state = "invalid" + return + try: + payload = json.loads(evidence_bytes.decode("utf-8")) + parsed = self._validate_evidence(payload) + except (OSError, UnicodeDecodeError, json.JSONDecodeError, RuntimeError, TypeError, ValueError): + self._latest_evidence = None + self._current_envelope = None + self._current_identity = None + self._evidence_state = "invalid" + return + self._latest_evidence, self._current_envelope, self._current_identity = parsed + self._evidence_state = self._loaded_evidence_state( + self._latest_evidence, + self._current_identity, + ) + + def _reload_current_evidence_locked(self) -> None: + """Reload peer state while retaining an exact in-process legacy proof.""" + + prior_token = self._legacy_proof_token + self._load_current_evidence_locked() + if ( + prior_token is not None + and self._evidence_state == "legacy_unproven" + and self._current_identity == prior_token.identity + ): + try: + current_token = self._capture_legacy_proof_token( + prior_token.identity + ) + except (OSError, RuntimeError): + current_token = None + if current_token == prior_token: + self._evidence_state = "legacy_proven" + return + self._legacy_proof_token = None + + def _capture_legacy_proof_token( + self, + identity: AuditPublicationIdentity, + ) -> _LegacyProofToken: + if self._current_envelope is None: + raise RuntimeError("legacy audit envelope is unavailable") + evidence_bytes, evidence_stat = self._read_owned_regular_bytes( + self._evidence_path + ) + envelope_bytes, envelope_stat = self._read_owned_regular_bytes( + self._current_envelope + ) + return _LegacyProofToken( + identity=identity, + evidence_file=_FileIdentity.from_stat(evidence_stat), + evidence_sha256=_sha256_bytes(evidence_bytes), + envelope_file=_FileIdentity.from_stat(envelope_stat), + envelope_sha256=_sha256_bytes(envelope_bytes), + ) + + def _validate_evidence( + self, + payload: object, + ) -> tuple[dict[str, Any], Path, AuditPublicationIdentity]: + if not isinstance(payload, dict) or payload.get("schema") != LIVE_EVIDENCE_SCHEMA: + raise RuntimeError("invalid evidence schema") + self._validate_root_identity() + raw_block_hash = payload.get("block_hash") + block_hash = _canonical_hex(raw_block_hash, name="block_hash") + if raw_block_hash != block_hash: + raise RuntimeError("evidence block hash is not canonical") + raw_block_height = payload.get("block_height") + if ( + isinstance(raw_block_height, bool) + or not isinstance(raw_block_height, int) + or raw_block_height < 0 + ): + raise RuntimeError("evidence block height is invalid") + block_height = raw_block_height + raw_envelope_path = payload.get("audit_bundle_path") + if not isinstance(raw_envelope_path, str): + raise RuntimeError("evidence envelope path is not canonical") + envelope_path = Path(raw_envelope_path).absolute() + expected_path = self.live_envelope_path( + block_height=block_height, + block_hash=block_hash, + ) + if envelope_path != expected_path or raw_envelope_path != str(expected_path): + raise RuntimeError("evidence envelope path is not canonical") + envelope_bytes, _envelope_stat = self._read_owned_regular_bytes(envelope_path) + envelope = json.loads(envelope_bytes.decode("utf-8")) + if ( + not isinstance(envelope, dict) + or envelope.get("schema") != LIVE_ENVELOPE_SCHEMA + or envelope.get("block_hash") != block_hash + or isinstance(envelope.get("block_height"), bool) + or not isinstance(envelope.get("block_height"), int) + or envelope.get("block_height") != block_height + ): + raise RuntimeError("evidence envelope identity mismatch") + report = payload.get("audit_report") + persistence = payload.get("persistence") + if not isinstance(report, dict) or not isinstance(persistence, dict): + raise RuntimeError("evidence report and persistence are required") + normalized_report = self._normalize_report_identity(report) + if report != normalized_report: + raise RuntimeError("evidence audit report is not canonical") + if normalized_report["block_height"] != block_height: + raise RuntimeError("evidence report block height mismatch") + report_digest = _canonical_hex( + normalized_report.get("audit_bundle_sha256_hex"), + name="audit report digest", + ) + raw_persistence_digest = persistence.get("audit_bundle_sha256") + persistence_digest = _canonical_hex( + raw_persistence_digest, + name="persistence digest", + ) + raw_envelope_digest = envelope.get("audit_bundle_sha256") + envelope_digest = _canonical_hex( + raw_envelope_digest, + name="envelope digest", + ) + if ( + raw_persistence_digest != persistence_digest + or raw_envelope_digest != envelope_digest + ): + raise RuntimeError("evidence digest is not canonical") + if not ( + hmac.compare_digest(report_digest, persistence_digest) + and hmac.compare_digest(report_digest, envelope_digest) + ): + raise RuntimeError("evidence digest mismatch") + raw_persistence_body_uri = persistence.get("body_uri") + raw_envelope_body_uri = envelope.get("body_uri") + if not isinstance(raw_persistence_body_uri, str) or not isinstance( + raw_envelope_body_uri, + str, + ): + raise RuntimeError("evidence body URI is not canonical") + persistence_body_uri = raw_persistence_body_uri + envelope_body_uri = raw_envelope_body_uri + if persistence_body_uri: + body_path = self.resolve_owned_path(persistence_body_uri) + expected_body_path = self.body_path(block_hash, persistence_digest) + if body_path != expected_body_path: + raise RuntimeError("evidence body URI identity mismatch") + canonical_body_uri = str(body_path) + if persistence_body_uri != canonical_body_uri: + raise RuntimeError("evidence body URI is not canonical") + persistence_body_uri = canonical_body_uri + if envelope_body_uri: + canonical_envelope_body_uri = str( + self.resolve_owned_path(envelope_body_uri) + ) + if envelope_body_uri != canonical_envelope_body_uri: + raise RuntimeError("envelope body URI is not canonical") + envelope_body_uri = canonical_envelope_body_uri + if ( + envelope_body_uri != persistence_body_uri + or envelope.get("body_filename") + != (Path(persistence_body_uri).name if persistence_body_uri else None) + ): + raise RuntimeError("evidence and envelope body URIs differ") + coinbase_bindings = ( + ("coinbase_txid", "coinbase_txid"), + ("coinbase_manifest_sha256_hex", "coinbase_manifest_sha256"), + ("coinbase_tx_hex", "coinbase_tx_hex"), + ("coinbase_value_sats", "coinbase_value_sats"), + ) + for report_key, envelope_key in coinbase_bindings: + expected_value = normalized_report[report_key] + evidence_value = payload.get(report_key) + envelope_value = envelope.get(envelope_key) + if report_key == "coinbase_value_sats": + if ( + isinstance(evidence_value, bool) + or not isinstance(evidence_value, int) + or isinstance(envelope_value, bool) + or not isinstance(envelope_value, int) + ): + raise RuntimeError("evidence coinbase value is invalid") + elif report_key == "coinbase_tx_hex": + canonical_evidence_value = _canonical_hex_bytes( + evidence_value, + name="evidence coinbase_tx_hex", + ) + canonical_envelope_value = _canonical_hex_bytes( + envelope_value, + name="envelope coinbase_tx_hex", + ) + if ( + evidence_value != canonical_evidence_value + or envelope_value != canonical_envelope_value + ): + raise RuntimeError("evidence coinbase identity is not canonical") + evidence_value = canonical_evidence_value + envelope_value = canonical_envelope_value + else: + canonical_evidence_value = _canonical_hex( + evidence_value, + name=report_key, + ) + canonical_envelope_value = _canonical_hex( + envelope_value, + name=envelope_key, + ) + if ( + evidence_value != canonical_evidence_value + or envelope_value != canonical_envelope_value + ): + raise RuntimeError("evidence coinbase identity is not canonical") + evidence_value = canonical_evidence_value + envelope_value = canonical_envelope_value + if evidence_value != expected_value or envelope_value != expected_value: + raise RuntimeError("evidence coinbase identity mismatch") + identity_payload = payload.get("audit_publication_identity") + if not isinstance(identity_payload, dict): + # Older durable evidence remains readable but starts at sequence 0. + identity = AuditPublicationIdentity(0, block_height, block_hash) + else: + raw_sequence = identity_payload.get("sequence") + raw_identity_height = identity_payload.get("block_height") + if ( + isinstance(raw_sequence, bool) + or not isinstance(raw_sequence, int) + or isinstance(raw_identity_height, bool) + or not isinstance(raw_identity_height, int) + ): + raise RuntimeError("invalid evidence publication identity") + identity = AuditPublicationIdentity( + raw_sequence, + raw_identity_height, + _canonical_hex( + identity_payload.get("block_hash"), + name="publication block hash", + ), + ) + if ( + identity.sequence < 0 + or identity.block_height != block_height + or identity.block_hash != block_hash + or identity_payload != identity.to_json() + ): + raise RuntimeError("invalid evidence publication identity") + normalized_payload = copy.deepcopy(payload) + verification_payload = payload.get("audit_verification_identity") + if identity.sequence > 0: + if ( + isinstance(verification_payload, dict) + and verification_payload.get("schema") + == LEGACY_VERIFICATION_UNAVAILABLE_SCHEMA + ): + normalized_verification = ( + self._normalize_legacy_verification_marker( + verification_payload, + identity=identity, + report=normalized_report, + ) + ) + else: + normalized_verification = self._normalize_verification_identity( + verification_payload, + report=normalized_report, + ) + normalized_payload["audit_verification_identity"] = ( + normalized_verification + ) + elif verification_payload is not None: + # Sequence-zero evidence is the only supported legacy form. If a + # producer supplied a verification identity anyway, validate it + # rather than allowing an unchecked field to become durable state. + normalized_payload["audit_verification_identity"] = ( + self._normalize_verification_identity( + verification_payload, + report=normalized_report, + ) + ) + self._validate_root_identity() + self._validate_evidence_parent_identity() + return normalized_payload, envelope_path, identity + + def adopt_legacy_publication_identity( + self, + identity: AuditPublicationIdentity, + *, + publication_floor_sequence: int, + ) -> None: + """Record a ledger proof that must be revalidated after every restart.""" + + self._require_publication_order_guard() + if ( + isinstance(publication_floor_sequence, bool) + or not isinstance(publication_floor_sequence, int) + or publication_floor_sequence < 0 + ): + raise ValueError("publication floor sequence must be a non-negative integer") + with self._lock: + self._validate_root_identity() + self._validate_evidence_parent_identity() + if not self._compatibility_evidence_override: + self._load_current_evidence_locked() + if self._evidence_state not in { + "legacy", + "legacy_unproven", + } or self._latest_evidence is None: + return + if identity.sequence > publication_floor_sequence: + raise RuntimeError( + "legacy audit publication sequence exceeds the durable ledger floor" + ) + if identity.sequence < publication_floor_sequence: + self.invalidate_unprovable_legacy_evidence() + return + if ( + identity.sequence <= 0 + or identity.block_hash != self._latest_evidence.get("block_hash") + or identity.block_height + != int(self._latest_evidence.get("block_height", -1)) + ): + raise RuntimeError("legacy evidence ledger identity mismatch") + upgraded = copy.deepcopy(self._latest_evidence) + upgraded["audit_publication_identity"] = identity.to_json() + report = upgraded.get("audit_report") + if not isinstance(report, dict): + raise RuntimeError("legacy audit report is unavailable") + upgraded["audit_verification_identity"] = ( + self._legacy_verification_marker( + identity=identity, + report=report, + ) + ) + old_evidence, _old_stat = self._read_owned_regular_bytes( + self._evidence_path + ) + published_identity = self._write_mutable_json( + self._evidence_path, + upgraded, + sort_keys=True, + ) + try: + self._validate_root_identity() + self._validate_evidence_parent_identity() + except BaseException: + self._rollback_mutable_replace( + self._evidence_path, + old_evidence, + published_identity, + allow_detached_authority=True, + ) + raise + self._latest_evidence = upgraded + self._current_identity = identity + self._evidence_state = "legacy_proven" + self._invalidated_legacy_identity = None + self._legacy_proof_token = self._capture_legacy_proof_token(identity) + + def invalidate_unprovable_legacy_evidence(self) -> None: + """Drop ordering/pin authority while allowing a new durable repair.""" + + with self._lock: + if self._evidence_state not in {"legacy", "legacy_unproven"}: + return + self._invalidated_legacy_identity = self._current_identity + self._legacy_proof_token = None + self._latest_evidence = None + self._current_envelope = None + self._current_identity = None + self._evidence_state = "invalid" + + def prune_best_effort( + self, + *, + keep_live_path: Path | None = None, + ) -> RetentionResult: + try: + with self.publication_order_guard(): + with self._lock: + if not self._compatibility_evidence_override: + # A different coordinator process may have published a + # newer reference. Reconcile it under the same flock + # immediately before deriving live-deletion pins. + self._reload_current_evidence_locked() + return self._prune_best_effort_guarded( + keep_live_path=keep_live_path, + ) + except (OSError, RuntimeError): + return RetentionResult(errors=1) + + def _prune_best_effort_guarded( + self, + *, + keep_live_path: Path | None = None, + ) -> RetentionResult: + self._require_publication_order_guard() + live_removed = 0 + candidate_removed = 0 + errors = 0 + with self._lock: + live_authorized = self._directory_authority_is_current() + pins = ( + { + path + for path in (self._current_envelope,) + if path is not None + } + if live_authorized + and self._evidence_state in {"valid", "legacy_proven"} + else set() + ) + if keep_live_path is not None: + raw_keep = Path(keep_live_path).absolute() + candidate_keep = raw_keep.parent.resolve() / raw_keep.name + if ( + candidate_keep.parent == self._root + and _LIVE_RE.fullmatch(candidate_keep.name) + ): + pins.add(candidate_keep) + active_paths = {entry[0] for entry in self._active_candidates.values()} + live_retention = self._live_bundle_retention + candidate_retention = self._candidate_retention_seconds + evidence_state = self._evidence_state if live_authorized else "invalid" + try: + self._validate_root_identity() + entries = sorted( + (self._root / name for name in self._scan_root_names()), + key=lambda value: value.name, + ) + except (OSError, RuntimeError): + return RetentionResult(errors=1) + live: list[tuple[int, int, str, Path, _FileIdentity]] = [] + now = self._wall_time() + for path in entries: + try: + value = self._owned_lstat(path) + except RuntimeError: + errors += 1 + return RetentionResult(0, candidate_removed, errors) + except (FileNotFoundError, OSError): + errors += 1 + continue + if not stat.S_ISREG(value.st_mode): + continue + live_match = _LIVE_RE.fullmatch(path.name) + if live_match: + live.append( + ( + value.st_mtime_ns, + int(live_match.group("height")), + live_match.group("block"), + path, + _FileIdentity.from_stat(value), + ) + ) + continue + if not ( + _CANDIDATE_RE.fullmatch(path.name) + or _LEGACY_CANDIDATE_RE.fullmatch(path.name) + or _LEGACY_HIDDEN_CANDIDATE_RE.fullmatch(path.name) + ): + continue + if path in active_paths: + continue + if candidate_retention != 0 and now - value.st_mtime <= candidate_retention: + continue + try: + with self._lock: + if not self._directory_authority_is_current(): + errors += 1 + break + current_active_paths = { + entry[0] for entry in self._active_candidates.values() + } + if path in current_active_paths: + continue + if self._unlink_scanned_owned( + path, + _FileIdentity.from_stat(value), + ): + candidate_removed += 1 + except (OSError, RuntimeError): + errors += 1 + if evidence_state not in {"valid", "legacy_proven"}: + return RetentionResult(0, candidate_removed, errors) + live.sort(key=lambda item: (item[0], item[1], item[2], item[3].name), reverse=True) + retained = 0 + for _mtime, _height, _block, path, identity in live: + if path in pins: + continue + if live_retention < 0 or retained < max(live_retention - len(pins), 0): + retained += 1 + continue + try: + with self._lock: + self._require_publication_order_guard() + if not self._directory_authority_is_current(): + errors += 1 + break + if path == self._current_envelope: + continue + if self._unlink_scanned_owned( + path, + identity, + require_all_authorities=True, + ): + live_removed += 1 + except (OSError, RuntimeError): + errors += 1 + try: + self._validate_root_identity() + self._validate_evidence_parent_identity() + except RuntimeError: + errors += 1 + return RetentionResult(live_removed, candidate_removed, errors) + + def _unlink_scanned_owned( + self, + path: Path, + identity: _FileIdentity, + *, + require_all_authorities: bool = False, + ) -> bool: + if require_all_authorities and not self._directory_authority_is_current(): + raise RuntimeError("audit directory authority changed before live prune") + return self._remove_identity_safe(path, identity) + + def _remove_identity_safe( + self, + path: Path, + identity: _FileIdentity, + *, + allow_detached_authority: bool = False, + ) -> bool: + if not allow_detached_authority: + self._validate_owned_parent(path) + try: + current = self._owned_lstat(path) + except FileNotFoundError: + return False + if not stat.S_ISREG(current.st_mode): + # Hard-link restoration is unavailable for directories and is not + # portable for symlinks. Never relocate an unowned nonregular + # replacement merely to discover that cleanup lacks authority. + return False + if not identity.matches(current): + # A replacement that was already present at the precheck is not + # ours to relocate, even temporarily. + return False + quarantine = path.with_name(f".{path.name}.{uuid.uuid4().hex}.cleanup") + try: + self._owned_replace(path, quarantine) + except FileNotFoundError: + return False + try: + moved = self._owned_lstat(quarantine) + if identity.matches(moved): + self._owned_unlink(quarantine) + return True + # A replacement won the race. Restore it when the original name is + # free; otherwise preserve it under the quarantine name. + try: + self._owned_link(quarantine, path) + except OSError: + return False + self._owned_unlink(quarantine) + return False + except BaseException: + # Never turn an uncertain identity into deletion authority. + raise + + def _write_mutable_json( + self, + path: Path, + payload: Mapping[str, Any], + *, + sort_keys: bool, + ) -> _FileIdentity: + body = _json_bytes(payload, sort_keys=sort_keys) + return self._write_mutable_bytes(path, body) + + def _write_mutable_bytes(self, path: Path, payload: bytes) -> _FileIdentity: + return self._write_mutable_chunks(path, lambda: (payload,)) + + def _write_mutable_chunks( + self, + path: Path, + chunks_factory: Callable[[], Iterable[bytes]], + ) -> _FileIdentity: + """Replace a mutable owned file with a chunk stream, atomically. + + ``chunks_factory`` returns a fresh bounded iterator each call, so a + whole-window payload is never held as one bytes object; the prior + bytes are still captured for rollback (bounded by the existing + file: a share slot or an evidence document). + """ + path = Path(path).absolute() + resolved_parent = path.parent.resolve(strict=True) + if not ( + resolved_parent == self._root + or path == self._evidence_path + ): + raise RuntimeError("mutable audit target is outside the owned store") + with self._lock: + if path == self._evidence_path: + self._validate_evidence_parent_identity() + else: + self._validate_root_identity() + old_bytes: bytes | None + try: + old_bytes, _old_stat = self._read_owned_regular_bytes(path) + except FileNotFoundError: + old_bytes = None + tmp_path = path.with_name(f".{path.name}.{uuid.uuid4().hex}.tmp") + temp_identity: _FileIdentity | None = None + published = False + try: + fd = self._owned_open( + tmp_path, + os.O_WRONLY | os.O_CREAT | os.O_EXCL | getattr(os, "O_NOFOLLOW", 0), + 0o600, + ) + temp_identity = _FileIdentity.from_stat(os.fstat(fd)) + with os.fdopen(fd, "wb") as handle: + try: + for chunk in chunks_factory(): + handle.write(chunk) + handle.flush() + os.fsync(handle.fileno()) + finally: + temp_identity = _FileIdentity.from_stat( + os.fstat(handle.fileno()) + ) + self._owned_replace(tmp_path, path) + published = True + try: + self._fsync_directory(path.parent) + if path == self._evidence_path: + self._validate_evidence_parent_identity() + else: + self._validate_root_identity() + except BaseException: + self._rollback_mutable_replace( + path, + old_bytes, + temp_identity, + allow_detached_authority=True, + ) + raise + return temp_identity + finally: + if not published and temp_identity is not None: + self._remove_identity_safe( + tmp_path, + temp_identity, + allow_detached_authority=True, + ) + + def _rollback_mutable_replace( + self, + path: Path, + old_bytes: bytes | None, + published_identity: _FileIdentity, + *, + allow_detached_authority: bool = False, + ) -> None: + removed = self._remove_identity_safe( + path, + published_identity, + allow_detached_authority=allow_detached_authority, + ) + if not removed: + # A replacement won the race. Preserve it and do not restore stale + # bytes over an inode this operation never owned. + return + if old_bytes is None: + if allow_detached_authority: + self._fsync_pinned_directory(path.parent) + else: + self._fsync_directory(path.parent) + return + rollback = path.with_name(f".{path.name}.{uuid.uuid4().hex}.rollback") + rollback_identity: _FileIdentity | None = None + try: + fd = self._owned_open( + rollback, + os.O_WRONLY | os.O_CREAT | os.O_EXCL | getattr(os, "O_NOFOLLOW", 0), + 0o600, + ) + rollback_identity = _FileIdentity.from_stat(os.fstat(fd)) + with os.fdopen(fd, "wb") as handle: + try: + handle.write(old_bytes) + handle.flush() + os.fsync(handle.fileno()) + finally: + rollback_identity = _FileIdentity.from_stat( + os.fstat(handle.fileno()) + ) + try: + self._owned_link(rollback, path) + except FileExistsError: + # Preserve a concurrent replacement rather than overwriting it. + return + if allow_detached_authority: + self._fsync_pinned_directory(path.parent) + else: + self._fsync_directory(path.parent) + finally: + if rollback_identity is not None: + self._remove_identity_safe( + rollback, + rollback_identity, + allow_detached_authority=allow_detached_authority, + ) + + def _write_immutable_bytes(self, path: Path, payload: bytes) -> None: + self._write_immutable_chunks(path, lambda: (payload,), len(payload)) + + def _write_immutable_chunks( + self, + path: Path, + chunks_factory: Callable[[], Iterable[bytes]], + size: int | None, + ) -> None: + """Publish an immutable owned artifact from a replayable chunk stream. + + ``chunks_factory`` must return the same bounded chunks on every call: + the stream is consumed once to write the temporary inode and again, + chunk by chunk against the existing file, whenever an equal artifact + may already be published. ``size`` is the stream's total byte count + when known in advance (None derives it from the stream). + """ + if getattr(self, "_read_only", False): + raise RuntimeError( + "audit artifact store is read-only: publication is not available" + ) + path = Path(path).absolute() + if ( + path.parent.resolve(strict=True) != self._root + or self.artifact_kind(path.name) + not in {"body", "share_segment", "canonical_bundle"} + ): + raise RuntimeError( + "immutable audit target is not an owned body, segment, or " + "canonical bundle" + ) + with self._lock: + self._validate_root_identity() + try: + existing = self._owned_lstat(path) + except FileNotFoundError: + existing = None + if existing is not None: + if not stat.S_ISREG(existing.st_mode) or not self._file_matches_chunks( + path, + chunks_factory, + size, + ): + raise RuntimeError( + f"existing audit artifact does not match payload at {path}" + ) + # The prior writer may have failed between link/rename and its + # directory fsync. An idempotent retry is the durability + # repair boundary, even though no bytes need to change. + self._fsync_directory(path.parent) + return + tmp_path = path.with_name(f".{path.name}.{uuid.uuid4().hex}.tmp") + temp_identity: _FileIdentity | None = None + linked_by_this_call = False + try: + fd = self._owned_open( + tmp_path, + os.O_WRONLY | os.O_CREAT | os.O_EXCL | getattr(os, "O_NOFOLLOW", 0), + 0o600, + ) + temp_identity = _FileIdentity.from_stat(os.fstat(fd)) + with os.fdopen(fd, "wb") as handle: + try: + written = 0 + for chunk in chunks_factory(): + handle.write(chunk) + written += len(chunk) + handle.flush() + os.fsync(handle.fileno()) + finally: + temp_identity = _FileIdentity.from_stat( + os.fstat(handle.fileno()) + ) + if size is not None and written != size: + raise RuntimeError( + f"audit artifact stream size changed while publishing {path}" + ) + try: + self._owned_link(tmp_path, path) + linked_by_this_call = True + except FileExistsError: + if not self._file_matches_chunks(path, chunks_factory, size): + raise RuntimeError( + f"existing audit artifact does not match payload at {path}" + ) + try: + self._fsync_directory(path.parent) + except BaseException: + if linked_by_this_call: + # Revoke only the inode linked by this call. A racing + # replacement is never deleted or overwritten. + self._remove_identity_safe( + path, + temp_identity, + allow_detached_authority=True, + ) + try: + self._fsync_directory(path.parent) + except BaseException: + # Preserve the original durability failure; the + # retry path always fsyncs an equal existing file. + pass + raise + self._validate_root_identity() + finally: + if temp_identity is not None: + self._remove_identity_safe( + tmp_path, + temp_identity, + allow_detached_authority=True, + ) + + def _fsync_directory(self, path: Path) -> None: + path = Path(path).absolute() + if path == self._root: + self._validate_root_identity() + os.fsync(self._root_fd) + self._validate_root_identity() + return + if path == self._evidence_path.parent: + self._validate_evidence_parent_identity() + os.fsync(self._evidence_parent_fd) + self._validate_evidence_parent_identity() + return + raise RuntimeError("audit fsync target has no pinned authority") + + def _fsync_pinned_directory(self, path: Path) -> None: + path = Path(path).absolute() + if path == self._root: + os.fsync(self._root_fd) + return + if path == self._evidence_path.parent: + os.fsync(self._evidence_parent_fd) + return + raise RuntimeError("audit fsync target has no pinned authority") + + def file_matches_bytes(self, path: Path, expected: bytes) -> bool: + return self._file_matches_chunks(path, lambda: (expected,), len(expected)) + + def _file_matches_chunks( + self, + path: Path, + chunks_factory: Callable[[], Iterable[bytes]], + size: int | None, + ) -> bool: + """Compare an owned file with a chunk stream without reading it whole. + + Bounded by one chunk at a time; the descriptor identity is checked + before and after the comparison exactly as the whole-file reader + does, so a file mutated mid-compare reports a mismatch. + """ + try: + parent_fd = self._owned_parent_fd(path) + if parent_fd is not None: + self._validate_owned_parent(path) + fd = self._owned_open( + path, + os.O_RDONLY | getattr(os, "O_NOFOLLOW", 0), + ) + except OSError: + return False + try: + before = os.fstat(fd) + if not stat.S_ISREG(before.st_mode): + return False + if size is not None and before.st_size != size: + return False + offset = 0 + equal = True + for chunk in chunks_factory(): + if not equal: + break + actual = _pread_exact(fd, offset, len(chunk)) + if not hmac.compare_digest(actual, chunk): + equal = False + offset += len(chunk) + if equal and offset != before.st_size: + equal = False + after = os.fstat(fd) + if ( + before.st_dev != after.st_dev + or before.st_ino != after.st_ino + or before.st_size != after.st_size + or before.st_mtime_ns != after.st_mtime_ns + or before.st_ctime_ns != after.st_ctime_ns + ): + return False + if parent_fd is not None: + self._validate_owned_parent(path) + return equal + except OSError: + return False + finally: + os.close(fd) + + @staticmethod + def file_sha256_hex(path: Path) -> str: + payload, _value = AuditArtifactStore.read_regular_bytes(path) + return _sha256_bytes(payload) + + @staticmethod + def read_regular_bytes(path: Path) -> tuple[bytes, os.stat_result]: + fd = os.open( + path, + os.O_RDONLY | getattr(os, "O_NOFOLLOW", 0), + ) + return AuditArtifactStore._read_regular_bytes_fd(fd) + + @staticmethod + def _read_regular_bytes_fd(fd: int) -> tuple[bytes, os.stat_result]: + try: + before = os.fstat(fd) + if not stat.S_ISREG(before.st_mode): + raise OSError("audit artifact is not a regular file") + chunks: list[bytes] = [] + while True: + chunk = os.read(fd, 1024 * 1024) + if not chunk: + break + chunks.append(chunk) + after = os.fstat(fd) + if ( + before.st_dev != after.st_dev + or before.st_ino != after.st_ino + or before.st_size != after.st_size + or before.st_mtime_ns != after.st_mtime_ns + or before.st_ctime_ns != after.st_ctime_ns + ): + raise OSError("audit artifact changed while reading") + return b"".join(chunks), after + finally: + os.close(fd) + + # External body and share-segment capability used by the ledger. Database + # lease checks remain in the ledger; paths, encodings, and bytes stay here. + + def canonical_audit_bundle_bytes(self, final_bundle: Mapping[str, Any]) -> bytes: + if isinstance(final_bundle, CanonicalAuditBundleView): + # The view *is* the canonical artifact, so its bytes are the + # canonical bytes by definition. This whole-body copy is the + # compatibility lane for callers that still want one bytes + # object; the bounded publication path streams + # ``final_bundle.iter_bytes()`` instead. + return b"".join(final_bundle.iter_bytes()) + return canonical_audit_bundle_bytes(final_bundle, self._canonicalizer) + + def canonical_audit_body_bytes_for_sha( + self, + final_bundle: Mapping[str, Any], + audit_bundle_sha256: str, + ) -> bytes: + expected = _canonical_hex( + audit_bundle_sha256, + name="audit_bundle_sha256", + ) + if isinstance(final_bundle, CanonicalAuditBundleView): + actual = final_bundle.sha256_hex + if not hmac.compare_digest(actual, expected): + raise RuntimeError( + f"audit bundle sha256 mismatch: expected {expected}, got {actual}" + ) + return b"".join(final_bundle.iter_bytes()) + body = self.canonical_audit_bundle_bytes(final_bundle) + actual = _sha256_bytes(body) + if not hmac.compare_digest(actual, expected): + raise RuntimeError( + f"audit bundle sha256 mismatch: expected {expected}, got {actual}" + ) + return body + + # Bounded canonical-source helpers (issue #255). The verified candidate + # artifact is the authority; these open it under the store's directory + # authority and expose it as a positional-read source so digests, gzip + # publication and logical comparison stream through bounded chunks. + + def _open_owned_artifact_source(self, path: Path) -> ArtifactSource: + """Open an owned regular file as a bounded positional-read source.""" + path = Path(path) + parent_fd = self._owned_parent_fd(path) + if parent_fd is not None: + self._validate_owned_parent(path) + fd = self._owned_open( + path, + os.O_RDONLY | getattr(os, "O_NOFOLLOW", 0), + ) + try: + source = ArtifactSource(fd, path=path) + except CanonicalArtifactError as exc: + raise OSError(str(exc)) from exc + if parent_fd is not None: + try: + self._validate_owned_parent(path) + except BaseException: + # The descriptor is retired here, not by a finalizer that a + # retained traceback could postpone. + source.close() + raise + return source + + def _scan_owned_artifact( + self, + path: Path, + *, + lazy_paths: Sequence[tuple[str, ...]], + ) -> CanonicalAuditBundleView: + """Scan an owned JSON artifact into a bounded view (closed by caller).""" + parent_fd = self._owned_parent_fd(path) + if parent_fd is not None: + self._validate_owned_parent(path) + fd = self._owned_open( + Path(path), + os.O_RDONLY | getattr(os, "O_NOFOLLOW", 0), + ) + view = CanonicalAuditBundleView.scan( + fd, + path=Path(path), + lazy_paths=lazy_paths, + ) + if parent_fd is not None: + try: + self._validate_owned_parent(path) + except BaseException: + # Retire the artifact and index descriptors now; a retained + # traceback must not keep them open until collection. + view.close() + raise + return view + + def _require_logical_match( + self, + source: ArtifactSource, + final_bundle: Mapping[str, Any], + ) -> None: + """Full logical comparison of a canonical source with ``final_bundle``. + + A view scanned from this very inode (same identity and digest) is + the artifact by construction. Any other bundle -- a dictionary from + a compatibility builder, or a view over a different file -- is + compared member by member against a bounded scan of the source, with + the window-sized arrays streamed record by record, which preserves + the whole-document equality check without ``json.loads`` of the + candidate. + """ + if ( + isinstance(final_bundle, CanonicalAuditBundleView) + and final_bundle.identity == source.identity + and hmac.compare_digest(final_bundle.sha256_hex, source.sha256_hex()) + ): + return + fd = os.dup(source.fileno()) + try: + scanned = CanonicalAuditBundleView.scan(fd, path=source.path) + except CanonicalArtifactError as exc: + raise RuntimeError("canonical audit candidate is not valid JSON") from exc + try: + if scanned != final_bundle: + raise RuntimeError( + "canonical audit candidate does not match logical bundle" + ) + finally: + scanned.close() + + def stored_canonical_bundle_sha256( + self, + block_hash: str, + audit_bundle_sha256: str, + ) -> str | None: + """Verify a stored canonical bundle by streaming, without its bytes. + + Returns the verified uncompressed digest, None when nothing is + stored, and raises :class:`CanonicalAuditBundleCorrupt` for damage, + exactly like :meth:`read_canonical_audit_bundle` -- but the artifact + is decompressed and hashed one bounded chunk at a time. + """ + + block_hash = _canonical_hex(block_hash, name="block_hash") + digest = _canonical_hex( + audit_bundle_sha256, + name="audit_bundle_sha256", + ) + path = self.canonical_bundle_path(block_hash, digest) + try: + source = self._open_owned_artifact_source(path) + except FileNotFoundError: + self._validate_root_identity() + return None + except OSError as exc: + raise CanonicalAuditBundleCorrupt( + f"canonical audit bundle is not retrievable at {path}: {exc}" + ) from exc + try: + hasher = hashlib.sha256() + try: + with gzip.GzipFile( + fileobj=_ChunkReader(source), + mode="rb", + ) as handle: + while True: + piece = handle.read(SCAN_CHUNK_BYTES_LOCAL) + if not piece: + break + hasher.update(piece) + source.verify_identity() + except (OSError, EOFError, zlib.error, CanonicalArtifactError) as exc: + raise CanonicalAuditBundleCorrupt( + f"canonical audit bundle does not decompress at {path}: {exc}" + ) from exc + actual = hasher.hexdigest() + if not hmac.compare_digest(actual, digest): + raise CanonicalAuditBundleCorrupt( + f"canonical audit bundle hash mismatch at {path}: " + f"expected {digest}, got {actual}" + ) + finally: + source.close() + self._validate_root_identity() + return actual + + def write_canonical_audit_bundle_from_source( + self, + block_hash: str, + audit_bundle_sha256: str, + source: ArtifactSource, + ) -> Path: + """Publish the canonical bundle behind ``source`` by streaming it. + + Same contract as :meth:`write_canonical_audit_bundle` -- verified + digest before any store side effect, reproducible compression, + immutable publication, and a post-publication re-verification -- + without ever holding the uncompressed or compressed body as one + bytes object. Republishing is a durable no-op when the stored + artifact decompresses to the advertised digest. + """ + + block_hash = _canonical_hex(block_hash, name="block_hash") + digest = _canonical_hex( + audit_bundle_sha256, + name="audit_bundle_sha256", + ) + actual = source.sha256_hex() + if not hmac.compare_digest(actual, digest): + raise RuntimeError( + f"audit bundle sha256 mismatch: expected {digest}, got {actual}" + ) + path = self.canonical_bundle_path(block_hash, digest) + with self._lock: + try: + stored = self.stored_canonical_bundle_sha256(block_hash, digest) + except CanonicalAuditBundleCorrupt: + # Preserve the immutable writer's existing conflict behavior + # for damaged files. It will refuse to replace their bytes. + stored = None + if stored is not None: + self._fsync_directory(path.parent) + self._validate_root_identity() + return path + self._write_immutable_chunks( + path, + lambda: _gzip_canonical_chunks(source.iter_bytes()), + None, + ) + self._fsync_directory(path.parent) + published = self.stored_canonical_bundle_sha256(block_hash, digest) + if published is None: + raise RuntimeError( + f"canonical audit bundle is absent after publication at {path}" + ) + self._validate_root_identity() + return path + + # Canonical bundle persistence. The public digest of a publication is the + # SHA-256 of these uncompressed bytes; keeping them on disk means a reader + # never has to trust a reconstruction to answer for that identity. + + def write_canonical_audit_bundle( + self, + block_hash: str, + audit_bundle_sha256: str, + canonical_bytes: bytes, + ) -> Path: + """Publish the exact canonical bundle bytes behind an advertised digest. + + Verifies the uncompressed digest before anything reaches the store, + compresses reproducibly, publishes immutably, then reopens the + published artifact and verifies it again. Republishing identical + bytes is a durable no-op; a conflicting artifact fails loudly. + """ + + block_hash = _canonical_hex(block_hash, name="block_hash") + digest = _canonical_hex( + audit_bundle_sha256, + name="audit_bundle_sha256", + ) + payload = bytes(canonical_bytes) + actual = _sha256_bytes(payload) + if not hmac.compare_digest(actual, digest): + raise RuntimeError( + f"audit bundle sha256 mismatch: expected {digest}, got {actual}" + ) + path = self.canonical_bundle_path(block_hash, digest) + with self._lock: + try: + existing_payload = self.read_canonical_audit_bundle( + block_hash, + digest, + ) + except CanonicalAuditBundleCorrupt: + # Preserve the immutable writer's existing conflict behavior + # for damaged files. It will refuse to replace their bytes. + existing_payload = None + if existing_payload is not None and hmac.compare_digest( + existing_payload, + payload, + ): + # The public identity covers the uncompressed canonical bytes, + # not a particular zlib encoding. A retry after a compressor + # upgrade is therefore the same durability-repair boundary as + # a byte-identical gzip republication. + self._fsync_directory(path.parent) + self._validate_root_identity() + return path + + # The immutable writer is the publication boundary: it links + # atomically and refuses to overwrite an artifact that does not + # represent this canonical payload. + self._write_immutable_bytes(path, gzip_canonical_bundle_bytes(payload)) + # `_write_immutable_bytes` already fsyncs the directory; restating + # the durability boundary here keeps it explicit for this artifact. + self._fsync_directory(path.parent) + published = self.read_canonical_audit_bundle(block_hash, digest) + if published is None: + raise RuntimeError( + f"canonical audit bundle is absent after publication at {path}" + ) + if not hmac.compare_digest(published, payload): + raise CanonicalAuditBundleCorrupt( + f"canonical audit bundle bytes changed during publication at {path}" + ) + self._validate_root_identity() + return path + + def read_canonical_audit_bundle( + self, + block_hash: str, + audit_bundle_sha256: str, + ) -> bytes | None: + """Return the stored canonical bytes, or None when none were stored. + + Damage raises `CanonicalAuditBundleCorrupt` rather than reporting a + miss, so a caller that falls back to reconstruction can log the loss + instead of silently absorbing it. + """ + + block_hash = _canonical_hex(block_hash, name="block_hash") + digest = _canonical_hex( + audit_bundle_sha256, + name="audit_bundle_sha256", + ) + path = self.canonical_bundle_path(block_hash, digest) + try: + compressed, _value = self._read_owned_regular_bytes(path) + except FileNotFoundError: + self._validate_root_identity() + return None + except OSError as exc: + # A symlink refused by O_NOFOLLOW, a non-regular file, or a file + # mutated mid-read is damage, not absence. + raise CanonicalAuditBundleCorrupt( + f"canonical audit bundle is not retrievable at {path}: {exc}" + ) from exc + try: + payload = gunzip_canonical_bundle_bytes(compressed) + except (OSError, EOFError, zlib.error) as exc: + raise CanonicalAuditBundleCorrupt( + f"canonical audit bundle does not decompress at {path}: {exc}" + ) from exc + actual = _sha256_bytes(payload) + if not hmac.compare_digest(actual, digest): + raise CanonicalAuditBundleCorrupt( + f"canonical audit bundle hash mismatch at {path}: " + f"expected {digest}, got {actual}" + ) + self._validate_root_identity() + return payload + + def _owned_external_body_bytes( + self, + body_uri: object, + *, + expected_sha256: str, + block_hash: str | None = None, + ) -> tuple[Path, bytes]: + """Resolve and read an owned external body, binding its filename identity.""" + + path = self.resolve_owned_path(body_uri) + match = _BODY_RE.fullmatch(path.name) + if match is None: + raise RuntimeError("body URI is not an owned canonical body") + if match.group("digest") != expected_sha256: + raise RuntimeError("body URI digest does not match expected digest") + if block_hash is not None and match.group("block") != block_hash: + raise RuntimeError("body URI block hash does not match expected block") + payload, _value = self._read_owned_regular_bytes(path) + return path, payload + + def literal_canonical_bundle_bytes( + self, + body_uri: object, + *, + expected_sha256: object, + ) -> bytes | None: + """Return a pre-v2 external body that already *is* the canonical bundle. + + Bodies written before the compact layouts stored the canonical JSON + verbatim. When the file digests to the advertised identity it is the + artifact we want, so take it directly instead of parsing and + reserializing a bundle that would only have to hash back to the same + bytes. Returns None when the body is a compact reference instead. + """ + + expected = _canonical_hex( + expected_sha256, + name="audit_bundle_sha256", + ) + _path, payload = self._owned_external_body_bytes( + body_uri, + expected_sha256=expected, + ) + if not hmac.compare_digest(_sha256_bytes(payload), expected): + return None + self._validate_root_identity() + return payload + + def canonical_bundle_bytes_from_external_body( + self, + *, + block_hash: str, + audit_bundle_sha256: str, + body_uri: object, + load_missing_range: Callable[..., list[Any]] | None = None, + ) -> bytes: + """Recover the canonical bytes for an already published body. + + Built for the historical backfill: it takes the pre-v2 literal-body + shortcut when it applies, and otherwise reconstructs the logical bundle + from the compact reference, repairing share segments that have since + left the disk through `load_missing_range`. Recovered shares are + canonicalized and checked against the digest the body itself already + commits to, and the assembled bundle is checked against the advertised + bundle digest, so database fallback bytes are never trusted on their + own. + """ + + block_hash = _canonical_hex(block_hash, name="block_hash") + expected = _canonical_hex( + audit_bundle_sha256, + name="audit_bundle_sha256", + ) + path, body_bytes = self._owned_external_body_bytes( + body_uri, + expected_sha256=expected, + block_hash=block_hash, + ) + if hmac.compare_digest(_sha256_bytes(body_bytes), expected): + self._validate_root_identity() + return body_bytes + try: + body = json.loads(body_bytes.decode("utf-8")) + except (UnicodeDecodeError, json.JSONDecodeError) as exc: + raise RuntimeError( + f"audit bundle body is not valid JSON at {path}: {exc}" + ) from exc + schema = body.get("schema") if isinstance(body, dict) else None + if schema == AUDIT_BODY_REF_SCHEMA: + bundle = self.resolve_audit_body_ref( + body, + expected_sha256=expected, + body_uri=str(path), + load_missing_range=load_missing_range, + ) + elif schema == AUDIT_BUNDLE_V2_SCHEMA: + bundle = self.resolve_audit_bundle_v2( + body, + expected_sha256=expected, + body_uri=str(path), + load_missing_range=load_missing_range, + ) + else: + raise RuntimeError( + f"audit bundle body is not a canonical bundle source at {path}" + ) + canonical = self.canonical_audit_bundle_bytes(bundle) + actual = _sha256_bytes(canonical) + if not hmac.compare_digest(actual, expected): + raise RuntimeError( + f"audit bundle body hash mismatch at {path}: " + f"expected {expected}, got {actual}" + ) + self._validate_root_identity() + return canonical + + def _read_share_part_with_recovery( + self, + part: Mapping[str, Any], + *, + parent_body_uri: object, + load_missing_range: Callable[..., list[Any]] | None, + ) -> list[Any]: + """Read one share part, repairing a segment that has left the disk. + + The digest the compact body already commits to is the only authority + for a recovered range, so a callback result is accepted solely because + it canonicalizes back to that digest. + """ + + try: + return self.read_audit_share_segment( + part, + parent_body_uri=parent_body_uri, + ) + except (OSError, RuntimeError) as exc: + if load_missing_range is None: + raise + unavailable = exc + kind = str(part.get("kind") or "") + digest_key = { + "segment": "sha256", + "segment_range": "range_sha256", + "segment_prefix": "prefix_sha256", + }.get(kind) + if digest_key is None: + raise unavailable + try: + expected = _canonical_hex(part.get(digest_key), name="share segment digest") + first_share_seq = int(part.get("first_share_seq") or 0) + last_share_seq = int(part.get("last_share_seq") or 0) + share_count = int(part.get("share_count") or 0) + except (TypeError, ValueError) as exc: + raise RuntimeError( + f"audit bundle body is not valid JSON at {parent_body_uri}: " + f"invalid share part: {exc}" + ) from unavailable + try: + recovered = list( + load_missing_range( + first_share_seq=first_share_seq, + last_share_seq=last_share_seq, + ) + ) + except _AuditShareSegmentConflict: + raise + except Exception as exc: + raise RuntimeError( + f"audit bundle body is not retrievable at {parent_body_uri}: " + f"share segment {part.get('body_uri')} could not be recovered: {exc}" + ) from unavailable + self._validate_share_range(first_share_seq, last_share_seq, recovered) + if len(recovered) != share_count: + raise RuntimeError( + f"audit bundle body is not valid JSON at {parent_body_uri}: " + f"recovered share segment {part.get('body_uri')} share count mismatch" + ) + actual = _sha256_bytes( + self.storage_json_bytes( + self.audit_share_segment_payload( + first_share_seq=first_share_seq, + last_share_seq=last_share_seq, + shares=recovered, + ) + ) + ) + if not hmac.compare_digest(actual, expected): + raise RuntimeError( + f"audit bundle body hash mismatch at {parent_body_uri}: " + f"recovered share segment {part.get('body_uri')} " + f"expected {expected}, got {actual}" + ) + return recovered + + @staticmethod + def storage_json_bytes(payload: Mapping[str, Any]) -> bytes: + return _json_bytes(payload) + + def audit_share_segment_payload( + self, + *, + first_share_seq: int, + last_share_seq: int, + shares: Sequence[Any], + ) -> dict[str, Any]: + return { + "schema": AUDIT_SHARE_SEGMENT_SCHEMA, + "first_share_seq": first_share_seq, + "last_share_seq": last_share_seq, + "share_count": len(shares), + "shares": shares, + } + + @staticmethod + def _storage_json_chunks(payload: Mapping[str, Any]) -> Callable[[], Iterator[bytes]]: + """A replayable factory of :meth:`storage_json_bytes` in bounded chunks. + + The stream reproduces ``storage_json_bytes(payload)`` exactly while + any lazy share sequence inside ``payload`` is encoded batch by batch + instead of being materialized; a plain list of records encodes the + same way. + """ + return lambda: iter_json_byte_chunks(payload) + + def write_audit_share_segment( + self, + *, + first_share_seq: int, + last_share_seq: int, + shares: Sequence[Any], + ) -> tuple[str, str]: + self._validate_share_range(first_share_seq, last_share_seq, shares) + segment = self.audit_share_segment_payload( + first_share_seq=first_share_seq, + last_share_seq=last_share_seq, + shares=shares, + ) + chunks = self._storage_json_chunks(segment) + digest, size = json_chunks_sha256_and_size(chunks()) + path = self._root / ( + f"prism-audit-share-segment-{first_share_seq}-{last_share_seq}-{digest}.json" + ) + if not _SHARE_CONTENT_RE.fullmatch(path.name): + raise RuntimeError("invalid audit share segment bounds") + self._write_immutable_chunks(path, chunks, size) + self._validate_root_identity() + return str(path), digest + + def write_audit_share_segment_range( + self, + *, + segment_first_share_seq: int, + segment_last_share_seq: int, + first_share_seq: int, + last_share_seq: int, + shares: Sequence[Any], + load_missing_range: Callable[..., list[Any]] | None = None, + ) -> tuple[str, str]: + if not shares: + raise RuntimeError("audit share segment range cannot be empty") + if ( + segment_first_share_seq <= 0 + or segment_last_share_seq < segment_first_share_seq + or first_share_seq < segment_first_share_seq + or last_share_seq > segment_last_share_seq + ): + raise RuntimeError("audit share range is outside its segment slot") + self._validate_share_range(first_share_seq, last_share_seq, shares) + path = self._root / ( + "prism-audit-share-segment-slot-" + f"{segment_first_share_seq}-{segment_last_share_seq}.json" + ) + if not _SHARE_SLOT_RE.fullmatch(path.name): + raise RuntimeError("invalid audit share segment slot bounds") + incoming = self.audit_share_segment_payload( + first_share_seq=first_share_seq, + last_share_seq=last_share_seq, + shares=shares, + ) + # Digest the incoming range in bounded chunks. Completed 10k slots + # dominate normal block persistence; byte equality lets them return + # without parsing, merging, deep-copying, or serializing the existing + # slot tree, and the chunk stream is replayed rather than held whole. + incoming_chunks = self._storage_json_chunks(incoming) + range_digest, incoming_size = json_chunks_sha256_and_size(incoming_chunks()) + with self._lock: + self._validate_root_identity() + if self._file_matches_chunks(path, incoming_chunks, incoming_size): + self._fsync_directory(path.parent) + return str(path), range_digest + existing_bytes: bytes | None = None + merged: list[Any] | None = shares + try: + value = self._owned_lstat(path) + except FileNotFoundError: + value = None + if value is not None: + if not stat.S_ISREG(value.st_mode): + raise RuntimeError( + f"existing audit share segment is not regular at {path}" + ) + try: + existing_bytes, existing_descriptor = self._read_owned_regular_bytes( + path + ) + if ( + existing_descriptor.st_dev != value.st_dev + or existing_descriptor.st_ino != value.st_ino + or existing_descriptor.st_mtime_ns != value.st_mtime_ns + ): + raise OSError("share segment identity changed") + existing = json.loads(existing_bytes) + except (OSError, UnicodeDecodeError, json.JSONDecodeError) as exc: + raise RuntimeError( + f"existing audit share segment is not valid JSON at {path}" + ) from exc + if ( + not isinstance(existing, dict) + or existing.get("schema") != AUDIT_SHARE_SEGMENT_SCHEMA + or not isinstance(existing.get("shares"), list) + ): + raise RuntimeError( + f"existing audit share segment has invalid schema at {path}" + ) + merged = self.merge_audit_share_ranges( + existing["shares"], + shares, + segment_path=path, + load_missing_range=load_missing_range, + ) + if merged is None: + # Keep prior body_uri references valid at the stable slot + # path, while the new bundle uses an immutable + # incoming-only segment. + fresh_uri, fresh_digest = self.write_audit_share_segment( + first_share_seq=first_share_seq, + last_share_seq=last_share_seq, + shares=shares, + ) + if fresh_digest != range_digest: + raise RuntimeError( + "audit share segment range hash changed during fallback" + ) + assert existing_bytes is not None + try: + self.quarantine_audit_share_segment( + path, + expected_bytes=existing_bytes, + ) + except (OSError, RuntimeError): + # The stable slot already preserves the old range, and + # the immutable file preserves the incoming one. A + # quarantine snapshot failure must not wedge block + # finalization. + pass + return fresh_uri, range_digest + segment_first = int(merged[0]["share_seq"]) + segment_last = int(merged[-1]["share_seq"]) + if ( + segment_first == first_share_seq + and segment_last == last_share_seq + and len(merged) == len(shares) + ): + segment_chunks = incoming_chunks + else: + segment = self.audit_share_segment_payload( + first_share_seq=segment_first, + last_share_seq=segment_last, + shares=merged, + ) + segment_chunks = self._storage_json_chunks(segment) + if existing_bytes is None or not _bytes_equal_chunks( + existing_bytes, + segment_chunks(), + ): + # Streamed chunk by chunk: a slot never becomes one bytes + # object here, whatever a single record's size. + self._write_mutable_chunks(path, segment_chunks) + else: + self._fsync_directory(path.parent) + self._validate_root_identity() + return str(path), range_digest + + @staticmethod + def _validate_share_range( + first_share_seq: int, + last_share_seq: int, + shares: Sequence[Any], + ) -> None: + if first_share_seq <= 0 or last_share_seq < first_share_seq or not shares: + raise RuntimeError("audit share range bounds are invalid") + # Walked once, record by record: a lazy slice streams from the + # artifact and never becomes a list of sequence numbers. + previous: int | None = None + first: int | None = None + try: + for share in shares: + share_seq = int(share["share_seq"]) + if first is None: + first = share_seq + elif previous is not None and previous + 1 != share_seq: + raise RuntimeError("audit share range is not exactly contiguous") + previous = share_seq + except (KeyError, TypeError, ValueError) as exc: + raise RuntimeError("audit share range has invalid share_seq") from exc + if first != first_share_seq or previous != last_share_seq: + raise RuntimeError("audit share range is not exactly contiguous") + + @staticmethod + def _is_share_sequence(value: object) -> bool: + """True for a list or any other replayable sequence of records.""" + return isinstance(value, Sequence) and not isinstance( + value, + (str, bytes, bytearray), + ) + + @staticmethod + def _contiguous_share_start(shares: Sequence[Any]) -> int | None: + """First share_seq when every record is a dict in strict +1 order. + + One streaming pass; None reproduces the historical "fall back to the + inline body-ref layout" verdict for a window that is not a + contiguous dictionary sequence. + """ + first: int | None = None + previous: int | None = None + for share in shares: + if not isinstance(share, dict): + return None + try: + share_seq = int(share["share_seq"]) + except (KeyError, TypeError, ValueError): + return None + if first is None: + first = share_seq + elif previous is not None and previous + 1 != share_seq: + return None + previous = share_seq + return first + + @staticmethod + def _iter_segment_slots( + first_seq: int, + count: int, + segment_size: int, + ) -> Iterator[tuple[int, int, int, int]]: + """(index, end_index, slot_start, slot_end) for a contiguous window. + + Because contiguity has been verified, record ``i`` carries sequence + ``first_seq + i``; the slot boundaries follow arithmetically without + a second pass over the records. + """ + index = 0 + while index < count: + seq = first_seq + index + slot_start = ((seq - 1) // segment_size) * segment_size + 1 + slot_end = slot_start + segment_size - 1 + end = min(count, index + (slot_end - seq + 1)) + yield index, end, slot_start, slot_end + index = end + + def merge_audit_share_ranges( + self, + existing_shares: list[Any], + incoming_shares: list[Any], + *, + segment_path: Path, + load_missing_range: Callable[..., list[Any]] | None = None, + ) -> list[Any] | None: + if not existing_shares: + return list(incoming_shares) + existing_by_seq = self.audit_shares_by_seq( + existing_shares, + segment_path=segment_path, + ) + incoming_by_seq = self.audit_shares_by_seq( + incoming_shares, + segment_path=segment_path, + ) + for share_seq, incoming in incoming_by_seq.items(): + existing = existing_by_seq.get(share_seq) + if existing is not None and existing != incoming: + raise _AuditShareSegmentConflict( + "existing audit share segment conflicts at share_seq " + f"{share_seq} in {segment_path}" + ) + merged_by_seq = {**existing_by_seq, **incoming_by_seq} + ordered_seqs = sorted(merged_by_seq) + if any(current + 1 != nxt for current, nxt in zip(ordered_seqs, ordered_seqs[1:])): + # A skipped finalize can leave an otherwise valid slot behind the + # incoming range. Repair that hole from the append-only ledger. + gap_before, gap_after = next( + (current, nxt) + for current, nxt in zip(ordered_seqs, ordered_seqs[1:]) + if current + 1 != nxt + ) + missing_first = gap_before + 1 + missing_last = gap_after - 1 + durable_shares: list[Any] | None = None + if load_missing_range is not None: + try: + durable_shares = load_missing_range( + first_share_seq=missing_first, + last_share_seq=missing_last, + ) + except _AuditShareSegmentConflict: + raise + except Exception: + # This read is a recovery aid; its failure must not turn + # the original slot gap into another permanent finalize + # failure. + durable_shares = None + if durable_shares is not None: + try: + durable_by_seq = self.audit_shares_by_seq( + durable_shares, + segment_path=segment_path, + require_contiguous=False, + ) + except _AuditShareSegmentConflict: + raise + except RuntimeError: + durable_by_seq = {} + if sorted(durable_by_seq) == list(range(missing_first, missing_last + 1)): + merged_by_seq.update(durable_by_seq) + ordered_seqs = sorted(merged_by_seq) + if not any( + current + 1 != nxt + for current, nxt in zip(ordered_seqs, ordered_seqs[1:]) + ): + return [merged_by_seq[share_seq] for share_seq in ordered_seqs] + # The incoming range is independently complete and remains usable. + # The caller preserves this slot and gives that range a fresh URI. + return None + return [merged_by_seq[share_seq] for share_seq in ordered_seqs] + + def quarantine_audit_share_segment( + self, + segment_path: Path, + *, + expected_bytes: bytes, + ) -> Path: + segment_path = Path(segment_path) + with self._lock: + self._validate_root_identity() + conflict_prefix = f"{segment_path.name}.conflict-" + for name in sorted(self._scan_root_names()): + if not name.startswith(conflict_prefix): + continue + if self.file_matches_bytes(self._root / name, expected_bytes): + return self._root / name + timestamp = time.time_ns() + quarantine_path = segment_path.with_name( + f"{segment_path.name}.conflict-{timestamp}" + ) + while True: + try: + self._owned_lstat(quarantine_path) + except FileNotFoundError: + break + timestamp += 1 + quarantine_path = segment_path.with_name( + f"{segment_path.name}.conflict-{timestamp}" + ) + # Snapshot rather than move: previously finalized bundles retain + # this stable slot URI, so it must remain readable across crashes + # and retries. + self._write_mutable_bytes(quarantine_path, expected_bytes) + return quarantine_path + + @staticmethod + def audit_shares_by_seq( + shares: list[Any], + *, + segment_path: Path, + require_contiguous: bool = True, + ) -> dict[int, Any]: + by_seq: dict[int, Any] = {} + for share in shares: + if not isinstance(share, dict): + raise RuntimeError( + f"audit share segment has invalid share payload at {segment_path}" + ) + try: + share_seq = int(share["share_seq"]) + except (KeyError, TypeError, ValueError) as exc: + raise RuntimeError( + f"audit share segment has invalid share_seq at {segment_path}" + ) from exc + existing = by_seq.get(share_seq) + if existing is not None and existing != share: + raise _AuditShareSegmentConflict( + "audit share segment has duplicate conflicting share_seq " + f"{share_seq} at {segment_path}" + ) + by_seq[share_seq] = share + ordered = sorted(by_seq) + if require_contiguous and any( + current + 1 != nxt for current, nxt in zip(ordered, ordered[1:]) + ): + raise RuntimeError( + f"audit share segment has non-contiguous share_seq values at {segment_path}" + ) + return by_seq + + @staticmethod + def _spool_share_parts( + parts: Iterable[Mapping[str, Any]], + scratch: list[CanonicalAuditBundleView] | None, + ) -> Sequence[Any]: + """The share-part index as a replayable on-disk sequence. + + Parts are written to an unlinked scratch document as they are + produced -- an inline part's lazy slice streams batch by batch -- + and read back lazily, so a small configured segment size cannot + turn the index into a window-scaled list. The scratch view is + appended to ``scratch`` for the caller to close; without a scratch + list it retires with the last reference to the sequence. + """ + + def chunks() -> Iterator[bytes]: + yield b'{"share_parts":[' + first = True + for part in parts: + if not first: + yield b"," + first = False + yield from iter_json_byte_chunks(part) + yield b"]}" + + view = spool_json_document(chunks(), lazy_paths=(("share_parts",),)) + if scratch is not None: + scratch.append(view) + return view["share_parts"] + + @staticmethod + def share_parts_digest_hex(parts: Sequence[Any]) -> str: + """``sha256(storage_json_bytes({"share_parts": parts}))``, streamed.""" + digest, _size = json_chunks_sha256_and_size( + iter_json_byte_chunks({"share_parts": parts}) + ) + return digest + + def _iter_share_range_parts( + self, + shares: Sequence[Any], + first_seq: int, + *, + load_missing_range: Callable[..., list[Any]] | None, + ) -> Iterator[dict[str, Any]]: + segment_size = self._share_segment_size + for index, end, segment_start, segment_end in self._iter_segment_slots( + first_seq, + len(shares), + segment_size, + ): + chunk = shares[index:end] + chunk_first = first_seq + index + chunk_last = first_seq + end - 1 + uri, digest = self.write_audit_share_segment_range( + segment_first_share_seq=segment_start, + segment_last_share_seq=segment_end, + first_share_seq=chunk_first, + last_share_seq=chunk_last, + shares=chunk, + load_missing_range=load_missing_range, + ) + yield { + "kind": "segment_range", + "segment_first_share_seq": segment_start, + "segment_last_share_seq": segment_end, + "first_share_seq": chunk_first, + "last_share_seq": chunk_last, + "share_count": end - index, + "range_sha256": digest, + "body_uri": uri, + } + + def audit_share_range_parts( + self, + shares: Sequence[Any], + *, + load_missing_range: Callable[..., list[Any]] | None = None, + scratch: list[CanonicalAuditBundleView] | None = None, + ) -> Sequence[Any] | None: + """Slot-range parts for a contiguous window, one bounded slice each. + + Accepts a list or a lazy artifact sequence. The window is walked + once to prove contiguity, then each slot's records are handed to the + segment writer as a slice bounded by the configured segment size; + no list of sequence numbers or of records spanning the whole window + is built, and the parts themselves are spooled to disk as they are + produced and returned as a replayable sequence. + """ + if self._share_segment_size <= 0: + return None + if not self._is_share_sequence(shares): + return None + first_seq = self._contiguous_share_start(shares) + if first_seq is None: + return None + return self._spool_share_parts( + self._iter_share_range_parts( + shares, + first_seq, + load_missing_range=load_missing_range, + ), + scratch, + ) + + def _iter_share_parts( + self, + shares: Sequence[Any], + first_seq: int, + ) -> Iterator[dict[str, Any]]: + segment_size = self._share_segment_size + for index, end, segment_start, segment_end in self._iter_segment_slots( + first_seq, + len(shares), + segment_size, + ): + chunk = shares[index:end] + chunk_first = first_seq + index + chunk_last = first_seq + end - 1 + if ( + end - index == segment_size + and chunk_first == segment_start + and chunk_last == segment_end + ): + uri, digest = self.write_audit_share_segment( + first_share_seq=segment_start, + last_share_seq=segment_end, + shares=chunk, + ) + yield { + "kind": "segment", + "first_share_seq": segment_start, + "last_share_seq": segment_end, + "share_count": end - index, + "sha256": digest, + "body_uri": uri, + } + else: + yield { + "kind": "inline", + "first_share_seq": chunk_first, + "last_share_seq": chunk_last, + "share_count": end - index, + # A lazy slice stays lazy here; the spool encoder + # materializes it batch by batch. + "shares": chunk, + } + + def audit_share_parts( + self, + shares: Sequence[Any], + *, + scratch: list[CanonicalAuditBundleView] | None = None, + ) -> Sequence[Any] | None: + if self._share_segment_size <= 0: + return None + if not self._is_share_sequence(shares): + return None + first_seq = self._contiguous_share_start(shares) + if first_seq is None: + return None + return self._spool_share_parts(self._iter_share_parts(shares, first_seq), scratch) + + def audit_body_ref( + self, + *, + block_hash: str, + audit_bundle_sha256: str, + final_bundle: Mapping[str, Any], + scratch: list[CanonicalAuditBundleView] | None = None, + ) -> dict[str, Any] | None: + if self._share_segment_size <= 0: + return None + shares = final_bundle.get("shares") + if not self._is_share_sequence(shares) or not shares: + return None + parts = self.audit_share_parts(shares, scratch=scratch) + if parts is None or not any(part.get("kind") == "segment" for part in parts): + return None + without_shares = {key: value for key, value in final_bundle.items() if key != "shares"} + return { + "schema": AUDIT_BODY_REF_SCHEMA, + "block_hash": block_hash, + "audit_bundle_sha256": audit_bundle_sha256, + "audit_bundle_schema": str(final_bundle.get("schema") or ""), + "share_count": len(shares), + "share_segment_size": self._share_segment_size, + "shares_key_index": list(final_bundle).index("shares"), + "bundle_without_shares": without_shares, + "share_parts": parts, + } + + def audit_bundle_v2( + self, + *, + block_hash: str, + audit_bundle_sha256: str, + final_bundle: Mapping[str, Any], + load_missing_range: Callable[..., list[Any]] | None = None, + scratch: list[CanonicalAuditBundleView] | None = None, + ) -> dict[str, Any] | None: + if self._share_segment_size <= 0: + return None + shares = final_bundle.get("shares") + if not self._is_share_sequence(shares) or not shares: + return None + parts = self.audit_share_range_parts( + shares, + load_missing_range=load_missing_range, + scratch=scratch, + ) + if parts is None: + return None + proof: dict[str, Any] = { + "schema": AUDIT_WINDOW_COMPLETENESS_PROOF_SCHEMA, + "share_segment_size": self._share_segment_size, + "first_share_seq": int(shares[0]["share_seq"]), + "last_share_seq": int(shares[-1]["share_seq"]), + "share_count": len(shares), + # The parts serialization is hashed incrementally from the + # on-disk index; it equals the reader's whole-encoding digest. + "share_parts_digest_hex": self.share_parts_digest_hex(parts), + "share_parts": parts, + } + reward = final_bundle.get("reward_manifest") + if isinstance(reward, dict): + for key in ( + "anchor_job_issued_at_ms", + "anchor_share_seq", + "newest_share_seq", + "oldest_share_seq", + "included_share_count", + "requested_window_weight", + "counted_window_weight", + "share_slice_digest_hex", + ): + if key in reward: + proof[key] = reward[key] + return { + "schema": AUDIT_BUNDLE_V2_SCHEMA, + "block_hash": block_hash, + "audit_bundle_sha256": audit_bundle_sha256, + "logical_audit_bundle_schema": str(final_bundle.get("schema") or ""), + "share_count": len(shares), + "shares_key_index": list(final_bundle).index("shares"), + "bundle_without_shares": { + key: value for key, value in final_bundle.items() if key != "shares" + }, + "share_window_proof": proof, + } + + def externalize_audit_body( + self, + block_hash: str, + audit_bundle_sha256: str, + final_bundle: Mapping[str, Any], + ) -> str: + block_hash = _canonical_hex(block_hash, name="block_hash") + digest = _canonical_hex( + audit_bundle_sha256, + name="audit_bundle_sha256", + ) + path = self.body_path(block_hash, digest) + if isinstance(final_bundle, CanonicalAuditBundleView): + if not hmac.compare_digest(final_bundle.sha256_hex, digest): + raise RuntimeError( + "audit bundle sha256 mismatch: " + f"expected {digest}, got {final_bundle.sha256_hex}" + ) + final_bundle.verify_identity() + self._write_immutable_chunks( + path, + final_bundle.iter_bytes, + final_bundle.byte_length, + ) + else: + body = self.canonical_audit_body_bytes_for_sha(final_bundle, digest) + self._write_immutable_bytes(path, body) + self._validate_root_identity() + return str(path) + + def prepare_external_audit_body( + self, + payload: Mapping[str, Any], + final_bundle: Mapping[str, Any], + *, + body_uri: str | None, + canonical_bundle_path: Path | None = None, + load_missing_range: Callable[..., list[Any]] | None = None, + ) -> str | None: + """Publish the canonical bundle and its external body for one block. + + Bounded end to end (issue #255): the canonical candidate is opened + under the store's directory authority and digested, compared and + compressed through bounded chunks; a :class:`CanonicalAuditBundleView` + scanned from that very inode is accepted by identity, any other + logical bundle is compared member by member with the window arrays + streamed; compact share segments are cut from bounded slices; the + body document is encoded chunk by chunk; and every verification of + an existing or freshly published body streams the same way. The + digest, signature and coinbase checks, exact inode identity binding + and immutable-publication semantics are unchanged. + """ + block_hash = _canonical_hex(payload["block_hash"], name="block_hash") + expected = _canonical_hex( + payload["audit_bundle_sha256"], + name="audit_bundle_sha256", + ) + if body_uri is None: + return None + body_path = self.resolve_owned_path(body_uri) + canonical_path = self.body_path(block_hash, expected) + if body_path != canonical_path: + raise RuntimeError( + "existing audit bundle body pointer does not match canonical external path: " + f"{body_uri}" + ) + canonical: ArtifactSource | None = None + owned_canonical = False + canonical_bytes: bytes | None = None + source_identity: _FileIdentity | None = None + # Scratch views (the on-disk share-part index) retire with this call. + scratch: list[CanonicalAuditBundleView] = [] + try: + if canonical_bundle_path is not None: + source_path = Path(canonical_bundle_path) + try: + canonical = self._open_owned_artifact_source(source_path) + except OSError as exc: + raise RuntimeError( + f"canonical audit bundle is not retrievable at {source_path}: {exc}" + ) from exc + owned_canonical = True + source_identity = _FileIdentity( + canonical.identity.device, + canonical.identity.inode, + canonical.identity.mode, + canonical.identity.size, + canonical.identity.mtime_ns, + ) + actual = canonical.sha256_hex() + if not hmac.compare_digest(actual, expected): + raise RuntimeError( + f"audit bundle sha256 mismatch: expected {expected}, got {actual}" + ) + # Compact storage is derived from final_bundle, not necessarily + # from the supplied canonical path. Bind those logical inputs + # independently -- by identity for a view of this inode, + # otherwise by a streamed member-by-member comparison. + self._require_logical_match(canonical, final_bundle) + elif isinstance(final_bundle, CanonicalAuditBundleView): + if not hmac.compare_digest(final_bundle.sha256_hex, expected): + raise RuntimeError( + "audit bundle sha256 mismatch: " + f"expected {expected}, got {final_bundle.sha256_hex}" + ) + final_bundle.verify_identity() + canonical = final_bundle.source + else: + canonical_bytes = self.canonical_audit_body_bytes_for_sha( + final_bundle, + expected, + ) + # Persist the exact bytes the public digest is taken over, reusing + # the source already in hand. This runs before the compact body so + # every path through this call publishes it, including the ones + # that return early on an existing body. The caller's lease + # preflight has already completed, so this is still a fenced write. + if canonical is not None: + self.write_canonical_audit_bundle_from_source( + block_hash, + expected, + canonical, + ) + else: + assert canonical_bytes is not None + self.write_canonical_audit_bundle(block_hash, expected, canonical_bytes) + storage = self.audit_bundle_v2( + block_hash=block_hash, + audit_bundle_sha256=expected, + final_bundle=final_bundle, + load_missing_range=load_missing_range, + scratch=scratch, + ) + if storage is None: + storage = self.audit_body_ref( + block_hash=block_hash, + audit_bundle_sha256=expected, + final_bundle=final_bundle, + scratch=scratch, + ) + body_size: int | None + if storage is not None: + body_chunks = self._storage_json_chunks(storage) + body_size = None + elif canonical is not None: + body_chunks = canonical.iter_bytes + body_size = canonical.identity.size + else: + assert canonical_bytes is not None + literal = canonical_bytes + body_chunks = lambda: (literal,) # noqa: E731 - replayable single chunk + body_size = len(literal) + try: + existing = self._owned_lstat(body_path) + except FileNotFoundError: + existing = None + if existing is not None: + if not stat.S_ISREG(existing.st_mode): + raise RuntimeError( + f"existing audit bundle body is not regular at {body_path}" + ) + if storage is not None and self._file_matches_chunks( + body_path, + body_chunks, + body_size, + ): + if self._compact_body_reconstructs_to( + body_path, + expected=expected, + final_bundle=final_bundle, + ): + self._fsync_directory(body_path.parent) + return str(body_path) + raise RuntimeError( + f"existing audit bundle body does not match payload at {body_path}" + ) + if storage is None and self._literal_body_matches_sha(body_path, expected): + self._fsync_directory(body_path.parent) + return str(body_path) + # Layout upgrades may represent the same canonical logical + # bundle with different storage bytes. Verify by a streamed + # comparison against the logical bundle (a compact body) or by + # digest (a literal body). + if not self._existing_body_matches( + body_path, + expected=expected, + final_bundle=final_bundle, + ): + raise RuntimeError( + f"existing audit bundle body does not match payload at {body_path}" + ) + self._fsync_directory(body_path.parent) + return str(body_path) + self._write_immutable_chunks(body_path, body_chunks, body_size) + if canonical_bundle_path is not None and source_identity is not None: + source_after = self._owned_lstat(Path(canonical_bundle_path)) + if not source_identity.matches(source_after): + raise RuntimeError( + "canonical audit bundle identity changed after publication" + ) + # The compact storage bytes were derived from the already verified + # logical bundle. Validate the exact destination here, streaming + # the segments against the logical window. + if storage is not None: + valid = self._compact_body_reconstructs_to( + body_path, + expected=expected, + final_bundle=final_bundle, + ) + else: + valid = self._literal_body_matches_sha(body_path, expected) + if not valid: + raise RuntimeError("published audit bundle body failed digest verification") + self._validate_root_identity() + return str(body_path) + finally: + for view in scratch: + view.close() + if owned_canonical and canonical is not None: + canonical.close() + + def _literal_body_matches_sha(self, body_path: Path, expected: str) -> bool: + """True when the owned body file digests to ``expected``, streamed.""" + try: + source = self._open_owned_artifact_source(body_path) + except OSError: + return False + try: + return hmac.compare_digest(source.sha256_hex(), expected) + except (OSError, CanonicalArtifactError): + return False + finally: + source.close() + + def _existing_body_matches( + self, + body_path: Path, + *, + expected: str, + final_bundle: Mapping[str, Any], + ) -> bool: + """A published body of any layout represents ``final_bundle``. + + Compact layouts are compared record by record through their + segments; a literal body is verified by its digest. Nothing is + reconstructed into a whole-window object graph. + """ + if self._compact_body_reconstructs_to( + body_path, + expected=expected, + final_bundle=final_bundle, + ): + return True + return self._literal_body_matches_sha(body_path, expected) + + # Every window-, transaction- or recipient-scaled member of a compact + # body stays lazy: the header mirrors the artifact's lazy paths and the + # share-part index scales with the window over the segment size. + _BODY_LAZY_PATHS: tuple[tuple[str, ...], ...] = ( + ("bundle_without_shares", "shares"), + ("bundle_without_shares", "reward_manifest", "shares"), + ("bundle_without_shares", "reward_manifest", "entitlements"), + ("bundle_without_shares", "witness_merkle_leaves_hex"), + ("bundle_without_shares", "audit_commitment_leaves_hex"), + ("bundle_without_shares", "prior_balances"), + ("bundle_without_shares", "payout_policy_manifest", "accounts"), + ("bundle_without_shares", "payout_policy_manifest", "onchain_entitlements"), + ("bundle_without_shares", "settlement_mode_decision", "direct_recipients"), + ("bundle_without_shares", "settlement_mode_decision", "fanout_chunks"), + ("bundle_without_shares", "ctv_fanout_manifest_set", "manifests"), + ("share_window_proof", "share_parts"), + ("share_parts",), + ) + + def _compact_body_reconstructs_to( + self, + body_path: Path, + *, + expected: str, + final_bundle: Mapping[str, Any], + ) -> bool: + """The compact body at ``body_path`` reconstructs exactly to ``final_bundle``. + + Streams instead of reconstructing: the body is scanned with its + window-sized members left lazy, the header is compared member by + member (the reward manifest's counted shares record by record), and + every share part is read one bounded segment at a time and matched + against the corresponding records of the logical window. The same + wrapper identity, proof and part validations apply as in the + reconstruction readers; no recovery callback is consulted, so a + segment missing from disk is a mismatch exactly as before. + """ + try: + body = self._scan_owned_artifact(body_path, lazy_paths=self._BODY_LAZY_PATHS) + except (OSError, RuntimeError, CanonicalArtifactError): + return False + try: + schema = body.get("schema") + if schema == AUDIT_BODY_REF_SCHEMA: + return self._body_ref_matches( + body, + body_path=body_path, + expected=expected, + final_bundle=final_bundle, + ) + if schema == AUDIT_BUNDLE_V2_SCHEMA: + return self._bundle_v2_matches( + body, + body_path=body_path, + expected=expected, + final_bundle=final_bundle, + ) + return False + except ( + OSError, + RuntimeError, + TypeError, + ValueError, + UnicodeDecodeError, + json.JSONDecodeError, + ): + return False + finally: + body.close() + + def _header_matches( + self, + body: Mapping[str, Any], + final_bundle: Mapping[str, Any], + ) -> bool: + without_shares = body.get("bundle_without_shares") + if not isinstance(without_shares, Mapping): + return False + expected_without = { + key: value for key, value in final_bundle.items() if key != "shares" + } + if without_shares != expected_without: + return False + keys = list(final_bundle) + expected_index = keys.index("shares") if "shares" in keys else len(keys) + return int(body.get("shares_key_index", len(without_shares))) == expected_index + + def _parts_match_window( + self, + parts: Sequence[Any], + *, + body_path: Path, + final_bundle: Mapping[str, Any], + allow_inline: bool, + ) -> tuple[int, int | None, int | None] | None: + """Match ordered share parts against the logical window, streaming. + + Returns (count, first_share_seq, last_share_seq) of the matched + records, or None on any mismatch. Each segment part is read + bounded by the segment size; inline parts are bounded by the body. + """ + expected_shares = final_bundle.get("shares") + if not self._is_share_sequence(expected_shares): + return None + expected_iter = iter(expected_shares) + count = 0 + first_seq: int | None = None + last_seq: int | None = None + previous_last_share_seq: int | None = None + for part in parts: + if isinstance(part, RawJsonRecord): + # An inline part above the in-process limit: the part is + # the unit here, so it is decoded whole (documented). + part = materialize_json(part) + if not isinstance(part, Mapping): + return None + kind = part.get("kind") + segment_view: CanonicalAuditBundleView | None = None + if kind in {"segment", "segment_range", "segment_prefix"}: + segment_view, part_shares = self._open_share_segment_slice( + part, + parent_body_uri=str(body_path), + ) + elif ( + allow_inline + and kind == "inline" + and self._is_share_sequence(part.get("shares")) + ): + inline = part["shares"] + if len(inline) != int(part.get("share_count") or 0): + return None + self._validate_share_range( + int(part.get("first_share_seq") or 0), + int(part.get("last_share_seq") or 0), + inline, + ) + part_shares = inline + else: + return None + try: + first_share_seq = int(part.get("first_share_seq") or 0) + last_share_seq = int(part.get("last_share_seq") or 0) + if ( + previous_last_share_seq is not None + and first_share_seq <= previous_last_share_seq + ): + return None + previous_last_share_seq = last_share_seq + for record in part_shares: + try: + expected_record = next(expected_iter) + except StopIteration: + return None + if record != expected_record: + return None + if isinstance(record, dict): + try: + seq = int(record.get("share_seq") or 0) + except (TypeError, ValueError): + seq = 0 + if first_seq is None: + first_seq = seq + last_seq = seq + count += 1 + finally: + if segment_view is not None: + segment_view.close() + try: + next(expected_iter) + except StopIteration: + return count, first_seq, last_seq + return None + + def _body_ref_matches( + self, + body: Mapping[str, Any], + *, + body_path: Path, + expected: str, + final_bundle: Mapping[str, Any], + ) -> bool: + declared = _canonical_hex( + body.get("audit_bundle_sha256"), + name="audit_bundle_sha256", + ) + self._validate_body_wrapper_identity(body, str(body_path), declared) + if not hmac.compare_digest(declared, expected): + return False + parts = body.get("share_parts") + if not self._is_share_sequence(parts) or not self._header_matches(body, final_bundle): + return False + matched = self._parts_match_window( + parts, + body_path=body_path, + final_bundle=final_bundle, + allow_inline=True, + ) + if matched is None: + return False + return matched[0] == int(body.get("share_count") or 0) + + def _bundle_v2_matches( + self, + body: Mapping[str, Any], + *, + body_path: Path, + expected: str, + final_bundle: Mapping[str, Any], + ) -> bool: + declared = _canonical_hex( + body.get("audit_bundle_sha256"), + name="audit_bundle_sha256", + ) + self._validate_body_wrapper_identity(body, str(body_path), declared) + if not hmac.compare_digest(declared, expected): + return False + proof = body.get("share_window_proof") + if not isinstance(proof, dict): + return False + if proof.get("schema") != AUDIT_WINDOW_COMPLETENESS_PROOF_SCHEMA: + return False + if int(body.get("share_count") or 0) != int(proof.get("share_count") or 0): + return False + parts = proof.get("share_parts") + if not self._is_share_sequence(parts): + return False + expected_parts_digest = str(proof.get("share_parts_digest_hex") or "").lower() + actual_parts_digest = self.share_parts_digest_hex(parts) + if expected_parts_digest != actual_parts_digest: + return False + if not self._header_matches(body, final_bundle): + return False + matched = self._parts_match_window( + parts, + body_path=body_path, + final_bundle=final_bundle, + allow_inline=False, + ) + if matched is None: + return False + count, first_seq, last_seq = matched + if count != int(proof.get("share_count") or 0): + return False + if count and ( + (first_seq or 0) != int(proof.get("first_share_seq") or 0) + or (last_seq or 0) != int(proof.get("last_share_seq") or 0) + ): + return False + reward_manifest = final_bundle.get("reward_manifest") + copied_proof_fields = ( + "anchor_job_issued_at_ms", + "anchor_share_seq", + "newest_share_seq", + "oldest_share_seq", + "included_share_count", + "requested_window_weight", + "counted_window_weight", + "share_slice_digest_hex", + ) + if isinstance(reward_manifest, Mapping): + for field in copied_proof_fields: + if (field in proof) != (field in reward_manifest) or proof.get( + field + ) != reward_manifest.get(field): + return False + elif any(field in proof for field in copied_proof_fields): + return False + return True + + def validate_canonical_source( + self, + path: Path, + expected_sha256: str, + final_bundle: Mapping[str, Any] | None = None, + ) -> None: + expected = _canonical_hex( + expected_sha256, + name="audit_bundle_sha256", + ) + try: + source = self._open_owned_artifact_source(Path(path)) + except OSError as exc: + raise RuntimeError( + f"canonical audit bundle is not retrievable at {path}: {exc}" + ) from exc + try: + actual = source.sha256_hex() + if not hmac.compare_digest(actual, expected): + raise RuntimeError( + f"audit bundle sha256 mismatch: expected {expected}, got {actual}" + ) + if final_bundle is not None: + self._require_logical_match(source, final_bundle) + finally: + source.close() + self._validate_root_identity() + + def audit_body_byte_len( + self, + body_uri: object | None, + final_bundle: Mapping[str, Any], + canonical_bundle_path: Path | None = None, + ) -> int: + if body_uri: + value = self._owned_regular_size(self.resolve_owned_path(body_uri)) + self._validate_root_identity() + return value + if canonical_bundle_path is not None: + value = self._owned_regular_size(Path(canonical_bundle_path)) + self._validate_root_identity() + return value + if isinstance(final_bundle, CanonicalAuditBundleView): + final_bundle.verify_identity() + value = final_bundle.byte_length + else: + value = len(self.canonical_audit_bundle_bytes(final_bundle)) + self._validate_root_identity() + return value + + def _owned_regular_size(self, path: Path) -> int: + """Byte length of an owned regular file from its descriptor, unread.""" + source = self._open_owned_artifact_source(path) + try: + return source.identity.size + finally: + source.close() + + def read_external_body( + self, + body_uri: object, + *, + expected_sha256: object | None = None, + ) -> dict[str, object] | None: + if not body_uri: + return None + try: + path = self.resolve_owned_path(body_uri) + body_match = _BODY_RE.fullmatch(path.name) + if body_match is None: + raise RuntimeError("body URI is not an owned canonical body") + if expected_sha256 is not None and body_match.group("digest") != _canonical_hex( + expected_sha256, + name="audit_bundle_sha256", + ): + raise RuntimeError("body URI digest does not match expected digest") + body_bytes, _value = self._read_owned_regular_bytes(path) + except (OSError, RuntimeError, TypeError, ValueError, OverflowError) as exc: + raise RuntimeError( + f"audit bundle body is not retrievable at {body_uri}: {exc}" + ) from exc + try: + body = json.loads(body_bytes.decode("utf-8")) + except (UnicodeDecodeError, json.JSONDecodeError) as exc: + raise RuntimeError( + f"audit bundle body is not valid JSON at {body_uri}: {exc}" + ) from exc + try: + if isinstance(body, dict) and body.get("schema") == AUDIT_BODY_REF_SCHEMA: + resolved = self.resolve_audit_body_ref( + body, + expected_sha256=expected_sha256, + body_uri=body_uri, + ) + self._validate_root_identity() + return resolved + if isinstance(body, dict) and body.get("schema") == AUDIT_BUNDLE_V2_SCHEMA: + resolved = self.resolve_audit_bundle_v2( + body, + expected_sha256=expected_sha256, + body_uri=body_uri, + ) + self._validate_root_identity() + return resolved + if expected_sha256: + expected = _canonical_hex( + expected_sha256, + name="audit_bundle_sha256", + ) + actual = _sha256_bytes(body_bytes) + if not hmac.compare_digest(actual, expected): + raise RuntimeError( + f"audit bundle body hash mismatch at {body_uri}: " + f"expected {expected}, got {actual}" + ) + self._validate_root_identity() + return body + except RuntimeError: + raise + except (TypeError, ValueError, OverflowError) as exc: + raise RuntimeError( + f"audit bundle body is not valid JSON at {body_uri}: {exc}" + ) from exc + + def external_body_matches_sha(self, body_path: Path, expected: str) -> bool: + try: + self.read_external_body(str(body_path), expected_sha256=expected) + except ( + OSError, + RuntimeError, + TypeError, + ValueError, + OverflowError, + UnicodeDecodeError, + json.JSONDecodeError, + ): + return False + return True + + def external_body_available_for_sha(self, body_uri: object, expected: str) -> bool: + try: + expected = _canonical_hex(expected, name="audit_bundle_sha256") + self.read_external_body(body_uri, expected_sha256=expected) + except ( + OSError, + RuntimeError, + TypeError, + ValueError, + OverflowError, + UnicodeDecodeError, + json.JSONDecodeError, + ): + return False + return True + + def resolve_audit_body_ref( + self, + body_ref: Mapping[str, Any], + *, + expected_sha256: object | None, + body_uri: object, + verify_digest: bool = True, + load_missing_range: Callable[..., list[Any]] | None = None, + ) -> dict[str, object]: + expected = ( + _canonical_hex(expected_sha256, name="audit_bundle_sha256") + if expected_sha256 + else None + ) + declared = _canonical_hex( + body_ref.get("audit_bundle_sha256"), + name="audit_bundle_sha256", + ) + self._validate_body_wrapper_identity(body_ref, body_uri, declared) + if expected and not hmac.compare_digest(declared, expected): + raise RuntimeError( + f"audit bundle body hash mismatch at {body_uri}: expected {expected}, got {declared}" + ) + without_shares = body_ref.get("bundle_without_shares") + parts = body_ref.get("share_parts") + if not isinstance(without_shares, dict) or not isinstance(parts, list): + raise RuntimeError( + f"audit bundle body is not valid JSON at {body_uri}: invalid body reference" + ) + shares: list[Any] = [] + previous_last_share_seq: int | None = None + for part in parts: + if not isinstance(part, dict): + raise RuntimeError( + f"audit bundle body is not valid JSON at {body_uri}: invalid share part" + ) + kind = part.get("kind") + if kind in {"segment", "segment_range", "segment_prefix"}: + part_shares = self._read_share_part_with_recovery( + part, + parent_body_uri=body_uri, + load_missing_range=load_missing_range, + ) + elif kind == "inline" and isinstance(part.get("shares"), list): + inline = part["shares"] + if len(inline) != int(part.get("share_count") or 0): + raise RuntimeError( + f"audit bundle body is not valid JSON at {body_uri}: inline share count mismatch" + ) + self._validate_share_range( + int(part.get("first_share_seq") or 0), + int(part.get("last_share_seq") or 0), + inline, + ) + part_shares = inline + else: + raise RuntimeError( + f"audit bundle body is not valid JSON at {body_uri}: invalid share part kind" + ) + first_share_seq = int(part.get("first_share_seq") or 0) + last_share_seq = int(part.get("last_share_seq") or 0) + if ( + previous_last_share_seq is not None + and first_share_seq <= previous_last_share_seq + ): + raise RuntimeError( + f"audit bundle body is not valid JSON at {body_uri}: share parts overlap or are out of order" + ) + previous_last_share_seq = last_share_seq + shares.extend(part_shares) + if len(shares) != int(body_ref.get("share_count") or 0): + raise RuntimeError( + f"audit bundle body is not valid JSON at {body_uri}: share count mismatch" + ) + bundle = self._insert_shares( + without_shares, + shares, + int(body_ref.get("shares_key_index", len(without_shares))), + ) + if verify_digest: + actual = _sha256_bytes(self.canonical_audit_bundle_bytes(bundle)) + if not hmac.compare_digest(actual, declared): + raise RuntimeError( + f"audit bundle body hash mismatch at {body_uri}: expected {declared}, got {actual}" + ) + return bundle + + def resolve_audit_bundle_v2( + self, + body: Mapping[str, Any], + *, + expected_sha256: object | None, + body_uri: object, + verify_digest: bool = True, + load_missing_range: Callable[..., list[Any]] | None = None, + ) -> dict[str, object]: + expected = ( + _canonical_hex(expected_sha256, name="audit_bundle_sha256") + if expected_sha256 + else None + ) + declared = _canonical_hex( + body.get("audit_bundle_sha256"), + name="audit_bundle_sha256", + ) + self._validate_body_wrapper_identity(body, body_uri, declared) + if expected and not hmac.compare_digest(declared, expected): + raise RuntimeError( + f"audit bundle body hash mismatch at {body_uri}: expected {expected}, got {declared}" + ) + without_shares = body.get("bundle_without_shares") + proof = body.get("share_window_proof") + if not isinstance(without_shares, dict) or not isinstance(proof, dict): + raise RuntimeError( + f"audit bundle body is not valid JSON at {body_uri}: invalid v2 body" + ) + if proof.get("schema") != AUDIT_WINDOW_COMPLETENESS_PROOF_SCHEMA: + raise RuntimeError( + f"audit bundle body is not valid JSON at {body_uri}: invalid proof schema" + ) + if int(body.get("share_count") or 0) != int(proof.get("share_count") or 0): + raise RuntimeError( + f"audit bundle body is not valid JSON at {body_uri}: proof share count mismatch" + ) + parts = proof.get("share_parts") + if not isinstance(parts, list): + raise RuntimeError( + f"audit bundle body is not valid JSON at {body_uri}: missing share parts" + ) + expected_parts_digest = str(proof.get("share_parts_digest_hex") or "").lower() + actual_parts_digest = _sha256_bytes( + self.storage_json_bytes({"share_parts": parts}) + ) + if expected_parts_digest != actual_parts_digest: + raise RuntimeError( + f"audit bundle body is not valid JSON at {body_uri}: share parts digest mismatch" + ) + shares: list[Any] = [] + for part in parts: + if not isinstance(part, dict): + raise RuntimeError( + f"audit bundle body is not valid JSON at {body_uri}: invalid share part" + ) + shares.extend( + self._read_share_part_with_recovery( + part, + parent_body_uri=body_uri, + load_missing_range=load_missing_range, + ) + ) + if len(shares) != int(proof.get("share_count") or 0): + raise RuntimeError( + f"audit bundle body is not valid JSON at {body_uri}: share count mismatch" + ) + if shares and ( + int(shares[0].get("share_seq") or 0) + != int(proof.get("first_share_seq") or 0) + or int(shares[-1].get("share_seq") or 0) + != int(proof.get("last_share_seq") or 0) + ): + raise RuntimeError( + f"audit bundle body is not valid JSON at {body_uri}: proof range mismatch" + ) + bundle = self._insert_shares( + without_shares, + shares, + int(body.get("shares_key_index", len(without_shares))), + ) + reward_manifest = bundle.get("reward_manifest") + copied_proof_fields = ( + "anchor_job_issued_at_ms", + "anchor_share_seq", + "newest_share_seq", + "oldest_share_seq", + "included_share_count", + "requested_window_weight", + "counted_window_weight", + "share_slice_digest_hex", + ) + if isinstance(reward_manifest, dict): + for field in copied_proof_fields: + if (field in proof) != (field in reward_manifest) or proof.get( + field + ) != reward_manifest.get(field): + raise RuntimeError( + f"audit bundle body is not valid JSON at {body_uri}: proof {field} mismatch" + ) + elif any(field in proof for field in copied_proof_fields): + raise RuntimeError( + f"audit bundle body is not valid JSON at {body_uri}: proof has no reward manifest" + ) + if verify_digest: + actual = _sha256_bytes(self.canonical_audit_bundle_bytes(bundle)) + if not hmac.compare_digest(actual, declared): + raise RuntimeError( + f"audit bundle body hash mismatch at {body_uri}: expected {declared}, got {actual}" + ) + return bundle + + def _validate_body_wrapper_identity( + self, + body: Mapping[str, Any], + body_uri: object, + declared_digest: str, + ) -> None: + path = self.resolve_owned_path(body_uri) + match = _BODY_RE.fullmatch(path.name) + if match is None: + raise RuntimeError("compact audit body path is invalid") + block_hash = _canonical_hex(body.get("block_hash"), name="block_hash") + if ( + block_hash != match.group("block") + or declared_digest != match.group("digest") + ): + raise RuntimeError("compact audit body identity mismatch") + + @staticmethod + def _insert_shares( + without_shares: Mapping[str, Any], + shares: list[Any], + shares_key_index: int, + ) -> dict[str, object]: + bundle: dict[str, object] = {} + inserted = False + for index, (key, value) in enumerate(without_shares.items()): + if index == shares_key_index: + bundle["shares"] = shares + inserted = True + bundle[str(key)] = value + if not inserted: + bundle["shares"] = shares + return bundle + + def _resolve_share_segment_path( + self, + part: Mapping[str, Any], + ) -> tuple[Path, str, re.Match[str]]: + """Validate a share part's kind, owned path and filename bounds.""" + body_uri = part.get("body_uri") + kind = str(part.get("kind") or "") + path = self.resolve_owned_path(body_uri) + if self.artifact_kind(path.name) != "share_segment": + raise RuntimeError("not an owned share segment") + if kind == "segment": + match = _SHARE_CONTENT_RE.fullmatch(path.name) + if match is None: + raise RuntimeError("immutable share segment path is invalid") + if ( + int(match.group("first")) != int(part.get("first_share_seq") or 0) + or int(match.group("last")) != int(part.get("last_share_seq") or 0) + ): + raise RuntimeError("immutable share segment path bounds mismatch") + elif kind in {"segment_range", "segment_prefix"}: + match = _SHARE_SLOT_RE.fullmatch(path.name) + if match is None: + # An irreparable stable-slot merge publishes the incoming + # range at an immutable content-addressed URI instead of + # rewriting the slot; its filename bounds are the exact + # declared range. + content_match = _SHARE_CONTENT_RE.fullmatch(path.name) + if content_match is None: + raise RuntimeError("share segment slot path is invalid") + if ( + int(content_match.group("first")) + != int(part.get("first_share_seq") or 0) + or int(content_match.group("last")) + != int(part.get("last_share_seq") or 0) + ): + raise RuntimeError( + "immutable share segment range path bounds mismatch" + ) + match = content_match + else: + declared_slot_first = int( + part.get("segment_first_share_seq") + or match.group("first") + ) + declared_slot_last = int( + part.get("segment_last_share_seq") + or match.group("last") + ) + if ( + declared_slot_first != int(match.group("first")) + or declared_slot_last != int(match.group("last")) + ): + raise RuntimeError("share segment slot path bounds mismatch") + else: + raise RuntimeError("invalid share part kind") + return path, kind, match + + def _open_share_segment_slice( + self, + part: Mapping[str, Any], + *, + parent_body_uri: object, + ) -> tuple[CanonicalAuditBundleView, Sequence[Any]]: + """:meth:`read_audit_share_segment` with the segment left on disk. + + Performs the same path, digest, header, contiguity, count and range + checks, but the segment is scanned into a bounded view whose share + array stays lazy, and the selected range comes back as a slice of + that array. The caller closes the returned view. Used by the + publication-time reconstruction check so no segment -- and no single + oversized record inside one -- is loaded whole (#255). + """ + body_uri = part.get("body_uri") + kind = str(part.get("kind") or "") + if kind not in {"segment", "segment_range", "segment_prefix"}: + raise RuntimeError( + f"audit bundle body is not valid JSON at {parent_body_uri}: " + "invalid share part kind" + ) + try: + path, kind, match = self._resolve_share_segment_path(part) + view = self._scan_owned_artifact(path, lazy_paths=(("shares",),)) + except (OSError, RuntimeError, CanonicalArtifactError) as exc: + raise RuntimeError( + f"audit bundle body is not retrievable at {parent_body_uri}: " + f"share segment {body_uri}: {exc}" + ) from exc + try: + if kind == "segment": + expected = str(part.get("sha256") or "").lower() + actual = view.sha256_hex + if not hmac.compare_digest(actual, expected): + raise RuntimeError( + f"audit bundle body hash mismatch at {parent_body_uri}: " + f"share segment {body_uri} expected {expected}, got {actual}" + ) + segment_shares = view.get("shares") + if ( + view.get("schema") != AUDIT_SHARE_SEGMENT_SCHEMA + or not self._is_share_sequence(segment_shares) + ): + raise RuntimeError( + f"audit bundle body is not valid JSON at {parent_body_uri}: " + f"invalid share segment {body_uri}" + ) + try: + segment_first = int(view.get("first_share_seq")) + segment_last = int(view.get("last_share_seq")) + segment_count = int(view.get("share_count")) + self._validate_share_range( + segment_first, + segment_last, + segment_shares, + ) + except (TypeError, ValueError, RuntimeError) as exc: + raise RuntimeError( + f"audit bundle body is not valid JSON at {parent_body_uri}: " + f"share segment {body_uri} header mismatch" + ) from exc + if segment_count != len(segment_shares): + raise RuntimeError( + f"audit bundle body is not valid JSON at {parent_body_uri}: " + f"share segment {body_uri} share count mismatch" + ) + if kind == "segment": + if ( + segment_first != int(match.group("first")) + or segment_last != int(match.group("last")) + ): + raise RuntimeError("immutable share segment header bounds mismatch") + elif ( + segment_first < int(match.group("first")) + or segment_last > int(match.group("last")) + ): + raise RuntimeError("share segment header escapes slot bounds") + first_share_seq = int(part.get("first_share_seq") or 0) + last_share_seq = int(part.get("last_share_seq") or 0) + # The header check above proved strict +1 contiguity from + # segment_first, so the requested range is an index range. + if first_share_seq > last_share_seq: + selected: Sequence[Any] = [] + elif ( + first_share_seq < segment_first + or last_share_seq > segment_last + ): + raise RuntimeError( + f"audit bundle body is not valid JSON at {parent_body_uri}: " + f"share segment {body_uri} does not contain requested range" + ) + else: + selected = segment_shares[ + first_share_seq - segment_first : last_share_seq - segment_first + 1 + ] + if len(selected) != int(part.get("share_count") or 0): + raise RuntimeError( + f"audit bundle body is not valid JSON at {parent_body_uri}: " + f"share segment {body_uri} share count mismatch" + ) + if kind in {"segment_range", "segment_prefix"}: + key = "range_sha256" if kind == "segment_range" else "prefix_sha256" + actual, _size = json_chunks_sha256_and_size( + iter_json_byte_chunks( + self.audit_share_segment_payload( + first_share_seq=first_share_seq, + last_share_seq=last_share_seq, + shares=selected, + ) + ) + ) + expected = str(part.get(key) or "").lower() + if not hmac.compare_digest(actual, expected): + raise RuntimeError( + f"audit bundle body hash mismatch at {parent_body_uri}: " + f"share segment range {body_uri} expected {expected}, got {actual}" + ) + self._validate_root_identity() + return view, selected + except BaseException: + view.close() + raise + + def read_audit_share_segment( + self, + part: Mapping[str, Any], + *, + parent_body_uri: object, + ) -> list[Any]: + body_uri = part.get("body_uri") + kind = str(part.get("kind") or "") + if kind not in {"segment", "segment_range", "segment_prefix"}: + raise RuntimeError( + f"audit bundle body is not valid JSON at {parent_body_uri}: " + "invalid share part kind" + ) + try: + path, kind, match = self._resolve_share_segment_path(part) + segment_bytes, _value = self._read_owned_regular_bytes(path) + except (OSError, RuntimeError) as exc: + raise RuntimeError( + f"audit bundle body is not retrievable at {parent_body_uri}: " + f"share segment {body_uri}: {exc}" + ) from exc + if kind == "segment": + expected = str(part.get("sha256") or "").lower() + actual = _sha256_bytes(segment_bytes) + if not hmac.compare_digest(actual, expected): + raise RuntimeError( + f"audit bundle body hash mismatch at {parent_body_uri}: " + f"share segment {body_uri} expected {expected}, got {actual}" + ) + try: + segment = json.loads(segment_bytes) + except (UnicodeDecodeError, json.JSONDecodeError) as exc: + raise RuntimeError( + f"audit bundle body is not valid JSON at {parent_body_uri}: " + f"share segment {body_uri}: {exc}" + ) from exc + if ( + not isinstance(segment, dict) + or segment.get("schema") != AUDIT_SHARE_SEGMENT_SCHEMA + or not isinstance(segment.get("shares"), list) + ): + raise RuntimeError( + f"audit bundle body is not valid JSON at {parent_body_uri}: " + f"invalid share segment {body_uri}" + ) + segment_shares = segment["shares"] + try: + segment_first = int(segment.get("first_share_seq")) + segment_last = int(segment.get("last_share_seq")) + segment_count = int(segment.get("share_count")) + self._validate_share_range( + segment_first, + segment_last, + segment_shares, + ) + except (TypeError, ValueError, RuntimeError) as exc: + raise RuntimeError( + f"audit bundle body is not valid JSON at {parent_body_uri}: " + f"share segment {body_uri} header mismatch" + ) from exc + if segment_count != len(segment_shares): + raise RuntimeError( + f"audit bundle body is not valid JSON at {parent_body_uri}: " + f"share segment {body_uri} share count mismatch" + ) + if kind == "segment": + assert match is not None + if ( + segment_first != int(match.group("first")) + or segment_last != int(match.group("last")) + ): + raise RuntimeError("immutable share segment header bounds mismatch") + else: + assert match is not None + if ( + segment_first < int(match.group("first")) + or segment_last > int(match.group("last")) + ): + raise RuntimeError("share segment header escapes slot bounds") + selected = self.select_audit_share_segment_range( + segment_shares, + first_share_seq=int(part.get("first_share_seq") or 0), + last_share_seq=int(part.get("last_share_seq") or 0), + parent_body_uri=parent_body_uri, + body_uri=body_uri, + ) + if len(selected) != int(part.get("share_count") or 0): + raise RuntimeError( + f"audit bundle body is not valid JSON at {parent_body_uri}: " + f"share segment {body_uri} share count mismatch" + ) + if kind in {"segment_range", "segment_prefix"}: + key = "range_sha256" if kind == "segment_range" else "prefix_sha256" + actual = _sha256_bytes( + self.storage_json_bytes( + self.audit_share_segment_payload( + first_share_seq=int(part.get("first_share_seq") or 0), + last_share_seq=int(part.get("last_share_seq") or 0), + shares=selected, + ) + ) + ) + expected = str(part.get(key) or "").lower() + if not hmac.compare_digest(actual, expected): + raise RuntimeError( + f"audit bundle body hash mismatch at {parent_body_uri}: " + f"share segment range {body_uri} expected {expected}, got {actual}" + ) + elif kind != "segment": + raise RuntimeError( + f"audit bundle body is not valid JSON at {parent_body_uri}: invalid share part kind" + ) + self._validate_root_identity() + return selected + + @staticmethod + def select_audit_share_segment_range( + shares: list[Any], + *, + first_share_seq: int, + last_share_seq: int, + parent_body_uri: object, + body_uri: object, + ) -> list[Any]: + selected: list[Any] = [] + previous_seq: int | None = None + for share in shares: + if not isinstance(share, dict): + raise RuntimeError( + f"audit bundle body is not valid JSON at {parent_body_uri}: " + f"share segment {body_uri} has invalid share" + ) + share_seq = int(share.get("share_seq") or 0) + if previous_seq is not None and previous_seq + 1 != share_seq: + raise RuntimeError( + f"audit bundle body is not valid JSON at {parent_body_uri}: " + f"share segment {body_uri} is not contiguous" + ) + previous_seq = share_seq + if first_share_seq <= share_seq <= last_share_seq: + selected.append(share) + if selected: + if int(selected[0].get("share_seq") or 0) != first_share_seq: + selected = [] + elif int(selected[-1].get("share_seq") or 0) != last_share_seq: + selected = [] + if not selected and first_share_seq <= last_share_seq: + raise RuntimeError( + f"audit bundle body is not valid JSON at {parent_body_uri}: " + f"share segment {body_uri} does not contain requested range" + ) + return selected + + +SCAN_CHUNK_BYTES_LOCAL = 256 * 1024 + + +class _ChunkReader(io.RawIOBase): + """Sequential read adapter over an :class:`ArtifactSource`.""" + + def __init__(self, source: ArtifactSource) -> None: + super().__init__() + self._source = source + self._offset = 0 + + def readable(self) -> bool: + return True + + def readinto(self, buffer: Any) -> int: + view = memoryview(buffer) + chunk = self._source.pread(self._offset, len(view)) + view[: len(chunk)] = chunk + self._offset += len(chunk) + return len(chunk) + + +def _gzip_canonical_chunks(chunks: Iterable[bytes]) -> Iterator[bytes]: + """:func:`gzip_canonical_bundle_bytes` over a chunk stream, chunk by chunk. + + Same compressor, level, header and (absent) mtime, so equal inputs give + the same compressed bytes as the whole-body helper; deflate output does + not depend on how the input was split. + """ + + sink = io.BytesIO() + with gzip.GzipFile( + fileobj=sink, + mode="wb", + compresslevel=CANONICAL_BUNDLE_GZIP_LEVEL, + mtime=0, + ) as handle: + for chunk in chunks: + handle.write(chunk) + if sink.tell(): + piece = sink.getvalue() + sink.seek(0) + sink.truncate() + yield piece + piece = sink.getvalue() + if piece: + yield piece + + +def _bytes_equal_chunks(expected: bytes, chunks: Iterable[bytes]) -> bool: + """``expected == b"".join(chunks)`` without joining the chunks.""" + offset = 0 + view = memoryview(expected) + for chunk in chunks: + end = offset + len(chunk) + if end > len(view) or not hmac.compare_digest(bytes(view[offset:end]), chunk): + return False + offset = end + return offset == len(view) + + +def _pread_exact(fd: int, offset: int, size: int) -> bytes: + """Read exactly ``size`` bytes at ``offset``; short at end of file.""" + pieces: list[bytes] = [] + remaining = size + while remaining > 0: + try: + chunk = os.pread(fd, remaining, offset) + except InterruptedError: + continue + if not chunk: + break + pieces.append(chunk) + offset += len(chunk) + remaining -= len(chunk) + return b"".join(pieces) if len(pieces) != 1 else pieces[0] + + +def _canonical_hex_bytes(value: object, *, name: str) -> str: + text = str(value) + if len(text) % 2: + raise ValueError(f"{name} must contain complete hexadecimal bytes") + try: + bytes.fromhex(text) + except ValueError as exc: + raise ValueError(f"{name} must be hexadecimal") from exc + return text.lower() diff --git a/lab/prism/audit_bundle_view.py b/lab/prism/audit_bundle_view.py new file mode 100644 index 000000000..ad9a9b63f --- /dev/null +++ b/lab/prism/audit_bundle_view.py @@ -0,0 +1,2366 @@ +#!/usr/bin/env python3 +"""Bounded views over an immutable canonical audit artifact (issue #255). + +The canonical audit bundle the Rust builder writes for a found block carries +two window-sized arrays: the top-level ``shares`` (one record per accepted +share in the payout window) and ``reward_manifest.shares`` (one record per +counted share). Loading that artifact with ``json.load`` materialized both +as Python dictionaries inside the lease-bearing coordinator process, and +every downstream consumer -- the logical comparison against the candidate +file, compact-body segmentation, reconstruction verification, and the +accepted-block persistence payload -- parsed or encoded the whole window +again. + +This module keeps the verified artifact file as the sole authority and +exposes it through bounded objects: + +* :class:`CanonicalAuditBundleView` is an immutable mapping of the bundle's + top-level members. Small members (found block, balances, policy and + coinbase manifests) are parsed eagerly; the window-sized arrays are + exposed as :class:`LazyRecordSequence` objects that decode records on + demand from the file through bounded read windows. +* :class:`LazyRecordSequence` is a replayable ``collections.abc.Sequence`` + of dictionaries: every iteration re-reads the artifact, random access + seeks to a checkpoint and scans at most one stride of records, and no + window-sized list, tuple, string or dictionary is ever built. Equality + against a list or another lazy sequence streams both sides record by + record. +* :func:`iter_json_chunks` encodes a payload that embeds such sequences into + the exact ``json.dumps(payload, separators=(",", ":"))`` text in bounded + chunks, so compact bodies and share segments can be written and digested + without a whole-window string. + +The scanner never rejects a valid document because of its size: a record +larger than the scan window grows that one decode to the record's own size +and the window is trimmed again afterwards, so an oversized single field is +isolated to its own decode rather than refused. + +The byte contract is the Rust typed canonical field order. Nothing here +sorts keys or otherwise re-canonicalizes; the v1 candidate-identity JSON +(sorted keys, ``default=str``) is a different encoder with a different +digest and is never derived from these views. +""" + +from __future__ import annotations + +from contextlib import contextmanager + +import codecs +from collections.abc import Mapping, Sequence +from dataclasses import dataclass +import hashlib +import json +import os +from pathlib import Path +import re +import stat +import struct +import subprocess +import sys +import tempfile +import threading +import time +from typing import Any, Callable, Iterable, Iterator +import weakref + +from lab.prism.share_json_stream import iter_json_array_text_chunks + + +# Bytes read per positional read while scanning or replaying an artifact. +# Also the ceiling on decoded text retained between records for ordinary +# records; one oversized record grows the window to its own size only. +SCAN_CHUNK_BYTES = 256 * 1024 +# Records between byte-offset checkpoints of a lazy sequence. Random access +# seeks to the nearest checkpoint and skips at most this many records. +RECORD_INDEX_STRIDE = 256 +# Records per ``json.dumps`` call inside the streaming encoder. +JSON_BATCH_RECORDS = 256 +# Target size of one yielded encoder chunk. +JSON_CHUNK_CHARS = 64 * 1024 +# Member paths of a canonical audit bundle whose length scales with the +# window (one entry per accepted or counted share), with the block's +# transactions (one leaf per transaction) or with recipient cardinality +# (balances, accounts, entitlements, settlement recipients and fanout +# manifests). Recipient cardinality is itself bounded only by the window, +# so none of these is a byte bound; every one stays on disk. What remains +# eager is fixed-size: the found block, policy, attestation, digests and +# the signed coinbase manifest, whose outputs are capped by configuration. +CANONICAL_BUNDLE_LAZY_PATHS: tuple[tuple[str, ...], ...] = ( + ("shares",), + ("reward_manifest", "shares"), + ("reward_manifest", "entitlements"), + ("witness_merkle_leaves_hex",), + ("audit_commitment_leaves_hex",), + ("prior_balances",), + ("payout_policy_manifest", "accounts"), + ("payout_policy_manifest", "onchain_entitlements"), + ("settlement_mode_decision", "direct_recipients"), + ("settlement_mode_decision", "fanout_chunks"), + ("ctv_fanout_manifest_set", "manifests"), +) +# Largest single JSON value the coordinator decodes in-process with one +# ``raw_decode`` call. A record above this limit is normalized by an +# isolated helper process instead (see RawJsonRecord); it is never +# rejected. Values above the limit are also counted in the scan statistics. +RECORD_DECODE_SOFT_LIMIT_BYTES = 1024 * 1024 +# Members of an isolated record that are kept in the coordinator as plain +# values: any member whose compact encoding is at most this many bytes. +RAW_RECORD_MEMBER_LIMIT_BYTES = 4096 +# Wall-clock ceiling for one isolated record normalization, admission wait +# included. +RAW_RECORD_HELPER_TIMEOUT_SECONDS = 300.0 +# Isolated helper processes admitted at once: each one may hold a whole +# record (and two encodings of it), so the slots bound helper memory. +RAW_RECORD_HELPER_SLOTS = 1 +RAW_RECORD_HELPER_MEMORY_BYTES = 4 * 1024 * 1024 * 1024 +RAW_RECORD_HELPER_HEADER_BYTES = 1024 * 1024 +# Supervisor poll interval while a helper runs or a slot is awaited. +RAW_RECORD_HELPER_POLL_SECONDS = 0.05 +# Bytes of a helper's diagnostics retained (the tail) for the error report. +RAW_RECORD_HELPER_DIAGNOSTIC_BYTES = 64 * 1024 + +_WHITESPACE = re.compile(r"[ \t\n\r]*") +# Strict JSON token grammar (RFC 8259 plus the standard decoder's NaN and +# Infinity constants). The quantifiers are possessive: when a window ends +# inside a long token the match must fail in one pass instead of +# backtracking through every prefix, which would allocate a regex state +# stack proportional to the window. +_WS = r"[ \t\n\r]*+" +_STRING_TOKEN = r'"(?:[^"\\\x00-\x1f]++|\\(?:["\\/bfnrt]|u[0-9a-fA-F]{4}))*+"' +_NUMBER_TOKEN = r"-?(?:0|[1-9][0-9]*+)(?:\.[0-9]++)?+(?:[eE][+-]?[0-9]++)?+" +_SCALAR_TOKEN = rf"(?:{_STRING_TOKEN}|{_NUMBER_TOKEN}|true|false|null|NaN|Infinity|-Infinity)" +# One complete JSON object with no nested containers: every share record +# in a canonical bundle. Matching it validates and skips the record at C +# speed without building a dictionary; anything it does not match (a +# record cut by the window, a nested value, or a malformed one) goes to +# the streaming validator below, which reports malformed input. +_FLAT_OBJECT = re.compile( + r"\{" + + _WS + + rf"(?:{_STRING_TOKEN}{_WS}:{_WS}{_SCALAR_TOKEN}" + + rf"(?:{_WS},{_WS}{_STRING_TOKEN}{_WS}:{_WS}{_SCALAR_TOKEN})*+)?+" + + _WS + + r"\}" +) +_NON_STRING_SCALAR = re.compile( + rf"(?:{_NUMBER_TOKEN}|true|false|null|NaN|Infinity|-Infinity)\Z" +) +# Characters that end a string's ordinary run: the closing quote, an +# escape, or a control character the grammar forbids. +_STRING_STRUCTURAL = re.compile(r'["\\\x00-\x1f]') +_SCALAR_END = re.compile(r"[,\]}: \t\n\r\"{\[]") +_ESCAPE_TOKEN = re.compile(r'["\\/bfnrt]|u[0-9a-fA-F]{4}') +# Characters that can extend a JSON number token past a decoded prefix. +_NUMBER_CONTINUATION = frozenset("0123456789.eE+-") +_DECODER = json.JSONDecoder() +_COMPACT_SEPARATORS = (",", ":") +# Checkpoint entries buffered before they are written to the index file. +_INDEX_FLUSH_BYTES = 4096 + + +def _close_fd(fd: int) -> None: + try: + os.close(fd) + except OSError: + pass + + +def _scratch_fd(purpose: str) -> int: + """An unlinked temporary file's descriptor, or resource pressure.""" + try: + handle = tempfile.TemporaryFile() + except OSError as exc: + raise ArtifactResourcePressure( + f"cannot create the {purpose} scratch file: {exc}" + ) from exc + try: + return os.dup(handle.fileno()) + except OSError as exc: + raise ArtifactResourcePressure( + f"cannot duplicate the {purpose} scratch descriptor: {exc}" + ) from exc + finally: + handle.close() + + +def _pwrite_all(fd: int, data: bytes, offset: int, *, purpose: str) -> None: + view = memoryview(data) + while view: + try: + written = os.pwrite(fd, view, offset) + except InterruptedError: + continue + except OSError as exc: + raise ArtifactResourcePressure( + f"cannot write the {purpose} scratch file: {exc}" + ) from exc + view = view[written:] + offset += written + + +class _CheckpointIndex: + """Fixed-width record-offset checkpoints for the lazy arrays of one scan. + + Entries are 8-byte little-endian byte offsets appended while scanning + and read back with positional reads, so the index scales with the + window on disk, never in memory: an unlinked temporary file holds it + (the share-window spool's approach) and its descriptor closes with the + last owner. There is no in-memory fallback: when the scratch file + cannot be created or written, :class:`ArtifactResourcePressure` is + raised so the caller retries later rather than growing the process. + """ + + __slots__ = ( + "__weakref__", + "_count", + "_fd", + "_finalizer", + "_pending", + "_written", + ) + + ENTRY = struct.Struct(" None: + self._count = 0 + self._written = 0 + self._pending = bytearray() + self._fd = _scratch_fd("checkpoint index") + self._finalizer = weakref.finalize(self, _close_fd, self._fd) + + @property + def count(self) -> int: + return self._count + + def append(self, offset: int) -> None: + self._pending += self.ENTRY.pack(int(offset)) + if len(self._pending) >= _INDEX_FLUSH_BYTES: + self.flush() + self._count += 1 + + def flush(self) -> None: + if not self._pending: + return + if not self._finalizer.alive: + raise CanonicalArtifactError("canonical audit artifact index is closed") + _pwrite_all( + self._fd, + bytes(self._pending), + self._written * self.ENTRY.size, + purpose="checkpoint index", + ) + self._written += len(self._pending) // self.ENTRY.size + self._pending = bytearray() + + def get(self, index: int) -> int: + if not 0 <= index < self._count: + raise IndexError("checkpoint index out of range") + self.flush() + if not self._finalizer.alive: + raise CanonicalArtifactError("canonical audit artifact index is closed") + entry = os.pread(self._fd, self.ENTRY.size, index * self.ENTRY.size) + if len(entry) != self.ENTRY.size: + raise CanonicalArtifactError("canonical audit artifact index is truncated") + return int(self.ENTRY.unpack(entry)[0]) + + def close(self) -> None: + self._finalizer() + + +class ArtifactResourcePressure(Exception): + """A scratch resource (checkpoint index, record spool, helper) is unavailable. + + Raised instead of degrading to an unbounded in-memory structure. The + artifact itself is intact, so this is deliberately neither a + :class:`CanonicalArtifactError` nor a :class:`json.JSONDecodeError`: + callers must not classify it as corruption or mismatch, and the + operation is retry-safe once the pressure passes. + """ + + +class CanonicalArtifactError(ValueError): + """The artifact cannot be used: identity changed, closed, or not a file.""" + + +class CanonicalArtifactSyntaxError(CanonicalArtifactError, json.JSONDecodeError): + """The artifact is not a well-formed JSON document. + + Also a :class:`json.JSONDecodeError`, so callers that classified the + historical ``json.load`` failure keep classifying this one the same way. + """ + + def __init__(self, msg: str, doc: str = "", pos: int = 0) -> None: + json.JSONDecodeError.__init__(self, msg, doc, pos) + + def __str__(self) -> str: + return str(self.msg) + + +@dataclass(frozen=True) +class ArtifactIdentity: + """The stat identity a view binds to; mirrors the store's file identity.""" + + device: int + inode: int + mode: int + size: int + mtime_ns: int + + @classmethod + def from_stat(cls, value: os.stat_result) -> ArtifactIdentity: + return cls( + value.st_dev, + value.st_ino, + value.st_mode, + value.st_size, + value.st_mtime_ns, + ) + + def matches(self, value: os.stat_result) -> bool: + return ( + stat.S_ISREG(value.st_mode) + and self.device == value.st_dev + and self.inode == value.st_ino + and self.mode == value.st_mode + and self.size == value.st_size + and self.mtime_ns == value.st_mtime_ns + ) + + +class ArtifactSource: + """One open regular file read only through bounded positional reads. + + Owns its descriptor: the file stays readable after its path is unlinked + or replaced, and every read re-checks the descriptor's identity so a + truncated or rewritten inode is reported rather than silently reread. + The descriptor closes when the source is closed or garbage collected. + """ + + def __init__(self, fd: int, *, path: Path | None = None) -> None: + value = os.fstat(fd) + if not stat.S_ISREG(value.st_mode): + os.close(fd) + raise CanonicalArtifactError("canonical audit artifact is not a regular file") + self._fd = fd + self.path = path + self.identity = ArtifactIdentity.from_stat(value) + self._finalizer = weakref.finalize(self, os.close, fd) + + @property + def closed(self) -> bool: + return not self._finalizer.alive + + def close(self) -> None: + self._finalizer() + + def fileno(self) -> int: + if self.closed: + raise CanonicalArtifactError("canonical audit artifact source is closed") + return self._fd + + def verify_identity(self) -> None: + if self.closed: + raise CanonicalArtifactError("canonical audit artifact source is closed") + if not self.identity.matches(os.fstat(self._fd)): + raise CanonicalArtifactError("canonical audit artifact changed after it was scanned") + + def pread(self, offset: int, size: int) -> bytes: + """Read up to ``size`` bytes at ``offset``; empty at end of file.""" + fd = self.fileno() + pieces: list[bytes] = [] + remaining = size + while remaining > 0: + try: + chunk = os.pread(fd, remaining, offset) + except InterruptedError: + continue + if not chunk: + break + pieces.append(chunk) + offset += len(chunk) + remaining -= len(chunk) + return b"".join(pieces) if len(pieces) != 1 else pieces[0] + + def iter_bytes(self, *, chunk_bytes: int = SCAN_CHUNK_BYTES) -> Iterator[bytes]: + """The complete file in bounded chunks; identity checked at both ends.""" + self.verify_identity() + offset = 0 + while True: + chunk = self.pread(offset, chunk_bytes) + if not chunk: + break + offset += len(chunk) + yield chunk + self.verify_identity() + if offset != self.identity.size: + raise CanonicalArtifactError("canonical audit artifact size changed while reading") + + def sha256_hex(self, *, chunk_bytes: int = SCAN_CHUNK_BYTES) -> str: + digest = hashlib.sha256() + for chunk in self.iter_bytes(chunk_bytes=chunk_bytes): + digest.update(chunk) + return digest.hexdigest() + + +class ScanStats: + """Exact allocation maxima observed while scanning one artifact. + + ``max_value_chars`` is the largest single JSON value handed to one + ``raw_decode`` call (a lazy-array record or an eager member); + ``window_high_water_chars`` is the largest decoded text window held at + any moment (at most one read chunk beyond the largest value); + ``oversized_values`` counts values above + :data:`RECORD_DECODE_SOFT_LIMIT_BYTES`. + """ + + __slots__ = ( + "checkpoint_entries", + "isolated_records", + "max_value_chars", + "oversized_values", + "values_decoded", + "window_high_water_chars", + ) + + def __init__(self) -> None: + self.max_value_chars = 0 + self.oversized_values = 0 + self.values_decoded = 0 + self.window_high_water_chars = 0 + self.checkpoint_entries = 0 + # Records normalized by the isolated helper instead of in-process. + self.isolated_records = 0 + + def note_value(self, chars: int) -> None: + """Record one value's size: decoded text characters, or bytes when + the value was skipped structurally (equal for ASCII artifacts).""" + self.values_decoded += 1 + if chars > self.max_value_chars: + self.max_value_chars = chars + if chars > RECORD_DECODE_SOFT_LIMIT_BYTES: + self.oversized_values += 1 + + def note_window(self, chars: int) -> None: + if chars > self.window_high_water_chars: + self.window_high_water_chars = chars + + def as_dict(self) -> dict[str, int]: + return { + "max_value_chars": self.max_value_chars, + "oversized_values": self.oversized_values, + "values_decoded": self.values_decoded, + "window_high_water_chars": self.window_high_water_chars, + "checkpoint_entries": self.checkpoint_entries, + "isolated_records": self.isolated_records, + } + + +class _Cursor: + """A decoded text window over an artifact, advanced record by record. + + ``text`` holds a bounded decoded window starting at ``window_start`` + bytes into the file; ``pos`` is the character index of the next token. + Reads happen only when the parser needs more text, feed the optional + hasher, and invoke the optional checkpoint callable so a cancellation + check runs at least once per chunk. + """ + + __slots__ = ( + "_ascii", + "_checkpoint", + "_chunk_bytes", + "_decoder", + "_hasher", + "_next_read", + "_source", + "_window_start", + "eof", + "generation", + "pos", + "stats", + "text", + ) + + def __init__( + self, + source: ArtifactSource, + offset: int, + *, + chunk_bytes: int, + hasher: Any | None = None, + checkpoint: Callable[[], None] | None = None, + stats: ScanStats | None = None, + ) -> None: + self._source = source + self._chunk_bytes = max(1, int(chunk_bytes)) + self._hasher = hasher + self._checkpoint = checkpoint + self._decoder = codecs.getincrementaldecoder("utf-8")() + self._window_start = int(offset) + self._next_read = int(offset) + self._ascii = True + self.text = "" + self.pos = 0 + self.eof = False + # Bumped whenever consumed text is dropped, so a caller can tell + # whether a slice of ``text`` taken before a skip is still valid. + self.generation = 0 + self.stats = stats if stats is not None else ScanStats() + + @property + def byte_pos(self) -> int: + """Byte offset in the file of ``text[pos]``.""" + if self._ascii: + return self._window_start + self.pos + return self._window_start + len(self.text[: self.pos].encode("utf-8")) + + def _append(self, piece: str) -> None: + if not piece: + return + if self._ascii and not piece.isascii(): + self._ascii = False + self.text = self.text + piece if self.text else piece + self.stats.note_window(len(self.text)) + + def fill(self, min_ahead: int) -> None: + """Ensure ``min_ahead`` characters follow ``pos`` unless at EOF.""" + while not self.eof and len(self.text) - self.pos < min_ahead: + chunk = self._source.pread(self._next_read, self._chunk_bytes) + if self._checkpoint is not None: + self._checkpoint() + if not chunk: + self.eof = True + try: + tail = self._decoder.decode(b"", final=True) + except UnicodeDecodeError as exc: + raise CanonicalArtifactSyntaxError( + f"canonical audit artifact is not valid UTF-8: {exc}" + ) from exc + self._append(tail) + break + self._next_read += len(chunk) + if self._hasher is not None: + self._hasher.update(chunk) + try: + self._append(self._decoder.decode(chunk)) + except UnicodeDecodeError as exc: + raise CanonicalArtifactSyntaxError( + f"canonical audit artifact is not valid UTF-8: {exc}" + ) from exc + + def trim(self, *, force: bool = False) -> None: + """Drop consumed text once it exceeds one chunk; keeps windows bounded.""" + if self.pos < self._chunk_bytes and not force: + return + if self.pos == 0: + return + self._window_start = self.byte_pos + self.text = self.text[self.pos :] + self.pos = 0 + self._ascii = self.text.isascii() + self.generation += 1 + + def skip_ws(self) -> None: + while True: + self.pos = _WHITESPACE.match(self.text, self.pos).end() + if self.pos < len(self.text) or self.eof: + return + self.fill(1) + + def peek(self) -> str: + self.fill(1) + return self.text[self.pos] if self.pos < len(self.text) else "" + + def expect(self, char: str) -> None: + if self.peek() != char: + raise CanonicalArtifactSyntaxError( + f"canonical audit artifact is malformed at byte {self.byte_pos}: " + f"expected {char!r}" + ) + self.pos += 1 + + def at_eof(self) -> bool: + self.fill(1) + return self.eof and self.pos >= len(self.text) + + def decode_value(self) -> Any: + """Decode one JSON value at ``pos`` (whitespace already skipped). + + A value cut by the window boundary extends the window until it fits; + the growth is geometric so an oversized field costs a handful of + reads, and nothing beyond that one value is retained. + + Token completion is checked explicitly for numbers: ``raw_decode`` + happily returns the prefix ``1`` of a window ending in ``1e+`` or + ``1.`` without touching the window end, so a number is complete + only when the character after it cannot continue a number (or the + file has ended). A continuation character with the whole file in + view is a malformed number, exactly as the standard decoder reports + for the complete document. + """ + growth = self._chunk_bytes + while True: + self.fill(1) + try: + value, end = _DECODER.raw_decode(self.text, self.pos) + except json.JSONDecodeError as exc: + if self.eof: + raise CanonicalArtifactSyntaxError( + f"canonical audit artifact is malformed: {exc}" + ) from exc + self.fill(len(self.text) - self.pos + growth) + growth *= 2 + continue + if end >= len(self.text): + if not self.eof: + # A number or literal may continue in the next chunk. + self.fill(len(self.text) - self.pos + growth) + growth *= 2 + continue + elif ( + isinstance(value, (int, float)) + and not isinstance(value, bool) + and self.text[end] in _NUMBER_CONTINUATION + ): + if not self.eof: + self.fill(len(self.text) - self.pos + growth) + growth *= 2 + continue + raise CanonicalArtifactSyntaxError( + f"canonical audit artifact is malformed at byte {self.byte_pos}: " + "invalid number" + ) + self.stats.note_value(end - self.pos) + self.pos = end + return value + + def skip_value(self) -> None: + """Validate and advance past one JSON value without retaining it. + + A flat record that fits the window is validated and skipped by one + strict-grammar regex match. Anything else -- a record straddling + the window, a nested value, a string or scalar of any size -- is + walked by a streaming token validator window by window, so the text + held never exceeds about two read chunks even for a value far larger + than the chunk. The validation is the standard decoder's: token + syntax, escapes, control characters, separators, container balance + and termination all fail closed here, at scan time. + """ + char = self.peek() + if char == "{": + match = _FLAT_OBJECT.match(self.text, self.pos) + if match is not None: + self.stats.note_value(match.end() - self.pos) + self.pos = match.end() + return + start = self.byte_pos + self._walk_value() + self.stats.note_value(self.byte_pos - start) + + def _malformed(self, detail: str) -> CanonicalArtifactSyntaxError: + return CanonicalArtifactSyntaxError( + f"canonical audit artifact is malformed at byte {self.byte_pos}: {detail}" + ) + + def _advance_window(self) -> None: + """Consume the whole window and read the next chunk.""" + self.pos = len(self.text) + self.trim(force=True) + self.fill(1) + if self.eof and self.pos >= len(self.text): + raise CanonicalArtifactSyntaxError( + "canonical audit artifact is malformed: unterminated value" + ) + + def _walk_value(self) -> None: + """Streaming validator for one JSON value starting at ``pos``.""" + stack: list[str] = [] + state = "value" + while True: + self.skip_ws() + char = self.peek() + if state == "value": + if char == '"': + self.pos += 1 + self._skip_string_body() + state = "after" + elif char == "{": + self.pos += 1 + stack.append("O") + state = "key_or_end" + elif char == "[": + self.pos += 1 + stack.append("A") + state = "value_or_end" + else: + self._skip_scalar() + state = "after" + elif state == "value_or_end": + if char == "]": + self.pos += 1 + stack.pop() + state = "after" + else: + state = "value" + continue + elif state == "key_or_end": + if char == "}": + self.pos += 1 + stack.pop() + state = "after" + elif char == '"': + self.pos += 1 + self._skip_string_body() + state = "colon" + else: + raise self._malformed("expected a member name or '}'") + elif state == "key": + if char != '"': + raise self._malformed("expected a member name") + self.pos += 1 + self._skip_string_body() + state = "colon" + elif state == "colon": + if char != ":": + raise self._malformed("expected ':'") + self.pos += 1 + state = "value" + else: # after a complete value + if not stack: + return + top = stack[-1] + if char == ",": + self.pos += 1 + state = "key" if top == "O" else "value" + elif char == "]" and top == "A": + self.pos += 1 + stack.pop() + elif char == "}" and top == "O": + self.pos += 1 + stack.pop() + elif char == "": + raise CanonicalArtifactSyntaxError( + "canonical audit artifact is malformed: unterminated value" + ) + else: + raise self._malformed("expected ',' or a closing bracket") + if self.pos >= self._chunk_bytes: + self.trim() + + def _skip_string_body(self) -> None: + """Validate to just past the closing quote; ``pos`` is inside the string.""" + while True: + match = _STRING_STRUCTURAL.search(self.text, self.pos) + if match is None: + self._advance_window() + continue + char = match.group() + self.pos = match.end() + if char == '"': + return + if char != "\\": + self.pos -= 1 + raise self._malformed("control character in string") + # An escape needs up to five more characters; make them visible + # before validating so a window boundary cannot split it. + self.fill(5) + escape = _ESCAPE_TOKEN.match(self.text, self.pos) + if escape is None: + raise self._malformed("invalid escape sequence") + self.pos = escape.end() + if self.pos >= self._chunk_bytes: + self.trim() + + def _skip_scalar(self) -> None: + """Validate one number or literal token; it may span windows.""" + start = self.byte_pos + pieces: list[str] = [] + while True: + match = _SCALAR_END.search(self.text, self.pos) + if match is None: + if self.eof: + pieces.append(self.text[self.pos :]) + self.pos = len(self.text) + break + pieces.append(self.text[self.pos :]) + self._advance_window() + continue + pieces.append(self.text[self.pos : match.start()]) + self.pos = match.start() + break + token = "".join(pieces) + if not token: + raise CanonicalArtifactSyntaxError( + f"canonical audit artifact is malformed at byte {start}: expected a value" + ) + if _NON_STRING_SCALAR.match(token) is None: + raise CanonicalArtifactSyntaxError( + f"canonical audit artifact is malformed at byte {start}: invalid token {token[:32]!r}" + ) + + +def _is_plain(value: Any, depth: int) -> bool: + """True when ``json.dumps`` can encode ``value`` without a lazy member.""" + if value is None or isinstance(value, (str, int, float, bool)): + return True + if depth <= 0: + return isinstance(value, (dict, list, tuple)) + if isinstance(value, dict): + return all(_is_plain(item, depth - 1) for item in value.values()) + if isinstance(value, (list, tuple)): + return all(_is_plain(item, depth - 1) for item in value) + return False + + +class RawJsonDocument: + """A pre-encoded JSON value: a byte span of a source emitted verbatim. + + The streaming encoder copies its bytes chunk by chunk, so a whole + canonical artifact can be embedded in a larger JSON document (the + legacy inline persistence lane) without decoding or re-encoding it. + """ + + __slots__ = ("_end", "_source", "_start") + + def __init__(self, source: ArtifactSource, *, start: int = 0, end: int | None = None) -> None: + self._source = source + self._start = int(start) + self._end = int(source.identity.size if end is None else end) + + @property + def byte_length(self) -> int: + return self._end - self._start + + def iter_byte_chunks(self, *, chunk_bytes: int = SCAN_CHUNK_BYTES) -> Iterator[bytes]: + self._source.verify_identity() + offset = self._start + while offset < self._end: + chunk = self._source.pread(offset, min(chunk_bytes, self._end - offset)) + if not chunk: + raise CanonicalArtifactError("canonical audit artifact span is truncated") + offset += len(chunk) + yield chunk + self._source.verify_identity() + + def iter_text_chunks(self, *, chunk_bytes: int = SCAN_CHUNK_BYTES) -> Iterator[str]: + decoder = codecs.getincrementaldecoder("utf-8")() + try: + for chunk in self.iter_byte_chunks(chunk_bytes=chunk_bytes): + text = decoder.decode(chunk) + if text: + yield text + tail = decoder.decode(b"", final=True) + except UnicodeDecodeError as exc: + raise CanonicalArtifactSyntaxError( + f"canonical audit artifact is not valid UTF-8: {exc}" + ) from exc + if tail: + yield tail + + +class StreamedJsonString: + """A JSON string value whose text is produced chunk by chunk. + + The streaming encoder escapes each chunk as it goes, so a canonical + encoding of a recipient-scaled member can be embedded as a *string* + member of a larger document (the CTV recovery payload's + ``manifest_set_json``) without ever holding the text whole. Chunks are + Python text, so a boundary never splits a code point and chunk-wise + escaping concatenates exactly. + """ + + __slots__ = ("_chunks_factory",) + + def __init__(self, chunks_factory: Callable[[], Iterable[str]]) -> None: + self._chunks_factory = chunks_factory + + def iter_text_chunks(self) -> Iterator[str]: + return iter(self._chunks_factory()) + + def iter_encoded_chunks(self) -> Iterator[str]: + yield '"' + for chunk in self._chunks_factory(): + if chunk: + yield json.dumps(chunk)[1:-1] + yield '"' + + +class MappedSequence(Sequence): + """A replayable sequence applying ``transform`` to another sequence's items. + + Iteration and indexing apply the transform on demand, so a derived + per-item structure (the CTV recovery artifacts derived from lazy + manifests) never exists as a whole list. + """ + + __slots__ = ("_base", "_transform") + + def __init__(self, base: Sequence[Any], transform: Callable[[Any], Any]) -> None: + self._base = base + self._transform = transform + + def __len__(self) -> int: + return len(self._base) + + def __iter__(self) -> Iterator[Any]: + for item in self._base: + yield self._transform(item) + + def __getitem__(self, index: int | slice) -> Any: + if isinstance(index, slice): + return MappedSequence(self._base[index], self._transform) + return self._transform(self._base[index]) + + def __eq__(self, other: object) -> bool: + if other is self: + return True + return _sequences_equal(self, other) + + def __ne__(self, other: object) -> bool: + result = self.__eq__(other) + if result is NotImplemented: + return result # type: ignore[return-value] + return not result + + __hash__ = None # type: ignore[assignment] + + +class RawJsonRecord(Mapping): + """One JSON value normalized outside the coordinator's decoder. + + Built for records above :data:`RECORD_DECODE_SOFT_LIMIT_BYTES`: an + isolated helper process decodes the record and returns its compact + encodings (original key order, and sorted keys for equality) plus the + members small enough to keep, which are spooled to an unlinked + temporary file. The coordinator never calls ``json.loads`` on the + record. Small members read like a mapping; a member above + :data:`RAW_RECORD_MEMBER_LIMIT_BYTES` is reachable only through the + streamed encodings and raises :class:`CanonicalArtifactError` when + indexed. Equality with a mapping or another record compares the + sorted-key encodings record by record; the streaming encoder emits the + original-order encoding verbatim. + """ + + __slots__ = ( + "__weakref__", + "_is_object", + "_keys", + "_members", + "_omitted", + "_ordered", + "_sorted", + "_spool", + ) + + def __init__( + self, + spool: ArtifactSource, + *, + ordered: tuple[int, int], + sorted_span: tuple[int, int], + keys: Sequence[str], + members: Mapping[str, Any], + omitted: Iterable[str], + is_object: bool, + ) -> None: + self._spool = spool + self._ordered = ordered + self._sorted = sorted_span + self._keys = tuple(keys) + self._members = dict(members) + self._omitted = frozenset(omitted) + self._is_object = bool(is_object) + + @property + def is_object(self) -> bool: + return self._is_object + + @property + def omitted_members(self) -> frozenset[str]: + return self._omitted + + @property + def encoded_length(self) -> int: + return self._ordered[1] - self._ordered[0] + + def __getitem__(self, key: str) -> Any: + if key in self._members: + return self._members[key] + if key in self._omitted: + raise CanonicalArtifactError( + f"record member {key!r} exceeds the in-process decode limit; " + "it is available only through the streamed encodings" + ) + raise KeyError(key) + + def __iter__(self) -> Iterator[str]: + return iter(self._keys) + + def __len__(self) -> int: + return len(self._keys) + + def __contains__(self, key: object) -> bool: + return key in self._members or key in self._omitted + + def __repr__(self) -> str: + return f"RawJsonRecord(members={list(self._keys)}, bytes={self.encoded_length})" + + def ordered_text_chunks(self) -> Iterator[str]: + return RawJsonDocument( + self._spool, + start=self._ordered[0], + end=self._ordered[1], + ).iter_text_chunks() + + def sorted_text_chunks(self) -> Iterator[str]: + return RawJsonDocument( + self._spool, + start=self._sorted[0], + end=self._sorted[1], + ).iter_text_chunks() + + def __eq__(self, other: object) -> bool: + if other is self: + return True + if isinstance(other, RawJsonRecord): + return _text_streams_equal(self.sorted_text_chunks(), other.sorted_text_chunks()) + if isinstance(other, Mapping) or ( + isinstance(other, Sequence) and not isinstance(other, (str, bytes, bytearray)) + ) or other is None or isinstance(other, (str, int, float, bool)): + try: + other_chunks = iter_json_chunks(other, sort_keys=True) + except (TypeError, ValueError): + return False + return _text_streams_equal(self.sorted_text_chunks(), other_chunks) + return NotImplemented + + def __ne__(self, other: object) -> bool: + result = self.__eq__(other) + if result is NotImplemented: + return result # type: ignore[return-value] + return not result + + __hash__ = None # type: ignore[assignment] + + def close(self) -> None: + self._spool.close() + + +def _text_streams_equal(left: Iterable[str], right: Iterable[str]) -> bool: + """Compare two text streams without joining either.""" + left_iter = iter(left) + right_iter = iter(right) + left_buffer = "" + right_buffer = "" + left_done = right_done = False + while True: + while not left_buffer and not left_done: + try: + left_buffer = next(left_iter) + except StopIteration: + left_done = True + while not right_buffer and not right_done: + try: + right_buffer = next(right_iter) + except StopIteration: + right_done = True + if left_done or right_done: + return left_done and right_done and not left_buffer and not right_buffer + size = min(len(left_buffer), len(right_buffer)) + if left_buffer[:size] != right_buffer[:size]: + return False + left_buffer = left_buffer[size:] + right_buffer = right_buffer[size:] + + +def _helper_main(argv: Sequence[str]) -> int: + """Isolated record normalization: stdin record -> header + encodings. + + Runs in a child process so a record of any size is decoded outside the + lease-bearing coordinator. Exit status 2 marks a malformed record. + """ + import resource + + resource.setrlimit(resource.RLIMIT_AS, (RAW_RECORD_HELPER_MEMORY_BYTES, RAW_RECORD_HELPER_MEMORY_BYTES)) + member_limit = RAW_RECORD_MEMBER_LIMIT_BYTES + if len(argv) >= 2 and argv[0] == "--normalize-record": + member_limit = int(argv[1]) + elif argv != ["--normalize-record"]: + sys.stderr.write("usage: --normalize-record [member-limit-bytes]\n") + return 64 + raw = sys.stdin.buffer.read() + try: + value = json.loads(raw) + except (UnicodeDecodeError, ValueError) as exc: + sys.stderr.write(f"malformed record: {exc}\n") + return 2 + ordered = json.dumps(value, separators=_COMPACT_SEPARATORS).encode("utf-8") + sorted_text = json.dumps(value, sort_keys=True, separators=_COMPACT_SEPARATORS).encode("utf-8") + keys: list[str] = [] + members: dict[str, Any] = {} + omitted: list[str] = [] + is_object = isinstance(value, dict) + if is_object: + for key, item in value.items(): + keys.append(str(key)) + if len(json.dumps(item, separators=_COMPACT_SEPARATORS)) <= member_limit: + members[str(key)] = item + else: + omitted.append(str(key)) + header = json.dumps( + { + "is_object": is_object, + "keys": keys, + "members": members, + "omitted": omitted, + "ordered_size": len(ordered), + "sorted_size": len(sorted_text), + }, + separators=_COMPACT_SEPARATORS, + ).encode("utf-8") + out = sys.stdout.buffer + out.write(header + b"\n") + out.write(ordered) + out.write(sorted_text) + out.flush() + return 0 + + +class HelperAdmission: + """Bounded admission slots shared by isolated helper processes. + + Every helper kind that decodes a whole document outside the coordinator + (the record normalizer here, the legacy candidate helper) can share one + instance so their combined memory stays bounded by the slot count. + Waiting is supervised by the caller's deadline/cancellation check. + """ + + __slots__ = ("_semaphore", "slots") + + def __init__(self, slots: int = RAW_RECORD_HELPER_SLOTS) -> None: + self.slots = max(1, int(slots)) + self._semaphore = threading.BoundedSemaphore(self.slots) + + def acquire( + self, + check: Callable[[], None], + *, + poll_seconds: float = RAW_RECORD_HELPER_POLL_SECONDS, + ) -> None: + """Take a slot; ``check`` runs between polls and raises to give up.""" + while not self._semaphore.acquire(timeout=max(0.001, float(poll_seconds))): + check() + + def release(self) -> None: + self._semaphore.release() + + @contextmanager + def hold(self, check: Callable[[], None]) -> Iterator[None]: + self.acquire(check) + try: + check() + yield + finally: + self.release() + + +RECORD_HELPER_ADMISSION = HelperAdmission() + + +def _feed_helper_stdin(process: subprocess.Popen[bytes], source: ArtifactSource, start: int, end: int) -> list[BaseException]: + errors: list[BaseException] = [] + assert process.stdin is not None + try: + offset = start + while offset < end: + chunk = source.pread(offset, min(SCAN_CHUNK_BYTES, end - offset)) + if not chunk: + raise CanonicalArtifactError("canonical audit artifact span is truncated") + process.stdin.write(chunk) + offset += len(chunk) + process.stdin.flush() + except BrokenPipeError: + pass + except BaseException as exc: # surfaced by the supervisor + errors.append(exc) + finally: + try: + process.stdin.close() + except OSError: + pass + return errors + + +def _read_exact(stream: Any, size: int) -> Iterator[bytes]: + remaining = size + while remaining > 0: + chunk = stream.read(min(SCAN_CHUNK_BYTES, remaining)) + if not chunk: + raise ArtifactResourcePressure("record helper output ended early") + remaining -= len(chunk) + yield chunk + + +class _HelperOutput: + """What the stdout reader thread collected: header facts and the spool.""" + + __slots__ = ("error", "header", "no_header", "spool_fd") + + def __init__(self) -> None: + self.header: dict[str, Any] | None = None + self.no_header = False + self.spool_fd: int | None = None + self.error: BaseException | None = None + + +def _read_helper_output(stream: Any, output: _HelperOutput) -> None: + """Read the header line, then stream both encodings into a scratch file.""" + try: + header_line = stream.readline(RAW_RECORD_HELPER_HEADER_BYTES + 1) + if len(header_line) > RAW_RECORD_HELPER_HEADER_BYTES: + raise ArtifactResourcePressure("record helper metadata exceeds its byte limit") + if not header_line: + output.no_header = True + return + try: + header = json.loads(header_line) + parsed = { + "ordered_size": int(header["ordered_size"]), + "sorted_size": int(header["sorted_size"]), + "keys": [str(key) for key in header["keys"]], + "members": dict(header["members"]), + "omitted": [str(key) for key in header["omitted"]], + "is_object": bool(header["is_object"]), + } + except (ValueError, KeyError, TypeError) as exc: + raise ArtifactResourcePressure(f"record helper header is malformed: {exc}") from exc + output.spool_fd = _scratch_fd("record spool") + offset = 0 + for chunk in _read_exact(stream, parsed["ordered_size"] + parsed["sorted_size"]): + _pwrite_all(output.spool_fd, chunk, offset, purpose="record spool") + offset += len(chunk) + output.header = parsed + except BaseException as exc: # classified by the supervisor + output.error = exc + + +def _drain_helper_stderr(stream: Any, tail: bytearray, limit: int) -> None: + """Drain diagnostics so a noisy child never blocks; keep only the tail.""" + try: + while True: + chunk = stream.read(8192) + if not chunk: + return + tail.extend(chunk) + if len(tail) > limit: + del tail[:-limit] + except OSError: + return + + +def normalize_record_isolated( + source: ArtifactSource, + start: int, + end: int, + *, + member_limit: int = RAW_RECORD_MEMBER_LIMIT_BYTES, + timeout_seconds: float = RAW_RECORD_HELPER_TIMEOUT_SECONDS, + cancellation: Callable[[], None] | None = None, + admission: HelperAdmission | None = None, +) -> RawJsonRecord: + """Normalize the record at ``source[start:end)`` in a helper process. + + The record bytes stream to the helper through a pipe and its encodings + stream back into an unlinked scratch file, so the coordinator holds at + most one read chunk of the record at any time. Every protocol step -- + the wait for an admission slot, feeding stdin, reading the header and + the encodings, draining diagnostics -- runs under one deadline + (``timeout_seconds`` from the call) and the optional ``cancellation`` + check, which raises to abandon the helper. On every exit the child is + killed if still alive, reaped, and its pipes closed; a scratch + descriptor not handed to the record is closed. No ``preexec_fn`` is + used: the coordinator is multithreaded. + + A malformed record is a :class:`CanonicalArtifactSyntaxError`. A helper + that cannot be admitted or spawned in time, exits abnormally, times out, + or a scratch file that cannot be written is + :class:`ArtifactResourcePressure`: retryable, never corruption. + """ + deadline = time.monotonic() + float(timeout_seconds) + + def check() -> None: + if cancellation is not None: + cancellation() + if time.monotonic() >= deadline: + raise ArtifactResourcePressure("record helper exceeded its deadline") + + slots = admission if admission is not None else RECORD_HELPER_ADMISSION + slots.acquire(check) + try: + return _run_record_helper(source, start, end, member_limit=member_limit, check=check) + finally: + slots.release() + + +def _run_record_helper( + source: ArtifactSource, + start: int, + end: int, + *, + member_limit: int, + check: Callable[[], None], +) -> RawJsonRecord: + check() + command = [sys.executable, "-m", __name__, "--normalize-record", str(int(member_limit))] + try: + process = subprocess.Popen( + command, + stdin=subprocess.PIPE, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + close_fds=True, + ) + except OSError as exc: + raise ArtifactResourcePressure(f"cannot start the record helper: {exc}") from exc + assert process.stdin is not None and process.stdout is not None and process.stderr is not None + output = _HelperOutput() + feeder_errors: list[BaseException] = [] + diagnostics = bytearray() + threads = [ + threading.Thread( + target=lambda: feeder_errors.extend(_feed_helper_stdin(process, source, start, end)), + name="prism-audit-record-helper-feed", + daemon=True, + ), + threading.Thread( + target=_read_helper_output, + args=(process.stdout, output), + name="prism-audit-record-helper-output", + daemon=True, + ), + threading.Thread( + target=_drain_helper_stderr, + args=(process.stderr, diagnostics, RAW_RECORD_HELPER_DIAGNOSTIC_BYTES), + name="prism-audit-record-helper-stderr", + daemon=True, + ), + ] + try: + for thread in threads: + thread.start() + while process.poll() is None: + check() + if output.error is not None or feeder_errors: + break + time.sleep(RAW_RECORD_HELPER_POLL_SECONDS) + if process.poll() is None: + # A transport error while the child still runs: stop it now so + # the pipe threads unblock, then report the error below. + process.kill() + process.wait() + for thread in threads: + while thread.is_alive(): + check() + thread.join(RAW_RECORD_HELPER_POLL_SECONDS) + tail = diagnostics.decode("utf-8", "replace").strip() + if process.returncode == 2: + raise CanonicalArtifactSyntaxError( + "canonical audit artifact record is malformed: " + tail + ) + if process.returncode != 0: + raise ArtifactResourcePressure( + f"record helper exited with status {process.returncode}: {tail}" + ) + if feeder_errors: + raise feeder_errors[0] + if output.error is not None: + raise output.error + if output.no_header or output.header is None or output.spool_fd is None: + raise ArtifactResourcePressure("record helper produced no header: " + tail) + header = output.header + spool_fd = output.spool_fd + output.spool_fd = None + try: + spool = ArtifactSource(spool_fd, path=None) + except BaseException: + _close_fd(spool_fd) + raise + return RawJsonRecord( + spool, + ordered=(0, header["ordered_size"]), + sorted_span=(header["ordered_size"], header["ordered_size"] + header["sorted_size"]), + keys=header["keys"], + members=header["members"], + omitted=header["omitted"], + is_object=header["is_object"], + ) + finally: + if process.poll() is None: + try: + process.kill() + except OSError: + pass + try: + process.wait() + except OSError: + pass + for stream in (process.stdin, process.stdout, process.stderr): + try: + stream.close() + except OSError: + pass + for thread in threads: + thread.join(timeout=5.0) + if output.spool_fd is not None: + _close_fd(output.spool_fd) + + +def streamed_sha256_json_hex(value: Any) -> str: + """``sha256(json.dumps(value, sort_keys=True, separators=(",", ":")))``, streamed. + + The ledger's ``sha256_json_hex`` over a value that may embed lazy + sequences or isolated records, without building the encoding whole. + """ + digest = hashlib.sha256() + for chunk in iter_json_byte_chunks(value, sort_keys=True): + digest.update(chunk) + return digest.hexdigest() + + +def materialize_json(value: Any) -> Any: + """A plain JSON object graph for consumers that need one. + + Lazy sequences become lists, views become dictionaries, and an isolated + record is decoded in-process from its ordered encoding. This is the + explicit boundary for statement builders that must embed a whole + recipient-scaled member; it is never applied to a window-sized member + on the finalization path. + """ + if isinstance(value, RawJsonRecord): + return json.loads("".join(value.ordered_text_chunks())) + if isinstance(value, RawJsonDocument): + return json.loads("".join(value.iter_text_chunks())) + if isinstance(value, StreamedJsonString): + return "".join(value.iter_text_chunks()) + if not isinstance(value, (Mapping, Sequence)): + # A streamed scalar from another bounded view family (a candidate + # spool string or raw token): text when it has some, else its JSON. + if callable(getattr(value, "iter_text_chunks", None)): + return "".join(value.iter_text_chunks()) + if callable(getattr(value, "iter_encoded_chunks", None)): + return json.loads("".join(value.iter_encoded_chunks())) + if isinstance(value, Mapping): + return {str(key): materialize_json(item) for key, item in value.items()} + if isinstance(value, (str, bytes, bytearray)) or not isinstance(value, Sequence): + return value + return [materialize_json(item) for item in value] + + +class _Coalescer: + __slots__ = ("_chunk_chars", "_pending", "_pending_chars") + + def __init__(self, chunk_chars: int) -> None: + self._chunk_chars = int(chunk_chars) + self._pending: list[str] = [] + self._pending_chars = 0 + + def push(self, text: str) -> str | None: + if not text: + return None + self._pending.append(text) + self._pending_chars += len(text) + if self._pending_chars < self._chunk_chars: + return None + return self.flush() + + def flush(self) -> str | None: + if not self._pending: + return None + chunk = self._pending[0] if len(self._pending) == 1 else "".join(self._pending) + self._pending = [] + self._pending_chars = 0 + return chunk + + +def _encode_record_batches( + records: Iterable[Any], + *, + sort_keys: bool, + batch_records: int, + chunk_chars: int, +) -> Iterator[str]: + """Array body (no brackets) of a record stream, isolated records verbatim. + + Plain records are encoded ``batch_records`` at a time through one + ``json.dumps``; an isolated :class:`RawJsonRecord` streams its own + encoding between batches. + """ + batch: list[Any] = [] + first = True + + def flush() -> Iterator[str]: + nonlocal batch, first + if not batch: + return + text = json.dumps(batch, sort_keys=sort_keys, separators=_COMPACT_SEPARATORS)[1:-1] + batch = [] + if not first: + text = "," + text + first = False + yield text + + for record in records: + if isinstance(record, RawJsonRecord): + yield from flush() + if not first: + yield "," + first = False + yield from (record.sorted_text_chunks() if sort_keys else record.ordered_text_chunks()) + continue + if not _is_plain(record, 2): + yield from flush() + if not first: + yield "," + first = False + yield from _encode_pieces( + record, + sort_keys=sort_keys, + batch_records=batch_records, + chunk_chars=chunk_chars, + ) + continue + batch.append(record) + if len(batch) >= batch_records: + yield from flush() + yield from flush() + + +def _encode_pieces( + value: Any, + *, + sort_keys: bool, + batch_records: int, + chunk_chars: int, +) -> Iterator[str]: + if isinstance(value, RawJsonRecord): + yield from (value.sorted_text_chunks() if sort_keys else value.ordered_text_chunks()) + return + if isinstance(value, RawJsonDocument): + if sort_keys: + raise TypeError("a pre-encoded document cannot be re-sorted") + yield from value.iter_text_chunks() + return + if isinstance(value, StreamedJsonString): + yield from value.iter_encoded_chunks() + return + if not isinstance(value, (Mapping, Sequence)) and callable( + getattr(value, "iter_encoded_chunks", None) + ): + # A streamed scalar from another bounded view family (a candidate + # spool string or raw token) is emitted verbatim; its encoding does + # not depend on key order. Lazy containers take the generic Mapping + # and Sequence routes below, member by member. + yield from value.iter_encoded_chunks() + return + if isinstance(value, Mapping): + keys = list(value) + if any(not isinstance(key, str) for key in keys): + yield json.dumps( + value if isinstance(value, dict) else dict(value), + sort_keys=sort_keys, + separators=_COMPACT_SEPARATORS, + ) + return + if sort_keys: + keys.sort() + yield "{" + first = True + for key in keys: + yield ("" if first else ",") + json.dumps(key) + ":" + first = False + yield from _encode_pieces( + value[key], + sort_keys=sort_keys, + batch_records=batch_records, + chunk_chars=chunk_chars, + ) + yield "}" + return + if isinstance(value, (str, bytes, bytearray)) or not isinstance(value, Sequence): + yield json.dumps(value, sort_keys=sort_keys, separators=_COMPACT_SEPARATORS) + return + yield "[" + if isinstance(value, (LazyRecordSequence, LazyRecordSlice, MappedSequence)): + yield from _encode_record_batches( + value, + sort_keys=sort_keys, + batch_records=batch_records, + chunk_chars=chunk_chars, + ) + elif isinstance(value, (list, tuple)) and _is_plain(value, 2): + yield from iter_json_array_text_chunks( + value, + sort_keys=sort_keys, + batch_records=batch_records, + chunk_chars=chunk_chars, + ) + else: + first = True + for item in value: + if not first: + yield "," + first = False + yield from _encode_pieces( + item, + sort_keys=sort_keys, + batch_records=batch_records, + chunk_chars=chunk_chars, + ) + yield "]" + + +def iter_json_chunks( + value: Any, + *, + sort_keys: bool = False, + batch_records: int = JSON_BATCH_RECORDS, + chunk_chars: int = JSON_CHUNK_CHARS, +) -> Iterator[str]: + """``json.dumps(value, separators=(",", ":"), sort_keys=...)`` in bounded text chunks. + + Mappings are encoded member by member, lazy record sequences and plain + lists batch by batch (each ``json.dumps`` covers at most + ``batch_records`` items), isolated records and pre-encoded documents + verbatim from their scratch files, and every other value through one + ``json.dumps`` bounded by that value's own size. Concatenating the + chunks reproduces the compact encoding byte for byte; every chunk is + ASCII unless a pre-encoded document carries raw UTF-8. + """ + coalescer = _Coalescer(chunk_chars) + for piece in _encode_pieces( + value, + sort_keys=sort_keys, + batch_records=batch_records, + chunk_chars=chunk_chars, + ): + chunk = coalescer.push(piece) + if chunk is not None: + yield chunk + chunk = coalescer.flush() + if chunk is not None: + yield chunk + + +def iter_json_byte_chunks( + value: Any, + *, + sort_keys: bool = False, + batch_records: int = JSON_BATCH_RECORDS, + chunk_chars: int = JSON_CHUNK_CHARS, +) -> Iterator[bytes]: + """:func:`iter_json_chunks`, UTF-8 encoded chunk by chunk.""" + for chunk in iter_json_chunks( + value, + sort_keys=sort_keys, + batch_records=batch_records, + chunk_chars=chunk_chars, + ): + yield chunk.encode("utf-8") + + +def json_chunks_sha256_and_size(chunks: Iterable[bytes]) -> tuple[str, int]: + """Digest and byte count of a chunk stream without joining it.""" + digest = hashlib.sha256() + size = 0 + for chunk in chunks: + digest.update(chunk) + size += len(chunk) + return digest.hexdigest(), size + + +def _sequences_equal(left: Sequence[Any], right: Any) -> bool: + if isinstance(right, (str, bytes, bytearray)) or not isinstance(right, Sequence): + return NotImplemented # type: ignore[return-value] + if len(left) != len(right): + return False + right_iter = iter(right) + for item in left: + try: + other = next(right_iter) + except StopIteration: + return False + if item != other: + return False + try: + next(right_iter) + except StopIteration: + return True + return False + + +class LazyRecordSequence(Sequence): + """A replayable, read-only sequence of JSON records inside an artifact. + + Records live between ``start`` (the byte after the opening bracket) and + ``end`` (the byte after the closing bracket) of one JSON array in the + source. Iteration streams the array through bounded windows; indexing + seeks to the checkpoint preceding the index and skips forward. Slices + with unit step return a :class:`LazyRecordSlice` over the same file; + other slices return a list bounded by the slice itself. + """ + + __slots__ = ( + "__weakref__", + "_cancellation", + "_chunk_bytes", + "_count", + "_end", + "_index", + "_index_base", + "_source", + "_start", + "_stats", + "_stride", + ) + + def __init__( + self, + source: ArtifactSource, + *, + start: int, + end: int, + count: int, + index: _CheckpointIndex, + index_base: int, + stride: int = RECORD_INDEX_STRIDE, + chunk_bytes: int = SCAN_CHUNK_BYTES, + stats: ScanStats | None = None, + cancellation: Callable[[], None] | None = None, + ) -> None: + self._source = source + self._start = int(start) + self._end = int(end) + self._count = int(count) + self._index = index + self._index_base = int(index_base) + self._stride = max(1, int(stride)) + self._chunk_bytes = int(chunk_bytes) + self._stats = stats if stats is not None else ScanStats() + # Runs once per read chunk of every lazy read and before each + # isolated-helper step; raises to abandon the work. + self._cancellation = cancellation + + def _decode_record(self, cursor: _Cursor) -> Any: + """Decode the record at the cursor, isolating an oversized one. + + The record is first skipped structurally (bounded), which yields its + byte span. A record within the in-process limit is decoded from the + window text when it is still held, otherwise from one positional + read of exactly that span; a larger record goes to the isolated + helper and comes back as a :class:`RawJsonRecord`. + """ + cursor.skip_ws() + text_start = cursor.pos + generation = cursor.generation + start = cursor.byte_pos + cursor.skip_value() + end = cursor.byte_pos + size = end - start + if size <= RECORD_DECODE_SOFT_LIMIT_BYTES: + if cursor.generation == generation: + text = cursor.text[text_start : cursor.pos] + else: + text = self._source.pread(start, size).decode("utf-8") + try: + return json.loads(text) + except ValueError as exc: + raise CanonicalArtifactSyntaxError( + f"canonical audit artifact record is malformed: {exc}" + ) from exc + self._stats.isolated_records += 1 + return normalize_record_isolated( + self._source, start, end, cancellation=self._cancellation + ) + + @property + def source(self) -> ArtifactSource: + return self._source + + @property + def byte_range(self) -> tuple[int, int]: + return self._start, self._end + + def __len__(self) -> int: + return self._count + + def __repr__(self) -> str: + return f"LazyRecordSequence(count={self._count}, bytes={self._end - self._start})" + + def _cursor(self, offset: int) -> _Cursor: + return _Cursor( + self._source, + offset, + chunk_bytes=self._chunk_bytes, + checkpoint=self._cancellation, + ) + + def iter_range(self, start: int, stop: int) -> Iterator[Any]: + """Records ``start`` (inclusive) to ``stop`` (exclusive), streamed.""" + start = max(0, int(start)) + stop = min(self._count, int(stop)) + if start >= stop: + return + self._source.verify_identity() + block = start // self._stride + cursor = self._cursor(self._index.get(self._index_base + block)) + index = block * self._stride + while index < start: + cursor.skip_ws() + cursor.skip_value() + cursor.skip_ws() + cursor.expect(",") + cursor.trim() + index += 1 + while index < stop: + yield self._decode_record(cursor) + index += 1 + cursor.trim() + if index < self._count: + cursor.skip_ws() + cursor.expect(",") + if stop == self._count: + cursor.skip_ws() + cursor.expect("]") + self._source.verify_identity() + + def __iter__(self) -> Iterator[Any]: + return self.iter_range(0, self._count) + + def __getitem__(self, index: int | slice) -> Any: + if isinstance(index, slice): + start, stop, step = index.indices(self._count) + if step == 1: + return LazyRecordSlice(self, start, max(start, stop)) + return [self[position] for position in range(start, stop, step)] + if isinstance(index, bool) or not isinstance(index, int): + raise TypeError("lazy record indices must be integers or slices") + if index < 0: + index += self._count + if not 0 <= index < self._count: + raise IndexError("lazy record index out of range") + for record in self.iter_range(index, index + 1): + return record + raise IndexError("lazy record index out of range") + + def batches(self, size: int) -> Iterator[LazyRecordSlice]: + """Consecutive slices of at most ``size`` records.""" + size = max(1, int(size)) + for start in range(0, self._count, size): + yield LazyRecordSlice(self, start, min(self._count, start + size)) + + def __eq__(self, other: object) -> bool: + if other is self: + return True + return _sequences_equal(self, other) + + def __ne__(self, other: object) -> bool: + result = self.__eq__(other) + if result is NotImplemented: + return result # type: ignore[return-value] + return not result + + __hash__ = None # type: ignore[assignment] + + +class LazyRecordSlice(Sequence): + """A bounded window of a :class:`LazyRecordSequence`, streamed on demand.""" + + __slots__ = ("_parent", "_start", "_stop") + + def __init__(self, parent: LazyRecordSequence, start: int, stop: int) -> None: + self._parent = parent + self._start = int(start) + self._stop = int(stop) + + def __len__(self) -> int: + return self._stop - self._start + + def __repr__(self) -> str: + return f"LazyRecordSlice({self._start}:{self._stop})" + + def __iter__(self) -> Iterator[Any]: + return self._parent.iter_range(self._start, self._stop) + + def __getitem__(self, index: int | slice) -> Any: + length = len(self) + if isinstance(index, slice): + start, stop, step = index.indices(length) + if step == 1: + return LazyRecordSlice( + self._parent, + self._start + start, + self._start + max(start, stop), + ) + return [self[position] for position in range(start, stop, step)] + if isinstance(index, bool) or not isinstance(index, int): + raise TypeError("lazy record indices must be integers or slices") + if index < 0: + index += length + if not 0 <= index < length: + raise IndexError("lazy record index out of range") + return self._parent[self._start + index] + + def __eq__(self, other: object) -> bool: + if other is self: + return True + return _sequences_equal(self, other) + + def __ne__(self, other: object) -> bool: + result = self.__eq__(other) + if result is NotImplemented: + return result # type: ignore[return-value] + return not result + + __hash__ = None # type: ignore[assignment] + + +def is_lazy_sequence(value: object) -> bool: + return isinstance(value, (LazyRecordSequence, LazyRecordSlice)) + + +def _scan_array( + cursor: _Cursor, + *, + source: ArtifactSource, + index: _CheckpointIndex, + stride: int, + chunk_bytes: int, + cancellation: Callable[[], None] | None = None, +) -> Any: + cursor.skip_ws() + if cursor.peek() != "[": + return cursor.decode_value() + cursor.pos += 1 + start = cursor.byte_pos + index_base = index.count + count = 0 + cursor.skip_ws() + if cursor.peek() == "]": + cursor.pos += 1 + return LazyRecordSequence( + source, + start=start, + end=cursor.byte_pos, + count=0, + index=index, + index_base=index_base, + stride=stride, + chunk_bytes=chunk_bytes, + stats=cursor.stats, + cancellation=cancellation, + ) + while True: + cursor.skip_ws() + if count % stride == 0: + index.append(cursor.byte_pos) + cursor.skip_value() + count += 1 + cursor.trim() + cursor.skip_ws() + char = cursor.peek() + if char == ",": + cursor.pos += 1 + continue + if char == "]": + cursor.pos += 1 + break + raise CanonicalArtifactSyntaxError( + f"canonical audit artifact is malformed at byte {cursor.byte_pos}: " + "expected ',' or ']'" + ) + return LazyRecordSequence( + source, + start=start, + end=cursor.byte_pos, + count=count, + index=index, + index_base=index_base, + stride=stride, + chunk_bytes=chunk_bytes, + stats=cursor.stats, + cancellation=cancellation, + ) + + +def _parse_object( + cursor: _Cursor, + path: tuple[str, ...], + *, + lazy_paths: Sequence[tuple[str, ...]], + source: ArtifactSource, + index: _CheckpointIndex, + stride: int, + chunk_bytes: int, + cancellation: Callable[[], None] | None = None, +) -> dict[str, Any]: + cursor.skip_ws() + cursor.expect("{") + members: dict[str, Any] = {} + cursor.skip_ws() + if cursor.peek() == "}": + cursor.pos += 1 + return members + while True: + cursor.skip_ws() + if cursor.peek() != '"': + raise CanonicalArtifactSyntaxError( + f"canonical audit artifact is malformed at byte {cursor.byte_pos}: " + "expected a member name" + ) + key = cursor.decode_value() + cursor.skip_ws() + cursor.expect(":") + member_path = path + (str(key),) + if member_path in lazy_paths: + value = _scan_array( + cursor, + source=source, + index=index, + stride=stride, + chunk_bytes=chunk_bytes, + cancellation=cancellation, + ) + elif any( + len(lazy) > len(member_path) and lazy[: len(member_path)] == member_path + for lazy in lazy_paths + ): + cursor.skip_ws() + if cursor.peek() == "{": + value = _parse_object( + cursor, + member_path, + lazy_paths=lazy_paths, + source=source, + index=index, + stride=stride, + chunk_bytes=chunk_bytes, + cancellation=cancellation, + ) + else: + value = cursor.decode_value() + else: + cursor.skip_ws() + value = cursor.decode_value() + members[str(key)] = value + cursor.trim() + cursor.skip_ws() + char = cursor.peek() + if char == ",": + cursor.pos += 1 + continue + if char == "}": + cursor.pos += 1 + return members + raise CanonicalArtifactSyntaxError( + f"canonical audit artifact is malformed at byte {cursor.byte_pos}: " + "expected ',' or '}'" + ) + + +class CanonicalAuditBundleView(Mapping): + """Bounded, immutable mapping view of one canonical audit artifact. + + Construct with :meth:`scan`, which reads the file exactly once through + bounded windows, digests the raw bytes as it goes, parses every member + outside ``lazy_paths`` eagerly and indexes the lazy arrays. The view + binds the file identity it scanned; every later lazy read re-checks it. + + Equality with a dictionary (either operand order) compares member by + member, streaming the lazy arrays record by record, so callers keep the + full logical comparison the dictionary path performed without building + a window-sized object graph. + """ + + def __init__( + self, + source: ArtifactSource, + members: dict[str, Any], + *, + sha256_hex: str, + lazy_paths: tuple[tuple[str, ...], ...], + scan_stats: ScanStats | None = None, + index: _CheckpointIndex | None = None, + ) -> None: + self._source = source + self._members = members + self._sha256_hex = sha256_hex + self._lazy_paths = lazy_paths + self._scan_stats = scan_stats if scan_stats is not None else ScanStats() + self._index = index + + @classmethod + def scan( + cls, + fd: int, + *, + path: Path | None = None, + cancellation: Callable[[], None] | None = None, + chunk_bytes: int = SCAN_CHUNK_BYTES, + stride: int = RECORD_INDEX_STRIDE, + lazy_paths: Sequence[tuple[str, ...]] = CANONICAL_BUNDLE_LAZY_PATHS, + ) -> CanonicalAuditBundleView: + """Scan the artifact behind ``fd``; the view takes ownership of ``fd``. + + ``cancellation`` runs once per read chunk so a cancelled build stops + within one bounded read; the lazy sequences the view hands out keep + it, so later lazy reads and isolated-helper runs honour it as well. + The whole file is consumed so the returned digest covers every + byte, and trailing garbage is rejected. + """ + source = ArtifactSource(fd, path=path) + try: + index = _CheckpointIndex() + except BaseException: + source.close() + raise + try: + hasher = hashlib.sha256() + stats = ScanStats() + cursor = _Cursor( + source, + 0, + chunk_bytes=chunk_bytes, + hasher=hasher, + checkpoint=cancellation, + stats=stats, + ) + lazy = tuple(tuple(str(part) for part in item) for item in lazy_paths) + members = _parse_object( + cursor, + (), + lazy_paths=lazy, + source=source, + index=index, + stride=stride, + chunk_bytes=chunk_bytes, + cancellation=cancellation, + ) + cursor.skip_ws() + if not cursor.at_eof(): + raise CanonicalArtifactSyntaxError( + "canonical audit artifact has trailing data after the bundle" + ) + source.verify_identity() + if cursor.byte_pos != source.identity.size: + raise CanonicalArtifactError( + "canonical audit artifact size does not match its scan" + ) + index.flush() + stats.checkpoint_entries = index.count + except BaseException: + index.close() + source.close() + raise + return cls( + source, + members, + sha256_hex=hasher.hexdigest(), + lazy_paths=lazy, + scan_stats=stats, + index=index, + ) + + @classmethod + def scan_path( + cls, + path: Path, + *, + cancellation: Callable[[], None] | None = None, + chunk_bytes: int = SCAN_CHUNK_BYTES, + stride: int = RECORD_INDEX_STRIDE, + lazy_paths: Sequence[tuple[str, ...]] = CANONICAL_BUNDLE_LAZY_PATHS, + ) -> CanonicalAuditBundleView: + """Open ``path`` without following symlinks and scan it.""" + fd = os.open(path, os.O_RDONLY | getattr(os, "O_NOFOLLOW", 0)) + return cls.scan( + fd, + path=Path(path), + cancellation=cancellation, + chunk_bytes=chunk_bytes, + stride=stride, + lazy_paths=lazy_paths, + ) + + # Mapping protocol. + + def __getitem__(self, key: str) -> Any: + return self._members[key] + + def __iter__(self) -> Iterator[str]: + return iter(self._members) + + def __len__(self) -> int: + return len(self._members) + + def __contains__(self, key: object) -> bool: + return key in self._members + + def __repr__(self) -> str: + return ( + f"CanonicalAuditBundleView(sha256={self._sha256_hex}, " + f"bytes={self.byte_length}, members={list(self._members)})" + ) + + def __eq__(self, other: object) -> bool: + if other is self: + return True + if not isinstance(other, Mapping): + return NotImplemented + if len(self._members) != len(other): + return False + for key, value in self._members.items(): + if key not in other: + return False + if value != other[key]: + return False + return True + + def __ne__(self, other: object) -> bool: + result = self.__eq__(other) + if result is NotImplemented: + return result # type: ignore[return-value] + return not result + + __hash__ = None # type: ignore[assignment] + + # Artifact binding. + + @property + def source(self) -> ArtifactSource: + return self._source + + @property + def path(self) -> Path | None: + return self._source.path + + @property + def identity(self) -> ArtifactIdentity: + return self._source.identity + + @property + def sha256_hex(self) -> str: + return self._sha256_hex + + @property + def byte_length(self) -> int: + return self._source.identity.size + + @property + def lazy_paths(self) -> tuple[tuple[str, ...], ...]: + return self._lazy_paths + + @property + def scan_stats(self) -> ScanStats: + """Allocation maxima observed by the initial scan of this artifact.""" + return self._scan_stats + + def verify_identity(self) -> None: + self._source.verify_identity() + + def iter_bytes(self, *, chunk_bytes: int = SCAN_CHUNK_BYTES) -> Iterator[bytes]: + return self._source.iter_bytes(chunk_bytes=chunk_bytes) + + def key_index(self, key: str) -> int: + return list(self._members).index(key) + + def without(self, *keys: str) -> dict[str, Any]: + """Ordered copy of the members minus ``keys``; values are shared.""" + return { + key: value for key, value in self._members.items() if key not in keys + } + + def close(self) -> None: + """Retire the artifact and index descriptors this view owns. + + Lazy sequences handed out earlier keep their own references to the + source and index and fail closed once these are gone. + """ + if self._index is not None: + self._index.close() + self._source.close() + + @property + def closed(self) -> bool: + return self._source.closed + + +def compare_streaming(left: Any, right: Any) -> bool: + """``left == right`` where either side may embed lazy sequences or views.""" + return bool(left == right) + + +def spool_json_document( + chunks: Iterable[bytes], + *, + lazy_paths: Sequence[tuple[str, ...]], + stride: int = RECORD_INDEX_STRIDE, +) -> CanonicalAuditBundleView: + """Write a JSON document to an unlinked scratch file and scan it lazily. + + Used for derived metadata that would otherwise become a window-scaled + list -- the compact body's share-part index -- so it stays replayable + on disk with the same bounded readers as the artifact itself. + """ + fd = _scratch_fd("json spool") + try: + offset = 0 + for chunk in chunks: + _pwrite_all(fd, chunk, offset, purpose="json spool") + offset += len(chunk) + except BaseException: + _close_fd(fd) + raise + return CanonicalAuditBundleView.scan(fd, lazy_paths=lazy_paths, stride=stride) + + +__all__ = [ + "ArtifactIdentity", + "ArtifactResourcePressure", + "ArtifactSource", + "CANONICAL_BUNDLE_LAZY_PATHS", + "CanonicalArtifactError", + "CanonicalArtifactSyntaxError", + "CanonicalAuditBundleView", + "HelperAdmission", + "JSON_BATCH_RECORDS", + "JSON_CHUNK_CHARS", + "LazyRecordSequence", + "LazyRecordSlice", + "MappedSequence", + "RAW_RECORD_HELPER_DIAGNOSTIC_BYTES", + "RAW_RECORD_HELPER_POLL_SECONDS", + "RAW_RECORD_HELPER_SLOTS", + "RAW_RECORD_HELPER_TIMEOUT_SECONDS", + "RAW_RECORD_MEMBER_LIMIT_BYTES", + "RECORD_DECODE_SOFT_LIMIT_BYTES", + "RECORD_HELPER_ADMISSION", + "RECORD_INDEX_STRIDE", + "RawJsonDocument", + "RawJsonRecord", + "SCAN_CHUNK_BYTES", + "ScanStats", + "StreamedJsonString", + "compare_streaming", + "is_lazy_sequence", + "iter_json_byte_chunks", + "iter_json_chunks", + "json_chunks_sha256_and_size", + "materialize_json", + "normalize_record_isolated", + "spool_json_document", + "streamed_sha256_json_hex", +] + + +if __name__ == "__main__": + raise SystemExit(_helper_main(sys.argv[1:])) diff --git a/lab/prism/audit_http.py b/lab/prism/audit_http.py new file mode 100644 index 000000000..9f5e4ff14 --- /dev/null +++ b/lab/prism/audit_http.py @@ -0,0 +1,706 @@ +"""Operator audit/ops HTTP routing and bounded server lifecycle for PRISM. + +The public ``/public/v1`` read surface is served by its own process +(``lab/prism/public_read_service.py``), so nothing here imports +``lab.prism.public_api``: an unknown ``/public/v1`` request reaches this +listener's ordinary unknown-endpoint 404. +""" + +from __future__ import annotations + +from dataclasses import dataclass +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer +import json +import threading +import time +from typing import Mapping, Protocol +import urllib.parse + + +class MetricsResponse(Protocol): + """What ``/metrics`` needs from a cached metrics read (issue #184). + + Structural on purpose: this module stays free of the observability policy + that decides the state and names the headers, and only writes what it is + handed. + """ + + @property + def status(self) -> int: ... + + @property + def body(self) -> str: ... + + def response_headers(self) -> Mapping[str, str]: ... + + +class AuditHttpPort(Protocol): + """Purpose-specific application reads exposed to the HTTP facade.""" + + def cached_health_payload(self) -> tuple[int, Mapping[str, object]]: ... + + def cached_metrics_payload(self) -> MetricsResponse: ... + + def latest_evidence_payload(self) -> Mapping[str, object] | None: ... + + def owed_balances_payload(self) -> Mapping[str, object]: ... + + def carry_forward_integrity_payload(self) -> Mapping[str, object]: ... + + def miner_status_payload(self, recipient_id: str) -> Mapping[str, object]: ... + + def ledger_backend(self) -> str: ... + + def audit_share_window( + self, + *, + anchor_job_issued_at_ms: int, + network_difficulty: int, + ) -> list[dict[str, object]]: ... + + def audit_block_payouts(self, *, block_hash: str) -> list[dict[str, object]]: ... + + def audit_ctv_fanouts(self, *, block_hash: str) -> list[dict[str, object]]: ... + + def audit_ctv_fanout_manifest_set( + self, + *, + block_hash: str, + ) -> Mapping[str, object] | None: ... + + def ctv_fanout_status( + self, + *, + fanout_txid: str, + ) -> Mapping[str, object] | None: ... + + def pending_ctv_fanout_statuses( + self, + *, + limit: int, + ) -> list[dict[str, object]]: ... + + def audit_bundle( + self, + *, + block_hash: str, + ) -> Mapping[str, object] | None: ... + + def audit_bundle_by_commitment( + self, + *, + commitment_leaf_hex: str, + ) -> Mapping[str, object] | None: ... + + +@dataclass(frozen=True) +class AuditHttpConfig: + bind: str + port: int + thread_name: str = "prism-audit-http" + join_timeout_seconds: float = 1.0 + + def __post_init__(self) -> None: + if not self.bind: + raise ValueError("audit HTTP bind must not be empty") + if self.port < 0 or self.port > 65_535: + raise ValueError("audit HTTP port must be between 0 and 65535") + if not self.thread_name: + raise ValueError("audit HTTP thread name must not be empty") + if self.join_timeout_seconds < 0: + raise ValueError("audit HTTP join timeout must be nonnegative") + + +@dataclass(frozen=True) +class AuditHttpState: + lifecycle: str + bound_address: tuple[str, int] | None + thread_alive: bool + + +class _BoundedThreadingHttpServer(ThreadingHTTPServer): + daemon_threads = True + block_on_close = False + allow_reuse_address = True + + def __init__( + self, + server_address: tuple[str, int], + handler_type: type[BaseHTTPRequestHandler], + ) -> None: + self.ready = threading.Event() + self._startup_lock = threading.Lock() + self._startup_cancelled = False + self._serve_committed = False + super().__init__(server_address, handler_type) + + def service_actions(self) -> None: + self.ready.set() + + def serve_unless_startup_cancelled(self, *, poll_interval: float) -> None: + """Commit atomically to serving or honor startup cancellation.""" + + with self._startup_lock: + if self._startup_cancelled: + return + self._serve_committed = True + self.serve_forever(poll_interval=poll_interval) + + def cancel_startup(self) -> bool: + """Cancel an uncommitted start and report whether serving committed. + + A True result means ``serve_forever()`` runs (or already ran), so + ``BaseServer.shutdown()`` will be answered; a False result means the + serve thread is guaranteed to bail out before entering the loop, so + blocking ``shutdown()`` must never be called for this server. + """ + + with self._startup_lock: + self._startup_cancelled = True + return self._serve_committed + + +class AuditHttpFacade: + """Own exact route behavior plus the listener socket and server thread.""" + + def __init__( + self, + port: AuditHttpPort, + config: AuditHttpConfig | None = None, + ) -> None: + self.port = port + self.config = config + self._lifecycle_lock = threading.Lock() + self._server: _BoundedThreadingHttpServer | None = None + self._thread: threading.Thread | None = None + self._serve_exited: threading.Event | None = None + self._lifecycle = "new" + self._handler_type = self._make_handler_type() + + def handler_type(self) -> type[BaseHTTPRequestHandler]: + return self._handler_type + + def state(self) -> AuditHttpState: + with self._lifecycle_lock: + return self._state_locked() + + def start(self) -> AuditHttpState: + config = self.config + if config is None: + raise RuntimeError("audit HTTP lifecycle requires configuration") + with self._lifecycle_lock: + if self._thread is not None and self._thread.is_alive(): + return self._state_locked() + server = _BoundedThreadingHttpServer( + (config.bind, config.port), + self._handler_type, + ) + serve_exited = threading.Event() + thread = threading.Thread( + target=self._serve, + args=(server, serve_exited), + name=config.thread_name, + daemon=True, + ) + self._server = server + self._thread = thread + self._serve_exited = serve_exited + self._lifecycle = "starting" + try: + thread.start() + except Exception: + server.server_close() + self._server = None + self._thread = None + self._serve_exited = None + self._lifecycle = "stopped" + raise + try: + if not server.ready.wait(1.0): + raise RuntimeError("audit HTTP server did not enter its serve loop") + except Exception: + self._request_serve_exit_bounded( + server, + thread, + serve_exited, + config.join_timeout_seconds, + ) + server.server_close() + thread.join(timeout=config.join_timeout_seconds) + with self._lifecycle_lock: + if self._server is server: + self._server = None + self._thread = None + self._serve_exited = None + self._lifecycle = ( + "stopped" if not thread.is_alive() else "stop_timeout" + ) + elif ( + self._server is None + and self._thread is None + and not thread.is_alive() + ): + self._lifecycle = "stopped" + raise + with self._lifecycle_lock: + if self._server is not server or self._thread is not thread: + raise RuntimeError("audit HTTP server stopped during startup") + if not thread.is_alive(): + self._lifecycle = "exited" + raise RuntimeError("audit HTTP server exited during startup") + self._lifecycle = "running" + return self._state_locked() + + def _serve( + self, + server: _BoundedThreadingHttpServer, + serve_exited: threading.Event, + ) -> None: + try: + server.serve_unless_startup_cancelled(poll_interval=0.05) + finally: + try: + server.server_close() + finally: + serve_exited.set() + with self._lifecycle_lock: + if self._server is server: + self._server = None + self._thread = None + self._serve_exited = None + if self._lifecycle in {"starting", "running"}: + self._lifecycle = "exited" + + @staticmethod + def _request_serve_exit_bounded( + server: _BoundedThreadingHttpServer, + thread: threading.Thread, + serve_exited: threading.Event, + timeout: float, + ) -> None: + """Ask the serve loop to exit without ever blocking unboundedly. + + ``BaseServer.shutdown()`` waits for ``serve_forever()`` to exit, so + calling it directly can block forever when a stop races startup + before the serve loop commits (the documented old-layer gap). + Cancel the startup window atomically first; only a committed loop is + woken, from a disposable daemon thread this caller never joins, and + the wait for the loop's exit is bounded by ``timeout``. + """ + + if not server.cancel_startup(): + # The serve thread is guaranteed to observe the cancellation + # before entering serve_forever(); there is no loop to wake. + return + if not thread.is_alive(): + return + threading.Thread( + target=server.shutdown, + name=f"{thread.name}-shutdown", + daemon=True, + ).start() + serve_exited.wait(timeout) + + def stop(self) -> bool: + with self._lifecycle_lock: + server = self._server + thread = self._thread + serve_exited = self._serve_exited + if server is None or thread is None: + self._lifecycle = "stopped" + return True + self._lifecycle = "stopping" + join_timeout = self.config.join_timeout_seconds if self.config else 1.0 + deadline = time.monotonic() + join_timeout + if serve_exited is not None: + self._request_serve_exit_bounded( + server, + thread, + serve_exited, + max(0.0, deadline - time.monotonic()), + ) + server.server_close() + thread.join(timeout=max(0.0, deadline - time.monotonic())) + stopped = not thread.is_alive() + with self._lifecycle_lock: + if stopped and self._server is server: + self._server = None + self._thread = None + self._serve_exited = None + self._lifecycle = "stopped" if stopped else "stop_timeout" + return stopped + + def _state_locked(self) -> AuditHttpState: + server = self._server + thread = self._thread + return AuditHttpState( + lifecycle=self._lifecycle, + bound_address=( + None + if server is None + else (str(server.server_address[0]), int(server.server_address[1])) + ), + thread_alive=bool(thread is not None and thread.is_alive()), + ) + + def _make_handler_type(self) -> type[BaseHTTPRequestHandler]: + facade = self + + class AuditHandler(BaseHTTPRequestHandler): + server_version = "QbitPrismAudit/0.1" + + def do_GET(self) -> None: + parsed = urllib.parse.urlparse(self.path) + path = parsed.path.rstrip("/") or "/" + query = urllib.parse.parse_qs(parsed.query) + try: + if path == "/healthz": + status, payload = facade.port.cached_health_payload() + self.write_json(status, payload) + return + if path == "/metrics": + metrics = facade.port.cached_metrics_payload() + self.write_text( + metrics.status, + metrics.body, + "text/plain; version=0.0.4", + # Freshness is stated in the response metadata as + # well as the body, so a scraper or an operator can + # tell a stale payload from a current one without + # parsing Prometheus text (issue #184). + headers=metrics.response_headers(), + ) + return + if path == "/audit/latest": + payload = facade.port.latest_evidence_payload() + if payload is None: + self.write_json( + 404, + {"error": "no PRISM evidence has been produced"}, + ) + else: + self.write_json(200, payload) + return + if path in {"/owed", "/owed-balances"}: + self.write_json(200, facade.port.owed_balances_payload()) + return + if path in { + "/audit/carry-forward-integrity", + "/audit/ledger-integrity", + }: + self.write_json( + 200, + facade.port.carry_forward_integrity_payload(), + ) + return + if path.startswith("/miners/") and path.endswith("/status"): + recipient_id = urllib.parse.unquote( + path.removeprefix("/miners/").removesuffix("/status") + ) + self.write_json( + 200, + facade.port.miner_status_payload(recipient_id), + ) + return + if path.startswith("/payouts/") and path.endswith("/status"): + recipient_id = urllib.parse.unquote( + path.removeprefix("/payouts/").removesuffix("/status") + ) + self.write_json( + 200, + facade.port.miner_status_payload(recipient_id), + ) + return + if path == "/audit/share-window": + self.handle_share_window(query) + return + if path.startswith("/audit/blocks/") and path.endswith( + "/payouts" + ): + block_hash = path.removeprefix( + "/audit/blocks/" + ).removesuffix("/payouts") + self.handle_block_payouts(block_hash) + return + if path.startswith("/audit/blocks/") and path.endswith( + "/ctv-fanouts" + ): + block_hash = path.removeprefix( + "/audit/blocks/" + ).removesuffix("/ctv-fanouts") + self.handle_block_ctv_fanouts(block_hash) + return + if path.startswith("/audit/blocks/") and path.endswith( + "/ctv-fanout-manifest-set" + ): + block_hash = path.removeprefix( + "/audit/blocks/" + ).removesuffix("/ctv-fanout-manifest-set") + self.handle_block_ctv_fanout_manifest_set(block_hash) + return + if path == "/audit/fanouts/pending": + self.handle_pending_ctv_fanouts(query) + return + if path.startswith("/audit/fanouts/") and path.endswith( + "/status" + ): + fanout_txid = path.removeprefix( + "/audit/fanouts/" + ).removesuffix("/status") + self.handle_ctv_fanout_status(fanout_txid) + return + if path.startswith("/audit/commitments/") and path.endswith( + "/bundle" + ): + commitment_leaf_hex = path.removeprefix( + "/audit/commitments/" + ).removesuffix("/bundle") + self.handle_commitment_bundle(commitment_leaf_hex) + return + if path.startswith("/audit/block/"): + block_hash = path.removeprefix("/audit/block/") + self.handle_block_payouts(block_hash) + return + if path.startswith("/audit/blocks/") and path.endswith( + "/bundle" + ): + block_hash = path.removeprefix( + "/audit/blocks/" + ).removesuffix("/bundle") + self.handle_block_bundle(block_hash) + return + self.write_json(404, {"error": "unknown endpoint"}) + except ValueError as exc: + self.write_json(400, {"error": str(exc)}) + except Exception as exc: + self.write_json(500, {"error": str(exc)}) + + def handle_share_window(self, query: dict[str, list[str]]) -> None: + anchor_raw = self.first_query_value( + query, + "anchor_job_issued_at_ms", + "anchor", + ) + difficulty_raw = self.first_query_value( + query, + "network_difficulty", + ) + if anchor_raw is None or difficulty_raw is None: + raise ValueError( + "anchor_job_issued_at_ms and network_difficulty are required" + ) + rows = facade.port.audit_share_window( + anchor_job_issued_at_ms=int(anchor_raw), + network_difficulty=int(difficulty_raw), + ) + self.write_json( + 200, + { + "schema": "qbit.prism.audit-share-window.v1", + "ledger_backend": facade.port.ledger_backend(), + "rows": rows, + }, + ) + + def handle_block_payouts(self, block_hash: str) -> None: + block_hash = self.clean_hash(block_hash) + rows = facade.port.audit_block_payouts(block_hash=block_hash) + if not rows: + self.write_json( + 404, + { + "error": "unknown PRISM block", + "block_hash": block_hash, + }, + ) + return + self.write_json( + 200, + { + "schema": "qbit.prism.audit-block-payouts.v1", + "ledger_backend": facade.port.ledger_backend(), + "block_hash": block_hash, + "rows": rows, + }, + ) + + def handle_block_ctv_fanouts(self, block_hash: str) -> None: + block_hash = self.clean_hash(block_hash, name="block hash") + rows = facade.port.audit_ctv_fanouts(block_hash=block_hash) + if not rows: + self.write_json( + 404, + { + "error": "unknown CTV fanout block", + "block_hash": block_hash, + }, + ) + return + self.write_json( + 200, + { + "schema": "qbit.prism.audit-ctv-fanouts.v1", + "ledger_backend": facade.port.ledger_backend(), + "block_hash": block_hash, + "rows": rows, + }, + ) + + def handle_block_ctv_fanout_manifest_set(self, block_hash: str) -> None: + block_hash = self.clean_hash(block_hash, name="block hash") + payload = facade.port.audit_ctv_fanout_manifest_set( + block_hash=block_hash + ) + if payload is None: + self.write_json( + 404, + { + "error": "unknown CTV fanout block", + "block_hash": block_hash, + }, + ) + return + self.write_json(200, payload) + + def handle_ctv_fanout_status(self, fanout_txid: str) -> None: + fanout_txid = self.clean_hash(fanout_txid, name="fanout txid") + payload = facade.port.ctv_fanout_status(fanout_txid=fanout_txid) + if payload is None: + self.write_json( + 404, + { + "error": "unknown CTV fanout", + "fanout_txid": fanout_txid, + }, + ) + return + self.write_json(200, payload) + + def handle_pending_ctv_fanouts( + self, + query: dict[str, list[str]], + ) -> None: + limit_raw = self.first_query_value(query, "limit") + limit = int(limit_raw) if limit_raw is not None else 100 + rows = facade.port.pending_ctv_fanout_statuses(limit=limit) + self.write_json( + 200, + { + "schema": "qbit.prism.pending-ctv-fanouts.v1", + "ledger_backend": facade.port.ledger_backend(), + "count": len(rows), + "rows": rows, + }, + ) + + def handle_block_bundle(self, block_hash: str) -> None: + block_hash = self.clean_hash(block_hash, name="block hash") + payload = facade.port.audit_bundle(block_hash=block_hash) + if payload is None: + self.write_json( + 404, + { + "error": "unknown PRISM block", + "block_hash": block_hash, + }, + ) + return + self.write_json(200, payload) + + def handle_commitment_bundle(self, commitment_leaf_hex: str) -> None: + commitment_leaf_hex = self.clean_hash( + commitment_leaf_hex, + name="audit commitment leaf", + ) + payload = facade.port.audit_bundle_by_commitment( + commitment_leaf_hex=commitment_leaf_hex + ) + if payload is None: + self.write_json( + 404, + { + "error": "unknown PRISM audit commitment", + "audit_commitment_leaf_hex": commitment_leaf_hex, + }, + ) + return + self.write_json(200, payload) + + def write_json( + self, + status: int, + payload: object, + headers: dict[str, str] | None = None, + ) -> None: + # The one content-addressed payload this listener used to + # serve (#154's /public/v1/artifacts/{sha256} RawJsonBody) + # left with the public read surface, so no route reaching + # here carries pre-serialized bytes and this module needs no + # public-API import at all. + body = json.dumps(payload, sort_keys=True).encode() + b"\n" + try: + self.send_response(status) + self.send_header("Content-Type", "application/json") + self.send_header("Content-Length", str(len(body))) + for key, value in (headers or {}).items(): + self.send_header(key, value) + self.end_headers() + self.wfile.write(body) + except (BrokenPipeError, ConnectionResetError): + # The client (typically a health checker with a short + # timeout) hung up before the response was written; + # nothing to salvage. + return + + def write_text( + self, + status: int, + payload: str, + content_type: str, + headers: Mapping[str, str] | None = None, + ) -> None: + body = payload.encode() + try: + self.send_response(status) + self.send_header("Content-Type", content_type) + self.send_header("Content-Length", str(len(body))) + for key, value in (headers or {}).items(): + self.send_header(key, value) + self.end_headers() + self.wfile.write(body) + except (BrokenPipeError, ConnectionResetError): + return + + def log_message(self, format: str, *args: object) -> None: + return + + @staticmethod + def first_query_value( + query: dict[str, list[str]], + *keys: str, + ) -> str | None: + for key in keys: + values = query.get(key) + if values: + return values[0] + return None + + @staticmethod + def clean_hash(value: str, *, name: str = "block hash") -> str: + value = urllib.parse.unquote(value).strip() + if len(value) != 64 or any( + char not in "0123456789abcdefABCDEF" for char in value + ): + raise ValueError(f"{name} must be 64 hex characters") + return value.lower() + + return AuditHandler + + +__all__ = [ + "AuditHttpConfig", + "AuditHttpFacade", + "AuditHttpPort", + "AuditHttpState", + "MetricsResponse", +] diff --git a/lab/prism/backfill_audit_bundle_canonical.py b/lab/prism/backfill_audit_bundle_canonical.py new file mode 100644 index 000000000..1c4fae8ae --- /dev/null +++ b/lab/prism/backfill_audit_bundle_canonical.py @@ -0,0 +1,1117 @@ +#!/usr/bin/env python3 +"""Backfill immutable canonical audit-bundle artifacts for existing rows. + +The live publication path owns one immutable canonical artifact per +``(block_hash, audit_bundle_sha256)``: the exact byte sequence the advertised +digest was computed over, published once and never rewritten. Rows written +before that artifact existed still advertise a digest with nothing canonical +behind it. Three shapes are in the table today: + +* a pre-v2 literal external body, whose ``body_uri`` file *is* the canonical + byte sequence -- recovering it is a copy, not a re-serialization; +* a modern compact/v2 external body, which stores the bundle without its + shares plus content-addressed share segments, and has to be reconstructed + through the artifact store before it can be canonicalized; +* an inline ``qbit_pool_audit_bundles.audit_bundle`` JSONB body, the oldest + layout, which is canonicalized with the repository's own canonicalizer. + +This tool walks those rows in primary-key order, recovers the canonical bytes +for each, and verifies the advertised digest *before* anything reaches the +filesystem. It never rewrites a digest: a row whose recovered bytes do not +hash to the advertised value is reported on stderr, left untouched, and turned +into a nonzero exit status once the requested range has been processed. + +Every publication is preceded, immediately, by the ledger's pre-publication +preflight (writer-lease refresh plus current row identity confirmation), so a +writer that has lost its lease -- or a row that changed underneath this scan -- +cannot publish. + +The run is idempotent and resumable. Rows are paged by ``block_hash`` (the +primary key), the last visited hash is reported as ``last_checkpoint``, and +``--start-after`` resumes from it. Revisiting an already published row is +safe: its canonical artifact is recognized and skipped. Failed row identities +and reasons are retained in the machine-readable ``failed_rows`` summary. A +checkpoint advances past failures, so retry them by starting before the first +failed hash (or by rerunning the idempotent scan from the beginning). + +Publish mode owns the coordinator writer lease exclusively. Operators must +stop the coordinator before publishing; the command refuses lease waits rather +than sitting behind a live same-identity owner. Once the run finishes or +raises, it releases the exact lease and closes both ledger and artifact-store +resources before returning. +""" + +from __future__ import annotations + +import argparse +import hashlib +import hmac +import inspect +import json +import os +import sys +from dataclasses import asdict, dataclass +from pathlib import Path +from typing import Any, Callable, Iterator, Mapping, Sequence + +if not __package__: + sys.path.insert(0, str(Path(__file__).resolve().parents[2])) + +from lab.prism.audit_artifacts import ( + AuditArtifactConfig, + AuditArtifactStore, + DEFAULT_AUDIT_SHARE_SEGMENT_SIZE, +) +from lab.prism.backfill_ctv_fanouts import psql_command_from_env +from lab.prism.bundle_compiler import canonical_bundle_bytes +from lab.prism.coordinator_config import ( + DEFAULT_LEASE_ACQUIRE_ATTEMPTS, + DEFAULT_LEASE_ACQUIRE_LOCK_TIMEOUT_SECONDS, + DEFAULT_POSTGRES_IDLE_IN_TRANSACTION_TIMEOUT_SECONDS, + DEFAULT_POSTGRES_TCP_KEEPALIVES_COUNT, + DEFAULT_POSTGRES_TCP_KEEPALIVES_IDLE_SECONDS, + DEFAULT_POSTGRES_TCP_KEEPALIVES_INTERVAL_SECONDS, + env, + env_nonnegative_int, + env_positive_float, + env_positive_int, +) +from lab.prism.share_ledger import PsqlShareLedger, canonical_hex + + +BACKFILL_SUMMARY_SCHEMA = "qbit.prism.audit-bundle-canonical-backfill.v1" +DEFAULT_BATCH_SIZE = 100 + +# Every capability this tool needs from phase 2 is resolved by name at +# construction, first match wins, and the resolved names are reported in the +# summary. Reconciling a final signature difference is then a one-line edit +# here plus a summary line that says exactly what the run bound to. +READ_CANONICAL_CANDIDATES = ("read_canonical_audit_bundle",) +WRITE_CANONICAL_CANDIDATES = ("write_canonical_audit_bundle",) +LITERAL_BODY_CANDIDATES = ( + "literal_canonical_bundle_bytes", + "read_literal_audit_body", + "read_verified_literal_audit_body", + "read_pre_v2_audit_body", + "read_literal_external_audit_body", +) +RECONSTRUCT_CANDIDATES = ( + "canonical_bundle_bytes_from_external_body", + "reconstruct_external_audit_body", + "read_external_body", +) +LEDGER_PREFLIGHT_CANDIDATES = ( + "preflight_canonical_bundle_publication", + "preflight_audit_bundle_publication", + "confirm_audit_bundle_publication", + "refresh_lease_and_confirm_audit_bundle", +) +LEDGER_SHARE_RANGE_CANDIDATES = ( + "load_audit_share_ledger_range", + "_load_audit_share_ledger_range", +) +LEDGER_READ_JSON_CANDIDATES = ("_run_read_json", "_run_json") +PREFLIGHT_DIGEST_KEYWORDS = ( + "audit_bundle_sha256", + "digest", + "audit_bundle_sha256_hex", +) +# Whatever the reconstruction helper hands its missing-range callback, one of +# these carries the segment digest the recovered range must reproduce. +SEGMENT_DIGEST_KEYS = ( + "expected_sha256", + "range_sha256", + "prefix_sha256", + "sha256", + "segment_sha256", +) + + +class CanonicalBackfillMismatch(RuntimeError): + """Recovered bytes do not reproduce the advertised digest.""" + + +class CanonicalBackfillUnrecoverable(RuntimeError): + """No source for this row yielded bytes that could be verified.""" + + +class LedgerPreflightRefused(RuntimeError): + """The ledger declined to authorize this publication.""" + + +class MissingCanonicalCapability(RuntimeError): + """A required phase-2 method is not present on the injected object.""" + + +class BackfillWriterLeaseUnavailable(RuntimeError): + """Publish mode could not take exclusive ownership of the writer lease.""" + + +def _sha256_hex(payload: bytes) -> str: + return hashlib.sha256(payload).hexdigest() + + +def _accepts_keyword(func: Callable[..., Any], name: str) -> bool: + """Report whether ``func`` takes ``name`` as a keyword argument. + + The phase-2 signatures are assumed, not observed, so every optional + keyword this tool would like to pass is probed rather than assumed. An + unintrospectable callable (a C builtin, an exotic mock) is given the + benefit of the doubt: the documented signature is the contract. + """ + + try: + signature = inspect.signature(func) + except (TypeError, ValueError): + return True + for parameter in signature.parameters.values(): + if parameter.kind is inspect.Parameter.VAR_KEYWORD: + return True + if parameter.name == name and parameter.kind in ( + inspect.Parameter.POSITIONAL_OR_KEYWORD, + inspect.Parameter.KEYWORD_ONLY, + ): + return True + return False + + +def _resolve_capability( + owner: object, + candidates: Sequence[str], + *, + description: str, +) -> tuple[str, Callable[..., Any]]: + for name in candidates: + value = getattr(owner, name, None) + if callable(value): + return name, value + raise MissingCanonicalCapability( + f"{description} is unavailable on {type(owner).__name__}; " + f"tried: {', '.join(candidates)}" + ) + + +def _sql_text_literal(value: str) -> str: + return "'" + str(value).replace("'", "''") + "'" + + +def audit_bundle_page_sql( + *, + start_after: str | None, + limit: int, + text_literal: Callable[[str], str] = _sql_text_literal, +) -> str: + """One bounded, resumable page of ``qbit_pool_audit_bundles``. + + ``block_hash`` is the table's primary key, so ordering by it is stable and + ``block_hash > `` resumes exactly where the previous page stopped. + The range predicate and the ordering share the column's collation, so the + pagination stays self-consistent whatever that collation is. + """ + + if limit <= 0: + raise ValueError("audit bundle page limit must be positive") + predicate = ( + "" if start_after is None else f" WHERE block_hash > {text_literal(start_after)}\n" + ) + return f""" +SELECT COALESCE(json_agg(json_build_object( + 'block_hash', page.block_hash, + 'audit_bundle_sha256', page.audit_bundle_sha256, + 'body_uri', page.body_uri, + 'audit_bundle', page.audit_bundle +) ORDER BY page.block_hash ASC), '[]'::json) +FROM ( + SELECT block_hash, audit_bundle_sha256, body_uri, audit_bundle + FROM qbit_pool_audit_bundles +{predicate} ORDER BY block_hash ASC + LIMIT {int(limit)} +) AS page; +""" + + +@dataclass(frozen=True) +class AuditBundleRow: + block_hash: str + audit_bundle_sha256: str + body_uri: str | None + audit_bundle: dict[str, Any] | None + + +@dataclass(frozen=True) +class RecoveredCanonical: + canonical_bytes: bytes + source: str + + +class CanonicalArtifactAdapter: + """The phase-2 canonical artifact surface, bound by name in one place. + + Phase 2 owns ``read_canonical_audit_bundle`` / + ``write_canonical_audit_bundle``, a verified pre-v2 literal-body reader, + and a verified external-body reconstruction helper that accepts a + missing-range callback. This adapter is the only place that names them, + so a final signature difference is reconciled here and nowhere else. + """ + + def __init__(self, store: object) -> None: + self._store = store + self._read_name, self._read = _resolve_capability( + store, + READ_CANONICAL_CANDIDATES, + description="canonical audit bundle reader", + ) + self._write_name, self._write = _resolve_capability( + store, + WRITE_CANONICAL_CANDIDATES, + description="canonical audit bundle writer", + ) + self._literal_name, self._literal = _resolve_capability( + store, + LITERAL_BODY_CANDIDATES, + description="verified pre-v2 literal audit body reader", + ) + self._reconstruct_name, self._reconstruct = _resolve_capability( + store, + RECONSTRUCT_CANDIDATES, + description="verified external audit body reconstruction helper", + ) + self._canonicalize_name, self._canonicalize = _resolve_capability( + store, + ("canonical_audit_bundle_bytes",), + description="audit bundle canonicalizer", + ) + self._literal_takes_digest = _accepts_keyword(self._literal, "expected_sha256") + self._reconstruct_takes_digest = _accepts_keyword( + self._reconstruct, + "expected_sha256", + ) + self._reconstruct_takes_callback = _accepts_keyword( + self._reconstruct, + "load_missing_range", + ) + + def resolved_interface(self) -> dict[str, str]: + return { + "read_canonical": self._read_name, + "write_canonical": self._write_name, + "literal_body_reader": self._literal_name, + "external_body_reconstruction": self._reconstruct_name, + "canonicalizer": self._canonicalize_name, + } + + def read_canonical(self, block_hash: str, digest: str) -> bytes | None: + payload = self._read(block_hash, digest) + return None if payload is None else bytes(payload) + + def write_canonical(self, block_hash: str, digest: str, payload: bytes) -> Any: + return self._write(block_hash, digest, payload) + + def read_literal_body(self, body_uri: object, digest: str) -> bytes | None: + """Return the pre-v2 body bytes verbatim, or None when it is not one. + + A compact or v2 wrapper is not the canonical byte sequence, so the + reader is expected to decline it rather than hand back wrapper bytes. + """ + + if self._literal_takes_digest: + payload = self._literal(body_uri, expected_sha256=digest) + else: + payload = self._literal(body_uri) + return None if payload is None else bytes(payload) + + def canonical_bytes_from_external_body( + self, + block_hash: str, + body_uri: object, + digest: str, + load_missing_range: Callable[..., list[Any]], + ) -> bytes | None: + if self._reconstruct_name == "canonical_bundle_bytes_from_external_body": + payload = self._reconstruct( + block_hash=block_hash, + audit_bundle_sha256=digest, + body_uri=body_uri, + load_missing_range=load_missing_range, + ) + return None if payload is None else bytes(payload) + kwargs: dict[str, Any] = {} + if self._reconstruct_takes_digest: + kwargs["expected_sha256"] = digest + if self._reconstruct_takes_callback: + kwargs["load_missing_range"] = load_missing_range + bundle = self._reconstruct(body_uri, **kwargs) + if bundle is None: + return None + if not isinstance(bundle, dict): + raise CanonicalBackfillUnrecoverable( + "external audit body reconstruction did not return a bundle object" + ) + return self.canonical_bytes(bundle) + + def canonical_bytes(self, bundle: Mapping[str, Any]) -> bytes: + return bytes(self._canonicalize(dict(bundle))) + + def segment_storage_bytes( + self, + *, + first_share_seq: int, + last_share_seq: int, + shares: list[Any], + ) -> bytes: + """Canonicalize a recovered share range with the store's own encoder.""" + + payload = self._store.audit_share_segment_payload( + first_share_seq=first_share_seq, + last_share_seq=last_share_seq, + shares=shares, + ) + return bytes(self._store.storage_json_bytes(payload)) + + +class AuditBundleLedgerAdapter: + """The PostgreSQL surface this backfill needs, bound by name in one place.""" + + def __init__(self, ledger: object) -> None: + self._ledger = ledger + self._read_json_name, self._read_json = _resolve_capability( + ledger, + LEDGER_READ_JSON_CANDIDATES, + description="ledger read-slot JSON runner", + ) + self._share_range_name, self._share_range = _resolve_capability( + ledger, + LEDGER_SHARE_RANGE_CANDIDATES, + description="ledger audit share range reader", + ) + self._preflight_name, self._preflight = _resolve_capability( + ledger, + LEDGER_PREFLIGHT_CANDIDATES, + description="ledger publication preflight", + ) + self._preflight_digest_keyword = next( + ( + keyword + for keyword in PREFLIGHT_DIGEST_KEYWORDS + if _accepts_keyword(self._preflight, keyword) + ), + PREFLIGHT_DIGEST_KEYWORDS[0], + ) + text_literal = getattr(ledger, "_text_literal", None) + self._text_literal = ( + text_literal if callable(text_literal) else _sql_text_literal + ) + + def resolved_interface(self) -> dict[str, str]: + return { + "read_json": self._read_json_name, + "share_range_reader": self._share_range_name, + "publication_preflight": self._preflight_name, + "preflight_digest_keyword": self._preflight_digest_keyword, + } + + def page(self, *, start_after: str | None, limit: int) -> list[AuditBundleRow]: + sql = audit_bundle_page_sql( + start_after=start_after, + limit=limit, + text_literal=self._text_literal, + ) + rows = self._read_json(sql) + if not isinstance(rows, list): + raise RuntimeError("audit bundle page did not return a JSON array") + page: list[AuditBundleRow] = [] + for row in rows: + if not isinstance(row, Mapping): + raise RuntimeError("audit bundle page returned a non-object row") + bundle = row.get("audit_bundle") + page.append( + AuditBundleRow( + block_hash=str(row.get("block_hash") or ""), + audit_bundle_sha256=str(row.get("audit_bundle_sha256") or ""), + body_uri=( + None if row.get("body_uri") is None else str(row["body_uri"]) + ), + audit_bundle=dict(bundle) if isinstance(bundle, Mapping) else None, + ) + ) + return page + + def load_share_ledger_range( + self, + *, + first_share_seq: int, + last_share_seq: int, + ) -> list[Any]: + shares = self._share_range( + first_share_seq=first_share_seq, + last_share_seq=last_share_seq, + ) + if not isinstance(shares, list): + raise RuntimeError("audit share ledger range did not return a list") + return shares + + def preflight_publication(self, *, block_hash: str, digest: str) -> Any: + """Refresh the writer lease and confirm this row's identity. + + Called immediately before every filesystem publication. A refusal -- + raised, reported through an ``error`` key, or an explicit falsy + ``confirmed`` -- withholds the publication. + """ + + result = self._preflight( + block_hash=block_hash, + **{self._preflight_digest_keyword: digest}, + ) + if isinstance(result, Mapping): + error = result.get("error") + if error: + raise LedgerPreflightRefused(str(error)) + confirmed = result.get("confirmed") + if confirmed is not None and not confirmed: + raise LedgerPreflightRefused( + "ledger preflight did not confirm the audit bundle row identity" + ) + elif result is False: + raise LedgerPreflightRefused( + "ledger preflight did not confirm the audit bundle row identity" + ) + return result + + +@dataclass +class BackfillCounters: + visited: int = 0 + written: int = 0 + already_present: int = 0 + dry_run: int = 0 + literal_freebie: int = 0 + reconstructed: int = 0 + inline_canonicalized: int = 0 + db_segment_reloads: int = 0 + mismatch: int = 0 + errors: int = 0 + + +class CanonicalBundleBackfill: + """Recover and publish canonical bytes for existing audit bundle rows.""" + + def __init__( + self, + *, + artifacts: CanonicalArtifactAdapter, + ledger: AuditBundleLedgerAdapter, + dry_run: bool = False, + batch_size: int = DEFAULT_BATCH_SIZE, + limit: int | None = None, + start_after: str | None = None, + stderr: Any = None, + ) -> None: + batch_size = int(batch_size) + if batch_size <= 0: + raise ValueError("batch_size must be positive") + if limit is not None: + limit = int(limit) + if limit <= 0: + raise ValueError("limit must be positive when given") + self._artifacts = artifacts + self._ledger = ledger + self._dry_run = bool(dry_run) + self._batch_size = batch_size + self._limit = limit + self._start_after = start_after + self._stderr = sys.stderr if stderr is None else stderr + self._counters = BackfillCounters() + self._checkpoint = start_after + self._failed_rows: list[dict[str, str | None]] = [] + + def run(self) -> dict[str, Any]: + for row in self._iter_rows(): + self._counters.visited += 1 + # The checkpoint advances over every visited row, including the + # ones this run refuses to publish. Those are reported loudly and + # make the run exit nonzero; silently rewinding to them on the next + # resume would stall the scan on a row that needs an operator. + self._checkpoint = row.block_hash + self._process_row(row) + return self.summary() + + def summary(self) -> dict[str, Any]: + counters = asdict(self._counters) + return { + "schema": BACKFILL_SUMMARY_SCHEMA, + **counters, + "last_checkpoint": self._checkpoint, + "start_after": self._start_after, + "batch_size": self._batch_size, + "limit": self._limit, + "dry_run_mode": self._dry_run, + "exit_code": self.exit_code(), + "failed_rows": [dict(failure) for failure in self._failed_rows], + "interface": { + "artifacts": self._artifacts.resolved_interface(), + "ledger": self._ledger.resolved_interface(), + }, + } + + def exit_code(self) -> int: + return 1 if (self._counters.mismatch or self._counters.errors) else 0 + + def _iter_rows(self) -> Iterator[AuditBundleRow]: + cursor = self._start_after + remaining = self._limit + while remaining is None or remaining > 0: + page_size = ( + self._batch_size if remaining is None else min(self._batch_size, remaining) + ) + page = self._ledger.page(start_after=cursor, limit=page_size) + if not page: + return + for row in page: + yield row + next_cursor = page[-1].block_hash + # The scan only terminates while the cursor strictly advances. A + # page whose last row carries no usable key would otherwise reissue + # the same query forever; the primary key makes that unreachable in + # a healthy database, so it is reported rather than tolerated. + if not next_cursor or (cursor is not None and next_cursor <= cursor): + self._counters.errors += 1 + self._report( + "error", + page[-1], + "audit bundle page did not advance the resume checkpoint", + ) + return + cursor = next_cursor + if remaining is not None: + remaining -= len(page) + if len(page) < page_size: + return + + def _report(self, kind: str, row: AuditBundleRow, reason: str) -> None: + self._failed_rows.append( + { + "kind": kind, + "block_hash": row.block_hash or None, + "audit_bundle_sha256": row.audit_bundle_sha256 or None, + "reason": reason, + } + ) + print( + f"prism canonical audit bundle backfill {kind} " + f"block={row.block_hash or ''} " + f"digest={row.audit_bundle_sha256 or ''} " + f"reason={reason}", + file=self._stderr, + flush=True, + ) + + def _process_row(self, row: AuditBundleRow) -> None: + try: + block_hash = canonical_hex( + row.block_hash, + name="block_hash", + expected_bytes=32, + ) + digest = canonical_hex( + row.audit_bundle_sha256, + name="audit_bundle_sha256", + expected_bytes=32, + ) + except (ValueError, TypeError) as exc: + self._counters.errors += 1 + self._report("error", row, f"row identity is not canonical: {exc}") + return + + try: + if self._already_published(block_hash, digest): + self._counters.already_present += 1 + return + recovered = self._recover(row, digest) + actual = _sha256_hex(recovered.canonical_bytes) + if not hmac.compare_digest(actual, digest): + raise CanonicalBackfillMismatch( + f"recovered {recovered.source} bytes hash to {actual}" + ) + except CanonicalBackfillMismatch as exc: + self._counters.mismatch += 1 + self._report("mismatch", row, str(exc)) + return + except Exception as exc: # noqa: BLE001 - one bad row must not end the range + self._counters.errors += 1 + self._report("error", row, f"{type(exc).__name__}: {exc}") + return + + if recovered.source == "literal": + self._counters.literal_freebie += 1 + elif recovered.source == "reconstructed": + self._counters.reconstructed += 1 + else: + self._counters.inline_canonicalized += 1 + + if self._dry_run: + self._counters.dry_run += 1 + return + + try: + # Preflight is the last thing before the filesystem is touched: + # the lease is refreshed and the row identity re-confirmed with no + # verification work left to run in between. + self._ledger.preflight_publication(block_hash=block_hash, digest=digest) + self._artifacts.write_canonical( + block_hash, + digest, + recovered.canonical_bytes, + ) + except Exception as exc: # noqa: BLE001 - report and keep scanning + self._counters.errors += 1 + self._report("error", row, f"{type(exc).__name__}: {exc}") + return + self._counters.written += 1 + + def _already_published(self, block_hash: str, digest: str) -> bool: + """Report whether a verified canonical artifact already exists. + + An unreadable artifact is treated as absent -- publication is + idempotent, so recovering and republishing is the useful answer. An + artifact that reads back with the wrong digest is not absent; it is a + corrupt publication, and it is reported rather than overwritten. + """ + + try: + existing = self._artifacts.read_canonical(block_hash, digest) + except Exception: # noqa: BLE001 - fall through to recovery + return False + if existing is None: + return False + actual = _sha256_hex(existing) + if not hmac.compare_digest(actual, digest): + raise CanonicalBackfillMismatch( + f"existing canonical artifact hashes to {actual}" + ) + return True + + def _recover(self, row: AuditBundleRow, digest: str) -> RecoveredCanonical: + reasons: list[str] = [] + if row.body_uri: + literal = self._recover_literal(row, digest, reasons) + if literal is not None: + return literal + canonical = self._reconstruct(row, digest, reasons) + if canonical is not None: + actual = _sha256_hex(canonical) + if not hmac.compare_digest(actual, digest): + raise CanonicalBackfillMismatch( + f"reconstructed external body canonicalizes to {actual}" + ) + return RecoveredCanonical(canonical, "reconstructed") + if row.audit_bundle is not None: + canonical = self._artifacts.canonical_bytes(row.audit_bundle) + actual = _sha256_hex(canonical) + if not hmac.compare_digest(actual, digest): + raise CanonicalBackfillMismatch( + f"inline bundle canonicalizes to {actual}" + ) + return RecoveredCanonical(canonical, "inline") + if not row.body_uri: + reasons.append("row has neither an inline bundle nor an external body") + else: + reasons.append("external body did not yield the advertised bytes") + raise CanonicalBackfillUnrecoverable("; ".join(reasons)) + + def _recover_literal( + self, + row: AuditBundleRow, + digest: str, + reasons: list[str], + ) -> RecoveredCanonical | None: + """Promote a pre-v2 literal body verbatim when it already verifies. + + These bytes *are* the canonical serialization the digest was computed + over, so they are copied, never parsed and re-emitted: re-serializing + would substitute this process's encoder for the one that produced the + advertised digest. + """ + + try: + literal = self._artifacts.read_literal_body(row.body_uri, digest) + except Exception as exc: # noqa: BLE001 - a compact body is not a failure + reasons.append(f"literal body reader: {type(exc).__name__}: {exc}") + return None + if literal is None: + return None + actual = _sha256_hex(literal) + if hmac.compare_digest(actual, digest): + return RecoveredCanonical(literal, "literal") + # Not a mismatch on its own: a compact or v2 wrapper never hashes to + # the bundle digest, and reconstruction is the path that reads it. + reasons.append(f"external body bytes hash to {actual}") + return None + + def _reconstruct( + self, + row: AuditBundleRow, + digest: str, + reasons: list[str], + ) -> bytes | None: + try: + return self._artifacts.canonical_bytes_from_external_body( + row.block_hash, + row.body_uri, + digest, + self._load_missing_range, + ) + except CanonicalBackfillMismatch: + raise + except Exception as exc: # noqa: BLE001 - recorded, then inline is tried + reasons.append( + f"external body reconstruction: {type(exc).__name__}: {exc}" + ) + return None + + def _load_missing_range( + self, + *, + first_share_seq: int, + last_share_seq: int, + **extra: Any, + ) -> list[Any]: + """Recover an absent on-disk share segment from the append-only ledger. + + The range is read from PostgreSQL, canonicalized with the store's own + segment encoder, and checked against the segment digest the bundle + embeds. A range that does not reproduce that digest is refused here, + before it can reach a reconstruction the final bundle digest would + then have to catch. + """ + + first_share_seq = int(first_share_seq) + last_share_seq = int(last_share_seq) + shares = self._ledger.load_share_ledger_range( + first_share_seq=first_share_seq, + last_share_seq=last_share_seq, + ) + try: + sequences = [int(share["share_seq"]) for share in shares] + except (KeyError, TypeError, ValueError) as exc: + raise CanonicalBackfillUnrecoverable( + "ledger share range has an invalid share_seq" + ) from exc + if sequences != list(range(first_share_seq, last_share_seq + 1)): + raise CanonicalBackfillUnrecoverable( + "ledger share range " + f"{first_share_seq}-{last_share_seq} is incomplete or out of order" + ) + segment_bytes = self._artifacts.segment_storage_bytes( + first_share_seq=first_share_seq, + last_share_seq=last_share_seq, + shares=shares, + ) + expected = _expected_segment_digest(extra) + if expected is not None: + actual = _sha256_hex(segment_bytes) + if not hmac.compare_digest(actual, expected): + raise CanonicalBackfillMismatch( + f"ledger share range {first_share_seq}-{last_share_seq} " + f"canonicalizes to {actual}, not the embedded {expected}" + ) + self._counters.db_segment_reloads += 1 + return shares + + +def _expected_segment_digest(extra: Mapping[str, Any]) -> str | None: + """Pull the embedded segment digest out of the callback's extra kwargs.""" + + for key in SEGMENT_DIGEST_KEYS: + value = extra.get(key) + if value: + return str(value).lower() + part = extra.get("part") + if isinstance(part, Mapping): + for key in SEGMENT_DIGEST_KEYS: + value = part.get(key) + if value: + return str(value).lower() + return None + + +def build_audit_store_from_env( + environ: Mapping[str, str] | None = None, + *, + read_only: bool = False, +) -> AuditArtifactStore: + """Open the audit artifact root the way every PRISM entry point does.""" + + source = os.environ if environ is None else environ + evidence_path = Path( + env("PRISM_EVIDENCE_PATH", "prism-live-evidence.json", environ=source) + ) + root = Path(env("PRISM_AUDIT_DIR", str(evidence_path.parent), environ=source)) + return AuditArtifactStore( + AuditArtifactConfig( + root=root, + evidence_path=evidence_path, + share_segment_size=env_nonnegative_int( + "PRISM_AUDIT_SHARE_SEGMENT_SIZE", + DEFAULT_AUDIT_SHARE_SEGMENT_SIZE, + environ=source, + ), + ), + canonicalizer=canonical_bundle_bytes, + read_only=read_only, + ) + + +def build_parser() -> argparse.ArgumentParser: + parser = argparse.ArgumentParser( + description=( + "Backfill immutable canonical audit-bundle artifacts for existing " + "qbit_pool_audit_bundles rows." + ) + ) + parser.add_argument( + "--start-after", + default=None, + help="resume after this block hash checkpoint (see last_checkpoint)", + ) + parser.add_argument( + "--batch-size", + type=int, + default=env_positive_int( + "PRISM_AUDIT_CANONICAL_BACKFILL_BATCH_SIZE", + DEFAULT_BATCH_SIZE, + ), + help="rows fetched per page", + ) + parser.add_argument( + "--limit", + type=int, + default=None, + help="stop after visiting this many rows in this run", + ) + parser.add_argument( + "--dry-run", + action="store_true", + help="recover and verify every row without publishing anything", + ) + parser.add_argument( + "--psql-command", + help="psql command; defaults to PRISM_POSTGRES_PSQL_COMMAND/PRISM_DATABASE_URL", + ) + parser.add_argument( + "--writer-id", + default=os.environ.get( + "PRISM_LEDGER_WRITER_ID", + "prism-audit-canonical-backfill", + ), + ) + parser.add_argument( + "--writer-epoch", + type=int, + default=int(os.environ.get("PRISM_LEDGER_WRITER_EPOCH", "1")), + ) + parser.add_argument( + "--writer-session-token", + default=os.environ.get("PRISM_LEDGER_WRITER_SESSION_TOKEN"), + ) + parser.add_argument( + "--lease-ttl-seconds", + type=float, + default=float(os.environ.get("PRISM_LEDGER_LEASE_TTL_SECONDS", "60")), + ) + parser.add_argument( + "--lease-acquire-lock-timeout-seconds", + type=float, + default=env_positive_float( + "PRISM_LEDGER_LEASE_ACQUIRE_LOCK_TIMEOUT_SECONDS", + DEFAULT_LEASE_ACQUIRE_LOCK_TIMEOUT_SECONDS, + ), + ) + parser.add_argument( + "--lease-acquire-attempts", + type=int, + default=env_positive_int( + "PRISM_LEDGER_LEASE_ACQUIRE_ATTEMPTS", + DEFAULT_LEASE_ACQUIRE_ATTEMPTS, + ), + ) + parser.add_argument( + "--postgres-idle-in-transaction-timeout-seconds", + type=float, + default=env_positive_float( + "PRISM_POSTGRES_IDLE_IN_TRANSACTION_TIMEOUT_SECONDS", + DEFAULT_POSTGRES_IDLE_IN_TRANSACTION_TIMEOUT_SECONDS, + ), + ) + parser.add_argument( + "--postgres-tcp-keepalives-idle-seconds", + type=int, + default=env_positive_int( + "PRISM_POSTGRES_TCP_KEEPALIVES_IDLE_SECONDS", + DEFAULT_POSTGRES_TCP_KEEPALIVES_IDLE_SECONDS, + ), + ) + parser.add_argument( + "--postgres-tcp-keepalives-interval-seconds", + type=int, + default=env_positive_int( + "PRISM_POSTGRES_TCP_KEEPALIVES_INTERVAL_SECONDS", + DEFAULT_POSTGRES_TCP_KEEPALIVES_INTERVAL_SECONDS, + ), + ) + parser.add_argument( + "--postgres-tcp-keepalives-count", + type=int, + default=env_positive_int( + "PRISM_POSTGRES_TCP_KEEPALIVES_COUNT", + DEFAULT_POSTGRES_TCP_KEEPALIVES_COUNT, + ), + ) + parser.add_argument( + "--no-init-schema", + action="store_true", + help="do not run the idempotent PRISM schema repair before backfilling", + ) + return parser + + +def ledger_from_args( + args: argparse.Namespace, + store: AuditArtifactStore, +) -> PsqlShareLedger: + return PsqlShareLedger( + psql_command=psql_command_from_env(args.psql_command), + writer_id=args.writer_id, + writer_epoch=args.writer_epoch, + writer_session_token=args.writer_session_token, + lease_retry_sleep=( + None if args.dry_run else _refuse_publish_writer_lease_wait + ), + initialize_schema=not args.no_init_schema and not args.dry_run, + read_only=args.dry_run, + lease_ttl_seconds=args.lease_ttl_seconds, + lease_acquire_lock_timeout_seconds=args.lease_acquire_lock_timeout_seconds, + lease_acquire_attempts=args.lease_acquire_attempts, + postgres_idle_in_transaction_timeout_seconds=( + args.postgres_idle_in_transaction_timeout_seconds + ), + postgres_tcp_keepalives_idle_seconds=args.postgres_tcp_keepalives_idle_seconds, + postgres_tcp_keepalives_interval_seconds=( + args.postgres_tcp_keepalives_interval_seconds + ), + postgres_tcp_keepalives_count=args.postgres_tcp_keepalives_count, + audit_artifact_store=store, + audit_bundle_canonicalizer=canonical_bundle_bytes, + ) + + +def _refuse_publish_writer_lease_wait(wait_seconds: float) -> None: + """Refuse to wait behind a live same-identity coordinator lease.""" + + raise BackfillWriterLeaseUnavailable( + "publish-mode canonical audit-bundle backfill requires exclusive " + "writer-lease ownership; stop the coordinator before publishing " + f"(lease acquisition requested a {float(wait_seconds):g}s wait)" + ) + + +def _cleanup_backfill_resources( + *, + ledger: PsqlShareLedger | None, + store: AuditArtifactStore | None, + publish_mode: bool, +) -> list[str]: + """Release the lease, then close every constructed resource.""" + + failures: list[str] = [] + if ledger is not None: + if publish_mode: + try: + released = ledger.release_writer_lease() + if released is False: + raise RuntimeError( + "the exact backfill writer lease was no longer held" + ) + except Exception as exc: # noqa: BLE001 - close must still run + failures.append( + "writer lease release failed: " + f"{type(exc).__name__}: {exc}" + ) + try: + ledger.close() + except Exception as exc: # noqa: BLE001 - store close must still run + failures.append(f"ledger close failed: {type(exc).__name__}: {exc}") + if store is not None: + try: + store.close() + except Exception as exc: # noqa: BLE001 - report after all cleanup + failures.append( + f"audit artifact store close failed: {type(exc).__name__}: {exc}" + ) + return failures + + +def main(argv: list[str] | None = None) -> int: + args = build_parser().parse_args(argv) + if args.limit is not None and args.limit <= 0: + raise SystemExit("--limit must be positive") + if args.batch_size <= 0: + raise SystemExit("--batch-size must be positive") + try: + start_after = ( + None + if args.start_after is None + else canonical_hex(args.start_after, name="start_after", expected_bytes=32) + ) + except ValueError as exc: + raise SystemExit(f"--start-after is invalid: {exc}") from exc + + store: AuditArtifactStore | None = None + ledger: PsqlShareLedger | None = None + summary: dict[str, Any] | None = None + primary_error: BaseException | None = None + cleanup_failures: list[str] = [] + try: + store = build_audit_store_from_env(read_only=args.dry_run) + # Bind the canonical artifact API before opening the database: a + # missing phase-2 capability fails without first taking a writer lease. + artifacts = CanonicalArtifactAdapter(store) + try: + ledger = ledger_from_args(args, store) + except Exception as exc: + if args.dry_run: + raise + raise SystemExit( + "publish-mode canonical audit-bundle backfill could not acquire " + "exclusive writer-lease ownership; stop the coordinator before " + f"publishing and retry: {exc}" + ) from exc + backfill = CanonicalBundleBackfill( + artifacts=artifacts, + ledger=AuditBundleLedgerAdapter(ledger), + dry_run=args.dry_run, + batch_size=args.batch_size, + limit=args.limit, + start_after=start_after, + ) + summary = backfill.run() + except BaseException as exc: + primary_error = exc + finally: + cleanup_failures = _cleanup_backfill_resources( + ledger=ledger, + store=store, + publish_mode=not args.dry_run, + ) + if primary_error is not None: + for failure in cleanup_failures: + primary_error.add_note(f"backfill cleanup: {failure}") + raise primary_error + if cleanup_failures: + raise RuntimeError("; ".join(cleanup_failures)) + assert summary is not None + print(json.dumps(summary, indent=2, sort_keys=True)) + return int(summary["exit_code"]) + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/lab/prism/backfill_ctv_fanouts.py b/lab/prism/backfill_ctv_fanouts.py index db7c79cbc..31433acfe 100644 --- a/lab/prism/backfill_ctv_fanouts.py +++ b/lab/prism/backfill_ctv_fanouts.py @@ -16,6 +16,16 @@ if not __package__: sys.path.insert(0, str(Path(__file__).resolve().parents[2])) +from lab.prism.coordinator_config import ( + DEFAULT_LEASE_ACQUIRE_ATTEMPTS, + DEFAULT_LEASE_ACQUIRE_LOCK_TIMEOUT_SECONDS, + DEFAULT_POSTGRES_IDLE_IN_TRANSACTION_TIMEOUT_SECONDS, + DEFAULT_POSTGRES_TCP_KEEPALIVES_COUNT, + DEFAULT_POSTGRES_TCP_KEEPALIVES_IDLE_SECONDS, + DEFAULT_POSTGRES_TCP_KEEPALIVES_INTERVAL_SECONDS, + env_positive_float, + env_positive_int, +) from lab.prism.share_ledger import ( AUDIT_BODY_REF_SCHEMA, AUDIT_BUNDLE_V2_SCHEMA, @@ -238,6 +248,54 @@ def build_parser() -> argparse.ArgumentParser: type=float, default=float(os.environ.get("PRISM_LEDGER_LEASE_TTL_SECONDS", "60")), ) + parser.add_argument( + "--lease-acquire-lock-timeout-seconds", + type=float, + default=env_positive_float( + "PRISM_LEDGER_LEASE_ACQUIRE_LOCK_TIMEOUT_SECONDS", + DEFAULT_LEASE_ACQUIRE_LOCK_TIMEOUT_SECONDS, + ), + ) + parser.add_argument( + "--lease-acquire-attempts", + type=int, + default=env_positive_int( + "PRISM_LEDGER_LEASE_ACQUIRE_ATTEMPTS", + DEFAULT_LEASE_ACQUIRE_ATTEMPTS, + ), + ) + parser.add_argument( + "--postgres-idle-in-transaction-timeout-seconds", + type=float, + default=env_positive_float( + "PRISM_POSTGRES_IDLE_IN_TRANSACTION_TIMEOUT_SECONDS", + DEFAULT_POSTGRES_IDLE_IN_TRANSACTION_TIMEOUT_SECONDS, + ), + ) + parser.add_argument( + "--postgres-tcp-keepalives-idle-seconds", + type=int, + default=env_positive_int( + "PRISM_POSTGRES_TCP_KEEPALIVES_IDLE_SECONDS", + DEFAULT_POSTGRES_TCP_KEEPALIVES_IDLE_SECONDS, + ), + ) + parser.add_argument( + "--postgres-tcp-keepalives-interval-seconds", + type=int, + default=env_positive_int( + "PRISM_POSTGRES_TCP_KEEPALIVES_INTERVAL_SECONDS", + DEFAULT_POSTGRES_TCP_KEEPALIVES_INTERVAL_SECONDS, + ), + ) + parser.add_argument( + "--postgres-tcp-keepalives-count", + type=int, + default=env_positive_int( + "PRISM_POSTGRES_TCP_KEEPALIVES_COUNT", + DEFAULT_POSTGRES_TCP_KEEPALIVES_COUNT, + ), + ) parser.add_argument( "--no-init-schema", action="store_true", @@ -246,6 +304,23 @@ def build_parser() -> argparse.ArgumentParser: return parser +def ledger_from_args(args: argparse.Namespace) -> PsqlShareLedger: + return PsqlShareLedger( + psql_command=psql_command_from_env(args.psql_command), + writer_id=args.writer_id, + writer_epoch=args.writer_epoch, + writer_session_token=args.writer_session_token, + initialize_schema=not args.no_init_schema, + lease_ttl_seconds=args.lease_ttl_seconds, + lease_acquire_lock_timeout_seconds=args.lease_acquire_lock_timeout_seconds, + lease_acquire_attempts=args.lease_acquire_attempts, + postgres_idle_in_transaction_timeout_seconds=args.postgres_idle_in_transaction_timeout_seconds, + postgres_tcp_keepalives_idle_seconds=args.postgres_tcp_keepalives_idle_seconds, + postgres_tcp_keepalives_interval_seconds=args.postgres_tcp_keepalives_interval_seconds, + postgres_tcp_keepalives_count=args.postgres_tcp_keepalives_count, + ) + + def main(argv: list[str] | None = None) -> int: args = build_parser().parse_args(argv) if len(args.paths) != 1 and (args.path_block_hash or args.path_manifest_set_sha256): @@ -266,14 +341,7 @@ def main(argv: list[str] | None = None) -> int: if not path_inputs and not db_hashes and not args.db_block_height: raise SystemExit("provide at least one path, --db-block-hash, or --db-block-height") - ledger = PsqlShareLedger( - psql_command=psql_command_from_env(args.psql_command), - writer_id=args.writer_id, - writer_epoch=args.writer_epoch, - writer_session_token=args.writer_session_token, - initialize_schema=not args.no_init_schema, - lease_ttl_seconds=args.lease_ttl_seconds, - ) + ledger = ledger_from_args(args) for height in args.db_block_height: height_hashes = db_block_hashes_for_height(ledger, height) diff --git a/lab/prism/background_services.py b/lab/prism/background_services.py new file mode 100644 index 000000000..e0bb9b621 --- /dev/null +++ b/lab/prism/background_services.py @@ -0,0 +1,2285 @@ +"""Named lifecycle registry for PRISM process-level background loops. + +The registry deliberately has no ``start_all`` operation. The coordinator +starts named services at the existing recovery boundaries, while this module +owns the start-once state and the exact thread handles used during shutdown. + +This module also owns :class:`LedgerLeaseHeartbeatService`, the sole owner of +the ledger writer-lease heartbeat/monitor threads, their freshness state, and +the synchronous exact-session external-effect fence. Because its startup is +conditional, creates two coupled threads, and waits for the first proof, it is +started directly at the serve boundary rather than modeled as two independent +registry specifications. + +:class:`WatchdogService` owns watchdog policy: the coordination-blocked +streak, coordination-versus-publication classification, the liveness and +fatal-stop decisions, failure detail/diagnostics, and the bounded fresh-thread +shutdown/lease-release attempt followed by an unconditional process exit. +Generic liveness heartbeat registration/record/pause maps stay coordinator +runtime state and reach the watchdog through ports. Both WatchdogService.run +and the lease service trigger the dynamic coordinator ``_watchdog_hard_exit`` +seam so per-instance monkeypatches intercept every exit path; the coordinator +delegate routes back into :meth:`WatchdogService.hard_exit`. +""" + +from __future__ import annotations + +from dataclasses import dataclass +import gc +import inspect +import os +import queue +import sys +import threading +import time +from typing import Any, Callable, Iterable + +from lab.prism.coordinator_shutdown import ShutdownInProgress +from lab.prism.process_telemetry import PRISM_GC_GENERATIONS +from lab.prism.share_ledger import ( + DEFAULT_WRITER_LEASE_ADOPTION_SILENCE_SECONDS, + WRITER_LEASE_VERIFICATION_MAX_STATEMENTS, + WriterLeaseRenewalDeferred, +) +from lab.prism.writer_lease_timing import ( + WRITER_LEASE_HEARTBEAT_SCHEDULER_SLACK_SECONDS, + LEASE_HEARTBEAT_MODE_FENCE, + LEASE_HEARTBEAT_MODE_PROOF, + LEASE_HEARTBEAT_MODE_RENEW, + LEASE_HEARTBEAT_MODES, + LEASE_HEARTBEAT_OUTCOME_DEFERRED, + LEASE_HEARTBEAT_OUTCOME_FAILED, + LEASE_HEARTBEAT_OUTCOME_PROVEN, + LEASE_HEARTBEAT_OUTCOME_RENEWAL_DUE, + LEASE_HEARTBEAT_OUTCOME_RENEWED, + LEASE_HEARTBEAT_OUTCOMES, + LEASE_MONITOR_LATE_WAKE_SLACK_FRACTIONS, + LEASE_MONITOR_STALL_PROBE_MAX_SAMPLES_PER_WINDOW, + LEASE_MONITOR_STALL_PROBE_TRIGGER_SLACK_FRACTION, + LEASE_MONITOR_STALL_PROBE_WINDOW_SECONDS, + LEASE_MONITOR_WAKE_DELAY_BUCKETS, + LEASE_MONITOR_WAKE_DELAY_WINDOW_SECONDS, + LEASE_MONITOR_WAKE_DELAY_WINDOW_SLICES, + UNATTRIBUTED_PHASES, + WriterLeaseHeartbeatPolicy, + WriterLeaseVerificationAttempt, + WriterLeaseVerificationPhases, +) + +# GC pause histogram buckets (issue #227). A closed set, like every other +# bucket vocabulary on /metrics. Distinct from the WP1a gc families in +# lab.prism.process_telemetry, which export gc.get_stats() counts (passes, +# objects collected, uncollectable) and say nothing about how long a pass +# held the interpreter; these are the pause durations, measured from the +# collector's own start/stop callbacks. +PRISM_GC_PAUSE_SECONDS_BUCKETS = ( + 0.001, + 0.0025, + 0.005, + 0.01, + 0.025, + 0.05, + 0.1, + 0.25, + 0.5, + 1.0, + 2.5, +) + +# Bounds on one stall-probe stack sample, so a sample of a 64-thread +# process stays a few kilobytes: the innermost frames of each thread, and +# at most this many threads. +LEASE_MONITOR_STALL_PROBE_MAX_FRAMES_PER_THREAD = 12 +LEASE_MONITOR_STALL_PROBE_MAX_THREADS = 64 + + +@dataclass(frozen=True, slots=True) +class BackgroundServiceSpec: + """Immutable construction and shutdown policy for one background loop.""" + + name: str + thread_name: str + target: Callable[[], None] + daemon: bool + join_timeout: float + watchdog_monitored: bool + registration_identity: object | None = None + + def __post_init__(self) -> None: + if not self.name: + raise ValueError("background service name must not be empty") + if not self.thread_name: + raise ValueError("background service thread name must not be empty") + if self.join_timeout < 0: + raise ValueError("background service join timeout must be nonnegative") + + +@dataclass(frozen=True, slots=True) +class BackgroundServiceSnapshot: + """Read-only lifecycle state returned without exposing registry records.""" + + specification: BackgroundServiceSpec + started: bool + thread: threading.Thread | None + + +@dataclass(slots=True) +class _BackgroundServiceRecord: + specification: BackgroundServiceSpec + started: bool = False + thread: threading.Thread | None = None + start_hook_completed: bool = False + + +ThreadFactory = Callable[..., threading.Thread] + + +class BackgroundServiceRegistry: + """Start named process services once and retain their drain handles.""" + + def __init__( + self, + specifications: Iterable[BackgroundServiceSpec] = (), + *, + thread_factory: ThreadFactory = threading.Thread, + ) -> None: + self._lock = threading.Lock() + self._thread_factory = thread_factory + self._records: dict[str, _BackgroundServiceRecord] = {} + self._thread_names: set[str] = set() + for specification in specifications: + self.register(specification) + + def register(self, specification: BackgroundServiceSpec) -> None: + """Register one service without starting it.""" + with self._lock: + self._register_locked(specification) + + def register_if_absent(self, specification: BackgroundServiceSpec) -> bool: + """Atomically install an equivalent dynamic service at most once. + + Returns true when this call registered the service. Concurrent callers + describing the same lifecycle and registration identity receive false; + a conflicting reuse of either name still fails explicitly. + """ + with self._lock: + existing = self._records.get(specification.name) + if existing is not None: + if self._equivalent(existing.specification, specification): + return False + raise ValueError( + "incompatible background service registration for name: " + f"{specification.name}" + ) + self._register_locked(specification) + return True + + def _register_locked(self, specification: BackgroundServiceSpec) -> None: + if specification.name in self._records: + raise ValueError( + f"background service is already registered: {specification.name}" + ) + if specification.thread_name in self._thread_names: + raise ValueError( + "background service thread name is already registered: " + f"{specification.thread_name}" + ) + self._records[specification.name] = _BackgroundServiceRecord( + specification=specification + ) + self._thread_names.add(specification.thread_name) + + @staticmethod + def _equivalent( + existing: BackgroundServiceSpec, + candidate: BackgroundServiceSpec, + ) -> bool: + target_equivalent = ( + existing.target is candidate.target + if existing.registration_identity is None + and candidate.registration_identity is None + else existing.registration_identity == candidate.registration_identity + and existing.registration_identity is not None + and candidate.registration_identity is not None + ) + return bool( + target_equivalent + and existing.name == candidate.name + and existing.thread_name == candidate.thread_name + and existing.daemon == candidate.daemon + and existing.join_timeout == candidate.join_timeout + and existing.watchdog_monitored == candidate.watchdog_monitored + ) + + def contains(self, name: str) -> bool: + with self._lock: + return name in self._records + + def service_names(self) -> tuple[str, ...]: + with self._lock: + return tuple(self._records) + + def snapshot(self, name: str) -> BackgroundServiceSnapshot: + with self._lock: + record = self._records[name] + return BackgroundServiceSnapshot( + specification=record.specification, + started=record.started, + thread=record.thread, + ) + + def start( + self, + name: str, + *, + on_started: Callable[[BackgroundServiceSpec], None] | None = None, + ) -> threading.Thread: + """Start a named service once, returning the same thread thereafter.""" + with self._lock: + record = self._records[name] + if record.thread is not None: + if not record.start_hook_completed and on_started is not None: + on_started(record.specification) + record.start_hook_completed = True + return record.thread + specification = record.specification + thread = self._thread_factory( + target=specification.target, + name=specification.thread_name, + daemon=specification.daemon, + ) + thread.start() + record.thread = thread + record.started = True + if on_started is None: + record.start_hook_completed = True + else: + # The live thread remains registered for shutdown if this + # nonblocking side-effect fails. A later start call retries + # only the hook and never creates a second worker. + on_started(specification) + record.start_hook_completed = True + return thread + + def threads_to_drain(self) -> tuple[tuple[threading.Thread, float], ...]: + """Return started threads in stable registration/shutdown order.""" + with self._lock: + return tuple( + (record.thread, record.specification.join_timeout) + for record in self._records.values() + if record.started and record.thread is not None + ) + + def watchdog_service_names(self, *, started_only: bool = False) -> tuple[str, ...]: + """Derive watchdog keys from the same records used to start loops.""" + with self._lock: + return tuple( + record.specification.name + for record in self._records.values() + if record.specification.watchdog_monitored + and (record.started or not started_only) + ) + + +@dataclass(frozen=True, slots=True) +class WatchdogPorts: + """Dynamic process-supervision capabilities used by the watchdog owner. + + ``publication_state``, ``hard_exit``, ``overdue_heartbeats``, and + ``publication_failure_expired`` resolve coordinator methods at call time + so per-instance monkeypatches intercept them; the interval/budget/enabled + reads stay live per iteration for focused timing overrides. + """ + + wait_for_stop: Callable[[float], bool] + interval_seconds: Callable[[], float] + fatal_exit_requested: Callable[[], bool] + publication_state: Callable[[float], tuple[str | None, float, float, float]] + hard_exit: Callable[[str], None] + liveness_enabled: Callable[[], bool] + overdue_heartbeats: Callable[[float], list[str]] + liveness_timeout_seconds: Callable[[], float] + coordination_budget_seconds: Callable[[], float] + publication_budget_seconds: Callable[[], float] + ensure_job_cache_state: Callable[[], None] + publication_failure_expired: Callable[[float], bool] + publication_divergence_since: Callable[[], float | None] + lease_release_timeout_seconds: Callable[[], float] + shutdown_controller: Callable[[], Any] + request_shutdown: Callable[[], None] + release_ledger_lease: Callable[[float], bool] + lease_failure_reason: Callable[[], str | None] + exit_process: Callable[[int], None] + log: Callable[[str], None] + + +class WatchdogService: + """Own watchdog classification, streak state, and bounded emergency exit. + + Progress-health continues to own the publication-divergence timestamps; + the classification reads a locked snapshot through a port and rechecks it + while claiming an exit. Lock order is the service's coordination lock, + then the progress-health lock — progress-health must never call the + watchdog while holding its own lock. + """ + + def __init__(self, ports: WatchdogPorts) -> None: + self._ports = ports + self._coordination_blocked_lock = threading.Lock() + self._coordination_blocked_since_monotonic: float | None = None + self._publication_watchdog_exit_claimed = False + self.failure_detail: str | None = None + + def record_coordination_blocked_refresh(self, now: float) -> None: + with self._coordination_blocked_lock: + if self._publication_watchdog_exit_claimed: + return + if self._coordination_blocked_since_monotonic is None: + self._coordination_blocked_since_monotonic = now + + def clear_coordination_blocked_streak(self) -> None: + with self._coordination_blocked_lock: + # Once the watchdog claims an expired generation, its hard exit is + # the terminal action. Otherwise a refresh completing between the + # deadline check and os._exit could appear to cancel an exit that + # already won the streak lock. + if not self._publication_watchdog_exit_claimed: + self._coordination_blocked_since_monotonic = None + + def coordination_blocked_streak_age_seconds( + self, + now: float | None = None, + ) -> float: + now = time.monotonic() if now is None else now + with self._coordination_blocked_lock: + started = self._coordination_blocked_since_monotonic + return 0.0 if started is None else max(0.0, now - started) + + def coordination_blocked_streak_expired(self, now: float) -> bool: + budget = float(self._ports.coordination_budget_seconds()) + with self._coordination_blocked_lock: + started = self._coordination_blocked_since_monotonic + return bool( + started is not None + and (budget <= 0 or now - started >= budget) + ) + + def publication_watchdog_state( + self, + now: float, + ) -> tuple[str | None, float, float, float]: + """Arbitrate coordination and ordinary publication deadlines. + + The final decision is serialized with coordination streak changes. + A streak recorded while the ordinary publication check is in flight + therefore owns the longer coordination deadline; once either deadline + is claimed, later refresh results cannot cancel the terminal exit. + """ + coordination_budget = float(self._ports.coordination_budget_seconds()) + publication_budget = float(self._ports.publication_budget_seconds()) + self._ports.ensure_job_cache_state() + + # This preflight keeps the common healthy case cheap. Its result is + # revalidated under both state locks before an ordinary exit is + # claimed, so a concurrent publication cannot leave a stale verdict. + publication_expired = self._ports.publication_failure_expired(now) + with self._coordination_blocked_lock: + started = self._coordination_blocked_since_monotonic + if started is not None: + age = max(0.0, now - started) + if coordination_budget <= 0 or age >= coordination_budget: + self._publication_watchdog_exit_claimed = True + return ( + "coordination", + age, + coordination_budget, + publication_budget, + ) + return None, age, coordination_budget, publication_budget + + if publication_expired: + # Recheck the divergence timestamp under the progress-health + # lock (inside the port) while the coordination lock is held. + divergence_since = self._ports.publication_divergence_since() + publication_expired = bool( + publication_budget > 0 + and divergence_since is not None + and now - divergence_since >= publication_budget + ) + if publication_expired: + self._publication_watchdog_exit_claimed = True + return ( + "publication", + 0.0, + coordination_budget, + publication_budget, + ) + return None, 0.0, coordination_budget, publication_budget + + def run(self) -> None: + while True: + if self._ports.wait_for_stop(self._ports.interval_seconds()): + if self._ports.fatal_exit_requested(): + self._ports.log( + "prism coordinator: fatal block-work restart requested; " + "exiting non-zero even if the main thread is blocked" + ) + self._ports.exit_process(1) + return + now = time.monotonic() + ( + publication_failure, + coordination_age, + coordination_budget, + publication_budget, + ) = self._ports.publication_state(now) + if publication_failure == "coordination": + self.failure_detail = ( + "prism coordinator: publication-progress watchdog firing; " + "template refresh remained coordination-blocked past the " + f"coordination budget={coordination_budget:g}s " + f"streak_age={coordination_age:.3f}s" + ) + self._ports.hard_exit("coordination") + if publication_failure == "publication": + self.failure_detail = ( + "prism coordinator: publication-progress watchdog firing; " + "current tip/generation remained unpublished past the " + f"template refresh failure budget={publication_budget:g}s" + ) + self._ports.hard_exit("publication") + overdue = ( + self._ports.overdue_heartbeats(now) + if self._ports.liveness_enabled() + else [] + ) + if overdue: + self.failure_detail = ( + "prism coordinator: liveness watchdog firing; unresponsive " + f"subsystems={overdue} " + f"timeout={self._ports.liveness_timeout_seconds():g}s" + ) + # Queued shares have not been acknowledged. Miners reconnect + # and retry them after restart; exact-payload replay is + # idempotent if Postgres committed just before this exit. + self._ports.hard_exit("liveness") + + def hard_exit( + self, + reason: str, + *, + timeout_seconds: float | None = None, + ) -> None: + """Bound a fresh-thread lease release, then terminate unconditionally.""" + try: + if timeout_seconds is None: + timeout_seconds = float( + self._ports.lease_release_timeout_seconds() + ) + deadline = time.monotonic() + max(0.0, timeout_seconds) + release_thread = threading.Thread( + target=self._release_ledger_lease, + args=(reason, deadline), + name="prism-watchdog-lease-release", + daemon=True, + ) + release_thread.start() + release_thread.join(max(0.0, deadline - time.monotonic())) + finally: + # Nothing, including timeout logging or thread-start failure, may + # extend or suppress the watchdog's terminal action. Exiting also + # discards the in-memory vardiff retention store (see + # vardiff_service.SessionDifficultyStore), so the post-restart + # reconnect wave is not smoothed by difficulty resume. + self._ports.exit_process(1) + + def _release_ledger_lease(self, reason: str, deadline: float) -> bool: + """Use only the shutdown controller and a fresh DB connection. + + Do not call ``shutdown`` here: cancellation takes the coordinator + control-plane lock, which may belong to the subsystem that triggered + the watchdog. Closing writer admission plus the controller's + tracked-writer barrier retains the graceful path's release-withheld + invariant without that lock. + + Any best-effort diagnostic is emitted only after lease handling on this + daemon worker. A blocked container log pipe may park the worker, but it + cannot precede the release attempt or extend the caller's hard deadline. + """ + controller = self._ports.shutdown_controller() + self._ports.request_shutdown() + if not controller.begin_shutdown(f"watchdog_{reason}"): + handled = controller.wait_for_lease_handling() + self._exit_diagnostic(reason, lease_handled=handled) + return handled + + quiesced, _elapsed, _blockers = controller.wait_for_writer_quiescence( + max(0.0, deadline - time.monotonic()) + ) + if not quiesced: + self._exit_diagnostic(reason, lease_handled=False) + return False + if time.monotonic() >= deadline: + self._exit_diagnostic(reason, lease_handled=False) + return False + handled = self._ports.release_ledger_lease(deadline) + self._exit_diagnostic(reason, lease_handled=handled) + return handled + + def _exit_diagnostic(self, reason: str, *, lease_handled: bool) -> None: + """Best-effort logging after the watchdog's safety-critical work.""" + detail = ( + self._ports.lease_failure_reason() + if reason == "lease_heartbeat" + else self.failure_detail + ) + try: + self._ports.log( + (detail or f"prism coordinator: {reason} watchdog firing") + + ". Exiting non-zero so the restart policy recovers the process. " + + f"lease_handled={lease_handled}" + ) + except Exception: + pass + + +def guard_session_verifier(ledger: object) -> Callable[..., object] | None: + """Return the guarded-session liveness check for ``ledger``. + + Prefer ``verify_writer_lease_guard_session``: it proves the dedicated + guard session is live and renews the lease TTL only via SKIP LOCKED, + so it can never queue behind the row lock a long fenced transaction + (``persist_accepted_block``) holds until commit and then die on the + guard's statement timeout, yet an idle coordinator still keeps its + lease from expiring under different-identity claimants. + ``renew_writer_lease_heartbeat`` is the legacy embedder spelling and + does wait on the lease tuple; it is only a fallback for ledgers that + predate the non-blocking verification. + """ + verify = getattr(ledger, "verify_writer_lease_guard_session", None) + if verify is not None: + return verify + return getattr(ledger, "renew_writer_lease_heartbeat", None) + + +def guard_session_prover(ledger: object) -> Callable[..., object] | None: + """Return the cheap read-only ownership proof for ``ledger``, if any. + + ``prove_writer_lease_guard_session`` answers only "does this backend + still hold the advisory guard, and does the committed lease row still + name this exact session?" on one non-blocking statement. It is the + frequent half of the issue #212 split: the heartbeat needs ownership + several times per adoption-silence window, but the lease TTL only needs + a writer-side refresh well before it lapses, and paying for renewal + (a ``SKIP LOCKED`` row lock, a ``pg_stat_activity`` locker attribution, + and sometimes a second statement) on every beat is what pushed the + frequent statement toward the guard's 500ms statement timeout during a + rapid-block burst. + + A ledger that does not offer it — an embedder's, or a test fake — + simply keeps running the full verification on every beat, which is the + behavior that predates the split. + """ + return getattr(ledger, "prove_writer_lease_guard_session", None) + + +def verifier_callbacks(verify: Callable[..., object]) -> frozenset[str]: + """Which progress callbacks ``verify`` accepts. + + Resolved once per loop rather than per beat: a ledger's verification + signature cannot change under a running heartbeat, and + ``inspect.signature`` is far too expensive for a 0.25s cadence. + """ + try: + parameters = inspect.signature(verify).parameters + except (TypeError, ValueError): + return frozenset() + return frozenset( + name + for name in ( + "on_query_start", + "on_statement_progress", + "on_statement_end", + "on_statement_server_seconds", + ) + if name in parameters + ) + + +def _print_line(line: str) -> None: + """Default log port for the monitor's diagnostics: a flushed print. + + Only ever invoked from the monitor's diagnostics thread + (:meth:`LedgerLeaseHeartbeatService._run_diagnostics`), never from the + monitor thread itself. The monitor is the thread that must notice a + dead guard session and hard-exit before a successor adopts, so a full + container log pipe stalling this flush must stall a throwaway thread, + not the thread that decides the exit. + """ + try: + print(line, flush=True) + except Exception: + pass + + +class _WindowRateLimiter: + """At most ``max_events`` admissions per ``window_seconds``, then refuse. + + A fixed window keyed on the first admission, not a token bucket: the + property the stall probe needs is a hard ceiling on the number of + samples any interval can produce, so that a stall can never be turned + into a stall storm by the instrument that reports it. + """ + + __slots__ = ("max_events", "window_seconds", "_window_started", "_admitted") + + def __init__(self, *, max_events: int, window_seconds: float) -> None: + self.max_events = int(max_events) + self.window_seconds = float(window_seconds) + self._window_started: float | None = None + self._admitted = 0 + + def admit(self, now: float) -> bool: + if ( + self._window_started is None + or now - self._window_started >= self.window_seconds + ): + self._window_started = now + self._admitted = 0 + if self._admitted >= self.max_events: + return False + self._admitted += 1 + return True + + +class MonitorWakeTelemetry: + """The heartbeat monitor's own lateness, as signals that can re-arm. + + Issue #227: the only export of the monitor's lateness was a lifetime + high-water mark. Every stall smaller than the record was invisible, the + alert built on it could fire once per process lifetime, and a hard-exit + message quoted a possibly hours-old worst case. This keeps the lifetime + maximum for dashboard continuity and adds the signals an alert can + actually be expressed on: + + * a fixed-bucket histogram of every wake's lateness; + * counters of wakes at or above 0.5x, 0.8x and 1.0x the policy's + scheduler slack (closed label set); + * a rolling-window maximum that falls back once a stall ages out; + * the age of the current lifetime record. + + Written only by the monitor thread; read by the metrics thread. No + lock: every field is a Python int or float, so a scrape sees either the + value before or after one ``observe``, never a torn one. + """ + + __slots__ = ( + "_buckets", + "_bucket_counts", + "_count", + "_sum", + "lifetime_max_seconds", + "_record_monotonic", + "_late", + "_slice_seconds", + "_slice_index", + "_slice_max", + ) + + def __init__( + self, + *, + buckets: tuple[float, ...] = LEASE_MONITOR_WAKE_DELAY_BUCKETS, + window_seconds: float = LEASE_MONITOR_WAKE_DELAY_WINDOW_SECONDS, + window_slices: int = LEASE_MONITOR_WAKE_DELAY_WINDOW_SLICES, + ) -> None: + self._buckets = tuple(float(bucket) for bucket in buckets) + # Cumulative, in Prometheus's own convention: bucket i counts every + # observation <= its upper bound. + self._bucket_counts = [0 for _ in self._buckets] + self._count = 0 + self._sum = 0.0 + self.lifetime_max_seconds = 0.0 + self._record_monotonic: float | None = None + self._late = { + fraction: 0 for fraction in LEASE_MONITOR_LATE_WAKE_SLACK_FRACTIONS + } + slices = max(1, int(window_slices)) + self._slice_seconds = float(window_seconds) / slices + # A ring of per-slice maxima keyed by absolute slice number, so an + # old slice is recognised as stale rather than read as current. + self._slice_index = [-1 for _ in range(slices)] + self._slice_max = [0.0 for _ in range(slices)] + + @property + def buckets(self) -> tuple[float, ...]: + return self._buckets + + def observe( + self, + wake_delay_seconds: float, + *, + slack_seconds: float, + now: float, + ) -> None: + delay = max(0.0, float(wake_delay_seconds)) + self._count += 1 + self._sum += delay + for index, bound in enumerate(self._buckets): + if delay <= bound: + self._bucket_counts[index] += 1 + for fraction in self._late: + if delay >= float(fraction) * float(slack_seconds): + self._late[fraction] += 1 + if delay > self.lifetime_max_seconds: + self.lifetime_max_seconds = delay + self._record_monotonic = now + absolute = int(now // self._slice_seconds) + slot = absolute % len(self._slice_max) + if self._slice_index[slot] != absolute: + self._slice_index[slot] = absolute + self._slice_max[slot] = delay + elif delay > self._slice_max[slot]: + self._slice_max[slot] = delay + + def window_max_seconds(self, now: float) -> float: + """Largest lateness observed inside the trailing window.""" + oldest = int(now // self._slice_seconds) - len(self._slice_max) + 1 + worst = 0.0 + for index, value in zip(self._slice_index, self._slice_max): + if index >= oldest and value > worst: + worst = value + return worst + + def record_age_seconds(self, now: float) -> float: + """Seconds since the lifetime maximum was last raised (0 before any).""" + if self._record_monotonic is None: + return 0.0 + return max(0.0, now - self._record_monotonic) + + def snapshot(self, now: float) -> dict[str, object]: + """Fixed-key view for the metrics surface.""" + return { + "buckets": { + bound: int(count) + for bound, count in zip(self._buckets, self._bucket_counts) + }, + "count": int(self._count), + "sum": float(self._sum), + "late_wakes": dict(self._late), + "lifetime_max_seconds": float(self.lifetime_max_seconds), + "window_max_seconds": self.window_max_seconds(now), + "record_age_seconds": self.record_age_seconds(now), + } + + +class GcPauseTelemetry: + """Cyclic-collector pause durations per generation, from ``gc.callbacks``. + + Issue #227's stall attribution needs to know whether the interpreter + was inside a collection when the monitor could not run. The WP1a + families (``qbit_prism_process_gc_*`` in ``process_telemetry``) export + ``gc.get_stats()`` — how many passes ran and what they freed — and + cannot say how long any pass held the interpreter. This is the missing + duration: the collector calls back at the start and stop of every + pass with the generation, and the difference is the pause. + + The callback must never raise (an exception in a GC callback is + reported as unraisable and can hide the pass) and must never take a + lock: a collection can start on any thread, including one that holds + a lock the callback would need. Every field is a plain int or float, + so the metrics thread reads them lock-free like the rest of the + heartbeat's counters. + + ``generation`` is CPython's closed three-value set. Anything else + (there is nothing else on CPython) is ignored rather than allowed to + grow the label set. + """ + + def __init__( + self, + *, + clock: Callable[[], float] = time.perf_counter, + buckets: tuple[float, ...] = PRISM_GC_PAUSE_SECONDS_BUCKETS, + ) -> None: + self._clock = clock + self._buckets = tuple(float(bucket) for bucket in buckets) + self._bucket_counts = { + generation: [0 for _ in self._buckets] + for generation in PRISM_GC_GENERATIONS + } + self._count = {generation: 0 for generation in PRISM_GC_GENERATIONS} + self._sum = {generation: 0.0 for generation in PRISM_GC_GENERATIONS} + self._last = {generation: 0.0 for generation in PRISM_GC_GENERATIONS} + self._max = {generation: 0.0 for generation in PRISM_GC_GENERATIONS} + self._started_at: float | None = None + self._started_generation: str | None = None + self._installed = False + + @property + def buckets(self) -> tuple[float, ...]: + return self._buckets + + @property + def installed(self) -> bool: + return self._installed + + def install(self) -> None: + """Register on ``gc.callbacks`` once; idempotent.""" + if self._installed: + return + gc.callbacks.append(self._callback) + self._installed = True + + def uninstall(self) -> None: + if not self._installed: + return + self._installed = False + try: + gc.callbacks.remove(self._callback) + except ValueError: + pass + + def _callback(self, phase: str, info: dict[str, Any]) -> None: + try: + if phase == "start": + self._started_at = self._clock() + self._started_generation = str(info.get("generation", "")) + elif phase == "stop": + started = self._started_at + if started is None: + return + self._started_at = None + generation = str( + info.get("generation", self._started_generation or "") + ) + self.record(generation, self._clock() - started) + except Exception: + # Never let telemetry disturb a collection. + self._started_at = None + + def record(self, generation: str, pause_seconds: float) -> None: + """Account one pause to ``generation``; unknown generations are dropped. + + Ordering matters because nothing locks against the scrape thread: + the count is bumped *before* the buckets here and read *after* + them in :meth:`snapshot`, so whichever way a scrape interleaves it + sees every ``le`` bucket at or below ``+Inf``. The other order + renders a bucket larger than the count, which Prometheus ingests + as a malformed histogram (the same discipline as + :meth:`MonitorWakeTelemetry.observe`). + """ + if generation not in self._count: + return + pause = max(0.0, float(pause_seconds)) + self._count[generation] += 1 + self._sum[generation] += pause + counts = self._bucket_counts[generation] + for index, bound in enumerate(self._buckets): + if pause <= bound: + counts[index] += 1 + self._last[generation] = pause + if pause > self._max[generation]: + self._max[generation] = pause + + def worst(self) -> tuple[str | None, float]: + """The longest pause seen and its generation (``None`` before any).""" + generation: str | None = None + worst = 0.0 + for candidate in PRISM_GC_GENERATIONS: + if self._count[candidate] and ( + generation is None or self._max[candidate] > worst + ): + generation = candidate + worst = self._max[candidate] + return generation, worst + + def snapshot(self) -> dict[str, dict[str, object]]: + """Fixed-key view: one entry per generation, each with fixed keys.""" + return { + generation: { + "buckets": { + bound: int(count) + for bound, count in zip( + self._buckets, self._bucket_counts[generation] + ) + }, + "count": int(self._count[generation]), + "sum": float(self._sum[generation]), + "last_seconds": float(self._last[generation]), + "max_seconds": float(self._max[generation]), + } + for generation in PRISM_GC_GENERATIONS + } + + +class StallProbe: + """Rate-limited stack sample of the running threads during a stall. + + When the monitor wakes at least half a scheduler slack late, the + question is *what held the interpreter*. ``sys._current_frames()`` + answers it at the only moment it can be asked — the instant the + monitor gets the GIL back — at a cost of one frame walk per thread. + + The rate limit is load-bearing, not decorative. A stall storm is by + definition many late wakes in a row, and an instrument that paid a + frame walk plus a log line for each of them would deepen the storm it + is reporting. ``LEASE_MONITOR_STALL_PROBE_MAX_SAMPLES_PER_WINDOW`` + samples per ``LEASE_MONITOR_STALL_PROBE_WINDOW_SECONDS`` is the ceiling; + every trigger beyond it is counted as suppressed and costs nothing. + """ + + def __init__( + self, + *, + max_samples_per_window: int = LEASE_MONITOR_STALL_PROBE_MAX_SAMPLES_PER_WINDOW, + window_seconds: float = LEASE_MONITOR_STALL_PROBE_WINDOW_SECONDS, + max_threads: int = LEASE_MONITOR_STALL_PROBE_MAX_THREADS, + max_frames_per_thread: int = LEASE_MONITOR_STALL_PROBE_MAX_FRAMES_PER_THREAD, + ) -> None: + self._limiter = _WindowRateLimiter( + max_events=max_samples_per_window, + window_seconds=window_seconds, + ) + self._max_threads = int(max_threads) + self._max_frames = int(max_frames_per_thread) + self.samples_total = 0 + self.suppressed_total = 0 + self.last_sample: dict[str, object] | None = None + + @property + def max_samples_per_window(self) -> int: + return self._limiter.max_events + + @property + def window_seconds(self) -> float: + return self._limiter.window_seconds + + def admit(self, now: float) -> bool: + """O(1): may a sample be taken now? Counts the refusals. + + This is the only part of the probe the monitor thread runs; the + frame walk itself is deferred to the diagnostics thread through + :meth:`capture`. + """ + if self._limiter.admit(now): + return True + self.suppressed_total += 1 + return False + + def capture(self, *, now: float, wake_delay_seconds: float) -> str: + """Walk the other threads' stacks for an already-admitted sample.""" + stacks = self.capture_stacks() + self.samples_total += 1 + self.last_sample = { + "monotonic": float(now), + "wake_delay_seconds": float(wake_delay_seconds), + "stacks": stacks, + } + return stacks + + def sample(self, *, now: float, wake_delay_seconds: float) -> str | None: + """Admit and capture in one step, or ``None`` when rate-limited.""" + if not self.admit(now): + return None + return self.capture(now=now, wake_delay_seconds=wake_delay_seconds) + + def capture_stacks(self) -> str: + """One line per other thread: name, ident, innermost frames first. + + Walks the frame objects directly — ``f_code.co_name``, the basename + of ``f_code.co_filename``, ``f_lineno``, ``f_back`` — and never + touches ``linecache``. ``traceback.extract_stack`` would stat every + file and read every source line (milliseconds, and megabytes kept + in ``linecache``) for text this sample does not print; even + ``StackSummary.extract(lookup_lines=False)`` still calls + ``linecache.checkcache`` per file. Nothing here reads a file. + """ + current = threading.get_ident() + names = {thread.ident: thread.name for thread in threading.enumerate()} + frame = walker = None + frames = sys._current_frames() + try: + lines: list[str] = [] + listed = 0 + for ident, frame in frames.items(): + if ident == current: + continue + if listed >= self._max_threads: + lines.append( + f"... {len(frames) - 1 - listed} more threads not listed" + ) + break + listed += 1 + entries: list[str] = [] + walker = frame + while walker is not None and len(entries) < self._max_frames: + code = walker.f_code + entries.append( + f"{code.co_name}@{os.path.basename(code.co_filename)}" + f":{walker.f_lineno}" + ) + walker = walker.f_back + lines.append( + f"thread {names.get(ident, '?')} ({ident}): " + " <- ".join(entries) + ) + return "\n".join(lines) + finally: + # The snapshot includes this frame: keeping it forms a cycle + # retaining every sampled thread's locals until cyclic GC. Drop + # both the snapshot and traversal references, even on an error or + # a sampling cap. The loop can bind frame to this frame even when + # skipping the current thread. Never clear live frames themselves. + frames.clear() + frame = walker = None + + +def _default_monotonic() -> float: + """Read ``time.monotonic`` at call time, not at class-definition time. + + The lease lifecycle's documented test seam is patching ``monotonic`` on + the shared ``time`` module object. A dataclass default captured when this + module is imported would keep reading the real clock behind such a patch, + which is exactly the kind of silent divergence between test and + production that the phase attribution exists to prevent. + """ + return time.monotonic() + + +@dataclass(frozen=True) +class LedgerLeaseHeartbeatPorts: + """Call-time-resolved coordinator seams for the lease heartbeat owner. + + Every callable resolves live coordinator state (ledger identity, test + timing overrides, patched hard-exit methods) when invoked, preserving the + documented monkeypatch seams. ``lease_hard_exit`` routes through the + coordinator's ``_ledger_lease_heartbeat_hard_exit`` delegate so instance + patches intercept every internal failure path; the real exit body then + reaches process termination only through the dynamic coordinator + ``_watchdog_hard_exit`` port (PR 80 redirects that seam). + """ + + ledger: Callable[[], object] + heartbeat_seconds: Callable[[], float] + failure_seconds: Callable[[], float] + monitor_seconds: Callable[[], float] + exit_timeout_seconds: Callable[[], float] + external_fence_timeout_seconds: Callable[[], float] + lease_hard_exit: Callable[..., None] + watchdog_hard_exit: Callable[..., None] + heartbeat_loop: Callable[[], None] + monitor_loop: Callable[[], None] + # Every interval this service measures about itself reads this clock, so + # a deterministic test can drive the whole envelope — idle waits, phase + # attribution, staleness, the monitor's own lateness — by advancing a + # virtual clock instead of sleeping and hoping. Production passes + # nothing and gets time.monotonic. + monotonic: Callable[[], float] = _default_monotonic + # The process-side scheduling allowance, resolved like every other + # timing term rather than read from a module constant: it appears on + # both sides of the safety inequality (the heartbeat's stamping path + # and the monitor's own poll), so a deployment or a scaled-down test + # that changes it must change what the policy validates. + scheduler_slack_seconds: Callable[[], float] = ( + lambda: WRITER_LEASE_HEARTBEAT_SCHEDULER_SLACK_SECONDS + ) + # Where the monitor's stall diagnostics go (issue #227). Invoked only + # from the diagnostics thread the service spawns after the exit + # decision, never from the monitor thread, so a blocking default is + # safe here; see LedgerLeaseHeartbeatService._run_diagnostics. Tests + # substitute a list append to assert on the structured warning. + log: Callable[[str], None] = _print_line + + +class LedgerLeaseHeartbeatService: + """Sole owner of ledger-lease heartbeat state, threads, and fencing. + + Owns the success/progress/server-proven timestamps and their timing + envelope, guarded-session verifier selection, the heartbeat and monitor + threads, startup readiness/fail-closed state, the stop event, the failure + reason, the exit-thread exemption, and the synchronous exact-session + external-effect fence. It owns no writer-admission or lease-release-claim + state; that remains with ``CoordinatorShutdownController``. + """ + + def __init__(self, ports: LedgerLeaseHeartbeatPorts) -> None: + self._ports = ports + self.freshness_lock = threading.Lock() + self.failure_lock = threading.Lock() + self.last_success_monotonic: float | None = None + self.last_progress_monotonic: float | None = None + self.last_server_proven_monotonic: float | None = None + self.failed: threading.Event | None = None + self.ready: threading.Event | None = None + self.stop_event: threading.Event | None = None + self.thread: threading.Thread | None = None + self.monitor_thread: threading.Thread | None = None + self.exit_thread: threading.Thread | None = None + self.failure_reason: str | None = None + self.failure_has_traceback = False + # Phase attribution (issue #212). Written by whichever thread ran the + # attempt and published as one immutable snapshot, so the monitor can + # quote the last attempt in a hard-exit reason without locking. + self.last_phases: WriterLeaseVerificationPhases = UNATTRIBUTED_PHASES + self.worst_phases: WriterLeaseVerificationPhases = UNATTRIBUTED_PHASES + self.attempt_counts: dict[str, int] = { + mode: 0 for mode in LEASE_HEARTBEAT_MODES + } + self.outcome_counts: dict[str, int] = { + outcome: 0 for outcome in LEASE_HEARTBEAT_OUTCOMES + } + # The monitor's own lateness. It decides the hard exit, so a stalled + # monitor thread consumes envelope exactly like a stalled heartbeat + # thread and must be attributable separately from database latency. + # This is the lifetime high-water mark, kept for the existing gauge + # and exit message; the histogram, threshold counters, rolling + # maximum and record age that issue #227 asked for live in + # ``monitor_wakes``. + self.monitor_wake_delay_seconds = 0.0 + self.monitor_wakes = MonitorWakeTelemetry() + # Issue #227's response to lateness beyond slack: a wake later than + # the policy's max_guaranteed_monitor_lateness is a beat on which + # exit-before-adoption was not guaranteed. Counted, with the worst + # overrun kept, and warned about (rate-limited, never blocking). + self.exit_guarantee_breaches = 0 + self.worst_exit_guarantee_overrun_seconds = 0.0 + self.last_slack_breach: dict[str, object] | None = None + self.suppressed_slack_breach_warnings = 0 + self._slack_breach_warnings = _WindowRateLimiter( + max_events=LEASE_MONITOR_STALL_PROBE_MAX_SAMPLES_PER_WINDOW, + window_seconds=LEASE_MONITOR_STALL_PROBE_WINDOW_SECONDS, + ) + self.stall_probe = StallProbe() + # The most recent diagnostics thread (stack walk, formatting and + # the log write all happen there, after the exit decision); kept so + # a test can join it before asserting on what it logged. + self.diagnostics_thread: threading.Thread | None = None + # Installed on gc.callbacks by start() and removed by stop(), so a + # coordinator that never arms the heartbeat registers nothing. + self.gc_pauses = GcPauseTelemetry() + + def record_success(self, renewal_started_monotonic: float) -> None: + """Advance the conservative freshness edge without regression.""" + with self.freshness_lock: + last_success = self.last_success_monotonic + if ( + last_success is None + or renewal_started_monotonic > last_success + ): + self.last_success_monotonic = renewal_started_monotonic + + def record_phases(self, phases: WriterLeaseVerificationPhases) -> None: + """Publish one attempt's phase breakdown for exits and metrics.""" + self.last_phases = phases + if phases.total_seconds >= self.worst_phases.total_seconds: + self.worst_phases = phases + counts = self.attempt_counts + if phases.mode in counts: + counts[phases.mode] += 1 + outcomes = self.outcome_counts + if phases.outcome in outcomes: + outcomes[phases.outcome] += 1 + + def record_progress(self) -> None: + """Stamp mid-verification progress for the staleness monitor only. + + Fires from inside verify_writer_lease_guard_session when its + attribution recheck runs a second statement: the first statement's + round trip completed, proving the heartbeat machinery is making + progress. Deliberately separate from the success timestamp — the + verification has not yet reached a verdict, so nothing that treats + success as authorization freshness may observe this mark. Only the + heartbeat monitor reads it, so a lawful two-statement verification + is not mistaken for a wedged heartbeat while a genuinely stuck + statement still ages toward the failure budget unimpeded. + """ + self.last_progress_monotonic = self._ports.monotonic() + + def record_server_proven( + self, + proven_monotonic: float | None = None, + ) -> None: + """Stamp a completed server round trip: progress plus the envelope edge. + + Client-side progress marks (attempt start, query-slot acquisition) + keep a lawfully slow verification alive under the monitor, but they + can postdate a silent guard-session death by up to one idle + interval, so they cannot carry the adoption-envelope guarantee + (hard exit completes before a replacement's silence window can + elapse). Only a response actually received from PostgreSQL — a + completed statement round trip or a whole verification — proves + the session was alive approximately now, so only these sites stamp + the server-proven edge the monitor's envelope cap measures from. + + ``proven_monotonic`` is the *send* edge of the statement whose + response just arrived: a response proves the session was alive at + some instant between the statement leaving this process and the + answer arriving, and the conservative end of that interval is when + it left. Stamping receipt time instead would let scheduler delay + between the response and this call make the session look fresher + than PostgreSQL proved, spending adoption envelope that the exit + ordering argument budgets for the monitor's own poll. Omitting the + argument keeps the old receipt-time behavior for callers that + cannot observe the send edge. + + The stamp never regresses: the heartbeat and a concurrent external + fence both publish here, and an older send edge arriving second + must not undo a newer proof. + """ + now = self._ports.monotonic() + self.last_progress_monotonic = now + proven = now if proven_monotonic is None else float(proven_monotonic) + with self.freshness_lock: + last_proven = self.last_server_proven_monotonic + if last_proven is None or proven > last_proven: + self.last_server_proven_monotonic = proven + + def adoption_silence_seconds(self) -> float: + """Resolve the ledger's adoption-silence window for timing checks. + + PsqlShareLedger keeps the value on a private attribute with no + public alias; read both spellings so operator overrides actually + reach the misconfiguration warning and the monitor's envelope cap + instead of silently comparing against the compiled default. + """ + ledger = self._ports.ledger() + for name in ( + "lease_adoption_silence_seconds", + "_lease_adoption_silence_seconds", + ): + value = getattr(ledger, name, None) + if value: + return float(value) + return float(DEFAULT_WRITER_LEASE_ADOPTION_SILENCE_SECONDS) + + def activity_age_seconds(self) -> float: + """Age of the newest monitor-visible heartbeat activity mark. + + The success timestamp is deliberately the call-start edge (authority + freshness must never look newer than what PostgreSQL proved), so it + alone cannot answer "is the heartbeat advancing?": back-to-back + verifications that each lawfully approach the statement budget age + the freshest success edge by two call durations plus the idle + interval, and a budget sized for one statement would hard-exit a + healthy sole writer. Staleness enforcers therefore measure silence + from the newest of the success edge and the progress mark, which + carries attempt starts, query-slot acquisition, statement round + trips, and completions. A verification that stops advancing stops + stamping, so a wedged statement still ages out on the same budget. + """ + last_success = float( + self.last_success_monotonic + if self.last_success_monotonic is not None + else self._ports.monotonic() + ) + last_progress = float( + self.last_progress_monotonic + if self.last_progress_monotonic is not None + else last_success + ) + return max(0.0, self._ports.monotonic() - max(last_success, last_progress)) + + def server_proven_age_seconds(self) -> float: + """Age of the newest completed guard round trip. + + The envelope edge: unlike the client-side activity marks this can + never postdate a silent guard-session death, so it is what the + monitor's adoption cap measures from. + """ + last_server_proven = float( + self.last_server_proven_monotonic + if self.last_server_proven_monotonic is not None + else self._ports.monotonic() + ) + return max(0.0, self._ports.monotonic() - last_server_proven) + + def policy(self) -> WriterLeaseHeartbeatPolicy: + """Resolve the five coupled timings as one validated policy. + + Built from live ports and the ledger's own adoption silence, so an + operator override reaches the same inequality the compiled defaults + satisfy rather than being compared against a stale constant. + """ + return WriterLeaseHeartbeatPolicy( + adoption_silence_seconds=self.adoption_silence_seconds(), + heartbeat_interval_seconds=float(self._ports.heartbeat_seconds()), + failure_budget_seconds=float(self._ports.failure_seconds()), + monitor_interval_seconds=float(self._ports.monitor_seconds()), + exit_margin_seconds=float(self._ports.exit_timeout_seconds()), + scheduler_slack_seconds=float( + self._ports.scheduler_slack_seconds() + ), + ) + + def snapshot(self) -> dict[str, object]: + """Read-only heartbeat attribution for the metrics surface. + + Every key is fixed and every nested map has a fixed key set, so the + rendered metric cardinality cannot grow with traffic. + """ + policy = self.policy() + last = self.last_phases + worst = self.worst_phases + thread = self.thread + now = self._ports.monotonic() + return { + "running": bool(thread is not None and thread.is_alive()), + "activity_age_seconds": self.activity_age_seconds(), + "server_proven_age_seconds": self.server_proven_age_seconds(), + "monitor_wake_delay_seconds": self.monitor_wake_delay_seconds, + # Issue #227: the re-armable signals. Bucket, fraction and + # generation vocabularies are closed sets from + # writer_lease_timing / process_telemetry. + "monitor_wakes": self.monitor_wakes.snapshot(now), + "exit_guarantee_breaches": int(self.exit_guarantee_breaches), + "worst_exit_guarantee_overrun_seconds": float( + self.worst_exit_guarantee_overrun_seconds + ), + "stall_probe": { + "samples": int(self.stall_probe.samples_total), + "suppressed": int(self.stall_probe.suppressed_total), + }, + "slack_breach_warnings_suppressed": int( + self.suppressed_slack_breach_warnings + ), + "gc_pauses": self.gc_pauses.snapshot(), + "attempts": dict(self.attempt_counts), + "outcomes": dict(self.outcome_counts), + "last_phase_seconds": last.phase_seconds(), + "worst_phase_seconds": worst.phase_seconds(), + "last_statement_count": last.statement_count, + "policy_seconds": { + "adoption_silence": policy.adoption_silence_seconds, + "heartbeat_interval": policy.heartbeat_interval_seconds, + "failure_budget": policy.failure_budget_seconds, + "monitor_interval": policy.monitor_interval_seconds, + "exit_margin": policy.exit_margin_seconds, + "guard_statement_timeout": ( + policy.guard_statement_timeout_seconds + ), + "scheduler_slack": policy.scheduler_slack_seconds, + "exit_envelope": policy.exit_envelope_seconds, + "server_proven_cap": policy.server_proven_cap_seconds, + "max_healthy_server_gap": policy.max_healthy_server_gap_seconds, + "stability_surplus": policy.stability_surplus_seconds, + "max_guaranteed_monitor_lateness": ( + policy.max_guaranteed_monitor_lateness_seconds + ), + }, + } + + def start(self) -> threading.Thread | None: + """Arm both coupled threads and wait for the first liveness proof.""" + existing = self.thread + if existing is not None and existing.is_alive(): + return existing + ledger = self._ports.ledger() + guard_required = bool( + getattr(ledger, "writer_lease_guard_required", False) + ) + if not bool( + getattr(ledger, "writer_lease_fast_adoption_capable", False) + ): + if guard_required: + # The session token published to the lease row promises a + # guarded heartbeat session, but the advisory guard is already + # gone (for example the dedicated connection dropped between + # ledger construction and serve()). Continuing would let a + # replacement adopt the lease after one silence window while + # this process keeps running as a fenced zombie. + self._ports.lease_hard_exit( + "prism coordinator: writer lease guard was lost before " + "the lease heartbeat started; hard-exiting so a " + "replacement does not adopt while this process is live", + include_traceback=False, + ) + return None + verify = guard_session_verifier(ledger) + if verify is None: + if guard_required: + self._ports.lease_hard_exit( + "prism coordinator: writer lease guard requires a " + "heartbeat but the ledger cannot verify the guarded " + "session; hard-exiting instead of running unguarded", + include_traceback=False, + ) + return None + # One validated policy, resolved before anything is armed. An unsafe + # combination is refused outright rather than warned about and run: + # the whole content of a violation is "this process may still be + # live when a replacement adopts the lease", which is the split-brain + # the guard exists to prevent, and running anyway trades a loud + # startup failure for a silent double-writer window. Refusing here — + # before the threads exist and before serve() admits a writer — is + # the fail-closed reading. + policy = self.policy() + violations = policy.violations() + if violations: + self._ports.lease_hard_exit( + "prism coordinator: refusing to start the ledger lease " + "heartbeat; its timing policy cannot guarantee hard exit " + "before replacement adoption (" + + "; ".join(violations) + + f"). Policy: {policy.describe()}", + include_traceback=False, + ) + return None + for advisory in policy.advisories(): + # Safe but unstable: no double-writer window, but ordinary + # verification tail latency will hard-exit a healthy + # coordinator. A lab or test policy is allowed to make that + # trade deliberately, so this stays a warning. + print( + "prism coordinator: ledger lease heartbeat timing has no " + f"tail-latency headroom: {advisory}. Policy: " + f"{policy.describe()}", + flush=True, + ) + armed_started_monotonic = self._ports.monotonic() + self.last_success_monotonic = armed_started_monotonic + self.last_server_proven_monotonic = armed_started_monotonic + self.failed = threading.Event() + self.ready = threading.Event() + self.stop_event = threading.Event() + # GC pause durations are process-wide but the stall attribution is + # this service's question, so the collector callbacks are armed + # with the threads and disarmed with them (see stop()). + self.gc_pauses.install() + thread = threading.Thread( + target=self._ports.heartbeat_loop, + name="prism-ledger-lease-heartbeat", + daemon=True, + ) + self.thread = thread + thread.start() + monitor_thread = threading.Thread( + target=self._ports.monitor_loop, + name="prism-ledger-lease-heartbeat-monitor", + daemon=True, + ) + monitor_thread.start() + self.monitor_thread = monitor_thread + failure_seconds = policy.failure_budget_seconds + # The startup fencing deadline measures silence from the newest + # monitor-visible activity, not from arming: a first verification + # that is legally slower than the whole budget end-to-end but + # demonstrably advancing (queue wait, then a statement round trip + # near the guard's statement timeout) is not a wedged first beat. + # A genuinely stuck first statement stamps nothing and ages out on + # the same budget the monitor enforces. + while not self.ready.wait(0.01): + if self.failed.is_set(): + return None + if self.activity_age_seconds() < failure_seconds: + continue + self._ports.lease_hard_exit( + "prism coordinator: initial ledger lease heartbeat did not " + "complete before startup fencing deadline " + f"({self.last_phases.summary()})", + include_traceback=False, + ) + return None + return thread + + def hard_exit(self, message: str, *, include_traceback: bool) -> None: + with self.failure_lock: + failed = self.failed + if failed is None: + failed = threading.Event() + self.failed = failed + if failed.is_set(): + return + failed.set() + # This thread now proceeds to _watchdog_hard_exit and stays parked + # there until os._exit; it can never issue another renewal. The + # release worker must not join it, or a heartbeat/monitor-driven + # exit deadlocks against its own release and burns the exit budget + # before the fresh-connection lease release can run. + self.exit_thread = threading.current_thread() + # Never write to stdout/stderr before arming the hard-exit path. A full + # container log pipe can block a flush forever, leaving writer + # admission open and the old process alive. Keep the detail in memory + # for tests/embedders whose patched hard-exit returns; the real process + # exits and Docker's die event is the authoritative diagnostic. + self.failure_reason = message + self.failure_has_traceback = include_traceback + self._ports.watchdog_hard_exit( + "lease_heartbeat", + timeout_seconds=float(self._ports.exit_timeout_seconds()), + ) + + def run_guard_attempt( + self, + call: Callable[..., object], + callbacks: frozenset[str], + mode: str, + ) -> object: + """Run one guarded call, attributing its phases, and publish them. + + Every progress mark this installs feeds the same two clocks the + monitor reads, and the attempt object turns the same marks into the + phase breakdown an operator needs to see in the exit reason: + queue wait for the guard's serialized slot, guard SQL across the + round trips, and the residual — Python scheduling and GIL + contention — that neither of those explains. + + Progress marks feed only the staleness monitor; authority freshness + still moves exclusively on the conservative success edge the caller + records. The attempt-start and slot-acquisition marks are + client-side and can postdate a silent guard-session death by at + most one idle interval, so they carry only the liveness budget. The + adoption envelope is enforced separately by the monitor's + server-proven cap, which measures from completed round trips at + their conservative send edge. During any hang the external-effect + fence also queues behind the hung statement and fails closed, so no + new external effect authorizes while the monitor ages out. + + Raises whatever the guarded call raises, after publishing the + phases of the failed attempt — the exit reason is built from them. + """ + attempt = WriterLeaseVerificationAttempt( + mode, + monotonic=self._ports.monotonic, + ) + + def on_query_start() -> None: + attempt.slot_acquired() + self.record_progress() + + def on_statement_end() -> None: + self.record_server_proven(attempt.statement_completed()) + + kwargs: dict[str, Callable[..., None]] = {} + if "on_query_start" in callbacks: + kwargs["on_query_start"] = on_query_start + if "on_statement_end" in callbacks: + kwargs["on_statement_end"] = on_statement_end + elif "on_statement_progress" in callbacks: + # The legacy spelling: fires only between a verification's two + # statements, which is enough to keep a lawful attribution + # recheck from looking wedged but carries no phase timing. + kwargs["on_statement_progress"] = self.record_server_proven + if "on_statement_server_seconds" in callbacks: + # Issue #227: the statement's own server-side execution time, + # so a GIL stall between PostgreSQL answering and this thread + # resuming is attributed to this process rather than to SQL. + # Timing only; the proven edge above stays the send edge. + kwargs["on_statement_server_seconds"] = ( + attempt.statement_server_seconds + ) + try: + result = call(**kwargs) + except BaseException: + self.record_phases(attempt.finish(LEASE_HEARTBEAT_OUTCOME_FAILED)) + raise + self.record_phases(attempt.finish(self._attempt_outcome(mode, result))) + # Completion is a real server response even when the call reported + # no per-statement marks; stamp the conservative edge so a verifier + # without callbacks still keeps the envelope fed. + self.record_server_proven(attempt.proven_edge_monotonic) + return result + + @staticmethod + def _attempt_outcome(mode: str, result: object) -> str: + """Classify one guarded call for attribution and metrics.""" + if not isinstance(result, dict): + return ( + LEASE_HEARTBEAT_OUTCOME_PROVEN + if mode == LEASE_HEARTBEAT_MODE_PROOF + else LEASE_HEARTBEAT_OUTCOME_RENEWED + ) + if mode == LEASE_HEARTBEAT_MODE_PROOF: + if result.get("lease_renewal_due"): + return LEASE_HEARTBEAT_OUTCOME_RENEWAL_DUE + return LEASE_HEARTBEAT_OUTCOME_PROVEN + if result.get("renewal_deferred_to_own_write"): + return LEASE_HEARTBEAT_OUTCOME_DEFERRED + try: + renewed = int(result.get("renewed_count", 0)) + except (TypeError, ValueError): + renewed = 0 + if renewed > 0: + return LEASE_HEARTBEAT_OUTCOME_RENEWED + return LEASE_HEARTBEAT_OUTCOME_PROVEN + + def heartbeat_loop(self) -> None: + """Keep proving the guarded session live on an isolated DB path. + + The heartbeat must never wait on the lease tuple that fenced writes + hold for whole transactions, or a long ``persist_accepted_block`` + would time out the heartbeat statement and hard-exit a healthy + coordinator (the block-39416 restart loop). It still renews the + lease TTL whenever that tuple is uncontended, so an idle coordinator + (no fenced writes, CTV broadcaster disabled) does not let + ``lease_expires_at`` lapse into different-identity expiry claims. + + Ownership and renewal run at different rates (issue #212). Each beat + asks the cheap read-only ownership question — one non-blocking + statement, no row lock and no ``pg_stat_activity`` scan — and the + full renewing verification runs on the first beat and thereafter + only on a beat whose proof reports the committed row inside the + own-write authority margin. That escalation happens on the *same* + beat, so renewal is never later than it was before the split, and + while renewals are actually being skipped (a long own fenced write) + every beat runs the same fail-closed verification it always did. + A ledger with no proof method keeps running the full verification + on every beat. + """ + ledger = self._ports.ledger() + verify = guard_session_verifier(ledger) + if verify is None: + return + prove = guard_session_prover(ledger) + verify_callbacks = verifier_callbacks(verify) + prove_callbacks = ( + verifier_callbacks(prove) if prove is not None else frozenset() + ) + interval_seconds = float(self._ports.heartbeat_seconds()) + heartbeat_stop = self.stop_event + if heartbeat_stop is None: + heartbeat_stop = threading.Event() + self.stop_event = heartbeat_stop + # The first beat renews: startup readiness must prove the renewing + # path works, not merely that the session answers. + renew_this_beat = True + while not heartbeat_stop.is_set(): + beat_started_monotonic = self._ports.monotonic() + # An advancing verification is monitor-visible activity, and the + # attempt start is its first client-side mark. + self.record_progress() + try: + if prove is not None and not renew_this_beat: + proof = self.run_guard_attempt( + prove, + prove_callbacks, + LEASE_HEARTBEAT_MODE_PROOF, + ) + renew_this_beat = not isinstance(proof, dict) or bool( + proof.get("lease_renewal_due") + ) + if renew_this_beat or prove is None: + self.record_progress() + self.run_guard_attempt( + verify, + verify_callbacks, + LEASE_HEARTBEAT_MODE_RENEW, + ) + # A landed renewal (or a deferral behind this writer's + # own in-flight write) puts the answer back in the + # committed row: the next beat's proof re-reads it and + # escalates again if renewal is still outstanding. + renew_this_beat = False + except Exception: + self._ports.lease_hard_exit( + "prism coordinator: ledger lease heartbeat failed; " + "hard-exiting so this process cannot outlive its " + "fast-adoptable session " + f"({self.last_phases.summary()})", + include_traceback=True, + ) + return + # The session was proven live no earlier than the beat's start. + # Using that conservative edge prevents a delayed response from + # making local freshness look newer than what PostgreSQL proved. + self.record_success(beat_started_monotonic) + ready = self.ready + if ready is not None: + ready.set() + if heartbeat_stop.wait(interval_seconds): + return + + def monitor_loop(self) -> None: + # One resolved policy for the whole loop: the monitor's two bounds + # and the exit reason all quote the same inequality. + policy = self.policy() + failure_seconds = policy.failure_budget_seconds + monitor_seconds = policy.monitor_interval_seconds + heartbeat_stop = self.stop_event + if heartbeat_stop is None: + heartbeat_stop = threading.Event() + self.stop_event = heartbeat_stop + # The adoption-envelope cap: the client-side progress marks that + # keep a lawfully slow verification alive can postdate a silent + # guard-session death by up to one idle interval, which with a + # silence sized at interval + budget would let the hard exit land + # at or after a replacement's CAS eligibility. Server-proven marks + # cannot postdate the death, so a second bound measured from them + # restores exit-before-adoption regardless of client marks. What + # the cap must leave free inside the silence is the whole exit + # envelope: the hard-exit budget, one poll of granularity, this + # thread's own budgeted lateness, and one further poll of strict + # reserve (see WriterLeaseHeartbeatPolicy.exit_envelope_seconds). + # Reserving only the poll intervals — as this did before the + # scheduler-slack term was added — lets a maximally late monitor + # begin the exit after the successor is already eligible. The cap + # is floored at the failure budget so a deliberately tiny (advised) + # silence degrades to the plain liveness bound instead of killing + # lawful single-statement verifications. + server_cap_seconds = policy.server_proven_cap_seconds + while True: + wait_started_monotonic = self._ports.monotonic() + if heartbeat_stop.wait(monitor_seconds): + return + # The monitor decides the hard exit, so its own lateness is + # spent envelope exactly like a slow statement is. Attributing + # it separately is what lets an operator tell "PostgreSQL was + # slow" from "this process could not get scheduled". + wake_delay = max( + 0.0, + self._ports.monotonic() - wait_started_monotonic - monitor_seconds, + ) + if wake_delay > self.monitor_wake_delay_seconds: + self.monitor_wake_delay_seconds = wake_delay + # Issue #227: the lifetime maximum above is a signal that can + # fire once per process. Feed the histogram, the slack-fraction + # counters, the rolling maximum and the record age, and decide + # whether the documented response is due. All of that is O(1) + # arithmetic; anything heavier (the stack walk, the formatting, + # the log write) comes back as a deferred job and runs only + # *after* the exit decision below, and never on this thread. + # The instrument must not extend what it measures: work done + # here would consume the very envelope it is reporting, and + # work done between polls is not measured as lateness at all. + diagnostics = self._observe_monitor_wake(wake_delay, policy) + # Mid-verification progress counts: the budget — sized for one + # statement plus one idle wait plus scheduler slack so it stays + # under the adoption silence — cannot absorb a whole multi-step + # verification, and each demonstrable step (attempt start, slot + # acquisition, statement round trip, completion) is the + # advancing-heartbeat evidence this monitor watches for. A + # wedged statement records nothing and still ages out. + age_seconds = self.activity_age_seconds() + server_age_seconds = self.server_proven_age_seconds() + if ( + age_seconds < failure_seconds + and server_age_seconds < server_cap_seconds + ): + if diagnostics is not None: + self._run_diagnostics(diagnostics) + continue + # Exiting: the deferred diagnostics are dropped. The exit reason + # below already carries the wake-delay figures and the worst GC + # pause, and nothing may run ahead of the exit. + # Name the bound that tripped and the phase that consumed it. + # Issue #212 was unresolvable in production precisely because + # the exit said only how stale the heartbeat was, never where + # the time went. + if age_seconds >= failure_seconds: + tripped = ( + f"activity {age_seconds:.3f}s >= failure budget " + f"{failure_seconds:.3f}s" + ) + else: + tripped = ( + f"server-proven {server_age_seconds:.3f}s >= adoption " + f"envelope cap {server_cap_seconds:.3f}s" + ) + # The wake-delay figures are the lifetime record, the trailing + # window's worst and the record's age, so an operator can tell + # a stall that just happened from one hours old (issue #227). + exit_now = self._ports.monotonic() + gc_generation, gc_worst = self.gc_pauses.worst() + self._ports.lease_hard_exit( + "prism coordinator: ledger lease heartbeat stopped making " + f"progress for {age_seconds:.3f}s " + f"(server-proven {server_age_seconds:.3f}s); hard-exiting " + "before its session becomes fast-adoptable. " + f"Tripped: {tripped}. Last attempt: " + f"{self.last_phases.summary()}. Worst attempt: " + f"{self.worst_phases.summary()}. Monitor wake delay: " + f"{self.monitor_wake_delay_seconds:.3f}s lifetime record " + f"({self.monitor_wakes.record_age_seconds(exit_now):.1f}s old), " + f"{self.monitor_wakes.window_max_seconds(exit_now):.3f}s in the " + f"last {LEASE_MONITOR_WAKE_DELAY_WINDOW_SECONDS:g}s, " + f"{self.exit_guarantee_breaches} exit-guarantee breach(es). " + f"Worst GC pause: {gc_worst:.3f}s " + f"(generation {gc_generation if gc_generation is not None else 'none'}). " + f"Policy: {policy.describe()}", + include_traceback=False, + ) + return + + def _observe_monitor_wake( + self, + wake_delay: float, + policy: WriterLeaseHeartbeatPolicy, + ) -> Callable[[], None] | None: + """Account one monitor wake in O(1); return the deferred diagnostics. + + The decision issue #227 asked for, argued in the + ``writer_lease_timing`` module docstring: lateness beyond the slack + is *accepted* rather than turned into a tightened cap (which would + hard-exit healthy coordinators under a transient stall — issue + #212 — without moving the exit earlier on the beat that breached), + because every effect that could escape during the un-covered + interval is fenced at the row or behind a synchronous + exact-session verification. What changes is that the breach is no + longer invisible: it is counted, its overrun past the + exit-guarantee bound is kept, one structured warning names it, and + a stack sample says what held the interpreter. + + Three tiers, all keyed on the policy's ``scheduler_slack``: + + * at or above half the slack: take a rate-limited stack sample; + * above the slack: the budgeted assumption was exceeded — record + the breach and warn (rate-limited); + * above ``max_guaranteed_monitor_lateness`` (slack plus one + monitor interval): exit-before-adoption was not guaranteed for + this beat — count it and keep the worst overrun. + + This runs on the monitor thread *before* the staleness decision, so + it does arithmetic only: the histogram, the counters, the + rate-limit admissions and a handful of float reads. Everything that + walks frames, formats text or writes a log line is returned as a + closure for :meth:`monitor_loop` to hand to + :meth:`_run_diagnostics` after the exit decision — or to drop, when + the decision is to exit. The instrument must not extend the + envelope it reports on. + """ + now = self._ports.monotonic() + slack = float(policy.scheduler_slack_seconds) + self.monitor_wakes.observe(wake_delay, slack_seconds=slack, now=now) + if wake_delay < LEASE_MONITOR_STALL_PROBE_TRIGGER_SLACK_FRACTION * slack: + return None + probe_admitted = self.stall_probe.admit(now) + if wake_delay <= slack: + if not probe_admitted: + return None + last_summary = self.last_phases.summary() + return lambda: self._emit_probe_sample( + now=now, + wake_delay=wake_delay, + slack=slack, + last_summary=last_summary, + ) + bound = policy.max_guaranteed_monitor_lateness_seconds + overrun = policy.exit_guarantee_overrun_seconds(wake_delay) + if overrun > 0.0: + self.exit_guarantee_breaches += 1 + if overrun > self.worst_exit_guarantee_overrun_seconds: + self.worst_exit_guarantee_overrun_seconds = overrun + # The figures the warning quotes are read now, at the wake, not + # when the diagnostics thread gets around to formatting them. + server_proven_age = self.server_proven_age_seconds() + activity_age = self.activity_age_seconds() + window_max = self.monitor_wakes.window_max_seconds(now) + breaches = self.exit_guarantee_breaches + self.last_slack_breach = { + "monotonic": now, + "wake_delay_seconds": float(wake_delay), + "overrun_seconds": float(overrun), + "server_proven_age_seconds": server_proven_age, + "stacks": None, + } + warning_admitted = self._slack_breach_warnings.admit(now) + if not warning_admitted: + self.suppressed_slack_breach_warnings += 1 + if not probe_admitted and not warning_admitted: + return None + return lambda: self._emit_slack_breach( + now=now, + wake_delay=wake_delay, + slack=slack, + bound=bound, + overrun=overrun, + breaches=breaches, + server_proven_age=server_proven_age, + activity_age=activity_age, + window_max=window_max, + policy=policy, + probe_admitted=probe_admitted, + warning_admitted=warning_admitted, + ) + + def _run_diagnostics(self, job: Callable[[], None]) -> None: + """Run deferred wake diagnostics on a throwaway daemon thread. + + Never on the monitor thread. The monitor's poll spacing is one + interval plus its lateness plus whatever it does between polls, + and that last term is not measured as lateness, so it has to stay + O(1): a frame walk, string formatting or a print here would be + unmeasured envelope consumption on the thread that decides the + hard exit. The rate limits upstream (probe and warning admissions) + bound how many of these threads a stall storm can create. + """ + thread = threading.Thread( + target=job, + name="prism-ledger-lease-monitor-diagnostics", + daemon=True, + ) + self.diagnostics_thread = thread + try: + thread.start() + except Exception: + pass + + def _emit_probe_sample( + self, + *, + now: float, + wake_delay: float, + slack: float, + last_summary: str, + ) -> None: + """Diagnostics thread: an admitted half-slack sample and its log line.""" + stacks = self.stall_probe.capture(now=now, wake_delay_seconds=wake_delay) + self._ports.log( + "prism coordinator: ledger lease heartbeat monitor woke " + f"{wake_delay:.3f}s late (scheduler slack {slack:.3f}s); " + "stall probe sampled the running threads. Last attempt: " + f"{last_summary}. Threads:\n{stacks}" + ) + + def _emit_slack_breach( + self, + *, + now: float, + wake_delay: float, + slack: float, + bound: float, + overrun: float, + breaches: int, + server_proven_age: float, + activity_age: float, + window_max: float, + policy: WriterLeaseHeartbeatPolicy, + probe_admitted: bool, + warning_admitted: bool, + ) -> None: + """Diagnostics thread: the sample and the structured breach warning.""" + stacks: str | None = None + if probe_admitted: + stacks = self.stall_probe.capture(now=now, wake_delay_seconds=wake_delay) + breach = self.last_slack_breach + if breach is not None and breach.get("monotonic") == now: + breach["stacks"] = stacks + if not warning_admitted: + return + if overrun > 0.0: + verdict = ( + "exit-before-adoption was NOT guaranteed for this beat: " + f"the worst-case exit lands {overrun:.3f}s after the " + f"adoption edge (bound {bound:.3f}s)" + ) + else: + verdict = ( + "exit-before-adoption still held for this beat, with the " + "strictness reserve consumed " + f"(bound {bound:.3f}s)" + ) + gc_generation, gc_worst = self.gc_pauses.worst() + self._ports.log( + "prism coordinator: WARNING ledger lease heartbeat monitor wake " + f"delay {wake_delay:.3f}s exceeded the {slack:.3f}s scheduler " + "slack the timing policy budgets (issue #227 lateness-beyond-" + f"slack breach; {breaches} exit-guarantee " + f"breach(es) this process); {verdict}. The stall was in-process " + "(this thread could not be scheduled); every ledger write is " + "session-token fenced and every external effect is behind an " + "exact-session verification, so no effect escapes, but the " + "process may have outlived the adoption edge had its guard " + "session died. " + f"server_proven_age={server_proven_age:.3f}s " + f"activity_age={activity_age:.3f}s " + f"window_max={window_max:.3f}s " + f"worst_gc_pause={gc_worst:.3f}s" + f"(generation {gc_generation if gc_generation is not None else 'none'}). " + f"Last attempt: {self.last_phases.summary()}. Worst attempt: " + f"{self.worst_phases.summary()}. Policy: {policy.describe()}." + + (f" Threads:\n{stacks}" if stacks is not None else "") + ) + + def stop(self, *, deadline: float | None = None) -> bool: + heartbeat_stop = self.stop_event + if heartbeat_stop is None: + return True + heartbeat_stop.set() + self.gc_pauses.uninstall() + threads = (self.thread, self.monitor_thread) + # A heartbeat or monitor thread that armed the hard exit is blocked in + # _watchdog_hard_exit joining the very release worker that runs this + # method. It has already stopped renewing forever, so treat it as + # stopped instead of deadlocking against it until the deadline. + exit_thread = self.exit_thread + if deadline is None: + deadline = ( + self._ports.monotonic() + + DEFAULT_WRITER_LEASE_ADOPTION_SILENCE_SECONDS + ) + for thread in threads: + if ( + thread is None + or thread is threading.current_thread() + or thread is exit_thread + ): + continue + thread.join(max(0.0, deadline - self._ports.monotonic())) + return all( + thread is None + or thread is threading.current_thread() + or thread is exit_thread + or not thread.is_alive() + for thread in threads + ) + + def require_fresh_lease_for_external_side_effect(self, component: str) -> None: + """Synchronously verify the exact guarded session before an RPC effect. + + The periodic heartbeat is an early process-liveness detector, not an + authorization oracle: CLOCK_MONOTONIC may not advance across host + suspend, and a PostgreSQL connection object does not necessarily know + its server session died until the next I/O. Every external mutation + therefore performs a bounded non-blocking verification on the session + holding the advisory guard. It must not wait on the lease row: a long + fenced transaction (``persist_accepted_block``) holds that tuple's + row lock until commit, and queueing behind it would time out and + fence a healthy coordinator (the TTL is renewed opportunistically via + SKIP LOCKED instead). A daemon worker bounds a dead network path even + when the driver's server-side statement timeout cannot. + + PostgreSQL and qbitd are independent systems, so this remains a + preflight fence rather than an atomic transaction with the subsequent + RPC. The advisory session remains held across the RPC in the ordinary + case; see the deployment guide for the residual post-check pause risk. + + A verification that reports ``renewal_deferred_to_own_write`` proves + liveness but not authority: the writer's own fenced write holds the + expired lease row, and only its commit guarantees the lease survives. + That is enough for the heartbeat, which merely keeps a healthy + process from self-fencing, but not for an RPC — a rollback would let + a queued claimant take the lease while the effect is in flight. This + fence then raises :class:`WriterLeaseRenewalDeferred` without arming + the hard exit; callers already retry deferred work on their own + cadence. + """ + ledger = self._ports.ledger() + if not bool( + getattr(ledger, "writer_lease_guard_required", False) + ): + return + verify = guard_session_verifier(ledger) + if verify is None: + self._ports.lease_hard_exit( + "prism coordinator: refusing external side effect from " + f"{component}; exact-session verification is unavailable", + include_traceback=False, + ) + raise ShutdownInProgress( + f"writer lease guard verification is unavailable before {component}" + ) + + # The configured fence timeout budgets one statement; the + # verification may lawfully run a second inside the same guarded + # slot (the attribution recheck), and killing that recheck at the + # single-statement deadline would hard-exit a coordinator the + # recheck was about to prove healthy. Two moderately slow but + # in-deadline statements must both fit. + timeout_seconds = WRITER_LEASE_VERIFICATION_MAX_STATEMENTS * max( + 0.0, + float(self._ports.external_fence_timeout_seconds()), + ) + # The guarded session serializes this verification behind the periodic + # heartbeat and any concurrent fence, and the execution budget above + # tracks the guard's statement timeout. Waiting for the serialized + # query slot must not consume that budget, or a fence arriving during + # an in-flight renewal hard-exits a coordinator whose session is still + # healthy. The queue wait gets the heartbeat failure budget instead: a + # guard session that cannot complete queued work within it is already + # being declared dead by the heartbeat monitor. + queue_timeout_seconds = max( + timeout_seconds, + float(self._ports.failure_seconds()), + ) + query_started = threading.Event() + callbacks = verifier_callbacks(verify) + verify_reports_query_start = "on_query_start" in callbacks + # This verification holds the guard's serialized slot for its whole + # statement sequence, so the periodic heartbeat queues behind it + # and records nothing meanwhile. Its lawful second statement (the + # attribution recheck) can therefore age the heartbeat's last + # success past the monitor's failure budget — the same hazard the + # heartbeat loop's own recheck had — so this caller must feed the + # monitor its completed round trips too. + verify_reports_statement_end = "on_statement_end" in callbacks + verify_reports_statement_progress = ( + not verify_reports_statement_end + and "on_statement_progress" in callbacks + ) + verify_reports_server_seconds = ( + "on_statement_server_seconds" in callbacks + ) + # The fence is a guard-slot caller like the heartbeat, so its phases + # belong in the same attribution: a fence that spent its budget + # queueing behind an in-flight renewal reads very differently from + # one that spent it inside PostgreSQL. + attempt = WriterLeaseVerificationAttempt( + LEASE_HEARTBEAT_MODE_FENCE, + monotonic=self._ports.monotonic, + ) + if not verify_reports_query_start: + # A verification that cannot report when its query slot was + # acquired keeps the previous behavior: the whole budget covers + # both. + query_started.set() + + outcome: queue.Queue[BaseException | None] = queue.Queue(maxsize=1) + verification_result: list[object] = [] + verification_started_monotonic = attempt.started_monotonic + + def on_query_start() -> None: + attempt.slot_acquired() + query_started.set() + + def on_statement_end() -> None: + self.record_server_proven(attempt.statement_completed()) + + def verify_exact_session() -> None: + try: + verify_kwargs: dict[str, Callable[..., None]] = {} + if verify_reports_query_start: + verify_kwargs["on_query_start"] = on_query_start + if verify_reports_statement_end: + verify_kwargs["on_statement_end"] = on_statement_end + elif verify_reports_statement_progress: + verify_kwargs["on_statement_progress"] = ( + self.record_server_proven + ) + if verify_reports_server_seconds: + verify_kwargs["on_statement_server_seconds"] = ( + attempt.statement_server_seconds + ) + result = verify(**verify_kwargs) + except BaseException as exc: + outcome.put(exc) + else: + verification_result.append(result) + outcome.put(None) + + try: + thread = threading.Thread( + target=verify_exact_session, + name="prism-ledger-lease-external-fence", + daemon=True, + ) + thread.start() + except BaseException as exc: + self._ports.lease_hard_exit( + "prism coordinator: refusing external side effect from " + f"{component}; could not start exact-session verification", + include_traceback=False, + ) + raise ShutdownInProgress( + f"writer lease guard verification could not start before {component}" + ) from exc + + def refuse(detail: str) -> str: + """Fail closed, naming the phase that consumed the fence budget.""" + phases = attempt.finish(LEASE_HEARTBEAT_OUTCOME_FAILED) + self.record_phases(phases) + return ( + "prism coordinator: refusing external side effect from " + f"{component}; {detail} ({phases.summary()})" + ) + + queue_deadline = verification_started_monotonic + queue_timeout_seconds + while not query_started.is_set() and thread.is_alive(): + remaining = queue_deadline - self._ports.monotonic() + if remaining <= 0.0: + self._ports.lease_hard_exit( + refuse( + "exact-session verification could not start within " + f"{queue_timeout_seconds:g}s" + ), + include_traceback=False, + ) + raise ShutdownInProgress( + f"writer lease guard verification timed out before {component}" + ) + thread.join(min(0.01, remaining)) + thread.join(timeout_seconds) + if thread.is_alive(): + self._ports.lease_hard_exit( + refuse( + "exact-session verification exceeded " + f"{timeout_seconds:g}s" + ), + include_traceback=False, + ) + raise ShutdownInProgress( + f"writer lease guard verification timed out before {component}" + ) + + try: + error = outcome.get_nowait() + except queue.Empty as exc: + self._ports.lease_hard_exit( + refuse("exact-session verification returned no result"), + include_traceback=False, + ) + raise ShutdownInProgress( + f"writer lease guard verification failed before {component}" + ) from exc + if error is not None: + self._ports.lease_hard_exit( + refuse("exact-session verification failed"), + include_traceback=False, + ) + raise ShutdownInProgress( + f"writer lease guard verification failed before {component}" + ) from error + + # Use the call-start time so scheduler delay never makes this success + # appear fresher than the database response actually proves. The + # envelope edge takes the conservative send edge of the newest + # completed round trip for the same reason. + self.record_success(verification_started_monotonic) + self.record_server_proven(attempt.proven_edge_monotonic) + + # A verification that proved liveness only because the writer's own + # fenced write holds the expired lease row is not authority for an + # external side effect: that survival argument assumes the write + # commits, and a rollback would hand the row to a queued + # different-identity claimant while the RPC is in flight. Refuse the + # side effect without fencing the process — the session is healthy + # and the write's commit will land the next renewal; callers retry + # on their own cadence (broadcast interval, candidate outbox replay). + result = verification_result[0] if verification_result else None + self.record_phases( + attempt.finish( + self._attempt_outcome(LEASE_HEARTBEAT_MODE_FENCE, result) + ) + ) + if isinstance(result, dict) and result.get( + "renewal_deferred_to_own_write" + ): + raise WriterLeaseRenewalDeferred( + f"withholding {component}: writer lease renewal is deferred " + "behind this coordinator's own in-flight fenced write; " + "retry after that transaction commits and a renewal lands" + ) + + +class _LeaseHeartbeatStateField: + """Descriptor routing a legacy coordinator attribute to lease ownership. + + The lease service holds the only copy of each mutable field; the + coordinator class keeps the historical private attribute names readable + and writable for tests and embedders. Reads and writes construct the + lazy service on first touch so pre-service fixture assignments are + adopted rather than duplicated. + """ + + def __init__(self, attribute: str): + self.attribute = attribute + + def __get__(self, instance: Any, owner: type[Any] | None = None) -> Any: + if instance is None: + return self + return getattr( + instance._ensure_lease_heartbeat_service(), self.attribute + ) + + def __set__(self, instance: Any, value: Any) -> None: + setattr( + instance._ensure_lease_heartbeat_service(), self.attribute, value + ) diff --git a/lab/prism/block_candidates.py b/lab/prism/block_candidates.py new file mode 100644 index 000000000..3880de25f --- /dev/null +++ b/lab/prism/block_candidates.py @@ -0,0 +1,8967 @@ +"""Durable PRISM block-candidate codec, replay queue, and submitter service. + +B1 owner module: the node-offer fast lane, the same-hash disposition guard, +the accounting actor and its handoff queues, startup/outbox replay and +quarantine, retry merge/pacing/finalize-only state, acceptance evidence, +block-work liveness heartbeats, and the bounded DB/RPC call workers all live +here. Post-offer accepted-block finalization (``submit_block_candidate``, +``_submit_block_candidate_serialized``, ``_land_and_confirm_block_candidate``) +stays coordinator-owned at this layer and is reached through the injected +coordinator reference; a later layer moves it to its own finalization owner. +This module never imports ``prism_coordinator``. +""" + +from __future__ import annotations + +import dataclasses +import inspect +import itertools +import json +import math +import queue +import threading +import time +import traceback +from contextlib import AbstractContextManager, contextmanager +from dataclasses import dataclass, field, replace as dataclass_replace +from types import SimpleNamespace +from collections.abc import Mapping, Sequence +from typing import Any, Callable, Iterable, Iterator, Protocol + +from lab.prism import direct_stratum +from lab.prism.audit_bundle_view import MappedSequence +from lab.prism.accepted_preview_telemetry import ( + LANDING_PHASE_LANE_WAIT, + LANDING_PHASE_PREVIEW_PREPARE, + LANDING_PHASE_PREVIEW_PUBLISH, + PRISM_ACCEPTED_LANDING_PHASES, + RECONCILE_CALLER_LANDING, + AcceptedPreviewPublicationDiagnostic, + ensure_accepted_preview_telemetry, +) +from lab.prism.coordinator_config import ( + BLOCK_LANDING_DB_TIMEOUT_WATCHDOG_FRACTION, + DEFAULT_ACCEPTED_PARENT_REDRIVE_ATTEMPT_MAX, + DEFAULT_ACCEPTED_PARENT_REDRIVE_DEFER_THRESHOLD, + DEFAULT_BLOCK_CANDIDATE_CLEANUP_RETRY_BACKLOG_MAX, + DEFAULT_BLOCK_LANDING_DB_TIMEOUT_MAX_SECONDS, + DEFAULT_BLOCK_LANDING_DB_TIMEOUT_SECONDS, + DEFAULT_BLOCK_SUBMIT_DB_TIMEOUT_SECONDS, + DEFAULT_BLOCK_SUBMIT_LOCK_WAIT_LOG_SECONDS, + DEFAULT_BLOCK_SUBMIT_RPC_TIMEOUT_SECONDS, + DEFAULT_BLOCK_SUBMIT_STUCK_CALL_EXIT_SECONDS, + DEFAULT_PRISM_OBSERVED_TIP_ACCEPT_WINDOW_SECONDS, + DEFAULT_PRISM_WATCHDOG_TIMEOUT_SECONDS, + MAX_BLOCK_CANDIDATE_CLEANUP_RETRY_BACKLOG_MAX, + MAX_BLOCK_REPLAY_PAGE_SIZE, +) +from lab.prism.candidate_codec import ( + CandidateBodyIntegrityError, + CandidateCodecError, + PreparedCandidateIntent, + prepare_candidate_intent, +) +from lab.prism.candidate_store import ( + HEADER_PAGE_MAX_BYTES, + HEADER_PAGE_MAX_ROWS, + CandidateBodyRef, + CandidateBodyUnavailable, + DurableCandidateDescriptor, +) +from lab.prism.coordinator_shutdown import ShutdownInProgress +from lab.prism.job_bundle import PRISM_JOB_BUILD_SECONDS_BUCKETS +from lab.prism.job_delivery import PrismJobContext +from lab.prism.rpc import JsonRpc +from lab.prism.share_ledger import LedgerOperationTimeout, PendingShare +from lab.prism.stratum_session import ClientState, WorkerIdentity + + +# Block candidates queue to a dedicated submitter thread so the miner's share +# ack never waits on audit/submitblock after the share and intent commit. The +# bound limits RAM; overflow only coalesces a wakeup because Postgres retains +# the authoritative pending candidate. +MAX_PENDING_BLOCK_CANDIDATES = 32 +# Startup replay must observe every durable pending candidate before child +# job builds unblock (issue #188 fix 4): a truncated enumeration could hide +# the active parent whose payout transition is not yet armed. A full batch +# therefore re-queries with a doubled window until the result is provably +# untruncated. This cap bounds one enumeration pass's memory; at the cap the +# gate simply stays closed while the queued batch drains, and the submitter +# loop re-enumerates the shrinking remainder. +MAX_BLOCK_REPLAY_ENUMERATION_ROWS = MAX_BLOCK_REPLAY_PAGE_SIZE +# Issue #255. Metadata-first replay keeps at most this many small durable +# descriptors in memory; rows beyond it register their accepted-parent and +# credit-floor obligations and stay in the outbox, which a later poll +# re-reads once the window drains. A descriptor is a few hundred bytes, so +# the window is well inside the 4 MiB header/queue budget. +MAX_BLOCK_REPLAY_DESCRIPTORS_IN_MEMORY = 1024 +# Rows per metadata-first replay page: the configured replay page size is +# additionally clamped to the #255 header page cap. +BLOCK_REPLAY_HEADER_PAGE_ROWS = min(MAX_BLOCK_REPLAY_ENUMERATION_ROWS, HEADER_PAGE_MAX_ROWS) +# Minimum spacing between two janitor steps on the submitter loop. +BLOCK_CANDIDATE_BODY_JANITOR_INTERVAL_SECONDS = 5.0 +# Ancestor re-drive bookkeeping (issue #190) is keyed by block hash and +# dropped the moment the blocking transition resolves; this bound only +# guards against a pathological stream of distinct never-resolving +# ancestors leaking entries forever. Eviction is oldest-first, and every +# deferral re-inserts its ancestor's record and its child's blocking +# entry, so for a wedge the still-blocking pair is the last thing evicted. +MAX_ANCESTOR_REDRIVE_TRACKED_HASHES = 64 + + +@dataclass +class _AncestorRedriveRecord: + """Per-ancestor re-drive bookkeeping (issue #190). + + One record holds every fact the mechanism tracks about an ancestor, so + recency (the record's insertion position), the armed request, and the + exhaustion latch can never disagree the way parallel registries could. + Mutated only under ``coordinator.lock``. + """ + + # Consecutive finalization deferrals since the last armed pass. Frozen + # while a request is armed-but-unconsumed: deferrals during that window + # prove nothing the armed pass will not already act on, and counting + # them would let the next attempts arm on single deferrals right after + # consumption, exhausting the cap before the first pass's adoption had + # a chance to resolve the ancestor. + streak: int = 0 + # Re-drive passes armed against the per-ancestor cap. + attempts: int = 0 + # Armed passes the submitter actually consumed. The resolved counter + # keys off this, not ``attempts``: a transition that resolved through + # the ordinary landing tail while a request sat unconsumed is the + # mechanism standing by, not succeeding. + consumed: int = 0 + # A forced durable-replay pass is waiting for the submitter loop. + armed: bool = False + # The cap is spent; deferrals fall back to exactly the pre-#190 + # behavior with the publication-progress watchdog as the backstop. + exhausted: bool = False + + +DEFAULT_BLOCK_CANDIDATE_RETRY_INITIAL_SECONDS = 0.25 +DEFAULT_BLOCK_CANDIDATE_RETRY_MAX_SECONDS = 30.0 +# The primary accounting handoff queue must be bounded or the documented +# result-preserving spillover ordering can never engage; the overflow queue +# stays unbounded by design so an already-offered block is never converted +# back into a raw-submit retry. +DEFAULT_BLOCK_ACCOUNTING_QUEUE_DEPTH = 8 +# How many definitive-acceptance accounting tasks the lane may dispatch back +# to back while non-accepted work is waiting. The accepted lane exists so a +# decided block's accounting never queues behind stale replay (see +# _enqueue_block_accounting_task), but "never behind" must not become "stale +# replay never runs": a stale accounting task that is starved leaves its +# durable outbox row pending, its offer-time accepted-block payout-preview +# barrier armed, and its disposition lease held, which is a different bug +# rather than a fixed one. The quota therefore states the fairness bound +# outright -- a non-accepted task at the head of its lane waits behind at +# most this many accepted services, whatever the accepted arrival rate. +# +# Four, not one: one accepted tip can produce more than one accepted-evidence +# task (the winner, plus a same-height idempotent replay of it), and a quota +# of one would interleave a stale task's ~6 RPCs, two ledger writes and a +# payout-balance mutation into the middle of that acceptance tail -- the very +# interleaving the lane was added to remove. Four covers a normal tip's tail +# while still bounding stale delay at four accepted services. +BLOCK_ACCOUNTING_ACCEPTED_DISPATCH_QUOTA = 4 +# A collapsed row whose post-write cleanup failed has no durable replay +# source left, so the accounting lane is the only thing that can ever finish +# it. Draining it purely from that lane's idle branch made it hostage to the +# lane staying idle: sustained ordinary accounting traffic, or a +# continuously replenished invalid-candidate quarantine queue, starved the +# retry for as long as the traffic lasted. The lane therefore also offers +# one bounded retry after at most this many completed work items, where a +# work item is one finished accounting task from any of the handoff queues +# or one finished quarantine item. The offer runs at most one hash's +# still-owed steps and the cadence resets on every offer -- including one +# that finds nothing due -- so neither lane can starve the other. +DEFAULT_BLOCK_ACCOUNTING_CLEANUP_RETRY_WORK_ITEMS = 8 +# Issue #198. A cleanup-retry record is the only replay source a durably +# terminal row has left, so the registry never sheds one; under a systemic +# cleanup fault it therefore grows by one record per row the collapse wins. +# At the observed 312,000-candidate storm that is an unbounded in-memory +# backlog -- each record retaining its exact pending-share floor holders and +# pinning its terminal-outcome fence -- driven by a lane that retries one +# hash per pass. The bound is applied to *admission* instead: once the +# backlog holds ``_collapse_cleanup_retry_backlog_max()`` records +# (PRISM_BLOCK_CANDIDATE_CLEANUP_RETRY_BACKLOG_MAX, default +# DEFAULT_BLOCK_CANDIDATE_CLEANUP_RETRY_BACKLOG_MAX) the collapse hands no +# further rows to its fenced bulk terminalization, and every row it declines +# stays durable and pending for the ordinary per-row path. Nothing already +# terminal loses its record, its holders, or its fence. +# +# A persistently engaged backpressure would otherwise print one line per +# page or per dequeued sibling; the warning is rate limited to this interval +# while the ``backlog_deferred`` counter still moves per row. +BLOCK_CANDIDATE_CLEANUP_BACKPRESSURE_LOG_SECONDS = 60.0 +# The closed vocabulary the backpressure warning names its caller from, so +# the field can never carry a hash, a page cursor, or a free-form reason. +PRISM_BLOCK_CANDIDATE_CLEANUP_BACKPRESSURE_CALLERS = ( + # The replay-adoption page walk (#183/#196). + "replay-page", + # The dequeue-time stale sibling skip (#181 item 2). + "dequeue", +) +# The same-hash disposition guard remembers every terminal outcome so a late +# duplicate -- queued, replayed, retried, parked, or quarantined -- joins the +# decision instead of re-offering a block qbitd already answered for. That +# memory was process-lifetime historical state: one testnet4 storm left +# 312,000 entries behind, and every decided-height collapse poll copied the +# whole registry under the global lock (~124 ms at that size, growing with +# every block the process ever disposed) while unrelated share acks waited. +# +# The registry is bounded instead, and eviction is oldest-first over the +# entries that no in-memory copy still needs: every lane that can hold a +# candidate object (or its still-owed terminal work) pins its hash and is +# never evicted, so a fenced hash cannot become offerable again by being +# forgotten. That includes the instant between lanes -- a candidate the +# submitter has taken out of a holder or queue but not yet handed to its +# disposition flight is named by the dequeued-hash pin, moved in and out +# under the same lock holds that empty and refill the lanes around it. The +# bound sits far above the number of copies the process can +# hold at once -- the bounded admission queue +# (MAX_PENDING_BLOCK_CANDIDATES), one replay enumeration batch +# (MAX_BLOCK_REPLAY_ENUMERATION_ROWS), and the single-slot retry holders -- +# so ordinary operation never even reaches an eviction. +MAX_BLOCK_CANDIDATE_TERMINAL_OUTCOMES = 8192 +# One insert examines at most this many of the oldest entries beyond the +# overflow it has to clear, so trimming stays O(1) amortized under the global +# lock even when a run of pinned hashes sits at the front of the registry. +# A pinned entry the scan passes is moved behind the window rather than +# dropped, so every pass makes progress against the next one. +BLOCK_CANDIDATE_TERMINAL_OUTCOME_EVICTION_SCAN = 128 +BLOCK_SUBMITTER_WAIT_HEARTBEAT_SLICE_SECONDS = 0.25 +MAX_BLOCK_SUBMITTER_STUCK_CALL_WORKERS = 2 +BLOCK_CANDIDATE_RETRY_HEARTBEAT_SLICE_SECONDS = 0.25 +BLOCK_CANDIDATE_INTENT_SCHEMA = "qbit.prism.block-candidate-intent.v1" +PRISM_STALE_JOB_ABANDON_CLASSES = ( + "tip_moved", + "balance_stale", + "append_epoch_stale", +) +# A would-be terminal abandonment refused because the candidate's own block +# hash is (or was recently observed as) part of the active chain: qbitd +# accepted the block even though this process has not completed the accepted +# success tail (for example when the direct submitblock ack was lost in +# transport and acceptance was learned from a blockwait tip observation). +# The candidate defers and retries until the tail finalizes it as submitted. +PRISM_REJECTION_BLOCK_ACCEPT_PENDING = "accepted-pending-finalization" +# The ledger reported the candidate's row terminally disposed (reorg +# quarantine, rejection, or reversal) before the confirmation landed. That is +# a routine race around a chain reorganization: terminal for this candidate, +# benign for the pool, so it must never escalate to a coordinator shutdown +# the way an unexplained confirmation failure does. +PRISM_REJECTION_LEDGER_CONFIRMATION_SUPERSEDED = "ledger-confirmation-superseded" +# Used only by lightweight embedders that bypass dataclass/coordinator +# construction. Production instances adopt their eagerly installed state at +# service construction. Serializing the fallback prevents concurrent +# first-touch callers from ever publishing different containers for the same +# state. +_STATE_BACKFILL_LOCK = threading.Lock() + +# -- decided-height candidate collapse (#183) -------------------------- +# Bounded, fixed-label outcome keys for the collapse selector/apply. They +# are the whole metric label space: no block hash, parent hash, or job ID +# ever becomes a Prometheus label. +PRISM_BLOCK_CANDIDATE_COLLAPSE_OUTCOMES = ( + # Durable pending rows the selector examined. + "considered", + # Rows preserved because their height was never probed: the walk had + # spent MAX_BLOCK_CANDIDATE_COLLAPSE_HEIGHT_PROBES distinct heights. + "height_deferred", + # Rows that satisfied every predicate clause. + "selected", + # Selected rows whose disposition lease was held elsewhere. + "lease_skipped", + # Leased rows the immediate pre-write re-read disqualified. + "revalidation_dropped", + # Qualified rows the fenced batch write did not transition. + "write_lost", + # Rows the fenced batch write actually transitioned. + "abandoned", + # Rows preserved because a page-scope read or write failed closed. + "fail_closed", + # Won rows whose post-write cleanup raised. + "cleanup_failed", + # Deferred cleanups a later bounded retry pass finished in full. + "cleanup_recovered", + # Retry passes that ran but still left at least one step owed. + "cleanup_retry_failed", + # -- issue #198: cleanup-retry backlog backpressure ----------------------- + # Rows that satisfied (or were never asked) predicate S and were + # preserved -- left durable and pending for the per-row path -- because + # the cleanup-retry backlog was at its configured admission bound. Shared + # by the replay-page walk and the dequeue-time skip like the outcomes + # above; the warning names the caller, the counter counts rows. + "backlog_deferred", + # -- issue #181 item 2: the dequeue-time stale sibling skip ------------- + # These three name the *population* the skip judged, one dequeued + # candidate at a time, and partition it exactly: + # dequeue_considered = dequeue_skipped + dequeue_preserved. The outcomes + # above stay shared with the replay-adoption page path, because they + # describe the same machinery (one selection, one lease set, one fenced + # write, one cleanup) whichever caller drove it. + # + # Dequeued candidates predicate S was actually asked about: the ledger + # can answer a fenced batch abandonment and a pool-block probe, and the + # candidate carries a readable hash and parent. + "dequeue_considered", + # Dequeued candidates terminalized before any node offer. + "dequeue_skipped", + # Dequeued candidates handed to the ordinary offer path, for any reason + # at all -- current work, offer evidence, an unprobed height, a lighter + # occupant, a lost fenced write, or a read that failed closed. + "dequeue_preserved", +) +# Issue #181 item 3: the interval from definitive node acceptance (the +# submitblock RPC returning None on the submitter thread) to the accepted +# block's payout preview becoming visible to waiting child work. Both exits +# of _publish_accepted_block_payout_preview publish -- the atomic generation +# publication, and issue #188's fenced local-retention branch, which installs +# the compact preview and notifies waiters without installing a generation -- +# so the label set separates them rather than merging a degraded publication +# into the healthy one. +PRISM_ACCEPTED_BLOCK_PREVIEW_PUBLICATION_RESULTS = ( + "published", + "degraded", +) +# The acceptance criterion is a p95 below the 5 s child wait budget +# (DEFAULT_ACCEPTED_BLOCK_PAYOUT_PREVIEW_WAIT_SECONDS), so the boundary at +# exactly 5.0 has to exist and the approach to it has to be resolved. The +# shared PRISM_JOB_BUILD_SECONDS_BUCKETS tuple is tuned for millisecond job +# builds and steps 1.0 -> 2.5 -> 5.0 -> 10.0 across the whole decision range; +# this dedicated tuple keeps that one unwidened while still extending well +# past the budget so a saturated tail is bucketed instead of folded into +# +Inf. +PRISM_ACCEPTED_BLOCK_PREVIEW_PUBLICATION_SECONDS_BUCKETS = ( + 0.01, + 0.05, + 0.1, + 0.25, + 0.5, + 1.0, + 1.5, + 2.0, + 3.0, + 4.0, + 5.0, + 7.5, + 10.0, + 20.0, + 30.0, + 60.0, +) +# Acceptance stamps are keyed by block hash and consumed by the first +# publication observed for that hash. A hash that never reaches publication +# -- a terminal accounting failure, a withdrawn acceptance -- would otherwise +# retain its stamp forever, so the oldest entries are evicted past this cap. +# Max-block admission and the physical block rate keep the live population +# far below it; this is a leak bound, not a working set. +MAX_ACCEPTED_BLOCK_PREVIEW_PUBLICATION_STAMPS = 512 +# Issue #224: the event name of the one bounded structured line each closed +# acceptance-to-publication interval emits. It is the only place in this +# lane's output where a block hash and height appear alongside the timings; +# the phase metrics carry the fixed phase label and nothing else. +PRISM_ACCEPTED_LANDING_LOG_EVENT = "accepted_block_preview_landing" +# The terminal cleanup one won row owes, in the order the apply runs it. +# A deferred retry replays exactly the steps its own hash still owes, so +# the step names are a closed set: a retry can never repeat a step that +# already completed, and never invent one the apply does not run. +BLOCK_CANDIDATE_COLLAPSE_CLEANUP_PAYOUT_STEP = "payout-preview-withdrawal" +BLOCK_CANDIDATE_COLLAPSE_CLEANUP_FLOOR_STEP = "pending-share-floor" +BLOCK_CANDIDATE_COLLAPSE_CLEANUP_STEPS = ( + BLOCK_CANDIDATE_COLLAPSE_CLEANUP_PAYOUT_STEP, + "finalize-retry", + "retry-state", + "outstanding-and-tip-observation", + "terminal-outcome", + "abandonment-accounting", + BLOCK_CANDIDATE_COLLAPSE_CLEANUP_FLOOR_STEP, +) +# The per-row path abandons a candidate whose height was decided by another +# block as a tip-moved stale job; the bulk path must land in the same +# bounded reason/class buckets or the abandonment series splits in two. +# ``share_submission`` imports this module, so its PRISM_REJECTION_STALE_JOB +# is restated here rather than imported; a collapse test pins them equal. +PRISM_BLOCK_CANDIDATE_COLLAPSE_REASON = "stale-job" +PRISM_BLOCK_CANDIDATE_COLLAPSE_STALE_JOB_CLASS = "tip_moved" +# A storm collapses thousands of rows in one apply. Logs stay bounded: at +# most this many parent/job/reason groups are printed, each with at most +# this many sample hashes, plus one remainder line. +BLOCK_CANDIDATE_COLLAPSE_LOG_GROUPS = 8 +BLOCK_CANDIDATE_COLLAPSE_LOG_SAMPLE_HASHES = 3 +# A systemic cleanup fault would otherwise print one line and one +# traceback per row of a storm-sized page; only the first few are +# detailed and the rest are summarized by count. +BLOCK_CANDIDATE_COLLAPSE_LOG_FAILURES = 3 +# A ledger or node that cannot answer the page-scope reads fails closed on +# every poll. The counter still moves per row; the log line is rate limited +# so a persistently degraded read cannot flood the journal. +BLOCK_CANDIDATE_COLLAPSE_FAIL_CLOSED_LOG_SECONDS = 60.0 +# Chain-height probes one replay walk may spend on collapse, counted in +# *distinct heights* rather than rows. +# +# The storm this path exists for is one decided height behind thousands of +# rows, and it costs a single probe no matter how wide the page is. A page +# whose rows span many heights is the opposite shape: selection would read +# ``getblockhash`` (and then the occupant's ``getblockheader``) once per +# distinct height, sequentially, on the sole block-submitter thread. A page +# holds up to MAX_BLOCK_REPLAY_ENUMERATION_ROWS rows, so that is up to 1,024 +# heights and ~2,048 round trips; at a production RPC deadline the thread +# would sit in collapse for hours while a live solve waited behind it for +# submitblock. The bound makes the chain cost of a walk a constant -- +# at most this many ``getblockhash`` reads and at most one occupant header +# each -- independent of how many rows or heights the backlog carries. +# +# Rows at a height the walk did not probe are simply never selected. They +# stay durable, keep every piece of evidence they had, and are adopted by +# the ordinary per-row disposition path, which disposed of them exactly this +# way before collapse existed; a later walk starts with a fresh budget and +# may probe their heights. Collapse is an optimisation over that path, so +# spending less of it is always safe and never terminalizes anything. +MAX_BLOCK_CANDIDATE_COLLAPSE_HEIGHT_PROBES = 8 +# PRISM records every network difficulty in scaled integer units: the +# coordinator stamps a candidate's ``found_block.network_difficulty`` with +# ``template_artifacts.scaled_network_difficulty(template["bits"])``, which is +# ``pow_limit_target * 1_000_000 // template_target`` -- roughly the raw +# Bitcoin-style difficulty multiplied by a million. The collapse selector +# compares that stored value against the occupying block's work, so the +# occupant has to be converted into the same units from the same inputs. +# These restate the scale and the qbit powLimit rather than importing them +# from ``template_artifacts`` (or ``public_api``): both sit above this module +# in the import graph, and B1 must not grow an edge to the template/payout +# layer for two constants. ``_collapse_scaled_difficulty`` reapplies the +# formula, and a collapse test pins it equal to the production helper. +COLLAPSE_DIFFICULTY_SCALE = 1_000_000 +COLLAPSE_POW_LIMIT_BITS = "207fffff" +COLLAPSE_POW_LIMIT_TARGET = direct_stratum.target_from_compact_hex( + COLLAPSE_POW_LIMIT_BITS +) + + +def _pending_rows_accepts_cursor(pending_rows: Callable[..., Any]) -> bool: + """Report whether a ledger's pending-row reader can paginate. + + Signature introspection runs first because the alternative — probing + with the keyword and catching TypeError — spends a bounded ledger + worker and records a landing-class ledger-call sample on every startup + against a ledger that never supported cursors. The probe still runs + (and still falls back) whenever a callable cannot be introspected; this + only skips it where the answer is already knowable. + """ + try: + parameters = inspect.signature(pending_rows).parameters.values() + except (TypeError, ValueError): + return True + return any( + parameter.name == "after_cursor" + or parameter.kind is inspect.Parameter.VAR_KEYWORD + for parameter in parameters + ) + + +def _block_replay_cursor_key(after_cursor: object | None) -> str | None: + """Fold one opaque enumeration cursor into a hashable dedupe-key part. + + The cursor is a JSON-safe list, so it cannot go into a tuple key as-is. + Only identity matters here: two calls share an in-flight ledger worker + exactly when they are the same query. + """ + if after_cursor is None: + return None + return json.dumps(after_cursor, separators=(",", ":"), default=repr) + + +def _collapse_block_hash(value: object) -> str | None: + """Normalize one block hash, or report it unusable. + + Every collapse comparison is between hashes from three unrelated + sources -- a durable intent, a durable row key, and a qbitd RPC -- so + they are folded to one canonical spelling here. Anything that is not a + 64-character hex string is unusable rather than merely differently + spelled, and its row is never selected. + """ + if not isinstance(value, str): + return None + key = value.strip().lower() + if len(key) != 64: + return None + try: + int(key, 16) + except ValueError: + return None + return key + + +def _block_candidate_hash_of(candidate: object) -> str | None: + """The canonical block hash one candidate object carries, if any.""" + return _collapse_block_hash( + getattr( + getattr(candidate, "submission", None), + "block_hash_hex", + None, + ) + ) + + +def _collapse_height(value: object) -> int | None: + """Parse one block height, rejecting booleans and inexact numbers. + + ``bool`` is an ``int`` subclass, so a stray ``True`` would otherwise + parse as height 1 and compare against a real tip height. + """ + if isinstance(value, bool) or value is None: + return None + if isinstance(value, int): + return int(value) + if isinstance(value, float): + if not math.isfinite(value) or value != int(value): + return None + return int(value) + if isinstance(value, str): + try: + return int(value.strip(), 10) + except ValueError: + return None + return None + + +def _collapse_row_height(durable_row: object) -> int | None: + """The height one durable page row claims, or None when it states none. + + The single place that knows where a fetched row records its height, so + the cheap pre-selection peek in ``_collapse_superseded_block_candidates`` + and the decode in ``_superseded_candidate_row`` can never drift apart on + it. This is only a peek: it validates nothing beyond the parse, and every + fact it reads is re-read and cross-checked against the row's template by + the decode, which is what actually decides the row. + """ + if not isinstance(durable_row, dict): + return None + intent = durable_row.get("candidate") + if not isinstance(intent, Mapping): + return None + return _collapse_height(intent.get("expected_height")) + + +def _collapse_difficulty(value: object) -> int | float | None: + """Parse one stored work value, rejecting booleans and non-finite numbers. + + A NaN compares false against every threshold, so an unparsed NaN would + silently answer "not enough work" (safe) or, inverted, "enough work" + (not safe). Rejecting it outright keeps the clause decidable. + + An integral spelling is kept as an ``int`` rather than widened to + ``float``. PRISM stores ``found_block.network_difficulty`` as the scaled + *integer* ``scaled_network_difficulty`` returns, and above 2**53 a float + conversion rounds it -- often upwards, which would make a candidate look + strictly heavier than the identical work the chain reports and defeat + exactly the equal-work case clause 4b/5 has to accept. Python compares + ``int`` and ``float`` exactly, so a non-integral spelling stays usable + without dragging the integral ones through binary floating point. + """ + if isinstance(value, bool) or value is None: + return None + number: int | float + if isinstance(value, int): + number = value + elif isinstance(value, float): + number = value + elif isinstance(value, str): + text = value.strip() + try: + number = int(text, 10) + except ValueError: + try: + number = float(text) + except (OverflowError, ValueError): + return None + else: + return None + if isinstance(number, float) and not math.isfinite(number): + return None + if number <= 0: + return None + return number + + +def _collapse_scaled_difficulty(bits: object) -> int | None: + """Convert one header's compact ``bits`` to PRISM-scaled difficulty units. + + This is deliberately the same integer formula + ``template_artifacts.scaled_network_difficulty`` applies to the + template's bits when the coordinator stamps a candidate's + ``found_block.network_difficulty`` -- recomputed here from + ``direct_stratum`` rather than imported, because this module must not + grow an edge to the template/payout layer. Both sides of clause 4b/5 + therefore land on the identical integer for identical work, with no + float rounding at the equality boundary the clause turns on. + + Returns ``None`` for anything that is not an 8-character compact hex + string with a positive target, so a header missing ``bits``, or + carrying a malformed or zero one, fails its page closed instead of + being read as "no work". + """ + if not isinstance(bits, str): + return None + compact = bits.strip().lower() + if len(compact) != 8 or any(char not in "0123456789abcdef" for char in compact): + return None + try: + target = direct_stratum.target_from_compact_hex(compact) + except ValueError: + return None + if target <= 0: + return None + return max( + 1, + (COLLAPSE_POW_LIMIT_TARGET * COLLAPSE_DIFFICULTY_SCALE) // target, + ) + + +class _BlockCandidateCollapseFailedClosed(Exception): + """A page-scope chain read was unavailable or proved nothing. + + Raised, not returned, because every caller's answer is the same: this + page selects nothing and every one of its rows is preserved for the + ordinary per-row path. + """ + + +@dataclass +class _CollapsedCandidateCleanup: + """One terminal hash's outstanding cleanup, kept for a later retry. + + After the fenced batch write the durable row is terminal, so the row is + partitioned out of the replay page and no later enumeration will ever + return it: a cleanup step that failed has no durable replay source and + would otherwise leave its in-memory state -- a payout preview or its + tombstone, a pending-share floor holder, an outstanding-hash marker -- + installed for the process lifetime. This record is that replay source. + + It holds no candidate object, no durable row, and no node submission, + so a deferred cleanup can only finish tearing state down; there is + nothing here to re-adopt or re-offer with. ``shares`` carries the exact + pending-share floor holders the apply resolved, because the floor keys + holders by object identity and the queues they were read from are + drained long before a retry runs. ``shares_resolved`` is false only + when the apply aborted before it could index them, in which case the + retry re-runs the (bounded, in-memory) scan itself. + """ + + block_hash: str + steps: frozenset[str] + shares: tuple[PendingShare, ...] = () + shares_resolved: bool = True + attempts: int = 0 + delay_seconds: float = 0.0 + not_before_monotonic: float = 0.0 + # When this hash's cleanup was first deferred. Kept across merges and + # reschedules so the oldest-age gauge measures how long the row has + # owed cleanup, not how long since its last failed attempt. + deferred_monotonic: float = 0.0 + + +@dataclass(frozen=True) +class _SupersededCandidateRow: + """One durable pending row reduced to the facts predicate S reads.""" + + block_hash: str + parent_hash: str + height: int + # PRISM-scaled units (see COLLAPSE_DIFFICULTY_SCALE), as stamped by the + # coordinator. Comparable only against a chain-side work value converted + # into the same units by ``_collapse_scaled_difficulty``. + network_difficulty: int | float + job_id: str + + +@dataclass +class _CollapseHeightProbeBudget: + """Distinct chain heights a collapse walk may still read, and what it read. + + One budget is created per replay walk and shared by that walk's pages, + so the walk's chain cost is the bound whether the backlog arrives as one + page or fifty. A pass handed no budget (a direct call, or the legacy + single-page shape) gets a fresh full one, which is the same ceiling + applied to a walk of one. + + Only heights are charged. The occupant header a probed height leads to is + read at most once per probed height, so charging the height bounds both + round trips at once and keeps the accounting to the single number the + comment on MAX_BLOCK_CANDIDATE_COLLAPSE_HEIGHT_PROBES explains. + + The two caches live here, beside the counter, for the same reason the + counter does: a walk that has paid for a height must not pay for it + again on its next page. Each page builds a fresh + ``_BlockCandidateChainView`` over this one budget, and a view that + cached its reads only for itself would recharge the walk for the same + height on every page -- so the one decided height behind a storm would + exhaust the budget after eight pages and defer the entire remainder of + the backlog. ``difficulty`` is keyed by block hash, whose work is + immutable, so sharing it can only remove round trips. Sharing ``active`` + widens the window between a selection read and the write it feeds, which + is precisely the window the pre-write revalidation exists to close: that + pass builds its own budget, so its caches -- and therefore its chain + reads -- stay fresh and independent of this walk's. + """ + + remaining: int = MAX_BLOCK_CANDIDATE_COLLAPSE_HEIGHT_PROBES + # Height -> the active-chain block this walk read there. + active: dict[int, str] = field(default_factory=dict) + # Block hash -> its PRISM-scaled work, re-derived from its header bits. + difficulty: dict[str, int] = field(default_factory=dict) + + @property + def exhausted(self) -> bool: + return self.remaining <= 0 + + def spend(self) -> bool: + """Claim one height probe; False when the walk has none left.""" + if self.remaining <= 0: + return False + self.remaining -= 1 + return True + + +class _BlockCandidateChainView: + """Bounded chain reads for one collapse pass. + + The best tip is read once for the whole page and the active block at a + height once per distinct candidate height, so a 3,120-row storm at one + decided height costs a fixed handful of RPCs instead of one round trip + per row. Every miss or failure is a fail-closed page, never a guess. + + A height-diverse page cannot turn "once per distinct height" into a walk + of the node as long as the page: ``probe_budget`` caps the distinct + heights this view will ever read, and a height past the cap is reported + as unprobed rather than guessed at. + + "Once per distinct height" is a property of the *walk*, not of this + view. The caches are the budget's, so a page-two view over the same + budget already knows every height page one probed: a storm at one + decided height costs the walk one ``getblockhash`` and one + ``getblockheader`` however many pages it spans. Only the page-scope tip + reads are per view, which is what keeps each page's selection running + against a freshly read tip. + """ + + def __init__( + self, + service: "BlockCandidateService", + *, + probe_budget: _CollapseHeightProbeBudget | None = None, + ) -> None: + self._service = service + self._probe_budget = ( + probe_budget + if probe_budget is not None + else _CollapseHeightProbeBudget() + ) + self._active: dict[int, str] = self._probe_budget.active + self._difficulty: dict[str, int] = self._probe_budget.difficulty + # Per view, never on the budget: the tip is the one fact a view must + # not inherit from an earlier page, and keeping the memo here is what + # preserves "each page's selection runs against a freshly read tip" + # while letting one page read it once. Both reads were already made + # at most once per view by the page selector and the pre-write + # revalidation, so memoizing changes no existing caller's round + # trips; it exists so a caller that needs the tip *before* handing + # this view to selection -- the dequeue-time skip, which gates on + # clause 3 before it spends a durable read -- does not pay for a + # second one. + self._tip: str | None = None + self._tip_height: int | None = None + + def _call(self, method: str, params: list[object] | None = None) -> object: + coordinator = self._service._coordinator + coordinator._record_block_submitter_phase(f"replay-collapse-{method}") + try: + if params is None: + return coordinator.rpc.call(method) + return coordinator.rpc.call(method, params) + except Exception as exc: + raise _BlockCandidateCollapseFailedClosed( + f"{method} failed during candidate collapse: {exc}" + ) from exc + finally: + coordinator._record_block_submitter_phase( + f"replay-collapse-{method}:complete" + ) + + def best_tip(self) -> str: + """The chain's best tip, read once for the life of this view.""" + if self._tip is not None: + return self._tip + tip = _collapse_block_hash(self._call("getbestblockhash")) + if tip is None: + raise _BlockCandidateCollapseFailedClosed( + "best tip hash is not a block hash" + ) + self._tip = tip + return tip + + def tip_height(self) -> int: + """The best tip's height, read once for the life of this view.""" + if self._tip_height is not None: + return self._tip_height + height = _collapse_height(self._call("getblockcount")) + if height is None or height < 0: + raise _BlockCandidateCollapseFailedClosed( + "best tip height is not an integer" + ) + self._tip_height = height + return height + + def active_at(self, height: int) -> str | None: + """The active-chain block occupying ``height``, read once per height. + + ``None`` means this walk has spent its height-probe budget and the + height was never read. It is not "no occupant" and it is not a + fail-closed read: it says only that the chain was never asked, so + every caller must preserve the row rather than decide anything about + it. A height this walk has already read is free -- on this page and + on every later page of the same walk -- and stays answerable after + the budget is gone. + """ + cached = self._active.get(height) + if cached is not None: + return cached + if not self._probe_budget.spend(): + return None + active = _collapse_block_hash(self._call("getblockhash", [int(height)])) + if active is None: + raise _BlockCandidateCollapseFailedClosed( + f"active block at height {height} is not a block hash" + ) + self._active[height] = active + return active + + def cached_difficulty(self, block_hash: str) -> int | None: + """The work this walk already read for a hash, without a new call.""" + return self._difficulty.get(block_hash) + + def difficulty_of(self, block_hash: str) -> int: + """One occupying block's work, in PRISM-scaled units, once per hash. + + Read from the header's compact ``bits``, not its ``difficulty`` + field. The candidate row this value is compared against carries + ``scaled_network_difficulty(template["bits"])`` -- the raw + difficulty times COLLAPSE_DIFFICULTY_SCALE -- while + ``getblockheader.difficulty`` is the raw float. Comparing the two + directly made an equal-work occupant read as a millionth of the + candidate's work, so clause 4b/5 rejected it and a decided height + never collapsed. Recomputing from bits with the production formula + puts both sides on the same integer scale, and keeping it integral + means the equal-work boundary the clause turns on is decided + exactly rather than by float comparison. + """ + cached = self._difficulty.get(block_hash) + if cached is not None: + return cached + header = self._call("getblockheader", [block_hash]) + if not isinstance(header, dict): + raise _BlockCandidateCollapseFailedClosed( + f"header for {block_hash} is not an object" + ) + difficulty = _collapse_scaled_difficulty(header.get("bits")) + if difficulty is None: + raise _BlockCandidateCollapseFailedClosed( + f"header for {block_hash} carries no usable compact bits" + ) + self._difficulty[block_hash] = difficulty + return difficulty + + +@dataclass +class _AcceptedLandingAttribution: + """One landing's in-flight phase attribution, keyed by block hash. + + Issue #224. A landing's sub-phases are measured on the accounting and + block-work threads, while the interval they belong to is owned by the + acceptance stamp taken on the submitter thread; the block hash is the only + identity both sides share, exactly as it is for the stamp itself. + + ``phase_seconds`` accumulates only while that hash's interval is open, so + a stretch that ran before definitive acceptance -- the fallback landing + that submits inside the balance serializer reaches its reconcile and + chain-probe stretches before the offer resolves -- and a retry's stretch + after the interval already closed both land in the process-wide phase + family without ever being claimed by a landing's diagnostic. + """ + + block_height: int | None = None + phase_seconds: dict[str, float] = field(default_factory=dict) + lane_wait_recorded: bool = False + publish_started_monotonic: float | None = None + + +def _accepted_landing_log(event: str, **fields: object) -> None: + """Single-line JSON lifecycle log, the payout-artifact line's shape.""" + print( + "prism coordinator: " + + json.dumps({"event": event, **fields}, sort_keys=True), + flush=True, + ) + + +def _accepted_landing_candidate_identity( + candidate: Any, +) -> tuple[str | None, int | None]: + """This candidate's hash and height, or ``None`` when unreadable. + + Attribution is observability: a candidate shape this cannot read must + never turn the start of a landing into an exception. + """ + try: + block_hash = str(candidate.submission.block_hash_hex).lower() + except Exception: + return (None, None) + if not block_hash: + return (None, None) + try: + block_height: int | None = int(candidate.context.template["height"]) + except Exception: + block_height = None + return (block_hash, block_height) + + +@dataclass(frozen=True) +class PrismBlockCandidate: + """A block-worthy submission queued for the block-submitter thread. + + A share that met its target is acknowledged and credited on the client + thread, then queued here for the submitter to land the block off the hot + path. When the hash solved the block but missed the share target (floor + above network difficulty), credit_share_on_accept is set and the candidate + is instead submitted synchronously by handle_submit: that share is valid + only if the block lands, so its credit and the miner's accept/reject follow + the block outcome directly rather than being queued. + """ + + context: PrismJobContext + submission: direct_stratum.DirectQbitSubmission + extranonce1_hex: str + extranonce2_hex: str + pending_share: PendingShare + client: ClientState + credit_share_on_accept: bool = False + durable_replay: bool = False + # When this in-process attempt became runnable: live candidates stamp + # share-accept time, durable outbox replays stamp row-restore time. The + # block-submit histogram measures from here to submitblock's return -- + # the race-critical interval a lost block round is decided in. + landed_monotonic: float = field(default_factory=time.monotonic) + + +@dataclass +class _BlockCandidateDispositionFlight: + """One same-hash submission guard shared by its holder and waiters.""" + + # The node-offer thread acquires this guard and the accounting thread + # releases it after durable finalization. A plain Lock permits that + # deliberate ownership transfer; RLock does not. + lock: threading.Lock = field(default_factory=threading.Lock) + users: int = 0 + + +@dataclass(frozen=True) +class _BlockCandidateDispositionLease: + """A same-hash guard held across node offer and durable finalization.""" + + block_hash: str + flight: _BlockCandidateDispositionFlight + + +@dataclass(frozen=True) +class _BlockCandidateNodeSubmission: + """Result of the latency-critical qbitd fast-lane call.""" + + attempted: bool + result: object = None + error: BaseException | None = None + + +def _is_definitive_node_acceptance( + node_submission: _BlockCandidateNodeSubmission | None, +) -> bool: + """Report whether an offer is a *definitive* node acceptance. + + qbitd's ``submitblock`` returns JSON ``null`` for a block it accepted and + a rejection string ("duplicate", "inconclusive", a validation reason) for + anything else, so the only shape that names an accepted block is an + attempted offer that returned no result and raised nothing. + + All three clauses are load-bearing, and ``error is None`` is the one that + is easy to drop by accident: the transport failure path in + :meth:`_submit_block_candidate_to_node` builds + ``_BlockCandidateNodeSubmission(attempted=True, error=exc)``, which leaves + ``result`` at its ``None`` default. A two-clause test would therefore read + a *failed* offer as an acceptance -- the single worst misclassification + available here, because every consumer of this predicate treats a true + answer as "the node already has this block". + + Consumers must share this one definition rather than re-spelling it: + :meth:`_stash_retained_block_candidate_node_submission` decides whether an + offer may be replayed from memory, and the accounting handoff decides + which lane a task joins. Those two answers must never diverge. + """ + if node_submission is None: + return False + return ( + bool(node_submission.attempted) + and node_submission.error is None + and node_submission.result is None + ) + + +@dataclass +class _BlockSubmitterLedgerCall: + """One still-running direct outbox call, reused across paced retries.""" + + started_monotonic: float = field(default_factory=time.monotonic) + done: threading.Event = field(default_factory=threading.Event) + result: object = None + error: BaseException | None = None + + +@dataclass +class _BlockSubmitterRpcCall: + """One hard-deadline, single-flight submitblock transport call.""" + + started_monotonic: float = field(default_factory=time.monotonic) + done: threading.Event = field(default_factory=threading.Event) + result: object = None + error: BaseException | None = None + + +@dataclass(frozen=True) +class _BlockCandidateAccountingTask: + """A node-offered candidate awaiting serialized durable accounting.""" + + candidate: PrismBlockCandidate + node_submission: _BlockCandidateNodeSubmission + disposition_lease: _BlockCandidateDispositionLease + # Set by the accounting lane's own exception path once it has retained + # this task's candidate for retry while still holding the disposition. + # The loop's catch reads it so a candidate the lane retained is not + # retained (and counted) a second time, while a failure that fired + # before the lane could retain -- a delegate raising ahead of it -- is + # still retained there. + retained_for_retry: threading.Event = field( + default_factory=threading.Event, + compare=False, + repr=False, + ) + + +class BlockSubmitterDatabaseTimeout(TimeoutError): + """A submitter ledger phase exceeded its coordinator-side deadline.""" + + +@dataclass(frozen=True) +class BlockCandidateAttemptResult: + """Structured result of the landing callback used by retry/terminalization.""" + + accepted: bool + reason: str | None + error: str + + def retryable(self, retryable_reasons: frozenset[str]) -> bool: + return not self.accepted and ( + self.reason is None or self.reason in retryable_reasons + ) + + +@dataclass(frozen=True) +class BlockCandidateRunResult: + """Result returned after one in-memory wakeup or retry slot is consumed.""" + + ran: bool + refresh_client: Any | None = None + + +class BlockCandidateSubmitPort(Protocol): + """Land one candidate through the coordinator's submit entrypoint. + + The landing tail has two forms and only the coordinator can tell them + apart: embedders and tests replace the bound ``submit_block_candidate`` + to stand in for the node submission, and that replacement is installed on + the instance *after* this service is constructed. The port therefore + carries the whole landing call shape -- an already-created + ``node_submission`` when the caller made one, and whether the caller + already owns the same-hash disposition -- and lets the coordinator resolve + the entrypoint per call, instead of the service inspecting it through the + runtime seam. + """ + + def __call__( + self, + candidate: PrismBlockCandidate, + *, + node_submission: _BlockCandidateNodeSubmission | None = None, + disposition_held: bool = False, + ) -> bool: ... + + +@dataclass(frozen=True) +class BlockCandidatePorts: + """Callback seams back into the coordinator, resolved at use time. + + ``runtime`` is the owning coordinator; service bodies reach every + cross-owner concern (and every monkeypatch-sensitive sibling delegate) + through it at call time, per the in-tree Runtime seam convention. The + named callables preserve the historical narrow port surface for focused + embedders and the layer-original B1 ownership tests. + """ + + runtime: Any + ledger: Callable[[], Any] + stop_event: Callable[[], threading.Event] + writer_operation: Callable[[str], AbstractContextManager[object]] + submit_candidate: BlockCandidateSubmitPort + reject_terminal_prepared: Callable[[PrismBlockCandidate], None] + begin_preview: Callable[[str, int], None] + clear_preview: Callable[[str, bool], None] + share_writer: Callable[[], Any] + finish_pending_candidate: Callable[[PendingShare], None] + refresh_after_accept: Callable[[Any], None] + record_heartbeat: Callable[[str], None] + replay_entrypoint: Callable[[], int] + submit_next_entrypoint: Callable[[float | None], bool] + next_retry_delay: Callable[[str], float] + log: Callable[[str], None] + + +def _candidate_block_hex(candidate: PrismBlockCandidate) -> Any: + """Return the candidate's serialized block, never an unmaterialized one. + + ``assemble_submission`` materializes ``block_hex`` only for a hash that + solved the block, which every candidate route requires, so a real + submission always carries the bytes by the time a candidate exists. If one + ever does not, the cause is a defect in that coupling and the block is + already lost -- but an empty value written to the durable intent loses it + twice, resurrecting after every restart as a candidate that can never be + offered to the node. Fail on the client thread instead, before the intent + is written. Duck-typed embedders that retain no block bytes at all keep + the historical empty-string behaviour: the invariant is a property of the + real submission type, and the landing path already guards them separately. + + The guard deliberately does not apply to a durable replay. A replayed + candidate's intent already exists, and ``block_candidate_from_intent`` + rebuilds every one of them as a real ``DirectQbitSubmission`` -- so an + embedder's tolerated empty value comes back wearing the strict type. The + only caller that re-encodes a replayed candidate is the credit-on-accept + append, which runs *after* the block has landed; raising there would fail + a credit for a block already in the chain, which is strictly worse than + re-persisting the same empty value the row already holds. Re-persisting an + existing intent is not the write this guard exists to stop. + + A hydrated replay can carry the block as a streamed ``SpoolStringView`` + (a raw block of roughly 512 KiB or more). It is returned as is: + ``str()`` refuses it, and the codec writes its stored encoding verbatim, + byte-identical to the plain string (#255). + """ + submission = candidate.submission + block_hex = getattr(submission, "block_hex", "") + if callable(getattr(block_hex, "iter_encoded_chunks", None)): + return block_hex + block_hex = str(block_hex) + if ( + not block_hex + and not candidate.durable_replay + and isinstance(submission, direct_stratum.DirectQbitSubmission) + ): + raise ValueError("block candidate submission carries no serialized block") + return block_hex + + +def block_candidate_intent(candidate: PrismBlockCandidate) -> PreparedCandidateIntent: + """Return the immutable intent needed to resume a candidate after restart. + + Issue #255. The result is a :class:`PreparedCandidateIntent`: a mapping + with the historical keys, whose ``shares_json`` is the job's immutable + share sequence itself (a page-backed window, a daemon mirror or a plain + list) rather than a list copy, and whose body streams the v1 identity + JSON in bounded chunks. Preparation validates every field synchronously + on the calling thread -- the client thread, before any share credit -- + exactly where the historical whole-document ``json.dumps`` did, but in + bounded C calls, and it refuses what PostgreSQL's jsonb parser used to + refuse (NaN/Infinity, ``\\u0000``, unpaired surrogates). + """ + context = candidate.context + submission = candidate.submission + intent = { + "schema": BLOCK_CANDIDATE_INTENT_SCHEMA, + "block_hash_hex": str(submission.block_hash_hex).lower(), + "block_hex": _candidate_block_hex(candidate), + "coinbase_tx_hex": str(getattr(submission, "coinbase_tx_hex", "")), + "parent_hash": str(context.template["previousblockhash"]).lower(), + "expected_height": int(context.template["height"]), + "template": { + "previousblockhash": context.template["previousblockhash"], + "height": int(context.template["height"]), + "coinbasevalue": int(context.template["coinbasevalue"]), + }, + # The immutable source sequence, never ``list(...)``: the codec + # streams it (verbatim page bytes where the sequence carries them). + "shares_json": context.shares_json, + "prior_balances": context.prior_balances, + "found_block": context.found_block, + "prospective_prior_balances": getattr(context, "prospective_prior_balances", None), + "witness_merkle_leaves_hex": direct_stratum.witness_merkle_leaves_hex( + getattr(context.job, "transaction_hexes", ()) + ), + "extranonce1_hex": candidate.extranonce1_hex, + "extranonce2_hex": candidate.extranonce2_hex, + "username": context.worker.username, + "pending_share": dataclasses.asdict(candidate.pending_share), + "credit_share_on_accept": candidate.credit_share_on_accept, + "collection_only": bool(context.collection_only), + } + # Fail on the client thread before committing a share if a field carries + # a value that cannot survive the durable boundary. One bounded encode + # pass replaces the historical whole-document ``json.dumps``; the bytes + # are discarded and only the manifest (digest, chunk digests, share + # span) is kept until storage asks for the chunks. + return prepare_candidate_intent(intent) + + +def _replayable_share_sequence(shares: Any) -> Any: + """The share sequence a decoded intent adopts, without a list copy. + + A plain list or tuple is adopted as is (the durable decode owns it), and + any other sequence -- a page-backed window, a daemon mirror, a spool + sequence -- is adopted as the immutable, replayable object it already + is. Anything that is not a sequence at all fails exactly as ``list()`` + used to. + """ + if isinstance(shares, (str, bytes)) or not isinstance(shares, Sequence): + raise TypeError("block candidate intent shares_json is not a sequence") + return shares + + +def _collapse_row_from_header(row: Mapping[str, Any]) -> dict[str, Any]: + """Shape one metadata-first header row as the page row predicate S reads. + + The collapse selector (``_superseded_candidate_row``) validates a page + row's ``candidate`` facts strictly and fails its page closed on any + malformed one; the header carries exactly those facts as an explicit + projection, so they are restated here under the same keys and nothing + else. The header row itself is preserved on the result so descriptor + registration reads the same object the collapse decided on. + """ + header = row.get("header") + if not isinstance(header, Mapping): + header = {} + template = header.get("template") + found_block = header.get("found_block") + pending_share = header.get("pending_share") + return { + **row, + "candidate": { + "block_hash_hex": header.get("block_hash_hex"), + "parent_hash": header.get("parent_hash"), + "expected_height": header.get("expected_height"), + "template": ( + { + "previousblockhash": template.get("previousblockhash"), + "height": template.get("height"), + } + if isinstance(template, Mapping) + else template + ), + "found_block": ( + {"network_difficulty": found_block.get("network_difficulty")} + if isinstance(found_block, Mapping) + else found_block + ), + "pending_share": ( + {"job_id": pending_share.get("job_id", "")} + if isinstance(pending_share, Mapping) + else pending_share + ), + }, + } + + +def _dequeued_candidate_collapse_row( + candidate: PrismBlockCandidate, + *, + pool_block_exists: bool, +) -> dict[str, Any]: + """Shape one dequeued candidate as the durable page row predicate S reads. + + The dequeue-time skip (issue #181 item 2) judges an in-memory candidate, + not a fetched outbox row, so it has to present one. It deliberately does + *not* call :func:`block_candidate_intent` to do that: that function + re-serializes the whole block, recomputes every witness merkle leaf, and + round-trips the result through ``json.dumps`` to validate it -- work + proportional to the block, paid per dequeued sibling, to read four + scalars. This builds only the fields ``_superseded_candidate_row`` + actually decodes. + + Every value here is copied from the same place the durable intent copies + it from, so the row this returns and the row the outbox holds for the + same candidate carry identical facts; a regression pins that field by + field rather than trusting the restatement. No predicate input is + coerced or defaulted: a candidate whose context cannot answer one of + them raises, and the caller fails that candidate closed onto the + ordinary offer path. ``job_id`` is the one exception, because it is not + a predicate input at all -- it only groups the collapse log line. + """ + context = candidate.context + template = context.template + return { + "block_hash": str(candidate.submission.block_hash_hex).lower(), + "candidate": { + "block_hash_hex": str(candidate.submission.block_hash_hex).lower(), + "parent_hash": str(template["previousblockhash"]).lower(), + "expected_height": int(template["height"]), + "template": { + "previousblockhash": template["previousblockhash"], + "height": int(template["height"]), + }, + "found_block": context.found_block, + "pending_share": { + "job_id": getattr(candidate.pending_share, "job_id", ""), + }, + }, + # Read durably by the caller, never assumed: clause 2 is the fact + # that keeps an offered, landed candidate out of a terminal set. + "pool_block_exists": bool(pool_block_exists), + } + + +def _joined_intent_text(value: Any) -> str: + """A hydrated string fact the consumer needs whole, joined explicitly. + + Only the fixed-metadata fields are always decoded as ``str``; any other + string above the view threshold stays a streamed ``SpoolStringView``, + which refuses ``str()`` (#255). Everything else keeps ``str()``. + """ + if callable(getattr(value, "iter_text_chunks", None)): + return "".join(value.iter_text_chunks()) + return str(value) + + +def block_candidate_from_intent(intent: Mapping[str, Any]) -> PrismBlockCandidate: + """Decode and validate a durable candidate intent without side effects. + + Accepts the historical dict and the prepared/hydrated mapping alike. + ``shares_json`` is adopted as the sequence the intent carries -- a spool + sequence decodes page by page on demand -- and is never copied into a + second list (#255). + """ + if not isinstance(intent, Mapping): + raise TypeError("block candidate intent must be an object") + if intent.get("schema") != BLOCK_CANDIDATE_INTENT_SCHEMA: + raise ValueError("unsupported block candidate intent schema") + block_hash = str(intent["block_hash_hex"]).lower() + # Not fixed metadata: a username with a JSON encoding over 1 MiB is + # hydrated as a streamed view, and the worker identity needs it whole. + username = _joined_intent_text(intent["username"]) + template = dict(intent["template"]) + if str(template.get("previousblockhash", "")).lower() != str(intent["parent_hash"]).lower(): + raise ValueError("block candidate parent hash does not match template") + if int(template.get("height", -1)) != int(intent["expected_height"]): + raise ValueError("block candidate height does not match template") + submission = direct_stratum.DirectQbitSubmission( + coinbase_tx_hex=str(intent["coinbase_tx_hex"]), + coinbase_txid_preimage_hex="", + header_hex="", + block_hex=(intent["block_hex"] if callable(getattr(intent["block_hex"], "iter_encoded_chunks", None)) + else str(intent["block_hex"])), + block_hash_hex=block_hash, + block_hash_int=int(block_hash, 16), + share_pass=True, + block_pass=True, + applied_version_hex="", + ) + prospective = intent.get("prospective_prior_balances") + if isinstance(prospective, Sequence) and not isinstance(prospective, (str, bytes)): + def normalized_balance(row: Any) -> tuple[str, str, str, int]: + return str(row[0]), str(row[1]), str(row[2]), int(row[3]) + + # Preserve eager validation without retaining a second collection. + for row in prospective: + normalized_balance(row) + prospective = MappedSequence(prospective, normalized_balance) + else: + prospective = None + context = PrismJobContext( + job=SimpleNamespace( + transaction_hexes=(), + witness_merkle_leaves_hex=_replayable_share_sequence( + intent.get("witness_merkle_leaves_hex", []) + ), + ), + template=template, + shares_json=_replayable_share_sequence(intent["shares_json"]), + prior_balances=_replayable_share_sequence(intent["prior_balances"]), + found_block=dict(intent["found_block"]), + share_weight=0, + collection_only=bool(intent.get("collection_only", False)), + worker=WorkerIdentity( + username=username, + payout_address="", + worker_name=None, + script_pubkey_hex="", + p2mr_program_hex="", + ), + issued_at_ms=0, + prospective_prior_balances=prospective, + # Append-invalidation epochs are process-local counters, so a + # stamp from the process that built this candidate is meaningless + # after a restart. The negative sentinel tells the landing epoch + # fence to stand down and instead revalidate the recorded window + # against the durable ledger at its declared anchor (carry + # balances do not move on a share append, so the prior-balance + # fence alone cannot see an omitted late row). + payout_append_invalidation_epoch=-1, + ) + return PrismBlockCandidate( + context=context, + submission=submission, + extranonce1_hex=str(intent["extranonce1_hex"]), + extranonce2_hex=str(intent["extranonce2_hex"]), + pending_share=PendingShare(**dict(intent["pending_share"])), + client=SimpleNamespace(username=username), + credit_share_on_accept=bool(intent.get("credit_share_on_accept", False)), + ) + + +class BlockCandidateService: + """Own the durable replay queue, retry ordering, and submitter lifecycle.""" + + def __init__( + self, + ports: BlockCandidatePorts, + *, + candidate_queue: queue.Queue[PrismBlockCandidate] | None = None, + retry_initial_seconds: float = DEFAULT_BLOCK_CANDIDATE_RETRY_INITIAL_SECONDS, + retry_max_seconds: float = DEFAULT_BLOCK_CANDIDATE_RETRY_MAX_SECONDS, + retryable_reasons: frozenset[str] = frozenset(), + submit_seconds_buckets: tuple[float, ...] | None = None, + ) -> None: + self.ports = ports + self._coordinator = ports.runtime + self.candidate_queue = candidate_queue or queue.Queue( + maxsize=MAX_PENDING_BLOCK_CANDIDATES + ) + self.retry_initial_seconds = max(0.0, float(retry_initial_seconds)) + self.retry_max_seconds = max( + self.retry_initial_seconds, + float(retry_max_seconds), + ) + self.retryable_reasons = frozenset(retryable_reasons) + self.retry_delays: dict[str, float] = {} + self.finalize_retries: dict[str, tuple[bool, str]] = {} + self.retry_candidate: PrismBlockCandidate | None = None + self.wakeups_coalesced = 0 + self.retries = 0 + self.poisoned = 0 + self.dropped = 0 + self.abandoned_counts: dict[str, int] = {} + self.outcome = threading.local() + self._state_lock = threading.RLock() + self._backoff_started_monotonic: float | None = None + self._backoff_deadline_monotonic: float | None = None + self._backoff_delay_seconds = 0.0 + # Hashes of block candidates this process may still land (durable + # outbox pending, queued, retained for retry, or mid-disposition). + # Membership lets every tip-observation channel recognize the pool's + # own block the moment it becomes the chain tip. + self._outstanding_block_candidate_hashes: set[str] = set() + # Outstanding candidate hashes observed as the chain tip + # (hash -> monotonic stamp). qbitd only reports a candidate hash as + # its tip after accepting that block, so an entry here is acceptance + # evidence that outlives transient fork views and lost submitblock + # acks; disposition/abandon paths consult it before treating any + # instantaneous chain probe as terminal truth. + self._tip_observed_accepted_block_hashes: dict[str, float] = {} + # Durable cleanup can fail after a terminal decision and force the + # same hash through that decision again; abandonment metrics count + # candidates, not cleanup attempts. Retired by the terminal-outcome + # eviction pass, which is the one place that has already proved a + # hash unpinned by every live and cleanup-owing lane. + self._counted_block_candidate_abandonments: set[str] = set() + self._block_submit_metrics_lock = threading.Lock() + if submit_seconds_buckets is None: + submit_seconds_buckets = tuple(PRISM_JOB_BUILD_SECONDS_BUCKETS) + # Landed candidate -> submitblock-return interval. Post-submit + # bookkeeping (audit build, persistence, outbox finalize) is + # deliberately excluded; the outbox created_at/completed_at span + # already covers it. + self.block_submit_seconds_histogram: dict[str, Any] = { + "buckets": {bucket: 0 for bucket in submit_seconds_buckets}, + "sum": 0.0, + "count": 0, + } + # Issue #181 item 3. Definitive node acceptance is learned on the + # submitter thread and the preview is published on the accounting + # thread, so the start stamp has to outlive the offer: the hash is + # the only identity both sides share. A stamp of None is the + # already-observed tombstone -- it keeps a re-offer of the same hash + # from restarting an interval that has already been measured -- and + # both the stamps and the histograms live under one leaf lock that + # never nests, so an observation can be taken while the publication + # path still holds the payout-balance mutation lock. + self._accepted_block_preview_publication_lock = threading.Lock() + self.accepted_block_preview_publication_seconds_histogram: dict[ + str, dict[str, Any] + ] = { + result: { + "buckets": { + bucket: 0 + for bucket in ( + PRISM_ACCEPTED_BLOCK_PREVIEW_PUBLICATION_SECONDS_BUCKETS + ) + }, + "sum": 0.0, + "count": 0, + } + for result in PRISM_ACCEPTED_BLOCK_PREVIEW_PUBLICATION_RESULTS + } + self._accepted_block_preview_acceptance_monotonic: dict[ + str, float | None + ] = {} + # Issue #224: the landing phase attribution for the hashes whose + # intervals are open, under the same leaf lock as the stamps above and + # bounded by the same cap with the same oldest-first eviction. A record + # is popped by the publication that closes its interval and discarded + # on terminal abandonment, so like the stamps this cap is a leak bound + # rather than a working set. + self._accepted_landing_attributions: dict[ + str, _AcceptedLandingAttribution + ] = {} + self._block_replay_enumeration_owed_flag = False + # A forced enumeration that meets cleanup backpressure adopts at + # most that one bounded page, then lets the submitter drain it before + # another outbox walk. This flag prevents an eager re-entry from + # growing the otherwise-unbounded replay queue page by page. + self._block_replay_backpressure_drain_pending = False + # Targeted ancestor re-drive state (issue #190), all under + # coordinator.lock. A found block whose finalization keeps deferring + # on the same unresolved accepted-ancestor payout transition arms a + # forced durable-replay pass for that ancestor after a configured + # deferral streak; the counters feed the qbit_prism_accepted_parent_ + # redrive_* metrics. One record per ancestor plus one child-keyed + # blocking index, each insertion-ordered by recency for eviction. + self._ancestor_redrive_records: dict[str, _AncestorRedriveRecord] = {} + self._ancestor_redrive_last_blocking: dict[str, str] = {} + self.accepted_parent_redrive_attempt_count = 0 + self.accepted_parent_redrive_resolved_count = 0 + self.accepted_parent_redrive_exhausted_count = 0 + # Fixed-key selector/apply outcome counters for the decided-height + # collapse. The key set is closed so the rendered series carries no + # block hash, parent hash, or job ID in a label. + self._block_candidate_collapse_counts: dict[str, int] = { + outcome: 0 for outcome in PRISM_BLOCK_CANDIDATE_COLLAPSE_OUTCOMES + } + self._block_candidate_collapse_fail_closed_logged_monotonic: ( + float | None + ) = None + # Terminal hashes whose collapse cleanup did not complete, keyed by + # hash so the backlog is bounded by the affected rows themselves and + # a repeated failure re-registers rather than accumulating. + self._block_candidate_collapse_cleanup_retries: dict[ + str, + _CollapsedCandidateCleanup, + ] = {} + + # -- twelve historical field aliases ----------------------------------- + + @property + def _retry_block_candidate(self) -> PrismBlockCandidate | None: + """Historical attribute spelling used by the retry-merge plumbing.""" + return self.retry_candidate + + @_retry_block_candidate.setter + def _retry_block_candidate(self, value: PrismBlockCandidate | None) -> None: + self.retry_candidate = value + + # -- replayed-candidate credit floor ----------------------------------- + + def adopt_replayed_candidate(self, candidate: PrismBlockCandidate) -> None: + """Adopt a decoded credit-bearing candidate's snapshot-floor holder. + + A below-target candidate can credit its older accepted stamp after + durable replay. Registering the reconstructed PendingShare on the S3 + pending floor before the candidate becomes visible to job issuance + keeps startup prewarm anchored below that stamp. + """ + if candidate.credit_share_on_accept: + self.ports.share_writer().adopt_pending_share(candidate.pending_share) + + def _release_dropped_duplicate_candidate_floor( + self, + candidate: PrismBlockCandidate, + ) -> None: + """Release a duplicate-dropped credit candidate's floor holder. + + Same-hash duplicates are distinct objects (each durable-replay decode + adopts a freshly reconstructed PendingShare; each live stamp registers + its own), and the pending floor keys holders by object identity, so + the object whose disposition actually lands releases only its own + holder. Dropping a credit-bearing duplicate without this release + pins the job/payout snapshot anchor below its stamp until restart. + Non-credit candidates are exempt: their holder belongs to the + share-append commit path, which may still be in flight. + """ + if getattr(candidate, "credit_share_on_accept", False): + self._coordinator._finish_pending_share_candidate( + candidate.pending_share + ) + + # -- queue admission --------------------------------------------------- + + def enqueue(self, candidate: PrismBlockCandidate) -> bool: + # Outstanding from admission (even when the wakeup coalesces below: + # the durable outbox row keeps the candidate replayable), so a tip + # observation can register acceptance before the submitter drains it. + self._register_outstanding_block_candidate( + str(candidate.submission.block_hash_hex) + ) + queue_obj = self.candidate_queue + if queue_obj is None: + queue_obj = queue.Queue(maxsize=MAX_PENDING_BLOCK_CANDIDATES) + self.candidate_queue = queue_obj + try: + queue_obj.put_nowait(candidate) + return True + except queue.Full: + # The candidate is already durable. A full queue merely coalesces + # this wakeup; the submitter re-reads pending outbox rows whenever + # it drains the queue, so no candidate is discarded. + with self._coordinator.lock: + self.wakeups_coalesced = int(self.wakeups_coalesced) + 1 + print( + "prism coordinator: block candidate wakeup coalesced " + f"hash={candidate.submission.block_hash_hex} (submitter queue full)", + flush=True, + ) + return False + + # -- startup enumeration gate (#120) ----------------------------------- + + def _note_block_replay_enumeration_owed(self) -> None: + """Mark that pending durable candidates have not been enumerated yet.""" + with self._coordinator.lock: + self._block_replay_enumeration_owed_flag = True + + def _clear_block_replay_enumeration_owed(self) -> None: + with self._coordinator.lock: + self._block_replay_enumeration_owed_flag = False + + def _block_replay_enumeration_owed(self) -> bool: + with self._coordinator.lock: + return bool(self._block_replay_enumeration_owed_flag) + + def _run_startup_block_candidate_replay(self) -> bool: + """Run best-effort pre-accept replay without dying on a slow ledger.""" + # Until the durable outbox has been enumerated once, this process + # cannot know whether a pending accepted candidate exists. Child job + # builds fail closed on that uncertainty instead of snapshotting a + # payout base that may omit a pending parent's carry (issue #188); + # replay_pending_block_candidates clears the flag on success. + self._coordinator._note_block_replay_enumeration_owed() + try: + return self._coordinator._run_startup_writer_replay( + self._coordinator.replay_pending_block_candidates + ) + except TimeoutError: + print( + "prism coordinator: startup block candidate replay timed out " + "phase=replay-outbox-query " + f"timeout={self._coordinator._block_landing_db_timeout():g}s; " + "continuing startup; block submitter loop will retry and " + "job builds stay blocked until pending candidates are known", + flush=True, + ) + return True + + # -- durable replay and quarantine ------------------------------------- + + def _ensure_block_replay_state(self) -> None: + """Backfill replay/maintenance queues for lightweight coordinators.""" + with _STATE_BACKFILL_LOCK: + if not hasattr(self, "_block_replay_candidate_queue"): + self._block_replay_candidate_queue = queue.Queue() + if not hasattr(self, "_block_replay_inflight_hashes"): + self._block_replay_inflight_hashes: set[str] = set() + if not hasattr(self, "_block_quarantine_queue"): + self._block_quarantine_queue = queue.Queue() + if not hasattr(self, "_block_quarantine_hashes"): + self._block_quarantine_hashes: set[str] = set() + if not hasattr(self, "_block_replay_backpressure_drain_pending"): + self._block_replay_backpressure_drain_pending = False + + def _enqueue_replayed_block_candidate( + self, + candidate: PrismBlockCandidate, + ) -> bool: + """Queue one durable replay behind live solves, once per process.""" + self._ensure_block_replay_state() + block_hash = str(candidate.submission.block_hash_hex).lower() + with self._coordinator.lock: + duplicate = ( + block_hash in self._block_replay_inflight_hashes + or block_hash + in getattr(self, "_block_candidate_terminal_outcomes", {}) + # A hash a single-slot retry holder still carries has a live + # in-memory copy whose durable row is deliberately still + # pending. Before the #190 re-drive, the retry short-circuit + # kept enumeration from ever seeing such a row mid-retention; + # a forced pass reaches it, and adopting it would install a + # second in-process copy (and re-arm its payout barrier) + # under the holder's. Skipping is safe: the row stays durable + # and pending, and a later pass re-reads it once the holder + # lets go. Only the holders qualify -- the wider lane + # registries (the outstanding set above all) name every + # admitted hash, and skipping those would break the + # queue-overflow recovery this enumeration exists for. + or block_hash in self._held_block_candidate_retry_hashes() + ) + if not duplicate: + self._block_replay_inflight_hashes.add(block_hash) + if duplicate: + # The instance decode already adopted this object's credit floor + # holder; the earlier same-hash flight owns the disposition and + # releases only its own holder. + self._release_dropped_duplicate_candidate_floor(candidate) + return False + try: + # This is an in-memory condition update, not accounting. Install + # it before making the candidate visible to the raw lane so + # startup prewarm cannot build a child from the old payout base + # after qbitd accepts but before the RPC response returns. + self._coordinator._begin_accepted_block_payout_preview( + block_hash, + block_height=int(candidate.context.template["height"]), + ) + self._block_replay_candidate_queue.put_nowait(candidate) + except BaseException: + with self._coordinator.lock: + self._block_replay_inflight_hashes.discard(block_hash) + raise + return True + + def _queue_invalid_block_candidate_for_quarantine( + self, + block_hash: str, + error: str, + *, + pending_share: PendingShare | None = None, + ) -> None: + """Move malformed-row cleanup off the node-offer lane.""" + if not block_hash: + return + self._ensure_block_replay_state() + key = block_hash.lower() + with self._coordinator.lock: + if key in self._block_quarantine_hashes: + return + self._block_quarantine_hashes.add(key) + self._block_quarantine_queue.put_nowait((key, error, pending_share)) + + def _adopt_durable_block_candidate_rows( + self, + durable_rows: list[Any], + ) -> int: + """Decode and queue one fetched batch, quarantining malformed rows. + + Shared by both enumeration shapes (keyset pages and the legacy + widening window) so a fallback pass restores rows through exactly + the same decode, dedupe, and poison path. + """ + queued = 0 + self._coordinator._record_block_submitter_phase("replay-restore") + for durable_row in durable_rows: + durable_block_hash = "" + # Published to poison cleanup only after its durable credit + # holder was adopted successfully (the instance decode adopts + # the credit floor before the candidate is visible anywhere). + decoded_candidate: PrismBlockCandidate | None = None + try: + if not isinstance(durable_row, dict): + raise ValueError("durable block candidate row is not an object") + durable_block_hash = str(durable_row["block_hash"]).lower() + if "header" in durable_row: + # Metadata-first row (#255). A ledger whose bodies live + # in storage registers a small descriptor now (its + # accepted-parent and credit-floor obligations included) + # and hydrates the body only when the descriptor is + # dequeued; an in-memory ledger hands the intent back + # immediately and takes the classic decode below. + if self._replay_hydration_deferred(): + if self._register_durable_block_candidate_header_row( + durable_row + ): + queued += 1 + if self._block_replay_backpressure_drain_pending: + break + continue + intent = self._coordinator.ledger.hydrate_block_candidate_intent( + durable_row + ) + else: + intent = durable_row["candidate"] + if not isinstance(intent, Mapping): + raise ValueError("durable block candidate intent is not an object") + intent_block_hash = str(intent.get("block_hash_hex", "")).lower() + if not durable_block_hash or intent_block_hash != durable_block_hash: + raise ValueError("durable block candidate row key does not match intent") + decoded_candidate = dataclass_replace( + self._coordinator.block_candidate_from_intent(intent), + durable_replay=True, + ) + # Durable acceptance-state reads stay in accounting. The + # separate replay queue keeps these recovered rows behind any + # live solve while still exposing the whole batch to qbitd. + if self._coordinator._enqueue_replayed_block_candidate( + decoded_candidate + ): + queued += 1 + except Exception: + print("prism coordinator: invalid durable block candidate intent", flush=True) + traceback.print_exc() + self._coordinator._queue_invalid_block_candidate_for_quarantine( + durable_block_hash, + "invalid durable candidate intent", + pending_share=( + decoded_candidate.pending_share + if decoded_candidate is not None + and decoded_candidate.credit_share_on_accept + else None + ), + ) + return queued + + # -- metadata-first descriptors and bounded hydration (#255) ----------- + + def _replay_hydration_deferred(self) -> bool: + """Whether this ledger's bodies must be hydrated at dequeue time.""" + return bool( + getattr(self._coordinator.ledger, "candidate_hydration_deferred", False) + ) + + def _ensure_replay_descriptor_state(self) -> None: + with _STATE_BACKFILL_LOCK: + if not hasattr(self, "_block_replay_floor_holders"): + self._block_replay_floor_holders: dict[str, PendingShare] = {} + if not hasattr(self, "_block_replay_descriptor_overflow_count"): + self._block_replay_descriptor_overflow_count = 0 + + def _durable_candidate_descriptor( + self, + row: Mapping[str, Any], + ) -> DurableCandidateDescriptor: + """Strictly type one header row; any malformed fact raises. + + The same facts the collapse selector validates, plus the ones + registration and hydration need: a decodable ``PendingShare``, the + credit flag, and for a chunked body a sealed reference whose digest + agrees with the row. A row that fails here enters the existing + quarantine path exactly as a malformed intent did. + """ + block_hash = _collapse_block_hash(row.get("block_hash")) + header = row.get("header") + if block_hash is None or not isinstance(header, Mapping): + raise ValueError("durable block candidate header is not an object") + if _collapse_block_hash(header.get("block_hash_hex")) != block_hash: + raise ValueError("durable block candidate row key does not match its header") + parent_hash = _collapse_block_hash(header.get("parent_hash")) + height = _collapse_height(header.get("expected_height")) + if parent_hash is None or height is None or height < 0: + raise ValueError( + "durable block candidate header carries no usable parent or height" + ) + template = header.get("template") + if not isinstance(template, Mapping): + raise ValueError("durable block candidate header carries no template") + if _collapse_block_hash(template.get("previousblockhash")) != parent_hash: + raise ValueError("durable block candidate header template parent disagrees") + if _collapse_height(template.get("height")) != height: + raise ValueError("durable block candidate header template height disagrees") + pending_share = header.get("pending_share") + if not isinstance(pending_share, Mapping): + raise ValueError("durable block candidate header carries no pending share") + # Validates the field set exactly as the historical decode did. + PendingShare(**dict(pending_share)) + credit = header.get("credit_share_on_accept") + if not isinstance(credit, bool): + raise ValueError("durable block candidate header carries no credit flag") + # An oversized header replaces an over-bound fact with None and defers + # it to the body (``replay_header_from_fields``); a long worker name + # does this to ``username`` and ``pending_share.share_id``. That + # placeholder is not corruption: hydration reads the real value and + # the identity digest verifies it. + oversized = header.get("oversized") is True + username = header.get("username") + if not isinstance(username, str) and not (oversized and username is None): + raise ValueError("durable block candidate header carries no username") + storage_version = int(row.get("storage_version") or 1) + candidate_sha256 = str(row.get("candidate_sha256") or "") + body: CandidateBodyRef | None = None + if storage_version == 2: + body_row = row.get("body") + if not isinstance(body_row, Mapping): + raise ValueError("durable block candidate row carries no body reference") + if body_row.get("state") != "sealed": + raise ValueError("durable block candidate body is not sealed") + body = CandidateBodyRef.from_row(body_row) + if body.candidate_sha256 != candidate_sha256: + raise ValueError("durable block candidate body digest disagrees with its row") + elif storage_version != 1: + raise ValueError(f"unsupported candidate storage version {storage_version}") + return DurableCandidateDescriptor( + block_hash=block_hash, + block_height=int(height), + parent_hash=parent_hash, + candidate_sha256=candidate_sha256, + storage_version=storage_version, + credit_share_on_accept=credit, + collection_only=bool(header.get("collection_only", False)), + username=username, + pending_share=dict(pending_share), + accepted_at_present=bool(header.get("accepted_at_present")), + accepted_at_ms=pending_share.get("accepted_at_ms"), + body=body, + cursor=row.get("cursor"), + row=dict(row), + header_oversized=oversized, + ) + + def _register_durable_block_candidate_header_row( + self, + row: Mapping[str, Any], + ) -> bool: + return self._register_replayed_block_candidate_descriptor( + self._durable_candidate_descriptor(row) + ) + + def _adopt_replay_descriptor_floor( + self, + descriptor: DurableCandidateDescriptor, + ) -> PendingShare: + """Hold the credit-bearing stamp's snapshot floor for one hash. + + One holder per hash for as long as the row is pending and owned + here; a descriptor re-registered after an overflow or a paced retry + reuses it rather than stacking a second holder under the first. + """ + self._ensure_replay_descriptor_state() + with self._coordinator.lock: + existing = self._block_replay_floor_holders.get(descriptor.block_hash) + if existing is not None: + return existing + holder = PendingShare(**descriptor.pending_share) + self._block_replay_floor_holders[descriptor.block_hash] = holder + self.ports.share_writer().adopt_pending_share(holder) + return holder + + def _take_replay_floor_holder(self, block_hash: str) -> PendingShare | None: + """Hand the holder to the object that will release it; no release.""" + self._ensure_replay_descriptor_state() + with self._coordinator.lock: + return self._block_replay_floor_holders.pop(block_hash, None) + + def _release_replay_floor_holder(self, block_hash: str) -> None: + holder = self._take_replay_floor_holder(block_hash) + if holder is not None: + self._coordinator._finish_pending_share_candidate(holder) + + def _register_replayed_block_candidate_descriptor( + self, + descriptor: DurableCandidateDescriptor, + ) -> bool: + """Register one durable row's obligations and queue its descriptor. + + Mirrors ``_enqueue_replayed_block_candidate`` for a row whose body + has not been read: the same duplicate rules, the same payout-preview + registration before the row is visible to the submitter, and the + credit floor adopted from the header's pending share. When the + in-memory window is full enumeration pauses before adopting any + further obligations. The enumeration-owed gate protects job builds + until the queue drains and the remaining durable rows are adopted. + """ + self._ensure_block_replay_state() + self._ensure_replay_descriptor_state() + block_hash = descriptor.block_hash + overflow = False + with self._coordinator.lock: + duplicate = ( + block_hash in self._block_replay_inflight_hashes + or block_hash in getattr(self, "_block_candidate_terminal_outcomes", {}) + or block_hash in self._held_block_candidate_retry_hashes() + ) + if not duplicate: + overflow = ( + self._block_replay_candidate_queue.qsize() + >= MAX_BLOCK_REPLAY_DESCRIPTORS_IN_MEMORY + ) + if not overflow: + self._block_replay_inflight_hashes.add(block_hash) + if duplicate: + return False + if overflow: + with self._coordinator.lock: + self._block_replay_descriptor_overflow_count += 1 + self._block_replay_backpressure_drain_pending = True + self._coordinator._note_block_replay_enumeration_owed() + return False + try: + if descriptor.credit_share_on_accept: + self._adopt_replay_descriptor_floor(descriptor) + self._coordinator._begin_accepted_block_payout_preview( + block_hash, + block_height=descriptor.block_height, + ) + self._block_replay_candidate_queue.put_nowait(descriptor) + except BaseException: + with self._coordinator.lock: + self._block_replay_inflight_hashes.discard(block_hash) + raise + return True + + @staticmethod + def _replay_descriptor_ready(descriptor: DurableCandidateDescriptor) -> bool: + return time.monotonic() >= float(descriptor.not_before_monotonic) + + def _hydrate_replay_descriptor( + self, + descriptor: DurableCandidateDescriptor, + ) -> PrismBlockCandidate | None: + """Read one descriptor's body and decode it, off every lock. + + At most one hydration runs at a time (this is the submitter thread), + through the bounded ledger call so a wedged read stays registered + under a key that names the body rather than reusing another page's + call. Outcomes: a candidate; ``None`` after the descriptor was + dropped (its row is no longer pending), quarantined (malformed + durable content) or requeued for a paced retry (transport failure, + spool backpressure, deadline). + """ + coordinator = self._coordinator + block_hash = descriptor.block_hash + coordinator._record_block_submitter_phase("replay-hydrate") + hydrate = getattr(coordinator.ledger, "hydrate_block_candidate_intent", None) + intent = None + transferred = False + try: + if not callable(hydrate): + raise CandidateBodyUnavailable("ledger cannot hydrate durable candidates") + intent = coordinator._run_block_submitter_ledger_call( + ("replay-hydrate", block_hash, descriptor.body_key), + "replay-hydrate", + lambda: hydrate( + descriptor.row, + cancelled=coordinator.stop_event.is_set, + ), + timeout_seconds=coordinator._block_landing_db_timeout(), + call_class="landing", + ) + candidate = block_candidate_from_intent(intent) + if str(candidate.submission.block_hash_hex).lower() != block_hash: + raise ValueError("hydrated block candidate does not match its row") + if candidate.credit_share_on_accept: + holder = self._take_replay_floor_holder(block_hash) + if holder is None: + holder = self._adopt_replay_descriptor_floor(descriptor) + self._take_replay_floor_holder(block_hash) + body_share = candidate.pending_share + if body_share is not None and body_share != holder: + # The header's copy held the floor until the body was + # read, but an oversized header carries None + # placeholders. The digest-verified body is + # authoritative: adopt it before releasing the stand-in + # so the floor never lapses. + self.ports.share_writer().adopt_pending_share(body_share) + self._coordinator._finish_pending_share_candidate(holder) + holder = body_share + candidate = dataclass_replace(candidate, pending_share=holder) + candidate = dataclass_replace(candidate, durable_replay=True) + transferred = True + return candidate + except ShutdownInProgress: + self._requeue_replay_descriptor(descriptor) + raise + except CandidateBodyUnavailable as exc: + print( + "prism coordinator: durable block candidate is no longer pending; " + f"dropping its replay descriptor hash={block_hash} ({exc})", + flush=True, + ) + self._drop_replay_descriptor(descriptor) + return None + except ( + CandidateBodyIntegrityError, + CandidateCodecError, + ValueError, + TypeError, + KeyError, + ): + print("prism coordinator: invalid durable block candidate body", flush=True) + traceback.print_exc() + self._quarantine_replay_descriptor(descriptor) + return None + except Exception: + print( + "prism coordinator: durable block candidate hydration failed " + f"hash={block_hash}; the row stays pending and is retried", + flush=True, + ) + traceback.print_exc() + self._requeue_replay_descriptor(descriptor) + return None + + finally: + if not transferred and isinstance(intent, PreparedCandidateIntent): + intent.release() + + def _drop_replay_descriptor(self, descriptor: DurableCandidateDescriptor) -> None: + block_hash = descriptor.block_hash + with self._coordinator.lock: + self._block_replay_inflight_hashes.discard(block_hash) + self._release_replay_floor_holder(block_hash) + self._coordinator._clear_accepted_block_payout_preview(block_hash) + + def _quarantine_replay_descriptor( + self, + descriptor: DurableCandidateDescriptor, + ) -> None: + block_hash = descriptor.block_hash + with self._coordinator.lock: + self._block_replay_inflight_hashes.discard(block_hash) + # The quarantine step releases the holder after the row is terminal. + holder = self._take_replay_floor_holder(block_hash) + self._coordinator._queue_invalid_block_candidate_for_quarantine( + block_hash, + "invalid durable candidate intent", + pending_share=holder, + ) + + def _requeue_replay_descriptor(self, descriptor: DurableCandidateDescriptor) -> None: + """Park the descriptor at the back of the window with backoff.""" + attempts = int(descriptor.attempts) + 1 + delay = min( + float(self.retry_initial_seconds) * (2 ** min(attempts, 16)), + float(self.retry_max_seconds), + ) + self._block_replay_candidate_queue.put_nowait( + dataclass_replace( + descriptor, + attempts=attempts, + not_before_monotonic=time.monotonic() + delay, + ) + ) + + def replay_descriptor_snapshot(self) -> dict[str, int]: + """Bounded gauges for the metadata-first replay window.""" + self._ensure_block_replay_state() + self._ensure_replay_descriptor_state() + with self._coordinator.lock: + return { + "queued": int(self._block_replay_candidate_queue.qsize()), + "floor_holders": len(self._block_replay_floor_holders), + "overflow": int(self._block_replay_descriptor_overflow_count), + } + + def _run_block_candidate_body_janitor_step(self) -> None: + """One bounded chunk-page reclaim of a retired body, rate limited. + + Separate from every terminalization statement and never under + ``coordinator.lock``: the fenced terminal write only detaches and + retires a manifest, and this step drains its chunks a page at a + time. A failure is logged and the loop moves on. + """ + coordinator = self._coordinator + reap = getattr(coordinator.ledger, "reap_retired_candidate_bodies", None) + if not callable(reap): + return + now = time.monotonic() + last = float(getattr(self, "_block_candidate_body_janitor_last_monotonic", 0.0)) + if now - last < BLOCK_CANDIDATE_BODY_JANITOR_INTERVAL_SECONDS: + return + self._block_candidate_body_janitor_last_monotonic = now + try: + outcome = coordinator._run_block_submitter_ledger_call( + ("candidate-body-janitor",), + "candidate-body-janitor", + lambda: reap(), + ) + except ShutdownInProgress: + raise + except Exception: + print("prism coordinator: candidate body janitor step failed", flush=True) + traceback.print_exc() + return + if isinstance(outcome, dict) and ( + outcome.get("deleted_chunks") + or outcome.get("deleted_pages") + or outcome.get("deleted_spans") + or outcome.get("deleted_bodies") + or outcome.get("pending") + ): + # A retired body still has parts (or a manifest) to reclaim on + # a later step: let the next loop iteration step again. + self._block_candidate_body_janitor_last_monotonic = 0.0 + if any( + outcome.get(key) + for key in ("deleted_chunks", "deleted_pages", "deleted_spans", "deleted_bodies") + ): + print( + "prism coordinator: candidate body janitor reclaimed " + f"chunks={outcome.get('deleted_chunks', 0)} " + f"pages={outcome.get('deleted_pages', 0)} " + f"spans={outcome.get('deleted_spans', 0)} " + f"bodies={outcome.get('deleted_bodies', 0)} " + f"body_id={outcome.get('body_id')}", + flush=True, + ) + + # -- decided-height collapse (#183) ------------------------------------ + + def _ensure_block_candidate_collapse_state(self) -> None: + """Backfill the collapse counters and cleanup-retry registry.""" + if ( + hasattr(self, "_block_candidate_collapse_counts") + and hasattr(self, "_block_candidate_collapse_cleanup_retries") + and hasattr(self, "_block_candidate_collapse_cleanup_inflight") + and hasattr(self, "_block_candidate_cleanup_backpressure_engagements") + ): + return + with _STATE_BACKFILL_LOCK: + if not hasattr(self, "_block_candidate_collapse_counts"): + self._block_candidate_collapse_counts: dict[str, int] = { + outcome: 0 + for outcome in PRISM_BLOCK_CANDIDATE_COLLAPSE_OUTCOMES + } + if not hasattr(self, "_block_candidate_collapse_cleanup_retries"): + self._block_candidate_collapse_cleanup_retries: dict[ + str, + _CollapsedCandidateCleanup, + ] = {} + if not hasattr(self, "_block_candidate_collapse_cleanup_inflight"): + # A retry moves its record here for the duration of the + # attempt. It must remain part of both the admission depth + # and the terminal-outcome pin set while the accounting lane + # is running cleanup outside coordinator.lock. + self._block_candidate_collapse_cleanup_inflight: dict[ + str, + _CollapsedCandidateCleanup, + ] = {} + if not hasattr(self, "_block_candidate_cleanup_backpressure_engagements"): + # Occasions on which the admission bound withheld at least + # one row from bulk terminalization, and when the last + # bounded warning was printed. Both under coordinator.lock. + self._block_candidate_cleanup_backpressure_engagements = 0 + self._block_candidate_cleanup_backpressure_logged_monotonic: ( + float | None + ) = None + + def _record_block_candidate_collapse( + self, + outcome: str, + count: int = 1, + ) -> None: + if outcome not in PRISM_BLOCK_CANDIDATE_COLLAPSE_OUTCOMES: + raise ValueError( + f"unknown block candidate collapse outcome: {outcome}" + ) + if count <= 0: + return + self._ensure_block_candidate_collapse_state() + with self._coordinator.lock: + counts = self._block_candidate_collapse_counts + counts[outcome] = int(counts.get(outcome, 0)) + int(count) + + def block_candidate_collapse_snapshot(self) -> dict[str, int]: + """Copied fixed-key collapse counters for metrics rendering. + + The key set is closed and carries no hash, parent, or job ID, so the + renderer can label by outcome without any unbounded label space. + """ + self._ensure_block_candidate_collapse_state() + with self._coordinator.lock: + counts = dict(self._block_candidate_collapse_counts) + return { + outcome: int(counts.get(outcome, 0)) + for outcome in PRISM_BLOCK_CANDIDATE_COLLAPSE_OUTCOMES + } + + def _block_candidate_collapse_evidence( + self, + hashes: Iterable[object], + *, + ignore_leases: frozenset[str] = frozenset(), + ) -> frozenset[str]: + """Which of ``hashes`` some evidence says were offered to qbitd, or are being. + + Answers membership for the hashes the caller is actually deciding -- + one durable page during selection, the leased subset during + revalidation -- instead of materializing the union of every registry. + The registries this reads are not all page-sized: + ``_block_candidate_terminal_outcomes`` is historical state that grows + with every block the process ever disposed (312,000 entries after one + testnet4 storm), and copying it per poll cost ~124 ms with + ``coordinator.lock`` held against every unrelated share ack. Probing + the page's own hashes keeps the cost O(page * registries) and makes + it independent of how much disposition history has accumulated. + + Deliberately excludes ``_outstanding_block_candidate_hashes`` and + ``_block_replay_inflight_hashes``. Both mean "this process still owns + eventual disposition", not "a node offer happened": live admission + marks a candidate outstanding before it even reaches the bounded + queue, and replay adoption marks every restored row inflight, so + folding either one in excludes an entire storm and makes the whole + selector a safe no-op. ``attempt_count`` is excluded for the mirror + reason: it is stamped only *after* the node offer, so at the instant + that matters it proves nothing. + + ``ignore_leases`` lets the pre-write revalidation re-read this + evidence without the apply's own disposition leases disqualifying + the very rows they were claimed for. + + Every registry read here is keyed by the canonical lowercase hash the + rest of the submitter compares on -- the same spelling + ``_collapse_block_hash`` folds a page row to -- which is what lets a + membership probe replace the old lowercasing walk without changing + which hashes count as evidence. + """ + coordinator = self._coordinator + coordinator._ensure_job_cache_state() + self._ensure_block_candidate_disposition_state() + probes: set[str] = set() + for value in hashes: + key = _collapse_block_hash(value) + if key is not None: + probes.add(key) + if not probes: + return frozenset() + evidence: set[str] = set() + with self._block_submitter_lock( + self._block_candidate_disposition_registry_lock, + "candidate-disposition-registry", + ): + flights = self._block_candidate_disposition_flights + evidence.update( + key + for key in probes + if key in flights and key not in ignore_leases + ) + with coordinator.lock: + for holder in ( + self.retry_candidate, + getattr(self, "_block_accounting_deferred_retry_candidate", None), + ): + if holder is None: + continue + held = _collapse_block_hash( + getattr( + getattr(holder, "submission", None), + "block_hash_hex", + None, + ) + ) + if held is not None and held in probes: + evidence.add(held) + for registry in ( + self._block_disposition_waiting_retries, + self.finalize_retries, + getattr(self, "_block_candidate_retained_node_submissions", None), + self._tip_observed_accepted_block_hashes, + getattr(coordinator, "_accounted_accepted_block_hashes", None), + self._block_candidate_terminal_outcomes, + ): + if not registry: + continue + evidence.update(key for key in probes if key in registry) + return frozenset(evidence) + + def _collapse_pool_block_exists(self, durable_row: object) -> bool: + """Read one row's durable pool-block fact, or fail the page closed. + + A missing key is not "no pool block": it is a page that cannot + answer the one fact keeping an offered, landed candidate out of a + terminal set. Reading it as false would abandon exactly the rows + that must never be abandoned, so the whole page fails closed. The + fenced batch write re-asks under the writer fence regardless. + """ + if ( + not isinstance(durable_row, dict) + or "pool_block_exists" not in durable_row + ): + raise _BlockCandidateCollapseFailedClosed( + "durable page row carries no pool block existence fact" + ) + exists = durable_row["pool_block_exists"] + if not isinstance(exists, bool): + raise _BlockCandidateCollapseFailedClosed( + "durable page row pool block existence is not a boolean" + ) + return exists + + def _superseded_candidate_row( + self, + durable_row: object, + ) -> _SupersededCandidateRow: + """Reduce one page row to S's inputs, or fail its page closed. + + The preserved page still reaches ordinary decode/quarantine, but no + sibling may be terminalized from a snapshot containing a malformed + candidate fact. + """ + if not isinstance(durable_row, dict): + raise _BlockCandidateCollapseFailedClosed( + "durable candidate row is not an object" + ) + block_hash = _collapse_block_hash(durable_row.get("block_hash")) + intent = durable_row.get("candidate") + if block_hash is None or not isinstance(intent, Mapping): + raise _BlockCandidateCollapseFailedClosed( + "durable candidate row carries no usable hash or intent" + ) + if _collapse_block_hash(intent.get("block_hash_hex")) != block_hash: + raise _BlockCandidateCollapseFailedClosed( + "durable candidate row key does not match its intent" + ) + parent_hash = _collapse_block_hash(intent.get("parent_hash")) + height = _collapse_row_height(durable_row) + if parent_hash is None or height is None or height < 0: + raise _BlockCandidateCollapseFailedClosed( + "durable candidate intent carries no usable parent or height" + ) + template = intent.get("template") + if not isinstance(template, dict): + raise _BlockCandidateCollapseFailedClosed( + "durable candidate intent carries no template" + ) + # The decode path refuses an intent whose template disagrees with + # its own parent/height fields. Reading either half of a + # disagreeing pair here would compare the chain against a fact the + # candidate does not actually carry. + if _collapse_block_hash(template.get("previousblockhash")) != parent_hash: + raise _BlockCandidateCollapseFailedClosed( + "durable candidate template parent disagrees with its intent" + ) + if _collapse_height(template.get("height")) != height: + raise _BlockCandidateCollapseFailedClosed( + "durable candidate template height disagrees with its intent" + ) + found_block = intent.get("found_block") + if not isinstance(found_block, dict): + raise _BlockCandidateCollapseFailedClosed( + "durable candidate intent carries no found-block facts" + ) + # PRISM-scaled units; see COLLAPSE_DIFFICULTY_SCALE and + # _BlockCandidateChainView.difficulty_of for the chain-side half of + # the comparison this feeds. + network_difficulty = _collapse_difficulty( + found_block.get("network_difficulty") + ) + if network_difficulty is None: + raise _BlockCandidateCollapseFailedClosed( + "durable candidate intent carries no usable network difficulty" + ) + pending_share = intent.get("pending_share") + job_id = ( + str(pending_share.get("job_id", "")) + if isinstance(pending_share, dict) + else "" + ) + return _SupersededCandidateRow( + block_hash=block_hash, + parent_hash=parent_hash, + height=height, + network_difficulty=network_difficulty, + job_id=job_id, + ) + + def _select_superseded_block_candidates( + self, + durable_rows: list[Any], + chain: _BlockCandidateChainView, + *, + ignore_leases: frozenset[str] = frozenset(), + ) -> list[_SupersededCandidateRow]: + """Apply predicate S to one fetched page of durable pending rows. + + Every clause is conjunctive and every unknown is a rejection. The + chain view bounds the cost: one best-tip and one tip-height read for + the page, one active-block and one header read per distinct + candidate height, and at most + MAX_BLOCK_CANDIDATE_COLLAPSE_HEIGHT_PROBES distinct heights for the + whole walk. A row whose height the walk cannot afford to probe is + counted and skipped, which preserves it for the per-row path exactly + as any other rejected clause does. + + ``ignore_leases`` names disposition flights the *caller itself* + holds, exactly as it does for the pre-write revalidation: those + flights are not treated as offer evidence, and every other member of + the evidence set still rejects. The replay-adoption page path passes + nothing and is unchanged; the dequeue-time skip (issue #181 item 2) + passes the single hash whose lease ``submit_next`` claimed before it + called here, because that flight is this pass and this pass has + made no node offer. See + ``_skip_superseded_block_candidate_at_dequeue`` for why no other + flight can hide behind it. + """ + # Clause 2 is read first, for the whole page, before a single chain + # round trip: a page that cannot answer it selects nothing at all, + # and a ledger whose page reader never carries the fact (a + # compatibility intent-only reader) must not spend a tip read per + # poll to discover that again. The list is page-bounded. + pool_block_facts = [ + self._collapse_pool_block_exists(durable_row) + for durable_row in durable_rows + ] + # Evidence is asked about this page's hashes only. A malformed row + # contributes no probe and is failed closed by + # ``_superseded_candidate_row`` below regardless; every row that + # survives that decode carries the row key probed here, because the + # decode refuses an intent whose hash disagrees with it. + evidence = self._block_candidate_collapse_evidence( + ( + durable_row.get("block_hash") + for durable_row in durable_rows + if isinstance(durable_row, dict) + ), + ignore_leases=ignore_leases, + ) + tip = chain.best_tip() + tip_height = chain.tip_height() + selected: list[_SupersededCandidateRow] = [] + seen: set[str] = set() + height_deferred = 0 + for pool_block_exists, durable_row in zip(pool_block_facts, durable_rows): + row = self._superseded_candidate_row(durable_row) + if row.block_hash in seen: + continue + seen.add(row.block_hash) + if pool_block_exists: + continue + if row.block_hash in evidence: + continue + # Clause 3: a candidate building on the current best tip is the + # next block, not a superseded sibling. + if row.parent_hash == tip: + continue + # Clause 4: only a height the chain has already decided, decided + # by somebody else. + if row.height > tip_height: + continue + active = chain.active_at(row.height) + if active is None: + # The walk has spent its bounded height probes. Nothing is + # known about this height, so nothing may be concluded about + # this row: it stays durable, keeps its evidence, and reaches + # the per-row path this pass and the next walk's collapse + # after that. + height_deferred += 1 + continue + if active == row.block_hash: + continue + # Clause 4b/5: a lower-work occupant is not a decision. The + # per-row path can still reorg a heavier sibling into the chain, + # and a terminal batch write would destroy that block. Both + # sides are PRISM-scaled integers: the row as the coordinator + # stamped it, the occupant as re-derived from its header bits. + if chain.difficulty_of(active) < row.network_difficulty: + continue + selected.append(row) + self._record_block_candidate_collapse("height_deferred", height_deferred) + return selected + + def _revalidate_superseded_block_candidates( + self, + leased: list[_SupersededCandidateRow], + chain: _BlockCandidateChainView, + ) -> tuple[list[_SupersededCandidateRow], str]: + """Re-read the chain under the held leases, immediately before the write. + + Selection ran before any lease existed, so a selected candidate + could have been offered, accepted, and become the active block in + the gap. Re-reading the best tip and each retained height's occupant + with the leases held closes it: nothing else can drive this hash + through a node offer while the lease is ours, and the fenced write + is the very next thing that happens. + + The occupant's header is re-read only when the occupying hash + changed; an unchanged occupant keeps the work value this walk + already read for that hash, so the steady case adds no header round + trip. + """ + # Budgeted for exactly the heights selection already probed and no + # more. The leased set is a subset of one bounded selection, so this + # can never refuse a re-read the write depends on, and it cannot draw + # on the walk's budget either -- the walk's remaining probes are for + # pages this apply has not seen. A budget of its own is a cache of + # its own as well: the walk's active-height reads, which may have + # been made pages ago, are never reused here, so every leased height + # is genuinely re-read from the chain under the held leases. + fresh = _BlockCandidateChainView( + self, + probe_budget=_CollapseHeightProbeBudget( + remaining=len({row.height for row in leased}), + ), + ) + tip = fresh.best_tip() + leased_hashes = frozenset(row.block_hash for row in leased) + evidence = self._block_candidate_collapse_evidence( + leased_hashes, + ignore_leases=leased_hashes, + ) + qualified: list[_SupersededCandidateRow] = [] + for row in leased: + if row.block_hash in evidence: + continue + if row.parent_hash == tip or row.block_hash == tip: + continue + # An occupied height is by construction at or below the tip, so + # this read subsumes the height bound as well as clause 4b. + active = fresh.active_at(row.height) + if active is None or active == row.block_hash: + # Unreachable while the budget above matches the leased + # heights, and a drop either way: an unprobed height is a row + # this write must not carry. + continue + # Same clause, same units as selection: a cache hit reuses the + # scaled integer that pass derived, and a changed occupant is + # re-derived from its own header bits rather than compared as a + # raw float against the row's scaled value. + difficulty = chain.cached_difficulty(active) + if difficulty is None: + difficulty = fresh.difficulty_of(active) + if difficulty < row.network_difficulty: + continue + qualified.append(row) + return qualified, tip + + def _note_block_candidate_collapse_fail_closed( + self, + rows: int, + detail: object, + ) -> bool: + """Count a fail-closed page and log it at a bounded rate. + + Returns whether this occasion logged, so a caller holding an + exception can attach its traceback under the same rate limit + instead of flooding a persistently degraded read. + """ + self._record_block_candidate_collapse("fail_closed", rows) + now = time.monotonic() + with self._coordinator.lock: + last = getattr( + self, + "_block_candidate_collapse_fail_closed_logged_monotonic", + None, + ) + due = ( + last is None + or (now - float(last)) + >= BLOCK_CANDIDATE_COLLAPSE_FAIL_CLOSED_LOG_SECONDS + ) + if due: + self._block_candidate_collapse_fail_closed_logged_monotonic = now + if due: + print( + "prism coordinator: superseded block candidate collapse failed " + f"closed rows={rows}: {detail}; every row keeps the per-row " + "path", + flush=True, + ) + return due + + def _collapsed_candidate_floor_holders( + self, + abandoned: Iterable[str], + ) -> dict[str, list[PrismBlockCandidate]]: + """Index the in-memory candidate objects owning each collapsed hash. + + The pending-share floor keys holders by object identity, so only the + exact object a lane is holding may be released and never a same-hash + sibling's. A durable row that was never adopted owns no object and + no holder, and none is invented for it: decoding one just to release + it would adopt a fresh floor holder in order to drop it. + """ + wanted = frozenset(abandoned) + holders: dict[str, list[PrismBlockCandidate]] = {} + if not wanted: + return holders + self._ensure_block_replay_state() + for queue_obj in ( + self.candidate_queue, + self._block_replay_candidate_queue, + ): + if queue_obj is None: + continue + with queue_obj.mutex: + queued = list(queue_obj.queue) + for item in queued: + key = _collapse_block_hash( + getattr( + getattr(item, "submission", None), + "block_hash_hex", + None, + ) + ) + if key in wanted: + holders.setdefault(key, []).append(item) + return holders + + def _run_collapsed_candidate_cleanup_steps( + self, + abandoned: tuple[str, ...], + *, + owed: dict[str, frozenset[str]], + shares: dict[str, tuple[PendingShare, ...]] | None = None, + ) -> tuple[dict[str, frozenset[str]], dict[str, tuple[PendingShare, ...]]]: + """Run each hash's owed cleanup steps and report what it still owes. + + Every step is idempotent and independently guarded, so a hash whose + step raises cannot strand the rest of the page mid-way, and a step + the caller does not list is simply not run -- which is what lets a + deferred retry replay only the remainder. + + Returns the still-owed steps per hash (absent means fully clean) and + the floor holders this pass resolved, so a caller that has to defer + the remainder keeps the exact holder objects it would otherwise lose + when the queues drain. + """ + coordinator = self._coordinator + coordinator._record_block_submitter_phase("replay-collapse-cleanup") + remaining: dict[str, set[str]] = {} + detailed_failures = 0 + + def note_failure(block_hash: str, step: str) -> None: + nonlocal detailed_failures + detailed = detailed_failures < BLOCK_CANDIDATE_COLLAPSE_LOG_FAILURES + remaining.setdefault(block_hash, set()).add(step) + if not detailed: + return + detailed_failures += 1 + print( + "prism coordinator: collapsed block candidate cleanup failed " + f"step={step} hash={block_hash}", + flush=True, + ) + traceback.print_exc() + + # One outer acquisition for the whole page. Each withdrawal takes + # this lock anyway (it is re-entrant); acquiring it per hash would + # hand a storm-sized page thousands of chances to interleave with + # descendant payout work between two halves of one hash's cleanup. + with self._block_submitter_lock( + coordinator._payout_balance_mutation_lock, + "payout-balance-mutation", + ): + for block_hash in abandoned: + if ( + BLOCK_CANDIDATE_COLLAPSE_CLEANUP_PAYOUT_STEP + not in owed.get(block_hash, frozenset()) + ): + continue + try: + # Withdraw with published invalidation, then drop the + # tombstone: the durable row is already terminal, so no + # replay source is left for the tombstone to fence, and + # a retained one is exactly the preview wait storm this + # collapse exists to end. + coordinator._clear_accepted_block_payout_preview( + block_hash, + invalidate_published=True, + ) + coordinator._clear_accepted_block_payout_preview(block_hash) + self._discard_accepted_landing_attribution(block_hash) + except Exception: + note_failure( + block_hash, + BLOCK_CANDIDATE_COLLAPSE_CLEANUP_PAYOUT_STEP, + ) + outcome = SimpleNamespace( + reason=PRISM_BLOCK_CANDIDATE_COLLAPSE_REASON, + stale_job_class=PRISM_BLOCK_CANDIDATE_COLLAPSE_STALE_JOB_CLASS, + ) + if shares is None: + # Unguarded on purpose: a page-scope indexing fault proves + # nothing about any hash's remaining steps, so the caller must + # see it as an abort and defer the whole won set. + shares = { + block_hash: tuple( + candidate.pending_share for candidate in candidates + ) + for block_hash, candidates in ( + self._collapsed_candidate_floor_holders(abandoned).items() + ) + } + for block_hash in abandoned: + steps = owed.get(block_hash, frozenset()) + + def run_step(step: str, action: Callable[[], None]) -> None: + if step not in steps: + return + try: + action() + except Exception: + note_failure(block_hash, step) + + def clear_finalize_retry() -> None: + with coordinator.lock: + self.finalize_retries.pop(block_hash, None) + + run_step("finalize-retry", clear_finalize_retry) + run_step( + "retry-state", + lambda: coordinator._clear_block_candidate_retry_state(block_hash), + ) + # Stops matching tip observations and drops the tip observation + # stamp in the same critical section. + run_step( + "outstanding-and-tip-observation", + lambda: coordinator._discard_outstanding_block_candidate(block_hash), + ) + # Also drops the fast-lane reservation, replay-inflight marker, + # and any parked same-hash retry object's own floor holder. + run_step( + "terminal-outcome", + lambda: coordinator._record_block_candidate_terminal_outcome( + block_hash, + accepted=False, + ), + ) + run_step( + "abandonment-accounting", + lambda: coordinator._record_committed_block_candidate_abandonment( + block_hash, + outcome, + ), + ) + for pending_share in shares.get(block_hash, ()): + run_step( + BLOCK_CANDIDATE_COLLAPSE_CLEANUP_FLOOR_STEP, + lambda pending_share=pending_share: ( + coordinator._finish_pending_share_commit(pending_share) + ), + ) + return ( + { + block_hash: frozenset(steps) + for block_hash, steps in remaining.items() + }, + shares, + ) + + def _publish_collapsed_candidate_terminal_fence( + self, + abandoned: Iterable[str], + ) -> None: + """Fence every won hash from node submission before any cleanup runs. + + The fenced batch write is the moment a row becomes durably terminal, + and the ``terminal-outcome`` cleanup step is what normally publishes + that fact in memory. That step can fail, or a page-level abort can + stop the pass before it -- and the failure leaves the hash in the + cleanup-retry registry while a same-hash candidate may still be + sitting in the live, replay, retry, or waiting lane. Once the apply + releases its disposition lease that candidate is dequeued, finds no + terminal outcome, and offers a durably abandoned block to the node. + + Publishing the outcome here closes that window with the fence the + rest of the submitter already honours. The apply holds every won + hash's disposition lease from before the write until after this + call, so no same-hash lane can be inside its own guarded region + while this runs, and none can enter one afterwards without seeing + the fence. It is a lock-guarded assignment into a dict the + disposition state already owns: no durable read or write, no queue + admission, and nothing that could re-adopt or re-offer a row. + + The full ``terminal-outcome`` step still runs in cleanup for its + other side effects -- the fast-lane reservation, the replay-inflight + marker, and the parked same-hash retry's own floor holder -- and + remains owed, and retried, whenever it fails. + + Contained so it cannot raise: this sits between a won write and the + page partition, and an exception escaping here would fail the page + open and replay-adopt rows whose durable outbox entries are gone. + """ + try: + self._ensure_block_candidate_disposition_state() + published: set[str] = set() + with self._coordinator.lock: + outcomes = self._block_candidate_terminal_outcomes + for block_hash in abandoned: + self._stamp_block_candidate_terminal_outcome( + outcomes, + block_hash, + False, + ) + published.add(block_hash) + self._bound_block_candidate_terminal_outcomes( + frozenset(published) + ) + except Exception: + # Degrades to the pre-publication behaviour: the cleanup step + # still owes the outcome and the retry registry still carries it. + print( + "prism coordinator: collapsed block candidate terminal fence " + "could not be published", + flush=True, + ) + traceback.print_exc() + + def _clean_up_collapsed_block_candidates( + self, + abandoned: tuple[str, ...], + ) -> frozenset[str]: + """Mirror per-row terminal abandonment cleanup for the rows we won. + + Driven by the hashes the fenced write returned, never by the hashes + it was asked about: a requested-but-absent row was won by somebody + else (or already terminal), and running this cleanup for it would + discard state its real owner still needs. + + Returns the hashes whose cleanup did not complete. The apply owns + the ``cleanup_failed`` accounting for the whole page: counting a + contained failure here and a later page-level abort there would + report one affected hash twice. Each of those hashes keeps its + still-owed steps in the cleanup-retry registry, because its durable + row is terminal and no enumeration will ever hand it back. Their + terminal fence is already published by the apply, so a step this + pass leaves owed can never leave one of them offerable to the node. + """ + every_step = frozenset(BLOCK_CANDIDATE_COLLAPSE_CLEANUP_STEPS) + remaining, shares = self._run_collapsed_candidate_cleanup_steps( + abandoned, + owed={block_hash: every_step for block_hash in abandoned}, + ) + for block_hash in abandoned: + owed = remaining.get(block_hash) + if owed: + self._defer_collapsed_candidate_cleanup( + block_hash, + owed, + shares=shares.get(block_hash, ()), + ) + else: + # A hash the fenced write can only win once, but a direct + # caller may repeat the cleanup; a completed pass discharges + # whatever an earlier one left owed. + self._discharge_collapsed_candidate_cleanup(block_hash) + if remaining: + print( + "prism coordinator: collapsed block candidate cleanup failed " + f"rows={len(remaining)} of {len(abandoned)}; their durable rows " + "are already terminal and their cleanup is retried", + flush=True, + ) + return frozenset(remaining) + + # -- deferred cleanup retry -------------------------------------------- + + def _collapsed_candidate_cleanup_registry( + self, + ) -> dict[str, _CollapsedCandidateCleanup]: + self._ensure_block_candidate_collapse_state() + return self._block_candidate_collapse_cleanup_retries + + def _defer_collapsed_candidate_cleanup( + self, + block_hash: str, + steps: Iterable[str], + *, + shares: Iterable[PendingShare] = (), + shares_resolved: bool = True, + ) -> None: + """Keep one terminal hash's owed cleanup steps for a later retry. + + Contained so it cannot raise: the apply's abort path calls this on + its way to returning the won set, and an exception escaping there + would fail the whole page closed and replay-adopt rows whose durable + outbox entries are already gone. + """ + try: + owed = frozenset(steps).intersection( + BLOCK_CANDIDATE_COLLAPSE_CLEANUP_STEPS + ) + if not owed: + return + registry = self._collapsed_candidate_cleanup_registry() + now = time.monotonic() + with self._coordinator.lock: + record = registry.pop(block_hash, None) + if record is None: + record = _CollapsedCandidateCleanup( + block_hash=block_hash, + steps=owed, + shares=tuple(shares), + shares_resolved=bool(shares_resolved), + delay_seconds=max(0.0, float(self.retry_initial_seconds)), + # The first attempt is due immediately; only a failed + # attempt starts the backoff. + not_before_monotonic=now, + deferred_monotonic=now, + ) + else: + # The floor keys holders by object identity, so merge by + # identity rather than by value: two reconstructed shares + # for one hash are two distinct holders. + by_identity = {id(share): share for share in record.shares} + for share in shares: + by_identity.setdefault(id(share), share) + record.steps = record.steps | owed + record.shares = tuple(by_identity.values()) + record.shares_resolved = record.shares_resolved and bool( + shares_resolved + ) + record.not_before_monotonic = min( + record.not_before_monotonic, + now, + ) + if not record.deferred_monotonic: + record.deferred_monotonic = now + # Re-inserted at the tail so a storm-sized backlog is retried + # round-robin instead of starving behind its oldest entry. + registry[block_hash] = record + except Exception: + print( + "prism coordinator: collapsed block candidate cleanup could " + f"not be deferred hash={block_hash}", + flush=True, + ) + traceback.print_exc() + + def _discharge_collapsed_candidate_cleanup(self, block_hash: str) -> bool: + """Drop one hash's retry record; True when one was still owed.""" + registry = self._collapsed_candidate_cleanup_registry() + with self._coordinator.lock: + return registry.pop(block_hash, None) is not None + + def collapsed_candidate_cleanup_backlog(self) -> dict[str, frozenset[str]]: + """The cleanup steps each terminal hash still owes (diagnostics).""" + registry = self._collapsed_candidate_cleanup_registry() + with self._coordinator.lock: + combined: dict[str, frozenset[str]] = {} + for source in ( + registry, + self._block_candidate_collapse_cleanup_inflight, + ): + for block_hash, record in source.items(): + combined[block_hash] = ( + combined.get(block_hash, frozenset()) | record.steps + ) + return combined + + # -- cleanup-retry backlog bound (#198) -------------------------------- + + def _collapse_cleanup_retry_backlog_max(self) -> int: + """Retry records the backlog may hold before admission stops. + + Read from the coordinator attribute first, so an embedder or a test + can pin it the way the sibling accounting knobs are pinned, then + from the loaded ``BlockConfig``, then the shipped default. Clamp it + into the same safe range enforced by startup validation. A direct + runtime value below one degrades into "admit one row at a time while + the backlog is empty"; one above the fence registry's capacity can + never weaken the hard memory bound. + """ + coordinator = self._coordinator + value = getattr( + coordinator, + "block_candidate_cleanup_retry_backlog_max", + None, + ) + if value is None: + block_config = getattr(getattr(coordinator, "config", None), "block", None) + value = getattr( + block_config, + "candidate_cleanup_retry_backlog_max", + None, + ) + if value is None: + value = DEFAULT_BLOCK_CANDIDATE_CLEANUP_RETRY_BACKLOG_MAX + return min( + MAX_BLOCK_CANDIDATE_CLEANUP_RETRY_BACKLOG_MAX, + max(1, int(value)), + ) + + def _collapse_cleanup_admission_headroom(self) -> tuple[int, int, int]: + """(rows the collapse may still admit, backlog depth, bound). + + Only the collapse callers on the block-submitter thread add records. + The accounting lane moves one record into the in-flight registry + while retrying it, so the owed authority remains counted until the + attempt either discharges or re-registers it. + """ + registry = self._collapsed_candidate_cleanup_registry() + maximum = self._collapse_cleanup_retry_backlog_max() + with self._coordinator.lock: + depth = len(registry) + len( + self._block_candidate_collapse_cleanup_inflight + ) + return max(0, maximum - depth), depth, maximum + + def _collapse_cleanup_backpressure_engagement_count(self) -> int: + """Return the monotonic engagement counter without walking records.""" + self._ensure_block_candidate_collapse_state() + with self._coordinator.lock: + return int(self._block_candidate_cleanup_backpressure_engagements) + + def collapsed_candidate_cleanup_backlog_snapshot(self) -> dict[str, Any]: + """Fixed-key gauges over the cleanup-retry backlog, for metrics. + + The key set is closed and carries no hash or step name. The walk is + O(backlog) under ``coordinator.lock``, which the admission bound + keeps at a few thousand entries at most; every value is a copy, so + the renderer never holds a reference into the registry. + + ``terminal_outcome_pins`` counts the records whose hash currently + holds a published terminal-outcome fence -- the entries the + registry forbids the fence eviction from dropping. It equals the + depth whenever the apply's fence publication succeeded; a smaller + value names records whose fence is still owed to the + ``terminal-outcome`` step. + """ + self._ensure_block_candidate_collapse_state() + self._ensure_block_candidate_disposition_state() + maximum = self._collapse_cleanup_retry_backlog_max() + now = time.monotonic() + with self._coordinator.lock: + registry = self._block_candidate_collapse_cleanup_retries + inflight = self._block_candidate_collapse_cleanup_inflight + outcomes = self._block_candidate_terminal_outcomes + records = tuple(registry.values()) + tuple(inflight.values()) + depth = len(records) + holder_ids: set[int] = set() + pinned_hashes: set[str] = set() + oldest: float | None = None + for record in records: + holder_ids.update(id(share) for share in record.shares) + if record.block_hash in outcomes: + pinned_hashes.add(record.block_hash) + stamp = float(record.deferred_monotonic) + if oldest is None or stamp < oldest: + oldest = stamp + engagements = int( + self._block_candidate_cleanup_backpressure_engagements + ) + return { + "depth": depth, + "backlog_max": maximum, + "oldest_age_seconds": ( + -1.0 if oldest is None else max(0.0, now - oldest) + ), + "pending_share_holders": len(holder_ids), + "terminal_outcome_pins": len(pinned_hashes), + "backpressure_active": depth >= maximum, + "backpressure_engagements": engagements, + } + + def _note_block_candidate_cleanup_backpressure( + self, + *, + caller: str, + rows: int, + admitted: int, + depth: int, + maximum: int, + ) -> None: + """Count rows the admission bound preserved and warn at a bounded rate. + + Every field of the warning is drawn from a closed vocabulary or is a + count: the caller name, the rows preserved, the rows this same pass + still admitted, the backlog depth and bound at the decision, and the + oldest record's age. No hash, parent, page cursor, or step name ever + appears, so a storm-long engagement cannot grow the log line's shape + and a persistent one cannot flood the journal. + """ + if caller not in PRISM_BLOCK_CANDIDATE_CLEANUP_BACKPRESSURE_CALLERS: + raise ValueError(f"unknown cleanup backpressure caller: {caller}") + if rows <= 0: + return + self._record_block_candidate_collapse("backlog_deferred", rows) + now = time.monotonic() + with self._coordinator.lock: + self._block_candidate_cleanup_backpressure_engagements = ( + int(self._block_candidate_cleanup_backpressure_engagements) + 1 + ) + last = self._block_candidate_cleanup_backpressure_logged_monotonic + due = ( + last is None + or (now - float(last)) + >= BLOCK_CANDIDATE_CLEANUP_BACKPRESSURE_LOG_SECONDS + ) + if due: + self._block_candidate_cleanup_backpressure_logged_monotonic = now + if not due: + return + oldest_age = float( + self.collapsed_candidate_cleanup_backlog_snapshot()["oldest_age_seconds"] + ) + print( + "prism coordinator: collapsed block candidate cleanup backpressure " + f"engaged caller={caller} rows_preserved={rows} admitted={admitted} " + f"backlog={depth} backlog_max={maximum} " + f"oldest_seconds={oldest_age:.3f}; admitting them to bulk " + "terminalization would take the cleanup-retry backlog past its " + "bound, so they stay durable and pending for the per-row path " + "until it drains", + flush=True, + ) + + def _run_one_collapsed_block_candidate_cleanup_retry(self) -> bool: + """Retry one deferred collapse cleanup; True when one was attempted. + + Driven from the accounting lane, because a terminal hash's cleanup + is accounting work and the durable row behind it is already gone. + The lane offers it two ways: immediately whenever the lane is truly + idle, and otherwise on an explicit cadence after at most + ``_block_accounting_cleanup_retry_work_items()`` completed work + items, so neither sustained accounting traffic nor a continuously + replenished quarantine queue can starve a due record. The pass is + bounded three ways: at most one hash per call, only that hash's + still-owed steps, and only once its own backoff deadline has passed. + It never reads or writes the outbox and never enqueues a candidate, + so it can only finish tearing state down -- a terminal row can be + neither re-adopted nor re-offered from here. + """ + registry = self._collapsed_candidate_cleanup_registry() + inflight = self._block_candidate_collapse_cleanup_inflight + now = time.monotonic() + with self._coordinator.lock: + due: _CollapsedCandidateCleanup | None = None + for record in registry.values(): + if ( + record.block_hash not in inflight + and record.not_before_monotonic <= now + ): + due = record + break + if due is None: + return False + # Move, rather than drop, the authority record for the duration + # of the attempt. Admission, gauges, and terminal-outcome + # eviction all count the in-flight registry, so cleanup can run + # outside coordinator.lock without opening an undercount or an + # unfenced window. A second lane also skips the same hash. + registry.pop(due.block_hash, None) + inflight[due.block_hash] = due + due.attempts += 1 + attempt_returned = False + try: + result = self._attempt_collapsed_block_candidate_cleanup_retry(due) + attempt_returned = True + return result + finally: + with self._coordinator.lock: + retained = inflight.pop(due.block_hash, None) + if ( + not attempt_returned + and retained is not None + and due.block_hash not in registry + ): + # An unexpected exception outside the contained step + # runner must not discard the only cleanup authority. + registry[due.block_hash] = retained + + def _attempt_collapsed_block_candidate_cleanup_retry( + self, + due: _CollapsedCandidateCleanup, + ) -> bool: + """Run one claimed retry while its authority remains in-flight.""" + self._coordinator._record_block_submitter_phase( + "replay-collapse-cleanup-retry" + ) + block_hash = due.block_hash + owed = due.steps + shares = due.shares + shares_resolved = due.shares_resolved + if ( + BLOCK_CANDIDATE_COLLAPSE_CLEANUP_FLOOR_STEP in owed + and not shares_resolved + ): + # The apply aborted before it could index the floor holders. The + # scan reads only the in-memory queues, so repeating it is cheap + # and safe; a candidate already dropped from them released its + # own holder on the way out. + try: + holders = self._collapsed_candidate_floor_holders((block_hash,)) + except Exception: + print( + "prism coordinator: collapsed block candidate cleanup " + f"retry could not index floor holders hash={block_hash}", + flush=True, + ) + traceback.print_exc() + self._record_block_candidate_collapse("cleanup_retry_failed") + self._reschedule_collapsed_candidate_cleanup( + due, + owed, + shares=shares, + shares_resolved=False, + ) + return True + by_identity = {id(share): share for share in shares} + for candidate in holders.get(block_hash, ()): + by_identity.setdefault( + id(candidate.pending_share), + candidate.pending_share, + ) + shares = tuple(by_identity.values()) + shares_resolved = True + try: + remaining, _shares = self._run_collapsed_candidate_cleanup_steps( + (block_hash,), + owed={block_hash: owed}, + shares={block_hash: shares}, + ) + still_owed = remaining.get(block_hash, frozenset()) + except Exception: + # Same reasoning as the apply's abort: a pass that died proves + # nothing about the steps it skipped, so the hash owes them all + # again. + print( + "prism coordinator: collapsed block candidate cleanup retry " + f"aborted hash={block_hash}", + flush=True, + ) + traceback.print_exc() + still_owed = owed + if still_owed: + self._record_block_candidate_collapse("cleanup_retry_failed") + self._reschedule_collapsed_candidate_cleanup( + due, + still_owed, + shares=shares, + shares_resolved=shares_resolved, + ) + return True + # Counted once per hash: the record is gone, so no later pass can + # report the same recovery twice, and the series can never exceed + # the cleanup_failed set that created these records. + self._record_block_candidate_collapse("cleanup_recovered") + print( + "prism coordinator: collapsed block candidate cleanup recovered " + f"hash={block_hash} attempts={due.attempts}", + flush=True, + ) + return True + + def _reschedule_collapsed_candidate_cleanup( + self, + record: _CollapsedCandidateCleanup, + steps: Iterable[str], + *, + shares: Iterable[PendingShare], + shares_resolved: bool, + ) -> None: + """Re-register a retried hash behind its own doubled backoff. + + The pacing mirrors ``next_retry_delay``: the configured candidate + retry backoff, doubled per attempt and capped, so a systemically + broken cleanup cannot spin the accounting lane while a transient one + still recovers on its next tick. + """ + initial = max(0.0, float(self.retry_initial_seconds)) + maximum = max(initial, float(self.retry_max_seconds)) + delay = min(maximum, max(initial, float(record.delay_seconds) * 2)) + self._defer_collapsed_candidate_cleanup( + record.block_hash, + steps, + shares=shares, + shares_resolved=shares_resolved, + ) + registry = self._collapsed_candidate_cleanup_registry() + with self._coordinator.lock: + rescheduled = registry.get(record.block_hash) + if rescheduled is None: + return + rescheduled.attempts = max(rescheduled.attempts, record.attempts) + rescheduled.delay_seconds = delay + rescheduled.not_before_monotonic = time.monotonic() + delay + # The retry took the record out of the registry before the + # attempt, so the re-registration above minted a fresh stamp; + # the age gauge measures from the first deferral, not the + # latest failure. + if record.deferred_monotonic: + rescheduled.deferred_monotonic = min( + rescheduled.deferred_monotonic or record.deferred_monotonic, + record.deferred_monotonic, + ) + + def _log_collapsed_block_candidates( + self, + qualified: list[_SupersededCandidateRow], + abandoned: frozenset[str], + *, + considered: int, + selected: int, + lease_skipped: int, + revalidation_dropped: int, + ) -> None: + """Emit one bounded summary instead of one line per candidate.""" + groups: dict[tuple[str, str], list[str]] = {} + for row in qualified: + if row.block_hash not in abandoned: + continue + groups.setdefault((row.parent_hash, row.job_id[:64]), []).append( + row.block_hash + ) + print( + "prism coordinator: collapsed superseded block candidates " + f"considered={considered} selected={selected} " + f"lease_skipped={lease_skipped} " + f"revalidation_dropped={revalidation_dropped} " + f"abandoned={len(abandoned)} " + f"reason={PRISM_BLOCK_CANDIDATE_COLLAPSE_REASON} " + f"stale_job_class={PRISM_BLOCK_CANDIDATE_COLLAPSE_STALE_JOB_CLASS} " + f"groups={len(groups)}", + flush=True, + ) + ordered = sorted(groups.items(), key=lambda item: (-len(item[1]), item[0])) + for (parent_hash, job_id), hashes in ( + ordered[:BLOCK_CANDIDATE_COLLAPSE_LOG_GROUPS] + ): + sample = ",".join( + sorted(hashes)[:BLOCK_CANDIDATE_COLLAPSE_LOG_SAMPLE_HASHES] + ) + print( + "prism coordinator: collapsed candidate group " + f"parent={parent_hash} job={job_id} " + f"reason={PRISM_BLOCK_CANDIDATE_COLLAPSE_REASON} " + f"count={len(hashes)} sample={sample}", + flush=True, + ) + remainder = len(ordered) - BLOCK_CANDIDATE_COLLAPSE_LOG_GROUPS + if remainder > 0: + print( + "prism coordinator: collapsed candidate groups not shown " + f"count={remainder}", + flush=True, + ) + + def _apply_superseded_block_candidate_collapse( + self, + selected: list[_SupersededCandidateRow], + chain: _BlockCandidateChainView, + *, + page_rows: int, + timeout_seconds: float | None, + call_class: str, + held_leases: frozenset[str] = frozenset(), + ) -> frozenset[str]: + """Lease, revalidate, write once, then clean up exactly what we won. + + ``held_leases`` names hashes whose disposition lease the caller + already holds and will release itself. They are neither claimed nor + released here, and they count as leased for every step that follows + -- including the pre-write revalidation's own self-exemption, which + already ignores the leases this apply is operating under. The + replay-adoption page path passes nothing and behaves exactly as + before; the dequeue-time skip passes the one hash ``submit_next`` + leased before it dequeued anything. + """ + coordinator = self._coordinator + coordinator._record_block_submitter_phase("replay-collapse-lease") + leases: dict[str, _BlockCandidateDispositionLease] = {} + try: + for row in selected: + if row.block_hash in held_leases: + continue + # Never block: a synchronous submit that has persisted its + # intent but not yet claimed its own lease must not queue + # behind a page of maintenance work. + lease = coordinator._claim_block_candidate_disposition( + row.block_hash, + blocking=False, + ) + if lease is not None: + leases[row.block_hash] = lease + leased = [ + row + for row in selected + if row.block_hash in leases or row.block_hash in held_leases + ] + self._record_block_candidate_collapse( + "lease_skipped", + len(selected) - len(leased), + ) + if not leased: + return frozenset() + coordinator._record_block_submitter_phase("replay-collapse-revalidate") + try: + qualified, tip = self._revalidate_superseded_block_candidates( + leased, + chain, + ) + except Exception as exc: + logged = self._note_block_candidate_collapse_fail_closed( + page_rows, + exc, + ) + if logged and not isinstance( + exc, + _BlockCandidateCollapseFailedClosed, + ): + traceback.print_exc() + return frozenset() + self._record_block_candidate_collapse( + "revalidation_dropped", + len(leased) - len(qualified), + ) + if not qualified: + return frozenset() + hashes = tuple(row.block_hash for row in qualified) + error = f"tip moved before submit: {tip}" + coordinator._record_block_submitter_phase("replay-collapse-write") + try: + mark = coordinator.ledger.mark_block_candidates_abandoned + returned = coordinator._run_block_submitter_ledger_call( + ("collapse-superseded", hashes), + "collapse-superseded", + lambda: mark(block_hashes=hashes, error=error), + timeout_seconds=timeout_seconds, + call_class=call_class, + ) + except Exception as exc: + # The rows stay pending, so nothing is lost: the whole page + # is preserved and the per-row path disposes of it. A + # timed-out call may still land on its bounded worker; the + # rows it transitions simply replay as terminal, which is + # the same convergence every other outbox mutation has. + if self._note_block_candidate_collapse_fail_closed(page_rows, exc): + traceback.print_exc() + return frozenset() + if not isinstance(returned, (list, tuple, set, frozenset)): + # The contract is a hash set, not a count or a boolean. A + # ledger that answers with something else has told us + # nothing about which rows it won. + self._note_block_candidate_collapse_fail_closed( + page_rows, + "fenced batch abandonment did not return a hash set", + ) + return frozenset() + requested = frozenset(hashes) + abandoned = frozenset( + key + for key in ( + _collapse_block_hash(value) for value in (returned or ()) + ) + # A returned hash outside the request would mean the ledger + # transitioned a row this apply never leased; refuse to run + # cleanup for it rather than tear down somebody else's state. + if key is not None and key in requested + ) + self._record_block_candidate_collapse( + "write_lost", + len(requested - abandoned), + ) + if not abandoned: + return frozenset() + self._record_block_candidate_collapse("abandoned", len(abandoned)) + # Before any cleanup, and while every won hash's lease is still + # held: a cleanup that fails or aborts must not leave a durably + # terminal row offerable to the node. + self._publish_collapsed_candidate_terminal_fence(abandoned) + try: + cleanup_failed = self._clean_up_collapsed_block_candidates( + tuple( + row.block_hash + for row in qualified + if row.block_hash in abandoned + ) + ) + except Exception: + # The rows are durably terminal whatever happened here, so + # the caller must still partition them out of the page: a + # cleanup fault cannot be allowed to replay-adopt a row + # whose outbox entry is gone. An abort proves nothing about + # the steps it skipped, so the whole won set is affected -- + # which subsumes any hash a contained step already failed. + cleanup_failed = abandoned + # Every won hash owes every step again, and the floor + # holders were never indexed, so the retry re-runs that scan + # too. Deferral is contained, so this stays on the path that + # returns the won set. + for block_hash in abandoned: + self._defer_collapsed_candidate_cleanup( + block_hash, + BLOCK_CANDIDATE_COLLAPSE_CLEANUP_STEPS, + shares_resolved=False, + ) + print( + "prism coordinator: collapsed block candidate cleanup " + f"aborted rows={len(abandoned)}; their durable rows are " + "already terminal and their cleanup is retried", + flush=True, + ) + traceback.print_exc() + if cleanup_failed: + # Counted once per apply over distinct hashes. Contained + # per-step failures and a later abort describe the same + # rows, so this series can never exceed the won set. + self._record_block_candidate_collapse( + "cleanup_failed", + len(cleanup_failed), + ) + try: + self._log_collapsed_block_candidates( + qualified, + abandoned, + considered=page_rows, + selected=len(selected), + lease_skipped=len(selected) - len(leased), + revalidation_dropped=len(leased) - len(qualified), + ) + except Exception: + # Diagnostics are not cleanup state. Keep a formatter fault + # visible without claiming hashes whose terminal cleanup + # completed successfully. + print( + "prism coordinator: collapsed block candidate logging " + f"failed rows={len(abandoned)}", + flush=True, + ) + traceback.print_exc() + return abandoned + finally: + for lease in leases.values(): + coordinator._release_block_candidate_disposition(lease) + + def _collapse_superseded_block_candidates( + self, + durable_rows: list[Any], + *, + timeout_seconds: float | None, + call_class: str, + probe_budget: _CollapseHeightProbeBudget | None = None, + ) -> list[Any]: + """Terminalize this page's decided-height siblings before adoption. + + Returns the rows that still have to be adopted: exactly the page + minus the hashes the fenced write actually transitioned. A won row's + fetched payload is a stale copy of an intent whose durable row is + now terminal, so it must never be replay-adopted or re-arm a payout + barrier. Anything short of a won row -- a fail-closed read, an + unclaimable lease, a revalidation drop, an unprobed height, a partial + return, a failed write -- preserves the row and leaves it to the + per-row path. + + ``probe_budget`` is the enumeration walk's shared chain-height + budget, carrying both the walk's remaining probes and the reads it + has already made; passes given none get a fresh one of their own. A + spent budget ends this page only when the page has nothing at a + height the walk already read: the counter bounds new heights, and a + cached height costs nothing to answer from however late in the walk + it is asked. + """ + if not durable_rows: + return durable_rows + if not callable( + getattr( + self._coordinator.ledger, + "mark_block_candidates_abandoned", + None, + ) + ): + # A ledger with no fenced batch abandonment has no safe bulk + # form at all. That is a structural absence, not a fail-closed + # read, so it neither counts nor spends a chain round trip: the + # per-row path disposes of every row exactly as before. + return durable_rows + if ( + probe_budget is not None + and probe_budget.exhausted + and not any( + _collapse_row_height(durable_row) in probe_budget.active + for durable_row in durable_rows + ) + ): + # This walk has spent its chain-height budget on earlier pages + # *and* not one row here sits at a height those pages already + # read. A spent counter only forbids reading a new height; a + # height already in the walk's cache stays answerable for free, + # and a page holding such rows still qualifies them below. This + # page holds none, so selection could not qualify a single row + # and the two page-scope tip reads it opens with would be two + # more round trips for a pass that is already decided. Every row + # is preserved. + # + # The peek is deliberately lenient: a row it cannot read a height + # from contributes no cached height, which at worst preserves a + # page the decode would have failed closed on anyway -- exactly + # what the unconditional early return did for every such page. + self._record_block_candidate_collapse( + "height_deferred", + len(durable_rows), + ) + return durable_rows + headroom, backlog_depth, backlog_max = ( + self._collapse_cleanup_admission_headroom() + ) + if headroom <= 0: + # Issue #198. The cleanup-retry backlog is at its bound: every + # row the fenced write could win here would add one more record + # that only the accounting lane can ever drain. Nothing is + # selected and no chain read is spent; every row stays durable + # and pending for the per-row path, exactly as an unprobed + # height leaves it. Rows already terminal keep their records, + # holders, and fences untouched -- this bounds admission, never + # cleanup authority. + self._note_block_candidate_cleanup_backpressure( + caller="replay-page", + rows=len(durable_rows), + admitted=0, + depth=backlog_depth, + maximum=backlog_max, + ) + return durable_rows + self._coordinator._record_block_submitter_phase("replay-collapse-select") + self._record_block_candidate_collapse("considered", len(durable_rows)) + chain = _BlockCandidateChainView(self, probe_budget=probe_budget) + try: + selected = self._select_superseded_block_candidates( + durable_rows, + chain, + ) + except Exception as exc: + logged = self._note_block_candidate_collapse_fail_closed( + len(durable_rows), + exc, + ) + if logged and not isinstance(exc, _BlockCandidateCollapseFailedClosed): + traceback.print_exc() + return durable_rows + if not selected: + return durable_rows + self._record_block_candidate_collapse("selected", len(selected)) + if len(selected) > headroom: + # Predicate S is unchanged: every one of these rows satisfied + # it and is counted as selected. Admission is what is bounded: + # the fenced write may win at most ``headroom`` more rows before + # the backlog reaches its bound, so only that many, in page + # order, are handed on. The remainder is preserved, not + # disposed of. + self._note_block_candidate_cleanup_backpressure( + caller="replay-page", + rows=len(selected) - headroom, + admitted=headroom, + depth=backlog_depth, + maximum=backlog_max, + ) + selected = selected[:headroom] + try: + abandoned = self._apply_superseded_block_candidate_collapse( + selected, + chain, + page_rows=len(durable_rows), + timeout_seconds=timeout_seconds, + call_class=call_class, + ) + except Exception as exc: + if self._note_block_candidate_collapse_fail_closed( + len(durable_rows), + exc, + ): + traceback.print_exc() + return durable_rows + if not abandoned: + return durable_rows + return [ + durable_row + for durable_row in durable_rows + if _collapse_block_hash( + durable_row.get("block_hash") + if isinstance(durable_row, dict) + else None + ) + not in abandoned + ] + + # -- dequeue-time stale sibling skip (issue #181 item 2) ---------------- + + def _block_candidate_dequeue_chain(self) -> _BlockCandidateChainView: + """A chain view over the submitter's own tip-epoch height cache. + + The replay walk shares one ``_CollapseHeightProbeBudget`` across its + pages so a storm at one decided height costs the walk one + ``getblockhash`` and one ``getblockheader`` however many pages it + spans. A dequeue burst has exactly that shape -- hundreds of + siblings of one decided height, arriving one candidate at a time -- + so it shares a budget the same way, and for the same reason: without + it every sibling would re-read the occupant and its header. + + The cache is retired by :meth:`_retire_block_candidate_dequeue_chain` + the moment the best tip changes, so its lifetime is exactly one tip + epoch. Only the height caches are shared: the tip itself is memoized + per view, so every candidate's selection still runs against a + freshly read best tip, and the pre-write revalidation builds its own + budget and re-reads the occupant under the held lease regardless. + + Submitter-thread state, like the replay walk's budget: ``submit_next`` + is the only caller and the block-submitter thread is the only thread + that reaches it. + """ + budget = getattr(self, "_block_candidate_dequeue_probe_budget", None) + if budget is None: + budget = _CollapseHeightProbeBudget() + self._block_candidate_dequeue_probe_budget = budget + self._block_candidate_dequeue_probe_tip: str | None = None + return _BlockCandidateChainView(self, probe_budget=budget) + + def _retire_block_candidate_dequeue_chain(self, tip: str) -> None: + """Drop every height this cache read under an earlier best tip. + + The best tip names the whole active chain beneath it: a block at any + height can only change by changing every descendant, so while the + tip hash is unchanged every ``getblockhash(H)`` below it answers + identically. Retiring exactly on a tip change therefore makes the + cache valid for precisely as long as it is kept, and returns the + bounded probe allowance so a long-lived submitter never stops + probing new heights. + + That is a stronger statement than the walk's cross-page sharing + needs, and the pre-write revalidation is unaffected either way: it + builds its own budget and re-reads the occupant from the chain under + the held lease before anything terminal happens. + """ + if getattr(self, "_block_candidate_dequeue_probe_tip", None) == tip: + return + budget = self._block_candidate_dequeue_probe_budget + budget.active.clear() + budget.difficulty.clear() + budget.remaining = MAX_BLOCK_CANDIDATE_COLLAPSE_HEIGHT_PROBES + self._block_candidate_dequeue_probe_tip = tip + + def _skip_superseded_block_candidate_at_dequeue( + self, + candidate: PrismBlockCandidate, + *, + timeout_seconds: float | None = None, + call_class: str = "fast", + ) -> bool: + """Terminalize one provably-stale dequeued candidate before any offer. + + Returns True when the durable row was abandoned and every piece of + this candidate's in-memory state was torn down, so ``submit_next`` + must release its lease and consume the wakeup without offering + anything. False means "offer it", for any reason at all: this is an + optimisation over the per-row path and declining it is always safe. + + Why this exists (issue #181, the 2026-08-20 spike): a candidate + whose parent is no longer the best tip and that carries no offer + evidence is provably stale *before* the offer, and the per-row path + spends one ``submitblock``, ~6 chain reads and two ledger writes + discovering that -- plus an accounting task, a fast-lane + reservation, and an accepted-block payout-preview barrier armed and + withdrawn -- for each one. Those are the individual + ``block candidate abandoned reason=stale-job: tip moved before + submit`` bursts the 2026-08-21 validation still showed for + live/fast-path siblings after #196 removed the replay population. + + The predicate is #196's, unchanged: same evidence set, same + clauses, same fencing, same cleanup, same terminal write. This is a + second *caller*, not a second notion of staleness. The one + divergence is the self-exemption below. + + **The self-exemption, and why nothing else can hide behind it.** + ``submit_next`` claims this hash's disposition lease before it gets + here, so the hash is in ``_block_candidate_disposition_flights`` -- + a member of #196's evidence set E -- and #196's selector would + reject the row on clause 1 forever. Exactly one flight is exempted: + the one this pass holds the lock of, named by this candidate's own + hash and passed as ``ignore_leases``/``held_leases``. No other + flight can be exempted, because no other hash is ever in that set; + and no other *holder* of this flight can hide behind the exemption, + because a flight's registry entry is shared by its holder and its + waiters while its lock has exactly one owner -- this pass. A pass + merely waiting on the lock has offered nothing, and once the fenced + write lands, ``_publish_collapsed_candidate_terminal_fence`` stamps + the terminal outcome while the lease is still ours, so the waiter + wakes into the fence rather than into an offer. Every other member + of E still rejects this hash: retry holders, the deferred accounting + retry, ``_block_disposition_waiting_retries``, ``finalize_retries``, + ``_block_candidate_retained_node_submissions``, + ``_tip_observed_accepted_block_hashes``, + ``_accounted_accepted_block_hashes``, + ``_block_candidate_terminal_outcomes``, and a ``qbit_pool_blocks`` + row in any state. + + **Chain, not observation set.** Replay adoption does not register a + hash outstanding, so a blockwait observation of a replayed hash is + dropped until dequeue and the in-memory tip view says nothing about + it. Clause 3 is therefore decided against a freshly read + ``getbestblockhash``, never against the coordinator's published tip. + + **Fail closed.** Any unreadable fact -- a chain read, the pool-block + probe, an intent this candidate cannot answer, a fenced write that + did not return this hash -- preserves the candidate for the offer + path. Nothing is ever abandoned on an unknown. + + One diagnostic note: because this reuses #196's selection and apply + verbatim, the submitter phase stamps those emit still read + ``replay-collapse-*``. They name the machinery, which is shared, not + the caller. The ``dequeue_considered``/``dequeue_skipped``/ + ``dequeue_preserved`` counters are what separate this caller's + population from the replay walk's, and the one phase this method + stamps itself -- ``dequeue-collapse-pool-block`` -- names the only + durable read it adds. + """ + coordinator = self._coordinator + ledger = coordinator.ledger + if not callable(getattr(ledger, "mark_block_candidates_abandoned", None)): + # A ledger with no fenced batch abandonment has no safe bulk form + # at all; a structural absence, so it neither counts nor spends a + # round trip. Same reasoning as the page-level driver. + return False + pool_block_reader = getattr(ledger, "pool_block_state", None) + if not callable(pool_block_reader): + # Clause 2 has no durable answer here. Reading it as false would + # abandon exactly the rows that must never be abandoned. + return False + block_hash = _block_candidate_hash_of(candidate) + if block_hash is None: + return False + headroom, backlog_depth, backlog_max = ( + self._collapse_cleanup_admission_headroom() + ) + if headroom <= 0: + # Issue #198: same bound, same answer as the page walk. The + # candidate is not considered at all -- the dequeue partition + # counters stay exact -- and it takes the ordinary offer path, + # whose per-row disposition has its own cleanup contract. + self._note_block_candidate_cleanup_backpressure( + caller="dequeue", + rows=1, + admitted=0, + depth=backlog_depth, + maximum=backlog_max, + ) + return False + held = frozenset((block_hash,)) + self._record_block_candidate_collapse("dequeue_considered") + try: + if self._block_candidate_collapse_evidence(held, ignore_leases=held): + # Clause 1 first, because it is the only clause that costs + # nothing: the selector re-asks it below, but asking here + # keeps both the chain round trip and the durable pool-block + # probe off every candidate some evidence already excludes. + self._record_block_candidate_collapse("dequeue_preserved") + return False + parent_hash = _collapse_block_hash( + candidate.context.template["previousblockhash"] + ) + if parent_hash is None: + raise _BlockCandidateCollapseFailedClosed( + "dequeued candidate carries no usable parent hash" + ) + chain = self._block_candidate_dequeue_chain() + tip = chain.best_tip() + self._retire_block_candidate_dequeue_chain(tip) + if parent_hash == tip: + # Clause 3, short-circuited before any durable read: this is + # the block waiting to be offered, not a superseded sibling. + # Keeping it first is what holds the acceptance path's added + # cost to a single getbestblockhash. + # + # That read is not replaceable by the coordinator's published + # tip (``_current_published_tip_hash_locked``), tempting as a + # free in-memory gate is. Published work is what a *blocked* + # refresh stops updating, and a blocked refresh is #181's + # symptom: during the incident the published tip sits at the + # very parent every stale sibling names, so a gate on it + # would decline to skip exactly the population this exists + # for. The chain is asked instead, which is also what D5 + # requires for replay-adopted rows. + self._record_block_candidate_collapse("dequeue_preserved") + return False + pool_block_exists = ( + coordinator._run_block_submitter_ledger_call( + ("dequeue-collapse-pool-block", block_hash), + "dequeue-collapse-pool-block", + lambda: pool_block_reader(block_hash=block_hash), + timeout_seconds=timeout_seconds, + call_class=call_class, + ) + is not None + ) + selected = self._select_superseded_block_candidates( + [ + _dequeued_candidate_collapse_row( + candidate, + pool_block_exists=pool_block_exists, + ) + ], + chain, + ignore_leases=held, + ) + except Exception as exc: + logged = self._note_block_candidate_collapse_fail_closed(1, exc) + if logged and not isinstance(exc, _BlockCandidateCollapseFailedClosed): + traceback.print_exc() + self._record_block_candidate_collapse("dequeue_preserved") + return False + if not selected: + self._record_block_candidate_collapse("dequeue_preserved") + return False + self._record_block_candidate_collapse("selected", len(selected)) + try: + abandoned = self._apply_superseded_block_candidate_collapse( + selected, + chain, + page_rows=1, + timeout_seconds=timeout_seconds, + call_class=call_class, + held_leases=held, + ) + except Exception as exc: + if self._note_block_candidate_collapse_fail_closed(1, exc): + traceback.print_exc() + self._record_block_candidate_collapse("dequeue_preserved") + return False + if block_hash not in abandoned: + # Anything short of a won row -- an unclaimable lease, a + # revalidation drop, a lost fenced write -- is preserved. + self._record_block_candidate_collapse("dequeue_preserved") + return False + # The floor holder is bound to the *queued object's* identity, and + # this object has already left its lane, so the apply's own + # pending-share step -- which indexes holders by scanning the live + # and replay queues -- cannot see it. Release it here, through the + # same seam ``submit_next`` uses for a dropped duplicate. Correct + # whether the apply's cleanup completed or was deferred: a deferred + # retry has no queue left to find this object in either. + self._release_dropped_duplicate_candidate_floor(candidate) + self._record_block_candidate_collapse("dequeue_skipped") + return True + + def _block_replay_should_yield_to_live_candidates( + self, + probe_budget: _CollapseHeightProbeBudget, + ) -> bool: + """Whether this walk should hand the submitter back before another page. + + The block-submitter thread that walks these pages is the only thread + that reaches ``submitblock``, so a backlog it keeps walking is a + backlog that delays a live solve. Between pages -- never inside one + page's fenced collapse, its write, or its adoption -- the walk gives + that thread back as soon as two things hold at once: + + * a live candidate or a retry is waiting, so there is something + strictly more valuable than finishing the enumeration; and + * the walk has already spent its chain-height probes, so the pages it + would go on to fetch can no longer decide a height it has not + already read. + + Both halves matter. Yielding on a waiting candidate alone would cut + short the same-height storm this path exists for, which walks several + pages on a single probe. Yielding on a spent budget alone would leave + the enumeration owed -- and job builds blocked -- for a walk that had + nothing better to do than finish. The caller re-enumerates from the + outbox on its next pass with a fresh budget, and every row this walk + fetched was adopted before the yield, so the backlog still shrinks. + + A spent budget is not the same as nothing left to collapse: the pages + past the yield may hold siblings at heights this walk already read, + and those would have collapsed for free. What the yield gives up is + therefore bounded and recovered -- those rows stay durable and pending, + and the next walk re-probes their heights on a fresh budget -- while + what it buys is the only thread that reaches ``submitblock``. + """ + if not probe_budget.exhausted: + return False + with self._coordinator.lock: + if self.retry_candidate is not None: + return True + queue_obj = self.candidate_queue + return queue_obj is not None and not queue_obj.empty() + + # -- targeted ancestor re-drive (issue #190) --------------------------- + + def _ancestor_redrive_defer_threshold(self) -> int: + return max( + 1, + int( + getattr( + self._coordinator, + "accepted_parent_redrive_defer_threshold", + DEFAULT_ACCEPTED_PARENT_REDRIVE_DEFER_THRESHOLD, + ) + ), + ) + + def _ancestor_redrive_attempt_cap(self) -> int: + """Re-drives one ancestor may trigger; zero disables the mechanism.""" + return max( + 0, + int( + getattr( + self._coordinator, + "accepted_parent_redrive_attempt_max", + DEFAULT_ACCEPTED_PARENT_REDRIVE_ATTEMPT_MAX, + ) + ), + ) + + def _evict_stale_redrive_entries_locked(self) -> None: + """Bound per-hash re-drive bookkeeping. Caller holds the runtime lock. + + Entries are dropped on resolution, so eviction only engages under a + pathological stream of distinct never-resolving ancestors. The + deferral path re-inserts its ancestor's record and its child's + blocking entry on every deferral, so an actively wedged pair sits at + the back of each registry and is the last thing an oldest-first + eviction touches -- exhausted records included, whose survival is + what keeps the per-ancestor cap a lifetime bound. The two registries + are keyed by different hash spaces (ancestors vs children), so each + is trimmed strictly against itself: a hash that is both a child and + an ancestor must not lose its ancestor record to the eviction of a + stale child entry. + """ + while len(self._ancestor_redrive_records) > MAX_ANCESTOR_REDRIVE_TRACKED_HASHES: + self._ancestor_redrive_records.pop( + next(iter(self._ancestor_redrive_records)), None + ) + while ( + len(self._ancestor_redrive_last_blocking) + > MAX_ANCESTOR_REDRIVE_TRACKED_HASHES + ): + self._ancestor_redrive_last_blocking.pop( + next(iter(self._ancestor_redrive_last_blocking)), None + ) + + def note_pending_parent_transition_deferral( + self, + block_hash: str, + ancestor_hash: str, + ) -> None: + """Track one finalization deferral against its blocking ancestor. + + Called by the coordinator's pending-parent fence each time a + candidate's finalization defers because ``ancestor_hash``'s accepted + payout transition is unresolved. The deferral itself only re-checks + the transition, so a streak of them proves the retry loop cannot + resolve the ancestor on its own; on crossing the configured + threshold a targeted durable-replay pass is armed for the submitter + loop. Bounded per ancestor by the attempt cap, past which deferrals + fall back to exactly the pre-#190 behavior with the + publication-progress watchdog as the backstop. + """ + cap = self._ancestor_redrive_attempt_cap() + if cap <= 0: + # Zero disables the mechanism outright: no streaks, no armed + # passes, and no exhaustion accounting. A disabled install must + # not emit the alert-worthy exhausted signal a spent cap means. + return + child = str(block_hash).lower() + ancestor = str(ancestor_hash).lower() + threshold = self._ancestor_redrive_defer_threshold() + requested = False + exhausted = False + attempts = 0 + streak = 0 + self._ensure_block_replay_state() + with self._coordinator.lock: + # Re-inserted (not updated in place) so insertion order tracks + # recency and the eviction above stays oldest-first. + self._ancestor_redrive_last_blocking.pop(child, None) + self._ancestor_redrive_last_blocking[child] = ancestor + record = self._ancestor_redrive_records.pop(ancestor, None) + if record is None: + record = _AncestorRedriveRecord() + self._ancestor_redrive_records[ancestor] = record + # A replay-lane copy of the ancestor means in-process resolution + # is already underway -- an earlier re-drive's adoption, or + # ordinary replay. Deferrals during that window prove nothing a + # new pass could act on (it would no-op on the adopted check), + # so they must not bank toward attempts: burning the cap against + # a mechanism that is mid-fix would exhaust it before slow-but- + # working finalization completes, standing the re-drive down for + # an ancestor it had already reached. + ancestor_inflight = ancestor in self._block_replay_inflight_hashes + if not record.armed and not record.exhausted and not ancestor_inflight: + record.streak += 1 + streak = record.streak + if record.streak >= threshold: + if record.attempts < cap: + record.attempts += 1 + record.armed = True + record.streak = 0 + attempts = record.attempts + self.accepted_parent_redrive_attempt_count = ( + int(self.accepted_parent_redrive_attempt_count) + 1 + ) + requested = True + else: + record.exhausted = True + attempts = record.attempts + self.accepted_parent_redrive_exhausted_count = ( + int(self.accepted_parent_redrive_exhausted_count) + 1 + ) + exhausted = True + self._evict_stale_redrive_entries_locked() + if requested: + print( + "prism coordinator: arming in-process ancestor re-drive " + f"ancestor={ancestor} child={child} deferral_streak={streak} " + f"attempt={attempts}/{cap}", + flush=True, + ) + if exhausted: + print( + "prism coordinator: ancestor re-drive attempts exhausted " + f"ancestor={ancestor} child={child} attempts={attempts}; " + "deferrals continue and the publication-progress watchdog " + "remains the backstop", + flush=True, + ) + + def note_pending_parent_transition_resolved(self, block_hash: str) -> None: + """Drop re-drive bookkeeping once a child's ancestor fence passes.""" + child = str(block_hash).lower() + resolved_ancestor: str | None = None + with self._coordinator.lock: + ancestor = self._ancestor_redrive_last_blocking.pop(child, None) + if ancestor is None: + return + record = self._ancestor_redrive_records.pop(ancestor, None) + for key in [ + key + for key, value in self._ancestor_redrive_last_blocking.items() + if value == ancestor + ]: + self._ancestor_redrive_last_blocking.pop(key, None) + if record is not None and record.consumed > 0: + # Counted only when a forced pass actually ran: a transition + # that resolved through the ordinary landing tail while a + # request sat armed-but-unconsumed is the mechanism standing + # by, not succeeding, and must not inflate the resolved + # series operators compare against attempts. + self.accepted_parent_redrive_resolved_count = ( + int(self.accepted_parent_redrive_resolved_count) + 1 + ) + resolved_ancestor = ancestor + if resolved_ancestor is not None: + print( + "prism coordinator: pending ancestor payout transition " + "resolved after in-process re-drive " + f"ancestor={resolved_ancestor} child={child}", + flush=True, + ) + + def _ancestor_redrive_owed(self) -> bool: + """Whether a forced durable-replay pass is armed for any ancestor.""" + with self._coordinator.lock: + return any( + record.armed + for record in self._ancestor_redrive_records.values() + ) + + def _consume_ancestor_redrive_requests(self) -> tuple[str, ...]: + """Take (and clear) every armed re-drive; one forced pass serves all. + + Consumed up front deliberately: a pass whose enumeration fails burns + the attempt rather than re-running unbounded, a fresh deferral + streak re-arms the next attempt, and the per-ancestor cap bounds the + total. + """ + with self._coordinator.lock: + consumed = tuple( + ancestor + for ancestor, record in self._ancestor_redrive_records.items() + if record.armed + ) + for ancestor in consumed: + record = self._ancestor_redrive_records[ancestor] + record.armed = False + record.consumed = record.attempts + return consumed + + def _block_candidate_owned_in_process(self, block_hash: str) -> bool | None: + """Whether any live lane still names this hash, or None if unreadable. + + Reads exactly the pin set the terminal-outcome eviction trusts, via + the shared collector, so the two proofs cannot drift apart. None + means a leaf lane could not be read without waiting under the global + lock; callers treat that as owned, which is the safe direction. + """ + key = str(block_hash).lower() + self._ensure_block_candidate_disposition_state() + self._ensure_block_replay_state() + with self._coordinator.lock: + pins = self._collect_live_block_candidate_pins() + if pins is None: + return None + pinned, held = pins + if key in held: + return True + return any(key in registry for registry in pinned) + + def _resolve_unreplayable_ancestor_transition(self, block_hash: str) -> None: + """Converge a stuck transition with durable state when replay cannot. + + A forced enumeration that adopts the ancestor's pending outbox row is + the ordinary re-drive; this handles the remaining wedge shape, where + the armed transition has no pending durable row left -- the state a + process restart resolves simply by not rebuilding the transition. It + is cleared here only under the same proof a fresh startup replay + would compute: the completed enumeration found no pending row for + the hash, no in-process lane still owns a copy that could finish (or + withdraw) the transition itself, the durable pool-block row reports + the block confirmed and not reversed, and a fresh chain probe places + the block's own hash on the active chain at its height -- the exact + predicate the landing's already-confirmed branch clears the preview + under, which consults the durable row only for a block it has just + proven active. The probe matters because a reorg qbitd has seen but + the reconciler has not yet written back (reconciliation fails closed + while a landed transition exists) leaves the durable row + stale-confirmed, and clearing on the row alone would unfence + descendants onto the orphaned block's carry. Anything short of the + full proof leaves the transition alone and the watchdog remains the + backstop. + """ + coordinator = self._coordinator + coordinator._ensure_job_cache_state() + key = str(block_hash).lower() + with coordinator._accepted_block_payout_preview_condition: + transition = coordinator._accepted_block_payout_previews.get(key) + if transition is None: + return + if self._block_candidate_owned_in_process(key) is not False: + print( + "prism coordinator: ancestor re-drive left the transition to " + f"its in-process owner hash={key}", + flush=True, + ) + return + block_state_reader = getattr(coordinator.ledger, "pool_block_state", None) + if not callable(block_state_reader): + return + try: + block_state = coordinator._run_block_submitter_ledger_call( + ("redrive-pool-block-state", key), + "redrive-pool-block-state", + lambda: block_state_reader(block_hash=key), + ) + except Exception: + print( + "prism coordinator: ancestor re-drive pool-block state read " + f"failed hash={key}", + flush=True, + ) + traceback.print_exc() + return + confirmed = ( + isinstance(block_state, dict) + and str(block_state.get("chain_state", "")) == "confirmed" + and str(block_state.get("maturity_state", "")) != "reversed" + ) + if not confirmed: + print( + "prism coordinator: ancestor re-drive could not resolve the " + f"transition hash={key} (no pending outbox row, pool block " + "not durably confirmed); watchdog remains the backstop", + flush=True, + ) + return + # The durable row alone is not proof against a reorg the fenced + # reconciler has not written back yet; require the same node-side + # activity the landing proves before it trusts this row. Anything + # but a definite True (probe failure included) stands down. + chain_probe = self._block_candidate_chain_probe( + key, + expected_height=getattr(transition, "block_height", None), + ) + if chain_probe is not True: + print( + "prism coordinator: ancestor re-drive could not prove the " + f"durably confirmed block active on the node's chain " + f"hash={key} probe={chain_probe}; watchdog remains the " + "backstop", + flush=True, + ) + return + # Durable state already includes this block's payout carry, so + # removing the in-memory override changes no logical payout state -- + # the same clear the landing performs for an already-confirmed + # exact-idempotent replay. + coordinator._clear_accepted_block_payout_preview(key) + print( + "prism coordinator: ancestor re-drive cleared a stale transition " + f"for durably confirmed block hash={key}", + flush=True, + ) + + def _conclude_ancestor_redrive_pass( + self, + redrive_hashes: tuple[str, ...], + enumeration_truncated: bool, + ) -> None: + """Disposition each consumed re-drive after its enumeration ran. + + A truncated pass proved nothing about missing rows, so every + transition is left for the next attempt (a fresh deferral streak + re-arms it) or the watchdog. A complete pass either finds the + ancestor (re-)adopted -- ordinary replay finalization owns it from + there -- or runs the guarded stale-transition sweep. + """ + if not redrive_hashes: + return + if enumeration_truncated: + print( + "prism coordinator: ancestor re-drive enumeration was " + "truncated; leaving pending transitions for the next " + "attempt or the watchdog", + flush=True, + ) + return + for redrive_hash in redrive_hashes: + key = str(redrive_hash).lower() + with self._coordinator.lock: + adopted = key in self._block_replay_inflight_hashes + if adopted: + # The pending outbox row was (re-)adopted -- by this pass + # or an earlier one -- so ordinary replay finalization now + # owns resolving the transition. + print( + "prism coordinator: ancestor re-drive enumerated a " + f"pending candidate hash={key}", + flush=True, + ) + continue + self._resolve_unreplayable_ancestor_transition(key) + + def replay_pending(self) -> int: + """Queue durable candidate intents not completed by an earlier process.""" + self._coordinator._record_block_submitter_phase("replay-check-memory") + self._ensure_block_replay_state() + # The normal submitter loop drains replay work before asking the + # outbox again. Enforce that ordering here too for direct callers: a + # forced walk paused by cleanup backpressure must not append another + # page to the unbounded replay queue before its bounded page drains. + if self._block_replay_backpressure_drain_pending: + if not self._block_replay_candidate_queue.empty(): + return 0 + self._block_replay_backpressure_drain_pending = False + # While startup enumeration is still owed, correctness requires the + # outbox query even if live candidates are queued: job builds stay + # blocked until pending candidates are known, and only a successful + # enumeration can unblock them. + enumeration_owed = self._coordinator._block_replay_enumeration_owed() + # A targeted ancestor re-drive (issue #190) must reach the outbox + # query even while a retained retry or queued live work exists -- + # those short-circuits are exactly what starved this path during the + # observed wedge -- so an armed request bypasses each of them below. + redrive_hashes = self._consume_ancestor_redrive_requests() + redrive_owed = bool(redrive_hashes) + # One flag for "this pass must reach and complete the outbox query". + # The startup gate and a targeted re-drive need the same enumeration + # semantics -- the short-circuit bypasses, the query budget, the + # keyset pagination, and the widening-window exit below all key off + # it together -- so the re-drive genuinely runs the code startup + # replay runs rather than a poll-budget approximation of it. + forced_enumeration = enumeration_owed or redrive_owed + with self._coordinator.lock: + if not forced_enumeration and self.retry_candidate is not None: + return 0 + # A live wakeup is already the lowest-latency route to qbitd. Never + # park it behind the outbox query that exists only to recover missing + # wakeups after queue pressure or restart. + queue_obj = self.candidate_queue + if ( + not forced_enumeration + and queue_obj is not None + and not queue_obj.empty() + ): + return 0 + if ( + not forced_enumeration + and not self._block_replay_candidate_queue.empty() + ): + return 0 + # A forced enumeration gates wedge resolution (startup: job + # issuance; re-drive: a stuck payout transition), so it runs with + # the landing-class budget instead of the poll budget (issue #188 + # fix 4); the periodic steady-state poll keeps the tight budget. The + # metrics class follows the budget so a slow or timed-out forced + # enumeration surfaces on the landing series the alerts watch. + replay_query_timeout = ( + self._coordinator._block_landing_db_timeout() + if forced_enumeration + else None + ) + replay_query_call_class = "landing" if forced_enumeration else "fast" + pending_rows = getattr( + self._coordinator.ledger, + "pending_block_candidate_rows", + None, + ) + fetch_durable_page: Callable[..., list[Any]] | None = None + # Issue #255: a ledger that can answer metadata-first pages is + # enumerated through them -- typed headers only, a hard row and byte + # cap per page, and exhaustion answered by the outbox rather than + # inferred from a short page. Bodies are hydrated one at a time when + # a descriptor is dequeued, never during enumeration. + pending_headers = getattr( + self._coordinator.ledger, + "pending_block_candidate_headers", + None, + ) + fetch_header_page: Callable[..., Any] | None = None + # A ledger whose row query was replaced by a cursorless double keeps + # exactly the windowed semantics its replacement models. + if callable(pending_headers) and ( + not callable(pending_rows) or _pending_rows_accepts_cursor(pending_rows) + ): + + def fetch_header_page( + limit: int, + *, + page: int, + after_cursor: object | None, + ) -> Any: + return self._coordinator._run_block_submitter_ledger_call( + ( + "replay-outbox-query", + limit, + page, + _block_replay_cursor_key(after_cursor), + ), + "replay-outbox-query", + lambda: pending_headers( + limit=limit, + after_cursor=after_cursor, + max_bytes=HEADER_PAGE_MAX_BYTES, + ), + timeout_seconds=replay_query_timeout, + call_class=replay_query_call_class, + ) + + def fetch_durable_rows(limit: int) -> list[Any]: + header_limit = min(int(limit), HEADER_PAGE_MAX_ROWS) + header_page = self._coordinator._run_block_submitter_ledger_call( + ("replay-outbox-query", header_limit), + "replay-outbox-query", + lambda: pending_headers( + limit=header_limit, + after_cursor=None, + max_bytes=HEADER_PAGE_MAX_BYTES, + ), + timeout_seconds=replay_query_timeout, + call_class=replay_query_call_class, + ) + return [_collapse_row_from_header(row) for row in header_page.rows] + + elif callable(pending_rows): + + def fetch_durable_rows(limit: int) -> list[Any]: + return self._coordinator._run_block_submitter_ledger_call( + ("replay-outbox-query", limit), + "replay-outbox-query", + # Restore a batch with no per-row database work. In-flight + # dedupe lets later rows reach qbitd even while the oldest + # candidate is still accounting. + lambda: pending_rows(limit=limit), + timeout_seconds=replay_query_timeout, + call_class=replay_query_call_class, + ) + + if _pending_rows_accepts_cursor(pending_rows): + + def fetch_durable_page( + limit: int, + *, + page: int, + after_cursor: object | None, + ) -> list[Any]: + return self._coordinator._run_block_submitter_ledger_call( + # The page ordinal and its cursor belong in the + # dedupe key: a timed-out call stays registered for + # the next paced retry to reuse, and reusing page + # N's in-flight call to answer page N+1 would + # silently drop a whole page of pending candidates + # from the enumeration. + ( + "replay-outbox-query", + limit, + page, + _block_replay_cursor_key(after_cursor), + ), + "replay-outbox-query", + lambda: pending_rows( + limit=limit, + after_cursor=after_cursor, + ), + timeout_seconds=replay_query_timeout, + call_class=replay_query_call_class, + ) + + else: + pending = getattr( + self._coordinator.ledger, + "pending_block_candidates", + None, + ) + if not callable(pending): + self._coordinator._clear_block_replay_enumeration_owed() + return 0 + + def fetch_durable_rows(limit: int) -> list[Any]: + pending_intents = self._coordinator._run_block_submitter_ledger_call( + ("replay-outbox-query", limit), + "replay-outbox-query", + lambda: pending(limit=limit), + timeout_seconds=replay_query_timeout, + call_class=replay_query_call_class, + ) + return [ + { + "block_hash": ( + intent.get("block_hash_hex", "") + if isinstance(intent, Mapping) + else "" + ), + "candidate": intent, + } + for intent in pending_intents + ] + + queued = 0 + enumeration_truncated = False + enumeration_paginated = False + # One chain-height budget for the whole walk -- both its remaining + # probes and the reads they bought -- so a backlog split across fifty + # pages costs the node what a single page does. + collapse_probe_budget = _CollapseHeightProbeBudget() + if forced_enumeration and ( + fetch_header_page is not None or fetch_durable_page is not None + ): + # A row can embed an entire payout window. Smaller pages reduce + # the uninterrupted JSON decode without weakening the complete + # cursor walk, the legacy window cap, or cleanup admission bounds. + block_config = getattr( + getattr(self._coordinator, "config", None), "block", None + ) + configured_page_size = getattr( + block_config, "replay_page_size", MAX_BLOCK_REPLAY_ENUMERATION_ROWS + ) + page_size = min( + MAX_BLOCK_REPLAY_ENUMERATION_ROWS, + max(1, int(configured_page_size)), + ) + # Header pages honour the configured size and, on top of it, + # the hard #255 row cap; the byte cap is enforced by the ledger. + # Neither bound weakens the completeness proof: the page reports + # exhaustion explicitly, so a page cut short by either cap is + # never mistaken for the end of the backlog. + header_page_size = min(page_size, HEADER_PAGE_MAX_ROWS) + # Pagination, not a widening window: the doubling loop below + # fails closed once one page would have to hold the entire + # backlog, so a backlog larger than the cap kept enumeration + # owed (and every job build blocked) until it drained under the + # cap on its own. A keyset cursor walks a backlog of any size + # with the same bounded per-query cost, and only a page proven + # short ends the walk. + page = 0 + after_cursor: object | None = None + while True: + page += 1 + header_page: Any = None + if fetch_header_page is not None: + header_page = fetch_header_page( + header_page_size, + page=page, + after_cursor=after_cursor, + ) + durable_rows = [ + _collapse_row_from_header(row) for row in header_page.rows + ] + else: + try: + durable_rows = fetch_durable_page( + page_size, + page=page, + after_cursor=after_cursor, + ) + except TypeError: + if page > 1: + # Cursor support was already proven by an earlier + # page, so this is a real fault and not a legacy + # ledger; adopted rows must not be re-adopted by a + # fallback pass that starts over from the top. + raise + # A ledger without cursor support keeps exactly today's + # windowed semantics, fail-closed truncation included. + break + enumeration_paginated = True + # Collapse this page's decided-height siblings before any of + # it is adopted, so a row this apply terminalizes is never + # queued and never re-arms a payout barrier. The cursor and + # short-page proof stay bound to the *fetched* page: what the + # collapse removes was removed from the outbox too, so the + # enumeration is still complete. + backpressure_before = ( + self._collapse_cleanup_backpressure_engagement_count() + ) + retained_rows = self._collapse_superseded_block_candidates( + durable_rows, + timeout_seconds=replay_query_timeout, + call_class=replay_query_call_class, + probe_budget=collapse_probe_budget, + ) + cleanup_backpressured = ( + self._collapse_cleanup_backpressure_engagement_count() + != backpressure_before + ) + queued += self._adopt_durable_block_candidate_rows(retained_rows) + if self._block_replay_backpressure_drain_pending: + # A descriptor was withheld before its floor/preview was + # adopted. Even an exhausted SQL page is incomplete here. + enumeration_truncated = True + self._coordinator._note_block_replay_enumeration_owed() + break + print( + "prism coordinator: pending block candidate enumeration " + f"page={page} rows={len(durable_rows)} " + f"limit={header_page_size if header_page is not None else page_size}", + flush=True, + ) + if header_page is not None: + if header_page.exhausted: + # The outbox answered that no pending row followed + # this page, which is the completeness the job-build + # gate waits on. A short byte-capped page does not + # say that, and does not end the walk. + break + elif len(durable_rows) < page_size: + # A short page proves no pending row followed it at query + # time, which is the completeness the job-build gate waits + # on. + break + if cleanup_backpressured: + # A full preserved page may have more pending rows behind + # it. Stop here rather than materializing every later page + # in the unbounded replay queue; the enumeration stays + # owed and resumes only after this bounded page drains. + enumeration_truncated = True + self._block_replay_backpressure_drain_pending = True + print( + "prism coordinator: pending block candidate " + f"enumeration page={page} paused after cleanup " + "backpressure; replay queue drains before enumeration " + "resumes and job builds stay blocked until a complete " + "enumeration succeeds", + flush=True, + ) + break + if self._block_replay_should_yield_to_live_candidates( + collapse_probe_budget + ): + enumeration_truncated = True + print( + "prism coordinator: pending block candidate " + f"enumeration page={page} yielded to a waiting live " + "candidate; job builds stay blocked until a complete " + "enumeration succeeds", + flush=True, + ) + break + if header_page is not None: + next_cursor = header_page.next_cursor + else: + next_cursor = ( + durable_rows[-1].get("cursor") + if durable_rows and isinstance(durable_rows[-1], dict) + else None + ) + if next_cursor is None or next_cursor == after_cursor: + # Either the ledger accepted the keyword without keying + # its rows, or it accepted the cursor without honouring + # it (a **kwargs double, say) and re-served the same + # page. Both leave the walk unable to advance, so fail + # closed instead of looping forever or declaring an + # unproven enumeration complete. + enumeration_truncated = True + print( + "prism coordinator: pending block candidate " + f"enumeration page={page} did not advance its " + "cursor; job builds stay blocked until a complete " + "enumeration succeeds", + flush=True, + ) + break + after_cursor = next_cursor + if not enumeration_paginated: + enumeration_limit = MAX_PENDING_BLOCK_CANDIDATES + while True: + durable_rows = fetch_durable_rows(enumeration_limit) + # Same collapse-then-adopt order on the legacy widening + # window; the truncation proof below still measures the + # fetched window, not the surviving remainder. + backpressure_before = ( + self._collapse_cleanup_backpressure_engagement_count() + ) + retained_rows = self._collapse_superseded_block_candidates( + durable_rows, + timeout_seconds=replay_query_timeout, + call_class=replay_query_call_class, + probe_budget=collapse_probe_budget, + ) + cleanup_backpressured = ( + self._collapse_cleanup_backpressure_engagement_count() + != backpressure_before + ) + queued += self._adopt_durable_block_candidate_rows(retained_rows) + if self._block_replay_backpressure_drain_pending: + # A descriptor was withheld before its floor/preview was + # adopted. Even an exhausted SQL page is incomplete here. + enumeration_truncated = True + self._coordinator._note_block_replay_enumeration_owed() + break + if len(durable_rows) < enumeration_limit or not forced_enumeration: + # A short page proves no further pending row existed at + # query time -- the completeness the re-drive's + # stale-transition sweep below also relies on. + break + if cleanup_backpressured: + enumeration_truncated = True + self._block_replay_backpressure_drain_pending = True + print( + "prism coordinator: pending block candidate " + f"enumeration at {enumeration_limit} rows paused after " + "cleanup backpressure; replay queue drains before " + "enumeration resumes and job builds stay blocked until " + "a complete enumeration succeeds", + flush=True, + ) + break + if self._block_replay_should_yield_to_live_candidates( + collapse_probe_budget + ): + enumeration_truncated = True + print( + "prism coordinator: pending block candidate " + f"enumeration at {enumeration_limit} rows yielded to " + "a waiting live candidate; job builds stay blocked " + "until a complete enumeration succeeds", + flush=True, + ) + break + # A full batch may hide more pending rows, and a hidden row could + # be the active parent whose carry a child job must not omit. + # Re-query with a doubled window until the result is provably + # untruncated; in-flight dedupe makes re-seen rows free. + if enumeration_limit >= MAX_BLOCK_REPLAY_ENUMERATION_ROWS: + enumeration_truncated = True + print( + "prism coordinator: pending block candidate enumeration " + f"still truncated at {enumeration_limit} rows; job builds " + "stay blocked until a complete enumeration succeeds", + flush=True, + ) + break + enumeration_limit = min( + enumeration_limit * 2, + MAX_BLOCK_REPLAY_ENUMERATION_ROWS, + ) + if not enumeration_truncated: + # Every pending candidate is now known and its payout barrier armed + # (or quarantined), so child job builds may proceed. A truncated + # pass instead leaves enumeration owed: the queued batch drains, + # and the submitter loop re-enumerates the remainder. + self._coordinator._clear_block_replay_enumeration_owed() + self._coordinator._record_startup_phase_once("block_replay_enumerated") + self._conclude_ancestor_redrive_pass(redrive_hashes, enumeration_truncated) + if queued: + print( + f"prism coordinator: replayed {queued} pending block candidate(s)", + flush=True, + ) + return queued + + def _run_one_invalid_block_candidate_quarantine(self) -> bool: + self._ensure_block_replay_state() + try: + item = self._block_quarantine_queue.get_nowait() + except queue.Empty: + return False + block_hash, error = item[0], item[1] + pending_share = item[2] if len(item) > 2 else None + completed = False + try: + self._coordinator._record_block_submitter_phase("replay-quarantine") + quarantine = getattr( + self._coordinator.ledger, + "mark_block_candidate_abandoned", + None, + ) + if callable(quarantine): + quarantined = self._coordinator._run_block_submitter_ledger_call( + ("replay-quarantine", block_hash), + "replay-quarantine", + lambda: quarantine(block_hash=block_hash, error=error), + ) + self._coordinator._clear_accepted_block_payout_preview(block_hash) + if pending_share is not None: + # The row's durable credit holder was adopted before the + # failure. Its outbox row is now terminal, so a later + # successful replay re-creates a fresh holder; release + # this one instead of clamping snapshot anchors forever. + self._coordinator._finish_pending_share_candidate(pending_share) + if quarantined: + self._coordinator._clear_block_candidate_retry_state(block_hash) + self._coordinator._discard_outstanding_block_candidate(block_hash) + with self._coordinator.lock: + self.poisoned = int(self.poisoned) + 1 + completed = True + return True + except Exception: + print( + "prism coordinator: invalid candidate quarantine failed " + f"hash={block_hash}", + flush=True, + ) + traceback.print_exc() + return True + finally: + self._block_quarantine_queue.task_done() + if completed: + with self._coordinator.lock: + self._block_quarantine_hashes.discard(block_hash) + elif not self._coordinator.stop_event.is_set(): + self._block_quarantine_queue.put_nowait( + (block_hash, error, pending_share) + ) + + # -- retry state ------------------------------------------------------- + + def wait_for_retry(self, delay_seconds: float) -> bool: + """Wait for intentional backoff without impersonating stuck work. + + Retry waits heartbeat in bounded slices. Direct outbox calls and lock + admission use the same phase-aware pattern; work that is not covered + by an explicit deadline remains watchdog-eligible. + """ + delay_seconds = max(0.0, float(delay_seconds)) + if delay_seconds <= 0: + return self._coordinator.stop_event.is_set() + started = time.monotonic() + with self._state_lock: + self._backoff_started_monotonic = started + self._backoff_deadline_monotonic = started + delay_seconds + self._backoff_delay_seconds = delay_seconds + remaining = delay_seconds + try: + while remaining > 0: + self._coordinator._record_block_submitter_wait("retry-backoff") + wait_slice = min(remaining, self._block_work_wait_slice()) + if self._coordinator.stop_event.wait(wait_slice): + return True + remaining = max(0.0, remaining - wait_slice) + self._coordinator._record_block_submitter_wait("retry-backoff:complete") + return False + finally: + with self._state_lock: + self._backoff_started_monotonic = None + self._backoff_deadline_monotonic = None + self._backoff_delay_seconds = 0.0 + + def backoff_snapshot(self) -> tuple[bool, float, float]: + now = time.monotonic() + with self._state_lock: + deadline = self._backoff_deadline_monotonic + return ( + deadline is not None, + max(0.0, deadline - now) if deadline is not None else 0.0, + self._backoff_delay_seconds, + ) + + def block_submit_seconds_snapshot(self) -> tuple[dict[float, int], float, int]: + """Copied landed-to-RPC histogram state for metrics rendering.""" + with self._block_submit_metrics_lock: + histogram = self.block_submit_seconds_histogram + return ( + dict(histogram["buckets"]), + float(histogram["sum"]), + int(histogram["count"]), + ) + + def accepted_block_preview_publication_snapshot( + self, + ) -> dict[str, dict[str, Any]]: + """Copied acceptance-to-preview-publication histograms, by result.""" + with self._accepted_block_preview_publication_lock: + return { + result: { + "buckets": dict(histogram["buckets"]), + "sum": float(histogram["sum"]), + "count": int(histogram["count"]), + } + for result, histogram in ( + self.accepted_block_preview_publication_seconds_histogram + ).items() + } + + def next_retry_delay(self, block_hash: str) -> float: + initial = max(0.0, float(self.retry_initial_seconds)) + maximum = max(initial, float(self.retry_max_seconds)) + with self._coordinator.lock: + delays = self.retry_delays + if delays is None: + delays = {} + self.retry_delays = delays + delay = float(delays.get(block_hash, initial)) + delays[block_hash] = min(maximum, max(initial, delay * 2)) + return min(delay, maximum) + + def clear_retry_state(self, block_hash: str) -> None: + with self._coordinator.lock: + delays = self.retry_delays + if delays is not None: + delays.pop(block_hash, None) + landing_timeouts = getattr(self, "_block_landing_timeout_counts", None) + if landing_timeouts is not None: + landing_timeouts.pop(block_hash, None) + not_before = getattr( + self, + "_block_candidate_retry_not_before", + None, + ) + if not_before is not None: + not_before.pop(block_hash, None) + retained = getattr( + self, + "_block_candidate_retained_node_submissions", + None, + ) + if retained is not None: + retained.pop(str(block_hash).lower(), None) + stamped = getattr( + self, + "_block_candidate_retained_submission_monotonic", + None, + ) + if stamped is not None: + stamped.pop(str(block_hash).lower(), None) + + def mark_attempted(self, block_hash: str) -> None: + mark_attempted = getattr( + self._coordinator.ledger, + "mark_block_candidate_attempted", + None, + ) + if callable(mark_attempted): + self._coordinator._run_block_submitter_ledger_call( + ("mark-attempted", block_hash), + "mark-attempted", + lambda: mark_attempted(block_hash=block_hash), + ) + + def _merge_block_candidate_retry_locked( + self, + attribute: str, + candidate: PrismBlockCandidate, + ) -> None: + """Merge one retry by parent-first order. Caller holds the runtime lock.""" + candidate_height = int(candidate.context.template["height"]) + candidate_hash = str(candidate.submission.block_hash_hex).lower() + existing = getattr(self, attribute, None) + if existing is None: + setattr(self, attribute, candidate) + return + existing_height = int(existing.context.template["height"]) + existing_hash = str(existing.submission.block_hash_hex).lower() + if candidate_hash == existing_hash: + setattr(self, attribute, candidate) + if existing is not candidate: + # The newer same-hash object takes the slot; the displaced + # duplicate is dropped and carries its own floor holder. + self._release_dropped_duplicate_candidate_floor(existing) + return + if attribute != "_retry_block_candidate": + if candidate_height < existing_height: + setattr(self, attribute, candidate) + return + + # The raw lane has one parent-first head slot, but every displaced + # hash still needs an in-memory wakeup. Durable replay dedupe keeps a + # replayed descendant marked in-flight, so relying on a later outbox + # scan here could otherwise suppress it forever. + self._ensure_block_candidate_disposition_state() + waiting = self._block_disposition_waiting_retries + if candidate_height < existing_height: + waiting[existing_hash] = existing + setattr(self, attribute, candidate) + else: + waiting[candidate_hash] = candidate + + def retain_for_retry(self, candidate: PrismBlockCandidate) -> None: + """Keep the oldest unresolved candidate ahead of queued descendants.""" + candidate_hash = str(candidate.submission.block_hash_hex).lower() + # A retained candidate will be re-disposed, so the disposition seal + # (which stopped tip-observation matching at a terminal commit) no + # longer applies: the terminal work did not complete. Re-register + # immediately -- not at the next disposition -- so acceptance + # evidence arriving during the retry backoff is not lost. + self._coordinator._register_outstanding_block_candidate(candidate_hash) + with self._coordinator.lock: + self.retries = int(self.retries) + 1 + accounting_owner = ( + threading.get_ident() + == getattr(self, "_block_accounting_thread_ident", None) + and bool( + getattr( + self, + "_block_accounting_holds_disposition", + False, + ) + ) + ) + retry_attribute = ( + "_block_accounting_deferred_retry_candidate" + if accounting_owner + else "_retry_block_candidate" + ) + self._merge_block_candidate_retry_locked( + retry_attribute, + candidate, + ) + + def _pace_block_candidate_retry(self, block_hash: str) -> None: + """Apply per-candidate retry backoff without convoying accounting. + + On the block_accounting thread the disposition lease and writer + admission stay held until the accounting task's finally clause, so + sleeping here would stall every queued accounting task and keep an + armed payout barrier blocking balance mutation for the whole backoff + window. Record a not-before deadline instead; the dequeue path honors + it, and replay_pending_block_candidates already short-circuits while + the retained candidate occupies the retry slot. + """ + delay_seconds = self._coordinator._next_block_candidate_retry_delay(block_hash) + accounting_owner = ( + threading.get_ident() + == getattr(self, "_block_accounting_thread_ident", None) + and bool( + getattr( + self, + "_block_accounting_holds_disposition", + False, + ) + ) + ) + if not accounting_owner: + self._coordinator._wait_for_block_candidate_retry(delay_seconds) + return + with self._coordinator.lock: + not_before = getattr( + self, + "_block_candidate_retry_not_before", + None, + ) + if not_before is None: + not_before = {} + self._block_candidate_retry_not_before = not_before + not_before[str(block_hash).lower()] = ( + time.monotonic() + delay_seconds + ) + + def _block_candidate_retry_ready_locked( + self, + candidate: PrismBlockCandidate, + ) -> bool: + """Return whether a parked retry's backoff deadline has passed. + + Caller holds the runtime lock. A ready entry is dropped so a candidate + that later lands terminally leaves no stale pacing behind. + """ + not_before = getattr(self, "_block_candidate_retry_not_before", None) + if not not_before: + return True + block_hash = str(candidate.submission.block_hash_hex).lower() + deadline = not_before.get(block_hash) + if deadline is None: + return True + if time.monotonic() < deadline: + return False + not_before.pop(block_hash, None) + return True + + # -- deadline-classed ledger call workers ------------------------------ + + def _ensure_block_submitter_ledger_call_state(self) -> None: + if not hasattr(self, "_block_submitter_ledger_calls_lock"): + self._block_submitter_ledger_calls_lock = threading.Lock() + if not hasattr(self, "_block_submitter_ledger_calls"): + self._block_submitter_ledger_calls = {} + if not hasattr(self, "_block_submitter_ledger_worker_slots"): + self._block_submitter_ledger_worker_slots = threading.BoundedSemaphore( + MAX_BLOCK_SUBMITTER_STUCK_CALL_WORKERS + ) + + def _block_submitter_db_timeout(self) -> float: + return max( + 0.001, + float( + getattr( + self._coordinator, + "block_submit_db_timeout_seconds", + DEFAULT_BLOCK_SUBMIT_DB_TIMEOUT_SECONDS, + ) + ), + ) + + def _block_landing_watchdog_ceiling(self) -> float: + """Largest landing budget the configured watchdog can tolerate. + + Landing-class work runs on the block-work thread the watchdog + monitors, so the landing budget and the watchdog tolerance are one + system, not two independent settings. Deriving the ceiling from the + configured tolerance keeps them in step through every override + instead of pinning a second literal that silently goes stale. + + A deployment that turned the watchdog off has no hard-exit hazard to + stay under, and clamping it anyway would cost the operator budget for + no safety at all -- so that case gets no ceiling. The attribute is + read defensively and defaults to *enabled*: clamping is the safe + answer when the setting cannot be determined. + """ + if not bool(getattr(self._coordinator, "watchdog_enabled", True)): + return float("inf") + return max( + 0.001, + float( + getattr( + self._coordinator, + "watchdog_timeout_seconds", + DEFAULT_PRISM_WATCHDOG_TIMEOUT_SECONDS, + ) + ) + * BLOCK_LANDING_DB_TIMEOUT_WATCHDOG_FRACTION, + ) + + def _note_block_landing_budget_clamped( + self, + *, + configured_base: float, + configured_cap: float, + ceiling: float, + ) -> None: + """Say once that the watchdog is granting less than was configured. + + Without this the operator's configured landing budget is quietly + reduced -- at the 120s default tolerance the reviewed 120s cap + becomes 60s -- and the two states an operator most needs to tell + apart become indistinguishable in the logs: escalation exhausted at + the configured cap, versus escalation that never reached it because + the ceiling stopped it first. Emitted once per process because + _block_landing_db_timeout runs on every landing attempt and a + per-attempt line would bury the landing's own diagnostics; the flag + flips under the coordinator lock so concurrent first landings still + print exactly one line. The print stays outside that lock -- no I/O + under a coordinator-wide lock on the landing path. + """ + with self._coordinator.lock: + if getattr(self, "_block_landing_budget_clamp_logged", False): + return + self._block_landing_budget_clamp_logged = True + watchdog_seconds = float( + getattr( + self._coordinator, + "watchdog_timeout_seconds", + DEFAULT_PRISM_WATCHDOG_TIMEOUT_SECONDS, + ) + ) + print( + "prism coordinator: landing db budget clamped by watchdog " + f"configured_base={configured_base:g}s " + f"configured_max={configured_cap:g}s " + f"granted_base={min(configured_base, ceiling):g}s " + f"granted_max={min(configured_cap, ceiling):g}s " + f"ceiling={ceiling:g}s " + f"watchdog_timeout={watchdog_seconds:g}s " + f"fraction={BLOCK_LANDING_DB_TIMEOUT_WATCHDOG_FRACTION:g}", + flush=True, + ) + + def _block_landing_db_timeout(self, block_hash: str | None = None) -> float: + """Landing-class deadline, escalated after observed landing timeouts. + + The first attempt already receives the full landing budget; a known + landing-class operation never begins at the one-second poll budget. + Escalation doubles per timed-out landing attempt for the same block + hash up to the reviewed cap. + + The reviewed cap is an upper bound, not the granted budget: every + value here is clamped to the watchdog-derived ceiling, which can only + lower it. Landing steps are spent on the watchdog-monitored block-work + thread, so an escalated budget that outruns the watchdog tolerance is + not a longer attempt but a hard exit mid-landing -- and because the + escalation counts live only in memory, the restart drops back to the + base budget and repeats the same doomed cycle (issue #125). At the + 120s default tolerance the ceiling is 60s and escalation runs + 30s -> 60s; at the 300s production tolerance the ceiling is 150s and + the configured 120s cap is unchanged. A deployment running with the + watchdog disabled has no ceiling at all and keeps its configured + values in full. Any clamp is announced once (see + _note_block_landing_budget_clamped) so a reduced budget is never a + silent behavior change. + """ + ceiling = self._block_landing_watchdog_ceiling() + configured_base = max( + 0.001, + float( + getattr( + self._coordinator, + "block_landing_db_timeout_seconds", + DEFAULT_BLOCK_LANDING_DB_TIMEOUT_SECONDS, + ) + ), + ) + configured_cap = max( + configured_base, + float( + getattr( + self._coordinator, + "block_landing_db_timeout_max_seconds", + DEFAULT_BLOCK_LANDING_DB_TIMEOUT_MAX_SECONDS, + ) + ), + ) + base = min(configured_base, ceiling) + cap = min(configured_cap, ceiling) + if base < configured_base or cap < configured_cap: + # An infinite ceiling (watchdog disabled) can never compare below + # a finite configured value, so a deployment that opted out of the + # watchdog never reaches this line. + self._note_block_landing_budget_clamped( + configured_base=configured_base, + configured_cap=configured_cap, + ceiling=ceiling, + ) + timeouts = 0 + if block_hash is not None: + with self._coordinator.lock: + counts = getattr(self, "_block_landing_timeout_counts", None) + if counts is not None: + timeouts = int(counts.get(block_hash, 0)) + return min(cap, base * (2.0 ** min(timeouts, 8))) + + def _note_block_landing_timeout(self, block_hash: str | None) -> None: + if block_hash is None: + return + with self._coordinator.lock: + counts = getattr(self, "_block_landing_timeout_counts", None) + if counts is None: + counts = {} + self._block_landing_timeout_counts = counts + counts[block_hash] = int(counts.get(block_hash, 0)) + 1 + + def _ensure_block_ledger_call_metrics(self) -> None: + if not hasattr(self, "_block_ledger_call_metrics_lock"): + self._block_ledger_call_metrics_lock = threading.Lock() + if not hasattr(self, "_block_ledger_call_metrics"): + self._block_ledger_call_metrics = {} + + def _record_block_ledger_call( + self, + *, + call_class: str, + budget_seconds: float, + duration_seconds: float, + timed_out: bool, + ) -> None: + """Track per-call-class submitter ledger latency and timeout counts.""" + self._ensure_block_ledger_call_metrics() + with self._block_ledger_call_metrics_lock: + stats = self._block_ledger_call_metrics.setdefault( + call_class, + { + "calls_total": 0, + "timeouts_total": 0, + "last_budget_seconds": 0.0, + "last_duration_seconds": 0.0, + "max_duration_seconds": 0.0, + }, + ) + stats["calls_total"] = int(stats["calls_total"]) + 1 + if timed_out: + stats["timeouts_total"] = int(stats["timeouts_total"]) + 1 + stats["last_budget_seconds"] = float(budget_seconds) + stats["last_duration_seconds"] = float(duration_seconds) + stats["max_duration_seconds"] = max( + float(stats["max_duration_seconds"]), float(duration_seconds) + ) + + def block_ledger_call_class_metrics(self) -> dict[str, dict[str, float | int]]: + self._ensure_block_ledger_call_metrics() + with self._block_ledger_call_metrics_lock: + return { + call_class: dict(stats) + for call_class, stats in self._block_ledger_call_metrics.items() + } + + def _block_submitter_stuck_call_exit_timeout(self) -> float: + return max( + 0.001, + float( + getattr( + self._coordinator, + "block_submit_stuck_call_exit_seconds", + DEFAULT_BLOCK_SUBMIT_STUCK_CALL_EXIT_SECONDS, + ) + ), + ) + + def _maybe_restart_for_stuck_block_call( + self, + *, + kind: str, + started_monotonic: float, + ) -> None: + """Fail stop when a poisoned worker pool stays exhausted.""" + age_seconds = max(0.0, time.monotonic() - started_monotonic) + exit_seconds = self._block_submitter_stuck_call_exit_timeout() + if age_seconds < exit_seconds: + return + stop_event = getattr(self._coordinator, "stop_event", None) + if stop_event is not None and stop_event.is_set(): + return + print( + "prism coordinator: block work call remained stuck; requesting " + f"restart kind={kind} age={age_seconds:.3f}s " + f"budget={exit_seconds:g}s", + flush=True, + ) + self._coordinator._fatal_exit_requested = True + self._coordinator.request_shutdown() + + def _maybe_restart_for_exhausted_block_call_pool( + self, + *, + kind: str, + calls_lock: threading.Lock, + calls: dict[Any, Any], + ) -> None: + """Age an exhausted pool even when retries reuse existing calls.""" + with calls_lock: + active_starts = [ + pending.started_monotonic + for pending in calls.values() + if not pending.done.is_set() + ] + if len(active_starts) < MAX_BLOCK_SUBMITTER_STUCK_CALL_WORKERS: + return + self._maybe_restart_for_stuck_block_call( + kind=kind, + started_monotonic=min(active_starts), + ) + + def _retire_finished_block_submitter_ledger_call( + self, + key: tuple[object, ...], + call: _BlockSubmitterLedgerCall, + ) -> None: + """Take one finished call out of the registry, by identity. + + A finished call has no live need: it is registered only so that a + same-key caller arriving while the worker is *still out* joins that + one call instead of starting a second one, and a worker that has + published ``done`` is not out any more. Leaving it behind would wait + on an invocation that may never come -- the exact key can be a whole + enumeration page of block hashes, and the late write itself is what + empties that page out of the outbox -- so the registry would keep + the completed call, and its hash tuple, for the life of the process. + + Both the worker and a waiter that observed the completion call this, + whichever gets there first; the identity check is what keeps either + of them from dropping a *different* call that has since taken the + key, and the ``done`` check is what keeps them from ever dropping a + worker that is still out. Waiters hold the call directly, so a + removal never costs anybody the result or error it is owed: it only + settles that the next caller for that key replays the idempotent + operation on a fresh call. + """ + with self._block_submitter_ledger_calls_lock: + if not call.done.is_set(): + return + if self._block_submitter_ledger_calls.get(key) is call: + del self._block_submitter_ledger_calls[key] + + @contextmanager + def _block_submitter_ledger_timeout_scope( + self, + timeout_seconds: float | None = None, + ) -> Iterator[None]: + """Apply the submitter's PostgreSQL deadline when the ledger supports it.""" + operation_timeout = getattr( + self._coordinator.ledger, + "operation_timeout", + None, + ) + if not callable(operation_timeout): + yield + return + with operation_timeout( + self._block_submitter_db_timeout() + if timeout_seconds is None + else timeout_seconds + ): + yield + + @contextmanager + def _block_work_ledger_progress_scope(self, phase: str) -> Iterator[None]: + """Keep a ledger admission wait visible to the block-work watchdog. + + A statement deadline only bounds work the server can cancel. Before + any SQL is sent the ledger must first win its own writer lock or read + semaphore, and that wait is local: no statement exists to cancel and + nothing reports until admission succeeds. The scopes below run on the + block-work owner thread the watchdog monitors, so an admission wait + that stamps nothing is indistinguishable from a wedged thread and can + cost the coordinator a hard exit while it is merely queued behind + another writer. + + Installing the ledger's progress hook makes that wait heartbeat in + watchdog-sized slices, using the same phase-stamping helper the + bounded-call wait loop already uses (it no-ops safely off the owner + thread). Ledgers predating the hook are left exactly as they were. + """ + operation_progress = getattr( + self._coordinator.ledger, + "operation_progress", + None, + ) + if not callable(operation_progress): + yield + return + with operation_progress( + lambda: self._coordinator._record_block_submitter_wait(phase), + slice_seconds=self._block_work_wait_slice(), + ): + yield + + @contextmanager + def _block_submitter_ledger_statement_timeout_scope(self) -> Iterator[None]: + """Give each post-submit ledger step a fresh short deadline.""" + statement_timeout = getattr( + self._coordinator.ledger, + "statement_timeout", + None, + ) + if callable(statement_timeout): + with self._block_work_ledger_progress_scope( + "wait-ledger-admission:submit" + ): + with statement_timeout(self._block_submitter_db_timeout()): + yield + return + # Duck-typed ledgers predating per-statement scopes still receive a + # bounded operation, even though their budget spans the whole tail. + with self._block_submitter_ledger_timeout_scope(): + yield + + @contextmanager + def _block_landing_ledger_statement_timeout_scope( + self, + block_hash: str | None = None, + ) -> Iterator[None]: + """Give each landing-class ledger step the landing budget. + + The accounting tail that lands an accepted block (persist, prior + balances, confirm, and the prepared-state rejection of a terminal + candidate) runs under this scope instead of the poll-class one. + Timed-out steps are recorded so the next attempt for the same block + hash escalates its budget; the ledger backends already guarantee + server-side cancellation completes and the pooled session is rolled + back or replaced before the paced retry re-enters here. + + The guarded body also runs node RPCs and audit/build work. Only a + ledger-originated deadline may escalate the next landing budget or + fire the landing-timeout alert: a node RPC timeout is not a database + cancellation, and escalating on it would page and widen PostgreSQL + deadlines for a database that never missed one. + """ + timeout_seconds = self._coordinator._block_landing_db_timeout(block_hash) + scope = getattr(self._coordinator.ledger, "statement_timeout", None) + if not callable(scope): + scope = getattr(self._coordinator.ledger, "operation_timeout", None) + started = time.monotonic() + timed_out = False + try: + if callable(scope): + with self._block_work_ledger_progress_scope( + "wait-ledger-admission:landing" + ): + with scope(timeout_seconds): + yield + else: + yield + except (LedgerOperationTimeout, BlockSubmitterDatabaseTimeout): + timed_out = True + self._coordinator._note_block_landing_timeout(block_hash) + raise + finally: + self._coordinator._record_block_ledger_call( + call_class="landing", + budget_seconds=timeout_seconds, + duration_seconds=max(0.0, time.monotonic() - started), + timed_out=timed_out, + ) + + def _run_block_submitter_ledger_call( + self, + key: tuple[object, ...], + phase: str, + operation: Callable[[], Any], + *, + timeout_seconds: float | None = None, + call_class: str = "fast", + ) -> Any: + """Run one direct outbox call without letting its driver wedge us. + + A timed-out call stays registered for as long as its worker is + still out, and the next paced retry joins that same call. This + bounds the coordinator-side wait without spawning an unbounded pile + of threads when a fake/misbehaving driver ignores the real + PostgreSQL statement deadline. Candidate outbox mutations are + idempotent, so a late completion converges with replay. + + The registry therefore holds in-flight calls and nothing else: the + worker retires its own entry the moment it publishes ``done`` (see + _retire_finished_block_submitter_ledger_call), so a key whose retry + never comes back cannot pin a completed call and its page of block + hashes for the life of the process. Waiters already parked on the + call hold it directly and still get its result or error; a caller + arriving after the removal simply replays the operation. + + call_class labels the per-class latency/timeout metrics and must + match the budget in use: a call given the landing deadline records + as "landing" so the landing-timeout alert covers it, instead of + inflating the fast-call budget gauge. + """ + if timeout_seconds is None: + timeout_seconds = self._block_submitter_db_timeout() + else: + timeout_seconds = max(0.001, float(timeout_seconds)) + self._ensure_block_submitter_ledger_call_state() + with self._block_submitter_ledger_calls_lock: + call = self._block_submitter_ledger_calls.get(key) + if call is None: + if not self._block_submitter_ledger_worker_slots.acquire( + blocking=False + ): + oldest_started = min( + ( + pending.started_monotonic + for pending in self._block_submitter_ledger_calls.values() + if not pending.done.is_set() + ), + default=time.monotonic(), + ) + self._maybe_restart_for_stuck_block_call( + kind="ledger-worker-pool", + started_monotonic=oldest_started, + ) + raise BlockSubmitterDatabaseTimeout( + f"{phase} could not acquire a bounded ledger worker" + ) + call = _BlockSubmitterLedgerCall() + self._block_submitter_ledger_calls[key] = call + + def run() -> None: + try: + with self._block_submitter_ledger_timeout_scope( + timeout_seconds + ): + call.result = operation() + except BaseException as exc: + call.error = exc + finally: + call.done.set() + # The slot goes back before the removal: a caller + # that arrives to find this key already retired must + # not then be told the bounded pool is exhausted. + self._block_submitter_ledger_worker_slots.release() + self._retire_finished_block_submitter_ledger_call( + key, + call, + ) + + threading.Thread( + target=run, + name=f"prism-block-ledger-{phase}", + daemon=True, + ).start() + + deadline = time.monotonic() + timeout_seconds + while not call.done.is_set(): + self._coordinator._record_block_submitter_wait(phase) + remaining = deadline - time.monotonic() + if remaining <= 0: + self._maybe_restart_for_exhausted_block_call_pool( + kind="ledger-worker-pool", + calls_lock=self._block_submitter_ledger_calls_lock, + calls=self._block_submitter_ledger_calls, + ) + print( + "prism coordinator: block submitter ledger phase timed out " + f"phase={phase} timeout={timeout_seconds:g}s", + flush=True, + ) + self._coordinator._record_block_ledger_call( + call_class=call_class, + budget_seconds=timeout_seconds, + duration_seconds=timeout_seconds, + timed_out=True, + ) + raise BlockSubmitterDatabaseTimeout( + f"{phase} exceeded {timeout_seconds:g}s" + ) + call.done.wait( + min( + remaining, + self._block_work_wait_slice(), + ) + ) + self._coordinator._record_block_submitter_wait(f"{phase}:complete") + # A server-side deadline normally completes the worker with a ledger + # timeout error before the coordinator-side wait expires. Every + # operation behind this wrapper is a database call, so a completed + # call carrying a timeout error is still a timed-out call for the + # per-class alert series. + self._coordinator._record_block_ledger_call( + call_class=call_class, + budget_seconds=timeout_seconds, + duration_seconds=max(0.0, time.monotonic() - call.started_monotonic), + timed_out=isinstance(call.error, TimeoutError), + ) + self._retire_finished_block_submitter_ledger_call(key, call) + if call.error is not None: + raise call.error + return call.result + + # -- submitblock transport --------------------------------------------- + + def _rpc_call_with_timeout( + self, + method: str, + params: list[object], + *, + timeout_seconds: float, + ) -> Any: + """Pass an explicit timeout to production RPCs and capable test doubles.""" + call = self._coordinator.rpc.call + supports_timeout = isinstance(self._coordinator.rpc, JsonRpc) + if not supports_timeout: + try: + parameters = inspect.signature(call).parameters.values() + supports_timeout = any( + parameter.name == "timeout" + or parameter.kind is inspect.Parameter.VAR_KEYWORD + for parameter in parameters + ) + except (TypeError, ValueError): + supports_timeout = False + if supports_timeout: + return call(method, params, timeout=timeout_seconds) + return call(method, params) + + def _ensure_block_submitter_rpc_call_state(self) -> None: + if not hasattr(self, "_block_submitter_rpc_calls_lock"): + self._block_submitter_rpc_calls_lock = threading.Lock() + if not hasattr(self, "_block_submitter_rpc_calls"): + self._block_submitter_rpc_calls = {} + if not hasattr(self, "_block_submitter_rpc_worker_slots"): + self._block_submitter_rpc_worker_slots = threading.BoundedSemaphore( + MAX_BLOCK_SUBMITTER_STUCK_CALL_WORKERS + ) + + def _run_submitblock_rpc_with_hard_deadline( + self, + *, + block_hash: str, + block_hex: str, + timeout_seconds: float, + ) -> Any: + """Bound wall time even when an RPC adapter ignores its timeout.""" + self._ensure_block_submitter_rpc_call_state() + with self._block_submitter_rpc_calls_lock: + call = self._block_submitter_rpc_calls.get(block_hash) + if call is None: + if not self._block_submitter_rpc_worker_slots.acquire( + blocking=False + ): + oldest_started = min( + ( + pending.started_monotonic + for pending in self._block_submitter_rpc_calls.values() + if not pending.done.is_set() + ), + default=time.monotonic(), + ) + self._maybe_restart_for_stuck_block_call( + kind="rpc-worker-pool", + started_monotonic=oldest_started, + ) + raise TimeoutError( + "submitblock could not acquire a bounded RPC worker" + ) + call = _BlockSubmitterRpcCall() + self._block_submitter_rpc_calls[block_hash] = call + + def run() -> None: + try: + call.result = self._rpc_call_with_timeout( + "submitblock", + [block_hex], + timeout_seconds=timeout_seconds, + ) + except BaseException as exc: + call.error = exc + finally: + call.done.set() + self._block_submitter_rpc_worker_slots.release() + + threading.Thread( + target=run, + name=f"prism-block-rpc-{block_hash[:12]}", + daemon=True, + ).start() + + deadline = time.monotonic() + timeout_seconds + while not call.done.is_set(): + self._coordinator._record_block_submitter_wait("submitblock-rpc") + remaining = deadline - time.monotonic() + if remaining <= 0: + self._maybe_restart_for_exhausted_block_call_pool( + kind="rpc-worker-pool", + calls_lock=self._block_submitter_rpc_calls_lock, + calls=self._block_submitter_rpc_calls, + ) + raise TimeoutError( + f"submitblock exceeded {timeout_seconds:g}s" + ) + call.done.wait( + min(remaining, self._block_work_wait_slice()) + ) + with self._block_submitter_rpc_calls_lock: + if self._block_submitter_rpc_calls.get(block_hash) is call: + self._block_submitter_rpc_calls.pop(block_hash, None) + if call.error is not None: + raise call.error + return call.result + + # -- node offer -------------------------------------------------------- + + def _arm_block_candidate_after_node_offer( + self, + candidate: PrismBlockCandidate, + node_submission: _BlockCandidateNodeSubmission, + ) -> None: + """Fence child payout work as soon as node acceptance is possible.""" + self._coordinator._stash_retained_block_candidate_node_submission( + str(candidate.submission.block_hash_hex), + node_submission, + ) + ambiguous_or_landed = ( + node_submission.error is not None + or node_submission.result in (None, "duplicate") + ) + if not ambiguous_or_landed: + self._coordinator._release_block_fast_lane_slot( + str(candidate.submission.block_hash_hex) + ) + return + block_hash = str(candidate.submission.block_hash_hex).lower() + expected_height = int(candidate.context.template["height"]) + self._coordinator._begin_accepted_block_payout_preview( + block_hash, + block_height=expected_height, + ) + + def _submit_block_candidate_to_node( + self, + candidate: PrismBlockCandidate, + ) -> _BlockCandidateNodeSubmission: + """Offer the durable candidate to qbitd before any accounting work.""" + block_hash = str(candidate.submission.block_hash_hex).lower() + self._coordinator._begin_accepted_block_payout_preview( + block_hash, + block_height=int(candidate.context.template["height"]), + ) + self._coordinator._register_outstanding_block_candidate(block_hash) + self._coordinator._record_block_submitter_phase("submitblock-rpc") + timeout_seconds = max( + 0.001, + float( + getattr( + self._coordinator, + "block_submit_rpc_timeout_seconds", + DEFAULT_BLOCK_SUBMIT_RPC_TIMEOUT_SECONDS, + ) + ), + ) + try: + result = self._coordinator._run_submitblock_rpc_with_hard_deadline( + block_hash=block_hash, + block_hex=candidate.submission.block_hex, + timeout_seconds=timeout_seconds, + ) + except BaseException as exc: + self._coordinator._record_block_submitter_phase("submitblock-rpc:error") + node_submission = _BlockCandidateNodeSubmission( + attempted=True, + error=exc, + ) + self._coordinator._arm_block_candidate_after_node_offer( + candidate, + node_submission, + ) + return node_submission + self._coordinator._record_block_submitter_phase("submitblock-rpc:complete") + landed_monotonic = getattr(candidate, "landed_monotonic", None) + if landed_monotonic is not None: + self._coordinator._observe_block_submit_seconds( + time.monotonic() - float(landed_monotonic) + ) + node_submission = _BlockCandidateNodeSubmission( + attempted=True, + result=result, + ) + if _is_definitive_node_acceptance(node_submission): + # Stamp the start of issue #181's acceptance-to-publication + # interval from the same predicate the lane routing and the + # retained-offer stash read, so "the node has this block" has + # exactly one definition in this module. + self._note_accepted_block_preview_acceptance(block_hash) + self._coordinator._arm_block_candidate_after_node_offer( + candidate, + node_submission, + ) + return node_submission + + def _node_submission_for_candidate( + self, + candidate: PrismBlockCandidate, + ) -> _BlockCandidateNodeSubmission: + """Choose the node fast lane unless the pool was already closed.""" + block_hash = str(candidate.submission.block_hash_hex).lower() + self._coordinator._record_block_submitter_phase("fast-lane-admission") + with self._coordinator.lock: + accounted_hashes = getattr( + self._coordinator, + "_accounted_accepted_block_hashes", + set(), + ) + accepted_count = int(self._coordinator.accepted_block_count) + pool_closed = block_hash not in accounted_hashes and ( + accepted_count >= int(self._coordinator.max_blocks) + or ( + bool(self._coordinator.stop_after_block) + and accepted_count >= 1 + ) + ) + if pool_closed: + return _BlockCandidateNodeSubmission(attempted=False) + return self._coordinator._submit_block_candidate_to_node(candidate) + + def _node_submission_for_candidate_or_retained( + self, + candidate: PrismBlockCandidate, + ) -> _BlockCandidateNodeSubmission: + """Reuse a retained definitive acceptance instead of re-offering. + + An in-process retry of a candidate whose earlier offer already + returned success must not ask the node again: the re-offer answers + "duplicate", which downgrades the classification to the moved live + tip and leans on chain probes that may be unavailable under the + same saturation that caused the retry. The stashed result reruns + the landing tail as if the first pass had continued. + """ + retained = self._coordinator._retained_block_candidate_node_submission( + str(candidate.submission.block_hash_hex) + ) + if retained is not None: + return retained + return self._coordinator._node_submission_for_candidate(candidate) + + def _node_submission_for_direct_candidate( + self, + candidate: PrismBlockCandidate, + ) -> _BlockCandidateNodeSubmission: + """Preserve active-replay semantics for non-queue embedders. + + The dedicated submitter always uses the unconditional fast lane. A + direct caller can instead be resuming a durable active ancestor, for + which another submit is unnecessary and some integrations do not + retain block bytes. This compatibility probe is not on the incident + queue-to-node path. + """ + block_hash = str(candidate.submission.block_hash_hex).lower() + expected_height = int(candidate.context.template["height"]) + try: + if str(self._coordinator.rpc.call("getbestblockhash")).lower() == block_hash: + return _BlockCandidateNodeSubmission(attempted=False) + except Exception: + pass + try: + if ( + self._coordinator.active_block_candidate_height(block_hash) + == expected_height + ): + return _BlockCandidateNodeSubmission(attempted=False) + except Exception: + pass + if not hasattr(candidate.submission, "block_hex"): + return _BlockCandidateNodeSubmission(attempted=False) + return self._coordinator._node_submission_for_candidate(candidate) + + def _account_block_candidate_after_node_submit( + self, + candidate: PrismBlockCandidate, + node_submission: _BlockCandidateNodeSubmission, + ) -> bool: + """Pass fast-lane evidence while tolerating legacy test embedders.""" + submit = self._coordinator.submit_block_candidate + supports_node_submission = True + try: + parameters = inspect.signature(submit).parameters.values() + supports_node_submission = any( + parameter.name == "node_submission" + or parameter.kind is inspect.Parameter.VAR_KEYWORD + for parameter in parameters + ) + except (TypeError, ValueError): + pass + if supports_node_submission: + return bool(submit(candidate, node_submission=node_submission)) + return bool(submit(candidate)) + + def _submit_synchronous_block_candidate( + self, + candidate: PrismBlockCandidate, + ) -> bool: + """Run the rare miner-facing path under one same-hash disposition.""" + coordinator = self._coordinator + block_hash = str(candidate.submission.block_hash_hex).lower() + with coordinator._block_candidate_disposition(block_hash): + terminal_outcome = coordinator._block_candidate_terminal_outcome( + block_hash + ) + if terminal_outcome is not None: + coordinator._finish_pending_share_commit(candidate.pending_share) + return terminal_outcome + outcome = self.outcome + # The accepted/rejected accounting tail may already be complete + # while only its durable outbox terminal update is retrying. A + # synchronous same-hash waiter must join that finalize-only state; + # another node offer/accounting pass could invert the outcome. + with coordinator.lock: + pending_finalize = self.finalize_retries.get(block_hash) + if pending_finalize is not None: + accepted, error = pending_finalize + coordinator._finalize_block_candidate( + candidate, + block_hash=block_hash, + accepted=accepted, + error=error, + outcome=outcome, + ) + return accepted + try: + if not coordinator._reserve_block_fast_lane_slot(block_hash): + raise RuntimeError( + "block candidate is waiting for fast-lane capacity" + ) + # Everything from definitive acceptance to here was queue, + # disposition and admission time the landing itself did not + # control (issue #224). A candidate whose earlier pass already + # stamped acceptance closes its lane here, before the retained + # offer is even looked up, so none of this landing's own work + # can reach ``lane_wait``. + self._note_accepted_landing_lane_start(candidate) + node_submission = coordinator._node_submission_for_candidate_or_retained(candidate) + # A *fresh* candidate has no stamp until the call above makes + # the offer, so the close before it found no interval to + # measure. Close again now that one may exist: it is a no-op + # once this interval's lane was recorded, so the retained case + # keeps the earlier, tighter sample and the fresh case reports + # the zero queue time it genuinely had rather than no sample + # at all. + self._note_accepted_landing_lane_start(candidate) + # Both closes sit above the attempt marker on purpose. The + # marker is this landing's own PostgreSQL write: charging it + # to ``lane_wait`` would grow the one stretch that exonerates + # the landing exactly when the writer is degraded, and a + # marker that timed out would keep growing it across the + # retry backoff. + coordinator._mark_block_candidate_attempted(block_hash) + with coordinator._block_landing_ledger_statement_timeout_scope(block_hash): + # The same-hash disposition is already held here, so the + # coordinator may run its serialized inner tail rather + # than the public entrypoint that would take that guard + # again. Which of the two applies is its call to make. + block_landed = self.ports.submit_candidate( + candidate, + node_submission=node_submission, + disposition_held=True, + ) + except BaseException: + coordinator._retain_block_candidate_for_retry(candidate) + raise + + if not block_landed: + reason = getattr(outcome, "reason", None) + if reason in {None, *self.retryable_reasons}: + coordinator._retain_block_candidate_for_retry(candidate) + raise RuntimeError( + "block candidate outcome is pending durable retry" + ) + abandon_error = getattr(outcome, "error", None) + try: + coordinator._record_block_submitter_phase( + "reject-prepared-block" + ) + with coordinator._block_landing_ledger_statement_timeout_scope( + block_hash + ): + coordinator._reject_terminal_prepared_block_candidate(candidate) + coordinator._record_block_submitter_phase( + "reject-prepared-block:complete" + ) + except Exception as exc: + # A prior attempt may have persisted prepared payout rows + # before this synchronous resubmit reached a false terminal + # verdict. Keep the outbox pending until those rows can be + # rejected; otherwise restart replay is removed while its + # balance transition remains live. + coordinator._defer_block_candidate( + "backend-rpc-unavailable", + "could not reject prepared state for terminal candidate", + worker=candidate.client.username or None, + ) + coordinator._retain_block_candidate_for_retry(candidate) + raise RuntimeError( + "could not reject prepared state for terminal candidate" + ) from exc + coordinator._finalize_block_candidate( + candidate, + block_hash=block_hash, + accepted=False, + error=str(abandon_error or reason), + outcome=outcome, + ) + return False + + coordinator._finalize_block_candidate( + candidate, + block_hash=block_hash, + accepted=True, + error="", + outcome=outcome, + ) + return True + + # -- same-hash disposition guard --------------------------------------- + + def _ensure_block_candidate_disposition_state(self) -> None: + """Backfill same-hash submission guards for lightweight embedders.""" + if ( + hasattr(self, "_block_candidate_disposition_registry_lock") + and hasattr(self, "_block_candidate_disposition_flights") + and hasattr(self, "_block_candidate_terminal_outcomes") + and hasattr(self, "_block_fast_lane_reservations") + and hasattr(self, "_block_disposition_waiting_retries") + and hasattr(self, "_block_candidate_dequeued_hashes") + ): + return + with _STATE_BACKFILL_LOCK: + if not hasattr(self, "_block_candidate_disposition_registry_lock"): + self._block_candidate_disposition_registry_lock = threading.Lock() + if not hasattr(self, "_block_candidate_disposition_flights"): + self._block_candidate_disposition_flights: dict[ + str, _BlockCandidateDispositionFlight + ] = {} + if not hasattr(self, "_block_candidate_terminal_outcomes"): + self._block_candidate_terminal_outcomes: dict[str, bool] = {} + if not hasattr(self, "_block_fast_lane_reservations"): + self._block_fast_lane_reservations: set[str] = set() + if not hasattr(self, "_block_disposition_waiting_retries"): + self._block_disposition_waiting_retries: dict[ + str, PrismBlockCandidate + ] = {} + if not hasattr(self, "_block_candidate_dequeued_hashes"): + # hash -> number of dequeued same-hash candidate objects that + # are between a lane and their disposition flight (or the + # waiting registry). Read and written under coordinator.lock. + self._block_candidate_dequeued_hashes: dict[str, int] = {} + + def _claim_block_candidate_disposition( + self, + block_hash: str, + *, + blocking: bool, + ) -> _BlockCandidateDispositionLease | None: + """Claim one hash without making unrelated node offers wait.""" + key = block_hash.lower() + self._ensure_block_candidate_disposition_state() + with self._block_submitter_lock( + self._block_candidate_disposition_registry_lock, + "candidate-disposition-registry", + ): + flight = self._block_candidate_disposition_flights.get(key) + if flight is None: + flight = _BlockCandidateDispositionFlight() + self._block_candidate_disposition_flights[key] = flight + flight.users += 1 + if blocking: + self._acquire_block_submitter_lock( + flight.lock, + f"candidate-disposition:{key}", + ) + acquired = True + else: + acquired = flight.lock.acquire(blocking=False) + if acquired: + return _BlockCandidateDispositionLease(key, flight) + self._drop_block_candidate_disposition_user(key, flight) + return None + + def _drop_block_candidate_disposition_user( + self, + key: str, + flight: _BlockCandidateDispositionFlight, + ) -> None: + with self._block_submitter_lock( + self._block_candidate_disposition_registry_lock, + "candidate-disposition-registry", + ): + flight.users -= 1 + if ( + flight.users == 0 + and self._block_candidate_disposition_flights.get(key) is flight + ): + self._block_candidate_disposition_flights.pop(key, None) + + def _release_block_candidate_disposition( + self, + lease: _BlockCandidateDispositionLease, + ) -> None: + lease.flight.lock.release() + self._drop_block_candidate_disposition_user( + lease.block_hash, + lease.flight, + ) + + # -- dequeued-hash pin ------------------------------------------------- + + def _pin_dequeued_block_candidate_locked( + self, + candidate: PrismBlockCandidate, + ) -> str: + """Name a candidate's hash from the instant it leaves its lane. + + Caller holds ``coordinator.lock`` and, in the same critical section, + has just taken ``candidate`` out of the retry holder, a candidate + queue, or the waiting registry. Until ``submit_next`` hands the hash + to its disposition flight or back to the waiting registry -- again + under the lock -- the candidate object lives only in that method's + local variable, and this pin is the only thing that tells the + terminal-outcome eviction a same-hash copy is still in memory. The + lanes and the flight each pin the hash in their own right; without + this entry a terminal duplicate's fence is the oldest unpinned + outcome exactly while the pass that is about to read it holds the + candidate in hand. + + Counted rather than set-valued so two dequeued same-hash objects + (a second caller of ``submit_next``) cannot unpin each other. + Returns the lowercase key the rest of the pass compares on. + """ + block_hash = str(candidate.submission.block_hash_hex).lower() + pins = self._block_candidate_dequeued_hashes + pins[block_hash] = pins.get(block_hash, 0) + 1 + return block_hash + + def _unpin_dequeued_block_candidate_locked(self, block_hash: str) -> None: + """Drop one dequeue's pin. Caller holds ``coordinator.lock``. + + Called only in the critical section that hands the hash on -- to the + flight the claim installed, or to the waiting registry -- or where + nothing needs it any more because the candidate object left with an + exception and no in-memory copy remains to offer the block. + """ + pins = self._block_candidate_dequeued_hashes + remaining = pins.get(block_hash, 0) - 1 + if remaining > 0: + pins[block_hash] = remaining + else: + pins.pop(block_hash, None) + + def _live_block_candidate_hash_registries(self) -> tuple[Any, ...]: + """Registries whose named hash forbids evicting its terminal outcome. + + Each one either holds a candidate object that has not reached the + node yet or the still-owed terminal work of one, and every such copy + reads the terminal outcome before it may offer the block again. + Dropping the outcome under one of them is exactly the window + ``_publish_collapsed_candidate_terminal_fence`` exists to close. + The dequeued-hash pin is the lane between lanes: the candidate + ``submit_next`` has taken out of a holder, queue, or the waiting + registry and not yet handed to its disposition flight. + + Read with ``coordinator.lock`` held. Backfilled state is fetched by + ``getattr`` rather than by an ``_ensure_...`` call, which would take + the state-backfill lock underneath the global one. + """ + return tuple( + registry + for registry in ( + getattr(self, "_outstanding_block_candidate_hashes", None), + getattr(self, "_block_replay_inflight_hashes", None), + getattr(self, "_block_quarantine_hashes", None), + getattr(self, "_block_fast_lane_reservations", None), + getattr(self, "_block_disposition_waiting_retries", None), + getattr(self, "finalize_retries", None), + getattr(self, "_block_candidate_retained_node_submissions", None), + getattr(self, "_block_candidate_collapse_cleanup_retries", None), + getattr(self, "_block_candidate_collapse_cleanup_inflight", None), + getattr(self, "_block_candidate_dequeued_hashes", None), + ) + if registry + ) + + def _held_block_candidate_retry_hashes(self) -> frozenset[str]: + """The hashes the single-slot retry holders are still carrying.""" + held: set[str] = set() + for holder in ( + getattr(self, "retry_candidate", None), + getattr(self, "_block_accounting_deferred_retry_candidate", None), + ): + key = _block_candidate_hash_of(holder) + if key is not None: + held.add(key) + return frozenset(held) + + @staticmethod + def _queued_block_candidate_hashes(queue_obj: object) -> frozenset[str] | None: + """The hashes still sitting in one candidate queue, or None if unread. + + The registries above stop naming a hash the moment its terminal + outcome is recorded -- ``_record_block_candidate_terminal_outcome`` + and the collapse cleanup's ``outstanding-and-tip-observation`` step + clear the outstanding and replay-inflight markers by design -- so a + *duplicate* copy of that hash still queued behind the pass that + finished would be named by nothing. That copy is precisely the one + the fence protects, so the queues themselves are read. + + The underlying deque is read under the queue's own mutex, because + iterating it while a producer or consumer mutates it is not safe. + The acquire is non-blocking: this runs with ``coordinator.lock`` + held, and waiting there for a queue's leaf lock would convoy the + whole coordinator behind an unrelated enqueue. Failing to take it + returns None and the caller skips the eviction pass entirely, which + only leaves the registry over its bound until the next outcome. + """ + if queue_obj is None: + return frozenset() + mutex = getattr(queue_obj, "mutex", None) + entries = getattr(queue_obj, "queue", None) + if mutex is None or entries is None: + return frozenset() + if not mutex.acquire(blocking=False): + return None + try: + items = list(entries) + finally: + mutex.release() + queued: set[str] = set() + for item in items: + key = _block_candidate_hash_of(item) + if key is not None: + queued.add(key) + return frozenset(queued) + + def _in_flight_block_candidate_hashes(self) -> frozenset[str] | None: + """The hashes with a live same-hash disposition flight, or None. + + A claimed flight means a pass is between its node offer and its + durable finalization for that hash. Same non-blocking rule as the + queues: the disposition registry lock is taken under + ``coordinator.lock`` here, the reverse of the ordinary order, so it + is only ever tried, never waited on. + """ + lock = getattr(self, "_block_candidate_disposition_registry_lock", None) + flights = getattr(self, "_block_candidate_disposition_flights", None) + if lock is None or flights is None: + return frozenset() + if not lock.acquire(blocking=False): + return None + try: + return frozenset(flights) + finally: + lock.release() + + def _collect_live_block_candidate_pins( + self, + ) -> tuple[tuple[Any, ...], frozenset[str]] | None: + """The one live pin set every hash-liveness proof reads. + + Returns the lane registries (checked by membership) and the union of + the retry-holder, queued (live and replay), and claimed-flight + hashes, or None when a leaf lane could not be read without waiting + under the global lock. Read with ``coordinator.lock`` held. Both the + terminal-outcome eviction and the ancestor re-drive's ownership + check (issue #190) sit on this collector, deliberately: each one's + safety argument is that a hash named by none of these sources has no + in-memory copy left, and a lane added to one consumer but not the + other would silently void the proof of whichever fell behind. + """ + pinned = self._live_block_candidate_hash_registries() + held = set(self._held_block_candidate_retry_hashes()) + for live in ( + self._queued_block_candidate_hashes( + getattr(self, "candidate_queue", None) + ), + self._queued_block_candidate_hashes( + getattr(self, "_block_replay_candidate_queue", None) + ), + self._in_flight_block_candidate_hashes(), + ): + if live is None: + return None + held |= live + return pinned, frozenset(held) + + @staticmethod + def _stamp_block_candidate_terminal_outcome( + outcomes: dict[str, bool], + block_hash: str, + accepted: bool, + ) -> None: + """Record one outcome as the newest entry of the FIFO registry. + + Re-recording an existing hash would otherwise keep its original + insertion position, leaving a freshly reasserted outcome at the + front of the eviction scan. + """ + outcomes.pop(block_hash, None) + outcomes[block_hash] = accepted + + def _bound_block_candidate_terminal_outcomes( + self, + protect: frozenset[str] = frozenset(), + ) -> int: + """Trim the terminal-outcome registry to its bound; return evictions. + + Called by the two writers with ``coordinator.lock`` already held and + their own entries already stamped. ``protect`` names the hashes this + very call published: a bulk fence page larger than the whole bound + must not evict the outcomes it just wrote, so the registry is left + temporarily over its bound instead. + + Eviction is oldest-first and skips every pinned hash, so the bound + is a target rather than a hard ceiling -- a registry whose entries + are all still live simply stays large, which is the safe direction. + The pins are themselves bounded state, so the total stays bounded. + + The pin set is the union of the lane registries (including the + dequeued-hash pin), the single-slot retry holders, the candidate + objects still queued on the live and replay lanes, and the hashes + with a claimed disposition flight. A hash held by none of those has + no in-memory copy left that could reach a node offer; anything + arriving for it afterwards is a fresh admission, which re-reads the + durable outbox row -- terminal there either way -- before it can + offer anything. + + That same proof retires the hash's counted-abandonment dedup key. + The key exists so a re-run terminal disposition -- a collapse whose + cleanup failed, a finalize-only replay -- counts its candidate once + rather than once per attempt, so it may only be dropped when no + copy is left to re-run anything. Both writer orderings are covered + by the one rule: the direct finalize path counts before it publishes + its terminal outcome, so its key is retired by a later pass over the + outcome it goes on to publish, and an ambiguous false finalize + failure has no outcome to evict at all while ``finalize_retries`` + pins it; the collapse publishes the outcome first and counts during + cleanup, and holds the hash's disposition lease -- an in-flight pin + -- across both. Dropping the key here therefore never lets a lane + that is still owed accounting count its candidate twice. + """ + outcomes = self._block_candidate_terminal_outcomes + overflow = len(outcomes) - MAX_BLOCK_CANDIDATE_TERMINAL_OUTCOMES + if overflow <= 0: + return 0 + pins = self._collect_live_block_candidate_pins() + if pins is None: + # A lane could not be read without waiting under the global + # lock. Evicting against an incomplete pin set could unfence + # a live copy, so nothing is evicted this round. + return 0 + pinned, held = pins + counted = getattr(self, "_counted_block_candidate_abandonments", None) + window = overflow + BLOCK_CANDIDATE_TERMINAL_OUTCOME_EVICTION_SCAN + dropped = 0 + for key in list(itertools.islice(outcomes, window)): + if dropped >= overflow: + break + if ( + key in protect + or key in held + or any(key in registry for registry in pinned) + ): + # A live, replayed, retried, parked, quarantined, or + # cleanup-owing copy of this hash is still in memory and + # will read this outcome before it could offer the block. + # Move it behind the scan window so it stops blocking the + # next pass instead of dropping it. + outcomes[key] = outcomes.pop(key) + continue + del outcomes[key] + if counted is not None: + # Retired under the very proof that let the fence go: a + # dedup key outliving its fence is what let this set grow + # without bound under a collapsed storm. + counted.discard(key) + dropped += 1 + return dropped + + def _block_candidate_terminal_outcome(self, block_hash: str) -> bool | None: + self._ensure_block_candidate_disposition_state() + with self._coordinator.lock: + return self._block_candidate_terminal_outcomes.get(block_hash.lower()) + + def _record_block_candidate_terminal_outcome( + self, + block_hash: str, + *, + accepted: bool, + ) -> None: + self._ensure_block_candidate_disposition_state() + dropped_waiting: PrismBlockCandidate | None = None + with self._coordinator.lock: + key = block_hash.lower() + self._stamp_block_candidate_terminal_outcome( + self._block_candidate_terminal_outcomes, + key, + accepted, + ) + self._bound_block_candidate_terminal_outcomes(frozenset((key,))) + self._block_fast_lane_reservations.discard(key) + replay_hashes = getattr(self, "_block_replay_inflight_hashes", None) + if replay_hashes is not None: + replay_hashes.discard(key) + waiting = getattr(self, "_block_disposition_waiting_retries", None) + if waiting is not None: + dropped_waiting = waiting.pop(key, None) + if dropped_waiting is not None: + # The parked same-hash wakeup dies with the terminal outcome + # recorded; its floor holder must not outlive it. + self._release_dropped_duplicate_candidate_floor(dropped_waiting) + + def _record_committed_block_candidate_abandonment( + self, + block_hash: str, + outcome: threading.local, + ) -> None: + """Count an abandonment only after its terminal cleanup is fixed. + + ``_abandon_block_candidate`` seals a proposed rejection before any + prepared payout rows are removed. That cleanup can fail, in which + case the candidate is deliberately re-registered and can still prove + accepted on a later pass. Counting at the seal would then expose the + same hash as both abandoned and accepted. The writer calls this only + after cleanup succeeds or after a false finalize-only disposition is + installed; direct accounting callers invoke it only after the full + serialized rejection path returns. + + The dedup key that makes this once-per-candidate is retired by + ``_bound_block_candidate_terminal_outcomes`` when that pass evicts + the hash's terminal outcome, which is the only place that has proved + no lane can re-enter this method for the hash. + """ + reason = getattr(outcome, "reason", None) + if not isinstance(reason, str) or not reason: + return + if reason in self.retryable_reasons: + return + # Terminal for this hash: no later attempt can close its interval, so + # the phase state it accumulated has no consumer left (issue #224). + self._discard_accepted_landing_attribution(block_hash) + stale_job_class = getattr(outcome, "stale_job_class", None) + key = block_hash.lower() + with self._coordinator.lock: + counted_abandonments = self._counted_block_candidate_abandonments + if key in counted_abandonments: + return + counted_abandonments.add(key) + counts = self.abandoned_counts + if counts is None: + counts = {} + self.abandoned_counts = counts + counts[reason] = int(counts.get(reason, 0)) + 1 + if stale_job_class is not None: + stale_counts = getattr(self, "stale_job_abandon_counts", None) + if stale_counts is None: + stale_counts = { + abandon_class: 0 + for abandon_class in PRISM_STALE_JOB_ABANDON_CLASSES + } + self.stale_job_abandon_counts = stale_counts + stale_counts[stale_job_class] = ( + int(stale_counts.get(stale_job_class, 0)) + 1 + ) + + def _reserve_block_fast_lane_slot(self, block_hash: str) -> bool: + """Reserve pool capacity while a node offer awaits terminal accounting.""" + key = block_hash.lower() + self._ensure_block_candidate_disposition_state() + with self._coordinator.lock: + reservations = self._block_fast_lane_reservations + if key in reservations: + return True + accepted_count = int( + getattr(self._coordinator, "accepted_block_count", 0) + ) + capacity = int(getattr(self._coordinator, "max_blocks", 2**31 - 1)) + stop_after_one = bool( + getattr(self._coordinator, "stop_after_block", False) + ) + reserved_count = len(reservations) + if accepted_count + reserved_count >= capacity: + return False + if stop_after_one and accepted_count + reserved_count >= 1: + return False + reservations.add(key) + return True + + def _release_block_fast_lane_slot(self, block_hash: str) -> None: + self._ensure_block_candidate_disposition_state() + with self._coordinator.lock: + reservations = self._block_fast_lane_reservations + if reservations is not None: + reservations.discard(block_hash.lower()) + + @contextmanager + def _block_candidate_disposition( + self, + block_hash: str, + ) -> Iterator[_BlockCandidateDispositionLease]: + """Serialize the full accepted/abandoned decision for one hash. + + A below-share-target solve submits synchronously while the durable + outbox can concurrently replay that same candidate. Keep both attempts + ordered until the accepted success tail records its process-local + completion; otherwise the replay can terminally abandon the outbox + during the gap after durable confirmation but before audit/share + evidence is complete. + """ + lease = self._coordinator._claim_block_candidate_disposition( + block_hash, + blocking=True, + ) + assert lease is not None + try: + yield lease + finally: + self._coordinator._release_block_candidate_disposition(lease) + + # -- accounting actor -------------------------------------------------- + + def _ensure_block_accounting_state(self) -> None: + if not hasattr(self, "_block_accounting_state_lock"): + self._block_accounting_state_lock = threading.Lock() + if not hasattr(self, "_block_accounting_queue"): + depth = max( + 1, + int( + getattr( + self._coordinator, + "block_accounting_queue_depth", + DEFAULT_BLOCK_ACCOUNTING_QUEUE_DEPTH, + ) + ), + ) + self._block_accounting_queue = queue.PriorityQueue(maxsize=depth) + if not hasattr(self, "_block_accounting_overflow_queue"): + self._block_accounting_overflow_queue = queue.PriorityQueue() + if not hasattr(self, "_block_accounting_accepted_queue"): + # Deliberately unbounded, for the reason the overflow queue is: + # a node offer has already happened and must never be converted + # back into a raw-submit retry. A maxsize here would need a spill + # target, and any queue-state-dependent spill is exactly the + # inversion this lane removes. Max-block admission + # (_reserve_block_fast_lane_slot, max_blocks, stop_after_block) + # and the physical block-acceptance rate bound how many + # unresolved real offers can exist at once. + self._block_accounting_accepted_queue = queue.PriorityQueue() + if not hasattr(self, "_block_accounting_sequence"): + self._block_accounting_sequence = 0 + if not hasattr(self, "_block_accounting_thread"): + self._block_accounting_thread = None + + def _start_block_accounting_thread(self) -> threading.Thread: + self._ensure_block_accounting_state() + with self._block_accounting_state_lock: + thread = self._block_accounting_thread + if thread is not None and thread.is_alive(): + return thread + self._record_block_work_heartbeat("block_accounting", "starting") + thread = threading.Thread( + target=self._coordinator.block_accounting_loop, + name="prism-block-accounting", + daemon=True, + ) + self._block_accounting_thread = thread + thread.start() + return thread + + def _enqueue_block_accounting_task( + self, + task: _BlockCandidateAccountingTask, + ) -> bool: + self._ensure_block_accounting_state() + with self._block_accounting_state_lock: + sequence = self._block_accounting_sequence + self._block_accounting_sequence += 1 + priority = int(task.candidate.context.template["height"]) + item = (priority, sequence, task) + if _is_definitive_node_acceptance(task.node_submission): + # Definitive acceptance leaves the primary/overflow population + # entirely. It does not get a better key inside it -- a key there + # is inert exactly when it matters. Under a burst the spillover + # rule below has already engaged, so the accepted task would be + # put behind the spill while the stale entries still sitting in + # the bounded primary queue -- up to + # DEFAULT_BLOCK_ACCOUNTING_QUEUE_DEPTH of them -- are all served + # ahead of it, each costing ~6 RPCs, two ledger writes and a + # payout-balance mutation. Priority only ever orders one heap; the + # primary-then-overflow dequeue discipline is what inverts it. + # + # Membership is what changes here, not the spillover rule. That + # rule is unchanged below, and the property it exists to protect -- + # once spillover begins, later handoffs join it rather than + # repeatedly refilling the bounded primary, so older spill entries + # are not starved by newer arrivals -- still governs exactly the + # population it was written for: non-definitive accounting work. + # + # A separate lane makes this class structurally un-spillable: no + # queue-state-dependent rule can place it behind stale work, + # because it shares no queue with stale work. Fairness for the + # stale lanes stops being an emergent property of two heaps and + # becomes the explicit dispatch quota + # BLOCK_ACCOUNTING_ACCEPTED_DISPATCH_QUOTA, enforced in + # block_accounting_loop and bounded there rather than argued for. + self._block_accounting_accepted_queue.put_nowait(item) + return True + if not self._block_accounting_overflow_queue.empty(): + # Once spillover begins, keep later handoffs behind it instead of + # repeatedly refilling the primary queue and starving older spill + # entries. + self._block_accounting_overflow_queue.put_nowait(item) + return True + try: + self._block_accounting_queue.put_nowait(item) + return True + except queue.Full: + # A node offer has already happened and must never be converted + # back into a raw-submit retry. Preserve its result and lease in + # an unbounded, process-local overflow queue; max-block admission + # bounds the number of unresolved real offers. + self._block_accounting_overflow_queue.put_nowait(item) + print( + "prism coordinator: block accounting handoff spilled " + f"hash={task.candidate.submission.block_hash_hex}", + flush=True, + ) + return True + + def _call_block_candidate_writer( + self, + candidate: PrismBlockCandidate, + *, + node_submission: _BlockCandidateNodeSubmission, + disposition_held: bool, + ) -> bool: + """Invoke the writer while preserving duck-typed test integrations.""" + writer = self._coordinator._submit_next_block_candidate_writer + supports_disposition_held = True + try: + parameters = inspect.signature(writer).parameters.values() + supports_disposition_held = any( + parameter.name == "disposition_held" + or parameter.kind is inspect.Parameter.VAR_KEYWORD + for parameter in parameters + ) + except (TypeError, ValueError): + pass + if supports_disposition_held: + return bool( + writer( + candidate, + node_submission=node_submission, + disposition_held=disposition_held, + ) + ) + return bool(writer(candidate, node_submission=node_submission)) + + def _restore_replayed_candidate_acceptance_evidence( + self, + candidate: PrismBlockCandidate, + ) -> None: + if not candidate.durable_replay: + return + block_hash = str(candidate.submission.block_hash_hex).lower() + block_state = None + state_read_failed = False + state_reader = getattr(self._coordinator.ledger, "pool_block_state", None) + if callable(state_reader): + try: + block_state = self._coordinator._run_block_submitter_ledger_call( + ("replay-pool-block-state", block_hash), + "replay-pool-block-state", + lambda: state_reader(block_hash=block_hash), + ) + except Exception: + traceback.print_exc() + state_read_failed = True + durable_chain_state = ( + str(block_state.get("chain_state", "")) + if block_state is not None + else "" + ) + if state_read_failed or durable_chain_state in {"prepared", "confirmed"}: + self._coordinator._register_outstanding_block_candidate(block_hash) + with self._coordinator.lock: + self._tip_observed_accepted_block_hashes[block_hash] = ( + time.monotonic() + ) + print( + "prism coordinator: restored acceptance evidence for " + f"replayed block candidate hash={block_hash} " + + ( + "after a failed durable-state read" + if state_read_failed + else f"chain_state={durable_chain_state}" + ), + flush=True, + ) + + def _run_block_accounting_task( + self, + task: _BlockCandidateAccountingTask, + ) -> None: + candidate = task.candidate + with self._coordinator.lock: + self._block_accounting_holds_disposition = True + self._block_accounting_deferred_retry_candidate = None + outcome = self.outcome + outcome.refresh_client = None + try: + self._coordinator._restore_replayed_candidate_acceptance_evidence( + candidate + ) + with self._coordinator._writer_operation("accepted_block_handling"): + self._coordinator._call_block_candidate_writer( + candidate, + node_submission=task.node_submission, + disposition_held=True, + ) + refresh_client = getattr(outcome, "refresh_client", None) + outcome.refresh_client = None + except ShutdownInProgress: + return + except Exception: + # An unexpected failure leaves the candidate to the retry lane. + # Retain it here, while this thread still owns the disposition: + # an accounting-owner retain routes into the deferred holder, + # which names the hash until the finally below merges it into + # the retry slot under coordinator.lock -- after the lease is + # released, but with no instant in between where nothing names + # it. The loop's catch used to retain only after this finally + # had already released the lease, and that instant was exactly + # the dequeued-candidate gap in its accounting-lane form: a + # terminal-outcome eviction there could unfence the hash the + # retried candidate was about to be re-disposed against. + self._coordinator._retain_block_candidate_for_retry(candidate) + task.retained_for_retry.set() + raise + finally: + self._coordinator._release_block_candidate_disposition( + task.disposition_lease + ) + with self._coordinator.lock: + self._block_accounting_holds_disposition = False + deferred_retry = getattr( + self, + "_block_accounting_deferred_retry_candidate", + None, + ) + self._block_accounting_deferred_retry_candidate = None + if deferred_retry is not None: + self._merge_block_candidate_retry_locked( + "_retry_block_candidate", + deferred_retry, + ) + if refresh_client is not None and not self._coordinator.stop_event.is_set(): + self._coordinator._record_block_submitter_phase("refresh-jobs") + self._coordinator.refresh_jobs_after_pending_accepted_block( + refresh_client, + heartbeat_name="block_accounting", + ) + self._coordinator._record_block_submitter_phase("refresh-jobs:complete") + + def _block_accounting_cleanup_retry_work_items(self) -> int: + """Completed work items the lane may run between cleanup offers. + + One work item is one finished ordinary accounting task from any of + the handoff queues, or one finished invalid-candidate quarantine + item. Bounded below at one so a misconfigured value degrades into + offering the retry after every item rather than never offering it at + all. + """ + return max( + 1, + int( + getattr( + self._coordinator, + "block_accounting_cleanup_retry_work_items", + DEFAULT_BLOCK_ACCOUNTING_CLEANUP_RETRY_WORK_ITEMS, + ) + ), + ) + + def _block_accounting_accepted_dispatch_quota(self) -> int: + """Consecutive accepted dispatches allowed while stale work waits. + + Bounded below at one so a misconfigured value degrades into strict + alternation -- one accepted task per stale task -- rather than into + an accepted lane that can never dispatch while any stale work exists. + """ + return max( + 1, + int( + getattr( + self._coordinator, + "block_accounting_accepted_dispatch_quota", + BLOCK_ACCOUNTING_ACCEPTED_DISPATCH_QUOTA, + ) + ), + ) + + def block_accounting_loop(self) -> None: + self._block_accounting_thread_ident = threading.get_ident() + self._ensure_block_accounting_state() + # Work items completed since the last deferred-cleanup offer. The + # idle branch below cannot be the only driver: either lane running + # continuously would hold a due terminal cleanup off for as long as + # the traffic lasted, and its durable row is already gone. Counting + # items here bounds that wait at one offer per configured cadence + # whichever lane the traffic is on. + work_items = 0 + # Definitive-acceptance tasks dispatched back to back since the last + # non-accepted dispatch. Counted separately from work_items on + # purpose: work_items paces the deferred-cleanup offer against *all* + # completed work, while this counter measures one thing only -- how + # long the accepted lane has been holding the stale lanes off. Never + # fold the two together. + accepted_run = 0 + while not self._coordinator.stop_event.is_set(): + self._coordinator._record_block_submitter_phase("accounting-queue") + source_queue = None + task = None + # Selection order is the accepted lane's whole contract, and it + # is stated as a bound rather than as a preference: + # + # whenever non-accepted accounting work is available, at most + # `quota` accepted tasks are dispatched between two consecutive + # dispatches of non-accepted work. + # + # Equivalently, a non-accepted task at the head of its lane waits + # behind at most `quota` accepted services, never behind an + # unbounded stream of them, whatever the accepted arrival rate. + # Starving it is not an option: a stale accounting task that never + # runs leaves its outbox row pending, its offer-time payout-preview + # barrier armed and its disposition lease held. + if accepted_run < self._block_accounting_accepted_dispatch_quota(): + try: + _priority, _sequence, task = ( + self._block_accounting_accepted_queue.get_nowait() + ) + source_queue = self._block_accounting_accepted_queue + accepted_run += 1 + except queue.Empty: + pass + if source_queue is None: + # The stale population, dequeued with exactly the discipline + # it has always had: bounded primary first, then the + # unbounded spillover behind it. + try: + _priority, _sequence, task = ( + self._block_accounting_queue.get_nowait() + ) + source_queue = self._block_accounting_queue + except queue.Empty: + try: + _priority, _sequence, task = ( + self._block_accounting_overflow_queue.get_nowait() + ) + source_queue = self._block_accounting_overflow_queue + except queue.Empty: + pass + if source_queue is not None: + accepted_run = 0 + if source_queue is None: + # No stale work exists, so the quota has nothing to yield to + # and is honoured vacuously; anything the quota was holding + # back runs now, and the run counter restarts from this + # dispatch rather than carrying an exhausted count forward. + try: + _priority, _sequence, task = ( + self._block_accounting_accepted_queue.get_nowait() + ) + source_queue = self._block_accounting_accepted_queue + accepted_run = 0 + except queue.Empty: + pass + if source_queue is None: + if self._coordinator._run_one_invalid_block_candidate_quarantine(): + work_items += 1 + if ( + work_items + >= self._block_accounting_cleanup_retry_work_items() + ): + work_items = 0 + self._run_one_collapsed_block_candidate_cleanup_retry() + continue + # A collapsed row's failed cleanup has no durable replay + # source left, so the accounting lane is the only thing + # that will ever finish it. A truly idle lane offers it + # here immediately rather than waiting out a cadence; the + # work-item cadence exists for the busy lanes, which + # never reach this branch at all. + work_items = 0 + if self._run_one_collapsed_block_candidate_cleanup_retry(): + continue + self._coordinator.stop_event.wait(self._block_work_wait_slice()) + continue + try: + self._coordinator._run_block_accounting_task(task) + except Exception: + print( + "prism coordinator: block accounting iteration failed; " + "durable candidate remains pending", + flush=True, + ) + traceback.print_exc() + retained = getattr(task, "retained_for_retry", None) + if retained is None or not retained.is_set(): + # The failure fired before _run_block_accounting_task's + # own exception path could retain the candidate -- a + # delegate raising ahead of it -- so it is retained here + # as before. A candidate the lane already retained (and + # merged into the retry slot) is not retained again. + self._coordinator._retain_block_candidate_for_retry( + task.candidate + ) + finally: + assert source_queue is not None + source_queue.task_done() + work_items += 1 + if work_items >= self._block_accounting_cleanup_retry_work_items(): + # The counter resets on every offer, due record or not: a + # cleanup backlog may therefore spend at most one attempt per + # cadence, so it can never starve accounting or quarantine in + # return. Deliberately left uncontained and outside the task's + # try/finally -- the retry pops its record before attempting + # it, and only its own failure paths re-register it, so an + # outer catch here could swallow the last reference to a + # terminal hash's owed steps. + work_items = 0 + self._run_one_collapsed_block_candidate_cleanup_retry() + + # -- submitter loop ---------------------------------------------------- + + def run(self) -> None: + # Boundary stamps in _record_block_candidate_progress are gated to + # this thread so client-thread dispositions cannot refresh the + # submitter's liveness budget on its behalf. + self._block_submitter_thread_ident = threading.get_ident() + self._coordinator._start_block_accounting_thread() + while not self.ports.stop_event().is_set(): + self._coordinator._record_block_submitter_phase("loop") + try: + # The in-memory wakeup is already backed by the durable + # outbox. Drain it before any recovery query so a saturated + # database cannot delay the first node submission. + with self._coordinator.lock: + retry_ready = self.retry_candidate is not None + queue_obj = self.candidate_queue + replay_queue = getattr( + self, + "_block_replay_candidate_queue", + None, + ) + wakeup_ready = bool( + (queue_obj is not None and not queue_obj.empty()) + or (replay_queue is not None and not replay_queue.empty()) + ) + if ( + retry_ready or wakeup_ready + ) and self._coordinator.submit_next_block_candidate( + defer_accounting=True + ): + if not self._ancestor_redrive_owed(): + continue + # An armed ancestor re-drive (issue #190) falls through + # to the replay entrypoint even though immediate work + # succeeded: sustained live traffic taking this + # `continue` every pass is exactly how the wedge starved + # the one path that resolves a stuck ancestor. + self.ports.replay_entrypoint() + self._run_block_candidate_body_janitor_step() + self._coordinator.submit_next_block_candidate( + timeout=1.0, + defer_accounting=True, + ) + except ShutdownInProgress: + # Admission can close after the loop condition. Durable block + # candidates remain in the outbox for the replacement writer. + return + except Exception: + phase = getattr(self, "_block_submitter_phase", "unknown") + print( + "prism coordinator: block submitter iteration failed " + f"phase={phase}; durable candidates remain pending", + flush=True, + ) + traceback.print_exc() + retry_delay = float(self.retry_initial_seconds) + if self._coordinator._wait_for_block_candidate_retry(retry_delay): + return + + def submit_next( + self, + timeout: float | None = None, + *, + defer_accounting: bool = False, + ) -> BlockCandidateRunResult: + """Dequeue and land one block candidate; ``ran`` is True when one ran. + + The block-submitter loop calls this continuously; tests call it + directly (through the coordinator delegate) to drain the queue + deterministically. + """ + coordinator = self._coordinator + coordinator._record_block_submitter_phase("dequeue-retry") + # The disposition state owns the dequeued-hash pin, and its backfill + # takes the state-backfill lock, so it runs before any coordinator + # lock hold below rather than underneath one. + self._ensure_block_candidate_disposition_state() + # Every dequeue below leaves its lane and enters the dequeued-hash + # pin in one coordinator.lock critical section. The lanes pin a + # terminal duplicate's outcome against eviction while it sits in + # them and the disposition flight pins it from the claim onward; + # between the two the candidate object is only this method's local + # variable, and the pin is what keeps the fence it is about to read + # from being the oldest unpinned entry in the registry. + with coordinator.lock: + candidate = self.retry_candidate + if candidate is not None and not self._block_candidate_retry_ready_locked( + candidate + ): + # Parked by _pace_block_candidate_retry: honor the backoff + # deadline without sleeping on the submitter or accounting + # lane. + candidate = None + if candidate is not None: + self.retry_candidate = None + block_hash = self._pin_dequeued_block_candidate_locked(candidate) + if candidate is None: + queue_obj = self.candidate_queue + self._ensure_block_replay_state() + replay_queue = self._block_replay_candidate_queue + if queue_obj is None and replay_queue is None: + return BlockCandidateRunResult(False) + deadline = ( + None + if timeout is None + else time.monotonic() + max(0.0, timeout) + ) + while candidate is None: + coordinator._record_block_submitter_phase("dequeue-queue") + descriptor: DurableCandidateDescriptor | None = None + with coordinator.lock: + # Live discoveries always outrank durable restart replay. + # The non-blocking get takes the queue's own mutex under + # coordinator.lock -- the order the eviction's queue read + # already establishes -- so the entry leaves the queue + # and enters the pin without a gap between them. + for candidate_queue in (queue_obj, replay_queue): + if candidate_queue is None: + continue + try: + item = candidate_queue.get_nowait() + except queue.Empty: + continue + if isinstance(item, DurableCandidateDescriptor): + if not self._replay_descriptor_ready(item): + # Parked by a paced hydration retry: rotate + # it to the back and look at the next lane. + candidate_queue.put_nowait(item) + continue + descriptor = item + else: + candidate = item + break + if candidate is None and descriptor is None: + waiting = self._block_disposition_waiting_retries + ready_hashes = [ + key + for key in waiting + if self._block_candidate_retry_ready_locked( + waiting[key] + ) + ] + if ready_hashes: + waiting_hash = min( + ready_hashes, + key=lambda key: int( + waiting[key].context.template["height"] + ), + ) + candidate = waiting.pop(waiting_hash) + if candidate is not None: + block_hash = self._pin_dequeued_block_candidate_locked( + candidate + ) + if descriptor is not None: + # Body-sized work: hydrate outside coordinator.lock, one + # descriptor at a time. The hash stays named by the + # replay-inflight registry throughout, so the terminal + # outcome fence it will read cannot be evicted meanwhile. + candidate = self._hydrate_replay_descriptor(descriptor) + if candidate is None: + return BlockCandidateRunResult(True) + with coordinator.lock: + block_hash = self._pin_dequeued_block_candidate_locked( + candidate + ) + if candidate is not None: + break + if deadline is None: + return BlockCandidateRunResult(False) + remaining = deadline - time.monotonic() + if remaining <= 0: + return BlockCandidateRunResult(False) + if coordinator.stop_event.wait( + min(remaining, self._block_work_wait_slice()) + ): + return BlockCandidateRunResult(False) + + try: + lease = coordinator._claim_block_candidate_disposition( + block_hash, + blocking=not defer_accounting, + ) + except BaseException: + # The candidate object leaves with the exception (its durable + # outbox row still replays), so no in-memory copy remains that + # the pin would be protecting. + with coordinator.lock: + self._unpin_dequeued_block_candidate_locked(block_hash) + raise + if lease is None: + # Another same-hash pass already spans node offer through durable + # finalization. Keep this wakeup outside the global parent retry + # slot until that lease transfers/releases: consuming it can lose + # an accounting retry, while repeatedly prioritizing it can starve + # unrelated live blocks. The waiting registry takes over the pin + # in the same critical section. + with coordinator.lock: + self._block_disposition_waiting_retries[block_hash] = candidate + self._unpin_dequeued_block_candidate_locked(block_hash) + coordinator._wait_for_block_candidate_retry( + float(self.retry_initial_seconds) + ) + return BlockCandidateRunResult(True) + # The claim installed this pass's disposition flight, which the + # eviction reads as a pin of its own; hand the hash over to it. + with coordinator.lock: + self._unpin_dequeued_block_candidate_locked(block_hash) + transferred = False + if coordinator._block_candidate_terminal_outcome(block_hash) is not None: + coordinator._release_block_candidate_disposition(lease) + # Another same-hash object's disposition landed first; this + # duplicate is dropped here and nothing later can release the + # floor holder it carries. + self._release_dropped_duplicate_candidate_floor(candidate) + return BlockCandidateRunResult(True) + + outcome = self.outcome + outcome.refresh_client = None + coordinator._record_block_submitter_phase("finalize-registry") + with coordinator.lock: + pending_finalize = self.finalize_retries.get(block_hash) + if pending_finalize is None: + # Issue #181 item 2. A candidate whose parent is no longer the + # best tip, whose height the chain has decided in favour of a + # block of at least its work, and which carries no offer + # evidence, is provably stale before any offer: skip it here + # rather than pay a submitblock, ~6 chain reads, two ledger + # writes, an accounting task and a payout-preview barrier to + # learn the same thing per row. + # + # Placed after the lease claim and the terminal-outcome fence -- + # both of which this depends on -- and before the fast-lane + # reservation, so a sibling on its way to being abandoned never + # consumes max-block capacity. The skip terminalizes the durable + # row itself (it does not drop the wakeup): the lease it holds + # is already in #196's evidence set, so a dropped row would be + # rejected by that selector forever, and the dequeued object's + # pending-share floor holder can only be released by draining + # this object. + # + # Guarded exactly as the node offer below is: the skip contains + # its own failures and answers False for every one of them, but + # an escape here would leave this hash's lease held forever and + # drop the only object that can release its floor holder. + try: + stale = self._skip_superseded_block_candidate_at_dequeue( + candidate + ) + except BaseException: + try: + coordinator._retain_block_candidate_for_retry(candidate) + finally: + coordinator._release_block_candidate_disposition(lease) + raise + if stale: + coordinator._release_block_candidate_disposition(lease) + return BlockCandidateRunResult(True) + permanently_closed = False + already_accounted = False + if defer_accounting: + with coordinator.lock: + accounted_hashes = getattr( + coordinator, + "_accounted_accepted_block_hashes", + set(), + ) + already_accounted = block_hash in accounted_hashes + accepted_count = int( + getattr(coordinator, "accepted_block_count", 0) + ) + permanently_closed = not already_accounted and ( + accepted_count + >= int(getattr(coordinator, "max_blocks", 2**31 - 1)) + or ( + bool(getattr(coordinator, "stop_after_block", False)) + and accepted_count >= 1 + ) + ) + if permanently_closed or already_accounted: + # Accounting must terminalize a durable outbox row even + # after pool capacity closes. An already-accounted hash + # likewise needs only its exact-idempotent/finalize tail. + node_submission = _BlockCandidateNodeSubmission( + attempted=False + ) + elif not coordinator._reserve_block_fast_lane_slot(block_hash): + # Capacity is provisionally occupied by another unresolved + # node offer. Preserve strict max-block semantics until + # that offer either accounts or terminates. Retained + # before the lease goes, as on the exception paths, so + # the retry holder names the hash before the flight + # stops doing so. + coordinator._retain_block_candidate_for_retry(candidate) + coordinator._release_block_candidate_disposition(lease) + coordinator._wait_for_block_candidate_retry( + float(self.retry_initial_seconds) + ) + return BlockCandidateRunResult(True) + else: + try: + node_submission = coordinator._node_submission_for_candidate_or_retained(candidate) + except BaseException: + try: + coordinator._retain_block_candidate_for_retry(candidate) + finally: + coordinator._release_block_candidate_disposition(lease) + raise + else: + try: + node_submission = coordinator._node_submission_for_candidate_or_retained(candidate) + except BaseException: + try: + coordinator._retain_block_candidate_for_retry(candidate) + finally: + coordinator._release_block_candidate_disposition(lease) + raise + else: + node_submission = _BlockCandidateNodeSubmission(attempted=False) + + if defer_accounting: + task = _BlockCandidateAccountingTask( + candidate=candidate, + node_submission=node_submission, + disposition_lease=lease, + ) + try: + enqueued = coordinator._enqueue_block_accounting_task(task) + except BaseException: + try: + coordinator._retain_block_candidate_for_retry(candidate) + finally: + coordinator._release_block_candidate_disposition(lease) + raise + if enqueued: + transferred = True + return BlockCandidateRunResult(True) + coordinator._retain_block_candidate_for_retry(candidate) + coordinator._release_block_candidate_disposition(lease) + coordinator._wait_for_block_candidate_retry( + float(self.retry_initial_seconds) + ) + return BlockCandidateRunResult(True) + + try: + with coordinator._writer_operation("accepted_block_handling"): + ran = coordinator._call_block_candidate_writer( + candidate, + node_submission=node_submission, + disposition_held=True, + ) + refresh_client = getattr(outcome, "refresh_client", None) + outcome.refresh_client = None + except ShutdownInProgress: + # The durable outbox remains pending and the replacement process + # will replay it. Dequeuing the in-memory wakeup during the + # admission-close race cannot lose candidate work. + return BlockCandidateRunResult(False) + finally: + if not transferred: + coordinator._release_block_candidate_disposition(lease) + # Fresh-job fanout is deliberately outside the writer admission. Once + # the candidate outbox is finalized it cannot mutate the ledger, so a + # blocked client send must not hold the writer lease during shutdown. + if refresh_client is not None and not coordinator.stop_event.is_set(): + coordinator._record_block_submitter_phase("refresh-jobs") + coordinator.refresh_jobs_after_pending_accepted_block( + refresh_client, + heartbeat_name="block_submitter", + ) + coordinator._record_block_submitter_phase("refresh-jobs:complete") + return BlockCandidateRunResult(ran, refresh_client) + + def attempt(self, candidate: PrismBlockCandidate) -> BlockCandidateAttemptResult: + """Run one direct landing attempt and structure its outcome.""" + self.outcome.reason = None + error = "candidate became stale or submission failed" + self._note_accepted_landing_lane_start(candidate) + try: + accepted = self.ports.submit_candidate(candidate) + except Exception: + accepted = False + error = "candidate submission raised an exception" + self.ports.log( + "prism coordinator: block candidate submission failed " + f"hash={candidate.submission.block_hash_hex}" + ) + traceback.print_exc() + return BlockCandidateAttemptResult( + accepted=accepted, + reason=getattr(self.outcome, "reason", None), + error=error, + ) + + def submit_writer( + self, + candidate: PrismBlockCandidate, + *, + node_submission: _BlockCandidateNodeSubmission | None = None, + disposition_held: bool = False, + ) -> bool: + """Land one dequeued block candidate inside writer admission.""" + coordinator = self._coordinator + block_hash = str(candidate.submission.block_hash_hex).lower() + if not disposition_held: + # Preserve the historical direct-writer seam while keeping its + # node offer and terminal outbox decision inside the same-hash + # guard. Production queue/accounting calls transfer an existing + # lease and skip this wrapper. + with coordinator._block_candidate_disposition(block_hash): + terminal_outcome = coordinator._block_candidate_terminal_outcome( + block_hash + ) + if terminal_outcome is not None: + return terminal_outcome + if node_submission is None: + node_submission = ( + coordinator._node_submission_for_candidate_or_retained( + candidate + ) + ) + return coordinator._submit_next_block_candidate_writer( + candidate, + node_submission=node_submission, + disposition_held=True, + ) + outcome = self.outcome + outcome.reason = None + outcome.error = None + outcome.stale_job_class = None + coordinator._record_block_submitter_phase("finalize-registry") + with coordinator.lock: + pending_finalize = self.finalize_retries.get(block_hash) + if pending_finalize is not None: + # Finalize-only replay: node submission, terminal accounting, and + # payout persistence already completed on the pass that armed + # this entry. It bypasses both submitblock and attempt marking. + accepted, error = pending_finalize + return coordinator._finalize_block_candidate( + candidate, + block_hash=block_hash, + accepted=accepted, + error=error, + outcome=outcome, + ) + # The queue, disposition and admission time ends here: everything + # below is work this landing runs (issue #224). A candidate whose + # earlier pass already stamped acceptance closes its lane here, before + # any retained offer is resolved. + self._note_accepted_landing_lane_start(candidate) + if node_submission is None: + node_submission = coordinator._node_submission_for_candidate_or_retained(candidate) + # A fresh candidate is stamped by the offer that call makes, so the + # close above found no interval. Close again against the stamp that + # now exists; it is a no-op once this interval's lane was recorded. + self._note_accepted_landing_lane_start(candidate) + # Both closes sit above the attempt marker on purpose. The marker is + # this landing's own PostgreSQL write, and a degraded writer must not + # inflate the one stretch that exonerates the landing -- nor keep + # inflating it across the retry backoff a timed-out marker triggers. + try: + coordinator._mark_block_candidate_attempted(block_hash) + except Exception: + print( + "prism coordinator: could not record block candidate attempt " + f"hash={block_hash}", + flush=True, + ) + traceback.print_exc() + coordinator._retain_block_candidate_for_retry(candidate) + coordinator._pace_block_candidate_retry(block_hash) + return True + accepted = False + error = "candidate became stale or submission failed" + try: + coordinator._record_block_submitter_phase("accounting") + with coordinator._block_landing_ledger_statement_timeout_scope(block_hash): + # ``disposition_held`` states that the serialized inner tail + # is available; a ``node_submission`` of None still selects + # the historical bare entrypoint call. Both distinctions are + # carried to the port rather than decided from here. + accepted = self.ports.submit_candidate( + candidate, + node_submission=node_submission, + disposition_held=disposition_held, + ) + except Exception: + error = "candidate submission raised an exception" + print( + "prism coordinator: block candidate submission failed " + f"hash={candidate.submission.block_hash_hex}", + flush=True, + ) + traceback.print_exc() + abandon_reason = getattr(outcome, "reason", None) if outcome is not None else None + abandon_error = getattr(outcome, "error", None) if outcome is not None else None + if not accepted and abandon_error: + error = str(abandon_error) + retryable = not accepted and ( + abandon_reason is None + or abandon_reason in self.retryable_reasons + ) + if retryable: + # Leave the outbox row pending. It will replay after a short pause + # or on process restart. Keep this parent ahead of queued children: + # a child built from its prospective balances cannot be validated + # against the database until the parent confirmation catches up. + print( + "prism coordinator: retained block candidate for retry " + f"hash={block_hash} reason={abandon_reason or 'exception'}", + flush=True, + ) + coordinator._retain_block_candidate_for_retry(candidate) + coordinator._pace_block_candidate_retry(block_hash) + return True + if not accepted: + try: + coordinator._record_block_submitter_phase("reject-prepared-block") + with coordinator._block_landing_ledger_statement_timeout_scope(block_hash): + coordinator._reject_terminal_prepared_block_candidate(candidate) + coordinator._record_block_submitter_phase( + "reject-prepared-block:complete" + ) + except Exception: + # Persistence may have committed before a later RPC/transport + # failure. Do not terminally discard the outbox row until its + # prepared balance deltas have also reached a terminal state. + print( + "prism coordinator: prepared block cleanup failed " + f"hash={block_hash}", + flush=True, + ) + traceback.print_exc() + coordinator._defer_block_candidate( + "backend-rpc-unavailable", + "could not reject prepared state for terminal candidate", + worker=candidate.client.username or None, + ) + coordinator._retain_block_candidate_for_retry(candidate) + coordinator._wait_for_block_candidate_retry( + coordinator._next_block_candidate_retry_delay(block_hash) + ) + return True + return coordinator._finalize_block_candidate( + candidate, + block_hash=block_hash, + accepted=accepted, + error=error, + outcome=outcome, + ) + + def finalize( + self, + candidate: PrismBlockCandidate, + *, + block_hash: str, + accepted: bool, + error: str, + outcome: threading.local, + ) -> bool: + """Drive a terminal candidate's durable outbox update, with backoff. + + Failure retains the candidate as a finalize-only replay: the next + paced attempt re-enters here directly, never submit_block_candidate, + so terminal abandonment accounting stays once-per-candidate and an + accepted candidate's audit/persist work is not redone per retry. + """ + coordinator = self._coordinator + coordinator._record_block_submitter_phase("finalize-preview") + coordinator._clear_accepted_block_payout_preview( + block_hash, + invalidate_published=not accepted, + ) + finish_name = ( + "mark_block_candidate_submitted" + if accepted + else "mark_block_candidate_abandoned" + ) + finish = getattr(coordinator.ledger, finish_name, None) + if callable(finish): + try: + if accepted: + coordinator._run_block_submitter_ledger_call( + ("finalize", block_hash, "submitted"), + "finalize-outbox-submitted", + lambda: finish(block_hash=block_hash), + ) + else: + coordinator._run_block_submitter_ledger_call( + ("finalize", block_hash, "abandoned"), + "finalize-outbox-abandoned", + lambda: finish(block_hash=block_hash, error=error), + ) + coordinator._record_committed_block_candidate_abandonment( + block_hash, + outcome, + ) + # The invalidation tombstone is needed until the durable + # outbox becomes terminal. A normal return (including an + # already-terminal/missing row) means there is no pending + # replay source left for this process to guard. + coordinator._clear_accepted_block_payout_preview(block_hash) + except Exception: + # Keep the coordinator alive. The terminal-state update + # failed, so the durable row stays pending and its replay + # must pace like any other retained retry. + print( + "prism coordinator: could not finalize durable block candidate " + f"hash={block_hash}", + flush=True, + ) + traceback.print_exc() + with coordinator.lock: + registry = self.finalize_retries + first_failure = block_hash not in registry + registry[block_hash] = (accepted, error) + if not accepted: + # The durable update is ambiguous, but this process has + # now frozen a false finalize-only disposition. It cannot + # return to chain-state evaluation until restart, where + # these process-local counters start fresh. + coordinator._record_committed_block_candidate_abandonment( + block_hash, + outcome, + ) + # The share row already reached its terminal outcome in this + # process; only the outbox mark is pending. Release the + # snapshot anchor floor now (idempotent) -- holding it across + # paced retries would clamp job snapshot anchors and + # under-count already-durable shares in reward windows. + coordinator._finish_pending_share_commit(candidate.pending_share) + coordinator._retain_block_candidate_for_retry(candidate) + if accepted and first_failure: + # The block is active regardless of the outbox update; + # post-accept fleet refresh must wait for neither ledger + # recovery nor the first backoff. Return unpaced so the + # caller refreshes immediately; the paced ladder starts + # from the first finalize-only replay. + outcome.refresh_client = candidate.client + return True + coordinator._wait_for_block_candidate_retry( + coordinator._next_block_candidate_retry_delay(block_hash) + ) + return True + elif not accepted: + # Compatibility ledgers without a durable candidate outbox have + # no restart replay source that could require the tombstone. + coordinator._clear_accepted_block_payout_preview(block_hash) + coordinator._record_committed_block_candidate_abandonment( + block_hash, + outcome, + ) + with coordinator.lock: + self.finalize_retries.pop(block_hash, None) + coordinator._clear_block_candidate_retry_state(block_hash) + coordinator._discard_outstanding_block_candidate(block_hash) + coordinator._record_block_candidate_terminal_outcome( + block_hash, + accepted=accepted, + ) + # Terminal for this process either way: an accepted candidate credited + # its share during the success tail (a no-op release here), and an + # abandoned one can only be credited by restart replay, which stamps a + # fresh PendingShare. Stop holding the snapshot anchor floor. + coordinator._finish_pending_share_commit(candidate.pending_share) + if accepted: + outcome.refresh_client = candidate.client + return True + + # -- retained node acceptance ------------------------------------------ + + def _stash_retained_block_candidate_node_submission( + self, + block_hash: str, + node_submission: _BlockCandidateNodeSubmission | None, + ) -> None: + """Record a definitive node acceptance for in-process retries. + + Recorded at the offer itself (the universal post-offer hook) so + every retention path — writer failures, defer-accounting handoff + failures, the accounting loop — is covered without each site having + to remember. A retryable failure after a successful offer would + otherwise re-offer on retry and read "duplicate", which classifies + against the moved live tip and can only rescue the block through + chain probes that may be unavailable under the same saturation. + The entry is read without consuming and lives until the candidate + reaches a terminal outcome, so repeated retryable failures keep + reusing the same known acceptance. + """ + if not _is_definitive_node_acceptance(node_submission): + # Only a definitive success is safe to reuse: an ambiguous or + # rejected offer must be re-offered so the node can resolve it. + # The predicate is shared with the accounting handoff's lane + # routing so the two readings of "the node has this block" cannot + # drift apart. + return + with self._coordinator.lock: + retained = getattr( + self, + "_block_candidate_retained_node_submissions", + None, + ) + if retained is None: + retained = {} + self._block_candidate_retained_node_submissions = retained + retained[str(block_hash).lower()] = node_submission + stamped = getattr( + self, + "_block_candidate_retained_submission_monotonic", + None, + ) + if stamped is None: + stamped = {} + self._block_candidate_retained_submission_monotonic = stamped + stamped[str(block_hash).lower()] = time.monotonic() + + def _retained_block_candidate_node_submission( + self, + block_hash: str, + ) -> _BlockCandidateNodeSubmission | None: + with self._coordinator.lock: + retained = getattr( + self, + "_block_candidate_retained_node_submissions", + None, + ) + if not retained: + return None + return retained.get(str(block_hash).lower()) + + def _block_candidate_acceptance_retained(self, block_hash: str) -> bool: + """Whether this process holds fresh first-party acceptance evidence. + + The retained stash records only definitive submitblock successes, + so its presence proves qbitd accepted this candidate — evidence of + the same strength as a recent own-hash tip observation, and + available precisely when saturation makes the chain probes answer + "unknown" (the observation registry can be empty after a definitive + ack: blockwait only reports the newest of rapid connects). It ages + on the same window as tip observations: acceptance at offer time + does not prove the block stayed canonical, and an orphaned block + never probes False — it is merely absent — so a candidate whose + probes stay inconclusive past the window must regain + abandonability instead of deferring forever behind a stale ack. + """ + with self._coordinator.lock: + stamped = getattr( + self, + "_block_candidate_retained_submission_monotonic", + None, + ) + if not stamped: + return False + recorded = stamped.get(str(block_hash).lower()) + if recorded is None: + return False + window = float( + getattr( + self._coordinator, + "observed_tip_accept_window_seconds", + DEFAULT_PRISM_OBSERVED_TIP_ACCEPT_WINDOW_SECONDS, + ) + ) + if window <= 0: + return True + return (time.monotonic() - recorded) <= window + + # -- outcome recording ------------------------------------------------- + + def record_deferred( + self, + reason: str, + message: str, + *, + worker: str | None, + ) -> None: + """Record a retryable outcome without counting a terminal abandonment.""" + del worker + outcome = self.outcome + outcome.reason = reason + outcome.error = None + outcome.stale_job_class = None + print( + f"prism coordinator: block candidate deferred reason={reason}: {message}", + flush=True, + ) + + # -- acceptance evidence ----------------------------------------------- + + def _block_candidate_acceptance_recorded(self, block_hash: str) -> bool: + """Return whether this process completed the candidate success tail.""" + self._coordinator._ensure_job_cache_state() + with self._coordinator.lock: + return ( + block_hash.lower() + in self._coordinator._accounted_accepted_block_hashes + ) + + def _register_outstanding_block_candidate(self, block_hash: str) -> None: + """Track a candidate this process may still land, for tip matching.""" + self._coordinator._ensure_job_cache_state() + with self._coordinator.lock: + self._outstanding_block_candidate_hashes.add(block_hash.lower()) + + def _discard_outstanding_block_candidate(self, block_hash: str) -> None: + """Stop matching tip observations once a candidate is terminal.""" + self._coordinator._ensure_job_cache_state() + key = block_hash.lower() + with self._coordinator.lock: + self._outstanding_block_candidate_hashes.discard(key) + self._tip_observed_accepted_block_hashes.pop(key, None) + + def _note_tip_observation_for_candidates(self, tip_hash: str) -> None: + """Register a tip observation that matches an outstanding candidate. + + qbitd only ever reports the pool's own candidate hash as its chain + tip after accepting that block, so the observation itself is + acceptance evidence -- even when the direct submitblock ack was lost + in transport and the accepted success tail has not run (blockwait + typically learns of the tip before, or instead of, the ack). Every + tip-observation channel funnels through here so later disposition + and abandon checks can outlive transient fork views. + """ + key = tip_hash.lower() + self._coordinator._ensure_job_cache_state() + newly_observed = False + with self._coordinator.lock: + if key in self._outstanding_block_candidate_hashes: + newly_observed = ( + key not in self._tip_observed_accepted_block_hashes + ) + self._tip_observed_accepted_block_hashes[key] = time.monotonic() + if newly_observed: + print( + "prism coordinator: chain tip observation matches pool block " + f"candidate hash={key}; acceptance registered pending " + "finalization", + flush=True, + ) + + def _block_candidate_acceptance_observed(self, block_hash: str) -> bool: + """Whether a recent tip observation already proved this candidate landed.""" + self._coordinator._ensure_job_cache_state() + with self._coordinator.lock: + observed = self._tip_observed_accepted_block_hashes.get( + block_hash.lower() + ) + if observed is None: + return False + window = float( + getattr( + self._coordinator, + "observed_tip_accept_window_seconds", + DEFAULT_PRISM_OBSERVED_TIP_ACCEPT_WINDOW_SECONDS, + ) + ) + if window <= 0: + return True + return (time.monotonic() - observed) <= window + + def _block_candidate_chain_probe( + self, + block_hash: str, + *, + expected_height: int | None = None, + ) -> bool | None: + """Fresh chain verdict for a candidate: proven active, proven wrong, or unknown. + + Returns True when the candidate's own hash is the fresh best tip or + an active chain header at its expected height, False when it is + provably active at the wrong height (a corrupt intent, never a tip + race), and None when this instantaneous view proves nothing (the + hash absent during a tip race, or the probe itself failing). + """ + key = block_hash.lower() + # The two probes are independent: a best-tip lookup failure must not + # suppress the active-header check, which subsumes it (the tip block + # itself reports one confirmation) and can prove acceptance alone. + try: + if str(self._coordinator.rpc.call("getbestblockhash")).lower() == key: + self._coordinator._note_tip_observation_for_candidates(key) + return True + except Exception: + print( + "prism coordinator: acceptance re-check best-tip probe " + f"failed hash={key}; trying the active-header probe", + flush=True, + ) + traceback.print_exc() + height: int | None = None + try: + height = self._coordinator.active_block_candidate_height(key) + except Exception: + print( + "prism coordinator: acceptance re-check header probe failed " + f"hash={key}; falling back to tip-observation evidence", + flush=True, + ) + traceback.print_exc() + if height is None: + return None + if expected_height is None or int(height) == int(expected_height): + self._coordinator._note_tip_observation_for_candidates(key) + return True + return False + + def _block_candidate_acceptance_pending( + self, + block_hash: str, + *, + expected_height: int | None = None, + ) -> bool: + """Return whether abandoning this candidate would discard an accepted block. + + A fresh probe wins in both directions: a candidate proven active is + accepted even if its tip observation was missed, and a candidate + proven active at the wrong height stays abandonable. When the probe + cannot prove either way, a recent own-hash tip observation keeps the + candidate deferring instead of terminal: qbitd accepted it once, so + only a durably settled chain view may discard it. + """ + probe = self._coordinator._block_candidate_chain_probe( + block_hash, + expected_height=expected_height, + ) + if probe is not None: + return probe + return self._coordinator._block_candidate_acceptance_observed(block_hash) + + def _count_accept_pending_defer(self) -> None: + with self._coordinator.lock: + self.block_candidate_accept_pending_defer_count = int( + getattr(self, "block_candidate_accept_pending_defer_count", 0) + ) + 1 + + def record_abandoned( + self, + reason: str, + message: str, + *, + block_hash: str, + worker: str | None, + preserve_if_accepted: bool = False, + expected_height: int | None = None, + stale_job_class: str | None = None, + ) -> bool: + """Record a lost/failed block candidate as a BLOCK-path event. + + The share that produced the candidate was acknowledged and, when it met + the share target, credited at submit time; the block losing its race + afterwards does not un-earn it and is NOT a share rejection. It is + counted under a dedicated block-abandonment counter (by reason, so a + benign 'tip moved' race is distinguishable from a real + submitblock-rejected/ledger failure) rather than the share-reject + counters, which stay a true measure of shares refused to miners. + + Every terminal abandonment withdraws its payout-preview transition + before the outcome becomes final. ``preserve_if_accepted`` closes the + moved-tip race: if another attempt completed this hash's accepted + success tail while withdrawal was in flight, the accepted disposition + wins and the caller must finalize the outbox as submitted. Returns + whether that accepted disposition won. + + Independent of that completed-tail record, a candidate whose own + block hash is the fresh best tip, an active chain header at its + expected height, a recent own-hash tip observation, or a fresh + retained definitive submitblock success is an ACCEPTED block whose + finalization is still pending (for example after a lost submitblock + ack, or when saturation makes both probes answer "unknown" at this + instant). Terminal abandonment would discard its payout + accounting and withdraw its landed preview -- fencing payout + publication for work qbitd already accepted -- so such candidates + defer for retry instead; only hashes provably absent from the active + chain (past the observation and retained-acceptance windows) + abandon terminally. The terminal + seal re-reads observation evidence atomically, so callers can order + follow-up durable work (rejecting prepared payout rows) strictly + afterward. Abandonment metrics commit only once that cleanup succeeds + or a false finalize-only disposition is frozen. + """ + coordinator = self._coordinator + if reason in self.retryable_reasons: + coordinator._defer_block_candidate(reason, message, worker=worker) + return False + if ( + stale_job_class is not None + and stale_job_class not in PRISM_STALE_JOB_ABANDON_CLASSES + ): + raise ValueError( + f"unknown stale job abandon class: {stale_job_class}" + ) + outcome = self.outcome + if ( + preserve_if_accepted + and coordinator._block_candidate_acceptance_recorded(block_hash) + ): + # Durable accepted state already equals the prospective view, so + # any transition recreated by the losing attempt is a no-op + # override, not a withdrawal. + coordinator._clear_accepted_block_payout_preview(block_hash) + outcome.reason = None + outcome.error = None + return True + chain_probe = coordinator._block_candidate_chain_probe( + block_hash, + expected_height=expected_height, + ) + if chain_probe is True or ( + chain_probe is None + and ( + coordinator._block_candidate_acceptance_observed(block_hash) + or coordinator._block_candidate_acceptance_retained(block_hash) + ) + ): + self._count_accept_pending_defer() + coordinator._defer_block_candidate( + PRISM_REJECTION_BLOCK_ACCEPT_PENDING, + "candidate is on (or was recently observed or definitively " + "accepted as) the active chain; refusing terminal " + f"abandonment (was {reason}: {message})", + worker=worker, + ) + return False + + # Own the cleanup invariant here rather than relying on every caller to + # remember it. Invalidation can block behind another candidate pass; + # recheck the accepted record afterward before committing abandonment. + # Capture the transition first: if late acceptance evidence forces a + # defer below, its published preview must be restorable. + with coordinator._accepted_block_payout_preview_condition: + withdrawn_transition = coordinator._accepted_block_payout_previews.get( + block_hash.lower() + ) + coordinator._clear_accepted_block_payout_preview( + block_hash, + invalidate_published=True, + ) + # The invalidation above can block long enough for the chain view to + # heal (a buried accepted block is not always re-observed as the tip + # while blockwait only reports the newest of rapid connects), so an + # unknown pre-withdrawal verdict must re-probe before the terminal + # commit. A provably wrong-height verdict is immutable (headers + # cannot change height) and is never re-probed. + late_probe = ( + chain_probe + if chain_probe is False + else coordinator._block_candidate_chain_probe( + block_hash, + expected_height=expected_height, + ) + ) + with coordinator.lock: + accepted_race_won = bool( + preserve_if_accepted + and block_hash.lower() + in coordinator._accounted_accepted_block_hashes + ) + # A blockwait observation can also register during the blocking + # invalidation. The disposition seal must consult that evidence + # atomically or the same blind spot reopens inside this window; + # the probe still wins both directions. Metrics are committed + # later, after prepared-state cleanup can no longer reverse this + # decision. + late_acceptance_observed = bool( + not accepted_race_won + and ( + late_probe is True + or ( + late_probe is not False + and ( + coordinator._block_candidate_acceptance_observed( + block_hash + ) + or coordinator._block_candidate_acceptance_retained( + block_hash + ) + ) + ) + ) + ) + if not accepted_race_won and not late_acceptance_observed: + outcome.reason = reason + outcome.error = message + outcome.stale_job_class = stale_job_class + # Seal the disposition in the same critical section that + # commits it: stop matching tip observations for this hash so + # no acceptance evidence can register between this terminal + # decision and the caller's follow-up durable work (rejecting + # prepared payout rows). Observation registration takes this + # same lock, so exclusion across the gap is total. A crash + # before the durable outbox update replays the candidate, + # which re-registers and re-evaluates from live chain state. + self._outstanding_block_candidate_hashes.discard( + block_hash.lower() + ) + self._tip_observed_accepted_block_hashes.pop( + block_hash.lower(), + None, + ) + if accepted_race_won: + coordinator._clear_accepted_block_payout_preview(block_hash) + outcome.reason = None + outcome.error = None + return True + if late_acceptance_observed: + # Restore the landed barrier the withdrawal just removed (and pop + # its fail-closed tombstone): the candidate is still an accepted + # block pending finalization, so descendant builders must keep + # waiting on its preview -- not fail closed -- until the deferred + # retry's accepted tail republishes it. Without this, retries + # that keep deferring on observation evidence alone would leave + # the tombstone fencing template refreshes, recreating the + # coordination-blocked stall this path exists to prevent. + coordinator._begin_accepted_block_payout_preview( + block_hash, + block_height=expected_height, + ) + if expected_height is not None: + coordinator._mark_accepted_block_payout_landed( + block_hash, + block_height=expected_height, + ) + if ( + withdrawn_transition is not None + and withdrawn_transition.preview is not None + ): + # The withdrawal superseded payout publication and, on a + # lost publication race, left delivery fenced. Republish the + # withdrawn preview now so admission reopens with the defer + # rather than staying coordination-blocked across the + # deferral cycles until the accepted tail republishes it. + try: + with self._accepted_landing_phase( + block_hash, + LANDING_PHASE_PREVIEW_PREPARE, + block_height=expected_height, + ): + materialized = coordinator._materialize_prior_balance_preview( + withdrawn_transition.preview + ) + with self._accepted_landing_publish_span( + block_hash, + block_height=expected_height, + ): + coordinator._publish_accepted_block_payout_preview( + block_hash, + materialized, + ) + except Exception: + print( + "prism coordinator: could not republish withdrawn " + f"payout preview hash={block_hash}; the scheduled " + "refresh will retry publication", + flush=True, + ) + traceback.print_exc() + self._count_accept_pending_defer() + coordinator._defer_block_candidate( + PRISM_REJECTION_BLOCK_ACCEPT_PENDING, + "acceptance evidence arrived during payout-preview " + f"withdrawal; refusing terminal abandonment (was {reason}: " + f"{message})", + worker=worker, + ) + outcome.error = None + return False + + print( + f"prism coordinator: block candidate abandoned reason={reason}: {message}", + flush=True, + ) + return False + + # -- block-work liveness ----------------------------------------------- + + def _block_work_heartbeat_owner(self) -> tuple[str, str] | None: + """Return the independent heartbeat/phase slots owned by this thread.""" + current = threading.get_ident() + if current == getattr(self, "_block_submitter_thread_ident", None): + return "block_submitter", "_block_submitter_phase" + if current == getattr(self, "_block_accounting_thread_ident", None): + return "block_accounting", "_block_accounting_phase" + return None + + def _record_block_work_heartbeat(self, name: str, phase: str) -> None: + """Record a phase while preserving one-argument heartbeat embedders.""" + heartbeat = self._coordinator._record_heartbeat + try: + heartbeat(name, phase=phase) + except TypeError as exc: + # Preserve the historical one-argument heartbeat seam used by + # focused embedders. Do not hide TypeErrors raised by a heartbeat + # implementation that did accept the keyword. + if "unexpected keyword argument 'phase'" not in str(exc): + raise + heartbeat(name) + + def _record_block_submitter_phase(self, phase: str) -> None: + """Stamp a named phase only from a dedicated block-work owner.""" + owner = self._block_work_heartbeat_owner() + if owner is None: + return + heartbeat_name, phase_attribute = owner + setattr(self, phase_attribute, phase) + self._record_block_work_heartbeat(heartbeat_name, phase) + + def _record_block_submitter_wait(self, phase: str) -> None: + """Heartbeat owner waits while preserving lightweight test behavior.""" + owner = self._block_work_heartbeat_owner() + if owner is None and not hasattr(self, "_block_submitter_thread_ident"): + self._coordinator._record_heartbeat("block_submitter") + return + self._coordinator._record_block_submitter_phase(phase) + + def _block_work_wait_slice(self) -> float: + """Choose a polling slice that stays inside the configured watchdog.""" + watchdog_budget = max( + 0.001, + float(getattr(self._coordinator, "watchdog_timeout_seconds", 120.0)), + ) + return min( + BLOCK_SUBMITTER_WAIT_HEARTBEAT_SLICE_SECONDS, + max(0.001, watchdog_budget * 0.9), + ) + + def _observe_coordinator_lock_wait(self, elapsed_seconds: float) -> None: + """Keep a sliced coordinator-lock wait visible and watchdog-safe.""" + owner = self._block_work_heartbeat_owner() + if owner is None: + return + heartbeat_name, phase_attribute = owner + current_phase = getattr(self, phase_attribute, "unknown") + wait_phase = f"wait-lock:coordinator-state:{current_phase}" + self._record_block_work_heartbeat(heartbeat_name, wait_phase) + now = time.monotonic() + log_interval = float( + getattr( + self._coordinator, + "block_submit_lock_wait_log_seconds", + DEFAULT_BLOCK_SUBMIT_LOCK_WAIT_LOG_SECONDS, + ) + ) + last_log = float( + getattr(self, "_block_submitter_last_lock_wait_log_monotonic", 0.0) + ) + if last_log <= 0 or now - last_log >= log_interval: + self._block_submitter_last_lock_wait_log_monotonic = now + print( + "prism coordinator: block submitter waiting on lock " + f"lock=coordinator-state phase={current_phase} " + f"elapsed={elapsed_seconds:.3f}s", + flush=True, + ) + + def _acquire_block_submitter_lock(self, lock: Any, name: str) -> None: + """Acquire a submit-path lock in heartbeat/logging slices.""" + owner = self._block_work_heartbeat_owner() + if owner is None and ( + hasattr(self, "_block_submitter_thread_ident") + or hasattr(self, "_block_accounting_thread_ident") + ): + lock.acquire() + return + started = time.monotonic() + last_log = started + log_interval = float( + getattr( + self._coordinator, + "block_submit_lock_wait_log_seconds", + DEFAULT_BLOCK_SUBMIT_LOCK_WAIT_LOG_SECONDS, + ) + ) + while not lock.acquire(timeout=self._block_work_wait_slice()): + phase = f"wait-lock:{name}" + self._coordinator._record_block_submitter_wait(phase) + now = time.monotonic() + if now - last_log >= log_interval: + print( + "prism coordinator: block submitter waiting on lock " + f"lock={name} elapsed={now - started:.3f}s", + flush=True, + ) + last_log = now + + @contextmanager + def _block_submitter_lock(self, lock: Any, name: str) -> Iterator[None]: + self._coordinator._acquire_block_submitter_lock(lock, name) + try: + yield + finally: + lock.release() + + def _record_block_candidate_progress( + self, + phase: str = "accounting-progress", + ) -> None: + """Stamp the submitter heartbeat at a candidate-disposition boundary. + + Stamps come only from the dedicated submitter thread: dispositions + also run on client connection threads (synchronous below-target + solves), and a stamp from those threads would refresh the + ``block_submitter`` budget while the dedicated thread might be + wedged elsewhere. A disposition crosses several ledger and + filesystem writes that slow down together under database pressure; + stamping each completed phase on the owner thread keeps a + progressing disposition inside the liveness budget while a wedged + phase still leaves the watchdog able to recover the process -- the + same shape as the CTV broadcaster's per-row stamping, whose name + likewise maps to a single thread. + """ + self._coordinator._record_block_submitter_phase(phase) + + def _observe_block_submit_seconds(self, elapsed_seconds: float) -> None: + self._coordinator._ensure_job_cache_state() + with self._block_submit_metrics_lock: + histogram = self.block_submit_seconds_histogram + histogram["count"] = int(histogram["count"]) + 1 + histogram["sum"] = float(histogram["sum"]) + max( + 0.0, elapsed_seconds + ) + buckets = histogram["buckets"] + assert isinstance(buckets, dict) + for bucket in tuple(buckets): + if elapsed_seconds <= bucket: + buckets[bucket] = int(buckets.get(bucket, 0)) + 1 + + def _note_accepted_block_preview_acceptance(self, block_hash: str) -> None: + """Stamp the moment definitive node acceptance became known. + + Deliberately not ``AcceptedBlockPayoutTransition.landed_monotonic``: + that stamp is armed *before* the RPC (see + ``_unmark_accepted_block_payout_landed``, "the next attempt re-arms + landed before its own RPC"), so an interval measured from it would + include the offer itself and would restart on every retry. This one + is taken on the submitter thread at the instant the offer resolved + definitively, which is the earliest moment acceptance is knowable + without a further chain probe. + """ + key = str(block_hash).lower() + stamped = time.monotonic() + with self._accepted_block_preview_publication_lock: + stamps = self._accepted_block_preview_acceptance_monotonic + if key in stamps: + # Either an unpublished stamp from an earlier definitive + # offer of this hash, or its observed tombstone. Both mean + # the interval is already owned; a re-offer must not restart + # a measurement that is running or already recorded. + return + stamps[key] = stamped + overflow = len(stamps) - MAX_ACCEPTED_BLOCK_PREVIEW_PUBLICATION_STAMPS + if overflow > 0: + for stale in list(stamps)[:overflow]: + del stamps[stale] + + def _accepted_landing_attribution_locked( + self, + key: str, + *, + block_height: int | None = None, + ) -> _AcceptedLandingAttribution: + """Get or create one hash's record; the caller holds the leaf lock. + + Bounded exactly as the acceptance stamps are: past the shared cap the + oldest entries are evicted. The record returned here was just inserted + at the newest end, so the trim can never drop the one in use. + """ + records = self._accepted_landing_attributions + record = records.get(key) + if record is None: + record = _AcceptedLandingAttribution() + records[key] = record + overflow = ( + len(records) - MAX_ACCEPTED_BLOCK_PREVIEW_PUBLICATION_STAMPS + ) + if overflow > 0: + for stale in list(records)[:overflow]: + del records[stale] + if block_height is not None and record.block_height is None: + record.block_height = int(block_height) + return record + + def _accepted_landing_interval_open_locked(self, key: str) -> bool: + """Whether this hash's acceptance-to-publication interval is running.""" + stamps = self._accepted_block_preview_acceptance_monotonic + return stamps.get(key) is not None + + def _commit_accepted_landing_phase( + self, + telemetry: Any, + key: str, + phase: str, + seconds: float, + *, + block_height: int | None = None, + ) -> None: + """Record one measured sub-phase of a landing. + + The family observation is unconditional -- every stretch a landing ran + belongs in the phase histogram, and its only label is the fixed phase + name. Only the per-landing diagnostic is conditional, because a stretch + outside an open interval was not part of the latency that interval + measures. + """ + elapsed = max(0.0, float(seconds)) + with self._accepted_block_preview_publication_lock: + if self._accepted_landing_interval_open_locked(key): + record = self._accepted_landing_attribution_locked( + key, + block_height=block_height, + ) + record.phase_seconds[phase] = ( + float(record.phase_seconds.get(phase, 0.0)) + elapsed + ) + # Deliberately outside the leaf lock: the shared telemetry owner has a + # lock of its own, and this one never nests. + telemetry.observe_landing_phase(phase, elapsed) + + @contextmanager + def _accepted_landing_phase( + self, + block_hash: str, + phase: str, + *, + block_height: int | None = None, + ) -> Iterator[None]: + """Time one landing sub-phase; records on every exit, raising or not. + + The phase name is validated and the shared owner resolved before the + guarded body, so everything the ``finally`` below runs is arithmetic + under two leaf locks and can never replace the exception a failing + stretch is raising. + """ + if phase not in PRISM_ACCEPTED_LANDING_PHASES: + raise ValueError(f"unknown accepted landing phase: {phase!r}") + telemetry = ensure_accepted_preview_telemetry(self._coordinator) + key = str(block_hash).lower() + started = time.monotonic() + try: + yield + finally: + self._commit_accepted_landing_phase( + telemetry, + key, + phase, + time.monotonic() - started, + block_height=block_height, + ) + + def _close_accepted_landing_lane_wait( + self, + block_hash: str, + *, + block_height: int | None = None, + ) -> None: + """Close ``lane_wait`` where the lane starts this hash's landing task. + + Once per open interval. A retried landing re-enters the lane, and + measuring again from the same acceptance stamp would charge the retry's + queue time to a stretch the first attempt already owns -- and would + keep growing that stretch for as long as the candidate kept retrying. + """ + key = str(block_hash).lower() + now = time.monotonic() + with self._accepted_block_preview_publication_lock: + started = self._accepted_block_preview_acceptance_monotonic.get(key) + if started is None: + # No definitive acceptance was stamped for this hash in this + # process, or its interval already closed. There is no lane + # wait to measure either way. + return + record = self._accepted_landing_attribution_locked( + key, + block_height=block_height, + ) + if record.lane_wait_recorded: + return + record.lane_wait_recorded = True + elapsed = max(0.0, now - float(started)) + record.phase_seconds[LANDING_PHASE_LANE_WAIT] = elapsed + ensure_accepted_preview_telemetry(self._coordinator).observe_landing_phase( + LANDING_PHASE_LANE_WAIT, + elapsed, + ) + + def _note_accepted_landing_lane_start(self, candidate: Any) -> None: + """Close ``lane_wait`` for the candidate whose landing starts now.""" + block_hash, block_height = _accepted_landing_candidate_identity(candidate) + if block_hash is None: + return + self._close_accepted_landing_lane_wait( + block_hash, + block_height=block_height, + ) + + @contextmanager + def _accepted_landing_publish_span( + self, + block_hash: str, + *, + block_height: int | None = None, + ) -> Iterator[None]: + """Time ``preview_publish`` up to the preview's visibility instant. + + The publication path closes the interval from inside the publish call, + so a plain timer around that call would record its phase after the + diagnostic had already been built from an empty cell. The span is + opened here and closed by whichever comes first: the observer below, at + the visibility instant it already measures, or this context manager's + exit when the call published nothing this hash was waiting for. Never + both -- the start value is the span's token. + + The record is allocated only for an open interval, exactly as every + other stretch allocates one. A startup replay or a chain-proven + candidate publishes without an in-process acceptance stamp, and the + observer returns before it can pop anything; a record opened for one of + those would be reported as in flight forever and would spend a slot the + bound reserves for live landings. The family observation below stays + unconditional either way -- every stretch a landing ran belongs in the + phase histogram. + """ + telemetry = ensure_accepted_preview_telemetry(self._coordinator) + key = str(block_hash).lower() + started = time.monotonic() + with self._accepted_block_preview_publication_lock: + attributed = self._accepted_landing_interval_open_locked(key) + if attributed: + record = self._accepted_landing_attribution_locked( + key, + block_height=block_height, + ) + record.publish_started_monotonic = started + try: + yield + finally: + self._close_accepted_landing_publish_span( + telemetry, + key, + started, + attributed=attributed, + ) + + def _close_accepted_landing_publish_span( + self, + telemetry: Any, + key: str, + started: float, + *, + attributed: bool = True, + ) -> None: + """Close a publish span this caller still owns, if it still owns it. + + ``attributed`` is what the span found when it opened: False means no + interval was running and no record was allocated, so there is nothing + to own and nothing the observer could have taken -- only the family + histogram is owed a sample. + """ + ended = time.monotonic() + elapsed = max(0.0, ended - started) + with self._accepted_block_preview_publication_lock: + if attributed: + record = self._accepted_landing_attributions.get(key) + if record is None or record.publish_started_monotonic != started: + # The observer closed this span at the visibility instant + # and recorded it there, or terminal cleanup discarded the + # record while the publication was in flight. + return + record.publish_started_monotonic = None + if self._accepted_landing_interval_open_locked(key): + record.phase_seconds[LANDING_PHASE_PREVIEW_PUBLISH] = ( + float( + record.phase_seconds.get( + LANDING_PHASE_PREVIEW_PUBLISH, 0.0 + ) + ) + + elapsed + ) + else: + # The interval closed under this publication without the + # observer taking the span: the acceptance stamp aged out + # of the shared cap. Nothing can build a diagnostic from + # this record now, so drop it rather than leave it in + # flight for the bound to evict a live landing over. + self._accepted_landing_attributions.pop(key, None) + telemetry.observe_landing_phase(LANDING_PHASE_PREVIEW_PUBLISH, elapsed) + + def _discard_accepted_landing_attribution(self, block_hash: str) -> None: + """Drop one hash's in-flight phase state on terminal disposition. + + Nothing can close the interval of a hash that will never publish, so + its record has no consumer left. The cap above would evict it + eventually; dropping it here keeps a storm of terminal candidates from + pushing live landings out of the bound. + """ + key = str(block_hash).lower() + with self._accepted_block_preview_publication_lock: + self._accepted_landing_attributions.pop(key, None) + + def accepted_landing_attribution_snapshot(self) -> dict[str, dict[str, float]]: + """Copied in-flight per-hash phase state, oldest first.""" + with self._accepted_block_preview_publication_lock: + return { + key: dict(record.phase_seconds) + for key, record in self._accepted_landing_attributions.items() + } + + def _record_accepted_landing_diagnostic( + self, + key: str, + *, + result: str, + elapsed: float, + record: _AcceptedLandingAttribution | None, + publish_seconds: float | None, + ) -> None: + """Retain and log one closed landing, outside the leaf lock. + + The rescan and ledger totals stay at the contract's neutral values. + This lane can obtain no per-landing causal evidence for either without + reading a process-wide delta, and such a read here would claim whatever + else the process happened to be doing -- a concurrent job build's + rescan, another lane's ledger reads -- as this block's work. The + separately attributed rescan and ledger-read families answer those + questions without the race. + """ + telemetry = ensure_accepted_preview_telemetry(self._coordinator) + if publish_seconds is not None: + telemetry.observe_landing_phase( + LANDING_PHASE_PREVIEW_PUBLISH, + publish_seconds, + ) + try: + diagnostic = AcceptedPreviewPublicationDiagnostic( + block_hash=key, + block_height=None if record is None else record.block_height, + result=result, + acceptance_to_publication_seconds=elapsed, + phase_seconds=( + dict(record.phase_seconds) if record is not None else {} + ), + reconcile_caller=RECONCILE_CALLER_LANDING, + ) + telemetry.record_publication_diagnostic(diagnostic) + _accepted_landing_log( + PRISM_ACCEPTED_LANDING_LOG_EVENT, + **diagnostic.log_fields(), + ) + except Exception: + # Attribution is observability. A record the contract refuses, or + # a log line that cannot be written, must not fail a publication + # that has already made the preview visible to waiting children. + traceback.print_exc() + + def _observe_accepted_block_preview_publication( + self, + block_hash: str, + *, + result: str, + ) -> None: + """Close the acceptance-to-publication interval for one hash. + + Only the first publication of a hash is measured: it is the one that + made the preview visible to children already waiting on the + transition, which is the latency the 5 s child wait budget is spent + against. Later matching republications observe nothing. + + That same first publication owns the landing's attribution (issue + #224): it is the only moment at which the interval, the phases + measured under it, and the publication result are all known and + final, so the diagnostic is built here and nowhere else. + """ + key = str(block_hash).lower() + published = time.monotonic() + with self._accepted_block_preview_publication_lock: + stamps = self._accepted_block_preview_acceptance_monotonic + if key not in stamps: + # No definitive offer of this hash was stamped in this + # process: a replayed candidate confirmed by chain probe, or + # a preview republished for an acceptance this process never + # made. There is no interval to close, and inventing one from + # the publication alone would report a zero. + return + started = stamps[key] + if started is None: + return + histogram = self.accepted_block_preview_publication_seconds_histogram.get( + result + ) + if histogram is None: + # The label set is closed by + # PRISM_ACCEPTED_BLOCK_PREVIEW_PUBLICATION_RESULTS. Drop the + # observation rather than growing series cardinality, and + # leave the stamp for a labelled publication to close. + return + stamps[key] = None + elapsed = max(0.0, published - float(started)) + histogram["count"] = int(histogram["count"]) + 1 + histogram["sum"] = float(histogram["sum"]) + elapsed + buckets = histogram["buckets"] + assert isinstance(buckets, dict) + for bucket in tuple(buckets): + if elapsed <= bucket: + buckets[bucket] = int(buckets.get(bucket, 0)) + 1 + # Pop the landing's attribution under the same lock that + # tombstoned the stamp, so a retry or a later republication of + # this hash can never build a second diagnostic for it. An open + # publish span belongs to this call: the preview is visible to + # waiting children exactly now. + record = self._accepted_landing_attributions.pop(key, None) + publish_seconds: float | None = None + if record is not None and record.publish_started_monotonic is not None: + publish_seconds = max( + 0.0, + published - float(record.publish_started_monotonic), + ) + record.publish_started_monotonic = None + record.phase_seconds[LANDING_PHASE_PREVIEW_PUBLISH] = ( + float( + record.phase_seconds.get(LANDING_PHASE_PREVIEW_PUBLISH, 0.0) + ) + + publish_seconds + ) + # Outside the leaf lock: the telemetry owner locks for itself and the + # lifecycle line is I/O. + self._record_accepted_landing_diagnostic( + key, + result=result, + elapsed=elapsed, + record=record, + publish_seconds=publish_seconds, + ) + + +class BlockCandidateCompatibilityField: + """Route temporary coordinator fields to the B1 service owner.""" + + def __init__(self, name: str, default: Any) -> None: + self.name = name + self.default = default + + def __get__(self, instance: Any, owner: type[Any]) -> Any: + if instance is None: + return self + service = instance.__dict__.get("_block_candidate_service") + if service is None: + value = instance.__dict__.get(self.name, self.default) + if callable(value) and getattr(value, "__candidate_default_factory__", False): + value = value() + instance.__dict__[self.name] = value + return value + return _compat_get(service, self.name) + + def __set__(self, instance: Any, value: Any) -> None: + service = instance.__dict__.get("_block_candidate_service") + if service is None: + instance.__dict__[self.name] = value + return + _compat_set(service, self.name, value) + + +def compatibility_default(factory: Callable[[], Any]) -> Callable[[], Any]: + setattr(factory, "__candidate_default_factory__", True) + return factory + + +class BlockCandidateStateField: + """Route one #113-era B1 coordinator field to the service owner. + + The descriptor keeps the historical attribute name readable and writable + on the coordinator while :class:`BlockCandidateService` owns the single + mutable copy, mirroring the S1/S2/S3/J1/P1/G1 extraction pattern. Fields + the service creates lazily preserve the pre-extraction ``hasattr``/ + ``getattr``-with-default semantics exactly: reading an unset field raises + AttributeError until first write. + """ + + def __init__(self, name: str, attribute: str | None = None) -> None: + self.name = name + self.attribute = attribute or name + + def __get__(self, instance: Any, owner: type[Any] | None = None) -> Any: + if instance is None: + return self + return getattr( + instance._ensure_block_candidate_service(), + self.attribute, + ) + + def __set__(self, instance: Any, value: Any) -> None: + setattr( + instance._ensure_block_candidate_service(), + self.attribute, + value, + ) + + +_COMPATIBILITY_FIELD_MAP = { + "block_candidate_queue": "candidate_queue", + "block_candidates_dropped": "dropped", + "block_candidate_wakeups_coalesced": "wakeups_coalesced", + "block_candidate_retry_count": "retries", + "block_candidate_poisoned_count": "poisoned", + "block_candidate_retry_initial_seconds": "retry_initial_seconds", + "block_candidate_retry_max_seconds": "retry_max_seconds", + "block_candidate_retry_delays": "retry_delays", + "_block_candidate_finalize_retries": "finalize_retries", + "block_candidate_abandoned_counts": "abandoned_counts", + "_retry_block_candidate": "retry_candidate", + "_block_candidate_outcome": "outcome", +} + +# The #113-era coordinator fields routed one-to-one onto the service. The +# handful whose service spelling differs (the pre-#113 backoff quartet) map +# through their historical service names. +_STATE_FIELD_MAP = { + "_block_replay_candidate_queue": "_block_replay_candidate_queue", + "_block_replay_inflight_hashes": "_block_replay_inflight_hashes", + "_block_quarantine_queue": "_block_quarantine_queue", + "_block_quarantine_hashes": "_block_quarantine_hashes", + "_block_candidate_disposition_registry_lock": "_block_candidate_disposition_registry_lock", + "_block_candidate_disposition_flights": "_block_candidate_disposition_flights", + "_block_candidate_terminal_outcomes": "_block_candidate_terminal_outcomes", + "_block_fast_lane_reservations": "_block_fast_lane_reservations", + "_block_disposition_waiting_retries": "_block_disposition_waiting_retries", + "_block_candidate_dequeued_hashes": "_block_candidate_dequeued_hashes", + "_block_accounting_state_lock": "_block_accounting_state_lock", + "_block_accounting_queue": "_block_accounting_queue", + "_block_accounting_overflow_queue": "_block_accounting_overflow_queue", + "_block_accounting_accepted_queue": "_block_accounting_accepted_queue", + "_block_accounting_sequence": "_block_accounting_sequence", + "_block_accounting_thread": "_block_accounting_thread", + "_block_accounting_thread_ident": "_block_accounting_thread_ident", + "_block_accounting_holds_disposition": "_block_accounting_holds_disposition", + "_block_accounting_deferred_retry_candidate": "_block_accounting_deferred_retry_candidate", + "_block_accounting_phase": "_block_accounting_phase", + "_block_submitter_thread_ident": "_block_submitter_thread_ident", + "_block_submitter_phase": "_block_submitter_phase", + "_block_submitter_last_lock_wait_log_monotonic": "_block_submitter_last_lock_wait_log_monotonic", + "_block_submitter_retry_state_lock": "_state_lock", + "_block_submitter_backoff_started_monotonic": "_backoff_started_monotonic", + "_block_submitter_backoff_deadline_monotonic": "_backoff_deadline_monotonic", + "_block_submitter_backoff_delay_seconds": "_backoff_delay_seconds", + "_block_submitter_ledger_calls_lock": "_block_submitter_ledger_calls_lock", + "_block_submitter_ledger_calls": "_block_submitter_ledger_calls", + "_block_submitter_ledger_worker_slots": "_block_submitter_ledger_worker_slots", + "_block_submitter_rpc_calls_lock": "_block_submitter_rpc_calls_lock", + "_block_submitter_rpc_calls": "_block_submitter_rpc_calls", + "_block_submitter_rpc_worker_slots": "_block_submitter_rpc_worker_slots", + "_block_landing_timeout_counts": "_block_landing_timeout_counts", + "_block_ledger_call_metrics_lock": "_block_ledger_call_metrics_lock", + "_block_ledger_call_metrics": "_block_ledger_call_metrics", + "_block_candidate_retry_not_before": "_block_candidate_retry_not_before", + "_block_candidate_retained_node_submissions": "_block_candidate_retained_node_submissions", + "_block_candidate_retained_submission_monotonic": "_block_candidate_retained_submission_monotonic", + "_counted_block_candidate_abandonments": "_counted_block_candidate_abandonments", + "_outstanding_block_candidate_hashes": "_outstanding_block_candidate_hashes", + "_tip_observed_accepted_block_hashes": "_tip_observed_accepted_block_hashes", + "block_candidate_accept_pending_defer_count": "block_candidate_accept_pending_defer_count", + "accepted_parent_redrive_attempt_count": "accepted_parent_redrive_attempt_count", + "accepted_parent_redrive_resolved_count": "accepted_parent_redrive_resolved_count", + "accepted_parent_redrive_exhausted_count": "accepted_parent_redrive_exhausted_count", + "stale_job_abandon_counts": "stale_job_abandon_counts", + "_block_submit_metrics_lock": "_block_submit_metrics_lock", + "block_submit_seconds_histogram": "block_submit_seconds_histogram", + "_accepted_block_preview_publication_lock": ( + "_accepted_block_preview_publication_lock" + ), + "accepted_block_preview_publication_seconds_histogram": ( + "accepted_block_preview_publication_seconds_histogram" + ), + "_accepted_block_preview_acceptance_monotonic": ( + "_accepted_block_preview_acceptance_monotonic" + ), + "_block_replay_enumeration_owed_flag": "_block_replay_enumeration_owed_flag", +} + + +def _compat_get(service: BlockCandidateService, name: str) -> Any: + return getattr(service, _COMPATIBILITY_FIELD_MAP[name]) + + +def _compat_set(service: BlockCandidateService, name: str, value: Any) -> None: + setattr(service, _COMPATIBILITY_FIELD_MAP[name], value) diff --git a/lab/prism/block_finalization.py b/lab/prism/block_finalization.py new file mode 100644 index 000000000..b3f46117a --- /dev/null +++ b/lab/prism/block_finalization.py @@ -0,0 +1,2365 @@ +"""Measured, replay-safe PRISM accepted-block finalization. + +B3 owner module: correctness after node-offer evidence exists. It owns the +post-offer admission/finalization policy of ``submit_block_candidate`` and +``_submit_block_candidate_serialized``, the replay-window decision that +consumes P1's durable window proof, ``_land_and_confirm_block_candidate``, +the CTV/share-credit tail, ordered PR 75 audit evidence publication, and +accepted-block process accounting. The node offer, same-hash disposition, +accounting handoff, and durable outbox terminalization stay with B1 +(``lab/prism/block_candidates.py``); payout previews, balance serialization, +append/anchor fences, and window proofs stay with P1 +(``lab/prism/payout_state.py``). The service accepts B1's prepared +``node_submission`` and never creates a second transport path — its rare +fenced fallback calls B1's node-offer seam through the runtime. +""" + +from __future__ import annotations + +from contextlib import AbstractContextManager, ExitStack, contextmanager +from dataclasses import dataclass +import os +from pathlib import Path +import sys +import threading +import time +import traceback +from typing import Any, Callable, Iterator, Mapping, Protocol + +from lab.prism import direct_stratum, public_api +from lab.prism.accepted_preview_telemetry import ( + LANDING_PHASE_BALANCE_LOCK_WAIT, + LANDING_PHASE_CHAIN_PROBE, + LANDING_PHASE_PREVIEW_PREPARE, + LANDING_PHASE_PRIOR_BALANCES_CHECK, + LANDING_PHASE_RECONCILE, +) +from lab.prism.audit_artifacts import AuditArtifactStore, AuditPublicationIdentity +from lab.prism.audit_bundle_view import streamed_sha256_json_hex +from lab.prism.block_candidates import ( + PRISM_REJECTION_LEDGER_CONFIRMATION_SUPERSEDED, + PrismBlockCandidate, + _BlockCandidateNodeSubmission, +) +from lab.prism.coordinator_shutdown import ShutdownInProgress +from lab.prism.share_ledger import WriterLeaseRenewalDeferred +from lab.prism.share_submission import ( + PRISM_REJECTION_BACKEND_RPC_UNAVAILABLE, + PRISM_REJECTION_POOL_CLOSED, + PRISM_REJECTION_STALE_JOB, +) + + +PRISM_REJECTION_CANDIDATE_AUDIT_MISMATCH = "candidate-audit-mismatch" +PRISM_REJECTION_SUBMITBLOCK_REJECTED = "submitblock-rejected" +PRISM_REJECTION_BLOCK_STALE = "block-stale" +PRISM_REJECTION_LEDGER_CONFIRMATION_FAILED = "ledger-confirmation-failed" +FINALIZATION_PHASES = ( + "admission", + "land_confirm", + "ctv_credit", + "evidence", + "audit_publish", + "accounting", +) + + +def _declared_anchor_ms(context: Any) -> int | None: + """The payout window anchor this candidate's job declared, if any. + + ``found_block`` records the window the issued job was built from, and its + anchor is what scopes an append-invalidation question to this candidate + rather than to whatever else moved the global epoch. A candidate that + carries no anchor -- a reconstruction whose intent predates the field -- + cannot scope the question at all, and every fence that asks reads ``None`` + as invalidated. + """ + found_block = getattr(context, "found_block", None) + if not isinstance(found_block, dict): + return None + anchor_ms = found_block.get("anchor_job_issued_at_ms") + return None if anchor_ms is None else int(anchor_ms) + + +@dataclass(frozen=True) +class FinalizationAdmission: + """Immutable result of candidate admission and active-chain classification. + + ``current_tip`` records the observed post-probe tip that payout + reconciliation must follow; the fresh-attempt classification against the + stamped parent stays local to admission. ``node_submission`` carries B1's + fast-lane offer evidence into the landing phase unchanged. + """ + + candidate: PrismBlockCandidate + context: Any + submission: direct_stratum.DirectQbitSubmission + worker: str | None + expected_height: int + block_hash: str + parent_hash: str + current_tip: str + already_active: bool + node_submission: _BlockCandidateNodeSubmission + + +@dataclass(frozen=True) +class LandedCandidate: + """Durable landing outputs consumed by the remaining ordered phases. + + Combines the landing's four accounting mappings with PR 75's ordered + audit publication identity and the verifier's identity evidence. + """ + + # For a canonical build this is the bounded artifact view returned by + # the bundle compiler: its small members are parsed, its window-sized + # share arrays stay on disk. The later phases read only header members + # (coinbase manifest, CTV manifest set), and the view is closed once the + # ordered phases finish (issue #255). + final_bundle: Mapping[str, Any] + report: dict[str, Any] + persistence: dict[str, Any] + confirmation: dict[str, Any] + audit_publication_identity: AuditPublicationIdentity + audit_verification_identity: dict[str, Any] + + +@dataclass(frozen=True) +class FinalizationEvidence: + """Evidence body and its normalized publication persistence identity.""" + + evidence: dict[str, Any] + publication_persistence: dict[str, Any] + + +class BlockFinalizationPort(Protocol): + """Explicit coordinator capabilities required by finalization. + + Every runtime attribute and method the service touches is declared here, + with one intentional exception: the ``verify_bundle`` verifier override is + consumed only through ``runtime.__dict__`` as a per-instance escape hatch + and deliberately stays out of the port. + """ + + ledger: Any + lock: Any + rpc: Any + accepted_block_count: int + latest_coinbase_size_bytes: int | None + ledger_writer_public_key_hex: str | None + max_blocks: int + reorg_reconciler_enabled: bool + stop_after_block: bool + _accounted_accepted_block_hashes: set[str] + _block_candidate_outcome: threading.local | None + _block_fast_lane_reservations: set[str] + _job_cache_lock: Any + _payout_append_landing_fence_lock: Any + _payout_balance_mutation_lock: Any + _payout_ledger_append_invalidation_epoch: int + _payout_state_generation: int + _payout_state_source: tuple[int, str | None, str, float] + _template_artifacts: Any + + def _abandon_block_candidate( + self, + reason: str, + message: str, + *, + block_hash: str, + worker: str | None, + preserve_if_accepted: bool = False, + expected_height: int | None = None, + stale_job_class: str | None = None, + ) -> bool: ... + + def _accepted_block_payout_preview_from_bundle( + self, + final_bundle: Mapping[str, Any], + *, + prior_balances: list[dict[str, object]] | None = None, + ) -> list[dict[str, object]]: ... + + def _accepted_block_payout_transition_landed(self, block_hash: str) -> bool: ... + + def _append_epoch_invalidated_declared_anchor( + self, + *, + baseline_epoch: int, + live_epoch: int, + declared_anchor_ms: int | None, + ) -> bool: ... + + def _audit_publication_identity( + self, + *, + block_hash: str, + block_height: int, + confirmation: Mapping[str, Any], + ) -> AuditPublicationIdentity: ... + + def _await_unfenced_appends_predating_anchor(self, anchor_ms: int) -> None: ... + + def _begin_accepted_block_payout_preview( + self, + block_hash: str, + *, + block_height: int | None = None, + ) -> None: ... + + def _block_candidate_acceptance_recorded(self, block_hash: str) -> bool: ... + + def _block_candidate_chain_probe( + self, + block_hash: str, + *, + expected_height: int | None = None, + ) -> bool | None: ... + + def _block_candidate_disposition( + self, + block_hash: str, + ) -> AbstractContextManager[Any]: ... + + def _block_candidate_terminal_outcome(self, block_hash: str) -> bool | None: ... + + def _block_payout_state_publication( + self, + *, + force: bool = False, + supersede_with: tuple[int, str | None, str, float] | None = None, + ) -> None: ... + + def _block_submitter_lock( + self, + lock: Any, + name: str, + ) -> AbstractContextManager[None]: ... + + def _cancel_obsolete_job_builds( + self, + reason: str, + *, + keep_published_snapshot: bool = False, + ) -> None: ... + + def _capture_payout_state_source( + self, + ) -> tuple[int, int, str | None, str, float]: ... + + def _clear_accepted_block_payout_preview( + self, + block_hash: str, + *, + invalidate_published: bool = False, + ) -> None: ... + + def _defer_for_pending_parent_payout_transition( + self, + *, + block_hash: str, + parent_hash: str, + parent_height: int, + worker: str | None, + active_candidate_hash: str | None = None, + active_candidate_height: int | None = None, + ) -> bool: ... + + def _ensure_audit_artifact_store(self) -> AuditArtifactStore: ... + + def _ensure_block_candidate_service(self) -> Any: ... + + def _ensure_job_cache_state(self) -> None: ... + + def _expose_inflight_scan_anchor(self, anchor_ms: int) -> int: ... + + def _land_and_confirm_block_candidate( + self, + candidate: PrismBlockCandidate, + *, + current_tip: str, + already_active: bool, + worker: str | None, + node_submission: _BlockCandidateNodeSubmission, + revalidated_append_epoch: int | None = None, + ) -> tuple[ + dict[str, Any], + dict[str, Any], + dict[str, Any], + dict[str, Any], + AuditPublicationIdentity, + dict[str, Any], + ] | None: ... + + def _mark_accepted_block_payout_landed( + self, + block_hash: str, + *, + block_height: int, + ) -> None: ... + + def _mark_tip_refresh_pending(self, _observation: object) -> int: ... + + def _materialize_prior_balance_preview( + self, + preview: tuple[tuple[str, str, str, int], ...], + ) -> list[dict[str, object]]: ... + + def _node_submission_for_direct_candidate( + self, + candidate: PrismBlockCandidate, + ) -> _BlockCandidateNodeSubmission: ... + + def _note_tip_observation_for_candidates(self, tip_hash: str) -> None: ... + + def _observe_payout_state_seconds( + self, + name: str, + elapsed_seconds: float, + *, + relation: str | None = None, + ) -> None: ... + + def _payout_balance_serializer_released( + self, + ) -> AbstractContextManager[None]: ... + + def _payout_source_requires_publication(self, candidate: Any = None) -> bool: ... + + def _payout_state_publication_fenced(self) -> bool: ... + + def _publish_accepted_block_payout_preview( + self, + block_hash: str, + balances: list[dict[str, object]], + ) -> list[dict[str, object]]: ... + + def _publish_current_payout_state_with_retry_budget( + self, + *, + initial_attempted: bool = False, + ) -> int | None: ... + + def _record_block_candidate_progress( + self, + phase: str = "accounting-progress", + ) -> None: ... + + def _record_block_submitter_phase(self, phase: str) -> None: ... + + def _record_committed_block_candidate_abandonment( + self, + block_hash: str, + outcome: threading.local, + ) -> None: ... + + def _register_outstanding_block_candidate(self, block_hash: str) -> None: ... + + def _replayed_payout_window_reproducible(self, context: Any) -> bool: ... + + def _require_fresh_ledger_lease_for_external_side_effect( + self, + component: str, + ) -> None: ... + + def _retire_inflight_scan_anchor(self, token: int | None) -> None: ... + + def _schedule_current_payout_ledger_artifact_if_missing(self) -> None: ... + + def _schedule_payout_ledger_artifact_preparation( + self, + payout_state_generation: int, + network_difficulty: int, + *, + min_interval_seconds: float | None = None, + bypass_build_interval: bool = False, + ) -> None: ... + + def _schedule_tip_refresh_retry(self) -> None: ... + + def _submit_block_candidate_serialized( + self, + candidate: PrismBlockCandidate, + *, + node_submission: _BlockCandidateNodeSubmission, + ) -> bool: ... + + def _submit_block_candidate_to_node( + self, + candidate: PrismBlockCandidate, + ) -> _BlockCandidateNodeSubmission: ... + + def _unmark_accepted_block_payout_landed(self, block_hash: str) -> None: ... + + def accepted_share_stats(self) -> tuple[int, int]: ... + + def active_block_candidate_height(self, block_hash: str) -> int | None: ... + + def append_accepted_share( + self, + client: Any, + context: Any, + submission: direct_stratum.DirectQbitSubmission, + pending_share: Any, + *, + credit_policy: str | None = None, + candidate_intent: dict[str, Any] | None = None, + ) -> str | None: ... + + def block_candidate_intent( + self, + candidate: PrismBlockCandidate, + ) -> dict[str, Any]: ... + + def build_audit_bundle( + self, + *, + shares: list[dict[str, object]], + found_block: dict[str, object], + prior_balances: list[dict[str, object]], + coinbase_script_sig_suffix_hex: str, + witness_merkle_leaves_hex: list[str] | None = None, + ctv_fee_parent_hash: str | None = None, + canonical_output_path: Path | None = None, + canonical_output_parent_fd: int | None = None, + canonical_output_adopter: Callable[[Path, os.stat_result], None] | None = None, + ) -> Mapping[str, Any]: ... + + def coinbase_script_sig_suffix_hex( + self, + extranonce1_hex: str, + extranonce2_hex: str, + ) -> str: ... + + def ensure_reorg_reconciled_for_tip( + self, + tip_hash: str, + *, + _coalesce_same_tip: bool = True, + ) -> bool: ... + + def normalized_prior_balances( + self, + balances: list[dict[str, object]], + ) -> list[dict[str, object]]: ... + + def prior_balances_match_current( + self, + prior_balances: list[dict[str, object]], + ) -> bool: ... + + def reconcile_prism_pool_blocks_once( + self, + *, + tip_hash: str | None = None, + _force_publish: bool = False, + _source_reserved: bool = False, + _wait_for_same_tip_flight: bool = True, + ) -> dict[str, object]: ... + + def reject_prepared_block( + self, + *, + block_hash: str, + active_tip_height: int, + ) -> dict[str, int | str]: ... + + def request_shutdown(self, signum: int | None = None) -> None: ... + + def settlement_balances_by_program( + self, + balances: list[dict[str, object]], + ) -> dict[str, int]: ... + + def trusted_ledger_writer_public_key_hex( + self, + bundle: Mapping[str, Any], + ) -> str: ... + + +class BlockFinalizationService: + """Own accepted-block finalization over explicit infrastructure ports. + + The service deliberately receives the whole coordinator as its runtime: + the finalization tail touches many coordinator-owned seams that focused + tests monkeypatch per instance, so every runtime capability is reached + explicitly through ``self.runtime`` against the typed + ``BlockFinalizationPort`` while the bounded phase metrics stay + service-owned. + """ + + runtime: BlockFinalizationPort + + def __init__(self, runtime: BlockFinalizationPort) -> None: + self.runtime = runtime + self._metrics_lock = threading.Lock() + self._phase_metrics = { + phase: {"count": 0, "sum": 0.0, "max": 0.0} + for phase in FINALIZATION_PHASES + } + self._last_candidate_started: float | None = None + self._candidate_intervals: dict[str, int | float | None] = { + "count": 0, + "sum": 0.0, + "min": None, + } + + @contextmanager + def _phase(self, name: str) -> Iterator[None]: + started = time.monotonic() + try: + yield + finally: + elapsed = max(0.0, time.monotonic() - started) + with self._metrics_lock: + metric = self._phase_metrics[name] + metric["count"] = int(metric["count"]) + 1 + metric["sum"] = float(metric["sum"]) + elapsed + metric["max"] = max(float(metric["max"]), elapsed) + + def _record_block_candidate_progress( + self, + phase: str = "accounting-progress", + ) -> None: + # Service-level compatibility seam: progress marks stay dispatched + # through the service so focused tests can intercept them with a + # subclass override; the base implementation forwards to the port. + self.runtime._record_block_candidate_progress(phase) + + def _accepted_landing_owner(self) -> Any | None: + """The B1 owner of per-landing phase attribution (issue #224). + + The landing's sub-phases are measured here, but the interval they + belong to is opened by B1 at definitive node acceptance and closed by + B1 at the first publication, so the per-hash state lives beside B1's + acceptance stamps rather than in this service. A runtime without that + seam -- the focused fakes that drive this service directly -- records + nothing and behaves exactly as before. + """ + ensure = getattr(self.runtime, "_ensure_block_candidate_service", None) + if not callable(ensure): + return None + return ensure() + + @contextmanager + def _landing_phase( + self, + phase: str, + *, + block_hash: str | None, + block_height: int | None = None, + ) -> Iterator[None]: + """Time one accepted-landing sub-phase through the B1 owner. + + The owner is resolved before the guarded body, so the recording it + runs on the way out is arithmetic under leaf locks and can never + replace the exception a failing stretch is raising. A call with no + block hash -- the disposition-path reread that is not landing a + candidate -- times nothing. + """ + owner = None if block_hash is None else self._accepted_landing_owner() + if owner is None: + yield + return + with owner._accepted_landing_phase( + block_hash, + phase, + block_height=block_height, + ): + yield + + @contextmanager + def _landing_publish_span( + self, + *, + block_hash: str, + block_height: int | None = None, + ) -> Iterator[None]: + """Time ``preview_publish`` up to the preview's visibility instant. + + The publication closes the interval from inside the publish call, so + B1 owns both ends of this span: it is opened here and closed by + whichever comes first, the publication observer or this exit. + """ + owner = self._accepted_landing_owner() + if owner is None: + yield + return + with owner._accepted_landing_publish_span( + block_hash, + block_height=block_height, + ): + yield + + def _stamped_prior_balances_match_current( + self, + prior_balances: list[dict[str, object]], + *, + block_hash: str | None = None, + block_height: int | None = None, + ) -> bool: + """Compare the job's payout base against the ledger, bracketed. + + The comparison is a ledger read run inline on the landing's + block-work thread, between the reorg-reconcile stretch and the + audit build. Bracketing it keeps that statement attributable: an + owner parked in ``prior-balances-check`` is waiting on a known + database read rather than sitting in an unnamed silence the + watchdog cannot tell from a wedge. Both landing call sites route + through here so the pair of stamps cannot drift apart. The stampers + no-op off the block-work owner thread, so client-thread + dispositions are unaffected. + """ + self.runtime._record_block_submitter_phase("prior-balances-check") + with self._landing_phase( + LANDING_PHASE_PRIOR_BALANCES_CHECK, + block_hash=block_hash, + block_height=block_height, + ): + matched = bool(self.runtime.prior_balances_match_current(prior_balances)) + self.runtime._record_block_submitter_phase("prior-balances-check:complete") + return matched + + def _note_candidate_started(self) -> None: + now = time.monotonic() + with self._metrics_lock: + previous = self._last_candidate_started + self._last_candidate_started = now + if previous is None: + return + interval = max(0.0, now - previous) + metric = self._candidate_intervals + metric["count"] = int(metric["count"]) + 1 + metric["sum"] = float(metric["sum"]) + interval + current_min = metric["min"] + metric["min"] = ( + interval if current_min is None else min(float(current_min), interval) + ) + + def metrics_snapshot(self) -> dict[str, Any]: + with self._metrics_lock: + return { + "phases": { + name: dict(value) for name, value in self._phase_metrics.items() + }, + "candidate_intervals": dict(self._candidate_intervals), + } + + def metrics_lines(self) -> list[str]: + snapshot = self.metrics_snapshot() + phases = snapshot["phases"] + intervals = snapshot["candidate_intervals"] + lines = [ + "# HELP qbit_prism_block_finalization_phase_seconds Accepted-block finalization wall time by ordered phase.", + "# TYPE qbit_prism_block_finalization_phase_seconds summary", + ] + for phase in FINALIZATION_PHASES: + metric = phases[phase] + lines.extend( + [ + f'qbit_prism_block_finalization_phase_seconds_sum{{phase="{phase}"}} {float(metric["sum"]):.6f}', + f'qbit_prism_block_finalization_phase_seconds_count{{phase="{phase}"}} {int(metric["count"])}', + f'qbit_prism_block_finalization_phase_seconds_max{{phase="{phase}"}} {float(metric["max"]):.6f}', + ] + ) + interval_count = int(intervals["count"]) + interval_sum = float(intervals["sum"]) + interval_min = intervals["min"] + lines.extend( + [ + "# HELP qbit_prism_block_candidate_interarrival_seconds Time between finalization starts.", + "# TYPE qbit_prism_block_candidate_interarrival_seconds summary", + f"qbit_prism_block_candidate_interarrival_seconds_sum {interval_sum:.6f}", + f"qbit_prism_block_candidate_interarrival_seconds_count {interval_count}", + "qbit_prism_block_candidate_interarrival_seconds_min " + + ("0.000000" if interval_min is None else f"{float(interval_min):.6f}"), + ] + ) + return lines + + def _land_and_confirm_block_candidate( + self, + candidate: PrismBlockCandidate, + *, + current_tip: str, + already_active: bool, + worker: str | None, + node_submission: _BlockCandidateNodeSubmission, + revalidated_append_epoch: int | None = None, + ) -> tuple[ + Mapping[str, Any], + dict[str, Any], + dict[str, Any], + dict[str, Any], + AuditPublicationIdentity, + dict[str, Any], + ] | None: + """Land, verify, publish, persist, and confirm one candidate. + + The balance serializer spans the last prior-state check through durable + confirmation. Reconciliation therefore cannot change the base beneath + the accepted coinbase, while ordinary job delivery remains unblocked. + The caller has already run submitblock on the lock/DB-free fast lane. + The audit bundle build and verification execute with the serializer + temporarily released (the landed fence stays armed), so neither block + announcement nor job delivery waits on audit construction. + """ + context = candidate.context + submission = candidate.submission + expected_height = int(context.template["height"]) + block_hash = str(submission.block_hash_hex).lower() + parent_hash = str(context.template["previousblockhash"]) + self.runtime._ensure_job_cache_state() + durable_payout_state = bool( + getattr(self.runtime.ledger, "durable_payout_state", False) + ) + with ExitStack() as balance_serializer: + # The wait for the balance serializer is a landing stretch of its + # own (issue #224). The stack holds the lock for exactly the block + # the plain ``with`` held it for, so only the acquisition is timed + # and the release point is unchanged. + with self._landing_phase( + LANDING_PHASE_BALANCE_LOCK_WAIT, + block_hash=block_hash, + block_height=expected_height, + ): + balance_serializer.enter_context( + self.runtime._block_submitter_lock( + self.runtime._payout_balance_mutation_lock, + "payout-balance-mutation", + ) + ) + if self.runtime._defer_for_pending_parent_payout_transition( + block_hash=block_hash, + parent_hash=parent_hash, + parent_height=expected_height - 1, + worker=worker, + active_candidate_hash=block_hash if already_active else None, + active_candidate_height=expected_height if already_active else None, + ): + return None + block_state: dict[str, object] | None = None + block_state_reader = getattr(self.runtime.ledger, "pool_block_state", None) + transition_already_landed = self.runtime._accepted_block_payout_transition_landed( + block_hash + ) + reorg_reconciled: bool | None = None + if already_active and not transition_already_landed: + # A replayed active ancestor may coexist with balances from an + # orphaned pool block. Reconcile that global state before this + # transition becomes a landed barrier and before validating its + # payout base. + try: + # Reconciliation is the longest heartbeat-silent stretch + # in the landing scope: a chain walk of ledger statements + # and getblockhash calls, one pair per watched block. The + # pass stamps its own per-row progress internally; these + # boundary stamps mark the stretch itself so a landing + # that enters it is never mistaken for a wedged thread. + self.runtime._record_block_submitter_phase("reorg-reconcile") + with self._landing_phase( + LANDING_PHASE_RECONCILE, + block_hash=block_hash, + block_height=expected_height, + ): + reorg_reconciled = ( + self.runtime.ensure_reorg_reconciled_for_tip( + current_tip, + _coalesce_same_tip=False, + ) + ) + self.runtime._record_block_submitter_phase( + "reorg-reconcile:complete" + ) + except Exception: + traceback.print_exc() + self.runtime._abandon_block_candidate( + PRISM_REJECTION_BACKEND_RPC_UNAVAILABLE, + "reorg reconciliation failed before block replay", + block_hash=block_hash, + worker=worker, + ) + return None + if not reorg_reconciled: + self.runtime._abandon_block_candidate( + PRISM_REJECTION_BACKEND_RPC_UNAVAILABLE, + "reorg reconciliation reported an untrusted chain view", + block_hash=block_hash, + worker=worker, + ) + return None + if already_active and callable(block_state_reader): + self.runtime._record_block_submitter_phase("pool-block-state") + with self._landing_phase( + LANDING_PHASE_CHAIN_PROBE, + block_hash=block_hash, + block_height=expected_height, + ): + block_state = block_state_reader(block_hash=block_hash) + self.runtime._record_block_submitter_phase("pool-block-state:complete") + already_confirmed = bool( + block_state is not None + and str(block_state.get("chain_state", "")) == "confirmed" + and str(block_state.get("maturity_state", "")) != "reversed" + ) + if already_confirmed: + # The outbox terminal update can fail after a fully durable + # confirmation. Do not replace later global balances with an + # ancestor-only preview during exact-idempotent replay. + self.runtime._clear_accepted_block_payout_preview(block_hash) + reorg_reconciled = True + elif already_active: + self.runtime._begin_accepted_block_payout_preview( + block_hash, + block_height=expected_height, + ) + self.runtime._mark_accepted_block_payout_landed( + block_hash, + block_height=expected_height, + ) + reorg_reconciled = True + elif transition_already_landed: + # A prior attempt reached submitblock while holding this + # serializer. External reconciliation is barred until it + # confirms or is withdrawn, so retry its durable steps directly. + reorg_reconciled = True + else: + try: + # Same unbounded chain walk as the replay path above. + self.runtime._record_block_submitter_phase("reorg-reconcile") + with self._landing_phase( + LANDING_PHASE_RECONCILE, + block_hash=block_hash, + block_height=expected_height, + ): + reorg_reconciled = ( + self.runtime.ensure_reorg_reconciled_for_tip( + current_tip, + _coalesce_same_tip=False, + ) + ) + self.runtime._record_block_submitter_phase( + "reorg-reconcile:complete" + ) + except Exception: + traceback.print_exc() + self.runtime._abandon_block_candidate( + PRISM_REJECTION_BACKEND_RPC_UNAVAILABLE, + "reorg reconciliation failed before block submit", + block_hash=block_hash, + worker=worker, + ) + return None + if not reorg_reconciled: + self.runtime._abandon_block_candidate( + PRISM_REJECTION_BACKEND_RPC_UNAVAILABLE, + "reorg reconciliation reported an untrusted chain view", + block_hash=block_hash, + worker=worker, + ) + return None + if ( + already_active + and not already_confirmed + and self.runtime._defer_for_pending_parent_payout_transition( + block_hash=block_hash, + parent_hash=parent_hash, + parent_height=expected_height - 1, + worker=worker, + ) + ): + return None + # A late-visible append advances only the append-invalidation + # epoch: the tip, payout generation, and balance digest all + # survive, yet this candidate's coinbase pays from a window that + # omitted the late row. The refresh wave retires the job + # asynchronously; until it does, membership admission still lets + # the pre-append job submit, so landing fails closed here. + # Collection candidates settle solver-pays-all and are exempt, as + # at every other epoch fence. A negative stamp is a candidate + # reconstructed from durable intent: the caller revalidated its + # recorded window against the durable ledger before this landing + # and rebased it onto the live epoch sequence at that read, so + # from here both kinds of candidate share one epoch fence. A + # reconstructed candidate on a backend that cannot revalidate + # carries no epoch and the fence stands down. + # + # The question is asked per anchor, not per counter. The epoch is + # global, so a bare inequality also fires for an append that only + # invalidated some newer live window -- and on this lane, where + # nothing has been offered, that abandoned a block before qbitd + # ever saw it (issue #126). The predicate forgives a moved epoch + # only when every bump between this candidate's baseline and the + # live value was caused by a row that provably does not predate + # this candidate's own declared anchor, and fails closed + # otherwise -- including when the anchor is unknown, which is + # what the bare comparison did for every candidate. + context_append_epoch = int( + getattr(context, "payout_append_invalidation_epoch", 0) + ) + effective_append_epoch = ( + context_append_epoch + if context_append_epoch >= 0 + else revalidated_append_epoch + ) + declared_anchor_ms = _declared_anchor_ms(context) + with self.runtime._job_cache_lock: + live_append_epoch = int( + self.runtime._payout_ledger_append_invalidation_epoch + ) + if ( + not already_active + and not getattr(context, "collection_only", False) + and not node_submission.attempted + and effective_append_epoch is not None + and self.runtime._append_epoch_invalidated_declared_anchor( + baseline_epoch=effective_append_epoch, + live_epoch=live_append_epoch, + declared_anchor_ms=declared_anchor_ms, + ) + ): + # Fail closed only while nothing has been offered: a + # fast-lane candidate already reached qbitd, so a moved + # epoch can no longer withhold its coinbase — the offer's + # own classification below decides (an error retries + # duplicate-safe, a rejection stays terminal, and an + # accepted or duplicate result proceeds to the post-offer + # fence, which keeps the as-issued snapshot for + # accounting). + self.runtime._abandon_block_candidate( + PRISM_REJECTION_STALE_JOB, + "payout window was invalidated by a late-visible share append", + block_hash=block_hash, + worker=worker, + expected_height=expected_height, + stale_job_class="append_epoch_stale", + ) + return None + if ( + durable_payout_state + and not already_active + and not self._stamped_prior_balances_match_current( + context.prior_balances, + block_hash=block_hash, + block_height=expected_height, + ) + ): + self.runtime._abandon_block_candidate( + PRISM_REJECTION_STALE_JOB, + "prior balances changed since the job was issued", + block_hash=block_hash, + worker=worker, + expected_height=expected_height, + stale_job_class="balance_stale", + ) + return None + if not already_active and not node_submission.attempted: + # One RPC round trip, but it runs between two already-stamped + # stretches on the block-work thread; leaving it unnamed puts + # an unbounded node response inside somebody else's phase. + self.runtime._record_block_submitter_phase("tip-height-rpc") + with self._landing_phase( + LANDING_PHASE_CHAIN_PROBE, + block_hash=block_hash, + block_height=expected_height, + ): + before_height = int(self.runtime.rpc.call("getblockcount")) + self.runtime._record_block_submitter_phase("tip-height-rpc:complete") + if before_height + 1 != expected_height: + self.runtime._abandon_block_candidate( + PRISM_REJECTION_BLOCK_STALE, + f"stale block height: template={expected_height} tip={before_height}", + block_hash=block_hash, + worker=worker, + expected_height=expected_height, + ) + return None + if not already_active: + # Register before a fallback submitblock can expose this hash as the new + # tip. Child builders will wait for the verified preview rather + # than reading balances that omit their new parent. + self.runtime._begin_accepted_block_payout_preview( + block_hash, + block_height=expected_height, + ) + # Treat the submit outcome as uncertain before entering RPC. + # If transport fails after qbitd accepted the block, this + # conservative barrier preserves the coinbase's payout base. + self.runtime._mark_accepted_block_payout_landed( + block_hash, + block_height=expected_height, + ) + fallback_submit_under_fence = not node_submission.attempted + if not node_submission.attempted: + + def _verify_lease_before_submitblock() -> None: + # Runs at the RPC boundary — after any wait on the + # landing fence lock below — so the lease runway the + # verification proves is still intact when the RPC + # starts. A predating append_batch can hold that + # lock across its ledger commit and epoch bump far + # longer than any scheduling headroom, and a + # verification taken before the wait would measure + # runway the wait then consumes. + try: + self.runtime._require_fresh_ledger_lease_for_external_side_effect( + "submitblock" + ) + except WriterLeaseRenewalDeferred: + if not transition_already_landed: + # The refusal fired before submitblock, so + # this attempt's outcome is not uncertain: + # qbitd provably never saw the block (no + # fast-lane node offer either — attempted is + # False). Unwind the landed bar this attempt + # armed — leaving it would bar reconciliation + # and payout-state publication for as long as + # the writer's own fenced write keeps the + # renewal deferred, though nothing needs + # preserving. The begun preview stays and the + # retry re-arms both. A bar landed by an + # earlier attempt that did reach submitblock + # keeps standing for that attempt's uncertain + # outcome. + self.runtime._unmark_accepted_block_payout_landed( + block_hash + ) + raise + + # The epoch fence above is advisory: it releases the lock + # after one read, so an append-side bump could still commit + # between that read and the RPC below. This one is + # authoritative -- the bump acquires the same fence lock, so + # holding it across submitblock means no late-visible append + # can advance the epoch between this comparison and the + # block entering qbitd. The lock spans the lease + # verification and one RPC, and only on this boundary; + # ordinary share commits never touch it (the append side + # takes it only for rows that predate a live anchor, and + # this landing's own declared anchor stays exposed for + # the landing's duration). + append_epoch_raced = False + if ( + effective_append_epoch is None + or getattr(context, "collection_only", False) + ): + _verify_lease_before_submitblock() + node_submission = self.runtime._submit_block_candidate_to_node( + candidate + ) + else: + with self.runtime._payout_append_landing_fence_lock: + with self.runtime._job_cache_lock: + live_append_epoch = int( + self.runtime._payout_ledger_append_invalidation_epoch + ) + # Anchor-scoped, exactly as the advisory read + # above: a bump that commits inside this window + # is terminal only if the row that caused it + # predates this candidate's declared anchor. + # Two guarantees make reading recorded bumps + # sufficient. The fence lock makes the answer + # authoritative: the bump takes the same lock, + # so none can commit between this comparison + # and the RPC below. And every append that + # predates this candidate's window has a bump + # to read: the landing's own anchor is exposed + # for the landing's duration, and the anchor- + # set maximum never decreases while any job is + # landable (an armed artifact's anchor is + # folded into the never-retired published- + # window watermark before the slot can drop + # it), so a predating row always finds an + # anchor to predate and stamps the epoch it + # advances -- never a silent commit this read + # would mistake for harmless history. + if self.runtime._append_epoch_invalidated_declared_anchor( + baseline_epoch=effective_append_epoch, + live_epoch=live_append_epoch, + declared_anchor_ms=declared_anchor_ms, + ): + append_epoch_raced = True + else: + _verify_lease_before_submitblock() + node_submission = ( + self.runtime._submit_block_candidate_to_node(candidate) + ) + if append_epoch_raced: + self.runtime._abandon_block_candidate( + PRISM_REJECTION_STALE_JOB, + "payout window was invalidated by a late-visible share append", + block_hash=block_hash, + worker=worker, + expected_height=expected_height, + stale_job_class="append_epoch_stale", + ) + return None + if node_submission.error is not None: + raise node_submission.error + result = node_submission.result + if result not in (None, "duplicate"): + self.runtime._abandon_block_candidate( + PRISM_REJECTION_SUBMITBLOCK_REJECTED, + f"submitblock rejected candidate: {result}", + block_hash=block_hash, + worker=worker, + expected_height=expected_height, + ) + return None + if ( + not fallback_submit_under_fence + and effective_append_epoch is not None + and not getattr(context, "collection_only", False) + ): + # The fast lane offered this block to qbitd without any + # ledger synchronization, so the landing fence can no + # longer gate submitblock itself. It still orders the + # accounting decision: wait out any fenced predating + # append whose durable commit is in flight (the commit + # holds this lock across its epoch bump). A moved epoch + # is no longer terminal here, though: every candidate + # reaching this point was accepted or already known by + # the node (rejections returned above), so its coinbase + # paid the as-issued window the moment it landed and no + # rebuild can change it. Abandoning would permanently + # discard payout accounting for a live block — the + # ledger would then re-pay the same carried balances in + # a later block — so the as-issued snapshot is kept for + # accounting and the predating share simply rides the + # next window (its ledger credit is untouched). The + # active-height check below still owns the + # did-it-actually-land verdict, and the pre-offer fence + # above still fails closed while nothing has been + # submitted. + with self.runtime._payout_append_landing_fence_lock: + with self.runtime._job_cache_lock: + live_append_epoch = int( + self.runtime._payout_ledger_append_invalidation_epoch + ) + if live_append_epoch != effective_append_epoch: + print( + "prism coordinator: payout window was " + "invalidated by a late-visible share append " + f"after the node offer hash={block_hash}; " + "keeping the as-issued payout snapshot for " + "accounting", + flush=True, + ) + active_hash = str( + self.runtime.rpc.call("getblockhash", [expected_height]) + ).lower() + if active_hash != block_hash: + self.runtime._abandon_block_candidate( + PRISM_REJECTION_SUBMITBLOCK_REJECTED, + f"submitted block is not active at height {expected_height}", + block_hash=block_hash, + worker=worker, + expected_height=expected_height, + ) + return None + self.runtime._cancel_obsolete_job_builds("direct PRISM block accepted") + self.runtime._mark_tip_refresh_pending(block_hash) + self.runtime._schedule_tip_refresh_retry() + + preview: list[dict[str, object]] | None = None + issued_preview = getattr(context, "prospective_prior_balances", None) + if not already_confirmed and issued_preview is not None: + # The compact preview came from the immutable issued job + # summary. Publish it before rebuilding/canonicalizing the full + # audit bundle, without retaining that bundle's shares tree. + with self._landing_phase( + LANDING_PHASE_PREVIEW_PREPARE, + block_hash=block_hash, + block_height=expected_height, + ): + preview = self.runtime._materialize_prior_balance_preview( + issued_preview + ) + if durable_payout_state and not ( + self._stamped_prior_balances_match_current( + context.prior_balances, + block_hash=block_hash, + block_height=expected_height, + ) + ): + self.runtime.request_shutdown() + self.runtime._clear_accepted_block_payout_preview( + block_hash, + invalidate_published=True, + ) + self.runtime._abandon_block_candidate( + PRISM_REJECTION_LEDGER_CONFIRMATION_FAILED, + "accepted block payout base changed before preview publication", + block_hash=block_hash, + worker=worker, + ) + return None + with self._landing_publish_span( + block_hash=block_hash, + block_height=expected_height, + ): + self.runtime._publish_accepted_block_payout_preview( + block_hash, + preview, + ) + + self.runtime._record_block_submitter_phase("audit-build") + # The bundle derives only from inputs frozen on the candidate + # (share window, prior balances, extranonces, template fields), + # so the serializer is released around the builder/verifier + # subprocess work: submitblock above has already run for fresh + # candidates -- announcement is never delayed by audit + # construction -- and the landed fence keeps reconciliation out + # while job delivery proceeds. Child builds consume the compact + # preview published above until the verified preview lands. + with self.runtime._payout_balance_serializer_released(): + audit_store = self.runtime._ensure_audit_artifact_store() + candidate_artifact = audit_store.issue_candidate( + block_hash=submission.block_hash_hex + ) + candidate_bundle_path = candidate_artifact.path + compiler_transferred_candidate = False + + def adopt_compiler_output(path: Path, value: os.stat_result) -> None: + nonlocal compiler_transferred_candidate + audit_store.adopt_compiler_candidate( + candidate_artifact, + path=path, + value=value, + ) + compiler_transferred_candidate = True + + compiler_parent_fd = audit_store.duplicate_root_directory_fd() + try: + final_bundle = self.runtime.build_audit_bundle( + shares=context.shares_json, + found_block=context.found_block, + prior_balances=context.prior_balances, + coinbase_script_sig_suffix_hex=self.runtime.coinbase_script_sig_suffix_hex( + candidate.extranonce1_hex, + candidate.extranonce2_hex, + ), + witness_merkle_leaves_hex=list( + getattr(context.job, "witness_merkle_leaves_hex", ()) + ) + or direct_stratum.witness_merkle_leaves_hex( + getattr(context.job, "transaction_hexes", ()) + ), + ctv_fee_parent_hash=parent_hash, + canonical_output_path=candidate_bundle_path, + canonical_output_parent_fd=compiler_parent_fd, + canonical_output_adopter=adopt_compiler_output, + ) + except BaseException: + audit_store.discard_candidate(candidate_artifact) + raise + finally: + os.close(compiler_parent_fd) + # A real build returns the bundle compiler's bounded view of + # the canonical artifact it just wrote: header members are + # parsed, the window-sized share arrays remain on disk, and + # every consumer below (coinbase check, trusted key, payout + # preview, persistence) reads it as a mapping (#255). + # Compatibility builders used by tests and older integrations + # may ignore canonical_output_path. Persist their logical + # bundle via the normal canonicalization fallback without + # mislabeling bytes. + try: + if not candidate_bundle_path.exists(): + candidate_bundle_path = audit_store.write_compatibility_candidate( + candidate_artifact, + final_bundle, + ) + else: + if not compiler_transferred_candidate: + raise RuntimeError( + "audit builder created an output path without exact inode transfer" + ) + final_manifest = final_bundle["signed_coinbase_manifest"]["manifest"] + final_coinbase_tx_hex_raw = final_manifest["coinbase_tx_hex"] + if not isinstance(final_coinbase_tx_hex_raw, str): + raise ValueError( + "final audit bundle coinbase_tx_hex is not a string" + ) + final_coinbase_tx_hex = final_coinbase_tx_hex_raw.lower() + except BaseException: + audit_store.discard_candidate(candidate_artifact) + raise + if final_coinbase_tx_hex != submission.coinbase_tx_hex.lower(): + audit_store.discard_candidate(candidate_artifact) + self.runtime.request_shutdown() + self.runtime._clear_accepted_block_payout_preview( + block_hash, + invalidate_published=True, + ) + self.runtime._abandon_block_candidate( + PRISM_REJECTION_CANDIDATE_AUDIT_MISMATCH, + "final audit bundle coinbase does not match submitted coinbase", + block_hash=block_hash, + worker=worker, + ) + return None + payout_commit_started: float | None = None + payout_commit_source: int | None = None + try: + with self.runtime._payout_balance_serializer_released(): + self.runtime._record_block_submitter_phase("audit-verify") + # Intentional escape hatch outside the typed port: only a + # per-instance override placed in the runtime's __dict__ + # replaces the verifier, never a class-level attribute. + verifier_override = self.runtime.__dict__.get("verify_bundle") + configured_writer_key = getattr( + self.runtime, + "ledger_writer_public_key_hex", + None, + ) + verified_audit = audit_store.verify_candidate( + candidate_artifact, + coinbase_tx_hex=submission.coinbase_tx_hex, + expected_coinbase_value_sats=int( + context.template["coinbasevalue"] + ), + expected_block_height=expected_height, + trusted_writer_public_key_hex=( + self.runtime.trusted_ledger_writer_public_key_hex(final_bundle) + ), + trust_source=( + "configured" + if configured_writer_key is not None + else "embedded_test_only" + ), + verifier=( + verifier_override + if callable(verifier_override) + else None + ), + ) + audit_store.require_current_verified_candidate( + verified_audit, + candidate_artifact, + ) + report = dict(verified_audit.report) + persistence_canonical_bundle_path = ( + candidate_bundle_path + if verified_audit.canonical_copy_eligible + else None + ) + with self._landing_phase( + LANDING_PHASE_PREVIEW_PREPARE, + block_hash=block_hash, + block_height=expected_height, + ): + # The preview walks the bundle's accounts row by + # row; on a bounded view they stream from disk. + verified_preview = ( + self.runtime._accepted_block_payout_preview_from_bundle( + final_bundle, + prior_balances=context.prior_balances, + ) + ) + self.runtime._record_block_submitter_phase("audit-verify:complete") + if not already_confirmed: + if preview is None and durable_payout_state: + # A prior-balances reread, never a payout-window + # rescan: the ledger read below re-observes the base + # this candidate's coinbase pays from. + with self._landing_phase( + LANDING_PHASE_PRIOR_BALANCES_CHECK, + block_hash=block_hash, + block_height=expected_height, + ): + live_prior_balances = ( + self.runtime.settlement_balances_by_program( + self.runtime.ledger.current_prior_balances() + ) + ) + expected_prior_balances = ( + self.runtime.settlement_balances_by_program( + context.prior_balances + ) + ) + if live_prior_balances != expected_prior_balances: + self.runtime.request_shutdown() + self.runtime._clear_accepted_block_payout_preview( + block_hash, + invalidate_published=True, + ) + self.runtime._abandon_block_candidate( + PRISM_REJECTION_LEDGER_CONFIRMATION_FAILED, + "accepted block payout base changed before preview publication", + block_hash=block_hash, + worker=worker, + ) + return None + try: + with self._landing_publish_span( + block_hash=block_hash, + block_height=expected_height, + ): + self.runtime._publish_accepted_block_payout_preview( + block_hash, + verified_preview, + ) + except RuntimeError as exc: + self.runtime.request_shutdown() + self.runtime._clear_accepted_block_payout_preview( + block_hash, + invalidate_published=True, + ) + self.runtime._abandon_block_candidate( + PRISM_REJECTION_CANDIDATE_AUDIT_MISMATCH, + "verified final payout preview does not match the " + f"issued block job: {exc}", + block_hash=block_hash, + worker=worker, + ) + return None + preview = verified_preview + + # The verified preview is now the effective balance snapshot, + # so persistence can do canonicalization, body writes, copies, + # and bulk SQL without owning the delivery gate. + payout_commit_started = time.monotonic() + payout_commit_source = self.runtime._capture_payout_state_source()[1] + self.runtime._record_block_submitter_phase("persist-accepted-block") + persistence = self.runtime.ledger.persist_accepted_block( + block_hash=submission.block_hash_hex, + block_height=expected_height, + parent_hash=parent_hash, + final_bundle=final_bundle, + audit_report=report, + canonical_bundle_path=persistence_canonical_bundle_path, + ) + self.runtime._record_block_submitter_phase("persist-accepted-block:complete") + active_hash = str( + self.runtime.rpc.call("getblockhash", [expected_height]) + ).lower() + if active_hash != block_hash: + if already_confirmed: + self.runtime._abandon_block_candidate( + PRISM_REJECTION_BACKEND_RPC_UNAVAILABLE, + "accepted ancestor left the active chain during replay", + block_hash=block_hash, + worker=worker, + ) + return None + # Seal the disposition BEFORE touching the prepared + # payout rows: a rejected row can never be promoted by a + # later confirmation (and reactivation only covers + # inactive rows), so rejection must follow -- never + # precede -- a terminal decision. The abandon defers on + # live acceptance evidence; its terminal commit consults + # that evidence atomically with the commit AND seals the + # hash against further observation matching, so no + # evidence can register anywhere in the gap between the + # sealed decision and this rejection. A crash in between + # leaves the outbox row pending; restart replay + # re-registers the hash and re-evaluates from live chain + # state before rejecting. + self.runtime._abandon_block_candidate( + PRISM_REJECTION_BLOCK_STALE, + "accepted block left the active chain before ledger confirmation", + block_hash=block_hash, + worker=worker, + expected_height=expected_height, + ) + outcome = getattr( + self.runtime, "_block_candidate_outcome", None + ) + sealed_reason = ( + getattr(outcome, "reason", None) + if outcome is not None + else None + ) + if sealed_reason == PRISM_REJECTION_BLOCK_STALE: + active_tip_height = int(self.runtime.rpc.call("getblockcount")) + self.runtime.reject_prepared_block( + block_hash=block_hash, + active_tip_height=active_tip_height, + ) + return None + self.runtime._record_block_submitter_phase("confirm-accepted-block") + with audit_store.publication_order_guard(): + confirmation = self.runtime.ledger.confirm_accepted_block( + block_hash=block_hash, + # The ledger confirmation function matches this value + # against the candidate row's own height. An accepted + # ancestor can be finalized after newer blocks arrive. + active_tip_height=expected_height, + ) + confirmed_count = int(confirmation.get("confirmed_count", 0)) + # 1 is the fresh flip and 2 the idempotent replay of a row + # this writer already confirmed. Both leave a confirmed + # row addressed by the ordinal the flip allocated -- the + # replay allocates nothing -- so both carry a publication + # identity and both complete the candidate. + if confirmed_count in {1, 2}: + audit_publication_identity = ( + self.runtime._audit_publication_identity( + block_hash=block_hash, + block_height=expected_height, + confirmation=confirmation, + ) + ) + self.runtime._record_block_submitter_phase( + "confirm-accepted-block:complete" + ) + # 2 is the ledger's idempotent disposition: this writer had + # already confirmed the row, so the call changed nothing. It + # completes the candidate exactly like a fresh flip, and its + # payout publication falls through the covered-replay checks + # below -- a covered confirmation forces no publication of its + # own, because the state it would republish is the state its + # original flip already published (issue #61). Those checks are + # not weakened by it: a source left unpublished or a leaked + # delivery fence still publishes here, exactly as before. + if confirmed_count not in {1, 2}: + # -1 is the ledger's superseded disposition: the row was + # terminally disposed (reorg quarantine, rejection, or + # reversal) before this confirmation landed. That is + # terminal for the candidate but benign for the pool, so + # the coordinator keeps serving; the shutdown escalation + # is reserved for a genuinely unexplained failure. + superseded = confirmed_count == -1 + if not superseded: + self.runtime.request_shutdown() + self.runtime._clear_accepted_block_payout_preview( + block_hash, + invalidate_published=True, + ) + if superseded: + self.runtime._abandon_block_candidate( + PRISM_REJECTION_LEDGER_CONFIRMATION_SUPERSEDED, + "ledger row for accepted block " + f"{block_hash} was superseded before confirmation", + block_hash=block_hash, + worker=worker, + ) + else: + self.runtime._abandon_block_candidate( + PRISM_REJECTION_LEDGER_CONFIRMATION_FAILED, + f"ledger did not confirm accepted block {block_hash}", + block_hash=block_hash, + worker=worker, + ) + return None + + if durable_payout_state: + # A fresh confirmation just made this block the ledger's + # newest confirmed pool block: the landing holds the + # balance serializer and every younger landing defers on + # this block's unresolved payout transition (re-armed by + # startup replay enumeration after a restart), so the + # transactionally maintained current-balance summary IS + # the active-chain view as of this block. Read that + # O(recipients) summary instead of the O(history) carry + # scan, whose growth past the parent-preview wait stalled + # back-to-back landings (issue #209). Only an exact replay + # of an already-confirmed ancestor can run after a later + # pool block changed global balances; that replay still + # compares against its height-bounded active-chain view, + # which also preserves the invariant across restart after + # a prior post-confirm mismatch instead of silently + # accepting the already-confirmed row on the next attempt. + as_of_reader = ( + getattr( + self.runtime.ledger, + "prior_balances_after_pool_block", + None, + ) + if already_confirmed + else None + ) + confirmed_balances = self.runtime.normalized_prior_balances( + as_of_reader(block_hash=block_hash) + if callable(as_of_reader) + else self.runtime.ledger.current_prior_balances() + ) + if self.runtime.settlement_balances_by_program( + confirmed_balances + ) != self.runtime.settlement_balances_by_program(preview): + self.runtime.request_shutdown() + self.runtime._clear_accepted_block_payout_preview( + block_hash, + invalidate_published=True, + ) + self.runtime._abandon_block_candidate( + PRISM_REJECTION_LEDGER_CONFIRMATION_FAILED, + "confirmed payout balances do not match the published " + f"preview for accepted block {block_hash}", + block_hash=block_hash, + worker=worker, + ) + return None + # Durability caught up to the already-published logical state; + # clearing the parent override needs no second generation bump. + self.runtime._clear_accepted_block_payout_preview(block_hash) + self.runtime._schedule_current_payout_ledger_artifact_if_missing() + payout_publication_required = ( + self.runtime._payout_source_requires_publication() + ) + payout_publication_fenced = ( + self.runtime._payout_state_publication_fenced() + ) + # These two are the whole publication decision, for a fresh + # confirmation and a covered replay alike: nothing pending and + # nothing fenced means the published payout state already + # covers this candidate, so republishing it would bump the + # payout generation, wipe the job-bundle cache, and abort every + # in-flight refresh for identical state. Never make the + # disposition alone force a publication past them. + if payout_publication_required or payout_publication_fenced: + # A covered replay normally has no publication work. The + # exception is a leaked delivery fence whose source already + # published: force one republish so the replay heals it. + covered_replay_fence = ( + payout_publication_fenced + and not payout_publication_required + ) + with self.runtime.lock: + pending_cause = self.runtime._payout_state_source[2] + # A bounded preview-publication loss already left the gate + # fenced and its retry scheduled. Do not monopolize the + # submitter with a second retry budget. Uncertain commits, + # ordinary unfenced tip sources, and a covered replay's + # leaked fence still reconcile now. + publish_now = ( + covered_replay_fence + or pending_cause == "direct_block_uncertain" + or not payout_publication_fenced + ) + published: int | None = None + if publish_now and getattr( + self.runtime, + "reorg_reconciler_enabled", + True, + ): + with self.runtime.lock: + latest_tip = self.runtime._payout_state_source[1] + summary = self.runtime.reconcile_prism_pool_blocks_once( + tip_hash=latest_tip, + _force_publish=True, + _source_reserved=True, + ) + reconciled_generation = summary.get("published_generation") + if isinstance(reconciled_generation, int): + published = reconciled_generation + elif publish_now: + published = ( + self.runtime._publish_current_payout_state_with_retry_budget() + ) + if publish_now and published is None: + # The block is durably confirmed; only the payout + # publication lost its race. Aborting would keep the + # outbox row pending and replay persist/confirm churn + # for an already-final block. Keep delivery fenced and + # let the scheduled tip refresh publish the newest + # source; this candidate's durable work is complete. + self.runtime._block_payout_state_publication() + print( + "prism coordinator: accepted block confirmed " + "durably; payout publication deferred to the " + f"scheduled refresh hash={block_hash}", + flush=True, + ) + return ( + final_bundle, + report, + persistence, + confirmation, + audit_publication_identity, + dict(verified_audit.verification_identity), + ) + except Exception: + if payout_commit_started is not None and payout_commit_source is not None: + # Persistence/confirmation can report failure after a + # durable partial commit. Supersede every prepared source + # and keep all delivery fenced until replay/reconciliation + # proves the resulting ledger state. + self.runtime._block_payout_state_publication( + supersede_with=( + payout_commit_source, + block_hash, + "direct_block_uncertain", + payout_commit_started, + ) + ) + raise + finally: + if payout_commit_started is not None: + self.runtime._observe_payout_state_seconds( + "preparation", + max(0.0, time.monotonic() - payout_commit_started), + ) + audit_store.discard_candidate(candidate_artifact) + + def _admit_candidate( + self, + candidate: PrismBlockCandidate, + *, + node_submission: _BlockCandidateNodeSubmission, + ) -> FinalizationAdmission | bool | None: + """Classify one post-offer candidate against pool and chain state. + + Returns an admission for the ordered phases, a terminal bool for an + outcome the disposition can finalize directly (an already-recorded + acceptance or an accepted race win), or ``None`` for a terminal + abandonment already recorded through the outcome seam. + """ + outcome = getattr(self.runtime, "_block_candidate_outcome", None) + if outcome is None: + outcome = threading.local() + self.runtime._block_candidate_outcome = outcome + outcome.reason = None + outcome.error = None + outcome.stale_job_class = None + context = candidate.context + submission = candidate.submission + worker = candidate.client.username or None + expected_height = int(context.template["height"]) + block_hash = str(submission.block_hash_hex).lower() + parent_hash = str(context.template["previousblockhash"]) + self.runtime._ensure_job_cache_state() + # Every disposition (queue drain, synchronous below-target submit, + # outbox replay, retained retry) marks its hash outstanding so tip + # observations arriving on other threads can register acceptance. + self.runtime._register_outstanding_block_candidate(block_hash) + self._record_block_candidate_progress("disposition-start") + if ( + self.runtime._block_candidate_acceptance_recorded(block_hash) + and node_submission.error is not None + ): + # A concurrent same-hash pass completed the success tail while + # this duplicate-safe node offer waited for disposition. Do not + # recreate its payout transition or accounting work. + self.runtime._clear_accepted_block_payout_preview(block_hash) + return True + with self.runtime.lock: + accepted_count = int(self.runtime.accepted_block_count) + pool_closed = ( + block_hash not in self.runtime._accounted_accepted_block_hashes + and ( + accepted_count >= int(self.runtime.max_blocks) + or (bool(self.runtime.stop_after_block) and accepted_count >= 1) + ) + ) + if pool_closed: + with self._landing_phase( + LANDING_PHASE_CHAIN_PROBE, + block_hash=block_hash, + block_height=expected_height, + ): + closed_pool_chain_probe = self.runtime._block_candidate_chain_probe( + block_hash, + expected_height=expected_height, + ) + if closed_pool_chain_probe is True: + # The chain provably contains this block; its payout accounting + # must complete regardless of when the pool stopped accepting + # new work. Fall through to the normal disposition below, which + # resumes the accepted success tail. Observation evidence alone + # deliberately does not open this gate -- an unprovable view + # defers via the abandon path instead, so a closed pool can + # never fall through to submitblock on stale evidence. + pool_closed = False + if pool_closed: + self.runtime._abandon_block_candidate( + PRISM_REJECTION_POOL_CLOSED, + "pool is no longer accepting blocks", + block_hash=block_hash, + worker=worker, + expected_height=expected_height, + ) + return None + self._record_block_candidate_progress("current-tip-rpc") + observed_tip = str(self.runtime.rpc.call("getbestblockhash")) + self._record_block_candidate_progress("current-tip-rpc:complete") + # A successful or transport-ambiguous fast-lane call can change the + # tip before this post-submit probe. It is still a *fresh* attempt, + # not an active replay: run the normal validation/persistence tail + # against the candidate's stamped parent. A later getblockhash check + # proves a successful acknowledgement, while an ambiguous transport + # outcome stays pending for duplicate-safe replay. Duplicate replies + # are replay evidence and retain the live-tip classification; the + # abandonment tie-breaker separately consults this process's own + # recorded offer evidence so an unprovable chain view cannot + # terminally discard a block the node already told us it has. + fresh_or_uncertain_submit = bool( + node_submission.attempted + and ( + node_submission.error is not None + or node_submission.result is None + ) + ) + current_tip = parent_hash if fresh_or_uncertain_submit else observed_tip + landed_height: int | None = None + if current_tip.lower() == block_hash: + landed_height = expected_height + elif current_tip != parent_hash: + try: + landed_height = self.runtime.active_block_candidate_height(block_hash) + except Exception: + traceback.print_exc() + self.runtime._abandon_block_candidate( + PRISM_REJECTION_BACKEND_RPC_UNAVAILABLE, + "could not determine whether a prior candidate is active", + block_hash=block_hash, + worker=worker, + ) + return None + already_active = landed_height == expected_height + if landed_height is not None and not already_active: + self.runtime._abandon_block_candidate( + PRISM_REJECTION_BLOCK_STALE, + f"candidate active at unexpected height {landed_height}", + block_hash=block_hash, + worker=worker, + expected_height=expected_height, + ) + return None + if already_active: + # A disposition probe is a tip observation too: remember it so a + # later attempt cannot terminally abandon this hash on a racing + # chain snapshot after this attempt fails mid-tail. + self.runtime._note_tip_observation_for_candidates(block_hash) + print( + "prism coordinator: resuming finalization for active block candidate " + f"height={landed_height} hash={submission.block_hash_hex}", + flush=True, + ) + elif parent_hash != current_tip: + if self.runtime._block_candidate_acceptance_recorded(block_hash): + # A duplicate wakeup can reach this check after the accepted + # success tail but after a newer tip hides the candidate from + # the active-header probe. Its durable work is already done; + # let the caller finalize the outbox as submitted. + self.runtime._clear_accepted_block_payout_preview(block_hash) + return True + accepted_race_won = self.runtime._abandon_block_candidate( + PRISM_REJECTION_STALE_JOB, + f"tip moved before submit: {current_tip}", + block_hash=block_hash, + worker=worker, + preserve_if_accepted=True, + expected_height=expected_height, + stale_job_class="tip_moved", + ) + return bool(accepted_race_won) + if ( + node_submission.attempted + and node_submission.error is None + and node_submission.result not in (None, "duplicate") + and not already_active + ): + self.runtime._abandon_block_candidate( + PRISM_REJECTION_SUBMITBLOCK_REJECTED, + f"submitblock rejected candidate: {node_submission.result}", + block_hash=block_hash, + worker=worker, + expected_height=expected_height, + ) + return None + return FinalizationAdmission( + candidate=candidate, + context=context, + submission=submission, + worker=worker, + expected_height=expected_height, + block_hash=block_hash, + parent_hash=parent_hash, + # Fresh-attempt classification may use the stamped parent, but + # payout reconciliation must follow the observed post-submit tip. + current_tip=observed_tip, + already_active=already_active, + node_submission=node_submission, + ) + + def _land_candidate( + self, + admission: FinalizationAdmission, + ) -> LandedCandidate | None: + """Decide replay validity, fence the anchor, and land the candidate.""" + candidate = admission.candidate + context = admission.context + worker = admission.worker + expected_height = admission.expected_height + block_hash = admission.block_hash + node_submission = admission.node_submission + # A reconstructed candidate revalidates BEFORE the balance + # serializer: the audit share-window replay is the slow oracle walk + # and takes the ledger writer lock, so running it inside + # _payout_balance_mutation_lock would stall share persistence and + # every other landing for the walk's duration. The read is safe out + # here because the window is append-only content -- a pass can only + # be invalidated by a later append, and any such append after the + # baseline epoch captured below advances the live epoch (this + # landing's declared anchor stays exposed to the append-side + # predates() checks), which the landing's fences compare against. + collection_only = bool(getattr(context, "collection_only", False)) + context_append_epoch = int( + getattr(context, "payout_append_invalidation_epoch", 0) + ) + revalidated_append_epoch: int | None = None + landing_anchor_token: int | None = None + if not admission.already_active and not collection_only: + declared_anchor_ms = _declared_anchor_ms(context) + if declared_anchor_ms is not None: + # With no armed artifact and no in-flight walk (an outbox + # replay at startup, or a landing after the artifact was + # disarmed), a replay-shaped append would skip the epoch + # bump entirely; exposing the landing window's own anchor + # guarantees the bump the fences below check for. + landing_anchor_token = self.runtime._expose_inflight_scan_anchor( + int(declared_anchor_ms) + ) + # An append classified as unfenced before that exposure + # commits outside the landing fence, so its row could + # become durable mid-submitblock while its epoch bump + # queues behind the fence this landing holds across the + # RPC. Wait those commits (and their bump attempts + # against the now-exposed anchor) out before any window + # revalidation or epoch fence runs; no fence is held + # here, so the bump can always proceed. + self.runtime._await_unfenced_appends_predating_anchor( + int(declared_anchor_ms) + ) + try: + reconstructed_needs_revalidation = ( + not admission.already_active + and not collection_only + and context_append_epoch < 0 + and bool(getattr(self.runtime.ledger, "durable_payout_state", False)) + ) + if reconstructed_needs_revalidation and node_submission.attempted: + # Revalidation guards an offer the node has not yet seen: a + # reconstructed window that omits a durably appended share + # must not mint a coinbase. Once the fast lane has offered + # the durable bytes, the coinbase is the node's to judge — + # an accepted or duplicate result proceeds with the + # as-issued snapshot (the post-offer epoch fences log + # rather than abandon), a rejection stayed terminal above, + # and an ambiguous transport error re-offers duplicate- + # safely on retry. Abandoning an already-offered candidate + # here would permanently discard payout accounting for a + # block qbitd may have accepted, and the audit walk is the + # slow oracle whose deadline under saturation would + # otherwise defer-loop an accepted block forever. + print( + "prism coordinator: reconstructed candidate was already " + f"offered to the node hash={block_hash}; keeping the " + "as-issued payout snapshot (window revalidation " + "skipped)", + flush=True, + ) + if ( + reconstructed_needs_revalidation + and not node_submission.attempted + ): + with self.runtime._job_cache_lock: + revalidation_base_epoch = int( + self.runtime._payout_ledger_append_invalidation_epoch + ) + try: + window_reproducible = ( + self.runtime._replayed_payout_window_reproducible(context) + ) + except Exception: + traceback.print_exc() + self.runtime._abandon_block_candidate( + PRISM_REJECTION_BACKEND_RPC_UNAVAILABLE, + "durable share-window replay failed for the " + "reconstructed candidate", + block_hash=block_hash, + worker=worker, + ) + return None + if not window_reproducible: + self.runtime._abandon_block_candidate( + PRISM_REJECTION_STALE_JOB, + "replayed payout window omits a durably appended share", + block_hash=block_hash, + worker=worker, + expected_height=expected_height, + stale_job_class="append_epoch_stale", + ) + return None + revalidated_append_epoch = revalidation_base_epoch + # Land through the coordinator/runtime seam so per-instance + # monkeypatches on _land_and_confirm_block_candidate intercept. + landed = self.runtime._land_and_confirm_block_candidate( + candidate, + current_tip=admission.current_tip, + already_active=admission.already_active, + worker=worker, + node_submission=node_submission, + revalidated_append_epoch=revalidated_append_epoch, + ) + finally: + self.runtime._retire_inflight_scan_anchor(landing_anchor_token) + if landed is None: + return None + return LandedCandidate(*landed) + + def _candidate_already_accounted(self, block_hash: str) -> bool: + with self.runtime.lock: + return block_hash in self.runtime._accounted_accepted_block_hashes + + def _persist_ctv_and_credit( + self, + admission: FinalizationAdmission, + landed: LandedCandidate, + ) -> dict[str, Any] | None: + candidate = admission.candidate + block_hash = admission.block_hash + ctv_persistence = None + # On a bounded view the manifest set's fanout manifests stay lazy on + # disk; the digest streams the sorted canonical encoding and the + # ledger statement streams the set into its literal (#255). + ctv_manifest_set = landed.final_bundle.get("ctv_fanout_manifest_set") + if isinstance(ctv_manifest_set, Mapping): + self._record_block_candidate_progress("ctv-manifest-persist") + ctv_persistence = self.runtime.ledger.persist_ctv_fanout_manifest_set( + block_hash=block_hash, + manifest_set=ctv_manifest_set, + manifest_set_sha256=streamed_sha256_json_hex(ctv_manifest_set), + ) + self._record_block_candidate_progress("ctv-manifest-persist:complete") + if candidate.credit_share_on_accept: + self._record_block_candidate_progress("accepted-share-credit") + self.runtime.append_accepted_share( + candidate.client, + admission.context, + admission.submission, + candidate.pending_share, + candidate_intent=self.runtime.block_candidate_intent(candidate), + ) + # The preview window was intentionally clamped below this pending + # winning share. Once the append is durable, enqueue an urgent + # delta fold so a rapidly found child does not wait for the normal + # 60-second cadence to include its payout obligation. + with self.runtime._job_cache_lock: + payout_generation = self.runtime._payout_state_generation + template_artifacts = self.runtime._template_artifacts + if template_artifacts is not None: + self.runtime._schedule_payout_ledger_artifact_preparation( + payout_generation, + template_artifacts.network_difficulty, + bypass_build_interval=True, + ) + self._record_block_candidate_progress("accepted-share-credit:complete") + return ctv_persistence + + def _build_finalization_evidence( + self, + admission: FinalizationAdmission, + landed: LandedCandidate, + ctv_persistence: dict[str, Any] | None, + ) -> FinalizationEvidence: + # Aggregate counts only: materializing the whole share history + # (all_shares) here would scan the full ledger twice per block, + # and would grow without bound as the ledger grows. The counters are + # served from the ledger's maintained cache; a cold cache (first + # read after process start) runs the exact aggregate synchronously + # and stays watchdog-eligible on purpose, so a wedged read keeps the + # exit-and-replay recovery path instead of hanging the disposition + # invisibly. + self._record_block_candidate_progress("accepted-share-stats") + evidence_share_count, evidence_distinct_miners = self.runtime.accepted_share_stats() + self._record_block_candidate_progress("accepted-share-stats:complete") + evidence = { + "schema": "qbit.prism.live-stratum-evidence.v1", + "block_hash": admission.block_hash, + "block_height": admission.expected_height, + "coinbase_tx_hex": admission.submission.coinbase_tx_hex, + "audit_report": landed.report, + "ledger_backend": self.runtime.ledger.backend_name, + "persistence": landed.persistence, + "confirmation": landed.confirmation, + "audit_verification_identity": landed.audit_verification_identity, + "ctv_persistence": ctv_persistence, + "accepted_share_count": evidence_share_count, + "distinct_miner_count": evidence_distinct_miners, + "job_share_count": len(admission.context.shares_json), + } + publication_persistence = dict(landed.persistence) + publication_persistence.setdefault( + "audit_bundle_sha256", + landed.report.get("audit_bundle_sha256_hex"), + ) + publication_persistence.setdefault("body_uri", "") + evidence["persistence"] = publication_persistence + return FinalizationEvidence( + evidence=evidence, + publication_persistence=publication_persistence, + ) + + def _publish_finalization_evidence( + self, + landed: LandedCandidate, + prepared: FinalizationEvidence, + ) -> dict[str, Any]: + audit_store = self.runtime._ensure_audit_artifact_store() + self._record_block_candidate_progress("evidence-write") + # This is the live writer finalizing a block it confirmed itself in + # this landing. Two distinct found blocks can land inside one + # confirm->publish window -- the synchronous client-thread tail and + # the accounting actor do not serialise across hashes -- so this + # publication can arrive already superseded on the publication + # ordinal. It must then still write its own height+hash live + # envelope, which is the block's only published evidence pointer. + # + # Confirming with a valid lease does not prove the lease survived the + # confirm->publish gap: a writer deposed inside the gap would reach + # this call site with only stale reads behind it. Restoring a + # superseded envelope is a mutation of the shared audit root that the + # current evidence reference never re-validates, so restore authority + # requires a fresh bounded non-blocking proof of the exact guarded + # session (a no-op for ledgers without the writer lease guard). The + # proof gates only the restore: withholding authority degrades to the + # pre-restore behaviour of skipping the superseded envelope write, + # which is always safe, whereas failing the publication on a lease + # hiccup would be a live regression on the found-block path. The + # fence may arm a hard exit internally for a genuinely dead lease; + # that exit proceeds on its own and is not undone here. + restore_superseded_envelope = True + lease_fence_component = "superseded_audit_envelope_restore" + require_fresh_lease = getattr( + self.runtime, + "_require_fresh_ledger_lease_for_external_side_effect", + None, + ) + withheld_restore_authority: Exception | None = None + with self.runtime._payout_balance_mutation_lock: + with audit_store.publication_order_guard(): + # The proof runs with both locks held: the wait on either + # lock is unbounded, so a writer deposed while queued must + # not restore on authority it proved before the wait. + if callable(require_fresh_lease): + try: + require_fresh_lease(lease_fence_component) + except (ShutdownInProgress, WriterLeaseRenewalDeferred) as exc: + restore_superseded_envelope = False + withheld_restore_authority = exc + publication_floor_reader = getattr( + self.runtime.ledger, + "audit_publication_sequence_floor", + None, + ) + if callable(publication_floor_reader): + # This is deliberately a fresh durable-row read immediately + # before A1 publication. Confirmation-time state or a raw + # sequence value cannot fence rollback gaps and restart + # replays. P1's local serializer plus A1's process guard + # prevent another confirmation from allocating between this + # read and the durable publication decision. Reactivation + # preserves its block's already-published ordinal. + publication_floor_sequence = publication_floor_reader() + else: + # Compatibility-only ledgers used by legacy embeddings/tests + # do not own durable ordinal state. Production memory/Postgres + # backends implement the reader above. + publication_floor_sequence = ( + landed.audit_publication_identity.sequence + ) + publication = audit_store.publish_success( + identity=landed.audit_publication_identity, + publication_floor_sequence=publication_floor_sequence, + report=landed.report, + persistence=prepared.publication_persistence, + evidence=prepared.evidence, + verification_identity=landed.audit_verification_identity, + created_at=public_api.utc_now_iso(), + # Granted by the fresh-lease proof above; a deposed writer + # replaying stale state never receives restore authority. + restore_superseded_envelope=restore_superseded_envelope, + ) + if withheld_restore_authority is not None: + # Withheld authority silently reverts to the historical lossy + # behaviour for a superseded publication, so it must be + # visible. Diagnostics go to stderr because worker processes + # reserve stdout for a strict JSON-lines protocol, and are + # emitted only after the store locks are released (see below). + print( + "prism coordinator: withholding superseded-envelope " + f"restore authority component={lease_fence_component} " + f"error={type(withheld_restore_authority).__name__}: " + f"{withheld_restore_authority}", + file=sys.stderr, + flush=True, + ) + if publication.superseded_envelope_written: + # Emitted only after the balance lock and the publication order + # guard have been released: an undrained stderr must never block + # the found-block path while a store lock is held. + print( + "prism coordinator: wrote live envelope for superseded " + f"publication sequence={publication.identity.sequence} " + f"floor_sequence={publication_floor_sequence} " + f"height={publication.identity.block_height} " + f"hash={publication.identity.block_hash}", + file=sys.stderr, + flush=True, + ) + published_evidence = dict(publication.evidence) + self._record_block_candidate_progress("evidence-write:complete") + return published_evidence + + def _account_finalized_candidate( + self, + admission: FinalizationAdmission, + landed: LandedCandidate, + published_evidence: dict[str, Any], + ) -> bool: + # The copied publication is intentionally consumed before accounting; + # converting an invalid publication remains a finalization failure. + del published_evidence + block_hash = admission.block_hash + with self.runtime.lock: + newly_accounted = block_hash not in self.runtime._accounted_accepted_block_hashes + if newly_accounted: + self.runtime._accounted_accepted_block_hashes.add(block_hash) + self.runtime.accepted_block_count += 1 + # Replace this hash's provisional capacity reservation with + # its durable accounted slot atomically. Keeping both until + # the outbox terminal write would double-count the block and + # unnecessarily reject an unrelated next solve. + self.runtime._block_fast_lane_reservations.discard(block_hash) + self.runtime.latest_coinbase_size_bytes = len( + str( + landed.final_bundle["signed_coinbase_manifest"]["manifest"][ + "coinbase_tx_hex" + ] + ) + ) // 2 + should_stop = ( + newly_accounted + and (self.runtime.stop_after_block or self.runtime.accepted_block_count >= self.runtime.max_blocks) + ) + if not newly_accounted: + return True + print( + "prism coordinator: qbit accepted direct PRISM block " + f"height={admission.expected_height} hash={block_hash}", + flush=True, + ) + if should_stop: + self.runtime.request_shutdown() + else: + # The public submitter wrapper performs this fanout only after its + # writer scope (including outbox finalization) exits. The rare + # synchronous share path consumes the same marker after sending + # the Stratum result. + admission.candidate.client.post_accept_refresh_block = ( + admission.expected_height, + block_hash, + ) + return True + + def _submit_block_candidate_serialized( + self, + candidate: PrismBlockCandidate, + *, + node_submission: _BlockCandidateNodeSubmission, + ) -> bool: + """Process a candidate while its same-hash disposition guard is held. + + Runs the ordered, replay-safe post-offer finalization phases. B1's + accounting actor enters here directly with the disposition already + held; the decorated public wrapper reaches the same runner through + the coordinator seam. + """ + self._note_candidate_started() + with self._phase("admission"): + admission = self._admit_candidate( + candidate, + node_submission=node_submission, + ) + if admission is None: + return False + if isinstance(admission, bool): + return admission + with self._phase("land_confirm"): + landed = self._land_candidate(admission) + if landed is None: + return False + try: + self._record_block_candidate_progress("durable-accounting:complete") + if self._candidate_already_accounted(admission.block_hash): + # The previous attempt completed every success side effect + # but its durable outbox terminal update failed. submit_next + # will retry that update after this exact-idempotent + # confirmation without double counting the block or + # replacing newer evidence/work. + return True + with self._phase("ctv_credit"): + ctv_persistence = self._persist_ctv_and_credit(admission, landed) + with self._phase("evidence"): + prepared = self._build_finalization_evidence( + admission, + landed, + ctv_persistence, + ) + with self._phase("audit_publish"): + published_evidence = self._publish_finalization_evidence( + landed, + prepared, + ) + with self._phase("accounting"): + return self._account_finalized_candidate( + admission, + landed, + published_evidence, + ) + finally: + # The landed bundle is the last legitimate owner of the bounded + # artifact view; retire its descriptor on every exit so neither + # a failed phase nor a completed one leaves the candidate file + # pinned (#255). Compatibility dictionaries have nothing to + # close. + close_view = getattr( + getattr(landed, "final_bundle", None), + "close", + None, + ) + if callable(close_view): + close_view() + + def submit_block_candidate( + self, + candidate: PrismBlockCandidate, + *, + node_submission: _BlockCandidateNodeSubmission | None = None, + ) -> bool: + """Land one block candidate, then finalize its audit and payout state. + + Runs on the block-submitter thread (tests call it synchronously). It + never raises for a lost race and holds self.runtime.lock only for short + in-memory state mutation -- never across RPC, psql, subprocess, or + file I/O -- so share acks and job pushes stay fast while a block + lands. The durable candidate outbox is the pre-submit recovery boundary; + full audit and payout persistence happens after the latency-sensitive + ``submitblock`` call and is replayable after a crash. Returns True only + after that finalization completes. + """ + block_hash = str(candidate.submission.block_hash_hex).lower() + with self.runtime._block_candidate_disposition(block_hash): + terminal_outcome = self.runtime._block_candidate_terminal_outcome(block_hash) + if terminal_outcome is not None: + return terminal_outcome + if node_submission is None: + node_submission = self.runtime._node_submission_for_direct_candidate(candidate) + accepted = self.runtime._submit_block_candidate_serialized( + candidate, + node_submission=node_submission, + ) + if not accepted: + outcome = getattr(self.runtime, "_block_candidate_outcome", None) + if outcome is not None: + # Direct embedders do not use the outbox-finalization + # wrapper. A normal return means the serialized path also + # completed any prepared-state rejection it initiated. + self.runtime._record_committed_block_candidate_abandonment( + block_hash, + outcome, + ) + return accepted diff --git a/lab/prism/bounded_executor.py b/lab/prism/bounded_executor.py new file mode 100644 index 000000000..862f7719e --- /dev/null +++ b/lab/prism/bounded_executor.py @@ -0,0 +1,166 @@ +"""Bounded priority executor used for PRISM job delivery.""" + +from __future__ import annotations + +from concurrent.futures import Future +import heapq +import math +import queue +import threading +from typing import Any, Callable + + +class _DeliveryQueueFull(RuntimeError): + """The bounded delivery executor cannot admit another task.""" + + +class _BoundedPriorityExecutor: + """Small Future-compatible executor with bounded, priority-ordered work.""" + + def __init__( + self, + *, + max_workers: int, + max_queue_size: int, + thread_name_prefix: str = "prism-job-delivery", + ) -> None: + self.max_workers = max_workers + self.max_queue_size = max_queue_size + self._queue: queue.PriorityQueue[tuple[object, ...]] = queue.PriorityQueue( + maxsize=max_queue_size + ) + self._lock = threading.Lock() + self._sequence = 0 + self._active_workers = 0 + self._shutdown = False + self._threads = [ + threading.Thread( + target=self._worker, + name=f"{thread_name_prefix}-{index + 1}", + daemon=True, + ) + for index in range(max_workers) + ] + for thread in self._threads: + thread.start() + + def submit( + self, + function: Callable[..., Any], + /, + *args: object, + # Same-tip refreshes are the lowest-priority delivery class; the + # numeric default stays owner-local so this executor never imports + # the delivery-priority registry. + priority: int = 2, + **kwargs: object, + ) -> Future[Any]: + future: Future[Any] = Future() + with self._lock: + if self._shutdown: + raise RuntimeError("delivery executor is shut down") + self._sequence += 1 + item = ( + int(priority), + self._sequence, + future, + function, + args, + kwargs, + ) + try: + self._queue.put_nowait(item) + except queue.Full as exc: + raise _DeliveryQueueFull("delivery executor queue is full") from exc + return future + + def _worker(self) -> None: + while True: + item = self._queue.get() + try: + _, _, future, function, args, kwargs = item + if function is None: + return + assert isinstance(future, Future) + if not future.set_running_or_notify_cancel(): + continue + with self._lock: + self._active_workers += 1 + try: + result = function(*args, **kwargs) + except BaseException as exc: + future.set_exception(exc) + else: + future.set_result(result) + finally: + with self._lock: + self._active_workers -= 1 + finally: + self._queue.task_done() + # An idle worker's frame stays alive in queue.get(). Release + # the previous task outside all locks before waiting: delivery + # arguments can own an entire superseded payout window. Cover + # failed and cancelled tasks too, and drop any prior result. + item = future = function = args = kwargs = result = None + + def cancel(self, future: Future[Any]) -> bool: + """Cancel ``future`` and immediately discard it when still queued. + + ``Future.cancel`` alone leaves the cancelled entry in PriorityQueue + until a worker dequeues it. Removing the exact entry while holding the + queue mutex makes bounded admission available to a replacement at the + cancellation boundary. A worker that already dequeued the entry owns + the normal ``task_done`` path, so the two paths cannot double-release. + """ + removed = False + with self._queue.mutex: + queued_items = self._queue.queue + for index, item in enumerate(queued_items): + if item[2] is not future: + continue + queued_items.pop(index) + heapq.heapify(queued_items) + self._queue.unfinished_tasks -= 1 + if self._queue.unfinished_tasks == 0: + self._queue.all_tasks_done.notify_all() + self._queue.not_full.notify() + removed = True + break + # Invoke callbacks only after releasing the queue mutex. Initial-job + # cancellation callbacks may submit a replacement to this executor. + future.cancel() + return removed + + def stats(self) -> tuple[int, int]: + with self._lock: + return self._queue.qsize(), self._active_workers + + def shutdown(self, *, wait: bool = True, cancel_futures: bool = False) -> None: + with self._lock: + if self._shutdown: + threads = list(self._threads) + already_shutdown = True + else: + self._shutdown = True + threads = list(self._threads) + already_shutdown = False + if already_shutdown: + if wait: + for thread in threads: + thread.join() + return + if cancel_futures: + while True: + try: + item = self._queue.get_nowait() + except queue.Empty: + break + future = item[2] + if isinstance(future, Future): + future.cancel() + self._queue.task_done() + for index in range(len(threads)): + self._queue.put((math.inf, index, None, None, (), {})) + if wait: + for thread in threads: + thread.join() diff --git a/lab/prism/bundle_compiler.py b/lab/prism/bundle_compiler.py new file mode 100644 index 000000000..d3479a7e1 --- /dev/null +++ b/lab/prism/bundle_compiler.py @@ -0,0 +1,2639 @@ +#!/usr/bin/env python3 +"""Cancelable subprocess and daemon transport for PRISM audit-bundle builds. + +The compiler owns the complete builder wire/process domain: the compact +audit payload encoding, the share-window spool representation and its +leases, the one-shot subprocess build, and the persistent ``--serve`` daemon +protocol. It never imports ``prism_coordinator`` or ``job_bundle``: shared +scheduler state and refresh metrics are reached through the +:class:`BundleCompilerRuntime` port (resolved at call time so coordinator +monkeypatch seams keep working), and the job-bundle exception/control types +are injected at construction exactly like the historical layer's +``superseded_error`` port. +""" + +from __future__ import annotations + +from collections import OrderedDict +from contextlib import ExitStack, contextmanager +from dataclasses import dataclass, field +import json +import os +from pathlib import Path +import selectors +import stat +import subprocess +import tempfile +import threading +import time +import uuid +from collections.abc import Mapping, Sequence +from typing import Any, Callable, Iterable, Iterator, Protocol + +from lab.prism.audit_bundle_view import CanonicalAuditBundleView +from lab.prism.coordinator_config import ( + DEFAULT_PRISM_BUNDLE_BUILD_TIMEOUT_SECONDS, + DEFAULT_PRISM_JOB_BUILD_CANCEL_GRACE_SECONDS, + env_bool, +) +from lab.prism.prism_tools import prism_tool_command +from lab.prism.share_json_stream import ( + SHARE_JSON_BATCH_RECORDS, + SHARE_JSON_CHUNK_BYTES, + iter_json_array_text_chunks, + iter_json_object_text_chunks, +) +from lab.prism.share_ledger import DaemonWindowMirrorDivergence + + +PRISM_BUILDER_PHASE_METRICS_PREFIX = "qbit-prism-build-phase-metrics " +# Owner-local duplicate of the coordinator's admission poll cadence; the +# compiler must not import the coordinator for one pacing constant. +PRISM_TIP_REFRESH_ADMISSION_POLL_SECONDS = 0.05 +# Longest one pipe readiness wait may block before a transport loop +# re-checks cancellation, supersession and its absolute deadline. This is +# the cadence the retired fixed post-EAGAIN sleeps enforced; a readiness +# wait wakes the moment the pipe can progress and only sleeps this long +# when it cannot. +PRISM_BUILDER_PIPE_WAIT_SLICE_SECONDS = min( + 0.02, + PRISM_TIP_REFRESH_ADMISSION_POLL_SECONDS, +) +# Largest single os.read from a builder pipe. Matches the default Linux +# pipe capacity so one wakeup drains one full pipe. +PRISM_BUILDER_PIPE_READ_CHUNK_BYTES = 1 << 16 +# Upper bound for one kernel transfer from the share-window spool file into +# the audit builder's stdin pipe. The kernel clamps each call to the free +# pipe capacity anyway; the bound only paces cancellation checkpoints. +PRISM_SPOOL_SPLICE_CHUNK_BYTES = 1 << 20 +# JSONL protocol version this coordinator speaks with the --serve audit +# builder. The daemon announces its version in a startup handshake; any +# mismatch retires the daemon and every build falls back to one-shot mode. +# Version 2 adds the prepare_window request (window fold, canonical digest, +# and incremental advance daemon-side) and the append-invalidation epoch tag. +PRISM_SERVE_BUILDER_PROTOCOL_VERSION = 2 +# Parsed share windows the --serve daemon retains; the coordinator mirrors +# the bound to predict which uploads the daemon still holds. +PRISM_SERVE_BUILDER_WINDOW_CACHE_ENTRIES = 2 + + +def canonical_bundle_bytes(bundle: dict[str, Any]) -> bytes: + """Use J1's typed Rust adapter for canonical audit-bundle bytes. + + Generic compact JSON is not equivalent: serde restores typed field order, + emits UTF-8, and omits optional defaults. Keep those digest semantics in J1. + """ + + completed = subprocess.run( + prism_tool_command("qbit-prism-audit-canonicalize") + ["--input", "-"], + input=json.dumps(bundle).encode("utf-8"), + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + check=False, + ) + if completed.returncode != 0: + stderr = completed.stderr.decode(errors="replace").strip() + raise RuntimeError(f"qbit-prism-audit-canonicalize failed: {stderr}") + return completed.stdout + + +class CancellationPort(Protocol): + """Cooperative-cancellation surface consumed by the compiler.""" + + def is_set(self) -> bool: ... + + def raise_if_cancelled(self, phase: str) -> None: ... + + +class BundleBuildControlPort(Protocol): + """Registered build control consumed for supersession checks.""" + + cancel_event: Any + process: Any + + +class BundleCompilerRuntime(Protocol): + """Typed port over the coordinator, resolved at call time.""" + + signing_seed_hex: str + ledger_attestation_signing_seed_hex: str + + def _ensure_job_cache_state(self) -> None: ... + + def _ensure_tip_refresh_state(self) -> None: ... + + def _job_build_checkpoint(self, phase: str, cancellation: Any) -> None: ... + + def _job_build_phases(self) -> dict[str, float]: ... + + def prism_payout_policy(self) -> dict[str, object]: ... + + def prism_ctv_settlement_config( + self, + *, + block_height: int, + parent_hash: str | None, + ) -> dict[str, object] | None: ... + + def _observe_tip_refresh_build_phase(self, name: str, elapsed: float) -> None: ... + + def _record_tip_refresh_ipc_bytes(self, direction: str, byte_count: int) -> None: ... + + def _register_job_bundle_process(self, control: Any, process: Any) -> None: ... + + def _unregister_job_bundle_process(self, control: Any, process: Any) -> None: ... + + def _note_window_mirror_divergence(self) -> None: ... + + +def _compact_share_payload( + shares: list[dict[str, object]], +) -> tuple[list[tuple[str, str, str]], list[tuple[object, ...]]]: + """Deduplicate share identities into the audit-builder compact form.""" + + identity_indexes: dict[tuple[str, str, str], int] = {} + identities: list[tuple[str, str, str]] = [] + compact_shares: list[tuple[object, ...]] = [] + for share in shares: + identity = ( + str(share["miner_id"]), + str(share["order_key"]), + str(share["p2mr_program_hex"]), + ) + identity_index = identity_indexes.get(identity) + if identity_index is None: + identity_index = len(identities) + identity_indexes[identity] = identity_index + identities.append(identity) + compact_shares.append( + ( + share["share_seq"], + share["share_id"], + identity_index, + share["share_difficulty"], + share["job_issued_at_ms"], + share["accepted_at_ms"], + share.get("credit_policy"), + ) + ) + return identities, compact_shares + + +def _compact_share_tail_chunks( + shares: Iterable[dict[str, object]], + *, + batch_records: int = SHARE_JSON_BATCH_RECORDS, + chunk_chars: int = SHARE_JSON_CHUNK_BYTES, +) -> tuple[str, ...]: + """The audit-builder payload tail as bounded ASCII chunks. + + Concatenated, the chunks are byte-identical to the historical tail -- + ``',"compact_share_identities":' + json.dumps(identities) + + ',"compact_shares":' + json.dumps(compact_shares) + "}"`` with compact + separators over :func:`_compact_share_payload`'s two lists, the exact + bytes the spool held and the in-memory fallback wrote -- but no + ``json.dumps`` call covers more than ``batch_records`` records, so + encoding a 200k-share window no longer holds the GIL in one piece + (#236). Identities are indexed in first-seen order exactly as before; + the share rows are encoded as they are walked, and the identity table, + complete only once the walk ends, is placed ahead of them in the + returned order. + """ + identity_indexes: dict[tuple[str, str, str], int] = {} + identities: list[tuple[str, str, str]] = [] + + def compact_rows() -> Iterator[tuple[object, ...]]: + for share in shares: + identity = ( + str(share["miner_id"]), + str(share["order_key"]), + str(share["p2mr_program_hex"]), + ) + identity_index = identity_indexes.get(identity) + if identity_index is None: + identity_index = len(identities) + identity_indexes[identity] = identity_index + identities.append(identity) + yield ( + share["share_seq"], + share["share_id"], + identity_index, + share["share_difficulty"], + share["job_issued_at_ms"], + share["accepted_at_ms"], + share.get("credit_policy"), + ) + + share_chunks = tuple( + iter_json_array_text_chunks( + compact_rows(), + batch_records=batch_records, + chunk_chars=chunk_chars, + ) + ) + identity_chunks = tuple( + iter_json_array_text_chunks( + identities, + batch_records=batch_records, + chunk_chars=chunk_chars, + ) + ) + return ( + ',"compact_share_identities":[', + *identity_chunks, + '],"compact_shares":[', + *share_chunks, + "]}", + ) + + +PRISM_BUILDER_INPUT_ARRAY_KEYS = ( + "shares", + "prior_balances", + "witness_merkle_leaves_hex", + "compact_share_identities", + "compact_shares", +) + + +def _iter_share_window_items( + value: Any, + *, + batch_records: int, + chunk_chars: int, +) -> Iterator[str | bytes]: + """The ``shares`` array body from the cheapest exact source. + + A daemon-mirror sequence carries the window already encoded as + ``canonical_items`` (sorted-key records with numeric difficulties, the + bytes the builder daemon itself parses); a page-backed window carries + one such fragment per page. Both are spliced in bounded slices without + parsing a record -- in particular without touching the mirror's parsed + tuple cache. Any other replayable sequence is walked record by record. + The builder deserializes by field name, so record key order is + irrelevant to the canonical output. + """ + items = getattr(value, "canonical_items", None) + if isinstance(items, (bytes, bytearray, memoryview)): + view = memoryview(items) + for offset in range(0, len(view), chunk_chars): + yield bytes(view[offset : offset + chunk_chars]) + return + pages = getattr(value, "pages", None) + if isinstance(pages, (list, tuple)) and pages and all( + isinstance(getattr(page, "canonical_json_items", None), (bytes, bytearray, memoryview)) + for page in pages + ): + needs_separator = False + for page in pages: + page_items = memoryview(page.canonical_json_items) + if not len(page_items): + continue + if needs_separator: + yield b"," + for offset in range(0, len(page_items), chunk_chars): + yield bytes(page_items[offset : offset + chunk_chars]) + needs_separator = True + return + yield from iter_json_array_text_chunks( + value, + batch_records=batch_records, + chunk_chars=chunk_chars, + ) + + +def _iter_build_input_chunks( + payload: Mapping[str, object], + *, + array_keys: Iterable[str] = PRISM_BUILDER_INPUT_ARRAY_KEYS, + batch_records: int = SHARE_JSON_BATCH_RECORDS, + chunk_chars: int = SHARE_JSON_CHUNK_BYTES, +) -> Iterator[str | bytes]: + """The one-shot builder input in bounded chunks, every array streamed. + + Byte-identical to ``json.dumps(payload, separators=(",", ":"))`` for a + payload of plain lists. :func:`iter_json_object_text_chunks` streams + only ``list``/``tuple`` members; here every member named in + ``array_keys`` streams whatever replayable sequence it holds (each + ``json.dumps`` covers at most ``batch_records`` records), the ``shares`` + member additionally splices pre-encoded window bytes when the sequence + carries them, and the remaining members (found block, policy, script + suffix, settlement configuration) are fixed-size and take one + ``json.dumps`` each. Chunks are ``str`` except spliced window bytes, + which are yielded as ``bytes`` (#255). + """ + keys = list(payload) + if any(not isinstance(key, str) for key in keys): + yield json.dumps(dict(payload), separators=(",", ":")) + return + streamed = set(array_keys) + pending: list[str] = [] + pending_chars = 0 + + def flush() -> Iterator[str]: + nonlocal pending, pending_chars + if pending: + chunk = "".join(pending) + pending = [] + pending_chars = 0 + yield chunk + + def push(text: str) -> Iterator[str]: + nonlocal pending_chars + if not text: + return + pending.append(text) + pending_chars += len(text) + if pending_chars >= chunk_chars: + yield from flush() + + yield from push("{") + first = True + for key in keys: + value = payload[key] + prefix = ("" if first else ",") + json.dumps(key) + ":" + first = False + if ( + key in streamed + and not isinstance(value, (str, bytes, bytearray, Mapping)) + and isinstance(value, Iterable) + ): + yield from push(prefix + "[") + pieces: Iterable[str | bytes] + if key == "shares": + pieces = _iter_share_window_items( + value, + batch_records=batch_records, + chunk_chars=chunk_chars, + ) + else: + pieces = iter_json_array_text_chunks( + value, + batch_records=batch_records, + chunk_chars=chunk_chars, + ) + for piece in pieces: + if isinstance(piece, (bytes, bytearray)): + yield from flush() + yield bytes(piece) + else: + yield from push(piece) + yield from push("]") + else: + yield from push(prefix + json.dumps(value, separators=(",", ":"))) + yield from push("}") + yield from flush() + + +def _iter_prepare_window_request_chunks( + request_fields: dict[str, object], +) -> Iterator[bytes]: + """One prepare_window request line in bounded chunks, newline included. + + Byte-identical to ``json.dumps(request_fields, separators=(",", ":"))`` + plus the terminating newline: the envelope members are each one small + ``json.dumps`` and the ``records`` array is encoded batch by batch. + """ + for chunk in iter_json_object_text_chunks( + request_fields, + array_keys=("records",), + ): + yield chunk.encode("utf-8") + yield b"\n" + + +def _share_window_spool_file() -> Any: + """Anonymous spool file for the serialized share-window payload tail. + + Created in the same temporary filesystem the builder's captured output + and stderr already use, and unlinked from birth: a crashed coordinator + can never strand a multi-megabyte window on disk, and closing the last + descriptor is the entire cleanup story. + """ + return tempfile.TemporaryFile() + + +@dataclass +class _ShareWindowSerialization: + """Derived share-window forms cached per canonical window. + + For a fixed (canonical digest, count, window weight) the serialized share + window is immutable: its canonical digest and the + audit-builder compact payload fragments are pure functions of the + artifact's share snapshot. Recomputing them inside every template-bump + build serialized the whole window under the GIL each time. The compact + fragments are derived lazily by the first audit-builder invocation -- + embedders that replace the builder never require the compact schema. + + The fragments' invariant byte encoding is additionally spooled to an + anonymous temp file once per key, so every build after the first can feed + the audit builder's stdin through kernel transfers instead of re-writing + megabytes of unchanged JSON through Python. Spool access is leased: + rotation to a new generation retires the spool, and the descriptor closes + when the last in-flight transfer releases it, never underneath one. + """ + + key: tuple[str, int, int] + share_count: int + share_snapshot_sha256: str + _source_artifact: Any = field( + default=None, + repr=False, + compare=False, + ) + _compact_lock: threading.Lock = field( + default_factory=threading.Lock, + repr=False, + ) + # The audit-builder payload tail (the compact identities and shares + # members plus the closing brace) as bounded ASCII chunks, encoded once + # per generation; see compact_tail_chunks. + _compact_tail_chunks: tuple[str, ...] | None = field( + default=None, + repr=False, + ) + _compact_tail_chars: int = field(default=0, repr=False) + _spool_lock: threading.Lock = field( + default_factory=threading.Lock, + repr=False, + ) + _spool_file: Any = field(default=None, repr=False) + _spool_size: int = field(default=0, repr=False) + _spool_failed: bool = field(default=False, repr=False) + _spool_retired: bool = field(default=False, repr=False) + _spool_leases: int = field(default=0, repr=False) + # Call-time-resolved spool-file factory. The coordinator wires a lambda + # that resolves its own module global so the historical + # ``prism_coordinator._share_window_spool_file`` patch seam keeps working. + _spool_factory: Callable[[], Any] | None = field( + default=None, + repr=False, + compare=False, + ) + + def compact_tail_chunks( + self, + shares: list[dict[str, object]], + ) -> tuple[str, ...]: + """Encoded payload tail chunks, computed once per generation. + + Concurrent builders block here instead of duplicating the encode; the + window is immutable for this key, so first-writer-wins is exact. + Every chunk is ASCII (``json.dumps``' ``ensure_ascii`` default), so + its character count is its byte count on every transport, and no + chunk splits a record. + """ + with self._compact_lock: + if self._compact_tail_chunks is None: + chunks = _compact_share_tail_chunks(shares) + self._compact_tail_chars = sum(len(chunk) for chunk in chunks) + self._compact_tail_chunks = chunks + return self._compact_tail_chunks + + @property + def compact_json_bytes(self) -> int: + """Resident bytes of the encoded tail; 0 until its first use.""" + return int(self._compact_tail_chars) + + def acquire_spooled_tail( + self, + shares: list[dict[str, object]], + ) -> tuple[Any, int] | None: + """Lease the spooled payload tail, writing it on first use. + + Returns the open spool file and its byte size, or None when spooling + is unavailable (creation failed earlier, a transfer poisoned it, or + the generation was retired). The caller must pair a successful lease + with release_spooled_tail once its transfer is finished. + """ + with self._spool_lock: + if self._spool_failed or self._spool_retired: + return None + if self._spool_file is None: + chunks = self.compact_tail_chunks(shares) + spool = None + spool_factory = self._spool_factory or _share_window_spool_file + try: + spool = spool_factory() + for chunk in chunks: + spool.write(chunk.encode("utf-8")) + spool.flush() + size = spool.seek(0, os.SEEK_END) + except (OSError, ValueError): + # Spooling is an optimization; the in-memory pipe path + # remains authoritative. Failure is sticky so a broken + # temp filesystem is not retried on every build. + self._spool_failed = True + if spool is not None: + try: + spool.close() + except OSError: + pass + return None + self._spool_file = spool + self._spool_size = int(size) + self._spool_leases += 1 + return self._spool_file, self._spool_size + + def release_spooled_tail(self) -> None: + with self._spool_lock: + self._spool_leases -= 1 + self._close_spool_if_unused_locked() + + def mark_spool_failed(self) -> None: + """Poison the spool after a transfer-side failure (still leased).""" + with self._spool_lock: + self._spool_failed = True + + def retire_spool(self) -> None: + """Stop new leases and close the file once in-flight ones finish.""" + with self._spool_lock: + self._spool_retired = True + self._close_spool_if_unused_locked() + + def _close_spool_if_unused_locked(self) -> None: + if ( + self._spool_leases > 0 + or self._spool_file is None + or not (self._spool_retired or self._spool_failed) + ): + return + spool, self._spool_file = self._spool_file, None + self._spool_size = 0 + try: + spool.close() + except OSError: + pass + + +class _PipeReadinessWaiter: + """Readiness wait for one non-blocking pipe end of a builder process. + + The transport loops drive ``os.read``/``os.write``/``os.splice`` on + non-blocking descriptors so that cancellation, supersession and the + absolute deadline are checked between syscalls. They used to sleep a + fixed 20 ms after every EAGAIN, charging one sleep per drained pipe + capacity (64 KiB on Linux): a multi-megabyte window exchange then paid + orders of magnitude more in idle waits than in copying (#236). A waiter + blocks in the platform selector instead, bounded by BOTH the remaining + deadline and the cancellation-check cadence, so a waiting build wakes as + soon as bytes arrive or drain and still re-checks its control signals on + the old schedule when nothing happens. + + One instance serves one transport operation. Registration is lazy -- an + operation satisfied from buffered bytes or by syscalls that never block + costs no selector -- and ``close`` releases the registration on every + exit path. A descriptor the selector refuses (a regular file standing + in for the pipe, a closed or fake descriptor) degrades to the historical + bounded sleep so the retry loop keeps working; so does one the selector + registers but cannot poll (``SelectSelector`` over a Windows pipe + handle, a descriptor torn down under the registration): the selector is + released once, the descriptor stays unwatchable for the rest of the + operation, and the sleep takes only whatever budget the failed poll + left. Readiness reported by the selector may be spurious and EINTR is + never a wait: both are resolved by the caller retrying the syscall after + its control checks. + """ + + __slots__ = ("_events", "_file_descriptor", "_selector", "_unwatchable", "waits") + + def __init__(self, file_descriptor: int, events: int) -> None: + self._file_descriptor = file_descriptor + self._events = events + self._selector: selectors.BaseSelector | None = None + self._unwatchable = False + # Blocked waits taken by this operation; read by tests and the + # transport benchmark, never by production metrics. + self.waits = 0 + + def __enter__(self) -> _PipeReadinessWaiter: + return self + + def __exit__(self, *exc_info: object) -> None: + self.close() + + def wait(self, deadline: float) -> bool: + """Block until ready, the cadence slice ends, or the deadline passes. + + Returns True when readiness was reported (possibly spuriously) and + False when the slice or the deadline elapsed first. It never raises + for an exhausted deadline: the calling loop owns the ordering of its + cancellation, supersession and deadline checks and performs them + before the next syscall. + """ + self.waits += 1 + now = time.monotonic() + timeout = min(PRISM_BUILDER_PIPE_WAIT_SLICE_SECONDS, deadline - now) + if timeout <= 0: + return False + # This wait may consume no more than one slice, however it ends. + slice_deadline = min(now + timeout, deadline) + selector = self._selector + if selector is None and not self._unwatchable: + try: + selector = selectors.DefaultSelector() + try: + selector.register(self._file_descriptor, self._events) + except BaseException: + selector.close() + raise + except (OSError, ValueError, KeyError): + selector = None + self._unwatchable = True + else: + self._selector = selector + if selector is not None: + try: + # select() retries EINTR itself with the remaining timeout + # (PEP 475); a signal handler that raises propagates as is. + return bool(selector.select(timeout)) + except InterruptedError: + # Not a readiness verdict and not a fault: the caller + # re-checks its control signals and retries the syscall. + return False + except (OSError, ValueError): + # Registered, but the selector cannot poll this descriptor. + # Release it exactly once, remember that for the rest of the + # operation, and finish THIS wait with the bounded sleep + # over whatever budget the failed poll left -- never a fresh + # full slice, never past the absolute deadline. + self._detach_selector() + timeout = slice_deadline - time.monotonic() + if timeout <= 0: + return False + time.sleep(timeout) + return False + + def _detach_selector(self) -> None: + self._unwatchable = True + selector, self._selector = self._selector, None + if selector is not None: + try: + selector.close() + except OSError: + pass + + def close(self) -> None: + selector, self._selector = self._selector, None + if selector is not None: + selector.close() + + +class _ServeBuilderUnavailable(RuntimeError): + """Daemon anomaly; the build must transparently use one-shot mode.""" + + +@dataclass(frozen=True) +class PreparedWindowOutcome: + """One prepare_window round trip's typed result. + + ``status`` is one of: + + - ``"prepared"`` -- the daemon holds the folded window; the digest, + record count, advance stats, and the canonical-items byte payload + (full stream for ``full`` mode, prefix-drop plus appended suffix for + ``advance``) are populated. + - ``"needs_full"`` -- the daemon does not hold ``base_digest`` (respawn, + eviction, epoch change); the caller re-sends a full preparation. + Ordinary control flow, never a daemon anomaly. + - ``"fallback"`` -- an advance invariant failed daemon-side; the caller + takes exactly its IncrementalWindowFallback path (clear the cache, + full rescan). Ordinary control flow, never a daemon anomaly. + - ``"fold_invalid"`` -- the full snapshot itself violates the fold's + invariants; the in-process oracle raises the same rejection, so the + caller falls through to it for the authoritative error. + - ``"out_of_range"`` -- the request carries an integer outside the + daemon's declared widths (``share_seq`` u64, ``share_difficulty`` and + ``network_difficulty`` u128, ``template_height`` u64, ``ntime`` u32, + millisecond stamps i64, ``window_weight`` u128) or a difficulty total + that leaves u128. The daemon is healthy, keeps its state and stays + registered; this materialization degrades to the in-process pipeline, + whose unbounded integers fold the window exactly. Ordinary control + flow, never a daemon anomaly -- distinct from a malformed request, + which does retire the daemon. + - ``"busy"`` -- another build owns the daemon and it could not be had + within this preparation's budget. The daemon is healthy and stays + registered; this materialization degrades to the in-process pipeline. + Ordinary control flow, never a daemon anomaly. + + ``rejection`` is populated for ``fallback`` and ``fold_invalid``: the + daemon's stable, machine-readable category for the refused condition + (the parity oracle's vocabulary), never to be inferred from ``error``, + whose text is diagnostic and free to drift. + """ + + status: str + share_snapshot_sha256: str | None = None + record_count: int = 0 + added_rows: int = 0 + expired_rows: int = 0 + touched_pages: int = 0 + window_items: bytes | None = field(default=None, repr=False) + retained_drop_bytes: int = 0 + appended_items: bytes = field(default=b"", repr=False) + error: str | None = None + rejection: str | None = None + + +@dataclass +class _ServeBuilderClient: + """Line-oriented I/O state for one long-lived --serve audit builder.""" + + process: subprocess.Popen[bytes] + stdout_buffer: bytearray = field(default_factory=bytearray, repr=False) + # True from the first byte of a request until its terminating newline. + # A request that escapes half-written leaves the daemon's stdin holding + # an unterminated line, and the NEXT request would concatenate onto it, + # so the daemon must be retired rather than reused. + request_incomplete: bool = False + # Mirrors the daemon's bounded LRU so requests can predict whether the + # window must ride along. Divergence is repaired by the daemon's + # needs_window response, never trusted blindly. + uploaded_windows: OrderedDict[str, None] = field( + default_factory=OrderedDict, + repr=False, + ) + + def note_uploaded_window(self, share_snapshot_sha256: str) -> None: + self.uploaded_windows.pop(share_snapshot_sha256, None) + self.uploaded_windows[share_snapshot_sha256] = None + while len(self.uploaded_windows) > PRISM_SERVE_BUILDER_WINDOW_CACHE_ENTRIES: + self.uploaded_windows.popitem(last=False) + + def close(self) -> None: + # Tolerates lightweight process fakes used by embedders and tests, + # which do not necessarily expose the full Popen surface. + process = self.process + poll = getattr(process, "poll", None) + wait = getattr(process, "wait", None) + if callable(poll): + if poll() is None: + try: + process.kill() + except (ProcessLookupError, OSError, AttributeError): + pass + if callable(wait): + try: + wait(timeout=5.0) + except ( + subprocess.TimeoutExpired, + OSError, + TypeError, + ValueError, + AttributeError, + ): + pass + for stream in ( + getattr(process, "stdin", None), + getattr(process, "stdout", None), + ): + if stream is None or not hasattr(stream, "close"): + continue + try: + stream.close() + except OSError: + pass + + +class BundleCompiler: + """Compile summaries or canonical bundles with exact cancellation rules. + + Owns the persistent ``--serve`` daemon state and its metrics; every + scheduler counter, refresh metric, and payload input is reached through + the runtime port so the coordinator remains the single owner of that + state and its monkeypatch seams. + """ + + def __init__( + self, + runtime: BundleCompilerRuntime, + *, + superseded_error: Callable[[str], BaseException], + cancellation_error_types: tuple[type[BaseException], ...], + build_control_type: type, + tool_command: Callable[[str], list[str]] | None = None, + ) -> None: + self._runtime = runtime + self._superseded_error = superseded_error + self._cancellation_error_types = tuple(cancellation_error_types) + self._build_control_type = build_control_type + # Call-time-resolved builder-command factory: the coordinator wires a + # lambda over its own module global so the historical + # ``prism_coordinator.prism_tool_command`` patch seam keeps working. + self._tool_command = tool_command or prism_tool_command + # Serializes daemon ownership per request. Contended builds do + # not queue behind the daemon; they take the one-shot path. + self._serve_builder_lock = threading.Lock() + self._serve_builder: _ServeBuilderClient | None = None + self._serve_builder_shutdown = False + # Counters get their own short-lived lock so a metrics scrape + # never waits behind _serve_builder_lock, which one request can + # hold for a whole daemon round trip. + self._serve_builder_metrics_lock = threading.Lock() + # Guarded by _serve_builder_metrics_lock. + self.serve_builder_counts = { + "requests": 0, + "fallbacks": 0, + "spawns": 0, + "window_uploads": 0, + "window_prepares": 0, + "window_prepare_contended": 0, + "window_prepare_out_of_range": 0, + } + # Guarded by _serve_builder_metrics_lock. + self.serve_builder_window_cache_counts = {"hits": 0, "misses": 0} + + @contextmanager + def _routed_window_mirror_divergence(self) -> Iterator[None]: + """Report a refuted daemon window mirror, then let the error stand. + + A mirror reconciles its bytes, digest and record count when it is + built, so nothing here should ever fire; if it does, the build that + touched the mirror is not the place the problem started, and it must + not be the only place the problem is visible either. Invalidating + the cached window keeps the NEXT build off the same bytes, and the + counted event turns a lone traceback into something a scrape sees. + """ + try: + yield + except DaemonWindowMirrorDivergence: + note = getattr( + self._runtime, + "_note_window_mirror_divergence", + None, + ) + if callable(note): + note() + raise + + def shutdown_serve_builder(self) -> None: + """Retire the persistent audit builder; builds revert to one-shot.""" + self._runtime._ensure_job_cache_state() + with self._serve_builder_lock: + self._serve_builder_shutdown = True + client, self._serve_builder = self._serve_builder, None + if client is not None: + client.close() + + def _retire_serve_builder_locked(self) -> None: + client, self._serve_builder = self._serve_builder, None + if client is not None: + client.close() + + def _record_live_serve_builder_termination( + self, + client: _ServeBuilderClient | None, + ) -> None: + """Account for deliberately killing a still-running daemon. + + Crash paths count themselves when the EOF surfaces; a live worker + retired for a timeout, malformed response, protocol mismatch, or + spool failure must land in the termination counters so its + replacement reads as a restart, mirroring the cancellation path. + """ + runtime = self._runtime + if client is None: + return + poll = getattr(client.process, "poll", None) + if not callable(poll) or poll() is not None: + return + with runtime._job_build_scheduler_lock: + runtime.job_build_worker_counts["terminations"] += 1 + runtime._job_build_worker_restart_pending = True + + def _observe_builder_phase_metrics(self, metrics: dict[str, Any]) -> None: + """Apply one build's builder-side phase timings to refresh metrics. + + Shared by the one-shot stderr metrics line and the --serve response + payload so serialization_copy and the builder phase histograms stay + comparable regardless of the transport that ran the build. + """ + runtime = self._runtime + phase_seconds = metrics.get("phases_seconds", {}) + if isinstance(phase_seconds, dict): + for phase in ( + "payout_state_derivation", + "ctv_manifest_construction", + "coinbase_bundle_construction", + "signing_verification", + ): + elapsed = phase_seconds.get(phase) + if isinstance(elapsed, (int, float)): + runtime._observe_tip_refresh_build_phase( + phase, + float(elapsed), + ) + rust_serialization = sum( + float(metrics.get(name, 0.0)) + for name in ( + "input_deserialization_seconds", + "output_serialization_seconds", + ) + ) + runtime._observe_tip_refresh_build_phase( + "serialization_copy", + rust_serialization, + ) + + def _spawn_serve_builder_locked( + self, + deadline: float, + cancellation: CancellationPort | None, + ) -> _ServeBuilderClient: + runtime = self._runtime + command = self._tool_command("qbit-prism-build-audit-bundle") + [ + "--serve", + "--signing-key-seed-hex", + runtime.signing_seed_hex, + "--ledger-signing-key-seed-hex", + runtime.ledger_attestation_signing_seed_hex, + ] + try: + process = subprocess.Popen( + command, + stdin=subprocess.PIPE, + stdout=subprocess.PIPE, + # The daemon inherits the coordinator's stderr so a crash's + # diagnostics land in the journal instead of a per-request + # capture file that dies with the request. + stderr=None, + close_fds=True, + ) + except OSError as exc: + raise _ServeBuilderUnavailable( + f"audit-builder daemon spawn failed: {exc}" + ) from exc + # The start (and any pending restart it satisfies) is recorded as + # soon as the worker exists, exactly like the one-shot path: a + # handshake-phase death or protocol mismatch must not leave the + # lifecycle totals describing fewer workers than were launched. + restarted = False + with runtime._job_build_scheduler_lock: + if runtime._job_build_worker_restart_pending: + runtime.job_build_worker_counts["restarts"] += 1 + runtime._job_build_worker_restart_pending = False + restarted = True + runtime.job_build_worker_counts["starts"] += 1 + if restarted: + with runtime._tip_refresh_metrics_lock: + runtime.tip_refresh_worker_restarts += 1 + client = _ServeBuilderClient(process=process) + try: + assert process.stdin is not None and process.stdout is not None + os.set_blocking(process.stdin.fileno(), False) + os.set_blocking(process.stdout.fileno(), False) + handshake_line = self._serve_builder_read_line( + client, + deadline, + cancellation, + None, + ) + handshake = json.loads(handshake_line) + if ( + not isinstance(handshake, dict) + or handshake.get("event") != "handshake" + or handshake.get("protocol") + != PRISM_SERVE_BUILDER_PROTOCOL_VERSION + ): + raise _ServeBuilderUnavailable( + "audit-builder daemon announced an unsupported protocol" + ) + except _ServeBuilderUnavailable: + self._record_live_serve_builder_termination(client) + client.close() + raise + except (OSError, ValueError, AttributeError) as exc: + self._record_live_serve_builder_termination(client) + client.close() + raise _ServeBuilderUnavailable( + f"audit-builder daemon handshake failed: {exc}" + ) from exc + except BaseException: + self._record_live_serve_builder_termination(client) + client.close() + raise + return client + + def _serve_builder_read_line( + self, + client: _ServeBuilderClient, + deadline: float, + cancellation: CancellationPort | None, + build_control: BundleBuildControlPort | None, + ) -> bytes: + runtime = self._runtime + stdout = client.process.stdout + assert stdout is not None + file_descriptor = stdout.fileno() + # Bytes before this offset were already scanned for the newline; the + # buffer only grows until the line is cut, so a long response line + # is scanned once rather than once per received chunk. + scanned = 0 + with _PipeReadinessWaiter(file_descriptor, selectors.EVENT_READ) as waiter: + while True: + newline_index = client.stdout_buffer.find(b"\n", scanned) + if newline_index >= 0: + line = bytes(client.stdout_buffer[:newline_index]) + del client.stdout_buffer[: newline_index + 1] + return line + scanned = len(client.stdout_buffer) + if cancellation is not None: + cancellation.raise_if_cancelled("serve builder response") + if ( + build_control is not None + and build_control.cancel_event.is_set() + ): + raise self._superseded_error( + "audit-builder daemon request was canceled after supersession" + ) + if time.monotonic() >= deadline: + # Not counted as a worker failure here: the fallback + # one-shot runs against this same exhausted deadline and + # its own timeout path records the failure exactly once. + raise _ServeBuilderUnavailable( + "audit-builder daemon timed out" + ) + try: + chunk = os.read( + file_descriptor, + PRISM_BUILDER_PIPE_READ_CHUNK_BYTES, + ) + except InterruptedError: + # Not a wait: retried straight after the control checks. + continue + except BlockingIOError: + waiter.wait(deadline) + continue + except OSError as exc: + raise _ServeBuilderUnavailable( + f"audit-builder daemon read failed: {exc}" + ) from exc + if not chunk: + if ( + build_control is not None + and build_control.cancel_event.is_set() + ): + raise self._superseded_error( + "audit-builder daemon was terminated after supersession" + ) + with runtime._job_build_scheduler_lock: + runtime.job_build_worker_counts["crashes"] += 1 + runtime._job_build_worker_restart_pending = True + raise _ServeBuilderUnavailable( + "audit-builder daemon exited mid-request" + ) + client.stdout_buffer += chunk + + def _serve_builder_read_exact( + self, + client: _ServeBuilderClient, + byte_count: int, + deadline: float, + cancellation: CancellationPort | None, + build_control: BundleBuildControlPort | None, + ) -> bytes: + """Read exactly byte_count raw bytes following a response line. + + prepare_window responses append a raw canonical-items section after + the JSON envelope; the section length rides in the envelope, so this + drains the line reader's buffer first and then reads from the pipe + with the same deadline/cancellation discipline as line reads. + """ + runtime = self._runtime + stdout = client.process.stdout + assert stdout is not None + file_descriptor = stdout.fileno() + out = bytearray() + if client.stdout_buffer: + take = min(byte_count, len(client.stdout_buffer)) + out += client.stdout_buffer[:take] + del client.stdout_buffer[:take] + with _PipeReadinessWaiter(file_descriptor, selectors.EVENT_READ) as waiter: + while len(out) < byte_count: + if cancellation is not None: + cancellation.raise_if_cancelled("serve builder response") + if ( + build_control is not None + and build_control.cancel_event.is_set() + ): + raise self._superseded_error( + "audit-builder daemon request was canceled after supersession" + ) + if time.monotonic() >= deadline: + raise _ServeBuilderUnavailable( + "audit-builder daemon timed out" + ) + try: + chunk = os.read( + file_descriptor, + min( + PRISM_BUILDER_PIPE_READ_CHUNK_BYTES, + byte_count - len(out), + ), + ) + except InterruptedError: + continue + except BlockingIOError: + waiter.wait(deadline) + continue + except OSError as exc: + raise _ServeBuilderUnavailable( + f"audit-builder daemon read failed: {exc}" + ) from exc + if not chunk: + with runtime._job_build_scheduler_lock: + runtime.job_build_worker_counts["crashes"] += 1 + runtime._job_build_worker_restart_pending = True + raise _ServeBuilderUnavailable( + "audit-builder daemon exited mid-request" + ) + out += chunk + return bytes(out) + + def _serve_builder_write( + self, + client: _ServeBuilderClient, + data: bytes, + deadline: float, + cancellation: CancellationPort | None, + build_control: BundleBuildControlPort | None, + ) -> int: + stdin = client.process.stdin + assert stdin is not None + file_descriptor = stdin.fileno() + remaining = memoryview(data) + written_total = 0 + with _PipeReadinessWaiter(file_descriptor, selectors.EVENT_WRITE) as waiter: + while remaining: + if cancellation is not None: + cancellation.raise_if_cancelled("serve builder request") + if ( + build_control is not None + and build_control.cancel_event.is_set() + ): + raise self._superseded_error( + "audit-builder daemon request was canceled after supersession" + ) + if time.monotonic() >= deadline: + # Not counted as a worker failure here: the fallback + # one-shot runs against this same exhausted deadline and + # its own timeout path records the failure exactly once. + raise _ServeBuilderUnavailable( + "audit-builder daemon timed out" + ) + try: + written = os.write(file_descriptor, remaining) + except InterruptedError: + continue + except BlockingIOError: + waiter.wait(deadline) + continue + except BrokenPipeError as exc: + if ( + build_control is not None + and build_control.cancel_event.is_set() + ): + raise self._superseded_error( + "audit-builder daemon was terminated after supersession" + ) from exc + raise _ServeBuilderUnavailable( + "audit-builder daemon input pipe closed" + ) from exc + except OSError as exc: + raise _ServeBuilderUnavailable( + f"audit-builder daemon write failed: {exc}" + ) from exc + if written <= 0: + raise _ServeBuilderUnavailable( + "audit-builder daemon input pipe closed" + ) + written_total += written + remaining = remaining[written:] + return written_total + + def _serve_builder_splice_spool( + self, + client: _ServeBuilderClient, + share_serialization: _ShareWindowSerialization, + spool_file: Any, + spool_size: int, + deadline: float, + cancellation: CancellationPort | None, + build_control: BundleBuildControlPort | None, + ) -> int: + stdin = client.process.stdin + assert stdin is not None + stdin_fd = stdin.fileno() + spool_fd = spool_file.fileno() + offset = 0 + # EAGAIN here always means the pipe is full (the spool is a regular + # file), so the wait is for the daemon's stdin to drain. A splice + # that raised moved nothing -- offset_src is explicit -- so retrying + # from the same offset never repeats a byte, and a splice that moved + # part of its request advances the offset by exactly that much. + with _PipeReadinessWaiter(stdin_fd, selectors.EVENT_WRITE) as waiter: + while offset < spool_size: + if cancellation is not None: + cancellation.raise_if_cancelled("serve builder request") + if ( + build_control is not None + and build_control.cancel_event.is_set() + ): + raise self._superseded_error( + "audit-builder daemon request was canceled after supersession" + ) + if time.monotonic() >= deadline: + # Not counted as a worker failure here: the fallback + # one-shot runs against this same exhausted deadline and + # its own timeout path records the failure exactly once. + raise _ServeBuilderUnavailable( + "audit-builder daemon timed out" + ) + try: + moved = os.splice( + spool_fd, + stdin_fd, + min(PRISM_SPOOL_SPLICE_CHUNK_BYTES, spool_size - offset), + offset_src=offset, + ) + except InterruptedError: + continue + except BlockingIOError: + waiter.wait(deadline) + continue + except BrokenPipeError as exc: + # The daemon's stdin closed; the spool itself is fine. + if ( + build_control is not None + and build_control.cancel_event.is_set() + ): + raise self._superseded_error( + "audit-builder daemon was terminated after supersession" + ) from exc + raise _ServeBuilderUnavailable( + "audit-builder daemon input pipe closed" + ) from exc + except OSError as exc: + if ( + build_control is not None + and build_control.cancel_event.is_set() + ): + raise self._superseded_error( + "audit-builder daemon was terminated after supersession" + ) from exc + # The spool itself cannot stream. Poison it so the + # one-shot fallback -- and every later build -- writes + # the cached in-memory fragments instead of retrying the + # same transfer and failing mid-stream. Whatever prefix + # already reached the daemon is never re-sent: the + # anomaly retires this daemon with the request marked + # incomplete. + share_serialization.mark_spool_failed() + raise _ServeBuilderUnavailable( + f"audit-builder daemon spool transfer failed: {exc}" + ) from exc + if moved <= 0: + raise _ServeBuilderUnavailable( + "audit-builder daemon input pipe closed" + ) + offset += moved + return offset + + def _serve_builder_request_locked( + self, + client: _ServeBuilderClient, + *, + deadline: float, + payload: dict[str, object], + shares: list[dict[str, object]], + precomposed: Callable[[], tuple[str, ...]], + share_serialization: _ShareWindowSerialization, + cancellation: CancellationPort | None, + build_control: BundleBuildControlPort | None, + record_phase_metrics: bool, + append_invalidation_epoch: int | None = None, + ) -> dict[str, Any]: + """One JSONL round trip; raises _ServeBuilderUnavailable on anomaly.""" + runtime = self._runtime + share_snapshot_sha256 = share_serialization.share_snapshot_sha256 + request_fields = dict(payload) + request_fields["window_key"] = { + "share_snapshot_sha256": share_snapshot_sha256, + } + if append_invalidation_epoch is not None: + # Lets the daemon drop prepared window state from an older + # append-invalidation epoch; see the prepare_window protocol. + request_fields["append_invalidation_epoch"] = int( + append_invalidation_epoch + ) + prefix = json.dumps(request_fields, separators=(",", ":")).encode( + "utf-8" + ) + phases = runtime._job_build_phases() + input_bytes = 0 + input_serialization_elapsed = 0.0 + + def send_request(upload_window: bool) -> None: + nonlocal input_bytes, input_serialization_elapsed + serialization_started = time.monotonic() + client.request_incomplete = True + try: + if not upload_window: + input_bytes += self._serve_builder_write( + client, + prefix + b"\n", + deadline, + cancellation, + build_control, + ) + client.request_incomplete = False + return + input_bytes += self._serve_builder_write( + client, + prefix[:-1], + deadline, + cancellation, + build_control, + ) + with self._routed_window_mirror_divergence(): + lease = ( + share_serialization.acquire_spooled_tail(shares) + if hasattr(os, "splice") + else None + ) + if lease is not None: + try: + spool_file, spool_size = lease + input_bytes += self._serve_builder_splice_spool( + client, + share_serialization, + spool_file, + spool_size, + deadline, + cancellation, + build_control, + ) + finally: + share_serialization.release_spooled_tail() + else: + # Spooling unavailable: the cached tail streams from + # memory chunk by chunk, each one bounded, so this + # fallback never re-encodes the window whole either. + for chunk in precomposed(): + input_bytes += self._serve_builder_write( + client, + chunk.encode("utf-8"), + deadline, + cancellation, + build_control, + ) + input_bytes += self._serve_builder_write( + client, + b"\n", + deadline, + cancellation, + build_control, + ) + client.request_incomplete = False + finally: + elapsed = time.monotonic() - serialization_started + phases["input_serialization"] = ( + phases.get("input_serialization", 0.0) + elapsed + ) + # Accumulated across a possible needs_window re-send and + # observed once per build so serialization_copy stays + # comparable with the one-shot transport. + input_serialization_elapsed += elapsed + + def read_response() -> tuple[dict[str, Any], bytes]: + worker_started = time.monotonic() + line = self._serve_builder_read_line( + client, + deadline, + cancellation, + build_control, + ) + phases["worker"] = phases.get("worker", 0.0) + ( + time.monotonic() - worker_started + ) + output_started = time.monotonic() + try: + value = json.loads(line) + except ValueError as exc: + raise _ServeBuilderUnavailable( + f"audit-builder daemon response was malformed: {exc}" + ) from exc + finally: + phases["output_serialization"] = phases.get( + "output_serialization", + 0.0, + ) + (time.monotonic() - output_started) + if not isinstance(value, dict): + raise _ServeBuilderUnavailable( + "audit-builder daemon response was not an object" + ) + return value, line + + upload_window = share_snapshot_sha256 not in client.uploaded_windows + send_request(upload_window) + response, response_line = read_response() + if ( + response.get("ok") is not True + and bool(response.get("needs_window")) + and not upload_window + ): + # The daemon evicted this window (respawn or generation churn the + # coordinator's mirror missed); repeat the request with the + # window riding along. + upload_window = True + send_request(True) + response, response_line = read_response() + if response.get("ok") is not True: + raise _ServeBuilderUnavailable( + "audit-builder daemon error: " + f"{response.get('error', 'unknown')}" + ) + summary = response.get("summary") + if not isinstance(summary, dict): + raise _ServeBuilderUnavailable( + "audit-builder daemon returned a malformed summary" + ) + # Every successful response promotes the window in the mirror, hits + # included: the daemon moves a hit entry to most-recent position, and + # a mirror that only tracked uploads would evict a different key and + # either re-upload a still-cached window or bounce on needs_window. + client.note_uploaded_window(share_snapshot_sha256) + if upload_window: + with self._serve_builder_metrics_lock: + self.serve_builder_counts["window_uploads"] += 1 + window_cache = response.get("window_cache") + if isinstance(window_cache, dict): + outcome = "hits" if bool(window_cache.get("hit")) else "misses" + with self._serve_builder_metrics_lock: + self.serve_builder_window_cache_counts[outcome] += 1 + if record_phase_metrics: + runtime._observe_tip_refresh_build_phase( + "serialization_copy", + input_serialization_elapsed, + ) + metrics_value = response.get("metrics") + if isinstance(metrics_value, dict): + try: + self._observe_builder_phase_metrics(metrics_value) + except (TypeError, ValueError): + # Metrics are diagnostic only. A malformed timing payload + # must never invalidate an otherwise valid summary. + pass + runtime._record_tip_refresh_ipc_bytes("input", input_bytes) + runtime._record_tip_refresh_ipc_bytes("output", len(response_line)) + return summary + + def _build_audit_bundle_via_serve_builder( + self, + *, + payload: dict[str, object], + shares: list[dict[str, object]], + precomposed: Callable[[], tuple[str, ...]], + share_serialization: _ShareWindowSerialization, + cancellation: CancellationPort | None, + record_phase_metrics: bool, + deadline: float, + append_invalidation_epoch: int | None = None, + ) -> dict[str, Any] | None: + """Build through the persistent daemon; None means use one-shot. + + Any daemon anomaly -- spawn failure, handshake or protocol mismatch, + crash, timeout, malformed response -- retires the daemon and returns + None so the caller's one-shot path runs unchanged against the same + deadline (whatever budget the daemon consumed stays consumed). + Cancellation and supersession raise exactly like the one-shot path + instead of falling back. + """ + runtime = self._runtime + if not env_bool("PRISM_BUILDER_SERVE", "1"): + return None + runtime._ensure_job_cache_state() + if not getattr(runtime, "signing_seed_hex", None) or not getattr( + runtime, + "ledger_attestation_signing_seed_hex", + None, + ): + return None + if not self._serve_builder_lock.acquire(blocking=False): + # A concurrent build owns the daemon. One-shot is cheaper than + # queueing behind a multi-second build. + return None + try: + if self._serve_builder_shutdown: + return None + build_control = getattr( + runtime._job_build_phase_local, + "bundle_build_control", + None, + ) + if not isinstance(build_control, self._build_control_type): + build_control = None + client: _ServeBuilderClient | None = None + try: + client = self._serve_builder + if client is not None and client.process.poll() is not None: + # The daemon died between requests (its own crash or an + # external kill). Record the lifecycle transition before + # retiring it so the immediate respawn below reads as a + # crash-and-restart rather than an ordinary start. + with runtime._job_build_scheduler_lock: + runtime.job_build_worker_counts["crashes"] += 1 + runtime._job_build_worker_restart_pending = True + self._retire_serve_builder_locked() + client = None + if client is None: + client = self._spawn_serve_builder_locked( + deadline, + cancellation, + ) + self._serve_builder = client + with self._serve_builder_metrics_lock: + self.serve_builder_counts["spawns"] += 1 + if build_control is not None: + runtime._register_job_bundle_process( + build_control, + client.process, # type: ignore[arg-type] + ) + try: + summary = self._serve_builder_request_locked( + client, + deadline=deadline, + payload=payload, + shares=shares, + precomposed=precomposed, + share_serialization=share_serialization, + cancellation=cancellation, + build_control=build_control, + record_phase_metrics=record_phase_metrics, + append_invalidation_epoch=append_invalidation_epoch, + ) + finally: + if build_control is not None: + runtime._unregister_job_bundle_process( + build_control, + client.process, + ) + except self._cancellation_error_types: + # The daemon stream is indeterminate mid-request; retire it + # so the replacement build starts clean. + self._retire_serve_builder_locked() + with runtime._job_build_scheduler_lock: + runtime.job_build_worker_counts["terminations"] += 1 + runtime._job_build_worker_restart_pending = True + raise + except _ServeBuilderUnavailable: + self._record_live_serve_builder_termination( + self._serve_builder + ) + self._retire_serve_builder_locked() + with self._serve_builder_metrics_lock: + self.serve_builder_counts["fallbacks"] += 1 + return None + except (OSError, ValueError): + self._record_live_serve_builder_termination( + self._serve_builder + ) + self._retire_serve_builder_locked() + with self._serve_builder_metrics_lock: + self.serve_builder_counts["fallbacks"] += 1 + return None + except BaseException: + # Anything else -- a payload the serializer refused, a + # divergence raised while composing the window fragments -- + # is not a daemon anomaly, so it propagates. But if it + # escaped a half-written request, the daemon's stdin now + # holds an unterminated line and the next request would + # concatenate onto it, so the daemon goes with the error + # rather than answering "malformed serve request" to some + # innocent later build. + if client is not None and client.request_incomplete: + self._record_live_serve_builder_termination(client) + self._retire_serve_builder_locked() + with self._serve_builder_metrics_lock: + self.serve_builder_counts["fallbacks"] += 1 + raise + else: + with self._serve_builder_metrics_lock: + self.serve_builder_counts["requests"] += 1 + return summary + finally: + self._serve_builder_lock.release() + + def prepare_payout_window( + self, + *, + mode: str, + records_json: list[dict[str, object]], + anchor_job_issued_at_ms: int, + append_invalidation_epoch: int, + window_weight: int | None = None, + page_size: int | None = None, + base_digest: str | None = None, + wait_for_daemon: bool = True, + ) -> PreparedWindowOutcome | None: + """One prepare_window round trip; None means use in-process Python. + + Called once per window generation from artifact materialization, + never per build. Any daemon anomaly -- spawn failure, handshake or + protocol mismatch, crash, timeout, malformed response -- retires the + daemon and returns None so the caller's in-process pipeline runs + unchanged. The ``needs_full``/``fallback``/``fold_invalid``/ + ``out_of_range`` outcomes are ordinary control flow and never retire + the daemon; ``out_of_range`` in particular used to arrive as a + malformed-request error and retire a healthy daemon once per + materialization for as long as the window held the value. + + ``wait_for_daemon=False`` refuses to queue at all: urgent + materializations (the accepted-block preview) must never wait behind + another build's round trip, so a contended daemon answers ``busy`` + immediately. Either way, losing the race for the daemon is not a + daemon fault and is never recorded as one. + """ + runtime = self._runtime + if not env_bool("PRISM_BUILDER_SERVE", "1"): + return None + runtime._ensure_job_cache_state() + if not getattr(runtime, "signing_seed_hex", None) or not getattr( + runtime, + "ledger_attestation_signing_seed_hex", + None, + ): + return None + request_fields: dict[str, object] = { + "request": "prepare_window", + "mode": mode, + "append_invalidation_epoch": int(append_invalidation_epoch), + "anchor_job_issued_at_ms": int(anchor_job_issued_at_ms), + "records": records_json, + } + if window_weight is not None: + request_fields["window_weight"] = int(window_weight) + if page_size is not None: + request_fields["page_size"] = int(page_size) + if base_digest is not None: + request_fields["base_digest"] = base_digest + budget = float( + getattr( + runtime, + "bundle_build_timeout_seconds", + DEFAULT_PRISM_BUNDLE_BUILD_TIMEOUT_SECONDS, + ) + ) + # Unlike builds (which prefer one-shot over queueing), a routine + # window preparation has no cheaper transport to defect to, so it + # will wait for the daemon -- but only for its own budget, and the + # exchange's deadline is stamped AFTER the wait. Charging a build's + # runtime against this preparation's budget would make the first + # write inside the lock read as a daemon timeout, and the anomaly + # handler would SIGKILL a daemon that had done nothing wrong. + if wait_for_daemon: + acquired = self._serve_builder_lock.acquire(timeout=budget) + else: + acquired = self._serve_builder_lock.acquire(blocking=False) + if not acquired: + with self._serve_builder_metrics_lock: + self.serve_builder_counts["window_prepare_contended"] = ( + self.serve_builder_counts.get( + "window_prepare_contended", + 0, + ) + + 1 + ) + return PreparedWindowOutcome( + status="busy", + error="audit-builder daemon is owned by another build", + ) + deadline = time.monotonic() + budget + try: + if self._serve_builder_shutdown: + return None + try: + client = self._serve_builder + if client is not None and client.process.poll() is not None: + with runtime._job_build_scheduler_lock: + runtime.job_build_worker_counts["crashes"] += 1 + runtime._job_build_worker_restart_pending = True + self._retire_serve_builder_locked() + client = None + if client is None: + client = self._spawn_serve_builder_locked(deadline, None) + self._serve_builder = client + with self._serve_builder_metrics_lock: + self.serve_builder_counts["spawns"] += 1 + # The request streams in bounded chunks: each envelope + # member is one small json.dumps and the record array is + # encoded batch by batch, so a 200k-record preparation never + # holds the GIL for one whole-window encode (#236). Same + # bytes as one json.dumps of the fields. From the first byte + # until the terminating newline the daemon's stdin holds a + # partial line; anything that escapes in between retires + # the daemon below rather than letting the next request + # concatenate onto the fragment. + client.request_incomplete = True + for chunk in _iter_prepare_window_request_chunks( + request_fields + ): + self._serve_builder_write( + client, + chunk, + deadline, + None, + None, + ) + client.request_incomplete = False + envelope_line = self._serve_builder_read_line( + client, + deadline, + None, + None, + ) + envelope = json.loads(envelope_line) + if ( + not isinstance(envelope, dict) + or envelope.get("request") != "prepare_window" + ): + raise _ServeBuilderUnavailable( + "audit-builder daemon returned a malformed" + " prepare_window response" + ) + if envelope.get("ok") is not True: + rejection = envelope.get("rejection") + if not isinstance(rejection, str): + rejection = None + if bool(envelope.get("needs_full")): + outcome = PreparedWindowOutcome( + status="needs_full", + error=str(envelope.get("error", "")), + ) + elif bool(envelope.get("fallback")): + outcome = PreparedWindowOutcome( + status="fallback", + error=str(envelope.get("error", "")), + rejection=rejection, + ) + elif bool(envelope.get("fold_invalid")): + outcome = PreparedWindowOutcome( + status="fold_invalid", + error=str(envelope.get("error", "")), + rejection=rejection, + ) + elif bool(envelope.get("out_of_range")): + # The daemon declined a value outside its declared + # widths. It is healthy and keeps its state; count + # the decline so the degraded materializations are + # visible, and let the in-process fold take over. + with self._serve_builder_metrics_lock: + self.serve_builder_counts[ + "window_prepare_out_of_range" + ] = ( + self.serve_builder_counts.get( + "window_prepare_out_of_range", + 0, + ) + + 1 + ) + outcome = PreparedWindowOutcome( + status="out_of_range", + error=str(envelope.get("error", "")), + ) + else: + raise _ServeBuilderUnavailable( + "audit-builder daemon prepare_window error: " + f"{envelope.get('error', 'unknown')}" + ) + else: + digest = envelope.get("share_snapshot_sha256") + if not isinstance(digest, str) or not digest: + raise _ServeBuilderUnavailable( + "audit-builder daemon prepare_window response" + " carries no digest" + ) + window_items: bytes | None = None + retained_drop_bytes = 0 + appended_items = b"" + if "window_items_len" in envelope: + window_items = self._serve_builder_read_exact( + client, + int(envelope["window_items_len"]), + deadline, + None, + None, + ) + else: + retained_drop_bytes = int( + envelope["retained_drop_bytes"] + ) + appended_items = self._serve_builder_read_exact( + client, + int(envelope["appended_items_len"]), + deadline, + None, + None, + ) + # The raw section's terminating newline. + self._serve_builder_read_exact( + client, + 1, + deadline, + None, + None, + ) + # The prepared window doubles as the build cache entry; + # note it in the LRU mirror so the next build for this + # digest skips its window upload entirely. + client.note_uploaded_window(digest) + outcome = PreparedWindowOutcome( + status="prepared", + share_snapshot_sha256=digest, + record_count=int(envelope.get("record_count", 0)), + added_rows=int(envelope.get("added_rows", 0)), + expired_rows=int(envelope.get("expired_rows", 0)), + touched_pages=int(envelope.get("touched_pages", 0)), + window_items=window_items, + retained_drop_bytes=retained_drop_bytes, + appended_items=appended_items, + ) + except _ServeBuilderUnavailable: + self._record_live_serve_builder_termination( + self._serve_builder + ) + self._retire_serve_builder_locked() + with self._serve_builder_metrics_lock: + self.serve_builder_counts["fallbacks"] += 1 + return None + except (OSError, ValueError, KeyError, TypeError): + self._record_live_serve_builder_termination( + self._serve_builder + ) + self._retire_serve_builder_locked() + with self._serve_builder_metrics_lock: + self.serve_builder_counts["fallbacks"] += 1 + return None + except BaseException: + # Not a daemon anomaly, so it propagates -- but if it + # escaped a half-written request the daemon's stdin holds + # an unterminated line, and the daemon goes with the error + # rather than answering "malformed serve request" to some + # innocent later build. + client = self._serve_builder + if client is not None and client.request_incomplete: + self._record_live_serve_builder_termination(client) + self._retire_serve_builder_locked() + with self._serve_builder_metrics_lock: + self.serve_builder_counts["fallbacks"] += 1 + raise + else: + with self._serve_builder_metrics_lock: + self.serve_builder_counts["window_prepares"] = ( + self.serve_builder_counts.get("window_prepares", 0) + 1 + ) + return outcome + finally: + self._serve_builder_lock.release() + + def build_audit_bundle( + self, + *, + shares: list[dict[str, object]], + found_block: dict[str, object], + prior_balances: list[dict[str, object]], + coinbase_script_sig_suffix_hex: str, + witness_merkle_leaves_hex: list[str] | None = None, + ctv_fee_parent_hash: str | None = None, + canonical_output_path: Path | None = None, + canonical_output_parent_fd: int | None = None, + canonical_output_adopter: Callable[[Path, os.stat_result], None] | None = None, + summary_only: bool = False, + payout_policy: dict[str, object] | None = None, + ctv_settlement: dict[str, object] | None = None, + cancellation: CancellationPort | None = None, + share_serialization: _ShareWindowSerialization | None = None, + append_invalidation_epoch: int | None = None, + ) -> dict[str, Any]: + runtime = self._runtime + runtime._ensure_job_cache_state() + runtime._ensure_tip_refresh_state() + if cancellation is not None: + runtime._job_build_checkpoint("serialization", cancellation) + payload: dict[str, object] = { + "found_block": found_block, + "prior_balances": prior_balances, + "payout_policy": ( + runtime.prism_payout_policy() + if payout_policy is None + else payout_policy + ), + "coinbase_script_sig_suffix_hex": coinbase_script_sig_suffix_hex, + "witness_merkle_leaves_hex": witness_merkle_leaves_hex or [], + } + job_build_phase_local = getattr(runtime, "_job_build_phase_local", None) + record_phase_metrics = bool( + getattr(job_build_phase_local, "tip_refresh_metrics", False) + ) + precomposed: Callable[[], tuple[str, ...]] | None = None + if summary_only: + artifact_started = time.monotonic() + if ( + share_serialization is not None + and share_serialization.share_count == len(shares) + ): + # The compact conversion and its encoded fragments are pure + # per-generation functions of the share window, resolved + # lazily: a daemon build that hits prepared/uploaded window + # state never sends the window, so walking the shares (which + # may be a lazily-parsed daemon mirror) is deferred until a + # transport actually needs the compact bytes. + serialization = share_serialization + + def precompose() -> tuple[str, ...]: + with self._routed_window_mirror_divergence(): + return serialization.compact_tail_chunks(shares) + + precomposed = precompose + else: + with self._routed_window_mirror_divergence(): + identities, compact_shares = _compact_share_payload(shares) + payload["compact_share_identities"] = identities + payload["compact_shares"] = compact_shares + if record_phase_metrics: + runtime._observe_tip_refresh_build_phase( + "serialization_copy", + time.monotonic() - artifact_started, + ) + else: + # Canonical builds serialize the full share window inline. Any + # replayable sequence -- a list, the daemon mirror, a page-backed + # window -- is walked record by record while the builder input + # is written; it is never copied into a list first (#255). + if isinstance(shares, (str, bytes, bytearray)) or not isinstance( + shares, + (Sequence, Iterable), + ): + raise TypeError("shares must be a sequence of share records") + payload["shares"] = shares + if ctv_settlement is None and payout_policy is None: + ctv_settlement = runtime.prism_ctv_settlement_config( + block_height=int(found_block["block_height"]), + parent_hash=ctv_fee_parent_hash, + ) + if ctv_settlement is not None: + payload["ctv_settlement"] = ctv_settlement + if canonical_output_path is not None and summary_only: + raise ValueError("canonical output and job summary output are mutually exclusive") + # One deadline covers the whole build regardless of transport: a + # daemon anomaly falls back to the one-shot subprocess with only the + # remaining budget, so a hung daemon cannot double the configured + # limit while progress health already treats one limit as stuck. + build_deadline = time.monotonic() + float( + getattr( + runtime, + "bundle_build_timeout_seconds", + DEFAULT_PRISM_BUNDLE_BUILD_TIMEOUT_SECONDS, + ) + ) + if ( + summary_only + and canonical_output_path is None + and precomposed is not None + and share_serialization is not None + ): + # The persistent builder serves the artifact-backed summary path, + # whose window identity and cached fragments it can key on. Every + # anomaly falls back to the one-shot subprocess below. + served = self._build_audit_bundle_via_serve_builder( + payload=payload, + shares=shares, + precomposed=precomposed, + share_serialization=share_serialization, + cancellation=cancellation, + record_phase_metrics=record_phase_metrics, + deadline=build_deadline, + append_invalidation_epoch=append_invalidation_epoch, + ) + if served is not None: + return served + command = self._tool_command("qbit-prism-build-audit-bundle") + [ + "--input", + "-", + "--signing-key-seed-hex", + runtime.signing_seed_hex, + "--ledger-signing-key-seed-hex", + runtime.ledger_attestation_signing_seed_hex, + ] + command.append("--job-summary-output" if summary_only else "--canonical-output") + if record_phase_metrics: + command.append("--phase-metrics") + if ( + canonical_output_path is not None + and canonical_output_parent_fd is None + ): + canonical_output_path.parent.mkdir(parents=True, exist_ok=True) + succeeded = False + created_output = False + created_output_fd: int | None = None + try: + with ExitStack() as stack: + if canonical_output_path is None: + output = stack.enter_context( + tempfile.TemporaryFile(mode="w+", encoding="utf-8") + ) + else: + if canonical_output_parent_fd is None: + output = stack.enter_context( + canonical_output_path.open("x+", encoding="utf-8") + ) + else: + output_fd = os.open( + canonical_output_path.name, + os.O_RDWR + | os.O_CREAT + | os.O_EXCL + | getattr(os, "O_NOFOLLOW", 0), + 0o666, + dir_fd=canonical_output_parent_fd, + ) + output = stack.enter_context( + os.fdopen(output_fd, "w+", encoding="utf-8") + ) + created_output = True + # Keep the created inode allocated until cleanup compares + # it with the published name. This closes the Linux inode- + # reuse window after an adopter replaces the path. + created_output_fd = os.dup(output.fileno()) + stderr = stack.enter_context( + tempfile.TemporaryFile(mode="w+", encoding="utf-8") + ) + process = subprocess.Popen( + command, + stdin=subprocess.PIPE, + stdout=output, + stderr=stderr, + text=True, + encoding="utf-8", + close_fds=True, + ) + build_control = getattr( + job_build_phase_local, + "bundle_build_control", + None, + ) + if isinstance(build_control, self._build_control_type): + runtime._register_job_bundle_process(build_control, process) + with runtime._job_build_scheduler_lock: + if runtime._job_build_worker_restart_pending: + runtime.job_build_worker_counts["restarts"] += 1 + runtime._job_build_worker_restart_pending = False + runtime.job_build_worker_counts["starts"] += 1 + assert process.stdin is not None + input_byte_count = 0 + worker_deadline = build_deadline + killed_process_wait_seconds = max( + 0.001, + float( + getattr( + runtime, + "job_build_cancel_grace_seconds", + DEFAULT_PRISM_JOB_BUILD_CANCEL_GRACE_SECONDS, + ) + ), + ) + coordinator = runtime + build_control_type = self._build_control_type + superseded_error = self._superseded_error + + class _CancelableInput: + def __init__(self, stream: Any) -> None: + self.stream = stream + self.waiter: _PipeReadinessWaiter | None = None + try: + file_descriptor = int(stream.fileno()) + except (AttributeError, OSError, TypeError, ValueError): + # Lightweight process fakes used by embedders and + # tests do not necessarily expose an OS pipe. + self.file_descriptor: int | None = None + else: + os.set_blocking(file_descriptor, False) + self.file_descriptor = file_descriptor + self.waiter = _PipeReadinessWaiter( + file_descriptor, + selectors.EVENT_WRITE, + ) + + def wait_writable(self) -> None: + """Block until the child's stdin can drain. + + Bounded by the build deadline and the cancellation + cadence exactly like the daemon transport; the + caller re-runs check_cancelled before retrying. + """ + if self.waiter is not None: + self.waiter.wait(worker_deadline) + + def close(self) -> None: + waiter, self.waiter = self.waiter, None + if waiter is not None: + waiter.close() + + def check_cancelled(self) -> None: + if cancellation is not None: + cancellation.raise_if_cancelled( + "builder input serialization" + ) + if ( + isinstance(build_control, build_control_type) + and build_control.cancel_event.is_set() + ): + raise superseded_error( + "audit-builder input was canceled after supersession" + ) + if time.monotonic() >= worker_deadline: + with coordinator._tip_refresh_metrics_lock: + coordinator.tip_refresh_worker_failures += 1 + raise RuntimeError( + "qbit-prism-build-audit-bundle timed out" + ) + + def write(self, value: str | bytes) -> int: + nonlocal input_byte_count + self.check_cancelled() + if self.file_descriptor is None: + text = ( + value.decode("utf-8") + if isinstance(value, (bytes, bytearray)) + else value + ) + written = int(self.stream.write(text)) + input_byte_count += len(text[:written].encode("utf-8")) + return written + encoded = ( + bytes(value) + if isinstance(value, (bytes, bytearray)) + else value.encode("utf-8") + ) + remaining = memoryview(encoded) + while remaining: + self.check_cancelled() + try: + written = os.write( + self.file_descriptor, + remaining, + ) + except InterruptedError: + continue + except BlockingIOError: + self.wait_writable() + continue + if written <= 0: + raise BrokenPipeError( + "audit-builder input pipe closed" + ) + input_byte_count += written + remaining = remaining[written:] + return len(value) + + serialization_started = time.monotonic() + spool_lease: tuple[Any, int] | None = None + sink: _CancelableInput | None = None + try: + sink = _CancelableInput(process.stdin) + + def write_precomposed_tail() -> None: + assert precomposed is not None + for chunk in precomposed(): + sink.write(chunk) + + if ( + precomposed is not None + and share_serialization is not None + and sink.file_descriptor is not None + and hasattr(os, "splice") + ): + spool_lease = share_serialization.acquire_spooled_tail( + shares + ) + if precomposed is not None: + # Per-build fields are encoded fresh; the dominant + # share-window fragments stream from the spool file + # written once per generation (kernel moves the bytes + # into the pipe), falling back to the cached strings + # whenever no spool is available. + prefix = json.dumps(payload, separators=(",", ":")) + sink.write(prefix[:-1]) + spool_transferred = False + if spool_lease is not None: + spool_file, spool_size = spool_lease + spool_fd = spool_file.fileno() + offset = 0 + while offset < spool_size: + sink.check_cancelled() + try: + moved = os.splice( + spool_fd, + sink.file_descriptor, + min( + PRISM_SPOOL_SPLICE_CHUNK_BYTES, + spool_size - offset, + ), + offset_src=offset, + ) + except InterruptedError: + continue + except BlockingIOError: + sink.wait_writable() + continue + except BrokenPipeError: + # The child's stdin closed; the spool + # itself is fine and the surrounding + # pipe-error handling owns this. + raise + except OSError: + # The spool cannot stream (unsupported + # filesystem or an I/O error); the cached + # in-memory fragments stay authoritative + # for this and every later build. + share_serialization.mark_spool_failed() + if offset: + # Part of the tail already reached the + # pipe; this build cannot be repaired + # by re-writing it from memory. + raise + break + if moved <= 0: + raise BrokenPipeError( + "audit-builder input pipe closed" + ) + offset += moved + input_byte_count += moved + spool_transferred = ( + spool_size > 0 and offset >= spool_size + ) + if not spool_transferred: + write_precomposed_tail() + else: + # The share arrays (the canonical build's full + # window, or an artifact-less summary's compact + # form) stream in bounded batches and every other + # member is one small json.dumps: the same bytes as + # one json.dump of the payload, without the + # pure-Python encoder's per-token writes or a + # whole-window C call. A daemon-mirror window is + # walked here, so a refuted mirror is routed exactly + # as the retired list() copy was. + with self._routed_window_mirror_divergence(): + for chunk in _iter_build_input_chunks(payload): + sink.write(chunk) + except BrokenPipeError: + # Prefer the builder's diagnostic below. + pass + except BaseException as exc: + try: + process.kill() + except ProcessLookupError: + pass + try: + process.wait(timeout=killed_process_wait_seconds) + except subprocess.TimeoutExpired: + pass + if isinstance(exc, self._cancellation_error_types): + with runtime._job_build_scheduler_lock: + runtime.job_build_worker_counts["terminations"] += 1 + runtime._job_build_worker_restart_pending = True + raise + finally: + if sink is not None: + sink.close() + if spool_lease is not None and share_serialization is not None: + share_serialization.release_spooled_tail() + phases = runtime._job_build_phases() + phases["input_serialization"] = phases.get( + "input_serialization", + 0.0, + ) + (time.monotonic() - serialization_started) + try: + process.stdin.close() + except (BlockingIOError, BrokenPipeError): + pass + if record_phase_metrics: + runtime._observe_tip_refresh_build_phase( + "serialization_copy", + time.monotonic() - serialization_started, + ) + runtime._record_tip_refresh_ipc_bytes("input", input_byte_count) + worker_started = time.monotonic() + terminated = False + returncode: int | None = None + if cancellation is None and not hasattr(process, "poll"): + returncode = process.wait( + timeout=max(0.001, worker_deadline - time.monotonic()) + ) + else: + while returncode is None: + returncode = process.poll() + if returncode is not None: + break + cancelled_by_control = ( + isinstance(build_control, self._build_control_type) + and build_control.cancel_event.is_set() + ) + cancelled_by_request = ( + cancellation is not None and cancellation.is_set() + ) + if cancelled_by_control or cancelled_by_request: + terminated = True + process.terminate() + try: + returncode = process.wait( + timeout=max( + 0.0, + float( + getattr( + runtime, + "job_build_cancel_grace_seconds", + DEFAULT_PRISM_JOB_BUILD_CANCEL_GRACE_SECONDS, + ) + ), + ) + ) + except subprocess.TimeoutExpired: + process.kill() + try: + returncode = process.wait( + timeout=killed_process_wait_seconds + ) + except subprocess.TimeoutExpired: + returncode = process.poll() + with runtime._job_build_scheduler_lock: + runtime.job_build_worker_counts["terminations"] += 1 + runtime._job_build_worker_restart_pending = True + break + if time.monotonic() >= worker_deadline: + process.kill() + try: + returncode = process.wait( + timeout=killed_process_wait_seconds + ) + except subprocess.TimeoutExpired: + returncode = process.poll() + with runtime._tip_refresh_metrics_lock: + runtime.tip_refresh_worker_failures += 1 + raise RuntimeError( + "qbit-prism-build-audit-bundle timed out" + ) + time.sleep(min(0.02, PRISM_TIP_REFRESH_ADMISSION_POLL_SECONDS)) + phases["worker"] = phases.get("worker", 0.0) + ( + time.monotonic() - worker_started + ) + if terminated and cancellation is not None: + if cancellation.is_set(): + cancellation.raise_if_cancelled("builder worker") + if ( + terminated + and isinstance(build_control, self._build_control_type) + and build_control.cancel_event.is_set() + ): + raise self._superseded_error( + "audit-builder subprocess was canceled after supersession" + ) + stderr.seek(0) + error_text = stderr.read() + if returncode != 0: + with runtime._job_build_scheduler_lock: + runtime.job_build_worker_counts["crashes"] += 1 + runtime._job_build_worker_restart_pending = True + if record_phase_metrics: + with runtime._tip_refresh_metrics_lock: + runtime.tip_refresh_worker_failures += 1 + raise RuntimeError( + f"qbit-prism-build-audit-bundle failed: {error_text}" + ) + if ( + isinstance(build_control, self._build_control_type) + and build_control.cancel_event.is_set() + ): + raise self._superseded_error( + "audit-builder result completed after supersession" + ) + output.flush() + output_size = os.fstat(output.fileno()).st_size + if record_phase_metrics: + runtime._record_tip_refresh_ipc_bytes("output", output_size) + for line in error_text.splitlines(): + if not line.startswith(PRISM_BUILDER_PHASE_METRICS_PREFIX): + continue + raw_metrics = line.removeprefix( + PRISM_BUILDER_PHASE_METRICS_PREFIX + ) + try: + metrics = json.loads(raw_metrics) + if isinstance(metrics, dict): + self._observe_builder_phase_metrics(metrics) + except (TypeError, ValueError, json.JSONDecodeError): + # Metrics are diagnostic only. A malformed timing + # line must never invalidate an otherwise valid + # signed bundle. + pass + if canonical_output_path is not None: + os.fsync(output.fileno()) + output.seek(0) + output_started = time.monotonic() + if cancellation is not None: + cancellation.raise_if_cancelled("builder output serialization") + bundle: Any + if canonical_output_path is not None: + # The canonical artifact on disk is the authority. It is + # scanned once through bounded windows into a view whose + # window-sized arrays (``shares`` and the reward + # manifest's counted shares) stay on disk and replay on + # demand; nothing window-sized is decoded into Python + # objects in this process (#255). The descriptor is + # duplicated so the view outlives this stack; the file + # may be unlinked later without invalidating the view. + scan_checkpoint: Callable[[], None] | None = None + if cancellation is not None: + scan_checkpoint = lambda: cancellation.raise_if_cancelled( # noqa: E731 + "builder output scan" + ) + bundle = CanonicalAuditBundleView.scan( + os.dup(output.fileno()), + path=canonical_output_path, + cancellation=scan_checkpoint, + ) + scan_stats = bundle.scan_stats + if scan_stats.oversized_values: + # One JSON value is the decoder's unit, so a record + # above the soft limit was decoded whole. Valid work + # is never refused for its size; the event is made + # visible instead. + print( + "prism coordinator: canonical audit artifact " + f"scan decoded {scan_stats.oversized_values} " + "oversized JSON value(s) " + f"max_value_chars={scan_stats.max_value_chars} " + "window_high_water_chars=" + f"{scan_stats.window_high_water_chars} " + f"path={canonical_output_path.name}", + flush=True, + ) + else: + # Embedders without a candidate artifact receive the + # decoded document, as before; the finalization path + # always names an artifact. + bundle = json.load(output) + phases["output_serialization"] = phases.get( + "output_serialization", + 0.0, + ) + (time.monotonic() - output_started) + try: + if cancellation is not None: + cancellation.raise_if_cancelled("builder verification") + if ( + canonical_output_path is not None + and canonical_output_adopter is not None + ): + canonical_output_adopter( + canonical_output_path, + os.fstat(output.fileno()), + ) + except BaseException: + close_view = getattr(bundle, "close", None) + if callable(close_view): + close_view() + raise + succeeded = True + return bundle + finally: + try: + if ( + canonical_output_path is not None + and created_output + and not succeeded + ): + created_output_identity = ( + None + if created_output_fd is None + else os.fstat(created_output_fd) + ) + self._remove_created_output_if_same( + canonical_output_path, + created_output_identity, + parent_fd=canonical_output_parent_fd, + ) + finally: + if created_output_fd is not None: + os.close(created_output_fd) + + @staticmethod + def _remove_created_output_if_same( + path: Path, + identity: os.stat_result | None, + *, + parent_fd: int | None = None, + ) -> None: + if identity is None: + return + try: + current = ( + path.lstat() + if parent_fd is None + else os.stat(path.name, dir_fd=parent_fd, follow_symlinks=False) + ) + except FileNotFoundError: + return + if not stat.S_ISREG(current.st_mode): + # Cleanup has no portable no-replace restore primitive for a + # directory/symlink. Reject it before moving an unowned path. + return + if not BundleCompiler._output_identity_matches(identity, current): + # Preserve a replacement already visible at the canonical name. + return + quarantine = path.with_name(f".{path.name}.{uuid.uuid4().hex}.cleanup") + try: + if parent_fd is None: + os.replace(path, quarantine) + else: + os.replace( + path.name, + quarantine.name, + src_dir_fd=parent_fd, + dst_dir_fd=parent_fd, + ) + except FileNotFoundError: + return + moved = ( + quarantine.lstat() + if parent_fd is None + else os.stat( + quarantine.name, + dir_fd=parent_fd, + follow_symlinks=False, + ) + ) + if BundleCompiler._output_identity_matches(identity, moved): + if parent_fd is None: + quarantine.unlink() + else: + os.unlink(quarantine.name, dir_fd=parent_fd) + return + try: + if parent_fd is None: + os.link(quarantine, path, follow_symlinks=False) + else: + os.link( + quarantine.name, + path.name, + src_dir_fd=parent_fd, + dst_dir_fd=parent_fd, + follow_symlinks=False, + ) + except OSError: + # A second replacement won after quarantine. Keep both artifacts; + # an unowned non-regular replacement is likewise preserved there. + return + if parent_fd is None: + quarantine.unlink() + else: + os.unlink(quarantine.name, dir_fd=parent_fd) + + @staticmethod + def _output_identity_matches( + identity: os.stat_result, + value: os.stat_result, + ) -> bool: + return bool( + value.st_dev == identity.st_dev + and value.st_ino == identity.st_ino + and value.st_mode == identity.st_mode + and value.st_size == identity.st_size + and value.st_mtime_ns == identity.st_mtime_ns + and stat.S_ISREG(value.st_mode) + ) + + +__all__ = [ + "BundleBuildControlPort", + "BundleCompiler", + "BundleCompilerRuntime", + "CancellationPort", + "PreparedWindowOutcome", + "PRISM_BUILDER_INPUT_ARRAY_KEYS", + "PRISM_BUILDER_PHASE_METRICS_PREFIX", + "PRISM_SERVE_BUILDER_PROTOCOL_VERSION", + "PRISM_SERVE_BUILDER_WINDOW_CACHE_ENTRIES", + "PRISM_SPOOL_SPLICE_CHUNK_BYTES", + "PRISM_TIP_REFRESH_ADMISSION_POLL_SECONDS", + "_ServeBuilderClient", + "_ServeBuilderUnavailable", + "_ShareWindowSerialization", + "_iter_build_input_chunks", + "_iter_share_window_items", + "_compact_share_payload", + "_compact_share_tail_chunks", + "_iter_prepare_window_request_chunks", + "_share_window_spool_file", + "canonical_bundle_bytes", +] diff --git a/lab/prism/candidate_codec.py b/lab/prism/candidate_codec.py new file mode 100644 index 000000000..a171c760a --- /dev/null +++ b/lab/prism/candidate_codec.py @@ -0,0 +1,2238 @@ +#!/usr/bin/env python3 +"""Bounded codec and immutable body handles for durable block candidates. + +Issue #255. A block-candidate intent carries the whole payout window under +``shares_json`` (375,000 shares and ~232 MB of JSON at the stress size), and +the historical path materialized it three times in the lease-bearing +coordinator: ``list(context.shares_json)``, a whole ``json.dumps`` to +validate it, and a second whole ``json.dumps`` to inline it as a ``jsonb`` +literal. Each of those is one C call that holds the GIL for the duration +while the writer-lease monitor waits to run. + +This module replaces the whole-body calls with a streaming encoder whose +individual C calls are bounded (at most :data:`CODEC_BATCH_RECORDS` records +or roughly :data:`CODEC_BATCH_TARGET_BYTES` of output per ``json.dumps``; +strings over :data:`CODEC_STRING_SLICE_CHARS` code points are escaped in +slices) and hands the result to storage as fixed +:data:`CANDIDATE_BODY_CHUNK_BYTES` byte chunks with per-chunk digests. + +Nothing the encoder keeps scales with the body. It emits three kinds of +events -- chunks, per-field *spans* (one per large top-level field) and +*page* entries (one per bounded batch of a large array) -- and retains only +a scalar :class:`CandidateBodyManifest`. A reader gets the same facts back +from storage in bounded pages and keeps the page index on disk next to the +spool file, bisecting it with ``pread`` instead of holding a list. + +Byte identity is the contract. The body bytes are exactly the historical +v1 identity JSON -- ``json.dumps(block_candidate_identity(intent), +sort_keys=True, separators=(",", ":"))`` with the pending acknowledgment +stamp neutralized -- so ``candidate_sha256`` is unchanged for every existing +row and every existing oracle. Chunk boundaries, spans, pages and the +storage version never enter that digest. + +Three body shapes exist: + +* :class:`SourceCandidateBody` re-encodes on demand from the immutable + share sequence the live job pinned (a page-backed window, wave B's lazy + daemon sequence, or a plain list). It retains no encoded copy. +* :class:`SpoolCandidateBody` is a durable replay body hydrated to a local + spool file in bounded pages; :class:`SpoolJsonArraySequence` decodes one + bounded page per ``json.loads`` from it and digests the share array + without decoding it at all. +* :class:`PreparedCandidateIntent` wraps either one behind the historical + mapping interface (``intent["shares_json"]`` is the immutable sequence, + never a list copy) so dict-facing callers keep working. + +This module is a leaf: standard library only. +""" + +from __future__ import annotations + +import hashlib +import io +import json +import os +import re +import struct +import threading +import weakref +from collections.abc import Iterable, Iterator, Mapping, MutableMapping, Sequence +from dataclasses import dataclass, replace +from typing import Any, Callable + +# Storage format version of a chunked body. Distinct from the v1 mining +# intent schema string (``qbit.prism.block-candidate-intent.v1``), which is +# neither renamed nor re-hashed. +CANDIDATE_BODY_STORAGE_VERSION = 2 +LEGACY_CANDIDATE_STORAGE_VERSION = 1 +# Fixed byte size of every body chunk but the last. Storage, transport and +# the reader all enforce it independently. +CANDIDATE_BODY_CHUNK_BYTES = 256 * 1024 +# Records per ``json.dumps`` call on the fast path, and the output size the +# adaptive batch aims for. +CODEC_BATCH_RECORDS = 256 +CODEC_BATCH_TARGET_BYTES = 256 * 1024 +# Output of one batch above which its items are re-encoded one by one +# through the incremental path (a giant field inside a record). +CODEC_BATCH_OVERSIZED_BYTES = 4 * 1024 * 1024 +# Ordinary per-value ``json.dumps``/``json.loads`` input ceiling. Longer +# strings are escaped and decoded slice by slice; a top-level field whose +# encoding is larger than this becomes a *large field* with its own span. +# Nothing is rejected for size alone. +CODEC_STRING_SLICE_CHARS = 4096 +CODEC_FAST_PATH_BYTES = 64 * 1024 +# Historical reader ceilings, retained for callers that import them. The +# reader no longer decodes a page or a record whole above +# ``candidate_spool_view.SPOOL_VIEW_DECODE_BYTES``: an oversized record is a +# lazy view, and no record size is rejected. +CODEC_PAGE_DECODE_BYTES = 16 * 1024 * 1024 +CODEC_WALK_RECORD_MAX_BYTES = 256 * 1024 * 1024 +# Records decoded per page when a body carries no page index for a span. +SPOOL_WALK_BATCH_RECORDS = 256 +# Bytes read from a spool per refill while walking records. +SPOOL_READ_BYTES = 256 * 1024 +# Page entries written per index statement / read per index page. +INDEX_PAGE_ENTRIES = 256 +# Span rows read per statement. +INDEX_SPAN_ROWS = 64 + +CANDIDATE_SHARES_KEY = "shares_json" +CANDIDATE_INTENT_SCHEMA = "qbit.prism.block-candidate-intent.v1" + +_COMPACT_SEPARATORS = (",", ":") +# One on-disk page index record: body offset, record index (u64, u64). +_INDEX_RECORD = struct.Struct(" int: + return len(self.data) + + +@dataclass(frozen=True) +class FieldSpan: + """Byte span of one large top-level field inside the body. + + ``kind`` is ``array`` (decoded page by page), ``string`` (decoded slice + by slice) or ``value`` (any other JSON value, decoded in one call bounded + by the field's own size). ``item_count`` is the array length for arrays. + ``pages_exact`` is False when the page entries are boundary *hints* + (verbatim daemon bytes scanned rather than encoded record by record); a + reader validates every hint by decoding and falls back to the walker. + """ + + field: str + kind: str + start: int + end: int + item_count: int + page_count: int + pages_exact: bool + + def to_json(self) -> dict[str, Any]: + return { + "field": self.field, + "kind": self.kind, + "start": self.start, + "end": self.end, + "item_count": self.item_count, + "page_count": self.page_count, + "pages_exact": self.pages_exact, + } + + @classmethod + def from_json(cls, payload: Mapping[str, Any]) -> FieldSpan: + span = cls( + field=str(payload["field"]), + kind=str(payload["kind"]), + start=int(payload["start"]), + end=int(payload["end"]), + item_count=int(payload.get("item_count") or 0), + page_count=int(payload.get("page_count") or 0), + pages_exact=bool(payload.get("pages_exact", True)), + ) + if span.kind not in {"array", "string", "value"}: + raise CandidateBodyIntegrityError("body span has an unknown kind") + if span.start < 0 or span.end < span.start or span.item_count < 0 or span.page_count < 0: + raise CandidateBodyIntegrityError("body span is out of range") + return span + + +@dataclass(frozen=True) +class PageEntry: + """Where one bounded page of a large array starts inside the body.""" + + field: str + ordinal: int + offset: int + record_index: int + + +@dataclass(frozen=True) +class CandidateBodyManifest: + """Scalar facts about a body; nothing here scales with its size.""" + + storage_version: int + candidate_sha256: str + byte_count: int + chunk_count: int + chunk_bytes: int + share_count: int + shares_offset: int + shares_end: int + span_count: int + page_count: int + + def to_json(self) -> dict[str, Any]: + return { + "storage_version": self.storage_version, + "candidate_sha256": self.candidate_sha256, + "byte_count": self.byte_count, + "chunk_count": self.chunk_count, + "chunk_bytes": self.chunk_bytes, + "share_count": self.share_count, + "shares_offset": self.shares_offset, + "shares_end": self.shares_end, + "span_count": self.span_count, + "page_count": self.page_count, + } + + @classmethod + def from_json(cls, payload: Mapping[str, Any]) -> CandidateBodyManifest: + try: + manifest = cls( + storage_version=int(payload["storage_version"]), + candidate_sha256=str(payload["candidate_sha256"]), + byte_count=int(payload["byte_count"]), + chunk_count=int(payload["chunk_count"]), + chunk_bytes=int(payload["chunk_bytes"]), + share_count=int(payload["share_count"]), + shares_offset=int(payload["shares_offset"]), + shares_end=int(payload["shares_end"]), + span_count=int(payload.get("span_count") or 0), + page_count=int(payload.get("page_count") or 0), + ) + except (KeyError, TypeError, ValueError) as exc: + raise CandidateBodyIntegrityError("manifest is malformed") from exc + manifest.validate() + return manifest + + def validate(self) -> None: + if self.storage_version != CANDIDATE_BODY_STORAGE_VERSION: + raise CandidateBodyIntegrityError( + f"unsupported candidate body storage version {self.storage_version}" + ) + if self.chunk_bytes <= 0 or self.chunk_bytes > CANDIDATE_BODY_CHUNK_BYTES: + raise CandidateBodyIntegrityError("manifest chunk size is out of range") + if self.byte_count < 0 or self.chunk_count < 0: + raise CandidateBodyIntegrityError("manifest counts are negative") + if self.chunk_count != chunk_ordinals_for(self.byte_count, self.chunk_bytes): + raise CandidateBodyIntegrityError( + "manifest chunk count disagrees with its byte count" + ) + # A body without a share array (a candidate-only intent, as some + # tools and tests persist) carries an empty span at offset 0. + if not (0 <= self.shares_offset <= self.shares_end <= self.byte_count): + raise CandidateBodyIntegrityError("manifest share span is out of range") + if self.shares_offset == self.shares_end and self.share_count: + raise CandidateBodyIntegrityError("manifest declares shares without a span") + if len(self.candidate_sha256) != 64: + raise CandidateBodyIntegrityError("manifest digest is malformed") + if self.span_count < 0 or self.page_count < 0: + raise CandidateBodyIntegrityError("manifest index counts are negative") + + +def chunk_ordinals_for(byte_count: int, chunk_bytes: int = CANDIDATE_BODY_CHUNK_BYTES) -> int: + return -(-int(byte_count) // int(chunk_bytes)) + + +# -------------------------------------------------------------------------- +# jsonb compatibility +# -------------------------------------------------------------------------- + +# PostgreSQL ``jsonb`` rejects two things CPython's encoder happily emits: +# the ``\\u0000`` escape (text cannot hold NUL) and an unpaired UTF-16 +# surrogate escape. The historical route inlined the intent as a jsonb +# literal, so both failed on the server before any credit; a chunked body +# never reaches a jsonb parser, so the same rejection happens here, at +# preparation. Escape-aware: a backslash run of even length before ``\u`` +# is escaped backslashes, not an escape start. +_ESCAPED_NUL_RE = re.compile(r"(? None: + """Refuse encoded JSON text PostgreSQL's jsonb parser would refuse.""" + if "\\u0000" in text and _ESCAPED_NUL_RE.search(text): + raise CandidateCodecError( + "unsupported Unicode escape sequence: \\u0000 cannot be converted to text" + ) + if "\\ud" not in text: + return + expected_low_at = -1 + for match in _SURROGATE_ESCAPE_RE.finditer(text): + unit = match.group(1) + escape_start = match.end() - 6 + if unit[1] in "89ab": + if expected_low_at == escape_start: + raise CandidateCodecError("Unicode high surrogate must not follow a high surrogate") + following = text[match.end() : match.end() + 6] + if not _LOW_SURROGATE_ESCAPE_RE.fullmatch(following): + raise CandidateCodecError("Unicode high surrogate must not be alone") + expected_low_at = match.end() + elif expected_low_at == escape_start: + expected_low_at = -1 + else: + raise CandidateCodecError("Unicode low surrogate must follow a high surrogate") + + +def _reject_jsonb_incompatible_str(value: str) -> None: + """The Python-string form of :func:`_reject_jsonb_incompatible_text`.""" + if "\x00" in value: + raise CandidateCodecError( + "unsupported Unicode escape sequence: \\u0000 cannot be converted to text" + ) + if _LONE_SURROGATE_RE.search(value): + raise CandidateCodecError("unpaired UTF-16 surrogate cannot be stored as jsonb") + + +# -------------------------------------------------------------------------- +# streaming encoder +# -------------------------------------------------------------------------- + + +class _ChunkSink: + """Turns encoded text into fixed-size chunks and a running digest.""" + + __slots__ = ( + "_chunk_bytes", + "_on_chunk", + "_buffer", + "_buffered", + "offset", + "_digest", + "_ordinal", + ) + + def __init__( + self, + chunk_bytes: int, + on_chunk: Callable[[BodyChunk], None] | None, + ) -> None: + self._chunk_bytes = int(chunk_bytes) + self._on_chunk = on_chunk + self._buffer: list[bytes] = [] + self._buffered = 0 + self.offset = 0 + self._digest = hashlib.sha256() + self._ordinal = 0 + + @property + def chunk_count(self) -> int: + return self._ordinal + + def write_text(self, text: str) -> None: + if text: + self.write_bytes(text.encode("utf-8")) + + def write_bytes(self, data: bytes) -> None: + if not data: + return + self._digest.update(data) + self.offset += len(data) + # Fill the current chunk to exactly its size before emitting, and + # never re-join a remainder: a verbatim page of any size is consumed + # in one linear pass. + view = memoryview(data) + while len(view): + room = self._chunk_bytes - self._buffered + piece = view[:room] + self._buffer.append(bytes(piece)) + self._buffered += len(piece) + view = view[room:] + if self._buffered >= self._chunk_bytes: + self._emit() + + def _emit(self) -> None: + chunk_data = self._buffer[0] if len(self._buffer) == 1 else b"".join(self._buffer) + self._buffer = [] + self._buffered = 0 + chunk = BodyChunk( + ordinal=self._ordinal, + data=chunk_data, + sha256=hashlib.sha256(chunk_data).hexdigest(), + ) + self._ordinal += 1 + if self._on_chunk is not None: + self._on_chunk(chunk) + + def finish(self) -> str: + if self._buffered: + self._emit() + return self._digest.hexdigest() + + +class _IndexEvents: + """Collects span/page events for the field being encoded, bounded. + + Page entries are handed to ``on_page`` in batches of at most + :data:`INDEX_PAGE_ENTRIES`; the span is handed to ``on_span`` when the + field ends. Only the current batch is ever held. + """ + + __slots__ = ("on_span", "on_page", "_field", "_pages", "_page_ordinal", "_start", "span_count", "page_count", "_kind") + + def __init__( + self, + on_span: Callable[[FieldSpan], None] | None, + on_page: Callable[[list[PageEntry]], None] | None, + ) -> None: + self.on_span = on_span + self.on_page = on_page + self._field = "" + self._kind = "" + self._pages: list[PageEntry] = [] + self._page_ordinal = 0 + self._start = 0 + self.span_count = 0 + self.page_count = 0 + + def begin(self, field_name: str, kind: str, start: int) -> None: + self._field = field_name + self._kind = kind + self._start = start + self._pages = [] + self._page_ordinal = 0 + + def page(self, offset: int, record_index: int) -> None: + self._pages.append( + PageEntry( + field=self._field, + ordinal=self._page_ordinal, + offset=offset, + record_index=record_index, + ) + ) + self._page_ordinal += 1 + if len(self._pages) >= INDEX_PAGE_ENTRIES: + self._flush_pages() + + def _flush_pages(self) -> None: + if not self._pages: + return + if self.on_page is not None: + self.on_page(self._pages) + self.page_count += len(self._pages) + self._pages = [] + + def end(self, end: int, *, item_count: int, pages_exact: bool, emit: bool) -> None: + if not emit: + # A small field: no span, no pages -- decoded from the skeleton. + self._pages = [] + self._page_ordinal = 0 + return + self._flush_pages() + span = FieldSpan( + field=self._field, + kind=self._kind, + start=self._start, + end=end, + item_count=item_count, + page_count=self._page_ordinal, + pages_exact=pages_exact, + ) + self.span_count += 1 + if self.on_span is not None: + self.on_span(span) + + +def _dumps_scalar(value: Any) -> str: + # ``allow_nan=False`` is the deliberate difference from the historical + # ``json.dumps``: NaN/Infinity used to pass the client-side validation + # and then fail as a jsonb literal on the server, after the share was + # already inside the fenced statement. A chunked body never reaches a + # jsonb parser, so the rejection has to happen here, before credit. + text = json.dumps(value, allow_nan=False, sort_keys=True, separators=_COMPACT_SEPARATORS) + if isinstance(value, (dict, list, tuple)): + _reject_jsonb_incompatible_text(text) + return text + + +def _dumps_key(key: str) -> str: + _reject_jsonb_incompatible_str(key) + return json.dumps(key) + + +def _write_string(value: str, sink: _ChunkSink) -> None: + if len(value) <= CODEC_STRING_SLICE_CHARS: + _reject_jsonb_incompatible_str(value) + sink.write_text(json.dumps(value)) + return + # ``ensure_ascii`` escapes code point by code point (a non-BMP character + # is one code point and becomes one surrogate pair), so slicing by code + # point and escaping each slice reproduces the whole-string escape. + sink.write_text('"') + start = 0 + while start < len(value): + end = min(len(value), start + CODEC_STRING_SLICE_CHARS) + if end < len(value) and "\ud800" <= value[end - 1] <= "\udbff" and "\udc00" <= value[end] <= "\udfff": + end += 1 + piece = value[start:end] + _reject_jsonb_incompatible_str(piece) + sink.write_text(json.dumps(piece)[1:-1]) + start = end + sink.write_text('"') + + +def _write_value(value: Any, sink: _ChunkSink) -> None: + """Write ``value`` exactly as ``json.dumps(value, sort_keys=True)`` would.""" + encoded_chunks = getattr(value, "iter_byte_chunks", None) + if callable(encoded_chunks): + for chunk in encoded_chunks(): + sink.write_bytes(chunk) + return + if value is None or value is True or value is False: + sink.write_text(_dumps_scalar(value)) + return + if isinstance(value, str): + _write_string(value, sink) + return + if isinstance(value, (int, float)): + sink.write_text(_dumps_scalar(value)) + return + if isinstance(value, dict): + keys = list(value) + if not keys: + sink.write_text("{}") + return + # Sort before coercion, matching CPython's failure on incomparable + # key types. Even a numeric-key extension can contain a giant value; + # it must use the same streamed value encoder as ordinary fields. + keys.sort() + sink.write_text("{") + first = True + for key in keys: + if not first: + sink.write_text(",") + if isinstance(key, str): + encoded_key = key + elif key is None or isinstance(key, (bool, int, float)): + encoded_key = _dumps_scalar(key) + else: + raise TypeError(f"keys must be str, int, float, bool or None, not {type(key).__name__}") + _write_string(encoded_key, sink) + sink.write_text(":") + first = False + _write_value(value[key], sink) + sink.write_text("}") + return + if isinstance(value, (list, tuple)): + _write_array(value, sink, on_page=None) + return + raise TypeError(f"Object of type {type(value).__name__} is not JSON serializable") + + +def _batch_text(batch: list[Any]) -> str: + # ``_dumps_scalar`` already ran the jsonb-compatibility scan over the + # batch text (whole records, so escape runs never straddle a boundary). + return _dumps_scalar(batch)[1:-1] + + +def _encoded_size_bound(value: Any, budget: int) -> int | None: + """Conservatively bound an ordinary value *before* calling the C codec. + + Stop inspecting once the byte budget is spent. In particular, an oversized + string needs only its length checked; measuring it must not encode it first. + Twelve ASCII bytes per code point covers ensure_ascii's surrogate pairs. + """ + if budget < 2: + return None + if isinstance(value, str): + size = 2 + 12 * len(value) + elif value is None or isinstance(value, bool): + size = 5 + elif isinstance(value, int): + size = 2 + value.bit_length() // 3 + elif isinstance(value, float): + size = 32 + elif isinstance(value, (dict, list, tuple)): + size = 2 + if isinstance(value, dict): + for key, item in value.items(): + if not isinstance(key, str): + return None + key_size = 2 + 12 * len(key) + item_size = _encoded_size_bound(item, budget - size - key_size - 2) + if item_size is None: + return None + size += key_size + item_size + 2 + else: + for item in value: + item_size = _encoded_size_bound(item, budget - size - 1) + if item_size is None: + return None + size += item_size + 1 + else: + return None + return size if size <= budget else None + + +def _write_array( + items: Iterable[Any], + sink: _ChunkSink, + *, + on_page: Callable[[int, int], None] | None, +) -> int: + """Write one JSON array in bounded batches; returns the item count. + + ``on_page(offset, record_index)`` fires at the start of every batch. + """ + sink.write_text("[") + batch: list[Any] = [] + batch_bytes = 0 + first = True + record_index = 0 + + def flush(batch: list[Any]) -> None: + nonlocal first, record_index + if not first: + sink.write_text(",") + first = False + if on_page is not None: + on_page(sink.offset, record_index) + text = _batch_text(batch) + sink.write_text(text) + record_index += len(batch) + + for item in items: + size = _encoded_size_bound(item, CODEC_BATCH_TARGET_BYTES - 2) + if batch and (size is None or batch_bytes + size + 1 > CODEC_BATCH_TARGET_BYTES - 2): + flush(batch) + batch = [] + batch_bytes = 0 + if size is None: + if not first: + sink.write_text(",") + first = False + if on_page is not None: + on_page(sink.offset, record_index) + _write_value(item, sink) + record_index += 1 + continue + batch.append(item) + batch_bytes += size + 1 + if len(batch) >= CODEC_BATCH_RECORDS: + flush(batch) + batch = [] + batch_bytes = 0 + if batch: + flush(batch) + sink.write_text("]") + return record_index + + +def _canonical_share_pages(shares: Any) -> Iterator[tuple[bytes, int | None]] | None: + """Already-encoded share pages, when the sequence can supply them. + + Returns ``None`` for a sequence that has to be iterated and re-encoded. + Three shapes are consumed verbatim: + + * the ledger's page-backed window (``pages[*].canonical_json_items``), + whose bytes CPython's own encoder produced from the very records the + sequence iterates; + * a spool body written by this codec (``canonical_share_pages``); + * the daemon mirror (``canonical_items`` + ``record_count``), whose + items stream is the canonical encoding the mirror reconciled against + its record count and digest at construction. Copying it is the only + way to stage a daemon-owned window without the sequence's whole + parse (its ``__iter__`` on the current tree materializes every + record); wave B of #254 keeps the same bytes contract. Record + boundaries are not known for this shape, so its page entries are + scanned *hints* that a reader validates by decoding. + """ + pages_method = getattr(shares, "canonical_share_pages", None) + if callable(pages_method): + return pages_method() + canonical_items = getattr(shares, "canonical_items", None) + if isinstance(canonical_items, bytes) and isinstance( + getattr(shares, "record_count", None), int + ): + + def iter_daemon_items() -> Iterator[tuple[bytes, int | None]]: + # One raw slice: the sink consumes it linearly and the digest + # update releases the GIL for a buffer this size. No copy of + # the stream is made beyond the chunk pieces themselves. + yield canonical_items, None + + return iter_daemon_items() + pages = getattr(shares, "pages", None) + if isinstance(pages, tuple) and all( + isinstance(getattr(page, "canonical_json_items", None), bytes) + and isinstance(getattr(page, "prism_json_records", None), tuple) + for page in pages + ): + + def iter_pages() -> Iterator[tuple[bytes, int | None]]: + for page in pages: + if page.canonical_json_items: + yield page.canonical_json_items, len(page.prism_json_records) + + return iter_pages() + return None + + +_JSON_STRUCTURE = re.compile(rb'["\\{}\[\],]') + + +class _RecordPageScanner: + """Locate exact top-level separators using bounded byte scans. + + Quoted text and nested arrays/objects cannot become page boundaries. + State persists across source chunks, including a split escape pair. + """ + + def __init__(self, on_page: Callable[[int, int], None]) -> None: + self.on_page = on_page + self.started = False + self.in_string = False + self.depth = 0 + self.escape_at = -2 + self.record_index = 0 + self.page_index = 0 + self.page_start = 0 + + def feed(self, data: bytes, base_offset: int) -> None: + if not data: + return + if not self.started: + self.started = True + self.page_start = base_offset + self.on_page(base_offset, 0) + for offset in range(0, len(data), SPOOL_READ_BYTES): + piece = data[offset:offset + SPOOL_READ_BYTES] + for match in _JSON_STRUCTURE.finditer(piece): + position = base_offset + offset + match.start() + token = match[0] + if self.in_string: + if position == self.escape_at + 1: + self.escape_at = -2 + elif token == b"\\": + self.escape_at = position + elif token == b'"': + self.in_string = False + elif token == b'"': + self.in_string = True + elif token in (b"{", b"["): + self.depth += 1 + elif token in (b"}", b"]"): + self.depth -= 1 + if self.depth < 0: + raise CandidateBodyIntegrityError("array source has unmatched delimiters") + elif token == b"," and self.depth == 0: + self.record_index += 1 + if (self.record_index - self.page_index >= CODEC_BATCH_RECORDS + or position + 1 - self.page_start >= CODEC_BATCH_TARGET_BYTES): + self.page_start = position + 1 + self.page_index = self.record_index + self.on_page(self.page_start, self.page_index) + + def finish(self) -> int: + if self.in_string or self.depth: + raise CandidateBodyIntegrityError("array source ends inside a record") + return self.record_index + int(self.started) + + +def _write_sequence(sequence: Any, sink: _ChunkSink, events: _IndexEvents) -> tuple[int, bool]: + """Copy canonical items with exact page boundaries and no decoded mirror.""" + pages = _canonical_share_pages(sequence) + if pages is None: + return _write_array(iter(sequence), sink, on_page=events.page), True + sink.write_text("[") + scanner = _RecordPageScanner(events.page) + first = True + raw_mode = False + for data, record_count in pages: + if not data: + continue + if not first and (record_count is not None or not raw_mode): + scanner.feed(b",", sink.offset) + sink.write_text(",") + raw_mode = record_count is None + first = False + scanner.feed(data, sink.offset) + sink.write_bytes(data) + count = scanner.finish() + if count != len(sequence): + raise CandidateBodyIntegrityError("array pages disagree with the sequence length") + sink.write_text("]") + return count, True + + +def _write_shares(shares: Any, sink: _ChunkSink, events: _IndexEvents) -> tuple[int, int, int, bool]: + """Write the share array; returns (start, end, count, pages_exact).""" + start = sink.offset + events.begin(CANDIDATE_SHARES_KEY, "array", start) + count, pages_exact = _write_sequence(shares, sink, events) + end = sink.offset + events.end(end, item_count=count, pages_exact=pages_exact, emit=True) + return start, end, count, pages_exact + + +def _write_top_level_field(key: str, value: Any, sink: _ChunkSink, events: _IndexEvents) -> None: + """Write one non-share top-level field, spanning it when it is large. + + Any sequence that is not a string is an array here -- a list, a tuple, + or a spool-backed array adapter from a hydrated body -- so a hydrated + intent re-stages without materializing its large fields. + """ + start = sink.offset + if isinstance(value, Sequence) and not isinstance(value, (str, bytes, bytearray)): + events.begin(key, "array", start) + count, pages_exact = _write_sequence(value, sink, events) + end = sink.offset + events.end(end, item_count=count, pages_exact=pages_exact, emit=(end - start) > CODEC_FAST_PATH_BYTES) + return + if isinstance(value, str): + events.begin(key, "string", start) + _write_string(value, sink) + end = sink.offset + events.end(end, item_count=0, pages_exact=True, emit=(end - start) > CODEC_FAST_PATH_BYTES) + return + events.begin(key, "value", start) + _write_value(value, sink) + end = sink.offset + events.end(end, item_count=0, pages_exact=True, emit=(end - start) > CODEC_FAST_PATH_BYTES) + + +def encode_identity_body( + identity: Mapping[str, Any], + shares: Any, + *, + chunk_bytes: int = CANDIDATE_BODY_CHUNK_BYTES, + on_chunk: Callable[[BodyChunk], None] | None = None, + on_span: Callable[[FieldSpan], None] | None = None, + on_page: Callable[[list[PageEntry]], None] | None = None, +) -> CandidateBodyManifest: + """Encode ``{**identity, "shares_json": shares}`` in bounded pieces. + + ``identity`` is the small-field mapping with the acknowledgment stamp + already neutralized. The concatenated chunks handed to ``on_chunk`` are + byte-identical to ``json.dumps({**identity, "shares_json": + list(shares)}, sort_keys=True, separators=(",", ":"))``. ``on_span`` and + ``on_page`` receive the index events; the returned manifest is scalar. + """ + if shares is not None and ( + isinstance(shares, (str, bytes)) or not isinstance(shares, Sequence) + ): + raise CandidateCodecError("shares_json must be a sequence of records") + keys = list(identity) + if CANDIDATE_SHARES_KEY in keys: + raise CandidateCodecError("identity fields must not carry shares_json") + if any(not isinstance(key, str) for key in keys): + raise CandidateCodecError("candidate intent keys must be strings") + if shares is not None: + keys.append(CANDIDATE_SHARES_KEY) + keys.sort() + sink = _ChunkSink(chunk_bytes, on_chunk) + events = _IndexEvents(on_span, on_page) + sink.write_text("{") + first = True + shares_start = shares_end = 0 + share_count = 0 + for key in keys: + sink.write_text(("" if first else ",") + _dumps_key(key) + ":") + first = False + if key == CANDIDATE_SHARES_KEY: + shares_start, shares_end, share_count, _exact = _write_shares(shares, sink, events) + else: + _write_top_level_field(key, identity[key], sink, events) + sink.write_text("}") + digest = sink.finish() + manifest = CandidateBodyManifest( + storage_version=CANDIDATE_BODY_STORAGE_VERSION, + candidate_sha256=digest, + byte_count=sink.offset, + chunk_count=sink.chunk_count, + chunk_bytes=int(chunk_bytes), + share_count=share_count, + shares_offset=shares_start, + shares_end=shares_end, + span_count=events.span_count, + page_count=events.page_count, + ) + manifest.validate() + return manifest + + +# -------------------------------------------------------------------------- +# identity normalization and replay header +# -------------------------------------------------------------------------- + + +def split_candidate_fields( + fields: Mapping[str, Any], +) -> tuple[dict[str, Any], Any]: + """Return ``(small facts, shares sequence or None)`` without copying. + + ``None`` means the intent carries no ``shares_json`` key at all (a + candidate-only intent as some tools persist); the historical identity + JSON of such a document has no share member and is reproduced as is. + """ + if not isinstance(fields, Mapping): + raise TypeError("block candidate intent must be an object") + facts = { + str(key): value + for key, value in fields.items() + if key != CANDIDATE_SHARES_KEY + } + return facts, fields.get(CANDIDATE_SHARES_KEY) if CANDIDATE_SHARES_KEY in fields else None + + +def neutralized_identity_facts(facts: Mapping[str, Any]) -> tuple[dict[str, Any], bool, Any]: + """Apply ``block_candidate_identity``'s stamp rule to the small facts. + + Returns ``(identity facts, stamp present, original stamp)``. The rule + is exactly the historical one: only when ``pending_share`` is a mapping + that carries ``accepted_at_ms`` is that one value replaced by ``None``. + """ + pending_share = facts.get("pending_share") + if isinstance(pending_share, dict) and "accepted_at_ms" in pending_share: + identity = { + **facts, + "pending_share": {**pending_share, "accepted_at_ms": None}, + } + return identity, True, pending_share["accepted_at_ms"] + return dict(facts), False, None + + +def restore_pending_stamp( + facts: Mapping[str, Any], + *, + present: bool, + accepted_at_ms: Any, +) -> dict[str, Any]: + """Undo :func:`neutralized_identity_facts` for a body read back.""" + restored = dict(facts) + pending_share = restored.get("pending_share") + if present and isinstance(pending_share, dict): + restored["pending_share"] = {**pending_share, "accepted_at_ms": accepted_at_ms} + return restored + + +def _bounded_text(value: Any, limit: int) -> Any: + """A string fact within ``limit`` bytes of JSON text, else None.""" + if not isinstance(value, str) or len(value) > limit: + return None + if len(json.dumps(value)) > limit: + return None + return value + + +def _bounded_number(value: Any, limit: int) -> Any: + """An int/float fact within ``limit`` bytes of JSON text, else None.""" + if isinstance(value, bool) or not isinstance(value, (int, float)): + return None + if len(json.dumps(value)) > limit: + return None + return value + + +_PENDING_SHARE_TEXT_FIELDS = ("share_id", "miner_id", "order_key", "p2mr_program_hex", "job_id", "credit_policy") +_PENDING_SHARE_NUMBER_FIELDS = ( + "share_difficulty", + "network_difficulty", + "template_height", + "job_issued_at_ms", + "accepted_at_ms", + "ntime", +) + + +def replay_header_from_fields(fields: Mapping[str, Any]) -> dict[str, Any]: + """The small typed summary an outbox row carries beside its body. + + Explicit projection with per-fact byte bounds, never subtraction: + every key is named and every fact is either within its bound or + replaced by ``None`` with ``oversized`` set, so a page of headers has a + provable maximum size regardless of what the body holds. An oversized + fact is deferred to the body (hydration reads the real value); nothing + is rejected here. ``pending_share`` keeps the original acknowledgment + stamp. Mirrors ``qbit_prism_bounded_replay_header`` in SQL. + """ + template = fields.get("template") + found_block = fields.get("found_block") + pending_share = fields.get("pending_share") + oversized = False + + def text(value: Any, limit: int = HEADER_TEXT_FIELD_BYTES) -> Any: + nonlocal oversized + bounded = _bounded_text(value, limit) + if bounded is None and value is not None: + oversized = True + return bounded + + def number(value: Any, limit: int = HEADER_NUMERIC_FIELD_BYTES) -> Any: + nonlocal oversized + bounded = _bounded_number(value, limit) + if bounded is None and value is not None: + oversized = True + return bounded + + bounded_pending: dict[str, Any] | None = None + if isinstance(pending_share, Mapping): + bounded_pending = {} + for key, value in pending_share.items(): + if not isinstance(key, str) or len(key) > 64 or len(key.encode("utf-8")) > 64: + oversized = True + continue + if key in _PENDING_SHARE_NUMBER_FIELDS: + bounded_pending[str(key)] = number(value) + elif key in _PENDING_SHARE_TEXT_FIELDS: + bounded_pending[str(key)] = text(value) + elif isinstance(value, str): + bounded_pending[str(key)] = text(value) + elif isinstance(value, (int, float)) and not isinstance(value, bool): + bounded_pending[str(key)] = number(value) + elif value is None or isinstance(value, bool): + bounded_pending[str(key)] = value + else: + oversized = True + bounded_pending[str(key)] = None + if len(bounded_pending) > 32: + oversized = True + break + elif pending_share is not None: + oversized = True + header = { + "schema": text(fields.get("schema")), + "block_hash_hex": text(fields.get("block_hash_hex"), HEADER_HASH_FIELD_BYTES + 2), + "parent_hash": text(fields.get("parent_hash"), HEADER_HASH_FIELD_BYTES + 2), + "expected_height": number(fields.get("expected_height")), + "template": ( + { + "previousblockhash": text(template.get("previousblockhash"), HEADER_HASH_FIELD_BYTES + 2), + "height": number(template.get("height")), + "coinbasevalue": number(template.get("coinbasevalue")), + } + if isinstance(template, Mapping) + else None + ), + "found_block": ( + {"network_difficulty": number(found_block.get("network_difficulty"))} + if isinstance(found_block, Mapping) + else None + ), + "pending_share": bounded_pending, + "credit_share_on_accept": ( + fields.get("credit_share_on_accept") + if isinstance(fields.get("credit_share_on_accept"), bool) + else None + ), + "collection_only": ( + fields.get("collection_only") + if isinstance(fields.get("collection_only"), bool) + else None + ), + "username": text(fields.get("username")), + "accepted_at_present": bool( + isinstance(pending_share, Mapping) and "accepted_at_ms" in pending_share + ), + "oversized": oversized, + } + return header + + +# -------------------------------------------------------------------------- +# on-disk page index +# -------------------------------------------------------------------------- + + +class SpoolFieldIndex: + """Spans and page entries of one spool body, kept on disk. + + The index file holds every field's page entries as fixed 16-byte + records (`` None: + self.path = path + self._spans: dict[str, FieldSpan] = {} + self._regions: dict[str, int] = {} + self._lock = threading.Lock() + + @classmethod + def create(cls, path: str) -> SpoolFieldIndex: + with open(path, "wb"): + pass + return cls(path) + + @classmethod + def open_written(cls, path: str, spans_in_order: Sequence[FieldSpan]) -> SpoolFieldIndex: + """Attach to an index file another process wrote span by span. + + The writer appended each field's region in the order it began the + spans, so the regions are recovered from the page counts alone and + the file size is checked against them. + """ + index = cls(path) + position = 0 + for span in spans_in_order: + index._spans[span.field] = span + index._regions[span.field] = position + position += span.page_count * _INDEX_RECORD.size + if os.path.getsize(path) != position: + raise CandidateBodyIntegrityError("page index size disagrees with its spans") + return index + + def spans(self) -> dict[str, FieldSpan]: + with self._lock: + return dict(self._spans) + + def span(self, field_name: str) -> FieldSpan | None: + with self._lock: + return self._spans.get(field_name) + + def begin_field(self, span: FieldSpan) -> None: + """Declare a field's span. + + The encoder emits a field's page batches *before* its span (the + span is known only when the field ends), so the region start is + pinned by whichever comes first: the first page append, or this + declaration for a field with no pages. + """ + with self._lock: + if span.field in self._spans: + raise CandidateBodyIntegrityError("duplicate body span") + self._spans[span.field] = span + self._regions.setdefault(span.field, os.path.getsize(self.path)) + + def append_pages(self, field_name: str, entries: Iterable[tuple[int, int]]) -> None: + """Append ``(offset, record_index)`` records for the field being written.""" + with self._lock: + self._regions.setdefault(field_name, os.path.getsize(self.path)) + packed = b"".join(_INDEX_RECORD.pack(int(offset), int(index)) for offset, index in entries) + with open(self.path, "ab") as handle: + handle.write(packed) + + def page_count(self, field_name: str) -> int: + return self._spans[field_name].page_count + + def entry(self, field_name: str, ordinal: int) -> tuple[int, int]: + span = self._spans[field_name] + if not (0 <= ordinal < span.page_count): + raise IndexError(ordinal) + position = self._regions[field_name] + ordinal * _INDEX_RECORD.size + with open(self.path, "rb") as handle: + handle.seek(position) + raw = handle.read(_INDEX_RECORD.size) + if len(raw) != _INDEX_RECORD.size: + raise CandidateBodyIntegrityError("page index is truncated") + offset, record_index = _INDEX_RECORD.unpack(raw) + return int(offset), int(record_index) + + def set_record_index(self, field_name: str, ordinal: int, record_index: int) -> None: + """Correct one hint's record index after an exact decode.""" + offset, _ = self.entry(field_name, ordinal) + position = self._regions[field_name] + ordinal * _INDEX_RECORD.size + with self._lock, open(self.path, "r+b") as handle: + handle.seek(position) + handle.write(_INDEX_RECORD.pack(int(offset), int(record_index))) + + def bisect_record(self, field_name: str, record_index: int) -> int: + """The page ordinal holding ``record_index`` (exact pages only).""" + low, high = 0, self._spans[field_name].page_count + while high - low > 1: + middle = (low + high) // 2 + if self.entry(field_name, middle)[1] <= record_index: + low = middle + else: + high = middle + return low + + def mark_exact(self, field_name: str) -> None: + with self._lock: + span = self._spans[field_name] + self._spans[field_name] = replace(span, pages_exact=True) + + def close(self) -> None: + try: + os.unlink(self.path) + except OSError: + pass + + +# -------------------------------------------------------------------------- +# body handles +# -------------------------------------------------------------------------- + + +class CandidateBody: + """Replayable immutable body bytes plus the manifest describing them. + + Chunks are delivered push-style, one at a time, to a consumer callable: + every consumer this codebase has (validation, storage upload, spool + write, helper pipe) is push-shaped, and a push interface keeps at most + one chunk plus the encoder's bounded batch alive at any instant. Index + events (spans, page batches) ride the same pass. + """ + + manifest: CandidateBodyManifest + + def write_chunks( + self, + consumer: Callable[[BodyChunk], None], + *, + on_span: Callable[[FieldSpan], None] | None = None, + on_page: Callable[[list[PageEntry]], None] | None = None, + ) -> None: + raise NotImplementedError + + def close(self) -> None: + """Release what this handle owns; idempotent.""" + + +class SourceCandidateBody(CandidateBody): + """A live body: re-encoded from the pinned source sequence on demand. + + Nothing encoded is retained between passes. Every pass re-derives the + bytes, and the rolling digest of what was handed out must equal the + manifest's digest, so a source that changed underneath is detected + before the caller seals anything. + """ + + __slots__ = ("manifest", "_identity", "_shares") + + def __init__( + self, + identity: Mapping[str, Any], + shares: Any, + manifest: CandidateBodyManifest, + ) -> None: + self.manifest = manifest + self._identity = identity + self._shares = shares + + def write_chunks( + self, + consumer: Callable[[BodyChunk], None], + *, + on_span: Callable[[FieldSpan], None] | None = None, + on_page: Callable[[list[PageEntry]], None] | None = None, + ) -> None: + manifest = encode_identity_body( + self._identity, + self._shares, + chunk_bytes=self.manifest.chunk_bytes, + on_chunk=consumer, + on_span=on_span, + on_page=on_page, + ) + if manifest != self.manifest: + raise CandidateBodyIntegrityError( + "candidate body source changed after validation" + ) + + def close(self) -> None: + self._shares = () + self._identity = {} + + +class SpoolCandidateBody(CandidateBody): + """A durable body materialized to a local spool file with a page index. + + The spool holds exactly the body bytes and the index its spans and + pages. Chunk iteration re-reads and re-hashes the file, so a reader + never trusts a stale manifest over the bytes on disk. Both files are + unlinked when the last owner releases the handle (a ``weakref.finalize`` + on the handle, plus explicit ``close``), never while a sequence still + reads them. + """ + + __slots__ = ("manifest", "path", "index", "_finalizer") + + def __init__( + self, + path: str, + manifest: CandidateBodyManifest, + index: SpoolFieldIndex, + *, + release: Callable[[], None] | None = None, + ) -> None: + self.manifest = manifest + self.path = path + self.index = index + self._finalizer = weakref.finalize(self, _unlink_spool, path, index.path, release) + + def iter_chunks(self) -> Iterator[BodyChunk]: + chunk_bytes = self.manifest.chunk_bytes + digest = hashlib.sha256() + with open(self.path, "rb") as handle: + ordinal = 0 + while True: + data = handle.read(chunk_bytes) + if not data: + break + if ordinal >= self.manifest.chunk_count: + raise CandidateBodyIntegrityError("spool body is longer than its manifest") + digest.update(data) + yield BodyChunk(ordinal=ordinal, data=data, sha256=hashlib.sha256(data).hexdigest()) + ordinal += 1 + if ordinal != self.manifest.chunk_count or digest.hexdigest() != self.manifest.candidate_sha256: + raise CandidateBodyIntegrityError("spool body disagrees with its manifest") + + def write_chunks( + self, + consumer: Callable[[BodyChunk], None], + *, + on_span: Callable[[FieldSpan], None] | None = None, + on_page: Callable[[list[PageEntry]], None] | None = None, + ) -> None: + for chunk in self.iter_chunks(): + consumer(chunk) + if on_span is None and on_page is None: + return + for field_name, span in sorted(self.index.spans().items()): + if on_span is not None: + on_span(span) + if on_page is None: + continue + batch: list[PageEntry] = [] + for ordinal in range(span.page_count): + offset, record_index = self.index.entry(field_name, ordinal) + batch.append(PageEntry(field=field_name, ordinal=ordinal, offset=offset, record_index=record_index)) + if len(batch) >= INDEX_PAGE_ENTRIES: + on_page(batch) + batch = [] + if batch: + on_page(batch) + + def read_span(self, start: int, end: int) -> bytes: + if start < 0 or end < start or end > self.manifest.byte_count: + raise ValueError("spool span out of range") + with open(self.path, "rb") as handle: + handle.seek(start) + data = handle.read(end - start) + if len(data) != end - start: + raise CandidateBodyIntegrityError("spool body is shorter than its manifest") + return data + + def close(self) -> None: + self._finalizer() + + @property + def alive(self) -> bool: + return self._finalizer.alive + + +def _unlink_spool(path: str, index_path: str, release: Callable[[], None] | None) -> None: + for target in (path, index_path): + try: + os.unlink(target) + except OSError: + pass + if release is not None: + try: + release() + except Exception: # noqa: BLE001 - finalizers must not raise + pass + + +# -------------------------------------------------------------------------- +# spool decoding +# -------------------------------------------------------------------------- + + +def _views() -> Any: + """The lazy-view module (``candidate_spool_view``), imported on first use. + + That module builds on this one and on the audit adapters' retryable + resource-pressure class, so the import is deferred to keep this module + a standard-library leaf at import time. + """ + from lab.prism import candidate_spool_view + + return candidate_spool_view + + +def _walk_record_batches( + body: SpoolCandidateBody, + start: int, + end: int, + *, + batch_records: int = SPOOL_WALK_BATCH_RECORDS, +) -> Iterator[tuple[int, int, tuple[Any, ...]]]: + """``(batch_start, batch_end, records)`` over ``body[start:end]``. + + The range holds records joined by ``,``. Record boundaries come from the + structural byte scanner, so no record is decoded to find its end, and + each record follows the view module's decode policy: one bounded + ``json.loads`` when it fits :data:`~lab.prism.candidate_spool_view.SPOOL_VIEW_DECODE_BYTES`, + a lazy view when it does not. There is no per-record size ceiling. + """ + views = _views() + batch: list[Any] = [] + batch_start = start + batch_end = start + for item_start, item_end in views.iter_item_spans(body, start, end): + if not batch: + batch_start = item_start + batch.append(views.decode_body_span(body, item_start, item_end)) + batch_end = item_end + if len(batch) >= batch_records: + yield batch_start, batch_end, tuple(batch) + batch = [] + if batch: + yield batch_start, batch_end, tuple(batch) + + +def _walk_records( + body: SpoolCandidateBody, + start: int, + end: int, + *, + batch_records: int = SPOOL_WALK_BATCH_RECORDS, +) -> Iterator[tuple[Any, ...]]: + """Decode ``body[start:end]`` (records joined by ``,``) in bounded batches.""" + for _batch_start, _batch_end, records in _walk_record_batches( + body, start, end, batch_records=batch_records + ): + yield records + + +class SpoolJsonArraySequence(Sequence): + """One large JSON array field decoded page by page from a spool body. + + Iteration decodes one page per ``json.loads`` from the on-disk page + index while the page fits the view module's decode threshold; a larger + page (one holding an oversized record) is split by the structural + scanner and each record follows the decode policy, so an oversized + record is a lazy view rather than a whole decode or a rejection. Random + access bisects the index with ``pread``. A span whose page entries are + not exact (legacy scanned hints) is walked sequentially once, ignoring + the hints, and the exact page starts recovered by that walk serve random + access from then on. Only the page last decoded is cached. + """ + + __slots__ = ("_body", "_span", "_lock", "_cached_page", "_cached_records", "_walk_index") + + def __init__(self, body: SpoolCandidateBody, span: FieldSpan) -> None: + self._body = body + self._span = span + self._lock = threading.Lock() + self._cached_page = -1 + self._cached_records: tuple[Any, ...] = () + self._walk_index: Any = None + + @property + def body(self) -> SpoolCandidateBody: + return self._body + + @property + def span(self) -> FieldSpan: + return self._body.index.span(self._span.field) or self._span + + @property + def byte_range(self) -> tuple[int, int]: + return self._span.start, self._span.end + + @property + def byte_length(self) -> int: + return self._span.end - self._span.start + + def __len__(self) -> int: + return self._span.item_count + + # -- pages ------------------------------------------------------------------ + + def _indexed(self) -> bool: + """True when the on-disk page index can serve random access.""" + span = self.span + return span.page_count > 0 and span.pages_exact + + def _index_entry(self, ordinal: int) -> tuple[int, int]: + try: + return self._body.index.entry(self._span.field, ordinal) + except OSError as exc: + raise CandidateBodyIntegrityError(f"spool page index is unreadable: {exc}") from exc + + def _page_span(self, page: int) -> tuple[int, int, int]: + """(start, end, record_index) of one page; ``end`` excludes the separator.""" + start, record_index = self._index_entry(page) + if page + 1 < self._span.page_count: + end = self._index_entry(page + 1)[0] - 1 + else: + end = self._span.end - 1 + if not (self._span.start < start <= end < self._span.end): + raise CandidateBodyIntegrityError("spool page index points outside its span") + return start, end, record_index + + def _decode_records(self, start: int, end: int) -> tuple[Any, ...]: + """Records in ``body[start:end]``: one bounded call, or the walk.""" + views = _views() + if end <= start: + return () + if end - start <= views.SPOOL_VIEW_DECODE_BYTES: + data = views.read_body_span(self._body, start, end) + try: + records = json.loads(b"[" + data + b"]") + except ValueError as exc: + raise CandidateBodyIntegrityError("spool page is not valid JSON") from exc + return tuple(records) + return tuple( + record + for _batch_start, _batch_end, batch in _walk_record_batches(self._body, start, end) + for record in batch + ) + + def _decode_page(self, page: int) -> tuple[Any, ...]: + with self._lock: + if page == self._cached_page: + return self._cached_records + start, end, _ = self._page_span(page) + decoded = self._decode_records(start, end) + with self._lock: + self._cached_page = page + self._cached_records = decoded + return decoded + + def _walk_pages(self, span: FieldSpan) -> Iterator[tuple[Any, ...]]: + """Sequential walk of a span without a usable page index. + + Records the exact batch starts in a private bounded index so later + random access bisects instead of walking again. + """ + views = _views() + with self._lock: + index = None if self._walk_index is not None else views._OffsetIndex(2) + running = 0 + try: + for batch_start, _batch_end, records in _walk_record_batches( + self._body, span.start + 1, span.end - 1 + ): + if index is not None: + index.append(batch_start, running) + running += len(records) + yield records + if running != span.item_count: + raise CandidateBodyIntegrityError( + f"spool array holds {running} records where {span.item_count} were declared" + ) + except BaseException: + if index is not None: + index.close() + raise + if index is not None: + with self._lock: + if self._walk_index is None: + self._walk_index = index + index = None + if index is not None: + index.close() + + def iter_pages(self) -> Iterator[tuple[Any, ...]]: + """Decoded pages in order; each one bounded C call (or a walk).""" + span = self.span + if not self._indexed(): + yield from self._walk_pages(span) + return + running = 0 + for page in range(span.page_count): + records = self._decode_page(page) + running += len(records) + yield records + if running != span.item_count: + raise CandidateBodyIntegrityError( + f"spool array holds {running} records where {span.item_count} were declared" + ) + + # -- Sequence protocol ---------------------------------------------------------- + + def __iter__(self) -> Iterator[Any]: + for page in self.iter_pages(): + yield from page + + def __getitem__(self, index: int | slice) -> Any: + if isinstance(index, slice): + start, stop, step = index.indices(len(self)) + return tuple(self[position] for position in range(start, stop, step)) + if isinstance(index, bool) or not isinstance(index, int): + raise TypeError("spool array indices must be integers or slices") + resolved = index + len(self) if index < 0 else index + if not 0 <= resolved < len(self): + raise IndexError(index) + span = self.span + if self._indexed(): + try: + page = self._body.index.bisect_record(self._span.field, resolved) + except OSError as exc: + raise CandidateBodyIntegrityError(f"spool page index is unreadable: {exc}") from exc + records = self._decode_page(page) + first = self._index_entry(page)[1] + return records[resolved - first] + walk = self._walk_index + if walk is None: + for _page in self.iter_pages(): + pass + walk = self._walk_index + if walk is None: + raise IndexError(index) + low, high = 0, walk.count + while high - low > 1: + middle = (low + high) // 2 + if walk.get(middle)[1] <= resolved: + low = middle + else: + high = middle + batch_start, first = walk.get(low) + batch_end = walk.get(low + 1)[0] - 1 if low + 1 < walk.count else span.end - 1 + records = self._decode_records(batch_start, batch_end) + return records[resolved - first] + + def __eq__(self, other: object) -> bool: + if other is self: + return True + if isinstance(other, (str, bytes, bytearray)) or not isinstance(other, Sequence): + return NotImplemented + if len(self) != len(other): + return False + other_iter = iter(other) + for item in self: + try: + candidate = next(other_iter) + except StopIteration: + return False + if item != candidate: + return False + return True + + def __ne__(self, other: object) -> bool: + result = self.__eq__(other) + if result is NotImplemented: + return result # type: ignore[return-value] + return not result + + __hash__ = None # type: ignore[assignment] + + def __repr__(self) -> str: + return f"SpoolJsonArraySequence(field={self._span.field!r}, items={len(self)}, bytes={self.byte_length})" + + # -- verbatim bytes -------------------------------------------------------------- + + def iter_byte_chunks(self, *, chunk_bytes: int | None = None) -> Iterator[bytes]: + """The array's exact encoding as stored, brackets included.""" + return _views().iter_body_bytes(self._body, self._span.start, self._span.end, chunk_bytes=chunk_bytes) + + def iter_encoded_chunks(self) -> Iterator[str]: + return _views().iter_body_text(self._body, self._span.start, self._span.end) + + def canonical_json_sha256(self) -> str: + digest = hashlib.sha256() + for chunk in self.iter_byte_chunks(): + digest.update(chunk) + return digest.hexdigest() + + def canonical_share_pages(self) -> Iterator[tuple[bytes, int | None]]: + """The encoded pages, verbatim, for re-staging this body. + + A page within the read size is one complete page with its record + count; a larger page (an oversized record) and a span without a + usable index stream as raw continuation pieces, which the encoder's + structural scanner re-pages exactly. Nothing page-sized is held. + """ + views = _views() + span = self.span + if not self._indexed(): + for chunk in views.iter_body_bytes(self._body, span.start + 1, span.end - 1): + yield chunk, None + return + for page in range(span.page_count): + start, end, record_index = self._page_span(page) + if end - start > views.SPOOL_VIEW_READ_BYTES: + for chunk in views.iter_body_bytes(self._body, start, end): + yield chunk, None + continue + next_index = ( + self._index_entry(page + 1)[1] + if page + 1 < span.page_count + else span.item_count + ) + yield views.read_body_span(self._body, start, end), next_index - record_index + + def close(self) -> None: + """Release the private walk index, if one was built; the body is untouched.""" + with self._lock: + walk = self._walk_index + if walk is not None: + walk.close() + + +# Historical name kept for the share array. +SpoolShareJsonSequence = SpoolJsonArraySequence + + +def decode_json_string_span(body: SpoolCandidateBody, span: FieldSpan) -> str: + """The whole text of one large JSON string field, decoded in slices. + + Every C call is one slice; the join is bounded only by the field, which + is why hydration uses this for the consensus-bounded metadata fields + alone and hands any other oversized string out as a streamed view. + """ + return _views().decode_json_string(body, span.start, span.end) + + +def decode_spool_small_fields(body: SpoolCandidateBody) -> dict[str, Any]: + """The body's fields other than the share array, bounded per field. + + Builds a *skeleton* of the body in which every spanned (large) field is + replaced by an empty placeholder and decodes it with one ``json.loads`` + bounded by the view decode ceiling, falling back to per-member reads + if the aggregate metadata exceeds that ceiling. Each + large field then follows the view module's policy: an array becomes a + :class:`SpoolJsonArraySequence`; a string in + :data:`~lab.prism.candidate_spool_view.SPOOL_DECODED_METADATA_FIELDS` + is decoded whole in slices (its size is bounded by consensus and its + consumers need the text); any other value is a plain value within the + decode threshold and a lazy view above it. Nothing is rejected for size. + """ + views = _views() + manifest = body.manifest + spans = sorted(body.index.spans().values(), key=lambda span: span.start) + validated_end = 0 + for span in spans: + if span.start < validated_end or span.end > manifest.byte_count: + raise CandidateBodyIntegrityError("spool spans overlap or exceed the body") + validated_end = span.end + + class SkeletonFull(Exception): + pass + + class BoundedSkeleton(io.BytesIO): + def write(self, data: bytes) -> int: + if self.tell() + len(data) > views.SPOOL_VIEW_DECODE_BYTES: + raise SkeletonFull + return super().write(data) + + skeleton = BoundedSkeleton() + cursor = 0 + try: + for span in spans: + _copy_span(body, cursor, span.start, skeleton) + skeleton.write(b"[]" if span.kind == "array" else b'""' if span.kind == "string" else b"null") + cursor = span.end + _copy_span(body, cursor, manifest.byte_count, skeleton) + fields = json.loads(skeleton.getvalue()) + except SkeletonFull: + # Many individually small metadata fields can exceed the aggregate + # fast-path budget. Decode those members separately through the same + # view policy instead of growing one native JSON call. + root = views.SpoolObjectView(body, 0, manifest.byte_count) + try: + fields = dict(root.items()) + finally: + root.close() + except ValueError as exc: + raise CandidateBodyIntegrityError("spool body skeleton is not valid JSON") from exc + if not isinstance(fields, dict): + raise CandidateBodyIntegrityError("spool body does not decode to an intent") + for span in spans: + if span.field == CANDIDATE_SHARES_KEY: + continue + if span.kind == "array": + fields[span.field] = SpoolJsonArraySequence(body, span) + elif span.kind == "string" and span.field in views.SPOOL_DECODED_METADATA_FIELDS: + fields[span.field] = views.decode_json_string(body, span.start, span.end) + else: + fields[span.field] = views.decode_body_span(body, span.start, span.end) + fields.pop(CANDIDATE_SHARES_KEY, None) + return fields + + +def _copy_span(body: SpoolCandidateBody, start: int, end: int, sink: io.BytesIO) -> None: + for chunk in _views().iter_body_bytes(body, start, end): + sink.write(chunk) + + +# -------------------------------------------------------------------------- +# prepared intent +# -------------------------------------------------------------------------- + + +class PreparedCandidateIntent(MutableMapping): + """A validated candidate intent behind the historical mapping interface. + + ``facts`` holds every field but ``shares_json`` with the pending share's + original acknowledgment stamp; ``shares`` is the immutable sequence + (never a list copy); ``body`` streams the v1 identity bytes; the + remaining attributes are the small typed facts storage needs. Mapping + access (``intent["shares_json"]``, ``dict(intent)``, ``intent.get``) + behaves as the historical dict did. Mutation (``pop``, item + assignment) is supported for dict-facing callers and tools; it marks + the intent *dirty*, and the next preparation re-derives the manifest + from the mutated fields rather than trusting the stale one. + """ + + __slots__ = ( + "facts", + "shares", + "body", + "block_hash", + "candidate_sha256", + "accepted_at_present", + "accepted_at_ms", + "_key_order", + "dirty", + ) + + def __init__( + self, + *, + facts: dict[str, Any], + shares: Any, + body: CandidateBody, + accepted_at_present: bool, + accepted_at_ms: Any, + key_order: tuple[str, ...], + ) -> None: + self.facts = facts + self.shares = shares + self.body = body + self.block_hash = str(facts.get("block_hash_hex", "")).lower() + self.candidate_sha256 = body.manifest.candidate_sha256 + self.accepted_at_present = accepted_at_present + self.accepted_at_ms = accepted_at_ms + self._key_order = key_order + self.dirty = False + + def __setitem__(self, key: str, value: Any) -> None: + key = str(key) + if key == CANDIDATE_SHARES_KEY: + self.shares = value + else: + self.facts[key] = value + if key not in self._key_order: + self._key_order = (*self._key_order, key) + self.dirty = True + + def __delitem__(self, key: str) -> None: + if key == CANDIDATE_SHARES_KEY: + if self.shares is None: + raise KeyError(key) + self.shares = None + else: + del self.facts[key] + self._key_order = tuple(name for name in self._key_order if name != key) + self.dirty = True + + @property + def manifest(self) -> CandidateBodyManifest: + return self.body.manifest + + @property + def storage_version(self) -> int: + return self.body.manifest.storage_version + + @property + def has_shares(self) -> bool: + return self.shares is not None + + def __getitem__(self, key: str) -> Any: + if key == CANDIDATE_SHARES_KEY: + if self.shares is None: + raise KeyError(key) + return self.shares + return self.facts[key] + + def __iter__(self) -> Iterator[str]: + return iter(self._key_order) + + def __len__(self) -> int: + return len(self._key_order) + + def __contains__(self, key: object) -> bool: + if key == CANDIDATE_SHARES_KEY: + return self.shares is not None + return key in self.facts + + def __repr__(self) -> str: + return ( + f"PreparedCandidateIntent(block_hash={self.block_hash!r}, " + f"candidate_sha256={self.candidate_sha256!r}, " + f"shares={self.manifest.share_count}, bytes={self.manifest.byte_count})" + ) + + def replay_header(self) -> dict[str, Any]: + """The bounded typed summary the outbox row carries beside the body.""" + return replay_header_from_fields({**self.facts, "block_hash_hex": self.block_hash}) + + def release(self) -> None: + """Drop the body's hold on its source; the facts stay readable.""" + self.body.close() + + +def prepare_candidate_intent( + fields: Mapping[str, Any], + *, + chunk_bytes: int = CANDIDATE_BODY_CHUNK_BYTES, +) -> PreparedCandidateIntent: + """Validate and index an intent mapping without materializing its body. + + One bounded encode pass runs now -- the synchronous validation that + used to be a whole ``json.dumps`` -- producing the scalar manifest and + discarding the bytes and index events. The returned intent re-encodes + from the same immutable source whenever storage asks for chunks, and + refuses to publish if the source changed. + """ + if isinstance(fields, PreparedCandidateIntent): + if not fields.dirty: + return fields + # Mutated after preparation: re-derive from the current fields. + fields = dict(fields) + facts, shares = split_candidate_fields(fields) + block_hash = str(facts.get("block_hash_hex", "")).lower() + if not block_hash: + raise CandidateCodecError("block candidate is missing block_hash_hex") + facts["block_hash_hex"] = block_hash + identity, present, stamp = neutralized_identity_facts(facts) + try: + manifest = encode_identity_body(identity, shares, chunk_bytes=chunk_bytes) + except CandidateCodecError: + raise + except TypeError: + # Exactly the exception the historical whole-document json.dumps + # raised for an unsupported value type; callers key on it. + raise + except ValueError as exc: + raise CandidateCodecError(str(exc)) from exc + key_order = tuple(str(key) for key in fields.keys()) + return PreparedCandidateIntent( + facts=facts, + shares=shares, + body=SourceCandidateBody(identity, shares, manifest), + accepted_at_present=present, + accepted_at_ms=stamp, + key_order=key_order, + ) + + +def prepared_intent_from_spool( + body: SpoolCandidateBody, + *, + accepted_at_present: bool, + accepted_at_ms: Any, +) -> PreparedCandidateIntent: + """Rebuild the mapping view of a hydrated body. + + Facts stay a plain dict; each value follows the view module's decode + policy. ``pending_share`` alone is shallow-copied into a dict when it + hydrates as a lazy object, so the acknowledgment stamp is restored the + historical way (its members keep the policy: an oversized ``share_id`` + is a streamed string). + """ + identity = decode_spool_small_fields(body) + pending_share = identity.get("pending_share") + if ( + accepted_at_present + and isinstance(pending_share, Mapping) + and not isinstance(pending_share, dict) + ): + try: + identity["pending_share"] = dict(pending_share.items()) + except BaseException: + # The view's scratch index must not outlive this failure even + # while a retained traceback keeps the view itself reachable. + pending_share.close() + raise + facts = restore_pending_stamp( + identity, + present=accepted_at_present, + accepted_at_ms=accepted_at_ms, + ) + shares_span = body.index.span(CANDIDATE_SHARES_KEY) + if shares_span is None: + if body.manifest.share_count or body.manifest.shares_end > body.manifest.shares_offset: + raise CandidateBodyIntegrityError("spool body carries no share span") + shares = None + keys = sorted(facts.keys()) + else: + shares = SpoolJsonArraySequence(body, shares_span) + keys = sorted([*facts.keys(), CANDIDATE_SHARES_KEY]) + return PreparedCandidateIntent( + facts=facts, + shares=shares, + body=body, + accepted_at_present=accepted_at_present, + accepted_at_ms=accepted_at_ms, + key_order=tuple(keys), + ) + + +class SpoolWriter: + """Write verified chunks and index events to a spool + index pair.""" + + def __init__(self, path: str, index_path: str, manifest: CandidateBodyManifest) -> None: + self.manifest = manifest + self.path = path + self.index = SpoolFieldIndex.create(index_path) + self._handle = open(path, "wb") + self._digest = hashlib.sha256() + self._written = 0 + self._ordinal = 0 + self._span_count = 0 + self._page_count = 0 + self._open_field: str | None = None + + def chunk(self, chunk: BodyChunk) -> None: + manifest = self.manifest + if chunk.ordinal != self._ordinal: + raise CandidateBodyIntegrityError("spool chunks arrived out of order") + if self._ordinal >= manifest.chunk_count: + raise CandidateBodyIntegrityError("more chunks than the manifest declares") + expected_length = ( + manifest.chunk_bytes + if self._ordinal + 1 < manifest.chunk_count + else manifest.byte_count - manifest.chunk_bytes * (manifest.chunk_count - 1) + ) + if len(chunk.data) != expected_length: + raise CandidateBodyIntegrityError("spool chunk length disagrees with the manifest") + if hashlib.sha256(chunk.data).hexdigest() != chunk.sha256: + raise CandidateBodyIntegrityError("spool chunk digest disagrees with its bytes") + self._handle.write(chunk.data) + self._digest.update(chunk.data) + self._written += len(chunk.data) + self._ordinal += 1 + + def span(self, span: FieldSpan) -> None: + self.index.begin_field(span) + self._span_count += 1 + + def pages(self, entries: Iterable[PageEntry]) -> None: + batch = list(entries) + if not batch: + return + field_name = batch[0].field + if any(entry.field != field_name for entry in batch): + raise CandidateBodyIntegrityError("page batch spans several fields") + self.index.append_pages(field_name, ((entry.offset, entry.record_index) for entry in batch)) + self._page_count += len(batch) + + def finish(self, *, release: Callable[[], None] | None = None) -> SpoolCandidateBody: + """Verify and hand over the one body handle that owns the files. + + Exactly one :class:`SpoolCandidateBody` is created here: its + finalizer unlinks the spool pair, so a second handle on the same + path would delete the files as soon as the first was dropped. + """ + self._handle.close() + manifest = self.manifest + if self._ordinal != manifest.chunk_count or self._written != manifest.byte_count: + raise CandidateBodyIntegrityError("spool body is incomplete") + if self._digest.hexdigest() != manifest.candidate_sha256: + raise CandidateBodyIntegrityError("spool body digest disagrees with the manifest") + if self._span_count != manifest.span_count or self._page_count != manifest.page_count: + raise CandidateBodyIntegrityError("spool index disagrees with the manifest") + return SpoolCandidateBody(self.path, manifest, self.index, release=release) + + def abort(self) -> None: + try: + self._handle.close() + finally: + for target in (self.path, self.index.path): + try: + os.unlink(target) + except OSError: + pass + + +def write_spool_from_body( + path: str, + index_path: str, + body: CandidateBody, +) -> SpoolCandidateBody: + """Materialize any body (with its index events) to a spool pair.""" + writer = SpoolWriter(path, index_path, body.manifest) + try: + body.write_chunks(writer.chunk, on_span=writer.span, on_page=writer.pages) + return writer.finish() + except BaseException: + writer.abort() + raise + + +# -------------------------------------------------------------------------- +# legacy (v1 jsonb) normalization -- runs in the isolated helper only +# -------------------------------------------------------------------------- + + +def legacy_json_to_spool( + text: str, + path: str, + index_path: str, + *, + chunk_bytes: int = CANDIDATE_BODY_CHUNK_BYTES, + spool_limit_bytes: int | None = None, +) -> tuple[CandidateBodyManifest, list[FieldSpan], bool, Any]: + """Normalize one legacy ``candidate`` JSON text into a spool body pair. + + Whole-document work by design: this runs only inside the compatibility + helper process, never in the lease-bearing coordinator. Returns the + scalar manifest, the span rows (one per large field), and the pending + stamp's presence and original value; the page entries are in the index + file. + """ + document = json.loads(text) + if not isinstance(document, dict): + raise CandidateCodecError("legacy candidate is not an object") + facts, shares = split_candidate_fields(document) + identity, present, stamp = neutralized_identity_facts(facts) + index = SpoolFieldIndex.create(index_path) + spans: list[FieldSpan] = [] + written = 0 + + def admit(size: int) -> None: + nonlocal written + if spool_limit_bytes is not None and written + size > spool_limit_bytes: + raise OSError("legacy candidate spool reservation exhausted") + written += size + + def on_span(span: FieldSpan) -> None: + index.begin_field(span) + spans.append(span) + + with open(path, "wb") as handle: + def write_chunk(chunk: BodyChunk) -> None: + admit(len(chunk.data)) + handle.write(chunk.data) + + def write_pages(batch: list[PageEntry]) -> None: + admit(len(batch) * _INDEX_RECORD.size) + index.append_pages(batch[0].field, ((entry.offset, entry.record_index) for entry in batch)) + + manifest = encode_identity_body( + identity, + shares, + chunk_bytes=chunk_bytes, + on_chunk=write_chunk, + on_span=on_span, + on_page=write_pages, + ) + return manifest, spans, present, stamp + + +def jsonb_equivalent(left: Any, right: Any) -> bool: + """PostgreSQL ``jsonb`` equality over two decoded documents. + + Numbers compare as exact decimals (``1.0 = 1``, ``1e16 = 10^16``), the + boolean/null/number/string/array/object type classes never cross, and + objects compare as key sets regardless of order. Decode both sides with + ``parse_float=decimal.Decimal`` for exactness. + """ + from decimal import Decimal + + def kind(value: Any) -> str: + if value is None: + return "null" + if isinstance(value, bool): + return "bool" + if isinstance(value, (int, float, Decimal)): + return "number" + if isinstance(value, str): + return "string" + if isinstance(value, list): + return "array" + if isinstance(value, dict): + return "object" + raise TypeError(f"not a JSON value: {type(value).__name__}") + + left_kind = kind(left) + if left_kind != kind(right): + return False + if left_kind == "number": + return Decimal(str(left)) == Decimal(str(right)) + if left_kind in {"null", "bool", "string"}: + return left == right + if left_kind == "array": + return len(left) == len(right) and all( + jsonb_equivalent(a, b) for a, b in zip(left, right, strict=True) + ) + if set(left) != set(right): + return False + return all(jsonb_equivalent(left[key], right[key]) for key in left) + + +__all__ = [ + "CANDIDATE_BODY_CHUNK_BYTES", + "CANDIDATE_BODY_STORAGE_VERSION", + "CANDIDATE_INTENT_SCHEMA", + "CANDIDATE_SHARES_KEY", + "CODEC_BATCH_RECORDS", + "CODEC_BATCH_TARGET_BYTES", + "CODEC_FAST_PATH_BYTES", + "CODEC_STRING_SLICE_CHARS", + "HEADER_TEXT_FIELD_BYTES", + "INDEX_PAGE_ENTRIES", + "INDEX_SPAN_ROWS", + "LEGACY_CANDIDATE_STORAGE_VERSION", + "BodyChunk", + "CandidateBody", + "CandidateBodyIntegrityError", + "CandidateBodyManifest", + "CandidateCodecError", + "FieldSpan", + "PageEntry", + "PreparedCandidateIntent", + "SourceCandidateBody", + "SpoolCandidateBody", + "SpoolFieldIndex", + "SpoolJsonArraySequence", + "SpoolShareJsonSequence", + "SpoolWriter", + "chunk_ordinals_for", + "decode_json_string_span", + "decode_spool_small_fields", + "encode_identity_body", + "jsonb_equivalent", + "legacy_json_to_spool", + "neutralized_identity_facts", + "prepare_candidate_intent", + "prepared_intent_from_spool", + "replay_header_from_fields", + "restore_pending_stamp", + "split_candidate_fields", + "write_spool_from_body", +] diff --git a/lab/prism/candidate_spool_view.py b/lab/prism/candidate_spool_view.py new file mode 100644 index 000000000..7d9608d19 --- /dev/null +++ b/lab/prism/candidate_spool_view.py @@ -0,0 +1,1029 @@ +#!/usr/bin/env python3 +"""Bounded lazy views over a hydrated candidate spool body (issue #255). + +A hydrated block-candidate body is the v1 identity JSON on a local spool +file. The historical layout puts no bound on any single value: a share's +``share_id``, the miner ``username`` or a ``recipient_id`` may be of any +length, and ``found_block`` may carry nested arrays. The first reader +decoded every value whole, so one oversized field became one unbounded +``json.loads`` call in the lease-bearing coordinator, and the size ceilings +that guarded those calls rejected valid bodies as corruption. + +This module replaces the whole-value decode with one policy applied to +every JSON value in the body: + +* a value whose encoding is at most :data:`SPOOL_VIEW_DECODE_BYTES` is + decoded with one ``json.loads`` bounded by that size and returned as the + plain Python value it always was; +* a larger value becomes a lazy view over its own byte range: a + :class:`SpoolObjectView` (a ``Mapping``), a :class:`SpoolArrayView` (a + ``Sequence``), a :class:`SpoolStringView` (a streamed string) or a + :class:`SpoolRawValue` (an oversized scalar token). Members and items of + a lazy container are decoded on access under the same policy, so the + largest C call the reader ever makes is bounded by the threshold, at any + nesting depth, for a body of any size. + +Nothing is rejected for size. Every view exposes its original encoding +verbatim (``iter_byte_chunks``, ``iter_encoded_chunks``) so digests and +re-staging never need the decoded value, and a streamed string exposes its +text through ``iter_text_chunks`` in pieces that never split an escape or a +surrogate pair. The streaming protocol is the audit adapters' one +(``StreamedJsonString``, ``RawJsonDocument`` in ``audit_bundle_view``), so +the compact-body encoder embeds these views without materializing them. + +Structure is located with a stateful byte scanner (quoted text and nested +containers are skipped, escape pairs and multi-byte UTF-8 sequences are +carried across read boundaries), and per-container item indices live in +memory only up to a small cap, spilling to an unlinked scratch file beyond +it. A scratch file that cannot be created is +:class:`~lab.prism.audit_bundle_view.ArtifactResourcePressure` (retryable, +never corruption); a corrupt, truncated or closed body is +:class:`~lab.prism.candidate_codec.CandidateBodyIntegrityError`. + +Ownership: every view holds the :class:`SpoolCandidateBody` it reads and +nothing holds a view back, so there are no reference cycles and a dropped +view releases its scratch descriptor by reference count alone, with the +cyclic collector disabled. ``PreparedCandidateIntent.release()`` closes +the body; views read after that fail closed. +""" + +from __future__ import annotations + +import codecs +import hashlib +import json +import os +import re +import struct +import tempfile +import threading +import weakref +from collections.abc import ItemsView, Iterator, Mapping, Sequence, ValuesView +from typing import Any + +from lab.prism.audit_bundle_view import ArtifactResourcePressure +from lab.prism.candidate_codec import ( + SPOOL_READ_BYTES, + CandidateBodyIntegrityError, + SpoolCandidateBody, +) + +# Hard ceiling on the encoded size of any value decoded in-process with one +# ``json.loads``. A larger value is a lazy view, never a Python object. +SPOOL_VIEW_DECODE_BYTES = 1024 * 1024 +# Bytes of encoded string text decoded per ``json.loads`` while streaming. +SPOOL_VIEW_STRING_SLICE_BYTES = 64 * 1024 +# Bytes read from the spool per positional read while scanning or copying. +SPOOL_VIEW_READ_BYTES = SPOOL_READ_BYTES +# Item-index entries held in memory before the index spills to a scratch file. +SPOOL_VIEW_INDEX_MEMORY_ENTRIES = 256 +# Object member keys are cached in memory only while the object has at most +# this many members, each key at most this many encoded bytes. +SPOOL_VIEW_KEY_CACHE_ENTRIES = 1024 +SPOOL_VIEW_KEY_CACHE_BYTES = 4096 +# Top-level string fields hydrated as ``str`` whatever their size, each +# decoded in bounded slices and joined. Every one has a bound of its own +# that is not this reader's to impose -- the coinbase transaction and the +# hashes by consensus (the block), the extranonces by the Stratum session, +# the schema by the protocol -- and its consumers need the text whole (hex +# parsing, txid computation, identity checks). The raw block is absent on +# purpose: consensus bounds it too, but a submitter can stream it, so above +# the threshold it stays a streamed view. This is the explicit decoded +# fixed-metadata boundary; nothing outside it is ever joined implicitly. +SPOOL_DECODED_METADATA_FIELDS = frozenset( + { + "schema", + "block_hash_hex", + "parent_hash", + "coinbase_tx_hex", + "extranonce1_hex", + "extranonce2_hex", + } +) +# Pending index bytes buffered before they are written to the scratch file. +_INDEX_FLUSH_BYTES = 64 * 1024 + +_STRUCTURAL = re.compile(rb'["\\{}\[\],:]') +_HIGH_SURROGATE_TAIL = re.compile(r"\\u[dD][89abAB][0-9a-fA-F]{2}$") + +_QUOTE = 0x22 +_BACKSLASH = 0x5C +_COMMA = 0x2C +_COLON = 0x3A +_OPEN = (0x7B, 0x5B) +_CLOSE = (0x7D, 0x5D) + + +# -------------------------------------------------------------------------- +# bounded body access +# -------------------------------------------------------------------------- + + +def read_body_span(body: SpoolCandidateBody, start: int, end: int) -> bytes: + """Body bytes ``[start, end)``; a closed or unreadable body fails closed.""" + if end <= start: + return b"" + if not body.alive: + raise CandidateBodyIntegrityError("spool body is closed") + try: + return body.read_span(start, end) + except ValueError as exc: + raise CandidateBodyIntegrityError(f"spool span is out of range: {exc}") from exc + except OSError as exc: + raise CandidateBodyIntegrityError(f"spool body is unreadable: {exc}") from exc + + +def iter_body_bytes( + body: SpoolCandidateBody, + start: int, + end: int, + *, + chunk_bytes: int | None = None, +) -> Iterator[bytes]: + """Body bytes ``[start, end)`` in bounded pieces.""" + size = max(1, int(chunk_bytes if chunk_bytes is not None else SPOOL_VIEW_READ_BYTES)) + position = start + while position < end: + stop = min(end, position + size) + yield read_body_span(body, position, stop) + position = stop + + +def iter_body_text(body: SpoolCandidateBody, start: int, end: int) -> Iterator[str]: + """Body bytes ``[start, end)`` decoded as UTF-8 text, chunk by chunk.""" + decoder = codecs.getincrementaldecoder("utf-8")() + try: + for chunk in iter_body_bytes(body, start, end): + text = decoder.decode(chunk) + if text: + yield text + tail = decoder.decode(b"", final=True) + except UnicodeDecodeError as exc: + raise CandidateBodyIntegrityError(f"spool body is not valid UTF-8: {exc}") from exc + if tail: + yield tail + + +def _loads(data: bytes) -> Any: + try: + return json.loads(data) + except ValueError as exc: + raise CandidateBodyIntegrityError(f"spool value is not valid JSON: {exc}") from exc + + +# -------------------------------------------------------------------------- +# structural scanner +# -------------------------------------------------------------------------- + + +def _iter_top_level_tokens( + body: SpoolCandidateBody, + start: int, + end: int, +) -> Iterator[tuple[int, int]]: + """``(position, byte)`` of every ``,`` and ``:`` at depth 0 in ``[start, end)``. + + Quoted text (escape-aware, including an escape pair split by a read + boundary) and nested containers are skipped; multi-byte UTF-8 never + matches a structural byte, so raw UTF-8 text is safe. The range must + end outside any string or container. + """ + in_string = False + escape_at = -2 + depth = 0 + position = start + read_bytes = max(1, int(SPOOL_VIEW_READ_BYTES)) + while position < end: + stop = min(end, position + read_bytes) + piece = read_body_span(body, position, stop) + for match in _STRUCTURAL.finditer(piece): + at = position + match.start() + token = piece[match.start()] + if in_string: + if at == escape_at + 1: + escape_at = -2 + elif token == _BACKSLASH: + escape_at = at + elif token == _QUOTE: + in_string = False + elif token == _QUOTE: + in_string = True + elif token in _OPEN: + depth += 1 + elif token in _CLOSE: + depth -= 1 + if depth < 0: + raise CandidateBodyIntegrityError( + f"spool value has unmatched delimiters at byte {at}" + ) + elif depth == 0: + yield at, token + position = stop + if in_string or depth: + raise CandidateBodyIntegrityError("spool value ends inside a string or container") + + +def iter_item_spans( + body: SpoolCandidateBody, + start: int, + end: int, +) -> Iterator[tuple[int, int]]: + """``(item_start, item_end)`` of the comma-joined values in ``[start, end)``. + + The range is an array body without its brackets or a page of records + (the codec's page index points at exactly such ranges). An empty range + holds no items. + """ + if end <= start: + return + item_start = start + for at, token in _iter_top_level_tokens(body, start, end): + if token == _COMMA: + yield item_start, at + item_start = at + 1 + else: + raise CandidateBodyIntegrityError(f"spool array holds a stray colon at byte {at}") + yield item_start, end + + +def iter_member_spans( + body: SpoolCandidateBody, + start: int, + end: int, +) -> Iterator[tuple[int, int, int, int]]: + """``(key_start, key_end, value_start, value_end)`` of the members in ``[start, end)``. + + The range is an object body without its braces. An empty range holds + no members. + """ + if end <= start: + return + member_start = start + colon = -1 + for at, token in _iter_top_level_tokens(body, start, end): + if token == _COLON: + if colon >= 0: + raise CandidateBodyIntegrityError(f"spool object member has two colons at byte {at}") + colon = at + else: + if colon < 0: + raise CandidateBodyIntegrityError(f"spool object member has no colon at byte {at}") + yield member_start, colon, colon + 1, at + member_start = at + 1 + colon = -1 + if colon < 0: + raise CandidateBodyIntegrityError("spool object member has no colon") + yield member_start, colon, colon + 1, end + + +# -------------------------------------------------------------------------- +# disk-spilling offset index +# -------------------------------------------------------------------------- + + +class _OffsetIndex: + """Append-only fixed-width offset entries, bounded in memory. + + Entries stay in a bytearray up to :data:`SPOOL_VIEW_INDEX_MEMORY_ENTRIES` + and move to an unlinked scratch file beyond that, read back with + ``pread``. The scratch descriptor closes with the last owner and on + :meth:`close`; a scratch file that cannot be created or written is + resource pressure, not corruption. + """ + + __slots__ = ( + "__weakref__", + "_closed", + "_count", + "_file", + "_finalizer", + "_memory", + "_memory_entries", + "_pending", + "_struct", + "_written", + ) + + def __init__(self, fields: int, *, memory_entries: int | None = None) -> None: + self._struct = struct.Struct("<" + "Q" * int(fields)) + self._memory_entries = int( + SPOOL_VIEW_INDEX_MEMORY_ENTRIES if memory_entries is None else memory_entries + ) + self._memory = bytearray() + self._pending = bytearray() + self._count = 0 + self._written = 0 + self._file: Any = None + self._finalizer: weakref.finalize | None = None + self._closed = False + + @property + def count(self) -> int: + return self._count + + @property + def spilled(self) -> bool: + return self._file is not None + + def fileno(self) -> int | None: + return None if self._file is None else self._file.fileno() + + def append(self, *values: int) -> None: + if self._closed: + raise CandidateBodyIntegrityError("spool view index is closed") + packed = self._struct.pack(*values) + if self._file is None: + if self._count < self._memory_entries: + self._memory += packed + self._count += 1 + return + self._spill() + self._pending += packed + self._count += 1 + if len(self._pending) >= _INDEX_FLUSH_BYTES: + self._flush() + + def _spill(self) -> None: + try: + handle = tempfile.TemporaryFile() + except OSError as exc: + raise ArtifactResourcePressure( + f"cannot create the spool view index scratch file: {exc}" + ) from exc + self._file = handle + self._finalizer = weakref.finalize(self, handle.close) + self._pending = bytearray(self._memory) + self._memory = bytearray() + self._flush() + + def _flush(self) -> None: + if not self._pending: + return + if self._file is None or self._file.closed: + raise CandidateBodyIntegrityError("spool view index is closed") + data = memoryview(bytes(self._pending)) + offset = self._written * self._struct.size + fd = self._file.fileno() + while data: + try: + written = os.pwrite(fd, data, offset) + except InterruptedError: + continue + except OSError as exc: + raise ArtifactResourcePressure( + f"cannot write the spool view index scratch file: {exc}" + ) from exc + data = data[written:] + offset += written + self._written += len(self._pending) // self._struct.size + self._pending = bytearray() + + def get(self, index: int) -> tuple[int, ...]: + if not 0 <= index < self._count: + raise IndexError(index) + if self._closed: + raise CandidateBodyIntegrityError("spool view index is closed") + if self._file is None: + return self._struct.unpack_from(self._memory, index * self._struct.size) + self._flush() + if self._file.closed: + raise CandidateBodyIntegrityError("spool view index is closed") + position = index * self._struct.size + try: + raw = os.pread(self._file.fileno(), self._struct.size, position) + except OSError as exc: + raise ArtifactResourcePressure( + f"cannot read the spool view index scratch file: {exc}" + ) from exc + if len(raw) != self._struct.size: + raise CandidateBodyIntegrityError("spool view index is truncated") + return self._struct.unpack(raw) + + def close(self) -> None: + self._closed = True + self._memory = bytearray() + self._pending = bytearray() + if self._finalizer is not None: + self._finalizer() + + @property + def closed(self) -> bool: + return self._closed + + +# -------------------------------------------------------------------------- +# views +# -------------------------------------------------------------------------- + + +class _SpoolView: + """One byte range of a body, exposed verbatim.""" + + __slots__ = ("_body", "_end", "_start") + + def __init__(self, body: SpoolCandidateBody, start: int, end: int) -> None: + self._body = body + self._start = int(start) + self._end = int(end) + + @property + def body(self) -> SpoolCandidateBody: + return self._body + + @property + def byte_range(self) -> tuple[int, int]: + return self._start, self._end + + @property + def byte_length(self) -> int: + return self._end - self._start + + def iter_byte_chunks(self, *, chunk_bytes: int | None = None) -> Iterator[bytes]: + """The value's exact encoding as stored, in bounded pieces.""" + return iter_body_bytes(self._body, self._start, self._end, chunk_bytes=chunk_bytes) + + def iter_encoded_chunks(self) -> Iterator[str]: + """The value's JSON encoding as text, in bounded pieces.""" + return iter_body_text(self._body, self._start, self._end) + + def canonical_json_sha256(self) -> str: + digest = hashlib.sha256() + for chunk in self.iter_byte_chunks(): + digest.update(chunk) + return digest.hexdigest() + + def close(self) -> None: + """Release scratch resources this view owns; the body is untouched.""" + + __hash__ = None # type: ignore[assignment] + + +def _text_stream_equals_str(chunks: Iterator[str], other: str) -> bool: + offset = 0 + for chunk in chunks: + stop = offset + len(chunk) + if stop > len(other) or other[offset:stop] != chunk: + return False + offset = stop + return offset == len(other) + + +def _text_streams_equal(left: Iterator[str], right: Iterator[str]) -> bool: + left_buffer = "" + right_buffer = "" + left_done = right_done = False + while True: + while not left_buffer and not left_done: + try: + left_buffer = next(left) + except StopIteration: + left_done = True + while not right_buffer and not right_done: + try: + right_buffer = next(right) + except StopIteration: + right_done = True + if left_done or right_done: + return left_done and right_done and not left_buffer and not right_buffer + size = min(len(left_buffer), len(right_buffer)) + if left_buffer[:size] != right_buffer[:size]: + return False + left_buffer = left_buffer[size:] + right_buffer = right_buffer[size:] + + +class SpoolStringView(_SpoolView): + """A JSON string above the decode threshold, streamed from the body. + + ``iter_text_chunks`` yields the decoded value in pieces cut only at + points that split neither an escape sequence nor a UTF-16 surrogate + pair; ``iter_encoded_chunks`` and ``iter_byte_chunks`` yield the stored + encoding. Equality with a ``str`` streams the comparison. ``str()`` is + refused deliberately: a consumer that needs the text whole must say so + through :func:`materialize_spool_views` (or join the chunks itself) so + a repr can never be stored where the value belongs. + """ + + __slots__ = () + + def iter_text_chunks(self, *, slice_bytes: int | None = None) -> Iterator[str]: + return iter_json_string_text_chunks( + self._body, self._start, self._end, slice_bytes=slice_bytes + ) + + def text_length(self) -> int: + """Length in code points, computed by streaming.""" + return sum(len(chunk) for chunk in self.iter_text_chunks()) + + def __eq__(self, other: object) -> bool: + if other is self: + return True + if isinstance(other, str): + return _text_stream_equals_str(self.iter_text_chunks(), other) + if isinstance(other, SpoolStringView): + return _text_streams_equal(self.iter_text_chunks(), other.iter_text_chunks()) + return NotImplemented + + def __ne__(self, other: object) -> bool: + result = self.__eq__(other) + if result is NotImplemented: + return result # type: ignore[return-value] + return not result + + def __bool__(self) -> bool: + return True + + def __str__(self) -> str: + raise TypeError( + "SpoolStringView is a streamed string; iterate iter_text_chunks() " + "or call materialize_spool_views() to obtain the text" + ) + + def __repr__(self) -> str: + return f"SpoolStringView(bytes={self.byte_length})" + + __hash__ = None # type: ignore[assignment] + + +class SpoolRawValue(_SpoolView): + """An oversized non-string scalar token (a number or literal). + + This encoder never produces one (integers are bounded by the + interpreter's digit limit), so it exists only so an unexpected body is + reported faithfully rather than refused: the token is reachable through + its encoding and equality compares encodings. + """ + + __slots__ = () + + def __eq__(self, other: object) -> bool: + if other is self: + return True + if isinstance(other, SpoolRawValue): + return _text_streams_equal(self.iter_encoded_chunks(), other.iter_encoded_chunks()) + return NotImplemented + + def __ne__(self, other: object) -> bool: + result = self.__eq__(other) + if result is NotImplemented: + return result # type: ignore[return-value] + return not result + + def __repr__(self) -> str: + return f"SpoolRawValue(bytes={self.byte_length})" + + __hash__ = None # type: ignore[assignment] + + +class _ObjectItemsView(ItemsView): + def __iter__(self) -> Iterator[tuple[str, Any]]: + return self._mapping._iter_items() # type: ignore[attr-defined] + + +class _ObjectValuesView(ValuesView): + def __iter__(self) -> Iterator[Any]: + for _key, value in self._mapping._iter_items(): # type: ignore[attr-defined] + yield value + + +class SpoolObjectView(_SpoolView, Mapping): + """A JSON object above the decode threshold, decoded member by member. + + The member index (key and value byte spans) is built by one structural + scan on first access and kept bounded in memory; keys are cached only + for objects with few, short keys, and otherwise looked up by scanning. + Each member value follows the module's decode policy, so a member can + itself be a view. Equality with a mapping compares member by member. + """ + + __slots__ = ("_index", "_keys", "_lock") + + def __init__(self, body: SpoolCandidateBody, start: int, end: int) -> None: + super().__init__(body, start, end) + self._index: _OffsetIndex | None = None + self._keys: dict[str, int] | None = None + self._lock = threading.Lock() + + def _ensure_indexed(self) -> _OffsetIndex: + index = self._index + if index is not None: + return index + with self._lock: + if self._index is not None: + return self._index + index = _OffsetIndex(4) + keys: dict[str, int] | None = {} + try: + for ordinal, (key_start, key_end, value_start, value_end) in enumerate( + iter_member_spans(self._body, self._start + 1, self._end - 1) + ): + index.append(key_start, key_end, value_start, value_end) + if keys is not None: + if ( + ordinal >= SPOOL_VIEW_KEY_CACHE_ENTRIES + or key_end - key_start > SPOOL_VIEW_KEY_CACHE_BYTES + ): + keys = None + else: + keys[self._decode_key(key_start, key_end)] = ordinal + except BaseException: + index.close() + raise + self._index = index + self._keys = keys + return index + + def _decode_key(self, start: int, end: int) -> str: + return decode_json_string(self._body, start, end) + + def _value_at(self, index: _OffsetIndex, ordinal: int) -> Any: + _key_start, _key_end, value_start, value_end = index.get(ordinal) + return decode_body_span(self._body, value_start, value_end) + + def _ordinal_of(self, key: str) -> int | None: + index = self._ensure_indexed() + if self._keys is not None: + return self._keys.get(key) + found: int | None = None + for ordinal in range(index.count): + key_start, key_end, _value_start, _value_end = index.get(ordinal) + if SpoolStringView(self._body, key_start, key_end) == key: + found = ordinal + return found + + def _iter_items(self) -> Iterator[tuple[str, Any]]: + index = self._ensure_indexed() + for ordinal in range(index.count): + key_start, key_end, value_start, value_end = index.get(ordinal) + yield self._decode_key(key_start, key_end), decode_body_span( + self._body, value_start, value_end + ) + + def __getitem__(self, key: str) -> Any: + if not isinstance(key, str): + raise KeyError(key) + ordinal = self._ordinal_of(key) + if ordinal is None: + raise KeyError(key) + return self._value_at(self._ensure_indexed(), ordinal) + + def __iter__(self) -> Iterator[str]: + index = self._ensure_indexed() + for ordinal in range(index.count): + key_start, key_end, _value_start, _value_end = index.get(ordinal) + yield self._decode_key(key_start, key_end) + + def __len__(self) -> int: + return self._ensure_indexed().count + + def __contains__(self, key: object) -> bool: + return isinstance(key, str) and self._ordinal_of(key) is not None + + def items(self) -> ItemsView: + return _ObjectItemsView(self) + + def values(self) -> ValuesView: + return _ObjectValuesView(self) + + def __eq__(self, other: object) -> bool: + if other is self: + return True + if not isinstance(other, Mapping): + return NotImplemented + if len(self) != len(other): + return False + for key, value in self._iter_items(): + if key not in other: + return False + if value != other[key]: + return False + return True + + def __ne__(self, other: object) -> bool: + result = self.__eq__(other) + if result is NotImplemented: + return result # type: ignore[return-value] + return not result + + def __repr__(self) -> str: + members = "?" if self._index is None else str(self._index.count) + return f"SpoolObjectView(bytes={self.byte_length}, members={members})" + + def close(self) -> None: + with self._lock: + index = self._index + if index is not None: + index.close() + + __hash__ = None # type: ignore[assignment] + + +class SpoolArrayView(_SpoolView, Sequence): + """A JSON array above the decode threshold, decoded item by item. + + The item index is built by one structural scan on first access and + kept bounded in memory (spilling to a scratch file). Items follow the + module's decode policy. ``canonical_share_pages`` hands the stored item + bytes to the codec's verbatim re-staging path, and equality with a + sequence compares item by item. + """ + + __slots__ = ("_index", "_lock") + + def __init__(self, body: SpoolCandidateBody, start: int, end: int) -> None: + super().__init__(body, start, end) + self._index: _OffsetIndex | None = None + self._lock = threading.Lock() + + def _ensure_indexed(self) -> _OffsetIndex: + index = self._index + if index is not None: + return index + with self._lock: + if self._index is not None: + return self._index + index = _OffsetIndex(2) + try: + for item_start, item_end in iter_item_spans( + self._body, self._start + 1, self._end - 1 + ): + index.append(item_start, item_end) + except BaseException: + index.close() + raise + self._index = index + return index + + def __len__(self) -> int: + return self._ensure_indexed().count + + def __iter__(self) -> Iterator[Any]: + index = self._ensure_indexed() + for ordinal in range(index.count): + item_start, item_end = index.get(ordinal) + yield decode_body_span(self._body, item_start, item_end) + + def __getitem__(self, index: int | slice) -> Any: + count = len(self) + if isinstance(index, slice): + start, stop, step = index.indices(count) + return tuple(self[position] for position in range(start, stop, step)) + if isinstance(index, bool) or not isinstance(index, int): + raise TypeError("spool array indices must be integers or slices") + resolved = index + count if index < 0 else index + if not 0 <= resolved < count: + raise IndexError(index) + item_start, item_end = self._ensure_indexed().get(resolved) + return decode_body_span(self._body, item_start, item_end) + + def canonical_share_pages(self) -> Iterator[tuple[bytes, int | None]]: + """The stored item bytes, verbatim, for the codec's re-staging path.""" + for chunk in iter_body_bytes(self._body, self._start + 1, self._end - 1): + yield chunk, None + + def __eq__(self, other: object) -> bool: + if other is self: + return True + if isinstance(other, (str, bytes, bytearray)) or not isinstance(other, Sequence): + return NotImplemented + if len(self) != len(other): + return False + other_iter = iter(other) + for item in self: + try: + candidate = next(other_iter) + except StopIteration: + return False + if item != candidate: + return False + return True + + def __ne__(self, other: object) -> bool: + result = self.__eq__(other) + if result is NotImplemented: + return result # type: ignore[return-value] + return not result + + def __repr__(self) -> str: + items = "?" if self._index is None else str(self._index.count) + return f"SpoolArrayView(bytes={self.byte_length}, items={items})" + + def close(self) -> None: + with self._lock: + index = self._index + if index is not None: + index.close() + + __hash__ = None # type: ignore[assignment] + + +# -------------------------------------------------------------------------- +# decode policy +# -------------------------------------------------------------------------- + + +def decode_body_span(body: SpoolCandidateBody, start: int, end: int) -> Any: + """One JSON value at ``[start, end)``: plain when small, a view when not. + + A value within :data:`SPOOL_VIEW_DECODE_BYTES` is one bounded + ``json.loads``; a larger one is classified by its first and last byte + without reading anything in between. + """ + size = end - start + if size <= 0: + raise CandidateBodyIntegrityError("spool value is empty") + if size <= SPOOL_VIEW_DECODE_BYTES: + return _loads(read_body_span(body, start, end)) + first = read_body_span(body, start, start + 1)[0] + last = read_body_span(body, end - 1, end)[0] + if first == 0x7B and last == 0x7D: + return SpoolObjectView(body, start, end) + if first == 0x5B and last == 0x5D: + return SpoolArrayView(body, start, end) + if first == _QUOTE and last == _QUOTE: + return SpoolStringView(body, start, end) + if first in (0x7B, 0x5B, _QUOTE) or last in (0x7D, 0x5D, _QUOTE): + raise CandidateBodyIntegrityError("spool value delimiters do not match") + return SpoolRawValue(body, start, end) + + +def _escape_is_real(text: str, at: int) -> bool: + """The backslash at ``at`` starts an escape (an even run precedes it).""" + preceding = at - len(text[:at].rstrip("\\")) + return preceding % 2 == 0 + + +def _safe_cut(text: str) -> int: + """Largest prefix of encoded string text that ends no escape mid-way. + + Three cuts, applied in order: an odd trailing backslash run leaves its + last backslash for the next piece; an incomplete ``\\uXXXX`` in the last + five characters moves whole; and a complete high-surrogate escape at the + end moves whole so its low half is decoded in the same ``json.loads`` + (two lone surrogates concatenated are not the code point they encode). + """ + cut = len(text) + run = cut - len(text.rstrip("\\")) + if run % 2 == 1: + cut -= 1 + else: + at = text.rfind("\\u", max(0, cut - 5)) + if at >= 0 and _escape_is_real(text, at): + cut = at + head = text[:cut] + match = _HIGH_SURROGATE_TAIL.search(head) + if match is not None and _escape_is_real(head, match.start()): + cut = match.start() + return cut + + +def iter_json_string_text_chunks( + body: SpoolCandidateBody, + start: int, + end: int, + *, + slice_bytes: int | None = None, +) -> Iterator[str]: + """Decoded text of the JSON string at ``[start, end)`` (quotes included). + + Each piece is one ``json.loads`` over at most ``slice_bytes`` of encoded + text plus a short carry; raw UTF-8 is decoded incrementally so a read + boundary never splits a code point either. + """ + # Clamped to the decode threshold so no call exceeds it however the + # knobs are set; at least 32 so a carried escape always makes progress. + size = max( + 32, + min( + int(slice_bytes if slice_bytes is not None else SPOOL_VIEW_STRING_SLICE_BYTES), + int(SPOOL_VIEW_DECODE_BYTES), + ), + ) + if end - start < 2: + raise CandidateBodyIntegrityError("spool string span is too short") + if read_body_span(body, start, start + 1) != b'"' or read_body_span(body, end - 1, end) != b'"': + raise CandidateBodyIntegrityError("spool string span is not quoted") + decoder = codecs.getincrementaldecoder("utf-8")() + position = start + 1 + stop = end - 1 + carry = "" + while position < stop: + read_to = min(stop, position + size) + try: + text = carry + decoder.decode(read_body_span(body, position, read_to), final=read_to >= stop) + except UnicodeDecodeError as exc: + raise CandidateBodyIntegrityError(f"spool string is not valid UTF-8: {exc}") from exc + position = read_to + if position < stop: + cut = _safe_cut(text) + carry = text[cut:] + text = text[:cut] + else: + carry = "" + if text: + try: + piece = json.loads('"' + text + '"') + except ValueError as exc: + raise CandidateBodyIntegrityError(f"spool string is not valid JSON: {exc}") from exc + yield piece + if carry: + raise CandidateBodyIntegrityError("spool string ends inside an escape") + + +def decode_json_string(body: SpoolCandidateBody, start: int, end: int) -> str: + """The whole text of the JSON string at ``[start, end)``. + + The explicit materialization boundary for callers that need the value + whole (for example a bounded metadata field); every C call stays a slice. + """ + return "".join(iter_json_string_text_chunks(body, start, end)) + + +# -------------------------------------------------------------------------- +# consumer helpers +# -------------------------------------------------------------------------- + + +def is_spool_view(value: object) -> bool: + return isinstance(value, _SpoolView) + + +def iter_string_text_chunks(value: Any) -> Iterator[str]: + """Text of a ``str`` or a :class:`SpoolStringView`, chunk by chunk. + + The one call a consumer needs to accept either shape (a COPY writer + streaming ``share_id`` values, a hashing loop). + """ + if isinstance(value, str): + if value: + yield value + return + if isinstance(value, SpoolStringView): + yield from value.iter_text_chunks() + return + raise TypeError(f"expected a string or a SpoolStringView, not {type(value).__name__}") + + +def materialize_spool_views(value: Any) -> Any: + """A plain Python object graph with every view decoded whole. + + The explicit boundary for consumers that must hold a value entire. + Strings are joined from their chunks, containers rebuilt member by + member; nothing here is bounded by anything but the value's own size, + which is why the call is explicit. + """ + if isinstance(value, SpoolStringView): + return "".join(value.iter_text_chunks()) + if isinstance(value, SpoolRawValue): + return json.loads("".join(value.iter_encoded_chunks())) + if isinstance(value, Mapping): + return {str(key): materialize_spool_views(item) for key, item in value.items()} + if isinstance(value, (str, bytes, bytearray)) or not isinstance(value, Sequence): + return value + return [materialize_spool_views(item) for item in value] + + +def close_spool_views(value: Any) -> None: + """Release the scratch resources of every view reachable from ``value``. + + Walks mappings and non-string sequences shallowly enough to find the + views a hydrated intent's facts hold; a page-indexed share sequence is + left alone (it owns nothing but the body reference). + """ + if isinstance(value, _SpoolView): + value.close() + return + if isinstance(value, Mapping): + for item in list(value.values()) if isinstance(value, dict) else (): + close_spool_views(item) + return + if isinstance(value, (list, tuple)): + for item in value: + close_spool_views(item) + + +__all__ = [ + "SPOOL_DECODED_METADATA_FIELDS", + "SPOOL_VIEW_DECODE_BYTES", + "SPOOL_VIEW_INDEX_MEMORY_ENTRIES", + "SPOOL_VIEW_KEY_CACHE_BYTES", + "SPOOL_VIEW_KEY_CACHE_ENTRIES", + "SPOOL_VIEW_READ_BYTES", + "SPOOL_VIEW_STRING_SLICE_BYTES", + "SpoolArrayView", + "SpoolObjectView", + "SpoolRawValue", + "SpoolStringView", + "close_spool_views", + "decode_body_span", + "decode_json_string", + "is_spool_view", + "iter_body_bytes", + "iter_body_text", + "iter_item_spans", + "iter_json_string_text_chunks", + "iter_member_spans", + "iter_string_text_chunks", + "materialize_spool_views", + "read_body_span", +] diff --git a/lab/prism/candidate_store.py b/lab/prism/candidate_store.py new file mode 100644 index 000000000..82fbabc80 --- /dev/null +++ b/lab/prism/candidate_store.py @@ -0,0 +1,1687 @@ +#!/usr/bin/env python3 +"""Chunked candidate body storage: statements, hydration, legacy lane, janitor. + +Issue #255. The ledger's candidate rows used to carry the whole intent as +one ``jsonb`` value, so every durable write inlined a payout-window-sized +literal, every replay page ``json_agg``-ed complete candidates, and psycopg +decoded them in one GIL-held call. This module owns the storage side of the +bounded replacement: + +* **Staging.** A body is uploaded as fixed byte chunks into + ``qbit_block_candidate_body_chunk`` under a manifest row whose owner is + the exact producer session, with its page index in + ``qbit_block_candidate_body_span``/``_page`` written in bounded batches. + Staging grants nothing: no share credit, no replay eligibility, no + lease-row touch. The seal statement locks the manifest row, proves + completeness from the chunk and index rows themselves and re-hashes + every chunk on the server; from then on the database's own triggers + refuse any chunk insert, update or delete until authoritative + retirement. +* **Publication.** The ledger's fenced statements (share append, standalone + intent persist) reference a sealed manifest by id and compare-and-swap on + the outbox row's observed version; the exact-body precomparison runs + here, outside that critical section, as a paged server-side chunk digest + check. +* **Metadata-first replay.** ``header_page_sql`` projects only a bounded + typed summary per row (``qbit_prism_bounded_replay_header``), at most + :data:`HEADER_PAGE_MAX_ROWS` rows and :data:`HEADER_PAGE_MAX_BYTES` per + page, and answers exhaustion explicitly. +* **Hydration.** :class:`CandidateBodyHydrator` reads the scalar manifest, + the span rows and the page rows in bounded pages (the page index goes + straight to disk), then at most :data:`BODY_READ_MAX_CHUNKS` chunks per + statement, verifying state, digests and a rolling body digest as it + goes. A body that turns terminal or corrupt mid-read is discarded. +* **Legacy lane.** Pending v1 ``jsonb`` rows are converted by an isolated + helper process (``python -m lab.prism.candidate_store``) that owns its + own database connection, cannot take the writer lease, is memory- and + deadline-limited, and hands back a spool/index pair plus a scalar + manifest, never a Python object graph. +* **Janitor.** Two separate statements: a tiny fenced compare-and-swap that + retires orphaned staging/sealed bodies (``retire_orphan_bodies_sql``), + and a lease-free bounded deletion of one retired body's chunks a page at + a time (``reap_retired_chunks_sql``). ``retired`` is permanent, and a + publication can only reference a ``sealed`` body, so nothing revives a + body once it is reapable. +""" + +from __future__ import annotations + +import base64 +import binascii +import json +import os +import shlex +import subprocess +import sys +import tempfile +import threading +import time +import uuid +from collections.abc import Callable, Mapping +from dataclasses import dataclass, field +from pathlib import Path +from typing import Any + +from lab.prism.candidate_codec import ( + CANDIDATE_BODY_CHUNK_BYTES, + CANDIDATE_BODY_STORAGE_VERSION, + INDEX_PAGE_ENTRIES, + INDEX_SPAN_ROWS, + LEGACY_CANDIDATE_STORAGE_VERSION, + BodyChunk, + CandidateBody, + CandidateBodyIntegrityError, + CandidateBodyManifest, + FieldSpan, + PageEntry, + PreparedCandidateIntent, + SpoolWriter, + jsonb_equivalent, + legacy_json_to_spool, + prepared_intent_from_spool, +) + +# Chunks per body read statement; three 256 KiB chunks base64-encode to +# exactly 1 MiB of returned text, which is the transport ceiling. +BODY_READ_MAX_CHUNKS = 3 +BODY_READ_MAX_BYTES = 1024 * 1024 +# Base64 text for one chunk: 4/3 of the bytes plus one newline per 76 chars. +CHUNK_BASE64_MAX_CHARS = ((CANDIDATE_BODY_CHUNK_BYTES + 2) // 3) * 4 + CANDIDATE_BODY_CHUNK_BYTES // 57 + 2 +# Header enumeration page: rows and running header bytes. The byte cap is +# the bound that matters for the native decoder (every header is a bounded +# projection, so a 256 KiB page holds a few hundred of them at most); the +# row cap matches the configured replay page maximum rather than a smaller +# number, because every page costs the collapse walk two chain tip reads and +# a probe budget, and a 64-row cap multiplied a storm's chain reads by +# sixteen in the existing cost contracts. +HEADER_PAGE_MAX_ROWS = 1024 +HEADER_PAGE_MAX_BYTES = 256 * 1024 +# Per-header ceiling enforced on read; the bounded projection is far below. +HEADER_MAX_BYTES = 8 * 1024 +# Local spool admission for hydrated bodies (operator-reviewed reservation). +DEFAULT_SPOOL_RESERVATION_BYTES = 4 * 1024 * 1024 * 1024 +# Chunk deletions per janitor statement and the age after which an +# unreferenced staging/sealed body is an orphan. +JANITOR_CHUNKS_PER_STEP = 64 +STALE_STAGING_SECONDS = 15 * 60 +# Legacy helper process limits. +LEGACY_HELPER_MEMORY_BYTES = 4 * 1024 * 1024 * 1024 +LEGACY_HELPER_TIMEOUT_SECONDS = 600.0 +LEGACY_HELPER_POLL_SECONDS = 0.2 +# The schema capability this code understands; a database that declares a +# higher one is newer than this process and must not be written by it. +CANDIDATE_SCHEMA_CAPABILITY = "candidate_storage_version" + +CHUNK_LITERAL_TAG = "qbit_prism_chunk" + + +class CandidateStorageError(RuntimeError): + """A storage operation failed in a way that is not a transport retry.""" + + +class CandidateBodyUnavailable(CandidateStorageError): + """The body no longer exists as a sealed, referenced manifest.""" + + +class SpoolAdmissionExhausted(CandidateStorageError): + """Local spool reservation would be exceeded; backpressure, not loss.""" + + +class IncompatibleCandidateSchema(CandidateStorageError): + """The database's candidate schema is newer than this process, or unmigrated.""" + + +def new_body_id() -> str: + return uuid.uuid4().hex + + +# -------------------------------------------------------------------------- +# typed rows +# -------------------------------------------------------------------------- + + +@dataclass(frozen=True) +class CandidateBodyRef: + """Small reference to a stored body, carried by a header row.""" + + body_id: str + storage_version: int + candidate_sha256: str + byte_count: int + chunk_count: int + chunk_bytes: int + share_count: int + + @classmethod + def from_row(cls, row: Mapping[str, Any]) -> CandidateBodyRef: + body_id = row.get("body_id") + if not isinstance(body_id, str) or len(body_id) != 32: + raise ValueError("candidate body reference carries no body id") + return cls( + body_id=body_id, + storage_version=int(row.get("storage_version", CANDIDATE_BODY_STORAGE_VERSION)), + candidate_sha256=str(row["candidate_sha256"]), + byte_count=int(row["byte_count"]), + chunk_count=int(row["chunk_count"]), + chunk_bytes=int(row["chunk_bytes"]), + share_count=int(row["share_count"]), + ) + + def to_json(self) -> dict[str, Any]: + return { + "body_id": self.body_id, + "storage_version": self.storage_version, + "candidate_sha256": self.candidate_sha256, + "byte_count": self.byte_count, + "chunk_count": self.chunk_count, + "chunk_bytes": self.chunk_bytes, + "share_count": self.share_count, + } + + +@dataclass(frozen=True) +class CandidateHeaderPage: + """One metadata-first replay page with an explicit completeness fact.""" + + rows: tuple[dict[str, Any], ...] + next_cursor: Any + exhausted: bool + fetched: int + truncated_by_bytes: bool + + def __iter__(self): # noqa: ANN204 - Sequence-like convenience + return iter(self.rows) + + def __len__(self) -> int: + return len(self.rows) + + def __getitem__(self, index: int) -> dict[str, Any]: + return self.rows[index] + + +@dataclass(frozen=True) +class DurableCandidateDescriptor: + """Everything replay needs about a pending row before its body exists. + + Built from one header row after stale collapse and strict typing. The + descriptor is what the bounded replay window holds; the body is + hydrated only when the descriptor is dequeued. + """ + + block_hash: str + block_height: int + parent_hash: str + candidate_sha256: str + storage_version: int + credit_share_on_accept: bool + collection_only: bool + # None only when the header is oversized: the fact was deferred to the + # body, which hydration reads (``replay_header_from_fields``). + username: str | None + pending_share: dict[str, Any] + accepted_at_present: bool + accepted_at_ms: Any + body: CandidateBodyRef | None + cursor: Any + # The header row this descriptor was built from, handed back to the + # ledger at hydration time (it carries the body reference and the + # original pending stamp). + row: dict[str, Any] = field(default_factory=dict, compare=False, repr=False) + header_oversized: bool = False + attempts: int = 0 + not_before_monotonic: float = 0.0 + + @property + def submission(self) -> Any: + """Duck-typed hash carrier for registries that read ``.submission``.""" + from types import SimpleNamespace + + return SimpleNamespace(block_hash_hex=self.block_hash) + + @property + def body_key(self) -> str: + return self.body.body_id if self.body is not None else "legacy" + + +# -------------------------------------------------------------------------- +# statement builders +# -------------------------------------------------------------------------- + +Literal = Callable[[str], str] +JsonbLiteral = Callable[[object], str] + + +def chunk_hex_literal(data: bytes) -> str: + return f"decode(${CHUNK_LITERAL_TAG}${data.hex()}${CHUNK_LITERAL_TAG}$, 'hex')" + + +def schema_capability_sql() -> str: + """Read the candidate storage version the schema declares.""" + return f""" +SELECT json_build_object( + 'declared', ( + SELECT capability_value + FROM qbit_prism_schema_capabilities + WHERE capability = '{CANDIDATE_SCHEMA_CAPABILITY}' + ) +); +""" + + +def candidate_schema_refusal(declared: int | None, has_body_table: bool) -> str | None: + """Why this process must not use the database's candidate schema, or None. + + Every storage version needs the 002 migration: replay projects headers + through its function and the terminal outbox statements retire body + references in its columns, so ``PRISM_CANDIDATE_STORAGE_VERSION=1`` + stops version-2 writes but never makes an unmigrated database usable. + The capability row is declared last, so its absence also marks a + partially applied migration. A declared version above this process's + is the rollback floor. Shared by the coordinator boot check and the + offline recovery reader. + """ + if declared is not None and declared > CANDIDATE_BODY_STORAGE_VERSION: + return ( + f"database declares candidate storage version {declared}; this " + f"process understands up to {CANDIDATE_BODY_STORAGE_VERSION} and " + "must not write to it (rollback floor)" + ) + if not has_body_table or declared is None or declared < CANDIDATE_BODY_STORAGE_VERSION: + return ( + "the database lacks the 002_candidate_bodies.sql migration; apply it " + "(PRISM_POSTGRES_INIT_SCHEMA=1 does so at startup) before starting " + "this release. PRISM_CANDIDATE_STORAGE_VERSION=1 only stops " + "version-2 writes: replay and terminal outbox statements use the " + "migrated schema for every storage version" + ) + return None + + +def stage_body_sql( + payload: Mapping[str, Any], + *, + jsonb: JsonbLiteral, +) -> str: + """Create one staging manifest; idempotent on ``body_id``. + + No lease CTE: staging is private preparation and must not touch the + lease row. The producer identity is recorded so chunk uploads and the + seal can require the same session, and so an orphan can be identified + by the janitor. + """ + return f""" +WITH input AS ( + SELECT {jsonb(payload)} AS data +), +inserted AS ( + INSERT INTO qbit_block_candidate_body ( + body_id, storage_version, block_hash, candidate_sha256, + byte_count, chunk_count, chunk_bytes, share_count, + shares_offset, shares_end, span_count, page_count, + staging_writer_id, staging_writer_epoch, staging_session_token + ) + SELECT + data->>'body_id', (data->>'storage_version')::integer, + data->>'block_hash', data->>'candidate_sha256', + (data->>'byte_count')::bigint, (data->>'chunk_count')::integer, + (data->>'chunk_bytes')::integer, (data->>'share_count')::bigint, + (data->>'shares_offset')::bigint, (data->>'shares_end')::bigint, + (data->>'span_count')::integer, (data->>'page_count')::bigint, + data->>'writer_id', (data->>'writer_epoch')::bigint, + data->>'writer_session_token' + FROM input + ON CONFLICT (body_id) DO NOTHING + RETURNING body_id +) +SELECT json_build_object( + 'staged', (SELECT count(*) FROM inserted), + -- A row this statement inserted is invisible to its own final SELECT + -- (one snapshot per statement), so the fresh case is answered from the + -- RETURNING CTE. + 'state', COALESCE( + ( + SELECT state FROM qbit_block_candidate_body body, input + WHERE body.body_id = input.data->>'body_id' + ), + CASE WHEN EXISTS (SELECT 1 FROM inserted) THEN 'staging' END + ) +); +""" + + +def _owner_cte(alias: str = "owner") -> str: + return f""" +{alias} AS ( + SELECT body.body_id + FROM qbit_block_candidate_body body, input + WHERE body.body_id = input.data->>'body_id' + AND body.state = 'staging' + AND body.staging_writer_id = input.data->>'writer_id' + AND body.staging_writer_epoch = (input.data->>'writer_epoch')::bigint + AND body.staging_session_token = input.data->>'writer_session_token' +)""" + + +def body_chunk_sql( + payload: Mapping[str, Any], + data: bytes, + *, + jsonb: JsonbLiteral, +) -> str: + """Upload one chunk; idempotent on ``(body_id, ordinal)``. + + The insert is guarded by the manifest still being in ``staging`` under + the uploading session (and, independently, by the chunk table's + trigger), so a sealed or retired body can never gain a chunk and a + foreign session cannot fill someone else's staging. + """ + return f""" +WITH input AS ( + SELECT {jsonb(payload)} AS data +),{_owner_cte()}, +inserted AS ( + INSERT INTO qbit_block_candidate_body_chunk (body_id, ordinal, chunk, chunk_sha256) + SELECT + owner.body_id, (input.data->>'ordinal')::integer, + {chunk_hex_literal(data)}, input.data->>'chunk_sha256' + FROM owner, input + ON CONFLICT (body_id, ordinal) DO NOTHING + RETURNING ordinal +) +SELECT json_build_object( + 'owned', EXISTS (SELECT 1 FROM owner), + 'inserted', (SELECT count(*) FROM inserted) +); +""" + + +def body_index_rows_sql( + payload: Mapping[str, Any], + *, + jsonb: JsonbLiteral, +) -> str: + """Upload one bounded batch of span rows and/or page rows. + + ``payload['spans']`` holds at most :data:`INDEX_SPAN_ROWS` span objects + and ``payload['pages']`` at most :data:`INDEX_PAGE_ENTRIES` page arrays + ``[field, ordinal, offset, record_index]``. Idempotent on the primary + keys, and owner-guarded exactly like a chunk upload. + """ + return f""" +WITH input AS ( + SELECT {jsonb(payload)} AS data +),{_owner_cte()}, +inserted_spans AS ( + INSERT INTO qbit_block_candidate_body_span ( + body_id, field, kind, start_offset, end_offset, item_count, page_count, pages_exact + ) + SELECT + owner.body_id, span->>'field', span->>'kind', + (span->>'start')::bigint, (span->>'end')::bigint, + (span->>'item_count')::bigint, (span->>'page_count')::bigint, + (span->>'pages_exact')::boolean + FROM owner, input, jsonb_array_elements(COALESCE(input.data->'spans', '[]'::jsonb)) AS span + ON CONFLICT (body_id, field) DO NOTHING + RETURNING field +), +inserted_pages AS ( + INSERT INTO qbit_block_candidate_body_page ( + body_id, field, page_ordinal, body_offset, record_index + ) + SELECT + owner.body_id, page->>0, (page->>1)::bigint, (page->>2)::bigint, (page->>3)::bigint + FROM owner, input, jsonb_array_elements(COALESCE(input.data->'pages', '[]'::jsonb)) AS page + ON CONFLICT (body_id, field, page_ordinal) DO NOTHING + RETURNING page_ordinal +) +SELECT json_build_object( + 'owned', EXISTS (SELECT 1 FROM owner), + 'spans', (SELECT count(*) FROM inserted_spans), + 'pages', (SELECT count(*) FROM inserted_pages) +); +""" + + +def seal_body_sql( + payload: Mapping[str, Any], + *, + jsonb: JsonbLiteral, +) -> str: + """Seal a complete, verified staging body under its producer session. + + The manifest row is locked (``FOR UPDATE``) so a concurrent chunk or + index upload serializes behind the seal and then finds the body sealed + (its trigger refuses the insert). Completeness is proven from the rows + themselves (chunk count, byte sum, ordinal range, span and page counts) + and integrity by re-hashing every stored chunk on the server against + the digest the uploader recorded. Nothing here trusts a client-side + upload count. + """ + return f""" +WITH input AS ( + SELECT {jsonb(payload)} AS data +), +target AS ( + SELECT body.* + FROM qbit_block_candidate_body body, input + WHERE body.body_id = input.data->>'body_id' + FOR UPDATE +), +observed AS ( + SELECT + count(*) AS chunk_count, + COALESCE(sum(octet_length(chunk)), 0) AS byte_count, + COALESCE(min(ordinal), 0) AS min_ordinal, + COALESCE(max(ordinal), -1) AS max_ordinal, + COALESCE(bool_and(sha256(chunk) = decode(chunk_sha256, 'hex')), true) AS digests_ok, + COALESCE(bool_and( + octet_length(chunk) = (SELECT chunk_bytes FROM target) + OR ordinal = (SELECT chunk_count FROM target) - 1 + ), true) AS lengths_ok + FROM qbit_block_candidate_body_chunk chunk + WHERE chunk.body_id = (SELECT body_id FROM target) +), +observed_index AS ( + SELECT + (SELECT count(*) FROM qbit_block_candidate_body_span span + WHERE span.body_id = (SELECT body_id FROM target)) AS span_count, + (SELECT count(*) FROM qbit_block_candidate_body_page page + WHERE page.body_id = (SELECT body_id FROM target)) AS page_count +), +complete AS ( + SELECT target.body_id + FROM target, observed, observed_index, input + WHERE target.state = 'staging' + AND target.staging_writer_id = input.data->>'writer_id' + AND target.staging_writer_epoch = (input.data->>'writer_epoch')::bigint + AND target.staging_session_token = input.data->>'writer_session_token' + AND target.candidate_sha256 = input.data->>'candidate_sha256' + AND observed.chunk_count = target.chunk_count + AND observed.byte_count = target.byte_count + AND (target.chunk_count = 0 OR observed.min_ordinal = 0) + AND observed.max_ordinal = target.chunk_count - 1 + AND observed.digests_ok + AND observed.lengths_ok + AND observed_index.span_count = target.span_count + AND observed_index.page_count = target.page_count +), +sealed AS ( + UPDATE qbit_block_candidate_body + SET state = 'sealed', sealed_at = clock_timestamp() + FROM complete + WHERE qbit_block_candidate_body.body_id = complete.body_id + RETURNING qbit_block_candidate_body.body_id +) +SELECT json_build_object( + 'sealed', (SELECT count(*) FROM sealed), + 'state', (SELECT state FROM target), + 'observed_chunk_count', (SELECT chunk_count FROM observed), + 'observed_byte_count', (SELECT byte_count FROM observed), + 'observed_span_count', (SELECT span_count FROM observed_index), + 'observed_page_count', (SELECT page_count FROM observed_index), + 'digests_ok', (SELECT digests_ok FROM observed), + 'lengths_ok', (SELECT lengths_ok FROM observed) +); +""" + + +def retire_staging_body_sql(body_id: str, *, text: Literal) -> str: + """Hand a failed staging body to the janitor (no lease; own body only).""" + return f""" +WITH retired AS ( + UPDATE qbit_block_candidate_body + SET state = 'retired', retired_at = clock_timestamp() + WHERE body_id = {text(body_id)} + AND state = 'staging' + RETURNING body_id +) +SELECT json_build_object('retired', (SELECT count(*) FROM retired)); +""" + + +def body_manifest_sql(body_id: str, *, text: Literal) -> str: + """Read one scalar manifest with its live state and reference fact.""" + return f""" +SELECT json_build_object( + 'found', EXISTS (SELECT 1 FROM qbit_block_candidate_body WHERE body_id = {text(body_id)}), + 'manifest', ( + SELECT json_build_object( + 'body_id', body_id, + 'storage_version', storage_version, + 'block_hash', block_hash, + 'candidate_sha256', candidate_sha256, + 'byte_count', byte_count, + 'chunk_count', chunk_count, + 'chunk_bytes', chunk_bytes, + 'share_count', share_count, + 'shares_offset', shares_offset, + 'shares_end', shares_end, + 'span_count', span_count, + 'page_count', page_count, + 'state', state, + 'referenced', EXISTS ( + SELECT 1 FROM qbit_block_candidate_outbox outbox + WHERE outbox.body_id = body.body_id AND outbox.state = 'pending' + ) + ) + FROM qbit_block_candidate_body body + WHERE body_id = {text(body_id)} + ) +); +""" + + +def body_spans_sql(body_id: str, after_field: str | None, *, text: Literal) -> str: + """Read at most :data:`INDEX_SPAN_ROWS` span rows, keyset by field name.""" + after = "" if after_field is None else f"\n AND field > {text(after_field)}" + return f""" +SELECT json_build_object( + 'state', (SELECT state FROM qbit_block_candidate_body WHERE body_id = {text(body_id)}), + 'spans', ( + SELECT COALESCE(json_agg(json_build_object( + 'field', span.field, 'kind', span.kind, + 'start', span.start_offset, 'end', span.end_offset, + 'item_count', span.item_count, 'page_count', span.page_count, + 'pages_exact', span.pages_exact + ) ORDER BY span.field), '[]'::json) + FROM ( + SELECT * FROM qbit_block_candidate_body_span + WHERE body_id = {text(body_id)}{after} + ORDER BY field + LIMIT {INDEX_SPAN_ROWS} + ) span + ) +); +""" + + +def body_pages_sql(body_id: str, field_name: str, from_ordinal: int, *, text: Literal) -> str: + """Read at most :data:`INDEX_PAGE_ENTRIES` page rows of one field.""" + return f""" +SELECT json_build_object( + 'state', (SELECT state FROM qbit_block_candidate_body WHERE body_id = {text(body_id)}), + 'pages', ( + SELECT COALESCE(json_agg(json_build_array( + page.page_ordinal, page.body_offset, page.record_index + ) ORDER BY page.page_ordinal), '[]'::json) + FROM ( + SELECT * FROM qbit_block_candidate_body_page + WHERE body_id = {text(body_id)} + AND field = {text(field_name)} + AND page_ordinal >= {int(from_ordinal)} + ORDER BY page_ordinal + LIMIT {INDEX_PAGE_ENTRIES} + ) page + ) +); +""" + + +def body_page_sql( + body_id: str, + from_ordinal: int, + max_chunks: int, + *, + text: Literal, +) -> str: + """Read at most ``max_chunks`` consecutive chunks of one sealed body. + + Bounded by the SELECT itself (``LIMIT``), so the PGresult is bounded + too -- not merely the client's fetch size. Base64 keeps the transport + text at four thirds of the raw bytes. + """ + max_chunks = max(1, min(int(max_chunks), BODY_READ_MAX_CHUNKS)) + return f""" +SELECT json_build_object( + 'state', (SELECT state FROM qbit_block_candidate_body WHERE body_id = {text(body_id)}), + 'referenced', EXISTS ( + SELECT 1 FROM qbit_block_candidate_outbox outbox + WHERE outbox.body_id = {text(body_id)} AND outbox.state = 'pending' + ), + 'chunks', ( + SELECT COALESCE(json_agg(json_build_object( + 'ordinal', page.ordinal, + 'sha256', page.chunk_sha256, + 'base64', encode(page.chunk, 'base64') + ) ORDER BY page.ordinal), '[]'::json) + FROM ( + SELECT ordinal, chunk_sha256, chunk + FROM qbit_block_candidate_body_chunk + WHERE body_id = {text(body_id)} + AND ordinal >= {int(from_ordinal)} + ORDER BY ordinal + LIMIT {max_chunks} + ) page + ) +); +""" + + +def header_page_sql( + limit: int, + after_cursor: tuple[str, str] | None, + max_bytes: int, + *, + text: Literal, +) -> str: + """Metadata-first pending page with explicit exhaustion. + + Projects the bounded typed replay header for both storage versions + through ``qbit_prism_bounded_replay_header``: v2 rows carry the + writer's projection in ``replay_header``; v1 rows have the same named + keys projected out of the legacy jsonb (explicit field projection, + never ``- 'shares_json'`` over an arbitrary document). Every fact is + bounded on the server, so a header has a provable maximum size and + the byte cap is a real bound even for the page's first row. The inner + query fetches one row beyond the row cap so the page can prove + exhaustion, and the running byte total cuts the page at ``max_bytes`` + while reporting that it did. + """ + limit = max(1, min(int(limit), HEADER_PAGE_MAX_ROWS)) + max_bytes = max(1, min(int(max_bytes), HEADER_PAGE_MAX_BYTES)) + after_predicate = "" + if after_cursor is not None: + created_at_text, cursor_block_hash = after_cursor + after_predicate = ( + "\n AND (created_at, block_hash) > " + f"({text(created_at_text)}::timestamptz, {text(cursor_block_hash)})" + ) + return f""" +WITH fetched AS ( + SELECT + block_hash, + storage_version, + candidate_sha256, + body_id, + created_at, + to_char( + created_at AT TIME ZONE 'UTC', + 'YYYY-MM-DD"T"HH24:MI:SS.US"Z"' + ) AS cursor_created_at, + qbit_prism_bounded_replay_header( + CASE + WHEN storage_version = 2 THEN replay_header + ELSE jsonb_build_object( + 'schema', candidate->'schema', + 'block_hash_hex', candidate->'block_hash_hex', + 'parent_hash', candidate->'parent_hash', + 'expected_height', candidate->'expected_height', + 'template', candidate->'template', + 'found_block', jsonb_build_object( + 'network_difficulty', candidate->'found_block'->'network_difficulty' + ), + 'pending_share', candidate->'pending_share', + 'credit_share_on_accept', candidate->'credit_share_on_accept', + 'collection_only', candidate->'collection_only', + 'username', candidate->'username', + 'accepted_at_present', (candidate->'pending_share') ? 'accepted_at_ms' + ) + END + ) AS header, + EXISTS ( + SELECT 1 + FROM qbit_pool_blocks pool + WHERE pool.block_hash = qbit_block_candidate_outbox.block_hash + ) AS pool_block_exists + FROM qbit_block_candidate_outbox + WHERE state = 'pending'{after_predicate} + ORDER BY created_at, block_hash + LIMIT {limit + 1} +), +measured AS ( + SELECT + fetched.*, + row_number() OVER (ORDER BY created_at, block_hash) AS row_position, + sum(octet_length(header::text)) OVER ( + ORDER BY created_at, block_hash + ROWS BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW + ) AS running_bytes + FROM fetched +), +page AS ( + SELECT measured.* + FROM measured + WHERE row_position <= {limit} + AND (row_position = 1 OR running_bytes <= {max_bytes}) +) +SELECT json_build_object( + 'fetched', (SELECT count(*) FROM fetched), + 'returned', (SELECT count(*) FROM page), + 'rows', ( + SELECT COALESCE(json_agg(json_build_object( + 'block_hash', page.block_hash, + 'storage_version', page.storage_version, + 'candidate_sha256', page.candidate_sha256, + 'header', page.header, + 'header_bytes', octet_length(page.header::text), + 'body', ( + SELECT json_build_object( + 'body_id', body.body_id, + 'storage_version', body.storage_version, + 'candidate_sha256', body.candidate_sha256, + 'byte_count', body.byte_count, + 'chunk_count', body.chunk_count, + 'chunk_bytes', body.chunk_bytes, + 'share_count', body.share_count, + 'state', body.state + ) + FROM qbit_block_candidate_body body + WHERE body.body_id = page.body_id + ), + 'pool_block_exists', page.pool_block_exists, + 'cursor', json_build_array(page.cursor_created_at, page.block_hash) + ) ORDER BY page.created_at, page.block_hash), '[]'::json) + FROM page + ) +); +""" + + +def retire_orphan_bodies_sql(payload: Mapping[str, Any], *, jsonb: JsonbLiteral) -> str: + """Tiny fenced compare-and-swap: orphaned bodies become ``retired``. + + Under the writer fence (a deposed session retires nothing), but the + only row this touches besides the lease is one manifest row: a + ``staging`` body older than ``stale_staging_seconds`` owned by another + session, or a ``sealed`` body older than that age that no outbox row + references. ``retired`` is permanent -- the manifest trigger permits no + transition out of it -- and publication references only ``sealed`` + bodies, so a retired body can never be revived. + + The race with a publication in flight is settled by the database, not + by this statement's snapshot: publication locks the body row ``FOR + SHARE`` while it checks the body is sealed, this statement locks it + ``FOR UPDATE``, and the manifest trigger refuses to retire a body any + outbox row references (its check runs with a fresh snapshot after the + lock wait). Whichever side loses the lock re-evaluates and fails + closed; in-process, both statements also serialize behind the ledger's + writer gate. + """ + return f""" +WITH input AS ( + SELECT {jsonb(payload)} AS data +), +lease AS ( + UPDATE qbit_ledger_writer_lease + SET updated_at = clock_timestamp() + FROM input + WHERE singleton + AND writer_id = input.data->>'writer_id' + AND writer_epoch = (input.data->>'writer_epoch')::bigint + AND writer_session_token = input.data->>'writer_session_token' + RETURNING writer_id +), +target AS ( + SELECT body.body_id + FROM qbit_block_candidate_body body, input + WHERE EXISTS (SELECT 1 FROM lease) + AND ( + ( + body.state = 'staging' + AND body.created_at < clock_timestamp() + - (input.data->>'stale_staging_seconds')::double precision * interval '1 second' + AND NOT ( + body.staging_writer_id = input.data->>'writer_id' + AND body.staging_session_token = input.data->>'writer_session_token' + ) + ) + OR ( + body.state = 'sealed' + AND body.sealed_at < clock_timestamp() + - (input.data->>'stale_staging_seconds')::double precision * interval '1 second' + ) + ) + AND NOT EXISTS ( + SELECT 1 FROM qbit_block_candidate_outbox outbox + WHERE outbox.body_id = body.body_id + ) + ORDER BY body.created_at, body.body_id + LIMIT 1 + FOR UPDATE OF body +), +retired AS ( + UPDATE qbit_block_candidate_body + SET state = 'retired', retired_at = clock_timestamp() + FROM target + WHERE qbit_block_candidate_body.body_id = target.body_id + AND qbit_block_candidate_body.state IN ('staging', 'sealed') + AND NOT EXISTS ( + SELECT 1 FROM qbit_block_candidate_outbox outbox + WHERE outbox.body_id = qbit_block_candidate_body.body_id + ) + RETURNING qbit_block_candidate_body.body_id +) +SELECT CASE + WHEN NOT EXISTS (SELECT 1 FROM lease) THEN + json_build_object('error', 'writer lease is not active') + ELSE json_build_object('retired', COALESCE((SELECT json_agg(body_id) FROM retired), '[]'::json)) +END; +""" + + +def reap_retired_chunks_sql(payload: Mapping[str, Any], *, jsonb: JsonbLiteral) -> str: + """Delete one bounded page of one retired body's parts; no lease touch. + + Runs entirely outside the lease row: ``retired`` is a permanent state + that nothing publishes from, so reclaiming its bytes needs no fence + and a deposed session doing it cannot harm anyone. + + Every step is bounded and every probe is an existence test, never a + count: at most ``max_chunks`` chunk rows go, then (once the statement's + snapshot shows no chunk left) at most ``max_chunks`` page rows, then + span rows, and the manifest itself only on a later step whose snapshot + finds every child table empty. The returned ``remaining`` flags let the + caller keep stepping until the body is gone, so a body with zero chunks + but many page rows never stalls. + """ + return f""" +WITH input AS ( + SELECT {jsonb(payload)} AS data +), +target AS ( + SELECT body.body_id + FROM qbit_block_candidate_body body + WHERE body.state = 'retired' + AND NOT EXISTS ( + SELECT 1 FROM qbit_block_candidate_outbox outbox + WHERE outbox.body_id = body.body_id + ) + ORDER BY body.retired_at, body.body_id + LIMIT 1 +), +before AS ( + SELECT + EXISTS (SELECT 1 FROM qbit_block_candidate_body_chunk chunk, target + WHERE chunk.body_id = target.body_id) AS chunks, + EXISTS (SELECT 1 FROM qbit_block_candidate_body_page page, target + WHERE page.body_id = target.body_id) AS pages, + EXISTS (SELECT 1 FROM qbit_block_candidate_body_span span, target + WHERE span.body_id = target.body_id) AS spans +), +deleted_chunks AS ( + DELETE FROM qbit_block_candidate_body_chunk + WHERE ctid IN ( + SELECT chunk.ctid + FROM qbit_block_candidate_body_chunk chunk, target + WHERE chunk.body_id = target.body_id + ORDER BY chunk.ordinal + LIMIT (SELECT (data->>'max_chunks')::integer FROM input) + ) + RETURNING body_id +), +deleted_pages AS ( + DELETE FROM qbit_block_candidate_body_page + WHERE NOT (SELECT chunks FROM before) + AND ctid IN ( + SELECT page.ctid + FROM qbit_block_candidate_body_page page, target + WHERE page.body_id = target.body_id + ORDER BY page.field, page.page_ordinal + LIMIT (SELECT (data->>'max_chunks')::integer FROM input) + ) + RETURNING body_id +), +deleted_spans AS ( + DELETE FROM qbit_block_candidate_body_span + WHERE NOT (SELECT chunks FROM before) + AND NOT (SELECT pages FROM before) + AND ctid IN ( + SELECT span.ctid + FROM qbit_block_candidate_body_span span, target + WHERE span.body_id = target.body_id + ORDER BY span.field + LIMIT (SELECT (data->>'max_chunks')::integer FROM input) + ) + RETURNING body_id +), +deleted_bodies AS ( + DELETE FROM qbit_block_candidate_body + WHERE body_id IN (SELECT body_id FROM target) + AND state = 'retired' + AND NOT (SELECT chunks FROM before) + AND NOT (SELECT pages FROM before) + AND NOT (SELECT spans FROM before) + RETURNING body_id +) +SELECT json_build_object( + 'body_id', (SELECT body_id FROM target), + 'deleted_chunks', (SELECT count(*) FROM deleted_chunks), + 'deleted_pages', (SELECT count(*) FROM deleted_pages), + 'deleted_spans', (SELECT count(*) FROM deleted_spans), + 'deleted_bodies', (SELECT count(*) FROM deleted_bodies), + 'remaining', (SELECT json_build_object('chunks', chunks, 'pages', pages, 'spans', spans) FROM before) +); +""" + + +# -------------------------------------------------------------------------- +# spool admission +# -------------------------------------------------------------------------- + + +class SpoolAdmission: + """Global byte accounting for hydrated spool files.""" + + def __init__( + self, + directory: str | Path | None = None, + *, + limit_bytes: int = DEFAULT_SPOOL_RESERVATION_BYTES, + ) -> None: + self._lock = threading.Lock() + self._limit = int(limit_bytes) + self._reserved = 0 + self._directory = ( + Path(directory) + if directory is not None + else Path(tempfile.gettempdir()) / "qbit-prism-candidate-spool" + ) + + @property + def directory(self) -> Path: + return self._directory + + def snapshot(self) -> dict[str, int]: + with self._lock: + return {"reserved_bytes": self._reserved, "limit_bytes": self._limit} + + def reserve(self, byte_count: int) -> Callable[[], None]: + """Reserve ``byte_count``; returns the idempotent release callable.""" + byte_count = max(0, int(byte_count)) + with self._lock: + if self._reserved + byte_count > self._limit: + raise SpoolAdmissionExhausted( + f"candidate spool reservation exhausted: {self._reserved + byte_count} " + f"> {self._limit} bytes" + ) + self._reserved += byte_count + released = False + lock = self._lock + + def release() -> None: + nonlocal released + with lock: + if not released: + released = True + self._reserved -= byte_count + + return release + + def new_spool_paths(self, block_hash: str) -> tuple[str, str]: + """A fresh ``(body path, index path)`` pair in the spool directory.""" + self._directory.mkdir(parents=True, exist_ok=True) + handle, path = tempfile.mkstemp( + prefix=f"candidate-{block_hash[:16]}-", + suffix=".body", + dir=str(self._directory), + ) + os.close(handle) + index_path = path[: -len(".body")] + ".idx" + try: + with open(index_path, "wb"): + pass + except BaseException: + os.unlink(path) + raise + return path, index_path + + +# -------------------------------------------------------------------------- +# hydration +# -------------------------------------------------------------------------- + + +def parse_manifest_row(row: Mapping[str, Any]) -> tuple[CandidateBodyManifest, str, bool]: + """Typed manifest, live state and referenced flag from a manifest read.""" + if not isinstance(row, Mapping) or not row.get("found"): + raise CandidateBodyUnavailable("candidate body manifest does not exist") + manifest_json = row.get("manifest") + if not isinstance(manifest_json, Mapping): + raise CandidateBodyUnavailable("candidate body manifest is unreadable") + manifest = CandidateBodyManifest.from_json(manifest_json) + state = str(manifest_json.get("state")) + referenced = bool(manifest_json.get("referenced")) + return manifest, state, referenced + + +def _strict_base64(encoded: str, *, max_chars: int) -> bytes: + """Decode PostgreSQL ``encode(..., 'base64')`` output strictly. + + The size is checked before any decoding allocates. PostgreSQL wraps at + 76 characters with ``\\n``; only that whitespace is tolerated, and the + alphabet is validated rather than silently filtered. + """ + if not isinstance(encoded, str): + raise CandidateBodyIntegrityError("candidate body page chunk carries no bytes") + if len(encoded) > max_chars: + raise CandidateBodyIntegrityError("candidate body page chunk exceeds the transport bound") + compact = encoded.replace("\n", "") + if "\r" in compact or " " in compact: + raise CandidateBodyIntegrityError("candidate body page chunk carries unexpected whitespace") + try: + return base64.b64decode(compact.encode("ascii"), validate=True) + except (binascii.Error, ValueError, UnicodeEncodeError) as exc: + raise CandidateBodyIntegrityError("candidate body page chunk is not valid base64") from exc + + +def parse_body_page(row: Mapping[str, Any], *, max_chunks: int = BODY_READ_MAX_CHUNKS) -> tuple[str | None, bool, list[BodyChunk]]: + """Decode one page read: state, referenced flag and verified chunks. + + The chunk count and every chunk's encoded length are checked against + the page bound before any base64 allocation; the decoded total is + checked against the read bound as well. + """ + if not isinstance(row, Mapping): + raise CandidateBodyIntegrityError("candidate body page is not an object") + state = row.get("state") + chunks_json = row.get("chunks") + if not isinstance(chunks_json, list): + raise CandidateBodyIntegrityError("candidate body page carries no chunk list") + if len(chunks_json) > max(1, min(int(max_chunks), BODY_READ_MAX_CHUNKS)): + raise CandidateBodyIntegrityError("candidate body page exceeds the chunk bound") + chunks: list[BodyChunk] = [] + total = 0 + for entry in chunks_json: + if not isinstance(entry, Mapping): + raise CandidateBodyIntegrityError("candidate body page chunk is malformed") + data = _strict_base64(entry.get("base64"), max_chars=CHUNK_BASE64_MAX_CHARS) + total += len(data) + if len(data) > CANDIDATE_BODY_CHUNK_BYTES or total > BODY_READ_MAX_BYTES: + raise CandidateBodyIntegrityError("candidate body page exceeds the read bound") + chunks.append( + BodyChunk( + ordinal=int(entry.get("ordinal", -1)), + data=data, + sha256=str(entry.get("sha256", "")), + ) + ) + return (None if state is None else str(state)), bool(row.get("referenced")), chunks + + +def parse_span_rows(row: Mapping[str, Any]) -> tuple[str | None, list[FieldSpan]]: + if not isinstance(row, Mapping) or not isinstance(row.get("spans"), list): + raise CandidateBodyIntegrityError("candidate body span page is malformed") + spans = row["spans"] + if len(spans) > INDEX_SPAN_ROWS: + raise CandidateBodyIntegrityError("candidate body span page exceeds its bound") + state = row.get("state") + return (None if state is None else str(state)), [FieldSpan.from_json(span) for span in spans] + + +def parse_page_rows(row: Mapping[str, Any], field_name: str) -> tuple[str | None, list[PageEntry]]: + if not isinstance(row, Mapping) or not isinstance(row.get("pages"), list): + raise CandidateBodyIntegrityError("candidate body page-index page is malformed") + pages = row["pages"] + if len(pages) > INDEX_PAGE_ENTRIES: + raise CandidateBodyIntegrityError("candidate body page-index page exceeds its bound") + entries: list[PageEntry] = [] + for page in pages: + if not isinstance(page, list) or len(page) != 3: + raise CandidateBodyIntegrityError("candidate body page-index entry is malformed") + entries.append( + PageEntry(field=field_name, ordinal=int(page[0]), offset=int(page[1]), record_index=int(page[2])) + ) + state = row.get("state") + return (None if state is None else str(state)), entries + + +class CandidateBodyHydrator: + """Hydrate one sealed body to a spool/index pair in bounded pages.""" + + def __init__( + self, + *, + read_manifest: Callable[[str], Any], + read_spans: Callable[[str, str | None], Any], + read_pages: Callable[[str, str, int], Any], + read_page: Callable[[str, int, int], Any], + admission: SpoolAdmission, + cancelled: Callable[[], bool] | None = None, + max_chunks_per_page: int = BODY_READ_MAX_CHUNKS, + ) -> None: + self._read_manifest = read_manifest + self._read_spans = read_spans + self._read_pages = read_pages + self._read_page = read_page + self._admission = admission + self._cancelled = cancelled or (lambda: False) + self._max_chunks = max(1, min(int(max_chunks_per_page), BODY_READ_MAX_CHUNKS)) + + def hydrate( + self, + ref: CandidateBodyRef, + *, + block_hash: str, + accepted_at_present: bool, + accepted_at_ms: Any, + ) -> PreparedCandidateIntent: + manifest, state, referenced = parse_manifest_row(self._read_manifest(ref.body_id)) + if state != "sealed" or not referenced: + raise CandidateBodyUnavailable( + f"candidate body {ref.body_id} is {state} and " + f"{'referenced' if referenced else 'unreferenced'}" + ) + if manifest.candidate_sha256 != ref.candidate_sha256: + raise CandidateBodyIntegrityError("candidate body manifest digest differs from its outbox row") + if manifest.byte_count != ref.byte_count or manifest.chunk_count != ref.chunk_count: + raise CandidateBodyIntegrityError("candidate body manifest disagrees with its outbox row") + release = self._admission.reserve(manifest.byte_count + manifest.page_count * 16) + paths: tuple[str, str] = () + writer = None + body = None + try: + paths = self._admission.new_spool_paths(block_hash) + writer = SpoolWriter(*paths, manifest) + self._spool_index(ref.body_id, manifest, writer) + self._spool_chunks(ref.body_id, manifest, writer) + body = writer.finish(release=release) + return prepared_intent_from_spool( + body, + accepted_at_present=accepted_at_present, + accepted_at_ms=accepted_at_ms, + ) + except BaseException: + if body is not None: + body.close() + elif writer is not None: + writer.abort() + else: + for path in paths: + try: + os.unlink(path) + except FileNotFoundError: + pass + release() + raise + + def _check_state(self, state: str | None) -> None: + if state != "sealed": + raise CandidateBodyUnavailable("candidate body turned terminal during hydration") + + def _spool_index(self, body_id: str, manifest: CandidateBodyManifest, writer: SpoolWriter) -> None: + after: str | None = None + seen = 0 + while True: + if self._cancelled(): + raise CandidateStorageError("candidate body hydration cancelled") + state, spans = parse_span_rows(self._read_spans(body_id, after)) + self._check_state(state) + for span in spans: + writer.span(span) + self._spool_pages(body_id, span, writer) + seen += len(spans) + if len(spans) < INDEX_SPAN_ROWS: + break + after = spans[-1].field + if seen != manifest.span_count: + raise CandidateBodyIntegrityError("candidate body span rows disagree with the manifest") + + def _spool_pages(self, body_id: str, span: FieldSpan, writer: SpoolWriter) -> None: + ordinal = 0 + while ordinal < span.page_count: + if self._cancelled(): + raise CandidateStorageError("candidate body hydration cancelled") + state, entries = parse_page_rows(self._read_pages(body_id, span.field, ordinal), span.field) + self._check_state(state) + if not entries: + raise CandidateBodyIntegrityError("candidate body page index is incomplete") + for entry in entries: + if entry.ordinal != ordinal: + raise CandidateBodyIntegrityError("candidate body page index is not contiguous") + ordinal += 1 + writer.pages(entries) + if ordinal != span.page_count: + raise CandidateBodyIntegrityError("candidate body page index disagrees with its span") + + def _spool_chunks(self, body_id: str, manifest: CandidateBodyManifest, writer: SpoolWriter) -> None: + ordinal = 0 + while ordinal < manifest.chunk_count: + if self._cancelled(): + raise CandidateStorageError("candidate body hydration cancelled") + state, referenced, chunks = parse_body_page( + self._read_page(body_id, ordinal, self._max_chunks), + max_chunks=self._max_chunks, + ) + if state != "sealed" or not referenced: + raise CandidateBodyUnavailable("candidate body turned terminal during hydration") + if not chunks: + raise CandidateBodyIntegrityError("candidate body page returned no chunks") + for chunk in chunks: + if chunk.ordinal != ordinal: + raise CandidateBodyIntegrityError("candidate body chunks are not contiguous") + writer.chunk(chunk) + ordinal += 1 + + +# -------------------------------------------------------------------------- +# legacy helper (isolated process) +# -------------------------------------------------------------------------- + + +@dataclass(frozen=True) +class LegacyTransport: + """How the helper reaches PostgreSQL: a psql command or a DSN.""" + + psql_command: str | None = None + database_url: str | None = None + + def to_json(self) -> dict[str, Any]: + return {"psql_command": self.psql_command, "database_url": self.database_url} + + +@dataclass +class LegacyHelperResult: + manifest: CandidateBodyManifest + spans: list[FieldSpan] + accepted_at_present: bool + accepted_at_ms: Any + row_candidate_sha256: str | None + identity_matches_row: bool + + +class LegacyCandidateHelper: + """Run the isolated legacy conversion/comparison child process.""" + + def __init__( + self, + transport: LegacyTransport, + *, + python: str | None = None, + memory_limit_bytes: int = LEGACY_HELPER_MEMORY_BYTES, + timeout_seconds: float = LEGACY_HELPER_TIMEOUT_SECONDS, + cancelled: Callable[[], bool] | None = None, + ) -> None: + self._transport = transport + self._python = python or sys.executable + self._memory_limit = int(memory_limit_bytes) + self._timeout = float(timeout_seconds) + self._cancelled = cancelled or (lambda: False) + self._lock = threading.Lock() # admit at most one conversion + + def with_cancellation(self, cancelled: Callable[[], bool]) -> LegacyCandidateHelper: + helper = LegacyCandidateHelper( + self._transport, + python=self._python, + memory_limit_bytes=self._memory_limit, + timeout_seconds=self._timeout, + cancelled=cancelled, + ) + # Cancellation wrappers share the same admission slot. + helper._lock = self._lock + return helper + + def _run( + self, + request: Mapping[str, Any], + body: CandidateBody | None = None, + ) -> dict[str, Any]: + """Supervise input, output, cancellation and retirement together. + + A helper may wait on PostgreSQL before reading stdin. The deadline + therefore starts before sending body bytes, and a blocked pipe writer + runs alongside the supervisor. Diagnostic pipes are drained with fixed + retention limits so a failed child cannot deadlock or grow the parent. + """ + deadline = time.monotonic() + self._timeout + + def check_deadline() -> None: + if self._cancelled(): + raise CandidateStorageError("legacy candidate helper cancelled") + if time.monotonic() >= deadline: + raise CandidateStorageError("legacy candidate helper exceeded its deadline") + + while not self._lock.acquire(timeout=LEGACY_HELPER_POLL_SECONDS): + check_deadline() + process = None + threads: list[threading.Thread] = [] + output = bytearray() + errors = bytearray() + io_errors: list[BaseException] = [] + limit = 64 * 1024 + from lab.prism.audit_bundle_view import RECORD_HELPER_ADMISSION + + admitted = False + try: + RECORD_HELPER_ADMISSION.acquire(check_deadline) + admitted = True + check_deadline() + repo_root = Path(__file__).resolve().parents[2] + env = dict(os.environ) + env["PYTHONPATH"] = str(repo_root) + ( + os.pathsep + env["PYTHONPATH"] if env.get("PYTHONPATH") else "" + ) + # Apply limits in the freshly exec'd child. preexec_fn is unsafe + # in this multithreaded process, before Python locks are reset. + process = subprocess.Popen( + [self._python, "-m", "lab.prism.candidate_store"], + stdin=subprocess.PIPE, stdout=subprocess.PIPE, + stderr=subprocess.PIPE, cwd=str(repo_root), env=env, + ) + assert process.stdin and process.stdout and process.stderr + + def drain(pipe: Any, target: bytearray) -> None: + try: + while chunk := pipe.read(8192): + target.extend(chunk) + if len(target) > limit: + del target[:-limit] + except BaseException as exc: + io_errors.append(exc) + + def produce() -> None: + try: + assert process is not None and process.stdin is not None + process.stdin.write((json.dumps({ + **request, "memory_limit_bytes": self._memory_limit, + }) + "\n").encode("utf-8")) + process.stdin.flush() + if body is not None: + def write_chunk(chunk: BodyChunk) -> None: + check_deadline() + process.stdin.write(chunk.data) + body.write_chunks(write_chunk) + process.stdin.flush() + except BaseException as exc: + io_errors.append(exc) + finally: + try: + process.stdin.close() + except OSError: + pass + + for target, args, name in ( + (drain, (process.stdout, output), "stdout"), + (drain, (process.stderr, errors), "stderr"), + (produce, (), "input"), + ): + thread = threading.Thread(target=target, args=args, name=f"prism-legacy-helper-{name}") + threads.append(thread) + thread.start() + while process.poll() is None: + check_deadline() + if io_errors: + raise CandidateStorageError("legacy candidate helper transport failed") from io_errors[0] + time.sleep(LEGACY_HELPER_POLL_SECONDS) + for thread in threads: + thread.join() + if process.returncode != 0: + raise CandidateStorageError( + "legacy candidate helper failed: " + errors.decode("utf-8", "replace")[-2000:] + ) + if io_errors: + raise CandidateStorageError("legacy candidate helper transport failed") from io_errors[0] + try: + result = json.loads(output.decode("utf-8").strip().splitlines()[-1]) + except (ValueError, IndexError) as exc: + raise CandidateStorageError("legacy candidate helper returned no verdict") from exc + if not isinstance(result, dict): + raise CandidateStorageError("legacy candidate helper verdict is malformed") + if "error" in result: + raise CandidateStorageError(f"legacy candidate helper: {result['error']}") + return result + finally: + if process is not None: + if process.poll() is None: + process.kill() + process.wait() + for thread in threads: + thread.join() + for pipe in (process.stdin, process.stdout, process.stderr): + if pipe is not None: + pipe.close() + if admitted: + RECORD_HELPER_ADMISSION.release() + self._lock.release() + + def convert(self, block_hash: str, spool_path: str, index_path: str, *, spool_limit_bytes: int | None = None) -> LegacyHelperResult: + result = self._run({ + "mode": "convert", + "transport": self._transport.to_json(), + "block_hash": block_hash, + "spool_path": spool_path, + "index_path": index_path, + "chunk_bytes": CANDIDATE_BODY_CHUNK_BYTES, + "spool_limit_bytes": spool_limit_bytes, + }) + manifest = CandidateBodyManifest.from_json(result["manifest"]) + spans_json = result.get("spans") + if not isinstance(spans_json, list) or len(spans_json) > INDEX_SPAN_ROWS * 64: + raise CandidateStorageError("legacy candidate helper returned malformed spans") + return LegacyHelperResult( + manifest=manifest, + spans=[FieldSpan.from_json(span) for span in spans_json], + accepted_at_present=bool(result.get("accepted_at_present")), + accepted_at_ms=result.get("accepted_at_ms"), + row_candidate_sha256=result.get("row_candidate_sha256"), + identity_matches_row=bool(result.get("identity_matches_row")), + ) + + def compare(self, block_hash: str, body: CandidateBody) -> bool: + """jsonb-equivalence of the legacy row's identity and ``body``.""" + return self._compare("compare", block_hash, body, body_id=None) + + def compare_stored_body(self, body_id: str, block_hash: str, body: CandidateBody) -> bool: + """jsonb-equivalence of a stored chunked body and ``body``. + + The fallback behind the byte comparison in + ``PsqlShareLedger.compare_candidate_body``: the helper reads the + stored chunks itself (page by page, its own connection), decodes + both documents whole -- isolated, memory- and deadline-limited -- + and answers PostgreSQL's jsonb equality. + """ + return self._compare("compare_stored", block_hash, body, body_id=body_id) + + def _compare( + self, + mode: str, + block_hash: str, + body: CandidateBody, + *, + body_id: str | None, + ) -> bool: + result = self._run({ + "mode": mode, + "transport": self._transport.to_json(), + "block_hash": block_hash, + "body_id": body_id, + "body_bytes": body.manifest.byte_count, + }, body) + return bool(result.get("equal")) + + +def _helper_query_json(transport: Mapping[str, Any], sql: str) -> Any: + """Run one statement that yields a single JSON value, in the helper.""" + database_url = transport.get("database_url") + psql_command = transport.get("psql_command") + if database_url: + try: + import psycopg + except ImportError: + psycopg = None + if psycopg is not None: + with psycopg.connect(database_url, autocommit=True) as connection: + row = connection.execute(sql).fetchone() + if row is None: + return None + value = row[0] + return json.loads(value) if isinstance(value, str) else value + if not psql_command: + raise CandidateStorageError("legacy helper has no database transport") + completed = subprocess.run( + [*shlex.split(psql_command), "--tuples-only", "--no-align", "--quiet"], + input=sql, + text=True, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + check=False, + ) + if completed.returncode != 0: + raise CandidateStorageError(f"helper psql read failed: {completed.stderr.strip()}") + output = completed.stdout.strip() + if not output: + return None + return json.loads(output.splitlines()[-1]) + + +def _helper_fetch_legacy_text(transport: Mapping[str, Any], block_hash: str) -> tuple[str | None, str | None]: + """Fetch ``candidate::text`` and ``candidate_sha256`` for one v1 row.""" + quoted = block_hash.lower().replace(chr(39), chr(39) * 2) + document = _helper_query_json( + transport, + "SELECT json_build_object('candidate', candidate::text, 'sha', candidate_sha256) " + "FROM qbit_block_candidate_outbox " + f"WHERE block_hash = '{quoted}' " + f"AND storage_version = {LEGACY_CANDIDATE_STORAGE_VERSION} AND candidate IS NOT NULL;", + ) + if not isinstance(document, dict): + return None, None + return document.get("candidate"), document.get("sha") + + +def _helper_fetch_stored_body(transport: Mapping[str, Any], body_id: str) -> bytes | None: + """Read one stored chunked body in the helper, three chunks per statement.""" + quoted = body_id.replace(chr(39), chr(39) * 2) + header = _helper_query_json( + transport, + "SELECT json_build_object('state', state, 'chunk_count', chunk_count) " + f"FROM qbit_block_candidate_body WHERE body_id = '{quoted}';", + ) + if not isinstance(header, dict) or header.get("state") != "sealed": + return None + chunk_count = int(header.get("chunk_count") or 0) + pieces: list[bytes] = [] + ordinal = 0 + while ordinal < chunk_count: + page = _helper_query_json( + transport, + "SELECT COALESCE(json_agg(json_build_object('ordinal', ordinal, 'base64', encode(chunk, 'base64')) " + "ORDER BY ordinal), '[]'::json) FROM (SELECT ordinal, chunk FROM qbit_block_candidate_body_chunk " + f"WHERE body_id = '{quoted}' AND ordinal >= {ordinal} ORDER BY ordinal LIMIT {BODY_READ_MAX_CHUNKS}) page;", + ) + if not isinstance(page, list) or not page: + return None + for entry in page: + if int(entry.get("ordinal", -1)) != ordinal: + return None + pieces.append(_strict_base64(entry.get("base64"), max_chars=CHUNK_BASE64_MAX_CHARS)) + ordinal += 1 + return b"".join(pieces) + + +def helper_main(stdin_stream: Any = None, stdout_stream: Any = None) -> int: + """Entry point of the isolated helper process.""" + stdin_buffer = (stdin_stream or sys.stdin.buffer) + stdout_text = stdout_stream or sys.stdout + header = stdin_buffer.readline() + try: + request = json.loads(header.decode("utf-8")) + if os.name == "posix": + import resource + memory_limit = int(request.get("memory_limit_bytes", LEGACY_HELPER_MEMORY_BYTES)) + resource.setrlimit(resource.RLIMIT_AS, (memory_limit, memory_limit)) + mode = request["mode"] + transport = request["transport"] + block_hash = str(request["block_hash"]).lower() + if mode == "compare_stored": + from decimal import Decimal + + stored = _helper_fetch_stored_body(transport, str(request["body_id"])) + if stored is None: + raise CandidateBodyUnavailable("stored candidate body is not sealed") + ours_bytes = stdin_buffer.read() + theirs = json.loads(stored.decode("utf-8"), parse_float=Decimal) + ours = json.loads(ours_bytes.decode("utf-8"), parse_float=Decimal) + result = {"equal": jsonb_equivalent(theirs, ours)} + stdout_text.write(json.dumps(result) + "\n") + stdout_text.flush() + return 0 + legacy_text, row_sha = _helper_fetch_legacy_text(transport, block_hash) + if legacy_text is None: + raise CandidateBodyUnavailable("legacy candidate row is not pending") + if mode == "convert": + manifest, spans, present, stamp = legacy_json_to_spool( + legacy_text, + str(request["spool_path"]), + str(request["index_path"]), + chunk_bytes=int(request.get("chunk_bytes", CANDIDATE_BODY_CHUNK_BYTES)), + spool_limit_bytes=request.get("spool_limit_bytes"), + ) + # The spans are a handful of rows (one per large field); the + # page entries stay in the index file. + result = { + "manifest": manifest.to_json(), + "spans": [span.to_json() for span in spans], + "accepted_at_present": present, + "accepted_at_ms": stamp, + "row_candidate_sha256": row_sha, + "identity_matches_row": manifest.candidate_sha256 == row_sha, + } + elif mode == "compare": + from decimal import Decimal + + ours_bytes = stdin_buffer.read() + legacy = json.loads(legacy_text, parse_float=Decimal) + ours = json.loads(ours_bytes.decode("utf-8"), parse_float=Decimal) + for document in (legacy, ours): + pending = document.get("pending_share") if isinstance(document, dict) else None + if isinstance(pending, dict) and "accepted_at_ms" in pending: + pending["accepted_at_ms"] = None + result = {"equal": jsonb_equivalent(legacy, ours)} + else: + raise CandidateStorageError(f"unknown helper mode {mode!r}") + except Exception as exc: # noqa: BLE001 - reported to the parent + stdout_text.write(json.dumps({"error": f"{type(exc).__name__}: {exc}"}) + "\n") + stdout_text.flush() + return 1 + stdout_text.write(json.dumps(result) + "\n") + stdout_text.flush() + return 0 + + +if __name__ == "__main__": # pragma: no cover - exercised as a subprocess + sys.exit(helper_main()) + + +__all__ = [ + "BODY_READ_MAX_BYTES", + "BODY_READ_MAX_CHUNKS", + "CANDIDATE_SCHEMA_CAPABILITY", + "HEADER_MAX_BYTES", + "HEADER_PAGE_MAX_BYTES", + "HEADER_PAGE_MAX_ROWS", + "JANITOR_CHUNKS_PER_STEP", + "STALE_STAGING_SECONDS", + "CandidateBodyHydrator", + "CandidateBodyRef", + "CandidateBodyUnavailable", + "CandidateHeaderPage", + "CandidateStorageError", + "DurableCandidateDescriptor", + "IncompatibleCandidateSchema", + "LegacyCandidateHelper", + "LegacyHelperResult", + "LegacyTransport", + "SpoolAdmission", + "SpoolAdmissionExhausted", + "body_chunk_sql", + "body_index_rows_sql", + "body_manifest_sql", + "body_page_sql", + "body_pages_sql", + "body_spans_sql", + "candidate_schema_refusal", + "chunk_hex_literal", + "header_page_sql", + "helper_main", + "new_body_id", + "parse_body_page", + "parse_manifest_row", + "parse_page_rows", + "parse_span_rows", + "reap_retired_chunks_sql", + "retire_orphan_bodies_sql", + "retire_staging_body_sql", + "schema_capability_sql", + "seal_body_sql", + "stage_body_sql", +] diff --git a/lab/prism/candidate_window.py b/lab/prism/candidate_window.py new file mode 100644 index 000000000..f7a8a4911 --- /dev/null +++ b/lab/prism/candidate_window.py @@ -0,0 +1,266 @@ +"""Bounded, exact membership checks for a replayed candidate's share window. + +Only omissions matter: the durable window must be a subset of the recorded +window. PostgreSQL performs that comparison against a private temporary table +fed through bounded COPY writes, returning a boolean rather than a window-sized +JSON value. No ledger mutation or writer-lease lock is involved. The in-memory +ledger compatibility path uses an on-disk index instead of a Python set. +""" + +from __future__ import annotations + +from collections.abc import Callable, Iterable, Iterator, Mapping +import hashlib +import json +from pathlib import Path +import sqlite3 +import subprocess +import tempfile +from typing import Any + + +COPY_TEXT_CHARACTERS = 4096 +COPY_TEXT_MAX_BYTES = COPY_TEXT_CHARACTERS * 4 +COPY_TEXT_MAX_RECORDS = 256 +_CREATE = """ +CREATE TEMPORARY TABLE qbit_candidate_recorded_ids (share_id text COLLATE "C" NOT NULL) +ON COMMIT DROP; +SET LOCAL work_mem = '8MB'; +""" +_COPY = "COPY pg_temp.qbit_candidate_recorded_ids (share_id) FROM STDIN" + + +def recorded_share_ids(shares: Iterable[Any]) -> Iterator[Any]: + """Keep the historical dictionary/type/string conversion semantics.""" + for row in shares: + if isinstance(row, Mapping): + value = row.get("share_id") + yield value if callable(getattr(value, "iter_text_chunks", None)) else str(value) + + +def _text_pieces(value: Any) -> Iterator[str]: + streamed = getattr(value, "iter_text_chunks", None) + chunks = streamed() if callable(streamed) else (value,) + for chunk in chunks: + for offset in range(0, len(chunk), COPY_TEXT_CHARACTERS): + yield chunk[offset:offset + COPY_TEXT_CHARACTERS] + + +def copy_text_chunks( + values: Iterable[Any], *, check: Callable[[], Any] = lambda: None, +) -> Iterator[bytes]: + """Encode COPY text without encoding or escaping an entire field at once. + + A UTF-8 character is at most four bytes; every escaped ASCII character + uses two. Thus even an unusually long share ID obeys the same write bound. + Literal backslashes (including \\N and \\.) cannot become COPY commands. + """ + buffer = bytearray() + records = 0 + for value in values: + for part in _text_pieces(value): + if "\x00" in part: + # PostgreSQL text cannot contain NUL. In psql's COPY input, + # it can also truncate the stream without a useful diagnostic. + raise ValueError("candidate share ID contains a NUL character") + part = part.replace("\\", "\\\\").replace("\t", "\\t") + part = part.replace("\n", "\\n").replace("\r", "\\r") + encoded = part.encode("utf-8") + if buffer and len(buffer) + len(encoded) > COPY_TEXT_MAX_BYTES: + check() + yield bytes(buffer) + buffer.clear() + records = 0 + buffer.extend(encoded) + if len(buffer) == COPY_TEXT_MAX_BYTES: + check() + yield bytes(buffer) + buffer.clear() + records = 0 + buffer.append(10) + records += 1 + if records == COPY_TEXT_MAX_RECORDS: + check() + yield bytes(buffer) + buffer.clear() + records = 0 + if buffer: + check() + yield bytes(buffer) + + +def disk_window_covers(recorded: Iterable[Any], durable: Iterable[Any]) -> bool: + """Compatibility path for in-memory ledgers; never retain the ID set.""" + with tempfile.TemporaryDirectory(prefix="prism-candidate-membership-") as root, tempfile.TemporaryFile() as values: + connection = sqlite3.connect(Path(root) / "ids.sqlite") + try: + connection.execute("PRAGMA cache_size = -2048") + connection.execute("PRAGMA temp_store = FILE") + connection.execute( + "CREATE TABLE recorded (digest BLOB, start INTEGER, length INTEGER)" + ) + connection.execute("CREATE INDEX recorded_digest ON recorded (digest)") + for value in recorded: + digest = hashlib.sha256() + start = values.tell() + for piece in _text_pieces(value): + encoded = piece.encode("utf-8") + digest.update(encoded) + values.write(encoded) + connection.execute("INSERT INTO recorded VALUES (?, ?, ?)", + (digest.digest(), start, values.tell() - start)) + for value in recorded_share_ids(durable): + digest = hashlib.sha256() + for piece in _text_pieces(value): + digest.update(piece.encode("utf-8")) + matched = False + for start, length in connection.execute( + "SELECT start, length FROM recorded WHERE digest = ?", (digest.digest(),), + ): + # A digest only selects possible matches. Equality still + # compares every byte, including under a hash collision. + values.seek(start) + remaining = length + for piece in _text_pieces(value): + encoded = piece.encode("utf-8") + if len(encoded) > remaining or values.read(len(encoded)) != encoded: + break + remaining -= len(encoded) + else: + if remaining == 0: + matched = True + break + if not matched: + return False + return True + finally: + connection.close() + + +def _query(anchor_job_issued_at_ms: int, network_difficulty: int) -> str: + return f""" +SELECT json_build_object('reproducible', NOT EXISTS ( + SELECT 1 FROM qbit_audit_share_window( + to_timestamp(({int(anchor_job_issued_at_ms)}::double precision / 1000.0)), + {int(network_difficulty)}::numeric + ) AS durable + WHERE NOT EXISTS ( + SELECT 1 FROM pg_temp.qbit_candidate_recorded_ids AS recorded + WHERE recorded.share_id = durable.share_id + ) +)); +""" + + +def _flushing_copy_writer(cursor: Any) -> Any: + # Keep psycopg optional for the psql-only deployment. LibpqWriter's + # standard finish handles CopyDone/CopyFail and cancellation correctly. + from psycopg.copy import LibpqWriter + from psycopg.generators import copy_to + + class FlushingWriter(LibpqWriter): + def write(self, data: bytes) -> None: + # On Linux the default writer can grow libpq's output buffer with + # the entire window while PostgreSQL is not consuming the socket. + # Our producer caps each write at 16 KiB; flush it before advancing. + self.connection.wait(copy_to(self._pgconn, data, flush=True)) + + return FlushingWriter(cursor) + + +def postgres_window_covers( + ledger: Any, + shares: Iterable[Any], + *, + anchor_job_issued_at_ms: int, + network_difficulty: int, +) -> bool: + """Use one durable snapshot, bounded transport, and exact text equality. + + The COPY table is connection-private and disappears on commit or rollback. + Server join memory can spill under work_mem. It grants neither credit nor + replay eligibility. A lost connection fails the read; the caller's existing + candidate retry repeats it with a fresh table and its immutable share view. + """ + from lab.prism.share_ledger import LedgerOperationTimeout, parse_single_json_value + + query = _query(anchor_job_issued_at_ms, network_difficulty) + with ledger._operation_gate(ledger._read_semaphore, "read slot"): + native = getattr(ledger, "_native", None) + if native is not None and callable(getattr(native, "connection", None)): + timeout = ledger._remaining_operation_timeout() + kwargs = {} if timeout is None else {"timeout_seconds": timeout} + try: + with native.connection(**kwargs) as connection: + with connection.transaction(): + def arm_statement() -> None: + remaining = ledger._remaining_operation_timeout() + if remaining is None: + return + milliseconds = max(1, int(remaining * 1000)) + connection.execute( + f"SET LOCAL statement_timeout = '{milliseconds}ms'" + ) + connection.execute( + f"SET LOCAL lock_timeout = '{milliseconds}ms'" + ) + + arm_statement() + connection.execute(_CREATE) + ledger._note_operation_progress() + with connection.cursor() as cursor: + arm_statement() + with cursor.copy(_COPY, writer=_flushing_copy_writer(cursor)) as copy: + for chunk in copy_text_chunks( + recorded_share_ids(shares), + check=ledger._note_json_row_batch, + ): + copy.write(chunk) + ledger._note_json_row_batch() + # Refresh the statement budget after streaming, so + # each phase shares the caller's original deadline. + arm_statement() + connection.execute("ANALYZE pg_temp.qbit_candidate_recorded_ids") + ledger._note_json_row_batch() + arm_statement() + row = connection.execute(query).fetchone() + result = parse_single_json_value(row[0] if row else None) + ledger._remaining_operation_timeout() + except native._psycopg.OperationalError as exc: + from lab.prism.share_ledger import _is_postgres_deadline_error + + if _is_postgres_deadline_error(exc): + raise LedgerOperationTimeout("candidate window check timed out") from exc + raise RuntimeError(f"postgres candidate window check failed: {exc}") from exc + else: + # psql reads a boundedly-written file. communicate(input=...) would + # reconstitute the entire recorded window in a single bytes object. + with tempfile.TemporaryFile() as source, tempfile.TemporaryFile() as errors: + source.write(_CREATE.encode("ascii")) + source.write((_COPY + ";\n").encode("ascii")) + for chunk in copy_text_chunks( + recorded_share_ids(shares), check=ledger._note_json_row_batch, + ): + source.write(chunk) + source.write(b"\\.\n") + source.write(b"ANALYZE pg_temp.qbit_candidate_recorded_ids;\n") + source.write(query.encode("ascii")) + source.seek(0) + command, kwargs, timeout = ledger._psql_invocation() + try: + completed = subprocess.run( + command, stdin=source, stdout=subprocess.PIPE, stderr=errors, + check=False, **kwargs, + ) + except subprocess.TimeoutExpired as exc: + raise LedgerOperationTimeout("candidate window check timed out") from exc + errors.seek(0) + # COPY errors can echo a whole input line. Keep diagnostics + # bounded as well; the spool itself dies with this operation. + error_text = errors.read(64 * 1024).decode("utf-8", errors="replace") + ledger._check_psql_exit(completed.returncode, error_text, timeout) + ledger._remaining_operation_timeout() + result = json.loads(completed.stdout) + if not isinstance(result, dict) or type(result.get("reproducible")) is not bool: + raise RuntimeError("candidate window check returned no boolean result") + return result["reproducible"] diff --git a/lab/prism/coordinator_config.py b/lab/prism/coordinator_config.py new file mode 100644 index 000000000..45c03291e --- /dev/null +++ b/lab/prism/coordinator_config.py @@ -0,0 +1,2252 @@ +"""Immutable PRISM coordinator configuration and environment loading.""" + +from __future__ import annotations + +import json +import math +import shlex +import sys +from os import environ as _PROCESS_ENVIRON +from dataclasses import dataclass, replace as dataclass_replace +from decimal import Decimal, InvalidOperation +from pathlib import Path +from typing import Mapping + +from lab.auxpow import vardiff +from lab.prism import direct_stratum +from lab.prism.ctv_broadcaster_daemon import MAX_CTV_FANOUT_BROADCASTER_CHUNK_SIZE +from lab.prism.process_telemetry import HeapCensusConfig +from lab.prism.share_ledger import ( + DEFAULT_AUDIT_SHARE_SEGMENT_SIZE, + DEFAULT_CTV_BROADCAST_ATTEMPT_DETAIL_LIMIT, + DEFAULT_CTV_BROADCAST_RETRY_BACKOFF_SECONDS, + DEFAULT_LEASE_ACQUIRE_ATTEMPTS, + DEFAULT_LEASE_ACQUIRE_LOCK_TIMEOUT_SECONDS, + DEFAULT_POSTGRES_IDLE_IN_TRANSACTION_TIMEOUT_SECONDS, + DEFAULT_POSTGRES_TCP_KEEPALIVES_COUNT, + DEFAULT_POSTGRES_TCP_KEEPALIVES_IDLE_SECONDS, + DEFAULT_POSTGRES_TCP_KEEPALIVES_INTERVAL_SECONDS, +) +from lab.prism.writer_lease_timing import DEFAULT_WRITER_LEASE_HEARTBEAT_POLICY + + +DEFAULT_P2MR_SPEND_INPUT_BYTES = 3_680 +DEFAULT_MIN_OUTPUT_FEERATE_SATS_PER_BYTE = 1 +DEFAULT_MIN_OUTPUT_SAFETY_MULTIPLIER = 4 +DEFAULT_TESTNET_USERNAME_FALLBACK_ADDRESS = ( + "tq1zlsq9dpxz8mennhdpr9nf9s0f2tjtq6gxs9m84k6xglhkfp92q2zszzu4m3" +) +DEFAULT_PRISM_COINBASE_TAG = "/PRISM/" +MAX_PRISM_COINBASE_TAG_BYTES = 40 +DEFAULT_DIRECT_COINBASE_PAYOUT_FLOOR_SATS = 10_485_760 +DEFAULT_MAX_COINBASE_SETTLEMENT_OUTPUTS = 16 +DEFAULT_MAX_DIRECT_COINBASE_OUTPUTS = 12 +DEFAULT_MAX_CTV_FANOUT_RECIPIENTS_PER_TRANSACTION = 1_000 +DEFAULT_CTV_FANOUT_FEE_PREMIUM_BPS = 12_000 +TESTNET_QBIT_CHAINS = {"testnet", "testnet3", "testnet4", "signet"} +DEFAULT_PRISM_BLOCKPOLL_SECONDS = 2.0 +DEFAULT_PRISM_BLOCKWAIT_TIMEOUT_SECONDS = 5.0 +DEFAULT_PRISM_TIP_REFRESH_FAILURE_HOLDOFF_SECONDS = 1.0 +DEFAULT_PRISM_JOB_BUNDLE_CACHE_SECONDS = 10.0 +DEFAULT_PRISM_BUNDLE_BUILD_TIMEOUT_SECONDS = 60.0 +# Minimum spacing between speculative payout-ledger-artifact rebuilds armed by +# a failed reuse fence. Every fence failure wants a fresh artifact; the floor +# prevents repeated delta/carry reads and also paces any cold/full fallback. +DEFAULT_PRISM_PAYOUT_ARTIFACT_REARM_MIN_SECONDS = 5.0 +# How long an armed artifact keeps serving NEW reuse decisions, measured +# from its install (prepared_monotonic), never from its anchor: the reward +# window walk itself takes multiple seconds at production share volume, so +# a bound measured from the wall-clock anchor let a slow build arm an +# artifact that was already expired on arrival -- the born-expired rebuild +# livelock behind the 2026-07-29 rollback of the first anchor-scoped +# deploy. Freshness starts when the window becomes reusable. +DEFAULT_PRISM_PAYOUT_ARTIFACT_REANCHOR_SECONDS = 60.0 +# Normal generations may reuse the last materialized window until this floor +# elapses. Found-block settlement bypasses the floor; the default follows the +# existing re-anchor cadence so one knob continues to describe the expected +# freshness rhythm when the new setting is absent. +DEFAULT_PRISM_PAYOUT_ARTIFACT_MIN_BUILD_INTERVAL_SECONDS = 60.0 +# A full oracle pass periodically checks the append-only delta assumption. +# Reorg/carry mutations and invariant failures force this path immediately. +DEFAULT_PRISM_PAYOUT_ARTIFACT_FULL_RESCAN_SECONDS = 3_600.0 +# Route the payout-window fold, canonical digest, and incremental advance +# through the persistent --serve audit builder instead of in-process Python. +# Defaults to Python: the Rust path is adopted deliberately, per #131's +# per-component-switch method, and any daemon anomaly still degrades to the +# in-process pipeline for that materialization without flipping this. +DEFAULT_PRISM_WINDOW_PIPELINE_RUST = False +# Issue #226: bind glibc mallinfo2 through ctypes on every /metrics scrape +# and export arena/in-use/free/mmapped bytes beside the interpreter's own +# allocation and GC counters. On by default: it is the one always-on reading +# that separates glibc arena fragmentation from Python retention on a +# running coordinator. The switch exists because mallinfo2 walks glibc's +# free lists under each arena lock -- negligible on a healthy heap, but the +# only reading here that is not a plain counter load -- so an operator can +# stop that call without a redeploy. Off renders the allocator gauges as +# unavailable (availability 0, bytes -1); the interpreter gauges stay on. +DEFAULT_PRISM_MALLOC_TELEMETRY = True +# Issue #226 part 2: the operator-triggered heap census. Off by default and +# armed only by this switch: the walk (gc.get_objects plus a pass over every +# tracked object) holds the GIL for seconds on a multi-gigabyte heap, so it +# must never be reachable by accident. On, SIGUSR1 writes one bounded JSON +# report per signal to PRISM_HEAP_CENSUS_DIR; nothing else changes. +DEFAULT_PRISM_HEAP_CENSUS = False +# Where reports land: a subdirectory of the audit directory, which is the +# one persistent volume the compose stack mounts into the coordinator. +DEFAULT_PRISM_HEAP_CENSUS_SUBDIR = "heap-census" +# The bounds. Top-N caps both type lists and both tracemalloc lists; the +# byte cap is enforced by halving top-N until the JSON fits, never by cutting +# the document; the walk cap stops the per-object pass and marks the report +# truncated. The ceilings below are hard: a value above them refuses startup +# rather than promising a "bounded" census that is not. +DEFAULT_PRISM_HEAP_CENSUS_TOP_N = 40 +MAX_PRISM_HEAP_CENSUS_TOP_N = 200 +DEFAULT_PRISM_HEAP_CENSUS_MAX_BYTES = 262_144 +MIN_PRISM_HEAP_CENSUS_MAX_BYTES = 4_096 +MAX_PRISM_HEAP_CENSUS_MAX_BYTES = 4 * 1024 * 1024 +DEFAULT_PRISM_HEAP_CENSUS_MAX_SECONDS = 10.0 +MAX_PRISM_HEAP_CENSUS_MAX_SECONDS = 120.0 +# Signals arriving closer together than this coalesce into one action, so a +# repeated kill cannot stack heap walks or arena-lock trims. +DEFAULT_PRISM_HEAP_CENSUS_MIN_INTERVAL_SECONDS = 60.0 +# tracemalloc has a standing cost on every allocation for as long as it +# traces, so it is a second, separate opt-in that only means anything with +# the census on; on without the census refuses startup. +DEFAULT_PRISM_HEAP_CENSUS_TRACEMALLOC = False +# malloc_trim on SIGRTMIN+1. It takes every glibc arena lock in turn while +# it walks that arena's free lists, so it is an operator action, not a +# default; the periodic variant is paced and floored so it cannot become a +# steady stall. 0 disables the periodic trim. +DEFAULT_PRISM_MALLOC_TRIM_SIGNAL = False +DEFAULT_PRISM_MALLOC_TRIM_INTERVAL_SECONDS = 0.0 +MIN_PRISM_MALLOC_TRIM_INTERVAL_SECONDS = 60.0 + + +def validate_payout_artifact_age_bounds( + max_anchor_age_seconds: float, + refresh_floor_seconds: float, +) -> None: + """Refuse an anchor ceiling the background re-anchor cannot beat. + + A ceiling at or below the re-anchor floor plus the debounce and the + multi-second window walk turns every reuse probe into a ceiling + rejection while the rebuild runs debounced in parallel -- mechanically + the 2026-07-30 walk-per-build brownout. The 2x floor requirement keeps + comfortable margin over that sum at any plausible walk duration. + """ + if max_anchor_age_seconds <= 2.0 * refresh_floor_seconds: + raise SystemExit( + "PRISM_PAYOUT_ARTIFACT_MAX_ANCHOR_AGE_SECONDS must exceed " + "2x the payout-artifact refresh floor" + ) + + +# Audit-facing ceiling on how far a served window's declared anchor may +# trail the live ledger, in wall-clock terms. Deliberately much looser than +# the reuse-staleness knob: shares stamped after an anchor deterministically +# belong to the next window (never lost; bundles declare the artifact's own +# anchor, so reused windows stay audit-reproducible), and the ceiling only +# caps worst-case payout-preview lag under pathological build latency or a +# wedged pending-commit floor. An artifact whose anchor is already past the +# ceiling at install time is born expired: discarded, and counted as a +# preparation failure for re-arm backoff pacing. +DEFAULT_PRISM_PAYOUT_ARTIFACT_MAX_ANCHOR_AGE_SECONDS = 300.0 +# Bounds one routine/initial admission wait behind publication-priority +# builds. A parked request past the deadline fails fast with a visible log +# line and counter instead of silently riding the client's own much longer +# timeout; the initial-job runner retries with its normal pacing. +DEFAULT_PRISM_ROUTINE_ADMISSION_DEADLINE_SECONDS = 15.0 +DEFAULT_PRISM_CTV_BROADCASTER_CHUNK_SIZE = 5 +DEFAULT_PRISM_REORG_RECONCILE_CACHE_SECONDS = 5.0 +DEFAULT_PRISM_HEALTH_REFRESH_SECONDS = 5.0 +DEFAULT_PRISM_METRICS_REFRESH_SECONDS = 5.0 +DEFAULT_PRISM_HEALTH_PENDING_REFRESH_MAX_AGE_SECONDS = 15.0 +DEFAULT_PRISM_HEALTH_TIP_POLL_MAX_AGE_SECONDS = 15.0 +DEFAULT_PRISM_STRATUM_SEND_TIMEOUT_SECONDS = 20.0 +DEFAULT_PRISM_STRATUM_MAX_CONNECTIONS = 384 +DEFAULT_PRISM_STRATUM_MAX_CONNECTIONS_PER_USERNAME = 0 +DEFAULT_PRISM_STRATUM_MAX_PENDING_INITIAL_JOBS = 128 +DEFAULT_PRISM_STRATUM_INITIAL_JOB_TIMEOUT_SECONDS = 30.0 +DEFAULT_PRISM_MINING_HEALTH_STARTUP_GRACE_SECONDS = 30.0 +DEFAULT_PRISM_STRATUM_ACCEPT_RESOURCE_EXHAUSTION_BACKOFF_SECONDS = 1.0 +# Kernel accept backlog per stratum listener. Sized so a whole fleet +# reconnecting inside a restart window parks in the backlog instead of being +# SYN-dropped; the kernel caps the effective value at net.core.somaxconn. +DEFAULT_PRISM_STRATUM_LISTEN_BACKLOG = 1024 +# How long bind() retries EADDRINUSE at startup, covering overlap with a +# predecessor process that is still draining its shutdown while holding the +# port. Zero fails fast (the historical behavior). +DEFAULT_PRISM_STRATUM_BIND_RETRY_SECONDS = 10.0 +DEFAULT_PRISM_PAYOUT_ADDRESS_CACHE_MAX_ENTRIES = 4_096 +DEFAULT_PRISM_PAYOUT_ADDRESS_CACHE_TTL_SECONDS = 3_600.0 +DEFAULT_PRISM_STALE_GRACE_SECONDS = 3.0 +# How old the refresh-published tip may be before mining.submit stops trusting +# it and falls back to a live getbestblockhash per share. Healthy coordinators +# republish/reconfirm it every blockpoll interval; a bounded divergence lease +# separately covers replacement construction without failing open forever. +DEFAULT_PRISM_SUBMIT_TIP_MAX_AGE_SECONDS = 10.0 +DEFAULT_PRISM_SAME_TIP_JOB_RETENTION_SECONDS = 30.0 +DEFAULT_PRISM_SAME_TIP_JOB_RETENTION_PER_CONNECTION = 64 +# How many same-tip retained contexts may outlive their connection so a +# reconnecting client (same authorized username) can submit work computed +# against the prior connection's jobs instead of eating unknown-job rejects +# across a proxy flap. Entries keep the normal same-tip TTL; 0 restores the +# historical purge-on-disconnect behavior. +DEFAULT_PRISM_DISCONNECTED_JOB_RETENTION = 16_384 +DEFAULT_PRISM_TIP_REFRESH_MAX_WORKERS = 16 +DEFAULT_PRISM_INITIAL_JOB_MAX_WORKERS = 4 +DEFAULT_PRISM_JOB_BUILD_TIMEOUT_SECONDS = 60.0 +DEFAULT_PRISM_JOB_BUILD_CANCEL_GRACE_SECONDS = 0.25 +DEFAULT_PRISM_JOB_BUILD_EXECUTOR_WORKERS = 2 +# Upper bound for PRISM_PYTHON_SWITCH_INTERVAL_SECONDS. The interpreter default +# is 5 ms, so anything approaching a second is a misplaced decimal point rather +# than a tuning intent -- and the failure it would cause is the one hardest to +# read back from a running pool: threads holding the GIL for whole seconds, +# presenting as ack-latency and lock-convoy symptoms with no obvious cause. +# Reject at startup instead. +MAX_PRISM_PYTHON_SWITCH_INTERVAL_SECONDS = 1.0 +# Lower bound, and it is the interpreter's own resolution rather than a policy +# choice: CPython stores this interval in whole microseconds, so anything below +# one truncates to *zero* -- sys.setswitchinterval(9e-07) leaves +# sys.getswitchinterval() reporting 0.0. A zero interval asks for a switch check +# at every opportunity, which is the opposite of what someone lowering the value +# is reaching for and is worst precisely here, in a thread-heavy coordinator. +# The same truncation applies harmlessly above the bound: a requested 1.5us +# applies as 1us, so a tuned value may read back slightly lower than it was set. +MIN_PRISM_PYTHON_SWITCH_INTERVAL_SECONDS = 1e-06 +DEFAULT_PRISM_VARDIFF_IDLE_SWEEP_SECONDS = 15.0 +# Reconnect difficulty retention: a reconnecting worker (same listener + +# exact username) resumes at its last converged difficulty instead of the +# lane start, so a mass reconnect does not re-flood the share path while +# every session re-climbs. The TTL bounds how stale a resumed value may be, +# the entry cap bounds retained state (2x the deployed 4096 connection cap), +# and the start factor caps a resume at a multiple of the lane's own start +# difficulty (1024 = five 4x retarget steps, the climb reconnects previously +# repeated). +DEFAULT_PRISM_VARDIFF_RESUME_TTL_SECONDS = 900.0 +DEFAULT_PRISM_VARDIFF_RESUME_MAX_ENTRIES = 8192 +DEFAULT_PRISM_VARDIFF_RESUME_MAX_START_FACTOR = Decimal("1024") +# Fast-arrival initial convergence defaults (#132). 64x in one move is what +# lets a standard-lane session started at 16384 cross the 500000 a rental +# router filters at without waiting out several ordinary 4x retargets; 8 +# accepted shares over at least 1 second, showing at least a 4x gap, is the +# evidence required before that one move is allowed. +DEFAULT_PRISM_VARDIFF_INITIAL_MAX_STEP_UP = Decimal("64") +DEFAULT_PRISM_VARDIFF_INITIAL_MIN_SHARES = 8 +DEFAULT_PRISM_VARDIFF_INITIAL_MIN_STEP_UP = Decimal("4") +DEFAULT_PRISM_VARDIFF_INITIAL_MIN_SECONDS = Decimal("1") +DEFAULT_PRISM_WORKER_METRICS_LIMIT = 100 +DEFAULT_SHARE_COMMIT_BATCH_SIZE = 64 +DEFAULT_SHARE_COMMIT_LINGER_MILLISECONDS = 5.0 +DEFAULT_SHARE_COMMIT_TIMEOUT_SECONDS = 15.0 +DEFAULT_PRISM_WRITER_QUIESCENCE_TIMEOUT_SECONDS = 15.0 +DEFAULT_PRISM_TEMPLATE_MAX_AGE_SECONDS = 120 +DEFAULT_PRISM_COORDINATION_BLOCKED_EXIT_SECONDS = 900.0 +# How long a monitored coordinator thread may stay heartbeat-silent before the +# watchdog hard-exits the process. Deadlines that are spent on a monitored +# thread derive their ceiling from this value rather than repeating the number, +# so retuning the watchdog cannot leave a budget stranded above its tolerance. +DEFAULT_PRISM_WATCHDOG_TIMEOUT_SECONDS = 120.0 +# A watchdog release must never turn a wedged DB path into a second outage. +# The release worker is daemonized and the watchdog hard-exits at this total +# wall-clock deadline whether quiescence or the fresh DB connection completes. +DEFAULT_PRISM_WATCHDOG_LEASE_RELEASE_TIMEOUT_SECONDS = 5.0 +# A fast-adoptable coordinator proves its guarded session live several times +# inside the silence proof required by PsqlShareLedger. This runs through a +# dedicated connection, not the ordinary ledger lock or shared native pool, +# and must never wait on the lease tuple's row lock: fenced writes hold it +# for whole transactions (persist_accepted_block can exceed the guard's +# statement timeout many times over). The lease TTL is renewed only when +# that tuple is uncontended (SKIP LOCKED), so an idle coordinator still +# keeps lease_expires_at ahead of different-identity expiry claims. +# +# The five values below are not independent knobs. They are the terms of one +# inequality — exit before adoption — owned by +# lab.prism.writer_lease_timing.WriterLeaseHeartbeatPolicy, which derives +# them from measurable phase budgets (the guard session's statement timeout, +# the heartbeat interval, and a process scheduler-slack allowance) and +# validates them at import. They are read from that policy rather than +# re-derived here, so retuning one term cannot silently break the argument +# the others depend on. See writer_lease_timing for the derivation and for +# why the shipped adoption silence is what it is (issue #212). +DEFAULT_PRISM_LEDGER_LEASE_HEARTBEAT_SECONDS = ( + DEFAULT_WRITER_LEASE_HEARTBEAT_POLICY.heartbeat_interval_seconds +) +# The monitor kills a coordinator before PostgreSQL can observe the full +# adoption silence if the heartbeat query itself wedges. Its short release +# budget keeps the old process gone before a successor is allowed to CAS. +DEFAULT_PRISM_LEDGER_LEASE_HEARTBEAT_FAILURE_SECONDS = ( + DEFAULT_WRITER_LEASE_HEARTBEAT_POLICY.failure_budget_seconds +) +DEFAULT_PRISM_LEDGER_LEASE_HEARTBEAT_MONITOR_SECONDS = ( + DEFAULT_WRITER_LEASE_HEARTBEAT_POLICY.monitor_interval_seconds +) +DEFAULT_PRISM_LEDGER_LEASE_HEARTBEAT_EXIT_TIMEOUT_SECONDS = ( + DEFAULT_WRITER_LEASE_HEARTBEAT_POLICY.exit_margin_seconds +) +DEFAULT_PRISM_LEDGER_LEASE_EXTERNAL_FENCE_TIMEOUT_SECONDS = 0.5 +# Headroom added on top of the longest guarded RPC deadline when sizing the +# ledger's own-write deferral margin. It absorbs what the deadline itself +# cannot: the scheduling gap between the fence verification returning and +# the RPC's first byte, coordinator-vs-DB clock drift (the lease runway is +# measured by clock_timestamp()), and the node's application tail — the +# client-side deadline severs the socket, which bounds transmission but +# cannot cancel a handler qbitd already received in full, and these RPC +# handlers apply within milliseconds of complete receipt. The client side +# of the RPC can never exceed its wall-clock deadline (mutating methods are +# in _QBIT_RPC_NO_TRANSPORT_RETRY_METHODS and never retry; retried read +# paths share one deadline across attempts), so a multiplicative factor +# here would only make the deferral needlessly eager as operators raise +# deadlines. A node that stalls longer than this headroom after fully +# receiving a mutating request is the documented residual of preflight +# fencing between independent systems (see the external-side-effect fence +# docstring); no client-side teardown can bound it. +LEASE_AUTHORITY_MARGIN_HEADROOM_SECONDS = 5.0 +# The node fast lane is intentionally shorter than the normal ten-second RPC +# budget: an ambiguous timeout leaves the durable outbox pending and replay +# safely submits the same hash again. +DEFAULT_BLOCK_SUBMIT_RPC_TIMEOUT_SECONDS = 1.0 +# Every PostgreSQL operation reached by the submitter inherits this deadline. +# Direct outbox calls also run behind a coordinator-side single-flight guard, +# so a driver that ignores the deadline cannot freeze the submitter thread. +DEFAULT_BLOCK_SUBMIT_DB_TIMEOUT_SECONDS = 1.0 +# Landing-class deadline. The post-acceptance landing/accounting tail +# (persist, prior balances, confirm, prepared-state rejection of a terminal +# candidate) must not share the one-second budget meant for cheap outbox +# polls and fast-lane-adjacent calls: a structurally slow landing under that +# ceiling is statement-canceled on every attempt and can never complete +# (issue #188 — mainnet block 42743's landing was canceled 13 times while +# Postgres was otherwise idle). The first landing attempt receives this full +# budget; retries escalate after observed landing timeouts. +DEFAULT_BLOCK_LANDING_DB_TIMEOUT_SECONDS = 30.0 +# Reviewed escalation cap for landing retries. Per-attempt wall clock stays +# bounded by the escalated statement budget, cycles are paced by the +# candidate retry backoff, and the stuck-call/coordination watchdogs remain +# the overall bound. An accepted block's landing is never abandoned outright; +# it converges through durable replay. This is an upper bound only: landing +# steps run on watchdog-monitored block-work threads, so the watchdog-derived +# ceiling below can lower the budget actually granted, and only lower it. +DEFAULT_BLOCK_LANDING_DB_TIMEOUT_MAX_SECONDS = 120.0 +# Fraction of the configured watchdog tolerance a landing budget may occupy. +# A landing step spends its budget on the block-work thread the watchdog +# monitors, so the reviewed cap and the watchdog are not independent knobs: +# a cap at or above the tolerance lets a legal, still-running call trip the +# watchdog, and the in-memory escalation state dies with the process, so the +# restart replays the same doomed attempt from the base budget forever +# (issue #125). The landing scope is bracketed to keep that span short: the +# ledger admission wait stamps in slices, gate bodies that hold their lock +# across two statements report between them, and the reorg walk, prior- +# balance checks, tip-height RPC and unfenced-append drain each stamp their +# own phases. What remains between two stamps is normally one server-side +# statement, so half the tolerance leaves roughly a statement of margin +# before the watchdog fires, and the two values can no longer drift apart +# when either is retuned. It is a margin, not a guarantee: a network stall +# mid-statement has no client-side socket-read deadline on the native +# psycopg path and can outlast any budget -- that case is a genuinely +# wedged process, and a watchdog exit is the correct outcome. +BLOCK_LANDING_DB_TIMEOUT_WATCHDOG_FRACTION = 0.5 +DEFAULT_BLOCK_SUBMIT_LOCK_WAIT_LOG_SECONDS = 5.0 +DEFAULT_BLOCK_SUBMIT_STUCK_CALL_EXIT_SECONDS = 30.0 +# How long an own-hash tip observation keeps protecting a block candidate +# from terminal abandonment while instantaneous chain probes disagree +# (transient sibling/fork views during quick-succession blocks, RPC blips). +# Within the window the candidate defers and retries until a fresh probe +# proves it active again (the accepted tail then finalizes it as submitted); +# once the window expires with the hash still absent from the active chain, +# the candidate is genuinely stale and abandons terminally. +DEFAULT_PRISM_OBSERVED_TIP_ACCEPT_WINDOW_SECONDS = 300.0 +# Fail-closed bound on unresolved accepted-parent transitions (landed +# candidates whose durable bookkeeping has not reached a terminal state). +# Child jobs may keep issuing against published previews while landings +# complete, but once this many transitions are unresolved at once, new job +# builds block instead of stacking further prospective balance chains. The +# durable outbox (MAX_PENDING_BLOCK_CANDIDATES) remains the wider bound. +DEFAULT_ACCEPTED_PARENT_UNRESOLVED_DEPTH_MAX = 8 +# In-process targeted ancestor re-drive (issue #190). A found block whose +# finalization keeps deferring on the same unresolved accepted-ancestor +# payout transition can only be resolved by re-driving that ancestor through +# the durable-replay path -- the deferral itself merely re-checks the +# transition. After this many consecutive deferrals on one ancestor the +# submitter forces a durable-outbox enumeration (the same code startup +# replay runs) so the stuck ancestor is re-adopted and finalized in-process +# instead of waiting out the publication-progress watchdog restart. +DEFAULT_ACCEPTED_PARENT_REDRIVE_DEFER_THRESHOLD = 3 +# How many re-drives one ancestor may trigger before the mechanism stands +# down for that hash. On exhaustion deferrals continue exactly as before +# the fix and the watchdog restart remains the backstop, so the re-drive +# itself can never livelock the submitter. +DEFAULT_ACCEPTED_PARENT_REDRIVE_ATTEMPT_MAX = 3 +# Issue #198: the decided-height collapse keeps one in-memory retry record +# per terminal row whose post-write cleanup failed (see +# ``_CollapsedCandidateCleanup`` in ``lab/prism/block_candidates.py``). Those +# records are the only replay source the row has left, so they are never +# dropped; what is bounded instead is *admission*: once the backlog holds +# this many records the collapse stops handing new rows to its fenced bulk +# terminalization and every unselected row stays durable and pending for +# the ordinary per-row path. +# +# The default is derived from the candidate-storm measurement recorded in +# ``docs/prism-overload-alerts.md``: one record costs on the order of a +# kilobyte plus the pending-share holders it retains, a drained backlog +# recovers at thousands of records per second on an idle accounting lane, +# and the registry pins one terminal-outcome fence per record. The bound is +# half the terminal-outcome registry (MAX_BLOCK_CANDIDATE_TERMINAL_OUTCOMES, +# 8,192) so a saturated backlog can never pin the whole fence registry, and +# it absorbs the observed 3,120-candidate storm (3,119 siblings) plus one +# full replay page (1,024 rows) without engaging. +DEFAULT_BLOCK_CANDIDATE_CLEANUP_RETRY_BACKLOG_MAX = 4096 +# The knob may not exceed the terminal-outcome registry bound: every retry +# record pins its hash's fence, and a backlog allowed to grow past the +# registry would leave the fence eviction with nothing it may drop. Restated +# here rather than imported because ``block_candidates`` imports this module; +# a block-candidate test pins the two equal. +MAX_BLOCK_CANDIDATE_CLEANUP_RETRY_BACKLOG_MAX = 8192 +# Maximum payload rows in a forced replay query. Operators may reduce this +# independently of the cleanup bounds when candidate JSON is very large. +MAX_BLOCK_REPLAY_PAGE_SIZE = 1024 +DEFAULT_HIGHDIFF_DIFFICULTY = "500000" +DEFAULT_HIGHDIFF_MAX_DIFFICULTY = "4294967296" +# Coinbase output ordering policies accepted by PRISM_COINBASE_OUTPUT_POLICY. +# "canonical" preserves the historical lexicographic output ordering; +# "pool-fee-first" reserves a direct settlement slot for a positive pool fee +# and emits it at coinbase vout 0. +VALID_COINBASE_OUTPUT_POLICIES = frozenset({"canonical", "pool-fee-first"}) + + +Env = Mapping[str, str] + + +def _current_environ(environ: Env | None) -> Env: + return _PROCESS_ENVIRON if environ is None else environ + + +def env(name: str, default: str | None = None, *, environ: Env | None = None) -> str: + value = _current_environ(environ).get(name, default) + if value is None or value == "": + raise SystemExit(f"{name} is required") + return value + + +def env_int(name: str, default: int, *, environ: Env | None = None) -> int: + return int(env(name, str(default), environ=environ)) + + +def env_positive_int(name: str, default: int, *, environ: Env | None = None) -> int: + try: + value = env_int(name, default, environ=environ) + except ValueError as exc: + raise SystemExit(f"{name} must be an integer") from exc + if value <= 0: + raise SystemExit(f"{name} must be positive") + return value + + +def env_positive_int_with_legacy( + primary_name: str, + legacy_name: str, + default: int, + *, + environ: Env | None = None, +) -> int: + if env_optional(primary_name, environ=environ) is not None: + return env_positive_int(primary_name, default, environ=environ) + return env_positive_int(legacy_name, default, environ=environ) + + +def env_nonnegative_int(name: str, default: int, *, environ: Env | None = None) -> int: + try: + value = env_int(name, default, environ=environ) + except ValueError as exc: + raise SystemExit(f"{name} must be an integer") from exc + if value < 0: + raise SystemExit(f"{name} must be non-negative") + return value + + +def env_nonnegative_int_with_legacy( + primary_name: str, + legacy_name: str, + default: int, + *, + environ: Env | None = None, +) -> int: + if env_optional(primary_name, environ=environ) is not None: + return env_nonnegative_int(primary_name, default, environ=environ) + return env_nonnegative_int(legacy_name, default, environ=environ) + + +def env_positive_float(name: str, default: float, *, environ: Env | None = None) -> float: + try: + value = float(env(name, str(default), environ=environ)) + except ValueError as exc: + raise SystemExit(f"{name} must be a number") from exc + if not math.isfinite(value): + raise SystemExit(f"{name} must be finite") + if value <= 0: + raise SystemExit(f"{name} must be positive") + return value + + +def env_nonnegative_float(name: str, default: float, *, environ: Env | None = None) -> float: + try: + value = float(env(name, str(default), environ=environ)) + except ValueError as exc: + raise SystemExit(f"{name} must be a number") from exc + if not math.isfinite(value): + raise SystemExit(f"{name} must be finite") + if value < 0: + raise SystemExit(f"{name} must be non-negative") + return value + + +def env_optional_positive_int(name: str, *, environ: Env | None = None) -> int | None: + raw = env_optional(name, environ=environ) + if raw is None: + return None + try: + value = int(raw) + except ValueError as exc: + raise SystemExit(f"{name} must be an integer") from exc + if value <= 0: + raise SystemExit(f"{name} must be positive") + return value + + +def env_optional_positive_float(name: str, *, environ: Env | None = None) -> float | None: + raw = env_optional(name, environ=environ) + if raw is None: + return None + try: + value = float(raw) + except ValueError as exc: + raise SystemExit(f"{name} must be a number") from exc + if not math.isfinite(value): + raise SystemExit(f"{name} must be finite") + if value <= 0: + raise SystemExit(f"{name} must be positive") + return value + + +def env_optional_positive_int_with_legacy( + primary_name: str, + legacy_name: str, + *, + environ: Env | None = None, +) -> int | None: + value = env_optional_positive_int(primary_name, environ=environ) + if value is not None: + return value + return env_optional_positive_int(legacy_name, environ=environ) + + +def env_decimal(name: str, default: str, *, environ: Env | None = None) -> Decimal: + try: + value = Decimal(env(name, default, environ=environ)) + except InvalidOperation as exc: + raise SystemExit(f"{name} must be a decimal number") from exc + if not value.is_finite(): + raise SystemExit(f"{name} must be finite") + if value <= 0: + raise SystemExit(f"{name} must be positive") + return value + + +def env_bool(name: str, default: str, *, environ: Env | None = None) -> bool: + return env(name, default, environ=environ).lower() in {"1", "true", "yes", "on"} + + +_STRICT_BOOL_TRUE = frozenset({"1", "true", "yes", "on"}) +_STRICT_BOOL_FALSE = frozenset({"0", "false", "no", "off"}) + + +def env_strict_bool(name: str, default: bool, *, environ: Env | None = None) -> bool: + """Boolean that fails closed: anything non-boolean refuses startup. + + ``env_bool`` silently reads unrecognized values as false, which is the + wrong failure mode for a pipeline-routing switch -- a typo must be a + visible startup error, never a silent flip to either side. + """ + raw = _current_environ(environ).get(name) + if raw is None or raw == "": + return default + normalized = raw.lower() + if normalized in _STRICT_BOOL_TRUE: + return True + if normalized in _STRICT_BOOL_FALSE: + return False + raise SystemExit(f"{name} must be a boolean (0/1/true/false/yes/no/on/off)") + + +def env_window_pipeline_rust(*, environ: Env | None = None) -> bool: + """Whether the payout-window pipeline routes through the Rust daemon. + + Independent of PRISM_BUILDER_SERVE: with the daemon transport disabled + this switch is inert (the materialization path checks the transport + itself), not an error. + """ + return env_strict_bool( + "PRISM_WINDOW_PIPELINE_RUST", + DEFAULT_PRISM_WINDOW_PIPELINE_RUST, + environ=environ, + ) + + +def env_malloc_telemetry(*, environ: Env | None = None) -> bool: + """Whether /metrics binds glibc mallinfo2 for the allocator gauges. + + Strict like the Rust switch: a non-boolean value refuses startup through + the coordinator config loader rather than silently landing on either + side. The metrics renderer reads the same strict switch when it is + constructed, so it can only ever see a value the loader already + validated in this process's environment. + """ + return env_strict_bool( + "PRISM_MALLOC_TELEMETRY", + DEFAULT_PRISM_MALLOC_TELEMETRY, + environ=environ, + ) + + +def env_heap_census_config( + *, + environ: Env | None = None, + audit_dir: Path | None = None, +) -> HeapCensusConfig: + """The issue #226 census, trim, and tracing switches, validated fail-closed. + + Every switch is strict like the Rust and malloc-telemetry switches, and + every bound is range-checked against the ceilings beside its default: a + census that is "bounded" by a top-N of a million or a walk cap of an hour + is not bounded, so those values refuse startup. ``audit_dir`` is the + resolved ``PRISM_AUDIT_DIR`` when the coordinator loader already has it; + standalone callers get the same derivation the loader uses. + """ + source = _current_environ(environ) + if audit_dir is None: + evidence_path = Path(env("PRISM_EVIDENCE_PATH", "prism-live-evidence.json", environ=source)) + audit_dir = Path(env("PRISM_AUDIT_DIR", str(evidence_path.parent), environ=source)) + enabled = env_strict_bool("PRISM_HEAP_CENSUS", DEFAULT_PRISM_HEAP_CENSUS, environ=source) + # Optional: the compose passthrough delivers an empty string when the + # deploy dotenv leaves it unset, and empty must mean the default, not a + # refused startup. + output_dir = env_optional("PRISM_HEAP_CENSUS_DIR", environ=source) or str( + audit_dir / DEFAULT_PRISM_HEAP_CENSUS_SUBDIR + ) + top_n = env_positive_int( + "PRISM_HEAP_CENSUS_TOP_N", DEFAULT_PRISM_HEAP_CENSUS_TOP_N, environ=source + ) + if top_n > MAX_PRISM_HEAP_CENSUS_TOP_N: + raise SystemExit( + f"PRISM_HEAP_CENSUS_TOP_N must be at most {MAX_PRISM_HEAP_CENSUS_TOP_N}" + ) + max_bytes = env_positive_int( + "PRISM_HEAP_CENSUS_MAX_BYTES", DEFAULT_PRISM_HEAP_CENSUS_MAX_BYTES, environ=source + ) + if not MIN_PRISM_HEAP_CENSUS_MAX_BYTES <= max_bytes <= MAX_PRISM_HEAP_CENSUS_MAX_BYTES: + raise SystemExit( + "PRISM_HEAP_CENSUS_MAX_BYTES must be between " + f"{MIN_PRISM_HEAP_CENSUS_MAX_BYTES} and {MAX_PRISM_HEAP_CENSUS_MAX_BYTES}" + ) + max_seconds = env_positive_float( + "PRISM_HEAP_CENSUS_MAX_SECONDS", DEFAULT_PRISM_HEAP_CENSUS_MAX_SECONDS, environ=source + ) + if max_seconds > MAX_PRISM_HEAP_CENSUS_MAX_SECONDS: + raise SystemExit( + f"PRISM_HEAP_CENSUS_MAX_SECONDS must be at most {MAX_PRISM_HEAP_CENSUS_MAX_SECONDS:g}" + ) + min_interval = env_nonnegative_float( + "PRISM_HEAP_CENSUS_MIN_INTERVAL_SECONDS", + DEFAULT_PRISM_HEAP_CENSUS_MIN_INTERVAL_SECONDS, + environ=source, + ) + tracemalloc_enabled = env_strict_bool( + "PRISM_HEAP_CENSUS_TRACEMALLOC", DEFAULT_PRISM_HEAP_CENSUS_TRACEMALLOC, environ=source + ) + if tracemalloc_enabled and not enabled: + raise SystemExit("PRISM_HEAP_CENSUS_TRACEMALLOC requires PRISM_HEAP_CENSUS=1") + trim_signal = env_strict_bool( + "PRISM_MALLOC_TRIM_SIGNAL", DEFAULT_PRISM_MALLOC_TRIM_SIGNAL, environ=source + ) + trim_interval = env_nonnegative_float( + "PRISM_MALLOC_TRIM_INTERVAL_SECONDS", + DEFAULT_PRISM_MALLOC_TRIM_INTERVAL_SECONDS, + environ=source, + ) + if 0 < trim_interval < MIN_PRISM_MALLOC_TRIM_INTERVAL_SECONDS: + raise SystemExit( + "PRISM_MALLOC_TRIM_INTERVAL_SECONDS must be 0 or at least " + f"{MIN_PRISM_MALLOC_TRIM_INTERVAL_SECONDS:g}" + ) + return HeapCensusConfig( + enabled=enabled, + output_dir=output_dir, + top_n=top_n, + max_bytes=max_bytes, + max_seconds=max_seconds, + min_interval_seconds=min_interval, + tracemalloc_enabled=tracemalloc_enabled, + malloc_trim_signal_enabled=trim_signal, + malloc_trim_interval_seconds=trim_interval, + ) + + +# The census as shipped: every switch off, every bound at its default. The +# lifecycle config carries this so a positional construction keeps working. +DEFAULT_PRISM_HEAP_CENSUS_CONFIG = HeapCensusConfig( + enabled=DEFAULT_PRISM_HEAP_CENSUS, + output_dir=str(Path("prism-live-evidence.json").parent / DEFAULT_PRISM_HEAP_CENSUS_SUBDIR), + top_n=DEFAULT_PRISM_HEAP_CENSUS_TOP_N, + max_bytes=DEFAULT_PRISM_HEAP_CENSUS_MAX_BYTES, + max_seconds=DEFAULT_PRISM_HEAP_CENSUS_MAX_SECONDS, + min_interval_seconds=DEFAULT_PRISM_HEAP_CENSUS_MIN_INTERVAL_SECONDS, + tracemalloc_enabled=DEFAULT_PRISM_HEAP_CENSUS_TRACEMALLOC, + malloc_trim_signal_enabled=DEFAULT_PRISM_MALLOC_TRIM_SIGNAL, + malloc_trim_interval_seconds=DEFAULT_PRISM_MALLOC_TRIM_INTERVAL_SECONDS, +) + + +def env_optional_bool(name: str, *, environ: Env | None = None) -> bool | None: + raw = env_optional(name, environ=environ) + if raw is None: + return None + return raw.lower() in {"1", "true", "yes", "on"} + + +def env_optional(name: str, *, environ: Env | None = None) -> str | None: + value = _current_environ(environ).get(name) + if value is None or value == "": + return None + return value + + +def production_mode(*, environ: Env | None = None) -> bool: + return ( + env_bool("QBIT_PRODUCTION", "0", environ=environ) + or env_bool("QBIT_TOOLS_PRODUCTION", "0", environ=environ) + or env("QBIT_CHAIN", "regtest", environ=environ).lower() in {"main", "mainnet"} + ) + + +def validate_same_tip_job_retention_limits( + *, + retention_seconds: float, + per_connection: int, + max_connections: int, + production: bool, +) -> None: + if retention_seconds <= 0: + return + if per_connection <= 0: + raise SystemExit( + "PRISM_STRATUM_SAME_TIP_JOB_RETENTION_PER_CONNECTION must be positive " + "when same-tip retention is enabled" + ) + if production and max_connections <= 0: + raise SystemExit( + "production mode requires a positive PRISM_STRATUM_MAX_CONNECTIONS " + "when same-tip retention is enabled" + ) + + +def validate_block_candidate_cleanup_retry_backlog_max(value: int) -> int: + """Refuse a cleanup-retry backlog bound the fence registry cannot carry. + + Positivity is checked by the env reader; this is the upper bound. The + error names both the limit and why it exists so an operator retuning + the knob learns the contract rather than a bare number. + """ + if int(value) > MAX_BLOCK_CANDIDATE_CLEANUP_RETRY_BACKLOG_MAX: + raise SystemExit( + "PRISM_BLOCK_CANDIDATE_CLEANUP_RETRY_BACKLOG_MAX cannot exceed " + f"{MAX_BLOCK_CANDIDATE_CLEANUP_RETRY_BACKLOG_MAX}: every retry " + "record pins a terminal-outcome fence and the fence registry is " + "bounded at that size" + ) + return int(value) + + +def require_production_env(name: str, *, environ: Env | None = None) -> str: + value = env_optional(name, environ=environ) + if value is None: + raise SystemExit(f"production mode requires {name}") + return value + + +def validate_prism_production_gate(*, environ: Env | None = None) -> None: + if not production_mode(environ=environ): + return + + for name in ( + "PRISM_ALLOW_MEMORY_LEDGER", + "PRISM_ALLOW_TEST_SIGNING_SEEDS", + "PRISM_ALLOW_BUNDLE_EMBEDDED_LEDGER_KEY", + "PRISM_ALLOW_FIXED_LEDGER_SESSION_TOKEN", + ): + if env_bool(name, "0", environ=environ): + raise SystemExit(f"production mode rejects {name}=1") + + # Stale grace is deliberately not pinned to zero on mainnet: with a zero + # window every in-flight prior-tip share is rejected (Stratum error 21) at + # each block, which miners read as pool failure rather than as stale + # work. The bounded default credits those shares instead (see + # docs/prism-rejections.md; bundles with credited rows use the v1.1 + # audit schema, which this release's verifier accepts). + + production_difficulties: dict[str, Decimal] = {} + for name in ( + "PRISM_STRATUM_SHARE_DIFF", + "PRISM_STRATUM_VARDIFF_MIN_DIFF", + "PRISM_STRATUM_VARDIFF_START_DIFF", + "PRISM_STRATUM_VARDIFF_MAX_DIFF", + ): + raw_value = require_production_env(name, environ=environ) + if not raw_value: + raise SystemExit(f"production mode requires an explicit {name}") + try: + value = Decimal(raw_value) + except InvalidOperation as exc: + raise SystemExit(f"{name} must be a decimal number") from exc + if not value.is_finite() or value <= 0: + raise SystemExit(f"{name} must be positive") + if value == Decimal("0.000000001"): + raise SystemExit(f"{name} cannot use the lab-only 1e-9 difficulty") + production_difficulties[name] = value + if ( + production_difficulties["PRISM_STRATUM_VARDIFF_MIN_DIFF"] + > production_difficulties["PRISM_STRATUM_VARDIFF_START_DIFF"] + ): + raise SystemExit("production vardiff minimum exceeds its start difficulty") + if ( + production_difficulties["PRISM_STRATUM_VARDIFF_START_DIFF"] + > production_difficulties["PRISM_STRATUM_VARDIFF_MAX_DIFF"] + ): + raise SystemExit("production vardiff start exceeds its maximum difficulty") + + prism_database_url = env_optional("PRISM_DATABASE_URL", environ=environ) + if prism_database_url is None and env_optional("PRISM_POSTGRES_PSQL_COMMAND", environ=environ) is None: + raise SystemExit("production mode requires PRISM_DATABASE_URL or PRISM_POSTGRES_PSQL_COMMAND") + if env_optional("PRISM_POSTGRES_PASSWORD", environ=environ) == "change-this": + raise SystemExit("production mode requires a non-default PRISM_POSTGRES_PASSWORD") + if prism_database_url is not None and "change-this" in prism_database_url: + raise SystemExit("production mode requires a non-default PRISM_DATABASE_URL") + + require_production_env("PRISM_MANIFEST_SIGNING_SEED_HEX", environ=environ) + require_production_env("PRISM_LEDGER_ATTESTATION_SIGNING_SEED_HEX", environ=environ) + require_production_env("PRISM_LEDGER_WRITER_PUBLIC_KEY_HEX", environ=environ) + require_production_env("PRISM_LEDGER_WRITER_ID", environ=environ) + require_production_env("PRISM_LEDGER_WRITER_EPOCH", environ=environ) + require_production_env("PRISM_AUDIT_DIR", environ=environ) + require_production_env("PRISM_EVIDENCE_PATH", environ=environ) + + if env_optional("PRISM_LEDGER_WRITER_SESSION_TOKEN", environ=environ) is not None: + raise SystemExit("production mode requires managed ledger session tokens; unset PRISM_LEDGER_WRITER_SESSION_TOKEN") + + if env_nonnegative_int( + "PRISM_STRATUM_MAX_CONNECTIONS", + DEFAULT_PRISM_STRATUM_MAX_CONNECTIONS, + environ=environ, + ) <= 0: + raise SystemExit( + "production mode requires a positive PRISM_STRATUM_MAX_CONNECTIONS" + ) + env_positive_int( + "PRISM_STRATUM_MAX_PENDING_INITIAL_JOBS", + DEFAULT_PRISM_STRATUM_MAX_PENDING_INITIAL_JOBS, + environ=environ, + ) + if env_nonnegative_float( + "PRISM_STRATUM_INITIAL_JOB_TIMEOUT_SECONDS", + DEFAULT_PRISM_STRATUM_INITIAL_JOB_TIMEOUT_SECONDS, + environ=environ, + ) <= 0: + raise SystemExit( + "production mode requires a positive " + "PRISM_STRATUM_INITIAL_JOB_TIMEOUT_SECONDS" + ) + + require_production_env("QBIT_RPC_USER", environ=environ) + qbit_rpc_password = require_production_env("QBIT_RPC_PASSWORD", environ=environ) + if qbit_rpc_password == "change-this": + raise SystemExit("production mode requires a non-default QBIT_RPC_PASSWORD") + + if env("QBIT_CHAIN", "regtest", environ=environ).lower() in {"main", "mainnet"} and env_bool( + "PRISM_CTV_SETTLEMENT_ENABLED", "0", environ=environ + ): + require_production_env( + "PRISM_CTV_FANOUT_FEE_MARKET_RATE_BITS_PER_1000_WEIGHT", environ=environ + ) + env_positive_int( + "PRISM_CTV_FANOUT_FEE_MARKET_RATE_BITS_PER_1000_WEIGHT", 0, environ=environ + ) + + validate_same_tip_job_retention_limits( + retention_seconds=env_nonnegative_float( + "PRISM_STRATUM_SAME_TIP_JOB_RETENTION_SECONDS", + DEFAULT_PRISM_SAME_TIP_JOB_RETENTION_SECONDS, + environ=environ, + ), + per_connection=env_nonnegative_int( + "PRISM_STRATUM_SAME_TIP_JOB_RETENTION_PER_CONNECTION", + DEFAULT_PRISM_SAME_TIP_JOB_RETENTION_PER_CONNECTION, + environ=environ, + ), + max_connections=env_nonnegative_int( + "PRISM_STRATUM_MAX_CONNECTIONS", + DEFAULT_PRISM_STRATUM_MAX_CONNECTIONS, + environ=environ, + ), + production=True, + ) + + +def validate_hex(value: str, *, name: str, expected_bytes: int | None = None) -> str: + try: + bytes.fromhex(value) + except ValueError as exc: + raise SystemExit(f"{name} must be hex") from exc + if expected_bytes is not None and len(value) != expected_bytes * 2: + raise SystemExit(f"{name} must be {expected_bytes * 2} hex chars") + return value.lower() + + +def env_seed_hex(name: str, *, test_default: str, environ: Env | None = None) -> str: + value = env_optional(name, environ=environ) + if value is None: + if env_bool("PRISM_ALLOW_TEST_SIGNING_SEEDS", "0", environ=environ): + value = test_default + else: + raise SystemExit(f"{name} is required") + return validate_hex(value, name=name, expected_bytes=32) + + +def validate_initial_job_max_workers(workers: int, max_pending: int) -> int: + """Reject first-job worker counts the pending-client bound cannot feed. + + The dedicated executor's queue is bounded by + PRISM_STRATUM_MAX_PENDING_INITIAL_JOBS, so workers beyond that count can + never receive a task; unbounded values would also risk partially failed + thread creation on constrained hosts. + """ + if workers <= 0: + raise SystemExit("PRISM_INITIAL_JOB_MAX_WORKERS must be positive") + if workers > max_pending: + raise SystemExit( + "PRISM_INITIAL_JOB_MAX_WORKERS cannot exceed " + "PRISM_STRATUM_MAX_PENDING_INITIAL_JOBS" + ) + return workers + + +def resolve_initial_job_max_workers( + explicit: int | None, + max_pending: int, +) -> int: + """Resolve the first-job worker count against the pending-client bound. + + Deployments that bound pending clients below the default worker count + were valid before the worker knob existed; the implicit default must cap + itself to the pending bound rather than fail their startup. An explicit + setting keeps strict validation. + """ + if explicit is None: + return min(DEFAULT_PRISM_INITIAL_JOB_MAX_WORKERS, int(max_pending)) + return validate_initial_job_max_workers(explicit, max_pending) + + +def validate_job_build_executor_workers(workers: int) -> int: + """Reject executor widths the build scheduler can never use. + + The scheduler admits at most two concurrent build flights (one active, + one retiring); extra executor threads would never receive work, so a + higher setting would be a silent no-op rather than a tuning gain. + """ + if workers <= 0: + raise SystemExit("PRISM_JOB_BUILD_EXECUTOR_WORKERS must be positive") + if workers > DEFAULT_PRISM_JOB_BUILD_EXECUTOR_WORKERS: + raise SystemExit( + "PRISM_JOB_BUILD_EXECUTOR_WORKERS cannot exceed " + f"{DEFAULT_PRISM_JOB_BUILD_EXECUTOR_WORKERS}" + ) + return workers + + +def resolve_python_switch_interval_seconds( + *, environ: Env | None = None +) -> float | None: + """Return the configured interpreter switch interval, or None to leave it. + + Unset means the interpreter keeps whatever CPython defaults to (5 ms on + every version this pool has run), which is the deployed behaviour today. + Absent is therefore not "use 5 ms" but "do not call + ``sys.setswitchinterval`` at all" -- the two differ if a future + interpreter changes its own default, and this knob should not pin a value + the tree never chose. + """ + interval = env_optional_positive_float( + "PRISM_PYTHON_SWITCH_INTERVAL_SECONDS", environ=environ + ) + if interval is None: + return None + if interval < MIN_PRISM_PYTHON_SWITCH_INTERVAL_SECONDS: + raise SystemExit( + "PRISM_PYTHON_SWITCH_INTERVAL_SECONDS cannot be below " + f"{MIN_PRISM_PYTHON_SWITCH_INTERVAL_SECONDS} " + "(the interpreter would truncate it to zero)" + ) + if interval > MAX_PRISM_PYTHON_SWITCH_INTERVAL_SECONDS: + raise SystemExit( + "PRISM_PYTHON_SWITCH_INTERVAL_SECONDS cannot exceed " + f"{MAX_PRISM_PYTHON_SWITCH_INTERVAL_SECONDS}" + ) + return interval + + +def apply_python_switch_interval(*, environ: Env | None = None) -> float | None: + """Apply the configured switch interval; return what was applied, if any. + + Must run before any thread starts. The interval decides how long a + CPU-bound thread may hold the GIL before the interpreter forces a switch, + so it is process-global state that existing threads observe unevenly if it + changes underneath them. + + #143's gate evidence measured this as a real lever rather than a + micro-optimisation: a sweep moved throughput 15% and CPU-per-share 13%, + and the 5 ms default had the worst ack tail of the three settings tested. + The right value is host-specific -- it trades share-ack tail latency + against interpreter overhead, and the balance shifts with core count and + per-core speed -- so this ships as a knob with no default rather than as a + value picked on one machine and inherited everywhere. + """ + interval = resolve_python_switch_interval_seconds(environ=environ) + if interval is None: + return None + sys.setswitchinterval(interval) + return interval + + +def load_prism_vardiff_config( + startup_difficulty: Decimal, *, environ: Env | None = None +) -> vardiff.VardiffConfig: + initial_convergence_enabled = env_bool( + "PRISM_STRATUM_VARDIFF_INITIAL_CONVERGENCE", + "1", + environ=environ, + ) + high_diff_arrival_threshold = env_decimal( + "PRISM_STRATUM_VARDIFF_HIGH_DIFF_ARRIVAL_DIFF", + DEFAULT_HIGHDIFF_DIFFICULTY, + environ=environ, + ) + max_step_factor = env_decimal( + "PRISM_STRATUM_VARDIFF_MAX_STEP_UP", "4", environ=environ + ) + if initial_convergence_enabled: + initial_min_shares = env_int( + "PRISM_STRATUM_VARDIFF_INITIAL_MIN_SHARES", + DEFAULT_PRISM_VARDIFF_INITIAL_MIN_SHARES, + environ=environ, + ) + if initial_min_shares < 1: + raise SystemExit( + "PRISM_STRATUM_VARDIFF_INITIAL_MIN_SHARES must be at least 1" + ) + initial_max_step_factor = env_decimal( + "PRISM_STRATUM_VARDIFF_INITIAL_MAX_STEP_UP", + str(DEFAULT_PRISM_VARDIFF_INITIAL_MAX_STEP_UP), + environ=environ, + ) + initial_min_step_factor = env_decimal( + "PRISM_STRATUM_VARDIFF_INITIAL_MIN_STEP_UP", + str(DEFAULT_PRISM_VARDIFF_INITIAL_MIN_STEP_UP), + environ=environ, + ) + if initial_min_step_factor < 1: + raise SystemExit( + "PRISM_STRATUM_VARDIFF_INITIAL_MIN_STEP_UP must be at least 1" + ) + initial_min_seconds = env_decimal( + "PRISM_STRATUM_VARDIFF_INITIAL_MIN_SECONDS", + str(DEFAULT_PRISM_VARDIFF_INITIAL_MIN_SECONDS), + environ=environ, + ) + if initial_max_step_factor < max_step_factor: + raise SystemExit( + "PRISM_STRATUM_VARDIFF_INITIAL_MAX_STEP_UP is below " + "PRISM_STRATUM_VARDIFF_MAX_STEP_UP" + ) + else: + # The relaxation is off, so do not parse its environment settings: + # initial_convergence_ready() refuses immediately on + # initial_convergence_enabled, and calculate_next_difficulty() keeps + # the ordinary max_step_factor. Defaults keep VardiffConfig's + # unconditional invariants satisfied without allowing dead settings + # (including malformed strings) to prevent startup. + initial_min_shares = DEFAULT_PRISM_VARDIFF_INITIAL_MIN_SHARES + initial_min_step_factor = DEFAULT_PRISM_VARDIFF_INITIAL_MIN_STEP_UP + initial_max_step_factor = max( + DEFAULT_PRISM_VARDIFF_INITIAL_MAX_STEP_UP, + max_step_factor, + ) + initial_min_seconds = DEFAULT_PRISM_VARDIFF_INITIAL_MIN_SECONDS + return vardiff.VardiffConfig( + enabled=env_bool("PRISM_STRATUM_VARDIFF", "1", environ=environ), + target_share_interval_seconds=env_decimal( + "PRISM_STRATUM_VARDIFF_TARGET_SECONDS", "15", environ=environ + ), + min_difficulty=env_decimal( + "PRISM_STRATUM_VARDIFF_MIN_DIFF", str(startup_difficulty), environ=environ + ), + max_difficulty=env_decimal("PRISM_STRATUM_VARDIFF_MAX_DIFF", "1024", environ=environ), + retarget_interval_seconds=env_decimal( + "PRISM_STRATUM_VARDIFF_RETARGET_SECONDS", "90", environ=environ + ), + max_step_factor=max_step_factor, + startup_difficulty=env_decimal( + "PRISM_STRATUM_VARDIFF_START_DIFF", str(startup_difficulty), environ=environ + ), + max_step_down_factor=env_decimal( + "PRISM_STRATUM_VARDIFF_MAX_STEP_DOWN", "4", environ=environ + ), + ewma_alpha=env_decimal("PRISM_STRATUM_VARDIFF_EWMA_ALPHA", "0.4", environ=environ), + retarget_tolerance=env_decimal( + "PRISM_STRATUM_VARDIFF_RETARGET_TOLERANCE", "0.25", environ=environ + ), + initial_convergence_enabled=initial_convergence_enabled, + initial_max_step_factor=initial_max_step_factor, + initial_min_accepted_shares=initial_min_shares, + initial_min_step_factor=initial_min_step_factor, + initial_min_elapsed_seconds=initial_min_seconds, + high_diff_arrival_threshold=high_diff_arrival_threshold, + ) + + +@dataclass(frozen=True) +class StratumListenerProfile: + """One stratum listener with its own difficulty policy. + + Every listener feeds the same coordinator, ledger, and settlement path; + the profile only decides where to listen and which difficulty bounds its + clients get. + """ + + name: str + bind: str + port: int + share_difficulty: Decimal + vardiff_config: vardiff.VardiffConfig + heartbeat_name: str + # Difficulty this listener never advertises below, even when the qbit + # network target is easier. Zero means no floor: stamped jobs keep the + # network cap (a share is never required to be harder than a block). The + # high-diff listener sets its configured minimum because marketplace + # verification (NiceHash-style) checks the first advertised difficulty, + # which must hold even while network difficulty sits below the floor. + minimum_advertised_difficulty: Decimal = Decimal("0") + + +def load_prism_highdiff_listener( + base_bind: str, + base_vardiff_config: vardiff.VardiffConfig, + *, + environ: Env | None = None, +) -> StratumListenerProfile | None: + """Optional high-difficulty listener for rental-scale miners. + + Disabled unless PRISM_STRATUM_HIGHDIFF_PORT is set. The 500k default floor + matches the NiceHash SHA-256 pool-verification minimum, which must hold + from the first mining.set_difficulty a client sees. + """ + + port_value = env_optional("PRISM_STRATUM_HIGHDIFF_PORT", environ=environ) + if port_value is None: + return None + try: + port = int(port_value) + except ValueError as exc: + raise SystemExit("PRISM_STRATUM_HIGHDIFF_PORT must be an integer") from exc + if not 0 < port < 65536: + raise SystemExit("PRISM_STRATUM_HIGHDIFF_PORT must be a valid TCP port") + min_difficulty = env_decimal( + "PRISM_STRATUM_HIGHDIFF_MIN_DIFF", DEFAULT_HIGHDIFF_DIFFICULTY, environ=environ + ) + start_difficulty = env_decimal( + "PRISM_STRATUM_HIGHDIFF_START_DIFF", DEFAULT_HIGHDIFF_DIFFICULTY, environ=environ + ) + max_difficulty = env_decimal( + "PRISM_STRATUM_HIGHDIFF_MAX_DIFF", DEFAULT_HIGHDIFF_MAX_DIFFICULTY, environ=environ + ) + if min_difficulty > start_difficulty: + raise SystemExit("PRISM_STRATUM_HIGHDIFF_MIN_DIFF exceeds PRISM_STRATUM_HIGHDIFF_START_DIFF") + if start_difficulty > max_difficulty: + raise SystemExit("PRISM_STRATUM_HIGHDIFF_START_DIFF exceeds PRISM_STRATUM_HIGHDIFF_MAX_DIFF") + # The fixed difficulty (used when vardiff is disabled) tracks the start + # difficulty unless explicitly set, and must respect the listener bounds: + # advertising below the floor would break the marketplace verification + # this listener exists for. + share_value = env_optional("PRISM_STRATUM_HIGHDIFF_SHARE_DIFF", environ=environ) + if share_value is None: + share_difficulty = start_difficulty + else: + try: + share_difficulty = Decimal(share_value) + except Exception as exc: + raise SystemExit("PRISM_STRATUM_HIGHDIFF_SHARE_DIFF must be a decimal") from exc + if not share_difficulty.is_finite() or share_difficulty <= 0: + raise SystemExit("PRISM_STRATUM_HIGHDIFF_SHARE_DIFF must be positive") + if share_difficulty < min_difficulty: + raise SystemExit("PRISM_STRATUM_HIGHDIFF_SHARE_DIFF is below PRISM_STRATUM_HIGHDIFF_MIN_DIFF") + if share_difficulty > max_difficulty: + raise SystemExit("PRISM_STRATUM_HIGHDIFF_SHARE_DIFF exceeds PRISM_STRATUM_HIGHDIFF_MAX_DIFF") + try: + config = dataclass_replace( + base_vardiff_config, + min_difficulty=min_difficulty, + max_difficulty=max_difficulty, + startup_difficulty=start_difficulty, + ) + except ValueError as exc: + raise SystemExit(f"invalid PRISM_STRATUM_HIGHDIFF_* difficulty bounds: {exc}") from exc + return StratumListenerProfile( + name="highdiff", + # env_optional so an empty value (compose default passthrough) means + # "inherit the default listener bind" instead of a startup failure. + bind=env_optional("PRISM_STRATUM_HIGHDIFF_BIND", environ=environ) or base_bind, + port=port, + share_difficulty=share_difficulty, + vardiff_config=config, + heartbeat_name="stratum_accept_highdiff", + minimum_advertised_difficulty=min_difficulty, + ) + + +def default_prism_payout_policy(*, environ: Env | None = None) -> dict[str, object]: + policy: dict[str, object] = { + "p2mr_spend_input_bytes": env_positive_int( + "PRISM_PAYOUT_P2MR_SPEND_INPUT_BYTES", + DEFAULT_P2MR_SPEND_INPUT_BYTES, + environ=environ, + ), + "target_feerate_sats_per_byte": env_positive_int_with_legacy( + "PRISM_PAYOUT_TARGET_FEERATE_BITS_PER_BYTE", + "PRISM_PAYOUT_TARGET_FEERATE_SATS_PER_BYTE", + DEFAULT_MIN_OUTPUT_FEERATE_SATS_PER_BYTE, + environ=environ, + ), + "safety_multiplier": env_positive_int( + "PRISM_PAYOUT_SAFETY_MULTIPLIER", + DEFAULT_MIN_OUTPUT_SAFETY_MULTIPLIER, + environ=environ, + ), + } + min_output_sats = env_optional_positive_int_with_legacy( + "PRISM_PAYOUT_MIN_OUTPUT_BITS", + "PRISM_PAYOUT_MIN_OUTPUT_SATS", + environ=environ, + ) + if min_output_sats is not None: + policy["min_output_sats"] = min_output_sats + return policy + + +def default_prism_coinbase_tag_hex(*, environ: Env | None = None) -> str: + tag = _current_environ(environ).get("PRISM_COINBASE_TAG", DEFAULT_PRISM_COINBASE_TAG) + try: + tag_bytes = tag.encode("ascii") + except UnicodeEncodeError as exc: + raise SystemExit("PRISM_COINBASE_TAG must be ASCII") from exc + if len(tag_bytes) > MAX_PRISM_COINBASE_TAG_BYTES: + raise SystemExit( + f"PRISM_COINBASE_TAG must be at most {MAX_PRISM_COINBASE_TAG_BYTES} bytes" + ) + if any(byte < 0x20 or byte > 0x7e for byte in tag_bytes): + raise SystemExit("PRISM_COINBASE_TAG must contain printable ASCII only") + return tag_bytes.hex() + + +def default_prism_username_fallback_address(*, environ: Env | None = None) -> str | None: + configured = env_optional("PRISM_USERNAME_FALLBACK_ADDRESS", environ=environ) + if configured is not None: + return configured + if (_current_environ(environ).get("QBIT_CHAIN") or "regtest").lower() in TESTNET_QBIT_CHAINS: + return DEFAULT_TESTNET_USERNAME_FALLBACK_ADDRESS + return None + + +def _parse_share_weights(raw: str) -> tuple[tuple[str, int], ...]: + if not raw: + return () + try: + parsed = json.loads(raw) + except json.JSONDecodeError as exc: + raise SystemExit(f"PRISM_STRATUM_SHARE_WEIGHTS_JSON is not valid JSON: {exc}") from exc + if not isinstance(parsed, dict): + raise SystemExit("PRISM_STRATUM_SHARE_WEIGHTS_JSON must be an object") + weights: list[tuple[str, int]] = [] + for username, weight in parsed.items(): + parsed_weight = int(weight) + if parsed_weight <= 0: + raise SystemExit(f"share weight for {username} must be positive") + weights.append((str(username), parsed_weight)) + return tuple(weights) + + +def load_share_weights(*, environ: Env | None = None) -> dict[str, int]: + """Load the legacy username-weight mapping for compatibility callers.""" + + source = _current_environ(environ) + return dict(_parse_share_weights(source.get("PRISM_STRATUM_SHARE_WEIGHTS_JSON", ""))) + + +@dataclass(frozen=True) +class RpcConfig: + host: str + port: int + user: str + password: str + chain: str + expected_genesis_hash: str | None + minimum_peers_raw: str | None + + +@dataclass(frozen=True) +class StratumConfig: + bind: str + port: int + extranonce2_size: int + stale_grace_seconds: float + send_timeout_seconds: float + max_connections: int + max_connections_per_username: int + max_pending_initial_jobs: int + initial_job_timeout_seconds: float + initial_job_max_workers: int + accept_resource_exhaustion_backoff_seconds: float + listen_backlog: int + bind_retry_seconds: float + share_difficulty: Decimal + vardiff_config: vardiff.VardiffConfig + vardiff_idle_sweep_seconds: float + listener_profiles: tuple[StratumListenerProfile, ...] + default_share_weight: int + share_weights_by_username: tuple[tuple[str, int], ...] + username_fallback_address: str | None + fallback_version_mask: int + same_tip_job_retention_seconds: float + same_tip_job_retention_per_connection: int + disconnected_job_retention: int + payout_address_cache_max_entries: int + payout_address_cache_ttl_seconds: float + # Reconnect difficulty retention knobs, appended with defaults so existing + # positional constructions keep working. + vardiff_resume_enabled: bool = True + vardiff_resume_ttl_seconds: float = DEFAULT_PRISM_VARDIFF_RESUME_TTL_SECONDS + vardiff_resume_max_entries: int = DEFAULT_PRISM_VARDIFF_RESUME_MAX_ENTRIES + vardiff_resume_max_start_factor: Decimal = ( + DEFAULT_PRISM_VARDIFF_RESUME_MAX_START_FACTOR + ) + + +@dataclass(frozen=True) +class JobPipelineConfig: + blockpoll_seconds: float + blockwait_enabled: bool + blockwait_timeout_seconds: float + tip_refresh_failure_holdoff_seconds: float + submit_tip_max_age_seconds: float + tip_refresh_max_workers: int + tip_refresh_epoch_fanout: bool + job_build_timeout_seconds: float + job_build_cancel_grace_seconds: float + job_build_executor_workers: int + routine_admission_deadline_seconds: float + worker_metrics_limit: int + reorg_reconciler_enabled: bool + job_bundle_cache_seconds: float + bundle_build_timeout_seconds: float + payout_artifact_rearm_min_seconds: float + payout_artifact_reuse_enabled: bool + payout_artifact_reanchor_seconds: float + payout_artifact_min_build_interval_seconds: float + payout_artifact_full_rescan_seconds: float + payout_artifact_max_anchor_age_seconds: float + template_cache_seconds: float + template_refresh_failure_exit_seconds: float + reorg_reconcile_cache_seconds: float + min_ready_miners: int + payout_environment: tuple[tuple[str, str], ...] + pool_fee_enabled_raw: str | None + pool_fee_bps_raw: str | None + pool_fee_address: str | None + pool_fee_program_hex: str | None + pool_fee_recipient_id: str | None + pool_fee_order_key: str | None + template_max_age_raw: str | None + # Appended with a default so existing positional constructions keep + # working. The loader passes the strict env read, so a non-boolean + # PRISM_WINDOW_PIPELINE_RUST refuses startup rather than starting on + # either side of the switch. + window_pipeline_rust_enabled: bool = DEFAULT_PRISM_WINDOW_PIPELINE_RUST + + +@dataclass(frozen=True) +class BlockConfig: + submit_rpc_timeout_seconds: float + submit_db_timeout_seconds: float + landing_db_timeout_seconds: float + landing_db_timeout_max_seconds: float + submit_lock_wait_log_seconds: float + submit_stuck_call_exit_seconds: float + observed_tip_accept_window_seconds: float + accepted_parent_unresolved_depth_max: int + # Defaulted so existing positional constructions keep working. + accepted_parent_redrive_defer_threshold: int = ( + DEFAULT_ACCEPTED_PARENT_REDRIVE_DEFER_THRESHOLD + ) + accepted_parent_redrive_attempt_max: int = ( + DEFAULT_ACCEPTED_PARENT_REDRIVE_ATTEMPT_MAX + ) + # Issue #198: retry records the collapse cleanup backlog may hold before + # bulk terminalization stops admitting new rows. + candidate_cleanup_retry_backlog_max: int = ( + DEFAULT_BLOCK_CANDIDATE_CLEANUP_RETRY_BACKLOG_MAX + ) + replay_page_size: int = MAX_BLOCK_REPLAY_PAGE_SIZE + + +@dataclass(frozen=True) +class LedgerConfig: + psql_command: str + database_url: str | None + allow_memory_ledger: bool + native_client_mode: str + writer_id: str + writer_epoch: int + writer_session_token: str | None + initialize_schema: bool + lease_ttl_seconds: float + lease_acquire_lock_timeout_seconds: float + lease_acquire_attempts: int + postgres_idle_in_transaction_timeout_seconds: float + postgres_tcp_keepalives_idle_seconds: int + postgres_tcp_keepalives_interval_seconds: int + postgres_tcp_keepalives_count: int + read_concurrency: int + accepted_stats_cache_seconds: float + reward_window_cache_seconds: float + signing_seed_hex: str + attestation_signing_seed_hex: str + writer_public_key_hex: str | None + share_commit_batch_size: int + share_commit_linger_seconds: float + share_commit_timeout_seconds: float + share_recovery_path: Path + # Issue #255: chunked candidate bodies. Version 2 writes bounded chunk + # bodies; version 1 keeps the legacy whole-jsonb write for the + # compatibility/rollback release. The reader handles both. + candidate_storage_version: int = 2 + candidate_spool_dir: str | None = None + candidate_spool_reservation_bytes: int = 4 * 1024 * 1024 * 1024 + + +@dataclass(frozen=True) +class AuditConfig: + evidence_path: Path + directory: Path + share_segment_size: int + live_bundle_retention: int + candidate_retention_seconds: int + bind: str | None + port: int + + +@dataclass(frozen=True) +class CtvConfig: + settlement_enabled_raw: str | None + broadcaster_enabled: bool + broadcaster_wallet: str | None + broadcaster_fee_sats: int + broadcaster_limit: int + broadcaster_chunk_size: int + broadcaster_interval_seconds: float + broadcast_attempt_detail_limit: int + broadcast_retry_backoff_seconds: int + settlement_environment: tuple[tuple[str, str], ...] + + +@dataclass(frozen=True) +class LifecycleConfig: + health_refresh_seconds: float + metrics_refresh_seconds: float + pending_refresh_health_deadline_seconds: float + coherent_tip_poll_health_deadline_seconds: float + mining_health_startup_grace_seconds: float + writer_quiescence_timeout_seconds: float + coordination_blocked_exit_seconds: float + watchdog_enabled: bool + watchdog_timeout_seconds: float + watchdog_interval_seconds: float + watchdog_lease_release_timeout_seconds: float + ledger_lease_heartbeat_seconds: float + ledger_lease_heartbeat_failure_seconds: float + ledger_lease_heartbeat_monitor_seconds: float + ledger_lease_heartbeat_exit_timeout_seconds: float + ledger_lease_external_fence_timeout_seconds: float + # Appended with a default so existing positional constructions keep + # working. The loader passes the strict env read, so a non-boolean + # PRISM_MALLOC_TELEMETRY refuses startup. The metrics renderer resolves + # the same switch from the environment itself; this field records the + # validated value in the config snapshot. + malloc_telemetry_enabled: bool = DEFAULT_PRISM_MALLOC_TELEMETRY + # Issue #226 part 2. Validated by the loader so a bad census bound + # refuses startup. prism_coordinator.main re-reads the same strict + # switches from the environment when it arms the census (the + # malloc-telemetry precedent above), so this field records the validated + # value in the config snapshot and can only ever agree with what main saw. + heap_census: HeapCensusConfig = DEFAULT_PRISM_HEAP_CENSUS_CONFIG + + +@dataclass(frozen=True) +class CoordinatorConfig: + rpc: RpcConfig + stratum: StratumConfig + jobs: JobPipelineConfig + block: BlockConfig + ledger: LedgerConfig + audit: AuditConfig + ctv: CtvConfig + lifecycle: LifecycleConfig + production: bool + hot_path_log_enabled: bool + coinbase_tag_hex: str + stop_after_block: bool + max_blocks: int + + +def _selected_environment(source: Env, names: tuple[str, ...]) -> tuple[tuple[str, str], ...]: + return tuple((name, source[name]) for name in names if name in source) + + +def load_coordinator_config(environ: Env | None = None) -> CoordinatorConfig: + """Load and validate one immutable configuration snapshot. + + Passing a mapping gives tests and embedding callers a no-global-environment + construction path. The zero-argument production path snapshots ``os.environ``. + """ + + source: Env = dict(_PROCESS_ENVIRON) if environ is None else dict(environ) + validate_prism_production_gate(environ=source) + production = production_mode(environ=source) + + rpc = RpcConfig( + host=env("QBIT_RPC_HOST", environ=source), + port=env_int("QBIT_RPC_PORT", 18452, environ=source), + user=env("QBIT_RPC_USER", environ=source), + password=env("QBIT_RPC_PASSWORD", environ=source), + chain=env("QBIT_CHAIN", "regtest", environ=source), + expected_genesis_hash=env_optional("QBIT_EXPECTED_GENESIS_HASH", environ=source), + minimum_peers_raw=source.get("PRISM_MIN_PEERS"), + ) + + blockpoll_seconds = env_positive_float( + "PRISM_BLOCKPOLL_SECONDS", DEFAULT_PRISM_BLOCKPOLL_SECONDS, environ=source + ) + same_tip_seconds = env_nonnegative_float( + "PRISM_STRATUM_SAME_TIP_JOB_RETENTION_SECONDS", + DEFAULT_PRISM_SAME_TIP_JOB_RETENTION_SECONDS, + environ=source, + ) + same_tip_per_connection = env_nonnegative_int( + "PRISM_STRATUM_SAME_TIP_JOB_RETENTION_PER_CONNECTION", + DEFAULT_PRISM_SAME_TIP_JOB_RETENTION_PER_CONNECTION, + environ=source, + ) + max_connections = env_nonnegative_int( + "PRISM_STRATUM_MAX_CONNECTIONS", DEFAULT_PRISM_STRATUM_MAX_CONNECTIONS, environ=source + ) + max_pending_initial_jobs = env_positive_int( + "PRISM_STRATUM_MAX_PENDING_INITIAL_JOBS", + DEFAULT_PRISM_STRATUM_MAX_PENDING_INITIAL_JOBS, + environ=source, + ) + if max_connections > 0 and max_pending_initial_jobs > max_connections: + raise SystemExit( + "PRISM_STRATUM_MAX_PENDING_INITIAL_JOBS cannot exceed " + "PRISM_STRATUM_MAX_CONNECTIONS" + ) + validate_same_tip_job_retention_limits( + retention_seconds=same_tip_seconds, + per_connection=same_tip_per_connection, + max_connections=max_connections, + production=production, + ) + initial_job_max_workers = resolve_initial_job_max_workers( + env_optional_positive_int("PRISM_INITIAL_JOB_MAX_WORKERS", environ=source), + max_pending_initial_jobs, + ) + job_build_executor_workers = validate_job_build_executor_workers( + env_positive_int( + "PRISM_JOB_BUILD_EXECUTOR_WORKERS", + DEFAULT_PRISM_JOB_BUILD_EXECUTOR_WORKERS, + environ=source, + ) + ) + tip_refresh_max_workers = env_positive_int( + "PRISM_TIP_REFRESH_MAX_WORKERS", DEFAULT_PRISM_TIP_REFRESH_MAX_WORKERS, environ=source + ) + if tip_refresh_max_workers > DEFAULT_PRISM_TIP_REFRESH_MAX_WORKERS: + raise SystemExit( + "PRISM_TIP_REFRESH_MAX_WORKERS cannot exceed " + f"{DEFAULT_PRISM_TIP_REFRESH_MAX_WORKERS}" + ) + template_refresh_failure_exit_seconds = env_nonnegative_float( + "PRISM_TEMPLATE_REFRESH_FAILURE_EXIT_SECONDS", + DEFAULT_PRISM_TEMPLATE_MAX_AGE_SECONDS, + environ=source, + ) + if production and template_refresh_failure_exit_seconds <= 0: + raise SystemExit( + "production mode requires a positive PRISM_TEMPLATE_REFRESH_FAILURE_EXIT_SECONDS" + ) + payout_artifact_rearm_min_seconds = env_positive_float( + "PRISM_PAYOUT_ARTIFACT_REARM_MIN_SECONDS", + DEFAULT_PRISM_PAYOUT_ARTIFACT_REARM_MIN_SECONDS, + environ=source, + ) + payout_artifact_reanchor_seconds = env_positive_float( + "PRISM_PAYOUT_ARTIFACT_REANCHOR_SECONDS", + DEFAULT_PRISM_PAYOUT_ARTIFACT_REANCHOR_SECONDS, + environ=source, + ) + payout_artifact_min_build_interval_seconds = env_positive_float( + "PRISM_PAYOUT_ARTIFACT_MIN_BUILD_INTERVAL_SECONDS", + payout_artifact_reanchor_seconds, + environ=source, + ) + payout_artifact_max_anchor_age_seconds = env_positive_float( + "PRISM_PAYOUT_ARTIFACT_MAX_ANCHOR_AGE_SECONDS", + DEFAULT_PRISM_PAYOUT_ARTIFACT_MAX_ANCHOR_AGE_SECONDS, + environ=source, + ) + validate_payout_artifact_age_bounds( + payout_artifact_max_anchor_age_seconds, + max( + payout_artifact_reanchor_seconds, + payout_artifact_min_build_interval_seconds, + ), + ) + + bind = env("PRISM_STRATUM_BIND", "127.0.0.1", environ=source) + port = env_int("PRISM_STRATUM_PORT", 3340, environ=source) + share_difficulty = env_decimal("PRISM_STRATUM_SHARE_DIFF", "0.000000001", environ=source) + vardiff_config = load_prism_vardiff_config(share_difficulty, environ=source) + listener_profiles = [ + StratumListenerProfile( + name="default", + bind=bind, + port=port, + share_difficulty=share_difficulty, + vardiff_config=vardiff_config, + heartbeat_name="stratum_accept", + ) + ] + highdiff_profile = load_prism_highdiff_listener(bind, vardiff_config, environ=source) + if highdiff_profile is not None: + if highdiff_profile.port == port and highdiff_profile.bind == bind: + raise SystemExit("PRISM_STRATUM_HIGHDIFF_PORT must differ from PRISM_STRATUM_PORT") + listener_profiles.append(highdiff_profile) + default_share_weight = env_int("PRISM_STRATUM_SHARE_WEIGHT", 1, environ=source) + if default_share_weight <= 0: + raise SystemExit("PRISM_STRATUM_SHARE_WEIGHT must be positive") + try: + fallback_version_mask = direct_stratum.normalize_version_rolling_mask( + env( + "PRISM_VERSION_ROLLING_MASK", + direct_stratum.QBIT_VERSION_ROLLING_MASK_HEX, + environ=source, + ), + field_name="PRISM_VERSION_ROLLING_MASK", + ) + except ValueError as exc: + raise SystemExit(str(exc)) from exc + + vardiff_resume_max_start_factor = env_decimal( + "PRISM_STRATUM_VARDIFF_RESUME_MAX_START_FACTOR", + str(DEFAULT_PRISM_VARDIFF_RESUME_MAX_START_FACTOR), + environ=source, + ) + if vardiff_resume_max_start_factor < 1: + raise SystemExit( + "PRISM_STRATUM_VARDIFF_RESUME_MAX_START_FACTOR must be at least 1" + ) + + stratum = StratumConfig( + bind=bind, + port=port, + extranonce2_size=env_int("PRISM_STRATUM_EXTRANONCE2_SIZE", 8, environ=source), + stale_grace_seconds=env_nonnegative_float( + "PRISM_STRATUM_STALE_GRACE_SECONDS", DEFAULT_PRISM_STALE_GRACE_SECONDS, environ=source + ), + send_timeout_seconds=env_nonnegative_float( + "PRISM_STRATUM_SEND_TIMEOUT_SECONDS", + DEFAULT_PRISM_STRATUM_SEND_TIMEOUT_SECONDS, + environ=source, + ), + max_connections=max_connections, + max_connections_per_username=env_nonnegative_int( + "PRISM_STRATUM_MAX_CONNECTIONS_PER_USERNAME", + DEFAULT_PRISM_STRATUM_MAX_CONNECTIONS_PER_USERNAME, + environ=source, + ), + max_pending_initial_jobs=max_pending_initial_jobs, + initial_job_timeout_seconds=env_nonnegative_float( + "PRISM_STRATUM_INITIAL_JOB_TIMEOUT_SECONDS", + DEFAULT_PRISM_STRATUM_INITIAL_JOB_TIMEOUT_SECONDS, + environ=source, + ), + initial_job_max_workers=initial_job_max_workers, + accept_resource_exhaustion_backoff_seconds=env_positive_float( + "PRISM_STRATUM_ACCEPT_RESOURCE_EXHAUSTION_BACKOFF_SECONDS", + DEFAULT_PRISM_STRATUM_ACCEPT_RESOURCE_EXHAUSTION_BACKOFF_SECONDS, + environ=source, + ), + listen_backlog=env_positive_int( + "PRISM_STRATUM_LISTEN_BACKLOG", DEFAULT_PRISM_STRATUM_LISTEN_BACKLOG, environ=source + ), + bind_retry_seconds=env_nonnegative_float( + "PRISM_STRATUM_BIND_RETRY_SECONDS", + DEFAULT_PRISM_STRATUM_BIND_RETRY_SECONDS, + environ=source, + ), + share_difficulty=share_difficulty, + vardiff_config=vardiff_config, + vardiff_idle_sweep_seconds=env_nonnegative_float( + "PRISM_STRATUM_VARDIFF_IDLE_SWEEP_SECONDS", + DEFAULT_PRISM_VARDIFF_IDLE_SWEEP_SECONDS, + environ=source, + ), + listener_profiles=tuple(listener_profiles), + default_share_weight=default_share_weight, + share_weights_by_username=_parse_share_weights( + source.get("PRISM_STRATUM_SHARE_WEIGHTS_JSON", "") + ), + username_fallback_address=default_prism_username_fallback_address(environ=source), + fallback_version_mask=fallback_version_mask, + same_tip_job_retention_seconds=same_tip_seconds, + same_tip_job_retention_per_connection=same_tip_per_connection, + disconnected_job_retention=env_nonnegative_int( + "PRISM_STRATUM_DISCONNECTED_JOB_RETENTION", + DEFAULT_PRISM_DISCONNECTED_JOB_RETENTION, + environ=source, + ), + payout_address_cache_max_entries=env_nonnegative_int( + "PRISM_PAYOUT_ADDRESS_CACHE_MAX_ENTRIES", + DEFAULT_PRISM_PAYOUT_ADDRESS_CACHE_MAX_ENTRIES, + environ=source, + ), + payout_address_cache_ttl_seconds=env_nonnegative_float( + "PRISM_PAYOUT_ADDRESS_CACHE_TTL_SECONDS", + DEFAULT_PRISM_PAYOUT_ADDRESS_CACHE_TTL_SECONDS, + environ=source, + ), + vardiff_resume_enabled=env_bool( + "PRISM_STRATUM_VARDIFF_RESUME", "1", environ=source + ), + vardiff_resume_ttl_seconds=env_nonnegative_float( + "PRISM_STRATUM_VARDIFF_RESUME_TTL_SECONDS", + DEFAULT_PRISM_VARDIFF_RESUME_TTL_SECONDS, + environ=source, + ), + vardiff_resume_max_entries=env_nonnegative_int( + "PRISM_STRATUM_VARDIFF_RESUME_MAX_ENTRIES", + DEFAULT_PRISM_VARDIFF_RESUME_MAX_ENTRIES, + environ=source, + ), + vardiff_resume_max_start_factor=vardiff_resume_max_start_factor, + ) + + jobs = JobPipelineConfig( + blockpoll_seconds=blockpoll_seconds, + blockwait_enabled=env_bool("PRISM_BLOCKWAIT_ENABLED", "1", environ=source), + blockwait_timeout_seconds=env_positive_float( + "PRISM_BLOCKWAIT_TIMEOUT_SECONDS", + DEFAULT_PRISM_BLOCKWAIT_TIMEOUT_SECONDS, + environ=source, + ), + tip_refresh_failure_holdoff_seconds=env_nonnegative_float( + "PRISM_TIP_REFRESH_FAILURE_HOLDOFF_SECONDS", + DEFAULT_PRISM_TIP_REFRESH_FAILURE_HOLDOFF_SECONDS, + environ=source, + ), + submit_tip_max_age_seconds=env_nonnegative_float( + "PRISM_SUBMIT_TIP_MAX_AGE_SECONDS", + DEFAULT_PRISM_SUBMIT_TIP_MAX_AGE_SECONDS, + environ=source, + ), + tip_refresh_max_workers=tip_refresh_max_workers, + tip_refresh_epoch_fanout=env_bool( + "PRISM_TIP_REFRESH_EPOCH_FANOUT", "0", environ=source + ), + job_build_timeout_seconds=env_positive_float( + "PRISM_JOB_BUILD_TIMEOUT_SECONDS", DEFAULT_PRISM_JOB_BUILD_TIMEOUT_SECONDS, environ=source + ), + job_build_cancel_grace_seconds=env_nonnegative_float( + "PRISM_JOB_BUILD_CANCEL_GRACE_SECONDS", + DEFAULT_PRISM_JOB_BUILD_CANCEL_GRACE_SECONDS, + environ=source, + ), + job_build_executor_workers=job_build_executor_workers, + routine_admission_deadline_seconds=env_positive_float( + "PRISM_ROUTINE_ADMISSION_DEADLINE_SECONDS", + DEFAULT_PRISM_ROUTINE_ADMISSION_DEADLINE_SECONDS, + environ=source, + ), + worker_metrics_limit=env_nonnegative_int( + "PRISM_WORKER_METRICS_LIMIT", DEFAULT_PRISM_WORKER_METRICS_LIMIT, environ=source + ), + reorg_reconciler_enabled=env_bool("PRISM_REORG_RECONCILER_ENABLED", "1", environ=source), + job_bundle_cache_seconds=env_nonnegative_float( + "PRISM_JOB_BUNDLE_CACHE_SECONDS", + DEFAULT_PRISM_JOB_BUNDLE_CACHE_SECONDS, + environ=source, + ), + bundle_build_timeout_seconds=env_positive_float( + "PRISM_BUNDLE_BUILD_TIMEOUT_SECONDS", + DEFAULT_PRISM_BUNDLE_BUILD_TIMEOUT_SECONDS, + environ=source, + ), + payout_artifact_rearm_min_seconds=payout_artifact_rearm_min_seconds, + payout_artifact_reuse_enabled=env_bool( + "PRISM_PAYOUT_ARTIFACT_REUSE", "1", environ=source + ), + window_pipeline_rust_enabled=env_window_pipeline_rust(environ=source), + payout_artifact_reanchor_seconds=payout_artifact_reanchor_seconds, + payout_artifact_min_build_interval_seconds=( + payout_artifact_min_build_interval_seconds + ), + payout_artifact_full_rescan_seconds=env_positive_float( + "PRISM_PAYOUT_ARTIFACT_FULL_RESCAN_SECONDS", + DEFAULT_PRISM_PAYOUT_ARTIFACT_FULL_RESCAN_SECONDS, + environ=source, + ), + payout_artifact_max_anchor_age_seconds=payout_artifact_max_anchor_age_seconds, + template_cache_seconds=env_nonnegative_float( + "PRISM_TEMPLATE_CACHE_SECONDS", blockpoll_seconds, environ=source + ), + template_refresh_failure_exit_seconds=template_refresh_failure_exit_seconds, + reorg_reconcile_cache_seconds=env_nonnegative_float( + "PRISM_REORG_RECONCILE_CACHE_SECONDS", + DEFAULT_PRISM_REORG_RECONCILE_CACHE_SECONDS, + environ=source, + ), + min_ready_miners=env_int("PRISM_MIN_READY_MINERS", 3, environ=source), + payout_environment=_selected_environment( + source, + ( + "PRISM_PAYOUT_P2MR_SPEND_INPUT_BYTES", + "PRISM_PAYOUT_TARGET_FEERATE_BITS_PER_BYTE", + "PRISM_PAYOUT_TARGET_FEERATE_SATS_PER_BYTE", + "PRISM_PAYOUT_SAFETY_MULTIPLIER", + "PRISM_PAYOUT_MIN_OUTPUT_BITS", + "PRISM_PAYOUT_MIN_OUTPUT_SATS", + ), + ), + pool_fee_enabled_raw=source.get("PRISM_POOL_FEE_ENABLED"), + pool_fee_bps_raw=env_optional("PRISM_POOL_FEE_BPS", environ=source), + pool_fee_address=env_optional("PRISM_POOL_FEE_ADDRESS", environ=source), + pool_fee_program_hex=env_optional("PRISM_POOL_FEE_P2MR_PROGRAM_HEX", environ=source), + pool_fee_recipient_id=env_optional("PRISM_POOL_FEE_RECIPIENT_ID", environ=source), + pool_fee_order_key=env_optional("PRISM_POOL_FEE_ORDER_KEY", environ=source), + template_max_age_raw=source.get("PRISM_TEMPLATE_MAX_AGE_SECONDS"), + ) + + block = BlockConfig( + replay_page_size=env_positive_int( + "PRISM_BLOCK_REPLAY_PAGE_SIZE", MAX_BLOCK_REPLAY_PAGE_SIZE, environ=source + ), + submit_rpc_timeout_seconds=env_positive_float( + "PRISM_BLOCK_SUBMIT_RPC_TIMEOUT_SECONDS", + DEFAULT_BLOCK_SUBMIT_RPC_TIMEOUT_SECONDS, + environ=source, + ), + submit_db_timeout_seconds=env_positive_float( + "PRISM_BLOCK_SUBMIT_DB_TIMEOUT_SECONDS", + DEFAULT_BLOCK_SUBMIT_DB_TIMEOUT_SECONDS, + environ=source, + ), + landing_db_timeout_seconds=env_positive_float( + "PRISM_BLOCK_LANDING_DB_TIMEOUT_SECONDS", + DEFAULT_BLOCK_LANDING_DB_TIMEOUT_SECONDS, + environ=source, + ), + landing_db_timeout_max_seconds=env_positive_float( + "PRISM_BLOCK_LANDING_DB_TIMEOUT_MAX_SECONDS", + DEFAULT_BLOCK_LANDING_DB_TIMEOUT_MAX_SECONDS, + environ=source, + ), + submit_lock_wait_log_seconds=env_positive_float( + "PRISM_BLOCK_SUBMIT_LOCK_WAIT_LOG_SECONDS", + DEFAULT_BLOCK_SUBMIT_LOCK_WAIT_LOG_SECONDS, + environ=source, + ), + submit_stuck_call_exit_seconds=env_positive_float( + "PRISM_BLOCK_SUBMIT_STUCK_CALL_EXIT_SECONDS", + DEFAULT_BLOCK_SUBMIT_STUCK_CALL_EXIT_SECONDS, + environ=source, + ), + observed_tip_accept_window_seconds=env_positive_float( + "PRISM_OBSERVED_TIP_ACCEPT_WINDOW_SECONDS", + DEFAULT_PRISM_OBSERVED_TIP_ACCEPT_WINDOW_SECONDS, + environ=source, + ), + accepted_parent_unresolved_depth_max=env_positive_int( + "PRISM_ACCEPTED_PARENT_UNRESOLVED_DEPTH_MAX", + DEFAULT_ACCEPTED_PARENT_UNRESOLVED_DEPTH_MAX, + environ=source, + ), + accepted_parent_redrive_defer_threshold=env_positive_int( + "PRISM_ACCEPTED_PARENT_REDRIVE_DEFER_THRESHOLD", + DEFAULT_ACCEPTED_PARENT_REDRIVE_DEFER_THRESHOLD, + environ=source, + ), + accepted_parent_redrive_attempt_max=env_nonnegative_int( + "PRISM_ACCEPTED_PARENT_REDRIVE_ATTEMPT_MAX", + DEFAULT_ACCEPTED_PARENT_REDRIVE_ATTEMPT_MAX, + environ=source, + ), + candidate_cleanup_retry_backlog_max=( + validate_block_candidate_cleanup_retry_backlog_max( + env_positive_int( + "PRISM_BLOCK_CANDIDATE_CLEANUP_RETRY_BACKLOG_MAX", + DEFAULT_BLOCK_CANDIDATE_CLEANUP_RETRY_BACKLOG_MAX, + environ=source, + ) + ) + ), + ) + + if block.replay_page_size > MAX_BLOCK_REPLAY_PAGE_SIZE: + raise SystemExit( + "PRISM_BLOCK_REPLAY_PAGE_SIZE cannot exceed " + f"{MAX_BLOCK_REPLAY_PAGE_SIZE}" + ) + + evidence_path = Path(env("PRISM_EVIDENCE_PATH", "prism-live-evidence.json", environ=source)) + audit_dir = Path(env("PRISM_AUDIT_DIR", str(evidence_path.parent), environ=source)) + audit = AuditConfig( + evidence_path=evidence_path, + directory=audit_dir, + share_segment_size=env_nonnegative_int( + "PRISM_AUDIT_SHARE_SEGMENT_SIZE", DEFAULT_AUDIT_SHARE_SEGMENT_SIZE, environ=source + ), + live_bundle_retention=env_nonnegative_int( + "PRISM_AUDIT_LIVE_BUNDLE_RETENTION", 5, environ=source + ), + candidate_retention_seconds=env_nonnegative_int( + "PRISM_AUDIT_CANDIDATE_RETENTION_SECONDS", 24 * 60 * 60, environ=source + ), + bind=source.get("PRISM_AUDIT_BIND"), + port=int(source.get("PRISM_AUDIT_PORT", "0") or "0"), + ) + + configured_writer_public_key = env_optional( + "PRISM_LEDGER_WRITER_PUBLIC_KEY_HEX", environ=source + ) + if configured_writer_public_key is not None: + writer_public_key = validate_hex( + configured_writer_public_key, + name="PRISM_LEDGER_WRITER_PUBLIC_KEY_HEX", + expected_bytes=32, + ) + elif env_bool("PRISM_ALLOW_BUNDLE_EMBEDDED_LEDGER_KEY", "0", environ=source): + writer_public_key = None + else: + raise SystemExit( + "PRISM_LEDGER_WRITER_PUBLIC_KEY_HEX is required; " + "set PRISM_ALLOW_BUNDLE_EMBEDDED_LEDGER_KEY=1 only for local tests" + ) + psql_command = source.get("PRISM_POSTGRES_PSQL_COMMAND", "") + database_url = source.get("PRISM_DATABASE_URL", "") + if not psql_command and database_url: + psql_command = f"psql {shlex.quote(database_url)}" + writer_session_token = env_optional("PRISM_LEDGER_WRITER_SESSION_TOKEN", environ=source) + if writer_session_token is not None and not env_bool( + "PRISM_ALLOW_FIXED_LEDGER_SESSION_TOKEN", "0", environ=source + ): + raise SystemExit( + "PRISM_LEDGER_WRITER_SESSION_TOKEN requires " + "PRISM_ALLOW_FIXED_LEDGER_SESSION_TOKEN=1 for local tests" + ) + lease_acquire_lock_timeout_seconds = env_positive_float( + "PRISM_LEDGER_LEASE_ACQUIRE_LOCK_TIMEOUT_SECONDS", + DEFAULT_LEASE_ACQUIRE_LOCK_TIMEOUT_SECONDS, + environ=source, + ) + lease_acquire_attempts = env_positive_int( + "PRISM_LEDGER_LEASE_ACQUIRE_ATTEMPTS", + DEFAULT_LEASE_ACQUIRE_ATTEMPTS, + environ=source, + ) + postgres_idle_in_transaction_timeout_seconds = env_positive_float( + "PRISM_POSTGRES_IDLE_IN_TRANSACTION_TIMEOUT_SECONDS", + DEFAULT_POSTGRES_IDLE_IN_TRANSACTION_TIMEOUT_SECONDS, + environ=source, + ) + lease_acquire_budget_seconds = ( + lease_acquire_attempts * lease_acquire_lock_timeout_seconds + ) + if postgres_idle_in_transaction_timeout_seconds > lease_acquire_budget_seconds: + print( + "prism coordinator: " + "PRISM_POSTGRES_IDLE_IN_TRANSACTION_TIMEOUT_SECONDS " + f"({postgres_idle_in_transaction_timeout_seconds:g}s) exceeds the " + "writer-lease acquire budget " + f"PRISM_LEDGER_LEASE_ACQUIRE_ATTEMPTS ({lease_acquire_attempts}) x " + "PRISM_LEDGER_LEASE_ACQUIRE_LOCK_TIMEOUT_SECONDS " + f"({lease_acquire_lock_timeout_seconds:g}s) = " + f"{lease_acquire_budget_seconds:g}s: an orphaned " + "idle-in-transaction backend holding the lease row lock is not " + "reaped mid-budget, so startup exhausts every acquire attempt at " + "the same locked row instead of self-healing; raise the acquire " + "lock deadline or attempts, or lower the idle-in-transaction " + "timeout", + file=sys.stderr, + flush=True, + ) + ledger = LedgerConfig( + psql_command=psql_command, + database_url=database_url or None, + allow_memory_ledger=env_bool("PRISM_ALLOW_MEMORY_LEDGER", "0", environ=source), + native_client_mode=env("PRISM_POSTGRES_NATIVE_CLIENT", "auto", environ=source), + writer_id=env("PRISM_LEDGER_WRITER_ID", "prism-coordinator", environ=source), + writer_epoch=env_int("PRISM_LEDGER_WRITER_EPOCH", 1, environ=source), + writer_session_token=writer_session_token, + initialize_schema=env("PRISM_POSTGRES_INIT_SCHEMA", "0", environ=source) + in {"1", "true", "yes"}, + lease_ttl_seconds=env_positive_float( + "PRISM_LEDGER_LEASE_TTL_SECONDS", 60.0, environ=source + ), + lease_acquire_lock_timeout_seconds=lease_acquire_lock_timeout_seconds, + lease_acquire_attempts=lease_acquire_attempts, + postgres_idle_in_transaction_timeout_seconds=( + postgres_idle_in_transaction_timeout_seconds + ), + postgres_tcp_keepalives_idle_seconds=env_positive_int( + "PRISM_POSTGRES_TCP_KEEPALIVES_IDLE_SECONDS", + DEFAULT_POSTGRES_TCP_KEEPALIVES_IDLE_SECONDS, + environ=source, + ), + postgres_tcp_keepalives_interval_seconds=env_positive_int( + "PRISM_POSTGRES_TCP_KEEPALIVES_INTERVAL_SECONDS", + DEFAULT_POSTGRES_TCP_KEEPALIVES_INTERVAL_SECONDS, + environ=source, + ), + postgres_tcp_keepalives_count=env_positive_int( + "PRISM_POSTGRES_TCP_KEEPALIVES_COUNT", + DEFAULT_POSTGRES_TCP_KEEPALIVES_COUNT, + environ=source, + ), + read_concurrency=env_positive_int("PRISM_POSTGRES_READ_CONCURRENCY", 4, environ=source), + accepted_stats_cache_seconds=env_nonnegative_float( + "PRISM_ACCEPTED_STATS_CACHE_SECONDS", 60.0, environ=source + ), + reward_window_cache_seconds=env_nonnegative_float( + "PRISM_PUBLIC_REWARD_WINDOW_CACHE_SECONDS", 30.0, environ=source + ), + signing_seed_hex=env_seed_hex( + "PRISM_MANIFEST_SIGNING_SEED_HEX", test_default="42" * 32, environ=source + ), + attestation_signing_seed_hex=env_seed_hex( + "PRISM_LEDGER_ATTESTATION_SIGNING_SEED_HEX", + test_default="43" * 32, + environ=source, + ), + writer_public_key_hex=writer_public_key, + share_commit_batch_size=env_positive_int( + "PRISM_SHARE_COMMIT_BATCH_SIZE", DEFAULT_SHARE_COMMIT_BATCH_SIZE, environ=source + ), + share_commit_linger_seconds=env_nonnegative_float( + "PRISM_SHARE_COMMIT_LINGER_MILLISECONDS", + DEFAULT_SHARE_COMMIT_LINGER_MILLISECONDS, + environ=source, + ) + / 1000.0, + share_commit_timeout_seconds=env_positive_float( + "PRISM_SHARE_COMMIT_TIMEOUT_SECONDS", DEFAULT_SHARE_COMMIT_TIMEOUT_SECONDS, environ=source + ), + candidate_storage_version=env_positive_int( + "PRISM_CANDIDATE_STORAGE_VERSION", 2, environ=source + ), + candidate_spool_dir=env_optional("PRISM_CANDIDATE_SPOOL_DIR", environ=source) or None, + candidate_spool_reservation_bytes=env_positive_int( + "PRISM_CANDIDATE_SPOOL_RESERVATION_BYTES", + 4 * 1024 * 1024 * 1024, + environ=source, + ), + share_recovery_path=Path( + env( + "PRISM_SHARE_RECOVERY_PATH", + str(audit_dir / "prism-unpersisted-shares.jsonl"), + environ=source, + ) + ), + ) + + configured_broadcaster_enabled = env_optional_bool( + "PRISM_CTV_BROADCASTER_ENABLED", environ=source + ) + broadcaster_enabled = ( + configured_broadcaster_enabled + if configured_broadcaster_enabled is not None + else env_bool("PRISM_CTV_SETTLEMENT_ENABLED", "0", environ=source) + ) + broadcaster_wallet = env_optional("PRISM_CTV_BROADCASTER_WALLET", environ=source) + broadcaster_fee_sats = env_nonnegative_int_with_legacy( + "PRISM_CTV_BROADCASTER_FEE_BITS", + "PRISM_CTV_BROADCASTER_FEE_SATS", + 0, + environ=source, + ) + if broadcaster_enabled and broadcaster_fee_sats > 0 and not broadcaster_wallet: + raise SystemExit( + "PRISM_CTV_BROADCASTER_WALLET is required when " + "PRISM_CTV_BROADCASTER_FEE_BITS is positive" + ) + broadcaster_chunk_size = env_positive_int( + "PRISM_CTV_BROADCASTER_CHUNK_SIZE", + DEFAULT_PRISM_CTV_BROADCASTER_CHUNK_SIZE, + environ=source, + ) + if broadcaster_chunk_size > MAX_CTV_FANOUT_BROADCASTER_CHUNK_SIZE: + raise SystemExit( + "PRISM_CTV_BROADCASTER_CHUNK_SIZE must be at most " + f"{MAX_CTV_FANOUT_BROADCASTER_CHUNK_SIZE}" + ) + ctv = CtvConfig( + settlement_enabled_raw=source.get("PRISM_CTV_SETTLEMENT_ENABLED"), + broadcaster_enabled=broadcaster_enabled, + broadcaster_wallet=broadcaster_wallet, + broadcaster_fee_sats=broadcaster_fee_sats, + broadcaster_limit=env_positive_int("PRISM_CTV_BROADCASTER_LIMIT", 100, environ=source), + broadcaster_chunk_size=broadcaster_chunk_size, + broadcaster_interval_seconds=env_positive_float( + "PRISM_CTV_BROADCASTER_INTERVAL_SECONDS", 30.0, environ=source + ), + broadcast_attempt_detail_limit=env_nonnegative_int( + "PRISM_CTV_BROADCAST_ATTEMPT_DETAIL_LIMIT", + DEFAULT_CTV_BROADCAST_ATTEMPT_DETAIL_LIMIT, + environ=source, + ), + broadcast_retry_backoff_seconds=env_nonnegative_int( + "PRISM_CTV_BROADCAST_RETRY_BACKOFF_SECONDS", + DEFAULT_CTV_BROADCAST_RETRY_BACKOFF_SECONDS, + environ=source, + ), + settlement_environment=_selected_environment( + source, + ( + "PRISM_DIRECT_COINBASE_PAYOUT_FLOOR_BITS", + "PRISM_DIRECT_COINBASE_PAYOUT_FLOOR_SATS", + "PRISM_RESERVED_COINBASE_OUTPUTS", + "PRISM_MAX_COINBASE_SETTLEMENT_OUTPUTS", + "PRISM_MAX_DIRECT_COINBASE_OUTPUTS", + "PRISM_MAX_CTV_FANOUT_RECIPIENTS_PER_TRANSACTION", + "PRISM_CTV_FANOUT_FEE_MARKET_RATE_BITS_PER_1000_WEIGHT", + "PRISM_CTV_FANOUT_FEE_MARKET_RATE_SATS_PER_1000_WEIGHT", + "PRISM_CTV_FANOUT_FEE_ESTIMATE_TARGET_BLOCKS", + "PRISM_CTV_FANOUT_FEE_PREMIUM_BPS", + ), + ), + ) + + watchdog_timeout_seconds = env_positive_float( + "PRISM_WATCHDOG_TIMEOUT_SECONDS", + DEFAULT_PRISM_WATCHDOG_TIMEOUT_SECONDS, + environ=source, + ) + if watchdog_timeout_seconds < 1.0: + raise SystemExit( + "PRISM_WATCHDOG_TIMEOUT_SECONDS must be at least 1 so the " + "escalated landing-budget ceiling derived from half of it stays " + "below the tolerance" + ) + + lifecycle = LifecycleConfig( + health_refresh_seconds=env_positive_float( + "PRISM_HEALTH_REFRESH_SECONDS", DEFAULT_PRISM_HEALTH_REFRESH_SECONDS, environ=source + ), + metrics_refresh_seconds=env_positive_float( + "PRISM_METRICS_REFRESH_SECONDS", + DEFAULT_PRISM_METRICS_REFRESH_SECONDS, + environ=source, + ), + pending_refresh_health_deadline_seconds=env_positive_float( + "PRISM_HEALTH_PENDING_REFRESH_MAX_AGE_SECONDS", + DEFAULT_PRISM_HEALTH_PENDING_REFRESH_MAX_AGE_SECONDS, + environ=source, + ), + coherent_tip_poll_health_deadline_seconds=env_positive_float( + "PRISM_HEALTH_TIP_POLL_MAX_AGE_SECONDS", + DEFAULT_PRISM_HEALTH_TIP_POLL_MAX_AGE_SECONDS, + environ=source, + ), + mining_health_startup_grace_seconds=env_nonnegative_float( + "PRISM_MINING_HEALTH_STARTUP_GRACE_SECONDS", + DEFAULT_PRISM_MINING_HEALTH_STARTUP_GRACE_SECONDS, + environ=source, + ), + writer_quiescence_timeout_seconds=env_positive_float( + "PRISM_WRITER_QUIESCENCE_TIMEOUT_SECONDS", + DEFAULT_PRISM_WRITER_QUIESCENCE_TIMEOUT_SECONDS, + environ=source, + ), + coordination_blocked_exit_seconds=env_positive_float( + "PRISM_COORDINATION_BLOCKED_EXIT_SECONDS", + DEFAULT_PRISM_COORDINATION_BLOCKED_EXIT_SECONDS, + environ=source, + ), + watchdog_enabled=env_bool("PRISM_WATCHDOG_ENABLED", "1", environ=source), + watchdog_timeout_seconds=watchdog_timeout_seconds, + watchdog_interval_seconds=env_positive_float( + "PRISM_WATCHDOG_INTERVAL_SECONDS", 15.0, environ=source + ), + # Lease/watchdog timing envelopes are derived, not environment-facing: + # the values enter the immutable snapshot so services consume typed + # configuration instead of rereading process state at run time. + watchdog_lease_release_timeout_seconds=( + DEFAULT_PRISM_WATCHDOG_LEASE_RELEASE_TIMEOUT_SECONDS + ), + ledger_lease_heartbeat_seconds=DEFAULT_PRISM_LEDGER_LEASE_HEARTBEAT_SECONDS, + ledger_lease_heartbeat_failure_seconds=( + DEFAULT_PRISM_LEDGER_LEASE_HEARTBEAT_FAILURE_SECONDS + ), + ledger_lease_heartbeat_monitor_seconds=( + DEFAULT_PRISM_LEDGER_LEASE_HEARTBEAT_MONITOR_SECONDS + ), + ledger_lease_heartbeat_exit_timeout_seconds=( + DEFAULT_PRISM_LEDGER_LEASE_HEARTBEAT_EXIT_TIMEOUT_SECONDS + ), + ledger_lease_external_fence_timeout_seconds=( + DEFAULT_PRISM_LEDGER_LEASE_EXTERNAL_FENCE_TIMEOUT_SECONDS + ), + malloc_telemetry_enabled=env_malloc_telemetry(environ=source), + heap_census=env_heap_census_config(environ=source, audit_dir=audit_dir), + ) + max_blocks = env_int("PRISM_MAX_BLOCKS", 1, environ=source) + if max_blocks <= 0: + raise SystemExit("PRISM_MAX_BLOCKS must be positive") + return CoordinatorConfig( + rpc=rpc, + stratum=stratum, + jobs=jobs, + block=block, + ledger=ledger, + audit=audit, + ctv=ctv, + lifecycle=lifecycle, + production=production, + hot_path_log_enabled=env_bool("PRISM_HOT_PATH_LOG", "0", environ=source), + coinbase_tag_hex=default_prism_coinbase_tag_hex(environ=source), + stop_after_block=env("PRISM_STOP_AFTER_BLOCK", "1", environ=source) + in {"1", "true", "yes"}, + max_blocks=max_blocks, + ) diff --git a/lab/prism/coordinator_shutdown.py b/lab/prism/coordinator_shutdown.py new file mode 100644 index 000000000..0d858e948 --- /dev/null +++ b/lab/prism/coordinator_shutdown.py @@ -0,0 +1,300 @@ +"""Writer admission and shutdown state for the PRISM coordinator. + +The controller is the sole owner of the writer barrier, the one-shot lease +release claim/outcome, and the drain metrics. It owns no heartbeat I/O, +thread, or process-exit state: lease heartbeat threads belong to +``lab.prism.background_services.LedgerLeaseHeartbeatService`` and the +coordinator keeps the orchestration adapters (``request_shutdown``, +``shutdown``, ``release_ledger_lease``, ``drain_non_writer_components``, +``shutdown_metrics_lines``) around controller decisions. +""" + +from __future__ import annotations + +from contextlib import AbstractContextManager +from functools import wraps +import signal +import threading +import time +from typing import Any, Callable, Protocol + +# Owner-local copy of the block submitter's admission-wait heartbeat slice. +# The submitter machinery itself remains coordinator-owned at this layer, so +# the coordinator keeps its own module constant; this leaf module must not +# import the coordinator to share one. +BLOCK_SUBMITTER_WAIT_HEARTBEAT_SLICE_SECONDS = 0.25 + + +class ShutdownInProgress(RuntimeError): + """Raised when work that could mutate the ledger arrives after shutdown.""" + + +class _WriterOperationToken: + """One transferable writer admission held until durable work completes.""" + + def __init__(self, controller: "CoordinatorShutdownController", component: str): + self.controller = controller + self.component = component + self.finished = False + + def finish(self) -> None: + self.controller.finish_token(self) + + +class CoordinatorShutdownController: + """Coordinates the writer barrier, one-shot lease release, and final drain. + + Writer operations enter through :meth:`enter_writer` before shutdown or + inherit an already-admitted operation on the same thread. Queue admissions + use transferable tokens so a share remains visible to the barrier while it + moves from a client thread to the group-commit writer. + """ + + def __init__(self, writer_quiescence_timeout_seconds: float): + self.writer_quiescence_timeout_seconds = writer_quiescence_timeout_seconds + self.condition = threading.Condition(threading.RLock()) + self.local = threading.local() + self.phase = "running" + self.reason: str | None = None + self.signal_number: int | None = None + self.sigterm_monotonic: float | None = None + self.shutdown_started_monotonic: float | None = None + self.active_writers: dict[str, int] = {} + self.shutdowns_total = 0 + self.writer_quiescence_outcomes = {"success": 0, "timeout": 0} + self.writer_quiescence_seconds = 0.0 + self.lease_release_attempts_total = 0 + self.lease_release_outcomes = { + "success": 0, + "not_held": 0, + "unsupported": 0, + "failure": 0, + } + self.lease_release_seconds = 0.0 + self.lease_release_attempted = False + self.lease_release_succeeded = False + self.lease_release_withheld = False + self.sigterm_to_lease_release_seconds = 0.0 + self.sigterm_release_observed = False + self.release_withheld_total = 0 + self.non_writer_drain_seconds = 0.0 + self.non_writer_drains_total = 0 + self._drain_claimed = False + + def request_shutdown(self, signum: int | None) -> None: + """Close admission atomically; the caller only needs to set its event.""" + now = time.monotonic() + with self.condition: + if signum == signal.SIGTERM and self.sigterm_monotonic is None: + self.sigterm_monotonic = now + if self.signal_number is None and signum is not None: + self.signal_number = signum + if self.phase == "running": + self.phase = "requested" + self.condition.notify_all() + + def begin_shutdown(self, reason: str) -> bool: + with self.condition: + if self.phase not in {"running", "requested"}: + return False + self.phase = "quiescing_writers" + self.reason = reason + self.shutdown_started_monotonic = time.monotonic() + self.shutdowns_total += 1 + self.condition.notify_all() + return True + + def wait_for_lease_handling(self) -> bool: + """Wait for the one shutdown owner to release or safely withhold.""" + in_progress = { + "requested", + "quiescing_writers", + "writers_quiesced", + "releasing_lease", + } + with self.condition: + while self.phase in in_progress: + self.condition.wait() + return self.lease_release_succeeded + + def _thread_writer_depth(self) -> int: + return int(getattr(self.local, "writer_depth", 0)) + + def _admit_writer_locked(self, component: str, *, inherited: bool) -> _WriterOperationToken: + if self.lease_release_attempted: + raise ShutdownInProgress("PRISM writer lease release has already started") + if self.phase != "running" and not inherited: + raise ShutdownInProgress("PRISM coordinator is shutting down") + self.active_writers[component] = self.active_writers.get(component, 0) + 1 + return _WriterOperationToken(self, component) + + def enter_writer( + self, + component: str, + *, + wait_callback: Callable[[], None] | None = None, + ) -> _WriterOperationToken: + depth = self._thread_writer_depth() + if wait_callback is None: + with self.condition: + token = self._admit_writer_locked(component, inherited=depth > 0) + else: + while not self.condition.acquire( + timeout=BLOCK_SUBMITTER_WAIT_HEARTBEAT_SLICE_SECONDS + ): + wait_callback() + try: + token = self._admit_writer_locked(component, inherited=depth > 0) + finally: + self.condition.release() + self.local.writer_depth = depth + 1 + return token + + def exit_writer(self, token: _WriterOperationToken) -> None: + depth = self._thread_writer_depth() + self.local.writer_depth = max(0, depth - 1) + token.finish() + + def reserve_writer(self, component: str) -> _WriterOperationToken: + """Reserve work that will finish on another thread.""" + with self.condition: + return self._admit_writer_locked( + component, + inherited=self._thread_writer_depth() > 0, + ) + + def finish_token(self, token: _WriterOperationToken) -> None: + with self.condition: + if token.finished: + return + token.finished = True + remaining = self.active_writers.get(token.component, 0) - 1 + if remaining > 0: + self.active_writers[token.component] = remaining + else: + self.active_writers.pop(token.component, None) + self.condition.notify_all() + + def has_active_writer(self, components: set[str]) -> bool: + with self.condition: + return any(self.active_writers.get(component, 0) for component in components) + + def writer_admission_closed(self) -> bool: + with self.condition: + return self.phase != "running" + + def wait_for_writer_quiescence( + self, + timeout_seconds: float | None = None, + ) -> tuple[bool, float, dict[str, int]]: + started = time.monotonic() + timeout = ( + self.writer_quiescence_timeout_seconds + if timeout_seconds is None + else max(0.0, float(timeout_seconds)) + ) + deadline = started + timeout + with self.condition: + while self.active_writers: + remaining = deadline - time.monotonic() + if remaining <= 0: + break + self.condition.wait(remaining) + elapsed = max(0.0, time.monotonic() - started) + quiesced = not self.active_writers + blockers = dict(sorted(self.active_writers.items())) + outcome = "success" if quiesced else "timeout" + self.writer_quiescence_outcomes[outcome] += 1 + self.writer_quiescence_seconds = elapsed + if quiesced: + self.phase = "writers_quiesced" + else: + self.phase = "release_withheld" + self.lease_release_withheld = True + self.release_withheld_total += 1 + self.condition.notify_all() + return quiesced, elapsed, blockers + + def claim_lease_release(self) -> tuple[bool, dict[str, int]]: + with self.condition: + if self.lease_release_attempted or self.lease_release_withheld: + return False, {} + if self.active_writers: + return False, dict(sorted(self.active_writers.items())) + self.lease_release_attempted = True + self.lease_release_attempts_total += 1 + self.phase = "releasing_lease" + self.condition.notify_all() + return True, {} + + def finish_lease_release(self, outcome: str, elapsed: float) -> None: + with self.condition: + self.lease_release_outcomes[outcome] += 1 + self.lease_release_seconds = elapsed + self.lease_release_succeeded = outcome != "failure" + self.phase = "lease_released" if outcome != "failure" else "lease_release_failed" + if outcome != "failure" and self.sigterm_monotonic is not None: + self.sigterm_to_lease_release_seconds = max( + 0.0, + time.monotonic() - self.sigterm_monotonic, + ) + self.sigterm_release_observed = True + self.condition.notify_all() + + def claim_non_writer_drain(self) -> bool: + with self.condition: + if self._drain_claimed: + return False + if self.phase not in { + "lease_released", + "lease_release_failed", + "release_withheld", + }: + return False + self._drain_claimed = True + self.phase = "draining_non_writers" + return True + + def finish_non_writer_drain(self, elapsed: float) -> None: + with self.condition: + self.non_writer_drain_seconds = elapsed + self.non_writer_drains_total += 1 + self.phase = "complete" + self.condition.notify_all() + + def snapshot(self) -> dict[str, Any]: + with self.condition: + return { + "phase": self.phase, + "active_writers": dict(self.active_writers), + "shutdowns_total": self.shutdowns_total, + "writer_quiescence_outcomes": dict(self.writer_quiescence_outcomes), + "writer_quiescence_seconds": self.writer_quiescence_seconds, + "lease_release_attempts_total": self.lease_release_attempts_total, + "lease_release_outcomes": dict(self.lease_release_outcomes), + "lease_release_seconds": self.lease_release_seconds, + "lease_release_withheld": self.lease_release_withheld, + "sigterm_to_lease_release_seconds": self.sigterm_to_lease_release_seconds, + "sigterm_release_observed": self.sigterm_release_observed, + "release_withheld_total": self.release_withheld_total, + "non_writer_drain_seconds": self.non_writer_drain_seconds, + "non_writer_drains_total": self.non_writer_drains_total, + } + + +class _WriterOperationOwner(Protocol): + def _writer_operation(self, component: str) -> AbstractContextManager[object]: ... + + +def ledger_writer_operation(component: str) -> Callable[[Callable[..., Any]], Callable[..., Any]]: + """Decorate an entry point that can mutate the PRISM ledger.""" + + def decorate(method: Callable[..., Any]) -> Callable[..., Any]: + @wraps(method) + def guarded(self: _WriterOperationOwner, *args: Any, **kwargs: Any) -> Any: + with self._writer_operation(component): + return method(self, *args, **kwargs) + + return guarded + + return decorate diff --git a/lab/prism/ctv_runtime.py b/lab/prism/ctv_runtime.py new file mode 100644 index 000000000..17e5973ca --- /dev/null +++ b/lab/prism/ctv_runtime.py @@ -0,0 +1,411 @@ +"""Coordinator-facing lifecycle and metrics for the CTV broadcaster daemon. + +The CTV broadcast engine and durable daemon remain in their existing modules. +This service owns only the process runtime seam: daemon construction, writer +admission, the dedicated loop, and its bounded-cardinality metrics. +""" + +from __future__ import annotations + +from contextlib import AbstractContextManager +from dataclasses import dataclass, replace +import threading +import time +import traceback +from typing import Any, Callable, Protocol + +from lab.prism.coordinator_config import CtvConfig +from lab.prism.ctv_broadcaster import CtvFanoutBroadcaster +from lab.prism.ctv_broadcaster_daemon import ( + CtvFanoutBroadcastDaemon, + CtvFanoutChunkResult, + CtvFanoutDaemonResult, +) +from lab.prism.share_ledger import WriterLeaseRenewalDeferred + + +CTV_FANOUT_BROADCASTER_SERVICE_NAME = "ctv_fanout_broadcaster" +CTV_BROADCAST_STATE_COMPONENT = "ctv_broadcast_state" +PRISM_CTV_BROADCASTER_SECONDS_BUCKETS = ( + 1.0, + 5.0, + 10.0, + 30.0, + 60.0, + 120.0, + 300.0, + 600.0, +) +PRISM_CTV_BROADCASTER_CHUNK_SECONDS_BUCKETS = ( + 0.01, + 0.025, + 0.05, + 0.1, + 0.25, + 0.5, + 1.0, + 2.5, + 5.0, + 10.0, + 30.0, +) +PRISM_CTV_BROADCASTER_CHUNK_ROWS_BUCKETS = (1, 2, 5, 10, 25, 50, 100) + + +class StopSignal(Protocol): + def is_set(self) -> bool: ... + + def wait(self, timeout: float) -> bool: ... + + +@dataclass(frozen=True, slots=True) +class CtvRuntimeConfig: + enabled: bool + wallet: str | None + fee_sats: int + limit: int + chunk_size: int + interval_seconds: float + + @classmethod + def from_coordinator_config(cls, config: CtvConfig) -> "CtvRuntimeConfig": + return cls( + enabled=config.broadcaster_enabled, + wallet=config.broadcaster_wallet, + fee_sats=config.broadcaster_fee_sats, + limit=config.broadcaster_limit, + chunk_size=config.broadcaster_chunk_size, + interval_seconds=config.broadcaster_interval_seconds, + ) + + +@dataclass(frozen=True, slots=True) +class CtvRuntimeMetricsSnapshot: + pass_seconds_bucket_counts: dict[float, int] + pass_seconds_sum: float + pass_count: int + processed_rows_total: int + yielded_total: int + chunk_seconds_bucket_counts: dict[float, int] + chunk_rows_bucket_counts: dict[int, int] + chunk_seconds_sum: float + chunk_rows_sum: int + chunk_count: int + + +class CtvRuntimeService: + """Own the coordinator's CTV daemon instance, loop, and metric state.""" + + def __init__( + self, + *, + rpc_call: Callable[..., Any], + ledger_source: Callable[[], object], + writer_admission: Callable[[str], AbstractContextManager[object]], + tip_refresh_pending: Callable[[], bool], + heartbeat: Callable[[], None], + stop_signal: Callable[[], StopSignal], + config: CtvRuntimeConfig, + # Fenced external-effect verification for broadcast RPCs. The port + # resolves the coordinator seam at call time so lease-service tests + # keep their monkeypatch surface. + before_external_side_effect: Callable[..., Any] | None = None, + # The coordinator-owned shutdown-admission error type. Injected until + # the shutdown controller owns its own module; the loop treats it as a + # silent clean exit exactly like the historical inline loop. + shutdown_exception: type[BaseException] = RuntimeError, + daemon_type: Callable[..., CtvFanoutBroadcastDaemon] = CtvFanoutBroadcastDaemon, + broadcaster_type: Callable[..., CtvFanoutBroadcaster] = CtvFanoutBroadcaster, + monotonic: Callable[[], float] = time.monotonic, + print_exception: Callable[[], None] = traceback.print_exc, + ) -> None: + self._rpc_call = rpc_call + self._ledger_source = ledger_source + self._writer_admission = writer_admission + self._tip_refresh_pending = tip_refresh_pending + self._heartbeat = heartbeat + self._stop_signal = stop_signal + self._before_external_side_effect = before_external_side_effect + self._shutdown_exception = shutdown_exception + self._daemon_type = daemon_type + self._broadcaster_type = broadcaster_type + self._monotonic = monotonic + self._print_exception = print_exception + self._config_lock = threading.Lock() + self._config = config + + self._daemon_lock = threading.Lock() + self._daemon: CtvFanoutBroadcastDaemon | None = None + self._metrics_lock = threading.Lock() + self._pass_seconds_bucket_counts = { + bucket: 0 for bucket in PRISM_CTV_BROADCASTER_SECONDS_BUCKETS + } + self._pass_seconds_sum = 0.0 + self._pass_count = 0 + self._processed_rows_total = 0 + self._yielded_total = 0 + self._chunk_seconds_bucket_counts = { + bucket: 0 for bucket in PRISM_CTV_BROADCASTER_CHUNK_SECONDS_BUCKETS + } + self._chunk_rows_bucket_counts = { + bucket: 0 for bucket in PRISM_CTV_BROADCASTER_CHUNK_ROWS_BUCKETS + } + self._chunk_seconds_sum = 0.0 + self._chunk_rows_sum = 0 + self._chunk_count = 0 + + @property + def daemon(self) -> CtvFanoutBroadcastDaemon | None: + with self._daemon_lock: + return self._daemon + + @daemon.setter + def daemon(self, daemon: CtvFanoutBroadcastDaemon | None) -> None: + with self._daemon_lock: + self._daemon = daemon + + @property + def config(self) -> CtvRuntimeConfig: + """Return one immutable configuration snapshot under service ownership.""" + with self._config_lock: + return self._config + + def replace_config(self, **changes: object) -> None: + """Support temporary coordinator compatibility properties.""" + with self._daemon_lock: + with self._config_lock: + previous = self._config + replacement = replace(previous, **changes) + self._config = replacement + if ( + previous.wallet != replacement.wallet + or previous.fee_sats != replacement.fee_sats + ): + self._daemon = None + + def make_daemon( + self, + config: CtvRuntimeConfig | None = None, + ) -> CtvFanoutBroadcastDaemon: + config = self.config if config is None else config + if config.fee_sats > 0 and not config.wallet: + raise ValueError( + "ctv_broadcaster_wallet is required when ctv_broadcaster_fee_sats is positive" + ) + broadcaster_kwargs: dict[str, Any] = {"funding_wallet": config.wallet} + if self._before_external_side_effect is not None: + broadcaster_kwargs["before_external_side_effect"] = ( + self._before_external_side_effect + ) + broadcaster = self._broadcaster_type( + self._rpc_call, + **broadcaster_kwargs, + ) + return self._daemon_type( + self._ledger_source(), + broadcaster, + fee_sats=config.fee_sats, + ) + + def _configured_daemon( + self, + ) -> tuple[CtvRuntimeConfig, CtvFanoutBroadcastDaemon]: + with self._daemon_lock: + with self._config_lock: + config = self._config + if self._daemon is None: + self._daemon = self.make_daemon(config) + return config, self._daemon + + def run_once( + self, + *, + progress_callback: Callable[[], None] | None = None, + chunk_callback: Callable[[CtvFanoutChunkResult], None] | None = None, + ) -> CtvFanoutDaemonResult: + with self._writer_admission(CTV_BROADCAST_STATE_COMPONENT): + config, daemon = self._configured_daemon() + return daemon.run_once( + limit=config.limit, + progress_callback=progress_callback, + chunk_size=config.chunk_size, + tip_refresh_pending=self._tip_refresh_pending, + chunk_callback=( + self.observe_chunk if chunk_callback is None else chunk_callback + ), + ) + + def record_progress(self) -> None: + self._heartbeat() + with self._metrics_lock: + self._processed_rows_total += 1 + + def observe_pass(self, elapsed_seconds: float) -> None: + with self._metrics_lock: + self._pass_count += 1 + self._pass_seconds_sum += elapsed_seconds + for bucket in PRISM_CTV_BROADCASTER_SECONDS_BUCKETS: + if elapsed_seconds <= bucket: + self._pass_seconds_bucket_counts[bucket] += 1 + + def observe_chunk(self, result: CtvFanoutChunkResult) -> None: + self._heartbeat() + with self._metrics_lock: + self._chunk_count += 1 + self._chunk_seconds_sum += result.elapsed_seconds + self._chunk_rows_sum += result.processed_count + for bucket in PRISM_CTV_BROADCASTER_CHUNK_SECONDS_BUCKETS: + if result.elapsed_seconds <= bucket: + self._chunk_seconds_bucket_counts[bucket] += 1 + for bucket in PRISM_CTV_BROADCASTER_CHUNK_ROWS_BUCKETS: + if result.processed_count <= bucket: + self._chunk_rows_bucket_counts[bucket] += 1 + + def record_yield(self) -> None: + with self._metrics_lock: + self._yielded_total += 1 + + def loop( + self, + *, + run_once: Callable[..., CtvFanoutDaemonResult] | None = None, + progress_callback: Callable[[], None] | None = None, + observe_pass: Callable[[float], None] | None = None, + record_yield: Callable[[], None] | None = None, + ) -> None: + """Run the dedicated broadcaster loop until shutdown. + + The optional call ports retain the coordinator's temporary method-level + compatibility seam; normal registered service threads use the owned + methods directly. + """ + pass_runner = self.run_once if run_once is None else run_once + progress_recorder = ( + self.record_progress if progress_callback is None else progress_callback + ) + pass_observer = self.observe_pass if observe_pass is None else observe_pass + yield_recorder = self.record_yield if record_yield is None else record_yield + while not self._stop_signal().is_set(): + self._heartbeat() + started = self._monotonic() + shutdown_admission_closed = False + try: + try: + result = pass_runner(progress_callback=progress_recorder) + except self._shutdown_exception: + shutdown_admission_closed = True + return + finally: + # Stamp completion before logging or entering the interval + # wait. A blocked row never reaches this finally clause, so + # the watchdog remains able to recover a wedged operation. + self._heartbeat() + if not shutdown_admission_closed: + pass_observer(max(0.0, self._monotonic() - started)) + except WriterLeaseRenewalDeferred as exc: + # A preflight refusal, not a failure: the writer's own + # fenced write is holding the lease renewal, and the fence + # withheld the guarded RPC before it ran. The next pass on + # the ordinary interval retries once the write commits and + # a renewal lands. + print( + "prism coordinator: CTV fanout broadcaster pass " + f"withheld: {exc}", + flush=True, + ) + except Exception: + print("prism coordinator: CTV fanout broadcaster pass failed", flush=True) + self._print_exception() + else: + if result.yielded_to_tip_refresh: + yield_recorder() + if result.scanned_count or result.submitted_count or result.failed_count: + print( + "prism coordinator: CTV fanout broadcaster " + f"scanned={result.scanned_count} " + f"submitted={result.submitted_count} " + f"updated={result.updated_count} " + f"failed={result.failed_count}", + flush=True, + ) + if self._stop_signal().wait(self.config.interval_seconds): + break + + def startup_summary(self) -> str: + config = self.config + return ( + "prism coordinator: CTV fanout broadcaster enabled " + f"mode={'cpfp' if config.fee_sats > 0 else 'direct'} " + f"fee_bits={config.fee_sats} " + f"wallet={'configured' if config.wallet else 'none'} " + f"interval={config.interval_seconds:g}s " + f"limit={config.limit} " + f"chunk_size={config.chunk_size}" + ) + + def metrics_snapshot(self) -> CtvRuntimeMetricsSnapshot: + with self._metrics_lock: + return CtvRuntimeMetricsSnapshot( + pass_seconds_bucket_counts=dict(self._pass_seconds_bucket_counts), + pass_seconds_sum=self._pass_seconds_sum, + pass_count=self._pass_count, + processed_rows_total=self._processed_rows_total, + yielded_total=self._yielded_total, + chunk_seconds_bucket_counts=dict(self._chunk_seconds_bucket_counts), + chunk_rows_bucket_counts=dict(self._chunk_rows_bucket_counts), + chunk_seconds_sum=self._chunk_seconds_sum, + chunk_rows_sum=self._chunk_rows_sum, + chunk_count=self._chunk_count, + ) + + @property + def processed_rows_total(self) -> int: + return self.metrics_snapshot().processed_rows_total + + @property + def pass_count(self) -> int: + return self.metrics_snapshot().pass_count + + def metrics_lines(self) -> list[str]: + snapshot = self.metrics_snapshot() + metric_name = "qbit_prism_ctv_fanout_broadcaster_pass_seconds" + chunk_seconds_name = "qbit_prism_ctv_fanout_broadcaster_chunk_seconds" + chunk_rows_name = "qbit_prism_ctv_fanout_broadcaster_chunk_rows" + return [ + "# HELP qbit_prism_ctv_fanout_broadcaster_processed_rows_total CTV fanout rows completed by the broadcaster loop.", + "# TYPE qbit_prism_ctv_fanout_broadcaster_processed_rows_total counter", + f"qbit_prism_ctv_fanout_broadcaster_processed_rows_total {snapshot.processed_rows_total}", + "# HELP qbit_prism_ctv_fanout_broadcaster_pass_seconds CTV fanout broadcaster pass wall time.", + "# TYPE qbit_prism_ctv_fanout_broadcaster_pass_seconds histogram", + *[ + f'{metric_name}_bucket{{le="{bucket:g}"}} ' + f"{snapshot.pass_seconds_bucket_counts.get(bucket, 0)}" + for bucket in PRISM_CTV_BROADCASTER_SECONDS_BUCKETS + ], + f'{metric_name}_bucket{{le="+Inf"}} {snapshot.pass_count}', + f"{metric_name}_sum {snapshot.pass_seconds_sum:.6f}", + f"{metric_name}_count {snapshot.pass_count}", + "# HELP qbit_prism_ctv_fanout_broadcaster_tip_refresh_yields_total CTV broadcaster passes yielding between committed chunks for a pending tip refresh.", + "# TYPE qbit_prism_ctv_fanout_broadcaster_tip_refresh_yields_total counter", + f"qbit_prism_ctv_fanout_broadcaster_tip_refresh_yields_total {snapshot.yielded_total}", + "# HELP qbit_prism_ctv_fanout_broadcaster_chunk_seconds CTV broadcaster committed chunk wall time.", + "# TYPE qbit_prism_ctv_fanout_broadcaster_chunk_seconds histogram", + *[ + f'{chunk_seconds_name}_bucket{{le="{bucket:g}"}} ' + f"{snapshot.chunk_seconds_bucket_counts.get(bucket, 0)}" + for bucket in PRISM_CTV_BROADCASTER_CHUNK_SECONDS_BUCKETS + ], + f'{chunk_seconds_name}_bucket{{le="+Inf"}} {snapshot.chunk_count}', + f"{chunk_seconds_name}_sum {snapshot.chunk_seconds_sum:.6f}", + f"{chunk_seconds_name}_count {snapshot.chunk_count}", + "# HELP qbit_prism_ctv_fanout_broadcaster_chunk_rows Rows processed per committed CTV broadcaster chunk.", + "# TYPE qbit_prism_ctv_fanout_broadcaster_chunk_rows histogram", + *[ + f'{chunk_rows_name}_bucket{{le="{bucket}"}} ' + f"{snapshot.chunk_rows_bucket_counts.get(bucket, 0)}" + for bucket in PRISM_CTV_BROADCASTER_CHUNK_ROWS_BUCKETS + ], + f'{chunk_rows_name}_bucket{{le="+Inf"}} {snapshot.chunk_count}', + f"{chunk_rows_name}_sum {snapshot.chunk_rows_sum}", + f"{chunk_rows_name}_count {snapshot.chunk_count}", + ] diff --git a/lab/prism/direct_stratum.py b/lab/prism/direct_stratum.py index a2735aba3..b882ff9c8 100644 --- a/lab/prism/direct_stratum.py +++ b/lab/prism/direct_stratum.py @@ -493,15 +493,30 @@ def assemble_submission( if strip_witness_transaction(full_coinbase_hex) != coinbase_without_witness: raise ValueError("full coinbase witness transaction does not match Stratum txid preimage") header_hash_int = stratum_codec.header_hash_int(header) + block_pass = header_hash_int <= job.qbit_target return DirectQbitSubmission( coinbase_tx_hex=full_coinbase_hex, coinbase_txid_preimage_hex=coinbase_without_witness.hex(), header_hex=header.hex(), - block_hex=serialize_block(header, full_coinbase_hex, job.transaction_hexes).hex(), + # Serializing the block costs O(block bytes) of GIL-held decode, + # concatenate and re-encode, and only the block-landing path ever + # reads the result, so only a hash that actually solved the block + # pays for it. ``block_pass`` is the exact precondition rather than a + # heuristic: share_submission.classify_submit_work makes it necessary + # for every route that builds a PrismBlockCandidate, so the bytes are + # always materialized here -- eagerly, on the client thread, inside + # the call that already validates the coinbase -- before a candidate + # exists and long before its durable intent is written. An ordinary + # share can never need them. + block_hex=( + serialize_block(header, full_coinbase_hex, job.transaction_hexes).hex() + if block_pass + else "" + ), block_hash_hex=stratum_codec.header_hash_hex(header), block_hash_int=header_hash_int, share_pass=header_hash_int <= job.share_target, - block_pass=header_hash_int <= job.qbit_target, + block_pass=block_pass, applied_version_hex=version_hex, ) diff --git a/lab/prism/endpoint_registry.py b/lab/prism/endpoint_registry.py new file mode 100644 index 000000000..9f4543d79 --- /dev/null +++ b/lab/prism/endpoint_registry.py @@ -0,0 +1,805 @@ +"""Declarative classification of every endpoint the PRISM HTTP surface serves. + +The coordinator process answers three unrelated kinds of request on one +listener (``lab/prism/audit_http.py``): a public read surface anyone on the +internet may poll, an operator surface describing this process instance, and a +set of payout-operations reads that contend with the writer lease the +block-landing path holds. Issue #145 extracts the first of those into its own +process; this module is the boundary made explicit so the split can be reviewed +one route at a time before anything moves. + +Nothing here changes request handling. ``AuditHttpFacade`` still owns dispatch, +and this registry is asserted against it in +``tests/test_prism_endpoint_registry.py`` so a new route cannot be added without +being classified. + +The boundary is not invented here. ``PRISM.md`` already states it: the +coordinator "exposes a private audit/ops listener and a dashboard-safe public +API from the same process", lists ``/audit/*``, ``/healthz``, ``/metrics`` and +``/owed-balances`` as private/internal, and concludes that "operators can expose +only ``/public/v1`` through a reverse proxy or dashboard frontend". +``docs/public-dashboard-api/README.md`` repeats it as a contract -- "the public +API must not expose ``/audit/*``, ``/metrics``, ``/healthz``, operator controls +..." -- and ``tests/test_public_dashboard_api_contract.py`` enforces it by +asserting the OpenAPI document contains no ``/audit/`` path. + +So published audit evidence reaches the public through the content-addressed +``/public/v1/artifacts/{sha256}`` route and the artifact links that reference +it, not through ``/audit/*``. The evidence content is public; the ``/audit/*`` +routes are internal tooling over the same data, with raw shapes +(``fanout_tx_hex``, ``manifest_set_json``, full broadcast-attempt histories), +raw ``sats`` units, no pagination, and in one case a caller-controlled unbounded +window. + +The three axes a classification answers: + +``Audience`` + ``PUBLIC_READ`` is the documented, versioned dashboard contract. + ``OPERATOR`` describes a process instance or the private audit/ops tooling. + ``PAYOUT_AFFECTING`` reports live payout obligations -- the state the + settlement machinery itself acts on. + +``LedgerAccess`` + How the response reaches durable state. ``WRITER_LOCK`` reads take + ``share_ledger._operation_gate(self._lock, "writer lock")`` and therefore + serialize against the lease-holding writer -- public volume on one of these + is share-ingest backpressure. ``READ_SLOT`` reads take the bounded read + semaphore (``PRISM_POSTGRES_READ_CONCURRENCY``) and are replica-eligible. + ``ARTIFACT_FILE`` reads come from the published audit artifacts, which are + already the durable public record. ``PROCESS_STATE`` reads in-memory + coordinator state and cannot leave the process at all. + +``Disposition`` + Where the route lands after issue #145. + +``max_staleness_seconds`` + How old an answer an extracted route may serve before it must refuse. + +Staleness is a contract, not a nicety: a silently stale dashboard is worse than +an honest one. ``lab/prism/observability.py`` set the precedent for this +process -- the cached ``/metrics`` endpoint returns 503 rather than serve an +unbounded-stale snapshot, using ``stale_after = max(3 * refresh_seconds, +MINIMUM_HEALTH_STALE_SECONDS)`` with a 15-second floor, and always publishes +the observed age. The extracted public tier follows the same discipline with +budgets derived from the caches that actually sit under each route rather than +hand-picked per endpoint: + + budget = max(3 * (cache_ttl_seconds + underlying_cache_seconds), + MINIMUM_PUBLIC_STALE_SECONDS) + +``cache_ttl_seconds`` is the route's own shared-response TTL from +``public_api.public_cache_policy`` at its documented default: 5s for the plain +row reads, 30s for the pool-wide aggregates (``/public/v1/pool-summary``, +``/public/v1/hashrate-series``, ``/public/v1/miners/{recipient_id}/workers``), +and 300s for ``/public/v1/mining-configuration``. + +``underlying_cache_seconds`` is any second cache stacked beneath that one. Only +one exists today: the ledger's pool reward-window aggregate +(``share_ledger._pool_reward_window_aggregate``, +``PRISM_PUBLIC_REWARD_WINDOW_CACHE_SECONDS``, default 30s), reached from +``dashboard_miner_reward_window`` and therefore counted only for the miner page +``/public/v1/miners/{recipient_id}``. Two caches in series compound, so the +budget must cover their sum, not the larger of the two. + +The factor of 3 and the 15-second floor are the precedent's, deliberately: a +budget of exactly one TTL would refuse a response that is merely due for +refresh, and a sub-15s budget on a 5s TTL would make ordinary scheduling jitter +look like an outage. + +``/public/v1/artifacts/{sha256}`` is the one route with no budget at all, and +says so through ``immutable_content`` rather than by omission. Its content is +immutable and content-addressed: a body that hashes to the requested sha256 is +correct no matter how old it is, so refusing it for age would be refusing a +correct answer. The only observable staleness is the 404 -> 200 transition for +an artifact published moments ago, which no budget would fix. + +Replica lag is the third staleness source named in +``docs/prism-public-operator-endpoint-split.md``. No replica exists yet, so it +contributes 0 here; when one lands it becomes another +``underlying_cache_seconds`` term rather than a new mechanism. +""" + +from __future__ import annotations + +from dataclasses import dataclass +from enum import Enum + + +class Audience(Enum): + PUBLIC_READ = "public_read" + OPERATOR = "operator" + PAYOUT_AFFECTING = "payout_affecting" + + +class LedgerAccess(Enum): + WRITER_LOCK = "writer_lock" + READ_SLOT = "read_slot" + ARTIFACT_FILE = "artifact_file" + PROCESS_STATE = "process_state" + QBIT_RPC = "qbit_rpc" + NONE = "none" + + +class Disposition(Enum): + EXTRACT = "extract" + RETAIN = "retain" + + +# The staleness floor, matching observability.MINIMUM_HEALTH_STALE_SECONDS. +MINIMUM_PUBLIC_STALE_SECONDS = 15 + +# Documented defaults of the caches a public response can sit behind. These are +# the defaults the budgets below are derived from; an operator who raises a TTL +# past its budget is told so by tests/test_prism_endpoint_registry.py. +PUBLIC_CACHE_TTL_DEFAULT_SECONDS = 5 +PUBLIC_AGGREGATE_CACHE_TTL_DEFAULT_SECONDS = 30 +PUBLIC_CONFIG_CACHE_TTL_DEFAULT_SECONDS = 300 +POOL_REWARD_WINDOW_CACHE_DEFAULT_SECONDS = 30 + + +def staleness_budget_seconds( + *, + cache_ttl_seconds: int, + underlying_cache_seconds: int = 0, +) -> int: + """Derive one route's staleness budget from the caches beneath it. + + See the module docstring: three refresh intervals of every cache in the + series, floored at MINIMUM_PUBLIC_STALE_SECONDS. + """ + + if cache_ttl_seconds < 0 or underlying_cache_seconds < 0: + raise ValueError("cache seconds must be nonnegative") + return max( + 3 * (cache_ttl_seconds + underlying_cache_seconds), + MINIMUM_PUBLIC_STALE_SECONDS, + ) + + +@dataclass(frozen=True) +class Endpoint: + """One classified route. + + ``paths`` lists every request path that reaches the same handler, so an + alias cannot be silently dropped by the extraction. + """ + + paths: tuple[str, ...] + audience: Audience + disposition: Disposition + access: tuple[LedgerAccess, ...] + rationale: str + # Ledger methods reached, as evidence for the access classification. + ledger_methods: tuple[str, ...] = () + # Set when a route's audience and its current implementation disagree -- + # the work the extraction must do before the route can move. + extraction_blocker: str | None = None + # How old a response this route may serve before it must refuse with 503. + # Derived through staleness_budget_seconds(), never hand-picked. None means + # no budget, which is only legitimate together with immutable_content. + max_staleness_seconds: float | None = None + # Content-addressed, immutable bodies: correct at any age, so this route + # never refuses for staleness. Stated rather than left implicit. + immutable_content: bool = False + + def __post_init__(self) -> None: + if not self.paths: + raise ValueError("endpoint must list at least one path") + for path in self.paths: + if not path.startswith("/"): + raise ValueError(f"endpoint path must be absolute: {path!r}") + if not self.access: + raise ValueError(f"endpoint {self.paths[0]} must declare ledger access") + if not self.rationale: + raise ValueError(f"endpoint {self.paths[0]} must carry a rationale") + if (self.audience is Audience.PUBLIC_READ) != ( + self.disposition is Disposition.EXTRACT + ): + raise ValueError( + f"endpoint {self.paths[0]}: exactly the public read surface is " + "extracted, so audience and disposition must agree" + ) + if ( + LedgerAccess.WRITER_LOCK in self.access + and self.disposition is Disposition.EXTRACT + and self.extraction_blocker is None + ): + raise ValueError( + f"endpoint {self.paths[0]} takes the writer lock and cannot be " + "extracted without recording the blocker that must be removed" + ) + if self.immutable_content and self.disposition is not Disposition.EXTRACT: + raise ValueError( + f"endpoint {self.paths[0]}: immutable_content describes the " + "extracted public contract only" + ) + if self.immutable_content and self.max_staleness_seconds is not None: + raise ValueError( + f"endpoint {self.paths[0]}: immutable content is correct at any " + "age, so it must not also carry a staleness budget" + ) + if self.disposition is Disposition.EXTRACT: + if self.max_staleness_seconds is None and not self.immutable_content: + raise ValueError( + f"endpoint {self.paths[0]}: every extracted route must state " + "the staleness it tolerates, or declare immutable_content" + ) + if ( + self.max_staleness_seconds is not None + and self.max_staleness_seconds < MINIMUM_PUBLIC_STALE_SECONDS + ): + raise ValueError( + f"endpoint {self.paths[0]}: staleness budget must not fall " + f"below {MINIMUM_PUBLIC_STALE_SECONDS}s" + ) + elif self.max_staleness_seconds is not None: + raise ValueError( + f"endpoint {self.paths[0]}: the staleness contract covers the " + "extracted public surface; retained routes answer for " + "themselves" + ) + + @property + def primary_path(self) -> str: + return self.paths[0] + + +# --- Operator surface ------------------------------------------------------- +# +# Process state, plus the private audit/ops listener. PRISM.md lists these as +# private/internal and instructs operators not to expose them to the internet. + +_OPERATOR: tuple[Endpoint, ...] = ( + Endpoint( + paths=("/healthz",), + audience=Audience.OPERATOR, + disposition=Disposition.RETAIN, + access=(LedgerAccess.PROCESS_STATE,), + rationale=( + "Liveness of this coordinator instance: job-build progress, tip " + "refresh, writer-lease state. Answered from an in-memory snapshot " + "refreshed by a background thread, and overlaid with live progress " + "state on every request so a cached ok=true cannot mask a failed " + "refresh. A different process cannot answer it. Wired to the " + "compose healthcheck." + ), + ), + Endpoint( + paths=("/metrics",), + audience=Audience.OPERATOR, + disposition=Disposition.RETAIN, + access=(LedgerAccess.PROCESS_STATE,), + rationale=( + "Prometheus exposition of this process's counters and gauges. " + "Serves only the complete cached snapshot and returns 503 once the " + "snapshot exceeds max(3 * refresh, 15s) -- the staleness precedent " + "the extracted service follows." + ), + ), + Endpoint( + paths=("/audit/latest",), + audience=Audience.OPERATOR, + disposition=Disposition.RETAIN, + access=(LedgerAccess.ARTIFACT_FILE,), + rationale=( + "The most recently published evidence envelope, read from the " + "audit artifact directory. Public consumers reach the same " + "evidence content-addressed via /public/v1/artifacts/{sha256}; " + "this route is the operator's view of the live envelope and is " + "listed as private/internal in PRISM.md." + ), + ), + Endpoint( + paths=("/audit/share-window",), + audience=Audience.OPERATOR, + disposition=Disposition.RETAIN, + access=(LedgerAccess.WRITER_LOCK,), + ledger_methods=("audit_share_window",), + rationale=( + "Raw per-share rows for a caller-supplied anchor. The window is " + "caller-controlled and unbounded, and the read is fenced behind " + "the writer lock, so this is an operator reconciliation tool -- " + "the public equivalent is the aggregated reward leaderboard." + ), + ), + Endpoint( + paths=("/audit/blocks/{block_hash}/payouts", "/audit/block/{block_hash}"), + audience=Audience.OPERATOR, + disposition=Disposition.RETAIN, + access=(LedgerAccess.WRITER_LOCK,), + ledger_methods=("audit_block_payouts",), + rationale=( + "Per-recipient payout rows for a settled block in raw sats, with " + "maturity_state and chain_state. The public per-miner slice is " + "/public/v1/miners/{recipient_id}/payouts. The short " + "/audit/block/{block_hash} form is a legacy alias for the same " + "handler." + ), + ), + Endpoint( + paths=("/audit/blocks/{block_hash}/bundle",), + audience=Audience.OPERATOR, + disposition=Disposition.RETAIN, + access=(LedgerAccess.WRITER_LOCK, LedgerAccess.ARTIFACT_FILE), + ledger_methods=("audit_bundle",), + rationale=( + "The full inline audit bundle plus coinbase_tx_hex for one block. " + "The bundle body itself is public, content-addressed at " + "/public/v1/artifacts/{sha256} and linked from the public " + "settlement-artifacts route; this is the internal by-block " + "lookup." + ), + ), + Endpoint( + paths=("/audit/commitments/{commitment_leaf_hex}/bundle",), + audience=Audience.OPERATOR, + disposition=Disposition.RETAIN, + access=(LedgerAccess.WRITER_LOCK, LedgerAccess.ARTIFACT_FILE), + ledger_methods=("audit_bundle_by_commitment",), + rationale=( + "The same bundle addressed by audit commitment leaf. No public " + "commitment-leaf lookup exists; a verifier holding a leaf resolves " + "the bundle sha256 from the block's public settlement artifacts " + "and fetches it content-addressed." + ), + ), + Endpoint( + paths=("/audit/blocks/{block_hash}/ctv-fanout-manifest-set",), + audience=Audience.OPERATOR, + disposition=Disposition.RETAIN, + access=(LedgerAccess.READ_SLOT,), + ledger_methods=("audit_ctv_fanout_manifest_set",), + rationale=( + "The full CTV recovery payload, including parent_coinbase_tx_hex " + "and manifest_set_json -- broadcast material, not a dashboard " + "read. /public/v1/blocks/{block_hash}/settlement-artifacts is the " + "public reshaping of the same ledger read." + ), + ), + Endpoint( + paths=("/audit/blocks/{block_hash}/ctv-fanouts",), + audience=Audience.OPERATOR, + disposition=Disposition.RETAIN, + access=(LedgerAccess.READ_SLOT,), + ledger_methods=("audit_ctv_fanouts", "audit_ctv_fanout_manifest_set"), + rationale=( + "Raw artifact dicts with fanout_tx_hex and sats fields, delegating " + "to the manifest-set read above." + ), + ), + Endpoint( + paths=("/audit/fanouts/pending",), + audience=Audience.OPERATOR, + disposition=Disposition.RETAIN, + access=(LedgerAccess.READ_SLOT,), + ledger_methods=("pending_ctv_fanout_statuses",), + rationale=( + "The broadcaster's work queue, filtered to fanouts due now, with " + "full broadcast-attempt histories. /public/v1/fanouts/pending is " + "the paginated public view built from a different read model." + ), + ), + Endpoint( + paths=("/audit/fanouts/{fanout_txid}/status",), + audience=Audience.OPERATOR, + disposition=Disposition.RETAIN, + access=(LedgerAccess.READ_SLOT,), + ledger_methods=("ctv_fanout_status",), + rationale=( + "Raw broadcast status including submit_result and attempt detail. " + "/public/v1/fanouts/{fanout_txid} wraps the same ledger read in " + "the public schema." + ), + ), +) + + +# --- Payout-affecting surface ---------------------------------------------- +# +# Live payout obligations: the state the settlement machinery acts on. Out of +# scope for extraction by the terms of issue #145. + +_PAYOUT_AFFECTING: tuple[Endpoint, ...] = ( + Endpoint( + paths=("/audit/carry-forward-integrity", "/audit/ledger-integrity"), + audience=Audience.PAYOUT_AFFECTING, + disposition=Disposition.RETAIN, + access=(LedgerAccess.WRITER_LOCK,), + ledger_methods=("carry_forward_integrity_report",), + rationale=( + "The drift count and mismatch list over active carry-forward rows, " + "plus the audit_head_sha256 operators are told to mirror after " + "payout-affecting blocks. It reports whether the payout ledger " + "disagrees with itself. It runs two statements under one held " + "writer gate, so it is also the worst route on the surface to " + "expose to uncontrolled volume." + ), + ), + Endpoint( + paths=("/owed", "/owed-balances"), + audience=Audience.PAYOUT_AFFECTING, + disposition=Disposition.RETAIN, + access=(LedgerAccess.WRITER_LOCK,), + ledger_methods=("current_owed_balances",), + rationale=( + "Every recipient's outstanding owed balance -- the obligations the " + "next settlement will pay. O(recipients) under the writer lock. A " + "miner's own balance is available per-recipient from the public " + "tier; there is no public all-recipients dump." + ), + ), + Endpoint( + paths=("/miners/{recipient_id}/status", "/payouts/{recipient_id}/status"), + audience=Audience.PAYOUT_AFFECTING, + disposition=Disposition.RETAIN, + access=(LedgerAccess.WRITER_LOCK,), + ledger_methods=("current_owed_balances", "recipient_payout_history"), + rationale=( + "The pre-dashboard payout-operations view: owed balances plus " + "payout history in raw sats. Both aliases reach one handler. " + "Superseded for public consumption by " + "/public/v1/miners/{recipient_id}, which reports bits." + ), + ), +) + + +# --- Public read surface ---------------------------------------------------- +# +# The documented, versioned dashboard contract. Exactly this moves. + +_PUBLIC_READ: tuple[Endpoint, ...] = ( + Endpoint( + paths=("/public/v1/pool-summary",), + audience=Audience.PUBLIC_READ, + disposition=Disposition.EXTRACT, + access=(LedgerAccess.READ_SLOT, LedgerAccess.QBIT_RPC), + ledger_methods=("dashboard_pool_snapshot",), + rationale=( + "Pool-wide headline figures. Needs a qbit RPC client for the " + "network summary, which the extracted service gets its own copy " + "of -- it is a node read, not coordinator state." + ), + # Pool-wide aggregate: the 30s aggregate TTL, nothing beneath it. + max_staleness_seconds=staleness_budget_seconds( + cache_ttl_seconds=PUBLIC_AGGREGATE_CACHE_TTL_DEFAULT_SECONDS, + ), + ), + Endpoint( + paths=("/public/v1/blocks",), + audience=Audience.PUBLIC_READ, + disposition=Disposition.EXTRACT, + access=(LedgerAccess.READ_SLOT,), + ledger_methods=("dashboard_blocks",), + rationale="Paginated found-block history.", + max_staleness_seconds=staleness_budget_seconds( + cache_ttl_seconds=PUBLIC_CACHE_TTL_DEFAULT_SECONDS, + ), + ), + Endpoint( + paths=("/public/v1/leaderboard",), + audience=Audience.PUBLIC_READ, + disposition=Disposition.EXTRACT, + access=(LedgerAccess.READ_SLOT, LedgerAccess.QBIT_RPC), + ledger_methods=("dashboard_leaderboard", "dashboard_reward_leaderboard"), + rationale=( + "Both windows. window=3h is a plain read-slot aggregate; " + "window=reward additionally needs the network difficulty from RPC " + "to size the reward window." + ), + # Both windows read the ledger directly. dashboard_reward_leaderboard + # runs its own qbit_prism_window scan rather than going through the + # cached pool aggregate, so nothing stacks under the response TTL. + max_staleness_seconds=staleness_budget_seconds( + cache_ttl_seconds=PUBLIC_CACHE_TTL_DEFAULT_SECONDS, + ), + ), + Endpoint( + paths=("/public/v1/hashrate-series",), + audience=Audience.PUBLIC_READ, + disposition=Disposition.EXTRACT, + access=(LedgerAccess.READ_SLOT,), + ledger_methods=("dashboard_hashrate_series",), + rationale=( + "Bucketed hashrate history for the pool or one miner. The widest " + "ledger scan on the public surface and the strongest single " + "argument for moving this traffic off the primary." + ), + max_staleness_seconds=staleness_budget_seconds( + cache_ttl_seconds=PUBLIC_AGGREGATE_CACHE_TTL_DEFAULT_SECONDS, + ), + ), + Endpoint( + paths=("/public/v1/block-markers",), + audience=Audience.PUBLIC_READ, + disposition=Disposition.EXTRACT, + access=(LedgerAccess.READ_SLOT,), + ledger_methods=("dashboard_block_markers",), + rationale=( + "Found-block markers pre-aggregated onto the hashrate chart's " + "bucket grid. The pool finds hundreds of blocks a day, so the " + "dashboard cannot place markers on long timeframes by paging " + "/public/v1/blocks; this groups the small qbit_pool_blocks table " + "per bucket instead -- a cheap read-slot scan, so it carries the " + "plain row-read TTL rather than the aggregate one." + ), + max_staleness_seconds=staleness_budget_seconds( + cache_ttl_seconds=PUBLIC_CACHE_TTL_DEFAULT_SECONDS, + ), + ), + Endpoint( + paths=("/public/v1/mining-configuration",), + audience=Audience.PUBLIC_READ, + disposition=Disposition.EXTRACT, + access=(LedgerAccess.NONE,), + rationale=( + "Stratum endpoints and pool fee. Touches no ledger at all -- it " + "is assembled from environment configuration." + ), + extraction_blocker=( + "Falls back to getattr(coordinator, 'port') and " + "getattr(coordinator, 'bind') when the public URL env vars are " + "unset. The extracted service has no coordinator, so " + "PRISM_PUBLIC_STRATUM_URL / PRISM_STRATUM_PORT must be supplied " + "explicitly rather than inferred." + ), + # Environment-derived and near-static, so it carries the longest + # response TTL on the surface and, from it, the longest budget. + max_staleness_seconds=staleness_budget_seconds( + cache_ttl_seconds=PUBLIC_CONFIG_CACHE_TTL_DEFAULT_SECONDS, + ), + ), + Endpoint( + paths=("/public/v1/miners/{recipient_id}",), + audience=Audience.PUBLIC_READ, + disposition=Disposition.EXTRACT, + access=( + LedgerAccess.WRITER_LOCK, + LedgerAccess.READ_SLOT, + LedgerAccess.QBIT_RPC, + ), + ledger_methods=( + "current_owed_balances", + "dashboard_miner_share_summary", + "dashboard_miner_reward_window", + "dashboard_miner_payout_rows", + "dashboard_miner_worker_rows", + "dashboard_miner_lifetime_earnings_bits", + "dashboard_miner_pending_maturity_bits", + "dashboard_blocks", + ), + rationale=( + "A miner's own dashboard page. The most-polled route on the " + "surface and the clearest public read." + ), + extraction_blocker=( + "owed_balance_for_recipient() calls current_owed_balances(), a " + "writer-lock read that returns every recipient's balance and then " + "filters in Python. So the single most-polled public route takes " + "the writer lock today. The extracted service needs a " + "per-recipient owed-balance read that does not fence." + ), + # The one route with a second cache beneath the response cache: its + # reward-window slice comes from the shared pool aggregate, so the two + # TTLs compound. + max_staleness_seconds=staleness_budget_seconds( + cache_ttl_seconds=PUBLIC_CACHE_TTL_DEFAULT_SECONDS, + underlying_cache_seconds=POOL_REWARD_WINDOW_CACHE_DEFAULT_SECONDS, + ), + ), + Endpoint( + paths=("/public/v1/miners/{recipient_id}/earnings",), + audience=Audience.PUBLIC_READ, + disposition=Disposition.EXTRACT, + access=(LedgerAccess.READ_SLOT,), + ledger_methods=("dashboard_miner_earning_rows",), + rationale="Per-block earnings history for one miner.", + max_staleness_seconds=staleness_budget_seconds( + cache_ttl_seconds=PUBLIC_CACHE_TTL_DEFAULT_SECONDS, + ), + ), + Endpoint( + paths=("/public/v1/miners/{recipient_id}/payouts",), + audience=Audience.PUBLIC_READ, + disposition=Disposition.EXTRACT, + access=(LedgerAccess.READ_SLOT,), + ledger_methods=("dashboard_miner_payout_rows",), + rationale="Per-payout history for one miner.", + max_staleness_seconds=staleness_budget_seconds( + cache_ttl_seconds=PUBLIC_CACHE_TTL_DEFAULT_SECONDS, + ), + ), + Endpoint( + paths=("/public/v1/miners/{recipient_id}/workers",), + audience=Audience.PUBLIC_READ, + disposition=Disposition.EXTRACT, + access=(LedgerAccess.READ_SLOT,), + ledger_methods=("dashboard_miner_worker_rows",), + rationale="Worker list for one miner.", + # Classified as a pool-wide aggregate by public_cache_policy: it + # aggregates a miner's whole recent share history. + max_staleness_seconds=staleness_budget_seconds( + cache_ttl_seconds=PUBLIC_AGGREGATE_CACHE_TTL_DEFAULT_SECONDS, + ), + ), + Endpoint( + paths=("/public/v1/blocks/{block_hash}/settlement-artifacts",), + audience=Audience.PUBLIC_READ, + disposition=Disposition.EXTRACT, + access=( + LedgerAccess.WRITER_LOCK, + LedgerAccess.READ_SLOT, + LedgerAccess.ARTIFACT_FILE, + ), + ledger_methods=( + "audit_ctv_fanout_manifest_set", + "audit_bundle", + "ctv_fanout_status", + "dashboard_public_artifact_exists", + ), + rationale=( + "The public settlement view of a block, linking to the artifacts " + "that prove it. This is how published evidence reaches the public " + "without exposing /audit/*." + ), + extraction_blocker=( + "The CTV path is read-slot, but a direct-coinbase block falls " + "through to direct_coinbase_settlement_payload(), which calls " + "audit_bundle() under the writer lock. The extracted service must " + "take the direct-coinbase branch from the artifact file." + ), + max_staleness_seconds=staleness_budget_seconds( + cache_ttl_seconds=PUBLIC_CACHE_TTL_DEFAULT_SECONDS, + ), + ), + Endpoint( + paths=("/public/v1/fanouts/pending",), + audience=Audience.PUBLIC_READ, + disposition=Disposition.EXTRACT, + access=(LedgerAccess.READ_SLOT,), + ledger_methods=("dashboard_pending_fanout_rows",), + rationale="Public paginated view of fanouts awaiting broadcast.", + max_staleness_seconds=staleness_budget_seconds( + cache_ttl_seconds=PUBLIC_CACHE_TTL_DEFAULT_SECONDS, + ), + ), + Endpoint( + paths=("/public/v1/fanouts/{fanout_txid}",), + audience=Audience.PUBLIC_READ, + disposition=Disposition.EXTRACT, + access=(LedgerAccess.READ_SLOT,), + ledger_methods=("ctv_fanout_status",), + rationale="Public view of one fanout's broadcast state.", + max_staleness_seconds=staleness_budget_seconds( + cache_ttl_seconds=PUBLIC_CACHE_TTL_DEFAULT_SECONDS, + ), + ), + Endpoint( + paths=("/public/v1/artifacts/{sha256}",), + audience=Audience.PUBLIC_READ, + disposition=Disposition.EXTRACT, + access=(LedgerAccess.READ_SLOT, LedgerAccess.ARTIFACT_FILE), + ledger_methods=("dashboard_public_artifact",), + rationale=( + "Content-addressed fetch of a published artifact -- the sanctioned " + "public path to audit evidence, and the reason the public API can " + "keep /audit/* internal. Already resolves the body from body_uri " + "on disk when the inline column is NULL, verified against the " + "requested sha256. Its PRISM_PUBLIC_ARTIFACT_CACHE_* knobs move " + "with the extracted service. Content-addressed and immutable: a " + "body that hashes to the requested sha256 is correct at any age, " + "so this route carries no staleness budget and must never refuse " + "for staleness." + ), + immutable_content=True, + ), +) + + +ENDPOINTS: tuple[Endpoint, ...] = _OPERATOR + _PAYOUT_AFFECTING + _PUBLIC_READ + + +def endpoints_by_audience(audience: Audience) -> tuple[Endpoint, ...]: + return tuple(item for item in ENDPOINTS if item.audience is audience) + + +def endpoints_by_disposition(disposition: Disposition) -> tuple[Endpoint, ...]: + return tuple(item for item in ENDPOINTS if item.disposition is disposition) + + +def extracted_paths() -> tuple[str, ...]: + """Every request path the extracted public service must answer.""" + + return tuple( + path + for item in endpoints_by_disposition(Disposition.EXTRACT) + for path in item.paths + ) + + +def retained_paths() -> tuple[str, ...]: + """Every request path the coordinator keeps.""" + + return tuple( + path + for item in endpoints_by_disposition(Disposition.RETAIN) + for path in item.paths + ) + + +def extraction_blockers() -> tuple[tuple[str, str], ...]: + """(path, blocker) for each route whose implementation must change first.""" + + return tuple( + (item.primary_path, item.extraction_blocker) + for item in ENDPOINTS + if item.extraction_blocker is not None + ) + + +def _template_matches(template: str, path: str) -> bool: + """True when a concrete request path fills one registry path template.""" + + template_parts = template.split("/") + path_parts = path.split("/") + if len(template_parts) != len(path_parts): + return False + for expected, actual in zip(template_parts, path_parts): + if expected.startswith("{") and expected.endswith("}"): + if not actual: + return False + continue + if expected != actual: + return False + return True + + +def endpoint_for_request_path(path: str) -> Endpoint | None: + """Classify one concrete request path, or None when it is not a route. + + Literal paths win over templates so /public/v1/fanouts/pending resolves to + the pending list rather than to /public/v1/fanouts/{fanout_txid}, which is + the same precedence dispatch() applies. + """ + + for endpoint in ENDPOINTS: + if path in endpoint.paths: + return endpoint + for endpoint in ENDPOINTS: + for template in endpoint.paths: + if "{" in template and _template_matches(template, path): + return endpoint + return None + + +def writer_lock_paths() -> tuple[str, ...]: + """Paths that currently serialize against the lease-holding writer.""" + + return tuple( + path + for item in ENDPOINTS + if LedgerAccess.WRITER_LOCK in item.access + for path in item.paths + ) + + +__all__ = [ + "Audience", + "Disposition", + "ENDPOINTS", + "Endpoint", + "LedgerAccess", + "MINIMUM_PUBLIC_STALE_SECONDS", + "POOL_REWARD_WINDOW_CACHE_DEFAULT_SECONDS", + "PUBLIC_AGGREGATE_CACHE_TTL_DEFAULT_SECONDS", + "PUBLIC_CACHE_TTL_DEFAULT_SECONDS", + "PUBLIC_CONFIG_CACHE_TTL_DEFAULT_SECONDS", + "endpoint_for_request_path", + "endpoints_by_audience", + "endpoints_by_disposition", + "extracted_paths", + "extraction_blockers", + "retained_paths", + "staleness_budget_seconds", + "writer_lock_paths", +] diff --git a/lab/prism/future_callbacks.py b/lab/prism/future_callbacks.py new file mode 100644 index 000000000..24ef6ff54 --- /dev/null +++ b/lab/prism/future_callbacks.py @@ -0,0 +1,31 @@ +"""Completion callbacks whose captured work is released after invocation.""" + +from __future__ import annotations + +from concurrent.futures import Future +from typing import Callable, TypeVar + + +_Result = TypeVar("_Result") + + +def add_releasing_done_callback( + future: Future[_Result], callback: Callable[[Future[_Result]], object] +) -> None: + """Keep a callback alive until completion, then release its captures. + + Future retains registered callbacks after invoking them. A callback that + captures a request holding that future therefore retains the request and + result until cyclic GC, even when all external owners release them. + Leave only this empty wrapper registered once the callback runs, including + when it raises. Future still owns callback ordering and error handling. + """ + pending: Callable[[Future[_Result]], object] | None = callback + + def finish(completed: Future[_Result]) -> None: + nonlocal pending + completing, pending = pending, None + if completing is not None: + completing(completed) + + future.add_done_callback(finish) diff --git a/lab/prism/job_build_benchmark.py b/lab/prism/job_build_benchmark.py index 0282f9173..880665f71 100644 --- a/lab/prism/job_build_benchmark.py +++ b/lab/prism/job_build_benchmark.py @@ -42,12 +42,12 @@ from lab.auxpow import vardiff from lab.prism import direct_stratum +from lab.prism.coordinator_config import default_prism_coinbase_tag_hex from lab.prism.prism_coordinator import ( ClientState, PRISM_REJECTION_REASON_IDS, PrismCoordinator, WorkerIdentity, - default_prism_coinbase_tag_hex, ) EXTRANONCE2_SIZE = 8 @@ -461,7 +461,7 @@ def main() -> int: if not server.ensure_reorg_reconciled_for_tip(rpc.template["previousblockhash"]): raise RuntimeError("benchmark payout reconciliation failed") server._prepare_payout_ledger_artifact( - server._payout_state_generation, + server._ensure_payout_state_service().snapshot().generation, artifacts.network_difficulty, ) # The benchmark's timed path models an already-published artifact, not @@ -500,6 +500,10 @@ def observing(elapsed_seconds: float, phases: dict[str, float]) -> None: finally: pool.close() + # Read the delivery/cache metrics from their owner services instead of + # coordinator-private fields; the coordinator facade only routes here. + tip_refresh_service = server._ensure_tip_refresh_service() + job_bundle_service = server._ensure_job_bundle_service() report = { "schema": "qbit.prism.job-build-benchmark.v1", "mode": args.mode, @@ -514,23 +518,21 @@ def observing(elapsed_seconds: float, phases: dict[str, float]) -> None: "sent_job_elapsed_seconds": summarize(samples), "full_refresh_wall_seconds": summarize(flip_wall_times), "tip_to_first_delivery_seconds": summarize_histogram( - server.tip_refresh_histograms["first_delivery"] + tip_refresh_service.tip_refresh_histograms["first_delivery"] ), "tip_to_last_delivery_seconds": summarize_histogram( - server.tip_refresh_histograms["last_delivery"] + tip_refresh_service.tip_refresh_histograms["last_delivery"] ), "phase_totals_seconds": {k: round(v, 3) for k, v in sorted(phase_totals.items())}, "bundle_phase_totals_seconds": { phase: round(float(histogram["sum"]), 4) - for phase, histogram in getattr( - server, - "tip_refresh_build_phase_histograms", - {}, - ).items() + for phase, histogram in ( + tip_refresh_service.tip_refresh_build_phase_histograms.items() + ) }, - "builder_ipc_bytes": dict(getattr(server, "tip_refresh_ipc_bytes", {})), - "cache_hits": dict(server.job_cache_hit_counts), - "cache_misses": dict(server.job_cache_miss_counts), + "builder_ipc_bytes": dict(tip_refresh_service.tip_refresh_ipc_bytes), + "cache_hits": dict(job_bundle_service.job_cache_hit_counts), + "cache_misses": dict(job_bundle_service.job_cache_miss_counts), "getblocktemplate_calls": rpc.calls.get("getblocktemplate", 0), } text = json.dumps(report, indent=2) diff --git a/lab/prism/job_bundle.py b/lab/prism/job_bundle.py new file mode 100644 index 000000000..7471acf7a --- /dev/null +++ b/lab/prism/job_bundle.py @@ -0,0 +1,3628 @@ +#!/usr/bin/env python3 +"""Immutable PRISM job bundles, their cache, and the bounded build scheduler. + +The service owns the shared job-bundle cache, the bounded latest-wins build +scheduler (active/retiring/pending flights, priority and routine admission, +promise cleanup, orphan sweep), the per-generation issued-at anchors, the +share-window serialization cache slot, and the template-artifact repository. + +It never imports ``prism_coordinator``. Every cross-domain fact -- payout +state, tip-refresh authority, ledger snapshots, refresh metrics, live +configuration attributes -- is reached through the :class:`JobBundleRuntime` +typed port, resolved at call time so the historical coordinator monkeypatch +seams (including the instance-level facade patches used by the current test +suite) keep intercepting exactly as before the extraction. +""" + +from __future__ import annotations + +from collections import OrderedDict +from concurrent.futures import ( + FIRST_COMPLETED, + CancelledError, + Future, + InvalidStateError, + ThreadPoolExecutor, + wait, +) +from dataclasses import dataclass, field, replace as dataclass_replace +from decimal import Context, getcontext, localcontext +import hashlib +import json +import subprocess +import threading +import time +from types import FrameType +import weakref +from typing import Any, Callable, Protocol + +from lab.prism import direct_stratum +from lab.prism.bundle_compiler import _ShareWindowSerialization +from lab.prism.future_callbacks import add_releasing_done_callback +from lab.prism.share_json_stream import share_array_json_view +from lab.prism.share_ledger import DaemonShareJsonSequence +from lab.prism.coordinator_config import ( + DEFAULT_PRISM_JOB_BUILD_CANCEL_GRACE_SECONDS, + DEFAULT_PRISM_JOB_BUILD_EXECUTOR_WORKERS, + DEFAULT_PRISM_JOB_BUILD_TIMEOUT_SECONDS, + DEFAULT_PRISM_JOB_BUNDLE_CACHE_SECONDS, + DEFAULT_PRISM_ROUTINE_ADMISSION_DEADLINE_SECONDS, +) +from lab.prism.template_artifacts import ( + CachedTemplateArtifacts, + PayoutStatePublicationBlocked, + QbitTipTemplateSnapshot, + TemplateArtifactRepository, + TemplateRefreshBlocked, + TemplateRefreshSuperseded, +) + + +# Owner-local copies of the still-coordinator-visible constants; these are +# compatibility duplicates by design, not missed removals. +MAX_PRISM_JOB_BUNDLE_CACHE_ENTRIES = 128 +PRISM_JOB_EXTRANONCE1_PLACEHOLDER_HEX = "00000000" +PRISM_REWARD_WINDOW_MULTIPLIER = 8 +PRISM_SNAPSHOT_WINDOW_MARGIN = 2 +PRISM_JOB_BUILD_SECONDS_BUCKETS = ( + 0.01, + 0.025, + 0.05, + 0.1, + 0.25, + 0.5, + 1.0, + 2.5, + 5.0, + 10.0, + 30.0, +) +PRISM_JOB_BUILD_PHASES = ( + "reorg", + "template", + "merkle", + "ledger", + "payout_artifact", + "payout", + "ctv", + "input_serialization", + "worker", + "output_serialization", + "assembly", + "bundle", + "preparation_wait", + "executor_queue", + "client_lock", + "payout_gate", + "stamp", + "socket_send", + "send", +) +PRISM_JOB_CACHE_KINDS = ("template", "bundle") +# A flight whose executor future is finished normally leaves its slot inside +# the future's done callback. The admission sweep only treats such a flight +# as orphaned after this grace, so a callback that is merely between future +# completion and slot cleanup is never mistaken for a dead one. +PRISM_JOB_BUILD_ORPHAN_SWEEP_GRACE_SECONDS = 1.0 +# Cancellation-check slice while an initial request rides a subscribed +# publication-priority build promise. Promise completion wakes the waiter +# immediately; this only bounds how stale a cancellation can go unnoticed. +PRISM_INITIAL_JOB_SUBSCRIBE_POLL_SECONDS = 0.25 + + +def now_ms() -> int: + return int(time.time() * 1000) + + +def canonical_json_text(value: object) -> str: + return json.dumps(value, sort_keys=True, separators=(",", ":"), default=str) + + +def canonical_json_sha256(value: object) -> str: + incremental_digest = getattr(value, "canonical_json_sha256", None) + if callable(incremental_digest): + return str(incremental_digest()) + return hashlib.sha256(canonical_json_text(value).encode()).hexdigest() + + +class WorkerIdentityPort(Protocol): + """The exact worker facts a collection-mode build binds into its work.""" + + payout_address: str + p2mr_program_hex: str + + +class JobBuildCancelled(TemplateRefreshBlocked): + """An immutable build was cancelled or timed out.""" + + +class JobBuildSuperseded(JobBuildCancelled, TemplateRefreshSuperseded): + """A coordination race cooperatively cancelled immutable construction.""" + + +class JobBundleBuildSuperseded(JobBuildSuperseded): + """A newer tip or payout generation canceled this deterministic build.""" + + +class CollectionIdentityUnavailable(TemplateRefreshBlocked): + """Current collection work is waiting for an authorized worker identity.""" + + +class JobBuildAdmissionDeadlineExceeded(TemplateRefreshBlocked): + """A routine request parked behind publication priority past its bound. + + Subclasses TemplateRefreshBlocked so callers retry with their normal + pacing; the raise site logs and counts the expiry so parked admission is + visible instead of silently riding the client's own longer timeout. + """ + + +class JobBuildWaiterCancelled(RuntimeError): + """A bundle waiter became obsolete before it acquired preparation.""" + + +class JobBuildCancellation: + def __init__(self, *, timeout_seconds: float) -> None: + self._event = threading.Event() + self._lock = threading.Lock() + self.started_monotonic = time.monotonic() + self.deadline_monotonic = self.started_monotonic + timeout_seconds + self.last_checkpoint_monotonic = self.started_monotonic + self.cancelled_monotonic: float | None = None + self.reason: str | None = None + + def cancel(self, reason: str) -> bool: + with self._lock: + if self._event.is_set(): + return False + self.reason = reason + self.cancelled_monotonic = time.monotonic() + self._event.set() + return True + + def is_set(self) -> bool: + if self._event.is_set(): + return True + if time.monotonic() >= self.deadline_monotonic: + self.cancel("timeout") + return True + return False + + def raise_if_cancelled(self, phase: str) -> None: + if self.is_set(): + reason = self.reason or "cancelled" + if reason == "timeout": + raise JobBuildCancelled( + f"job build timeout at {phase}; immediate retry scheduled" + ) + raise JobBuildSuperseded( + f"job build {reason} at {phase}; immediate retry scheduled" + ) + with self._lock: + self.last_checkpoint_monotonic = time.monotonic() + + +# The executor task a build flight submits. The coordinator facade and the +# service method share this name, so a traceback carrying it belongs to a +# build that ran on the job-build executor. +_JOB_BUILD_EXECUTOR_ENTRY = "_execute_job_build_request" + + +def _clear_finished_frame(frame: FrameType) -> None: + try: + frame.clear() + except RuntimeError: + # frame.clear() refuses an executing frame and leaves it intact. The + # frames selected below have all returned, so this only guards the + # promise resolution against a foreign frame spliced into a chained + # or re-raised exception by code outside the producer. + pass + + +def _release_finished_job_build_frames(error: BaseException) -> None: + """Drop the locals of the finished producer frames a build error retains. + + A build that observes its cancellation raises from deep inside the + executor task. The stored exception's traceback references every frame it + unwound through, from the executor's task plumbing and the entry point + down to the checkpoint, and those frames still own the request (as a + local and inside the task's argument tuple), its template artifacts, the + ledger snapshot, and the converted share rows. The request owns the + promise that stores the exception, so the whole graph is a cycle that + only cyclic GC reclaims. + + The traceback's first entry is the executor's work-item frame that caught + the error and stored it on the future. That frame is still executing + while done callbacks run and is never touched. Every entry below it has + returned: the error could only reach the work item by propagating out of + all of them, including the executor context frame that unpacked the task + arguments. Those finished frames are cleared, and so are the finished + producer frames recorded by the error's ``__cause__`` and ``__context__`` + chain (a compiler failure re-raised as supersession keeps the failure as + its cause, whose own traceback still reaches the request). A chained + entry is only cleared from the first frame the cancellation itself + unwound through, so a frame that belongs to some other execution is left + alone. Clearing a finished frame drops only its locals; formatted + tracebacks keep their files, line numbers, and source lines, and the + chain itself is preserved. + """ + + head = error.__traceback__ + entry = head + while ( + entry is not None + and entry.tb_frame.f_code.co_name != _JOB_BUILD_EXECUTOR_ENTRY + ): + entry = entry.tb_next + if head is None or entry is None: + return + finished: set[FrameType] = set() + node = head.tb_next + while node is not None: + finished.add(node.tb_frame) + _clear_finished_frame(node.tb_frame) + node = node.tb_next + seen = {id(error)} + pending = [error] + while pending: + current = pending.pop() + for chained in (current.__cause__, current.__context__): + if chained is None or id(chained) in seen: + continue + seen.add(id(chained)) + pending.append(chained) + node = chained.__traceback__ + while node is not None and node.tb_frame not in finished: + node = node.tb_next + while node is not None: + finished.add(node.tb_frame) + _clear_finished_frame(node.tb_frame) + node = node.tb_next + + +def _job_build_error_for_waiter(error: JobBuildCancelled) -> JobBuildCancelled: + """Return one waiter's private copy of a promise's cancellation error. + + The copy carries the same type, message, diagnostic attributes, notes, + and ``__cause__``/``__context__`` chain, and shares the producer + traceback entries, so a formatted traceback still shows where the build + observed its cancellation and why. Raising a private instance leaves the + stored exception's traceback untouched: no waiter frame is ever appended + to it, and concurrent waiters cannot interleave their frames in each + other's diagnostics. + """ + + private = type(error)(*error.args) + private.__dict__.update(error.__dict__) + notes = getattr(error, "__notes__", None) + if notes is not None: + private.__notes__ = list(notes) + private.__cause__ = error.__cause__ + private.__context__ = error.__context__ + private.__suppress_context__ = error.__suppress_context__ + return private.with_traceback(error.__traceback__) + + +def _await_job_build_promise( + promise: "Future[CachedJobBundle]", + timeout: float, +) -> "CachedJobBundle": + """Wait for a shared build promise without re-raising its stored error. + + ``Future.result`` re-raises the stored instance, which prepends the + waiter's frames to the exception's traceback. A waiter that owns the + request or promise while it unwinds -- the retrying bundle loop does -- + thereby rebuilds the request -> promise -> exception -> traceback -> + request cycle for every cancelled build it observes. Cancellation + outcomes are raised as waiter-private copies instead; successful builds + and unexpected errors keep ``Future.result`` semantics exactly. + """ + + error = promise.exception(timeout=timeout) + if isinstance(error, JobBuildCancelled): + raise _job_build_error_for_waiter(error) + return promise.result() + + +@dataclass(frozen=True) +class JobBuildKey: + """Every immutable input that can affect one constructed mining job.""" + + best_tip_hash: str + previous_block_hash: str + template_fingerprint: str + template_generation: int + payout_state_generation: int + payout_artifact_sha256: str + mode: str + collection_identity: tuple[str, str] | None + block_height: int + coinbase_value_sats: int + network_difficulty: int + issued_at_ms: int + payout_policy_sha256: str + ctv_settlement_sha256: str | None + witness_merkle_sha256: str + transaction_set_sha256: str + coinbase_suffix_hex: str + signing_key_sha256: str + ledger_signing_key_sha256: str + numeric_context_sha256: str + share_snapshot_sha256: str = "" + payout_append_invalidation_epoch: int = 0 + + +@dataclass(frozen=True) +class CachedJobBundle: + """One heavy job build (ledger snapshot + signed manifest + base job) + shared across every client on the same template. + + The base job is built with the extranonce1 placeholder; per-client jobs + are stamped from it by swapping job_id, extranonce1, difficulty, and the + clean_jobs flag. All other fields are byte-identical across clients + because the stratum coinbase split excludes the extranonce window. + """ + + key: tuple[object, ...] + template: dict[str, Any] + template_fingerprint: str + coinbase_manifest: dict[str, Any] + shares_json: list[dict[str, object]] + prior_balances: list[dict[str, object]] + found_block: dict[str, object] + collection_only: bool + issued_at_ms: int + base_job: direct_stratum.DirectQbitStratumJob + built_monotonic: float + template_generation: int = 0 + payout_state_generation: int = 0 + payout_artifact_generation: int = 0 + # Collection coinbases commit a synthetic share to this exact payout + # identity. Ready bundles have no worker-specific inputs and keep this + # unset, which makes accidental cross-worker stamping fail closed. + collection_identity: tuple[str, str] | None = None + # Compact carry-forward state derived from the immutable job summary. It + # lets an accepted block publish child payout state without retaining the + # full audit bundle (and its duplicate shares tree) in every cached job. + prospective_prior_balances: tuple[tuple[str, str, str, int], ...] | None = None + build_key: JobBuildKey | None = None + # A synchronous ready build publishes its fenced ledger window for reuse + # only once this bundle wins cache publication (or is served as pinned + # snapshot work). Installing mid-build would re-key concurrent lookups + # away from the in-flight single flight and fork redundant builds. + prepared_ledger_artifact: "PayoutLedgerArtifact | None" = field( + default=None, + compare=False, + repr=False, + ) + + +@dataclass +class JobBuildRequest: + key: JobBuildKey + cache_key: tuple[object, ...] + equivalence_key: tuple[object, ...] + artifacts: CachedTemplateArtifacts + template_json: str + transaction_hexes: tuple[str, ...] + witness_merkle_leaves_hex: tuple[str, ...] + worker: "WorkerIdentityPort | None" + mode: str + payout_artifact: "PayoutStateArtifact" + payout_ledger_artifact: "PayoutLedgerArtifact | None" + payout_policy_json: str + ctv_settlement_json: str | None + decimal_context: Context = field(repr=False) + cancellation: JobBuildCancellation + idle_retarget: bool = False + publication_critical: bool = False + request_source: str = "routine" + priority_admission_recorded: bool = False + promise: Future[CachedJobBundle] = field(default_factory=Future) + requested_monotonic: float = field(default_factory=time.monotonic) + superseded_monotonic: float | None = None + # Exposed in-flight scan anchor owned by this build (see + # _expose_inflight_scan_anchor). Set by the synchronous ledger read so + # the executor wrapper can retire the exposure when the build dies + # before handing the token to its seeded artifact. + inflight_scan_anchor_token: int | None = None + + +@dataclass(eq=False) +class JobBuildFlight: + request: JobBuildRequest + future: Future[CachedJobBundle] | None = None + # First time the admission sweep saw this flight finished but still + # occupying its slot; eviction waits out the sweep grace from here. + orphan_observed_monotonic: float | None = None + + +@dataclass +class JobBundleBuildControl: + key: tuple[object, ...] + previousblockhash: str + payout_state_generation: int + payout_artifact_generation: int + cancel_event: threading.Event = field(default_factory=threading.Event) + process: subprocess.Popen[str] | None = None + + +@dataclass +class _SharedBundlePreparationFlight: + event: threading.Event = field(default_factory=threading.Event) + result: CachedJobBundle | None = None + error: BaseException | None = None + waiters: int = 0 + + +class JobBundleRuntime(Protocol): + """Typed port over the coordinator, resolved at call time. + + Every member is looked up on the live coordinator object when used, so + instance monkeypatches (``server.build_audit_bundle = ...`` and friends) + and coordinator-owned live configuration attributes keep working exactly + as before the extraction. Owner facades route back into this service; + that round trip is deliberate: it is what preserves the current + white-box patch surface until later stack layers repoint those tests. + """ + + # Cross-domain objects and live configuration attributes. + lock: Any + rpc: Any + ledger: Any + stop_event: Any + extranonce2_size: int + share_difficulty: Any + min_ready_miners: int + job_build_timeout_seconds: float + job_build_cancel_grace_seconds: float + + # First-touch initializers. + def _ensure_job_cache_state(self) -> None: ... + + def _ensure_tip_refresh_state(self) -> None: ... + + # Payout-state facades (owner extracted by a later chunk). + def _current_payout_state_artifact(self, cancellation: Any = None) -> Any: ... + + def _usable_payout_ledger_artifact( + self, + payout_state_generation: int, + network_difficulty: int, + ) -> Any: ... + + def _await_pending_parent_payout_preview( + self, + parent_hash: str, + *, + parent_height: int, + ) -> Any: ... + + def _prior_balances_for_job_parent( + self, + parent_hash: str, + *, + parent_height: int, + fallback_balances: list[dict[str, object]], + ) -> list[dict[str, object]]: ... + + def _serialize_prior_balance_preview(self, balances: Any) -> Any: ... + + def _accepted_block_payout_preview_from_bundle( + self, + bundle: dict[str, Any], + *, + prior_balances: list[dict[str, object]], + ) -> Any: ... + + def _install_payout_ledger_artifact(self, artifact: Any) -> Any: ... + + def _payout_artifact_reuse_active(self) -> bool: ... + + def _payout_artifact_max_anchor_age_ms(self) -> float: ... + + def _payout_artifact_reanchor_seconds(self) -> float: ... + + def _expose_inflight_scan_anchor(self, anchor_ms: int) -> int: ... + + def _retire_inflight_scan_anchor(self, token: int | None) -> None: ... + + def _publish_seedless_job_window_anchor_locked(self, anchor_ms: int) -> None: ... + + # Tip-refresh facades (owner extracted by a later chunk). + def _newest_observed_tip_locked(self) -> str | None: ... + + def _artifacts_buildable_locked(self, artifacts: CachedTemplateArtifacts) -> bool: ... + + def _published_snapshot_artifacts_locked( + self, + artifacts: CachedTemplateArtifacts, + ) -> bool: ... + + def _published_tip_authoritative_locked(self, now: float) -> bool: ... + + def _schedule_tip_refresh_retry(self) -> None: ... + + def _observe_tip_refresh_build_phase(self, name: str, elapsed: float) -> None: ... + + def _vardiff_idle_tip_divergence_locked(self) -> bool: ... + + # Shared readiness/stats and builder facades. + def pool_readiness_latched(self) -> bool: ... + + def accepted_share_stats(self) -> tuple[int, int]: ... + + def prism_payout_policy(self) -> dict[str, object]: ... + + def prism_ctv_settlement_config( + self, + *, + block_height: int, + parent_hash: str | None, + ) -> dict[str, object] | None: ... + + def coinbase_script_sig_suffix_hex( + self, + extranonce1_hex: str, + extranonce2_hex: str, + ) -> str: ... + + def _job_snapshot_anchor_ms(self, issued_at_ms: int) -> int: ... + + def _job_build_checkpoint(self, phase: str, cancellation: Any) -> None: ... + + def build_audit_bundle(self, **kwargs: Any) -> dict[str, Any]: ... + + +class JobBundleService: + """Sole owner of the job-bundle cache, scheduler, and readiness seams.""" + + def __init__( + self, + runtime: JobBundleRuntime, + *, + spool_factory: Callable[[], Any] | None = None, + payout_ledger_artifact_type: type | None = None, + now_ms: Callable[[], int] | None = None, + canonical_json_sha256_override: Callable[[object], str] | None = None, + ) -> None: + self._runtime = runtime + # Call-time-resolved wall clock: the coordinator wires a lambda over + # its own module global so the historical ``prism_coordinator.now_ms`` + # patch seam keeps steering anchor selection. + self._now_ms = now_ms if now_ms is not None else globals()["now_ms"] + # Same seam for the canonical digest: white-box tests intercept it + # through the coordinator module global. + self._canonical_json_sha256 = ( + canonical_json_sha256 + if canonical_json_sha256_override is None + else canonical_json_sha256_override + ) + # Call-time-resolved spool-file factory (see bundle_compiler); the + # coordinator wires its own module global through here so the + # historical patch seam keeps steering spool creation. + self._spool_factory = spool_factory + # The payout-ledger artifact remains coordinator-owned until the + # payout-state owner is extracted; the class is injected so seeded + # artifacts stay type-identical with coordinator-built ones. + self._payout_ledger_artifact_type = payout_ledger_artifact_type + self._job_cache_lock = threading.Lock() + self._active_job_bundle_builds: dict[ + tuple[object, ...], JobBundleBuildControl + ] = {} + # Compatibility seam for embedders/tests. Expensive construction is + # coordinated by the bounded latest-wins scheduler below, not held + # under this lock. + self._job_build_lock = threading.Lock() + self._job_build_scheduler_lock = threading.RLock() + self._job_build_priority_preparations: dict[int, float] = {} + self._job_build_priority_preparation_sequence = 0 + self._job_build_routine_preparations: dict[ + int, + weakref.ReferenceType[JobBuildCancellation], + ] = {} + self._job_build_routine_preparation_sequence = 0 + self._job_build_priority_changed = threading.Event() + self._job_build_executor: ThreadPoolExecutor | None = None + self._job_build_executor_shutdown = False + self._job_build_active: JobBuildFlight | None = None + self._job_build_retiring: JobBuildFlight | None = None + self._job_build_pending: JobBuildRequest | None = None + self._job_build_issued_at_ms: OrderedDict[int, int] = OrderedDict() + self._job_bundle_cache: OrderedDict[ + tuple[object, ...], CachedJobBundle + ] = OrderedDict() + self._job_build_phase_local = threading.local() + self.job_cache_hit_counts = {kind: 0 for kind in PRISM_JOB_CACHE_KINDS} + self.job_cache_miss_counts = {kind: 0 for kind in PRISM_JOB_CACHE_KINDS} + self.job_build_seconds_bucket_counts = { + bucket: 0 for bucket in PRISM_JOB_BUILD_SECONDS_BUCKETS + } + self.job_build_seconds_sum = 0.0 + self.job_build_count = 0 + self.job_build_phase_seconds = {phase: 0.0 for phase in PRISM_JOB_BUILD_PHASES} + self.job_build_scheduler_counts = { + "requests": 0, + "starts": 0, + "completions": 0, + "supersessions": 0, + "obsolete_results": 0, + "orphan_evicted": 0, + } + self.job_build_priority_counts = { + result: 0 + for result in ( + "started", + "coalesced", + "queued", + "routine_deferred", + "routine_preempted", + ) + } + self.job_build_priority_admission_seconds = { + "sum": 0.0, + "count": 0, + } + self.initial_job_prepared_work_counts = { + result: 0 + for result in ( + "cache_hit", + "singleflight", + "deferred", + "subscribed", + "admission_deadline", + ) + } + self._admission_deadline_last_log_monotonic: float | None = None + self.job_build_cancellation_seconds = { + "sum": 0.0, + "count": 0, + } + self.job_build_replacement_start_seconds = { + "sum": 0.0, + "count": 0, + } + self.job_build_worker_counts = { + "starts": 0, + "terminations": 0, + "crashes": 0, + "restarts": 0, + } + self._job_build_worker_restart_pending = False + self._share_window_serialization_lock = threading.Lock() + self._share_window_serialization: _ShareWindowSerialization | None = None + # Keyed shared-preparation flights absorbed from the historical + # coordinator field pair; retained for the still-coordinator-side + # metrics reader until later chunks retire them. + self._bundle_preparation_lock = threading.Lock() + self._bundle_preparation_flights: dict[ + tuple[object, ...], _SharedBundlePreparationFlight + ] = {} + self.shared_bundle_build_counts = { + outcome: 0 + for outcome in ("started", "completed", "superseded", "failed") + } + self.shared_bundle_preparation_seconds_sum = 0.0 + self.shared_bundle_preparation_count = 0 + self.shared_bundle_preparation_waiters = 0 + self._prepared_ready_bundle: CachedJobBundle | None = None + self._prepared_ready_snapshot: QbitTipTemplateSnapshot | None = None + self.job_preparation_pending = False + # The template-artifact repository shares the job-cache lock: the + # historical layout guarded template currency and bundle admission + # with one lock and remaining coordinator domains still rely on it. + self.template_repository = TemplateArtifactRepository( + runtime, + lock=self._job_cache_lock, + ) + + # -- phase bookkeeping ------------------------------------------------- + + def _job_build_phases(self) -> dict[str, float]: + """Per-thread scratch dict of phase timings for the current build.""" + self._runtime._ensure_job_cache_state() + phases = getattr(self._job_build_phase_local, "phases", None) + if phases is None: + phases = {} + self._job_build_phase_local.phases = phases + return phases + + def observe_job_build_elapsed( + self, + elapsed_seconds: float, + phases: dict[str, float], + ) -> None: + self._runtime._ensure_job_cache_state() + with self._job_cache_lock: + self.job_build_count += 1 + self.job_build_seconds_sum += elapsed_seconds + for bucket in PRISM_JOB_BUILD_SECONDS_BUCKETS: + if elapsed_seconds <= bucket: + self.job_build_seconds_bucket_counts[bucket] += 1 + for phase, duration in phases.items(): + if phase in self.job_build_phase_seconds: + self.job_build_phase_seconds[phase] += duration + + def _flush_job_build_phases(self, phases: dict[str, float]) -> None: + """Fold a worker thread's phase accruals into the exported counters. + + Only the per-client delivery entry points clear and flush the + thread-local phase dict; the job-build executor and payout-artifact + preparation workers never passed through them, so the template/ + ledger/assembly/bundle seconds they accrued were exported as + exactly 0.0 while production paid multi-second walks -- the blind + spot that misdirected the 2026-07-31 incident review. Build counts + and the duration histogram stay per-delivery; this folds phase + seconds only, and clears the dict so a long-lived worker thread + cannot double-report an earlier request's accruals. + """ + if not phases: + return + self._runtime._ensure_job_cache_state() + with self._job_cache_lock: + for phase, duration in phases.items(): + if phase in self.job_build_phase_seconds: + self.job_build_phase_seconds[phase] += duration + phases.clear() + + def _record_job_cache_event(self, kind: str, *, hit: bool) -> None: + self._runtime._ensure_job_cache_state() + with self._job_cache_lock: + counts = self.job_cache_hit_counts if hit else self.job_cache_miss_counts + counts[kind] = int(counts.get(kind, 0)) + 1 + + # -- build-control registry -------------------------------------------- + + def _cancel_obsolete_job_bundle_builds( + self, + *, + current_tip: str | None = None, + payout_state_generation: int | None = None, + ) -> None: + """Cancel only builds proven obsolete by a newer exact generation.""" + self._runtime._ensure_job_cache_state() + processes: list[subprocess.Popen[str]] = [] + with self._job_cache_lock: + for control in self._active_job_bundle_builds.values(): + obsolete = ( + current_tip is not None + and control.previousblockhash != current_tip + ) or ( + payout_state_generation is not None + and control.payout_state_generation + != int(payout_state_generation) + ) + if not obsolete or control.cancel_event.is_set(): + continue + control.cancel_event.set() + if control.process is not None: + processes.append(control.process) + for process in processes: + if process.poll() is not None: + continue + try: + process.terminate() + except ProcessLookupError: + pass + + def _register_job_bundle_process( + self, + control: JobBundleBuildControl, + process: subprocess.Popen[str], + ) -> None: + terminate = False + with self._job_cache_lock: + if ( + self._active_job_bundle_builds.get(control.key) is not control + or control.cancel_event.is_set() + ): + terminate = True + else: + control.process = process + if terminate and process.poll() is None: + try: + process.terminate() + except ProcessLookupError: + pass + + def _unregister_job_bundle_process( + self, + control: JobBundleBuildControl, + process: Any, + ) -> None: + """Detach a shared daemon from a build control after its request. + + Registration makes supersession terminate the in-flight process. + Once this build's daemon request has completed, a later supersession + of the build must not kill the shared daemon out from under + whichever request owns it next. + """ + with self._job_cache_lock: + if control.process is process: + control.process = None + + # -- bounded latest-wins scheduler ------------------------------------- + + def _job_build_executor_locked(self) -> ThreadPoolExecutor: + runtime = self._runtime + if self._job_build_executor_shutdown: + raise RuntimeError("job build executor is shut down") + executor = self._job_build_executor + if executor is None: + executor = ThreadPoolExecutor( + max_workers=int( + getattr( + runtime, + "job_build_executor_workers", + DEFAULT_PRISM_JOB_BUILD_EXECUTOR_WORKERS, + ) + ), + thread_name_prefix="prism-job-build", + ) + self._job_build_executor = executor + return executor + + def _start_job_build_locked(self, request: JobBuildRequest) -> JobBuildFlight: + runtime = self._runtime + executor = runtime._job_build_executor_locked() + flight = JobBuildFlight(request=request) + self.job_build_scheduler_counts["starts"] += 1 + self.shared_bundle_build_counts["started"] += 1 + if request.superseded_monotonic is not None: + elapsed = max(0.0, time.monotonic() - request.superseded_monotonic) + self.job_build_replacement_start_seconds["sum"] += elapsed + self.job_build_replacement_start_seconds["count"] += 1 + future = executor.submit(runtime._execute_job_build_request, request) + flight.future = future + runtime._record_priority_admission_locked(request, "started") + return flight + + def _arm_job_build_locked(self, flight: JobBuildFlight) -> None: + runtime = self._runtime + future = flight.future + assert future is not None + add_releasing_done_callback( + future, + lambda completed: runtime._job_build_done(flight, completed), + ) + + def _execute_job_build_request( + self, + request: JobBuildRequest, + ) -> CachedJobBundle: + runtime = self._runtime + request.cancellation.raise_if_cancelled("start") + control = JobBundleBuildControl( + key=request.equivalence_key, + previousblockhash=request.key.previous_block_hash, + payout_state_generation=request.key.payout_state_generation, + payout_artifact_generation=( + request.payout_ledger_artifact.generation + if request.payout_ledger_artifact is not None + else 0 + ), + ) + with self._job_cache_lock: + self._active_job_bundle_builds[control.key] = control + previous_control = getattr( + self._job_build_phase_local, + "bundle_build_control", + None, + ) + self._job_build_phase_local.bundle_build_control = control + phases = runtime._job_build_phases() + phases.clear() + try: + with localcontext(request.decimal_context): + return runtime.build_shared_job_bundle( + request.artifacts, + request.worker, + mode=request.mode, + payout_state_generation=request.key.payout_state_generation, + payout_artifact=request.payout_ledger_artifact, + key=request.cache_key, + build_request=request, + ) + except BaseException: + # A build that died past its synchronous ledger read can no + # longer hand its exposed scan anchor to a seeded artifact; + # retire it here so a dead walk does not keep advancing the + # append-invalidation epoch. Successful builds either retired + # the token at the publication fence or handed it to the seed, + # whose install fence retires it. + runtime._retire_inflight_scan_anchor( + request.inflight_scan_anchor_token + ) + raise + finally: + runtime._flush_job_build_phases(phases) + self._job_build_phase_local.bundle_build_control = previous_control + with self._job_cache_lock: + if self._active_job_bundle_builds.get(control.key) is control: + self._active_job_bundle_builds.pop(control.key, None) + control.process = None + + @staticmethod + def _collection_job_builds_are_independent( + first: JobBuildRequest, + second: JobBuildRequest, + ) -> bool: + """Distinct workers in one immutable collection cohort are peers.""" + + return ( + first.mode == "collection" + and second.mode == "collection" + and first.key.collection_identity != second.key.collection_identity + and dataclass_replace(first.key, collection_identity=None) + == dataclass_replace(second.key, collection_identity=None) + ) + + @staticmethod + def _job_build_requests_can_share( + first: JobBuildRequest, + second: JobBuildRequest, + ) -> bool: + """Share exact builds plus ready work stable across clock-only refreshes.""" + + return first.equivalence_key == second.equivalence_key or ( + first.mode == "ready" + and second.mode == "ready" + and first.cache_key == second.cache_key + ) + + @staticmethod + def _ready_job_build_precedes_collection( + first: JobBuildRequest, + second: JobBuildRequest, + ) -> bool: + """Live ready work cannot be displaced by a collection-mode retry.""" + + return ( + first.mode == "ready" + and second.mode == "collection" + and not first.cancellation.is_set() + ) + + @staticmethod + def _defer_job_build_locked( + *blockers: Future[CachedJobBundle], + ) -> Future[CachedJobBundle]: + """Wake a bounded lower-priority waiter when occupied capacity exits.""" + + deferred: Future[CachedJobBundle] = Future() + wake_lock = threading.Lock() + + def wake_for_retry(_completed: Future[CachedJobBundle]) -> None: + with wake_lock: + if not deferred.done(): + deferred.set_exception( + JobBuildSuperseded( + "job build capacity became available; retrying" + ) + ) + + for blocker in blockers: + blocker.add_done_callback(wake_for_retry) + return deferred + + @staticmethod + def _job_build_is_publication_critical(request: object) -> bool: + return bool(getattr(request, "publication_critical", False)) + + @staticmethod + def _job_build_promise_done(request: object) -> bool: + """Whether a request's shared promise has already been resolved. + + Reads defensively like the other request predicates: embedders and + tests plant lightweight request doubles without a promise, which are + never resolved-and-abandoned flights. + """ + + promise = getattr(request, "promise", None) + return promise is not None and promise.done() + + def _record_priority_admission_locked( + self, + request: JobBuildRequest, + result: str, + ) -> None: + """Observe first builder admission from publication-priority reservation.""" + + runtime = self._runtime + if not runtime._job_build_is_publication_critical(request): + return + self.job_build_priority_counts[result] += 1 + if result not in {"started", "coalesced"}: + return + if request.priority_admission_recorded: + return + request.priority_admission_recorded = True + elapsed = max(0.0, time.monotonic() - request.requested_monotonic) + self.job_build_priority_admission_seconds["sum"] += elapsed + self.job_build_priority_admission_seconds["count"] += 1 + + def _record_initial_prepared_work_locked(self, result: str) -> None: + self.initial_job_prepared_work_counts[result] += 1 + + def _new_job_build_cancellation(self) -> JobBuildCancellation: + return JobBuildCancellation( + timeout_seconds=max( + 0.001, + float( + getattr( + self._runtime, + "job_build_timeout_seconds", + DEFAULT_PRISM_JOB_BUILD_TIMEOUT_SECONDS, + ) + ), + ) + ) + + def _begin_job_build_priority_preparation( + self, + requested_monotonic: float | None = None, + ) -> tuple[int, float]: + """Reserve publication priority before immutable request construction.""" + + started = ( + time.monotonic() + if requested_monotonic is None + else requested_monotonic + ) + with self._job_build_scheduler_lock: + self._job_build_priority_preparation_sequence += 1 + token = self._job_build_priority_preparation_sequence + self._job_build_priority_preparations[token] = started + for routine_cancellation_ref in tuple( + self._job_build_routine_preparations.values() + ): + routine_cancellation = routine_cancellation_ref() + if routine_cancellation is not None: + routine_cancellation.cancel("publication priority") + self._job_build_routine_preparations.clear() + self._job_build_priority_changed.set() + return token, started + + def _finish_job_build_priority_preparation(self, token: int) -> None: + with self._job_build_scheduler_lock: + self._job_build_priority_preparations.pop(token, None) + self._job_build_priority_changed.set() + + def _begin_routine_job_build_preparation( + self, + *, + request_source: str, + cancelled: Callable[[], bool] | None, + ) -> tuple[int, JobBuildCancellation, CachedJobBundle | None]: + """Atomically admit cancellable routine request construction. + + Initial requests do not poll out the publication-priority window: + they subscribe to the in-flight priority build's promise, wake when + it completes, and hand its bundle back to the caller. Riding the + result that displaced them replaces the old defer-then-rebuild cycle + (and its wakeup storm) with one shared build. + """ + + runtime = self._runtime + deferred_recorded = False + subscription_recorded = False + subscribed_bundle: CachedJobBundle | None = None + admission_started_monotonic = time.monotonic() + admission_deadline_seconds = float( + getattr( + runtime, + "routine_admission_deadline_seconds", + DEFAULT_PRISM_ROUTINE_ADMISSION_DEADLINE_SECONDS, + ) + ) + while True: + self._job_build_priority_changed.clear() + priority_promises: tuple[Future[CachedJobBundle], ...] = () + with self._job_build_scheduler_lock: + if not runtime._publication_priority_scheduled_locked(): + self._job_build_routine_preparation_sequence += 1 + token = self._job_build_routine_preparation_sequence + preparation_cancellation = ( + runtime._new_job_build_cancellation() + ) + coordinator_ref = weakref.ref(runtime) + + def remove_dead_preparation( + dead_ref: weakref.ReferenceType[ + JobBuildCancellation + ], + *, + preparation_token: int = token, + ) -> None: + coordinator = coordinator_ref() + if coordinator is None: + return + with coordinator._job_build_scheduler_lock: + if ( + coordinator._job_build_routine_preparations.get( + preparation_token + ) + is dead_ref + ): + coordinator._job_build_routine_preparations.pop( + preparation_token, + None, + ) + + self._job_build_routine_preparations[token] = weakref.ref( + preparation_cancellation, + remove_dead_preparation, + ) + return token, preparation_cancellation, subscribed_bundle + if not deferred_recorded: + self.job_build_priority_counts["routine_deferred"] += 1 + if request_source == "initial": + runtime._record_initial_prepared_work_locked("deferred") + deferred_recorded = True + if request_source == "initial": + priority_promises = ( + runtime._publication_priority_promises_locked() + ) + if priority_promises and not subscription_recorded: + runtime._record_initial_prepared_work_locked("subscribed") + subscription_recorded = True + if cancelled is not None and cancelled(): + raise JobBuildWaiterCancelled( + "job bundle request was cancelled behind publication priority" + ) + stop_event = getattr(runtime, "stop_event", None) + if stop_event is not None and stop_event.is_set(): + raise JobBuildWaiterCancelled( + "coordinator stopped behind publication priority" + ) + parked_seconds = time.monotonic() - admission_started_monotonic + if parked_seconds > admission_deadline_seconds: + # Unbounded parking here is how the 2026-07-29 incident hid: + # initial jobs rode the client's own 90s timeout with zero + # tracebacks. Fail fast and visibly instead; callers retry + # with their normal pacing. + with self._job_build_scheduler_lock: + if request_source == "initial": + runtime._record_initial_prepared_work_locked( + "admission_deadline" + ) + now_monotonic = time.monotonic() + should_log = ( + self._admission_deadline_last_log_monotonic is None + or now_monotonic + - self._admission_deadline_last_log_monotonic >= 5.0 + ) + if should_log: + self._admission_deadline_last_log_monotonic = ( + now_monotonic + ) + if should_log: + print( + "prism coordinator: job admission deadline exceeded " + f"source={request_source} " + f"parked_seconds={parked_seconds:.1f}", + flush=True, + ) + raise JobBuildAdmissionDeadlineExceeded( + "job bundle admission parked behind publication priority " + f"for {parked_seconds:.1f}s" + ) + if priority_promises: + for promise in priority_promises: + if ( + promise.done() + and not promise.cancelled() + and promise.exception() is None + ): + subscribed_bundle = promise.result() + pending_promises = tuple( + promise + for promise in priority_promises + if not promise.done() + ) + if pending_promises: + done, _pending = wait( + pending_promises, + timeout=PRISM_INITIAL_JOB_SUBSCRIBE_POLL_SECONDS, + return_when=FIRST_COMPLETED, + ) + for completed in done: + if ( + completed.cancelled() + or completed.exception() is not None + ): + continue + subscribed_bundle = completed.result() + else: + # Every live priority promise has resolved; the scheduler + # is about to sweep the finished flights. Waiting on the + # event instead of the done promises avoids a hot loop. + self._job_build_priority_changed.wait(0.05) + else: + # Reservation-only window: the priority request object (and + # its promise) does not exist yet. This window only spans + # immutable request construction, never the build itself. + self._job_build_priority_changed.wait(0.05) + + def _publication_priority_promises_locked( + self, + ) -> tuple[Future[CachedJobBundle], ...]: + """Promises of live publication-critical build requests.""" + + runtime = self._runtime + promises: list[Future[CachedJobBundle]] = [] + pending = self._job_build_pending + if ( + pending is not None + and not pending.cancellation.is_set() + and runtime._job_build_is_publication_critical(pending) + ): + promises.append(pending.promise) + for flight in (self._job_build_active, self._job_build_retiring): + if ( + flight is not None + and not flight.request.cancellation.is_set() + and runtime._job_build_is_publication_critical(flight.request) + ): + promises.append(flight.request.promise) + return tuple(promises) + + def _finish_routine_job_build_preparation(self, token: int) -> None: + with self._job_build_scheduler_lock: + self._job_build_routine_preparations.pop(token, None) + + def _publication_priority_scheduled_locked(self) -> bool: + runtime = self._runtime + if self._job_build_priority_preparations: + return True + pending = self._job_build_pending + if ( + pending is not None + and not pending.cancellation.is_set() + and not runtime._job_build_promise_done(pending) + and runtime._job_build_is_publication_critical(pending) + ): + return True + return any( + flight is not None + and not flight.request.cancellation.is_set() + and not runtime._job_build_promise_done(flight.request) + and runtime._job_build_is_publication_critical(flight.request) + for flight in ( + self._job_build_active, + self._job_build_retiring, + ) + ) + + def _job_build_can_inherit_publication_priority( + self, + existing: JobBuildRequest, + incoming: JobBuildRequest, + ) -> bool: + """Reject an almost-expired or stalled routine flight as a critical owner.""" + + runtime = self._runtime + if ( + not runtime._job_build_is_publication_critical(incoming) + or runtime._job_build_is_publication_critical(existing) + ): + return True + cancellation = existing.cancellation + total_budget = max( + 0.001, + cancellation.deadline_monotonic - cancellation.started_monotonic, + ) + remaining_budget = cancellation.deadline_monotonic - time.monotonic() + progress_budget = max( + 0.001, + float( + getattr( + runtime, + "job_build_cancel_grace_seconds", + DEFAULT_PRISM_JOB_BUILD_CANCEL_GRACE_SECONDS, + ) + ), + ) + progress_age = ( + time.monotonic() + - float( + getattr( + cancellation, + "last_checkpoint_monotonic", + cancellation.started_monotonic, + ) + ) + ) + return ( + remaining_budget >= total_budget / 2.0 + and progress_age <= progress_budget + ) + + @staticmethod + def _resolve_cancelled_job_build_promise( + request: JobBuildRequest, + reason: str, + ) -> None: + """Wake every waiter on a terminated flight's promise immediately. + + Safe against the build's own completion: the done callback checks + ``promise.done()`` before resolving, so the first resolution wins and + a late build result for a cancelled flight is dropped exactly as it + was when the callback produced the exception itself. + """ + + promise = getattr(request, "promise", None) + if promise is None or promise.done(): + return + if reason == "timeout": + promise.set_exception( + JobBuildCancelled("job build flight cancelled by timeout") + ) + return + promise.set_exception( + JobBuildSuperseded(f"job build flight cancelled: {reason}") + ) + + @staticmethod + def _job_build_flight_outcome( + request: JobBuildRequest, + future: Future[CachedJobBundle], + ) -> tuple[CachedJobBundle | None, BaseException | None]: + """Map a finished executor future onto the shared promise outcome.""" + + # Inspect the terminal error without re-raising it here: this frame + # owns both the request and future, so adding it to a stored error's + # traceback would retain the completed build until cyclic GC. + if future.cancelled(): + return None, CancelledError() + error = future.exception() + if error is not None: + if isinstance(error, JobBuildCancelled): + # The finished producer frames still own the request and its + # payout window through this traceback. Release them before + # the error is published to waiters; unexpected failures keep + # their full traceback for post-mortem inspection. + _release_finished_job_build_frames(error) + return None, error + result = future.result() + if request.cancellation.is_set(): + if request.cancellation.reason == "timeout": + error = JobBuildCancelled( + "job build completed after its timeout" + ) + else: + error = JobBuildSuperseded( + "obsolete job build completed after cancellation" + ) + return result, error + + def _evict_orphaned_job_build_flights_locked(self) -> list[str]: + """Evict finished flights whose completion never released their slot. + + A flight's done callback resolves its promise, vacates its slot, and + promotes pending work. Done callbacks swallow exceptions, so if one + dies mid-pass the flight wedges: admission keeps treating the slot as + occupied, deferred requesters chain onto a promise nobody will ever + resolve, and promotion never runs again even though the executor is + idle. Sweep such flights out at admission time: resolve the promise + from the finished future so parked waiters wake, free the slot so the + next requester becomes the new owner, and promote any parked pending + request. A flight whose future is still queued or executing is never + touched; its own completion performs the normal cleanup. + + Returns the eviction log lines instead of printing them: the caller + holds the scheduler lock, and a blocked stdout must not stall + admission (the admission-deadline log follows the same discipline). + """ + + runtime = self._runtime + eviction_logs: list[str] = [] + grace_seconds = float( + getattr( + runtime, + "job_build_orphan_sweep_grace_seconds", + PRISM_JOB_BUILD_ORPHAN_SWEEP_GRACE_SECONDS, + ) + ) + now = time.monotonic() + evicted = False + for slot_name in ("_job_build_active", "_job_build_retiring"): + flight: JobBuildFlight | None = getattr(self, slot_name) + if flight is None: + continue + # Embedders and tests plant lightweight flight doubles; read the + # lifecycle fields defensively like the rest of the scheduler. + future = getattr(flight, "future", None) + if future is not None and not future.done(): + continue + observed = getattr(flight, "orphan_observed_monotonic", None) + if observed is None: + try: + flight.orphan_observed_monotonic = now + except AttributeError: + pass + continue + if now - observed < grace_seconds: + continue + request = flight.request + promise = getattr(request, "promise", None) + if promise is not None and not promise.done(): + if future is None: + runtime._resolve_cancelled_job_build_promise( + request, + "superseded", + ) + else: + result, error = runtime._job_build_flight_outcome( + request, + future, + ) + try: + if error is not None: + promise.set_exception(error) + else: + assert result is not None + promise.set_result(result) + except InvalidStateError: + # The flight's own done callback won the resolution + # race between our done() check and this set. + pass + setattr(self, slot_name, None) + self.job_build_scheduler_counts["orphan_evicted"] += 1 + evicted = True + eviction_logs.append( + "prism coordinator: evicted orphaned job build flight " + f"slot={slot_name.removeprefix('_job_build_')} " + f"cancelled={flight.request.cancellation.is_set()} " + f"started={future is not None}" + ) + if evicted: + runtime._promote_pending_job_build_locked() + self._job_build_priority_changed.set() + return eviction_logs + + def _cancel_job_build_flight_locked( + self, + flight: JobBuildFlight, + reason: str, + *, + now: float | None = None, + ) -> bool: + runtime = self._runtime + if not flight.request.cancellation.cancel(reason): + # Already cancelled (possibly by its own deadline). A flight that + # never reached the executor still has no other resolver, so + # settle its waiters here instead of leaving them to the sweep. + if getattr(flight, "future", None) is None: + runtime._resolve_cancelled_job_build_promise( + flight.request, + flight.request.cancellation.reason or reason, + ) + return False + try: + flight.request.superseded_monotonic = ( + time.monotonic() if now is None else now + ) + self.job_build_scheduler_counts["supersessions"] += 1 + if reason == "publication priority": + self.job_build_priority_counts["routine_preempted"] += 1 + finally: + # A submitted future delivers the promise from its done callback + # when the build observes the cancellation and drains -- refresh + # drivers rely on that ordering to keep one heavy build in + # flight, and a completed-but-unswept future is the admission + # sweep's to resolve. A flight that never reached the executor + # has no callback at all: cancellation must wake its waiters + # itself or they burn their full wait deadline. + if getattr(flight, "future", None) is None: + runtime._resolve_cancelled_job_build_promise( + flight.request, + reason, + ) + self._job_build_priority_changed.set() + return True + + def _promote_pending_job_build_locked(self) -> None: + runtime = self._runtime + pending = self._job_build_pending + if pending is None: + return + if runtime._job_build_promise_done(pending): + # Every waiter has already been answered (eviction or a raced + # resolution); building it would spend a bounded executor slot on + # work nobody consumes. + self._job_build_pending = None + return + active = self._job_build_active + retiring = self._job_build_retiring + if ( + not runtime._job_build_is_publication_critical(pending) + and any( + flight is not None + and not flight.request.cancellation.is_set() + and not runtime._job_build_promise_done(flight.request) + and runtime._job_build_is_publication_critical(flight.request) + for flight in (active, retiring) + ) + ): + # The pending request may predate an exact-flight priority upgrade. + # Preserve the same admission invariant as _request_job_build: + # routine work cannot displace scheduled publication work. + return + if active is not None: + if retiring is not None: + return + if runtime._ready_job_build_precedes_collection( + active.request, + pending, + ) and not runtime._job_build_is_publication_critical(pending): + return + if not runtime._collection_job_builds_are_independent( + active.request, + pending, + ): + reason = ( + "publication priority" + if runtime._job_build_is_publication_critical(pending) + and not runtime._job_build_is_publication_critical(active.request) + else "superseded" + ) + runtime._cancel_job_build_flight_locked(active, reason) + self._job_build_retiring = active + self._job_build_active = None + elif retiring is not None: + if runtime._ready_job_build_precedes_collection( + retiring.request, + pending, + ) and not runtime._job_build_is_publication_critical(pending): + return + if not runtime._collection_job_builds_are_independent( + retiring.request, + pending, + ): + reason = ( + "publication priority" + if runtime._job_build_is_publication_critical(pending) + and not runtime._job_build_is_publication_critical(retiring.request) + else "superseded" + ) + runtime._cancel_job_build_flight_locked(retiring, reason) + self._job_build_pending = None + try: + flight = runtime._start_job_build_locked(pending) + except BaseException: + # The pending slot is already vacated: with no flight there is no + # done callback left that could ever resolve this promise for its + # waiters. + runtime._resolve_cancelled_job_build_promise(pending, "superseded") + raise + self._job_build_active = flight + runtime._arm_job_build_locked(flight) + + def _job_build_done( + self, + flight: JobBuildFlight, + future: Future[CachedJobBundle], + ) -> None: + runtime = self._runtime + request = flight.request + result, error = runtime._job_build_flight_outcome(request, future) + try: + with self._job_build_scheduler_lock: + self.job_build_scheduler_counts["completions"] += 1 + self.shared_bundle_preparation_count += 1 + self.shared_bundle_preparation_seconds_sum += max( + 0.0, + time.monotonic() - request.cancellation.started_monotonic, + ) + if request.cancellation.cancelled_monotonic is not None: + elapsed = max( + 0.0, + time.monotonic() + - request.cancellation.cancelled_monotonic, + ) + self.job_build_cancellation_seconds["sum"] += elapsed + self.job_build_cancellation_seconds["count"] += 1 + coordination_cancelled = isinstance( + error, JobBuildSuperseded + ) or ( + request.cancellation.is_set() + and request.cancellation.reason != "timeout" + ) + if error is not None and coordination_cancelled: + self.job_build_scheduler_counts["obsolete_results"] += 1 + self.shared_bundle_build_counts["superseded"] += 1 + with runtime._tip_refresh_metrics_lock: + runtime.tip_refresh_superseded_results += 1 + elif error is not None: + self.shared_bundle_build_counts["failed"] += 1 + else: + self.shared_bundle_build_counts["completed"] += 1 + if self._job_build_active is flight: + self._job_build_active = None + if self._job_build_retiring is flight: + self._job_build_retiring = None + runtime._promote_pending_job_build_locked() + finally: + # Done callbacks swallow exceptions, so a raise anywhere in the + # bookkeeping above must not skip promise resolution: an orphaned + # promise strands every joiner until its wait deadline and leaves + # a finished flight wedged in its slot with the executor idle. + try: + if not request.promise.done(): + if error is not None: + request.promise.set_exception(error) + else: + assert result is not None + request.promise.set_result(result) + except InvalidStateError: + # The admission sweep can resolve a wedged flight between the + # done() check and this set; either resolution wakes waiters. + pass + self._job_build_priority_changed.set() + + def _request_job_build(self, request: JobBuildRequest) -> Future[CachedJobBundle]: + runtime = self._runtime + runtime._ensure_job_cache_state() + with self._job_build_scheduler_lock: + eviction_logs = runtime._evict_orphaned_job_build_flights_locked() + for eviction_log in eviction_logs: + print(eviction_log, flush=True) + with self._job_build_scheduler_lock: + if getattr(request, "idle_retarget", False): + with runtime.lock: + defer_idle = runtime._vardiff_idle_tip_divergence_locked() + if defer_idle: + request.cancellation.cancel( + "idle retarget deferred during unpublished tip refresh" + ) + if not request.promise.done(): + request.promise.set_exception( + JobBuildSuperseded( + "idle retarget deferred during unpublished tip refresh" + ) + ) + return request.promise + self.job_build_scheduler_counts["requests"] += 1 + active = self._job_build_active + retiring = self._job_build_retiring + pending = self._job_build_pending + publication_critical = runtime._job_build_is_publication_critical( + request + ) + if ( + active is not None + and not active.request.cancellation.is_set() + and not runtime._job_build_promise_done(active.request) + and runtime._job_build_requests_can_share(active.request, request) + and runtime._job_build_can_inherit_publication_priority( + active.request, + request, + ) + ): + if publication_critical: + active.request.publication_critical = True + active.request.request_source = request.request_source + active.request.requested_monotonic = request.requested_monotonic + active.request.priority_admission_recorded = True + runtime._record_priority_admission_locked(request, "coalesced") + if request.request_source == "initial": + runtime._record_initial_prepared_work_locked("singleflight") + return active.request.promise + if ( + retiring is not None + and not retiring.request.cancellation.is_set() + and not runtime._job_build_promise_done(retiring.request) + and runtime._job_build_requests_can_share(retiring.request, request) + and runtime._job_build_can_inherit_publication_priority( + retiring.request, + request, + ) + ): + if publication_critical: + retiring.request.publication_critical = True + retiring.request.request_source = request.request_source + retiring.request.requested_monotonic = request.requested_monotonic + retiring.request.priority_admission_recorded = True + runtime._record_priority_admission_locked(request, "coalesced") + if request.request_source == "initial": + runtime._record_initial_prepared_work_locked("singleflight") + return retiring.request.promise + if ( + pending is not None + and not pending.cancellation.is_set() + and not runtime._job_build_promise_done(pending) + and runtime._job_build_requests_can_share(pending, request) + and runtime._job_build_can_inherit_publication_priority( + pending, + request, + ) + ): + if publication_critical: + pending.publication_critical = True + pending.request_source = request.request_source + pending.requested_monotonic = request.requested_monotonic + runtime._record_priority_admission_locked(request, "queued") + now = time.monotonic() + for occupied in (active, retiring): + if ( + occupied is not None + and not runtime._job_build_requests_can_share( + occupied.request, + pending, + ) + and not runtime._job_build_is_publication_critical( + occupied.request + ) + ): + runtime._cancel_job_build_flight_locked( + occupied, + "publication priority", + now=now, + ) + runtime._promote_pending_job_build_locked() + if request.request_source == "initial": + runtime._record_initial_prepared_work_locked("singleflight") + return pending.promise + + if not publication_critical: + priority_blockers = tuple( + blocker + for blocker in ( + active.request if active is not None else None, + retiring.request if retiring is not None else None, + pending, + ) + if blocker is not None + and not blocker.cancellation.is_set() + # A resolved promise is not live priority work: deferring + # on it wakes instantly and spins until the sweep evicts. + and not runtime._job_build_promise_done(blocker) + and runtime._job_build_is_publication_critical(blocker) + ) + if priority_blockers: + self.job_build_priority_counts["routine_deferred"] += 1 + if request.request_source == "initial": + runtime._record_initial_prepared_work_locked("deferred") + return runtime._defer_job_build_locked( + *(blocker.promise for blocker in priority_blockers) + ) + + if request.mode == "collection": + possible_blockers = ( + pending, + active.request if active is not None else None, + retiring.request if retiring is not None else None, + ) + for blocker in possible_blockers: + if ( + blocker is not None + and not blocker.cancellation.is_set() + and not runtime._job_build_promise_done(blocker) + and runtime._ready_job_build_precedes_collection( + blocker, + request, + ) + and not ( + publication_critical + and not runtime._job_build_is_publication_critical( + blocker + ) + ) + ): + return runtime._defer_job_build_locked(blocker.promise) + if active is None: + if pending is not None: + if runtime._collection_job_builds_are_independent( + pending, + request, + ): + self._job_build_pending = None + try: + flight = runtime._start_job_build_locked(pending) + except BaseException: + # The pending slot is already vacated: with no + # flight there is no done callback left that + # could resolve this promise for its waiters. + runtime._resolve_cancelled_job_build_promise( + pending, + "superseded", + ) + raise + if retiring is None: + try: + replacement = runtime._start_job_build_locked( + request + ) + except BaseException: + # The first start succeeded: slot and arm it + # so its completion still resolves and sweeps + # normally before this failure propagates. + self._job_build_active = flight + runtime._arm_job_build_locked(flight) + raise + self._job_build_retiring = flight + self._job_build_active = replacement + runtime._arm_job_build_locked(flight) + runtime._arm_job_build_locked(replacement) + return request.promise + self._job_build_active = flight + runtime._arm_job_build_locked(flight) + return runtime._defer_job_build_locked( + flight.request.promise, + retiring.request.promise, + ) + pending.cancellation.cancel("superseded while pending") + if not pending.promise.done(): + pending.promise.set_exception( + JobBuildSuperseded("pending job build was superseded") + ) + self._job_build_pending = None + self.job_build_scheduler_counts["supersessions"] += 1 + if ( + retiring is not None + and not runtime._collection_job_builds_are_independent( + retiring.request, + request, + ) + ): + now = time.monotonic() + reason = ( + "publication priority" + if publication_critical + and not runtime._job_build_is_publication_critical( + retiring.request + ) + else "superseded" + ) + if runtime._cancel_job_build_flight_locked( + retiring, + reason, + now=now, + ): + request.superseded_monotonic = now + flight = runtime._start_job_build_locked(request) + self._job_build_active = flight + runtime._arm_job_build_locked(flight) + return request.promise + + if runtime._collection_job_builds_are_independent( + active.request, + request, + ): + if self._job_build_retiring is None: + self._job_build_retiring = active + flight = runtime._start_job_build_locked(request) + self._job_build_active = flight + runtime._arm_job_build_locked(flight) + return request.promise + if pending is None: + self._job_build_pending = request + if publication_critical: + runtime._record_priority_admission_locked(request, "queued") + now = time.monotonic() + for occupied in (active, self._job_build_retiring): + if ( + occupied is not None + and not runtime._job_build_is_publication_critical( + occupied.request + ) + ): + runtime._cancel_job_build_flight_locked( + occupied, + "publication priority", + now=now, + ) + return request.promise + assert retiring is not None + if publication_critical: + pending.cancellation.cancel("superseded while pending") + if not pending.promise.done(): + pending.promise.set_exception( + JobBuildSuperseded( + "pending job build was superseded by publication priority" + ) + ) + self.job_build_scheduler_counts["supersessions"] += 1 + self._job_build_pending = request + runtime._record_priority_admission_locked(request, "queued") + now = time.monotonic() + for occupied in (active, retiring): + if not runtime._job_build_is_publication_critical( + occupied.request + ): + runtime._cancel_job_build_flight_locked( + occupied, + "publication priority", + now=now, + ) + return request.promise + return runtime._defer_job_build_locked( + active.request.promise, + retiring.request.promise, + ) + + now = time.monotonic() + for obsolete in (active, retiring): + if obsolete is not None: + reason = ( + "publication priority" + if publication_critical + and not runtime._job_build_is_publication_critical( + obsolete.request + ) + else "superseded" + ) + runtime._cancel_job_build_flight_locked( + obsolete, + reason, + now=now, + ) + request.superseded_monotonic = now + if retiring is None: + self._job_build_retiring = active + flight = runtime._start_job_build_locked(request) + self._job_build_active = flight + runtime._arm_job_build_locked(flight) + return request.promise + + previous_pending = self._job_build_pending + if previous_pending is not None: + previous_pending.cancellation.cancel("superseded while pending") + if not previous_pending.promise.done(): + previous_pending.promise.set_exception( + JobBuildSuperseded("pending job build was superseded") + ) + self.job_build_scheduler_counts["supersessions"] += 1 + self._job_build_pending = request + if publication_critical: + runtime._record_priority_admission_locked(request, "queued") + return request.promise + + def _cancel_obsolete_job_builds( + self, + reason: str, + *, + keep_published_snapshot: bool = False, + ) -> None: + runtime = self._runtime + runtime._ensure_job_cache_state() + published_parent: str | None = None + published_fingerprint: str | None = None + if keep_published_snapshot: + # A per-client build for exactly the published snapshot is still + # valid, creditable work when a same-tip template bump sweeps old + # fingerprints. Detection-time sweeps deliberately do not use this: + # once a replacement tip is detected, even published-tip builds + # must vacate the single-flight lane for the replacement's work. + # Snapshot the identity outside the scheduler lock so the keep + # check stays lock-free below. + with runtime.lock: + published = getattr(runtime, "current_tip_first_seen", None) + snapshot = getattr(runtime, "tip_template_snapshot", None) + snapshot_tip = getattr(snapshot, "bestblockhash", None) + snapshot_fingerprint = getattr( + snapshot, "template_fingerprint", None + ) + if ( + published is not None + and snapshot_tip == published[0] + and snapshot_fingerprint is not None + and runtime._published_tip_authoritative_locked(time.monotonic()) + ): + published_parent = published[0] + published_fingerprint = snapshot_fingerprint + + def keep(request: JobBuildRequest) -> bool: + return bool( + published_fingerprint is not None + and request.artifacts.previousblockhash == published_parent + and request.artifacts.fingerprint == published_fingerprint + ) + + with self._job_build_scheduler_lock: + for flight in (self._job_build_active, self._job_build_retiring): + if flight is not None and not keep(flight.request): + runtime._cancel_job_build_flight_locked(flight, reason) + pending = self._job_build_pending + if pending is not None and not keep(pending): + pending.cancellation.cancel(reason) + if not pending.promise.done(): + pending.promise.set_exception( + JobBuildSuperseded(f"pending job build {reason}") + ) + self._job_build_pending = None + self.job_build_scheduler_counts["supersessions"] += 1 + + def shutdown_job_build_executor(self) -> None: + runtime = self._runtime + runtime._ensure_job_cache_state() + with self._job_build_scheduler_lock: + for flight in (self._job_build_active, self._job_build_retiring): + if flight is None: + continue + flight.request.cancellation.cancel("shutdown") + future = getattr(flight, "future", None) + if future is None or future.done(): + # A terminal future's callback has already run (or died) + # and executor shutdown will not fire it again: this is + # the last chance to settle the flight's waiters. + runtime._resolve_cancelled_job_build_promise( + flight.request, + "shutdown", + ) + pending = self._job_build_pending + if pending is not None: + pending.cancellation.cancel("shutdown") + if not pending.promise.done(): + pending.promise.set_exception( + JobBuildSuperseded("pending job build cancelled by shutdown") + ) + self._job_build_pending = None + executor = self._job_build_executor + self._job_build_executor = None + self._job_build_executor_shutdown = True + if executor is not None: + executor.shutdown(wait=True, cancel_futures=True) + + # -- cache identity and admission -------------------------------------- + + def _job_bundle_payout_state_current(self, bundle: CachedJobBundle) -> bool: + runtime = self._runtime + runtime._ensure_job_cache_state() + with self._job_cache_lock: + artifact = runtime._published_payout_state.artifact + return bool( + bundle.payout_state_generation == runtime._payout_state_generation + and bundle.build_key is not None + and ( + bundle.collection_only + or int( + getattr( + bundle.build_key, + "payout_append_invalidation_epoch", + 0, + ) + ) + == runtime._payout_ledger_append_invalidation_epoch + ) + and artifact is not None + and bundle.build_key.payout_artifact_sha256 + == artifact.prior_balances_sha256 + ) + + @staticmethod + def _collection_bundle_identity(worker: WorkerIdentityPort) -> tuple[str, str]: + return worker.payout_address, worker.p2mr_program_hex + + def _job_bundle_key( + self, + artifacts: CachedTemplateArtifacts, + *, + mode: str, + payout_state_generation: int, + payout_artifact_generation: int = 0, + worker: WorkerIdentityPort | None, + ) -> tuple[object, ...]: + runtime = self._runtime + if mode == "ready": + return ( + artifacts.fingerprint, + artifacts.previousblockhash, + "ready", + payout_state_generation, + payout_artifact_generation, + ) + if mode != "collection": + raise ValueError(f"unknown PRISM job-bundle mode: {mode}") + if worker is None: + raise CollectionIdentityUnavailable( + "collection-mode worker identity is temporarily unavailable" + ) + return ( + artifacts.fingerprint, + artifacts.previousblockhash, + "collection", + artifacts.generation, + payout_state_generation, + payout_artifact_generation, + *runtime._collection_bundle_identity(worker), + ) + + def _job_bundle_mode(self, requested_mode: str | None) -> str: + runtime = self._runtime + if requested_mode is not None: + if requested_mode not in {"ready", "collection"}: + raise ValueError( + f"unknown PRISM job-bundle mode: {requested_mode}" + ) + return requested_mode + return "ready" if runtime.pool_readiness_latched() else "collection" + + def _lookup_job_bundle( + self, + key: tuple[object, ...], + ) -> CachedJobBundle | None: + runtime = self._runtime + ttl = getattr(runtime, "job_bundle_cache_seconds", DEFAULT_PRISM_JOB_BUNDLE_CACHE_SECONDS) + now = time.monotonic() + with self._job_cache_lock: + if ttl <= 0: + self._job_bundle_cache.clear() + return None + # The entry-count cap is not a memory bound: one production entry + # can reference more than 100k shares. Expired entries must release + # their snapshots instead of remaining resident until count eviction. + expired = [ + cache_key + for cache_key, entry in self._job_bundle_cache.items() + if now - entry.built_monotonic > ttl + ] + for cache_key in expired: + self._job_bundle_cache.pop(cache_key, None) + return self._job_bundle_cache.get(key) + return None + + def _no_artifact_job_bundle_key( + self, + artifacts: CachedTemplateArtifacts, + *, + mode: str, + payout_state_generation: int, + worker: WorkerIdentityPort | None, + ) -> tuple[object, ...]: + """Fallback cache identity for work built before an artifact re-key. + + A synchronous build caches its bundle under the no-artifact key and + then publishes its anchored window as the payout ledger artifact. The + next lookup is keyed to that artifact and would miss the still-fresh + bundle and rebuild identical work: within one payout generation and + template identity the no-artifact bundle binds the same balances, + window anchor, and policy inputs, so it remains a correct cache + identity to serve -- but only while its window digest matches the + armed artifact (see _no_artifact_bundle_matches_artifact); a fence + re-arm can install a fresher window that cached work must not shadow. + """ + return self._runtime._job_bundle_key( + artifacts, + mode=mode, + payout_state_generation=payout_state_generation, + payout_artifact_generation=0, + worker=worker, + ) + + @staticmethod + def _no_artifact_bundle_matches_artifact( + cached: CachedJobBundle, + payout_artifact: Any, + ) -> bool: + """Whether pre-re-key cached work carries exactly the artifact's window. + + A speculative rebuild can arm a fresher artifact while an older + no-artifact bundle is still inside its cache TTL. Serving that bundle + would shadow the fresher window behind cached work, so the fallback + identity is honored only when the window digests are byte-identical + -- which the bundle that published the artifact satisfies by + construction. + """ + return ( + payout_artifact is not None + and payout_artifact.share_snapshot_sha256 is not None + and cached.build_key is not None + and cached.build_key.share_snapshot_sha256 + == payout_artifact.share_snapshot_sha256 + ) + + def _job_bundle_entry_usable( + self, + cached: CachedJobBundle | None, + artifacts: CachedTemplateArtifacts, + ) -> bool: + """Re-validate freshness and readiness for cached bundles. + + Readiness is monotonic in practice (the distinct accepted-miner count + only grows), so submit-capable ready bundles are served as-is while + their declared window anchor stays inside the artifact staleness + bound. A cached collection bundle is re-checked against the cheap + aggregate stats: once the pool is ready it must stop being served, or + jobs would keep collecting winning shares without submitting blocks + for up to the cache TTL. + """ + runtime = self._runtime + if cached is None: + return False + # Usability follows the same split-authority rule as construction: a + # cached bundle for the newest detected tip stays reusable across + # refresh retries while the previous tip is still published, and the + # published snapshot's own bundle stays servable for pinned issuance. + with runtime.lock: + parent_usable = runtime._artifacts_buildable_locked(artifacts) + if not parent_usable: + return False + with self._job_cache_lock: + if runtime._payout_state_publication_blocked: + return False + if not runtime._job_bundle_payout_state_current(cached): + return False + if not cached.collection_only: + # Cached-ready-bundle validity follows the same ceiling/floor + # split as artifact reuse. The declared anchor is gated only by + # the audit CEILING: the anchor is frozen per template + # generation and predates the window walk, so a tight + # wall-clock gate here declared every rebuilt bundle dead on + # arrival once its template generation outlived the bound + # (2026-07-29 rollback). + found_block = getattr(cached, "found_block", None) + declared_anchor_ms = ( + found_block.get("anchor_job_issued_at_ms") + if isinstance(found_block, dict) + else None + ) + if declared_anchor_ms is not None and ( + self._now_ms() - int(declared_anchor_ms) + > runtime._payout_artifact_max_anchor_age_ms() + ): + return False + # A bundle keyed to the currently armed artifact generation + # carries exactly the armed window, so it serves for as long as + # reuse itself does (the armed artifact is event-invalidated by + # the payout fences and re-anchored in the background). Every + # other ready bundle (no-artifact builds and survivors of a + # re-key) carries a window nothing will re-anchor, so it keeps + # serving only inside the re-anchor floor -- its declared + # anchor is already ceiling-gated above, and a wall-clock + # budget tighter than the build cadence is exactly the + # 2026-07-30 brownout (every consecutive build fell back to the + # synchronous reward-window walk). + if int(getattr(cached, "payout_artifact_generation", 0)) > 0: + with self._job_cache_lock: + armed = runtime._payout_ledger_artifact + if ( + armed is not None + and int(armed.generation) + == int(cached.payout_artifact_generation) + ): + return True + return ( + time.monotonic() - float(cached.built_monotonic) + <= runtime._payout_artifact_reanchor_seconds() + ) + # Collection bundles sign a synthetic bootstrap share containing the + # exact template ntime. A clock-only observation keeps the stable work + # fingerprint, but it must rebuild this signed bundle instead of + # rebinding the old manifest to a new template generation. + if ( + cached.template is not artifacts.template + or cached.template_generation != artifacts.generation + ): + return False + try: + _, ready_miner_count = runtime.accepted_share_stats() + except Exception: + # If readiness cannot be proven, force the normal build path. That + # path will either build an up-to-date bundle or surface the ledger + # failure instead of continuing to issue no-submit collection jobs. + return False + return ready_miner_count < runtime.min_ready_miners + + def _bind_cached_bundle_to_artifacts( + self, + cached: CachedJobBundle, + artifacts: CachedTemplateArtifacts, + ) -> CachedJobBundle: + """Return the cached heavy bundle bound to this exact observation. + + Clock-only template changes intentionally keep the stable fingerprint. + Ready bundles may reuse their ledger snapshot and signed manifest, but + the Stratum base job must still carry the observing template's exact + ntime and generation. Collection bundles are filtered before this point + because their signed synthetic share contains the template ntime. + """ + runtime = self._runtime + if ( + cached.template is artifacts.template + and cached.template_generation == artifacts.generation + ): + return cached + manifest = cached.coinbase_manifest + base_job = direct_stratum.make_job_from_builder_manifest( + job_id="prism-template-base", + template=artifacts.template, + manifest=manifest, + extranonce1_hex=PRISM_JOB_EXTRANONCE1_PLACEHOLDER_HEX, + extranonce2_size=runtime.extranonce2_size, + desired_share_difficulty=runtime.share_difficulty, + clean_jobs=True, + transaction_hexes=artifacts.transaction_hexes, + ) + return dataclass_replace( + cached, + template=artifacts.template, + base_job=base_job, + template_generation=artifacts.generation, + build_key=( + dataclass_replace( + cached.build_key, + best_tip_hash=artifacts.previousblockhash, + previous_block_hash=artifacts.previousblockhash, + template_generation=artifacts.generation, + block_height=int(artifacts.template["height"]), + coinbase_value_sats=int(artifacts.template["coinbasevalue"]), + ) + if cached.build_key is not None + else None + ), + ) + + def _new_job_build_request( + self, + artifacts: CachedTemplateArtifacts, + worker: WorkerIdentityPort | None, + *, + mode: str, + payout_state_generation: int, + cache_key: tuple[object, ...], + payout_ledger_artifact: Any = None, + idle_retarget: bool = False, + publication_critical: bool = False, + request_source: str = "routine", + priority_requested_monotonic: float | None = None, + preparation_cancellation: JobBuildCancellation | None = None, + ) -> JobBuildRequest: + runtime = self._runtime + cancellation = ( + runtime._new_job_build_cancellation() + if preparation_cancellation is None + else preparation_cancellation + ) + cancellation.raise_if_cancelled("immutable snapshot") + # A pending accepted-parent transition owns the balances children must + # observe. Wait for its preview before acquiring any confirmed payout + # input so a child build cannot take -- or cache into the published + # artifact -- confirmed state that omits its new parent. The preview + # itself is re-resolved consistently during construction. + runtime._await_pending_parent_payout_preview( + artifacts.previousblockhash, + parent_height=int(artifacts.template["height"]) - 1, + ) + payout_artifact = runtime._current_payout_state_artifact(cancellation) + if payout_artifact.generation != payout_state_generation: + raise JobBuildSuperseded( + "payout artifact generation changed before build request" + ) + + phases = runtime._job_build_phases() + payout_started = time.monotonic() + payout_policy_json = canonical_json_text(runtime.prism_payout_policy()) + phases["payout"] = phases.get("payout", 0.0) + ( + time.monotonic() - payout_started + ) + cancellation.raise_if_cancelled("payout policy") + ctv_started = time.monotonic() + ctv_settlement = runtime.prism_ctv_settlement_config( + block_height=int(artifacts.template["height"]), + parent_hash=artifacts.previousblockhash, + ) + ctv_settlement_json = ( + canonical_json_text(ctv_settlement) + if ctv_settlement is not None + else None + ) + phases["ctv"] = phases.get("ctv", 0.0) + ( + time.monotonic() - ctv_started + ) + cancellation.raise_if_cancelled("CTV configuration") + + suffix_hex = runtime.coinbase_script_sig_suffix_hex( + PRISM_JOB_EXTRANONCE1_PLACEHOLDER_HEX, + "00" * runtime.extranonce2_size, + ) + collection_identity = ( + runtime._collection_bundle_identity(worker) + if mode == "collection" and worker is not None + else None + ) + decimal_context = getcontext().copy() + numeric_context_sha256 = self._canonical_json_sha256( + { + "precision": decimal_context.prec, + "rounding": decimal_context.rounding, + "minimum_exponent": decimal_context.Emin, + "maximum_exponent": decimal_context.Emax, + "capitals": decimal_context.capitals, + "clamp": decimal_context.clamp, + } + ) + with self._job_cache_lock: + issued_at_ms = self._job_build_issued_at_ms.get(artifacts.generation) + current_append_invalidation_epoch = int( + runtime._payout_ledger_append_invalidation_epoch + ) + if issued_at_ms is None: + # The issued time doubles as the audit window anchor, so it + # must not cover a stamped share whose commit is still in + # flight: the frozen anchor stays reproducible from the + # durable ledger for every rebuild of this generation. The + # pending-commit clamp is pure anchor selection -- it freezes + # the highest anchor whose covered shares are all durable, and + # shares stamped above it deterministically belong to the next + # window -- so nothing else needs to be captured with it. + candidate_anchor_ms = runtime._job_snapshot_anchor_ms(self._now_ms()) + with self._job_cache_lock: + issued_at_ms = self._job_build_issued_at_ms.get( + artifacts.generation + ) + if issued_at_ms is None: + issued_at_ms = candidate_anchor_ms + self._job_build_issued_at_ms[artifacts.generation] = ( + issued_at_ms + ) + while len(self._job_build_issued_at_ms) > 128: + self._job_build_issued_at_ms.popitem(last=False) + build_append_invalidation_epoch = ( + int(payout_ledger_artifact.append_invalidation_epoch) + if mode == "ready" and payout_ledger_artifact is not None + else current_append_invalidation_epoch + if mode == "ready" + else 0 + ) + build_key = JobBuildKey( + best_tip_hash=artifacts.previousblockhash, + previous_block_hash=artifacts.previousblockhash, + template_fingerprint=artifacts.fingerprint, + template_generation=artifacts.generation, + payout_state_generation=payout_state_generation, + payout_artifact_sha256=payout_artifact.prior_balances_sha256, + mode=mode, + collection_identity=collection_identity, + block_height=int(artifacts.template["height"]), + coinbase_value_sats=int(artifacts.template["coinbasevalue"]), + network_difficulty=int(artifacts.network_difficulty), + issued_at_ms=issued_at_ms, + payout_policy_sha256=hashlib.sha256( + payout_policy_json.encode() + ).hexdigest(), + ctv_settlement_sha256=( + hashlib.sha256(ctv_settlement_json.encode()).hexdigest() + if ctv_settlement_json is not None + else None + ), + witness_merkle_sha256=self._canonical_json_sha256( + artifacts.witness_merkle_leaves_hex + ), + transaction_set_sha256=self._canonical_json_sha256( + artifacts.transaction_hexes + ), + coinbase_suffix_hex=suffix_hex, + signing_key_sha256=hashlib.sha256( + str(getattr(runtime, "signing_seed_hex", "")).encode() + ).hexdigest(), + ledger_signing_key_sha256=hashlib.sha256( + str( + getattr( + runtime, + "ledger_attestation_signing_seed_hex", + "", + ) + ).encode() + ).hexdigest(), + numeric_context_sha256=numeric_context_sha256, + payout_append_invalidation_epoch=( + build_append_invalidation_epoch + ), + ) + immutable_identity: tuple[object, ...] = ( + cache_key, + artifacts.generation, + issued_at_ms, + payout_artifact.prior_balances_sha256, + build_key.payout_policy_sha256, + build_key.ctv_settlement_sha256, + build_key.witness_merkle_sha256, + build_key.transaction_set_sha256, + build_key.coinbase_suffix_hex, + build_key.signing_key_sha256, + build_key.ledger_signing_key_sha256, + build_key.numeric_context_sha256, + build_key.payout_append_invalidation_epoch, + ) + return JobBuildRequest( + key=build_key, + cache_key=cache_key, + equivalence_key=immutable_identity, + artifacts=artifacts, + template_json=canonical_json_text(artifacts.template), + transaction_hexes=artifacts.transaction_hexes, + witness_merkle_leaves_hex=artifacts.witness_merkle_leaves_hex, + worker=worker, + mode=mode, + payout_artifact=payout_artifact, + payout_ledger_artifact=payout_ledger_artifact, + payout_policy_json=payout_policy_json, + ctv_settlement_json=ctv_settlement_json, + decimal_context=decimal_context, + cancellation=cancellation, + idle_retarget=idle_retarget, + publication_critical=publication_critical, + request_source=request_source, + requested_monotonic=( + cancellation.started_monotonic + if priority_requested_monotonic is None + else priority_requested_monotonic + ), + ) + + def _cache_job_bundle_if_current( + self, + built: CachedJobBundle, + artifacts: CachedTemplateArtifacts, + ) -> bool: + """Cache only current state; report whether payout state stayed valid.""" + runtime = self._runtime + with self._job_cache_lock: + with runtime.lock: + buildable = runtime._artifacts_buildable_locked(artifacts) + snapshot_pinned = runtime._published_snapshot_artifacts_locked( + artifacts + ) + published_artifact = runtime._published_payout_state.artifact + if not buildable: + return False + if ( + built.payout_state_generation != runtime._payout_state_generation + or built.build_key is None + or ( + not built.collection_only + and built.build_key.payout_append_invalidation_epoch + != runtime._payout_ledger_append_invalidation_epoch + ) + or published_artifact is None + or built.build_key.payout_artifact_sha256 + != published_artifact.prior_balances_sha256 + ): + return False + current = runtime._template_artifacts + globally_current = ( + current is not None + and current.fingerprint == artifacts.fingerprint + and current.previousblockhash == artifacts.previousblockhash + and ( + not built.collection_only + or current.generation == artifacts.generation + ) + ) + if not globally_current and not snapshot_pinned: + # Only the current template observation may win the cache + # race; the sole exception is a pinned rebuild of exactly the + # published snapshot, which repeated direct issuance would + # otherwise rebuild for the rest of the unpublished window. + return False + self._job_bundle_cache[built.key] = built + self._job_bundle_cache.move_to_end(built.key) + while len(self._job_bundle_cache) > MAX_PRISM_JOB_BUNDLE_CACHE_ENTRIES: + oldest_key = next(iter(self._job_bundle_cache)) + self._job_bundle_cache.pop(oldest_key, None) + # Arm the build's fenced ledger window only after the bundle itself + # won cache publication: lookups re-keyed by the fresh artifact fall + # back to exactly this cached no-artifact entry instead of forking a + # redundant build. + if built.prepared_ledger_artifact is not None: + runtime._install_payout_ledger_artifact(built.prepared_ledger_artifact) + return True + + def _probe_initial_job_bundle( + self, + artifacts: CachedTemplateArtifacts, + worker: WorkerIdentityPort | None, + mode: str | None, + ) -> CachedJobBundle | None: + """Serve an initial request straight from the bundle cache. + + Routine admission exists to keep CPU-heavy request construction from + competing with a publication-critical build; a cache probe does none + of that work. Probing before admission lets a first job ship the + already-published bundle immediately instead of waiting out an entire + priority build it would never have contended with. + """ + runtime = self._runtime + resolved_mode = runtime._job_bundle_mode(mode) + if resolved_mode == "collection" and worker is None: + raise CollectionIdentityUnavailable( + "collection-mode worker identity is temporarily unavailable" + ) + with self._job_cache_lock: + payout_state_generation = runtime._payout_state_generation + payout_artifact = ( + runtime._usable_payout_ledger_artifact( + payout_state_generation, + artifacts.network_difficulty, + ) + if resolved_mode == "ready" + else None + ) + key = runtime._job_bundle_key( + artifacts, + mode=resolved_mode, + payout_state_generation=payout_state_generation, + payout_artifact_generation=( + payout_artifact.generation if payout_artifact is not None else 0 + ), + worker=worker, + ) + cached = runtime._lookup_job_bundle(key) + if cached is None and payout_artifact is not None: + fallback = runtime._lookup_job_bundle( + runtime._no_artifact_job_bundle_key( + artifacts, + mode=resolved_mode, + payout_state_generation=payout_state_generation, + worker=worker, + ) + ) + if fallback is not None and runtime._no_artifact_bundle_matches_artifact( + fallback, + payout_artifact, + ): + cached = fallback + if not runtime._job_bundle_entry_usable(cached, artifacts): + return None + runtime._record_job_cache_event("bundle", hit=True) + with self._job_build_scheduler_lock: + self.initial_job_prepared_work_counts["cache_hit"] += 1 + assert cached is not None + return runtime._bind_cached_bundle_to_artifacts(cached, artifacts) + + def shared_job_bundle( + self, + artifacts: CachedTemplateArtifacts, + worker: WorkerIdentityPort | None = None, + *, + mode: str | None = None, + cancelled: Callable[[], bool] | None = None, + retry_superseded: bool = True, + idle_retarget: bool = False, + publication_critical: bool = False, + request_source: str = "routine", + priority_requested_monotonic: float | None = None, + ) -> CachedJobBundle: + """Return one immutable heavy build through a work-identity flight. + + Equivalent callers share one future. Publication-critical refreshes + can cancel routine work and start in the bounded replacement slot; + reconnect, Vardiff, authorization, and same-tip work must defer while + that priority work is active. Only the latest pending request is + retained during repeated supersession. + """ + runtime = self._runtime + runtime._ensure_job_cache_state() + runtime._ensure_tip_refresh_state() + priority_token: int | None = None + if publication_critical: + ( + priority_token, + priority_requested_monotonic, + ) = runtime._begin_job_build_priority_preparation( + priority_requested_monotonic + ) + try: + return runtime._shared_job_bundle_after_priority_admission( + artifacts, + worker, + mode=mode, + cancelled=cancelled, + retry_superseded=retry_superseded, + idle_retarget=idle_retarget, + publication_critical=publication_critical, + request_source=request_source, + priority_requested_monotonic=priority_requested_monotonic, + ) + finally: + if priority_token is not None: + runtime._finish_job_build_priority_preparation(priority_token) + + def _shared_job_bundle_after_priority_admission( + self, + artifacts: CachedTemplateArtifacts, + worker: WorkerIdentityPort | None = None, + *, + mode: str | None = None, + cancelled: Callable[[], bool] | None = None, + retry_superseded: bool = True, + idle_retarget: bool = False, + publication_critical: bool = False, + request_source: str = "routine", + priority_requested_monotonic: float | None = None, + ) -> CachedJobBundle: + runtime = self._runtime + while True: + routine_preparation_token: int | None = None + preparation_cancellation: JobBuildCancellation | None = None + subscribed_bundle: CachedJobBundle | None = None + if not publication_critical: + if request_source == "initial": + probed = runtime._probe_initial_job_bundle( + artifacts, + worker, + mode, + ) + if probed is not None: + return probed + ( + routine_preparation_token, + preparation_cancellation, + subscribed_bundle, + ) = runtime._begin_routine_job_build_preparation( + request_source=request_source, + cancelled=cancelled, + ) + resolved_mode = runtime._job_bundle_mode(mode) + if preparation_cancellation is not None: + preparation_cancellation.raise_if_cancelled( + "request preparation admission" + ) + if resolved_mode == "collection" and worker is None: + raise CollectionIdentityUnavailable( + "collection-mode worker identity is temporarily unavailable" + ) + with self._job_cache_lock: + payout_state_generation = runtime._payout_state_generation + payout_artifact = ( + runtime._usable_payout_ledger_artifact( + payout_state_generation, + artifacts.network_difficulty, + ) + if resolved_mode == "ready" + else None + ) + if preparation_cancellation is not None: + preparation_cancellation.raise_if_cancelled( + "payout artifact lookup" + ) + payout_artifact_generation = ( + payout_artifact.generation if payout_artifact is not None else 0 + ) + key = runtime._job_bundle_key( + artifacts, + mode=resolved_mode, + payout_state_generation=payout_state_generation, + payout_artifact_generation=payout_artifact_generation, + worker=worker, + ) + no_artifact_key = ( + runtime._no_artifact_job_bundle_key( + artifacts, + mode=resolved_mode, + payout_state_generation=payout_state_generation, + worker=worker, + ) + if payout_artifact_generation != 0 + else None + ) + cached = runtime._lookup_job_bundle(key) + if ( + cached is None + and subscribed_bundle is not None + and ( + subscribed_bundle.key == key + or ( + no_artifact_key is not None + and subscribed_bundle.key == no_artifact_key + and runtime._no_artifact_bundle_matches_artifact( + subscribed_bundle, + payout_artifact, + ) + ) + ) + ): + # The subscribed priority result may not have won the global + # cache-publication race yet; consume it directly. Usability + # below applies exactly the checks a cache entry would face. + cached = subscribed_bundle + if cached is None and no_artifact_key is not None: + fallback = runtime._lookup_job_bundle(no_artifact_key) + if ( + fallback is not None + and runtime._no_artifact_bundle_matches_artifact( + fallback, + payout_artifact, + ) + ): + cached = fallback + if runtime._job_bundle_entry_usable(cached, artifacts): + if preparation_cancellation is not None: + preparation_cancellation.raise_if_cancelled( + "bundle cache lookup" + ) + if routine_preparation_token is not None: + runtime._finish_routine_job_build_preparation( + routine_preparation_token + ) + runtime._record_job_cache_event("bundle", hit=True) + if request_source == "initial": + with self._job_build_scheduler_lock: + self.initial_job_prepared_work_counts["cache_hit"] += 1 + assert cached is not None + return runtime._bind_cached_bundle_to_artifacts(cached, artifacts) + if runtime._job_bundle_mode(mode) != resolved_mode: + if routine_preparation_token is not None: + runtime._finish_routine_job_build_preparation( + routine_preparation_token + ) + continue + runtime._record_job_cache_event("bundle", hit=False) + try: + request = runtime._new_job_build_request( + artifacts, + worker, + mode=resolved_mode, + payout_state_generation=payout_state_generation, + cache_key=key, + payout_ledger_artifact=payout_artifact, + idle_retarget=idle_retarget, + publication_critical=publication_critical, + request_source=request_source, + priority_requested_monotonic=( + priority_requested_monotonic + ), + preparation_cancellation=preparation_cancellation, + ) + # Preserve the historical readiness handoff without holding a + # lock across construction: only admission and the final mode + # re-selection are serialized here. + with self._job_build_lock: + with self._job_build_scheduler_lock: + if routine_preparation_token is not None: + runtime._finish_routine_job_build_preparation( + routine_preparation_token + ) + routine_preparation_token = None + request.cancellation.raise_if_cancelled( + "scheduler admission" + ) + if runtime._job_bundle_mode(mode) != resolved_mode: + request.cancellation.cancel( + "worker mode superseded" + ) + continue + if cancelled is not None and cancelled(): + raise JobBuildWaiterCancelled( + "job bundle request was cancelled before preparation" + ) + promise = runtime._request_job_build(request) + wait_deadline = time.monotonic() + max( + 0.001, + float(runtime.job_build_timeout_seconds) + + float(runtime.job_build_cancel_grace_seconds) + + 1.0, + ) + while True: + if cancelled is not None and cancelled(): + raise JobBuildWaiterCancelled( + "job bundle waiter was cancelled during preparation" + ) + try: + built = _await_job_build_promise( + promise, + min( + 0.1, + max(0.001, wait_deadline - time.monotonic()), + ), + ) + break + except TimeoutError: + if time.monotonic() >= wait_deadline: + raise + except TimeoutError as exc: + request.cancellation.cancel("timeout") + runtime._schedule_tip_refresh_retry() + raise JobBuildCancelled( + "job build timed out; immediate retry scheduled" + ) from exc + except JobBuildCancelled: + if routine_preparation_token is not None: + runtime._finish_routine_job_build_preparation( + routine_preparation_token + ) + runtime._schedule_tip_refresh_retry() + if not retry_superseded: + raise + with runtime.lock: + buildable = runtime._artifacts_buildable_locked(artifacts) + if not buildable: + raise + with self._job_cache_lock: + current = runtime._template_artifacts + payout_current = ( + payout_state_generation == runtime._payout_state_generation + ) + if current is artifacts and payout_current: + continue + if current is artifacts: + continue + raise + built = runtime._bind_cached_bundle_to_artifacts(built, artifacts) + if not runtime._cache_job_bundle_if_current(built, artifacts): + with self._job_build_scheduler_lock: + self.job_build_scheduler_counts["obsolete_results"] += 1 + with runtime.lock: + buildable = runtime._artifacts_buildable_locked(artifacts) + if not buildable: + with runtime._tip_refresh_metrics_lock: + runtime.tip_refresh_superseded_results += 1 + raise JobBuildSuperseded( + "observed tip changed before cache publication" + ) + with self._job_cache_lock: + payout_current = ( + built.payout_state_generation + == runtime._payout_state_generation + and ( + built.collection_only + or ( + built.build_key is not None + and built.build_key.payout_append_invalidation_epoch + == runtime._payout_ledger_append_invalidation_epoch + ) + ) + ) + with runtime.lock: + published_snapshot = getattr( + runtime, + "tip_template_snapshot", + None, + ) + published_artifacts = ( + published_snapshot.template_artifacts + if published_snapshot is not None + else None + ) + if ( + payout_current + and built.template is artifacts.template + and built.template_generation == artifacts.generation + and ( + not retry_superseded + or published_artifacts is artifacts + ) + ): + # Snapshot-owned work may outlive an unrelated global + # cache fill. Return it only to its refresh validator or + # retained collection delivery; never retain it globally. + if built.prepared_ledger_artifact is not None: + runtime._install_payout_ledger_artifact( + built.prepared_ledger_artifact + ) + return built + if retry_superseded: + with self._job_cache_lock: + current = runtime._template_artifacts + if current is artifacts: + continue + raise JobBuildSuperseded( + "job build key changed before cache publication" + ) + return built + + def _share_window_serialization_for_artifact( + self, + payout_artifact: Any, + shares: list[dict[str, object]], + ) -> _ShareWindowSerialization: + """Cached digest and builder fragments for the artifact's share window. + + Single-slot cache keyed by the canonical content digest and share + count. A debounced payout generation can retag an + unchanged window, so generation identity would unnecessarily rebuild + and re-spool the same multi-megabyte compact payload. The compute + deliberately takes no cancellation checkpoints -- the result outlives + any one requester and is served to whichever build wins next. + """ + self._runtime._ensure_job_cache_state() + # The digest is computed under the slot lock on purpose: the two + # bounded build flights can request the same window concurrently, and + # letting both walk the share tree would duplicate exactly the + # GIL-heavy pass this cache removes. The loser waits briefly and + # reuses the winner's entry; a different-key waiter is serialized + # behind at most one O(window) pass. + with self._share_window_serialization_lock: + cached = self._share_window_serialization + if ( + payout_artifact.share_snapshot_sha256 is None + and cached is not None + and cached._source_artifact is payout_artifact + and cached.share_count == len(shares) + ): + return cached + share_snapshot_sha256 = ( + payout_artifact.share_snapshot_sha256 + or self._canonical_json_sha256(share_array_json_view(shares)) + ) + # The compact payload is a pure function of the shares (see the + # class docstring), so the digest and count fully identify it. + # The live-difficulty window weight was deliberately dropped + # from this key: ASERT retargets every block, so a weight + # component rotated the key -- and re-spooled an identical + # multi-megabyte payload -- per block. + key = (share_snapshot_sha256, len(shares)) + if ( + cached is not None + and cached.key == key + and cached.share_count == len(shares) + ): + cached._source_artifact = payout_artifact + return cached + serialization = _ShareWindowSerialization( + key=key, + share_count=len(shares), + share_snapshot_sha256=share_snapshot_sha256, + _source_artifact=payout_artifact, + _spool_factory=self._spool_factory, + ) + self._share_window_serialization = serialization + if cached is not None: + # Generation rotation retires the replaced spool; a build still + # holding a lease keeps the descriptor alive until it releases. + cached.retire_spool() + return serialization + + def retire_share_window_spool(self) -> None: + """Release the cached share-window spool during shutdown. + + Runs after the build executors quiesce, so any lease still held by a + draining transfer defers the close to its own release. + """ + self._runtime._ensure_job_cache_state() + with self._share_window_serialization_lock: + serialization = self._share_window_serialization + if serialization is not None: + serialization.retire_spool() + + def build_shared_job_bundle( + self, + artifacts: CachedTemplateArtifacts, + worker: WorkerIdentityPort | None = None, + *, + mode: str | None = None, + payout_state_generation: int | None = None, + payout_artifact: Any = None, + key: tuple[object, ...] | None = None, + build_request: JobBuildRequest | None = None, + ) -> CachedJobBundle: + runtime = self._runtime + phases = runtime._job_build_phases() + resolved_mode = runtime._job_bundle_mode(mode) + if resolved_mode == "collection" and worker is None: + raise CollectionIdentityUnavailable( + "collection-mode worker identity is temporarily unavailable" + ) + with self._job_cache_lock: + publication_blocked = runtime._payout_state_publication_blocked + if payout_state_generation is None: + payout_state_generation = runtime._payout_state_generation + if publication_blocked: + raise PayoutStatePublicationBlocked( + "payout state invalidation is pending publication" + ) + if key is None: + key = runtime._job_bundle_key( + artifacts, + mode=resolved_mode, + payout_state_generation=payout_state_generation, + payout_artifact_generation=( + payout_artifact.generation + if payout_artifact is not None + else 0 + ), + worker=worker, + ) + if build_request is None: + build_request = runtime._new_job_build_request( + artifacts, + worker, + mode=resolved_mode, + payout_state_generation=payout_state_generation, + cache_key=key, + payout_ledger_artifact=payout_artifact, + ) + else: + payout_artifact = build_request.payout_ledger_artifact + cancellation = build_request.cancellation + runtime._job_build_checkpoint("ledger_snapshot", cancellation) + template_value = json.loads(build_request.template_json) + if not isinstance(template_value, dict): + raise RuntimeError("immutable job template is not an object") + template: dict[str, Any] = template_value + issued_at_ms = build_request.key.issued_at_ms + started = time.monotonic() + snapshot_window_weight = ( + PRISM_REWARD_WINDOW_MULTIPLIER + * PRISM_SNAPSHOT_WINDOW_MARGIN + * int(build_request.key.network_difficulty) + ) + prepared_ledger_artifact: Any = None + snapshot_accepted_count: int | None = None + inflight_scan_anchor_token: int | None = None + if payout_artifact is not None: + # The reuse decision was made at request preparation. Nothing + # that happens to the ARMED slot afterwards -- a fresher window + # installing (generation re-key), a same-window anchor refresh + # swapping the instance, or the artifact simply aging out of new + # reuse decisions -- invalidates the copy this build already + # selected: its window stays audit-reproducible at its declared + # anchor. Scrapping admitted work on any of those (as the first + # anchor-scoped deploy did on the generation re-key) turns + # routine churn into a rebuild storm. Only two fences gate + # completion: the payout generation and the published balances + # this artifact binds. + with self._job_cache_lock: + current_payout_state_generation = runtime._payout_state_generation + published_artifact = runtime._published_payout_state.artifact + current_append_invalidation_epoch = ( + runtime._payout_ledger_append_invalidation_epoch + ) + if current_payout_state_generation != payout_state_generation: + raise JobBuildSuperseded( + "payout generation changed before construction" + ) + if ( + payout_artifact.append_invalidation_epoch + != current_append_invalidation_epoch + ): + raise JobBuildSuperseded( + "payout window invalidated before construction" + ) + if published_artifact is None: + try: + published_artifact = runtime._current_payout_state_artifact() + except Exception as exc: + raise JobBuildSuperseded( + "published payout state unavailable before construction" + ) from exc + artifact_balances_sha256 = ( + payout_artifact.prior_balances_sha256 + or self._canonical_json_sha256(payout_artifact.prior_balances) + ) + if ( + artifact_balances_sha256 + != published_artifact.prior_balances_sha256 + ): + raise JobBuildSuperseded( + "precomputed payout artifact changed before construction" + ) + prior_balances = list(payout_artifact.prior_balances) + if ( + artifact_balances_sha256 + != build_request.key.payout_artifact_sha256 + ): + raise JobBuildSuperseded( + "precomputed payout artifact does not match payout generation" + ) + # A daemon-mirror sequence is kept as-is rather than copied into + # a list: it parses its dicts lazily, and the routine build path + # must not materialize dicts it never consumes. The rare + # consumers that need real dicts (the durable candidate intent, + # the canonical audit build) coerce at their own call sites. + # Every other artifact sequence keeps the historical list copy. + artifact_shares_json = payout_artifact.shares_json + shares = ( + artifact_shares_json + if isinstance(artifact_shares_json, DaemonShareJsonSequence) + else list(artifact_shares_json) + ) + # The artifact binds the key above; the balances actually used may + # still be an accepted parent's prospective carry state. + prior_balances = runtime._prior_balances_for_job_parent( + str(template["previousblockhash"]), + parent_height=int(template["height"]) - 1, + fallback_balances=prior_balances, + ) + else: + with runtime._payout_state_prepare_lock: + with self._job_cache_lock: + published_artifact = runtime._published_payout_state.artifact + if runtime._payout_state_publication_blocked: + raise PayoutStatePublicationBlocked( + "payout state invalidation is pending publication" + ) + if ( + payout_state_generation != runtime._payout_state_generation + or build_request.key.payout_append_invalidation_epoch + != runtime._payout_ledger_append_invalidation_epoch + or published_artifact is None + or published_artifact.prior_balances_sha256 + != build_request.key.payout_artifact_sha256 + ): + raise JobBuildSuperseded( + "payout generation changed before ledger snapshot" + ) + if resolved_mode == "ready": + try: + accepted_now, _ = runtime.accepted_share_stats() + except Exception: + # Artifact seeding is speculative; snapshot errors + # stay owned by the bundle build itself. Without an + # observed count the seed below is skipped rather + # than armed with a fabricated one. + accepted_now = None + if accepted_now is not None: + # Informational only: the window is scoped by the + # frozen anchor (shares committing during the read + # land above it and belong to the next window), so + # no bracket around the read is needed. + snapshot_accepted_count = int(accepted_now) + if resolved_mode == "ready": + # Publish the read's frozen anchor: a late-visible + # append committing mid-read predates it without + # holding the pending floor, and this read's database + # snapshot may already exclude the row. The recorded + # invalidation epoch keeps the resulting bundle and + # its seeded artifact from arming or serving. The + # exposure must outlive this read: nothing else exposes + # this window's anchor until the seeded artifact passes + # its install fence, so an append committing during + # bundle construction or cache publication would + # otherwise advance no epoch and the seed would arm a + # window that delta reads can never rediscover the row + # from. The token therefore rides the seeded artifact + # (retired by its install fence); a build that raises, + # or seeds nothing, retires it on the spot. + inflight_scan_anchor_token = ( + runtime._expose_inflight_scan_anchor(issued_at_ms) + ) + # Mirrored onto the request so the executor wrapper can + # retire the exposure when this build dies anywhere past + # the read (checkpoint cancellation, manifest assembly, + # the publication fence). + build_request.inflight_scan_anchor_token = ( + inflight_scan_anchor_token + ) + try: + records = runtime.ledger.snapshot_at_job_issue( + issued_at_ms, + window_weight=snapshot_window_weight, + ) + except BaseException: + runtime._retire_inflight_scan_anchor( + inflight_scan_anchor_token + ) + raise + else: + records = [] + # An accepted parent's prospective carry state supersedes the + # published artifact for children built on that parent; the + # published balances remain the fallback for ordinary tips. + prior_balances = runtime._prior_balances_for_job_parent( + str(template["previousblockhash"]), + parent_height=int(template["height"]) - 1, + fallback_balances=build_request.payout_artifact.prior_balances(), + ) + runtime._job_build_checkpoint("ledger_snapshot_complete", cancellation) + shares = [] + for index, record in enumerate(records): + if index % 256 == 0: + runtime._job_build_checkpoint( + "ledger_snapshot_conversion", + cancellation, + ) + shares.append(record.to_prism_json()) + # A reused artifact carries shares snapshotted at its own earlier + # anchor. The bundle must declare that anchor: replaying the audit + # window at this job's fresher anchor could include a share that was + # already durable at artifact build time but stamped after the + # artifact's clamped anchor, and the artifact's share set excludes it + # by construction. + bundle_anchor_ms = ( + payout_artifact.snapshot_anchor_ms + if payout_artifact is not None + and payout_artifact.snapshot_anchor_ms is not None + else issued_at_ms + ) + ledger_elapsed = time.monotonic() - started + phases["ledger"] = phases.get("ledger", 0.0) + ledger_elapsed + if resolved_mode == "ready": + runtime._observe_tip_refresh_build_phase( + "ledger_snapshot", + ledger_elapsed, + ) + share_serialization: _ShareWindowSerialization | None = None + if resolved_mode == "ready" and payout_artifact is not None: + share_serialization = runtime._share_window_serialization_for_artifact( + payout_artifact, + shares, + ) + share_snapshot_sha256 = share_serialization.share_snapshot_sha256 + else: + # A plain list digests through the streaming view so the hook's + # fallback never serializes the whole window in one call. + share_snapshot_sha256 = self._canonical_json_sha256( + share_array_json_view(shares) + ) + if ( + payout_artifact is None + and snapshot_accepted_count is not None + and shares + and runtime._payout_artifact_reuse_active() + ): + # This build already paid the full ledger read; carry the window + # it produced so cache publication can arm it for builds arriving + # while its anchor stays inside the staleness bound. The artifact + # carries the published payout-state balances, not this bundle's + # possibly parent-adjusted view: reuse re-applies the parent + # override itself and the reuse fence hashes the artifact + # balances against the published payout artifact. Skipped + # entirely under the kill-switch: nothing may arm while reuse + # is disabled. + assert self._payout_ledger_artifact_type is not None, ( + "job bundle service requires the payout-ledger artifact type " + "to seed reusable windows" + ) + seed_prior_balances = tuple( + build_request.payout_artifact.prior_balances() + ) + prepared_ledger_artifact = self._payout_ledger_artifact_type( + generation=0, + payout_state_generation=payout_state_generation, + network_difficulty=int(build_request.key.network_difficulty), + accepted_share_count=snapshot_accepted_count, + shares_json=tuple(shares), + prior_balances=seed_prior_balances, + prepared_monotonic=time.monotonic(), + snapshot_anchor_ms=issued_at_ms, + share_snapshot_sha256=share_snapshot_sha256, + prior_balances_sha256=self._canonical_json_sha256( + seed_prior_balances + ), + append_invalidation_epoch=( + build_request.key.payout_append_invalidation_epoch + ), + inflight_scan_anchor_token=inflight_scan_anchor_token, + ) + final_build_key = dataclass_replace( + build_request.key, + share_snapshot_sha256=share_snapshot_sha256, + ) + runtime._job_build_checkpoint("payout_derivation", cancellation) + started = time.monotonic() + placeholder_suffix_hex = final_build_key.coinbase_suffix_hex + collection_identity: tuple[str, str] | None = None + previous_metrics_scope = bool( + getattr(self._job_build_phase_local, "tip_refresh_metrics", False) + ) + self._job_build_phase_local.tip_refresh_metrics = resolved_mode == "ready" + try: + if resolved_mode == "ready": + if not shares: + raise RuntimeError( + "ready-pool ledger snapshot contained no payout shares" + ) + runtime._job_build_checkpoint("ctv_manifest", cancellation) + runtime._job_build_checkpoint("signing_verification", cancellation) + bundle = runtime.build_audit_bundle( + shares=shares, + found_block={ + "block_height": int(template["height"]), + "coinbase_value_sats": int(template["coinbasevalue"]), + "network_difficulty": artifacts.network_difficulty, + "anchor_job_issued_at_ms": bundle_anchor_ms, + }, + prior_balances=prior_balances, + coinbase_script_sig_suffix_hex=placeholder_suffix_hex, + witness_merkle_leaves_hex=list( + build_request.witness_merkle_leaves_hex + ), + ctv_fee_parent_hash=str(template["previousblockhash"]), + summary_only=True, + payout_policy=json.loads(build_request.payout_policy_json), + ctv_settlement=( + json.loads(build_request.ctv_settlement_json) + if build_request.ctv_settlement_json is not None + else None + ), + cancellation=cancellation, + share_serialization=share_serialization, + append_invalidation_epoch=( + payout_artifact.append_invalidation_epoch + if payout_artifact is not None + else None + ), + ) + collection_only = False + else: + assert worker is not None + runtime._job_build_checkpoint("ctv_manifest", cancellation) + runtime._job_build_checkpoint("signing_verification", cancellation) + bundle = runtime.build_collection_bundle( + template=template, + transaction_hexes=build_request.transaction_hexes, + worker=worker, + network_difficulty=final_build_key.network_difficulty, + issued_at_ms=issued_at_ms, + suffix_hex=placeholder_suffix_hex, + summary_only=True, + payout_policy=json.loads(build_request.payout_policy_json), + ctv_settlement=( + json.loads(build_request.ctv_settlement_json) + if build_request.ctv_settlement_json is not None + else None + ), + cancellation=cancellation, + ) + shares = [] + collection_only = True + collection_identity = runtime._collection_bundle_identity(worker) + finally: + self._job_build_phase_local.tip_refresh_metrics = previous_metrics_scope + manifest = bundle["signed_coinbase_manifest"]["manifest"] + prospective_prior_balances: ( + tuple[tuple[str, str, str, int], ...] | None + ) = None + payout_policy_manifest = bundle.get("payout_policy_manifest") + if isinstance(payout_policy_manifest, dict) and isinstance( + payout_policy_manifest.get("accounts"), + list, + ): + prospective_prior_balances = runtime._serialize_prior_balance_preview( + runtime._accepted_block_payout_preview_from_bundle( + bundle, + prior_balances=prior_balances, + ) + ) + runtime._job_build_checkpoint("bundle_assembly", cancellation) + assembly_started = time.monotonic() + base_job = direct_stratum.make_job_from_builder_manifest( + job_id="prism-template-base", + template=template, + manifest=manifest, + extranonce1_hex=PRISM_JOB_EXTRANONCE1_PLACEHOLDER_HEX, + extranonce2_size=runtime.extranonce2_size, + desired_share_difficulty=runtime.share_difficulty, + clean_jobs=True, + transaction_hexes=build_request.transaction_hexes, + ) + phases["assembly"] = phases.get("assembly", 0.0) + ( + time.monotonic() - assembly_started + ) + runtime._job_build_checkpoint("serialization", cancellation) + runtime._job_build_checkpoint("bundle_publication", cancellation) + if resolved_mode == "ready": + with self._job_cache_lock: + if ( + final_build_key.payout_append_invalidation_epoch + != runtime._payout_ledger_append_invalidation_epoch + ): + raise JobBuildSuperseded( + "payout window invalidated before bundle publication" + ) + if ( + inflight_scan_anchor_token is not None + and prepared_ledger_artifact is None + ): + # The fence above just proved, under this same lock hold, + # that no append predated this build's window. Seeded + # builds keep their exposure until the artifact's + # install fence settles; with no seed, nothing else + # exposes this window's anchor once the walk exposure + # retires, yet jobs stamped from this bundle keep + # serving the window until they retire. Hand the anchor + # to the published-window watermark under this same lock + # hold, so a replay-shaped append committing after this + # return still predates a live anchor and advances the + # epoch those jobs' landing fences check. + runtime._publish_seedless_job_window_anchor_locked( + issued_at_ms + ) + runtime._payout_window_inflight_scan_anchors.pop( + inflight_scan_anchor_token, + None, + ) + inflight_scan_anchor_token = None + build_request.inflight_scan_anchor_token = None + phases["bundle"] = phases.get("bundle", 0.0) + (time.monotonic() - started) + return CachedJobBundle( + key=key, + # Construction consumed the detached canonical template above; + # publication retains the exact snapshot-owned object identity used + # by the existing validation token. + template=artifacts.template, + template_fingerprint=artifacts.fingerprint, + # Only this manifest is needed to bind later clock-only template + # observations. Retaining the returned logical bundle duplicated + # the entire shares tree already held in shares_json. + coinbase_manifest=manifest, + shares_json=shares, + prior_balances=prior_balances, + found_block=bundle["found_block"], + collection_only=collection_only, + issued_at_ms=issued_at_ms, + base_job=base_job, + built_monotonic=time.monotonic(), + template_generation=artifacts.generation, + payout_state_generation=payout_state_generation, + payout_artifact_generation=( + payout_artifact.generation if payout_artifact is not None else 0 + ), + collection_identity=collection_identity, + prospective_prior_balances=prospective_prior_balances, + build_key=final_build_key, + prepared_ledger_artifact=prepared_ledger_artifact, + ) + + def build_collection_bundle( + self, + *, + template: dict[str, Any], + transaction_hexes: tuple[str, ...], + worker: WorkerIdentityPort, + network_difficulty: int, + issued_at_ms: int, + suffix_hex: str, + summary_only: bool = False, + payout_policy: dict[str, object] | None = None, + ctv_settlement: dict[str, object] | None = None, + cancellation: JobBuildCancellation | None = None, + ) -> dict[str, Any]: + runtime = self._runtime + if cancellation is not None: + cancellation.raise_if_cancelled("collection payout derivation") + share = { + "share_seq": 1, + "share_id": "bootstrap-share", + "miner_id": worker.payout_address, + "order_key": worker.payout_address, + "p2mr_program_hex": worker.p2mr_program_hex, + "share_difficulty": network_difficulty, + "network_difficulty": network_difficulty, + "template_height": int(template["height"]) - 1, + "job_id": "bootstrap-job", + "job_issued_at_ms": issued_at_ms, + "accepted_at_ms": issued_at_ms, + "ntime": int(template["curtime"]), + } + return runtime.build_audit_bundle( + shares=[share], + found_block={ + "block_height": int(template["height"]), + "coinbase_value_sats": int(template["coinbasevalue"]), + "network_difficulty": network_difficulty, + "anchor_job_issued_at_ms": issued_at_ms, + }, + prior_balances=[], + coinbase_script_sig_suffix_hex=suffix_hex, + witness_merkle_leaves_hex=direct_stratum.witness_merkle_leaves_hex(transaction_hexes), + ctv_fee_parent_hash=str(template["previousblockhash"]), + summary_only=summary_only, + payout_policy=payout_policy, + ctv_settlement=ctv_settlement, + cancellation=cancellation, + ) + + +__all__ = [ + "CachedJobBundle", + "CollectionIdentityUnavailable", + "JobBuildAdmissionDeadlineExceeded", + "JobBuildCancellation", + "JobBuildCancelled", + "JobBuildFlight", + "JobBuildKey", + "JobBuildRequest", + "JobBuildSuperseded", + "JobBuildWaiterCancelled", + "JobBundleBuildControl", + "JobBundleBuildSuperseded", + "JobBundleRuntime", + "JobBundleService", + "MAX_PRISM_JOB_BUNDLE_CACHE_ENTRIES", + "PRISM_INITIAL_JOB_SUBSCRIBE_POLL_SECONDS", + "PRISM_JOB_BUILD_ORPHAN_SWEEP_GRACE_SECONDS", + "PRISM_JOB_BUILD_PHASES", + "PRISM_JOB_BUILD_SECONDS_BUCKETS", + "PRISM_JOB_CACHE_KINDS", + "PRISM_JOB_EXTRANONCE1_PLACEHOLDER_HEX", + "PRISM_REWARD_WINDOW_MULTIPLIER", + "PRISM_SNAPSHOT_WINDOW_MARGIN", + "WorkerIdentityPort", + "canonical_json_sha256", + "canonical_json_text", + "now_ms", +] diff --git a/lab/prism/job_delivery.py b/lab/prism/job_delivery.py new file mode 100644 index 000000000..ce7861d98 --- /dev/null +++ b/lab/prism/job_delivery.py @@ -0,0 +1,3042 @@ +#!/usr/bin/env python3 +"""Per-session PRISM job delivery, initial jobs, and retained-job ownership. + +The service owns the S2 domain: job stamping/registration, initial-job +scheduling and its bounded executor, prepared-job delivery admission (epoch +admission floor and post-write proof), difficulty adjacency, and all +retained/evicted-job state including the #100 cross-connection disconnected +retention authority. + +It never imports ``prism_coordinator``. Session membership, Stratum socket +writes, payout delivery gates, job-bundle preparation, and live configuration +attributes are reached through the :class:`JobDeliveryRuntime` typed port, +resolved at call time so the historical coordinator monkeypatch seams +(including the instance-level facade patches used by the current test suite) +keep intercepting exactly as before the extraction. ``ClientState`` remains +coordinator-owned at this layer (it moves with the session owner); client +objects are used strictly through duck typing, exactly as the current code +does. +""" + +from __future__ import annotations + +from collections import OrderedDict +from collections.abc import Mapping +from concurrent.futures import Future +from contextlib import contextmanager, nullcontext +from dataclasses import dataclass, field, replace as dataclass_replace +from decimal import Decimal +import json +import threading +import time +import traceback +from typing import Any, Callable, Iterator, Protocol + +from lab.auxpow import vardiff +from lab.prism import direct_stratum +from lab.prism.bounded_executor import _BoundedPriorityExecutor, _DeliveryQueueFull +from lab.prism.future_callbacks import add_releasing_done_callback +from lab.prism.coordinator_config import ( + DEFAULT_PRISM_DISCONNECTED_JOB_RETENTION, + DEFAULT_PRISM_INITIAL_JOB_MAX_WORKERS, # noqa: F401 - compatibility re-export + DEFAULT_PRISM_SAME_TIP_JOB_RETENTION_PER_CONNECTION, + DEFAULT_PRISM_SAME_TIP_JOB_RETENTION_SECONDS, + DEFAULT_PRISM_STALE_GRACE_SECONDS, +) +from lab.prism.job_bundle import ( + CachedJobBundle, + JobBuildWaiterCancelled, + PRISM_JOB_BUILD_PHASES, # noqa: F401 - owner-shared phase registry +) +# Imported from its owning module rather than the template_artifacts +# compatibility surface: this type is new, so it has no historical import +# path to preserve there. +from lab.prism.payout_state import AcceptedParentPayoutPreviewPending +from lab.prism.template_artifacts import ( + CachedTemplateArtifacts, + QbitTipTemplateSnapshot, + TemplateRefreshBlocked, + TemplateRefreshSuperseded, + qbit_template_fingerprint, +) +from lab.prism.tip_refresh import ( + FanoutCancellation, + RefreshResult, + TipRefreshValidationToken, + _TipRefreshTrustBlocked, +) + + +DEFAULT_PRISM_EVICTED_JOB_PRUNE_INTERVAL_SECONDS = 1.0 +# Owner-local copy of the still-coordinator-visible constant; a compatibility +# duplicate by design. This module is the coordinator's compatibility import +# source for the constant. +PRISM_TIP_REFRESH_ADMISSION_POLL_SECONDS = 0.05 +MAX_ACTIVE_PRISM_JOBS_PER_CLIENT = 16 +# Never-served clients outrank the fleet's tip-refresh wave: a client with no +# active job is producing nothing until its first notify, while a client with +# stale-tip work keeps mining (stale-grace credits it) for the wave's duration. +PRISM_DELIVERY_PRIORITY_INITIAL = 0 +PRISM_DELIVERY_PRIORITY_NEW_TIP = 1 +PRISM_DELIVERY_PRIORITY_SAME_TIP = 2 +PRISM_EVICTED_JOB_CLASSES = ("same_tip", "stale_grace") +PRISM_EVICTED_JOB_SUBMIT_OUTCOMES = ( + "accepted_same_tip", + "credited_stale_grace", + "accepted_same_tip_cross_connection", +) +PRISM_EVICTED_JOB_CAPACITY_SCOPES = ("connection", "disconnected") +# Credit policies recorded on accepted ledger rows. Normal shares carry no +# policy; a policy marks a share that was credited by an explicit pool rule +# (documented in docs/prism-rejections.md) so audits can distinguish them. +PRISM_CREDIT_POLICY_STALE_GRACE = "stale-grace" + + +@dataclass(frozen=True) +class PrismJobContext: + job: direct_stratum.DirectQbitStratumJob + template: dict[str, Any] + shares_json: list[dict[str, object]] + prior_balances: list[dict[str, object]] + found_block: dict[str, object] + share_weight: int + collection_only: bool + worker: WorkerIdentity + issued_at_ms: int + template_fingerprint: str | None = None + template_generation: int = 0 + payout_state_generation: int = 0 + prospective_prior_balances: tuple[tuple[str, str, str, int], ...] | None = None + payout_artifact_generation: int = 0 + payout_append_invalidation_epoch: int = 0 + # Digest of the published prior-balance snapshot this job's payouts are + # keyed to. A periodic self-check repair replaces the published snapshot + # in place -- the payout generation and append epoch both survive the + # swap -- so this digest is the only fence that identifies an active job + # still carrying refuted balances. + payout_artifact_sha256: str | None = None + connection_id: int = 0 + authorization_generation: int = 0 + difficulty_generation: int = 0 + tip_refresh_epoch_sequence: int = 0 + # Version-rolling mask negotiated on the connection this job was + # delivered to. Cross-connection resumes validate in-flight version bits + # against this mask, not the replacement connection's (often still 0). + version_mask: int = 0 + + +@dataclass(frozen=True) +class EvictedJobEntry: + context: PrismJobContext + connection_id: int + evicted_monotonic: float + previousblockhash: str + client: ClientState | None = None + + +@dataclass(eq=False) +class PendingInitialJob: + client: ClientState + authorization_generation: int + worker: WorkerIdentity + requested_monotonic: float + deadline_monotonic: float | None + connection_id: int | None = None + difficulty_generation: int | None = None + cancelled: threading.Event = field(default_factory=threading.Event) + future: Future[bool] | None = None + predecessor: Future[bool] | None = None + + +@dataclass(frozen=True) +class InitialJobAdmissionSnapshot: + """One internally consistent copy of the first-job admission domain. + + Metrics and health readers take this instead of reaching into the live + admission state, so an observability refresh never has to hold the + coordinator lock and the admission lock at the same time (#159). + """ + + pending: dict[Any, PendingInitialJob] + queue_rejections: int + timeout_disconnects: int + cancelled: int + coalesced: int + queue_capacity_reclaimed: int + last_delivery_monotonic: float | None + + +class PendingInitialJobsView(Mapping): + """Read-only, lock-protected view of the first-job admission map. + + The mutable mapping is private to :class:`JobDeliveryService` and is only + ever read or written under the admission lock. Every compatibility reader + -- the session owner's reauthorization capacity pre-check, health and + metrics, focused tests -- reaches the queue through this view, so an + external ``len()``, ``in``, lookup, or iteration is itself taken under the + owning lock. Such a read can therefore never observe a replacement or a + cancellation half applied, and can never mutate the queue. + + ``len`` and ``in`` stay O(1) under the lock rather than copying the map: + the reauthorization pre-check runs once per re-authorize, and a snapshot + copy there would put O(N) work back inside a lock a reconnect herd + contends on -- the exact shape #159 exists to remove. Bulk materialization + (``copy``, ``items``, ``values``, ``keys``, iteration) takes one locked + copy, so it cannot tear against concurrent admission. + + These reads are exact at the instant they are taken, but they are not a + reservation: the authoritative capacity decision is always re-taken under + the same lock in :meth:`JobDeliveryService.schedule_initial_job`. + + The view is strictly immutable: it exposes no ``__setitem__``, + ``__delitem__``, ``pop``, ``clear``, ``update``, or ``setdefault``, so no + caller outside this service can reach admission state to write it. + Embedders and focused tests that need to install a request seed the whole + mapping through the ``pending_initial_jobs`` setter, which adopts its + contents under the admission lock. + """ + + __slots__ = ("_service",) + + def __init__(self, service: "JobDeliveryService") -> None: + self._service = service + + def copy(self) -> dict[Any, PendingInitialJob]: + """Return one locked, internally consistent copy of the queue.""" + service = self._service + with service._initial_job_admission_lock: + return dict(service._pending_initial_jobs) + + def __getitem__(self, key: Any) -> PendingInitialJob: + service = self._service + with service._initial_job_admission_lock: + return service._pending_initial_jobs[key] + + def __contains__(self, key: object) -> bool: + service = self._service + with service._initial_job_admission_lock: + return key in service._pending_initial_jobs + + def __len__(self) -> int: + service = self._service + with service._initial_job_admission_lock: + return len(service._pending_initial_jobs) + + def get( + self, + key: Any, + default: Any = None, + ) -> PendingInitialJob | None: + service = self._service + with service._initial_job_admission_lock: + return service._pending_initial_jobs.get(key, default) + + def __iter__(self) -> Iterator[Any]: + return iter(self.copy()) + + def keys(self) -> Any: + return self.copy().keys() + + def values(self) -> Any: + return self.copy().values() + + def items(self) -> Any: + return self.copy().items() + + def __eq__(self, other: object) -> bool: + return self.copy() == other + + def __ne__(self, other: object) -> bool: + return not self.__eq__(other) + + __hash__ = None # type: ignore[assignment] + + def __repr__(self) -> str: + return f"{type(self).__name__}({self.copy()!r})" + + +class JobBuildFailed(RuntimeError): + """Internal signal used to distinguish a skipped build from a no-op.""" + + +class JobDeliveryRuntime(Protocol): + """Typed port over the coordinator, resolved at call time. + + Every member is looked up on the live coordinator object when used, so + instance monkeypatches (``server.maybe_send_job = ...`` and friends) and + coordinator-owned live configuration attributes keep working exactly as + before the extraction. Owner facades route back into this service; that + round trip is deliberate: it is what preserves the current white-box + patch surface until later stack layers repoint those tests. The legacy + S2 field names also resolve here -- coordinator class descriptors route + them to this service's single mutable copy -- and the R1 published-tip + fields resolve through the tip-refresh owner's descriptors the same way. + """ + + # Cross-domain objects and live configuration attributes. + _job_cache_lock: Any + _payout_ledger_append_invalidation_epoch: Any + _payout_state_delivery_gate: Any + _payout_state_generation: Any + _payout_state_publication_blocked: Any + _pool_ready_latched: Any + _published_payout_state: Any + _template_artifacts: Any + _tip_refresh_epoch_payout_generation: Any + _tip_refresh_epoch_sequence: Any + _tip_refresh_epoch_tip_hash: Any + clients: Any + current_tip_first_seen: Any + current_tip_parent: Any + disconnected_job_retention: Any + hot_path_log_enabled: Any + initial_job_max_workers: Any + job_build_failure_count: Any + lock: Any + reorg_reconciler_enabled: Any + rpc: Any + same_tip_job_retention_per_connection: Any + same_tip_job_retention_seconds: Any + stale_grace_seconds: Any + stop_event: Any + stratum_initial_job_timeout_seconds: Any + stratum_max_pending_initial_jobs: Any + tip_refresh_epoch_fanout: Any + tip_template_snapshot: Any + vardiff_config: Any + + def _client_vardiff_lock(self, *args: Any, **kwargs: Any) -> Any: ... + + def _collection_bundle_identity(self, *args: Any, **kwargs: Any) -> Any: ... + + def _current_published_tip_hash_locked(self, *args: Any, **kwargs: Any) -> Any: ... + + def _ensure_evicted_job_state(self, *args: Any, **kwargs: Any) -> Any: ... + + def _ensure_initial_job_state(self, *args: Any, **kwargs: Any) -> Any: ... + + def _ensure_job_cache_state(self, *args: Any, **kwargs: Any) -> Any: ... + + def _idle_bundle_current_locked(self, *args: Any, **kwargs: Any) -> Any: ... + + def _issuance_artifacts_current(self, *args: Any, **kwargs: Any) -> Any: ... + + def _job_build_phases(self, *args: Any, **kwargs: Any) -> Any: ... + + def _observe_payout_gate_admission(self, *args: Any, **kwargs: Any) -> Any: ... + + def _payout_delivery(self, *args: Any, **kwargs: Any) -> Any: ... + + def _progress_bundle_build_finished(self, *args: Any, **kwargs: Any) -> Any: ... + + def _progress_bundle_build_started(self, *args: Any, **kwargs: Any) -> Any: ... + + def _record_first_payout_delivery(self, *args: Any, **kwargs: Any) -> Any: ... + + def _record_progress_delivery(self, *args: Any, **kwargs: Any) -> Any: ... + + def _record_startup_phase_once(self, *args: Any, **kwargs: Any) -> Any: ... + + def _submit_delivery_task(self, *args: Any, **kwargs: Any) -> Any: ... + + def bury_evicted_job(self, *args: Any, **kwargs: Any) -> Any: ... + + def client_minimum_advertised_difficulty(self, *args: Any, **kwargs: Any) -> Any: ... + + def desired_client_share_difficulty(self, *args: Any, **kwargs: Any) -> Any: ... + + def difficulty_payload(self, *args: Any, **kwargs: Any) -> Any: ... + + def disconnect_client(self, *args: Any, **kwargs: Any) -> Any: ... + + def ensure_reorg_reconciled_for_current_tip(self, *args: Any, **kwargs: Any) -> Any: ... + + def job_issuance_template_artifacts(self, *args: Any, **kwargs: Any) -> Any: ... + + def job_payload(self, *args: Any, **kwargs: Any) -> Any: ... + + def observe_job_build_elapsed(self, *args: Any, **kwargs: Any) -> Any: ... + + def qbit_chain_view_untrusted(self, *args: Any, **kwargs: Any) -> Any: ... + + def send_difficulty(self, *args: Any, **kwargs: Any) -> Any: ... + + def send_job(self, *args: Any, **kwargs: Any) -> Any: ... + + def share_weight_for_worker(self, *args: Any, **kwargs: Any) -> Any: ... + + def shared_job_bundle(self, *args: Any, **kwargs: Any) -> Any: ... + + +class JobDeliveryService: + """Sole owner of S2 job delivery, initial jobs, and retained jobs.""" + + def __init__( + self, + runtime: JobDeliveryRuntime, + *, + stratum_error: type[BaseException], + ) -> None: + self._runtime = runtime + # The Stratum error type remains coordinator-owned until the session + # owner is extracted; it is injected so this leaf module never + # imports prism_coordinator. + self._stratum_error = stratum_error + self.jobs: dict[str, PrismJobContext] = {} + self.job_counter = 0 + # Private: every read and write of this mapping happens under the + # admission lock below. Callers outside this service reach it only + # through the read-only ``pending_initial_jobs`` view. + self._pending_initial_jobs: dict[Any, PendingInitialJob] = {} + # Sole owner of the first-job admission domain: the admission map + # itself, its exact capacity decision, request replacement/coalescing/ + # cancellation, and every admission counter and timestamp below. + # + # It is the innermost lock in the process. Nothing acquires another + # lock while holding it, and it is never held across a socket write, a + # disconnect, a job build, an executor shutdown wait, or a future + # cancellation (``Future.cancel`` runs done-callbacks inline, and this + # module's callback re-enters admission). Cancellation callbacks are + # therefore always invoked after the admission state is committed and + # the lock is released. Splitting this domain out of ``runtime.lock`` + # is what stops a reconnect herd's queue bookkeeping from convoying + # behind fleet-wide coordinator work (#159). + self._initial_job_admission_lock = threading.Lock() + self._pending_initial_jobs_view = PendingInitialJobsView(self) + self._initial_job_executor_lock = threading.Lock() + self._initial_job_executor: _BoundedPriorityExecutor | None = None + self._initial_job_executor_shutdown = False + self.initial_job_queue_rejection_count = 0 + self.initial_job_timeout_count = 0 + self.initial_job_cancelled_count = 0 + self.initial_job_coalesced_count = 0 + self.initial_job_queue_capacity_reclaimed_count = 0 + self.initial_job_sent_count = 0 + self.initial_job_failed_count = 0 + self.initial_job_superseded_count = 0 + self.initial_job_delivery_latency_seconds_sum = 0.0 + self.initial_job_delivery_latency_count = 0 + self.last_initial_job_delivery_monotonic: float | None = None + # Globally insertion ordered, with per-connection indexes for the + # independent TTL and capacity limits. Prior-tip entries never consume + # the same-tip cap while stale-grace still protects them. + self.evicted_job_graveyard: OrderedDict[str, EvictedJobEntry] = OrderedDict() + self.evicted_jobs_by_connection: dict[int, OrderedDict[str, None]] = {} + self.evicted_same_tip_by_connection: dict[int, OrderedDict[str, None]] = {} + self.evicted_same_tip_job_ids: OrderedDict[str, None] = OrderedDict() + self.evicted_job_index_tip_hash: str | None = None + self.evicted_job_next_prune_monotonic = 0.0 + self.evicted_job_expiration_counts = { + job_class: 0 for job_class in PRISM_EVICTED_JOB_CLASSES + } + self.evicted_job_capacity_eviction_counts = { + scope: 0 for scope in PRISM_EVICTED_JOB_CAPACITY_SCOPES + } + self.evicted_job_submit_counts = { + outcome: 0 for outcome in PRISM_EVICTED_JOB_SUBMIT_OUTCOMES + } + # Insertion-ordered ids of graveyard entries whose connection is + # gone (entry.client is None); bounds cross-connection retention + # independently of live connections' per-connection caps. + self._disconnected_evicted_job_ids: OrderedDict[str, None] = OrderedDict() + + # -- state adoption ---------------------------------------------------- + + @property + def pending_initial_jobs(self) -> PendingInitialJobsView: + """Read-only compatibility surface over the admission map. + + The legacy attribute name is coordinator-visible (and reaches the + session owner's reauthorization capacity pre-check through the + coordinator descriptor), so it must not hand out the mutable mapping: + that is what left a capacity read outside the lock that owns it. + """ + return self._pending_initial_jobs_view + + @pending_initial_jobs.setter + def pending_initial_jobs(self, value: Any) -> None: + """Adopt a mapping assigned through the legacy name. + + Construction and focused tests assign through this name; adopt the + contents into the private map instead of retaining the caller's + mutable object, which would put admission state back outside the lock. + """ + with self._initial_job_admission_lock: + self._pending_initial_jobs.clear() + if value: + self._pending_initial_jobs.update(value) + + def ensure_evicted_job_state(self) -> None: + """Ensure/adopt the retained-job index, converting legacy seeded state.""" + runtime = self._runtime + graveyard = getattr(self, "evicted_job_graveyard", None) + rebuild_indexes = False + if not isinstance(graveyard, OrderedDict): + converted: OrderedDict[str, EvictedJobEntry] = OrderedDict() + for job_id, entry in (graveyard or {}).items(): + if isinstance(entry, EvictedJobEntry): + converted[job_id] = entry + continue + context, connection_id, evicted_monotonic = entry + client = next( + ( + candidate + for candidate in getattr(runtime, "clients", ()) + if candidate.connection_id == connection_id + ), + None, + ) + converted[job_id] = EvictedJobEntry( + context=context, + connection_id=connection_id, + evicted_monotonic=evicted_monotonic, + previousblockhash=str(context.template["previousblockhash"]), + client=client, + ) + self.evicted_job_graveyard = converted + rebuild_indexes = True + if not hasattr(runtime, "same_tip_job_retention_seconds"): + runtime.same_tip_job_retention_seconds = ( + DEFAULT_PRISM_SAME_TIP_JOB_RETENTION_SECONDS + ) + if not hasattr(runtime, "same_tip_job_retention_per_connection"): + runtime.same_tip_job_retention_per_connection = ( + DEFAULT_PRISM_SAME_TIP_JOB_RETENTION_PER_CONNECTION + ) + if not hasattr(runtime, "disconnected_job_retention"): + runtime.disconnected_job_retention = ( + DEFAULT_PRISM_DISCONNECTED_JOB_RETENTION + ) + current_tip = runtime._current_published_tip_hash_locked() + if self.evicted_job_index_tip_hash != current_tip: + rebuild_indexes = True + if rebuild_indexes: + self._rebuild_evicted_job_indexes_locked() + + # -- narrow adapters for other owners ---------------------------------- + + def next_job_id(self) -> str: + """Allocate the next delivery job id under the coordinator lock.""" + runtime = self._runtime + with runtime.lock: + self.job_counter += 1 + return f"prism-{self.job_counter}" + + def complete_delivery( + self, + client: Any, + context: PrismJobContext, + delivered_monotonic: float, + ) -> None: + """Commit S2 delivery bookkeeping after a successful socket write.""" + runtime = self._runtime + runtime.note_tip_work_delivered( + client, + str(context.template["previousblockhash"]), + ) + runtime._record_progress_delivery(client, context, delivered_monotonic) + + def stamp_job_for_client( + self, + client: ClientState, + cached: CachedJobBundle, + *, + clean_jobs: bool, + tip_refresh_epoch_sequence: int | None = None, + ) -> PrismJobContext: + runtime = self._runtime + if client.worker is None: + raise self._stratum_error(20, "client is not authorized") + if cached.collection_only and cached.collection_identity != ( + runtime._collection_bundle_identity(client.worker) + ): + raise self._stratum_error( + 20, + "collection bundle payout identity no longer matches client authorization", + ) + with runtime.lock: + self.job_counter += 1 + job_id = f"prism-{self.job_counter}" + if tip_refresh_epoch_sequence is None: + tip_refresh_epoch_sequence = ( + runtime._tip_refresh_epoch_for_bundle_locked(cached) + ) + share_target = direct_stratum.effective_share_target( + runtime.desired_client_share_difficulty(client), + cached.base_job.qbit_target, + minimum_advertised_difficulty=runtime.client_minimum_advertised_difficulty(client), + ) + job = dataclass_replace( + cached.base_job, + job_id=job_id, + extranonce1_hex=client.extranonce1_hex, + share_target=share_target, + share_difficulty=direct_stratum.target_difficulty(share_target), + clean_jobs=clean_jobs, + ) + return PrismJobContext( + job=job, + template=cached.template, + shares_json=cached.shares_json, + prior_balances=cached.prior_balances, + found_block=cached.found_block, + share_weight=runtime.share_weight_for_worker(client.worker), + collection_only=cached.collection_only, + worker=client.worker, + issued_at_ms=cached.issued_at_ms, + template_fingerprint=cached.template_fingerprint, + template_generation=cached.template_generation, + payout_state_generation=cached.payout_state_generation, + prospective_prior_balances=cached.prospective_prior_balances, + payout_artifact_generation=cached.payout_artifact_generation, + payout_append_invalidation_epoch=( + int( + getattr( + cached.build_key, + "payout_append_invalidation_epoch", + 0, + ) + ) + if cached.build_key is not None + else 0 + ), + payout_artifact_sha256=( + cached.build_key.payout_artifact_sha256 + if cached.build_key is not None + else None + ), + connection_id=client.connection_id, + authorization_generation=int( + getattr(client, "authorization_generation", 0) + ), + difficulty_generation=int( + getattr(client, "difficulty_generation", 0) + ), + tip_refresh_epoch_sequence=int(tip_refresh_epoch_sequence), + version_mask=int(getattr(client, "version_mask", 0)), + ) + + def initial_job_executor(self) -> _BoundedPriorityExecutor: + runtime = self._runtime + runtime._ensure_initial_job_state() + with self._initial_job_executor_lock: + if self._initial_job_executor_shutdown: + raise RuntimeError("initial job executor is shut down") + executor = self._initial_job_executor + if executor is None: + executor = _BoundedPriorityExecutor( + max_workers=runtime.initial_job_max_workers, + max_queue_size=runtime.stratum_max_pending_initial_jobs, + thread_name_prefix="prism-initial-job-delivery", + ) + self._initial_job_executor = executor + return executor + + def shutdown_initial_job_executor(self) -> None: + runtime = self._runtime + runtime._ensure_initial_job_state() + # Reclaim the whole admission domain in one constant-shape critical + # section, then cancel the futures outside it: every cancellation runs + # this module's done-callback inline, and that callback re-enters + # admission. + with self._initial_job_admission_lock: + reclaimed = tuple(self._pending_initial_jobs.values()) + self._pending_initial_jobs.clear() + futures = [] + for request in reclaimed: + request.cancelled.set() + if request.future is not None: + futures.append(request.future) + self.initial_job_cancelled_count += 1 + for future in futures: + runtime._cancel_initial_job_future(future) + with self._initial_job_executor_lock: + executor = self._initial_job_executor + self._initial_job_executor = None + self._initial_job_executor_shutdown = True + if executor is not None: + # Running workers observe request cancellation before shutdown + # returns; queued reconnect work is cancelled without starting. + executor.shutdown(wait=True, cancel_futures=True) + + def initial_job_admission_snapshot(self) -> InitialJobAdmissionSnapshot: + """Return one internally consistent copy of the admission domain. + + Health and metrics readers call this instead of walking the live + admission state under the coordinator lock, so a refresh never holds + two locks at once and never observes a half-committed replacement. + """ + with self._initial_job_admission_lock: + return InitialJobAdmissionSnapshot( + pending=dict(self._pending_initial_jobs), + queue_rejections=self.initial_job_queue_rejection_count, + timeout_disconnects=self.initial_job_timeout_count, + cancelled=self.initial_job_cancelled_count, + coalesced=self.initial_job_coalesced_count, + queue_capacity_reclaimed=( + self.initial_job_queue_capacity_reclaimed_count + ), + last_delivery_monotonic=self.last_initial_job_delivery_monotonic, + ) + + def _cancel_initial_job_future(self, future: Future[bool]) -> bool: + """Cancel one initial-job future and account physical queue removal.""" + executor = getattr(self, "_initial_job_executor", None) + reclaimed = bool( + executor is not None + and executor.cancel(future) + ) + if executor is None: + future.cancel() + if reclaimed: + # Admission accounting only; the cancellation callbacks that + # ``executor.cancel`` ran inline above have already completed, so + # taking the admission lock here cannot re-enter it. + with self._initial_job_admission_lock: + self.initial_job_queue_capacity_reclaimed_count += 1 + return reclaimed + + def _initial_request_current_locked(self, request: PendingInitialJob) -> bool: + """Coordinator-domain currency for one first-job request. + + Called with ``runtime.lock`` held (session membership and the + published-tip identity are read here). The pending-slot term is read + under the admission lock -- taken and released before the rest of the + predicate, and innermost as always -- so it cannot observe a + replacement or cancellation half applied. + """ + runtime = self._runtime + client = request.client + deadline = request.deadline_monotonic + with self._initial_job_admission_lock: + owns_pending_slot = self._pending_initial_jobs.get(client) is request + return ( + owns_pending_slot + and client in runtime.clients + and not getattr(client, "closing", False) + and ( + request.connection_id is None + or client.connection_id == request.connection_id + ) + and client.authorized + and client.subscribed + and client.worker == request.worker + and int(getattr(client, "authorization_generation", 0)) + == request.authorization_generation + and ( + request.difficulty_generation is None + or int(getattr(client, "difficulty_generation", 0)) + == request.difficulty_generation + ) + and (deadline is None or time.monotonic() < deadline) + and not request.cancelled.is_set() + ) + + def _initial_request_cancelled(self, request: PendingInitialJob) -> bool: + runtime = self._runtime + if request.cancelled.is_set() or runtime.stop_event.is_set(): + return True + with runtime.lock: + runtime._ensure_initial_job_state() + return not runtime._initial_request_current_locked(request) + + def _cancel_pending_initial_job_locked( + self, + client: ClientState, + *, + count: bool, + ) -> PendingInitialJob | None: + """Release this client's pending first-job slot, if it owns one. + + The historical ``_locked`` name records that the disconnect path calls + this with ``runtime.lock`` held; the admission state it mutates is now + owned by the admission lock, which is taken (and released) inside. The + future is cancelled after the slot is committed so the inline + done-callback observes the released slot and cannot deadlock. + """ + runtime = self._runtime + runtime._ensure_initial_job_state() + with self._initial_job_admission_lock: + request = self._pending_initial_jobs.pop(client, None) + if request is None: + return None + request.cancelled.set() + future = request.future + if count: + self.initial_job_cancelled_count += 1 + if future is not None: + runtime._cancel_initial_job_future(future) + return request + + def _client_has_current_tip_job_locked(self, client: ClientState) -> bool: + runtime = self._runtime + context = client.active_job + if context is None: + return False + payout_generation = int(getattr(runtime, "_payout_state_generation", 0)) + if int(getattr(context, "payout_state_generation", payout_generation)) != payout_generation: + return False + current_tip = runtime._current_published_tip_hash_locked() + if current_tip is None: + # An active job is not proof that tip observation is alive. Keep + # coverage fail-closed until blockpoll/blockwait has published the + # tip that makes the job current. + return False + snapshot = getattr(runtime, "tip_template_snapshot", None) + if snapshot is None: + # Focused embedders may only publish the observed tip. Production + # startup and blockpoll publish a full snapshot, in which case the + # exact template identity checks below are mandatory. + return str(context.template.get("previousblockhash", "")) == current_tip + if snapshot.bestblockhash != current_tip or snapshot.template_artifacts is None: + return False + return bool( + str(context.template.get("previousblockhash", "")) == current_tip + and getattr(context, "template_fingerprint", None) + == snapshot.template_fingerprint + and int(getattr(context, "template_generation", 0)) + == snapshot.template_generation + and context.template is snapshot.template_artifacts.template + and int(getattr(context, "connection_id", client.connection_id)) + == client.connection_id + and int(getattr(context, "authorization_generation", 0)) + == int(getattr(client, "authorization_generation", 0)) + and int(getattr(context, "difficulty_generation", 0)) + == int(getattr(client, "difficulty_generation", 0)) + ) + + def note_initial_job_delivered( + self, + client: ClientState, + *, + validated_current: bool = False, + ) -> None: + runtime = self._runtime + runtime._ensure_initial_job_state() + if not validated_current: + # Coordinator-domain question (published tip identity, payout + # generation): a short constant-time read, taken and released + # before any admission state is touched. + with runtime.lock: + if not runtime._client_has_current_tip_job_locked(client): + return + future: Future[bool] | None = None + completed = False + with self._initial_job_admission_lock: + request = self._pending_initial_jobs.pop(client, None) + if request is not None: + request.cancelled.set() + future = request.future + delivered = time.monotonic() + self.initial_job_sent_count += 1 + self.initial_job_delivery_latency_seconds_sum += max( + 0.0, delivered - request.requested_monotonic + ) + self.initial_job_delivery_latency_count += 1 + self.last_initial_job_delivery_monotonic = delivered + completed = True + if future is not None: + runtime._cancel_initial_job_future(future) + if completed: + runtime._record_startup_phase_once("first_job_delivered") + + def schedule_initial_job(self, client: ClientState) -> bool: + """Coalesce and enqueue one first-job request without blocking its handler.""" + runtime = self._runtime + # Focused tests and embedders replace maybe_send_job on the instance as + # a synchronous seam. Preserve it without affecting the production + # class path, which always uses the bounded executor below. + if "maybe_send_job" in runtime.__dict__: + return bool(runtime.maybe_send_job(client, clean_jobs=True)) + + now = time.monotonic() + runtime._ensure_initial_job_state() + if ( + not client.subscribed + or not client.authorized + or client.worker is None + or getattr(client, "closing", False) + ): + return True + generation = int(getattr(client, "authorization_generation", 0)) + difficulty_generation = int(getattr(client, "difficulty_generation", 0)) + worker = client.worker + connection_id = client.connection_id + + def represents_this_authorization(candidate: PendingInitialJob) -> bool: + return ( + candidate.connection_id == connection_id + and candidate.authorization_generation == generation + and candidate.difficulty_generation == difficulty_generation + and candidate.worker == worker + ) + + # Pure queue bookkeeping: a live request already represents exactly + # this authorization. A reconnect herd's repeat requests settle here + # without ever reaching the coordinator lock. + with self._initial_job_admission_lock: + existing = self._pending_initial_jobs.get(client) + if existing is not None and represents_this_authorization(existing): + self.initial_job_coalesced_count += 1 + return True + + # Coordinator-domain snapshot: whether this connection already holds + # current published-tip work is global tip identity, so it stays under + # runtime.lock -- taken alone, for a constant-time read, and released + # before the admission commit below. + with runtime.lock: + has_current_tip_job = runtime._client_has_current_tip_job_locked(client) + + # Configuration is read outside the admission lock so nothing but + # local state is touched inside it. + capacity = int(runtime.stratum_max_pending_initial_jobs) + timeout = float(runtime.stratum_initial_job_timeout_seconds) + + reject = False + deferred = False + superseded_future: Future[bool] | None = None + request: PendingInitialJob | None = None + with self._initial_job_admission_lock: + if getattr(client, "closing", False): + # Re-read under the admission lock: retirement and the timeout + # sweep both commit ``closing`` before releasing this client's + # slot, so an installation that observed a stale False here + # would strand a request behind a session already being torn + # down. + return True + existing = self._pending_initial_jobs.get(client) + if existing is not None and represents_this_authorization(existing): + self.initial_job_coalesced_count += 1 + return True + if existing is not None: + existing.cancelled.set() + superseded_future = existing.future + self.initial_job_cancelled_count += 1 + self.initial_job_superseded_count += 1 + if has_current_tip_job: + self._pending_initial_jobs.pop(client, None) + if not has_current_tip_job: + if ( + existing is None + and len(self._pending_initial_jobs) >= capacity + ): + self.initial_job_queue_rejection_count += 1 + reject = True + else: + predecessor = None + if existing is not None: + for candidate in (existing.future, existing.predecessor): + if candidate is not None and not candidate.done(): + predecessor = candidate + break + request = PendingInitialJob( + client=client, + connection_id=connection_id, + authorization_generation=generation, + difficulty_generation=difficulty_generation, + worker=worker, + requested_monotonic=now, + deadline_monotonic=now + timeout if timeout > 0 else None, + predecessor=predecessor, + ) + # Install the replacement before releasing the lock: the + # superseded future's cancellation callback runs inline + # below and must observe the replacement already holding + # the client slot. + self._pending_initial_jobs[client] = request + deferred = predecessor is not None + if superseded_future is not None: + # Cancellation callbacks run inline here, with the admission state + # already committed and the admission lock released; the + # predecessor callback then hands off exactly one client slot + # instead of mistaking the obsolete request for a terminal failure. + runtime._cancel_initial_job_future(superseded_future) + if has_current_tip_job: + return True + if reject or request is None: + runtime.disconnect_client(client) + return False + if deferred: + return True + + return runtime._submit_initial_job_request(request) + + def request_initial_job_delivery(self, client: ClientState) -> bool: + """Compatibility name for the single bounded initial-job pipeline.""" + runtime = self._runtime + return runtime.schedule_initial_job(client) + + def cancel_initial_job_delivery(self, client: ClientState) -> None: + runtime = self._runtime + runtime._ensure_initial_job_state() + runtime._cancel_pending_initial_job_locked(client, count=True) + + def _submit_initial_job_request(self, request: PendingInitialJob) -> bool: + runtime = self._runtime + client = request.client + try: + future = runtime._submit_delivery_task( + runtime.initial_job_executor(), + runtime._run_initial_job, + request, + priority=PRISM_DELIVERY_PRIORITY_INITIAL, + ) + except (_DeliveryQueueFull, RuntimeError): + disconnect = False + with self._initial_job_admission_lock: + if self._pending_initial_jobs.get(client) is request: + self._pending_initial_jobs.pop(client, None) + request.cancelled.set() + self.initial_job_queue_rejection_count += 1 + disconnect = True + if disconnect: + runtime.disconnect_client(client) + # An obsolete submit can race its already-installed replacement. + # It owns neither the client slot nor the right to retire the live + # session when admission fails. + return not disconnect + orphan: Future[bool] | None = None + with self._initial_job_admission_lock: + if self._pending_initial_jobs.get(client) is request: + request.future = future + else: + orphan = future + if orphan is not None: + # A replacement already owns the client slot. Reclaim the queue + # entry outside the admission lock; the done-callback installed + # just below then fires immediately and hands the slot on. + runtime._cancel_initial_job_future(orphan) + add_releasing_done_callback( + future, + lambda completed: runtime._initial_job_future_finished(request, completed) + ) + return True + + def _initial_job_future_finished( + self, + request: PendingInitialJob, + future: Future[bool], + ) -> None: + """Release failed first-job requests instead of stranding capacity.""" + runtime = self._runtime + delivered = False + if not future.cancelled(): + # Re-raising a stored task error here would add this request-owning + # frame to its traceback, retaining the client through the future. + error = future.exception() + if error is None: + try: + delivered = bool(future.result()) + except Exception as exc: + error = exc + if error is not None: + if not isinstance(error, Exception): + raise error + with runtime.lock: + runtime.job_build_failure_count = int( + getattr(runtime, "job_build_failure_count", 0) + ) + 1 + print( + "prism coordinator: initial job task failed " + f"connection={request.client.connection_id}", + flush=True, + ) + traceback.print_exception(error) + error = None + + # Coordinator-domain question, asked only on the delivered path and + # only for as long as the constant-time read takes. + proven_delivered = False + if delivered: + with runtime.lock: + runtime._ensure_initial_job_state() + proven_delivered = bool( + runtime._initial_request_current_locked(request) + and runtime._client_has_current_tip_job_locked(request.client) + ) + + disconnect = False + first_delivery = False + replacement: PendingInitialJob | None = None + with self._initial_job_admission_lock: + current = self._pending_initial_jobs.get(request.client) + if current is not request: + if ( + current is not None + and current.future is None + and current.predecessor is future + ): + current.predecessor = None + replacement = current + elif proven_delivered: + self._pending_initial_jobs.pop(request.client, None) + request.cancelled.set() + self.last_initial_job_delivery_monotonic = time.monotonic() + first_delivery = True + else: + self._pending_initial_jobs.pop(request.client, None) + request.cancelled.set() + if ( + request.deadline_monotonic is not None + and request.deadline_monotonic <= time.monotonic() + ): + # Commit teardown while this request still owns the pending + # slot: a concurrent reauthorization re-reads ``closing`` + # under this lock and cannot install a replacement between + # expiry and disconnect. + request.client.closing = True + self.initial_job_timeout_count += 1 + self.initial_job_cancelled_count += 1 + else: + self.initial_job_failed_count += 1 + disconnect = True + if first_delivery: + runtime._record_startup_phase_once("first_job_delivered") + if replacement is not None: + runtime._submit_initial_job_request(replacement) + if disconnect: + runtime.disconnect_client(request.client) + + def _log_accepted_parent_preview_pending( + self, + exc: AcceptedParentPayoutPreviewPending, + *, + connection_id: object, + ) -> None: + """Rate-limited retry note for accepted-parent preview backpressure. + + Kept separate from the generic build-failure log (and its own 5s + stamp) so benign backpressure never crowds out, or is mistaken for, + a counted job-build failure. + """ + now = time.monotonic() + last = getattr(self, "_last_preview_pending_log_monotonic", None) + if last is not None and now - last < 5.0: + return + self._last_preview_pending_log_monotonic = now + print( + "prism coordinator: job delivery waiting on accepted-parent payout " + f"preview reason={exc.retry_reason} connection={connection_id}; " + f"retrying: {exc}", + flush=True, + ) + + def _run_initial_job(self, request: PendingInitialJob) -> bool: + """Prepare outside client locks, then atomically stamp and send current work.""" + runtime = self._runtime + retry_delay = 0.05 + last_failure_log_monotonic: float | None = None + + def retry_later() -> bool: + nonlocal retry_delay + request.cancelled.wait(retry_delay) + retry_delay = min(1.0, retry_delay * 2) + return not runtime._initial_request_cancelled(request) + + try: + while not runtime._initial_request_cancelled(request): + try: + if not runtime.ensure_reorg_reconciled_for_current_tip(): + if not retry_later(): + return False + continue + artifacts = runtime.job_issuance_template_artifacts() + if runtime._initial_request_cancelled(request): + return False + bundle = runtime.shared_job_bundle( + artifacts, + request.worker, + request_source="initial", + cancelled=lambda: ( + runtime._initial_request_cancelled(request) + or not runtime._issuance_artifacts_current(artifacts) + ), + ) + live_tip = str(runtime.rpc.call("getbestblockhash")) + if artifacts.previousblockhash != live_tip: + # Feed the observation into detection like every other + # live-tip reader; the refresh path owns publication. + runtime.observe_tip_for_refresh(live_tip) + with runtime.lock: + published = getattr(runtime, "current_tip_first_seen", None) + pinned_authoritative = bool( + published is not None + and artifacts.previousblockhash == published[0] + and runtime._published_tip_authoritative_locked( + time.monotonic() + ) + ) + if not pinned_authoritative: + # Authority lapsed: submit classification is on the + # live RPC read now, so published-tip work would be + # rejected. Drop it and rebuild from live state. + with runtime._job_cache_lock: + if runtime._template_artifacts is artifacts: + runtime._template_artifacts = None + if not retry_later(): + return False + continue + except JobBuildWaiterCancelled: + if runtime._initial_request_cancelled(request): + return False + if not retry_later(): + return False + continue + except AcceptedParentPayoutPreviewPending as exc: + # Coordination backpressure, not a build failure: the + # accepted parent's preview has simply not published yet. + # Coalesce onto the same bounded retry as any other + # blocked refresh and stay out of + # ``job_build_failure_count``. The raise site already + # counted the timeout and scheduled the tip-refresh + # retry, so sustained pendency still escalates through + # the coordination-blocked watchdog instead of retrying + # silently forever. + if runtime._initial_request_cancelled(request): + return False + self._log_accepted_parent_preview_pending( + exc, + connection_id=request.client.connection_id, + ) + if not retry_later(): + return False + continue + except TemplateRefreshBlocked: + if runtime._initial_request_cancelled(request): + return False + if not retry_later(): + return False + continue + except Exception: + with runtime.lock: + runtime.job_build_failure_count = int( + getattr(runtime, "job_build_failure_count", 0) + ) + 1 + now = time.monotonic() + if ( + last_failure_log_monotonic is None + or now - last_failure_log_monotonic >= 5.0 + ): + last_failure_log_monotonic = now + print( + "prism coordinator: initial job preparation failed " + f"connection={request.client.connection_id}; retrying", + flush=True, + ) + traceback.print_exc() + if not retry_later(): + return False + continue + delivered = runtime._deliver_initial_bundle(request, artifacts, bundle) + if delivered is None: + if not retry_later(): + return False + continue + return delivered + return False + except OSError: + runtime.disconnect_client(request.client) + return False + + def _deliver_initial_bundle( + self, + request: PendingInitialJob, + artifacts: CachedTemplateArtifacts, + bundle: CachedJobBundle, + ) -> bool | None: + runtime = self._runtime + client = request.client + + def cancelled() -> bool: + return runtime._initial_request_cancelled(request) + + def bundle_payout_stale() -> bool: + # Unlocked at the gate (a cancellation hint, exactly like direct + # delivery's callback); authoritative only when re-read under + # _job_cache_lock at the commit boundary below. + published_artifact = runtime._published_payout_state.artifact + return ( + bundle.payout_state_generation != runtime._payout_state_generation + or bundle.build_key is None + # A periodic self-check repair replaces the published balance + # snapshot in place: the generation and append epoch both + # survive the swap, so only the digest identifies work keyed + # to the refuted balances. Every other serving decision + # (cache usability, idle issuance, cache publication) already + # applies this fence; the initial-delivery commit must too, + # or a bundle probed before the repair becomes the miner's + # active job with the refuted balances. + or published_artifact is None + or bundle.build_key.payout_artifact_sha256 + != published_artifact.prior_balances_sha256 + or ( + not bundle.collection_only + and int( + getattr( + bundle.build_key, + "payout_append_invalidation_epoch", + 0, + ) + ) + != runtime._payout_ledger_append_invalidation_epoch + ) + ) + + if not self._acquire_client_job_lock(client, cancelled): + return False + try: + if cancelled(): + return False + if not runtime._issuance_artifacts_current(artifacts): + return None + runtime._ensure_job_cache_state() + gate_started = time.monotonic() + with runtime._payout_delivery( + lambda: cancelled() or bundle_payout_stale(), + generation=bundle.payout_state_generation, + ) as admitted: + runtime._observe_payout_gate_admission( + admitted, + generation=bundle.payout_state_generation, + fallback_wait_seconds=time.monotonic() - gate_started, + ) + if cancelled(): + return False + if not admitted: + # Non-admission with a bundle in hand is transient gate + # state: a payout invalidation pending publication, or + # this bundle's generation going stale while waiting. + # Rebuild and retry within the request deadline; treating + # it as terminal turned every unlucky join during a payout + # publication into a disconnect-reconnect-rebuild loop. + return None + with runtime._job_cache_lock: + payout_current = not bundle_payout_stale() + if not payout_current or not runtime._issuance_artifacts_current(artifacts): + return None + # Difficulty state precedes coordinator admission everywhere. + # A share holding this client's Vardiff lock may delay only + # this delivery; it must never make initial-job delivery hold + # the global control-plane lock while waiting. + with runtime._client_vardiff_lock(client): + # The Vardiff wait happens after the payout check above; a + # late-visible append can invalidate the bundle during it. + # Mirror direct delivery: the live epoch re-read and the + # stamp are one commit boundary under _job_cache_lock. + with runtime._job_cache_lock: + if bundle_payout_stale(): + return None + with runtime.lock: + if not runtime._initial_request_current_locked(request): + return False + context = runtime.stamp_job_for_client( + client, + bundle, + clean_jobs=True, + ) + if not runtime._admit_client_tip_refresh_epoch_locked( + client, + context.tip_refresh_epoch_sequence, + ): + # The fence blocks only strictly older epochs, so + # this connection already registered newer refresh + # work whose delivery the epoch machinery owns. + # Retrying here can spin until the initial-job + # deadline disconnects an already-served client; + # complete a request that newer current-tip work + # satisfies instead. + runtime.note_initial_job_delivered(client) + with self._initial_job_admission_lock: + still_owns_slot = ( + self._pending_initial_jobs.get(client) + is request + ) + if not still_owns_slot: + return False + return None + client.active_job = context + for job_id in tuple(client.active_job_ids): + runtime.bury_evicted_job(client, job_id, prune=False) + self.jobs.pop(job_id, None) + client.active_job_ids.clear() + runtime.prune_evicted_job_graveyard(force=False) + self.jobs[context.job.job_id] = context + client.active_job_ids.add(context.job.job_id) + runtime.prune_client_active_jobs(client) + + runtime.send_job_update(client, context.job) + mark_delivered = getattr(admitted, "mark_delivered", None) + if callable(mark_delivered): + mark_delivered() + runtime.apply_job_difficulty(client, context.job) + runtime.note_tip_work_delivered( + client, + str(context.template["previousblockhash"]), + ) + delivered_monotonic = time.monotonic() + runtime._record_first_payout_delivery( + bundle.payout_state_generation, + delivered_monotonic, + ) + runtime._record_progress_delivery( + client, + context, + delivered_monotonic, + ) + runtime.note_initial_job_delivered(client, validated_current=True) + return True + finally: + client.job_update_lock.release() + + def sweep_initial_job_timeouts(self, *, now: float | None = None) -> int: + runtime = self._runtime + runtime._ensure_initial_job_state() + now = time.monotonic() if now is None else now + timed_out: list[PendingInitialJob] = [] + futures: list[Future[bool]] = [] + with self._initial_job_admission_lock: + expired = [ + request + for request in self._pending_initial_jobs.values() + if request.deadline_monotonic is not None + and request.deadline_monotonic <= now + ] + for request in expired: + self._pending_initial_jobs.pop(request.client, None) + request.cancelled.set() + if request.future is not None: + futures.append(request.future) + # Commit teardown while this request still owns the pending + # slot. A concurrent reauthorization re-reads closing under + # this lock and cannot install a replacement between expiry + # and disconnect. + request.client.closing = True + self.initial_job_timeout_count += 1 + self.initial_job_cancelled_count += 1 + timed_out.append(request) + # Queue reclamation runs cancellation callbacks inline; both it and + # the disconnects below stay outside the admission lock. + for future in futures: + runtime._cancel_initial_job_future(future) + for request in timed_out: + runtime.disconnect_client(request.client) + return len(timed_out) + + def initial_job_timeout_loop(self) -> None: + runtime = self._runtime + while not runtime.stop_event.wait(1.0): + runtime.sweep_initial_job_timeouts() + + def send_prepared_job( + self, + client: ClientState, + bundle: CachedJobBundle, + snapshot: QbitTipTemplateSnapshot, + validation_token: TipRefreshValidationToken, + expected_connection_id: int, + expected_active_job: PrismJobContext | None, + cancel_event: FanoutCancellation | None = None, + submitted_monotonic: float | None = None, + ) -> RefreshResult: + runtime = self._runtime + worker_started = time.monotonic() + started = worker_started if submitted_monotonic is None else submitted_monotonic + phases = runtime._job_build_phases() + phases.clear() + + def cancelled() -> bool: + return runtime._prepared_tip_refresh_obsolete( + validation_token, + bundle, + snapshot, + cancel_event, + ) or getattr(client, "closing", False) + + phases["executor_queue"] = max(0.0, worker_started - started) + client_lock_started = worker_started + client_lock_acquired = False + client_lock_attempted = False + try: + while True: + with runtime.lock: + if ( + client not in runtime.clients + or client.connection_id != expected_connection_id + or getattr(client, "closing", False) + ): + return RefreshResult("disconnected") + if cancelled(): + phases["client_lock"] = max( + 0.0, + time.monotonic() - client_lock_started, + ) + runtime._record_tip_refresh_cancellation( + "client_lock" if client_lock_attempted else "executor_queue" + ) + return RefreshResult("skipped") + client_lock_attempted = True + client_lock_acquired = client.job_update_lock.acquire( + timeout=PRISM_TIP_REFRESH_ADMISSION_POLL_SECONDS + ) + if client_lock_acquired: + break + phases["client_lock"] = max( + 0.0, + time.monotonic() - client_lock_started, + ) + if cancelled(): + runtime._record_tip_refresh_cancellation("client_lock") + return RefreshResult("skipped") + with runtime.lock: + if ( + client not in runtime.clients + or client.connection_id != expected_connection_id + ): + return RefreshResult("disconnected") + active_epoch = int( + getattr( + getattr(client, "active_job", None), + "tip_refresh_epoch_sequence", + 0, + ) + ) + epoch_fanout = bool( + getattr(runtime, "tip_refresh_epoch_fanout", False) + ) + if ( + not runtime.client_can_receive_jobs(client) + or ( + epoch_fanout + and runtime._client_tip_refresh_epoch_blocked_locked( + client, + validation_token.epoch_sequence, + ) + ) + or ( + ( + not epoch_fanout + or active_epoch >= validation_token.epoch_sequence + ) + and runtime.intervening_job_supersedes_snapshot( + client.active_job, + expected_active_job, + snapshot, + ) + ) + or not runtime.client_needs_tip_template_refresh(client, snapshot) + ): + return RefreshResult("skipped") + runtime._ensure_job_cache_state() + payout_gate_started = time.monotonic() + with runtime._payout_state_delivery_gate.delivery_cancelable( + cancelled, + generation=bundle.payout_state_generation, + priority=True, + ) as payout_admitted: + phases["payout_gate"] = max( + 0.0, + time.monotonic() - payout_gate_started, + ) + runtime._observe_payout_gate_admission( + payout_admitted, + generation=bundle.payout_state_generation, + fallback_wait_seconds=phases["payout_gate"], + ) + if not payout_admitted or cancelled(): + runtime._record_tip_refresh_cancellation("payout_gate") + return RefreshResult("skipped") + fanout_admitted = ( + cancel_event is None or cancel_event.begin_delivery() + ) + if not fanout_admitted: + runtime._record_tip_refresh_cancellation("payout_gate") + return RefreshResult("skipped") + try: + # The validation token binds this immutable bundle to the + # exact observed artifact object. Fanout tasks consult only + # in-memory publication/cancellation state, never RPC or + # the mutable cache. + # Preserve one coherent difficulty snapshot through stamp + # and commit, but acquire it before global control-plane + # admission so a concurrent share cannot form the inverse + # coordinator -> Vardiff edge. + with runtime._client_vardiff_lock(client): + with runtime.lock: + token_current = runtime._tip_refresh_token_current_locked( + validation_token, + bundle, + snapshot, + ) + if not token_current: + if cancel_event is not None: + cancel_event.cancel() + return RefreshResult("skipped") + if ( + client not in runtime.clients + or client.connection_id != expected_connection_id + ): + return RefreshResult("disconnected") + active_epoch = int( + getattr( + getattr(client, "active_job", None), + "tip_refresh_epoch_sequence", + 0, + ) + ) + epoch_fanout = bool( + getattr(runtime, "tip_refresh_epoch_fanout", False) + ) + if ( + not runtime.client_can_receive_jobs(client) + or ( + epoch_fanout + and runtime._client_tip_refresh_epoch_blocked_locked( + client, + validation_token.epoch_sequence, + ) + ) + or ( + ( + not epoch_fanout + or active_epoch + >= validation_token.epoch_sequence + ) + and runtime.intervening_job_supersedes_snapshot( + client.active_job, + expected_active_job, + snapshot, + ) + ) + or not runtime.client_needs_tip_template_refresh( + client, + snapshot, + ) + ): + return RefreshResult("skipped") + if not runtime._admit_client_tip_refresh_epoch_locked( + client, + validation_token.epoch_sequence, + ): + return RefreshResult("skipped") + clean_jobs = runtime.client_tip_changed_for_snapshot( + client, + snapshot, + ) + stamp_started = time.monotonic() + context = runtime.stamp_job_for_client( + client, + bundle, + clean_jobs=clean_jobs, + tip_refresh_epoch_sequence=( + validation_token.epoch_sequence + ), + ) + phases["stamp"] = time.monotonic() - stamp_started + client.active_job = context + if clean_jobs: + for job_id in tuple(client.active_job_ids): + runtime.bury_evicted_job(client, job_id, prune=False) + self.jobs.pop(job_id, None) + client.active_job_ids.clear() + runtime.prune_evicted_job_graveyard(force=False) + self.jobs[context.job.job_id] = context + client.active_job_ids.add(context.job.job_id) + runtime.prune_client_active_jobs(client) + + socket_send_started = time.monotonic() + try: + runtime.send_job_update(client, context.job) + payout_admitted.mark_delivered() + finally: + socket_send_finished = time.monotonic() + phases["socket_send"] = max( + 0.0, + socket_send_finished - socket_send_started, + ) + runtime.apply_job_difficulty(client, context.job) + runtime.note_tip_work_delivered( + client, + str(context.template["previousblockhash"]), + ) + runtime.note_initial_job_delivered(client, validated_current=True) + delivered_monotonic = time.monotonic() + runtime._record_first_payout_delivery( + context.payout_state_generation, + delivered_monotonic, + ) + runtime._record_progress_delivery( + client, + context, + delivered_monotonic, + ) + if getattr(runtime, "hot_path_log_enabled", False): + print( + "prism coordinator: sent prepared job " + f"connection={client.connection_id} username={client.username} " + f"job={context.job.job_id} " + f"elapsed={delivered_monotonic - started:.3f}s", + flush=True, + ) + return RefreshResult("sent", delivered_monotonic) + finally: + if cancel_event is not None: + cancel_event.end_delivery() + finally: + if client_lock_acquired: + client.job_update_lock.release() + runtime.observe_job_build_elapsed( + max(0.0, time.monotonic() - started), + phases, + ) + + def note_tip_work_delivered(self, client: ClientState, job_parent_hash: str) -> None: + """Record the first time this connection was sent work for a tip. + + First delivery wins per tip: same-tip template refreshes must not slide + the connection's stale-grace anchor forward. + """ + now = time.monotonic() + # Per-connection state only. The coordinator lock is deliberately + # absent: a delivery to one connection must never wait on fleet-wide + # control-plane work, and the O(N) current-tip coverage census that + # used to run here now runs once per observability refresh instead of + # once per delivery (#159). + # + # Anchoring is serialized by this connection's job-update lock. Every + # production delivery path already holds it across send-then-anchor, so + # re-entering it costs nothing and extends the same serialization to + # the direct compatibility callers. Lightweight embedders and focused + # doubles may supply a minimal lock exposing only the timed + # acquire/release pair the delivery path uses; those callers hold it + # already, so anchor without re-entering rather than failing on the + # missing protocol. + lock = getattr(client, "job_update_lock", None) + with lock if hasattr(lock, "__enter__") else nullcontext(): + delivered = client.tip_work_delivered + if delivered is None or delivered[0] != job_parent_hash: + client.tip_work_delivered = (job_parent_hash, now) + + def _evicted_job_class_locked(self, entry: EvictedJobEntry) -> str: + runtime = self._runtime + current_tip = runtime._current_published_tip_hash_locked() + if current_tip is None or entry.previousblockhash == current_tip: + return "same_tip" + return "stale_grace" + + def bury_evicted_job( + self, + client: ClientState, + job_id: str, + *, + now: float | None = None, + prune: bool = True, + ) -> None: + runtime = self._runtime + with runtime.lock: + runtime._ensure_evicted_job_state() + context = self.jobs.get(job_id) + if context is None: + return + self._remove_evicted_job_locked(job_id) + self.evicted_job_graveyard[job_id] = EvictedJobEntry( + context=context, + connection_id=client.connection_id, + evicted_monotonic=time.monotonic() if now is None else now, + previousblockhash=str(context.template["previousblockhash"]), + client=client, + ) + self._index_evicted_job_locked(job_id, self.evicted_job_graveyard[job_id]) + self._enforce_evicted_same_tip_capacity_locked(client.connection_id) + if prune: + runtime.prune_evicted_job_graveyard(now=now, force=False) + + def prune_evicted_job_graveyard( + self, + *, + now: float | None = None, + force: bool = True, + ) -> None: + runtime = self._runtime + with runtime.lock: + runtime._ensure_evicted_job_state() + if not self.evicted_job_graveyard: + return + now = time.monotonic() if now is None else now + if not force and now < self.evicted_job_next_prune_monotonic: + return + self.evicted_job_next_prune_monotonic = ( + now + DEFAULT_PRISM_EVICTED_JOB_PRUNE_INTERVAL_SECONDS + ) + for job_id, entry in tuple(self.evicted_job_graveyard.items()): + job_class, expired = self._evicted_job_expired_locked(entry, now=now) + if expired: + self._remove_evicted_job_locked(job_id) + self.evicted_job_expiration_counts[job_class] += 1 + + def evicted_job_entry( + self, + client: ClientState, + job_id: str, + ) -> EvictedJobEntry | None: + runtime = self._runtime + with runtime.lock: + runtime._ensure_evicted_job_state() + entry = getattr(self, "evicted_job_graveyard", {}).get(job_id) + if entry is None: + return None + if ( + entry.connection_id != client.connection_id + and not self._cross_connection_evicted_entry_allowed_locked( + entry, + client, + ) + ): + return None + job_class, expired = self._evicted_job_expired_locked( + entry, + now=time.monotonic(), + ) + if expired: + self._remove_evicted_job_locked(job_id) + self.evicted_job_expiration_counts[job_class] += 1 + return None + return entry + + def note_evicted_job_submit( + self, + credit_policy: str | None, + *, + cross_connection: bool = False, + ) -> None: + runtime = self._runtime + if credit_policy == PRISM_CREDIT_POLICY_STALE_GRACE: + outcome = "credited_stale_grace" + elif cross_connection: + outcome = "accepted_same_tip_cross_connection" + else: + outcome = "accepted_same_tip" + with runtime.lock: + runtime._ensure_evicted_job_state() + self.evicted_job_submit_counts[outcome] += 1 + + def client_can_receive_jobs(self, client: ClientState) -> bool: + return ( + not getattr(client, "closing", False) + and client.subscribed + and client.authorized + and client.worker is not None + ) + + def client_needs_tip_template_refresh( + self, + client: ClientState, + snapshot: QbitTipTemplateSnapshot, + ) -> bool: + runtime = self._runtime + context = client.active_job + if context is None: + return True + if getattr(runtime, "tip_refresh_epoch_fanout", False): + current_epoch = int( + getattr(runtime, "_tip_refresh_epoch_sequence", 0) + ) + context_epoch = int( + getattr(context, "tip_refresh_epoch_sequence", 0) + ) + if context_epoch > current_epoch: + return False + if context_epoch < current_epoch: + return True + current_epoch_tip = getattr( + runtime, + "_tip_refresh_epoch_tip_hash", + None, + ) + if ( + current_epoch_tip is not None + and str(context.template.get("previousblockhash", "")) + != current_epoch_tip + ): + return True + if getattr(context, "collection_only", False) and getattr(runtime, "_pool_ready_latched", False + ): + # The pool crossed min_ready_miners after this job was issued. + # A collection job keeps settling solved blocks solver-pays-all, + # so replace it with windowed work on the next poller pass. + return True + template = context.template + previousblockhash = str(template.get("previousblockhash", "")) + context_fingerprint = getattr(context, "template_fingerprint", None) + if context_fingerprint is None: + context_fingerprint = qbit_template_fingerprint(template) + context_payout_generation = int( + getattr(context, "payout_state_generation", 0) + ) + context_payout_artifact_sha256 = getattr( + context, + "payout_artifact_sha256", + None, + ) + published_payout_state = getattr(runtime, "_published_payout_state", None) + published_payout_artifact = ( + published_payout_state.artifact + if published_payout_state is not None + else None + ) + active_needs_refresh = ( + previousblockhash != snapshot.bestblockhash + or previousblockhash != snapshot.previousblockhash + or context_fingerprint != snapshot.template_fingerprint + or context_payout_generation + != int(getattr(runtime, "_payout_state_generation", 0)) + # A self-check repair swaps the published balance snapshot in + # place: the generation and append epoch both survive, so only + # the digest identifies an active job still mining the refuted + # balances. Jobs stamped before the digest existed (or while no + # artifact is published mid-publication) defer to the generation + # fence above. + or ( + context_payout_artifact_sha256 is not None + and published_payout_artifact is not None + and context_payout_artifact_sha256 + != published_payout_artifact.prior_balances_sha256 + ) + # A late-visible append advances only the epoch: the generation + # and digest survive, yet the job's window omitted the late row + # (possibly a replayed winning share). Collection jobs settle + # solver-pays-all and are exempt, as at every other epoch fence. + or ( + not getattr(context, "collection_only", False) + and int( + getattr(context, "payout_append_invalidation_epoch", 0) + ) + != int( + getattr( + runtime, + "_payout_ledger_append_invalidation_epoch", + 0, + ) + ) + ) + ) + if active_needs_refresh: + return True + if not getattr(runtime, "tip_refresh_epoch_fanout", False): + return False + delivered = getattr(client, "_progress_delivered_context", None) + if delivered is None: + return True + delivered_fingerprint = getattr( + delivered, + "template_fingerprint", + None, + ) + if delivered_fingerprint is None: + delivered_fingerprint = qbit_template_fingerprint( + delivered.template + ) + # Reselection must cover every condition the epoch fixpoint counts + # as undelivered, or the owning wave re-enters forever with no + # candidate to serve. A delivered collection job after the pool + # latches ready is one such condition even when its epoch, tip, + # payout generation, and fingerprint all remain current, because + # registration can advance past a delivery whose send failed. + # A client whose current-epoch delivery is registered but unproven + # is deliberately reselected here (the fence admits equal epochs, + # so the re-serve is idempotent on the wire). Deduplicating on the + # admitted epoch instead would be wrong: admission is monotonic + # and survives a failed send, so an admitted-equal skip would + # deselect the client while the fixpoint still counts it + # undelivered. + return ( + int(getattr(delivered, "connection_id", 0)) + != int(client.connection_id) + or int(getattr(delivered, "tip_refresh_epoch_sequence", 0)) + != current_epoch + or str(delivered.template.get("previousblockhash", "")) + != current_epoch_tip + or int(getattr(delivered, "payout_state_generation", 0)) + != int(getattr(runtime, "_tip_refresh_epoch_payout_generation", 0)) + or delivered_fingerprint != snapshot.template_fingerprint + or ( + bool(getattr(delivered, "collection_only", False)) + and bool(getattr(runtime, "_pool_ready_latched", False)) + ) + ) + + def intervening_job_supersedes_snapshot( + self, + active_job: PrismJobContext | None, + expected_active_job: PrismJobContext | None, + snapshot: QbitTipTemplateSnapshot, + ) -> bool: + runtime = self._runtime + if active_job is expected_active_job or active_job is None: + return False + active_payout_generation = int( + getattr(active_job, "payout_state_generation", 0) + ) + if active_payout_generation < int( + getattr(runtime, "_payout_state_generation", 0) + ): + # Template ordering cannot make a payout-stale intervening job + # authoritative. Let the reconciled refresh replace it. + return False + active_parent_hash = str( + getattr(active_job, "template", {}).get("previousblockhash", "") + ) + if ( + active_parent_hash != snapshot.bestblockhash + or active_parent_hash != snapshot.previousblockhash + ): + # Artifact generations order fetch starts, not chain tips. A fetch + # for the old tip can start after this exact new-tip observation + # and therefore carry a larger generation; it must not prevent + # the new-tip snapshot from replacing that stale work. + return False + active_generation = int(getattr(active_job, "template_generation", 0)) + snapshot_generation = int(getattr(snapshot, "template_generation", 0)) + if active_generation <= 0 or snapshot_generation <= 0: + # Legacy/test contexts without ordering metadata retain the safe + # behavior: never overwrite an unclassified intervening job. + return True + return active_generation >= snapshot_generation + + def client_tip_changed_for_snapshot( + self, + client: ClientState, + snapshot: QbitTipTemplateSnapshot, + ) -> bool: + runtime = self._runtime + context = client.active_job + if context is None: + return True + previousblockhash = str(context.template.get("previousblockhash", "")) + context_payout_artifact_sha256 = getattr( + context, + "payout_artifact_sha256", + None, + ) + published_payout_state = getattr(runtime, "_published_payout_state", None) + published_payout_artifact = ( + published_payout_state.artifact + if published_payout_state is not None + else None + ) + return ( + previousblockhash != snapshot.bestblockhash + or previousblockhash != snapshot.previousblockhash + or int(getattr(context, "payout_state_generation", 0)) + != int(getattr(runtime, "_payout_state_generation", 0)) + # A refresh selected because a self-check repair replaced the + # published balance digest must also retire the refuted job: a + # clean_jobs=False replacement leaves the old job ID admissible, + # so a late block submission could still commit the refuted + # allocation. Same None guards as the reselection predicate. + or ( + context_payout_artifact_sha256 is not None + and published_payout_artifact is not None + and context_payout_artifact_sha256 + != published_payout_artifact.prior_balances_sha256 + ) + # Likewise for append-epoch skew: the replacement window covers + # the late-visible row, so the job mining the pre-append window + # must not stay admissible beside it. + or ( + not getattr(context, "collection_only", False) + and int( + getattr(context, "payout_append_invalidation_epoch", 0) + ) + != int( + getattr( + runtime, + "_payout_ledger_append_invalidation_epoch", + 0, + ) + ) + ) + ) + + def maybe_send_job( + self, + client: ClientState, + *, + clean_jobs: bool, + raise_on_reorg_failure: bool = False, + raise_on_build_failure: bool = False, + tip_refresh_snapshot: QbitTipTemplateSnapshot | None = None, + tip_refresh_observation_sequence: int | None = None, + ) -> bool: + runtime = self._runtime + with client.job_update_lock: + return runtime._maybe_send_job_locked( + client, + clean_jobs=clean_jobs, + raise_on_reorg_failure=raise_on_reorg_failure, + raise_on_build_failure=raise_on_build_failure, + tip_refresh_snapshot=tip_refresh_snapshot, + tip_refresh_observation_sequence=tip_refresh_observation_sequence, + ) + + def _maybe_send_job_locked( + self, + client: ClientState, + *, + clean_jobs: bool, + raise_on_reorg_failure: bool = False, + raise_on_build_failure: bool = False, + tip_refresh_snapshot: QbitTipTemplateSnapshot | None = None, + tip_refresh_observation_sequence: int | None = None, + prepared_bundle: CachedJobBundle | None = None, + commit_guard: Callable[[], bool] | None = None, + commit_guard_lock: threading.RLock | None = None, + prepared_bundle_allow_uncached: bool = False, + ) -> bool: + runtime = self._runtime + if not client.subscribed or not client.authorized or client.worker is None: + return False + runtime._ensure_job_cache_state() + started = time.monotonic() + phases = runtime._job_build_phases() + phases.clear() + if getattr(runtime, "hot_path_log_enabled", False): + print( + f"prism coordinator: building job connection={client.connection_id} username={client.username}", + flush=True, + ) + phase_started = time.monotonic() + guarded_refresh = tip_refresh_snapshot is not None + if guarded_refresh != (tip_refresh_observation_sequence is not None): + raise ValueError("tip refresh snapshot and observation sequence must be paired") + if prepared_bundle is not None and guarded_refresh: + raise ValueError("prepared idle bundles cannot be combined with tip refresh guards") + if prepared_bundle_allow_uncached and prepared_bundle is None: + raise ValueError("uncached prepared delivery requires a prepared bundle") + if prepared_bundle is not None: + pass + elif guarded_refresh: + assert tip_refresh_snapshot is not None + assert tip_refresh_observation_sequence is not None + with runtime.lock: + refresh_current = runtime._tip_refresh_snapshot_current_locked( + tip_refresh_snapshot, + tip_refresh_observation_sequence, + ) + if not refresh_current: + runtime._schedule_tip_refresh_retry() + raise TemplateRefreshSuperseded( + "tip refresh snapshot was superseded before client job build" + ) + try: + chain_view_untrusted = bool( + getattr(runtime, "reorg_reconciler_enabled", True) + and runtime.qbit_chain_view_untrusted() + ) + except Exception as exc: + runtime._schedule_tip_refresh_retry() + raise _TipRefreshTrustBlocked( + "qbit chain trust check failed before sequential client job build" + ) from exc + if chain_view_untrusted: + runtime._schedule_tip_refresh_retry() + raise _TipRefreshTrustBlocked( + "qbit chain view became untrusted before sequential client job build" + ) + else: + try: + if not runtime.ensure_reorg_reconciled_for_current_tip(): + if raise_on_reorg_failure: + raise _TipRefreshTrustBlocked( + "qbit chain view became untrusted before client job build" + ) + return False + except AcceptedParentPayoutPreviewPending: + # Propagated verbatim: a late accepted-parent preview must + # never be relabelled into a reorg-reconciliation failure by + # the generic clause below, which would arm the ordinary + # template-refresh failure budget. + raise + except TemplateRefreshBlocked: + raise + except Exception as exc: + print( + f"prism coordinator: reorg reconciliation failed before job build " + f"connection={client.connection_id} username={client.username}; skipping this job", + flush=True, + ) + traceback.print_exc() + if raise_on_reorg_failure: + raise TemplateRefreshBlocked( + "reorg reconciliation failed before client job build" + ) from exc + return False + phases["reorg"] = time.monotonic() - phase_started + built_from_guarded_artifacts = bool( + guarded_refresh + and tip_refresh_snapshot.template_artifacts is not None + and "build_job_for_client" not in runtime.__dict__ + ) + try: + if prepared_bundle is not None: + context = runtime.stamp_job_for_client( + client, + prepared_bundle, + clean_jobs=clean_jobs, + ) + elif built_from_guarded_artifacts: + assert tip_refresh_snapshot is not None + assert tip_refresh_snapshot.template_artifacts is not None + context = runtime.build_job_for_client_from_artifacts( + client, + tip_refresh_snapshot.template_artifacts, + clean_jobs=clean_jobs, + publication_critical=True, + request_source="tip_refresh", + ) + else: + context = runtime.build_job_for_client(client, clean_jobs=clean_jobs) + except AcceptedParentPayoutPreviewPending as exc: + # Same coalescing retry as any other blocked build, but typed all + # the way up: the caller's coordination-blocked handling keeps it + # out of ``job_build_failure_count`` and the ordinary + # template-refresh failure budget, while its own longer streak + # deadline preserves the existing watchdog/health escalation. + runtime._schedule_tip_refresh_retry() + self._log_accepted_parent_preview_pending( + exc, + connection_id=client.connection_id, + ) + if guarded_refresh or raise_on_reorg_failure or raise_on_build_failure: + raise + return False + except TemplateRefreshBlocked: + runtime._schedule_tip_refresh_retry() + if guarded_refresh or raise_on_reorg_failure or raise_on_build_failure: + raise + return False + except Exception as exc: + # A single bad template (e.g. a coinbase whose bytes collide with the + # extranonce placeholder, or a transient getblocktemplate failure) must + # never tear down the miner's connection. Log it, count it, and skip + # this job; the next share/retarget or block change rebuilds a fresh one. + # Only the build is isolated: nothing has been registered or sent yet, so + # there is no stale job state. Downstream send failures still surface to + # handle_client, which disconnects the (now dead) socket and cleans up. + with runtime.lock: + runtime.job_build_failure_count += 1 + print( + f"prism coordinator: job build failed connection={client.connection_id} " + f"username={client.username}; keeping client connected and skipping this template", + flush=True, + ) + traceback.print_exc() + if raise_on_build_failure: + raise JobBuildFailed( + f"job build failed for connection {client.connection_id}" + ) from exc + return False + # Linearize direct delivery against the immutable publication pointer. + # Expensive build and ledger reads happened under the preparation lock, + # outside this admission boundary. + with runtime._job_cache_lock: + current_payout_generation = runtime._payout_state_generation + current_append_invalidation_epoch = ( + runtime._payout_ledger_append_invalidation_epoch + ) + published_tip = runtime._published_payout_state.source_tip_hash + publication_blocked = runtime._payout_state_publication_blocked + context_payout_generation = int( + getattr( + context, + "payout_state_generation", + current_payout_generation, + ) + ) + context_append_invalidation_epoch = int( + getattr(context, "payout_append_invalidation_epoch", 0) + ) + context_collection_only = bool( + getattr(context, "collection_only", False) + ) + context_payout_artifact_sha256 = getattr( + context, + "payout_artifact_sha256", + None, + ) + + def context_payout_digest_stale() -> bool: + # A periodic self-check repair swaps the published balance + # snapshot in place: the payout generation and append epoch both + # survive the swap, so only the digest identifies a context still + # keyed to the refuted balances. Contexts without a digest defer + # to the generation fence. Unlocked at the gate (a cancellation + # hint); authoritative when re-read under _job_cache_lock at the + # commit boundary below. + if context_payout_artifact_sha256 is None: + return False + published_artifact = runtime._published_payout_state.artifact + return ( + published_artifact is None + or context_payout_artifact_sha256 + != published_artifact.prior_balances_sha256 + ) + + context_template = getattr(context, "template", None) + context_parent = ( + str(context_template.get("previousblockhash", "")) + if isinstance(context_template, dict) + else "" + ) + with runtime.lock: + published_authority = getattr(runtime, "current_tip_first_seen", None) + published_authoritative = runtime._published_tip_authoritative_locked( + time.monotonic() + ) + pinned_published_delivery = bool( + context_parent + and published_authority is not None + and context_parent == published_authority[0] + and published_authoritative + ) + lapsed_live_validated = False + if ( + not guarded_refresh + and context_parent + and published_authority is not None + and not published_authoritative + ): + # The published authority lapsed, so per-share classification has + # fallen back to the live RPC read. Mirror the initial-job path: + # revalidate against the live tip (outside every lock), record + # the observation for the refresh machinery, and drop work that + # would be stale on arrival. + try: + lapsed_live_tip = str(runtime.rpc.call("getbestblockhash")) + except Exception: + lapsed_live_tip = None + if lapsed_live_tip is not None: + runtime.observe_tip_for_refresh(lapsed_live_tip) + if context_parent != lapsed_live_tip: + runtime._schedule_tip_refresh_retry() + return False + lapsed_live_validated = True + priority_delivery = ( + not publication_blocked + and context_payout_generation == current_payout_generation + and ( + context_collection_only + or context_append_invalidation_epoch + == current_append_invalidation_epoch + ) + and ( + published_tip is None + or context_parent == published_tip + # Reconciliation can source payout state at the detected tip + # before submit authority flips. Pinned published-tip work is + # still the currently creditable work, so it must keep the + # priority lane instead of tripping the gate's non-priority + # same-generation rejection for the whole unpublished window. + or pinned_published_delivery + ) + ) + payout_gate_started = time.monotonic() + with runtime._payout_state_delivery_gate.delivery_cancelable( + lambda: ( + context_payout_generation != runtime._payout_state_generation + or context_payout_digest_stale() + or ( + not context_collection_only + and context_append_invalidation_epoch + != runtime._payout_ledger_append_invalidation_epoch + ) + ), + generation=context_payout_generation, + priority=priority_delivery, + ) as payout_admitted: + payout_gate_wait = max(0.0, time.monotonic() - payout_gate_started) + phases["payout_gate"] = phases.get("payout_gate", 0.0) + payout_gate_wait + runtime._observe_payout_gate_admission( + payout_admitted, + generation=context_payout_generation, + fallback_wait_seconds=payout_gate_wait, + ) + if not payout_admitted: + runtime._schedule_tip_refresh_retry() + if guarded_refresh: + raise TemplateRefreshSuperseded( + "payout state changed during client job build" + ) + return False + def commit_context_locked() -> bool: + if getattr(client, "closing", False): + return False + if commit_guard is not None and not commit_guard(): + return False + if guarded_refresh: + assert tip_refresh_snapshot is not None + assert tip_refresh_observation_sequence is not None + if not runtime._tip_refresh_snapshot_current_locked( + tip_refresh_snapshot, + tip_refresh_observation_sequence, + ): + runtime._schedule_tip_refresh_retry() + raise TemplateRefreshSuperseded( + "tip refresh snapshot was superseded during client job build" + ) + artifacts = tip_refresh_snapshot.template_artifacts + if built_from_guarded_artifacts and artifacts is not None and ( + context.template is not artifacts.template + or context.template_fingerprint != artifacts.fingerprint + or context.template_generation != artifacts.generation + ): + raise TemplateRefreshBlocked( + "client job build did not use the guarded refresh artifacts" + ) + else: + published_now = getattr(runtime, "current_tip_first_seen", None) + if published_now is None: + # Bootstrap: nothing published yet, so there is no + # authority for this work to contradict. + pass + elif runtime._published_tip_authoritative_locked( + time.monotonic() + ): + if context_parent and context_parent != published_now[0]: + # Authority moved while this direct build waited + # on the payout gate or client lock. Sending now + # would advertise work that classifies stale-job + # on arrival; skip and let the refresh fanout (or + # the pending retry) deliver current work. + runtime._schedule_tip_refresh_retry() + return False + elif context_parent and not lapsed_live_validated: + # The lease lapsed during the wait and this context + # never passed a live-tip revalidation: cached + # observations may be blind to the tip mining.submit + # now classifies against. Defer to a fresh pass whose + # pre-wait live check settles it without holding the + # coordinator lock across an RPC. + runtime._schedule_tip_refresh_retry() + return False + if not runtime._admit_client_tip_refresh_epoch_locked( + client, + int(getattr(context, "tip_refresh_epoch_sequence", 0)), + ): + runtime._schedule_tip_refresh_retry() + return False + client.active_job = context + if clean_jobs: + for job_id in client.active_job_ids: + runtime.bury_evicted_job(client, job_id, prune=False) + self.jobs.pop(job_id, None) + client.active_job_ids.clear() + runtime.prune_evicted_job_graveyard(force=False) + self.jobs[context.job.job_id] = context + client.active_job_ids.add(context.job.job_id) + runtime.prune_client_active_jobs(client) + return True + + def commit_context() -> bool: + if prepared_bundle is not None: + # Exact cache identity and the client/window guard are one + # commit boundary. A tip or payout publication that wins + # this race makes the idle task a no-op instead of + # delivering stale work. + with runtime._job_cache_lock: + if not runtime._idle_bundle_current_locked( + client, + prepared_bundle, + allow_uncached=prepared_bundle_allow_uncached, + ): + return False + with runtime.lock: + return commit_context_locked() + with runtime._job_cache_lock: + if context_payout_digest_stale() or ( + not context_collection_only + and context_append_invalidation_epoch + != runtime._payout_ledger_append_invalidation_epoch + ): + return False + with runtime.lock: + return commit_context_locked() + + if commit_guard_lock is None: + committed = commit_context() + else: + # Per-client state is acquired before coordinator admission. + # A busy share can delay only this delivery; it cannot hold + # runtime.lock while the delivery waits for Vardiff state. + with commit_guard_lock: + committed = commit_context() + if not committed: + return False + phase_started = time.monotonic() + runtime.send_job_update(client, context.job) + payout_admitted.mark_delivered() + runtime.apply_job_difficulty(client, context.job) + runtime.note_tip_work_delivered(client, str(context.template["previousblockhash"])) + delivered_monotonic = time.monotonic() + runtime._record_first_payout_delivery( + context_payout_generation, + delivered_monotonic, + ) + runtime._record_progress_delivery( + client, + context, + delivered_monotonic, + ) + runtime._consume_retained_collection_refresh(context) + runtime.note_initial_job_delivered( + client, + validated_current=guarded_refresh, + ) + phases["send"] = delivered_monotonic - phase_started + elapsed = time.monotonic() - started + runtime.observe_job_build_elapsed(elapsed, phases) + if getattr(runtime, "hot_path_log_enabled", False): + phase_report = ",".join( + f"{phase}:{phases[phase]:.3f}" + for phase in PRISM_JOB_BUILD_PHASES + if phase in phases + ) + print( + f"prism coordinator: sent job connection={client.connection_id} username={client.username} " + f"job={context.job.job_id} collection={context.collection_only} elapsed={elapsed:.3f}s " + f"phases={phase_report}", + flush=True, + ) + return True + + def prune_client_active_jobs(self, client: ClientState) -> None: + runtime = self._runtime + for job_id in tuple(client.active_job_ids): + if job_id not in self.jobs: + client.active_job_ids.discard(job_id) + ordered_active_job_ids = [ + job_id for job_id in self.jobs if job_id in client.active_job_ids + ] + while len(ordered_active_job_ids) > MAX_ACTIVE_PRISM_JOBS_PER_CLIENT: + oldest_job_id = ordered_active_job_ids.pop(0) + client.active_job_ids.remove(oldest_job_id) + runtime.bury_evicted_job(client, oldest_job_id) + self.jobs.pop(oldest_job_id, None) + + def apply_job_difficulty(self, client: ClientState, job: direct_stratum.DirectQbitStratumJob) -> None: + runtime = self._runtime + with runtime._client_vardiff_lock(client): + config = ( + client.vardiff_config + or client.listener_vardiff_config + or runtime.vardiff_config + ) + if not config.enabled: + client.share_difficulty = job.share_difficulty + client.pending_share_difficulty = None + return + pending = client.pending_share_difficulty + client.share_difficulty = job.share_difficulty + if pending is not None and job.share_difficulty == pending: + client.pending_share_difficulty = None + + def apply_client_difficulty_requests(self, client: ClientState) -> Decimal | None: + """Specialize the client's difficulty policy from its recorded requests + (password ``d=``/``md=`` and ``mining.suggest_difficulty``), clamped to + the pristine listener bounds. The listener floor always wins: on a + high-diff listener no request can drop a client below the configured + minimum. Explicit ``d=`` outranks a suggestion. Returns the resolved + target difficulty, or None when the client requested nothing.""" + runtime = self._runtime + with runtime._client_vardiff_lock(client): + base = client.listener_vardiff_config or runtime.vardiff_config + requested = ( + client.requested_difficulty + if client.requested_difficulty is not None + else client.suggested_difficulty + ) + if requested is None and client.requested_min_difficulty is None: + # No live requests: drop any stale specialization so the client + # falls back to the pristine listener policy. + client.vardiff_config = None + return None + floor = base.min_difficulty + if client.requested_min_difficulty is not None: + floor = vardiff.clamp( + client.requested_min_difficulty, + base.min_difficulty, + base.max_difficulty, + ) + if requested is None: + requested = client.share_difficulty + target = vardiff.clamp(requested, floor, base.max_difficulty) + client.vardiff_config = dataclass_replace( + base, + min_difficulty=floor, + startup_difficulty=target, + ) + return target + + def advertise_client_difficulty(self, client: ClientState, target: Decimal) -> bool: + """Move a client to an explicitly requested difficulty. + + Before the client can receive jobs the value is applied directly (the + first set_difficulty/notify pair picks it up). Afterwards it uses the + same job-gated pending mechanism as vardiff retargets: the difficulty + is advertised together with the job it applies to, or not at all. + Returns True only when a fresh set_difficulty/notify pair went out, so + callers about to send their own job can skip a duplicate pair.""" + runtime = self._runtime + with client.job_update_lock: + return runtime._advertise_client_difficulty_locked(client, target) + + def _advertise_client_difficulty_locked( + self, + client: ClientState, + target: Decimal, + ) -> bool: + runtime = self._runtime + applied_directly = False + schedule_initial = False + with runtime._client_vardiff_lock(client): + current = client.pending_share_difficulty or client.share_difficulty + if target == current: + return False + if not (client.subscribed and client.authorized) or ( + client.active_job is None and "maybe_send_job" not in runtime.__dict__ + ): + client.share_difficulty = target + client.pending_share_difficulty = None + client.difficulty_generation = int( + getattr(client, "difficulty_generation", 0) + ) + 1 + applied_directly = True + schedule_initial = bool( + client.subscribed + and client.authorized + and client.worker is not None + ) + else: + prior_pending = client.pending_share_difficulty + prior_generation = int( + getattr(client, "difficulty_generation", 0) + ) + advertised_generation = prior_generation + 1 + client.pending_share_difficulty = target + client.difficulty_generation = advertised_generation + if applied_directly: + if schedule_initial: + # A pending first-job request captured the previous difficulty + # generation. Replace it atomically so its cancellation callback + # hands the client slot to current work instead of disconnecting. + runtime.request_initial_job_delivery(client) + return False + runtime._ensure_initial_job_state() + # Lock-protected membership read; the replacement decision itself is + # re-taken exactly under the same lock in schedule_initial_job. + with self._initial_job_admission_lock: + initial_pending = client in self._pending_initial_jobs + if initial_pending: + runtime.request_initial_job_delivery(client) + return False + if not runtime.stop_event.is_set() and runtime.maybe_send_job(client, clean_jobs=True): + return True + with runtime._client_vardiff_lock(client): + if ( + client.pending_share_difficulty == target + and int(getattr(client, "difficulty_generation", 0)) + == advertised_generation + ): + client.pending_share_difficulty = prior_pending + client.difficulty_generation = prior_generation + return False + + def send_job_update( + self, + client: ClientState, + job: direct_stratum.DirectQbitStratumJob, + ) -> None: + runtime = self._runtime + # Preserve instance-level send method replacements used by focused + # tests; normal coordinators use the atomic socket batch below. + if "send_difficulty" in runtime.__dict__ or "send_job" in runtime.__dict__: + runtime.send_difficulty(client, job) + runtime.send_job(client, job) + return + payloads = [ + runtime.difficulty_payload(job.share_difficulty), + runtime.job_payload(job), + ] + shared_params = getattr(job, "notify_shared_params_json", None) + if shared_params is None: + client.send_batch(payloads) + return + # Splice the per-client fields around the fragment serialized once + # per bundle build; at fleet scale the coinb1/coinb2/merkle parts + # dominate the notify and are byte-identical for every client. + notify_line = ( + '{"id": null, "method": "mining.notify", "params": [' + + json.dumps(job.job_id) + + ", " + + shared_params + + ", " + + ("true" if job.clean_jobs else "false") + + "]}" + ) + client.send_batch( + payloads, + preserialized=( + json.dumps(payloads[0]).encode() + + b"\n" + + notify_line.encode() + + b"\n" + ), + ) + + def build_job_for_client(self, client: ClientState, *, clean_jobs: bool) -> PrismJobContext: + runtime = self._runtime + if client.worker is None: + raise self._stratum_error(20, "client is not authorized") + runtime._ensure_job_cache_state() + artifacts = ( + runtime._retained_collection_artifacts() + or runtime.job_issuance_template_artifacts() + ) + return runtime.build_job_for_client_from_artifacts( + client, + artifacts, + clean_jobs=clean_jobs, + ) + + def build_job_for_client_from_artifacts( + self, + client: ClientState, + artifacts: CachedTemplateArtifacts, + *, + clean_jobs: bool, + publication_critical: bool = False, + request_source: str = "routine", + ) -> PrismJobContext: + runtime = self._runtime + runtime._ensure_job_cache_state() + phases = runtime._job_build_phases() + while True: + worker = client.worker + if worker is None: + raise self._stratum_error(20, "client is not authorized") + progress_build_token = runtime._progress_bundle_build_started() + try: + cached_bundle = runtime.shared_job_bundle( + artifacts, + worker, + publication_critical=publication_critical, + request_source=request_source, + ) + finally: + runtime._progress_bundle_build_finished(progress_build_token) + current_worker = client.worker + if not cached_bundle.collection_only or current_worker == worker: + break + # Reauthorization changed a genuine collection input while the + # worker-specific bundle was being built. Re-select the latest + # identity without refetching or discarding the exact artifacts. + stamp_started = time.monotonic() + context = runtime.stamp_job_for_client(client, cached_bundle, clean_jobs=clean_jobs) + phases["stamp"] = phases.get("stamp", 0.0) + (time.monotonic() - stamp_started) + return context + + def _acquire_client_job_lock( + self, + client: ClientState, + cancelled: Callable[[], bool], + ) -> bool: + while not cancelled(): + if client.job_update_lock.acquire(timeout=0.1): + return True + return False + + @contextmanager + def _cancellable_client_job_lock( + self, + client: ClientState, + cancelled: Callable[[], bool], + ) -> Iterator[bool]: + acquired = self._acquire_client_job_lock(client, cancelled) + try: + yield acquired + finally: + if acquired: + client.job_update_lock.release() + + @staticmethod + def _client_has_delivered_work_locked(client: ClientState) -> bool: + """Return whether a socket write completed for any usable job.""" + + return bool( + client.tip_work_delivered is not None + or client._progress_delivered_context is not None + ) + + def _enforce_evicted_same_tip_capacity_locked( + self, + connection_id: int | None = None, + ) -> None: + runtime = self._runtime + connection_ids = ( + (connection_id,) + if connection_id is not None + else tuple(self.evicted_same_tip_by_connection) + ) + per_connection_cap = int(runtime.same_tip_job_retention_per_connection) + for candidate_connection_id in connection_ids: + job_ids = self.evicted_same_tip_by_connection.get(candidate_connection_id) + while job_ids is not None and len(job_ids) > per_connection_cap: + oldest_job_id = next(iter(job_ids)) + self._remove_evicted_job_locked(oldest_job_id) + self.evicted_job_capacity_eviction_counts["connection"] += 1 + job_ids = self.evicted_same_tip_by_connection.get(candidate_connection_id) + + def _evicted_job_expired_locked( + self, + entry: EvictedJobEntry, + *, + now: float, + ) -> tuple[str, bool]: + runtime = self._runtime + job_class = runtime._evicted_job_class_locked(entry) + if job_class == "same_tip": + ttl = float(runtime.same_tip_job_retention_seconds) + return job_class, ttl <= 0 or now - entry.evicted_monotonic > ttl + return job_class, self._stale_grace_entry_expired_locked( + entry, + now=now, + ttl=float( + getattr( + runtime, + "stale_grace_seconds", + DEFAULT_PRISM_STALE_GRACE_SECONDS, + ) + ), + ) + + def _index_evicted_job_locked(self, job_id: str, entry: EvictedJobEntry) -> None: + runtime = self._runtime + self.evicted_jobs_by_connection.setdefault( + entry.connection_id, + OrderedDict(), + )[job_id] = None + if runtime._evicted_job_class_locked(entry) != "same_tip": + return + self.evicted_same_tip_by_connection.setdefault( + entry.connection_id, + OrderedDict(), + )[job_id] = None + self.evicted_same_tip_job_ids[job_id] = None + + def _rebuild_evicted_job_indexes_locked(self) -> None: + runtime = self._runtime + self.evicted_jobs_by_connection = {} + self.evicted_same_tip_by_connection = {} + self.evicted_same_tip_job_ids = OrderedDict() + for job_id, entry in self.evicted_job_graveyard.items(): + self._index_evicted_job_locked(job_id, entry) + self.evicted_job_index_tip_hash = runtime._current_published_tip_hash_locked() + self._enforce_evicted_same_tip_capacity_locked() + + def _remove_evicted_job_locked(self, job_id: str) -> EvictedJobEntry | None: + entry = self.evicted_job_graveyard.pop(job_id, None) + if entry is None: + return None + connection_jobs = self.evicted_jobs_by_connection.get(entry.connection_id) + if connection_jobs is not None: + connection_jobs.pop(job_id, None) + if not connection_jobs: + self.evicted_jobs_by_connection.pop(entry.connection_id, None) + connection_jobs = self.evicted_same_tip_by_connection.get(entry.connection_id) + if connection_jobs is not None: + connection_jobs.pop(job_id, None) + if not connection_jobs: + self.evicted_same_tip_by_connection.pop(entry.connection_id, None) + self.evicted_same_tip_job_ids.pop(job_id, None) + self._disconnected_evicted_job_ids.pop(job_id, None) + return entry + + def _stale_grace_entry_expired_locked( + self, + entry: EvictedJobEntry, + *, + now: float, + ttl: float, + ) -> bool: + runtime = self._runtime + current_tip = runtime._current_published_tip_hash_locked() + first_seen = getattr(runtime, "current_tip_first_seen", None) + if ( + ttl <= 0 + or current_tip is None + or first_seen is None + or str(first_seen[0]) != current_tip + or first_seen[1] is None + ): + return True + + # Submit eligibility is exactly one chain parent behind, so pruning + # must use that same relationship. The prior poll observation can lag + # (for example when authorize/vardiff issued work on an intermediate + # tip), and using it here would drop work submit would still credit. + # Until the parent RPC has populated the cache, retain conservatively; + # submit classification fetches it before granting stale grace. + cached_parent = getattr(runtime, "current_tip_parent", None) + if ( + cached_parent is not None + and cached_parent[0] == current_tip + and entry.previousblockhash != cached_parent[1] + ): + return True + + client = entry.client + if client is not None: + delivered = client.tip_work_delivered + if delivered is None or delivered[0] != current_tip: + # Match stale_grace_deadline_open: prior-tip shares stay in + # flight until this connection receives replacement work. + return False + anchor = delivered[1] + else: + # Disconnect normally removes these entries. Keep legacy/test + # orphan state bounded from the refresh path's tip-flip anchor. + anchor = float(first_seen[1]) + return now - anchor > ttl + + def _enforce_disconnected_evicted_capacity_locked(self) -> None: + runtime = self._runtime + cap = int( + getattr( + runtime, + "disconnected_job_retention", + DEFAULT_PRISM_DISCONNECTED_JOB_RETENTION, + ) + ) + while len(self._disconnected_evicted_job_ids) > max(0, cap): + oldest_job_id = next(iter(self._disconnected_evicted_job_ids)) + self._remove_evicted_job_locked(oldest_job_id) + self.evicted_job_capacity_eviction_counts["disconnected"] += 1 + + def cleanup_disconnected_client(self, client: ClientState) -> None: + """Retire a disconnected client's job/retention state (S1 port). + + The caller (the session owner's disconnect path) already holds + ``client.job_update_lock``; every mixed lock path uses + ``job_update_lock -> coordinator lock`` and this method takes the + coordinator lock itself. + """ + runtime = self._runtime + with runtime.lock: + runtime._ensure_evicted_job_state() + retention_cap = int( + getattr( + runtime, + "disconnected_job_retention", + DEFAULT_PRISM_DISCONNECTED_JOB_RETENTION, + ) + ) + if retention_cap > 0: + # Devices behind a flapping proxy keep mining the + # jobs this connection delivered; bury the active + # ones so their in-flight shares can resume against + # the retained context after the reconnect. + for job_id in tuple(client.active_job_ids): + runtime.bury_evicted_job(client, job_id, prune=False) + for job_id in tuple(client.active_job_ids): + self.jobs.pop(job_id, None) + client.active_job_ids.clear() + client.active_job = None + for job_id in tuple( + self.evicted_jobs_by_connection.get(client.connection_id, ()) + ): + entry = self.evicted_job_graveyard.get(job_id) + if entry is None: + continue + if ( + retention_cap <= 0 + or self._evicted_job_class_locked(entry) != "same_tip" + ): + # Stale-grace work stays tied to the connection + # that received it (its expiry anchors on that + # connection's delivered work); only fully valid + # same-tip contexts survive the disconnect. + self._remove_evicted_job_locked(job_id) + continue + # Detach from the dead connection object: expiry then + # anchors on the entry's own eviction time (same-tip + # TTL) or the published tip flip, never on delivery + # state this connection can no longer advance. + self.evicted_job_graveyard[job_id] = dataclass_replace( + entry, + client=None, + ) + self._disconnected_evicted_job_ids[job_id] = None + self._enforce_disconnected_evicted_capacity_locked() + client.authorized = False + client.worker = None + client.username = "" + + def _cross_connection_evicted_entry_allowed_locked( + self, + entry: EvictedJobEntry, + client: ClientState, + ) -> bool: + """Whether a retained context of a dead connection may serve this + submitter. + + Only same-tip contexts qualify (they are fully valid current work; + stale-grace expiry anchors on the original connection's delivery + state, which a dead connection can never advance), and only for an + authorized reconnect of the same username -- the credit goes to the + context's original worker identity either way, so a foreign + submitter could at most donate work, but refusing keeps job ids + unguessable-by-effect and the accounting per miner honest. Replays + stay rejected by the (username, header) dedup key. + """ + runtime = self._runtime + if entry.client is not None: + # The owning connection is still alive; job ids stay scoped to it. + return False + if ( + int( + getattr( + runtime, + "disconnected_job_retention", + DEFAULT_PRISM_DISCONNECTED_JOB_RETENTION, + ) + ) + <= 0 + ): + return False + if not client.authorized or not client.username: + return False + worker = getattr(entry.context, "worker", None) + username = getattr(worker, "username", None) + if not username or username != client.username: + return False + return runtime._evicted_job_class_locked(entry) == "same_tip" + + +__all__ = [ + "DEFAULT_PRISM_EVICTED_JOB_PRUNE_INTERVAL_SECONDS", + "DEFAULT_PRISM_INITIAL_JOB_MAX_WORKERS", + "EvictedJobEntry", + "JobBuildFailed", + "JobDeliveryRuntime", + "JobDeliveryService", + "MAX_ACTIVE_PRISM_JOBS_PER_CLIENT", + "PRISM_CREDIT_POLICY_STALE_GRACE", + "PRISM_DELIVERY_PRIORITY_INITIAL", + "PRISM_DELIVERY_PRIORITY_NEW_TIP", + "PRISM_DELIVERY_PRIORITY_SAME_TIP", + "PRISM_EVICTED_JOB_CAPACITY_SCOPES", + "PRISM_EVICTED_JOB_CLASSES", + "PRISM_EVICTED_JOB_SUBMIT_OUTCOMES", + "PRISM_TIP_REFRESH_ADMISSION_POLL_SECONDS", + "PendingInitialJob", + "PrismJobContext", +] diff --git a/lab/prism/metrics.py b/lab/prism/metrics.py new file mode 100644 index 000000000..9cb4bd21f --- /dev/null +++ b/lab/prism/metrics.py @@ -0,0 +1,2031 @@ +"""Read-only Prometheus assembly for the PRISM coordinator. + +The renderer owns the complete live/base metrics document and the formatter +bodies PR 80 extracted from the coordinator. It holds no producer state: +counters, histograms, and label registries stay with their owner services +(J1, B1, share submission, P1, reorg, ledger, observability) and reach the +renderer through the :class:`MetricsPort` as copied snapshots or bounded +per-domain ``*_metrics_lines()`` formatters. The renderer never calls the +cached-metrics path, so the snapshot refresher can call it without recursion. +""" + +from __future__ import annotations + +import time +from typing import Any, Protocol + +from lab.prism.accepted_preview_telemetry import ( + PRISM_ACCEPTED_LANDING_PHASES, + PRISM_PAYOUT_WINDOW_FULL_RESCAN_PATHS, + PRISM_PAYOUT_WINDOW_FULL_RESCAN_REASONS, + PRISM_REORG_RECONCILE_CALLERS, + PRISM_REORG_RECONCILE_STEPS, + ensure_accepted_preview_telemetry, + fold_ledger_read_stats, +) +from lab.prism.block_candidates import ( + PRISM_ACCEPTED_BLOCK_PREVIEW_PUBLICATION_RESULTS, + PRISM_ACCEPTED_BLOCK_PREVIEW_PUBLICATION_SECONDS_BUCKETS, + PRISM_BLOCK_CANDIDATE_COLLAPSE_OUTCOMES, + PRISM_STALE_JOB_ABANDON_CLASSES, +) +from lab.prism.coordinator_config import ( + DEFAULT_PRISM_INITIAL_JOB_MAX_WORKERS, + DEFAULT_PRISM_JOB_BUILD_EXECUTOR_WORKERS, + DEFAULT_PRISM_MALLOC_TELEMETRY, + env_malloc_telemetry, +) +from lab.prism.job_bundle import ( + PRISM_JOB_BUILD_PHASES, + PRISM_JOB_BUILD_SECONDS_BUCKETS, + PRISM_JOB_CACHE_KINDS, +) +from lab.prism.job_delivery import ( + PRISM_EVICTED_JOB_CAPACITY_SCOPES, + PRISM_EVICTED_JOB_CLASSES, + PRISM_EVICTED_JOB_SUBMIT_OUTCOMES, +) +from lab.prism.background_services import PRISM_GC_PAUSE_SECONDS_BUCKETS +from lab.prism.process_telemetry import ( + PRISM_GC_GENERATIONS, + ProcessHeapTelemetry, +) +from lab.prism.reorg_reconciler import ( + PRISM_REORG_RECONCILE_LOOKUP_PATHS, + PRISM_REORG_RECONCILE_LOOKUP_SOURCES, +) +from lab.prism.share_ledger import DaemonShareWindowMirror +from lab.prism.share_submission import PRISM_SHARE_ACK_RESULTS +from lab.prism.vardiff_service import PRISM_VARDIFF_RESUME_OUTCOMES +from lab.prism.writer_lease_timing import ( + LEASE_HEARTBEAT_MODES, + LEASE_HEARTBEAT_OUTCOMES, + LEASE_HEARTBEAT_PHASES, + LEASE_HEARTBEAT_POLICY_TERMS, + LEASE_MONITOR_LATE_WAKE_SLACK_FRACTIONS, + LEASE_MONITOR_WAKE_DELAY_BUCKETS, +) + +# Issue #226 (and the gauge list issue #185 asked for): the closed component +# label set of qbit_prism_component_entries. Every entry is one len() or +# Queue.qsize() over a structure its owner constructs in __init__ (or +# backfills through its own _ensure_* method), so a scrape costs a fixed +# number of constant-time reads and the series set cannot grow with jobs, +# tips, candidates, connections, or generations. Extend this tuple +# deliberately: tests/test_prism_metrics.py pins it as the family's exact +# series set. +PRISM_COMPONENT_ENTRY_KINDS = ( + # The payout window: the in-process incremental window's pages and + # records, the armed ledger artifact's share sequence, and the + # payout-state maps that are keyed by hash, epoch, or token. + "payout_window_pages", + "payout_window_records", + "payout_ledger_artifact_shares", + "accepted_block_payout_previews", + "invalidated_accepted_block_payout_previews", + "payout_append_invalidation_stamps", + "payout_window_inflight_scan_anchors", + "payout_unfenced_append_inflight_stamps", + # The single-slot share-window serialization cache: shares it covers. + "share_window_serialization_shares", + # Job contexts and the bundle cache. + "job_contexts", + "job_bundle_cache", + "job_build_issued_stamps", + "bundle_preparation_flights", + "active_job_bundle_builds", + # The evicted-job graveyard. + "evicted_job_graveyard", + "evicted_same_tip_job_ids", + # Candidate and replay state: the live, replay, and quarantine lanes + # plus every hash-keyed registry the B1 owner retains. + "block_candidate_queue", + "block_replay_queue", + "block_replay_inflight_hashes", + "block_quarantine_queue", + "block_quarantine_hashes", + "outstanding_block_candidate_hashes", + "tip_observed_accepted_block_hashes", + "counted_block_candidate_abandonments", + "accepted_block_preview_stamps", + "ancestor_redrive_records", + "block_candidate_terminal_outcomes", + "block_candidate_disposition_flights", + "block_disposition_waiting_retries", + "block_candidate_dequeued_hashes", + "accounted_accepted_block_hashes", + # Reconcile flights and the trusted-tip memo. + "reconcile_flights", + "reconcile_trusted_memo", + # Pending-share holders on the snapshot-anchor floor. + "pending_share_commit_floor", + # The daemon mirror: records in a mirror-backed window, and the + # compiler's bounded mirror of the daemon's uploaded-window LRU. + "daemon_window_mirror_records", + "daemon_uploaded_windows", +) +# The closed component label set of qbit_prism_component_bytes: the retained +# byte-sized payloads whose size, not count, is what scales with the window. +PRISM_COMPONENT_BYTE_KINDS = ( + "payout_window_canonical_json", + "daemon_window_mirror_canonical_items", + "share_window_serialization_spool", + "share_window_serialization_compact_json", +) + + +class MetricsPort(Protocol): + """Explicit read capabilities used to assemble one metrics document. + + The renderer is observational but not pure: it still performs live + RPC/ledger reads and calls bounded retained-job pruning, and several + ``_ensure_*`` calls can initialize owner state on first touch. The + issue #224 attribution owner is likewise attached to the port's + instance ``__dict__`` on first render + (``ensure_accepted_preview_telemetry``), so the port needs an + instance dictionary but no dedicated attribute or method for it. + """ + + ledger: Any + rpc: Any + lock: Any + clients: Any + connection_limit_rejection_counts: Any + evicted_job_capacity_eviction_counts: Any + evicted_job_expiration_counts: Any + evicted_job_graveyard: Any + evicted_job_submit_counts: Any + evicted_same_tip_job_ids: Any + worker_metrics_lock: Any + worker_share_counts: Any + worker_rejection_counts: Any + accepted_block_count: Any + matured_payout_count: Any + post_accept_refresh_failure_count: Any + reorg_inactive_block_count: Any + reorg_reactivated_block_count: Any + reorg_reconcile_error_count: Any + reorg_reconcile_skip_count: Any + reorg_reconcile_lookup_counts: Any + started_monotonic: Any + tip_refresh_job_count: Any + vardiff_config: Any + listener_profiles: Any + accept_resource_exhaustion_count: Any + block_candidate_abandoned_counts: Any + block_candidate_accept_pending_defer_count: Any + block_candidate_poisoned_count: Any + block_candidate_retry_count: Any + block_candidate_wakeups_coalesced: Any + block_candidates_dropped: Any + block_solves_dropped_counts: Any + stale_job_abandon_counts: Any + connection_setup_failure_count: Any + idle_retarget_count: Any + jobs: Any + jobs_lock: Any + job_build_failure_count: Any + latest_coinbase_size_bytes: Any + share_append_queue: Any + share_append_failure_count: Any + shares_recovered_to_disk: Any + shares_replayed: Any + share_replay_conflicts: Any + initial_job_sent_count: Any + initial_job_cancelled_count: Any + initial_job_coalesced_count: Any + initial_job_failed_count: Any + initial_job_superseded_count: Any + initial_job_delivery_latency_seconds_sum: Any + initial_job_delivery_latency_count: Any + initial_job_queue_capacity_reclaimed_count: Any + initial_job_max_workers: Any + _initial_job_executor: Any + _bundle_preparation_lock: Any + shared_bundle_build_counts: Any + shared_bundle_preparation_seconds_sum: Any + shared_bundle_preparation_count: Any + shared_bundle_preparation_waiters: Any + _job_cache_lock: Any + job_build_seconds_bucket_counts: Any + job_build_seconds_sum: Any + job_build_count: Any + job_build_phase_seconds: Any + job_cache_hit_counts: Any + job_cache_miss_counts: Any + _job_build_scheduler_lock: Any + job_build_scheduler_counts: Any + job_build_priority_counts: Any + job_build_priority_admission_seconds: Any + initial_job_prepared_work_counts: Any + job_build_cancellation_seconds: Any + job_build_replacement_start_seconds: Any + job_build_worker_counts: Any + job_build_executor_workers: Any + _job_build_active: Any + _job_build_pending: Any + _job_build_retiring: Any + _job_build_priority_preparations: Any + _accepted_parent_preview_wait_timeouts: Any + accepted_parent_redrive_attempt_count: Any + accepted_parent_redrive_resolved_count: Any + accepted_parent_redrive_exhausted_count: Any + _accounted_accepted_block_hashes: Any + + def _accepted_parent_unresolved_depth(self) -> int: ... + def _accepted_parent_unresolved_depth_cap(self) -> int: ... + def _ensure_block_candidate_service(self) -> Any: ... + def _ensure_bundle_compiler(self) -> Any: ... + def _ensure_connection_capacity_state(self) -> Any: ... + def _ensure_evicted_job_state(self) -> Any: ... + def _ensure_initial_job_state(self) -> Any: ... + def _ensure_job_bundle_service(self) -> Any: ... + def _ensure_job_cache_state(self) -> Any: ... + def _ensure_observability_service(self) -> Any: ... + def _ensure_lease_heartbeat_service(self) -> Any: ... + def _ensure_payout_state_service(self) -> Any: ... + def _ensure_share_writer_service(self) -> Any: ... + def _ensure_shutdown_controller(self) -> Any: ... + def _ensure_worker_metrics_state(self) -> Any: ... + def _job_build_is_publication_critical(self, request: Any) -> bool: ... + def accepted_share_stats(self) -> Any: ... + def accepted_parent_unresolved_ages_seconds(self) -> list[float]: ... + def audit_artifact_metrics(self) -> Any: ... + def block_candidate_collapse_snapshot(self) -> dict[str, int]: ... + def block_finalization_metrics_lines(self) -> Any: ... + def block_ledger_call_class_metrics(self) -> Any: ... + def block_submitter_snapshot(self) -> dict[str, object]: ... + def coordinator_lock_contention_snapshot(self) -> tuple[int, int, float, float]: ... + def ctv_fanout_broadcaster_metrics_lines(self) -> Any: ... + def mining_delivery_snapshot(self) -> Any: ... + def payout_state_metrics_lines(self) -> Any: ... + def process_resource_metrics(self) -> Any: ... + def progress_health_metrics_lines(self) -> Any: ... + def prometheus_label_value(self, value: str) -> str: ... + def prune_evicted_job_graveyard(self, *, force: bool) -> Any: ... + def rejection_reason_ids(self) -> tuple[str, ...]: ... + def share_accounting_snapshot(self) -> dict[str, object]: ... + def share_ack_snapshot(self) -> dict[str, dict[str, Any]]: ... + def startup_phase_seconds(self) -> dict[str, float]: ... + def tip_refresh_metrics_lines(self) -> Any: ... + def vardiff_convergence_snapshot(self) -> Any: ... + def vardiff_idle_metrics_lines(self) -> Any: ... + + +class MetricsRenderer: + """Collect and format one complete metrics generation.""" + + def __init__( + self, + port: MetricsPort, + *, + process_telemetry: ProcessHeapTelemetry | None = None, + ) -> None: + self.port = port + if process_telemetry is None: + # Resolution order, most authoritative first. A port attribute + # lets an embedder or a test pin the switch outright. Otherwise + # the validated lifecycle snapshot the coordinator was built with + # wins: a caller that passed an explicit CoordinatorConfig with + # telemetry disabled must not have mallinfo2 run anyway because + # the ambient environment happens to say otherwise. Only with no + # snapshot at all do we read the environment. Resolved once here + # so a scrape never touches any of them. + enabled = getattr(port, "malloc_telemetry_enabled", None) + if enabled is None: + config = getattr(port, "config", None) + lifecycle = getattr(config, "lifecycle", None) + enabled = getattr(lifecycle, "malloc_telemetry_enabled", None) + if enabled is None: + try: + enabled = env_malloc_telemetry() + except BaseException: + # The config loader validated this switch at startup, so + # reaching here means the environment changed underneath a + # running process. env_malloc_telemetry is fail-closed and + # exits on a bad value, which is right at startup and wrong + # on a metrics thread: killing the coordinator over a + # telemetry switch would turn an observability knob into an + # outage. Fall back to the shipped default instead. + enabled = DEFAULT_PRISM_MALLOC_TELEMETRY + process_telemetry = ProcessHeapTelemetry(malloc_enabled=bool(enabled)) + self._process_telemetry = process_telemetry + + def render(self) -> str: + ledger_metrics = self.port.ledger.metrics() + audit_metrics = self.port.audit_artifact_metrics() + mining_metrics = self.port.mining_delivery_snapshot() + process_rss_bytes, process_open_fds = self.port.process_resource_metrics() + accepted_share_count = self.port.accepted_share_stats()[0] + share_accounting = self.port.share_accounting_snapshot() + submitted_share_count = int(share_accounting["submitted"]) + stale_share_count = int(share_accounting["stale"]) + duplicate_share_count = int(share_accounting["duplicate"]) + low_difficulty_share_count = int(share_accounting["low_difficulty"]) + collection_block_submission_count = int( + share_accounting["collection_block"] + ) + rejection_counts = share_accounting["rejections"] + assert isinstance(rejection_counts, dict) + grace_credited_share_count = int(share_accounting["grace_credited"]) + # The submission owner's copied snapshot (via the routing descriptor) + # is read outside the accounting lock so owner locks never nest. + block_solves_dropped_counts = dict( + getattr( + self.port, + "block_solves_dropped_counts", + {"stale_grace": 0}, + ) + ) + stale_percent = 0.0 + if submitted_share_count > 0: + stale_percent = (stale_share_count / submitted_share_count) * 100.0 + idle_retarget_count = int(getattr(self.port, "idle_retarget_count", 0)) + with self.port.lock: + self.port._ensure_connection_capacity_state() + active_connection_count = len(self.port.clients) + connection_limit_rejection_counts = dict( + self.port.connection_limit_rejection_counts + ) + accept_resource_exhaustion_count = int( + getattr(self.port, "accept_resource_exhaustion_count", 0) + ) + connection_setup_failure_count = int( + getattr(self.port, "connection_setup_failure_count", 0) + ) + self.port._ensure_evicted_job_state() + self.port.prune_evicted_job_graveyard(force=False) + same_tip_context_count = len(self.port.evicted_same_tip_job_ids) + evicted_job_context_counts = { + "same_tip": same_tip_context_count, + "stale_grace": len(self.port.evicted_job_graveyard) - same_tip_context_count, + } + evicted_job_submit_counts = dict(self.port.evicted_job_submit_counts) + evicted_job_expiration_counts = dict(self.port.evicted_job_expiration_counts) + evicted_job_capacity_eviction_counts = dict( + self.port.evicted_job_capacity_eviction_counts + ) + stale_job_abandon_counts = dict( + getattr( + self.port, + "stale_job_abandon_counts", + { + abandon_class: 0 + for abandon_class in PRISM_STALE_JOB_ABANDON_CLASSES + }, + ) + ) + self.port._ensure_worker_metrics_state() + with self.port.worker_metrics_lock: + worker_share_counts = { + label: dict(counts) + for label, counts in self.port.worker_share_counts.items() + } + worker_rejection_counts = dict(self.port.worker_rejection_counts) + coinbase_weight_headroom = 2_000_000 + latest_coinbase_size_bytes = getattr( + self.port, "latest_coinbase_size_bytes", None + ) + if latest_coinbase_size_bytes is not None: + coinbase_weight_headroom = 2_000_000 - int(latest_coinbase_size_bytes) + ctv_pending = 0 + ctv_broadcastable = 0 + ctv_failed = 0 + pending_ctv_fanouts = getattr( + self.port.ledger, "pending_ctv_fanout_statuses", None + ) + if callable(pending_ctv_fanouts): + try: + for fanout in pending_ctv_fanouts(limit=1_000): + ctv_pending += 1 + status = str(fanout.get("settlement_status", "")) + if status == "broadcastable": + ctv_broadcastable += 1 + elif status == "failed": + ctv_failed += 1 + except Exception: + ctv_pending = -1 + ctv_broadcastable = -1 + ctv_failed = -1 + if ctv_failed >= 0: + ctv_failed = int(ledger_metrics.get("ctv_fanouts_failed", ctv_failed)) + ibd = 0 + peers = 0 + try: + blockchain_info = self.port.rpc.call("getblockchaininfo") + if isinstance(blockchain_info, dict) and blockchain_info.get("initialblockdownload"): + ibd = 1 + except Exception: + ibd = -1 + try: + network_info = self.port.rpc.call("getnetworkinfo") + if isinstance(network_info, dict): + peers = int(network_info.get("connections", 0)) + except Exception: + peers = -1 + rejection_reason_ids = self.port.rejection_reason_ids() + lines = [ + "# HELP qbit_prism_accepted_shares_total Accepted shares recorded by the canonical PRISM ledger.", + "# TYPE qbit_prism_accepted_shares_total counter", + f"qbit_prism_accepted_shares_total {accepted_share_count}", + *self.accepted_stats_reconcile_metric_lines(), + *self.landing_observability_metrics_lines(), + "# HELP qbit_prism_submitted_shares_total Stratum share submissions seen by the PRISM coordinator.", + "# TYPE qbit_prism_submitted_shares_total counter", + f"qbit_prism_submitted_shares_total {submitted_share_count}", + "# HELP qbit_prism_stratum_active_connections Active admitted Stratum connections across all listeners.", + "# TYPE qbit_prism_stratum_active_connections gauge", + f"qbit_prism_stratum_active_connections {active_connection_count}", + "# HELP qbit_prism_stratum_connection_limit Configured global Stratum connection ceiling; zero means unlimited.", + "# TYPE qbit_prism_stratum_connection_limit gauge", + f"qbit_prism_stratum_connection_limit {mining_metrics['connection_capacity']}", + "# HELP qbit_prism_stratum_peak_active_connections Peak admitted Stratum connections since process start.", + "# TYPE qbit_prism_stratum_peak_active_connections gauge", + f"qbit_prism_stratum_peak_active_connections {mining_metrics['peak_active_connections']}", + "# HELP qbit_prism_stratum_subscribed_connections Active subscribed Stratum connections.", + "# TYPE qbit_prism_stratum_subscribed_connections gauge", + f"qbit_prism_stratum_subscribed_connections {mining_metrics['subscribed_connections']}", + "# HELP qbit_prism_stratum_authorized_connections Active subscribed and authorized Stratum connections.", + "# TYPE qbit_prism_stratum_authorized_connections gauge", + f"qbit_prism_stratum_authorized_connections {mining_metrics['authorized_connections']}", + "# HELP qbit_prism_stratum_pending_initial_jobs Authorized clients awaiting their first usable current-tip job.", + "# TYPE qbit_prism_stratum_pending_initial_jobs gauge", + f"qbit_prism_stratum_pending_initial_jobs {mining_metrics['pending_initial_jobs']}", + "# HELP qbit_prism_stratum_pending_initial_job_limit Configured bound for clients awaiting their first usable job.", + "# TYPE qbit_prism_stratum_pending_initial_job_limit gauge", + f"qbit_prism_stratum_pending_initial_job_limit {mining_metrics['pending_initial_job_capacity']}", + "# HELP qbit_prism_stratum_oldest_pending_initial_job_seconds Age of the oldest pending first-job request.", + "# TYPE qbit_prism_stratum_oldest_pending_initial_job_seconds gauge", + f"qbit_prism_stratum_oldest_pending_initial_job_seconds {mining_metrics['oldest_pending_initial_job_age_seconds']}", + "# HELP qbit_prism_stratum_oldest_genuinely_pending_initial_job_seconds Age of the oldest authorized client that has never received usable work.", + "# TYPE qbit_prism_stratum_oldest_genuinely_pending_initial_job_seconds gauge", + f"qbit_prism_stratum_oldest_genuinely_pending_initial_job_seconds {mining_metrics['oldest_genuinely_pending_initial_job_age_seconds']}", + "# HELP qbit_prism_stratum_current_tip_coverage_gap_seconds Continuous age of current-tip job coverage below 95 percent.", + "# TYPE qbit_prism_stratum_current_tip_coverage_gap_seconds gauge", + f"qbit_prism_stratum_current_tip_coverage_gap_seconds {mining_metrics['current_tip_coverage_gap_age_seconds']}", + "# HELP qbit_prism_stratum_initial_job_queue_rejections_total Sessions closed because bounded first-job delivery was full.", + "# TYPE qbit_prism_stratum_initial_job_queue_rejections_total counter", + f"qbit_prism_stratum_initial_job_queue_rejections_total {mining_metrics['initial_job_queue_rejections']}", + "# HELP qbit_prism_stratum_initial_job_timeouts_total Sessions disconnected after first-job delivery timed out.", + "# TYPE qbit_prism_stratum_initial_job_timeouts_total counter", + f"qbit_prism_stratum_initial_job_timeouts_total {mining_metrics['initial_job_timeout_disconnects']}", + "# HELP qbit_prism_stratum_initial_job_tasks_total First-job tasks canceled or coalesced before duplicate work.", + "# TYPE qbit_prism_stratum_initial_job_tasks_total counter", + f'qbit_prism_stratum_initial_job_tasks_total{{result="cancelled"}} {mining_metrics["initial_job_cancelled_tasks"]}', + f'qbit_prism_stratum_initial_job_tasks_total{{result="coalesced"}} {mining_metrics["initial_job_coalesced_tasks"]}', + "# HELP qbit_prism_stratum_initial_job_queue_capacity_reclaimed_total Queued first-job admission slots reclaimed immediately by cancellation.", + "# TYPE qbit_prism_stratum_initial_job_queue_capacity_reclaimed_total counter", + f'qbit_prism_stratum_initial_job_queue_capacity_reclaimed_total {mining_metrics["initial_job_queue_capacity_reclaimed"]}', + "# HELP qbit_prism_stratum_clients_with_current_tip_jobs Authorized clients holding usable current-tip work.", + "# TYPE qbit_prism_stratum_clients_with_current_tip_jobs gauge", + f"qbit_prism_stratum_clients_with_current_tip_jobs {mining_metrics['clients_with_current_tip_jobs']}", + "# HELP qbit_prism_stratum_current_tip_job_coverage Ratio of authorized clients holding current-tip work.", + "# TYPE qbit_prism_stratum_current_tip_job_coverage gauge", + f"qbit_prism_stratum_current_tip_job_coverage {mining_metrics['current_tip_job_coverage']}", + "# HELP qbit_prism_stratum_semantic_current_work_ratio Ratio of authorized clients whose work matches the current template fingerprint, payout generation, and client session.", + "# TYPE qbit_prism_stratum_semantic_current_work_ratio gauge", + f"qbit_prism_stratum_semantic_current_work_ratio {mining_metrics['semantic_current_work_ratio']}", + "# HELP qbit_prism_stratum_handler_threads Active per-connection Stratum handler threads.", + "# TYPE qbit_prism_stratum_handler_threads gauge", + f"qbit_prism_stratum_handler_threads {mining_metrics['handler_threads']}", + "# HELP qbit_prism_job_delivery_queue_depth Current bounded delivery executor queue depth.", + "# TYPE qbit_prism_job_delivery_queue_depth gauge", + f"qbit_prism_job_delivery_queue_depth {mining_metrics['delivery_executor_queue_depth']}", + "# HELP qbit_prism_job_delivery_active_workers Delivery executor workers currently running tasks.", + "# TYPE qbit_prism_job_delivery_active_workers gauge", + f"qbit_prism_job_delivery_active_workers {mining_metrics['delivery_executor_active_workers']}", + "# HELP qbit_prism_process_resident_memory_bytes Current process RSS bytes, or -1 when unavailable.", + "# TYPE qbit_prism_process_resident_memory_bytes gauge", + f"qbit_prism_process_resident_memory_bytes {process_rss_bytes}", + "# HELP qbit_prism_process_open_file_descriptors Current process open descriptor count, or -1 when unavailable.", + "# TYPE qbit_prism_process_open_file_descriptors gauge", + f"qbit_prism_process_open_file_descriptors {process_open_fds}", + "# HELP qbit_prism_stratum_connection_limit_rejections_total Stratum connections rejected by an explicitly configured admission limit.", + "# TYPE qbit_prism_stratum_connection_limit_rejections_total counter", + *[ + f'qbit_prism_stratum_connection_limit_rejections_total{{scope="{scope}"}} {int(connection_limit_rejection_counts.get(scope, 0))}' + for scope in ("global", "username") + ], + "# HELP qbit_prism_stratum_accept_resource_exhaustions_total Recoverable Stratum accept or client-setup failures caused by process or system descriptor exhaustion.", + "# TYPE qbit_prism_stratum_accept_resource_exhaustions_total counter", + f"qbit_prism_stratum_accept_resource_exhaustions_total {accept_resource_exhaustion_count}", + "# HELP qbit_prism_stratum_connection_setup_failures_total Admitted Stratum connections cleaned up after socket or handler-thread setup failure.", + "# TYPE qbit_prism_stratum_connection_setup_failures_total counter", + f"qbit_prism_stratum_connection_setup_failures_total {connection_setup_failure_count}", + "# HELP qbit_prism_stale_shares_total Stratum shares rejected or ignored as stale.", + "# TYPE qbit_prism_stale_shares_total counter", + f"qbit_prism_stale_shares_total {stale_share_count}", + "# HELP qbit_prism_duplicate_shares_total Duplicate Stratum shares rejected.", + "# TYPE qbit_prism_duplicate_shares_total counter", + f"qbit_prism_duplicate_shares_total {duplicate_share_count}", + "# HELP qbit_prism_low_difficulty_shares_total Low-difficulty Stratum shares rejected.", + "# TYPE qbit_prism_low_difficulty_shares_total counter", + f"qbit_prism_low_difficulty_shares_total {low_difficulty_share_count}", + "# HELP qbit_prism_collection_block_submissions_total Solver-pays-all block candidates submitted from collection-mode jobs.", + "# TYPE qbit_prism_collection_block_submissions_total counter", + f"qbit_prism_collection_block_submissions_total {collection_block_submission_count}", + "# HELP qbit_prism_grace_credited_shares_total Accepted shares credited by the stale-grace policy.", + "# TYPE qbit_prism_grace_credited_shares_total counter", + f"qbit_prism_grace_credited_shares_total {grace_credited_share_count}", + "# HELP qbit_prism_block_solves_dropped_total Block-passing submissions intentionally excluded from block submission by a bounded policy reason.", + "# TYPE qbit_prism_block_solves_dropped_total counter", + *[ + f'qbit_prism_block_solves_dropped_total{{reason="{reason}"}} {int(block_solves_dropped_counts.get(reason, 0))}' + for reason in ("stale_grace",) + ], + "# HELP qbit_prism_rejections_total PRISM share or block rejections by canonical reason ID.", + "# TYPE qbit_prism_rejections_total counter", + *[ + f'qbit_prism_rejections_total{{reason_id="{reason}"}} {int(rejection_counts.get(reason, 0))}' + for reason in rejection_reason_ids + ], + "# HELP qbit_prism_worker_submitted_shares_total Stratum share submissions by bounded worker label.", + "# TYPE qbit_prism_worker_submitted_shares_total counter", + *[ + f'qbit_prism_worker_submitted_shares_total{{worker="{self.port.prometheus_label_value(label)}"}} {int(counts.get("submitted", 0))}' + for label, counts in sorted(worker_share_counts.items()) + ], + "# HELP qbit_prism_worker_accepted_shares_total Accepted shares by bounded worker label.", + "# TYPE qbit_prism_worker_accepted_shares_total counter", + *[ + f'qbit_prism_worker_accepted_shares_total{{worker="{self.port.prometheus_label_value(label)}"}} {int(counts.get("accepted", 0))}' + for label, counts in sorted(worker_share_counts.items()) + ], + "# HELP qbit_prism_worker_grace_credited_shares_total Stale-grace credited shares by bounded worker label.", + "# TYPE qbit_prism_worker_grace_credited_shares_total counter", + *[ + f'qbit_prism_worker_grace_credited_shares_total{{worker="{self.port.prometheus_label_value(label)}"}} {int(counts.get("grace", 0))}' + for label, counts in sorted(worker_share_counts.items()) + ], + "# HELP qbit_prism_worker_rejections_total PRISM share or block rejections by bounded worker label and reason ID.", + "# TYPE qbit_prism_worker_rejections_total counter", + *[ + f'qbit_prism_worker_rejections_total{{worker="{self.port.prometheus_label_value(label)}",reason_id="{reason}"}} {int(count)}' + for (label, reason), count in sorted(worker_rejection_counts.items()) + ], + "# HELP qbit_prism_job_build_failures_total Job builds skipped after a template/coinbase error without dropping the client.", + "# TYPE qbit_prism_job_build_failures_total counter", + f"qbit_prism_job_build_failures_total {self.port.job_build_failure_count}", + "# HELP qbit_prism_block_candidates_dropped_total Legacy counter; durable candidate outbox rows are never dropped on queue overflow.", + "# TYPE qbit_prism_block_candidates_dropped_total counter", + f"qbit_prism_block_candidates_dropped_total {int(getattr(self.port, 'block_candidates_dropped', 0))}", + "# HELP qbit_prism_block_candidate_wakeups_coalesced_total Candidate queue wakeups coalesced while the durable outbox retained the work.", + "# TYPE qbit_prism_block_candidate_wakeups_coalesced_total counter", + f"qbit_prism_block_candidate_wakeups_coalesced_total {int(getattr(self.port, 'block_candidate_wakeups_coalesced', 0))}", + "# HELP qbit_prism_block_candidate_retries_total Transient candidate outcomes retained for durable retry.", + "# TYPE qbit_prism_block_candidate_retries_total counter", + f"qbit_prism_block_candidate_retries_total {int(getattr(self.port, 'block_candidate_retry_count', 0))}", + "# HELP qbit_prism_block_candidate_accept_pending_defers_total Terminal abandonments refused because the candidate is (or was recently observed as) an active chain block; the candidate retries until its accepted success tail finalizes it as submitted.", + "# TYPE qbit_prism_block_candidate_accept_pending_defers_total counter", + f"qbit_prism_block_candidate_accept_pending_defers_total {int(getattr(self.port, 'block_candidate_accept_pending_defer_count', 0))}", + "# HELP qbit_prism_block_candidate_poisoned_total Invalid durable candidate intents quarantined from replay.", + "# TYPE qbit_prism_block_candidate_poisoned_total counter", + f"qbit_prism_block_candidate_poisoned_total {int(getattr(self.port, 'block_candidate_poisoned_count', 0))}", + "# HELP qbit_prism_block_candidates_abandoned_total Block candidates that did not land (lost tip race or failed submit), by reason. Not share rejections: the underlying share was accepted.", + "# TYPE qbit_prism_block_candidates_abandoned_total counter", + *[ + f'qbit_prism_block_candidates_abandoned_total{{reason_id="{reason}"}} {int(count)}' + for reason, count in sorted(getattr(self.port, "block_candidate_abandoned_counts", {}).items()) + ], + "# HELP qbit_prism_stale_job_abandons_total Terminal stale-job block candidate abandonments by bounded cause.", + "# TYPE qbit_prism_stale_job_abandons_total counter", + *[ + f'qbit_prism_stale_job_abandons_total{{class="{abandon_class}"}} {int(stale_job_abandon_counts.get(abandon_class, 0))}' + for abandon_class in PRISM_STALE_JOB_ABANDON_CLASSES + ], + "# HELP qbit_prism_share_append_queue_depth Accepted shares waiting on the ledger writer thread.", + "# TYPE qbit_prism_share_append_queue_depth gauge", + f"qbit_prism_share_append_queue_depth {self.port.share_append_queue.qsize() if getattr(self.port, 'share_append_queue', None) is not None else 0}", + "# HELP qbit_prism_share_append_failures_total Shares in group commits that failed before acknowledgement.", + "# TYPE qbit_prism_share_append_failures_total counter", + f"qbit_prism_share_append_failures_total {int(getattr(self.port, 'share_append_failure_count', 0))}", + "# HELP qbit_prism_shares_recovered_to_disk_total Legacy pre-commit-ACK shares written to the upgrade recovery file.", + "# TYPE qbit_prism_shares_recovered_to_disk_total counter", + f"qbit_prism_shares_recovered_to_disk_total {int(getattr(self.port, 'shares_recovered_to_disk', 0))}", + "# HELP qbit_prism_shares_replayed_total Recovery-file shares replayed into the ledger at startup.", + "# TYPE qbit_prism_shares_replayed_total counter", + f"qbit_prism_shares_replayed_total {int(getattr(self.port, 'shares_replayed', 0))}", + "# HELP qbit_prism_share_replay_conflicts_total Recovery-file rows quarantined because the durable row disagrees with the journal payload.", + "# TYPE qbit_prism_share_replay_conflicts_total counter", + f"qbit_prism_share_replay_conflicts_total {int(getattr(self.port, 'share_replay_conflicts', 0))}", + "# HELP qbit_prism_tip_refresh_jobs_total Client jobs refreshed after qbit tip/template changes.", + "# TYPE qbit_prism_tip_refresh_jobs_total counter", + f"qbit_prism_tip_refresh_jobs_total {self.port.tip_refresh_job_count}", + "# HELP qbit_prism_active_job_contexts Current retained PRISM job contexts.", + "# TYPE qbit_prism_active_job_contexts gauge", + f"qbit_prism_active_job_contexts {len(getattr(self.port, 'jobs', {}))}", + "# HELP qbit_prism_evicted_job_contexts Evicted job contexts retained by safety class.", + "# TYPE qbit_prism_evicted_job_contexts gauge", + *[ + f'qbit_prism_evicted_job_contexts{{class="{job_class}"}} {evicted_job_context_counts[job_class]}' + for job_class in PRISM_EVICTED_JOB_CLASSES + ], + "# HELP qbit_prism_evicted_job_submits_total Accepted submits validated against an evicted job context.", + "# TYPE qbit_prism_evicted_job_submits_total counter", + *[ + f'qbit_prism_evicted_job_submits_total{{outcome="{outcome}"}} {int(evicted_job_submit_counts.get(outcome, 0))}' + for outcome in PRISM_EVICTED_JOB_SUBMIT_OUTCOMES + ], + "# HELP qbit_prism_evicted_job_expirations_total Retained job contexts removed after their class TTL.", + "# TYPE qbit_prism_evicted_job_expirations_total counter", + *[ + f'qbit_prism_evicted_job_expirations_total{{class="{job_class}"}} {int(evicted_job_expiration_counts.get(job_class, 0))}' + for job_class in PRISM_EVICTED_JOB_CLASSES + ], + "# HELP qbit_prism_evicted_job_capacity_evictions_total Same-tip retained contexts removed by a configured count limit.", + "# TYPE qbit_prism_evicted_job_capacity_evictions_total counter", + *[ + f'qbit_prism_evicted_job_capacity_evictions_total{{scope="{scope}"}} {int(evicted_job_capacity_eviction_counts.get(scope, 0))}' + for scope in PRISM_EVICTED_JOB_CAPACITY_SCOPES + ], + "# HELP qbit_prism_post_accept_refresh_failures_total Immediate clean-job refreshes that failed after direct block acceptance.", + "# TYPE qbit_prism_post_accept_refresh_failures_total counter", + f"qbit_prism_post_accept_refresh_failures_total {self.port.post_accept_refresh_failure_count}", + "# HELP qbit_prism_reorg_inactive_blocks_total PRISM pool blocks quarantined after leaving the active chain.", + "# TYPE qbit_prism_reorg_inactive_blocks_total counter", + f"qbit_prism_reorg_inactive_blocks_total {self.port.reorg_inactive_block_count}", + "# HELP qbit_prism_reorg_reactivated_blocks_total Quarantined PRISM pool blocks restored after returning to the active chain.", + "# TYPE qbit_prism_reorg_reactivated_blocks_total counter", + f"qbit_prism_reorg_reactivated_blocks_total {self.port.reorg_reactivated_block_count}", + "# HELP qbit_prism_reorg_reconcile_skips_total Reorg reconciliation passes skipped because qbitd chain view was not trusted.", + "# TYPE qbit_prism_reorg_reconcile_skips_total counter", + f"qbit_prism_reorg_reconcile_skips_total {self.port.reorg_reconcile_skip_count}", + "# HELP qbit_prism_reorg_reconcile_errors_total Reorg reconciliation errors that prevented ordered job issuance.", + "# TYPE qbit_prism_reorg_reconcile_errors_total counter", + f"qbit_prism_reorg_reconcile_errors_total {self.port.reorg_reconcile_error_count}", + "# HELP qbit_prism_reorg_reconcile_lookups_total Reconcile demand by caller path and the source that satisfied it.", + "# TYPE qbit_prism_reorg_reconcile_lookups_total counter", + *[ + f'qbit_prism_reorg_reconcile_lookups_total{{path="{path}",source="{source}"}} ' + f"{int(getattr(self.port, 'reorg_reconcile_lookup_counts', {}).get((path, source), 0))}" + for path in PRISM_REORG_RECONCILE_LOOKUP_PATHS + for source in PRISM_REORG_RECONCILE_LOOKUP_SOURCES + ], + "# HELP qbit_prism_matured_payouts_total Payout entries marked mature by the coordinator tip reconciliation path.", + "# TYPE qbit_prism_matured_payouts_total counter", + f"qbit_prism_matured_payouts_total {self.port.matured_payout_count}", + "# HELP qbit_prism_vardiff_idle_retargets_total Vardiff retargets triggered by the idle zero-accepted-share sweep.", + "# TYPE qbit_prism_vardiff_idle_retargets_total counter", + f"qbit_prism_vardiff_idle_retargets_total {idle_retarget_count}", + # qbit_prism_shares_per_second is deliberately absent. It divided + # the ledger's lifetime accepted count by this process's uptime -- + # different domains, so the value tracked 1/uptime rather than any + # share rate: enormous right after a restart, decaying toward zero + # as the process aged, and never once the pool's throughput. Use + # rate(qbit_prism_accepted_shares_total[5m]), which is what the + # counter above exists for, or sum the per-lane gauge for this + # process's own since-start commit rate. + "# HELP qbit_prism_stale_share_percent Percent of submitted shares classified stale.", + "# TYPE qbit_prism_stale_share_percent gauge", + f"qbit_prism_stale_share_percent {stale_percent:.12g}", + "# HELP qbit_prism_blocks_accepted_total Blocks accepted through the PRISM coordinator.", + "# TYPE qbit_prism_blocks_accepted_total counter", + f"qbit_prism_blocks_accepted_total {self.port.accepted_block_count}", + "# HELP qbit_prism_persisted_blocks Persisted PRISM pool block rows.", + "# TYPE qbit_prism_persisted_blocks gauge", + f"qbit_prism_persisted_blocks {ledger_metrics['blocks']}", + "# HELP qbit_prism_inactive_pool_blocks PRISM pool block rows currently quarantined as inactive.", + "# TYPE qbit_prism_inactive_pool_blocks gauge", + f"qbit_prism_inactive_pool_blocks {ledger_metrics.get('inactive_blocks', 0)}", + "# HELP qbit_prism_reversed_pool_blocks PRISM pool block rows terminally reversed.", + "# TYPE qbit_prism_reversed_pool_blocks gauge", + f"qbit_prism_reversed_pool_blocks {ledger_metrics.get('reversed_blocks', 0)}", + "# HELP qbit_prism_rejected_pool_blocks PRISM pool block rows rejected before confirmation.", + "# TYPE qbit_prism_rejected_pool_blocks gauge", + f"qbit_prism_rejected_pool_blocks {ledger_metrics.get('rejected_blocks', 0)}", + "# HELP qbit_prism_owed_accounts Current accounts with positive carried owed balances.", + "# TYPE qbit_prism_owed_accounts gauge", + f"qbit_prism_owed_accounts {ledger_metrics['owed_accounts']}", + "# HELP qbit_prism_coinbase_weight_headroom_bytes Remaining qbit block weight bytes after the latest pool coinbase.", + "# TYPE qbit_prism_coinbase_weight_headroom_bytes gauge", + f"qbit_prism_coinbase_weight_headroom_bytes {coinbase_weight_headroom}", + "# HELP qbit_prism_ctv_fanouts_pending Pending non-terminal CTV fanouts known to the ledger, or -1 if unavailable.", + "# TYPE qbit_prism_ctv_fanouts_pending gauge", + f"qbit_prism_ctv_fanouts_pending {ctv_pending}", + "# HELP qbit_prism_ctv_fanouts_broadcastable CTV fanouts that are mature enough to broadcast, or -1 if unavailable.", + "# TYPE qbit_prism_ctv_fanouts_broadcastable gauge", + f"qbit_prism_ctv_fanouts_broadcastable {ctv_broadcastable}", + "# HELP qbit_prism_ctv_fanouts_failed CTV fanouts with failed or rejected broadcast state, or -1 if unavailable.", + "# TYPE qbit_prism_ctv_fanouts_failed gauge", + f"qbit_prism_ctv_fanouts_failed {ctv_failed}", + "# HELP qbit_prism_vardiff_enabled Whether PRISM Stratum vardiff is enabled.", + "# TYPE qbit_prism_vardiff_enabled gauge", + f"qbit_prism_vardiff_enabled {1 if self.port.vardiff_config.enabled else 0}", + "# HELP qbit_prism_qbitd_initial_block_download qbitd initialblockdownload status, or -1 if unavailable.", + "# TYPE qbit_prism_qbitd_initial_block_download gauge", + f"qbit_prism_qbitd_initial_block_download {ibd}", + "# HELP qbit_prism_qbitd_peers qbitd peer count, or -1 if unavailable.", + "# TYPE qbit_prism_qbitd_peers gauge", + f"qbit_prism_qbitd_peers {peers}", + "# HELP qbit_prism_audit_artifact_bytes Bytes used by PRISM audit artifacts in PRISM_AUDIT_DIR by artifact kind.", + "# TYPE qbit_prism_audit_artifact_bytes gauge", + *[ + f'qbit_prism_audit_artifact_bytes{{kind="{kind}"}} {audit_metrics[kind]["bytes"]}' + for kind in ("body", "share_segment", "live_bundle", "candidate", "other") + ], + "# HELP qbit_prism_audit_artifact_files PRISM audit artifact file count in PRISM_AUDIT_DIR by artifact kind.", + "# TYPE qbit_prism_audit_artifact_files gauge", + *[ + f'qbit_prism_audit_artifact_files{{kind="{kind}"}} {audit_metrics[kind]["files"]}' + for kind in ("body", "share_segment", "live_bundle", "candidate", "other") + ], + "# HELP qbit_prism_audit_artifact_scan_error Whether the latest PRISM_AUDIT_DIR metric scan failed.", + "# TYPE qbit_prism_audit_artifact_scan_error gauge", + f"qbit_prism_audit_artifact_scan_error {audit_metrics['scan_error']}", + ] + lines.extend(self.lease_heartbeat_metrics_lines()) + lines.extend(self.shutdown_metrics_lines()) + lines.extend(self.coordinator_lock_metrics_lines()) + lines.extend(self.block_submitter_metrics_lines()) + lines.extend(self.block_candidate_cleanup_backlog_metrics_lines()) + lines.extend(self.share_ack_metrics_lines()) + lines.extend(self.port.ctv_fanout_broadcaster_metrics_lines()) + lines.extend(self.port.vardiff_idle_metrics_lines()) + lines.extend(self.vardiff_convergence_metrics_lines()) + lines.extend(self.port.block_finalization_metrics_lines()) + lines.extend(self.job_build_metrics_lines()) + lines.extend(self.port.tip_refresh_metrics_lines()) + lines.extend(self.port.payout_state_metrics_lines()) + lines.extend(self.initial_delivery_metrics_lines()) + lines.extend(self.port.progress_health_metrics_lines()) + lines.extend(self.accepted_preview_attribution_metrics_lines()) + lines.extend(self.gc_pause_metrics_lines()) + lines.extend(self.process_heap_metrics_lines()) + lines.extend(self.component_cardinality_metrics_lines()) + return "\n".join(lines) + "\n" + + def gc_pause_metrics_lines(self) -> list[str]: + """Issue #227: cyclic-collector pause durations, per generation. + + Three families, each labelled only by the closed + PRISM_GC_GENERATIONS set. They sit immediately before the issue + #226 heap block whose ``qbit_prism_process_gc_*`` families they + complement: those export ``gc.get_stats()`` -- how many passes ran + and what they freed -- and cannot say how long any pass held the + interpreter; these are the durations, from ``gc.callbacks``. The + recorder is armed with the lease heartbeat (it is the stall + attribution's instrument), so every series is zero on a + coordinator that never armed the heartbeat. + """ + pauses = self.port._ensure_lease_heartbeat_service().snapshot()["gc_pauses"] + assert isinstance(pauses, dict) + lines = [ + "# HELP qbit_prism_process_gc_pause_seconds Cyclic collector pause duration per generation, from gc.callbacks; the qbit_prism_process_gc_* count families say how many passes ran, this says how long each held the interpreter.", + "# TYPE qbit_prism_process_gc_pause_seconds histogram", + ] + for generation in PRISM_GC_GENERATIONS: + entry = pauses[generation] + buckets = entry["buckets"] + lines.extend( + f'qbit_prism_process_gc_pause_seconds_bucket{{generation="{generation}",le="{bucket:g}"}} {int(buckets.get(bucket, 0))}' + for bucket in PRISM_GC_PAUSE_SECONDS_BUCKETS + ) + lines.append( + f'qbit_prism_process_gc_pause_seconds_bucket{{generation="{generation}",le="+Inf"}} {int(entry["count"])}' + ) + lines.append( + f'qbit_prism_process_gc_pause_seconds_sum{{generation="{generation}"}} {float(entry["sum"]):.6f}' + ) + lines.append( + f'qbit_prism_process_gc_pause_seconds_count{{generation="{generation}"}} {int(entry["count"])}' + ) + lines.extend( + [ + "# HELP qbit_prism_process_gc_last_pause_seconds Duration of the most recent cyclic collector pass per generation.", + "# TYPE qbit_prism_process_gc_last_pause_seconds gauge", + *[ + f'qbit_prism_process_gc_last_pause_seconds{{generation="{generation}"}} {float(pauses[generation]["last_seconds"]):.6f}' + for generation in PRISM_GC_GENERATIONS + ], + "# HELP qbit_prism_process_gc_max_pause_seconds Longest cyclic collector pass observed per generation since the lease heartbeat armed.", + "# TYPE qbit_prism_process_gc_max_pause_seconds gauge", + *[ + f'qbit_prism_process_gc_max_pause_seconds{{generation="{generation}"}} {float(pauses[generation]["max_seconds"]):.6f}' + for generation in PRISM_GC_GENERATIONS + ], + ] + ) + return lines + + def share_ack_metrics_lines(self) -> list[str]: + histograms = self.port.share_ack_snapshot() + lines = [ + "# HELP qbit_prism_share_ack_seconds mining.submit line arrival to Stratum response, by outcome.", + "# TYPE qbit_prism_share_ack_seconds histogram", + ] + for result in PRISM_SHARE_ACK_RESULTS: + histogram = histograms[result] + buckets = histogram["buckets"] + lines.extend( + f'qbit_prism_share_ack_seconds_bucket{{result="{result}",le="{bucket:g}"}} {int(buckets.get(bucket, 0))}' + for bucket in PRISM_JOB_BUILD_SECONDS_BUCKETS + ) + lines.append( + f'qbit_prism_share_ack_seconds_bucket{{result="{result}",le="+Inf"}} {histogram["count"]}' + ) + lines.append( + f'qbit_prism_share_ack_seconds_sum{{result="{result}"}} {histogram["sum"]:.6f}' + ) + lines.append( + f'qbit_prism_share_ack_seconds_count{{result="{result}"}} {histogram["count"]}' + ) + return lines + + def vardiff_convergence_metrics_lines(self) -> list[str]: + """Vardiff convergence and reconnect-resume observability. + + The lane label set is deterministic: configured listener profiles in + order, then any additional lane names observed in the counters, + sorted. The per-second gauge divides a process-local count by this + process's uptime, so numerator and denominator share a domain -- but + that numerator is narrower than "accepted". VardiffService.note_accepted + runs only where the share writer sees a newly inserted ledger row, and + the startup recovery-journal replay reaches the ledger without passing + through it at all, so an exact replay and a recovered share are both + acked accepted while never being lane-counted. The lanes therefore sum + to this coordinator's own commit rate, not to its accepted-ack count. + """ + snapshot = self.port.vardiff_convergence_snapshot() + lane_accepted = snapshot["lane_accepted_shares"] + assert isinstance(lane_accepted, dict) + resume_outcomes = snapshot["resume_outcomes"] + assert isinstance(resume_outcomes, dict) + lanes = [ + str(profile.name) + for profile in getattr(self.port, "listener_profiles", ()) or () + ] + lanes.extend(sorted(lane for lane in lane_accepted if lane not in lanes)) + elapsed = max(0.001, time.monotonic() - self.port.started_monotonic) + return [ + "# HELP qbit_prism_vardiff_sessions_at_max_difficulty Sessions whose current difficulty sits at their effective vardiff ceiling.", + "# TYPE qbit_prism_vardiff_sessions_at_max_difficulty gauge", + f"qbit_prism_vardiff_sessions_at_max_difficulty {int(snapshot['sessions_at_max_difficulty'])}", + "# HELP qbit_prism_vardiff_lane_accepted_shares_total Accepted shares by Stratum lane.", + "# TYPE qbit_prism_vardiff_lane_accepted_shares_total counter", + *[ + f'qbit_prism_vardiff_lane_accepted_shares_total{{lane="{self.port.prometheus_label_value(lane)}"}} {int(lane_accepted.get(lane, 0))}' + for lane in lanes + ], + "# HELP qbit_prism_vardiff_lane_accepted_shares_per_second Accepted shares per second by Stratum lane, averaged since coordinator start; a long-run average cannot show a transient reconnect storm -- use rate(qbit_prism_vardiff_lane_accepted_shares_total[5m]) for that. The numerator counts shares this process newly committed on the live submission path, not the ledger lifetime total published by qbit_prism_accepted_shares_total: an exact replay of an already-durable share and a startup recovery-journal replay are both acked accepted without being lane-counted, so the lanes sum to this coordinator's own commit rate and need not reconcile against an accepted-ack count.", + "# TYPE qbit_prism_vardiff_lane_accepted_shares_per_second gauge", + *[ + f'qbit_prism_vardiff_lane_accepted_shares_per_second{{lane="{self.port.prometheus_label_value(lane)}"}} {int(lane_accepted.get(lane, 0)) / elapsed:.12g}' + for lane in lanes + ], + "# HELP qbit_prism_vardiff_resume_total Reconnect difficulty resume attempts by outcome; clamped means the retained value was pulled into the lane's plausibility bounds, overridden means an adopted value was superseded by an explicit difficulty request in the same authorize, so resumed + clamped - overridden is the number that stuck.", + "# TYPE qbit_prism_vardiff_resume_total counter", + *[ + f'qbit_prism_vardiff_resume_total{{outcome="{outcome}"}} {int(resume_outcomes.get(outcome, 0))}' + for outcome in PRISM_VARDIFF_RESUME_OUTCOMES + ], + "# HELP qbit_prism_vardiff_resume_retained_sessions Worker sessions holding a retained difficulty eligible for resume.", + "# TYPE qbit_prism_vardiff_resume_retained_sessions gauge", + f"qbit_prism_vardiff_resume_retained_sessions {int(snapshot['retained_sessions'])}", + ] + + def coordinator_lock_metrics_lines(self) -> list[str]: + acquisition_count, contention_count, wait_sum, wait_max = ( + self.port.coordinator_lock_contention_snapshot() + ) + return [ + "# HELP qbit_prism_coordinator_lock_acquisitions_total Coordinator control-plane lock acquisitions granted, contended or not; divide qbit_prism_coordinator_lock_contentions_total by this for the contended fraction. A timed-out wait is counted as contention without an acquisition, so that fraction is an upper bound where a caller passes a timeout.", + "# TYPE qbit_prism_coordinator_lock_acquisitions_total counter", + f"qbit_prism_coordinator_lock_acquisitions_total {int(acquisition_count)}", + "# HELP qbit_prism_coordinator_lock_contentions_total Coordinator control-plane lock acquisitions that had to wait.", + "# TYPE qbit_prism_coordinator_lock_contentions_total counter", + f"qbit_prism_coordinator_lock_contentions_total {int(contention_count)}", + "# HELP qbit_prism_coordinator_lock_wait_seconds Coordinator control-plane lock wait duration for contended acquisitions.", + "# TYPE qbit_prism_coordinator_lock_wait_seconds summary", + f"qbit_prism_coordinator_lock_wait_seconds_sum {float(wait_sum):.6f}", + f"qbit_prism_coordinator_lock_wait_seconds_count {int(contention_count)}", + "# HELP qbit_prism_coordinator_lock_wait_seconds_max Longest observed coordinator control-plane lock wait.", + "# TYPE qbit_prism_coordinator_lock_wait_seconds_max gauge", + f"qbit_prism_coordinator_lock_wait_seconds_max {float(wait_max):.6f}", + ] + + @staticmethod + def _accepted_block_preview_publication_lines( + histograms: dict[str, Any], + ) -> list[str]: + """Per-result bucket/sum/count lines for the publication histogram.""" + lines: list[str] = [] + for result in PRISM_ACCEPTED_BLOCK_PREVIEW_PUBLICATION_RESULTS: + histogram = histograms[result] + buckets = histogram["buckets"] + count = int(histogram["count"]) + lines.extend( + f"qbit_prism_accepted_block_preview_publication_seconds_bucket" + f'{{result="{result}",le="{bucket:g}"}} ' + f"{int(buckets.get(bucket, 0))}" + for bucket in ( + PRISM_ACCEPTED_BLOCK_PREVIEW_PUBLICATION_SECONDS_BUCKETS + ) + ) + lines.append( + f"qbit_prism_accepted_block_preview_publication_seconds_bucket" + f'{{result="{result}",le="+Inf"}} {count}' + ) + lines.append( + f"qbit_prism_accepted_block_preview_publication_seconds_sum" + f'{{result="{result}"}} {float(histogram["sum"]):.6f}' + ) + lines.append( + f"qbit_prism_accepted_block_preview_publication_seconds_count" + f'{{result="{result}"}} {count}' + ) + return lines + + def block_submitter_metrics_lines(self) -> list[str]: + snapshot = self.port.block_submitter_snapshot() + submit_buckets = snapshot["submit_seconds_buckets"] + assert isinstance(submit_buckets, dict) + submit_sum = float(snapshot["submit_seconds_sum"]) + submit_count = int(snapshot["submit_seconds_count"]) + backoff_active = bool(snapshot["backoff_active"]) + backoff_remaining = float(snapshot["backoff_remaining_seconds"]) + backoff_delay = float(snapshot["backoff_delay_seconds"]) + # The B1 owner copies these under the coordinator lock; the key set + # is fixed by PRISM_BLOCK_CANDIDATE_COLLAPSE_OUTCOMES so the series + # cardinality cannot grow with the candidate population. + collapse_counts = self.port.block_candidate_collapse_snapshot() + # Issue #181 item 3: the label set is closed by + # PRISM_ACCEPTED_BLOCK_PREVIEW_PUBLICATION_RESULTS, so this series + # carries no block hash or height and cannot grow with the candidate + # population. + preview_publication = snapshot["accepted_preview_publication"] + assert isinstance(preview_publication, dict) + return [ + "# HELP qbit_prism_block_submit_seconds Seconds from a block candidate landing in this process to its submitblock RPC returning.", + "# TYPE qbit_prism_block_submit_seconds histogram", + *[ + f'qbit_prism_block_submit_seconds_bucket{{le="{bucket:g}"}} {int(submit_buckets.get(bucket, 0))}' + for bucket in PRISM_JOB_BUILD_SECONDS_BUCKETS + ], + f'qbit_prism_block_submit_seconds_bucket{{le="+Inf"}} {submit_count}', + f"qbit_prism_block_submit_seconds_sum {submit_sum:.6f}", + f"qbit_prism_block_submit_seconds_count {submit_count}", + "# HELP qbit_prism_accepted_block_preview_publication_seconds Seconds from definitive qbitd acceptance of a block candidate to its payout preview becoming visible to waiting child work, by publication result.", + "# TYPE qbit_prism_accepted_block_preview_publication_seconds histogram", + *self._accepted_block_preview_publication_lines(preview_publication), + "# HELP qbit_prism_block_candidates_pending Durable block candidates awaiting a terminal outcome, or -1 if unavailable.", + "# TYPE qbit_prism_block_candidates_pending gauge", + f"qbit_prism_block_candidates_pending {int(snapshot['pending_count'])}", + "# HELP qbit_prism_block_candidate_oldest_pending_seconds Age of the oldest durable pending block candidate, or -1 if unavailable.", + "# TYPE qbit_prism_block_candidate_oldest_pending_seconds gauge", + f"qbit_prism_block_candidate_oldest_pending_seconds {float(snapshot['oldest_pending_age_seconds']):.6f}", + "# HELP qbit_prism_block_candidate_oldest_unattempted_seconds Age of the oldest durable candidate that has never entered processing, or -1 if unavailable.", + "# TYPE qbit_prism_block_candidate_oldest_unattempted_seconds gauge", + f"qbit_prism_block_candidate_oldest_unattempted_seconds {float(snapshot['oldest_unattempted_age_seconds']):.6f}", + "# HELP qbit_prism_block_submitter_retry_backoff_active Whether the submitter is in an intentional interruptible retry wait.", + "# TYPE qbit_prism_block_submitter_retry_backoff_active gauge", + f"qbit_prism_block_submitter_retry_backoff_active {1 if backoff_active else 0}", + "# HELP qbit_prism_block_submitter_retry_backoff_remaining_seconds Remaining intentional submitter retry wait.", + "# TYPE qbit_prism_block_submitter_retry_backoff_remaining_seconds gauge", + f"qbit_prism_block_submitter_retry_backoff_remaining_seconds {backoff_remaining:.6f}", + "# HELP qbit_prism_block_submitter_retry_backoff_seconds Current intentional submitter retry delay.", + "# TYPE qbit_prism_block_submitter_retry_backoff_seconds gauge", + f"qbit_prism_block_submitter_retry_backoff_seconds {backoff_delay:.6f}", + "# HELP qbit_prism_block_candidate_collapse_total Durable pending block candidates handled by the decided-height collapse, by bounded outcome. The outcome label set is closed: no block hash, parent hash, or job ID is ever a label.", + "# TYPE qbit_prism_block_candidate_collapse_total counter", + *[ + f'qbit_prism_block_candidate_collapse_total{{outcome="{outcome}"}} {int(collapse_counts.get(outcome, 0))}' + for outcome in PRISM_BLOCK_CANDIDATE_COLLAPSE_OUTCOMES + ], + ] + + def block_candidate_cleanup_backlog_metrics_lines(self) -> list[str]: + """Issue #198: the collapse cleanup-retry backlog and its bound. + + Seven single-series families, none labelled: the B1 owner copies + the values under the coordinator lock and the key set of its + snapshot is closed, so the export cannot grow with the candidate + population or carry a hash. The bound is exported beside the depth + so an alert can be a ratio against the configured contract instead + of a deployment-specific literal (the same shape as the + accepted-parent depth cap). + """ + snapshot = ( + self.port._ensure_block_candidate_service() + .collapsed_candidate_cleanup_backlog_snapshot() + ) + return [ + "# HELP qbit_prism_block_candidate_cleanup_retry_backlog Durably terminal collapsed block candidates whose in-memory cleanup is still owed and retried by the accounting lane.", + "# TYPE qbit_prism_block_candidate_cleanup_retry_backlog gauge", + f"qbit_prism_block_candidate_cleanup_retry_backlog {int(snapshot['depth'])}", + "# HELP qbit_prism_block_candidate_cleanup_retry_backlog_max Configured cleanup-retry backlog depth at which the decided-height collapse stops admitting rows to bulk terminalization.", + "# TYPE qbit_prism_block_candidate_cleanup_retry_backlog_max gauge", + f"qbit_prism_block_candidate_cleanup_retry_backlog_max {int(snapshot['backlog_max'])}", + "# HELP qbit_prism_block_candidate_cleanup_retry_oldest_seconds Age of the oldest owed collapse cleanup since it was first deferred, or -1 when none is owed.", + "# TYPE qbit_prism_block_candidate_cleanup_retry_oldest_seconds gauge", + f"qbit_prism_block_candidate_cleanup_retry_oldest_seconds {float(snapshot['oldest_age_seconds']):.6f}", + "# HELP qbit_prism_block_candidate_cleanup_retry_pending_share_holders Pending-share floor holders retained by owed collapse cleanups, by exact object identity.", + "# TYPE qbit_prism_block_candidate_cleanup_retry_pending_share_holders gauge", + f"qbit_prism_block_candidate_cleanup_retry_pending_share_holders {int(snapshot['pending_share_holders'])}", + "# HELP qbit_prism_block_candidate_cleanup_retry_terminal_outcome_pins Terminal-outcome fences the cleanup-retry backlog pins against eviction.", + "# TYPE qbit_prism_block_candidate_cleanup_retry_terminal_outcome_pins gauge", + f"qbit_prism_block_candidate_cleanup_retry_terminal_outcome_pins {int(snapshot['terminal_outcome_pins'])}", + "# HELP qbit_prism_block_candidate_cleanup_backpressure_active Whether the cleanup-retry backlog is at its bound and bulk terminalization is refusing new rows.", + "# TYPE qbit_prism_block_candidate_cleanup_backpressure_active gauge", + f"qbit_prism_block_candidate_cleanup_backpressure_active {1 if snapshot['backpressure_active'] else 0}", + "# HELP qbit_prism_block_candidate_cleanup_backpressure_total Occasions on which the cleanup-retry backlog bound preserved at least one row from bulk terminalization.", + "# TYPE qbit_prism_block_candidate_cleanup_backpressure_total counter", + f"qbit_prism_block_candidate_cleanup_backpressure_total {int(snapshot['backpressure_engagements'])}", + ] + + def process_heap_metrics_lines(self) -> list[str]: + """Issue #226: always-on interpreter and glibc allocator readings. + + Eleven families. Only the three GC families carry a label, and its + value set is the closed PRISM_GC_GENERATIONS; nothing here can grow + with traffic. The collector is a snapshot read -- allocation and + GC counters, the thread count, and one mallinfo2 call -- and never + walks the heap. Where mallinfo2 is unavailable (musl, macOS, glibc + before 2.33, or PRISM_MALLOC_TELEMETRY=0) the availability gauge + renders 0 and the four byte gauges render -1, never 0. + """ + sample = self._process_telemetry.sample() + return [ + "# HELP qbit_prism_process_allocated_blocks Memory blocks currently allocated by the CPython allocator (sys.getallocatedblocks); a live-object proxy that rises with retention and stays flat under pure allocator fragmentation.", + "# TYPE qbit_prism_process_allocated_blocks gauge", + f"qbit_prism_process_allocated_blocks {int(sample.allocated_blocks)}", + "# HELP qbit_prism_process_gc_trigger_count gc.get_count(): the cyclic collector's per-generation trigger counters, compared against gc.get_threshold() to decide when to collect. Generation 0 is allocations minus deallocations, NOT a count of retained objects. CPython 3.13 made the collector incremental, so the third entry is unused and reads 0; the series is kept for a fixed family set.", + "# TYPE qbit_prism_process_gc_trigger_count gauge", + *[ + f'qbit_prism_process_gc_trigger_count{{generation="{generation}"}} {int(count)}' + for generation, count in zip(PRISM_GC_GENERATIONS, sample.gc_trigger_count) + ], + "# HELP qbit_prism_process_gc_collections_total Cyclic collector passes completed per generation since process start (gc.get_stats).", + "# TYPE qbit_prism_process_gc_collections_total counter", + *[ + f'qbit_prism_process_gc_collections_total{{generation="{generation}"}} {int(count)}' + for generation, count in zip(PRISM_GC_GENERATIONS, sample.gc_collections) + ], + "# HELP qbit_prism_process_gc_collected_objects_total Objects the cyclic collector freed per generation since process start.", + "# TYPE qbit_prism_process_gc_collected_objects_total counter", + *[ + f'qbit_prism_process_gc_collected_objects_total{{generation="{generation}"}} {int(count)}' + for generation, count in zip(PRISM_GC_GENERATIONS, sample.gc_collected) + ], + "# HELP qbit_prism_process_gc_uncollectable_objects_total Objects the cyclic collector found unreachable but could not free per generation since process start; growth is a leak the collector already knows about.", + "# TYPE qbit_prism_process_gc_uncollectable_objects_total counter", + *[ + f'qbit_prism_process_gc_uncollectable_objects_total{{generation="{generation}"}} {int(count)}' + for generation, count in zip(PRISM_GC_GENERATIONS, sample.gc_uncollectable) + ], + "# HELP qbit_prism_process_threads Live Python threads (threading.active_count); each one is a candidate glibc malloc arena.", + "# TYPE qbit_prism_process_threads gauge", + f"qbit_prism_process_threads {int(sample.threads)}", + "# HELP qbit_prism_process_malloc_info_available Whether glibc mallinfo2 is bound and PRISM_MALLOC_TELEMETRY is on (1), or the allocator byte gauges are unavailable (0): the symbol is glibc 2.33+ only and absent on musl and macOS. While 0 the byte gauges render -1, never 0.", + "# TYPE qbit_prism_process_malloc_info_available gauge", + f"qbit_prism_process_malloc_info_available {1 if sample.malloc_info_available else 0}", + "# HELP qbit_prism_process_malloc_arena_bytes Heap space glibc malloc holds for its arenas across every thread arena, excluding mmapped chunks (mallinfo2.arena), or -1 when unavailable.", + "# TYPE qbit_prism_process_malloc_arena_bytes gauge", + f"qbit_prism_process_malloc_arena_bytes {int(sample.malloc_arena_bytes)}", + "# HELP qbit_prism_process_malloc_in_use_bytes Bytes in allocated chunks across every glibc malloc arena (mallinfo2.uordblks), or -1 when unavailable; arena minus this is the fragmentation glibc has not returned to the kernel.", + "# TYPE qbit_prism_process_malloc_in_use_bytes gauge", + f"qbit_prism_process_malloc_in_use_bytes {int(sample.malloc_in_use_bytes)}", + "# HELP qbit_prism_process_malloc_free_bytes Bytes in free chunks glibc malloc retains across every arena (mallinfo2.fordblks), or -1 when unavailable.", + "# TYPE qbit_prism_process_malloc_free_bytes gauge", + f"qbit_prism_process_malloc_free_bytes {int(sample.malloc_free_bytes)}", + "# HELP qbit_prism_process_malloc_mmapped_bytes Bytes in chunks glibc malloc served directly through mmap (mallinfo2.hblkhd), or -1 when unavailable.", + "# TYPE qbit_prism_process_malloc_mmapped_bytes gauge", + f"qbit_prism_process_malloc_mmapped_bytes {int(sample.malloc_mmapped_bytes)}", + ] + + def component_cardinality_metrics_lines(self) -> list[str]: + """Issue #226 (and #185's gauge list): retained entries per component. + + Two families, each a closed ``component`` label set: entry counts and + retained bytes over the state that plausibly scales with the payout + window, the job population, or a candidate storm. The reads are + deliberately direct attribute loads on the owning services rather + than tolerant ``getattr`` defaults: a renamed owner field must fail + the shipped-owner test loudly, never report a silent zero. The owners + are the ones the renderer already constructs through + ``_ensure_job_cache_state``; the reorg reconciler is read only when + it exists, because the coordinator builds it lazily and it is the + sole holder of its flights, so an absent service is a true zero. + + Every read is one ``len()``, ``qsize()``, or attribute load, atomic + under the GIL, so no owner lock is taken beyond the coordinator lock + the renderer already holds for the graveyard: a gauge that + tolerates one entry of skew must never queue the scrape behind a + daemon round trip or a payout-window walk. + + The one exception is the page-backed payout window, where records and + canonical bytes are summed over the window's pages: O(pages), about + 120 iterations of two ``len()`` calls for a 63k-share window at the + default page size, not O(shares). A mirror-backed window costs neither + sum. This is a bounded per-scrape cost, not a heap walk. + """ + port = self.port + port._ensure_job_cache_state() + bundles = port._ensure_job_bundle_service() + payout = port._ensure_payout_state_service() + writer = port._ensure_share_writer_service() + compiler = port._ensure_bundle_compiler() + candidates = port._ensure_block_candidate_service() + candidates._ensure_block_replay_state() + candidates._ensure_block_candidate_disposition_state() + reconciler = getattr(port, "__dict__", {}).get("_reorg_reconciler_service") + serialization = bundles._share_window_serialization + cached_window = payout._incremental_payout_artifact_window + artifact = payout._payout_ledger_artifact + daemon_client = compiler._serve_builder + window = cached_window.window if cached_window is not None else None + window_pages = window_records = window_canonical_bytes = 0 + mirror_records = mirror_bytes = 0 + # Exactly one component accounts for the cached window, because these + # gauges exist to attribute resident bytes and an operator summing the + # family must not count the same object twice. The two backings are + # mutually exclusive -- the mirror *is* the cached window on the Rust + # path -- so a mirror-backed window reports zero pages, records, and + # canonical bytes under payout_window_*, and its size appears only + # under daemon_window_mirror_*. Which pair is non-zero is also how an + # operator reads the backing off /metrics. + if isinstance(window, DaemonShareWindowMirror): + mirror_records = int(window.record_count) + mirror_bytes = len(window.canonical_items) + elif window is not None: + pages = window.pages + window_pages = len(pages) + window_records = sum(len(page.records) for page in pages) + window_canonical_bytes = sum( + len(page.canonical_json_items) for page in pages + ) + with port.lock: + port._ensure_evicted_job_state() + graveyard_entries = len(port.evicted_job_graveyard) + same_tip_entries = len(port.evicted_same_tip_job_ids) + outstanding_hashes = len(candidates._outstanding_block_candidate_hashes) + tip_observed_hashes = len(candidates._tip_observed_accepted_block_hashes) + counted_abandonments = len(candidates._counted_block_candidate_abandonments) + ancestor_redrives = len(candidates._ancestor_redrive_records) + terminal_outcomes = len(candidates._block_candidate_terminal_outcomes) + waiting_retries = len(candidates._block_disposition_waiting_retries) + dequeued_hashes = len(candidates._block_candidate_dequeued_hashes) + accounted_hashes = len(port._accounted_accepted_block_hashes) + trusted_memo = ( + len(reconciler._reorg_reconcile_trusted_memo) + if reconciler is not None + else 0 + ) + entries = { + "payout_window_pages": window_pages, + "payout_window_records": window_records, + "payout_ledger_artifact_shares": ( + len(artifact.shares_json) if artifact is not None else 0 + ), + "accepted_block_payout_previews": len(payout._accepted_block_payout_previews), + "invalidated_accepted_block_payout_previews": len( + payout._invalidated_accepted_block_payout_previews + ), + "payout_append_invalidation_stamps": len( + payout._payout_append_invalidation_stamps + ), + "payout_window_inflight_scan_anchors": len( + payout._payout_window_inflight_scan_anchors + ), + "payout_unfenced_append_inflight_stamps": len( + payout._payout_unfenced_append_inflight_stamps + ), + "share_window_serialization_shares": ( + int(serialization.share_count) if serialization is not None else 0 + ), + "job_contexts": len(port.jobs), + "job_bundle_cache": len(bundles._job_bundle_cache), + "job_build_issued_stamps": len(bundles._job_build_issued_at_ms), + "bundle_preparation_flights": len(bundles._bundle_preparation_flights), + "active_job_bundle_builds": len(bundles._active_job_bundle_builds), + "evicted_job_graveyard": graveyard_entries, + "evicted_same_tip_job_ids": same_tip_entries, + "block_candidate_queue": candidates.candidate_queue.qsize(), + "block_replay_queue": candidates._block_replay_candidate_queue.qsize(), + "block_replay_inflight_hashes": len(candidates._block_replay_inflight_hashes), + "block_quarantine_queue": candidates._block_quarantine_queue.qsize(), + "block_quarantine_hashes": len(candidates._block_quarantine_hashes), + "outstanding_block_candidate_hashes": outstanding_hashes, + "tip_observed_accepted_block_hashes": tip_observed_hashes, + "counted_block_candidate_abandonments": counted_abandonments, + "accepted_block_preview_stamps": len( + candidates._accepted_block_preview_acceptance_monotonic + ), + "ancestor_redrive_records": ancestor_redrives, + "block_candidate_terminal_outcomes": terminal_outcomes, + "block_candidate_disposition_flights": len( + candidates._block_candidate_disposition_flights + ), + "block_disposition_waiting_retries": waiting_retries, + "block_candidate_dequeued_hashes": dequeued_hashes, + "accounted_accepted_block_hashes": accounted_hashes, + "reconcile_flights": ( + len(reconciler._reconcile_flights) if reconciler is not None else 0 + ), + "reconcile_trusted_memo": trusted_memo, + "pending_share_commit_floor": len(writer._pending_share_commit_floor), + "daemon_window_mirror_records": mirror_records, + "daemon_uploaded_windows": ( + len(daemon_client.uploaded_windows) if daemon_client is not None else 0 + ), + } + byte_sizes = { + "payout_window_canonical_json": window_canonical_bytes, + "daemon_window_mirror_canonical_items": mirror_bytes, + "share_window_serialization_spool": ( + int(serialization._spool_size) if serialization is not None else 0 + ), + # json.dumps with its ASCII default, so characters are bytes. + "share_window_serialization_compact_json": ( + int(serialization.compact_json_bytes) + if serialization is not None + else 0 + ), + } + return [ + "# HELP qbit_prism_component_entries Entries retained by each in-process coordinator component; the component label set is closed and carries no job id, tip hash, generation, or worker name.", + "# TYPE qbit_prism_component_entries gauge", + *[ + f'qbit_prism_component_entries{{component="{kind}"}} {int(entries[kind])}' + for kind in PRISM_COMPONENT_ENTRY_KINDS + ], + "# HELP qbit_prism_component_bytes Bytes retained by each byte-sized coordinator payload (the cached payout window's canonical JSON, the daemon window mirror, and the share-window serialization compact strings); the component label set is closed. All are resident bytes except share_window_serialization_spool, which is an on-disk temporary file and is not part of RSS.", + "# TYPE qbit_prism_component_bytes gauge", + *[ + f'qbit_prism_component_bytes{{component="{kind}"}} {int(byte_sizes[kind])}' + for kind in PRISM_COMPONENT_BYTE_KINDS + ], + ] + + def landing_observability_metrics_lines(self) -> list[str]: + """Landing-path metrics for issue #188's pre-deadline alerting. + + The prior-balances read crossed the one-second submitter deadline + silently over several weeks; these series make that growth, landing + timeouts, and unresolved accepted-parent age visible before they + become an outage. + """ + lines: list[str] = [ + "# HELP qbit_prism_block_ledger_calls_total Submitter ledger calls by deadline class.", + "# TYPE qbit_prism_block_ledger_calls_total counter", + "# HELP qbit_prism_block_ledger_call_timeouts_total Submitter ledger deadline expiries by deadline class.", + "# TYPE qbit_prism_block_ledger_call_timeouts_total counter", + "# HELP qbit_prism_block_ledger_call_budget_seconds Most recent statement budget applied per deadline class (escalates for retried landings).", + "# TYPE qbit_prism_block_ledger_call_budget_seconds gauge", + "# HELP qbit_prism_block_ledger_call_last_duration_seconds Duration of the most recent call per deadline class.", + "# TYPE qbit_prism_block_ledger_call_last_duration_seconds gauge", + "# HELP qbit_prism_block_ledger_call_max_duration_seconds Longest observed call per deadline class since process start.", + "# TYPE qbit_prism_block_ledger_call_max_duration_seconds gauge", + ] + for call_class, stats in sorted( + self.port.block_ledger_call_class_metrics().items() + ): + label = self.port.prometheus_label_value(call_class) + lines.extend( + [ + f'qbit_prism_block_ledger_calls_total{{call_class="{label}"}} {int(stats["calls_total"])}', + f'qbit_prism_block_ledger_call_timeouts_total{{call_class="{label}"}} {int(stats["timeouts_total"])}', + f'qbit_prism_block_ledger_call_budget_seconds{{call_class="{label}"}} {float(stats["last_budget_seconds"]):.6f}', + f'qbit_prism_block_ledger_call_last_duration_seconds{{call_class="{label}"}} {float(stats["last_duration_seconds"]):.6f}', + f'qbit_prism_block_ledger_call_max_duration_seconds{{call_class="{label}"}} {float(stats["max_duration_seconds"]):.6f}', + ] + ) + # The depth gauge must equal what the admission fence counts. The + # fence counts every landed transition; the age list can only report + # the ones carrying a monotonic landing stamp. A degraded preview + # publication and a replayed durable candidate both arm ``landed`` + # without that stamp, so deriving depth from the ages would read zero + # while the fence already refuses admission at depth one. + unresolved_depth = self.port._accepted_parent_unresolved_depth() + unresolved_depth_cap = self.port._accepted_parent_unresolved_depth_cap() + unresolved_ages = self.port.accepted_parent_unresolved_ages_seconds() + oldest_unresolved = max(unresolved_ages) if unresolved_ages else -1.0 + with self.port.lock: + preview_wait_timeouts = int( + getattr(self.port, "_accepted_parent_preview_wait_timeouts", 0) + ) + lines.extend( + [ + "# HELP qbit_prism_accepted_parent_unresolved_transitions Landed accepted-block transitions whose durable bookkeeping is unresolved.", + "# TYPE qbit_prism_accepted_parent_unresolved_transitions gauge", + f"qbit_prism_accepted_parent_unresolved_transitions {unresolved_depth}", + "# HELP qbit_prism_accepted_parent_unresolved_depth_max Configured unresolved accepted-parent depth at which job admission fails closed.", + "# TYPE qbit_prism_accepted_parent_unresolved_depth_max gauge", + f"qbit_prism_accepted_parent_unresolved_depth_max {unresolved_depth_cap}", + "# HELP qbit_prism_accepted_parent_unresolved_oldest_seconds Age of the oldest unresolved accepted-parent transition, or -1 when none.", + "# TYPE qbit_prism_accepted_parent_unresolved_oldest_seconds gauge", + f"qbit_prism_accepted_parent_unresolved_oldest_seconds {oldest_unresolved:.6f}", + "# HELP qbit_prism_accepted_parent_preview_wait_timeouts_total Child job builds that timed out waiting for an accepted-parent payout preview.", + "# TYPE qbit_prism_accepted_parent_preview_wait_timeouts_total counter", + f"qbit_prism_accepted_parent_preview_wait_timeouts_total {preview_wait_timeouts}", + "# HELP qbit_prism_accepted_parent_redrive_attempts_total In-process ancestor re-drives armed after repeated finalization deferrals on one pending accepted-parent transition.", + "# TYPE qbit_prism_accepted_parent_redrive_attempts_total counter", + "qbit_prism_accepted_parent_redrive_attempts_total " + f"{int(getattr(self.port, 'accepted_parent_redrive_attempt_count', 0))}", + "# HELP qbit_prism_accepted_parent_redrive_resolved_total Pending accepted-parent transitions that resolved after at least one in-process re-drive attempt.", + "# TYPE qbit_prism_accepted_parent_redrive_resolved_total counter", + "qbit_prism_accepted_parent_redrive_resolved_total " + f"{int(getattr(self.port, 'accepted_parent_redrive_resolved_count', 0))}", + "# HELP qbit_prism_accepted_parent_redrive_exhausted_total Ancestors whose re-drive attempt cap was exhausted with the transition still pending; the publication-progress watchdog remains the backstop.", + "# TYPE qbit_prism_accepted_parent_redrive_exhausted_total counter", + "qbit_prism_accepted_parent_redrive_exhausted_total " + f"{int(getattr(self.port, 'accepted_parent_redrive_exhausted_count', 0))}", + ] + ) + prior_stats_fn = getattr(self.port.ledger, "prior_balances_read_stats", None) + if callable(prior_stats_fn): + prior_stats = prior_stats_fn() + lines.extend( + [ + "# HELP qbit_prism_prior_balances_reads_total Prior-balances reads served by the ledger.", + "# TYPE qbit_prism_prior_balances_reads_total counter", + f"qbit_prism_prior_balances_reads_total {int(prior_stats['reads_total'])}", + "# HELP qbit_prism_prior_balances_read_last_seconds Duration of the most recent prior-balances read.", + "# TYPE qbit_prism_prior_balances_read_last_seconds gauge", + f"qbit_prism_prior_balances_read_last_seconds {float(prior_stats['last_seconds']):.6f}", + "# HELP qbit_prism_prior_balances_read_max_seconds Longest prior-balances read since process start.", + "# TYPE qbit_prism_prior_balances_read_max_seconds gauge", + f"qbit_prism_prior_balances_read_max_seconds {float(prior_stats['max_seconds']):.6f}", + ] + ) + lines.extend(self._ledger_read_gate_metric_lines()) + startup_phases = self.port.startup_phase_seconds() + if startup_phases: + lines.extend( + [ + "# HELP qbit_prism_startup_phase_seconds Seconds from serve() start to each startup phase, recorded once.", + "# TYPE qbit_prism_startup_phase_seconds gauge", + ] + ) + for phase, seconds in sorted(startup_phases.items()): + label = self.port.prometheus_label_value(phase) + lines.append( + f'qbit_prism_startup_phase_seconds{{phase="{label}"}} {float(seconds):.6f}' + ) + return lines + + def _ledger_read_gate_metric_lines(self) -> list[str]: + """Split read-slot admission wait from SQL execution, per operation. + + A single duration per ledger call cannot say whether a budget went to + coordinator-local admission or to PostgreSQL, and issue #211 turned on + exactly that distinction: the replay enumeration reported + ``exceeded 5s`` while its statement deadline was barely touched and + the database showed no blocked backends. These two families answer it + from a scrape -- ``..._gate_wait_seconds_*`` is contention inside this + process, ``..._execute_seconds_*`` is the server (a cancelled + statement's return tail included). The timeout counters are split the + same way, so an admission expiry and a statement expiry never land on + one series. + """ + stats_fn = getattr(self.port.ledger, "ledger_read_gate_stats", None) + if not callable(stats_fn): + return [] + # Issue #224 Wave 0: the operation label is bounded by + # PRISM_LEDGER_READ_OPERATIONS. Names outside the contract fold + # into ``other`` (summed) instead of opening a series; members + # pass through untouched, so the shipped operation renders + # byte-for-byte as before. + stats_by_operation = fold_ledger_read_stats(stats_fn() or {}) + if not stats_by_operation: + return [] + # Samples stay grouped under their own family: the exposition format + # wants every line of a metric emitted as one block, so the loop is + # over families and the operations are the inner dimension. + families: tuple[tuple[str, str, str, str, bool], ...] = ( + ( + "qbit_prism_ledger_read_calls_total", + "counter", + "Ledger read-slot operations completed or failed, by operation.", + "calls_total", + True, + ), + ( + "qbit_prism_ledger_read_gate_wait_seconds_total", + "counter", + "Cumulative coordinator-local read-slot admission wait, by operation.", + "gate_wait_seconds_total", + False, + ), + ( + "qbit_prism_ledger_read_gate_wait_seconds_max", + "gauge", + "Longest coordinator-local read-slot admission wait since process start, by operation.", + "gate_wait_seconds_max", + False, + ), + ( + "qbit_prism_ledger_read_gate_timeouts_total", + "counter", + "Read-slot operations whose deadline expired before admission, so no statement was ever sent.", + "gate_timeouts_total", + True, + ), + ( + "qbit_prism_ledger_read_execute_seconds_total", + "counter", + "Cumulative PostgreSQL execution time for admitted read-slot statements, by operation.", + "execute_seconds_total", + False, + ), + ( + "qbit_prism_ledger_read_execute_seconds_max", + "gauge", + "Longest PostgreSQL execution time for an admitted read-slot statement since process start, by operation.", + "execute_seconds_max", + False, + ), + ( + "qbit_prism_ledger_read_execute_timeouts_total", + "counter", + "Admitted read-slot statements whose deadline expired inside PostgreSQL, cancel lag included.", + "execute_timeouts_total", + True, + ), + ) + operations = [ + (self.port.prometheus_label_value(str(operation)), stats) + for operation, stats in sorted(stats_by_operation.items()) + ] + lines: list[str] = [] + for name, metric_type, help_text, field, integral in families: + lines.append(f"# HELP {name} {help_text}") + lines.append(f"# TYPE {name} {metric_type}") + for label, stats in operations: + value = stats.get(field, 0) + rendered = ( + f"{int(value)}" if integral else f"{float(value):.6f}" + ) + lines.append(f'{name}{{operation="{label}"}} {rendered}') + return lines + + @staticmethod + def _duration_summary_lines( + name: str, + labels: str, + stats: dict[str, Any], + ) -> list[str]: + """``_sum`` / ``_count`` / ``_max`` for one labelled cell.""" + return [ + f'{name}_sum{{{labels}}} {float(stats["sum"]):.6f}', + f'{name}_count{{{labels}}} {int(stats["count"])}', + f'{name}_max{{{labels}}} {float(stats["max"]):.6f}', + ] + + def accepted_preview_attribution_metrics_lines(self) -> list[str]: + """Issue #224 Wave 0: the shared acceptance-to-preview attribution. + + Four summary families, one owner snapshot per render, every label + drawn from a closed vocabulary in + ``lab.prism.accepted_preview_telemetry`` and every cell of every + product rendered from zero. The export is therefore constant-size + regardless of block cadence and carries no hash or height; the + per-publication record that may carry those is the owner's + diagnostics ring, which is deliberately not a metric. + """ + snapshot = ensure_accepted_preview_telemetry(self.port).snapshot() + landing = snapshot["landing_phases"] + passes = snapshot["reconcile_passes"] + steps = snapshot["reconcile_steps"] + rescans = snapshot["full_rescans"] + landing_name = "qbit_prism_accepted_block_landing_phase_seconds" + pass_name = "qbit_prism_reorg_reconcile_pass_seconds" + step_name = "qbit_prism_reorg_reconcile_step_seconds" + rescan_name = "qbit_prism_payout_window_full_rescan_seconds" + lines = [ + f"# HELP {landing_name} Accepted-block landing wall time between definitive node acceptance and payout preview publication, by bounded sub-phase; prior_balances_check is a prior-balances reread, never a payout-window rescan.", + f"# TYPE {landing_name} summary", + ] + for phase in PRISM_ACCEPTED_LANDING_PHASES: + lines.extend( + self._duration_summary_lines( + landing_name, f'phase="{phase}"', landing[phase] + ) + ) + lines.extend( + [ + f"# HELP {pass_name} Reorg reconciliation passes by bounded caller; _count is the pass count, and a pass minus the sum of its steps is the unattributed remainder.", + f"# TYPE {pass_name} summary", + ] + ) + for caller in PRISM_REORG_RECONCILE_CALLERS: + lines.extend( + self._duration_summary_lines( + pass_name, f'caller="{caller}"', passes[caller] + ) + ) + lines.extend( + [ + f"# HELP {step_name} Reorg reconciliation wall time by bounded caller and step; admission_wait is writer admission plus the payout-balance mutation lock, and candidate_prepare is where a reconcile_invalidation full rescan executes.", + f"# TYPE {step_name} summary", + ] + ) + for caller in PRISM_REORG_RECONCILE_CALLERS: + for step in PRISM_REORG_RECONCILE_STEPS: + lines.extend( + self._duration_summary_lines( + step_name, + f'caller="{caller}",step="{step}"', + steps[(caller, step)], + ) + ) + lines.extend( + [ + f"# HELP {rescan_name} Full payout-window oracle rescans by bounded reason and window pipeline path; a prior-balances reread is a ledger read operation, not an observation here.", + f"# TYPE {rescan_name} summary", + ] + ) + for reason in PRISM_PAYOUT_WINDOW_FULL_RESCAN_REASONS: + for path in PRISM_PAYOUT_WINDOW_FULL_RESCAN_PATHS: + lines.extend( + self._duration_summary_lines( + rescan_name, + f'reason="{reason}",path="{path}"', + rescans[(reason, path)], + ) + ) + return lines + + def accepted_stats_reconcile_metric_lines(self) -> list[str]: + """Surface reconcile liveness now that failures no longer raise. + + Serving maintained counters means a failing or wedged reconcile is + invisible to callers, so its age and failure count must be visible + to scrapes for alerting instead. + """ + status_fn = getattr(self.port.ledger, "accepted_stats_reconcile_status", None) + if not callable(status_fn): + return [] + status = status_fn() + lines = [ + "# HELP qbit_prism_accepted_stats_reconcile_failures_total Failed background accepted-stats reconcile passes.", + "# TYPE qbit_prism_accepted_stats_reconcile_failures_total counter", + f"qbit_prism_accepted_stats_reconcile_failures_total {int(status.get('failures') or 0)}", + ] + age = status.get("age_seconds") + if age is not None: + lines.extend( + [ + "# HELP qbit_prism_accepted_stats_reconcile_age_seconds Seconds since the accepted-share counters were last reconciled against the ledger.", + "# TYPE qbit_prism_accepted_stats_reconcile_age_seconds gauge", + f"qbit_prism_accepted_stats_reconcile_age_seconds {float(age):.6f}", + ] + ) + return lines + + def lease_heartbeat_metrics_lines(self) -> list[str]: + """Writer-lease heartbeat phase attribution and its timing policy. + + Issue #212 was diagnosable only from exit messages after the fact: + nothing exported where a heartbeat's wall-clock went, or how much + envelope the shipped policy actually leaves. Every series here has + a fixed label vocabulary (modes, outcomes, phases and policy terms + are closed sets in ``lab.prism.writer_lease_timing``), so scrape + cardinality is constant regardless of traffic or restarts. + """ + snapshot = self.port._ensure_lease_heartbeat_service().snapshot() + attempts = snapshot["attempts"] + outcomes = snapshot["outcomes"] + last_phases = snapshot["last_phase_seconds"] + worst_phases = snapshot["worst_phase_seconds"] + policy_seconds = snapshot["policy_seconds"] + wakes = snapshot["monitor_wakes"] + probe = snapshot["stall_probe"] + assert isinstance(attempts, dict) + assert isinstance(outcomes, dict) + assert isinstance(last_phases, dict) + assert isinstance(worst_phases, dict) + assert isinstance(policy_seconds, dict) + assert isinstance(wakes, dict) + assert isinstance(probe, dict) + wake_buckets = wakes["buckets"] + late_wakes = wakes["late_wakes"] + assert isinstance(wake_buckets, dict) + assert isinstance(late_wakes, dict) + return [ + "# HELP qbit_prism_lease_heartbeat_running Whether the writer-lease heartbeat thread is alive.", + "# TYPE qbit_prism_lease_heartbeat_running gauge", + f"qbit_prism_lease_heartbeat_running {int(bool(snapshot['running']))}", + "# HELP qbit_prism_lease_heartbeat_attempts_total Guard verifications by mode.", + "# TYPE qbit_prism_lease_heartbeat_attempts_total counter", + *[ + f'qbit_prism_lease_heartbeat_attempts_total{{mode="{mode}"}} {int(attempts.get(mode, 0))}' + for mode in LEASE_HEARTBEAT_MODES + ], + "# HELP qbit_prism_lease_heartbeat_outcomes_total Guard verification outcomes.", + "# TYPE qbit_prism_lease_heartbeat_outcomes_total counter", + *[ + f'qbit_prism_lease_heartbeat_outcomes_total{{outcome="{outcome}"}} {int(outcomes.get(outcome, 0))}' + for outcome in LEASE_HEARTBEAT_OUTCOMES + ], + "# HELP qbit_prism_lease_heartbeat_phase_seconds Phase breakdown of the latest guard verification.", + "# TYPE qbit_prism_lease_heartbeat_phase_seconds gauge", + *[ + f'qbit_prism_lease_heartbeat_phase_seconds{{phase="{phase}"}} {float(last_phases.get(phase, 0.0)):.6f}' + for phase in LEASE_HEARTBEAT_PHASES + ], + "# HELP qbit_prism_lease_heartbeat_worst_phase_seconds Phase breakdown of the slowest guard verification observed.", + "# TYPE qbit_prism_lease_heartbeat_worst_phase_seconds gauge", + *[ + f'qbit_prism_lease_heartbeat_worst_phase_seconds{{phase="{phase}"}} {float(worst_phases.get(phase, 0.0)):.6f}' + for phase in LEASE_HEARTBEAT_PHASES + ], + "# HELP qbit_prism_lease_heartbeat_activity_age_seconds Age of the newest monitor-visible heartbeat activity mark.", + "# TYPE qbit_prism_lease_heartbeat_activity_age_seconds gauge", + f"qbit_prism_lease_heartbeat_activity_age_seconds {float(snapshot['activity_age_seconds']):.6f}", + "# HELP qbit_prism_lease_heartbeat_server_proven_age_seconds Age of the newest completed guard round trip.", + "# TYPE qbit_prism_lease_heartbeat_server_proven_age_seconds gauge", + f"qbit_prism_lease_heartbeat_server_proven_age_seconds {float(snapshot['server_proven_age_seconds']):.6f}", + "# HELP qbit_prism_lease_heartbeat_monitor_wake_delay_seconds Worst observed lateness of the heartbeat monitor's own poll.", + "# TYPE qbit_prism_lease_heartbeat_monitor_wake_delay_seconds gauge", + f"qbit_prism_lease_heartbeat_monitor_wake_delay_seconds {float(snapshot['monitor_wake_delay_seconds']):.6f}", + "# HELP qbit_prism_lease_heartbeat_policy_seconds Terms of the validated writer-lease heartbeat timing policy.", + "# TYPE qbit_prism_lease_heartbeat_policy_seconds gauge", + *[ + f'qbit_prism_lease_heartbeat_policy_seconds{{term="{term}"}} {float(policy_seconds.get(term, 0.0)):.6f}' + for term in LEASE_HEARTBEAT_POLICY_TERMS + ], + # Issue #227: the monitor's lateness as signals that re-arm. The + # lifetime gauge above never resets; these do the alerting. + "# HELP qbit_prism_lease_heartbeat_monitor_wake_lateness_seconds Lateness of every heartbeat-monitor poll wake (issue #227); express the wake-delay alert on this or on the late-wake counters, not on the lifetime gauge.", + "# TYPE qbit_prism_lease_heartbeat_monitor_wake_lateness_seconds histogram", + *[ + f'qbit_prism_lease_heartbeat_monitor_wake_lateness_seconds_bucket{{le="{bucket:g}"}} {int(wake_buckets.get(bucket, 0))}' + for bucket in LEASE_MONITOR_WAKE_DELAY_BUCKETS + ], + f'qbit_prism_lease_heartbeat_monitor_wake_lateness_seconds_bucket{{le="+Inf"}} {int(wakes["count"])}', + f'qbit_prism_lease_heartbeat_monitor_wake_lateness_seconds_sum {float(wakes["sum"]):.6f}', + f'qbit_prism_lease_heartbeat_monitor_wake_lateness_seconds_count {int(wakes["count"])}', + "# HELP qbit_prism_lease_heartbeat_monitor_late_wakes_total Monitor wakes at least the labelled fraction of the policy's scheduler slack late.", + "# TYPE qbit_prism_lease_heartbeat_monitor_late_wakes_total counter", + *[ + f'qbit_prism_lease_heartbeat_monitor_late_wakes_total{{slack_fraction="{fraction}"}} {int(late_wakes.get(fraction, 0))}' + for fraction in LEASE_MONITOR_LATE_WAKE_SLACK_FRACTIONS + ], + "# HELP qbit_prism_lease_heartbeat_monitor_wake_delay_window_max_seconds Worst monitor wake lateness inside the trailing 300s window; falls back once a stall ages out, unlike the lifetime gauge.", + "# TYPE qbit_prism_lease_heartbeat_monitor_wake_delay_window_max_seconds gauge", + f"qbit_prism_lease_heartbeat_monitor_wake_delay_window_max_seconds {float(wakes['window_max_seconds']):.6f}", + "# HELP qbit_prism_lease_heartbeat_monitor_wake_delay_record_age_seconds Seconds since the lifetime monitor wake-delay record was last raised.", + "# TYPE qbit_prism_lease_heartbeat_monitor_wake_delay_record_age_seconds gauge", + f"qbit_prism_lease_heartbeat_monitor_wake_delay_record_age_seconds {float(wakes['record_age_seconds']):.6f}", + "# HELP qbit_prism_lease_heartbeat_monitor_exit_guarantee_breaches_total Monitor wakes later than the policy's max_guaranteed_monitor_lateness: beats on which exit-before-adoption was not guaranteed (issue #227).", + "# TYPE qbit_prism_lease_heartbeat_monitor_exit_guarantee_breaches_total counter", + f"qbit_prism_lease_heartbeat_monitor_exit_guarantee_breaches_total {int(snapshot['exit_guarantee_breaches'])}", + "# HELP qbit_prism_lease_heartbeat_monitor_worst_exit_guarantee_overrun_seconds Worst amount by which a late monitor wake could have placed the hard exit after the successor's adoption edge.", + "# TYPE qbit_prism_lease_heartbeat_monitor_worst_exit_guarantee_overrun_seconds gauge", + f"qbit_prism_lease_heartbeat_monitor_worst_exit_guarantee_overrun_seconds {float(snapshot['worst_exit_guarantee_overrun_seconds']):.6f}", + "# HELP qbit_prism_lease_heartbeat_stall_probe_samples_total Stack samples the stall probe took on monitor wakes at least half a scheduler slack late.", + "# TYPE qbit_prism_lease_heartbeat_stall_probe_samples_total counter", + f"qbit_prism_lease_heartbeat_stall_probe_samples_total {int(probe['samples'])}", + "# HELP qbit_prism_lease_heartbeat_stall_probe_suppressed_total Stall-probe triggers refused by its rate limit (at most 3 samples per 60s).", + "# TYPE qbit_prism_lease_heartbeat_stall_probe_suppressed_total counter", + f"qbit_prism_lease_heartbeat_stall_probe_suppressed_total {int(probe['suppressed'])}", + "# HELP qbit_prism_lease_heartbeat_monitor_breach_warnings_suppressed_total Lateness-beyond-slack warnings the monitor refused under its rate limit (at most 3 per 60s); the breaches themselves are still counted.", + "# TYPE qbit_prism_lease_heartbeat_monitor_breach_warnings_suppressed_total counter", + f"qbit_prism_lease_heartbeat_monitor_breach_warnings_suppressed_total {int(snapshot['slack_breach_warnings_suppressed'])}", + ] + + def shutdown_metrics_lines(self) -> list[str]: + snapshot = self.port._ensure_shutdown_controller().snapshot() + quiescence = snapshot["writer_quiescence_outcomes"] + release = snapshot["lease_release_outcomes"] + active = snapshot["active_writers"] + assert isinstance(quiescence, dict) + assert isinstance(release, dict) + assert isinstance(active, dict) + return [ + "# HELP qbit_prism_shutdowns_total Controlled coordinator shutdown sequences started.", + "# TYPE qbit_prism_shutdowns_total counter", + f"qbit_prism_shutdowns_total {int(snapshot['shutdowns_total'])}", + "# HELP qbit_prism_shutdown_writer_operations Active admitted ledger-mutating operations by component.", + "# TYPE qbit_prism_shutdown_writer_operations gauge", + *[ + f'qbit_prism_shutdown_writer_operations{{component="{self.port.prometheus_label_value(str(component))}"}} {int(count)}' + for component, count in sorted(active.items()) + ], + "# HELP qbit_prism_shutdown_writer_quiescence_total Writer-quiescence outcomes.", + "# TYPE qbit_prism_shutdown_writer_quiescence_total counter", + *[ + f'qbit_prism_shutdown_writer_quiescence_total{{outcome="{outcome}"}} {int(quiescence.get(outcome, 0))}' + for outcome in ("success", "timeout") + ], + "# HELP qbit_prism_shutdown_writer_quiescence_seconds Duration of the latest writer-quiescence barrier.", + "# TYPE qbit_prism_shutdown_writer_quiescence_seconds gauge", + f"qbit_prism_shutdown_writer_quiescence_seconds {float(snapshot['writer_quiescence_seconds']):.6f}", + "# HELP qbit_prism_shutdown_lease_release_attempts_total Writer-lease release attempts.", + "# TYPE qbit_prism_shutdown_lease_release_attempts_total counter", + f"qbit_prism_shutdown_lease_release_attempts_total {int(snapshot['lease_release_attempts_total'])}", + "# HELP qbit_prism_shutdown_lease_release_total Writer-lease release outcomes.", + "# TYPE qbit_prism_shutdown_lease_release_total counter", + *[ + f'qbit_prism_shutdown_lease_release_total{{outcome="{outcome}"}} {int(release.get(outcome, 0))}' + for outcome in ("success", "not_held", "unsupported", "failure") + ], + "# HELP qbit_prism_shutdown_lease_release_seconds Duration of the latest writer-lease release attempt.", + "# TYPE qbit_prism_shutdown_lease_release_seconds gauge", + f"qbit_prism_shutdown_lease_release_seconds {float(snapshot['lease_release_seconds']):.6f}", + "# HELP qbit_prism_shutdown_sigterm_to_lease_release_seconds Time from SIGTERM admission close to safe lease release, or -1 if unobserved.", + "# TYPE qbit_prism_shutdown_sigterm_to_lease_release_seconds gauge", + "qbit_prism_shutdown_sigterm_to_lease_release_seconds " + + ( + f"{float(snapshot['sigterm_to_lease_release_seconds']):.6f}" + if snapshot["sigterm_release_observed"] + else "-1" + ), + "# HELP qbit_prism_shutdown_release_withheld_total Shutdowns that withheld lease release because a writer did not quiesce.", + "# TYPE qbit_prism_shutdown_release_withheld_total counter", + f"qbit_prism_shutdown_release_withheld_total {int(snapshot['release_withheld_total'])}", + "# HELP qbit_prism_shutdown_non_writer_drain_seconds Duration of cleanup after writer lease handling.", + "# TYPE qbit_prism_shutdown_non_writer_drain_seconds gauge", + f"qbit_prism_shutdown_non_writer_drain_seconds {float(snapshot['non_writer_drain_seconds']):.6f}", + ] + + def initial_delivery_metrics_lines(self) -> list[str]: + self.port._ensure_initial_job_state() + mining = self.port.mining_delivery_snapshot() + with self.port.lock: + counts = { + "sent": self.port.initial_job_sent_count, + "cancelled": self.port.initial_job_cancelled_count, + "coalesced": self.port.initial_job_coalesced_count, + "failed": self.port.initial_job_failed_count, + "superseded": self.port.initial_job_superseded_count, + } + latency_sum = self.port.initial_job_delivery_latency_seconds_sum + latency_count = self.port.initial_job_delivery_latency_count + queue_capacity_reclaimed = ( + self.port.initial_job_queue_capacity_reclaimed_count + ) + executor = getattr(self.port, "_initial_job_executor", None) + queued, slots = executor.stats() if executor is not None else (0, 0) + configured_workers = int( + getattr( + self.port, + "initial_job_max_workers", + DEFAULT_PRISM_INITIAL_JOB_MAX_WORKERS, + ) + ) + with self.port._bundle_preparation_lock: + build_counts = dict(self.port.shared_bundle_build_counts) + preparation_sum = self.port.shared_bundle_preparation_seconds_sum + preparation_count = self.port.shared_bundle_preparation_count + waiters = self.port.shared_bundle_preparation_waiters + return [ + "# HELP qbit_prism_stratum_subscribed_clients Subscribed Stratum clients.", + "# TYPE qbit_prism_stratum_subscribed_clients gauge", + f'qbit_prism_stratum_subscribed_clients {mining["subscribed_clients"]}', + "# HELP qbit_prism_stratum_authorized_clients Subscribed and authorized Stratum clients.", + "# TYPE qbit_prism_stratum_authorized_clients gauge", + f'qbit_prism_stratum_authorized_clients {mining["authorized_clients"]}', + "# HELP qbit_prism_clients_without_current_tip_job Authorized clients without usable current-tip work.", + "# TYPE qbit_prism_clients_without_current_tip_job gauge", + f'qbit_prism_clients_without_current_tip_job {mining["clients_without_current_tip_job"]}', + "# HELP qbit_prism_clients_with_no_active_job Authorized clients with no active job at all.", + "# TYPE qbit_prism_clients_with_no_active_job gauge", + f'qbit_prism_clients_with_no_active_job {mining["clients_with_no_active_job"]}', + "# HELP qbit_prism_clients_with_current_tip_job Authorized clients with usable current-tip work.", + "# TYPE qbit_prism_clients_with_current_tip_job gauge", + f'qbit_prism_clients_with_current_tip_job {mining["clients_with_current_tip_job"]}', + "# HELP qbit_prism_current_tip_job_coverage_ratio Fraction of authorized clients with current-tip work.", + "# TYPE qbit_prism_current_tip_job_coverage_ratio gauge", + f'qbit_prism_current_tip_job_coverage_ratio {float(mining["current_tip_job_coverage_ratio"]):.12g}', + "# HELP qbit_prism_initial_job_deliveries_pending Coalesced initial deliveries queued or running.", + "# TYPE qbit_prism_initial_job_deliveries_pending gauge", + f'qbit_prism_initial_job_deliveries_pending {mining["clients_pending_initial_job"]}', + "# HELP qbit_prism_initial_job_delivery_tasks_inflight Bounded shared delivery slots currently occupied.", + "# TYPE qbit_prism_initial_job_delivery_tasks_inflight gauge", + f"qbit_prism_initial_job_delivery_tasks_inflight {slots}", + "# HELP qbit_prism_initial_job_delivery_queue_depth Initial-job tasks waiting for a dedicated worker.", + "# TYPE qbit_prism_initial_job_delivery_queue_depth gauge", + f"qbit_prism_initial_job_delivery_queue_depth {queued}", + "# HELP qbit_prism_initial_job_delivery_active_workers Dedicated initial-job workers currently running tasks.", + "# TYPE qbit_prism_initial_job_delivery_active_workers gauge", + f"qbit_prism_initial_job_delivery_active_workers {slots}", + "# HELP qbit_prism_initial_job_delivery_configured_workers Configured dedicated initial-job worker count.", + "# TYPE qbit_prism_initial_job_delivery_configured_workers gauge", + f"qbit_prism_initial_job_delivery_configured_workers {configured_workers}", + "# HELP qbit_prism_initial_job_delivery_seconds Authorization-to-current-job latency.", + "# TYPE qbit_prism_initial_job_delivery_seconds summary", + f"qbit_prism_initial_job_delivery_seconds_sum {latency_sum:.6f}", + f"qbit_prism_initial_job_delivery_seconds_count {latency_count}", + "# HELP qbit_prism_initial_job_requests_total Initial delivery outcomes.", + "# TYPE qbit_prism_initial_job_requests_total counter", + *[ + f'qbit_prism_initial_job_requests_total{{result="{result}"}} {count}' + for result, count in sorted(counts.items()) + ], + "# HELP qbit_prism_initial_job_queue_capacity_reclaimed_total Queued initial-job slots reclaimed immediately by cancellation.", + "# TYPE qbit_prism_initial_job_queue_capacity_reclaimed_total counter", + f"qbit_prism_initial_job_queue_capacity_reclaimed_total {queue_capacity_reclaimed}", + "# HELP qbit_prism_shared_bundle_preparation_seconds Heavy shared bundle preparation wall time.", + "# TYPE qbit_prism_shared_bundle_preparation_seconds summary", + f"qbit_prism_shared_bundle_preparation_seconds_sum {preparation_sum:.6f}", + f"qbit_prism_shared_bundle_preparation_seconds_count {preparation_count}", + "# HELP qbit_prism_shared_bundle_preparation_waiters Callers waiting on the keyed shared preparation flight.", + "# TYPE qbit_prism_shared_bundle_preparation_waiters gauge", + f"qbit_prism_shared_bundle_preparation_waiters {waiters}", + "# HELP qbit_prism_shared_bundle_builds_total Shared bundle builds by terminal outcome.", + "# TYPE qbit_prism_shared_bundle_builds_total counter", + *[ + f'qbit_prism_shared_bundle_builds_total{{result="{result}"}} {count}' + for result, count in sorted(build_counts.items()) + ], + ] + + def job_build_metrics_lines(self) -> list[str]: + self.port._ensure_job_cache_state() + with self.port._job_cache_lock: + bucket_counts = dict(self.port.job_build_seconds_bucket_counts) + build_sum = self.port.job_build_seconds_sum + build_count = self.port.job_build_count + phase_seconds = dict(self.port.job_build_phase_seconds) + hit_counts = dict(self.port.job_cache_hit_counts) + miss_counts = dict(self.port.job_cache_miss_counts) + # Health-refresh failure counts are owner state; read them from + # the observability service directly (no coordinator property). + health_refresh_failures = ( + self.port._ensure_observability_service() + .state() + .health_snapshot_refresh_failure_count + ) + with self.port._job_build_scheduler_lock: + scheduler_counts = dict(self.port.job_build_scheduler_counts) + priority_counts = dict(self.port.job_build_priority_counts) + priority_admission_seconds = dict( + self.port.job_build_priority_admission_seconds + ) + initial_prepared_counts = dict( + self.port.initial_job_prepared_work_counts + ) + cancellation_seconds = dict(self.port.job_build_cancellation_seconds) + replacement_seconds = dict( + self.port.job_build_replacement_start_seconds + ) + worker_counts = dict(self.port.job_build_worker_counts) + active_builds = int(self.port._job_build_active is not None) + pending_builds = int(self.port._job_build_pending is not None) + priority_requests = tuple( + request + for request in ( + ( + self.port._job_build_active.request + if self.port._job_build_active is not None + else None + ), + ( + self.port._job_build_retiring.request + if self.port._job_build_retiring is not None + else None + ), + self.port._job_build_pending, + ) + if request is not None + and not request.cancellation.is_set() + and self.port._job_build_is_publication_critical(request) + ) + priority_preparations = tuple( + self.port._job_build_priority_preparations.values() + ) + priority_active = int( + bool(priority_requests or priority_preparations) + ) + priority_age_seconds = max( + ( + time.monotonic() - request.requested_monotonic + for request in priority_requests + ), + default=0.0, + ) + priority_age_seconds = max( + priority_age_seconds, + max( + ( + time.monotonic() - started + for started in priority_preparations + ), + default=0.0, + ), + ) + lock = getattr(self.port, "lock", None) + if lock is not None: + with lock: + connected_clients = len(getattr(self.port, "clients", ())) + else: + connected_clients = len(getattr(self.port, "clients", ())) + lines = [ + "# HELP qbit_prism_job_build_seconds Wall time from client job build or prepared submission to completion, including skipped prepared tasks.", + "# TYPE qbit_prism_job_build_seconds histogram", + ] + for bucket in PRISM_JOB_BUILD_SECONDS_BUCKETS: + lines.append( + f'qbit_prism_job_build_seconds_bucket{{le="{bucket:g}"}} {bucket_counts.get(bucket, 0)}' + ) + lines.extend( + [ + f'qbit_prism_job_build_seconds_bucket{{le="+Inf"}} {build_count}', + f"qbit_prism_job_build_seconds_sum {build_sum:.6f}", + f"qbit_prism_job_build_seconds_count {build_count}", + "# HELP qbit_prism_job_build_phase_seconds_total Cumulative job build wall time by phase.", + "# TYPE qbit_prism_job_build_phase_seconds_total counter", + *[ + f'qbit_prism_job_build_phase_seconds_total{{phase="{phase}"}} {phase_seconds.get(phase, 0.0):.6f}' + for phase in PRISM_JOB_BUILD_PHASES + ], + "# HELP qbit_prism_job_cache_hits_total Job build cache hits by cache kind.", + "# TYPE qbit_prism_job_cache_hits_total counter", + *[ + f'qbit_prism_job_cache_hits_total{{cache="{kind}"}} {int(hit_counts.get(kind, 0))}' + for kind in PRISM_JOB_CACHE_KINDS + ], + "# HELP qbit_prism_job_cache_misses_total Job build cache misses by cache kind.", + "# TYPE qbit_prism_job_cache_misses_total counter", + *[ + f'qbit_prism_job_cache_misses_total{{cache="{kind}"}} {int(miss_counts.get(kind, 0))}' + for kind in PRISM_JOB_CACHE_KINDS + ], + "# HELP qbit_prism_health_snapshot_refresh_failures_total Background health snapshot refreshes that raised.", + "# TYPE qbit_prism_health_snapshot_refresh_failures_total counter", + f"qbit_prism_health_snapshot_refresh_failures_total {health_refresh_failures}", + "# HELP qbit_prism_connected_clients Currently connected Stratum clients.", + "# TYPE qbit_prism_connected_clients gauge", + f"qbit_prism_connected_clients {connected_clients}", + "# HELP qbit_prism_job_build_requests_total Immutable job build requests admitted to the latest-wins scheduler.", + "# TYPE qbit_prism_job_build_requests_total counter", + f'qbit_prism_job_build_requests_total {int(scheduler_counts.get("requests", 0))}', + "# HELP qbit_prism_job_build_starts_total Immutable job builds started by the bounded executor.", + "# TYPE qbit_prism_job_build_starts_total counter", + f'qbit_prism_job_build_starts_total {int(scheduler_counts.get("starts", 0))}', + "# HELP qbit_prism_job_build_completions_total Immutable job build executions completed.", + "# TYPE qbit_prism_job_build_completions_total counter", + f'qbit_prism_job_build_completions_total {int(scheduler_counts.get("completions", 0))}', + "# HELP qbit_prism_job_build_supersessions_total Active or pending builds replaced by a newer immutable key.", + "# TYPE qbit_prism_job_build_supersessions_total counter", + f'qbit_prism_job_build_supersessions_total {int(scheduler_counts.get("supersessions", 0))}', + "# HELP qbit_prism_job_build_obsolete_results_total Obsolete build results discarded before cache or delivery.", + "# TYPE qbit_prism_job_build_obsolete_results_total counter", + f'qbit_prism_job_build_obsolete_results_total {int(scheduler_counts.get("obsolete_results", 0))}', + "# HELP qbit_prism_job_build_orphan_evicted_total Finished flights evicted from scheduler slots after their completion never released them.", + "# TYPE qbit_prism_job_build_orphan_evicted_total counter", + f'qbit_prism_job_build_orphan_evicted_total {int(scheduler_counts.get("orphan_evicted", 0))}', + "# HELP qbit_prism_job_build_active Current latest-generation build executions.", + "# TYPE qbit_prism_job_build_active gauge", + f"qbit_prism_job_build_active {active_builds}", + "# HELP qbit_prism_job_build_pending Newest build request waiting for a bounded executor slot.", + "# TYPE qbit_prism_job_build_pending gauge", + f"qbit_prism_job_build_pending {pending_builds}", + "# HELP qbit_prism_job_build_cancellation_seconds Cancellation signal to obsolete execution completion.", + "# TYPE qbit_prism_job_build_cancellation_seconds summary", + f'qbit_prism_job_build_cancellation_seconds_sum {float(cancellation_seconds.get("sum", 0.0)):.6f}', + f'qbit_prism_job_build_cancellation_seconds_count {int(cancellation_seconds.get("count", 0))}', + "# HELP qbit_prism_job_build_replacement_start_seconds Supersession signal to replacement build start.", + "# TYPE qbit_prism_job_build_replacement_start_seconds summary", + f'qbit_prism_job_build_replacement_start_seconds_sum {float(replacement_seconds.get("sum", 0.0)):.6f}', + f'qbit_prism_job_build_replacement_start_seconds_count {int(replacement_seconds.get("count", 0))}', + "# HELP qbit_prism_job_build_priority_events_total Publication-critical scheduler admissions and routine-work displacement.", + "# TYPE qbit_prism_job_build_priority_events_total counter", + *[ + f'qbit_prism_job_build_priority_events_total{{result="{result}"}} {int(priority_counts.get(result, 0))}' + for result in ( + "started", + "coalesced", + "queued", + "routine_deferred", + "routine_preempted", + ) + ], + "# HELP qbit_prism_job_build_priority_admission_seconds Publication-priority reservation to builder start or exact-flight coalescing.", + "# TYPE qbit_prism_job_build_priority_admission_seconds summary", + f'qbit_prism_job_build_priority_admission_seconds_sum {float(priority_admission_seconds.get("sum", 0.0)):.6f}', + f'qbit_prism_job_build_priority_admission_seconds_count {int(priority_admission_seconds.get("count", 0))}', + "# HELP qbit_prism_job_build_priority_active Whether publication-critical build work is preparing, running, retiring, or pending.", + "# TYPE qbit_prism_job_build_priority_active gauge", + f"qbit_prism_job_build_priority_active {priority_active}", + "# HELP qbit_prism_job_build_priority_age_seconds Age of the oldest admitted publication-critical build request.", + "# TYPE qbit_prism_job_build_priority_age_seconds gauge", + f"qbit_prism_job_build_priority_age_seconds {priority_age_seconds:.6f}", + "# HELP qbit_prism_initial_job_prepared_work_total Initial jobs that reused, coalesced behind, or deferred to prepared shared work.", + "# TYPE qbit_prism_initial_job_prepared_work_total counter", + *[ + f'qbit_prism_initial_job_prepared_work_total{{result="{result}"}} {int(initial_prepared_counts.get(result, 0))}' + for result in ( + "cache_hit", + "singleflight", + "deferred", + "subscribed", + "admission_deadline", + ) + ], + "# HELP qbit_prism_job_build_worker_events_total Pure builder subprocess lifecycle events.", + "# TYPE qbit_prism_job_build_worker_events_total counter", + *[ + f'qbit_prism_job_build_worker_events_total{{event="{event}"}} {int(worker_counts.get(event, 0))}' + for event in ("starts", "terminations", "crashes", "restarts") + ], + "# HELP qbit_prism_job_build_configured_workers Configured bounded job-build executor worker count.", + "# TYPE qbit_prism_job_build_configured_workers gauge", + ( + "qbit_prism_job_build_configured_workers " + f"{int(getattr(self.port, 'job_build_executor_workers', DEFAULT_PRISM_JOB_BUILD_EXECUTOR_WORKERS))}" + ), + ] + ) + return lines diff --git a/lab/prism/observability.py b/lab/prism/observability.py new file mode 100644 index 000000000..b5a9c37bc --- /dev/null +++ b/lab/prism/observability.py @@ -0,0 +1,780 @@ +"""Cached PRISM base health with a fresh monotonic progress overlay.""" + +from __future__ import annotations + +from dataclasses import dataclass +import threading +from typing import Callable, Iterator, Mapping, Protocol + +from lab.prism.progress_health import overlay_progress_health + + +HEALTH_SCHEMA = "qbit.prism.audit-health.v1" +MINIMUM_HEALTH_STALE_SECONDS = 15.0 +METRICS_FAILURE_CLASSES = ("exception", "invalid_payload") + +# What /metrics says about the payload it just returned, so a consumer can tell +# fresh from stale from unavailable without parsing the Prometheus body (issue +# #184). Same shape as #164's database-state header: one bounded value an +# operator greps and a dashboard frontend switches a banner on, rather than a +# warn-code line to parse or a gauge to scrape out of the body it qualifies. +METRICS_STATE_HEADER = "X-Prism-Metrics-State" +METRICS_STATE_FRESH = "fresh" +METRICS_STATE_STALE = "stale" +METRICS_STATE_UNAVAILABLE = "unavailable" +METRICS_STATES = ( + METRICS_STATE_FRESH, + METRICS_STATE_STALE, + METRICS_STATE_UNAVAILABLE, +) + +# Warning 110 is the registered "Response is Stale" warn-code (RFC 7234 5.5.1), +# with this service named as the agent that added it, exactly as #164 says it +# on a cached public response. Said only on a stale payload this endpoint +# actually served: the warm-up refusal has no cached body to qualify. +METRICS_STALE_WARNING_HEADER = "Warning" +METRICS_STALE_WARNING = ( + '110 qbit-prism "metrics snapshot is stale; serving last complete payload"' +) + + +@dataclass(frozen=True) +class MiningDeliveryInputs: + """One immutable collection of coordinator facts used for health policy.""" + + active_connections: int + connection_capacity: int + peak_active_connections: int + subscribed_connections: int + authorized_connections: int + pending_initial_jobs: int + pending_initial_job_capacity: int + oldest_pending_initial_job_age_seconds: float + oldest_genuinely_pending_initial_job_age_seconds: float + clients_with_current_tip_jobs: int + clients_with_semantically_current_work: int + clients_with_no_active_job: int + last_initial_job_delivery_monotonic: float | None + initial_job_timeout_seconds: float + initial_job_queue_rejections: int + initial_job_timeout_disconnects: int + initial_job_cancelled_tasks: int + initial_job_coalesced_tasks: int + initial_job_queue_capacity_reclaimed: int + handler_threads: int + delivery_executor_queue_depth: int + delivery_executor_active_workers: int + started_monotonic: float + startup_grace_seconds: float + stale_unknown_rejections: int + submitted_shares: int + job_preparation_pending: bool + current_observed_tip: str | None + prepared_bundle_current: bool + prepared_bundle_tip: str | None + prepared_bundle_template_generation: int | None + prepared_bundle_payout_generation: int | None + + +class ObservabilityPort(Protocol): + """Narrow coordinator adapter required by health collection.""" + + def monotonic(self) -> float: ... + + def mining_delivery_inputs(self, now: float) -> MiningDeliveryInputs: ... + + def accepted_share_stats(self) -> tuple[int, int]: ... + + def ledger_backend(self) -> str: ... + + def block_counts(self) -> tuple[int, int]: ... + + def progress_health(self) -> Mapping[str, object]: ... + + def health_refresh_seconds(self) -> float: ... + + def render_metrics_payload(self) -> str: ... + + def metrics_refresh_seconds(self) -> float: ... + + def record_startup_phase(self, phase: str) -> None: ... + + def stop_requested(self) -> bool: ... + + def wait_for_stop(self, timeout: float) -> bool: ... + + def log(self, message: str) -> None: ... + + def log_exception(self) -> None: ... + + +@dataclass(frozen=True) +class ObservabilityState: + health_snapshot: dict[str, object] | None + health_snapshot_monotonic: float | None + health_refresh_loop_running: bool + health_snapshot_refresh_failure_count: int + mining_overload_started_monotonic: float | None + mining_delivery_failure_started_monotonic: float | None + mining_semantic_work_gap_started_monotonic: float | None + + +@dataclass(frozen=True) +class MetricsObservabilityState: + metrics_snapshot: str | None + metrics_snapshot_monotonic: float | None + metrics_refresh_loop_running: bool + metrics_collection_success_count: int + metrics_collection_failure_count: int + metrics_collection_generation: int + metrics_failure_exception_count: int + metrics_failure_invalid_payload_count: int + metrics_last_failure_class: str | None + + +@dataclass(frozen=True) +class MetricsSnapshotResponse: + """One /metrics answer: the bytes to send, and what they are (issue #184). + + Iterable as the ``(status, body)`` pair this read used to return, so the + callers that only ever needed those two keep working unchanged while the + HTTP layer gets the freshness facts it must publish out of band. + """ + + status: int + body: str + state: str + age_seconds: int | None + + def __iter__(self) -> Iterator[object]: + yield self.status + yield self.body + + def response_headers(self) -> dict[str, str]: + """State this answer's freshness where no body parsing is needed.""" + + headers = { + # A scrape answer is never re-servable: an intermediary holding a + # stale body would republish it under this response's own Age. + "Cache-Control": "no-store", + METRICS_STATE_HEADER: self.state, + } + if self.age_seconds is not None: + headers["Age"] = str(self.age_seconds) + if self.state == METRICS_STATE_STALE: + headers[METRICS_STALE_WARNING_HEADER] = METRICS_STALE_WARNING + return headers + + +class _InvalidMetricsPayload(ValueError): + pass + + +class ObservabilityService: + """Own health policy, the cached base snapshot, and refresher lifecycle.""" + + def __init__( + self, + port: ObservabilityPort, + *, + lock_factory: Callable[[], threading.RLock] = threading.RLock, + ) -> None: + self.port = port + self._lock: threading.RLock = lock_factory() + self._health_snapshot: dict[str, object] | None = None + self._health_snapshot_monotonic: float | None = None + self._health_refresh_loop_running = False + self._health_snapshot_refresh_failure_count = 0 + self._mining_overload_started_monotonic: float | None = None + self._mining_delivery_failure_started_monotonic: float | None = None + self._mining_semantic_work_gap_started_monotonic: float | None = None + self._mining_delivery_lock = threading.Lock() + self._metrics_lock = threading.RLock() + self._metrics_collection_lock = threading.Lock() + self._metrics_snapshot: str | None = None + self._metrics_snapshot_monotonic: float | None = None + self._metrics_refresh_loop_running = False + self._metrics_collection_success_count = 0 + self._metrics_collection_failure_count = 0 + self._metrics_collection_generation = 0 + self._metrics_failure_counts = { + failure_class: 0 for failure_class in METRICS_FAILURE_CLASSES + } + self._metrics_last_failure_class: str | None = None + + def state(self) -> ObservabilityState: + with self._lock: + return ObservabilityState( + health_snapshot=( + None + if self._health_snapshot is None + else dict(self._health_snapshot) + ), + health_snapshot_monotonic=self._health_snapshot_monotonic, + health_refresh_loop_running=self._health_refresh_loop_running, + health_snapshot_refresh_failure_count=( + self._health_snapshot_refresh_failure_count + ), + mining_overload_started_monotonic=( + self._mining_overload_started_monotonic + ), + mining_delivery_failure_started_monotonic=( + self._mining_delivery_failure_started_monotonic + ), + mining_semantic_work_gap_started_monotonic=( + self._mining_semantic_work_gap_started_monotonic + ), + ) + + def metrics_state(self) -> MetricsObservabilityState: + with self._metrics_lock: + return MetricsObservabilityState( + metrics_snapshot=self._metrics_snapshot, + metrics_snapshot_monotonic=self._metrics_snapshot_monotonic, + metrics_refresh_loop_running=self._metrics_refresh_loop_running, + metrics_collection_success_count=( + self._metrics_collection_success_count + ), + metrics_collection_failure_count=( + self._metrics_collection_failure_count + ), + metrics_collection_generation=self._metrics_collection_generation, + metrics_failure_exception_count=self._metrics_failure_counts[ + "exception" + ], + metrics_failure_invalid_payload_count=self._metrics_failure_counts[ + "invalid_payload" + ], + metrics_last_failure_class=self._metrics_last_failure_class, + ) + + def replace_lock_for_test(self, lock: threading.RLock) -> None: + self._lock = lock + + def clear_health_snapshot_for_test(self) -> None: + with self._lock: + self._health_snapshot = None + + def set_health_snapshot_monotonic_for_test( + self, + value: float | None, + ) -> None: + with self._lock: + self._health_snapshot_monotonic = value + + def set_loop_running_for_test(self, value: bool) -> None: + with self._lock: + self._health_refresh_loop_running = bool(value) + + def set_delivery_failure_started_monotonic_for_test( + self, + value: float | None, + ) -> None: + # A test that starts past the deadline means the whole delivery gap: + # seed the semantic timer alongside the strict tip-hash one. + with self._lock: + self._mining_delivery_failure_started_monotonic = value + self._mining_semantic_work_gap_started_monotonic = value + + def reset_delivery_failure(self) -> None: + """Start the next current-tip coverage gap with a fresh grace window.""" + + with self._lock: + self._mining_delivery_failure_started_monotonic = None + + def mining_delivery_snapshot( + self, + *, + now: float | None = None, + ) -> dict[str, object]: + with self._mining_delivery_lock: + return self._mining_delivery_snapshot_serialized(now=now) + + def _mining_delivery_snapshot_serialized( + self, + *, + now: float | None, + ) -> dict[str, object]: + now = self.port.monotonic() if now is None else now + inputs = self.port.mining_delivery_inputs(now) + authorized = inputs.authorized_connections + current = inputs.clients_with_current_tip_jobs + semantic_current = inputs.clients_with_semantically_current_work + coverage = current / authorized if authorized else 1.0 + semantic_coverage = ( + semantic_current / authorized if authorized else 1.0 + ) + cap_saturated = ( + inputs.connection_capacity > 0 + and inputs.active_connections >= inputs.connection_capacity + ) + pending_saturated = ( + inputs.pending_initial_jobs >= inputs.pending_initial_job_capacity + ) + # A reconnect incident is operationally significant well before + # nearly every miner is missing work. Treat any sustained loss of + # at least five percent of current-job coverage as degraded. + poor_coverage = authorized > 0 and coverage < 0.95 + # Exact-tip churn alone must not fail health (#216): a fleet holding + # work for the current template content is being served even while + # the strict tip-hash gauge lags the observed tip, so the delivery + # stall is timed on semantic coverage. Boundary: overload_now, + # reject_storm, and the cap-saturated reason deliberately stay on + # exact-tip coverage. Each already needs independent corroboration + # (queue/cap saturation or an observed rejection storm) and + # diagnoses strict-tip operational pressure; semantic coverage + # changes only the delivery-stalled predicate. + poor_semantic_coverage = authorized > 0 and semantic_coverage < 0.95 + overload_now = pending_saturated or (cap_saturated and poor_coverage) + with self._lock: + if poor_coverage: + if self._mining_delivery_failure_started_monotonic is None: + self._mining_delivery_failure_started_monotonic = now + else: + self._mining_delivery_failure_started_monotonic = None + delivery_failure_started = ( + self._mining_delivery_failure_started_monotonic + ) + if poor_semantic_coverage: + if self._mining_semantic_work_gap_started_monotonic is None: + self._mining_semantic_work_gap_started_monotonic = now + else: + self._mining_semantic_work_gap_started_monotonic = None + semantic_gap_started = ( + self._mining_semantic_work_gap_started_monotonic + ) + if overload_now: + if self._mining_overload_started_monotonic is None: + self._mining_overload_started_monotonic = now + else: + self._mining_overload_started_monotonic = None + overload_started = self._mining_overload_started_monotonic + delivery_failure_age = ( + max(0.0, now - delivery_failure_started) + if delivery_failure_started is not None + else 0.0 + ) + semantic_gap_age = ( + max(0.0, now - semantic_gap_started) + if semantic_gap_started is not None + else 0.0 + ) + overload_age = ( + max(0.0, now - overload_started) + if overload_started is not None + else 0.0 + ) + deadline = ( + inputs.initial_job_timeout_seconds + if inputs.initial_job_timeout_seconds > 0 + else None + ) + startup_age = max(0.0, now - inputs.started_monotonic) + in_startup_grace = startup_age < inputs.startup_grace_seconds + # A nonzero genuine pending age implies at least one genuinely + # pending authorized client existed at capture time, so the age + # threshold alone is the starvation predicate. + initial_job_starved = bool( + deadline is not None + and inputs.oldest_genuinely_pending_initial_job_age_seconds >= deadline + ) + # The semantic gap must be sustained for a full deadline, and the + # strict gap alone can never fire it. Exact current-tip work is + # checked against the same template fingerprint, payout generation, + # and client-session generations whenever a template snapshot is + # published, so in production it is a subset of the semantic count + # and this reduces to the semantic gap; an embedder that publishes + # only the observed tip has no semantic evidence, and its exact-tip + # work must still count as delivered. + current_work_coverage_stalled = bool( + deadline is not None + and poor_semantic_coverage + and semantic_gap_age >= deadline + and poor_coverage + and delivery_failure_age >= deadline + ) + no_delivery_progress = bool( + initial_job_starved or current_work_coverage_stalled + ) + reject_storm = ( + poor_coverage + and inputs.submitted_shares > 0 + and inputs.stale_unknown_rejections / inputs.submitted_shares >= 0.95 + ) + persistent_overload = deadline is not None and overload_age >= deadline + unhealthy_reasons: list[str] = [] + if not in_startup_grace: + if no_delivery_progress: + unhealthy_reasons.append("initial-delivery-stalled") + if pending_saturated and persistent_overload: + unhealthy_reasons.append("pending-initial-jobs-saturated") + if cap_saturated and poor_coverage and persistent_overload: + unhealthy_reasons.append("connection-capacity-saturated") + if reject_storm: + unhealthy_reasons.append("stale-unknown-rejection-storm") + mining_ready = not unhealthy_reasons + return { + "mining_ready": mining_ready, + "mining_delivery_healthy": mining_ready, + "mining_health_startup_grace": in_startup_grace, + "active_connections": inputs.active_connections, + "connection_capacity": inputs.connection_capacity, + "peak_active_connections": inputs.peak_active_connections, + "subscribed_connections": inputs.subscribed_connections, + "authorized_connections": authorized, + "pending_initial_jobs": inputs.pending_initial_jobs, + "pending_initial_job_capacity": inputs.pending_initial_job_capacity, + "oldest_pending_initial_job_age_seconds": round( + inputs.oldest_pending_initial_job_age_seconds, + 3, + ), + "oldest_genuinely_pending_initial_job_age_seconds": round( + inputs.oldest_genuinely_pending_initial_job_age_seconds, + 3, + ), + "clients_with_current_tip_jobs": current, + "current_tip_job_coverage": round(coverage, 6), + "clients_with_semantically_current_work": semantic_current, + "semantic_current_work_ratio": round(semantic_coverage, 6), + "current_tip_coverage_gap_age_seconds": round( + delivery_failure_age, + 3, + ), + "semantic_current_work_gap_age_seconds": round( + semantic_gap_age, + 3, + ), + "connection_capacity_saturated": cap_saturated, + "pending_initial_jobs_saturated": pending_saturated, + "initial_delivery_stalled": no_delivery_progress, + "overload": bool(overload_now or reject_storm), + "overload_age_seconds": round(overload_age, 3), + "unhealthy_reasons": unhealthy_reasons, + "initial_job_queue_rejections": inputs.initial_job_queue_rejections, + "initial_job_timeout_disconnects": ( + inputs.initial_job_timeout_disconnects + ), + "initial_job_cancelled_tasks": inputs.initial_job_cancelled_tasks, + "initial_job_coalesced_tasks": inputs.initial_job_coalesced_tasks, + "initial_job_queue_capacity_reclaimed": ( + inputs.initial_job_queue_capacity_reclaimed + ), + "handler_threads": inputs.handler_threads, + "delivery_executor_queue_depth": ( + inputs.delivery_executor_queue_depth + ), + "delivery_executor_active_workers": ( + inputs.delivery_executor_active_workers + ), + # Compatibility aliases and preparation visibility introduced by + # the prewarm work. These retain the original bounded-pipeline + # names above for existing dashboards. + "subscribed_clients": inputs.subscribed_connections, + "authorized_clients": authorized, + "clients_with_no_active_job": inputs.clients_with_no_active_job, + "clients_without_current_tip_job": authorized - current, + "clients_with_current_tip_job": current, + "clients_pending_initial_job": inputs.pending_initial_jobs, + "current_tip_job_coverage_ratio": coverage, + # Compatibility alias: this now reports only genuine first-job + # starvation. Current-tip fanout lag has its own age above. + "oldest_initial_job_pending_seconds": round( + inputs.oldest_genuinely_pending_initial_job_age_seconds, + 3, + ), + "job_preparation_pending": inputs.job_preparation_pending, + "current_observed_tip": inputs.current_observed_tip, + "prepared_bundle_current": inputs.prepared_bundle_current, + "prepared_bundle_tip": inputs.prepared_bundle_tip, + "prepared_bundle_template_generation": ( + inputs.prepared_bundle_template_generation + ), + "prepared_bundle_payout_generation": ( + inputs.prepared_bundle_payout_generation + ), + } + + def base_health_payload(self) -> dict[str, object]: + accepted_share_count, ready_miner_count = self.port.accepted_share_stats() + mining = self.mining_delivery_snapshot() + accepted_block_count, max_blocks = self.port.block_counts() + return { + "ok": bool(mining["mining_ready"]), + "schema": HEALTH_SCHEMA, + "ledger_backend": self.port.ledger_backend(), + "accepted_share_count": accepted_share_count, + "ready_miner_count": ready_miner_count, + "accepted_block": accepted_block_count > 0, + "accepted_block_count": accepted_block_count, + "max_blocks": max_blocks, + **mining, + } + + def _with_current_progress( + self, + base_health: Mapping[str, object], + ) -> dict[str, object]: + return self.apply_progress_health( + base_health, + self.port.progress_health(), + ) + + @staticmethod + def apply_progress_health( + base_health: Mapping[str, object], + progress: Mapping[str, object], + ) -> dict[str, object]: + """Retain the existing overlay contract behind the new owner.""" + + return dict(overlay_progress_health(base_health, progress)) + + def health_payload(self) -> dict[str, object]: + return self._with_current_progress(self.base_health_payload()) + + def refresh_health_snapshot(self) -> dict[str, object]: + # Cache only ledger/session-backed base health. Progress is monotonic, + # in-memory state and is deliberately re-read for every response. + base_health = self.base_health_payload() + with self._lock: + self._health_snapshot = base_health + self._health_snapshot_monotonic = self.port.monotonic() + self.port.record_startup_phase("health_snapshot_warm") + return self._with_current_progress(base_health) + + def cached_health_payload(self) -> tuple[int, dict[str, object]]: + with self._lock: + snapshot = ( + None + if self._health_snapshot is None + else dict(self._health_snapshot) + ) + snapshot_monotonic = self._health_snapshot_monotonic + loop_running = self._health_refresh_loop_running + if snapshot is None or snapshot_monotonic is None: + if not loop_running: + # No refresher (tests, or audit HTTP without serve()): compute + # inline like the legacy endpoint did. + payload = self.refresh_health_snapshot() + return (200 if payload.get("ok") else 503), payload + # The refresher owns the cold accepted-share warm-up; until its + # first snapshot lands, report an explicit starting state instead + # of running ledger aggregates on the handler thread (issue #188 + # fix 4). + payload = self._with_current_progress( + { + "ok": False, + "schema": HEALTH_SCHEMA, + "state": "starting", + "error": "health snapshot warm-up has not completed yet", + } + ) + payload["ok"] = False + return 503, payload + age_seconds = self.port.monotonic() - snapshot_monotonic + stale_after = max( + 3 * self.port.health_refresh_seconds(), + MINIMUM_HEALTH_STALE_SECONDS, + ) + if age_seconds > stale_after: + payload = self._with_current_progress( + { + "ok": False, + "schema": HEALTH_SCHEMA, + "error": "health snapshot is stale", + "snapshot_age_seconds": round(age_seconds, 3), + } + ) + payload["ok"] = False + return 503, payload + # Ledger-backed fields stay cached, but progress state is an in-memory + # monotonic snapshot and must be overlaid on every request. Otherwise a + # cached ok=true response can mask a known failed refresh for another + # full cache cycle (the production incident this endpoint must expose). + payload = self._with_current_progress(snapshot) + payload["snapshot_age_seconds"] = round(age_seconds, 3) + return (200 if payload.get("ok") else 503), payload + + def begin_refresh_loop(self) -> bool: + with self._lock: + if self._health_refresh_loop_running: + return False + self._health_refresh_loop_running = True + return True + + def health_snapshot_loop(self) -> None: + # The running flag is a one-shot latch (upstream #120): it is armed + # before the refresher thread can be dispatched and deliberately + # never cleared, so a request racing loop exit (or arriving during + # shutdown drain) fails closed through the cached path instead of + # re-opening cached_health_payload's legacy inline aggregate. + while not self.port.stop_requested(): + try: + self.refresh_health_snapshot() + except Exception: + with self._lock: + self._health_snapshot_refresh_failure_count += 1 + self.port.log("prism coordinator: health snapshot refresh failed") + self.port.log_exception() + if self.port.wait_for_stop(self.port.health_refresh_seconds()): + break + + @staticmethod + def _metrics_failure_class(exc: Exception) -> str: + if isinstance(exc, _InvalidMetricsPayload): + return "invalid_payload" + return "exception" + + def refresh_metrics_snapshot(self) -> str: + """Collect and atomically publish one complete Prometheus document.""" + + try: + with self._metrics_collection_lock: + payload = self.port.render_metrics_payload() + if not isinstance(payload, str) or not payload or not payload.endswith( + "\n" + ): + raise _InvalidMetricsPayload( + "metrics renderer must return non-empty newline-terminated text" + ) + collected_monotonic = self.port.monotonic() + with self._metrics_lock: + self._metrics_snapshot = payload + self._metrics_snapshot_monotonic = collected_monotonic + self._metrics_collection_success_count += 1 + self._metrics_collection_generation += 1 + self._metrics_last_failure_class = None + return payload + except Exception as exc: + failure_class = self._metrics_failure_class(exc) + with self._metrics_lock: + self._metrics_collection_failure_count += 1 + self._metrics_failure_counts[failure_class] += 1 + self._metrics_last_failure_class = failure_class + raise + + def _metrics_snapshot_response( + self, + *, + now: float, + refresh_seconds: float, + ) -> MetricsSnapshotResponse: + with self._metrics_lock: + snapshot = self._metrics_snapshot + snapshot_monotonic = self._metrics_snapshot_monotonic + running = self._metrics_refresh_loop_running + successes = self._metrics_collection_success_count + generation = self._metrics_collection_generation + failure_counts = dict(self._metrics_failure_counts) + age_seconds = ( + -1.0 + if snapshot_monotonic is None + else max(0.0, now - snapshot_monotonic) + ) + stale_after = max(3 * refresh_seconds, MINIMUM_HEALTH_STALE_SECONDS) + stale = snapshot is None or age_seconds > stale_after + diagnostic_lines = [ + "# HELP qbit_prism_metrics_snapshot_available Whether a complete metrics snapshot is available.", + "# TYPE qbit_prism_metrics_snapshot_available gauge", + f"qbit_prism_metrics_snapshot_available {1 if snapshot is not None else 0}", + "# HELP qbit_prism_metrics_snapshot_stale Whether the complete metrics snapshot exceeded its staleness budget.", + "# TYPE qbit_prism_metrics_snapshot_stale gauge", + f"qbit_prism_metrics_snapshot_stale {1 if stale else 0}", + "# HELP qbit_prism_metrics_snapshot_age_seconds Age of the complete metrics snapshot, or -1 before first success.", + "# TYPE qbit_prism_metrics_snapshot_age_seconds gauge", + f"qbit_prism_metrics_snapshot_age_seconds {age_seconds:.3f}", + "# HELP qbit_prism_metrics_collector_running Whether the background metrics collector is running.", + "# TYPE qbit_prism_metrics_collector_running gauge", + f"qbit_prism_metrics_collector_running {1 if running else 0}", + "# HELP qbit_prism_metrics_collection_successes_total Complete metrics snapshots published.", + "# TYPE qbit_prism_metrics_collection_successes_total counter", + f"qbit_prism_metrics_collection_successes_total {successes}", + "# HELP qbit_prism_metrics_collection_failures_total Metrics collection failures by bounded class.", + "# TYPE qbit_prism_metrics_collection_failures_total counter", + *( + f'qbit_prism_metrics_collection_failures_total{{class="{failure_class}"}} {failure_counts[failure_class]}' + for failure_class in METRICS_FAILURE_CLASSES + ), + "# HELP qbit_prism_metrics_snapshot_generation Complete metrics snapshot generation.", + "# TYPE qbit_prism_metrics_snapshot_generation gauge", + f"qbit_prism_metrics_snapshot_generation {generation}", + ] + prefix = "" if snapshot is None else snapshot + body = prefix + "\n".join(diagnostic_lines) + "\n" + if snapshot is None: + # Nothing complete has ever been published, so there is no payload + # to serve and no truthful age to publish beside it. Refuse, and + # say so out of band too: this body is diagnostics only, and a + # scraper that stored it would be storing a metrics document that + # is missing every metric the process actually reports. + return MetricsSnapshotResponse( + status=503, + body=body, + state=METRICS_STATE_UNAVAILABLE, + age_seconds=None, + ) + # A complete payload exists, so it is served -- stale or not (#184). + # Prometheus discards the body of a non-200, so answering 503 here threw + # the last known-good document away at exactly the moment it was the + # only thing left to look at, and took the diagnostic gauges above with + # it. The staleness is not hidden: it travels in the response metadata + # and in snapshot_stale/snapshot_age_seconds. Nothing is rendered or + # collected on this thread either way -- the published generation is + # returned as it stands, so a collector wedged on the GIL or on the + # renderer lock cannot delay or re-enter this path. + return MetricsSnapshotResponse( + status=200, + body=body, + state=METRICS_STATE_STALE if stale else METRICS_STATE_FRESH, + age_seconds=int(age_seconds), + ) + + def cached_metrics_payload(self) -> MetricsSnapshotResponse: + """Return the cached answer using only monotonic time and metrics state.""" + + now = self.port.monotonic() + refresh_seconds = self.port.metrics_refresh_seconds() + return self._metrics_snapshot_response( + now=now, + refresh_seconds=refresh_seconds, + ) + + def begin_metrics_refresh_loop(self) -> bool: + with self._metrics_lock: + if self._metrics_refresh_loop_running: + return False + self._metrics_refresh_loop_running = True + return True + + def metrics_snapshot_loop(self) -> None: + try: + while not self.port.stop_requested(): + try: + self.refresh_metrics_snapshot() + except Exception: + self.port.log("prism coordinator: metrics snapshot refresh failed") + self.port.log_exception() + if self.port.wait_for_stop(self.port.metrics_refresh_seconds()): + break + finally: + with self._metrics_lock: + self._metrics_refresh_loop_running = False + + +__all__ = [ + "HEALTH_SCHEMA", + "MINIMUM_HEALTH_STALE_SECONDS", + "METRICS_FAILURE_CLASSES", + "METRICS_STALE_WARNING", + "METRICS_STALE_WARNING_HEADER", + "METRICS_STATES", + "METRICS_STATE_FRESH", + "METRICS_STATE_HEADER", + "METRICS_STATE_STALE", + "METRICS_STATE_UNAVAILABLE", + "MetricsObservabilityState", + "MetricsSnapshotResponse", + "MiningDeliveryInputs", + "ObservabilityPort", + "ObservabilityService", + "ObservabilityState", +] diff --git a/lab/prism/payout_state.py b/lab/prism/payout_state.py new file mode 100644 index 000000000..e6f2fdb17 --- /dev/null +++ b/lab/prism/payout_state.py @@ -0,0 +1,5944 @@ +#!/usr/bin/env python3 +"""PRISM payout-state ownership: generations, artifacts, previews, fencing. + +This module owns the P1 domain: the payout-state generation and its +publication/delivery gate, the payout ledger artifact (incremental windows, +reuse/debounce/re-anchor policy and its background preparation executor), +accepted-block payout previews and their landing transitions, and the #112 +append/anchor fencing (inflight scan anchors, the published seedless job +window anchor, the append invalidation epoch, the rare append-vs-submit +landing fence, and the unfenced-append registry/drain). + +It never imports ``prism_coordinator``. The ledger handle, J1 job cache and +scheduler, R1 refresh scheduling, G1 progress recording, S3's pending-commit +floor, the shutdown controller, and live configuration attributes are reached +through the :class:`PayoutStateRuntime` typed port, resolved at call time so +the historical coordinator monkeypatch seams (including the instance-level +facade patches used by the current test suite) keep intercepting exactly as +before the extraction. Owner facades route back into this service; that +round trip is deliberate. + +Cross-owner lock order is unchanged by the extraction: payout publication and +delivery use this owner's delivery gate and balance-mutation lock; append +classification briefly takes and releases the state lock, then takes the +append-landing fence around the ledger commit; landing drains unfenced +appends without the fence, then holds fence -> state for the final epoch +check; no socket or RPC work runs while registry/retained locks are held. +""" + +from __future__ import annotations + +from concurrent.futures import Future, ThreadPoolExecutor +from contextlib import contextmanager +from dataclasses import dataclass, field, replace as dataclass_replace +import hashlib +import json +import threading +import time +from typing import Any, Callable, Iterator, Mapping, Protocol, Sequence + +from lab.prism.accepted_preview_telemetry import ( + FULL_RESCAN_PATH_DAEMON, + FULL_RESCAN_PATH_IN_PROCESS, + ensure_accepted_preview_telemetry, +) +from lab.prism.candidate_window import disk_window_covers, recorded_share_ids +from lab.prism.coordinator_config import ( + DEFAULT_ACCEPTED_PARENT_UNRESOLVED_DEPTH_MAX, + DEFAULT_PRISM_PAYOUT_ARTIFACT_FULL_RESCAN_SECONDS, + DEFAULT_PRISM_PAYOUT_ARTIFACT_MAX_ANCHOR_AGE_SECONDS, + DEFAULT_PRISM_PAYOUT_ARTIFACT_REANCHOR_SECONDS, + DEFAULT_PRISM_PAYOUT_ARTIFACT_REARM_MIN_SECONDS, + env_bool, + env_window_pipeline_rust, +) +from lab.prism.coordinator_shutdown import ( + BLOCK_SUBMITTER_WAIT_HEARTBEAT_SLICE_SECONDS, +) +from lab.prism.share_json_stream import ( + release_share_list_incrementally, + share_array_json_view, +) +from lab.prism.share_ledger import ( + DEFAULT_INCREMENTAL_SHARE_WINDOW_PAGE_SIZE, + DaemonShareWindowMirror, + DaemonWindowMirrorDivergence, + IncrementalShareJsonSequence, # noqa: F401 - annotation/compatibility export + IncrementalShareWindow, + IncrementalWindowAdvanceStats, + IncrementalWindowFallback, + PendingShare, # noqa: F401 - annotation/compatibility export +) + + +DEFAULT_ACCEPTED_BLOCK_PAYOUT_PREVIEW_WAIT_SECONDS = 5.0 +DEFAULT_PRISM_PAYOUT_RECONCILE_SUPERSESSION_RETRIES = 8 +PRISM_PAYOUT_DELIVERY_GENERATIONS = ("current", "stale", "future") +# Consecutive aborted speculative rebuilds double the re-arm interval up to +# this multiplier (80s at the default floor). Rebuilds abort on generation +# supersession, snapshot errors, empty windows, or a pathologically old +# pending-commit floor; backing off stops database preparation from cycling -- +# and from holding the payout preparation lock tip builds also need -- +# while the underlying condition persists. Any successful install or +# event-driven preparation resets the backoff. +PRISM_PAYOUT_ARTIFACT_REARM_BACKOFF_CAP = 16 +# How many append-invalidation bumps keep the stamp of the row that caused +# them. The stamps are what let a landing ask whether a bump between its own +# baseline epoch and the live one invalidated ITS window, instead of reading +# the bare counter and abandoning for an append that only invalidated a newer +# window (issue #126). Only a replay-shaped append -- a durable row that +# predates a live anchor -- ever adds an entry, and a landing's baseline is +# stamped when its job was issued and stops mattering the moment the block +# lands, so a few hundred spans far more such appends than any candidate can +# outlive. Falling off the end costs nothing but the relaxation: a baseline +# older than the retained history fails closed, exactly as the bare counter +# did. +PRISM_PAYOUT_APPEND_INVALIDATION_STAMP_HISTORY = 256 +# Owner-local copies of the still-coordinator-visible reward-window constants +# and the admission poll cadence; compatibility duplicates by design so this +# leaf module never imports upward. +PRISM_REWARD_WINDOW_MULTIPLIER = 8 +PRISM_SNAPSHOT_WINDOW_MARGIN = 2 + +# qbit retargets every block (ASERT), so the snapshot weight -- margin times +# the 8 x difficulty reward window -- drifts a fraction of a percent with +# every tip change. Treating any weight delta as a cache invalidation +# therefore forced the full oracle rescan (a multi-second ledger read under +# the postgres writer lock) on effectively every block, which is exactly the +# hold that starves the block submitter's landing-window reads and delays +# accepted-block preview publication (mainnet alert #236, 2026-08-25). +# +# The snapshot deliberately retains PRISM_SNAPSHOT_WINDOW_MARGIN times the +# reward window, and every consumer re-derives the exact 8 x difficulty +# cutoff from its own live network_difficulty (_prism_window_shares and the +# fold's requested_window_weight); the cached superset therefore stays +# payout-correct while it still covers the live reward window. Hold the +# cached weight inside a band instead of demanding equality: rescan only +# when the cached weight falls below 5/8 of the live snapshot weight (the +# 8 x reward window plus 25% headroom out of the 2 x margin) or exceeds +# twice it (bounding how oversized a stale superset may grow). Forced +# rescans -- landing reconcile invalidation, cold start, invariant +# failures, daemon rebuilds -- still re-center the weight at the live +# difficulty, and the periodic runtime-check validates the pipeline at the +# cached weight so its match verdict stays like-for-like. +PRISM_SNAPSHOT_WINDOW_WEIGHT_BAND_FLOOR_NUM = 5 +PRISM_SNAPSHOT_WINDOW_WEIGHT_BAND_FLOOR_DEN = 8 +PRISM_SNAPSHOT_WINDOW_WEIGHT_BAND_CEILING = 2 + +# Payout coverage depends on the floor retaining at least the whole reward +# window out of the margined snapshot: FLOOR_NUM/FLOOR_DEN >= 1/MARGIN. +# The fold has no error path for an under-covering superset -- it would +# silently truncate the window tail and misdistribute the coinbase -- so a +# margin retune that breaks the relationship must fail at import, not at +# payout time. Not an `assert`: python -O would strip it. +if ( + PRISM_SNAPSHOT_WINDOW_WEIGHT_BAND_FLOOR_NUM * PRISM_SNAPSHOT_WINDOW_MARGIN + < PRISM_SNAPSHOT_WINDOW_WEIGHT_BAND_FLOOR_DEN +): + raise AssertionError( + "snapshot window weight band floor no longer covers the reward " + "window at the configured snapshot margin" + ) + + +def _snapshot_window_weight_within_band( + cached_weight: int, live_weight: int +) -> bool: + """Whether a cached snapshot weight still serves the live reward window.""" + if cached_weight <= 0 or live_weight <= 0: + return False + if cached_weight == live_weight: + return True + if ( + cached_weight * PRISM_SNAPSHOT_WINDOW_WEIGHT_BAND_FLOOR_DEN + < live_weight * PRISM_SNAPSHOT_WINDOW_WEIGHT_BAND_FLOOR_NUM + ): + return False + return cached_weight <= live_weight * PRISM_SNAPSHOT_WINDOW_WEIGHT_BAND_CEILING + + +PRISM_TIP_REFRESH_ADMISSION_POLL_SECONDS = 0.05 +# Payout preparation/publication/first-delivery latency buckets; value-equal +# to the tip-refresh owner's seconds buckets so exposition is unchanged. +PRISM_PAYOUT_SECONDS_BUCKETS = ( + 0.01, + 0.025, + 0.05, + 0.1, + 0.25, + 0.5, + 1.0, + 2.5, + 5.0, + 10.0, + 30.0, +) +# Where one payout-window build spends its time, as a closed phase set +# (issue #228). ``ledger_read`` is every PostgreSQL read the build issues +# (full oracle, delta, periodic self-check, carry balances); +# ``record_conversion`` is folding records into pages, canonical JSON and +# digests in-process; ``daemon_prepare`` is the Rust daemon round trip +# (prepare or upload); ``lock_wait`` is time queued for the preparation +# lock. Fixed cardinality: no per-tip, per-generation or per-reason label. +PRISM_PAYOUT_WINDOW_BUILD_PHASES = ( + "ledger_read", + "record_conversion", + "daemon_prepare", + "lock_wait", +) +# Staleness the coordinator accepts on the tip-refresh path (issue #228). +# A routine candidate preparation that finds the preparation lock busy +# reuses the armed share window instead of queueing behind the lock +# holder, provided the window's anchor is no older than this; and a +# candidate build on that path defers an overdue periodic self-check +# while its cached window is inside the same bound, for at most this long +# past the check's due time (measured from the last self-check attempt, +# so build frequency cannot extend it). The number is one background +# re-anchor interval (DEFAULT_PRISM_PAYOUT_ARTIFACT_REANCHOR_SECONDS): in +# steady state the re-anchor keeps the armed window younger than this, +# so a window inside the bound is one the delivery path is already +# serving to new jobs as fresh, and a fanout that reuses it declares an +# anchor no older than reuse itself declares. Past the bound the wave +# pays the (incremental) build instead: the re-anchor has already flagged +# that window for replacement, and carrying it into a new payout +# generation would let a stalled background walker pin a stale anchor +# for every following generation. It sits well inside the 300 s audit +# ceiling that bounds how far any served window may trail the live +# ledger. +PRISM_TIP_REFRESH_WINDOW_STALENESS_SECONDS = 60.0 +# How one payout-window build ended, as a closed set (issue #228). The +# phase family is observed under this label on every exit, so a build that +# lost its generation race, hit the audit ceiling, found an empty window or +# died with the ledger still contributes the time it measured; those are +# exactly the slow builds the split exists to attribute. +PRISM_PAYOUT_WINDOW_BUILD_OUTCOMES = ( + "completed", + "debounced", + "aborted", + "failed", +) + + +def canonical_json_text(value: object) -> str: + """Owner-local canonical serialization (no reverse imports by design).""" + return json.dumps(value, sort_keys=True, separators=(",", ":"), default=str) + + +def canonical_json_sha256(value: object) -> str: + """Owner-local canonical digest (no reverse imports by design).""" + incremental_digest = getattr(value, "canonical_json_sha256", None) + if callable(incremental_digest): + return str(incremental_digest()) + return hashlib.sha256(canonical_json_text(value).encode()).hexdigest() + + +class TemplateRefreshBlocked(RuntimeError): + """A live template was fetched, but safe work could not be issued.""" + + +class TemplateRefreshSuperseded(TemplateRefreshBlocked): + """Concurrent tip/payout progress invalidated this refresh attempt. + + Raised only for coordination races that a scheduled retry resolves on its + own: the tip advanced mid-refresh, the payout-state generation moved, or a + newer observation superseded the prepared work. Unlike its parent, this + subclass never arms the ordinary template-refresh failure budget. Repeated + coordination blocks are tracked by their own longer deadline; a genuine + RPC/build/trust failure must raise plain TemplateRefreshBlocked so it still + takes the ordinary budgeted restart path. + """ + + +class PayoutStatePublicationBlocked(TemplateRefreshBlocked): + """Job construction is waiting for a prepared payout publication.""" + + +class AcceptedParentPayoutPreviewPending(PayoutStatePublicationBlocked): + """A child build's bounded wait expired with its parent's preview unpublished. + + Raised only for the one benign backpressure condition: an armed, + un-withdrawn accepted-parent transition whose prospective balances have + not published within ``accepted_block_payout_preview_wait_seconds``. The + parent is still landing normally, so nothing has failed -- the child is + simply early. + + Subclassing PayoutStatePublicationBlocked keeps this inside the + coordination-blocked family: the retry coalesces with the scheduled tip + refresh, it never arms ``job_build_failure_count`` or the ordinary + template-refresh failure budget, and a sustained continuous streak still + exits through the coordination-blocked watchdog rather than retrying + silently forever. Callers that catch either parent keep working + unchanged. + + Deliberately *not* used for any other block: RPC/build/trust failures, + withdrawn previews, unknown replay-enumeration state, unresolved-depth + caps, and invariant violations keep their existing types so their + persistence still takes the ordinary budgeted restart path. + """ + + #: Stable machine-readable reason for retry logs and delivery telemetry. + retry_reason = "accepted_parent_preview_pending" + + def __init__( + self, + message: str = "accepted parent payout preview is not ready yet", + *, + parent_hash: str | None = None, + waited_seconds: float | None = None, + timeout_count: int | None = None, + ) -> None: + super().__init__(message) + self.parent_hash = parent_hash + self.waited_seconds = waited_seconds + self.timeout_count = timeout_count + + +@dataclass(frozen=True) +class PayoutStateArtifact: + """Immutable ledger-backed inputs published with one payout generation.""" + + generation: int + source_generation: int + prior_balances_json: str = field(repr=False) + prior_balances_sha256: str + prepared_monotonic: float + + def prior_balances(self) -> list[dict[str, object]]: + value = json.loads(self.prior_balances_json) + if not isinstance(value, list): + raise RuntimeError("published payout artifact is not a balance list") + return value + + +@dataclass(frozen=True) +class AcceptedBlockPayoutTransition: + """Prospective balances for one durable candidate across its landing seam.""" + + block_height: int | None = None + landed: bool = False + preview: tuple[tuple[str, str, str, int], ...] | None = None + published_generation: int | None = None + landed_monotonic: float | None = None + + +@dataclass(frozen=True) +class PayoutStateCandidate: + """Immutable result of payout work prepared outside delivery admission.""" + + base_generation: int + source_generation: int + source_tip_hash: str | None + cause: str + invalidated_monotonic: float + prepared_monotonic: float + accepted_block_hash: str | None = None + accepted_block_preview: tuple[tuple[str, str, str, int], ...] | None = None + accepted_block_withdrawal: bool = False + accepted_block_height: int | None = None + ledger_artifact: PayoutLedgerArtifact | None = field( + default=None, + compare=False, + repr=False, + ) + + +@dataclass(frozen=True) +class PublishedPayoutState: + """The payout snapshot identity to which cached jobs are stamped.""" + + generation: int + source_generation: int + source_tip_hash: str | None + published_monotonic: float + artifact: PayoutStateArtifact | None = field(default=None, repr=False) + + +@dataclass(frozen=True) +class PayoutLedgerArtifact: + """Immutable ledger input prepared independently of a qbit template.""" + + generation: int + payout_state_generation: int + network_difficulty: int + # Durable accepted-share total observed near the snapshot. Informational + # (diagnostics only): validity is scoped to snapshot_anchor_ms, and the + # count cannot be scoped to a clamped anchor because share stamping and + # writer enqueue are not atomic. + accepted_share_count: int + shares_json: Sequence[dict[str, object]] = field(repr=False) + prior_balances: tuple[dict[str, object], ...] = field(repr=False) + prepared_monotonic: float + # The anchor the share snapshot was actually taken at. Bundles built from + # this artifact must declare it as anchor_job_issued_at_ms: an auditor + # replaying qbit_audit_share_window at the declared anchor must reproduce + # exactly these shares, which only holds at the snapshot's own anchor. + # That declaration is also what makes reuse valid while shares keep + # landing: a share stamped after this anchor deterministically belongs to + # the next window (it is never lost), so the artifact stays + # audit-reproducible for as long as it serves. Validity is event-driven + # (the payout-generation and balances fences); wall-clock time only + # enters through the loose anchor ceiling that backstops how far a + # served window may trail the live ledger, never through install age. + snapshot_anchor_ms: int | None = None + # Canonical digest of shares_json. Cached bundles built before this + # artifact was armed may only keep serving re-keyed lookups when their + # own window digest matches; anything else must rebuild from the + # artifact so a fresher window is never shadowed by older cached work. + share_snapshot_sha256: str | None = None + # Canonical digest of prior_balances, memoized at construction. The + # reuse probe hashes the balances on every serving decision and the + # reused-build fences hash them twice more; the tuple is immutable, so + # the O(accounts) canonicalization is paid once here instead. + prior_balances_sha256: str | None = None + # Internal coordination token. A late-visible append can invalidate a + # previously exact anchored window without changing payout generation. + # Artifacts prepared before that append may never be armed again. + append_invalidation_epoch: int = field( + default=0, + compare=False, + repr=False, + ) + # Exposed in-flight scan anchor carried from the synchronous bundle + # build that seeded this artifact (see _expose_inflight_scan_anchor). + # Nothing else exposes this window's anchor between the ledger read and + # the install fence, so the exposure rides here and the install retires + # it once the fence settles either way. + inflight_scan_anchor_token: int | None = field( + default=None, + compare=False, + repr=False, + ) + # Build-path diagnostics only. They do not participate in artifact + # equality or any signed/wire representation. + window_build_mode: str | None = field(default=None, compare=False, repr=False) + window_delta_rows: int = field(default=0, compare=False, repr=False) + window_expired_rows: int = field(default=0, compare=False, repr=False) + window_touched_pages: int = field(default=0, compare=False, repr=False) + window_build_seconds: float = field(default=0.0, compare=False, repr=False) + window_full_rescan_reason: str | None = field( + default=None, + compare=False, + repr=False, + ) + window_self_check_recentered: bool = field( + default=False, + compare=False, + repr=False, + ) + + +@dataclass(frozen=True) +class _IncrementalPayoutArtifactWindow: + """Coordinator-owned exact window plus its canonical materialization.""" + + window: IncrementalShareWindow + shares_json: IncrementalShareJsonSequence = field(repr=False) + share_snapshot_sha256: str + refreshed_monotonic: float + full_rescan_monotonic: float + full_rescan_attempt_monotonic: float + append_invalidation_epoch: int = 0 + # Set when a daemon mirror was adopted from the in-process oracle + # without the daemon being prepared for its digest (issue #207): a + # periodic self-check re-center or repair whose daemon preparation + # was declined. The next advance then answers ``needs_full`` for a + # deliberate coordinator decision, and this names it in the rebuild + # reason instead of reporting a daemon eviction. + daemon_unprepared_reason: str | None = None + + +class _DaemonWindowRebuildRequired(Exception): + """The daemon path cannot advance; rebuild from the full oracle. + + Raised only inside the payout-window materialization: ``reason`` becomes + the full-rescan reason routed through the existing invalidation + machinery, and ``clear_cache`` distinguishes a refuted local state (the + mirror diverged, or the daemon transport vanished under a mirror-typed + cache) from a daemon that merely lost its own copy while the coordinator + mirror stays valid. + """ + + def __init__(self, reason: str, *, clear_cache: bool) -> None: + super().__init__(reason) + self.reason = reason + self.clear_cache = clear_cache + + +@dataclass(frozen=True) +class _PayoutWindowMaterialization: + """One exact share-window materialization and bounded-work metadata.""" + + shares_json: Sequence[dict[str, object]] = field(repr=False) + share_snapshot_sha256: str + snapshot_anchor_ms: int + mode: str + record_count: int + stats: IncrementalWindowAdvanceStats + full_rescan_reason: str | None = None + balance_check_prior_balances: tuple[dict[str, object], ...] | None = field( + default=None, + repr=False, + ) + balance_check_mismatch: bool = False + # An overdue periodic self-check was left for the next build off the + # critical path because this build's cached window was inside + # PRISM_TIP_REFRESH_WINDOW_STALENESS_SECONDS (issue #228). + self_check_deferred: bool = False + # The periodic self-check re-centered an oversized window at the live + # weight (issue #207); a deliberate coordinator decision, flagged apart + # from any daemon reason. + self_check_recentered: bool = False + + +class _ReservedPayoutSource(tuple): + """The source tuple handed back to the caller that reserved it. + + Unpacks, indexes and compares exactly like the plain five-tuple a + capture returns; the difference is the one-shot entitlement it carries + (issue #228). The reserving caller's own preparation acts on the cause + it reserved by presenting this object, whatever other preparations + have done at any generation in between, so no shared bookkeeping and + no pruning rule can take that turn away. The entitlement is consumed + the first time it is honoured; a second presentation inherits the + tuple like any capture. + """ + + def __new__( + cls, + base_generation: int, + source_generation: int, + tip_hash: str | None, + cause: str, + invalidated_monotonic: float, + ) -> "_ReservedPayoutSource": + self = super().__new__( + cls, + ( + base_generation, + source_generation, + tip_hash, + cause, + invalidated_monotonic, + ), + ) + self._entitled = True + return self + + def consume_entitlement(self) -> bool: + """Take the reserver's one honoured turn; caller holds runtime.lock.""" + if self._entitled: + self._entitled = False + return True + return False + + +@dataclass(frozen=True) +class _PayoutWindowBuildAttribution: + """Why one candidate build was requested, for its log line (issue #228). + + Set on the building thread by ``_prepared_payout_state_candidate`` and + read by ``_build_payout_ledger_artifact``, which the coordinator + delegate reaches with a fixed positional signature; the thread-local + carries what that signature cannot. + """ + + # The cause string in the captured source tuple, verbatim. + cause: str + # True when the capture found a cause another preparation had already + # acted on: the tuple was inherited for supersession identity only. + cause_inherited: bool + # "caller_flag", "reserved_cause", "caller_flag+reserved_cause" or + # "none": what, if anything, marked this build as a payout mutation. + force_origin: str + # A routine candidate on the tip-refresh path: an overdue periodic + # self-check is deferred while the cached window is inside the + # accepted staleness bound. + critical_path: bool + + +@dataclass +class PayoutDeliveryAdmission: + admitted: bool + wait_seconds: float + generation: int + published_generation: int + relation: str + delivered: bool = False + + def __bool__(self) -> bool: + return self.admitted + + def mark_delivered(self) -> None: + if not self.admitted: + raise RuntimeError("payout delivery completed without admission") + self.delivered = True + + +class PayoutStateDeliveryGate: + """Order delivery admission around a very short payout publication. + + A publisher first closes admission and drains sends that already crossed + the boundary. It does not own the atomic publication section while that + drain is in progress. Once drained, publication ownership is transferred + to the caller for the generation/cache pointer swap only. + """ + + def __init__(self) -> None: + self._condition = threading.Condition() + self._active_deliveries = 0 + self._publisher_waiting = False + self._mutation_owner: int | None = None + self._mutation_depth = 0 + self._published_generation = 0 + self._priority_generation: int | None = None + self._delivery_blocked = False + + @staticmethod + def _generation_relation(generation: int, published_generation: int) -> str: + if generation < published_generation: + return "stale" + if generation > published_generation: + return "future" + return "current" + + @contextmanager + def delivery(self) -> Iterator[None]: + with self.delivery_cancelable(lambda: False, priority=True) as admission: + if not admission: + raise RuntimeError("uncancelled payout delivery was not admitted") + yield + admission.mark_delivered() + + @contextmanager + def delivery_cancelable( + self, + cancelled: Callable[[], bool], + *, + generation: int | None = None, + priority: bool = False, + poll_seconds: float = PRISM_TIP_REFRESH_ADMISSION_POLL_SECONDS, + ) -> Iterator[PayoutDeliveryAdmission]: + """Admit a delivery unless cancellation wins while mutation owns the gate.""" + + started = time.monotonic() + admitted = False + with self._condition: + if generation is None: + generation = self._published_generation + while True: + if cancelled(): + break + if self._delivery_blocked: + break + if generation < self._published_generation: + break + publication_blocked = ( + self._publisher_waiting or self._mutation_owner is not None + ) + priority_blocked = ( + self._priority_generation is not None + and generation == self._priority_generation + and not priority + ) + if priority_blocked: + # A same-generation job that is not for the published tip + # must not occupy a waiter slot indefinitely. Reject it so + # its caller can rebuild current-tip work; the reserved + # first-delivery lane remains available to priority work. + break + future_blocked = generation > self._published_generation + if ( + not publication_blocked + and not future_blocked + ): + self._active_deliveries += 1 + admitted = True + break + self._condition.wait(timeout=poll_seconds) + published_generation = self._published_generation + relation = self._generation_relation(generation, published_generation) + admission = PayoutDeliveryAdmission( + admitted=admitted, + wait_seconds=max(0.0, time.monotonic() - started), + generation=generation, + published_generation=published_generation, + relation=relation, + ) + try: + yield admission + finally: + if admitted: + with self._condition: + if self._active_deliveries <= 0: + raise RuntimeError("payout delivery gate released without admission") + self._active_deliveries -= 1 + if ( + priority + and admission.delivered + and generation == self._priority_generation + ): + # Keep routine same-generation sends queued until the + # first prioritized current-tip socket delivery exits. + # Privileged synchronization admissions do not consume + # this reservation merely by leaving the gate. + self._priority_generation = None + if self._active_deliveries == 0: + self._condition.notify_all() + elif self._priority_generation is None: + self._condition.notify_all() + + @contextmanager + def publication(self) -> Iterator[None]: + owner = threading.get_ident() + with self._condition: + if self._mutation_owner == owner: + self._mutation_depth += 1 + else: + while self._mutation_owner is not None or self._publisher_waiting: + self._condition.wait() + self._publisher_waiting = True + while self._active_deliveries: + self._condition.wait() + self._mutation_owner = owner + self._mutation_depth = 1 + self._publisher_waiting = False + try: + yield + finally: + with self._condition: + if self._mutation_owner != owner or self._mutation_depth <= 0: + raise RuntimeError("payout mutation gate released by non-owner") + self._mutation_depth -= 1 + if self._mutation_depth == 0: + self._mutation_owner = None + self._condition.notify_all() + + def publish_generation(self, generation: int, *, prioritize_delivery: bool) -> None: + owner = threading.get_ident() + with self._condition: + if self._mutation_owner != owner: + raise RuntimeError("payout generation published outside atomic section") + if generation <= self._published_generation: + raise RuntimeError("payout generation did not advance") + self._published_generation = generation + self._priority_generation = generation if prioritize_delivery else None + self._delivery_blocked = False + + def block_delivery( + self, + mark_blocked: Callable[[], bool] | None = None, + ) -> bool: + """Reject admission until publication, atomically with caller state.""" + + with self._condition: + # A publisher drops the condition while swapping its immutable + # pointer. Wait only for that short section, never for admitted + # socket sends or fanout cancellation. + while self._mutation_owner is not None: + self._condition.wait() + if mark_blocked is not None and not mark_blocked(): + return False + self._delivery_blocked = True + self._condition.notify_all() + return True + + @contextmanager + def mutation(self) -> Iterator[None]: + """Compatibility alias for tests and callers that only need exclusion.""" + + with self.publication(): + yield + + +@dataclass(frozen=True) +class PayoutStateSnapshot: + """Copied identity facts for cross-owner delivery decisions.""" + + generation: int + published: PublishedPayoutState + publication_blocked: bool + + +class PayoutStateRuntime(Protocol): + """Typed port over the coordinator, resolved at call time. + + Every member is looked up on the live coordinator object when used, so + instance monkeypatches (``server._prepared_payout_state_candidate = ...`` + and friends) and coordinator-owned live configuration attributes keep + working exactly as before the extraction. Owner facades route back into + this service; that round trip is deliberate. The legacy P1 field names + (``_payout_state_generation`` through ``payout_artifact_event_counts``) + also resolve here -- coordinator class descriptors route them to this + service's single mutable copy. + """ + + # Cross-domain objects and live configuration attributes. + _job_cache_lock: Any + ledger: Any + lock: Any + stop_event: Any + + def _ensure_job_cache_state(self, *args: Any, **kwargs: Any) -> Any: ... + + def _ensure_shutdown_controller(self, *args: Any, **kwargs: Any) -> Any: ... + + def _record_heartbeat(self, *args: Any, **kwargs: Any) -> Any: ... + + def _record_block_submitter_phase(self, *args: Any, **kwargs: Any) -> Any: ... + + def _block_work_wait_slice(self, *args: Any, **kwargs: Any) -> Any: ... + + def _record_progress_payout_generation(self, *args: Any, **kwargs: Any) -> Any: ... + + def _observe_accepted_block_preview_publication( + self, + *args: Any, + **kwargs: Any, + ) -> Any: ... + + def _writer_operation(self, *args: Any, **kwargs: Any) -> Any: ... + + +class PayoutStateService: + """Sole owner of P1 payout state, artifacts, previews, and fencing.""" + + def __init__( + self, + runtime: PayoutStateRuntime, + *, + shutdown_error: type[BaseException], + now_ms: Callable[[], int], + canonical_json_text_override: Callable[[object], str] = canonical_json_text, + canonical_json_sha256_override: Callable[[object], str] = canonical_json_sha256, + ) -> None: + self._runtime = runtime + # The shutdown exception type stays coordinator-owned until the + # shutdown controller is extracted, the wall-clock stamp and the + # canonical serialization resolve the coordinator module's globals at + # call time (tests patch them there); all are injected so this leaf + # module never imports prism_coordinator. + self._shutdown_error = shutdown_error + self._now_ms = now_ms + self._canonical_json_text = canonical_json_text_override + self._canonical_json_sha256 = canonical_json_sha256_override + self._payout_state_generation = 0 + # Ledger mutations and the ledger reads used to build signed jobs + # share this lock. They may be expensive, but they never block + # delivery of an already-published immutable generation. + self._payout_state_prepare_lock = threading.RLock() + self._payout_state_source: tuple[int, str | None, str, float] = ( + 0, + None, + "startup", + time.monotonic(), + ) + self._published_payout_state = PublishedPayoutState( + generation=self._payout_state_generation, + source_generation=0, + source_tip_hash=None, + published_monotonic=time.monotonic(), + ) + self._payout_ledger_artifact: PayoutLedgerArtifact | None = None + self._payout_ledger_artifact_generation = 0 + # Guarded by _job_cache_lock. Unlike payout generation, this + # advances only when a newly visible row predates an anchored + # share window and makes pre-append work unsafe to publish. + self._payout_ledger_append_invalidation_epoch = 0 + # Guarded by _job_cache_lock. Per advanced epoch, the stamp of the + # row whose durability caused that bump: max(job_issued_at_ms, + # accepted_at_ms), which is exactly the quantity + # _pending_share_predates_anchor tests against an anchor -- a row + # predates an anchor iff both its stamps do, i.e. iff this max does. + # Bounded to the most recent + # PRISM_PAYOUT_APPEND_INVALIDATION_STAMP_HISTORY bumps; + # _payout_append_invalidation_oldest_epoch names the oldest epoch + # still retained (the epoch the next bump would take, while there + # has been none), so a landing whose baseline fell off the end is + # told the question cannot be answered rather than answered from a + # gap. + self._payout_append_invalidation_stamps: dict[int, int] = {} + self._payout_append_invalidation_oldest_epoch = 1 + # Serializes the append-side epoch bump against the landing's + # final epoch-check-and-submitblock boundary. Ordinary share + # commits never touch it: the append side acquires it only for + # rows that predate a live anchor (the rare replay-shaped + # append), and the landing holds it across exactly one RPC, so + # an epoch advance can never slip between the landing's last + # epoch read and the block entering qbitd. Ordered strictly + # outside _job_cache_lock. + self._payout_append_landing_fence_lock = threading.Lock() + # Guarded by _job_cache_lock. The anchors of ledger window walks + # whose results are not yet visible anywhere else, published + # before each read begins. A late-visible append can predate such + # an anchor while no completed window or armed artifact exposes + # one, and the walk's database snapshot may already exclude the + # row; the append-side invalidation must still advance the epoch + # so the walk's result cannot arm or serve. A synchronous bundle + # build keeps its entry exposed past its own return -- until the + # seeded artifact's install fence settles -- because nothing else + # exposes that window's anchor in the meantime. + self._payout_window_inflight_scan_anchors: dict[int, int] = {} + self._payout_window_inflight_scan_anchor_token = 0 + # Guarded by _job_cache_lock. One entry per durable append + # currently committing OUTSIDE the landing fence, holding the + # batch's most predating row stamp (min over rows of + # max(job_issued_at_ms, accepted_at_ms)). The unfenced + # classification is a one-time predicate: an anchor exposed + # after it cannot retroactively fence the in-flight commit, so + # a landing that just exposed its declared anchor drains + # matching entries before its epoch fences arm. The condition + # shares _job_cache_lock and signals every deregistration. + self._payout_unfenced_append_inflight_stamps: dict[int, int] = {} + self._payout_unfenced_append_inflight_token = 0 + self._payout_unfenced_append_drained = threading.Condition( + runtime._job_cache_lock + ) + # Guarded by _job_cache_lock. The highest declared anchor among + # published job windows that nothing else keeps exposed: a + # seedless publication hands its anchor over at the publication + # fence (a build under the reuse kill-switch, or one with nothing + # to seed), and the armed artifact's anchor is folded in before + # the slot can drop or replace it + # (_replace_payout_ledger_artifact_locked). Jobs stamped from such + # a window keep serving it until they retire, and nothing else + # exposes its anchor between publication and the landing's own + # exposure -- so the anchor must stay visible to the append-side + # predates() checks: a replay-shaped append committing in that gap + # has to advance the epoch those jobs' landing fences compare + # against. Monotonic and never retired; ordinary share commits can + # never match it because every job anchor is clamped below the + # pending-commit floor. + self._payout_published_job_window_anchor_ms: int | None = None + # Guarded by _payout_state_prepare_lock. It is independent of the + # published artifact generation: normal generation bumps retag + # the same immutable share window without another ledger walk. + self._incremental_payout_artifact_window: ( + _IncrementalPayoutArtifactWindow | None + ) = None + # Guarded by _payout_state_prepare_lock. Explicit append-side + # invalidations leave a one-shot diagnostic for the oracle build + # that replaces the discarded incremental window. + self._incremental_payout_artifact_window_invalidation_reason: ( + str | None + ) = None + # Guarded by _payout_state_prepare_lock. A periodic balance oracle + # mismatch keeps the stale published digest off the reuse path + # until a later publication installs a different digest. + self._payout_prior_balances_reuse_invalidated_sha256: str | None = None + self._payout_artifact_executor_lock = threading.Lock() + self._payout_artifact_executor: ThreadPoolExecutor | None = None + self._payout_artifact_future: Future[None] | None = None + self._payout_artifact_requested: tuple[int, int] | None = None + self._payout_artifact_requested_bypass = False + self._payout_artifact_executor_shutdown = False + # Guarded by _payout_artifact_executor_lock. Stamped by every + # accepted preparation request so fence-failure re-arms measure + # their debounce from the newest scheduled rebuild, whichever + # path requested it. + self._payout_artifact_last_schedule_monotonic: float | None = None + # Guarded by _payout_artifact_executor_lock. + self._payout_artifact_rearm_backoff = 1 + # Orders reconciliation mutations against final job-delivery + # admission while preserving parallel sends to different miners. + self._payout_state_delivery_gate = PayoutStateDeliveryGate() + # Keep durable payout-balance transitions serialized even when their + # preparation intentionally runs outside the delivery gate. The + # accepted-block path can hold this lock across expensive writes + # without preventing replacement jobs from reaching miners. + self._payout_balance_mutation_lock = threading.RLock() + self._accepted_block_payout_preview_condition = threading.Condition() + # Durable replay registers an unlanded transition. Once its block + # is active, reconciliation is barred and child/descendant builders + # wait for or consume its verified prospective balance snapshot. + self._accepted_block_payout_previews: dict[ + str, AcceptedBlockPayoutTransition + ] = {} + # A landed transition can be withdrawn before durability catches + # up. Keep a height-bearing tombstone so a new exact/descendant + # build cannot fall through to database balances that omit it in + # the gap before durable retry re-registers the candidate. + self._invalidated_accepted_block_payout_previews: dict[ + str, int | None + ] = {} + self._payout_state_metrics_lock = threading.Lock() + self.payout_state_histograms = { + name: { + "buckets": { + bucket: 0 for bucket in PRISM_PAYOUT_SECONDS_BUCKETS + }, + "sum": 0.0, + "count": 0, + } + for name in ("preparation", "publish", "first_delivery") + } + self.payout_gate_wait_histograms = { + relation: { + "buckets": { + bucket: 0 for bucket in PRISM_PAYOUT_SECONDS_BUCKETS + }, + "sum": 0.0, + "count": 0, + } + for relation in PRISM_PAYOUT_DELIVERY_GENERATIONS + } + self.payout_state_candidates_discarded = 0 + # Guarded by _payout_state_metrics_lock. Where builds spend their + # time, by the closed phase set and the closed outcome set (issue + # #228). + self.payout_window_build_phase_histograms = { + (phase, outcome): { + "buckets": { + bucket: 0 for bucket in PRISM_PAYOUT_SECONDS_BUCKETS + }, + "sum": 0.0, + "count": 0, + } + for phase in PRISM_PAYOUT_WINDOW_BUILD_PHASES + for outcome in PRISM_PAYOUT_WINDOW_BUILD_OUTCOMES + } + # Per-thread build state: the attribution a candidate preparation + # hands its build, and the phase accumulator of the build in + # progress on this thread. + self._payout_window_build_context = threading.local() + # Guarded by runtime.lock. The highest source generation any + # candidate preparation has acted on; it never moves backwards. A + # capture at that generation or below inherits the tuple for + # supersession identity only, so the cause a withdrawal reserved is + # acted on once (issue #228). The reserver's own turn does not live + # here: it travels in the _ReservedPayoutSource its reservation + # returned. The tuple itself is never cleared, because publication + # supersession and the finalizer's pending-cause read depend on it + # staying readable. + self._payout_source_cause_high_water: int | None = None + # Guarded by runtime.lock. A critical-path build deferred an overdue + # periodic self-check; the next publication requests a background + # preparation so the check runs off the critical path promptly. + self._payout_self_check_owed = False + self._payout_first_delivery_pending: tuple[int, float] | None = None + self._payout_state_publication_blocked = False + # Artifact lifecycle observability (the 2026-07-29 incident was + # invisible precisely here). Guarded by + # _payout_artifact_executor_lock. + self.payout_artifact_event_counts = { + event: 0 + for event in ( + "built", + "build_aborted", + "debounced", + "incremental", + "full_rescan", + "self_check_match", + "self_check_mismatch", + "self_check_failed", + "balance_check_mismatch", + "found_block_cached", + "tip_refresh_cached", + "installed", + "refreshed", + "already_current", + "discarded", + "born_expired", + "rearm_scheduled", + ) + } + + def snapshot(self) -> PayoutStateSnapshot: + """Copied cross-owner identity facts (memo rule: no shared mutables).""" + with self._runtime.lock: + return PayoutStateSnapshot( + generation=int(self._payout_state_generation), + published=self._published_payout_state, + publication_blocked=bool(self._payout_state_publication_blocked), + ) + + @staticmethod + def _incremental_window_records_supported(records: Sequence[object]) -> bool: + """Whether a ledger snapshot exposes the append-only window contract.""" + + required = ( + "share_seq", + "share_id", + "share_difficulty", + "job_issued_at_ms", + "accepted_at_ms", + "to_prism_json", + ) + return bool(records) and all( + all(hasattr(record, attribute) for attribute in required) + for record in records + ) + + def _observe_payout_window_full_rescan( + self, + reason: str, + path: str, + seconds: float, + ) -> None: + """Record one executed full oracle rescan in the #224 owner. + + Only the two places that actually run the whole-window oracle call + this: ``_full_payout_window_materialization`` and the periodic + self-check inside the incremental materialization. A prior-balances + reread (``force_prior_balances_read``) never reaches here. + """ + ensure_accepted_preview_telemetry( + self._runtime + ).observe_payout_window_full_rescan(reason, path, seconds) + + def _full_payout_window_materialization( + self, + *, + snapshot_anchor_ms: int, + snapshot_window_weight: int, + reason: str, + observed_monotonic: float, + append_invalidation_epoch: int, + bypass_build_interval: bool = False, + ) -> _PayoutWindowMaterialization: + """Run the exact ledger oracle and atomically replace cached pages.""" + rescan_started = time.monotonic() + # The path is chosen by which pipeline folded the window; the ledger + # read and any daemon decline that degrades to the in-process fold + # belong to the rescan either way. Recorded on every exit so a + # rescan that dies with the ledger still owns its wall-clock. + rescan_path = FULL_RESCAN_PATH_IN_PROCESS + try: + materialized, rescan_path = self._full_payout_window_oracle( + snapshot_anchor_ms=snapshot_anchor_ms, + snapshot_window_weight=snapshot_window_weight, + reason=reason, + observed_monotonic=observed_monotonic, + append_invalidation_epoch=append_invalidation_epoch, + bypass_build_interval=bypass_build_interval, + ) + finally: + self._observe_payout_window_full_rescan( + reason, + rescan_path, + time.monotonic() - rescan_started, + ) + return materialized + + def _full_payout_window_oracle( + self, + *, + snapshot_anchor_ms: int, + snapshot_window_weight: int, + reason: str, + observed_monotonic: float, + append_invalidation_epoch: int, + bypass_build_interval: bool = False, + ) -> tuple[_PayoutWindowMaterialization, str]: + """The oracle read plus fold; returns the window and the fold path.""" + runtime = self._runtime + + # Timed in a finally: a read that dies with the ledger is exactly + # the slow read the phase family exists to attribute. + read_started = time.monotonic() + try: + records = list( + runtime.ledger.snapshot_at_job_issue( + snapshot_anchor_ms, + window_weight=snapshot_window_weight, + ) + ) + finally: + self._note_window_build_phase( + "ledger_read", + time.monotonic() - read_started, + ) + zero_stats = IncrementalWindowAdvanceStats( + added_rows=0, + expired_rows=0, + touched_pages=0, + ) + if not runtime._incremental_window_records_supported(records): + # Small test/custom ledgers predating the delta contract retain the + # legacy full-read behavior. Production ledger records always + # expose the complete AcceptedShareRecord shape. + runtime._incremental_payout_artifact_window = None + conversion_started = time.monotonic() + shares_json = tuple(record.to_prism_json() for record in records) + # The digest hook prefers a sequence's own canonical digest; + # the view streams the freshly converted tuple record-batch by + # record-batch instead of one whole-window json.dumps. + digest = self._canonical_json_sha256( + share_array_json_view(shares_json) + ) + self._note_window_build_phase( + "record_conversion", + time.monotonic() - conversion_started, + ) + return ( + _PayoutWindowMaterialization( + shares_json=shares_json, + share_snapshot_sha256=digest, + snapshot_anchor_ms=int(snapshot_anchor_ms), + mode="full_rescan", + record_count=len(records), + stats=zero_stats, + full_rescan_reason=reason, + ), + FULL_RESCAN_PATH_IN_PROCESS, + ) + + if self._window_pipeline_rust_enabled(): + materialized = self._daemon_full_window_materialization( + records=records, + snapshot_anchor_ms=snapshot_anchor_ms, + snapshot_window_weight=snapshot_window_weight, + reason=reason, + observed_monotonic=observed_monotonic, + append_invalidation_epoch=append_invalidation_epoch, + bypass_build_interval=bypass_build_interval, + ) + if materialized is not None: + # The daemon holds the window as bytes now; the ledger rows + # are finished with. Drop them in bounded slices so their + # release is not one refcount cascade under the GIL (#236). + release_share_list_incrementally(records) + return materialized, FULL_RESCAN_PATH_DAEMON + # Any daemon outcome that cannot produce a window here -- an + # anomaly (already retired), the transport disabled, a daemon + # another build owns, a snapshot the fold must reject, or a + # value outside the daemon's declared integer widths (the + # daemon declines it and stays healthy) -- degrades to the + # in-process pipeline below for this materialization, which is + # also where an invalid snapshot raises its authoritative error + # and where unbounded integers fold the out-of-range window. + + conversion_started = time.monotonic() + window = IncrementalShareWindow.from_full_snapshot( + records, + anchor_job_issued_at_ms=snapshot_anchor_ms, + window_weight=snapshot_window_weight, + ) + shares_json = window.json_records() + digest = self._canonical_json_sha256(shares_json) + self._note_window_build_phase( + "record_conversion", + time.monotonic() - conversion_started, + ) + runtime._incremental_payout_artifact_window = ( + _IncrementalPayoutArtifactWindow( + window=window, + shares_json=shares_json, + share_snapshot_sha256=digest, + refreshed_monotonic=observed_monotonic, + full_rescan_monotonic=observed_monotonic, + full_rescan_attempt_monotonic=observed_monotonic, + append_invalidation_epoch=append_invalidation_epoch, + ) + ) + return ( + _PayoutWindowMaterialization( + shares_json=shares_json, + share_snapshot_sha256=digest, + snapshot_anchor_ms=int(snapshot_anchor_ms), + mode="full_rescan", + record_count=len(shares_json), + stats=zero_stats, + full_rescan_reason=reason, + ), + FULL_RESCAN_PATH_IN_PROCESS, + ) + + def _window_pipeline_rust_enabled(self) -> bool: + """Whether window materialization routes through the --serve daemon. + + Fail-closed at the env layer (a non-boolean PRISM_WINDOW_PIPELINE_RUST + refuses startup via SystemExit); inert -- not an error -- when the + daemon transport itself is disabled or when the runtime exposes no + prepare port (embedders and slim test coordinators). + """ + if not env_window_pipeline_rust(): + return False + if not env_bool("PRISM_BUILDER_SERVE", "1"): + return False + return callable(getattr(self._runtime, "prepare_payout_window", None)) + + def _daemon_full_window_materialization( + self, + *, + records: Sequence[Any], + snapshot_anchor_ms: int, + snapshot_window_weight: int, + reason: str, + observed_monotonic: float, + append_invalidation_epoch: int, + bypass_build_interval: bool = False, + ) -> _PayoutWindowMaterialization | None: + """Fold one full snapshot through the daemon; None means fold here. + + The daemon returns the canonical digest plus the full canonical + items stream; the mirror construction reconciles those bytes with + both the digest and the record count before anything downstream may + consume them. Every non-prepared outcome -- a daemon anomaly (already + retired by the transport), a daemon another build owns, the fold + rejecting the snapshot (the in-process oracle must raise the + authoritative error), an integer outside the daemon's declared + widths (``out_of_range``: the daemon is kept, the window is folded + here), or a refuted byte mirror -- degrades to the in-process + pipeline for this materialization. + """ + runtime = self._runtime + prepare = getattr(runtime, "prepare_payout_window", None) + if not callable(prepare): + return None + conversion_started = time.monotonic() + records_json = [record.to_prism_json() for record in records] + self._note_window_build_phase( + "record_conversion", + time.monotonic() - conversion_started, + ) + daemon_started = time.monotonic() + try: + outcome = prepare( + mode="full", + records_json=records_json, + anchor_job_issued_at_ms=int(snapshot_anchor_ms), + append_invalidation_epoch=int(append_invalidation_epoch), + window_weight=int(snapshot_window_weight), + page_size=DEFAULT_INCREMENTAL_SHARE_WINDOW_PAGE_SIZE, + # An accepted-block preview publishes on the critical path; + # it folds in-process rather than queueing behind another + # build. + wait_for_daemon=not bypass_build_interval, + ) + finally: + self._note_window_build_phase( + "daemon_prepare", + time.monotonic() - daemon_started, + ) + # The converted dicts were consumed by the request; drop them in + # bounded slices rather than in one refcount cascade (#236). + release_share_list_incrementally(records_json) + if outcome is None or outcome.status != "prepared": + return None + conversion_started = time.monotonic() + try: + mirror = DaemonShareWindowMirror.from_full_items( + anchor_job_issued_at_ms=int(snapshot_anchor_ms), + window_weight=int(snapshot_window_weight), + page_size=DEFAULT_INCREMENTAL_SHARE_WINDOW_PAGE_SIZE, + record_count=int(outcome.record_count), + canonical_items=outcome.window_items or b"", + share_snapshot_sha256=outcome.share_snapshot_sha256 or "", + ) + except DaemonWindowMirrorDivergence: + runtime._note_window_mirror_divergence() + return None + shares_json = mirror.json_records() + self._note_window_build_phase( + "record_conversion", + time.monotonic() - conversion_started, + ) + runtime._incremental_payout_artifact_window = ( + _IncrementalPayoutArtifactWindow( + window=mirror, + shares_json=shares_json, + share_snapshot_sha256=mirror.share_snapshot_sha256, + refreshed_monotonic=observed_monotonic, + full_rescan_monotonic=observed_monotonic, + full_rescan_attempt_monotonic=observed_monotonic, + append_invalidation_epoch=append_invalidation_epoch, + ) + ) + return _PayoutWindowMaterialization( + shares_json=shares_json, + share_snapshot_sha256=mirror.share_snapshot_sha256, + snapshot_anchor_ms=int(snapshot_anchor_ms), + mode="full_rescan", + record_count=mirror.record_count, + stats=IncrementalWindowAdvanceStats( + added_rows=0, + expired_rows=0, + touched_pages=0, + ), + full_rescan_reason=reason, + ) + + def _daemon_advance_window( + self, + mirror: DaemonShareWindowMirror, + delta_records: Sequence[Any], + *, + snapshot_anchor_ms: int, + append_invalidation_epoch: int, + bypass_build_interval: bool = False, + unprepared_reason: str | None = None, + ) -> tuple[DaemonShareWindowMirror, IncrementalWindowAdvanceStats]: + """Advance the daemon-prepared window by one append-only delta. + + Outcome mapping, mirroring the in-process contract exactly: + ``fallback`` raises IncrementalWindowFallback (same handler, same + ``incremental_invariant_failed`` full rescan, daemon not retired); + ``needs_full`` and every anomaly raise _DaemonWindowRebuildRequired + so the caller rebuilds through the full oracle path. ``busy`` is a + healthy daemon another build owns: it rebuilds too, because opaque + mirror bytes cannot advance in-process, but it keeps its own reason + so a contended daemon is never mistaken for an unavailable one. + ``out_of_range`` is a healthy daemon declining a delta outside its + declared integer widths: the rebuild goes through the full path, + where the daemon declines the snapshot the same way and the + in-process fold takes the window -- reasoned + ``window_value_out_of_range``, the daemon untouched. + + ``unprepared_reason`` is the cached entry's own account of why the + daemon may not hold this base: a periodic self-check adopted the + mirror from the in-process oracle and its daemon preparation was + declined (issue #207). A ``needs_full`` for such a mirror is that + deliberate decision surfacing, and is reasoned with it rather than + reported as daemon state loss. + """ + runtime = self._runtime + prepare = getattr(runtime, "prepare_payout_window", None) + if not callable(prepare) or not self._window_pipeline_rust_enabled(): + # The switch or transport flipped between the cache-mode check + # and this advance; opaque bytes cannot advance in-process. + raise _DaemonWindowRebuildRequired( + "window_pipeline_mode_changed", + clear_cache=True, + ) + conversion_started = time.monotonic() + records_json = [record.to_prism_json() for record in delta_records] + self._note_window_build_phase( + "record_conversion", + time.monotonic() - conversion_started, + ) + daemon_started = time.monotonic() + try: + outcome = prepare( + mode="advance", + records_json=records_json, + anchor_job_issued_at_ms=int(snapshot_anchor_ms), + append_invalidation_epoch=int(append_invalidation_epoch), + base_digest=mirror.share_snapshot_sha256, + wait_for_daemon=not bypass_build_interval, + ) + finally: + self._note_window_build_phase( + "daemon_prepare", + time.monotonic() - daemon_started, + ) + if outcome is None: + raise _DaemonWindowRebuildRequired( + "window_daemon_unavailable", + clear_cache=True, + ) + if outcome.status == "busy": + # Contention, not sickness: the daemon is untouched and still + # holds this base, so the oracle rebuild below re-prepares it + # without the cache being thrown away on a healthy daemon's + # behalf. + raise _DaemonWindowRebuildRequired( + "window_daemon_busy", + clear_cache=False, + ) + if outcome.status == "needs_full": + # The daemon no longer holds the base window. A mirror adopted + # without a daemon preparation names its own reason; anything + # else is a respawn or eviction. Either way the coordinator + # mirror is still digest-verified, so only the daemon needs + # re-seeding via the full path. + raise _DaemonWindowRebuildRequired( + unprepared_reason or "window_daemon_state_lost", + clear_cache=False, + ) + if outcome.status == "fallback": + raise IncrementalWindowFallback( + outcome.error or "daemon advance invariant failed" + ) + if outcome.status == "out_of_range": + # The mirror is opaque daemon bytes and the daemon has just + # declared it cannot hold this window, so the cache is useless + # here; the full path re-asks once (declined again) and folds + # in-process. Not an anomaly: the daemon stays registered. + raise _DaemonWindowRebuildRequired( + "window_value_out_of_range", + clear_cache=True, + ) + if outcome.status != "prepared": + raise _DaemonWindowRebuildRequired( + "window_daemon_unavailable", + clear_cache=True, + ) + conversion_started = time.monotonic() + try: + advanced = mirror.advanced( + anchor_job_issued_at_ms=int(snapshot_anchor_ms), + record_count=int(outcome.record_count), + retained_drop_bytes=int(outcome.retained_drop_bytes), + appended_items=outcome.appended_items, + share_snapshot_sha256=outcome.share_snapshot_sha256 or "", + ) + except DaemonWindowMirrorDivergence: + runtime._note_window_mirror_divergence() + raise _DaemonWindowRebuildRequired( + "window_mirror_divergence", + clear_cache=True, + ) from None + finally: + self._note_window_build_phase( + "record_conversion", + time.monotonic() - conversion_started, + ) + return advanced, IncrementalWindowAdvanceStats( + added_rows=int(outcome.added_rows), + expired_rows=int(outcome.expired_rows), + touched_pages=int(outcome.touched_pages), + ) + + def _daemon_adopted_window( + self, + *, + full_window: IncrementalShareWindow, + shares_json: IncrementalShareJsonSequence, + digest: str, + advanced_window: IncrementalShareWindow | DaemonShareWindowMirror, + snapshot_anchor_ms: int, + append_invalidation_epoch: int, + ) -> tuple[DaemonShareWindowMirror, str | None]: + """Adopt the self-check oracle's window as a mirror the daemon holds. + + Issue #207: the periodic self-check used to install a mirror built + from the oracle's own pages without telling the daemon, so after an + oversized re-center (or a repair after a mismatch or an outage) the + next advance answered ``needs_full`` and paid a second full + ``snapshot_at_job_issue`` under the writer lock, recorded as + ``window_daemon_state_lost``. When the daemon already holds the + adopted digest (its own advance produced it) the mirror is taken + from the oracle pages as before. Otherwise the daemon is prepared + for the adopted window here, once, so the next build advances + incrementally. If that preparation is declined -- busy, unavailable, + out of range, or a digest the in-process oracle refutes -- the + oracle mirror is installed anyway and tagged with the daemon's own + outcome reason (``window_daemon_busy``, ``window_daemon_unavailable``, + ``window_value_out_of_range`` or ``window_mirror_divergence``, all + members of the closed rescan-reason vocabulary), so the + ``needs_full`` it will draw is reported as what declined the + adoption and never as daemon state loss. + """ + runtime = self._runtime + + def mirror_from_oracle_pages() -> DaemonShareWindowMirror: + # The bytes come from the oracle's own pages (already encoded); + # the digest is a function of exactly the retained records. + return DaemonShareWindowMirror( + anchor_job_issued_at_ms=full_window.anchor_job_issued_at_ms, + window_weight=full_window.window_weight, + page_size=full_window.page_size, + record_count=len(shares_json), + canonical_items=b",".join( + page.canonical_json_items + for page in full_window.pages + if page.canonical_json_items + ), + share_snapshot_sha256=digest, + ) + + if ( + isinstance(advanced_window, DaemonShareWindowMirror) + and advanced_window.share_snapshot_sha256 == digest + ): + return mirror_from_oracle_pages(), None + prepare = getattr(runtime, "prepare_payout_window", None) + if not callable(prepare): + return mirror_from_oracle_pages(), "window_daemon_unavailable" + conversion_started = time.monotonic() + records_json = [record.to_prism_json() for record in full_window.records()] + self._note_window_build_phase( + "record_conversion", + time.monotonic() - conversion_started, + ) + daemon_started = time.monotonic() + try: + outcome = prepare( + mode="full", + records_json=records_json, + anchor_job_issued_at_ms=int(snapshot_anchor_ms), + append_invalidation_epoch=int(append_invalidation_epoch), + window_weight=int(full_window.window_weight), + page_size=int(full_window.page_size), + # The self-check only runs off the found-block critical path. + wait_for_daemon=True, + ) + finally: + self._note_window_build_phase( + "daemon_prepare", + time.monotonic() - daemon_started, + ) + release_share_list_incrementally(records_json) + if outcome is None: + return mirror_from_oracle_pages(), "window_daemon_unavailable" + if outcome.status == "busy": + return mirror_from_oracle_pages(), "window_daemon_busy" + if outcome.status == "out_of_range": + return mirror_from_oracle_pages(), "window_value_out_of_range" + if outcome.status != "prepared": + return mirror_from_oracle_pages(), "window_daemon_unavailable" + if outcome.share_snapshot_sha256 != digest: + # The daemon folded the same records to a different digest. + # The in-process oracle is authoritative here; surface the + # disagreement and keep its window. + runtime._record_payout_artifact_event("window_mirror_divergence") + return mirror_from_oracle_pages(), "window_mirror_divergence" + conversion_started = time.monotonic() + try: + mirror = DaemonShareWindowMirror.from_full_items( + anchor_job_issued_at_ms=int(snapshot_anchor_ms), + window_weight=int(full_window.window_weight), + page_size=int(full_window.page_size), + record_count=int(outcome.record_count), + canonical_items=outcome.window_items or b"", + share_snapshot_sha256=outcome.share_snapshot_sha256 or "", + ) + except DaemonWindowMirrorDivergence: + runtime._record_payout_artifact_event("window_mirror_divergence") + return mirror_from_oracle_pages(), "window_mirror_divergence" + finally: + self._note_window_build_phase( + "record_conversion", + time.monotonic() - conversion_started, + ) + return mirror, None + + def _incremental_payout_window_materialization( + self, + *, + snapshot_anchor_ms: int, + snapshot_window_weight: int, + force_full_rescan: bool, + bypass_build_interval: bool, + append_invalidation_epoch: int, + reused_prior_balances_sha256: str | None = None, + ) -> _PayoutWindowMaterialization: + """Return an exact window using debounce, delta folding, or the oracle. + + Must run under ``_payout_state_prepare_lock``. The cache replacement + happens only after a complete materialization, except an explicit + forced invalidation clears it before the oracle so a failed reorg + check cannot later resume from unverified append-only state. The + append-epoch check below is what keeps a late-visible append from + being advanced over; it does not depend on the request flag. + """ + runtime = self._runtime + + observed = time.monotonic() + cached = runtime._incremental_payout_artifact_window + full_reason: str | None = None + if force_full_rescan: + runtime._incremental_payout_artifact_window = None + runtime._incremental_payout_artifact_window_invalidation_reason = None + cached = None + full_reason = "reconcile_invalidation" + elif cached is None: + full_reason = ( + runtime._incremental_payout_artifact_window_invalidation_reason + or "cold_start" + ) + elif cached.append_invalidation_epoch != append_invalidation_epoch: + # Append invalidation is published without waiting for this + # preparation lock. A build that acquires the lock first must not + # advance the pre-append cache under the new epoch. + runtime._incremental_payout_artifact_window = None + cached = None + full_reason = "late_visible_append" + elif isinstance( + cached.window, DaemonShareWindowMirror + ) and not self._window_pipeline_rust_enabled(): + # The Rust switch went off while a daemon mirror was cached; the + # in-process pipeline cannot advance opaque bytes, so rebuild + # from the oracle. The converse (a Python window cached while + # the switch is on) deliberately keeps advancing in Python -- + # that is the degraded state while the daemon is unavailable -- + # and the periodic runtime-check converts it back. + runtime._incremental_payout_artifact_window = None + runtime._incremental_payout_artifact_window_invalidation_reason = None + cached = None + full_reason = "window_pipeline_mode_changed" + elif not _snapshot_window_weight_within_band( + int(cached.window.window_weight), int(snapshot_window_weight) + ): + runtime._incremental_payout_artifact_window = None + runtime._incremental_payout_artifact_window_invalidation_reason = None + cached = None + full_reason = "snapshot_window_weight_out_of_band" + elif snapshot_anchor_ms < cached.window.anchor_job_issued_at_ms: + runtime._incremental_payout_artifact_window = None + runtime._incremental_payout_artifact_window_invalidation_reason = None + cached = None + full_reason = "anchor_regression" + + if cached is None: + materialized = runtime._full_payout_window_materialization( + snapshot_anchor_ms=snapshot_anchor_ms, + snapshot_window_weight=snapshot_window_weight, + reason=full_reason or "cache_invalidated", + observed_monotonic=observed, + append_invalidation_epoch=append_invalidation_epoch, + bypass_build_interval=bypass_build_interval, + ) + runtime._incremental_payout_artifact_window_invalidation_reason = None + return materialized + + min_interval = runtime._payout_artifact_min_build_interval_seconds() + full_rescan_seconds = float( + getattr( + runtime, + "payout_artifact_full_rescan_seconds", + DEFAULT_PRISM_PAYOUT_ARTIFACT_FULL_RESCAN_SECONDS, + ) + ) + # Bypass builds (accepted-block previews) refresh + # refreshed_monotonic on every block yet never run the periodic + # oracle themselves, so a sustained sub-interval block cadence + # would otherwise send every routine build back from this + # debounce and postpone the configured runtime-check indefinitely. + # An overdue runtime-check disarms the debounce for routine builds; + # the urgent preview path stays debounce-free and oracle-free + # either way. + self_check_overdue = ( + observed - cached.full_rescan_attempt_monotonic + >= full_rescan_seconds + ) + # Issue #228: a routine candidate on the tip-refresh path must not + # pay the oracle while a valid recent window exists. If its cached + # window is inside the accepted staleness bound, an overdue + # self-check is left for a build off that path. The deferral is + # bounded in wall-clock terms against the last self-check attempt, + # not against the last build: once the check is a full staleness + # bound past due it runs here whatever the build frequency, so + # candidates arriving faster than the bound cannot defer it + # forever. A deferral also leaves the check owed, and the next + # publication requests a background preparation that normally runs + # it off the critical path before that grace expires. + attribution = getattr( + self._payout_window_build_context, "attribution", None + ) + critical_path = attribution is not None and attribution.critical_path + cached_window_fresh = ( + int(snapshot_anchor_ms) - int(cached.window.anchor_job_issued_at_ms) + <= PRISM_TIP_REFRESH_WINDOW_STALENESS_SECONDS * 1000.0 + ) + self_check_grace_exhausted = ( + observed - cached.full_rescan_attempt_monotonic + >= full_rescan_seconds + PRISM_TIP_REFRESH_WINDOW_STALENESS_SECONDS + ) + self_check_deferred = bool( + self_check_overdue + and not bypass_build_interval + and critical_path + and cached_window_fresh + and not self_check_grace_exhausted + ) + run_self_check = ( + not bypass_build_interval + and self_check_overdue + and not self_check_deferred + ) + if ( + not bypass_build_interval + and not self_check_overdue + and observed - cached.refreshed_monotonic < min_interval + ): + return _PayoutWindowMaterialization( + shares_json=cached.shares_json, + share_snapshot_sha256=cached.share_snapshot_sha256, + snapshot_anchor_ms=cached.window.anchor_job_issued_at_ms, + mode="debounced", + record_count=cached.window.record_count, + stats=IncrementalWindowAdvanceStats(0, 0, 0), + ) + + delta_reader = getattr(runtime.ledger, "snapshot_between_job_issues", None) + if not callable(delta_reader): + return runtime._full_payout_window_materialization( + snapshot_anchor_ms=snapshot_anchor_ms, + snapshot_window_weight=snapshot_window_weight, + reason="delta_api_unavailable", + observed_monotonic=observed, + append_invalidation_epoch=append_invalidation_epoch, + bypass_build_interval=bypass_build_interval, + ) + + try: + delta_started = time.monotonic() + try: + delta_records = list( + delta_reader( + cached.window.anchor_job_issued_at_ms, + snapshot_anchor_ms, + ) + ) + finally: + self._note_window_build_phase( + "ledger_read", + time.monotonic() - delta_started, + ) + if isinstance(cached.window, DaemonShareWindowMirror): + advanced_window, stats = self._daemon_advance_window( + cached.window, + delta_records, + snapshot_anchor_ms=snapshot_anchor_ms, + append_invalidation_epoch=append_invalidation_epoch, + bypass_build_interval=bypass_build_interval, + unprepared_reason=cached.daemon_unprepared_reason, + ) + else: + advance_started = time.monotonic() + try: + advanced_window, stats = cached.window.advance( + delta_records, + anchor_job_issued_at_ms=snapshot_anchor_ms, + ) + finally: + self._note_window_build_phase( + "record_conversion", + time.monotonic() - advance_started, + ) + except _DaemonWindowRebuildRequired as rebuild: + # Daemon window loss (respawn/eviction), a daemon anomaly, a + # daemon another build owns, or a refuted byte mirror. The cached + # mirror stays valid when the daemon merely lost state or was + # busy; the oracle rebuild below re-prepares it. + if rebuild.clear_cache: + runtime._incremental_payout_artifact_window = None + return runtime._full_payout_window_materialization( + snapshot_anchor_ms=snapshot_anchor_ms, + snapshot_window_weight=snapshot_window_weight, + reason=rebuild.reason, + observed_monotonic=observed, + append_invalidation_epoch=append_invalidation_epoch, + bypass_build_interval=bypass_build_interval, + ) + except (IncrementalWindowFallback, ValueError): + runtime._incremental_payout_artifact_window = None + return runtime._full_payout_window_materialization( + snapshot_anchor_ms=snapshot_anchor_ms, + snapshot_window_weight=snapshot_window_weight, + reason="incremental_invariant_failed", + observed_monotonic=observed, + append_invalidation_epoch=append_invalidation_epoch, + bypass_build_interval=bypass_build_interval, + ) + + if delta_records or stats.expired_rows: + # Each window page owns its immutable JSON representation. Page + # identities survive normal advancement, so this view and its + # digest convert only append/head boundary pages; retained share + # records are never flattened or re-encoded here. + conversion_started = time.monotonic() + shares_json = advanced_window.json_records() + digest = self._canonical_json_sha256(shares_json) + self._note_window_build_phase( + "record_conversion", + time.monotonic() - conversion_started, + ) + else: + shares_json = cached.shares_json + digest = cached.share_snapshot_sha256 + + advanced = _IncrementalPayoutArtifactWindow( + window=advanced_window, + shares_json=shares_json, + share_snapshot_sha256=digest, + refreshed_monotonic=observed, + full_rescan_monotonic=cached.full_rescan_monotonic, + full_rescan_attempt_monotonic=( + cached.full_rescan_attempt_monotonic + ), + append_invalidation_epoch=append_invalidation_epoch, + ) + mode = "incremental" + check_reason: str | None = None + self_check_recentered = False + balance_check_prior_balances: tuple[dict[str, object], ...] | None = None + balance_check_mismatch = False + if run_self_check: + # The self-check is a whole-window oracle read at the cached + # weight; it is recorded as a full rescan under its check reason + # (the periodic family) and always folds in-process. + self_check_started = time.monotonic() + try: + # Within the tolerance band the cached and live snapshot + # weights legitimately differ. The match verdict must + # compare like-for-like at the cached weight (a live-weight + # comparison would turn ordinary difficulty drift into + # systematic false self_check_mismatch), and on ordinary + # drift the adopted window must ALSO keep the cached weight: + # adopting a digest the daemon never prepared would answer + # needs_full on the next advance and pay a second full + # oracle read under the writer lock at every runtime-check. + # Re-centering is therefore one-sided and thresholded -- + # only when the cached weight sits more than 25% above the + # live snapshot weight, the oversize direction nothing else + # bounds (a falling difficulty could otherwise pin an + # up-to-2x oversized oracle scan); the undersize direction + # is safety-bounded by the band floor and re-centered by + # every forced rescan. The oversize case makes the cached + # weight the larger one, so the single oracle read at the + # cached weight covers both exact cutoffs. + cached_window_weight = int(advanced_window.window_weight) + live_window_weight = int(snapshot_window_weight) + recenter_oversized = ( + cached_window_weight * 4 > live_window_weight * 5 + ) + oracle_started = time.monotonic() + try: + full_records = list( + runtime.ledger.snapshot_at_job_issue( + snapshot_anchor_ms, + window_weight=cached_window_weight, + ) + ) + finally: + self._note_window_build_phase( + "ledger_read", + time.monotonic() - oracle_started, + ) + conversion_started = time.monotonic() + comparison_window = IncrementalShareWindow.from_full_snapshot( + full_records, + anchor_job_issued_at_ms=snapshot_anchor_ms, + window_weight=cached_window_weight, + ) + if recenter_oversized: + full_window = IncrementalShareWindow.from_full_snapshot( + full_records, + anchor_job_issued_at_ms=snapshot_anchor_ms, + window_weight=live_window_weight, + ) + else: + full_window = comparison_window + shares_json = full_window.json_records() + digest = self._canonical_json_sha256(shares_json) + if isinstance(advanced_window, DaemonShareWindowMirror): + # The mirror holds canonical bytes, not records; digest + # equality is the same comparison, since the canonical + # digest is a function of exactly the retained records. + comparison_digest = ( + digest + if full_window is comparison_window + else self._canonical_json_sha256( + comparison_window.json_records() + ) + ) + matched = ( + comparison_digest + == advanced_window.share_snapshot_sha256 + ) + else: + matched = ( + comparison_window.records() + == advanced_window.records() + ) + self._note_window_build_phase( + "record_conversion", + time.monotonic() - conversion_started, + ) + except Exception: + # The already-validated delta remains usable. Space failed + # oracle attempts by the configured runtime-check interval so a + # saturated/unavailable database cannot turn a periodic + # safety check into one full CTE retry every minute. + advanced = dataclass_replace( + advanced, + full_rescan_attempt_monotonic=observed, + ) + mode = "incremental_self_check_failed" + check_reason = "periodic_self_check_failed" + else: + oracle_window: ( + IncrementalShareWindow | DaemonShareWindowMirror + ) = full_window + daemon_unprepared_reason: str | None = None + if self._window_pipeline_rust_enabled(): + # Keep the cache in mirror form so the next delta build + # advances through the daemon rather than mode-cycling; + # this is also how a Python-advancing cache (the degraded + # state after a daemon outage) converts back. When the + # daemon does not already hold the adopted digest (an + # oversized re-center, a mismatch repair, or the + # conversion after an outage) it is prepared for it here + # (issue #207), so the next advance never answers + # needs_full for a decision the coordinator itself made. + oracle_window, daemon_unprepared_reason = ( + self._daemon_adopted_window( + full_window=full_window, + shares_json=shares_json, + digest=digest, + advanced_window=advanced_window, + snapshot_anchor_ms=snapshot_anchor_ms, + append_invalidation_epoch=append_invalidation_epoch, + ) + ) + advanced = _IncrementalPayoutArtifactWindow( + window=oracle_window, + shares_json=shares_json, + share_snapshot_sha256=digest, + refreshed_monotonic=observed, + full_rescan_monotonic=observed, + full_rescan_attempt_monotonic=observed, + append_invalidation_epoch=append_invalidation_epoch, + daemon_unprepared_reason=daemon_unprepared_reason, + ) + mode = "self_check_match" if matched else "self_check_mismatch" + check_reason = "periodic_self_check" + # A deliberate re-center is flagged on the build (issue + # #207) rather than given a reason of its own: the rescan + # reason vocabulary is closed in the #224 telemetry owner, + # and with the daemon prepared for the adopted digest the + # re-center draws no rebuild for a later build to name. + self_check_recentered = bool(recenter_oversized) + if reused_prior_balances_sha256 is not None: + try: + balances_started = time.monotonic() + try: + checked_balances = tuple( + runtime.ledger.current_prior_balances() + ) + finally: + self._note_window_build_phase( + "ledger_read", + time.monotonic() - balances_started, + ) + checked_balances_sha256 = self._canonical_json_sha256( + checked_balances + ) + if ( + checked_balances_sha256 + != reused_prior_balances_sha256 + ): + balance_check_prior_balances = checked_balances + balance_check_mismatch = True + runtime._payout_prior_balances_reuse_invalidated_sha256 = ( + reused_prior_balances_sha256 + ) + # Equal-window refreshes intentionally retain + # their armed balances. Drop the stale artifact so + # the fresh balance read cannot be discarded as a + # window-only no-op when this build is installed. + with runtime._job_cache_lock: + armed = runtime._payout_ledger_artifact + if armed is not None: + armed_balances_sha256 = ( + armed.prior_balances_sha256 + or self._canonical_json_sha256( + armed.prior_balances + ) + ) + if ( + armed_balances_sha256 + == reused_prior_balances_sha256 + ): + self._disarm_payout_ledger_artifact_locked() + except Exception: + # The carry backstop is independent of the window + # oracle that just succeeded. Reverting here would + # serve a window the oracle may have refuted for a + # whole rescan interval, so keep the full window and + # leave the reused balances unverified until the next + # runtime-check. + check_reason = ( + "periodic_self_check_balance_check_failed" + ) + if check_reason is not None: + self._observe_payout_window_full_rescan( + check_reason, + FULL_RESCAN_PATH_IN_PROCESS, + time.monotonic() - self_check_started, + ) + + runtime._incremental_payout_artifact_window = advanced + return _PayoutWindowMaterialization( + shares_json=advanced.shares_json, + share_snapshot_sha256=advanced.share_snapshot_sha256, + snapshot_anchor_ms=advanced.window.anchor_job_issued_at_ms, + mode=mode, + record_count=advanced.window.record_count, + stats=stats, + full_rescan_reason=check_reason, + balance_check_prior_balances=balance_check_prior_balances, + balance_check_mismatch=balance_check_mismatch, + self_check_deferred=self_check_deferred, + self_check_recentered=self_check_recentered, + ) + + def _build_payout_ledger_artifact( + self, + expected_payout_state_generation: int, + artifact_payout_state_generation: int, + network_difficulty: int, + force_full_rescan: bool = False, + bypass_build_interval: bool = False, + during_publication: bool = False, + *, + force_prior_balances_read: bool = False, + ) -> PayoutLedgerArtifact | None: + """Build a stable ledger snapshot without publishing it. + + The build proper is ``_build_payout_ledger_artifact_scoped``; this + seam owns the per-build phase accumulator and observes it on every + exit under a closed outcome (issue #228), so a build that aborts (a + lost generation race, an anchor past the audit ceiling, an empty + window) or dies with the ledger still contributes the time it + measured. The slow builds are exactly the ones that lose those + races, and a family that only counted finished builds would read + as healthy during the incident it exists for. + """ + phases = self._begin_window_build_phases() + outcome = "failed" + try: + artifact, outcome = self._build_payout_ledger_artifact_scoped( + expected_payout_state_generation, + artifact_payout_state_generation, + network_difficulty, + force_full_rescan, + bypass_build_interval, + during_publication, + force_prior_balances_read=force_prior_balances_read, + phases=phases, + ) + return artifact + finally: + self._finish_window_build_phases() + self._observe_window_build_phases(phases, outcome) + + def _build_payout_ledger_artifact_scoped( + self, + expected_payout_state_generation: int, + artifact_payout_state_generation: int, + network_difficulty: int, + force_full_rescan: bool = False, + bypass_build_interval: bool = False, + during_publication: bool = False, + *, + force_prior_balances_read: bool = False, + phases: dict[str, float], + ) -> tuple[PayoutLedgerArtifact | None, str]: + """The build proper; returns the artifact and its closed outcome. + + Validity is anchor-scoped, not count-scoped. The pending-commit clamp + selects the highest clean anchor: every share stamped at or below it + is already durable, so the window read at that anchor is exact and + reproducible no matter how many shares commit while the walk runs. + Shares stamped above the anchor deterministically belong to the next + window; concurrent writers therefore never invalidate this attempt. + + ``force_full_rescan`` discards the incremental window and runs the + oracle (``reconcile_invalidation``), which also rereads the prior + balances. ``force_prior_balances_read`` is the narrower #224 intent: + the share window is still exact (reconciliation never writes + ``qbit_share_ledger``) but the published prior balances may not be, + so the window is reused through the normal debounce/delta path and + only the carry aggregate is read live. It is never a full rescan. + + Issue #228 adds attribution: a candidate preparation on the calling + thread says which cause and flag asked for the build (read from the + thread-local context, since the coordinator delegate reaches this + seam with a fixed signature), and every build is timed by phase. + """ + runtime = self._runtime + runtime._ensure_job_cache_state() + # A candidate preparation on this thread says why it asked for + # this build; background preparations carry no attribution. + attribution = getattr( + self._payout_window_build_context, "attribution", None + ) + payout_mutated = bool(force_full_rescan or force_prior_balances_read) + build_started = time.monotonic() + ledger_started = time.monotonic() + materialized: _PayoutWindowMaterialization + try: + lock_wait_started = time.monotonic() + with runtime._payout_state_prepare_lock: + self._note_window_build_phase( + "lock_wait", + time.monotonic() - lock_wait_started, + ) + with runtime._job_cache_lock: + if ( + expected_payout_state_generation + != runtime._payout_state_generation + ): + return None, "aborted" + append_invalidation_epoch = int( + runtime._payout_ledger_append_invalidation_epoch + ) + published_payout_artifact = ( + runtime._published_payout_state.artifact + ) + invalidated_balances_sha256 = ( + runtime._payout_prior_balances_reuse_invalidated_sha256 + ) + if ( + invalidated_balances_sha256 is not None + and published_payout_artifact is not None + and published_payout_artifact.prior_balances_sha256 + != invalidated_balances_sha256 + ): + # A later publication installed a different balance + # snapshot, so the digest that failed its oracle check can + # no longer enter this generation's reuse path. + runtime._payout_prior_balances_reuse_invalidated_sha256 = None + invalidated_balances_sha256 = None + reuse_published_balances = bool( + not payout_mutated + and published_payout_artifact is not None + and published_payout_artifact.generation + == expected_payout_state_generation + and published_payout_artifact.prior_balances_sha256 + != invalidated_balances_sha256 + ) + snapshot_window_weight = ( + PRISM_REWARD_WINDOW_MULTIPLIER + * PRISM_SNAPSHOT_WINDOW_MARGIN + * int(network_difficulty) + ) + clamp_now_ms = self._now_ms() + snapshot_anchor_ms = runtime._job_snapshot_anchor_ms(clamp_now_ms) + if ( + clamp_now_ms - snapshot_anchor_ms + > runtime._payout_artifact_max_anchor_age_ms() + ): + # The floor is held this far below now only by a wedged + # writer or a leaked release; an artifact anchored there + # would already be past the audit ceiling -- born + # expired -- on arrival. Abort before paying the window + # walk: the re-arm backoff paces retries and the + # share-commit liveness watchdog owns recovery. + return None, "aborted" + window_started = time.monotonic() + # Publish the walk's anchor before the ledger read: a + # late-visible append committing mid-walk can predate it + # while a cold or forcibly cleared cache exposes no other + # anchor, and the walk's database snapshot may already + # exclude the row. Retiring in the finally is safe here + # because the materialization installs its incremental + # pointer -- the walk's durable visibility -- before this + # exposure is withdrawn. + inflight_anchor_token = runtime._expose_inflight_scan_anchor( + snapshot_anchor_ms + ) + try: + materialized = runtime._incremental_payout_window_materialization( + snapshot_anchor_ms=snapshot_anchor_ms, + snapshot_window_weight=snapshot_window_weight, + force_full_rescan=force_full_rescan, + bypass_build_interval=bypass_build_interval, + append_invalidation_epoch=append_invalidation_epoch, + reused_prior_balances_sha256=( + published_payout_artifact.prior_balances_sha256 + if reuse_published_balances + and published_payout_artifact is not None + else None + ), + ) + finally: + runtime._retire_inflight_scan_anchor(inflight_anchor_token) + if materialized.balance_check_mismatch: + # Emit at detection time: a later, unrelated artifact + # serialization or stats failure must not hide the oracle + # alarm that already invalidated reuse. + runtime._record_payout_artifact_event( + "balance_check_mismatch" + ) + runtime._payout_artifact_log( + "payout_artifact_balance_check_mismatch", + payout_state_generation=int( + artifact_payout_state_generation + ), + balance_check_mismatch=True, + ) + if ( + materialized.balance_check_mismatch + and materialized.balance_check_prior_balances is not None + and published_payout_artifact is not None + ): + # The oracle refuted the published balance snapshot, so + # invalidating reuse alone is not enough: consumers keyed + # to the published digest (the armed-artifact fence, new + # build keys, and the synchronous fallback) would keep + # serving the refuted bytes until an unrelated generation + # change. Publish the repaired balances for the same + # generation so this build's artifact can arm and serve. + runtime._publish_self_check_repaired_balances( + expected_payout_state_generation, + stale_prior_balances_sha256=( + published_payout_artifact.prior_balances_sha256 + ), + balances=materialized.balance_check_prior_balances, + ) + window_build_seconds = time.monotonic() - window_started + if materialized.balance_check_prior_balances is not None: + # The periodic oracle already paid for the live aggregate. + # Use that exact result rather than issuing a duplicate + # carry query in the mismatch build. + prior_balances = ( + materialized.balance_check_prior_balances + ) + prior_balances_source = "ledger" + elif reuse_published_balances: + assert published_payout_artifact is not None + # Prior balances are generation-owned immutable state. + # Normal share-window refreshes cannot change them, so + # reuse the exact published bytes instead of aggregating + # the 1.3M-row carry tables on every debounced generation. + # Reconcile/withdrawal mutations force a ledger read. + prior_balances = ( + published_payout_artifact.prior_balances() + ) + prior_balances_source = "published" + else: + balances_started = time.monotonic() + try: + prior_balances = runtime.ledger.current_prior_balances() + finally: + self._note_window_build_phase( + "ledger_read", + time.monotonic() - balances_started, + ) + prior_balances_source = "ledger" + accepted_share_count, _ = runtime.accepted_share_stats() + except Exception as exc: + # Artifact preparation is speculative. The synchronous bundle path + # still owns errors when current work actually requires a snapshot. + if during_publication: + runtime._record_payout_artifact_event("build_aborted") + runtime._payout_artifact_log( + "payout_artifact_build_aborted", + payout_state_generation=int( + artifact_payout_state_generation + ), + duration_seconds=round( + time.monotonic() - build_started, + 3, + ), + during_publication=True, + build_interval_bypassed=bool(bypass_build_interval), + abort_reason=type(exc).__name__, + phase_seconds={ + phase: round(float(phases.get(phase, 0.0)), 6) + for phase in PRISM_PAYOUT_WINDOW_BUILD_PHASES + }, + ) + return None, "failed" + finally: + runtime._observe_tip_refresh_build_phase( + "ledger_snapshot", + time.monotonic() - ledger_started, + ) + if materialized.self_check_deferred: + # Owed until a publication hands it to the background walker. + with runtime.lock: + self._payout_self_check_owed = True + if not materialized.shares_json: + return None, "aborted" + copy_started = time.monotonic() + shares_json = materialized.shares_json + frozen_balances = tuple(prior_balances) + runtime._observe_tip_refresh_build_phase( + "serialization_copy", + time.monotonic() - copy_started, + ) + artifact = PayoutLedgerArtifact( + generation=0, + payout_state_generation=artifact_payout_state_generation, + network_difficulty=int(network_difficulty), + accepted_share_count=accepted_share_count, + shares_json=shares_json, + prior_balances=frozen_balances, + prepared_monotonic=time.monotonic(), + snapshot_anchor_ms=materialized.snapshot_anchor_ms, + share_snapshot_sha256=materialized.share_snapshot_sha256, + prior_balances_sha256=self._canonical_json_sha256(frozen_balances), + append_invalidation_epoch=append_invalidation_epoch, + window_build_mode=materialized.mode, + window_delta_rows=materialized.stats.added_rows, + window_expired_rows=materialized.stats.expired_rows, + window_touched_pages=materialized.stats.touched_pages, + window_build_seconds=window_build_seconds, + window_full_rescan_reason=materialized.full_rescan_reason, + window_self_check_recentered=materialized.self_check_recentered, + ) + build_seconds = time.monotonic() - build_started + log_fields: dict[str, object] = { + "payout_state_generation": int(artifact_payout_state_generation), + "duration_seconds": round(build_seconds, 3), + "window_build_seconds": round(window_build_seconds, 3), + "window_build_mode": materialized.mode, + "window_shares": materialized.record_count, + "delta_rows": materialized.stats.added_rows, + "expired_rows": materialized.stats.expired_rows, + "touched_pages": materialized.stats.touched_pages, + "anchor_age_ms": self._now_ms() - materialized.snapshot_anchor_ms, + "during_publication": bool(during_publication), + "build_interval_bypassed": bool(bypass_build_interval), + "prior_balances_source": prior_balances_source, + "prior_balances_read_forced": bool(force_prior_balances_read), + "balance_check_mismatch": materialized.balance_check_mismatch, + # Issue #228: which caller forced this build and why, readable + # from the line alone. ``invalidation_cause`` is the captured + # source cause verbatim; ``force_origin`` says whether a payout + # mutation came from the caller's explicit flag, from the + # reserved cause, both, or neither; ``cause_inherited`` marks a + # capture that found a cause already acted on (the sticky-tuple + # shape that used to force full rescans); ``window_oracle_forced`` + # is the explicit exact-oracle request. + "invalidation_cause": ( + attribution.cause if attribution is not None else None + ), + "force_origin": ( + attribution.force_origin if attribution is not None else "none" + ), + "cause_inherited": bool( + attribution is not None and attribution.cause_inherited + ), + "payout_mutated": payout_mutated, + "window_oracle_forced": bool(force_full_rescan), + "self_check_deferred": bool(materialized.self_check_deferred), + "self_check_recentered": bool(materialized.self_check_recentered), + "phase_seconds": { + phase: round(float(phases.get(phase, 0.0)), 6) + for phase in PRISM_PAYOUT_WINDOW_BUILD_PHASES + }, + } + if materialized.full_rescan_reason is not None: + log_fields["full_rescan_reason"] = materialized.full_rescan_reason + if materialized.mode == "debounced": + runtime._record_payout_artifact_event("debounced") + runtime._payout_artifact_log( + "payout_artifact_build_debounced", + **log_fields, + ) + else: + runtime._record_payout_artifact_event("built") + if materialized.mode == "full_rescan": + runtime._record_payout_artifact_event("full_rescan") + elif materialized.mode == "self_check_match": + runtime._record_payout_artifact_event("self_check_match") + elif materialized.mode == "self_check_mismatch": + runtime._record_payout_artifact_event("self_check_mismatch") + elif materialized.mode == "incremental_self_check_failed": + runtime._record_payout_artifact_event("incremental") + runtime._record_payout_artifact_event("self_check_failed") + else: + runtime._record_payout_artifact_event("incremental") + runtime._payout_artifact_log("payout_artifact_built", **log_fields) + return artifact, ( + "debounced" if materialized.mode == "debounced" else "completed" + ) + + def _prepare_payout_ledger_artifact( + self, + payout_state_generation: int, + network_difficulty: int, + *, + bypass_build_interval: bool = False, + ) -> None: + """Prepare and atomically publish an artifact for a current generation.""" + runtime = self._runtime + build_started = time.monotonic() + artifact = runtime._build_payout_ledger_artifact( + payout_state_generation, + payout_state_generation, + network_difficulty, + False, + bypass_build_interval, + ) + build_seconds = time.monotonic() - build_started + if artifact is None: + # A superseding generation, a snapshot error, an empty window, or + # a pathologically old pending-commit floor discarded this + # attempt; that condition can persist, and each attempt performs + # ledger preparation under the shared lock. Back the + # fence-failure re-arm off until something arms. + runtime._record_payout_artifact_event("build_aborted") + runtime._payout_artifact_log( + "payout_artifact_build_aborted", + payout_state_generation=int(payout_state_generation), + duration_seconds=round(build_seconds, 3), + ) + with runtime._payout_artifact_executor_lock: + runtime._payout_artifact_rearm_backoff = min( + runtime._payout_artifact_rearm_backoff * 2, + PRISM_PAYOUT_ARTIFACT_REARM_BACKOFF_CAP, + ) + return + if not runtime._install_payout_ledger_artifact(artifact): + # A discarded install -- born expired after a slow walk, an + # ordering loss to a fresher window, or a generation change -- + # armed nothing. It must not collapse the re-arm interval: an + # unconditional reset here is exactly the reset-then-rewalk + # livelock that rolled back the first anchor-scoped deploy. + with runtime._payout_artifact_executor_lock: + runtime._payout_artifact_rearm_backoff = min( + runtime._payout_artifact_rearm_backoff * 2, + PRISM_PAYOUT_ARTIFACT_REARM_BACKOFF_CAP, + ) + + def _install_payout_ledger_artifact( + self, + artifact: PayoutLedgerArtifact, + ) -> bool: + """Atomically publish a prepared artifact for its own generation. + + Snapshot-freshness-ordered and idempotent: every snapshot is taken at + a clean anchor (strictly below all pending commits), and the durable + window at or below an anchor is immutable, so the anchor orders + snapshots even when a build delayed in window conversion finishes + after a later snapshot installed (completion time cannot order + snapshots). Equal-anchor installs with an identical window -- every + flight waiter re-runs cache publication with the same prepared + artifact, and a same-window speculative rebuild re-reads unchanged + state under an equal anchor -- keep the installed generation instead + of re-keying bundle lookups for nothing. + + Returns True when the window is armed (installed fresh, refreshed in + place, or confirmed already current); only that outcome resets the + re-arm backoff. A discarded install returns False and leaves the + backoff alone -- in particular a BORN-EXPIRED artifact, whose anchor + aged past the audit ceiling during its own window walk. Arming (or + crediting) such an install would collapse the re-arm interval and + rewalk the reward window continuously: the livelock behind the + 2026-07-29 rollback. + """ + runtime = self._runtime + try: + return runtime._install_payout_ledger_artifact_outcome(artifact) + finally: + # A seeded artifact carries the exposed scan anchor of the + # synchronous read that produced it. The install decision above + # is atomic under _job_cache_lock: either the window armed (its + # own anchor is now visible to append invalidation) or it was + # discarded and can never serve, so the exposure has done its + # job either way. + runtime._retire_inflight_scan_anchor( + artifact.inflight_scan_anchor_token + ) + + def _install_payout_ledger_artifact_outcome( + self, + artifact: PayoutLedgerArtifact, + ) -> bool: + runtime = self._runtime + if not runtime._payout_artifact_reuse_active(): + # Kill-switch: never arm. An armed artifact would churn install + # events while disabled and re-key the idle-bundle fast path to + # a generation the reuse probe refuses anyway. + return False + anchor_age_ms = ( + None + if artifact.snapshot_anchor_ms is None + else self._now_ms() - int(artifact.snapshot_anchor_ms) + ) + if ( + anchor_age_ms is None + or anchor_age_ms > runtime._payout_artifact_max_anchor_age_ms() + ): + runtime._record_payout_artifact_event("born_expired") + runtime._payout_artifact_log( + "payout_artifact_born_expired", + payout_state_generation=int(artifact.payout_state_generation), + anchor_age_ms=anchor_age_ms, + window_shares=len(artifact.shares_json), + ) + return False + with runtime._job_cache_lock: + outcome, armed_generation = ( + runtime._install_payout_ledger_artifact_locked(artifact) + ) + runtime._record_payout_artifact_event(outcome) + if outcome == "discarded": + runtime._payout_artifact_log( + "payout_artifact_discarded", + payout_state_generation=int(artifact.payout_state_generation), + anchor_age_ms=anchor_age_ms, + window_shares=len(artifact.shares_json), + ) + return False + if outcome in ("installed", "refreshed"): + runtime._payout_artifact_log( + "payout_artifact_" + outcome, + generation=armed_generation, + payout_state_generation=int(artifact.payout_state_generation), + anchor_age_ms=anchor_age_ms, + window_shares=len(artifact.shares_json), + ) + # An armed artifact -- installed fresh or confirmed already current + # -- proves preparation can succeed again; let the next fence-failure + # re-arm run at the configured floor. + with runtime._payout_artifact_executor_lock: + runtime._payout_artifact_rearm_backoff = 1 + return True + + def _install_payout_ledger_artifact_locked( + self, + artifact: PayoutLedgerArtifact, + ) -> tuple[str, int | None]: + """Ordering core of the install; returns (outcome, armed generation).""" + runtime = self._runtime + if ( + artifact.payout_state_generation != runtime._payout_state_generation + or artifact.append_invalidation_epoch + != runtime._payout_ledger_append_invalidation_epoch + ): + return "discarded", None + current = runtime._payout_ledger_artifact + if ( + current is not None + and current.payout_state_generation + == artifact.payout_state_generation + ): + current_anchor_ms = ( + -1 + if current.snapshot_anchor_ms is None + else int(current.snapshot_anchor_ms) + ) + artifact_anchor_ms = ( + -1 + if artifact.snapshot_anchor_ms is None + else int(artifact.snapshot_anchor_ms) + ) + if current_anchor_ms > artifact_anchor_ms: + return "discarded", None + if ( + current.network_difficulty == artifact.network_difficulty + and current.share_snapshot_sha256 is not None + and current.share_snapshot_sha256 + == artifact.share_snapshot_sha256 + ): + # The armed artifact already carries exactly this + # window; keep its generation (no lookup re-key) but + # still treat the preparation as a success, and advance + # the freshness clock in place either way -- otherwise a + # quiet share stream, or a pending-commit floor pinning + # the anchor so every re-prove lands at the SAME anchor, + # would cycle identical rebuilds forever without ever + # un-staling the armed artifact. No fresher window is + # constructible while the window bytes are unchanged, so + # the re-walk that just re-proved them earns the credit. + # The armed balances are kept: they are what the reuse + # fence hashes against the published payout state + # (including an accepted parent's preview patch). + anchor_advanced = artifact_anchor_ms > current_anchor_ms + self._replace_payout_ledger_artifact_locked( + dataclass_replace( + current, + accepted_share_count=artifact.accepted_share_count, + prepared_monotonic=time.monotonic(), + snapshot_anchor_ms=( + artifact.snapshot_anchor_ms + if anchor_advanced + else current.snapshot_anchor_ms + ), + ) + ) + if anchor_advanced: + return "refreshed", int(current.generation) + return "already_current", int(current.generation) + template_artifacts = getattr( + runtime, + "_template_artifacts", + None, + ) + if ( + current.network_difficulty != artifact.network_difficulty + and template_artifacts is not None + and current.network_difficulty + == int(template_artifacts.network_difficulty) + and artifact.network_difficulty + != int(template_artifacts.network_difficulty) + ): + # Whether a delayed pre-retarget build's anchor ties or + # leads, keep the artifact the live template difficulty + # can actually reuse: a wrong-difficulty install would + # fail every reuse probe on the difficulty check, which + # never re-arms, until the next synchronous build + # re-seeded the live window. + return "discarded", None + runtime._payout_ledger_artifact_generation += 1 + # Freshness runs from the moment the window becomes reusable, not + # from build completion: a sync-seeded artifact is constructed + # mid-bundle-build and reaches cache publication only after the + # audit builder, so stamping here keeps the budget honest at every + # install site. + self._replace_payout_ledger_artifact_locked( + dataclass_replace( + artifact, + generation=runtime._payout_ledger_artifact_generation, + prepared_monotonic=time.monotonic(), + ) + ) + return "installed", int(runtime._payout_ledger_artifact_generation) + + def _payout_artifact_preparation_loop(self) -> None: + runtime = self._runtime + while True: + with runtime._payout_artifact_executor_lock: + request = runtime._payout_artifact_requested + bypass_build_interval = ( + runtime._payout_artifact_requested_bypass + ) + runtime._payout_artifact_requested = None + runtime._payout_artifact_requested_bypass = False + if request is None: + runtime._payout_artifact_future = None + return + phases = runtime._job_build_phases() + phases.clear() + try: + runtime._prepare_payout_ledger_artifact( + *request, + bypass_build_interval=bypass_build_interval, + ) + finally: + runtime._flush_job_build_phases(phases) + + def _schedule_payout_ledger_artifact_preparation( + self, + payout_state_generation: int, + network_difficulty: int, + *, + min_interval_seconds: float | None = None, + bypass_build_interval: bool = False, + ) -> None: + """Latest-generation-wins scheduling with one worker and one slot. + + With min_interval_seconds, the debounce check, timestamp update, and + request enqueue are one atomic step under the executor lock: a + concurrent dequeue or a racing probe cannot slip a duplicate request + into the emptied slot and run the reward-window snapshot twice. + """ + runtime = self._runtime + runtime._ensure_job_cache_state() + if not runtime._payout_artifact_reuse_active(): + # Kill-switch: no background reward-window walks either. Off + # disables the reuse line -- probe refusals, no walks, no + # arming -- restoring pre-reuse delivery economics; the + # synchronous build path keeps the re-landed anchor-selection + # semantics rather than the pre-#102 count fences. + return + with runtime._payout_artifact_executor_lock: + if runtime._payout_artifact_executor_shutdown: + return + if min_interval_seconds is not None: + # Consecutive aborted rebuilds stretch the re-arm interval; + # event-driven preparations below reset the backoff because + # they mark a real state change worth retrying promptly for. + effective_interval = ( + min_interval_seconds * runtime._payout_artifact_rearm_backoff + ) + if ( + runtime._payout_artifact_last_schedule_monotonic is not None + and time.monotonic() + - runtime._payout_artifact_last_schedule_monotonic + < effective_interval + ): + return + rearm_backoff = runtime._payout_artifact_rearm_backoff + else: + runtime._payout_artifact_rearm_backoff = 1 + rearm_backoff = None + runtime._payout_artifact_requested = ( + int(payout_state_generation), + int(network_difficulty), + ) + runtime._payout_artifact_requested_bypass = ( + runtime._payout_artifact_requested_bypass + or bool(bypass_build_interval) + ) + runtime._payout_artifact_last_schedule_monotonic = time.monotonic() + if runtime._payout_artifact_future is None: + executor = runtime._payout_artifact_executor + if executor is None: + executor = ThreadPoolExecutor( + max_workers=1, + thread_name_prefix="prism-payout-artifact", + ) + runtime._payout_artifact_executor = executor + runtime._payout_artifact_future = executor.submit( + runtime._payout_artifact_preparation_loop + ) + if rearm_backoff is not None: + # Only debounced fence-failure re-arms log here; event-driven + # preparations ride payout publications, which are already + # visible. Recorded outside the executor lock the counter + # shares. + runtime._record_payout_artifact_event("rearm_scheduled") + runtime._payout_artifact_log( + "payout_artifact_rearm_scheduled", + payout_state_generation=int(payout_state_generation), + backoff=int(rearm_backoff), + ) + + def _payout_artifact_max_anchor_age_ms(self) -> float: + """Audit ceiling on a served window's wall-clock anchor age.""" + runtime = self._runtime + return ( + float( + getattr( + runtime, + "payout_artifact_max_anchor_age_seconds", + DEFAULT_PRISM_PAYOUT_ARTIFACT_MAX_ANCHOR_AGE_SECONDS, + ) + ) + * 1000.0 + ) + + def _payout_artifact_reanchor_seconds(self) -> float: + """Anchor age that triggers a background re-anchor while reuse keeps serving.""" + runtime = self._runtime + return float( + getattr( + runtime, + "payout_artifact_reanchor_seconds", + DEFAULT_PRISM_PAYOUT_ARTIFACT_REANCHOR_SECONDS, + ) + ) + + def _payout_artifact_min_build_interval_seconds(self) -> float: + """Minimum normal cadence; found-block settlement bypasses it.""" + runtime = self._runtime + + return float( + getattr( + runtime, + "payout_artifact_min_build_interval_seconds", + runtime._payout_artifact_reanchor_seconds(), + ) + ) + + def _payout_artifact_reuse_active(self) -> bool: + """Master kill-switch (PRISM_PAYOUT_ARTIFACT_REUSE) for the reuse line.""" + runtime = self._runtime + return bool(getattr(runtime, "payout_artifact_reuse_enabled", True)) + + @staticmethod + def _payout_artifact_log(event: str, **fields: object) -> None: + """Single-line JSON lifecycle log; the incident was invisible here.""" + print( + "prism coordinator: " + + json.dumps({"event": event, **fields}, sort_keys=True), + flush=True, + ) + + def _record_payout_artifact_event(self, event: str) -> None: + runtime = self._runtime + runtime._ensure_job_cache_state() + with runtime._payout_artifact_executor_lock: + counts = runtime.payout_artifact_event_counts + counts[event] = int(counts.get(event, 0)) + 1 + + def _note_window_mirror_divergence(self) -> None: + """Record a refuted mirror and make sure it cannot be advanced again. + + Mirror construction reconciles bytes, digest and record count + eagerly, so a divergence reaching a consumer means an assumption + this code makes about the daemon's output is wrong. Whatever raised + it is already failing its own operation; the job here is to make + sure the artifact window that produced the doubt is not carried into + the NEXT build, and that the event is visible in metrics rather than + only in whatever traceback the caller prints. + + Deliberately does not take _payout_state_prepare_lock: callers reach + this from build and landing paths that must not queue behind an + oracle rebuild, and the cost of racing a concurrent materialization + is one extra full rescan, never an unsound window. + """ + runtime = self._runtime + runtime._record_payout_artifact_event("window_mirror_divergence") + runtime._incremental_payout_artifact_window = None + runtime._incremental_payout_artifact_window_invalidation_reason = ( + "window_mirror_divergence" + ) + + def _usable_payout_ledger_artifact( + self, + payout_state_generation: int, + network_difficulty: int, + *, + rearm_on_fence_failure: bool = True, + ) -> PayoutLedgerArtifact | None: + """Return the armed artifact when reuse is valid for NEW work. + + Validity is anchor-scoped and EVENT-DRIVEN: the artifact's window is + exact at its own snapshot anchor (which reused bundles declare), so + shares landing after the anchor never invalidate it -- they belong + to the next window by construction -- and the artifact stays valid + until a payout event actually changes the state it snapshots (the + generation and balances fences below). Wall-clock time is NOT a + validity input below the audit ceiling: the 2026-07-29 rollback was + a 10s anchor-age budget rejecting every slow build on arrival, and + the 2026-07-30 brownout was the same budget moved to install time -- + at production cadence (a shared build every 10-14s, a 4-8s window + walk) a 10s budget expires between consecutive builds, so every + build fell back to the synchronous reward-window walk while the + re-arm worker walked the same window in parallel. Past the + REANCHOR floor the probe schedules a debounced background re-anchor + but KEEPS SERVING the armed window; only past the audit ceiling + (the bound on how far a paid window may trail the live ledger -- + post-anchor shares settle in the next window, none are lost) does + reuse fail closed. The ceiling governs NEW reuse and issuance + decisions; an in-flight build keeps the window it already selected, + so a declared anchor slightly past the ceiling can still publish. + """ + runtime = self._runtime + runtime._ensure_job_cache_state() + if not runtime._payout_artifact_reuse_active(): + return None + with runtime._job_cache_lock: + artifact = runtime._payout_ledger_artifact + published_artifact = runtime._published_payout_state.artifact + append_invalidation_epoch = ( + runtime._payout_ledger_append_invalidation_epoch + ) + if ( + artifact is None + or artifact.payout_state_generation != payout_state_generation + or artifact.network_difficulty != int(network_difficulty) + or artifact.append_invalidation_epoch + != append_invalidation_epoch + ): + return None + if artifact.snapshot_anchor_ms is None: + # Without a recorded anchor the artifact cannot declare the + # anchor reused bundles must stamp; fail closed. + return None + anchor_age_ms = self._now_ms() - int(artifact.snapshot_anchor_ms) + if anchor_age_ms > runtime._payout_artifact_max_anchor_age_ms(): + # The declared anchor crossed the audit ceiling; the window may + # no longer be served to new work. Queue a bounded speculative + # rebuild that re-anchors it; in-flight builds keep the copy + # they selected. + if rearm_on_fence_failure: + runtime._rearm_payout_ledger_artifact_after_fence_failure( + payout_state_generation, + network_difficulty, + ) + runtime._record_payout_artifact_event("probe_rejected_ceiling") + return None + if anchor_age_ms > runtime._payout_artifact_reanchor_seconds() * 1000.0: + # Aging but valid: schedule the debounced background re-anchor + # and keep serving. The delivery path must never pay the + # synchronous reward-window walk while a correct armed window + # exists; the re-anchor swaps in a fresher window off the + # critical path. Counted unconditionally so the canary can + # distinguish "serving past the floor, re-anchor pending" from + # "never crossing the floor" -- rearm_scheduled alone + # under-counts through the debounce and suppression paths. + runtime._record_payout_artifact_event("probe_past_floor") + if rearm_on_fence_failure: + runtime._rearm_payout_ledger_artifact_after_fence_failure( + payout_state_generation, + network_difficulty, + ) + if published_artifact is None: + try: + published_artifact = runtime._current_payout_state_artifact() + except Exception: + return None + balances_sha256 = artifact.prior_balances_sha256 or self._canonical_json_sha256( + artifact.prior_balances + ) + with runtime._job_cache_lock: + if ( + runtime._payout_state_generation != payout_state_generation + or runtime._published_payout_state.artifact is not published_artifact + or runtime._payout_ledger_append_invalidation_epoch + != artifact.append_invalidation_epoch + ): + return None + latest = runtime._payout_ledger_artifact + if latest is not artifact: + # An equal-window freshness restamp (already_current / + # refreshed) replaces the armed object while this probe was + # hashing balances. The restamp keeps the generation, the + # window bytes, and the balances object, and only moves the + # freshness stamp (and possibly the anchor) forward, so the + # validity established above still holds for the restamped + # copy; failing closed here would turn the intentional + # pinned-floor recovery path into a spurious synchronous + # reward-window walk. A real re-key changes the generation + # or the window sha and still fails closed. + if ( + latest is None + or latest.generation != artifact.generation + or latest.payout_state_generation + != artifact.payout_state_generation + or latest.network_difficulty != artifact.network_difficulty + or latest.share_snapshot_sha256 is None + or latest.share_snapshot_sha256 + != artifact.share_snapshot_sha256 + or latest.snapshot_anchor_ms is None + or latest.prior_balances is not artifact.prior_balances + or latest.append_invalidation_epoch + != artifact.append_invalidation_epoch + ): + return None + artifact = latest + if balances_sha256 != published_artifact.prior_balances_sha256: + # A candidate can carry a ledger snapshot prepared before its + # payout state is published. Never keep retrying that stale + # shortcut; the synchronous path will take a fresh snapshot. + self._disarm_payout_ledger_artifact_locked() + return None + served = artifact + # Recorded outside the cache lock the event counter's own lock + # ordering forbids nesting under. + runtime._record_payout_artifact_event("served_reuse") + return served + + def _rearm_payout_ledger_artifact_after_fence_failure( + self, + payout_state_generation: int, + network_difficulty: int, + ) -> None: + """Debounced rebuild scheduling for an artifact past the re-anchor floor or ceiling. + + The interval floor keeps delta/carry reads (and any full fallback) + from running continuously when artifacts age out faster than rebuilds + complete. + Landed accepted-block previews suppress the re-arm entirely: a + speculative rebuild in that window would read database balances the + published prospective state has already superseded, and the + durable-confirmation path resumes preparation itself once the gap + closes. + """ + runtime = self._runtime + runtime._ensure_job_cache_state() + with runtime._job_cache_lock: + if ( + runtime._payout_state_publication_blocked + or payout_state_generation != runtime._payout_state_generation + ): + return + with runtime._accepted_block_payout_preview_condition: + if any( + transition.landed + for transition in runtime._accepted_block_payout_previews.values() + ): + return + cached_window = getattr( + runtime, + "_incremental_payout_artifact_window", + None, + ) + if ( + cached_window is not None + and time.monotonic() - cached_window.refreshed_monotonic + < runtime._payout_artifact_min_build_interval_seconds() + ): + # A pinned/old declared anchor can cross the re-anchor floor while + # the window itself was just re-proved. Do not enqueue a worker + # every five seconds merely to retag the same pages and re-read + # carry balances; the next probe after the real cadence expires + # will schedule it. + return + runtime._schedule_payout_ledger_artifact_preparation( + payout_state_generation, + network_difficulty, + min_interval_seconds=float( + getattr( + runtime, + "payout_artifact_rearm_min_seconds", + DEFAULT_PRISM_PAYOUT_ARTIFACT_REARM_MIN_SECONDS, + ) + ), + ) + + def _schedule_current_payout_ledger_artifact_if_missing(self) -> None: + """Resume speculative preparation after a durable preview catches up.""" + runtime = self._runtime + runtime._ensure_job_cache_state() + with runtime._job_cache_lock: + payout_state_generation = runtime._payout_state_generation + template_artifacts = runtime._template_artifacts + if template_artifacts is None: + return + if runtime._usable_payout_ledger_artifact( + payout_state_generation, + template_artifacts.network_difficulty, + # This call site enqueues unconditionally below; letting the + # probe also re-arm would race the one-slot worker's dequeue and + # run the reward-window snapshot twice for one resumption. + rearm_on_fence_failure=False, + ) is not None: + return + runtime._schedule_payout_ledger_artifact_preparation( + payout_state_generation, + template_artifacts.network_difficulty, + ) + + def shutdown_payout_artifact_executor(self) -> None: + runtime = self._runtime + runtime._ensure_job_cache_state() + with runtime._payout_artifact_executor_lock: + executor = runtime._payout_artifact_executor + runtime._payout_artifact_executor = None + runtime._payout_artifact_executor_shutdown = True + runtime._payout_artifact_requested = None + if executor is not None: + executor.shutdown(wait=True, cancel_futures=True) + + def _prepare_payout_state_artifact( + self, + *, + generation: int, + source_generation: int, + cancellation: _JobBuildCancellation | None = None, + ) -> PayoutStateArtifact: + """Snapshot carry-forward balances once for a payout generation.""" + runtime = self._runtime + + started = time.monotonic() + if cancellation is not None: + cancellation.raise_if_cancelled("payout artifact read") + with runtime._payout_state_prepare_lock: + balances = runtime.ledger.current_prior_balances() + if cancellation is not None: + cancellation.raise_if_cancelled("payout artifact serialization") + artifact = runtime._payout_state_artifact_from_balances( + generation=generation, + source_generation=source_generation, + balances=balances, + ) + phases = runtime._job_build_phases() + phases["payout_artifact"] = phases.get("payout_artifact", 0.0) + ( + time.monotonic() - started + ) + return artifact + + def _payout_state_artifact_from_balances( + self, + *, + generation: int, + source_generation: int, + balances: list[dict[str, object]], + ) -> PayoutStateArtifact: + balances_json = self._canonical_json_text(balances) + return PayoutStateArtifact( + generation=generation, + source_generation=source_generation, + prior_balances_json=balances_json, + prior_balances_sha256=hashlib.sha256(balances_json.encode()).hexdigest(), + prepared_monotonic=time.monotonic(), + ) + + def _publish_self_check_repaired_balances( + self, + expected_payout_state_generation: int, + *, + stale_prior_balances_sha256: str, + balances: Sequence[dict[str, object]], + ) -> bool: + """Replace the published balance snapshot the periodic oracle refuted. + + The published artifact is otherwise immutable for its generation, but + a confirmed drift means its bytes no longer match the durable ledger + (a payout mutation whose response was lost). Swapping in the repaired + balances re-keys new work to the corrected digest, and every serving + decision for NOT-yet-stamped work re-checks that digest. Jobs already + stamped for miners have no such fence -- their admission compares + only the payout generation and append epoch, which both survive this + swap -- so the repair also schedules the same refresh wave a payout + publication schedules; digest-aware reselection then replaces exactly + the active jobs keyed to the refuted snapshot. Only the exact refuted + snapshot is replaced: any concurrent publication wins the guarded + swap (and drives its own wave). + """ + runtime = self._runtime + runtime._ensure_job_cache_state() + with runtime._job_cache_lock: + source_generation = runtime._published_payout_state.source_generation + repaired = runtime._payout_state_artifact_from_balances( + generation=int(expected_payout_state_generation), + source_generation=int(source_generation), + balances=list(balances), + ) + with runtime._job_cache_lock: + published = runtime._published_payout_state + current = published.artifact + if ( + runtime._payout_state_publication_blocked + or runtime._payout_state_generation + != expected_payout_state_generation + or published.generation != expected_payout_state_generation + or published.source_generation != source_generation + or current is None + or current.generation != expected_payout_state_generation + or current.prior_balances_sha256 + != stale_prior_balances_sha256 + ): + return False + runtime._published_payout_state = dataclass_replace( + published, + artifact=repaired, + ) + runtime._record_payout_artifact_event("balance_repair_published") + runtime._payout_artifact_log( + "payout_artifact_balance_repair_published", + payout_state_generation=int(expected_payout_state_generation), + stale_prior_balances_sha256=stale_prior_balances_sha256, + repaired_prior_balances_sha256=repaired.prior_balances_sha256, + ) + # Active jobs keyed to the refuted digest remain mineable until a + # wave replaces them -- a block solved from one would commit the + # stale allocation. The pending mark survives a superseded wave and + # the digest reselection is idempotent, so a wave that raced this + # repair converges on the next pass. + runtime._mark_tip_refresh_pending(int(expected_payout_state_generation)) + runtime._schedule_tip_refresh_retry() + return True + + def _current_payout_state_artifact( + self, + cancellation: _JobBuildCancellation | None = None, + ) -> PayoutStateArtifact: + """Return the immutable artifact published for the current generation.""" + runtime = self._runtime + + runtime._ensure_job_cache_state() + while True: + with runtime._job_cache_lock: + if runtime._payout_state_publication_blocked: + raise PayoutStatePublicationBlocked( + "payout state invalidation is pending publication" + ) + published = runtime._published_payout_state + if ( + published.generation == runtime._payout_state_generation + and published.artifact is not None + and published.artifact.generation == published.generation + ): + return published.artifact + generation = runtime._payout_state_generation + source_generation = published.source_generation + artifact = runtime._prepare_payout_state_artifact( + generation=generation, + source_generation=source_generation, + cancellation=cancellation, + ) + with runtime._job_cache_lock: + published = runtime._published_payout_state + if ( + runtime._payout_state_publication_blocked + or runtime._payout_state_generation != generation + or published.source_generation != source_generation + ): + if cancellation is not None: + cancellation.raise_if_cancelled( + "payout artifact publication race" + ) + continue + runtime._published_payout_state = dataclass_replace( + published, + artifact=artifact, + ) + return artifact + + @contextmanager + def _payout_balance_mutation(self) -> Iterator[None]: + """Serialize durable balance changes without excluding delivery.""" + runtime = self._runtime + runtime._ensure_job_cache_state() + with runtime._block_submitter_lock( + runtime._payout_balance_mutation_lock, + "payout-balance-mutation", + ): + with runtime._accepted_block_payout_preview_condition: + landed_transition = any( + transition.landed + for transition in runtime._accepted_block_payout_previews.values() + ) + if landed_transition: + raise PayoutStatePublicationBlocked( + "accepted block payout confirmation is still pending" + ) + yield + + def _begin_accepted_block_payout_preview( + self, + block_hash: str, + *, + block_height: int | None = None, + ) -> None: + """Prevent child work from snapshotting pre-accept balances.""" + runtime = self._runtime + runtime._ensure_job_cache_state() + key = block_hash.lower() + with runtime._accepted_block_payout_preview_condition: + runtime._invalidated_accepted_block_payout_previews.pop(key, None) + existing = runtime._accepted_block_payout_previews.get(key) + if existing is None: + runtime._accepted_block_payout_previews[key] = ( + AcceptedBlockPayoutTransition(block_height=block_height) + ) + elif ( + block_height is not None + and existing.block_height is not None + and existing.block_height != block_height + ): + raise RuntimeError("accepted block payout transition height changed") + elif existing.block_height is None and block_height is not None: + runtime._accepted_block_payout_previews[key] = dataclass_replace( + existing, + block_height=block_height, + ) + + def _mark_accepted_block_payout_landed( + self, + block_hash: str, + *, + block_height: int, + ) -> None: + """Bar reconciliation after submitblock makes a candidate active.""" + runtime = self._runtime + runtime._ensure_job_cache_state() + key = block_hash.lower() + with runtime._accepted_block_payout_preview_condition: + existing = runtime._accepted_block_payout_previews.get( + key, + AcceptedBlockPayoutTransition(block_height=block_height), + ) + if existing.block_height not in {None, block_height}: + raise RuntimeError("accepted block payout transition height changed") + runtime._accepted_block_payout_previews[key] = dataclass_replace( + existing, + block_height=block_height, + landed=True, + landed_monotonic=( + existing.landed_monotonic + if existing.landed_monotonic is not None + else time.monotonic() + ), + ) + runtime._accepted_block_payout_preview_condition.notify_all() + + def _unmark_accepted_block_payout_landed(self, block_hash: str) -> None: + """Withdraw a landed bar armed for an attempt that never reached RPC. + + Only sound while the submit outcome is NOT uncertain: the caller + must know submitblock was never invoked under this arming (a + preflight refusal such as ``WriterLeaseRenewalDeferred``), so there + is no possibly-active coinbase for the barrier to preserve and no + reason to keep reconciliation and payout-state publication barred + while the candidate waits for its retry. The pending preview entry + survives — child work keeps waiting instead of snapshotting + pre-accept balances, matching the startup-replay state a durable + candidate restores — and the next attempt re-arms ``landed`` before + its own RPC. + """ + runtime = self._runtime + runtime._ensure_job_cache_state() + key = block_hash.lower() + with runtime._accepted_block_payout_preview_condition: + existing = runtime._accepted_block_payout_previews.get(key) + if existing is None or not existing.landed: + return + runtime._accepted_block_payout_previews[key] = dataclass_replace( + existing, + landed=False, + landed_monotonic=None, + ) + runtime._accepted_block_payout_preview_condition.notify_all() + + def _publish_accepted_block_payout_preview( + self, + block_hash: str, + balances: list[dict[str, object]], + ) -> list[dict[str, object]]: + """Publish the balances child work must observe after confirmation. + + Canonicalization happens before the delivery boundary. The gate only + installs an immutable pointer and advances the generation, so no job + bound to the pre-accept balances can land after publication. + """ + runtime = self._runtime + normalized = runtime.normalized_prior_balances(balances) + serialized = runtime._serialize_prior_balance_preview(normalized) + key = block_hash.lower() + with runtime._block_submitter_lock( + runtime._payout_balance_mutation_lock, + "payout-balance-mutation", + ): + with runtime._accepted_block_payout_preview_condition: + existing = runtime._accepted_block_payout_previews.get(key) + existing_preview = existing.preview if existing is not None else None + if existing_preview is not None: + if existing_preview != serialized: + raise RuntimeError( + "accepted block payout preview changed during retry" + ) + if ( + existing is not None + and existing.published_generation is not None + ): + # A matching preview that already carries a published + # generation is visible to waiting children, so this + # call is the publication for issue #181's interval + # when it is the first one seen for the hash; the + # observer drops it otherwise. + runtime._observe_accepted_block_preview_publication( + key, + result="published", + ) + return runtime._materialize_prior_balance_preview( + existing_preview + ) + # A bounded publication loss retains the compact preview + # locally while delivery remains fenced. Matching retries + # must still cross the atomic publication boundary so they + # install a generation and reopen admission. + + published = None + try: + captured = runtime._capture_payout_state_source() + reserved = runtime._reserve_payout_state_source_if_current( + captured[1], + "accepted_block_preview", + tip_hash=key, + invalidated_monotonic=time.monotonic(), + ) + if reserved is None: + candidate = runtime._prepared_payout_state_candidate( + runtime._capture_payout_state_source(), + bypass_build_interval=True, + ) + else: + candidate = runtime._prepared_payout_state_candidate(reserved) + candidate = runtime._accepted_block_preview_candidate( + candidate, + block_hash=key, + preview=serialized, + ) + # Close delivery admission before exposing the preview. The + # generation publication below performs the only atomic pointer + # swap and reopens admission; no preparation lock is involved. + runtime._block_payout_state_publication(force=True) + published = runtime._publish_payout_state_candidate(candidate) + if published is None: + max_retries = max( + 0, + int( + getattr( + runtime, + "payout_reconcile_supersession_retries", + DEFAULT_PRISM_PAYOUT_RECONCILE_SUPERSESSION_RETRIES, + ) + ), + ) + for _attempt in range(max_retries): + candidate = runtime._accepted_block_preview_candidate( + runtime._prepared_payout_state_candidate( + runtime._capture_payout_state_source(), + bypass_build_interval=True, + ), + block_hash=key, + preview=serialized, + ) + published = runtime._publish_payout_state_candidate(candidate) + if published is not None: + break + except Exception as exc: + # Last-resort degraded retention (issue #188): candidate + # preparation already degrades a ledger-timeout window build + # to the cached armed window, so reaching here means an + # in-memory publication step itself failed. Fall through to + # the fenced local-retention branch below exactly like a + # lost atomic publication: admission is force-blocked, the + # immutable compact preview remains visible to children + # already waiting on the transition, and the tip-refresh + # retry loop must publish a fresh source before new job + # admission resumes. + published = None + print( + "prism coordinator: accepted block payout preview " + f"publication degraded hash={key}: {exc}", + flush=True, + ) + if published is None: + runtime._block_payout_state_publication(force=True) + # Admission is now globally fenced, so retaining the compact + # preview locally is safe even though it did not win an atomic + # generation publication. Finalization may finish; the normal + # retry loop will publish the newest source before delivery + # resumes. + with runtime._accepted_block_payout_preview_condition: + transition = runtime._accepted_block_payout_previews.get( + key, + AcceptedBlockPayoutTransition(landed=True), + ) + runtime._accepted_block_payout_previews[key] = dataclass_replace( + transition, + landed=True, + preview=serialized, + published_generation=None, + ) + runtime._accepted_block_payout_preview_condition.notify_all() + # Both exits above make the preview observable to children already + # waiting on the transition -- the generation publication through the + # atomic pointer swap, the fenced retention branch through the + # compact preview plus notify_all -- so both close issue #181's + # acceptance-to-publication interval. They are labelled apart because + # only one of them installed a generation. + runtime._observe_accepted_block_preview_publication( + key, + result="published" if published is not None else "degraded", + ) + return runtime._materialize_prior_balance_preview(serialized) + + def _accepted_block_preview_candidate( + self, + candidate: PayoutStateCandidate, + *, + block_hash: str, + preview: tuple[tuple[str, str, str, int], ...], + ) -> PayoutStateCandidate: + """Bind a compact preview to its prepared artifact before gating.""" + runtime = self._runtime + ledger_artifact = candidate.ledger_artifact + if ledger_artifact is not None: + # The artifact was prepared before accepted-block persistence, so + # replace only its balance view with the verified prospective + # state. This allocation must stay outside delivery publication. + # The memoized balances digest must be re-derived with the patch: + # a stale digest would fail the reuse probe's balances fence + # against the very state this patch installs. + patched_balances = tuple( + runtime._materialize_prior_balance_preview(preview) + ) + ledger_artifact = dataclass_replace( + ledger_artifact, + prior_balances=patched_balances, + prior_balances_sha256=self._canonical_json_sha256(patched_balances), + ) + return dataclass_replace( + candidate, + accepted_block_hash=block_hash, + accepted_block_preview=preview, + ledger_artifact=ledger_artifact, + ) + + @staticmethod + def _serialize_prior_balance_preview( + balances: list[dict[str, object]], + ) -> tuple[tuple[str, str, str, int], ...]: + return tuple( + ( + str(balance["recipient_id"]), + str(balance["order_key"]), + str(balance["p2mr_program_hex"]), + int(balance["balance_sats"]), + ) + for balance in balances + ) + + def _accepted_block_payout_preview_from_bundle( + self, + final_bundle: dict[str, Any], + *, + prior_balances: list[dict[str, object]] | None = None, + ) -> list[dict[str, object]]: + """Derive the confirmed carry-forward view from a verified bundle. + + The bundle may be a dictionary or a bounded artifact view whose + ``accounts`` member is a lazy on-disk sequence (issue #255); the + accounts are walked once, row by row, in either case. + """ + runtime = self._runtime + manifest = final_bundle.get("payout_policy_manifest") + accounts = manifest.get("accounts") if isinstance(manifest, Mapping) else None + if ( + accounts is None + or isinstance(accounts, (str, bytes, bytearray)) + or not isinstance(accounts, Sequence) + ): + raise RuntimeError("accepted block payout manifest is missing accounts") + prior_identities: dict[str, tuple[str, str]] = {} + for balance in prior_balances or []: + program = str(balance.get("p2mr_program_hex", "")).lower() + identity = ( + str(balance.get("order_key", "")), + str(balance.get("recipient_id", "")), + ) + prior_identities[program] = min( + identity, + prior_identities.get(program, identity), + ) + balances: list[dict[str, object]] = [] + for account in accounts: + if not isinstance(account, Mapping): + continue + if str(account.get("account_type", "miner")) == "pool_fee": + continue + balance_sats = int(account.get("carry_forward_balance_sats", 0)) + if balance_sats == 0: + continue + program = str(account.get("p2mr_program_hex", "")).lower() + account_identity = ( + str(account.get("order_key", "")), + str(account.get("recipient_id", "")), + ) + order_key, recipient_id = min( + account_identity, + prior_identities.get(program, account_identity), + ) + balances.append( + { + "recipient_id": recipient_id, + "order_key": order_key, + "p2mr_program_hex": program, + "balance_sats": balance_sats, + } + ) + return runtime.normalized_prior_balances(balances) + + @staticmethod + def _materialize_prior_balance_preview( + preview: tuple[tuple[str, str, str, int], ...], + ) -> list[dict[str, object]]: + return [ + { + "recipient_id": recipient_id, + "order_key": order_key, + "p2mr_program_hex": p2mr_program_hex, + "balance_sats": balance_sats, + } + for recipient_id, order_key, p2mr_program_hex, balance_sats in preview + ] + + def _clear_accepted_block_payout_preview( + self, + block_hash: str, + *, + invalidate_published: bool = False, + ) -> None: + runtime = self._runtime + runtime._ensure_job_cache_state() + key = block_hash.lower() + with runtime._block_submitter_lock( + runtime._payout_balance_mutation_lock, + "payout-balance-mutation", + ): + with runtime._accepted_block_payout_preview_condition: + existing = runtime._accepted_block_payout_previews.get(key) + if existing is None: + if not invalidate_published: + runtime._invalidated_accepted_block_payout_previews.pop( + key, + None, + ) + runtime._accepted_block_payout_preview_condition.notify_all() + return + if not invalidate_published: + # Durable state now equals the published prospective view; + # removing the override changes no logical payout state. + runtime._accepted_block_payout_previews.pop(key, None) + runtime._invalidated_accepted_block_payout_previews.pop(key, None) + runtime._accepted_block_payout_preview_condition.notify_all() + return + if existing.preview is None: + # Nothing crossed a generation boundary. A landed + # transition still becomes a tombstone so descendants + # cannot fall through to an uncertain database snapshot. + runtime._accepted_block_payout_previews.pop(key, None) + if existing.landed: + runtime._invalidated_accepted_block_payout_previews[key] = ( + existing.block_height + ) + runtime._accepted_block_payout_preview_condition.notify_all() + return + + captured = runtime._capture_payout_state_source() + reserved = runtime._reserve_payout_state_source_if_current( + captured[1], + "accepted_block_preview_withdrawn", + tip_hash=captured[2], + invalidated_monotonic=time.monotonic(), + ) + candidate = runtime._prepared_payout_state_candidate( + reserved if reserved is not None else runtime._capture_payout_state_source(), + force_full_window_rescan=True, + bypass_build_interval=True, + ) + candidate = dataclass_replace( + candidate, + accepted_block_hash=key, + accepted_block_withdrawal=True, + accepted_block_height=existing.block_height, + ) + runtime._block_payout_state_publication(force=True) + published = runtime._publish_payout_state_candidate(candidate) + if published is None: + max_retries = max( + 0, + int( + getattr( + runtime, + "payout_reconcile_supersession_retries", + DEFAULT_PRISM_PAYOUT_RECONCILE_SUPERSESSION_RETRIES, + ) + ), + ) + for _attempt in range(max_retries): + candidate = dataclass_replace( + runtime._prepared_payout_state_candidate( + runtime._capture_payout_state_source(), + force_full_window_rescan=True, + bypass_build_interval=True, + ), + accepted_block_hash=key, + accepted_block_withdrawal=True, + accepted_block_height=existing.block_height, + ) + published = runtime._publish_payout_state_candidate(candidate) + if published is not None: + break + if published is None: + # Delivery remains fenced. Install the tombstone immediately + # so even local builders cannot fall through while the newest + # source waits for a retry publication. + with runtime._accepted_block_payout_preview_condition: + runtime._accepted_block_payout_previews.pop(key, None) + runtime._invalidated_accepted_block_payout_previews[key] = ( + existing.block_height + ) + runtime._accepted_block_payout_preview_condition.notify_all() + + def _accepted_block_payout_transition_landed(self, block_hash: str) -> bool: + runtime = self._runtime + runtime._ensure_job_cache_state() + with runtime._accepted_block_payout_preview_condition: + transition = runtime._accepted_block_payout_previews.get(block_hash.lower()) + return transition is not None and transition.landed + + def _accepted_block_payout_transition_for_parent( + self, + parent_hash: str, + *, + parent_height: int | None = None, + ) -> tuple[str, bool] | None: + """Select the highest active exact/ancestor payout transition. + + The boolean result is true for an invalidated landed transition. The + caller decides whether to wait for a live preview or fail closed on its + tombstone. + """ + runtime = self._runtime + runtime._ensure_job_cache_state() + key = parent_hash.lower() + with runtime._accepted_block_payout_preview_condition: + exact_transition = runtime._accepted_block_payout_previews.get(key) + exact_invalidated = ( + key in runtime._invalidated_accepted_block_payout_previews + ) + fail_closed_candidate_hashes = { + candidate_hash + for candidate_hash, transition in ( + runtime._accepted_block_payout_previews.items() + ) + if transition.landed + } + fail_closed_candidate_hashes.update( + runtime._invalidated_accepted_block_payout_previews + ) + ancestor_candidates = [ + (candidate_hash, transition.block_height, False) + for candidate_hash, transition in runtime._accepted_block_payout_previews.items() + if exact_transition is None + and not exact_invalidated + and transition.block_height is not None + and parent_height is not None + and transition.block_height <= parent_height + ] + ancestor_candidates.extend( + ( + candidate_hash, + candidate_height, + True, + ) + for candidate_hash, candidate_height in ( + runtime._invalidated_accepted_block_payout_previews.items() + ) + if exact_transition is None + and not exact_invalidated + and candidate_height is not None + and parent_height is not None + and candidate_height <= parent_height + ) + if exact_transition is not None or exact_invalidated: + return key, exact_invalidated + if not ancestor_candidates: + return None + + active_ancestors: list[tuple[int, str, bool]] = [] + # One probe per distinct height, not per candidate. Every offered or + # replay-adopted block candidate arms a preview barrier, and siblings + # of one solved block all share a height -- so a candidate storm makes + # this loop issue thousands of identical getblockhash calls for a + # single answer, twice per built bundle, on the path every child build + # takes before it can wait for a preview. + # + # Scoped to this scan deliberately, and not memoized across calls: the + # active block at a height changes under reorgs, and a longer-lived + # cache would answer with a chain that no longer exists. Within one + # scan a single view is what the comparison below already assumes. + active_hash_by_height: dict[int, str] = {} + try: + for ( + candidate_hash, + candidate_height, + candidate_invalidated, + ) in ancestor_candidates: + assert candidate_height is not None + active_hash = active_hash_by_height.get(candidate_height) + if active_hash is None: + active_hash = str( + runtime.rpc.call("getblockhash", [candidate_height]) + ).lower() + active_hash_by_height[candidate_height] = active_hash + if active_hash == candidate_hash: + active_ancestors.append( + ( + candidate_height, + candidate_hash, + candidate_invalidated, + ) + ) + except Exception as exc: + runtime._schedule_tip_refresh_retry() + raise TemplateRefreshBlocked( + "could not validate an accepted payout preview on the active chain" + ) from exc + if not active_ancestors: + if any( + candidate_hash in fail_closed_candidate_hashes + for candidate_hash, _candidate_height, _invalidated in ( + ancestor_candidates + ) + ): + # A prepared artifact published with an accepted preview carries + # that prospective balance view. If its block left the active + # chain between reconciliation and this build, falling through + # to the artifact would stamp unrelated work with those balances. + # Wait for withdrawal/reconciliation to publish a new payout + # generation instead of using either the artifact or live DB. + runtime._schedule_tip_refresh_retry() + raise PayoutStatePublicationBlocked( + "accepted payout transition is no longer active" + ) + return None + _, selected_key, selected_invalidated = max(active_ancestors) + return selected_key, selected_invalidated + + def _accepted_parent_unresolved_depth_cap(self) -> int: + runtime = self._runtime + return max( + 1, + int( + getattr( + runtime, + "accepted_parent_unresolved_depth_max", + DEFAULT_ACCEPTED_PARENT_UNRESOLVED_DEPTH_MAX, + ) + ), + ) + + def accepted_parent_unresolved_ages_seconds(self) -> list[float]: + """Ages of landed transitions whose bookkeeping is still unresolved.""" + runtime = self._runtime + runtime._ensure_job_cache_state() + now = time.monotonic() + with runtime._accepted_block_payout_preview_condition: + return [ + max(0.0, now - transition.landed_monotonic) + for transition in runtime._accepted_block_payout_previews.values() + if transition.landed and transition.landed_monotonic is not None + ] + + def _accepted_parent_unresolved_depth(self) -> int: + runtime = self._runtime + runtime._ensure_job_cache_state() + with runtime._accepted_block_payout_preview_condition: + return sum( + 1 + for transition in runtime._accepted_block_payout_previews.values() + if transition.landed + ) + + def _await_pending_parent_payout_preview( + self, + parent_hash: str, + *, + parent_height: int | None = None, + ) -> list[dict[str, object]] | None: + """Wait out a pending accepted-parent transition, returning its preview. + + Returns None when no landed transition governs this parent, without + touching any confirmed payout input. Ordering this wait before every + confirmed read lets children of a pending parent block here instead of + taking (or caching) balances that omit their new parent. + """ + runtime = self._runtime + # Startup-replay window (issue #188 fix 4): before the durable outbox + # has been enumerated, an unarmed pending parent may exist. A child + # built now could pay balances that omit that parent's carry, so job + # builds fail closed until enumeration succeeds. + if runtime._block_replay_enumeration_owed(): + runtime._schedule_tip_refresh_retry() + raise TemplateRefreshBlocked( + "durable block candidate replay has not enumerated pending " + "candidates yet" + ) + selected = runtime._accepted_block_payout_transition_for_parent( + parent_hash, + parent_height=parent_height, + ) + if selected is None: + return None + # Fail closed once too many landed transitions remain unresolved: + # every published preview a child consumes stacks another prospective + # balance chain on unfinished bookkeeping, so the depth of that stack + # must stay bounded even while degraded previews keep jobs flowing. + # Issuance stops at the cap, not past it: a child issued while + # exactly depth_cap transitions are unresolved could land and create + # a (cap + 1)th, so the configured maximum would not actually bound + # the chain. + unresolved_depth = runtime._accepted_parent_unresolved_depth() + depth_cap = runtime._accepted_parent_unresolved_depth_cap() + if unresolved_depth >= depth_cap: + runtime._schedule_tip_refresh_retry() + raise TemplateRefreshBlocked( + "unresolved accepted-parent depth " + f"{unresolved_depth} meets or exceeds cap {depth_cap}" + ) + selected_key, selected_invalidated = selected + if selected_invalidated: + runtime._schedule_tip_refresh_retry() + raise TemplateRefreshBlocked( + "accepted parent payout preview was withdrawn" + ) + + wait_seconds = max( + 0.0, + float( + getattr( + runtime, + "accepted_block_payout_preview_wait_seconds", + DEFAULT_ACCEPTED_BLOCK_PAYOUT_PREVIEW_WAIT_SECONDS, + ) + ), + ) + deadline = time.monotonic() + wait_seconds + timed_out = False + invalidated = False + with runtime._accepted_block_payout_preview_condition: + while selected_key in runtime._accepted_block_payout_previews: + transition = runtime._accepted_block_payout_previews[selected_key] + if transition.preview is not None: + return runtime._materialize_prior_balance_preview( + transition.preview + ) + if runtime.stop_event.is_set(): + raise RuntimeError( + "coordinator stopped while accepted payout preview was pending" + ) + remaining = deadline - time.monotonic() + if remaining <= 0: + timed_out = True + break + runtime._accepted_block_payout_preview_condition.wait( + timeout=min(0.25, remaining) + ) + invalidated = ( + selected_key in runtime._invalidated_accepted_block_payout_previews + ) + if invalidated: + runtime._schedule_tip_refresh_retry() + raise TemplateRefreshBlocked( + "accepted parent payout preview was withdrawn" + ) + if timed_out: + with runtime.lock: + timeout_count = ( + int(getattr(runtime, "_accepted_parent_preview_wait_timeouts", 0)) + + 1 + ) + runtime._accepted_parent_preview_wait_timeouts = timeout_count + runtime._schedule_tip_refresh_retry() + # Typed backpressure, not a failure: the transition is armed and + # un-withdrawn, so the parent's preview is merely late. The typed + # raise keeps this out of the ordinary template-refresh failure + # budget while the counter above (and the coordination-blocked + # streak the delivery boundaries feed) keeps sustained pendency + # visible and still escalating. + raise AcceptedParentPayoutPreviewPending( + "accepted parent payout preview is not ready yet" + f" (parent={selected_key} waited={wait_seconds:.3f}s" + f" timeouts={timeout_count})", + parent_hash=selected_key, + waited_seconds=wait_seconds, + timeout_count=timeout_count, + ) + # The transition reached a terminal durable state while waiting; the + # caller's confirmed fallback now includes the parent. + return None + + def _prior_balances_for_job_parent( + self, + parent_hash: str, + *, + parent_height: int | None = None, + fallback_balances: Sequence[dict[str, object]] | None = None, + ) -> list[dict[str, object]]: + """Return prospective balances, otherwise a prepared/ledger fallback.""" + runtime = self._runtime + preview = runtime._await_pending_parent_payout_preview( + parent_hash, + parent_height=parent_height, + ) + if preview is not None: + return preview + return ( + list(fallback_balances) + if fallback_balances is not None + else runtime.ledger.current_prior_balances() + ) + + def _observe_payout_state_seconds( + self, + name: str, + elapsed_seconds: float, + *, + relation: str | None = None, + ) -> None: + runtime = self._runtime + runtime._ensure_job_cache_state() + with runtime._payout_state_metrics_lock: + if name == "gate_wait": + if relation not in PRISM_PAYOUT_DELIVERY_GENERATIONS: + raise ValueError(f"unknown payout delivery generation: {relation}") + histogram = runtime.payout_gate_wait_histograms[str(relation)] + else: + histogram = runtime.payout_state_histograms[name] + histogram["count"] = int(histogram["count"]) + 1 + histogram["sum"] = float(histogram["sum"]) + elapsed_seconds + buckets = histogram["buckets"] + assert isinstance(buckets, dict) + for bucket in PRISM_PAYOUT_SECONDS_BUCKETS: + if elapsed_seconds <= bucket: + buckets[bucket] = int(buckets.get(bucket, 0)) + 1 + + def _observe_payout_gate_admission( + self, + admission: object, + *, + generation: int, + fallback_wait_seconds: float, + ) -> None: + runtime = self._runtime + runtime._ensure_job_cache_state() + with runtime._job_cache_lock: + published_generation = runtime._payout_state_generation + relation = getattr(admission, "relation", None) + if relation not in PRISM_PAYOUT_DELIVERY_GENERATIONS: + relation = PayoutStateDeliveryGate._generation_relation( + generation, + published_generation, + ) + wait_seconds = float( + getattr(admission, "wait_seconds", fallback_wait_seconds) + ) + runtime._observe_payout_state_seconds( + "gate_wait", + max(0.0, wait_seconds), + relation=relation, + ) + + def _reserve_payout_state_source( + self, + cause: str, + *, + tip_hash: str | None = None, + invalidated_monotonic: float | None = None, + ) -> int: + runtime = self._runtime + runtime._ensure_job_cache_state() + invalidated = ( + time.monotonic() + if invalidated_monotonic is None + else invalidated_monotonic + ) + with runtime.lock: + generation = runtime._payout_state_source[0] + 1 + runtime._payout_state_source = ( + generation, + tip_hash, + cause, + invalidated, + ) + return generation + + def _reserve_payout_state_source_if_current( + self, + expected_source_generation: int, + cause: str, + *, + tip_hash: str | None = None, + invalidated_monotonic: float | None = None, + ) -> tuple[int, int, str | None, str, float] | None: + """Reserve and capture a source only if preparation was not superseded.""" + runtime = self._runtime + + runtime._ensure_job_cache_state() + invalidated = ( + time.monotonic() + if invalidated_monotonic is None + else invalidated_monotonic + ) + # Match publication's lock order so the returned base generation and + # newly reserved source form one atomic candidate identity. + with runtime._job_cache_lock: + with runtime.lock: + if runtime._payout_state_source[0] != expected_source_generation: + return None + source_generation = expected_source_generation + 1 + runtime._payout_state_source = ( + source_generation, + tip_hash, + cause, + invalidated, + ) + # The reserver's entitlement to act on this cause travels + # with the tuple it gets back (issue #228). + return _ReservedPayoutSource( + runtime._payout_state_generation, + source_generation, + tip_hash, + cause, + invalidated, + ) + + def _capture_payout_state_source( + self, + ) -> tuple[int, int, str | None, str, float]: + runtime = self._runtime + runtime._ensure_job_cache_state() + with runtime.lock: + source_generation, source_tip, cause, invalidated = ( + runtime._payout_state_source + ) + with runtime._job_cache_lock: + base_generation = runtime._payout_state_generation + return ( + base_generation, + source_generation, + source_tip, + cause, + invalidated, + ) + + def _consume_payout_source_cause( + self, + captured: tuple[int, int, str | None, str, float], + ) -> bool: + """Whether this preparation may act on the source's reserved cause. + + Two rules, and nothing shared between them that a pruning policy + could reach: + + - The reserver's own preparation presents the ``_ReservedPayoutSource`` + its reservation returned, and acts on the cause exactly once, + whatever other preparations have done at any generation since. + The entitlement lives in that object, so a foreign preparation + at a later generation cannot defeat it; its candidate may be + doomed at the publication check, but its build keeps the + found-block immediacy and degrade of issue #188. + - A plain capture acts only if its source generation is above the + high-water mark of everything acted on so far, and moves the mark + there. At the mark, someone already acted (the mark got there by + acting); below it, the capture is stale. Neither ever rewinds + the mark, so a stale preparation cannot re-enable a newer + generation's cause. + + The reserver's turn also lifts the mark to its generation when that + is newer, so a later capture at the same generation inherits. The + tuple itself is never cleared: candidate supersession and the + finalizer's pending-cause read depend on it staying readable. + """ + runtime = self._runtime + source_generation = int(captured[1]) + with runtime.lock: + high_water = self._payout_source_cause_high_water + newer = high_water is None or source_generation > high_water + if isinstance(captured, _ReservedPayoutSource): + if not captured.consume_entitlement(): + return False + if newer: + self._payout_source_cause_high_water = source_generation + return True + if newer: + self._payout_source_cause_high_water = source_generation + return True + return False + + def _begin_window_build_phases(self) -> dict[str, float]: + phases = {phase: 0.0 for phase in PRISM_PAYOUT_WINDOW_BUILD_PHASES} + self._payout_window_build_context.phases = phases + return phases + + def _finish_window_build_phases(self) -> None: + self._payout_window_build_context.phases = None + + def _note_window_build_phase(self, phase: str, elapsed_seconds: float) -> None: + """Attribute one timed slice of the build in progress on this thread.""" + if phase not in PRISM_PAYOUT_WINDOW_BUILD_PHASES: + raise ValueError(f"unknown payout window build phase: {phase}") + phases = getattr(self._payout_window_build_context, "phases", None) + if phases is not None: + phases[phase] = float(phases.get(phase, 0.0)) + max( + 0.0, float(elapsed_seconds) + ) + + def _observe_window_build_phases( + self, + phases: dict[str, float], + outcome: str, + ) -> None: + if outcome not in PRISM_PAYOUT_WINDOW_BUILD_OUTCOMES: + raise ValueError(f"unknown payout window build outcome: {outcome}") + runtime = self._runtime + with runtime._payout_state_metrics_lock: + for phase in PRISM_PAYOUT_WINDOW_BUILD_PHASES: + elapsed = max(0.0, float(phases.get(phase, 0.0))) + histogram = self.payout_window_build_phase_histograms[ + (phase, outcome) + ] + histogram["count"] = int(histogram["count"]) + 1 + histogram["sum"] = float(histogram["sum"]) + elapsed + buckets = histogram["buckets"] + assert isinstance(buckets, dict) + for bucket in PRISM_PAYOUT_SECONDS_BUCKETS: + if elapsed <= bucket: + buckets[bucket] = int(buckets.get(bucket, 0)) + 1 + + def _cached_tip_refresh_payout_artifact( + self, + *, + base_generation: int, + artifact_payout_state_generation: int, + network_difficulty: int, + ) -> PayoutLedgerArtifact | None: + """Reuse the armed window for a routine candidate when the lock is busy. + + The tip-refresh path's counterpart of the found-block degrade + (issue #188): a routine candidate must not queue behind another + build's oracle while an exact, still-valid armed window exists. + Reuse holds exactly where the delivery path's own reuse holds -- + current payout generation, current append epoch, balances equal + to the published bytes -- narrowed to an anchor no older than + PRISM_TIP_REFRESH_WINDOW_STALENESS_SECONDS and a live difficulty + inside the snapshot-weight band the incremental window already + serves across (in-band ASERT drift keeps the cached weight; the + artifact is tagged with the template difficulty either way). + Balances are the published bytes, which is what a routine build + would reuse itself. Anything else returns None and the caller + queues for a fresh build as before. + """ + runtime = self._runtime + if not runtime._payout_artifact_reuse_active(): + return None + with runtime._job_cache_lock: + artifact = runtime._payout_ledger_artifact + published_artifact = runtime._published_payout_state.artifact + generation_current = base_generation == runtime._payout_state_generation + append_epoch_current = bool( + artifact is not None + and artifact.append_invalidation_epoch + == runtime._payout_ledger_append_invalidation_epoch + ) + if ( + artifact is None + or published_artifact is None + or not generation_current + or not append_epoch_current + or artifact.payout_state_generation != base_generation + or artifact.snapshot_anchor_ms is None + ): + return None + anchor_age_ms = self._now_ms() - int(artifact.snapshot_anchor_ms) + if anchor_age_ms > PRISM_TIP_REFRESH_WINDOW_STALENESS_SECONDS * 1000.0: + return None + weight_per_difficulty = ( + PRISM_REWARD_WINDOW_MULTIPLIER * PRISM_SNAPSHOT_WINDOW_MARGIN + ) + if not _snapshot_window_weight_within_band( + weight_per_difficulty * int(artifact.network_difficulty), + weight_per_difficulty * int(network_difficulty), + ): + return None + balances_sha256 = artifact.prior_balances_sha256 or self._canonical_json_sha256( + artifact.prior_balances + ) + if balances_sha256 != published_artifact.prior_balances_sha256: + return None + reused = dataclass_replace( + artifact, + generation=0, + payout_state_generation=int(artifact_payout_state_generation), + network_difficulty=int(network_difficulty), + prepared_monotonic=time.monotonic(), + window_build_mode="tip_refresh_cached", + window_delta_rows=0, + window_expired_rows=0, + window_touched_pages=0, + window_build_seconds=0.0, + window_full_rescan_reason="prepare_lock_busy", + ) + runtime._record_payout_artifact_event("tip_refresh_cached") + runtime._payout_artifact_log( + "payout_artifact_tip_refresh_cached", + payout_state_generation=int(artifact_payout_state_generation), + duration_seconds=0.0, + window_build_mode="tip_refresh_cached", + window_shares=len(reused.shares_json), + anchor_age_ms=anchor_age_ms, + during_publication=True, + build_interval_bypassed=False, + fallback_reason="prepare_lock_busy", + ) + return reused + + def _prepared_payout_state_candidate( + self, + captured: tuple[int, int, str | None, str, float], + *, + force_full_window_rescan: bool = False, + force_prior_balances_read: bool = False, + bypass_build_interval: bool = False, + ) -> PayoutStateCandidate: + """Prepare a candidate, optionally forcing the window or balance read. + + ``force_full_window_rescan`` keeps its full fail-closed meaning (the + withdrawal path and every explicit caller). ``force_prior_balances_read`` + is the reconciler's intent after a confirmed payout mutation: reuse + the exact armed share window and reread the carry aggregate (#224). + """ + runtime = self._runtime + base_generation, source_generation, source_tip, cause, invalidated = captured + ledger_artifact: PayoutLedgerArtifact | None = None + with runtime._job_cache_lock: + template_artifacts = runtime._template_artifacts + if ( + template_artifacts is not None + and getattr(runtime, "_pool_ready_latched", False) + ): + # A reserved cause is acted on once, by the preparation that + # consumed it. Before this, a withdrawal's cause stayed readable + # in the sticky source tuple until the next tip displaced it, + # and every routine capture in between forced the full oracle + # for a window nothing had changed (issue #228). + cause_honored = self._consume_payout_source_cause(captured) + decisive_cause = cause if cause_honored else None + cause_forces_rescan = ( + decisive_cause == "accepted_block_preview_withdrawn" + ) + force_full = force_full_window_rescan or cause_forces_rescan + # Either flag invalidates the published balances; only the + # fail-closed full rescan touches the share window. The #224 + # intent (``force_prior_balances_read``) is what reconciliation + # asks for after a confirmed mutation, because it moves + # pool-block, payout-entry and carry rows and never the + # append-only share ledger. + payout_mutated = force_full or force_prior_balances_read + caller_flag = force_full_window_rescan or force_prior_balances_read + if caller_flag and cause_forces_rescan: + force_origin = "caller_flag+reserved_cause" + elif caller_flag: + force_origin = "caller_flag" + elif cause_forces_rescan: + force_origin = "reserved_cause" + else: + force_origin = "none" + bypass_interval = ( + bypass_build_interval + or decisive_cause + in { + "accepted_block_preview", + "accepted_block_preview_withdrawn", + } + ) + immediate_preview = ( + decisive_cause == "accepted_block_preview" + or ( + bypass_build_interval + and not force_full_window_rescan + and decisive_cause != "accepted_block_preview_withdrawn" + ) + ) + routine = not payout_mutated and not immediate_preview + self._payout_window_build_context.attribution = ( + _PayoutWindowBuildAttribution( + cause=cause, + cause_inherited=not cause_honored, + force_origin=force_origin, + critical_path=routine, + ) + ) + # Forwarded only when set: the build seam predates the intent + # and per-instance patches of it keep their exact signature. + build_intent: dict[str, bool] = ( + {"force_prior_balances_read": True} + if force_prior_balances_read + else {} + ) + try: + prepare_lock_acquired = True + lock_acquired_here = False + reused_armed_window = False + if immediate_preview or routine: + # A periodic oracle may already own this lock and its + # Postgres query cannot be cancelled safely. Found-block + # publication must not queue behind that 11--36 second + # scan (issue #188): use the last exact, still-valid + # armed window and patch its balances with the verified + # preview below. A routine tip-refresh candidate follows + # the same rule while an armed window inside the + # accepted staleness bound exists (issue #228); without + # one it queues for a fresh build as before. + prepare_lock_acquired = runtime._payout_state_prepare_lock.acquire( + blocking=False + ) + lock_acquired_here = prepare_lock_acquired + if routine and not prepare_lock_acquired: + ledger_artifact = self._cached_tip_refresh_payout_artifact( + base_generation=base_generation, + artifact_payout_state_generation=base_generation + 1, + network_difficulty=template_artifacts.network_difficulty, + ) + reused_armed_window = ledger_artifact is not None + prepare_lock_acquired = not reused_armed_window + if prepare_lock_acquired and not reused_armed_window: + try: + ledger_artifact = runtime._build_payout_ledger_artifact( + base_generation, + base_generation + 1, + template_artifacts.network_difficulty, + force_full, + bypass_interval, + True, + **build_intent, + ) + except self._shutdown_error: + raise + except Exception as exc: + if not immediate_preview: + raise + # Found-block publication exists to decouple child + # job issuance from slow landing bookkeeping (issue + # #188). A window build that dies with the ledger + # must not abort the atomic publication that reopens + # admission: degrade to the cached armed window + # below, exactly like a busy prepare lock. The + # preview itself never depends on this build; it is + # the verified balance state either way. + print( + "prism coordinator: found-block payout window build " + f"failed; degrading to cached armed window: {exc}", + flush=True, + ) + ledger_artifact = None + finally: + if lock_acquired_here: + runtime._payout_state_prepare_lock.release() + elif not reused_armed_window: + ledger_artifact = None + finally: + self._payout_window_build_context.attribution = None + if immediate_preview and ledger_artifact is None: + ledger_artifact = runtime._cached_found_block_payout_artifact( + base_generation=base_generation, + artifact_payout_state_generation=base_generation + 1, + network_difficulty=template_artifacts.network_difficulty, + fallback_reason=( + "build_failed" + if prepare_lock_acquired + else "prepare_lock_busy" + ), + ) + return PayoutStateCandidate( + base_generation=base_generation, + source_generation=source_generation, + source_tip_hash=source_tip, + cause=cause, + invalidated_monotonic=invalidated, + prepared_monotonic=time.monotonic(), + ledger_artifact=ledger_artifact, + ) + + def _cached_found_block_payout_artifact( + self, + *, + base_generation: int, + artifact_payout_state_generation: int, + network_difficulty: int, + fallback_reason: str, + ) -> PayoutLedgerArtifact | None: + """Retag an exact armed window when immediate preparation is unavailable. + + The caller always replaces its balance view with the accepted block's + verified prospective balances before publication. Only the share + window and its declared anchor are reused here. + """ + runtime = self._runtime + + with runtime._job_cache_lock: + artifact = runtime._payout_ledger_artifact + generation_current = base_generation == runtime._payout_state_generation + append_epoch_current = bool( + artifact is not None + and artifact.append_invalidation_epoch + == runtime._payout_ledger_append_invalidation_epoch + ) + anchor_age_ms = ( + None + if artifact is None or artifact.snapshot_anchor_ms is None + else self._now_ms() - int(artifact.snapshot_anchor_ms) + ) + if ( + not generation_current + or not append_epoch_current + or artifact is None + or artifact.payout_state_generation != base_generation + or artifact.network_difficulty != int(network_difficulty) + or anchor_age_ms is None + or anchor_age_ms > runtime._payout_artifact_max_anchor_age_ms() + ): + runtime._record_payout_artifact_event("build_aborted") + runtime._payout_artifact_log( + "payout_artifact_build_aborted", + payout_state_generation=int(artifact_payout_state_generation), + duration_seconds=0.0, + during_publication=True, + build_interval_bypassed=True, + abort_reason=f"{fallback_reason}_without_usable_window", + ) + return None + reused = dataclass_replace( + artifact, + generation=0, + payout_state_generation=int(artifact_payout_state_generation), + prepared_monotonic=time.monotonic(), + window_build_mode="found_block_cached", + window_delta_rows=0, + window_expired_rows=0, + window_touched_pages=0, + window_build_seconds=0.0, + window_full_rescan_reason=fallback_reason, + ) + runtime._record_payout_artifact_event("found_block_cached") + runtime._payout_artifact_log( + "payout_artifact_found_block_cached", + payout_state_generation=int(artifact_payout_state_generation), + duration_seconds=0.0, + window_build_mode="found_block_cached", + window_shares=len(reused.shares_json), + anchor_age_ms=anchor_age_ms, + during_publication=True, + build_interval_bypassed=True, + fallback_reason=fallback_reason, + ) + return reused + + def _current_payout_state_candidate(self) -> PayoutStateCandidate: + runtime = self._runtime + return runtime._prepared_payout_state_candidate( + runtime._capture_payout_state_source() + ) + + def _record_discarded_payout_candidate(self) -> None: + runtime = self._runtime + runtime._ensure_job_cache_state() + with runtime._payout_state_metrics_lock: + runtime.payout_state_candidates_discarded += 1 + + def _block_payout_state_publication( + self, + *, + force: bool = False, + supersede_with: tuple[int, str | None, str, float] | None = None, + ) -> None: + """Atomically close delivery, optionally reserving a newer source.""" + runtime = self._runtime + + runtime._ensure_job_cache_state() + + pending_source: int | None = None + + def mark_blocked() -> bool: + nonlocal pending_source + with runtime._job_cache_lock: + with runtime.lock: + if supersede_with is not None: + ( + expected_source, + fallback_tip, + cause, + invalidated, + ) = supersede_with + current_source, current_tip, _, _ = ( + runtime._payout_state_source + ) + # A newer tip/source wins its identity, but it must be + # superseded so no candidate prepared before an + # uncertain durable commit can publish afterward. + source_tip = ( + fallback_tip + if current_source == expected_source + else current_tip + ) + pending_source = current_source + 1 + runtime._payout_state_source = ( + pending_source, + source_tip, + cause, + invalidated, + ) + else: + pending_source = runtime._payout_state_source[0] + if ( + not force + and supersede_with is None + and pending_source + == runtime._published_payout_state.source_generation + ): + return False + runtime._payout_state_publication_blocked = True + runtime._job_bundle_cache.clear() + return True + + # Close fleet admission atomically with the cache fence. Escaped + # immutable bundles remain stamped with the old generation, but cannot + # cross the boundary while the ledger has newer unpublished state. + if not runtime._payout_state_delivery_gate.block_delivery(mark_blocked): + return + # Construction cancellation is independent of fanout activation. An + # obsolete builder may still be in its ledger or subprocess phase and + # must be preempted before a replacement request is submitted. + runtime._cancel_obsolete_job_builds("payout generation superseded") + with runtime._job_cache_lock: + next_payout_generation = runtime._payout_state_generation + 1 + with runtime.lock: + payout_invalidated_monotonic = runtime._payout_state_source[3] + runtime._record_progress_payout_generation( + next_payout_generation, + payout_invalidated_monotonic, + ) + with runtime.lock: + active = getattr(runtime, "_active_tip_refresh", None) + if ( + active is not None + and active[0].payout_state_generation < next_payout_generation + ): + active[1].cancel() + elif active is not None: + return + assert pending_source is not None + runtime._mark_tip_refresh_pending(next_payout_generation) + runtime._schedule_tip_refresh_retry() + + def _payout_state_publication_fenced(self) -> bool: + """Report whether delivery is still blocked awaiting a publication.""" + runtime = self._runtime + + runtime._ensure_job_cache_state() + with runtime._job_cache_lock: + return runtime._payout_state_publication_blocked + + def _payout_source_requires_publication( + self, + candidate: PayoutStateCandidate | None = None, + ) -> bool: + """Report whether an invalidation source still lacks a publication.""" + runtime = self._runtime + + runtime._ensure_job_cache_state() + with runtime._job_cache_lock: + published_source = runtime._published_payout_state.source_generation + if candidate is not None: + return candidate.source_generation != published_source + with runtime.lock: + return runtime._payout_state_source[0] != published_source + + def _captured_payout_source_requires_publication( + self, + captured: tuple[int, int, str | None, str, float], + ) -> bool: + """Publication check for a captured source, without a candidate. + + Answers exactly what _payout_source_requires_publication would answer + for a candidate prepared from ``captured``. Candidate preparation can + include a multi-second ledger snapshot, so decision sites must be able + to ask this first and prepare only when publication will follow. + """ + runtime = self._runtime + + runtime._ensure_job_cache_state() + with runtime._job_cache_lock: + return captured[1] != runtime._published_payout_state.source_generation + + def _publish_payout_state_candidate( + self, + candidate: PayoutStateCandidate, + ) -> int | None: + """Publish a prepared candidate, or reject it if its source moved.""" + runtime = self._runtime + + runtime._ensure_job_cache_state() + runtime._ensure_tip_refresh_state() + published_generation: int | None = None + schedule_retry = False + active_to_cancel: _FanoutCancellation | None = None + publication_born_expired: tuple[int | None, int] | None = None + publication_installed: tuple[int, int, int] | None = None + publish_started = 0.0 + with runtime._job_cache_lock: + with runtime.lock: + if ( + candidate.source_generation != runtime._payout_state_source[0] + or candidate.base_generation != runtime._payout_state_generation + ): + runtime._record_discarded_payout_candidate() + return None + try: + if ( + candidate.accepted_block_preview is not None + and not candidate.accepted_block_withdrawal + ): + # The preview IS the balance state this generation publishes. + # The durable ledger only catches up after persistence, so a + # ledger read here would cache pre-parent balances against the + # post-parent generation and outlive the transition override. + artifact = runtime._payout_state_artifact_from_balances( + generation=candidate.base_generation + 1, + source_generation=candidate.source_generation, + balances=runtime._materialize_prior_balance_preview( + candidate.accepted_block_preview + ), + ) + elif candidate.ledger_artifact is not None: + # Candidate preparation captured balances under the same + # ledger fence as its share window. Reuse that immutable copy + # instead of issuing the carry-forward aggregate twice during + # every publication. + artifact = runtime._payout_state_artifact_from_balances( + generation=candidate.base_generation + 1, + source_generation=candidate.source_generation, + balances=list(candidate.ledger_artifact.prior_balances), + ) + else: + artifact = runtime._prepare_payout_state_artifact( + generation=candidate.base_generation + 1, + source_generation=candidate.source_generation, + ) + except Exception: + # The delivery/cache fence remains closed. A later reconciliation + # retry can prepare and publish the artifact; no generation lacking + # immutable payout inputs is ever exposed to builders. + runtime._schedule_tip_refresh_retry() + raise + with runtime._job_cache_lock: + with runtime.lock: + if ( + candidate.source_generation != runtime._payout_state_source[0] + or candidate.base_generation != runtime._payout_state_generation + ): + runtime._record_discarded_payout_candidate() + return None + with runtime._payout_state_delivery_gate.publication(): + # publication() has already drained admitted old sends. Start the + # critical-section timer only now; drain latency is delivery wait, + # not time spent holding the atomic payout mutation section. + publish_started = time.monotonic() + with runtime._job_cache_lock: + with runtime.lock: + source_generation = runtime._payout_state_source[0] + if ( + candidate.source_generation == source_generation + and candidate.base_generation + == runtime._payout_state_generation + ): + published_generation = runtime._payout_state_generation + 1 + if candidate.accepted_block_hash is not None: + key = candidate.accepted_block_hash + with runtime._accepted_block_payout_preview_condition: + transition = runtime._accepted_block_payout_previews.get( + key, + AcceptedBlockPayoutTransition( + block_height=candidate.accepted_block_height, + landed=True, + ), + ) + if candidate.accepted_block_withdrawal: + runtime._accepted_block_payout_previews.pop(key, None) + runtime._invalidated_accepted_block_payout_previews[ + key + ] = ( + transition.block_height + if transition.block_height is not None + else candidate.accepted_block_height + ) + else: + existing_preview = transition.preview + if ( + existing_preview is not None + and existing_preview + != candidate.accepted_block_preview + ): + raise RuntimeError( + "accepted block payout preview changed " + "during atomic publication" + ) + runtime._invalidated_accepted_block_payout_previews.pop( + key, + None, + ) + runtime._accepted_block_payout_previews[key] = ( + dataclass_replace( + transition, + landed=True, + preview=candidate.accepted_block_preview, + published_generation=published_generation, + ) + ) + runtime._accepted_block_payout_preview_condition.notify_all() + runtime._payout_state_generation = published_generation + prepared_artifact = candidate.ledger_artifact + candidate_anchor_age_ms = ( + None + if prepared_artifact is None + or prepared_artifact.snapshot_anchor_ms is None + else self._now_ms() + - int(prepared_artifact.snapshot_anchor_ms) + ) + if ( + prepared_artifact is not None + and runtime._payout_artifact_reuse_active() + and prepared_artifact.payout_state_generation + == published_generation + and prepared_artifact.append_invalidation_epoch + == runtime._payout_ledger_append_invalidation_epoch + and candidate_anchor_age_ms is not None + and candidate_anchor_age_ms + <= runtime._payout_artifact_max_anchor_age_ms() + ): + runtime._payout_ledger_artifact_generation += 1 + # Restamp freshness at the install: the candidate + # artifact was built before publication, and the + # delivery-gate drain between the two can outlive + # the reuse budget -- a freshly published payout + # generation must never arm an already-stale + # artifact and force the next builds back through + # the synchronous reward-window walk. + self._replace_payout_ledger_artifact_locked( + dataclass_replace( + prepared_artifact, + generation=runtime._payout_ledger_artifact_generation, + prepared_monotonic=time.monotonic(), + ) + ) + publication_installed = ( + runtime._payout_ledger_artifact_generation, + candidate_anchor_age_ms, + len(prepared_artifact.shares_json), + ) + else: + if ( + prepared_artifact is not None + and runtime._payout_artifact_reuse_active() + and prepared_artifact.payout_state_generation + == published_generation + and prepared_artifact.append_invalidation_epoch + == runtime._payout_ledger_append_invalidation_epoch + ): + # Same admission rule as + # _install_payout_ledger_artifact: candidate + # construction plus the delivery-gate drain + # can push the declared anchor past the + # audit ceiling, and arming such a + # BORN-EXPIRED artifact would fail every + # reuse probe on anchor age -- with the + # re-arm suppressed while an accepted + # preview awaits durability. Discard it; the + # post-publication probe schedules + # preparation (or the durable-confirmation + # path resumes it once the preview lands). + publication_born_expired = ( + candidate_anchor_age_ms, + len(prepared_artifact.shares_json), + ) + self._disarm_payout_ledger_artifact_locked() + runtime._published_payout_state = PublishedPayoutState( + generation=published_generation, + source_generation=candidate.source_generation, + source_tip_hash=candidate.source_tip_hash, + published_monotonic=publish_started, + artifact=artifact, + ) + epoch_tip_hash = runtime._payout_epoch_tip_hash_locked() + if epoch_tip_hash is not None: + runtime._mint_tip_refresh_epoch_locked( + tip_hash=epoch_tip_hash, + payout_state_generation=published_generation, + started_monotonic=candidate.invalidated_monotonic, + ) + # Re-anchor the stamping identity to the minted + # epoch in the same lock hold. A stale identity + # stamps rebuilt bundles at epoch 0, which the + # fence blocks for every previously admitted + # connection until the next wave republishes. + # The publish helper no-ops when the epoch tip + # is not the published snapshot tip; the owning + # wave handles that transition. + published_snapshot = getattr( + runtime, + "tip_template_snapshot", + None, + ) + if published_snapshot is not None: + runtime._publish_tip_refresh_epoch_identity_locked( + published_snapshot + ) + runtime._payout_state_publication_blocked = False + runtime._job_bundle_cache.clear() + runtime._retained_collection_refresh = None + active = getattr(runtime, "_active_tip_refresh", None) + if active is None: + schedule_retry = True + elif active[0].payout_state_generation < published_generation: + # The payout gate itself rejects this old generation. + # Signal its fanout only after atomic publication. + active_to_cancel = active[1] + schedule_retry = True + with runtime._payout_state_metrics_lock: + runtime._payout_first_delivery_pending = ( + published_generation, + candidate.invalidated_monotonic, + ) + if published_generation is not None: + # The mutation owner still blocks every delivery admission, + # so the pointer swap and gate generation remain one atomic + # publication boundary. Do not acquire the gate condition + # while holding coordinator locks: cancellation callbacks take + # those locks after entering the gate wait loop. + runtime._payout_state_delivery_gate.publish_generation( + published_generation, + prioritize_delivery=True, + ) + runtime._observe_payout_state_seconds( + "publish", + max(0.0, time.monotonic() - publish_started), + ) + if published_generation is None: + runtime._record_discarded_payout_candidate() + return None + if publication_born_expired is not None: + born_expired_age_ms, born_expired_shares = publication_born_expired + runtime._record_payout_artifact_event("born_expired") + runtime._payout_artifact_log( + "payout_artifact_born_expired", + payout_state_generation=int(published_generation), + anchor_age_ms=born_expired_age_ms, + window_shares=born_expired_shares, + during_publication=True, + ) + if publication_installed is not None: + # The pointer swap above is an install site that bypasses + # _install_payout_ledger_artifact; record it in the same event + # family or the publication path stays invisible to the + # lifecycle observability this counter exists for. + installed_generation, installed_age_ms, installed_shares = ( + publication_installed + ) + runtime._record_payout_artifact_event("installed") + runtime._payout_artifact_log( + "payout_artifact_installed", + generation=int(installed_generation), + payout_state_generation=int(published_generation), + anchor_age_ms=installed_age_ms, + window_shares=installed_shares, + during_publication=True, + ) + # A publication is a fresh start for speculative preparation: the + # candidate artifact installs through the atomic pointer swap above + # (not _install_payout_ledger_artifact), so whatever re-arm backoff + # pre-publication share traffic accumulated must be released here or + # a share landing right after publication would stay suppressed for + # the whole scaled interval. + with runtime._payout_artifact_executor_lock: + runtime._payout_artifact_rearm_backoff = 1 + runtime._cancel_obsolete_job_bundle_builds( + payout_state_generation=published_generation + ) + runtime._record_progress_payout_generation( + published_generation, + candidate.invalidated_monotonic, + ) + if active_to_cancel is not None: + active_to_cancel.cancel() + runtime._cancel_obsolete_job_builds("payout generation published") + if schedule_retry: + runtime._mark_tip_refresh_pending(published_generation) + runtime._schedule_tip_refresh_retry() + with runtime._job_cache_lock: + current_artifacts = runtime._template_artifacts + published_artifact_usable = ( + runtime._usable_payout_ledger_artifact( + published_generation, + current_artifacts.network_difficulty, + ) + if current_artifacts is not None + else None + ) + accepted_preview_pending_durability = ( + candidate.accepted_block_hash is not None + and not candidate.accepted_block_withdrawal + ) + with runtime.lock: + self_check_owed = self._payout_self_check_owed + if ( + current_artifacts is not None + and (published_artifact_usable is None or self_check_owed) + and not accepted_preview_pending_durability + ): + # A background artifact built before accepted-block confirmation + # would carry the old database balances under the new payout + # generation. Child builders use the compact preview until the + # durable state catches up; artifact preparation resumes then. + # A self-check deferred by this generation's candidate build is + # handed to the same background walker here (issue #228), so + # it runs off the critical path instead of waiting for a + # re-anchor that fresh publications keep postponing. + with runtime.lock: + self._payout_self_check_owed = False + runtime._schedule_payout_ledger_artifact_preparation( + published_generation, + current_artifacts.network_difficulty, + ) + return published_generation + + def _record_first_payout_delivery( + self, + generation: int, + delivered_monotonic: float, + ) -> None: + runtime = self._runtime + runtime._ensure_job_cache_state() + elapsed: float | None = None + with runtime._payout_state_metrics_lock: + pending = runtime._payout_first_delivery_pending + if pending is not None and pending[0] == generation: + elapsed = max(0.0, delivered_monotonic - pending[1]) + runtime._payout_first_delivery_pending = None + if elapsed is not None: + runtime._observe_payout_state_seconds("first_delivery", elapsed) + + def _advance_payout_state_generation(self) -> int: + """Publish a payout-only invalidation with no expensive gate work.""" + runtime = self._runtime + # No production caller remains: direct-block finalization publishes + # through submit_block_candidate's reserve/publish path instead. Tests + # keep using this as the smallest complete reserve -> block -> publish + # invalidation cycle. + runtime._ensure_job_cache_state() + runtime._reserve_payout_state_source("payout_only") + prepared_started = time.monotonic() + with runtime._payout_state_prepare_lock: + # Close build/delivery admission before releasing snapshot readers. + # Publication may then drain already-admitted sends without holding + # the preparation lock needed by later ledger work. + runtime._block_payout_state_publication(force=True) + runtime._observe_payout_state_seconds( + "preparation", + max(0.0, time.monotonic() - prepared_started), + ) + generation = runtime._publish_current_payout_state_with_retry_budget() + if generation is None: + raise TemplateRefreshSuperseded( + "payout-only invalidation was superseded; immediate retry scheduled" + ) + return generation + + def _publish_current_payout_state_with_retry_budget( + self, + *, + initial_attempted: bool = False, + ) -> int | None: + """Publish the current source with a bounded supersession budget.""" + runtime = self._runtime + + max_retries = max( + 0, + int( + getattr( + runtime, + "payout_reconcile_supersession_retries", + DEFAULT_PRISM_PAYOUT_RECONCILE_SUPERSESSION_RETRIES, + ) + ), + ) + attempts = max_retries + (0 if initial_attempted else 1) + for _attempt in range(attempts): + candidate = runtime._current_payout_state_candidate() + published = runtime._publish_payout_state_candidate(candidate) + if published is not None: + return published + runtime._block_payout_state_publication() + return None + + def _payout_delivery( + self, + cancelled: Callable[[], bool], + *, + generation: int, + ) -> Any: + """Use cancellable admission while retaining focused gate test seams.""" + runtime = self._runtime + gate = runtime._payout_state_delivery_gate + delivery_cancelable = getattr(gate, "delivery_cancelable", None) + if callable(delivery_cancelable): + return delivery_cancelable( + cancelled, + generation=generation, + priority=True, + ) + delivery = gate.delivery + try: + return delivery(cancelled) + except TypeError: + return delivery() + + def _expose_inflight_scan_anchor(self, anchor_ms: int) -> int: + """Publish a ledger walk's anchor to the append-side invalidation. + + The entry must stay exposed for as long as the walk's result is + invisible to ``_invalidate_incremental_payout_window_for_append``: + from before the database read (whose snapshot may already exclude a + row committing mid-read) until either the result is discarded or its + window becomes visible through the incremental cache or the armed + artifact's own anchor. Entries older than the audit ceiling are + pruned lazily here: ``_install_payout_ledger_artifact`` rejects any + artifact whose anchor aged past that same ceiling as born-expired, + so a leaked exposure (a built bundle every waiter abandoned) can + never outlive the last install that could have used it. + """ + runtime = self._runtime + runtime._ensure_job_cache_state() + max_age_ms = runtime._payout_artifact_max_anchor_age_ms() + stale_before_ms = self._now_ms() - max_age_ms + with runtime._job_cache_lock: + for token, exposed_anchor_ms in tuple( + runtime._payout_window_inflight_scan_anchors.items() + ): + if exposed_anchor_ms < stale_before_ms: + runtime._payout_window_inflight_scan_anchors.pop(token, None) + runtime._payout_window_inflight_scan_anchor_token += 1 + token = runtime._payout_window_inflight_scan_anchor_token + runtime._payout_window_inflight_scan_anchors[token] = int(anchor_ms) + return token + + def _retire_inflight_scan_anchor(self, token: int | None) -> None: + """Withdraw an exposed walk anchor once its result became visible + (or provably never will). Idempotent; ``None`` is a no-op.""" + runtime = self._runtime + if token is None: + return + runtime._ensure_job_cache_state() + with runtime._job_cache_lock: + runtime._payout_window_inflight_scan_anchors.pop(int(token), None) + + def _fold_published_job_window_anchor_locked( + self, + anchor_ms: int | None, + ) -> None: + """Raise the published-window anchor watermark to ``anchor_ms``. + + Caller holds _job_cache_lock -- the lock every predates() read + takes -- so the raise is atomic with whatever exposure handed the + anchor over: no append can be classified against an anchor set + that has already lost the anchor but not yet gained the watermark. + Raising it early or redundantly is always safe, because a higher + watermark can only classify MORE rows as predating, and that is + the fail-closed direction: an extra bump is forgiven per anchor by + _append_epoch_invalidated_declared_anchor, while a missing one + leaves no stamp for a landing's fences to refuse. ``None`` is a + no-op. + """ + if anchor_ms is None: + return + runtime = self._runtime + runtime._payout_published_job_window_anchor_ms = max( + int(anchor_ms), + runtime._payout_published_job_window_anchor_ms or 0, + ) + + def _publish_seedless_job_window_anchor_locked(self, anchor_ms: int) -> None: + """Keep a seedless published window's anchor visible to appends. + + Caller holds _job_cache_lock, at the bundle publication fence of a + build whose walk exposure retires with no seeded artifact to carry + it (the reuse kill-switch, or nothing to seed). Jobs stamped from + that bundle keep serving the window until they retire, so the + declared anchor must stay in the predates() anchor set past the + exposure: a replay-shaped append that starts and finishes entirely + between publication and a landing's own anchor exposure would + otherwise see no live anchor, skip its epoch bump, and leave no + registry entry for the landing's drain to find. Monotonic and never + retired; the pending-commit floor keeps every ordinary share commit + above it. + """ + self._fold_published_job_window_anchor_locked(anchor_ms) + + def _replace_payout_ledger_artifact_locked( + self, + artifact: PayoutLedgerArtifact | None, + ) -> None: + """Every write to the armed-artifact slot goes through here. + + Caller holds _job_cache_lock. A seeded window's anchor may be + exposed to the append-side predates() checks by nothing but the + armed artifact, while jobs stamped from that window stay landable + after the slot changes: a disarm or replacement retires the cache, + never the jobs. Folding the outgoing artifact's anchor into the + never-retired published-window watermark BEFORE the slot moves -- + under the same lock every predates() read takes, so no append can + observe the anchor gone with the watermark not yet raised -- + preserves the invariant the landing fences assume: the anchor-set + maximum never decreases while any job is landable, so every append + that predates a live or seeded anchor bumps the epoch and leaves a + stamp for the fences to read. Skipping the fold is the disarm gap: + the bump that consults the seeded anchor also disarms the artifact, + emptying the anchor set, and a later row genuinely predating a + still-landable candidate's window then commits with no bump at all + -- nothing for the anchor-scoped predicate to refuse, so the + landing submits an invalidated window. The permanence is what the + per-anchor predicate makes affordable: a retired anchor forces + extra bumps only from replay-shaped rows (the pending-commit floor + keeps ordinary commits above every job anchor), and each is + forgiven per anchor instead of over-abandoning (issue #126). + """ + runtime = self._runtime + outgoing = runtime._payout_ledger_artifact + if outgoing is not None: + self._fold_published_job_window_anchor_locked( + outgoing.snapshot_anchor_ms + ) + runtime._payout_ledger_artifact = artifact + + def _disarm_payout_ledger_artifact_locked(self) -> None: + """Drop the armed artifact, keeping its anchor in the watermark.""" + self._replace_payout_ledger_artifact_locked(None) + + @staticmethod + def _pending_share_predates_anchor( + pending_share: PendingShare, + anchor_ms: int | None, + ) -> bool: + return bool( + anchor_ms is not None + and int(pending_share.job_issued_at_ms) <= anchor_ms + and int(pending_share.accepted_at_ms) <= anchor_ms + ) + + def _pending_share_predates_live_anchor_locked( + self, + pending_share: PendingShare, + ) -> bool: + # Caller holds _job_cache_lock. The incremental pointer is + # immutable and replaced atomically; its owning lock is + # deliberately not acquired here, because an urgent append must + # disarm the separately guarded artifact even while a long + # oracle build owns _payout_state_prepare_lock. Reading it under + # _job_cache_lock orders the read against the in-flight scan + # anchors: a window walk makes its result visible (the + # incremental pointer, or the armed artifact's install fence) + # before retiring its exposed anchor under this lock, so a walk + # that could have missed this row always exposes an anchor to + # the predates checks below. + runtime = self._runtime + predates = runtime._pending_share_predates_anchor + cached = getattr(runtime, "_incremental_payout_artifact_window", None) + artifact = runtime._payout_ledger_artifact + inflight_anchors_ms = tuple( + runtime._payout_window_inflight_scan_anchors.values() + ) + cached_anchor_ms = ( + None + if cached is None + else int(cached.window.anchor_job_issued_at_ms) + ) + artifact_anchor_ms = ( + None + if artifact is None or artifact.snapshot_anchor_ms is None + else int(artifact.snapshot_anchor_ms) + ) + published_job_anchor_ms = getattr( + runtime, "_payout_published_job_window_anchor_ms", None + ) + return bool( + predates(pending_share, cached_anchor_ms) + or predates(pending_share, artifact_anchor_ms) + or predates(pending_share, published_job_anchor_ms) + or any( + predates(pending_share, anchor_ms) + for anchor_ms in inflight_anchors_ms + ) + ) + + @contextmanager + def _landing_fence_for_predating_append( + self, + pending_shares: list[PendingShare], + ) -> Iterator[bool]: + """Hold the landing fence across a durable append of predating rows. + + An epoch bump that only starts after its row is durable leaves a + gap: a landing can acquire the fence, verify the pre-bump epoch, + and enter submitblock after the row committed but before the bump + starts, still landing a coinbase whose window omits the durable + share. Any append whose rows predate a live anchor therefore takes + the fence BEFORE the ledger commit and holds it through the epoch + bump, so the durable append and its invalidation land on the same + side of a landing's fence-guarded epoch-check-and-submit boundary. + Ordinary share commits stay off the lock entirely: the unfenced + peek here fires only for the rare replay-shaped append. Yields + whether the fence is held so the caller can thread it into + _record_late_visible_payout_append. + + The unfenced classification is a one-time predicate, so an anchor + exposed after it (a landing publishing its declared anchor) cannot + retroactively fence this commit. An unfenced batch therefore + registers itself, atomically with the classification, in + _payout_unfenced_append_inflight_stamps and deregisters only after + the caller's post-commit epoch-bump attempt ran inside this block; + _await_unfenced_appends_predating_anchor lets a landing wait those + commits out before its epoch fences arm. + """ + runtime = self._runtime + runtime._ensure_job_cache_state() + unfenced_token: int | None = None + with runtime._job_cache_lock: + fence_needed = any( + runtime._pending_share_predates_live_anchor_locked(pending) + for pending in pending_shares + ) + if not fence_needed and pending_shares: + runtime._payout_unfenced_append_inflight_token += 1 + unfenced_token = runtime._payout_unfenced_append_inflight_token + runtime._payout_unfenced_append_inflight_stamps[ + unfenced_token + ] = min( + max( + int(pending.job_issued_at_ms), + int(pending.accepted_at_ms), + ) + for pending in pending_shares + ) + if not fence_needed: + try: + yield False + finally: + if unfenced_token is not None: + with runtime._payout_unfenced_append_drained: + runtime._payout_unfenced_append_inflight_stamps.pop( + unfenced_token, None + ) + runtime._payout_unfenced_append_drained.notify_all() + return + with runtime._payout_append_landing_fence_lock: + yield True + + def _block_work_wait_slice(self) -> float: + """Slice length for a condition wait taken on the block-work thread. + + The block-candidate owner derives this from the configured watchdog + tolerance, so reusing it keeps every sliced wait on that thread in + step with the watchdog through operator overrides instead of pinning + a second literal that goes stale. Duck-typed runtimes in focused + tests do not carry the helper; they fall back to the same + admission-wait slice the shutdown controller keeps its own copy of. + """ + wait_slice = getattr(self._runtime, "_block_work_wait_slice", None) + if callable(wait_slice): + return max(0.001, float(wait_slice())) + return BLOCK_SUBMITTER_WAIT_HEARTBEAT_SLICE_SECONDS + + def _record_block_work_phase(self, phase: str) -> None: + """Stamp a block-work liveness phase when the runtime records them. + + The recorder itself decides whether the calling thread is a + registered block-work owner and records nothing otherwise, which is + what keeps a share-writer or client thread from refreshing the + budget of an owner that is genuinely wedged. Duck-typed runtimes + without the recorder simply do not stamp. + """ + recorder = getattr(self._runtime, "_record_block_submitter_phase", None) + if callable(recorder): + recorder(phase) + + def _await_unfenced_appends_predating_anchor(self, anchor_ms: int) -> None: + """Wait until no unfenced in-flight append can predate ``anchor_ms``. + + Called by a landing right after exposing its declared anchor. An + append classified as unfenced before that exposure commits outside + the landing fence, so holding the fence across submitblock would + not exclude its durable commit -- and its post-commit epoch bump + would queue behind the very fence the landing holds, arriving only + after the block entered qbitd. Draining here restores the fence + contract: each matching append finishes its commit and its bump + attempt against the now-exposed anchor before this returns, so a + predating row advances the epoch the landing's fences check, and + every append classified after the exposure sees the anchor and + takes the fence itself. Appends whose rows cannot predate the + anchor never block this wait, and the registry holds entries only + for the duration of a ledger commit, so the common path is one + locked emptiness check. Commit-scoped retention is sound because an + append that completed before this drain cannot have predated + ``anchor_ms`` silently: every landable declared anchor is exposed + from publication onward (the armed artifact's anchor -- which the + published-window watermark absorbs before the slot can drop it -- + or that same watermark a seedless build hands its anchor to at + the publication fence), so such an append classified as fenced and + advanced the epoch this landing's fences compare against. + """ + runtime = self._runtime + runtime._ensure_job_cache_state() + anchor = int(anchor_ms) + # The drain is unbounded by design and stays that way: it enforces + # the fencing contract above, so a deadline here could let a landing + # proceed past appends whose epoch bumps its own fences must see. + # What an untimed wait may not do is run heartbeat-silent -- this + # call happens inline on the watchdog-monitored block-work thread, + # where an unbroken silence longer than the tolerance is a hard exit + # mid-landing (issue #125). Waiting in watchdog-sized slices and + # stamping each one that still finds the predicate true keeps the + # wait exactly as long as it was while making it visible; the + # ``while`` re-check is what makes a timed wait semantically + # transparent. The common path -- nothing in flight predates the + # anchor -- never enters the loop and so stamps nothing at all. + slice_seconds = self._block_work_wait_slice() + with runtime._payout_unfenced_append_drained: + while any( + stamp_ms <= anchor + for stamp_ms in ( + runtime._payout_unfenced_append_inflight_stamps.values() + ) + ): + self._record_block_work_phase("wait-unfenced-append-drain") + runtime._payout_unfenced_append_drained.wait( + timeout=slice_seconds + ) + + def _record_late_visible_payout_append( + self, + pending_share: PendingShare, + *, + landing_fence_owned: bool = False, + ) -> int | None: + """Advance the append-invalidation epoch for a newly durable row. + + Two-phase bump. The unfenced peek keeps ordinary share commits off + the landing fence lock entirely; only a row that predates a live + anchor re-checks and bumps under it. Holding that lock for the bump + is what makes the landing's final epoch fence authoritative: a bump + cannot commit while a landing sits between its last epoch read and + submitblock, so the invalidation lands either before that read + (candidate abandoned) or after the RPC returns (the refresh wave + retires the job). The predicate re-runs under the lock because the + anchor set may have moved during the unfenced gap. A caller that + already holds the fence around the durable commit itself + (landing_fence_owned) bumps without re-acquiring. + + Returns the advanced epoch, or None when no live anchor predates + the row. The caller must follow a non-None return with + _retire_payout_windows_for_late_append OUTSIDE the fence: that step + waits on _payout_state_prepare_lock, which a long oracle build can + hold, and the wait must not block landings. + """ + runtime = self._runtime + runtime._ensure_job_cache_state() + + def bump_if_predating_locked() -> int | None: + with runtime._job_cache_lock: + if not runtime._pending_share_predates_live_anchor_locked( + pending_share + ): + return None + runtime._payout_ledger_append_invalidation_epoch += 1 + epoch = int(runtime._payout_ledger_append_invalidation_epoch) + self._record_append_invalidation_stamp_locked( + epoch, pending_share + ) + # The fold inside the disarm is load-bearing here: this bump + # may be consulting the armed artifact's anchor as the last + # live exposure of a seeded window, and dropping it bare + # would let the NEXT predating row commit against an empty + # anchor set -- silently, with no stamp for a still-landable + # candidate's fences to refuse. + self._disarm_payout_ledger_artifact_locked() + return epoch + + if landing_fence_owned: + return bump_if_predating_locked() + with runtime._job_cache_lock: + if not runtime._pending_share_predates_live_anchor_locked( + pending_share + ): + return None + with runtime._payout_append_landing_fence_lock: + return bump_if_predating_locked() + + def _record_append_invalidation_stamp_locked( + self, + epoch: int, + pending_share: PendingShare, + ) -> None: + """Remember which row caused ``epoch``, and prune the oldest. + + Caller holds _job_cache_lock and has just advanced the epoch to + ``epoch``. The stamp recorded is the row's own predating quantity, + so a landing can compare it against its declared anchor with the + same test the append side used against the live anchor set. + + Pruning drops entries from the old end only, and advances the + retained floor by exactly what it dropped. Acceptance therefore + never widens: a baseline that can no longer be answered is refused + by _append_epoch_invalidated_declared_anchor rather than treated as + a bump-free stretch. + """ + stamps = self._payout_append_invalidation_stamps + stamps[epoch] = max( + int(pending_share.job_issued_at_ms), + int(pending_share.accepted_at_ms), + ) + oldest_retained = epoch - PRISM_PAYOUT_APPEND_INVALIDATION_STAMP_HISTORY + 1 + if oldest_retained > self._payout_append_invalidation_oldest_epoch: + for dropped in range( + self._payout_append_invalidation_oldest_epoch, oldest_retained + ): + stamps.pop(dropped, None) + self._payout_append_invalidation_oldest_epoch = oldest_retained + + def _append_epoch_invalidated_declared_anchor( + self, + *, + baseline_epoch: int, + live_epoch: int, + declared_anchor_ms: int | None, + ) -> bool: + """Did an append between two epochs invalidate THIS window? + + The invalidation epoch is one global counter, so a landing that + compares its baseline against the live value learns only that some + window somewhere was invalidated. An append stamped after this + candidate's declared anchor but before a newer live window's anchor + bumps that counter without touching this candidate's payout window, + and on the fallback submit path the bare comparison abandoned the + block before it ever reached qbitd (issue #126). Scoping the + question to the anchor answers what the fence actually needs. + + Fails closed on anything it cannot prove: an unknown declared + anchor is as terminal as it was before, and so is a baseline older + than the retained stamp history or an epoch that somehow moved + backwards. Only a bump whose recorded row provably does not predate + this anchor is forgiven. + + Must not be called while holding _job_cache_lock; it takes that + lock itself. The caller that needs the answer to be authoritative + holds _payout_append_landing_fence_lock across its live-epoch read + and this call, which is what stops a bump from landing in between. + """ + baseline = int(baseline_epoch) + live = int(live_epoch) + if baseline == live: + return False + if live < baseline: + return True + if declared_anchor_ms is None: + return True + anchor = int(declared_anchor_ms) + runtime = self._runtime + runtime._ensure_job_cache_state() + with runtime._job_cache_lock: + if baseline + 1 < self._payout_append_invalidation_oldest_epoch: + return True + stamps = self._payout_append_invalidation_stamps + # Bounded by the retention above: the floor check just proved + # every epoch in this range is retained, so the walk is at most + # PRISM_PAYOUT_APPEND_INVALIDATION_STAMP_HISTORY entries long. + for epoch in range(baseline + 1, live + 1): + stamp = stamps.get(epoch) + if stamp is None or stamp <= anchor: + return True + return False + + def _invalidate_incremental_payout_window_for_append( + self, + pending_share: PendingShare, + ) -> None: + """Disarm payout work that predates a newly visible eligible row. + + Normal submissions hold the pending-share commit floor, so their + accepted timestamp remains above the cached anchor until the row is + durable. Durable replay reconstructs the original timestamps without + that process-local floor. If both stamps are already covered by an + incremental or armed-artifact anchor, the snapshot cannot discover + the late-visible row. Publish the invalidation without waiting for an + in-flight oracle walk, then clear the incremental cache under its own + lock; the epoch prevents pre-append work from re-arming meanwhile. + + This wrapper serves post-commit callers that do not hold the landing + fence; the durable append paths take the fence around the commit via + _landing_fence_for_predating_append and call the bump and retire + steps directly. + """ + runtime = self._runtime + invalidation_epoch = runtime._record_late_visible_payout_append( + pending_share + ) + if invalidation_epoch is None: + return + runtime._retire_payout_windows_for_late_append( + pending_share, invalidation_epoch + ) + + def _retire_payout_windows_for_late_append( + self, + pending_share: PendingShare, + invalidation_epoch: int, + ) -> None: + """Disarm cached payout windows after a late-append epoch bump. + + Runs outside the landing fence. Active jobs stamped against the + pre-append window still pass the generation and digest fences; only + epoch-aware reselection replaces them. Schedule the same refresh wave + a repair publication schedules, and do it before the prepare-lock + wait below so a long oracle build holding that lock cannot delay + superseding the stale jobs. + """ + runtime = self._runtime + + def predates(anchor_ms: int | None) -> bool: + return runtime._pending_share_predates_anchor( + pending_share, anchor_ms + ) + + runtime._mark_tip_refresh_pending(invalidation_epoch) + runtime._schedule_tip_refresh_retry() + + with runtime._payout_state_prepare_lock: + cached = runtime._incremental_payout_artifact_window + if cached is None: + runtime._incremental_payout_artifact_window_invalidation_reason = ( + "late_visible_append" + ) + return + anchor_ms = int(cached.window.anchor_job_issued_at_ms) + if cached.append_invalidation_epoch >= invalidation_epoch: + # A post-invalidation build already replaced the cache. + return + if predates(anchor_ms): + runtime._incremental_payout_artifact_window = None + runtime._incremental_payout_artifact_window_invalidation_reason = ( + "late_visible_append" + ) + else: + # The armed artifact was affected but this older incremental + # anchor was not. Retag it so a later delta build can preserve + # the exact append-only window instead of forcing an oracle. + runtime._incremental_payout_artifact_window = dataclass_replace( + cached, + append_invalidation_epoch=invalidation_epoch, + ) + + @contextmanager + def _payout_balance_serializer_released(self) -> Iterator[None]: + """Temporarily release the balance serializer around candidate audit + construction. + + The caller is the landing, which holds the serializer exactly once + (its single outer acquisition; no caller of + _land_and_confirm_block_candidate holds it). The landed-transition + fence armed before the release keeps reconciliation from mutating + balances while the serializer is free, so job delivery proceeds + during the expensive builder/verifier subprocess work instead of + queueing for its whole duration. + """ + runtime = self._runtime + runtime._payout_balance_mutation_lock.release() + try: + yield + finally: + runtime._acquire_block_submitter_lock( + runtime._payout_balance_mutation_lock, + "payout-balance-mutation", + ) + + def _replayed_payout_window_reproducible( + self, + context: PrismJobContext, + ) -> bool: + """Whether a reconstructed candidate's payout window replays intact. + + Append-invalidation epochs are process-local, so the landing epoch + fence stands down for a candidate rebuilt from durable intent. The + durable ledger is the surviving authority instead: the reward-window + contract keeps the share window replayable at the artifact's declared + anchor, so a share row that became durably visible only after the + recorded window walk -- on either side of a restart -- appears in the + replayed window while the recorded coinbase omits it. Only omissions + fail the candidate: an appended row leaves the tip, height, and carry + balances untouched, which is exactly why no other landing fence can + see it, while a recorded row absent from the replay is not this + hazard (share rows are append-only, and drift from settled payout + state is governed by the reorg and prior-balance fences). + """ + runtime = self._runtime + found_block = getattr(context, "found_block", None) + anchor_ms = ( + found_block.get("anchor_job_issued_at_ms") + if isinstance(found_block, dict) + else None + ) + network_difficulty = ( + found_block.get("network_difficulty") + if isinstance(found_block, dict) + else None + ) + audit_share_window = getattr(runtime.ledger, "audit_share_window", None) + window_covers = getattr(runtime.ledger, "candidate_window_covers", None) + if ( + anchor_ms is None + or network_difficulty is None + or not (callable(window_covers) or callable(audit_share_window)) + ): + # Fail closed: a candidate whose window cannot be replayed at a + # declared anchor cannot prove its coinbase pays the window the + # durable ledger requires. + return False + try: + if callable(window_covers): + return window_covers( + context.shares_json, + anchor_job_issued_at_ms=int(anchor_ms), + network_difficulty=int(network_difficulty), + ) + # Compatibility for in-memory ledgers and older test adapters. + # The PostgreSQL path returns only a boolean, never durable rows. + durable_rows = audit_share_window( + anchor_job_issued_at_ms=int(anchor_ms), + network_difficulty=int(network_difficulty), + ) + return disk_window_covers(recorded_share_ids(context.shares_json), durable_rows) + except DaemonWindowMirrorDivergence: + # Fail closed, exactly like a candidate whose window cannot be + # replayed at all: a recorded window the coordinator can no + # longer read cannot prove the coinbase pays it. + self._note_window_mirror_divergence() + return False + + def normalized_prior_balances(self, balances: list[dict[str, object]]) -> list[dict[str, object]]: + rows = [ + { + "recipient_id": str(balance.get("recipient_id", "")), + "order_key": str(balance.get("order_key", "")), + "p2mr_program_hex": str(balance.get("p2mr_program_hex", "")), + "balance_sats": int(balance.get("balance_sats", 0)), + } + for balance in balances + ] + rows.sort( + key=lambda row: ( + row["order_key"], + row["recipient_id"], + row["p2mr_program_hex"], + row["balance_sats"], + ) + ) + return rows + + @staticmethod + def settlement_balances_by_program( + balances: list[dict[str, object]], + ) -> dict[str, int]: + """Aggregate owed balances by P2MR program for settlement equality. + + A miner may re-authorize the same payout address under any valid + spelling (bech32 permits an all-uppercase form), which changes the + recipient/order-key identity strings while committing to the same + payout program. Ledger and preview views can then label one program + with different identity strings while agreeing on every settled + amount, so settlement equality must be judged on the program and + amount alone. Identity strings stay display metadata. Zero totals + are dropped to match both the ledger reader (nonzero HAVING filter) + and the preview builder (zero balances skipped). + """ + aggregated: dict[str, int] = {} + for balance in balances: + program = str(balance.get("p2mr_program_hex", "")).lower() + aggregated[program] = aggregated.get(program, 0) + int( + balance.get("balance_sats", 0) + ) + return {program: total for program, total in aggregated.items() if total != 0} + + def prior_balances_match_current(self, prior_balances: list[dict[str, object]]) -> bool: + runtime = self._runtime + return runtime.settlement_balances_by_program( + prior_balances + ) == runtime.settlement_balances_by_program(runtime.ledger.current_prior_balances()) + + def payout_state_metrics_lines(self) -> list[str]: + runtime = self._runtime + runtime._ensure_job_cache_state() + with runtime._payout_state_metrics_lock: + state_histograms = { + name: { + "buckets": dict(histogram["buckets"]), + "sum": float(histogram["sum"]), + "count": int(histogram["count"]), + } + for name, histogram in runtime.payout_state_histograms.items() + } + gate_histograms = { + relation: { + "buckets": dict(histogram["buckets"]), + "sum": float(histogram["sum"]), + "count": int(histogram["count"]), + } + for relation, histogram in runtime.payout_gate_wait_histograms.items() + } + phase_histograms = { + key: { + "buckets": dict(histogram["buckets"]), + "sum": float(histogram["sum"]), + "count": int(histogram["count"]), + } + for key, histogram in self.payout_window_build_phase_histograms.items() + } + discarded = runtime.payout_state_candidates_discarded + + metric_names = { + "preparation": "qbit_prism_payout_preparation_seconds", + "publish": "qbit_prism_payout_publish_seconds", + "first_delivery": "qbit_prism_payout_invalidation_first_delivery_seconds", + } + descriptions = { + "preparation": "Payout reconciliation and candidate preparation outside delivery publication.", + "publish": "Atomic payout generation/cache publication gate-hold time.", + "first_delivery": "Payout invalidation to first delivery of the published generation.", + } + lines: list[str] = [] + for name, metric_name in metric_names.items(): + histogram = state_histograms[name] + buckets = histogram["buckets"] + assert isinstance(buckets, dict) + lines.extend( + [ + f"# HELP {metric_name} {descriptions[name]}", + f"# TYPE {metric_name} histogram", + *[ + f'{metric_name}_bucket{{le="{bucket:g}"}} {int(buckets.get(bucket, 0))}' + for bucket in PRISM_PAYOUT_SECONDS_BUCKETS + ], + f'{metric_name}_bucket{{le="+Inf"}} {histogram["count"]}', + f'{metric_name}_sum {float(histogram["sum"]):.6f}', + f'{metric_name}_count {histogram["count"]}', + ] + ) + + gate_name = "qbit_prism_payout_gate_wait_seconds" + lines.extend( + [ + "# HELP qbit_prism_payout_gate_wait_seconds Delivery admission wait by generation relationship to the published payout state.", + "# TYPE qbit_prism_payout_gate_wait_seconds histogram", + ] + ) + for relation in PRISM_PAYOUT_DELIVERY_GENERATIONS: + histogram = gate_histograms[relation] + buckets = histogram["buckets"] + assert isinstance(buckets, dict) + lines.extend( + [ + *[ + f'{gate_name}_bucket{{generation="{relation}",le="{bucket:g}"}} {int(buckets.get(bucket, 0))}' + for bucket in PRISM_PAYOUT_SECONDS_BUCKETS + ], + f'{gate_name}_bucket{{generation="{relation}",le="+Inf"}} {histogram["count"]}', + f'{gate_name}_sum{{generation="{relation}"}} {float(histogram["sum"]):.6f}', + f'{gate_name}_count{{generation="{relation}"}} {histogram["count"]}', + ] + ) + phase_name = "qbit_prism_payout_window_build_phase_seconds" + lines.extend( + [ + f"# HELP {phase_name} Payout-window build time by phase (PostgreSQL reads, in-process record conversion, Rust daemon prepare or upload, preparation-lock waits) and by how the build ended, observed on every exit so aborted and failed builds keep the time they measured. Both label sets are closed.", + f"# TYPE {phase_name} histogram", + ] + ) + for phase in PRISM_PAYOUT_WINDOW_BUILD_PHASES: + for outcome in PRISM_PAYOUT_WINDOW_BUILD_OUTCOMES: + histogram = phase_histograms[(phase, outcome)] + buckets = histogram["buckets"] + assert isinstance(buckets, dict) + labels = f'phase="{phase}",outcome="{outcome}"' + lines.extend( + [ + *[ + f'{phase_name}_bucket{{{labels},le="{bucket:g}"}} {int(buckets.get(bucket, 0))}' + for bucket in PRISM_PAYOUT_SECONDS_BUCKETS + ], + f'{phase_name}_bucket{{{labels},le="+Inf"}} {histogram["count"]}', + f'{phase_name}_sum{{{labels}}} {float(histogram["sum"]):.6f}', + f'{phase_name}_count{{{labels}}} {histogram["count"]}', + ] + ) + lines.extend( + [ + "# HELP qbit_prism_payout_candidates_discarded_total Prepared payout candidates discarded after source supersession.", + "# TYPE qbit_prism_payout_candidates_discarded_total counter", + f"qbit_prism_payout_candidates_discarded_total {discarded}", + ] + ) + return lines + + +__all__ = [ + "AcceptedBlockPayoutTransition", + "AcceptedParentPayoutPreviewPending", + "DEFAULT_ACCEPTED_BLOCK_PAYOUT_PREVIEW_WAIT_SECONDS", + "DEFAULT_PRISM_PAYOUT_RECONCILE_SUPERSESSION_RETRIES", + "PRISM_PAYOUT_ARTIFACT_REARM_BACKOFF_CAP", + "PRISM_PAYOUT_DELIVERY_GENERATIONS", + "PRISM_PAYOUT_SECONDS_BUCKETS", + "PRISM_PAYOUT_WINDOW_BUILD_OUTCOMES", + "PRISM_PAYOUT_WINDOW_BUILD_PHASES", + "PRISM_TIP_REFRESH_WINDOW_STALENESS_SECONDS", + "PayoutDeliveryAdmission", + "PayoutLedgerArtifact", + "PayoutStateArtifact", + "PayoutStateCandidate", + "PayoutStateDeliveryGate", + "PayoutStatePublicationBlocked", + "PayoutStateRuntime", + "PayoutStateService", + "PayoutStateSnapshot", + "PublishedPayoutState", + "TemplateRefreshBlocked", + "TemplateRefreshSuperseded", + "_IncrementalPayoutArtifactWindow", + "_PayoutWindowMaterialization", + "canonical_json_sha256", + "canonical_json_text", +] diff --git a/lab/prism/prism_coordinator.py b/lab/prism/prism_coordinator.py index 7a51cb215..fee9b8fa7 100644 --- a/lab/prism/prism_coordinator.py +++ b/lab/prism/prism_coordinator.py @@ -3,50 +3,33 @@ from __future__ import annotations -import base64 import copy from collections import OrderedDict from concurrent.futures import ( - FIRST_COMPLETED, - CancelledError as FuturesCancelledError, Future, - InvalidStateError, ThreadPoolExecutor, - wait, ) from contextlib import ExitStack, contextmanager -import dataclasses -import errno import faulthandler -from functools import wraps import hashlib -import heapq -import http.client -import inspect import json -import math import os import queue -import random +import random # noqa: F401 - patch seam for tip-refresh holdoff jitter tests import shlex import signal import socket -import struct import subprocess import tempfile import threading import time import traceback -import urllib.parse -import urllib.request import uuid -import weakref -from types import SimpleNamespace -from dataclasses import dataclass, field, replace as dataclass_replace -from decimal import Context, Decimal, InvalidOperation, ROUND_CEILING, getcontext, localcontext -from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer +from dataclasses import replace as dataclass_replace +from decimal import Decimal, ROUND_CEILING +from http.server import BaseHTTPRequestHandler from pathlib import Path -from typing import Any, Callable, Iterator, Sequence +from typing import Any, Callable, Iterator, Mapping, Sequence import sys @@ -54,264 +37,389 @@ sys.path.insert(0, str(Path(__file__).resolve().parents[2])) from lab.auxpow import stratum_codec, vardiff -from lab.prism import direct_stratum, public_api +from lab.prism import direct_stratum +# Compatibility re-exports; new callers should import lab.prism.bounded_executor. +from lab.prism.bounded_executor import ( + _BoundedPriorityExecutor, # noqa: F401 - compatibility re-export + _DeliveryQueueFull, # noqa: F401 - compatibility re-export +) from lab.prism.prism_tools import prism_tool_command +# Compatibility re-exports; new callers should import lab.prism.background_services. +from lab.prism.writer_lease_timing import ( + WRITER_LEASE_HEARTBEAT_SCHEDULER_SLACK_SECONDS, +) +from lab.prism.background_services import ( + BackgroundServiceRegistry, # noqa: F401 - compatibility re-export + BackgroundServiceSpec, # noqa: F401 - compatibility re-export + LedgerLeaseHeartbeatPorts, + LedgerLeaseHeartbeatService, + WatchdogPorts, + WatchdogService, + _LeaseHeartbeatStateField, + guard_session_prover as _lease_guard_session_prover, + guard_session_verifier as _lease_guard_session_verifier, +) +# Compatibility re-exports; new callers should import lab.prism.coordinator_shutdown. +from lab.prism.coordinator_shutdown import ( + CoordinatorShutdownController, # noqa: F401 - compatibility re-export + ShutdownInProgress, # noqa: F401 - compatibility re-export + ledger_writer_operation, # noqa: F401 - compatibility re-export +) +from lab.prism.coordinator_config import ( + BLOCK_LANDING_DB_TIMEOUT_WATCHDOG_FRACTION, # noqa: F401 + CoordinatorConfig, + apply_python_switch_interval, + DEFAULT_ACCEPTED_PARENT_UNRESOLVED_DEPTH_MAX, # noqa: F401 + DEFAULT_BLOCK_LANDING_DB_TIMEOUT_MAX_SECONDS, # noqa: F401 + DEFAULT_BLOCK_LANDING_DB_TIMEOUT_SECONDS, # noqa: F401 + DEFAULT_BLOCK_SUBMIT_LOCK_WAIT_LOG_SECONDS, # noqa: F401 + DEFAULT_BLOCK_SUBMIT_RPC_TIMEOUT_SECONDS, + DEFAULT_BLOCK_SUBMIT_STUCK_CALL_EXIT_SECONDS, # noqa: F401 + DEFAULT_CTV_FANOUT_FEE_PREMIUM_BPS, + DEFAULT_DIRECT_COINBASE_PAYOUT_FLOOR_SATS, + DEFAULT_MAX_COINBASE_SETTLEMENT_OUTPUTS, + DEFAULT_MAX_CTV_FANOUT_RECIPIENTS_PER_TRANSACTION, + DEFAULT_MAX_DIRECT_COINBASE_OUTPUTS, + DEFAULT_PRISM_BUNDLE_BUILD_TIMEOUT_SECONDS, + DEFAULT_PRISM_COORDINATION_BLOCKED_EXIT_SECONDS, + DEFAULT_PRISM_CTV_BROADCASTER_CHUNK_SIZE, + DEFAULT_PRISM_DISCONNECTED_JOB_RETENTION, # noqa: F401 + DEFAULT_PRISM_HEALTH_PENDING_REFRESH_MAX_AGE_SECONDS, + DEFAULT_PRISM_HEALTH_REFRESH_SECONDS, # noqa: F401 - compatibility re-export + DEFAULT_PRISM_HEALTH_TIP_POLL_MAX_AGE_SECONDS, + DEFAULT_PRISM_INITIAL_JOB_MAX_WORKERS, + DEFAULT_PRISM_JOB_BUILD_CANCEL_GRACE_SECONDS, + DEFAULT_PRISM_JOB_BUILD_EXECUTOR_WORKERS, + DEFAULT_PRISM_JOB_BUILD_TIMEOUT_SECONDS, + DEFAULT_PRISM_JOB_BUNDLE_CACHE_SECONDS, + DEFAULT_PRISM_LEDGER_LEASE_EXTERNAL_FENCE_TIMEOUT_SECONDS, + DEFAULT_PRISM_LEDGER_LEASE_HEARTBEAT_EXIT_TIMEOUT_SECONDS, + DEFAULT_PRISM_LEDGER_LEASE_HEARTBEAT_FAILURE_SECONDS, + DEFAULT_PRISM_LEDGER_LEASE_HEARTBEAT_MONITOR_SECONDS, + DEFAULT_PRISM_LEDGER_LEASE_HEARTBEAT_SECONDS, + DEFAULT_PRISM_METRICS_REFRESH_SECONDS, # noqa: F401 - compatibility re-export + DEFAULT_PRISM_MINING_HEALTH_STARTUP_GRACE_SECONDS, # noqa: F401 + DEFAULT_PRISM_OBSERVED_TIP_ACCEPT_WINDOW_SECONDS, # noqa: F401 + DEFAULT_PRISM_PAYOUT_ARTIFACT_FULL_RESCAN_SECONDS, # noqa: F401 + DEFAULT_PRISM_PAYOUT_ARTIFACT_MAX_ANCHOR_AGE_SECONDS, # noqa: F401 + DEFAULT_PRISM_PAYOUT_ARTIFACT_MIN_BUILD_INTERVAL_SECONDS, # noqa: F401 + DEFAULT_PRISM_PAYOUT_ARTIFACT_REANCHOR_SECONDS, # noqa: F401 + DEFAULT_PRISM_PAYOUT_ARTIFACT_REARM_MIN_SECONDS, # noqa: F401 + DEFAULT_PRISM_REORG_RECONCILE_CACHE_SECONDS, + DEFAULT_PRISM_ROUTINE_ADMISSION_DEADLINE_SECONDS, # noqa: F401 + DEFAULT_PRISM_STALE_GRACE_SECONDS, + DEFAULT_PRISM_STRATUM_ACCEPT_RESOURCE_EXHAUSTION_BACKOFF_SECONDS, # noqa: F401 + DEFAULT_PRISM_STRATUM_BIND_RETRY_SECONDS, # noqa: F401 + DEFAULT_PRISM_STRATUM_INITIAL_JOB_TIMEOUT_SECONDS, + DEFAULT_PRISM_STRATUM_LISTEN_BACKLOG, # noqa: F401 + DEFAULT_PRISM_STRATUM_MAX_CONNECTIONS, + DEFAULT_PRISM_STRATUM_MAX_CONNECTIONS_PER_USERNAME, # noqa: F401 + DEFAULT_PRISM_STRATUM_MAX_PENDING_INITIAL_JOBS, + DEFAULT_PRISM_STRATUM_SEND_TIMEOUT_SECONDS, + DEFAULT_PRISM_TEMPLATE_MAX_AGE_SECONDS, + DEFAULT_PRISM_TIP_REFRESH_FAILURE_HOLDOFF_SECONDS, # noqa: F401 + DEFAULT_PRISM_TIP_REFRESH_MAX_WORKERS, + DEFAULT_PRISM_WATCHDOG_LEASE_RELEASE_TIMEOUT_SECONDS, + DEFAULT_PRISM_WATCHDOG_TIMEOUT_SECONDS, # noqa: F401 + DEFAULT_PRISM_WORKER_METRICS_LIMIT, + DEFAULT_PRISM_WRITER_QUIESCENCE_TIMEOUT_SECONDS, + LEASE_AUTHORITY_MARGIN_HEADROOM_SECONDS, + DEFAULT_SHARE_COMMIT_BATCH_SIZE, # noqa: F401 - compatibility re-export + DEFAULT_SHARE_COMMIT_LINGER_MILLISECONDS, # noqa: F401 - compatibility re-export + DEFAULT_SHARE_COMMIT_TIMEOUT_SECONDS, # noqa: F401 - compatibility re-export + StratumListenerProfile, # noqa: F401 - compatibility re-export + TESTNET_QBIT_CHAINS, # noqa: F401 - compatibility re-export + VALID_COINBASE_OUTPUT_POLICIES, + default_prism_payout_policy, + default_prism_username_fallback_address, # noqa: F401 - compatibility re-export + env, + env_bool, + env_heap_census_config, + env_int, + env_nonnegative_float, + env_nonnegative_int, + env_optional, + env_optional_positive_int, + env_optional_positive_int_with_legacy, + env_positive_float, + env_positive_int, + env_positive_int_with_legacy, + load_coordinator_config, + load_share_weights, + resolve_initial_job_max_workers, # noqa: F401 - compatibility re-export + validate_hex, + validate_initial_job_max_workers, # noqa: F401 - compatibility re-export + validate_job_build_executor_workers, # noqa: F401 - compatibility re-export + validate_payout_artifact_age_bounds, # noqa: F401 - compatibility re-export +) from lab.prism.ctv_broadcaster import CtvFanoutBroadcaster from lab.prism.ctv_broadcaster_daemon import ( CtvFanoutBroadcastDaemon, CtvFanoutChunkResult, CtvFanoutDaemonResult, - MAX_CTV_FANOUT_BROADCASTER_CHUNK_SIZE, ) +# Compatibility re-exports; new callers should import lab.prism.ctv_runtime. +from lab.prism.ctv_runtime import ( + CtvRuntimeConfig, + CtvRuntimeService, +) +# Compatibility re-exports; new callers should import the owning J1 modules. +from lab.prism.audit_artifacts import ( + AuditArtifactConfig, + AuditArtifactStore, + AuditPublicationIdentity, +) +from lab.prism.audit_http import ( + AuditHttpConfig, + AuditHttpFacade, + AuditHttpPort, +) +from lab.prism.observability import ( + METRICS_STATE_FRESH, + METRICS_STATE_UNAVAILABLE, + MetricsSnapshotResponse, + MiningDeliveryInputs, + ObservabilityPort, + ObservabilityService, +) +from lab.prism.block_candidates import ( + BLOCK_SUBMITTER_WAIT_HEARTBEAT_SLICE_SECONDS, + BlockCandidateAttemptResult, # noqa: F401 - compatibility re-export + BlockCandidateCompatibilityField, + BlockCandidatePorts, + BlockCandidateRunResult, # noqa: F401 - compatibility re-export + BlockCandidateService, + BlockCandidateStateField, + BlockSubmitterDatabaseTimeout, # noqa: F401 - compatibility re-export + DEFAULT_BLOCK_ACCOUNTING_QUEUE_DEPTH, # noqa: F401 - compatibility re-export + DEFAULT_BLOCK_CANDIDATE_RETRY_INITIAL_SECONDS, + DEFAULT_BLOCK_CANDIDATE_RETRY_MAX_SECONDS, + MAX_BLOCK_SUBMITTER_STUCK_CALL_WORKERS, # noqa: F401 - compatibility re-export + MAX_PENDING_BLOCK_CANDIDATES, + PRISM_REJECTION_BLOCK_ACCEPT_PENDING, + PRISM_REJECTION_LEDGER_CONFIRMATION_SUPERSEDED, + PRISM_STALE_JOB_ABANDON_CLASSES, + PrismBlockCandidate, + _BlockCandidateAccountingTask, # noqa: F401 - compatibility re-export + _BlockCandidateDispositionFlight, # noqa: F401 - compatibility re-export + _BlockCandidateDispositionLease, # noqa: F401 - compatibility re-export + _BlockCandidateNodeSubmission, + _BlockSubmitterLedgerCall, # noqa: F401 - compatibility re-export + _BlockSubmitterRpcCall, # noqa: F401 - compatibility re-export + _STATE_FIELD_MAP as _BLOCK_CANDIDATE_STATE_FIELD_MAP, + block_candidate_from_intent as decode_block_candidate_intent, + block_candidate_intent as encode_block_candidate_intent, + compatibility_default as candidate_compatibility_default, +) +from lab.prism.block_finalization import ( + PRISM_REJECTION_BLOCK_STALE, + PRISM_REJECTION_CANDIDATE_AUDIT_MISMATCH, + PRISM_REJECTION_LEDGER_CONFIRMATION_FAILED, + PRISM_REJECTION_SUBMITBLOCK_REJECTED, + BlockFinalizationService, +) +from lab.prism.accepted_preview_telemetry import ( + ensure_accepted_preview_telemetry, +) +from lab.prism.metrics import MetricsRenderer +from lab.prism.reorg_reconciler import ( + DEFAULT_PRISM_RECONCILE_FLIGHT_WAIT_SECONDS, + PRISM_RECONCILE_PREFETCH_JOIN_TIMEOUT_SECONDS, + ReorgCompatibilityField, + ReorgPorts, + ReorgReconcilerService, + qbit_chain_view_untrusted as reorg_chain_view_untrusted, +) +from lab.prism.vardiff_service import ( + PRISM_VARDIFF_IDLE_SECONDS_BUCKETS, + PRISM_VARDIFF_IDLE_SKIP_REASONS, + VARDIFF_COMPATIBILITY_FIELDS, + IdleRetargetRequest as _IdleRetargetRequest, + VardiffCompatibilityField, + VardiffService, +) +from lab.prism.share_submission import ( + PRISM_REJECTION_BACKEND_RPC_UNAVAILABLE, + PRISM_REJECTION_DUPLICATE_SHARE, + PRISM_REJECTION_INVALID_EXTRANONCE, + PRISM_REJECTION_INVALID_NTIME_OR_NONCE, + PRISM_REJECTION_LOW_DIFFICULTY, + PRISM_REJECTION_MALFORMED_SUBMIT, + PRISM_REJECTION_POOL_CLOSED, + PRISM_REJECTION_STALE_JOB, + PRISM_REJECTION_UNAUTHORIZED_WORKER, + PRISM_REJECTION_UNKNOWN_JOB, + BlockSolvesDroppedCompatibilityField, + RecentShareCompatibilityField, + RecentShareIndex, + ShareSubmissionPorts, + ShareSubmissionService, + SubmitControlSnapshot, + empty_share_ack_histograms, +) +from lab.prism.bundle_compiler import ( + BundleCompiler, + PRISM_BUILDER_PHASE_METRICS_PREFIX, # noqa: F401 - compatibility re-export + PRISM_SERVE_BUILDER_PROTOCOL_VERSION, # noqa: F401 - compatibility re-export + PRISM_SERVE_BUILDER_WINDOW_CACHE_ENTRIES, # noqa: F401 - compatibility re-export + PRISM_SPOOL_SPLICE_CHUNK_BYTES, # noqa: F401 - compatibility re-export + _ServeBuilderClient, # noqa: F401 - compatibility re-export + _ServeBuilderUnavailable, # noqa: F401 - compatibility re-export + _ShareWindowSerialization, # noqa: F401 - compatibility re-export + _compact_share_payload, # noqa: F401 - compatibility re-export + _share_window_spool_file, # noqa: F401 - compatibility re-export + canonical_bundle_bytes, +) +from lab.prism.job_bundle import ( + CachedJobBundle, + JobBuildAdmissionDeadlineExceeded, # noqa: F401 - compatibility re-export + PRISM_JOB_BUILD_SECONDS_BUCKETS, + JobBuildCancellation as _JobBuildCancellation, + JobBuildCancelled, + JobBuildFlight as _JobBuildFlight, + JobBuildRequest as _JobBuildRequest, + JobBuildSuperseded, + JobBundleBuildControl as _JobBundleBuildControl, + JobBundleBuildSuperseded as _JobBundleBuildSuperseded, + JobBundleService, +) +from lab.prism.job_delivery import ( + EvictedJobEntry, + JobBuildFailed as _JobBuildFailed, + JobDeliveryService, + PRISM_CREDIT_POLICY_STALE_GRACE, + PRISM_DELIVERY_PRIORITY_INITIAL, + PRISM_DELIVERY_PRIORITY_NEW_TIP, + PRISM_DELIVERY_PRIORITY_SAME_TIP, + PRISM_EVICTED_JOB_CAPACITY_SCOPES, + PRISM_EVICTED_JOB_CLASSES, + PRISM_EVICTED_JOB_SUBMIT_OUTCOMES, + PendingInitialJob, + PrismJobContext, +) +from lab.prism.payout_state import ( + PayoutStatePublicationBlocked, + TemplateRefreshBlocked, + TemplateRefreshSuperseded, +) +from lab.prism.template_artifacts import ( + CachedTemplateArtifacts, + QbitTipTemplateSnapshot, + TemplateArtifactRepository, # noqa: F401 - compatibility re-export +) +from lab.prism.tip_refresh import ( + FanoutCancellation as _FanoutCancellation, + PRISM_TIP_REFRESH_BUILD_PHASES, # noqa: F401 - compatibility re-export + PRISM_TIP_REFRESH_COVERAGE_TARGETS, # noqa: F401 - compatibility re-export + PRISM_TIP_REFRESH_FAILURE_HOLDOFF_JITTER_FRACTION, # noqa: F401 - compatibility re-export + PRISM_TIP_REFRESH_REENTRY_BACKOFF_SECONDS, # noqa: F401 - compatibility re-export + PRISM_TIP_REFRESH_SECONDS_BUCKETS, # noqa: F401 - compatibility re-export + PRISM_TIP_REFRESH_WAVE_OUTCOMES, # noqa: F401 - compatibility re-export + PRISM_TIP_REFRESH_WAVE_PASS_BUDGET, # noqa: F401 - compatibility re-export + RefreshResult, + RetainedCollectionRefresh, + TipRefreshService, + TipRefreshValidationToken, + _TipRefreshEpochCoverage, # noqa: F401 - compatibility re-export + _TipRefreshFanoutSuperseded, # noqa: F401 - compatibility re-export + _TipRefreshTrustBlocked, # noqa: F401 - compatibility re-export +) +# Compatibility re-exports; new callers should import lab.prism.payout_state. +from lab.prism.payout_state import ( + DEFAULT_ACCEPTED_BLOCK_PAYOUT_PREVIEW_WAIT_SECONDS, # noqa: F401 - compatibility re-export + DEFAULT_PRISM_PAYOUT_RECONCILE_SUPERSESSION_RETRIES, + PRISM_PAYOUT_ARTIFACT_REARM_BACKOFF_CAP, # noqa: F401 - compatibility re-export + PRISM_PAYOUT_DELIVERY_GENERATIONS, # noqa: F401 - compatibility re-export + PayoutLedgerArtifact, + PayoutStateArtifact, + PayoutStateCandidate, + PayoutStateService, + PayoutStateSnapshot, # noqa: F401 - compatibility re-export + _IncrementalPayoutArtifactWindow, # noqa: F401 - compatibility re-export + _PayoutWindowMaterialization, +) +from lab.prism.process_telemetry import install_heap_census +# Compatibility re-exports; new callers should import lab.prism.share_writer. +from lab.prism.share_writer import ( + MAX_PENDING_SHARE_APPENDS, + PENDING_SHARE_COMMIT_WARN_SECONDS as PRISM_PENDING_SHARE_COMMIT_WARN_SECONDS, # noqa: F401 + PendingShareAppend, + ShareWriter, + ShareWriterCompatibilityField, + ShareWriterError, # noqa: F401 - compatibility re-export + ShareWriterQueueFull, # noqa: F401 - compatibility re-export +) +# Compatibility re-exports; new callers should import lab.prism.stratum_session. +from lab.prism.stratum_session import ( + ClientState, + P2mrAddressValidator, + SessionRegistry, # noqa: F401 - compatibility re-export + StratumError, + StratumSessionService, + WorkerIdentity, + _P2mrAddressValidationFlight, # noqa: F401 - compatibility re-export + apply_stratum_send_timeout as apply_socket_send_timeout, + client_vardiff_lock, + difficulty_payload as stratum_difficulty_payload, + error_payload as stratum_error_payload, + job_payload as stratum_job_payload, + result_payload as stratum_result_payload, + stratum_accept_heartbeat_names as configured_accept_heartbeat_names, +) +# Compatibility re-exports; new callers should import lab.prism.progress_health. +from lab.prism.progress_health import ( + EligibilitySnapshot, # noqa: F401 - compatibility re-export + ProgressHealthConfig, + ProgressHealthService, + ProgressHealthSnapshot, + WorkGeneration, +) +from lab.prism.rpc import ( + DEFAULT_QBIT_RPC_CALL_TIMEOUT_SECONDS, + JsonRpc, + _QBIT_RPC_NO_TRANSPORT_RETRY_METHODS, # noqa: F401 - compatibility re-export +) +from lab.prism.candidate_store import IncompatibleCandidateSchema from lab.prism.share_ledger import ( DEFAULT_AUDIT_SHARE_SEGMENT_SIZE, DEFAULT_CTV_BROADCAST_ATTEMPT_DETAIL_LIMIT, DEFAULT_CTV_BROADCAST_RETRY_BACKOFF_SECONDS, - DEFAULT_WRITER_LEASE_ADOPTION_SILENCE_SECONDS, - IncrementalShareWindow, - IncrementalShareJsonSequence, - IncrementalWindowAdvanceStats, - IncrementalWindowFallback, - LedgerOperationTimeout, + DEFAULT_LEASE_ACQUIRE_ATTEMPTS, + DEFAULT_LEASE_ACQUIRE_LOCK_TIMEOUT_SECONDS, + DEFAULT_POSTGRES_IDLE_IN_TRANSACTION_TIMEOUT_SECONDS, + DEFAULT_POSTGRES_TCP_KEEPALIVES_COUNT, + DEFAULT_POSTGRES_TCP_KEEPALIVES_IDLE_SECONDS, + DEFAULT_POSTGRES_TCP_KEEPALIVES_INTERVAL_SECONDS, + DEFAULT_WRITER_LEASE_ADOPTION_SILENCE_SECONDS, # noqa: F401 - compatibility re-export + DaemonWindowMirrorDivergence, PendingShare, PsqlShareLedger, SingleWriterShareLedger, WRITER_LEASE_HEARTBEAT_SESSION_PREFIX, - WRITER_LEASE_VERIFICATION_MAX_STATEMENTS, - WriterLeaseRenewalDeferred, - sha256_json_hex, + WRITER_LEASE_VERIFICATION_MAX_STATEMENTS, # noqa: F401 - compatibility re-export + WriterLeaseRenewalDeferred, # noqa: F401 - compatibility re-export ) -DEFAULT_P2MR_SPEND_INPUT_BYTES = 3_680 -DEFAULT_MIN_OUTPUT_FEERATE_SATS_PER_BYTE = 1 -DEFAULT_MIN_OUTPUT_SAFETY_MULTIPLIER = 4 -DEFAULT_TESTNET_USERNAME_FALLBACK_ADDRESS = ( - "tq1zlsq9dpxz8mennhdpr9nf9s0f2tjtq6gxs9m84k6xglhkfp92q2zszzu4m3" -) -DEFAULT_PRISM_COINBASE_TAG = "/PRISM/" -MAX_PRISM_COINBASE_TAG_BYTES = 40 -DEFAULT_DIRECT_COINBASE_PAYOUT_FLOOR_SATS = 10_485_760 -DEFAULT_MAX_COINBASE_SETTLEMENT_OUTPUTS = 16 -DEFAULT_MAX_DIRECT_COINBASE_OUTPUTS = 12 -DEFAULT_MAX_CTV_FANOUT_RECIPIENTS_PER_TRANSACTION = 1_000 -DEFAULT_CTV_FANOUT_FEE_PREMIUM_BPS = 12_000 -TESTNET_QBIT_CHAINS = {"testnet", "testnet3", "testnet4", "signet"} -DEFAULT_PRISM_BLOCKPOLL_SECONDS = 2.0 -DEFAULT_PRISM_BLOCKWAIT_TIMEOUT_SECONDS = 5.0 -DEFAULT_PRISM_TIP_REFRESH_FAILURE_HOLDOFF_SECONDS = 1.0 -# Fraction of the holdoff added as random jitter so coordinators sharing one -# qbitd do not phase-lock their blocked-refresh re-attempts. -PRISM_TIP_REFRESH_FAILURE_HOLDOFF_JITTER_FRACTION = 0.25 -DEFAULT_PRISM_JOB_BUNDLE_CACHE_SECONDS = 10.0 -DEFAULT_PRISM_BUNDLE_BUILD_TIMEOUT_SECONDS = 60.0 -# Minimum spacing between speculative payout-ledger-artifact rebuilds armed by -# a failed reuse fence. Every fence failure wants a fresh artifact; the floor -# prevents repeated delta/carry reads and also paces any cold/full fallback. -DEFAULT_PRISM_PAYOUT_ARTIFACT_REARM_MIN_SECONDS = 5.0 -# Consecutive aborted speculative rebuilds double the re-arm interval up to -# this multiplier (80s at the default floor). Rebuilds abort on generation -# supersession, snapshot errors, empty windows, or a pathologically old -# pending-commit floor; backing off stops database preparation from cycling -- -# and from holding the payout preparation lock tip builds also need -- -# while the underlying condition persists. Any successful install or -# event-driven preparation resets the backoff. -PRISM_PAYOUT_ARTIFACT_REARM_BACKOFF_CAP = 16 -# How long an armed artifact keeps serving NEW reuse decisions, measured -# from its install (prepared_monotonic), never from its anchor: the reward -# window walk itself takes multiple seconds at production share volume, so -# a bound measured from the wall-clock anchor let a slow build arm an -# artifact that was already expired on arrival -- the born-expired rebuild -# livelock behind the 2026-07-29 rollback of the first anchor-scoped -# deploy. Freshness starts when the window becomes reusable. -DEFAULT_PRISM_PAYOUT_ARTIFACT_REANCHOR_SECONDS = 60.0 -# Normal generations may reuse the last materialized window until this floor -# elapses. Found-block settlement bypasses the floor; the default follows the -# existing re-anchor cadence so one knob continues to describe the expected -# freshness rhythm when the new setting is absent. -DEFAULT_PRISM_PAYOUT_ARTIFACT_MIN_BUILD_INTERVAL_SECONDS = 60.0 -# A full oracle pass periodically checks the append-only delta assumption. -# Reorg/carry mutations and invariant failures force this path immediately. -DEFAULT_PRISM_PAYOUT_ARTIFACT_FULL_RESCAN_SECONDS = 3_600.0 - - -def validate_payout_artifact_age_bounds( - max_anchor_age_seconds: float, - refresh_floor_seconds: float, -) -> None: - """Refuse an anchor ceiling the background re-anchor cannot beat. - - A ceiling at or below the re-anchor floor plus the debounce and the - multi-second window walk turns every reuse probe into a ceiling - rejection while the rebuild runs debounced in parallel -- mechanically - the 2026-07-30 walk-per-build brownout. The 2x floor requirement keeps - comfortable margin over that sum at any plausible walk duration. - """ - if max_anchor_age_seconds <= 2.0 * refresh_floor_seconds: - raise SystemExit( - "PRISM_PAYOUT_ARTIFACT_MAX_ANCHOR_AGE_SECONDS must exceed " - "2x the payout-artifact refresh floor" - ) -# Audit-facing ceiling on how far a served window's declared anchor may -# trail the live ledger, in wall-clock terms. Deliberately much looser than -# the reuse-staleness knob: shares stamped after an anchor deterministically -# belong to the next window (never lost; bundles declare the artifact's own -# anchor, so reused windows stay audit-reproducible), and the ceiling only -# caps worst-case payout-preview lag under pathological build latency or a -# wedged pending-commit floor. An artifact whose anchor is already past the -# ceiling at install time is born expired: discarded, and counted as a -# preparation failure for re-arm backoff pacing. -DEFAULT_PRISM_PAYOUT_ARTIFACT_MAX_ANCHOR_AGE_SECONDS = 300.0 -# Bounds one routine/initial admission wait behind publication-priority -# builds. A parked request past the deadline fails fast with a visible log -# line and counter instead of silently riding the client's own much longer -# timeout; the initial-job runner retries with its normal pacing. -DEFAULT_PRISM_ROUTINE_ADMISSION_DEADLINE_SECONDS = 15.0 -# Upper bound for one kernel transfer from the share-window spool file into -# the audit builder's stdin pipe. The kernel clamps each call to the free -# pipe capacity anyway; the bound only paces cancellation checkpoints. -PRISM_SPOOL_SPLICE_CHUNK_BYTES = 1 << 20 MAX_PRISM_JOB_BUNDLE_CACHE_ENTRIES = 128 -DEFAULT_PRISM_CTV_BROADCASTER_CHUNK_SIZE = 5 -DEFAULT_PRISM_REORG_RECONCILE_CACHE_SECONDS = 5.0 -# Trusted reconcile outcomes are memoized per tip so an untrusted outcome for -# one tip cannot unarm the cache for every other tip. The map only ever -# answers for the current best hash; a small bound merely caps growth while -# stale entries age out of the TTL. -PRISM_REORG_RECONCILE_MEMO_MAX_TIPS = 8 -# Same-tip reconcile callers coalesce behind one in-flight pass. The wait is -# a liveness backstop, not a pacing knob: a follower that outwaits it simply -# runs its own serialized pass, exactly as every caller did before flights. -DEFAULT_PRISM_RECONCILE_FLIGHT_WAIT_SECONDS = 30.0 -# Reconcile demand observability: which caller lane asked, and what satisfied -# it. The tip-refresh lane can be satisfied by the per-tip trusted memo, by a -# pass overlapped with the template fetch on the prefetch worker, or by a -# serial pass on the refresh thread; per-client job builds only ever see a -# memo hit or their own serial pass. -PRISM_REORG_RECONCILE_LOOKUP_PATHS = ("tip_refresh", "job_build") -PRISM_REORG_RECONCILE_LOOKUP_SOURCES = ("memo_hit", "overlap", "serial") -# Read-to-ack latency labels for mining.submit. Accepted shares wait for the -# group commit before their ack; rejected shares (measured when the reject -# decision is made) skip it, so the pair separates commit pressure from -# thread-scheduling/GIL pressure as connection count grows. -PRISM_SHARE_ACK_RESULTS = ("accepted", "rejected") -DEFAULT_PRISM_HEALTH_REFRESH_SECONDS = 5.0 -DEFAULT_PRISM_HEALTH_PENDING_REFRESH_MAX_AGE_SECONDS = 15.0 -DEFAULT_PRISM_HEALTH_TIP_POLL_MAX_AGE_SECONDS = 15.0 -DEFAULT_PRISM_STRATUM_SEND_TIMEOUT_SECONDS = 20.0 -DEFAULT_PRISM_STRATUM_MAX_CONNECTIONS = 384 -DEFAULT_PRISM_STRATUM_MAX_CONNECTIONS_PER_USERNAME = 0 -DEFAULT_PRISM_STRATUM_MAX_PENDING_INITIAL_JOBS = 128 -DEFAULT_PRISM_STRATUM_INITIAL_JOB_TIMEOUT_SECONDS = 30.0 -DEFAULT_PRISM_MINING_HEALTH_STARTUP_GRACE_SECONDS = 30.0 -DEFAULT_PRISM_STRATUM_ACCEPT_RESOURCE_EXHAUSTION_BACKOFF_SECONDS = 1.0 -# Kernel accept backlog per stratum listener. Sized so a whole fleet -# reconnecting inside a restart window parks in the backlog instead of being -# SYN-dropped; the kernel caps the effective value at net.core.somaxconn. -DEFAULT_PRISM_STRATUM_LISTEN_BACKLOG = 1024 -# How long bind() retries EADDRINUSE at startup, covering overlap with a -# predecessor process that is still draining its shutdown while holding the -# port. Zero fails fast (the historical behavior). -DEFAULT_PRISM_STRATUM_BIND_RETRY_SECONDS = 10.0 -DEFAULT_PRISM_PAYOUT_ADDRESS_CACHE_MAX_ENTRIES = 4_096 -DEFAULT_PRISM_PAYOUT_ADDRESS_CACHE_TTL_SECONDS = 3_600.0 -DEFAULT_PRISM_STALE_GRACE_SECONDS = 3.0 -# How old the refresh-published tip may be before mining.submit stops trusting -# it and falls back to a live getbestblockhash per share. Healthy coordinators -# republish/reconfirm it every blockpoll interval; a bounded divergence lease -# separately covers replacement construction without failing open forever. -DEFAULT_PRISM_SUBMIT_TIP_MAX_AGE_SECONDS = 10.0 -DEFAULT_PRISM_SAME_TIP_JOB_RETENTION_SECONDS = 30.0 -DEFAULT_PRISM_SAME_TIP_JOB_RETENTION_PER_CONNECTION = 64 -DEFAULT_PRISM_EVICTED_JOB_PRUNE_INTERVAL_SECONDS = 1.0 -# How many same-tip retained contexts may outlive their connection so a -# reconnecting client (same authorized username) can submit work computed -# against the prior connection's jobs instead of eating unknown-job rejects -# across a proxy flap. Entries keep the normal same-tip TTL; 0 restores the -# historical purge-on-disconnect behavior. -DEFAULT_PRISM_DISCONNECTED_JOB_RETENTION = 16_384 -DEFAULT_PRISM_TIP_REFRESH_MAX_WORKERS = 16 -DEFAULT_PRISM_INITIAL_JOB_MAX_WORKERS = 4 -PRISM_TIP_REFRESH_ADMISSION_POLL_SECONDS = 0.05 -PRISM_TIP_REFRESH_REENTRY_BACKOFF_SECONDS = 0.05 -PRISM_TIP_REFRESH_WAVE_PASS_BUDGET = 64 -# The overlapped reconcile pass runs ledger reads that can crawl while the -# chain churns. The tip-refresh join must never park the poll loop on it -# past the liveness budget: a bounded join leaves the pass running in its -# single slot (the paced retry re-joins the same future) and keeps the poll -# loop's heartbeat live while the pass catches up. -PRISM_RECONCILE_PREFETCH_JOIN_TIMEOUT_SECONDS = 20.0 -# Ceiling for operator overrides of the join budget: a value at or above the -# poll loop's failure/watchdog budgets would silently reinstate the very park -# the bound exists to prevent. -PRISM_RECONCILE_PREFETCH_JOIN_TIMEOUT_CEILING_SECONDS = 60.0 +# The reorg reconcile memo/flight/prefetch constants and machinery moved to +# lab.prism.reorg_reconciler; the two runtime-override defaults the port +# wiring reads are re-imported above. +# Read-to-ack latency labels for mining.submit now live with the +# share-submission owner (PRISM_SHARE_ACK_RESULTS is re-exported above). # Cancellation-check slice while an initial request rides a subscribed # publication-priority build promise. Promise completion wakes the waiter # immediately; this only bounds how stale a cancellation can go unnoticed. PRISM_INITIAL_JOB_SUBSCRIBE_POLL_SECONDS = 0.25 -DEFAULT_PRISM_JOB_BUILD_TIMEOUT_SECONDS = 60.0 -DEFAULT_PRISM_JOB_BUILD_CANCEL_GRACE_SECONDS = 0.25 -DEFAULT_PRISM_JOB_BUILD_EXECUTOR_WORKERS = 2 # A flight whose executor future is finished normally leaves its slot inside # the future's done callback. The admission sweep only treats such a flight # as orphaned after this grace, so a callback that is merely between future # completion and slot cleanup is never mistaken for a dead one. PRISM_JOB_BUILD_ORPHAN_SWEEP_GRACE_SECONDS = 1.0 -DEFAULT_PRISM_VARDIFF_IDLE_SWEEP_SECONDS = 15.0 -PRISM_VARDIFF_IDLE_RETARGET_MAX_WORKERS = 2 -MAX_PENDING_VARDIFF_IDLE_RETARGETS = 8 -DEFAULT_PRISM_WORKER_METRICS_LIMIT = 100 -MAX_ACTIVE_PRISM_JOBS_PER_CLIENT = 16 -# Block candidates queue to a dedicated submitter thread so the miner's share -# ack never waits on audit/submitblock after the share and intent commit. The -# bound limits RAM; overflow only coalesces a wakeup because Postgres retains -# the authoritative pending candidate. -MAX_PENDING_BLOCK_CANDIDATES = 32 -# Startup replay must observe every durable pending candidate before child -# job builds unblock (issue #188 fix 4): a truncated enumeration could hide -# the active parent whose payout transition is not yet armed. A full batch -# therefore re-queries with a doubled window until the result is provably -# untruncated. This cap bounds one enumeration pass's memory; at the cap the -# gate simply stays closed while the queued batch drains, and the submitter -# loop re-enumerates the shrinking remainder. -MAX_BLOCK_REPLAY_ENUMERATION_ROWS = 1024 +# The vardiff idle worker/pending limits moved to lab.prism.vardiff_service; +# the bucket/skip registries are re-imported above for metric consumers. # Accepted shares use a small, bounded group-commit queue. Every submitter # waits for its batch's Postgres commit before receiving Stratum success, so # this is a latency-smoothing bound rather than a durable backlog. -MAX_PENDING_SHARE_APPENDS = 4_096 -# A pending share commit normally clears the snapshot anchor floor within one -# group-commit linger. Holding it longer than this is a wedged writer or a -# leaked release path and is logged loudly (once per share). -PRISM_PENDING_SHARE_COMMIT_WARN_SECONDS = 30.0 -DEFAULT_SHARE_COMMIT_BATCH_SIZE = 64 -DEFAULT_SHARE_COMMIT_LINGER_MILLISECONDS = 5.0 -DEFAULT_SHARE_COMMIT_TIMEOUT_SECONDS = 15.0 -DEFAULT_PRISM_WRITER_QUIESCENCE_TIMEOUT_SECONDS = 15.0 -# A watchdog release must never turn a wedged DB path into a second outage. -# The release worker is daemonized and the watchdog hard-exits at this total -# wall-clock deadline whether quiescence or the fresh DB connection completes. -DEFAULT_PRISM_WATCHDOG_LEASE_RELEASE_TIMEOUT_SECONDS = 5.0 -# A fast-adoptable coordinator proves its guarded session live four times -# inside the one-second silence proof required by PsqlShareLedger. This runs -# through a dedicated connection, not the ordinary ledger lock or shared -# native pool, and must never wait on the lease tuple's row lock: fenced -# writes hold it for whole transactions (persist_accepted_block can exceed -# the guard's statement timeout many times over). The lease TTL is renewed -# only when that tuple is uncontended (SKIP LOCKED), so an idle coordinator -# still keeps lease_expires_at ahead of different-identity expiry claims. -DEFAULT_PRISM_LEDGER_LEASE_HEARTBEAT_SECONDS = ( - DEFAULT_WRITER_LEASE_ADOPTION_SILENCE_SECONDS / 4 -) -# The monitor kills a coordinator before PostgreSQL can observe the full -# adoption silence if the heartbeat query itself wedges. Its short release -# budget keeps the old process gone before a successor is allowed to CAS. -DEFAULT_PRISM_LEDGER_LEASE_HEARTBEAT_FAILURE_SECONDS = ( - DEFAULT_WRITER_LEASE_ADOPTION_SILENCE_SECONDS * 0.75 -) -DEFAULT_PRISM_LEDGER_LEASE_HEARTBEAT_MONITOR_SECONDS = 0.05 -DEFAULT_PRISM_LEDGER_LEASE_HEARTBEAT_EXIT_TIMEOUT_SECONDS = 0.1 -DEFAULT_PRISM_LEDGER_LEASE_EXTERNAL_FENCE_TIMEOUT_SECONDS = 0.5 -DEFAULT_PRISM_TEMPLATE_MAX_AGE_SECONDS = 120 -DEFAULT_PRISM_COORDINATION_BLOCKED_EXIT_SECONDS = 900.0 # The reward window is 8x network difficulty (must match PRISM_WINDOW_MULTIPLIER # in crates/qbit-prism/src/lib.rs and the SQL). The job-build snapshot only needs # the shares that window can cover; requesting a margin above it returns a @@ -328,143 +436,13 @@ def validate_payout_artifact_age_bounds( # reaches miners; real connections stamp their own extranonce1 into the job. # Client extranonce1 values start at 1, so the placeholder never collides. PRISM_JOB_EXTRANONCE1_PLACEHOLDER_HEX = "00000000" -PRISM_JOB_BUILD_SECONDS_BUCKETS = ( - 0.01, - 0.025, - 0.05, - 0.1, - 0.25, - 0.5, - 1.0, - 2.5, - 5.0, - 10.0, - 30.0, -) -PRISM_CTV_BROADCASTER_SECONDS_BUCKETS = ( - 1.0, - 5.0, - 10.0, - 30.0, - 60.0, - 120.0, - 300.0, - 600.0, -) -PRISM_CTV_BROADCASTER_CHUNK_SECONDS_BUCKETS = PRISM_JOB_BUILD_SECONDS_BUCKETS -PRISM_CTV_BROADCASTER_CHUNK_ROWS_BUCKETS = (1, 2, 5, 10, 25, 50, 100) -PRISM_TIP_REFRESH_SECONDS_BUCKETS = PRISM_JOB_BUILD_SECONDS_BUCKETS -PRISM_TIP_REFRESH_BUILD_PHASES = ( - "reorg_reconcile", - "ledger_snapshot", - "payout_state_derivation", - "ctv_manifest_construction", - "coinbase_bundle_construction", - "signing_verification", - "serialization_copy", - "singleflight_wait", -) -PRISM_BUILDER_PHASE_METRICS_PREFIX = "qbit-prism-build-phase-metrics " -PRISM_VARDIFF_IDLE_SECONDS_BUCKETS = PRISM_JOB_BUILD_SECONDS_BUCKETS -PRISM_VARDIFF_IDLE_SKIP_REASONS = ( - "busy", - "disconnected", - "not_idle", - "cache_miss", - "queue_full", - "superseded", -) -PRISM_PAYOUT_DELIVERY_GENERATIONS = ("current", "stale", "future") -DEFAULT_PRISM_PAYOUT_RECONCILE_SUPERSESSION_RETRIES = 8 -PRISM_JOB_BUILD_PHASES = ( - "reorg", - "template", - "merkle", - "ledger", - "payout_artifact", - "payout", - "ctv", - "input_serialization", - "worker", - "output_serialization", - "assembly", - "bundle", - "preparation_wait", - "executor_queue", - "client_lock", - "payout_gate", - "stamp", - "socket_send", - "send", -) -PRISM_JOB_CACHE_KINDS = ("template", "bundle") -PRISM_TIP_REFRESH_RESULTS = ("sent", "skipped", "disconnected", "failed") -PRISM_TIP_REFRESH_CANCELLATION_STAGES = ( - "executor_queue", - "client_lock", - "payout_gate", -) -PRISM_TIP_REFRESH_WAVE_OUTCOMES = ( - "completed", - "fanout_superseded", - "build_superseded", - "payout_blocked", - "trust_blocked", - "shutdown", - "error", -) -PRISM_TIP_REFRESH_COVERAGE_TARGETS = ( - ("50", Decimal("0.50")), - ("95", Decimal("0.95")), - ("99", Decimal("0.99")), -) -PRISM_STALE_JOB_ABANDON_CLASSES = ( - "tip_moved", - "balance_stale", - "append_epoch_stale", -) -# Never-served clients outrank the fleet's tip-refresh wave: a client with no -# active job is producing nothing until its first notify, while a client with -# stale-tip work keeps mining (stale-grace credits it) for the wave's duration. -PRISM_DELIVERY_PRIORITY_INITIAL = 0 -PRISM_DELIVERY_PRIORITY_NEW_TIP = 1 -PRISM_DELIVERY_PRIORITY_SAME_TIP = 2 -PRISM_PROGRESS_HEALTH_REASONS = ( - "tip_poll_stale", - "refresh_pending_too_long", - "current_generation_not_published", - "current_generation_not_delivered", - "bundle_build_stuck", -) -PRISM_EVICTED_JOB_CLASSES = ("same_tip", "stale_grace") -PRISM_EVICTED_JOB_SUBMIT_OUTCOMES = ( - "accepted_same_tip", - "credited_stale_grace", - "accepted_same_tip_cross_connection", -) -PRISM_EVICTED_JOB_CAPACITY_SCOPES = ("connection", "disconnected") -PRISM_REJECTION_STALE_JOB = "stale-job" -PRISM_REJECTION_DUPLICATE_SHARE = "duplicate-share" -PRISM_REJECTION_LOW_DIFFICULTY = "low-difficulty" -PRISM_REJECTION_MALFORMED_SUBMIT = "malformed-submit" -PRISM_REJECTION_UNAUTHORIZED_WORKER = "unauthorized-worker" -PRISM_REJECTION_UNKNOWN_JOB = "unknown-job" -PRISM_REJECTION_INVALID_EXTRANONCE = "invalid-extranonce" -PRISM_REJECTION_INVALID_NTIME_OR_NONCE = "invalid-ntime-or-nonce" -PRISM_REJECTION_CANDIDATE_AUDIT_MISMATCH = "candidate-audit-mismatch" -PRISM_REJECTION_SUBMITBLOCK_REJECTED = "submitblock-rejected" -PRISM_REJECTION_BACKEND_RPC_UNAVAILABLE = "backend-rpc-unavailable" +# The job-build seconds buckets, phase registry, and cache-kind registry are +# canonical in lab.prism.job_bundle; the metrics renderer imports them there. +# The ten submit-path rejection reasons moved to lab.prism.share_submission +# and the four accepted-block finalization reasons moved to +# lab.prism.block_finalization; both groups are re-exported above. Only the +# process-internal reason stays here. PRISM_REJECTION_INTERNAL_ERROR = "internal-error" -PRISM_REJECTION_POOL_CLOSED = "pool-closed" -PRISM_REJECTION_BLOCK_STALE = "block-stale" -PRISM_REJECTION_LEDGER_CONFIRMATION_FAILED = "ledger-confirmation-failed" -# A would-be terminal abandonment refused because the candidate's own block -# hash is (or was recently observed as) part of the active chain: qbitd -# accepted the block even though this process has not completed the accepted -# success tail (for example when the direct submitblock ack was lost in -# transport and acceptance was learned from a blockwait tip observation). -# The candidate defers and retries until the tail finalizes it as submitted. -PRISM_REJECTION_BLOCK_ACCEPT_PENDING = "accepted-pending-finalization" PRISM_RETRYABLE_BLOCK_CANDIDATE_REASONS = frozenset( { PRISM_REJECTION_BACKEND_RPC_UNAVAILABLE, @@ -474,88 +452,11 @@ def validate_payout_artifact_age_bounds( PRISM_REJECTION_BLOCK_ACCEPT_PENDING, } ) -DEFAULT_BLOCK_CANDIDATE_RETRY_INITIAL_SECONDS = 0.25 -DEFAULT_BLOCK_CANDIDATE_RETRY_MAX_SECONDS = 30.0 -# The primary accounting handoff queue must be bounded or the documented -# result-preserving spillover ordering can never engage; the overflow queue -# stays unbounded by design so an already-offered block is never converted -# back into a raw-submit retry. -DEFAULT_BLOCK_ACCOUNTING_QUEUE_DEPTH = 8 -# The default deadline for every JsonRpc.call without an explicit timeout, -# including the fence-guarded CTV broadcast RPCs. The ledger's own-write -# deferral margin is derived from the guarded deadlines, so route changes -# through this constant rather than the call signature's literal. -DEFAULT_QBIT_RPC_CALL_TIMEOUT_SECONDS = 10.0 -# Headroom added on top of the longest guarded RPC deadline when sizing the -# ledger's own-write deferral margin. It absorbs what the deadline itself -# cannot: the scheduling gap between the fence verification returning and -# the RPC's first byte, coordinator-vs-DB clock drift (the lease runway is -# measured by clock_timestamp()), and the node's application tail — the -# client-side deadline severs the socket, which bounds transmission but -# cannot cancel a handler qbitd already received in full, and these RPC -# handlers apply within milliseconds of complete receipt. The client side -# of the RPC can never exceed its wall-clock deadline (mutating methods are -# in _QBIT_RPC_NO_TRANSPORT_RETRY_METHODS and never retry; retried read -# paths share one deadline across attempts), so a multiplicative factor -# here would only make the deferral needlessly eager as operators raise -# deadlines. A node that stalls longer than this headroom after fully -# receiving a mutating request is the documented residual of preflight -# fencing between independent systems (see the external-side-effect fence -# docstring); no client-side teardown can bound it. -LEASE_AUTHORITY_MARGIN_HEADROOM_SECONDS = 5.0 -# The node fast lane is intentionally shorter than the normal ten-second RPC -# budget: an ambiguous timeout leaves the durable outbox pending and replay -# safely submits the same hash again. -DEFAULT_BLOCK_SUBMIT_RPC_TIMEOUT_SECONDS = 1.0 -# Every PostgreSQL operation reached by the submitter inherits this deadline. -# Direct outbox calls also run behind a coordinator-side single-flight guard, -# so a driver that ignores the deadline cannot freeze the submitter thread. -DEFAULT_BLOCK_SUBMIT_DB_TIMEOUT_SECONDS = 1.0 -# Landing-class deadline. The post-acceptance landing/accounting tail -# (persist, prior balances, confirm, prepared-state rejection of a terminal -# candidate) must not share the one-second budget meant for cheap outbox -# polls and fast-lane-adjacent calls: a structurally slow landing under that -# ceiling is statement-canceled on every attempt and can never complete -# (issue #188 — mainnet block 42743's landing was canceled 13 times while -# Postgres was otherwise idle). The first landing attempt receives this full -# budget; retries escalate after observed landing timeouts. -DEFAULT_BLOCK_LANDING_DB_TIMEOUT_SECONDS = 30.0 -# Reviewed escalation cap for landing retries. Per-attempt wall clock stays -# bounded by the escalated statement budget, cycles are paced by the -# candidate retry backoff, and the stuck-call/coordination watchdogs remain -# the overall bound. An accepted block's landing is never abandoned outright; -# it converges through durable replay. -DEFAULT_BLOCK_LANDING_DB_TIMEOUT_MAX_SECONDS = 120.0 -BLOCK_SUBMITTER_WAIT_HEARTBEAT_SLICE_SECONDS = 0.25 -DEFAULT_BLOCK_SUBMIT_LOCK_WAIT_LOG_SECONDS = 5.0 -DEFAULT_BLOCK_SUBMIT_STUCK_CALL_EXIT_SECONDS = 30.0 -MAX_BLOCK_SUBMITTER_STUCK_CALL_WORKERS = 2 -# How long an own-hash tip observation keeps protecting a block candidate -# from terminal abandonment while instantaneous chain probes disagree -# (transient sibling/fork views during quick-succession blocks, RPC blips). -# Within the window the candidate defers and retries until a fresh probe -# proves it active again (the accepted tail then finalizes it as submitted); -# once the window expires with the hash still absent from the active chain, -# the candidate is genuinely stale and abandons terminally. -DEFAULT_PRISM_OBSERVED_TIP_ACCEPT_WINDOW_SECONDS = 300.0 -BLOCK_CANDIDATE_RETRY_HEARTBEAT_SLICE_SECONDS = 0.25 # Used only by lightweight embedders that bypass dataclass/coordinator # construction. Production instances install these locks eagerly in __init__. # Serializing the fallback prevents concurrent first-touch callers from ever # publishing different lock objects for the same state. _HOT_PATH_LOCK_INITIALIZATION_LOCK = threading.Lock() -DEFAULT_ACCEPTED_BLOCK_PAYOUT_PREVIEW_WAIT_SECONDS = 5.0 -# Fail-closed bound on unresolved accepted-parent transitions (landed -# candidates whose durable bookkeeping has not reached a terminal state). -# Child jobs may keep issuing against published previews while landings -# complete, but once this many transitions are unresolved at once, new job -# builds block instead of stacking further prospective balance chains. The -# durable outbox (MAX_PENDING_BLOCK_CANDIDATES) remains the wider bound. -DEFAULT_ACCEPTED_PARENT_UNRESOLVED_DEPTH_MAX = 8 -# Credit policies recorded on accepted ledger rows. Normal shares carry no -# policy; a policy marks a share that was credited by an explicit pool rule -# (documented in docs/prism-rejections.md) so audits can distinguish them. -PRISM_CREDIT_POLICY_STALE_GRACE = "stale-grace" # Aggregation bucket for per-worker share metrics once the distinct-worker # label budget is exhausted. PRISM_WORKER_METRICS_OVERFLOW_LABEL = "_other" @@ -575,6 +476,7 @@ def validate_payout_artifact_age_bounds( PRISM_REJECTION_POOL_CLOSED, PRISM_REJECTION_BLOCK_STALE, PRISM_REJECTION_LEDGER_CONFIRMATION_FAILED, + PRISM_REJECTION_LEDGER_CONFIRMATION_SUPERSEDED, ) PRISM_TEMPLATE_FINGERPRINT_VOLATILE_KEYS = frozenset( { @@ -589,12 +491,41 @@ def validate_payout_artifact_age_bounds( class _ObservedRLock: - """RLock with zero shared-metrics work on uncontended acquisitions. - - The coordinator lock protects control-plane publication state. Its - contention counters intentionally record only acquisitions that fail an - immediate probe, so observing it cannot recreate the share-path convoy the - metrics are meant to diagnose. + """RLock with no mutex acquisition on uncontended acquisitions. + + The coordinator lock protects control-plane publication state. Its wait + counters intentionally record only acquisitions that fail an immediate + probe, and the shared mutex guarding them is therefore never taken on an + uncontended acquisition, so observing this lock cannot recreate the + share-path convoy the metrics are meant to diagnose. + + A contended count on its own is an absolute number with no denominator, so + a contended percentage cannot be derived from a live process. The + acquisition counter that supplies that denominator does run on every + acquisition, contended or not, and preserves the property above by where + it runs rather than by how often: it is incremented only while the + underlying RLock is already held, which puts it inside a critical section + the lock has already serialised. It therefore adds no mutex acquisition + and no cross-thread ordering that the lock did not already impose, and it + cannot convoy the fast path. + + That placement is also why this counter does not rest on an increment + being atomic under the GIL -- the lock, not the interpreter, is what makes + it safe, so it stays correct on a free-threaded build (see #150, which + audits that class of assumption in this tree). The one unsynchronised + access is the read in contention_snapshot, which holds only + _metrics_lock: it assumes an attribute read yields a whole value rather + than a torn one, and may observe the counter one increment stale. Both are + acceptable for a monotonic counter read by a metrics scrape, and neither + is load-bearing for correctness anywhere else. + + Only granted acquisitions are counted. A wait that times out records + contention without an acquisition, so where a caller passes a timeout the + derived contended percentage is an upper bound. Hold *duration* is + deliberately not instrumented: a re-entrant lock would need per-thread + depth tracking to avoid double counting nested holds, and a clock read on + every acquire and release is exactly the fast-path work this class exists + to avoid. """ def __init__( @@ -604,6 +535,9 @@ def __init__( ) -> None: self._lock = threading.RLock() self._metrics_lock = threading.Lock() + # Guarded by _lock itself, never by _metrics_lock. See the class + # docstring: every write below happens with _lock held. + self._acquisition_count = 0 self._contention_count = 0 self._wait_seconds_sum = 0.0 self._wait_seconds_max = 0.0 @@ -611,8 +545,12 @@ def __init__( def acquire(self, blocking: bool = True, timeout: float = -1) -> bool: if not blocking: - return self._lock.acquire(blocking=False) + if not self._lock.acquire(blocking=False): + return False + self._acquisition_count += 1 + return True if self._lock.acquire(blocking=False): + self._acquisition_count += 1 return True started = time.monotonic() observer = self._wait_observer @@ -635,6 +573,9 @@ def acquire(self, blocking: bool = True, timeout: float = -1) -> bool: if not acquired: observer(max(0.0, time.monotonic() - started)) waited = max(0.0, time.monotonic() - started) + if acquired: + # Still under _lock, as on the uncontended path above. + self._acquisition_count += 1 with self._metrics_lock: self._contention_count += 1 self._wait_seconds_sum += waited @@ -654,501 +595,26 @@ def __exit__(self, *_args: object) -> None: def _is_owned(self) -> bool: return self._lock._is_owned() # type: ignore[attr-defined] - def contention_snapshot(self) -> tuple[int, float, float]: + def contention_snapshot(self) -> tuple[int, int, float, float]: + """Return (acquisitions, contentions, wait_sum, wait_max). + + The acquisition count is read without _lock on purpose -- taking the + coordinator lock to publish a metric would be the observation cost + this class refuses to pay. + """ with self._metrics_lock: return ( + self._acquisition_count, self._contention_count, self._wait_seconds_sum, self._wait_seconds_max, ) -def env(name: str, default: str | None = None) -> str: - value = os.environ.get(name, default) - if value is None or value == "": - raise SystemExit(f"{name} is required") - return value - - -def env_int(name: str, default: int) -> int: - return int(env(name, str(default))) - - -def env_positive_int(name: str, default: int) -> int: - try: - value = env_int(name, default) - except ValueError as exc: - raise SystemExit(f"{name} must be an integer") from exc - if value <= 0: - raise SystemExit(f"{name} must be positive") - return value - - -def env_positive_int_with_legacy(primary_name: str, legacy_name: str, default: int) -> int: - if env_optional(primary_name) is not None: - return env_positive_int(primary_name, default) - return env_positive_int(legacy_name, default) - - -def env_nonnegative_int(name: str, default: int) -> int: - try: - value = env_int(name, default) - except ValueError as exc: - raise SystemExit(f"{name} must be an integer") from exc - if value < 0: - raise SystemExit(f"{name} must be non-negative") - return value - - -def env_nonnegative_int_with_legacy(primary_name: str, legacy_name: str, default: int) -> int: - if env_optional(primary_name) is not None: - return env_nonnegative_int(primary_name, default) - return env_nonnegative_int(legacy_name, default) - - -def env_positive_float(name: str, default: float) -> float: - try: - value = float(env(name, str(default))) - except ValueError as exc: - raise SystemExit(f"{name} must be a number") from exc - if not math.isfinite(value): - raise SystemExit(f"{name} must be finite") - if value <= 0: - raise SystemExit(f"{name} must be positive") - return value - - -def env_nonnegative_float(name: str, default: float) -> float: - try: - value = float(env(name, str(default))) - except ValueError as exc: - raise SystemExit(f"{name} must be a number") from exc - if not math.isfinite(value): - raise SystemExit(f"{name} must be finite") - if value < 0: - raise SystemExit(f"{name} must be non-negative") - return value - - -def env_optional_positive_int(name: str) -> int | None: - raw = env_optional(name) - if raw is None: - return None - try: - value = int(raw) - except ValueError as exc: - raise SystemExit(f"{name} must be an integer") from exc - if value <= 0: - raise SystemExit(f"{name} must be positive") - return value - - -def env_optional_positive_int_with_legacy(primary_name: str, legacy_name: str) -> int | None: - value = env_optional_positive_int(primary_name) - if value is not None: - return value - return env_optional_positive_int(legacy_name) - - -def env_decimal(name: str, default: str) -> Decimal: - value = Decimal(env(name, default)) - if value <= 0: - raise SystemExit(f"{name} must be positive") - return value - - -def env_bool(name: str, default: str) -> bool: - return env(name, default).lower() in {"1", "true", "yes", "on"} - - -def env_optional_bool(name: str) -> bool | None: - raw = env_optional(name) - if raw is None: - return None - return raw.lower() in {"1", "true", "yes", "on"} - - -def env_optional(name: str) -> str | None: - value = os.environ.get(name) - if value is None or value == "": - return None - return value - - -def production_mode() -> bool: - return ( - env_bool("QBIT_PRODUCTION", "0") - or env_bool("QBIT_TOOLS_PRODUCTION", "0") - or env("QBIT_CHAIN", "regtest").lower() in {"main", "mainnet"} - ) - - -def validate_same_tip_job_retention_limits( - *, - retention_seconds: float, - per_connection: int, - max_connections: int, - production: bool, -) -> None: - if retention_seconds <= 0: - return - if per_connection <= 0: - raise SystemExit( - "PRISM_STRATUM_SAME_TIP_JOB_RETENTION_PER_CONNECTION must be positive " - "when same-tip retention is enabled" - ) - if production and max_connections <= 0: - raise SystemExit( - "production mode requires a positive PRISM_STRATUM_MAX_CONNECTIONS " - "when same-tip retention is enabled" - ) - - -def require_production_env(name: str) -> str: - value = env_optional(name) - if value is None: - raise SystemExit(f"production mode requires {name}") - return value - - -def validate_prism_production_gate() -> None: - if not production_mode(): - return - - for name in ( - "PRISM_ALLOW_MEMORY_LEDGER", - "PRISM_ALLOW_TEST_SIGNING_SEEDS", - "PRISM_ALLOW_BUNDLE_EMBEDDED_LEDGER_KEY", - "PRISM_ALLOW_FIXED_LEDGER_SESSION_TOKEN", - ): - if env_bool(name, "0"): - raise SystemExit(f"production mode rejects {name}=1") - - if env("QBIT_CHAIN", "regtest").lower() in {"main", "mainnet"} and env_nonnegative_float( - "PRISM_STRATUM_STALE_GRACE_SECONDS", - DEFAULT_PRISM_STALE_GRACE_SECONDS, - ) != 0: - raise SystemExit( - "mainnet requires PRISM_STRATUM_STALE_GRACE_SECONDS=0" - ) - - production_difficulties: dict[str, Decimal] = {} - for name in ( - "PRISM_STRATUM_SHARE_DIFF", - "PRISM_STRATUM_VARDIFF_MIN_DIFF", - "PRISM_STRATUM_VARDIFF_START_DIFF", - "PRISM_STRATUM_VARDIFF_MAX_DIFF", - ): - raw_value = require_production_env(name) - if not raw_value: - raise SystemExit(f"production mode requires an explicit {name}") - try: - value = Decimal(raw_value) - except InvalidOperation as exc: - raise SystemExit(f"{name} must be a decimal number") from exc - if not value.is_finite() or value <= 0: - raise SystemExit(f"{name} must be positive") - if value == Decimal("0.000000001"): - raise SystemExit(f"{name} cannot use the lab-only 1e-9 difficulty") - production_difficulties[name] = value - if ( - production_difficulties["PRISM_STRATUM_VARDIFF_MIN_DIFF"] - > production_difficulties["PRISM_STRATUM_VARDIFF_START_DIFF"] - ): - raise SystemExit("production vardiff minimum exceeds its start difficulty") - if ( - production_difficulties["PRISM_STRATUM_VARDIFF_START_DIFF"] - > production_difficulties["PRISM_STRATUM_VARDIFF_MAX_DIFF"] - ): - raise SystemExit("production vardiff start exceeds its maximum difficulty") - - prism_database_url = env_optional("PRISM_DATABASE_URL") - if prism_database_url is None and env_optional("PRISM_POSTGRES_PSQL_COMMAND") is None: - raise SystemExit("production mode requires PRISM_DATABASE_URL or PRISM_POSTGRES_PSQL_COMMAND") - if env_optional("PRISM_POSTGRES_PASSWORD") == "change-this": - raise SystemExit("production mode requires a non-default PRISM_POSTGRES_PASSWORD") - if prism_database_url is not None and "change-this" in prism_database_url: - raise SystemExit("production mode requires a non-default PRISM_DATABASE_URL") - - require_production_env("PRISM_MANIFEST_SIGNING_SEED_HEX") - require_production_env("PRISM_LEDGER_ATTESTATION_SIGNING_SEED_HEX") - require_production_env("PRISM_LEDGER_WRITER_PUBLIC_KEY_HEX") - require_production_env("PRISM_LEDGER_WRITER_ID") - require_production_env("PRISM_LEDGER_WRITER_EPOCH") - require_production_env("PRISM_AUDIT_DIR") - require_production_env("PRISM_EVIDENCE_PATH") - - if env_optional("PRISM_LEDGER_WRITER_SESSION_TOKEN") is not None: - raise SystemExit("production mode requires managed ledger session tokens; unset PRISM_LEDGER_WRITER_SESSION_TOKEN") - - if env_nonnegative_int( - "PRISM_STRATUM_MAX_CONNECTIONS", - DEFAULT_PRISM_STRATUM_MAX_CONNECTIONS, - ) <= 0: - raise SystemExit( - "production mode requires a positive PRISM_STRATUM_MAX_CONNECTIONS" - ) - env_positive_int( - "PRISM_STRATUM_MAX_PENDING_INITIAL_JOBS", - DEFAULT_PRISM_STRATUM_MAX_PENDING_INITIAL_JOBS, - ) - if env_nonnegative_float( - "PRISM_STRATUM_INITIAL_JOB_TIMEOUT_SECONDS", - DEFAULT_PRISM_STRATUM_INITIAL_JOB_TIMEOUT_SECONDS, - ) <= 0: - raise SystemExit( - "production mode requires a positive " - "PRISM_STRATUM_INITIAL_JOB_TIMEOUT_SECONDS" - ) - - require_production_env("QBIT_RPC_USER") - qbit_rpc_password = require_production_env("QBIT_RPC_PASSWORD") - if qbit_rpc_password == "change-this": - raise SystemExit("production mode requires a non-default QBIT_RPC_PASSWORD") - - if env("QBIT_CHAIN", "regtest").lower() in {"main", "mainnet"} and env_bool( - "PRISM_CTV_SETTLEMENT_ENABLED", "0" - ): - require_production_env("PRISM_CTV_FANOUT_FEE_MARKET_RATE_BITS_PER_1000_WEIGHT") - env_positive_int("PRISM_CTV_FANOUT_FEE_MARKET_RATE_BITS_PER_1000_WEIGHT", 0) - - validate_same_tip_job_retention_limits( - retention_seconds=env_nonnegative_float( - "PRISM_STRATUM_SAME_TIP_JOB_RETENTION_SECONDS", - DEFAULT_PRISM_SAME_TIP_JOB_RETENTION_SECONDS, - ), - per_connection=env_nonnegative_int( - "PRISM_STRATUM_SAME_TIP_JOB_RETENTION_PER_CONNECTION", - DEFAULT_PRISM_SAME_TIP_JOB_RETENTION_PER_CONNECTION, - ), - max_connections=env_nonnegative_int( - "PRISM_STRATUM_MAX_CONNECTIONS", - DEFAULT_PRISM_STRATUM_MAX_CONNECTIONS, - ), - production=True, - ) - - -def validate_hex(value: str, *, name: str, expected_bytes: int | None = None) -> str: - try: - bytes.fromhex(value) - except ValueError as exc: - raise SystemExit(f"{name} must be hex") from exc - if expected_bytes is not None and len(value) != expected_bytes * 2: - raise SystemExit(f"{name} must be {expected_bytes * 2} hex chars") - return value.lower() - - -def env_seed_hex(name: str, *, test_default: str) -> str: - value = env_optional(name) - if value is None: - if env_bool("PRISM_ALLOW_TEST_SIGNING_SEEDS", "0"): - value = test_default - else: - raise SystemExit(f"{name} is required") - return validate_hex(value, name=name, expected_bytes=32) - - def now_ms() -> int: return int(time.time() * 1000) -def load_prism_vardiff_config(startup_difficulty: Decimal) -> vardiff.VardiffConfig: - return vardiff.VardiffConfig( - enabled=env_bool("PRISM_STRATUM_VARDIFF", "1"), - target_share_interval_seconds=env_decimal("PRISM_STRATUM_VARDIFF_TARGET_SECONDS", "15"), - min_difficulty=env_decimal("PRISM_STRATUM_VARDIFF_MIN_DIFF", str(startup_difficulty)), - max_difficulty=env_decimal("PRISM_STRATUM_VARDIFF_MAX_DIFF", "1024"), - retarget_interval_seconds=env_decimal("PRISM_STRATUM_VARDIFF_RETARGET_SECONDS", "90"), - max_step_factor=env_decimal("PRISM_STRATUM_VARDIFF_MAX_STEP_UP", "4"), - startup_difficulty=env_decimal("PRISM_STRATUM_VARDIFF_START_DIFF", str(startup_difficulty)), - max_step_down_factor=env_decimal("PRISM_STRATUM_VARDIFF_MAX_STEP_DOWN", "4"), - ewma_alpha=env_decimal("PRISM_STRATUM_VARDIFF_EWMA_ALPHA", "0.4"), - retarget_tolerance=env_decimal("PRISM_STRATUM_VARDIFF_RETARGET_TOLERANCE", "0.25"), - ) - - -@dataclass(frozen=True) -class StratumListenerProfile: - """One stratum listener with its own difficulty policy. - - Every listener feeds the same coordinator, ledger, and settlement path; - the profile only decides where to listen and which difficulty bounds its - clients get. - """ - - name: str - bind: str - port: int - share_difficulty: Decimal - vardiff_config: vardiff.VardiffConfig - heartbeat_name: str - # Difficulty this listener never advertises below, even when the qbit - # network target is easier. Zero means no floor: stamped jobs keep the - # network cap (a share is never required to be harder than a block). The - # high-diff listener sets its configured minimum because marketplace - # verification (NiceHash-style) checks the first advertised difficulty, - # which must hold even while network difficulty sits below the floor. - minimum_advertised_difficulty: Decimal = Decimal("0") - - -DEFAULT_HIGHDIFF_DIFFICULTY = "500000" -DEFAULT_HIGHDIFF_MAX_DIFFICULTY = "4294967296" - - -def load_prism_highdiff_listener( - base_bind: str, - base_vardiff_config: vardiff.VardiffConfig, -) -> StratumListenerProfile | None: - """Optional high-difficulty listener for rental-scale miners. - - Disabled unless PRISM_STRATUM_HIGHDIFF_PORT is set. The 500k default floor - matches the NiceHash SHA-256 pool-verification minimum, which must hold - from the first mining.set_difficulty a client sees. - """ - - port_value = env_optional("PRISM_STRATUM_HIGHDIFF_PORT") - if port_value is None: - return None - try: - port = int(port_value) - except ValueError as exc: - raise SystemExit("PRISM_STRATUM_HIGHDIFF_PORT must be an integer") from exc - if not 0 < port < 65536: - raise SystemExit("PRISM_STRATUM_HIGHDIFF_PORT must be a valid TCP port") - min_difficulty = env_decimal("PRISM_STRATUM_HIGHDIFF_MIN_DIFF", DEFAULT_HIGHDIFF_DIFFICULTY) - start_difficulty = env_decimal("PRISM_STRATUM_HIGHDIFF_START_DIFF", DEFAULT_HIGHDIFF_DIFFICULTY) - max_difficulty = env_decimal("PRISM_STRATUM_HIGHDIFF_MAX_DIFF", DEFAULT_HIGHDIFF_MAX_DIFFICULTY) - if min_difficulty > start_difficulty: - raise SystemExit("PRISM_STRATUM_HIGHDIFF_MIN_DIFF exceeds PRISM_STRATUM_HIGHDIFF_START_DIFF") - if start_difficulty > max_difficulty: - raise SystemExit("PRISM_STRATUM_HIGHDIFF_START_DIFF exceeds PRISM_STRATUM_HIGHDIFF_MAX_DIFF") - # The fixed difficulty (used when vardiff is disabled) tracks the start - # difficulty unless explicitly set, and must respect the listener bounds: - # advertising below the floor would break the marketplace verification - # this listener exists for. - share_value = env_optional("PRISM_STRATUM_HIGHDIFF_SHARE_DIFF") - if share_value is None: - share_difficulty = start_difficulty - else: - try: - share_difficulty = Decimal(share_value) - except Exception as exc: - raise SystemExit("PRISM_STRATUM_HIGHDIFF_SHARE_DIFF must be a decimal") from exc - if not share_difficulty.is_finite() or share_difficulty <= 0: - raise SystemExit("PRISM_STRATUM_HIGHDIFF_SHARE_DIFF must be positive") - if share_difficulty < min_difficulty: - raise SystemExit("PRISM_STRATUM_HIGHDIFF_SHARE_DIFF is below PRISM_STRATUM_HIGHDIFF_MIN_DIFF") - if share_difficulty > max_difficulty: - raise SystemExit("PRISM_STRATUM_HIGHDIFF_SHARE_DIFF exceeds PRISM_STRATUM_HIGHDIFF_MAX_DIFF") - try: - config = dataclass_replace( - base_vardiff_config, - min_difficulty=min_difficulty, - max_difficulty=max_difficulty, - startup_difficulty=start_difficulty, - ) - except ValueError as exc: - raise SystemExit(f"invalid PRISM_STRATUM_HIGHDIFF_* difficulty bounds: {exc}") from exc - return StratumListenerProfile( - name="highdiff", - # env_optional so an empty value (compose default passthrough) means - # "inherit the default listener bind" instead of a startup failure. - bind=env_optional("PRISM_STRATUM_HIGHDIFF_BIND") or base_bind, - port=port, - share_difficulty=share_difficulty, - vardiff_config=config, - heartbeat_name="stratum_accept_highdiff", - minimum_advertised_difficulty=min_difficulty, - ) - - -def parse_stratum_password_options(password: str) -> tuple[Decimal | None, Decimal | None]: - """Extract the pool-side d=N / md=N difficulty convention from a password. - - Unknown tokens and malformed values are ignored: miners routinely send - junk passwords ("x") and rejecting them would break every such rig. - Returns (requested_difficulty, requested_min_difficulty). - """ - - requested: Decimal | None = None - requested_min: Decimal | None = None - for token in password.split(","): - key, separator, raw_value = token.strip().partition("=") - if not separator: - continue - key = key.strip().lower() - if key not in {"d", "md"}: - continue - try: - value = Decimal(raw_value.strip()) - except Exception: - continue - if not value.is_finite() or value <= 0: - continue - if key == "d": - requested = value - else: - requested_min = value - return requested, requested_min - - -# Coinbase output ordering policies accepted by PRISM_COINBASE_OUTPUT_POLICY. -# "canonical" preserves the historical lexicographic output ordering; -# "pool-fee-first" reserves a direct settlement slot for a positive pool fee -# and emits it at coinbase vout 0. -VALID_COINBASE_OUTPUT_POLICIES = frozenset({"canonical", "pool-fee-first"}) - - -def default_prism_payout_policy() -> dict[str, object]: - policy: dict[str, object] = { - "p2mr_spend_input_bytes": env_positive_int( - "PRISM_PAYOUT_P2MR_SPEND_INPUT_BYTES", - DEFAULT_P2MR_SPEND_INPUT_BYTES, - ), - "target_feerate_sats_per_byte": env_positive_int_with_legacy( - "PRISM_PAYOUT_TARGET_FEERATE_BITS_PER_BYTE", - "PRISM_PAYOUT_TARGET_FEERATE_SATS_PER_BYTE", - DEFAULT_MIN_OUTPUT_FEERATE_SATS_PER_BYTE, - ), - "safety_multiplier": env_positive_int( - "PRISM_PAYOUT_SAFETY_MULTIPLIER", - DEFAULT_MIN_OUTPUT_SAFETY_MULTIPLIER, - ), - } - min_output_sats = env_optional_positive_int_with_legacy( - "PRISM_PAYOUT_MIN_OUTPUT_BITS", - "PRISM_PAYOUT_MIN_OUTPUT_SATS", - ) - if min_output_sats is not None: - policy["min_output_sats"] = min_output_sats - return policy - - -def default_prism_coinbase_tag_hex() -> str: - tag = os.environ.get("PRISM_COINBASE_TAG", DEFAULT_PRISM_COINBASE_TAG) - try: - tag_bytes = tag.encode("ascii") - except UnicodeEncodeError as exc: - raise SystemExit("PRISM_COINBASE_TAG must be ASCII") from exc - if len(tag_bytes) > MAX_PRISM_COINBASE_TAG_BYTES: - raise SystemExit( - f"PRISM_COINBASE_TAG must be at most {MAX_PRISM_COINBASE_TAG_BYTES} bytes" - ) - if any(byte < 0x20 or byte > 0x7e for byte in tag_bytes): - raise SystemExit("PRISM_COINBASE_TAG must contain printable ASCII only") - return tag_bytes.hex() - - -def default_prism_username_fallback_address() -> str | None: - configured = env_optional("PRISM_USERNAME_FALLBACK_ADDRESS") - if configured is not None: - return configured - if (os.environ.get("QBIT_CHAIN") or "regtest").lower() in TESTNET_QBIT_CHAINS: - return DEFAULT_TESTNET_USERNAME_FALLBACK_ADDRESS - return None - - def qbit_gbt_rules(chain: str) -> list[str]: rules = ["segwit"] if chain.strip().lower() == "signet": @@ -1182,2075 +648,1686 @@ def canonical_json_sha256(value: object) -> str: return hashlib.sha256(canonical_json_text(value).encode()).hexdigest() -def validate_initial_job_max_workers(workers: int, max_pending: int) -> int: - """Reject first-job worker counts the pending-client bound cannot feed. +class _BundlePreparationSuperseded(TemplateRefreshSuperseded): + """The exact work identity lost to a newer tip/template observation. - The dedicated executor's queue is bounded by - PRISM_STRATUM_MAX_PENDING_INITIAL_JOBS, so workers beyond that count can - never receive a task; unbounded values would also risk partially failed - thread creation on constrained hosts. + Subclasses TemplateRefreshSuperseded: losing the shared-bundle build race + to a newer tip/template observation is coordination churn, so it escapes + the poll without arming the template-refresh failure budget. """ - if workers <= 0: - raise SystemExit("PRISM_INITIAL_JOB_MAX_WORKERS must be positive") - if workers > max_pending: - raise SystemExit( - "PRISM_INITIAL_JOB_MAX_WORKERS cannot exceed " - "PRISM_STRATUM_MAX_PENDING_INITIAL_JOBS" - ) - return workers -def resolve_initial_job_max_workers( - explicit: int | None, - max_pending: int, -) -> int: - """Resolve the first-job worker count against the pending-client bound. +class _ProgressHealthStateField: + """Route a legacy raw ``_progress_*`` coordinator attribute to its owner. - Deployments that bound pending clients below the default worker count - were valid before the worker knob existed; the implicit default must cap - itself to the pending bound rather than fail their startup. An explicit - setting keeps strict validation. + G1 owns the aggregate progress state. Reads and writes deliberately take + no lock: the historical white-box protocol holds + ``coordinator._progress_health_lock`` (the service lock) around direct + field access, and the service's own methods lock internally. """ - if explicit is None: - return min(DEFAULT_PRISM_INITIAL_JOB_MAX_WORKERS, int(max_pending)) - return validate_initial_job_max_workers(explicit, max_pending) + def __init__(self, attribute: str) -> None: + self._attribute = attribute -def validate_job_build_executor_workers(workers: int) -> int: - """Reject executor widths the build scheduler can never use. - - The scheduler admits at most two concurrent build flights (one active, - one retiring); extra executor threads would never receive work, so a - higher setting would be a silent no-op rather than a tuning gain. - """ - if workers <= 0: - raise SystemExit("PRISM_JOB_BUILD_EXECUTOR_WORKERS must be positive") - if workers > DEFAULT_PRISM_JOB_BUILD_EXECUTOR_WORKERS: - raise SystemExit( - "PRISM_JOB_BUILD_EXECUTOR_WORKERS cannot exceed " - f"{DEFAULT_PRISM_JOB_BUILD_EXECUTOR_WORKERS}" - ) - return workers - - -def _compact_share_payload( - shares: list[dict[str, object]], -) -> tuple[list[tuple[str, str, str]], list[tuple[object, ...]]]: - """Deduplicate share identities into the audit-builder compact form.""" - - identity_indexes: dict[tuple[str, str, str], int] = {} - identities: list[tuple[str, str, str]] = [] - compact_shares: list[tuple[object, ...]] = [] - for share in shares: - identity = ( - str(share["miner_id"]), - str(share["order_key"]), - str(share["p2mr_program_hex"]), - ) - identity_index = identity_indexes.get(identity) - if identity_index is None: - identity_index = len(identities) - identity_indexes[identity] = identity_index - identities.append(identity) - compact_shares.append( - ( - share["share_seq"], - share["share_id"], - identity_index, - share["share_difficulty"], - share["job_issued_at_ms"], - share["accepted_at_ms"], - share.get("credit_policy"), - ) + def __get__( + self, + instance: "PrismCoordinator | None", + owner: type | None = None, + ) -> object: + if instance is None: + return self + return getattr( + instance._ensure_progress_health_service(), + self._attribute, ) - return identities, compact_shares - -_QBIT_RPC_NO_TRANSPORT_RETRY_METHODS = frozenset( - { - "getnewaddress", - "sendrawtransaction", - "signrawtransactionwithwallet", - "submitblock", - "submitpackage", - } -) + def __set__(self, instance: "PrismCoordinator", value: object) -> None: + setattr( + instance._ensure_progress_health_service(), + self._attribute, + value, + ) -class JsonRpc: - def __init__(self, *, host: str, port: int, user: str, password: str): - self.host = host - self.port = port - self.url = f"http://{host}:{port}" - credentials = f"{user}:{password}".encode() - self.auth = f"Basic {base64.b64encode(credentials).decode()}" - # Keep-alive connections, one per calling thread. qbitd is called on - # the hot share/block paths (a fresh getaddrinfo + TCP connect per call - # was ~seconds of overhead under load); reusing the connection removes - # that. threading.local keeps each thread's HTTPConnection private, so - # concurrent callers never share a non-thread-safe connection. - self._connections = threading.local() - - def _acquire_connection(self, timeout: float) -> http.client.HTTPConnection: - conn = getattr(self._connections, "conn", None) - if conn is None: - conn = http.client.HTTPConnection(self.host, self.port, timeout=timeout) - # Never let http.client resurrect a severed connection: with - # auto_open, request() silently reconnects when the deadline - # watchdog's close() lands between the pre-send check and the - # send, and that implicit fresh socket lives until the next - # watchdog sweep — long enough for a short mutating POST to - # reach qbitd after a caller released its ordering locks. With - # auto_open off a cleared socket makes send() raise - # NotConnected instead; call() connects explicitly under the - # watchdog and rechecks the deadline before any byte goes out. - conn.auto_open = 0 - self._connections.conn = conn - else: - # Reuse: refresh the deadline for this call on the live socket. - conn.timeout = timeout - if conn.sock is not None: - conn.sock.settimeout(timeout) - return conn - - def _drop_connection(self) -> None: - conn = getattr(self._connections, "conn", None) - if conn is not None: - try: - conn.close() - except Exception: - pass - self._connections.conn = None +class _JobBundleStateField: + """Route one legacy J1 coordinator field to its owner service. - def call( - self, - method: str, - params: list[object] | None = None, - *, - wallet: str | None = None, - timeout: float = DEFAULT_QBIT_RPC_CALL_TIMEOUT_SECONDS, - ) -> Any: - body = json.dumps( - { - "jsonrpc": "1.0", - "id": method, - "method": method, - "params": params or [], - } - ).encode() - path = "/" - if wallet is not None: - path = f"/wallet/{urllib.parse.quote(wallet, safe='')}" - headers = { - "Authorization": self.auth, - "Content-Type": "application/json", - "User-Agent": "qbit-prism-coordinator/0.1", - } - # Read-only calls retry once with a fresh connection after a transport - # error, normally an idle keep-alive that qbitd closed. Mutating calls - # never retry inside this method: their writer-lease fence applies to - # the outer call, so an invisible second POST could otherwise run after - # that lease was lost. Durable block/CTV workflows retry later as a new - # fully fenced operation and reconcile an uncertain first result. - last_exc: Exception | None = None - attempt_count = ( - 1 if method in _QBIT_RPC_NO_TRANSPORT_RETRY_METHODS else 2 - ) - deadline = time.monotonic() + max(0.001, float(timeout)) - for attempt in range(attempt_count): - remaining = deadline - time.monotonic() - if remaining <= 0: - if last_exc is not None: - raise last_exc - raise TimeoutError(f"qbit RPC {method} timed out") - conn = self._acquire_connection(remaining) - # The socket timeout bounds each individual socket operation, - # not the call: a response body arriving one packet per - # interval extends the call arbitrarily past its nominal - # deadline. Lease authority margins are sized from this - # timeout as a wall-clock bound on fence-guarded effects, so - # enforce it end-to-end: past the deadline a watchdog severs - # the connection's socket — repeatedly, because a stalled name - # resolution has no socket yet and the one connect() creates - # afterwards must not carry the RPC onward — until the attempt - # observes the abort as a transport error. For mutating calls - # that is the same uncertain outcome a socket timeout already - # produces, reconciled by durable replay as a new fully fenced - # operation. - watchdog_fired = threading.Event() - attempt_finished = threading.Event() - - # The allowance is the attempt's exact remaining wall clock, no - # grace: callers like the submitblock hard-deadline wrapper - # abandon the call and release ordering locks (the payout - # landing fence) the instant their deadline passes, so a byte - # transmitted in any grace window would race work those callers - # already unblocked. - def _enforce_deadline( - doomed: http.client.HTTPConnection = conn, - fired: threading.Event = watchdog_fired, - finished: threading.Event = attempt_finished, - allowance: float = remaining, - ) -> None: - if finished.wait(allowance): - return - fired.set() - while True: - sock = getattr(doomed, "sock", None) - if sock is not None: - try: - sock.shutdown(socket.SHUT_RDWR) - except OSError: - pass - try: - doomed.close() - except Exception: - pass - if finished.wait(0.1): - return - - watchdog = threading.Thread( - target=_enforce_deadline, - name="qbit-rpc-deadline", - daemon=True, - ) - watchdog.start() - try: - # Establish the connection (name resolution + TCP) under - # the watchdog and re-check the deadline before any request - # byte is sent. Left to conn.request(), a name resolution - # returning after the deadline resumes straight into - # sendall(), and a short mutating POST can reach qbitd - # between the watchdog's periodic sweeps; the explicit - # check makes the late-connect case lose deterministically. - # The check is against the wall clock, not just the - # watchdog flag: the flag lags the deadline by thread - # scheduling, and a caller that abandoned this call at its - # deadline may already have released ordering locks that - # a post-deadline send would race. - if getattr(conn, "sock", None) is None: - conn_connect = getattr(conn, "connect", None) - if conn_connect is not None: - conn_connect() - if watchdog_fired.is_set() or time.monotonic() >= deadline: - raise TimeoutError(f"qbit RPC {method} timed out") - conn.request("POST", path, body=body, headers=headers) - response = conn.getresponse() - data = response.read() # drain so the connection can be reused - except (http.client.HTTPException, OSError) as exc: - self._drop_connection() - if watchdog_fired.is_set(): - raise TimeoutError(f"qbit RPC {method} timed out") from exc - last_exc = exc - if attempt + 1 < attempt_count: - continue - raise - finally: - attempt_finished.set() - if response.status != 200: - # Non-200 bodies may hold a JSON-RPC error (qbitd returns the - # error object with a 500 for some methods); surface it as the - # same RuntimeError text callers already match on (e.g. the - # "-32601 / Method not found" blockwait-unsupported probe). - self._drop_connection() - detail = data.decode("utf-8", "replace") - try: - error = json.loads(detail).get("error") - except Exception: - error = None - if error is not None: - raise RuntimeError(f"qbit RPC {method} failed: {error}") - raise RuntimeError(f"qbit RPC {method} HTTP {response.status}: {detail[:200]}") - payload = json.loads(data) - if payload["error"] is not None: - raise RuntimeError(f"qbit RPC {method} failed: {payload['error']}") - return payload["result"] - raise last_exc if last_exc is not None else RuntimeError("qbit RPC call failed") - - -@dataclass(frozen=True) -class WorkerIdentity: - username: str - payout_address: str - worker_name: str | None - script_pubkey_hex: str - p2mr_program_hex: str - - -@dataclass -class _P2mrAddressValidationFlight: - event: threading.Event = field(default_factory=threading.Event) - result: tuple[str, str] | None = None - error: BaseException | None = None - waiters: int = 0 - - -@dataclass(frozen=True) -class PrismJobContext: - job: direct_stratum.DirectQbitStratumJob - template: dict[str, Any] - shares_json: list[dict[str, object]] - prior_balances: list[dict[str, object]] - found_block: dict[str, object] - share_weight: int - collection_only: bool - worker: WorkerIdentity - issued_at_ms: int - template_fingerprint: str | None = None - template_generation: int = 0 - payout_state_generation: int = 0 - prospective_prior_balances: tuple[tuple[str, str, str, int], ...] | None = None - payout_artifact_generation: int = 0 - payout_append_invalidation_epoch: int = 0 - # Digest of the published prior-balance snapshot this job's payouts are - # keyed to. A periodic self-check repair replaces the published snapshot - # in place -- the payout generation and append epoch both survive the - # swap -- so this digest is the only fence that identifies an active job - # still carrying refuted balances. - payout_artifact_sha256: str | None = None - connection_id: int = 0 - authorization_generation: int = 0 - difficulty_generation: int = 0 - tip_refresh_epoch_sequence: int = 0 - # Version-rolling mask negotiated on the connection this job was - # delivered to. Cross-connection resumes validate in-flight version bits - # against this mask, not the replacement connection's (often still 0). - version_mask: int = 0 - - -@dataclass -class PendingShareAppend: - """A share waiting for the ledger group-commit writer. - - The client thread does not count or acknowledge this share until - ``committed`` is set successfully. A block candidate intent, when - present, is inserted in the same transaction as the share. + The descriptor keeps the historical attribute name readable and writable + on the coordinator while :class:`JobBundleService` owns the single + mutable copy, mirroring the progress-health extraction pattern. Direct + test assignment through the legacy name is thereby adopted by the owner. """ - pending_share: PendingShare - username: str - job_id: str - block_hash_hex: str - collection_only: bool - credit_policy: str | None - candidate_intent: dict[str, Any] | None = None - committed: threading.Event = field(default_factory=threading.Event) - record: Any | None = None - error: BaseException | None = None - writer_token: _WriterOperationToken | None = None - - -@dataclass(frozen=True) -class PrismBlockCandidate: - """A block-worthy submission queued for the block-submitter thread. - - A share that met its target is acknowledged and credited on the client - thread, then queued here for the submitter to land the block off the hot - path. When the hash solved the block but missed the share target (floor - above network difficulty), credit_share_on_accept is set and the candidate - is instead submitted synchronously by handle_submit: that share is valid - only if the block lands, so its credit and the miner's accept/reject follow - the block outcome directly rather than being queued. - """ + def __init__(self, attribute: str) -> None: + self._attribute = attribute - context: PrismJobContext - submission: direct_stratum.DirectQbitSubmission - extranonce1_hex: str - extranonce2_hex: str - pending_share: PendingShare - client: ClientState - credit_share_on_accept: bool = False - durable_replay: bool = False - # When this in-process attempt became runnable: live candidates stamp - # share-accept time, durable outbox replays stamp row-restore time. The - # block-submit histogram measures from here to submitblock's return -- - # the race-critical interval a lost block round is decided in. - landed_monotonic: float = field(default_factory=time.monotonic) + def __get__( + self, + instance: "PrismCoordinator | None", + owner: type | None = None, + ) -> object: + if instance is None: + return self + return getattr( + instance._ensure_job_bundle_service(), + self._attribute, + ) + def __set__(self, instance: "PrismCoordinator", value: object) -> None: + setattr( + instance._ensure_job_bundle_service(), + self._attribute, + value, + ) -@dataclass -class _BlockCandidateDispositionFlight: - """One same-hash submission guard shared by its holder and waiters.""" - # The node-offer thread acquires this guard and the accounting thread - # releases it after durable finalization. A plain Lock permits that - # deliberate ownership transfer; RLock does not. - lock: threading.Lock = field(default_factory=threading.Lock) - users: int = 0 +class _TemplateArtifactStateField: + """Route the legacy template-artifact fields to their repository owner.""" + def __init__(self, attribute: str) -> None: + self._attribute = attribute -@dataclass(frozen=True) -class _BlockCandidateDispositionLease: - """A same-hash guard held across node offer and durable finalization.""" + def __get__( + self, + instance: "PrismCoordinator | None", + owner: type | None = None, + ) -> object: + if instance is None: + return self + repository = instance._ensure_job_bundle_service().template_repository + return getattr(repository, self._attribute) - block_hash: str - flight: _BlockCandidateDispositionFlight + def __set__(self, instance: "PrismCoordinator", value: object) -> None: + repository = instance._ensure_job_bundle_service().template_repository + if self._attribute == "_template_artifacts": + repository.adopt_template_artifacts(value) + return + repository.adopt_template_artifact_generation(value) -@dataclass(frozen=True) -class _BlockCandidateNodeSubmission: - """Result of the latency-critical qbitd fast-lane call.""" +class _BundleCompilerStateField: + """Route the legacy serve-builder daemon fields to the compiler owner.""" - attempted: bool - result: object = None - error: BaseException | None = None + def __init__(self, attribute: str) -> None: + self._attribute = attribute + def __get__( + self, + instance: "PrismCoordinator | None", + owner: type | None = None, + ) -> object: + if instance is None: + return self + return getattr(instance._ensure_bundle_compiler(), self._attribute) -@dataclass -class _BlockSubmitterLedgerCall: - """One still-running direct outbox call, reused across paced retries.""" + def __set__(self, instance: "PrismCoordinator", value: object) -> None: + setattr(instance._ensure_bundle_compiler(), self._attribute, value) - started_monotonic: float = field(default_factory=time.monotonic) - done: threading.Event = field(default_factory=threading.Event) - result: object = None - error: BaseException | None = None +class _TipRefreshStateField: + """Route one legacy R1 coordinator field to its owner service. -@dataclass -class _BlockSubmitterRpcCall: - """One hard-deadline, single-flight submitblock transport call.""" + The descriptor keeps the historical attribute name readable and writable + on the coordinator while :class:`TipRefreshService` owns the single + mutable copy. The twelve historical plain fields (and the tip-detection + epoch) are lazily created on the service, so ``hasattr``/``getattr`` + -with-default semantics of the pre-extraction coordinator are preserved + exactly: reading an unset field raises AttributeError until first write. + """ - started_monotonic: float = field(default_factory=time.monotonic) - done: threading.Event = field(default_factory=threading.Event) - result: object = None - error: BaseException | None = None + def __init__(self, attribute: str) -> None: + self._attribute = attribute + def __get__( + self, + instance: "PrismCoordinator | None", + owner: type | None = None, + ) -> object: + if instance is None: + return self + return getattr( + instance._ensure_tip_refresh_service(), + self._attribute, + ) -@dataclass(frozen=True) -class _BlockCandidateAccountingTask: - """A node-offered candidate awaiting serialized durable accounting.""" + def __set__(self, instance: "PrismCoordinator", value: object) -> None: + setattr( + instance._ensure_tip_refresh_service(), + self._attribute, + value, + ) - candidate: PrismBlockCandidate - node_submission: _BlockCandidateNodeSubmission - disposition_lease: _BlockCandidateDispositionLease +class _JobDeliveryStateField: + """Route one legacy S2 coordinator field to its owner service. -class BlockSubmitterDatabaseTimeout(TimeoutError): - """A submitter ledger phase exceeded its coordinator-side deadline.""" + The descriptor keeps the historical attribute name readable and writable + on the coordinator while :class:`JobDeliveryService` owns the single + mutable copy, mirroring the J1/G1 extraction pattern. Direct test + assignment through the legacy name is thereby adopted by the owner. + """ + def __init__(self, attribute: str) -> None: + self._attribute = attribute -@dataclass(frozen=True) -class CachedTemplateArtifacts: - """Template plus everything derivable from it alone, shared by all clients. + def __get__( + self, + instance: "PrismCoordinator | None", + owner: type | None = None, + ) -> object: + if instance is None: + return self + return getattr( + instance._ensure_job_delivery_service(), + self._attribute, + ) - Derived fields are keyed by the template fingerprint: a refetch whose - fingerprint matches (only clock fields moved) reuses the previously - computed transaction hexes and witness merkle leaves instead of re-hashing - the full template. Generation records observation-start order so a slow, - older fetch cannot supersede a newer observation merely by finishing last. - """ + def __set__(self, instance: "PrismCoordinator", value: object) -> None: + setattr( + instance._ensure_job_delivery_service(), + self._attribute, + value, + ) - template: dict[str, Any] - fingerprint: str - previousblockhash: str - transaction_hexes: tuple[str, ...] - witness_merkle_leaves_hex: tuple[str, ...] - network_difficulty: int - fetched_monotonic: float - generation: int = 0 - - -@dataclass(frozen=True) -class QbitTipTemplateSnapshot: - bestblockhash: str - previousblockhash: str - template_fingerprint: str - template_generation: int = 0 - # The observation owns this exact artifact object. It deliberately does - # not participate in snapshot equality: callers compare the stable - # identity fields above, while refresh preparation consumes the exact - # template and derivations that were observed even if the mutable cache's - # current pointer is replaced concurrently. - template_artifacts: CachedTemplateArtifacts | None = field( - default=None, - compare=False, - repr=False, - ) +class _PayoutStateStateField: + """Route one legacy P1 coordinator field to its owner service. -@dataclass(frozen=True) -class RetainedCollectionRefresh: - """Current immutable preparation waiting for a collection identity.""" - - snapshot: QbitTipTemplateSnapshot - observation_sequence: int - payout_state_generation: int - - -@dataclass(frozen=True) -class PayoutStateArtifact: - """Immutable ledger-backed inputs published with one payout generation.""" - - generation: int - source_generation: int - prior_balances_json: str = field(repr=False) - prior_balances_sha256: str - prepared_monotonic: float - - def prior_balances(self) -> list[dict[str, object]]: - value = json.loads(self.prior_balances_json) - if not isinstance(value, list): - raise RuntimeError("published payout artifact is not a balance list") - return value - - -@dataclass(frozen=True) -class JobBuildKey: - """Every immutable input that can affect one constructed mining job.""" - - best_tip_hash: str - previous_block_hash: str - template_fingerprint: str - template_generation: int - payout_state_generation: int - payout_artifact_sha256: str - mode: str - collection_identity: tuple[str, str] | None - block_height: int - coinbase_value_sats: int - network_difficulty: int - issued_at_ms: int - payout_policy_sha256: str - ctv_settlement_sha256: str | None - witness_merkle_sha256: str - transaction_set_sha256: str - coinbase_suffix_hex: str - signing_key_sha256: str - ledger_signing_key_sha256: str - numeric_context_sha256: str - share_snapshot_sha256: str = "" - payout_append_invalidation_epoch: int = 0 - - -@dataclass(frozen=True) -class CachedJobBundle: - """One heavy job build (ledger snapshot + signed manifest + base job) - shared across every client on the same template. - - The base job is built with the extranonce1 placeholder; per-client jobs - are stamped from it by swapping job_id, extranonce1, difficulty, and the - clean_jobs flag. All other fields are byte-identical across clients - because the stratum coinbase split excludes the extranonce window. + The descriptor keeps the historical attribute name readable and writable + on the coordinator while :class:`PayoutStateService` owns the single + mutable copy, mirroring the S1/S2/S3/J1/G1 extraction pattern. Direct + test assignment through the legacy name is thereby adopted by the owner. """ - key: tuple[object, ...] - template: dict[str, Any] - template_fingerprint: str - coinbase_manifest: dict[str, Any] - shares_json: list[dict[str, object]] - prior_balances: list[dict[str, object]] - found_block: dict[str, object] - collection_only: bool - issued_at_ms: int - base_job: direct_stratum.DirectQbitStratumJob - built_monotonic: float - template_generation: int = 0 - payout_state_generation: int = 0 - payout_artifact_generation: int = 0 - # Collection coinbases commit a synthetic share to this exact payout - # identity. Ready bundles have no worker-specific inputs and keep this - # unset, which makes accidental cross-worker stamping fail closed. - collection_identity: tuple[str, str] | None = None - # Compact carry-forward state derived from the immutable job summary. It - # lets an accepted block publish child payout state without retaining the - # full audit bundle (and its duplicate shares tree) in every cached job. - prospective_prior_balances: tuple[tuple[str, str, str, int], ...] | None = None - build_key: JobBuildKey | None = None - # A synchronous ready build publishes its fenced ledger window for reuse - # only once this bundle wins cache publication (or is served as pinned - # snapshot work). Installing mid-build would re-key concurrent lookups - # away from the in-flight single flight and fork redundant builds. - prepared_ledger_artifact: PayoutLedgerArtifact | None = field( - default=None, - compare=False, - repr=False, - ) - + def __init__(self, attribute: str) -> None: + self._attribute = attribute -@dataclass(frozen=True) -class _AcceptedBlockPayoutTransition: - """Prospective balances for one durable candidate across its landing seam.""" - - block_height: int | None = None - landed: bool = False - preview: tuple[tuple[str, str, str, int], ...] | None = None - published_generation: int | None = None - landed_monotonic: float | None = None - - -@dataclass(frozen=True) -class _IdleRetargetRequest: - """Immutable idle-window identity captured by one bounded sweep.""" - - client: ClientState - connection_id: int - worker: WorkerIdentity - active_job: PrismJobContext - window_started_monotonic: float - current_difficulty: Decimal - elapsed_seconds: Decimal - - -@dataclass(frozen=True) -class EvictedJobEntry: - context: PrismJobContext - connection_id: int - evicted_monotonic: float - previousblockhash: str - client: ClientState | None = None - - -@dataclass(frozen=True) -class RefreshResult: - result: str - delivered_monotonic: float | None = None - - -@dataclass(frozen=True, eq=False) -class TipRefreshValidationToken: - """Immutable proof that one prepared refresh passed its expensive guard.""" - - tip_hash: str - template_fingerprint: str - template_generation: int - payout_state_generation: int - observation_sequence: int - build_key: JobBuildKey - snapshot: QbitTipTemplateSnapshot = field(repr=False) - epoch_sequence: int = 0 - - -@dataclass -class _TipRefreshEpochCoverage: - """Client cohort snapshot used to observe fixed-cardinality coverage latency.""" - - sequence: int - tip_hash: str - payout_state_generation: int - started_monotonic: float - client_weights: dict[int, Decimal] - total_weight: Decimal - delivered_weight: Decimal = Decimal(0) - delivered_clients: set[int] = field(default_factory=set) - recorded_targets: set[str] = field(default_factory=set) - - -@dataclass(frozen=True) -class PayoutStateCandidate: - """Immutable result of payout work prepared outside delivery admission.""" - - base_generation: int - source_generation: int - source_tip_hash: str | None - cause: str - invalidated_monotonic: float - prepared_monotonic: float - accepted_block_hash: str | None = None - accepted_block_preview: tuple[tuple[str, str, str, int], ...] | None = None - accepted_block_withdrawal: bool = False - accepted_block_height: int | None = None - ledger_artifact: PayoutLedgerArtifact | None = field( - default=None, - compare=False, - repr=False, - ) + def __get__( + self, + instance: "PrismCoordinator | None", + owner: type | None = None, + ) -> object: + if instance is None: + return self + return getattr( + instance._ensure_payout_state_service(), + self._attribute, + ) + def __set__(self, instance: "PrismCoordinator", value: object) -> None: + setattr( + instance._ensure_payout_state_service(), + self._attribute, + value, + ) -@dataclass(frozen=True) -class PublishedPayoutState: - """The payout snapshot identity to which cached jobs are stamped.""" - - generation: int - source_generation: int - source_tip_hash: str | None - published_monotonic: float - artifact: PayoutStateArtifact | None = field(default=None, repr=False) - - -@dataclass(frozen=True) -class PayoutLedgerArtifact: - """Immutable ledger input prepared independently of a qbit template.""" - - generation: int - payout_state_generation: int - network_difficulty: int - # Durable accepted-share total observed near the snapshot. Informational - # (diagnostics only): validity is scoped to snapshot_anchor_ms, and the - # count cannot be scoped to a clamped anchor because share stamping and - # writer enqueue are not atomic. - accepted_share_count: int - shares_json: Sequence[dict[str, object]] = field(repr=False) - prior_balances: tuple[dict[str, object], ...] = field(repr=False) - prepared_monotonic: float - # The anchor the share snapshot was actually taken at. Bundles built from - # this artifact must declare it as anchor_job_issued_at_ms: an auditor - # replaying qbit_audit_share_window at the declared anchor must reproduce - # exactly these shares, which only holds at the snapshot's own anchor. - # That declaration is also what makes reuse valid while shares keep - # landing: a share stamped after this anchor deterministically belongs to - # the next window (it is never lost), so the artifact stays - # audit-reproducible for as long as it serves. Validity is event-driven - # (the payout-generation and balances fences); wall-clock time only - # enters through the loose anchor ceiling that backstops how far a - # served window may trail the live ledger, never through install age. - snapshot_anchor_ms: int | None = None - # Canonical digest of shares_json. Cached bundles built before this - # artifact was armed may only keep serving re-keyed lookups when their - # own window digest matches; anything else must rebuild from the - # artifact so a fresher window is never shadowed by older cached work. - share_snapshot_sha256: str | None = None - # Canonical digest of prior_balances, memoized at construction. The - # reuse probe hashes the balances on every serving decision and the - # reused-build fences hash them twice more; the tuple is immutable, so - # the O(accounts) canonicalization is paid once here instead. - prior_balances_sha256: str | None = None - # Internal coordination token. A late-visible append can invalidate a - # previously exact anchored window without changing payout generation. - # Artifacts prepared before that append may never be armed again. - append_invalidation_epoch: int = field( - default=0, - compare=False, - repr=False, - ) - # Exposed in-flight scan anchor carried from the synchronous bundle - # build that seeded this artifact (see _expose_inflight_scan_anchor). - # Nothing else exposes this window's anchor between the ledger read and - # the install fence, so the exposure rides here and the install retires - # it once the fence settles either way. - inflight_scan_anchor_token: int | None = field( - default=None, - compare=False, - repr=False, - ) - # Build-path diagnostics only. They do not participate in artifact - # equality or any signed/wire representation. - window_build_mode: str | None = field(default=None, compare=False, repr=False) - window_delta_rows: int = field(default=0, compare=False, repr=False) - window_expired_rows: int = field(default=0, compare=False, repr=False) - window_touched_pages: int = field(default=0, compare=False, repr=False) - window_build_seconds: float = field(default=0.0, compare=False, repr=False) - window_full_rescan_reason: str | None = field( - default=None, - compare=False, - repr=False, - ) +class _StratumSessionStateField: + """Route one legacy S1 coordinator field to its owner service. -@dataclass(frozen=True) -class _IncrementalPayoutArtifactWindow: - """Coordinator-owned exact window plus its canonical materialization.""" - - window: IncrementalShareWindow - shares_json: IncrementalShareJsonSequence = field(repr=False) - share_snapshot_sha256: str - refreshed_monotonic: float - full_rescan_monotonic: float - full_rescan_attempt_monotonic: float - append_invalidation_epoch: int = 0 - - -@dataclass(frozen=True) -class _PayoutWindowMaterialization: - """One exact share-window materialization and bounded-work metadata.""" - - shares_json: Sequence[dict[str, object]] = field(repr=False) - share_snapshot_sha256: str - snapshot_anchor_ms: int - mode: str - record_count: int - stats: IncrementalWindowAdvanceStats - full_rescan_reason: str | None = None - balance_check_prior_balances: tuple[dict[str, object], ...] | None = field( - default=None, - repr=False, - ) - balance_check_mismatch: bool = False + The descriptor keeps the historical attribute name readable and writable + on the coordinator while :class:`StratumSessionService` (its + ``SessionRegistry`` and ``P2mrAddressValidator``) owns the single mutable + copy, mirroring the S2/J1/G1 extraction pattern. Direct test assignment + through the legacy name is thereby adopted by the owner. + """ + def __init__(self, attribute: str) -> None: + self._attribute = attribute -def _share_window_spool_file() -> Any: - """Anonymous spool file for the serialized share-window payload tail. + def __get__( + self, + instance: "PrismCoordinator | None", + owner: type | None = None, + ) -> object: + if instance is None: + return self + return getattr( + instance._ensure_stratum_session_service(), + self._attribute, + ) - Created in the same temporary filesystem the builder's captured output - and stderr already use, and unlinked from birth: a crashed coordinator - can never strand a multi-megabyte window on disk, and closing the last - descriptor is the entire cleanup story. - """ - return tempfile.TemporaryFile() - - -@dataclass -class _ShareWindowSerialization: - """Derived share-window forms cached per canonical window. - - For a fixed (canonical digest, count, window weight) the serialized share - window is immutable: its canonical digest and the - audit-builder compact payload fragments are pure functions of the - artifact's share snapshot. Recomputing them inside every template-bump - build serialized the whole window under the GIL each time. The compact - fragments are derived lazily by the first audit-builder invocation -- - embedders that replace the builder never require the compact schema. - - The fragments' invariant byte encoding is additionally spooled to an - anonymous temp file once per key, so every build after the first can feed - the audit builder's stdin through kernel transfers instead of re-writing - megabytes of unchanged JSON through Python. Spool access is leased: - rotation to a new generation retires the spool, and the descriptor closes - when the last in-flight transfer releases it, never underneath one. - """ + def __set__(self, instance: "PrismCoordinator", value: object) -> None: + setattr( + instance._ensure_stratum_session_service(), + self._attribute, + value, + ) - key: tuple[str, int, int] - share_count: int - share_snapshot_sha256: str - _source_artifact: PayoutLedgerArtifact | None = field( - default=None, - repr=False, - compare=False, - ) - _compact_lock: threading.Lock = field( - default_factory=threading.Lock, - repr=False, - ) - _compact_share_identities_json: str | None = field( - default=None, - repr=False, - ) - _compact_shares_json: str | None = field(default=None, repr=False) - _spool_lock: threading.Lock = field( - default_factory=threading.Lock, - repr=False, - ) - _spool_file: Any = field(default=None, repr=False) - _spool_size: int = field(default=0, repr=False) - _spool_failed: bool = field(default=False, repr=False) - _spool_retired: bool = field(default=False, repr=False) - _spool_leases: int = field(default=0, repr=False) - def compact_fragments( - self, - shares: list[dict[str, object]], - ) -> tuple[str, str]: - """Encoded compact fragments, computed once per generation. +class _CoordinatorObservability(ObservabilityPort): + """Collect immutable health inputs without exporting coordinator state.""" - Concurrent builders block here instead of duplicating the encode; the - window is immutable for this key, so first-writer-wins is exact. - """ - with self._compact_lock: - if ( - self._compact_share_identities_json is None - or self._compact_shares_json is None - ): - identities, compact_shares = _compact_share_payload(shares) - self._compact_share_identities_json = json.dumps( - identities, - separators=(",", ":"), - ) - self._compact_shares_json = json.dumps( - compact_shares, - separators=(",", ":"), - ) - return ( - self._compact_share_identities_json, - self._compact_shares_json, - ) + def __init__(self, coordinator: "PrismCoordinator") -> None: + self.coordinator = coordinator - def acquire_spooled_tail( - self, - shares: list[dict[str, object]], - ) -> tuple[Any, int] | None: - """Lease the spooled payload tail, writing it on first use. + def monotonic(self) -> float: + return time.monotonic() - Returns the open spool file and its byte size, or None when spooling - is unavailable (creation failed earlier, a transfer poisoned it, or - the generation was retired). The caller must pair a successful lease - with release_spooled_tail once its transfer is finished. - """ - with self._spool_lock: - if self._spool_failed or self._spool_retired: - return None - if self._spool_file is None: - identities_json, compact_shares_json = self.compact_fragments( - shares - ) - spool = None - try: - spool = _share_window_spool_file() - spool.write(b',"compact_share_identities":') - spool.write(identities_json.encode("utf-8")) - spool.write(b',"compact_shares":') - spool.write(compact_shares_json.encode("utf-8")) - spool.write(b"}") - spool.flush() - size = spool.seek(0, os.SEEK_END) - except (OSError, ValueError): - # Spooling is an optimization; the in-memory pipe path - # remains authoritative. Failure is sticky so a broken - # temp filesystem is not retried on every build. - self._spool_failed = True - if spool is not None: - try: - spool.close() - except OSError: - pass - return None - self._spool_file = spool - self._spool_size = int(size) - self._spool_leases += 1 - return self._spool_file, self._spool_size - - def release_spooled_tail(self) -> None: - with self._spool_lock: - self._spool_leases -= 1 - self._close_spool_if_unused_locked() - - def mark_spool_failed(self) -> None: - """Poison the spool after a transfer-side failure (still leased).""" - with self._spool_lock: - self._spool_failed = True - - def retire_spool(self) -> None: - """Stop new leases and close the file once in-flight ones finish.""" - with self._spool_lock: - self._spool_retired = True - self._close_spool_if_unused_locked() - - def _close_spool_if_unused_locked(self) -> None: - if ( - self._spool_leases > 0 - or self._spool_file is None - or not (self._spool_retired or self._spool_failed) - ): - return - spool, self._spool_file = self._spool_file, None - self._spool_size = 0 - try: - spool.close() - except OSError: - pass - - -# JSONL protocol version this coordinator speaks with the --serve audit -# builder. The daemon announces its version in a startup handshake; any -# mismatch retires the daemon and every build falls back to one-shot mode. -PRISM_SERVE_BUILDER_PROTOCOL_VERSION = 1 -# Parsed share windows the --serve daemon retains; the coordinator mirrors -# the bound to predict which uploads the daemon still holds. -PRISM_SERVE_BUILDER_WINDOW_CACHE_ENTRIES = 2 - - -class _ServeBuilderUnavailable(RuntimeError): - """Daemon anomaly; the build must transparently use one-shot mode.""" - - -@dataclass -class _ServeBuilderClient: - """Line-oriented I/O state for one long-lived --serve audit builder.""" + def mining_delivery_inputs(self, now: float) -> MiningDeliveryInputs: + coordinator = self.coordinator + coordinator._ensure_initial_job_state() + # The first-job admission domain is owned by its own lock in the S2 + # service. Copy it once, before the coordinator lock is taken, so this + # refresh never holds both locks and never reads a half-committed + # replacement (#159). + admission = ( + coordinator._ensure_job_delivery_service() + .initial_job_admission_snapshot() + ) + pending_jobs = admission.pending + with coordinator.lock: + active = len(coordinator.clients) + current_tip = coordinator._current_published_tip_hash_locked() + published_snapshot = getattr(coordinator, "tip_template_snapshot", None) + subscribed = sum( + 1 for client in coordinator.clients if client.subscribed + ) + authorized_clients = [ + client + for client in coordinator.clients + if client.subscribed + and client.authorized + and client.worker is not None + ] + authorized = len(authorized_clients) + clients_with_current_work = [ + client + for client in authorized_clients + if coordinator._client_has_current_tip_job_locked(client) + ] + current = len(clients_with_current_work) + # Semantic currency alongside the strict gauge above: template + # fingerprint and payout generation, plus the connection, + # authorization, and difficulty generations that make the job + # usable by this client. Template-generation and object-identity + # churn remain intentionally excluded. The strict predicate reads + # 0 whenever a template generation was minted since the last + # fanout; this gauge instead answers whether miners actually hold + # usable work for the current template content. + payout_generation_now = int( + getattr(coordinator, "_payout_state_generation", 0) + ) + semantic_current = sum( + 1 + for client in authorized_clients + if client.active_job is not None + and published_snapshot is not None + and getattr(client.active_job, "template_fingerprint", None) + == published_snapshot.template_fingerprint + and int( + getattr(client.active_job, "payout_state_generation", -1) + ) + == payout_generation_now + and int( + getattr( + client.active_job, + "connection_id", + client.connection_id, + ) + ) + == client.connection_id + and int( + getattr(client.active_job, "authorization_generation", 0) + ) + == int(getattr(client, "authorization_generation", 0)) + and int( + getattr(client.active_job, "difficulty_generation", 0) + ) + == int(getattr(client, "difficulty_generation", 0)) + ) + clients_with_no_active_job = sum( + 1 for client in authorized_clients if client.active_job is None + ) + pending_requests = list(pending_jobs.values()) + pending = len(pending_requests) + oldest_age = max( + ( + max(0.0, now - request.requested_monotonic) + for request in pending_requests + ), + default=0.0, + ) + genuinely_pending_initial_clients = [ + client + for client in authorized_clients + if not JobDeliveryService._client_has_delivered_work_locked(client) + ] + genuine_initial_started = [ + started + for client in genuinely_pending_initial_clients + for started in ( + client.authorized_monotonic, + ( + pending_jobs[client].requested_monotonic + if client in pending_jobs + else None + ), + ) + if started is not None + ] + oldest_genuine_initial_age = max( + (max(0.0, now - started) for started in genuine_initial_started), + default=0.0, + ) + connection_limit = int( + getattr( + coordinator, + "stratum_max_connections", + DEFAULT_PRISM_STRATUM_MAX_CONNECTIONS, + ) + ) + pending_limit = int(coordinator.stratum_max_pending_initial_jobs) + timeout = float(coordinator.stratum_initial_job_timeout_seconds) + timeout_disconnects = admission.timeout_disconnects + queue_rejections = admission.queue_rejections + cancelled = admission.cancelled + coalesced = admission.coalesced + queue_capacity_reclaimed = admission.queue_capacity_reclaimed + peak = coordinator.peak_active_connection_count + handlers = coordinator.handler_thread_count + last_delivery = admission.last_delivery_monotonic + + # This census is the fleet's only current-tip coverage scan: job + # delivery used to repeat it per delivered job under the coordinator + # lock, which made every reconnect herd O(N^2) inside the lock the herd + # was already queued on (#159). Restored coverage resets the delivery + # failure here, after the snapshot and outside the lock. + if authorized == 0 or current / authorized >= 0.95: + coordinator._ensure_observability_service().reset_delivery_failure() + + coordinator._ensure_job_cache_state() + with coordinator._job_cache_lock: + prepared_bundle = coordinator._prepared_ready_bundle + prepared_snapshot = coordinator._prepared_ready_snapshot + preparation_pending = bool(coordinator.job_preparation_pending) + payout_generation = int(coordinator._payout_state_generation) + prepared_current = bool( + prepared_bundle is not None + and prepared_snapshot is published_snapshot + and published_snapshot is not None + and current_tip is not None + and published_snapshot.bestblockhash == current_tip + and published_snapshot.template_artifacts is not None + and not prepared_bundle.collection_only + and prepared_bundle.template + is published_snapshot.template_artifacts.template + and prepared_bundle.template_fingerprint + == published_snapshot.template_fingerprint + and prepared_bundle.template_generation + == published_snapshot.template_generation + and prepared_bundle.payout_state_generation == payout_generation + ) + rejection_counts = getattr(coordinator, "rejection_counts_by_reason", {}) + stale_unknown = int( + rejection_counts.get(PRISM_REJECTION_STALE_JOB, 0) + ) + int(rejection_counts.get(PRISM_REJECTION_UNKNOWN_JOB, 0)) + executor = getattr(coordinator, "_tip_refresh_executor", None) + queue_depth, active_workers = ( + executor.stats() if executor is not None else (0, 0) + ) + return MiningDeliveryInputs( + active_connections=active, + connection_capacity=connection_limit, + peak_active_connections=peak, + subscribed_connections=subscribed, + authorized_connections=authorized, + pending_initial_jobs=pending, + pending_initial_job_capacity=pending_limit, + oldest_pending_initial_job_age_seconds=oldest_age, + oldest_genuinely_pending_initial_job_age_seconds=( + oldest_genuine_initial_age + ), + clients_with_current_tip_jobs=current, + clients_with_semantically_current_work=semantic_current, + clients_with_no_active_job=clients_with_no_active_job, + last_initial_job_delivery_monotonic=last_delivery, + initial_job_timeout_seconds=timeout, + initial_job_queue_rejections=queue_rejections, + initial_job_timeout_disconnects=timeout_disconnects, + initial_job_cancelled_tasks=cancelled, + initial_job_coalesced_tasks=coalesced, + initial_job_queue_capacity_reclaimed=queue_capacity_reclaimed, + handler_threads=handlers, + delivery_executor_queue_depth=queue_depth, + delivery_executor_active_workers=active_workers, + started_monotonic=float(getattr(coordinator, "started_monotonic", now)), + startup_grace_seconds=float( + getattr( + coordinator, + "mining_health_startup_grace_seconds", + DEFAULT_PRISM_MINING_HEALTH_STARTUP_GRACE_SECONDS, + ) + ), + stale_unknown_rejections=stale_unknown, + submitted_shares=int(getattr(coordinator, "submitted_share_count", 0)), + job_preparation_pending=preparation_pending, + current_observed_tip=current_tip, + prepared_bundle_current=prepared_current, + prepared_bundle_tip=( + prepared_snapshot.bestblockhash + if prepared_snapshot is not None + else None + ), + prepared_bundle_template_generation=( + prepared_bundle.template_generation + if prepared_bundle is not None + else None + ), + prepared_bundle_payout_generation=( + prepared_bundle.payout_state_generation + if prepared_bundle is not None + else None + ), + ) - process: subprocess.Popen[bytes] - stdout_buffer: bytearray = field(default_factory=bytearray, repr=False) - # Mirrors the daemon's bounded LRU so requests can predict whether the - # window must ride along. Divergence is repaired by the daemon's - # needs_window response, never trusted blindly. - uploaded_windows: OrderedDict[str, None] = field( - default_factory=OrderedDict, - repr=False, - ) + def accepted_share_stats(self) -> tuple[int, int]: + return self.coordinator.accepted_share_stats() - def note_uploaded_window(self, share_snapshot_sha256: str) -> None: - self.uploaded_windows.pop(share_snapshot_sha256, None) - self.uploaded_windows[share_snapshot_sha256] = None - while len(self.uploaded_windows) > PRISM_SERVE_BUILDER_WINDOW_CACHE_ENTRIES: - self.uploaded_windows.popitem(last=False) - - def close(self) -> None: - # Tolerates lightweight process fakes used by embedders and tests, - # which do not necessarily expose the full Popen surface. - process = self.process - poll = getattr(process, "poll", None) - wait = getattr(process, "wait", None) - if callable(poll): - if poll() is None: - try: - process.kill() - except (ProcessLookupError, OSError, AttributeError): - pass - if callable(wait): - try: - wait(timeout=5.0) - except ( - subprocess.TimeoutExpired, - OSError, - TypeError, - ValueError, - AttributeError, - ): - pass - for stream in ( - getattr(process, "stdin", None), - getattr(process, "stdout", None), - ): - if stream is None or not hasattr(stream, "close"): - continue - try: - stream.close() - except OSError: - pass + def ledger_backend(self) -> str: + return str(self.coordinator.ledger.backend_name) + def block_counts(self) -> tuple[int, int]: + coordinator = self.coordinator + lock = getattr(coordinator, "lock", None) + if lock is None: + return int(coordinator.accepted_block_count), int(coordinator.max_blocks) + with lock: + return int(coordinator.accepted_block_count), int(coordinator.max_blocks) -@dataclass -class _PayoutDeliveryAdmission: - admitted: bool - wait_seconds: float - generation: int - published_generation: int - relation: str - delivered: bool = False + def progress_health(self) -> Mapping[str, object]: + return self.coordinator.progress_health_snapshot() - def __bool__(self) -> bool: - return self.admitted + def health_refresh_seconds(self) -> float: + return float( + getattr( + self.coordinator, + "health_refresh_seconds", + DEFAULT_PRISM_HEALTH_REFRESH_SECONDS, + ) + ) - def mark_delivered(self) -> None: - if not self.admitted: - raise RuntimeError("payout delivery completed without admission") - self.delivered = True + def render_metrics_payload(self) -> str: + return self.coordinator._render_metrics_payload() + def metrics_refresh_seconds(self) -> float: + return float( + getattr( + self.coordinator, + "metrics_refresh_seconds", + DEFAULT_PRISM_METRICS_REFRESH_SECONDS, + ) + ) -@dataclass(eq=False) -class PendingInitialJob: - client: ClientState - authorization_generation: int - worker: WorkerIdentity - requested_monotonic: float - deadline_monotonic: float | None - connection_id: int | None = None - difficulty_generation: int | None = None - cancelled: threading.Event = field(default_factory=threading.Event) - future: Future[bool] | None = None - predecessor: Future[bool] | None = None + def record_startup_phase(self, phase: str) -> None: + # The startup-phase recorder stays coordinator runtime state; owners + # stamp through this dynamic seam (upstream #120, issue #188). + self.coordinator._record_startup_phase_once(phase) + def stop_requested(self) -> bool: + return self.coordinator.stop_event.is_set() -class _DeliveryQueueFull(RuntimeError): - """The bounded delivery executor cannot admit another task.""" + def wait_for_stop(self, timeout: float) -> bool: + return self.coordinator.stop_event.wait(timeout) + def log(self, message: str) -> None: + print(message, flush=True) -class _JobBuildCancelled(RuntimeError): - """A bundle waiter became obsolete before it acquired preparation.""" + def log_exception(self) -> None: + traceback.print_exc() -class _BoundedPriorityExecutor: - """Small Future-compatible executor with bounded, priority-ordered work.""" +class _CoordinatorAuditHttp(AuditHttpPort): + """Expose purpose-specific dynamic reads to the HTTP facade.""" def __init__( self, + coordinator: "PrismCoordinator", *, - max_workers: int, - max_queue_size: int, - thread_name_prefix: str = "prism-job-delivery", + allow_uncached_compatibility: bool = False, ) -> None: - self.max_workers = max_workers - self.max_queue_size = max_queue_size - self._queue: queue.PriorityQueue[tuple[object, ...]] = queue.PriorityQueue( - maxsize=max_queue_size - ) - self._lock = threading.Lock() - self._sequence = 0 - self._active_workers = 0 - self._shutdown = False - self._threads = [ - threading.Thread( - target=self._worker, - name=f"{thread_name_prefix}-{index + 1}", - daemon=True, - ) - for index in range(max_workers) - ] - for thread in self._threads: - thread.start() + self.coordinator = coordinator + self.allow_uncached_compatibility = allow_uncached_compatibility + + def cached_health_payload(self) -> tuple[int, Mapping[str, object]]: + cached = getattr(self.coordinator, "cached_health_payload", None) + if callable(cached): + return cached() + if self.allow_uncached_compatibility: + return 200, self.coordinator.health_payload() + raise RuntimeError("cached health is unavailable") + + def cached_metrics_payload(self) -> MetricsSnapshotResponse: + cached = getattr(self.coordinator, "cached_metrics_payload", None) + if callable(cached): + response = self._as_metrics_response(cached()) + if response.status != 503 or not self.allow_uncached_compatibility: + # Carried through whole rather than unpacked: the freshness + # state and age this read decided are what /metrics publishes + # out of band (issue #184), and rebuilding them here from the + # status alone would lose the fresh/stale distinction that + # both now answer 200. + return response + if self.allow_uncached_compatibility: + # Legacy make_audit_handler path only: no refresher owns the + # snapshot, so this renders inline and the result is by definition + # freshly collected. + return MetricsSnapshotResponse( + status=200, + body=self.coordinator.metrics_payload(), + state=METRICS_STATE_FRESH, + age_seconds=0, + ) + raise RuntimeError("cached metrics are unavailable") - def submit( - self, - function: Callable[..., Any], - /, - *args: object, - priority: int = PRISM_DELIVERY_PRIORITY_SAME_TIP, - **kwargs: object, - ) -> Future[Any]: - future: Future[Any] = Future() - with self._lock: - if self._shutdown: - raise RuntimeError("delivery executor is shut down") - self._sequence += 1 - item = ( - int(priority), - self._sequence, - future, - function, - args, - kwargs, - ) - try: - self._queue.put_nowait(item) - except queue.Full as exc: - raise _DeliveryQueueFull("delivery executor queue is full") from exc - return future - - def _worker(self) -> None: - while True: - item = self._queue.get() - _, _, future, function, args, kwargs = item - if function is None: - self._queue.task_done() - return - assert isinstance(future, Future) - if not future.set_running_or_notify_cancel(): - self._queue.task_done() - continue - with self._lock: - self._active_workers += 1 - try: - result = function(*args, **kwargs) - except BaseException as exc: - future.set_exception(exc) - else: - future.set_result(result) - finally: - with self._lock: - self._active_workers -= 1 - self._queue.task_done() - - def cancel(self, future: Future[Any]) -> bool: - """Cancel ``future`` and immediately discard it when still queued. - - ``Future.cancel`` alone leaves the cancelled entry in PriorityQueue - until a worker dequeues it. Removing the exact entry while holding the - queue mutex makes bounded admission available to a replacement at the - cancellation boundary. A worker that already dequeued the entry owns - the normal ``task_done`` path, so the two paths cannot double-release. - """ - removed = False - with self._queue.mutex: - queued_items = self._queue.queue - for index, item in enumerate(queued_items): - if item[2] is not future: - continue - queued_items.pop(index) - heapq.heapify(queued_items) - self._queue.unfinished_tasks -= 1 - if self._queue.unfinished_tasks == 0: - self._queue.all_tasks_done.notify_all() - self._queue.not_full.notify() - removed = True - break - # Invoke callbacks only after releasing the queue mutex. Initial-job - # cancellation callbacks may submit a replacement to this executor. - future.cancel() - return removed - - def stats(self) -> tuple[int, int]: - with self._lock: - return self._queue.qsize(), self._active_workers - - def shutdown(self, *, wait: bool = True, cancel_futures: bool = False) -> None: - with self._lock: - if self._shutdown: - threads = list(self._threads) - already_shutdown = True - else: - self._shutdown = True - threads = list(self._threads) - already_shutdown = False - if already_shutdown: - if wait: - for thread in threads: - thread.join() - return - if cancel_futures: - while True: - try: - item = self._queue.get_nowait() - except queue.Empty: - break - future = item[2] - if isinstance(future, Future): - future.cancel() - self._queue.task_done() - for index in range(len(threads)): - self._queue.put((math.inf, index, None, None, (), {})) - if wait: - for thread in threads: - thread.join() - - -class _FanoutCancellation: - """Cancel a fanout without racing already-admitted deliveries. - - ``cancel`` closes admission without waiting, so workers can call it while - holding a client lock. The fanout coordinator calls ``set`` outside client - locks to wait for deliveries that already passed the final gate. - """ + @staticmethod + def _as_metrics_response(cached: object) -> MetricsSnapshotResponse: + """Accept the response type, or adapt a legacy ``(status, body)`` pair. - def __init__(self) -> None: - self._condition = threading.Condition() - self._cancelling = False - self._active_deliveries = 0 + Coordinator doubles predating the #184 response type still answer with + the bare pair. A pair carries no freshness beyond its status, so the + state is derived from exactly that and nothing is invented. + """ - def is_set(self) -> bool: - with self._condition: - return self._cancelling + if isinstance(cached, MetricsSnapshotResponse): + return cached + status, body = cached # type: ignore[misc] + return MetricsSnapshotResponse( + status=int(status), + body=str(body), + state=( + METRICS_STATE_UNAVAILABLE + if int(status) == 503 + else METRICS_STATE_FRESH + ), + age_seconds=None if int(status) == 503 else 0, + ) - def begin_delivery(self) -> bool: - with self._condition: - if self._cancelling: - return False - self._active_deliveries += 1 - return True + def latest_evidence_payload(self) -> Mapping[str, object] | None: + return self.coordinator.latest_evidence_payload() - def end_delivery(self) -> None: - with self._condition: - if self._active_deliveries <= 0: - raise RuntimeError("fanout delivery gate released without admission") - self._active_deliveries -= 1 - if self._active_deliveries == 0: - self._condition.notify_all() - - def cancel(self) -> None: - with self._condition: - self._cancelling = True - - def set(self) -> None: - self.cancel() - with self._condition: - while self._active_deliveries: - self._condition.wait() - - -@dataclass -class _JobBundleBuildControl: - key: tuple[object, ...] - previousblockhash: str - payout_state_generation: int - payout_artifact_generation: int - cancel_event: threading.Event = field(default_factory=threading.Event) - process: subprocess.Popen[str] | None = None - - -class _PayoutStateDeliveryGate: - """Order delivery admission around a very short payout publication. - - A publisher first closes admission and drains sends that already crossed - the boundary. It does not own the atomic publication section while that - drain is in progress. Once drained, publication ownership is transferred - to the caller for the generation/cache pointer swap only. - """ + def owed_balances_payload(self) -> Mapping[str, object]: + return self.coordinator.owed_balances_payload() - def __init__(self) -> None: - self._condition = threading.Condition() - self._active_deliveries = 0 - self._publisher_waiting = False - self._mutation_owner: int | None = None - self._mutation_depth = 0 - self._published_generation = 0 - self._priority_generation: int | None = None - self._delivery_blocked = False + def carry_forward_integrity_payload(self) -> Mapping[str, object]: + return self.coordinator.carry_forward_integrity_payload() - @staticmethod - def _generation_relation(generation: int, published_generation: int) -> str: - if generation < published_generation: - return "stale" - if generation > published_generation: - return "future" - return "current" + def miner_status_payload(self, recipient_id: str) -> Mapping[str, object]: + return self.coordinator.miner_status_payload(recipient_id) - @contextmanager - def delivery(self) -> Iterator[None]: - with self.delivery_cancelable(lambda: False, priority=True) as admission: - if not admission: - raise RuntimeError("uncancelled payout delivery was not admitted") - yield - admission.mark_delivered() + def ledger_backend(self) -> str: + return str(self.coordinator.ledger.backend_name) - @contextmanager - def delivery_cancelable( + def audit_share_window( self, - cancelled: Callable[[], bool], *, - generation: int | None = None, - priority: bool = False, - poll_seconds: float = PRISM_TIP_REFRESH_ADMISSION_POLL_SECONDS, - ) -> Iterator[_PayoutDeliveryAdmission]: - """Admit a delivery unless cancellation wins while mutation owns the gate.""" - - started = time.monotonic() - admitted = False - with self._condition: - if generation is None: - generation = self._published_generation - while True: - if cancelled(): - break - if self._delivery_blocked: - break - if generation < self._published_generation: - break - publication_blocked = ( - self._publisher_waiting or self._mutation_owner is not None - ) - priority_blocked = ( - self._priority_generation is not None - and generation == self._priority_generation - and not priority - ) - if priority_blocked: - # A same-generation job that is not for the published tip - # must not occupy a waiter slot indefinitely. Reject it so - # its caller can rebuild current-tip work; the reserved - # first-delivery lane remains available to priority work. - break - future_blocked = generation > self._published_generation - if ( - not publication_blocked - and not future_blocked - ): - self._active_deliveries += 1 - admitted = True - break - self._condition.wait(timeout=poll_seconds) - published_generation = self._published_generation - relation = self._generation_relation(generation, published_generation) - admission = _PayoutDeliveryAdmission( - admitted=admitted, - wait_seconds=max(0.0, time.monotonic() - started), - generation=generation, - published_generation=published_generation, - relation=relation, - ) - try: - yield admission - finally: - if admitted: - with self._condition: - if self._active_deliveries <= 0: - raise RuntimeError("payout delivery gate released without admission") - self._active_deliveries -= 1 - if ( - priority - and admission.delivered - and generation == self._priority_generation - ): - # Keep routine same-generation sends queued until the - # first prioritized current-tip socket delivery exits. - # Privileged synchronization admissions do not consume - # this reservation merely by leaving the gate. - self._priority_generation = None - if self._active_deliveries == 0: - self._condition.notify_all() - elif self._priority_generation is None: - self._condition.notify_all() + anchor_job_issued_at_ms: int, + network_difficulty: int, + ) -> list[dict[str, object]]: + return self.coordinator.ledger.audit_share_window( + anchor_job_issued_at_ms=anchor_job_issued_at_ms, + network_difficulty=network_difficulty, + ) - @contextmanager - def publication(self) -> Iterator[None]: - owner = threading.get_ident() - with self._condition: - if self._mutation_owner == owner: - self._mutation_depth += 1 - else: - while self._mutation_owner is not None or self._publisher_waiting: - self._condition.wait() - self._publisher_waiting = True - while self._active_deliveries: - self._condition.wait() - self._mutation_owner = owner - self._mutation_depth = 1 - self._publisher_waiting = False - try: - yield - finally: - with self._condition: - if self._mutation_owner != owner or self._mutation_depth <= 0: - raise RuntimeError("payout mutation gate released by non-owner") - self._mutation_depth -= 1 - if self._mutation_depth == 0: - self._mutation_owner = None - self._condition.notify_all() - - def publish_generation(self, generation: int, *, prioritize_delivery: bool) -> None: - owner = threading.get_ident() - with self._condition: - if self._mutation_owner != owner: - raise RuntimeError("payout generation published outside atomic section") - if generation <= self._published_generation: - raise RuntimeError("payout generation did not advance") - self._published_generation = generation - self._priority_generation = generation if prioritize_delivery else None - self._delivery_blocked = False - - def block_delivery( + def audit_block_payouts( self, - mark_blocked: Callable[[], bool] | None = None, - ) -> bool: - """Reject admission until publication, atomically with caller state.""" - - with self._condition: - # A publisher drops the condition while swapping its immutable - # pointer. Wait only for that short section, never for admitted - # socket sends or fanout cancellation. - while self._mutation_owner is not None: - self._condition.wait() - if mark_blocked is not None and not mark_blocked(): - return False - self._delivery_blocked = True - self._condition.notify_all() - return True - - @contextmanager - def mutation(self) -> Iterator[None]: - """Compatibility alias for tests and callers that only need exclusion.""" - - with self.publication(): - yield + *, + block_hash: str, + ) -> list[dict[str, object]]: + return self.coordinator.ledger.audit_block_payouts(block_hash=block_hash) + def audit_ctv_fanouts( + self, + *, + block_hash: str, + ) -> list[dict[str, object]]: + return self.coordinator.ledger.audit_ctv_fanouts(block_hash=block_hash) -@dataclass -class _SharedBundlePreparationFlight: - event: threading.Event = field(default_factory=threading.Event) - result: CachedJobBundle | None = None - error: BaseException | None = None - waiters: int = 0 - - -@dataclass(eq=False) -class ClientState: - sock: socket.socket - address: tuple[str, int] - connection_id: int - extranonce1_hex: str - subscribed: bool = False - authorized: bool = False - authorization_generation: int = 0 - difficulty_generation: int = 0 - authorized_monotonic: float | None = None - username: str = "" - worker: WorkerIdentity | None = None - version_mask: int = 0 - active_job: PrismJobContext | None = None - # active_job is registered before the potentially blocking socket write. - # These delivery-proof fields advance only after send_job_update succeeds. - _progress_delivered_context: PrismJobContext | None = None - _progress_delivered_template_fingerprint: str | None = None - _progress_delivered_template_generation: int = 0 - _progress_delivered_payout_generation: int = -1 - _progress_delivered_monotonic: float | None = None - # Admission ordering only. Delivery proof remains in the progress fields - # above and advances exclusively after the socket write succeeds. - _tip_refresh_admitted_epoch_sequence: int = 0 - listener_name: str = "default" - # Pristine difficulty policy of the accepting listener; never mutated. - listener_vardiff_config: vardiff.VardiffConfig | None = None - # Floor below which stamped jobs never advertise, copied from the - # accepting listener profile. Zero (default listener) keeps the network - # cap authoritative. - minimum_advertised_difficulty: Decimal = Decimal("0") - # Per-client specialization of the listener policy (password d=/md= or - # mining.suggest_difficulty); recomputed from the pristine base on every - # request so repeat applications cannot compound. - vardiff_config: vardiff.VardiffConfig | None = None - requested_difficulty: Decimal | None = None - requested_min_difficulty: Decimal | None = None - suggested_difficulty: Decimal | None = None - share_difficulty: Decimal = Decimal("1") - pending_share_difficulty: Decimal | None = None - vardiff_window_started_monotonic: float = field(default_factory=time.monotonic) - vardiff_window_accepted: int = 0 - vardiff_window_submitted: int = 0 - vardiff_window_work: Decimal = Decimal("0") - vardiff_difficulty_estimate: Decimal | None = None - # Owns all per-client difficulty policy, estimates, and Vardiff windows. - # Share handlers release it before acquiring job_update_lock or the - # coordinator lock; job delivery may acquire it while job_update_lock is - # held. Keeping this lock per connection prevents one miner's accounting - # or slow socket from convoying tip publication for the whole coordinator. - vardiff_lock: threading.RLock = field(default_factory=threading.RLock) - active_job_ids: set[str] = field(default_factory=set) - post_accept_refresh_block: tuple[int, str] | None = None - # (job previousblockhash, monotonic) of the FIRST job this connection was - # sent for that tip. Anchors the per-connection stale-grace window: a - # prior-tip share is in flight until shortly after this connection - # received replacement work, however long the refresh pass took to reach - # it. See stale_grace_deadline_open. - tip_work_delivered: tuple[str, float] | None = None - # Protected by the coordinator lock. Disconnect retirement sets this before - # waiting for any per-client job update so queued work can reject the client. - closing: bool = False - # Stamped by the handler thread when a request line arrives, before JSON - # parsing. Only that thread reads it; mining.submit uses it to observe - # read-to-ack latency (the share-ingest saturation instrument). - request_received_monotonic: float | None = None - # Serializes every job build/register/send transition for this connection. - # The coordinator lock may be acquired while this lock is held, never in - # the reverse order. RLock permits authorize/retarget helpers to call the - # common maybe_send_job path while retaining the same serialization scope. - job_update_lock: threading.RLock = field(default_factory=threading.RLock) - send_lock: threading.Lock = field(default_factory=threading.Lock) - handler_thread_registered: bool = False - - def send(self, payload: dict[str, object]) -> None: - data = json.dumps(payload).encode() + b"\n" - with self.send_lock: - self.sock.sendall(data) - - def send_batch( + def audit_ctv_fanout_manifest_set( self, - payloads: list[dict[str, object]], *, - preserialized: bytes | None = None, - ) -> None: - # Tests and embedders may replace ``send`` with an in-memory recorder; - # retain that seam while production sockets write the whole difficulty - # + notify pair under one send lock with no response interleaving. - # ``preserialized`` must be the exact serialization of ``payloads``; - # callers with a shared precomposed fragment pass it so a fleet-wide - # wave does not re-serialize identical coinbase parts per client. - if "send" in self.__dict__: - for payload in payloads: - self.send(payload) - return - data = ( - preserialized - if preserialized is not None - else b"".join( - json.dumps(payload).encode() + b"\n" for payload in payloads - ) + block_hash: str, + ) -> Mapping[str, object] | None: + return self.coordinator.ledger.audit_ctv_fanout_manifest_set( + block_hash=block_hash ) - with self.send_lock: - self.sock.sendall(data) - - def close(self) -> None: - try: - self.sock.shutdown(socket.SHUT_RDWR) - except OSError: - pass - try: - self.sock.close() - except OSError: - pass - -class StratumError(RuntimeError): - def __init__( + def ctv_fanout_status( self, - code: int, - message: str, *, - reason: str | None = None, - disconnect: bool = False, - ): - super().__init__(message) - self.code = code - self.message = message - self.reason = reason - self.disconnect = disconnect - - -class ShutdownInProgress(RuntimeError): - """Raised when work that could mutate the ledger arrives after shutdown.""" - - -class _WriterOperationToken: - """One transferable writer admission held until durable work completes.""" - - def __init__(self, controller: "CoordinatorShutdownController", component: str): - self.controller = controller - self.component = component - self.finished = False - - def finish(self) -> None: - self.controller.finish_token(self) - - -class CoordinatorShutdownController: - """Coordinates the writer barrier, one-shot lease release, and final drain. - - Writer operations enter through :meth:`enter_writer` before shutdown or - inherit an already-admitted operation on the same thread. Queue admissions - use transferable tokens so a share remains visible to the barrier while it - moves from a client thread to the group-commit writer. - """ - - def __init__(self, writer_quiescence_timeout_seconds: float): - self.writer_quiescence_timeout_seconds = writer_quiescence_timeout_seconds - self.condition = threading.Condition(threading.RLock()) - self.local = threading.local() - self.phase = "running" - self.reason: str | None = None - self.signal_number: int | None = None - self.sigterm_monotonic: float | None = None - self.shutdown_started_monotonic: float | None = None - self.active_writers: dict[str, int] = {} - self.shutdowns_total = 0 - self.writer_quiescence_outcomes = {"success": 0, "timeout": 0} - self.writer_quiescence_seconds = 0.0 - self.lease_release_attempts_total = 0 - self.lease_release_outcomes = { - "success": 0, - "not_held": 0, - "unsupported": 0, - "failure": 0, - } - self.lease_release_seconds = 0.0 - self.lease_release_attempted = False - self.lease_release_succeeded = False - self.lease_release_withheld = False - self.sigterm_to_lease_release_seconds = 0.0 - self.sigterm_release_observed = False - self.release_withheld_total = 0 - self.non_writer_drain_seconds = 0.0 - self.non_writer_drains_total = 0 - self._drain_claimed = False - - def request_shutdown(self, signum: int | None) -> None: - """Close admission atomically; the caller only needs to set its event.""" - now = time.monotonic() - with self.condition: - if signum == signal.SIGTERM and self.sigterm_monotonic is None: - self.sigterm_monotonic = now - if self.signal_number is None and signum is not None: - self.signal_number = signum - if self.phase == "running": - self.phase = "requested" - self.condition.notify_all() - - def begin_shutdown(self, reason: str) -> bool: - with self.condition: - if self.phase not in {"running", "requested"}: - return False - self.phase = "quiescing_writers" - self.reason = reason - self.shutdown_started_monotonic = time.monotonic() - self.shutdowns_total += 1 - self.condition.notify_all() - return True + fanout_txid: str, + ) -> Mapping[str, object] | None: + return self.coordinator.ledger.ctv_fanout_status(fanout_txid=fanout_txid) - def wait_for_lease_handling(self) -> bool: - """Wait for the one shutdown owner to release or safely withhold.""" - in_progress = { - "requested", - "quiescing_writers", - "writers_quiesced", - "releasing_lease", - } - with self.condition: - while self.phase in in_progress: - self.condition.wait() - return self.lease_release_succeeded - - def _thread_writer_depth(self) -> int: - return int(getattr(self.local, "writer_depth", 0)) - - def _admit_writer_locked(self, component: str, *, inherited: bool) -> _WriterOperationToken: - if self.lease_release_attempted: - raise ShutdownInProgress("PRISM writer lease release has already started") - if self.phase != "running" and not inherited: - raise ShutdownInProgress("PRISM coordinator is shutting down") - self.active_writers[component] = self.active_writers.get(component, 0) + 1 - return _WriterOperationToken(self, component) - - def enter_writer( + def pending_ctv_fanout_statuses( self, - component: str, *, - wait_callback: Callable[[], None] | None = None, - ) -> _WriterOperationToken: - depth = self._thread_writer_depth() - if wait_callback is None: - with self.condition: - token = self._admit_writer_locked(component, inherited=depth > 0) - else: - while not self.condition.acquire( - timeout=BLOCK_SUBMITTER_WAIT_HEARTBEAT_SLICE_SECONDS - ): - wait_callback() - try: - token = self._admit_writer_locked(component, inherited=depth > 0) - finally: - self.condition.release() - self.local.writer_depth = depth + 1 - return token - - def exit_writer(self, token: _WriterOperationToken) -> None: - depth = self._thread_writer_depth() - self.local.writer_depth = max(0, depth - 1) - token.finish() - - def reserve_writer(self, component: str) -> _WriterOperationToken: - """Reserve work that will finish on another thread.""" - with self.condition: - return self._admit_writer_locked( - component, - inherited=self._thread_writer_depth() > 0, - ) - - def finish_token(self, token: _WriterOperationToken) -> None: - with self.condition: - if token.finished: - return - token.finished = True - remaining = self.active_writers.get(token.component, 0) - 1 - if remaining > 0: - self.active_writers[token.component] = remaining - else: - self.active_writers.pop(token.component, None) - self.condition.notify_all() - - def has_active_writer(self, components: set[str]) -> bool: - with self.condition: - return any(self.active_writers.get(component, 0) for component in components) - - def writer_admission_closed(self) -> bool: - with self.condition: - return self.phase != "running" + limit: int, + ) -> list[dict[str, object]]: + return self.coordinator.ledger.pending_ctv_fanout_statuses(limit=limit) - def wait_for_writer_quiescence( + def audit_bundle( self, - timeout_seconds: float | None = None, - ) -> tuple[bool, float, dict[str, int]]: - started = time.monotonic() - timeout = ( - self.writer_quiescence_timeout_seconds - if timeout_seconds is None - else max(0.0, float(timeout_seconds)) - ) - deadline = started + timeout - with self.condition: - while self.active_writers: - remaining = deadline - time.monotonic() - if remaining <= 0: - break - self.condition.wait(remaining) - elapsed = max(0.0, time.monotonic() - started) - quiesced = not self.active_writers - blockers = dict(sorted(self.active_writers.items())) - outcome = "success" if quiesced else "timeout" - self.writer_quiescence_outcomes[outcome] += 1 - self.writer_quiescence_seconds = elapsed - if quiesced: - self.phase = "writers_quiesced" - else: - self.phase = "release_withheld" - self.lease_release_withheld = True - self.release_withheld_total += 1 - self.condition.notify_all() - return quiesced, elapsed, blockers - - def claim_lease_release(self) -> tuple[bool, dict[str, int]]: - with self.condition: - if self.lease_release_attempted or self.lease_release_withheld: - return False, {} - if self.active_writers: - return False, dict(sorted(self.active_writers.items())) - self.lease_release_attempted = True - self.lease_release_attempts_total += 1 - self.phase = "releasing_lease" - self.condition.notify_all() - return True, {} - - def finish_lease_release(self, outcome: str, elapsed: float) -> None: - with self.condition: - self.lease_release_outcomes[outcome] += 1 - self.lease_release_seconds = elapsed - self.lease_release_succeeded = outcome != "failure" - self.phase = "lease_released" if outcome != "failure" else "lease_release_failed" - if outcome != "failure" and self.sigterm_monotonic is not None: - self.sigterm_to_lease_release_seconds = max( - 0.0, - time.monotonic() - self.sigterm_monotonic, - ) - self.sigterm_release_observed = True - self.condition.notify_all() - - def claim_non_writer_drain(self) -> bool: - with self.condition: - if self._drain_claimed: - return False - if self.phase not in { - "lease_released", - "lease_release_failed", - "release_withheld", - }: - return False - self._drain_claimed = True - self.phase = "draining_non_writers" - return True - - def finish_non_writer_drain(self, elapsed: float) -> None: - with self.condition: - self.non_writer_drain_seconds = elapsed - self.non_writer_drains_total += 1 - self.phase = "complete" - self.condition.notify_all() + *, + block_hash: str, + ) -> Mapping[str, object] | None: + return self.coordinator.ledger.audit_bundle(block_hash=block_hash) - def snapshot(self) -> dict[str, Any]: - with self.condition: - return { - "phase": self.phase, - "active_writers": dict(self.active_writers), - "shutdowns_total": self.shutdowns_total, - "writer_quiescence_outcomes": dict(self.writer_quiescence_outcomes), - "writer_quiescence_seconds": self.writer_quiescence_seconds, - "lease_release_attempts_total": self.lease_release_attempts_total, - "lease_release_outcomes": dict(self.lease_release_outcomes), - "lease_release_seconds": self.lease_release_seconds, - "lease_release_withheld": self.lease_release_withheld, - "sigterm_to_lease_release_seconds": self.sigterm_to_lease_release_seconds, - "sigterm_release_observed": self.sigterm_release_observed, - "release_withheld_total": self.release_withheld_total, - "non_writer_drain_seconds": self.non_writer_drain_seconds, - "non_writer_drains_total": self.non_writer_drains_total, - } + def audit_bundle_by_commitment( + self, + *, + commitment_leaf_hex: str, + ) -> Mapping[str, object] | None: + return self.coordinator.ledger.audit_bundle_by_commitment( + commitment_leaf_hex=commitment_leaf_hex + ) -def ledger_writer_operation(component: str) -> Callable[[Callable[..., Any]], Callable[..., Any]]: - """Decorate an entry point that can mutate the PRISM ledger.""" - - def decorate(method: Callable[..., Any]) -> Callable[..., Any]: - @wraps(method) - def guarded(self: "PrismCoordinator", *args: Any, **kwargs: Any) -> Any: - with self._writer_operation(component): - return method(self, *args, **kwargs) - - return guarded - - return decorate - - -class TemplateRefreshBlocked(RuntimeError): - """A live template was fetched, but safe work could not be issued.""" - - -class TemplateRefreshSuperseded(TemplateRefreshBlocked): - """Concurrent tip/payout progress invalidated this refresh attempt. - - Raised only for coordination races that a scheduled retry resolves on its - own: the tip advanced mid-refresh, the payout-state generation moved, or a - newer observation superseded the prepared work. Unlike its parent, this - subclass never arms the ordinary template-refresh failure budget. Repeated - coordination blocks are tracked by their own longer deadline; a genuine - RPC/build/trust failure must raise plain TemplateRefreshBlocked so it still - takes the ordinary budgeted restart path. - """ +class PrismCoordinator: + # Share-submission owner state routed through descriptors; the + # ShareSubmissionService holds the single mutable copy once it exists + # (see lab/prism/share_submission.py). Pre-service writes land in the + # instance dict and are adopted at service construction. + recent_share_keys = RecentShareCompatibilityField() + block_solves_dropped_counts = BlockSolvesDroppedCompatibilityField() + + # Reorg reconciler owner state routed through descriptors; the + # ReorgReconcilerService holds the single mutable copy once it exists + # (see lab/prism/reorg_reconciler.py). Pre-service writes land in + # backing slots and are adopted at service construction. + reorg_reconciler_enabled = ReorgCompatibilityField("enabled", True) + reorg_reconcile_cache_seconds = ReorgCompatibilityField( + "cache_seconds", + DEFAULT_PRISM_REORG_RECONCILE_CACHE_SECONDS, + ) + reorg_inactive_block_count = ReorgCompatibilityField( + "inactive_block_count", + 0, + ) + reorg_reactivated_block_count = ReorgCompatibilityField( + "reactivated_block_count", + 0, + ) + reorg_reconcile_skip_count = ReorgCompatibilityField( + "reconcile_skip_count", + 0, + ) + reorg_reconcile_error_count = ReorgCompatibilityField( + "reconcile_error_count", + 0, + ) + matured_payout_count = ReorgCompatibilityField("matured_payout_count", 0) + last_reorg_reconciled_tip_hash = ReorgCompatibilityField( + "last_tip_hash", + None, + ) + last_reorg_reconciled_trusted = ReorgCompatibilityField( + "last_trusted", + False, + ) + last_reorg_reconciled_monotonic = ReorgCompatibilityField( + "last_monotonic", + None, + ) + # J1/template/compiler owner state routed through descriptors; the owner + # services hold the single mutable copy (see lab/prism/job_bundle.py, + # lab/prism/template_artifacts.py, and lab/prism/bundle_compiler.py). + _job_cache_lock = _JobBundleStateField("_job_cache_lock") + _active_job_bundle_builds = _JobBundleStateField("_active_job_bundle_builds") + _job_build_lock = _JobBundleStateField("_job_build_lock") + _job_build_scheduler_lock = _JobBundleStateField("_job_build_scheduler_lock") + _job_build_priority_preparations = _JobBundleStateField( + "_job_build_priority_preparations" + ) + _job_build_priority_preparation_sequence = _JobBundleStateField( + "_job_build_priority_preparation_sequence" + ) + _job_build_routine_preparations = _JobBundleStateField( + "_job_build_routine_preparations" + ) + _job_build_routine_preparation_sequence = _JobBundleStateField( + "_job_build_routine_preparation_sequence" + ) + _job_build_priority_changed = _JobBundleStateField("_job_build_priority_changed") + _job_build_executor = _JobBundleStateField("_job_build_executor") + _job_build_executor_shutdown = _JobBundleStateField("_job_build_executor_shutdown") + _job_build_active = _JobBundleStateField("_job_build_active") + _job_build_retiring = _JobBundleStateField("_job_build_retiring") + _job_build_pending = _JobBundleStateField("_job_build_pending") + _job_build_issued_at_ms = _JobBundleStateField("_job_build_issued_at_ms") + _job_bundle_cache = _JobBundleStateField("_job_bundle_cache") + _job_build_phase_local = _JobBundleStateField("_job_build_phase_local") + job_cache_hit_counts = _JobBundleStateField("job_cache_hit_counts") + job_cache_miss_counts = _JobBundleStateField("job_cache_miss_counts") + job_build_seconds_bucket_counts = _JobBundleStateField( + "job_build_seconds_bucket_counts" + ) + job_build_seconds_sum = _JobBundleStateField("job_build_seconds_sum") + job_build_count = _JobBundleStateField("job_build_count") + job_build_phase_seconds = _JobBundleStateField("job_build_phase_seconds") + job_build_scheduler_counts = _JobBundleStateField("job_build_scheduler_counts") + job_build_priority_counts = _JobBundleStateField("job_build_priority_counts") + job_build_priority_admission_seconds = _JobBundleStateField( + "job_build_priority_admission_seconds" + ) + initial_job_prepared_work_counts = _JobBundleStateField( + "initial_job_prepared_work_counts" + ) + _admission_deadline_last_log_monotonic = _JobBundleStateField( + "_admission_deadline_last_log_monotonic" + ) + job_build_cancellation_seconds = _JobBundleStateField( + "job_build_cancellation_seconds" + ) + job_build_replacement_start_seconds = _JobBundleStateField( + "job_build_replacement_start_seconds" + ) + job_build_worker_counts = _JobBundleStateField("job_build_worker_counts") + _job_build_worker_restart_pending = _JobBundleStateField( + "_job_build_worker_restart_pending" + ) + _share_window_serialization_lock = _JobBundleStateField( + "_share_window_serialization_lock" + ) + _share_window_serialization = _JobBundleStateField("_share_window_serialization") + _bundle_preparation_lock = _JobBundleStateField("_bundle_preparation_lock") + _bundle_preparation_flights = _JobBundleStateField("_bundle_preparation_flights") + shared_bundle_build_counts = _JobBundleStateField("shared_bundle_build_counts") + shared_bundle_preparation_seconds_sum = _JobBundleStateField( + "shared_bundle_preparation_seconds_sum" + ) + shared_bundle_preparation_count = _JobBundleStateField( + "shared_bundle_preparation_count" + ) + shared_bundle_preparation_waiters = _JobBundleStateField( + "shared_bundle_preparation_waiters" + ) + _prepared_ready_bundle = _JobBundleStateField("_prepared_ready_bundle") + _prepared_ready_snapshot = _JobBundleStateField("_prepared_ready_snapshot") + job_preparation_pending = _JobBundleStateField("job_preparation_pending") + _template_artifacts = _TemplateArtifactStateField("_template_artifacts") + _template_artifact_generation = _TemplateArtifactStateField( + "_template_artifact_generation" + ) + _serve_builder_lock = _BundleCompilerStateField("_serve_builder_lock") + _serve_builder = _BundleCompilerStateField("_serve_builder") + _serve_builder_shutdown = _BundleCompilerStateField("_serve_builder_shutdown") + _serve_builder_metrics_lock = _BundleCompilerStateField( + "_serve_builder_metrics_lock" + ) + serve_builder_counts = _BundleCompilerStateField("serve_builder_counts") + serve_builder_window_cache_counts = _BundleCompilerStateField( + "serve_builder_window_cache_counts" + ) -class _TipRefreshFanoutSuperseded(TemplateRefreshSuperseded): - """A published fanout lost to a newer tip-refresh epoch.""" + # R1 owner state routed through descriptors; TipRefreshService holds + # the single mutable copy (see lab/prism/tip_refresh.py). The twelve + # historical plain fields and the tip-detection epoch stay lazily + # created on the owner, preserving legacy hasattr semantics. + _tip_refresh_lock = _TipRefreshStateField("_tip_refresh_lock") + _tip_refresh_singleflight_lock = _TipRefreshStateField( + "_tip_refresh_singleflight_lock" + ) + _tip_refresh_executor_lock = _TipRefreshStateField("_tip_refresh_executor_lock") + _tip_refresh_executor = _TipRefreshStateField("_tip_refresh_executor") + _tip_refresh_executor_shutdown = _TipRefreshStateField( + "_tip_refresh_executor_shutdown" + ) + _tip_refresh_metrics_lock = _TipRefreshStateField("_tip_refresh_metrics_lock") + tip_refresh_histograms = _TipRefreshStateField("tip_refresh_histograms") + tip_refresh_build_phase_histograms = _TipRefreshStateField( + "tip_refresh_build_phase_histograms" + ) + tip_refresh_client_counts = _TipRefreshStateField("tip_refresh_client_counts") + tip_refresh_cancellation_counts = _TipRefreshStateField( + "tip_refresh_cancellation_counts" + ) + tip_refresh_wave_outcome_counts = _TipRefreshStateField( + "tip_refresh_wave_outcome_counts" + ) + tip_refresh_coverage_histograms = _TipRefreshStateField( + "tip_refresh_coverage_histograms" + ) + tip_refresh_inflight = _TipRefreshStateField("tip_refresh_inflight") + tip_refresh_build_inflight = _TipRefreshStateField("tip_refresh_build_inflight") + tip_refresh_build_queue_depth = _TipRefreshStateField( + "tip_refresh_build_queue_depth" + ) + tip_refresh_singleflight_hits = _TipRefreshStateField( + "tip_refresh_singleflight_hits" + ) + tip_refresh_superseded_results = _TipRefreshStateField( + "tip_refresh_superseded_results" + ) + tip_refresh_worker_failures = _TipRefreshStateField("tip_refresh_worker_failures") + tip_refresh_worker_restarts = _TipRefreshStateField("tip_refresh_worker_restarts") + tip_refresh_ipc_bytes = _TipRefreshStateField("tip_refresh_ipc_bytes") + _tip_refresh_pending_event = _TipRefreshStateField("_tip_refresh_pending_event") + _tip_refresh_pending_counter = _TipRefreshStateField("_tip_refresh_pending_counter") + _tip_refresh_pending_token = _TipRefreshStateField("_tip_refresh_pending_token") + _tip_refresh_retry = _TipRefreshStateField("_tip_refresh_retry") + _tip_refresh_retry_counter = _TipRefreshStateField("_tip_refresh_retry_counter") + _tip_refresh_retry_consumed = _TipRefreshStateField("_tip_refresh_retry_consumed") + _tip_refresh_failure_holdoff_until = _TipRefreshStateField( + "_tip_refresh_failure_holdoff_until" + ) + _tip_refresh_failure_tip = _TipRefreshStateField("_tip_refresh_failure_tip") + _active_tip_refresh = _TipRefreshStateField("_active_tip_refresh") + _tip_refresh_epoch_sequence = _TipRefreshStateField("_tip_refresh_epoch_sequence") + _tip_refresh_epoch_tip_hash = _TipRefreshStateField("_tip_refresh_epoch_tip_hash") + _tip_refresh_epoch_payout_generation = _TipRefreshStateField( + "_tip_refresh_epoch_payout_generation" + ) + _tip_refresh_epoch_coverage = _TipRefreshStateField("_tip_refresh_epoch_coverage") + _published_tip_refresh_epoch_identity = _TipRefreshStateField( + "_published_tip_refresh_epoch_identity" + ) + _retained_collection_refresh = _TipRefreshStateField("_retained_collection_refresh") + current_tip_first_seen = _TipRefreshStateField("current_tip_first_seen") + current_tip_parent = _TipRefreshStateField("current_tip_parent") + current_tip_observation_sequence = _TipRefreshStateField( + "current_tip_observation_sequence" + ) + current_tip_observed_monotonic = _TipRefreshStateField( + "current_tip_observed_monotonic" + ) + tip_template_snapshot = _TipRefreshStateField("tip_template_snapshot") + latest_detected_tip = _TipRefreshStateField("latest_detected_tip") + tip_refresh_divergence_started_monotonic = _TipRefreshStateField( + "tip_refresh_divergence_started_monotonic" + ) + tip_observation_sequence = _TipRefreshStateField("tip_observation_sequence") + last_successful_template_refresh_monotonic = _TipRefreshStateField( + "last_successful_template_refresh_monotonic" + ) + template_refresh_failure_started_monotonic = _TipRefreshStateField( + "template_refresh_failure_started_monotonic" + ) + tip_refresh_job_count = _TipRefreshStateField("tip_refresh_job_count") + post_accept_refresh_failure_count = _TipRefreshStateField( + "post_accept_refresh_failure_count" + ) + tip_detection_epoch = _TipRefreshStateField("tip_detection_epoch") + # S2 owner state routed through descriptors; JobDeliveryService holds + # the single mutable copy (see lab/prism/job_delivery.py). + pending_initial_jobs = _JobDeliveryStateField("pending_initial_jobs") + initial_job_queue_rejection_count = _JobDeliveryStateField( + "initial_job_queue_rejection_count" + ) + initial_job_timeout_count = _JobDeliveryStateField("initial_job_timeout_count") + initial_job_cancelled_count = _JobDeliveryStateField("initial_job_cancelled_count") + initial_job_coalesced_count = _JobDeliveryStateField("initial_job_coalesced_count") + initial_job_queue_capacity_reclaimed_count = _JobDeliveryStateField( + "initial_job_queue_capacity_reclaimed_count" + ) + initial_job_sent_count = _JobDeliveryStateField("initial_job_sent_count") + initial_job_failed_count = _JobDeliveryStateField("initial_job_failed_count") + initial_job_superseded_count = _JobDeliveryStateField( + "initial_job_superseded_count" + ) + initial_job_delivery_latency_seconds_sum = _JobDeliveryStateField( + "initial_job_delivery_latency_seconds_sum" + ) + initial_job_delivery_latency_count = _JobDeliveryStateField( + "initial_job_delivery_latency_count" + ) + last_initial_job_delivery_monotonic = _JobDeliveryStateField( + "last_initial_job_delivery_monotonic" + ) + _initial_job_executor_lock = _JobDeliveryStateField("_initial_job_executor_lock") + _initial_job_executor = _JobDeliveryStateField("_initial_job_executor") + _initial_job_executor_shutdown = _JobDeliveryStateField( + "_initial_job_executor_shutdown" + ) + jobs = _JobDeliveryStateField("jobs") + job_counter = _JobDeliveryStateField("job_counter") + evicted_job_graveyard = _JobDeliveryStateField("evicted_job_graveyard") + evicted_jobs_by_connection = _JobDeliveryStateField("evicted_jobs_by_connection") + evicted_same_tip_by_connection = _JobDeliveryStateField( + "evicted_same_tip_by_connection" + ) + evicted_same_tip_job_ids = _JobDeliveryStateField("evicted_same_tip_job_ids") + evicted_job_index_tip_hash = _JobDeliveryStateField("evicted_job_index_tip_hash") + evicted_job_next_prune_monotonic = _JobDeliveryStateField( + "evicted_job_next_prune_monotonic" + ) + evicted_job_expiration_counts = _JobDeliveryStateField( + "evicted_job_expiration_counts" + ) + evicted_job_capacity_eviction_counts = _JobDeliveryStateField( + "evicted_job_capacity_eviction_counts" + ) + evicted_job_submit_counts = _JobDeliveryStateField("evicted_job_submit_counts") + _disconnected_evicted_job_ids = _JobDeliveryStateField( + "_disconnected_evicted_job_ids" + ) -class _TipRefreshTrustBlocked(TemplateRefreshBlocked): - """Refresh work stopped because the live chain view was not trusted.""" + # S1 owner state routed through descriptors; StratumSessionService (its + # SessionRegistry and P2mrAddressValidator) holds the single mutable copy + # (see lab/prism/stratum_session.py). + clients = _StratumSessionStateField("clients") + connection_counter = _StratumSessionStateField("connection_counter") + connection_limit_rejection_counts = _StratumSessionStateField( + "connection_limit_rejection_counts" + ) + peak_active_connection_count = _StratumSessionStateField( + "peak_active_connection_count" + ) + handler_thread_count = _StratumSessionStateField("handler_thread_count") + accept_resource_exhaustion_count = _StratumSessionStateField( + "accept_resource_exhaustion_count" + ) + connection_setup_failure_count = _StratumSessionStateField( + "connection_setup_failure_count" + ) + _p2mr_address_cache_lock = _StratumSessionStateField("_p2mr_address_cache_lock") + _p2mr_address_cache = _StratumSessionStateField("_p2mr_address_cache") + _p2mr_address_validation_inflight = _StratumSessionStateField( + "_p2mr_address_validation_inflight" + ) + # S3 owner state routed through descriptors; ShareWriter holds the single + # mutable copy (see lab/prism/share_writer.py). + share_append_queue = ShareWriterCompatibilityField("share_append_queue") + share_commit_batch_size = ShareWriterCompatibilityField("share_commit_batch_size") + share_commit_linger_seconds = ShareWriterCompatibilityField( + "share_commit_linger_seconds" + ) + share_commit_timeout_seconds = ShareWriterCompatibilityField( + "share_commit_timeout_seconds" + ) + share_writer_active = ShareWriterCompatibilityField("share_writer_active") + share_append_failure_count = ShareWriterCompatibilityField( + "share_append_failure_count" + ) + share_recovery_path = ShareWriterCompatibilityField("share_recovery_path") + share_recovery_lock = ShareWriterCompatibilityField("share_recovery_lock") + shares_recovered_to_disk = ShareWriterCompatibilityField( + "shares_recovered_to_disk" + ) + shares_replayed = ShareWriterCompatibilityField("shares_replayed") + share_replay_conflicts = ShareWriterCompatibilityField( + "share_replay_conflicts" + ) + _pending_share_commit_lock = ShareWriterCompatibilityField( + "_pending_share_commit_lock" + ) + _pending_share_commit_floor = ShareWriterCompatibilityField( + "_pending_share_commit_floor" + ) -class JobBuildCancelled(TemplateRefreshBlocked): - """An immutable build was cancelled or timed out.""" + # P1 owner state routed through descriptors; PayoutStateService holds + # the single mutable copy (see lab/prism/payout_state.py). + _payout_state_generation = _PayoutStateStateField( + "_payout_state_generation" + ) + _payout_state_prepare_lock = _PayoutStateStateField( + "_payout_state_prepare_lock" + ) + _payout_state_source = _PayoutStateStateField("_payout_state_source") + _published_payout_state = _PayoutStateStateField("_published_payout_state") + _payout_ledger_artifact = _PayoutStateStateField("_payout_ledger_artifact") + _payout_ledger_artifact_generation = _PayoutStateStateField( + "_payout_ledger_artifact_generation" + ) + _payout_ledger_append_invalidation_epoch = _PayoutStateStateField( + "_payout_ledger_append_invalidation_epoch" + ) + _payout_append_landing_fence_lock = _PayoutStateStateField( + "_payout_append_landing_fence_lock" + ) + _payout_window_inflight_scan_anchors = _PayoutStateStateField( + "_payout_window_inflight_scan_anchors" + ) + _payout_window_inflight_scan_anchor_token = _PayoutStateStateField( + "_payout_window_inflight_scan_anchor_token" + ) + _payout_unfenced_append_inflight_stamps = _PayoutStateStateField( + "_payout_unfenced_append_inflight_stamps" + ) + _payout_unfenced_append_inflight_token = _PayoutStateStateField( + "_payout_unfenced_append_inflight_token" + ) + _payout_unfenced_append_drained = _PayoutStateStateField( + "_payout_unfenced_append_drained" + ) + _payout_published_job_window_anchor_ms = _PayoutStateStateField( + "_payout_published_job_window_anchor_ms" + ) + _incremental_payout_artifact_window = _PayoutStateStateField( + "_incremental_payout_artifact_window" + ) + _incremental_payout_artifact_window_invalidation_reason = _PayoutStateStateField( + "_incremental_payout_artifact_window_invalidation_reason" + ) + _payout_prior_balances_reuse_invalidated_sha256 = _PayoutStateStateField( + "_payout_prior_balances_reuse_invalidated_sha256" + ) + _payout_artifact_executor_lock = _PayoutStateStateField( + "_payout_artifact_executor_lock" + ) + _payout_artifact_executor = _PayoutStateStateField( + "_payout_artifact_executor" + ) + _payout_artifact_future = _PayoutStateStateField("_payout_artifact_future") + _payout_artifact_requested = _PayoutStateStateField( + "_payout_artifact_requested" + ) + _payout_artifact_requested_bypass = _PayoutStateStateField( + "_payout_artifact_requested_bypass" + ) + _payout_artifact_executor_shutdown = _PayoutStateStateField( + "_payout_artifact_executor_shutdown" + ) + _payout_artifact_last_schedule_monotonic = _PayoutStateStateField( + "_payout_artifact_last_schedule_monotonic" + ) + _payout_artifact_rearm_backoff = _PayoutStateStateField( + "_payout_artifact_rearm_backoff" + ) + _payout_state_delivery_gate = _PayoutStateStateField( + "_payout_state_delivery_gate" + ) + _payout_balance_mutation_lock = _PayoutStateStateField( + "_payout_balance_mutation_lock" + ) + _accepted_block_payout_preview_condition = _PayoutStateStateField( + "_accepted_block_payout_preview_condition" + ) + _accepted_block_payout_previews = _PayoutStateStateField( + "_accepted_block_payout_previews" + ) + _invalidated_accepted_block_payout_previews = _PayoutStateStateField( + "_invalidated_accepted_block_payout_previews" + ) + _payout_state_metrics_lock = _PayoutStateStateField( + "_payout_state_metrics_lock" + ) + payout_state_histograms = _PayoutStateStateField("payout_state_histograms") + payout_gate_wait_histograms = _PayoutStateStateField( + "payout_gate_wait_histograms" + ) + payout_state_candidates_discarded = _PayoutStateStateField( + "payout_state_candidates_discarded" + ) + _payout_first_delivery_pending = _PayoutStateStateField( + "_payout_first_delivery_pending" + ) + _payout_state_publication_blocked = _PayoutStateStateField( + "_payout_state_publication_blocked" + ) + payout_artifact_event_counts = _PayoutStateStateField( + "payout_artifact_event_counts" + ) + _progress_current_template_generation = _ProgressHealthStateField( + "_current_template_generation" + ) + _progress_current_template_fingerprint = _ProgressHealthStateField( + "_current_template_fingerprint" + ) + _progress_current_payout_generation = _ProgressHealthStateField( + "_current_payout_generation" + ) + _progress_published_template_generation = _ProgressHealthStateField( + "_published_template_generation" + ) + _progress_published_template_fingerprint = _ProgressHealthStateField( + "_published_template_fingerprint" + ) + _progress_published_payout_generation = _ProgressHealthStateField( + "_published_payout_generation" + ) + _progress_has_published_work = _ProgressHealthStateField("_has_published_work") + _progress_last_tip_poll_monotonic = _ProgressHealthStateField( + "_last_tip_poll_monotonic" + ) + _progress_last_delivery_template_generation = _ProgressHealthStateField( + "_last_delivery_template_generation" + ) + _progress_last_delivery_template_fingerprint = _ProgressHealthStateField( + "_last_delivery_template_fingerprint" + ) + _progress_last_delivery_payout_generation = _ProgressHealthStateField( + "_last_delivery_payout_generation" + ) + _progress_last_delivery_monotonic = _ProgressHealthStateField( + "_last_delivery_monotonic" + ) + _progress_pending_since_monotonic = _ProgressHealthStateField( + "_pending_since_monotonic" + ) + _progress_publication_divergence_since_monotonic = _ProgressHealthStateField( + "_publication_divergence_since_monotonic" + ) + _progress_refresh_signal_pending = _ProgressHealthStateField( + "_refresh_signal_pending" + ) + _progress_active_refresh_count = _ProgressHealthStateField( + "_active_refresh_count" + ) + _progress_last_refresh_activity_monotonic = _ProgressHealthStateField( + "_last_refresh_activity_monotonic" + ) + _progress_bundle_build_counter = _ProgressHealthStateField( + "_bundle_build_counter" + ) -class JobBuildSuperseded(JobBuildCancelled, TemplateRefreshSuperseded): - """A coordination race cooperatively cancelled immutable construction.""" + @property + def _progress_health_lock(self) -> threading.Lock: + return self._ensure_progress_health_service()._lock + @property + def _progress_bundle_builds(self) -> dict[int, float]: + return self._ensure_progress_health_service()._bundle_builds -class _JobBuildCancellation: - def __init__(self, *, timeout_seconds: float) -> None: - self._event = threading.Event() - self._lock = threading.Lock() - self.started_monotonic = time.monotonic() - self.deadline_monotonic = self.started_monotonic + timeout_seconds - self.last_checkpoint_monotonic = self.started_monotonic - self.cancelled_monotonic: float | None = None - self.reason: str | None = None - - def cancel(self, reason: str) -> bool: - with self._lock: - if self._event.is_set(): - return False - self.reason = reason - self.cancelled_monotonic = time.monotonic() - self._event.set() - return True + block_candidate_queue = BlockCandidateCompatibilityField( + "block_candidate_queue", + candidate_compatibility_default( + lambda: queue.Queue(maxsize=MAX_PENDING_BLOCK_CANDIDATES) + ), + ) + block_candidates_dropped = BlockCandidateCompatibilityField( + "block_candidates_dropped", 0 + ) + block_candidate_wakeups_coalesced = BlockCandidateCompatibilityField( + "block_candidate_wakeups_coalesced", 0 + ) + block_candidate_retry_count = BlockCandidateCompatibilityField( + "block_candidate_retry_count", 0 + ) + block_candidate_poisoned_count = BlockCandidateCompatibilityField( + "block_candidate_poisoned_count", 0 + ) + block_candidate_retry_initial_seconds = BlockCandidateCompatibilityField( + "block_candidate_retry_initial_seconds", + DEFAULT_BLOCK_CANDIDATE_RETRY_INITIAL_SECONDS, + ) + block_candidate_retry_max_seconds = BlockCandidateCompatibilityField( + "block_candidate_retry_max_seconds", + DEFAULT_BLOCK_CANDIDATE_RETRY_MAX_SECONDS, + ) + block_candidate_retry_delays = BlockCandidateCompatibilityField( + "block_candidate_retry_delays", + candidate_compatibility_default(lambda: {}), + ) + block_candidate_abandoned_counts = BlockCandidateCompatibilityField( + "block_candidate_abandoned_counts", + candidate_compatibility_default(lambda: {}), + ) + _retry_block_candidate = BlockCandidateCompatibilityField( + "_retry_block_candidate", None + ) + _block_candidate_outcome = BlockCandidateCompatibilityField( + "_block_candidate_outcome", + candidate_compatibility_default(lambda: threading.local()), + ) + _block_candidate_finalize_retries = BlockCandidateCompatibilityField( + "_block_candidate_finalize_retries", + candidate_compatibility_default(lambda: {}), + ) + # The #113-era block-candidate fields route to the B1 owner one-to-one + # (the pre-#113 backoff quartet keeps its historical service spelling). + _block_replay_candidate_queue = BlockCandidateStateField( + "_block_replay_candidate_queue" + ) + _block_replay_inflight_hashes = BlockCandidateStateField( + "_block_replay_inflight_hashes" + ) + _block_quarantine_queue = BlockCandidateStateField("_block_quarantine_queue") + _block_quarantine_hashes = BlockCandidateStateField("_block_quarantine_hashes") + _block_candidate_disposition_registry_lock = BlockCandidateStateField( + "_block_candidate_disposition_registry_lock" + ) + _block_candidate_disposition_flights = BlockCandidateStateField( + "_block_candidate_disposition_flights" + ) + _block_candidate_terminal_outcomes = BlockCandidateStateField( + "_block_candidate_terminal_outcomes" + ) + _block_fast_lane_reservations = BlockCandidateStateField( + "_block_fast_lane_reservations" + ) + _block_disposition_waiting_retries = BlockCandidateStateField( + "_block_disposition_waiting_retries" + ) + _block_candidate_dequeued_hashes = BlockCandidateStateField( + "_block_candidate_dequeued_hashes" + ) + _block_accounting_state_lock = BlockCandidateStateField( + "_block_accounting_state_lock" + ) + _block_accounting_queue = BlockCandidateStateField("_block_accounting_queue") + _block_accounting_overflow_queue = BlockCandidateStateField( + "_block_accounting_overflow_queue" + ) + _block_accounting_accepted_queue = BlockCandidateStateField( + "_block_accounting_accepted_queue" + ) + _block_accounting_sequence = BlockCandidateStateField( + "_block_accounting_sequence" + ) + _block_accounting_thread = BlockCandidateStateField("_block_accounting_thread") + _block_accounting_thread_ident = BlockCandidateStateField( + "_block_accounting_thread_ident" + ) + _block_accounting_holds_disposition = BlockCandidateStateField( + "_block_accounting_holds_disposition" + ) + _block_accounting_deferred_retry_candidate = BlockCandidateStateField( + "_block_accounting_deferred_retry_candidate" + ) + _block_accounting_phase = BlockCandidateStateField("_block_accounting_phase") + _block_submitter_thread_ident = BlockCandidateStateField( + "_block_submitter_thread_ident" + ) + _block_submitter_phase = BlockCandidateStateField("_block_submitter_phase") + _block_submitter_last_lock_wait_log_monotonic = BlockCandidateStateField( + "_block_submitter_last_lock_wait_log_monotonic" + ) + _block_submitter_retry_state_lock = BlockCandidateStateField( + "_block_submitter_retry_state_lock", "_state_lock" + ) + _block_submitter_backoff_started_monotonic = BlockCandidateStateField( + "_block_submitter_backoff_started_monotonic", "_backoff_started_monotonic" + ) + _block_submitter_backoff_deadline_monotonic = BlockCandidateStateField( + "_block_submitter_backoff_deadline_monotonic", "_backoff_deadline_monotonic" + ) + _block_submitter_backoff_delay_seconds = BlockCandidateStateField( + "_block_submitter_backoff_delay_seconds", "_backoff_delay_seconds" + ) + _block_submitter_ledger_calls_lock = BlockCandidateStateField( + "_block_submitter_ledger_calls_lock" + ) + _block_submitter_ledger_calls = BlockCandidateStateField( + "_block_submitter_ledger_calls" + ) + _block_submitter_ledger_worker_slots = BlockCandidateStateField( + "_block_submitter_ledger_worker_slots" + ) + _block_submitter_rpc_calls_lock = BlockCandidateStateField( + "_block_submitter_rpc_calls_lock" + ) + _block_submitter_rpc_calls = BlockCandidateStateField( + "_block_submitter_rpc_calls" + ) + _block_submitter_rpc_worker_slots = BlockCandidateStateField( + "_block_submitter_rpc_worker_slots" + ) + _block_landing_timeout_counts = BlockCandidateStateField( + "_block_landing_timeout_counts" + ) + _block_ledger_call_metrics_lock = BlockCandidateStateField( + "_block_ledger_call_metrics_lock" + ) + _block_ledger_call_metrics = BlockCandidateStateField( + "_block_ledger_call_metrics" + ) + _block_candidate_retry_not_before = BlockCandidateStateField( + "_block_candidate_retry_not_before" + ) + _block_candidate_retained_node_submissions = BlockCandidateStateField( + "_block_candidate_retained_node_submissions" + ) + _block_candidate_retained_submission_monotonic = BlockCandidateStateField( + "_block_candidate_retained_submission_monotonic" + ) + _counted_block_candidate_abandonments = BlockCandidateStateField( + "_counted_block_candidate_abandonments" + ) + _outstanding_block_candidate_hashes = BlockCandidateStateField( + "_outstanding_block_candidate_hashes" + ) + _tip_observed_accepted_block_hashes = BlockCandidateStateField( + "_tip_observed_accepted_block_hashes" + ) + block_candidate_accept_pending_defer_count = BlockCandidateStateField( + "block_candidate_accept_pending_defer_count" + ) + accepted_parent_redrive_attempt_count = BlockCandidateStateField( + "accepted_parent_redrive_attempt_count" + ) + accepted_parent_redrive_resolved_count = BlockCandidateStateField( + "accepted_parent_redrive_resolved_count" + ) + accepted_parent_redrive_exhausted_count = BlockCandidateStateField( + "accepted_parent_redrive_exhausted_count" + ) + stale_job_abandon_counts = BlockCandidateStateField("stale_job_abandon_counts") + _block_submit_metrics_lock = BlockCandidateStateField( + "_block_submit_metrics_lock" + ) + block_submit_seconds_histogram = BlockCandidateStateField( + "block_submit_seconds_histogram" + ) + _accepted_block_preview_publication_lock = BlockCandidateStateField( + "_accepted_block_preview_publication_lock" + ) + accepted_block_preview_publication_seconds_histogram = ( + BlockCandidateStateField( + "accepted_block_preview_publication_seconds_histogram" + ) + ) + _accepted_block_preview_acceptance_monotonic = BlockCandidateStateField( + "_accepted_block_preview_acceptance_monotonic" + ) + _block_replay_enumeration_owed_flag = BlockCandidateStateField( + "_block_replay_enumeration_owed_flag" + ) - def is_set(self) -> bool: - if self._event.is_set(): - return True - if time.monotonic() >= self.deadline_monotonic: - self.cancel("timeout") - return True - return False + # Vardiff owner state routed through descriptors; VardiffService holds + # the single mutable copy (see lab/prism/vardiff_service.py). Before the + # lazy service exists, reads/writes land in the instance dict and are + # adopted at single-flight construction, preserving embedders, bare + # __new__ test instances, and concurrent first use. + idle_retarget_count = VardiffCompatibilityField( + "idle_retarget_count", + lambda: 0, + ) + _vardiff_idle_lock = VardiffCompatibilityField( + "_vardiff_idle_lock", + threading.Lock, + ) + _vardiff_idle_executor = VardiffCompatibilityField( + "_vardiff_idle_executor", + lambda: None, + ) + _vardiff_idle_executor_shutdown = VardiffCompatibilityField( + "_vardiff_idle_executor_shutdown", + lambda: False, + ) + _vardiff_idle_pending = VardiffCompatibilityField( + "_vardiff_idle_pending", + set, + ) + vardiff_idle_queue_depth = VardiffCompatibilityField( + "vardiff_idle_queue_depth", + lambda: 0, + ) + vardiff_idle_inflight = VardiffCompatibilityField( + "vardiff_idle_inflight", + lambda: 0, + ) + vardiff_idle_clients_inspected = VardiffCompatibilityField( + "vardiff_idle_clients_inspected", + lambda: 0, + ) + vardiff_idle_skip_counts = VardiffCompatibilityField( + "vardiff_idle_skip_counts", + lambda: {reason: 0 for reason in PRISM_VARDIFF_IDLE_SKIP_REASONS}, + ) + vardiff_idle_task_failures = VardiffCompatibilityField( + "vardiff_idle_task_failures", + lambda: 0, + ) + vardiff_idle_sweep_histogram = VardiffCompatibilityField( + "vardiff_idle_sweep_histogram", + lambda: { + "buckets": { + bucket: 0 for bucket in PRISM_VARDIFF_IDLE_SECONDS_BUCKETS + }, + "sum": 0.0, + "count": 0, + }, + ) + vardiff_idle_task_histogram = VardiffCompatibilityField( + "vardiff_idle_task_histogram", + lambda: { + "buckets": { + bucket: 0 for bucket in PRISM_VARDIFF_IDLE_SECONDS_BUCKETS + }, + "sum": 0.0, + "count": 0, + }, + ) - def raise_if_cancelled(self, phase: str) -> None: - if self.is_set(): - reason = self.reason or "cancelled" - if reason == "timeout": - raise JobBuildCancelled( - f"job build timeout at {phase}; immediate retry scheduled" - ) - raise JobBuildSuperseded( - f"job build {reason} at {phase}; immediate retry scheduled" + def _ensure_block_candidate_service(self) -> BlockCandidateService: + service = self.__dict__.get("_block_candidate_service") + if service is not None: + return service + init_lock = self.__dict__.setdefault( + "_block_candidate_service_init_lock", + threading.Lock(), + ) + with init_lock: + service = self.__dict__.get("_block_candidate_service") + if service is not None: + return service + candidate_queue = self.__dict__.get("block_candidate_queue") + if candidate_queue is None: + candidate_queue = queue.Queue(maxsize=MAX_PENDING_BLOCK_CANDIDATES) + stop_event = self.__dict__.get("stop_event") + if stop_event is None: + stop_event = threading.Event() + self.stop_event = stop_event + service = BlockCandidateService( + BlockCandidatePorts( + runtime=self, + ledger=lambda: self.ledger, + stop_event=lambda: self.stop_event, + writer_operation=lambda component: self._writer_operation(component), + submit_candidate=( + lambda candidate, *, node_submission=None, disposition_held=False: ( + self._land_block_candidate_submission( + candidate, + node_submission=node_submission, + disposition_held=disposition_held, + ) + ) + ), + reject_terminal_prepared=( + lambda candidate: self._reject_terminal_prepared_block_candidate( + candidate + ) + ), + begin_preview=lambda block_hash, block_height: ( + self._begin_accepted_block_payout_preview( + block_hash, + block_height=block_height, + ) + ), + clear_preview=lambda block_hash, invalidate: ( + self._clear_accepted_block_payout_preview( + block_hash, + invalidate_published=invalidate, + ) + ), + share_writer=lambda: self._ensure_share_writer_service(), + finish_pending_candidate=( + lambda pending: self._finish_pending_share_candidate(pending) + ), + refresh_after_accept=lambda client: ( + self.refresh_jobs_after_pending_accepted_block( + client, + heartbeat_name="block_submitter", + ) + ), + record_heartbeat=lambda name: self._record_heartbeat(name), + replay_entrypoint=lambda: self.replay_pending_block_candidates(), + submit_next_entrypoint=( + lambda timeout: self.submit_next_block_candidate(timeout=timeout) + ), + next_retry_delay=lambda block_hash: ( + self._next_block_candidate_retry_delay(block_hash) + ), + log=lambda message: print(message, flush=True), + ), + candidate_queue=candidate_queue, + retry_initial_seconds=float( + self.__dict__.get( + "block_candidate_retry_initial_seconds", + DEFAULT_BLOCK_CANDIDATE_RETRY_INITIAL_SECONDS, + ) + ), + retry_max_seconds=float( + self.__dict__.get( + "block_candidate_retry_max_seconds", + DEFAULT_BLOCK_CANDIDATE_RETRY_MAX_SECONDS, + ) + ), + retryable_reasons=PRISM_RETRYABLE_BLOCK_CANDIDATE_REASONS, + submit_seconds_buckets=PRISM_JOB_BUILD_SECONDS_BUCKETS, ) - with self._lock: - self.last_checkpoint_monotonic = time.monotonic() - - -@dataclass -class _JobBuildRequest: - key: JobBuildKey - cache_key: tuple[object, ...] - equivalence_key: tuple[object, ...] - artifacts: CachedTemplateArtifacts - template_json: str - transaction_hexes: tuple[str, ...] - witness_merkle_leaves_hex: tuple[str, ...] - worker: WorkerIdentity | None - mode: str - payout_artifact: PayoutStateArtifact - payout_ledger_artifact: PayoutLedgerArtifact | None - payout_policy_json: str - ctv_settlement_json: str | None - decimal_context: Context = field(repr=False) - cancellation: _JobBuildCancellation - idle_retarget: bool = False - publication_critical: bool = False - request_source: str = "routine" - priority_admission_recorded: bool = False - promise: Future[CachedJobBundle] = field(default_factory=Future) - requested_monotonic: float = field(default_factory=time.monotonic) - superseded_monotonic: float | None = None - # Exposed in-flight scan anchor owned by this build (see - # _expose_inflight_scan_anchor). Set by the synchronous ledger read so - # the executor wrapper can retire the exposure when the build dies - # before handing the token to its seeded artifact. - inflight_scan_anchor_token: int | None = None - - -@dataclass(eq=False) -class _JobBuildFlight: - request: _JobBuildRequest - future: Future[CachedJobBundle] | None = None - # First time the admission sweep saw this flight finished but still - # occupying its slot; eviction waits out the sweep grace from here. - orphan_observed_monotonic: float | None = None - - -class _PayoutStatePublicationBlocked(TemplateRefreshBlocked): - """Job construction is waiting for a prepared payout publication.""" - - -class _ReconcileFlight: - """One in-flight reconcile pass shared by concurrent same-tip callers.""" - - __slots__ = ("event", "summary", "exception") - - def __init__(self) -> None: - self.event = threading.Event() - self.summary: dict[str, object] | None = None - self.exception: BaseException | None = None - - -class _JobBundleBuildSuperseded(JobBuildSuperseded): - """A newer tip or payout generation canceled this deterministic build.""" - - -class CollectionIdentityUnavailable(TemplateRefreshBlocked): - """Current collection work is waiting for an authorized worker identity.""" - - -class JobBuildAdmissionDeadlineExceeded(TemplateRefreshBlocked): - """A routine request parked behind publication priority past its bound. - - Subclasses TemplateRefreshBlocked so callers retry with their normal - pacing; the raise site logs and counts the expiry so parked admission is - visible instead of silently riding the client's own longer timeout. - """ - - -class _JobBuildFailed(RuntimeError): - """Internal signal used to distinguish a skipped build from a no-op.""" - + service.dropped = int( + self.__dict__.get("block_candidates_dropped", 0) + ) + service.wakeups_coalesced = int( + self.__dict__.get("block_candidate_wakeups_coalesced", 0) + ) + service.retries = int( + self.__dict__.get("block_candidate_retry_count", 0) + ) + service.poisoned = int( + self.__dict__.get("block_candidate_poisoned_count", 0) + ) + service.retry_delays = self.__dict__.get( + "block_candidate_retry_delays", + service.retry_delays, + ) + service.finalize_retries = self.__dict__.get( + "_block_candidate_finalize_retries", + service.finalize_retries, + ) + service.abandoned_counts = self.__dict__.get( + "block_candidate_abandoned_counts", + service.abandoned_counts, + ) + service.retry_candidate = self.__dict__.get( + "_retry_block_candidate" + ) + service.outcome = self.__dict__.get( + "_block_candidate_outcome", + service.outcome, + ) + # Adopt any remaining pre-service legacy values (direct __dict__ + # installs by focused embedders) before removing them; ordinary + # attribute writes already route through the class descriptors. + for legacy_name, service_attribute in ( + _BLOCK_CANDIDATE_STATE_FIELD_MAP.items() + ): + if legacy_name in self.__dict__: + setattr(service, service_attribute, self.__dict__[legacy_name]) + self.__dict__["_block_candidate_service"] = service + for name in ( + "block_candidate_queue", + "block_candidates_dropped", + "block_candidate_wakeups_coalesced", + "block_candidate_retry_count", + "block_candidate_poisoned_count", + "block_candidate_retry_initial_seconds", + "block_candidate_retry_max_seconds", + "block_candidate_retry_delays", + "_block_candidate_finalize_retries", + "block_candidate_abandoned_counts", + "_retry_block_candidate", + "_block_candidate_outcome", + *_BLOCK_CANDIDATE_STATE_FIELD_MAP, + ): + self.__dict__.pop(name, None) + return service + + def _ensure_vardiff_service(self) -> VardiffService: + service = self.__dict__.get("_vardiff_service") + if service is not None: + return service + init_lock = self.__dict__.setdefault( + "_vardiff_service_init_lock", + threading.RLock(), + ) + with init_lock: + service = self.__dict__.get("_vardiff_service") + if service is not None: + return service + # Adopt pre-service legacy fields (production __init__ writes and + # direct __dict__ installs by focused embedders) before removing + # them; ordinary attribute access already routes through the + # class descriptors under this same lock. + initial = { + name: self.__dict__.pop(name) + for name in VARDIFF_COMPATIBILITY_FIELDS + if name in self.__dict__ + } + service = VardiffService(self) + for name, value in initial.items(): + setattr(service, name, value) + self.__dict__["_vardiff_service"] = service + return service + + def _ensure_block_finalization_service(self) -> BlockFinalizationService: + service = self.__dict__.get("_block_finalization_service") + if service is not None: + return service + init_lock = self.__dict__.setdefault( + "_block_finalization_service_init_lock", + threading.Lock(), + ) + with init_lock: + service = self.__dict__.get("_block_finalization_service") + if service is None: + service = BlockFinalizationService(self) + self.__dict__["_block_finalization_service"] = service + return service + + # The five PR 79 health compatibility properties were removed by this + # layer; health snapshot state is read and poked only through the + # observability service's state() and narrow *_for_test accessors. + + def __init__(self, config: CoordinatorConfig | None = None) -> None: + self.config = load_coordinator_config() if config is None else config + rpc_config = self.config.rpc + stratum_config = self.config.stratum + job_config = self.config.jobs + block_config = self.config.block + ledger_config = self.config.ledger + audit_config = self.config.audit + ctv_config = self.config.ctv + lifecycle_config = self.config.lifecycle -class _BundlePreparationSuperseded(TemplateRefreshSuperseded): - """The exact work identity lost to a newer tip/template observation. - - Subclasses TemplateRefreshSuperseded: losing the shared-bundle build race - to a newer tip/template observation is coordination churn, so it escapes - the poll without arming the template-refresh failure budget. - """ - - -def parse_worker_username(username: str) -> tuple[str, str | None]: - payout_address, worker_name = split_worker_username(username) - if not payout_address: - raise StratumError(20, "username base is empty") - return payout_address, worker_name - - -def split_worker_username(username: str) -> tuple[str, str | None]: - payout_address, separator, worker_name = username.partition(".") - return payout_address, worker_name if separator else None - - -class PrismCoordinator: - def __init__(self) -> None: - validate_prism_production_gate() self.rpc = JsonRpc( - host=env("QBIT_RPC_HOST"), - port=env_int("QBIT_RPC_PORT", 18452), - user=env("QBIT_RPC_USER"), - password=env("QBIT_RPC_PASSWORD"), - ) - self.qbit_chain = env("QBIT_CHAIN", "regtest") - self.bind = env("PRISM_STRATUM_BIND", "127.0.0.1") - self.port = env_int("PRISM_STRATUM_PORT", 3340) - self.extranonce2_size = env_int("PRISM_STRATUM_EXTRANONCE2_SIZE", 8) - self.blockpoll_seconds = env_positive_float( - "PRISM_BLOCKPOLL_SECONDS", - DEFAULT_PRISM_BLOCKPOLL_SECONDS, - ) + host=rpc_config.host, + port=rpc_config.port, + user=rpc_config.user, + password=rpc_config.password, + ) + self.qbit_chain = rpc_config.chain + self.bind = stratum_config.bind + self.port = stratum_config.port + self.extranonce2_size = stratum_config.extranonce2_size + self.blockpoll_seconds = job_config.blockpoll_seconds # Push-style tip detection rides waitfornewblock; the poll loop above # stays as the fallback and still covers same-tip template refreshes. - self.blockwait_enabled = env_bool("PRISM_BLOCKWAIT_ENABLED", "1") - self.blockwait_timeout_seconds = env_positive_float( - "PRISM_BLOCKWAIT_TIMEOUT_SECONDS", - DEFAULT_PRISM_BLOCKWAIT_TIMEOUT_SECONDS, - ) - self.block_submit_rpc_timeout_seconds = env_positive_float( - "PRISM_BLOCK_SUBMIT_RPC_TIMEOUT_SECONDS", - DEFAULT_BLOCK_SUBMIT_RPC_TIMEOUT_SECONDS, - ) - self.block_submit_db_timeout_seconds = env_positive_float( - "PRISM_BLOCK_SUBMIT_DB_TIMEOUT_SECONDS", - DEFAULT_BLOCK_SUBMIT_DB_TIMEOUT_SECONDS, + self.blockwait_enabled = job_config.blockwait_enabled + self.blockwait_timeout_seconds = job_config.blockwait_timeout_seconds + self.block_submit_rpc_timeout_seconds = block_config.submit_rpc_timeout_seconds + self.block_submit_db_timeout_seconds = block_config.submit_db_timeout_seconds + self.block_landing_db_timeout_seconds = block_config.landing_db_timeout_seconds + self.block_landing_db_timeout_max_seconds = ( + block_config.landing_db_timeout_max_seconds ) - self.block_landing_db_timeout_seconds = env_positive_float( - "PRISM_BLOCK_LANDING_DB_TIMEOUT_SECONDS", - DEFAULT_BLOCK_LANDING_DB_TIMEOUT_SECONDS, + self.accepted_parent_unresolved_depth_max = ( + block_config.accepted_parent_unresolved_depth_max ) - self.block_landing_db_timeout_max_seconds = env_positive_float( - "PRISM_BLOCK_LANDING_DB_TIMEOUT_MAX_SECONDS", - DEFAULT_BLOCK_LANDING_DB_TIMEOUT_MAX_SECONDS, + self.accepted_parent_redrive_defer_threshold = ( + block_config.accepted_parent_redrive_defer_threshold ) - self.accepted_parent_unresolved_depth_max = env_positive_int( - "PRISM_ACCEPTED_PARENT_UNRESOLVED_DEPTH_MAX", - DEFAULT_ACCEPTED_PARENT_UNRESOLVED_DEPTH_MAX, + self.accepted_parent_redrive_attempt_max = ( + block_config.accepted_parent_redrive_attempt_max ) - self.block_submit_lock_wait_log_seconds = env_positive_float( - "PRISM_BLOCK_SUBMIT_LOCK_WAIT_LOG_SECONDS", - DEFAULT_BLOCK_SUBMIT_LOCK_WAIT_LOG_SECONDS, + self.block_submit_lock_wait_log_seconds = ( + block_config.submit_lock_wait_log_seconds ) - self.block_submit_stuck_call_exit_seconds = env_positive_float( - "PRISM_BLOCK_SUBMIT_STUCK_CALL_EXIT_SECONDS", - DEFAULT_BLOCK_SUBMIT_STUCK_CALL_EXIT_SECONDS, + self.block_submit_stuck_call_exit_seconds = ( + block_config.submit_stuck_call_exit_seconds ) # An own-hash tip observation is acceptance evidence even when the # direct submitblock ack was lost; this window bounds how long that # evidence blocks a terminal abandonment while fresh chain probes # cannot prove the candidate active. - self.observed_tip_accept_window_seconds = env_positive_float( - "PRISM_OBSERVED_TIP_ACCEPT_WINDOW_SECONDS", - DEFAULT_PRISM_OBSERVED_TIP_ACCEPT_WINDOW_SECONDS, + self.observed_tip_accept_window_seconds = ( + block_config.observed_tip_accept_window_seconds ) # After a FAILED refresh pass, the fallback poller re-attempts no # sooner than this many seconds (plus jitter) while the tip the failed @@ -3258,344 +2335,186 @@ def __init__(self) -> None: # sustained payout churn costs ~1 attempt/second instead of one per # 0.25s trigger slice. A successful pass or a newly observed tip # re-arms immediately. Zero restores unspaced retries. - self.tip_refresh_failure_holdoff_seconds = env_nonnegative_float( - "PRISM_TIP_REFRESH_FAILURE_HOLDOFF_SECONDS", - DEFAULT_PRISM_TIP_REFRESH_FAILURE_HOLDOFF_SECONDS, + self.tip_refresh_failure_holdoff_seconds = ( + job_config.tip_refresh_failure_holdoff_seconds ) # Zero disables stale-grace crediting (every prior-tip share rejects, # the pre-grace behavior). - self.stale_grace_seconds = env_nonnegative_float( - "PRISM_STRATUM_STALE_GRACE_SECONDS", - DEFAULT_PRISM_STALE_GRACE_SECONDS, - ) + self.stale_grace_seconds = stratum_config.stale_grace_seconds # Per-share/per-job stdout logging is debug-only: at production share # rates each print is a journald flush on the Stratum hot path. - self.hot_path_log_enabled = env_bool("PRISM_HOT_PATH_LOG", "0") + self.hot_path_log_enabled = self.config.hot_path_log_enabled # Zero disables the observed-tip reuse (every submit re-reads the tip # over RPC, the legacy behavior). - self.submit_tip_max_age_seconds = env_nonnegative_float( - "PRISM_SUBMIT_TIP_MAX_AGE_SECONDS", - DEFAULT_PRISM_SUBMIT_TIP_MAX_AGE_SECONDS, - ) - self.same_tip_job_retention_seconds = env_nonnegative_float( - "PRISM_STRATUM_SAME_TIP_JOB_RETENTION_SECONDS", - DEFAULT_PRISM_SAME_TIP_JOB_RETENTION_SECONDS, - ) - self.same_tip_job_retention_per_connection = env_nonnegative_int( - "PRISM_STRATUM_SAME_TIP_JOB_RETENTION_PER_CONNECTION", - DEFAULT_PRISM_SAME_TIP_JOB_RETENTION_PER_CONNECTION, - ) - self.disconnected_job_retention = env_nonnegative_int( - "PRISM_STRATUM_DISCONNECTED_JOB_RETENTION", - DEFAULT_PRISM_DISCONNECTED_JOB_RETENTION, - ) - self.tip_refresh_max_workers = env_positive_int( - "PRISM_TIP_REFRESH_MAX_WORKERS", - DEFAULT_PRISM_TIP_REFRESH_MAX_WORKERS, - ) - if self.tip_refresh_max_workers > DEFAULT_PRISM_TIP_REFRESH_MAX_WORKERS: - raise SystemExit( - "PRISM_TIP_REFRESH_MAX_WORKERS cannot exceed " - f"{DEFAULT_PRISM_TIP_REFRESH_MAX_WORKERS}" - ) - self.tip_refresh_epoch_fanout = env_bool( - "PRISM_TIP_REFRESH_EPOCH_FANOUT", - "0", - ) - self.job_build_timeout_seconds = env_positive_float( - "PRISM_JOB_BUILD_TIMEOUT_SECONDS", - DEFAULT_PRISM_JOB_BUILD_TIMEOUT_SECONDS, - ) - self.job_build_cancel_grace_seconds = env_nonnegative_float( - "PRISM_JOB_BUILD_CANCEL_GRACE_SECONDS", - DEFAULT_PRISM_JOB_BUILD_CANCEL_GRACE_SECONDS, - ) - self.vardiff_idle_sweep_seconds = env_nonnegative_float( - "PRISM_STRATUM_VARDIFF_IDLE_SWEEP_SECONDS", - DEFAULT_PRISM_VARDIFF_IDLE_SWEEP_SECONDS, + self.submit_tip_max_age_seconds = job_config.submit_tip_max_age_seconds + self.same_tip_job_retention_seconds = ( + stratum_config.same_tip_job_retention_seconds + ) + self.same_tip_job_retention_per_connection = ( + stratum_config.same_tip_job_retention_per_connection + ) + self.disconnected_job_retention = stratum_config.disconnected_job_retention + self.tip_refresh_max_workers = job_config.tip_refresh_max_workers + self.tip_refresh_epoch_fanout = job_config.tip_refresh_epoch_fanout + self.job_build_timeout_seconds = job_config.job_build_timeout_seconds + self.job_build_cancel_grace_seconds = job_config.job_build_cancel_grace_seconds + self.vardiff_idle_sweep_seconds = stratum_config.vardiff_idle_sweep_seconds + # Reconnect difficulty retention: a reconnecting worker resumes at + # its last converged difficulty instead of the lane start. Off + # restores lane-start behavior exactly. + self.vardiff_resume_enabled = stratum_config.vardiff_resume_enabled + self.vardiff_resume_ttl_seconds = stratum_config.vardiff_resume_ttl_seconds + self.vardiff_resume_max_entries = stratum_config.vardiff_resume_max_entries + self.vardiff_resume_max_start_factor = ( + stratum_config.vardiff_resume_max_start_factor ) # Zero collapses every worker into the overflow label (per-worker # metrics effectively off) without touching the aggregate counters. - self.worker_metrics_limit = env_nonnegative_int( - "PRISM_WORKER_METRICS_LIMIT", - DEFAULT_PRISM_WORKER_METRICS_LIMIT, - ) - self.reorg_reconciler_enabled = env_bool("PRISM_REORG_RECONCILER_ENABLED", "1") + self.worker_metrics_limit = job_config.worker_metrics_limit + self.reorg_reconciler_enabled = job_config.reorg_reconciler_enabled # Per-template job caching. A zero disables the corresponding cache # (every build redoes that stage), which is also the legacy behavior. - self.job_bundle_cache_seconds = env_nonnegative_float( - "PRISM_JOB_BUNDLE_CACHE_SECONDS", - DEFAULT_PRISM_JOB_BUNDLE_CACHE_SECONDS, - ) - self.bundle_build_timeout_seconds = env_positive_float( - "PRISM_BUNDLE_BUILD_TIMEOUT_SECONDS", - DEFAULT_PRISM_BUNDLE_BUILD_TIMEOUT_SECONDS, - ) - self.payout_artifact_rearm_min_seconds = env_positive_float( - "PRISM_PAYOUT_ARTIFACT_REARM_MIN_SECONDS", - DEFAULT_PRISM_PAYOUT_ARTIFACT_REARM_MIN_SECONDS, + self.job_bundle_cache_seconds = job_config.job_bundle_cache_seconds + self.bundle_build_timeout_seconds = job_config.bundle_build_timeout_seconds + self.payout_artifact_rearm_min_seconds = ( + job_config.payout_artifact_rearm_min_seconds ) # Master kill-switch for the payout-artifact reuse line. Off # restores the pre-reuse behavior (every ready build pays the # synchronous reward-window walk and no background walks run), so a # production regression is an env flip instead of a rollback. - self.payout_artifact_reuse_enabled = env_bool( - "PRISM_PAYOUT_ARTIFACT_REUSE", - "1", - ) - self.payout_artifact_reanchor_seconds = env_positive_float( - "PRISM_PAYOUT_ARTIFACT_REANCHOR_SECONDS", - DEFAULT_PRISM_PAYOUT_ARTIFACT_REANCHOR_SECONDS, - ) - self.payout_artifact_min_build_interval_seconds = env_positive_float( - "PRISM_PAYOUT_ARTIFACT_MIN_BUILD_INTERVAL_SECONDS", - self.payout_artifact_reanchor_seconds, - ) - self.payout_artifact_full_rescan_seconds = env_positive_float( - "PRISM_PAYOUT_ARTIFACT_FULL_RESCAN_SECONDS", - DEFAULT_PRISM_PAYOUT_ARTIFACT_FULL_RESCAN_SECONDS, - ) - self.payout_artifact_max_anchor_age_seconds = env_positive_float( - "PRISM_PAYOUT_ARTIFACT_MAX_ANCHOR_AGE_SECONDS", - DEFAULT_PRISM_PAYOUT_ARTIFACT_MAX_ANCHOR_AGE_SECONDS, - ) - validate_payout_artifact_age_bounds( - self.payout_artifact_max_anchor_age_seconds, - max( - self.payout_artifact_reanchor_seconds, - self.payout_artifact_min_build_interval_seconds, - ), - ) - self.routine_admission_deadline_seconds = env_positive_float( - "PRISM_ROUTINE_ADMISSION_DEADLINE_SECONDS", - DEFAULT_PRISM_ROUTINE_ADMISSION_DEADLINE_SECONDS, + self.payout_artifact_reuse_enabled = job_config.payout_artifact_reuse_enabled + self.payout_artifact_reanchor_seconds = ( + job_config.payout_artifact_reanchor_seconds ) - self.template_cache_seconds = env_nonnegative_float( - "PRISM_TEMPLATE_CACHE_SECONDS", - self.blockpoll_seconds, + self.payout_artifact_min_build_interval_seconds = ( + job_config.payout_artifact_min_build_interval_seconds ) - self.template_refresh_failure_exit_seconds = env_nonnegative_float( - "PRISM_TEMPLATE_REFRESH_FAILURE_EXIT_SECONDS", - DEFAULT_PRISM_TEMPLATE_MAX_AGE_SECONDS, - ) - self.coordination_blocked_exit_seconds = env_positive_float( - "PRISM_COORDINATION_BLOCKED_EXIT_SECONDS", - DEFAULT_PRISM_COORDINATION_BLOCKED_EXIT_SECONDS, - ) - if production_mode() and self.template_refresh_failure_exit_seconds <= 0: - raise SystemExit( - "production mode requires a positive " - "PRISM_TEMPLATE_REFRESH_FAILURE_EXIT_SECONDS" - ) - self.last_successful_template_refresh_monotonic: float | None = None - self.template_refresh_failure_started_monotonic: float | None = None - self._coordination_blocked_lock = threading.Lock() - self._coordination_blocked_since_monotonic: float | None = None - self._publication_watchdog_exit_claimed = False - self.reorg_reconcile_cache_seconds = env_nonnegative_float( - "PRISM_REORG_RECONCILE_CACHE_SECONDS", - DEFAULT_PRISM_REORG_RECONCILE_CACHE_SECONDS, + self.payout_artifact_full_rescan_seconds = ( + job_config.payout_artifact_full_rescan_seconds ) - self.health_refresh_seconds = env_positive_float( - "PRISM_HEALTH_REFRESH_SECONDS", - DEFAULT_PRISM_HEALTH_REFRESH_SECONDS, + self.payout_artifact_max_anchor_age_seconds = ( + job_config.payout_artifact_max_anchor_age_seconds ) - self.health_pending_refresh_max_age_seconds = env_positive_float( - "PRISM_HEALTH_PENDING_REFRESH_MAX_AGE_SECONDS", - DEFAULT_PRISM_HEALTH_PENDING_REFRESH_MAX_AGE_SECONDS, + self.routine_admission_deadline_seconds = ( + job_config.routine_admission_deadline_seconds ) - self.health_tip_poll_max_age_seconds = env_positive_float( - "PRISM_HEALTH_TIP_POLL_MAX_AGE_SECONDS", - DEFAULT_PRISM_HEALTH_TIP_POLL_MAX_AGE_SECONDS, + self.template_cache_seconds = job_config.template_cache_seconds + self.template_refresh_failure_exit_seconds = ( + job_config.template_refresh_failure_exit_seconds ) - self.stratum_send_timeout_seconds = env_nonnegative_float( - "PRISM_STRATUM_SEND_TIMEOUT_SECONDS", - DEFAULT_PRISM_STRATUM_SEND_TIMEOUT_SECONDS, + self.coordination_blocked_exit_seconds = ( + lifecycle_config.coordination_blocked_exit_seconds ) + self.last_successful_template_refresh_monotonic: float | None = None + self.template_refresh_failure_started_monotonic: float | None = None + # Coordination-blocked streak state moved to the WatchdogService + # owner (see lab/prism/background_services.py). + self.reorg_reconcile_cache_seconds = job_config.reorg_reconcile_cache_seconds + self.health_refresh_seconds = lifecycle_config.health_refresh_seconds + self.metrics_refresh_seconds = lifecycle_config.metrics_refresh_seconds + self.health_pending_refresh_max_age_seconds = ( + lifecycle_config.pending_refresh_health_deadline_seconds + ) + self.health_tip_poll_max_age_seconds = ( + lifecycle_config.coherent_tip_poll_health_deadline_seconds + ) + self.stratum_send_timeout_seconds = stratum_config.send_timeout_seconds # Zero remains an explicit unlimited option for focused local/regtest # use. Deployments default to a conservative ceiling above PRISM's # normal 200-250 connection population. - self.stratum_max_connections = env_nonnegative_int( - "PRISM_STRATUM_MAX_CONNECTIONS", - DEFAULT_PRISM_STRATUM_MAX_CONNECTIONS, - ) - self.stratum_max_connections_per_username = env_nonnegative_int( - "PRISM_STRATUM_MAX_CONNECTIONS_PER_USERNAME", - DEFAULT_PRISM_STRATUM_MAX_CONNECTIONS_PER_USERNAME, - ) - self.stratum_max_pending_initial_jobs = env_positive_int( - "PRISM_STRATUM_MAX_PENDING_INITIAL_JOBS", - DEFAULT_PRISM_STRATUM_MAX_PENDING_INITIAL_JOBS, - ) - self.initial_job_max_workers = resolve_initial_job_max_workers( - env_optional_positive_int("PRISM_INITIAL_JOB_MAX_WORKERS"), - self.stratum_max_pending_initial_jobs, - ) - self.job_build_executor_workers = validate_job_build_executor_workers( - env_positive_int( - "PRISM_JOB_BUILD_EXECUTOR_WORKERS", - DEFAULT_PRISM_JOB_BUILD_EXECUTOR_WORKERS, + self.stratum_max_connections = stratum_config.max_connections + self.stratum_max_connections_per_username = ( + stratum_config.max_connections_per_username + ) + self.stratum_max_pending_initial_jobs = stratum_config.max_pending_initial_jobs + if config is None: + # The worker knobs stay visibly env-wired in this module for the + # zero-argument production path (an upstream lost-fix detector + # asserts the source); the values are identical to what the + # snapshot loader just validated. A supplied snapshot never + # rereads the environment. + self.initial_job_max_workers = resolve_initial_job_max_workers( + env_optional_positive_int("PRISM_INITIAL_JOB_MAX_WORKERS"), + self.stratum_max_pending_initial_jobs, + ) + self.job_build_executor_workers = validate_job_build_executor_workers( + env_positive_int( + "PRISM_JOB_BUILD_EXECUTOR_WORKERS", + DEFAULT_PRISM_JOB_BUILD_EXECUTOR_WORKERS, + ) ) + else: + self.initial_job_max_workers = stratum_config.initial_job_max_workers + self.job_build_executor_workers = job_config.job_build_executor_workers + self.stratum_initial_job_timeout_seconds = ( + stratum_config.initial_job_timeout_seconds + ) + self.mining_health_startup_grace_seconds = ( + lifecycle_config.mining_health_startup_grace_seconds + ) + self.stratum_accept_resource_exhaustion_backoff_seconds = ( + stratum_config.accept_resource_exhaustion_backoff_seconds + ) + self.stratum_listen_backlog = stratum_config.listen_backlog + self.stratum_bind_retry_seconds = stratum_config.bind_retry_seconds + self.payout_address_cache_max_entries = ( + stratum_config.payout_address_cache_max_entries + ) + self.payout_address_cache_ttl_seconds = ( + stratum_config.payout_address_cache_ttl_seconds + ) + self.coinbase_tag_hex = self.config.coinbase_tag_hex + self.share_difficulty = stratum_config.share_difficulty + self.vardiff_config = stratum_config.vardiff_config + self.listener_profiles = list(stratum_config.listener_profiles) + self.default_share_weight = stratum_config.default_share_weight + self.share_weights_by_username = dict(stratum_config.share_weights_by_username) + self.username_fallback_address = stratum_config.username_fallback_address + self.min_ready_miners = job_config.min_ready_miners + self.signing_seed_hex = ledger_config.signing_seed_hex + self.ledger_attestation_signing_seed_hex = ( + ledger_config.attestation_signing_seed_hex + ) + self.ledger_writer_public_key_hex = ledger_config.writer_public_key_hex + self.evidence_path = audit_config.evidence_path + self.audit_dir = audit_config.directory + self.audit_share_segment_size = audit_config.share_segment_size + self.audit_live_bundle_retention = audit_config.live_bundle_retention + self.audit_candidate_retention_seconds = ( + audit_config.candidate_retention_seconds + ) + self.ctv_broadcast_attempt_detail_limit = ( + ctv_config.broadcast_attempt_detail_limit + ) + self.ctv_broadcast_retry_backoff_seconds = ( + ctv_config.broadcast_retry_backoff_seconds + ) + self.audit_bind = audit_config.bind + self.audit_port = audit_config.port + self.stop_after_block = self.config.stop_after_block + self.max_blocks = self.config.max_blocks + self.version_mask_selection = self.resolve_version_rolling_mask( + stratum_config.fallback_version_mask ) - if ( - self.stratum_max_connections > 0 - and self.stratum_max_pending_initial_jobs > self.stratum_max_connections - ): - raise SystemExit( - "PRISM_STRATUM_MAX_PENDING_INITIAL_JOBS cannot exceed " - "PRISM_STRATUM_MAX_CONNECTIONS" - ) - self.stratum_initial_job_timeout_seconds = env_nonnegative_float( - "PRISM_STRATUM_INITIAL_JOB_TIMEOUT_SECONDS", - DEFAULT_PRISM_STRATUM_INITIAL_JOB_TIMEOUT_SECONDS, - ) - self.mining_health_startup_grace_seconds = env_nonnegative_float( - "PRISM_MINING_HEALTH_STARTUP_GRACE_SECONDS", - DEFAULT_PRISM_MINING_HEALTH_STARTUP_GRACE_SECONDS, - ) - validate_same_tip_job_retention_limits( - retention_seconds=self.same_tip_job_retention_seconds, - per_connection=self.same_tip_job_retention_per_connection, - max_connections=self.stratum_max_connections, - production=production_mode(), - ) - self.stratum_accept_resource_exhaustion_backoff_seconds = env_positive_float( - "PRISM_STRATUM_ACCEPT_RESOURCE_EXHAUSTION_BACKOFF_SECONDS", - DEFAULT_PRISM_STRATUM_ACCEPT_RESOURCE_EXHAUSTION_BACKOFF_SECONDS, - ) - self.stratum_listen_backlog = env_positive_int( - "PRISM_STRATUM_LISTEN_BACKLOG", - DEFAULT_PRISM_STRATUM_LISTEN_BACKLOG, - ) - self.stratum_bind_retry_seconds = env_nonnegative_float( - "PRISM_STRATUM_BIND_RETRY_SECONDS", - DEFAULT_PRISM_STRATUM_BIND_RETRY_SECONDS, - ) - self.payout_address_cache_max_entries = env_nonnegative_int( - "PRISM_PAYOUT_ADDRESS_CACHE_MAX_ENTRIES", - DEFAULT_PRISM_PAYOUT_ADDRESS_CACHE_MAX_ENTRIES, - ) - self.payout_address_cache_ttl_seconds = env_nonnegative_float( - "PRISM_PAYOUT_ADDRESS_CACHE_TTL_SECONDS", - DEFAULT_PRISM_PAYOUT_ADDRESS_CACHE_TTL_SECONDS, - ) - self.coinbase_tag_hex = default_prism_coinbase_tag_hex() - self.share_difficulty = env_decimal("PRISM_STRATUM_SHARE_DIFF", "0.000000001") - self.vardiff_config = load_prism_vardiff_config(self.share_difficulty) - self.listener_profiles = [ - StratumListenerProfile( - name="default", - bind=self.bind, - port=self.port, - share_difficulty=self.share_difficulty, - vardiff_config=self.vardiff_config, - heartbeat_name="stratum_accept", - ) - ] - highdiff_profile = load_prism_highdiff_listener(self.bind, self.vardiff_config) - if highdiff_profile is not None: - if highdiff_profile.port == self.port and highdiff_profile.bind == self.bind: - raise SystemExit("PRISM_STRATUM_HIGHDIFF_PORT must differ from PRISM_STRATUM_PORT") - self.listener_profiles.append(highdiff_profile) - self.default_share_weight = env_int("PRISM_STRATUM_SHARE_WEIGHT", 1) - if self.default_share_weight <= 0: - raise SystemExit("PRISM_STRATUM_SHARE_WEIGHT must be positive") - self.share_weights_by_username = self.parse_share_weights() - self.username_fallback_address = default_prism_username_fallback_address() - self.min_ready_miners = env_int("PRISM_MIN_READY_MINERS", 3) - self.signing_seed_hex = env_seed_hex( - "PRISM_MANIFEST_SIGNING_SEED_HEX", - test_default="42" * 32, - ) - self.ledger_attestation_signing_seed_hex = env_seed_hex( - "PRISM_LEDGER_ATTESTATION_SIGNING_SEED_HEX", - test_default="43" * 32, - ) - self.ledger_writer_public_key_hex = self.load_trusted_ledger_writer_public_key() - self.evidence_path = Path(env("PRISM_EVIDENCE_PATH", "prism-live-evidence.json")) - self.audit_dir = Path(env("PRISM_AUDIT_DIR", str(self.evidence_path.parent))) - self.audit_dir.mkdir(parents=True, exist_ok=True) - self.audit_share_segment_size = env_nonnegative_int( - "PRISM_AUDIT_SHARE_SEGMENT_SIZE", - DEFAULT_AUDIT_SHARE_SEGMENT_SIZE, - ) - self.audit_live_bundle_retention = env_nonnegative_int("PRISM_AUDIT_LIVE_BUNDLE_RETENTION", 5) - self.audit_candidate_retention_seconds = env_nonnegative_int( - "PRISM_AUDIT_CANDIDATE_RETENTION_SECONDS", - 24 * 60 * 60, - ) - self.ctv_broadcast_attempt_detail_limit = env_nonnegative_int( - "PRISM_CTV_BROADCAST_ATTEMPT_DETAIL_LIMIT", - DEFAULT_CTV_BROADCAST_ATTEMPT_DETAIL_LIMIT, - ) - self.ctv_broadcast_retry_backoff_seconds = env_nonnegative_int( - "PRISM_CTV_BROADCAST_RETRY_BACKOFF_SECONDS", - DEFAULT_CTV_BROADCAST_RETRY_BACKOFF_SECONDS, - ) - self.audit_bind = os.environ.get("PRISM_AUDIT_BIND") - self.audit_port = int(os.environ.get("PRISM_AUDIT_PORT", "0") or "0") - self.stop_after_block = env("PRISM_STOP_AFTER_BLOCK", "1") in {"1", "true", "yes"} - self.max_blocks = env_int("PRISM_MAX_BLOCKS", 1) - if self.max_blocks <= 0: - raise SystemExit("PRISM_MAX_BLOCKS must be positive") - try: - fallback_version_mask = direct_stratum.normalize_version_rolling_mask( - env("PRISM_VERSION_ROLLING_MASK", direct_stratum.QBIT_VERSION_ROLLING_MASK_HEX), - field_name="PRISM_VERSION_ROLLING_MASK", - ) - except ValueError as exc: - raise SystemExit(str(exc)) from exc - self.version_mask_selection = self.resolve_version_rolling_mask(fallback_version_mask) self.version_mask = self.version_mask_selection.selected_mask - self.writer_quiescence_timeout_seconds = env_positive_float( - "PRISM_WRITER_QUIESCENCE_TIMEOUT_SECONDS", - DEFAULT_PRISM_WRITER_QUIESCENCE_TIMEOUT_SECONDS, + self.writer_quiescence_timeout_seconds = ( + lifecycle_config.writer_quiescence_timeout_seconds ) self.ledger = self.make_ledger() - configured_ctv_broadcaster_enabled = env_optional_bool("PRISM_CTV_BROADCASTER_ENABLED") - self.ctv_broadcaster_enabled = ( - configured_ctv_broadcaster_enabled - if configured_ctv_broadcaster_enabled is not None - else env_bool("PRISM_CTV_SETTLEMENT_ENABLED", "0") - ) - self.ctv_broadcaster_wallet = env_optional("PRISM_CTV_BROADCASTER_WALLET") - self.ctv_broadcaster_fee_sats = env_nonnegative_int_with_legacy( - "PRISM_CTV_BROADCASTER_FEE_BITS", - "PRISM_CTV_BROADCASTER_FEE_SATS", - 0, - ) - if ( - self.ctv_broadcaster_enabled - and self.ctv_broadcaster_fee_sats > 0 - and not self.ctv_broadcaster_wallet - ): - raise SystemExit( - "PRISM_CTV_BROADCASTER_WALLET is required when " - "PRISM_CTV_BROADCASTER_FEE_BITS is positive" - ) - self.ctv_broadcaster_limit = env_positive_int("PRISM_CTV_BROADCASTER_LIMIT", 100) - self.ctv_broadcaster_chunk_size = env_positive_int( - "PRISM_CTV_BROADCASTER_CHUNK_SIZE", - DEFAULT_PRISM_CTV_BROADCASTER_CHUNK_SIZE, - ) - if self.ctv_broadcaster_chunk_size > MAX_CTV_FANOUT_BROADCASTER_CHUNK_SIZE: - raise SystemExit( - "PRISM_CTV_BROADCASTER_CHUNK_SIZE must be at most " - f"{MAX_CTV_FANOUT_BROADCASTER_CHUNK_SIZE}" - ) - self.ctv_broadcaster_interval_seconds = env_positive_float( - "PRISM_CTV_BROADCASTER_INTERVAL_SECONDS", - 30.0, - ) - self._ctv_broadcaster_metrics_lock = threading.Lock() - self.ctv_broadcaster_pass_seconds_bucket_counts = { - bucket: 0 for bucket in PRISM_CTV_BROADCASTER_SECONDS_BUCKETS - } - self.ctv_broadcaster_pass_seconds_sum = 0.0 - self.ctv_broadcaster_pass_count = 0 - self.ctv_broadcaster_processed_rows_total = 0 + # Incremental hashrate rollup maintenance rides the writer because + # only the lease holder may mutate ledger-derived state; the public + # read tier never runs it. The flag and cadence stay plain + # attributes so tests and embedders can retune a live coordinator. + self.hashrate_rollup_enabled = env_bool("PRISM_HASHRATE_ROLLUP_ENABLED", "1") + self.hashrate_rollup_interval_seconds = env_positive_float( + "PRISM_HASHRATE_ROLLUP_INTERVAL_SECONDS", 15.0 + ) + self.hashrate_rollup_batch_shares = env_positive_int( + "PRISM_HASHRATE_ROLLUP_BATCH_SHARES", 50000 + ) + self._upgrade_legacy_audit_evidence() self._ctv_fanout_market_fee_rate_cache: dict[tuple[int | None, str | None], int] = {} - self.ctv_fanout_broadcast_daemon: CtvFanoutBroadcastDaemon | None = None self.lock = _ObservedRLock( wait_observer=self._observe_coordinator_lock_wait, ) @@ -3612,8 +2531,6 @@ def __init__(self) -> None: self.initial_job_coalesced_count = 0 self.initial_job_queue_capacity_reclaimed_count = 0 self.last_initial_job_delivery_monotonic: float | None = None - self._mining_overload_started_monotonic: float | None = None - self._mining_delivery_failure_started_monotonic: float | None = None self._p2mr_address_cache_lock = threading.Lock() self._p2mr_address_cache: OrderedDict[ str, tuple[float, tuple[str, str]] @@ -3624,8 +2541,8 @@ def __init__(self) -> None: self.jobs: dict[str, PrismJobContext] = {} # Share-path ownership is deliberately disjoint from the coordinator # control-plane lock. Deduplication remains process-wide so an exact - # header replay across reauthorization/connections is still rejected. - self._recent_share_lock = threading.Lock() + # header replay across reauthorization/connections is still rejected; + # the submission owner adopts this seed on first hot-path touch. self.recent_share_keys: set[tuple[object, ...]] = set() self._share_accounting_lock = threading.Lock() self.connection_counter = 0 @@ -3719,6 +2636,12 @@ def __init__(self) -> None: self._block_accounting_overflow_queue: queue.PriorityQueue[ tuple[int, int, _BlockCandidateAccountingTask] ] = queue.PriorityQueue() + # Definitive node acceptances only, and unbounded for the same reason + # the overflow queue is: the offer already happened, so it may never + # be converted back into a raw-submit retry. + self._block_accounting_accepted_queue: queue.PriorityQueue[ + tuple[int, int, _BlockCandidateAccountingTask] + ] = queue.PriorityQueue() self._block_accounting_sequence = 0 self._block_accounting_state_lock = threading.Lock() self._block_accounting_thread: threading.Thread | None = None @@ -3786,31 +2709,20 @@ def __init__(self) -> None: self.share_append_queue: queue.Queue[PendingShareAppend] = queue.Queue( maxsize=MAX_PENDING_SHARE_APPENDS ) - self.share_commit_batch_size = env_positive_int( - "PRISM_SHARE_COMMIT_BATCH_SIZE", DEFAULT_SHARE_COMMIT_BATCH_SIZE - ) - self.share_commit_linger_seconds = ( - env_nonnegative_float( - "PRISM_SHARE_COMMIT_LINGER_MILLISECONDS", - DEFAULT_SHARE_COMMIT_LINGER_MILLISECONDS, - ) - / 1000.0 - ) - self.share_commit_timeout_seconds = env_positive_float( - "PRISM_SHARE_COMMIT_TIMEOUT_SECONDS", DEFAULT_SHARE_COMMIT_TIMEOUT_SECONDS - ) + self.share_commit_batch_size = ledger_config.share_commit_batch_size + self.share_commit_linger_seconds = ledger_config.share_commit_linger_seconds + self.share_commit_timeout_seconds = ledger_config.share_commit_timeout_seconds self.share_writer_active = False self.share_append_failure_count = 0 # Retain the historical recovery-file reader for clean upgrades from a # release that could acknowledge before Postgres commit. New shares # are never written here: an unavailable ledger produces no success # acknowledgement and an exact retry is idempotent. - self.share_recovery_path = Path( - env("PRISM_SHARE_RECOVERY_PATH", str(self.audit_dir / "prism-unpersisted-shares.jsonl")) - ) + self.share_recovery_path = ledger_config.share_recovery_path self.share_recovery_lock = threading.Lock() self.shares_recovered_to_disk = 0 self.shares_replayed = 0 + self.share_replay_conflicts = 0 self.job_build_failure_count = 0 self.tip_refresh_job_count = 0 self.post_accept_refresh_failure_count = 0 @@ -3819,7 +2731,6 @@ def __init__(self) -> None: self.reorg_reconcile_skip_count = 0 self.reorg_reconcile_error_count = 0 self.matured_payout_count = 0 - self.latest_evidence: dict[str, Any] | None = None # The full accepted-block bundle is durable in the audit store. Keeping # it here only to derive one metric pinned the complete share window for # the lifetime of the coordinator. @@ -3834,5285 +2745,1881 @@ def __init__(self) -> None: self._initial_job_executor: _BoundedPriorityExecutor | None = None self._initial_job_executor_shutdown = False self._tip_refresh_pending_event = threading.Event() - self._tip_refresh_pending_counter = 0 - self._tip_refresh_pending_token: int | None = None - self._tip_refresh_retry = threading.Event() - self._tip_refresh_retry_counter = 0 - self._tip_refresh_retry_consumed = 0 - self._tip_refresh_failure_holdoff_until: float | None = None - self._tip_refresh_failure_tip: str | None = None - self._active_tip_refresh: tuple[ - TipRefreshValidationToken, - _FanoutCancellation, - ] | None = None - self._retained_collection_refresh: RetainedCollectionRefresh | None = None - self.last_reorg_reconciled_tip_hash: str | None = None - self.last_reorg_reconciled_trusted = False - self.last_reorg_reconciled_monotonic: float | None = None - self._prism_payout_policy_cache: dict[str, object] | None = None - self._ensure_job_cache_state() - self.stop_event = threading.Event() - self._shutdown_controller = CoordinatorShutdownController( - self.writer_quiescence_timeout_seconds - ) - # Liveness watchdog: each monitored loop stamps a monotonic heartbeat; - # if any goes stale past the timeout the process exits non-zero so the - # container/systemd restart policy recovers a *hung* coordinator (a - # healthcheck alone does not restart it under plain compose). - self._heartbeats: dict[str, float] = {} - self._heartbeat_phases: dict[str, str] = {} - self._watchdog_pauses: dict[str, int] = {} - self._heartbeats_lock = threading.Lock() - self.watchdog_enabled = env_bool("PRISM_WATCHDOG_ENABLED", "1") - self.watchdog_timeout_seconds = env_positive_float("PRISM_WATCHDOG_TIMEOUT_SECONDS", 120.0) - self.watchdog_interval_seconds = env_positive_float("PRISM_WATCHDOG_INTERVAL_SECONDS", 15.0) - - def _ensure_share_hot_path_state(self) -> None: - """Backfill dedicated hot-path locks for lightweight test embedders.""" - if ( - hasattr(self, "_recent_share_lock") - and hasattr(self, "_share_accounting_lock") - and hasattr(self, "recent_share_keys") - and hasattr(self, "share_ack_histograms") - ): - return - with _HOT_PATH_LOCK_INITIALIZATION_LOCK: - if not hasattr(self, "_recent_share_lock"): - self._recent_share_lock = threading.Lock() - if not hasattr(self, "_share_accounting_lock"): - self._share_accounting_lock = threading.Lock() - if not hasattr(self, "recent_share_keys"): - self.recent_share_keys = set() - if not hasattr(self, "share_ack_histograms"): - self.share_ack_histograms = { - result: { - "buckets": { - bucket: 0 - for bucket in PRISM_JOB_BUILD_SECONDS_BUCKETS - }, - "sum": 0.0, - "count": 0, - } - for result in PRISM_SHARE_ACK_RESULTS - } - - def _observe_share_ack_seconds( - self, - result: str, - elapsed_seconds: float, - ) -> None: - if result not in PRISM_SHARE_ACK_RESULTS: - raise ValueError(f"unknown share ack result: {result}") - self._ensure_share_hot_path_state() - with self._share_accounting_lock: - histogram = self.share_ack_histograms[result] - histogram["count"] = int(histogram["count"]) + 1 - histogram["sum"] = float(histogram["sum"]) + max( - 0.0, elapsed_seconds - ) - buckets = histogram["buckets"] - assert isinstance(buckets, dict) - for bucket in PRISM_JOB_BUILD_SECONDS_BUCKETS: - if elapsed_seconds <= bucket: - buckets[bucket] = int(buckets.get(bucket, 0)) + 1 - - def share_ack_metrics_lines(self) -> list[str]: - self._ensure_share_hot_path_state() - with self._share_accounting_lock: - histograms = { - result: { - "buckets": dict(histogram["buckets"]), - "sum": float(histogram["sum"]), - "count": int(histogram["count"]), - } - for result, histogram in self.share_ack_histograms.items() - } - lines = [ - "# HELP qbit_prism_share_ack_seconds mining.submit line arrival to Stratum response, by outcome.", - "# TYPE qbit_prism_share_ack_seconds histogram", - ] - for result in PRISM_SHARE_ACK_RESULTS: - histogram = histograms[result] - buckets = histogram["buckets"] - lines.extend( - f'qbit_prism_share_ack_seconds_bucket{{result="{result}",le="{bucket:g}"}} {int(buckets.get(bucket, 0))}' - for bucket in PRISM_JOB_BUILD_SECONDS_BUCKETS - ) - lines.append( - f'qbit_prism_share_ack_seconds_bucket{{result="{result}",le="+Inf"}} {histogram["count"]}' - ) - lines.append( - f'qbit_prism_share_ack_seconds_sum{{result="{result}"}} {histogram["sum"]:.6f}' - ) - lines.append( - f'qbit_prism_share_ack_seconds_count{{result="{result}"}} {histogram["count"]}' - ) - return lines - - @staticmethod - def _client_vardiff_lock(client: ClientState) -> threading.RLock: - # Production ClientState instances always have this field. The lazy - # fallback preserves focused tests and embedders that construct a - # ClientState with __new__ and populate only the attributes they use. - lock = getattr(client, "vardiff_lock", None) - if lock is not None: - return lock - with _HOT_PATH_LOCK_INITIALIZATION_LOCK: - lock = getattr(client, "vardiff_lock", None) - if lock is None: - lock = threading.RLock() - client.vardiff_lock = lock - return lock - - def _reserve_recent_share_key(self, share_key: tuple[object, ...]) -> bool: - self._ensure_share_hot_path_state() - with self._recent_share_lock: - if share_key in self.recent_share_keys: - return False - if len(self.recent_share_keys) > 50_000: - self.recent_share_keys.clear() - self.recent_share_keys.add(share_key) - return True - - def _forget_recent_share_key(self, share_key: tuple[object, ...]) -> None: - self._ensure_share_hot_path_state() - with self._recent_share_lock: - self.recent_share_keys.discard(share_key) - - def record_rejection(self, reason: str, *, worker: str | None = None) -> None: - if reason not in PRISM_REJECTION_REASON_IDS: - raise ValueError(f"unknown PRISM rejection reason: {reason}") - self._ensure_share_hot_path_state() - with self._share_accounting_lock: - counts = getattr(self, "rejection_counts_by_reason", None) - if counts is None: - counts = {reason_id: 0 for reason_id in PRISM_REJECTION_REASON_IDS} - self.rejection_counts_by_reason = counts - counts[reason] = int(counts.get(reason, 0)) + 1 - if reason in {PRISM_REJECTION_STALE_JOB, PRISM_REJECTION_UNKNOWN_JOB, PRISM_REJECTION_BLOCK_STALE}: - self.stale_share_count += 1 - elif reason == PRISM_REJECTION_DUPLICATE_SHARE: - self.duplicate_share_count += 1 - elif reason == PRISM_REJECTION_LOW_DIFFICULTY: - self.low_difficulty_share_count += 1 - if worker is not None: - self._ensure_worker_metrics_state() - with self.worker_metrics_lock: - label = self._worker_metric_label_locked(worker) - key = (label, reason) - self.worker_rejection_counts[key] = ( - int(self.worker_rejection_counts.get(key, 0)) + 1 - ) - - def reject_stratum(self, code: int, reason: str, message: str, *, worker: str | None = None) -> None: - self.record_rejection(reason, worker=worker) - raise StratumError(code, message, reason=reason) - - def worker_metric_label(self, worker: str) -> str: - """Metric label for one worker, from a bounded label set. - - The label is the stratum username as authorized (payout address plus - optional worker suffix). Usernames are miner-supplied, so the set of - distinct labels is capped: new workers past the cap aggregate into the - overflow label instead of growing metric cardinality without bound. - """ - self._ensure_worker_metrics_state() - with self.worker_metrics_lock: - return self._worker_metric_label_locked(worker) - - def _worker_metric_label_locked(self, worker: str) -> str: - label = worker or "_unauthenticated" - if len(label) > 128: - label = label[:128] - share_counts = self.worker_share_counts - if label in share_counts: - return label - limit = getattr(self, "worker_metrics_limit", DEFAULT_PRISM_WORKER_METRICS_LIMIT) - if len(share_counts) >= max(0, int(limit)): - label = PRISM_WORKER_METRICS_OVERFLOW_LABEL - share_counts.setdefault(label, {"submitted": 0, "accepted": 0, "grace": 0}) - return label - - def note_worker_submitted_share(self, worker: str) -> None: - self._ensure_worker_metrics_state() - with self.worker_metrics_lock: - label = self._worker_metric_label_locked(worker) - self.worker_share_counts[label]["submitted"] += 1 - - def note_worker_accepted_share(self, worker: str, credit_policy: str | None) -> None: - self._ensure_worker_metrics_state() - with self.worker_metrics_lock: - label = self._worker_metric_label_locked(worker) - counts = self.worker_share_counts[label] - counts["accepted"] += 1 - if credit_policy == PRISM_CREDIT_POLICY_STALE_GRACE: - counts["grace"] += 1 - if credit_policy == PRISM_CREDIT_POLICY_STALE_GRACE: - self._ensure_share_hot_path_state() - with self._share_accounting_lock: - self.grace_credited_share_count = ( - int(getattr(self, "grace_credited_share_count", 0)) + 1 - ) - - @staticmethod - def prometheus_label_value(value: str) -> str: - return ( - value.replace("\\", "\\\\") - .replace("\n", "\\n") - .replace("\r", "\\r") - .replace('"', '\\"') - ) - - def resolve_version_rolling_mask(self, fallback_mask: int) -> direct_stratum.VersionRollingMaskSelection: - try: - template = self.rpc.call("getblocktemplate", [{"rules": qbit_gbt_rules(self.qbit_chain)}]) - if not isinstance(template, dict): - raise RuntimeError("getblocktemplate returned non-object") - except Exception as exc: - return direct_stratum.VersionRollingMaskSelection( - fallback_mask, - "fallback", - f"probe_error:{exc}", - ) - try: - return direct_stratum.select_version_rolling_mask(template, fallback_mask) - except ValueError as exc: - raise SystemExit(str(exc)) from exc - - def parse_share_weights(self) -> dict[str, int]: - raw = os.environ.get("PRISM_STRATUM_SHARE_WEIGHTS_JSON", "") - if not raw: - return {} - try: - parsed = json.loads(raw) - except json.JSONDecodeError as exc: - raise SystemExit(f"PRISM_STRATUM_SHARE_WEIGHTS_JSON is not valid JSON: {exc}") from exc - if not isinstance(parsed, dict): - raise SystemExit("PRISM_STRATUM_SHARE_WEIGHTS_JSON must be an object") - weights: dict[str, int] = {} - for username, weight in parsed.items(): - parsed_weight = int(weight) - if parsed_weight <= 0: - raise SystemExit(f"share weight for {username} must be positive") - weights[str(username)] = parsed_weight - return weights - - def share_weight_for_worker(self, worker: WorkerIdentity) -> int: - return self.share_weights_by_username.get( - worker.username, - self.share_weights_by_username.get(worker.payout_address, self.default_share_weight), - ) - - def make_ledger(self) -> SingleWriterShareLedger | PsqlShareLedger: - psql_command = os.environ.get("PRISM_POSTGRES_PSQL_COMMAND", "") - database_url = os.environ.get("PRISM_DATABASE_URL", "") - if not psql_command and database_url: - psql_command = f"psql {shlex.quote(database_url)}" - if not psql_command: - if not env_bool("PRISM_ALLOW_MEMORY_LEDGER", "0"): - raise SystemExit( - "PRISM_DATABASE_URL or PRISM_POSTGRES_PSQL_COMMAND is required; " - "set PRISM_ALLOW_MEMORY_LEDGER=1 only for local tests" - ) - return SingleWriterShareLedger( - ctv_broadcast_attempt_detail_limit=getattr( - self, - "ctv_broadcast_attempt_detail_limit", - DEFAULT_CTV_BROADCAST_ATTEMPT_DETAIL_LIMIT, - ), - ctv_broadcast_retry_backoff_seconds=getattr( - self, - "ctv_broadcast_retry_backoff_seconds", - DEFAULT_CTV_BROADCAST_RETRY_BACKOFF_SECONDS, - ), - ) - writer_session_token = env_optional("PRISM_LEDGER_WRITER_SESSION_TOKEN") - if writer_session_token is not None and not env_bool("PRISM_ALLOW_FIXED_LEDGER_SESSION_TOKEN", "0"): - raise SystemExit( - "PRISM_LEDGER_WRITER_SESSION_TOKEN requires " - "PRISM_ALLOW_FIXED_LEDGER_SESSION_TOKEN=1 for local tests" - ) - if writer_session_token is None: - writer_session_token = ( - f"{WRITER_LEASE_HEARTBEAT_SESSION_PREFIX}{uuid.uuid4().hex}" - ) - audit_body_dir = getattr(self, "audit_dir", None) - return PsqlShareLedger( - psql_command=psql_command, - database_url=database_url or None, - native_client_mode=env("PRISM_POSTGRES_NATIVE_CLIENT", "auto"), - writer_id=env("PRISM_LEDGER_WRITER_ID", "prism-coordinator"), - writer_epoch=env_int("PRISM_LEDGER_WRITER_EPOCH", 1), - writer_session_token=writer_session_token, - initialize_schema=env("PRISM_POSTGRES_INIT_SCHEMA", "0") in {"1", "true", "yes"}, - lease_ttl_seconds=env_positive_float("PRISM_LEDGER_LEASE_TTL_SECONDS", 60.0), - # The own-write deferral margin must cover the longest RPC the - # lease fence can authorize — submitblock's dedicated deadline - # and the CTV broadcast RPCs riding JsonRpc.call's default — - # plus fixed headroom for fence-to-RPC scheduling and clock - # drift (see LEASE_AUTHORITY_MARGIN_HEADROOM_SECONDS; the RPCs - # themselves cannot outlive their wall-clock deadlines). The - # ledger floors this at half the TTL, so short deadlines keep - # today's behavior and only an operator raising a guarded - # deadline widens the deferral window with it. - lease_authority_margin_seconds=( - LEASE_AUTHORITY_MARGIN_HEADROOM_SECONDS - + max( - float( - getattr( - self, - "block_submit_rpc_timeout_seconds", - DEFAULT_BLOCK_SUBMIT_RPC_TIMEOUT_SECONDS, - ) - ), - DEFAULT_QBIT_RPC_CALL_TIMEOUT_SECONDS, - ) - ), - read_concurrency=env_positive_int("PRISM_POSTGRES_READ_CONCURRENCY", 4), - accepted_stats_cache_seconds=env_nonnegative_float("PRISM_ACCEPTED_STATS_CACHE_SECONDS", 60.0), - reward_window_cache_seconds=env_nonnegative_float("PRISM_PUBLIC_REWARD_WINDOW_CACHE_SECONDS", 30.0), - audit_body_dir=str(audit_body_dir) if audit_body_dir is not None else None, - audit_share_segment_size=getattr(self, "audit_share_segment_size", DEFAULT_AUDIT_SHARE_SEGMENT_SIZE), - ctv_broadcast_attempt_detail_limit=getattr( - self, - "ctv_broadcast_attempt_detail_limit", - DEFAULT_CTV_BROADCAST_ATTEMPT_DETAIL_LIMIT, - ), - ctv_broadcast_retry_backoff_seconds=getattr( - self, - "ctv_broadcast_retry_backoff_seconds", - DEFAULT_CTV_BROADCAST_RETRY_BACKOFF_SECONDS, - ), - ) - - def load_trusted_ledger_writer_public_key(self) -> str | None: - configured = env_optional("PRISM_LEDGER_WRITER_PUBLIC_KEY_HEX") - if configured is not None: - return validate_hex(configured, name="PRISM_LEDGER_WRITER_PUBLIC_KEY_HEX", expected_bytes=32) - if env_bool("PRISM_ALLOW_BUNDLE_EMBEDDED_LEDGER_KEY", "0"): - return None - raise SystemExit( - "PRISM_LEDGER_WRITER_PUBLIC_KEY_HEX is required; " - "set PRISM_ALLOW_BUNDLE_EMBEDDED_LEDGER_KEY=1 only for local tests" - ) - - def prism_payout_policy(self) -> dict[str, object]: - cached = getattr(self, "_prism_payout_policy_cache", None) - if cached is not None: - return cached - - policy = default_prism_payout_policy() - output_policy = env_optional("PRISM_COINBASE_OUTPUT_POLICY") or "canonical" - if output_policy not in VALID_COINBASE_OUTPUT_POLICIES: - raise SystemExit( - "PRISM_COINBASE_OUTPUT_POLICY must be one of: " - + ", ".join(sorted(VALID_COINBASE_OUTPUT_POLICIES)) - ) - fee_bps_raw = env_optional("PRISM_POOL_FEE_BPS") - fee_enabled = env_bool("PRISM_POOL_FEE_ENABLED", "0") - fee_address = env_optional("PRISM_POOL_FEE_ADDRESS") - fee_program_hex = env_optional("PRISM_POOL_FEE_P2MR_PROGRAM_HEX") - fee_recipient_id = env_optional("PRISM_POOL_FEE_RECIPIENT_ID") - fee_order_key = env_optional("PRISM_POOL_FEE_ORDER_KEY") - has_fee_config = any( - value is not None - for value in (fee_bps_raw, fee_address, fee_program_hex, fee_recipient_id, fee_order_key) - ) - if not fee_enabled: - if has_fee_config: - raise SystemExit("set PRISM_POOL_FEE_ENABLED=1 when configuring pool fees") - if output_policy == "pool-fee-first": - raise SystemExit( - "PRISM_COINBASE_OUTPUT_POLICY=pool-fee-first requires PRISM_POOL_FEE_ENABLED=1" - " and a configured pool fee" - ) - self._prism_payout_policy_cache = policy - return policy - if fee_bps_raw is None: - raise SystemExit("PRISM_POOL_FEE_BPS is required when pool fees are enabled") - - try: - fee_bps = int(fee_bps_raw) - except ValueError as exc: - raise SystemExit("PRISM_POOL_FEE_BPS must be an integer") from exc - if fee_bps < 0 or fee_bps > 10_000: - raise SystemExit("PRISM_POOL_FEE_BPS must be between 0 and 10000") - if (fee_address is None) == (fee_program_hex is None): - raise SystemExit("set exactly one of PRISM_POOL_FEE_ADDRESS or PRISM_POOL_FEE_P2MR_PROGRAM_HEX") - - if fee_address is not None: - validation = self.rpc.call("validateaddress", [fee_address]) - if not isinstance(validation, dict) or not validation.get("isvalid"): - raise SystemExit(f"PRISM_POOL_FEE_ADDRESS is not a valid qbit address: {fee_address}") - script = str(validation.get("scriptPubKey") or "") - if not script.startswith("5220") or len(script) != 68: - raise SystemExit("PRISM_POOL_FEE_ADDRESS must resolve to a P2MR script") - fee_policy = { - "fee_bps": fee_bps, - "recipient_id": fee_address, - "order_key": fee_address, - "p2mr_program_hex": script[4:], - } - else: - program_hex = validate_hex( - fee_program_hex or "", - name="PRISM_POOL_FEE_P2MR_PROGRAM_HEX", - expected_bytes=32, - ) - if fee_recipient_id is None: - raise SystemExit("PRISM_POOL_FEE_RECIPIENT_ID is required with PRISM_POOL_FEE_P2MR_PROGRAM_HEX") - fee_policy = { - "fee_bps": fee_bps, - "recipient_id": fee_recipient_id, - "order_key": fee_order_key or fee_recipient_id, - "p2mr_program_hex": program_hex, - } - - policy["pool_fee_policy"] = fee_policy - if output_policy != "canonical": - # Omitted for canonical so canonical payloads and signed audit - # artifacts keep their historical bytes. - policy["coinbase_output_policy"] = output_policy - self._prism_payout_policy_cache = policy - return policy - - def prism_ctv_settlement_config( - self, - *, - block_height: int | None = None, - parent_hash: str | None = None, - ) -> dict[str, object] | None: - if not env_bool("PRISM_CTV_SETTLEMENT_ENABLED", "0"): - return None - direct_floor_sats = env_positive_int_with_legacy( - "PRISM_DIRECT_COINBASE_PAYOUT_FLOOR_BITS", - "PRISM_DIRECT_COINBASE_PAYOUT_FLOOR_SATS", - DEFAULT_DIRECT_COINBASE_PAYOUT_FLOOR_SATS, - ) - reserved_coinbase_outputs = env_int("PRISM_RESERVED_COINBASE_OUTPUTS", 0) - if reserved_coinbase_outputs < 0: - raise SystemExit("PRISM_RESERVED_COINBASE_OUTPUTS must be non-negative") - config: dict[str, object] = { - "direct_floor_sats": direct_floor_sats, - "config": { - "max_coinbase_settlement_outputs": env_positive_int( - "PRISM_MAX_COINBASE_SETTLEMENT_OUTPUTS", - DEFAULT_MAX_COINBASE_SETTLEMENT_OUTPUTS, - ), - "max_direct_coinbase_outputs": env_positive_int( - "PRISM_MAX_DIRECT_COINBASE_OUTPUTS", - DEFAULT_MAX_DIRECT_COINBASE_OUTPUTS, - ), - "max_fanout_recipients_per_transaction": env_positive_int( - "PRISM_MAX_CTV_FANOUT_RECIPIENTS_PER_TRANSACTION", - DEFAULT_MAX_CTV_FANOUT_RECIPIENTS_PER_TRANSACTION, - ), - "reserved_coinbase_outputs": reserved_coinbase_outputs, - }, - } - config["fanout_fee_rate_policy"] = { - "market_fee_rate_sats_per_1000_weight": self.ctv_fanout_market_fee_rate_bits_per_1000_weight( - block_height=block_height, - parent_hash=parent_hash, - ), - "premium_bps": env_positive_int( - "PRISM_CTV_FANOUT_FEE_PREMIUM_BPS", - DEFAULT_CTV_FANOUT_FEE_PREMIUM_BPS, - ), - } - return config - - def ctv_fanout_market_fee_rate_bits_per_1000_weight( - self, - *, - block_height: int | None = None, - parent_hash: str | None = None, - ) -> int: - configured_rate = env_optional_positive_int_with_legacy( - "PRISM_CTV_FANOUT_FEE_MARKET_RATE_BITS_PER_1000_WEIGHT", - "PRISM_CTV_FANOUT_FEE_MARKET_RATE_SATS_PER_1000_WEIGHT", - ) - if configured_rate is not None: - return configured_rate - fee_rate_cache = getattr(self, "_ctv_fanout_market_fee_rate_cache", None) - if fee_rate_cache is None: - fee_rate_cache = {} - self._ctv_fanout_market_fee_rate_cache = fee_rate_cache - cache_key = (block_height, parent_hash) - if cache_key in fee_rate_cache: - return fee_rate_cache[cache_key] - try: - estimate = self.rpc.call( - "estimatesmartfee", - [env_positive_int("PRISM_CTV_FANOUT_FEE_ESTIMATE_TARGET_BLOCKS", 2)], - ) - if not isinstance(estimate, dict): - raise RuntimeError("estimatesmartfee returned non-object") - errors = estimate.get("errors") - if errors: - raise RuntimeError(f"estimatesmartfee returned errors: {errors}") - feerate = Decimal(str(estimate.get("feerate", ""))) - if not feerate.is_finite() or feerate <= 0: - raise RuntimeError(f"estimatesmartfee returned invalid feerate: {estimate.get('feerate')!r}") - rate = int((feerate * Decimal(100_000_000)).to_integral_value(rounding=ROUND_CEILING)) - if rate <= 0: - raise RuntimeError("estimatesmartfee rounded to a non-positive rate") - except Exception as exc: - raise RuntimeError( - "unable to compute PRISM CTV fanout fee rate; set " - "PRISM_CTV_FANOUT_FEE_MARKET_RATE_BITS_PER_1000_WEIGHT or fix " - f"estimatesmartfee: {exc}" - ) from exc - fee_rate_cache[cache_key] = rate - return rate - - def _ensure_job_cache_state(self) -> None: - if not hasattr(self, "_job_cache_lock"): - self._job_cache_lock = threading.Lock() - if not hasattr(self, "_active_job_bundle_builds"): - self._active_job_bundle_builds: dict[ - tuple[object, ...], _JobBundleBuildControl - ] = {} - if not hasattr(self, "_job_build_lock"): - # Compatibility seam for embedders/tests. Expensive construction is - # coordinated by the bounded latest-wins scheduler below, not held - # under this lock. - self._job_build_lock = threading.Lock() - if not hasattr(self, "_job_build_scheduler_lock"): - self._job_build_scheduler_lock = threading.RLock() - if not hasattr(self, "_job_build_priority_preparations"): - self._job_build_priority_preparations: dict[int, float] = {} - if not hasattr(self, "_job_build_priority_preparation_sequence"): - self._job_build_priority_preparation_sequence = 0 - if not hasattr(self, "_job_build_routine_preparations"): - self._job_build_routine_preparations: dict[ - int, - weakref.ReferenceType[_JobBuildCancellation], - ] = {} - if not hasattr(self, "_job_build_routine_preparation_sequence"): - self._job_build_routine_preparation_sequence = 0 - if not hasattr(self, "_job_build_priority_changed"): - self._job_build_priority_changed = threading.Event() - if not hasattr(self, "_job_build_executor"): - self._job_build_executor: ThreadPoolExecutor | None = None - if not hasattr(self, "_job_build_executor_shutdown"): - self._job_build_executor_shutdown = False - if not hasattr(self, "_job_build_active"): - self._job_build_active: _JobBuildFlight | None = None - if not hasattr(self, "_job_build_retiring"): - self._job_build_retiring: _JobBuildFlight | None = None - if not hasattr(self, "_job_build_pending"): - self._job_build_pending: _JobBuildRequest | None = None - if not hasattr(self, "_job_build_issued_at_ms"): - self._job_build_issued_at_ms: OrderedDict[int, int] = OrderedDict() - if not hasattr(self, "job_build_timeout_seconds"): - self.job_build_timeout_seconds = DEFAULT_PRISM_JOB_BUILD_TIMEOUT_SECONDS - if not hasattr(self, "job_build_cancel_grace_seconds"): - self.job_build_cancel_grace_seconds = ( - DEFAULT_PRISM_JOB_BUILD_CANCEL_GRACE_SECONDS - ) - if not hasattr(self, "_template_artifacts"): - self._template_artifacts: CachedTemplateArtifacts | None = None - if not hasattr(self, "_template_artifact_generation"): - self._template_artifact_generation = int( - getattr(self._template_artifacts, "generation", 0) - ) - if not hasattr(self, "_job_bundle_cache"): - self._job_bundle_cache: OrderedDict[ - tuple[object, ...], CachedJobBundle - ] = OrderedDict() - if not hasattr(self, "_payout_state_generation"): - self._payout_state_generation = 0 - if not hasattr(self, "_payout_state_prepare_lock"): - # Ledger mutations and the ledger reads used to build signed jobs - # share this lock. They may be expensive, but they never block - # delivery of an already-published immutable generation. - self._payout_state_prepare_lock = threading.RLock() - if not hasattr(self, "_payout_state_source"): - self._payout_state_source: tuple[int, str | None, str, float] = ( - 0, - None, - "startup", - time.monotonic(), - ) - if not hasattr(self, "_published_payout_state"): - self._published_payout_state = PublishedPayoutState( - generation=self._payout_state_generation, - source_generation=0, - source_tip_hash=None, - published_monotonic=time.monotonic(), - ) - if not hasattr(self, "_payout_ledger_artifact"): - self._payout_ledger_artifact: PayoutLedgerArtifact | None = None - if not hasattr(self, "_payout_ledger_artifact_generation"): - self._payout_ledger_artifact_generation = 0 - if not hasattr(self, "_payout_ledger_append_invalidation_epoch"): - # Guarded by _job_cache_lock. Unlike payout generation, this - # advances only when a newly visible row predates an anchored - # share window and makes pre-append work unsafe to publish. - self._payout_ledger_append_invalidation_epoch = 0 - if not hasattr(self, "_payout_append_landing_fence_lock"): - # Serializes the append-side epoch bump against the landing's - # final epoch-check-and-submitblock boundary. Ordinary share - # commits never touch it: the append side acquires it only for - # rows that predate a live anchor (the rare replay-shaped - # append), and the landing holds it across exactly one RPC, so - # an epoch advance can never slip between the landing's last - # epoch read and the block entering qbitd. Ordered strictly - # outside _job_cache_lock. - self._payout_append_landing_fence_lock = threading.Lock() - if not hasattr(self, "_payout_window_inflight_scan_anchors"): - # Guarded by _job_cache_lock. The anchors of ledger window walks - # whose results are not yet visible anywhere else, published - # before each read begins. A late-visible append can predate such - # an anchor while no completed window or armed artifact exposes - # one, and the walk's database snapshot may already exclude the - # row; the append-side invalidation must still advance the epoch - # so the walk's result cannot arm or serve. A synchronous bundle - # build keeps its entry exposed past its own return -- until the - # seeded artifact's install fence settles -- because nothing else - # exposes that window's anchor in the meantime. - self._payout_window_inflight_scan_anchors: dict[int, int] = {} - self._payout_window_inflight_scan_anchor_token = 0 - if not hasattr(self, "_payout_unfenced_append_inflight_stamps"): - # Guarded by _job_cache_lock. One entry per durable append - # currently committing OUTSIDE the landing fence, holding the - # batch's most predating row stamp (min over rows of - # max(job_issued_at_ms, accepted_at_ms)). The unfenced - # classification is a one-time predicate: an anchor exposed - # after it cannot retroactively fence the in-flight commit, so - # a landing that just exposed its declared anchor drains - # matching entries before its epoch fences arm. The condition - # shares _job_cache_lock and signals every deregistration. - self._payout_unfenced_append_inflight_stamps: dict[int, int] = {} - self._payout_unfenced_append_inflight_token = 0 - self._payout_unfenced_append_drained = threading.Condition( - self._job_cache_lock - ) - if not hasattr(self, "_payout_published_job_window_anchor_ms"): - # Guarded by _job_cache_lock. The highest declared anchor among - # published job windows whose walk exposure retired without a - # seeded artifact to carry it (a build under the reuse - # kill-switch, or one with nothing to seed). Jobs stamped from - # such a bundle keep serving the window until they retire, and - # nothing else exposes its anchor between publication and the - # landing's own exposure -- so the anchor must stay visible to - # the append-side predates() checks: a replay-shaped append - # committing in that gap has to advance the epoch those jobs' - # landing fences compare against. Monotonic and never retired; - # ordinary share commits can never match it because every job - # anchor is clamped below the pending-commit floor. - self._payout_published_job_window_anchor_ms: int | None = None - if not hasattr(self, "_incremental_payout_artifact_window"): - # Guarded by _payout_state_prepare_lock. It is independent of the - # published artifact generation: normal generation bumps retag - # the same immutable share window without another ledger walk. - self._incremental_payout_artifact_window: ( - _IncrementalPayoutArtifactWindow | None - ) = None - if not hasattr( - self, - "_incremental_payout_artifact_window_invalidation_reason", - ): - # Guarded by _payout_state_prepare_lock. Explicit append-side - # invalidations leave a one-shot diagnostic for the oracle build - # that replaces the discarded incremental window. - self._incremental_payout_artifact_window_invalidation_reason: ( - str | None - ) = None - if not hasattr( - self, - "_payout_prior_balances_reuse_invalidated_sha256", - ): - # Guarded by _payout_state_prepare_lock. A periodic balance oracle - # mismatch keeps the stale published digest off the reuse path - # until a later publication installs a different digest. - self._payout_prior_balances_reuse_invalidated_sha256: str | None = ( - None - ) - if not hasattr(self, "_pending_share_commit_lock"): - self._pending_share_commit_lock = threading.Lock() - if not hasattr(self, "_pending_share_commit_floor"): - # Shares whose accepted_at_ms has been assigned but whose ledger - # row has not reached a terminal outcome in this process. Job and - # payout-artifact snapshot anchors clamp below every entry so a - # bundle's eligible-share set stays exactly reproducible from the - # durable ledger after those rows commit. - self._pending_share_commit_floor: dict[int, list[object]] = {} - if not hasattr(self, "_payout_artifact_executor_lock"): - self._payout_artifact_executor_lock = threading.Lock() - if not hasattr(self, "_payout_artifact_executor"): - self._payout_artifact_executor: ThreadPoolExecutor | None = None - if not hasattr(self, "_payout_artifact_future"): - self._payout_artifact_future: Future[None] | None = None - if not hasattr(self, "_payout_artifact_requested"): - self._payout_artifact_requested: tuple[int, int] | None = None - if not hasattr(self, "_payout_artifact_requested_bypass"): - self._payout_artifact_requested_bypass = False - if not hasattr(self, "_payout_artifact_executor_shutdown"): - self._payout_artifact_executor_shutdown = False - if not hasattr(self, "_payout_artifact_last_schedule_monotonic"): - # Guarded by _payout_artifact_executor_lock. Stamped by every - # accepted preparation request so fence-failure re-arms measure - # their debounce from the newest scheduled rebuild, whichever - # path requested it. - self._payout_artifact_last_schedule_monotonic: float | None = None - if not hasattr(self, "_payout_artifact_rearm_backoff"): - # Guarded by _payout_artifact_executor_lock. - self._payout_artifact_rearm_backoff = 1 - if not hasattr(self, "_serve_builder_lock"): - # Serializes daemon ownership per request. Contended builds do - # not queue behind the daemon; they take the one-shot path. - self._serve_builder_lock = threading.Lock() - if not hasattr(self, "_serve_builder"): - self._serve_builder: _ServeBuilderClient | None = None - if not hasattr(self, "_serve_builder_shutdown"): - self._serve_builder_shutdown = False - if not hasattr(self, "_serve_builder_metrics_lock"): - # Counters get their own short-lived lock so a metrics scrape - # never waits behind _serve_builder_lock, which one request can - # hold for a whole daemon round trip. - self._serve_builder_metrics_lock = threading.Lock() - if not hasattr(self, "serve_builder_counts"): - # Guarded by _serve_builder_metrics_lock. - self.serve_builder_counts = { - "requests": 0, - "fallbacks": 0, - "spawns": 0, - "window_uploads": 0, - } - if not hasattr(self, "serve_builder_window_cache_counts"): - # Guarded by _serve_builder_metrics_lock. - self.serve_builder_window_cache_counts = {"hits": 0, "misses": 0} - if not hasattr(self, "_payout_state_delivery_gate"): - # Orders reconciliation mutations against final job-delivery - # admission while preserving parallel sends to different miners. - self._payout_state_delivery_gate = _PayoutStateDeliveryGate() - if not hasattr(self, "_payout_balance_mutation_lock"): - # Keep durable payout-balance transitions serialized even when their - # preparation intentionally runs outside the delivery gate. The - # accepted-block path can hold this lock across expensive writes - # without preventing replacement jobs from reaching miners. - self._payout_balance_mutation_lock = threading.RLock() - if not hasattr(self, "_reconcile_flight_lock"): - self._reconcile_flight_lock = threading.Lock() - if not hasattr(self, "_reconcile_flights"): - # In-flight reconcile passes keyed by tip hash. Unflagged callers - # for a tip already being reconciled await that pass's summary - # instead of queueing a redundant serialized pass of their own. - self._reconcile_flights: dict[str, _ReconcileFlight] = {} - if not hasattr(self, "_reorg_reconcile_trusted_memo"): - # Monotonic completion time of the last trusted reconcile pass, - # per tip hash, guarded by self.lock. Entries are unarmed only by - # an untrusted outcome for their own tip (or a reconcile error, - # which clears the whole map); see _note_reorg_reconcile_outcome. - self._reorg_reconcile_trusted_memo: OrderedDict[str, float] = ( - OrderedDict() - ) - if not hasattr(self, "_reconcile_prefetch_executor_lock"): - self._reconcile_prefetch_executor_lock = threading.Lock() - if not hasattr(self, "_reconcile_prefetch_executor"): - # Single-worker lane that overlaps the tip-refresh reconcile pass - # with the template fetch. Only the refresh singleflight owner - # submits, so at most one prefetched pass is in flight; same-tip - # followers still coalesce through _reconcile_flights. - self._reconcile_prefetch_executor: ThreadPoolExecutor | None = None - if not hasattr(self, "_reconcile_prefetch_executor_shutdown"): - self._reconcile_prefetch_executor_shutdown = False - if not hasattr(self, "_reconcile_prefetch_pending"): - # At most one outstanding prefetch, keyed by tip, guarded by - # _reconcile_prefetch_executor_lock. Failed refresh attempts - # (for example a template-RPC outage) reuse it instead of - # queueing another serialized pass per retry. - self._reconcile_prefetch_pending: ( - tuple[str, Future[bool], bool] | None - ) = None - if not hasattr(self, "reorg_reconcile_lookup_counts"): - # Reconcile demand by (caller path, satisfying source), guarded - # by self.lock. - self.reorg_reconcile_lookup_counts = { - (path, source): 0 - for path in PRISM_REORG_RECONCILE_LOOKUP_PATHS - for source in PRISM_REORG_RECONCILE_LOOKUP_SOURCES - } - if not hasattr(self, "_block_submit_metrics_lock"): - self._block_submit_metrics_lock = threading.Lock() - if not hasattr(self, "block_submit_seconds_histogram"): - # Landed candidate -> submitblock-return interval. Post-submit - # bookkeeping (audit build, persistence, outbox finalize) is - # deliberately excluded; the outbox created_at/completed_at span - # already covers it. - self.block_submit_seconds_histogram = { - "buckets": { - bucket: 0 for bucket in PRISM_JOB_BUILD_SECONDS_BUCKETS - }, - "sum": 0.0, - "count": 0, - } - if not hasattr(self, "_accepted_block_payout_preview_condition"): - self._accepted_block_payout_preview_condition = threading.Condition() - if not hasattr(self, "_accepted_block_payout_previews"): - # Durable replay registers an unlanded transition. Once its block - # is active, reconciliation is barred and child/descendant builders - # wait for or consume its verified prospective balance snapshot. - self._accepted_block_payout_previews: dict[ - str, _AcceptedBlockPayoutTransition - ] = {} - if not hasattr(self, "_invalidated_accepted_block_payout_previews"): - # A landed transition can be withdrawn before durability catches - # up. Keep a height-bearing tombstone so a new exact/descendant - # build cannot fall through to database balances that omit it in - # the gap before durable retry re-registers the candidate. - self._invalidated_accepted_block_payout_previews: dict[ - str, int | None - ] = {} - if not hasattr(self, "_accounted_accepted_block_hashes"): - self._accounted_accepted_block_hashes: set[str] = set() - if not hasattr(self, "_counted_block_candidate_abandonments"): - self._counted_block_candidate_abandonments: set[str] = set() - if not hasattr(self, "_outstanding_block_candidate_hashes"): - self._outstanding_block_candidate_hashes: set[str] = set() - if not hasattr(self, "_tip_observed_accepted_block_hashes"): - self._tip_observed_accepted_block_hashes: dict[str, float] = {} - if not hasattr(self, "_payout_state_metrics_lock"): - self._payout_state_metrics_lock = threading.Lock() - if not hasattr(self, "payout_state_histograms"): - self.payout_state_histograms = { - name: { - "buckets": { - bucket: 0 for bucket in PRISM_TIP_REFRESH_SECONDS_BUCKETS - }, - "sum": 0.0, - "count": 0, - } - for name in ("preparation", "publish", "first_delivery") - } - if not hasattr(self, "payout_gate_wait_histograms"): - self.payout_gate_wait_histograms = { - relation: { - "buckets": { - bucket: 0 for bucket in PRISM_TIP_REFRESH_SECONDS_BUCKETS - }, - "sum": 0.0, - "count": 0, - } - for relation in PRISM_PAYOUT_DELIVERY_GENERATIONS - } - if not hasattr(self, "payout_state_candidates_discarded"): - self.payout_state_candidates_discarded = 0 - if not hasattr(self, "_payout_first_delivery_pending"): - self._payout_first_delivery_pending: tuple[int, float] | None = None - if not hasattr(self, "_payout_state_publication_blocked"): - self._payout_state_publication_blocked = False - if not hasattr(self, "_job_build_phase_local"): - self._job_build_phase_local = threading.local() - if not hasattr(self, "job_cache_hit_counts"): - self.job_cache_hit_counts = {kind: 0 for kind in PRISM_JOB_CACHE_KINDS} - if not hasattr(self, "job_cache_miss_counts"): - self.job_cache_miss_counts = {kind: 0 for kind in PRISM_JOB_CACHE_KINDS} - if not hasattr(self, "job_build_seconds_bucket_counts"): - self.job_build_seconds_bucket_counts = { - bucket: 0 for bucket in PRISM_JOB_BUILD_SECONDS_BUCKETS - } - if not hasattr(self, "job_build_seconds_sum"): - self.job_build_seconds_sum = 0.0 - if not hasattr(self, "job_build_count"): - self.job_build_count = 0 - if not hasattr(self, "job_build_phase_seconds"): - self.job_build_phase_seconds = {phase: 0.0 for phase in PRISM_JOB_BUILD_PHASES} - if not hasattr(self, "job_build_scheduler_counts"): - self.job_build_scheduler_counts = { - "requests": 0, - "starts": 0, - "completions": 0, - "supersessions": 0, - "obsolete_results": 0, - "orphan_evicted": 0, - } - if not hasattr(self, "job_build_priority_counts"): - self.job_build_priority_counts = { - result: 0 - for result in ( - "started", - "coalesced", - "queued", - "routine_deferred", - "routine_preempted", - ) - } - if not hasattr(self, "job_build_priority_admission_seconds"): - self.job_build_priority_admission_seconds = { - "sum": 0.0, - "count": 0, - } - if not hasattr(self, "initial_job_prepared_work_counts"): - self.initial_job_prepared_work_counts = { - result: 0 - for result in ( - "cache_hit", - "singleflight", - "deferred", - "subscribed", - "admission_deadline", - ) - } - if not hasattr(self, "payout_artifact_event_counts"): - # Artifact lifecycle observability (the 2026-07-29 incident was - # invisible precisely here). Guarded by - # _payout_artifact_executor_lock. - self.payout_artifact_event_counts = { - event: 0 - for event in ( - "built", - "build_aborted", - "debounced", - "incremental", - "full_rescan", - "self_check_match", - "self_check_mismatch", - "self_check_failed", - "balance_check_mismatch", - "found_block_cached", - "installed", - "refreshed", - "already_current", - "discarded", - "born_expired", - "rearm_scheduled", - ) - } - if not hasattr(self, "_admission_deadline_last_log_monotonic"): - self._admission_deadline_last_log_monotonic: float | None = None - if not hasattr(self, "job_build_cancellation_seconds"): - self.job_build_cancellation_seconds = { - "sum": 0.0, - "count": 0, - } - if not hasattr(self, "job_build_replacement_start_seconds"): - self.job_build_replacement_start_seconds = { - "sum": 0.0, - "count": 0, - } - if not hasattr(self, "job_build_worker_counts"): - self.job_build_worker_counts = { - "starts": 0, - "terminations": 0, - "crashes": 0, - "restarts": 0, - } - if not hasattr(self, "_job_build_worker_restart_pending"): - self._job_build_worker_restart_pending = False - if not hasattr(self, "_health_snapshot"): - self._health_snapshot: dict[str, object] | None = None - if not hasattr(self, "_health_snapshot_monotonic"): - self._health_snapshot_monotonic: float | None = None - if not hasattr(self, "_health_refresh_loop_running"): - self._health_refresh_loop_running = False - if not hasattr(self, "health_snapshot_refresh_failure_count"): - self.health_snapshot_refresh_failure_count = 0 - if not hasattr(self, "_share_window_serialization_lock"): - self._share_window_serialization_lock = threading.Lock() - if not hasattr(self, "_share_window_serialization"): - self._share_window_serialization: ( - _ShareWindowSerialization | None - ) = None - if not hasattr(self, "_bundle_preparation_lock"): - self._bundle_preparation_lock = threading.Lock() - if not hasattr(self, "_bundle_preparation_flights"): - self._bundle_preparation_flights: dict[ - tuple[object, ...], _SharedBundlePreparationFlight - ] = {} - if not hasattr(self, "shared_bundle_build_counts"): - self.shared_bundle_build_counts = { - outcome: 0 - for outcome in ("started", "completed", "superseded", "failed") - } - if not hasattr(self, "shared_bundle_preparation_seconds_sum"): - self.shared_bundle_preparation_seconds_sum = 0.0 - if not hasattr(self, "shared_bundle_preparation_count"): - self.shared_bundle_preparation_count = 0 - if not hasattr(self, "shared_bundle_preparation_waiters"): - self.shared_bundle_preparation_waiters = 0 - if not hasattr(self, "_prepared_ready_bundle"): - self._prepared_ready_bundle: CachedJobBundle | None = None - if not hasattr(self, "_prepared_ready_snapshot"): - self._prepared_ready_snapshot: QbitTipTemplateSnapshot | None = None - if not hasattr(self, "job_preparation_pending"): - self.job_preparation_pending = False - if not hasattr(self, "_progress_health_lock"): - self._progress_health_lock = threading.Lock() - if not hasattr(self, "_progress_current_template_generation"): - self._progress_current_template_generation = 0 - if not hasattr(self, "_progress_current_template_fingerprint"): - self._progress_current_template_fingerprint: str | None = None - if not hasattr(self, "_progress_current_payout_generation"): - self._progress_current_payout_generation = self._payout_state_generation - if not hasattr(self, "_progress_published_template_generation"): - self._progress_published_template_generation = 0 - if not hasattr(self, "_progress_published_template_fingerprint"): - self._progress_published_template_fingerprint: str | None = None - if not hasattr(self, "_progress_published_payout_generation"): - self._progress_published_payout_generation = 0 - if not hasattr(self, "_progress_has_published_work"): - self._progress_has_published_work = False - if not hasattr(self, "_progress_last_tip_poll_monotonic"): - self._progress_last_tip_poll_monotonic: float | None = None - if not hasattr(self, "_progress_last_delivery_template_generation"): - self._progress_last_delivery_template_generation = 0 - if not hasattr(self, "_progress_last_delivery_template_fingerprint"): - self._progress_last_delivery_template_fingerprint: str | None = None - if not hasattr(self, "_progress_last_delivery_payout_generation"): - self._progress_last_delivery_payout_generation = 0 - if not hasattr(self, "_progress_last_delivery_monotonic"): - self._progress_last_delivery_monotonic: float | None = None - if not hasattr(self, "_progress_pending_since_monotonic"): - self._progress_pending_since_monotonic: float | None = float( - getattr(self, "started_monotonic", time.monotonic()) - ) - if not hasattr(self, "_progress_publication_divergence_since_monotonic"): - self._progress_publication_divergence_since_monotonic: float | None = ( - None - if self._progress_has_published_work - else float(getattr(self, "started_monotonic", time.monotonic())) - ) - if not hasattr(self, "_progress_refresh_signal_pending"): - self._progress_refresh_signal_pending = False - if not hasattr(self, "_progress_active_refresh_count"): - self._progress_active_refresh_count = 0 - if not hasattr(self, "_progress_last_refresh_activity_monotonic"): - self._progress_last_refresh_activity_monotonic: float | None = None - if not hasattr(self, "_progress_bundle_build_counter"): - self._progress_bundle_build_counter = 0 - if not hasattr(self, "_progress_bundle_builds"): - self._progress_bundle_builds: dict[int, float] = {} - - def _job_build_phases(self) -> dict[str, float]: - """Per-thread scratch dict of phase timings for the current build.""" - self._ensure_job_cache_state() - phases = getattr(self._job_build_phase_local, "phases", None) - if phases is None: - phases = {} - self._job_build_phase_local.phases = phases - return phases - - def _cancel_obsolete_job_bundle_builds( - self, - *, - current_tip: str | None = None, - payout_state_generation: int | None = None, - ) -> None: - """Cancel only builds proven obsolete by a newer exact generation.""" - self._ensure_job_cache_state() - processes: list[subprocess.Popen[str]] = [] - with self._job_cache_lock: - for control in self._active_job_bundle_builds.values(): - obsolete = ( - current_tip is not None - and control.previousblockhash != current_tip - ) or ( - payout_state_generation is not None - and control.payout_state_generation - != int(payout_state_generation) - ) - if not obsolete or control.cancel_event.is_set(): - continue - control.cancel_event.set() - if control.process is not None: - processes.append(control.process) - for process in processes: - if process.poll() is not None: - continue - try: - process.terminate() - except ProcessLookupError: - pass - - def _register_job_bundle_process( - self, - control: _JobBundleBuildControl, - process: subprocess.Popen[str], - ) -> None: - terminate = False - with self._job_cache_lock: - if ( - self._active_job_bundle_builds.get(control.key) is not control - or control.cancel_event.is_set() - ): - terminate = True - else: - control.process = process - if terminate and process.poll() is None: - try: - process.terminate() - except ProcessLookupError: - pass - - def _unregister_job_bundle_process( - self, - control: _JobBundleBuildControl, - process: Any, - ) -> None: - """Detach a shared daemon from a build control after its request. - - Registration makes supersession terminate the in-flight process. - Once this build's daemon request has completed, a later supersession - of the build must not kill the shared daemon out from under - whichever request owns it next. - """ - with self._job_cache_lock: - if control.process is process: - control.process = None - - @staticmethod - def _incremental_window_records_supported(records: Sequence[object]) -> bool: - """Whether a ledger snapshot exposes the append-only window contract.""" - - required = ( - "share_seq", - "share_id", - "share_difficulty", - "job_issued_at_ms", - "accepted_at_ms", - "to_prism_json", - ) - return bool(records) and all( - all(hasattr(record, attribute) for attribute in required) - for record in records - ) - - def _full_payout_window_materialization( - self, - *, - snapshot_anchor_ms: int, - snapshot_window_weight: int, - reason: str, - observed_monotonic: float, - append_invalidation_epoch: int, - ) -> _PayoutWindowMaterialization: - """Run the exact ledger oracle and atomically replace cached pages.""" - - records = list( - self.ledger.snapshot_at_job_issue( - snapshot_anchor_ms, - window_weight=snapshot_window_weight, - ) - ) - zero_stats = IncrementalWindowAdvanceStats( - added_rows=0, - expired_rows=0, - touched_pages=0, - ) - if not self._incremental_window_records_supported(records): - # Small test/custom ledgers predating the delta contract retain the - # legacy full-read behavior. Production ledger records always - # expose the complete AcceptedShareRecord shape. - self._incremental_payout_artifact_window = None - shares_json = tuple(record.to_prism_json() for record in records) - return _PayoutWindowMaterialization( - shares_json=shares_json, - share_snapshot_sha256=canonical_json_sha256(shares_json), - snapshot_anchor_ms=int(snapshot_anchor_ms), - mode="full_rescan", - record_count=len(records), - stats=zero_stats, - full_rescan_reason=reason, - ) - - window = IncrementalShareWindow.from_full_snapshot( - records, - anchor_job_issued_at_ms=snapshot_anchor_ms, - window_weight=snapshot_window_weight, - ) - shares_json = window.json_records() - digest = canonical_json_sha256(shares_json) - self._incremental_payout_artifact_window = ( - _IncrementalPayoutArtifactWindow( - window=window, - shares_json=shares_json, - share_snapshot_sha256=digest, - refreshed_monotonic=observed_monotonic, - full_rescan_monotonic=observed_monotonic, - full_rescan_attempt_monotonic=observed_monotonic, - append_invalidation_epoch=append_invalidation_epoch, - ) - ) - return _PayoutWindowMaterialization( - shares_json=shares_json, - share_snapshot_sha256=digest, - snapshot_anchor_ms=int(snapshot_anchor_ms), - mode="full_rescan", - record_count=len(shares_json), - stats=zero_stats, - full_rescan_reason=reason, - ) - - def _incremental_payout_window_materialization( - self, - *, - snapshot_anchor_ms: int, - snapshot_window_weight: int, - force_full_rescan: bool, - bypass_build_interval: bool, - append_invalidation_epoch: int, - reused_prior_balances_sha256: str | None = None, - ) -> _PayoutWindowMaterialization: - """Return an exact window using debounce, delta folding, or the oracle. - - Must run under ``_payout_state_prepare_lock``. The cache replacement - happens only after a complete materialization, except an explicit - forced invalidation clears it before the oracle so a failed reorg - check cannot later resume from unverified append-only state. - """ - - observed = time.monotonic() - cached = self._incremental_payout_artifact_window - full_reason: str | None = None - if force_full_rescan: - self._incremental_payout_artifact_window = None - self._incremental_payout_artifact_window_invalidation_reason = None - cached = None - full_reason = "reconcile_invalidation" - elif cached is None: - full_reason = ( - self._incremental_payout_artifact_window_invalidation_reason - or "cold_start" - ) - elif cached.append_invalidation_epoch != append_invalidation_epoch: - # Append invalidation is published without waiting for this - # preparation lock. A build that acquires the lock first must not - # advance the pre-append cache under the new epoch. - self._incremental_payout_artifact_window = None - cached = None - full_reason = "late_visible_append" - elif cached.window.window_weight != int(snapshot_window_weight): - self._incremental_payout_artifact_window = None - self._incremental_payout_artifact_window_invalidation_reason = None - cached = None - full_reason = "network_difficulty_changed" - elif snapshot_anchor_ms < cached.window.anchor_job_issued_at_ms: - self._incremental_payout_artifact_window = None - self._incremental_payout_artifact_window_invalidation_reason = None - cached = None - full_reason = "anchor_regression" - - if cached is None: - materialized = self._full_payout_window_materialization( - snapshot_anchor_ms=snapshot_anchor_ms, - snapshot_window_weight=snapshot_window_weight, - reason=full_reason or "cache_invalidated", - observed_monotonic=observed, - append_invalidation_epoch=append_invalidation_epoch, - ) - self._incremental_payout_artifact_window_invalidation_reason = None - return materialized - - min_interval = self._payout_artifact_min_build_interval_seconds() - full_rescan_seconds = float( - getattr( - self, - "payout_artifact_full_rescan_seconds", - DEFAULT_PRISM_PAYOUT_ARTIFACT_FULL_RESCAN_SECONDS, - ) - ) - # Bypass builds (accepted-block previews) refresh - # refreshed_monotonic on every block yet never run the periodic - # oracle themselves, so a sustained sub-interval block cadence - # would otherwise send every routine build back from this - # debounce and postpone the configured self-check indefinitely. - # An overdue self-check disarms the debounce for routine builds; - # the urgent preview path stays debounce-free and oracle-free - # either way. - self_check_overdue = ( - observed - cached.full_rescan_attempt_monotonic - >= full_rescan_seconds - ) - if ( - not bypass_build_interval - and not self_check_overdue - and observed - cached.refreshed_monotonic < min_interval - ): - return _PayoutWindowMaterialization( - shares_json=cached.shares_json, - share_snapshot_sha256=cached.share_snapshot_sha256, - snapshot_anchor_ms=cached.window.anchor_job_issued_at_ms, - mode="debounced", - record_count=sum( - len(page.records) for page in cached.window.pages - ), - stats=IncrementalWindowAdvanceStats(0, 0, 0), - ) - - delta_reader = getattr(self.ledger, "snapshot_between_job_issues", None) - if not callable(delta_reader): - return self._full_payout_window_materialization( - snapshot_anchor_ms=snapshot_anchor_ms, - snapshot_window_weight=snapshot_window_weight, - reason="delta_api_unavailable", - observed_monotonic=observed, - append_invalidation_epoch=append_invalidation_epoch, - ) - - try: - delta_records = list( - delta_reader( - cached.window.anchor_job_issued_at_ms, - snapshot_anchor_ms, - ) - ) - advanced_window, stats = cached.window.advance( - delta_records, - anchor_job_issued_at_ms=snapshot_anchor_ms, - ) - except (IncrementalWindowFallback, ValueError): - self._incremental_payout_artifact_window = None - return self._full_payout_window_materialization( - snapshot_anchor_ms=snapshot_anchor_ms, - snapshot_window_weight=snapshot_window_weight, - reason="incremental_invariant_failed", - observed_monotonic=observed, - append_invalidation_epoch=append_invalidation_epoch, - ) - - if delta_records or stats.expired_rows: - # Each window page owns its immutable JSON representation. Page - # identities survive normal advancement, so this view and its - # digest convert only append/head boundary pages; retained share - # records are never flattened or re-encoded here. - shares_json = advanced_window.json_records() - digest = canonical_json_sha256(shares_json) - else: - shares_json = cached.shares_json - digest = cached.share_snapshot_sha256 - - advanced = _IncrementalPayoutArtifactWindow( - window=advanced_window, - shares_json=shares_json, - share_snapshot_sha256=digest, - refreshed_monotonic=observed, - full_rescan_monotonic=cached.full_rescan_monotonic, - full_rescan_attempt_monotonic=( - cached.full_rescan_attempt_monotonic - ), - append_invalidation_epoch=append_invalidation_epoch, - ) - mode = "incremental" - check_reason: str | None = None - balance_check_prior_balances: tuple[dict[str, object], ...] | None = None - balance_check_mismatch = False - if not bypass_build_interval and self_check_overdue: - try: - full_records = list( - self.ledger.snapshot_at_job_issue( - snapshot_anchor_ms, - window_weight=snapshot_window_weight, - ) - ) - full_window = IncrementalShareWindow.from_full_snapshot( - full_records, - anchor_job_issued_at_ms=snapshot_anchor_ms, - window_weight=snapshot_window_weight, - ) - full_retained = full_window.records() - matched = full_retained == advanced_window.records() - shares_json = full_window.json_records() - digest = canonical_json_sha256(shares_json) - except Exception: - # The already-validated delta remains usable. Space failed - # oracle attempts by the configured self-check interval so a - # saturated/unavailable database cannot turn a periodic - # safety check into one full CTE retry every minute. - advanced = dataclass_replace( - advanced, - full_rescan_attempt_monotonic=observed, - ) - mode = "incremental_self_check_failed" - check_reason = "periodic_self_check_failed" - else: - advanced = _IncrementalPayoutArtifactWindow( - window=full_window, - shares_json=shares_json, - share_snapshot_sha256=digest, - refreshed_monotonic=observed, - full_rescan_monotonic=observed, - full_rescan_attempt_monotonic=observed, - append_invalidation_epoch=append_invalidation_epoch, - ) - mode = "self_check_match" if matched else "self_check_mismatch" - check_reason = "periodic_self_check" - if reused_prior_balances_sha256 is not None: - try: - checked_balances = tuple( - self.ledger.current_prior_balances() - ) - checked_balances_sha256 = canonical_json_sha256( - checked_balances - ) - if ( - checked_balances_sha256 - != reused_prior_balances_sha256 - ): - balance_check_prior_balances = checked_balances - balance_check_mismatch = True - self._payout_prior_balances_reuse_invalidated_sha256 = ( - reused_prior_balances_sha256 - ) - # Equal-window refreshes intentionally retain - # their armed balances. Drop the stale artifact so - # the fresh balance read cannot be discarded as a - # window-only no-op when this build is installed. - with self._job_cache_lock: - armed = self._payout_ledger_artifact - if armed is not None: - armed_balances_sha256 = ( - armed.prior_balances_sha256 - or canonical_json_sha256( - armed.prior_balances - ) - ) - if ( - armed_balances_sha256 - == reused_prior_balances_sha256 - ): - self._payout_ledger_artifact = None - except Exception: - # The carry backstop is independent of the window - # oracle that just succeeded. Reverting here would - # serve a window the oracle may have refuted for a - # whole rescan interval, so keep the full window and - # leave the reused balances unverified until the next - # self-check. - check_reason = ( - "periodic_self_check_balance_check_failed" - ) - - self._incremental_payout_artifact_window = advanced - return _PayoutWindowMaterialization( - shares_json=advanced.shares_json, - share_snapshot_sha256=advanced.share_snapshot_sha256, - snapshot_anchor_ms=advanced.window.anchor_job_issued_at_ms, - mode=mode, - record_count=sum(len(page.records) for page in advanced.window.pages), - stats=stats, - full_rescan_reason=check_reason, - balance_check_prior_balances=balance_check_prior_balances, - balance_check_mismatch=balance_check_mismatch, - ) - - def _build_payout_ledger_artifact( - self, - expected_payout_state_generation: int, - artifact_payout_state_generation: int, - network_difficulty: int, - force_full_rescan: bool = False, - bypass_build_interval: bool = False, - during_publication: bool = False, - ) -> PayoutLedgerArtifact | None: - """Build a stable ledger snapshot without publishing it. - - Validity is anchor-scoped, not count-scoped. The pending-commit clamp - selects the highest clean anchor: every share stamped at or below it - is already durable, so the window read at that anchor is exact and - reproducible no matter how many shares commit while the walk runs. - Shares stamped above the anchor deterministically belong to the next - window; concurrent writers therefore never invalidate this attempt. - """ - self._ensure_job_cache_state() - build_started = time.monotonic() - ledger_started = time.monotonic() - materialized: _PayoutWindowMaterialization - try: - with self._payout_state_prepare_lock: - with self._job_cache_lock: - if ( - expected_payout_state_generation - != self._payout_state_generation - ): - return None - append_invalidation_epoch = int( - self._payout_ledger_append_invalidation_epoch - ) - published_payout_artifact = ( - self._published_payout_state.artifact - ) - invalidated_balances_sha256 = ( - self._payout_prior_balances_reuse_invalidated_sha256 - ) - if ( - invalidated_balances_sha256 is not None - and published_payout_artifact is not None - and published_payout_artifact.prior_balances_sha256 - != invalidated_balances_sha256 - ): - # A later publication installed a different balance - # snapshot, so the digest that failed its oracle check can - # no longer enter this generation's reuse path. - self._payout_prior_balances_reuse_invalidated_sha256 = None - invalidated_balances_sha256 = None - reuse_published_balances = bool( - not force_full_rescan - and published_payout_artifact is not None - and published_payout_artifact.generation - == expected_payout_state_generation - and published_payout_artifact.prior_balances_sha256 - != invalidated_balances_sha256 - ) - snapshot_window_weight = ( - PRISM_REWARD_WINDOW_MULTIPLIER - * PRISM_SNAPSHOT_WINDOW_MARGIN - * int(network_difficulty) - ) - clamp_now_ms = now_ms() - snapshot_anchor_ms = self._job_snapshot_anchor_ms(clamp_now_ms) - if ( - clamp_now_ms - snapshot_anchor_ms - > self._payout_artifact_max_anchor_age_ms() - ): - # The floor is held this far below now only by a wedged - # writer or a leaked release; an artifact anchored there - # would already be past the audit ceiling -- born - # expired -- on arrival. Abort before paying the window - # walk: the re-arm backoff paces retries and the - # share-commit liveness watchdog owns recovery. - return None - window_started = time.monotonic() - # Publish the walk's anchor before the ledger read: a - # late-visible append committing mid-walk can predate it - # while a cold or forcibly cleared cache exposes no other - # anchor, and the walk's database snapshot may already - # exclude the row. Retiring in the finally is safe here - # because the materialization installs its incremental - # pointer -- the walk's durable visibility -- before this - # exposure is withdrawn. - inflight_anchor_token = self._expose_inflight_scan_anchor( - snapshot_anchor_ms - ) - try: - materialized = self._incremental_payout_window_materialization( - snapshot_anchor_ms=snapshot_anchor_ms, - snapshot_window_weight=snapshot_window_weight, - force_full_rescan=force_full_rescan, - bypass_build_interval=bypass_build_interval, - append_invalidation_epoch=append_invalidation_epoch, - reused_prior_balances_sha256=( - published_payout_artifact.prior_balances_sha256 - if reuse_published_balances - and published_payout_artifact is not None - else None - ), - ) - finally: - self._retire_inflight_scan_anchor(inflight_anchor_token) - if materialized.balance_check_mismatch: - # Emit at detection time: a later, unrelated artifact - # serialization or stats failure must not hide the oracle - # alarm that already invalidated reuse. - self._record_payout_artifact_event( - "balance_check_mismatch" - ) - self._payout_artifact_log( - "payout_artifact_balance_check_mismatch", - payout_state_generation=int( - artifact_payout_state_generation - ), - balance_check_mismatch=True, - ) - if ( - materialized.balance_check_mismatch - and materialized.balance_check_prior_balances is not None - and published_payout_artifact is not None - ): - # The oracle refuted the published balance snapshot, so - # invalidating reuse alone is not enough: consumers keyed - # to the published digest (the armed-artifact fence, new - # build keys, and the synchronous fallback) would keep - # serving the refuted bytes until an unrelated generation - # change. Publish the repaired balances for the same - # generation so this build's artifact can arm and serve. - self._publish_self_check_repaired_balances( - expected_payout_state_generation, - stale_prior_balances_sha256=( - published_payout_artifact.prior_balances_sha256 - ), - balances=materialized.balance_check_prior_balances, - ) - window_build_seconds = time.monotonic() - window_started - if materialized.balance_check_prior_balances is not None: - # The periodic oracle already paid for the live aggregate. - # Use that exact result rather than issuing a duplicate - # carry query in the mismatch build. - prior_balances = ( - materialized.balance_check_prior_balances - ) - prior_balances_source = "ledger" - elif reuse_published_balances: - assert published_payout_artifact is not None - # Prior balances are generation-owned immutable state. - # Normal share-window refreshes cannot change them, so - # reuse the exact published bytes instead of aggregating - # the 1.3M-row carry tables on every debounced generation. - # Reconcile/withdrawal mutations force a ledger read. - prior_balances = ( - published_payout_artifact.prior_balances() - ) - prior_balances_source = "published" - else: - prior_balances = self.ledger.current_prior_balances() - prior_balances_source = "ledger" - accepted_share_count, _ = self.accepted_share_stats() - except Exception as exc: - # Artifact preparation is speculative. The synchronous bundle path - # still owns errors when current work actually requires a snapshot. - if during_publication: - self._record_payout_artifact_event("build_aborted") - self._payout_artifact_log( - "payout_artifact_build_aborted", - payout_state_generation=int( - artifact_payout_state_generation - ), - duration_seconds=round( - time.monotonic() - build_started, - 3, - ), - during_publication=True, - build_interval_bypassed=bool(bypass_build_interval), - abort_reason=type(exc).__name__, - ) - return None - finally: - self._observe_tip_refresh_build_phase( - "ledger_snapshot", - time.monotonic() - ledger_started, - ) - if not materialized.shares_json: - return None - copy_started = time.monotonic() - shares_json = materialized.shares_json - frozen_balances = tuple(prior_balances) - self._observe_tip_refresh_build_phase( - "serialization_copy", - time.monotonic() - copy_started, - ) - artifact = PayoutLedgerArtifact( - generation=0, - payout_state_generation=artifact_payout_state_generation, - network_difficulty=int(network_difficulty), - accepted_share_count=accepted_share_count, - shares_json=shares_json, - prior_balances=frozen_balances, - prepared_monotonic=time.monotonic(), - snapshot_anchor_ms=materialized.snapshot_anchor_ms, - share_snapshot_sha256=materialized.share_snapshot_sha256, - prior_balances_sha256=canonical_json_sha256(frozen_balances), - append_invalidation_epoch=append_invalidation_epoch, - window_build_mode=materialized.mode, - window_delta_rows=materialized.stats.added_rows, - window_expired_rows=materialized.stats.expired_rows, - window_touched_pages=materialized.stats.touched_pages, - window_build_seconds=window_build_seconds, - window_full_rescan_reason=materialized.full_rescan_reason, - ) - build_seconds = time.monotonic() - build_started - log_fields: dict[str, object] = { - "payout_state_generation": int(artifact_payout_state_generation), - "duration_seconds": round(build_seconds, 3), - "window_build_seconds": round(window_build_seconds, 3), - "window_build_mode": materialized.mode, - "window_shares": materialized.record_count, - "delta_rows": materialized.stats.added_rows, - "expired_rows": materialized.stats.expired_rows, - "touched_pages": materialized.stats.touched_pages, - "anchor_age_ms": now_ms() - materialized.snapshot_anchor_ms, - "during_publication": bool(during_publication), - "build_interval_bypassed": bool(bypass_build_interval), - "prior_balances_source": prior_balances_source, - "balance_check_mismatch": materialized.balance_check_mismatch, - } - if materialized.full_rescan_reason is not None: - log_fields["full_rescan_reason"] = materialized.full_rescan_reason - if materialized.mode == "debounced": - self._record_payout_artifact_event("debounced") - self._payout_artifact_log( - "payout_artifact_build_debounced", - **log_fields, - ) - else: - self._record_payout_artifact_event("built") - if materialized.mode == "full_rescan": - self._record_payout_artifact_event("full_rescan") - elif materialized.mode == "self_check_match": - self._record_payout_artifact_event("self_check_match") - elif materialized.mode == "self_check_mismatch": - self._record_payout_artifact_event("self_check_mismatch") - elif materialized.mode == "incremental_self_check_failed": - self._record_payout_artifact_event("incremental") - self._record_payout_artifact_event("self_check_failed") - else: - self._record_payout_artifact_event("incremental") - self._payout_artifact_log("payout_artifact_built", **log_fields) - return artifact - - def _prepare_payout_ledger_artifact( - self, - payout_state_generation: int, - network_difficulty: int, - *, - bypass_build_interval: bool = False, - ) -> None: - """Prepare and atomically publish an artifact for a current generation.""" - build_started = time.monotonic() - artifact = self._build_payout_ledger_artifact( - payout_state_generation, - payout_state_generation, - network_difficulty, - False, - bypass_build_interval, - ) - build_seconds = time.monotonic() - build_started - if artifact is None: - # A superseding generation, a snapshot error, an empty window, or - # a pathologically old pending-commit floor discarded this - # attempt; that condition can persist, and each attempt performs - # ledger preparation under the shared lock. Back the - # fence-failure re-arm off until something arms. - self._record_payout_artifact_event("build_aborted") - self._payout_artifact_log( - "payout_artifact_build_aborted", - payout_state_generation=int(payout_state_generation), - duration_seconds=round(build_seconds, 3), - ) - with self._payout_artifact_executor_lock: - self._payout_artifact_rearm_backoff = min( - self._payout_artifact_rearm_backoff * 2, - PRISM_PAYOUT_ARTIFACT_REARM_BACKOFF_CAP, - ) - return - if not self._install_payout_ledger_artifact(artifact): - # A discarded install -- born expired after a slow walk, an - # ordering loss to a fresher window, or a generation change -- - # armed nothing. It must not collapse the re-arm interval: an - # unconditional reset here is exactly the reset-then-rewalk - # livelock that rolled back the first anchor-scoped deploy. - with self._payout_artifact_executor_lock: - self._payout_artifact_rearm_backoff = min( - self._payout_artifact_rearm_backoff * 2, - PRISM_PAYOUT_ARTIFACT_REARM_BACKOFF_CAP, - ) - - def _install_payout_ledger_artifact( - self, - artifact: PayoutLedgerArtifact, - ) -> bool: - """Atomically publish a prepared artifact for its own generation. - - Snapshot-freshness-ordered and idempotent: every snapshot is taken at - a clean anchor (strictly below all pending commits), and the durable - window at or below an anchor is immutable, so the anchor orders - snapshots even when a build delayed in window conversion finishes - after a later snapshot installed (completion time cannot order - snapshots). Equal-anchor installs with an identical window -- every - flight waiter re-runs cache publication with the same prepared - artifact, and a same-window speculative rebuild re-reads unchanged - state under an equal anchor -- keep the installed generation instead - of re-keying bundle lookups for nothing. - - Returns True when the window is armed (installed fresh, refreshed in - place, or confirmed already current); only that outcome resets the - re-arm backoff. A discarded install returns False and leaves the - backoff alone -- in particular a BORN-EXPIRED artifact, whose anchor - aged past the audit ceiling during its own window walk. Arming (or - crediting) such an install would collapse the re-arm interval and - rewalk the reward window continuously: the livelock behind the - 2026-07-29 rollback. - """ - try: - return self._install_payout_ledger_artifact_outcome(artifact) - finally: - # A seeded artifact carries the exposed scan anchor of the - # synchronous read that produced it. The install decision above - # is atomic under _job_cache_lock: either the window armed (its - # own anchor is now visible to append invalidation) or it was - # discarded and can never serve, so the exposure has done its - # job either way. - self._retire_inflight_scan_anchor( - artifact.inflight_scan_anchor_token - ) - - def _install_payout_ledger_artifact_outcome( - self, - artifact: PayoutLedgerArtifact, - ) -> bool: - if not self._payout_artifact_reuse_active(): - # Kill-switch: never arm. An armed artifact would churn install - # events while disabled and re-key the idle-bundle fast path to - # a generation the reuse probe refuses anyway. - return False - anchor_age_ms = ( - None - if artifact.snapshot_anchor_ms is None - else now_ms() - int(artifact.snapshot_anchor_ms) - ) - if ( - anchor_age_ms is None - or anchor_age_ms > self._payout_artifact_max_anchor_age_ms() - ): - self._record_payout_artifact_event("born_expired") - self._payout_artifact_log( - "payout_artifact_born_expired", - payout_state_generation=int(artifact.payout_state_generation), - anchor_age_ms=anchor_age_ms, - window_shares=len(artifact.shares_json), - ) - return False - with self._job_cache_lock: - outcome, armed_generation = ( - self._install_payout_ledger_artifact_locked(artifact) - ) - self._record_payout_artifact_event(outcome) - if outcome == "discarded": - self._payout_artifact_log( - "payout_artifact_discarded", - payout_state_generation=int(artifact.payout_state_generation), - anchor_age_ms=anchor_age_ms, - window_shares=len(artifact.shares_json), - ) - return False - if outcome in ("installed", "refreshed"): - self._payout_artifact_log( - "payout_artifact_" + outcome, - generation=armed_generation, - payout_state_generation=int(artifact.payout_state_generation), - anchor_age_ms=anchor_age_ms, - window_shares=len(artifact.shares_json), - ) - # An armed artifact -- installed fresh or confirmed already current - # -- proves preparation can succeed again; let the next fence-failure - # re-arm run at the configured floor. - with self._payout_artifact_executor_lock: - self._payout_artifact_rearm_backoff = 1 - return True - - def _install_payout_ledger_artifact_locked( - self, - artifact: PayoutLedgerArtifact, - ) -> tuple[str, int | None]: - """Ordering core of the install; returns (outcome, armed generation).""" - if ( - artifact.payout_state_generation != self._payout_state_generation - or artifact.append_invalidation_epoch - != self._payout_ledger_append_invalidation_epoch - ): - return "discarded", None - current = self._payout_ledger_artifact - if ( - current is not None - and current.payout_state_generation - == artifact.payout_state_generation - ): - current_anchor_ms = ( - -1 - if current.snapshot_anchor_ms is None - else int(current.snapshot_anchor_ms) - ) - artifact_anchor_ms = ( - -1 - if artifact.snapshot_anchor_ms is None - else int(artifact.snapshot_anchor_ms) - ) - if current_anchor_ms > artifact_anchor_ms: - return "discarded", None - if ( - current.network_difficulty == artifact.network_difficulty - and current.share_snapshot_sha256 is not None - and current.share_snapshot_sha256 - == artifact.share_snapshot_sha256 - ): - # The armed artifact already carries exactly this - # window; keep its generation (no lookup re-key) but - # still treat the preparation as a success, and advance - # the freshness clock in place either way -- otherwise a - # quiet share stream, or a pending-commit floor pinning - # the anchor so every re-prove lands at the SAME anchor, - # would cycle identical rebuilds forever without ever - # un-staling the armed artifact. No fresher window is - # constructible while the window bytes are unchanged, so - # the re-walk that just re-proved them earns the credit. - # The armed balances are kept: they are what the reuse - # fence hashes against the published payout state - # (including an accepted parent's preview patch). - anchor_advanced = artifact_anchor_ms > current_anchor_ms - self._payout_ledger_artifact = dataclass_replace( - current, - accepted_share_count=artifact.accepted_share_count, - prepared_monotonic=time.monotonic(), - snapshot_anchor_ms=( - artifact.snapshot_anchor_ms - if anchor_advanced - else current.snapshot_anchor_ms - ), - ) - if anchor_advanced: - return "refreshed", int(current.generation) - return "already_current", int(current.generation) - template_artifacts = getattr( - self, - "_template_artifacts", - None, - ) - if ( - current.network_difficulty != artifact.network_difficulty - and template_artifacts is not None - and current.network_difficulty - == int(template_artifacts.network_difficulty) - and artifact.network_difficulty - != int(template_artifacts.network_difficulty) - ): - # Whether a delayed pre-retarget build's anchor ties or - # leads, keep the artifact the live template difficulty - # can actually reuse: a wrong-difficulty install would - # fail every reuse probe on the difficulty check, which - # never re-arms, until the next synchronous build - # re-seeded the live window. - return "discarded", None - self._payout_ledger_artifact_generation += 1 - # Freshness runs from the moment the window becomes reusable, not - # from build completion: a sync-seeded artifact is constructed - # mid-bundle-build and reaches cache publication only after the - # audit builder, so stamping here keeps the budget honest at every - # install site. - self._payout_ledger_artifact = dataclass_replace( - artifact, - generation=self._payout_ledger_artifact_generation, - prepared_monotonic=time.monotonic(), - ) - return "installed", int(self._payout_ledger_artifact_generation) - - def _payout_artifact_preparation_loop(self) -> None: - while True: - with self._payout_artifact_executor_lock: - request = self._payout_artifact_requested - bypass_build_interval = ( - self._payout_artifact_requested_bypass - ) - self._payout_artifact_requested = None - self._payout_artifact_requested_bypass = False - if request is None: - self._payout_artifact_future = None - return - phases = self._job_build_phases() - phases.clear() - try: - self._prepare_payout_ledger_artifact( - *request, - bypass_build_interval=bypass_build_interval, - ) - finally: - self._flush_job_build_phases(phases) - - def _schedule_payout_ledger_artifact_preparation( - self, - payout_state_generation: int, - network_difficulty: int, - *, - min_interval_seconds: float | None = None, - bypass_build_interval: bool = False, - ) -> None: - """Latest-generation-wins scheduling with one worker and one slot. - - With min_interval_seconds, the debounce check, timestamp update, and - request enqueue are one atomic step under the executor lock: a - concurrent dequeue or a racing probe cannot slip a duplicate request - into the emptied slot and run the reward-window snapshot twice. - """ - self._ensure_job_cache_state() - if not self._payout_artifact_reuse_active(): - # Kill-switch: no background reward-window walks either. Off - # disables the reuse line -- probe refusals, no walks, no - # arming -- restoring pre-reuse delivery economics; the - # synchronous build path keeps the re-landed anchor-selection - # semantics rather than the pre-#102 count fences. - return - with self._payout_artifact_executor_lock: - if self._payout_artifact_executor_shutdown: - return - if min_interval_seconds is not None: - # Consecutive aborted rebuilds stretch the re-arm interval; - # event-driven preparations below reset the backoff because - # they mark a real state change worth retrying promptly for. - effective_interval = ( - min_interval_seconds * self._payout_artifact_rearm_backoff - ) - if ( - self._payout_artifact_last_schedule_monotonic is not None - and time.monotonic() - - self._payout_artifact_last_schedule_monotonic - < effective_interval - ): - return - rearm_backoff = self._payout_artifact_rearm_backoff - else: - self._payout_artifact_rearm_backoff = 1 - rearm_backoff = None - self._payout_artifact_requested = ( - int(payout_state_generation), - int(network_difficulty), - ) - self._payout_artifact_requested_bypass = ( - self._payout_artifact_requested_bypass - or bool(bypass_build_interval) - ) - self._payout_artifact_last_schedule_monotonic = time.monotonic() - if self._payout_artifact_future is None: - executor = self._payout_artifact_executor - if executor is None: - executor = ThreadPoolExecutor( - max_workers=1, - thread_name_prefix="prism-payout-artifact", - ) - self._payout_artifact_executor = executor - self._payout_artifact_future = executor.submit( - self._payout_artifact_preparation_loop - ) - if rearm_backoff is not None: - # Only debounced fence-failure re-arms log here; event-driven - # preparations ride payout publications, which are already - # visible. Recorded outside the executor lock the counter - # shares. - self._record_payout_artifact_event("rearm_scheduled") - self._payout_artifact_log( - "payout_artifact_rearm_scheduled", - payout_state_generation=int(payout_state_generation), - backoff=int(rearm_backoff), - ) - - def _payout_artifact_max_anchor_age_ms(self) -> float: - """Audit ceiling on a served window's wall-clock anchor age.""" - return ( - float( - getattr( - self, - "payout_artifact_max_anchor_age_seconds", - DEFAULT_PRISM_PAYOUT_ARTIFACT_MAX_ANCHOR_AGE_SECONDS, - ) - ) - * 1000.0 - ) - - def _payout_artifact_reanchor_seconds(self) -> float: - """Anchor age that triggers a background re-anchor while reuse keeps serving.""" - return float( - getattr( - self, - "payout_artifact_reanchor_seconds", - DEFAULT_PRISM_PAYOUT_ARTIFACT_REANCHOR_SECONDS, - ) - ) - - def _payout_artifact_min_build_interval_seconds(self) -> float: - """Minimum normal cadence; found-block settlement bypasses it.""" - - return float( - getattr( - self, - "payout_artifact_min_build_interval_seconds", - self._payout_artifact_reanchor_seconds(), - ) - ) - - def _payout_artifact_reuse_active(self) -> bool: - """Master kill-switch (PRISM_PAYOUT_ARTIFACT_REUSE) for the reuse line.""" - return bool(getattr(self, "payout_artifact_reuse_enabled", True)) - - @staticmethod - def _payout_artifact_log(event: str, **fields: object) -> None: - """Single-line JSON lifecycle log; the incident was invisible here.""" - print( - "prism coordinator: " - + json.dumps({"event": event, **fields}, sort_keys=True), - flush=True, - ) - - def _record_payout_artifact_event(self, event: str) -> None: - self._ensure_job_cache_state() - with self._payout_artifact_executor_lock: - counts = self.payout_artifact_event_counts - counts[event] = int(counts.get(event, 0)) + 1 - - def _usable_payout_ledger_artifact( - self, - payout_state_generation: int, - network_difficulty: int, - *, - rearm_on_fence_failure: bool = True, - ) -> PayoutLedgerArtifact | None: - """Return the armed artifact when reuse is valid for NEW work. - - Validity is anchor-scoped and EVENT-DRIVEN: the artifact's window is - exact at its own snapshot anchor (which reused bundles declare), so - shares landing after the anchor never invalidate it -- they belong - to the next window by construction -- and the artifact stays valid - until a payout event actually changes the state it snapshots (the - generation and balances fences below). Wall-clock time is NOT a - validity input below the audit ceiling: the 2026-07-29 rollback was - a 10s anchor-age budget rejecting every slow build on arrival, and - the 2026-07-30 brownout was the same budget moved to install time -- - at production cadence (a shared build every 10-14s, a 4-8s window - walk) a 10s budget expires between consecutive builds, so every - build fell back to the synchronous reward-window walk while the - re-arm worker walked the same window in parallel. Past the - REANCHOR floor the probe schedules a debounced background re-anchor - but KEEPS SERVING the armed window; only past the audit ceiling - (the bound on how far a paid window may trail the live ledger -- - post-anchor shares settle in the next window, none are lost) does - reuse fail closed. The ceiling governs NEW reuse and issuance - decisions; an in-flight build keeps the window it already selected, - so a declared anchor slightly past the ceiling can still publish. - """ - self._ensure_job_cache_state() - if not self._payout_artifact_reuse_active(): - return None - with self._job_cache_lock: - artifact = self._payout_ledger_artifact - published_artifact = self._published_payout_state.artifact - append_invalidation_epoch = ( - self._payout_ledger_append_invalidation_epoch - ) - if ( - artifact is None - or artifact.payout_state_generation != payout_state_generation - or artifact.network_difficulty != int(network_difficulty) - or artifact.append_invalidation_epoch - != append_invalidation_epoch - ): - return None - if artifact.snapshot_anchor_ms is None: - # Without a recorded anchor the artifact cannot declare the - # anchor reused bundles must stamp; fail closed. - return None - anchor_age_ms = now_ms() - int(artifact.snapshot_anchor_ms) - if anchor_age_ms > self._payout_artifact_max_anchor_age_ms(): - # The declared anchor crossed the audit ceiling; the window may - # no longer be served to new work. Queue a bounded speculative - # rebuild that re-anchors it; in-flight builds keep the copy - # they selected. - if rearm_on_fence_failure: - self._rearm_payout_ledger_artifact_after_fence_failure( - payout_state_generation, - network_difficulty, - ) - self._record_payout_artifact_event("probe_rejected_ceiling") - return None - if anchor_age_ms > self._payout_artifact_reanchor_seconds() * 1000.0: - # Aging but valid: schedule the debounced background re-anchor - # and keep serving. The delivery path must never pay the - # synchronous reward-window walk while a correct armed window - # exists; the re-anchor swaps in a fresher window off the - # critical path. Counted unconditionally so the canary can - # distinguish "serving past the floor, re-anchor pending" from - # "never crossing the floor" -- rearm_scheduled alone - # under-counts through the debounce and suppression paths. - self._record_payout_artifact_event("probe_past_floor") - if rearm_on_fence_failure: - self._rearm_payout_ledger_artifact_after_fence_failure( - payout_state_generation, - network_difficulty, - ) - if published_artifact is None: - try: - published_artifact = self._current_payout_state_artifact() - except Exception: - return None - balances_sha256 = artifact.prior_balances_sha256 or canonical_json_sha256( - artifact.prior_balances - ) - with self._job_cache_lock: - if ( - self._payout_state_generation != payout_state_generation - or self._published_payout_state.artifact is not published_artifact - or self._payout_ledger_append_invalidation_epoch - != artifact.append_invalidation_epoch - ): - return None - latest = self._payout_ledger_artifact - if latest is not artifact: - # An equal-window freshness restamp (already_current / - # refreshed) replaces the armed object while this probe was - # hashing balances. The restamp keeps the generation, the - # window bytes, and the balances object, and only moves the - # freshness stamp (and possibly the anchor) forward, so the - # validity established above still holds for the restamped - # copy; failing closed here would turn the intentional - # pinned-floor recovery path into a spurious synchronous - # reward-window walk. A real re-key changes the generation - # or the window sha and still fails closed. - if ( - latest is None - or latest.generation != artifact.generation - or latest.payout_state_generation - != artifact.payout_state_generation - or latest.network_difficulty != artifact.network_difficulty - or latest.share_snapshot_sha256 is None - or latest.share_snapshot_sha256 - != artifact.share_snapshot_sha256 - or latest.snapshot_anchor_ms is None - or latest.prior_balances is not artifact.prior_balances - or latest.append_invalidation_epoch - != artifact.append_invalidation_epoch - ): - return None - artifact = latest - if balances_sha256 != published_artifact.prior_balances_sha256: - # A candidate can carry a ledger snapshot prepared before its - # payout state is published. Never keep retrying that stale - # shortcut; the synchronous path will take a fresh snapshot. - self._payout_ledger_artifact = None - return None - served = artifact - # Recorded outside the cache lock the event counter's own lock - # ordering forbids nesting under. - self._record_payout_artifact_event("served_reuse") - return served - - def _rearm_payout_ledger_artifact_after_fence_failure( - self, - payout_state_generation: int, - network_difficulty: int, - ) -> None: - """Debounced rebuild scheduling for an artifact past the re-anchor floor or ceiling. - - The interval floor keeps delta/carry reads (and any full fallback) - from running continuously when artifacts age out faster than rebuilds - complete. - Landed accepted-block previews suppress the re-arm entirely: a - speculative rebuild in that window would read database balances the - published prospective state has already superseded, and the - durable-confirmation path resumes preparation itself once the gap - closes. - """ + self._tip_refresh_pending_counter = 0 + self._tip_refresh_pending_token: int | None = None + self._tip_refresh_retry = threading.Event() + self._tip_refresh_retry_counter = 0 + self._tip_refresh_retry_consumed = 0 + self._tip_refresh_failure_holdoff_until: float | None = None + self._tip_refresh_failure_tip: str | None = None + self._active_tip_refresh: tuple[ + TipRefreshValidationToken, + _FanoutCancellation, + ] | None = None + self._retained_collection_refresh: RetainedCollectionRefresh | None = None + self.last_reorg_reconciled_tip_hash: str | None = None + self.last_reorg_reconciled_trusted = False + self.last_reorg_reconciled_monotonic: float | None = None + self._prism_payout_policy_cache: dict[str, object] | None = None self._ensure_job_cache_state() - with self._job_cache_lock: - if ( - self._payout_state_publication_blocked - or payout_state_generation != self._payout_state_generation - ): - return - with self._accepted_block_payout_preview_condition: - if any( - transition.landed - for transition in self._accepted_block_payout_previews.values() - ): - return - cached_window = getattr( - self, - "_incremental_payout_artifact_window", - None, - ) - if ( - cached_window is not None - and time.monotonic() - cached_window.refreshed_monotonic - < self._payout_artifact_min_build_interval_seconds() - ): - # A pinned/old declared anchor can cross the re-anchor floor while - # the window itself was just re-proved. Do not enqueue a worker - # every five seconds merely to retag the same pages and re-read - # carry balances; the next probe after the real cadence expires - # will schedule it. - return - self._schedule_payout_ledger_artifact_preparation( - payout_state_generation, - network_difficulty, - min_interval_seconds=float( - getattr( - self, - "payout_artifact_rearm_min_seconds", - DEFAULT_PRISM_PAYOUT_ARTIFACT_REARM_MIN_SECONDS, - ) - ), + # Eager reorg owner root: the reconciler service adopts the compat + # seed values assigned above and owns every flight/memo/lookup/ + # prefetch container from here on. + self._ensure_reorg_reconciler_service() + self.stop_event = threading.Event() + self._shutdown_controller = CoordinatorShutdownController( + self.writer_quiescence_timeout_seconds ) + # Liveness watchdog: each monitored loop stamps a monotonic heartbeat; + # if any goes stale past the timeout the process exits non-zero so the + # container/systemd restart policy recovers a *hung* coordinator (a + # healthcheck alone does not restart it under plain compose). + self._heartbeats: dict[str, float] = {} + self._heartbeat_phases: dict[str, str] = {} + self._watchdog_pauses: dict[str, int] = {} + self._heartbeats_lock = threading.Lock() + self.watchdog_enabled = lifecycle_config.watchdog_enabled + self.watchdog_timeout_seconds = lifecycle_config.watchdog_timeout_seconds + self.watchdog_interval_seconds = lifecycle_config.watchdog_interval_seconds + # Lease/watchdog timing envelopes are projected from the immutable + # snapshot; services read the live attributes only because focused + # timing tests override them per instance after construction. + self.watchdog_lease_release_timeout_seconds = ( + lifecycle_config.watchdog_lease_release_timeout_seconds + ) + self.ledger_lease_heartbeat_seconds = ( + lifecycle_config.ledger_lease_heartbeat_seconds + ) + self.ledger_lease_heartbeat_failure_seconds = ( + lifecycle_config.ledger_lease_heartbeat_failure_seconds + ) + self.ledger_lease_heartbeat_monitor_seconds = ( + lifecycle_config.ledger_lease_heartbeat_monitor_seconds + ) + self.ledger_lease_heartbeat_exit_timeout_seconds = ( + lifecycle_config.ledger_lease_heartbeat_exit_timeout_seconds + ) + self.ledger_lease_external_fence_timeout_seconds = ( + lifecycle_config.ledger_lease_external_fence_timeout_seconds + ) + self._ctv_runtime_init_lock = threading.Lock() + self._ctv_runtime = self._make_ctv_runtime_service( + CtvRuntimeConfig.from_coordinator_config(ctv_config) + ) + # Eager B1 owner root: the block-candidate service adopted the legacy + # fields installed above (the first routed write constructs it) and + # owns every candidate queue/retry/disposition container from here on. + self._ensure_block_candidate_service() + # Eager B3 owner root: post-offer accepted-block finalization and its + # bounded phase/interarrival metrics live on the finalization service; + # B1's accounting actor reaches it through the coordinator seams. + self._ensure_block_finalization_service() + # Eager background-service root: process loop specifications, their + # start-once state, and the exact drain handles live in the registry; + # serve() starts named services at the existing recovery boundaries. + self._background_services = self._make_background_service_registry() - def _schedule_current_payout_ledger_artifact_if_missing(self) -> None: - """Resume speculative preparation after a durable preview catches up.""" - self._ensure_job_cache_state() - with self._job_cache_lock: - payout_state_generation = self._payout_state_generation - template_artifacts = self._template_artifacts - if template_artifacts is None: - return - if self._usable_payout_ledger_artifact( - payout_state_generation, - template_artifacts.network_difficulty, - # This call site enqueues unconditionally below; letting the - # probe also re-arm would race the one-slot worker's dequeue and - # run the reward-window snapshot twice for one resumption. - rearm_on_fence_failure=False, - ) is not None: - return - self._schedule_payout_ledger_artifact_preparation( - payout_state_generation, - template_artifacts.network_difficulty, - ) + def _ensure_share_hot_path_state(self) -> None: + """Backfill dedicated accounting state for lightweight embedders. - def shutdown_payout_artifact_executor(self) -> None: - self._ensure_job_cache_state() - with self._payout_artifact_executor_lock: - executor = self._payout_artifact_executor - self._payout_artifact_executor = None - self._payout_artifact_executor_shutdown = True - self._payout_artifact_requested = None - if executor is not None: - executor.shutdown(wait=True, cancel_futures=True) + Reduced first-touch shim: the recent-share duplicate window and the + share-ACK histograms are owned by the share-submission service, so + only the legacy accounting-counter lock is ensured here. + """ + if hasattr(self, "_share_accounting_lock"): + return + with _HOT_PATH_LOCK_INITIALIZATION_LOCK: + if not hasattr(self, "_share_accounting_lock"): + self._share_accounting_lock = threading.Lock() - def _job_build_checkpoint( + def _observe_share_ack_seconds( self, - phase: str, - cancellation: _JobBuildCancellation, + result: str, + elapsed_seconds: float, ) -> None: - """Cooperative boundary around coordinator and isolated-worker phases.""" + # Coordinator/session routing seam: the session owner stamps request + # receipt and reports the accepted/rejected response boundary here; + # the share-submission owner holds the histogram state. + self._ensure_share_submission_service().observe_share_ack_seconds( + result, + elapsed_seconds, + ) - cancellation.raise_if_cancelled(phase) + def share_ack_snapshot(self) -> dict[str, dict[str, Any]]: + """Copied share-ACK histograms for the metrics renderer. - def _record_job_cache_event(self, kind: str, *, hit: bool) -> None: - self._ensure_job_cache_state() - with self._job_cache_lock: - counts = self.job_cache_hit_counts if hit else self.job_cache_miss_counts - counts[kind] = int(counts.get(kind, 0)) + 1 + The data is the submission owner's copied snapshot. A coordinator + whose submission service was never touched has observed nothing, so + it reports zeroed histograms (or a legacy embedder's seeded state) + without forcing service construction. + """ + service = self.__dict__.get("_share_submission_service") + if service is not None: + return service.share_ack_snapshot() + legacy = self.__dict__.get("share_ack_histograms") + return ( + { + result: { + "buckets": dict(histogram["buckets"]), + "sum": float(histogram["sum"]), + "count": int(histogram["count"]), + } + for result, histogram in legacy.items() + } + if legacy is not None + else empty_share_ack_histograms() + ) - def _prepare_payout_state_artifact( - self, - *, - generation: int, - source_generation: int, - cancellation: _JobBuildCancellation | None = None, - ) -> PayoutStateArtifact: - """Snapshot carry-forward balances once for a payout generation.""" + def share_ack_metrics_lines(self) -> list[str]: + # Compatibility wrapper for the extracted metrics owner. + return self._ensure_metrics_renderer().share_ack_metrics_lines() - started = time.monotonic() - if cancellation is not None: - cancellation.raise_if_cancelled("payout artifact read") - with self._payout_state_prepare_lock: - balances = self.ledger.current_prior_balances() - if cancellation is not None: - cancellation.raise_if_cancelled("payout artifact serialization") - artifact = self._payout_state_artifact_from_balances( - generation=generation, - source_generation=source_generation, - balances=balances, + @staticmethod + def _client_vardiff_lock(client: ClientState) -> threading.RLock: + return client_vardiff_lock(client) + + def _reserve_recent_share_key(self, share_key: tuple[object, ...]) -> bool: + self._ensure_share_hot_path_state() + return self._ensure_share_submission_service().recent_shares.reserve( + share_key # type: ignore[arg-type] ) - phases = self._job_build_phases() - phases["payout_artifact"] = phases.get("payout_artifact", 0.0) + ( - time.monotonic() - started + + def _forget_recent_share_key(self, share_key: tuple[object, ...]) -> None: + self._ensure_share_submission_service().recent_shares.release( + share_key # type: ignore[arg-type] ) - return artifact - def _payout_state_artifact_from_balances( + def _share_submit_control_snapshot( self, - *, - generation: int, - source_generation: int, - balances: list[dict[str, object]], - ) -> PayoutStateArtifact: - balances_json = canonical_json_text(balances) - return PayoutStateArtifact( - generation=generation, - source_generation=source_generation, - prior_balances_json=balances_json, - prior_balances_sha256=hashlib.sha256(balances_json.encode()).hexdigest(), - prepared_monotonic=time.monotonic(), + client: ClientState, + job_id: str, + ) -> SubmitControlSnapshot: + pool_closed, context, published_tip = self._submit_control_snapshot( + client, + job_id, ) + return SubmitControlSnapshot( + pool_open=not pool_closed, + active_context=context, + published_tip=published_tip, + ) + + def _release_submit_share_key(self, share_key: tuple[str, str]) -> None: + self._ensure_share_submission_service().recent_shares.release(share_key) - def _publish_self_check_repaired_balances( + def _note_collection_block_candidate( self, - expected_payout_state_generation: int, - *, - stale_prior_balances_sha256: str, - balances: Sequence[dict[str, object]], - ) -> bool: - """Replace the published balance snapshot the periodic oracle refuted. - - The published artifact is otherwise immutable for its generation, but - a confirmed drift means its bytes no longer match the durable ledger - (a payout mutation whose response was lost). Swapping in the repaired - balances re-keys new work to the corrected digest, and every serving - decision for NOT-yet-stamped work re-checks that digest. Jobs already - stamped for miners have no such fence -- their admission compares - only the payout generation and append epoch, which both survive this - swap -- so the repair also schedules the same refresh wave a payout - publication schedules; digest-aware reselection then replaces exactly - the active jobs keyed to the refuted snapshot. Only the exact refuted - snapshot is replaced: any concurrent publication wins the guarded - swap (and drives its own wave). - """ - self._ensure_job_cache_state() - with self._job_cache_lock: - source_generation = self._published_payout_state.source_generation - repaired = self._payout_state_artifact_from_balances( - generation=int(expected_payout_state_generation), - source_generation=int(source_generation), - balances=list(balances), - ) - with self._job_cache_lock: - published = self._published_payout_state - current = published.artifact - if ( - self._payout_state_publication_blocked - or self._payout_state_generation - != expected_payout_state_generation - or published.generation != expected_payout_state_generation - or published.source_generation != source_generation - or current is None - or current.generation != expected_payout_state_generation - or current.prior_balances_sha256 - != stale_prior_balances_sha256 - ): - return False - self._published_payout_state = dataclass_replace( - published, - artifact=repaired, + context: PrismJobContext, + submission: Any, + ) -> None: + self._ensure_share_hot_path_state() + with self._share_accounting_lock: + self.collection_block_submission_count = ( + getattr(self, "collection_block_submission_count", 0) + 1 ) - self._record_payout_artifact_event("balance_repair_published") - self._payout_artifact_log( - "payout_artifact_balance_repair_published", - payout_state_generation=int(expected_payout_state_generation), - stale_prior_balances_sha256=stale_prior_balances_sha256, - repaired_prior_balances_sha256=repaired.prior_balances_sha256, - ) - # Active jobs keyed to the refuted digest remain mineable until a - # wave replaces them -- a block solved from one would commit the - # stale allocation. The pending mark survives a superseded wave and - # the digest reselection is idempotent, so a wave that raced this - # repair converges on the next pass. - self._mark_tip_refresh_pending(int(expected_payout_state_generation)) - self._schedule_tip_refresh_retry() - return True + print( + f"prism coordinator: collection-mode block candidate settles " + f"solver-pays-all miner={context.worker.payout_address} " + f"hash={submission.block_hash_hex}", + flush=True, + ) - def _current_payout_state_artifact( + def _note_submit_accounting( self, - cancellation: _JobBuildCancellation | None = None, - ) -> PayoutStateArtifact: - """Return the immutable artifact published for the current generation.""" + worker_name: str, + client: ClientState, + ) -> None: + self.note_worker_submitted_share(worker_name) + self.note_vardiff_submitted_share(client) - self._ensure_job_cache_state() - while True: - with self._job_cache_lock: - if self._payout_state_publication_blocked: - raise _PayoutStatePublicationBlocked( - "payout state invalidation is pending publication" + def _ensure_share_submission_service(self) -> ShareSubmissionService: + service = self.__dict__.get("_share_submission_service") + if service is not None: + return service + init_lock = self.__dict__.setdefault( + "_share_submission_service_init_lock", + threading.Lock(), + ) + with init_lock: + service = self.__dict__.get("_share_submission_service") + if service is not None: + return service + initial_recent_shares = self.__dict__.pop("recent_share_keys", set()) + initial_dropped_solves = self.__dict__.pop( + "block_solves_dropped_counts", None + ) + initial_share_ack = self.__dict__.pop("share_ack_histograms", None) + service = ShareSubmissionService( + ShareSubmissionPorts( + reject=lambda rejected, worker: self.reject_stratum( + rejected.code, + rejected.reason, + rejected.message, + worker=worker, + ), + control_snapshot=self._share_submit_control_snapshot, + note_submitted=self._note_submit_accounting, + retained_entry=lambda client, job_id: self.evicted_job_entry( + client, + job_id, + ), + live_tip=lambda: str(self.rpc.call("getbestblockhash")), + stale_grace_eligible=( + lambda client, context, current_tip: ( + self.context_eligible_for_stale_grace( + client, + context, + current_tip, + ) ) - published = self._published_payout_state - if ( - published.generation == self._payout_state_generation - and published.artifact is not None - and published.artifact.generation == published.generation - ): - return published.artifact - generation = self._payout_state_generation - source_generation = published.source_generation - artifact = self._prepare_payout_state_artifact( - generation=generation, - source_generation=source_generation, - cancellation=cancellation, - ) - with self._job_cache_lock: - published = self._published_payout_state - if ( - self._payout_state_publication_blocked - or self._payout_state_generation != generation - or published.source_generation != source_generation - ): - if cancellation is not None: - cancellation.raise_if_cancelled( - "payout artifact publication race" + ), + assemble=lambda client, context, request, version_mask: ( + direct_stratum.assemble_submission( + context.job, + extranonce2_hex=request.extranonce2_hex, + ntime_hex=request.ntime_hex, + nonce_hex=request.nonce_hex, + version_bits_hex=request.version_bits_hex, + version_mask=version_mask, + ) + ), + pending_share=lambda context, submission, ntime_hex, credit_policy: ( + self.pending_share_from_submission( + context=context, + submission=submission, + ntime_hex=ntime_hex, + credit_policy=credit_policy, + ) + ), + append_share=( + lambda client, context, submission, pending, policy, intent: ( + self.append_accepted_share( + client, + context, + submission, + pending, + credit_policy=policy, + candidate_intent=intent, + ) + ) + ), + note_retained_submit=( + lambda policy, cross_connection: self.note_evicted_job_submit( + policy, + cross_connection=cross_connection, + ) + ), + note_collection_candidate=( + lambda context, submission: self._note_collection_block_candidate( + context, + submission, + ) + ), + candidate_intent=lambda candidate: self.block_candidate_intent( + candidate + ), + finish_pending_commit=lambda pending: ( + self._finish_pending_share_commit(pending) + ), + record_terminal_outcome=lambda block_hash, accepted: ( + self._record_block_candidate_terminal_outcome( + block_hash, + accepted=accepted, + ) + ), + submit_synchronous_candidate=( + lambda candidate, share_key, worker, retained, policy: ( + self._submit_synchronous_credit_candidate( + candidate, + share_key=share_key, + worker_name=worker, + evicted_entry=retained, + credit_policy=policy, ) - continue - self._published_payout_state = dataclass_replace( - published, - artifact=artifact, - ) - return artifact - - def _job_build_executor_locked(self) -> ThreadPoolExecutor: - if self._job_build_executor_shutdown: - raise RuntimeError("job build executor is shut down") - executor = self._job_build_executor - if executor is None: - executor = ThreadPoolExecutor( - max_workers=int( - getattr( - self, - "job_build_executor_workers", - DEFAULT_PRISM_JOB_BUILD_EXECUTOR_WORKERS, ) ), - thread_name_prefix="prism-job-build", + enqueue_candidate=lambda candidate: self.enqueue_block_candidate( + candidate + ), + log=lambda message: print(message, flush=True), + log_exception=traceback.print_exc, + ), + extranonce2_size=int(self.extranonce2_size), + recent_shares=RecentShareIndex(initial=initial_recent_shares), ) - self._job_build_executor = executor - return executor + if initial_dropped_solves is not None: + service.replace_dropped_solves(initial_dropped_solves) + if initial_share_ack is not None: + service.replace_share_ack_histograms(initial_share_ack) + self.__dict__["_share_submission_service"] = service + return service - def _start_job_build_locked(self, request: _JobBuildRequest) -> _JobBuildFlight: - executor = self._job_build_executor_locked() - flight = _JobBuildFlight(request=request) - self.job_build_scheduler_counts["starts"] += 1 - self.shared_bundle_build_counts["started"] += 1 - if request.superseded_monotonic is not None: - elapsed = max(0.0, time.monotonic() - request.superseded_monotonic) - self.job_build_replacement_start_seconds["sum"] += elapsed - self.job_build_replacement_start_seconds["count"] += 1 - future = executor.submit(self._execute_job_build_request, request) - flight.future = future - self._record_priority_admission_locked(request, "started") - return flight + def record_rejection(self, reason: str, *, worker: str | None = None) -> None: + if reason not in PRISM_REJECTION_REASON_IDS: + raise ValueError(f"unknown PRISM rejection reason: {reason}") + self._ensure_share_hot_path_state() + with self._share_accounting_lock: + counts = getattr(self, "rejection_counts_by_reason", None) + if counts is None: + counts = {reason_id: 0 for reason_id in PRISM_REJECTION_REASON_IDS} + self.rejection_counts_by_reason = counts + counts[reason] = int(counts.get(reason, 0)) + 1 + if reason in {PRISM_REJECTION_STALE_JOB, PRISM_REJECTION_UNKNOWN_JOB, PRISM_REJECTION_BLOCK_STALE}: + self.stale_share_count += 1 + elif reason == PRISM_REJECTION_DUPLICATE_SHARE: + self.duplicate_share_count += 1 + elif reason == PRISM_REJECTION_LOW_DIFFICULTY: + self.low_difficulty_share_count += 1 + if worker is not None: + self._ensure_worker_metrics_state() + with self.worker_metrics_lock: + label = self._worker_metric_label_locked(worker) + key = (label, reason) + self.worker_rejection_counts[key] = ( + int(self.worker_rejection_counts.get(key, 0)) + 1 + ) - def _arm_job_build_locked(self, flight: _JobBuildFlight) -> None: - future = flight.future - assert future is not None - future.add_done_callback( - lambda completed, build_flight=flight: self._job_build_done( - build_flight, - completed, - ) - ) + def reject_stratum(self, code: int, reason: str, message: str, *, worker: str | None = None) -> None: + self.record_rejection(reason, worker=worker) + raise StratumError(code, message, reason=reason) - def _execute_job_build_request( - self, - request: _JobBuildRequest, - ) -> CachedJobBundle: - request.cancellation.raise_if_cancelled("start") - control = _JobBundleBuildControl( - key=request.equivalence_key, - previousblockhash=request.key.previous_block_hash, - payout_state_generation=request.key.payout_state_generation, - payout_artifact_generation=( - request.payout_ledger_artifact.generation - if request.payout_ledger_artifact is not None - else 0 - ), - ) - with self._job_cache_lock: - self._active_job_bundle_builds[control.key] = control - previous_control = getattr( - self._job_build_phase_local, - "bundle_build_control", - None, - ) - self._job_build_phase_local.bundle_build_control = control - phases = self._job_build_phases() - phases.clear() - try: - with localcontext(request.decimal_context): - return self.build_shared_job_bundle( - request.artifacts, - request.worker, - mode=request.mode, - payout_state_generation=request.key.payout_state_generation, - payout_artifact=request.payout_ledger_artifact, - key=request.cache_key, - build_request=request, + def worker_metric_label(self, worker: str) -> str: + """Metric label for one worker, from a bounded label set. + + The label is the stratum username as authorized (payout address plus + optional worker suffix). Usernames are miner-supplied, so the set of + distinct labels is capped: new workers past the cap aggregate into the + overflow label instead of growing metric cardinality without bound. + """ + self._ensure_worker_metrics_state() + with self.worker_metrics_lock: + return self._worker_metric_label_locked(worker) + + def _worker_metric_label_locked(self, worker: str) -> str: + label = worker or "_unauthenticated" + if len(label) > 128: + label = label[:128] + share_counts = self.worker_share_counts + if label in share_counts: + return label + limit = getattr(self, "worker_metrics_limit", DEFAULT_PRISM_WORKER_METRICS_LIMIT) + if len(share_counts) >= max(0, int(limit)): + label = PRISM_WORKER_METRICS_OVERFLOW_LABEL + share_counts.setdefault(label, {"submitted": 0, "accepted": 0, "grace": 0}) + return label + + def note_worker_submitted_share(self, worker: str) -> None: + self._ensure_worker_metrics_state() + with self.worker_metrics_lock: + label = self._worker_metric_label_locked(worker) + self.worker_share_counts[label]["submitted"] += 1 + + def note_worker_accepted_share(self, worker: str, credit_policy: str | None) -> None: + self._ensure_worker_metrics_state() + with self.worker_metrics_lock: + label = self._worker_metric_label_locked(worker) + counts = self.worker_share_counts[label] + counts["accepted"] += 1 + if credit_policy == PRISM_CREDIT_POLICY_STALE_GRACE: + counts["grace"] += 1 + if credit_policy == PRISM_CREDIT_POLICY_STALE_GRACE: + self._ensure_share_hot_path_state() + with self._share_accounting_lock: + self.grace_credited_share_count = ( + int(getattr(self, "grace_credited_share_count", 0)) + 1 ) - except BaseException: - # A build that died past its synchronous ledger read can no - # longer hand its exposed scan anchor to a seeded artifact; - # retire it here so a dead walk does not keep advancing the - # append-invalidation epoch. Successful builds either retired - # the token at the publication fence or handed it to the seed, - # whose install fence retires it. - self._retire_inflight_scan_anchor( - request.inflight_scan_anchor_token - ) - raise - finally: - self._flush_job_build_phases(phases) - self._job_build_phase_local.bundle_build_control = previous_control - with self._job_cache_lock: - if self._active_job_bundle_builds.get(control.key) is control: - self._active_job_bundle_builds.pop(control.key, None) - control.process = None @staticmethod - def _collection_job_builds_are_independent( - first: _JobBuildRequest, - second: _JobBuildRequest, - ) -> bool: - """Distinct workers in one immutable collection cohort are peers.""" - + def prometheus_label_value(value: str) -> str: return ( - first.mode == "collection" - and second.mode == "collection" - and first.key.collection_identity != second.key.collection_identity - and dataclass_replace(first.key, collection_identity=None) - == dataclass_replace(second.key, collection_identity=None) + value.replace("\\", "\\\\") + .replace("\n", "\\n") + .replace("\r", "\\r") + .replace('"', '\\"') ) - @staticmethod - def _job_build_requests_can_share( - first: _JobBuildRequest, - second: _JobBuildRequest, - ) -> bool: - """Share exact builds plus ready work stable across clock-only refreshes.""" - - return first.equivalence_key == second.equivalence_key or ( - first.mode == "ready" - and second.mode == "ready" - and first.cache_key == second.cache_key - ) + def resolve_version_rolling_mask(self, fallback_mask: int) -> direct_stratum.VersionRollingMaskSelection: + try: + template = self.rpc.call("getblocktemplate", [{"rules": qbit_gbt_rules(self.qbit_chain)}]) + if not isinstance(template, dict): + raise RuntimeError("getblocktemplate returned non-object") + except Exception as exc: + return direct_stratum.VersionRollingMaskSelection( + fallback_mask, + "fallback", + f"probe_error:{exc}", + ) + try: + return direct_stratum.select_version_rolling_mask(template, fallback_mask) + except ValueError as exc: + raise SystemExit(str(exc)) from exc - @staticmethod - def _ready_job_build_precedes_collection( - first: _JobBuildRequest, - second: _JobBuildRequest, - ) -> bool: - """Live ready work cannot be displaced by a collection-mode retry.""" + def parse_share_weights(self) -> dict[str, int]: + config = getattr(self, "config", None) + if config is not None: + return dict(config.stratum.share_weights_by_username) + return load_share_weights() - return ( - first.mode == "ready" - and second.mode == "collection" - and not first.cancellation.is_set() + def share_weight_for_worker(self, worker: WorkerIdentity) -> int: + return self.share_weights_by_username.get( + worker.username, + self.share_weights_by_username.get(worker.payout_address, self.default_share_weight), ) - @staticmethod - def _defer_job_build_locked( - *blockers: Future[CachedJobBundle], - ) -> Future[CachedJobBundle]: - """Wake a bounded lower-priority waiter when occupied capacity exits.""" - - deferred: Future[CachedJobBundle] = Future() - wake_lock = threading.Lock() - - def wake_for_retry(_completed: Future[CachedJobBundle]) -> None: - with wake_lock: - if not deferred.done(): - deferred.set_exception( - JobBuildSuperseded( - "job build capacity became available; retrying" - ) - ) - - for blocker in blockers: - blocker.add_done_callback(wake_for_retry) - return deferred - - @staticmethod - def _job_build_is_publication_critical(request: object) -> bool: - return bool(getattr(request, "publication_critical", False)) - - @staticmethod - def _job_build_promise_done(request: object) -> bool: - """Whether a request's shared promise has already been resolved. - - Reads defensively like the other request predicates: embedders and - tests plant lightweight request doubles without a promise, which are - never resolved-and-abandoned flights. - """ - - promise = getattr(request, "promise", None) - return promise is not None and promise.done() - - def _record_priority_admission_locked( - self, - request: _JobBuildRequest, - result: str, - ) -> None: - """Observe first builder admission from publication-priority reservation.""" - - if not self._job_build_is_publication_critical(request): - return - self.job_build_priority_counts[result] += 1 - if result not in {"started", "coalesced"}: - return - if request.priority_admission_recorded: - return - request.priority_admission_recorded = True - elapsed = max(0.0, time.monotonic() - request.requested_monotonic) - self.job_build_priority_admission_seconds["sum"] += elapsed - self.job_build_priority_admission_seconds["count"] += 1 - - def _record_initial_prepared_work_locked(self, result: str) -> None: - self.initial_job_prepared_work_counts[result] += 1 - - def _new_job_build_cancellation(self) -> _JobBuildCancellation: - return _JobBuildCancellation( - timeout_seconds=max( + def _ensure_audit_artifact_store(self) -> AuditArtifactStore: + init_lock = self.__dict__.setdefault( + "_audit_artifact_store_init_lock", + threading.Lock(), + ) + assert isinstance(init_lock, type(threading.Lock())) + with init_lock: + configured_audit_dir = self.__dict__.get("audit_dir") + if configured_audit_dir is None: + # Normal construction always sets audit_dir before any store + # use; only bare test instances reach this fallback. Root it + # under a per-instance temporary directory so a focused test + # never litters the process working directory. + fallback_root = self.__dict__.get("_audit_artifact_fallback_root") + if fallback_root is None: + fallback_root = Path( + tempfile.mkdtemp(prefix="prism-audit-fallback-") + ) + self.__dict__["_audit_artifact_fallback_root"] = fallback_root + configured_audit_dir = Path(fallback_root) / "prism-audit" + audit_dir = Path(configured_audit_dir) + evidence_path = Path( + self.__dict__.get( + "evidence_path", + audit_dir / "prism-live-stratum-evidence.json", + ) + ) + live_retention = int( + self.__dict__.get("audit_live_bundle_retention", 5) + ) + candidate_retention = int( + self.__dict__.get( + "audit_candidate_retention_seconds", + 24 * 60 * 60, + ) + ) + share_segment_size = int( + self.__dict__.get( + "audit_share_segment_size", + DEFAULT_AUDIT_SHARE_SEGMENT_SIZE, + ) + ) + # The verifier subprocess honors the coordinator's configured + # bundle-build budget rather than a store-local fixed default. + verifier_timeout_seconds = max( 0.001, float( - getattr( - self, - "job_build_timeout_seconds", - DEFAULT_PRISM_JOB_BUILD_TIMEOUT_SECONDS, + self.__dict__.get( + "bundle_build_timeout_seconds", + DEFAULT_PRISM_BUNDLE_BUILD_TIMEOUT_SECONDS, ) ), ) - ) - - def _begin_job_build_priority_preparation( - self, - requested_monotonic: float | None = None, - ) -> tuple[int, float]: - """Reserve publication priority before immutable request construction.""" - - started = ( - time.monotonic() - if requested_monotonic is None - else requested_monotonic - ) - with self._job_build_scheduler_lock: - self._job_build_priority_preparation_sequence += 1 - token = self._job_build_priority_preparation_sequence - self._job_build_priority_preparations[token] = started - for routine_cancellation_ref in tuple( - self._job_build_routine_preparations.values() - ): - routine_cancellation = routine_cancellation_ref() - if routine_cancellation is not None: - routine_cancellation.cancel("publication priority") - self._job_build_routine_preparations.clear() - self._job_build_priority_changed.set() - return token, started - - def _finish_job_build_priority_preparation(self, token: int) -> None: - with self._job_build_scheduler_lock: - self._job_build_priority_preparations.pop(token, None) - self._job_build_priority_changed.set() - - def _begin_routine_job_build_preparation( - self, - *, - request_source: str, - cancelled: Callable[[], bool] | None, - ) -> tuple[int, _JobBuildCancellation, CachedJobBundle | None]: - """Atomically admit cancellable routine request construction. - - Initial requests do not poll out the publication-priority window: - they subscribe to the in-flight priority build's promise, wake when - it completes, and hand its bundle back to the caller. Riding the - result that displaced them replaces the old defer-then-rebuild cycle - (and its wakeup storm) with one shared build. - """ - - deferred_recorded = False - subscription_recorded = False - subscribed_bundle: CachedJobBundle | None = None - admission_started_monotonic = time.monotonic() - admission_deadline_seconds = float( - getattr( - self, - "routine_admission_deadline_seconds", - DEFAULT_PRISM_ROUTINE_ADMISSION_DEADLINE_SECONDS, - ) - ) - while True: - self._job_build_priority_changed.clear() - priority_promises: tuple[Future[CachedJobBundle], ...] = () - with self._job_build_scheduler_lock: - if not self._publication_priority_scheduled_locked(): - self._job_build_routine_preparation_sequence += 1 - token = self._job_build_routine_preparation_sequence - preparation_cancellation = ( - self._new_job_build_cancellation() - ) - coordinator_ref = weakref.ref(self) - - def remove_dead_preparation( - dead_ref: weakref.ReferenceType[ - _JobBuildCancellation - ], - *, - preparation_token: int = token, - ) -> None: - coordinator = coordinator_ref() - if coordinator is None: - return - with coordinator._job_build_scheduler_lock: - if ( - coordinator._job_build_routine_preparations.get( - preparation_token - ) - is dead_ref - ): - coordinator._job_build_routine_preparations.pop( - preparation_token, - None, - ) - - self._job_build_routine_preparations[token] = weakref.ref( - preparation_cancellation, - remove_dead_preparation, - ) - return token, preparation_cancellation, subscribed_bundle - if not deferred_recorded: - self.job_build_priority_counts["routine_deferred"] += 1 - if request_source == "initial": - self._record_initial_prepared_work_locked("deferred") - deferred_recorded = True - if request_source == "initial": - priority_promises = ( - self._publication_priority_promises_locked() - ) - if priority_promises and not subscription_recorded: - self._record_initial_prepared_work_locked("subscribed") - subscription_recorded = True - if cancelled is not None and cancelled(): - raise _JobBuildCancelled( - "job bundle request was cancelled behind publication priority" - ) - stop_event = getattr(self, "stop_event", None) - if stop_event is not None and stop_event.is_set(): - raise _JobBuildCancelled( - "coordinator stopped behind publication priority" + store = self.__dict__.get("_audit_artifact_store") + if not isinstance(store, AuditArtifactStore): + store = AuditArtifactStore( + AuditArtifactConfig( + root=audit_dir, + evidence_path=evidence_path, + live_bundle_retention=live_retention, + candidate_retention_seconds=candidate_retention, + share_segment_size=share_segment_size, + verifier_timeout_seconds=verifier_timeout_seconds, + ), + canonicalizer=canonical_bundle_bytes, ) - parked_seconds = time.monotonic() - admission_started_monotonic - if parked_seconds > admission_deadline_seconds: - # Unbounded parking here is how the 2026-07-29 incident hid: - # initial jobs rode the client's own 90s timeout with zero - # tracebacks. Fail fast and visibly instead; callers retry - # with their normal pacing. - with self._job_build_scheduler_lock: - if request_source == "initial": - self._record_initial_prepared_work_locked( - "admission_deadline" - ) - now_monotonic = time.monotonic() - should_log = ( - self._admission_deadline_last_log_monotonic is None - or now_monotonic - - self._admission_deadline_last_log_monotonic >= 5.0 - ) - if should_log: - self._admission_deadline_last_log_monotonic = ( - now_monotonic - ) - if should_log: - print( - "prism coordinator: job admission deadline exceeded " - f"source={request_source} " - f"parked_seconds={parked_seconds:.1f}", - flush=True, + self.__dict__["_audit_artifact_store"] = store + if "_audit_latest_evidence_seed" in self.__dict__: + store.set_latest_evidence_for_compatibility( + self.__dict__.pop("_audit_latest_evidence_seed") ) - raise JobBuildAdmissionDeadlineExceeded( - "job bundle admission parked behind publication priority " - f"for {parked_seconds:.1f}s" + else: + updates: dict[str, Any] = {} + if store.root != audit_dir.expanduser().absolute().resolve(): + updates["root"] = audit_dir + expected_evidence = ( + evidence_path.expanduser().absolute().parent.resolve() + / evidence_path.name + ) + if store.evidence_path != expected_evidence: + updates["evidence_path"] = evidence_path + if store.live_bundle_retention != live_retention: + updates["live_bundle_retention"] = live_retention + if store.candidate_retention_seconds != candidate_retention: + updates["candidate_retention_seconds"] = candidate_retention + if store.share_segment_size != share_segment_size: + updates["share_segment_size"] = share_segment_size + if store.verifier_timeout_seconds != verifier_timeout_seconds: + updates["verifier_timeout_seconds"] = verifier_timeout_seconds + if updates: + store.reconfigure(**updates) + return store + + def _upgrade_legacy_audit_evidence(self) -> None: + store = self._ensure_audit_artifact_store() + with self._payout_balance_mutation_lock: + with store.publication_order_guard(): + legacy = store.legacy_evidence_identity() + if legacy is None: + return + reader = getattr(self.ledger, "pool_block_state", None) + floor_reader = getattr( + self.ledger, + "audit_publication_sequence_floor", + None, ) - if priority_promises: - for promise in priority_promises: - if ( - promise.done() - and not promise.cancelled() - and promise.exception() is None - ): - subscribed_bundle = promise.result() - pending_promises = tuple( - promise - for promise in priority_promises - if not promise.done() + if not callable(reader) or not callable(floor_reader): + store.invalidate_unprovable_legacy_evidence() + return + state = reader(block_hash=legacy.block_hash) + if not isinstance(state, dict): + store.invalidate_unprovable_legacy_evidence() + return + sequence = state.get("audit_publication_sequence") + state_block_hash = state.get("block_hash") + state_block_height = state.get("block_height") + if ( + sequence is None + or isinstance(sequence, bool) + or not isinstance(sequence, int) + or sequence <= 0 + or not isinstance(state_block_hash, str) + or state_block_hash != legacy.block_hash + or isinstance(state_block_height, bool) + or not isinstance(state_block_height, int) + or state_block_height != legacy.block_height + or str(state.get("chain_state") or "") != "confirmed" + or str(state.get("maturity_state") or "") + not in {"immature", "mature"} + ): + store.invalidate_unprovable_legacy_evidence() + return + publication_floor_sequence = floor_reader() + store.adopt_legacy_publication_identity( + AuditPublicationIdentity( + int(sequence), + legacy.block_height, + legacy.block_hash, + ), + publication_floor_sequence=publication_floor_sequence, ) - if pending_promises: - done, _pending = wait( - pending_promises, - timeout=PRISM_INITIAL_JOB_SUBSCRIBE_POLL_SECONDS, - return_when=FIRST_COMPLETED, - ) - for completed in done: - if ( - completed.cancelled() - or completed.exception() is not None - ): - continue - subscribed_bundle = completed.result() - else: - # Every live priority promise has resolved; the scheduler - # is about to sweep the finished flights. Waiting on the - # event instead of the done promises avoids a hot loop. - self._job_build_priority_changed.wait(0.05) - else: - # Reservation-only window: the priority request object (and - # its promise) does not exist yet. This window only spans - # immutable request construction, never the build itself. - self._job_build_priority_changed.wait(0.05) - def _publication_priority_promises_locked( + def _audit_publication_identity( self, - ) -> tuple[Future[CachedJobBundle], ...]: - """Promises of live publication-critical build requests.""" - - promises: list[Future[CachedJobBundle]] = [] - pending = self._job_build_pending - if ( - pending is not None - and not pending.cancellation.is_set() - and self._job_build_is_publication_critical(pending) - ): - promises.append(pending.promise) - for flight in (self._job_build_active, self._job_build_retiring): - if ( - flight is not None - and not flight.request.cancellation.is_set() - and self._job_build_is_publication_critical(flight.request) - ): - promises.append(flight.request.promise) - return tuple(promises) - - def _finish_routine_job_build_preparation(self, token: int) -> None: - with self._job_build_scheduler_lock: - self._job_build_routine_preparations.pop(token, None) + *, + block_hash: str, + block_height: int, + confirmation: Mapping[str, Any], + ) -> AuditPublicationIdentity: + sequence = confirmation.get("audit_publication_sequence") + if sequence is None: + # Compatibility-only fake ledgers in unit tests predate the durable + # ordinal. Production Postgres and the memory ledger always return + # it; never synthesize for an identified durable backend. + if str(confirmation.get("backend") or "") not in {"", "fake"}: + raise RuntimeError( + "ledger confirmation omitted audit publication sequence" + ) + sequences = self.__dict__.setdefault( + "_compat_audit_publication_sequences", + {}, + ) + assert isinstance(sequences, dict) + sequence = sequences.get(block_hash) + if sequence is None: + sequence = max( + [ + self._ensure_audit_artifact_store().publication_sequence_floor(), + *(int(value) for value in sequences.values()), + ] + ) + 1 + sequences[block_hash] = sequence + if isinstance(sequence, bool) or not isinstance(sequence, int): + raise RuntimeError("ledger confirmation returned invalid publication sequence") + if isinstance(block_height, bool) or not isinstance(block_height, int): + raise RuntimeError("ledger confirmation returned invalid block height") + canonical_block_hash = str(block_hash).lower() + if canonical_block_hash != block_hash: + raise RuntimeError("ledger confirmation returned non-canonical block hash") + return AuditPublicationIdentity( + sequence, + block_height, + canonical_block_hash, + ) - def _publication_priority_scheduled_locked(self) -> bool: - if self._job_build_priority_preparations: - return True - pending = self._job_build_pending + def make_ledger( + self, + *, + lease_retry_sleep: Callable[[float], None] | None = None, + ) -> SingleWriterShareLedger | PsqlShareLedger: + config = getattr(self, "config", None) + ledger_config = config.ledger if config is not None else None + psql_command = ( + ledger_config.psql_command + if ledger_config is not None + else os.environ.get("PRISM_POSTGRES_PSQL_COMMAND", "") + ) + database_url = ( + ledger_config.database_url or "" + if ledger_config is not None + else os.environ.get("PRISM_DATABASE_URL", "") + ) + if not psql_command and database_url: + psql_command = f"psql {shlex.quote(database_url)}" + if not psql_command: + allow_memory_ledger = ( + ledger_config.allow_memory_ledger + if ledger_config is not None + else env_bool("PRISM_ALLOW_MEMORY_LEDGER", "0") + ) + if not allow_memory_ledger: + raise SystemExit( + "PRISM_DATABASE_URL or PRISM_POSTGRES_PSQL_COMMAND is required; " + "set PRISM_ALLOW_MEMORY_LEDGER=1 only for local tests" + ) + return SingleWriterShareLedger( + ctv_broadcast_attempt_detail_limit=getattr( + self, + "ctv_broadcast_attempt_detail_limit", + DEFAULT_CTV_BROADCAST_ATTEMPT_DETAIL_LIMIT, + ), + ctv_broadcast_retry_backoff_seconds=getattr( + self, + "ctv_broadcast_retry_backoff_seconds", + DEFAULT_CTV_BROADCAST_RETRY_BACKOFF_SECONDS, + ), + ) + writer_session_token = ( + ledger_config.writer_session_token + if ledger_config is not None + else env_optional("PRISM_LEDGER_WRITER_SESSION_TOKEN") + ) if ( - pending is not None - and not pending.cancellation.is_set() - and not self._job_build_promise_done(pending) - and self._job_build_is_publication_critical(pending) + ledger_config is None + and writer_session_token is not None + and not env_bool("PRISM_ALLOW_FIXED_LEDGER_SESSION_TOKEN", "0") ): - return True - return any( - flight is not None - and not flight.request.cancellation.is_set() - and not self._job_build_promise_done(flight.request) - and self._job_build_is_publication_critical(flight.request) - for flight in ( - self._job_build_active, - self._job_build_retiring, + raise SystemExit( + "PRISM_LEDGER_WRITER_SESSION_TOKEN requires " + "PRISM_ALLOW_FIXED_LEDGER_SESSION_TOKEN=1 for local tests" + ) + if writer_session_token is None: + writer_session_token = ( + f"{WRITER_LEASE_HEARTBEAT_SESSION_PREFIX}{uuid.uuid4().hex}" ) + audit_store = self._ensure_audit_artifact_store() + ledger = self._construct_psql_share_ledger( + psql_command, + database_url, + ledger_config, + writer_session_token, + audit_store, + lease_retry_sleep=lease_retry_sleep, ) + try: + # Issue #255 rollback floor: refuse a database whose candidate + # storage is newer than this process, or one without the 002 + # migration (every storage version needs it), before any share + # or candidate is written. + ledger.verify_candidate_schema() + except IncompatibleCandidateSchema as exc: + ledger.close() + raise SystemExit(f"prism coordinator: {exc}") from exc + except Exception as exc: # noqa: BLE001 - a transport failure is not a verdict + print( + "prism coordinator: candidate schema capability check could not " + f"run at boot ({type(exc).__name__}: {exc}); a version-2 candidate " + "write against an unmigrated database fails closed on its own", + flush=True, + ) + return ledger - def _job_build_can_inherit_publication_priority( + def _construct_psql_share_ledger( self, - existing: _JobBuildRequest, - incoming: _JobBuildRequest, - ) -> bool: - """Reject an almost-expired or stalled routine flight as a critical owner.""" - - if ( - not self._job_build_is_publication_critical(incoming) - or self._job_build_is_publication_critical(existing) - ): - return True - cancellation = existing.cancellation - total_budget = max( - 0.001, - cancellation.deadline_monotonic - cancellation.started_monotonic, - ) - remaining_budget = cancellation.deadline_monotonic - time.monotonic() - progress_budget = max( - 0.001, - float( - getattr( - self, - "job_build_cancel_grace_seconds", - DEFAULT_PRISM_JOB_BUILD_CANCEL_GRACE_SECONDS, + psql_command: str, + database_url: str, + ledger_config: Any, + writer_session_token: str | None, + audit_store: Any, + *, + lease_retry_sleep: Callable[[float], None] | None = None, + ) -> PsqlShareLedger: + config = getattr(self, "config", None) + return PsqlShareLedger( + psql_command=psql_command, + lease_retry_sleep=lease_retry_sleep, + database_url=database_url or None, + native_client_mode=( + ledger_config.native_client_mode + if ledger_config is not None + else env("PRISM_POSTGRES_NATIVE_CLIENT", "auto") + ), + writer_id=( + ledger_config.writer_id + if ledger_config is not None + else env("PRISM_LEDGER_WRITER_ID", "prism-coordinator") + ), + writer_epoch=( + ledger_config.writer_epoch + if ledger_config is not None + else env_int("PRISM_LEDGER_WRITER_EPOCH", 1) + ), + writer_session_token=writer_session_token, + initialize_schema=( + ledger_config.initialize_schema + if ledger_config is not None + else env("PRISM_POSTGRES_INIT_SCHEMA", "0") in {"1", "true", "yes"} + ), + lease_ttl_seconds=( + ledger_config.lease_ttl_seconds + if ledger_config is not None + else env_positive_float("PRISM_LEDGER_LEASE_TTL_SECONDS", 60.0) + ), + lease_acquire_lock_timeout_seconds=( + ledger_config.lease_acquire_lock_timeout_seconds + if ledger_config is not None + else env_positive_float( + "PRISM_LEDGER_LEASE_ACQUIRE_LOCK_TIMEOUT_SECONDS", + DEFAULT_LEASE_ACQUIRE_LOCK_TIMEOUT_SECONDS, ) ), - ) - progress_age = ( - time.monotonic() - - float( - getattr( - cancellation, - "last_checkpoint_monotonic", - cancellation.started_monotonic, + lease_acquire_attempts=( + ledger_config.lease_acquire_attempts + if ledger_config is not None + else env_positive_int( + "PRISM_LEDGER_LEASE_ACQUIRE_ATTEMPTS", + DEFAULT_LEASE_ACQUIRE_ATTEMPTS, ) - ) - ) - return ( - remaining_budget >= total_budget / 2.0 - and progress_age <= progress_budget + ), + postgres_idle_in_transaction_timeout_seconds=( + ledger_config.postgres_idle_in_transaction_timeout_seconds + if ledger_config is not None + else env_positive_float( + "PRISM_POSTGRES_IDLE_IN_TRANSACTION_TIMEOUT_SECONDS", + DEFAULT_POSTGRES_IDLE_IN_TRANSACTION_TIMEOUT_SECONDS, + ) + ), + postgres_tcp_keepalives_idle_seconds=( + ledger_config.postgres_tcp_keepalives_idle_seconds + if ledger_config is not None + else env_positive_int( + "PRISM_POSTGRES_TCP_KEEPALIVES_IDLE_SECONDS", + DEFAULT_POSTGRES_TCP_KEEPALIVES_IDLE_SECONDS, + ) + ), + postgres_tcp_keepalives_interval_seconds=( + ledger_config.postgres_tcp_keepalives_interval_seconds + if ledger_config is not None + else env_positive_int( + "PRISM_POSTGRES_TCP_KEEPALIVES_INTERVAL_SECONDS", + DEFAULT_POSTGRES_TCP_KEEPALIVES_INTERVAL_SECONDS, + ) + ), + postgres_tcp_keepalives_count=( + ledger_config.postgres_tcp_keepalives_count + if ledger_config is not None + else env_positive_int( + "PRISM_POSTGRES_TCP_KEEPALIVES_COUNT", + DEFAULT_POSTGRES_TCP_KEEPALIVES_COUNT, + ) + ), + # The own-write deferral margin must cover the longest RPC the + # lease fence can authorize — submitblock's dedicated deadline + # and the CTV broadcast RPCs riding JsonRpc.call's default — + # plus fixed headroom for fence-to-RPC scheduling and clock + # drift (see LEASE_AUTHORITY_MARGIN_HEADROOM_SECONDS; the RPCs + # themselves cannot outlive their wall-clock deadlines). The + # ledger floors this at half the TTL, so short deadlines keep + # today's behavior and only an operator raising a guarded + # deadline widens the deferral window with it. + lease_authority_margin_seconds=( + LEASE_AUTHORITY_MARGIN_HEADROOM_SECONDS + + max( + float( + config.block.submit_rpc_timeout_seconds + if config is not None + else getattr( + self, + "block_submit_rpc_timeout_seconds", + DEFAULT_BLOCK_SUBMIT_RPC_TIMEOUT_SECONDS, + ) + ), + DEFAULT_QBIT_RPC_CALL_TIMEOUT_SECONDS, + ) + ), + read_concurrency=( + ledger_config.read_concurrency + if ledger_config is not None + else env_positive_int("PRISM_POSTGRES_READ_CONCURRENCY", 4) + ), + candidate_storage_version=( + ledger_config.candidate_storage_version + if ledger_config is not None + else env_positive_int("PRISM_CANDIDATE_STORAGE_VERSION", 2) + ), + candidate_spool_dir=( + ledger_config.candidate_spool_dir + if ledger_config is not None + else (env_optional("PRISM_CANDIDATE_SPOOL_DIR") or None) + ), + candidate_spool_reservation_bytes=( + ledger_config.candidate_spool_reservation_bytes + if ledger_config is not None + else env_positive_int( + "PRISM_CANDIDATE_SPOOL_RESERVATION_BYTES", + 4 * 1024 * 1024 * 1024, + ) + ), + accepted_stats_cache_seconds=( + ledger_config.accepted_stats_cache_seconds + if ledger_config is not None + else env_nonnegative_float("PRISM_ACCEPTED_STATS_CACHE_SECONDS", 60.0) + ), + reward_window_cache_seconds=( + ledger_config.reward_window_cache_seconds + if ledger_config is not None + else env_nonnegative_float( + "PRISM_PUBLIC_REWARD_WINDOW_CACHE_SECONDS", 30.0 + ) + ), + audit_artifact_store=audit_store, + ctv_broadcast_attempt_detail_limit=getattr( + self, + "ctv_broadcast_attempt_detail_limit", + DEFAULT_CTV_BROADCAST_ATTEMPT_DETAIL_LIMIT, + ), + ctv_broadcast_retry_backoff_seconds=getattr( + self, + "ctv_broadcast_retry_backoff_seconds", + DEFAULT_CTV_BROADCAST_RETRY_BACKOFF_SECONDS, + ), ) - @staticmethod - def _resolve_cancelled_job_build_promise( - request: _JobBuildRequest, - reason: str, - ) -> None: - """Wake every waiter on a terminated flight's promise immediately. + def load_trusted_ledger_writer_public_key(self) -> str | None: + config = getattr(self, "config", None) + if config is not None: + return config.ledger.writer_public_key_hex + configured = env_optional("PRISM_LEDGER_WRITER_PUBLIC_KEY_HEX") + if configured is not None: + return validate_hex(configured, name="PRISM_LEDGER_WRITER_PUBLIC_KEY_HEX", expected_bytes=32) + if env_bool("PRISM_ALLOW_BUNDLE_EMBEDDED_LEDGER_KEY", "0"): + return None + raise SystemExit( + "PRISM_LEDGER_WRITER_PUBLIC_KEY_HEX is required; " + "set PRISM_ALLOW_BUNDLE_EMBEDDED_LEDGER_KEY=1 only for local tests" + ) - Safe against the build's own completion: the done callback checks - ``promise.done()`` before resolving, so the first resolution wins and - a late build result for a cancelled flight is dropped exactly as it - was when the callback produced the exception itself. - """ + def prism_payout_policy(self) -> dict[str, object]: + cached = getattr(self, "_prism_payout_policy_cache", None) + if cached is not None: + return cached - promise = getattr(request, "promise", None) - if promise is None or promise.done(): - return - if reason == "timeout": - promise.set_exception( - JobBuildCancelled("job build flight cancelled by timeout") + policy = default_prism_payout_policy() + output_policy = env_optional("PRISM_COINBASE_OUTPUT_POLICY") or "canonical" + if output_policy not in VALID_COINBASE_OUTPUT_POLICIES: + raise SystemExit( + "PRISM_COINBASE_OUTPUT_POLICY must be one of: " + + ", ".join(sorted(VALID_COINBASE_OUTPUT_POLICIES)) ) - return - promise.set_exception( - JobBuildSuperseded(f"job build flight cancelled: {reason}") + fee_bps_raw = env_optional("PRISM_POOL_FEE_BPS") + fee_enabled = env_bool("PRISM_POOL_FEE_ENABLED", "0") + fee_address = env_optional("PRISM_POOL_FEE_ADDRESS") + fee_program_hex = env_optional("PRISM_POOL_FEE_P2MR_PROGRAM_HEX") + fee_recipient_id = env_optional("PRISM_POOL_FEE_RECIPIENT_ID") + fee_order_key = env_optional("PRISM_POOL_FEE_ORDER_KEY") + has_fee_config = any( + value is not None + for value in (fee_bps_raw, fee_address, fee_program_hex, fee_recipient_id, fee_order_key) ) + if not fee_enabled: + if has_fee_config: + raise SystemExit("set PRISM_POOL_FEE_ENABLED=1 when configuring pool fees") + if output_policy == "pool-fee-first": + raise SystemExit( + "PRISM_COINBASE_OUTPUT_POLICY=pool-fee-first requires PRISM_POOL_FEE_ENABLED=1" + " and a configured pool fee" + ) + self._prism_payout_policy_cache = policy + return policy + if fee_bps_raw is None: + raise SystemExit("PRISM_POOL_FEE_BPS is required when pool fees are enabled") - @staticmethod - def _job_build_flight_outcome( - request: _JobBuildRequest, - future: Future[CachedJobBundle], - ) -> tuple[CachedJobBundle | None, BaseException | None]: - """Map a finished executor future onto the shared promise outcome.""" - - result: CachedJobBundle | None = None - error: BaseException | None = None try: - result = future.result() - if request.cancellation.is_set(): - if request.cancellation.reason == "timeout": - error = JobBuildCancelled( - "job build completed after its timeout" - ) - else: - error = JobBuildSuperseded( - "obsolete job build completed after cancellation" - ) - except BaseException as exc: # noqa: BLE001 - delivered to all waiters - error = exc - return result, error - - def _evict_orphaned_job_build_flights_locked(self) -> list[str]: - """Evict finished flights whose completion never released their slot. - - A flight's done callback resolves its promise, vacates its slot, and - promotes pending work. Done callbacks swallow exceptions, so if one - dies mid-pass the flight wedges: admission keeps treating the slot as - occupied, deferred requesters chain onto a promise nobody will ever - resolve, and promotion never runs again even though the executor is - idle. Sweep such flights out at admission time: resolve the promise - from the finished future so parked waiters wake, free the slot so the - next requester becomes the new owner, and promote any parked pending - request. A flight whose future is still queued or executing is never - touched; its own completion performs the normal cleanup. - - Returns the eviction log lines instead of printing them: the caller - holds the scheduler lock, and a blocked stdout must not stall - admission (the admission-deadline log follows the same discipline). - """ + fee_bps = int(fee_bps_raw) + except ValueError as exc: + raise SystemExit("PRISM_POOL_FEE_BPS must be an integer") from exc + if fee_bps < 0 or fee_bps > 10_000: + raise SystemExit("PRISM_POOL_FEE_BPS must be between 0 and 10000") + if (fee_address is None) == (fee_program_hex is None): + raise SystemExit("set exactly one of PRISM_POOL_FEE_ADDRESS or PRISM_POOL_FEE_P2MR_PROGRAM_HEX") - eviction_logs: list[str] = [] - grace_seconds = float( - getattr( - self, - "job_build_orphan_sweep_grace_seconds", - PRISM_JOB_BUILD_ORPHAN_SWEEP_GRACE_SECONDS, + if fee_address is not None: + validation = self.rpc.call("validateaddress", [fee_address]) + if not isinstance(validation, dict) or not validation.get("isvalid"): + raise SystemExit(f"PRISM_POOL_FEE_ADDRESS is not a valid qbit address: {fee_address}") + script = str(validation.get("scriptPubKey") or "") + if not script.startswith("5220") or len(script) != 68: + raise SystemExit("PRISM_POOL_FEE_ADDRESS must resolve to a P2MR script") + fee_policy = { + "fee_bps": fee_bps, + "recipient_id": fee_address, + "order_key": fee_address, + "p2mr_program_hex": script[4:], + } + else: + program_hex = validate_hex( + fee_program_hex or "", + name="PRISM_POOL_FEE_P2MR_PROGRAM_HEX", + expected_bytes=32, ) + if fee_recipient_id is None: + raise SystemExit("PRISM_POOL_FEE_RECIPIENT_ID is required with PRISM_POOL_FEE_P2MR_PROGRAM_HEX") + fee_policy = { + "fee_bps": fee_bps, + "recipient_id": fee_recipient_id, + "order_key": fee_order_key or fee_recipient_id, + "p2mr_program_hex": program_hex, + } + + policy["pool_fee_policy"] = fee_policy + if output_policy != "canonical": + # Omitted for canonical so canonical payloads and signed audit + # artifacts keep their historical bytes. + policy["coinbase_output_policy"] = output_policy + self._prism_payout_policy_cache = policy + return policy + + def prism_ctv_settlement_config( + self, + *, + block_height: int | None = None, + parent_hash: str | None = None, + ) -> dict[str, object] | None: + if not env_bool("PRISM_CTV_SETTLEMENT_ENABLED", "0"): + return None + direct_floor_sats = env_positive_int_with_legacy( + "PRISM_DIRECT_COINBASE_PAYOUT_FLOOR_BITS", + "PRISM_DIRECT_COINBASE_PAYOUT_FLOOR_SATS", + DEFAULT_DIRECT_COINBASE_PAYOUT_FLOOR_SATS, ) - now = time.monotonic() - evicted = False - for slot_name in ("_job_build_active", "_job_build_retiring"): - flight: _JobBuildFlight | None = getattr(self, slot_name) - if flight is None: - continue - # Embedders and tests plant lightweight flight doubles; read the - # lifecycle fields defensively like the rest of the scheduler. - future = getattr(flight, "future", None) - if future is not None and not future.done(): - continue - observed = getattr(flight, "orphan_observed_monotonic", None) - if observed is None: - try: - flight.orphan_observed_monotonic = now - except AttributeError: - pass - continue - if now - observed < grace_seconds: - continue - request = flight.request - promise = getattr(request, "promise", None) - if promise is not None and not promise.done(): - if future is None: - self._resolve_cancelled_job_build_promise( - request, - "superseded", - ) - else: - result, error = self._job_build_flight_outcome( - request, - future, - ) - try: - if error is not None: - promise.set_exception(error) - else: - assert result is not None - promise.set_result(result) - except InvalidStateError: - # The flight's own done callback won the resolution - # race between our done() check and this set. - pass - setattr(self, slot_name, None) - self.job_build_scheduler_counts["orphan_evicted"] += 1 - evicted = True - eviction_logs.append( - "prism coordinator: evicted orphaned job build flight " - f"slot={slot_name.removeprefix('_job_build_')} " - f"cancelled={flight.request.cancellation.is_set()} " - f"started={future is not None}" - ) - if evicted: - self._promote_pending_job_build_locked() - self._job_build_priority_changed.set() - return eviction_logs + reserved_coinbase_outputs = env_int("PRISM_RESERVED_COINBASE_OUTPUTS", 0) + if reserved_coinbase_outputs < 0: + raise SystemExit("PRISM_RESERVED_COINBASE_OUTPUTS must be non-negative") + config: dict[str, object] = { + "direct_floor_sats": direct_floor_sats, + "config": { + "max_coinbase_settlement_outputs": env_positive_int( + "PRISM_MAX_COINBASE_SETTLEMENT_OUTPUTS", + DEFAULT_MAX_COINBASE_SETTLEMENT_OUTPUTS, + ), + "max_direct_coinbase_outputs": env_positive_int( + "PRISM_MAX_DIRECT_COINBASE_OUTPUTS", + DEFAULT_MAX_DIRECT_COINBASE_OUTPUTS, + ), + "max_fanout_recipients_per_transaction": env_positive_int( + "PRISM_MAX_CTV_FANOUT_RECIPIENTS_PER_TRANSACTION", + DEFAULT_MAX_CTV_FANOUT_RECIPIENTS_PER_TRANSACTION, + ), + "reserved_coinbase_outputs": reserved_coinbase_outputs, + }, + } + config["fanout_fee_rate_policy"] = { + "market_fee_rate_sats_per_1000_weight": self.ctv_fanout_market_fee_rate_bits_per_1000_weight( + block_height=block_height, + parent_hash=parent_hash, + ), + "premium_bps": env_positive_int( + "PRISM_CTV_FANOUT_FEE_PREMIUM_BPS", + DEFAULT_CTV_FANOUT_FEE_PREMIUM_BPS, + ), + } + return config - def _cancel_job_build_flight_locked( + def ctv_fanout_market_fee_rate_bits_per_1000_weight( self, - flight: _JobBuildFlight, - reason: str, *, - now: float | None = None, - ) -> bool: - if not flight.request.cancellation.cancel(reason): - # Already cancelled (possibly by its own deadline). A flight that - # never reached the executor still has no other resolver, so - # settle its waiters here instead of leaving them to the sweep. - if getattr(flight, "future", None) is None: - self._resolve_cancelled_job_build_promise( - flight.request, - flight.request.cancellation.reason or reason, - ) - return False + block_height: int | None = None, + parent_hash: str | None = None, + ) -> int: + configured_rate = env_optional_positive_int_with_legacy( + "PRISM_CTV_FANOUT_FEE_MARKET_RATE_BITS_PER_1000_WEIGHT", + "PRISM_CTV_FANOUT_FEE_MARKET_RATE_SATS_PER_1000_WEIGHT", + ) + if configured_rate is not None: + return configured_rate + fee_rate_cache = getattr(self, "_ctv_fanout_market_fee_rate_cache", None) + if fee_rate_cache is None: + fee_rate_cache = {} + self._ctv_fanout_market_fee_rate_cache = fee_rate_cache + cache_key = (block_height, parent_hash) + if cache_key in fee_rate_cache: + return fee_rate_cache[cache_key] try: - flight.request.superseded_monotonic = ( - time.monotonic() if now is None else now + estimate = self.rpc.call( + "estimatesmartfee", + [env_positive_int("PRISM_CTV_FANOUT_FEE_ESTIMATE_TARGET_BLOCKS", 2)], ) - self.job_build_scheduler_counts["supersessions"] += 1 - if reason == "publication priority": - self.job_build_priority_counts["routine_preempted"] += 1 - finally: - # A submitted future delivers the promise from its done callback - # when the build observes the cancellation and drains -- refresh - # drivers rely on that ordering to keep one heavy build in - # flight, and a completed-but-unswept future is the admission - # sweep's to resolve. A flight that never reached the executor - # has no callback at all: cancellation must wake its waiters - # itself or they burn their full wait deadline. - if getattr(flight, "future", None) is None: - self._resolve_cancelled_job_build_promise( - flight.request, - reason, - ) - self._job_build_priority_changed.set() - return True + if not isinstance(estimate, dict): + raise RuntimeError("estimatesmartfee returned non-object") + errors = estimate.get("errors") + if errors: + raise RuntimeError(f"estimatesmartfee returned errors: {errors}") + feerate = Decimal(str(estimate.get("feerate", ""))) + if not feerate.is_finite() or feerate <= 0: + raise RuntimeError(f"estimatesmartfee returned invalid feerate: {estimate.get('feerate')!r}") + rate = int((feerate * Decimal(100_000_000)).to_integral_value(rounding=ROUND_CEILING)) + if rate <= 0: + raise RuntimeError("estimatesmartfee rounded to a non-positive rate") + except Exception as exc: + raise RuntimeError( + "unable to compute PRISM CTV fanout fee rate; set " + "PRISM_CTV_FANOUT_FEE_MARKET_RATE_BITS_PER_1000_WEIGHT or fix " + f"estimatesmartfee: {exc}" + ) from exc + fee_rate_cache[cache_key] = rate + return rate - def _promote_pending_job_build_locked(self) -> None: - pending = self._job_build_pending - if pending is None: - return - if self._job_build_promise_done(pending): - # Every waiter has already been answered (eviction or a raced - # resolution); building it would spend a bounded executor slot on - # work nobody consumes. - self._job_build_pending = None - return - active = self._job_build_active - retiring = self._job_build_retiring - if ( - not self._job_build_is_publication_critical(pending) - and any( - flight is not None - and not flight.request.cancellation.is_set() - and not self._job_build_promise_done(flight.request) - and self._job_build_is_publication_critical(flight.request) - for flight in (active, retiring) + def _ensure_job_bundle_service(self) -> JobBundleService: + service = self.__dict__.get("job_bundle_service") + if service is not None: + return service + init_lock = self.__dict__.get("_job_bundle_service_init_lock") + if init_lock is None: + # CPython's setdefault is atomic under the GIL. Focused tests may + # construct through __new__, while normal instances build the + # service on the first legacy-field touch during __init__. + init_lock = self.__dict__.setdefault( + "_job_bundle_service_init_lock", + threading.Lock(), + ) + with init_lock: + service = self.__dict__.get("job_bundle_service") + if service is not None: + return service + service = JobBundleService( + self, + # Resolve the spool-file factory through this module's global + # at call time so the historical + # ``prism_coordinator._share_window_spool_file`` patch seam + # keeps steering spool creation. + spool_factory=lambda: _share_window_spool_file(), + payout_ledger_artifact_type=PayoutLedgerArtifact, + now_ms=lambda: now_ms(), + canonical_json_sha256_override=( + lambda value: canonical_json_sha256(value) + ), ) - ): - # The pending request may predate an exact-flight priority upgrade. - # Preserve the same admission invariant as _request_job_build: - # routine work cannot displace scheduled publication work. - return - if active is not None: - if retiring is not None: - return - if self._ready_job_build_precedes_collection( - active.request, - pending, - ) and not self._job_build_is_publication_critical(pending): - return - if not self._collection_job_builds_are_independent( - active.request, - pending, - ): - reason = ( - "publication priority" - if self._job_build_is_publication_critical(pending) - and not self._job_build_is_publication_critical(active.request) - else "superseded" - ) - self._cancel_job_build_flight_locked(active, reason) - self._job_build_retiring = active - self._job_build_active = None - elif retiring is not None: - if self._ready_job_build_precedes_collection( - retiring.request, - pending, - ) and not self._job_build_is_publication_critical(pending): - return - if not self._collection_job_builds_are_independent( - retiring.request, - pending, - ): - reason = ( - "publication priority" - if self._job_build_is_publication_critical(pending) - and not self._job_build_is_publication_critical(retiring.request) - else "superseded" - ) - self._cancel_job_build_flight_locked(retiring, reason) - self._job_build_pending = None - try: - flight = self._start_job_build_locked(pending) - except BaseException: - # The pending slot is already vacated: with no flight there is no - # done callback left that could ever resolve this promise for its - # waiters. - self._resolve_cancelled_job_build_promise(pending, "superseded") - raise - self._job_build_active = flight - self._arm_job_build_locked(flight) - - def _job_build_done( - self, - flight: _JobBuildFlight, - future: Future[CachedJobBundle], - ) -> None: - request = flight.request - result, error = self._job_build_flight_outcome(request, future) - try: - with self._job_build_scheduler_lock: - self.job_build_scheduler_counts["completions"] += 1 - self.shared_bundle_preparation_count += 1 - self.shared_bundle_preparation_seconds_sum += max( - 0.0, - time.monotonic() - request.cancellation.started_monotonic, - ) - if request.cancellation.cancelled_monotonic is not None: - elapsed = max( - 0.0, - time.monotonic() - - request.cancellation.cancelled_monotonic, - ) - self.job_build_cancellation_seconds["sum"] += elapsed - self.job_build_cancellation_seconds["count"] += 1 - coordination_cancelled = isinstance( - error, JobBuildSuperseded - ) or ( - request.cancellation.is_set() - and request.cancellation.reason != "timeout" - ) - if error is not None and coordination_cancelled: - self.job_build_scheduler_counts["obsolete_results"] += 1 - self.shared_bundle_build_counts["superseded"] += 1 - with self._tip_refresh_metrics_lock: - self.tip_refresh_superseded_results += 1 - elif error is not None: - self.shared_bundle_build_counts["failed"] += 1 - else: - self.shared_bundle_build_counts["completed"] += 1 - if self._job_build_active is flight: - self._job_build_active = None - if self._job_build_retiring is flight: - self._job_build_retiring = None - self._promote_pending_job_build_locked() - finally: - # Done callbacks swallow exceptions, so a raise anywhere in the - # bookkeeping above must not skip promise resolution: an orphaned - # promise strands every joiner until its wait deadline and leaves - # a finished flight wedged in its slot with the executor idle. - try: - if not request.promise.done(): - if error is not None: - request.promise.set_exception(error) - else: - assert result is not None - request.promise.set_result(result) - except InvalidStateError: - # The admission sweep can resolve a wedged flight between the - # done() check and this set; either resolution wakes waiters. - pass - self._job_build_priority_changed.set() - - def _request_job_build(self, request: _JobBuildRequest) -> Future[CachedJobBundle]: - self._ensure_job_cache_state() - with self._job_build_scheduler_lock: - eviction_logs = self._evict_orphaned_job_build_flights_locked() - for eviction_log in eviction_logs: - print(eviction_log, flush=True) - with self._job_build_scheduler_lock: - if getattr(request, "idle_retarget", False): - with self.lock: - defer_idle = self._vardiff_idle_tip_divergence_locked() - if defer_idle: - request.cancellation.cancel( - "idle retarget deferred during unpublished tip refresh" - ) - if not request.promise.done(): - request.promise.set_exception( - JobBuildSuperseded( - "idle retarget deferred during unpublished tip refresh" - ) - ) - return request.promise - self.job_build_scheduler_counts["requests"] += 1 - active = self._job_build_active - retiring = self._job_build_retiring - pending = self._job_build_pending - publication_critical = self._job_build_is_publication_critical( - request + self.__dict__["job_bundle_service"] = service + return service + + def _ensure_bundle_compiler(self) -> BundleCompiler: + compiler = self.__dict__.get("bundle_compiler") + if compiler is not None: + return compiler + init_lock = self.__dict__.get("_bundle_compiler_init_lock") + if init_lock is None: + init_lock = self.__dict__.setdefault( + "_bundle_compiler_init_lock", + threading.Lock(), + ) + with init_lock: + compiler = self.__dict__.get("bundle_compiler") + if compiler is not None: + return compiler + compiler = BundleCompiler( + self, + superseded_error=_JobBundleBuildSuperseded, + cancellation_error_types=( + JobBuildCancelled, + _JobBundleBuildSuperseded, + ), + build_control_type=_JobBundleBuildControl, + # Resolve the builder command through this module's global at + # call time so the historical patch seam keeps working. + tool_command=lambda name: prism_tool_command(name), + ) + self.__dict__["bundle_compiler"] = compiler + return compiler + + def _ensure_tip_refresh_service(self) -> TipRefreshService: + service = self.__dict__.get("tip_refresh_service") + if service is not None: + return service + init_lock = self.__dict__.get("_tip_refresh_service_init_lock") + if init_lock is None: + # CPython's setdefault is atomic under the GIL. Focused tests may + # construct through __new__, while normal instances build the + # service on the first legacy-field touch during __init__. + init_lock = self.__dict__.setdefault( + "_tip_refresh_service_init_lock", + threading.Lock(), + ) + with init_lock: + service = self.__dict__.get("tip_refresh_service") + if service is not None: + return service + service = TipRefreshService( + self, + # Coordinator-owned exception types and the S2 delivery + # priorities are injected so the R1 owner never imports this + # module (or job_delivery, which sits above it). + shutdown_error=ShutdownInProgress, + job_build_failed_error=_JobBuildFailed, + delivery_priority_initial=PRISM_DELIVERY_PRIORITY_INITIAL, + delivery_priority_new_tip=PRISM_DELIVERY_PRIORITY_NEW_TIP, + delivery_priority_same_tip=PRISM_DELIVERY_PRIORITY_SAME_TIP, + ) + self.__dict__["tip_refresh_service"] = service + return service + + def _ensure_job_delivery_service(self) -> JobDeliveryService: + service = self.__dict__.get("job_delivery_service") + if service is not None: + return service + init_lock = self.__dict__.get("_job_delivery_service_init_lock") + if init_lock is None: + init_lock = self.__dict__.setdefault( + "_job_delivery_service_init_lock", + threading.Lock(), + ) + with init_lock: + service = self.__dict__.get("job_delivery_service") + if service is not None: + return service + service = JobDeliveryService( + self, + # The Stratum error type is owned by the session module; it + # is injected so the S2 owner never imports the session owner + # (or this module). + stratum_error=StratumError, + ) + self.__dict__["job_delivery_service"] = service + return service + + def _ensure_payout_state_service(self) -> PayoutStateService: + service = self.__dict__.get("payout_state_service") + if service is not None: + return service + init_lock = self.__dict__.get("_payout_state_service_init_lock") + if init_lock is None: + init_lock = self.__dict__.setdefault( + "_payout_state_service_init_lock", + threading.Lock(), + ) + with init_lock: + service = self.__dict__.get("payout_state_service") + if service is not None: + return service + service = PayoutStateService( + self, + # The shutdown exception type stays coordinator-owned until + # the shutdown controller is extracted; the wall-clock stamp + # and canonical serialization resolve this module's globals + # at call time so existing monkeypatches keep intercepting. + shutdown_error=ShutdownInProgress, + now_ms=lambda: now_ms(), + canonical_json_text_override=( + lambda value: canonical_json_text(value) + ), + canonical_json_sha256_override=( + lambda value: canonical_json_sha256(value) + ), ) - if ( - active is not None - and not active.request.cancellation.is_set() - and not self._job_build_promise_done(active.request) - and self._job_build_requests_can_share(active.request, request) - and self._job_build_can_inherit_publication_priority( - active.request, - request, - ) - ): - if publication_critical: - active.request.publication_critical = True - active.request.request_source = request.request_source - active.request.requested_monotonic = request.requested_monotonic - active.request.priority_admission_recorded = True - self._record_priority_admission_locked(request, "coalesced") - if request.request_source == "initial": - self._record_initial_prepared_work_locked("singleflight") - return active.request.promise - if ( - retiring is not None - and not retiring.request.cancellation.is_set() - and not self._job_build_promise_done(retiring.request) - and self._job_build_requests_can_share(retiring.request, request) - and self._job_build_can_inherit_publication_priority( - retiring.request, - request, - ) - ): - if publication_critical: - retiring.request.publication_critical = True - retiring.request.request_source = request.request_source - retiring.request.requested_monotonic = request.requested_monotonic - retiring.request.priority_admission_recorded = True - self._record_priority_admission_locked(request, "coalesced") - if request.request_source == "initial": - self._record_initial_prepared_work_locked("singleflight") - return retiring.request.promise - if ( - pending is not None - and not pending.cancellation.is_set() - and not self._job_build_promise_done(pending) - and self._job_build_requests_can_share(pending, request) - and self._job_build_can_inherit_publication_priority( - pending, - request, - ) - ): - if publication_critical: - pending.publication_critical = True - pending.request_source = request.request_source - pending.requested_monotonic = request.requested_monotonic - self._record_priority_admission_locked(request, "queued") - now = time.monotonic() - for occupied in (active, retiring): - if ( - occupied is not None - and not self._job_build_requests_can_share( - occupied.request, - pending, - ) - and not self._job_build_is_publication_critical( - occupied.request - ) - ): - self._cancel_job_build_flight_locked( - occupied, - "publication priority", - now=now, - ) - self._promote_pending_job_build_locked() - if request.request_source == "initial": - self._record_initial_prepared_work_locked("singleflight") - return pending.promise - - if not publication_critical: - priority_blockers = tuple( - blocker - for blocker in ( - active.request if active is not None else None, - retiring.request if retiring is not None else None, - pending, - ) - if blocker is not None - and not blocker.cancellation.is_set() - # A resolved promise is not live priority work: deferring - # on it wakes instantly and spins until the sweep evicts. - and not self._job_build_promise_done(blocker) - and self._job_build_is_publication_critical(blocker) - ) - if priority_blockers: - self.job_build_priority_counts["routine_deferred"] += 1 - if request.request_source == "initial": - self._record_initial_prepared_work_locked("deferred") - return self._defer_job_build_locked( - *(blocker.promise for blocker in priority_blockers) - ) - - if request.mode == "collection": - possible_blockers = ( - pending, - active.request if active is not None else None, - retiring.request if retiring is not None else None, - ) - for blocker in possible_blockers: - if ( - blocker is not None - and not blocker.cancellation.is_set() - and not self._job_build_promise_done(blocker) - and self._ready_job_build_precedes_collection( - blocker, - request, - ) - and not ( - publication_critical - and not self._job_build_is_publication_critical( - blocker - ) + self.__dict__["payout_state_service"] = service + return service + + def _ensure_share_writer_service(self) -> ShareWriter: + service = self.__dict__.get("share_writer_service") + if service is not None: + return service + init_lock = self.__dict__.get("_share_writer_service_init_lock") + if init_lock is None: + init_lock = self.__dict__.setdefault( + "_share_writer_service_init_lock", + threading.Lock(), + ) + with init_lock: + service = self.__dict__.get("share_writer_service") + if service is not None: + return service + service = ShareWriter( + self, + # The rejection-reason registry stays coordinator-owned; the + # wall-clock stamp resolves this module's now_ms global at + # call time so existing monkeypatches keep intercepting. + internal_error_reason=PRISM_REJECTION_INTERNAL_ERROR, + now_ms=lambda: now_ms(), + ) + self.__dict__["share_writer_service"] = service + return service + + def _ensure_stratum_session_service(self) -> StratumSessionService: + service = self.__dict__.get("stratum_session_service") + if service is not None: + return service + init_lock = self.__dict__.get("_stratum_session_service_init_lock") + if init_lock is None: + init_lock = self.__dict__.setdefault( + "_stratum_session_service_init_lock", + threading.Lock(), + ) + with init_lock: + service = self.__dict__.get("stratum_session_service") + if service is not None: + return service + service = StratumSessionService( + self, + # The shutdown exception type stays coordinator-owned until + # the shutdown controller is extracted; inject it so the S1 + # owner never imports this module. + shutdown_error=ShutdownInProgress, + pool_closed_reason=PRISM_REJECTION_POOL_CLOSED, + ) + self.__dict__["stratum_session_service"] = service + return service + + def _ensure_reorg_reconciler_service(self) -> ReorgReconcilerService: + service = self.__dict__.get("_reorg_reconciler_service") + if service is not None: + return service + init_lock = self.__dict__.setdefault( + "_reorg_reconciler_service_init_lock", + threading.Lock(), + ) + with init_lock: + service = self.__dict__.get("_reorg_reconciler_service") + if service is not None: + return service + service = ReorgReconcilerService( + ReorgPorts( + rpc_call=lambda *args, **kwargs: self.rpc.call( + *args, + **kwargs, + ), + ledger=lambda: self.ledger, + ensure_job_cache_state=self._ensure_job_cache_state, + # The memo/counter state lock is the coordinator's + # control-plane lock (resolved live: focused tests + # replace it), preserving the atomicity tip observation + # relies on when it evicts memo entries under self.lock. + state_lock=lambda: self.lock, + source_tip=self._reorg_payout_source_tip, + reserve_external_tip=lambda tip: self._reserve_payout_state_source( + "external_tip", + tip_hash=tip, + ), + max_supersession_retries=lambda: getattr( + self, + "payout_reconcile_supersession_retries", + DEFAULT_PRISM_PAYOUT_RECONCILE_SUPERSESSION_RETRIES, + ), + # #113's heartbeat-aware acquisition of the payout-state + # preparation lock. + prepare_lock=lambda: self._block_submitter_lock( + self._payout_state_prepare_lock, + "payout-state-prepare", + ), + capture_source=lambda: self._capture_payout_state_source(), + prepared_candidate=lambda captured, **kwargs: ( + self._prepared_payout_state_candidate( + captured, + **kwargs, ) - ): - return self._defer_job_build_locked(blocker.promise) - if active is None: - if pending is not None: - if self._collection_job_builds_are_independent( - pending, - request, - ): - self._job_build_pending = None - try: - flight = self._start_job_build_locked(pending) - except BaseException: - # The pending slot is already vacated: with no - # flight there is no done callback left that - # could resolve this promise for its waiters. - self._resolve_cancelled_job_build_promise( - pending, - "superseded", - ) - raise - if retiring is None: - try: - replacement = self._start_job_build_locked( - request - ) - except BaseException: - # The first start succeeded: slot and arm it - # so its completion still resolves and sweeps - # normally before this failure propagates. - self._job_build_active = flight - self._arm_job_build_locked(flight) - raise - self._job_build_retiring = flight - self._job_build_active = replacement - self._arm_job_build_locked(flight) - self._arm_job_build_locked(replacement) - return request.promise - self._job_build_active = flight - self._arm_job_build_locked(flight) - return self._defer_job_build_locked( - flight.request.promise, - retiring.request.promise, + ), + captured_publication_required=lambda captured: ( + self._captured_payout_source_requires_publication( + captured ) - pending.cancellation.cancel("superseded while pending") - if not pending.promise.done(): - pending.promise.set_exception( - JobBuildSuperseded("pending job build was superseded") + ), + block_publication=lambda **kwargs: ( + self._block_payout_state_publication(**kwargs) + ), + publication_guard=lambda: ( + self._ensure_audit_artifact_store().publication_order_guard() + ), + publish_candidate=lambda candidate: ( + self._publish_payout_state_candidate(candidate) + ), + observe_preparation=lambda elapsed: ( + self._observe_payout_state_seconds( + "preparation", + elapsed, ) - self._job_build_pending = None - self.job_build_scheduler_counts["supersessions"] += 1 - if ( - retiring is not None - and not self._collection_job_builds_are_independent( - retiring.request, - request, - ) - ): - now = time.monotonic() - reason = ( - "publication priority" - if publication_critical - and not self._job_build_is_publication_critical( - retiring.request + ), + chain_view_untrusted=lambda: self.qbit_chain_view_untrusted(), + reorg_proof_snapshot=lambda: ( + self._ensure_tip_refresh_service().reorg_proof_snapshot() + ), + flight_wait_seconds=lambda: float( + getattr( + self, + "reconcile_flight_wait_seconds", + DEFAULT_PRISM_RECONCILE_FLIGHT_WAIT_SECONDS, ) - else "superseded" - ) - if self._cancel_job_build_flight_locked( - retiring, - reason, - now=now, - ): - request.superseded_monotonic = now - flight = self._start_job_build_locked(request) - self._job_build_active = flight - self._arm_job_build_locked(flight) - return request.promise - - if self._collection_job_builds_are_independent( - active.request, - request, + ), + prefetch_join_timeout_seconds=lambda: getattr( + self, + "reconcile_prefetch_join_timeout_seconds", + PRISM_RECONCILE_PREFETCH_JOIN_TIMEOUT_SECONDS, + ), + reconcile_with_admission=lambda **kwargs: ( + self.reconcile_prism_pool_blocks_once(**kwargs) + ), + reconcile_serialized=lambda **kwargs: ( + self._reconcile_prism_pool_blocks_serialized(**kwargs) + ), + ensure_tip=lambda tip: self.ensure_reorg_reconciled_for_tip( + tip + ), + # Accepted-block landings run a reconcile pass inline on + # the watchdog-monitored block-work thread. The recorder + # stamps only from that thread's registered owner, so + # background reconciliation passes and per-client callers + # keep recording nothing at all. + record_progress=lambda phase: ( + self._record_block_submitter_phase(phase) + ), + # Issue #224: the one attribution owner every lane and + # the renderer share; resolved live so it is the same + # object the metrics port reads. + accepted_preview_telemetry=lambda: ( + ensure_accepted_preview_telemetry(self) + ), + ), + enabled=bool(self.reorg_reconciler_enabled), + cache_seconds=self.reorg_reconcile_cache_seconds, + inactive_block_count=int(self.reorg_inactive_block_count), + reactivated_block_count=int(self.reorg_reactivated_block_count), + reconcile_skip_count=int(self.reorg_reconcile_skip_count), + reconcile_error_count=int(self.reorg_reconcile_error_count), + matured_payout_count=int(self.matured_payout_count), + last_tip_hash=self.last_reorg_reconciled_tip_hash, + last_trusted=bool(self.last_reorg_reconciled_trusted), + last_monotonic=self.last_reorg_reconciled_monotonic, + ) + self.__dict__["_reorg_reconciler_service"] = service + for name in ( + "enabled", + "cache_seconds", + "inactive_block_count", + "reactivated_block_count", + "reconcile_skip_count", + "reconcile_error_count", + "matured_payout_count", + "last_tip_hash", + "last_trusted", + "last_monotonic", ): - if self._job_build_retiring is None: - self._job_build_retiring = active - flight = self._start_job_build_locked(request) - self._job_build_active = flight - self._arm_job_build_locked(flight) - return request.promise - if pending is None: - self._job_build_pending = request - if publication_critical: - self._record_priority_admission_locked(request, "queued") - now = time.monotonic() - for occupied in (active, self._job_build_retiring): - if ( - occupied is not None - and not self._job_build_is_publication_critical( - occupied.request - ) - ): - self._cancel_job_build_flight_locked( - occupied, - "publication priority", - now=now, - ) - return request.promise - assert retiring is not None - if publication_critical: - pending.cancellation.cancel("superseded while pending") - if not pending.promise.done(): - pending.promise.set_exception( - JobBuildSuperseded( - "pending job build was superseded by publication priority" - ) - ) - self.job_build_scheduler_counts["supersessions"] += 1 - self._job_build_pending = request - self._record_priority_admission_locked(request, "queued") - now = time.monotonic() - for occupied in (active, retiring): - if not self._job_build_is_publication_critical( - occupied.request - ): - self._cancel_job_build_flight_locked( - occupied, - "publication priority", - now=now, - ) - return request.promise - return self._defer_job_build_locked( - active.request.promise, - retiring.request.promise, + self.__dict__.pop(f"_reorg_compat_{name}", None) + return service + + def _reorg_payout_source_tip(self) -> str | None: + """Payout-state snapshot source tip under the control-plane lock.""" + with self.lock: + return self._payout_state_source[1] + + def _ensure_job_cache_state(self) -> None: + # J1 scheduler/cache/template state lives in its owner service; the + # legacy raw fields route there through class descriptors and the + # service adopts direct test assignments through those descriptors. + self._ensure_job_bundle_service() + if not hasattr(self, "job_build_timeout_seconds"): + self.job_build_timeout_seconds = DEFAULT_PRISM_JOB_BUILD_TIMEOUT_SECONDS + if not hasattr(self, "job_build_cancel_grace_seconds"): + self.job_build_cancel_grace_seconds = ( + DEFAULT_PRISM_JOB_BUILD_CANCEL_GRACE_SECONDS + ) + # P1 payout state lives in its owner service; the legacy raw + # fields route there through class descriptors and the service + # adopts direct test assignments through those descriptors (see + # lab/prism/payout_state.py). + self._ensure_payout_state_service() + # The S3 owner holds the pending-commit floor lock and dictionary + # (see lab/prism/share_writer.py); ensure it exists for anchor reads. + self._ensure_share_writer_service() + # The serve-builder daemon state lives in the compiler owner; the + # legacy raw fields route there through class descriptors. + self._ensure_bundle_compiler() + # Reorg flight/memo/lookup/prefetch state lives in its owner service + # (see lab/prism/reorg_reconciler.py); the service constructor is the + # single initializer for those containers. + if not hasattr(self, "_accounted_accepted_block_hashes"): + self._accounted_accepted_block_hashes: set[str] = set() + # The B1 owner (created on first touch) holds the block-submit + # histogram, acceptance-evidence containers, and abandonment dedupe + # set; the legacy raw fields route there through class descriptors. + self._ensure_block_candidate_service() + # Health snapshot and mining-delivery timer state lives in the + # observability owner; the legacy raw fields route there through the + # class compatibility properties. + self._ensure_observability_service() + # The aggregate progress-health state lives in its owner service; the + # legacy raw ``_progress_*`` fields route there through descriptors. + self._ensure_progress_health_service() + + def _ensure_observability_service(self) -> ObservabilityService: + service = self.__dict__.get("_observability_service") + if service is not None: + return service + init_lock = self.__dict__.setdefault( + "_observability_service_init_lock", + threading.Lock(), + ) + with init_lock: + service = self.__dict__.get("_observability_service") + if service is None: + service = ObservabilityService(_CoordinatorObservability(self)) + self.__dict__["_observability_service"] = service + return service + + def _ensure_audit_http_facade(self) -> AuditHttpFacade: + facade = self.__dict__.get("_audit_http_facade") + if facade is not None: + return facade + init_lock = self.__dict__.setdefault( + "_audit_http_facade_init_lock", + threading.Lock(), + ) + with init_lock: + facade = self.__dict__.get("_audit_http_facade") + if facade is None: + facade = AuditHttpFacade( + _CoordinatorAuditHttp(self), + AuditHttpConfig( + bind=str(getattr(self, "audit_bind", None) or "127.0.0.1"), + port=int(getattr(self, "audit_port", 0)), + ), ) + self.__dict__["_audit_http_facade"] = facade + return facade - now = time.monotonic() - for obsolete in (active, retiring): - if obsolete is not None: - reason = ( - "publication priority" - if publication_critical - and not self._job_build_is_publication_critical( - obsolete.request - ) - else "superseded" - ) - self._cancel_job_build_flight_locked( - obsolete, - reason, - now=now, - ) - request.superseded_monotonic = now - if retiring is None: - self._job_build_retiring = active - flight = self._start_job_build_locked(request) - self._job_build_active = flight - self._arm_job_build_locked(flight) - return request.promise - - previous_pending = self._job_build_pending - if previous_pending is not None: - previous_pending.cancellation.cancel("superseded while pending") - if not previous_pending.promise.done(): - previous_pending.promise.set_exception( - JobBuildSuperseded("pending job build was superseded") - ) - self.job_build_scheduler_counts["supersessions"] += 1 - self._job_build_pending = request - if publication_critical: - self._record_priority_admission_locked(request, "queued") - return request.promise + def _job_build_phases(self) -> dict[str, float]: + """Per-thread scratch dict of phase timings for the current build.""" + return self._ensure_job_bundle_service()._job_build_phases() - def _cancel_obsolete_job_builds( + def _cancel_obsolete_job_bundle_builds( self, - reason: str, *, - keep_published_snapshot: bool = False, + current_tip: str | None = None, + payout_state_generation: int | None = None, ) -> None: - self._ensure_job_cache_state() - published_parent: str | None = None - published_fingerprint: str | None = None - if keep_published_snapshot: - # A per-client build for exactly the published snapshot is still - # valid, creditable work when a same-tip template bump sweeps old - # fingerprints. Detection-time sweeps deliberately do not use this: - # once a replacement tip is detected, even published-tip builds - # must vacate the single-flight lane for the replacement's work. - # Snapshot the identity outside the scheduler lock so the keep - # check stays lock-free below. - with self.lock: - published = getattr(self, "current_tip_first_seen", None) - snapshot = getattr(self, "tip_template_snapshot", None) - snapshot_tip = getattr(snapshot, "bestblockhash", None) - snapshot_fingerprint = getattr( - snapshot, "template_fingerprint", None - ) - if ( - published is not None - and snapshot_tip == published[0] - and snapshot_fingerprint is not None - and self._published_tip_authoritative_locked(time.monotonic()) - ): - published_parent = published[0] - published_fingerprint = snapshot_fingerprint + """Cancel only builds proven obsolete by a newer exact generation.""" + self._ensure_job_bundle_service()._cancel_obsolete_job_bundle_builds( + current_tip=current_tip, + payout_state_generation=payout_state_generation, + ) - def keep(request: _JobBuildRequest) -> bool: - return bool( - published_fingerprint is not None - and request.artifacts.previousblockhash == published_parent - and request.artifacts.fingerprint == published_fingerprint - ) + def _register_job_bundle_process( + self, + control: _JobBundleBuildControl, + process: subprocess.Popen[str], + ) -> None: + self._ensure_job_bundle_service()._register_job_bundle_process(control, process) - with self._job_build_scheduler_lock: - for flight in (self._job_build_active, self._job_build_retiring): - if flight is not None and not keep(flight.request): - self._cancel_job_build_flight_locked(flight, reason) - pending = self._job_build_pending - if pending is not None and not keep(pending): - pending.cancellation.cancel(reason) - if not pending.promise.done(): - pending.promise.set_exception( - JobBuildSuperseded(f"pending job build {reason}") - ) - self._job_build_pending = None - self.job_build_scheduler_counts["supersessions"] += 1 + def _unregister_job_bundle_process( + self, + control: _JobBundleBuildControl, + process: Any, + ) -> None: + """Detach a shared daemon from a build control after its request.""" + self._ensure_job_bundle_service()._unregister_job_bundle_process(control, process) - def shutdown_job_build_executor(self) -> None: - self._ensure_job_cache_state() - with self._job_build_scheduler_lock: - for flight in (self._job_build_active, self._job_build_retiring): - if flight is None: - continue - flight.request.cancellation.cancel("shutdown") - future = getattr(flight, "future", None) - if future is None or future.done(): - # A terminal future's callback has already run (or died) - # and executor shutdown will not fire it again: this is - # the last chance to settle the flight's waiters. - self._resolve_cancelled_job_build_promise( - flight.request, - "shutdown", - ) - pending = self._job_build_pending - if pending is not None: - pending.cancellation.cancel("shutdown") - if not pending.promise.done(): - pending.promise.set_exception( - JobBuildSuperseded("pending job build cancelled by shutdown") - ) - self._job_build_pending = None - executor = self._job_build_executor - self._job_build_executor = None - self._job_build_executor_shutdown = True - if executor is not None: - executor.shutdown(wait=True, cancel_futures=True) + @staticmethod + def _incremental_window_records_supported(records: Sequence[object]) -> bool: + """Whether a ledger snapshot exposes the append-only window contract.""" + return PayoutStateService._incremental_window_records_supported(records) - def _job_bundle_payout_state_current(self, bundle: CachedJobBundle) -> bool: - self._ensure_job_cache_state() - with self._job_cache_lock: - artifact = self._published_payout_state.artifact - return bool( - bundle.payout_state_generation == self._payout_state_generation - and bundle.build_key is not None - and ( - bundle.collection_only - or int( - getattr( - bundle.build_key, - "payout_append_invalidation_epoch", - 0, - ) - ) - == self._payout_ledger_append_invalidation_epoch - ) - and artifact is not None - and bundle.build_key.payout_artifact_sha256 - == artifact.prior_balances_sha256 - ) + def _full_payout_window_materialization(self, *, snapshot_anchor_ms: int, snapshot_window_weight: int, reason: str, observed_monotonic: float, append_invalidation_epoch: int, bypass_build_interval: bool=False) -> _PayoutWindowMaterialization: + """Run the exact ledger oracle and atomically replace cached pages.""" + return self._ensure_payout_state_service()._full_payout_window_materialization(snapshot_anchor_ms=snapshot_anchor_ms, snapshot_window_weight=snapshot_window_weight, reason=reason, observed_monotonic=observed_monotonic, append_invalidation_epoch=append_invalidation_epoch, bypass_build_interval=bypass_build_interval) - @contextmanager - def _payout_balance_mutation(self) -> Iterator[None]: - """Serialize durable balance changes without excluding delivery.""" - self._ensure_job_cache_state() - with self._block_submitter_lock( - self._payout_balance_mutation_lock, - "payout-balance-mutation", - ): - with self._accepted_block_payout_preview_condition: - landed_transition = any( - transition.landed - for transition in self._accepted_block_payout_previews.values() - ) - if landed_transition: - raise _PayoutStatePublicationBlocked( - "accepted block payout confirmation is still pending" - ) - yield + def _incremental_payout_window_materialization(self, *, snapshot_anchor_ms: int, snapshot_window_weight: int, force_full_rescan: bool, bypass_build_interval: bool, append_invalidation_epoch: int, reused_prior_balances_sha256: str | None=None) -> _PayoutWindowMaterialization: + """Return an exact window using debounce, delta folding, or the oracle.""" + return self._ensure_payout_state_service()._incremental_payout_window_materialization(snapshot_anchor_ms=snapshot_anchor_ms, snapshot_window_weight=snapshot_window_weight, force_full_rescan=force_full_rescan, bypass_build_interval=bypass_build_interval, append_invalidation_epoch=append_invalidation_epoch, reused_prior_balances_sha256=reused_prior_balances_sha256) - def _begin_accepted_block_payout_preview( - self, - block_hash: str, - *, - block_height: int | None = None, - ) -> None: - """Prevent child work from snapshotting pre-accept balances.""" - self._ensure_job_cache_state() - key = block_hash.lower() - with self._accepted_block_payout_preview_condition: - self._invalidated_accepted_block_payout_previews.pop(key, None) - existing = self._accepted_block_payout_previews.get(key) - if existing is None: - self._accepted_block_payout_previews[key] = ( - _AcceptedBlockPayoutTransition(block_height=block_height) - ) - elif ( - block_height is not None - and existing.block_height is not None - and existing.block_height != block_height - ): - raise RuntimeError("accepted block payout transition height changed") - elif existing.block_height is None and block_height is not None: - self._accepted_block_payout_previews[key] = dataclass_replace( - existing, - block_height=block_height, - ) + def _build_payout_ledger_artifact(self, expected_payout_state_generation: int, artifact_payout_state_generation: int, network_difficulty: int, force_full_rescan: bool=False, bypass_build_interval: bool=False, during_publication: bool=False, *, force_prior_balances_read: bool=False) -> PayoutLedgerArtifact | None: + """Build a stable ledger snapshot without publishing it.""" + return self._ensure_payout_state_service()._build_payout_ledger_artifact(expected_payout_state_generation, artifact_payout_state_generation, network_difficulty, force_full_rescan, bypass_build_interval, during_publication, force_prior_balances_read=force_prior_balances_read) - def _mark_accepted_block_payout_landed( - self, - block_hash: str, - *, - block_height: int, - ) -> None: - """Bar reconciliation after submitblock makes a candidate active.""" - self._ensure_job_cache_state() - key = block_hash.lower() - with self._accepted_block_payout_preview_condition: - existing = self._accepted_block_payout_previews.get( - key, - _AcceptedBlockPayoutTransition(block_height=block_height), - ) - if existing.block_height not in {None, block_height}: - raise RuntimeError("accepted block payout transition height changed") - self._accepted_block_payout_previews[key] = dataclass_replace( - existing, - block_height=block_height, - landed=True, - landed_monotonic=( - existing.landed_monotonic - if existing.landed_monotonic is not None - else time.monotonic() - ), - ) - self._accepted_block_payout_preview_condition.notify_all() + def _prepare_payout_ledger_artifact(self, payout_state_generation: int, network_difficulty: int, *, bypass_build_interval: bool=False) -> None: + """Prepare and atomically publish an artifact for a current generation.""" + return self._ensure_payout_state_service()._prepare_payout_ledger_artifact(payout_state_generation, network_difficulty, bypass_build_interval=bypass_build_interval) - def _unmark_accepted_block_payout_landed(self, block_hash: str) -> None: - """Withdraw a landed bar armed for an attempt that never reached RPC. - - Only sound while the submit outcome is NOT uncertain: the caller - must know submitblock was never invoked under this arming (a - preflight refusal such as ``WriterLeaseRenewalDeferred``), so there - is no possibly-active coinbase for the barrier to preserve and no - reason to keep reconciliation and payout-state publication barred - while the candidate waits for its retry. The pending preview entry - survives — child work keeps waiting instead of snapshotting - pre-accept balances, matching the startup-replay state a durable - candidate restores — and the next attempt re-arms ``landed`` before - its own RPC. - """ - self._ensure_job_cache_state() - key = block_hash.lower() - with self._accepted_block_payout_preview_condition: - existing = self._accepted_block_payout_previews.get(key) - if existing is None or not existing.landed: - return - self._accepted_block_payout_previews[key] = dataclass_replace( - existing, - landed=False, - landed_monotonic=None, - ) - self._accepted_block_payout_preview_condition.notify_all() + def _install_payout_ledger_artifact(self, artifact: PayoutLedgerArtifact) -> bool: + """Atomically publish a prepared artifact for its own generation.""" + return self._ensure_payout_state_service()._install_payout_ledger_artifact(artifact) - def _publish_accepted_block_payout_preview( - self, - block_hash: str, - balances: list[dict[str, object]], - ) -> list[dict[str, object]]: - """Publish the balances child work must observe after confirmation. + def _install_payout_ledger_artifact_outcome(self, artifact: PayoutLedgerArtifact) -> bool: + return self._ensure_payout_state_service()._install_payout_ledger_artifact_outcome(artifact) - Canonicalization happens before the delivery boundary. The gate only - installs an immutable pointer and advances the generation, so no job - bound to the pre-accept balances can land after publication. - """ - normalized = self.normalized_prior_balances(balances) - serialized = self._serialize_prior_balance_preview(normalized) - key = block_hash.lower() - with self._block_submitter_lock( - self._payout_balance_mutation_lock, - "payout-balance-mutation", - ): - with self._accepted_block_payout_preview_condition: - existing = self._accepted_block_payout_previews.get(key) - existing_preview = existing.preview if existing is not None else None - if existing_preview is not None: - if existing_preview != serialized: - raise RuntimeError( - "accepted block payout preview changed during retry" - ) - if ( - existing is not None - and existing.published_generation is not None - ): - return self._materialize_prior_balance_preview( - existing_preview - ) - # A bounded publication loss retains the compact preview - # locally while delivery remains fenced. Matching retries - # must still cross the atomic publication boundary so they - # install a generation and reopen admission. + def _install_payout_ledger_artifact_locked(self, artifact: PayoutLedgerArtifact) -> tuple[str, int | None]: + """Ordering core of the install; returns (outcome, armed generation).""" + return self._ensure_payout_state_service()._install_payout_ledger_artifact_locked(artifact) - published = None - try: - captured = self._capture_payout_state_source() - reserved = self._reserve_payout_state_source_if_current( - captured[1], - "accepted_block_preview", - tip_hash=key, - invalidated_monotonic=time.monotonic(), - ) - if reserved is None: - candidate = self._prepared_payout_state_candidate( - self._capture_payout_state_source(), - bypass_build_interval=True, - ) - else: - candidate = self._prepared_payout_state_candidate(reserved) - candidate = self._accepted_block_preview_candidate( - candidate, - block_hash=key, - preview=serialized, - ) - # Close delivery admission before exposing the preview. The - # generation publication below performs the only atomic pointer - # swap and reopens admission; no preparation lock is involved. - self._block_payout_state_publication(force=True) - published = self._publish_payout_state_candidate(candidate) - if published is None: - max_retries = max( - 0, - int( - getattr( - self, - "payout_reconcile_supersession_retries", - DEFAULT_PRISM_PAYOUT_RECONCILE_SUPERSESSION_RETRIES, - ) - ), - ) - for _attempt in range(max_retries): - candidate = self._accepted_block_preview_candidate( - self._prepared_payout_state_candidate( - self._capture_payout_state_source(), - bypass_build_interval=True, - ), - block_hash=key, - preview=serialized, - ) - published = self._publish_payout_state_candidate(candidate) - if published is not None: - break - except Exception as exc: - # Last-resort degraded retention (issue #188): candidate - # preparation already degrades a ledger-timeout window build - # to the cached armed window, so reaching here means an - # in-memory publication step itself failed. Fall through to - # the fenced local-retention branch below exactly like a - # lost atomic publication: admission is force-blocked, the - # immutable compact preview remains visible to children - # already waiting on the transition, and the tip-refresh - # retry loop must publish a fresh source before new job - # admission resumes. - published = None - print( - "prism coordinator: accepted block payout preview " - f"publication degraded hash={key}: {exc}", - flush=True, - ) - if published is None: - self._block_payout_state_publication(force=True) - # Admission is now globally fenced, so retaining the compact - # preview locally is safe even though it did not win an atomic - # generation publication. Finalization may finish; the normal - # retry loop will publish the newest source before delivery - # resumes. - with self._accepted_block_payout_preview_condition: - transition = self._accepted_block_payout_previews.get( - key, - _AcceptedBlockPayoutTransition(landed=True), - ) - self._accepted_block_payout_previews[key] = dataclass_replace( - transition, - landed=True, - preview=serialized, - published_generation=None, - ) - self._accepted_block_payout_preview_condition.notify_all() - return self._materialize_prior_balance_preview(serialized) + def _payout_artifact_preparation_loop(self) -> None: + return self._ensure_payout_state_service()._payout_artifact_preparation_loop() - def _accepted_block_preview_candidate( - self, - candidate: PayoutStateCandidate, - *, - block_hash: str, - preview: tuple[tuple[str, str, str, int], ...], - ) -> PayoutStateCandidate: - """Bind a compact preview to its prepared artifact before gating.""" - ledger_artifact = candidate.ledger_artifact - if ledger_artifact is not None: - # The artifact was prepared before accepted-block persistence, so - # replace only its balance view with the verified prospective - # state. This allocation must stay outside delivery publication. - # The memoized balances digest must be re-derived with the patch: - # a stale digest would fail the reuse probe's balances fence - # against the very state this patch installs. - patched_balances = tuple( - self._materialize_prior_balance_preview(preview) - ) - ledger_artifact = dataclass_replace( - ledger_artifact, - prior_balances=patched_balances, - prior_balances_sha256=canonical_json_sha256(patched_balances), - ) - return dataclass_replace( - candidate, - accepted_block_hash=block_hash, - accepted_block_preview=preview, - ledger_artifact=ledger_artifact, - ) + def _schedule_payout_ledger_artifact_preparation(self, payout_state_generation: int, network_difficulty: int, *, min_interval_seconds: float | None=None, bypass_build_interval: bool=False) -> None: + """Latest-generation-wins scheduling with one worker and one slot.""" + return self._ensure_payout_state_service()._schedule_payout_ledger_artifact_preparation(payout_state_generation, network_difficulty, min_interval_seconds=min_interval_seconds, bypass_build_interval=bypass_build_interval) - @staticmethod - def _serialize_prior_balance_preview( - balances: list[dict[str, object]], - ) -> tuple[tuple[str, str, str, int], ...]: - return tuple( - ( - str(balance["recipient_id"]), - str(balance["order_key"]), - str(balance["p2mr_program_hex"]), - int(balance["balance_sats"]), - ) - for balance in balances - ) + def _payout_artifact_max_anchor_age_ms(self) -> float: + """Audit ceiling on a served window's wall-clock anchor age.""" + return self._ensure_payout_state_service()._payout_artifact_max_anchor_age_ms() - def _accepted_block_payout_preview_from_bundle( - self, - final_bundle: dict[str, Any], - *, - prior_balances: list[dict[str, object]] | None = None, - ) -> list[dict[str, object]]: - """Derive the confirmed carry-forward view from a verified bundle.""" - manifest = final_bundle.get("payout_policy_manifest") - if not isinstance(manifest, dict) or not isinstance(manifest.get("accounts"), list): - raise RuntimeError("accepted block payout manifest is missing accounts") - prior_identities: dict[str, tuple[str, str]] = {} - for balance in prior_balances or []: - program = str(balance.get("p2mr_program_hex", "")).lower() - identity = ( - str(balance.get("order_key", "")), - str(balance.get("recipient_id", "")), - ) - prior_identities[program] = min( - identity, - prior_identities.get(program, identity), - ) - balances: list[dict[str, object]] = [] - for account in manifest["accounts"]: - if not isinstance(account, dict): - continue - if str(account.get("account_type", "miner")) == "pool_fee": - continue - balance_sats = int(account.get("carry_forward_balance_sats", 0)) - if balance_sats == 0: - continue - program = str(account.get("p2mr_program_hex", "")).lower() - account_identity = ( - str(account.get("order_key", "")), - str(account.get("recipient_id", "")), - ) - order_key, recipient_id = min( - account_identity, - prior_identities.get(program, account_identity), - ) - balances.append( - { - "recipient_id": recipient_id, - "order_key": order_key, - "p2mr_program_hex": program, - "balance_sats": balance_sats, - } - ) - return self.normalized_prior_balances(balances) + def _payout_artifact_reanchor_seconds(self) -> float: + """Anchor age that triggers a background re-anchor while reuse keeps serving.""" + return self._ensure_payout_state_service()._payout_artifact_reanchor_seconds() + + def _payout_artifact_min_build_interval_seconds(self) -> float: + """Minimum normal cadence; found-block settlement bypasses it.""" + return self._ensure_payout_state_service()._payout_artifact_min_build_interval_seconds() + + def _payout_artifact_reuse_active(self) -> bool: + """Master kill-switch (PRISM_PAYOUT_ARTIFACT_REUSE) for the reuse line.""" + return self._ensure_payout_state_service()._payout_artifact_reuse_active() @staticmethod - def _materialize_prior_balance_preview( - preview: tuple[tuple[str, str, str, int], ...], - ) -> list[dict[str, object]]: - return [ - { - "recipient_id": recipient_id, - "order_key": order_key, - "p2mr_program_hex": p2mr_program_hex, - "balance_sats": balance_sats, - } - for recipient_id, order_key, p2mr_program_hex, balance_sats in preview - ] + def _payout_artifact_log(event: str, **fields: object) -> None: + """Single-line JSON lifecycle log; the incident was invisible here.""" + return PayoutStateService._payout_artifact_log(event, **fields) + + def _record_payout_artifact_event(self, event: str) -> None: + return self._ensure_payout_state_service()._record_payout_artifact_event(event) - def _clear_accepted_block_payout_preview( + def _note_window_mirror_divergence(self) -> None: + """Invalidate a refuted daemon window mirror and count the event.""" + return self._ensure_payout_state_service()._note_window_mirror_divergence() + + def _usable_payout_ledger_artifact(self, payout_state_generation: int, network_difficulty: int, *, rearm_on_fence_failure: bool=True) -> PayoutLedgerArtifact | None: + """Return the armed artifact when reuse is valid for NEW work.""" + return self._ensure_payout_state_service()._usable_payout_ledger_artifact(payout_state_generation, network_difficulty, rearm_on_fence_failure=rearm_on_fence_failure) + + def _rearm_payout_ledger_artifact_after_fence_failure(self, payout_state_generation: int, network_difficulty: int) -> None: + """Debounced rebuild scheduling for an artifact past the re-anchor floor or ceiling.""" + return self._ensure_payout_state_service()._rearm_payout_ledger_artifact_after_fence_failure(payout_state_generation, network_difficulty) + + def _schedule_current_payout_ledger_artifact_if_missing(self) -> None: + """Resume speculative preparation after a durable preview catches up.""" + return self._ensure_payout_state_service()._schedule_current_payout_ledger_artifact_if_missing() + + def shutdown_payout_artifact_executor(self) -> None: + return self._ensure_payout_state_service().shutdown_payout_artifact_executor() + + def _job_build_checkpoint( self, - block_hash: str, - *, - invalidate_published: bool = False, + phase: str, + cancellation: _JobBuildCancellation, ) -> None: - self._ensure_job_cache_state() - key = block_hash.lower() - with self._block_submitter_lock( - self._payout_balance_mutation_lock, - "payout-balance-mutation", - ): - with self._accepted_block_payout_preview_condition: - existing = self._accepted_block_payout_previews.get(key) - if existing is None: - if not invalidate_published: - self._invalidated_accepted_block_payout_previews.pop( - key, - None, - ) - self._accepted_block_payout_preview_condition.notify_all() - return - if not invalidate_published: - # Durable state now equals the published prospective view; - # removing the override changes no logical payout state. - self._accepted_block_payout_previews.pop(key, None) - self._invalidated_accepted_block_payout_previews.pop(key, None) - self._accepted_block_payout_preview_condition.notify_all() - return - if existing.preview is None: - # Nothing crossed a generation boundary. A landed - # transition still becomes a tombstone so descendants - # cannot fall through to an uncertain database snapshot. - self._accepted_block_payout_previews.pop(key, None) - if existing.landed: - self._invalidated_accepted_block_payout_previews[key] = ( - existing.block_height - ) - self._accepted_block_payout_preview_condition.notify_all() - return + """Cooperative boundary around coordinator and isolated-worker phases.""" - captured = self._capture_payout_state_source() - reserved = self._reserve_payout_state_source_if_current( - captured[1], - "accepted_block_preview_withdrawn", - tip_hash=captured[2], - invalidated_monotonic=time.monotonic(), - ) - candidate = self._prepared_payout_state_candidate( - reserved if reserved is not None else self._capture_payout_state_source(), - force_full_window_rescan=True, - bypass_build_interval=True, - ) - candidate = dataclass_replace( - candidate, - accepted_block_hash=key, - accepted_block_withdrawal=True, - accepted_block_height=existing.block_height, - ) - self._block_payout_state_publication(force=True) - published = self._publish_payout_state_candidate(candidate) - if published is None: - max_retries = max( - 0, - int( - getattr( - self, - "payout_reconcile_supersession_retries", - DEFAULT_PRISM_PAYOUT_RECONCILE_SUPERSESSION_RETRIES, - ) - ), - ) - for _attempt in range(max_retries): - candidate = dataclass_replace( - self._prepared_payout_state_candidate( - self._capture_payout_state_source(), - force_full_window_rescan=True, - bypass_build_interval=True, - ), - accepted_block_hash=key, - accepted_block_withdrawal=True, - accepted_block_height=existing.block_height, - ) - published = self._publish_payout_state_candidate(candidate) - if published is not None: - break - if published is None: - # Delivery remains fenced. Install the tombstone immediately - # so even local builders cannot fall through while the newest - # source waits for a retry publication. - with self._accepted_block_payout_preview_condition: - self._accepted_block_payout_previews.pop(key, None) - self._invalidated_accepted_block_payout_previews[key] = ( - existing.block_height - ) - self._accepted_block_payout_preview_condition.notify_all() + cancellation.raise_if_cancelled(phase) + + def _record_job_cache_event(self, kind: str, *, hit: bool) -> None: + self._ensure_job_bundle_service()._record_job_cache_event(kind, hit=hit) + + def _prepare_payout_state_artifact(self, *, generation: int, source_generation: int, cancellation: _JobBuildCancellation | None=None) -> PayoutStateArtifact: + """Snapshot carry-forward balances once for a payout generation.""" + return self._ensure_payout_state_service()._prepare_payout_state_artifact(generation=generation, source_generation=source_generation, cancellation=cancellation) + + def _payout_state_artifact_from_balances(self, *, generation: int, source_generation: int, balances: list[dict[str, object]]) -> PayoutStateArtifact: + return self._ensure_payout_state_service()._payout_state_artifact_from_balances(generation=generation, source_generation=source_generation, balances=balances) + + def _publish_self_check_repaired_balances(self, expected_payout_state_generation: int, *, stale_prior_balances_sha256: str, balances: Sequence[dict[str, object]]) -> bool: + """Replace the published balance snapshot the periodic oracle refuted.""" + return self._ensure_payout_state_service()._publish_self_check_repaired_balances(expected_payout_state_generation, stale_prior_balances_sha256=stale_prior_balances_sha256, balances=balances) + + def _current_payout_state_artifact(self, cancellation: _JobBuildCancellation | None=None) -> PayoutStateArtifact: + """Return the immutable artifact published for the current generation.""" + return self._ensure_payout_state_service()._current_payout_state_artifact(cancellation) + + def _job_build_executor_locked(self) -> ThreadPoolExecutor: + return self._ensure_job_bundle_service()._job_build_executor_locked() + + def _start_job_build_locked(self, request: _JobBuildRequest) -> _JobBuildFlight: + return self._ensure_job_bundle_service()._start_job_build_locked(request) - def _accepted_block_payout_transition_landed(self, block_hash: str) -> bool: - self._ensure_job_cache_state() - with self._accepted_block_payout_preview_condition: - transition = self._accepted_block_payout_previews.get(block_hash.lower()) - return transition is not None and transition.landed + def _arm_job_build_locked(self, flight: _JobBuildFlight) -> None: + self._ensure_job_bundle_service()._arm_job_build_locked(flight) - def _accepted_block_payout_transition_for_parent( + def _execute_job_build_request( self, - parent_hash: str, - *, - parent_height: int | None = None, - ) -> tuple[str, bool] | None: - """Select the highest active exact/ancestor payout transition. + request: _JobBuildRequest, + ) -> CachedJobBundle: + return self._ensure_job_bundle_service()._execute_job_build_request(request) - The boolean result is true for an invalidated landed transition. The - caller decides whether to wait for a live preview or fail closed on its - tombstone. - """ - self._ensure_job_cache_state() - key = parent_hash.lower() - with self._accepted_block_payout_preview_condition: - exact_transition = self._accepted_block_payout_previews.get(key) - exact_invalidated = ( - key in self._invalidated_accepted_block_payout_previews - ) - fail_closed_candidate_hashes = { - candidate_hash - for candidate_hash, transition in ( - self._accepted_block_payout_previews.items() - ) - if transition.landed - } - fail_closed_candidate_hashes.update( - self._invalidated_accepted_block_payout_previews - ) - ancestor_candidates = [ - (candidate_hash, transition.block_height, False) - for candidate_hash, transition in self._accepted_block_payout_previews.items() - if exact_transition is None - and not exact_invalidated - and transition.block_height is not None - and parent_height is not None - and transition.block_height <= parent_height - ] - ancestor_candidates.extend( - ( - candidate_hash, - candidate_height, - True, - ) - for candidate_hash, candidate_height in ( - self._invalidated_accepted_block_payout_previews.items() - ) - if exact_transition is None - and not exact_invalidated - and candidate_height is not None - and parent_height is not None - and candidate_height <= parent_height - ) - if exact_transition is not None or exact_invalidated: - return key, exact_invalidated - if not ancestor_candidates: - return None + _collection_job_builds_are_independent = staticmethod( + JobBundleService._collection_job_builds_are_independent + ) - active_ancestors: list[tuple[int, str, bool]] = [] - try: - for ( - candidate_hash, - candidate_height, - candidate_invalidated, - ) in ancestor_candidates: - assert candidate_height is not None - active_hash = str( - self.rpc.call("getblockhash", [candidate_height]) - ).lower() - if active_hash == candidate_hash: - active_ancestors.append( - ( - candidate_height, - candidate_hash, - candidate_invalidated, - ) - ) - except Exception as exc: - self._schedule_tip_refresh_retry() - raise TemplateRefreshBlocked( - "could not validate an accepted payout preview on the active chain" - ) from exc - if not active_ancestors: - if any( - candidate_hash in fail_closed_candidate_hashes - for candidate_hash, _candidate_height, _invalidated in ( - ancestor_candidates - ) - ): - # A prepared artifact published with an accepted preview carries - # that prospective balance view. If its block left the active - # chain between reconciliation and this build, falling through - # to the artifact would stamp unrelated work with those balances. - # Wait for withdrawal/reconciliation to publish a new payout - # generation instead of using either the artifact or live DB. - self._schedule_tip_refresh_retry() - raise _PayoutStatePublicationBlocked( - "accepted payout transition is no longer active" - ) - return None - _, selected_key, selected_invalidated = max(active_ancestors) - return selected_key, selected_invalidated + _job_build_requests_can_share = staticmethod( + JobBundleService._job_build_requests_can_share + ) - def _accepted_parent_unresolved_depth_cap(self) -> int: - return max( - 1, - int( - getattr( - self, - "accepted_parent_unresolved_depth_max", - DEFAULT_ACCEPTED_PARENT_UNRESOLVED_DEPTH_MAX, - ) - ), - ) + _ready_job_build_precedes_collection = staticmethod( + JobBundleService._ready_job_build_precedes_collection + ) - def accepted_parent_unresolved_ages_seconds(self) -> list[float]: - """Ages of landed transitions whose bookkeeping is still unresolved.""" - self._ensure_job_cache_state() - now = time.monotonic() - with self._accepted_block_payout_preview_condition: - return [ - max(0.0, now - transition.landed_monotonic) - for transition in self._accepted_block_payout_previews.values() - if transition.landed and transition.landed_monotonic is not None - ] + _defer_job_build_locked = staticmethod( + JobBundleService._defer_job_build_locked + ) - def _accepted_parent_unresolved_depth(self) -> int: - self._ensure_job_cache_state() - with self._accepted_block_payout_preview_condition: - return sum( - 1 - for transition in self._accepted_block_payout_previews.values() - if transition.landed - ) + _job_build_is_publication_critical = staticmethod( + JobBundleService._job_build_is_publication_critical + ) + + _job_build_promise_done = staticmethod( + JobBundleService._job_build_promise_done + ) - def _await_pending_parent_payout_preview( + def _record_priority_admission_locked( self, - parent_hash: str, - *, - parent_height: int | None = None, - ) -> list[dict[str, object]] | None: - """Wait out a pending accepted-parent transition, returning its preview. - - Returns None when no landed transition governs this parent, without - touching any confirmed payout input. Ordering this wait before every - confirmed read lets children of a pending parent block here instead of - taking (or caching) balances that omit their new parent. - """ - # Startup-replay window (issue #188 fix 4): before the durable outbox - # has been enumerated, an unarmed pending parent may exist. A child - # built now could pay balances that omit that parent's carry, so job - # builds fail closed until enumeration succeeds. - if self._block_replay_enumeration_owed(): - self._schedule_tip_refresh_retry() - raise TemplateRefreshBlocked( - "durable block candidate replay has not enumerated pending " - "candidates yet" - ) - selected = self._accepted_block_payout_transition_for_parent( - parent_hash, - parent_height=parent_height, - ) - if selected is None: - return None - # Fail closed once too many landed transitions remain unresolved: - # every published preview a child consumes stacks another prospective - # balance chain on unfinished bookkeeping, so the depth of that stack - # must stay bounded even while degraded previews keep jobs flowing. - # Issuance stops at the cap, not past it: a child issued while - # exactly depth_cap transitions are unresolved could land and create - # a (cap + 1)th, so the configured maximum would not actually bound - # the chain. - unresolved_depth = self._accepted_parent_unresolved_depth() - depth_cap = self._accepted_parent_unresolved_depth_cap() - if unresolved_depth >= depth_cap: - self._schedule_tip_refresh_retry() - raise TemplateRefreshBlocked( - "unresolved accepted-parent depth " - f"{unresolved_depth} meets or exceeds cap {depth_cap}" - ) - selected_key, selected_invalidated = selected - if selected_invalidated: - self._schedule_tip_refresh_retry() - raise TemplateRefreshBlocked( - "accepted parent payout preview was withdrawn" - ) + request: _JobBuildRequest, + result: str, + ) -> None: + """Observe first builder admission from publication-priority reservation.""" + self._ensure_job_bundle_service()._record_priority_admission_locked(request, result) - wait_seconds = max( - 0.0, - float( - getattr( - self, - "accepted_block_payout_preview_wait_seconds", - DEFAULT_ACCEPTED_BLOCK_PAYOUT_PREVIEW_WAIT_SECONDS, - ) - ), - ) - deadline = time.monotonic() + wait_seconds - timed_out = False - invalidated = False - with self._accepted_block_payout_preview_condition: - while selected_key in self._accepted_block_payout_previews: - transition = self._accepted_block_payout_previews[selected_key] - if transition.preview is not None: - return self._materialize_prior_balance_preview( - transition.preview - ) - if self.stop_event.is_set(): - raise RuntimeError( - "coordinator stopped while accepted payout preview was pending" - ) - remaining = deadline - time.monotonic() - if remaining <= 0: - timed_out = True - break - self._accepted_block_payout_preview_condition.wait( - timeout=min(0.25, remaining) - ) - invalidated = ( - selected_key in self._invalidated_accepted_block_payout_previews - ) - if invalidated: - self._schedule_tip_refresh_retry() - raise TemplateRefreshBlocked( - "accepted parent payout preview was withdrawn" - ) - if timed_out: - with self.lock: - self._accepted_parent_preview_wait_timeouts = ( - int(getattr(self, "_accepted_parent_preview_wait_timeouts", 0)) - + 1 - ) - self._schedule_tip_refresh_retry() - raise TemplateRefreshBlocked( - "accepted parent payout preview is not ready yet" - ) - # The transition reached a terminal durable state while waiting; the - # caller's confirmed fallback now includes the parent. - return None + def _record_initial_prepared_work_locked(self, result: str) -> None: + self._ensure_job_bundle_service()._record_initial_prepared_work_locked(result) - def _prior_balances_for_job_parent( - self, - parent_hash: str, - *, - parent_height: int | None = None, - fallback_balances: Sequence[dict[str, object]] | None = None, - ) -> list[dict[str, object]]: - """Return prospective balances, otherwise a prepared/ledger fallback.""" - preview = self._await_pending_parent_payout_preview( - parent_hash, - parent_height=parent_height, - ) - if preview is not None: - return preview - return ( - list(fallback_balances) - if fallback_balances is not None - else self.ledger.current_prior_balances() - ) + def _new_job_build_cancellation(self) -> _JobBuildCancellation: + return self._ensure_job_bundle_service()._new_job_build_cancellation() - def _observe_payout_state_seconds( + def _begin_job_build_priority_preparation( self, - name: str, - elapsed_seconds: float, - *, - relation: str | None = None, - ) -> None: - self._ensure_job_cache_state() - with self._payout_state_metrics_lock: - if name == "gate_wait": - if relation not in PRISM_PAYOUT_DELIVERY_GENERATIONS: - raise ValueError(f"unknown payout delivery generation: {relation}") - histogram = self.payout_gate_wait_histograms[str(relation)] - else: - histogram = self.payout_state_histograms[name] - histogram["count"] = int(histogram["count"]) + 1 - histogram["sum"] = float(histogram["sum"]) + elapsed_seconds - buckets = histogram["buckets"] - assert isinstance(buckets, dict) - for bucket in PRISM_TIP_REFRESH_SECONDS_BUCKETS: - if elapsed_seconds <= bucket: - buckets[bucket] = int(buckets.get(bucket, 0)) + 1 - - def _observe_payout_gate_admission( + requested_monotonic: float | None = None, + ) -> tuple[int, float]: + """Reserve publication priority before immutable request construction.""" + return self._ensure_job_bundle_service()._begin_job_build_priority_preparation(requested_monotonic) + + def _finish_job_build_priority_preparation(self, token: int) -> None: + self._ensure_job_bundle_service()._finish_job_build_priority_preparation(token) + + def _begin_routine_job_build_preparation( self, - admission: object, *, - generation: int, - fallback_wait_seconds: float, - ) -> None: - self._ensure_job_cache_state() - with self._job_cache_lock: - published_generation = self._payout_state_generation - relation = getattr(admission, "relation", None) - if relation not in PRISM_PAYOUT_DELIVERY_GENERATIONS: - relation = _PayoutStateDeliveryGate._generation_relation( - generation, - published_generation, - ) - wait_seconds = float( - getattr(admission, "wait_seconds", fallback_wait_seconds) - ) - self._observe_payout_state_seconds( - "gate_wait", - max(0.0, wait_seconds), - relation=relation, + request_source: str, + cancelled: Callable[[], bool] | None, + ) -> tuple[int, _JobBuildCancellation, CachedJobBundle | None]: + """Atomically admit cancellable routine request construction.""" + return self._ensure_job_bundle_service()._begin_routine_job_build_preparation( + request_source=request_source, + cancelled=cancelled, ) - def _reserve_payout_state_source( + def _publication_priority_promises_locked( self, - cause: str, - *, - tip_hash: str | None = None, - invalidated_monotonic: float | None = None, - ) -> int: - self._ensure_job_cache_state() - invalidated = ( - time.monotonic() - if invalidated_monotonic is None - else invalidated_monotonic + ) -> tuple[Future[CachedJobBundle], ...]: + """Promises of live publication-critical build requests.""" + return self._ensure_job_bundle_service()._publication_priority_promises_locked() + + def _finish_routine_job_build_preparation(self, token: int) -> None: + self._ensure_job_bundle_service()._finish_routine_job_build_preparation(token) + + def _publication_priority_scheduled_locked(self) -> bool: + return self._ensure_job_bundle_service()._publication_priority_scheduled_locked() + + def _job_build_can_inherit_publication_priority( + self, + existing: _JobBuildRequest, + incoming: _JobBuildRequest, + ) -> bool: + """Reject an almost-expired or stalled routine flight as a critical owner.""" + return self._ensure_job_bundle_service()._job_build_can_inherit_publication_priority( + existing, + incoming, ) - with self.lock: - generation = self._payout_state_source[0] + 1 - self._payout_state_source = ( - generation, - tip_hash, - cause, - invalidated, - ) - return generation - def _reserve_payout_state_source_if_current( + _resolve_cancelled_job_build_promise = staticmethod( + JobBundleService._resolve_cancelled_job_build_promise + ) + + _job_build_flight_outcome = staticmethod( + JobBundleService._job_build_flight_outcome + ) + + def _evict_orphaned_job_build_flights_locked(self) -> list[str]: + """Evict finished flights whose completion never released their slot.""" + return self._ensure_job_bundle_service()._evict_orphaned_job_build_flights_locked() + + def _cancel_job_build_flight_locked( self, - expected_source_generation: int, - cause: str, + flight: _JobBuildFlight, + reason: str, *, - tip_hash: str | None = None, - invalidated_monotonic: float | None = None, - ) -> tuple[int, int, str | None, str, float] | None: - """Reserve and capture a source only if preparation was not superseded.""" + now: float | None = None, + ) -> bool: + return self._ensure_job_bundle_service()._cancel_job_build_flight_locked(flight, reason, now=now) - self._ensure_job_cache_state() - invalidated = ( - time.monotonic() - if invalidated_monotonic is None - else invalidated_monotonic - ) - # Match publication's lock order so the returned base generation and - # newly reserved source form one atomic candidate identity. - with self._job_cache_lock: - with self.lock: - if self._payout_state_source[0] != expected_source_generation: - return None - source_generation = expected_source_generation + 1 - self._payout_state_source = ( - source_generation, - tip_hash, - cause, - invalidated, - ) - return ( - self._payout_state_generation, - source_generation, - tip_hash, - cause, - invalidated, - ) + def _promote_pending_job_build_locked(self) -> None: + self._ensure_job_bundle_service()._promote_pending_job_build_locked() - def _capture_payout_state_source( + def _job_build_done( self, - ) -> tuple[int, int, str | None, str, float]: - self._ensure_job_cache_state() - with self.lock: - source_generation, source_tip, cause, invalidated = ( - self._payout_state_source - ) - with self._job_cache_lock: - base_generation = self._payout_state_generation - return ( - base_generation, - source_generation, - source_tip, - cause, - invalidated, - ) + flight: _JobBuildFlight, + future: Future[CachedJobBundle], + ) -> None: + self._ensure_job_bundle_service()._job_build_done(flight, future) + + def _request_job_build(self, request: _JobBuildRequest) -> Future[CachedJobBundle]: + return self._ensure_job_bundle_service()._request_job_build(request) - def _prepared_payout_state_candidate( + def _cancel_obsolete_job_builds( self, - captured: tuple[int, int, str | None, str, float], + reason: str, *, - force_full_window_rescan: bool = False, - bypass_build_interval: bool = False, - ) -> PayoutStateCandidate: - base_generation, source_generation, source_tip, cause, invalidated = captured - ledger_artifact: PayoutLedgerArtifact | None = None - with self._job_cache_lock: - template_artifacts = self._template_artifacts - if ( - template_artifacts is not None - and getattr(self, "_pool_ready_latched", False) - ): - force_full = ( - force_full_window_rescan - or cause == "accepted_block_preview_withdrawn" - ) - bypass_interval = ( - bypass_build_interval - or cause - in { - "accepted_block_preview", - "accepted_block_preview_withdrawn", - } - ) - immediate_preview = ( - cause == "accepted_block_preview" - or ( - bypass_build_interval - and not force_full_window_rescan - and cause != "accepted_block_preview_withdrawn" - ) - ) - prepare_lock_acquired = True - if immediate_preview: - # A routine periodic oracle may already own this lock and its - # Postgres query cannot be cancelled safely. Found-block - # publication must not queue behind that 11--36 second scan: - # use the last exact, still-valid armed window and patch its - # balances with the verified preview below. - prepare_lock_acquired = self._payout_state_prepare_lock.acquire( - blocking=False - ) - if prepare_lock_acquired: - try: - ledger_artifact = self._build_payout_ledger_artifact( - base_generation, - base_generation + 1, - template_artifacts.network_difficulty, - force_full, - bypass_interval, - True, - ) - except ShutdownInProgress: - raise - except Exception as exc: - if not immediate_preview: - raise - # Found-block publication exists to decouple child job - # issuance from slow landing bookkeeping (issue #188). A - # window build that dies with the ledger must not abort - # the atomic publication that reopens admission: degrade - # to the cached armed window below, exactly like a busy - # prepare lock. The preview itself never depends on this - # build; it is the verified balance state either way. - print( - "prism coordinator: found-block payout window build " - f"failed; degrading to cached armed window: {exc}", - flush=True, - ) - ledger_artifact = None - finally: - if immediate_preview: - self._payout_state_prepare_lock.release() - else: - ledger_artifact = None - if immediate_preview and ledger_artifact is None: - ledger_artifact = self._cached_found_block_payout_artifact( - base_generation=base_generation, - artifact_payout_state_generation=base_generation + 1, - network_difficulty=template_artifacts.network_difficulty, - fallback_reason=( - "build_failed" - if prepare_lock_acquired - else "prepare_lock_busy" - ), - ) - return PayoutStateCandidate( - base_generation=base_generation, - source_generation=source_generation, - source_tip_hash=source_tip, - cause=cause, - invalidated_monotonic=invalidated, - prepared_monotonic=time.monotonic(), - ledger_artifact=ledger_artifact, + keep_published_snapshot: bool = False, + ) -> None: + self._ensure_job_bundle_service()._cancel_obsolete_job_builds( + reason, + keep_published_snapshot=keep_published_snapshot, ) - def _cached_found_block_payout_artifact( - self, - *, - base_generation: int, - artifact_payout_state_generation: int, - network_difficulty: int, - fallback_reason: str, - ) -> PayoutLedgerArtifact | None: - """Retag an exact armed window when immediate preparation is unavailable. + def shutdown_job_build_executor(self) -> None: + self._ensure_job_bundle_service().shutdown_job_build_executor() - The caller always replaces its balance view with the accepted block's - verified prospective balances before publication. Only the share - window and its declared anchor are reused here. - """ + def _job_bundle_payout_state_current(self, bundle: CachedJobBundle) -> bool: + return self._ensure_job_bundle_service()._job_bundle_payout_state_current(bundle) - with self._job_cache_lock: - artifact = self._payout_ledger_artifact - generation_current = base_generation == self._payout_state_generation - append_epoch_current = bool( - artifact is not None - and artifact.append_invalidation_epoch - == self._payout_ledger_append_invalidation_epoch - ) - anchor_age_ms = ( - None - if artifact is None or artifact.snapshot_anchor_ms is None - else now_ms() - int(artifact.snapshot_anchor_ms) - ) - if ( - not generation_current - or not append_epoch_current - or artifact is None - or artifact.payout_state_generation != base_generation - or artifact.network_difficulty != int(network_difficulty) - or anchor_age_ms is None - or anchor_age_ms > self._payout_artifact_max_anchor_age_ms() - ): - self._record_payout_artifact_event("build_aborted") - self._payout_artifact_log( - "payout_artifact_build_aborted", - payout_state_generation=int(artifact_payout_state_generation), - duration_seconds=0.0, - during_publication=True, - build_interval_bypassed=True, - abort_reason=f"{fallback_reason}_without_usable_window", - ) - return None - reused = dataclass_replace( - artifact, - generation=0, - payout_state_generation=int(artifact_payout_state_generation), - prepared_monotonic=time.monotonic(), - window_build_mode="found_block_cached", - window_delta_rows=0, - window_expired_rows=0, - window_touched_pages=0, - window_build_seconds=0.0, - window_full_rescan_reason=fallback_reason, - ) - self._record_payout_artifact_event("found_block_cached") - self._payout_artifact_log( - "payout_artifact_found_block_cached", - payout_state_generation=int(artifact_payout_state_generation), - duration_seconds=0.0, - window_build_mode="found_block_cached", - window_shares=len(reused.shares_json), - anchor_age_ms=anchor_age_ms, - during_publication=True, - build_interval_bypassed=True, - fallback_reason=fallback_reason, - ) - return reused + def _payout_balance_mutation(self) -> Iterator[None]: + """Serialize durable balance changes without excluding delivery.""" + return self._ensure_payout_state_service()._payout_balance_mutation() - def _current_payout_state_candidate(self) -> PayoutStateCandidate: - return self._prepared_payout_state_candidate( - self._capture_payout_state_source() - ) + def _begin_accepted_block_payout_preview(self, block_hash: str, *, block_height: int | None=None) -> None: + """Prevent child work from snapshotting pre-accept balances.""" + return self._ensure_payout_state_service()._begin_accepted_block_payout_preview(block_hash, block_height=block_height) - def _record_discarded_payout_candidate(self) -> None: - self._ensure_job_cache_state() - with self._payout_state_metrics_lock: - self.payout_state_candidates_discarded += 1 + def _mark_accepted_block_payout_landed(self, block_hash: str, *, block_height: int) -> None: + """Bar reconciliation after submitblock makes a candidate active.""" + return self._ensure_payout_state_service()._mark_accepted_block_payout_landed(block_hash, block_height=block_height) - def _block_payout_state_publication( - self, - *, - force: bool = False, - supersede_with: tuple[int, str | None, str, float] | None = None, - ) -> None: - """Atomically close delivery, optionally reserving a newer source.""" + def _unmark_accepted_block_payout_landed(self, block_hash: str) -> None: + """Withdraw a landed bar armed for an attempt that never reached RPC.""" + return self._ensure_payout_state_service()._unmark_accepted_block_payout_landed(block_hash) - self._ensure_job_cache_state() + def _publish_accepted_block_payout_preview(self, block_hash: str, balances: list[dict[str, object]]) -> list[dict[str, object]]: + """Publish the balances child work must observe after confirmation.""" + return self._ensure_payout_state_service()._publish_accepted_block_payout_preview(block_hash, balances) - pending_source: int | None = None - - def mark_blocked() -> bool: - nonlocal pending_source - with self._job_cache_lock: - with self.lock: - if supersede_with is not None: - ( - expected_source, - fallback_tip, - cause, - invalidated, - ) = supersede_with - current_source, current_tip, _, _ = ( - self._payout_state_source - ) - # A newer tip/source wins its identity, but it must be - # superseded so no candidate prepared before an - # uncertain durable commit can publish afterward. - source_tip = ( - fallback_tip - if current_source == expected_source - else current_tip - ) - pending_source = current_source + 1 - self._payout_state_source = ( - pending_source, - source_tip, - cause, - invalidated, - ) - else: - pending_source = self._payout_state_source[0] - if ( - not force - and supersede_with is None - and pending_source - == self._published_payout_state.source_generation - ): - return False - self._payout_state_publication_blocked = True - self._job_bundle_cache.clear() - return True - - # Close fleet admission atomically with the cache fence. Escaped - # immutable bundles remain stamped with the old generation, but cannot - # cross the boundary while the ledger has newer unpublished state. - if not self._payout_state_delivery_gate.block_delivery(mark_blocked): - return - # Construction cancellation is independent of fanout activation. An - # obsolete builder may still be in its ledger or subprocess phase and - # must be preempted before a replacement request is submitted. - self._cancel_obsolete_job_builds("payout generation superseded") - with self._job_cache_lock: - next_payout_generation = self._payout_state_generation + 1 - with self.lock: - payout_invalidated_monotonic = self._payout_state_source[3] - self._record_progress_payout_generation( - next_payout_generation, - payout_invalidated_monotonic, - ) - with self.lock: - active = getattr(self, "_active_tip_refresh", None) - if ( - active is not None - and active[0].payout_state_generation < next_payout_generation - ): - active[1].cancel() - elif active is not None: - return - assert pending_source is not None - self._mark_tip_refresh_pending(next_payout_generation) - self._schedule_tip_refresh_retry() + def _accepted_block_preview_candidate(self, candidate: PayoutStateCandidate, *, block_hash: str, preview: tuple[tuple[str, str, str, int], ...]) -> PayoutStateCandidate: + """Bind a compact preview to its prepared artifact before gating.""" + return self._ensure_payout_state_service()._accepted_block_preview_candidate(candidate, block_hash=block_hash, preview=preview) + + @staticmethod + def _serialize_prior_balance_preview(balances: list[dict[str, object]]) -> tuple[tuple[str, str, str, int], ...]: + return PayoutStateService._serialize_prior_balance_preview(balances) + + def _accepted_block_payout_preview_from_bundle(self, final_bundle: dict[str, Any], *, prior_balances: list[dict[str, object]] | None=None) -> list[dict[str, object]]: + """Derive the confirmed carry-forward view from a verified bundle.""" + return self._ensure_payout_state_service()._accepted_block_payout_preview_from_bundle(final_bundle, prior_balances=prior_balances) + + @staticmethod + def _materialize_prior_balance_preview(preview: tuple[tuple[str, str, str, int], ...]) -> list[dict[str, object]]: + return PayoutStateService._materialize_prior_balance_preview(preview) + + def _clear_accepted_block_payout_preview(self, block_hash: str, *, invalidate_published: bool=False) -> None: + return self._ensure_payout_state_service()._clear_accepted_block_payout_preview(block_hash, invalidate_published=invalidate_published) + + def _accepted_block_payout_transition_landed(self, block_hash: str) -> bool: + return self._ensure_payout_state_service()._accepted_block_payout_transition_landed(block_hash) + + def _accepted_block_payout_transition_for_parent(self, parent_hash: str, *, parent_height: int | None=None) -> tuple[str, bool] | None: + """Select the highest active exact/ancestor payout transition.""" + return self._ensure_payout_state_service()._accepted_block_payout_transition_for_parent(parent_hash, parent_height=parent_height) + + def _accepted_parent_unresolved_depth_cap(self) -> int: + return self._ensure_payout_state_service()._accepted_parent_unresolved_depth_cap() + + def accepted_parent_unresolved_ages_seconds(self) -> list[float]: + """Ages of landed transitions whose bookkeeping is still unresolved.""" + return self._ensure_payout_state_service().accepted_parent_unresolved_ages_seconds() + + def _accepted_parent_unresolved_depth(self) -> int: + return self._ensure_payout_state_service()._accepted_parent_unresolved_depth() + + def _await_pending_parent_payout_preview(self, parent_hash: str, *, parent_height: int | None=None) -> list[dict[str, object]] | None: + """Wait out a pending accepted-parent transition, returning its preview.""" + return self._ensure_payout_state_service()._await_pending_parent_payout_preview(parent_hash, parent_height=parent_height) + + def _prior_balances_for_job_parent(self, parent_hash: str, *, parent_height: int | None=None, fallback_balances: Sequence[dict[str, object]] | None=None) -> list[dict[str, object]]: + """Return prospective balances, otherwise a prepared/ledger fallback.""" + return self._ensure_payout_state_service()._prior_balances_for_job_parent(parent_hash, parent_height=parent_height, fallback_balances=fallback_balances) + + def _observe_payout_state_seconds(self, name: str, elapsed_seconds: float, *, relation: str | None=None) -> None: + return self._ensure_payout_state_service()._observe_payout_state_seconds(name, elapsed_seconds, relation=relation) + + def _observe_payout_gate_admission(self, admission: object, *, generation: int, fallback_wait_seconds: float) -> None: + return self._ensure_payout_state_service()._observe_payout_gate_admission(admission, generation=generation, fallback_wait_seconds=fallback_wait_seconds) + + def _reserve_payout_state_source(self, cause: str, *, tip_hash: str | None=None, invalidated_monotonic: float | None=None) -> int: + return self._ensure_payout_state_service()._reserve_payout_state_source(cause, tip_hash=tip_hash, invalidated_monotonic=invalidated_monotonic) + + def _reserve_payout_state_source_if_current(self, expected_source_generation: int, cause: str, *, tip_hash: str | None=None, invalidated_monotonic: float | None=None) -> tuple[int, int, str | None, str, float] | None: + """Reserve and capture a source only if preparation was not superseded.""" + return self._ensure_payout_state_service()._reserve_payout_state_source_if_current(expected_source_generation, cause, tip_hash=tip_hash, invalidated_monotonic=invalidated_monotonic) + + def _capture_payout_state_source(self) -> tuple[int, int, str | None, str, float]: + return self._ensure_payout_state_service()._capture_payout_state_source() - def _payout_state_publication_fenced(self) -> bool: - """Report whether delivery is still blocked awaiting a publication.""" + def _prepared_payout_state_candidate(self, captured: tuple[int, int, str | None, str, float], *, force_full_window_rescan: bool=False, force_prior_balances_read: bool=False, bypass_build_interval: bool=False) -> PayoutStateCandidate: + return self._ensure_payout_state_service()._prepared_payout_state_candidate(captured, force_full_window_rescan=force_full_window_rescan, force_prior_balances_read=force_prior_balances_read, bypass_build_interval=bypass_build_interval) - self._ensure_job_cache_state() - with self._job_cache_lock: - return self._payout_state_publication_blocked + def _cached_found_block_payout_artifact(self, *, base_generation: int, artifact_payout_state_generation: int, network_difficulty: int, fallback_reason: str) -> PayoutLedgerArtifact | None: + """Retag an exact armed window when immediate preparation is unavailable.""" + return self._ensure_payout_state_service()._cached_found_block_payout_artifact(base_generation=base_generation, artifact_payout_state_generation=artifact_payout_state_generation, network_difficulty=network_difficulty, fallback_reason=fallback_reason) - def _payout_source_requires_publication( - self, - candidate: PayoutStateCandidate | None = None, - ) -> bool: - """Report whether an invalidation source still lacks a publication.""" + def _current_payout_state_candidate(self) -> PayoutStateCandidate: + return self._ensure_payout_state_service()._current_payout_state_candidate() - self._ensure_job_cache_state() - with self._job_cache_lock: - published_source = self._published_payout_state.source_generation - if candidate is not None: - return candidate.source_generation != published_source - with self.lock: - return self._payout_state_source[0] != published_source + def _record_discarded_payout_candidate(self) -> None: + return self._ensure_payout_state_service()._record_discarded_payout_candidate() - def _captured_payout_source_requires_publication( - self, - captured: tuple[int, int, str | None, str, float], - ) -> bool: - """Publication check for a captured source, without a candidate. + def _block_payout_state_publication(self, *, force: bool=False, supersede_with: tuple[int, str | None, str, float] | None=None) -> None: + """Atomically close delivery, optionally reserving a newer source.""" + return self._ensure_payout_state_service()._block_payout_state_publication(force=force, supersede_with=supersede_with) - Answers exactly what _payout_source_requires_publication would answer - for a candidate prepared from ``captured``. Candidate preparation can - include a multi-second ledger snapshot, so decision sites must be able - to ask this first and prepare only when publication will follow. - """ + def _payout_state_publication_fenced(self) -> bool: + """Report whether delivery is still blocked awaiting a publication.""" + return self._ensure_payout_state_service()._payout_state_publication_fenced() - self._ensure_job_cache_state() - with self._job_cache_lock: - return captured[1] != self._published_payout_state.source_generation + def _payout_source_requires_publication(self, candidate: PayoutStateCandidate | None=None) -> bool: + """Report whether an invalidation source still lacks a publication.""" + return self._ensure_payout_state_service()._payout_source_requires_publication(candidate) - def _publish_payout_state_candidate( - self, - candidate: PayoutStateCandidate, - ) -> int | None: - """Publish a prepared candidate, or reject it if its source moved.""" + def _captured_payout_source_requires_publication(self, captured: tuple[int, int, str | None, str, float]) -> bool: + """Publication check for a captured source, without a candidate.""" + return self._ensure_payout_state_service()._captured_payout_source_requires_publication(captured) - self._ensure_job_cache_state() - self._ensure_tip_refresh_state() - published_generation: int | None = None - schedule_retry = False - active_to_cancel: _FanoutCancellation | None = None - publication_born_expired: tuple[int | None, int] | None = None - publication_installed: tuple[int, int, int] | None = None - publish_started = 0.0 - with self._job_cache_lock: - with self.lock: - if ( - candidate.source_generation != self._payout_state_source[0] - or candidate.base_generation != self._payout_state_generation - ): - self._record_discarded_payout_candidate() - return None - try: - if ( - candidate.accepted_block_preview is not None - and not candidate.accepted_block_withdrawal - ): - # The preview IS the balance state this generation publishes. - # The durable ledger only catches up after persistence, so a - # ledger read here would cache pre-parent balances against the - # post-parent generation and outlive the transition override. - artifact = self._payout_state_artifact_from_balances( - generation=candidate.base_generation + 1, - source_generation=candidate.source_generation, - balances=self._materialize_prior_balance_preview( - candidate.accepted_block_preview - ), - ) - elif candidate.ledger_artifact is not None: - # Candidate preparation captured balances under the same - # ledger fence as its share window. Reuse that immutable copy - # instead of issuing the carry-forward aggregate twice during - # every publication. - artifact = self._payout_state_artifact_from_balances( - generation=candidate.base_generation + 1, - source_generation=candidate.source_generation, - balances=list(candidate.ledger_artifact.prior_balances), - ) - else: - artifact = self._prepare_payout_state_artifact( - generation=candidate.base_generation + 1, - source_generation=candidate.source_generation, - ) - except Exception: - # The delivery/cache fence remains closed. A later reconciliation - # retry can prepare and publish the artifact; no generation lacking - # immutable payout inputs is ever exposed to builders. - self._schedule_tip_refresh_retry() - raise - with self._job_cache_lock: - with self.lock: - if ( - candidate.source_generation != self._payout_state_source[0] - or candidate.base_generation != self._payout_state_generation - ): - self._record_discarded_payout_candidate() - return None - with self._payout_state_delivery_gate.publication(): - # publication() has already drained admitted old sends. Start the - # critical-section timer only now; drain latency is delivery wait, - # not time spent holding the atomic payout mutation section. - publish_started = time.monotonic() - with self._job_cache_lock: - with self.lock: - source_generation = self._payout_state_source[0] - if ( - candidate.source_generation == source_generation - and candidate.base_generation - == self._payout_state_generation - ): - published_generation = self._payout_state_generation + 1 - if candidate.accepted_block_hash is not None: - key = candidate.accepted_block_hash - with self._accepted_block_payout_preview_condition: - transition = self._accepted_block_payout_previews.get( - key, - _AcceptedBlockPayoutTransition( - block_height=candidate.accepted_block_height, - landed=True, - ), - ) - if candidate.accepted_block_withdrawal: - self._accepted_block_payout_previews.pop(key, None) - self._invalidated_accepted_block_payout_previews[ - key - ] = ( - transition.block_height - if transition.block_height is not None - else candidate.accepted_block_height - ) - else: - existing_preview = transition.preview - if ( - existing_preview is not None - and existing_preview - != candidate.accepted_block_preview - ): - raise RuntimeError( - "accepted block payout preview changed " - "during atomic publication" - ) - self._invalidated_accepted_block_payout_previews.pop( - key, - None, - ) - self._accepted_block_payout_previews[key] = ( - dataclass_replace( - transition, - landed=True, - preview=candidate.accepted_block_preview, - published_generation=published_generation, - ) - ) - self._accepted_block_payout_preview_condition.notify_all() - self._payout_state_generation = published_generation - prepared_artifact = candidate.ledger_artifact - candidate_anchor_age_ms = ( - None - if prepared_artifact is None - or prepared_artifact.snapshot_anchor_ms is None - else now_ms() - - int(prepared_artifact.snapshot_anchor_ms) - ) - if ( - prepared_artifact is not None - and self._payout_artifact_reuse_active() - and prepared_artifact.payout_state_generation - == published_generation - and prepared_artifact.append_invalidation_epoch - == self._payout_ledger_append_invalidation_epoch - and candidate_anchor_age_ms is not None - and candidate_anchor_age_ms - <= self._payout_artifact_max_anchor_age_ms() - ): - self._payout_ledger_artifact_generation += 1 - # Restamp freshness at the install: the candidate - # artifact was built before publication, and the - # delivery-gate drain between the two can outlive - # the reuse budget -- a freshly published payout - # generation must never arm an already-stale - # artifact and force the next builds back through - # the synchronous reward-window walk. - self._payout_ledger_artifact = dataclass_replace( - prepared_artifact, - generation=self._payout_ledger_artifact_generation, - prepared_monotonic=time.monotonic(), - ) - publication_installed = ( - self._payout_ledger_artifact_generation, - candidate_anchor_age_ms, - len(prepared_artifact.shares_json), - ) - else: - if ( - prepared_artifact is not None - and self._payout_artifact_reuse_active() - and prepared_artifact.payout_state_generation - == published_generation - and prepared_artifact.append_invalidation_epoch - == self._payout_ledger_append_invalidation_epoch - ): - # Same admission rule as - # _install_payout_ledger_artifact: candidate - # construction plus the delivery-gate drain - # can push the declared anchor past the - # audit ceiling, and arming such a - # BORN-EXPIRED artifact would fail every - # reuse probe on anchor age -- with the - # re-arm suppressed while an accepted - # preview awaits durability. Discard it; the - # post-publication probe schedules - # preparation (or the durable-confirmation - # path resumes it once the preview lands). - publication_born_expired = ( - candidate_anchor_age_ms, - len(prepared_artifact.shares_json), - ) - self._payout_ledger_artifact = None - self._published_payout_state = PublishedPayoutState( - generation=published_generation, - source_generation=candidate.source_generation, - source_tip_hash=candidate.source_tip_hash, - published_monotonic=publish_started, - artifact=artifact, - ) - epoch_tip_hash = self._payout_epoch_tip_hash_locked() - if epoch_tip_hash is not None: - self._mint_tip_refresh_epoch_locked( - tip_hash=epoch_tip_hash, - payout_state_generation=published_generation, - started_monotonic=candidate.invalidated_monotonic, - ) - # Re-anchor the stamping identity to the minted - # epoch in the same lock hold. A stale identity - # stamps rebuilt bundles at epoch 0, which the - # fence blocks for every previously admitted - # connection until the next wave republishes. - # The publish helper no-ops when the epoch tip - # is not the published snapshot tip; the owning - # wave handles that transition. - published_snapshot = getattr( - self, - "tip_template_snapshot", - None, - ) - if published_snapshot is not None: - self._publish_tip_refresh_epoch_identity_locked( - published_snapshot - ) - self._payout_state_publication_blocked = False - self._job_bundle_cache.clear() - self._retained_collection_refresh = None - active = getattr(self, "_active_tip_refresh", None) - if active is None: - schedule_retry = True - elif active[0].payout_state_generation < published_generation: - # The payout gate itself rejects this old generation. - # Signal its fanout only after atomic publication. - active_to_cancel = active[1] - schedule_retry = True - with self._payout_state_metrics_lock: - self._payout_first_delivery_pending = ( - published_generation, - candidate.invalidated_monotonic, - ) - if published_generation is not None: - # The mutation owner still blocks every delivery admission, - # so the pointer swap and gate generation remain one atomic - # publication boundary. Do not acquire the gate condition - # while holding coordinator locks: cancellation callbacks take - # those locks after entering the gate wait loop. - self._payout_state_delivery_gate.publish_generation( - published_generation, - prioritize_delivery=True, - ) - self._observe_payout_state_seconds( - "publish", - max(0.0, time.monotonic() - publish_started), - ) - if published_generation is None: - self._record_discarded_payout_candidate() - return None - if publication_born_expired is not None: - born_expired_age_ms, born_expired_shares = publication_born_expired - self._record_payout_artifact_event("born_expired") - self._payout_artifact_log( - "payout_artifact_born_expired", - payout_state_generation=int(published_generation), - anchor_age_ms=born_expired_age_ms, - window_shares=born_expired_shares, - during_publication=True, - ) - if publication_installed is not None: - # The pointer swap above is an install site that bypasses - # _install_payout_ledger_artifact; record it in the same event - # family or the publication path stays invisible to the - # lifecycle observability this counter exists for. - installed_generation, installed_age_ms, installed_shares = ( - publication_installed - ) - self._record_payout_artifact_event("installed") - self._payout_artifact_log( - "payout_artifact_installed", - generation=int(installed_generation), - payout_state_generation=int(published_generation), - anchor_age_ms=installed_age_ms, - window_shares=installed_shares, - during_publication=True, - ) - # A publication is a fresh start for speculative preparation: the - # candidate artifact installs through the atomic pointer swap above - # (not _install_payout_ledger_artifact), so whatever re-arm backoff - # pre-publication share traffic accumulated must be released here or - # a share landing right after publication would stay suppressed for - # the whole scaled interval. - with self._payout_artifact_executor_lock: - self._payout_artifact_rearm_backoff = 1 - self._cancel_obsolete_job_bundle_builds( - payout_state_generation=published_generation - ) - self._record_progress_payout_generation( - published_generation, - candidate.invalidated_monotonic, - ) - if active_to_cancel is not None: - active_to_cancel.cancel() - self._cancel_obsolete_job_builds("payout generation published") - if schedule_retry: - self._mark_tip_refresh_pending(published_generation) - self._schedule_tip_refresh_retry() - with self._job_cache_lock: - current_artifacts = self._template_artifacts - published_artifact_usable = ( - self._usable_payout_ledger_artifact( - published_generation, - current_artifacts.network_difficulty, - ) - if current_artifacts is not None - else None - ) - accepted_preview_pending_durability = ( - candidate.accepted_block_hash is not None - and not candidate.accepted_block_withdrawal - ) - if ( - current_artifacts is not None - and published_artifact_usable is None - and not accepted_preview_pending_durability - ): - # A background artifact built before accepted-block confirmation - # would carry the old database balances under the new payout - # generation. Child builders use the compact preview until the - # durable state catches up; artifact preparation resumes then. - self._schedule_payout_ledger_artifact_preparation( - published_generation, - current_artifacts.network_difficulty, - ) - return published_generation + def _publish_payout_state_candidate(self, candidate: PayoutStateCandidate) -> int | None: + """Publish a prepared candidate, or reject it if its source moved.""" + return self._ensure_payout_state_service()._publish_payout_state_candidate(candidate) - def _record_first_payout_delivery( - self, - generation: int, - delivered_monotonic: float, - ) -> None: - self._ensure_job_cache_state() - elapsed: float | None = None - with self._payout_state_metrics_lock: - pending = self._payout_first_delivery_pending - if pending is not None and pending[0] == generation: - elapsed = max(0.0, delivered_monotonic - pending[1]) - self._payout_first_delivery_pending = None - if elapsed is not None: - self._observe_payout_state_seconds("first_delivery", elapsed) + def _record_first_payout_delivery(self, generation: int, delivered_monotonic: float) -> None: + return self._ensure_payout_state_service()._record_first_payout_delivery(generation, delivered_monotonic) def _advance_payout_state_generation(self) -> int: """Publish a payout-only invalidation with no expensive gate work.""" - # No production caller remains: direct-block finalization publishes - # through submit_block_candidate's reserve/publish path instead. Tests - # keep using this as the smallest complete reserve -> block -> publish - # invalidation cycle. - self._ensure_job_cache_state() - self._reserve_payout_state_source("payout_only") - prepared_started = time.monotonic() - with self._payout_state_prepare_lock: - # Close build/delivery admission before releasing snapshot readers. - # Publication may then drain already-admitted sends without holding - # the preparation lock needed by later ledger work. - self._block_payout_state_publication(force=True) - self._observe_payout_state_seconds( - "preparation", - max(0.0, time.monotonic() - prepared_started), - ) - generation = self._publish_current_payout_state_with_retry_budget() - if generation is None: - raise TemplateRefreshSuperseded( - "payout-only invalidation was superseded; immediate retry scheduled" - ) - return generation + return self._ensure_payout_state_service()._advance_payout_state_generation() - def _publish_current_payout_state_with_retry_budget( - self, - *, - initial_attempted: bool = False, - ) -> int | None: + def _publish_current_payout_state_with_retry_budget(self, *, initial_attempted: bool=False) -> int | None: """Publish the current source with a bounded supersession budget.""" - - max_retries = max( - 0, - int( - getattr( - self, - "payout_reconcile_supersession_retries", - DEFAULT_PRISM_PAYOUT_RECONCILE_SUPERSESSION_RETRIES, - ) - ), - ) - attempts = max_retries + (0 if initial_attempted else 1) - for _attempt in range(attempts): - candidate = self._current_payout_state_candidate() - published = self._publish_payout_state_candidate(candidate) - if published is not None: - return published - self._block_payout_state_publication() - return None + return self._ensure_payout_state_service()._publish_current_payout_state_with_retry_budget(initial_attempted=initial_attempted) def observe_job_build_elapsed(self, elapsed_seconds: float, phases: dict[str, float]) -> None: - self._ensure_job_cache_state() - with self._job_cache_lock: - self.job_build_count += 1 - self.job_build_seconds_sum += elapsed_seconds - for bucket in PRISM_JOB_BUILD_SECONDS_BUCKETS: - if elapsed_seconds <= bucket: - self.job_build_seconds_bucket_counts[bucket] += 1 - for phase, duration in phases.items(): - if phase in self.job_build_phase_seconds: - self.job_build_phase_seconds[phase] += duration + self._ensure_job_bundle_service().observe_job_build_elapsed(elapsed_seconds, phases) def _flush_job_build_phases(self, phases: dict[str, float]) -> None: - """Fold a worker thread's phase accruals into the exported counters. - - Only the per-client delivery entry points clear and flush the - thread-local phase dict; the job-build executor and payout-artifact - preparation workers never passed through them, so the template/ - ledger/assembly/bundle seconds they accrued were exported as - exactly 0.0 while production paid multi-second walks -- the blind - spot that misdirected the 2026-07-31 incident review. Build counts - and the duration histogram stay per-delivery; this folds phase - seconds only, and clears the dict so a long-lived worker thread - cannot double-report an earlier request's accruals. - """ - if not phases: - return - self._ensure_job_cache_state() - with self._job_cache_lock: - for phase, duration in phases.items(): - if phase in self.job_build_phase_seconds: - self.job_build_phase_seconds[phase] += duration - phases.clear() + """Fold a worker thread's phase accruals into the exported counters.""" + self._ensure_job_bundle_service()._flush_job_build_phases(phases) def _reserve_template_artifact_generation(self) -> int: """Reserve template ordering when a fetch starts, not when it finishes.""" - self._ensure_job_cache_state() - with self._job_cache_lock: - self._template_artifact_generation += 1 - return self._template_artifact_generation + return self._ensure_job_bundle_service().template_repository.reserve_generation() def _derive_template_artifacts( self, @@ -9120,75 +4627,13 @@ def _derive_template_artifacts( *, generation: int, ) -> CachedTemplateArtifacts: - # Detach the observation from mutable RPC/test caller state. The - # dataclass then owns this exact tree for the lifetime of its snapshot. - template = copy.deepcopy(template) - fingerprint = qbit_template_fingerprint(template) - with self._job_cache_lock: - previous = self._template_artifacts - if previous is not None and previous.fingerprint == fingerprint: - return CachedTemplateArtifacts( - template=template, - fingerprint=fingerprint, - previousblockhash=str(template.get("previousblockhash", "")), - transaction_hexes=previous.transaction_hexes, - witness_merkle_leaves_hex=previous.witness_merkle_leaves_hex, - network_difficulty=previous.network_difficulty, - fetched_monotonic=time.monotonic(), - generation=generation, - ) - phases = self._job_build_phases() - started = time.monotonic() - transaction_hexes = direct_stratum.transaction_hexes_from_template(template) - witness_leaves = tuple(direct_stratum.witness_merkle_leaves_hex(transaction_hexes)) - network_difficulty = scaled_network_difficulty(str(template["bits"])) - phases["merkle"] = phases.get("merkle", 0.0) + (time.monotonic() - started) - return CachedTemplateArtifacts( - template=template, - fingerprint=fingerprint, - previousblockhash=str(template.get("previousblockhash", "")), - transaction_hexes=transaction_hexes, - witness_merkle_leaves_hex=witness_leaves, - network_difficulty=network_difficulty, - fetched_monotonic=time.monotonic(), - generation=generation, - ) + return self._ensure_job_bundle_service().template_repository.derive(template, generation=generation) def _store_template_artifacts( self, artifacts: CachedTemplateArtifacts, ) -> bool: - changed = False - with self._job_cache_lock: - previous = self._template_artifacts - if previous is not None and artifacts.generation < previous.generation: - return False - self._template_artifacts = artifacts - if previous is not None and previous.fingerprint != artifacts.fingerprint: - changed = True - # Retain the published snapshot's bundles alongside the new - # fingerprint: until the replacement tip is published, direct - # issuance still serves published-snapshot work, and pruning - # it here would force those paths to defer for the whole - # detected-but-unpublished window. Publication moves the - # snapshot, so the next fingerprint change drops the old - # entries. - keep_fingerprints = {artifacts.fingerprint} - with self.lock: - published_snapshot = getattr(self, "tip_template_snapshot", None) - if published_snapshot is not None: - keep_fingerprints.add(published_snapshot.template_fingerprint) - self._job_bundle_cache = OrderedDict( - (key, entry) - for key, entry in self._job_bundle_cache.items() - if entry.template_fingerprint in keep_fingerprints - ) - if changed: - self._cancel_obsolete_job_builds( - "template fingerprint superseded", - keep_published_snapshot=True, - ) - return True + return self._ensure_job_bundle_service().template_repository.store_artifacts(artifacts) def store_template_artifacts( self, @@ -9196,136 +4641,20 @@ def store_template_artifacts( *, generation: int | None = None, ) -> CachedTemplateArtifacts | None: - """Best-effort cache fill from an already-fetched template (blockpoll). - - Returns None instead of raising so a template the derivation cannot - digest degrades to the legacy per-build fetch path rather than failing - the poll. The returned artifacts describe this exact observation even - if a newer observation already won the cache-write race; blockpoll then - detects the mismatch before fanout. - """ - self._ensure_job_cache_state() - if generation is None: - generation = self._reserve_template_artifact_generation() - try: - artifacts = self._derive_template_artifacts( - template, - generation=generation, - ) - except Exception: - return None - self._store_template_artifacts(artifacts) - return artifacts + """Best-effort cache fill from an already-fetched template (blockpoll).""" + return self._ensure_job_bundle_service().template_repository.store(template, generation=generation) def job_issuance_template_artifacts(self) -> CachedTemplateArtifacts: - """Template artifacts for direct (non-refresh) job issuance. - - While a detected replacement tip is still unpublished, share - classification remains anchored to the published tip. Direct issuance - paths (initial delivery retries, Vardiff retargets, reauthorization) - must therefore keep handing out published-snapshot work; issuing - detected-tip work early would have every one of its shares rejected - as stale until the refresh publishes. Once the published authority - lapses (divergence lease expired), issuance falls through to the live - template exactly like submit classification falls back to the live - RPC read. - """ - self._ensure_tip_refresh_state() - with self.lock: - published = getattr(self, "current_tip_first_seen", None) - latest_detected = getattr(self, "latest_detected_tip", None) - published_snapshot = getattr(self, "tip_template_snapshot", None) - pinned = bool( - published is not None - and latest_detected is not None - and latest_detected[0] != published[0] - and published_snapshot is not None - and published_snapshot.bestblockhash == published[0] - and published_snapshot.template_artifacts is not None - and self._published_tip_authoritative_locked(time.monotonic()) - ) - if pinned: - assert published_snapshot is not None - assert published_snapshot.template_artifacts is not None - return published_snapshot.template_artifacts - artifacts = self.current_template_artifacts() - # The fetch above may itself be the first observation of a newer tip - # (a template-cache miss racing ahead of blockpoll/blockwait). The - # published tip still owns share classification, so serve its - # snapshot; the recorded detection has already armed the refresh. - with self.lock: - published = getattr(self, "current_tip_first_seen", None) - published_snapshot = getattr(self, "tip_template_snapshot", None) - repinned = bool( - published is not None - and artifacts.previousblockhash != published[0] - and published_snapshot is not None - and published_snapshot.bestblockhash == published[0] - and published_snapshot.template_artifacts is not None - and self._published_tip_authoritative_locked(time.monotonic()) - ) - if repinned: - assert published_snapshot is not None - assert published_snapshot.template_artifacts is not None - return published_snapshot.template_artifacts - return artifacts + """Template artifacts for direct (non-refresh) job issuance.""" + return self._ensure_job_bundle_service().template_repository.issuance() def current_template_artifacts(self) -> CachedTemplateArtifacts: """Return fresh template artifacts, fetching a template on cache miss.""" - self._ensure_job_cache_state() - ttl = getattr(self, "template_cache_seconds", DEFAULT_PRISM_BLOCKPOLL_SECONDS) - now = time.monotonic() - with self._job_cache_lock: - cached = self._template_artifacts - with self.lock: - observed_tip = self._newest_observed_tip_locked() - cached_tip_current = ( - observed_tip is None - or cached is None - or cached.previousblockhash == observed_tip - ) - if ( - cached is not None - and cached_tip_current - and ttl > 0 - and now - cached.fetched_monotonic <= ttl - ): - self._record_job_cache_event("template", hit=True) - return cached - self._record_job_cache_event("template", hit=False) - generation = self._reserve_template_artifact_generation() - phases = self._job_build_phases() - started = time.monotonic() - template = self.rpc.call( - "getblocktemplate", - [{"rules": qbit_gbt_rules(getattr(self, "qbit_chain", "regtest"))}], - ) - if not isinstance(template, dict): - raise RuntimeError("getblocktemplate returned non-object") - phases["template"] = phases.get("template", 0.0) + (time.monotonic() - started) - artifacts = self._derive_template_artifacts( - template, - generation=generation, - ) - if self._store_template_artifacts(artifacts): - # A direct fetch can be the first reader to see qbit advance. - # Record it as a detection like every other live-tip observation, - # or the buildability gates would treat the newer parent as - # unknown while pinned issuance still waits on blockpoll. - self.observe_tip_for_refresh(artifacts.previousblockhash) - return artifacts - # A later fetch completed first. Build from that current observation, - # never from the stale response that lost the cache-write race. - with self._job_cache_lock: - current = self._template_artifacts - if current is None: - raise RuntimeError("newer template artifacts disappeared after cache race") - self.observe_tip_for_refresh(current.previousblockhash) - return current + return self._ensure_job_bundle_service().template_repository.current() - @staticmethod - def _collection_bundle_identity(worker: WorkerIdentity) -> tuple[str, str]: - return worker.payout_address, worker.p2mr_program_hex + _collection_bundle_identity = staticmethod( + JobBundleService._collection_bundle_identity + ) def _job_bundle_key( self, @@ -9336,61 +4665,22 @@ def _job_bundle_key( payout_artifact_generation: int = 0, worker: WorkerIdentity | None, ) -> tuple[object, ...]: - if mode == "ready": - return ( - artifacts.fingerprint, - artifacts.previousblockhash, - "ready", - payout_state_generation, - payout_artifact_generation, - ) - if mode != "collection": - raise ValueError(f"unknown PRISM job-bundle mode: {mode}") - if worker is None: - raise CollectionIdentityUnavailable( - "collection-mode worker identity is temporarily unavailable" - ) - return ( - artifacts.fingerprint, - artifacts.previousblockhash, - "collection", - artifacts.generation, - payout_state_generation, - payout_artifact_generation, - *self._collection_bundle_identity(worker), + return self._ensure_job_bundle_service()._job_bundle_key( + artifacts, + mode=mode, + payout_state_generation=payout_state_generation, + payout_artifact_generation=payout_artifact_generation, + worker=worker, ) def _job_bundle_mode(self, requested_mode: str | None) -> str: - if requested_mode is not None: - if requested_mode not in {"ready", "collection"}: - raise ValueError( - f"unknown PRISM job-bundle mode: {requested_mode}" - ) - return requested_mode - return "ready" if self.pool_readiness_latched() else "collection" + return self._ensure_job_bundle_service()._job_bundle_mode(requested_mode) def _lookup_job_bundle( self, key: tuple[object, ...], ) -> CachedJobBundle | None: - ttl = getattr(self, "job_bundle_cache_seconds", DEFAULT_PRISM_JOB_BUNDLE_CACHE_SECONDS) - now = time.monotonic() - with self._job_cache_lock: - if ttl <= 0: - self._job_bundle_cache.clear() - return None - # The entry-count cap is not a memory bound: one production entry - # can reference more than 100k shares. Expired entries must release - # their snapshots instead of remaining resident until count eviction. - expired = [ - cache_key - for cache_key, entry in self._job_bundle_cache.items() - if now - entry.built_monotonic > ttl - ] - for cache_key in expired: - self._job_bundle_cache.pop(cache_key, None) - return self._job_bundle_cache.get(key) - return None + return self._ensure_job_bundle_service()._lookup_job_bundle(key) def _no_artifact_job_bundle_key( self, @@ -9400,382 +4690,61 @@ def _no_artifact_job_bundle_key( payout_state_generation: int, worker: WorkerIdentity | None, ) -> tuple[object, ...]: - """Fallback cache identity for work built before an artifact re-key. - - A synchronous build caches its bundle under the no-artifact key and - then publishes its anchored window as the payout ledger artifact. The - next lookup is keyed to that artifact and would miss the still-fresh - bundle and rebuild identical work: within one payout generation and - template identity the no-artifact bundle binds the same balances, - window anchor, and policy inputs, so it remains a correct cache - identity to serve -- but only while its window digest matches the - armed artifact (see _no_artifact_bundle_matches_artifact); a fence - re-arm can install a fresher window that cached work must not shadow. - """ - return self._job_bundle_key( + """Fallback cache identity for work built before an artifact re-key.""" + return self._ensure_job_bundle_service()._no_artifact_job_bundle_key( artifacts, mode=mode, payout_state_generation=payout_state_generation, - payout_artifact_generation=0, worker=worker, ) - @staticmethod - def _no_artifact_bundle_matches_artifact( - cached: CachedJobBundle, - payout_artifact: PayoutLedgerArtifact | None, - ) -> bool: - """Whether pre-re-key cached work carries exactly the artifact's window. - - A speculative rebuild can arm a fresher artifact while an older - no-artifact bundle is still inside its cache TTL. Serving that bundle - would shadow the fresher window behind cached work, so the fallback - identity is honored only when the window digests are byte-identical - -- which the bundle that published the artifact satisfies by - construction. - """ - return ( - payout_artifact is not None - and payout_artifact.share_snapshot_sha256 is not None - and cached.build_key is not None - and cached.build_key.share_snapshot_sha256 - == payout_artifact.share_snapshot_sha256 - ) + _no_artifact_bundle_matches_artifact = staticmethod( + JobBundleService._no_artifact_bundle_matches_artifact + ) def _job_bundle_entry_usable( self, cached: CachedJobBundle | None, artifacts: CachedTemplateArtifacts, ) -> bool: - """Re-validate freshness and readiness for cached bundles. - - Readiness is monotonic in practice (the distinct accepted-miner count - only grows), so submit-capable ready bundles are served as-is while - their declared window anchor stays inside the artifact staleness - bound. A cached collection bundle is re-checked against the cheap - aggregate stats: once the pool is ready it must stop being served, or - jobs would keep collecting winning shares without submitting blocks - for up to the cache TTL. - """ - if cached is None: - return False - # Usability follows the same split-authority rule as construction: a - # cached bundle for the newest detected tip stays reusable across - # refresh retries while the previous tip is still published, and the - # published snapshot's own bundle stays servable for pinned issuance. - with self.lock: - parent_usable = self._artifacts_buildable_locked(artifacts) - if not parent_usable: - return False - with self._job_cache_lock: - if self._payout_state_publication_blocked: - return False - if not self._job_bundle_payout_state_current(cached): - return False - if not cached.collection_only: - # Cached-ready-bundle validity follows the same ceiling/floor - # split as artifact reuse. The declared anchor is gated only by - # the audit CEILING: the anchor is frozen per template - # generation and predates the window walk, so a tight - # wall-clock gate here declared every rebuilt bundle dead on - # arrival once its template generation outlived the bound - # (2026-07-29 rollback). - found_block = getattr(cached, "found_block", None) - declared_anchor_ms = ( - found_block.get("anchor_job_issued_at_ms") - if isinstance(found_block, dict) - else None - ) - if declared_anchor_ms is not None and ( - now_ms() - int(declared_anchor_ms) - > self._payout_artifact_max_anchor_age_ms() - ): - return False - # A bundle keyed to the currently armed artifact generation - # carries exactly the armed window, so it serves for as long as - # reuse itself does (the armed artifact is event-invalidated by - # the payout fences and re-anchored in the background). Every - # other ready bundle (no-artifact builds and survivors of a - # re-key) carries a window nothing will re-anchor, so it keeps - # serving only inside the re-anchor floor -- its declared - # anchor is already ceiling-gated above, and a wall-clock - # budget tighter than the build cadence is exactly the - # 2026-07-30 brownout (every consecutive build fell back to the - # synchronous reward-window walk). - if int(getattr(cached, "payout_artifact_generation", 0)) > 0: - with self._job_cache_lock: - armed = self._payout_ledger_artifact - if ( - armed is not None - and int(armed.generation) - == int(cached.payout_artifact_generation) - ): - return True - return ( - time.monotonic() - float(cached.built_monotonic) - <= self._payout_artifact_reanchor_seconds() - ) - # Collection bundles sign a synthetic bootstrap share containing the - # exact template ntime. A clock-only observation keeps the stable work - # fingerprint, but it must rebuild this signed bundle instead of - # rebinding the old manifest to a new template generation. - if ( - cached.template is not artifacts.template - or cached.template_generation != artifacts.generation - ): - return False - try: - _, ready_miner_count = self.accepted_share_stats() - except Exception: - # If readiness cannot be proven, force the normal build path. That - # path will either build an up-to-date bundle or surface the ledger - # failure instead of continuing to issue no-submit collection jobs. - return False - return ready_miner_count < self.min_ready_miners + """Re-validate freshness and readiness for cached bundles.""" + return self._ensure_job_bundle_service()._job_bundle_entry_usable(cached, artifacts) def _bind_cached_bundle_to_artifacts( self, cached: CachedJobBundle, artifacts: CachedTemplateArtifacts, - ) -> CachedJobBundle: - """Return the cached heavy bundle bound to this exact observation. - - Clock-only template changes intentionally keep the stable fingerprint. - Ready bundles may reuse their ledger snapshot and signed manifest, but - the Stratum base job must still carry the observing template's exact - ntime and generation. Collection bundles are filtered before this point - because their signed synthetic share contains the template ntime. - """ - if ( - cached.template is artifacts.template - and cached.template_generation == artifacts.generation - ): - return cached - manifest = cached.coinbase_manifest - base_job = direct_stratum.make_job_from_builder_manifest( - job_id="prism-template-base", - template=artifacts.template, - manifest=manifest, - extranonce1_hex=PRISM_JOB_EXTRANONCE1_PLACEHOLDER_HEX, - extranonce2_size=self.extranonce2_size, - desired_share_difficulty=self.share_difficulty, - clean_jobs=True, - transaction_hexes=artifacts.transaction_hexes, - ) - return dataclass_replace( - cached, - template=artifacts.template, - base_job=base_job, - template_generation=artifacts.generation, - build_key=( - dataclass_replace( - cached.build_key, - best_tip_hash=artifacts.previousblockhash, - previous_block_hash=artifacts.previousblockhash, - template_generation=artifacts.generation, - block_height=int(artifacts.template["height"]), - coinbase_value_sats=int(artifacts.template["coinbasevalue"]), - ) - if cached.build_key is not None - else None - ), - ) - - def _new_job_build_request( - self, - artifacts: CachedTemplateArtifacts, - worker: WorkerIdentity | None, - *, - mode: str, - payout_state_generation: int, - cache_key: tuple[object, ...], - payout_ledger_artifact: PayoutLedgerArtifact | None = None, - idle_retarget: bool = False, - publication_critical: bool = False, - request_source: str = "routine", - priority_requested_monotonic: float | None = None, - preparation_cancellation: _JobBuildCancellation | None = None, - ) -> _JobBuildRequest: - cancellation = ( - self._new_job_build_cancellation() - if preparation_cancellation is None - else preparation_cancellation - ) - cancellation.raise_if_cancelled("immutable snapshot") - # A pending accepted-parent transition owns the balances children must - # observe. Wait for its preview before acquiring any confirmed payout - # input so a child build cannot take -- or cache into the published - # artifact -- confirmed state that omits its new parent. The preview - # itself is re-resolved consistently during construction. - self._await_pending_parent_payout_preview( - artifacts.previousblockhash, - parent_height=int(artifacts.template["height"]) - 1, - ) - payout_artifact = self._current_payout_state_artifact(cancellation) - if payout_artifact.generation != payout_state_generation: - raise JobBuildSuperseded( - "payout artifact generation changed before build request" - ) - - phases = self._job_build_phases() - payout_started = time.monotonic() - payout_policy_json = canonical_json_text(self.prism_payout_policy()) - phases["payout"] = phases.get("payout", 0.0) + ( - time.monotonic() - payout_started - ) - cancellation.raise_if_cancelled("payout policy") - ctv_started = time.monotonic() - ctv_settlement = self.prism_ctv_settlement_config( - block_height=int(artifacts.template["height"]), - parent_hash=artifacts.previousblockhash, - ) - ctv_settlement_json = ( - canonical_json_text(ctv_settlement) - if ctv_settlement is not None - else None - ) - phases["ctv"] = phases.get("ctv", 0.0) + ( - time.monotonic() - ctv_started - ) - cancellation.raise_if_cancelled("CTV configuration") - - suffix_hex = self.coinbase_script_sig_suffix_hex( - PRISM_JOB_EXTRANONCE1_PLACEHOLDER_HEX, - "00" * self.extranonce2_size, - ) - collection_identity = ( - self._collection_bundle_identity(worker) - if mode == "collection" and worker is not None - else None - ) - decimal_context = getcontext().copy() - numeric_context_sha256 = canonical_json_sha256( - { - "precision": decimal_context.prec, - "rounding": decimal_context.rounding, - "minimum_exponent": decimal_context.Emin, - "maximum_exponent": decimal_context.Emax, - "capitals": decimal_context.capitals, - "clamp": decimal_context.clamp, - } - ) - with self._job_cache_lock: - issued_at_ms = self._job_build_issued_at_ms.get(artifacts.generation) - current_append_invalidation_epoch = int( - self._payout_ledger_append_invalidation_epoch - ) - if issued_at_ms is None: - # The issued time doubles as the audit window anchor, so it - # must not cover a stamped share whose commit is still in - # flight: the frozen anchor stays reproducible from the - # durable ledger for every rebuild of this generation. The - # pending-commit clamp is pure anchor selection -- it freezes - # the highest anchor whose covered shares are all durable, and - # shares stamped above it deterministically belong to the next - # window -- so nothing else needs to be captured with it. - candidate_anchor_ms = self._job_snapshot_anchor_ms(now_ms()) - with self._job_cache_lock: - issued_at_ms = self._job_build_issued_at_ms.get( - artifacts.generation - ) - if issued_at_ms is None: - issued_at_ms = candidate_anchor_ms - self._job_build_issued_at_ms[artifacts.generation] = ( - issued_at_ms - ) - while len(self._job_build_issued_at_ms) > 128: - self._job_build_issued_at_ms.popitem(last=False) - build_append_invalidation_epoch = ( - int(payout_ledger_artifact.append_invalidation_epoch) - if mode == "ready" and payout_ledger_artifact is not None - else current_append_invalidation_epoch - if mode == "ready" - else 0 - ) - build_key = JobBuildKey( - best_tip_hash=artifacts.previousblockhash, - previous_block_hash=artifacts.previousblockhash, - template_fingerprint=artifacts.fingerprint, - template_generation=artifacts.generation, - payout_state_generation=payout_state_generation, - payout_artifact_sha256=payout_artifact.prior_balances_sha256, + ) -> CachedJobBundle: + """Return the cached heavy bundle bound to this exact observation.""" + return self._ensure_job_bundle_service()._bind_cached_bundle_to_artifacts(cached, artifacts) + + def _new_job_build_request( + self, + artifacts: CachedTemplateArtifacts, + worker: WorkerIdentity | None, + *, + mode: str, + payout_state_generation: int, + cache_key: tuple[object, ...], + payout_ledger_artifact: PayoutLedgerArtifact | None = None, + idle_retarget: bool = False, + publication_critical: bool = False, + request_source: str = "routine", + priority_requested_monotonic: float | None = None, + preparation_cancellation: _JobBuildCancellation | None = None, + ) -> _JobBuildRequest: + return self._ensure_job_bundle_service()._new_job_build_request( + artifacts, + worker, mode=mode, - collection_identity=collection_identity, - block_height=int(artifacts.template["height"]), - coinbase_value_sats=int(artifacts.template["coinbasevalue"]), - network_difficulty=int(artifacts.network_difficulty), - issued_at_ms=issued_at_ms, - payout_policy_sha256=hashlib.sha256( - payout_policy_json.encode() - ).hexdigest(), - ctv_settlement_sha256=( - hashlib.sha256(ctv_settlement_json.encode()).hexdigest() - if ctv_settlement_json is not None - else None - ), - witness_merkle_sha256=canonical_json_sha256( - artifacts.witness_merkle_leaves_hex - ), - transaction_set_sha256=canonical_json_sha256( - artifacts.transaction_hexes - ), - coinbase_suffix_hex=suffix_hex, - signing_key_sha256=hashlib.sha256( - str(getattr(self, "signing_seed_hex", "")).encode() - ).hexdigest(), - ledger_signing_key_sha256=hashlib.sha256( - str( - getattr( - self, - "ledger_attestation_signing_seed_hex", - "", - ) - ).encode() - ).hexdigest(), - numeric_context_sha256=numeric_context_sha256, - payout_append_invalidation_epoch=( - build_append_invalidation_epoch - ), - ) - immutable_identity: tuple[object, ...] = ( - cache_key, - artifacts.generation, - issued_at_ms, - payout_artifact.prior_balances_sha256, - build_key.payout_policy_sha256, - build_key.ctv_settlement_sha256, - build_key.witness_merkle_sha256, - build_key.transaction_set_sha256, - build_key.coinbase_suffix_hex, - build_key.signing_key_sha256, - build_key.ledger_signing_key_sha256, - build_key.numeric_context_sha256, - build_key.payout_append_invalidation_epoch, - ) - return _JobBuildRequest( - key=build_key, + payout_state_generation=payout_state_generation, cache_key=cache_key, - equivalence_key=immutable_identity, - artifacts=artifacts, - template_json=canonical_json_text(artifacts.template), - transaction_hexes=artifacts.transaction_hexes, - witness_merkle_leaves_hex=artifacts.witness_merkle_leaves_hex, - worker=worker, - mode=mode, - payout_artifact=payout_artifact, payout_ledger_artifact=payout_ledger_artifact, - payout_policy_json=payout_policy_json, - ctv_settlement_json=ctv_settlement_json, - decimal_context=decimal_context, - cancellation=cancellation, idle_retarget=idle_retarget, publication_critical=publication_critical, request_source=request_source, - requested_monotonic=( - cancellation.started_monotonic - if priority_requested_monotonic is None - else priority_requested_monotonic - ), + priority_requested_monotonic=priority_requested_monotonic, + preparation_cancellation=preparation_cancellation, ) def _newest_observed_tip_locked(self) -> str | None: @@ -9834,56 +4803,7 @@ def _cache_job_bundle_if_current( artifacts: CachedTemplateArtifacts, ) -> bool: """Cache only current state; report whether payout state stayed valid.""" - with self._job_cache_lock: - with self.lock: - buildable = self._artifacts_buildable_locked(artifacts) - snapshot_pinned = self._published_snapshot_artifacts_locked( - artifacts - ) - published_artifact = self._published_payout_state.artifact - if not buildable: - return False - if ( - built.payout_state_generation != self._payout_state_generation - or built.build_key is None - or ( - not built.collection_only - and built.build_key.payout_append_invalidation_epoch - != self._payout_ledger_append_invalidation_epoch - ) - or published_artifact is None - or built.build_key.payout_artifact_sha256 - != published_artifact.prior_balances_sha256 - ): - return False - current = self._template_artifacts - globally_current = ( - current is not None - and current.fingerprint == artifacts.fingerprint - and current.previousblockhash == artifacts.previousblockhash - and ( - not built.collection_only - or current.generation == artifacts.generation - ) - ) - if not globally_current and not snapshot_pinned: - # Only the current template observation may win the cache - # race; the sole exception is a pinned rebuild of exactly the - # published snapshot, which repeated direct issuance would - # otherwise rebuild for the rest of the unpublished window. - return False - self._job_bundle_cache[built.key] = built - self._job_bundle_cache.move_to_end(built.key) - while len(self._job_bundle_cache) > MAX_PRISM_JOB_BUNDLE_CACHE_ENTRIES: - oldest_key = next(iter(self._job_bundle_cache)) - self._job_bundle_cache.pop(oldest_key, None) - # Arm the build's fenced ledger window only after the bundle itself - # won cache publication: lookups re-keyed by the fresh artifact fall - # back to exactly this cached no-artifact entry instead of forking a - # redundant build. - if built.prepared_ledger_artifact is not None: - self._install_payout_ledger_artifact(built.prepared_ledger_artifact) - return True + return self._ensure_job_bundle_service()._cache_job_bundle_if_current(built, artifacts) def _probe_initial_job_bundle( self, @@ -9891,60 +4811,8 @@ def _probe_initial_job_bundle( worker: WorkerIdentity | None, mode: str | None, ) -> CachedJobBundle | None: - """Serve an initial request straight from the bundle cache. - - Routine admission exists to keep CPU-heavy request construction from - competing with a publication-critical build; a cache probe does none - of that work. Probing before admission lets a first job ship the - already-published bundle immediately instead of waiting out an entire - priority build it would never have contended with. - """ - resolved_mode = self._job_bundle_mode(mode) - if resolved_mode == "collection" and worker is None: - raise CollectionIdentityUnavailable( - "collection-mode worker identity is temporarily unavailable" - ) - with self._job_cache_lock: - payout_state_generation = self._payout_state_generation - payout_artifact = ( - self._usable_payout_ledger_artifact( - payout_state_generation, - artifacts.network_difficulty, - ) - if resolved_mode == "ready" - else None - ) - key = self._job_bundle_key( - artifacts, - mode=resolved_mode, - payout_state_generation=payout_state_generation, - payout_artifact_generation=( - payout_artifact.generation if payout_artifact is not None else 0 - ), - worker=worker, - ) - cached = self._lookup_job_bundle(key) - if cached is None and payout_artifact is not None: - fallback = self._lookup_job_bundle( - self._no_artifact_job_bundle_key( - artifacts, - mode=resolved_mode, - payout_state_generation=payout_state_generation, - worker=worker, - ) - ) - if fallback is not None and self._no_artifact_bundle_matches_artifact( - fallback, - payout_artifact, - ): - cached = fallback - if not self._job_bundle_entry_usable(cached, artifacts): - return None - self._record_job_cache_event("bundle", hit=True) - with self._job_build_scheduler_lock: - self.initial_job_prepared_work_counts["cache_hit"] += 1 - assert cached is not None - return self._bind_cached_bundle_to_artifacts(cached, artifacts) + """Serve an initial request straight from the bundle cache.""" + return self._ensure_job_bundle_service()._probe_initial_job_bundle(artifacts, worker, mode) def shared_job_bundle( self, @@ -9959,39 +4827,18 @@ def shared_job_bundle( request_source: str = "routine", priority_requested_monotonic: float | None = None, ) -> CachedJobBundle: - """Return one immutable heavy build through a work-identity flight. - - Equivalent callers share one future. Publication-critical refreshes - can cancel routine work and start in the bounded replacement slot; - reconnect, Vardiff, authorization, and same-tip work must defer while - that priority work is active. Only the latest pending request is - retained during repeated supersession. - """ - self._ensure_job_cache_state() - self._ensure_tip_refresh_state() - priority_token: int | None = None - if publication_critical: - ( - priority_token, - priority_requested_monotonic, - ) = self._begin_job_build_priority_preparation( - priority_requested_monotonic - ) - try: - return self._shared_job_bundle_after_priority_admission( - artifacts, - worker, - mode=mode, - cancelled=cancelled, - retry_superseded=retry_superseded, - idle_retarget=idle_retarget, - publication_critical=publication_critical, - request_source=request_source, - priority_requested_monotonic=priority_requested_monotonic, - ) - finally: - if priority_token is not None: - self._finish_job_build_priority_preparation(priority_token) + """Return one immutable heavy build through a work-identity flight.""" + return self._ensure_job_bundle_service().shared_job_bundle( + artifacts, + worker, + mode=mode, + cancelled=cancelled, + retry_superseded=retry_superseded, + idle_retarget=idle_retarget, + publication_critical=publication_critical, + request_source=request_source, + priority_requested_monotonic=priority_requested_monotonic, + ) def _shared_job_bundle_after_priority_admission( self, @@ -10006,778 +4853,48 @@ def _shared_job_bundle_after_priority_admission( request_source: str = "routine", priority_requested_monotonic: float | None = None, ) -> CachedJobBundle: - while True: - routine_preparation_token: int | None = None - preparation_cancellation: _JobBuildCancellation | None = None - subscribed_bundle: CachedJobBundle | None = None - if not publication_critical: - if request_source == "initial": - probed = self._probe_initial_job_bundle( - artifacts, - worker, - mode, - ) - if probed is not None: - return probed - ( - routine_preparation_token, - preparation_cancellation, - subscribed_bundle, - ) = self._begin_routine_job_build_preparation( - request_source=request_source, - cancelled=cancelled, - ) - resolved_mode = self._job_bundle_mode(mode) - if preparation_cancellation is not None: - preparation_cancellation.raise_if_cancelled( - "request preparation admission" - ) - if resolved_mode == "collection" and worker is None: - raise CollectionIdentityUnavailable( - "collection-mode worker identity is temporarily unavailable" - ) - with self._job_cache_lock: - payout_state_generation = self._payout_state_generation - payout_artifact = ( - self._usable_payout_ledger_artifact( - payout_state_generation, - artifacts.network_difficulty, - ) - if resolved_mode == "ready" - else None - ) - if preparation_cancellation is not None: - preparation_cancellation.raise_if_cancelled( - "payout artifact lookup" - ) - payout_artifact_generation = ( - payout_artifact.generation if payout_artifact is not None else 0 - ) - key = self._job_bundle_key( - artifacts, - mode=resolved_mode, - payout_state_generation=payout_state_generation, - payout_artifact_generation=payout_artifact_generation, - worker=worker, - ) - no_artifact_key = ( - self._no_artifact_job_bundle_key( - artifacts, - mode=resolved_mode, - payout_state_generation=payout_state_generation, - worker=worker, - ) - if payout_artifact_generation != 0 - else None - ) - cached = self._lookup_job_bundle(key) - if ( - cached is None - and subscribed_bundle is not None - and ( - subscribed_bundle.key == key - or ( - no_artifact_key is not None - and subscribed_bundle.key == no_artifact_key - and self._no_artifact_bundle_matches_artifact( - subscribed_bundle, - payout_artifact, - ) - ) - ) - ): - # The subscribed priority result may not have won the global - # cache-publication race yet; consume it directly. Usability - # below applies exactly the checks a cache entry would face. - cached = subscribed_bundle - if cached is None and no_artifact_key is not None: - fallback = self._lookup_job_bundle(no_artifact_key) - if ( - fallback is not None - and self._no_artifact_bundle_matches_artifact( - fallback, - payout_artifact, - ) - ): - cached = fallback - if self._job_bundle_entry_usable(cached, artifacts): - if preparation_cancellation is not None: - preparation_cancellation.raise_if_cancelled( - "bundle cache lookup" - ) - if routine_preparation_token is not None: - self._finish_routine_job_build_preparation( - routine_preparation_token - ) - self._record_job_cache_event("bundle", hit=True) - if request_source == "initial": - with self._job_build_scheduler_lock: - self.initial_job_prepared_work_counts["cache_hit"] += 1 - assert cached is not None - return self._bind_cached_bundle_to_artifacts(cached, artifacts) - if self._job_bundle_mode(mode) != resolved_mode: - if routine_preparation_token is not None: - self._finish_routine_job_build_preparation( - routine_preparation_token - ) - continue - self._record_job_cache_event("bundle", hit=False) - try: - request = self._new_job_build_request( - artifacts, - worker, - mode=resolved_mode, - payout_state_generation=payout_state_generation, - cache_key=key, - payout_ledger_artifact=payout_artifact, - idle_retarget=idle_retarget, - publication_critical=publication_critical, - request_source=request_source, - priority_requested_monotonic=( - priority_requested_monotonic - ), - preparation_cancellation=preparation_cancellation, - ) - # Preserve the historical readiness handoff without holding a - # lock across construction: only admission and the final mode - # re-selection are serialized here. - with self._job_build_lock: - with self._job_build_scheduler_lock: - if routine_preparation_token is not None: - self._finish_routine_job_build_preparation( - routine_preparation_token - ) - routine_preparation_token = None - request.cancellation.raise_if_cancelled( - "scheduler admission" - ) - if self._job_bundle_mode(mode) != resolved_mode: - request.cancellation.cancel( - "worker mode superseded" - ) - continue - if cancelled is not None and cancelled(): - raise _JobBuildCancelled( - "job bundle request was cancelled before preparation" - ) - promise = self._request_job_build(request) - wait_deadline = time.monotonic() + max( - 0.001, - float(self.job_build_timeout_seconds) - + float(self.job_build_cancel_grace_seconds) - + 1.0, - ) - while True: - if cancelled is not None and cancelled(): - raise _JobBuildCancelled( - "job bundle waiter was cancelled during preparation" - ) - try: - built = promise.result( - timeout=min( - 0.1, - max(0.001, wait_deadline - time.monotonic()), - ) - ) - break - except TimeoutError: - if time.monotonic() >= wait_deadline: - raise - except TimeoutError as exc: - request.cancellation.cancel("timeout") - self._schedule_tip_refresh_retry() - raise JobBuildCancelled( - "job build timed out; immediate retry scheduled" - ) from exc - except JobBuildCancelled: - if routine_preparation_token is not None: - self._finish_routine_job_build_preparation( - routine_preparation_token - ) - self._schedule_tip_refresh_retry() - if not retry_superseded: - raise - with self.lock: - buildable = self._artifacts_buildable_locked(artifacts) - if not buildable: - raise - with self._job_cache_lock: - current = self._template_artifacts - payout_current = ( - payout_state_generation == self._payout_state_generation - ) - if current is artifacts and payout_current: - continue - if current is artifacts: - continue - raise - built = self._bind_cached_bundle_to_artifacts(built, artifacts) - if not self._cache_job_bundle_if_current(built, artifacts): - with self._job_build_scheduler_lock: - self.job_build_scheduler_counts["obsolete_results"] += 1 - with self.lock: - buildable = self._artifacts_buildable_locked(artifacts) - if not buildable: - with self._tip_refresh_metrics_lock: - self.tip_refresh_superseded_results += 1 - raise JobBuildSuperseded( - "observed tip changed before cache publication" - ) - with self._job_cache_lock: - payout_current = ( - built.payout_state_generation - == self._payout_state_generation - and ( - built.collection_only - or ( - built.build_key is not None - and built.build_key.payout_append_invalidation_epoch - == self._payout_ledger_append_invalidation_epoch - ) - ) - ) - with self.lock: - published_snapshot = getattr( - self, - "tip_template_snapshot", - None, - ) - published_artifacts = ( - published_snapshot.template_artifacts - if published_snapshot is not None - else None - ) - if ( - payout_current - and built.template is artifacts.template - and built.template_generation == artifacts.generation - and ( - not retry_superseded - or published_artifacts is artifacts - ) - ): - # Snapshot-owned work may outlive an unrelated global - # cache fill. Return it only to its refresh validator or - # retained collection delivery; never retain it globally. - if built.prepared_ledger_artifact is not None: - self._install_payout_ledger_artifact( - built.prepared_ledger_artifact - ) - return built - if retry_superseded: - with self._job_cache_lock: - current = self._template_artifacts - if current is artifacts: - continue - raise JobBuildSuperseded( - "job build key changed before cache publication" - ) - return built - - def _share_window_serialization_for_artifact( - self, - payout_artifact: PayoutLedgerArtifact, - shares: list[dict[str, object]], - ) -> _ShareWindowSerialization: - """Cached digest and builder fragments for the artifact's share window. - - Single-slot cache keyed by the canonical content digest, share count, - and window weight. A debounced payout generation can retag an - unchanged window, so generation identity would unnecessarily rebuild - and re-spool the same multi-megabyte compact payload. The compute - deliberately takes no cancellation checkpoints -- the result outlives - any one requester and is served to whichever build wins next. - """ - self._ensure_job_cache_state() - window_weight = ( - PRISM_REWARD_WINDOW_MULTIPLIER - * PRISM_SNAPSHOT_WINDOW_MARGIN - * int(payout_artifact.network_difficulty) - ) - # The digest is computed under the slot lock on purpose: the two - # bounded build flights can request the same window concurrently, and - # letting both walk the share tree would duplicate exactly the - # GIL-heavy pass this cache removes. The loser waits briefly and - # reuses the winner's entry; a different-key waiter is serialized - # behind at most one O(window) pass. - with self._share_window_serialization_lock: - cached = self._share_window_serialization - if ( - payout_artifact.share_snapshot_sha256 is None - and cached is not None - and cached._source_artifact is payout_artifact - and cached.share_count == len(shares) - and cached.key[2] == window_weight - ): - return cached - share_snapshot_sha256 = ( - payout_artifact.share_snapshot_sha256 - or canonical_json_sha256(shares) - ) - key = (share_snapshot_sha256, len(shares), window_weight) - if ( - cached is not None - and cached.key == key - and cached.share_count == len(shares) - ): - cached._source_artifact = payout_artifact - return cached - serialization = _ShareWindowSerialization( - key=key, - share_count=len(shares), - share_snapshot_sha256=share_snapshot_sha256, - _source_artifact=payout_artifact, - ) - self._share_window_serialization = serialization - if cached is not None: - # Generation rotation retires the replaced spool; a build still - # holding a lease keeps the descriptor alive until it releases. - cached.retire_spool() - return serialization - - def build_shared_job_bundle( - self, - artifacts: CachedTemplateArtifacts, - worker: WorkerIdentity | None = None, - *, - mode: str | None = None, - payout_state_generation: int | None = None, - payout_artifact: PayoutLedgerArtifact | None = None, - key: tuple[object, ...] | None = None, - build_request: _JobBuildRequest | None = None, - ) -> CachedJobBundle: - phases = self._job_build_phases() - resolved_mode = self._job_bundle_mode(mode) - if resolved_mode == "collection" and worker is None: - raise CollectionIdentityUnavailable( - "collection-mode worker identity is temporarily unavailable" - ) - with self._job_cache_lock: - publication_blocked = self._payout_state_publication_blocked - if payout_state_generation is None: - payout_state_generation = self._payout_state_generation - if publication_blocked: - raise _PayoutStatePublicationBlocked( - "payout state invalidation is pending publication" - ) - if key is None: - key = self._job_bundle_key( - artifacts, - mode=resolved_mode, - payout_state_generation=payout_state_generation, - payout_artifact_generation=( - payout_artifact.generation - if payout_artifact is not None - else 0 - ), - worker=worker, - ) - if build_request is None: - build_request = self._new_job_build_request( - artifacts, - worker, - mode=resolved_mode, - payout_state_generation=payout_state_generation, - cache_key=key, - payout_ledger_artifact=payout_artifact, - ) - else: - payout_artifact = build_request.payout_ledger_artifact - cancellation = build_request.cancellation - self._job_build_checkpoint("ledger_snapshot", cancellation) - template_value = json.loads(build_request.template_json) - if not isinstance(template_value, dict): - raise RuntimeError("immutable job template is not an object") - template: dict[str, Any] = template_value - issued_at_ms = build_request.key.issued_at_ms - started = time.monotonic() - snapshot_window_weight = ( - PRISM_REWARD_WINDOW_MULTIPLIER - * PRISM_SNAPSHOT_WINDOW_MARGIN - * int(build_request.key.network_difficulty) - ) - prepared_ledger_artifact: PayoutLedgerArtifact | None = None - snapshot_accepted_count: int | None = None - inflight_scan_anchor_token: int | None = None - if payout_artifact is not None: - # The reuse decision was made at request preparation. Nothing - # that happens to the ARMED slot afterwards -- a fresher window - # installing (generation re-key), a same-window anchor refresh - # swapping the instance, or the artifact simply aging out of new - # reuse decisions -- invalidates the copy this build already - # selected: its window stays audit-reproducible at its declared - # anchor. Scrapping admitted work on any of those (as the first - # anchor-scoped deploy did on the generation re-key) turns - # routine churn into a rebuild storm. Only two fences gate - # completion: the payout generation and the published balances - # this artifact binds. - with self._job_cache_lock: - current_payout_state_generation = self._payout_state_generation - published_artifact = self._published_payout_state.artifact - current_append_invalidation_epoch = ( - self._payout_ledger_append_invalidation_epoch - ) - if current_payout_state_generation != payout_state_generation: - raise JobBuildSuperseded( - "payout generation changed before construction" - ) - if ( - payout_artifact.append_invalidation_epoch - != current_append_invalidation_epoch - ): - raise JobBuildSuperseded( - "payout window invalidated before construction" - ) - if published_artifact is None: - try: - published_artifact = self._current_payout_state_artifact() - except Exception as exc: - raise JobBuildSuperseded( - "published payout state unavailable before construction" - ) from exc - artifact_balances_sha256 = ( - payout_artifact.prior_balances_sha256 - or canonical_json_sha256(payout_artifact.prior_balances) - ) - if ( - artifact_balances_sha256 - != published_artifact.prior_balances_sha256 - ): - raise JobBuildSuperseded( - "precomputed payout artifact changed before construction" - ) - prior_balances = list(payout_artifact.prior_balances) - if ( - artifact_balances_sha256 - != build_request.key.payout_artifact_sha256 - ): - raise JobBuildSuperseded( - "precomputed payout artifact does not match payout generation" - ) - shares = list(payout_artifact.shares_json) - # The artifact binds the key above; the balances actually used may - # still be an accepted parent's prospective carry state. - prior_balances = self._prior_balances_for_job_parent( - str(template["previousblockhash"]), - parent_height=int(template["height"]) - 1, - fallback_balances=prior_balances, - ) - else: - with self._payout_state_prepare_lock: - with self._job_cache_lock: - published_artifact = self._published_payout_state.artifact - if self._payout_state_publication_blocked: - raise _PayoutStatePublicationBlocked( - "payout state invalidation is pending publication" - ) - if ( - payout_state_generation != self._payout_state_generation - or build_request.key.payout_append_invalidation_epoch - != self._payout_ledger_append_invalidation_epoch - or published_artifact is None - or published_artifact.prior_balances_sha256 - != build_request.key.payout_artifact_sha256 - ): - raise JobBuildSuperseded( - "payout generation changed before ledger snapshot" - ) - if resolved_mode == "ready": - try: - accepted_now, _ = self.accepted_share_stats() - except Exception: - # Artifact seeding is speculative; snapshot errors - # stay owned by the bundle build itself. Without an - # observed count the seed below is skipped rather - # than armed with a fabricated one. - accepted_now = None - if accepted_now is not None: - # Informational only: the window is scoped by the - # frozen anchor (shares committing during the read - # land above it and belong to the next window), so - # no bracket around the read is needed. - snapshot_accepted_count = int(accepted_now) - if resolved_mode == "ready": - # Publish the read's frozen anchor: a late-visible - # append committing mid-read predates it without - # holding the pending floor, and this read's database - # snapshot may already exclude the row. The recorded - # invalidation epoch keeps the resulting bundle and - # its seeded artifact from arming or serving. The - # exposure must outlive this read: nothing else exposes - # this window's anchor until the seeded artifact passes - # its install fence, so an append committing during - # bundle construction or cache publication would - # otherwise advance no epoch and the seed would arm a - # window that delta reads can never rediscover the row - # from. The token therefore rides the seeded artifact - # (retired by its install fence); a build that raises, - # or seeds nothing, retires it on the spot. - inflight_scan_anchor_token = ( - self._expose_inflight_scan_anchor(issued_at_ms) - ) - # Mirrored onto the request so the executor wrapper can - # retire the exposure when this build dies anywhere past - # the read (checkpoint cancellation, manifest assembly, - # the publication fence). - build_request.inflight_scan_anchor_token = ( - inflight_scan_anchor_token - ) - try: - records = self.ledger.snapshot_at_job_issue( - issued_at_ms, - window_weight=snapshot_window_weight, - ) - except BaseException: - self._retire_inflight_scan_anchor( - inflight_scan_anchor_token - ) - raise - else: - records = [] - # An accepted parent's prospective carry state supersedes the - # published artifact for children built on that parent; the - # published balances remain the fallback for ordinary tips. - prior_balances = self._prior_balances_for_job_parent( - str(template["previousblockhash"]), - parent_height=int(template["height"]) - 1, - fallback_balances=build_request.payout_artifact.prior_balances(), - ) - self._job_build_checkpoint("ledger_snapshot_complete", cancellation) - shares = [] - for index, record in enumerate(records): - if index % 256 == 0: - self._job_build_checkpoint( - "ledger_snapshot_conversion", - cancellation, - ) - shares.append(record.to_prism_json()) - # A reused artifact carries shares snapshotted at its own earlier - # anchor. The bundle must declare that anchor: replaying the audit - # window at this job's fresher anchor could include a share that was - # already durable at artifact build time but stamped after the - # artifact's clamped anchor, and the artifact's share set excludes it - # by construction. - bundle_anchor_ms = ( - payout_artifact.snapshot_anchor_ms - if payout_artifact is not None - and payout_artifact.snapshot_anchor_ms is not None - else issued_at_ms - ) - ledger_elapsed = time.monotonic() - started - phases["ledger"] = phases.get("ledger", 0.0) + ledger_elapsed - if resolved_mode == "ready": - self._observe_tip_refresh_build_phase( - "ledger_snapshot", - ledger_elapsed, - ) - share_serialization: _ShareWindowSerialization | None = None - if resolved_mode == "ready" and payout_artifact is not None: - share_serialization = self._share_window_serialization_for_artifact( - payout_artifact, - shares, - ) - share_snapshot_sha256 = share_serialization.share_snapshot_sha256 - else: - share_snapshot_sha256 = canonical_json_sha256(shares) - if ( - payout_artifact is None - and snapshot_accepted_count is not None - and shares - and self._payout_artifact_reuse_active() - ): - # This build already paid the full ledger read; carry the window - # it produced so cache publication can arm it for builds arriving - # while its anchor stays inside the staleness bound. The artifact - # carries the published payout-state balances, not this bundle's - # possibly parent-adjusted view: reuse re-applies the parent - # override itself and the reuse fence hashes the artifact - # balances against the published payout artifact. Skipped - # entirely under the kill-switch: nothing may arm while reuse - # is disabled. - seed_prior_balances = tuple( - build_request.payout_artifact.prior_balances() - ) - prepared_ledger_artifact = PayoutLedgerArtifact( - generation=0, - payout_state_generation=payout_state_generation, - network_difficulty=int(build_request.key.network_difficulty), - accepted_share_count=snapshot_accepted_count, - shares_json=tuple(shares), - prior_balances=seed_prior_balances, - prepared_monotonic=time.monotonic(), - snapshot_anchor_ms=issued_at_ms, - share_snapshot_sha256=share_snapshot_sha256, - prior_balances_sha256=canonical_json_sha256( - seed_prior_balances - ), - append_invalidation_epoch=( - build_request.key.payout_append_invalidation_epoch - ), - inflight_scan_anchor_token=inflight_scan_anchor_token, - ) - final_build_key = dataclass_replace( - build_request.key, - share_snapshot_sha256=share_snapshot_sha256, - ) - self._job_build_checkpoint("payout_derivation", cancellation) - started = time.monotonic() - placeholder_suffix_hex = final_build_key.coinbase_suffix_hex - collection_identity: tuple[str, str] | None = None - previous_metrics_scope = bool( - getattr(self._job_build_phase_local, "tip_refresh_metrics", False) + return self._ensure_job_bundle_service()._shared_job_bundle_after_priority_admission( + artifacts, + worker, + mode=mode, + cancelled=cancelled, + retry_superseded=retry_superseded, + idle_retarget=idle_retarget, + publication_critical=publication_critical, + request_source=request_source, + priority_requested_monotonic=priority_requested_monotonic, ) - self._job_build_phase_local.tip_refresh_metrics = resolved_mode == "ready" - try: - if resolved_mode == "ready": - if not shares: - raise RuntimeError( - "ready-pool ledger snapshot contained no payout shares" - ) - self._job_build_checkpoint("ctv_manifest", cancellation) - self._job_build_checkpoint("signing_verification", cancellation) - bundle = self.build_audit_bundle( - shares=shares, - found_block={ - "block_height": int(template["height"]), - "coinbase_value_sats": int(template["coinbasevalue"]), - "network_difficulty": artifacts.network_difficulty, - "anchor_job_issued_at_ms": bundle_anchor_ms, - }, - prior_balances=prior_balances, - coinbase_script_sig_suffix_hex=placeholder_suffix_hex, - witness_merkle_leaves_hex=list( - build_request.witness_merkle_leaves_hex - ), - ctv_fee_parent_hash=str(template["previousblockhash"]), - summary_only=True, - payout_policy=json.loads(build_request.payout_policy_json), - ctv_settlement=( - json.loads(build_request.ctv_settlement_json) - if build_request.ctv_settlement_json is not None - else None - ), - cancellation=cancellation, - share_serialization=share_serialization, - ) - collection_only = False - else: - assert worker is not None - self._job_build_checkpoint("ctv_manifest", cancellation) - self._job_build_checkpoint("signing_verification", cancellation) - bundle = self.build_collection_bundle( - template=template, - transaction_hexes=build_request.transaction_hexes, - worker=worker, - network_difficulty=final_build_key.network_difficulty, - issued_at_ms=issued_at_ms, - suffix_hex=placeholder_suffix_hex, - summary_only=True, - payout_policy=json.loads(build_request.payout_policy_json), - ctv_settlement=( - json.loads(build_request.ctv_settlement_json) - if build_request.ctv_settlement_json is not None - else None - ), - cancellation=cancellation, - ) - shares = [] - collection_only = True - collection_identity = self._collection_bundle_identity(worker) - finally: - self._job_build_phase_local.tip_refresh_metrics = previous_metrics_scope - manifest = bundle["signed_coinbase_manifest"]["manifest"] - prospective_prior_balances: ( - tuple[tuple[str, str, str, int], ...] | None - ) = None - payout_policy_manifest = bundle.get("payout_policy_manifest") - if isinstance(payout_policy_manifest, dict) and isinstance( - payout_policy_manifest.get("accounts"), - list, - ): - prospective_prior_balances = self._serialize_prior_balance_preview( - self._accepted_block_payout_preview_from_bundle( - bundle, - prior_balances=prior_balances, - ) - ) - self._job_build_checkpoint("bundle_assembly", cancellation) - assembly_started = time.monotonic() - base_job = direct_stratum.make_job_from_builder_manifest( - job_id="prism-template-base", - template=template, - manifest=manifest, - extranonce1_hex=PRISM_JOB_EXTRANONCE1_PLACEHOLDER_HEX, - extranonce2_size=self.extranonce2_size, - desired_share_difficulty=self.share_difficulty, - clean_jobs=True, - transaction_hexes=build_request.transaction_hexes, - ) - phases["assembly"] = phases.get("assembly", 0.0) + ( - time.monotonic() - assembly_started - ) - self._job_build_checkpoint("serialization", cancellation) - self._job_build_checkpoint("bundle_publication", cancellation) - if resolved_mode == "ready": - with self._job_cache_lock: - if ( - final_build_key.payout_append_invalidation_epoch - != self._payout_ledger_append_invalidation_epoch - ): - raise JobBuildSuperseded( - "payout window invalidated before bundle publication" - ) - if ( - inflight_scan_anchor_token is not None - and prepared_ledger_artifact is None - ): - # The fence above just proved, under this same lock hold, - # that no append predated this build's window. Seeded - # builds keep their exposure until the artifact's - # install fence settles; with no seed, nothing else - # exposes this window's anchor once the walk exposure - # retires, yet jobs stamped from this bundle keep - # serving the window until they retire. Hand the anchor - # to the published-window watermark under this same lock - # hold, so a replay-shaped append committing after this - # return still predates a live anchor and advances the - # epoch those jobs' landing fences check. - self._publish_seedless_job_window_anchor_locked( - issued_at_ms - ) - self._payout_window_inflight_scan_anchors.pop( - inflight_scan_anchor_token, - None, - ) - inflight_scan_anchor_token = None - build_request.inflight_scan_anchor_token = None - phases["bundle"] = phases.get("bundle", 0.0) + (time.monotonic() - started) - return CachedJobBundle( - key=key, - # Construction consumed the detached canonical template above; - # publication retains the exact snapshot-owned object identity used - # by the existing validation token. - template=artifacts.template, - template_fingerprint=artifacts.fingerprint, - # Only this manifest is needed to bind later clock-only template - # observations. Retaining the returned logical bundle duplicated - # the entire shares tree already held in shares_json. - coinbase_manifest=manifest, - shares_json=shares, - prior_balances=prior_balances, - found_block=bundle["found_block"], - collection_only=collection_only, - issued_at_ms=issued_at_ms, - base_job=base_job, - built_monotonic=time.monotonic(), - template_generation=artifacts.generation, + + def _share_window_serialization_for_artifact( + self, + payout_artifact: PayoutLedgerArtifact, + shares: list[dict[str, object]], + ) -> _ShareWindowSerialization: + """Cached digest and builder fragments for the artifact's share window.""" + return self._ensure_job_bundle_service()._share_window_serialization_for_artifact( + payout_artifact, + shares, + ) + + def build_shared_job_bundle( + self, + artifacts: CachedTemplateArtifacts, + worker: WorkerIdentity | None = None, + *, + mode: str | None = None, + payout_state_generation: int | None = None, + payout_artifact: PayoutLedgerArtifact | None = None, + key: tuple[object, ...] | None = None, + build_request: _JobBuildRequest | None = None, + ) -> CachedJobBundle: + return self._ensure_job_bundle_service().build_shared_job_bundle( + artifacts, + worker, + mode=mode, payout_state_generation=payout_state_generation, - payout_artifact_generation=( - payout_artifact.generation if payout_artifact is not None else 0 - ), - collection_identity=collection_identity, - prospective_prior_balances=prospective_prior_balances, - build_key=final_build_key, - prepared_ledger_artifact=prepared_ledger_artifact, + payout_artifact=payout_artifact, + key=key, + build_request=build_request, ) def stamp_job_for_client( @@ -10788,75 +4905,109 @@ def stamp_job_for_client( clean_jobs: bool, tip_refresh_epoch_sequence: int | None = None, ) -> PrismJobContext: - if client.worker is None: - raise StratumError(20, "client is not authorized") - if cached.collection_only and cached.collection_identity != ( - self._collection_bundle_identity(client.worker) - ): - raise StratumError( - 20, - "collection bundle payout identity no longer matches client authorization", - ) - with self.lock: - self.job_counter += 1 - job_id = f"prism-{self.job_counter}" - if tip_refresh_epoch_sequence is None: - tip_refresh_epoch_sequence = ( - self._tip_refresh_epoch_for_bundle_locked(cached) - ) - share_target = direct_stratum.effective_share_target( - self.desired_client_share_difficulty(client), - cached.base_job.qbit_target, - minimum_advertised_difficulty=self.client_minimum_advertised_difficulty(client), - ) - job = dataclass_replace( - cached.base_job, - job_id=job_id, - extranonce1_hex=client.extranonce1_hex, - share_target=share_target, - share_difficulty=direct_stratum.target_difficulty(share_target), + return self._ensure_job_delivery_service().stamp_job_for_client( + client, + cached, clean_jobs=clean_jobs, + tip_refresh_epoch_sequence=tip_refresh_epoch_sequence, ) - return PrismJobContext( - job=job, - template=cached.template, - shares_json=cached.shares_json, - prior_balances=cached.prior_balances, - found_block=cached.found_block, - share_weight=self.share_weight_for_worker(client.worker), - collection_only=cached.collection_only, - worker=client.worker, - issued_at_ms=cached.issued_at_ms, - template_fingerprint=cached.template_fingerprint, - template_generation=cached.template_generation, - payout_state_generation=cached.payout_state_generation, - prospective_prior_balances=cached.prospective_prior_balances, - payout_artifact_generation=cached.payout_artifact_generation, - payout_append_invalidation_epoch=( - int( - getattr( - cached.build_key, - "payout_append_invalidation_epoch", - 0, - ) - ) - if cached.build_key is not None - else 0 - ), - payout_artifact_sha256=( - cached.build_key.payout_artifact_sha256 - if cached.build_key is not None - else None - ), - connection_id=client.connection_id, - authorization_generation=int( - getattr(client, "authorization_generation", 0) - ), - difficulty_generation=int( - getattr(client, "difficulty_generation", 0) - ), - tip_refresh_epoch_sequence=int(tip_refresh_epoch_sequence), - version_mask=int(getattr(client, "version_mask", 0)), + + def _current_payout_generation(self) -> int: + """Narrow P1 adapter: the current payout-state generation.""" + return int(self._ensure_payout_state_service()._payout_state_generation) + + def _payout_delivery_snapshot(self) -> PayoutStateSnapshot: + """Narrow P1 adapter: copied identity facts for delivery decisions.""" + return self._ensure_payout_state_service().snapshot() + + def _payout_delivery_cancelable( + self, + cancelled: Callable[[], bool], + *, + generation: int, + priority: bool, + ) -> Any: + """Narrow P1 adapter: cancellable delivery-gate admission.""" + gate = self._ensure_payout_state_service()._payout_state_delivery_gate + return gate.delivery_cancelable( + cancelled, + generation=generation, + priority=priority, + ) + + def _job_delivery_observe_payout_admission( + self, + admission: Any, + *, + generation: int, + fallback_wait_seconds: float, + ) -> None: + """Narrow P1 adapter: record a delivery-gate admission observation.""" + self._observe_payout_gate_admission( + admission, + generation=generation, + fallback_wait_seconds=fallback_wait_seconds, + ) + + def _payout_template_network_difficulty(self) -> int | None: + """Narrow J1 adapter for P1: current template network difficulty.""" + service = self.__dict__.get("job_bundle_service") + if service is None: + return None + artifacts = service.template_repository.current_artifacts() + return None if artifacts is None else artifacts.network_difficulty + + def _invalidate_payout_job_cache(self) -> None: + """Narrow J1 adapter for P1: drop cached bundles after invalidation.""" + self._ensure_job_cache_state() + with self._job_cache_lock: + self._job_bundle_cache.clear() + + def _on_payout_state_invalidated( + self, + generation: int, + invalidated_monotonic: float, + ) -> None: + """Narrow G1/R1 adapter for P1: a payout generation was invalidated. + + The verbatim publication bodies keep their inline calls; this seam + exists for later stack layers that consume P1 through ports. + """ + self._record_progress_payout_generation( + generation, + invalidated_monotonic, + ) + + def _on_payout_state_published( + self, + generation: int, + invalidated_monotonic: float, + ) -> None: + """Narrow G1/R1 adapter for P1: a payout generation was published. + + The verbatim publication bodies keep their inline calls; this seam + exists for later stack layers that consume P1 through ports. + """ + self._record_progress_payout_generation( + generation, + invalidated_monotonic, + ) + + def _next_job_delivery_id(self) -> str: + """Allocate the next delivery job id through the S2 owner.""" + return self._ensure_job_delivery_service().next_job_id() + + def _complete_job_delivery( + self, + client: ClientState, + context: PrismJobContext, + delivered_monotonic: float, + ) -> None: + """Commit S2 delivery bookkeeping after a successful socket write.""" + self._ensure_job_delivery_service().complete_delivery( + client, + context, + delivered_monotonic, ) def accepted_share_stats(self) -> tuple[int, int]: @@ -10887,72 +5038,227 @@ def _ensure_watchdog_state(self) -> None: if not hasattr(self, "_watchdog_pauses"): self._watchdog_pauses = {} + def _legacy_ctv_runtime_config(self) -> CtvRuntimeConfig: + coordinator_config = getattr(self, "config", None) + ctv_config = getattr(coordinator_config, "ctv", None) + if ctv_config is not None: + config = CtvRuntimeConfig.from_coordinator_config(ctv_config) + else: + config = CtvRuntimeConfig( + enabled=False, + wallet=None, + fee_sats=0, + limit=100, + chunk_size=DEFAULT_PRISM_CTV_BROADCASTER_CHUNK_SIZE, + interval_seconds=30.0, + ) + overrides = self.__dict__.get("_ctv_runtime_compat_config", {}) + if overrides: + config = dataclass_replace(config, **overrides) + return config + + def _make_ctv_runtime_service( + self, + config: CtvRuntimeConfig | None = None, + ) -> CtvRuntimeService: + runtime = CtvRuntimeService( + rpc_call=lambda *args, **kwargs: self.rpc.call(*args, **kwargs), + ledger_source=lambda: getattr(self, "ledger", None), + writer_admission=lambda component: self._writer_operation(component), + tip_refresh_pending=lambda: self.tip_refresh_is_pending(), + heartbeat=lambda: self._record_heartbeat("ctv_fanout_broadcaster"), + stop_signal=lambda: self.stop_event, + config=self._legacy_ctv_runtime_config() if config is None else config, + before_external_side_effect=( + lambda *args, **kwargs: ( + self._require_fresh_ledger_lease_for_external_side_effect( + *args, **kwargs + ) + ) + ), + shutdown_exception=ShutdownInProgress, + # Resolve the process types through this module's globals at call + # time so existing patch("lab.prism.prism_coordinator....") test + # seams keep intercepting daemon/broadcaster construction. + daemon_type=lambda *args, **kwargs: CtvFanoutBroadcastDaemon( + *args, **kwargs + ), + broadcaster_type=lambda *args, **kwargs: CtvFanoutBroadcaster( + *args, **kwargs + ), + # Preserve temporary coordinator patch points for focused tests. + monotonic=lambda: time.monotonic(), + print_exception=lambda: traceback.print_exc(), + ) + compat_daemon = self.__dict__.pop("_ctv_runtime_compat_daemon", None) + if compat_daemon is not None: + runtime.daemon = compat_daemon + return runtime + + def _ensure_ctv_runtime(self) -> CtvRuntimeService: + runtime = self.__dict__.get("_ctv_runtime") + if runtime is not None: + return runtime + init_lock = self.__dict__.get("_ctv_runtime_init_lock") + if init_lock is None: + # CPython's setdefault is atomic under the GIL. Focused tests may + # construct through __new__, while normal instances install this + # lock in __init__ before any process thread can start. + init_lock = self.__dict__.setdefault( + "_ctv_runtime_init_lock", + threading.Lock(), + ) + with init_lock: + runtime = self.__dict__.get("_ctv_runtime") + if runtime is not None: + return runtime + config = self._legacy_ctv_runtime_config() + runtime = self._make_ctv_runtime_service(config) + self.__dict__["_ctv_runtime"] = runtime + # The service is now the sole configuration owner. Removing the + # pre-init store prevents an old override from being replayed if a + # later compatibility property updates the live service. + self.__dict__.pop("_ctv_runtime_compat_config", None) + return runtime + + def _ctv_runtime_config_value(self, name: str) -> object: + runtime = self.__dict__.get("_ctv_runtime") + if runtime is not None: + return getattr(runtime.config, name) + return getattr(self._legacy_ctv_runtime_config(), name) + + def _set_ctv_runtime_config_value(self, name: str, value: object) -> None: + runtime = self.__dict__.get("_ctv_runtime") + if runtime is not None: + runtime.replace_config(**{name: value}) + return + init_lock = self.__dict__.get("_ctv_runtime_init_lock") + if init_lock is None: + init_lock = self.__dict__.setdefault( + "_ctv_runtime_init_lock", + threading.Lock(), + ) + with init_lock: + runtime = self.__dict__.get("_ctv_runtime") + if runtime is not None: + runtime.replace_config(**{name: value}) + return + overrides = self.__dict__.setdefault("_ctv_runtime_compat_config", {}) + overrides[name] = value + + @property + def ctv_broadcaster_enabled(self) -> bool: + return bool(self._ctv_runtime_config_value("enabled")) + + @ctv_broadcaster_enabled.setter + def ctv_broadcaster_enabled(self, value: bool) -> None: + self._set_ctv_runtime_config_value("enabled", bool(value)) + + @property + def ctv_broadcaster_wallet(self) -> str | None: + value = self._ctv_runtime_config_value("wallet") + return None if value is None else str(value) + + @ctv_broadcaster_wallet.setter + def ctv_broadcaster_wallet(self, value: str | None) -> None: + self._set_ctv_runtime_config_value("wallet", value) + + @property + def ctv_broadcaster_fee_sats(self) -> int: + return int(self._ctv_runtime_config_value("fee_sats")) + + @ctv_broadcaster_fee_sats.setter + def ctv_broadcaster_fee_sats(self, value: int) -> None: + self._set_ctv_runtime_config_value("fee_sats", int(value)) + + @property + def ctv_broadcaster_limit(self) -> int: + return int(self._ctv_runtime_config_value("limit")) + + @ctv_broadcaster_limit.setter + def ctv_broadcaster_limit(self, value: int) -> None: + self._set_ctv_runtime_config_value("limit", int(value)) + + @property + def ctv_broadcaster_chunk_size(self) -> int: + return int(self._ctv_runtime_config_value("chunk_size")) + + @ctv_broadcaster_chunk_size.setter + def ctv_broadcaster_chunk_size(self, value: int) -> None: + self._set_ctv_runtime_config_value("chunk_size", int(value)) + + @property + def ctv_broadcaster_interval_seconds(self) -> float: + return float(self._ctv_runtime_config_value("interval_seconds")) + + @ctv_broadcaster_interval_seconds.setter + def ctv_broadcaster_interval_seconds(self, value: float) -> None: + self._set_ctv_runtime_config_value("interval_seconds", float(value)) + + @property + def ctv_fanout_broadcast_daemon(self) -> CtvFanoutBroadcastDaemon | None: + runtime = self.__dict__.get("_ctv_runtime") + if runtime is not None: + return runtime.daemon + init_lock = self.__dict__.get("_ctv_runtime_init_lock") + if init_lock is None: + init_lock = self.__dict__.setdefault( + "_ctv_runtime_init_lock", + threading.Lock(), + ) + with init_lock: + runtime = self.__dict__.get("_ctv_runtime") + if runtime is None: + return self.__dict__.get("_ctv_runtime_compat_daemon") + return runtime.daemon + + @ctv_fanout_broadcast_daemon.setter + def ctv_fanout_broadcast_daemon( + self, + daemon: CtvFanoutBroadcastDaemon | None, + ) -> None: + runtime = self.__dict__.get("_ctv_runtime") + if runtime is not None: + runtime.daemon = daemon + return + init_lock = self.__dict__.get("_ctv_runtime_init_lock") + if init_lock is None: + init_lock = self.__dict__.setdefault( + "_ctv_runtime_init_lock", + threading.Lock(), + ) + with init_lock: + runtime = self.__dict__.get("_ctv_runtime") + if runtime is None: + self.__dict__["_ctv_runtime_compat_daemon"] = daemon + else: + runtime.daemon = daemon + + @property + def ctv_broadcaster_processed_rows_total(self) -> int: + return self._ensure_ctv_runtime().processed_rows_total + + @property + def ctv_broadcaster_pass_count(self) -> int: + return self._ensure_ctv_runtime().pass_count + def _ensure_ctv_broadcaster_metrics_state(self) -> None: - if not hasattr(self, "_ctv_broadcaster_metrics_lock"): - self._ctv_broadcaster_metrics_lock = threading.Lock() - if not hasattr(self, "ctv_broadcaster_pass_seconds_bucket_counts"): - self.ctv_broadcaster_pass_seconds_bucket_counts = { - bucket: 0 for bucket in PRISM_CTV_BROADCASTER_SECONDS_BUCKETS - } - if not hasattr(self, "ctv_broadcaster_pass_seconds_sum"): - self.ctv_broadcaster_pass_seconds_sum = 0.0 - if not hasattr(self, "ctv_broadcaster_pass_count"): - self.ctv_broadcaster_pass_count = 0 - if not hasattr(self, "ctv_broadcaster_processed_rows_total"): - self.ctv_broadcaster_processed_rows_total = 0 - if not hasattr(self, "ctv_broadcaster_yielded_total"): - self.ctv_broadcaster_yielded_total = 0 - if not hasattr(self, "ctv_broadcaster_chunk_seconds_bucket_counts"): - self.ctv_broadcaster_chunk_seconds_bucket_counts = { - bucket: 0 for bucket in PRISM_CTV_BROADCASTER_CHUNK_SECONDS_BUCKETS - } - if not hasattr(self, "ctv_broadcaster_chunk_rows_bucket_counts"): - self.ctv_broadcaster_chunk_rows_bucket_counts = { - bucket: 0 for bucket in PRISM_CTV_BROADCASTER_CHUNK_ROWS_BUCKETS - } - if not hasattr(self, "ctv_broadcaster_chunk_seconds_sum"): - self.ctv_broadcaster_chunk_seconds_sum = 0.0 - if not hasattr(self, "ctv_broadcaster_chunk_rows_sum"): - self.ctv_broadcaster_chunk_rows_sum = 0 - if not hasattr(self, "ctv_broadcaster_chunk_count"): - self.ctv_broadcaster_chunk_count = 0 + self._ensure_ctv_runtime() def _record_ctv_fanout_broadcaster_progress(self) -> None: - self._record_heartbeat("ctv_fanout_broadcaster") - self._ensure_ctv_broadcaster_metrics_state() - with self._ctv_broadcaster_metrics_lock: - self.ctv_broadcaster_processed_rows_total += 1 + self._ensure_ctv_runtime().record_progress() def observe_ctv_fanout_broadcaster_pass(self, elapsed_seconds: float) -> None: - self._ensure_ctv_broadcaster_metrics_state() - with self._ctv_broadcaster_metrics_lock: - self.ctv_broadcaster_pass_count += 1 - self.ctv_broadcaster_pass_seconds_sum += elapsed_seconds - for bucket in PRISM_CTV_BROADCASTER_SECONDS_BUCKETS: - if elapsed_seconds <= bucket: - self.ctv_broadcaster_pass_seconds_bucket_counts[bucket] += 1 + self._ensure_ctv_runtime().observe_pass(elapsed_seconds) def observe_ctv_fanout_broadcaster_chunk( self, result: CtvFanoutChunkResult, ) -> None: - self._record_heartbeat("ctv_fanout_broadcaster") - self._ensure_ctv_broadcaster_metrics_state() - with self._ctv_broadcaster_metrics_lock: - self.ctv_broadcaster_chunk_count += 1 - self.ctv_broadcaster_chunk_seconds_sum += result.elapsed_seconds - self.ctv_broadcaster_chunk_rows_sum += result.processed_count - for bucket in PRISM_CTV_BROADCASTER_CHUNK_SECONDS_BUCKETS: - if result.elapsed_seconds <= bucket: - self.ctv_broadcaster_chunk_seconds_bucket_counts[bucket] += 1 - for bucket in PRISM_CTV_BROADCASTER_CHUNK_ROWS_BUCKETS: - if result.processed_count <= bucket: - self.ctv_broadcaster_chunk_rows_bucket_counts[bucket] += 1 + self._ensure_ctv_runtime().observe_chunk(result) def _record_ctv_fanout_broadcaster_yield(self) -> None: - self._ensure_ctv_broadcaster_metrics_state() - with self._ctv_broadcaster_metrics_lock: - self.ctv_broadcaster_yielded_total += 1 + self._ensure_ctv_runtime().record_yield() def _ensure_worker_metrics_state(self) -> None: if not hasattr(self, "worker_metrics_lock"): @@ -10963,8 +5269,11 @@ def _ensure_worker_metrics_state(self) -> None: self.worker_rejection_counts = {} def _ensure_initial_job_state(self) -> None: - if not hasattr(self, "pending_initial_jobs"): - self.pending_initial_jobs: dict[ClientState, PendingInitialJob] = {} + # S2 initial-job state lives in its owner service; the legacy raw + # fields route there through class descriptors and the service adopts + # direct test assignments through those descriptors. Only the live + # configuration attributes and S1/health fields remain seeded here. + self._ensure_job_delivery_service() if not hasattr(self, "stratum_max_pending_initial_jobs"): self.stratum_max_pending_initial_jobs = ( DEFAULT_PRISM_STRATUM_MAX_PENDING_INITIAL_JOBS @@ -10975,42 +5284,10 @@ def _ensure_initial_job_state(self) -> None: ) if not hasattr(self, "initial_job_max_workers"): self.initial_job_max_workers = DEFAULT_PRISM_INITIAL_JOB_MAX_WORKERS - if not hasattr(self, "_initial_job_executor_lock"): - self._initial_job_executor_lock = threading.Lock() - if not hasattr(self, "_initial_job_executor"): - self._initial_job_executor: _BoundedPriorityExecutor | None = None - if not hasattr(self, "_initial_job_executor_shutdown"): - self._initial_job_executor_shutdown = False - if not hasattr(self, "initial_job_queue_rejection_count"): - self.initial_job_queue_rejection_count = 0 - if not hasattr(self, "initial_job_timeout_count"): - self.initial_job_timeout_count = 0 - if not hasattr(self, "initial_job_cancelled_count"): - self.initial_job_cancelled_count = 0 - if not hasattr(self, "initial_job_coalesced_count"): - self.initial_job_coalesced_count = 0 - if not hasattr(self, "initial_job_queue_capacity_reclaimed_count"): - self.initial_job_queue_capacity_reclaimed_count = 0 - if not hasattr(self, "initial_job_sent_count"): - self.initial_job_sent_count = 0 - if not hasattr(self, "initial_job_failed_count"): - self.initial_job_failed_count = 0 - if not hasattr(self, "initial_job_superseded_count"): - self.initial_job_superseded_count = 0 - if not hasattr(self, "initial_job_delivery_latency_seconds_sum"): - self.initial_job_delivery_latency_seconds_sum = 0.0 - if not hasattr(self, "initial_job_delivery_latency_count"): - self.initial_job_delivery_latency_count = 0 - if not hasattr(self, "last_initial_job_delivery_monotonic"): - self.last_initial_job_delivery_monotonic = None if not hasattr(self, "handler_thread_count"): self.handler_thread_count = 0 if not hasattr(self, "peak_active_connection_count"): self.peak_active_connection_count = len(getattr(self, "clients", ())) - if not hasattr(self, "_mining_overload_started_monotonic"): - self._mining_overload_started_monotonic = None - if not hasattr(self, "_mining_delivery_failure_started_monotonic"): - self._mining_delivery_failure_started_monotonic = None def delivery_queue_limit(self) -> int: pending_limit = int( @@ -11034,165 +5311,38 @@ def delivery_queue_limit(self) -> int: ) def _ensure_tip_refresh_state(self) -> None: - if not hasattr(self, "_tip_refresh_lock"): - self._tip_refresh_lock = threading.Lock() - if not hasattr(self, "_tip_refresh_singleflight_lock"): - self._tip_refresh_singleflight_lock = threading.Lock() - if not hasattr(self, "_tip_refresh_executor_lock"): - self._tip_refresh_executor_lock = threading.Lock() - if not hasattr(self, "_tip_refresh_executor"): - self._tip_refresh_executor: _BoundedPriorityExecutor | None = None - if not hasattr(self, "_tip_refresh_executor_shutdown"): - self._tip_refresh_executor_shutdown = False + # R1 owner state lives in TipRefreshService; the legacy raw fields + # route there through class descriptors and the service adopts direct + # test assignments through those descriptors. Only the live + # configuration attributes and the two historically seeded lazy + # fields remain here. + self._ensure_tip_refresh_service() if not hasattr(self, "tip_refresh_max_workers"): self.tip_refresh_max_workers = DEFAULT_PRISM_TIP_REFRESH_MAX_WORKERS if not hasattr(self, "tip_refresh_epoch_fanout"): self.tip_refresh_epoch_fanout = False - if not hasattr(self, "_tip_refresh_metrics_lock"): - self._tip_refresh_metrics_lock = threading.Lock() - if not hasattr(self, "tip_refresh_histograms"): - self.tip_refresh_histograms = { - name: { - "buckets": {bucket: 0 for bucket in PRISM_TIP_REFRESH_SECONDS_BUCKETS}, - "sum": 0.0, - "count": 0, - } - for name in ( - "refresh", - "bundle_build", - "first_delivery", - "last_delivery", - "fanout_wave", - ) - } - if not hasattr(self, "tip_refresh_build_phase_histograms"): - self.tip_refresh_build_phase_histograms = { - phase: { - "buckets": { - bucket: 0 for bucket in PRISM_TIP_REFRESH_SECONDS_BUCKETS - }, - "sum": 0.0, - "count": 0, - } - for phase in PRISM_TIP_REFRESH_BUILD_PHASES - } - if not hasattr(self, "tip_refresh_client_counts"): - self.tip_refresh_client_counts = { - result: 0 for result in PRISM_TIP_REFRESH_RESULTS - } - if not hasattr(self, "tip_refresh_cancellation_counts"): - self.tip_refresh_cancellation_counts = { - stage: 0 for stage in PRISM_TIP_REFRESH_CANCELLATION_STAGES - } - if not hasattr(self, "tip_refresh_wave_outcome_counts"): - self.tip_refresh_wave_outcome_counts = { - outcome: 0 for outcome in PRISM_TIP_REFRESH_WAVE_OUTCOMES - } - if not hasattr(self, "tip_refresh_coverage_histograms"): - self.tip_refresh_coverage_histograms = { - target: { - "buckets": { - bucket: 0 for bucket in PRISM_TIP_REFRESH_SECONDS_BUCKETS - }, - "sum": 0.0, - "count": 0, - } - for target, _ratio in PRISM_TIP_REFRESH_COVERAGE_TARGETS - } - if not hasattr(self, "tip_refresh_inflight"): - self.tip_refresh_inflight = 0 - if not hasattr(self, "tip_refresh_build_inflight"): - self.tip_refresh_build_inflight = 0 - if not hasattr(self, "tip_refresh_build_queue_depth"): - self.tip_refresh_build_queue_depth = 0 - if not hasattr(self, "tip_refresh_singleflight_hits"): - self.tip_refresh_singleflight_hits = 0 - if not hasattr(self, "tip_refresh_superseded_results"): - self.tip_refresh_superseded_results = 0 - if not hasattr(self, "tip_refresh_worker_failures"): - self.tip_refresh_worker_failures = 0 - if not hasattr(self, "tip_refresh_worker_restarts"): - self.tip_refresh_worker_restarts = 0 - if not hasattr(self, "tip_refresh_ipc_bytes"): - self.tip_refresh_ipc_bytes = {"input": 0, "output": 0} - if not hasattr(self, "_tip_refresh_pending_event"): - self._tip_refresh_pending_event = threading.Event() - if not hasattr(self, "_tip_refresh_pending_counter"): - self._tip_refresh_pending_counter = 0 - if not hasattr(self, "_tip_refresh_pending_token"): - self._tip_refresh_pending_token: int | None = None - if not hasattr(self, "_tip_refresh_retry"): - self._tip_refresh_retry = threading.Event() - if not hasattr(self, "_tip_refresh_retry_counter"): - self._tip_refresh_retry_counter = 0 - if not hasattr(self, "_tip_refresh_retry_consumed"): - self._tip_refresh_retry_consumed = 0 - if not hasattr(self, "_tip_refresh_failure_holdoff_until"): - self._tip_refresh_failure_holdoff_until: float | None = None - if not hasattr(self, "_tip_refresh_failure_tip"): - self._tip_refresh_failure_tip: str | None = None - if not hasattr(self, "_active_tip_refresh"): - self._active_tip_refresh: tuple[ - TipRefreshValidationToken, - _FanoutCancellation, - ] | None = None - if not hasattr(self, "_tip_refresh_epoch_sequence"): - self._tip_refresh_epoch_sequence = 0 - if not hasattr(self, "_tip_refresh_epoch_tip_hash"): - self._tip_refresh_epoch_tip_hash: str | None = None - if not hasattr(self, "_tip_refresh_epoch_payout_generation"): - self._tip_refresh_epoch_payout_generation = 0 - if not hasattr(self, "_tip_refresh_epoch_coverage"): - self._tip_refresh_epoch_coverage: _TipRefreshEpochCoverage | None = None - if not hasattr(self, "_published_tip_refresh_epoch_identity"): - self._published_tip_refresh_epoch_identity: tuple[ - int, - str, - int, - str, - int, - ] | None = None - if not hasattr(self, "_retained_collection_refresh"): - self._retained_collection_refresh: RetainedCollectionRefresh | None = None if not hasattr(self, "latest_detected_tip"): - self.latest_detected_tip: tuple[str, int] | None = None + self.latest_detected_tip = None if not hasattr(self, "tip_refresh_divergence_started_monotonic"): - self.tip_refresh_divergence_started_monotonic: float | None = None + self.tip_refresh_divergence_started_monotonic = None - @staticmethod - def _tip_refresh_hashrate_proxy(client: ClientState) -> Decimal: - proxy = ( - getattr(client, "vardiff_difficulty_estimate", None) - or getattr(client, "share_difficulty", Decimal(0)) + def _tip_refresh_prune_evicted_jobs(self, now: float, force: bool) -> None: + """Resolve the retained-prune compatibility seam at call time.""" + override = self.__dict__.get("prune_evicted_job_graveyard") + if callable(override): + override(now=now, force=force) + return + self._ensure_job_delivery_service().prune_evicted_job_graveyard( + now=now, + force=force, ) - try: - resolved = Decimal(proxy) - except (InvalidOperation, TypeError, ValueError): - return Decimal(0) - return resolved if resolved.is_finite() and resolved > 0 else Decimal(0) + + _tip_refresh_hashrate_proxy = staticmethod( + TipRefreshService._tip_refresh_hashrate_proxy + ) def _payout_epoch_tip_hash_locked(self) -> str | None: - """Label a payout-minted epoch with the current refresh target tip. - - A payout publication advances the payout generation of the target - the fleet is already converging on; it must not move or invent the - tip. Candidate source hashes are unusable here: an accepted-block - preview stores the accepted block's own hash, which is not a - template parent and can trail an already-observed newer tip, so - labeling the epoch with it strands token currency, identity - publication, and the fixpoint until a later tip observation - remints. - """ - current_epoch_tip = getattr(self, "_tip_refresh_epoch_tip_hash", None) - if current_epoch_tip is not None: - return str(current_epoch_tip) - latest_detected = getattr(self, "latest_detected_tip", None) - if latest_detected is not None: - return str(latest_detected[0]) - published_tip = getattr(self, "current_tip_first_seen", None) - if published_tip is not None: - return str(published_tip[0]) - return None + return self._ensure_tip_refresh_service()._payout_epoch_tip_hash_locked() def _mint_tip_refresh_epoch_locked( self, @@ -11201,103 +5351,26 @@ def _mint_tip_refresh_epoch_locked( payout_state_generation: int, started_monotonic: float, ) -> int: - """Mint the next target epoch while the coordinator lock is held.""" - if not getattr(self, "tip_refresh_epoch_fanout", False): - return 0 - sequence = int(self._tip_refresh_epoch_sequence) + 1 - self._tip_refresh_epoch_sequence = sequence - self._tip_refresh_epoch_tip_hash = tip_hash - self._tip_refresh_epoch_payout_generation = int(payout_state_generation) - client_weights = { - int(client.connection_id): self._tip_refresh_hashrate_proxy(client) - for client in self.clients - if self.client_can_receive_jobs(client) - } - total_weight = sum( - ( - weight - for weight in client_weights.values() - if weight > 0 - ), - Decimal(0), - ) - self._tip_refresh_epoch_coverage = _TipRefreshEpochCoverage( - sequence=sequence, + return self._ensure_tip_refresh_service()._mint_tip_refresh_epoch_locked( tip_hash=tip_hash, - payout_state_generation=int(payout_state_generation), - started_monotonic=float(started_monotonic), - client_weights=client_weights, - total_weight=total_weight, - ) - if total_weight <= 0: - tracker = self._tip_refresh_epoch_coverage - with self._tip_refresh_metrics_lock: - for target, _ratio in PRISM_TIP_REFRESH_COVERAGE_TARGETS: - tracker.recorded_targets.add(target) - histogram = self.tip_refresh_coverage_histograms[target] - histogram["count"] = int(histogram["count"]) + 1 - buckets = histogram["buckets"] - assert isinstance(buckets, dict) - for bucket in PRISM_TIP_REFRESH_SECONDS_BUCKETS: - buckets[bucket] = int(buckets.get(bucket, 0)) + 1 - return sequence + payout_state_generation=payout_state_generation, + started_monotonic=started_monotonic, + ) def _tip_refresh_epoch_for_bundle_locked( self, cached: CachedJobBundle, ) -> int: - if not getattr(self, "tip_refresh_epoch_fanout", False): - return 0 - identity = getattr( - self, - "_published_tip_refresh_epoch_identity", - None, - ) - if identity is None: - return 0 - ( - sequence, - tip_hash, - payout_generation, - template_fingerprint, - template_generation, - ) = identity - parent_hash = str(cached.template.get("previousblockhash", "")) - if ( - parent_hash != tip_hash - or int(cached.payout_state_generation) - != payout_generation - or cached.template_fingerprint != template_fingerprint - or int(cached.template_generation) - != template_generation - ): - return 0 - return sequence + return self._ensure_tip_refresh_service()._tip_refresh_epoch_for_bundle_locked( + cached, + ) def _publish_tip_refresh_epoch_identity_locked( self, snapshot: QbitTipTemplateSnapshot, ) -> None: - if not getattr(self, "tip_refresh_epoch_fanout", False): - return - sequence = int(getattr(self, "_tip_refresh_epoch_sequence", 0)) - tip_hash = getattr(self, "_tip_refresh_epoch_tip_hash", None) - payout_generation = int( - getattr(self, "_tip_refresh_epoch_payout_generation", 0) - ) - if ( - sequence <= 0 - or tip_hash != snapshot.bestblockhash - or payout_generation - != int(getattr(self, "_payout_state_generation", 0)) - ): - return - self._published_tip_refresh_epoch_identity = ( - sequence, - tip_hash, - payout_generation, - snapshot.template_fingerprint, - snapshot.template_generation, + return self._ensure_tip_refresh_service()._publish_tip_refresh_epoch_identity_locked( + snapshot, ) def _admit_client_tip_refresh_epoch_locked( @@ -11305,36 +5378,14 @@ def _admit_client_tip_refresh_epoch_locked( client: ClientState, epoch_sequence: int, ) -> bool: - """Fence per-client registration so epochs cannot regress on the wire.""" - if not getattr(self, "tip_refresh_epoch_fanout", False): - return True - if self._client_tip_refresh_epoch_blocked_locked(client, epoch_sequence): - return False - admitted_epoch = int( - getattr(client, "_tip_refresh_admitted_epoch_sequence", 0) - ) - client._tip_refresh_admitted_epoch_sequence = max( - int(epoch_sequence), - admitted_epoch, + return self._ensure_tip_refresh_service()._admit_client_tip_refresh_epoch_locked( + client, + epoch_sequence, ) - return True - @staticmethod - def _client_tip_refresh_epoch_blocked_locked( - client: ClientState, - epoch_sequence: int, - ) -> bool: - active_epoch = int( - getattr( - getattr(client, "active_job", None), - "tip_refresh_epoch_sequence", - 0, - ) - ) - admitted_epoch = int( - getattr(client, "_tip_refresh_admitted_epoch_sequence", 0) - ) - return int(epoch_sequence) < max(active_epoch, admitted_epoch) + _client_tip_refresh_epoch_blocked_locked = staticmethod( + TipRefreshService._client_tip_refresh_epoch_blocked_locked + ) def _tip_refresh_epoch_coverage_reached_locked( self, @@ -11342,130 +5393,27 @@ def _tip_refresh_epoch_coverage_reached_locked( context: PrismJobContext, delivered_monotonic: float, ) -> list[tuple[str, float]]: - """Advance the active cohort while the coordinator lock is held.""" - if not getattr(self, "tip_refresh_epoch_fanout", False): - return [] - reached: list[tuple[str, float]] = [] - tracker = self._tip_refresh_epoch_coverage - connection_id = int(getattr(client, "connection_id", 0)) - if ( - tracker is None - or tracker.sequence != int(self._tip_refresh_epoch_sequence) - or tracker.total_weight <= 0 - or connection_id in tracker.delivered_clients - ): - return reached - weight = tracker.client_weights.get(connection_id) - if weight is None: - return reached - if ( - int(getattr(context, "connection_id", 0)) != connection_id - or int(getattr(context, "tip_refresh_epoch_sequence", 0)) - != tracker.sequence - or str(context.template.get("previousblockhash", "")) - != tracker.tip_hash - or int(getattr(context, "payout_state_generation", 0)) - != tracker.payout_state_generation - ): - return reached - tracker.delivered_clients.add(connection_id) - if weight <= 0: - return reached - tracker.delivered_weight += weight - coverage = tracker.delivered_weight / tracker.total_weight - elapsed = max( - 0.0, - float(delivered_monotonic) - tracker.started_monotonic, - ) - for target, ratio in PRISM_TIP_REFRESH_COVERAGE_TARGETS: - if target not in tracker.recorded_targets and coverage >= ratio: - tracker.recorded_targets.add(target) - reached.append((target, elapsed)) - return reached + return self._ensure_tip_refresh_service()._tip_refresh_epoch_coverage_reached_locked( + client, + context, + delivered_monotonic, + ) def _record_tip_refresh_epoch_coverage( self, reached: list[tuple[str, float]], ) -> None: - """Observe fixed cohort thresholds without per-connection series.""" - if not reached: - return - with self._tip_refresh_metrics_lock: - for target, elapsed in reached: - histogram = self.tip_refresh_coverage_histograms[target] - histogram["count"] = int(histogram["count"]) + 1 - histogram["sum"] = float(histogram["sum"]) + elapsed - buckets = histogram["buckets"] - assert isinstance(buckets, dict) - for bucket in PRISM_TIP_REFRESH_SECONDS_BUCKETS: - if elapsed <= bucket: - buckets[bucket] = int(buckets.get(bucket, 0)) + 1 + return self._ensure_tip_refresh_service()._record_tip_refresh_epoch_coverage( + reached, + ) def _record_tip_refresh_wave_outcome(self, outcome: str) -> None: - self._ensure_tip_refresh_state() - if outcome not in PRISM_TIP_REFRESH_WAVE_OUTCOMES: - raise ValueError(f"unknown tip refresh wave outcome: {outcome}") - with self._tip_refresh_metrics_lock: - self.tip_refresh_wave_outcome_counts[outcome] = ( - int(self.tip_refresh_wave_outcome_counts.get(outcome, 0)) + 1 - ) + return self._ensure_tip_refresh_service()._record_tip_refresh_wave_outcome( + outcome, + ) def _tip_refresh_epoch_fixpoint_reached(self) -> bool: - """Return whether every eligible connection has newest delivered work.""" - if not getattr(self, "tip_refresh_epoch_fanout", False): - return True - with self.lock: - sequence = int(getattr(self, "_tip_refresh_epoch_sequence", 0)) - tip_hash = getattr(self, "_tip_refresh_epoch_tip_hash", None) - payout_generation = int( - getattr(self, "_tip_refresh_epoch_payout_generation", 0) - ) - snapshot = getattr(self, "tip_template_snapshot", None) - ready_mode = bool(getattr(self, "_pool_ready_latched", False)) - for client in self.clients: - if not self.client_can_receive_jobs(client): - continue - context = getattr(client, "_progress_delivered_context", None) - if ( - context is None - or int(getattr(context, "connection_id", 0)) - != int(client.connection_id) - ): - return False - context_epoch = int( - getattr(context, "tip_refresh_epoch_sequence", 0) - ) - if context_epoch > sequence: - continue - if context_epoch < sequence: - return False - if str(context.template.get("previousblockhash", "")) != tip_hash: - return False - if ( - int(getattr(context, "payout_state_generation", 0)) - != payout_generation - ): - return False - if ready_mode and bool(getattr(context, "collection_only", False)): - return False - if ( - snapshot is not None - and snapshot.bestblockhash == tip_hash - and ( - getattr(context, "template_fingerprint", None) - != snapshot.template_fingerprint - ) - ): - return False - # Every eligible connection holds newest delivered work, so any - # refresh retry armed before this verification is satisfied. - # Consume it inside the same lock hold that proved convergence: - # signals armed afterwards stay armed, and blockpoll no longer - # re-runs a full wave against work a recovered supersession - # already finished. - self._tip_refresh_retry_consumed = self._tip_refresh_retry_counter - self._tip_refresh_retry.clear() - return True + return self._ensure_tip_refresh_service()._tip_refresh_epoch_fixpoint_reached() def _retain_collection_refresh( self, @@ -11473,499 +5421,155 @@ def _retain_collection_refresh( observation_sequence: int, payout_state_generation: int, ) -> None: - """Retain reusable current work until an eligible identity appears.""" - retained = RetainedCollectionRefresh( - snapshot=snapshot, - observation_sequence=observation_sequence, - payout_state_generation=payout_state_generation, + return self._ensure_tip_refresh_service()._retain_collection_refresh( + snapshot, + observation_sequence, + payout_state_generation, ) - should_log = False - with self.lock: - if not self._tip_refresh_snapshot_current_locked( - snapshot, - observation_sequence, - ): - return - if any( - self.client_can_receive_jobs(client) - for client in self.clients - ): - return - previous = self._retained_collection_refresh - self._retained_collection_refresh = retained - should_log = previous is None or ( - previous.snapshot.bestblockhash != snapshot.bestblockhash - or previous.payout_state_generation != payout_state_generation - ) - if should_log: - print( - "prism coordinator: collection refresh retained while no " - "authorized worker identity is available", - flush=True, - ) def _retained_collection_artifacts(self) -> CachedTemplateArtifacts | None: - """Return retained artifacts while their published work stays current. - - A same-tip poll advances its observation sequence before atomically - replacing ``tip_template_snapshot``. The published snapshot remains - reusable on both sides of that handoff even if the retained marker has - not yet been updated; a new tip or payout generation still invalidates - it immediately. - """ - self._ensure_job_cache_state() - self._ensure_tip_refresh_state() - with self._job_cache_lock: - payout_state_generation = self._payout_state_generation - with self.lock: - retained = self._retained_collection_refresh - if retained is None: - return None - if retained.payout_state_generation != payout_state_generation: - return None - current_tip = getattr(self, "current_tip_first_seen", None) - published_snapshot = self.tip_template_snapshot - if ( - published_snapshot is None - or current_tip is None - or current_tip[0] != published_snapshot.bestblockhash - ): - return None - return self._tip_refresh_artifacts(published_snapshot) + return self._ensure_tip_refresh_service()._retained_collection_artifacts() def _retain_current_collection_refresh_if_unrepresented(self) -> None: - """Keep the last published collection work when the fleet empties.""" - self._ensure_tip_refresh_state() - if getattr(self, "_pool_ready_latched", False): - return - with self.lock: - if any( - self.client_can_receive_jobs(client) - for client in self.clients - ): - return - snapshot = self.tip_template_snapshot - observation_sequence = int( - getattr(self, "current_tip_observation_sequence", 0) - ) - if snapshot is None: - return - self._ensure_job_cache_state() - with self._job_cache_lock: - payout_state_generation = self._payout_state_generation - self._retain_collection_refresh( - snapshot, - observation_sequence, - payout_state_generation, + return self._ensure_tip_refresh_service()._retain_current_collection_refresh_if_unrepresented( ) def _note_collection_identity_available(self, client: ClientState) -> None: - """Wake a retained collection refresh as soon as a client is eligible.""" - if not self.client_can_receive_jobs(client): - return - if self._retained_collection_artifacts() is None: - return - self._mark_tip_refresh_pending(client.connection_id) - self._schedule_tip_refresh_retry() + return self._ensure_tip_refresh_service()._note_collection_identity_available( + client, + ) def _consume_retained_collection_refresh( self, context: PrismJobContext, ) -> None: - """Consume retention only after its collection work was delivered.""" - if not context.collection_only: - return - with self.lock: - retained = self._retained_collection_refresh - published_snapshot = self.tip_template_snapshot - artifacts = ( - published_snapshot.template_artifacts - if published_snapshot is not None - else None - ) - if ( - retained is not None - and retained.payout_state_generation - == context.payout_state_generation - and artifacts is not None - and context.template is artifacts.template - and context.template_fingerprint == artifacts.fingerprint - and context.template_generation == artifacts.generation - ): - self._retained_collection_refresh = None + return self._ensure_tip_refresh_service()._consume_retained_collection_refresh( + context, + ) def tip_refresh_is_pending(self) -> bool: return self._tip_refresh_pending() def _tip_refresh_pending(self) -> bool: - self._ensure_tip_refresh_state() - return self._tip_refresh_pending_event.is_set() + return self._ensure_tip_refresh_service()._tip_refresh_pending() def _mark_tip_refresh_pending(self, _observation: object) -> int: - self._ensure_tip_refresh_state() - with self.lock: - self._tip_refresh_pending_counter += 1 - token = self._tip_refresh_pending_counter - self._tip_refresh_pending_token = token - self._tip_refresh_pending_event.set() - return token + return self._ensure_tip_refresh_service()._mark_tip_refresh_pending( + _observation, + ) def _claim_tip_refresh_pending(self) -> int | None: - """Snapshot pending work without replacing a newer producer's token.""" - self._ensure_tip_refresh_state() - with self.lock: - if not self._tip_refresh_pending_event.is_set(): - return None - return self._tip_refresh_pending_token + return self._ensure_tip_refresh_service()._claim_tip_refresh_pending() def _mark_tip_refresh_pending_for_poll( self, owned_token: int | None, _observation: object, ) -> int | None: - """Mark poll-owned work only while no newer producer has superseded it.""" - self._ensure_tip_refresh_state() - with self.lock: - if self._tip_refresh_pending_token != owned_token: - return owned_token - if owned_token is not None: - self._tip_refresh_pending_event.set() - return owned_token - self._tip_refresh_pending_counter += 1 - token = self._tip_refresh_pending_counter - self._tip_refresh_pending_token = token - self._tip_refresh_pending_event.set() - return token + return self._ensure_tip_refresh_service()._mark_tip_refresh_pending_for_poll( + owned_token, + _observation, + ) def _clear_tip_refresh_pending(self, token: int) -> None: - self._ensure_tip_refresh_state() - with self.lock: - if self._tip_refresh_pending_token == token: - self._tip_refresh_pending_token = None - self._tip_refresh_pending_event.clear() + return self._ensure_tip_refresh_service()._clear_tip_refresh_pending(token) def _clear_tip_refresh_pending_for_completed_refresh( self, - snapshot: QbitTipTemplateSnapshot, - observation_sequence: int, - payout_state_generation: int, - pending_signal_token: int | None = None, - ) -> bool: - """Atomically acknowledge pending work handled by a completed poll.""" - self._ensure_job_cache_state() - with self._payout_state_delivery_gate.delivery_cancelable( - lambda: self._payout_state_generation != payout_state_generation, - generation=payout_state_generation, - priority=True, - ) as admission: - if not admission: - return False - with self._job_cache_lock: - payout_state_current = ( - self._payout_state_generation == payout_state_generation - ) - with self.lock: - refresh_current = self._tip_refresh_snapshot_current_locked( - snapshot, - observation_sequence, - ) - pending_owned = ( - self._tip_refresh_pending_token == pending_signal_token - ) - if ( - not payout_state_current - or not refresh_current - or not pending_owned - ): - return False - self._tip_refresh_pending_token = None - self._tip_refresh_pending_event.clear() - with self._progress_health_lock: - published_current = bool( - self._progress_has_published_work - and self._progress_published_template_fingerprint - == snapshot.template_fingerprint - and self._progress_published_payout_generation - == payout_state_generation - ) - if published_current: - # The completion guard above proves this coherent - # snapshot still represents the latest detected and - # published tip. A transient A -> B -> A observation - # therefore closes its publication-divergence epoch - # even when existing A work needed no new delivery. - self._progress_refresh_signal_pending = False - self._progress_publication_divergence_since_monotonic = None - return True + snapshot: QbitTipTemplateSnapshot, + observation_sequence: int, + payout_state_generation: int, + pending_signal_token: int | None = None, + ) -> bool: + return self._ensure_tip_refresh_service()._clear_tip_refresh_pending_for_completed_refresh( + snapshot, + observation_sequence, + payout_state_generation, + pending_signal_token, + ) def _schedule_tip_refresh_retry(self) -> None: - self._ensure_tip_refresh_state() - # Pair the Event with a monotonic generation so a producer cannot set - # it between a waiter's wake and clear and lose the newest retry. The - # event remains the blocking primitive; the generation is the durable - # coalesced work marker. - with self.lock: - self._tip_refresh_retry_counter += 1 - self._tip_refresh_retry.set() + return self._ensure_tip_refresh_service()._schedule_tip_refresh_retry() def _consume_tip_refresh_retry(self) -> bool: - """Consume all retry signals visible at one atomic wake boundary.""" - self._ensure_tip_refresh_state() - with self.lock: - generation = self._tip_refresh_retry_counter - if generation == self._tip_refresh_retry_consumed: - return False - self._tip_refresh_retry_consumed = generation - self._tip_refresh_retry.clear() - return True + return self._ensure_tip_refresh_service()._consume_tip_refresh_retry() def _note_tip_refresh_attempt_failed( self, observed_tip: str | None = None, ) -> None: - """Stamp a failed refresh pass so the poller spaces its re-attempt. - - ``observed_tip`` is the tip the failed pass worked against; a pass - that failed before learning one stamps the current observation so an - RPC outage with a static tip still gets spaced. - """ - self._ensure_tip_refresh_state() - holdoff = float( - getattr( - self, - "tip_refresh_failure_holdoff_seconds", - DEFAULT_PRISM_TIP_REFRESH_FAILURE_HOLDOFF_SECONDS, - ) - ) - if holdoff <= 0: - return - holdoff += random.uniform( - 0.0, - holdoff * PRISM_TIP_REFRESH_FAILURE_HOLDOFF_JITTER_FRACTION, + return self._ensure_tip_refresh_service()._note_tip_refresh_attempt_failed( + observed_tip, ) - with self.lock: - if observed_tip is None: - observed_tip = self._newest_observed_tip_locked() - self._tip_refresh_failure_tip = observed_tip - self._tip_refresh_failure_holdoff_until = time.monotonic() + holdoff - - def _clear_tip_refresh_failure_holdoff(self) -> None: - self._ensure_tip_refresh_state() - with self.lock: - self._tip_refresh_failure_holdoff_until = None - self._tip_refresh_failure_tip = None def _tip_refresh_failure_holdoff_remaining(self) -> float: - """Seconds the poller must still wait before re-running a failed pass. - - Zero as soon as the newest known tip differs from the one the failed - pass worked against: spacing throttles re-attempts against an - unchanged, blocked or churning view, never the reaction to a - genuinely new tip (including one detected but not yet published). - """ - self._ensure_tip_refresh_state() - with self.lock: - deadline = self._tip_refresh_failure_holdoff_until - failed_tip = self._tip_refresh_failure_tip - current_hash = self._newest_observed_tip_locked() - if deadline is None: - return 0.0 - if current_hash != failed_tip: - return 0.0 - return max(0.0, deadline - time.monotonic()) + return self._ensure_tip_refresh_service()._tip_refresh_failure_holdoff_remaining( + ) def _observe_tip_refresh_seconds(self, name: str, elapsed_seconds: float) -> None: - self._ensure_tip_refresh_state() - with self._tip_refresh_metrics_lock: - histogram = self.tip_refresh_histograms[name] - histogram["count"] = int(histogram["count"]) + 1 - histogram["sum"] = float(histogram["sum"]) + elapsed_seconds - buckets = histogram["buckets"] - assert isinstance(buckets, dict) - for bucket in PRISM_TIP_REFRESH_SECONDS_BUCKETS: - if elapsed_seconds <= bucket: - buckets[bucket] = int(buckets.get(bucket, 0)) + 1 + return self._ensure_tip_refresh_service()._observe_tip_refresh_seconds( + name, + elapsed_seconds, + ) def _observe_tip_refresh_build_phase( self, phase: str, elapsed_seconds: float, ) -> None: - if phase not in PRISM_TIP_REFRESH_BUILD_PHASES: - raise ValueError(f"unknown tip refresh build phase: {phase}") - self._ensure_tip_refresh_state() - with self._tip_refresh_metrics_lock: - histogram = self.tip_refresh_build_phase_histograms[phase] - histogram["count"] = int(histogram["count"]) + 1 - histogram["sum"] = float(histogram["sum"]) + max( - 0.0, elapsed_seconds - ) - buckets = histogram["buckets"] - assert isinstance(buckets, dict) - for bucket in PRISM_TIP_REFRESH_SECONDS_BUCKETS: - if elapsed_seconds <= bucket: - buckets[bucket] = int(buckets.get(bucket, 0)) + 1 + return self._ensure_tip_refresh_service()._observe_tip_refresh_build_phase( + phase, + elapsed_seconds, + ) def _record_tip_refresh_ipc_bytes(self, direction: str, byte_count: int) -> None: - if direction not in {"input", "output"}: - raise ValueError(f"unknown tip refresh IPC direction: {direction}") - self._ensure_tip_refresh_state() - with self._tip_refresh_metrics_lock: - self.tip_refresh_ipc_bytes[direction] += max(0, int(byte_count)) + return self._ensure_tip_refresh_service()._record_tip_refresh_ipc_bytes( + direction, + byte_count, + ) def _record_tip_refresh_client_result(self, result: str) -> None: - if result not in PRISM_TIP_REFRESH_RESULTS: - raise ValueError(f"unknown tip refresh result: {result}") - self._ensure_tip_refresh_state() - with self._tip_refresh_metrics_lock: - self.tip_refresh_client_counts[result] += 1 + return self._ensure_tip_refresh_service()._record_tip_refresh_client_result( + result, + ) def _record_tip_refresh_cancellation(self, stage: str) -> None: - if stage not in PRISM_TIP_REFRESH_CANCELLATION_STAGES: - raise ValueError(f"unknown tip refresh cancellation stage: {stage}") - self._ensure_tip_refresh_state() - with self._tip_refresh_metrics_lock: - self.tip_refresh_cancellation_counts[stage] += 1 + return self._ensure_tip_refresh_service()._record_tip_refresh_cancellation( + stage, + ) def _tip_refresh_future_started(self) -> None: - self._ensure_tip_refresh_state() - with self._tip_refresh_metrics_lock: - self.tip_refresh_inflight += 1 + return self._ensure_tip_refresh_service()._tip_refresh_future_started() def _tip_refresh_future_finished(self, _future: Future[RefreshResult]) -> None: - self._ensure_tip_refresh_state() - with self._tip_refresh_metrics_lock: - self.tip_refresh_inflight = max(0, self.tip_refresh_inflight - 1) + return self._ensure_tip_refresh_service()._tip_refresh_future_finished(_future) def tip_refresh_executor(self) -> _BoundedPriorityExecutor: - self._ensure_tip_refresh_state() - with self._tip_refresh_executor_lock: - if self._tip_refresh_executor_shutdown: - raise RuntimeError("tip refresh executor is shut down") - executor = self._tip_refresh_executor - if executor is None: - executor = _BoundedPriorityExecutor( - max_workers=self.tip_refresh_max_workers, - max_queue_size=self.delivery_queue_limit(), - thread_name_prefix="prism-tip-refresh-delivery", - ) - self._tip_refresh_executor = executor - return executor + return self._ensure_tip_refresh_service().tip_refresh_executor() def initial_job_executor(self) -> _BoundedPriorityExecutor: - self._ensure_initial_job_state() - with self._initial_job_executor_lock: - if self._initial_job_executor_shutdown: - raise RuntimeError("initial job executor is shut down") - executor = self._initial_job_executor - if executor is None: - executor = _BoundedPriorityExecutor( - max_workers=self.initial_job_max_workers, - max_queue_size=self.stratum_max_pending_initial_jobs, - thread_name_prefix="prism-initial-job-delivery", - ) - self._initial_job_executor = executor - return executor + return self._ensure_job_delivery_service().initial_job_executor() def shutdown_initial_job_executor(self) -> None: - self._ensure_initial_job_state() - with self.lock: - for request in tuple(self.pending_initial_jobs.values()): - if self.pending_initial_jobs.get(request.client) is not request: - continue - self.pending_initial_jobs.pop(request.client, None) - request.cancelled.set() - if request.future is not None: - self._cancel_initial_job_future(request.future) - self.initial_job_cancelled_count += 1 - with self._initial_job_executor_lock: - executor = self._initial_job_executor - self._initial_job_executor = None - self._initial_job_executor_shutdown = True - if executor is not None: - # Running workers observe request cancellation before shutdown - # returns; queued reconnect work is cancelled without starting. - executor.shutdown(wait=True, cancel_futures=True) + return self._ensure_job_delivery_service().shutdown_initial_job_executor() def shutdown_tip_refresh_executor(self) -> None: - self._ensure_tip_refresh_state() - with self._tip_refresh_executor_lock: - executor = self._tip_refresh_executor - self._tip_refresh_executor = None - self._tip_refresh_executor_shutdown = True - if executor is not None: - # Stop queued publication work before waiting on reconnect workers. - executor.shutdown(wait=False, cancel_futures=True) - self.shutdown_initial_job_executor() - if executor is not None: - # Running workers may already hold client/job state or be inside a - # socket send. Drain them before serve returns and the writer lease - # is released; queued workers are cancelled without starting. - executor.shutdown(wait=True) - self.shutdown_job_build_executor() - self.shutdown_payout_artifact_executor() - self.shutdown_reconcile_prefetch_executor() - self.retire_share_window_spool() - self.shutdown_serve_builder() + return self._ensure_tip_refresh_service().shutdown_tip_refresh_executor() def retire_share_window_spool(self) -> None: - """Release the cached share-window spool during shutdown. - - Runs after the build executors quiesce, so any lease still held by a - draining transfer defers the close to its own release. - """ - self._ensure_job_cache_state() - with self._share_window_serialization_lock: - serialization = self._share_window_serialization - if serialization is not None: - serialization.retire_spool() + """Release the cached share-window spool during shutdown.""" + self._ensure_job_bundle_service().retire_share_window_spool() def _cancel_initial_job_future(self, future: Future[bool]) -> bool: - """Cancel one initial-job future and account physical queue removal.""" - executor = getattr(self, "_initial_job_executor", None) - reclaimed = bool( - executor is not None - and executor.cancel(future) - ) - if executor is None: - future.cancel() - if reclaimed: - with self.lock: - self._ensure_initial_job_state() - self.initial_job_queue_capacity_reclaimed_count += 1 - return reclaimed + return self._ensure_job_delivery_service()._cancel_initial_job_future(future) def _initial_request_current_locked(self, request: PendingInitialJob) -> bool: - client = request.client - deadline = request.deadline_monotonic - return ( - self.pending_initial_jobs.get(client) is request - and client in self.clients - and not getattr(client, "closing", False) - and ( - request.connection_id is None - or client.connection_id == request.connection_id - ) - and client.authorized - and client.subscribed - and client.worker == request.worker - and int(getattr(client, "authorization_generation", 0)) - == request.authorization_generation - and ( - request.difficulty_generation is None - or int(getattr(client, "difficulty_generation", 0)) - == request.difficulty_generation - ) - and (deadline is None or time.monotonic() < deadline) - and not request.cancelled.is_set() + return self._ensure_job_delivery_service()._initial_request_current_locked( + request, ) def _initial_request_cancelled(self, request: PendingInitialJob) -> bool: - if request.cancelled.is_set() or self.stop_event.is_set(): - return True - with self.lock: - self._ensure_initial_job_state() - return not self._initial_request_current_locked(request) + return self._ensure_job_delivery_service()._initial_request_cancelled(request) def _cancel_pending_initial_job_locked( self, @@ -11973,78 +5577,15 @@ def _cancel_pending_initial_job_locked( *, count: bool, ) -> PendingInitialJob | None: - self._ensure_initial_job_state() - request = self.pending_initial_jobs.pop(client, None) - if request is None: - return None - request.cancelled.set() - if request.future is not None: - self._cancel_initial_job_future(request.future) - if count: - self.initial_job_cancelled_count += 1 - return request - - def _client_has_current_tip_job_locked(self, client: ClientState) -> bool: - context = client.active_job - if context is None: - return False - payout_generation = int(getattr(self, "_payout_state_generation", 0)) - if int(getattr(context, "payout_state_generation", payout_generation)) != payout_generation: - return False - current_tip = self._current_published_tip_hash_locked() - if current_tip is None: - # An active job is not proof that tip observation is alive. Keep - # coverage fail-closed until blockpoll/blockwait has published the - # tip that makes the job current. - return False - snapshot = getattr(self, "tip_template_snapshot", None) - if snapshot is None: - # Focused embedders may only publish the observed tip. Production - # startup and blockpoll publish a full snapshot, in which case the - # exact template identity checks below are mandatory. - return str(context.template.get("previousblockhash", "")) == current_tip - if snapshot.bestblockhash != current_tip or snapshot.template_artifacts is None: - return False - return bool( - str(context.template.get("previousblockhash", "")) == current_tip - and getattr(context, "template_fingerprint", None) - == snapshot.template_fingerprint - and int(getattr(context, "template_generation", 0)) - == snapshot.template_generation - and context.template is snapshot.template_artifacts.template - and int(getattr(context, "connection_id", client.connection_id)) - == client.connection_id - and int(getattr(context, "authorization_generation", 0)) - == int(getattr(client, "authorization_generation", 0)) - and int(getattr(context, "difficulty_generation", 0)) - == int(getattr(client, "difficulty_generation", 0)) - ) - - @staticmethod - def _client_has_delivered_work_locked(client: ClientState) -> bool: - """Return whether a socket write completed for any usable job.""" - - return bool( - client.tip_work_delivered is not None - or client._progress_delivered_context is not None + return self._ensure_job_delivery_service()._cancel_pending_initial_job_locked( + client, + count=count, ) - def _reset_delivery_failure_if_coverage_restored_locked(self) -> None: - authorized_clients = [ - client - for client in self.clients - if client.subscribed and client.authorized and client.worker is not None - ] - if not authorized_clients: - self._mining_delivery_failure_started_monotonic = None - return - current = sum( - 1 - for client in authorized_clients - if self._client_has_current_tip_job_locked(client) + def _client_has_current_tip_job_locked(self, client: ClientState) -> bool: + return self._ensure_job_delivery_service()._client_has_current_tip_job_locked( + client, ) - if current / len(authorized_clients) >= 0.95: - self._mining_delivery_failure_started_monotonic = None def note_initial_job_delivered( self, @@ -12052,324 +5593,38 @@ def note_initial_job_delivered( *, validated_current: bool = False, ) -> None: - with self.lock: - self._ensure_initial_job_state() - if not validated_current and not self._client_has_current_tip_job_locked(client): - return - request = self.pending_initial_jobs.pop(client, None) - if request is not None: - request.cancelled.set() - if request.future is not None: - self._cancel_initial_job_future(request.future) - delivered = time.monotonic() - self.initial_job_sent_count += 1 - self.initial_job_delivery_latency_seconds_sum += max( - 0.0, delivered - request.requested_monotonic - ) - self.initial_job_delivery_latency_count += 1 - self.last_initial_job_delivery_monotonic = delivered - self._record_startup_phase_once("first_job_delivered") + return self._ensure_job_delivery_service().note_initial_job_delivered( + client, + validated_current=validated_current, + ) def schedule_initial_job(self, client: ClientState) -> bool: - """Coalesce and enqueue one first-job request without blocking its handler.""" - # Focused tests and embedders replace maybe_send_job on the instance as - # a synchronous seam. Preserve it without affecting the production - # class path, which always uses the bounded executor below. - if "maybe_send_job" in self.__dict__: - return bool(self.maybe_send_job(client, clean_jobs=True)) - - now = time.monotonic() - reject = False - deferred = False - superseded_future: Future[bool] | None = None - with self.lock: - self._ensure_initial_job_state() - if ( - not client.subscribed - or not client.authorized - or client.worker is None - or getattr(client, "closing", False) - ): - return True - generation = int(getattr(client, "authorization_generation", 0)) - difficulty_generation = int( - getattr(client, "difficulty_generation", 0) - ) - existing = self.pending_initial_jobs.get(client) - if ( - existing is not None - and existing.connection_id == client.connection_id - and existing.authorization_generation == generation - and existing.difficulty_generation == difficulty_generation - and existing.worker == client.worker - ): - self.initial_job_coalesced_count += 1 - return True - if existing is not None: - existing.cancelled.set() - superseded_future = existing.future - self.initial_job_cancelled_count += 1 - self.initial_job_superseded_count += 1 - if self._client_has_current_tip_job_locked(client): - if existing is not None: - self.pending_initial_jobs.pop(client, None) - if superseded_future is not None: - self._cancel_initial_job_future(superseded_future) - return True - if ( - existing is None - and len(self.pending_initial_jobs) - >= self.stratum_max_pending_initial_jobs - ): - self.initial_job_queue_rejection_count += 1 - reject = True - request = None - else: - timeout = float(self.stratum_initial_job_timeout_seconds) - predecessor = None - if existing is not None: - for candidate in (existing.future, existing.predecessor): - if candidate is not None and not candidate.done(): - predecessor = candidate - break - request = PendingInitialJob( - client=client, - connection_id=client.connection_id, - authorization_generation=generation, - difficulty_generation=difficulty_generation, - worker=client.worker, - requested_monotonic=now, - deadline_monotonic=now + timeout if timeout > 0 else None, - predecessor=predecessor, - ) - self.pending_initial_jobs[client] = request - deferred = predecessor is not None - if superseded_future is not None: - # Install the replacement before cancellation callbacks can run; - # the predecessor callback then hands off exactly one client slot - # instead of mistaking the obsolete request for a terminal failure. - self._cancel_initial_job_future(superseded_future) - if reject or request is None: - self.disconnect_client(client) - return False - if deferred: - return True - - return self._submit_initial_job_request(request) + return self._ensure_job_delivery_service().schedule_initial_job(client) def request_initial_job_delivery(self, client: ClientState) -> bool: - """Compatibility name for the single bounded initial-job pipeline.""" - return self.schedule_initial_job(client) + return self._ensure_job_delivery_service().request_initial_job_delivery(client) def cancel_initial_job_delivery(self, client: ClientState) -> None: - with self.lock: - self._ensure_initial_job_state() - self._cancel_pending_initial_job_locked(client, count=True) + return self._ensure_job_delivery_service().cancel_initial_job_delivery(client) def _submit_initial_job_request(self, request: PendingInitialJob) -> bool: - client = request.client - try: - future = self._submit_delivery_task( - self.initial_job_executor(), - self._run_initial_job, - request, - priority=PRISM_DELIVERY_PRIORITY_INITIAL, - ) - except (_DeliveryQueueFull, RuntimeError): - disconnect = False - with self.lock: - if self.pending_initial_jobs.get(client) is request: - self.pending_initial_jobs.pop(client, None) - request.cancelled.set() - self.initial_job_queue_rejection_count += 1 - disconnect = True - if disconnect: - self.disconnect_client(client) - # An obsolete submit can race its already-installed replacement. - # It owns neither the client slot nor the right to retire the live - # session when admission fails. - return not disconnect - with self.lock: - if self.pending_initial_jobs.get(client) is request: - request.future = future - else: - self._cancel_initial_job_future(future) - future.add_done_callback( - lambda completed: self._initial_job_future_finished(request, completed) - ) - return True + return self._ensure_job_delivery_service()._submit_initial_job_request(request) def _initial_job_future_finished( self, request: PendingInitialJob, future: Future[bool], ) -> None: - """Release failed first-job requests instead of stranding capacity.""" - delivered = False - if not future.cancelled(): - try: - delivered = bool(future.result()) - except Exception: - with self.lock: - self.job_build_failure_count = int( - getattr(self, "job_build_failure_count", 0) - ) + 1 - print( - "prism coordinator: initial job task failed " - f"connection={request.client.connection_id}", - flush=True, - ) - traceback.print_exc() - - disconnect = False - replacement: PendingInitialJob | None = None - with self.lock: - self._ensure_initial_job_state() - current = self.pending_initial_jobs.get(request.client) - if current is not request: - if ( - current is not None - and current.future is None - and current.predecessor is future - ): - current.predecessor = None - replacement = current - elif ( - delivered - and self._initial_request_current_locked(request) - and self._client_has_current_tip_job_locked(request.client) - ): - self.pending_initial_jobs.pop(request.client, None) - request.cancelled.set() - self.last_initial_job_delivery_monotonic = time.monotonic() - self._record_startup_phase_once("first_job_delivered") - elif current is request: - self.pending_initial_jobs.pop(request.client, None) - request.cancelled.set() - if ( - request.deadline_monotonic is not None - and request.deadline_monotonic <= time.monotonic() - ): - request.client.closing = True - self.initial_job_timeout_count += 1 - self.initial_job_cancelled_count += 1 - else: - self.initial_job_failed_count += 1 - disconnect = True - if replacement is not None: - self._submit_initial_job_request(replacement) - if disconnect: - self.disconnect_client(request.client) + return self._ensure_job_delivery_service()._initial_job_future_finished( + request, + future, + ) def _run_initial_job(self, request: PendingInitialJob) -> bool: - """Prepare outside client locks, then atomically stamp and send current work.""" - retry_delay = 0.05 - last_failure_log_monotonic: float | None = None - - def retry_later() -> bool: - nonlocal retry_delay - request.cancelled.wait(retry_delay) - retry_delay = min(1.0, retry_delay * 2) - return not self._initial_request_cancelled(request) - - try: - while not self._initial_request_cancelled(request): - try: - if not self.ensure_reorg_reconciled_for_current_tip(): - if not retry_later(): - return False - continue - artifacts = self.job_issuance_template_artifacts() - if self._initial_request_cancelled(request): - return False - bundle = self.shared_job_bundle( - artifacts, - request.worker, - request_source="initial", - cancelled=lambda: ( - self._initial_request_cancelled(request) - or not self._issuance_artifacts_current(artifacts) - ), - ) - live_tip = str(self.rpc.call("getbestblockhash")) - if artifacts.previousblockhash != live_tip: - # Feed the observation into detection like every other - # live-tip reader; the refresh path owns publication. - self.observe_tip_for_refresh(live_tip) - with self.lock: - published = getattr(self, "current_tip_first_seen", None) - pinned_authoritative = bool( - published is not None - and artifacts.previousblockhash == published[0] - and self._published_tip_authoritative_locked( - time.monotonic() - ) - ) - if not pinned_authoritative: - # Authority lapsed: submit classification is on the - # live RPC read now, so published-tip work would be - # rejected. Drop it and rebuild from live state. - with self._job_cache_lock: - if self._template_artifacts is artifacts: - self._template_artifacts = None - if not retry_later(): - return False - continue - except _JobBuildCancelled: - if self._initial_request_cancelled(request): - return False - if not retry_later(): - return False - continue - except TemplateRefreshBlocked: - if self._initial_request_cancelled(request): - return False - if not retry_later(): - return False - continue - except Exception: - with self.lock: - self.job_build_failure_count = int( - getattr(self, "job_build_failure_count", 0) - ) + 1 - now = time.monotonic() - if ( - last_failure_log_monotonic is None - or now - last_failure_log_monotonic >= 5.0 - ): - last_failure_log_monotonic = now - print( - "prism coordinator: initial job preparation failed " - f"connection={request.client.connection_id}; retrying", - flush=True, - ) - traceback.print_exc() - if not retry_later(): - return False - continue - delivered = self._deliver_initial_bundle(request, artifacts, bundle) - if delivered is None: - if not retry_later(): - return False - continue - return delivered - return False - except OSError: - self.disconnect_client(request.client) - return False + return self._ensure_job_delivery_service()._run_initial_job(request) def _template_artifacts_are_current(self, artifacts: CachedTemplateArtifacts) -> bool: - self._ensure_job_cache_state() - with self._job_cache_lock: - current = self._template_artifacts - return ( - current is artifacts - or ( - current is not None - and current.fingerprint == artifacts.fingerprint - and current.generation == artifacts.generation - ) - ) + return self._ensure_job_bundle_service().template_repository.artifacts_are_current(artifacts) def _issuance_artifacts_current(self, artifacts: CachedTemplateArtifacts) -> bool: """Issuance-side currency for direct job delivery. @@ -12396,49 +5651,9 @@ def _issuance_artifacts_current(self, artifacts: CachedTemplateArtifacts) -> boo and self._published_tip_authoritative_locked(time.monotonic()) ) - def _acquire_client_job_lock( - self, - client: ClientState, - cancelled: Callable[[], bool], - ) -> bool: - while not cancelled(): - if client.job_update_lock.acquire(timeout=0.1): - return True - return False - - @contextmanager - def _cancellable_client_job_lock( - self, - client: ClientState, - cancelled: Callable[[], bool], - ) -> Iterator[bool]: - acquired = self._acquire_client_job_lock(client, cancelled) - try: - yield acquired - finally: - if acquired: - client.job_update_lock.release() - - def _payout_delivery( - self, - cancelled: Callable[[], bool], - *, - generation: int, - ) -> Any: + def _payout_delivery(self, cancelled: Callable[[], bool], *, generation: int) -> Any: """Use cancellable admission while retaining focused gate test seams.""" - gate = self._payout_state_delivery_gate - delivery_cancelable = getattr(gate, "delivery_cancelable", None) - if callable(delivery_cancelable): - return delivery_cancelable( - cancelled, - generation=generation, - priority=True, - ) - delivery = gate.delivery - try: - return delivery(cancelled) - except TypeError: - return delivery() + return self._ensure_payout_state_service()._payout_delivery(cancelled, generation=generation) @staticmethod def _submit_delivery_task( @@ -12458,179 +5673,17 @@ def _deliver_initial_bundle( artifacts: CachedTemplateArtifacts, bundle: CachedJobBundle, ) -> bool | None: - client = request.client - - def cancelled() -> bool: - return self._initial_request_cancelled(request) - - def bundle_payout_stale() -> bool: - # Unlocked at the gate (a cancellation hint, exactly like direct - # delivery's callback); authoritative only when re-read under - # _job_cache_lock at the commit boundary below. - published_artifact = self._published_payout_state.artifact - return ( - bundle.payout_state_generation != self._payout_state_generation - or bundle.build_key is None - # A periodic self-check repair replaces the published balance - # snapshot in place: the generation and append epoch both - # survive the swap, so only the digest identifies work keyed - # to the refuted balances. Every other serving decision - # (cache usability, idle issuance, cache publication) already - # applies this fence; the initial-delivery commit must too, - # or a bundle probed before the repair becomes the miner's - # active job with the refuted balances. - or published_artifact is None - or bundle.build_key.payout_artifact_sha256 - != published_artifact.prior_balances_sha256 - or ( - not bundle.collection_only - and int( - getattr( - bundle.build_key, - "payout_append_invalidation_epoch", - 0, - ) - ) - != self._payout_ledger_append_invalidation_epoch - ) - ) - - if not self._acquire_client_job_lock(client, cancelled): - return False - try: - if cancelled(): - return False - if not self._issuance_artifacts_current(artifacts): - return None - self._ensure_job_cache_state() - gate_started = time.monotonic() - with self._payout_delivery( - lambda: cancelled() or bundle_payout_stale(), - generation=bundle.payout_state_generation, - ) as admitted: - self._observe_payout_gate_admission( - admitted, - generation=bundle.payout_state_generation, - fallback_wait_seconds=time.monotonic() - gate_started, - ) - if cancelled(): - return False - if not admitted: - # Non-admission with a bundle in hand is transient gate - # state: a payout invalidation pending publication, or - # this bundle's generation going stale while waiting. - # Rebuild and retry within the request deadline; treating - # it as terminal turned every unlucky join during a payout - # publication into a disconnect-reconnect-rebuild loop. - return None - with self._job_cache_lock: - payout_current = not bundle_payout_stale() - if not payout_current or not self._issuance_artifacts_current(artifacts): - return None - # Difficulty state precedes coordinator admission everywhere. - # A share holding this client's Vardiff lock may delay only - # this delivery; it must never make initial-job delivery hold - # the global control-plane lock while waiting. - with self._client_vardiff_lock(client): - # The Vardiff wait happens after the payout check above; a - # late-visible append can invalidate the bundle during it. - # Mirror direct delivery: the live epoch re-read and the - # stamp are one commit boundary under _job_cache_lock. - with self._job_cache_lock: - if bundle_payout_stale(): - return None - with self.lock: - if not self._initial_request_current_locked(request): - return False - context = self.stamp_job_for_client( - client, - bundle, - clean_jobs=True, - ) - if not self._admit_client_tip_refresh_epoch_locked( - client, - context.tip_refresh_epoch_sequence, - ): - # The fence blocks only strictly older epochs, so - # this connection already registered newer refresh - # work whose delivery the epoch machinery owns. - # Retrying here can spin until the initial-job - # deadline disconnects an already-served client; - # complete a request that newer current-tip work - # satisfies instead. - self.note_initial_job_delivered(client) - if ( - self.pending_initial_jobs.get(client) - is not request - ): - return False - return None - client.active_job = context - for job_id in tuple(client.active_job_ids): - self.bury_evicted_job(client, job_id, prune=False) - self.jobs.pop(job_id, None) - client.active_job_ids.clear() - self.prune_evicted_job_graveyard(force=False) - self.jobs[context.job.job_id] = context - client.active_job_ids.add(context.job.job_id) - self.prune_client_active_jobs(client) - - self.send_job_update(client, context.job) - mark_delivered = getattr(admitted, "mark_delivered", None) - if callable(mark_delivered): - mark_delivered() - self.apply_job_difficulty(client, context.job) - self.note_tip_work_delivered( - client, - str(context.template["previousblockhash"]), - ) - delivered_monotonic = time.monotonic() - self._record_first_payout_delivery( - bundle.payout_state_generation, - delivered_monotonic, - ) - self._record_progress_delivery( - client, - context, - delivered_monotonic, - ) - self.note_initial_job_delivered(client, validated_current=True) - return True - finally: - client.job_update_lock.release() + return self._ensure_job_delivery_service()._deliver_initial_bundle( + request, + artifacts, + bundle, + ) def sweep_initial_job_timeouts(self, *, now: float | None = None) -> int: - now = time.monotonic() if now is None else now - timed_out: list[PendingInitialJob] = [] - with self.lock: - self._ensure_initial_job_state() - expired = [ - request - for request in self.pending_initial_jobs.values() - if request.deadline_monotonic is not None - and request.deadline_monotonic <= now - ] - for request in expired: - if self.pending_initial_jobs.get(request.client) is not request: - continue - self.pending_initial_jobs.pop(request.client, None) - request.cancelled.set() - if request.future is not None: - self._cancel_initial_job_future(request.future) - # Commit teardown while this request still owns the pending - # slot. A concurrent reauthorization will observe closing and - # cannot install a replacement between expiry and disconnect. - request.client.closing = True - self.initial_job_timeout_count += 1 - self.initial_job_cancelled_count += 1 - timed_out.append(request) - for request in timed_out: - self.disconnect_client(request.client) - return len(timed_out) + return self._ensure_job_delivery_service().sweep_initial_job_timeouts(now=now) def initial_job_timeout_loop(self) -> None: - while not self.stop_event.wait(1.0): - self.sweep_initial_job_timeouts() + return self._ensure_job_delivery_service().initial_job_timeout_loop() def _record_heartbeat(self, name: str, *, phase: str | None = None) -> None: self._ensure_watchdog_state() @@ -12641,120 +5694,45 @@ def _record_heartbeat(self, name: str, *, phase: str | None = None) -> None: def _block_work_heartbeat_owner(self) -> tuple[str, str] | None: """Return the independent heartbeat/phase slots owned by this thread.""" - current = threading.get_ident() - if current == getattr(self, "_block_submitter_thread_ident", None): - return "block_submitter", "_block_submitter_phase" - if current == getattr(self, "_block_accounting_thread_ident", None): - return "block_accounting", "_block_accounting_phase" - return None + return self._ensure_block_candidate_service()._block_work_heartbeat_owner() def _record_block_work_heartbeat(self, name: str, phase: str) -> None: """Record a phase while preserving one-argument heartbeat embedders.""" - heartbeat = self._record_heartbeat - try: - heartbeat(name, phase=phase) - except TypeError as exc: - # Preserve the historical one-argument heartbeat seam used by - # focused embedders. Do not hide TypeErrors raised by a heartbeat - # implementation that did accept the keyword. - if "unexpected keyword argument 'phase'" not in str(exc): - raise - heartbeat(name) + self._ensure_block_candidate_service()._record_block_work_heartbeat( + name, + phase, + ) def _record_block_submitter_phase(self, phase: str) -> None: """Stamp a named phase only from a dedicated block-work owner.""" - owner = self._block_work_heartbeat_owner() - if owner is None: - return - heartbeat_name, phase_attribute = owner - setattr(self, phase_attribute, phase) - self._record_block_work_heartbeat(heartbeat_name, phase) + self._ensure_block_candidate_service()._record_block_submitter_phase(phase) def _record_block_submitter_wait(self, phase: str) -> None: """Heartbeat owner waits while preserving lightweight test behavior.""" - owner = self._block_work_heartbeat_owner() - if owner is None and not hasattr(self, "_block_submitter_thread_ident"): - self._record_heartbeat("block_submitter") - return - self._record_block_submitter_phase(phase) + self._ensure_block_candidate_service()._record_block_submitter_wait(phase) def _block_work_wait_slice(self) -> float: """Choose a polling slice that stays inside the configured watchdog.""" - watchdog_budget = max( - 0.001, - float(getattr(self, "watchdog_timeout_seconds", 120.0)), - ) - return min( - BLOCK_SUBMITTER_WAIT_HEARTBEAT_SLICE_SECONDS, - max(0.001, watchdog_budget * 0.9), - ) + return self._ensure_block_candidate_service()._block_work_wait_slice() def _observe_coordinator_lock_wait(self, elapsed_seconds: float) -> None: """Keep a sliced coordinator-lock wait visible and watchdog-safe.""" - owner = self._block_work_heartbeat_owner() - if owner is None: - return - heartbeat_name, phase_attribute = owner - current_phase = getattr(self, phase_attribute, "unknown") - wait_phase = f"wait-lock:coordinator-state:{current_phase}" - self._record_block_work_heartbeat(heartbeat_name, wait_phase) - now = time.monotonic() - log_interval = float( - getattr( - self, - "block_submit_lock_wait_log_seconds", - DEFAULT_BLOCK_SUBMIT_LOCK_WAIT_LOG_SECONDS, - ) - ) - last_log = float( - getattr(self, "_block_submitter_last_lock_wait_log_monotonic", 0.0) + self._ensure_block_candidate_service()._observe_coordinator_lock_wait( + elapsed_seconds ) - if last_log <= 0 or now - last_log >= log_interval: - self._block_submitter_last_lock_wait_log_monotonic = now - print( - "prism coordinator: block submitter waiting on lock " - f"lock=coordinator-state phase={current_phase} " - f"elapsed={elapsed_seconds:.3f}s", - flush=True, - ) def _acquire_block_submitter_lock(self, lock: Any, name: str) -> None: """Acquire a submit-path lock in heartbeat/logging slices.""" - owner = self._block_work_heartbeat_owner() - if owner is None and ( - hasattr(self, "_block_submitter_thread_ident") - or hasattr(self, "_block_accounting_thread_ident") - ): - lock.acquire() - return - started = time.monotonic() - last_log = started - log_interval = float( - getattr( - self, - "block_submit_lock_wait_log_seconds", - DEFAULT_BLOCK_SUBMIT_LOCK_WAIT_LOG_SECONDS, - ) + self._ensure_block_candidate_service()._acquire_block_submitter_lock( + lock, + name, ) - while not lock.acquire(timeout=self._block_work_wait_slice()): - phase = f"wait-lock:{name}" - self._record_block_submitter_wait(phase) - now = time.monotonic() - if now - last_log >= log_interval: - print( - "prism coordinator: block submitter waiting on lock " - f"lock={name} elapsed={now - started:.3f}s", - flush=True, - ) - last_log = now - @contextmanager def _block_submitter_lock(self, lock: Any, name: str) -> Iterator[None]: - self._acquire_block_submitter_lock(lock, name) - try: - yield - finally: - lock.release() + return self._ensure_block_candidate_service()._block_submitter_lock( + lock, + name, + ) def _overdue_heartbeats(self, now: float) -> list[str]: self._ensure_watchdog_state() @@ -12768,298 +5746,407 @@ def _overdue_heartbeats(self, now: float) -> list[str]: overdue.append(f"{name}:{phase}" if phase else name) return sorted(overdue) - def _pause_watchdog_heartbeat(self, name: str) -> None: - self._ensure_watchdog_state() - with self._heartbeats_lock: - self._watchdog_pauses[name] = self._watchdog_pauses.get(name, 0) + 1 - self._heartbeats[name] = time.monotonic() + def _pause_watchdog_heartbeat(self, name: str) -> None: + self._ensure_watchdog_state() + with self._heartbeats_lock: + self._watchdog_pauses[name] = self._watchdog_pauses.get(name, 0) + 1 + self._heartbeats[name] = time.monotonic() + + def _resume_watchdog_heartbeat(self, name: str) -> None: + self._ensure_watchdog_state() + with self._heartbeats_lock: + depth = self._watchdog_pauses.get(name, 0) + if depth <= 1: + self._watchdog_pauses.pop(name, None) + else: + self._watchdog_pauses[name] = depth - 1 + self._heartbeats[name] = time.monotonic() + + def _remove_watchdog_heartbeat(self, name: str) -> None: + self._ensure_watchdog_state() + with self._heartbeats_lock: + self._heartbeats.pop(name, None) + self._heartbeat_phases.pop(name, None) + self._watchdog_pauses.pop(name, None) + + def _registered_watchdog_heartbeat_names(self, *names: str) -> tuple[str, ...]: + self._ensure_watchdog_state() + with self._heartbeats_lock: + return tuple(name for name in names if name in self._heartbeats) + + def stratum_accept_heartbeat_names(self) -> tuple[str, ...]: + return configured_accept_heartbeat_names( + getattr(self, "listener_profiles", None) + ) + + def _make_background_service_registry(self) -> BackgroundServiceRegistry: + """Describe process loops in their historical shutdown-join order. + + Every specification is registered unconditionally with a + call-time-resolved target so post-construction test monkeypatches of + the loop methods and of the enable/interval attributes keep working; + serve() applies the live conditional at each named start. The + watchdog specification is deliberately absent here: serve() registers + it inline at its start boundary so the mandatory + publication-progress watchdog thread is created before any + synchronous startup prewarm/recovery work. The one capability + conditional is hashrate rollup maintenance: it exists only for + ledgers that expose ``advance_hashrate_rollups`` (the Postgres + backend), because a memory-backed ledger never grows that method at + runtime -- unlike an enable flag, the capability cannot flip after + construction, so registering an unstartable service would only + mislead the registry snapshot. + """ + specifications = [ + BackgroundServiceSpec( + name="qbit_blockpoll", + thread_name="prism-qbit-block-poll", + target=lambda: self.blockpoll_loop(), + daemon=True, + join_timeout=1.0, + watchdog_monitored=True, + registration_identity=("qbit_blockpoll",), + ), + BackgroundServiceSpec( + name="block_submitter", + thread_name="prism-block-submitter", + target=lambda: self.block_submit_loop(), + daemon=True, + join_timeout=1.0, + watchdog_monitored=True, + registration_identity=("block_submitter",), + ), + BackgroundServiceSpec( + name="qbit_blockwait", + thread_name="prism-qbit-block-wait", + target=lambda: self.blockwait_loop(), + daemon=True, + join_timeout=1.0, + watchdog_monitored=True, + registration_identity=("qbit_blockwait",), + ), + BackgroundServiceSpec( + name="vardiff_idle_sweep", + thread_name="prism-vardiff-idle-sweep", + target=lambda: self.vardiff_idle_sweep_loop(), + daemon=True, + join_timeout=1.0, + watchdog_monitored=True, + registration_identity=("vardiff_idle_sweep",), + ), + BackgroundServiceSpec( + name="initial_job_timeout_sweep", + thread_name="prism-initial-job-timeouts", + target=lambda: self.initial_job_timeout_loop(), + daemon=True, + join_timeout=1.0, + watchdog_monitored=False, + registration_identity=("initial_job_timeout_sweep",), + ), + BackgroundServiceSpec( + name="share_writer", + thread_name="prism-share-writer", + target=lambda: self.share_append_loop(), + daemon=True, + join_timeout=5.0, + watchdog_monitored=True, + registration_identity=("share_writer",), + ), + BackgroundServiceSpec( + name="ctv_fanout_broadcaster", + thread_name="prism-ctv-fanout-broadcaster", + target=lambda: self.ctv_fanout_broadcaster_loop(), + daemon=True, + join_timeout=1.0, + watchdog_monitored=True, + registration_identity=("ctv_fanout_broadcaster",), + ), + self._health_snapshot_service_spec(), + self._metrics_snapshot_service_spec(), + ] + if callable( + getattr(getattr(self, "ledger", None), "advance_hashrate_rollups", None) + ): + specifications.append( + BackgroundServiceSpec( + name="hashrate-rollup-maintenance", + thread_name="prism-hashrate-rollup-maintenance", + target=lambda: self.hashrate_rollup_maintenance_loop(), + daemon=True, + join_timeout=1.0, + watchdog_monitored=False, + registration_identity=("hashrate-rollup-maintenance",), + ) + ) + return BackgroundServiceRegistry(specifications) + + def _health_snapshot_service_spec(self) -> BackgroundServiceSpec: + return BackgroundServiceSpec( + name="health_snapshot_refresher", + thread_name="prism-health-snapshot-refresher", + target=lambda: self.health_snapshot_loop(), + daemon=True, + join_timeout=1.0, + watchdog_monitored=False, + registration_identity=("health_snapshot_refresher",), + ) + + def _metrics_snapshot_service_spec(self) -> BackgroundServiceSpec: + return BackgroundServiceSpec( + name="metrics_snapshot_refresher", + thread_name="prism-metrics-snapshot-refresher", + target=lambda: self.metrics_snapshot_loop(), + daemon=True, + join_timeout=1.0, + watchdog_monitored=False, + registration_identity=("metrics_snapshot_refresher",), + ) + + def _ensure_background_services(self) -> BackgroundServiceRegistry: + registry = getattr(self, "_background_services", None) + if registry is None: + registry = self._make_background_service_registry() + self._background_services = registry + return registry + + def _start_background_service(self, name: str) -> threading.Thread: + registry = self._ensure_background_services() + return registry.start( + name, + on_started=lambda specification: ( + self._record_heartbeat(specification.name) + if specification.watchdog_monitored + else None + ), + ) - def _resume_watchdog_heartbeat(self, name: str) -> None: - self._ensure_watchdog_state() - with self._heartbeats_lock: - depth = self._watchdog_pauses.get(name, 0) - if depth <= 1: - self._watchdog_pauses.pop(name, None) - else: - self._watchdog_pauses[name] = depth - 1 - self._heartbeats[name] = time.monotonic() + def _start_secondary_accept_service( + self, + server: socket.socket, + profile: StratumListenerProfile, + ) -> threading.Thread: + registry = self._ensure_background_services() + service_name = profile.heartbeat_name + registry.register_if_absent( + BackgroundServiceSpec( + name=service_name, + thread_name=f"prism-stratum-accept-{profile.name}", + target=lambda: self.accept_loop(server, profile), + daemon=True, + join_timeout=1.0, + watchdog_monitored=True, + registration_identity=( + "secondary_stratum_accept", + id(server), + id(profile), + ), + ) + ) + return self._start_background_service(service_name) - def _remove_watchdog_heartbeat(self, name: str) -> None: - self._ensure_watchdog_state() - with self._heartbeats_lock: - self._heartbeats.pop(name, None) - self._heartbeat_phases.pop(name, None) - self._watchdog_pauses.pop(name, None) + # Lease heartbeat state is owned by LedgerLeaseHeartbeatService; these + # descriptors keep the historical private coordinator attribute names + # readable/writable for tests and embedders with exactly one owner copy. + _ledger_lease_heartbeat_freshness_lock = _LeaseHeartbeatStateField( + "freshness_lock" + ) + _ledger_lease_heartbeat_failure_lock = _LeaseHeartbeatStateField( + "failure_lock" + ) + _ledger_lease_heartbeat_last_success_monotonic = _LeaseHeartbeatStateField( + "last_success_monotonic" + ) + _ledger_lease_heartbeat_last_progress_monotonic = _LeaseHeartbeatStateField( + "last_progress_monotonic" + ) + _ledger_lease_heartbeat_last_server_proven_monotonic = ( + _LeaseHeartbeatStateField("last_server_proven_monotonic") + ) + _ledger_lease_heartbeat_failed = _LeaseHeartbeatStateField("failed") + _ledger_lease_heartbeat_ready = _LeaseHeartbeatStateField("ready") + _ledger_lease_heartbeat_stop_event = _LeaseHeartbeatStateField("stop_event") + _ledger_lease_heartbeat_thread = _LeaseHeartbeatStateField("thread") + _ledger_lease_heartbeat_monitor_thread = _LeaseHeartbeatStateField( + "monitor_thread" + ) + _ledger_lease_heartbeat_exit_thread = _LeaseHeartbeatStateField( + "exit_thread" + ) + _ledger_lease_heartbeat_failure_reason = _LeaseHeartbeatStateField( + "failure_reason" + ) + _ledger_lease_heartbeat_failure_has_traceback = _LeaseHeartbeatStateField( + "failure_has_traceback" + ) - def _registered_watchdog_heartbeat_names(self, *names: str) -> tuple[str, ...]: - self._ensure_watchdog_state() - with self._heartbeats_lock: - return tuple(name for name in names if name in self._heartbeats) + def _ensure_lease_heartbeat_service(self) -> LedgerLeaseHeartbeatService: + """Single-flight lazy root for the lease heartbeat owner. - def stratum_accept_heartbeat_names(self) -> tuple[str, ...]: - profiles = getattr(self, "listener_profiles", None) - if not profiles: - return ("stratum_accept",) - return tuple(profile.heartbeat_name for profile in profiles) + Every port resolves live coordinator state at call time so the + documented monkeypatch seams (test timing attributes, patched + ``_ledger_lease_heartbeat_hard_exit`` / ``_watchdog_hard_exit`` + methods, replaced loop methods, swapped ledgers) keep working. + """ + service = self.__dict__.get("_lease_heartbeat_service") + if service is not None: + return service + # CPython's setdefault is atomic under the GIL. This lazy path exists + # for focused tests that construct a coordinator with __new__; normal + # instances reach it before any process thread starts. + init_lock = self.__dict__.setdefault( + "_lease_heartbeat_service_init_lock", + threading.Lock(), + ) + with init_lock: + service = self.__dict__.get("_lease_heartbeat_service") + if service is not None: + return service + service = LedgerLeaseHeartbeatService( + LedgerLeaseHeartbeatPorts( + ledger=lambda: getattr(self, "ledger", None), + heartbeat_seconds=lambda: float( + getattr( + self, + "ledger_lease_heartbeat_seconds", + DEFAULT_PRISM_LEDGER_LEASE_HEARTBEAT_SECONDS, + ) + ), + failure_seconds=lambda: float( + getattr( + self, + "ledger_lease_heartbeat_failure_seconds", + DEFAULT_PRISM_LEDGER_LEASE_HEARTBEAT_FAILURE_SECONDS, + ) + ), + monitor_seconds=lambda: float( + getattr( + self, + "ledger_lease_heartbeat_monitor_seconds", + DEFAULT_PRISM_LEDGER_LEASE_HEARTBEAT_MONITOR_SECONDS, + ) + ), + exit_timeout_seconds=lambda: float( + getattr( + self, + "ledger_lease_heartbeat_exit_timeout_seconds", + DEFAULT_PRISM_LEDGER_LEASE_HEARTBEAT_EXIT_TIMEOUT_SECONDS, + ) + ), + external_fence_timeout_seconds=lambda: float( + getattr( + self, + "ledger_lease_external_fence_timeout_seconds", + DEFAULT_PRISM_LEDGER_LEASE_EXTERNAL_FENCE_TIMEOUT_SECONDS, + ) + ), + lease_hard_exit=( + lambda message, *, include_traceback: ( + self._ledger_lease_heartbeat_hard_exit( + message, + include_traceback=include_traceback, + ) + ) + ), + watchdog_hard_exit=( + lambda reason, *, timeout_seconds: ( + self._watchdog_hard_exit( + reason, + timeout_seconds=timeout_seconds, + ) + ) + ), + scheduler_slack_seconds=lambda: float( + getattr( + self, + "ledger_lease_heartbeat_scheduler_slack_seconds", + WRITER_LEASE_HEARTBEAT_SCHEDULER_SLACK_SECONDS, + ) + ), + heartbeat_loop=lambda: self.ledger_lease_heartbeat_loop(), + monitor_loop=lambda: ( + self.ledger_lease_heartbeat_monitor_loop() + ), + ) + ) + # Adopt pre-service fixture fields once and remove the legacy + # keys; the descriptors above route all later access. + for legacy_key, attribute in ( + ("_ledger_lease_heartbeat_freshness_lock", "freshness_lock"), + ("_ledger_lease_heartbeat_failure_lock", "failure_lock"), + ( + "_ledger_lease_heartbeat_last_success_monotonic", + "last_success_monotonic", + ), + ( + "_ledger_lease_heartbeat_last_progress_monotonic", + "last_progress_monotonic", + ), + ( + "_ledger_lease_heartbeat_last_server_proven_monotonic", + "last_server_proven_monotonic", + ), + ("_ledger_lease_heartbeat_failed", "failed"), + ("_ledger_lease_heartbeat_ready", "ready"), + ("_ledger_lease_heartbeat_stop_event", "stop_event"), + ("_ledger_lease_heartbeat_thread", "thread"), + ("_ledger_lease_heartbeat_monitor_thread", "monitor_thread"), + ("_ledger_lease_heartbeat_exit_thread", "exit_thread"), + ("_ledger_lease_heartbeat_failure_reason", "failure_reason"), + ( + "_ledger_lease_heartbeat_failure_has_traceback", + "failure_has_traceback", + ), + ): + if legacy_key in self.__dict__: + setattr(service, attribute, self.__dict__.pop(legacy_key)) + self.__dict__["_lease_heartbeat_service"] = service + return service def _record_ledger_lease_heartbeat_success( self, renewal_started_monotonic: float, ) -> None: """Advance the conservative freshness edge without regression.""" - freshness_lock = self.__dict__.setdefault( - "_ledger_lease_heartbeat_freshness_lock", - threading.Lock(), + self._ensure_lease_heartbeat_service().record_success( + renewal_started_monotonic ) - with freshness_lock: - last_success = getattr( - self, - "_ledger_lease_heartbeat_last_success_monotonic", - None, - ) - if ( - last_success is None - or renewal_started_monotonic > last_success - ): - self._ledger_lease_heartbeat_last_success_monotonic = ( - renewal_started_monotonic - ) def _record_ledger_lease_heartbeat_progress(self) -> None: - """Stamp mid-verification progress for the staleness monitor only. - - Fires from inside verify_writer_lease_guard_session when its - attribution recheck runs a second statement: the first statement's - round trip completed, proving the heartbeat machinery is making - progress. Deliberately separate from the success timestamp — the - verification has not yet reached a verdict, so nothing that treats - success as authorization freshness may observe this mark. Only the - heartbeat monitor reads it, so a lawful two-statement verification - is not mistaken for a wedged heartbeat while a genuinely stuck - statement still ages toward the failure budget unimpeded. - """ - self._ledger_lease_heartbeat_last_progress_monotonic = time.monotonic() + """Stamp mid-verification progress for the staleness monitor only.""" + self._ensure_lease_heartbeat_service().record_progress() def _record_ledger_lease_heartbeat_server_proven(self) -> None: - """Stamp a completed server round trip: progress plus the envelope edge. - - Client-side progress marks (attempt start, query-slot acquisition) - keep a lawfully slow verification alive under the monitor, but they - can postdate a silent guard-session death by up to one idle - interval, so they cannot carry the adoption-envelope guarantee - (hard exit completes before a replacement's silence window can - elapse). Only a response actually received from PostgreSQL — a - completed statement round trip or a whole verification — proves - the session was alive approximately now, so only these sites stamp - the server-proven edge the monitor's envelope cap measures from. - """ - now = time.monotonic() - self._ledger_lease_heartbeat_last_progress_monotonic = now - self._ledger_lease_heartbeat_last_server_proven_monotonic = now + """Stamp a completed server round trip: progress plus envelope edge.""" + self._ensure_lease_heartbeat_service().record_server_proven() def _ledger_lease_adoption_silence_seconds(self) -> float: - """Resolve the ledger's adoption-silence window for timing checks. - - PsqlShareLedger keeps the value on a private attribute with no - public alias; read both spellings so operator overrides actually - reach the misconfiguration warning and the monitor's envelope cap - instead of silently comparing against the compiled default. - """ - ledger = getattr(self, "ledger", None) - for name in ( - "lease_adoption_silence_seconds", - "_lease_adoption_silence_seconds", - ): - value = getattr(ledger, name, None) - if value: - return float(value) - return float(DEFAULT_WRITER_LEASE_ADOPTION_SILENCE_SECONDS) + """Resolve the ledger's adoption-silence window for timing checks.""" + return self._ensure_lease_heartbeat_service().adoption_silence_seconds() def _ledger_lease_heartbeat_activity_age_seconds(self) -> float: - """Age of the newest monitor-visible heartbeat activity mark. - - The success timestamp is deliberately the call-start edge (authority - freshness must never look newer than what PostgreSQL proved), so it - alone cannot answer "is the heartbeat advancing?": back-to-back - verifications that each lawfully approach the statement budget age - the freshest success edge by two call durations plus the idle - interval, and a budget sized for one statement would hard-exit a - healthy sole writer. Staleness enforcers therefore measure silence - from the newest of the success edge and the progress mark, which - carries attempt starts, query-slot acquisition, statement round - trips, and completions. A verification that stops advancing stops - stamping, so a wedged statement still ages out on the same budget. - """ - last_success = float( - getattr( - self, - "_ledger_lease_heartbeat_last_success_monotonic", - time.monotonic(), - ) - ) - last_progress = float( - getattr( - self, - "_ledger_lease_heartbeat_last_progress_monotonic", - last_success, - ) - ) - return max(0.0, time.monotonic() - max(last_success, last_progress)) + """Age of the newest monitor-visible heartbeat activity mark.""" + return self._ensure_lease_heartbeat_service().activity_age_seconds() @staticmethod def _ledger_lease_guard_session_verifier( ledger: object, ) -> Callable[..., object] | None: - """Return the guarded-session liveness check for ``ledger``. - - Prefer ``verify_writer_lease_guard_session``: it proves the dedicated - guard session is live and renews the lease TTL only via SKIP LOCKED, - so it can never queue behind the row lock a long fenced transaction - (``persist_accepted_block``) holds until commit and then die on the - guard's statement timeout, yet an idle coordinator still keeps its - lease from expiring under different-identity claimants. - ``renew_writer_lease_heartbeat`` is the legacy embedder spelling and - does wait on the lease tuple; it is only a fallback for ledgers that - predate the non-blocking verification. - """ - verify = getattr(ledger, "verify_writer_lease_guard_session", None) - if verify is not None: - return verify - return getattr(ledger, "renew_writer_lease_heartbeat", None) + """Return the guarded-session liveness check for ``ledger``.""" + return _lease_guard_session_verifier(ledger) + + @staticmethod + def _ledger_lease_guard_session_prover( + ledger: object, + ) -> Callable[..., object] | None: + """Return the cheap read-only ownership proof for ``ledger``.""" + return _lease_guard_session_prover(ledger) + + def _ledger_lease_heartbeat_snapshot(self) -> dict[str, object]: + """Phase attribution and policy terms for the metrics surface.""" + return self._ensure_lease_heartbeat_service().snapshot() def _start_ledger_lease_heartbeat(self) -> threading.Thread | None: - existing = getattr(self, "_ledger_lease_heartbeat_thread", None) - if existing is not None and existing.is_alive(): - return existing - ledger = getattr(self, "ledger", None) - guard_required = bool( - getattr(ledger, "writer_lease_guard_required", False) - ) - if not bool( - getattr(ledger, "writer_lease_fast_adoption_capable", False) - ): - if guard_required: - # The session token published to the lease row promises a - # guarded heartbeat session, but the advisory guard is already - # gone (for example the dedicated connection dropped between - # ledger construction and serve()). Continuing would let a - # replacement adopt the lease after one silence window while - # this process keeps running as a fenced zombie. - self._ledger_lease_heartbeat_hard_exit( - "prism coordinator: writer lease guard was lost before " - "the lease heartbeat started; hard-exiting so a " - "replacement does not adopt while this process is live", - include_traceback=False, - ) - return None - verify = self._ledger_lease_guard_session_verifier(ledger) - if verify is None: - if guard_required: - self._ledger_lease_heartbeat_hard_exit( - "prism coordinator: writer lease guard requires a " - "heartbeat but the ledger cannot verify the guarded " - "session; hard-exiting instead of running unguarded", - include_traceback=False, - ) - return None - armed_started_monotonic = time.monotonic() - self._ledger_lease_heartbeat_last_success_monotonic = ( - armed_started_monotonic - ) - self._ledger_lease_heartbeat_last_server_proven_monotonic = ( - armed_started_monotonic - ) - self._ledger_lease_heartbeat_failed = threading.Event() - self._ledger_lease_heartbeat_failure_lock = threading.Lock() - self._ledger_lease_heartbeat_ready = threading.Event() - self._ledger_lease_heartbeat_stop_event = threading.Event() - thread = threading.Thread( - target=self.ledger_lease_heartbeat_loop, - name="prism-ledger-lease-heartbeat", - daemon=True, - ) - self._ledger_lease_heartbeat_thread = thread - thread.start() - monitor_thread = threading.Thread( - target=self.ledger_lease_heartbeat_monitor_loop, - name="prism-ledger-lease-heartbeat-monitor", - daemon=True, - ) - monitor_thread.start() - self._ledger_lease_heartbeat_monitor_thread = monitor_thread - failure_seconds = float( - getattr( - self, - "ledger_lease_heartbeat_failure_seconds", - DEFAULT_PRISM_LEDGER_LEASE_HEARTBEAT_FAILURE_SECONDS, - ) - ) - interval_seconds = float( - getattr( - self, - "ledger_lease_heartbeat_seconds", - DEFAULT_PRISM_LEDGER_LEASE_HEARTBEAT_SECONDS, - ) - ) - silence_seconds = self._ledger_lease_adoption_silence_seconds() - monitor_seconds = float( - getattr( - self, - "ledger_lease_heartbeat_monitor_seconds", - DEFAULT_PRISM_LEDGER_LEASE_HEARTBEAT_MONITOR_SECONDS, - ) - ) - exit_seconds = float( - getattr( - self, - "ledger_lease_heartbeat_exit_timeout_seconds", - DEFAULT_PRISM_LEDGER_LEASE_HEARTBEAT_EXIT_TIMEOUT_SECONDS, - ) - ) - if ( - failure_seconds <= interval_seconds - or silence_seconds - <= failure_seconds + exit_seconds + 2.0 * monitor_seconds - ): - # The hard-exit ordering argument needs interval < budget (one - # idle wait must not exhaust the budget) and enough headroom - # between the budget and the adoption silence for the monitor's - # server-proven envelope cap to fit its poll and exit costs - # (this process must be gone before a replacement may CAS). - # The derived defaults satisfy both; only operator overrides - # can break them, so warn rather than refuse. - print( - "prism coordinator: ledger lease heartbeat timing is " - f"misconfigured: failure budget {failure_seconds:g}s must " - f"exceed the heartbeat interval {interval_seconds:g}s and " - f"leave the adoption silence {silence_seconds:g}s at least " - f"{exit_seconds + 2.0 * monitor_seconds:g}s of envelope " - "headroom; continuing, but hard-exit is no longer " - "guaranteed to precede replacement adoption", - flush=True, - ) - # The startup fencing deadline measures silence from the newest - # monitor-visible activity, not from arming: a first verification - # that is legally slower than the whole budget end-to-end but - # demonstrably advancing (queue wait, then a statement round trip - # near the guard's statement timeout) is not a wedged first beat. - # A genuinely stuck first statement stamps nothing and ages out on - # the same budget the monitor enforces. - while not self._ledger_lease_heartbeat_ready.wait(0.01): - if self._ledger_lease_heartbeat_failed.is_set(): - return None - if ( - self._ledger_lease_heartbeat_activity_age_seconds() - < failure_seconds - ): - continue - self._ledger_lease_heartbeat_hard_exit( - "prism coordinator: initial ledger lease heartbeat did not " - "complete before startup fencing deadline", - include_traceback=False, - ) - return None - return thread + return self._ensure_lease_heartbeat_service().start() def _ledger_lease_heartbeat_hard_exit( self, @@ -13067,492 +6154,31 @@ def _ledger_lease_heartbeat_hard_exit( *, include_traceback: bool, ) -> None: - failure_lock = self.__dict__.setdefault( - "_ledger_lease_heartbeat_failure_lock", - threading.Lock(), - ) - with failure_lock: - failed = self.__dict__.setdefault( - "_ledger_lease_heartbeat_failed", - threading.Event(), - ) - if failed.is_set(): - return - failed.set() - # This thread now proceeds to _watchdog_hard_exit and stays parked - # there until os._exit; it can never issue another renewal. The - # release worker must not join it, or a heartbeat/monitor-driven - # exit deadlocks against its own release and burns the exit budget - # before the fresh-connection lease release can run. - self._ledger_lease_heartbeat_exit_thread = ( - threading.current_thread() - ) - # Never write to stdout/stderr before arming the hard-exit path. A full - # container log pipe can block a flush forever, leaving writer - # admission open and the old process alive. Keep the detail in memory - # for tests/embedders whose patched hard-exit returns; the real process - # exits and Docker's die event is the authoritative diagnostic. - self._ledger_lease_heartbeat_failure_reason = message - self._ledger_lease_heartbeat_failure_has_traceback = include_traceback - self._watchdog_hard_exit( - "lease_heartbeat", - timeout_seconds=float( - getattr( - self, - "ledger_lease_heartbeat_exit_timeout_seconds", - DEFAULT_PRISM_LEDGER_LEASE_HEARTBEAT_EXIT_TIMEOUT_SECONDS, - ) - ), + self._ensure_lease_heartbeat_service().hard_exit( + message, + include_traceback=include_traceback, ) def ledger_lease_heartbeat_loop(self) -> None: - """Keep proving the guarded session live on an isolated DB path. - - The heartbeat must never wait on the lease tuple that fenced writes - hold for whole transactions, or a long ``persist_accepted_block`` - would time out the heartbeat statement and hard-exit a healthy - coordinator (the block-39416 restart loop). It still renews the - lease TTL whenever that tuple is uncontended, so an idle coordinator - (no fenced writes, CTV broadcaster disabled) does not let - ``lease_expires_at`` lapse into different-identity expiry claims. - """ - ledger = getattr(self, "ledger", None) - verify = self._ledger_lease_guard_session_verifier(ledger) - if verify is None: - return - interval_seconds = float( - getattr( - self, - "ledger_lease_heartbeat_seconds", - DEFAULT_PRISM_LEDGER_LEASE_HEARTBEAT_SECONDS, - ) - ) - heartbeat_stop = getattr( - self, - "_ledger_lease_heartbeat_stop_event", - None, - ) - if heartbeat_stop is None: - heartbeat_stop = threading.Event() - self._ledger_lease_heartbeat_stop_event = heartbeat_stop - # The verification may lawfully run a second statement (the - # attribution recheck) whose duration the monitor's failure budget - # cannot absorb — that budget must stay under the adoption silence, - # so it is sized for one statement. A verification that reports - # per-statement progress lets the monitor count the completed first - # round trip instead of hard-exiting a healthy coordinator mid-way - # through a lawful recheck. - try: - verify_parameters = inspect.signature(verify).parameters - except (TypeError, ValueError): - verify_parameters = {} - verify_reports_query_start = "on_query_start" in verify_parameters - verify_reports_statement_progress = ( - "on_statement_progress" in verify_parameters - ) - while not heartbeat_stop.is_set(): - verify_started_monotonic = time.monotonic() - # An advancing verification is monitor-visible activity. The - # failure budget is sized for one statement round trip, so the - # monitor must measure silence between demonstrable steps of a - # verification (attempt start, query-slot acquisition, statement - # progress, completion), never between success edges: those are - # call-start times, so back-to-back verifications that each - # lawfully approach the statement budget age the freshest edge - # by two call durations plus the idle interval and the monitor - # would hard-exit a healthy sole writer. Progress marks feed - # only the staleness monitor — authority freshness still moves - # exclusively on the conservative success edge below. The - # attempt-start and slot-acquisition marks are client-side and - # can postdate a silent guard-session death by at most one idle - # interval, so they carry only the liveness budget; the - # adoption envelope (hard exit before a replacement's silence - # window elapses) is enforced separately by the monitor's - # server-proven cap, which measures from completed round trips - # alone. During any such hang the external-effect fence also - # queues behind the hung statement and fails closed, so no new - # external effect authorizes while the monitor ages out. - self._record_ledger_lease_heartbeat_progress() - verify_kwargs: dict[str, Callable[[], None]] = {} - if verify_reports_query_start: - verify_kwargs["on_query_start"] = ( - self._record_ledger_lease_heartbeat_progress - ) - if verify_reports_statement_progress: - verify_kwargs["on_statement_progress"] = ( - self._record_ledger_lease_heartbeat_server_proven - ) - try: - verify(**verify_kwargs) - except Exception: - self._ledger_lease_heartbeat_hard_exit( - "prism coordinator: ledger lease heartbeat failed; " - "hard-exiting so this process cannot outlive its " - "fast-adoptable session", - include_traceback=True, - ) - return - # The session was proven live no earlier than call start. Using - # that conservative edge prevents a delayed response from making - # local freshness look newer than what PostgreSQL actually proved. - self._record_ledger_lease_heartbeat_success( - verify_started_monotonic - ) - # Completion is the newest demonstrable activity and a real - # server response; the monitor may observe it even though - # authority freshness stays pinned to the call-start edge - # recorded above. - self._record_ledger_lease_heartbeat_server_proven() - ready = getattr(self, "_ledger_lease_heartbeat_ready", None) - if ready is not None: - ready.set() - if heartbeat_stop.wait(interval_seconds): - return + self._ensure_lease_heartbeat_service().heartbeat_loop() def ledger_lease_heartbeat_monitor_loop(self) -> None: - failure_seconds = float( - getattr( - self, - "ledger_lease_heartbeat_failure_seconds", - DEFAULT_PRISM_LEDGER_LEASE_HEARTBEAT_FAILURE_SECONDS, - ) - ) - monitor_seconds = float( - getattr( - self, - "ledger_lease_heartbeat_monitor_seconds", - DEFAULT_PRISM_LEDGER_LEASE_HEARTBEAT_MONITOR_SECONDS, - ) - ) - heartbeat_stop = getattr( - self, - "_ledger_lease_heartbeat_stop_event", - None, - ) - if heartbeat_stop is None: - heartbeat_stop = threading.Event() - self._ledger_lease_heartbeat_stop_event = heartbeat_stop - # The adoption-envelope cap: the client-side progress marks that - # keep a lawfully slow verification alive can postdate a silent - # guard-session death by up to one idle interval, which with the - # derived defaults (interval + budget = silence) would let the - # hard exit land at or after a replacement's CAS eligibility. - # Server-proven marks cannot postdate the death, so a second bound - # measured from them — sized to leave room for one monitor poll on - # each side plus the hard-exit budget inside the silence window — - # restores exit-before-adoption regardless of client marks. It is - # floored at the failure budget so a misconfigured (warned) tiny - # silence degrades to the plain liveness bound instead of killing - # lawful single-statement verifications. - exit_seconds = float( - getattr( - self, - "ledger_lease_heartbeat_exit_timeout_seconds", - DEFAULT_PRISM_LEDGER_LEASE_HEARTBEAT_EXIT_TIMEOUT_SECONDS, - ) - ) - server_cap_seconds = max( - failure_seconds, - self._ledger_lease_adoption_silence_seconds() - - exit_seconds - - 2.0 * monitor_seconds, - ) - while not heartbeat_stop.wait(monitor_seconds): - # Mid-verification progress counts: the budget — sized for one - # statement so it stays under the adoption silence — cannot - # absorb a whole multi-step verification, and each demonstrable - # step (attempt start, slot acquisition, statement round trip, - # completion) is the advancing-heartbeat evidence this monitor - # watches for. A wedged statement records nothing and still - # ages out. - age_seconds = self._ledger_lease_heartbeat_activity_age_seconds() - last_server_proven = float( - getattr( - self, - "_ledger_lease_heartbeat_last_server_proven_monotonic", - time.monotonic(), - ) - ) - server_age_seconds = max( - 0.0, - time.monotonic() - last_server_proven, - ) - if ( - age_seconds < failure_seconds - and server_age_seconds < server_cap_seconds - ): - continue - self._ledger_lease_heartbeat_hard_exit( - "prism coordinator: ledger lease heartbeat stopped making " - f"progress for {age_seconds:.3f}s " - f"(server-proven {server_age_seconds:.3f}s); hard-exiting " - "before its session becomes fast-adoptable", - include_traceback=False, - ) - return - - def _stop_ledger_lease_heartbeat( - self, - *, - deadline: float | None = None, - ) -> bool: - heartbeat_stop = getattr( - self, - "_ledger_lease_heartbeat_stop_event", - None, - ) - if heartbeat_stop is None: - return True - heartbeat_stop.set() - threads = ( - getattr(self, "_ledger_lease_heartbeat_thread", None), - getattr(self, "_ledger_lease_heartbeat_monitor_thread", None), - ) - # A heartbeat or monitor thread that armed the hard exit is blocked in - # _watchdog_hard_exit joining the very release worker that runs this - # method. It has already stopped renewing forever, so treat it as - # stopped instead of deadlocking against it until the deadline. - exit_thread = getattr( - self, - "_ledger_lease_heartbeat_exit_thread", - None, - ) - if deadline is None: - deadline = ( - time.monotonic() - + DEFAULT_WRITER_LEASE_ADOPTION_SILENCE_SECONDS - ) - for thread in threads: - if ( - thread is None - or thread is threading.current_thread() - or thread is exit_thread - ): - continue - thread.join(max(0.0, deadline - time.monotonic())) - return all( - thread is None - or thread is threading.current_thread() - or thread is exit_thread - or not thread.is_alive() - for thread in threads - ) - - def _require_fresh_ledger_lease_for_external_side_effect( - self, - component: str, - ) -> None: - """Synchronously verify the exact guarded session before an RPC effect. - - The periodic heartbeat is an early process-liveness detector, not an - authorization oracle: CLOCK_MONOTONIC may not advance across host - suspend, and a PostgreSQL connection object does not necessarily know - its server session died until the next I/O. Every external mutation - therefore performs a bounded non-blocking verification on the session - holding the advisory guard. It must not wait on the lease row: a long - fenced transaction (``persist_accepted_block``) holds that tuple's - row lock until commit, and queueing behind it would time out and - fence a healthy coordinator (the TTL is renewed opportunistically via - SKIP LOCKED instead). A daemon worker bounds a dead network path even - when the driver's server-side statement timeout cannot. - - PostgreSQL and qbitd are independent systems, so this remains a - preflight fence rather than an atomic transaction with the subsequent - RPC. The advisory session remains held across the RPC in the ordinary - case; see the deployment guide for the residual post-check pause risk. - - A verification that reports ``renewal_deferred_to_own_write`` proves - liveness but not authority: the writer's own fenced write holds the - expired lease row, and only its commit guarantees the lease survives. - That is enough for the heartbeat, which merely keeps a healthy - process from self-fencing, but not for an RPC — a rollback would let - a queued claimant take the lease while the effect is in flight. This - fence then raises :class:`WriterLeaseRenewalDeferred` without arming - the hard exit; callers already retry deferred work on their own - cadence. - """ - ledger = getattr(self, "ledger", None) - if not bool( - getattr(ledger, "writer_lease_guard_required", False) - ): - return - verify = self._ledger_lease_guard_session_verifier(ledger) - if verify is None: - self._ledger_lease_heartbeat_hard_exit( - "prism coordinator: refusing external side effect from " - f"{component}; exact-session verification is unavailable", - include_traceback=False, - ) - raise ShutdownInProgress( - f"writer lease guard verification is unavailable before {component}" - ) - - # The configured fence timeout budgets one statement; the - # verification may lawfully run a second inside the same guarded - # slot (the attribution recheck), and killing that recheck at the - # single-statement deadline would hard-exit a coordinator the - # recheck was about to prove healthy. Two moderately slow but - # in-deadline statements must both fit. - timeout_seconds = WRITER_LEASE_VERIFICATION_MAX_STATEMENTS * max( - 0.0, - float( - getattr( - self, - "ledger_lease_external_fence_timeout_seconds", - DEFAULT_PRISM_LEDGER_LEASE_EXTERNAL_FENCE_TIMEOUT_SECONDS, - ) - ), - ) - # The guarded session serializes this verification behind the periodic - # heartbeat and any concurrent fence, and the execution budget above - # tracks the guard's statement timeout. Waiting for the serialized - # query slot must not consume that budget, or a fence arriving during - # an in-flight renewal hard-exits a coordinator whose session is still - # healthy. The queue wait gets the heartbeat failure budget instead: a - # guard session that cannot complete queued work within it is already - # being declared dead by the heartbeat monitor. - queue_timeout_seconds = max( - timeout_seconds, - float( - getattr( - self, - "ledger_lease_heartbeat_failure_seconds", - DEFAULT_PRISM_LEDGER_LEASE_HEARTBEAT_FAILURE_SECONDS, - ) - ), - ) - query_started = threading.Event() - try: - verify_parameters = inspect.signature(verify).parameters - except (TypeError, ValueError): - verify_parameters = {} - verify_reports_query_start = "on_query_start" in verify_parameters - # This verification holds the guard's serialized slot for its whole - # statement sequence, so the periodic heartbeat queues behind it - # and records nothing meanwhile. Its lawful second statement (the - # attribution recheck) can therefore age the heartbeat's last - # success past the monitor's single-statement failure budget — the - # same hazard the heartbeat loop's own recheck had — so this - # caller must feed the monitor the completed first round trip too. - verify_reports_statement_progress = ( - "on_statement_progress" in verify_parameters - ) - if not verify_reports_query_start: - # A verification that cannot report when its query slot was - # acquired keeps the previous behavior: the whole budget covers - # both. - query_started.set() - - outcome: queue.Queue[BaseException | None] = queue.Queue(maxsize=1) - verification_result: list[object] = [] - verification_started_monotonic = time.monotonic() - - def verify_exact_session() -> None: - try: - verify_kwargs: dict[str, Callable[[], None]] = {} - if verify_reports_query_start: - verify_kwargs["on_query_start"] = query_started.set - if verify_reports_statement_progress: - verify_kwargs["on_statement_progress"] = ( - self._record_ledger_lease_heartbeat_server_proven - ) - result = verify(**verify_kwargs) - except BaseException as exc: - outcome.put(exc) - else: - verification_result.append(result) - outcome.put(None) - - try: - thread = threading.Thread( - target=verify_exact_session, - name="prism-ledger-lease-external-fence", - daemon=True, - ) - thread.start() - except BaseException as exc: - self._ledger_lease_heartbeat_hard_exit( - "prism coordinator: refusing external side effect from " - f"{component}; could not start exact-session verification", - include_traceback=False, - ) - raise ShutdownInProgress( - f"writer lease guard verification could not start before {component}" - ) from exc - - queue_deadline = verification_started_monotonic + queue_timeout_seconds - while not query_started.is_set() and thread.is_alive(): - remaining = queue_deadline - time.monotonic() - if remaining <= 0.0: - self._ledger_lease_heartbeat_hard_exit( - "prism coordinator: refusing external side effect from " - f"{component}; exact-session verification could not " - f"start within {queue_timeout_seconds:g}s", - include_traceback=False, - ) - raise ShutdownInProgress( - f"writer lease guard verification timed out before {component}" - ) - thread.join(min(0.01, remaining)) - thread.join(timeout_seconds) - if thread.is_alive(): - self._ledger_lease_heartbeat_hard_exit( - "prism coordinator: refusing external side effect from " - f"{component}; exact-session verification exceeded " - f"{timeout_seconds:g}s", - include_traceback=False, - ) - raise ShutdownInProgress( - f"writer lease guard verification timed out before {component}" - ) + self._ensure_lease_heartbeat_service().monitor_loop() - try: - error = outcome.get_nowait() - except queue.Empty as exc: - self._ledger_lease_heartbeat_hard_exit( - "prism coordinator: refusing external side effect from " - f"{component}; exact-session verification returned no result", - include_traceback=False, - ) - raise ShutdownInProgress( - f"writer lease guard verification failed before {component}" - ) from exc - if error is not None: - self._ledger_lease_heartbeat_hard_exit( - "prism coordinator: refusing external side effect from " - f"{component}; exact-session verification failed", - include_traceback=False, - ) - raise ShutdownInProgress( - f"writer lease guard verification failed before {component}" - ) from error - - # Use the call-start time so scheduler delay never makes this success - # appear fresher than the database response actually proves. The - # response itself is a fresh server round trip, so the monitor's - # envelope cap may take it at receipt time. - self._record_ledger_lease_heartbeat_success( - verification_started_monotonic - ) - self._record_ledger_lease_heartbeat_server_proven() - - # A verification that proved liveness only because the writer's own - # fenced write holds the expired lease row is not authority for an - # external side effect: that survival argument assumes the write - # commits, and a rollback would hand the row to a queued - # different-identity claimant while the RPC is in flight. Refuse the - # side effect without fencing the process — the session is healthy - # and the write's commit will land the next renewal; callers retry - # on their own cadence (broadcast interval, candidate outbox replay). - result = verification_result[0] if verification_result else None - if isinstance(result, dict) and result.get( - "renewal_deferred_to_own_write" - ): - raise WriterLeaseRenewalDeferred( - f"withholding {component}: writer lease renewal is deferred " - "behind this coordinator's own in-flight fenced write; " - "retry after that transaction commits and a renewal lands" - ) + def _stop_ledger_lease_heartbeat( + self, + *, + deadline: float | None = None, + ) -> bool: + return self._ensure_lease_heartbeat_service().stop(deadline=deadline) + + def _require_fresh_ledger_lease_for_external_side_effect( + self, + component: str, + ) -> None: + """Synchronously verify the exact guarded session before an RPC effect.""" + self._ensure_lease_heartbeat_service( + ).require_fresh_lease_for_external_side_effect(component) @contextmanager def _watchdog_paused(self, *names: str) -> Iterator[None]: @@ -13565,53 +6191,7 @@ def _watchdog_paused(self, *names: str) -> Iterator[None]: self._resume_watchdog_heartbeat(name) def watchdog_loop(self) -> None: - while True: - if self.stop_event.wait(self.watchdog_interval_seconds): - if getattr(self, "_fatal_exit_requested", False): - print( - "prism coordinator: fatal block-work restart requested; " - "exiting non-zero even if the main thread is blocked", - flush=True, - ) - os._exit(1) - return - now = time.monotonic() - ( - publication_failure, - coordination_age, - coordination_budget, - publication_budget, - ) = self._publication_watchdog_state(now) - if publication_failure == "coordination": - self._watchdog_failure_detail = ( - "prism coordinator: publication-progress watchdog firing; " - "template refresh remained coordination-blocked past the " - f"coordination budget={coordination_budget:g}s " - f"streak_age={coordination_age:.3f}s" - ) - self._watchdog_hard_exit("coordination") - if publication_failure == "publication": - self._watchdog_failure_detail = ( - "prism coordinator: publication-progress watchdog firing; " - "current tip/generation remained unpublished past the " - f"template refresh failure budget={publication_budget:g}s" - ) - self._watchdog_hard_exit("publication") - overdue = ( - self._overdue_heartbeats(now) - if getattr(self, "watchdog_enabled", True) - else [] - ) - if overdue: - self._watchdog_failure_detail = ( - "prism coordinator: liveness watchdog firing; unresponsive " - f"subsystems={overdue} " - f"timeout={self.watchdog_timeout_seconds:g}s" - ) - # Queued shares have not been acknowledged. Miners reconnect - # and retry them after restart; exact-payload replay is - # idempotent if Postgres committed just before this exit. - self._watchdog_hard_exit("liveness") + self._ensure_watchdog_service().run() def _watchdog_hard_exit( self, @@ -13619,92 +6199,98 @@ def _watchdog_hard_exit( *, timeout_seconds: float | None = None, ) -> None: - """Bound a fresh-thread lease release, then terminate unconditionally.""" - try: - if timeout_seconds is None: - timeout_seconds = float( - getattr( + """Dynamic hard-exit seam; the bounded-exit body is service-owned. + + Both WatchdogService.run and the ledger-lease heartbeat service + trigger this coordinator method at call time so per-instance + monkeypatches intercept every exit path. + """ + self._ensure_watchdog_service().hard_exit( + reason, + timeout_seconds=timeout_seconds, + ) + + def _ensure_watchdog_service(self) -> WatchdogService: + service = self.__dict__.get("_watchdog_service") + if service is None: + # Idempotent construction; no init lock is needed. + service = WatchdogService( + WatchdogPorts( + wait_for_stop=lambda timeout: self.stop_event.wait(timeout), + interval_seconds=lambda: self.watchdog_interval_seconds, + fatal_exit_requested=lambda: bool( + getattr(self, "_fatal_exit_requested", False) + ), + publication_state=lambda now: ( + self._publication_watchdog_state(now) + ), + hard_exit=lambda reason: self._watchdog_hard_exit(reason), + liveness_enabled=lambda: bool( + getattr(self, "watchdog_enabled", True) + ), + overdue_heartbeats=lambda now: self._overdue_heartbeats(now), + liveness_timeout_seconds=lambda: ( + self.watchdog_timeout_seconds + ), + coordination_budget_seconds=lambda: float( + getattr( + self, + "coordination_blocked_exit_seconds", + DEFAULT_PRISM_COORDINATION_BLOCKED_EXIT_SECONDS, + ) + ), + publication_budget_seconds=lambda: float( + getattr( + self, + "template_refresh_failure_exit_seconds", + DEFAULT_PRISM_TEMPLATE_MAX_AGE_SECONDS, + ) + ), + ensure_job_cache_state=self._ensure_job_cache_state, + publication_failure_expired=lambda now: ( + self.publication_progress_failure_expired(now) + ), + publication_divergence_since=( + self._publication_divergence_since_locked + ), + lease_release_timeout_seconds=lambda: float( + getattr( + self, + "watchdog_lease_release_timeout_seconds", + DEFAULT_PRISM_WATCHDOG_LEASE_RELEASE_TIMEOUT_SECONDS, + ) + ), + shutdown_controller=lambda: self._ensure_shutdown_controller(), + request_shutdown=lambda: self.request_shutdown(None), + release_ledger_lease=lambda deadline: ( + self.release_ledger_lease( + fresh_connection=True, + deadline=deadline, + emit_logs=False, + ) + ), + lease_failure_reason=lambda: getattr( self, - "watchdog_lease_release_timeout_seconds", - DEFAULT_PRISM_WATCHDOG_LEASE_RELEASE_TIMEOUT_SECONDS, - ) + "_ledger_lease_heartbeat_failure_reason", + None, + ), + exit_process=lambda code: os._exit(code), + log=lambda message: print(message, flush=True), ) - deadline = time.monotonic() + max(0.0, timeout_seconds) - release_thread = threading.Thread( - target=self._watchdog_release_ledger_lease, - args=(reason, deadline), - name="prism-watchdog-lease-release", - daemon=True, ) - release_thread.start() - release_thread.join(max(0.0, deadline - time.monotonic())) - finally: - # Nothing, including timeout logging or thread-start failure, may - # extend or suppress the watchdog's terminal action. - os._exit(1) - - def _watchdog_release_ledger_lease( - self, - reason: str, - deadline: float, - ) -> bool: - """Use only the shutdown controller and a fresh DB connection. + self.__dict__["_watchdog_service"] = service + return service - Do not call ``shutdown`` here: cancellation takes ``self.lock``, which - may belong to the subsystem that triggered the watchdog. Closing - writer admission plus the controller's tracked-writer barrier retains - the graceful path's release-withheld invariant without that lock. + def _publication_divergence_since_locked(self) -> float | None: + """Locked snapshot of the progress-health divergence timestamp. - Any best-effort diagnostic is emitted only after lease handling on this - daemon worker. A blocked container log pipe may park the worker, but it - cannot precede the release attempt or extend the caller's hard deadline. + Progress-health keeps ownership of the divergence state; the + watchdog reads it through this port while holding its own + coordination lock (watchdog lock before progress-health lock, + never the reverse). """ - controller = self._ensure_shutdown_controller() - controller.request_shutdown(None) - self.stop_event.set() - if not controller.begin_shutdown(f"watchdog_{reason}"): - handled = controller.wait_for_lease_handling() - self._watchdog_exit_diagnostic(reason, lease_handled=handled) - return handled - - quiesced, _elapsed, _blockers = controller.wait_for_writer_quiescence( - max(0.0, deadline - time.monotonic()) - ) - if not quiesced: - self._watchdog_exit_diagnostic(reason, lease_handled=False) - return False - if time.monotonic() >= deadline: - self._watchdog_exit_diagnostic(reason, lease_handled=False) - return False - handled = self.release_ledger_lease( - fresh_connection=True, - deadline=deadline, - emit_logs=False, - ) - self._watchdog_exit_diagnostic(reason, lease_handled=handled) - return handled - - def _watchdog_exit_diagnostic( - self, - reason: str, - *, - lease_handled: bool, - ) -> None: - """Best-effort logging after the watchdog's safety-critical work.""" - detail = ( - getattr(self, "_ledger_lease_heartbeat_failure_reason", None) - if reason == "lease_heartbeat" - else getattr(self, "_watchdog_failure_detail", None) - ) - try: - print( - (detail or f"prism coordinator: {reason} watchdog firing") - + ". Exiting non-zero so the restart policy recovers the process. " - + f"lease_handled={lease_handled}", - flush=True, - ) - except Exception: - pass + with self._progress_health_lock: + return self._progress_publication_divergence_since_monotonic def _ensure_shutdown_controller(self) -> CoordinatorShutdownController: controller = getattr(self, "_shutdown_controller", None) @@ -13760,12 +6346,8 @@ def _shutdown_log(event: str, **fields: object) -> None: ) def _cancel_active_tip_refresh_for_shutdown(self) -> None: - self._ensure_tip_refresh_state() - with self.lock: - active = self._active_tip_refresh - if active is not None: - active[1].cancel() - self._tip_refresh_retry.clear() + return self._ensure_tip_refresh_service()._cancel_active_tip_refresh_for_shutdown( + ) def shutdown(self, *, reason: str = "graceful") -> bool: """Quiesce every ledger writer and release its lease exactly once. @@ -13905,7 +6487,25 @@ def drain_non_writer_components( if not controller.claim_non_writer_drain(): return started = time.monotonic() - for thread, timeout in threads or []: + drain_threads: Sequence[tuple[threading.Thread, float]] + if threads is None: + # Registry-owned drain: only started registered services, in + # registration order. serve() passes an explicit list combining + # this with its non-registry threads; focused shutdown callers + # keep their historical explicit-thread seam. + drain_threads = self._ensure_background_services().threads_to_drain() + else: + drain_threads = threads + # The audit HTTP facade is demand-created; stop it only if it ever + # existed. Its bounded stop runs after lease release and before the + # non-writer thread joins so the listener port frees promptly. + audit_http = self.__dict__.get("_audit_http_facade") + if audit_http is not None and not audit_http.stop(): + self._shutdown_log( + "audit_http_stop", + outcome="timeout", + ) + for thread, timeout in drain_threads: thread.join(timeout=timeout) self.shutdown_vardiff_idle_executor() self.shutdown_tip_refresh_executor() @@ -13932,54 +6532,25 @@ def open_stratum_listeners( draining its shutdown. Returns None when shutdown is requested during the retry, so startup can abort gracefully. """ - backlog = int( - getattr(self, "stratum_listen_backlog", DEFAULT_PRISM_STRATUM_LISTEN_BACKLOG) - ) - retry_seconds = float( - getattr( - self, - "stratum_bind_retry_seconds", - DEFAULT_PRISM_STRATUM_BIND_RETRY_SECONDS, - ) + return StratumSessionService.open_stratum_listeners( + listener_stack, + self.listener_profiles, + backlog=int( + getattr( + self, + "stratum_listen_backlog", + DEFAULT_PRISM_STRATUM_LISTEN_BACKLOG, + ) + ), + retry_seconds=float( + getattr( + self, + "stratum_bind_retry_seconds", + DEFAULT_PRISM_STRATUM_BIND_RETRY_SECONDS, + ) + ), + stop_event=getattr(self, "stop_event", None), ) - listeners: list[tuple[socket.socket, StratumListenerProfile]] = [] - for profile in self.listener_profiles: - server = listener_stack.enter_context( - socket.socket(socket.AF_INET, socket.SOCK_STREAM) - ) - server.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) - bind_deadline = time.monotonic() + retry_seconds - warned = False - while True: - try: - server.bind((profile.bind, profile.port)) - break - except OSError as exc: - if exc.errno != errno.EADDRINUSE or time.monotonic() >= bind_deadline: - raise - stop_event = getattr(self, "stop_event", None) - if stop_event is not None and stop_event.is_set(): - # Shutting down mid-startup: stop contending for a - # port this process will never serve. - print( - f"prism coordinator: shutdown requested while waiting " - f"to bind {profile.bind}:{profile.port}; aborting startup", - flush=True, - ) - return None - if not warned: - print( - f"prism coordinator: {profile.name} listener port " - f"{profile.bind}:{profile.port} is busy; retrying bind " - f"for up to {retry_seconds:g}s", - flush=True, - ) - warned = True - time.sleep(0.1) - server.listen(backlog) - server.settimeout(1) - listeners.append((server, profile)) - return listeners def serve(self) -> None: with ExitStack() as listener_stack: @@ -14047,21 +6618,32 @@ def _serve_with_listener_stack(self, listener_stack: ExitStack) -> None: return self._record_block_work_heartbeat("block_submitter", "starting") block_accounting_thread = self._start_block_accounting_thread() - block_submitter_thread = threading.Thread( - target=self.block_submit_loop, - name="prism-block-submitter", - daemon=True, - ) - block_submitter_thread.start() - drain_threads: list[tuple[threading.Thread, float]] = [ - (block_submitter_thread, 1.0), + self._start_background_service("block_submitter") + # Threads owned outside the background-service registry (the block + # accounting worker and the lease heartbeat pair below) keep their + # historical bounded joins alongside the registry-owned drains. + extra_drain_threads: list[tuple[threading.Thread, float]] = [ (block_accounting_thread, 1.0), ] # Publication progress is mandatory even when the operator disables # ordinary heartbeat checks. Start the watchdog before any synchronous # startup prewarm/recovery work: a timeout-ignoring block call can ask # for fail-stop while the main thread is itself wedged in that work. - threading.Thread(target=self.watchdog_loop, daemon=True).start() + # The specification is registered at this exact boundary (not in + # _make_background_service_registry) so serve() keeps owning the + # watchdog thread's creation order. + self._ensure_background_services().register_if_absent( + BackgroundServiceSpec( + name="watchdog", + thread_name="prism-watchdog", + target=self.watchdog_loop, + daemon=True, + join_timeout=1.0, + watchdog_monitored=False, + registration_identity=("watchdog_loop",), + ) + ) + self._start_background_service("watchdog") prepared = self.prewarm_startup_jobs() print( "prism coordinator: startup job preparation " @@ -14070,74 +6652,41 @@ def _serve_with_listener_stack(self, listener_stack: ExitStack) -> None: f"tip={self.tip_template_snapshot.bestblockhash if self.tip_template_snapshot else 'unknown'}", flush=True, ) - # Seed liveness before starting monitored loops so the watchdog - # never fires during startup. + # Seed listener liveness before accepting so the watchdog never fires + # during startup. Background-loop heartbeats derive from their service + # specifications at each named start below. for _, profile in listeners: self._record_heartbeat(profile.heartbeat_name) - self._record_heartbeat("qbit_blockpoll") - blockpoll_thread = threading.Thread(target=self.blockpoll_loop, daemon=True) - blockpoll_thread.start() - blockwait_thread: threading.Thread | None = None + self._start_background_service("qbit_blockpoll") if self.blockwait_enabled: - self._record_heartbeat("qbit_blockwait") - blockwait_thread = threading.Thread(target=self.blockwait_loop, daemon=True) - blockwait_thread.start() - vardiff_idle_sweep_thread: threading.Thread | None = None + self._start_background_service("qbit_blockwait") if self.vardiff_idle_sweep_seconds > 0: - self._record_heartbeat("vardiff_idle_sweep") - vardiff_idle_sweep_thread = threading.Thread( - target=self.vardiff_idle_sweep_loop, - daemon=True, - ) - vardiff_idle_sweep_thread.start() - initial_job_timeout_thread: threading.Thread | None = None + self._start_background_service("vardiff_idle_sweep") if self.stratum_initial_job_timeout_seconds > 0: - initial_job_timeout_thread = threading.Thread( - target=self.initial_job_timeout_loop, - name="prism-initial-job-timeouts", - daemon=True, - ) - initial_job_timeout_thread.start() - drain_threads.append((blockpoll_thread, 1.0)) + self._start_background_service("initial_job_timeout_sweep") if lease_heartbeat_thread is not None: - drain_threads.append((lease_heartbeat_thread, 1.0)) + extra_drain_threads.append((lease_heartbeat_thread, 1.0)) lease_heartbeat_monitor_thread = getattr( self, "_ledger_lease_heartbeat_monitor_thread", None, ) if lease_heartbeat_monitor_thread is not None: - drain_threads.append((lease_heartbeat_monitor_thread, 1.0)) - if blockwait_thread is not None: - drain_threads.append((blockwait_thread, 1.0)) - if vardiff_idle_sweep_thread is not None: - drain_threads.append((vardiff_idle_sweep_thread, 1.0)) - if initial_job_timeout_thread is not None: - drain_threads.append((initial_job_timeout_thread, 1.0)) + extra_drain_threads.append((lease_heartbeat_monitor_thread, 1.0)) # Replay any shares stranded on disk by a prior ledger-outage # shutdown before serving, so no acked share is lost across restart. if not self._run_startup_writer_replay( self.replay_recovered_shares, - drain_threads=drain_threads, + drain_threads=[ + *self._ensure_background_services().threads_to_drain(), + *extra_drain_threads, + ], ): return - self._record_heartbeat("share_writer") self.share_writer_active = True - share_writer_thread = threading.Thread( - target=self.share_append_loop, - daemon=True, - ) - share_writer_thread.start() - drain_threads.append((share_writer_thread, 5.0)) - ctv_broadcaster_thread: threading.Thread | None = None + self._start_background_service("share_writer") if self.ctv_broadcaster_enabled: - self._record_heartbeat("ctv_fanout_broadcaster") - ctv_broadcaster_thread = threading.Thread( - target=self.ctv_fanout_broadcaster_loop, - daemon=True, - ) - ctv_broadcaster_thread.start() - drain_threads.append((ctv_broadcaster_thread, 1.0)) + self._start_background_service("ctv_fanout_broadcaster") print( "prism coordinator: CTV fanout broadcaster enabled " f"mode={'cpfp' if self.ctv_broadcaster_fee_sats > 0 else 'direct'} " @@ -14148,6 +6697,7 @@ def _serve_with_listener_stack(self, listener_stack: ExitStack) -> None: f"chunk_size={self.ctv_broadcaster_chunk_size}", flush=True, ) + self._start_hashrate_rollup_maintenance_if_enabled() if self.watchdog_enabled: print( "prism coordinator: liveness and publication-progress watchdog enabled " @@ -14165,12 +6715,14 @@ def _serve_with_listener_stack(self, listener_stack: ExitStack) -> None: f"interval={self.watchdog_interval_seconds:g}s", flush=True, ) + # Load durable worker difficulty evidence before either listener starts + # accepting. The service performs one bounded startup read and then + # serves authorizations from its in-memory cache, so a miner's first + # request never waits on PostgreSQL. + vardiff_service = self._ensure_vardiff_service() + vardiff_service.start_durable_prune_scheduler() for extra_server, extra_profile in listeners[1:]: - threading.Thread( - target=self.accept_loop, - args=(extra_server, extra_profile), - daemon=True, - ).start() + self._start_secondary_accept_service(extra_server, extra_profile) try: self.accept_loop(*listeners[0]) finally: @@ -14185,7 +6737,12 @@ def _serve_with_listener_stack(self, listener_stack: ExitStack) -> None: # joins and the tip-refresh executor drain: those may be stuck # in unrelated client delivery or obsolete fanout work. self.shutdown(reason="serve_exit") - self.drain_non_writer_components(drain_threads) + self.drain_non_writer_components( + [ + *self._ensure_background_services().threads_to_drain(), + *extra_drain_threads, + ] + ) def _record_startup_phase_once(self, phase: str) -> None: """Record seconds from serve() start to a named startup phase, once. @@ -14216,18 +6773,13 @@ def startup_phase_seconds(self) -> dict[str, float]: def _note_block_replay_enumeration_owed(self) -> None: """Mark that pending durable candidates have not been enumerated yet.""" - with self.lock: - self._block_replay_enumeration_owed_flag = True + self._ensure_block_candidate_service()._note_block_replay_enumeration_owed() def _clear_block_replay_enumeration_owed(self) -> None: - with self.lock: - self._block_replay_enumeration_owed_flag = False + self._ensure_block_candidate_service()._clear_block_replay_enumeration_owed() def _block_replay_enumeration_owed(self) -> bool: - with self.lock: - return bool( - getattr(self, "_block_replay_enumeration_owed_flag", False) - ) + return self._ensure_block_candidate_service()._block_replay_enumeration_owed() def _run_startup_writer_replay( self, @@ -14247,161 +6799,10 @@ def _run_startup_writer_replay( def _run_startup_block_candidate_replay(self) -> bool: """Run best-effort pre-accept replay without dying on a slow ledger.""" - # Until the durable outbox has been enumerated once, this process - # cannot know whether a pending accepted candidate exists. Child job - # builds fail closed on that uncertainty instead of snapshotting a - # payout base that may omit a pending parent's carry (issue #188); - # replay_pending_block_candidates clears the flag on success. - self._note_block_replay_enumeration_owed() - try: - return self._run_startup_writer_replay( - self.replay_pending_block_candidates - ) - except TimeoutError: - print( - "prism coordinator: startup block candidate replay timed out " - "phase=replay-outbox-query " - f"timeout={self._block_landing_db_timeout():g}s; " - "continuing startup; block submitter loop will retry and " - "job builds stay blocked until pending candidates are known", - flush=True, - ) - return True + return self._ensure_block_candidate_service()._run_startup_block_candidate_replay() def accept_loop(self, server: socket.socket, profile: StratumListenerProfile) -> None: - while not self.stop_event.is_set(): - self._record_heartbeat(profile.heartbeat_name) - try: - sock, address = server.accept() - except socket.timeout: - continue - except OSError as exc: - # The listener socket is torn down by serve()'s ExitStack on - # shutdown while secondary accept threads may still be blocked - # in accept(). Descriptor exhaustion is recoverable: keep the - # accept loop alive and refresh its watchdog heartbeat while - # waiting for client/RPC descriptors to drain. - if self.stop_event.is_set(): - return - if exc.errno in {errno.EMFILE, errno.ENFILE}: - self._record_stratum_resource_exhaustion( - listener_name=profile.name, - location="accept", - error_number=exc.errno, - ) - self._wait_after_stratum_resource_failure(profile.heartbeat_name) - continue - raise - - if ( - self.stop_event.is_set() - or self._ensure_shutdown_controller().phase != "running" - ): - try: - sock.close() - except OSError: - pass - return - - with self.lock: - if ( - self.stop_event.is_set() - or self._ensure_shutdown_controller().phase != "running" - ): - try: - sock.close() - except OSError: - pass - return - max_connections = int( - getattr( - self, - "stratum_max_connections", - DEFAULT_PRISM_STRATUM_MAX_CONNECTIONS, - ) - ) - if max_connections > 0 and len(self.clients) >= max_connections: - rejection_count = self._note_connection_limit_rejection_locked("global") - client = None - else: - self.connection_counter += 1 - connection_id = self.connection_counter - client = ClientState( - sock=sock, - address=address, - connection_id=connection_id, - extranonce1_hex=f"{connection_id & 0xFFFFFFFF:08x}", - listener_name=profile.name, - listener_vardiff_config=profile.vardiff_config, - minimum_advertised_difficulty=profile.minimum_advertised_difficulty, - share_difficulty=self.client_startup_difficulty(profile), - ) - self.clients.add(client) - self._ensure_initial_job_state() - self.peak_active_connection_count = max( - self.peak_active_connection_count, - len(self.clients), - ) - if client is None: - try: - sock.close() - except OSError: - pass - if rejection_count == 1 or rejection_count % 100 == 0: - print( - "prism coordinator: rejected stratum connection at global limit " - f"limit={max_connections} count={rejection_count}", - flush=True, - ) - continue - try: - sock.settimeout(None) - self.apply_stratum_send_timeout(sock) - thread = threading.Thread(target=self.handle_client, args=(client,), daemon=True) - with self.lock: - client.handler_thread_registered = True - self.handler_thread_count += 1 - thread.start() - except (OSError, RuntimeError) as exc: - # Admission is atomic with the global count. Undo it if socket - # setup or thread creation fails before a handler owns cleanup, - # then keep this listener alive for the next connection. - try: - with self.lock: - if client.handler_thread_registered: - client.handler_thread_registered = False - self.handler_thread_count = max( - 0, - self.handler_thread_count - 1, - ) - self.disconnect_client(client) - except Exception: - print( - "prism coordinator: failed to fully close rejected stratum client " - f"address={address}", - flush=True, - ) - traceback.print_exc() - with self.lock: - self.connection_setup_failure_count = int( - getattr(self, "connection_setup_failure_count", 0) - ) + 1 - setup_failure_count = self.connection_setup_failure_count - if isinstance(exc, OSError) and exc.errno in {errno.EMFILE, errno.ENFILE}: - self._record_stratum_resource_exhaustion( - listener_name=profile.name, - location="connection-setup", - error_number=exc.errno, - ) - if setup_failure_count == 1 or setup_failure_count % 100 == 0: - print( - "prism coordinator: stratum connection setup failed; backing off " - f"listener={profile.name} address={address} " - f"error={exc!r} count={setup_failure_count}", - flush=True, - ) - self._wait_after_stratum_resource_failure(profile.heartbeat_name) - continue + self._ensure_stratum_session_service().accept_loop(server, profile) def _record_stratum_resource_exhaustion( self, @@ -14410,89 +6811,33 @@ def _record_stratum_resource_exhaustion( location: str, error_number: int | None, ) -> int: - with self.lock: - self.accept_resource_exhaustion_count = int( - getattr(self, "accept_resource_exhaustion_count", 0) - ) + 1 - exhaustion_count = self.accept_resource_exhaustion_count - if exhaustion_count == 1 or exhaustion_count % 100 == 0: - print( - "prism coordinator: stratum resource exhaustion " - f"listener={listener_name} location={location} errno={error_number} " - f"count={exhaustion_count}", - flush=True, - ) - return exhaustion_count + return self._ensure_stratum_session_service().record_stratum_resource_exhaustion( + listener_name=listener_name, + location=location, + error_number=error_number, + ) def _wait_after_stratum_resource_failure(self, heartbeat_name: str) -> None: - backoff_seconds = getattr( - self, - "stratum_accept_resource_exhaustion_backoff_seconds", - DEFAULT_PRISM_STRATUM_ACCEPT_RESOURCE_EXHAUSTION_BACKOFF_SECONDS, + self._ensure_stratum_session_service().wait_after_stratum_resource_failure( + heartbeat_name ) - remaining_seconds = max(0.0, float(backoff_seconds)) - watchdog_timeout_seconds = max( - 0.001, - float(getattr(self, "watchdog_timeout_seconds", 120.0)), - ) - heartbeat_interval_seconds = max( - 0.001, - min(1.0, watchdog_timeout_seconds / 2.0), - ) - deadline = time.monotonic() + remaining_seconds - while not self.stop_event.is_set(): - self._record_heartbeat(heartbeat_name) - remaining_seconds = deadline - time.monotonic() - if remaining_seconds <= 0: - return - if self.stop_event.wait(min(remaining_seconds, heartbeat_interval_seconds)): - return def _ensure_connection_capacity_state(self) -> None: - if not hasattr(self, "connection_limit_rejection_counts"): - self.connection_limit_rejection_counts = {"global": 0, "username": 0} + self._ensure_stratum_session_service() def _note_connection_limit_rejection_locked(self, scope: str) -> int: - self._ensure_connection_capacity_state() - count = int(self.connection_limit_rejection_counts.get(scope, 0)) + 1 - self.connection_limit_rejection_counts[scope] = count - return count + return self._ensure_stratum_session_service().registry.note_rejection_locked( + scope + ) def reserve_client_username(self, client: ClientState, worker: WorkerIdentity) -> bool: """Atomically reserve an exact Stratum username for one connection.""" - with self.lock: - self._ensure_connection_capacity_state() - limit = int( - getattr( - self, - "stratum_max_connections_per_username", - DEFAULT_PRISM_STRATUM_MAX_CONNECTIONS_PER_USERNAME, - ) - ) - active_for_username = sum( - 1 - for other in self.clients - if ( - other is not client - and other.worker is not None - and other.username == worker.username - ) - ) - if limit > 0 and active_for_username >= limit: - rejection_count = self._note_connection_limit_rejection_locked("username") - if rejection_count == 1 or rejection_count % 100 == 0: - print( - "prism coordinator: rejected stratum authorization at username limit " - f"username={worker.username!r} limit={limit} count={rejection_count}", - flush=True, - ) - return False - client.worker = worker - client.username = worker.username - return True + return self._ensure_stratum_session_service().reserve_client_username( + client, worker + ) def start_audit_server(self) -> None: - # Mark the nonblocking refresher running before the listener can + # Mark the nonblocking refreshers running before the listener can # dispatch a request: a probe retrying connection-refused queues in # the accept backlog at listen() time, and a handler serving it # before the flag is set would take cached_health_payload's legacy @@ -14500,14 +6845,12 @@ def start_audit_server(self) -> None: # on the handler thread and racing the refresher with a duplicate # query instead of returning the documented starting state. self.start_health_snapshot_refresher() + self.start_metrics_snapshot_refresher() # Bind before the health snapshot warm-up completes: the cold # accepted-share aggregate can take minutes on a grown ledger, and # for that whole window the listener must answer with an explicit # starting state instead of connection refused (issue #188 fix 4). - handler_cls = make_audit_handler(self) - httpd = ThreadingHTTPServer((self.audit_bind or "127.0.0.1", self.audit_port), handler_cls) - thread = threading.Thread(target=httpd.serve_forever, daemon=True) - thread.start() + self._ensure_audit_http_facade().start() self._record_startup_phase_once("audit_listener_bound") print( f"prism coordinator: audit HTTP listening on {self.audit_bind}:{self.audit_port}", @@ -14524,360 +6867,66 @@ def apply_stratum_send_timeout(self, sock: socket.socket) -> None: A plain socket timeout is not usable here: it would also apply to recv, disconnecting idle-but-healthy miners. """ - timeout_seconds = getattr( - self, - "stratum_send_timeout_seconds", - DEFAULT_PRISM_STRATUM_SEND_TIMEOUT_SECONDS, + apply_socket_send_timeout( + sock, + getattr( + self, + "stratum_send_timeout_seconds", + DEFAULT_PRISM_STRATUM_SEND_TIMEOUT_SECONDS, + ), ) - if timeout_seconds <= 0: - return - seconds = int(timeout_seconds) - microseconds = int((timeout_seconds - seconds) * 1_000_000) - try: - sock.setsockopt( - socket.SOL_SOCKET, - socket.SO_SNDTIMEO, - struct.pack("ll", seconds, microseconds), - ) - except (AttributeError, OSError, struct.error): - # Platform without SO_SNDTIMEO support: keep legacy blocking sends. - return def _wait_for_blockpoll_trigger(self) -> bool: - """Wait for the normal interval or an immediate coalesced retry.""" - remaining = float(self.blockpoll_seconds) - while remaining > 0: - if self.stop_event.is_set(): - return False - # After a failed pass, leave retry signals unconsumed until the - # spacing window closes. Re-checked per slice: a newly observed - # tip zeroes the holdoff immediately, and signals arriving during - # the hold stay coalesced for the attempt that eventually runs. - holdoff = self._tip_refresh_failure_holdoff_remaining() - if holdoff <= 0 and self._consume_tip_refresh_retry(): - return not self.stop_event.is_set() - wait_seconds = min(remaining, 0.25) - if holdoff > 0: - # The retry event may already be set; pace on the stop event - # so the spacing window cannot be spun through instantly. - # Short slices keep a newly detected tip's release prompt. - # Beat per slice: a deliberately held poller is idle, not - # hung, and must not trip the watchdog when the configured - # holdoff exceeds its timeout. - self._record_heartbeat("qbit_blockpoll") - wait_seconds = min(wait_seconds, holdoff, 0.05) - self.stop_event.wait(wait_seconds) - else: - self._tip_refresh_retry.wait(wait_seconds) - remaining -= wait_seconds - # Interval-driven attempts respect the same spacing so a sub-holdoff - # blockpoll interval cannot re-run a failed pass early. - while not self.stop_event.is_set(): - holdoff = self._tip_refresh_failure_holdoff_remaining() - if holdoff <= 0: - break - self._record_heartbeat("qbit_blockpoll") - self.stop_event.wait(min(holdoff, 0.05)) - self._consume_tip_refresh_retry() - return not self.stop_event.is_set() + return self._ensure_tip_refresh_service()._wait_for_blockpoll_trigger() def blockpoll_loop(self) -> None: - self._ensure_tip_refresh_state() - while self._wait_for_blockpoll_trigger(): - # A superseding observation or post-fanout tip change wakes this - # fallback loop immediately. The event coalesces repeated signals; - # ordinary same-tip polling retains its configured interval. - # Heartbeat at the top of each iteration: reaching here proves the - # loop is alive. A transient qbit RPC error still loops and beats; a - # hung RPC call never returns, so the beat goes stale and the - # watchdog restarts the process. - self._record_heartbeat("qbit_blockpoll") - try: - refreshed = self.poll_qbit_tip_template_once() - if refreshed: - print( - f"prism coordinator: refreshed {refreshed} client job(s) after qbit tip/template change", - flush=True, - ) - except ShutdownInProgress: - # Admission can close after the loop condition but before a - # nested reconciliation enters the writer gate. That is an - # intentional shutdown stop, not a template-health failure. - return - except (TemplateRefreshSuperseded, _PayoutStatePublicationBlocked) as exc: - # Coordination-blocked attempts remain outside the ordinary - # template failure budget. Their separate continuous-streak - # budget is recorded by poll_qbit_tip_template_once and - # enforced by the mandatory publication-progress watchdog. - # The retry is already scheduled by the raise site. - print( - f"prism coordinator: tip/template refresh superseded; retrying: {exc}", - flush=True, - ) - except Exception: - print("prism coordinator: qbit tip/template poll failed", flush=True) - traceback.print_exc() - if self.template_refresh_failure_expired(time.monotonic()): - print( - "prism coordinator: template refresh failure budget exhausted; " - "exiting non-zero so the restart policy recovers the process", - flush=True, - ) - os._exit(1) + return self._ensure_tip_refresh_service().blockpoll_loop() def template_refresh_failure_expired(self, now: float) -> bool: - budget = float( - getattr( - self, - "template_refresh_failure_exit_seconds", - DEFAULT_PRISM_TEMPLATE_MAX_AGE_SECONDS, - ) - ) - if budget <= 0: - return False - failure_started = getattr(self, "template_refresh_failure_started_monotonic", None) - return failure_started is not None and now - failure_started >= budget + return self._ensure_tip_refresh_service().template_refresh_failure_expired(now) def publication_progress_failure_expired(self, now: float) -> bool: - """Bound sustained detected/current publication divergence. - - Retry-loop heartbeats prove only that the driver is scheduled. This - dedicated deadline is independent of client-delivery health and is - cleared only by current work publication/delivery, so old delivery - backlog cannot make a later legitimate divergence expire immediately. - """ - budget = float( - getattr( - self, - "template_refresh_failure_exit_seconds", - DEFAULT_PRISM_TEMPLATE_MAX_AGE_SECONDS, - ) - ) - if budget <= 0: - return False - self._ensure_job_cache_state() - with self._progress_health_lock: - divergence_since = ( - self._progress_publication_divergence_since_monotonic - ) - return bool( - divergence_since is not None - and now - divergence_since >= budget + return self._ensure_tip_refresh_service().publication_progress_failure_expired( + now, ) - def _ensure_coordination_blocked_state(self) -> None: - if not hasattr(self, "_coordination_blocked_lock"): - self._coordination_blocked_lock = threading.Lock() - if not hasattr(self, "_coordination_blocked_since_monotonic"): - self._coordination_blocked_since_monotonic: float | None = None - if not hasattr(self, "_publication_watchdog_exit_claimed"): - self._publication_watchdog_exit_claimed = False - def _record_coordination_blocked_refresh(self, now: float) -> None: - self._ensure_coordination_blocked_state() - with self._coordination_blocked_lock: - if self._publication_watchdog_exit_claimed: - return - if self._coordination_blocked_since_monotonic is None: - self._coordination_blocked_since_monotonic = now + self._ensure_watchdog_service().record_coordination_blocked_refresh(now) def _clear_coordination_blocked_streak(self) -> None: - self._ensure_coordination_blocked_state() - with self._coordination_blocked_lock: - # Once the watchdog claims an expired generation, its hard exit is - # the terminal action. Otherwise a refresh completing between the - # deadline check and os._exit could appear to cancel an exit that - # already won the streak lock. - if not self._publication_watchdog_exit_claimed: - self._coordination_blocked_since_monotonic = None + self._ensure_watchdog_service().clear_coordination_blocked_streak() def coordination_blocked_streak_age_seconds( self, now: float | None = None, ) -> float: - self._ensure_coordination_blocked_state() - now = time.monotonic() if now is None else now - with self._coordination_blocked_lock: - started = self._coordination_blocked_since_monotonic - return 0.0 if started is None else max(0.0, now - started) - - def coordination_blocked_streak_expired(self, now: float) -> bool: - budget = float( - getattr( - self, - "coordination_blocked_exit_seconds", - DEFAULT_PRISM_COORDINATION_BLOCKED_EXIT_SECONDS, + return ( + self._ensure_watchdog_service().coordination_blocked_streak_age_seconds( + now ) ) - self._ensure_coordination_blocked_state() - with self._coordination_blocked_lock: - started = self._coordination_blocked_since_monotonic - return bool( - started is not None - and (budget <= 0 or now - started >= budget) - ) - - def _publication_watchdog_state( - self, - now: float, - ) -> tuple[str | None, float, float, float]: - """Arbitrate coordination and ordinary publication deadlines. - The final decision is serialized with coordination streak changes. - A streak recorded while the ordinary publication check is in flight - therefore owns the longer coordination deadline; once either deadline - is claimed, later refresh results cannot cancel the terminal exit. - """ - coordination_budget = float( - getattr( - self, - "coordination_blocked_exit_seconds", - DEFAULT_PRISM_COORDINATION_BLOCKED_EXIT_SECONDS, - ) - ) - publication_budget = float( - getattr( - self, - "template_refresh_failure_exit_seconds", - DEFAULT_PRISM_TEMPLATE_MAX_AGE_SECONDS, + def coordination_blocked_streak_expired(self, now: float) -> bool: + return ( + self._ensure_watchdog_service().coordination_blocked_streak_expired( + now ) ) - self._ensure_job_cache_state() - self._ensure_coordination_blocked_state() - - # This preflight keeps the common healthy case cheap. Its result is - # revalidated under both state locks before an ordinary exit is - # claimed, so a concurrent publication cannot leave a stale verdict. - publication_expired = self.publication_progress_failure_expired(now) - with self._coordination_blocked_lock: - started = self._coordination_blocked_since_monotonic - if started is not None: - age = max(0.0, now - started) - if coordination_budget <= 0 or age >= coordination_budget: - self._publication_watchdog_exit_claimed = True - return ( - "coordination", - age, - coordination_budget, - publication_budget, - ) - return None, age, coordination_budget, publication_budget - if publication_expired: - with self._progress_health_lock: - divergence_since = ( - self._progress_publication_divergence_since_monotonic - ) - publication_expired = bool( - publication_budget > 0 - and divergence_since is not None - and now - divergence_since >= publication_budget - ) - if publication_expired: - self._publication_watchdog_exit_claimed = True - return ( - "publication", - 0.0, - coordination_budget, - publication_budget, - ) - return None, 0.0, coordination_budget, publication_budget + def _publication_watchdog_state( + self, + now: float, + ) -> tuple[str | None, float, float, float]: + return self._ensure_watchdog_service().publication_watchdog_state(now) def _record_template_refresh_failure(self, now: float) -> None: - budget = float( - getattr( - self, - "template_refresh_failure_exit_seconds", - DEFAULT_PRISM_TEMPLATE_MAX_AGE_SECONDS, - ) - ) - if ( - budget > 0 - and getattr(self, "template_refresh_failure_started_monotonic", None) is None - ): - self.template_refresh_failure_started_monotonic = now + return self._ensure_tip_refresh_service()._record_template_refresh_failure(now) def blockwait_once(self, known_tip: str) -> str: - """One waitfornewblock round: returns the tip after the wait. - - qbitd returns as soon as its tip differs from ``known_tip`` (or after - the server-side timeout, echoing the current tip), so a tip observed - between our last poll and this call is reported immediately rather - than being missed for a cycle. - """ - timeout_seconds = getattr( - self, - "blockwait_timeout_seconds", - DEFAULT_PRISM_BLOCKWAIT_TIMEOUT_SECONDS, - ) - watchdog_timeout = float(getattr(self, "watchdog_timeout_seconds", 120.0)) - max_rpc_timeout = max(1.0, watchdog_timeout * 0.8) - timeout_seconds = min(float(timeout_seconds), max(1.0, max_rpc_timeout - 1.0)) - result = self.rpc.call( - "waitfornewblock", - [max(1, int(timeout_seconds * 1000)), known_tip], - timeout=timeout_seconds + 10.0, - ) - if isinstance(result, dict): - new_tip = str(result.get("hash", "") or "") - if new_tip: - return new_tip - return known_tip + return self._ensure_tip_refresh_service().blockwait_once(known_tip) def blockwait_loop(self) -> None: - """Push-style tip detection alongside the interval poller. - - Stale rejects are dominated by the window between a block connecting - and miners receiving fresh work; the poller alone leaves up to a full - PRISM_BLOCKPOLL_SECONDS of that window. This loop parks inside - waitfornewblock and triggers the same refresh path within milliseconds - of a new tip. The poller stays on as the fallback and still owns - same-tip template refreshes, which waitfornewblock does not signal. - Disabled cleanly when qbitd does not support the RPC. - """ - known_tip: str | None = None - while not self.stop_event.is_set(): - self._record_heartbeat("qbit_blockwait") - try: - if known_tip is None: - known_tip = str(self.rpc.call("getbestblockhash")) - self.observe_tip_for_refresh(known_tip) - new_tip = self.blockwait_once(known_tip) - if new_tip == known_tip: - if self.stop_event.wait(0.25): - return - continue - # Advance the wait cursor before observation/logging. If either - # operation raises, the next wait must not rediscover the same - # already-seen transition and create a notification storm. - known_tip = new_tip - # Detection can supersede/cancel obsolete heavy work, but only - # the blockpoll single-flight owner may fetch, build, and - # publish replacement work. - try: - self.observe_tip_for_refresh(new_tip) - finally: - # Preserve the wake even if observation-side cancellation - # or bookkeeping raises after known_tip advanced. - self._schedule_tip_refresh_retry() - print( - f"prism coordinator: blockwait saw new tip {new_tip}; " - "single-flight refresh scheduled", - flush=True, - ) - except Exception as exc: - if known_tip is not None and self._blockwait_unsupported(exc): - print( - "prism coordinator: waitfornewblock unavailable on this qbitd; " - "tip detection falls back to blockpoll only", - flush=True, - ) - self._remove_watchdog_heartbeat("qbit_blockwait") - return - print("prism coordinator: blockwait pass failed", flush=True) - traceback.print_exc() - if self.stop_event.wait(min(5.0, self.blockpoll_seconds)): - return + return self._ensure_tip_refresh_service().blockwait_loop() @staticmethod def _blockwait_unsupported(exc: Exception) -> bool: @@ -14895,115 +6944,103 @@ def _blockwait_unsupported(exc: Exception) -> bool: ) def make_ctv_fanout_broadcast_daemon(self) -> CtvFanoutBroadcastDaemon: - if self.ctv_broadcaster_fee_sats > 0 and not self.ctv_broadcaster_wallet: - raise ValueError( - "ctv_broadcaster_wallet is required when ctv_broadcaster_fee_sats is positive" - ) - broadcaster = CtvFanoutBroadcaster( - self.rpc.call, - funding_wallet=self.ctv_broadcaster_wallet, - before_external_side_effect=( - self._require_fresh_ledger_lease_for_external_side_effect - ), - ) - return CtvFanoutBroadcastDaemon( - self.ledger, - broadcaster, - fee_sats=self.ctv_broadcaster_fee_sats, - ) + return self._ensure_ctv_runtime().make_daemon() - @ledger_writer_operation("ctv_broadcast_state") def run_ctv_fanout_broadcaster_once( self, *, progress_callback: Callable[[], None] | None = None, ) -> CtvFanoutDaemonResult: - if self.ctv_fanout_broadcast_daemon is None: - self.ctv_fanout_broadcast_daemon = self.make_ctv_fanout_broadcast_daemon() - return self.ctv_fanout_broadcast_daemon.run_once( - limit=self.ctv_broadcaster_limit, + return self._ensure_ctv_runtime().run_once( progress_callback=progress_callback, - chunk_size=int( - getattr( - self, - "ctv_broadcaster_chunk_size", - DEFAULT_PRISM_CTV_BROADCASTER_CHUNK_SIZE, - ) - ), - tip_refresh_pending=self.tip_refresh_is_pending, chunk_callback=self.observe_ctv_fanout_broadcaster_chunk, ) def ctv_fanout_broadcaster_loop(self) -> None: + self._ensure_ctv_runtime().loop( + run_once=self.run_ctv_fanout_broadcaster_once, + progress_callback=self._record_ctv_fanout_broadcaster_progress, + observe_pass=self.observe_ctv_fanout_broadcaster_pass, + record_yield=self._record_ctv_fanout_broadcaster_yield, + ) + + def _start_hashrate_rollup_maintenance_if_enabled(self) -> None: + """Start rollup maintenance when configured on and the ledger can. + + The capability check mirrors the registration conditional: a + memory-backed ledger has no ``advance_hashrate_rollups`` and no + registered service to start, so asking the registry for the name + would fail rather than no-op. Lightweight test/embedder + coordinators built without the full constructor carry neither the + flag nor a capable ledger, so an absent flag reads as disabled. + """ + if not getattr(self, "hashrate_rollup_enabled", False): + return + if not callable( + getattr(getattr(self, "ledger", None), "advance_hashrate_rollups", None) + ): + return + self._start_background_service("hashrate-rollup-maintenance") + print( + "prism coordinator: hashrate rollup maintenance enabled " + f"interval={self.hashrate_rollup_interval_seconds:g}s " + f"batch={self.hashrate_rollup_batch_shares}", + flush=True, + ) + + def hashrate_rollup_maintenance_loop(self) -> None: + """Advance the incremental hashrate rollups until shutdown. + + Each pass folds one watermarked batch into the rollup tables and + keeps going while the ledger reports more history behind the + watermark -- which is exactly how a cold database backfills itself: + the first passes replay all existing shares through the same + statement live shares take, so there is no separate migration to + operate. A failed pass is logged and retried on the next interval; + the watermark guarantees a retry can never double-count what a + partial pass already folded in. + + Every pass runs inside the writer-operation admission, like every + other mutation of ledger-derived state: shutdown quiescence must + count an in-flight pass, and admission must refuse a new one, or a + pass slipping past the stop check could renew the just-released + lease -- the maintenance statement renews on exact identity, which + an orderly release keeps -- and mutate rollups after shutdown + reported the writer quiesced. A refused admission is the orderly + stop, not a failure. + """ while not self.stop_event.is_set(): - self._record_heartbeat("ctv_fanout_broadcaster") - started = time.monotonic() - shutdown_admission_closed = False try: - try: - result = self.run_ctv_fanout_broadcaster_once( - progress_callback=self._record_ctv_fanout_broadcaster_progress, - ) - except ShutdownInProgress: - shutdown_admission_closed = True - return - finally: - # Stamp completion before logging or entering the interval - # wait. A blocked row never reaches this finally clause, so - # the watchdog remains able to recover a wedged operation. - self._record_heartbeat("ctv_fanout_broadcaster") - if not shutdown_admission_closed: - self.observe_ctv_fanout_broadcaster_pass( - max(0.0, time.monotonic() - started) + while not self.stop_event.is_set(): + with self._writer_operation("hashrate_rollup_maintenance"): + result = self.ledger.advance_hashrate_rollups( + batch_limit=self.hashrate_rollup_batch_shares, ) - except WriterLeaseRenewalDeferred as exc: - # A preflight refusal, not a failure: the writer's own - # fenced write is holding the lease renewal, and the fence - # withheld the guarded RPC before it ran. The next pass on - # the ordinary interval retries once the write commits and - # a renewal lands. + print( + "prism coordinator: hashrate rollup advance " + f"scanned={result['scanned']} " + f"last_share_seq={result['last_share_seq']} " + f"caught_up={'true' if result['caught_up'] else 'false'}", + flush=True, + ) + if result["caught_up"]: + break + except ShutdownInProgress: + return + except Exception: print( - "prism coordinator: CTV fanout broadcaster pass " - f"withheld: {exc}", + "prism coordinator: hashrate rollup maintenance pass failed", flush=True, ) - except Exception: - print("prism coordinator: CTV fanout broadcaster pass failed", flush=True) traceback.print_exc() - else: - if result.yielded_to_tip_refresh: - self._record_ctv_fanout_broadcaster_yield() - if result.scanned_count or result.submitted_count or result.failed_count: - print( - "prism coordinator: CTV fanout broadcaster " - f"scanned={result.scanned_count} " - f"submitted={result.submitted_count} " - f"updated={result.updated_count} " - f"failed={result.failed_count}", - flush=True, - ) - if self.stop_event.wait(self.ctv_broadcaster_interval_seconds): + if self.stop_event.wait(self.hashrate_rollup_interval_seconds): break def _tip_refresh_artifacts( self, snapshot: QbitTipTemplateSnapshot, ) -> CachedTemplateArtifacts: - artifacts = snapshot.template_artifacts - if ( - artifacts is None - or artifacts.fingerprint != snapshot.template_fingerprint - or artifacts.previousblockhash != snapshot.previousblockhash - or artifacts.generation != snapshot.template_generation - or snapshot.bestblockhash != snapshot.previousblockhash - or qbit_template_fingerprint(artifacts.template) != artifacts.fingerprint - or str(artifacts.template.get("previousblockhash", "")) - != artifacts.previousblockhash - ): - raise TemplateRefreshBlocked( - "tip/template snapshot does not own matching exact artifacts" - ) - return artifacts + return self._ensure_tip_refresh_service()._tip_refresh_artifacts(snapshot) def prepare_tip_refresh_bundle( self, @@ -15011,205 +7048,16 @@ def prepare_tip_refresh_bundle( *, priority_requested_monotonic: float | None = None, ) -> CachedJobBundle: - """Build ready-pool work from immutable shared inputs only. - - Client selection belongs exclusively to fanout. In particular, a - connection disappearing before or during this build cannot affect the - signed payout bundle or its cache lifetime. - """ - self._ensure_job_cache_state() - self._ensure_tip_refresh_state() - artifacts = self._tip_refresh_artifacts(snapshot) - build_started = time.monotonic() - progress_build_token = self._progress_bundle_build_started() - priority_token, priority_requested_monotonic = ( - self._begin_job_build_priority_preparation( - priority_requested_monotonic - ) + return self._ensure_tip_refresh_service().prepare_tip_refresh_bundle( + snapshot, + priority_requested_monotonic=priority_requested_monotonic, ) - try: - max_payout_retries = max( - 0, - int( - getattr( - self, - "payout_reconcile_supersession_retries", - DEFAULT_PRISM_PAYOUT_RECONCILE_SUPERSESSION_RETRIES, - ) - ), - ) - for attempt in range(max_payout_retries + 1): - with self._job_cache_lock: - payout_generation_before_build = ( - self._payout_state_generation - ) - try: - bundle = self.shared_job_bundle( - artifacts, - mode="ready", - retry_superseded=False, - publication_critical=True, - request_source="tip_refresh", - priority_requested_monotonic=( - priority_requested_monotonic - ), - ) - break - except JobBuildSuperseded: - if getattr(self, "tip_refresh_epoch_fanout", False): - raise - with self._job_cache_lock: - payout_generation_after_build = ( - self._payout_state_generation - ) - payout_publication_blocked = ( - self._payout_state_publication_blocked - ) - with self.lock: - artifacts_buildable = self._artifacts_buildable_locked( - artifacts - ) - if ( - attempt >= max_payout_retries - or payout_publication_blocked - or not artifacts_buildable - or payout_generation_after_build - == payout_generation_before_build - ): - raise - # Coalesce a completed payout generation into this same - # tip owner. Every abandoned attempt remains fenced by its - # immutable generation; only the latest coherent retry can - # reach final publication. - continue - else: # pragma: no cover - range always runs at least once - raise TemplateRefreshBlocked( - "payout generation did not stabilize during preparation" - ) - except TemplateRefreshBlocked: - raise - except Exception as exc: - with self.lock: - self.job_build_failure_count += 1 - raise TemplateRefreshBlocked("prepared refresh bundle build failed") from exc - finally: - self._finish_job_build_priority_preparation(priority_token) - self._progress_bundle_build_finished(progress_build_token) - self._observe_tip_refresh_seconds( - "bundle_build", - time.monotonic() - build_started, - ) - # Revalidate only the snapshot-owned object. A concurrent cache fill is - # unrelated to this refresh and cannot replace its exact artifacts. - artifacts = self._tip_refresh_artifacts(snapshot) - if ( - bundle.template is not artifacts.template - or bundle.template_fingerprint != artifacts.fingerprint - or bundle.template_generation != artifacts.generation - or str(bundle.template.get("previousblockhash", "")) - != artifacts.previousblockhash - ): - raise TemplateRefreshBlocked( - "prepared refresh bundle does not match exact template artifacts" - ) - if bundle.collection_only: - raise TemplateRefreshBlocked( - "ready-pool prepared refresh unexpectedly produced a collection bundle" - ) - return bundle def prewarm_current_tip_ready_bundle(self) -> CachedJobBundle | None: - """Publish one exact current-tip ready bundle before Stratum accepts.""" - self._ensure_job_cache_state() - with self._job_cache_lock: - self.job_preparation_pending = True - try: - observation_sequence = self._reserve_tip_observation_sequence() - snapshot = self.fetch_qbit_tip_template_snapshot() - self._record_progress_tip_poll(snapshot) - try: - reconciled = self.ensure_reorg_reconciled_for_tip( - snapshot.bestblockhash - ) - except Exception as exc: - raise TemplateRefreshBlocked( - "startup reorg reconciliation failed before job preparation" - ) from exc - if not reconciled: - raise TemplateRefreshBlocked( - "startup chain view remained untrusted during job preparation" - ) - - ready = self.pool_readiness_latched() - bundle: CachedJobBundle | None = None - if ready: - progress_build_token = self._progress_bundle_build_started() - try: - bundle = self.shared_job_bundle( - self._tip_refresh_artifacts(snapshot), - None, - publication_critical=True, - request_source="tip_refresh", - ) - finally: - self._progress_bundle_build_finished(progress_build_token) - if bundle.collection_only: - raise TemplateRefreshBlocked( - "startup ready preparation produced collection work" - ) - if bundle.payout_state_generation != int( - getattr(self, "_payout_state_generation", 0) - ): - raise TemplateRefreshSuperseded( - "payout state changed during startup job preparation" - ) - - if str(self.rpc.call("getbestblockhash")) != snapshot.bestblockhash: - raise TemplateRefreshSuperseded( - "qbit tip changed during startup job preparation" - ) - if not self.observe_tip_first_seen( - snapshot.bestblockhash, - observation_sequence=observation_sequence, - publish_refresh_observation=True, - published_snapshot=snapshot, - ): - raise TemplateRefreshSuperseded( - "startup job preparation was superseded before publication" - ) - with self._job_cache_lock: - self._prepared_ready_snapshot = snapshot if bundle is not None else None - self._prepared_ready_bundle = bundle - self._record_progress_publication( - snapshot, - ( - bundle.payout_state_generation - if bundle is not None - else int(getattr(self, "_payout_state_generation", 0)) - ), - ) - self.last_successful_template_refresh_monotonic = time.monotonic() - self._record_progress_tip_poll(snapshot) - return bundle - finally: - with self._job_cache_lock: - self.job_preparation_pending = False + return self._ensure_tip_refresh_service().prewarm_current_tip_ready_bundle() def prewarm_startup_jobs(self) -> CachedJobBundle | None: - """Best-effort startup prewarm; transient blocking defers to blockpoll.""" - try: - return self.prewarm_current_tip_ready_bundle() - except TemplateRefreshBlocked as exc: - # Startup prewarming is an optimization. A transient reconciliation, - # payout-generation, or tip race must not prevent Stratum listeners - # from opening; blockpoll and the bounded initial-job queue retry it. - self._schedule_tip_refresh_retry() - print( - "prism coordinator: startup job preparation deferred " - f"reason={exc}", - flush=True, - ) - return None + return self._ensure_tip_refresh_service().prewarm_startup_jobs() def _tip_refresh_token_current_locked( self, @@ -15217,16 +7065,10 @@ def _tip_refresh_token_current_locked( bundle: CachedJobBundle, snapshot: QbitTipTemplateSnapshot, ) -> bool: - return bool( - self._tip_refresh_token_prepublication_current_locked( - token, - bundle, - snapshot, - ) - and self._tip_refresh_snapshot_current_locked( - snapshot, - token.observation_sequence, - ) + return self._ensure_tip_refresh_service()._tip_refresh_token_current_locked( + token, + bundle, + snapshot, ) def _tip_refresh_token_prepublication_current_locked( @@ -15235,60 +7077,10 @@ def _tip_refresh_token_prepublication_current_locked( bundle: CachedJobBundle, snapshot: QbitTipTemplateSnapshot, ) -> bool: - published = getattr(self, "_published_payout_state", None) - return bool( - token.snapshot is snapshot - and token.tip_hash == snapshot.bestblockhash - and token.template_fingerprint == snapshot.template_fingerprint - and token.template_generation == snapshot.template_generation - and bundle.template_fingerprint == token.template_fingerprint - and bundle.template_generation == token.template_generation - and bundle.payout_state_generation == token.payout_state_generation - and bundle.build_key is token.build_key - and token.build_key.best_tip_hash == snapshot.bestblockhash - and token.build_key.previous_block_hash == snapshot.previousblockhash - and token.build_key.template_fingerprint - == snapshot.template_fingerprint - and token.build_key.template_generation == snapshot.template_generation - and token.build_key.payout_state_generation - == token.payout_state_generation - and token.build_key.payout_append_invalidation_epoch - == int( - getattr( - self, - "_payout_ledger_append_invalidation_epoch", - 0, - ) - ) - and token.payout_state_generation - == int(getattr(self, "_payout_state_generation", 0)) - and published is not None - and published.artifact is not None - and token.build_key.payout_artifact_sha256 - == published.artifact.prior_balances_sha256 - and snapshot.template_artifacts is not None - and bundle.template is snapshot.template_artifacts.template - and not self._detected_tip_supersedes_locked( - snapshot.bestblockhash, - token.observation_sequence, - ) - and ( - not getattr(self, "tip_refresh_epoch_fanout", False) - or ( - token.epoch_sequence - == int(getattr(self, "_tip_refresh_epoch_sequence", 0)) - and token.tip_hash - == getattr(self, "_tip_refresh_epoch_tip_hash", None) - and token.payout_state_generation - == int( - getattr( - self, - "_tip_refresh_epoch_payout_generation", - 0, - ) - ) - ) - ) + return self._ensure_tip_refresh_service()._tip_refresh_token_prepublication_current_locked( + token, + bundle, + snapshot, ) def _tip_refresh_snapshot_current_locked( @@ -15296,17 +7088,9 @@ def _tip_refresh_snapshot_current_locked( snapshot: QbitTipTemplateSnapshot, observation_sequence: int, ) -> bool: - current_tip = getattr(self, "current_tip_first_seen", None) - return bool( - self.tip_template_snapshot is snapshot - and current_tip is not None - and current_tip[0] == snapshot.bestblockhash - and int(getattr(self, "current_tip_observation_sequence", 0)) - == observation_sequence - and not self._detected_tip_supersedes_locked( - snapshot.bestblockhash, - observation_sequence, - ) + return self._ensure_tip_refresh_service()._tip_refresh_snapshot_current_locked( + snapshot, + observation_sequence, ) def _validate_prepared_tip_refresh( @@ -15315,76 +7099,11 @@ def _validate_prepared_tip_refresh( snapshot: QbitTipTemplateSnapshot, observation_sequence: int, ) -> TipRefreshValidationToken: - """Validate prepared work before publishing submit authority.""" - artifacts = self._tip_refresh_artifacts(snapshot) - if ( - bundle.template is not artifacts.template - or bundle.template_fingerprint != artifacts.fingerprint - or bundle.template_generation != artifacts.generation - or bundle.build_key is None - or bundle.build_key.best_tip_hash != snapshot.bestblockhash - or bundle.build_key.previous_block_hash != snapshot.previousblockhash - or bundle.build_key.template_fingerprint != artifacts.fingerprint - or bundle.build_key.template_generation != artifacts.generation - or bundle.build_key.mode != "ready" - ): - raise TemplateRefreshBlocked( - "prepared refresh bundle changed before final validation" - ) - try: - current_tip = str(self.rpc.call("getbestblockhash")) - except Exception as exc: - self._schedule_tip_refresh_retry() - raise TemplateRefreshBlocked( - "qbit tip validation failed before prepared fanout" - ) from exc - if current_tip != snapshot.bestblockhash: - self._schedule_tip_refresh_retry() - raise TemplateRefreshSuperseded( - "qbit tip changed before prepared fanout " - f"expected={snapshot.bestblockhash} current={current_tip}" - ) - try: - chain_view_untrusted = bool( - getattr(self, "reorg_reconciler_enabled", True) - and self.qbit_chain_view_untrusted() - ) - except Exception as exc: - self._schedule_tip_refresh_retry() - raise _TipRefreshTrustBlocked( - "qbit chain trust check failed before prepared fanout" - ) from exc - if chain_view_untrusted: - self._schedule_tip_refresh_retry() - raise _TipRefreshTrustBlocked( - "qbit chain view became untrusted before prepared fanout" - ) - with self.lock: - epoch_sequence = ( - int(getattr(self, "_tip_refresh_epoch_sequence", 0)) - if getattr(self, "tip_refresh_epoch_fanout", False) - else 0 - ) - token = TipRefreshValidationToken( - tip_hash=snapshot.bestblockhash, - template_fingerprint=artifacts.fingerprint, - template_generation=artifacts.generation, - payout_state_generation=bundle.payout_state_generation, - observation_sequence=observation_sequence, - epoch_sequence=epoch_sequence, - build_key=bundle.build_key, - snapshot=snapshot, - ) - if not self._tip_refresh_token_prepublication_current_locked( - token, - bundle, - snapshot, - ): - self._schedule_tip_refresh_retry() - raise TemplateRefreshSuperseded( - "prepared refresh was superseded before tip publication" - ) - return token + return self._ensure_tip_refresh_service()._validate_prepared_tip_refresh( + bundle, + snapshot, + observation_sequence, + ) def _activate_tip_refresh( self, @@ -15393,16 +7112,12 @@ def _activate_tip_refresh( snapshot: QbitTipTemplateSnapshot, cancel_event: _FanoutCancellation, ) -> None: - with self.lock: - if not self._tip_refresh_token_current_locked(token, bundle, snapshot): - self._schedule_tip_refresh_retry() - raise TemplateRefreshSuperseded( - "prepared refresh was superseded before cancellation registration" - ) - active = self._active_tip_refresh - if active is not None: - active[1].cancel() - self._active_tip_refresh = (token, cancel_event) + return self._ensure_tip_refresh_service()._activate_tip_refresh( + token, + bundle, + snapshot, + cancel_event, + ) def _publish_prepared_tip_refresh( self, @@ -15412,109 +7127,22 @@ def _publish_prepared_tip_refresh( *, parent_hash: str | None, ) -> _FanoutCancellation: - """Atomically publish prepared work and register its cancellation token.""" - now = time.monotonic() - cancel_event = _FanoutCancellation() - # Admit a synchronization-only reader of this payout generation. Do - # not mark it delivered: the gate's priority reservation belongs to - # the first current-tip socket delivery, not this publication fence. - with self._payout_state_delivery_gate.delivery_cancelable( - lambda: False, - generation=token.payout_state_generation, - priority=True, - ) as payout_admitted: - if not payout_admitted: - self._schedule_tip_refresh_retry() - raise TemplateRefreshSuperseded( - "prepared refresh was superseded before atomic publication" - ) - # Match payout publication's cache -> coordinator lock order. A - # payout fence can close admission after this reader entered, so - # recheck its blocked marker inside the same atomic section. - with self._job_cache_lock: - with self.lock: - current_sequence = int( - getattr(self, "current_tip_observation_sequence", 0) - ) - if ( - self._payout_state_publication_blocked - or current_sequence > token.observation_sequence - or not self._tip_refresh_token_prepublication_current_locked( - token, - bundle, - snapshot, - ) - ): - self._schedule_tip_refresh_retry() - raise TemplateRefreshSuperseded( - "prepared refresh was superseded before atomic publication" - ) - - first_seen = getattr(self, "current_tip_first_seen", None) - tip_changed = ( - first_seen is not None and first_seen[0] != token.tip_hash - ) - flip_stamp = ( - now - if tip_changed - else first_seen[1] - if first_seen is not None - else None - ) - self.current_tip_first_seen = ( - token.tip_hash, - flip_stamp, - ) - self.current_tip_observation_sequence = token.observation_sequence - self.current_tip_observed_monotonic = now - if parent_hash is not None: - self.current_tip_parent = (token.tip_hash, parent_hash) - else: - # The parent lookup is best-effort cleanup metadata. A - # transient RPC failure during a same-tip republication - # must not wipe the still-valid cached parent; only a - # parent belonging to a different tip is stale here. - prior_parent = getattr(self, "current_tip_parent", None) - if prior_parent is None or prior_parent[0] != token.tip_hash: - self.current_tip_parent = None - self.tip_template_snapshot = snapshot - self._record_startup_phase_once("first_tip_template_published") - self.tip_refresh_divergence_started_monotonic = None - if tip_changed: - # Retained collection work, the retained ready bundle, - # and graveyard classification all belong to the - # previously published tip. - self._retained_collection_refresh = None - self._prepared_ready_bundle = None - self._prepared_ready_snapshot = None - self.prune_evicted_job_graveyard(now=now, force=True) - - if not self._tip_refresh_token_current_locked( - token, - bundle, - snapshot, - ): - # The payout admission and both coordinator locks - # stabilize every field used by this predicate. - raise TemplateRefreshBlocked( - "prepared refresh publication did not produce a current token" - ) - self._publish_tip_refresh_epoch_identity_locked(snapshot) - active = self._active_tip_refresh - if active is not None: - active[1].cancel() - self._active_tip_refresh = (token, cancel_event) - return cancel_event + return self._ensure_tip_refresh_service()._publish_prepared_tip_refresh( + token, + bundle, + snapshot, + parent_hash=parent_hash, + ) def _clear_active_tip_refresh( self, token: TipRefreshValidationToken, cancel_event: _FanoutCancellation, ) -> None: - with self.lock: - active = self._active_tip_refresh - if active is not None and active[0] is token and active[1] is cancel_event: - self._active_tip_refresh = None + return self._ensure_tip_refresh_service()._clear_active_tip_refresh( + token, + cancel_event, + ) def _prepared_tip_refresh_obsolete( self, @@ -15523,19 +7151,12 @@ def _prepared_tip_refresh_obsolete( snapshot: QbitTipTemplateSnapshot, cancel_event: _FanoutCancellation | None, ) -> bool: - if self.stop_event.is_set() or ( - cancel_event is not None and cancel_event.is_set() - ): - return True - with self.lock: - current = self._tip_refresh_token_current_locked( - validation_token, - bundle, - snapshot, - ) - if not current and cancel_event is not None: - cancel_event.cancel() - return not current + return self._ensure_tip_refresh_service()._prepared_tip_refresh_obsolete( + validation_token, + bundle, + snapshot, + cancel_event, + ) def send_prepared_job( self, @@ -15548,255 +7169,16 @@ def send_prepared_job( cancel_event: _FanoutCancellation | None = None, submitted_monotonic: float | None = None, ) -> RefreshResult: - worker_started = time.monotonic() - started = worker_started if submitted_monotonic is None else submitted_monotonic - phases = self._job_build_phases() - phases.clear() - - def cancelled() -> bool: - return self._prepared_tip_refresh_obsolete( - validation_token, - bundle, - snapshot, - cancel_event, - ) or getattr(client, "closing", False) - - phases["executor_queue"] = max(0.0, worker_started - started) - client_lock_started = worker_started - client_lock_acquired = False - client_lock_attempted = False - try: - while True: - with self.lock: - if ( - client not in self.clients - or client.connection_id != expected_connection_id - or getattr(client, "closing", False) - ): - return RefreshResult("disconnected") - if cancelled(): - phases["client_lock"] = max( - 0.0, - time.monotonic() - client_lock_started, - ) - self._record_tip_refresh_cancellation( - "client_lock" if client_lock_attempted else "executor_queue" - ) - return RefreshResult("skipped") - client_lock_attempted = True - client_lock_acquired = client.job_update_lock.acquire( - timeout=PRISM_TIP_REFRESH_ADMISSION_POLL_SECONDS - ) - if client_lock_acquired: - break - phases["client_lock"] = max( - 0.0, - time.monotonic() - client_lock_started, - ) - if cancelled(): - self._record_tip_refresh_cancellation("client_lock") - return RefreshResult("skipped") - with self.lock: - if ( - client not in self.clients - or client.connection_id != expected_connection_id - ): - return RefreshResult("disconnected") - active_epoch = int( - getattr( - getattr(client, "active_job", None), - "tip_refresh_epoch_sequence", - 0, - ) - ) - epoch_fanout = bool( - getattr(self, "tip_refresh_epoch_fanout", False) - ) - if ( - not self.client_can_receive_jobs(client) - or ( - epoch_fanout - and self._client_tip_refresh_epoch_blocked_locked( - client, - validation_token.epoch_sequence, - ) - ) - or ( - ( - not epoch_fanout - or active_epoch >= validation_token.epoch_sequence - ) - and self.intervening_job_supersedes_snapshot( - client.active_job, - expected_active_job, - snapshot, - ) - ) - or not self.client_needs_tip_template_refresh(client, snapshot) - ): - return RefreshResult("skipped") - self._ensure_job_cache_state() - payout_gate_started = time.monotonic() - with self._payout_state_delivery_gate.delivery_cancelable( - cancelled, - generation=bundle.payout_state_generation, - priority=True, - ) as payout_admitted: - phases["payout_gate"] = max( - 0.0, - time.monotonic() - payout_gate_started, - ) - self._observe_payout_gate_admission( - payout_admitted, - generation=bundle.payout_state_generation, - fallback_wait_seconds=phases["payout_gate"], - ) - if not payout_admitted or cancelled(): - self._record_tip_refresh_cancellation("payout_gate") - return RefreshResult("skipped") - fanout_admitted = ( - cancel_event is None or cancel_event.begin_delivery() - ) - if not fanout_admitted: - self._record_tip_refresh_cancellation("payout_gate") - return RefreshResult("skipped") - try: - # The validation token binds this immutable bundle to the - # exact observed artifact object. Fanout tasks consult only - # in-memory publication/cancellation state, never RPC or - # the mutable cache. - # Preserve one coherent difficulty snapshot through stamp - # and commit, but acquire it before global control-plane - # admission so a concurrent share cannot form the inverse - # coordinator -> Vardiff edge. - with self._client_vardiff_lock(client): - with self.lock: - token_current = self._tip_refresh_token_current_locked( - validation_token, - bundle, - snapshot, - ) - if not token_current: - if cancel_event is not None: - cancel_event.cancel() - return RefreshResult("skipped") - if ( - client not in self.clients - or client.connection_id != expected_connection_id - ): - return RefreshResult("disconnected") - active_epoch = int( - getattr( - getattr(client, "active_job", None), - "tip_refresh_epoch_sequence", - 0, - ) - ) - epoch_fanout = bool( - getattr(self, "tip_refresh_epoch_fanout", False) - ) - if ( - not self.client_can_receive_jobs(client) - or ( - epoch_fanout - and self._client_tip_refresh_epoch_blocked_locked( - client, - validation_token.epoch_sequence, - ) - ) - or ( - ( - not epoch_fanout - or active_epoch - >= validation_token.epoch_sequence - ) - and self.intervening_job_supersedes_snapshot( - client.active_job, - expected_active_job, - snapshot, - ) - ) - or not self.client_needs_tip_template_refresh( - client, - snapshot, - ) - ): - return RefreshResult("skipped") - if not self._admit_client_tip_refresh_epoch_locked( - client, - validation_token.epoch_sequence, - ): - return RefreshResult("skipped") - clean_jobs = self.client_tip_changed_for_snapshot( - client, - snapshot, - ) - stamp_started = time.monotonic() - context = self.stamp_job_for_client( - client, - bundle, - clean_jobs=clean_jobs, - tip_refresh_epoch_sequence=( - validation_token.epoch_sequence - ), - ) - phases["stamp"] = time.monotonic() - stamp_started - client.active_job = context - if clean_jobs: - for job_id in tuple(client.active_job_ids): - self.bury_evicted_job(client, job_id, prune=False) - self.jobs.pop(job_id, None) - client.active_job_ids.clear() - self.prune_evicted_job_graveyard(force=False) - self.jobs[context.job.job_id] = context - client.active_job_ids.add(context.job.job_id) - self.prune_client_active_jobs(client) - - socket_send_started = time.monotonic() - try: - self.send_job_update(client, context.job) - payout_admitted.mark_delivered() - finally: - socket_send_finished = time.monotonic() - phases["socket_send"] = max( - 0.0, - socket_send_finished - socket_send_started, - ) - self.apply_job_difficulty(client, context.job) - self.note_tip_work_delivered( - client, - str(context.template["previousblockhash"]), - ) - self.note_initial_job_delivered(client, validated_current=True) - delivered_monotonic = time.monotonic() - self._record_first_payout_delivery( - context.payout_state_generation, - delivered_monotonic, - ) - self._record_progress_delivery( - client, - context, - delivered_monotonic, - ) - if getattr(self, "hot_path_log_enabled", False): - print( - "prism coordinator: sent prepared job " - f"connection={client.connection_id} username={client.username} " - f"job={context.job.job_id} " - f"elapsed={delivered_monotonic - started:.3f}s", - flush=True, - ) - return RefreshResult("sent", delivered_monotonic) - finally: - if cancel_event is not None: - cancel_event.end_delivery() - finally: - if client_lock_acquired: - client.job_update_lock.release() - self.observe_job_build_elapsed( - max(0.0, time.monotonic() - started), - phases, - ) + return self._ensure_job_delivery_service().send_prepared_job( + client, + bundle, + snapshot, + validation_token, + expected_connection_id, + expected_active_job, + cancel_event, + submitted_monotonic, + ) def _fanout_prepared_tip_refresh( self, @@ -15810,356 +7192,27 @@ def _fanout_prepared_tip_refresh( executor: ThreadPoolExecutor | None = None, expected_active_jobs: dict[ClientState, PrismJobContext | None] | None = None, heartbeat_name: str, - ) -> tuple[int, float | None, float | None, int]: - executor = executor or self.tip_refresh_executor() - cancel_event = preactivated_cancel_event or _FanoutCancellation() - if observation_sequence is None: - with self.lock: - observation_sequence = int( - getattr(self, "current_tip_observation_sequence", 0) - ) - if validation_token is None: - validation_token = self._validate_prepared_tip_refresh( - bundle, - snapshot, - observation_sequence, - ) - if preactivated_cancel_event is None: - self._activate_tip_refresh( - validation_token, - bundle, - snapshot, - cancel_event, - ) - futures: dict[Future[RefreshResult], ClientState] = {} - submitted_at: dict[Future[RefreshResult], float] = {} - queued_cancellations: set[Future[RefreshResult]] = set() - with self.lock: - if expected_active_jobs is None: - expected_active_jobs = { - client: client.active_job - for client in clients - } - # Vardiff drives every client toward the same share interval, so - # the difficulty a client sustains is proportional to its hashrate. - # Snapshot the proxies in the same locked pass as the job snapshot. - hashrate_proxies = { - client: ( - client.vardiff_difficulty_estimate or client.share_difficulty - ) - for client in clients - } - # Bounded submission admits at most max_inflight tasks at a time, so - # queue priority alone cannot lift one client over a fleet wave that - # has not been submitted yet. Order admission itself: stale-job - # clients by descending hashrate, job-less clients last. A stale fast - # client burns its full rate on old-tip work every second of refresh - # lag, while a job-less client burns nothing while it waits regardless - # of its configured difficulty (it also keeps its initial-delivery - # queue priority once admitted). - clients = sorted( + ) -> tuple[int, float | None, float | None, int]: + return self._ensure_tip_refresh_service()._fanout_prepared_tip_refresh( clients, - key=lambda ordered: ( - expected_active_jobs.get(ordered) is not None, - hashrate_proxies.get(ordered, Decimal(0)), - ), - reverse=True, + bundle, + snapshot, + observation_sequence=observation_sequence, + validation_token=validation_token, + preactivated_cancel_event=preactivated_cancel_event, + executor=executor, + expected_active_jobs=expected_active_jobs, + heartbeat_name=heartbeat_name, ) - clients_iter = iter(clients) - max_inflight = max(1, int(self.tip_refresh_max_workers)) - - def record_queued_cancellation(future: Future[RefreshResult]) -> None: - if future in queued_cancellations: - return - queued_cancellations.add(future) - elapsed = max(0.0, time.monotonic() - submitted_at[future]) - self.observe_job_build_elapsed(elapsed, {"executor_queue": elapsed}) - self._record_tip_refresh_cancellation("executor_queue") - - def cancel_pending_futures(pending: set[Future[RefreshResult]]) -> None: - cancel_event.cancel() - for future in tuple(pending): - if future.cancel(): - record_queued_cancellation(future) - self._record_tip_refresh_client_result("skipped") - pending.discard(future) - - def submit_available(pending: set[Future[RefreshResult]]) -> None: - while ( - len(pending) < max_inflight - and not self.stop_event.is_set() - and not cancel_event.is_set() - ): - with self.lock: - token_current = self._tip_refresh_token_current_locked( - validation_token, - bundle, - snapshot, - ) - if not token_current: - cancel_event.cancel() - return - try: - client = next(clients_iter) - except StopIteration: - return - submitted = time.monotonic() - active_job = expected_active_jobs.get(client) - if active_job is None: - priority = PRISM_DELIVERY_PRIORITY_INITIAL - elif self.client_tip_changed_for_snapshot(client, snapshot): - priority = PRISM_DELIVERY_PRIORITY_NEW_TIP - else: - priority = PRISM_DELIVERY_PRIORITY_SAME_TIP - future = self._submit_delivery_task( - executor, - self.send_prepared_job, - client, - bundle, - snapshot, - validation_token, - client.connection_id, - expected_active_jobs.get(client), - cancel_event, - submitted, - priority=priority, - ) - self._tip_refresh_future_started() - future.add_done_callback(self._tip_refresh_future_finished) - futures[future] = client - submitted_at[future] = submitted - pending.add(future) - - pending: set[Future[RefreshResult]] = set() - try: - sent = 0 - failed = 0 - first_delivery: float | None = None - last_delivery: float | None = None - invalidation: TemplateRefreshBlocked | None = None - last_live_trust_check = time.monotonic() - try: - submit_available(pending) - except RuntimeError: - cancel_pending_futures(pending) - cancel_event.set() - if pending: - wait(pending) - if not self.stop_event.is_set(): - self._schedule_tip_refresh_retry() - raise - while pending: - self._record_heartbeat(heartbeat_name) - if self.stop_event.is_set() or cancel_event.is_set(): - cancel_pending_futures(pending) - done, pending = wait( - pending, - timeout=PRISM_TIP_REFRESH_ADMISSION_POLL_SECONDS, - return_when=FIRST_COMPLETED, - ) - for future in done: - client = futures[future] - if future.cancelled(): - if future not in queued_cancellations: - record_queued_cancellation(future) - self._record_tip_refresh_client_result("skipped") - continue - try: - result = future.result() - except OSError: - self._record_tip_refresh_client_result("disconnected") - self.disconnect_client(client) - continue - except TemplateRefreshBlocked as exc: - self._record_tip_refresh_client_result("skipped") - invalidation = exc - cancel_pending_futures(pending) - continue - except Exception: - failed += 1 - self._record_tip_refresh_client_result("failed") - with self.lock: - self.job_build_failure_count += 1 - print( - "prism coordinator: prepared job fanout failed " - f"connection={client.connection_id} username={client.username}", - flush=True, - ) - traceback.print_exc() - continue - self._record_tip_refresh_client_result(result.result) - if result.result == "sent": - sent += 1 - delivered = result.delivered_monotonic - if delivered is not None: - first_delivery = ( - delivered - if first_delivery is None - else min(first_delivery, delivered) - ) - last_delivery = ( - delivered - if last_delivery is None - else max(last_delivery, delivered) - ) - if ( - pending - and invalidation is None - and not self.stop_event.is_set() - and time.monotonic() - last_live_trust_check >= 1.0 - ): - # Validation tokens keep queued per-client deliveries - # RPC-free, but they cannot observe headers advancing - # ahead of blocks while the best-block hash stays fixed. - # Recheck the live chain view from the fanout driver about - # once per second and cancel every delivery still queued - # if the view becomes untrusted. - try: - trusted = self.ensure_reorg_reconciled_for_current_tip( - expected_tip_hash=snapshot.bestblockhash, - ) - if not trusted: - raise _TipRefreshTrustBlocked( - "qbit chain view became untrusted during prepared fanout" - ) - last_live_trust_check = time.monotonic() - except ShutdownInProgress: - # Admission can close after the stop-event check above - # but before reconciliation enters its writer scope. - # Preserve the intentional shutdown signal so the - # poller cannot consume its template-failure budget. - cancel_pending_futures(pending) - raise - except TemplateRefreshBlocked as exc: - invalidation = exc - except Exception as exc: - invalidation = _TipRefreshTrustBlocked( - "qbit chain trust check failed during prepared fanout" - ) - invalidation.__cause__ = exc - if invalidation is not None: - cancel_pending_futures(pending) - if invalidation is None: - try: - submit_available(pending) - except RuntimeError: - cancel_pending_futures(pending) - cancel_event.set() - if pending: - wait(pending) - if not self.stop_event.is_set(): - self._schedule_tip_refresh_retry() - raise - if invalidation is not None: - cancel_event.set() - self._schedule_tip_refresh_retry() - raise invalidation - with self.lock: - token_current = self._tip_refresh_token_current_locked( - validation_token, - bundle, - snapshot, - ) - if not token_current: - self._schedule_tip_refresh_retry() - raise _TipRefreshFanoutSuperseded( - "prepared refresh was superseded during fanout; immediate retry scheduled" - ) - try: - post_fanout_tip = str(self.rpc.call("getbestblockhash")) - except Exception as exc: - self._schedule_tip_refresh_retry() - raise TemplateRefreshBlocked( - "qbit tip validation failed after prepared fanout; " - "immediate retry scheduled" - ) from exc - if post_fanout_tip != snapshot.bestblockhash: - cancel_event.set() - self._schedule_tip_refresh_retry() - raise _TipRefreshFanoutSuperseded( - "qbit tip changed during prepared fanout; immediate retry scheduled " - f"expected={snapshot.bestblockhash} current={post_fanout_tip}" - ) - try: - post_fanout_untrusted = bool( - getattr(self, "reorg_reconciler_enabled", True) - and self.qbit_chain_view_untrusted() - ) - except Exception as exc: - cancel_event.set() - self._schedule_tip_refresh_retry() - raise _TipRefreshTrustBlocked( - "qbit chain trust check failed after prepared fanout; " - "immediate retry scheduled" - ) from exc - if post_fanout_untrusted: - cancel_event.set() - self._schedule_tip_refresh_retry() - raise _TipRefreshTrustBlocked( - "qbit chain view became untrusted during prepared fanout; " - "immediate retry scheduled" - ) - with self.lock: - token_current = self._tip_refresh_token_current_locked( - validation_token, - bundle, - snapshot, - ) - if not token_current: - cancel_event.set() - self._schedule_tip_refresh_retry() - raise _TipRefreshFanoutSuperseded( - "prepared refresh payout state changed during post-fanout " - "validation; immediate retry scheduled" - ) - if last_delivery is not None and submitted_at: - # Wall-clock span of the wave itself (first task submission - # to last successful delivery), independent of the reconcile - # and bundle-build stages the first/last_delivery histograms - # include. At fleet scale this is the dominant staleness term - # bounded by the delivery worker pool. - self._observe_tip_refresh_seconds( - "fanout_wave", - max(0.0, last_delivery - min(submitted_at.values())), - ) - return sent, first_delivery, last_delivery, failed - finally: - self._clear_active_tip_refresh(validation_token, cancel_event) def _interrupted_wave_outcome(self, superseded_outcome: str) -> str: - """Classify a wave that declined re-entry. - - With epoch fanout active, a wave that would have re-entered but for - shutdown was terminated by shutdown, not by the supersession it - recovered; reporting it superseded would pollute the terminal - supersession counts that rollout evaluation watches. Legacy waves - never re-enter, so their supersession outcome stands. - """ - if ( - getattr(self, "tip_refresh_epoch_fanout", False) - and self.stop_event.is_set() - ): - return "shutdown" - return superseded_outcome + return self._ensure_tip_refresh_service()._interrupted_wave_outcome( + superseded_outcome, + ) def _tip_refresh_wave_reenters(self, completed_passes: int) -> bool: - """Gate owner-wave re-entry so tip churn cannot spin at RPC speed. - - Every re-entry pass costs template and reconcile RPC work against - qbitd exactly when block cadence is fastest. The inter-pass backoff - bounds that poll rate, and a wave that cannot converge within its - pass budget hands the remainder to the scheduled retry, whose fresh - wave restarts from the newest observed state. - """ - if not getattr(self, "tip_refresh_epoch_fanout", False): - return False - if self.stop_event.is_set(): - return False - if completed_passes >= PRISM_TIP_REFRESH_WAVE_PASS_BUDGET: - self._schedule_tip_refresh_retry() - return False - return not self.stop_event.wait( - PRISM_TIP_REFRESH_REENTRY_BACKOFF_SECONDS + return self._ensure_tip_refresh_service()._tip_refresh_wave_reenters( + completed_passes, ) def poll_qbit_tip_template_once( @@ -16167,137 +7220,9 @@ def poll_qbit_tip_template_once( *, heartbeat_name: str = "qbit_blockpoll", ) -> int: - """Drive one latest-wins refresh wave as its singleflight owner.""" - self._ensure_tip_refresh_state() - self._ensure_job_cache_state() - refresh_started = time.monotonic() - observed_best_tip: str | None = None - try: - observation_sequence = self._reserve_tip_observation_sequence() - observed_best_tip = str(self.rpc.call("getbestblockhash")) - if not self.observe_tip_for_refresh( - observed_best_tip, - observation_sequence=observation_sequence, - mark_pending=False, - ): - self._schedule_tip_refresh_retry() - raise TemplateRefreshSuperseded( - "tip/template poll was superseded before template fetch" - ) - except TemplateRefreshSuperseded: - if not getattr(self, "tip_refresh_epoch_fanout", False): - self._record_coordination_blocked_refresh(time.monotonic()) - self._note_tip_refresh_attempt_failed(observed_best_tip) - raise - except Exception: - self._clear_coordination_blocked_streak() - self._record_template_refresh_failure(time.monotonic()) - self._note_tip_refresh_attempt_failed(observed_best_tip) - raise - assert observed_best_tip is not None - if not self._tip_refresh_singleflight_lock.acquire(blocking=False): - # A losing observer still publishes newer detection to cancellation - # state, but it does not own (or count) a refresh wave. - self.observe_tip_for_refresh( - observed_best_tip, - observation_sequence=observation_sequence, - mark_pending=True, - ) - self._schedule_tip_refresh_retry() - return 0 - - total_refreshed = 0 - saw_build_supersession = False - saw_fanout_supersession = False - outcome = "error" - first_pass = True - completed_passes = 0 - try: - while True: - try: - total_refreshed += self._poll_qbit_tip_template_pass_once( - heartbeat_name=heartbeat_name, - refresh_started=refresh_started, - observation_sequence=( - observation_sequence if first_pass else None - ), - observed_best_tip=( - observed_best_tip if first_pass else None - ), - ) - first_pass = False - completed_passes += 1 - except _PayoutStatePublicationBlocked: - outcome = "payout_blocked" - raise - except _TipRefreshTrustBlocked: - outcome = "trust_blocked" - raise - except JobBuildSuperseded: - saw_build_supersession = True - first_pass = False - completed_passes += 1 - if self._tip_refresh_wave_reenters(completed_passes): - continue - outcome = self._interrupted_wave_outcome( - "build_superseded" - ) - raise - except TemplateRefreshSuperseded: - saw_fanout_supersession = True - first_pass = False - completed_passes += 1 - if self._tip_refresh_wave_reenters(completed_passes): - continue - outcome = self._interrupted_wave_outcome( - "fanout_superseded" - ) - raise - except Exception: - outcome = "error" - raise - - if ( - getattr(self, "tip_refresh_epoch_fanout", False) - and not self._tip_refresh_epoch_fixpoint_reached() - ): - saw_fanout_supersession = True - if self._tip_refresh_wave_reenters(completed_passes): - continue - outcome = self._interrupted_wave_outcome( - "fanout_superseded" - ) - return total_refreshed - outcome = ( - "build_superseded" - if saw_build_supersession - else "fanout_superseded" - if saw_fanout_supersession - else "completed" - ) - return total_refreshed - finally: - try: - self._record_tip_refresh_wave_outcome(outcome) - wave_elapsed = time.monotonic() - refresh_started - self._observe_tip_refresh_seconds( - "refresh", - wave_elapsed, - ) - if wave_elapsed > 1.0: - # One line per slow wave (~1/min at current cadence). - # Reconstructing wave latency by pairing blockwait lines - # with "refreshed N client job(s)" lines cannot - # attribute time or see superseded passes; this can. - print( - "prism coordinator: tip refresh wave " - f"outcome={outcome} refreshed={total_refreshed} " - f"passes={completed_passes} " - f"elapsed={wave_elapsed:.3f}s", - flush=True, - ) - finally: - self._tip_refresh_singleflight_lock.release() + return self._ensure_tip_refresh_service().poll_qbit_tip_template_once( + heartbeat_name=heartbeat_name, + ) def _poll_qbit_tip_template_pass_once( self, @@ -16307,789 +7232,24 @@ def _poll_qbit_tip_template_pass_once( observation_sequence: int | None = None, observed_best_tip: str | None = None, ) -> int: - self._ensure_tip_refresh_state() - self._ensure_job_cache_state() - publication_lock_acquired = False - progress_refresh_active = False - pending_signal_token: int | None = None - try: - observation_prevalidated = ( - observation_sequence is not None - and observed_best_tip is not None - ) - if observation_sequence is None: - observation_sequence = self._reserve_tip_observation_sequence() - # The interval poller has no push notification to mark priority for - # it. Probe the cheap best-tip RPC before fetching and deriving the - # template so CTV maintenance can yield as soon as a changed tip is - # observed, rather than after reconciliation or bundle preparation. - if observed_best_tip is None: - observed_best_tip = str(self.rpc.call("getbestblockhash")) - if ( - not observation_prevalidated - and not self.observe_tip_for_refresh( - observed_best_tip, - observation_sequence=observation_sequence, - mark_pending=False, - ) - ): - self._schedule_tip_refresh_retry() - raise TemplateRefreshSuperseded( - "tip/template poll was superseded before template fetch" - ) - pending_signal_token = self._claim_tip_refresh_pending() - with self.lock: - poll_start_clients = tuple( - client - for client in self.clients - if self.client_can_receive_jobs(client) - ) - with self.lock: - current_tip = getattr(self, "current_tip_first_seen", None) - if current_tip is not None and current_tip[0] != observed_best_tip: - pending_signal_token = self._mark_tip_refresh_pending_for_poll( - pending_signal_token, - observation_sequence, - ) - # Captured before reconciliation can begin: the pass may run on - # the prefetch worker while this thread fetches the template, so - # the mutation bracket must open before either starts. - payout_generation_before_reconciliation = int( - getattr(self, "_payout_state_generation", 0) - ) - # Reconciliation is keyed by tip hash and the probe above already - # observed the best hash, so a memo miss can run its full pass on - # the prefetch worker while this thread fetches and derives the - # template. Whether the pass can be skipped is decided at the - # join below against the memo's state AT THAT TIME -- a tip that - # flips away and back during the fetch evicts the entry so - # post-flip pool-block state is re-proven -- and publication - # still re-proves chain trust in _validate_prepared_tip_refresh - # before any fanout. With the memo disabled (TTL 0) there is no - # way to validate an overlapped pass across the fetch window, so - # the poll keeps its historical serial pass. - reconciler_enabled = bool( - getattr(self, "reorg_reconciler_enabled", True) - ) - reconcile_memo_enabled = reconciler_enabled and ( - float( - getattr( - self, - "reorg_reconcile_cache_seconds", - DEFAULT_PRISM_REORG_RECONCILE_CACHE_SECONDS, - ) - ) - > 0 - ) - reconcile_prefetch: Future[bool] | None = None - reconcile_probe_started = time.monotonic() - reconcile_detection_epoch = int( - getattr(self, "tip_detection_epoch", 0) - ) - if reconcile_memo_enabled and not self._reorg_reconcile_memo_fresh( - observed_best_tip - ): - reconcile_prefetch = self._submit_reconcile_prefetch( - observed_best_tip - ) - reconcile_probe_seconds = time.monotonic() - reconcile_probe_started - snapshot = self._reuse_current_tip_template_snapshot(observed_best_tip) - if snapshot is None: - self._record_job_cache_event("template", hit=False) - snapshot = self.fetch_qbit_tip_template_snapshot() - if not self.observe_tip_for_refresh( - snapshot.bestblockhash, - observation_sequence=observation_sequence, - mark_pending=False, - ): - self._schedule_tip_refresh_retry() - raise TemplateRefreshSuperseded( - "tip/template poll was superseded during template fetch" - ) - with self.lock: - published_after_fetch = getattr( - self, - "current_tip_first_seen", - None, - ) - if ( - published_after_fetch is not None - and published_after_fetch[0] != snapshot.bestblockhash - ): - pending_signal_token = self._mark_tip_refresh_pending_for_poll( - pending_signal_token, - observation_sequence, - ) - self._record_progress_tip_poll(snapshot) - self._progress_refresh_started() - progress_refresh_active = True - self.pool_readiness_latched() - with self.lock: - previous_snapshot = self.tip_template_snapshot - # Generation orders concurrent observations but is not itself - # a template change. Repeated observations of identical work - # must not trigger a clean fanout on every poll. - snapshot_changed = previous_snapshot is not None and ( - previous_snapshot.bestblockhash != snapshot.bestblockhash - or previous_snapshot.previousblockhash != snapshot.previousblockhash - or previous_snapshot.template_fingerprint - != snapshot.template_fingerprint - ) - if snapshot_changed: - clients = [ - client - for client in self.clients - if self.client_can_receive_jobs(client) - ] - else: - clients = [ - client - for client in self.clients - if self.client_can_receive_jobs(client) - and self.client_needs_tip_template_refresh(client, snapshot) - ] - # Capture the exact job each client had when this refresh pass - # selected it. A Vardiff/authorize path may install intervening - # work while the shared bundle is prepared or while its task - # waits in the executor queue. Artifact generations let the - # task replace stale intervening work while preserving work - # produced from a template stored after this snapshot. - expected_active_jobs = { - client: client.active_job - for client in clients - } - - if clients and snapshot_changed: - pending_signal_token = self._mark_tip_refresh_pending_for_poll( - pending_signal_token, - observation_sequence, - ) - - refreshed = 0 - build_failures = 0 - first_delivery: float | None = None - last_delivery: float | None = None - self._raise_if_tip_refresh_superseded( - snapshot, - observation_sequence, - ) - reconcile_source = "serial" - reconcile_join_started = time.monotonic() - try: - if ( - reconcile_memo_enabled - and int(getattr(self, "tip_detection_epoch", 0)) - == reconcile_detection_epoch - and self._reorg_reconcile_memo_fresh( - snapshot.bestblockhash - ) - ): - # Fresh at join time AND no detection interleaved the - # fetch. Both are required: a flip away and back evicts - # the entry, but a pass whose execution straddled the - # flip can re-arm it afterwards (the latest detected - # hash matches again), and its proof belongs to the - # closed epoch. Any epoch movement lands in the serial - # re-prove branches below. - reconcile_source = ( - "overlap" if reconcile_prefetch is not None else "memo_hit" - ) - if reconcile_prefetch is not None: - self._discard_stale_reconcile_prefetch( - reconcile_prefetch - ) - reconcile_prefetch = None - reorg_reconciled = True - elif ( - reconcile_prefetch is not None - and snapshot.bestblockhash == observed_best_tip - ): - reconcile_source = "overlap" - reorg_reconciled = self._join_reconcile_prefetch_bounded( - reconcile_prefetch - ) - if reorg_reconciled and ( - int(getattr(self, "tip_detection_epoch", 0)) - != reconcile_detection_epoch - ): - # A detection interleaved the fetch (a flip away, or - # away and back to this same hash): cached proofs - # were evicted and the overlapped pass may have run - # in the closed epoch. Re-prove off-thread with the - # same bounded join -- the crawl that slows the - # overlap join slows this re-prove identically. - reconcile_source = "serial" - reorg_reconciled = self._reconcile_snapshot_tip_bounded( - snapshot.bestblockhash - ) - # A trust flip after the pass completed (headers running - # ahead with no detection) is deliberately NOT re-checked - # here: prepared fanout re-proves the live chain view in - # _validate_prepared_tip_refresh, and sequential issuance - # re-proves it per client in - # ensure_reorg_reconciled_for_current_tip, whose trust - # check is never cached. A second live check here would - # break the one-trust-validation-per-refresh economy. - else: - if reconcile_prefetch is not None: - # The tip moved between the probe and the template - # fetch; the prefetched pass proved a superseded - # hash. Reconcile the snapshot tip off-thread with - # the same bounded join: the crawl that slows the - # overlap join reaches this branch through exactly - # the churn that moves tips mid-fetch, and the poll - # loop must never park on it here either. - self._discard_stale_reconcile_prefetch( - reconcile_prefetch - ) - reconcile_prefetch = None - reorg_reconciled = self._reconcile_snapshot_tip_bounded( - snapshot.bestblockhash - ) - except (ShutdownInProgress, FuturesCancelledError): - # Shutdown may close writer admission (or cancel the queued - # prefetch) after this refresh has fetched a snapshot. Leave - # the refresh incomplete and let the controlled shutdown - # proceed without consuming the template failure budget or - # taking the hard-exit path. - return 0 - except (TemplateRefreshSuperseded, _PayoutStatePublicationBlocked): - raise - except Exception as exc: - raise _TipRefreshTrustBlocked( - "qbit reorg reconciliation failed before refresh preparation" - ) from exc - finally: - # The effective serial reconcile cost of this refresh: the - # memo probe plus whatever wait the template fetch did not - # absorb. Observe failures too so a slow reconcile that - # blocks the refresh still shows up in the histogram. - self._observe_tip_refresh_build_phase( - "reorg_reconcile", - reconcile_probe_seconds - + (time.monotonic() - reconcile_join_started), - ) - if reconciler_enabled: - self._record_reorg_reconcile_lookup( - "tip_refresh", - reconcile_source, - ) - if not reorg_reconciled: - raise _TipRefreshTrustBlocked( - "qbit chain view remained untrusted after reorg reconciliation" - ) - payout_generation_after_reconciliation = int( - getattr(self, "_payout_state_generation", 0) - ) - if ( - payout_generation_after_reconciliation - != payout_generation_before_reconciliation - ): - # A same-tip reconciliation can invalidate signed payout state - # even when no client needed template work at initial - # selection. Reselect after the ledger mutation so every old- - # generation job is replaced from the post-reorg snapshot. - with self.lock: - clients = [ - client - for client in self.clients - if self.client_can_receive_jobs(client) - and self.client_needs_tip_template_refresh(client, snapshot) - ] - expected_active_jobs = { - client: client.active_job - for client in clients - } - # Reconciliation itself minted the payout-pending token and - # this pass has deliberately reselected clients from that new - # generation. Adopt exactly that token; completion still checks - # both token ownership and payout/detected-tip currentness, so - # a later producer cannot be cleared accidentally. - pending_signal_token = self._claim_tip_refresh_pending() - with self.lock: - selected_clients_list = list(clients) - selected_client_set = set(clients) - for client in poll_start_clients: - if client in selected_client_set: - continue - if snapshot_changed or self.client_needs_tip_template_refresh( - client, - snapshot, - ): - selected_clients_list.append(client) - selected_client_set.add(client) - expected_active_jobs[client] = client.active_job - selected_clients = tuple(selected_clients_list) - # Prepared-mode selection must cover every client this pass can - # end up serving. Revalidation only filters selected_clients, so - # deriving from the narrower initial list could publish authority - # through the sequential path with no shared bundle while - # poll-start targets still need prepared ready work. - use_prepared_fanout = bool( - selected_clients - and getattr(self, "_pool_ready_latched", False) - ) - ready_mode = bool(getattr(self, "_pool_ready_latched", False)) - bundle: CachedJobBundle | None = None - validation_token: TipRefreshValidationToken | None = None - preactivated_cancel_event: _FanoutCancellation | None = None - prepared_executor: ThreadPoolExecutor | None = None - if use_prepared_fanout: - self._raise_if_tip_refresh_superseded( - snapshot, - observation_sequence, - ) - try: - bundle = self.prepare_tip_refresh_bundle( - snapshot, - priority_requested_monotonic=refresh_started, - ) - except _PayoutStatePublicationBlocked: - for _client in selected_clients: - self._record_tip_refresh_client_result("skipped") - self._schedule_tip_refresh_retry() - raise - except TemplateRefreshBlocked: - for _client in selected_clients: - self._record_tip_refresh_client_result("failed") - raise - if ( - bundle.payout_state_generation - != payout_generation_after_reconciliation - ): - # Request construction can observe a payout generation - # published after reconciliation and still return a fully - # coherent newest-generation bundle. Adopt it only while - # it remains the current immutable payout pointer, then - # reselect the complete fleet so no old-generation client - # is omitted merely because selection preceded the build. - with self._job_cache_lock: - current_payout_generation = ( - self._payout_state_generation - ) - current_payout_artifact = ( - self._published_payout_state.artifact - ) - if ( - bundle.payout_state_generation - != current_payout_generation - or bundle.build_key is None - or current_payout_artifact is None - or bundle.build_key.payout_artifact_sha256 - != current_payout_artifact.prior_balances_sha256 - ): - self._schedule_tip_refresh_retry() - raise TemplateRefreshSuperseded( - "payout state changed after refresh preparation; " - "immediate retry scheduled" - ) - payout_generation_after_reconciliation = ( - current_payout_generation - ) - with self.lock: - selected_clients = tuple( - client - for client in self.clients - if self.client_can_receive_jobs(client) - and self.client_needs_tip_template_refresh( - client, - snapshot, - ) - ) - expected_active_jobs = { - client: client.active_job - for client in selected_clients - } - pending_signal_token = self._claim_tip_refresh_pending() - - self._raise_if_tip_refresh_superseded( - snapshot, - observation_sequence, - ) - # Construction is complete. Acquire the short publication lane - # only for final validation, snapshot publication, and current - # client selection. Other observations and replacement builds can - # progress while an obsolete builder is still unwinding. - while not self._tip_refresh_lock.acquire( - timeout=PRISM_TIP_REFRESH_ADMISSION_POLL_SECONDS - ): - self._record_heartbeat(heartbeat_name) - if self.stop_event.is_set(): - return 0 - self._probe_tip_while_refresh_waiting() - self._raise_if_tip_refresh_superseded( - snapshot, - observation_sequence, - ) - publication_lock_acquired = True - with self._job_cache_lock: - current_payout_generation = self._payout_state_generation - current_payout_artifact = self._published_payout_state.artifact - if ( - current_payout_generation - != payout_generation_after_reconciliation - ): - self._schedule_tip_refresh_retry() - raise TemplateRefreshSuperseded( - "payout state changed before refresh publication; " - "immediate retry scheduled" - ) - if ( - bundle is not None - and ( - current_payout_artifact is None - or bundle.build_key is None - or bundle.build_key.payout_artifact_sha256 - != current_payout_artifact.prior_balances_sha256 - ) - ): - self._schedule_tip_refresh_retry() - raise TemplateRefreshBlocked( - "complete build key changed before refresh publication" - ) - - # A ready-pool pass must build and validate its immutable shared - # bundle before publishing submit authority. Otherwise a cache, - # derivation, or final chain-validation failure can invalidate - # retained work without a deliverable replacement. Sequential / - # collection work has no shared preparation stage, so it commits - # here immediately before its worker-specific builds. - if use_prepared_fanout: - assert bundle is not None - # Acquire infrastructure before publishing authority. Once the - # active token is installed, every subsequent fanout exit is - # covered by its cancellation/cleanup finally block. - prepared_executor = self.tip_refresh_executor() - # Parent metadata is cleanup-only, but fetch it before the - # final live/trust guard so validation remains the last RPC - # boundary before atomic publication. - try: - prepared_parent_hash = self._fetch_tip_parent_hash( - snapshot.bestblockhash - ) - except Exception: - prepared_parent_hash = None - validation_token = self._validate_prepared_tip_refresh( - bundle, - snapshot, - observation_sequence, - ) - preactivated_cancel_event = self._publish_prepared_tip_refresh( - validation_token, - bundle, - snapshot, - parent_hash=prepared_parent_hash, - ) - else: - if not self.observe_tip_first_seen( - snapshot.bestblockhash, - observation_sequence=observation_sequence, - publish_refresh_observation=True, - published_snapshot=snapshot, - ): - raise TemplateRefreshSuperseded( - "tip/template poll was superseded by a newer tip observation" - ) - self.prune_evicted_job_graveyard(force=False) - with self.lock: - current_tip = getattr(self, "current_tip_first_seen", None) - if ( - current_tip is None - or current_tip[0] != snapshot.bestblockhash - or int(getattr(self, "current_tip_observation_sequence", 0)) - != observation_sequence - ): - raise TemplateRefreshSuperseded( - "tip/template poll was superseded before snapshot publication" - ) - if self.tip_template_snapshot is not snapshot: - raise TemplateRefreshBlocked( - "tip/template snapshot was not atomically published" - ) - - dropped_client_results: list[str] = [] - with self.lock: - # Revalidate the originally selected targets at publication. - # New connections keep their own pending/retained wake; this - # pass must not consume work that appeared after construction. - # Preserve the pre-build expected job pointer so an intervening - # authorize/Vardiff delivery is never overwritten. - current_clients: list[ClientState] = [] - for client in selected_clients: - if client not in self.clients: - dropped_client_results.append("disconnected") - elif not self.client_can_receive_jobs(client): - dropped_client_results.append("skipped") - elif not self.client_needs_tip_template_refresh(client, snapshot): - dropped_client_results.append("skipped") - else: - current_clients.append(client) - clients = current_clients - - for result in dropped_client_results: - self._record_tip_refresh_client_result(result) - - if bundle is not None and not bundle.collection_only: - with self._job_cache_lock: - if ( - bundle.payout_state_generation - == self._payout_state_generation - ): - self._prepared_ready_snapshot = snapshot - self._prepared_ready_bundle = bundle - - if ( - not use_prepared_fanout - and clients - and getattr(self, "_pool_ready_latched", False) - ): - # Publication already committed through the sequential path, - # so a ready-pool target that appeared after build selection - # has no shared bundle to receive. Leave the pending marker - # armed and let the immediate retry build for it. - self._mark_tip_refresh_pending(observation_sequence) - self._schedule_tip_refresh_retry() - raise TemplateRefreshBlocked( - "current clients appeared after build selection; retry scheduled" - ) - self._tip_refresh_lock.release() - publication_lock_acquired = False - - with self.lock: - progress_eligible_client = any( - self.client_can_receive_jobs(client) - for client in self.clients - ) - if use_prepared_fanout or not progress_eligible_client: - # Ready-mode work was prepared above. With no eligible clients, - # publishing the reconciled snapshot is sufficient; collection - # work with a live identity is not published until its worker- - # specific bundle is actually delivered below. - self._record_progress_publication( - snapshot, - payout_generation_after_reconciliation, - ) - - if not ready_mode: - with self.lock: - eligible_collection_client = any( - self.client_can_receive_jobs(client) - for client in self.clients - ) - if not eligible_collection_client: - self._retain_collection_refresh( - snapshot, - observation_sequence, - payout_generation_after_reconciliation, - ) - - if use_prepared_fanout: - assert bundle is not None - ( - refreshed, - first_delivery, - last_delivery, - build_failures, - ) = self._fanout_prepared_tip_refresh( - clients, - bundle, - snapshot, - observation_sequence=observation_sequence, - validation_token=validation_token, - preactivated_cancel_event=preactivated_cancel_event, - executor=prepared_executor, - expected_active_jobs=expected_active_jobs, - heartbeat_name=heartbeat_name, - ) - else: - for client in clients: - if self.stop_event.is_set(): - break - # Collection bundles are worker-specific, so build and - # validate each selected target independently. - self._record_heartbeat(heartbeat_name) - with self.lock: - target_connected = client in self.clients - target_eligible = ( - target_connected - and self.client_can_receive_jobs(client) - ) - if not target_connected: - self._record_tip_refresh_client_result("disconnected") - continue - if not target_eligible: - self._record_tip_refresh_client_result("skipped") - continue - try: - if self.maybe_send_job( - client, - clean_jobs=self.client_tip_changed_for_snapshot(client, snapshot), - raise_on_reorg_failure=True, - raise_on_build_failure=True, - tip_refresh_snapshot=snapshot, - tip_refresh_observation_sequence=observation_sequence, - ): - delivered = time.monotonic() - refreshed += 1 - first_delivery = ( - delivered - if first_delivery is None - else min(first_delivery, delivered) - ) - last_delivery = ( - delivered - if last_delivery is None - else max(last_delivery, delivered) - ) - self._record_tip_refresh_client_result("sent") - else: - self._record_tip_refresh_client_result("skipped") - except _JobBuildFailed: - build_failures += 1 - self._record_tip_refresh_client_result("failed") - except OSError: - self._record_tip_refresh_client_result("disconnected") - self.disconnect_client(client) - - if not ready_mode: - with self.lock: - eligible_collection_client = any( - self.client_can_receive_jobs(client) - for client in self.clients - ) - if not eligible_collection_client: - self._retain_collection_refresh( - snapshot, - observation_sequence, - payout_generation_after_reconciliation, - ) - self._record_progress_publication( - snapshot, - payout_generation_after_reconciliation, - ) - - if clients: - try: - post_fanout_tip = str(self.rpc.call("getbestblockhash")) - except Exception as exc: - self._schedule_tip_refresh_retry() - raise TemplateRefreshBlocked( - "qbit tip validation failed after sequential refresh; " - "immediate retry scheduled" - ) from exc - if post_fanout_tip != snapshot.bestblockhash: - self._schedule_tip_refresh_retry() - raise TemplateRefreshSuperseded( - "qbit tip changed during sequential refresh; " - "immediate retry scheduled " - f"expected={snapshot.bestblockhash} current={post_fanout_tip}" - ) - if int(getattr(self, "_payout_state_generation", 0)) != ( - payout_generation_after_reconciliation - ): - self._schedule_tip_refresh_retry() - raise TemplateRefreshSuperseded( - "payout state changed during sequential refresh; " - "immediate retry scheduled" - ) - - if refreshed == 0 and build_failures: - raise TemplateRefreshBlocked( - f"job builds failed for {build_failures} client(s); no refreshed work was issued" - ) - if refreshed: - with self.lock: - self.tip_refresh_job_count += refreshed - assert first_delivery is not None and last_delivery is not None - self._observe_tip_refresh_seconds( - "first_delivery", - first_delivery - refresh_started, - ) - self._observe_tip_refresh_seconds( - "last_delivery", - last_delivery - refresh_started, - ) - if not self._clear_tip_refresh_pending_for_completed_refresh( - snapshot, - observation_sequence, - payout_generation_after_reconciliation, - pending_signal_token, - ): - # A newer tip or payout mutation won after the last delivery - # guard. Preserve its pending token and retry immediately. - pending_signal_token = None - self._schedule_tip_refresh_retry() - raise TemplateRefreshSuperseded( - "tip or payout state changed before refresh completion; " - "immediate retry scheduled" - ) - self.last_successful_template_refresh_monotonic = time.monotonic() - self.template_refresh_failure_started_monotonic = None - self._clear_coordination_blocked_streak() - self._clear_tip_refresh_failure_holdoff() - # A completed pass reconfirms that the coherent snapshot remained - # current through publication and fanout. Refresh the liveness - # stamp so a legitimately long, progressing pass does not become - # stale the instant its active marker is cleared. - self._record_progress_tip_poll(snapshot) - return refreshed - except TemplateRefreshSuperseded: - # Coordination-blocked refreshes -- a superseded tip, a pending - # payout publication fence, a refresh raced by payout mutation -- - # are churn between healthy components, not qbitd unhealthiness. - # They do not arm the ordinary failure budget, but their oldest - # continuous streak has its own longer restart deadline. Re-raise - # so callers still schedule their immediate retry. Plain - # TemplateRefreshBlocked stays budgeted below: it also wraps - # genuine failures (job builds failing, malformed template - # artifacts, untrusted chain views) whose persistence must still - # take the ordinary restart path. - # The retry-spacing stamp still applies: churn against an - # unchanged tip re-arms the poller no faster than the holdoff, - # while a genuinely newer tip zeroes it immediately. - if not getattr(self, "tip_refresh_epoch_fanout", False): - self._record_coordination_blocked_refresh(time.monotonic()) - self._note_tip_refresh_attempt_failed(observed_best_tip) - raise - except _PayoutStatePublicationBlocked: - self._record_coordination_blocked_refresh(time.monotonic()) - self._note_tip_refresh_attempt_failed(observed_best_tip) - raise - except Exception: - self._clear_coordination_blocked_streak() - self._record_template_refresh_failure(time.monotonic()) - self._note_tip_refresh_attempt_failed(observed_best_tip) - raise - finally: - if publication_lock_acquired: - self._tip_refresh_lock.release() - if progress_refresh_active: - self._progress_refresh_finished() - - def _probe_tip_while_refresh_waiting(self) -> None: - """Detect a changed live tip without entering the heavy refresh lane.""" - observation_sequence = self._reserve_tip_observation_sequence() - try: - observed_tip = str(self.rpc.call("getbestblockhash")) - except Exception: - # The owning refresh still has to unwind or complete. Preserve its - # pending state and let the next bounded lock wait probe again. - return - self.observe_tip_for_refresh( - observed_tip, + return self._ensure_tip_refresh_service()._poll_qbit_tip_template_pass_once( + heartbeat_name=heartbeat_name, + refresh_started=refresh_started, observation_sequence=observation_sequence, + observed_best_tip=observed_best_tip, ) + def _probe_tip_while_refresh_waiting(self) -> None: + return self._ensure_tip_refresh_service()._probe_tip_while_refresh_waiting() + def _detected_tip_supersedes_locked( self, tip_hash: str, observation_sequence: int, ) -> bool: - latest = getattr(self, "latest_detected_tip", None) - return bool( - latest is not None - and latest[0] != tip_hash - and latest[1] > observation_sequence + return self._ensure_tip_refresh_service()._detected_tip_supersedes_locked( + tip_hash, + observation_sequence, ) def _raise_if_tip_refresh_superseded( @@ -17097,151 +7257,26 @@ def _raise_if_tip_refresh_superseded( snapshot: QbitTipTemplateSnapshot, observation_sequence: int, ) -> None: - """Stop obsolete work before entering another expensive phase.""" - with self.lock: - superseded = self._detected_tip_supersedes_locked( - snapshot.bestblockhash, - observation_sequence, - ) - if superseded: - self._schedule_tip_refresh_retry() - raise TemplateRefreshSuperseded( - "tip/template poll was superseded by a newer tip observation " - "before refresh preparation" - ) + return self._ensure_tip_refresh_service()._raise_if_tip_refresh_superseded( + snapshot, + observation_sequence, + ) def _reserve_tip_observation_sequence(self) -> int: - with self.lock: - sequence = int(getattr(self, "tip_observation_sequence", 0)) + 1 - self.tip_observation_sequence = sequence - return sequence + return self._ensure_tip_refresh_service()._reserve_tip_observation_sequence() def observe_tip_for_refresh( self, tip_hash: str, *, observation_sequence: int | None = None, - mark_pending: bool = True, - ) -> bool: - """Record tip detection without publishing share-validation authority. - - A waiter outside ``_tip_refresh_lock`` must be able to cancel obsolete - bundle construction and fanout promptly. It must not update - ``current_tip_first_seen``: that value invalidates old jobs, so the - winning refresh publishes it only after replacement work is prepared - and validated. - """ - if observation_sequence is None: - observation_sequence = self._reserve_tip_observation_sequence() - self._ensure_job_cache_state() - self._ensure_tip_refresh_state() - # Before any sequencing decision: an own-candidate hash arriving as - # the tip is acceptance evidence (blockwait can see it before -- or - # instead of -- the direct submitblock ack) and must be registered - # even when this observation loses the sequence race below. - self._note_tip_observation_for_candidates(tip_hash) - now = time.monotonic() - active_to_cancel: _FanoutCancellation | None = None - should_mark_pending = False - with self.lock: - latest = getattr(self, "latest_detected_tip", None) - if latest is not None and observation_sequence < latest[1]: - return latest[0] == tip_hash - published = getattr(self, "current_tip_first_seen", None) - prior_detected_hash = ( - latest[0] - if latest is not None - else published[0] - if published is not None - else None - ) - detection_changed = ( - prior_detected_hash is not None - and prior_detected_hash != tip_hash - ) - if detection_changed: - self._evict_reorg_reconcile_memo_for_new_tip_locked(tip_hash) - # Bumped exactly when cached reconcile proofs are dropped: - # a refresh's join compares this epoch to tell whether any - # flip (away, or away and back) interleaved its template - # fetch and invalidated a pass that ran concurrently. - self.tip_detection_epoch = ( - int(getattr(self, "tip_detection_epoch", 0)) + 1 - ) - if ( - detection_changed - and self._payout_state_source[1] != tip_hash - ): - # Supersede an in-progress immutable payout candidate as soon - # as the newer tip is detected, without publishing that tip as - # submit authority before replacement work is ready. - source_generation = self._payout_state_source[0] + 1 - self._payout_state_source = ( - source_generation, - tip_hash, - "external_tip", - now, - ) - if latest is None or detection_changed: - self._mint_tip_refresh_epoch_locked( - tip_hash=tip_hash, - payout_state_generation=int( - getattr(self, "_payout_state_generation", 0) - ), - started_monotonic=now, - ) - self.latest_detected_tip = (tip_hash, observation_sequence) - replacement_needed = published is None or published[0] != tip_hash - if published is not None and published[0] == tip_hash: - # A live observation has returned to (or reconfirmed) the - # published generation. This closes any unpublished divergence - # epoch without changing the published snapshot itself. - self.current_tip_observed_monotonic = now - self.tip_refresh_divergence_started_monotonic = None - elif ( - published is not None - and getattr( - self, - "tip_refresh_divergence_started_monotonic", - None, - ) - is None - ): - # Anchor the lease to the first departure from the published - # tip. Further B -> C observations deliberately do not renew it. - self.tip_refresh_divergence_started_monotonic = now - pending_already = bool( - getattr(self, "_tip_refresh_pending_event", None) - and self._tip_refresh_pending_event.is_set() - ) - active = getattr(self, "_active_tip_refresh", None) - if ( - active is not None - and active[0].tip_hash != tip_hash - and active[0].observation_sequence < observation_sequence - ): - active_to_cancel = active[1] - should_mark_pending = bool( - mark_pending - and ( - detection_changed - or (replacement_needed and not pending_already) - ) - ) - - if active_to_cancel is not None: - active_to_cancel.cancel() - if detection_changed: - self._progress_note_refresh_pending(now) - # In-flight constructions for older parents can no longer win; - # stop them at detection so replacement preparation starts - # immediately, well before publication moves submit authority. - self._cancel_obsolete_job_bundle_builds(current_tip=tip_hash) - self._cancel_obsolete_job_builds("chain tip superseded") - if should_mark_pending: - self._mark_tip_refresh_pending(observation_sequence) - self._schedule_tip_refresh_retry() - return True + mark_pending: bool = True, + ) -> bool: + return self._ensure_tip_refresh_service().observe_tip_for_refresh( + tip_hash, + observation_sequence=observation_sequence, + mark_pending=mark_pending, + ) def observe_tip_first_seen( self, @@ -17251,194 +7286,21 @@ def observe_tip_first_seen( publish_refresh_observation: bool = False, published_snapshot: QbitTipTemplateSnapshot | None = None, ) -> bool: - """Publish a tip (and, when supplied, its coherent template snapshot). - - Detection and publication are deliberately separate. Callers doing a - prepared refresh must finish bundle construction and final validation - before invoking this method. - """ - if ( - published_snapshot is not None - and published_snapshot.bestblockhash != tip_hash - ): - raise ValueError("published snapshot does not match tip hash") - if observation_sequence is None: - observation_sequence = self._reserve_tip_observation_sequence() - if not self.observe_tip_for_refresh( + return self._ensure_tip_refresh_service().observe_tip_first_seen( tip_hash, observation_sequence=observation_sequence, - mark_pending=False, - ): - return False - now = time.monotonic() - with self.lock: - current_sequence = int( - getattr(self, "current_tip_observation_sequence", 0) - ) - if ( - observation_sequence < current_sequence - or self._detected_tip_supersedes_locked( - tip_hash, - observation_sequence, - ) - ): - return False - first_seen = getattr(self, "current_tip_first_seen", None) - if first_seen is not None and first_seen[0] == tip_hash: - # A same-tip re-observation proves the tip view is live; the - # freshness stamp bounds submit_stale_check_tip reuse. - self.current_tip_observed_monotonic = now - active = getattr(self, "_active_tip_refresh", None) - # A routine blockwait/poll observation of the same hash carries - # no newer template. While that hash is actively fanning out, - # do not invalidate its token merely by advancing the global - # observation sequence. The next real refresh observation can - # advance it after the active fanout clears. - if publish_refresh_observation and ( - active is None or active[0].tip_hash != tip_hash - ): - self.current_tip_observation_sequence = observation_sequence - if published_snapshot is not None: - self.tip_template_snapshot = published_snapshot - self._publish_tip_refresh_epoch_identity_locked( - published_snapshot - ) - self.tip_refresh_divergence_started_monotonic = None - return True - - # Fetch optional cleanup metadata before the atomic publication. A slow - # parent RPC must not create a window where submits see the new tip but - # the winning refresh has not yet published its coherent snapshot. - try: - parent_hash = self._fetch_tip_parent_hash(tip_hash) - except Exception: - parent_hash = None - - with self.lock: - current_sequence = int( - getattr(self, "current_tip_observation_sequence", 0) - ) - if ( - observation_sequence < current_sequence - or self._detected_tip_supersedes_locked( - tip_hash, - observation_sequence, - ) - ): - return False - first_seen = getattr(self, "current_tip_first_seen", None) - if first_seen is not None and first_seen[0] == tip_hash: - self.current_tip_observed_monotonic = now - active = getattr(self, "_active_tip_refresh", None) - if publish_refresh_observation and ( - active is None or active[0].tip_hash != tip_hash - ): - self.current_tip_observation_sequence = observation_sequence - if published_snapshot is not None: - self.tip_template_snapshot = published_snapshot - self._publish_tip_refresh_epoch_identity_locked( - published_snapshot - ) - self.tip_refresh_divergence_started_monotonic = None - return True - - tip_changed = first_seen is not None - # The first tip this process publishes is a startup baseline, not - # a tip flip: a None stamp keeps stale grace closed. - self.current_tip_first_seen = ( - tip_hash, - now if tip_changed else None, - ) - self.current_tip_observation_sequence = observation_sequence - self.current_tip_observed_monotonic = now - if published_snapshot is not None: - self.tip_template_snapshot = published_snapshot - self._publish_tip_refresh_epoch_identity_locked( - published_snapshot - ) - self.tip_refresh_divergence_started_monotonic = None - # Retained collection work is reusable throughout detection and - # preparation, but never after authority moves to a different tip. - self._retained_collection_refresh = None - if parent_hash is None: - self.current_tip_parent = None - else: - self.current_tip_parent = (tip_hash, parent_hash) - # Reclassify formerly same-tip entries immediately. On mainnet the - # zero stale-grace TTL removes them in this pass; on other chains - # the actual chain parent removes multi-tip-behind entries while - # the independently configured grace lifetime protects one-back. - self.prune_evicted_job_graveyard(now=now, force=True) - if tip_changed: - # A retained ready bundle belongs to the previously published tip - # and can never satisfy the consumer's published-snapshot identity - # check again; release it as soon as authority moves. - with self._job_cache_lock: - self._prepared_ready_bundle = None - self._prepared_ready_snapshot = None - return True + publish_refresh_observation=publish_refresh_observation, + published_snapshot=published_snapshot, + ) def _fetch_tip_parent_hash(self, tip_hash: str) -> str | None: - block = self.rpc.call("getblock", [tip_hash]) - if not isinstance(block, dict): - return None - parent = str(block.get("previousblockhash", "") or "") - if not parent: - return None - return parent + return self._ensure_tip_refresh_service()._fetch_tip_parent_hash(tip_hash) def current_tip_parent_hash(self, tip_hash: str) -> str | None: - with self.lock: - cached = getattr(self, "current_tip_parent", None) - if cached is not None and cached[0] == tip_hash: - return cached[1] - first_seen = getattr(self, "current_tip_first_seen", None) - observed_sequence = ( - int(getattr(self, "current_tip_observation_sequence", 0)) - if first_seen is not None and first_seen[0] == tip_hash - else None - ) - parent = self._fetch_tip_parent_hash(tip_hash) - if parent is None: - return None - with self.lock: - current = getattr(self, "current_tip_first_seen", None) - if ( - observed_sequence is not None - and current is not None - and current[0] == tip_hash - and int(getattr(self, "current_tip_observation_sequence", 0)) - == observed_sequence - ): - self.current_tip_parent = (tip_hash, parent) - return parent + return self._ensure_tip_refresh_service().current_tip_parent_hash(tip_hash) def submit_stale_check_tip(self) -> str: - """Best-known chain tip for per-share submit classification. - - Prefers the tip for which the refresh path already published coherent - work (reconfirmed at least every PRISM_BLOCKPOLL_SECONDS while healthy) - so mining.submit never blocks on a getbestblockhash RPC per share. This - also removes the submit-races-ahead-of-the-refresh failure mode: a - submit-path RPC can observe a new tip seconds before jobs refresh, and - with PRISM_STRATUM_STALE_GRACE_SECONDS=0 (mainnet-forced) that - rejected every in-flight share on the old tip. Classifying against the - published tip keeps shares valid until the coordinator has prepared, - validated, and published the flip, and it is the same tip source the - stale-grace window and evicted-job classification are anchored to. - - During a detected-but-unpublished replacement, the published tip stays - authoritative beyond the ordinary freshness age so a large healthy - build cannot recreate the reject outage. That extension is bounded by - PRISM_TEMPLATE_REFRESH_FAILURE_EXIT_SECONDS and is anchored to the - first unpublished divergence; failed refreshes therefore still fall - back to the live RPC instead of accepting frozen work indefinitely. - """ - with self.lock: - published_tip = self._submit_stale_check_tip_locked(time.monotonic()) - if published_tip is not None: - return published_tip - return str(self.rpc.call("getbestblockhash")) + return self._ensure_tip_refresh_service().submit_stale_check_tip() def _submit_stale_check_tip_locked(self, now: float) -> str | None: """Return the authoritative published submit tip while holding self.lock.""" @@ -17470,48 +7332,8 @@ def _submit_control_snapshot( return pool_closed, context, published_tip def _published_tip_authoritative_locked(self, now: float) -> bool: - """True while the published tip still owns share classification. - - Either the ordinary freshness window (reconfirmed within - PRISM_SUBMIT_TIP_MAX_AGE_SECONDS) or the bounded detected-but- - unpublished replacement lease is open. Job issuance uses the same - predicate so work handed to miners is never classified against a - different tip than the one it was issued for. - """ - max_age = float( - getattr( - self, - "submit_tip_max_age_seconds", - DEFAULT_PRISM_SUBMIT_TIP_MAX_AGE_SECONDS, - ) - ) - if max_age <= 0: - return False - observed = getattr(self, "current_tip_first_seen", None) - if observed is None: - return False - observed_at = getattr(self, "current_tip_observed_monotonic", None) - if observed_at is not None and now - observed_at <= max_age: - return True - latest_detected = getattr(self, "latest_detected_tip", None) - divergence_started = getattr( - self, - "tip_refresh_divergence_started_monotonic", - None, - ) - divergence_budget = float( - getattr( - self, - "template_refresh_failure_exit_seconds", - DEFAULT_PRISM_TEMPLATE_MAX_AGE_SECONDS, - ) - ) - return bool( - latest_detected is not None - and latest_detected[0] != observed[0] - and divergence_started is not None - and divergence_budget > 0 - and now - divergence_started <= divergence_budget + return self._ensure_tip_refresh_service()._published_tip_authoritative_locked( + now, ) def stale_grace_deadline_open( @@ -17563,95 +7385,17 @@ def context_eligible_for_stale_grace( return bool(parent_hash) and str(context.template["previousblockhash"]) == parent_hash def note_tip_work_delivered(self, client: ClientState, job_parent_hash: str) -> None: - """Record the first time this connection was sent work for a tip. - - First delivery wins per tip: same-tip template refreshes must not slide - the connection's stale-grace anchor forward. - """ - now = time.monotonic() - with self.lock: - delivered = client.tip_work_delivered - if delivered is None or delivered[0] != job_parent_hash: - client.tip_work_delivered = (job_parent_hash, now) - self._ensure_initial_job_state() - if job_parent_hash == self._current_published_tip_hash_locked(): - self._reset_delivery_failure_if_coverage_restored_locked() + return self._ensure_job_delivery_service().note_tip_work_delivered( + client, + job_parent_hash, + ) def _ensure_evicted_job_state(self) -> None: - graveyard = getattr(self, "evicted_job_graveyard", None) - rebuild_indexes = False - if not isinstance(graveyard, OrderedDict): - converted: OrderedDict[str, EvictedJobEntry] = OrderedDict() - for job_id, entry in (graveyard or {}).items(): - if isinstance(entry, EvictedJobEntry): - converted[job_id] = entry - continue - context, connection_id, evicted_monotonic = entry - client = next( - ( - candidate - for candidate in getattr(self, "clients", ()) - if candidate.connection_id == connection_id - ), - None, - ) - converted[job_id] = EvictedJobEntry( - context=context, - connection_id=connection_id, - evicted_monotonic=evicted_monotonic, - previousblockhash=str(context.template["previousblockhash"]), - client=client, - ) - self.evicted_job_graveyard = converted - rebuild_indexes = True - if not hasattr(self, "evicted_jobs_by_connection"): - self.evicted_jobs_by_connection = {} - rebuild_indexes = True - if not hasattr(self, "evicted_same_tip_by_connection"): - self.evicted_same_tip_by_connection = {} - rebuild_indexes = True - if not hasattr(self, "evicted_same_tip_job_ids"): - self.evicted_same_tip_job_ids = OrderedDict() - rebuild_indexes = True - if not hasattr(self, "evicted_job_index_tip_hash"): - self.evicted_job_index_tip_hash = None - rebuild_indexes = True - if not hasattr(self, "evicted_job_next_prune_monotonic"): - self.evicted_job_next_prune_monotonic = 0.0 - if not hasattr(self, "evicted_job_expiration_counts"): - self.evicted_job_expiration_counts = { - job_class: 0 for job_class in PRISM_EVICTED_JOB_CLASSES - } - if not hasattr(self, "evicted_job_capacity_eviction_counts"): - self.evicted_job_capacity_eviction_counts = { - scope: 0 for scope in PRISM_EVICTED_JOB_CAPACITY_SCOPES - } - if not hasattr(self, "evicted_job_submit_counts"): - self.evicted_job_submit_counts = { - outcome: 0 for outcome in PRISM_EVICTED_JOB_SUBMIT_OUTCOMES - } - if not hasattr(self, "same_tip_job_retention_seconds"): - self.same_tip_job_retention_seconds = DEFAULT_PRISM_SAME_TIP_JOB_RETENTION_SECONDS - if not hasattr(self, "same_tip_job_retention_per_connection"): - self.same_tip_job_retention_per_connection = ( - DEFAULT_PRISM_SAME_TIP_JOB_RETENTION_PER_CONNECTION - ) - if not hasattr(self, "disconnected_job_retention"): - self.disconnected_job_retention = ( - DEFAULT_PRISM_DISCONNECTED_JOB_RETENTION - ) - if not hasattr(self, "_disconnected_evicted_job_ids"): - # Insertion-ordered ids of graveyard entries whose connection is - # gone (entry.client is None); bounds cross-connection retention - # independently of live connections' per-connection caps. - self._disconnected_evicted_job_ids: OrderedDict[str, None] = ( - OrderedDict() - ) - current_tip = self._current_published_tip_hash_locked() - if self.evicted_job_index_tip_hash != current_tip: - rebuild_indexes = True - if rebuild_indexes: - self._rebuild_evicted_job_indexes_locked() + # Compatibility initializer: the retained-job index is owned by + # JobDeliveryService. Ensuring it adopts legacy/test-seeded graveyard + # state (including pre-owner tuple entries) and rebuilds the + # per-connection/same-tip/disconnected indexes for the current tip. + self._ensure_job_delivery_service().ensure_evicted_job_state() def _current_published_tip_hash_locked(self) -> str | None: first_seen = getattr(self, "current_tip_first_seen", None) @@ -17663,127 +7407,7 @@ def _current_published_tip_hash_locked(self) -> str | None: return None def _evicted_job_class_locked(self, entry: EvictedJobEntry) -> str: - current_tip = self._current_published_tip_hash_locked() - if current_tip is None or entry.previousblockhash == current_tip: - return "same_tip" - return "stale_grace" - - def _remove_evicted_job_locked(self, job_id: str) -> EvictedJobEntry | None: - entry = self.evicted_job_graveyard.pop(job_id, None) - if entry is None: - return None - connection_jobs = self.evicted_jobs_by_connection.get(entry.connection_id) - if connection_jobs is not None: - connection_jobs.pop(job_id, None) - if not connection_jobs: - self.evicted_jobs_by_connection.pop(entry.connection_id, None) - connection_jobs = self.evicted_same_tip_by_connection.get(entry.connection_id) - if connection_jobs is not None: - connection_jobs.pop(job_id, None) - if not connection_jobs: - self.evicted_same_tip_by_connection.pop(entry.connection_id, None) - self.evicted_same_tip_job_ids.pop(job_id, None) - self._disconnected_evicted_job_ids.pop(job_id, None) - return entry - - def _index_evicted_job_locked(self, job_id: str, entry: EvictedJobEntry) -> None: - self.evicted_jobs_by_connection.setdefault( - entry.connection_id, - OrderedDict(), - )[job_id] = None - if self._evicted_job_class_locked(entry) != "same_tip": - return - self.evicted_same_tip_by_connection.setdefault( - entry.connection_id, - OrderedDict(), - )[job_id] = None - self.evicted_same_tip_job_ids[job_id] = None - - def _rebuild_evicted_job_indexes_locked(self) -> None: - self.evicted_jobs_by_connection = {} - self.evicted_same_tip_by_connection = {} - self.evicted_same_tip_job_ids = OrderedDict() - for job_id, entry in self.evicted_job_graveyard.items(): - self._index_evicted_job_locked(job_id, entry) - self.evicted_job_index_tip_hash = self._current_published_tip_hash_locked() - self._enforce_evicted_same_tip_capacity_locked() - - def _enforce_evicted_same_tip_capacity_locked( - self, - connection_id: int | None = None, - ) -> None: - connection_ids = ( - (connection_id,) - if connection_id is not None - else tuple(self.evicted_same_tip_by_connection) - ) - per_connection_cap = int(self.same_tip_job_retention_per_connection) - for candidate_connection_id in connection_ids: - job_ids = self.evicted_same_tip_by_connection.get(candidate_connection_id) - while job_ids is not None and len(job_ids) > per_connection_cap: - oldest_job_id = next(iter(job_ids)) - self._remove_evicted_job_locked(oldest_job_id) - self.evicted_job_capacity_eviction_counts["connection"] += 1 - job_ids = self.evicted_same_tip_by_connection.get(candidate_connection_id) - - def _enforce_disconnected_evicted_capacity_locked(self) -> None: - cap = int( - getattr( - self, - "disconnected_job_retention", - DEFAULT_PRISM_DISCONNECTED_JOB_RETENTION, - ) - ) - while len(self._disconnected_evicted_job_ids) > max(0, cap): - oldest_job_id = next(iter(self._disconnected_evicted_job_ids)) - self._remove_evicted_job_locked(oldest_job_id) - self.evicted_job_capacity_eviction_counts["disconnected"] += 1 - - def _stale_grace_entry_expired_locked( - self, - entry: EvictedJobEntry, - *, - now: float, - ttl: float, - ) -> bool: - current_tip = self._current_published_tip_hash_locked() - first_seen = getattr(self, "current_tip_first_seen", None) - if ( - ttl <= 0 - or current_tip is None - or first_seen is None - or str(first_seen[0]) != current_tip - or first_seen[1] is None - ): - return True - - # Submit eligibility is exactly one chain parent behind, so pruning - # must use that same relationship. The prior poll observation can lag - # (for example when authorize/vardiff issued work on an intermediate - # tip), and using it here would drop work submit would still credit. - # Until the parent RPC has populated the cache, retain conservatively; - # submit classification fetches it before granting stale grace. - cached_parent = getattr(self, "current_tip_parent", None) - if ( - cached_parent is not None - and cached_parent[0] == current_tip - and entry.previousblockhash != cached_parent[1] - ): - return True - - client = entry.client - if client is not None: - delivered = client.tip_work_delivered - if delivered is None or delivered[0] != current_tip: - # Match stale_grace_deadline_open: prior-tip shares stay in - # flight until this connection receives replacement work. - return False - anchor = delivered[1] - else: - # Disconnect normally removes these entries. Keep legacy/test - # orphan state bounded from the refresh path's tip-flip anchor. - anchor = float(first_seen[1]) - return now - anchor > ttl + return self._ensure_job_delivery_service()._evicted_job_class_locked(entry) def bury_evicted_job( self, @@ -17793,44 +7417,11 @@ def bury_evicted_job( now: float | None = None, prune: bool = True, ) -> None: - with self.lock: - self._ensure_evicted_job_state() - context = self.jobs.get(job_id) - if context is None: - return - self._remove_evicted_job_locked(job_id) - self.evicted_job_graveyard[job_id] = EvictedJobEntry( - context=context, - connection_id=client.connection_id, - evicted_monotonic=time.monotonic() if now is None else now, - previousblockhash=str(context.template["previousblockhash"]), - client=client, - ) - self._index_evicted_job_locked(job_id, self.evicted_job_graveyard[job_id]) - self._enforce_evicted_same_tip_capacity_locked(client.connection_id) - if prune: - self.prune_evicted_job_graveyard(now=now, force=False) - - def _evicted_job_expired_locked( - self, - entry: EvictedJobEntry, - *, - now: float, - ) -> tuple[str, bool]: - job_class = self._evicted_job_class_locked(entry) - if job_class == "same_tip": - ttl = float(self.same_tip_job_retention_seconds) - return job_class, ttl <= 0 or now - entry.evicted_monotonic > ttl - return job_class, self._stale_grace_entry_expired_locked( - entry, + return self._ensure_job_delivery_service().bury_evicted_job( + client, + job_id, now=now, - ttl=float( - getattr( - self, - "stale_grace_seconds", - DEFAULT_PRISM_STALE_GRACE_SECONDS, - ) - ), + prune=prune, ) def prune_evicted_job_graveyard( @@ -17839,88 +7430,17 @@ def prune_evicted_job_graveyard( now: float | None = None, force: bool = True, ) -> None: - with self.lock: - self._ensure_evicted_job_state() - if not self.evicted_job_graveyard: - return - now = time.monotonic() if now is None else now - if not force and now < self.evicted_job_next_prune_monotonic: - return - self.evicted_job_next_prune_monotonic = ( - now + DEFAULT_PRISM_EVICTED_JOB_PRUNE_INTERVAL_SECONDS - ) - for job_id, entry in tuple(self.evicted_job_graveyard.items()): - job_class, expired = self._evicted_job_expired_locked(entry, now=now) - if expired: - self._remove_evicted_job_locked(job_id) - self.evicted_job_expiration_counts[job_class] += 1 + return self._ensure_job_delivery_service().prune_evicted_job_graveyard( + now=now, + force=force, + ) def evicted_job_entry( self, client: ClientState, job_id: str, ) -> EvictedJobEntry | None: - with self.lock: - self._ensure_evicted_job_state() - entry = getattr(self, "evicted_job_graveyard", {}).get(job_id) - if entry is None: - return None - if ( - entry.connection_id != client.connection_id - and not self._cross_connection_evicted_entry_allowed_locked( - entry, - client, - ) - ): - return None - job_class, expired = self._evicted_job_expired_locked( - entry, - now=time.monotonic(), - ) - if expired: - self._remove_evicted_job_locked(job_id) - self.evicted_job_expiration_counts[job_class] += 1 - return None - return entry - - def _cross_connection_evicted_entry_allowed_locked( - self, - entry: EvictedJobEntry, - client: ClientState, - ) -> bool: - """Whether a retained context of a dead connection may serve this - submitter. - - Only same-tip contexts qualify (they are fully valid current work; - stale-grace expiry anchors on the original connection's delivery - state, which a dead connection can never advance), and only for an - authorized reconnect of the same username -- the credit goes to the - context's original worker identity either way, so a foreign - submitter could at most donate work, but refusing keeps job ids - unguessable-by-effect and the accounting per miner honest. Replays - stay rejected by the (username, header) dedup key. - """ - if entry.client is not None: - # The owning connection is still alive; job ids stay scoped to it. - return False - if ( - int( - getattr( - self, - "disconnected_job_retention", - DEFAULT_PRISM_DISCONNECTED_JOB_RETENTION, - ) - ) - <= 0 - ): - return False - if not client.authorized or not client.username: - return False - worker = getattr(entry.context, "worker", None) - username = getattr(worker, "username", None) - if not username or username != client.username: - return False - return self._evicted_job_class_locked(entry) == "same_tip" + return self._ensure_job_delivery_service().evicted_job_entry(client, job_id) def evicted_submit_context( self, @@ -17948,15 +7468,10 @@ def note_evicted_job_submit( *, cross_connection: bool = False, ) -> None: - if credit_policy == PRISM_CREDIT_POLICY_STALE_GRACE: - outcome = "credited_stale_grace" - elif cross_connection: - outcome = "accepted_same_tip_cross_connection" - else: - outcome = "accepted_same_tip" - with self.lock: - self._ensure_evicted_job_state() - self.evicted_job_submit_counts[outcome] += 1 + return self._ensure_job_delivery_service().note_evicted_job_submit( + credit_policy, + cross_connection=cross_connection, + ) def refresh_jobs_after_pending_accepted_block( self, @@ -17964,146 +7479,22 @@ def refresh_jobs_after_pending_accepted_block( *, heartbeat_name: str = "qbit_blockpoll", ) -> int: - with self.lock: - block = client.post_accept_refresh_block - client.post_accept_refresh_block = None - if block is None: - return 0 - block_height, block_hash = block - return self.refresh_jobs_after_accepted_block( - block_height=block_height, - block_hash=block_hash, + return self._ensure_tip_refresh_service().refresh_jobs_after_pending_accepted_block( + client, heartbeat_name=heartbeat_name, ) def refresh_jobs_after_accepted_block( self, *, block_height: int, block_hash: str, heartbeat_name: str = "qbit_blockpoll" ) -> int: - try: - self._record_heartbeat(heartbeat_name) - observation_sequence = self._reserve_tip_observation_sequence() - observed_tip = str(self.rpc.call("getbestblockhash")) - if not self.observe_tip_for_refresh( - observed_tip, - observation_sequence=observation_sequence, - ): - raise TemplateRefreshSuperseded( - "post-accept tip observation was superseded" - ) - except TemplateRefreshSuperseded: - # A newer observation won before this notification could publish - # its tip. The shared driver is woken below; this is coordination - # churn, not a post-accept refresh failure. - return 0 - except Exception: - with self.lock: - self.post_accept_refresh_failure_count += 1 - print( - "prism coordinator: post-accept clean job refresh failed after direct PRISM block " - f"height={block_height} hash={block_hash}", - flush=True, - ) - traceback.print_exc() - return 0 - finally: - # Accepted-block payout/template changes can require a same-tip - # rebuild. Wake the driver even when the immediate best-tip RPC or - # observation fails; the owner will refetch a coherent snapshot. - self._schedule_tip_refresh_retry() - print( - "prism coordinator: scheduled single-flight job refresh after " - f"direct PRISM block height={block_height} hash={block_hash} " - f"observed_tip={observed_tip}", - flush=True, - ) - return 0 - - def _reuse_current_tip_template_snapshot( - self, - observed_best_tip: str, - ) -> QbitTipTemplateSnapshot | None: - """Rebuild a snapshot from cached artifacts while their tip holds. - - Honors the PRISM_TEMPLATE_CACHE_SECONDS window that per-client job - builds already use: within it, a poll pass whose observed best tip - still equals the cached template's parent issues no getblocktemplate. - Rapid failed-refresh re-attempts then cost one cheap best-hash probe - each, while a changed tip or an expired window falls through to the - full fetch, so same-tip template rotation still lands on the normal - cadence. - """ - self._ensure_job_cache_state() - ttl = float( - getattr(self, "template_cache_seconds", DEFAULT_PRISM_BLOCKPOLL_SECONDS) - ) - if ttl <= 0: - return None - with self._job_cache_lock: - cached = self._template_artifacts - if cached is None: - return None - if time.monotonic() - cached.fetched_monotonic > ttl: - return None - if str(observed_best_tip) != cached.previousblockhash: - return None - self._record_job_cache_event("template", hit=True) - return QbitTipTemplateSnapshot( - bestblockhash=cached.previousblockhash, - previousblockhash=cached.previousblockhash, - template_fingerprint=cached.fingerprint, - template_generation=cached.generation, - template_artifacts=cached, + return self._ensure_tip_refresh_service().refresh_jobs_after_accepted_block( + block_height=block_height, + block_hash=block_hash, + heartbeat_name=heartbeat_name, ) def fetch_qbit_tip_template_snapshot(self) -> QbitTipTemplateSnapshot: - # Reserve ordering before either RPC: a fetch that started on an older - # view must not become "newer" merely because its template arrived last. - generation = self._reserve_template_artifact_generation() - template = self.rpc.call( - "getblocktemplate", - [{"rules": qbit_gbt_rules(getattr(self, "qbit_chain", "regtest"))}], - ) - if not isinstance(template, dict): - raise RuntimeError("getblocktemplate returned non-object") - previousblockhash = str(template.get("previousblockhash", "") or "") - if not previousblockhash: - raise RuntimeError("getblocktemplate omitted previousblockhash") - # The template parent is the tip this work actually extends. Validate - # it after fetching the template so a tip transition between these RPCs - # cannot produce an old bestblockhash paired with newer work. Reject a - # template that was superseded before it can enter the shared cache or - # drive tip observation/graveyard pruning. - bestblockhash = str(self.rpc.call("getbestblockhash")) - if bestblockhash != previousblockhash: - # Record the discovery before failing: the retry-spacing gate - # releases on a newer DETECTED tip, and without blockwait this - # mismatch is the only place the new tip becomes known. An - # unrecorded discovery would hold the immediate new-tip retry - # for the full failure holdoff. - self.observe_tip_for_refresh(bestblockhash) - self._schedule_tip_refresh_retry() - raise TemplateRefreshSuperseded( - "qbit tip changed while fetching block template " - f"template_parent={previousblockhash} current={bestblockhash}" - ) - # The poll already paid for this template; seed the job-build cache so - # client job builds triggered by the refresh below reuse it instead of - # refetching one template per client. - artifacts = self.store_template_artifacts( - template, - generation=generation, - ) - if artifacts is not None: - return QbitTipTemplateSnapshot( - bestblockhash=bestblockhash, - previousblockhash=artifacts.previousblockhash, - template_fingerprint=artifacts.fingerprint, - template_generation=artifacts.generation, - template_artifacts=artifacts, - ) - raise TemplateRefreshBlocked( - "unable to derive exact artifacts for observed qbit template" - ) + return self._ensure_job_bundle_service().template_repository.fetch_coherent_snapshot() def _note_reorg_reconcile_outcome( self, @@ -18114,372 +7505,100 @@ def _note_reorg_reconcile_outcome( evict_others: bool = False, proof_epoch: int | None = None, ) -> None: - """Record a reconcile outcome in the per-tip trusted memo. - - A trusted pass arms the memo for its own tip. An untrusted outcome - (superseded publication, untrusted chain view) unarms only its own - tip: it proves nothing about reconciliations already completed for - other tips, and unarming them globally forces every job build into a - redundant full pass. A pass that applied orphan/maturity row - mutations passes evict_others=True: cached proofs for other tips - were taken against pre-mutation rows and no longer hold, even if the - chain later flips back before any tip observation lands. A reconcile - error passes clear_memo=True; a partially applied ledger mutation - invalidates every cached outcome. ``proof_epoch`` carries the - tip-detection epoch the pass started its reads in: arming is refused - when the epoch moved during the pass, so a flip away and back can - never re-arm an entry with a proof from the closed epoch (the - latest-detected-hash guard alone cannot see the round trip). - """ - - self._ensure_job_cache_state() - now = time.monotonic() - with self.lock: - self.last_reorg_reconciled_tip_hash = tip_hash - self.last_reorg_reconciled_trusted = trusted - self.last_reorg_reconciled_monotonic = now - memo = self._reorg_reconcile_trusted_memo - if clear_memo: - memo.clear() - return - if evict_others: - for cached_tip in list(memo): - if cached_tip != tip_hash: - del memo[cached_tip] - if tip_hash is None: - return - if trusted: - latest = getattr(self, "latest_detected_tip", None) - if latest is not None and latest[0] != tip_hash: - # This pass finished for a tip that is no longer the - # newest detected one; its epoch is over. Arming would - # re-add an entry the newer observation already evicted - # and let a flip-back reuse a pre-flip outcome. - return - if proof_epoch is not None and proof_epoch != int( - getattr(self, "tip_detection_epoch", 0) - ): - # The pass spanned a detection cycle; every memo - # consumer (refresh joins, initial-job and vardiff-idle - # builds) must see a full re-proof instead. - return - memo[tip_hash] = now - memo.move_to_end(tip_hash) - while len(memo) > PRISM_REORG_RECONCILE_MEMO_MAX_TIPS: - memo.popitem(last=False) - else: - memo.pop(tip_hash, None) + """Compatibility seam for the extracted reorg reconciler owner.""" + self._ensure_reorg_reconciler_service().note_outcome( + tip_hash, + trusted=trusted, + clear_memo=clear_memo, + evict_others=evict_others, + proof_epoch=proof_epoch, + ) def _evict_reorg_reconcile_memo_for_new_tip_locked( self, tip_hash: str, ) -> None: - """Drop trusted-reconcile entries for every tip except ``tip_hash``. - - Called under self.lock when a newer tip is detected. A detected flip - ends the epoch of every previously cached outcome: if the chain later - flips back to an earlier hash within the cache TTL, pool-block chain - state must be re-proven by a fresh pass, not assumed from a pre-flip - reconciliation. Detection is observation-sequenced, so only genuinely - newer observations evict. - """ - - self._ensure_job_cache_state() - memo = self._reorg_reconcile_trusted_memo - for cached_tip in list(memo): - if cached_tip != tip_hash: - del memo[cached_tip] + self._ensure_reorg_reconciler_service().evict_memo_for_new_tip_locked( + tip_hash + ) def ensure_reorg_reconciled_for_current_tip( self, *, expected_tip_hash: str | None = None, ) -> bool: - reconciler_enabled = getattr(self, "reorg_reconciler_enabled", True) - if not reconciler_enabled and expected_tip_hash is None: - return True - current_tip = str(self.rpc.call("getbestblockhash")) - if expected_tip_hash is not None and current_tip != expected_tip_hash: - raise TemplateRefreshSuperseded( - "qbit tip changed while prepared work was queued " - f"expected={expected_tip_hash} current={current_tip}" - ) - if not reconciler_enabled: - return True - if self._reorg_reconcile_memo_fresh(current_tip): - self._record_reorg_reconcile_lookup("job_build", "memo_hit") - return True - self._record_reorg_reconcile_lookup("job_build", "serial") - return self.ensure_reorg_reconciled_for_tip(current_tip) + return self._ensure_reorg_reconciler_service().ensure_current( + expected_tip_hash=expected_tip_hash + ) def _reorg_reconcile_memo_fresh(self, tip_hash: str) -> bool: - """True when a trusted pass for ``tip_hash`` is inside the cache TTL - and the live chain view is still trusted. - - A fresh memo entry lets a caller reuse the completed pass instead of - queueing a redundant serialized one. The memo is per tip: an - untrusted outcome recorded for another tip never unarms this one. - The chain-view trust check is NOT cached: headers can run ahead of - the validated tip without the best block hash changing (an arriving - reorg), and job issuance must pause immediately, not a TTL later. - """ - ttl = getattr( - self, - "reorg_reconcile_cache_seconds", - DEFAULT_PRISM_REORG_RECONCILE_CACHE_SECONDS, - ) - if ttl <= 0: - return False - self._ensure_job_cache_state() - with self.lock: - reconciled_monotonic = self._reorg_reconcile_trusted_memo.get( - tip_hash - ) - return bool( - reconciled_monotonic is not None - and time.monotonic() - reconciled_monotonic <= ttl - and not self.qbit_chain_view_untrusted() - ) + return self._ensure_reorg_reconciler_service().memo_fresh(tip_hash) def _record_reorg_reconcile_lookup(self, path: str, source: str) -> None: - if path not in PRISM_REORG_RECONCILE_LOOKUP_PATHS: - raise ValueError(f"unknown reorg reconcile lookup path: {path}") - if source not in PRISM_REORG_RECONCILE_LOOKUP_SOURCES: - raise ValueError(f"unknown reorg reconcile lookup source: {source}") - self._ensure_job_cache_state() - with self.lock: - self.reorg_reconcile_lookup_counts[(path, source)] += 1 - - def _reconcile_prefetch_pass( - self, - tip_hash: str, - prove: bool = False, - ) -> bool: - """One prefetched reconcile, honoring the memo like the join does. - - A prefetch that queued behind a completed same-tip pass (abandoned - refresh attempts reuse the slot, but a replaced tip can leave one - queued) would otherwise re-run the full serialized pass for nothing. - A proving pass (the serial re-prove branches) bypasses the memo: - those branches exist precisely because the entry cannot be trusted. - """ - if not prove and self._reorg_reconcile_memo_fresh(tip_hash): - return True - return self.ensure_reorg_reconciled_for_tip(tip_hash) - - def _submit_reconcile_prefetch( - self, - tip_hash: str, - *, - prove: bool = False, - ) -> Future[bool] | None: - """Run one reconcile pass on the prefetch worker so it overlaps the - caller's template fetch. - - Returns ``None`` once shutdown has retired the executor; the caller - falls back to its serial pass. At most one prefetch is outstanding: - a refresh attempt that failed before its join (for example a - template-RPC outage) leaves its future in the slot, and the retry - reuses it for the same tip instead of queueing another serialized - pass behind the first. - """ - self._ensure_job_cache_state() - stale_future: Future[bool] | None = None - future: Future[bool] | None = None - with self._reconcile_prefetch_executor_lock: - if self._reconcile_prefetch_executor_shutdown: - return None - pending = self._reconcile_prefetch_pending - if pending is not None: - pending_tip, pending_future, pending_proves = pending - if ( - not pending_future.done() - and pending_tip == tip_hash - and (pending_proves or not prove) - ): - # A proving pass satisfies both kinds of caller; a - # memo-honoring pass cannot satisfy a prove request and - # is replaced below like a tip change. - return pending_future - # Replaced tip or completed future: hand the old future off - # for disposal outside this lock -- cancellation runs done - # callbacks inline on this thread, and _clear_slot below - # re-takes the lock. - self._reconcile_prefetch_pending = None - if not pending_future.done(): - stale_future = pending_future - executor = self._reconcile_prefetch_executor - if executor is None: - executor = ThreadPoolExecutor( - max_workers=1, - thread_name_prefix="prism-reconcile-prefetch", - ) - self._reconcile_prefetch_executor = executor - try: - future = executor.submit( - self._reconcile_prefetch_pass, tip_hash, prove - ) - self._reconcile_prefetch_pending = (tip_hash, future, prove) - except RuntimeError: - # Executor shutdown raced this submit; the serial path - # covers it (after the stale future is disposed below). - future = None - if stale_future is not None and not stale_future.cancel(): - # Already running for a replaced tip; let it finish detached. - # The slot holds the new tip, so at most one task ever waits - # behind the running one. - self._discard_stale_reconcile_prefetch(stale_future) - if future is None: - return None + self._ensure_reorg_reconciler_service().record_lookup(path, source) - def _clear_slot(done: Future[bool]) -> None: - with self._reconcile_prefetch_executor_lock: - pending_now = self._reconcile_prefetch_pending - if pending_now is not None and pending_now[1] is done: - self._reconcile_prefetch_pending = None + @property + def reorg_reconcile_lookup_counts(self) -> dict[tuple[str, str], int]: + """Copied lookup counters from the reorg owner (metrics consumer).""" + return ( + self._ensure_reorg_reconciler_service() + .reorg_reconcile_lookup_snapshot() + ) - # Registered outside the slot lock: a completed future runs the - # callback inline on this thread. - future.add_done_callback(_clear_slot) - return future + def _submit_reconcile_prefetch( + self, + tip_hash: str, + *, + prove: bool = False, + ) -> Future[bool] | None: + return self._ensure_reorg_reconciler_service().submit_prefetch( + tip_hash, + prove=prove, + ) @staticmethod def _discard_stale_reconcile_prefetch(future: Future[bool]) -> None: - """Detach a prefetched pass whose tip was superseded during the - template fetch. - - The pass cannot be cancelled mid-ledger-mutation and already records - its own outcome/error accounting; the serial pass for the current - tip re-surfaces any condition that still applies. Consuming the - result here only prevents an unretrieved-exception warning. - """ - - def _consume(done: Future[bool]) -> None: - try: - done.result() - except BaseException: - pass - - future.add_done_callback(_consume) + ReorgReconcilerService.discard_stale_prefetch(future) def _join_reconcile_prefetch_bounded(self, prefetch: Future[bool]) -> bool: - """Join a reconcile pass under the poll loop's bounded budget. - - On a genuine expiry the pass keeps running in its single prefetch - slot -- the paced retry re-joins the same future -- and the timeout - surfaces the normal blocked-retry path, keeping the poll loop's - liveness heartbeat fed while the pass catches up. The budget is - clamped below the loop's failure budget so a misconfigured override - cannot reinstate the park this bound exists to prevent. - """ - - join_timeout = min( - PRISM_RECONCILE_PREFETCH_JOIN_TIMEOUT_CEILING_SECONDS, - max( - 0.001, - float( - getattr( - self, - "reconcile_prefetch_join_timeout_seconds", - PRISM_RECONCILE_PREFETCH_JOIN_TIMEOUT_SECONDS, - ) - ), - ), + return self._ensure_reorg_reconciler_service().join_prefetch_bounded( + prefetch ) - try: - return prefetch.result(timeout=join_timeout) - except TimeoutError: - if prefetch.done(): - # The pass itself raised TimeoutError (socket.timeout is - # TimeoutError here): not a join expiry. Propagate silently - # so diagnosis points at the pass, not the join. - raise - print( - "prism coordinator: reconcile prefetch join exceeded " - f"{join_timeout:g}s; retrying refresh pass while it " - "completes", - flush=True, - ) - raise def _reconcile_snapshot_tip_bounded(self, tip_hash: str) -> bool: - """Run a snapshot-tip re-prove off-thread with the bounded join. - - The serial re-prove branches run the same crawling pass as the - overlapped prefetch; routing them through the prefetch slot gives - the poll loop one uniform bounded wait. Falls back to the direct - pass only when the prefetch executor has already been retired at - shutdown, whose exceptions the caller already maps to a clean exit. - """ - - prefetch = self._submit_reconcile_prefetch(tip_hash, prove=True) - if prefetch is None: - return self.ensure_reorg_reconciled_for_tip(tip_hash) - return self._join_reconcile_prefetch_bounded(prefetch) + return self._ensure_reorg_reconciler_service().snapshot_tip_bounded( + tip_hash + ) def shutdown_reconcile_prefetch_executor(self) -> None: - self._ensure_job_cache_state() - with self._reconcile_prefetch_executor_lock: - executor = self._reconcile_prefetch_executor - self._reconcile_prefetch_executor = None - self._reconcile_prefetch_executor_shutdown = True - self._reconcile_prefetch_pending = None - if executor is not None: - # A pass blocked on writer admission aborts via - # ShutdownInProgress on its own; never hold shutdown for it. - executor.shutdown(wait=False, cancel_futures=True) + self._ensure_reorg_reconciler_service().shutdown_prefetch_executor() def ensure_reorg_reconciled_for_tip( self, tip_hash: str, *, _coalesce_same_tip: bool = True, + _caller: str | None = None, ) -> bool: """Reconcile one tip, optionally bypassing same-tip flight reuse. Lock-owning accepted-block callers disable waiting for an existing leader because it may itself be waiting for the payout-balance mutation lock. Their own pass remains visible to ordinary followers. + ``_caller`` optionally names the #224 attribution caller; the + service infers ``landing`` from the flight bypass when omitted. """ - if not getattr(self, "reorg_reconciler_enabled", True): - return True - if _coalesce_same_tip: - summary = self.reconcile_prism_pool_blocks_once(tip_hash=tip_hash) - else: - summary = self.reconcile_prism_pool_blocks_once( - tip_hash=tip_hash, - _wait_for_same_tip_flight=False, - ) - return not bool(summary.get("untrusted") or summary.get("superseded")) + return self._ensure_reorg_reconciler_service().ensure_tip( + tip_hash, + _coalesce_same_tip=_coalesce_same_tip, + _caller=_caller, + ) def qbit_chain_view_untrusted(self) -> bool: - blockchain_info = self.rpc.call("getblockchaininfo") - if not isinstance(blockchain_info, dict): - raise RuntimeError("getblockchaininfo returned non-object") - public_chain = str(getattr(self, "qbit_chain", "regtest")).lower() in { - "main", - "mainnet", - *TESTNET_QBIT_CHAINS, - } - if ( - blockchain_info.get("initialblockdownload") is not False - if public_chain - else bool(blockchain_info.get("initialblockdownload")) - ): - return True - blocks_raw = blockchain_info.get("blocks") - headers_raw = blockchain_info.get("headers") - if public_chain and (blocks_raw is None or headers_raw is None): - return True - if blocks_raw is not None and headers_raw is not None: - try: - blocks = int(blocks_raw) - headers = int(headers_raw) - if blocks < 0 or headers < 0 or headers != blocks: - return True - except (TypeError, ValueError) as exc: - raise RuntimeError("getblockchaininfo blocks/headers are not integers") from exc - return False + return reorg_chain_view_untrusted( + lambda *args, **kwargs: self.rpc.call(*args, **kwargs), + str(getattr(self, "qbit_chain", "regtest")), + ) def validate_live_chain_identity(self) -> None: """Fail closed when a public-chain node is wrong, isolated, or behind.""" @@ -18604,69 +7723,23 @@ def reconcile_prism_pool_blocks_once( _force_publish: bool = False, _source_reserved: bool = False, _wait_for_same_tip_flight: bool = True, + _caller: str | None = None, ) -> dict[str, object]: - """Reconcile pool blocks, coalescing same-tip concurrent callers. - - Unflagged callers asking about a tip whose pass is already in flight - await that pass and share its summary instead of queueing another - full serialized pass. Callers carrying side-effect obligations (a - forced publication or an already-reserved source) and callers without - a tip always run their own pass. Lock-owning callers may disable - waiting for an existing flight while still registering as the visible - leader when no same-tip flight exists. + """Public reconcile entrypoint: service flight admission runs first. + + Same-tip flight coalescing happens *before* the serialized adapter's + writer admission so followers never queue behind it; see the service + docstring for the full contract. ``_caller`` optionally names the + #224 attribution caller; a forced publication without one is the + post-confirm path. """ - self._ensure_job_cache_state() - if tip_hash is None or _force_publish or _source_reserved: - return self._reconcile_prism_pool_blocks_serialized( - tip_hash=tip_hash, - _force_publish=_force_publish, - _source_reserved=_source_reserved, - ) - with self._reconcile_flight_lock: - flight = self._reconcile_flights.get(tip_hash) - leading = flight is None - if leading: - flight = _ReconcileFlight() - self._reconcile_flights[tip_hash] = flight - if not leading: - if not _wait_for_same_tip_flight: - return self._reconcile_prism_pool_blocks_serialized( - tip_hash=tip_hash - ) - wait_seconds = float( - getattr( - self, - "reconcile_flight_wait_seconds", - DEFAULT_PRISM_RECONCILE_FLIGHT_WAIT_SECONDS, - ) - ) - if flight.event.wait(timeout=wait_seconds): - exception = flight.exception - if exception is not None: - raise exception - summary = flight.summary - assert summary is not None - # Followers get a copy: summaries are mutable dicts and the - # leader's caller already holds the original. - return dict(summary) - # Liveness backstop: the leader outlived the wait. Run our own - # pass; the writer lock still serializes the actual work. - return self._reconcile_prism_pool_blocks_serialized( - tip_hash=tip_hash - ) - try: - summary = self._reconcile_prism_pool_blocks_serialized( - tip_hash=tip_hash - ) - flight.summary = summary - return summary - except BaseException as exc: - flight.exception = exc - raise - finally: - with self._reconcile_flight_lock: - self._reconcile_flights.pop(tip_hash, None) - flight.event.set() + return self._ensure_reorg_reconciler_service().reconcile_with_flights( + tip_hash=tip_hash, + _force_publish=_force_publish, + _source_reserved=_source_reserved, + _wait_for_same_tip_flight=_wait_for_same_tip_flight, + _caller=_caller, + ) @ledger_writer_operation("payout_reconciliation") def _reconcile_prism_pool_blocks_serialized( @@ -18676,7 +7749,13 @@ def _reconcile_prism_pool_blocks_serialized( _force_publish: bool = False, _source_reserved: bool = False, ) -> dict[str, object]: - """Serialize reconciliation against accepted-block finalization.""" + """Serialize reconciliation against accepted-block finalization. + + Cross-owner adapter retained by the coordinator: writer-admission + decorator, then the direct payout-balance mutation lock, then the + landed-preview fail-closed check, then the service core. Never + replace this sequence with the _payout_balance_mutation() wrapper. + """ self._ensure_job_cache_state() with self._payout_balance_mutation_lock: with self._accepted_block_payout_preview_condition: @@ -18684,355 +7763,17 @@ def _reconcile_prism_pool_blocks_serialized( transition.landed for transition in self._accepted_block_payout_previews.values() ): - raise _PayoutStatePublicationBlocked( + raise PayoutStatePublicationBlocked( "accepted block payout confirmation is still pending" ) - return self._reconcile_prism_pool_blocks_once( + return self._ensure_reorg_reconciler_service().reconcile( tip_hash=tip_hash, - _force_publish=_force_publish, - _source_reserved=_source_reserved, - ) - - def _reconcile_prism_pool_blocks_once( - self, - *, - tip_hash: str | None = None, - _force_publish: bool = False, - _source_reserved: bool = False, - ) -> dict[str, object]: - summary: dict[str, object] = { - "enabled": bool(getattr(self, "reorg_reconciler_enabled", True)), - "untrusted": False, - "superseded": False, - "published_generation": None, - "watched_blocks": 0, - "inactive_blocks": 0, - "reactivated_blocks": 0, - "matured_payouts": 0, - } - if not getattr(self, "reorg_reconciler_enabled", True): - return summary - self._ensure_job_cache_state() - if not _source_reserved and tip_hash is not None: - # Tip observation normally reserves this source before queueing - # reconciliation. Direct callers only need a new source when they - # are asking about a different tip; repeated reconciliation of the - # same tip must not supersede otherwise valid prepared work. - with self.lock: - current_source_tip = self._payout_state_source[1] - if current_source_tip != tip_hash: - self._reserve_payout_state_source( - "external_tip", - tip_hash=tip_hash, - ) - - inactive_blocks_total = 0 - reactivated_blocks_total = 0 - matured_payouts_total = 0 - supersession_retries = 0 - skip_recorded = False - max_supersession_retries = max( - 0, - int( - getattr( - self, - "payout_reconcile_supersession_retries", - DEFAULT_PRISM_PAYOUT_RECONCILE_SUPERSESSION_RETRIES, - ) - ), - ) - - proof_epoch = int(getattr(self, "tip_detection_epoch", 0)) - - def finish(*, trusted: bool) -> dict[str, object]: - with self.lock: - self.reorg_inactive_block_count += inactive_blocks_total - self.reorg_reactivated_block_count += reactivated_blocks_total - self.matured_payout_count += matured_payouts_total - self._note_reorg_reconcile_outcome( - tip_hash, - trusted=trusted, - # Row mutations invalidate proofs cached for other tips even - # when the mutating pass's tip was never observed (per-client - # callers reconcile straight off getbestblockhash). - evict_others=bool( - inactive_blocks_total - or reactivated_blocks_total - or matured_payouts_total - ), - proof_epoch=proof_epoch, + force_publish=_force_publish, + source_reserved=_source_reserved, ) - summary["inactive_blocks"] = inactive_blocks_total - summary["reactivated_blocks"] = reactivated_blocks_total - summary["matured_payouts"] = matured_payouts_total - return summary - - def retry_superseded_candidate() -> bool: - nonlocal supersession_retries, tip_hash - supersession_retries += 1 - if supersession_retries > max_supersession_retries: - summary["superseded"] = True - self._block_payout_state_publication() - return False - with self.lock: - latest_tip = self._payout_state_source[1] - tip_hash = latest_tip or tip_hash - return True - - while True: - candidate_to_publish: PayoutStateCandidate | None = None - error_candidate: PayoutStateCandidate | None = None - attempt_trusted = True - # The memo entry this attempt may arm must prove state for the - # epoch its reads happen in; a detection cycle during the pass - # (away, or away and back) refuses the arm in - # _note_reorg_reconcile_outcome. - proof_epoch = int(getattr(self, "tip_detection_epoch", 0)) - try: - with self._block_submitter_lock( - self._payout_state_prepare_lock, - "payout-state-prepare", - ): - prepared_started = time.monotonic() - captured_source = self._capture_payout_state_source() - payout_changed = False - payout_mutation_attempted = False - inactive_blocks = 0 - reactivated_blocks = 0 - matured_payouts = 0 - summary["untrusted"] = False - summary["watched_blocks"] = 0 - try: - if self.qbit_chain_view_untrusted(): - if not skip_recorded: - with self.lock: - self.reorg_reconcile_skip_count += 1 - skip_recorded = True - summary["untrusted"] = True - attempt_trusted = False - if _force_publish: - candidate_to_publish = ( - self._prepared_payout_state_candidate( - captured_source - ) - ) - else: - active_tip_height = int(self.rpc.call("getblockcount")) - watch_blocks = getattr( - self.ledger, - "reorg_watch_blocks", - None, - ) - if not callable(watch_blocks): - if ( - _force_publish - or self._captured_payout_source_requires_publication( - captured_source - ) - ): - candidate_to_publish = ( - self._prepared_payout_state_candidate( - captured_source, - force_full_window_rescan=payout_changed, - ) - ) - else: - rows = watch_blocks( - active_tip_height=active_tip_height - ) - summary["watched_blocks"] = len(rows) - - for row in rows: - block_height = int(row["block_height"]) - block_hash = str(row["block_hash"]).lower() - chain_state = str(row.get("chain_state", "")) - if block_height > active_tip_height: - if chain_state == "confirmed": - payout_mutation_attempted = True - inactive = ( - self.ledger.mark_pool_block_inactive( - block_hash=block_hash, - active_tip_height=active_tip_height, - ) - ) - inactive_count = int( - inactive.get("inactive_count", 0) - ) - inactive_blocks += inactive_count - payout_changed = ( - payout_changed - or bool(inactive_count) - ) - continue - active_hash = str( - self.rpc.call( - "getblockhash", - [block_height], - ) - ).lower() - on_active_chain = active_hash == block_hash - if ( - on_active_chain - and chain_state == "inactive" - ): - payout_mutation_attempted = True - reactivated = ( - self.ledger.reactivate_pool_block( - block_hash=block_hash, - active_tip_height=active_tip_height, - ) - ) - reactivated_count = int( - reactivated.get( - "reactivated_count", - 0, - ) - ) - reactivated_blocks += reactivated_count - payout_changed = ( - payout_changed - or bool(reactivated_count) - ) - elif ( - not on_active_chain - and chain_state == "confirmed" - ): - payout_mutation_attempted = True - inactive = ( - self.ledger.mark_pool_block_inactive( - block_hash=block_hash, - active_tip_height=active_tip_height, - ) - ) - inactive_count = int( - inactive.get("inactive_count", 0) - ) - inactive_blocks += inactive_count - payout_changed = ( - payout_changed - or bool(inactive_count) - ) - - mark_mature = getattr( - self.ledger, - "mark_mature_pool_payouts", - None, - ) - if callable(mark_mature): - payout_mutation_attempted = True - matured = mark_mature( - active_tip_height=active_tip_height - ) - matured_payouts = int( - matured.get("matured_count", 0) - ) - payout_changed = ( - payout_changed - or bool(matured_payouts) - ) - - inactive_blocks_total += inactive_blocks - reactivated_blocks_total += reactivated_blocks - matured_payouts_total += matured_payouts - if ( - payout_changed - or _force_publish - or self._captured_payout_source_requires_publication( - captured_source - ) - ): - # Candidate preparation embeds the ledger - # snapshot artifact; a pass that will not - # publish must not pay for one only to - # discard it. - candidate_to_publish = ( - self._prepared_payout_state_candidate( - captured_source, - force_full_window_rescan=payout_changed, - ) - ) - except Exception: - inactive_blocks_total += inactive_blocks - reactivated_blocks_total += reactivated_blocks - matured_payouts_total += matured_payouts - # Durable partial mutations close admission before the - # preparation lock is released. Publication drains old - # socket sends afterward without blocking new ledger - # preparation or snapshot acquisition. - if payout_mutation_attempted: - # A mutator can commit server-side and still raise - # locally when its response is lost. The observed - # row counts are then unavailable, so conservatively - # force the same ledger re-read as a confirmed - # mutation. Read-only failures never reach this flag. - payout_changed = True - if payout_changed: - error_candidate = ( - self._prepared_payout_state_candidate( - captured_source, - force_full_window_rescan=True, - ) - ) - self._block_payout_state_publication(force=True) - with self.lock: - self.reorg_inactive_block_count += ( - inactive_blocks_total - ) - self.reorg_reactivated_block_count += ( - reactivated_blocks_total - ) - self.matured_payout_count += matured_payouts_total - self.reorg_reconcile_error_count += 1 - # A pass that errored mid-mutation invalidates every - # cached outcome, not just its own tip's. - self._note_reorg_reconcile_outcome( - tip_hash, - trusted=False, - clear_memo=True, - ) - raise - finally: - self._observe_payout_state_seconds( - "preparation", - max(0.0, time.monotonic() - prepared_started), - ) - - if candidate_to_publish is not None: - # Atomically fence cache/build/delivery admission before - # releasing the ledger snapshot lock. The potentially - # slow drain then happens in publication() below. - self._block_payout_state_publication(force=True) - except Exception: - if error_candidate is not None: - if ( - self._publish_payout_state_candidate(error_candidate) - is None - ): - self._block_payout_state_publication() - raise - - if candidate_to_publish is not None: - published = self._publish_payout_state_candidate( - candidate_to_publish - ) - if published is None: - # Preserve durable counts and retry iteratively against the - # newest source. The explicit budget prevents tip churn - # from monopolizing preparation indefinitely; the fence - # stays closed between attempts. - if retry_superseded_candidate(): - continue - return finish(trusted=False) - summary["published_generation"] = published - return finish(trusted=attempt_trusted) def client_can_receive_jobs(self, client: ClientState) -> bool: - return ( - not getattr(client, "closing", False) - and client.subscribed - and client.authorized - and client.worker is not None - ) + return self._ensure_job_delivery_service().client_can_receive_jobs(client) def pool_readiness_latched(self) -> bool: """Latch, once, the transition past min_ready_miners. @@ -19067,136 +7808,9 @@ def client_needs_tip_template_refresh( client: ClientState, snapshot: QbitTipTemplateSnapshot, ) -> bool: - context = client.active_job - if context is None: - return True - if getattr(self, "tip_refresh_epoch_fanout", False): - current_epoch = int( - getattr(self, "_tip_refresh_epoch_sequence", 0) - ) - context_epoch = int( - getattr(context, "tip_refresh_epoch_sequence", 0) - ) - if context_epoch > current_epoch: - return False - if context_epoch < current_epoch: - return True - current_epoch_tip = getattr( - self, - "_tip_refresh_epoch_tip_hash", - None, - ) - if ( - current_epoch_tip is not None - and str(context.template.get("previousblockhash", "")) - != current_epoch_tip - ): - return True - if getattr(context, "collection_only", False) and getattr( - self, "_pool_ready_latched", False - ): - # The pool crossed min_ready_miners after this job was issued. - # A collection job keeps settling solved blocks solver-pays-all, - # so replace it with windowed work on the next poller pass. - return True - template = context.template - previousblockhash = str(template.get("previousblockhash", "")) - context_fingerprint = getattr(context, "template_fingerprint", None) - if context_fingerprint is None: - context_fingerprint = qbit_template_fingerprint(template) - context_payout_generation = int( - getattr(context, "payout_state_generation", 0) - ) - context_payout_artifact_sha256 = getattr( - context, - "payout_artifact_sha256", - None, - ) - published_payout_state = getattr(self, "_published_payout_state", None) - published_payout_artifact = ( - published_payout_state.artifact - if published_payout_state is not None - else None - ) - active_needs_refresh = ( - previousblockhash != snapshot.bestblockhash - or previousblockhash != snapshot.previousblockhash - or context_fingerprint != snapshot.template_fingerprint - or context_payout_generation - != int(getattr(self, "_payout_state_generation", 0)) - # A self-check repair swaps the published balance snapshot in - # place: the generation and append epoch both survive, so only - # the digest identifies an active job still mining the refuted - # balances. Jobs stamped before the digest existed (or while no - # artifact is published mid-publication) defer to the generation - # fence above. - or ( - context_payout_artifact_sha256 is not None - and published_payout_artifact is not None - and context_payout_artifact_sha256 - != published_payout_artifact.prior_balances_sha256 - ) - # A late-visible append advances only the epoch: the generation - # and digest survive, yet the job's window omitted the late row - # (possibly a replayed winning share). Collection jobs settle - # solver-pays-all and are exempt, as at every other epoch fence. - or ( - not getattr(context, "collection_only", False) - and int( - getattr(context, "payout_append_invalidation_epoch", 0) - ) - != int( - getattr( - self, - "_payout_ledger_append_invalidation_epoch", - 0, - ) - ) - ) - ) - if active_needs_refresh: - return True - if not getattr(self, "tip_refresh_epoch_fanout", False): - return False - delivered = getattr(client, "_progress_delivered_context", None) - if delivered is None: - return True - delivered_fingerprint = getattr( - delivered, - "template_fingerprint", - None, - ) - if delivered_fingerprint is None: - delivered_fingerprint = qbit_template_fingerprint( - delivered.template - ) - # Reselection must cover every condition the epoch fixpoint counts - # as undelivered, or the owning wave re-enters forever with no - # candidate to serve. A delivered collection job after the pool - # latches ready is one such condition even when its epoch, tip, - # payout generation, and fingerprint all remain current, because - # registration can advance past a delivery whose send failed. - # A client whose current-epoch delivery is registered but unproven - # is deliberately reselected here (the fence admits equal epochs, - # so the re-serve is idempotent on the wire). Deduplicating on the - # admitted epoch instead would be wrong: admission is monotonic - # and survives a failed send, so an admitted-equal skip would - # deselect the client while the fixpoint still counts it - # undelivered. - return ( - int(getattr(delivered, "connection_id", 0)) - != int(client.connection_id) - or int(getattr(delivered, "tip_refresh_epoch_sequence", 0)) - != current_epoch - or str(delivered.template.get("previousblockhash", "")) - != current_epoch_tip - or int(getattr(delivered, "payout_state_generation", 0)) - != int(getattr(self, "_tip_refresh_epoch_payout_generation", 0)) - or delivered_fingerprint != snapshot.template_fingerprint - or ( - bool(getattr(delivered, "collection_only", False)) - and bool(getattr(self, "_pool_ready_latched", False)) - ) + return self._ensure_job_delivery_service().client_needs_tip_template_refresh( + client, + snapshot, ) def intervening_job_supersedes_snapshot( @@ -19205,540 +7819,69 @@ def intervening_job_supersedes_snapshot( expected_active_job: PrismJobContext | None, snapshot: QbitTipTemplateSnapshot, ) -> bool: - if active_job is expected_active_job or active_job is None: - return False - active_payout_generation = int( - getattr(active_job, "payout_state_generation", 0) - ) - if active_payout_generation < int( - getattr(self, "_payout_state_generation", 0) - ): - # Template ordering cannot make a payout-stale intervening job - # authoritative. Let the reconciled refresh replace it. - return False - active_parent_hash = str( - getattr(active_job, "template", {}).get("previousblockhash", "") + return self._ensure_job_delivery_service().intervening_job_supersedes_snapshot( + active_job, + expected_active_job, + snapshot, ) - if ( - active_parent_hash != snapshot.bestblockhash - or active_parent_hash != snapshot.previousblockhash - ): - # Artifact generations order fetch starts, not chain tips. A fetch - # for the old tip can start after this exact new-tip observation - # and therefore carry a larger generation; it must not prevent - # the new-tip snapshot from replacing that stale work. - return False - active_generation = int(getattr(active_job, "template_generation", 0)) - snapshot_generation = int(getattr(snapshot, "template_generation", 0)) - if active_generation <= 0 or snapshot_generation <= 0: - # Legacy/test contexts without ordering metadata retain the safe - # behavior: never overwrite an unclassified intervening job. - return True - return active_generation >= snapshot_generation def client_tip_changed_for_snapshot( self, client: ClientState, snapshot: QbitTipTemplateSnapshot, ) -> bool: - context = client.active_job - if context is None: - return True - previousblockhash = str(context.template.get("previousblockhash", "")) - context_payout_artifact_sha256 = getattr( - context, - "payout_artifact_sha256", - None, - ) - published_payout_state = getattr(self, "_published_payout_state", None) - published_payout_artifact = ( - published_payout_state.artifact - if published_payout_state is not None - else None - ) - return ( - previousblockhash != snapshot.bestblockhash - or previousblockhash != snapshot.previousblockhash - or int(getattr(context, "payout_state_generation", 0)) - != int(getattr(self, "_payout_state_generation", 0)) - # A refresh selected because a self-check repair replaced the - # published balance digest must also retire the refuted job: a - # clean_jobs=False replacement leaves the old job ID admissible, - # so a late block submission could still commit the refuted - # allocation. Same None guards as the reselection predicate. - or ( - context_payout_artifact_sha256 is not None - and published_payout_artifact is not None - and context_payout_artifact_sha256 - != published_payout_artifact.prior_balances_sha256 - ) - # Likewise for append-epoch skew: the replacement window covers - # the late-visible row, so the job mining the pre-append window - # must not stay admissible beside it. - or ( - not getattr(context, "collection_only", False) - and int( - getattr(context, "payout_append_invalidation_epoch", 0) - ) - != int( - getattr( - self, - "_payout_ledger_append_invalidation_epoch", - 0, - ) - ) - ) + return self._ensure_job_delivery_service().client_tip_changed_for_snapshot( + client, + snapshot, ) def handle_client(self, client: ClientState) -> None: - reader = None - try: - reader = client.sock.makefile("r", encoding="utf-8", newline="\n") - for line in reader: - if self.stop_event.is_set(): - break - line = line.strip() - if not line: - continue - client.request_received_monotonic = time.monotonic() - request_id: object = None - request: object = None - try: - request = json.loads(line) - if not isinstance(request, dict): - raise StratumError(20, "request must be an object") - request_id = request.get("id") - self.handle_request(client, request) - except json.JSONDecodeError as exc: - self.send_error(client, request_id, 20, f"invalid JSON: {exc.msg}") - except StratumError as exc: - self.send_error(client, request_id, exc.code, exc.message, reason=exc.reason) - received_monotonic = client.request_received_monotonic - if ( - received_monotonic is not None - and isinstance(request, dict) - and request.get("method") == "mining.submit" - ): - # Symmetric with the accepted-path observation after - # send_result: both sides include the response write, - # so the accepted-vs-rejected comparison separates - # commit pressure from transport/thread saturation. - self._observe_share_ack_seconds( - "rejected", - time.monotonic() - received_monotonic, - ) - if exc.disconnect: - break - except Exception: - print( - f"prism coordinator: client thread failed address={client.address}", - flush=True, - ) - traceback.print_exc() - break - except (OSError, ValueError) as exc: - if isinstance(exc, OSError) and exc.errno in {errno.EMFILE, errno.ENFILE}: - self._record_stratum_resource_exhaustion( - listener_name=client.listener_name, - location="client-reader", - error_number=exc.errno, - ) - print( - "prism coordinator: stratum client socket failed " - f"address={client.address} error={exc!r}", - flush=True, - ) - finally: - try: - if reader is not None: - reader.close() - except (OSError, ValueError): - pass - finally: - self.disconnect_client(client) - with self.lock: - self._ensure_initial_job_state() - if getattr(client, "handler_thread_registered", False): - client.handler_thread_registered = False - self.handler_thread_count = max( - 0, - self.handler_thread_count - 1, - ) + self._ensure_stratum_session_service().handle_client(client) def disconnect_client(self, client: ClientState) -> None: # Retire admission and fanout eligibility without waiting behind job # delivery. Only the first caller owns socket close and final cleanup. - with self.lock: - # The timeout sweeper marks closing while leaving membership in - # place as its atomic handoff token. Whichever disconnect caller - # first removes that membership owns socket close and cleanup. - if getattr(client, "closing", False) and client not in self.clients: - return - client.closing = True - self.clients.discard(client) - self._cancel_pending_initial_job_locked(client, count=True) - - # Do not take send_lock here: shutdown must interrupt an in-flight - # sendall as well as the handler's blocking reader. - try: - client.close() - finally: - # Every mixed lock path uses job_update_lock -> coordinator lock. - # Retirement above holds neither while this potentially waits. - with client.job_update_lock: - with self.lock: - self._ensure_evicted_job_state() - retention_cap = int( - getattr( - self, - "disconnected_job_retention", - DEFAULT_PRISM_DISCONNECTED_JOB_RETENTION, - ) - ) - if retention_cap > 0: - # Devices behind a flapping proxy keep mining the - # jobs this connection delivered; bury the active - # ones so their in-flight shares can resume against - # the retained context after the reconnect. - for job_id in tuple(client.active_job_ids): - self.bury_evicted_job(client, job_id, prune=False) - for job_id in tuple(client.active_job_ids): - self.jobs.pop(job_id, None) - client.active_job_ids.clear() - client.active_job = None - for job_id in tuple( - self.evicted_jobs_by_connection.get(client.connection_id, ()) - ): - entry = self.evicted_job_graveyard.get(job_id) - if entry is None: - continue - if ( - retention_cap <= 0 - or self._evicted_job_class_locked(entry) != "same_tip" - ): - # Stale-grace work stays tied to the connection - # that received it (its expiry anchors on that - # connection's delivered work); only fully valid - # same-tip contexts survive the disconnect. - self._remove_evicted_job_locked(job_id) - continue - # Detach from the dead connection object: expiry then - # anchors on the entry's own eviction time (same-tip - # TTL) or the published tip flip, never on delivery - # state this connection can no longer advance. - self.evicted_job_graveyard[job_id] = dataclasses.replace( - entry, - client=None, - ) - self._disconnected_evicted_job_ids[job_id] = None - self._enforce_disconnected_evicted_capacity_locked() - client.authorized = False - client.worker = None - client.username = "" - self._retain_current_collection_refresh_if_unrepresented() + self._ensure_stratum_session_service().disconnect_client(client) def handle_request(self, client: ClientState, request: dict[str, object]) -> None: """Dispatch one request, translating shutdown races to Stratum errors.""" - try: - self._handle_request(client, request) - except ShutdownInProgress as exc: - # A request can pass the initial shutdown check immediately before - # writer admission closes. Preserve the normal protocol response - # instead of surfacing a generic client-thread failure. - raise StratumError( - 20, - "coordinator is shutting down", - reason=PRISM_REJECTION_POOL_CLOSED, - disconnect=True, - ) from exc + self._ensure_stratum_session_service().handle_request(client, request) def _handle_request(self, client: ClientState, request: dict[str, object]) -> None: - if self.stop_event.is_set() or self._ensure_shutdown_controller().phase != "running": - raise StratumError( - 20, - "coordinator is shutting down", - reason=PRISM_REJECTION_POOL_CLOSED, - disconnect=True, - ) - method = request.get("method") - params = request.get("params", []) - request_id = request.get("id") - if not isinstance(method, str): - raise StratumError(20, "missing method") - if not isinstance(params, list): - raise StratumError(20, "params must be an array") - - if method == "mining.configure": - self.handle_configure(client, request_id, params) - return - if method == "mining.subscribe": - with client.job_update_lock: - client.subscribed = True - self.send_result( - client, - request_id, - [[], client.extranonce1_hex, self.extranonce2_size], - ) - self._note_collection_identity_available(client) - needs_initial_job = client.authorized - if needs_initial_job: - self.request_initial_job_delivery(client) - return - if method == "mining.authorize": - username = str(params[0]) if params else "" - password = str(params[1]) if len(params) > 1 and params[1] is not None else "" - # Address validation may use RPC; it is unrelated to client job - # state and therefore stays outside the job-update lock. - worker = self.resolve_worker(username) - with client.job_update_lock: - was_authorized = client.authorized - if was_authorized: - with self.lock: - self._ensure_initial_job_state() - if ( - client not in self.pending_initial_jobs - and len(self.pending_initial_jobs) - >= self.stratum_max_pending_initial_jobs - and self._client_has_current_tip_job_locked(client) - ): - # Reject before mutating identity or difficulty. A - # working session keeps its current authorization - # when there is no live first-job slot for the - # superseding generation. - raise StratumError( - 20, - "initial job delivery capacity unavailable", - disconnect=False, - ) - if not self.reserve_client_username(client, worker): - raise StratumError( - 20, - "too many connections for username", - # A new connection has no useful session to preserve. A - # live miner re-authorizing to a full username does: keep - # its prior worker/session active after returning the - # capacity error. - disconnect=not client.authorized, - ) - # The password is authoritative for password-derived options: a - # re-authorize without d=/md= clears any prior override (a stored - # suggest_difficulty still applies via the request resolution). - with self._client_vardiff_lock(client): - client.requested_difficulty, client.requested_min_difficulty = ( - parse_stratum_password_options(password) - ) - target = self.apply_client_difficulty_requests(client) - if target is not None: - current = client.pending_share_difficulty or client.share_difficulty - if target != current: - if not was_authorized: - client.share_difficulty = target - client.pending_share_difficulty = None - else: - client.pending_share_difficulty = target - client.difficulty_generation = int( - getattr(client, "difficulty_generation", 0) - ) + 1 - client.authorization_generation = int( - getattr(client, "authorization_generation", 0) - ) + 1 - client.authorized = True - client.authorized_monotonic = time.monotonic() - self.send_result(client, request_id, True) - self._note_collection_identity_available(client) - # Exactly one coalesced delivery represents this authorization, - # including a password-derived difficulty change. - self.request_initial_job_delivery(client) - return - if method == "mining.extranonce.subscribe": - self.send_result(client, request_id, True) - return - if method == "mining.suggest_difficulty": - self.handle_suggest_difficulty(client, request_id, params) - return - if method == "mining.submit": - received_monotonic = getattr( - client, - "request_received_monotonic", - None, - ) - accepted_and_closed = self.handle_submit(client, params) - try: - self.send_result(client, request_id, True) - if received_monotonic is not None: - # Rejects are observed symmetrically in handle_client - # after their error response write. - self._observe_share_ack_seconds( - "accepted", - time.monotonic() - received_monotonic, - ) - finally: - self.refresh_jobs_after_pending_accepted_block(client) - if accepted_and_closed: - client.close() - return - raise StratumError(20, f"unsupported method {method}") + self._ensure_stratum_session_service()._handle_request(client, request) def handle_suggest_difficulty(self, client: ClientState, request_id: object, params: list[object]) -> None: - with client.job_update_lock: - suggested: Decimal | None = None - if params: - try: - suggested = Decimal(str(params[0])) - except Exception: - suggested = None - if suggested is not None and (not suggested.is_finite() or suggested <= 0): - suggested = None - if suggested is not None: - with self._client_vardiff_lock(client): - client.suggested_difficulty = suggested - target = self.apply_client_difficulty_requests(client) - if target is not None: - self.advertise_client_difficulty(client, target) - self.send_result(client, request_id, True) + self._ensure_stratum_session_service().handle_suggest_difficulty( + client, request_id, params + ) def handle_configure(self, client: ClientState, request_id: object, params: list[object]) -> None: - extensions = params[0] if params else [] - extension_params = params[1] if len(params) > 1 and isinstance(params[1], dict) else {} - result: dict[str, object] = {} - if isinstance(extensions, list): - for extension in extensions: - if extension == "version-rolling": - miner_mask = 0xFFFFFFFF - if "version-rolling.mask" in extension_params: - miner_mask = stratum_codec.parse_mask_hex( - extension_params["version-rolling.mask"], - field_name="version-rolling.mask", - ) - client.version_mask = self.version_mask & miner_mask - result["version-rolling"] = client.version_mask != 0 - result["version-rolling.mask"] = stratum_codec.format_mask_hex(client.version_mask) - else: - result[str(extension)] = False - self.send_result(client, request_id, result) - - def send_result(self, client: ClientState, request_id: object, result: object) -> None: - client.send({"id": request_id, "result": result, "error": None}) - - def send_error(self, client: ClientState, request_id: object, code: int, message: str, *, reason: str | None = None) -> None: - data = {"reason_id": reason} if reason is not None else None - client.send({"id": request_id, "result": None, "error": [code, message, data]}) - - def resolve_worker(self, username: str) -> WorkerIdentity: - payout_address, worker_name = split_worker_username(username) - try: - if not payout_address: - raise StratumError(20, "username base is empty") - script, p2mr_program_hex = self.validate_p2mr_address(payout_address, label="username base") - except StratumError as username_error: - fallback_address = getattr(self, "username_fallback_address", default_prism_username_fallback_address()) - if fallback_address is None: - raise username_error - print( - f"prism coordinator: username {username!r} cannot be used as a payout " - f"({username_error.message}); using fallback payout {fallback_address}", - flush=True, - ) - payout_address = fallback_address - script, p2mr_program_hex = self.validate_p2mr_address( - fallback_address, - label="PRISM_USERNAME_FALLBACK_ADDRESS", - ) - return WorkerIdentity( - username=username, - payout_address=payout_address, - worker_name=worker_name, - script_pubkey_hex=script, - p2mr_program_hex=p2mr_program_hex, + self._ensure_stratum_session_service().handle_configure( + client, request_id, params ) - def validate_p2mr_address(self, address: str, *, label: str) -> tuple[str, str]: - self._ensure_p2mr_address_cache_state() - with self._p2mr_address_cache_lock: - cached = self._p2mr_address_cache.get(address) - if cached is not None: - expires_monotonic, cached_result = cached - if expires_monotonic > time.monotonic(): - self._p2mr_address_cache.move_to_end(address) - return cached_result - self._p2mr_address_cache.pop(address, None) - pending = self._p2mr_address_validation_inflight.get(address) - is_leader = pending is None - if pending is None: - pending = _P2mrAddressValidationFlight() - self._p2mr_address_validation_inflight[address] = pending - else: - pending.waiters += 1 - - if not is_leader: - pending.event.wait() - if pending.result is not None: - return pending.result - if pending.error is not None: - self._raise_shared_p2mr_address_validation_error(pending.error) - raise RuntimeError("payout address validation completed without a result") + def send_result(self, client: ClientState, request_id: object, result: object) -> None: + client.send(stratum_result_payload(request_id, result)) - try: - validation = self.rpc.call("validateaddress", [address]) - if not isinstance(validation, dict) or not validation.get("isvalid"): - raise StratumError(20, f"{label} is not a valid qbit address: {address}") - script = str(validation.get("scriptPubKey") or "") - if not script.startswith("5220") or len(script) != 68: - raise StratumError(20, f"{label} does not resolve to a P2MR script: {address}") - result = (script, script[4:]) - with self._p2mr_address_cache_lock: - max_entries = int( - getattr( - self, - "payout_address_cache_max_entries", - DEFAULT_PRISM_PAYOUT_ADDRESS_CACHE_MAX_ENTRIES, - ) - ) - ttl_seconds = float( - getattr( - self, - "payout_address_cache_ttl_seconds", - DEFAULT_PRISM_PAYOUT_ADDRESS_CACHE_TTL_SECONDS, - ) - ) - if max_entries > 0 and ttl_seconds > 0: - self._p2mr_address_cache[address] = ( - time.monotonic() + ttl_seconds, - result, - ) - self._p2mr_address_cache.move_to_end(address) - while len(self._p2mr_address_cache) > max_entries: - self._p2mr_address_cache.popitem(last=False) - pending.result = result - return result - except BaseException as exc: - with self._p2mr_address_cache_lock: - pending.error = exc - raise - finally: - with self._p2mr_address_cache_lock: - if self._p2mr_address_validation_inflight.get(address) is pending: - self._p2mr_address_validation_inflight.pop(address, None) - pending.event.set() + def send_error(self, client: ClientState, request_id: object, code: int, message: str, *, reason: str | None = None) -> None: + client.send(stratum_error_payload(request_id, code, message, reason=reason)) + + def resolve_worker(self, username: str) -> WorkerIdentity: + return self._ensure_stratum_session_service().resolve_worker(username) + + def validate_p2mr_address(self, address: str, *, label: str) -> tuple[str, str]: + return self._ensure_stratum_session_service().address_validator.validate( + address, label=label + ) @staticmethod def _raise_shared_p2mr_address_validation_error(error: BaseException) -> None: - if isinstance(error, StratumError): - raise StratumError( - error.code, - error.message, - reason=error.reason, - disconnect=error.disconnect, - ) from error - raise RuntimeError(str(error)) from error + P2mrAddressValidator._raise_shared_error(error) def _ensure_p2mr_address_cache_state(self) -> None: - if not hasattr(self, "_p2mr_address_cache_lock"): - self._p2mr_address_cache_lock = threading.Lock() - if not hasattr(self, "_p2mr_address_cache"): - self._p2mr_address_cache = OrderedDict() - if not hasattr(self, "_p2mr_address_validation_inflight"): - self._p2mr_address_validation_inflight = {} + # The validator (created on first touch) owns the cache trio; legacy + # assignments through the descriptor-routed names are adopted by it. + self._ensure_stratum_session_service() def maybe_send_job( self, @@ -19750,15 +7893,14 @@ def maybe_send_job( tip_refresh_snapshot: QbitTipTemplateSnapshot | None = None, tip_refresh_observation_sequence: int | None = None, ) -> bool: - with client.job_update_lock: - return self._maybe_send_job_locked( - client, - clean_jobs=clean_jobs, - raise_on_reorg_failure=raise_on_reorg_failure, - raise_on_build_failure=raise_on_build_failure, - tip_refresh_snapshot=tip_refresh_snapshot, - tip_refresh_observation_sequence=tip_refresh_observation_sequence, - ) + return self._ensure_job_delivery_service().maybe_send_job( + client, + clean_jobs=clean_jobs, + raise_on_reorg_failure=raise_on_reorg_failure, + raise_on_build_failure=raise_on_build_failure, + tip_refresh_snapshot=tip_refresh_snapshot, + tip_refresh_observation_sequence=tip_refresh_observation_sequence, + ) def _maybe_send_job_locked( self, @@ -19774,413 +7916,21 @@ def _maybe_send_job_locked( commit_guard_lock: threading.RLock | None = None, prepared_bundle_allow_uncached: bool = False, ) -> bool: - if not client.subscribed or not client.authorized or client.worker is None: - return False - self._ensure_job_cache_state() - started = time.monotonic() - phases = self._job_build_phases() - phases.clear() - if getattr(self, "hot_path_log_enabled", False): - print( - f"prism coordinator: building job connection={client.connection_id} username={client.username}", - flush=True, - ) - phase_started = time.monotonic() - guarded_refresh = tip_refresh_snapshot is not None - if guarded_refresh != (tip_refresh_observation_sequence is not None): - raise ValueError("tip refresh snapshot and observation sequence must be paired") - if prepared_bundle is not None and guarded_refresh: - raise ValueError("prepared idle bundles cannot be combined with tip refresh guards") - if prepared_bundle_allow_uncached and prepared_bundle is None: - raise ValueError("uncached prepared delivery requires a prepared bundle") - if prepared_bundle is not None: - pass - elif guarded_refresh: - assert tip_refresh_snapshot is not None - assert tip_refresh_observation_sequence is not None - with self.lock: - refresh_current = self._tip_refresh_snapshot_current_locked( - tip_refresh_snapshot, - tip_refresh_observation_sequence, - ) - if not refresh_current: - self._schedule_tip_refresh_retry() - raise TemplateRefreshSuperseded( - "tip refresh snapshot was superseded before client job build" - ) - try: - chain_view_untrusted = bool( - getattr(self, "reorg_reconciler_enabled", True) - and self.qbit_chain_view_untrusted() - ) - except Exception as exc: - self._schedule_tip_refresh_retry() - raise _TipRefreshTrustBlocked( - "qbit chain trust check failed before sequential client job build" - ) from exc - if chain_view_untrusted: - self._schedule_tip_refresh_retry() - raise _TipRefreshTrustBlocked( - "qbit chain view became untrusted before sequential client job build" - ) - else: - try: - if not self.ensure_reorg_reconciled_for_current_tip(): - if raise_on_reorg_failure: - raise _TipRefreshTrustBlocked( - "qbit chain view became untrusted before client job build" - ) - return False - except TemplateRefreshBlocked: - raise - except Exception as exc: - print( - f"prism coordinator: reorg reconciliation failed before job build " - f"connection={client.connection_id} username={client.username}; skipping this job", - flush=True, - ) - traceback.print_exc() - if raise_on_reorg_failure: - raise TemplateRefreshBlocked( - "reorg reconciliation failed before client job build" - ) from exc - return False - phases["reorg"] = time.monotonic() - phase_started - built_from_guarded_artifacts = bool( - guarded_refresh - and tip_refresh_snapshot.template_artifacts is not None - and "build_job_for_client" not in self.__dict__ - ) - try: - if prepared_bundle is not None: - context = self.stamp_job_for_client( - client, - prepared_bundle, - clean_jobs=clean_jobs, - ) - elif built_from_guarded_artifacts: - assert tip_refresh_snapshot is not None - assert tip_refresh_snapshot.template_artifacts is not None - context = self.build_job_for_client_from_artifacts( - client, - tip_refresh_snapshot.template_artifacts, - clean_jobs=clean_jobs, - publication_critical=True, - request_source="tip_refresh", - ) - else: - context = self.build_job_for_client(client, clean_jobs=clean_jobs) - except TemplateRefreshBlocked: - self._schedule_tip_refresh_retry() - if guarded_refresh or raise_on_reorg_failure or raise_on_build_failure: - raise - return False - except Exception as exc: - # A single bad template (e.g. a coinbase whose bytes collide with the - # extranonce placeholder, or a transient getblocktemplate failure) must - # never tear down the miner's connection. Log it, count it, and skip - # this job; the next share/retarget or block change rebuilds a fresh one. - # Only the build is isolated: nothing has been registered or sent yet, so - # there is no stale job state. Downstream send failures still surface to - # handle_client, which disconnects the (now dead) socket and cleans up. - with self.lock: - self.job_build_failure_count += 1 - print( - f"prism coordinator: job build failed connection={client.connection_id} " - f"username={client.username}; keeping client connected and skipping this template", - flush=True, - ) - traceback.print_exc() - if raise_on_build_failure: - raise _JobBuildFailed( - f"job build failed for connection {client.connection_id}" - ) from exc - return False - # Linearize direct delivery against the immutable publication pointer. - # Expensive build and ledger reads happened under the preparation lock, - # outside this admission boundary. - with self._job_cache_lock: - current_payout_generation = self._payout_state_generation - current_append_invalidation_epoch = ( - self._payout_ledger_append_invalidation_epoch - ) - published_tip = self._published_payout_state.source_tip_hash - publication_blocked = self._payout_state_publication_blocked - context_payout_generation = int( - getattr( - context, - "payout_state_generation", - current_payout_generation, - ) - ) - context_append_invalidation_epoch = int( - getattr(context, "payout_append_invalidation_epoch", 0) - ) - context_collection_only = bool( - getattr(context, "collection_only", False) - ) - context_payout_artifact_sha256 = getattr( - context, - "payout_artifact_sha256", - None, - ) - - def context_payout_digest_stale() -> bool: - # A periodic self-check repair swaps the published balance - # snapshot in place: the payout generation and append epoch both - # survive the swap, so only the digest identifies a context still - # keyed to the refuted balances. Contexts without a digest defer - # to the generation fence. Unlocked at the gate (a cancellation - # hint); authoritative when re-read under _job_cache_lock at the - # commit boundary below. - if context_payout_artifact_sha256 is None: - return False - published_artifact = self._published_payout_state.artifact - return ( - published_artifact is None - or context_payout_artifact_sha256 - != published_artifact.prior_balances_sha256 - ) - - context_template = getattr(context, "template", None) - context_parent = ( - str(context_template.get("previousblockhash", "")) - if isinstance(context_template, dict) - else "" - ) - with self.lock: - published_authority = getattr(self, "current_tip_first_seen", None) - published_authoritative = self._published_tip_authoritative_locked( - time.monotonic() - ) - pinned_published_delivery = bool( - context_parent - and published_authority is not None - and context_parent == published_authority[0] - and published_authoritative - ) - lapsed_live_validated = False - if ( - not guarded_refresh - and context_parent - and published_authority is not None - and not published_authoritative - ): - # The published authority lapsed, so per-share classification has - # fallen back to the live RPC read. Mirror the initial-job path: - # revalidate against the live tip (outside every lock), record - # the observation for the refresh machinery, and drop work that - # would be stale on arrival. - try: - lapsed_live_tip = str(self.rpc.call("getbestblockhash")) - except Exception: - lapsed_live_tip = None - if lapsed_live_tip is not None: - self.observe_tip_for_refresh(lapsed_live_tip) - if context_parent != lapsed_live_tip: - self._schedule_tip_refresh_retry() - return False - lapsed_live_validated = True - priority_delivery = ( - not publication_blocked - and context_payout_generation == current_payout_generation - and ( - context_collection_only - or context_append_invalidation_epoch - == current_append_invalidation_epoch - ) - and ( - published_tip is None - or context_parent == published_tip - # Reconciliation can source payout state at the detected tip - # before submit authority flips. Pinned published-tip work is - # still the currently creditable work, so it must keep the - # priority lane instead of tripping the gate's non-priority - # same-generation rejection for the whole unpublished window. - or pinned_published_delivery - ) + return self._ensure_job_delivery_service()._maybe_send_job_locked( + client, + clean_jobs=clean_jobs, + raise_on_reorg_failure=raise_on_reorg_failure, + raise_on_build_failure=raise_on_build_failure, + tip_refresh_snapshot=tip_refresh_snapshot, + tip_refresh_observation_sequence=tip_refresh_observation_sequence, + prepared_bundle=prepared_bundle, + commit_guard=commit_guard, + commit_guard_lock=commit_guard_lock, + prepared_bundle_allow_uncached=prepared_bundle_allow_uncached, ) - payout_gate_started = time.monotonic() - with self._payout_state_delivery_gate.delivery_cancelable( - lambda: ( - context_payout_generation != self._payout_state_generation - or context_payout_digest_stale() - or ( - not context_collection_only - and context_append_invalidation_epoch - != self._payout_ledger_append_invalidation_epoch - ) - ), - generation=context_payout_generation, - priority=priority_delivery, - ) as payout_admitted: - payout_gate_wait = max(0.0, time.monotonic() - payout_gate_started) - phases["payout_gate"] = phases.get("payout_gate", 0.0) + payout_gate_wait - self._observe_payout_gate_admission( - payout_admitted, - generation=context_payout_generation, - fallback_wait_seconds=payout_gate_wait, - ) - if not payout_admitted: - self._schedule_tip_refresh_retry() - if guarded_refresh: - raise TemplateRefreshSuperseded( - "payout state changed during client job build" - ) - return False - def commit_context_locked() -> bool: - if getattr(client, "closing", False): - return False - if commit_guard is not None and not commit_guard(): - return False - if guarded_refresh: - assert tip_refresh_snapshot is not None - assert tip_refresh_observation_sequence is not None - if not self._tip_refresh_snapshot_current_locked( - tip_refresh_snapshot, - tip_refresh_observation_sequence, - ): - self._schedule_tip_refresh_retry() - raise TemplateRefreshSuperseded( - "tip refresh snapshot was superseded during client job build" - ) - artifacts = tip_refresh_snapshot.template_artifacts - if built_from_guarded_artifacts and artifacts is not None and ( - context.template is not artifacts.template - or context.template_fingerprint != artifacts.fingerprint - or context.template_generation != artifacts.generation - ): - raise TemplateRefreshBlocked( - "client job build did not use the guarded refresh artifacts" - ) - else: - published_now = getattr(self, "current_tip_first_seen", None) - if published_now is None: - # Bootstrap: nothing published yet, so there is no - # authority for this work to contradict. - pass - elif self._published_tip_authoritative_locked( - time.monotonic() - ): - if context_parent and context_parent != published_now[0]: - # Authority moved while this direct build waited - # on the payout gate or client lock. Sending now - # would advertise work that classifies stale-job - # on arrival; skip and let the refresh fanout (or - # the pending retry) deliver current work. - self._schedule_tip_refresh_retry() - return False - elif context_parent and not lapsed_live_validated: - # The lease lapsed during the wait and this context - # never passed a live-tip revalidation: cached - # observations may be blind to the tip mining.submit - # now classifies against. Defer to a fresh pass whose - # pre-wait live check settles it without holding the - # coordinator lock across an RPC. - self._schedule_tip_refresh_retry() - return False - if not self._admit_client_tip_refresh_epoch_locked( - client, - int(getattr(context, "tip_refresh_epoch_sequence", 0)), - ): - self._schedule_tip_refresh_retry() - return False - client.active_job = context - if clean_jobs: - for job_id in client.active_job_ids: - self.bury_evicted_job(client, job_id, prune=False) - self.jobs.pop(job_id, None) - client.active_job_ids.clear() - self.prune_evicted_job_graveyard(force=False) - self.jobs[context.job.job_id] = context - client.active_job_ids.add(context.job.job_id) - self.prune_client_active_jobs(client) - return True - - def commit_context() -> bool: - if prepared_bundle is not None: - # Exact cache identity and the client/window guard are one - # commit boundary. A tip or payout publication that wins - # this race makes the idle task a no-op instead of - # delivering stale work. - with self._job_cache_lock: - if not self._idle_bundle_current_locked( - client, - prepared_bundle, - allow_uncached=prepared_bundle_allow_uncached, - ): - return False - with self.lock: - return commit_context_locked() - with self._job_cache_lock: - if context_payout_digest_stale() or ( - not context_collection_only - and context_append_invalidation_epoch - != self._payout_ledger_append_invalidation_epoch - ): - return False - with self.lock: - return commit_context_locked() - - if commit_guard_lock is None: - committed = commit_context() - else: - # Per-client state is acquired before coordinator admission. - # A busy share can delay only this delivery; it cannot hold - # self.lock while the delivery waits for Vardiff state. - with commit_guard_lock: - committed = commit_context() - if not committed: - return False - phase_started = time.monotonic() - self.send_job_update(client, context.job) - payout_admitted.mark_delivered() - self.apply_job_difficulty(client, context.job) - self.note_tip_work_delivered(client, str(context.template["previousblockhash"])) - delivered_monotonic = time.monotonic() - self._record_first_payout_delivery( - context_payout_generation, - delivered_monotonic, - ) - self._record_progress_delivery( - client, - context, - delivered_monotonic, - ) - self._consume_retained_collection_refresh(context) - self.note_initial_job_delivered( - client, - validated_current=guarded_refresh, - ) - phases["send"] = delivered_monotonic - phase_started - elapsed = time.monotonic() - started - self.observe_job_build_elapsed(elapsed, phases) - if getattr(self, "hot_path_log_enabled", False): - phase_report = ",".join( - f"{phase}:{phases[phase]:.3f}" - for phase in PRISM_JOB_BUILD_PHASES - if phase in phases - ) - print( - f"prism coordinator: sent job connection={client.connection_id} username={client.username} " - f"job={context.job.job_id} collection={context.collection_only} elapsed={elapsed:.3f}s " - f"phases={phase_report}", - flush=True, - ) - return True def prune_client_active_jobs(self, client: ClientState) -> None: - for job_id in tuple(client.active_job_ids): - if job_id not in self.jobs: - client.active_job_ids.discard(job_id) - ordered_active_job_ids = [ - job_id for job_id in self.jobs if job_id in client.active_job_ids - ] - while len(ordered_active_job_ids) > MAX_ACTIVE_PRISM_JOBS_PER_CLIENT: - oldest_job_id = ordered_active_job_ids.pop(0) - client.active_job_ids.remove(oldest_job_id) - self.bury_evicted_job(client, oldest_job_id) - self.jobs.pop(oldest_job_id, None) + return self._ensure_job_delivery_service().prune_client_active_jobs(client) def send_difficulty(self, client: ClientState, job: direct_stratum.DirectQbitStratumJob) -> None: self.send_difficulty_value(client, job.share_difficulty) @@ -20190,2071 +7940,367 @@ def send_difficulty_value(self, client: ClientState, difficulty: Decimal) -> Non @staticmethod def difficulty_payload(difficulty: Decimal) -> dict[str, object]: - return { - "id": None, - "method": "mining.set_difficulty", - "params": [float(difficulty)], - } + return stratum_difficulty_payload(difficulty) def client_vardiff_config(self, client: ClientState) -> vardiff.VardiffConfig: - """The difficulty policy for one client: its per-client specialization - if any, else its listener profile, else the default listener's config - (clients created without one: tests, legacy callers).""" - with self._client_vardiff_lock(client): - return client.vardiff_config or client.listener_vardiff_config or self.vardiff_config + return self._ensure_vardiff_service().client_config(client) def client_startup_difficulty(self, profile: StratumListenerProfile | None = None) -> Decimal: - config = profile.vardiff_config if profile is not None else self.vardiff_config - fixed_difficulty = profile.share_difficulty if profile is not None else self.share_difficulty - if not config.enabled: - return fixed_difficulty - return vardiff.clamp( - config.startup_difficulty, - config.min_difficulty, - config.max_difficulty, - ) + return self._ensure_vardiff_service().startup_difficulty(profile) def desired_client_share_difficulty(self, client: ClientState) -> Decimal: - # pending_share_difficulty is set by vardiff retargets and by explicit - # difficulty requests (d=/suggest_difficulty); either way it applies to - # the next stamped job regardless of whether vardiff is enabled. - with self._client_vardiff_lock(client): - return client.pending_share_difficulty or client.share_difficulty + return self._ensure_vardiff_service().desired_difficulty(client) def client_minimum_advertised_difficulty(self, client: ClientState) -> Decimal: - """The difficulty stamped jobs never advertise below for this client. - - Zero everywhere except floor-bearing listeners (the high-diff port), - where the effective policy floor governs: the listener minimum, raised - by any md= specialization. The floor overrides the network-difficulty - cap because the listener's marketplace contract is checked against the - first advertised difficulty, even while qbit network difficulty sits - below the floor. - """ - with self._client_vardiff_lock(client): - if client.minimum_advertised_difficulty <= 0: - return Decimal("0") - config = ( - client.vardiff_config - or client.listener_vardiff_config - or self.vardiff_config - ) - return max(client.minimum_advertised_difficulty, config.min_difficulty) - - def apply_job_difficulty(self, client: ClientState, job: direct_stratum.DirectQbitStratumJob) -> None: - with self._client_vardiff_lock(client): - config = ( - client.vardiff_config - or client.listener_vardiff_config - or self.vardiff_config - ) - if not config.enabled: - client.share_difficulty = job.share_difficulty - client.pending_share_difficulty = None - return - pending = client.pending_share_difficulty - client.share_difficulty = job.share_difficulty - if pending is not None and job.share_difficulty == pending: - client.pending_share_difficulty = None - - def apply_client_difficulty_requests(self, client: ClientState) -> Decimal | None: - """Specialize the client's difficulty policy from its recorded requests - (password ``d=``/``md=`` and ``mining.suggest_difficulty``), clamped to - the pristine listener bounds. The listener floor always wins: on a - high-diff listener no request can drop a client below the configured - minimum. Explicit ``d=`` outranks a suggestion. Returns the resolved - target difficulty, or None when the client requested nothing.""" - with self._client_vardiff_lock(client): - base = client.listener_vardiff_config or self.vardiff_config - requested = ( - client.requested_difficulty - if client.requested_difficulty is not None - else client.suggested_difficulty - ) - if requested is None and client.requested_min_difficulty is None: - # No live requests: drop any stale specialization so the client - # falls back to the pristine listener policy. - client.vardiff_config = None - return None - floor = base.min_difficulty - if client.requested_min_difficulty is not None: - floor = vardiff.clamp( - client.requested_min_difficulty, - base.min_difficulty, - base.max_difficulty, - ) - if requested is None: - requested = client.share_difficulty - target = vardiff.clamp(requested, floor, base.max_difficulty) - client.vardiff_config = dataclass_replace( - base, - min_difficulty=floor, - startup_difficulty=target, - ) - return target - - def advertise_client_difficulty(self, client: ClientState, target: Decimal) -> bool: - """Move a client to an explicitly requested difficulty. - - Before the client can receive jobs the value is applied directly (the - first set_difficulty/notify pair picks it up). Afterwards it uses the - same job-gated pending mechanism as vardiff retargets: the difficulty - is advertised together with the job it applies to, or not at all. - Returns True only when a fresh set_difficulty/notify pair went out, so - callers about to send their own job can skip a duplicate pair.""" - with client.job_update_lock: - return self._advertise_client_difficulty_locked(client, target) - - def _advertise_client_difficulty_locked( - self, - client: ClientState, - target: Decimal, - ) -> bool: - applied_directly = False - schedule_initial = False - with self._client_vardiff_lock(client): - current = client.pending_share_difficulty or client.share_difficulty - if target == current: - return False - if not (client.subscribed and client.authorized) or ( - client.active_job is None and "maybe_send_job" not in self.__dict__ - ): - client.share_difficulty = target - client.pending_share_difficulty = None - client.difficulty_generation = int( - getattr(client, "difficulty_generation", 0) - ) + 1 - applied_directly = True - schedule_initial = bool( - client.subscribed - and client.authorized - and client.worker is not None - ) - else: - prior_pending = client.pending_share_difficulty - prior_generation = int( - getattr(client, "difficulty_generation", 0) - ) - advertised_generation = prior_generation + 1 - client.pending_share_difficulty = target - client.difficulty_generation = advertised_generation - if applied_directly: - if schedule_initial: - # A pending first-job request captured the previous difficulty - # generation. Replace it atomically so its cancellation callback - # hands the client slot to current work instead of disconnecting. - self.request_initial_job_delivery(client) - return False - with self.lock: - self._ensure_initial_job_state() - initial_pending = client in self.pending_initial_jobs - if initial_pending: - self.request_initial_job_delivery(client) - return False - if not self.stop_event.is_set() and self.maybe_send_job(client, clean_jobs=True): - return True - with self._client_vardiff_lock(client): - if ( - client.pending_share_difficulty == target - and int(getattr(client, "difficulty_generation", 0)) - == advertised_generation - ): - client.pending_share_difficulty = prior_pending - client.difficulty_generation = prior_generation - return False - - def normalized_prior_balances(self, balances: list[dict[str, object]]) -> list[dict[str, object]]: - rows = [ - { - "recipient_id": str(balance.get("recipient_id", "")), - "order_key": str(balance.get("order_key", "")), - "p2mr_program_hex": str(balance.get("p2mr_program_hex", "")), - "balance_sats": int(balance.get("balance_sats", 0)), - } - for balance in balances - ] - rows.sort( - key=lambda row: ( - row["order_key"], - row["recipient_id"], - row["p2mr_program_hex"], - row["balance_sats"], - ) - ) - return rows - - @staticmethod - def settlement_balances_by_program( - balances: list[dict[str, object]], - ) -> dict[str, int]: - """Aggregate owed balances by P2MR program for settlement equality. - - A miner may re-authorize the same payout address under any valid - spelling (bech32 permits an all-uppercase form), which changes the - recipient/order-key identity strings while committing to the same - payout program. Ledger and preview views can then label one program - with different identity strings while agreeing on every settled - amount, so settlement equality must be judged on the program and - amount alone. Identity strings stay display metadata. Zero totals - are dropped to match both the ledger reader (nonzero HAVING filter) - and the preview builder (zero balances skipped). - """ - aggregated: dict[str, int] = {} - for balance in balances: - program = str(balance.get("p2mr_program_hex", "")).lower() - aggregated[program] = aggregated.get(program, 0) + int( - balance.get("balance_sats", 0) - ) - return {program: total for program, total in aggregated.items() if total != 0} - - def prior_balances_match_current(self, prior_balances: list[dict[str, object]]) -> bool: - return self.settlement_balances_by_program( - prior_balances - ) == self.settlement_balances_by_program(self.ledger.current_prior_balances()) - - def send_job(self, client: ClientState, job: direct_stratum.DirectQbitStratumJob) -> None: - client.send(self.job_payload(job)) + return self._ensure_vardiff_service().minimum_advertised_difficulty(client) - @staticmethod - def job_payload(job: direct_stratum.DirectQbitStratumJob) -> dict[str, object]: - return { - "id": None, - "method": "mining.notify", - "params": [ - job.job_id, - job.prevhash, - job.coinb1, - job.coinb2, - list(job.merkle_branch), - job.version, - job.nbits, - job.ntime, - job.clean_jobs, - ], - } + def resume_client_difficulty(self, client: ClientState) -> Decimal | None: + return self._ensure_vardiff_service().apply_resumed_difficulty(client) - def send_job_update( - self, - client: ClientState, - job: direct_stratum.DirectQbitStratumJob, - ) -> None: - # Preserve instance-level send method replacements used by focused - # tests; normal coordinators use the atomic socket batch below. - if "send_difficulty" in self.__dict__ or "send_job" in self.__dict__: - self.send_difficulty(client, job) - self.send_job(client, job) - return - payloads = [ - self.difficulty_payload(job.share_difficulty), - self.job_payload(job), - ] - shared_params = getattr(job, "notify_shared_params_json", None) - if shared_params is None: - client.send_batch(payloads) - return - # Splice the per-client fields around the fragment serialized once - # per bundle build; at fleet scale the coinb1/coinb2/merkle parts - # dominate the notify and are byte-identical for every client. - notify_line = ( - '{"id": null, "method": "mining.notify", "params": [' - + json.dumps(job.job_id) - + ", " - + shared_params - + ", " - + ("true" if job.clean_jobs else "false") - + "]}" - ) - client.send_batch( - payloads, - preserialized=( - json.dumps(payloads[0]).encode() - + b"\n" - + notify_line.encode() - + b"\n" - ), - ) + def note_vardiff_resume_overridden(self) -> None: + self._ensure_vardiff_service().note_resume_overridden() - def build_job_for_client(self, client: ClientState, *, clean_jobs: bool) -> PrismJobContext: - if client.worker is None: - raise StratumError(20, "client is not authorized") - self._ensure_job_cache_state() - artifacts = ( - self._retained_collection_artifacts() - or self.job_issuance_template_artifacts() - ) - return self.build_job_for_client_from_artifacts( + def record_session_difficulty(self, client: ClientState) -> None: + # Only a connection that produced an accepted share may refresh the + # retained value's TTL. Without that evidence a reconnect loop of + # silent sessions would re-stamp the entry forever and defeat the TTL. + self._ensure_vardiff_service().record_session_difficulty( client, - artifacts, - clean_jobs=clean_jobs, - ) - - def build_job_for_client_from_artifacts( - self, - client: ClientState, - artifacts: CachedTemplateArtifacts, - *, - clean_jobs: bool, - publication_critical: bool = False, - request_source: str = "routine", - ) -> PrismJobContext: - self._ensure_job_cache_state() - phases = self._job_build_phases() - while True: - worker = client.worker - if worker is None: - raise StratumError(20, "client is not authorized") - progress_build_token = self._progress_bundle_build_started() - try: - cached_bundle = self.shared_job_bundle( - artifacts, - worker, - publication_critical=publication_critical, - request_source=request_source, - ) - finally: - self._progress_bundle_build_finished(progress_build_token) - current_worker = client.worker - if not cached_bundle.collection_only or current_worker == worker: - break - # Reauthorization changed a genuine collection input while the - # worker-specific bundle was being built. Re-select the latest - # identity without refetching or discarding the exact artifacts. - stamp_started = time.monotonic() - context = self.stamp_job_for_client(client, cached_bundle, clean_jobs=clean_jobs) - phases["stamp"] = phases.get("stamp", 0.0) + (time.monotonic() - stamp_started) - return context - - def build_collection_bundle( - self, - *, - template: dict[str, Any], - transaction_hexes: tuple[str, ...], - worker: WorkerIdentity, - network_difficulty: int, - issued_at_ms: int, - suffix_hex: str, - summary_only: bool = False, - payout_policy: dict[str, object] | None = None, - ctv_settlement: dict[str, object] | None = None, - cancellation: _JobBuildCancellation | None = None, - ) -> dict[str, Any]: - if cancellation is not None: - cancellation.raise_if_cancelled("collection payout derivation") - share = { - "share_seq": 1, - "share_id": "bootstrap-share", - "miner_id": worker.payout_address, - "order_key": worker.payout_address, - "p2mr_program_hex": worker.p2mr_program_hex, - "share_difficulty": network_difficulty, - "network_difficulty": network_difficulty, - "template_height": int(template["height"]) - 1, - "job_id": "bootstrap-job", - "job_issued_at_ms": issued_at_ms, - "accepted_at_ms": issued_at_ms, - "ntime": int(template["curtime"]), - } - return self.build_audit_bundle( - shares=[share], - found_block={ - "block_height": int(template["height"]), - "coinbase_value_sats": int(template["coinbasevalue"]), - "network_difficulty": network_difficulty, - "anchor_job_issued_at_ms": issued_at_ms, - }, - prior_balances=[], - coinbase_script_sig_suffix_hex=suffix_hex, - witness_merkle_leaves_hex=direct_stratum.witness_merkle_leaves_hex(transaction_hexes), - ctv_fee_parent_hash=str(template["previousblockhash"]), - summary_only=summary_only, - payout_policy=payout_policy, - ctv_settlement=ctv_settlement, - cancellation=cancellation, - ) - - def shutdown_serve_builder(self) -> None: - """Retire the persistent audit builder; builds revert to one-shot.""" - self._ensure_job_cache_state() - with self._serve_builder_lock: - self._serve_builder_shutdown = True - client, self._serve_builder = self._serve_builder, None - if client is not None: - client.close() - - def _retire_serve_builder_locked(self) -> None: - client, self._serve_builder = self._serve_builder, None - if client is not None: - client.close() - - def _record_live_serve_builder_termination( - self, - client: _ServeBuilderClient | None, - ) -> None: - """Account for deliberately killing a still-running daemon. - - Crash paths count themselves when the EOF surfaces; a live worker - retired for a timeout, malformed response, protocol mismatch, or - spool failure must land in the termination counters so its - replacement reads as a restart, mirroring the cancellation path. - """ - if client is None: - return - poll = getattr(client.process, "poll", None) - if not callable(poll) or poll() is not None: - return - with self._job_build_scheduler_lock: - self.job_build_worker_counts["terminations"] += 1 - self._job_build_worker_restart_pending = True - - def _observe_builder_phase_metrics(self, metrics: dict[str, Any]) -> None: - """Apply one build's builder-side phase timings to refresh metrics. - - Shared by the one-shot stderr metrics line and the --serve response - payload so serialization_copy and the builder phase histograms stay - comparable regardless of the transport that ran the build. - """ - phase_seconds = metrics.get("phases_seconds", {}) - if isinstance(phase_seconds, dict): - for phase in ( - "payout_state_derivation", - "ctv_manifest_construction", - "coinbase_bundle_construction", - "signing_verification", - ): - elapsed = phase_seconds.get(phase) - if isinstance(elapsed, (int, float)): - self._observe_tip_refresh_build_phase( - phase, - float(elapsed), - ) - rust_serialization = sum( - float(metrics.get(name, 0.0)) - for name in ( - "input_deserialization_seconds", - "output_serialization_seconds", - ) - ) - self._observe_tip_refresh_build_phase( - "serialization_copy", - rust_serialization, + share_backed=bool(getattr(client, "vardiff_accepted_any", False)), ) - def _spawn_serve_builder_locked( - self, - deadline: float, - cancellation: _JobBuildCancellation | None, - ) -> _ServeBuilderClient: - command = prism_tool_command("qbit-prism-build-audit-bundle") + [ - "--serve", - "--signing-key-seed-hex", - self.signing_seed_hex, - "--ledger-signing-key-seed-hex", - self.ledger_attestation_signing_seed_hex, - ] - try: - process = subprocess.Popen( - command, - stdin=subprocess.PIPE, - stdout=subprocess.PIPE, - # The daemon inherits the coordinator's stderr so a crash's - # diagnostics land in the journal instead of a per-request - # capture file that dies with the request. - stderr=None, - close_fds=True, - ) - except OSError as exc: - raise _ServeBuilderUnavailable( - f"audit-builder daemon spawn failed: {exc}" - ) from exc - # The start (and any pending restart it satisfies) is recorded as - # soon as the worker exists, exactly like the one-shot path: a - # handshake-phase death or protocol mismatch must not leave the - # lifecycle totals describing fewer workers than were launched. - restarted = False - with self._job_build_scheduler_lock: - if self._job_build_worker_restart_pending: - self.job_build_worker_counts["restarts"] += 1 - self._job_build_worker_restart_pending = False - restarted = True - self.job_build_worker_counts["starts"] += 1 - if restarted: - with self._tip_refresh_metrics_lock: - self.tip_refresh_worker_restarts += 1 - client = _ServeBuilderClient(process=process) - try: - assert process.stdin is not None and process.stdout is not None - os.set_blocking(process.stdin.fileno(), False) - os.set_blocking(process.stdout.fileno(), False) - handshake_line = self._serve_builder_read_line( - client, - deadline, - cancellation, - None, - ) - handshake = json.loads(handshake_line) - if ( - not isinstance(handshake, dict) - or handshake.get("event") != "handshake" - or handshake.get("protocol") - != PRISM_SERVE_BUILDER_PROTOCOL_VERSION - ): - raise _ServeBuilderUnavailable( - "audit-builder daemon announced an unsupported protocol" - ) - except _ServeBuilderUnavailable: - self._record_live_serve_builder_termination(client) - client.close() - raise - except (OSError, ValueError, AttributeError) as exc: - self._record_live_serve_builder_termination(client) - client.close() - raise _ServeBuilderUnavailable( - f"audit-builder daemon handshake failed: {exc}" - ) from exc - except BaseException: - self._record_live_serve_builder_termination(client) - client.close() - raise - return client + def apply_job_difficulty(self, client: ClientState, job: direct_stratum.DirectQbitStratumJob) -> None: + return self._ensure_job_delivery_service().apply_job_difficulty(client, job) - def _serve_builder_read_line( - self, - client: _ServeBuilderClient, - deadline: float, - cancellation: _JobBuildCancellation | None, - build_control: _JobBundleBuildControl | None, - ) -> bytes: - stdout = client.process.stdout - assert stdout is not None - file_descriptor = stdout.fileno() - while True: - newline_index = client.stdout_buffer.find(b"\n") - if newline_index >= 0: - line = bytes(client.stdout_buffer[:newline_index]) - del client.stdout_buffer[: newline_index + 1] - return line - if cancellation is not None: - cancellation.raise_if_cancelled("serve builder response") - if ( - build_control is not None - and build_control.cancel_event.is_set() - ): - raise _JobBundleBuildSuperseded( - "audit-builder daemon request was canceled after supersession" - ) - if time.monotonic() >= deadline: - # Not counted as a worker failure here: the fallback one-shot - # runs against this same exhausted deadline and its own - # timeout path records the failure exactly once. - raise _ServeBuilderUnavailable( - "audit-builder daemon timed out" - ) - try: - chunk = os.read(file_descriptor, 1 << 16) - except (BlockingIOError, InterruptedError): - time.sleep(min(0.02, PRISM_TIP_REFRESH_ADMISSION_POLL_SECONDS)) - continue - except OSError as exc: - raise _ServeBuilderUnavailable( - f"audit-builder daemon read failed: {exc}" - ) from exc - if not chunk: - if ( - build_control is not None - and build_control.cancel_event.is_set() - ): - raise _JobBundleBuildSuperseded( - "audit-builder daemon was terminated after supersession" - ) - with self._job_build_scheduler_lock: - self.job_build_worker_counts["crashes"] += 1 - self._job_build_worker_restart_pending = True - raise _ServeBuilderUnavailable( - "audit-builder daemon exited mid-request" - ) - client.stdout_buffer += chunk + def apply_client_difficulty_requests(self, client: ClientState) -> Decimal | None: + return self._ensure_job_delivery_service().apply_client_difficulty_requests( + client, + ) - def _serve_builder_write( - self, - client: _ServeBuilderClient, - data: bytes, - deadline: float, - cancellation: _JobBuildCancellation | None, - build_control: _JobBundleBuildControl | None, - ) -> int: - stdin = client.process.stdin - assert stdin is not None - file_descriptor = stdin.fileno() - remaining = memoryview(data) - written_total = 0 - while remaining: - if cancellation is not None: - cancellation.raise_if_cancelled("serve builder request") - if ( - build_control is not None - and build_control.cancel_event.is_set() - ): - raise _JobBundleBuildSuperseded( - "audit-builder daemon request was canceled after supersession" - ) - if time.monotonic() >= deadline: - # Not counted as a worker failure here: the fallback one-shot - # runs against this same exhausted deadline and its own - # timeout path records the failure exactly once. - raise _ServeBuilderUnavailable( - "audit-builder daemon timed out" - ) - try: - written = os.write(file_descriptor, remaining) - except (BlockingIOError, InterruptedError): - time.sleep(min(0.02, PRISM_TIP_REFRESH_ADMISSION_POLL_SECONDS)) - continue - except BrokenPipeError as exc: - if ( - build_control is not None - and build_control.cancel_event.is_set() - ): - raise _JobBundleBuildSuperseded( - "audit-builder daemon was terminated after supersession" - ) from exc - raise _ServeBuilderUnavailable( - "audit-builder daemon input pipe closed" - ) from exc - except OSError as exc: - raise _ServeBuilderUnavailable( - f"audit-builder daemon write failed: {exc}" - ) from exc - if written <= 0: - raise _ServeBuilderUnavailable( - "audit-builder daemon input pipe closed" - ) - written_total += written - remaining = remaining[written:] - return written_total + def advertise_client_difficulty(self, client: ClientState, target: Decimal) -> bool: + return self._ensure_job_delivery_service().advertise_client_difficulty( + client, + target, + ) - def _serve_builder_splice_spool( + def _advertise_client_difficulty_locked( self, - client: _ServeBuilderClient, - share_serialization: _ShareWindowSerialization, - spool_file: Any, - spool_size: int, - deadline: float, - cancellation: _JobBuildCancellation | None, - build_control: _JobBundleBuildControl | None, - ) -> int: - stdin = client.process.stdin - assert stdin is not None - stdin_fd = stdin.fileno() - spool_fd = spool_file.fileno() - offset = 0 - while offset < spool_size: - if cancellation is not None: - cancellation.raise_if_cancelled("serve builder request") - if ( - build_control is not None - and build_control.cancel_event.is_set() - ): - raise _JobBundleBuildSuperseded( - "audit-builder daemon request was canceled after supersession" - ) - if time.monotonic() >= deadline: - # Not counted as a worker failure here: the fallback one-shot - # runs against this same exhausted deadline and its own - # timeout path records the failure exactly once. - raise _ServeBuilderUnavailable( - "audit-builder daemon timed out" - ) - try: - moved = os.splice( - spool_fd, - stdin_fd, - min(PRISM_SPOOL_SPLICE_CHUNK_BYTES, spool_size - offset), - offset_src=offset, - ) - except (BlockingIOError, InterruptedError): - time.sleep(min(0.02, PRISM_TIP_REFRESH_ADMISSION_POLL_SECONDS)) - continue - except BrokenPipeError as exc: - # The daemon's stdin closed; the spool itself is fine. - if ( - build_control is not None - and build_control.cancel_event.is_set() - ): - raise _JobBundleBuildSuperseded( - "audit-builder daemon was terminated after supersession" - ) from exc - raise _ServeBuilderUnavailable( - "audit-builder daemon input pipe closed" - ) from exc - except OSError as exc: - if ( - build_control is not None - and build_control.cancel_event.is_set() - ): - raise _JobBundleBuildSuperseded( - "audit-builder daemon was terminated after supersession" - ) from exc - # The spool itself cannot stream. Poison it so the one-shot - # fallback -- and every later build -- writes the cached - # in-memory fragments instead of retrying the same transfer - # and failing mid-stream. - share_serialization.mark_spool_failed() - raise _ServeBuilderUnavailable( - f"audit-builder daemon spool transfer failed: {exc}" - ) from exc - if moved <= 0: - raise _ServeBuilderUnavailable( - "audit-builder daemon input pipe closed" - ) - offset += moved - return offset + client: ClientState, + target: Decimal, + ) -> bool: + return self._ensure_job_delivery_service()._advertise_client_difficulty_locked( + client, + target, + ) - def _serve_builder_request_locked( - self, - client: _ServeBuilderClient, - *, - deadline: float, - payload: dict[str, object], - shares: list[dict[str, object]], - precomposed: tuple[str, str], - share_serialization: _ShareWindowSerialization, - cancellation: _JobBuildCancellation | None, - build_control: _JobBundleBuildControl | None, - record_phase_metrics: bool, - ) -> dict[str, Any]: - """One JSONL round trip; raises _ServeBuilderUnavailable on anomaly.""" - share_snapshot_sha256 = share_serialization.share_snapshot_sha256 - request_fields = dict(payload) - request_fields["window_key"] = { - "share_snapshot_sha256": share_snapshot_sha256, - } - prefix = json.dumps(request_fields, separators=(",", ":")).encode( - "utf-8" + def normalized_prior_balances(self, balances: list[dict[str, object]]) -> list[dict[str, object]]: + return self._ensure_payout_state_service().normalized_prior_balances(balances) + + @staticmethod + def settlement_balances_by_program(balances: list[dict[str, object]]) -> dict[str, int]: + """Aggregate owed balances by P2MR program for settlement equality.""" + return PayoutStateService.settlement_balances_by_program(balances) + + def prior_balances_match_current(self, prior_balances: list[dict[str, object]]) -> bool: + ensure = getattr(self, "_ensure_payout_state_service", None) + if ensure is not None: + return ensure().prior_balances_match_current(prior_balances) + # Unbound duck-typed callers (the settlement comparison tests) supply + # only settlement_balances_by_program and the ledger; keep the exact + # #87 settlement-equality comparison for them. + return self.settlement_balances_by_program( + prior_balances + ) == self.settlement_balances_by_program( + self.ledger.current_prior_balances() ) - phases = self._job_build_phases() - input_bytes = 0 - input_serialization_elapsed = 0.0 - def send_request(upload_window: bool) -> None: - nonlocal input_bytes, input_serialization_elapsed - serialization_started = time.monotonic() - try: - if not upload_window: - input_bytes += self._serve_builder_write( - client, - prefix + b"\n", - deadline, - cancellation, - build_control, - ) - return - input_bytes += self._serve_builder_write( - client, - prefix[:-1], - deadline, - cancellation, - build_control, - ) - lease = ( - share_serialization.acquire_spooled_tail(shares) - if hasattr(os, "splice") - else None - ) - if lease is not None: - try: - spool_file, spool_size = lease - input_bytes += self._serve_builder_splice_spool( - client, - share_serialization, - spool_file, - spool_size, - deadline, - cancellation, - build_control, - ) - finally: - share_serialization.release_spooled_tail() - else: - identities_json, compact_shares_json = precomposed - for fragment in ( - b',"compact_share_identities":', - identities_json.encode("utf-8"), - b',"compact_shares":', - compact_shares_json.encode("utf-8"), - b"}", - ): - input_bytes += self._serve_builder_write( - client, - fragment, - deadline, - cancellation, - build_control, - ) - input_bytes += self._serve_builder_write( - client, - b"\n", - deadline, - cancellation, - build_control, - ) - finally: - elapsed = time.monotonic() - serialization_started - phases["input_serialization"] = ( - phases.get("input_serialization", 0.0) + elapsed - ) - # Accumulated across a possible needs_window re-send and - # observed once per build so serialization_copy stays - # comparable with the one-shot transport. - input_serialization_elapsed += elapsed - - def read_response() -> tuple[dict[str, Any], bytes]: - worker_started = time.monotonic() - line = self._serve_builder_read_line( - client, - deadline, - cancellation, - build_control, - ) - phases["worker"] = phases.get("worker", 0.0) + ( - time.monotonic() - worker_started - ) - output_started = time.monotonic() - try: - value = json.loads(line) - except ValueError as exc: - raise _ServeBuilderUnavailable( - f"audit-builder daemon response was malformed: {exc}" - ) from exc - finally: - phases["output_serialization"] = phases.get( - "output_serialization", - 0.0, - ) + (time.monotonic() - output_started) - if not isinstance(value, dict): - raise _ServeBuilderUnavailable( - "audit-builder daemon response was not an object" - ) - return value, line + def send_job(self, client: ClientState, job: direct_stratum.DirectQbitStratumJob) -> None: + client.send(self.job_payload(job)) - upload_window = share_snapshot_sha256 not in client.uploaded_windows - send_request(upload_window) - response, response_line = read_response() - if ( - response.get("ok") is not True - and bool(response.get("needs_window")) - and not upload_window - ): - # The daemon evicted this window (respawn or generation churn the - # coordinator's mirror missed); repeat the request with the - # window riding along. - upload_window = True - send_request(True) - response, response_line = read_response() - if response.get("ok") is not True: - raise _ServeBuilderUnavailable( - "audit-builder daemon error: " - f"{response.get('error', 'unknown')}" - ) - summary = response.get("summary") - if not isinstance(summary, dict): - raise _ServeBuilderUnavailable( - "audit-builder daemon returned a malformed summary" - ) - # Every successful response promotes the window in the mirror, hits - # included: the daemon moves a hit entry to most-recent position, and - # a mirror that only tracked uploads would evict a different key and - # either re-upload a still-cached window or bounce on needs_window. - client.note_uploaded_window(share_snapshot_sha256) - if upload_window: - with self._serve_builder_metrics_lock: - self.serve_builder_counts["window_uploads"] += 1 - window_cache = response.get("window_cache") - if isinstance(window_cache, dict): - outcome = "hits" if bool(window_cache.get("hit")) else "misses" - with self._serve_builder_metrics_lock: - self.serve_builder_window_cache_counts[outcome] += 1 - if record_phase_metrics: - self._observe_tip_refresh_build_phase( - "serialization_copy", - input_serialization_elapsed, - ) - metrics_value = response.get("metrics") - if isinstance(metrics_value, dict): - try: - self._observe_builder_phase_metrics(metrics_value) - except (TypeError, ValueError): - # Metrics are diagnostic only. A malformed timing payload - # must never invalidate an otherwise valid summary. - pass - self._record_tip_refresh_ipc_bytes("input", input_bytes) - self._record_tip_refresh_ipc_bytes("output", len(response_line)) - return summary + @staticmethod + def job_payload(job: direct_stratum.DirectQbitStratumJob) -> dict[str, object]: + return stratum_job_payload(job) + + def send_job_update( + self, + client: ClientState, + job: direct_stratum.DirectQbitStratumJob, + ) -> None: + return self._ensure_job_delivery_service().send_job_update(client, job) - def _build_audit_bundle_via_serve_builder( + def build_job_for_client(self, client: ClientState, *, clean_jobs: bool) -> PrismJobContext: + return self._ensure_job_delivery_service().build_job_for_client( + client, + clean_jobs=clean_jobs, + ) + + def build_job_for_client_from_artifacts( self, + client: ClientState, + artifacts: CachedTemplateArtifacts, *, - payload: dict[str, object], - shares: list[dict[str, object]], - precomposed: tuple[str, str], - share_serialization: _ShareWindowSerialization, - cancellation: _JobBuildCancellation | None, - record_phase_metrics: bool, - deadline: float, - ) -> dict[str, Any] | None: - """Build through the persistent daemon; None means use one-shot. - - Any daemon anomaly -- spawn failure, handshake or protocol mismatch, - crash, timeout, malformed response -- retires the daemon and returns - None so the caller's one-shot path runs unchanged against the same - deadline (whatever budget the daemon consumed stays consumed). - Cancellation and supersession raise exactly like the one-shot path - instead of falling back. - """ - if not env_bool("PRISM_BUILDER_SERVE", "1"): - return None - self._ensure_job_cache_state() - if not getattr(self, "signing_seed_hex", None) or not getattr( - self, - "ledger_attestation_signing_seed_hex", - None, - ): - return None - if not self._serve_builder_lock.acquire(blocking=False): - # A concurrent build owns the daemon. One-shot is cheaper than - # queueing behind a multi-second build. - return None - try: - if self._serve_builder_shutdown: - return None - build_control = getattr( - self._job_build_phase_local, - "bundle_build_control", - None, - ) - if not isinstance(build_control, _JobBundleBuildControl): - build_control = None - try: - client = self._serve_builder - if client is not None and client.process.poll() is not None: - # The daemon died between requests (its own crash or an - # external kill). Record the lifecycle transition before - # retiring it so the immediate respawn below reads as a - # crash-and-restart rather than an ordinary start. - with self._job_build_scheduler_lock: - self.job_build_worker_counts["crashes"] += 1 - self._job_build_worker_restart_pending = True - self._retire_serve_builder_locked() - client = None - if client is None: - client = self._spawn_serve_builder_locked( - deadline, - cancellation, - ) - self._serve_builder = client - with self._serve_builder_metrics_lock: - self.serve_builder_counts["spawns"] += 1 - if build_control is not None: - self._register_job_bundle_process( - build_control, - client.process, # type: ignore[arg-type] - ) - try: - summary = self._serve_builder_request_locked( - client, - deadline=deadline, - payload=payload, - shares=shares, - precomposed=precomposed, - share_serialization=share_serialization, - cancellation=cancellation, - build_control=build_control, - record_phase_metrics=record_phase_metrics, - ) - finally: - if build_control is not None: - self._unregister_job_bundle_process( - build_control, - client.process, - ) - except (JobBuildCancelled, _JobBundleBuildSuperseded): - # The daemon stream is indeterminate mid-request; retire it - # so the replacement build starts clean. - self._retire_serve_builder_locked() - with self._job_build_scheduler_lock: - self.job_build_worker_counts["terminations"] += 1 - self._job_build_worker_restart_pending = True - raise - except _ServeBuilderUnavailable: - self._record_live_serve_builder_termination( - self._serve_builder - ) - self._retire_serve_builder_locked() - with self._serve_builder_metrics_lock: - self.serve_builder_counts["fallbacks"] += 1 - return None - except (OSError, ValueError): - self._record_live_serve_builder_termination( - self._serve_builder - ) - self._retire_serve_builder_locked() - with self._serve_builder_metrics_lock: - self.serve_builder_counts["fallbacks"] += 1 - return None - else: - with self._serve_builder_metrics_lock: - self.serve_builder_counts["requests"] += 1 - return summary - finally: - self._serve_builder_lock.release() + clean_jobs: bool, + publication_critical: bool = False, + request_source: str = "routine", + ) -> PrismJobContext: + return self._ensure_job_delivery_service().build_job_for_client_from_artifacts( + client, + artifacts, + clean_jobs=clean_jobs, + publication_critical=publication_critical, + request_source=request_source, + ) - def build_audit_bundle( + def build_collection_bundle( self, *, - shares: list[dict[str, object]], - found_block: dict[str, object], - prior_balances: list[dict[str, object]], - coinbase_script_sig_suffix_hex: str, - witness_merkle_leaves_hex: list[str] | None = None, - ctv_fee_parent_hash: str | None = None, - canonical_output_path: Path | None = None, + template: dict[str, Any], + transaction_hexes: tuple[str, ...], + worker: WorkerIdentity, + network_difficulty: int, + issued_at_ms: int, + suffix_hex: str, summary_only: bool = False, payout_policy: dict[str, object] | None = None, ctv_settlement: dict[str, object] | None = None, cancellation: _JobBuildCancellation | None = None, - share_serialization: _ShareWindowSerialization | None = None, ) -> dict[str, Any]: - self._ensure_job_cache_state() - self._ensure_tip_refresh_state() - if cancellation is not None: - self._job_build_checkpoint("serialization", cancellation) - payload: dict[str, object] = { - "found_block": found_block, - "prior_balances": prior_balances, - "payout_policy": ( - self.prism_payout_policy() - if payout_policy is None - else payout_policy - ), - "coinbase_script_sig_suffix_hex": coinbase_script_sig_suffix_hex, - "witness_merkle_leaves_hex": witness_merkle_leaves_hex or [], - } - job_build_phase_local = getattr(self, "_job_build_phase_local", None) - record_phase_metrics = bool( - getattr(job_build_phase_local, "tip_refresh_metrics", False) - ) - precomposed: tuple[str, str] | None = None - if summary_only: - artifact_started = time.monotonic() - if ( - share_serialization is not None - and share_serialization.share_count == len(shares) - ): - # The compact conversion and its encoded fragments are pure - # per-generation functions of the share window; reuse the - # cached strings instead of re-walking the window. - precomposed = share_serialization.compact_fragments(shares) - else: - identities, compact_shares = _compact_share_payload(shares) - payload["compact_share_identities"] = identities - payload["compact_shares"] = compact_shares - if record_phase_metrics: - self._observe_tip_refresh_build_phase( - "serialization_copy", - time.monotonic() - artifact_started, - ) - else: - payload["shares"] = shares - if ctv_settlement is None and payout_policy is None: - ctv_settlement = self.prism_ctv_settlement_config( - block_height=int(found_block["block_height"]), - parent_hash=ctv_fee_parent_hash, - ) - if ctv_settlement is not None: - payload["ctv_settlement"] = ctv_settlement - if canonical_output_path is not None and summary_only: - raise ValueError("canonical output and job summary output are mutually exclusive") - # One deadline covers the whole build regardless of transport: a - # daemon anomaly falls back to the one-shot subprocess with only the - # remaining budget, so a hung daemon cannot double the configured - # limit while progress health already treats one limit as stuck. - build_deadline = time.monotonic() + float( - getattr( - self, - "bundle_build_timeout_seconds", - DEFAULT_PRISM_BUNDLE_BUILD_TIMEOUT_SECONDS, - ) - ) - if ( - summary_only - and canonical_output_path is None - and precomposed is not None - and share_serialization is not None - ): - # The persistent builder serves the artifact-backed summary path, - # whose window identity and cached fragments it can key on. Every - # anomaly falls back to the one-shot subprocess below. - served = self._build_audit_bundle_via_serve_builder( - payload=payload, - shares=shares, - precomposed=precomposed, - share_serialization=share_serialization, - cancellation=cancellation, - record_phase_metrics=record_phase_metrics, - deadline=build_deadline, - ) - if served is not None: - return served - command = prism_tool_command("qbit-prism-build-audit-bundle") + [ - "--input", - "-", - "--signing-key-seed-hex", - self.signing_seed_hex, - "--ledger-signing-key-seed-hex", - self.ledger_attestation_signing_seed_hex, - ] - command.append("--job-summary-output" if summary_only else "--canonical-output") - if record_phase_metrics: - command.append("--phase-metrics") - if canonical_output_path is not None: - canonical_output_path.parent.mkdir(parents=True, exist_ok=True) - succeeded = False - created_output = False - try: - with ExitStack() as stack: - if canonical_output_path is None: - output = stack.enter_context( - tempfile.TemporaryFile(mode="w+", encoding="utf-8") - ) - else: - output = stack.enter_context( - canonical_output_path.open("x+", encoding="utf-8") - ) - created_output = True - stderr = stack.enter_context( - tempfile.TemporaryFile(mode="w+", encoding="utf-8") - ) - process = subprocess.Popen( - command, - stdin=subprocess.PIPE, - stdout=output, - stderr=stderr, - text=True, - encoding="utf-8", - close_fds=True, - ) - build_control = getattr( - job_build_phase_local, - "bundle_build_control", - None, - ) - if isinstance(build_control, _JobBundleBuildControl): - self._register_job_bundle_process(build_control, process) - with self._job_build_scheduler_lock: - if self._job_build_worker_restart_pending: - self.job_build_worker_counts["restarts"] += 1 - self._job_build_worker_restart_pending = False - self.job_build_worker_counts["starts"] += 1 - assert process.stdin is not None - input_byte_count = 0 - worker_deadline = build_deadline - killed_process_wait_seconds = max( - 0.001, - float( - getattr( - self, - "job_build_cancel_grace_seconds", - DEFAULT_PRISM_JOB_BUILD_CANCEL_GRACE_SECONDS, - ) - ), - ) - coordinator = self - - class _CancelableInput: - def __init__(self, stream: Any) -> None: - self.stream = stream - try: - file_descriptor = int(stream.fileno()) - except (AttributeError, OSError, TypeError, ValueError): - # Lightweight process fakes used by embedders and - # tests do not necessarily expose an OS pipe. - self.file_descriptor: int | None = None - else: - os.set_blocking(file_descriptor, False) - self.file_descriptor = file_descriptor - - def check_cancelled(self) -> None: - if cancellation is not None: - cancellation.raise_if_cancelled( - "builder input serialization" - ) - if ( - isinstance(build_control, _JobBundleBuildControl) - and build_control.cancel_event.is_set() - ): - raise _JobBundleBuildSuperseded( - "audit-builder input was canceled after supersession" - ) - if time.monotonic() >= worker_deadline: - with coordinator._tip_refresh_metrics_lock: - coordinator.tip_refresh_worker_failures += 1 - raise RuntimeError( - "qbit-prism-build-audit-bundle timed out" - ) - - def write(self, value: str) -> int: - nonlocal input_byte_count - self.check_cancelled() - if self.file_descriptor is None: - written = int(self.stream.write(value)) - input_byte_count += len(value[:written].encode("utf-8")) - return written - encoded = value.encode("utf-8") - remaining = memoryview(encoded) - while remaining: - self.check_cancelled() - try: - written = os.write( - self.file_descriptor, - remaining, - ) - except (BlockingIOError, InterruptedError): - time.sleep( - min( - 0.02, - PRISM_TIP_REFRESH_ADMISSION_POLL_SECONDS, - ) - ) - continue - if written <= 0: - raise BrokenPipeError( - "audit-builder input pipe closed" - ) - input_byte_count += written - remaining = remaining[written:] - return len(value) - - serialization_started = time.monotonic() - spool_lease: tuple[Any, int] | None = None - try: - sink = _CancelableInput(process.stdin) - - def write_precomposed_tail() -> None: - assert precomposed is not None - identities_json, compact_shares_json = precomposed - sink.write(',"compact_share_identities":') - sink.write(identities_json) - sink.write(',"compact_shares":') - sink.write(compact_shares_json) - sink.write("}") - - if ( - precomposed is not None - and share_serialization is not None - and sink.file_descriptor is not None - and hasattr(os, "splice") - ): - spool_lease = share_serialization.acquire_spooled_tail( - shares - ) - if precomposed is not None: - # Per-build fields are encoded fresh; the dominant - # share-window fragments stream from the spool file - # written once per generation (kernel moves the bytes - # into the pipe), falling back to the cached strings - # whenever no spool is available. - prefix = json.dumps(payload, separators=(",", ":")) - sink.write(prefix[:-1]) - spool_transferred = False - if spool_lease is not None: - spool_file, spool_size = spool_lease - spool_fd = spool_file.fileno() - offset = 0 - while offset < spool_size: - sink.check_cancelled() - try: - moved = os.splice( - spool_fd, - sink.file_descriptor, - min( - PRISM_SPOOL_SPLICE_CHUNK_BYTES, - spool_size - offset, - ), - offset_src=offset, - ) - except (BlockingIOError, InterruptedError): - time.sleep( - min( - 0.02, - PRISM_TIP_REFRESH_ADMISSION_POLL_SECONDS, - ) - ) - continue - except BrokenPipeError: - # The child's stdin closed; the spool - # itself is fine and the surrounding - # pipe-error handling owns this. - raise - except OSError: - # The spool cannot stream (unsupported - # filesystem or an I/O error); the cached - # in-memory fragments stay authoritative - # for this and every later build. - share_serialization.mark_spool_failed() - if offset: - # Part of the tail already reached the - # pipe; this build cannot be repaired - # by re-writing it from memory. - raise - break - if moved <= 0: - raise BrokenPipeError( - "audit-builder input pipe closed" - ) - offset += moved - input_byte_count += moved - spool_transferred = ( - spool_size > 0 and offset >= spool_size - ) - if not spool_transferred: - write_precomposed_tail() - else: - # iterencode writes bounded fragments to the child - # instead of allocating a second full JSON - # representation in Python. - json.dump( - payload, - sink, - separators=(",", ":"), - ) - except BrokenPipeError: - # Prefer the builder's diagnostic below. - pass - except BaseException as exc: - try: - process.kill() - except ProcessLookupError: - pass - try: - process.wait(timeout=killed_process_wait_seconds) - except subprocess.TimeoutExpired: - pass - if isinstance( - exc, - (JobBuildCancelled, _JobBundleBuildSuperseded), - ): - with self._job_build_scheduler_lock: - self.job_build_worker_counts["terminations"] += 1 - self._job_build_worker_restart_pending = True - raise - finally: - if spool_lease is not None and share_serialization is not None: - share_serialization.release_spooled_tail() - phases = self._job_build_phases() - phases["input_serialization"] = phases.get( - "input_serialization", - 0.0, - ) + (time.monotonic() - serialization_started) - try: - process.stdin.close() - except (BlockingIOError, BrokenPipeError): - pass - if record_phase_metrics: - self._observe_tip_refresh_build_phase( - "serialization_copy", - time.monotonic() - serialization_started, - ) - self._record_tip_refresh_ipc_bytes("input", input_byte_count) - worker_started = time.monotonic() - terminated = False - returncode: int | None = None - if cancellation is None and not hasattr(process, "poll"): - returncode = process.wait( - timeout=max(0.001, worker_deadline - time.monotonic()) - ) - else: - while returncode is None: - returncode = process.poll() - if returncode is not None: - break - cancelled_by_control = ( - isinstance(build_control, _JobBundleBuildControl) - and build_control.cancel_event.is_set() - ) - cancelled_by_request = ( - cancellation is not None and cancellation.is_set() - ) - if cancelled_by_control or cancelled_by_request: - terminated = True - process.terminate() - try: - returncode = process.wait( - timeout=max( - 0.0, - float( - getattr( - self, - "job_build_cancel_grace_seconds", - DEFAULT_PRISM_JOB_BUILD_CANCEL_GRACE_SECONDS, - ) - ), - ) - ) - except subprocess.TimeoutExpired: - process.kill() - try: - returncode = process.wait( - timeout=killed_process_wait_seconds - ) - except subprocess.TimeoutExpired: - returncode = process.poll() - with self._job_build_scheduler_lock: - self.job_build_worker_counts["terminations"] += 1 - self._job_build_worker_restart_pending = True - break - if time.monotonic() >= worker_deadline: - process.kill() - try: - returncode = process.wait( - timeout=killed_process_wait_seconds - ) - except subprocess.TimeoutExpired: - returncode = process.poll() - with self._tip_refresh_metrics_lock: - self.tip_refresh_worker_failures += 1 - raise RuntimeError( - "qbit-prism-build-audit-bundle timed out" - ) - time.sleep(min(0.02, PRISM_TIP_REFRESH_ADMISSION_POLL_SECONDS)) - phases["worker"] = phases.get("worker", 0.0) + ( - time.monotonic() - worker_started - ) - if terminated and cancellation is not None: - if cancellation.is_set(): - cancellation.raise_if_cancelled("builder worker") - if ( - terminated - and isinstance(build_control, _JobBundleBuildControl) - and build_control.cancel_event.is_set() - ): - raise _JobBundleBuildSuperseded( - "audit-builder subprocess was canceled after supersession" - ) - stderr.seek(0) - error_text = stderr.read() - if returncode != 0: - with self._job_build_scheduler_lock: - self.job_build_worker_counts["crashes"] += 1 - self._job_build_worker_restart_pending = True - if record_phase_metrics: - with self._tip_refresh_metrics_lock: - self.tip_refresh_worker_failures += 1 - raise RuntimeError( - f"qbit-prism-build-audit-bundle failed: {error_text}" - ) - if ( - isinstance(build_control, _JobBundleBuildControl) - and build_control.cancel_event.is_set() - ): - raise _JobBundleBuildSuperseded( - "audit-builder result completed after supersession" - ) - output.flush() - output_size = os.fstat(output.fileno()).st_size - if record_phase_metrics: - self._record_tip_refresh_ipc_bytes("output", output_size) - for line in error_text.splitlines(): - if not line.startswith(PRISM_BUILDER_PHASE_METRICS_PREFIX): - continue - raw_metrics = line.removeprefix( - PRISM_BUILDER_PHASE_METRICS_PREFIX - ) - try: - metrics = json.loads(raw_metrics) - if isinstance(metrics, dict): - self._observe_builder_phase_metrics(metrics) - except (TypeError, ValueError, json.JSONDecodeError): - # Metrics are diagnostic only. A malformed timing - # line must never invalidate an otherwise valid - # signed bundle. - pass - if canonical_output_path is not None: - os.fsync(output.fileno()) - output.seek(0) - output_started = time.monotonic() - if cancellation is not None: - cancellation.raise_if_cancelled("builder output serialization") - bundle = json.load(output) - phases["output_serialization"] = phases.get( - "output_serialization", - 0.0, - ) + (time.monotonic() - output_started) - if cancellation is not None: - cancellation.raise_if_cancelled("builder verification") - succeeded = True - return bundle - finally: - if canonical_output_path is not None and created_output and not succeeded: - try: - canonical_output_path.unlink() - except FileNotFoundError: - pass - - def coinbase_script_sig_suffix_hex(self, extranonce1_hex: str, extranonce2_hex: str) -> str: - extranonce1_hex = validate_hex(extranonce1_hex, name="extranonce1") - extranonce2_hex = validate_hex(extranonce2_hex, name="extranonce2") - return self.coinbase_tag_hex + extranonce1_hex + extranonce2_hex - - @ledger_writer_operation("share_submission") - def handle_submit(self, client: ClientState, params: list[object]) -> bool: - if len(params) < 5: - self.reject_stratum( - 20, - PRISM_REJECTION_MALFORMED_SUBMIT, - "submit params are incomplete", - worker=client.username or None, - ) - worker_name, job_id, extranonce2_hex, ntime_hex, nonce_hex = [str(item) for item in params[:5]] - version_bits_hex = str(params[5]) if len(params) > 5 else None - if worker_name != client.username: - self.reject_stratum( - 20, - PRISM_REJECTION_UNAUTHORIZED_WORKER, - "submit username does not match authorized username", - worker=client.username or None, - ) - # Normal submits cross the coordinator lock once: pool admission, job - # membership, and published-tip authority are copied atomically. All - # miner bookkeeping and any live-tip RPC fallback stay outside it. - pool_closed, context, published_tip = self._submit_control_snapshot( - client, - job_id, - ) - # A closed pool rejects before any share accounting: post-close submits - # must not inflate global/per-worker submitted totals (the stale-percent - # denominator) or vardiff windows they can never contribute to. - if pool_closed: - self.reject_stratum( - 21, - PRISM_REJECTION_POOL_CLOSED, - "pool is no longer accepting shares", - worker=worker_name, - ) - if len(extranonce2_hex) != self.extranonce2_size * 2: - self.reject_stratum( - 20, - PRISM_REJECTION_INVALID_EXTRANONCE, - "unexpected extranonce2 size", - worker=worker_name, - ) - if len(ntime_hex) != 8 or len(nonce_hex) != 8: - self.reject_stratum( - 20, - PRISM_REJECTION_INVALID_NTIME_OR_NONCE, - "ntime and nonce must be 4-byte hex strings", - worker=worker_name, - ) - # Count submitted shares once, after the format checks, so the - # per-worker counter and the aggregate qbit_prism_submitted_shares_total - # (via note_vardiff_submitted_share) cover the same population; malformed - # extranonce/ntime submits are recorded only as rejections, not submits. - self.note_worker_submitted_share(worker_name) - self.note_vardiff_submitted_share(client) - credit_policy: str | None = None - evicted_entry: EvictedJobEntry | None = None - if context is None: - evicted_entry = self.evicted_job_entry(client, job_id) - if evicted_entry is None: - self.reject_stratum( - 21, - PRISM_REJECTION_UNKNOWN_JOB, - "stale job", - worker=worker_name, - ) - current_tip = ( - published_tip - if published_tip is not None - else str(self.rpc.call("getbestblockhash")) - ) - # Share classification (normal and stale-grace alike) is deliberately - # point-in-time against this single tip read: a tip that advances - # between here and the ledger append does not retroactively invalidate - # the share, exactly as a normal current-tip share stays credited when - # the tip moves during processing. Re-checking would add an RPC per - # share during post-block bursts only to reject valid work over - # processing latency. Block submission is different (chain state): - # submit_block_candidate re-checks the tip under lock before - # submitblock, and stale-grace shares never reach it. - if context is None: - try: - evicted_context = self.evicted_submit_context(client, evicted_entry, current_tip) - except Exception: - print("prism coordinator: failed to classify evicted submit context", flush=True) - traceback.print_exc() - self.reject_stratum( - 20, - PRISM_REJECTION_BACKEND_RPC_UNAVAILABLE, - "failed to classify stale-grace parent tip", - worker=worker_name, - ) - if evicted_context is None: - self.reject_stratum( - 21, - PRISM_REJECTION_STALE_JOB, - "stale job", - worker=worker_name, - ) - context, credit_policy = evicted_context - elif str(context.template["previousblockhash"]) != current_tip: - try: - eligible_for_grace = self.context_eligible_for_stale_grace(client, context, current_tip) - except Exception: - print("prism coordinator: failed to classify stale-grace parent tip", flush=True) - traceback.print_exc() - self.reject_stratum( - 20, - PRISM_REJECTION_BACKEND_RPC_UNAVAILABLE, - "failed to classify stale-grace parent tip", - worker=worker_name, - ) - if eligible_for_grace: - credit_policy = PRISM_CREDIT_POLICY_STALE_GRACE - else: - self.reject_stratum( - 21, - PRISM_REJECTION_STALE_JOB, - "stale job", - worker=worker_name, - ) - - submit_version_mask = client.version_mask - if ( - evicted_entry is not None - and evicted_entry.connection_id != client.connection_id - ): - # In-flight work from the dead connection was rolled under the - # mask negotiated there; the replacement's own mask (often still - # 0 before mining.configure) must not judge those version bits. - submit_version_mask = int(getattr(context, "version_mask", 0)) - try: - submission = direct_stratum.assemble_submission( - context.job, - extranonce2_hex=extranonce2_hex, - ntime_hex=ntime_hex, - nonce_hex=nonce_hex, - version_bits_hex=version_bits_hex, - version_mask=submit_version_mask, - ) - except ValueError as exc: - self.reject_stratum( - 20, - PRISM_REJECTION_MALFORMED_SUBMIT, - f"malformed submit: {exc}", - worker=worker_name, - ) - # A retained job keeps its original worker even if the connection is - # later re-authorized. Deduplication must use that immutable identity: - # otherwise the same header can be replayed under each new username. - share_key = (context.worker.username, submission.header_hex) - if not self._reserve_recent_share_key(share_key): - self.reject_stratum( - 22, - PRISM_REJECTION_DUPLICATE_SHARE, - "duplicate share", - worker=worker_name, - ) - # A floor-bearing listener holds the advertised share target above the - # qbit network target while network difficulty sits below the floor, - # so a submission can solve a block yet miss the share target. Never - # discard a block over share bookkeeping: reject as low-difficulty - # only when the hash is not block-worthy. Collection-mode jobs are - # block-worthy too: their signed bootstrap manifest already commits - # the whole coinbase to the submitting worker, so the solve settles - # solver-pays-all instead of being silently ledgered as a share -- a - # fresh ledger would otherwise withhold every solved block until some - # later job delivery, stalling a bootstrapping chain. - block_worthy = ( - submission.block_pass - and credit_policy != PRISM_CREDIT_POLICY_STALE_GRACE + return self._ensure_job_bundle_service().build_collection_bundle( + template=template, + transaction_hexes=transaction_hexes, + worker=worker, + network_difficulty=network_difficulty, + issued_at_ms=issued_at_ms, + suffix_hex=suffix_hex, + summary_only=summary_only, + payout_policy=payout_policy, + ctv_settlement=ctv_settlement, + cancellation=cancellation, ) - if ( - submission.block_pass - and credit_policy == PRISM_CREDIT_POLICY_STALE_GRACE - ): - self._ensure_share_hot_path_state() - with self._share_accounting_lock: - dropped_counts = getattr( - self, - "block_solves_dropped_counts", - None, - ) - if dropped_counts is None: - dropped_counts = {"stale_grace": 0} - self.block_solves_dropped_counts = dropped_counts - dropped_counts["stale_grace"] = ( - int(dropped_counts.get("stale_grace", 0)) + 1 - ) - print( - "prism coordinator: stale-grace block solve dropped " - f"hash={submission.block_hash_hex} " - f"parent={context.template['previousblockhash']}", - flush=True, - ) - if block_worthy and context.collection_only: - self._ensure_share_hot_path_state() - with self._share_accounting_lock: - self.collection_block_submission_count = ( - getattr(self, "collection_block_submission_count", 0) + 1 - ) - print( - f"prism coordinator: collection-mode block candidate settles " - f"solver-pays-all miner={context.worker.payout_address} " - f"hash={submission.block_hash_hex}", - flush=True, - ) - if not submission.share_pass and not block_worthy: - self.reject_stratum( - 23, - PRISM_REJECTION_LOW_DIFFICULTY, - "low difficulty share", - worker=worker_name, - ) - pending_share = self.pending_share_from_submission( - context=context, - submission=submission, - ntime_hex=ntime_hex, - credit_policy=credit_policy, + def shutdown_serve_builder(self) -> None: + """Retire the persistent audit builder; builds revert to one-shot.""" + self._ensure_bundle_compiler().shutdown_serve_builder() + + def build_audit_bundle( + self, + *, + shares: list[dict[str, object]], + found_block: dict[str, object], + prior_balances: list[dict[str, object]], + coinbase_script_sig_suffix_hex: str, + witness_merkle_leaves_hex: list[str] | None = None, + ctv_fee_parent_hash: str | None = None, + canonical_output_path: Path | None = None, + canonical_output_parent_fd: int | None = None, + canonical_output_adopter: Callable[[Path, os.stat_result], None] | None = None, + summary_only: bool = False, + payout_policy: dict[str, object] | None = None, + ctv_settlement: dict[str, object] | None = None, + cancellation: _JobBuildCancellation | None = None, + share_serialization: _ShareWindowSerialization | None = None, + append_invalidation_epoch: int | None = None, + ) -> dict[str, Any]: + return self._ensure_bundle_compiler().build_audit_bundle( + shares=shares, + found_block=found_block, + prior_balances=prior_balances, + coinbase_script_sig_suffix_hex=coinbase_script_sig_suffix_hex, + witness_merkle_leaves_hex=witness_merkle_leaves_hex, + ctv_fee_parent_hash=ctv_fee_parent_hash, + canonical_output_path=canonical_output_path, + canonical_output_parent_fd=canonical_output_parent_fd, + canonical_output_adopter=canonical_output_adopter, + summary_only=summary_only, + payout_policy=payout_policy, + ctv_settlement=ctv_settlement, + cancellation=cancellation, + share_serialization=share_serialization, + append_invalidation_epoch=append_invalidation_epoch, ) - if not block_worthy: - try: - self.append_accepted_share( - client, - context, - submission, - pending_share, - credit_policy=credit_policy, - ) - if evicted_entry is not None: - self.note_evicted_job_submit( - credit_policy, - cross_connection=( - evicted_entry.connection_id - != client.connection_id - ), - ) - except BaseException: - self._forget_recent_share_key(share_key) - raise - return False - candidate = PrismBlockCandidate( - context=context, - submission=submission, - # The mined coinbase embeds the extranonce1 the job was stamped - # with. A cross-connection resume submits through a client whose - # own extranonce1 differs from the retained job's, and the audit - # bundle suffix must match the coinbase actually in the block, - # so the job's value is authoritative whenever it carries one. - extranonce1_hex=str( - getattr(context.job, "extranonce1_hex", None) - or client.extranonce1_hex - ), - extranonce2_hex=extranonce2_hex, - pending_share=pending_share, - client=client, - credit_share_on_accept=not submission.share_pass, - ) - if candidate.credit_share_on_accept: - # The hash solved a block but missed the assigned share target - # (possible only while the listener floor sits above network - # difficulty). It is a valid share ONLY if the block lands, so land - # it synchronously: the miner's accept/reject and the ledger credit - # then both reflect the real outcome -- never an "accepted" ack with - # no ledger row. This path is rare (an honest miner does not submit - # below its assigned target), so it does not affect the async - # common-path latency. On failure the submitter already recorded - # the specific block-failure reason; reject the miner as - # low-difficulty (the share was, after all, below its target). The - # submitter already recorded the specific block-failure reason in - # block_candidate_abandoned_counts; reject_stratum additionally counts - # the miner-facing rejection (globally and per worker) so this rare - # synchronous path is not missing from the rejection metrics. - persist_intent = getattr(self.ledger, "persist_block_candidate_intent", None) - durable_candidate_state: str | None = None - try: - candidate_intent = self.block_candidate_intent(candidate) - if callable(persist_intent): - persist_result = self._run_block_submitter_ledger_call( - ( - "persist-candidate-intent", - str(candidate.submission.block_hash_hex).lower(), - ), - "persist-candidate-intent", - lambda: persist_intent(candidate_intent), - ) - result_state = getattr(persist_result, "state", None) - if result_state is not None: - durable_candidate_state = str(result_state) - except BaseException: - # No retry slot is safe until the pre-submit outbox boundary is - # durable. Let the miner retry this submission instead. Without - # a durable intent nothing can commit this stamped share, so - # stop holding the snapshot anchor floor under it. - self._finish_pending_share_commit(pending_share) - self._forget_recent_share_key(share_key) - raise - if durable_candidate_state in {"submitted", "abandoned"}: - # A process restart clears the in-memory disposition cache, - # but the existing outbox row remains authoritative. Join its - # terminal result before any new raw node offer. - block_landed = durable_candidate_state == "submitted" - self._record_block_candidate_terminal_outcome( - submission.block_hash_hex, - accepted=block_landed, - ) - self._finish_pending_share_commit(pending_share) - else: - try: - block_landed = self._submit_synchronous_block_candidate(candidate) - except BaseException: - self._forget_recent_share_key(share_key) - raise - if not block_landed: - self._forget_recent_share_key(share_key) - self.reject_stratum( - 23, - PRISM_REJECTION_LOW_DIFFICULTY, - "low difficulty share", - worker=worker_name, - ) - elif evicted_entry is not None: - self.note_evicted_job_submit( - credit_policy, - cross_connection=( - evicted_entry.connection_id != client.connection_id - ), - ) - return False - # A block-worthy submission that met the share target is a valid share - # regardless of the block's fate: credit it now, acknowledge the miner - # immediately, and land the block from the dedicated submitter thread - # (ckpool/btcpool/StratumV2 semantics). An orphaned candidate keeps its - # share credit. + + def prepare_payout_window( + self, + *, + mode: str, + records_json: list[dict[str, object]], + anchor_job_issued_at_ms: int, + append_invalidation_epoch: int, + window_weight: int | None = None, + page_size: int | None = None, + base_digest: str | None = None, + wait_for_daemon: bool = True, + ) -> Any: + """Fold/advance one payout window through the persistent builder.""" + return self._ensure_bundle_compiler().prepare_payout_window( + mode=mode, + records_json=records_json, + anchor_job_issued_at_ms=anchor_job_issued_at_ms, + append_invalidation_epoch=append_invalidation_epoch, + window_weight=window_weight, + page_size=page_size, + base_digest=base_digest, + wait_for_daemon=wait_for_daemon, + ) + + def coinbase_script_sig_suffix_hex(self, extranonce1_hex: str, extranonce2_hex: str) -> str: + extranonce1_hex = validate_hex(extranonce1_hex, name="extranonce1") + extranonce2_hex = validate_hex(extranonce2_hex, name="extranonce2") + return self.coinbase_tag_hex + extranonce1_hex + extranonce2_hex + + @ledger_writer_operation("share_submission") + def handle_submit(self, client: ClientState, params: list[object]) -> bool: + return self._ensure_share_submission_service().handle(client, params) + + def _submit_synchronous_credit_candidate( + self, + candidate: PrismBlockCandidate, + *, + share_key: tuple[str, str], + worker_name: str, + evicted_entry: EvictedJobEntry | None, + credit_policy: str | None, + ) -> bool: + """Land one below-target block synchronously with its share credit. + + The hash solved a block but missed the assigned share target + (possible only while the listener floor sits above network + difficulty). It is a valid share ONLY if the block lands, so land + it synchronously: the miner's accept/reject and the ledger credit + then both reflect the real outcome -- never an "accepted" ack with + no ledger row. This path is rare (an honest miner does not submit + below its assigned target), so it does not affect the async + common-path latency. On failure the submitter already recorded the + specific block-failure reason in block_candidate_abandoned_counts; + reject_stratum additionally counts the miner-facing rejection + (globally and per worker) so this rare synchronous path is not + missing from the rejection metrics. + """ + submission = candidate.submission + pending_share = candidate.pending_share + persist_intent = getattr(self.ledger, "persist_block_candidate_intent", None) + durable_candidate_state: str | None = None try: candidate_intent = self.block_candidate_intent(candidate) - durable_candidate_state = self.append_accepted_share( - client, - context, - submission, - pending_share, - credit_policy=credit_policy, - candidate_intent=candidate_intent, - ) - if evicted_entry is not None: - self.note_evicted_job_submit( - credit_policy, - cross_connection=( - evicted_entry.connection_id != client.connection_id + if callable(persist_intent): + persist_result = self._run_block_submitter_ledger_call( + ( + "persist-candidate-intent", + str(submission.block_hash_hex).lower(), ), + "persist-candidate-intent", + lambda: persist_intent(candidate_intent), ) + result_state = getattr(persist_result, "state", None) + if result_state is not None: + durable_candidate_state = str(result_state) except BaseException: - # Idempotent with append_accepted_share's own release; also covers - # an intent serialization failure before the append started. + # No retry slot is safe until the pre-submit outbox boundary is + # durable. Let the miner retry this submission instead. Without + # a durable intent nothing can commit this stamped share, so + # stop holding the snapshot anchor floor under it. self._finish_pending_share_commit(pending_share) - self._forget_recent_share_key(share_key) + self._release_submit_share_key(share_key) raise if durable_candidate_state in {"submitted", "abandoned"}: + # A process restart clears the in-memory disposition cache, + # but the existing outbox row remains authoritative. Join its + # terminal result before any new raw node offer. + block_landed = durable_candidate_state == "submitted" self._record_block_candidate_terminal_outcome( submission.block_hash_hex, - accepted=durable_candidate_state == "submitted", + accepted=block_landed, + ) + self._finish_pending_share_commit(pending_share) + else: + try: + block_landed = self._submit_synchronous_block_candidate(candidate) + except BaseException: + self._release_submit_share_key(share_key) + raise + if not block_landed: + self._release_submit_share_key(share_key) + self.reject_stratum( + 23, + PRISM_REJECTION_LOW_DIFFICULTY, + "low difficulty share", + worker=worker_name, + ) + elif evicted_entry is not None: + self.note_evicted_job_submit( + credit_policy, + cross_connection=( + evicted_entry.connection_id + != candidate.client.connection_id + ), ) - return False - self.enqueue_block_candidate(candidate) return False - @staticmethod - def block_candidate_intent(candidate: PrismBlockCandidate) -> dict[str, Any]: - """Return the immutable JSON needed to resume a candidate after restart.""" - context = candidate.context - submission = candidate.submission - intent = { - "schema": "qbit.prism.block-candidate-intent.v1", - "block_hash_hex": str(submission.block_hash_hex).lower(), - "block_hex": str(getattr(submission, "block_hex", "")), - "coinbase_tx_hex": str(getattr(submission, "coinbase_tx_hex", "")), - "parent_hash": str(context.template["previousblockhash"]).lower(), - "expected_height": int(context.template["height"]), - "template": { - "previousblockhash": context.template["previousblockhash"], - "height": int(context.template["height"]), - "coinbasevalue": int(context.template["coinbasevalue"]), - }, - "shares_json": context.shares_json, - "prior_balances": context.prior_balances, - "found_block": context.found_block, - "prospective_prior_balances": ( - [ - list(row) - for row in getattr( - context, - "prospective_prior_balances", - (), - ) - ] - if getattr(context, "prospective_prior_balances", None) is not None - else None - ), - "witness_merkle_leaves_hex": direct_stratum.witness_merkle_leaves_hex( - getattr(context.job, "transaction_hexes", ()) - ), - "extranonce1_hex": candidate.extranonce1_hex, - "extranonce2_hex": candidate.extranonce2_hex, - "username": context.worker.username, - "pending_share": dataclasses.asdict(candidate.pending_share), - "credit_share_on_accept": candidate.credit_share_on_accept, - "collection_only": bool(context.collection_only), - } - # Fail on the client thread before committing a share if a future field - # introduces a value that cannot survive the durable JSON boundary. - json.dumps(intent, separators=(",", ":"), sort_keys=True) - return intent + def block_candidate_intent(self, candidate: PrismBlockCandidate) -> Any: + """Return the immutable intent needed to resume a candidate after restart. + + The durable boundary is where a daemon-mirror share sequence used to + be forced to real dicts, so it is also where a refuted mirror would + first be seen on the landing path. The prepared intent (#255) now + streams the mirror's canonical bytes without parsing them, but the + route stays in place: a divergence raised while preparing must not + propagate, because the candidate cannot be persisted from a window + the coordinator no longer trusts, and the next build must not resume + from that window either. + """ + try: + return encode_block_candidate_intent(candidate) + except DaemonWindowMirrorDivergence: + self._note_window_mirror_divergence() + raise - @staticmethod - def block_candidate_from_intent(intent: dict[str, Any]) -> PrismBlockCandidate: - if intent.get("schema") != "qbit.prism.block-candidate-intent.v1": - raise ValueError("unsupported block candidate intent schema") - block_hash = str(intent["block_hash_hex"]).lower() - template = dict(intent["template"]) - if str(template.get("previousblockhash", "")).lower() != str(intent["parent_hash"]).lower(): - raise ValueError("block candidate parent hash does not match template") - if int(template.get("height", -1)) != int(intent["expected_height"]): - raise ValueError("block candidate height does not match template") - submission = direct_stratum.DirectQbitSubmission( - coinbase_tx_hex=str(intent["coinbase_tx_hex"]), - coinbase_txid_preimage_hex="", - header_hex="", - block_hex=str(intent["block_hex"]), - block_hash_hex=block_hash, - block_hash_int=int(block_hash, 16), - share_pass=True, - block_pass=True, - applied_version_hex="", - ) - context = PrismJobContext( - job=SimpleNamespace( - transaction_hexes=(), - witness_merkle_leaves_hex=tuple( - intent.get("witness_merkle_leaves_hex", []) - ), - ), - template=template, - shares_json=list(intent["shares_json"]), - prior_balances=list(intent["prior_balances"]), - found_block=dict(intent["found_block"]), - share_weight=0, - collection_only=bool(intent.get("collection_only", False)), - worker=WorkerIdentity( - username=str(intent["username"]), - payout_address="", - worker_name=None, - script_pubkey_hex="", - p2mr_program_hex="", - ), - issued_at_ms=0, - prospective_prior_balances=( - tuple( - (str(row[0]), str(row[1]), str(row[2]), int(row[3])) - for row in intent["prospective_prior_balances"] - ) - if isinstance(intent.get("prospective_prior_balances"), list) - else None - ), - # Append-invalidation epochs are process-local counters, so a - # stamp from the process that built this candidate is meaningless - # after a restart. The negative sentinel tells the landing epoch - # fence to stand down and instead revalidate the recorded window - # against the durable ledger at its declared anchor (carry - # balances do not move on a share append, so the prior-balance - # fence alone cannot see an omitted late row). - payout_append_invalidation_epoch=-1, - ) - return PrismBlockCandidate( - context=context, - submission=submission, - extranonce1_hex=str(intent["extranonce1_hex"]), - extranonce2_hex=str(intent["extranonce2_hex"]), - pending_share=PendingShare(**dict(intent["pending_share"])), - client=SimpleNamespace(username=str(intent["username"])), - credit_share_on_accept=bool(intent.get("credit_share_on_accept", False)), - ) + def block_candidate_from_intent( + self, + intent: Mapping[str, Any] | None = None, + ) -> PrismBlockCandidate: + # This helper was historically a static method. Preserve class-level + # decode calls while instance calls additionally adopt S3's durable + # credit-candidate holder before the reconstructed value is published. + coordinator: PrismCoordinator | None + if intent is None: + if not isinstance(self, Mapping): + raise TypeError("block candidate intent must be an object") + intent = self + coordinator = None + else: + coordinator = self + candidate = decode_block_candidate_intent(intent) + if candidate.credit_share_on_accept and coordinator is not None: + # A below-target candidate can credit this older accepted stamp + # after durable replay. Adopt its stable logical floor before + # startup prewarm/job issuance. Ordinary asynchronous candidates + # already committed their share and need no floor. + coordinator._ensure_block_candidate_service().adopt_replayed_candidate( + candidate + ) + return candidate def _ensure_pending_share_commit_state(self) -> None: - if not hasattr(self, "_pending_share_commit_lock"): - self._pending_share_commit_lock = threading.Lock() - if not hasattr(self, "_pending_share_commit_floor"): - self._pending_share_commit_floor = {} + # The S3 owner (created on first touch) holds the floor lock and the + # pending-commit floor; legacy assignments through the descriptor + # routed names are adopted by it. + self._ensure_share_writer_service() def _finish_pending_share_commit(self, pending_share: PendingShare) -> None: - """Drop a share from the snapshot anchor floor. + """Drop a share from the snapshot anchor floor.""" + self._ensure_share_writer_service().finish_pending_share(pending_share) - Called once the share's ledger row reached a terminal outcome in this - process: durably committed, rejected back to the miner, recovered to - the on-disk replay file, or its block candidate terminally abandoned. - Idempotent, and a no-op for shares this process never registered - (intent/journal replays re-create PendingShare objects from JSON). - """ - self._ensure_pending_share_commit_state() - with self._pending_share_commit_lock: - self._pending_share_commit_floor.pop(id(pending_share), None) + def _finish_pending_share_attempt(self, pending_share: PendingShare) -> None: + """Release one stamped submission attempt's floor authority (seam).""" + self._ensure_share_writer_service().finish_pending_attempt(pending_share) + + def _finish_pending_share_candidate(self, pending_share: PendingShare) -> None: + """Release the durable candidate actor's floor authority (seam).""" + self._ensure_share_writer_service().finish_pending_candidate(pending_share) def _job_snapshot_anchor_ms(self, issued_at_ms: int) -> int: - """Clamp a share-snapshot anchor below every coverable share stamp. - - The reward-window contract lets an auditor replay - qbit_audit_share_window(anchor) against the durable ledger and expect - exactly the shares the published bundle counted. Two hazards would - violate that. A share whose accepted_at_ms is already assigned but - whose row has not committed yet (group-commit queue, in-flight batch, - or a block-candidate credit linked after landing) is invisible to the - MVCC snapshot now but joins later replays at any anchor at or above - its accepted_at_ms. And a share stamped right after this clamp - returns can land in the same millisecond as the clamp instant -- - never protected by the pending floor -- so an anchor equal to that - instant would cover it too (the window predicate is - anchor-inclusive). Anchoring strictly below every pending share and - strictly below the clamp-time millisecond keeps the issued snapshot - reproducible without making job builds wait behind the writer - connection: any stamp assigned after this call is at or above the - clamp instant, hence above the anchor. - """ - self._ensure_pending_share_commit_state() - stale_share_ids: list[str] = [] - floor_ms: int | None = None - now_monotonic = time.monotonic() - with self._pending_share_commit_lock: - for entry in self._pending_share_commit_floor.values(): - share = entry[0] - accepted_at_ms = int(share.accepted_at_ms) - if floor_ms is None or accepted_at_ms < floor_ms: - floor_ms = accepted_at_ms - if ( - not entry[2] - and now_monotonic - float(entry[1]) - > PRISM_PENDING_SHARE_COMMIT_WARN_SECONDS - ): - entry[2] = True - stale_share_ids.append(str(share.share_id)) - for share_id in stale_share_ids: - # A long-held floor entry is a wedged writer or a leaked release - # path, not normal group-commit latency. The share-commit liveness - # watchdog owns recovery; this log makes the anchor clamp visible. - print( - "prism coordinator: pending share commit is holding the job " - f"snapshot anchor floor share_id={share_id}", - flush=True, - ) - if floor_ms is None: - return issued_at_ms - 1 - return min(issued_at_ms - 1, floor_ms - 1) + """Clamp a share-snapshot anchor below every coverable share stamp.""" + return self._ensure_share_writer_service().snapshot_anchor_ms(issued_at_ms) def pending_share_from_submission( self, @@ -22264,367 +8310,59 @@ def pending_share_from_submission( ntime_hex: str, credit_policy: str | None = None, ) -> PendingShare: - share_difficulty = self.accepted_share_difficulty(context) - self._ensure_pending_share_commit_state() - # Assign accepted_at_ms and register the commit-floor entry under one - # lock hold: a snapshot anchored between assignment and registration - # could otherwise anchor at or above this share and miss its later - # commit. Released via _finish_pending_share_commit. - with self._pending_share_commit_lock: - pending = PendingShare( - share_id=f"{context.worker.username}:{submission.block_hash_hex}", - miner_id=context.worker.payout_address, - order_key=context.worker.payout_address, - p2mr_program_hex=context.worker.p2mr_program_hex, - share_difficulty=share_difficulty, - network_difficulty=max(1, int(context.found_block["network_difficulty"])), - template_height=int(context.template["height"]) - 1, - job_id=context.job.job_id, - job_issued_at_ms=context.issued_at_ms, - accepted_at_ms=now_ms(), - ntime=int(ntime_hex, 16), - credit_policy=credit_policy, - ) - self._pending_share_commit_floor[id(pending)] = [ - pending, - time.monotonic(), - False, - ] - return pending + return self._ensure_share_writer_service().pending_share_from_submission( + context=context, + submission=submission, + ntime_hex=ntime_hex, + credit_policy=credit_policy, + ) def _expose_inflight_scan_anchor(self, anchor_ms: int) -> int: - """Publish a ledger walk's anchor to the append-side invalidation. - - The entry must stay exposed for as long as the walk's result is - invisible to ``_invalidate_incremental_payout_window_for_append``: - from before the database read (whose snapshot may already exclude a - row committing mid-read) until either the result is discarded or its - window becomes visible through the incremental cache or the armed - artifact's own anchor. Entries older than the audit ceiling are - pruned lazily here: ``_install_payout_ledger_artifact`` rejects any - artifact whose anchor aged past that same ceiling as born-expired, - so a leaked exposure (a built bundle every waiter abandoned) can - never outlive the last install that could have used it. - """ - self._ensure_job_cache_state() - max_age_ms = self._payout_artifact_max_anchor_age_ms() - stale_before_ms = now_ms() - max_age_ms - with self._job_cache_lock: - for token, exposed_anchor_ms in tuple( - self._payout_window_inflight_scan_anchors.items() - ): - if exposed_anchor_ms < stale_before_ms: - self._payout_window_inflight_scan_anchors.pop(token, None) - self._payout_window_inflight_scan_anchor_token += 1 - token = self._payout_window_inflight_scan_anchor_token - self._payout_window_inflight_scan_anchors[token] = int(anchor_ms) - return token + """Publish a ledger walk's anchor to the append-side invalidation.""" + return self._ensure_payout_state_service()._expose_inflight_scan_anchor(anchor_ms) def _retire_inflight_scan_anchor(self, token: int | None) -> None: - """Withdraw an exposed walk anchor once its result became visible - (or provably never will). Idempotent; ``None`` is a no-op.""" - if token is None: - return - self._ensure_job_cache_state() - with self._job_cache_lock: - self._payout_window_inflight_scan_anchors.pop(int(token), None) + """Withdraw an exposed walk anchor once its result became visible""" + return self._ensure_payout_state_service()._retire_inflight_scan_anchor(token) def _publish_seedless_job_window_anchor_locked(self, anchor_ms: int) -> None: - """Keep a seedless published window's anchor visible to appends. - - Caller holds _job_cache_lock, at the bundle publication fence of a - build whose walk exposure retires with no seeded artifact to carry - it (the reuse kill-switch, or nothing to seed). Jobs stamped from - that bundle keep serving the window until they retire, so the - declared anchor must stay in the predates() anchor set past the - exposure: a replay-shaped append that starts and finishes entirely - between publication and a landing's own anchor exposure would - otherwise see no live anchor, skip its epoch bump, and leave no - registry entry for the landing's drain to find. Monotonic and never - retired; the pending-commit floor keeps every ordinary share commit - above it. - """ - self._payout_published_job_window_anchor_ms = max( - int(anchor_ms), - self._payout_published_job_window_anchor_ms or 0, - ) + """Keep a seedless published window's anchor visible to appends.""" + return self._ensure_payout_state_service()._publish_seedless_job_window_anchor_locked(anchor_ms) @staticmethod - def _pending_share_predates_anchor( - pending_share: PendingShare, - anchor_ms: int | None, - ) -> bool: - return bool( - anchor_ms is not None - and int(pending_share.job_issued_at_ms) <= anchor_ms - and int(pending_share.accepted_at_ms) <= anchor_ms - ) + def _pending_share_predates_anchor(pending_share: PendingShare, anchor_ms: int | None) -> bool: + return PayoutStateService._pending_share_predates_anchor(pending_share, anchor_ms) - def _pending_share_predates_live_anchor_locked( - self, - pending_share: PendingShare, - ) -> bool: - # Caller holds _job_cache_lock. The incremental pointer is - # immutable and replaced atomically; its owning lock is - # deliberately not acquired here, because an urgent append must - # disarm the separately guarded artifact even while a long - # oracle build owns _payout_state_prepare_lock. Reading it under - # _job_cache_lock orders the read against the in-flight scan - # anchors: a window walk makes its result visible (the - # incremental pointer, or the armed artifact's install fence) - # before retiring its exposed anchor under this lock, so a walk - # that could have missed this row always exposes an anchor to - # the predates checks below. - predates = self._pending_share_predates_anchor - cached = getattr(self, "_incremental_payout_artifact_window", None) - artifact = self._payout_ledger_artifact - inflight_anchors_ms = tuple( - self._payout_window_inflight_scan_anchors.values() - ) - cached_anchor_ms = ( - None - if cached is None - else int(cached.window.anchor_job_issued_at_ms) - ) - artifact_anchor_ms = ( - None - if artifact is None or artifact.snapshot_anchor_ms is None - else int(artifact.snapshot_anchor_ms) - ) - published_job_anchor_ms = getattr( - self, "_payout_published_job_window_anchor_ms", None - ) - return bool( - predates(pending_share, cached_anchor_ms) - or predates(pending_share, artifact_anchor_ms) - or predates(pending_share, published_job_anchor_ms) - or any( - predates(pending_share, anchor_ms) - for anchor_ms in inflight_anchors_ms - ) - ) + def _pending_share_predates_live_anchor_locked(self, pending_share: PendingShare) -> bool: + return self._ensure_payout_state_service()._pending_share_predates_live_anchor_locked(pending_share) - @contextmanager - def _landing_fence_for_predating_append( - self, - pending_shares: list[PendingShare], - ) -> Iterator[bool]: - """Hold the landing fence across a durable append of predating rows. - - An epoch bump that only starts after its row is durable leaves a - gap: a landing can acquire the fence, verify the pre-bump epoch, - and enter submitblock after the row committed but before the bump - starts, still landing a coinbase whose window omits the durable - share. Any append whose rows predate a live anchor therefore takes - the fence BEFORE the ledger commit and holds it through the epoch - bump, so the durable append and its invalidation land on the same - side of a landing's fence-guarded epoch-check-and-submit boundary. - Ordinary share commits stay off the lock entirely: the unfenced - peek here fires only for the rare replay-shaped append. Yields - whether the fence is held so the caller can thread it into - _record_late_visible_payout_append. - - The unfenced classification is a one-time predicate, so an anchor - exposed after it (a landing publishing its declared anchor) cannot - retroactively fence this commit. An unfenced batch therefore - registers itself, atomically with the classification, in - _payout_unfenced_append_inflight_stamps and deregisters only after - the caller's post-commit epoch-bump attempt ran inside this block; - _await_unfenced_appends_predating_anchor lets a landing wait those - commits out before its epoch fences arm. - """ - self._ensure_job_cache_state() - unfenced_token: int | None = None - with self._job_cache_lock: - fence_needed = any( - self._pending_share_predates_live_anchor_locked(pending) - for pending in pending_shares - ) - if not fence_needed and pending_shares: - self._payout_unfenced_append_inflight_token += 1 - unfenced_token = self._payout_unfenced_append_inflight_token - self._payout_unfenced_append_inflight_stamps[ - unfenced_token - ] = min( - max( - int(pending.job_issued_at_ms), - int(pending.accepted_at_ms), - ) - for pending in pending_shares - ) - if not fence_needed: - try: - yield False - finally: - if unfenced_token is not None: - with self._payout_unfenced_append_drained: - self._payout_unfenced_append_inflight_stamps.pop( - unfenced_token, None - ) - self._payout_unfenced_append_drained.notify_all() - return - with self._payout_append_landing_fence_lock: - yield True + def _landing_fence_for_predating_append(self, pending_shares: list[PendingShare]) -> Iterator[bool]: + """Hold the landing fence across a durable append of predating rows.""" + return self._ensure_payout_state_service()._landing_fence_for_predating_append(pending_shares) def _await_unfenced_appends_predating_anchor(self, anchor_ms: int) -> None: - """Wait until no unfenced in-flight append can predate ``anchor_ms``. - - Called by a landing right after exposing its declared anchor. An - append classified as unfenced before that exposure commits outside - the landing fence, so holding the fence across submitblock would - not exclude its durable commit -- and its post-commit epoch bump - would queue behind the very fence the landing holds, arriving only - after the block entered qbitd. Draining here restores the fence - contract: each matching append finishes its commit and its bump - attempt against the now-exposed anchor before this returns, so a - predating row advances the epoch the landing's fences check, and - every append classified after the exposure sees the anchor and - takes the fence itself. Appends whose rows cannot predate the - anchor never block this wait, and the registry holds entries only - for the duration of a ledger commit, so the common path is one - locked emptiness check. Commit-scoped retention is sound because an - append that completed before this drain cannot have predated - ``anchor_ms`` silently: every landable declared anchor is exposed - from publication onward (the armed artifact's anchor, or the - published-window watermark a seedless build hands its anchor to at - the publication fence), so such an append classified as fenced and - advanced the epoch this landing's fences compare against. - """ - self._ensure_job_cache_state() - anchor = int(anchor_ms) - with self._payout_unfenced_append_drained: - while any( - stamp_ms <= anchor - for stamp_ms in ( - self._payout_unfenced_append_inflight_stamps.values() - ) - ): - self._payout_unfenced_append_drained.wait() - - def _record_late_visible_payout_append( - self, - pending_share: PendingShare, - *, - landing_fence_owned: bool = False, - ) -> int | None: - """Advance the append-invalidation epoch for a newly durable row. - - Two-phase bump. The unfenced peek keeps ordinary share commits off - the landing fence lock entirely; only a row that predates a live - anchor re-checks and bumps under it. Holding that lock for the bump - is what makes the landing's final epoch fence authoritative: a bump - cannot commit while a landing sits between its last epoch read and - submitblock, so the invalidation lands either before that read - (candidate abandoned) or after the RPC returns (the refresh wave - retires the job). The predicate re-runs under the lock because the - anchor set may have moved during the unfenced gap. A caller that - already holds the fence around the durable commit itself - (landing_fence_owned) bumps without re-acquiring. - - Returns the advanced epoch, or None when no live anchor predates - the row. The caller must follow a non-None return with - _retire_payout_windows_for_late_append OUTSIDE the fence: that step - waits on _payout_state_prepare_lock, which a long oracle build can - hold, and the wait must not block landings. - """ - self._ensure_job_cache_state() - - def bump_if_predating_locked() -> int | None: - with self._job_cache_lock: - if not self._pending_share_predates_live_anchor_locked( - pending_share - ): - return None - self._payout_ledger_append_invalidation_epoch += 1 - self._payout_ledger_artifact = None - return int(self._payout_ledger_append_invalidation_epoch) + """Wait until no unfenced in-flight append can predate ``anchor_ms``.""" + return self._ensure_payout_state_service()._await_unfenced_appends_predating_anchor(anchor_ms) - if landing_fence_owned: - return bump_if_predating_locked() - with self._job_cache_lock: - if not self._pending_share_predates_live_anchor_locked( - pending_share - ): - return None - with self._payout_append_landing_fence_lock: - return bump_if_predating_locked() + def _record_late_visible_payout_append(self, pending_share: PendingShare, *, landing_fence_owned: bool=False) -> int | None: + """Advance the append-invalidation epoch for a newly durable row.""" + return self._ensure_payout_state_service()._record_late_visible_payout_append(pending_share, landing_fence_owned=landing_fence_owned) - def _invalidate_incremental_payout_window_for_append( - self, - pending_share: PendingShare, - ) -> None: - """Disarm payout work that predates a newly visible eligible row. - - Normal submissions hold the pending-share commit floor, so their - accepted timestamp remains above the cached anchor until the row is - durable. Durable replay reconstructs the original timestamps without - that process-local floor. If both stamps are already covered by an - incremental or armed-artifact anchor, the snapshot cannot discover - the late-visible row. Publish the invalidation without waiting for an - in-flight oracle walk, then clear the incremental cache under its own - lock; the epoch prevents pre-append work from re-arming meanwhile. - - This wrapper serves post-commit callers that do not hold the landing - fence; the durable append paths take the fence around the commit via - _landing_fence_for_predating_append and call the bump and retire - steps directly. - """ - invalidation_epoch = self._record_late_visible_payout_append( - pending_share + def _append_epoch_invalidated_declared_anchor(self, *, baseline_epoch: int, live_epoch: int, declared_anchor_ms: int | None) -> bool: + """Whether an append between two epochs invalidated this declared anchor.""" + return self._ensure_payout_state_service()._append_epoch_invalidated_declared_anchor( + baseline_epoch=baseline_epoch, + live_epoch=live_epoch, + declared_anchor_ms=declared_anchor_ms, ) - if invalidation_epoch is None: - return - self._retire_payout_windows_for_late_append( - pending_share, invalidation_epoch - ) - - def _retire_payout_windows_for_late_append( - self, - pending_share: PendingShare, - invalidation_epoch: int, - ) -> None: - """Disarm cached payout windows after a late-append epoch bump. - - Runs outside the landing fence. Active jobs stamped against the - pre-append window still pass the generation and digest fences; only - epoch-aware reselection replaces them. Schedule the same refresh wave - a repair publication schedules, and do it before the prepare-lock - wait below so a long oracle build holding that lock cannot delay - superseding the stale jobs. - """ - - def predates(anchor_ms: int | None) -> bool: - return self._pending_share_predates_anchor( - pending_share, anchor_ms - ) - self._mark_tip_refresh_pending(invalidation_epoch) - self._schedule_tip_refresh_retry() + def _invalidate_incremental_payout_window_for_append(self, pending_share: PendingShare) -> None: + """Disarm payout work that predates a newly visible eligible row.""" + return self._ensure_payout_state_service()._invalidate_incremental_payout_window_for_append(pending_share) - with self._payout_state_prepare_lock: - cached = self._incremental_payout_artifact_window - if cached is None: - self._incremental_payout_artifact_window_invalidation_reason = ( - "late_visible_append" - ) - return - anchor_ms = int(cached.window.anchor_job_issued_at_ms) - if cached.append_invalidation_epoch >= invalidation_epoch: - # A post-invalidation build already replaced the cache. - return - if predates(anchor_ms): - self._incremental_payout_artifact_window = None - self._incremental_payout_artifact_window_invalidation_reason = ( - "late_visible_append" - ) - else: - # The armed artifact was affected but this older incremental - # anchor was not. Retag it so a later delta build can preserve - # the exact append-only window instead of forcing an oracle. - self._incremental_payout_artifact_window = dataclass_replace( - cached, - append_invalidation_epoch=invalidation_epoch, - ) + def _retire_payout_windows_for_late_append(self, pending_share: PendingShare, invalidation_epoch: int) -> None: + """Disarm cached payout windows after a late-append epoch bump.""" + return self._ensure_payout_state_service()._retire_payout_windows_for_late_append(pending_share, invalidation_epoch) @ledger_writer_operation("share_persistence") def append_accepted_share( @@ -22637,405 +8375,37 @@ def append_accepted_share( credit_policy: str | None = None, candidate_intent: dict[str, Any] | None = None, ) -> str | None: - entry = PendingShareAppend( - pending_share=pending_share, - username=context.worker.username, - job_id=context.job.job_id, - block_hash_hex=submission.block_hash_hex, - collection_only=bool(context.collection_only), + return self._ensure_share_writer_service().append_accepted_share( + client, + context, + submission, + pending_share, credit_policy=credit_policy, - candidate_intent=candidate_intent, - ) - try: - if getattr(self, "share_writer_active", False): - self.enqueue_share_append(entry, wait=True) - else: - self._append_share_entry(entry) - finally: - # The append reached a terminal outcome for this process: durably - # committed, or surfaced an error the miner will retry with a fresh - # share. Either way the stamped share no longer holds the snapshot - # anchor floor. Idempotent with the group-commit writer's release. - self._finish_pending_share_commit(pending_share) - record = entry.record - # Exact ledger replays return the original record. Their durable share - # must not increment process-local worker/vardiff counters again. - if record is None or bool(getattr(record, "newly_inserted", True)): - self.note_worker_accepted_share(context.worker.username, credit_policy) - self.note_vardiff_accepted_share(client, context.job) - if candidate_intent is None or record is None: - return None - candidate_state = getattr(record, "candidate_outbox_state", None) - return str(candidate_state) if candidate_state is not None else None - - def enqueue_share_append(self, entry: PendingShareAppend, *, wait: bool = False) -> None: - queue_obj = getattr(self, "share_append_queue", None) - if queue_obj is None: - queue_obj = queue.Queue(maxsize=MAX_PENDING_SHARE_APPENDS) - self.share_append_queue = queue_obj - if entry.writer_token is None: - entry.writer_token = self._ensure_shutdown_controller().reserve_writer( - "share_persistence" - ) - try: - if wait: - queue_obj.put( - entry, - timeout=getattr(self, "share_commit_timeout_seconds", 15.0), - ) - else: - queue_obj.put_nowait(entry) - except queue.Full: - entry.writer_token.finish() - entry.writer_token = None - raise StratumError( - 20, - "share ledger commit queue is full", - reason=PRISM_REJECTION_INTERNAL_ERROR, - ) - if not wait: - return - # Once admitted, wait for a definite transaction outcome. A local - # timeout is ambiguous because Postgres may commit immediately after - # it; the liveness watchdog owns recovery from a wedged writer. - entry.committed.wait() - if entry.error is not None: - raise StratumError( - 20, - f"share ledger commit failed: {entry.error}", - reason=PRISM_REJECTION_INTERNAL_ERROR, - ) - - def share_append_loop(self) -> None: - while True: - self._record_heartbeat("share_writer") - queue_obj = getattr(self, "share_append_queue", None) - if queue_obj is None: - queue_obj = queue.Queue(maxsize=MAX_PENDING_SHARE_APPENDS) - self.share_append_queue = queue_obj - stopping = self.stop_event.is_set() - try: - entry = queue_obj.get(timeout=0.2 if stopping else 1.0) - except queue.Empty: - controller = self._ensure_shutdown_controller() - if ( - stopping - and controller.writer_admission_closed() - and not controller.has_active_writer( - { - "share_submission", - "share_persistence", - "accepted_block_handling", - } - ) - ): - return - continue - batch = [entry] - batch_size = max(1, int(getattr(self, "share_commit_batch_size", 64))) - deadline = time.monotonic() + max( - 0.0, float(getattr(self, "share_commit_linger_seconds", 0.005)) - ) - if entry.candidate_intent is not None: - deadline = time.monotonic() - while len(batch) < batch_size: - remaining = deadline - time.monotonic() - if remaining <= 0: - break - try: - next_entry = queue_obj.get(timeout=remaining) - batch.append(next_entry) - if next_entry.candidate_intent is not None: - break - except queue.Empty: - break - self._append_share_batch(batch) + candidate_intent=candidate_intent, + ) + + def enqueue_share_append(self, entry: PendingShareAppend, *, wait: bool = False) -> None: + self._ensure_share_writer_service().enqueue_share_append(entry, wait=wait) + + def share_append_loop(self) -> None: + self._ensure_share_writer_service().share_append_loop() def _append_share_batch(self, batch: list[PendingShareAppend]) -> bool: """Commit a writer batch, then release every waiting submitter.""" - try: - invalidations: list[tuple[PendingShare, int]] = [] - # A batch holding a replay-shaped row commits under the landing - # fence: the durable append and its epoch bump must land on the - # same side of a landing's fence-guarded submit, or the landing - # can verify the pre-bump epoch after the row is already durable - # (see _landing_fence_for_predating_append). - with self._landing_fence_for_predating_append( - [entry.pending_share for entry in batch] - ) as fence_owned: - append_batch = getattr(self.ledger, "append_batch", None) - if callable(append_batch): - records = append_batch( - [(entry.pending_share, entry.candidate_intent) for entry in batch] - ) - else: - # Compatibility for lightweight test/tool ledgers. Production's - # Postgres ledger always supplies the atomic batch method. - records = [self.ledger.append(entry.pending_share) for entry in batch] - if len(records) != len(batch): - raise RuntimeError("share ledger returned an incomplete commit batch") - for entry, record in zip(batch, records, strict=True): - entry.record = record - invalidation_epoch = self._record_late_visible_payout_append( - entry.pending_share, - landing_fence_owned=fence_owned, - ) - if invalidation_epoch is not None: - invalidations.append( - (entry.pending_share, invalidation_epoch) - ) - for pending_share, invalidation_epoch in invalidations: - self._retire_payout_windows_for_late_append( - pending_share, invalidation_epoch - ) - if getattr(self, "hot_path_log_enabled", False): - for entry in batch: - record = entry.record - print( - "prism coordinator: accepted share " - f"seq={record.share_seq} miner={entry.username} job={entry.job_id} " - f"hash={entry.block_hash_hex} collection={entry.collection_only} " - f"credit_policy={entry.credit_policy or 'normal'}", - flush=True, - ) - return True - except Exception as exc: - self._ensure_share_hot_path_state() - with self._share_accounting_lock: - self.share_append_failure_count = ( - int(getattr(self, "share_append_failure_count", 0)) + len(batch) - ) - for entry in batch: - entry.error = exc - print( - f"prism coordinator: share ledger group commit failed count={len(batch)}", - flush=True, - ) - traceback.print_exc() - return False - finally: - for entry in batch: - self._finish_pending_share_commit(entry.pending_share) - entry.committed.set() - if entry.writer_token is not None: - entry.writer_token.finish() - entry.writer_token = None + return self._ensure_share_writer_service().append_share_batch(batch) def _recover_share_to_disk(self, entry: PendingShareAppend, reason: str) -> None: - """Durably capture an acked share the writer could not persist. - - Appends the canonical pending-share JSON to the recovery file (fsynced) - so a ledger outage or shutdown never silently loses a share the miner - was told was accepted; replayed on the next start. Best-effort: if even - the recovery write fails, log loudly rather than raise on the writer. - """ - path = getattr(self, "share_recovery_path", None) - if path is None: - print( - "prism coordinator: WOULD LOSE acked share (no recovery path) " - f"share_id={entry.pending_share.share_id} reason={reason}", - flush=True, - ) - return - try: - payload = json.dumps(dataclasses.asdict(entry.pending_share), separators=(",", ":")) - except Exception: - payload = None - with getattr(self, "share_recovery_lock", threading.Lock()): - try: - path.parent.mkdir(parents=True, exist_ok=True) - if payload is None: - raise ValueError("pending share is not serializable") - with open(path, "a", encoding="utf-8") as handle: - handle.write(payload + "\n") - handle.flush() - os.fsync(handle.fileno()) - self.shares_recovered_to_disk = ( - int(getattr(self, "shares_recovered_to_disk", 0)) + 1 - ) - print( - "prism coordinator: recovered unpersisted acked share to disk " - f"share_id={entry.pending_share.share_id} reason={reason}", - flush=True, - ) - except Exception: - print( - "prism coordinator: FAILED to recover acked share to disk; " - f"share may be lost share_id={entry.pending_share.share_id} reason={reason}", - flush=True, - ) - traceback.print_exc() + self._ensure_share_writer_service().recover_share_to_disk(entry, reason) @ledger_writer_operation("share_recovery_replay") def replay_recovered_shares(self) -> int: - """Replay any recovery-file shares into the ledger at startup. - - Idempotent: both ledgers raise on a duplicate share_id, so a row already - committed by an earlier partial replay is skipped (not double-counted) - and does not stop the pass. The file is cleared only after a clean pass, - so a transient failure here never drops shares. - """ - path = getattr(self, "share_recovery_path", None) - if path is None or not path.exists(): - return 0 - try: - lines = [line for line in path.read_text(encoding="utf-8").splitlines() if line.strip()] - except Exception: - print("prism coordinator: could not read share recovery file", flush=True) - traceback.print_exc() - return 0 - # Parse line-by-line and skip any single unparseable line rather than - # aborting the whole replay: a crash mid-append can leave the last line - # torn, and one torn line must not block the intact shares before it. - pendings: list[PendingShare] = [] - parse_failed = False - for line in lines: - try: - pendings.append(PendingShare(**json.loads(line))) - except Exception: - parse_failed = True - print("prism coordinator: skipping an unparseable recovered share line", flush=True) - traceback.print_exc() - # Replay in acceptance order. A share recovered out of FIFO order (a - # ledger flap during the shutdown drain, or an overflow-recovered newest - # share) otherwise sorts by file order; ordering by accepted_at_ms lands - # each share with a share_seq consistent with when it was accepted, so - # the reward window stays correctly ordered. - pendings.sort(key=lambda pending: pending.accepted_at_ms) - replayed = 0 - skipped_duplicates = 0 - for pending in pendings: - try: - # A recovered row reconstructs its pre-crash timestamps, so - # it routinely predates a live anchor; each row commits under - # the landing fence with its epoch bump so a concurrent - # landing can never verify a pre-bump epoch against an - # already-durable replayed share. The window retirement runs - # after the fence releases, per row, so replay never holds - # the fence across the prepare-lock wait. - invalidation_epoch: int | None = None - with self._landing_fence_for_predating_append( - [pending] - ) as fence_owned: - self.ledger.append(pending) - invalidation_epoch = self._record_late_visible_payout_append( - pending, - landing_fence_owned=fence_owned, - ) - if invalidation_epoch is not None: - self._retire_payout_windows_for_late_append( - pending, invalidation_epoch - ) - replayed += 1 - except Exception as exc: - if "duplicate share_id" in str(exc): - # Already committed by an earlier (partial) replay. Both - # ledgers raise on a duplicate share_id; treat it as done and - # keep going so replay is idempotent -- otherwise a retry - # after a partial pass would stop on the first committed row - # and strand every share after it. - skipped_duplicates += 1 - continue - print("prism coordinator: failed to replay a recovered share; keeping the file", flush=True) - traceback.print_exc() - self.shares_replayed = int(getattr(self, "shares_replayed", 0)) + replayed - return replayed - if skipped_duplicates: - print( - f"prism coordinator: skipped {skipped_duplicates} already-committed " - "recovered share(s) during replay", - flush=True, - ) - if parse_failed: - # Keep the file (with its intact-but-already-replayed lines, which - # the ledger dedups on a re-run) so the torn line is preserved for - # inspection rather than silently discarded. - self.shares_replayed = int(getattr(self, "shares_replayed", 0)) + replayed - if replayed: - print(f"prism coordinator: replayed {replayed} recovered share(s) into the ledger", flush=True) - return replayed - try: - path.unlink() - except FileNotFoundError: - pass - self.shares_replayed = int(getattr(self, "shares_replayed", 0)) + replayed - if replayed: - print(f"prism coordinator: replayed {replayed} recovered share(s) into the ledger", flush=True) - return replayed + """Replay any recovery-file shares into the ledger at startup.""" + return self._ensure_share_writer_service().replay_recovered_shares() def _append_share_entry(self, entry: PendingShareAppend, *, retry_until_stopped: bool = False) -> bool: - """Synchronously append one accepted share. - - On the writer thread a transient ledger failure retries with capped - backoff so ordering is preserved and nothing is silently lost; the - synchronous path (no writer) propagates the exception exactly as the - pre-async code did. - - Returns True when the share was persisted to the ledger, or False when - it was recovered to disk instead (ledger still down at shutdown). The - caller uses that to keep the shutdown drain in order. - """ - backoff_seconds = 0.5 - invalidation_epoch: int | None = None - while True: - try: - # A replay-shaped row commits under the landing fence with - # its epoch bump (see _landing_fence_for_predating_append). - # The fence wraps one attempt only: the backoff sleep below - # runs outside it, so a retrying ledger outage never holds - # up a landing. - with self._landing_fence_for_predating_append( - [entry.pending_share] - ) as fence_owned: - append_batch = getattr(self.ledger, "append_batch", None) - if callable(append_batch): - record = append_batch( - [(entry.pending_share, entry.candidate_intent)] - )[0] - else: - record = self.ledger.append(entry.pending_share) - entry.record = record - invalidation_epoch = self._record_late_visible_payout_append( - entry.pending_share, - landing_fence_owned=fence_owned, - ) - break - except Exception: - if not retry_until_stopped: - raise - self._ensure_share_hot_path_state() - with self._share_accounting_lock: - self.share_append_failure_count = ( - int(getattr(self, "share_append_failure_count", 0)) + 1 - ) - print( - "prism coordinator: ledger share append failed; retrying " - f"share_id={entry.pending_share.share_id}", - flush=True, - ) - traceback.print_exc() - if self.stop_event.wait(backoff_seconds): - # Shutting down mid-outage: do not silently drop this - # already-acked, already-counted share -- recover it to - # disk for replay on the next start. - self._recover_share_to_disk(entry, "ledger unavailable at shutdown") - return False - backoff_seconds = min(backoff_seconds * 2, 5.0) - self._record_heartbeat("share_writer") - if invalidation_epoch is not None: - self._retire_payout_windows_for_late_append( - entry.pending_share, invalidation_epoch - ) - if getattr(self, "hot_path_log_enabled", False): - print( - "prism coordinator: accepted share " - f"seq={record.share_seq} miner={entry.username} job={entry.job_id} " - f"hash={entry.block_hash_hex} collection={entry.collection_only} " - f"credit_policy={entry.credit_policy or 'normal'}", - flush=True, - ) - entry.committed.set() - return True + return self._ensure_share_writer_service().append_share_entry( + entry, retry_until_stopped=retry_until_stopped + ) def accepted_share_difficulty(self, context: PrismJobContext) -> int: override = self.share_weights_by_username.get( @@ -23050,106 +8420,19 @@ def note_vardiff_submitted_share(self, client: ClientState) -> None: self._ensure_share_hot_path_state() with self._share_accounting_lock: self.submitted_share_count += 1 - with self._client_vardiff_lock(client): - config = ( - client.vardiff_config - or client.listener_vardiff_config - or self.vardiff_config - ) - if not config.enabled: - return - client.vardiff_window_submitted += 1 + self._ensure_vardiff_service().note_submitted(client) def note_vardiff_accepted_share(self, client: ClientState, job: direct_stratum.DirectQbitStratumJob) -> None: - now = time.monotonic() - with self._client_vardiff_lock(client): - config = ( - client.vardiff_config - or client.listener_vardiff_config - or self.vardiff_config - ) - if not config.enabled: - return - client.vardiff_window_accepted += 1 - client.vardiff_window_work += job.share_difficulty - elapsed_seconds = Decimal(str(max(0.001, now - client.vardiff_window_started_monotonic))) - if elapsed_seconds < config.retarget_interval_seconds: - return - accepted_shares = client.vardiff_window_accepted - submitted_shares = client.vardiff_window_submitted - accepted_difficulty = client.vardiff_window_work - current_difficulty = client.pending_share_difficulty or client.share_difficulty - client.vardiff_window_started_monotonic = now - client.vardiff_window_accepted = 0 - client.vardiff_window_submitted = 0 - client.vardiff_window_work = Decimal("0") - self.retarget_client( - client, - current_difficulty=current_difficulty, - accepted_shares=accepted_shares, - submitted_shares=submitted_shares, - accepted_difficulty=accepted_difficulty, - elapsed_seconds=elapsed_seconds, - ) + self._ensure_vardiff_service().note_accepted(client, job.share_difficulty) def _ensure_vardiff_idle_state(self) -> None: - if not hasattr(self, "_vardiff_idle_lock"): - self._vardiff_idle_lock = threading.Lock() - if not hasattr(self, "_vardiff_idle_executor"): - self._vardiff_idle_executor: ThreadPoolExecutor | None = None - if not hasattr(self, "_vardiff_idle_executor_shutdown"): - self._vardiff_idle_executor_shutdown = False - if not hasattr(self, "_vardiff_idle_pending"): - self._vardiff_idle_pending: set[tuple[ClientState, int]] = set() - if not hasattr(self, "vardiff_idle_queue_depth"): - self.vardiff_idle_queue_depth = 0 - if not hasattr(self, "vardiff_idle_inflight"): - self.vardiff_idle_inflight = 0 - if not hasattr(self, "vardiff_idle_clients_inspected"): - self.vardiff_idle_clients_inspected = 0 - if not hasattr(self, "vardiff_idle_skip_counts"): - self.vardiff_idle_skip_counts = { - reason: 0 for reason in PRISM_VARDIFF_IDLE_SKIP_REASONS - } - if not hasattr(self, "vardiff_idle_task_failures"): - self.vardiff_idle_task_failures = 0 - for attribute in ( - "vardiff_idle_sweep_histogram", - "vardiff_idle_task_histogram", - ): - if not hasattr(self, attribute): - setattr( - self, - attribute, - { - "buckets": { - bucket: 0 - for bucket in PRISM_VARDIFF_IDLE_SECONDS_BUCKETS - }, - "sum": 0.0, - "count": 0, - }, - ) + self._ensure_vardiff_service() def _record_vardiff_idle_skip(self, reason: str) -> None: - if reason not in PRISM_VARDIFF_IDLE_SKIP_REASONS: - raise ValueError(f"unknown vardiff idle skip reason: {reason}") - self._ensure_vardiff_idle_state() - with self._vardiff_idle_lock: - self.vardiff_idle_skip_counts[reason] += 1 + self._ensure_vardiff_service().record_idle_skip(reason) def _observe_vardiff_idle_seconds(self, name: str, elapsed_seconds: float) -> None: - self._ensure_vardiff_idle_state() - if name not in {"sweep", "task"}: - raise ValueError(f"unknown vardiff idle histogram: {name}") - with self._vardiff_idle_lock: - histogram = getattr(self, f"vardiff_idle_{name}_histogram") - histogram["count"] = int(histogram["count"]) + 1 - histogram["sum"] = float(histogram["sum"]) + elapsed_seconds - buckets = histogram["buckets"] - for bucket in PRISM_VARDIFF_IDLE_SECONDS_BUCKETS: - if elapsed_seconds <= bucket: - buckets[bucket] = int(buckets.get(bucket, 0)) + 1 + self._ensure_vardiff_service().observe_idle_seconds(name, elapsed_seconds) def _idle_bundle_current_locked( self, @@ -23367,945 +8650,111 @@ def _build_idle_job_bundle( def _vardiff_idle_tip_divergence_locked(self) -> bool: """Whether detected tip work still lacks published submit authority.""" - published = getattr(self, "current_tip_first_seen", None) - latest_detected = getattr(self, "latest_detected_tip", None) - return bool( - latest_detected is not None - and (published is None or latest_detected[0] != published[0]) - ) + return self._ensure_vardiff_service().idle_tip_diverged_locked() def _idle_request_skip_reason( self, request: _IdleRetargetRequest, ) -> str | None: - client = request.client - # Take the per-client lock before coordinator admission. A share can - # delay this client's idle retarget, but it can never make the retarget - # hold the coordinator lock while waiting and convoy tip publication. - with self._client_vardiff_lock(client): - with self.lock: - if self._vardiff_idle_tip_divergence_locked(): - return "superseded" - if ( - client not in self.clients - or getattr(client, "closing", False) - or not self.client_can_receive_jobs(client) - ): - return "disconnected" - if ( - client.connection_id != request.connection_id - or client.worker != request.worker - or client.active_job is not request.active_job - or (client.pending_share_difficulty or client.share_difficulty) - != request.current_difficulty - ): - return "superseded" - if ( - client.vardiff_window_started_monotonic - != request.window_started_monotonic - or client.vardiff_window_accepted != 0 - or client.vardiff_window_submitted != 0 - ): - return "not_idle" - return None - - def _idle_request_pending(self, request: _IdleRetargetRequest) -> bool: - self._ensure_vardiff_idle_state() - with self._vardiff_idle_lock: - return ( - request.client, - request.connection_id, - ) in self._vardiff_idle_pending - - def _finish_idle_retarget_task( - self, - key: tuple[ClientState, int], - queued_monotonic: float, - *, - started: bool, - ) -> None: - self._ensure_vardiff_idle_state() - with self._vardiff_idle_lock: - if key not in self._vardiff_idle_pending: - return - self._vardiff_idle_pending.discard(key) - if started: - self.vardiff_idle_inflight = max(0, self.vardiff_idle_inflight - 1) - else: - self.vardiff_idle_queue_depth = max( - 0, - self.vardiff_idle_queue_depth - 1, - ) - self._observe_vardiff_idle_seconds( - "task", - max(0.0, time.monotonic() - queued_monotonic), - ) - - def _run_idle_retarget_task( - self, - request: _IdleRetargetRequest, - bundle: CachedJobBundle | None, - queued_monotonic: float, - ) -> None: - key = (request.client, request.connection_id) - self._ensure_vardiff_idle_state() - with self._vardiff_idle_lock: - if key not in self._vardiff_idle_pending: - return - self.vardiff_idle_queue_depth = max( - 0, - self.vardiff_idle_queue_depth - 1, - ) - self.vardiff_idle_inflight += 1 - client = request.client - delivery_attempted = False - try: - reason = self._idle_request_skip_reason(request) - if reason is not None: - self._record_vardiff_idle_skip(reason) - return - # Readiness may have crossed in the ledger after the sweep's - # cache-only snapshot. Refresh it on this bounded worker so a - # cached collection bundle cannot be delivered after the pool is - # ready for normal payout work. - self.pool_readiness_latched() - # Canonicalize the sweep's cache-only snapshot on the dedicated - # worker. shared_job_bundle() selects the current payout-artifact - # key and rebinds a ready heavy bundle to the latest same-tip - # template observation; a miss may build here, never on the sweep. - bundle = self._build_idle_job_bundle(request) - reason = self._idle_request_skip_reason(request) - if reason is not None: - self._record_vardiff_idle_skip(reason) - return - # Prepared bundles bypass _maybe_send_job_locked's normal build - # admission, so preserve its live reorg/headers/IBD trust guard on - # the dedicated worker before taking the client lock or sending. - if not self.ensure_reorg_reconciled_for_current_tip(): - self._record_vardiff_idle_skip("superseded") - return - if not client.job_update_lock.acquire(blocking=False): - self._record_vardiff_idle_skip("busy") - return - try: - reason = self._idle_request_skip_reason(request) - if reason is not None: - self._record_vardiff_idle_skip(reason) - return - with self._job_cache_lock: - bundle_current = self._idle_bundle_current_locked( - client, - bundle, - allow_uncached=True, - ) - if not bundle_current: - self._record_vardiff_idle_skip("superseded") - return - # Everything above this point is coordinator preparation. An - # OSError there belongs to qbit RPC/ledger I/O, not the miner - # socket. Only retire the connection after entering the paired - # client delivery path below. - delivery_attempted = True - retargeted = self._retarget_client_locked( - client, - current_difficulty=request.current_difficulty, - accepted_shares=0, - submitted_shares=0, - accepted_difficulty=Decimal("0"), - elapsed_seconds=request.elapsed_seconds, - require_idle=True, - prepared_bundle=bundle, - expected_connection_id=request.connection_id, - expected_worker=request.worker, - expected_active_job=request.active_job, - expected_window_started=request.window_started_monotonic, - prepared_bundle_allow_uncached=True, - ) - finally: - client.job_update_lock.release() - if retargeted: - with self.lock: - self.idle_retarget_count = int( - getattr(self, "idle_retarget_count", 0) - ) + 1 - return - reason = self._idle_request_skip_reason(request) - if reason is not None: - self._record_vardiff_idle_skip(reason) - return - with self._job_cache_lock: - bundle_current = self._idle_bundle_current_locked( - client, - bundle, - allow_uncached=True, - ) - if not bundle_current: - self._record_vardiff_idle_skip("superseded") - except JobBuildSuperseded: - self._record_vardiff_idle_skip("superseded") - except OSError: - with self._vardiff_idle_lock: - self.vardiff_idle_task_failures += 1 - if delivery_attempted: - self.disconnect_client(client) - return - print( - "prism coordinator: idle vardiff retarget preparation failed; " - "keeping client connected", - flush=True, - ) - traceback.print_exc() - except Exception: - with self._vardiff_idle_lock: - self.vardiff_idle_task_failures += 1 - print("prism coordinator: idle vardiff retarget task failed", flush=True) - traceback.print_exc() - - def _enqueue_idle_retarget( - self, - request: _IdleRetargetRequest, - bundle: CachedJobBundle | None, - ) -> str | None: - self._ensure_vardiff_idle_state() - key = (request.client, request.connection_id) - queued_monotonic = time.monotonic() - with self._vardiff_idle_lock: - if self._vardiff_idle_executor_shutdown or key in self._vardiff_idle_pending: - return "superseded" - if len(self._vardiff_idle_pending) >= MAX_PENDING_VARDIFF_IDLE_RETARGETS: - return "queue_full" - executor = self._vardiff_idle_executor - if executor is None: - executor = ThreadPoolExecutor( - max_workers=PRISM_VARDIFF_IDLE_RETARGET_MAX_WORKERS, - thread_name_prefix="prism-vardiff-idle", - ) - self._vardiff_idle_executor = executor - self._vardiff_idle_pending.add(key) - self.vardiff_idle_queue_depth += 1 - try: - future = executor.submit( - self._run_idle_retarget_task, - request, - bundle, - queued_monotonic, - ) - except RuntimeError: - self._vardiff_idle_pending.discard(key) - self.vardiff_idle_queue_depth = max( - 0, - self.vardiff_idle_queue_depth - 1, - ) - return "queue_full" - - def finish_task(completed: Future[None]) -> None: - self._finish_idle_retarget_task( - key, - queued_monotonic, - started=not completed.cancelled(), - ) - - future.add_done_callback(finish_task) - return None + return self._ensure_vardiff_service().request_skip_reason(request) def shutdown_vardiff_idle_executor(self) -> None: - self._ensure_vardiff_idle_state() - with self._vardiff_idle_lock: - executor = self._vardiff_idle_executor - self._vardiff_idle_executor = None - self._vardiff_idle_executor_shutdown = True - if executor is not None: - executor.shutdown(wait=True, cancel_futures=True) + self._ensure_vardiff_service().shutdown_idle_executor() def vardiff_idle_sweep_loop(self) -> None: - while not self.stop_event.wait(self.vardiff_idle_sweep_seconds): - self._record_heartbeat("vardiff_idle_sweep") - try: - queued = self.vardiff_idle_sweep_once() - if queued: - print( - f"prism coordinator: idle vardiff sweep queued {queued} client(s)", - flush=True, - ) - except Exception: - print("prism coordinator: idle vardiff sweep failed", flush=True) - traceback.print_exc() + self._ensure_vardiff_service().idle_sweep_loop() def vardiff_idle_sweep_once(self) -> int: - sweep_started = time.monotonic() - now = time.monotonic() - queued = 0 - try: - with self.lock: - clients = tuple(self.clients) - self._ensure_vardiff_idle_state() - with self._vardiff_idle_lock: - self.vardiff_idle_clients_inspected += len(clients) - for client in clients: - self._record_heartbeat("vardiff_idle_sweep") - with self._client_vardiff_lock(client), self.lock: - if self._vardiff_idle_tip_divergence_locked(): - reason = "superseded" - request = None - elif ( - client not in self.clients - or not self.client_can_receive_jobs(client) - ): - reason = "disconnected" - request = None - else: - active_job = client.active_job - worker = client.worker - config = ( - client.vardiff_config - or client.listener_vardiff_config - or self.vardiff_config - ) - if not config.enabled: - continue - if active_job is None or worker is None: - reason = "superseded" - request = None - else: - elapsed = Decimal( - str( - max( - 0.001, - now - - client.vardiff_window_started_monotonic, - ) - ) - ) - if ( - elapsed < config.retarget_interval_seconds - or client.vardiff_window_accepted != 0 - or client.vardiff_window_submitted != 0 - ): - reason = "not_idle" - request = None - else: - reason = None - request = _IdleRetargetRequest( - client=client, - connection_id=client.connection_id, - worker=worker, - active_job=active_job, - window_started_monotonic=( - client.vardiff_window_started_monotonic - ), - current_difficulty=( - client.pending_share_difficulty - or client.share_difficulty - ), - elapsed_seconds=elapsed, - ) - if reason is not None: - self._record_vardiff_idle_skip(reason) - continue - assert request is not None - if self._idle_request_pending(request): - self._record_vardiff_idle_skip("superseded") - continue - if not client.job_update_lock.acquire(blocking=False): - self._record_vardiff_idle_skip("busy") - continue - try: - reason = self._idle_request_skip_reason(request) - finally: - client.job_update_lock.release() - if reason is not None: - self._record_vardiff_idle_skip(reason) - continue - bundle = self._cached_idle_job_bundle(client) - if bundle is None: - # The sweep itself stays cache-only. A missing/expired - # bundle is rebuilt only by the dedicated bounded worker, - # so the client still makes eventual vardiff progress. - self._record_vardiff_idle_skip("cache_miss") - reason = self._enqueue_idle_retarget(request, bundle) - if reason is not None: - self._record_vardiff_idle_skip(reason) - continue - queued += 1 - return queued - finally: - self._record_heartbeat("vardiff_idle_sweep") - self._observe_vardiff_idle_seconds( - "sweep", - max(0.0, time.monotonic() - sweep_started), - ) + return self._ensure_vardiff_service().idle_sweep_once() def retarget_client( self, - client: ClientState, - *, - current_difficulty: Decimal, - accepted_shares: int, - submitted_shares: int, - accepted_difficulty: Decimal, - elapsed_seconds: Decimal, - require_idle: bool = False, - prepared_bundle: CachedJobBundle | None = None, - expected_connection_id: int | None = None, - expected_worker: WorkerIdentity | None = None, - expected_active_job: PrismJobContext | None = None, - expected_window_started: float | None = None, - ) -> bool: - acquired = client.job_update_lock.acquire(blocking=not require_idle) - if not acquired: - return False - try: - if require_idle and prepared_bundle is None: - prepared_bundle = self._cached_idle_job_bundle(client) - if prepared_bundle is None: - return False - return self._retarget_client_locked( - client, - current_difficulty=current_difficulty, - accepted_shares=accepted_shares, - submitted_shares=submitted_shares, - accepted_difficulty=accepted_difficulty, - elapsed_seconds=elapsed_seconds, - require_idle=require_idle, - prepared_bundle=prepared_bundle, - expected_connection_id=expected_connection_id, - expected_worker=expected_worker, - expected_active_job=expected_active_job, - expected_window_started=expected_window_started, - ) - finally: - client.job_update_lock.release() - - def _retarget_client_locked( - self, - client: ClientState, - *, - current_difficulty: Decimal, - accepted_shares: int, - submitted_shares: int, - accepted_difficulty: Decimal, - elapsed_seconds: Decimal, - require_idle: bool = False, - prepared_bundle: CachedJobBundle | None = None, - expected_connection_id: int | None = None, - expected_worker: WorkerIdentity | None = None, - expected_active_job: PrismJobContext | None = None, - expected_window_started: float | None = None, - prepared_bundle_allow_uncached: bool = False, - ) -> bool: - config = self.client_vardiff_config(client) - if not config.enabled: - return False - if require_idle: - if prepared_bundle is None: - return False - with self._client_vardiff_lock(client), self.lock: - if expected_connection_id is None: - expected_connection_id = client.connection_id - if expected_worker is None: - expected_worker = client.worker - if expected_active_job is None: - expected_active_job = client.active_job - if expected_window_started is None: - expected_window_started = client.vardiff_window_started_monotonic - if ( - client not in self.clients - or getattr(client, "closing", False) - or not self.client_can_receive_jobs(client) - or client.connection_id != expected_connection_id - or client.worker != expected_worker - or client.active_job is not expected_active_job - or client.vardiff_window_started_monotonic - != expected_window_started - or client.vardiff_window_accepted != 0 - or client.vardiff_window_submitted != 0 - ): - return False - observed_difficulty = vardiff.observed_difficulty( - accepted_difficulty=accepted_difficulty, - elapsed_seconds=elapsed_seconds, - target_share_interval_seconds=config.target_share_interval_seconds, - ) - with self._client_vardiff_lock(client): - previous_estimate = client.vardiff_difficulty_estimate - if observed_difficulty is None: - difficulty_estimate = None - with self._client_vardiff_lock(client): - client.vardiff_difficulty_estimate = None - else: - difficulty_estimate = vardiff.smooth_difficulty_estimate( - observed=observed_difficulty, - previous=previous_estimate, - config=config, - ) - with self._client_vardiff_lock(client): - client.vardiff_difficulty_estimate = difficulty_estimate - next_difficulty = vardiff.calculate_next_difficulty( - current_difficulty=current_difficulty, - accepted_shares=accepted_shares, - elapsed_seconds=elapsed_seconds, - config=config, - accepted_difficulty=accepted_difficulty, - difficulty_estimate=difficulty_estimate, - ) - if not vardiff.should_retarget( - current_difficulty, - next_difficulty, - config.retarget_tolerance, - ): - return False - idle_window_state: tuple[float, int, int, Decimal] | None = None - idle_window_reset_at: float | None = None - with self._client_vardiff_lock(client), self.lock: - previous_difficulty = client.pending_share_difficulty or client.share_difficulty - if previous_difficulty != current_difficulty: - return False - if require_idle and ( - client not in self.clients - or getattr(client, "closing", False) - or not self.client_can_receive_jobs(client) - or client.connection_id != expected_connection_id - or client.worker != expected_worker - or client.active_job is not expected_active_job - or client.vardiff_window_started_monotonic - != expected_window_started - or client.vardiff_window_accepted != 0 - or client.vardiff_window_submitted != 0 - ): - # A share landed since the idle snapshot; the accept path owns - # this window. Abort the speculative step-down rather than - # overriding a client that just resumed submitting. - return False - if require_idle: - idle_window_state = ( - client.vardiff_window_started_monotonic, - client.vardiff_window_accepted, - client.vardiff_window_submitted, - client.vardiff_window_work, - ) - prior_pending = client.pending_share_difficulty - client.pending_share_difficulty = next_difficulty - # Advertise the new difficulty only with its corresponding job. Idle - # retargets stamp an already-cached bundle; normal share-driven - # retargets retain the existing build path. Either path sends the pair - # together or restores the prior pending difficulty/window state. - def idle_commit_guard() -> bool: - nonlocal idle_window_reset_at - if not require_idle: - return True - # _maybe_send_job_locked holds vardiff_lock before entering the - # coordinator commit section, so this guard never waits under - # self.lock. - if ( - self.stop_event.is_set() - or client not in self.clients - or getattr(client, "closing", False) - or not self.client_can_receive_jobs(client) - or client.connection_id != expected_connection_id - or client.worker != expected_worker - or client.active_job is not expected_active_job - or client.vardiff_window_started_monotonic - != expected_window_started - or client.vardiff_window_accepted != 0 - or client.vardiff_window_submitted != 0 - or client.pending_share_difficulty != next_difficulty - ): - return False - idle_window_reset_at = time.monotonic() - client.vardiff_window_started_monotonic = idle_window_reset_at - client.vardiff_window_accepted = 0 - client.vardiff_window_submitted = 0 - client.vardiff_window_work = Decimal("0") - return True - - def restore_speculative_retarget() -> None: - with self._client_vardiff_lock(client), self.lock: - if client.pending_share_difficulty == next_difficulty: - client.pending_share_difficulty = prior_pending - self._restore_idle_window_state( - client, - idle_window_state, - idle_window_reset_at, - ) - - # A same-tip retarget only needs to flush in-flight work on a - # step-down: firmware that applies mining.set_difficulty retroactively - # would otherwise submit sub-target shares against the old job. On a - # step-up the old job stays valid at its own stamped share_target, so - # keeping it avoids discarding the miner's in-flight work. - clean_jobs = next_difficulty < current_difficulty - try: - if require_idle: - sent = self._maybe_send_job_locked( - client, - clean_jobs=clean_jobs, - raise_on_build_failure=True, - prepared_bundle=prepared_bundle, - commit_guard=idle_commit_guard, - commit_guard_lock=self._client_vardiff_lock(client), - prepared_bundle_allow_uncached=( - prepared_bundle_allow_uncached - ), - ) - else: - sent = bool( - client.authorized - and client.subscribed - and not self.stop_event.is_set() - and self.maybe_send_job(client, clean_jobs=clean_jobs) - ) - # A completed paired send is the commit point. Shutdown may race - # immediately afterward, but it cannot make already-delivered work - # speculative again. - if sent: - return True - except Exception: - # Cached stamping can surface _JobBuildFailed before delivery, and - # socket errors can surface during the paired send. Both must undo - # every speculative client mutation before the task reports failure. - restore_speculative_retarget() - raise - restore_speculative_retarget() - return False - - @staticmethod - def _restore_idle_window_state( - client: ClientState, - idle_window_state: tuple[float, int, int, Decimal] | None, - idle_window_reset_at: float | None, - ) -> None: - """Un-restart the idle vardiff window after a step-down that never - reached the miner (skipped build/send), so the next sweep can retry - immediately instead of waiting out another full retarget interval. - Caller must hold the client's vardiff_lock. No-op unless this retarget did the reset - and nothing else has restarted the window since.""" - if idle_window_reset_at is None or idle_window_state is None: - return - if ( - client.vardiff_window_started_monotonic == idle_window_reset_at - and client.vardiff_window_accepted == 0 - and client.vardiff_window_submitted == 0 - and client.vardiff_window_work == 0 - ): - ( - client.vardiff_window_started_monotonic, - client.vardiff_window_accepted, - client.vardiff_window_submitted, - client.vardiff_window_work, - ) = idle_window_state + client: ClientState, + *, + current_difficulty: Decimal, + accepted_shares: int, + submitted_shares: int, + accepted_difficulty: Decimal, + elapsed_seconds: Decimal, + require_idle: bool = False, + prepared_bundle: CachedJobBundle | None = None, + expected_connection_id: int | None = None, + expected_worker: WorkerIdentity | None = None, + expected_active_job: PrismJobContext | None = None, + expected_window_started: float | None = None, + initial_convergence: bool = False, + ) -> bool: + return self._ensure_vardiff_service().retarget( + client, + current_difficulty=current_difficulty, + accepted_shares=accepted_shares, + submitted_shares=submitted_shares, + accepted_difficulty=accepted_difficulty, + elapsed_seconds=elapsed_seconds, + require_idle=require_idle, + prepared_bundle=prepared_bundle, + expected_connection_id=expected_connection_id, + expected_worker=expected_worker, + expected_active_job=expected_active_job, + expected_window_started=expected_window_started, + initial_convergence=initial_convergence, + ) def enqueue_block_candidate(self, candidate: PrismBlockCandidate) -> bool: - # Outstanding from admission (even when the wakeup coalesces below: - # the durable outbox row keeps the candidate replayable), so a tip - # observation can register acceptance before the submitter drains it. - self._register_outstanding_block_candidate( - str(candidate.submission.block_hash_hex) - ) - queue_obj = getattr(self, "block_candidate_queue", None) - if queue_obj is None: - queue_obj = queue.Queue(maxsize=MAX_PENDING_BLOCK_CANDIDATES) - self.block_candidate_queue = queue_obj - try: - queue_obj.put_nowait(candidate) - return True - except queue.Full: - # The candidate is already durable. A full queue merely coalesces - # this wakeup; the submitter re-reads pending outbox rows whenever - # it drains the queue, so no candidate is discarded. - with self.lock: - self.block_candidate_wakeups_coalesced = int( - getattr(self, "block_candidate_wakeups_coalesced", 0) - ) + 1 - print( - "prism coordinator: block candidate wakeup coalesced " - f"hash={candidate.submission.block_hash_hex} (submitter queue full)", - flush=True, - ) - return False + return self._ensure_block_candidate_service().enqueue(candidate) def _ensure_block_replay_state(self) -> None: """Backfill replay/maintenance queues for lightweight coordinators.""" - with _HOT_PATH_LOCK_INITIALIZATION_LOCK: - if not hasattr(self, "_block_replay_candidate_queue"): - self._block_replay_candidate_queue = queue.Queue() - if not hasattr(self, "_block_replay_inflight_hashes"): - self._block_replay_inflight_hashes: set[str] = set() - if not hasattr(self, "_block_quarantine_queue"): - self._block_quarantine_queue = queue.Queue() - if not hasattr(self, "_block_quarantine_hashes"): - self._block_quarantine_hashes: set[str] = set() + self._ensure_block_candidate_service()._ensure_block_replay_state() def _enqueue_replayed_block_candidate( self, candidate: PrismBlockCandidate, ) -> bool: """Queue one durable replay behind live solves, once per process.""" - self._ensure_block_replay_state() - block_hash = str(candidate.submission.block_hash_hex).lower() - with self.lock: - if ( - block_hash in self._block_replay_inflight_hashes - or block_hash - in getattr(self, "_block_candidate_terminal_outcomes", {}) - ): - return False - self._block_replay_inflight_hashes.add(block_hash) - try: - # This is an in-memory condition update, not accounting. Install - # it before making the candidate visible to the raw lane so - # startup prewarm cannot build a child from the old payout base - # after qbitd accepts but before the RPC response returns. - self._begin_accepted_block_payout_preview( - block_hash, - block_height=int(candidate.context.template["height"]), - ) - self._block_replay_candidate_queue.put_nowait(candidate) - except BaseException: - with self.lock: - self._block_replay_inflight_hashes.discard(block_hash) - raise - return True + return self._ensure_block_candidate_service()._enqueue_replayed_block_candidate( + candidate + ) def _queue_invalid_block_candidate_for_quarantine( self, block_hash: str, error: str, + *, + pending_share: PendingShare | None = None, ) -> None: """Move malformed-row cleanup off the node-offer lane.""" - if not block_hash: - return - self._ensure_block_replay_state() - key = block_hash.lower() - with self.lock: - if key in self._block_quarantine_hashes: - return - self._block_quarantine_hashes.add(key) - self._block_quarantine_queue.put_nowait((key, error)) + self._ensure_block_candidate_service()._queue_invalid_block_candidate_for_quarantine( + block_hash, + error, + pending_share=pending_share, + ) @ledger_writer_operation("accepted_block_handling") def replay_pending_block_candidates(self) -> int: """Queue durable candidate intents not completed by an earlier process.""" - self._record_block_submitter_phase("replay-check-memory") - # While startup enumeration is still owed, correctness requires the - # outbox query even if live candidates are queued: job builds stay - # blocked until pending candidates are known, and only a successful - # enumeration can unblock them. - enumeration_owed = self._block_replay_enumeration_owed() - with self.lock: - if ( - not enumeration_owed - and getattr(self, "_retry_block_candidate", None) is not None - ): - return 0 - # A live wakeup is already the lowest-latency route to qbitd. Never - # park it behind the outbox query that exists only to recover missing - # wakeups after queue pressure or restart. - queue_obj = getattr(self, "block_candidate_queue", None) - if not enumeration_owed and queue_obj is not None and not queue_obj.empty(): - return 0 - self._ensure_block_replay_state() - if not enumeration_owed and not self._block_replay_candidate_queue.empty(): - return 0 - # The startup enumeration gates job issuance, so it runs with the - # landing-class budget instead of the poll budget (issue #188 fix 4); - # the periodic steady-state poll keeps the tight budget. The metrics - # class follows the budget so a slow or timed-out startup enumeration - # surfaces on the landing series the alerts watch. - replay_query_timeout = ( - self._block_landing_db_timeout() if enumeration_owed else None - ) - replay_query_call_class = "landing" if enumeration_owed else "fast" - pending_rows = getattr(self.ledger, "pending_block_candidate_rows", None) - if callable(pending_rows): - - def fetch_durable_rows(limit: int) -> list[Any]: - return self._run_block_submitter_ledger_call( - ("replay-outbox-query", limit), - "replay-outbox-query", - # Restore a batch with no per-row database work. In-flight - # dedupe lets later rows reach qbitd even while the oldest - # candidate is still accounting. - lambda: pending_rows(limit=limit), - timeout_seconds=replay_query_timeout, - call_class=replay_query_call_class, - ) - - else: - pending = getattr(self.ledger, "pending_block_candidates", None) - if not callable(pending): - self._clear_block_replay_enumeration_owed() - return 0 - - def fetch_durable_rows(limit: int) -> list[Any]: - pending_intents = self._run_block_submitter_ledger_call( - ("replay-outbox-query", limit), - "replay-outbox-query", - lambda: pending(limit=limit), - timeout_seconds=replay_query_timeout, - call_class=replay_query_call_class, - ) - return [ - { - "block_hash": ( - intent.get("block_hash_hex", "") - if isinstance(intent, dict) - else "" - ), - "candidate": intent, - } - for intent in pending_intents - ] - - queued = 0 - enumeration_truncated = False - enumeration_limit = MAX_PENDING_BLOCK_CANDIDATES - while True: - durable_rows = fetch_durable_rows(enumeration_limit) - self._record_block_submitter_phase("replay-restore") - for durable_row in durable_rows: - durable_block_hash = "" - try: - if not isinstance(durable_row, dict): - raise ValueError("durable block candidate row is not an object") - durable_block_hash = str(durable_row["block_hash"]).lower() - intent = durable_row["candidate"] - if not isinstance(intent, dict): - raise ValueError("durable block candidate intent is not an object") - intent_block_hash = str(intent.get("block_hash_hex", "")).lower() - if not durable_block_hash or intent_block_hash != durable_block_hash: - raise ValueError("durable block candidate row key does not match intent") - candidate = dataclass_replace( - self.block_candidate_from_intent(intent), - durable_replay=True, - ) - # Durable acceptance-state reads stay in accounting. The - # separate replay queue keeps these recovered rows behind any - # live solve while still exposing the whole batch to qbitd. - if self._enqueue_replayed_block_candidate(candidate): - queued += 1 - except Exception: - print("prism coordinator: invalid durable block candidate intent", flush=True) - traceback.print_exc() - self._queue_invalid_block_candidate_for_quarantine( - durable_block_hash, - "invalid durable candidate intent", - ) - if len(durable_rows) < enumeration_limit or not enumeration_owed: - break - # A full batch may hide more pending rows, and a hidden row could - # be the active parent whose carry a child job must not omit. - # Re-query with a doubled window until the result is provably - # untruncated; in-flight dedupe makes re-seen rows free. - if enumeration_limit >= MAX_BLOCK_REPLAY_ENUMERATION_ROWS: - enumeration_truncated = True - print( - "prism coordinator: pending block candidate enumeration " - f"still truncated at {enumeration_limit} rows; job builds " - "stay blocked until a complete enumeration succeeds", - flush=True, - ) - break - enumeration_limit = min( - enumeration_limit * 2, - MAX_BLOCK_REPLAY_ENUMERATION_ROWS, - ) - if not enumeration_truncated: - # Every pending candidate is now known and its payout barrier armed - # (or quarantined), so child job builds may proceed. A truncated - # pass instead leaves enumeration owed: the queued batch drains, - # and the submitter loop re-enumerates the remainder. - self._clear_block_replay_enumeration_owed() - self._record_startup_phase_once("block_replay_enumerated") - if queued: - print( - f"prism coordinator: replayed {queued} pending block candidate(s)", - flush=True, - ) - return queued + return self._ensure_block_candidate_service().replay_pending() def _ensure_block_submitter_retry_state(self) -> None: - if not hasattr(self, "_block_submitter_retry_state_lock"): - self._block_submitter_retry_state_lock = threading.Lock() - if not hasattr(self, "_block_submitter_backoff_started_monotonic"): - self._block_submitter_backoff_started_monotonic = None - if not hasattr(self, "_block_submitter_backoff_deadline_monotonic"): - self._block_submitter_backoff_deadline_monotonic = None - if not hasattr(self, "_block_submitter_backoff_delay_seconds"): - self._block_submitter_backoff_delay_seconds = 0.0 + self._ensure_block_candidate_service() def _ensure_block_submitter_ledger_call_state(self) -> None: - if not hasattr(self, "_block_submitter_ledger_calls_lock"): - self._block_submitter_ledger_calls_lock = threading.Lock() - if not hasattr(self, "_block_submitter_ledger_calls"): - self._block_submitter_ledger_calls = {} - if not hasattr(self, "_block_submitter_ledger_worker_slots"): - self._block_submitter_ledger_worker_slots = threading.BoundedSemaphore( - MAX_BLOCK_SUBMITTER_STUCK_CALL_WORKERS - ) + self._ensure_block_candidate_service()._ensure_block_submitter_ledger_call_state() def _block_submitter_db_timeout(self) -> float: - return max( - 0.001, - float( - getattr( - self, - "block_submit_db_timeout_seconds", - DEFAULT_BLOCK_SUBMIT_DB_TIMEOUT_SECONDS, - ) - ), - ) + return self._ensure_block_candidate_service()._block_submitter_db_timeout() def _block_landing_db_timeout(self, block_hash: str | None = None) -> float: - """Landing-class deadline, escalated after observed landing timeouts. - - The first attempt already receives the full landing budget; a known - landing-class operation never begins at the one-second poll budget. - Escalation doubles per timed-out landing attempt for the same block - hash up to the reviewed cap. - """ - base = max( - 0.001, - float( - getattr( - self, - "block_landing_db_timeout_seconds", - DEFAULT_BLOCK_LANDING_DB_TIMEOUT_SECONDS, - ) - ), - ) - cap = max( - base, - float( - getattr( - self, - "block_landing_db_timeout_max_seconds", - DEFAULT_BLOCK_LANDING_DB_TIMEOUT_MAX_SECONDS, - ) - ), + """Landing-class deadline, escalated after observed landing timeouts.""" + return self._ensure_block_candidate_service()._block_landing_db_timeout( + block_hash ) - timeouts = 0 - if block_hash is not None: - with self.lock: - counts = getattr(self, "_block_landing_timeout_counts", None) - if counts is not None: - timeouts = int(counts.get(block_hash, 0)) - return min(cap, base * (2.0 ** min(timeouts, 8))) def _note_block_landing_timeout(self, block_hash: str | None) -> None: - if block_hash is None: - return - with self.lock: - counts = getattr(self, "_block_landing_timeout_counts", None) - if counts is None: - counts = {} - self._block_landing_timeout_counts = counts - counts[block_hash] = int(counts.get(block_hash, 0)) + 1 + self._ensure_block_candidate_service()._note_block_landing_timeout(block_hash) def _ensure_block_ledger_call_metrics(self) -> None: - if not hasattr(self, "_block_ledger_call_metrics_lock"): - self._block_ledger_call_metrics_lock = threading.Lock() - if not hasattr(self, "_block_ledger_call_metrics"): - self._block_ledger_call_metrics = {} + self._ensure_block_candidate_service()._ensure_block_ledger_call_metrics() def _record_block_ledger_call( self, @@ -24316,46 +8765,21 @@ def _record_block_ledger_call( timed_out: bool, ) -> None: """Track per-call-class submitter ledger latency and timeout counts.""" - self._ensure_block_ledger_call_metrics() - with self._block_ledger_call_metrics_lock: - stats = self._block_ledger_call_metrics.setdefault( - call_class, - { - "calls_total": 0, - "timeouts_total": 0, - "last_budget_seconds": 0.0, - "last_duration_seconds": 0.0, - "max_duration_seconds": 0.0, - }, - ) - stats["calls_total"] = int(stats["calls_total"]) + 1 - if timed_out: - stats["timeouts_total"] = int(stats["timeouts_total"]) + 1 - stats["last_budget_seconds"] = float(budget_seconds) - stats["last_duration_seconds"] = float(duration_seconds) - stats["max_duration_seconds"] = max( - float(stats["max_duration_seconds"]), float(duration_seconds) - ) + self._ensure_block_candidate_service()._record_block_ledger_call( + call_class=call_class, + budget_seconds=budget_seconds, + duration_seconds=duration_seconds, + timed_out=timed_out, + ) def block_ledger_call_class_metrics(self) -> dict[str, dict[str, float | int]]: - self._ensure_block_ledger_call_metrics() - with self._block_ledger_call_metrics_lock: - return { - call_class: dict(stats) - for call_class, stats in self._block_ledger_call_metrics.items() - } + return self._ensure_block_candidate_service().block_ledger_call_class_metrics() + + def block_candidate_collapse_snapshot(self) -> dict[str, int]: + return self._ensure_block_candidate_service().block_candidate_collapse_snapshot() def _block_submitter_stuck_call_exit_timeout(self) -> float: - return max( - 0.001, - float( - getattr( - self, - "block_submit_stuck_call_exit_seconds", - DEFAULT_BLOCK_SUBMIT_STUCK_CALL_EXIT_SECONDS, - ) - ), - ) + return self._ensure_block_candidate_service()._block_submitter_stuck_call_exit_timeout() def _maybe_restart_for_stuck_block_call( self, @@ -24364,21 +8788,10 @@ def _maybe_restart_for_stuck_block_call( started_monotonic: float, ) -> None: """Fail stop when a poisoned worker pool stays exhausted.""" - age_seconds = max(0.0, time.monotonic() - started_monotonic) - exit_seconds = self._block_submitter_stuck_call_exit_timeout() - if age_seconds < exit_seconds: - return - stop_event = getattr(self, "stop_event", None) - if stop_event is not None and stop_event.is_set(): - return - print( - "prism coordinator: block work call remained stuck; requesting " - f"restart kind={kind} age={age_seconds:.3f}s " - f"budget={exit_seconds:g}s", - flush=True, + self._ensure_block_candidate_service()._maybe_restart_for_stuck_block_call( + kind=kind, + started_monotonic=started_monotonic, ) - self._fatal_exit_requested = True - self.request_shutdown() def _maybe_restart_for_exhausted_block_call_pool( self, @@ -24388,93 +8801,33 @@ def _maybe_restart_for_exhausted_block_call_pool( calls: dict[Any, Any], ) -> None: """Age an exhausted pool even when retries reuse existing calls.""" - with calls_lock: - active_starts = [ - pending.started_monotonic - for pending in calls.values() - if not pending.done.is_set() - ] - if len(active_starts) < MAX_BLOCK_SUBMITTER_STUCK_CALL_WORKERS: - return - self._maybe_restart_for_stuck_block_call( + self._ensure_block_candidate_service()._maybe_restart_for_exhausted_block_call_pool( kind=kind, - started_monotonic=min(active_starts), + calls_lock=calls_lock, + calls=calls, ) - @contextmanager def _block_submitter_ledger_timeout_scope( self, timeout_seconds: float | None = None, ) -> Iterator[None]: """Apply the submitter's PostgreSQL deadline when the ledger supports it.""" - operation_timeout = getattr(self.ledger, "operation_timeout", None) - if not callable(operation_timeout): - yield - return - with operation_timeout( - self._block_submitter_db_timeout() - if timeout_seconds is None - else timeout_seconds - ): - yield + return self._ensure_block_candidate_service()._block_submitter_ledger_timeout_scope( + timeout_seconds + ) - @contextmanager def _block_submitter_ledger_statement_timeout_scope(self) -> Iterator[None]: """Give each post-submit ledger step a fresh short deadline.""" - statement_timeout = getattr(self.ledger, "statement_timeout", None) - if callable(statement_timeout): - with statement_timeout(self._block_submitter_db_timeout()): - yield - return - # Duck-typed ledgers predating per-statement scopes still receive a - # bounded operation, even though their budget spans the whole tail. - with self._block_submitter_ledger_timeout_scope(): - yield + return self._ensure_block_candidate_service()._block_submitter_ledger_statement_timeout_scope() - @contextmanager def _block_landing_ledger_statement_timeout_scope( self, block_hash: str | None = None, ) -> Iterator[None]: - """Give each landing-class ledger step the landing budget. - - The accounting tail that lands an accepted block (persist, prior - balances, confirm, and the prepared-state rejection of a terminal - candidate) runs under this scope instead of the poll-class one. - Timed-out steps are recorded so the next attempt for the same block - hash escalates its budget; the ledger backends already guarantee - server-side cancellation completes and the pooled session is rolled - back or replaced before the paced retry re-enters here. - - The guarded body also runs node RPCs and audit/build work. Only a - ledger-originated deadline may escalate the next landing budget or - fire the landing-timeout alert: a node RPC timeout is not a database - cancellation, and escalating on it would page and widen PostgreSQL - deadlines for a database that never missed one. - """ - timeout_seconds = self._block_landing_db_timeout(block_hash) - scope = getattr(self.ledger, "statement_timeout", None) - if not callable(scope): - scope = getattr(self.ledger, "operation_timeout", None) - started = time.monotonic() - timed_out = False - try: - if callable(scope): - with scope(timeout_seconds): - yield - else: - yield - except (LedgerOperationTimeout, BlockSubmitterDatabaseTimeout): - timed_out = True - self._note_block_landing_timeout(block_hash) - raise - finally: - self._record_block_ledger_call( - call_class="landing", - budget_seconds=timeout_seconds, - duration_seconds=max(0.0, time.monotonic() - started), - timed_out=timed_out, - ) + """Give each landing-class ledger step the landing budget.""" + return self._ensure_block_candidate_service()._block_landing_ledger_statement_timeout_scope( + block_hash + ) def _run_block_submitter_ledger_call( self, @@ -24485,155 +8838,27 @@ def _run_block_submitter_ledger_call( timeout_seconds: float | None = None, call_class: str = "fast", ) -> Any: - """Run one direct outbox call without letting its driver wedge us. - - A timed-out call remains registered and is reused by the next paced - retry. This bounds the coordinator-side wait without spawning an - unbounded pile of threads when a fake/misbehaving driver ignores the - real PostgreSQL statement deadline. Candidate outbox mutations are - idempotent, so a late completion converges with replay. - - call_class labels the per-class latency/timeout metrics and must - match the budget in use: a call given the landing deadline records - as "landing" so the landing-timeout alert covers it, instead of - inflating the fast-call budget gauge. - """ - if timeout_seconds is None: - timeout_seconds = self._block_submitter_db_timeout() - else: - timeout_seconds = max(0.001, float(timeout_seconds)) - self._ensure_block_submitter_ledger_call_state() - with self._block_submitter_ledger_calls_lock: - call = self._block_submitter_ledger_calls.get(key) - if call is None: - if not self._block_submitter_ledger_worker_slots.acquire( - blocking=False - ): - oldest_started = min( - ( - pending.started_monotonic - for pending in self._block_submitter_ledger_calls.values() - if not pending.done.is_set() - ), - default=time.monotonic(), - ) - self._maybe_restart_for_stuck_block_call( - kind="ledger-worker-pool", - started_monotonic=oldest_started, - ) - raise BlockSubmitterDatabaseTimeout( - f"{phase} could not acquire a bounded ledger worker" - ) - call = _BlockSubmitterLedgerCall() - self._block_submitter_ledger_calls[key] = call - - def run() -> None: - try: - with self._block_submitter_ledger_timeout_scope( - timeout_seconds - ): - call.result = operation() - except BaseException as exc: - call.error = exc - finally: - call.done.set() - self._block_submitter_ledger_worker_slots.release() - - threading.Thread( - target=run, - name=f"prism-block-ledger-{phase}", - daemon=True, - ).start() - - deadline = time.monotonic() + timeout_seconds - while not call.done.is_set(): - self._record_block_submitter_wait(phase) - remaining = deadline - time.monotonic() - if remaining <= 0: - self._maybe_restart_for_exhausted_block_call_pool( - kind="ledger-worker-pool", - calls_lock=self._block_submitter_ledger_calls_lock, - calls=self._block_submitter_ledger_calls, - ) - print( - "prism coordinator: block submitter ledger phase timed out " - f"phase={phase} timeout={timeout_seconds:g}s", - flush=True, - ) - self._record_block_ledger_call( - call_class=call_class, - budget_seconds=timeout_seconds, - duration_seconds=timeout_seconds, - timed_out=True, - ) - raise BlockSubmitterDatabaseTimeout( - f"{phase} exceeded {timeout_seconds:g}s" - ) - call.done.wait( - min( - remaining, - self._block_work_wait_slice(), - ) - ) - self._record_block_submitter_wait(f"{phase}:complete") - # A server-side deadline normally completes the worker with a ledger - # timeout error before the coordinator-side wait expires. Every - # operation behind this wrapper is a database call, so a completed - # call carrying a timeout error is still a timed-out call for the - # per-class alert series. - self._record_block_ledger_call( + """Run one direct outbox call without letting its driver wedge us.""" + return self._ensure_block_candidate_service()._run_block_submitter_ledger_call( + key, + phase, + operation, + timeout_seconds=timeout_seconds, call_class=call_class, - budget_seconds=timeout_seconds, - duration_seconds=max(0.0, time.monotonic() - call.started_monotonic), - timed_out=isinstance(call.error, TimeoutError), ) - with self._block_submitter_ledger_calls_lock: - if self._block_submitter_ledger_calls.get(key) is call: - self._block_submitter_ledger_calls.pop(key, None) - if call.error is not None: - raise call.error - return call.result def _wait_for_block_candidate_retry(self, delay_seconds: float) -> bool: """Wait for intentional backoff without impersonating stuck work. - Retry waits heartbeat in bounded slices. Direct outbox calls and lock - admission use the same phase-aware pattern; work that is not covered - by an explicit deadline remains watchdog-eligible. + Only this bounded retry wait refreshes the submitter heartbeat by + itself; SQL, RPC and lock phases go through their own phase-aware + helpers, so a genuinely blocked candidate phase remains + watchdog-eligible. """ - delay_seconds = max(0.0, float(delay_seconds)) - if delay_seconds <= 0: - return self.stop_event.is_set() - self._ensure_block_submitter_retry_state() - started = time.monotonic() - with self._block_submitter_retry_state_lock: - self._block_submitter_backoff_started_monotonic = started - self._block_submitter_backoff_deadline_monotonic = started + delay_seconds - self._block_submitter_backoff_delay_seconds = delay_seconds - remaining = delay_seconds - try: - while remaining > 0: - self._record_block_submitter_wait("retry-backoff") - wait_slice = min(remaining, self._block_work_wait_slice()) - if self.stop_event.wait(wait_slice): - return True - remaining = max(0.0, remaining - wait_slice) - self._record_block_submitter_wait("retry-backoff:complete") - return False - finally: - with self._block_submitter_retry_state_lock: - self._block_submitter_backoff_started_monotonic = None - self._block_submitter_backoff_deadline_monotonic = None - self._block_submitter_backoff_delay_seconds = 0.0 + return self._ensure_block_candidate_service().wait_for_retry(delay_seconds) def _mark_block_candidate_attempted(self, block_hash: str) -> None: - mark_attempted = getattr(self.ledger, "mark_block_candidate_attempted", None) - if callable(mark_attempted): - self._run_block_submitter_ledger_call( - ("mark-attempted", block_hash), - "mark-attempted", - lambda: mark_attempted(block_hash=block_hash), - ) + self._ensure_block_candidate_service().mark_attempted(block_hash) def _rpc_call_with_timeout( self, @@ -24643,31 +8868,14 @@ def _rpc_call_with_timeout( timeout_seconds: float, ) -> Any: """Pass an explicit timeout to production RPCs and capable test doubles.""" - call = self.rpc.call - supports_timeout = isinstance(self.rpc, JsonRpc) - if not supports_timeout: - try: - parameters = inspect.signature(call).parameters.values() - supports_timeout = any( - parameter.name == "timeout" - or parameter.kind is inspect.Parameter.VAR_KEYWORD - for parameter in parameters - ) - except (TypeError, ValueError): - supports_timeout = False - if supports_timeout: - return call(method, params, timeout=timeout_seconds) - return call(method, params) + return self._ensure_block_candidate_service()._rpc_call_with_timeout( + method, + params, + timeout_seconds=timeout_seconds, + ) def _ensure_block_submitter_rpc_call_state(self) -> None: - if not hasattr(self, "_block_submitter_rpc_calls_lock"): - self._block_submitter_rpc_calls_lock = threading.Lock() - if not hasattr(self, "_block_submitter_rpc_calls"): - self._block_submitter_rpc_calls = {} - if not hasattr(self, "_block_submitter_rpc_worker_slots"): - self._block_submitter_rpc_worker_slots = threading.BoundedSemaphore( - MAX_BLOCK_SUBMITTER_STUCK_CALL_WORKERS - ) + self._ensure_block_candidate_service()._ensure_block_submitter_rpc_call_state() def _run_submitblock_rpc_with_hard_deadline( self, @@ -24677,72 +8885,11 @@ def _run_submitblock_rpc_with_hard_deadline( timeout_seconds: float, ) -> Any: """Bound wall time even when an RPC adapter ignores its timeout.""" - self._ensure_block_submitter_rpc_call_state() - with self._block_submitter_rpc_calls_lock: - call = self._block_submitter_rpc_calls.get(block_hash) - if call is None: - if not self._block_submitter_rpc_worker_slots.acquire( - blocking=False - ): - oldest_started = min( - ( - pending.started_monotonic - for pending in self._block_submitter_rpc_calls.values() - if not pending.done.is_set() - ), - default=time.monotonic(), - ) - self._maybe_restart_for_stuck_block_call( - kind="rpc-worker-pool", - started_monotonic=oldest_started, - ) - raise TimeoutError( - "submitblock could not acquire a bounded RPC worker" - ) - call = _BlockSubmitterRpcCall() - self._block_submitter_rpc_calls[block_hash] = call - - def run() -> None: - try: - call.result = self._rpc_call_with_timeout( - "submitblock", - [block_hex], - timeout_seconds=timeout_seconds, - ) - except BaseException as exc: - call.error = exc - finally: - call.done.set() - self._block_submitter_rpc_worker_slots.release() - - threading.Thread( - target=run, - name=f"prism-block-rpc-{block_hash[:12]}", - daemon=True, - ).start() - - deadline = time.monotonic() + timeout_seconds - while not call.done.is_set(): - self._record_block_submitter_wait("submitblock-rpc") - remaining = deadline - time.monotonic() - if remaining <= 0: - self._maybe_restart_for_exhausted_block_call_pool( - kind="rpc-worker-pool", - calls_lock=self._block_submitter_rpc_calls_lock, - calls=self._block_submitter_rpc_calls, - ) - raise TimeoutError( - f"submitblock exceeded {timeout_seconds:g}s" - ) - call.done.wait( - min(remaining, self._block_work_wait_slice()) - ) - with self._block_submitter_rpc_calls_lock: - if self._block_submitter_rpc_calls.get(block_hash) is call: - self._block_submitter_rpc_calls.pop(block_hash, None) - if call.error is not None: - raise call.error - return call.result + return self._ensure_block_candidate_service()._run_submitblock_rpc_with_hard_deadline( + block_hash=block_hash, + block_hex=block_hex, + timeout_seconds=timeout_seconds, + ) def _arm_block_candidate_after_node_offer( self, @@ -24750,144 +8897,46 @@ def _arm_block_candidate_after_node_offer( node_submission: _BlockCandidateNodeSubmission, ) -> None: """Fence child payout work as soon as node acceptance is possible.""" - self._stash_retained_block_candidate_node_submission( - str(candidate.submission.block_hash_hex), + self._ensure_block_candidate_service()._arm_block_candidate_after_node_offer( + candidate, node_submission, ) - ambiguous_or_landed = ( - node_submission.error is not None - or node_submission.result in (None, "duplicate") - ) - if not ambiguous_or_landed: - self._release_block_fast_lane_slot( - str(candidate.submission.block_hash_hex) - ) - return - block_hash = str(candidate.submission.block_hash_hex).lower() - expected_height = int(candidate.context.template["height"]) - self._begin_accepted_block_payout_preview( - block_hash, - block_height=expected_height, - ) def _submit_block_candidate_to_node( self, candidate: PrismBlockCandidate, ) -> _BlockCandidateNodeSubmission: """Offer the durable candidate to qbitd before any accounting work.""" - block_hash = str(candidate.submission.block_hash_hex).lower() - self._begin_accepted_block_payout_preview( - block_hash, - block_height=int(candidate.context.template["height"]), - ) - self._register_outstanding_block_candidate(block_hash) - self._record_block_submitter_phase("submitblock-rpc") - timeout_seconds = max( - 0.001, - float( - getattr( - self, - "block_submit_rpc_timeout_seconds", - DEFAULT_BLOCK_SUBMIT_RPC_TIMEOUT_SECONDS, - ) - ), - ) - try: - result = self._run_submitblock_rpc_with_hard_deadline( - block_hash=block_hash, - block_hex=str(candidate.submission.block_hex), - timeout_seconds=timeout_seconds, - ) - except BaseException as exc: - self._record_block_submitter_phase("submitblock-rpc:error") - node_submission = _BlockCandidateNodeSubmission( - attempted=True, - error=exc, - ) - self._arm_block_candidate_after_node_offer(candidate, node_submission) - return node_submission - self._record_block_submitter_phase("submitblock-rpc:complete") - landed_monotonic = getattr(candidate, "landed_monotonic", None) - if landed_monotonic is not None: - self._observe_block_submit_seconds( - time.monotonic() - float(landed_monotonic) - ) - node_submission = _BlockCandidateNodeSubmission( - attempted=True, - result=result, + return self._ensure_block_candidate_service()._submit_block_candidate_to_node( + candidate ) - self._arm_block_candidate_after_node_offer(candidate, node_submission) - return node_submission def _node_submission_for_candidate( self, candidate: PrismBlockCandidate, ) -> _BlockCandidateNodeSubmission: """Choose the node fast lane unless the pool was already closed.""" - block_hash = str(candidate.submission.block_hash_hex).lower() - self._record_block_submitter_phase("fast-lane-admission") - with self.lock: - accounted_hashes = getattr( - self, - "_accounted_accepted_block_hashes", - set(), - ) - accepted_count = int(self.accepted_block_count) - pool_closed = block_hash not in accounted_hashes and ( - accepted_count >= int(self.max_blocks) - or (bool(self.stop_after_block) and accepted_count >= 1) - ) - if pool_closed: - return _BlockCandidateNodeSubmission(attempted=False) - return self._submit_block_candidate_to_node(candidate) + return self._ensure_block_candidate_service()._node_submission_for_candidate( + candidate + ) def _node_submission_for_candidate_or_retained( self, candidate: PrismBlockCandidate, ) -> _BlockCandidateNodeSubmission: - """Reuse a retained definitive acceptance instead of re-offering. - - An in-process retry of a candidate whose earlier offer already - returned success must not ask the node again: the re-offer answers - "duplicate", which downgrades the classification to the moved live - tip and leans on chain probes that may be unavailable under the - same saturation that caused the retry. The stashed result reruns - the landing tail as if the first pass had continued. - """ - retained = self._retained_block_candidate_node_submission( - str(candidate.submission.block_hash_hex) + """Reuse a retained definitive acceptance instead of re-offering.""" + return self._ensure_block_candidate_service()._node_submission_for_candidate_or_retained( + candidate ) - if retained is not None: - return retained - return self._node_submission_for_candidate(candidate) def _node_submission_for_direct_candidate( self, candidate: PrismBlockCandidate, ) -> _BlockCandidateNodeSubmission: - """Preserve active-replay semantics for non-queue embedders. - - The dedicated submitter always uses the unconditional fast lane. A - direct caller can instead be resuming a durable active ancestor, for - which another submit is unnecessary and some integrations do not - retain block bytes. This compatibility probe is not on the incident - queue-to-node path. - """ - block_hash = str(candidate.submission.block_hash_hex).lower() - expected_height = int(candidate.context.template["height"]) - try: - if str(self.rpc.call("getbestblockhash")).lower() == block_hash: - return _BlockCandidateNodeSubmission(attempted=False) - except Exception: - pass - try: - if self.active_block_candidate_height(block_hash) == expected_height: - return _BlockCandidateNodeSubmission(attempted=False) - except Exception: - pass - if not hasattr(candidate.submission, "block_hex"): - return _BlockCandidateNodeSubmission(attempted=False) - return self._node_submission_for_candidate(candidate) + """Preserve active-replay semantics for non-queue embedders.""" + return self._ensure_block_candidate_service()._node_submission_for_direct_candidate( + candidate + ) def _account_block_candidate_after_node_submit( self, @@ -24895,220 +8944,58 @@ def _account_block_candidate_after_node_submit( node_submission: _BlockCandidateNodeSubmission, ) -> bool: """Pass fast-lane evidence while tolerating legacy test embedders.""" - submit = self.submit_block_candidate - supports_node_submission = True - try: - parameters = inspect.signature(submit).parameters.values() - supports_node_submission = any( - parameter.name == "node_submission" - or parameter.kind is inspect.Parameter.VAR_KEYWORD - for parameter in parameters - ) - except (TypeError, ValueError): - pass - if supports_node_submission: - return bool(submit(candidate, node_submission=node_submission)) - return bool(submit(candidate)) + return self._ensure_block_candidate_service()._account_block_candidate_after_node_submit( + candidate, + node_submission, + ) def _submit_synchronous_block_candidate( self, candidate: PrismBlockCandidate, ) -> bool: """Run the rare miner-facing path under one same-hash disposition.""" - block_hash = str(candidate.submission.block_hash_hex).lower() - with self._block_candidate_disposition(block_hash): - terminal_outcome = self._block_candidate_terminal_outcome(block_hash) - if terminal_outcome is not None: - self._finish_pending_share_commit(candidate.pending_share) - return terminal_outcome - outcome = getattr(self, "_block_candidate_outcome", None) - if outcome is None: - outcome = threading.local() - self._block_candidate_outcome = outcome - # The accepted/rejected accounting tail may already be complete - # while only its durable outbox terminal update is retrying. A - # synchronous same-hash waiter must join that finalize-only state; - # another node offer/accounting pass could invert the outcome. - with self.lock: - registry = getattr(self, "_block_candidate_finalize_retries", None) - pending_finalize = ( - registry.get(block_hash) if registry is not None else None - ) - if pending_finalize is not None: - accepted, error = pending_finalize - self._finalize_block_candidate( - candidate, - block_hash=block_hash, - accepted=accepted, - error=error, - outcome=outcome, - ) - return accepted - try: - if not self._reserve_block_fast_lane_slot(block_hash): - raise RuntimeError( - "block candidate is waiting for fast-lane capacity" - ) - node_submission = self._node_submission_for_candidate_or_retained(candidate) - self._mark_block_candidate_attempted(block_hash) - with self._block_landing_ledger_statement_timeout_scope(block_hash): - production_submit = ( - getattr(self.submit_block_candidate, "__func__", None) - is PrismCoordinator.submit_block_candidate - ) - if production_submit: - block_landed = self._submit_block_candidate_serialized( - candidate, - node_submission=node_submission, - ) - else: - block_landed = self._account_block_candidate_after_node_submit( - candidate, - node_submission, - ) - except BaseException: - self._retain_block_candidate_for_retry(candidate) - raise - - if not block_landed: - reason = getattr(outcome, "reason", None) - if reason in {None, *PRISM_RETRYABLE_BLOCK_CANDIDATE_REASONS}: - self._retain_block_candidate_for_retry(candidate) - raise RuntimeError( - "block candidate outcome is pending durable retry" - ) - abandon_error = getattr(outcome, "error", None) - try: - self._record_block_submitter_phase( - "reject-prepared-block" - ) - with self._block_landing_ledger_statement_timeout_scope( - block_hash - ): - self._reject_terminal_prepared_block_candidate(candidate) - self._record_block_submitter_phase( - "reject-prepared-block:complete" - ) - except Exception as exc: - # A prior attempt may have persisted prepared payout rows - # before this synchronous resubmit reached a false terminal - # verdict. Keep the outbox pending until those rows can be - # rejected; otherwise restart replay is removed while its - # balance transition remains live. - self._defer_block_candidate( - PRISM_REJECTION_BACKEND_RPC_UNAVAILABLE, - "could not reject prepared state for terminal candidate", - worker=candidate.client.username or None, - ) - self._retain_block_candidate_for_retry(candidate) - raise RuntimeError( - "could not reject prepared state for terminal candidate" - ) from exc - self._finalize_block_candidate( - candidate, - block_hash=block_hash, - accepted=False, - error=str(abandon_error or reason), - outcome=outcome, - ) - return False - - self._finalize_block_candidate( - candidate, - block_hash=block_hash, - accepted=True, - error="", - outcome=outcome, - ) - return True + return self._ensure_block_candidate_service()._submit_synchronous_block_candidate( + candidate + ) def _ensure_block_candidate_disposition_state(self) -> None: """Backfill same-hash submission guards for lightweight embedders.""" - if ( - hasattr(self, "_block_candidate_disposition_registry_lock") - and hasattr(self, "_block_candidate_disposition_flights") - and hasattr(self, "_block_candidate_terminal_outcomes") - and hasattr(self, "_block_fast_lane_reservations") - and hasattr(self, "_block_disposition_waiting_retries") - ): - return - with _HOT_PATH_LOCK_INITIALIZATION_LOCK: - if not hasattr(self, "_block_candidate_disposition_registry_lock"): - self._block_candidate_disposition_registry_lock = threading.Lock() - if not hasattr(self, "_block_candidate_disposition_flights"): - self._block_candidate_disposition_flights: dict[ - str, _BlockCandidateDispositionFlight - ] = {} - if not hasattr(self, "_block_candidate_terminal_outcomes"): - self._block_candidate_terminal_outcomes: dict[str, bool] = {} - if not hasattr(self, "_block_fast_lane_reservations"): - self._block_fast_lane_reservations: set[str] = set() - if not hasattr(self, "_block_disposition_waiting_retries"): - self._block_disposition_waiting_retries: dict[ - str, PrismBlockCandidate - ] = {} + self._ensure_block_candidate_service()._ensure_block_candidate_disposition_state() def _claim_block_candidate_disposition( self, block_hash: str, *, - blocking: bool, - ) -> _BlockCandidateDispositionLease | None: - """Claim one hash without making unrelated node offers wait.""" - key = block_hash.lower() - self._ensure_block_candidate_disposition_state() - with self._block_submitter_lock( - self._block_candidate_disposition_registry_lock, - "candidate-disposition-registry", - ): - flight = self._block_candidate_disposition_flights.get(key) - if flight is None: - flight = _BlockCandidateDispositionFlight() - self._block_candidate_disposition_flights[key] = flight - flight.users += 1 - if blocking: - self._acquire_block_submitter_lock( - flight.lock, - f"candidate-disposition:{key}", - ) - acquired = True - else: - acquired = flight.lock.acquire(blocking=False) - if acquired: - return _BlockCandidateDispositionLease(key, flight) - self._drop_block_candidate_disposition_user(key, flight) - return None + blocking: bool, + ) -> _BlockCandidateDispositionLease | None: + """Claim one hash without making unrelated node offers wait.""" + return self._ensure_block_candidate_service()._claim_block_candidate_disposition( + block_hash, + blocking=blocking, + ) def _drop_block_candidate_disposition_user( self, key: str, flight: _BlockCandidateDispositionFlight, ) -> None: - with self._block_submitter_lock( - self._block_candidate_disposition_registry_lock, - "candidate-disposition-registry", - ): - flight.users -= 1 - if ( - flight.users == 0 - and self._block_candidate_disposition_flights.get(key) is flight - ): - self._block_candidate_disposition_flights.pop(key, None) + self._ensure_block_candidate_service()._drop_block_candidate_disposition_user( + key, + flight, + ) def _release_block_candidate_disposition( self, lease: _BlockCandidateDispositionLease, ) -> None: - lease.flight.lock.release() - self._drop_block_candidate_disposition_user( - lease.block_hash, - lease.flight, + self._ensure_block_candidate_service()._release_block_candidate_disposition( + lease ) def _block_candidate_terminal_outcome(self, block_hash: str) -> bool | None: - self._ensure_block_candidate_disposition_state() - with self.lock: - return self._block_candidate_terminal_outcomes.get(block_hash.lower()) + return self._ensure_block_candidate_service()._block_candidate_terminal_outcome( + block_hash + ) def _record_block_candidate_terminal_outcome( self, @@ -25116,236 +9003,58 @@ def _record_block_candidate_terminal_outcome( *, accepted: bool, ) -> None: - self._ensure_block_candidate_disposition_state() - with self.lock: - key = block_hash.lower() - self._block_candidate_terminal_outcomes[key] = accepted - self._block_fast_lane_reservations.discard(key) - replay_hashes = getattr(self, "_block_replay_inflight_hashes", None) - if replay_hashes is not None: - replay_hashes.discard(key) - waiting = getattr(self, "_block_disposition_waiting_retries", None) - if waiting is not None: - waiting.pop(key, None) + self._ensure_block_candidate_service()._record_block_candidate_terminal_outcome( + block_hash, + accepted=accepted, + ) def _record_committed_block_candidate_abandonment( self, block_hash: str, outcome: threading.local, ) -> None: - """Count an abandonment only after its terminal cleanup is fixed. - - ``_abandon_block_candidate`` seals a proposed rejection before any - prepared payout rows are removed. That cleanup can fail, in which - case the candidate is deliberately re-registered and can still prove - accepted on a later pass. Counting at the seal would then expose the - same hash as both abandoned and accepted. The writer calls this only - after cleanup succeeds or after a false finalize-only disposition is - installed; direct accounting callers invoke it only after the full - serialized rejection path returns. - """ - reason = getattr(outcome, "reason", None) - if not isinstance(reason, str) or not reason: - return - if reason in PRISM_RETRYABLE_BLOCK_CANDIDATE_REASONS: - return - stale_job_class = getattr(outcome, "stale_job_class", None) - key = block_hash.lower() - with self.lock: - counted_abandonments = getattr( - self, - "_counted_block_candidate_abandonments", - None, - ) - if counted_abandonments is None: - counted_abandonments = set() - self._counted_block_candidate_abandonments = counted_abandonments - if key in counted_abandonments: - return - counted_abandonments.add(key) - counts = getattr(self, "block_candidate_abandoned_counts", None) - if counts is None: - counts = {} - self.block_candidate_abandoned_counts = counts - counts[reason] = int(counts.get(reason, 0)) + 1 - if stale_job_class is not None: - stale_counts = getattr(self, "stale_job_abandon_counts", None) - if stale_counts is None: - stale_counts = { - abandon_class: 0 - for abandon_class in PRISM_STALE_JOB_ABANDON_CLASSES - } - self.stale_job_abandon_counts = stale_counts - stale_counts[stale_job_class] = ( - int(stale_counts.get(stale_job_class, 0)) + 1 - ) + """Count an abandonment only after its terminal cleanup is fixed.""" + self._ensure_block_candidate_service()._record_committed_block_candidate_abandonment( + block_hash, + outcome, + ) def _reserve_block_fast_lane_slot(self, block_hash: str) -> bool: """Reserve pool capacity while a node offer awaits terminal accounting.""" - key = block_hash.lower() - with self.lock: - reservations = getattr(self, "_block_fast_lane_reservations", None) - if reservations is None: - reservations = set() - self._block_fast_lane_reservations = reservations - if key in reservations: - return True - accepted_count = int(getattr(self, "accepted_block_count", 0)) - capacity = int(getattr(self, "max_blocks", 2**31 - 1)) - stop_after_one = bool(getattr(self, "stop_after_block", False)) - reserved_count = len(reservations) - if accepted_count + reserved_count >= capacity: - return False - if stop_after_one and accepted_count + reserved_count >= 1: - return False - reservations.add(key) - return True + return self._ensure_block_candidate_service()._reserve_block_fast_lane_slot( + block_hash + ) def _release_block_fast_lane_slot(self, block_hash: str) -> None: - with self.lock: - reservations = getattr(self, "_block_fast_lane_reservations", None) - if reservations is not None: - reservations.discard(block_hash.lower()) + self._ensure_block_candidate_service()._release_block_fast_lane_slot( + block_hash + ) - @contextmanager def _block_candidate_disposition( self, block_hash: str, ) -> Iterator[_BlockCandidateDispositionLease]: - """Serialize the full accepted/abandoned decision for one hash. - - A below-share-target solve submits synchronously while the durable - outbox can concurrently replay that same candidate. Keep both attempts - ordered until the accepted success tail records its process-local - completion; otherwise the replay can terminally abandon the outbox - during the gap after durable confirmation but before audit/share - evidence is complete. - """ - lease = self._claim_block_candidate_disposition( - block_hash, - blocking=True, + """Serialize the full accepted/abandoned decision for one hash.""" + return self._ensure_block_candidate_service()._block_candidate_disposition( + block_hash ) - assert lease is not None - try: - yield lease - finally: - self._release_block_candidate_disposition(lease) def _ensure_block_accounting_state(self) -> None: - if not hasattr(self, "_block_accounting_state_lock"): - self._block_accounting_state_lock = threading.Lock() - if not hasattr(self, "_block_accounting_queue"): - depth = max( - 1, - int( - getattr( - self, - "block_accounting_queue_depth", - DEFAULT_BLOCK_ACCOUNTING_QUEUE_DEPTH, - ) - ), - ) - self._block_accounting_queue = queue.PriorityQueue(maxsize=depth) - if not hasattr(self, "_block_accounting_overflow_queue"): - self._block_accounting_overflow_queue = queue.PriorityQueue() - if not hasattr(self, "_block_accounting_sequence"): - self._block_accounting_sequence = 0 - if not hasattr(self, "_block_accounting_thread"): - self._block_accounting_thread = None + self._ensure_block_candidate_service()._ensure_block_accounting_state() def _start_block_accounting_thread(self) -> threading.Thread: - self._ensure_block_accounting_state() - with self._block_accounting_state_lock: - thread = self._block_accounting_thread - if thread is not None and thread.is_alive(): - return thread - self._record_block_work_heartbeat("block_accounting", "starting") - thread = threading.Thread( - target=self.block_accounting_loop, - name="prism-block-accounting", - daemon=True, - ) - self._block_accounting_thread = thread - thread.start() - return thread + return self._ensure_block_candidate_service()._start_block_accounting_thread() def _enqueue_block_accounting_task( self, task: _BlockCandidateAccountingTask, ) -> bool: - self._ensure_block_accounting_state() - with self._block_accounting_state_lock: - sequence = self._block_accounting_sequence - self._block_accounting_sequence += 1 - priority = int(task.candidate.context.template["height"]) - item = (priority, sequence, task) - if not self._block_accounting_overflow_queue.empty(): - # Once spillover begins, keep later handoffs behind it instead of - # repeatedly refilling the primary queue and starving older spill - # entries. - self._block_accounting_overflow_queue.put_nowait(item) - return True - try: - self._block_accounting_queue.put_nowait(item) - return True - except queue.Full: - # A node offer has already happened and must never be converted - # back into a raw-submit retry. Preserve its result and lease in - # an unbounded, process-local overflow queue; max-block admission - # bounds the number of unresolved real offers. - self._block_accounting_overflow_queue.put_nowait(item) - print( - "prism coordinator: block accounting handoff spilled " - f"hash={task.candidate.submission.block_hash_hex}", - flush=True, - ) - return True + return self._ensure_block_candidate_service()._enqueue_block_accounting_task( + task + ) def _run_one_invalid_block_candidate_quarantine(self) -> bool: - self._ensure_block_replay_state() - try: - block_hash, error = self._block_quarantine_queue.get_nowait() - except queue.Empty: - return False - completed = False - try: - self._record_block_submitter_phase("replay-quarantine") - quarantine = getattr( - self.ledger, - "mark_block_candidate_abandoned", - None, - ) - if callable(quarantine): - quarantined = self._run_block_submitter_ledger_call( - ("replay-quarantine", block_hash), - "replay-quarantine", - lambda: quarantine(block_hash=block_hash, error=error), - ) - self._clear_accepted_block_payout_preview(block_hash) - if quarantined: - self._clear_block_candidate_retry_state(block_hash) - self._discard_outstanding_block_candidate(block_hash) - with self.lock: - self.block_candidate_poisoned_count = int( - getattr(self, "block_candidate_poisoned_count", 0) - ) + 1 - completed = True - return True - except Exception: - print( - "prism coordinator: invalid candidate quarantine failed " - f"hash={block_hash}", - flush=True, - ) - traceback.print_exc() - return True - finally: - self._block_quarantine_queue.task_done() - if completed: - with self.lock: - self._block_quarantine_hashes.discard(block_hash) - elif not self.stop_event.is_set(): - self._block_quarantine_queue.put_nowait((block_hash, error)) + return self._ensure_block_candidate_service()._run_one_invalid_block_candidate_quarantine() def _call_block_candidate_writer( self, @@ -25355,600 +9064,60 @@ def _call_block_candidate_writer( disposition_held: bool, ) -> bool: """Invoke the writer while preserving duck-typed test integrations.""" - writer = self._submit_next_block_candidate_writer - supports_disposition_held = True - try: - parameters = inspect.signature(writer).parameters.values() - supports_disposition_held = any( - parameter.name == "disposition_held" - or parameter.kind is inspect.Parameter.VAR_KEYWORD - for parameter in parameters - ) - except (TypeError, ValueError): - pass - if supports_disposition_held: - return bool( - writer( - candidate, - node_submission=node_submission, - disposition_held=disposition_held, - ) - ) - return bool(writer(candidate, node_submission=node_submission)) + return self._ensure_block_candidate_service()._call_block_candidate_writer( + candidate, + node_submission=node_submission, + disposition_held=disposition_held, + ) def _restore_replayed_candidate_acceptance_evidence( self, candidate: PrismBlockCandidate, ) -> None: - if not candidate.durable_replay: - return - block_hash = str(candidate.submission.block_hash_hex).lower() - block_state = None - state_read_failed = False - state_reader = getattr(self.ledger, "pool_block_state", None) - if callable(state_reader): - try: - block_state = self._run_block_submitter_ledger_call( - ("replay-pool-block-state", block_hash), - "replay-pool-block-state", - lambda: state_reader(block_hash=block_hash), - ) - except Exception: - traceback.print_exc() - state_read_failed = True - durable_chain_state = ( - str(block_state.get("chain_state", "")) - if block_state is not None - else "" - ) - if state_read_failed or durable_chain_state in {"prepared", "confirmed"}: - self._register_outstanding_block_candidate(block_hash) - with self.lock: - self._tip_observed_accepted_block_hashes[block_hash] = ( - time.monotonic() - ) - print( - "prism coordinator: restored acceptance evidence for " - f"replayed block candidate hash={block_hash} " - + ( - "after a failed durable-state read" - if state_read_failed - else f"chain_state={durable_chain_state}" - ), - flush=True, - ) + self._ensure_block_candidate_service()._restore_replayed_candidate_acceptance_evidence( + candidate + ) def _run_block_accounting_task( self, task: _BlockCandidateAccountingTask, ) -> None: - candidate = task.candidate - with self.lock: - self._block_accounting_holds_disposition = True - self._block_accounting_deferred_retry_candidate = None - outcome = getattr(self, "_block_candidate_outcome", None) - if outcome is None: - outcome = threading.local() - self._block_candidate_outcome = outcome - outcome.refresh_client = None - try: - self._restore_replayed_candidate_acceptance_evidence(candidate) - with self._writer_operation("accepted_block_handling"): - self._call_block_candidate_writer( - candidate, - node_submission=task.node_submission, - disposition_held=True, - ) - refresh_client = getattr(outcome, "refresh_client", None) - outcome.refresh_client = None - except ShutdownInProgress: - return - finally: - self._release_block_candidate_disposition(task.disposition_lease) - with self.lock: - self._block_accounting_holds_disposition = False - deferred_retry = getattr( - self, - "_block_accounting_deferred_retry_candidate", - None, - ) - self._block_accounting_deferred_retry_candidate = None - if deferred_retry is not None: - self._merge_block_candidate_retry_locked( - "_retry_block_candidate", - deferred_retry, - ) - if refresh_client is not None and not self.stop_event.is_set(): - self._record_block_submitter_phase("refresh-jobs") - self.refresh_jobs_after_pending_accepted_block( - refresh_client, - heartbeat_name="block_accounting", - ) - self._record_block_submitter_phase("refresh-jobs:complete") + self._ensure_block_candidate_service()._run_block_accounting_task(task) def block_accounting_loop(self) -> None: - self._block_accounting_thread_ident = threading.get_ident() - self._ensure_block_accounting_state() - while not self.stop_event.is_set(): - self._record_block_submitter_phase("accounting-queue") - source_queue = None - try: - _priority, _sequence, task = self._block_accounting_queue.get_nowait() - source_queue = self._block_accounting_queue - except queue.Empty: - try: - _priority, _sequence, task = ( - self._block_accounting_overflow_queue.get_nowait() - ) - source_queue = self._block_accounting_overflow_queue - except queue.Empty: - if self._run_one_invalid_block_candidate_quarantine(): - continue - self.stop_event.wait(self._block_work_wait_slice()) - continue - try: - self._run_block_accounting_task(task) - except Exception: - print( - "prism coordinator: block accounting iteration failed; " - "durable candidate remains pending", - flush=True, - ) - traceback.print_exc() - self._retain_block_candidate_for_retry(task.candidate) - finally: - assert source_queue is not None - source_queue.task_done() + self._ensure_block_candidate_service().block_accounting_loop() def block_submit_loop(self) -> None: - # Boundary stamps in _record_block_candidate_progress are gated to - # this thread so client-thread dispositions cannot refresh the - # submitter's liveness budget on its behalf. - self._block_submitter_thread_ident = threading.get_ident() - self._start_block_accounting_thread() - while not self.stop_event.is_set(): - self._record_block_submitter_phase("loop") - try: - # The in-memory wakeup is already backed by the durable - # outbox. Drain it before any recovery query so a saturated - # database cannot delay the first node submission. - with self.lock: - retry_ready = ( - getattr(self, "_retry_block_candidate", None) is not None - ) - queue_obj = getattr(self, "block_candidate_queue", None) - replay_queue = getattr( - self, - "_block_replay_candidate_queue", - None, - ) - wakeup_ready = bool( - (queue_obj is not None and not queue_obj.empty()) - or (replay_queue is not None and not replay_queue.empty()) - ) - if (retry_ready or wakeup_ready) and self.submit_next_block_candidate( - defer_accounting=True - ): - continue - self.replay_pending_block_candidates() - self.submit_next_block_candidate( - timeout=1.0, - defer_accounting=True, - ) - except ShutdownInProgress: - # Admission can close after the loop condition. Durable block - # candidates remain in the outbox for the replacement writer. - return - except Exception: - phase = getattr(self, "_block_submitter_phase", "unknown") - print( - "prism coordinator: block submitter iteration failed " - f"phase={phase}; durable candidates remain pending", - flush=True, - ) - traceback.print_exc() - retry_delay = float( - getattr( - self, - "block_candidate_retry_initial_seconds", - DEFAULT_BLOCK_CANDIDATE_RETRY_INITIAL_SECONDS, - ) - ) - if self._wait_for_block_candidate_retry(retry_delay): - return - - def submit_next_block_candidate( - self, - timeout: float | None = None, - *, - defer_accounting: bool = False, - ) -> bool: - """Dequeue and land one block candidate; returns True when one ran. - - The block-submitter loop calls this continuously; tests call it - directly to drain the queue deterministically. - """ - self._record_block_submitter_phase("dequeue-retry") - with self.lock: - candidate = getattr(self, "_retry_block_candidate", None) - if candidate is not None and not self._block_candidate_retry_ready_locked( - candidate - ): - # Parked by _pace_block_candidate_retry: honor the backoff - # deadline without sleeping on the submitter or accounting - # lane. - candidate = None - if candidate is not None: - self._retry_block_candidate = None - if candidate is None: - queue_obj = getattr(self, "block_candidate_queue", None) - self._ensure_block_replay_state() - replay_queue = self._block_replay_candidate_queue - if queue_obj is None and replay_queue is None: - return False - deadline = ( - None - if timeout is None - else time.monotonic() + max(0.0, timeout) - ) - while candidate is None: - self._record_block_submitter_phase("dequeue-queue") - # Live discoveries always outrank durable restart replay. - for candidate_queue in (queue_obj, replay_queue): - if candidate_queue is None: - continue - try: - candidate = candidate_queue.get_nowait() - break - except queue.Empty: - pass - if candidate is None: - self._ensure_block_candidate_disposition_state() - with self.lock: - waiting = self._block_disposition_waiting_retries - ready_hashes = [ - key - for key in waiting - if self._block_candidate_retry_ready_locked( - waiting[key] - ) - ] - if ready_hashes: - waiting_hash = min( - ready_hashes, - key=lambda key: int( - waiting[key].context.template["height"] - ), - ) - candidate = waiting.pop(waiting_hash) - if candidate is not None: - break - if deadline is None: - return False - remaining = deadline - time.monotonic() - if remaining <= 0: - return False - if self.stop_event.wait( - min(remaining, self._block_work_wait_slice()) - ): - return False - - block_hash = str(candidate.submission.block_hash_hex).lower() - lease = self._claim_block_candidate_disposition( - block_hash, - blocking=not defer_accounting, - ) - if lease is None: - # Another same-hash pass already spans node offer through durable - # finalization. Keep this wakeup outside the global parent retry - # slot until that lease transfers/releases: consuming it can lose - # an accounting retry, while repeatedly prioritizing it can starve - # unrelated live blocks. - with self.lock: - self._block_disposition_waiting_retries[block_hash] = candidate - self._wait_for_block_candidate_retry( - float( - getattr( - self, - "block_candidate_retry_initial_seconds", - DEFAULT_BLOCK_CANDIDATE_RETRY_INITIAL_SECONDS, - ) - ) - ) - return True - transferred = False - if self._block_candidate_terminal_outcome(block_hash) is not None: - self._release_block_candidate_disposition(lease) - return True - - outcome = getattr(self, "_block_candidate_outcome", None) - if outcome is None: - outcome = threading.local() - self._block_candidate_outcome = outcome - outcome.refresh_client = None - self._record_block_submitter_phase("finalize-registry") - with self.lock: - registry = getattr(self, "_block_candidate_finalize_retries", None) - pending_finalize = ( - registry.get(block_hash) if registry is not None else None - ) - if pending_finalize is None: - permanently_closed = False - already_accounted = False - if defer_accounting: - with self.lock: - accounted_hashes = getattr( - self, - "_accounted_accepted_block_hashes", - set(), - ) - already_accounted = block_hash in accounted_hashes - accepted_count = int(getattr(self, "accepted_block_count", 0)) - permanently_closed = not already_accounted and ( - accepted_count >= int(getattr(self, "max_blocks", 2**31 - 1)) - or ( - bool(getattr(self, "stop_after_block", False)) - and accepted_count >= 1 - ) - ) - if permanently_closed or already_accounted: - # Accounting must terminalize a durable outbox row even - # after pool capacity closes. An already-accounted hash - # likewise needs only its exact-idempotent/finalize tail. - node_submission = _BlockCandidateNodeSubmission( - attempted=False - ) - elif not self._reserve_block_fast_lane_slot(block_hash): - # Capacity is provisionally occupied by another unresolved - # node offer. Preserve strict max-block semantics until - # that offer either accounts or terminates. - self._release_block_candidate_disposition(lease) - self._retain_block_candidate_for_retry(candidate) - self._wait_for_block_candidate_retry( - float( - getattr( - self, - "block_candidate_retry_initial_seconds", - DEFAULT_BLOCK_CANDIDATE_RETRY_INITIAL_SECONDS, - ) - ) - ) - return True - else: - try: - node_submission = self._node_submission_for_candidate_or_retained(candidate) - except BaseException: - try: - self._retain_block_candidate_for_retry(candidate) - finally: - self._release_block_candidate_disposition(lease) - raise - else: - try: - node_submission = self._node_submission_for_candidate_or_retained(candidate) - except BaseException: - try: - self._retain_block_candidate_for_retry(candidate) - finally: - self._release_block_candidate_disposition(lease) - raise - else: - node_submission = _BlockCandidateNodeSubmission(attempted=False) - - if defer_accounting: - task = _BlockCandidateAccountingTask( - candidate=candidate, - node_submission=node_submission, - disposition_lease=lease, - ) - try: - enqueued = self._enqueue_block_accounting_task(task) - except BaseException: - try: - self._retain_block_candidate_for_retry(candidate) - finally: - self._release_block_candidate_disposition(lease) - raise - if enqueued: - transferred = True - return True - self._release_block_candidate_disposition(lease) - self._retain_block_candidate_for_retry(candidate) - self._wait_for_block_candidate_retry( - float( - getattr( - self, - "block_candidate_retry_initial_seconds", - DEFAULT_BLOCK_CANDIDATE_RETRY_INITIAL_SECONDS, - ) - ) - ) - return True - - try: - with self._writer_operation("accepted_block_handling"): - ran = self._call_block_candidate_writer( - candidate, - node_submission=node_submission, - disposition_held=True, - ) - refresh_client = getattr(outcome, "refresh_client", None) - outcome.refresh_client = None - except ShutdownInProgress: - # The durable outbox remains pending and the replacement process - # will replay it. Dequeuing the in-memory wakeup during the - # admission-close race cannot lose candidate work. - return False - finally: - if not transferred: - self._release_block_candidate_disposition(lease) - # Fresh-job fanout is deliberately outside the writer admission. Once - # the candidate outbox is finalized it cannot mutate the ledger, so a - # blocked client send must not hold the writer lease during shutdown. - if refresh_client is not None and not self.stop_event.is_set(): - self._record_block_submitter_phase("refresh-jobs") - self.refresh_jobs_after_pending_accepted_block( - refresh_client, - heartbeat_name="block_submitter", - ) - self._record_block_submitter_phase("refresh-jobs:complete") - return ran + self._ensure_block_candidate_service().run() - def _submit_next_block_candidate_writer( - self, - candidate: PrismBlockCandidate, - *, - node_submission: _BlockCandidateNodeSubmission | None = None, - disposition_held: bool = False, - ) -> bool: - """Land one dequeued block candidate inside writer admission.""" - block_hash = str(candidate.submission.block_hash_hex).lower() - if not disposition_held: - # Preserve the historical direct-writer seam while keeping its - # node offer and terminal outbox decision inside the same-hash - # guard. Production queue/accounting calls transfer an existing - # lease and skip this wrapper. - with self._block_candidate_disposition(block_hash): - terminal_outcome = self._block_candidate_terminal_outcome( - block_hash - ) - if terminal_outcome is not None: - return terminal_outcome - if node_submission is None: - node_submission = ( - self._node_submission_for_candidate_or_retained( - candidate - ) - ) - return self._submit_next_block_candidate_writer( - candidate, - node_submission=node_submission, - disposition_held=True, - ) - outcome = getattr(self, "_block_candidate_outcome", None) - if outcome is None: - outcome = threading.local() - self._block_candidate_outcome = outcome - outcome.reason = None - outcome.error = None - outcome.stale_job_class = None - self._record_block_submitter_phase("finalize-registry") - with self.lock: - registry = getattr(self, "_block_candidate_finalize_retries", None) - pending_finalize = ( - registry.get(block_hash) if registry is not None else None - ) - if pending_finalize is not None: - # Finalize-only replay: node submission, terminal accounting, and - # payout persistence already completed on the pass that armed - # this entry. It bypasses both submitblock and attempt marking. - accepted, error = pending_finalize - return self._finalize_block_candidate( - candidate, - block_hash=block_hash, - accepted=accepted, - error=error, - outcome=outcome, - ) - if node_submission is None: - node_submission = self._node_submission_for_candidate_or_retained(candidate) - try: - self._mark_block_candidate_attempted(block_hash) - except Exception: - print( - "prism coordinator: could not record block candidate attempt " - f"hash={block_hash}", - flush=True, - ) - traceback.print_exc() - self._retain_block_candidate_for_retry(candidate) - self._pace_block_candidate_retry(block_hash) - return True - accepted = False - error = "candidate became stale or submission failed" - try: - self._record_block_submitter_phase("accounting") - with self._block_landing_ledger_statement_timeout_scope(block_hash): - production_submit = ( - getattr(self.submit_block_candidate, "__func__", None) - is PrismCoordinator.submit_block_candidate - ) - if disposition_held and production_submit: - assert node_submission is not None - accepted = self._submit_block_candidate_serialized( - candidate, - node_submission=node_submission, - ) - elif node_submission is None: - accepted = bool(self.submit_block_candidate(candidate)) - else: - accepted = self._account_block_candidate_after_node_submit( - candidate, - node_submission, - ) - except Exception: - error = "candidate submission raised an exception" - print( - "prism coordinator: block candidate submission failed " - f"hash={candidate.submission.block_hash_hex}", - flush=True, - ) - traceback.print_exc() - abandon_reason = getattr(outcome, "reason", None) if outcome is not None else None - abandon_error = getattr(outcome, "error", None) if outcome is not None else None - if not accepted and abandon_error: - error = str(abandon_error) - retryable = not accepted and ( - abandon_reason is None - or abandon_reason in PRISM_RETRYABLE_BLOCK_CANDIDATE_REASONS - ) - if retryable: - # Leave the outbox row pending. It will replay after a short pause - # or on process restart. Keep this parent ahead of queued children: - # a child built from its prospective balances cannot be validated - # against the database until the parent confirmation catches up. - print( - "prism coordinator: retained block candidate for retry " - f"hash={block_hash} reason={abandon_reason or 'exception'}", - flush=True, - ) - self._retain_block_candidate_for_retry(candidate) - self._pace_block_candidate_retry(block_hash) - return True - if not accepted: - try: - self._record_block_submitter_phase("reject-prepared-block") - with self._block_landing_ledger_statement_timeout_scope(block_hash): - self._reject_terminal_prepared_block_candidate(candidate) - self._record_block_submitter_phase( - "reject-prepared-block:complete" - ) - except Exception: - # Persistence may have committed before a later RPC/transport - # failure. Do not terminally discard the outbox row until its - # prepared balance deltas have also reached a terminal state. - print( - "prism coordinator: prepared block cleanup failed " - f"hash={block_hash}", - flush=True, - ) - traceback.print_exc() - self._defer_block_candidate( - PRISM_REJECTION_BACKEND_RPC_UNAVAILABLE, - "could not reject prepared state for terminal candidate", - worker=candidate.client.username or None, - ) - self._retain_block_candidate_for_retry(candidate) - self._wait_for_block_candidate_retry( - self._next_block_candidate_retry_delay(block_hash) - ) - return True - return self._finalize_block_candidate( + def submit_next_block_candidate( + self, + timeout: float | None = None, + *, + defer_accounting: bool = False, + ) -> bool: + """Dequeue and land one block candidate; returns True when one ran. + + The block-submitter loop calls this continuously; tests call it + directly to drain the queue deterministically. + """ + return self._ensure_block_candidate_service().submit_next( + timeout, + defer_accounting=defer_accounting, + ).ran + + def _submit_next_block_candidate_writer( + self, + candidate: PrismBlockCandidate, + *, + node_submission: _BlockCandidateNodeSubmission | None = None, + disposition_held: bool = False, + ) -> bool: + """Land one dequeued block candidate inside writer admission.""" + return self._ensure_block_candidate_service().submit_writer( candidate, - block_hash=block_hash, - accepted=accepted, - error=error, - outcome=outcome, + node_submission=node_submission, + disposition_held=disposition_held, ) def _finalize_block_candidate( @@ -25960,120 +9129,14 @@ def _finalize_block_candidate( error: str, outcome: threading.local, ) -> bool: - """Drive a terminal candidate's durable outbox update, with backoff. - - Failure retains the candidate as a finalize-only replay: the next - paced attempt re-enters here directly, never submit_block_candidate, - so terminal abandonment accounting stays once-per-candidate and an - accepted candidate's audit/persist work is not redone per retry. - """ - self._record_block_submitter_phase("finalize-preview") - self._clear_accepted_block_payout_preview( - block_hash, - invalidate_published=not accepted, - ) - finish_name = ( - "mark_block_candidate_submitted" - if accepted - else "mark_block_candidate_abandoned" - ) - finish = getattr(self.ledger, finish_name, None) - if callable(finish): - try: - if accepted: - self._run_block_submitter_ledger_call( - ("finalize", block_hash, "submitted"), - "finalize-outbox-submitted", - lambda: finish(block_hash=block_hash), - ) - else: - self._run_block_submitter_ledger_call( - ("finalize", block_hash, "abandoned"), - "finalize-outbox-abandoned", - lambda: finish(block_hash=block_hash, error=error), - ) - self._record_committed_block_candidate_abandonment( - block_hash, - outcome, - ) - # The invalidation tombstone is needed until the durable - # outbox becomes terminal. A normal return (including an - # already-terminal/missing row) means there is no pending - # replay source left for this process to guard. - self._clear_accepted_block_payout_preview(block_hash) - except Exception: - # Keep the coordinator alive. The terminal-state update - # failed, so the durable row stays pending and its replay - # must pace like any other retained retry. - print( - "prism coordinator: could not finalize durable block candidate " - f"hash={block_hash}", - flush=True, - ) - traceback.print_exc() - with self.lock: - registry = getattr( - self, "_block_candidate_finalize_retries", None - ) - if registry is None: - registry = {} - self._block_candidate_finalize_retries = registry - first_failure = block_hash not in registry - registry[block_hash] = (accepted, error) - if not accepted: - # The durable update is ambiguous, but this process has - # now frozen a false finalize-only disposition. It cannot - # return to chain-state evaluation until restart, where - # these process-local counters start fresh. - self._record_committed_block_candidate_abandonment( - block_hash, - outcome, - ) - # The share row already reached its terminal outcome in this - # process; only the outbox mark is pending. Release the - # snapshot anchor floor now (idempotent) -- holding it across - # paced retries would clamp job snapshot anchors and - # under-count already-durable shares in reward windows. - self._finish_pending_share_commit(candidate.pending_share) - self._retain_block_candidate_for_retry(candidate) - if accepted and first_failure: - # The block is active regardless of the outbox update; - # post-accept fleet refresh must wait for neither ledger - # recovery nor the first backoff. Return unpaced so the - # caller refreshes immediately; the paced ladder starts - # from the first finalize-only replay. - outcome.refresh_client = candidate.client - return True - self._wait_for_block_candidate_retry( - self._next_block_candidate_retry_delay(block_hash) - ) - return True - elif not accepted: - # Compatibility ledgers without a durable candidate outbox have - # no restart replay source that could require the tombstone. - self._clear_accepted_block_payout_preview(block_hash) - self._record_committed_block_candidate_abandonment( - block_hash, - outcome, - ) - with self.lock: - registry = getattr(self, "_block_candidate_finalize_retries", None) - if registry is not None: - registry.pop(block_hash, None) - self._clear_block_candidate_retry_state(block_hash) - self._discard_outstanding_block_candidate(block_hash) - self._record_block_candidate_terminal_outcome( - block_hash, + """Drive a terminal candidate's durable outbox update, with backoff.""" + return self._ensure_block_candidate_service().finalize( + candidate, + block_hash=block_hash, accepted=accepted, + error=error, + outcome=outcome, ) - # Terminal for this process either way: an accepted candidate credited - # its share during the success tail (a no-op release here), and an - # abandoned one can only be credited by restart replay, which stamps a - # fresh PendingShare. Stop holding the snapshot anchor floor. - self._finish_pending_share_commit(candidate.pending_share) - if accepted: - outcome.refresh_client = candidate.client - return True def _merge_block_candidate_retry_locked( self, @@ -26081,67 +9144,14 @@ def _merge_block_candidate_retry_locked( candidate: PrismBlockCandidate, ) -> None: """Merge one retry by parent-first order. Caller holds self.lock.""" - candidate_height = int(candidate.context.template["height"]) - candidate_hash = str(candidate.submission.block_hash_hex).lower() - existing = getattr(self, attribute, None) - if existing is None: - setattr(self, attribute, candidate) - return - existing_height = int(existing.context.template["height"]) - existing_hash = str(existing.submission.block_hash_hex).lower() - if candidate_hash == existing_hash: - setattr(self, attribute, candidate) - return - if attribute != "_retry_block_candidate": - if candidate_height < existing_height: - setattr(self, attribute, candidate) - return - - # The raw lane has one parent-first head slot, but every displaced - # hash still needs an in-memory wakeup. Durable replay dedupe keeps a - # replayed descendant marked in-flight, so relying on a later outbox - # scan here could otherwise suppress it forever. - self._ensure_block_candidate_disposition_state() - waiting = self._block_disposition_waiting_retries - if candidate_height < existing_height: - waiting[existing_hash] = existing - setattr(self, attribute, candidate) - else: - waiting[candidate_hash] = candidate + self._ensure_block_candidate_service()._merge_block_candidate_retry_locked( + attribute, + candidate, + ) def _retain_block_candidate_for_retry(self, candidate: PrismBlockCandidate) -> None: """Keep the oldest unresolved candidate ahead of queued descendants.""" - candidate_hash = str(candidate.submission.block_hash_hex).lower() - # A retained candidate will be re-disposed, so the disposition seal - # (which stopped tip-observation matching at a terminal commit) no - # longer applies: the terminal work did not complete. Re-register - # immediately -- not at the next disposition -- so acceptance - # evidence arriving during the retry backoff is not lost. - self._register_outstanding_block_candidate(candidate_hash) - with self.lock: - self.block_candidate_retry_count = int( - getattr(self, "block_candidate_retry_count", 0) - ) + 1 - accounting_owner = ( - threading.get_ident() - == getattr(self, "_block_accounting_thread_ident", None) - and bool( - getattr( - self, - "_block_accounting_holds_disposition", - False, - ) - ) - ) - retry_attribute = ( - "_block_accounting_deferred_retry_candidate" - if accounting_owner - else "_retry_block_candidate" - ) - self._merge_block_candidate_retry_locked( - retry_attribute, - candidate, - ) + self._ensure_block_candidate_service().retain_for_retry(candidate) def _reject_terminal_prepared_block_candidate( self, @@ -26169,304 +9179,86 @@ def _reject_terminal_prepared_block_candidate( ) def _next_block_candidate_retry_delay(self, block_hash: str) -> float: - initial = max( - 0.0, - float( - getattr( - self, - "block_candidate_retry_initial_seconds", - DEFAULT_BLOCK_CANDIDATE_RETRY_INITIAL_SECONDS, - ) - ), - ) - maximum = max( - initial, - float( - getattr( - self, - "block_candidate_retry_max_seconds", - DEFAULT_BLOCK_CANDIDATE_RETRY_MAX_SECONDS, - ) - ), - ) - with self.lock: - delays = getattr(self, "block_candidate_retry_delays", None) - if delays is None: - delays = {} - self.block_candidate_retry_delays = delays - delay = float(delays.get(block_hash, initial)) - delays[block_hash] = min(maximum, max(initial, delay * 2)) - return min(delay, maximum) + return self._ensure_block_candidate_service().next_retry_delay(block_hash) def _pace_block_candidate_retry(self, block_hash: str) -> None: - """Apply per-candidate retry backoff without convoying accounting. - - On the block_accounting thread the disposition lease and writer - admission stay held until the accounting task's finally clause, so - sleeping here would stall every queued accounting task and keep an - armed payout barrier blocking balance mutation for the whole backoff - window. Record a not-before deadline instead; the dequeue path honors - it, and replay_pending_block_candidates already short-circuits while - the retained candidate occupies the retry slot. - """ - delay_seconds = self._next_block_candidate_retry_delay(block_hash) - accounting_owner = ( - threading.get_ident() - == getattr(self, "_block_accounting_thread_ident", None) - and bool( - getattr( - self, - "_block_accounting_holds_disposition", - False, - ) - ) - ) - if not accounting_owner: - self._wait_for_block_candidate_retry(delay_seconds) - return - with self.lock: - not_before = getattr( - self, - "_block_candidate_retry_not_before", - None, - ) - if not_before is None: - not_before = {} - self._block_candidate_retry_not_before = not_before - not_before[str(block_hash).lower()] = ( - time.monotonic() + delay_seconds - ) + """Apply per-candidate retry backoff without convoying accounting.""" + self._ensure_block_candidate_service()._pace_block_candidate_retry(block_hash) def _block_candidate_retry_ready_locked( self, candidate: PrismBlockCandidate, ) -> bool: - """Return whether a parked retry's backoff deadline has passed. - - Caller holds self.lock. A ready entry is dropped so a candidate that - later lands terminally leaves no stale pacing behind. - """ - not_before = getattr(self, "_block_candidate_retry_not_before", None) - if not not_before: - return True - block_hash = str(candidate.submission.block_hash_hex).lower() - deadline = not_before.get(block_hash) - if deadline is None: - return True - if time.monotonic() < deadline: - return False - not_before.pop(block_hash, None) - return True + """Return whether a parked retry's backoff deadline has passed.""" + return self._ensure_block_candidate_service()._block_candidate_retry_ready_locked( + candidate + ) def _stash_retained_block_candidate_node_submission( self, block_hash: str, node_submission: _BlockCandidateNodeSubmission | None, ) -> None: - """Record a definitive node acceptance for in-process retries. - - Recorded at the offer itself (the universal post-offer hook) so - every retention path — writer failures, defer-accounting handoff - failures, the accounting loop — is covered without each site having - to remember. A retryable failure after a successful offer would - otherwise re-offer on retry and read "duplicate", which classifies - against the moved live tip and can only rescue the block through - chain probes that may be unavailable under the same saturation. - The entry is read without consuming and lives until the candidate - reaches a terminal outcome, so repeated retryable failures keep - reusing the same known acceptance. - """ - if node_submission is None or not node_submission.attempted: - return - if ( - node_submission.error is not None - or node_submission.result is not None - ): - # Only a definitive success is safe to reuse: an ambiguous or - # rejected offer must be re-offered so the node can resolve it. - return - with self.lock: - retained = getattr( - self, - "_block_candidate_retained_node_submissions", - None, - ) - if retained is None: - retained = {} - self._block_candidate_retained_node_submissions = retained - retained[str(block_hash).lower()] = node_submission - stamped = getattr( - self, - "_block_candidate_retained_submission_monotonic", - None, - ) - if stamped is None: - stamped = {} - self._block_candidate_retained_submission_monotonic = stamped - stamped[str(block_hash).lower()] = time.monotonic() + """Record a definitive node acceptance for in-process retries.""" + self._ensure_block_candidate_service()._stash_retained_block_candidate_node_submission( + block_hash, + node_submission, + ) def _retained_block_candidate_node_submission( self, block_hash: str, ) -> _BlockCandidateNodeSubmission | None: - with self.lock: - retained = getattr( - self, - "_block_candidate_retained_node_submissions", - None, - ) - if not retained: - return None - return retained.get(str(block_hash).lower()) + return self._ensure_block_candidate_service()._retained_block_candidate_node_submission( + block_hash + ) def _block_candidate_acceptance_retained(self, block_hash: str) -> bool: - """Whether this process holds fresh first-party acceptance evidence. - - The retained stash records only definitive submitblock successes, - so its presence proves qbitd accepted this candidate — evidence of - the same strength as a recent own-hash tip observation, and - available precisely when saturation makes the chain probes answer - "unknown" (the observation registry can be empty after a definitive - ack: blockwait only reports the newest of rapid connects). It ages - on the same window as tip observations: acceptance at offer time - does not prove the block stayed canonical, and an orphaned block - never probes False — it is merely absent — so a candidate whose - probes stay inconclusive past the window must regain - abandonability instead of deferring forever behind a stale ack. - """ - with self.lock: - stamped = getattr( - self, - "_block_candidate_retained_submission_monotonic", - None, - ) - if not stamped: - return False - recorded = stamped.get(str(block_hash).lower()) - if recorded is None: - return False - window = float( - getattr( - self, - "observed_tip_accept_window_seconds", - DEFAULT_PRISM_OBSERVED_TIP_ACCEPT_WINDOW_SECONDS, - ) + """Whether this process holds fresh first-party acceptance evidence.""" + return self._ensure_block_candidate_service()._block_candidate_acceptance_retained( + block_hash ) - if window <= 0: - return True - return (time.monotonic() - recorded) <= window def _clear_block_candidate_retry_state(self, block_hash: str) -> None: - with self.lock: - delays = getattr(self, "block_candidate_retry_delays", None) - if delays is not None: - delays.pop(block_hash, None) - landing_timeouts = getattr(self, "_block_landing_timeout_counts", None) - if landing_timeouts is not None: - landing_timeouts.pop(block_hash, None) - not_before = getattr( - self, - "_block_candidate_retry_not_before", - None, - ) - if not_before is not None: - not_before.pop(block_hash, None) - retained = getattr( - self, - "_block_candidate_retained_node_submissions", - None, - ) - if retained is not None: - retained.pop(str(block_hash).lower(), None) - stamped = getattr( - self, - "_block_candidate_retained_submission_monotonic", - None, - ) - if stamped is not None: - stamped.pop(str(block_hash).lower(), None) + self._ensure_block_candidate_service().clear_retry_state(block_hash) def _defer_block_candidate(self, reason: str, message: str, *, worker: str | None) -> None: """Record a retryable outcome without counting a terminal abandonment.""" - outcome = getattr(self, "_block_candidate_outcome", None) - if outcome is None: - outcome = threading.local() - self._block_candidate_outcome = outcome - outcome.reason = reason - outcome.error = None - outcome.stale_job_class = None - print( - f"prism coordinator: block candidate deferred reason={reason}: {message}", - flush=True, + self._ensure_block_candidate_service().record_deferred( + reason, + message, + worker=worker, ) def _block_candidate_acceptance_recorded(self, block_hash: str) -> bool: """Return whether this process completed the candidate success tail.""" - self._ensure_job_cache_state() - with self.lock: - return block_hash.lower() in self._accounted_accepted_block_hashes + return self._ensure_block_candidate_service()._block_candidate_acceptance_recorded( + block_hash + ) def _register_outstanding_block_candidate(self, block_hash: str) -> None: """Track a candidate this process may still land, for tip matching.""" - self._ensure_job_cache_state() - with self.lock: - self._outstanding_block_candidate_hashes.add(block_hash.lower()) + self._ensure_block_candidate_service()._register_outstanding_block_candidate( + block_hash + ) def _discard_outstanding_block_candidate(self, block_hash: str) -> None: """Stop matching tip observations once a candidate is terminal.""" - self._ensure_job_cache_state() - key = block_hash.lower() - with self.lock: - self._outstanding_block_candidate_hashes.discard(key) - self._tip_observed_accepted_block_hashes.pop(key, None) + self._ensure_block_candidate_service()._discard_outstanding_block_candidate( + block_hash + ) def _note_tip_observation_for_candidates(self, tip_hash: str) -> None: - """Register a tip observation that matches an outstanding candidate. - - qbitd only ever reports the pool's own candidate hash as its chain - tip after accepting that block, so the observation itself is - acceptance evidence -- even when the direct submitblock ack was lost - in transport and the accepted success tail has not run (blockwait - typically learns of the tip before, or instead of, the ack). Every - tip-observation channel funnels through here so later disposition - and abandon checks can outlive transient fork views. - """ - key = tip_hash.lower() - self._ensure_job_cache_state() - newly_observed = False - with self.lock: - if key in self._outstanding_block_candidate_hashes: - newly_observed = ( - key not in self._tip_observed_accepted_block_hashes - ) - self._tip_observed_accepted_block_hashes[key] = time.monotonic() - if newly_observed: - print( - "prism coordinator: chain tip observation matches pool block " - f"candidate hash={key}; acceptance registered pending " - "finalization", - flush=True, - ) + """Register a tip observation that matches an outstanding candidate.""" + self._ensure_block_candidate_service()._note_tip_observation_for_candidates( + tip_hash + ) def _block_candidate_acceptance_observed(self, block_hash: str) -> bool: """Whether a recent tip observation already proved this candidate landed.""" - self._ensure_job_cache_state() - with self.lock: - observed = self._tip_observed_accepted_block_hashes.get( - block_hash.lower() - ) - if observed is None: - return False - window = float( - getattr( - self, - "observed_tip_accept_window_seconds", - DEFAULT_PRISM_OBSERVED_TIP_ACCEPT_WINDOW_SECONDS, - ) + return self._ensure_block_candidate_service()._block_candidate_acceptance_observed( + block_hash ) - if window <= 0: - return True - return (time.monotonic() - observed) <= window def _block_candidate_chain_probe( self, @@ -26474,45 +9266,11 @@ def _block_candidate_chain_probe( *, expected_height: int | None = None, ) -> bool | None: - """Fresh chain verdict for a candidate: proven active, proven wrong, or unknown. - - Returns True when the candidate's own hash is the fresh best tip or - an active chain header at its expected height, False when it is - provably active at the wrong height (a corrupt intent, never a tip - race), and None when this instantaneous view proves nothing (the - hash absent during a tip race, or the probe itself failing). - """ - key = block_hash.lower() - # The two probes are independent: a best-tip lookup failure must not - # suppress the active-header check, which subsumes it (the tip block - # itself reports one confirmation) and can prove acceptance alone. - try: - if str(self.rpc.call("getbestblockhash")).lower() == key: - self._note_tip_observation_for_candidates(key) - return True - except Exception: - print( - "prism coordinator: acceptance re-check best-tip probe " - f"failed hash={key}; trying the active-header probe", - flush=True, - ) - traceback.print_exc() - height: int | None = None - try: - height = self.active_block_candidate_height(key) - except Exception: - print( - "prism coordinator: acceptance re-check header probe failed " - f"hash={key}; falling back to tip-observation evidence", - flush=True, - ) - traceback.print_exc() - if height is None: - return None - if expected_height is None or int(height) == int(expected_height): - self._note_tip_observation_for_candidates(key) - return True - return False + """Fresh chain verdict: proven active, proven wrong, or unknown.""" + return self._ensure_block_candidate_service()._block_candidate_chain_probe( + block_hash, + expected_height=expected_height, + ) def _block_candidate_acceptance_pending( self, @@ -26520,28 +9278,34 @@ def _block_candidate_acceptance_pending( *, expected_height: int | None = None, ) -> bool: - """Return whether abandoning this candidate would discard an accepted block. - - A fresh probe wins in both directions: a candidate proven active is - accepted even if its tip observation was missed, and a candidate - proven active at the wrong height stays abandonable. When the probe - cannot prove either way, a recent own-hash tip observation keeps the - candidate deferring instead of terminal: qbitd accepted it once, so - only a durably settled chain view may discard it. - """ - probe = self._block_candidate_chain_probe( + """Return whether abandoning this candidate would discard an accepted block.""" + return self._ensure_block_candidate_service()._block_candidate_acceptance_pending( block_hash, expected_height=expected_height, ) - if probe is not None: - return probe - return self._block_candidate_acceptance_observed(block_hash) def _count_accept_pending_defer(self) -> None: - with self.lock: - self.block_candidate_accept_pending_defer_count = int( - getattr(self, "block_candidate_accept_pending_defer_count", 0) - ) + 1 + self._ensure_block_candidate_service()._count_accept_pending_defer() + + def _note_pending_parent_payout_transition_deferred( + self, + block_hash: str, + ancestor_hash: str, + ) -> None: + """Track a finalization deferral against its blocking ancestor (#190).""" + self._ensure_block_candidate_service().note_pending_parent_transition_deferral( + block_hash, + ancestor_hash, + ) + + def _note_pending_parent_payout_transition_resolved( + self, + block_hash: str, + ) -> None: + """Drop ancestor re-drive bookkeeping once the fence passes (#190).""" + self._ensure_block_candidate_service().note_pending_parent_transition_resolved( + block_hash + ) def _abandon_block_candidate( self, @@ -26554,223 +9318,16 @@ def _abandon_block_candidate( expected_height: int | None = None, stale_job_class: str | None = None, ) -> bool: - """Record a lost/failed block candidate as a BLOCK-path event. - - The share that produced the candidate was acknowledged and, when it met - the share target, credited at submit time; the block losing its race - afterwards does not un-earn it and is NOT a share rejection. It is - counted under a dedicated block-abandonment counter (by reason, so a - benign 'tip moved' race is distinguishable from a real - submitblock-rejected/ledger failure) rather than the share-reject - counters, which stay a true measure of shares refused to miners. - - Every terminal abandonment withdraws its payout-preview transition - before the outcome becomes final. ``preserve_if_accepted`` closes the - moved-tip race: if another attempt completed this hash's accepted - success tail while withdrawal was in flight, the accepted disposition - wins and the caller must finalize the outbox as submitted. Returns - whether that accepted disposition won. - - Independent of that completed-tail record, a candidate whose own - block hash is the fresh best tip, an active chain header at its - expected height, a recent own-hash tip observation, or a fresh - retained definitive submitblock success is an ACCEPTED block whose - finalization is still pending (for example after a lost submitblock - ack, or when saturation makes both probes answer "unknown" at this - instant). Terminal abandonment would discard its payout - accounting and withdraw its landed preview -- fencing payout - publication for work qbitd already accepted -- so such candidates - defer for retry instead; only hashes provably absent from the active - chain (past the observation and retained-acceptance windows) - abandon terminally. The terminal - seal re-reads observation evidence atomically, so callers can order - follow-up durable work (rejecting prepared payout rows) strictly - afterward. Abandonment metrics commit only once that cleanup succeeds - or a false finalize-only disposition is frozen. - """ - if reason in PRISM_RETRYABLE_BLOCK_CANDIDATE_REASONS: - self._defer_block_candidate(reason, message, worker=worker) - return False - if ( - stale_job_class is not None - and stale_job_class not in PRISM_STALE_JOB_ABANDON_CLASSES - ): - raise ValueError( - f"unknown stale job abandon class: {stale_job_class}" - ) - outcome = getattr(self, "_block_candidate_outcome", None) - if outcome is None: - outcome = threading.local() - self._block_candidate_outcome = outcome - if ( - preserve_if_accepted - and self._block_candidate_acceptance_recorded(block_hash) - ): - # Durable accepted state already equals the prospective view, so - # any transition recreated by the losing attempt is a no-op - # override, not a withdrawal. - self._clear_accepted_block_payout_preview(block_hash) - outcome.reason = None - outcome.error = None - return True - chain_probe = self._block_candidate_chain_probe( - block_hash, - expected_height=expected_height, - ) - if chain_probe is True or ( - chain_probe is None - and ( - self._block_candidate_acceptance_observed(block_hash) - or self._block_candidate_acceptance_retained(block_hash) - ) - ): - self._count_accept_pending_defer() - self._defer_block_candidate( - PRISM_REJECTION_BLOCK_ACCEPT_PENDING, - "candidate is on (or was recently observed or definitively " - "accepted as) the active chain; refusing terminal " - f"abandonment (was {reason}: {message})", - worker=worker, - ) - return False - - # Own the cleanup invariant here rather than relying on every caller to - # remember it. Invalidation can block behind another candidate pass; - # recheck the accepted record afterward before committing abandonment. - # Capture the transition first: if late acceptance evidence forces a - # defer below, its published preview must be restorable. - with self._accepted_block_payout_preview_condition: - withdrawn_transition = self._accepted_block_payout_previews.get( - block_hash.lower() - ) - self._clear_accepted_block_payout_preview( - block_hash, - invalidate_published=True, - ) - # The invalidation above can block long enough for the chain view to - # heal (a buried accepted block is not always re-observed as the tip - # while blockwait only reports the newest of rapid connects), so an - # unknown pre-withdrawal verdict must re-probe before the terminal - # commit. A provably wrong-height verdict is immutable (headers - # cannot change height) and is never re-probed. - late_probe = ( - chain_probe - if chain_probe is False - else self._block_candidate_chain_probe( - block_hash, - expected_height=expected_height, - ) - ) - with self.lock: - accepted_race_won = bool( - preserve_if_accepted - and block_hash.lower() in self._accounted_accepted_block_hashes - ) - # A blockwait observation can also register during the blocking - # invalidation. The disposition seal must consult that evidence - # atomically or the same blind spot reopens inside this window; - # the probe still wins both directions. Metrics are committed - # later, after prepared-state cleanup can no longer reverse this - # decision. - late_acceptance_observed = bool( - not accepted_race_won - and ( - late_probe is True - or ( - late_probe is not False - and ( - self._block_candidate_acceptance_observed( - block_hash - ) - or self._block_candidate_acceptance_retained( - block_hash - ) - ) - ) - ) - ) - if not accepted_race_won and not late_acceptance_observed: - outcome.reason = reason - outcome.error = message - outcome.stale_job_class = stale_job_class - # Seal the disposition in the same critical section that - # commits it: stop matching tip observations for this hash so - # no acceptance evidence can register between this terminal - # decision and the caller's follow-up durable work (rejecting - # prepared payout rows). Observation registration takes this - # same lock, so exclusion across the gap is total. A crash - # before the durable outbox update replays the candidate, - # which re-registers and re-evaluates from live chain state. - self._outstanding_block_candidate_hashes.discard( - block_hash.lower() - ) - self._tip_observed_accepted_block_hashes.pop( - block_hash.lower(), - None, - ) - if accepted_race_won: - self._clear_accepted_block_payout_preview(block_hash) - outcome.reason = None - outcome.error = None - return True - if late_acceptance_observed: - # Restore the landed barrier the withdrawal just removed (and pop - # its fail-closed tombstone): the candidate is still an accepted - # block pending finalization, so descendant builders must keep - # waiting on its preview -- not fail closed -- until the deferred - # retry's accepted tail republishes it. Without this, retries - # that keep deferring on observation evidence alone would leave - # the tombstone fencing template refreshes, recreating the - # coordination-blocked stall this path exists to prevent. - self._begin_accepted_block_payout_preview( - block_hash, - block_height=expected_height, - ) - if expected_height is not None: - self._mark_accepted_block_payout_landed( - block_hash, - block_height=expected_height, - ) - if ( - withdrawn_transition is not None - and withdrawn_transition.preview is not None - ): - # The withdrawal superseded payout publication and, on a - # lost publication race, left delivery fenced. Republish the - # withdrawn preview now so admission reopens with the defer - # rather than staying coordination-blocked across the - # deferral cycles until the accepted tail republishes it. - try: - self._publish_accepted_block_payout_preview( - block_hash, - self._materialize_prior_balance_preview( - withdrawn_transition.preview - ), - ) - except Exception: - print( - "prism coordinator: could not republish withdrawn " - f"payout preview hash={block_hash}; the scheduled " - "refresh will retry publication", - flush=True, - ) - traceback.print_exc() - self._count_accept_pending_defer() - self._defer_block_candidate( - PRISM_REJECTION_BLOCK_ACCEPT_PENDING, - "acceptance evidence arrived during payout-preview " - f"withdrawal; refusing terminal abandonment (was {reason}: " - f"{message})", - worker=worker, - ) - outcome.error = None - return False - - print( - f"prism coordinator: block candidate abandoned reason={reason}: {message}", - flush=True, + """Record a lost/failed block candidate as a BLOCK-path event.""" + return self._ensure_block_candidate_service().record_abandoned( + reason, + message, + block_hash=block_hash, + worker=worker, + preserve_if_accepted=preserve_if_accepted, + expected_height=expected_height, + stale_job_class=stale_job_class, ) - return False def active_block_candidate_height(self, block_hash: str) -> int | None: """Return the active-chain height for a previously submitted candidate.""" @@ -26833,7 +9390,23 @@ def preserve_active_candidate_barrier() -> None: ) return True if pending_parent_transition is None: + self._note_pending_parent_payout_transition_resolved(block_hash) return False + pending_ancestor_hash, pending_ancestor_invalidated = ( + pending_parent_transition + ) + if not pending_ancestor_invalidated: + # Issue #190: the deferral below only re-checks the transition; + # it can never resolve the stuck ancestor. Count the deferral so + # a streak against one ancestor arms a targeted in-process + # re-drive through the durable-replay path. A tombstoned + # (invalidated) transition is a withdrawal in progress that the + # withdrawal retry publication owns, not a replay-resolvable + # pendency, so it stays outside the streak. + self._note_pending_parent_payout_transition_deferred( + block_hash, + pending_ancestor_hash, + ) preserve_active_candidate_barrier() self._abandon_block_candidate( PRISM_REJECTION_LEDGER_CONFIRMATION_FAILED, @@ -26843,82 +9416,13 @@ def preserve_active_candidate_barrier() -> None: ) return True - @contextmanager def _payout_balance_serializer_released(self) -> Iterator[None]: - """Temporarily release the balance serializer around candidate audit - construction. - - The caller is the landing, which holds the serializer exactly once - (its single outer acquisition; no caller of - _land_and_confirm_block_candidate holds it). The landed-transition - fence armed before the release keeps reconciliation from mutating - balances while the serializer is free, so job delivery proceeds - during the expensive builder/verifier subprocess work instead of - queueing for its whole duration. - """ - self._payout_balance_mutation_lock.release() - try: - yield - finally: - self._acquire_block_submitter_lock( - self._payout_balance_mutation_lock, - "payout-balance-mutation", - ) + """Temporarily release the balance serializer around candidate audit""" + return self._ensure_payout_state_service()._payout_balance_serializer_released() - def _replayed_payout_window_reproducible( - self, - context: PrismJobContext, - ) -> bool: - """Whether a reconstructed candidate's payout window replays intact. - - Append-invalidation epochs are process-local, so the landing epoch - fence stands down for a candidate rebuilt from durable intent. The - durable ledger is the surviving authority instead: the reward-window - contract keeps the share window replayable at the artifact's declared - anchor, so a share row that became durably visible only after the - recorded window walk -- on either side of a restart -- appears in the - replayed window while the recorded coinbase omits it. Only omissions - fail the candidate: an appended row leaves the tip, height, and carry - balances untouched, which is exactly why no other landing fence can - see it, while a recorded row absent from the replay is not this - hazard (share rows are append-only, and drift from settled payout - state is governed by the reorg and prior-balance fences). - """ - found_block = getattr(context, "found_block", None) - anchor_ms = ( - found_block.get("anchor_job_issued_at_ms") - if isinstance(found_block, dict) - else None - ) - network_difficulty = ( - found_block.get("network_difficulty") - if isinstance(found_block, dict) - else None - ) - audit_share_window = getattr(self.ledger, "audit_share_window", None) - if ( - anchor_ms is None - or network_difficulty is None - or not callable(audit_share_window) - ): - # Fail closed: a candidate whose window cannot be replayed at a - # declared anchor cannot prove its coinbase pays the window the - # durable ledger requires. - return False - durable_rows = audit_share_window( - anchor_job_issued_at_ms=int(anchor_ms), - network_difficulty=int(network_difficulty), - ) - recorded_share_ids = { - str(row.get("share_id")) - for row in context.shares_json - if isinstance(row, dict) - } - return all( - str(row.get("share_id")) in recorded_share_ids - for row in durable_rows - if isinstance(row, dict) - ) + def _replayed_payout_window_reproducible(self, context: PrismJobContext) -> bool: + """Whether a reconstructed candidate's payout window replays intact.""" + return self._ensure_payout_state_service()._replayed_payout_window_reproducible(context) def _land_and_confirm_block_candidate( self, @@ -26934,1206 +9438,92 @@ def _land_and_confirm_block_candidate( dict[str, Any], dict[str, Any], dict[str, Any], + AuditPublicationIdentity, + dict[str, Any], ] | None: - """Land, verify, publish, persist, and confirm one candidate. - - The balance serializer spans the last prior-state check through durable - confirmation. Reconciliation therefore cannot change the base beneath - the accepted coinbase, while ordinary job delivery remains unblocked. - The caller has already run submitblock on the lock/DB-free fast lane. - The audit bundle build and verification execute with the serializer - temporarily released (the landed fence stays armed), so neither block - announcement nor job delivery waits on audit construction. - """ - context = candidate.context - submission = candidate.submission - expected_height = int(context.template["height"]) - block_hash = str(submission.block_hash_hex).lower() - parent_hash = str(context.template["previousblockhash"]) - self._ensure_job_cache_state() - durable_payout_state = bool( - getattr(self.ledger, "durable_payout_state", False) + """Land, verify, publish, persist, and confirm one candidate (B3).""" + return self._ensure_block_finalization_service()._land_and_confirm_block_candidate( + candidate, + current_tip=current_tip, + already_active=already_active, + worker=worker, + node_submission=node_submission, + revalidated_append_epoch=revalidated_append_epoch, ) - with self._block_submitter_lock( - self._payout_balance_mutation_lock, - "payout-balance-mutation", - ): - if self._defer_for_pending_parent_payout_transition( - block_hash=block_hash, - parent_hash=parent_hash, - parent_height=expected_height - 1, - worker=worker, - active_candidate_hash=block_hash if already_active else None, - active_candidate_height=expected_height if already_active else None, - ): - return None - block_state: dict[str, object] | None = None - block_state_reader = getattr(self.ledger, "pool_block_state", None) - transition_already_landed = self._accepted_block_payout_transition_landed( - block_hash - ) - reorg_reconciled: bool | None = None - if already_active and not transition_already_landed: - # A replayed active ancestor may coexist with balances from an - # orphaned pool block. Reconcile that global state before this - # transition becomes a landed barrier and before validating its - # payout base. - try: - reorg_reconciled = self.ensure_reorg_reconciled_for_tip( - current_tip, - _coalesce_same_tip=False, - ) - except Exception: - traceback.print_exc() - self._abandon_block_candidate( - PRISM_REJECTION_BACKEND_RPC_UNAVAILABLE, - "reorg reconciliation failed before block replay", - block_hash=block_hash, - worker=worker, - ) - return None - if not reorg_reconciled: - self._abandon_block_candidate( - PRISM_REJECTION_BACKEND_RPC_UNAVAILABLE, - "reorg reconciliation reported an untrusted chain view", - block_hash=block_hash, - worker=worker, - ) - return None - if already_active and callable(block_state_reader): - self._record_block_submitter_phase("pool-block-state") - block_state = block_state_reader(block_hash=block_hash) - self._record_block_submitter_phase("pool-block-state:complete") - already_confirmed = bool( - block_state is not None - and str(block_state.get("chain_state", "")) == "confirmed" - and str(block_state.get("maturity_state", "")) != "reversed" - ) - if already_confirmed: - # The outbox terminal update can fail after a fully durable - # confirmation. Do not replace later global balances with an - # ancestor-only preview during exact-idempotent replay. - self._clear_accepted_block_payout_preview(block_hash) - reorg_reconciled = True - elif already_active: - self._begin_accepted_block_payout_preview( - block_hash, - block_height=expected_height, - ) - self._mark_accepted_block_payout_landed( - block_hash, - block_height=expected_height, - ) - reorg_reconciled = True - elif transition_already_landed: - # A prior attempt reached submitblock while holding this - # serializer. External reconciliation is barred until it - # confirms or is withdrawn, so retry its durable steps directly. - reorg_reconciled = True - else: - try: - reorg_reconciled = self.ensure_reorg_reconciled_for_tip( - current_tip, - _coalesce_same_tip=False, - ) - except Exception: - traceback.print_exc() - self._abandon_block_candidate( - PRISM_REJECTION_BACKEND_RPC_UNAVAILABLE, - "reorg reconciliation failed before block submit", - block_hash=block_hash, - worker=worker, - ) - return None - if not reorg_reconciled: - self._abandon_block_candidate( - PRISM_REJECTION_BACKEND_RPC_UNAVAILABLE, - "reorg reconciliation reported an untrusted chain view", - block_hash=block_hash, - worker=worker, - ) - return None - if ( - already_active - and not already_confirmed - and self._defer_for_pending_parent_payout_transition( - block_hash=block_hash, - parent_hash=parent_hash, - parent_height=expected_height - 1, - worker=worker, - ) - ): - return None - # A late-visible append advances only the append-invalidation - # epoch: the tip, payout generation, and balance digest all - # survive, yet this candidate's coinbase pays from a window that - # omitted the late row. The refresh wave retires the job - # asynchronously; until it does, membership admission still lets - # the pre-append job submit, so landing fails closed here. - # Collection candidates settle solver-pays-all and are exempt, as - # at every other epoch fence. A negative stamp is a candidate - # reconstructed from durable intent: the caller revalidated its - # recorded window against the durable ledger before this landing - # and rebased it onto the live epoch sequence at that read, so - # from here both kinds of candidate share one epoch fence. A - # reconstructed candidate on a backend that cannot revalidate - # carries no epoch and the fence stands down. - context_append_epoch = int( - getattr(context, "payout_append_invalidation_epoch", 0) - ) - effective_append_epoch = ( - context_append_epoch - if context_append_epoch >= 0 - else revalidated_append_epoch - ) - with self._job_cache_lock: - live_append_epoch = int( - self._payout_ledger_append_invalidation_epoch - ) - if ( - not already_active - and not getattr(context, "collection_only", False) - and not node_submission.attempted - and effective_append_epoch is not None - and effective_append_epoch != live_append_epoch - ): - # Fail closed only while nothing has been offered: a - # fast-lane candidate already reached qbitd, so a moved - # epoch can no longer withhold its coinbase — the offer's - # own classification below decides (an error retries - # duplicate-safe, a rejection stays terminal, and an - # accepted or duplicate result proceeds to the post-offer - # fence, which keeps the as-issued snapshot for - # accounting). - self._abandon_block_candidate( - PRISM_REJECTION_STALE_JOB, - "payout window was invalidated by a late-visible share append", - block_hash=block_hash, - worker=worker, - expected_height=expected_height, - stale_job_class="append_epoch_stale", - ) - return None - if ( - durable_payout_state - and not already_active - and not self.prior_balances_match_current(context.prior_balances) - ): - self._abandon_block_candidate( - PRISM_REJECTION_STALE_JOB, - "prior balances changed since the job was issued", - block_hash=block_hash, - worker=worker, - expected_height=expected_height, - stale_job_class="balance_stale", - ) - return None - if not already_active and not node_submission.attempted: - before_height = int(self.rpc.call("getblockcount")) - if before_height + 1 != expected_height: - self._abandon_block_candidate( - PRISM_REJECTION_BLOCK_STALE, - f"stale block height: template={expected_height} tip={before_height}", - block_hash=block_hash, - worker=worker, - expected_height=expected_height, - ) - return None - if not already_active: - # Register before a fallback submitblock can expose this hash as the new - # tip. Child builders will wait for the verified preview rather - # than reading balances that omit their new parent. - self._begin_accepted_block_payout_preview( - block_hash, - block_height=expected_height, - ) - # Treat the submit outcome as uncertain before entering RPC. - # If transport fails after qbitd accepted the block, this - # conservative barrier preserves the coinbase's payout base. - self._mark_accepted_block_payout_landed( - block_hash, - block_height=expected_height, - ) - fallback_submit_under_fence = not node_submission.attempted - if not node_submission.attempted: - - def _verify_lease_before_submitblock() -> None: - # Runs at the RPC boundary — after any wait on the - # landing fence lock below — so the lease runway the - # verification proves is still intact when the RPC - # starts. A predating append_batch can hold that - # lock across its ledger commit and epoch bump far - # longer than any scheduling headroom, and a - # verification taken before the wait would measure - # runway the wait then consumes. - try: - self._require_fresh_ledger_lease_for_external_side_effect( - "submitblock" - ) - except WriterLeaseRenewalDeferred: - if not transition_already_landed: - # The refusal fired before submitblock, so - # this attempt's outcome is not uncertain: - # qbitd provably never saw the block (no - # fast-lane node offer either — attempted is - # False). Unwind the landed bar this attempt - # armed — leaving it would bar reconciliation - # and payout-state publication for as long as - # the writer's own fenced write keeps the - # renewal deferred, though nothing needs - # preserving. The begun preview stays and the - # retry re-arms both. A bar landed by an - # earlier attempt that did reach submitblock - # keeps standing for that attempt's uncertain - # outcome. - self._unmark_accepted_block_payout_landed( - block_hash - ) - raise - - # The epoch fence above is advisory: it releases the lock - # after one read, so an append-side bump could still commit - # between that read and the RPC below. This one is - # authoritative -- the bump acquires the same fence lock, so - # holding it across submitblock means no late-visible append - # can advance the epoch between this comparison and the - # block entering qbitd. The lock spans the lease - # verification and one RPC, and only on this boundary; - # ordinary share commits never touch it (the append side - # takes it only for rows that predate a live anchor, and - # this landing's own declared anchor stays exposed for - # the landing's duration). - append_epoch_raced = False - if ( - effective_append_epoch is None - or getattr(context, "collection_only", False) - ): - _verify_lease_before_submitblock() - node_submission = self._submit_block_candidate_to_node( - candidate - ) - else: - with self._payout_append_landing_fence_lock: - with self._job_cache_lock: - live_append_epoch = int( - self._payout_ledger_append_invalidation_epoch - ) - if live_append_epoch != effective_append_epoch: - append_epoch_raced = True - else: - _verify_lease_before_submitblock() - node_submission = ( - self._submit_block_candidate_to_node(candidate) - ) - if append_epoch_raced: - self._abandon_block_candidate( - PRISM_REJECTION_STALE_JOB, - "payout window was invalidated by a late-visible share append", - block_hash=block_hash, - worker=worker, - expected_height=expected_height, - stale_job_class="append_epoch_stale", - ) - return None - if node_submission.error is not None: - raise node_submission.error - result = node_submission.result - if result not in (None, "duplicate"): - self._abandon_block_candidate( - PRISM_REJECTION_SUBMITBLOCK_REJECTED, - f"submitblock rejected candidate: {result}", - block_hash=block_hash, - worker=worker, - expected_height=expected_height, - ) - return None - if ( - not fallback_submit_under_fence - and effective_append_epoch is not None - and not getattr(context, "collection_only", False) - ): - # The fast lane offered this block to qbitd without any - # ledger synchronization, so the landing fence can no - # longer gate submitblock itself. It still orders the - # accounting decision: wait out any fenced predating - # append whose durable commit is in flight (the commit - # holds this lock across its epoch bump). A moved epoch - # is no longer terminal here, though: every candidate - # reaching this point was accepted or already known by - # the node (rejections returned above), so its coinbase - # paid the as-issued window the moment it landed and no - # rebuild can change it. Abandoning would permanently - # discard payout accounting for a live block — the - # ledger would then re-pay the same carried balances in - # a later block — so the as-issued snapshot is kept for - # accounting and the predating share simply rides the - # next window (its ledger credit is untouched). The - # active-height check below still owns the - # did-it-actually-land verdict, and the pre-offer fence - # above still fails closed while nothing has been - # submitted. - with self._payout_append_landing_fence_lock: - with self._job_cache_lock: - live_append_epoch = int( - self._payout_ledger_append_invalidation_epoch - ) - if live_append_epoch != effective_append_epoch: - print( - "prism coordinator: payout window was " - "invalidated by a late-visible share append " - f"after the node offer hash={block_hash}; " - "keeping the as-issued payout snapshot for " - "accounting", - flush=True, - ) - active_hash = str( - self.rpc.call("getblockhash", [expected_height]) - ).lower() - if active_hash != block_hash: - self._abandon_block_candidate( - PRISM_REJECTION_SUBMITBLOCK_REJECTED, - f"submitted block is not active at height {expected_height}", - block_hash=block_hash, - worker=worker, - expected_height=expected_height, - ) - return None - self._cancel_obsolete_job_builds("direct PRISM block accepted") - self._mark_tip_refresh_pending(block_hash) - self._schedule_tip_refresh_retry() - - preview: list[dict[str, object]] | None = None - issued_preview = getattr(context, "prospective_prior_balances", None) - if not already_confirmed and issued_preview is not None: - # The compact preview came from the immutable issued job - # summary. Publish it before rebuilding/canonicalizing the full - # audit bundle, without retaining that bundle's shares tree. - preview = self._materialize_prior_balance_preview(issued_preview) - if durable_payout_state and not self.prior_balances_match_current( - context.prior_balances - ): - self.request_shutdown() - self._clear_accepted_block_payout_preview( - block_hash, - invalidate_published=True, - ) - self._abandon_block_candidate( - PRISM_REJECTION_LEDGER_CONFIRMATION_FAILED, - "accepted block payout base changed before preview publication", - block_hash=block_hash, - worker=worker, - ) - return None - self._publish_accepted_block_payout_preview(block_hash, preview) - - self._record_block_submitter_phase("audit-build") - # The bundle derives only from inputs frozen on the candidate - # (share window, prior balances, extranonces, template fields), - # so the serializer is released around the builder/verifier - # subprocess work: submitblock above has already run for fresh - # candidates -- announcement is never delayed by audit - # construction -- and the landed fence keeps reconciliation out - # while job delivery proceeds. Child builds consume the compact - # preview published above until the verified preview lands. - with self._payout_balance_serializer_released(): - candidate_bundle_path = self.temporary_audit_bundle_path( - block_hash=submission.block_hash_hex - ) - final_bundle = self.build_audit_bundle( - shares=context.shares_json, - found_block=context.found_block, - prior_balances=context.prior_balances, - coinbase_script_sig_suffix_hex=self.coinbase_script_sig_suffix_hex( - candidate.extranonce1_hex, - candidate.extranonce2_hex, - ), - witness_merkle_leaves_hex=list( - getattr(context.job, "witness_merkle_leaves_hex", ()) - ) - or direct_stratum.witness_merkle_leaves_hex( - getattr(context.job, "transaction_hexes", ()) - ), - ctv_fee_parent_hash=parent_hash, - canonical_output_path=candidate_bundle_path, - ) - # Compatibility builders used by tests and older integrations - # may ignore canonical_output_path. Persist their logical - # bundle via the normal canonicalization fallback without - # mislabeling bytes. - if not candidate_bundle_path.exists(): - candidate_bundle_path = self.write_temporary_audit_bundle( - final_bundle, - block_hash=submission.block_hash_hex, - ) - try: - final_manifest = final_bundle["signed_coinbase_manifest"]["manifest"] - final_coinbase_tx_hex_raw = final_manifest["coinbase_tx_hex"] - if not isinstance(final_coinbase_tx_hex_raw, str): - raise ValueError( - "final audit bundle coinbase_tx_hex is not a string" - ) - final_coinbase_tx_hex = final_coinbase_tx_hex_raw.lower() - except BaseException: - try: - candidate_bundle_path.unlink() - except FileNotFoundError: - pass - raise - if final_coinbase_tx_hex != submission.coinbase_tx_hex.lower(): - try: - candidate_bundle_path.unlink() - except FileNotFoundError: - pass - self.request_shutdown() - self._clear_accepted_block_payout_preview( - block_hash, - invalidate_published=True, - ) - self._abandon_block_candidate( - PRISM_REJECTION_CANDIDATE_AUDIT_MISMATCH, - "final audit bundle coinbase does not match submitted coinbase", - block_hash=block_hash, - worker=worker, - ) - return None - payout_commit_started: float | None = None - payout_commit_source: int | None = None - try: - with self._payout_balance_serializer_released(): - self._record_block_submitter_phase("audit-verify") - report = self.verify_bundle( - candidate_bundle_path, - submission.coinbase_tx_hex, - self.trusted_ledger_writer_public_key_hex(final_bundle), - expected_coinbase_value_sats=int( - context.template["coinbasevalue"] - ), - ) - persistence_canonical_bundle_path = ( - self.verified_canonical_bundle_path( - candidate_bundle_path, - report, - ) - ) - verified_preview = ( - self._accepted_block_payout_preview_from_bundle( - final_bundle, - prior_balances=context.prior_balances, - ) - ) - self._record_block_submitter_phase("audit-verify:complete") - if not already_confirmed: - if preview is None and durable_payout_state: - live_prior_balances = self.settlement_balances_by_program( - self.ledger.current_prior_balances() - ) - expected_prior_balances = self.settlement_balances_by_program( - context.prior_balances - ) - if live_prior_balances != expected_prior_balances: - self.request_shutdown() - self._clear_accepted_block_payout_preview( - block_hash, - invalidate_published=True, - ) - self._abandon_block_candidate( - PRISM_REJECTION_LEDGER_CONFIRMATION_FAILED, - "accepted block payout base changed before preview publication", - block_hash=block_hash, - worker=worker, - ) - return None - try: - self._publish_accepted_block_payout_preview( - block_hash, - verified_preview, - ) - except RuntimeError as exc: - self.request_shutdown() - self._clear_accepted_block_payout_preview( - block_hash, - invalidate_published=True, - ) - self._abandon_block_candidate( - PRISM_REJECTION_CANDIDATE_AUDIT_MISMATCH, - "verified final payout preview does not match the " - f"issued block job: {exc}", - block_hash=block_hash, - worker=worker, - ) - return None - preview = verified_preview - - # The verified preview is now the effective balance snapshot, - # so persistence can do canonicalization, body writes, copies, - # and bulk SQL without owning the delivery gate. - payout_commit_started = time.monotonic() - payout_commit_source = self._capture_payout_state_source()[1] - self._record_block_submitter_phase("persist-accepted-block") - persistence = self.ledger.persist_accepted_block( - block_hash=submission.block_hash_hex, - block_height=expected_height, - parent_hash=parent_hash, - final_bundle=final_bundle, - audit_report=report, - canonical_bundle_path=persistence_canonical_bundle_path, - ) - self._record_block_submitter_phase("persist-accepted-block:complete") - active_hash = str( - self.rpc.call("getblockhash", [expected_height]) - ).lower() - if active_hash != block_hash: - if already_confirmed: - self._abandon_block_candidate( - PRISM_REJECTION_BACKEND_RPC_UNAVAILABLE, - "accepted ancestor left the active chain during replay", - block_hash=block_hash, - worker=worker, - ) - return None - # Seal the disposition BEFORE touching the prepared - # payout rows: a rejected row can never be promoted by a - # later confirmation (and reactivation only covers - # inactive rows), so rejection must follow -- never - # precede -- a terminal decision. The abandon defers on - # live acceptance evidence; its terminal commit consults - # that evidence atomically with the commit AND seals the - # hash against further observation matching, so no - # evidence can register anywhere in the gap between the - # sealed decision and this rejection. A crash in between - # leaves the outbox row pending; restart replay - # re-registers the hash and re-evaluates from live chain - # state before rejecting. - self._abandon_block_candidate( - PRISM_REJECTION_BLOCK_STALE, - "accepted block left the active chain before ledger confirmation", - block_hash=block_hash, - worker=worker, - expected_height=expected_height, - ) - outcome = getattr(self, "_block_candidate_outcome", None) - sealed_reason = ( - getattr(outcome, "reason", None) - if outcome is not None - else None - ) - if sealed_reason == PRISM_REJECTION_BLOCK_STALE: - active_tip_height = int(self.rpc.call("getblockcount")) - self.reject_prepared_block( - block_hash=block_hash, - active_tip_height=active_tip_height, - ) - return None - self._record_block_submitter_phase("confirm-accepted-block") - confirmation = self.ledger.confirm_accepted_block( - block_hash=block_hash, - # The ledger confirmation function matches this value - # against the candidate row's own height. An accepted - # ancestor can be finalized after newer blocks arrive. - active_tip_height=expected_height, - ) - self._record_block_submitter_phase( - "confirm-accepted-block:complete" - ) - confirmed_count = int(confirmation.get("confirmed_count", 0)) - if confirmed_count not in {0, 1}: - self.request_shutdown() - self._clear_accepted_block_payout_preview( - block_hash, - invalidate_published=True, - ) - self._abandon_block_candidate( - PRISM_REJECTION_LEDGER_CONFIRMATION_FAILED, - f"ledger did not confirm accepted block {block_hash}", - block_hash=block_hash, - worker=worker, - ) - return None - - if durable_payout_state: - # Compare the durable active-chain view as of this block, - # not the global latest view: an exact replay may finalize - # ancestor A after later pool block B is already confirmed. - # This also preserves the invariant across restart after a - # prior post-confirm mismatch instead of silently accepting - # the already-confirmed row on the next attempt. - as_of_reader = getattr( - self.ledger, - "prior_balances_after_pool_block", - None, - ) - confirmed_balances = self.normalized_prior_balances( - as_of_reader(block_hash=block_hash) - if callable(as_of_reader) - else self.ledger.current_prior_balances() - ) - if self.settlement_balances_by_program( - confirmed_balances - ) != self.settlement_balances_by_program(preview): - self.request_shutdown() - self._clear_accepted_block_payout_preview( - block_hash, - invalidate_published=True, - ) - self._abandon_block_candidate( - PRISM_REJECTION_LEDGER_CONFIRMATION_FAILED, - "confirmed payout balances do not match the published " - f"preview for accepted block {block_hash}", - block_hash=block_hash, - worker=worker, - ) - return None - # Durability caught up to the already-published logical state; - # clearing the parent override needs no second generation bump. - self._clear_accepted_block_payout_preview(block_hash) - self._schedule_current_payout_ledger_artifact_if_missing() - payout_publication_required = ( - self._payout_source_requires_publication() - ) - payout_publication_fenced = ( - self._payout_state_publication_fenced() - ) - if payout_publication_required or payout_publication_fenced: - # A covered replay normally has no publication work. The - # exception is a leaked delivery fence whose source already - # published: force one republish so the replay heals it. - covered_replay_fence = ( - payout_publication_fenced - and not payout_publication_required - ) - with self.lock: - pending_cause = self._payout_state_source[2] - # A bounded preview-publication loss already left the gate - # fenced and its retry scheduled. Do not monopolize the - # submitter with a second retry budget. Uncertain commits, - # ordinary unfenced tip sources, and a covered replay's - # leaked fence still reconcile now. - publish_now = ( - covered_replay_fence - or pending_cause == "direct_block_uncertain" - or not payout_publication_fenced - ) - published: int | None = None - if publish_now and getattr( - self, - "reorg_reconciler_enabled", - True, - ): - with self.lock: - latest_tip = self._payout_state_source[1] - summary = self.reconcile_prism_pool_blocks_once( - tip_hash=latest_tip, - _force_publish=True, - _source_reserved=True, - ) - reconciled_generation = summary.get("published_generation") - if isinstance(reconciled_generation, int): - published = reconciled_generation - elif publish_now: - published = ( - self._publish_current_payout_state_with_retry_budget() - ) - if publish_now and published is None: - # The block is durably confirmed; only the payout - # publication lost its race. Aborting would keep the - # outbox row pending and replay persist/confirm churn - # for an already-final block. Keep delivery fenced and - # let the scheduled tip refresh publish the newest - # source; this candidate's durable work is complete. - self._block_payout_state_publication() - print( - "prism coordinator: accepted block confirmed " - "durably; payout publication deferred to the " - f"scheduled refresh hash={block_hash}", - flush=True, - ) - return final_bundle, report, persistence, confirmation - except Exception: - if payout_commit_started is not None and payout_commit_source is not None: - # Persistence/confirmation can report failure after a - # durable partial commit. Supersede every prepared source - # and keep all delivery fenced until replay/reconciliation - # proves the resulting ledger state. - self._block_payout_state_publication( - supersede_with=( - payout_commit_source, - block_hash, - "direct_block_uncertain", - payout_commit_started, - ) - ) - raise - finally: - if payout_commit_started is not None: - self._observe_payout_state_seconds( - "preparation", - max(0.0, time.monotonic() - payout_commit_started), - ) - try: - candidate_bundle_path.unlink() - except FileNotFoundError: - pass - @ledger_writer_operation("accepted_block_handling") - def submit_block_candidate( + def _land_block_candidate_submission( self, candidate: PrismBlockCandidate, *, node_submission: _BlockCandidateNodeSubmission | None = None, + disposition_held: bool = False, ) -> bool: - """Land one block candidate, then finalize its audit and payout state. - - Runs on the block-submitter thread (tests call it synchronously). It - never raises for a lost race and holds self.lock only for short - in-memory state mutation -- never across RPC, psql, subprocess, or - file I/O -- so share acks and job pushes stay fast while a block - lands. The durable candidate outbox is the pre-submit recovery boundary; - full audit and payout persistence happens after the latency-sensitive - ``submitblock`` call and is replayable after a crash. Returns True only - after that finalization completes. + """Run one candidate's landing tail behind the named submit port. + + Embeddings and tests replace the bound ``submit_block_candidate`` to + stand in for the node submission, so which of the two landing tails + applies is a fact about this coordinator rather than about the + block-candidate owner. Deciding it here keeps that owner from reaching + back through the runtime seam to inspect the entrypoint, and keeps the + decision per call: the replacement is installed on the instance after + the block-candidate service is constructed. + + ``node_submission`` is an already-created offer result when the caller + made one, and None when none was created -- which keeps the historical + bare entrypoint call. ``disposition_held`` states that the caller + already owns the same-hash disposition, which is what makes the + serialized inner tail, rather than the public entrypoint that would + take that guard again, the correct production call. """ - block_hash = str(candidate.submission.block_hash_hex).lower() - with self._block_candidate_disposition(block_hash): - terminal_outcome = self._block_candidate_terminal_outcome(block_hash) - if terminal_outcome is not None: - return terminal_outcome - if node_submission is None: - node_submission = self._node_submission_for_direct_candidate(candidate) - accepted = self._submit_block_candidate_serialized( + submit = self.submit_block_candidate + production_submit = ( + getattr(submit, "__func__", None) + is PrismCoordinator.submit_block_candidate + ) + if disposition_held and production_submit: + assert node_submission is not None + return self._submit_block_candidate_serialized( candidate, node_submission=node_submission, ) - if not accepted: - outcome = getattr(self, "_block_candidate_outcome", None) - if outcome is not None: - # Direct embedders do not use the outbox-finalization - # wrapper. A normal return means the serialized path also - # completed any prepared-state rejection it initiated. - self._record_committed_block_candidate_abandonment( - block_hash, - outcome, - ) - return accepted - - def _record_block_candidate_progress( - self, - phase: str = "accounting-progress", - ) -> None: - """Stamp the submitter heartbeat at a candidate-disposition boundary. - - Stamps come only from the dedicated submitter thread: dispositions - also run on client connection threads (synchronous below-target - solves), and a stamp from those threads would refresh the - ``block_submitter`` budget while the dedicated thread might be - wedged elsewhere. A disposition crosses several ledger and - filesystem writes that slow down together under database pressure; - stamping each completed phase on the owner thread keeps a - progressing disposition inside the liveness budget while a wedged - phase still leaves the watchdog able to recover the process -- the - same shape as the CTV broadcaster's per-row stamping, whose name - likewise maps to a single thread. - """ - self._record_block_submitter_phase(phase) + if node_submission is None: + return bool(submit(candidate)) + return self._account_block_candidate_after_node_submit( + candidate, + node_submission, + ) - def _submit_block_candidate_serialized( + @ledger_writer_operation("accepted_block_handling") + def submit_block_candidate( self, candidate: PrismBlockCandidate, *, - node_submission: _BlockCandidateNodeSubmission, + node_submission: _BlockCandidateNodeSubmission | None = None, ) -> bool: - """Process a candidate while its same-hash disposition guard is held.""" - outcome = getattr(self, "_block_candidate_outcome", None) - if outcome is None: - outcome = threading.local() - self._block_candidate_outcome = outcome - outcome.reason = None - outcome.error = None - outcome.stale_job_class = None - context = candidate.context - submission = candidate.submission - worker = candidate.client.username or None - expected_height = int(context.template["height"]) - block_hash = str(submission.block_hash_hex).lower() - parent_hash = str(context.template["previousblockhash"]) - self._ensure_job_cache_state() - # Every disposition (queue drain, synchronous below-target submit, - # outbox replay, retained retry) marks its hash outstanding so tip - # observations arriving on other threads can register acceptance. - self._register_outstanding_block_candidate(block_hash) - self._record_block_candidate_progress("disposition-start") - if ( - self._block_candidate_acceptance_recorded(block_hash) - and node_submission.error is not None - ): - # A concurrent same-hash pass completed the success tail while - # this duplicate-safe node offer waited for disposition. Do not - # recreate its payout transition or accounting work. - self._clear_accepted_block_payout_preview(block_hash) - return True - with self.lock: - accepted_count = int(self.accepted_block_count) - pool_closed = ( - block_hash not in self._accounted_accepted_block_hashes - and ( - accepted_count >= int(self.max_blocks) - or (bool(self.stop_after_block) and accepted_count >= 1) - ) - ) - if pool_closed and self._block_candidate_chain_probe( - block_hash, - expected_height=expected_height, - ) is True: - # The chain provably contains this block; its payout accounting - # must complete regardless of when the pool stopped accepting - # new work. Fall through to the normal disposition below, which - # resumes the accepted success tail. Observation evidence alone - # deliberately does not open this gate -- an unprovable view - # defers via the abandon path instead, so a closed pool can - # never fall through to submitblock on stale evidence. - pool_closed = False - if pool_closed: - self._abandon_block_candidate( - PRISM_REJECTION_POOL_CLOSED, - "pool is no longer accepting blocks", - block_hash=block_hash, - worker=worker, - expected_height=expected_height, - ) - return False - self._record_block_candidate_progress("current-tip-rpc") - observed_tip = str(self.rpc.call("getbestblockhash")) - self._record_block_candidate_progress("current-tip-rpc:complete") - # A successful or transport-ambiguous fast-lane call can change the - # tip before this post-submit probe. It is still a *fresh* attempt, - # not an active replay: run the normal validation/persistence tail - # against the candidate's stamped parent. A later getblockhash check - # proves a successful acknowledgement, while an ambiguous transport - # outcome stays pending for duplicate-safe replay. Duplicate replies - # are replay evidence and retain the live-tip classification; the - # abandonment tie-breaker separately consults this process's own - # recorded offer evidence so an unprovable chain view cannot - # terminally discard a block the node already told us it has. - fresh_or_uncertain_submit = bool( - node_submission.attempted - and ( - node_submission.error is not None - or node_submission.result is None - ) + """Land one block candidate, then finalize its audit and payout state.""" + return self._ensure_block_finalization_service().submit_block_candidate( + candidate, + node_submission=node_submission, ) - current_tip = parent_hash if fresh_or_uncertain_submit else observed_tip - landed_height: int | None = None - if current_tip.lower() == block_hash: - landed_height = expected_height - elif current_tip != parent_hash: - try: - landed_height = self.active_block_candidate_height(block_hash) - except Exception: - traceback.print_exc() - self._abandon_block_candidate( - PRISM_REJECTION_BACKEND_RPC_UNAVAILABLE, - "could not determine whether a prior candidate is active", - block_hash=block_hash, - worker=worker, - ) - return False - already_active = landed_height == expected_height - if landed_height is not None and not already_active: - self._abandon_block_candidate( - PRISM_REJECTION_BLOCK_STALE, - f"candidate active at unexpected height {landed_height}", - block_hash=block_hash, - worker=worker, - expected_height=expected_height, - ) - return False - if already_active: - # A disposition probe is a tip observation too: remember it so a - # later attempt cannot terminally abandon this hash on a racing - # chain snapshot after this attempt fails mid-tail. - self._note_tip_observation_for_candidates(block_hash) - print( - "prism coordinator: resuming finalization for active block candidate " - f"height={landed_height} hash={submission.block_hash_hex}", - flush=True, - ) - elif parent_hash != current_tip: - if self._block_candidate_acceptance_recorded(block_hash): - # A duplicate wakeup can reach this check after the accepted - # success tail but after a newer tip hides the candidate from - # the active-header probe. Its durable work is already done; - # let the caller finalize the outbox as submitted. - self._clear_accepted_block_payout_preview(block_hash) - return True - accepted_race_won = self._abandon_block_candidate( - PRISM_REJECTION_STALE_JOB, - f"tip moved before submit: {current_tip}", - block_hash=block_hash, - worker=worker, - preserve_if_accepted=True, - expected_height=expected_height, - stale_job_class="tip_moved", - ) - return accepted_race_won - if ( - node_submission.attempted - and node_submission.error is None - and node_submission.result not in (None, "duplicate") - and not already_active - ): - self._abandon_block_candidate( - PRISM_REJECTION_SUBMITBLOCK_REJECTED, - f"submitblock rejected candidate: {node_submission.result}", - block_hash=block_hash, - worker=worker, - expected_height=expected_height, - ) - return False - # A reconstructed candidate revalidates BEFORE the balance - # serializer: the audit share-window replay is the slow oracle walk - # and takes the ledger writer lock, so running it inside - # _payout_balance_mutation_lock would stall share persistence and - # every other landing for the walk's duration. The read is safe out - # here because the window is append-only content -- a pass can only - # be invalidated by a later append, and any such append after the - # baseline epoch captured below advances the live epoch (this - # landing's declared anchor stays exposed to the append-side - # predates() checks), which the landing's fences compare against. - collection_only = bool(getattr(context, "collection_only", False)) - context_append_epoch = int( - getattr(context, "payout_append_invalidation_epoch", 0) - ) - revalidated_append_epoch: int | None = None - landing_anchor_token: int | None = None - if not already_active and not collection_only: - found_block = getattr(context, "found_block", None) - declared_anchor_ms = ( - found_block.get("anchor_job_issued_at_ms") - if isinstance(found_block, dict) - else None - ) - if declared_anchor_ms is not None: - # With no armed artifact and no in-flight walk (an outbox - # replay at startup, or a landing after the artifact was - # disarmed), a replay-shaped append would skip the epoch - # bump entirely; exposing the landing window's own anchor - # guarantees the bump the fences below check for. - landing_anchor_token = self._expose_inflight_scan_anchor( - int(declared_anchor_ms) - ) - # An append classified as unfenced before that exposure - # commits outside the landing fence, so its row could - # become durable mid-submitblock while its epoch bump - # queues behind the fence this landing holds across the - # RPC. Wait those commits (and their bump attempts - # against the now-exposed anchor) out before any window - # revalidation or epoch fence runs; no fence is held - # here, so the bump can always proceed. - self._await_unfenced_appends_predating_anchor( - int(declared_anchor_ms) - ) - try: - reconstructed_needs_revalidation = ( - not already_active - and not collection_only - and context_append_epoch < 0 - and bool(getattr(self.ledger, "durable_payout_state", False)) - ) - if reconstructed_needs_revalidation and node_submission.attempted: - # Revalidation guards an offer the node has not yet seen: a - # reconstructed window that omits a durably appended share - # must not mint a coinbase. Once the fast lane has offered - # the durable bytes, the coinbase is the node's to judge — - # an accepted or duplicate result proceeds with the - # as-issued snapshot (the post-offer epoch fences log - # rather than abandon), a rejection stayed terminal above, - # and an ambiguous transport error re-offers duplicate- - # safely on retry. Abandoning an already-offered candidate - # here would permanently discard payout accounting for a - # block qbitd may have accepted, and the audit walk is the - # slow oracle whose deadline under saturation would - # otherwise defer-loop an accepted block forever. - print( - "prism coordinator: reconstructed candidate was already " - f"offered to the node hash={block_hash}; keeping the " - "as-issued payout snapshot (window revalidation " - "skipped)", - flush=True, - ) - if ( - reconstructed_needs_revalidation - and not node_submission.attempted - ): - with self._job_cache_lock: - revalidation_base_epoch = int( - self._payout_ledger_append_invalidation_epoch - ) - try: - window_reproducible = ( - self._replayed_payout_window_reproducible(context) - ) - except Exception: - traceback.print_exc() - self._abandon_block_candidate( - PRISM_REJECTION_BACKEND_RPC_UNAVAILABLE, - "durable share-window replay failed for the " - "reconstructed candidate", - block_hash=block_hash, - worker=worker, - ) - return False - if not window_reproducible: - self._abandon_block_candidate( - PRISM_REJECTION_STALE_JOB, - "replayed payout window omits a durably appended share", - block_hash=block_hash, - worker=worker, - expected_height=expected_height, - stale_job_class="append_epoch_stale", - ) - return False - revalidated_append_epoch = revalidation_base_epoch - landed = self._land_and_confirm_block_candidate( - candidate, - # Fresh-attempt classification may use the stamped parent, but - # payout reconciliation must follow the observed post-submit tip. - current_tip=observed_tip, - already_active=already_active, - worker=worker, - node_submission=node_submission, - revalidated_append_epoch=revalidated_append_epoch, - ) - finally: - self._retire_inflight_scan_anchor(landing_anchor_token) - if landed is None: - return False - final_bundle, report, persistence, confirmation = landed - self._record_block_candidate_progress("durable-accounting:complete") - with self.lock: - already_accounted = block_hash in self._accounted_accepted_block_hashes - if already_accounted: - # The previous attempt completed every success side effect but its - # durable outbox terminal update failed. submit_next will retry that - # update after this exact-idempotent confirmation without double - # counting the block or replacing newer evidence/work. - return True - ctv_persistence = None - ctv_manifest_set = final_bundle.get("ctv_fanout_manifest_set") - if isinstance(ctv_manifest_set, dict): - self._record_block_candidate_progress("ctv-manifest-persist") - ctv_persistence = self.ledger.persist_ctv_fanout_manifest_set( - block_hash=block_hash, - manifest_set=ctv_manifest_set, - manifest_set_sha256=sha256_json_hex(ctv_manifest_set), - ) - self._record_block_candidate_progress("ctv-manifest-persist:complete") - final_bundle_path = ( - self.audit_dir - / f"prism-live-audit-bundle-{expected_height}-{block_hash}.json" - ) - self._record_block_candidate_progress("audit-envelope-write") - self.write_audit_bundle_envelope( - final_bundle_path, - block_hash=block_hash, - block_height=expected_height, - report=report, - persistence=persistence, - ) - self.prune_audit_artifacts(keep_live_path=final_bundle_path) - self._record_block_candidate_progress("audit-envelope-write:complete") - bundle_path = final_bundle_path - if candidate.credit_share_on_accept: - self._record_block_candidate_progress("accepted-share-credit") - self.append_accepted_share( - candidate.client, - context, - submission, - candidate.pending_share, - candidate_intent=self.block_candidate_intent(candidate), - ) - # The preview window was intentionally clamped below this pending - # winning share. Once the append is durable, enqueue an urgent - # delta fold so a rapidly found child does not wait for the normal - # 60-second cadence to include its payout obligation. - with self._job_cache_lock: - payout_generation = self._payout_state_generation - template_artifacts = self._template_artifacts - if template_artifacts is not None: - self._schedule_payout_ledger_artifact_preparation( - payout_generation, - template_artifacts.network_difficulty, - bypass_build_interval=True, - ) - self._record_block_candidate_progress("accepted-share-credit:complete") - # Aggregate counts only: materializing the whole share history - # (all_shares) here would scan the full ledger twice per block, - # and would grow without bound as the ledger grows. The counters are - # served from the ledger's maintained cache; a cold cache (first - # read after process start) runs the exact aggregate synchronously - # and stays watchdog-eligible on purpose, so a wedged read keeps the - # exit-and-replay recovery path instead of hanging the disposition - # invisibly. - self._record_block_candidate_progress("accepted-share-stats") - evidence_share_count, evidence_distinct_miners = self.accepted_share_stats() - self._record_block_candidate_progress("accepted-share-stats:complete") - evidence = { - "schema": "qbit.prism.live-stratum-evidence.v1", - "block_hash": block_hash, - "block_height": expected_height, - "coinbase_tx_hex": submission.coinbase_tx_hex, - "audit_bundle_path": str(bundle_path), - "audit_report": report, - "ledger_backend": self.ledger.backend_name, - "persistence": persistence, - "confirmation": confirmation, - "ctv_persistence": ctv_persistence, - "accepted_share_count": evidence_share_count, - "distinct_miner_count": evidence_distinct_miners, - "job_share_count": len(context.shares_json), - } - self._record_block_candidate_progress("evidence-write") - self.evidence_path.write_text(json.dumps(evidence, indent=2), encoding="utf-8") - self._record_block_candidate_progress("evidence-write:complete") - with self.lock: - newly_accounted = block_hash not in self._accounted_accepted_block_hashes - if newly_accounted: - self._accounted_accepted_block_hashes.add(block_hash) - self.accepted_block_count += 1 - # Replace this hash's provisional capacity reservation with - # its durable accounted slot atomically. Keeping both until - # the outbox terminal write would double-count the block and - # unnecessarily reject an unrelated next solve. - self._block_fast_lane_reservations.discard(block_hash) - self.latest_coinbase_size_bytes = len( - str( - final_bundle["signed_coinbase_manifest"]["manifest"][ - "coinbase_tx_hex" - ] - ) - ) // 2 - self.latest_evidence = evidence - should_stop = ( - newly_accounted - and (self.stop_after_block or self.accepted_block_count >= self.max_blocks) - ) - if not newly_accounted: - return True - print( - "prism coordinator: qbit accepted direct PRISM block " - f"height={expected_height} hash={block_hash}", - flush=True, + + def _record_block_candidate_progress( + self, + phase: str = "accounting-progress", + ) -> None: + """Stamp the submitter heartbeat at a candidate-disposition boundary.""" + self._ensure_block_candidate_service()._record_block_candidate_progress(phase) + + def _submit_block_candidate_serialized( + self, + candidate: PrismBlockCandidate, + *, + node_submission: _BlockCandidateNodeSubmission, + ) -> bool: + """Process a candidate while its same-hash disposition guard is held.""" + return self._ensure_block_finalization_service()._submit_block_candidate_serialized( + candidate, + node_submission=node_submission, ) - if should_stop: - self.request_shutdown() - else: - # The public submitter wrapper performs this fanout only after its - # writer scope (including outbox finalization) exits. The rare - # synchronous share path consumes the same marker after sending - # the Stratum result. - candidate.client.post_accept_refresh_block = ( - expected_height, - block_hash, - ) - return True @ledger_writer_operation("accepted_block_handling") def reject_prepared_block(self, *, block_hash: str, active_tip_height: int) -> dict[str, int | str]: @@ -28145,119 +9535,20 @@ def reject_prepared_block(self, *, block_hash: str, active_tip_height: int) -> d active_tip_height=active_tip_height, ) - def temporary_audit_bundle_path(self, *, block_hash: str) -> Path: - self.audit_dir.mkdir(parents=True, exist_ok=True) - return self.audit_dir / ( - f".prism-live-audit-bundle-candidate-{block_hash}-{uuid.uuid4().hex}.json.tmp" - ) - @staticmethod def verified_canonical_bundle_path( candidate_bundle_path: Path, report: dict[str, Any], ) -> Path | None: - expected_sha256 = str(report["audit_bundle_sha256_hex"]).lower() - digest = hashlib.sha256() - with candidate_bundle_path.open("rb") as handle: - while chunk := handle.read(1024 * 1024): - digest.update(chunk) - if digest.hexdigest() != expected_sha256: - return None - return candidate_bundle_path - - def write_temporary_audit_bundle(self, bundle: dict[str, Any], *, block_hash: str) -> Path: - path = self.temporary_audit_bundle_path(block_hash=block_hash) - with path.open("x", encoding="utf-8") as handle: - json.dump(bundle, handle, separators=(",", ":")) - handle.flush() - os.fsync(handle.fileno()) - return path - - def write_audit_bundle_envelope( - self, - path: Path, - *, - block_hash: str, - block_height: int, - report: dict[str, Any], - persistence: dict[str, Any], - ) -> None: - audit_bundle_sha256 = str( - persistence.get("audit_bundle_sha256") - or report.get("audit_bundle_sha256_hex") - or "" - ).lower() - body_uri = str(persistence.get("body_uri") or "") - envelope = { - "schema": "qbit.prism.live-audit-bundle-envelope.v1", - "block_hash": block_hash, - "block_height": block_height, - "audit_bundle_sha256": audit_bundle_sha256, - "body_uri": body_uri, - "body_filename": Path(body_uri).name if body_uri else None, - "coinbase_txid": report.get("coinbase_txid"), - "coinbase_manifest_sha256": report.get("coinbase_manifest_sha256_hex"), - "coinbase_tx_hex": report.get("coinbase_tx_hex"), - "created_at": public_api.utc_now_iso(), - } - self.write_json_atomically(path, envelope) - - def write_json_atomically(self, path: Path, payload: dict[str, Any]) -> None: - path.parent.mkdir(parents=True, exist_ok=True) - body = json.dumps(payload, separators=(",", ":"), sort_keys=True).encode("utf-8") - tmp_path = path.with_name(f".{path.name}.{uuid.uuid4().hex}.tmp") - try: - with tmp_path.open("xb") as handle: - handle.write(body) - handle.flush() - os.fsync(handle.fileno()) - tmp_path.replace(path) - finally: - try: - tmp_path.unlink() - except FileNotFoundError: - pass + return AuditArtifactStore.verified_canonical_bundle_path( + candidate_bundle_path, + report, + ) def prune_audit_artifacts(self, *, keep_live_path: Path | None = None) -> None: - self.prune_live_audit_envelopes(keep_path=keep_live_path) - self.prune_candidate_audit_bundles() - - def prune_live_audit_envelopes(self, *, keep_path: Path | None = None) -> None: - retention = int(getattr(self, "audit_live_bundle_retention", 5)) - if retention < 0: - return - keep_resolved = keep_path.resolve() if keep_path is not None else None - retained_non_keep = max(retention - 1, 0) if keep_resolved is not None else retention - paths = sorted( - self.audit_dir.glob("prism-live-audit-bundle-[0-9]*.json"), - key=lambda path: path.stat().st_mtime if path.exists() else 0, - reverse=True, - ) - retained_count = 0 - for path in paths: - if keep_resolved is not None and path.resolve() == keep_resolved: - continue - if retained_count < retained_non_keep: - retained_count += 1 - continue - try: - path.unlink() - except FileNotFoundError: - pass - - def prune_candidate_audit_bundles(self) -> None: - retention_seconds = int(getattr(self, "audit_candidate_retention_seconds", 24 * 60 * 60)) - now = time.time() - for pattern in ( - "prism-live-audit-bundle-candidate-*.json", - ".prism-live-audit-bundle-candidate-*.json.tmp", - ): - for path in self.audit_dir.glob(pattern): - try: - if retention_seconds == 0 or now - path.stat().st_mtime > retention_seconds: - path.unlink() - except FileNotFoundError: - pass + self._ensure_audit_artifact_store().prune_best_effort( + keep_live_path=keep_live_path + ) def verify_bundle( self, @@ -28266,10 +9557,78 @@ def verify_bundle( ledger_writer_public_key_hex: str, *, expected_coinbase_value_sats: int, + expected_block_height: int | None = None, ) -> dict[str, Any]: - timeout_seconds = max( - 0.001, - float( + return self._ensure_audit_artifact_store().verify_bundle( + bundle_path, + coinbase_tx_hex, + ledger_writer_public_key_hex, + expected_coinbase_value_sats=expected_coinbase_value_sats, + expected_block_height=expected_block_height, + ) + + def trusted_ledger_writer_public_key_hex(self, bundle: dict[str, Any]) -> str: + return AuditArtifactStore.trusted_writer_key( + getattr(self, "ledger_writer_public_key_hex", None), + bundle, + allow_embedded_test_key=( + getattr(self, "ledger_writer_public_key_hex", None) is None + ), + ) + + def _progress_now(self) -> float: + clock = getattr(self, "_progress_monotonic", None) + return float(clock() if callable(clock) else time.monotonic()) + + def _ensure_progress_health_service(self) -> ProgressHealthService: + service = self.__dict__.get("progress_health_service") + if service is not None: + return service + init_lock = self.__dict__.get("_progress_health_init_lock") + if init_lock is None: + # CPython's setdefault is atomic under the GIL. Focused tests may + # construct through __new__, while normal instances build the + # service during __init__ before any process thread can start. + init_lock = self.__dict__.setdefault( + "_progress_health_init_lock", + threading.Lock(), + ) + with init_lock: + service = self.__dict__.get("progress_health_service") + if service is not None: + return service + service = ProgressHealthService( + started_monotonic=float( + getattr(self, "started_monotonic", time.monotonic()) + ), + initial_payout_generation=int( + getattr(self, "_payout_state_generation", 0) + ), + # Resolve the clock through the coordinator seam at call time + # so the historical ``_progress_monotonic`` test override + # keeps steering the service after construction. + monotonic=self._progress_now, + ) + self.__dict__["progress_health_service"] = service + return service + + def _progress_health_config(self) -> ProgressHealthConfig: + return ProgressHealthConfig( + pending_refresh_deadline_seconds=float( + getattr( + self, + "health_pending_refresh_max_age_seconds", + DEFAULT_PRISM_HEALTH_PENDING_REFRESH_MAX_AGE_SECONDS, + ) + ), + tip_poll_deadline_seconds=float( + getattr( + self, + "health_tip_poll_max_age_seconds", + DEFAULT_PRISM_HEALTH_TIP_POLL_MAX_AGE_SECONDS, + ) + ), + bundle_build_deadline_seconds=float( getattr( self, "bundle_build_timeout_seconds", @@ -28277,57 +9636,23 @@ def verify_bundle( ) ), ) - try: - completed = subprocess.run( - prism_tool_command("qbit-prism-audit-verify") - + [ - str(bundle_path), - "--coinbase-tx-hex", - coinbase_tx_hex, - "--ledger-writer-public-key-hex", - ledger_writer_public_key_hex, - "--expected-coinbase-value-sats", - str(expected_coinbase_value_sats), - ], - text=True, - stdout=subprocess.PIPE, - stderr=subprocess.PIPE, - check=False, - timeout=timeout_seconds, - ) - except subprocess.TimeoutExpired as exc: - raise RuntimeError( - "qbit-prism-audit-verify timed out after " - f"{timeout_seconds:g}s" - ) from exc - if completed.returncode != 0: - raise RuntimeError(f"qbit-prism-audit-verify failed: {completed.stderr}") - return json.loads(completed.stdout) - def trusted_ledger_writer_public_key_hex(self, bundle: dict[str, Any]) -> str: - if self.ledger_writer_public_key_hex is not None: - return self.ledger_writer_public_key_hex - return validate_hex( - str(bundle["ledger_window_attestation"]["signature"]["public_key_hex"]), - name="bundle ledger public key", - expected_bytes=32, + def _progress_work_generation( + self, + snapshot: QbitTipTemplateSnapshot, + payout_generation: int, + ) -> WorkGeneration: + return WorkGeneration( + template_generation=int(snapshot.template_generation), + template_fingerprint=snapshot.template_fingerprint, + payout_generation=int(payout_generation), ) - def _progress_now(self) -> float: - clock = getattr(self, "_progress_monotonic", None) - return float(clock() if callable(clock) else time.monotonic()) - def _progress_note_refresh_pending(self, started_monotonic: float | None = None) -> None: self._ensure_job_cache_state() - started = self._progress_now() if started_monotonic is None else started_monotonic - with self._progress_health_lock: - pending_since = self._progress_pending_since_monotonic - if pending_since is None or started < pending_since: - self._progress_pending_since_monotonic = started - divergence_since = self._progress_publication_divergence_since_monotonic - if divergence_since is None or started < divergence_since: - self._progress_publication_divergence_since_monotonic = started - self._progress_refresh_signal_pending = True + self._ensure_progress_health_service().mark_refresh_pending( + started_monotonic + ) def _record_progress_tip_poll( self, @@ -28336,45 +9661,11 @@ def _record_progress_tip_poll( ) -> None: """Publish a coherent qbit tip/template observation to health state.""" self._ensure_job_cache_state() - observed = self._progress_now() if observed_monotonic is None else observed_monotonic payout_generation = int(getattr(self, "_payout_state_generation", 0)) - with self._progress_health_lock: - if snapshot.template_generation < self._progress_current_template_generation: - # A slower concurrent poll can finish after a newer coherent - # observation. It proves only that the obsolete generation was - # coherent, so it must not renew freshness for current work. - return - self._progress_current_template_generation = snapshot.template_generation - self._progress_current_template_fingerprint = snapshot.template_fingerprint - self._progress_current_payout_generation = max( - self._progress_current_payout_generation, - payout_generation, - ) - self._progress_last_tip_poll_monotonic = observed - same_published_work = bool( - self._progress_has_published_work - and self._progress_published_template_fingerprint - == snapshot.template_fingerprint - and self._progress_published_payout_generation - == self._progress_current_payout_generation - ) - if same_published_work: - # Observation generations order RPC races. When the semantic - # template fingerprint and payout generation are unchanged, - # already-issued work remains current and can be reconciled to - # the latest observation without a needless socket delivery. - self._progress_published_template_generation = ( - snapshot.template_generation - ) - else: - pending_since = self._progress_pending_since_monotonic - if pending_since is None or observed < pending_since: - self._progress_pending_since_monotonic = observed - divergence_since = ( - self._progress_publication_divergence_since_monotonic - ) - if divergence_since is None or observed < divergence_since: - self._progress_publication_divergence_since_monotonic = observed + self._ensure_progress_health_service().observe_tip( + self._progress_work_generation(snapshot, payout_generation), + observed_monotonic, + ) def _record_progress_payout_generation( self, @@ -28382,25 +9673,10 @@ def _record_progress_payout_generation( invalidated_monotonic: float | None = None, ) -> None: self._ensure_job_cache_state() - invalidated = ( - self._progress_now() - if invalidated_monotonic is None - else invalidated_monotonic + self._ensure_progress_health_service().observe_payout_generation( + generation, + invalidated_monotonic, ) - with self._progress_health_lock: - if generation < self._progress_current_payout_generation: - return - self._progress_current_payout_generation = generation - if generation != self._progress_published_payout_generation: - pending_since = self._progress_pending_since_monotonic - if pending_since is None or invalidated < pending_since: - self._progress_pending_since_monotonic = invalidated - divergence_since = ( - self._progress_publication_divergence_since_monotonic - ) - if divergence_since is None or invalidated < divergence_since: - self._progress_publication_divergence_since_monotonic = invalidated - self._progress_refresh_signal_pending = True def _record_progress_publication( self, @@ -28409,39 +9685,19 @@ def _record_progress_publication( ) -> None: """Record that current in-memory work is available for delivery.""" self._ensure_job_cache_state() - with self.lock, self._progress_health_lock: + service = self._ensure_progress_health_service() + with self.lock: latest_detected = getattr(self, "latest_detected_tip", None) - publication_matches_latest_tip = bool( - latest_detected is None - or latest_detected[0] == snapshot.bestblockhash - ) - if ( - snapshot.template_generation - < self._progress_published_template_generation - or payout_generation - < self._progress_published_payout_generation - ): - return - self._progress_published_template_generation = max( - self._progress_published_template_generation, - snapshot.template_generation, - ) - self._progress_published_template_fingerprint = snapshot.template_fingerprint - self._progress_published_payout_generation = max( - self._progress_published_payout_generation, - payout_generation, + published = service.publish_work( + self._progress_work_generation(snapshot, payout_generation), + matches_latest_tip=bool( + latest_detected is None + or latest_detected[0] == snapshot.bestblockhash + ), ) - self._progress_has_published_work = True - if ( - publication_matches_latest_tip - and self._progress_current_template_fingerprint - == snapshot.template_fingerprint - and self._progress_current_payout_generation == payout_generation - ): - self._progress_refresh_signal_pending = False - self._progress_publication_divergence_since_monotonic = None - self._progress_note_refresh_activity() - self._progress_reconcile_pending() + if published: + self._progress_note_refresh_activity() + self._progress_reconcile_pending() def _record_progress_delivery( self, @@ -28449,118 +9705,40 @@ def _record_progress_delivery( context: PrismJobContext, delivered_monotonic: float, ) -> None: - """Record a completed current-generation socket delivery.""" - self._ensure_job_cache_state() - fingerprint = getattr(context, "template_fingerprint", None) - if fingerprint is None: - fingerprint = qbit_template_fingerprint(context.template) - payout_generation = int(getattr(context, "payout_state_generation", 0)) - delivered_tip = str(context.template.get("previousblockhash", "")) - recorded = False - coverage_reached: list[tuple[str, float]] = [] - with self.lock, self._progress_health_lock: - client._progress_delivered_context = context - client._progress_delivered_template_fingerprint = fingerprint - client._progress_delivered_template_generation = int( - getattr(context, "template_generation", 0) - ) - client._progress_delivered_payout_generation = payout_generation - client._progress_delivered_monotonic = delivered_monotonic - ready_mode_required = bool( - getattr(self, "_pool_ready_latched", False) - ) - latest_detected = getattr(self, "latest_detected_tip", None) - delivery_matches_latest_tip = bool( - latest_detected is None or latest_detected[0] == delivered_tip - ) - if ( - fingerprint == self._progress_current_template_fingerprint - and payout_generation == self._progress_current_payout_generation - and not ( - ready_mode_required - and bool(getattr(context, "collection_only", False)) - ) - ): - self._progress_last_delivery_template_generation = int( - getattr(context, "template_generation", 0) - ) - self._progress_last_delivery_template_fingerprint = fingerprint - self._progress_last_delivery_payout_generation = payout_generation - self._progress_last_delivery_monotonic = delivered_monotonic - # A successful delivery proves publication for its coherent - # generation. Only the latest observed tip can close the - # outstanding divergence below. - self._progress_published_template_generation = max( - self._progress_published_template_generation, - self._progress_current_template_generation, - ) - self._progress_published_template_fingerprint = fingerprint - self._progress_published_payout_generation = max( - self._progress_published_payout_generation, - payout_generation, - ) - self._progress_has_published_work = True - self._progress_refresh_signal_pending = False - if delivery_matches_latest_tip: - self._progress_publication_divergence_since_monotonic = None - recorded = True - coverage_reached = ( - self._tip_refresh_epoch_coverage_reached_locked( - client, - context, - delivered_monotonic, - ) - ) - if recorded: - self._progress_note_refresh_activity(delivered_monotonic) - self._progress_reconcile_pending(now=delivered_monotonic) - self._record_tip_refresh_epoch_coverage(coverage_reached) + """Record a completed current-generation socket delivery. + + Cross-owner adapter by design: the session owner commits exact S1 + registry proof first, then reports the delivery to G1. + """ + self._ensure_stratum_session_service().record_successful_delivery( + client, context, delivered_monotonic + ) def _progress_refresh_started(self) -> None: """Track a coherent poll pass while it continues making progress.""" self._ensure_job_cache_state() - started = self._progress_now() - with self._progress_health_lock: - self._progress_active_refresh_count += 1 - self._progress_last_refresh_activity_monotonic = started + self._ensure_progress_health_service().refresh_started() def _progress_note_refresh_activity( self, observed_monotonic: float | None = None, ) -> None: self._ensure_job_cache_state() - observed = ( - self._progress_now() - if observed_monotonic is None - else observed_monotonic + self._ensure_progress_health_service().note_refresh_activity( + observed_monotonic ) - with self._progress_health_lock: - if self._progress_active_refresh_count > 0: - last_activity = self._progress_last_refresh_activity_monotonic - if last_activity is None or observed > last_activity: - self._progress_last_refresh_activity_monotonic = observed def _progress_refresh_finished(self) -> None: self._ensure_job_cache_state() - with self._progress_health_lock: - self._progress_active_refresh_count = max( - 0, - self._progress_active_refresh_count - 1, - ) + self._ensure_progress_health_service().refresh_finished() def _progress_bundle_build_started(self) -> int: self._ensure_job_cache_state() - started = self._progress_now() - with self._progress_health_lock: - self._progress_bundle_build_counter += 1 - token = self._progress_bundle_build_counter - self._progress_bundle_builds[token] = started - return token + return self._ensure_progress_health_service().bundle_build_started() def _progress_bundle_build_finished(self, token: int) -> None: self._ensure_job_cache_state() - with self._progress_health_lock: - self._progress_bundle_builds.pop(token, None) + self._ensure_progress_health_service().bundle_build_finished(token) def _progress_eligible_client_counts( self, @@ -28604,632 +9782,137 @@ def _progress_eligible_client_counts( ready_mode_required and bool( getattr(delivered_context, "collection_only", False) - ) - ) - ): - requiring_refresh += 1 - return len(eligible), requiring_refresh - - def _progress_reconcile_pending(self, *, now: float | None = None) -> None: - """Clear pending state exactly when publication/delivery is sufficient.""" - self._ensure_job_cache_state() - current = self._progress_now() if now is None else now - with self._progress_health_lock: - state_key = ( - self._progress_current_template_fingerprint, - self._progress_current_payout_generation, - ) - _, requiring_refresh = self._progress_eligible_client_counts( - *state_key - ) - with self._progress_health_lock: - if state_key != ( - self._progress_current_template_fingerprint, - self._progress_current_payout_generation, - ): - return - published_current = bool( - self._progress_has_published_work - and self._progress_published_template_fingerprint == state_key[0] - and self._progress_published_payout_generation == state_key[1] - ) - refresh_required = bool( - self._progress_refresh_signal_pending - or not published_current - or requiring_refresh > 0 - ) - if refresh_required: - if self._progress_pending_since_monotonic is None: - self._progress_pending_since_monotonic = current - else: - self._progress_pending_since_monotonic = None - - def progress_health_snapshot(self, *, now: float | None = None) -> dict[str, object]: - """Return a bounded, monotonic-only mining progress health snapshot.""" - self._ensure_job_cache_state() - current = self._progress_now() if now is None else now - payout_generation = int(getattr(self, "_payout_state_generation", 0)) - with self._progress_health_lock: - tracked_payout_generation = self._progress_current_payout_generation - if payout_generation > tracked_payout_generation: - self._record_progress_payout_generation(payout_generation, current) - self._progress_reconcile_pending(now=current) - - with self._progress_health_lock: - template_fingerprint = self._progress_current_template_fingerprint - current_template_generation = self._progress_current_template_generation - current_payout_generation = self._progress_current_payout_generation - published_template_generation = self._progress_published_template_generation - published_template_fingerprint = ( - self._progress_published_template_fingerprint - ) - published_payout_generation = self._progress_published_payout_generation - has_published_work = self._progress_has_published_work - last_tip_poll = self._progress_last_tip_poll_monotonic - last_delivery_fingerprint = ( - self._progress_last_delivery_template_fingerprint - ) - last_delivery_payout_generation = ( - self._progress_last_delivery_payout_generation - ) - last_delivery_monotonic = self._progress_last_delivery_monotonic - pending_since = self._progress_pending_since_monotonic - refresh_signal_pending = self._progress_refresh_signal_pending - active_refresh_count = self._progress_active_refresh_count - last_refresh_activity = ( - self._progress_last_refresh_activity_monotonic - ) - bundle_build_starts = tuple(self._progress_bundle_builds.values()) - - eligible_count, requiring_refresh = self._progress_eligible_client_counts( - template_fingerprint, - current_payout_generation, - ) - started = float(getattr(self, "started_monotonic", current)) - tip_poll_reference = started if last_tip_poll is None else last_tip_poll - tip_poll_age = max(0.0, current - tip_poll_reference) - refresh_activity_age = ( - None - if active_refresh_count <= 0 or last_refresh_activity is None - else max(0.0, current - last_refresh_activity) - ) - pending_age = ( - None if pending_since is None else max(0.0, current - pending_since) - ) - oldest_bundle_age = ( - 0.0 - if not bundle_build_starts - else max(0.0, current - min(bundle_build_starts)) - ) - published_current = bool( - has_published_work - and published_template_fingerprint == template_fingerprint - and published_payout_generation == current_payout_generation - ) - delivered_current = bool( - template_fingerprint is not None - and last_delivery_fingerprint == template_fingerprint - and last_delivery_payout_generation == current_payout_generation - ) - delivery_age = ( - max(0.0, current - last_delivery_monotonic) - if delivered_current and last_delivery_monotonic is not None - else None - ) - pending_deadline = float( - getattr( - self, - "health_pending_refresh_max_age_seconds", - DEFAULT_PRISM_HEALTH_PENDING_REFRESH_MAX_AGE_SECONDS, - ) - ) - tip_poll_deadline = float( - getattr( - self, - "health_tip_poll_max_age_seconds", - DEFAULT_PRISM_HEALTH_TIP_POLL_MAX_AGE_SECONDS, - ) - ) - bundle_build_deadline = float( - getattr( - self, - "bundle_build_timeout_seconds", - DEFAULT_PRISM_BUNDLE_BUILD_TIMEOUT_SECONDS, - ) - ) - reasons: list[str] = [] - refresh_is_progressing = bool( - active_refresh_count > 0 - and refresh_activity_age is not None - and refresh_activity_age <= tip_poll_deadline - ) - if tip_poll_age > tip_poll_deadline and not refresh_is_progressing: - reasons.append("tip_poll_stale") - if oldest_bundle_age > bundle_build_deadline: - reasons.append("bundle_build_stuck") - if not has_published_work: - reasons.append("current_generation_not_published") - elif pending_age is not None and pending_age > pending_deadline: - reasons.append("refresh_pending_too_long") - if refresh_signal_pending or not published_current: - reasons.append("current_generation_not_published") - elif requiring_refresh > 0: - reasons.append("current_generation_not_delivered") - reasons = list(dict.fromkeys(reasons)) - - def rounded(value: float | None) -> float | None: - return None if value is None else round(value, 3) - - return { - "ok": not reasons, - "reason": reasons[0] if reasons else None, - "reasons": reasons, - "pending_refresh": pending_since is not None, - "pending_refresh_age_seconds": rounded(pending_age), - "tip_poll_age_seconds": rounded(tip_poll_age), - "tip_refresh_in_progress": active_refresh_count > 0, - "tip_refresh_progress_age_seconds": rounded(refresh_activity_age), - "current_template_generation": current_template_generation, - "published_template_generation": published_template_generation, - "current_payout_generation": current_payout_generation, - "published_payout_generation": published_payout_generation, - "last_valid_delivery_age_seconds": rounded(delivery_age), - "eligible_client_count": eligible_count, - "eligible_clients_requiring_refresh": requiring_refresh, - "bundle_build_oldest_age_seconds": rounded(oldest_bundle_age), - } - - @staticmethod - def _apply_progress_health( - payload: dict[str, object], - progress: dict[str, object], - ) -> dict[str, object]: - # A cached snapshot's top-level ``ok`` already includes the progress - # state from snapshot time. Recompute that half from the stable mining - # readiness field so a fresh in-memory progress overlay can both fail - # and recover immediately without hiding an independent mining fault. - base_ok = bool(payload.get("mining_ready", payload.get("ok"))) - result = dict(payload) - result.update(progress) - result["ok"] = base_ok and bool(progress["ok"]) - if progress["ok"]: - result.pop("reason", None) - result.pop("reasons", None) - return result - - def progress_health_metrics_lines(self) -> list[str]: - progress = self.progress_health_snapshot() - pending_age = progress["pending_refresh_age_seconds"] - delivery_age = progress["last_valid_delivery_age_seconds"] - coordination_blocked_age = self.coordination_blocked_streak_age_seconds() - active_reasons = set(progress["reasons"]) - return [ - "# HELP qbit_prism_refresh_pending Whether current template or payout work still requires publication or delivery.", - "# TYPE qbit_prism_refresh_pending gauge", - f"qbit_prism_refresh_pending {1 if progress['pending_refresh'] else 0}", - "# HELP qbit_prism_refresh_pending_age_seconds Monotonic age of the oldest unresolved current-work refresh.", - "# TYPE qbit_prism_refresh_pending_age_seconds gauge", - f"qbit_prism_refresh_pending_age_seconds {float(pending_age or 0.0):.6f}", - "# HELP qbit_prism_tip_poll_age_seconds Monotonic age of the last coherent qbit tip/template poll.", - "# TYPE qbit_prism_tip_poll_age_seconds gauge", - f"qbit_prism_tip_poll_age_seconds {float(progress['tip_poll_age_seconds']):.6f}", - "# HELP qbit_prism_current_generation_delivery_age_seconds Monotonic age of the last valid current-generation delivery, or -1 when none exists.", - "# TYPE qbit_prism_current_generation_delivery_age_seconds gauge", - f"qbit_prism_current_generation_delivery_age_seconds {float(delivery_age) if delivery_age is not None else -1.0:.6f}", - "# HELP qbit_prism_bundle_build_oldest_age_seconds Monotonic age of the oldest active bundle build.", - "# TYPE qbit_prism_bundle_build_oldest_age_seconds gauge", - f"qbit_prism_bundle_build_oldest_age_seconds {float(progress['bundle_build_oldest_age_seconds']):.6f}", - "# HELP qbit_prism_template_refresh_coordination_blocked_age_seconds Monotonic age of the continuous coordination-blocked template-refresh streak, or 0 when clear.", - "# TYPE qbit_prism_template_refresh_coordination_blocked_age_seconds gauge", - f"qbit_prism_template_refresh_coordination_blocked_age_seconds {coordination_blocked_age:.6f}", - "# HELP qbit_prism_health_state Current progress-health state by bounded reason.", - "# TYPE qbit_prism_health_state gauge", - f'qbit_prism_health_state{{reason="healthy"}} {1 if progress["ok"] else 0}', - *[ - f'qbit_prism_health_state{{reason="{reason}"}} {1 if reason in active_reasons else 0}' - for reason in PRISM_PROGRESS_HEALTH_REASONS - ], - ] - - def ready_miner_count(self) -> int: - return self.accepted_share_stats()[1] - - def mining_delivery_snapshot(self, *, now: float | None = None) -> dict[str, object]: - now = time.monotonic() if now is None else now - with self.lock: - self._ensure_initial_job_state() - active = len(self.clients) - current_tip = self._current_published_tip_hash_locked() - published_snapshot = getattr(self, "tip_template_snapshot", None) - subscribed = sum(1 for client in self.clients if client.subscribed) - authorized_clients = [ - client - for client in self.clients - if client.subscribed and client.authorized and client.worker is not None - ] - authorized = len(authorized_clients) - clients_with_current_work = [ - client - for client in authorized_clients - if self._client_has_current_tip_job_locked(client) - ] - current = len(clients_with_current_work) - # Semantic currency alongside the strict gauge above: fingerprint - # and payout generation only, no template-generation or object - # identity terms. The strict predicate reads 0 whenever a - # generation was minted since the last fanout (it also gates - # mining health, so its fail-closed shape stays untouched); - # this gauge answers the operational question the 2026-07-31 - # review had to reconstruct from share-acceptance rates -- - # whether miners actually hold work for the current template - # content. - payout_generation_now = int( - getattr(self, "_payout_state_generation", 0) - ) - semantic_current = sum( - 1 - for client in authorized_clients - if client.active_job is not None - and published_snapshot is not None - and getattr(client.active_job, "template_fingerprint", None) - == published_snapshot.template_fingerprint - and int( - getattr(client.active_job, "payout_state_generation", -1) - ) - == payout_generation_now - ) - pending_requests = list(self.pending_initial_jobs.values()) - pending = len(pending_requests) - oldest_age = max( - ( - max(0.0, now - request.requested_monotonic) - for request in pending_requests - ), - default=0.0, - ) - genuinely_pending_initial_clients = [ - client - for client in authorized_clients - if not self._client_has_delivered_work_locked(client) - ] - genuine_initial_started = [ - started - for client in genuinely_pending_initial_clients - for started in ( - client.authorized_monotonic, - ( - self.pending_initial_jobs[client].requested_monotonic - if client in self.pending_initial_jobs - else None - ), - ) - if started is not None - ] - oldest_genuine_initial_age = max( - (max(0.0, now - started) for started in genuine_initial_started), - default=0.0, - ) - connection_limit = int( - getattr( - self, - "stratum_max_connections", - DEFAULT_PRISM_STRATUM_MAX_CONNECTIONS, - ) - ) - pending_limit = int(self.stratum_max_pending_initial_jobs) - coverage = current / authorized if authorized else 1.0 - semantic_coverage = ( - semantic_current / authorized if authorized else 1.0 - ) - cap_saturated = connection_limit > 0 and active >= connection_limit - pending_saturated = pending >= pending_limit - # A reconnect incident is operationally significant well before - # nearly every miner is missing work. Treat any sustained loss of - # at least five percent of current-job coverage as degraded. - poor_coverage = authorized > 0 and coverage < 0.95 - if poor_coverage: - if self._mining_delivery_failure_started_monotonic is None: - self._mining_delivery_failure_started_monotonic = now - else: - self._mining_delivery_failure_started_monotonic = None - delivery_failure_age = ( - max(0.0, now - self._mining_delivery_failure_started_monotonic) - if self._mining_delivery_failure_started_monotonic is not None - else 0.0 - ) - overload_now = pending_saturated or (cap_saturated and poor_coverage) - if overload_now: - if self._mining_overload_started_monotonic is None: - self._mining_overload_started_monotonic = now - else: - self._mining_overload_started_monotonic = None - overload_age = ( - max(0.0, now - self._mining_overload_started_monotonic) - if self._mining_overload_started_monotonic is not None - else 0.0 - ) - timeout = float(self.stratum_initial_job_timeout_seconds) - timeout_disconnects = self.initial_job_timeout_count - queue_rejections = self.initial_job_queue_rejection_count - cancelled = self.initial_job_cancelled_count - coalesced = self.initial_job_coalesced_count - queue_capacity_reclaimed = ( - self.initial_job_queue_capacity_reclaimed_count - ) - peak = self.peak_active_connection_count - handlers = self.handler_thread_count + ) + ) + ): + requiring_refresh += 1 + return len(eligible), requiring_refresh + def _progress_reconcile_pending(self, *, now: float | None = None) -> None: + """Clear pending state exactly when publication/delivery is sufficient.""" self._ensure_job_cache_state() - with self._job_cache_lock: - prepared_bundle = self._prepared_ready_bundle - prepared_snapshot = self._prepared_ready_snapshot - preparation_pending = bool(self.job_preparation_pending) - payout_generation = int(self._payout_state_generation) - prepared_current = bool( - prepared_bundle is not None - and prepared_snapshot is published_snapshot - and published_snapshot is not None - and current_tip is not None - and published_snapshot.bestblockhash == current_tip - and published_snapshot.template_artifacts is not None - and not prepared_bundle.collection_only - and prepared_bundle.template - is published_snapshot.template_artifacts.template - and prepared_bundle.template_fingerprint - == published_snapshot.template_fingerprint - and prepared_bundle.template_generation - == published_snapshot.template_generation - and prepared_bundle.payout_state_generation == payout_generation + self._ensure_progress_health_service().reconcile_pending( + self._progress_eligible_client_counts, + now=now, ) - deadline = timeout if timeout > 0 else None - startup_age = max(0.0, now - getattr(self, "started_monotonic", now)) - startup_grace = float( - getattr( - self, - "mining_health_startup_grace_seconds", - DEFAULT_PRISM_MINING_HEALTH_STARTUP_GRACE_SECONDS, - ) - ) - in_startup_grace = startup_age < startup_grace - initial_job_starved = bool( - deadline is not None - and oldest_genuine_initial_age >= deadline - and genuinely_pending_initial_clients - ) - current_tip_coverage_stalled = bool( - deadline is not None - and poor_coverage - and delivery_failure_age >= deadline - ) - no_delivery_progress = bool( - initial_job_starved or current_tip_coverage_stalled + def _progress_health_value( + self, + *, + now: float | None = None, + ) -> ProgressHealthSnapshot: + self._ensure_job_cache_state() + started_monotonic = getattr(self, "started_monotonic", None) + return self._ensure_progress_health_service().snapshot( + self._progress_eligible_client_counts, + int(getattr(self, "_payout_state_generation", 0)), + self._progress_health_config(), + None if started_monotonic is None else float(started_monotonic), + now=now, ) - stale_unknown = int( - getattr(self, "rejection_counts_by_reason", {}).get( - PRISM_REJECTION_STALE_JOB, - 0, - ) - ) + int( - getattr(self, "rejection_counts_by_reason", {}).get( - PRISM_REJECTION_UNKNOWN_JOB, - 0, + + def progress_health_snapshot(self, *, now: float | None = None) -> dict[str, object]: + """Return a bounded, monotonic-only mining progress health snapshot.""" + return self._progress_health_value(now=now).as_mapping() + + @staticmethod + def _apply_progress_health( + payload: dict[str, object], + progress: dict[str, object], + ) -> dict[str, object]: + return ObservabilityService.apply_progress_health(payload, progress) + + def progress_health_metrics_lines(self) -> list[str]: + return list( + ProgressHealthService.metrics_lines( + self._progress_health_value(), + coordination_blocked_age_seconds=( + self.coordination_blocked_streak_age_seconds() + ), ) ) - submitted = int(getattr(self, "submitted_share_count", 0)) - reject_storm = ( - poor_coverage - and submitted > 0 - and stale_unknown / submitted >= 0.95 - ) - persistent_overload = deadline is not None and overload_age >= deadline - unhealthy_reasons: list[str] = [] - if not in_startup_grace: - if no_delivery_progress: - unhealthy_reasons.append("initial-delivery-stalled") - if pending_saturated and persistent_overload: - unhealthy_reasons.append("pending-initial-jobs-saturated") - if cap_saturated and poor_coverage and persistent_overload: - unhealthy_reasons.append("connection-capacity-saturated") - if reject_storm: - unhealthy_reasons.append("stale-unknown-rejection-storm") - mining_ready = not unhealthy_reasons - executor = getattr(self, "_tip_refresh_executor", None) - queue_depth, active_workers = executor.stats() if executor is not None else (0, 0) - return { - "mining_ready": mining_ready, - "mining_delivery_healthy": mining_ready, - "mining_health_startup_grace": in_startup_grace, - "active_connections": active, - "connection_capacity": connection_limit, - "peak_active_connections": peak, - "subscribed_connections": subscribed, - "authorized_connections": authorized, - "pending_initial_jobs": pending, - "pending_initial_job_capacity": pending_limit, - "oldest_pending_initial_job_age_seconds": round(oldest_age, 3), - "oldest_genuinely_pending_initial_job_age_seconds": round( - oldest_genuine_initial_age, - 3, - ), - "clients_with_current_tip_jobs": current, - "current_tip_job_coverage": round(coverage, 6), - "clients_with_semantically_current_work": semantic_current, - "semantic_current_work_ratio": round(semantic_coverage, 6), - "current_tip_coverage_gap_age_seconds": round( - delivery_failure_age, - 3, - ), - "connection_capacity_saturated": cap_saturated, - "pending_initial_jobs_saturated": pending_saturated, - "initial_delivery_stalled": no_delivery_progress, - "overload": bool(overload_now or reject_storm), - "overload_age_seconds": round(overload_age, 3), - "unhealthy_reasons": unhealthy_reasons, - "initial_job_queue_rejections": queue_rejections, - "initial_job_timeout_disconnects": timeout_disconnects, - "initial_job_cancelled_tasks": cancelled, - "initial_job_coalesced_tasks": coalesced, - "initial_job_queue_capacity_reclaimed": queue_capacity_reclaimed, - "handler_threads": handlers, - "delivery_executor_queue_depth": queue_depth, - "delivery_executor_active_workers": active_workers, - # Compatibility aliases and preparation visibility introduced by - # the prewarm work. These retain the original bounded-pipeline - # names above for existing dashboards. - "subscribed_clients": subscribed, - "authorized_clients": authorized, - "clients_with_no_active_job": sum( - 1 for client in authorized_clients if client.active_job is None - ), - "clients_without_current_tip_job": authorized - current, - "clients_with_current_tip_job": current, - "clients_pending_initial_job": pending, - "current_tip_job_coverage_ratio": coverage, - # Compatibility alias: this now reports only genuine first-job - # starvation. Current-tip fanout lag has its own age above. - "oldest_initial_job_pending_seconds": round( - oldest_genuine_initial_age, - 3, - ), - "job_preparation_pending": preparation_pending, - "current_observed_tip": current_tip, - "prepared_bundle_current": prepared_current, - "prepared_bundle_tip": ( - prepared_snapshot.bestblockhash - if prepared_snapshot is not None - else None - ), - "prepared_bundle_template_generation": ( - prepared_bundle.template_generation - if prepared_bundle is not None - else None - ), - "prepared_bundle_payout_generation": ( - prepared_bundle.payout_state_generation - if prepared_bundle is not None - else None - ), - } + + def ready_miner_count(self) -> int: + return self.accepted_share_stats()[1] + + def mining_delivery_snapshot( + self, + *, + now: float | None = None, + ) -> dict[str, object]: + return self._ensure_observability_service().mining_delivery_snapshot(now=now) def health_payload(self) -> dict[str, object]: - accepted_share_count, ready_miner_count = self.accepted_share_stats() - mining = self.mining_delivery_snapshot() - payload = { - "ok": bool(mining["mining_ready"]), - "schema": "qbit.prism.audit-health.v1", - "ledger_backend": self.ledger.backend_name, - "accepted_share_count": accepted_share_count, - "ready_miner_count": ready_miner_count, - "accepted_block": self.accepted_block_count > 0, - "accepted_block_count": self.accepted_block_count, - "max_blocks": self.max_blocks, - **mining, - } - return self._apply_progress_health(payload, self.progress_health_snapshot()) + return self._ensure_observability_service().health_payload() def refresh_health_snapshot(self) -> dict[str, object]: - payload = self.health_payload() - self._ensure_job_cache_state() - with self._job_cache_lock: - self._health_snapshot = payload - self._health_snapshot_monotonic = time.monotonic() - self._record_startup_phase_once("health_snapshot_warm") - return payload + return self._ensure_observability_service().refresh_health_snapshot() def cached_health_payload(self) -> tuple[int, dict[str, object]]: - """Health response served from the background snapshot. - - The HTTP handler must never run ledger queries synchronously: under - job-build load those starve behind the GIL and the ledger lock, health - checks time out, and the container is flagged unhealthy exactly when - it is busiest. A snapshot older than the staleness budget flips the - endpoint to 503 so a genuinely wedged ledger still surfaces. - """ - self._ensure_job_cache_state() - refresh_seconds = getattr( - self, "health_refresh_seconds", DEFAULT_PRISM_HEALTH_REFRESH_SECONDS - ) - with self._job_cache_lock: - snapshot = self._health_snapshot - snapshot_monotonic = self._health_snapshot_monotonic - loop_running = self._health_refresh_loop_running - if snapshot is None or snapshot_monotonic is None: - if not loop_running: - # No refresher (tests, or audit HTTP without serve()): compute - # inline like the legacy endpoint did. - payload = self.refresh_health_snapshot() - return (200 if payload.get("ok") else 503), payload - payload = self._apply_progress_health( - { - "ok": False, - "schema": "qbit.prism.audit-health.v1", - "state": "starting", - "error": "health snapshot warm-up has not completed yet", - }, - self.progress_health_snapshot(), - ) - payload["ok"] = False - return 503, payload - age_seconds = time.monotonic() - snapshot_monotonic - stale_after = max(3 * refresh_seconds, 15.0) - if age_seconds > stale_after: - payload = self._apply_progress_health( - { - "ok": False, - "schema": "qbit.prism.audit-health.v1", - "error": "health snapshot is stale", - "snapshot_age_seconds": round(age_seconds, 3), - }, - self.progress_health_snapshot(), - ) - payload["ok"] = False - return 503, payload - # Ledger-backed fields stay cached, but progress state is an in-memory - # monotonic snapshot and must be overlaid on every request. Otherwise a - # cached ok=true response can mask a known failed refresh for another - # full cache cycle (the production incident this endpoint must expose). - payload = self._apply_progress_health( - snapshot, - self.progress_health_snapshot(), - ) - payload["snapshot_age_seconds"] = round(age_seconds, 3) - return (200 if payload.get("ok") else 503), payload + return self._ensure_observability_service().cached_health_payload() def health_snapshot_loop(self) -> None: - while not self.stop_event.is_set(): - try: - self.refresh_health_snapshot() - except Exception: - with self._job_cache_lock: - self.health_snapshot_refresh_failure_count += 1 - print("prism coordinator: health snapshot refresh failed", flush=True) - traceback.print_exc() - if self.stop_event.wait( - getattr(self, "health_refresh_seconds", DEFAULT_PRISM_HEALTH_REFRESH_SECONDS) - ): - break + self._ensure_observability_service().health_snapshot_loop() def start_health_snapshot_refresher(self) -> None: - self._ensure_job_cache_state() - with self._job_cache_lock: - if self._health_refresh_loop_running: - return - self._health_refresh_loop_running = True + service = self._ensure_observability_service() + if not service.begin_refresh_loop(): + return # The first refresh seeds the exact accepted-share aggregate, which # can take minutes on a grown ledger. It runs inside the background # loop so the audit listener bind path never blocks on it; until it # completes, cached_health_payload reports an explicit starting - # state (issue #188 fix 4). - threading.Thread(target=self.health_snapshot_loop, daemon=True).start() + # state (issue #188 fix 4). The running flag above is set before the + # registry can start the thread, preserving the #120 + # mark-before-listener-dispatch contract for start_audit_server. + self._start_background_service("health_snapshot_refresher") + + def refresh_metrics_snapshot(self) -> str: + return self._ensure_observability_service().refresh_metrics_snapshot() + + def cached_metrics_payload(self) -> MetricsSnapshotResponse: + return self._ensure_observability_service().cached_metrics_payload() + + def metrics_snapshot_loop(self) -> None: + self._ensure_observability_service().metrics_snapshot_loop() + + def start_metrics_snapshot_refresher(self) -> None: + service = self._ensure_observability_service() + if not service.begin_metrics_refresh_loop(): + return + # Like the health refresher above, the collector is marked running + # before its thread can be dispatched and never collects on the + # scrape path: /metrics serves only the cached complete snapshot. + self._start_background_service("metrics_snapshot_refresher") + + @property + def latest_evidence(self) -> dict[str, Any] | None: + if ( + "_audit_artifact_store" not in self.__dict__ + and "audit_dir" not in self.__dict__ + and "evidence_path" not in self.__dict__ + ): + value = self.__dict__.get("_audit_latest_evidence_seed") + return copy.deepcopy(value) if isinstance(value, dict) else None + return self._ensure_audit_artifact_store().latest_evidence() + + @latest_evidence.setter + def latest_evidence(self, payload: Mapping[str, Any] | None) -> None: + if ( + "_audit_artifact_store" not in self.__dict__ + and "audit_dir" not in self.__dict__ + and "evidence_path" not in self.__dict__ + ): + self.__dict__["_audit_latest_evidence_seed"] = ( + copy.deepcopy(dict(payload)) if payload is not None else None + ) + return + self._ensure_audit_artifact_store().set_latest_evidence_for_compatibility( + payload + ) def latest_evidence_payload(self) -> dict[str, object] | None: - with self.lock: - if self.latest_evidence is not None: - return dict(self.latest_evidence) - if self.evidence_path.exists(): - return json.loads(self.evidence_path.read_text(encoding="utf-8")) - return None + return self._ensure_audit_artifact_store().latest_evidence() def owed_balances_payload(self) -> dict[str, object]: return { @@ -29247,1752 +9930,223 @@ def miner_status_payload(self, recipient_id: str) -> dict[str, object]: recipient_id = recipient_id.strip() if not recipient_id: raise ValueError("recipient_id is required") - balances = [ - balance - for balance in self.ledger.current_owed_balances() - if str(balance.get("recipient_id", "")) == recipient_id - ] - owed_balance_sats = sum(int(balance.get("balance_sats", 0)) for balance in balances) - return { - "schema": "qbit.prism.miner-status.v1", - "ledger_backend": self.ledger.backend_name, - "recipient_id": recipient_id, - "owed_balance_sats": owed_balance_sats, - "owed_balances": balances, - "recent_payouts": self.ledger.recipient_payout_history(recipient_id=recipient_id), - } - - @staticmethod - def process_resource_metrics() -> tuple[int, int]: - """Return cheap Linux RSS/descriptor gauges without extra processes.""" - rss_bytes = -1 - open_descriptors = -1 - try: - statm = Path("/proc/self/statm").read_text(encoding="ascii").split() - rss_bytes = int(statm[1]) * int(os.sysconf("SC_PAGE_SIZE")) - except (OSError, ValueError, IndexError): - pass - try: - open_descriptors = len(tuple(Path("/proc/self/fd").iterdir())) - except OSError: - pass - return rss_bytes, open_descriptors - - def coordinator_lock_metrics_lines(self) -> list[str]: - snapshot = getattr(self.lock, "contention_snapshot", None) - if callable(snapshot): - contention_count, wait_sum, wait_max = snapshot() - else: - contention_count, wait_sum, wait_max = 0, 0.0, 0.0 - return [ - "# HELP qbit_prism_coordinator_lock_contentions_total Coordinator control-plane lock acquisitions that had to wait.", - "# TYPE qbit_prism_coordinator_lock_contentions_total counter", - f"qbit_prism_coordinator_lock_contentions_total {int(contention_count)}", - "# HELP qbit_prism_coordinator_lock_wait_seconds Coordinator control-plane lock wait duration for contended acquisitions.", - "# TYPE qbit_prism_coordinator_lock_wait_seconds summary", - f"qbit_prism_coordinator_lock_wait_seconds_sum {float(wait_sum):.6f}", - f"qbit_prism_coordinator_lock_wait_seconds_count {int(contention_count)}", - "# HELP qbit_prism_coordinator_lock_wait_seconds_max Longest observed coordinator control-plane lock wait.", - "# TYPE qbit_prism_coordinator_lock_wait_seconds_max gauge", - f"qbit_prism_coordinator_lock_wait_seconds_max {float(wait_max):.6f}", - ] - - def _observe_block_submit_seconds(self, elapsed_seconds: float) -> None: - self._ensure_job_cache_state() - with self._block_submit_metrics_lock: - histogram = self.block_submit_seconds_histogram - histogram["count"] = int(histogram["count"]) + 1 - histogram["sum"] = float(histogram["sum"]) + max( - 0.0, elapsed_seconds - ) - buckets = histogram["buckets"] - assert isinstance(buckets, dict) - for bucket in PRISM_JOB_BUILD_SECONDS_BUCKETS: - if elapsed_seconds <= bucket: - buckets[bucket] = int(buckets.get(bucket, 0)) + 1 - - def block_submitter_metrics_lines(self) -> list[str]: - pending_metrics = { - "pending_count": -1, - "oldest_pending_age_seconds": -1.0, - "oldest_unattempted_age_seconds": -1.0, - } - pending_snapshot = getattr(self.ledger, "block_candidate_pending_metrics", None) - if callable(pending_snapshot): - try: - pending_metrics.update(pending_snapshot()) - except Exception: - # Metrics collection is diagnostic. Candidate processing and - # its watchdog remain authoritative when this read is down. - pass - self._ensure_block_submitter_retry_state() - self._ensure_job_cache_state() - now = time.monotonic() - with self._block_submitter_retry_state_lock: - deadline = self._block_submitter_backoff_deadline_monotonic - backoff_active = deadline is not None - backoff_remaining = ( - max(0.0, float(deadline) - now) if deadline is not None else 0.0 - ) - backoff_delay = float(self._block_submitter_backoff_delay_seconds) - with self._block_submit_metrics_lock: - submit_buckets = dict(self.block_submit_seconds_histogram["buckets"]) - submit_sum = float(self.block_submit_seconds_histogram["sum"]) - submit_count = int(self.block_submit_seconds_histogram["count"]) - return [ - "# HELP qbit_prism_block_submit_seconds Seconds from a block candidate landing in this process to its submitblock RPC returning.", - "# TYPE qbit_prism_block_submit_seconds histogram", - *[ - f'qbit_prism_block_submit_seconds_bucket{{le="{bucket:g}"}} {int(submit_buckets.get(bucket, 0))}' - for bucket in PRISM_JOB_BUILD_SECONDS_BUCKETS - ], - f'qbit_prism_block_submit_seconds_bucket{{le="+Inf"}} {submit_count}', - f"qbit_prism_block_submit_seconds_sum {submit_sum:.6f}", - f"qbit_prism_block_submit_seconds_count {submit_count}", - "# HELP qbit_prism_block_candidates_pending Durable block candidates awaiting a terminal outcome, or -1 if unavailable.", - "# TYPE qbit_prism_block_candidates_pending gauge", - f"qbit_prism_block_candidates_pending {int(pending_metrics['pending_count'])}", - "# HELP qbit_prism_block_candidate_oldest_pending_seconds Age of the oldest durable pending block candidate, or -1 if unavailable.", - "# TYPE qbit_prism_block_candidate_oldest_pending_seconds gauge", - f"qbit_prism_block_candidate_oldest_pending_seconds {float(pending_metrics['oldest_pending_age_seconds']):.6f}", - "# HELP qbit_prism_block_candidate_oldest_unattempted_seconds Age of the oldest durable candidate that has never entered processing, or -1 if unavailable.", - "# TYPE qbit_prism_block_candidate_oldest_unattempted_seconds gauge", - f"qbit_prism_block_candidate_oldest_unattempted_seconds {float(pending_metrics['oldest_unattempted_age_seconds']):.6f}", - "# HELP qbit_prism_block_submitter_retry_backoff_active Whether the submitter is in an intentional interruptible retry wait.", - "# TYPE qbit_prism_block_submitter_retry_backoff_active gauge", - f"qbit_prism_block_submitter_retry_backoff_active {1 if backoff_active else 0}", - "# HELP qbit_prism_block_submitter_retry_backoff_remaining_seconds Remaining intentional submitter retry wait.", - "# TYPE qbit_prism_block_submitter_retry_backoff_remaining_seconds gauge", - f"qbit_prism_block_submitter_retry_backoff_remaining_seconds {backoff_remaining:.6f}", - "# HELP qbit_prism_block_submitter_retry_backoff_seconds Current intentional submitter retry delay.", - "# TYPE qbit_prism_block_submitter_retry_backoff_seconds gauge", - f"qbit_prism_block_submitter_retry_backoff_seconds {backoff_delay:.6f}", - ] - - def landing_observability_metrics_lines(self) -> list[str]: - """Landing-path metrics for issue #188's pre-deadline alerting. - - The prior-balances read crossed the one-second submitter deadline - silently over several weeks; these series make that growth, landing - timeouts, and unresolved accepted-parent age visible before they - become an outage. - """ - lines: list[str] = [ - "# HELP qbit_prism_block_ledger_calls_total Submitter ledger calls by deadline class.", - "# TYPE qbit_prism_block_ledger_calls_total counter", - "# HELP qbit_prism_block_ledger_call_timeouts_total Submitter ledger deadline expiries by deadline class.", - "# TYPE qbit_prism_block_ledger_call_timeouts_total counter", - "# HELP qbit_prism_block_ledger_call_budget_seconds Most recent statement budget applied per deadline class (escalates for retried landings).", - "# TYPE qbit_prism_block_ledger_call_budget_seconds gauge", - "# HELP qbit_prism_block_ledger_call_last_duration_seconds Duration of the most recent call per deadline class.", - "# TYPE qbit_prism_block_ledger_call_last_duration_seconds gauge", - "# HELP qbit_prism_block_ledger_call_max_duration_seconds Longest observed call per deadline class since process start.", - "# TYPE qbit_prism_block_ledger_call_max_duration_seconds gauge", - ] - for call_class, stats in sorted( - self.block_ledger_call_class_metrics().items() - ): - label = self.prometheus_label_value(call_class) - lines.extend( - [ - f'qbit_prism_block_ledger_calls_total{{call_class="{label}"}} {int(stats["calls_total"])}', - f'qbit_prism_block_ledger_call_timeouts_total{{call_class="{label}"}} {int(stats["timeouts_total"])}', - f'qbit_prism_block_ledger_call_budget_seconds{{call_class="{label}"}} {float(stats["last_budget_seconds"]):.6f}', - f'qbit_prism_block_ledger_call_last_duration_seconds{{call_class="{label}"}} {float(stats["last_duration_seconds"]):.6f}', - f'qbit_prism_block_ledger_call_max_duration_seconds{{call_class="{label}"}} {float(stats["max_duration_seconds"]):.6f}', - ] - ) - unresolved_ages = self.accepted_parent_unresolved_ages_seconds() - oldest_unresolved = max(unresolved_ages) if unresolved_ages else -1.0 - with self.lock: - preview_wait_timeouts = int( - getattr(self, "_accepted_parent_preview_wait_timeouts", 0) - ) - lines.extend( - [ - "# HELP qbit_prism_accepted_parent_unresolved_transitions Landed accepted-block transitions whose durable bookkeeping is unresolved.", - "# TYPE qbit_prism_accepted_parent_unresolved_transitions gauge", - f"qbit_prism_accepted_parent_unresolved_transitions {len(unresolved_ages)}", - "# HELP qbit_prism_accepted_parent_unresolved_oldest_seconds Age of the oldest unresolved accepted-parent transition, or -1 when none.", - "# TYPE qbit_prism_accepted_parent_unresolved_oldest_seconds gauge", - f"qbit_prism_accepted_parent_unresolved_oldest_seconds {oldest_unresolved:.6f}", - "# HELP qbit_prism_accepted_parent_preview_wait_timeouts_total Child job builds that timed out waiting for an accepted-parent payout preview.", - "# TYPE qbit_prism_accepted_parent_preview_wait_timeouts_total counter", - f"qbit_prism_accepted_parent_preview_wait_timeouts_total {preview_wait_timeouts}", - ] - ) - prior_stats_fn = getattr(self.ledger, "prior_balances_read_stats", None) - if callable(prior_stats_fn): - prior_stats = prior_stats_fn() - lines.extend( - [ - "# HELP qbit_prism_prior_balances_reads_total Prior-balances reads served by the ledger.", - "# TYPE qbit_prism_prior_balances_reads_total counter", - f"qbit_prism_prior_balances_reads_total {int(prior_stats['reads_total'])}", - "# HELP qbit_prism_prior_balances_read_last_seconds Duration of the most recent prior-balances read.", - "# TYPE qbit_prism_prior_balances_read_last_seconds gauge", - f"qbit_prism_prior_balances_read_last_seconds {float(prior_stats['last_seconds']):.6f}", - "# HELP qbit_prism_prior_balances_read_max_seconds Longest prior-balances read since process start.", - "# TYPE qbit_prism_prior_balances_read_max_seconds gauge", - f"qbit_prism_prior_balances_read_max_seconds {float(prior_stats['max_seconds']):.6f}", - ] - ) - startup_phases = self.startup_phase_seconds() - if startup_phases: - lines.extend( - [ - "# HELP qbit_prism_startup_phase_seconds Seconds from serve() start to each startup phase, recorded once.", - "# TYPE qbit_prism_startup_phase_seconds gauge", - ] - ) - for phase, seconds in sorted(startup_phases.items()): - label = self.prometheus_label_value(phase) - lines.append( - f'qbit_prism_startup_phase_seconds{{phase="{label}"}} {float(seconds):.6f}' - ) - return lines - - def _accepted_stats_reconcile_metric_lines(self) -> list[str]: - """Surface reconcile liveness now that failures no longer raise. - - Serving maintained counters means a failing or wedged reconcile is - invisible to callers, so its age and failure count must be visible - to scrapes for alerting instead. - """ - status_fn = getattr(self.ledger, "accepted_stats_reconcile_status", None) - if not callable(status_fn): - return [] - status = status_fn() - lines = [ - "# HELP qbit_prism_accepted_stats_reconcile_failures_total Failed background accepted-stats reconcile passes.", - "# TYPE qbit_prism_accepted_stats_reconcile_failures_total counter", - f"qbit_prism_accepted_stats_reconcile_failures_total {int(status.get('failures') or 0)}", - ] - age = status.get("age_seconds") - if age is not None: - lines.extend( - [ - "# HELP qbit_prism_accepted_stats_reconcile_age_seconds Seconds since the accepted-share counters were last reconciled against the ledger.", - "# TYPE qbit_prism_accepted_stats_reconcile_age_seconds gauge", - f"qbit_prism_accepted_stats_reconcile_age_seconds {float(age):.6f}", - ] - ) - return lines - - def metrics_payload(self) -> str: - ledger_metrics = self.ledger.metrics() - audit_metrics = self.audit_artifact_metrics() - mining_metrics = self.mining_delivery_snapshot() - process_rss_bytes, process_open_fds = self.process_resource_metrics() - accepted_share_count = self.accepted_share_stats()[0] - elapsed = max(0.001, time.monotonic() - self.started_monotonic) - shares_per_second = accepted_share_count / elapsed - self._ensure_share_hot_path_state() - with self._share_accounting_lock: - submitted_share_count = int(getattr(self, "submitted_share_count", 0)) - stale_share_count = int(getattr(self, "stale_share_count", 0)) - duplicate_share_count = int(getattr(self, "duplicate_share_count", 0)) - low_difficulty_share_count = int( - getattr(self, "low_difficulty_share_count", 0) - ) - collection_block_submission_count = int( - getattr(self, "collection_block_submission_count", 0) - ) - rejection_counts = dict( - getattr(self, "rejection_counts_by_reason", {}) - ) - grace_credited_share_count = int( - getattr(self, "grace_credited_share_count", 0) - ) - block_solves_dropped_counts = dict( - getattr( - self, - "block_solves_dropped_counts", - {"stale_grace": 0}, - ) - ) - stale_percent = 0.0 - if submitted_share_count > 0: - stale_percent = (stale_share_count / submitted_share_count) * 100.0 - idle_retarget_count = int(getattr(self, "idle_retarget_count", 0)) - with self.lock: - self._ensure_connection_capacity_state() - active_connection_count = len(self.clients) - connection_limit_rejection_counts = dict( - self.connection_limit_rejection_counts - ) - accept_resource_exhaustion_count = int( - getattr(self, "accept_resource_exhaustion_count", 0) - ) - connection_setup_failure_count = int( - getattr(self, "connection_setup_failure_count", 0) - ) - self._ensure_evicted_job_state() - self.prune_evicted_job_graveyard(force=False) - same_tip_context_count = len(self.evicted_same_tip_job_ids) - evicted_job_context_counts = { - "same_tip": same_tip_context_count, - "stale_grace": len(self.evicted_job_graveyard) - same_tip_context_count, - } - evicted_job_submit_counts = dict(self.evicted_job_submit_counts) - evicted_job_expiration_counts = dict(self.evicted_job_expiration_counts) - evicted_job_capacity_eviction_counts = dict( - self.evicted_job_capacity_eviction_counts - ) - stale_job_abandon_counts = dict( - getattr( - self, - "stale_job_abandon_counts", - { - abandon_class: 0 - for abandon_class in PRISM_STALE_JOB_ABANDON_CLASSES - }, - ) - ) - self._ensure_worker_metrics_state() - with self.worker_metrics_lock: - worker_share_counts = { - label: dict(counts) - for label, counts in self.worker_share_counts.items() - } - worker_rejection_counts = dict(self.worker_rejection_counts) - coinbase_weight_headroom = 2_000_000 - latest_coinbase_size_bytes = getattr(self, "latest_coinbase_size_bytes", None) - if latest_coinbase_size_bytes is not None: - coinbase_weight_headroom = 2_000_000 - int(latest_coinbase_size_bytes) - ctv_pending = 0 - ctv_broadcastable = 0 - ctv_failed = 0 - pending_ctv_fanouts = getattr(self.ledger, "pending_ctv_fanout_statuses", None) - if callable(pending_ctv_fanouts): - try: - for fanout in pending_ctv_fanouts(limit=1_000): - ctv_pending += 1 - status = str(fanout.get("settlement_status", "")) - if status == "broadcastable": - ctv_broadcastable += 1 - elif status == "failed": - ctv_failed += 1 - except Exception: - ctv_pending = -1 - ctv_broadcastable = -1 - ctv_failed = -1 - if ctv_failed >= 0: - ctv_failed = int(ledger_metrics.get("ctv_fanouts_failed", ctv_failed)) - ibd = 0 - peers = 0 - try: - blockchain_info = self.rpc.call("getblockchaininfo") - if isinstance(blockchain_info, dict) and blockchain_info.get("initialblockdownload"): - ibd = 1 - except Exception: - ibd = -1 - try: - network_info = self.rpc.call("getnetworkinfo") - if isinstance(network_info, dict): - peers = int(network_info.get("connections", 0)) - except Exception: - peers = -1 - lines = [ - "# HELP qbit_prism_accepted_shares_total Accepted shares recorded by the canonical PRISM ledger.", - "# TYPE qbit_prism_accepted_shares_total counter", - f"qbit_prism_accepted_shares_total {accepted_share_count}", - *self._accepted_stats_reconcile_metric_lines(), - *self.landing_observability_metrics_lines(), - "# HELP qbit_prism_submitted_shares_total Stratum share submissions seen by the PRISM coordinator.", - "# TYPE qbit_prism_submitted_shares_total counter", - f"qbit_prism_submitted_shares_total {submitted_share_count}", - "# HELP qbit_prism_stratum_active_connections Active admitted Stratum connections across all listeners.", - "# TYPE qbit_prism_stratum_active_connections gauge", - f"qbit_prism_stratum_active_connections {active_connection_count}", - "# HELP qbit_prism_stratum_connection_limit Configured global Stratum connection ceiling; zero means unlimited.", - "# TYPE qbit_prism_stratum_connection_limit gauge", - f"qbit_prism_stratum_connection_limit {mining_metrics['connection_capacity']}", - "# HELP qbit_prism_stratum_peak_active_connections Peak admitted Stratum connections since process start.", - "# TYPE qbit_prism_stratum_peak_active_connections gauge", - f"qbit_prism_stratum_peak_active_connections {mining_metrics['peak_active_connections']}", - "# HELP qbit_prism_stratum_subscribed_connections Active subscribed Stratum connections.", - "# TYPE qbit_prism_stratum_subscribed_connections gauge", - f"qbit_prism_stratum_subscribed_connections {mining_metrics['subscribed_connections']}", - "# HELP qbit_prism_stratum_authorized_connections Active subscribed and authorized Stratum connections.", - "# TYPE qbit_prism_stratum_authorized_connections gauge", - f"qbit_prism_stratum_authorized_connections {mining_metrics['authorized_connections']}", - "# HELP qbit_prism_stratum_pending_initial_jobs Authorized clients awaiting their first usable current-tip job.", - "# TYPE qbit_prism_stratum_pending_initial_jobs gauge", - f"qbit_prism_stratum_pending_initial_jobs {mining_metrics['pending_initial_jobs']}", - "# HELP qbit_prism_stratum_pending_initial_job_limit Configured bound for clients awaiting their first usable job.", - "# TYPE qbit_prism_stratum_pending_initial_job_limit gauge", - f"qbit_prism_stratum_pending_initial_job_limit {mining_metrics['pending_initial_job_capacity']}", - "# HELP qbit_prism_stratum_oldest_pending_initial_job_seconds Age of the oldest pending first-job request.", - "# TYPE qbit_prism_stratum_oldest_pending_initial_job_seconds gauge", - f"qbit_prism_stratum_oldest_pending_initial_job_seconds {mining_metrics['oldest_pending_initial_job_age_seconds']}", - "# HELP qbit_prism_stratum_oldest_genuinely_pending_initial_job_seconds Age of the oldest authorized client that has never received usable work.", - "# TYPE qbit_prism_stratum_oldest_genuinely_pending_initial_job_seconds gauge", - f"qbit_prism_stratum_oldest_genuinely_pending_initial_job_seconds {mining_metrics['oldest_genuinely_pending_initial_job_age_seconds']}", - "# HELP qbit_prism_stratum_current_tip_coverage_gap_seconds Continuous age of current-tip job coverage below 95 percent.", - "# TYPE qbit_prism_stratum_current_tip_coverage_gap_seconds gauge", - f"qbit_prism_stratum_current_tip_coverage_gap_seconds {mining_metrics['current_tip_coverage_gap_age_seconds']}", - "# HELP qbit_prism_stratum_initial_job_queue_rejections_total Sessions closed because bounded first-job delivery was full.", - "# TYPE qbit_prism_stratum_initial_job_queue_rejections_total counter", - f"qbit_prism_stratum_initial_job_queue_rejections_total {mining_metrics['initial_job_queue_rejections']}", - "# HELP qbit_prism_stratum_initial_job_timeouts_total Sessions disconnected after first-job delivery timed out.", - "# TYPE qbit_prism_stratum_initial_job_timeouts_total counter", - f"qbit_prism_stratum_initial_job_timeouts_total {mining_metrics['initial_job_timeout_disconnects']}", - "# HELP qbit_prism_stratum_initial_job_tasks_total First-job tasks canceled or coalesced before duplicate work.", - "# TYPE qbit_prism_stratum_initial_job_tasks_total counter", - f'qbit_prism_stratum_initial_job_tasks_total{{result="cancelled"}} {mining_metrics["initial_job_cancelled_tasks"]}', - f'qbit_prism_stratum_initial_job_tasks_total{{result="coalesced"}} {mining_metrics["initial_job_coalesced_tasks"]}', - "# HELP qbit_prism_stratum_initial_job_queue_capacity_reclaimed_total Queued first-job admission slots reclaimed immediately by cancellation.", - "# TYPE qbit_prism_stratum_initial_job_queue_capacity_reclaimed_total counter", - f'qbit_prism_stratum_initial_job_queue_capacity_reclaimed_total {mining_metrics["initial_job_queue_capacity_reclaimed"]}', - "# HELP qbit_prism_stratum_clients_with_current_tip_jobs Authorized clients holding usable current-tip work.", - "# TYPE qbit_prism_stratum_clients_with_current_tip_jobs gauge", - f"qbit_prism_stratum_clients_with_current_tip_jobs {mining_metrics['clients_with_current_tip_jobs']}", - "# HELP qbit_prism_stratum_current_tip_job_coverage Ratio of authorized clients holding current-tip work.", - "# TYPE qbit_prism_stratum_current_tip_job_coverage gauge", - f"qbit_prism_stratum_current_tip_job_coverage {mining_metrics['current_tip_job_coverage']}", - "# HELP qbit_prism_stratum_semantic_current_work_ratio Ratio of authorized clients whose work matches the current template fingerprint and payout generation.", - "# TYPE qbit_prism_stratum_semantic_current_work_ratio gauge", - f"qbit_prism_stratum_semantic_current_work_ratio {mining_metrics['semantic_current_work_ratio']}", - "# HELP qbit_prism_stratum_handler_threads Active per-connection Stratum handler threads.", - "# TYPE qbit_prism_stratum_handler_threads gauge", - f"qbit_prism_stratum_handler_threads {mining_metrics['handler_threads']}", - "# HELP qbit_prism_job_delivery_queue_depth Current bounded delivery executor queue depth.", - "# TYPE qbit_prism_job_delivery_queue_depth gauge", - f"qbit_prism_job_delivery_queue_depth {mining_metrics['delivery_executor_queue_depth']}", - "# HELP qbit_prism_job_delivery_active_workers Delivery executor workers currently running tasks.", - "# TYPE qbit_prism_job_delivery_active_workers gauge", - f"qbit_prism_job_delivery_active_workers {mining_metrics['delivery_executor_active_workers']}", - "# HELP qbit_prism_process_resident_memory_bytes Current process RSS bytes, or -1 when unavailable.", - "# TYPE qbit_prism_process_resident_memory_bytes gauge", - f"qbit_prism_process_resident_memory_bytes {process_rss_bytes}", - "# HELP qbit_prism_process_open_file_descriptors Current process open descriptor count, or -1 when unavailable.", - "# TYPE qbit_prism_process_open_file_descriptors gauge", - f"qbit_prism_process_open_file_descriptors {process_open_fds}", - "# HELP qbit_prism_stratum_connection_limit_rejections_total Stratum connections rejected by an explicitly configured admission limit.", - "# TYPE qbit_prism_stratum_connection_limit_rejections_total counter", - *[ - f'qbit_prism_stratum_connection_limit_rejections_total{{scope="{scope}"}} {int(connection_limit_rejection_counts.get(scope, 0))}' - for scope in ("global", "username") - ], - "# HELP qbit_prism_stratum_accept_resource_exhaustions_total Recoverable Stratum accept or client-setup failures caused by process or system descriptor exhaustion.", - "# TYPE qbit_prism_stratum_accept_resource_exhaustions_total counter", - f"qbit_prism_stratum_accept_resource_exhaustions_total {accept_resource_exhaustion_count}", - "# HELP qbit_prism_stratum_connection_setup_failures_total Admitted Stratum connections cleaned up after socket or handler-thread setup failure.", - "# TYPE qbit_prism_stratum_connection_setup_failures_total counter", - f"qbit_prism_stratum_connection_setup_failures_total {connection_setup_failure_count}", - "# HELP qbit_prism_stale_shares_total Stratum shares rejected or ignored as stale.", - "# TYPE qbit_prism_stale_shares_total counter", - f"qbit_prism_stale_shares_total {stale_share_count}", - "# HELP qbit_prism_duplicate_shares_total Duplicate Stratum shares rejected.", - "# TYPE qbit_prism_duplicate_shares_total counter", - f"qbit_prism_duplicate_shares_total {duplicate_share_count}", - "# HELP qbit_prism_low_difficulty_shares_total Low-difficulty Stratum shares rejected.", - "# TYPE qbit_prism_low_difficulty_shares_total counter", - f"qbit_prism_low_difficulty_shares_total {low_difficulty_share_count}", - "# HELP qbit_prism_collection_block_submissions_total Solver-pays-all block candidates submitted from collection-mode jobs.", - "# TYPE qbit_prism_collection_block_submissions_total counter", - f"qbit_prism_collection_block_submissions_total {collection_block_submission_count}", - "# HELP qbit_prism_grace_credited_shares_total Accepted shares credited by the stale-grace policy.", - "# TYPE qbit_prism_grace_credited_shares_total counter", - f"qbit_prism_grace_credited_shares_total {grace_credited_share_count}", - "# HELP qbit_prism_block_solves_dropped_total Block-passing submissions intentionally excluded from block submission by a bounded policy reason.", - "# TYPE qbit_prism_block_solves_dropped_total counter", - *[ - f'qbit_prism_block_solves_dropped_total{{reason="{reason}"}} {int(block_solves_dropped_counts.get(reason, 0))}' - for reason in ("stale_grace",) - ], - "# HELP qbit_prism_rejections_total PRISM share or block rejections by canonical reason ID.", - "# TYPE qbit_prism_rejections_total counter", - *[ - f'qbit_prism_rejections_total{{reason_id="{reason}"}} {int(rejection_counts.get(reason, 0))}' - for reason in PRISM_REJECTION_REASON_IDS - ], - "# HELP qbit_prism_worker_submitted_shares_total Stratum share submissions by bounded worker label.", - "# TYPE qbit_prism_worker_submitted_shares_total counter", - *[ - f'qbit_prism_worker_submitted_shares_total{{worker="{self.prometheus_label_value(label)}"}} {int(counts.get("submitted", 0))}' - for label, counts in sorted(worker_share_counts.items()) - ], - "# HELP qbit_prism_worker_accepted_shares_total Accepted shares by bounded worker label.", - "# TYPE qbit_prism_worker_accepted_shares_total counter", - *[ - f'qbit_prism_worker_accepted_shares_total{{worker="{self.prometheus_label_value(label)}"}} {int(counts.get("accepted", 0))}' - for label, counts in sorted(worker_share_counts.items()) - ], - "# HELP qbit_prism_worker_grace_credited_shares_total Stale-grace credited shares by bounded worker label.", - "# TYPE qbit_prism_worker_grace_credited_shares_total counter", - *[ - f'qbit_prism_worker_grace_credited_shares_total{{worker="{self.prometheus_label_value(label)}"}} {int(counts.get("grace", 0))}' - for label, counts in sorted(worker_share_counts.items()) - ], - "# HELP qbit_prism_worker_rejections_total PRISM share or block rejections by bounded worker label and reason ID.", - "# TYPE qbit_prism_worker_rejections_total counter", - *[ - f'qbit_prism_worker_rejections_total{{worker="{self.prometheus_label_value(label)}",reason_id="{reason}"}} {int(count)}' - for (label, reason), count in sorted(worker_rejection_counts.items()) - ], - "# HELP qbit_prism_job_build_failures_total Job builds skipped after a template/coinbase error without dropping the client.", - "# TYPE qbit_prism_job_build_failures_total counter", - f"qbit_prism_job_build_failures_total {self.job_build_failure_count}", - "# HELP qbit_prism_block_candidates_dropped_total Legacy counter; durable candidate outbox rows are never dropped on queue overflow.", - "# TYPE qbit_prism_block_candidates_dropped_total counter", - f"qbit_prism_block_candidates_dropped_total {int(getattr(self, 'block_candidates_dropped', 0))}", - "# HELP qbit_prism_block_candidate_wakeups_coalesced_total Candidate queue wakeups coalesced while the durable outbox retained the work.", - "# TYPE qbit_prism_block_candidate_wakeups_coalesced_total counter", - f"qbit_prism_block_candidate_wakeups_coalesced_total {int(getattr(self, 'block_candidate_wakeups_coalesced', 0))}", - "# HELP qbit_prism_block_candidate_retries_total Transient candidate outcomes retained for durable retry.", - "# TYPE qbit_prism_block_candidate_retries_total counter", - f"qbit_prism_block_candidate_retries_total {int(getattr(self, 'block_candidate_retry_count', 0))}", - "# HELP qbit_prism_block_candidate_accept_pending_defers_total Terminal abandonments refused because the candidate is (or was recently observed as) an active chain block; the candidate retries until its accepted success tail finalizes it as submitted.", - "# TYPE qbit_prism_block_candidate_accept_pending_defers_total counter", - f"qbit_prism_block_candidate_accept_pending_defers_total {int(getattr(self, 'block_candidate_accept_pending_defer_count', 0))}", - "# HELP qbit_prism_block_candidate_poisoned_total Invalid durable candidate intents quarantined from replay.", - "# TYPE qbit_prism_block_candidate_poisoned_total counter", - f"qbit_prism_block_candidate_poisoned_total {int(getattr(self, 'block_candidate_poisoned_count', 0))}", - "# HELP qbit_prism_block_candidates_abandoned_total Block candidates that did not land (lost tip race or failed submit), by reason. Not share rejections: the underlying share was accepted.", - "# TYPE qbit_prism_block_candidates_abandoned_total counter", - *[ - f'qbit_prism_block_candidates_abandoned_total{{reason_id="{reason}"}} {int(count)}' - for reason, count in sorted(getattr(self, "block_candidate_abandoned_counts", {}).items()) - ], - "# HELP qbit_prism_stale_job_abandons_total Terminal stale-job block candidate abandonments by bounded cause.", - "# TYPE qbit_prism_stale_job_abandons_total counter", - *[ - f'qbit_prism_stale_job_abandons_total{{class="{abandon_class}"}} {int(stale_job_abandon_counts.get(abandon_class, 0))}' - for abandon_class in PRISM_STALE_JOB_ABANDON_CLASSES - ], - "# HELP qbit_prism_share_append_queue_depth Accepted shares waiting on the ledger writer thread.", - "# TYPE qbit_prism_share_append_queue_depth gauge", - f"qbit_prism_share_append_queue_depth {self.share_append_queue.qsize() if getattr(self, 'share_append_queue', None) is not None else 0}", - "# HELP qbit_prism_share_append_failures_total Shares in group commits that failed before acknowledgement.", - "# TYPE qbit_prism_share_append_failures_total counter", - f"qbit_prism_share_append_failures_total {int(getattr(self, 'share_append_failure_count', 0))}", - "# HELP qbit_prism_shares_recovered_to_disk_total Legacy pre-commit-ACK shares written to the upgrade recovery file.", - "# TYPE qbit_prism_shares_recovered_to_disk_total counter", - f"qbit_prism_shares_recovered_to_disk_total {int(getattr(self, 'shares_recovered_to_disk', 0))}", - "# HELP qbit_prism_shares_replayed_total Recovery-file shares replayed into the ledger at startup.", - "# TYPE qbit_prism_shares_replayed_total counter", - f"qbit_prism_shares_replayed_total {int(getattr(self, 'shares_replayed', 0))}", - "# HELP qbit_prism_tip_refresh_jobs_total Client jobs refreshed after qbit tip/template changes.", - "# TYPE qbit_prism_tip_refresh_jobs_total counter", - f"qbit_prism_tip_refresh_jobs_total {self.tip_refresh_job_count}", - "# HELP qbit_prism_active_job_contexts Current retained PRISM job contexts.", - "# TYPE qbit_prism_active_job_contexts gauge", - f"qbit_prism_active_job_contexts {len(getattr(self, 'jobs', {}))}", - "# HELP qbit_prism_evicted_job_contexts Evicted job contexts retained by safety class.", - "# TYPE qbit_prism_evicted_job_contexts gauge", - *[ - f'qbit_prism_evicted_job_contexts{{class="{job_class}"}} {evicted_job_context_counts[job_class]}' - for job_class in PRISM_EVICTED_JOB_CLASSES - ], - "# HELP qbit_prism_evicted_job_submits_total Accepted submits validated against an evicted job context.", - "# TYPE qbit_prism_evicted_job_submits_total counter", - *[ - f'qbit_prism_evicted_job_submits_total{{outcome="{outcome}"}} {int(evicted_job_submit_counts.get(outcome, 0))}' - for outcome in PRISM_EVICTED_JOB_SUBMIT_OUTCOMES - ], - "# HELP qbit_prism_evicted_job_expirations_total Retained job contexts removed after their class TTL.", - "# TYPE qbit_prism_evicted_job_expirations_total counter", - *[ - f'qbit_prism_evicted_job_expirations_total{{class="{job_class}"}} {int(evicted_job_expiration_counts.get(job_class, 0))}' - for job_class in PRISM_EVICTED_JOB_CLASSES - ], - "# HELP qbit_prism_evicted_job_capacity_evictions_total Same-tip retained contexts removed by a configured count limit.", - "# TYPE qbit_prism_evicted_job_capacity_evictions_total counter", - *[ - f'qbit_prism_evicted_job_capacity_evictions_total{{scope="{scope}"}} {int(evicted_job_capacity_eviction_counts.get(scope, 0))}' - for scope in PRISM_EVICTED_JOB_CAPACITY_SCOPES - ], - "# HELP qbit_prism_post_accept_refresh_failures_total Immediate clean-job refreshes that failed after direct block acceptance.", - "# TYPE qbit_prism_post_accept_refresh_failures_total counter", - f"qbit_prism_post_accept_refresh_failures_total {self.post_accept_refresh_failure_count}", - "# HELP qbit_prism_reorg_inactive_blocks_total PRISM pool blocks quarantined after leaving the active chain.", - "# TYPE qbit_prism_reorg_inactive_blocks_total counter", - f"qbit_prism_reorg_inactive_blocks_total {self.reorg_inactive_block_count}", - "# HELP qbit_prism_reorg_reactivated_blocks_total Quarantined PRISM pool blocks restored after returning to the active chain.", - "# TYPE qbit_prism_reorg_reactivated_blocks_total counter", - f"qbit_prism_reorg_reactivated_blocks_total {self.reorg_reactivated_block_count}", - "# HELP qbit_prism_reorg_reconcile_skips_total Reorg reconciliation passes skipped because qbitd chain view was not trusted.", - "# TYPE qbit_prism_reorg_reconcile_skips_total counter", - f"qbit_prism_reorg_reconcile_skips_total {self.reorg_reconcile_skip_count}", - "# HELP qbit_prism_reorg_reconcile_errors_total Reorg reconciliation errors that prevented ordered job issuance.", - "# TYPE qbit_prism_reorg_reconcile_errors_total counter", - f"qbit_prism_reorg_reconcile_errors_total {self.reorg_reconcile_error_count}", - "# HELP qbit_prism_reorg_reconcile_lookups_total Reconcile demand by caller path and the source that satisfied it.", - "# TYPE qbit_prism_reorg_reconcile_lookups_total counter", - *[ - f'qbit_prism_reorg_reconcile_lookups_total{{path="{path}",source="{source}"}} ' - f"{int(getattr(self, 'reorg_reconcile_lookup_counts', {}).get((path, source), 0))}" - for path in PRISM_REORG_RECONCILE_LOOKUP_PATHS - for source in PRISM_REORG_RECONCILE_LOOKUP_SOURCES - ], - "# HELP qbit_prism_matured_payouts_total Payout entries marked mature by the coordinator tip reconciliation path.", - "# TYPE qbit_prism_matured_payouts_total counter", - f"qbit_prism_matured_payouts_total {self.matured_payout_count}", - "# HELP qbit_prism_vardiff_idle_retargets_total Vardiff retargets triggered by the idle zero-accepted-share sweep.", - "# TYPE qbit_prism_vardiff_idle_retargets_total counter", - f"qbit_prism_vardiff_idle_retargets_total {idle_retarget_count}", - "# HELP qbit_prism_shares_per_second Accepted shares per second since coordinator start.", - "# TYPE qbit_prism_shares_per_second gauge", - f"qbit_prism_shares_per_second {shares_per_second:.12g}", - "# HELP qbit_prism_stale_share_percent Percent of submitted shares classified stale.", - "# TYPE qbit_prism_stale_share_percent gauge", - f"qbit_prism_stale_share_percent {stale_percent:.12g}", - "# HELP qbit_prism_blocks_accepted_total Blocks accepted through the PRISM coordinator.", - "# TYPE qbit_prism_blocks_accepted_total counter", - f"qbit_prism_blocks_accepted_total {self.accepted_block_count}", - "# HELP qbit_prism_persisted_blocks Persisted PRISM pool block rows.", - "# TYPE qbit_prism_persisted_blocks gauge", - f"qbit_prism_persisted_blocks {ledger_metrics['blocks']}", - "# HELP qbit_prism_inactive_pool_blocks PRISM pool block rows currently quarantined as inactive.", - "# TYPE qbit_prism_inactive_pool_blocks gauge", - f"qbit_prism_inactive_pool_blocks {ledger_metrics.get('inactive_blocks', 0)}", - "# HELP qbit_prism_reversed_pool_blocks PRISM pool block rows terminally reversed.", - "# TYPE qbit_prism_reversed_pool_blocks gauge", - f"qbit_prism_reversed_pool_blocks {ledger_metrics.get('reversed_blocks', 0)}", - "# HELP qbit_prism_rejected_pool_blocks PRISM pool block rows rejected before confirmation.", - "# TYPE qbit_prism_rejected_pool_blocks gauge", - f"qbit_prism_rejected_pool_blocks {ledger_metrics.get('rejected_blocks', 0)}", - "# HELP qbit_prism_owed_accounts Current accounts with positive carried owed balances.", - "# TYPE qbit_prism_owed_accounts gauge", - f"qbit_prism_owed_accounts {ledger_metrics['owed_accounts']}", - "# HELP qbit_prism_coinbase_weight_headroom_bytes Remaining qbit block weight bytes after the latest pool coinbase.", - "# TYPE qbit_prism_coinbase_weight_headroom_bytes gauge", - f"qbit_prism_coinbase_weight_headroom_bytes {coinbase_weight_headroom}", - "# HELP qbit_prism_ctv_fanouts_pending Pending non-terminal CTV fanouts known to the ledger, or -1 if unavailable.", - "# TYPE qbit_prism_ctv_fanouts_pending gauge", - f"qbit_prism_ctv_fanouts_pending {ctv_pending}", - "# HELP qbit_prism_ctv_fanouts_broadcastable CTV fanouts that are mature enough to broadcast, or -1 if unavailable.", - "# TYPE qbit_prism_ctv_fanouts_broadcastable gauge", - f"qbit_prism_ctv_fanouts_broadcastable {ctv_broadcastable}", - "# HELP qbit_prism_ctv_fanouts_failed CTV fanouts with failed or rejected broadcast state, or -1 if unavailable.", - "# TYPE qbit_prism_ctv_fanouts_failed gauge", - f"qbit_prism_ctv_fanouts_failed {ctv_failed}", - "# HELP qbit_prism_vardiff_enabled Whether PRISM Stratum vardiff is enabled.", - "# TYPE qbit_prism_vardiff_enabled gauge", - f"qbit_prism_vardiff_enabled {1 if self.vardiff_config.enabled else 0}", - "# HELP qbit_prism_qbitd_initial_block_download qbitd initialblockdownload status, or -1 if unavailable.", - "# TYPE qbit_prism_qbitd_initial_block_download gauge", - f"qbit_prism_qbitd_initial_block_download {ibd}", - "# HELP qbit_prism_qbitd_peers qbitd peer count, or -1 if unavailable.", - "# TYPE qbit_prism_qbitd_peers gauge", - f"qbit_prism_qbitd_peers {peers}", - "# HELP qbit_prism_audit_artifact_bytes Bytes used by PRISM audit artifacts in PRISM_AUDIT_DIR by artifact kind.", - "# TYPE qbit_prism_audit_artifact_bytes gauge", - *[ - f'qbit_prism_audit_artifact_bytes{{kind="{kind}"}} {audit_metrics[kind]["bytes"]}' - for kind in ("body", "share_segment", "live_bundle", "candidate", "other") - ], - "# HELP qbit_prism_audit_artifact_files PRISM audit artifact file count in PRISM_AUDIT_DIR by artifact kind.", - "# TYPE qbit_prism_audit_artifact_files gauge", - *[ - f'qbit_prism_audit_artifact_files{{kind="{kind}"}} {audit_metrics[kind]["files"]}' - for kind in ("body", "share_segment", "live_bundle", "candidate", "other") - ], - "# HELP qbit_prism_audit_artifact_scan_error Whether the latest PRISM_AUDIT_DIR metric scan failed.", - "# TYPE qbit_prism_audit_artifact_scan_error gauge", - f"qbit_prism_audit_artifact_scan_error {audit_metrics['scan_error']}", - ] - lines.extend(self.shutdown_metrics_lines()) - lines.extend(self.coordinator_lock_metrics_lines()) - lines.extend(self.block_submitter_metrics_lines()) - lines.extend(self.share_ack_metrics_lines()) - lines.extend(self.ctv_fanout_broadcaster_metrics_lines()) - lines.extend(self.vardiff_idle_metrics_lines()) - lines.extend(self.job_build_metrics_lines()) - lines.extend(self.tip_refresh_metrics_lines()) - lines.extend(self.payout_state_metrics_lines()) - lines.extend(self.initial_delivery_metrics_lines()) - lines.extend(self.progress_health_metrics_lines()) - return "\n".join(lines) + "\n" - - def shutdown_metrics_lines(self) -> list[str]: - snapshot = self._ensure_shutdown_controller().snapshot() - quiescence = snapshot["writer_quiescence_outcomes"] - release = snapshot["lease_release_outcomes"] - active = snapshot["active_writers"] - assert isinstance(quiescence, dict) - assert isinstance(release, dict) - assert isinstance(active, dict) - return [ - "# HELP qbit_prism_shutdowns_total Controlled coordinator shutdown sequences started.", - "# TYPE qbit_prism_shutdowns_total counter", - f"qbit_prism_shutdowns_total {int(snapshot['shutdowns_total'])}", - "# HELP qbit_prism_shutdown_writer_operations Active admitted ledger-mutating operations by component.", - "# TYPE qbit_prism_shutdown_writer_operations gauge", - *[ - f'qbit_prism_shutdown_writer_operations{{component="{self.prometheus_label_value(str(component))}"}} {int(count)}' - for component, count in sorted(active.items()) - ], - "# HELP qbit_prism_shutdown_writer_quiescence_total Writer-quiescence outcomes.", - "# TYPE qbit_prism_shutdown_writer_quiescence_total counter", - *[ - f'qbit_prism_shutdown_writer_quiescence_total{{outcome="{outcome}"}} {int(quiescence.get(outcome, 0))}' - for outcome in ("success", "timeout") - ], - "# HELP qbit_prism_shutdown_writer_quiescence_seconds Duration of the latest writer-quiescence barrier.", - "# TYPE qbit_prism_shutdown_writer_quiescence_seconds gauge", - f"qbit_prism_shutdown_writer_quiescence_seconds {float(snapshot['writer_quiescence_seconds']):.6f}", - "# HELP qbit_prism_shutdown_lease_release_attempts_total Writer-lease release attempts.", - "# TYPE qbit_prism_shutdown_lease_release_attempts_total counter", - f"qbit_prism_shutdown_lease_release_attempts_total {int(snapshot['lease_release_attempts_total'])}", - "# HELP qbit_prism_shutdown_lease_release_total Writer-lease release outcomes.", - "# TYPE qbit_prism_shutdown_lease_release_total counter", - *[ - f'qbit_prism_shutdown_lease_release_total{{outcome="{outcome}"}} {int(release.get(outcome, 0))}' - for outcome in ("success", "not_held", "unsupported", "failure") - ], - "# HELP qbit_prism_shutdown_lease_release_seconds Duration of the latest writer-lease release attempt.", - "# TYPE qbit_prism_shutdown_lease_release_seconds gauge", - f"qbit_prism_shutdown_lease_release_seconds {float(snapshot['lease_release_seconds']):.6f}", - "# HELP qbit_prism_shutdown_sigterm_to_lease_release_seconds Time from SIGTERM admission close to safe lease release, or -1 if unobserved.", - "# TYPE qbit_prism_shutdown_sigterm_to_lease_release_seconds gauge", - "qbit_prism_shutdown_sigterm_to_lease_release_seconds " - + ( - f"{float(snapshot['sigterm_to_lease_release_seconds']):.6f}" - if snapshot["sigterm_release_observed"] - else "-1" - ), - "# HELP qbit_prism_shutdown_release_withheld_total Shutdowns that withheld lease release because a writer did not quiesce.", - "# TYPE qbit_prism_shutdown_release_withheld_total counter", - f"qbit_prism_shutdown_release_withheld_total {int(snapshot['release_withheld_total'])}", - "# HELP qbit_prism_shutdown_non_writer_drain_seconds Duration of cleanup after writer lease handling.", - "# TYPE qbit_prism_shutdown_non_writer_drain_seconds gauge", - f"qbit_prism_shutdown_non_writer_drain_seconds {float(snapshot['non_writer_drain_seconds']):.6f}", + balances = [ + balance + for balance in self.ledger.current_owed_balances() + if str(balance.get("recipient_id", "")) == recipient_id ] - - def audit_artifact_metrics(self) -> dict[str, dict[str, int] | int]: - metrics: dict[str, dict[str, int] | int] = { - kind: {"files": 0, "bytes": 0} - for kind in ("body", "share_segment", "live_bundle", "candidate", "other") + owed_balance_sats = sum(int(balance.get("balance_sats", 0)) for balance in balances) + return { + "schema": "qbit.prism.miner-status.v1", + "ledger_backend": self.ledger.backend_name, + "recipient_id": recipient_id, + "owed_balance_sats": owed_balance_sats, + "owed_balances": balances, + "recent_payouts": self.ledger.recipient_payout_history(recipient_id=recipient_id), } - metrics["scan_error"] = 0 - audit_dir = getattr(self, "audit_dir", None) - if audit_dir is None: - metrics["scan_error"] = 1 - return metrics - try: - paths = list(Path(audit_dir).iterdir()) - except OSError: - metrics["scan_error"] = 1 - return metrics - for path in paths: - try: - if not path.is_file(): - continue - size = path.stat().st_size - except OSError: - metrics["scan_error"] = 1 - continue - kind = self.audit_artifact_kind(path.name) - bucket = metrics[kind] - assert isinstance(bucket, dict) - bucket["files"] += 1 - bucket["bytes"] += size - return metrics @staticmethod - def audit_artifact_kind(name: str) -> str: - if name.startswith("prism-audit-bundle-body-") and name.endswith(".json"): - return "body" - if name.startswith("prism-audit-share-segment-") and name.endswith(".json"): - return "share_segment" - if name.startswith("prism-live-audit-bundle-candidate-") or name.startswith( - ".prism-live-audit-bundle-candidate-" - ): - return "candidate" - if name.startswith("prism-live-audit-bundle-") and name.endswith(".json"): - return "live_bundle" - return "other" + def process_resource_metrics() -> tuple[int, int]: + """Return cheap Linux RSS/descriptor gauges without extra processes.""" + rss_bytes = -1 + open_descriptors = -1 + try: + statm = Path("/proc/self/statm").read_text(encoding="ascii").split() + rss_bytes = int(statm[1]) * int(os.sysconf("SC_PAGE_SIZE")) + except (OSError, ValueError, IndexError): + pass + try: + open_descriptors = len(tuple(Path("/proc/self/fd").iterdir())) + except OSError: + pass + return rss_bytes, open_descriptors - def ctv_fanout_broadcaster_metrics_lines(self) -> list[str]: - self._ensure_ctv_broadcaster_metrics_state() - with self._ctv_broadcaster_metrics_lock: - bucket_counts = dict(self.ctv_broadcaster_pass_seconds_bucket_counts) - pass_sum = self.ctv_broadcaster_pass_seconds_sum - pass_count = self.ctv_broadcaster_pass_count - processed_rows_total = self.ctv_broadcaster_processed_rows_total - yielded_total = self.ctv_broadcaster_yielded_total - chunk_seconds_buckets = dict( - self.ctv_broadcaster_chunk_seconds_bucket_counts + def coordinator_lock_contention_snapshot(self) -> tuple[int, int, float, float]: + """Raw (acquisitions, contentions, wait_sum, wait_max) from the lock.""" + snapshot = getattr(self.lock, "contention_snapshot", None) + if callable(snapshot): + acquisition_count, contention_count, wait_sum, wait_max = snapshot() + return ( + int(acquisition_count), + int(contention_count), + float(wait_sum), + float(wait_max), ) - chunk_rows_buckets = dict(self.ctv_broadcaster_chunk_rows_bucket_counts) - chunk_seconds_sum = self.ctv_broadcaster_chunk_seconds_sum - chunk_rows_sum = self.ctv_broadcaster_chunk_rows_sum - chunk_count = self.ctv_broadcaster_chunk_count - metric_name = "qbit_prism_ctv_fanout_broadcaster_pass_seconds" - chunk_seconds_name = "qbit_prism_ctv_fanout_broadcaster_chunk_seconds" - chunk_rows_name = "qbit_prism_ctv_fanout_broadcaster_chunk_rows" - return [ - "# HELP qbit_prism_ctv_fanout_broadcaster_processed_rows_total CTV fanout rows completed by the broadcaster loop.", - "# TYPE qbit_prism_ctv_fanout_broadcaster_processed_rows_total counter", - f"qbit_prism_ctv_fanout_broadcaster_processed_rows_total {processed_rows_total}", - "# HELP qbit_prism_ctv_fanout_broadcaster_pass_seconds CTV fanout broadcaster pass wall time.", - "# TYPE qbit_prism_ctv_fanout_broadcaster_pass_seconds histogram", - *[ - f'{metric_name}_bucket{{le="{bucket:g}"}} {bucket_counts.get(bucket, 0)}' - for bucket in PRISM_CTV_BROADCASTER_SECONDS_BUCKETS - ], - f'{metric_name}_bucket{{le="+Inf"}} {pass_count}', - f"{metric_name}_sum {pass_sum:.6f}", - f"{metric_name}_count {pass_count}", - "# HELP qbit_prism_ctv_fanout_broadcaster_tip_refresh_yields_total CTV broadcaster passes yielding between committed chunks for a pending tip refresh.", - "# TYPE qbit_prism_ctv_fanout_broadcaster_tip_refresh_yields_total counter", - f"qbit_prism_ctv_fanout_broadcaster_tip_refresh_yields_total {yielded_total}", - "# HELP qbit_prism_ctv_fanout_broadcaster_chunk_seconds CTV broadcaster committed chunk wall time.", - "# TYPE qbit_prism_ctv_fanout_broadcaster_chunk_seconds histogram", - *[ - f'{chunk_seconds_name}_bucket{{le="{bucket:g}"}} {chunk_seconds_buckets.get(bucket, 0)}' - for bucket in PRISM_CTV_BROADCASTER_CHUNK_SECONDS_BUCKETS - ], - f'{chunk_seconds_name}_bucket{{le="+Inf"}} {chunk_count}', - f"{chunk_seconds_name}_sum {chunk_seconds_sum:.6f}", - f"{chunk_seconds_name}_count {chunk_count}", - "# HELP qbit_prism_ctv_fanout_broadcaster_chunk_rows Rows processed per committed CTV broadcaster chunk.", - "# TYPE qbit_prism_ctv_fanout_broadcaster_chunk_rows histogram", - *[ - f'{chunk_rows_name}_bucket{{le="{bucket}"}} {chunk_rows_buckets.get(bucket, 0)}' - for bucket in PRISM_CTV_BROADCASTER_CHUNK_ROWS_BUCKETS - ], - f'{chunk_rows_name}_bucket{{le="+Inf"}} {chunk_count}', - f"{chunk_rows_name}_sum {chunk_rows_sum}", - f"{chunk_rows_name}_count {chunk_count}", - ] + return 0, 0, 0.0, 0.0 - def initial_delivery_metrics_lines(self) -> list[str]: - self._ensure_initial_job_state() - mining = self.mining_delivery_snapshot() - with self.lock: - counts = { - "sent": self.initial_job_sent_count, - "cancelled": self.initial_job_cancelled_count, - "coalesced": self.initial_job_coalesced_count, - "failed": self.initial_job_failed_count, - "superseded": self.initial_job_superseded_count, - } - latency_sum = self.initial_job_delivery_latency_seconds_sum - latency_count = self.initial_job_delivery_latency_count - queue_capacity_reclaimed = ( - self.initial_job_queue_capacity_reclaimed_count - ) - executor = getattr(self, "_initial_job_executor", None) - queued, slots = executor.stats() if executor is not None else (0, 0) - configured_workers = int( - getattr( - self, - "initial_job_max_workers", - DEFAULT_PRISM_INITIAL_JOB_MAX_WORKERS, - ) + def coordinator_lock_metrics_lines(self) -> list[str]: + # Compatibility wrapper for the extracted metrics owner. + return self._ensure_metrics_renderer().coordinator_lock_metrics_lines() + + def _observe_block_submit_seconds(self, elapsed_seconds: float) -> None: + self._ensure_block_candidate_service()._observe_block_submit_seconds( + elapsed_seconds ) - with self._bundle_preparation_lock: - build_counts = dict(self.shared_bundle_build_counts) - preparation_sum = self.shared_bundle_preparation_seconds_sum - preparation_count = self.shared_bundle_preparation_count - waiters = self.shared_bundle_preparation_waiters - return [ - "# HELP qbit_prism_stratum_subscribed_clients Subscribed Stratum clients.", - "# TYPE qbit_prism_stratum_subscribed_clients gauge", - f'qbit_prism_stratum_subscribed_clients {mining["subscribed_clients"]}', - "# HELP qbit_prism_stratum_authorized_clients Subscribed and authorized Stratum clients.", - "# TYPE qbit_prism_stratum_authorized_clients gauge", - f'qbit_prism_stratum_authorized_clients {mining["authorized_clients"]}', - "# HELP qbit_prism_clients_without_current_tip_job Authorized clients without usable current-tip work.", - "# TYPE qbit_prism_clients_without_current_tip_job gauge", - f'qbit_prism_clients_without_current_tip_job {mining["clients_without_current_tip_job"]}', - "# HELP qbit_prism_clients_with_no_active_job Authorized clients with no active job at all.", - "# TYPE qbit_prism_clients_with_no_active_job gauge", - f'qbit_prism_clients_with_no_active_job {mining["clients_with_no_active_job"]}', - "# HELP qbit_prism_clients_with_current_tip_job Authorized clients with usable current-tip work.", - "# TYPE qbit_prism_clients_with_current_tip_job gauge", - f'qbit_prism_clients_with_current_tip_job {mining["clients_with_current_tip_job"]}', - "# HELP qbit_prism_current_tip_job_coverage_ratio Fraction of authorized clients with current-tip work.", - "# TYPE qbit_prism_current_tip_job_coverage_ratio gauge", - f'qbit_prism_current_tip_job_coverage_ratio {float(mining["current_tip_job_coverage_ratio"]):.12g}', - "# HELP qbit_prism_initial_job_deliveries_pending Coalesced initial deliveries queued or running.", - "# TYPE qbit_prism_initial_job_deliveries_pending gauge", - f'qbit_prism_initial_job_deliveries_pending {mining["clients_pending_initial_job"]}', - "# HELP qbit_prism_initial_job_delivery_tasks_inflight Bounded shared delivery slots currently occupied.", - "# TYPE qbit_prism_initial_job_delivery_tasks_inflight gauge", - f"qbit_prism_initial_job_delivery_tasks_inflight {slots}", - "# HELP qbit_prism_initial_job_delivery_queue_depth Initial-job tasks waiting for a dedicated worker.", - "# TYPE qbit_prism_initial_job_delivery_queue_depth gauge", - f"qbit_prism_initial_job_delivery_queue_depth {queued}", - "# HELP qbit_prism_initial_job_delivery_active_workers Dedicated initial-job workers currently running tasks.", - "# TYPE qbit_prism_initial_job_delivery_active_workers gauge", - f"qbit_prism_initial_job_delivery_active_workers {slots}", - "# HELP qbit_prism_initial_job_delivery_configured_workers Configured dedicated initial-job worker count.", - "# TYPE qbit_prism_initial_job_delivery_configured_workers gauge", - f"qbit_prism_initial_job_delivery_configured_workers {configured_workers}", - "# HELP qbit_prism_initial_job_delivery_seconds Authorization-to-current-job latency.", - "# TYPE qbit_prism_initial_job_delivery_seconds summary", - f"qbit_prism_initial_job_delivery_seconds_sum {latency_sum:.6f}", - f"qbit_prism_initial_job_delivery_seconds_count {latency_count}", - "# HELP qbit_prism_initial_job_requests_total Initial delivery outcomes.", - "# TYPE qbit_prism_initial_job_requests_total counter", - *[ - f'qbit_prism_initial_job_requests_total{{result="{result}"}} {count}' - for result, count in sorted(counts.items()) - ], - "# HELP qbit_prism_initial_job_queue_capacity_reclaimed_total Queued initial-job slots reclaimed immediately by cancellation.", - "# TYPE qbit_prism_initial_job_queue_capacity_reclaimed_total counter", - f"qbit_prism_initial_job_queue_capacity_reclaimed_total {queue_capacity_reclaimed}", - "# HELP qbit_prism_shared_bundle_preparation_seconds Heavy shared bundle preparation wall time.", - "# TYPE qbit_prism_shared_bundle_preparation_seconds summary", - f"qbit_prism_shared_bundle_preparation_seconds_sum {preparation_sum:.6f}", - f"qbit_prism_shared_bundle_preparation_seconds_count {preparation_count}", - "# HELP qbit_prism_shared_bundle_preparation_waiters Callers waiting on the keyed shared preparation flight.", - "# TYPE qbit_prism_shared_bundle_preparation_waiters gauge", - f"qbit_prism_shared_bundle_preparation_waiters {waiters}", - "# HELP qbit_prism_shared_bundle_builds_total Shared bundle builds by terminal outcome.", - "# TYPE qbit_prism_shared_bundle_builds_total counter", - *[ - f'qbit_prism_shared_bundle_builds_total{{result="{result}"}} {count}' - for result, count in sorted(build_counts.items()) - ], - ] - def vardiff_idle_metrics_lines(self) -> list[str]: - self._ensure_vardiff_idle_state() - with self._vardiff_idle_lock: - sweep = { - "buckets": dict(self.vardiff_idle_sweep_histogram["buckets"]), - "sum": float(self.vardiff_idle_sweep_histogram["sum"]), - "count": int(self.vardiff_idle_sweep_histogram["count"]), - } - task = { - "buckets": dict(self.vardiff_idle_task_histogram["buckets"]), - "sum": float(self.vardiff_idle_task_histogram["sum"]), - "count": int(self.vardiff_idle_task_histogram["count"]), - } - inspected = self.vardiff_idle_clients_inspected - skip_counts = dict(self.vardiff_idle_skip_counts) - queue_depth = self.vardiff_idle_queue_depth - inflight = self.vardiff_idle_inflight - failures = self.vardiff_idle_task_failures - - lines = [ - "# HELP qbit_prism_vardiff_idle_clients_inspected_total Clients inspected by bounded vardiff idle sweeps.", - "# TYPE qbit_prism_vardiff_idle_clients_inspected_total counter", - f"qbit_prism_vardiff_idle_clients_inspected_total {inspected}", - "# HELP qbit_prism_vardiff_idle_skips_total Idle retargets skipped by a bounded reason.", - "# TYPE qbit_prism_vardiff_idle_skips_total counter", - *[ - f'qbit_prism_vardiff_idle_skips_total{{reason="{reason}"}} {int(skip_counts.get(reason, 0))}' - for reason in PRISM_VARDIFF_IDLE_SKIP_REASONS - ], - "# HELP qbit_prism_vardiff_idle_queue_depth Cache-only idle retarget tasks waiting for a dedicated worker.", - "# TYPE qbit_prism_vardiff_idle_queue_depth gauge", - f"qbit_prism_vardiff_idle_queue_depth {queue_depth}", - "# HELP qbit_prism_vardiff_idle_inflight Cache-only idle retarget tasks currently running.", - "# TYPE qbit_prism_vardiff_idle_inflight gauge", - f"qbit_prism_vardiff_idle_inflight {inflight}", - "# HELP qbit_prism_vardiff_idle_task_failures_total Idle retarget tasks that failed during cached delivery.", - "# TYPE qbit_prism_vardiff_idle_task_failures_total counter", - f"qbit_prism_vardiff_idle_task_failures_total {failures}", - ] - for metric_name, description, histogram in ( - ( - "qbit_prism_vardiff_idle_sweep_seconds", - "Wall time of one bounded vardiff idle sweep.", - sweep, - ), - ( - "qbit_prism_vardiff_idle_retarget_task_seconds", - "Queue plus execution latency for cache-only idle retarget tasks.", - task, - ), - ): - lines.extend( - [ - f"# HELP {metric_name} {description}", - f"# TYPE {metric_name} histogram", - *[ - f'{metric_name}_bucket{{le="{bucket:g}"}} {histogram["buckets"].get(bucket, 0)}' - for bucket in PRISM_VARDIFF_IDLE_SECONDS_BUCKETS - ], - f'{metric_name}_bucket{{le="+Inf"}} {histogram["count"]}', - f'{metric_name}_sum {float(histogram["sum"]):.6f}', - f'{metric_name}_count {histogram["count"]}', - ] - ) - return lines - def tip_refresh_metrics_lines(self) -> list[str]: - self._ensure_tip_refresh_state() - with self._tip_refresh_executor_lock: - executor_workers = ( - self.tip_refresh_max_workers - if self._tip_refresh_executor is not None - else 0 - ) - with self._tip_refresh_metrics_lock: - histograms = { - name: { - "buckets": dict(histogram["buckets"]), - "sum": float(histogram["sum"]), - "count": int(histogram["count"]), - } - for name, histogram in self.tip_refresh_histograms.items() - } - phase_histograms = { - phase: { - "buckets": dict(histogram["buckets"]), - "sum": float(histogram["sum"]), - "count": int(histogram["count"]), - } - for phase, histogram in self.tip_refresh_build_phase_histograms.items() - } - client_counts = dict(self.tip_refresh_client_counts) - cancellation_counts = dict(self.tip_refresh_cancellation_counts) - wave_outcome_counts = dict(self.tip_refresh_wave_outcome_counts) - coverage_histograms = { - target: { - "buckets": dict(histogram["buckets"]), - "sum": float(histogram["sum"]), - "count": int(histogram["count"]), - } - for target, histogram in ( - self.tip_refresh_coverage_histograms.items() - ) - } - inflight = self.tip_refresh_inflight - build_inflight = self.tip_refresh_build_inflight - build_queue_depth = self.tip_refresh_build_queue_depth - singleflight_hits = self.tip_refresh_singleflight_hits - superseded_results = self.tip_refresh_superseded_results - worker_failures = self.tip_refresh_worker_failures - worker_restarts = self.tip_refresh_worker_restarts - ipc_bytes = dict(self.tip_refresh_ipc_bytes) - - metric_names = { - "refresh": "qbit_prism_tip_refresh_seconds", - "bundle_build": "qbit_prism_tip_refresh_bundle_build_seconds", - "first_delivery": "qbit_prism_tip_refresh_first_delivery_seconds", - "last_delivery": "qbit_prism_tip_refresh_last_delivery_seconds", - "fanout_wave": "qbit_prism_tip_refresh_fanout_wave_seconds", - } - descriptions = { - "refresh": "Full qbit tip/template refresh pass wall time.", - "bundle_build": "Shared ready-pool refresh bundle preparation wall time.", - "first_delivery": "Tip observation to first successful client delivery.", - "last_delivery": "Tip observation to last successful client delivery.", - "fanout_wave": "First task submission to last successful delivery of one completed prepared fanout wave.", + def _observe_accepted_block_preview_publication( + self, + block_hash: str, + *, + result: str, + ) -> None: + # Called from the P1 publication boundary; the B1 owner holds the + # acceptance stamp taken on the submitter thread. + service = self._ensure_block_candidate_service() + service._observe_accepted_block_preview_publication( + block_hash, + result=result, + ) + + def block_submitter_snapshot(self) -> dict[str, object]: + """Merged durable pending-candidate and B1 owner snapshot. + + Combines the ledger's durable pending aggregates with the block- + candidate service's copied backoff, submit-histogram, and + acceptance-to-preview-publication snapshots; each owner copies under + its own lock, so no locks nest here. + """ + pending_metrics: dict[str, object] = { + "pending_count": -1, + "oldest_pending_age_seconds": -1.0, + "oldest_unattempted_age_seconds": -1.0, } - lines: list[str] = [] - for name, metric_name in metric_names.items(): - histogram = histograms[name] - buckets = histogram["buckets"] - assert isinstance(buckets, dict) - lines.extend( - [ - f"# HELP {metric_name} {descriptions[name]}", - f"# TYPE {metric_name} histogram", - *[ - f'{metric_name}_bucket{{le="{bucket:g}"}} {int(buckets.get(bucket, 0))}' - for bucket in PRISM_TIP_REFRESH_SECONDS_BUCKETS - ], - f'{metric_name}_bucket{{le="+Inf"}} {histogram["count"]}', - f'{metric_name}_sum {float(histogram["sum"]):.6f}', - f'{metric_name}_count {histogram["count"]}', - ] - ) - phase_metric_name = "qbit_prism_tip_refresh_bundle_phase_seconds" - lines.extend( - [ - "# HELP qbit_prism_tip_refresh_bundle_phase_seconds Shared bundle-build phase wall time.", - "# TYPE qbit_prism_tip_refresh_bundle_phase_seconds histogram", - ] + pending_snapshot = getattr(self.ledger, "block_candidate_pending_metrics", None) + if callable(pending_snapshot): + try: + pending_metrics.update(pending_snapshot()) + except Exception: + # Metrics collection is diagnostic. Candidate processing and + # its watchdog remain authoritative when this read is down. + pass + service = self._ensure_block_candidate_service() + backoff_active, backoff_remaining, backoff_delay = ( + service.backoff_snapshot() ) - for phase in PRISM_TIP_REFRESH_BUILD_PHASES: - histogram = phase_histograms[phase] - buckets = histogram["buckets"] - assert isinstance(buckets, dict) - lines.extend( - [ - *[ - f'{phase_metric_name}_bucket{{phase="{phase}",le="{bucket:g}"}} {int(buckets.get(bucket, 0))}' - for bucket in PRISM_TIP_REFRESH_SECONDS_BUCKETS - ], - f'{phase_metric_name}_bucket{{phase="{phase}",le="+Inf"}} {histogram["count"]}', - f'{phase_metric_name}_sum{{phase="{phase}"}} {float(histogram["sum"]):.6f}', - f'{phase_metric_name}_count{{phase="{phase}"}} {histogram["count"]}', - ] - ) - lines.extend( - [ - "# HELP qbit_prism_tip_refresh_wave_outcomes_total Singleflight refresh waves by terminal or recovered supersession outcome.", - "# TYPE qbit_prism_tip_refresh_wave_outcomes_total counter", - *[ - f'qbit_prism_tip_refresh_wave_outcomes_total{{outcome="{outcome}"}} {int(wave_outcome_counts.get(outcome, 0))}' - for outcome in PRISM_TIP_REFRESH_WAVE_OUTCOMES - ], - "# HELP qbit_prism_tip_refresh_clients_total Client outcomes from tip/template refresh passes.", - "# TYPE qbit_prism_tip_refresh_clients_total counter", - *[ - f'qbit_prism_tip_refresh_clients_total{{result="{result}"}} {int(client_counts.get(result, 0))}' - for result in PRISM_TIP_REFRESH_RESULTS - ], - "# HELP qbit_prism_tip_refresh_cancellations_total Obsolete prepared refresh tasks canceled before delivery admission.", - "# TYPE qbit_prism_tip_refresh_cancellations_total counter", - *[ - f'qbit_prism_tip_refresh_cancellations_total{{stage="{stage}"}} {int(cancellation_counts.get(stage, 0))}' - for stage in PRISM_TIP_REFRESH_CANCELLATION_STAGES - ], - "# HELP qbit_prism_tip_refresh_inflight Prepared refresh client tasks currently queued or running.", - "# TYPE qbit_prism_tip_refresh_inflight gauge", - f"qbit_prism_tip_refresh_inflight {inflight}", - "# HELP qbit_prism_tip_refresh_executor_workers Configured persistent refresh executor workers, or zero before creation.", - "# TYPE qbit_prism_tip_refresh_executor_workers gauge", - f"qbit_prism_tip_refresh_executor_workers {executor_workers}", - "# HELP qbit_prism_tip_refresh_bundle_inflight Shared bundle builds currently running.", - "# TYPE qbit_prism_tip_refresh_bundle_inflight gauge", - f"qbit_prism_tip_refresh_bundle_inflight {build_inflight}", - "# HELP qbit_prism_tip_refresh_bundle_queue_depth Shared bundle callers waiting on bounded build admission or an identical single-flight.", - "# TYPE qbit_prism_tip_refresh_bundle_queue_depth gauge", - f"qbit_prism_tip_refresh_bundle_queue_depth {build_queue_depth}", - "# HELP qbit_prism_tip_refresh_bundle_singleflight_hits_total Shared bundle callers coalesced behind an identical build.", - "# TYPE qbit_prism_tip_refresh_bundle_singleflight_hits_total counter", - f"qbit_prism_tip_refresh_bundle_singleflight_hits_total {singleflight_hits}", - "# HELP qbit_prism_tip_refresh_bundle_superseded_results_total Completed or canceled shared bundles discarded after supersession.", - "# TYPE qbit_prism_tip_refresh_bundle_superseded_results_total counter", - f"qbit_prism_tip_refresh_bundle_superseded_results_total {superseded_results}", - "# HELP qbit_prism_tip_refresh_builder_worker_failures_total Audit-builder subprocess failures.", - "# TYPE qbit_prism_tip_refresh_builder_worker_failures_total counter", - f"qbit_prism_tip_refresh_builder_worker_failures_total {worker_failures}", - "# HELP qbit_prism_tip_refresh_builder_worker_restarts_total Long-lived builder worker restarts (persistent --serve daemon respawns after a crash or supersession).", - "# TYPE qbit_prism_tip_refresh_builder_worker_restarts_total counter", - f"qbit_prism_tip_refresh_builder_worker_restarts_total {worker_restarts}", - "# HELP qbit_prism_tip_refresh_builder_ipc_bytes_total Bytes copied across audit-builder subprocess IPC.", - "# TYPE qbit_prism_tip_refresh_builder_ipc_bytes_total counter", - *[ - f'qbit_prism_tip_refresh_builder_ipc_bytes_total{{direction="{direction}"}} {int(ipc_bytes.get(direction, 0))}' - for direction in ("input", "output") - ], - ] + submit_buckets, submit_sum, submit_count = ( + service.block_submit_seconds_snapshot() ) - coverage_metric_name = "qbit_prism_tip_refresh_hashrate_coverage_seconds" - lines.extend( - [ - "# HELP qbit_prism_tip_refresh_hashrate_coverage_seconds Time from epoch detection until a fixed share of the snapshotted hashrate proxy received current work.", - "# TYPE qbit_prism_tip_refresh_hashrate_coverage_seconds histogram", - ] + accepted_preview_publication = ( + service.accepted_block_preview_publication_snapshot() ) - for target, _ratio in PRISM_TIP_REFRESH_COVERAGE_TARGETS: - histogram = coverage_histograms[target] - buckets = histogram["buckets"] - assert isinstance(buckets, dict) - lines.extend( - [ - *[ - f'{coverage_metric_name}_bucket{{coverage="{target}",le="{bucket:g}"}} {int(buckets.get(bucket, 0))}' - for bucket in PRISM_TIP_REFRESH_SECONDS_BUCKETS - ], - f'{coverage_metric_name}_bucket{{coverage="{target}",le="+Inf"}} {histogram["count"]}', - f'{coverage_metric_name}_sum{{coverage="{target}"}} {float(histogram["sum"]):.6f}', - f'{coverage_metric_name}_count{{coverage="{target}"}} {histogram["count"]}', - ] - ) - self._ensure_job_cache_state() - with self._serve_builder_metrics_lock: - serve_counts = dict(self.serve_builder_counts) - serve_window_counts = dict(self.serve_builder_window_cache_counts) - with self._payout_artifact_executor_lock: - payout_artifact_events = dict(self.payout_artifact_event_counts) - lines.extend( - [ - "# HELP qbit_prism_serve_builder_events_total Persistent audit-builder daemon lifecycle and request outcomes.", - "# TYPE qbit_prism_serve_builder_events_total counter", - *[ - f'qbit_prism_serve_builder_events_total{{event="{event}"}} {int(serve_counts.get(event, 0))}' - for event in ( - "requests", - "fallbacks", - "spawns", - "window_uploads", - ) - ], - "# HELP qbit_prism_payout_artifact_events_total Payout-ledger-artifact lifecycle outcomes (build, install, reuse pacing).", - "# TYPE qbit_prism_payout_artifact_events_total counter", - *[ - f'qbit_prism_payout_artifact_events_total{{event="{event}"}} {int(payout_artifact_events.get(event, 0))}' - for event in ( - "built", - "build_aborted", - "debounced", - "incremental", - "full_rescan", - "self_check_match", - "self_check_mismatch", - "self_check_failed", - "balance_check_mismatch", - "found_block_cached", - "installed", - "refreshed", - "already_current", - "discarded", - "born_expired", - "rearm_scheduled", - "served_reuse", - "probe_rejected_ceiling", - "probe_past_floor", - ) - ], - "# HELP qbit_prism_serve_builder_window_cache_total Daemon parsed share-window cache outcomes.", - "# TYPE qbit_prism_serve_builder_window_cache_total counter", - *[ - f'qbit_prism_serve_builder_window_cache_total{{result="{result}"}} {int(serve_window_counts.get(counter_key, 0))}' - for result, counter_key in ( - ("hit", "hits"), - ("miss", "misses"), - ) - ], - ] + pending_metrics.update( + { + "backoff_active": backoff_active, + "backoff_remaining_seconds": backoff_remaining, + "backoff_delay_seconds": backoff_delay, + "submit_seconds_buckets": submit_buckets, + "submit_seconds_sum": submit_sum, + "submit_seconds_count": submit_count, + "accepted_preview_publication": accepted_preview_publication, + } ) - return lines + return pending_metrics - def job_build_metrics_lines(self) -> list[str]: - self._ensure_job_cache_state() - with self._job_cache_lock: - bucket_counts = dict(self.job_build_seconds_bucket_counts) - build_sum = self.job_build_seconds_sum - build_count = self.job_build_count - phase_seconds = dict(self.job_build_phase_seconds) - hit_counts = dict(self.job_cache_hit_counts) - miss_counts = dict(self.job_cache_miss_counts) - health_refresh_failures = self.health_snapshot_refresh_failure_count - with self._job_build_scheduler_lock: - scheduler_counts = dict(self.job_build_scheduler_counts) - priority_counts = dict(self.job_build_priority_counts) - priority_admission_seconds = dict( - self.job_build_priority_admission_seconds - ) - initial_prepared_counts = dict( - self.initial_job_prepared_work_counts - ) - cancellation_seconds = dict(self.job_build_cancellation_seconds) - replacement_seconds = dict(self.job_build_replacement_start_seconds) - worker_counts = dict(self.job_build_worker_counts) - active_builds = int(self._job_build_active is not None) - pending_builds = int(self._job_build_pending is not None) - priority_requests = tuple( - request - for request in ( - ( - self._job_build_active.request - if self._job_build_active is not None - else None - ), - ( - self._job_build_retiring.request - if self._job_build_retiring is not None - else None - ), - self._job_build_pending, - ) - if request is not None - and not request.cancellation.is_set() - and self._job_build_is_publication_critical(request) - ) - priority_preparations = tuple( - self._job_build_priority_preparations.values() - ) - priority_active = int( - bool(priority_requests or priority_preparations) - ) - priority_age_seconds = max( - ( - time.monotonic() - request.requested_monotonic - for request in priority_requests + def block_submitter_metrics_lines(self) -> list[str]: + # Compatibility wrapper for the extracted metrics owner. + return self._ensure_metrics_renderer().block_submitter_metrics_lines() + + def landing_observability_metrics_lines(self) -> list[str]: + # Compatibility wrapper for the extracted metrics owner. + return ( + self._ensure_metrics_renderer().landing_observability_metrics_lines() + ) + + def _accepted_stats_reconcile_metric_lines(self) -> list[str]: + # Compatibility wrapper for the extracted metrics owner. + return ( + self._ensure_metrics_renderer().accepted_stats_reconcile_metric_lines() + ) + + def share_accounting_snapshot(self) -> dict[str, object]: + """Copy the hot-path share counters under the accounting lock.""" + self._ensure_share_hot_path_state() + with self._share_accounting_lock: + return { + "submitted": int(getattr(self, "submitted_share_count", 0)), + "stale": int(getattr(self, "stale_share_count", 0)), + "duplicate": int(getattr(self, "duplicate_share_count", 0)), + "low_difficulty": int( + getattr(self, "low_difficulty_share_count", 0) ), - default=0.0, - ) - priority_age_seconds = max( - priority_age_seconds, - max( - ( - time.monotonic() - started - for started in priority_preparations - ), - default=0.0, + "collection_block": int( + getattr(self, "collection_block_submission_count", 0) ), - ) - lock = getattr(self, "lock", None) - if lock is not None: - with lock: - connected_clients = len(getattr(self, "clients", ())) - else: - connected_clients = len(getattr(self, "clients", ())) - lines = [ - "# HELP qbit_prism_job_build_seconds Wall time from client job build or prepared submission to completion, including skipped prepared tasks.", - "# TYPE qbit_prism_job_build_seconds histogram", - ] - for bucket in PRISM_JOB_BUILD_SECONDS_BUCKETS: - lines.append( - f'qbit_prism_job_build_seconds_bucket{{le="{bucket:g}"}} {bucket_counts.get(bucket, 0)}' - ) - lines.extend( - [ - f'qbit_prism_job_build_seconds_bucket{{le="+Inf"}} {build_count}', - f"qbit_prism_job_build_seconds_sum {build_sum:.6f}", - f"qbit_prism_job_build_seconds_count {build_count}", - "# HELP qbit_prism_job_build_phase_seconds_total Cumulative job build wall time by phase.", - "# TYPE qbit_prism_job_build_phase_seconds_total counter", - *[ - f'qbit_prism_job_build_phase_seconds_total{{phase="{phase}"}} {phase_seconds.get(phase, 0.0):.6f}' - for phase in PRISM_JOB_BUILD_PHASES - ], - "# HELP qbit_prism_job_cache_hits_total Job build cache hits by cache kind.", - "# TYPE qbit_prism_job_cache_hits_total counter", - *[ - f'qbit_prism_job_cache_hits_total{{cache="{kind}"}} {int(hit_counts.get(kind, 0))}' - for kind in PRISM_JOB_CACHE_KINDS - ], - "# HELP qbit_prism_job_cache_misses_total Job build cache misses by cache kind.", - "# TYPE qbit_prism_job_cache_misses_total counter", - *[ - f'qbit_prism_job_cache_misses_total{{cache="{kind}"}} {int(miss_counts.get(kind, 0))}' - for kind in PRISM_JOB_CACHE_KINDS - ], - "# HELP qbit_prism_health_snapshot_refresh_failures_total Background health snapshot refreshes that raised.", - "# TYPE qbit_prism_health_snapshot_refresh_failures_total counter", - f"qbit_prism_health_snapshot_refresh_failures_total {health_refresh_failures}", - "# HELP qbit_prism_connected_clients Currently connected Stratum clients.", - "# TYPE qbit_prism_connected_clients gauge", - f"qbit_prism_connected_clients {connected_clients}", - "# HELP qbit_prism_job_build_requests_total Immutable job build requests admitted to the latest-wins scheduler.", - "# TYPE qbit_prism_job_build_requests_total counter", - f'qbit_prism_job_build_requests_total {int(scheduler_counts.get("requests", 0))}', - "# HELP qbit_prism_job_build_starts_total Immutable job builds started by the bounded executor.", - "# TYPE qbit_prism_job_build_starts_total counter", - f'qbit_prism_job_build_starts_total {int(scheduler_counts.get("starts", 0))}', - "# HELP qbit_prism_job_build_completions_total Immutable job build executions completed.", - "# TYPE qbit_prism_job_build_completions_total counter", - f'qbit_prism_job_build_completions_total {int(scheduler_counts.get("completions", 0))}', - "# HELP qbit_prism_job_build_supersessions_total Active or pending builds replaced by a newer immutable key.", - "# TYPE qbit_prism_job_build_supersessions_total counter", - f'qbit_prism_job_build_supersessions_total {int(scheduler_counts.get("supersessions", 0))}', - "# HELP qbit_prism_job_build_obsolete_results_total Obsolete build results discarded before cache or delivery.", - "# TYPE qbit_prism_job_build_obsolete_results_total counter", - f'qbit_prism_job_build_obsolete_results_total {int(scheduler_counts.get("obsolete_results", 0))}', - "# HELP qbit_prism_job_build_orphan_evicted_total Finished flights evicted from scheduler slots after their completion never released them.", - "# TYPE qbit_prism_job_build_orphan_evicted_total counter", - f'qbit_prism_job_build_orphan_evicted_total {int(scheduler_counts.get("orphan_evicted", 0))}', - "# HELP qbit_prism_job_build_active Current latest-generation build executions.", - "# TYPE qbit_prism_job_build_active gauge", - f"qbit_prism_job_build_active {active_builds}", - "# HELP qbit_prism_job_build_pending Newest build request waiting for a bounded executor slot.", - "# TYPE qbit_prism_job_build_pending gauge", - f"qbit_prism_job_build_pending {pending_builds}", - "# HELP qbit_prism_job_build_cancellation_seconds Cancellation signal to obsolete execution completion.", - "# TYPE qbit_prism_job_build_cancellation_seconds summary", - f'qbit_prism_job_build_cancellation_seconds_sum {float(cancellation_seconds.get("sum", 0.0)):.6f}', - f'qbit_prism_job_build_cancellation_seconds_count {int(cancellation_seconds.get("count", 0))}', - "# HELP qbit_prism_job_build_replacement_start_seconds Supersession signal to replacement build start.", - "# TYPE qbit_prism_job_build_replacement_start_seconds summary", - f'qbit_prism_job_build_replacement_start_seconds_sum {float(replacement_seconds.get("sum", 0.0)):.6f}', - f'qbit_prism_job_build_replacement_start_seconds_count {int(replacement_seconds.get("count", 0))}', - "# HELP qbit_prism_job_build_priority_events_total Publication-critical scheduler admissions and routine-work displacement.", - "# TYPE qbit_prism_job_build_priority_events_total counter", - *[ - f'qbit_prism_job_build_priority_events_total{{result="{result}"}} {int(priority_counts.get(result, 0))}' - for result in ( - "started", - "coalesced", - "queued", - "routine_deferred", - "routine_preempted", - ) - ], - "# HELP qbit_prism_job_build_priority_admission_seconds Publication-priority reservation to builder start or exact-flight coalescing.", - "# TYPE qbit_prism_job_build_priority_admission_seconds summary", - f'qbit_prism_job_build_priority_admission_seconds_sum {float(priority_admission_seconds.get("sum", 0.0)):.6f}', - f'qbit_prism_job_build_priority_admission_seconds_count {int(priority_admission_seconds.get("count", 0))}', - "# HELP qbit_prism_job_build_priority_active Whether publication-critical build work is preparing, running, retiring, or pending.", - "# TYPE qbit_prism_job_build_priority_active gauge", - f"qbit_prism_job_build_priority_active {priority_active}", - "# HELP qbit_prism_job_build_priority_age_seconds Age of the oldest admitted publication-critical build request.", - "# TYPE qbit_prism_job_build_priority_age_seconds gauge", - f"qbit_prism_job_build_priority_age_seconds {priority_age_seconds:.6f}", - "# HELP qbit_prism_initial_job_prepared_work_total Initial jobs that reused, coalesced behind, or deferred to prepared shared work.", - "# TYPE qbit_prism_initial_job_prepared_work_total counter", - *[ - f'qbit_prism_initial_job_prepared_work_total{{result="{result}"}} {int(initial_prepared_counts.get(result, 0))}' - for result in ( - "cache_hit", - "singleflight", - "deferred", - "subscribed", - "admission_deadline", - ) - ], - "# HELP qbit_prism_job_build_worker_events_total Pure builder subprocess lifecycle events.", - "# TYPE qbit_prism_job_build_worker_events_total counter", - *[ - f'qbit_prism_job_build_worker_events_total{{event="{event}"}} {int(worker_counts.get(event, 0))}' - for event in ("starts", "terminations", "crashes", "restarts") - ], - "# HELP qbit_prism_job_build_configured_workers Configured bounded job-build executor worker count.", - "# TYPE qbit_prism_job_build_configured_workers gauge", - ( - "qbit_prism_job_build_configured_workers " - f"{int(getattr(self, 'job_build_executor_workers', DEFAULT_PRISM_JOB_BUILD_EXECUTOR_WORKERS))}" + "rejections": dict( + getattr(self, "rejection_counts_by_reason", {}) + ), + "grace_credited": int( + getattr(self, "grace_credited_share_count", 0) ), - ] - ) - return lines - - def payout_state_metrics_lines(self) -> list[str]: - self._ensure_job_cache_state() - with self._payout_state_metrics_lock: - state_histograms = { - name: { - "buckets": dict(histogram["buckets"]), - "sum": float(histogram["sum"]), - "count": int(histogram["count"]), - } - for name, histogram in self.payout_state_histograms.items() - } - gate_histograms = { - relation: { - "buckets": dict(histogram["buckets"]), - "sum": float(histogram["sum"]), - "count": int(histogram["count"]), - } - for relation, histogram in self.payout_gate_wait_histograms.items() } - discarded = self.payout_state_candidates_discarded - metric_names = { - "preparation": "qbit_prism_payout_preparation_seconds", - "publish": "qbit_prism_payout_publish_seconds", - "first_delivery": "qbit_prism_payout_invalidation_first_delivery_seconds", - } - descriptions = { - "preparation": "Payout reconciliation and candidate preparation outside delivery publication.", - "publish": "Atomic payout generation/cache publication gate-hold time.", - "first_delivery": "Payout invalidation to first delivery of the published generation.", - } - lines: list[str] = [] - for name, metric_name in metric_names.items(): - histogram = state_histograms[name] - buckets = histogram["buckets"] - assert isinstance(buckets, dict) - lines.extend( - [ - f"# HELP {metric_name} {descriptions[name]}", - f"# TYPE {metric_name} histogram", - *[ - f'{metric_name}_bucket{{le="{bucket:g}"}} {int(buckets.get(bucket, 0))}' - for bucket in PRISM_TIP_REFRESH_SECONDS_BUCKETS - ], - f'{metric_name}_bucket{{le="+Inf"}} {histogram["count"]}', - f'{metric_name}_sum {float(histogram["sum"]):.6f}', - f'{metric_name}_count {histogram["count"]}', - ] - ) + @staticmethod + def rejection_reason_ids() -> tuple[str, ...]: + # The renderer must not import coordinator globals. + return PRISM_REJECTION_REASON_IDS - gate_name = "qbit_prism_payout_gate_wait_seconds" - lines.extend( - [ - "# HELP qbit_prism_payout_gate_wait_seconds Delivery admission wait by generation relationship to the published payout state.", - "# TYPE qbit_prism_payout_gate_wait_seconds histogram", - ] - ) - for relation in PRISM_PAYOUT_DELIVERY_GENERATIONS: - histogram = gate_histograms[relation] - buckets = histogram["buckets"] - assert isinstance(buckets, dict) - lines.extend( - [ - *[ - f'{gate_name}_bucket{{generation="{relation}",le="{bucket:g}"}} {int(buckets.get(bucket, 0))}' - for bucket in PRISM_TIP_REFRESH_SECONDS_BUCKETS - ], - f'{gate_name}_bucket{{generation="{relation}",le="+Inf"}} {histogram["count"]}', - f'{gate_name}_sum{{generation="{relation}"}} {float(histogram["sum"]):.6f}', - f'{gate_name}_count{{generation="{relation}"}} {histogram["count"]}', - ] - ) - lines.extend( - [ - "# HELP qbit_prism_payout_candidates_discarded_total Prepared payout candidates discarded after source supersession.", - "# TYPE qbit_prism_payout_candidates_discarded_total counter", - f"qbit_prism_payout_candidates_discarded_total {discarded}", - ] - ) - return lines + def _ensure_metrics_renderer(self) -> MetricsRenderer: + renderer = self.__dict__.get("_metrics_renderer") + if renderer is None: + # Idempotent construction; no init lock is needed. + renderer = MetricsRenderer(self) + self.__dict__["_metrics_renderer"] = renderer + return renderer + def _render_metrics_payload(self) -> str: + return self._ensure_metrics_renderer().render() -def make_audit_handler(coordinator: PrismCoordinator) -> type[BaseHTTPRequestHandler]: - public_response_cache = public_api.PublicResponseCache() + def metrics_payload(self) -> str: + """Compatibility renderer; HTTP uses the complete cached snapshot.""" - class AuditHandler(BaseHTTPRequestHandler): - server_version = "QbitPrismAudit/0.1" + return self._render_metrics_payload() - def do_GET(self) -> None: - parsed = urllib.parse.urlparse(self.path) - path = parsed.path.rstrip("/") or "/" - query = urllib.parse.parse_qs(parsed.query) - try: - if path == "/healthz": - cached_health = getattr(coordinator, "cached_health_payload", None) - if callable(cached_health): - status, payload = cached_health() - self.write_json(status, payload) - else: - self.write_json(200, coordinator.health_payload()) - return - if path == "/metrics": - self.write_text(200, coordinator.metrics_payload(), "text/plain; version=0.0.4") - return - if path == "/public/v1" or path.startswith("/public/v1/"): - self.handle_public(path, query) - return - if path == "/audit/latest": - payload = coordinator.latest_evidence_payload() - if payload is None: - self.write_json(404, {"error": "no PRISM evidence has been produced"}) - else: - self.write_json(200, payload) - return - if path in {"/owed", "/owed-balances"}: - self.write_json(200, coordinator.owed_balances_payload()) - return - if path in {"/audit/carry-forward-integrity", "/audit/ledger-integrity"}: - self.write_json(200, coordinator.carry_forward_integrity_payload()) - return - if path.startswith("/miners/") and path.endswith("/status"): - recipient_id = urllib.parse.unquote(path.removeprefix("/miners/").removesuffix("/status")) - self.write_json(200, coordinator.miner_status_payload(recipient_id)) - return - if path.startswith("/payouts/") and path.endswith("/status"): - recipient_id = urllib.parse.unquote(path.removeprefix("/payouts/").removesuffix("/status")) - self.write_json(200, coordinator.miner_status_payload(recipient_id)) - return - if path == "/audit/share-window": - self.handle_share_window(query) - return - if path.startswith("/audit/blocks/") and path.endswith("/payouts"): - block_hash = path.removeprefix("/audit/blocks/").removesuffix("/payouts") - self.handle_block_payouts(block_hash) - return - if path.startswith("/audit/blocks/") and path.endswith("/ctv-fanouts"): - block_hash = path.removeprefix("/audit/blocks/").removesuffix("/ctv-fanouts") - self.handle_block_ctv_fanouts(block_hash) - return - if path.startswith("/audit/blocks/") and path.endswith("/ctv-fanout-manifest-set"): - block_hash = path.removeprefix("/audit/blocks/").removesuffix("/ctv-fanout-manifest-set") - self.handle_block_ctv_fanout_manifest_set(block_hash) - return - if path == "/audit/fanouts/pending": - self.handle_pending_ctv_fanouts(query) - return - if path.startswith("/audit/fanouts/") and path.endswith("/status"): - fanout_txid = path.removeprefix("/audit/fanouts/").removesuffix("/status") - self.handle_ctv_fanout_status(fanout_txid) - return - if path.startswith("/audit/commitments/") and path.endswith("/bundle"): - commitment_leaf_hex = path.removeprefix("/audit/commitments/").removesuffix("/bundle") - self.handle_commitment_bundle(commitment_leaf_hex) - return - if path.startswith("/audit/block/"): - block_hash = path.removeprefix("/audit/block/") - self.handle_block_payouts(block_hash) - return - if path.startswith("/audit/blocks/") and path.endswith("/bundle"): - block_hash = path.removeprefix("/audit/blocks/").removesuffix("/bundle") - self.handle_block_bundle(block_hash) - return - self.write_json(404, {"error": "unknown endpoint"}) - except public_api.PublicApiError as exc: - self.write_json( - exc.status, - public_api.error_payload(exc.code, exc.message), - headers=public_api.public_error_headers(), - ) - except ValueError as exc: - if path == "/public/v1" or path.startswith("/public/v1/"): - self.write_json( - 500, - public_api.error_payload("internal_error", "internal server error"), - headers=public_api.public_error_headers(), - ) - else: - self.write_json(400, {"error": str(exc)}) - except Exception as exc: - if path == "/public/v1" or path.startswith("/public/v1/"): - self.write_json( - 500, - public_api.error_payload("internal_error", "internal server error"), - headers=public_api.public_error_headers(), - ) - else: - self.write_json(500, {"error": str(exc)}) - - def handle_public(self, path: str, query: dict[str, list[str]]) -> None: - cache_policy = public_api.public_cache_policy(path) - status, payload, cache_state, age_seconds = public_response_cache.get_or_compute( - key=public_api.public_cache_key(path, query), - ttl_seconds=cache_policy.ttl_seconds, - compute=lambda: public_api.dispatch(coordinator, path, query), - ) - self.write_json( - status, - payload, - headers=public_api.public_cache_headers( - cache_policy, - cache_state=cache_state, - age_seconds=age_seconds, - ), - ) + def shutdown_metrics_lines(self) -> list[str]: + # Compatibility wrapper for the extracted metrics owner. + return self._ensure_metrics_renderer().shutdown_metrics_lines() - def handle_share_window(self, query: dict[str, list[str]]) -> None: - anchor_raw = self.first_query_value(query, "anchor_job_issued_at_ms", "anchor") - difficulty_raw = self.first_query_value(query, "network_difficulty") - if anchor_raw is None or difficulty_raw is None: - raise ValueError("anchor_job_issued_at_ms and network_difficulty are required") - rows = coordinator.ledger.audit_share_window( - anchor_job_issued_at_ms=int(anchor_raw), - network_difficulty=int(difficulty_raw), - ) - self.write_json( - 200, - { - "schema": "qbit.prism.audit-share-window.v1", - "ledger_backend": coordinator.ledger.backend_name, - "rows": rows, - }, - ) + def audit_artifact_metrics(self) -> dict[str, dict[str, int] | int]: + return self._ensure_audit_artifact_store().metrics_snapshot() - def handle_block_payouts(self, block_hash: str) -> None: - block_hash = self.clean_hash(block_hash) - rows = coordinator.ledger.audit_block_payouts(block_hash=block_hash) - if not rows: - self.write_json(404, {"error": "unknown PRISM block", "block_hash": block_hash}) - return - self.write_json( - 200, - { - "schema": "qbit.prism.audit-block-payouts.v1", - "ledger_backend": coordinator.ledger.backend_name, - "block_hash": block_hash, - "rows": rows, - }, - ) + @staticmethod + def audit_artifact_kind(name: str) -> str: + return AuditArtifactStore.artifact_kind(name) - def handle_block_ctv_fanouts(self, block_hash: str) -> None: - block_hash = self.clean_hash(block_hash, name="block hash") - rows = coordinator.ledger.audit_ctv_fanouts(block_hash=block_hash) - if not rows: - self.write_json(404, {"error": "unknown CTV fanout block", "block_hash": block_hash}) - return - self.write_json( - 200, - { - "schema": "qbit.prism.audit-ctv-fanouts.v1", - "ledger_backend": coordinator.ledger.backend_name, - "block_hash": block_hash, - "rows": rows, - }, - ) + def ctv_fanout_broadcaster_metrics_lines(self) -> list[str]: + return self._ensure_ctv_runtime().metrics_lines() - def handle_block_ctv_fanout_manifest_set(self, block_hash: str) -> None: - block_hash = self.clean_hash(block_hash, name="block hash") - payload = coordinator.ledger.audit_ctv_fanout_manifest_set(block_hash=block_hash) - if payload is None: - self.write_json(404, {"error": "unknown CTV fanout block", "block_hash": block_hash}) - return - self.write_json(200, payload) + def initial_delivery_metrics_lines(self) -> list[str]: + # Compatibility wrapper for the extracted metrics owner. + return self._ensure_metrics_renderer().initial_delivery_metrics_lines() - def handle_ctv_fanout_status(self, fanout_txid: str) -> None: - fanout_txid = self.clean_hash(fanout_txid, name="fanout txid") - payload = coordinator.ledger.ctv_fanout_status(fanout_txid=fanout_txid) - if payload is None: - self.write_json(404, {"error": "unknown CTV fanout", "fanout_txid": fanout_txid}) - return - self.write_json(200, payload) - - def handle_pending_ctv_fanouts(self, query: dict[str, list[str]]) -> None: - limit_raw = self.first_query_value(query, "limit") - limit = int(limit_raw) if limit_raw is not None else 100 - rows = coordinator.ledger.pending_ctv_fanout_statuses(limit=limit) - self.write_json( - 200, - { - "schema": "qbit.prism.pending-ctv-fanouts.v1", - "ledger_backend": coordinator.ledger.backend_name, - "count": len(rows), - "rows": rows, - }, - ) + def vardiff_idle_metrics_lines(self) -> list[str]: + return self._ensure_vardiff_service().metrics_lines() - def handle_block_bundle(self, block_hash: str) -> None: - block_hash = self.clean_hash(block_hash, name="block hash") - payload = coordinator.ledger.audit_bundle(block_hash=block_hash) - if payload is None: - self.write_json(404, {"error": "unknown PRISM block", "block_hash": block_hash}) - return - self.write_json(200, payload) - - def handle_commitment_bundle(self, commitment_leaf_hex: str) -> None: - commitment_leaf_hex = self.clean_hash(commitment_leaf_hex, name="audit commitment leaf") - payload = coordinator.ledger.audit_bundle_by_commitment(commitment_leaf_hex=commitment_leaf_hex) - if payload is None: - self.write_json( - 404, - { - "error": "unknown PRISM audit commitment", - "audit_commitment_leaf_hex": commitment_leaf_hex, - }, - ) - return - self.write_json(200, payload) + def vardiff_convergence_snapshot(self) -> dict[str, object]: + return self._ensure_vardiff_service().convergence_snapshot() - def write_json(self, status: int, payload: object, headers: dict[str, str] | None = None) -> None: - body = json.dumps(payload, sort_keys=True).encode() + b"\n" - try: - self.send_response(status) - self.send_header("Content-Type", "application/json") - self.send_header("Content-Length", str(len(body))) - for key, value in (headers or {}).items(): - self.send_header(key, value) - self.end_headers() - self.wfile.write(body) - except (BrokenPipeError, ConnectionResetError): - # The client (typically a health checker with a short timeout) - # hung up before the response was written; nothing to salvage. - return + def block_finalization_metrics_lines(self) -> list[str]: + return self._ensure_block_finalization_service().metrics_lines() - def write_text(self, status: int, payload: str, content_type: str) -> None: - body = payload.encode() - try: - self.send_response(status) - self.send_header("Content-Type", content_type) - self.send_header("Content-Length", str(len(body))) - self.end_headers() - self.wfile.write(body) - except (BrokenPipeError, ConnectionResetError): - return + def tip_refresh_metrics_lines(self) -> list[str]: + return self._ensure_tip_refresh_service().tip_refresh_metrics_lines() - def log_message(self, format: str, *args: object) -> None: - return + def job_build_metrics_lines(self) -> list[str]: + # Compatibility wrapper for the extracted metrics owner. + return self._ensure_metrics_renderer().job_build_metrics_lines() - @staticmethod - def first_query_value(query: dict[str, list[str]], *keys: str) -> str | None: - for key in keys: - values = query.get(key) - if values: - return values[0] - return None + def payout_state_metrics_lines(self) -> list[str]: + return self._ensure_payout_state_service().payout_state_metrics_lines() - @staticmethod - def clean_hash(value: str, *, name: str = "block hash") -> str: - value = urllib.parse.unquote(value).strip() - if len(value) != 64 or any(char not in "0123456789abcdefABCDEF" for char in value): - raise ValueError(f"{name} must be 64 hex characters") - return value.lower() - return AuditHandler +def make_audit_handler(coordinator: PrismCoordinator) -> type[BaseHTTPRequestHandler]: + """Compatibility factory backed by the coordinator-free H1 facade.""" + + return AuditHttpFacade( + _CoordinatorAuditHttp( + coordinator, + allow_uncached_compatibility=True, + ) + ).handler_type() def target_from_compact(bits_hex: str) -> int: @@ -31019,6 +10173,18 @@ def main() -> int: # all-thread dump for live triage without disturbing the process. faulthandler.enable(all_threads=True) faulthandler.register(signal.SIGUSR2, all_threads=True) + # Before the coordinator, because constructing it starts threads and the + # switch interval is process-global: applied afterwards, already-running + # threads would observe the change at an arbitrary point in their own + # scheduling. Silent when unset -- an unconfigured pool must not log a + # value it did not choose. + switch_interval = apply_python_switch_interval() + if switch_interval is not None: + print( + "prism interpreter switch interval set to " + f"{switch_interval:g}s (default is 5ms)", + flush=True, + ) coordinator = PrismCoordinator() def _request_shutdown(signum: int, _frame: Any) -> None: @@ -31029,6 +10195,15 @@ def _request_shutdown(signum: int, _frame: Any) -> None: signal.signal(signal.SIGTERM, _request_shutdown) signal.signal(signal.SIGINT, _request_shutdown) + # Issue #226 part 2: the operator-triggered heap census (SIGUSR1) and + # malloc_trim (SIGRTMIN+1). Nothing is registered unless the switches are + # on, and each handler only writes one byte to the census thread's pipe -- + # the SIGUSR2 shape above with the work moved off the handler. The strict + # reader here sees the same environment the config loader validated when + # the coordinator loaded its configuration (the malloc-telemetry + # precedent), so the hook needs nothing from the instance. The census + # itself lives in process_telemetry. + install_heap_census(env_heap_census_config()) try: coordinator.serve() finally: diff --git a/lab/prism/process_telemetry.py b/lab/prism/process_telemetry.py new file mode 100644 index 000000000..e1d7a0caa --- /dev/null +++ b/lab/prism/process_telemetry.py @@ -0,0 +1,1622 @@ +"""Cheap, always-on interpreter and allocator telemetry for ``/metrics``. + +Issue #226: the production coordinator's resident set grows by roughly +390 MB per hour and nothing in the tree can say whether that growth is live +Python objects, pymalloc fragmentation, or glibc per-thread arena +fragmentation across the process's 64 threads. This module collects the +fixed-cardinality, constant-cost signals that discriminate between those +hypotheses on a running coordinator: + +- ``sys.getallocatedblocks()`` -- the live allocation count. Rising with RSS + means retention; flat while RSS rises means fragmentation. +- ``gc.get_count()`` and ``gc.get_stats()`` -- collector pressure and the + uncollectable count, per generation. ``get_count()`` returns the collector's + *trigger* counters (generation 0 is allocations minus deallocations), which + it compares against ``gc.get_threshold()``; they are not a count of retained + objects and they reset on every collection. Only ``get_stats()``'s + ``uncollectable`` counts objects. +- ``threading.active_count()`` -- every thread is a candidate glibc arena. +- glibc ``mallinfo2`` -- arena, in-use, free, and mmapped bytes across every + arena, so arena-minus-in-use is the fragmentation glibc has not returned. + +Everything the scrape collector reads is a snapshot. It never walks the +heap: no ``gc.get_objects()``, no ``tracemalloc``. It runs on every scrape. + +``mallinfo2`` exists only in glibc 2.33 and later. musl and macOS have no +such symbol, and the older int-typed ``mallinfo`` is deliberately not used +as a fallback because its 32-bit fields wrap on a multi-gigabyte heap. The +binding is attempted once per collector, and an absent symbol is remembered +so a scrape never repeats the lookup and never logs. Absence renders as a +separate availability gauge at 0 with every byte gauge at -1; it never +renders as zero bytes, which an operator would read as an empty heap. + +The second half of this module (issue #226 part 2) is the part that *does* +walk the heap, and it is built so that it can only ever do so on purpose: + +- :class:`HeapCensus` takes a bounded type histogram over + ``gc.get_objects()`` and, when ``tracemalloc`` is tracing, a top-N of + allocation sites, and writes the result to a file. It is off unless + ``PRISM_HEAP_CENSUS=1``, it is triggered only by an operator signal + (``SIGUSR1``), it runs on its own thread (the signal handler only writes + one byte to a pipe), and it is capped on top-N, on output bytes, on walk + seconds, and on files kept. No HTTP route, Stratum message, or metrics + scrape can reach it. +- :class:`MallocTrimmer` calls glibc ``malloc_trim(0)`` and reports the + allocator's before/after readings through :class:`ProcessHeapTelemetry`. + ``malloc_trim`` takes every arena's lock in turn while it walks that + arena's free lists, so a thread allocating in that arena at that moment + waits; on a fragmented multi-gigabyte heap that wait is a stall of tens + to hundreds of milliseconds, not a wedge. It is exposed on ``SIGRTMIN+1`` + when ``PRISM_MALLOC_TRIM_SIGNAL=1`` and as a paced periodic action when + ``PRISM_MALLOC_TRIM_INTERVAL_SECONDS`` is set, never below the floor. +- :func:`evaluate_rss_bound` is the automated soak check: a resident-set + series must stay within a stated multiple of its post-warm-up baseline. + ``python3 -m lab.prism.process_telemetry rss-bound`` runs it. +""" + +from __future__ import annotations + +import argparse +import ctypes +import datetime as _datetime +import gc +import json +import os +import re +import select +import sys +import tempfile +import threading +import time +import tracemalloc +from dataclasses import dataclass, field +from pathlib import Path +from typing import Any, Callable, Iterable, Sequence + +# Closed label set. CPython reports three entries from both gc.get_count() +# and gc.get_stats(); an interpreter reporting fewer renders -1 for the +# missing generation rather than growing or shrinking the series set. Since +# CPython 3.13 the collector is incremental and the third threshold is +# unused, so get_count()'s third entry reads 0 on the shipped interpreter. +# The series is kept regardless: a fixed family set across interpreter +# versions is worth more than dropping a known-zero gauge. +PRISM_GC_GENERATIONS = ("0", "1", "2") + +# Field order of ``struct mallinfo2`` (glibc >= 2.33); every field is size_t. +_MALLINFO2_FIELDS = ( + "arena", + "ordblks", + "smblks", + "hblks", + "hblkhd", + "usmblks", + "fsmblks", + "uordblks", + "fordblks", + "keepcost", +) + + +class MallInfo2(ctypes.Structure): + """``struct mallinfo2`` as returned by glibc's ``mallinfo2()``.""" + + _fields_ = [(name, ctypes.c_size_t) for name in _MALLINFO2_FIELDS] + + +def resolve_mallinfo2() -> Callable[[], MallInfo2]: + """Bind glibc's ``mallinfo2`` through ctypes, or raise. + + Looks the symbol up in the running program's global namespace, which + is where the libc Python itself links resolves. Raises ``AttributeError`` + where the symbol is absent (musl, macOS, glibc before 2.33) and whatever + ``ctypes`` raises where the global handle cannot be opened at all. The + caller treats every exception as "unavailable". + """ + libc = ctypes.CDLL(None) + function = libc.mallinfo2 + function.restype = MallInfo2 + function.argtypes = () + return function + + +@dataclass(frozen=True) +class ProcessHeapSample: + """One scrape's worth of interpreter and allocator readings.""" + + allocated_blocks: int + gc_trigger_count: tuple[int, int, int] + gc_collections: tuple[int, int, int] + gc_collected: tuple[int, int, int] + gc_uncollectable: tuple[int, int, int] + threads: int + malloc_info_available: bool + # The four byte gauges are -1, never 0, while malloc_info_available is + # False: a zero arena on a 17 GB process is a lie an operator would act on. + malloc_arena_bytes: int + malloc_in_use_bytes: int + malloc_free_bytes: int + malloc_mmapped_bytes: int + + +def _per_generation(values: tuple[int, ...] | list[int]) -> tuple[int, int, int]: + """Pin a per-generation reading to the closed three-generation set.""" + padded = [int(value) for value in values[: len(PRISM_GC_GENERATIONS)]] + while len(padded) < len(PRISM_GC_GENERATIONS): + padded.append(-1) + return (padded[0], padded[1], padded[2]) + + +_UNRESOLVED = object() + + +class ProcessHeapTelemetry: + """Collector for :class:`ProcessHeapSample`; one per renderer. + + ``malloc_enabled`` is the ``PRISM_MALLOC_TELEMETRY`` switch: off skips the + ``mallinfo2`` call entirely (it walks glibc's free lists under each arena + lock, which is cheap on a healthy heap but is the one reading here that + is not a plain interpreter counter) and renders the allocator gauges as + unavailable. The interpreter readings have no switch; they are pure + counter loads and are always on. + """ + + def __init__( + self, + *, + malloc_enabled: bool = True, + mallinfo_resolver: Callable[[], Callable[[], MallInfo2]] = resolve_mallinfo2, + ) -> None: + self._malloc_enabled = bool(malloc_enabled) + self._mallinfo_resolver = mallinfo_resolver + # Resolved exactly once: the bound function, or None once the symbol + # has proven absent or broken. Never re-probed and never logged, so + # a platform without mallinfo2 costs nothing per scrape. + self._mallinfo2: object = _UNRESOLVED + self._resolve_lock = threading.Lock() + + @property + def malloc_enabled(self) -> bool: + return self._malloc_enabled + + def _resolved_mallinfo2(self) -> Callable[[], MallInfo2] | None: + if not self._malloc_enabled: + return None + resolved = self._mallinfo2 + if resolved is _UNRESOLVED: + with self._resolve_lock: + resolved = self._mallinfo2 + if resolved is _UNRESOLVED: + try: + resolved = self._mallinfo_resolver() + except Exception: + # Absent symbol, unsupported loader, or anything + # else the C boundary raises: unavailable, for good. + resolved = None + self._mallinfo2 = resolved + return resolved # type: ignore[return-value] + + def sample(self) -> ProcessHeapSample: + stats = gc.get_stats() + arena = in_use = free = mmapped = -1 + available = False + mallinfo2 = self._resolved_mallinfo2() + if mallinfo2 is not None: + try: + info = mallinfo2() + except Exception: + # A symbol that binds but fails at call time is treated the + # same as an absent one from now on: no per-scrape retry. + self._mallinfo2 = None + else: + available = True + arena = int(info.arena) + in_use = int(info.uordblks) + free = int(info.fordblks) + mmapped = int(info.hblkhd) + return ProcessHeapSample( + allocated_blocks=int(sys.getallocatedblocks()), + gc_trigger_count=_per_generation(gc.get_count()), + gc_collections=_per_generation( + [int(entry.get("collections", -1)) for entry in stats] + ), + gc_collected=_per_generation( + [int(entry.get("collected", -1)) for entry in stats] + ), + gc_uncollectable=_per_generation( + [int(entry.get("uncollectable", -1)) for entry in stats] + ), + threads=int(threading.active_count()), + malloc_info_available=available, + malloc_arena_bytes=arena, + malloc_in_use_bytes=in_use, + malloc_free_bytes=free, + malloc_mmapped_bytes=mmapped, + ) + + +# --------------------------------------------------------------------------- +# Resident set and address-space readings (census, trim, and the storm +# instrument; never the scrape, which has its own reader on the coordinator). +# --------------------------------------------------------------------------- + + +def read_resident_memory_bytes(pid: int | str = "self") -> int: + """RSS bytes from ``/proc//statm``, or -1 where that is unavailable.""" + try: + fields = Path(f"/proc/{pid}/statm").read_text(encoding="ascii").split() + return int(fields[1]) * int(os.sysconf("SC_PAGE_SIZE")) + except (OSError, ValueError, IndexError, AttributeError): + return -1 + + +_MIB = 1024 * 1024 +_GIB = 1024 * _MIB + +# The size bands the #226 capture was reported in: 658 of the 943 anonymous +# regions on union-mainnet fell in the 4-64 MiB band, which is the glibc +# per-thread heap (HEAP_MAX_SIZE = 64 MiB) signature. +ANONYMOUS_REGION_BANDS: tuple[tuple[str, int, int | None], ...] = ( + ("under_4mib", 0, 4 * _MIB), + ("4mib_to_64mib", 4 * _MIB, 64 * _MIB), + ("64mib_to_1gib", 64 * _MIB, _GIB), + ("over_1gib", _GIB, None), +) + + +@dataclass(frozen=True) +class AnonymousMapShape: + """Private writable anonymous regions of a process, by size band. + + ``heap_segment_bytes`` is the size of the ``[heap]`` (brk) segment, which + is glibc's main arena; the per-thread arenas are the bare anonymous + regions. Guard pages (``---p``), file mappings, and the stack are not + counted. ``available`` is False where ``/proc//maps`` cannot be read + and every count is then -1. + """ + + available: bool + regions: int + total_bytes: int + heap_segment_bytes: int + band_regions: dict[str, int] = field(default_factory=dict) + band_bytes: dict[str, int] = field(default_factory=dict) + + def as_dict(self) -> dict[str, Any]: + return { + "available": self.available, + "regions": self.regions, + "total_bytes": self.total_bytes, + "heap_segment_bytes": self.heap_segment_bytes, + "band_regions": dict(self.band_regions), + "band_bytes": dict(self.band_bytes), + } + + +def read_anonymous_map_shape(pid: int | str = "self") -> AnonymousMapShape: + """Histogram the private writable anonymous regions of ``/proc//maps``.""" + try: + text = Path(f"/proc/{pid}/maps").read_text(encoding="ascii", errors="replace") + except OSError: + return AnonymousMapShape( + available=False, + regions=-1, + total_bytes=-1, + heap_segment_bytes=-1, + ) + band_regions = {name: 0 for name, _low, _high in ANONYMOUS_REGION_BANDS} + band_bytes = {name: 0 for name, _low, _high in ANONYMOUS_REGION_BANDS} + regions = 0 + total = 0 + heap_segment = 0 + for line in text.splitlines(): + parts = line.split(None, 5) + if len(parts) < 5: + continue + span, perms = parts[0], parts[1] + pathname = parts[5].strip() if len(parts) > 5 else "" + if not perms.startswith("rw") or "p" not in perms: + continue + if pathname and pathname != "[heap]": + continue + try: + start_text, end_text = span.split("-", 1) + size = int(end_text, 16) - int(start_text, 16) + except ValueError: + continue + if pathname == "[heap]": + heap_segment += size + regions += 1 + total += size + for name, low, high in ANONYMOUS_REGION_BANDS: + if size >= low and (high is None or size < high): + band_regions[name] += 1 + band_bytes[name] += size + break + return AnonymousMapShape( + available=True, + regions=regions, + total_bytes=total, + heap_segment_bytes=heap_segment, + band_regions=band_regions, + band_bytes=band_bytes, + ) + + +@dataclass(frozen=True) +class MallocInfoSummary: + """glibc ``malloc_info`` folded across every arena. + + ``arenas`` is the arena count (main plus every per-thread arena ever + created; arenas outlive their threads). The free bytes are split by + where they sit, because that decides what ``malloc_trim`` can do: + ``bin_bytes`` are interior free chunks, which ``malloc_trim`` returns to + the kernel with ``madvise`` in every arena; ``top_bytes`` are the free + space at the end of each arena's heaps, which ``malloc_trim`` returns + only for the main arena -- a per-thread heap's top shrinks only when the + thread that owns the arena frees a chunk of 64 KiB or more while the top + exceeds ``M_TRIM_THRESHOLD``. All -1 when ``available`` is False. + """ + + available: bool + arenas: int + top_bytes: int + bin_bytes: int + fast_bytes: int + system_bytes: int + + def as_dict(self) -> dict[str, Any]: + return { + "available": self.available, + "arenas": self.arenas, + "top_bytes": self.top_bytes, + "bin_bytes": self.bin_bytes, + "fast_bytes": self.fast_bytes, + "system_bytes": self.system_bytes, + } + + +_MALLOC_INFO_UNAVAILABLE = MallocInfoSummary( + available=False, + arenas=-1, + top_bytes=-1, + bin_bytes=-1, + fast_bytes=-1, + system_bytes=-1, +) +_HEAP_ELEMENT = re.compile(r'(.*?)', re.S) +_REST_TOTAL = re.compile(r'') +_FAST_TOTAL = re.compile(r'') +_SYSTEM_CURRENT = re.compile(r'') +_SIZE_ROW = re.compile(r'') +_UNSORTED_ROW = re.compile(r'') + + +def _first_int(pattern: re.Pattern[str], body: str) -> int: + match = pattern.search(body) + return int(match.group(1)) if match else 0 + + +def parse_malloc_info_xml(text: str) -> MallocInfoSummary: + """Fold one ``malloc_info`` document into :class:`MallocInfoSummary`. + + Per ````: glibc prints one ```` row per non-empty size class + -- the fastbin classes first, then the small and large bins -- and the + unsorted list as its own ```` row. ```` is + exactly the sum of the fastbin rows, and ```` is the + bins (small, large, unsorted) plus the top chunk; fastbins are not part + of ``rest``. So the bin bytes are all rows plus unsorted minus the fast + total, and top is ``rest`` minus that. Summing the fastbin rows into the + bins would understate top by the fast total, and top is the number an + operator reads to decide whether ``malloc_trim`` can help. + """ + arenas = top = bins = fast = system = 0 + for heap in _HEAP_ELEMENT.finditer(text): + body = heap.group(1) + arenas += 1 + rest = _first_int(_REST_TOTAL, body) + fast_here = _first_int(_FAST_TOTAL, body) + rows = sum(int(total) for total in _SIZE_ROW.findall(body)) + sum( + int(total) for total in _UNSORTED_ROW.findall(body) + ) + in_bins = max(0, rows - fast_here) + bins += in_bins + top += max(0, rest - in_bins) + fast += fast_here + system += _first_int(_SYSTEM_CURRENT, body) + return MallocInfoSummary( + available=True, + arenas=arenas, + top_bytes=top, + bin_bytes=bins, + fast_bytes=fast, + system_bytes=system, + ) + + +def read_malloc_info_summary() -> MallocInfoSummary: + """Fold glibc ``malloc_info`` across arenas, or the unavailable sentinel. + + ``malloc_info`` writes one ```` element per arena. Like + ``mallinfo2`` it walks every arena under its lock -- and unlike it, it + prints a row per size class per arena -- so it is a census and + storm-instrument reading, never a scrape reading. The XML goes through a + temporary file and the C ``FILE*`` API because that is the only + interface glibc offers for it; :func:`parse_malloc_info_xml` does the + folding. + """ + try: + libc = ctypes.CDLL(None) + malloc_info = libc.malloc_info + fopen = libc.fopen + fclose = libc.fclose + except (OSError, AttributeError): + return _MALLOC_INFO_UNAVAILABLE + malloc_info.restype = ctypes.c_int + malloc_info.argtypes = (ctypes.c_int, ctypes.c_void_p) + fopen.restype = ctypes.c_void_p + fopen.argtypes = (ctypes.c_char_p, ctypes.c_char_p) + fclose.restype = ctypes.c_int + fclose.argtypes = (ctypes.c_void_p,) + try: + with tempfile.NamedTemporaryFile(prefix="prism-malloc-info-", suffix=".xml") as spool: + handle = fopen(spool.name.encode(), b"w") + if not handle: + return _MALLOC_INFO_UNAVAILABLE + try: + if malloc_info(0, handle) != 0: + return _MALLOC_INFO_UNAVAILABLE + finally: + fclose(handle) + text = Path(spool.name).read_text(encoding="ascii", errors="replace") + except (OSError, ValueError): + return _MALLOC_INFO_UNAVAILABLE + return parse_malloc_info_xml(text) + + +def read_malloc_arena_count() -> int: + """Count glibc malloc arenas through ``malloc_info``, or -1 when unavailable.""" + return read_malloc_info_summary().arenas + + +# --------------------------------------------------------------------------- +# Issue #226 part 2: the operator-triggered bounded heap census. +# --------------------------------------------------------------------------- + +HEAP_CENSUS_FORMAT = "qbit-prism-heap-census/v1" +# Files kept per output directory. Each is capped at the configured byte +# ceiling, so the disk footprint is bounded without a further knob. +HEAP_CENSUS_RETAINED_FILES = 32 +HEAP_CENSUS_FILE_PREFIX = "heap-census-" +HEAP_CENSUS_FILE_SUFFIX = ".json" +# Width of the per-process sequence in a report's file name; a million +# censuses at the 60 s floor is almost two years of one process. +HEAP_CENSUS_SEQUENCE_WIDTH = 6 +# The walk checks the clock every this many objects. 1024 objects is a few +# hundred microseconds, so the walk cap is honoured to well under a +# millisecond without paying a clock read per object. +_HEAP_CENSUS_CLOCK_STRIDE = 1024 +_HEAP_CENSUS_TYPE_NAME_LIMIT = 120 +_HEAP_CENSUS_SITE_LIMIT = 240 +# tracemalloc frame depth. One frame groups by file:line, which is what a +# top-N by site needs; deeper traces multiply the standing cost of tracing. +HEAP_CENSUS_TRACEMALLOC_FRAMES = 1 +# The signals. SIGUSR2 belongs to faulthandler (prism_coordinator.main); +# SIGUSR1 is free. The trim action takes the first real-time signal above +# SIGRTMIN so it cannot be confused with the census. +HEAP_CENSUS_SIGNAL_NAME = "SIGUSR1" +MALLOC_TRIM_SIGNAL_NAME = "SIGRTMIN+1" + + +@dataclass(frozen=True) +class HeapCensusConfig: + """The validated census, trim, and tracing switches and bounds. + + Built by ``lab.prism.coordinator_config.env_heap_census_config`` from the + ``PRISM_HEAP_CENSUS*`` and ``PRISM_MALLOC_TRIM*`` environment, which is + also where the defaults and their ceilings live. Every field is + deliberate: there is no default here, so a caller cannot construct an + enabled census without choosing its bounds. + """ + + enabled: bool + output_dir: str + top_n: int + max_bytes: int + max_seconds: float + min_interval_seconds: float + tracemalloc_enabled: bool + malloc_trim_signal_enabled: bool + malloc_trim_interval_seconds: float + + @property + def anything_armed(self) -> bool: + return bool( + self.enabled + or self.malloc_trim_signal_enabled + or self.malloc_trim_interval_seconds > 0 + ) + + +def _truncate(text: str, limit: int) -> str: + return text if len(text) <= limit else text[: limit - 3] + "..." + + +def _type_name(kind: type) -> str: + module = getattr(kind, "__module__", None) or "?" + qualname = getattr(kind, "__qualname__", None) or getattr(kind, "__name__", "?") + return _truncate(f"{module}.{qualname}", _HEAP_CENSUS_TYPE_NAME_LIMIT) + + +def _utc_stamp(now: float) -> str: + return ( + _datetime.datetime.fromtimestamp(now, tz=_datetime.timezone.utc) + .strftime("%Y%m%dT%H%M%SZ") + ) + + +def _log_line(message: str) -> None: + print(message, flush=True) + + +@dataclass(frozen=True) +class HeapCensusOutcome: + """What one census produced: where it went and the bounds it met.""" + + path: str + bytes_written: int + tracked_objects: int + objects_walked: int + walk_truncated: bool + top_n_written: int + seconds: float + tracemalloc_traced: bool + + +class HeapCensus: + """The bounded heap walk, and the file it writes. + + :meth:`take` is synchronous and is the only method that touches the heap. + It is what the census thread calls, what a test calls directly, and it + refuses to do anything at all when the configuration is not enabled -- + the entry point is inert, not merely unregistered, when the switch is + off. + + The walk is ``gc.get_objects()`` followed by one pass over the returned + list taking ``type()`` and ``sys.getsizeof()`` of each object. The + ``gc.get_objects()`` call itself is a single C step that allocates one + pointer per tracked object and holds the GIL for its duration; nothing + can interrupt it, and on a heap of tens of millions of tracked objects + it is a stall of seconds on every thread. The walk cap applies to the + pass that follows: the clock is checked every + ``_HEAP_CENSUS_CLOCK_STRIDE`` objects and the pass stops, marking the + report truncated, when ``max_seconds`` is used up. That is why the + census runs only on an operator's signal, on its own thread, and never + on a request path. + """ + + def __init__( + self, + config: HeapCensusConfig, + *, + telemetry: ProcessHeapTelemetry | None = None, + clock: Callable[[], float] = time.monotonic, + wall_clock: Callable[[], float] = time.time, + log: Callable[[str], None] = _log_line, + malloc_info_reader: Callable[[], MallocInfoSummary] = read_malloc_info_summary, + ) -> None: + self._config = config + self._telemetry = telemetry if telemetry is not None else ProcessHeapTelemetry() + self._clock = clock + self._wall_clock = wall_clock + self._log = log + self._malloc_info_reader = malloc_info_reader + self._lock = threading.Lock() + self._sequence = 0 + self.last_outcome: HeapCensusOutcome | None = None + + @property + def config(self) -> HeapCensusConfig: + return self._config + + @property + def enabled(self) -> bool: + return self._config.enabled + + def take(self, reason: str) -> HeapCensusOutcome | None: + """Walk the heap once, bounded, and write the report; None when disabled. + + Serialized: a second caller while one census is running waits for it + rather than starting another walk. Never raises past this frame -- + a census that fails is logged and returns None, because the census + thread must survive its own diagnostics. + """ + if not self._config.enabled: + return None + with self._lock: + try: + outcome = self._take_locked(reason) + except Exception as exc: # pragma: no cover - defensive; logged + self._log(f"prism heap census failed reason={reason} error={exc!r}") + return None + self.last_outcome = outcome + return outcome + + def _take_locked(self, reason: str) -> HeapCensusOutcome: + config = self._config + started = self._clock() + taken_at = self._wall_clock() + deadline = started + config.max_seconds + before = self._telemetry.sample() + resident_before = read_resident_memory_bytes() + malloc_info = self._malloc_info_reader() + map_shape = read_anonymous_map_shape() + get_objects_started = self._clock() + objects = gc.get_objects() + get_objects_seconds = self._clock() - get_objects_started + tracked = len(objects) + histogram, walked, truncated, distinct_type_objects = self._walk_types( + objects, deadline + ) + del objects + walk_seconds = self._clock() - started + traced = self._tracemalloc_section(config.top_n) + by_count = sorted(histogram.items(), key=lambda item: (-item[1][0], item[0])) + by_bytes = sorted(histogram.items(), key=lambda item: (-item[1][1], item[0])) + report: dict[str, Any] = { + "format": HEAP_CENSUS_FORMAT, + "reason": _truncate(str(reason), 80), + "taken_at": _datetime.datetime.fromtimestamp( + taken_at, tz=_datetime.timezone.utc + ).isoformat(), + "pid": os.getpid(), + "python": sys.version.split()[0], + "platform": sys.platform, + "bounds": { + "top_n": config.top_n, + "max_bytes": config.max_bytes, + "max_seconds": config.max_seconds, + }, + "walk": { + "tracked_objects": tracked, + "objects_walked": walked, + # Type objects seen, and the printed names they merged into + # (fewer when two types share a name). + "distinct_types": distinct_type_objects, + "distinct_type_names": len(histogram), + "truncated": truncated, + "get_objects_seconds": round(get_objects_seconds, 6), + "seconds": round(walk_seconds, 6), + }, + "process": { + "resident_memory_bytes": resident_before, + "allocated_blocks": before.allocated_blocks, + "threads": before.threads, + "gc_trigger_count": list(before.gc_trigger_count), + "gc_collections": list(before.gc_collections), + "gc_collected": list(before.gc_collected), + "gc_uncollectable": list(before.gc_uncollectable), + "malloc_info_available": before.malloc_info_available, + "malloc_arena_bytes": before.malloc_arena_bytes, + "malloc_in_use_bytes": before.malloc_in_use_bytes, + "malloc_free_bytes": before.malloc_free_bytes, + "malloc_mmapped_bytes": before.malloc_mmapped_bytes, + "malloc_arena_count": malloc_info.arenas, + # Where the free bytes sit decides what malloc_trim can do: + # bins it returns everywhere, per-thread heap tops it cannot. + "malloc_free_top_bytes": malloc_info.top_bytes, + "malloc_free_bin_bytes": malloc_info.bin_bytes, + "malloc_free_fast_bytes": malloc_info.fast_bytes, + "anonymous_map": map_shape.as_dict(), + }, + "tracemalloc": traced, + } + payload, top_n_written = self._fit_payload(report, by_count, by_bytes, config) + path = self._write(payload, taken_at) + self._retain_files(Path(config.output_dir)) + outcome = HeapCensusOutcome( + path=str(path), + bytes_written=len(payload), + tracked_objects=tracked, + objects_walked=walked, + walk_truncated=truncated, + top_n_written=top_n_written, + seconds=self._clock() - started, + tracemalloc_traced=bool(traced.get("tracing")), + ) + self._log( + "prism heap census written " + f"reason={report['reason']} path={outcome.path} bytes={outcome.bytes_written} " + f"tracked_objects={tracked} walked={walked} truncated={truncated} " + f"top_n={top_n_written} seconds={outcome.seconds:.3f} " + f"tracemalloc={'on' if outcome.tracemalloc_traced else 'off'}" + ) + return outcome + + def _walk_types( + self, + objects: list[Any], + deadline: float, + ) -> tuple[dict[str, list[int]], int, bool, int]: + histogram: dict[type, list[int]] = {} + getsizeof = sys.getsizeof + clock = self._clock + walked = 0 + truncated = False + for index, obj in enumerate(objects): + if index % _HEAP_CENSUS_CLOCK_STRIDE == 0 and index and clock() > deadline: + truncated = True + break + kind = type(obj) + entry = histogram.get(kind) + if entry is None: + entry = histogram[kind] = [0, 0] + entry[0] += 1 + try: + entry[1] += getsizeof(obj) + except Exception: + # A __sizeof__ that raises is not the census's problem; the + # count still stands. + pass + walked += 1 + # Two distinct type objects can share a printed name (a class defined + # twice, or two long names that truncate alike). Merge them rather + # than let one overwrite the other: a census that drops a retained + # type is worse than no census, and this is the instrument for + # finding the leak. + named: dict[str, list[int]] = {} + for kind, entry in histogram.items(): + name = _type_name(kind) + merged = named.get(name) + if merged is None: + named[name] = [entry[0], entry[1]] + else: + merged[0] += entry[0] + merged[1] += entry[1] + return named, walked, truncated, len(histogram) + + def _tracemalloc_section(self, top_n: int) -> dict[str, Any]: + if not tracemalloc.is_tracing(): + return {"tracing": False} + traced_now, traced_peak = tracemalloc.get_traced_memory() + snapshot = tracemalloc.take_snapshot().filter_traces( + (tracemalloc.Filter(False, tracemalloc.__file__),) + ) + by_line = [ + { + "site": _truncate(str(stat.traceback), _HEAP_CENSUS_SITE_LIMIT), + "bytes": int(stat.size), + "count": int(stat.count), + } + for stat in snapshot.statistics("lineno")[:top_n] + ] + by_file = [ + { + "site": _truncate(str(stat.traceback), _HEAP_CENSUS_SITE_LIMIT), + "bytes": int(stat.size), + "count": int(stat.count), + } + for stat in snapshot.statistics("filename")[:top_n] + ] + del snapshot + return { + "tracing": True, + "frames": tracemalloc.get_traceback_limit(), + "traced_bytes": int(traced_now), + "traced_peak_bytes": int(traced_peak), + "by_line": by_line, + "by_file": by_file, + } + + @staticmethod + def _fit_payload( + report: dict[str, Any], + by_count: list[tuple[str, list[int]]], + by_bytes: list[tuple[str, list[int]]], + config: HeapCensusConfig, + ) -> tuple[bytes, int]: + """Serialize under ``max_bytes`` by shrinking top-N, never by cutting JSON.""" + top_n = config.top_n + while True: + report["types_by_count"] = [ + {"type": name, "count": entry[0], "shallow_bytes": entry[1]} + for name, entry in by_count[:top_n] + ] + report["types_by_bytes"] = [ + {"type": name, "count": entry[0], "shallow_bytes": entry[1]} + for name, entry in by_bytes[:top_n] + ] + traced = report["tracemalloc"] + if traced.get("tracing"): + traced["by_line"] = traced["by_line"][:top_n] + traced["by_file"] = traced["by_file"][:top_n] + report["top_n_written"] = top_n + report["fit_to_max_bytes"] = top_n < config.top_n + payload = json.dumps(report, indent=1, sort_keys=True).encode("utf-8") + if len(payload) <= config.max_bytes or top_n == 0: + return payload, top_n + top_n //= 2 + + def _write(self, payload: bytes, taken_at: float) -> Path: + directory = Path(self._config.output_dir) + directory.mkdir(parents=True, exist_ok=True) + # heap-census---.json. The sequence is this + # process's census count, zero-padded, so plain name order is the + # order the reports were written: newest-first retention below never + # consults a filesystem timestamp, whose granularity can tie. + stem = f"{HEAP_CENSUS_FILE_PREFIX}{_utc_stamp(taken_at)}-{os.getpid()}" + while True: + self._sequence += 1 + path = directory / f"{stem}-{self._sequence:0{HEAP_CENSUS_SEQUENCE_WIDTH}d}{HEAP_CENSUS_FILE_SUFFIX}" + if not path.exists(): + break + temporary = path.with_suffix(".tmp") + temporary.write_bytes(payload) + os.replace(temporary, path) + return path + + @staticmethod + def _retain_files(directory: Path) -> None: + """Keep the newest ``HEAP_CENSUS_RETAINED_FILES`` census files. + + Newest by name: the stamp, then the pid, then the zero-padded + per-process sequence, so the order does not depend on the + filesystem's timestamp granularity. + """ + try: + names = sorted( + entry.name + for entry in directory.iterdir() + if entry.name.startswith(HEAP_CENSUS_FILE_PREFIX) + and entry.name.endswith(HEAP_CENSUS_FILE_SUFFIX) + ) + except OSError: + return + for name in names[: max(0, len(names) - HEAP_CENSUS_RETAINED_FILES)]: + try: + (directory / name).unlink() + except OSError: + pass + + +# --------------------------------------------------------------------------- +# malloc_trim, bounded and reported through the always-on telemetry. +# --------------------------------------------------------------------------- + + +def resolve_malloc_trim() -> Callable[[int], int]: + """Bind glibc's ``malloc_trim`` through ctypes, or raise where absent.""" + libc = ctypes.CDLL(None) + function = libc.malloc_trim + function.restype = ctypes.c_int + function.argtypes = (ctypes.c_size_t,) + return function + + +@dataclass(frozen=True) +class MallocTrimResult: + """One ``malloc_trim(0)`` call and the allocator readings around it. + + The deltas are the mechanism the trim is judged by: ``free`` bytes and + RSS should fall while ``in_use`` bytes are unchanged (a trim frees + nothing the program still holds). ``released`` is glibc's own return + value, 1 when any memory went back to the kernel. + """ + + reason: str + released: bool + seconds: float + resident_before: int + resident_after: int + before: ProcessHeapSample + after: ProcessHeapSample + + @property + def resident_delta(self) -> int: + if self.resident_before < 0 or self.resident_after < 0: + return 0 + return self.resident_after - self.resident_before + + @property + def arena_delta(self) -> int: + return self.after.malloc_arena_bytes - self.before.malloc_arena_bytes + + @property + def in_use_delta(self) -> int: + return self.after.malloc_in_use_bytes - self.before.malloc_in_use_bytes + + @property + def free_delta(self) -> int: + return self.after.malloc_free_bytes - self.before.malloc_free_bytes + + def as_dict(self) -> dict[str, Any]: + return { + "reason": self.reason, + "released": self.released, + "seconds": round(self.seconds, 6), + "resident_before": self.resident_before, + "resident_after": self.resident_after, + "resident_delta": self.resident_delta, + "malloc_info_available": self.before.malloc_info_available + and self.after.malloc_info_available, + "arena_before": self.before.malloc_arena_bytes, + "arena_after": self.after.malloc_arena_bytes, + "arena_delta": self.arena_delta, + "in_use_before": self.before.malloc_in_use_bytes, + "in_use_after": self.after.malloc_in_use_bytes, + "in_use_delta": self.in_use_delta, + "free_before": self.before.malloc_free_bytes, + "free_after": self.after.malloc_free_bytes, + "free_delta": self.free_delta, + } + + +class MallocTrimmer: + """Call ``malloc_trim(0)`` once per request and report its effect. + + Serialized by a lock so two trims never overlap. The symbol is resolved + once and its absence remembered, exactly like ``mallinfo2`` in the + collector: on musl or macOS :meth:`trim_once` returns None every time + and never logs more than once. + """ + + def __init__( + self, + telemetry: ProcessHeapTelemetry | None = None, + *, + resolver: Callable[[], Callable[[int], int]] = resolve_malloc_trim, + clock: Callable[[], float] = time.monotonic, + rss_reader: Callable[[], int] = read_resident_memory_bytes, + log: Callable[[str], None] = _log_line, + ) -> None: + self._telemetry = telemetry if telemetry is not None else ProcessHeapTelemetry() + self._resolver = resolver + self._clock = clock + self._rss_reader = rss_reader + self._log = log + self._lock = threading.Lock() + self._malloc_trim: object = _UNRESOLVED + self.last_result: MallocTrimResult | None = None + self.calls = 0 + + def _resolved(self) -> Callable[[int], int] | None: + resolved = self._malloc_trim + if resolved is _UNRESOLVED: + try: + resolved = self._resolver() + except Exception: + resolved = None + self._log("prism malloc_trim unavailable: glibc malloc_trim symbol not bound") + self._malloc_trim = resolved + return resolved # type: ignore[return-value] + + def trim_once(self, reason: str) -> MallocTrimResult | None: + with self._lock: + malloc_trim = self._resolved() + if malloc_trim is None: + return None + before = self._telemetry.sample() + resident_before = self._rss_reader() + started = self._clock() + try: + released = int(malloc_trim(0)) + except Exception as exc: + self._malloc_trim = None + self._log(f"prism malloc_trim failed reason={reason} error={exc!r}") + return None + seconds = self._clock() - started + after = self._telemetry.sample() + resident_after = self._rss_reader() + result = MallocTrimResult( + reason=_truncate(str(reason), 80), + released=bool(released), + seconds=seconds, + resident_before=resident_before, + resident_after=resident_after, + before=before, + after=after, + ) + self.calls += 1 + self.last_result = result + self._log( + "prism malloc_trim " + + " ".join(f"{key}={value}" for key, value in result.as_dict().items()) + ) + return result + + +# --------------------------------------------------------------------------- +# The service: one thread, one pipe, two signals, one periodic action. +# --------------------------------------------------------------------------- + +_CENSUS_BYTE = b"c" +_TRIM_BYTE = b"t" +_STOP_BYTE = b"q" + + +class HeapCensusService: + """Own the census thread and dispatch its requests with rate limits. + + A signal handler runs as Python code on the main thread between two + bytecodes, and a second signal can interrupt it at any bytecode, so the + handler must not take a lock it might already hold: ``Event.set`` would + do exactly that. The handlers therefore only store the reason string + (one attribute assignment) and write one byte to a non-blocking pipe, + which is the same self-pipe shape ``signal.set_wakeup_fd`` uses. A full + pipe means a request is already queued, and the byte is dropped: the + requests coalesce rather than pile up. + + The worker reads the pipe, applies ``min_interval_seconds`` per action + kind, runs the census or the trim, and runs the periodic trim when its + interval is set. Nothing here is reachable from the metrics renderer, + the audit HTTP handler, the public read service, or the Stratum + dispatcher; the coordinator only calls :func:`install_heap_census` from + ``main``. + """ + + THREAD_NAME = "prism-heap-census" + + def __init__( + self, + config: HeapCensusConfig, + *, + census: HeapCensus | None = None, + trimmer: MallocTrimmer | None = None, + telemetry: ProcessHeapTelemetry | None = None, + clock: Callable[[], float] = time.monotonic, + log: Callable[[str], None] = _log_line, + ) -> None: + self._config = config + telemetry = telemetry if telemetry is not None else ProcessHeapTelemetry() + self.census = census if census is not None else HeapCensus( + config, telemetry=telemetry, clock=clock, log=log + ) + self.trimmer = trimmer if trimmer is not None else MallocTrimmer( + telemetry, clock=clock, log=log + ) + self._clock = clock + self._log = log + self._read_fd, self._write_fd = os.pipe() + os.set_blocking(self._write_fd, False) + os.set_blocking(self._read_fd, False) + self._census_reason = "unset" + self._trim_reason = "unset" + self._last_census_monotonic: float | None = None + self._last_trim_monotonic: float | None = None + self._next_periodic_trim: float | None = ( + clock() + config.malloc_trim_interval_seconds + if config.malloc_trim_interval_seconds > 0 + else None + ) + self._thread: threading.Thread | None = None + self._closed = False + # (signal module, signum, previous handler) for every handler this + # service armed, so close() can disarm them before the pipe goes. + self._armed: list[tuple[Any, int, Any]] = [] + self.suppressed = {"census": 0, "trim": 0} + + @property + def config(self) -> HeapCensusConfig: + return self._config + + # -- signal registration ------------------------------------------------ + + def arm_signal(self, signal_module: Any, signum: int, handler: Any) -> None: + """Register ``handler`` and remember what it replaced.""" + getsignal = getattr(signal_module, "getsignal", None) + previous = getsignal(signum) if callable(getsignal) else None + signal_module.signal(signum, handler) + self._armed.append((signal_module, signum, previous)) + + def disarm_signals(self) -> None: + """Restore every handler :meth:`arm_signal` replaced.""" + while self._armed: + signal_module, signum, previous = self._armed.pop() + if previous is None: + previous = getattr(signal_module, "SIG_DFL", None) + if previous is None: + continue + try: + signal_module.signal(signum, previous) + except (OSError, ValueError, TypeError): + pass + + # -- signal-safe entry points ------------------------------------------- + + def _wake(self, byte: bytes) -> bool: + # After close() the descriptor numbers may belong to something else; + # a late signal must not write into a stranger's pipe. + if self._closed: + return False + try: + os.write(self._write_fd, byte) + except (BlockingIOError, InterruptedError, OSError): + return False + return True + + def request_census(self, reason: str) -> bool: + """Queue a census; safe to call from a signal handler.""" + if not self._config.enabled: + return False + self._census_reason = reason + return self._wake(_CENSUS_BYTE) + + def request_trim(self, reason: str) -> bool: + """Queue a trim; safe to call from a signal handler.""" + self._trim_reason = reason + return self._wake(_TRIM_BYTE) + + # -- dispatch with rate limits ------------------------------------------ + + def dispatch(self, kind: str, reason: str) -> object | None: + """Run one action now unless its rate limit suppresses it. + + Returns the census outcome, the trim result, or None when suppressed + or unavailable. The periodic trim goes through the same gate. + """ + now = self._clock() + floor = self._config.min_interval_seconds + if kind == "census": + if not self._config.enabled: + return None + last = self._last_census_monotonic + if last is not None and now - last < floor: + self.suppressed["census"] += 1 + self._log( + f"prism heap census suppressed reason={reason} " + f"min_interval_seconds={floor:g} since_last={now - last:.1f}" + ) + return None + outcome = self.census.take(reason) + self._last_census_monotonic = self._clock() + return outcome + if kind == "trim": + last = self._last_trim_monotonic + if last is not None and now - last < floor: + self.suppressed["trim"] += 1 + self._log( + f"prism malloc_trim suppressed reason={reason} " + f"min_interval_seconds={floor:g} since_last={now - last:.1f}" + ) + return None + result = self.trimmer.trim_once(reason) + self._last_trim_monotonic = self._clock() + return result + raise ValueError(f"unknown census action {kind!r}") + + def periodic_trim_due(self) -> bool: + return ( + self._next_periodic_trim is not None + and self._clock() >= self._next_periodic_trim + ) + + def _timeout(self) -> float | None: + if self._next_periodic_trim is None: + return None + return max(0.0, self._next_periodic_trim - self._clock()) + + def run_once(self, timeout: float | None) -> bool: + """One worker iteration: wait, then serve what arrived. False on stop.""" + try: + ready, _w, _x = select.select([self._read_fd], [], [], timeout) + except (OSError, ValueError): + return False + if ready: + try: + pending = os.read(self._read_fd, 64) + except (BlockingIOError, InterruptedError): + pending = b"" + except OSError: + return False + if _STOP_BYTE in pending: + return False + if _CENSUS_BYTE in pending: + self.dispatch("census", self._census_reason) + if _TRIM_BYTE in pending: + self.dispatch("trim", self._trim_reason) + if self.periodic_trim_due(): + self.dispatch("trim", "periodic") + self._next_periodic_trim = ( + self._clock() + self._config.malloc_trim_interval_seconds + ) + return True + + def run_forever(self) -> None: + while not self._closed: + try: + if not self.run_once(self._timeout()): + return + except Exception as exc: # pragma: no cover - defensive; logged + self._log(f"prism heap census thread error={exc!r}") + + def start(self) -> threading.Thread: + if self._thread is None: + self._thread = threading.Thread( + target=self.run_forever, + name=self.THREAD_NAME, + daemon=True, + ) + self._thread.start() + return self._thread + + def close(self) -> None: + """Disarm the signals, stop the worker, release the pipe. + + Order matters: handlers first, so no signal can reach ``_wake`` once + the descriptors are gone; then the stop byte while the pipe is still + open; then ``_closed`` so any handler already past the disarm is a + no-op; then the descriptors. Tests call this; in the coordinator the + daemon thread dies with the process. + """ + if self._closed: + return + self.disarm_signals() + self._wake(_STOP_BYTE) + self._closed = True + thread = self._thread + if thread is not None and thread is not threading.current_thread(): + thread.join(timeout=5.0) + for descriptor in (self._read_fd, self._write_fd): + try: + os.close(descriptor) + except OSError: + pass + + +def heap_census_signal(signal_module: Any) -> int | None: + return getattr(signal_module, "SIGUSR1", None) + + +def malloc_trim_signal(signal_module: Any) -> int | None: + base = getattr(signal_module, "SIGRTMIN", None) + if base is None: + return None + return int(base) + 1 + + +def install_heap_census( + config: HeapCensusConfig, + *, + signal_module: Any = None, + log: Callable[[str], None] = _log_line, + telemetry: ProcessHeapTelemetry | None = None, + start_thread: bool = True, +) -> HeapCensusService | None: + """Register the census and trim signals and start the worker; None when off. + + The one call ``prism_coordinator.main`` makes. With every switch off + nothing is registered, no thread starts, tracemalloc is not started, + and the return value is None: the walk is unreachable, not merely idle. + ``signal_module`` is injectable so a test can prove which handlers were + registered without sending real signals to itself. + """ + if not config.anything_armed: + return None + if signal_module is None: + import signal as signal_module # noqa: PLC0415 - registration only + if config.enabled and config.tracemalloc_enabled and not tracemalloc.is_tracing(): + tracemalloc.start(HEAP_CENSUS_TRACEMALLOC_FRAMES) + service = HeapCensusService(config, telemetry=telemetry, log=log) + armed: list[str] = [] + if config.enabled: + census_signal = heap_census_signal(signal_module) + if census_signal is None: + log("prism heap census: SIGUSR1 unavailable on this platform; census not armed") + else: + + def _on_census_signal(signum: int, _frame: Any) -> None: + service.request_census(f"signal:{signum}") + + service.arm_signal(signal_module, census_signal, _on_census_signal) + armed.append(f"census={HEAP_CENSUS_SIGNAL_NAME}") + if config.malloc_trim_signal_enabled: + trim_signal = malloc_trim_signal(signal_module) + if trim_signal is None: + log("prism malloc_trim: SIGRTMIN unavailable on this platform; trim signal not armed") + else: + + def _on_trim_signal(signum: int, _frame: Any) -> None: + service.request_trim(f"signal:{signum}") + + service.arm_signal(signal_module, trim_signal, _on_trim_signal) + armed.append(f"trim={MALLOC_TRIM_SIGNAL_NAME}") + if config.malloc_trim_interval_seconds > 0: + armed.append(f"periodic_trim={config.malloc_trim_interval_seconds:g}s") + if start_thread: + service.start() + log( + "prism heap census armed " + + " ".join(armed) + + f" dir={config.output_dir} top_n={config.top_n} max_bytes={config.max_bytes}" + f" max_seconds={config.max_seconds:g} min_interval_seconds={config.min_interval_seconds:g}" + f" tracemalloc={'on' if config.enabled and config.tracemalloc_enabled else 'off'}" + ) + return service + + +# --------------------------------------------------------------------------- +# The RSS bound: the automated soak check. +# --------------------------------------------------------------------------- + +# The stated bound. The baseline is the largest resident set seen during the +# warm-up (one hour: long enough for the payout window to be materialized +# and the first full rescan to run), and every later sample must stay under +# twice it. The #226 slope (390 MB/h from a 125-145 MiB start) breaches this +# inside the second hour; #185's post-storm drain (125 -> 613 MiB) would +# breach it only if the storm fell outside the warm-up and did not drain. +RSS_BOUND_DEFAULT_MULTIPLE = 2.0 +RSS_BOUND_DEFAULT_WARMUP_SECONDS = 3600.0 +# A 24 h soak with an hour of tolerance for a late first sample. +RSS_BOUND_DEFAULT_MIN_SPAN_SECONDS = 23 * 3600.0 + + +@dataclass(frozen=True) +class RssBoundVerdict: + passed: bool + reasons: tuple[str, ...] + multiple: float + warmup_seconds: float + samples: int + span_seconds: float + baseline_bytes: int + bound_bytes: int + peak_bytes: int + peak_at_seconds: float + peak_ratio: float + first_breach_at_seconds: float | None + slope_bytes_per_hour: float | None + + def as_dict(self) -> dict[str, Any]: + return { + "passed": self.passed, + "reasons": list(self.reasons), + "multiple": self.multiple, + "warmup_seconds": self.warmup_seconds, + "samples": self.samples, + "span_seconds": self.span_seconds, + "baseline_bytes": self.baseline_bytes, + "bound_bytes": self.bound_bytes, + "peak_bytes": self.peak_bytes, + "peak_at_seconds": self.peak_at_seconds, + "peak_ratio": self.peak_ratio, + "first_breach_at_seconds": self.first_breach_at_seconds, + "slope_bytes_per_hour": self.slope_bytes_per_hour, + } + + +def _least_squares_slope(points: Sequence[tuple[float, float]]) -> float | None: + if len(points) < 2: + return None + n = float(len(points)) + mean_x = sum(x for x, _y in points) / n + mean_y = sum(y for _x, y in points) / n + denominator = sum((x - mean_x) ** 2 for x, _y in points) + if denominator == 0: + return None + return sum((x - mean_x) * (y - mean_y) for x, y in points) / denominator + + +def evaluate_rss_bound( + samples: Iterable[tuple[float, int]], + *, + warmup_seconds: float = RSS_BOUND_DEFAULT_WARMUP_SECONDS, + multiple: float = RSS_BOUND_DEFAULT_MULTIPLE, + min_span_seconds: float = 0.0, + max_slope_bytes_per_hour: float | None = None, +) -> RssBoundVerdict: + """Judge a ``(seconds, rss_bytes)`` series against the stated multiple. + + Times may be absolute or relative; they are re-based on the first + sample. Samples with a negative RSS (the -1 "unavailable" sentinel) are + ignored. The verdict fails when any post-warm-up sample exceeds + ``multiple`` times the warm-up peak, when the series is too short to + contain both a warm-up and a post-warm-up sample, when it spans less + than ``min_span_seconds``, or when ``max_slope_bytes_per_hour`` is given + and the least-squares slope over the post-warm-up samples exceeds it. + """ + if multiple <= 1.0: + raise ValueError("multiple must be greater than 1.0") + if warmup_seconds < 0: + raise ValueError("warmup_seconds must be non-negative") + ordered = sorted( + (float(seconds), int(rss)) for seconds, rss in samples if int(rss) >= 0 + ) + reasons: list[str] = [] + if not ordered: + return RssBoundVerdict( + passed=False, + reasons=("no usable samples",), + multiple=multiple, + warmup_seconds=warmup_seconds, + samples=0, + span_seconds=0.0, + baseline_bytes=-1, + bound_bytes=-1, + peak_bytes=-1, + peak_at_seconds=0.0, + peak_ratio=0.0, + first_breach_at_seconds=None, + slope_bytes_per_hour=None, + ) + origin = ordered[0][0] + rebased = [(seconds - origin, rss) for seconds, rss in ordered] + span = rebased[-1][0] + warmup = [(t, r) for t, r in rebased if t <= warmup_seconds] + steady = [(t, r) for t, r in rebased if t > warmup_seconds] + if not warmup: + reasons.append("no sample inside the warm-up window") + if not steady: + reasons.append("no sample after the warm-up window") + if span < min_span_seconds: + reasons.append( + f"series spans {span:.0f}s, below the required {min_span_seconds:.0f}s" + ) + baseline = max((r for _t, r in warmup), default=-1) + bound = int(baseline * multiple) if baseline >= 0 else -1 + peak_at, peak = max(rebased, key=lambda item: (item[1], -item[0])) + peak_ratio = (peak / baseline) if baseline > 0 else 0.0 + first_breach: float | None = None + if baseline >= 0: + for t, r in steady: + if r > bound: + first_breach = t + break + if first_breach is not None: + reasons.append( + f"resident set exceeded {multiple:g}x the warm-up baseline " + f"({baseline} bytes) at {first_breach:.0f}s" + ) + slope_per_second = _least_squares_slope([(t, float(r)) for t, r in steady]) + slope_per_hour = slope_per_second * 3600.0 if slope_per_second is not None else None + if ( + max_slope_bytes_per_hour is not None + and slope_per_hour is not None + and slope_per_hour > max_slope_bytes_per_hour + ): + reasons.append( + f"post-warm-up slope {slope_per_hour:.0f} bytes/h exceeds " + f"{max_slope_bytes_per_hour:.0f} bytes/h" + ) + return RssBoundVerdict( + passed=not reasons, + reasons=tuple(reasons), + multiple=multiple, + warmup_seconds=warmup_seconds, + samples=len(rebased), + span_seconds=span, + baseline_bytes=baseline, + bound_bytes=bound, + peak_bytes=peak, + peak_at_seconds=peak_at, + peak_ratio=peak_ratio, + first_breach_at_seconds=first_breach, + slope_bytes_per_hour=slope_per_hour, + ) + + +def read_rss_samples(path: Path) -> list[tuple[float, int]]: + """Parse ``seconds,rss_bytes`` lines; blank lines and ``#`` comments are skipped.""" + samples: list[tuple[float, int]] = [] + for number, raw in enumerate(path.read_text(encoding="utf-8").splitlines(), 1): + line = raw.split("#", 1)[0].strip() + if not line: + continue + parts = [part.strip() for part in line.replace("\t", ",").split(",")] + if len(parts) < 2: + raise ValueError(f"{path}:{number}: expected 'seconds,rss_bytes'") + try: + samples.append((float(parts[0]), int(float(parts[1])))) + except ValueError as exc: + raise ValueError(f"{path}:{number}: {exc}") from exc + return samples + + +def _build_cli() -> argparse.ArgumentParser: + parser = argparse.ArgumentParser( + prog="python3 -m lab.prism.process_telemetry", + description="Issue #226 soak tooling: sample a resident set, judge a series.", + ) + commands = parser.add_subparsers(dest="command", required=True) + sample = commands.add_parser( + "rss-sample", + help="print one 'epoch_seconds,rss_bytes' line for a pid", + ) + sample.add_argument("--pid", default="self", help="process id (default: this process)") + bound = commands.add_parser( + "rss-bound", + help="judge a 'seconds,rss_bytes' series against the stated multiple", + ) + bound.add_argument("--samples", type=Path, required=True, help="CSV of seconds,rss_bytes") + bound.add_argument( + "--warmup-seconds", + type=float, + default=RSS_BOUND_DEFAULT_WARMUP_SECONDS, + help=f"baseline window (default {RSS_BOUND_DEFAULT_WARMUP_SECONDS:g})", + ) + bound.add_argument( + "--multiple", + type=float, + default=RSS_BOUND_DEFAULT_MULTIPLE, + help=f"allowed multiple of the warm-up peak (default {RSS_BOUND_DEFAULT_MULTIPLE:g})", + ) + bound.add_argument( + "--min-span-seconds", + type=float, + default=RSS_BOUND_DEFAULT_MIN_SPAN_SECONDS, + help=f"refuse a series shorter than this (default {RSS_BOUND_DEFAULT_MIN_SPAN_SECONDS:g})", + ) + bound.add_argument( + "--max-slope-bytes-per-hour", + type=float, + default=None, + help="also fail when the post-warm-up least-squares slope exceeds this", + ) + return parser + + +def main(argv: Sequence[str] | None = None) -> int: + args = _build_cli().parse_args(argv) + if args.command == "rss-sample": + rss = read_resident_memory_bytes(args.pid) + print(f"{time.time():.0f},{rss}") + return 0 if rss >= 0 else 1 + try: + samples = read_rss_samples(args.samples) + except (OSError, ValueError) as exc: + print(f"rss-bound: {exc}", file=sys.stderr) + return 2 + try: + verdict = evaluate_rss_bound( + samples, + warmup_seconds=args.warmup_seconds, + multiple=args.multiple, + min_span_seconds=args.min_span_seconds, + max_slope_bytes_per_hour=args.max_slope_bytes_per_hour, + ) + except ValueError as exc: + print(f"rss-bound: {exc}", file=sys.stderr) + return 2 + print(json.dumps(verdict.as_dict(), indent=2, sort_keys=True)) + return 0 if verdict.passed else 1 + + +__all__ = [ + "ANONYMOUS_REGION_BANDS", + "AnonymousMapShape", + "HEAP_CENSUS_FORMAT", + "HEAP_CENSUS_RETAINED_FILES", + "HEAP_CENSUS_SIGNAL_NAME", + "HEAP_CENSUS_TRACEMALLOC_FRAMES", + "HeapCensus", + "HeapCensusConfig", + "HeapCensusOutcome", + "HeapCensusService", + "MALLOC_TRIM_SIGNAL_NAME", + "MallInfo2", + "MallocInfoSummary", + "MallocTrimResult", + "MallocTrimmer", + "PRISM_GC_GENERATIONS", + "ProcessHeapSample", + "ProcessHeapTelemetry", + "RSS_BOUND_DEFAULT_MIN_SPAN_SECONDS", + "RSS_BOUND_DEFAULT_MULTIPLE", + "RSS_BOUND_DEFAULT_WARMUP_SECONDS", + "RssBoundVerdict", + "evaluate_rss_bound", + "heap_census_signal", + "install_heap_census", + "main", + "malloc_trim_signal", + "parse_malloc_info_xml", + "read_anonymous_map_shape", + "read_malloc_arena_count", + "read_malloc_info_summary", + "read_resident_memory_bytes", + "read_rss_samples", + "resolve_malloc_trim", + "resolve_mallinfo2", +] + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/lab/prism/progress_health.py b/lab/prism/progress_health.py new file mode 100644 index 000000000..a20ab4a70 --- /dev/null +++ b/lab/prism/progress_health.py @@ -0,0 +1,698 @@ +#!/usr/bin/env python3 +"""Monotonic mining-progress readiness state for the PRISM coordinator.""" + +from __future__ import annotations + +from dataclasses import dataclass +from types import MappingProxyType +import threading +import time +from typing import Callable, Mapping + + +PROGRESS_HEALTH_REASONS = ( + "tip_poll_stale", + "refresh_pending_too_long", + "current_generation_not_published", + "current_generation_not_delivered", + "bundle_build_stuck", +) + + +@dataclass(frozen=True) +class WorkGeneration: + template_generation: int + template_fingerprint: str | None + payout_generation: int + + +@dataclass(frozen=True) +class DeliveryProof: + connection_id: int + delivered_work: WorkGeneration + collection_only: bool + delivered_monotonic: float + + +@dataclass(frozen=True) +class EligibilitySnapshot: + eligible_connection_ids: tuple[int, ...] + delivery_proofs: tuple[DeliveryProof, ...] + ready_mode_required: bool + + +# Adapter signature used by the coordinator to supply live client facts: +# ``(template_fingerprint, payout_generation) -> (eligible, requiring_refresh)``. +EligibleClientCounts = Callable[[str | None, int], tuple[int, int]] + + +@dataclass(frozen=True) +class ProgressHealthConfig: + pending_refresh_deadline_seconds: float + tip_poll_deadline_seconds: float + bundle_build_deadline_seconds: float + + +@dataclass(frozen=True) +class ProgressHealthSnapshot: + ok: bool + reason: str | None + reasons: tuple[str, ...] + pending_refresh: bool + pending_refresh_age_seconds: float | None + tip_poll_age_seconds: float + tip_refresh_in_progress: bool + tip_refresh_progress_age_seconds: float | None + current_template_generation: int + published_template_generation: int + current_payout_generation: int + published_payout_generation: int + last_valid_delivery_age_seconds: float | None + eligible_client_count: int + eligible_clients_requiring_refresh: int + bundle_build_oldest_age_seconds: float + + def as_mapping(self) -> dict[str, object]: + """Return the existing mutable HTTP/test compatibility representation.""" + + return { + "ok": self.ok, + "reason": self.reason, + "reasons": list(self.reasons), + "pending_refresh": self.pending_refresh, + "pending_refresh_age_seconds": self.pending_refresh_age_seconds, + "tip_poll_age_seconds": self.tip_poll_age_seconds, + "tip_refresh_in_progress": self.tip_refresh_in_progress, + "tip_refresh_progress_age_seconds": self.tip_refresh_progress_age_seconds, + "current_template_generation": self.current_template_generation, + "published_template_generation": self.published_template_generation, + "current_payout_generation": self.current_payout_generation, + "published_payout_generation": self.published_payout_generation, + "last_valid_delivery_age_seconds": self.last_valid_delivery_age_seconds, + "eligible_client_count": self.eligible_client_count, + "eligible_clients_requiring_refresh": ( + self.eligible_clients_requiring_refresh + ), + "bundle_build_oldest_age_seconds": self.bundle_build_oldest_age_seconds, + } + + +@dataclass(frozen=True) +class _ProgressStateCopy: + current_template_generation: int + current_template_fingerprint: str | None + current_payout_generation: int + published_template_generation: int + published_template_fingerprint: str | None + published_payout_generation: int + has_published_work: bool + last_tip_poll_monotonic: float | None + last_delivery_template_fingerprint: str | None + last_delivery_payout_generation: int + last_delivery_monotonic: float | None + pending_since_monotonic: float | None + publication_divergence_since_monotonic: float | None + refresh_signal_pending: bool + active_refresh_count: int + last_refresh_activity_monotonic: float | None + bundle_build_starts: tuple[float, ...] + + +class RefreshActivityToken: + """One idempotent, context-managed refresh activity lifetime.""" + + def __init__(self, service: ProgressHealthService) -> None: + self._service = service + self._state_lock = threading.Lock() + self._finished = False + + def note_activity(self, observed_monotonic: float | None = None) -> None: + with self._state_lock: + if self._finished: + return + self._service.note_refresh_activity(observed_monotonic) + + def finish(self) -> None: + with self._state_lock: + if self._finished: + return + self._finished = True + self._service.refresh_finished() + + cancel = finish + + def __enter__(self) -> RefreshActivityToken: + return self + + def __exit__(self, *_args: object) -> None: + self.finish() + + +class BundleBuildToken: + """One idempotent, context-managed bundle construction lifetime.""" + + def __init__(self, service: ProgressHealthService, token_id: int) -> None: + self._service = service + self._token_id = token_id + self._state_lock = threading.Lock() + self._finished = False + + def finish(self) -> None: + with self._state_lock: + if self._finished: + return + self._finished = True + self._service.bundle_build_finished(self._token_id) + + cancel = finish + + def __enter__(self) -> BundleBuildToken: + return self + + def __exit__(self, *_args: object) -> None: + self.finish() + + +class ProgressHealthService: + """Own aggregate mining-progress state and evaluate bounded readiness. + + State fields are deliberately plain attributes matching the historical + coordinator layout one-for-one: the coordinator routes its legacy raw + ``_progress_*`` names here through class descriptors, and the historical + white-box protocol mutates them directly while holding ``_lock``. Service + methods take the lock themselves. + """ + + def __init__( + self, + *, + started_monotonic: float, + initial_payout_generation: int = 0, + monotonic: Callable[[], float] = time.monotonic, + ) -> None: + self._monotonic = monotonic + self._lock = threading.Lock() + self._current_template_generation = 0 + self._current_template_fingerprint: str | None = None + self._current_payout_generation = int(initial_payout_generation) + self._published_template_generation = 0 + self._published_template_fingerprint: str | None = None + self._published_payout_generation = 0 + self._has_published_work = False + self._last_tip_poll_monotonic: float | None = None + self._last_delivery_template_generation = 0 + self._last_delivery_template_fingerprint: str | None = None + self._last_delivery_payout_generation = 0 + self._last_delivery_monotonic: float | None = None + self._pending_since_monotonic: float | None = float(started_monotonic) + # Start of the current detected-vs-published divergence epoch. Cleared + # only by current-work publication/delivery that matches the latest + # detected tip, so old delivery backlog cannot renew or expire a later + # legitimate divergence (the publication watchdog reads it). + self._publication_divergence_since_monotonic: float | None = float( + started_monotonic + ) + self._refresh_signal_pending = False + self._active_refresh_count = 0 + self._last_refresh_activity_monotonic: float | None = None + self._bundle_build_counter = 0 + self._bundle_builds: dict[int, float] = {} + + def now(self) -> float: + return float(self._monotonic()) + + def mark_refresh_pending(self, started_monotonic: float | None = None) -> None: + started = self.now() if started_monotonic is None else started_monotonic + with self._lock: + pending_since = self._pending_since_monotonic + if pending_since is None or started < pending_since: + self._pending_since_monotonic = started + divergence_since = self._publication_divergence_since_monotonic + if divergence_since is None or started < divergence_since: + self._publication_divergence_since_monotonic = started + self._refresh_signal_pending = True + + def observe_tip( + self, + work: WorkGeneration, + observed_monotonic: float | None = None, + ) -> None: + """Publish a coherent qbit tip/template observation to health state.""" + observed = self.now() if observed_monotonic is None else observed_monotonic + with self._lock: + if work.template_generation < self._current_template_generation: + # A slower concurrent poll can finish after a newer coherent + # observation. It proves only that the obsolete generation was + # coherent, so it must not renew freshness for current work. + return + self._current_template_generation = work.template_generation + self._current_template_fingerprint = work.template_fingerprint + self._current_payout_generation = max( + self._current_payout_generation, + work.payout_generation, + ) + self._last_tip_poll_monotonic = observed + same_published_work = bool( + self._has_published_work + and self._published_template_fingerprint + == work.template_fingerprint + and self._published_payout_generation + == self._current_payout_generation + ) + if same_published_work: + # Observation generations order RPC races. When the semantic + # template fingerprint and payout generation are unchanged, + # already-issued work remains current and can be reconciled to + # the latest observation without a needless socket delivery. + self._published_template_generation = work.template_generation + else: + pending_since = self._pending_since_monotonic + if pending_since is None or observed < pending_since: + self._pending_since_monotonic = observed + divergence_since = self._publication_divergence_since_monotonic + if divergence_since is None or observed < divergence_since: + self._publication_divergence_since_monotonic = observed + + def observe_payout_generation( + self, + generation: int, + invalidated_monotonic: float | None = None, + ) -> None: + invalidated = ( + self.now() + if invalidated_monotonic is None + else invalidated_monotonic + ) + with self._lock: + if generation < self._current_payout_generation: + return + self._current_payout_generation = generation + if generation != self._published_payout_generation: + pending_since = self._pending_since_monotonic + if pending_since is None or invalidated < pending_since: + self._pending_since_monotonic = invalidated + divergence_since = self._publication_divergence_since_monotonic + if divergence_since is None or invalidated < divergence_since: + self._publication_divergence_since_monotonic = invalidated + self._refresh_signal_pending = True + + def publish_work( + self, + work: WorkGeneration, + *, + matches_latest_tip: bool = True, + ) -> bool: + """Record that current in-memory work is available for delivery.""" + with self._lock: + if ( + work.template_generation < self._published_template_generation + or work.payout_generation < self._published_payout_generation + ): + return False + self._published_template_generation = max( + self._published_template_generation, + work.template_generation, + ) + self._published_template_fingerprint = work.template_fingerprint + self._published_payout_generation = max( + self._published_payout_generation, + work.payout_generation, + ) + self._has_published_work = True + if ( + matches_latest_tip + and self._current_template_fingerprint == work.template_fingerprint + and self._current_payout_generation == work.payout_generation + ): + self._refresh_signal_pending = False + self._publication_divergence_since_monotonic = None + return True + + def record_delivery( + self, + proof: DeliveryProof, + ready_mode_required: bool, + *, + matches_latest_tip: bool = True, + ) -> bool: + """Record a completed current-generation socket delivery.""" + work = proof.delivered_work + with self._lock: + if ( + work.template_fingerprint == self._current_template_fingerprint + and work.payout_generation == self._current_payout_generation + and not (ready_mode_required and proof.collection_only) + ): + self._last_delivery_template_generation = work.template_generation + self._last_delivery_template_fingerprint = work.template_fingerprint + self._last_delivery_payout_generation = work.payout_generation + self._last_delivery_monotonic = proof.delivered_monotonic + # A successful delivery proves publication for its coherent + # generation. Only the latest observed tip can close the + # outstanding divergence below. + self._published_template_generation = max( + self._published_template_generation, + self._current_template_generation, + ) + self._published_template_fingerprint = work.template_fingerprint + self._published_payout_generation = max( + self._published_payout_generation, + work.payout_generation, + ) + self._has_published_work = True + self._refresh_signal_pending = False + if matches_latest_tip: + self._publication_divergence_since_monotonic = None + return True + return False + + def refresh_started(self) -> None: + """Track a coherent poll pass while it continues making progress.""" + started = self.now() + with self._lock: + self._active_refresh_count += 1 + self._last_refresh_activity_monotonic = started + + def note_refresh_activity( + self, + observed_monotonic: float | None = None, + ) -> None: + observed = ( + self.now() + if observed_monotonic is None + else observed_monotonic + ) + with self._lock: + if self._active_refresh_count > 0: + last_activity = self._last_refresh_activity_monotonic + if last_activity is None or observed > last_activity: + self._last_refresh_activity_monotonic = observed + + def refresh_finished(self) -> None: + with self._lock: + self._active_refresh_count = max(0, self._active_refresh_count - 1) + + def start_refresh(self) -> RefreshActivityToken: + self.refresh_started() + return RefreshActivityToken(self) + + def bundle_build_started(self) -> int: + started = self.now() + with self._lock: + self._bundle_build_counter += 1 + token = self._bundle_build_counter + self._bundle_builds[token] = started + return token + + def bundle_build_finished(self, token: int) -> None: + with self._lock: + self._bundle_builds.pop(token, None) + + def start_bundle_build(self) -> BundleBuildToken: + return BundleBuildToken(self, self.bundle_build_started()) + + @staticmethod + def requiring_refresh( + eligibility: EligibilitySnapshot, + work: WorkGeneration, + ) -> int: + proofs = { + proof.connection_id: proof + for proof in eligibility.delivery_proofs + } + requiring_refresh = 0 + for connection_id in eligibility.eligible_connection_ids: + proof = proofs.get(connection_id) + if ( + proof is None + or proof.delivered_work.template_fingerprint + != work.template_fingerprint + or proof.delivered_work.payout_generation + != work.payout_generation + or ( + eligibility.ready_mode_required + and proof.collection_only + ) + ): + requiring_refresh += 1 + return requiring_refresh + + def _copy_state_locked(self) -> _ProgressStateCopy: + return _ProgressStateCopy( + current_template_generation=self._current_template_generation, + current_template_fingerprint=self._current_template_fingerprint, + current_payout_generation=self._current_payout_generation, + published_template_generation=self._published_template_generation, + published_template_fingerprint=self._published_template_fingerprint, + published_payout_generation=self._published_payout_generation, + has_published_work=self._has_published_work, + last_tip_poll_monotonic=self._last_tip_poll_monotonic, + last_delivery_template_fingerprint=( + self._last_delivery_template_fingerprint + ), + last_delivery_payout_generation=self._last_delivery_payout_generation, + last_delivery_monotonic=self._last_delivery_monotonic, + pending_since_monotonic=self._pending_since_monotonic, + publication_divergence_since_monotonic=( + self._publication_divergence_since_monotonic + ), + refresh_signal_pending=self._refresh_signal_pending, + active_refresh_count=self._active_refresh_count, + last_refresh_activity_monotonic=( + self._last_refresh_activity_monotonic + ), + bundle_build_starts=tuple(self._bundle_builds.values()), + ) + + def publication_divergence_since(self) -> float | None: + """Locked read of the divergence stamp for the publication watchdog.""" + with self._lock: + return self._publication_divergence_since_monotonic + + def reconcile_pending( + self, + eligible_counts: EligibleClientCounts, + *, + now: float | None = None, + ) -> None: + """Clear pending state exactly when publication/delivery is sufficient.""" + current = self.now() if now is None else now + with self._lock: + state_key = ( + self._current_template_fingerprint, + self._current_payout_generation, + ) + _, requiring_refresh = eligible_counts(*state_key) + with self._lock: + if state_key != ( + self._current_template_fingerprint, + self._current_payout_generation, + ): + return + published_current = bool( + self._has_published_work + and self._published_template_fingerprint == state_key[0] + and self._published_payout_generation == state_key[1] + ) + refresh_required = bool( + self._refresh_signal_pending + or not published_current + or requiring_refresh > 0 + ) + if refresh_required: + if self._pending_since_monotonic is None: + self._pending_since_monotonic = current + else: + self._pending_since_monotonic = None + + def snapshot( + self, + eligible_counts: EligibleClientCounts, + current_payout_generation: int, + config: ProgressHealthConfig, + started_monotonic: float | None = None, + *, + now: float | None = None, + ) -> ProgressHealthSnapshot: + """Return a bounded, monotonic-only mining progress health snapshot.""" + current = self.now() if now is None else now + with self._lock: + tracked_payout_generation = self._current_payout_generation + if current_payout_generation > tracked_payout_generation: + self.observe_payout_generation(current_payout_generation, current) + self.reconcile_pending(eligible_counts, now=current) + + with self._lock: + state = self._copy_state_locked() + + eligible_count, requiring_refresh = eligible_counts( + state.current_template_fingerprint, + state.current_payout_generation, + ) + started = ( + current if started_monotonic is None else float(started_monotonic) + ) + tip_poll_reference = ( + started + if state.last_tip_poll_monotonic is None + else state.last_tip_poll_monotonic + ) + tip_poll_age = max(0.0, current - tip_poll_reference) + refresh_activity_age = ( + None + if ( + state.active_refresh_count <= 0 + or state.last_refresh_activity_monotonic is None + ) + else max(0.0, current - state.last_refresh_activity_monotonic) + ) + pending_age = ( + None + if state.pending_since_monotonic is None + else max(0.0, current - state.pending_since_monotonic) + ) + oldest_bundle_age = ( + 0.0 + if not state.bundle_build_starts + else max(0.0, current - min(state.bundle_build_starts)) + ) + published_current = bool( + state.has_published_work + and state.published_template_fingerprint + == state.current_template_fingerprint + and state.published_payout_generation + == state.current_payout_generation + ) + delivered_current = bool( + state.current_template_fingerprint is not None + and state.last_delivery_template_fingerprint + == state.current_template_fingerprint + and state.last_delivery_payout_generation + == state.current_payout_generation + ) + delivery_age = ( + max(0.0, current - state.last_delivery_monotonic) + if delivered_current and state.last_delivery_monotonic is not None + else None + ) + reasons: list[str] = [] + refresh_is_progressing = bool( + state.active_refresh_count > 0 + and refresh_activity_age is not None + and refresh_activity_age <= config.tip_poll_deadline_seconds + ) + if ( + tip_poll_age > config.tip_poll_deadline_seconds + and not refresh_is_progressing + ): + reasons.append("tip_poll_stale") + if oldest_bundle_age > config.bundle_build_deadline_seconds: + reasons.append("bundle_build_stuck") + if not state.has_published_work: + reasons.append("current_generation_not_published") + elif ( + pending_age is not None + and pending_age > config.pending_refresh_deadline_seconds + ): + reasons.append("refresh_pending_too_long") + if state.refresh_signal_pending or not published_current: + reasons.append("current_generation_not_published") + elif requiring_refresh > 0: + reasons.append("current_generation_not_delivered") + reasons = list(dict.fromkeys(reasons)) + + def rounded(value: float | None) -> float | None: + return None if value is None else round(value, 3) + + return ProgressHealthSnapshot( + ok=not reasons, + reason=reasons[0] if reasons else None, + reasons=tuple(reasons), + pending_refresh=state.pending_since_monotonic is not None, + pending_refresh_age_seconds=rounded(pending_age), + tip_poll_age_seconds=rounded(tip_poll_age), + tip_refresh_in_progress=state.active_refresh_count > 0, + tip_refresh_progress_age_seconds=rounded(refresh_activity_age), + current_template_generation=state.current_template_generation, + published_template_generation=state.published_template_generation, + current_payout_generation=state.current_payout_generation, + published_payout_generation=state.published_payout_generation, + last_valid_delivery_age_seconds=rounded(delivery_age), + eligible_client_count=eligible_count, + eligible_clients_requiring_refresh=requiring_refresh, + bundle_build_oldest_age_seconds=rounded(oldest_bundle_age), + ) + + @staticmethod + def overlay( + base_health: Mapping[str, object], + snapshot: ProgressHealthSnapshot, + ) -> Mapping[str, object]: + return overlay_progress_health(base_health, snapshot.as_mapping()) + + @staticmethod + def metrics_lines( + snapshot: ProgressHealthSnapshot, + *, + coordination_blocked_age_seconds: float, + ) -> tuple[str, ...]: + pending_age = snapshot.pending_refresh_age_seconds + delivery_age = snapshot.last_valid_delivery_age_seconds + active_reasons = set(snapshot.reasons) + return ( + "# HELP qbit_prism_refresh_pending Whether current template or payout work still requires publication or delivery.", + "# TYPE qbit_prism_refresh_pending gauge", + f"qbit_prism_refresh_pending {1 if snapshot.pending_refresh else 0}", + "# HELP qbit_prism_refresh_pending_age_seconds Monotonic age of the oldest unresolved current-work refresh.", + "# TYPE qbit_prism_refresh_pending_age_seconds gauge", + f"qbit_prism_refresh_pending_age_seconds {float(pending_age or 0.0):.6f}", + "# HELP qbit_prism_tip_poll_age_seconds Monotonic age of the last coherent qbit tip/template poll.", + "# TYPE qbit_prism_tip_poll_age_seconds gauge", + f"qbit_prism_tip_poll_age_seconds {float(snapshot.tip_poll_age_seconds):.6f}", + "# HELP qbit_prism_current_generation_delivery_age_seconds Monotonic age of the last valid current-generation delivery, or -1 when none exists.", + "# TYPE qbit_prism_current_generation_delivery_age_seconds gauge", + f"qbit_prism_current_generation_delivery_age_seconds {float(delivery_age) if delivery_age is not None else -1.0:.6f}", + "# HELP qbit_prism_bundle_build_oldest_age_seconds Monotonic age of the oldest active bundle build.", + "# TYPE qbit_prism_bundle_build_oldest_age_seconds gauge", + f"qbit_prism_bundle_build_oldest_age_seconds {float(snapshot.bundle_build_oldest_age_seconds):.6f}", + "# HELP qbit_prism_template_refresh_coordination_blocked_age_seconds Monotonic age of the continuous coordination-blocked template-refresh streak, or 0 when clear.", + "# TYPE qbit_prism_template_refresh_coordination_blocked_age_seconds gauge", + f"qbit_prism_template_refresh_coordination_blocked_age_seconds {coordination_blocked_age_seconds:.6f}", + "# HELP qbit_prism_health_state Current progress-health state by bounded reason.", + "# TYPE qbit_prism_health_state gauge", + f'qbit_prism_health_state{{reason="healthy"}} {1 if snapshot.ok else 0}', + *( + f'qbit_prism_health_state{{reason="{reason}"}} {1 if reason in active_reasons else 0}' + for reason in PROGRESS_HEALTH_REASONS + ), + ) + + +def overlay_progress_health( + base_health: Mapping[str, object], + progress: Mapping[str, object], +) -> Mapping[str, object]: + """Overlay current progress without masking an independent base failure.""" + + base_ok = bool(base_health.get("mining_ready", base_health.get("ok"))) + result = dict(base_health) + result.update(progress) + result["ok"] = base_ok and bool(progress["ok"]) + if progress["ok"]: + result.pop("reason", None) + result.pop("reasons", None) + return MappingProxyType(result) + + +__all__ = [ + "BundleBuildToken", + "DeliveryProof", + "EligibilitySnapshot", + "PROGRESS_HEALTH_REASONS", + "ProgressHealthConfig", + "ProgressHealthService", + "ProgressHealthSnapshot", + "RefreshActivityToken", + "WorkGeneration", + "overlay_progress_health", +] diff --git a/lab/prism/public_api.py b/lab/prism/public_api.py index 02695d046..5e263b290 100644 --- a/lab/prism/public_api.py +++ b/lab/prism/public_api.py @@ -3,6 +3,7 @@ from __future__ import annotations +import hashlib import json import math import os @@ -12,8 +13,8 @@ from collections import OrderedDict, deque from dataclasses import dataclass from datetime import datetime, timedelta, timezone -from decimal import Decimal -from typing import Any, Callable +from decimal import Decimal, InvalidOperation +from typing import Any, Callable, Iterator from lab.prism.ctv_broadcaster import COINBASE_MATURITY from lab.prism import direct_stratum @@ -27,6 +28,14 @@ ) MAX_SEARCH_LENGTH = 128 MAX_RECIPIENT_ID_LENGTH = 256 +BLOCKS_CHAIN_STATE_FILTERS = ("active", "all", "reversed") +# getnetworkhashps window and lane. The pool's shares are credited against the +# permissionless lane, so the ratio pool hashrate_ths / network hashrate_ths is +# only meaningful against that lane's estimate, not the all-lanes aggregate. +# 120 blocks and height -1 are the RPC's own documented defaults, passed +# explicitly because the lane argument is positional after them. +NETWORK_HASHRATE_BLOCK_WINDOW = 120 +NETWORK_HASHRATE_LANE = "permissionless" HASHRATE_SERIES_BUCKET_SECONDS = {"5m": 300, "1h": 3600, "1d": 86400} HASHRATE_SERIES_RANGE_SECONDS = { "1w": 7 * 86400, @@ -34,13 +43,47 @@ "6m": 180 * 86400, "all": None, } +# The bucket vocabulary each range may request. Chart resolution is a product +# choice, but request cost is not: the ledger scan and the response both grow +# with range/bucket, and the widest combinations (a 5m bucket over months of +# history) exceed PRISM_PUBLIC_CACHE_MAX_RESPONSE_BYTES, so every such request +# bypasses the response cache and re-scans -- an unbounded-cost endpoint any +# stranger can hit. The static vocabulary closes that: coarse ranges admit only +# buckets whose point counts stay dashboard-sized, and auto_bucket() always +# resolves inside these sets, so `bucket=auto` is never rejected. Shared by +# every bucketed public series endpoint, not just the hashrate series. +HASHRATE_SERIES_ALLOWED_BUCKETS: dict[str, frozenset[str]] = { + "1w": frozenset({"5m", "1h", "1d"}), + "1m": frozenset({"1h", "1d"}), + "6m": frozenset({"1d"}), + "all": frozenset({"1d"}), +} DEFAULT_HASHRATE_SMOOTHING_SECONDS = 30 * 60 +BLOCK_MARKERS_MAX_BLOCKS_PER_BUCKET = 3 PUBLIC_ERROR_CODES = { "not_found": "not_found", "rate_limited": "rate_limited", "internal_error": "internal_error", "qbit_rpc_unavailable": "upstream_unavailable", + # A cached response outlived its staleness budget and the public read + # service refused to serve it. Reported as upstream_unavailable, which is + # already in the documented error enum, because the condition is exactly + # that: the data behind this route is too old to answer with. + "stale_read_model": "upstream_unavailable", + # The public read tier's replica is missing, writable, or its replication + # stream has gone silent past the configured bound, so this route has no + # data it is willing to answer with. Same documented wire code as + # stale_read_model for the same reason: from a client's side the two are + # one condition. + "replica_unavailable": "upstream_unavailable", + # A public origin computation hit its per-request statement deadline + # (PRISM_PUBLIC_READ_STATEMENT_TIMEOUT_SECONDS) before the ledger + # answered. Its own wire code rather than upstream_unavailable: the + # database is reachable, this particular read was too expensive, and a + # retry a moment later can succeed off the cache -- which is not what + # upstream_unavailable tells a client. + "read_timeout": "read_timeout", } @@ -51,12 +94,37 @@ class PublicCachePolicy: immutable: bool = False +@dataclass(frozen=True) +class RawJsonBody: + """Pre-serialized JSON response body served byte-for-byte. + + Content-addressed artifact responses must satisfy + sha256(response bytes) == the advertised artifact sha256, so the exact + canonical bytes bypass write_json's generic sorted-key re-serialization + and its trailing newline. + """ + + body: bytes + + +@dataclass(frozen=True) +class UncacheableJsonBody: + """JSON payload whose degraded provenance must not enter shared caches.""" + + payload: object + reason: str + + @dataclass class _PublicCacheEntry: status: int payload: object stored_at: float expires_at: float + # When the entry stops being servable even as a stale-while-revalidate + # answer. Recorded at store time so capacity reaping can honor the window + # this entry was stored under without re-deriving the route's policy. + stale_until: float @dataclass @@ -66,6 +134,12 @@ class _PublicInflight: exception: BaseException | None = None +# Shared by every public response cache in this process. Admission is +# nonblocking: excess stale hits keep their bounded-age response without +# creating workers or queued refresh tasks behind the database read slots. +_PUBLIC_REFRESH_SLOTS = threading.BoundedSemaphore(4) + + class PublicResponseCache: """Small in-process cache for public read-model responses.""" @@ -86,64 +160,119 @@ def get_or_compute( key: tuple[str, tuple[tuple[str, tuple[str, ...]], ...]], ttl_seconds: int, compute: Callable[[], tuple[int, object]], + stale_while_revalidate_seconds: int = 0, ) -> tuple[int, object, str, int]: if ttl_seconds <= 0: status, payload = compute() return status, payload, "BYPASS", 0 now = time.monotonic() + stale_result: tuple[int, object, str, int] | None = None + refresh: _PublicInflight | None = None with self._lock: entry = self._entries.get(key) if entry is not None: if entry.expires_at > now: self._entries.move_to_end(key) return entry.status, entry.payload, "HIT", max(0, int(now - entry.stored_at)) - # Expired: drop the stale slot so it cannot count toward the bound. - del self._entries[key] - inflight = self._inflight.get(key) - if inflight is None: - inflight = _PublicInflight(event=threading.Event()) - self._inflight[key] = inflight - owner = True - else: - owner = False + if ( + stale_while_revalidate_seconds > 0 + and now <= entry.expires_at + stale_while_revalidate_seconds + ): + # Inside the stale-while-revalidate window the expired + # entry is served immediately -- with its honest age -- + # instead of blocking this requester on a recompute that + # can take seconds. The entry stays in place: it remains + # the answer for the rest of the window, including when + # the refresh below fails. At most one background refresh + # runs per key, deduped through the same inflight map that + # coalesces blocking misses, so a stampede of stale hits + # costs one origin call. + self._entries.move_to_end(key) + stale_result = ( + entry.status, + entry.payload, + "STALE", + max(0, int(now - entry.stored_at)), + ) + if key not in self._inflight and _PUBLIC_REFRESH_SLOTS.acquire(blocking=False): + refresh = _PublicInflight(event=threading.Event()) + self._inflight[key] = refresh + else: + # Expired past any revalidation window: drop the stale + # slot so it cannot count toward the bound. + del self._entries[key] + if stale_result is None: + inflight = self._inflight.get(key) + if inflight is None: + inflight = _PublicInflight(event=threading.Event()) + self._inflight[key] = inflight + owner = True + else: + owner = False + + if stale_result is not None: + if refresh is not None: + try: + threading.Thread( + target=self._refresh_entry_admitted, + args=(key, ttl_seconds, stale_while_revalidate_seconds, compute, refresh), + name="prism-public-cache-refresh", + daemon=True, + ).start() + except Exception: + # Thread creation can fail under resource pressure, after + # the refresh slot was already registered. Free the slot + # and wake any coalesced waiter, or a burst of stale keys + # would pin refreshes off forever and, past the window, + # leave blocking requests waiting on an event nobody can + # set. The stale answer below is still the right response; + # the next stale hit simply retries the refresh. + with self._lock: + if self._inflight.get(key) is refresh: + del self._inflight[key] + refresh.event.set() + _PUBLIC_REFRESH_SLOTS.release() + return stale_result if not owner: # Coalesce onto the owner's single origin call. Reuse its result even # when it was not cacheable (BYPASS), and re-raise its exception, so - # waiters never re-run an expensive or failing compute(). + # waiters never re-run an expensive or failing compute(). The owner + # may be a background refresh whose stale entry has since aged past + # the revalidation window; waiting for it is the same bargain. inflight.event.wait() - if inflight.exception is not None: - raise inflight.exception - if inflight.result is not None: - return inflight.result + exception = inflight.exception + result = inflight.result + # Drop this frame's name for the flight before re-raising (#251). + # Raising prepends this frame to the shared exception's traceback, + # and a frame still naming the flight would close the cycle + # flight -> exception -> traceback -> frame -> flight, keeping the + # owner's failed compute frame and its locals alive until cyclic + # GC. The flight itself keeps the exception for later observers. + del inflight + if exception is not None: + try: + raise exception + finally: + # Unbind the local for the same reason: the traceback's + # frame must not name the exception it belongs to. + del exception + if result is not None: + return result # Owner finished without recording a result; recompute defensively. return self.get_or_compute(key=key, ttl_seconds=ttl_seconds, compute=compute) try: status, payload = compute() if 200 <= status < 300 and cacheable_payload_size(payload): - now = time.monotonic() - with self._lock: - self._entries[key] = _PublicCacheEntry( - status=status, - payload=payload, - stored_at=now, - expires_at=now + ttl_seconds, - ) - self._entries.move_to_end(key) - max_entries = public_cache_max_entries() - if len(self._entries) > max_entries: - # Reap expired entries before LRU eviction so dead slots - # never push out still-fresh keys. - for stale_key in [ - stored_key - for stored_key, stored in self._entries.items() - if stored_key != key and stored.expires_at <= now - ]: - del self._entries[stale_key] - while len(self._entries) > max_entries: - self._entries.popitem(last=False) + self._store_entry( + key, + status=status, + payload=payload, + ttl_seconds=ttl_seconds, + stale_while_revalidate_seconds=stale_while_revalidate_seconds, + ) inflight.result = (status, payload, "MISS", 0) else: inflight.result = (status, payload, "BYPASS", 0) @@ -155,6 +284,103 @@ def get_or_compute( with self._lock: self._inflight.pop(key, None) inflight.event.set() + # Release the owner frame's reference last, after every waiter has + # been woken (#251). On failure the exception's traceback already + # holds this frame, so a frame that kept naming the flight would + # form flight -> exception -> traceback -> frame -> flight and pin + # the failed compute frame, with the origin call's locals, until + # cyclic GC. Waiters still hold the flight through their own + # frames until each has observed the recorded result or error. + del inflight + + def _store_entry( + self, + key: tuple[str, tuple[tuple[str, tuple[str, ...]], ...]], + *, + status: int, + payload: object, + ttl_seconds: int, + stale_while_revalidate_seconds: int = 0, + ) -> None: + now = time.monotonic() + with self._lock: + self._entries[key] = _PublicCacheEntry( + status=status, + payload=payload, + stored_at=now, + expires_at=now + ttl_seconds, + stale_until=now + ttl_seconds + max(0, stale_while_revalidate_seconds), + ) + self._entries.move_to_end(key) + max_entries = public_cache_max_entries() + if len(self._entries) > max_entries: + # Reap dead entries before LRU eviction so dead slots never + # push out still-fresh keys. Dead means past the entry's own + # recorded revalidation window, not merely past its TTL: an + # expired entry inside that window is still the promised + # instant answer for its next visitor, and discarding it here + # would turn that visitor's stale hit into a blocking miss. + for stale_key in [ + stored_key + for stored_key, stored in self._entries.items() + if stored_key != key and stored.stale_until <= now + ]: + del self._entries[stale_key] + while len(self._entries) > max_entries: + self._entries.popitem(last=False) + + def _refresh_entry_admitted(self, *args) -> None: + try: + self._refresh_entry(*args) + finally: + # A failed compute retains caller frames through f_back too. Drop + # the flight-bearing arguments before returning the admission slot. + del args + _PUBLIC_REFRESH_SLOTS.release() + + def _refresh_entry( + self, + key: tuple[str, tuple[tuple[str, tuple[str, ...]], ...]], + ttl_seconds: int, + stale_while_revalidate_seconds: int, + compute: Callable[[], tuple[int, object]], + inflight: _PublicInflight, + ) -> None: + """Recompute one stale-served key off the request thread. + + The same compute through the same status/size gates as a blocking + miss. Success replaces the entry; a failure or a non-cacheable result + leaves the stale entry exactly where it was -- still servable until + its revalidation window ends -- and the finally clause frees the + inflight slot either way, so a later stale hit can retry. A blocking + request that arrives beyond the window while this runs coalesces onto + it through the inflight map, exactly as it would onto a foreground + owner; the recorded result and exception exist for those waiters. + """ + try: + status, payload = compute() + if 200 <= status < 300 and cacheable_payload_size(payload): + self._store_entry( + key, + status=status, + payload=payload, + ttl_seconds=ttl_seconds, + stale_while_revalidate_seconds=stale_while_revalidate_seconds, + ) + inflight.result = (status, payload, "MISS", 0) + else: + inflight.result = (status, payload, "BYPASS", 0) + except BaseException as exc: + inflight.exception = exc + finally: + with self._lock: + self._inflight.pop(key, None) + inflight.event.set() + # Same ownership rule as the foreground owner (#251): the swallowed + # exception's traceback captured this frame on arrival, so the + # frame must not keep naming the flight that stores it, or the + # completed refresh pins its failed compute locals until cyclic GC. + del inflight def _is_hex64(value: str) -> bool: @@ -276,6 +502,10 @@ def cacheable_payload_size(payload: object) -> bool: max_bytes = env_nonnegative_int("PRISM_PUBLIC_CACHE_MAX_RESPONSE_BYTES", 1_048_576) if max_bytes <= 0: return False + if isinstance(payload, UncacheableJsonBody): + return False + if isinstance(payload, RawJsonBody): + return len(payload.body) <= max_bytes body_size = len(json.dumps(payload, sort_keys=True).encode()) + 1 return body_size <= max_bytes @@ -311,7 +541,14 @@ def dispatch(coordinator: Any, path: str, query: dict[str, list[str]]) -> tuple[ return 200, pool_summary(coordinator) if path == "/public/v1/blocks": page, limit = pagination_params(query) - return 200, blocks(coordinator, page=page, limit=limit) + chain_state = first_query_value(query, "chain_state") or "active" + if chain_state not in BLOCKS_CHAIN_STATE_FILTERS: + raise PublicApiError( + 400, + "invalid_chain_state", + "chain_state must be one of active, all, reversed", + ) + return 200, blocks(coordinator, page=page, limit=limit, chain_state=chain_state) if path == "/public/v1/leaderboard": page, limit = pagination_params(query) search = search_param(query) @@ -345,7 +582,12 @@ def dispatch(coordinator: Any, path: str, query: dict[str, list[str]]) -> tuple[ subject = first_query_value(query, "subject") or "pool" range_id = first_query_value(query, "range") or "1m" bucket = first_query_value(query, "bucket") or "auto" - return 200, hashrate_series(coordinator, subject=subject, range_id=range_id, bucket=bucket) + view = first_query_value(query, "view") + return 200, hashrate_series(coordinator, subject=subject, range_id=range_id, bucket=bucket, view=view) + if path == "/public/v1/block-markers": + range_id = first_query_value(query, "range") or "1m" + bucket = first_query_value(query, "bucket") or "auto" + return 200, block_markers(coordinator, range_id=range_id, bucket=bucket) if path == "/public/v1/mining-configuration": return 200, mining_configuration(coordinator) if path.startswith("/public/v1/miners/"): @@ -407,6 +649,10 @@ def error_payload(code: str, message: str) -> dict[str, object]: def pool_summary(coordinator: Any) -> dict[str, object]: generated_at = utc_now_iso() network = network_summary(coordinator) + # Looked up after network_summary so a missing-compact-bits 503 still + # short-circuits first; a failed hashrate lookup alone must never fail + # this endpoint (the field is nullable by contract). + network["hashrate_ths"] = network_hashrate_ths(coordinator) ledger_snapshot = coordinator.ledger.dashboard_pool_snapshot( current_network_difficulty=network["network_difficulty"], generated_at=generated_at, @@ -421,6 +667,8 @@ def pool_summary(coordinator: Any) -> dict[str, object]: "participants_3h": ledger_snapshot["participants_3h"], "blocks_found_total": ledger_snapshot["blocks_found_total"], "prism_blocks_total": ledger_snapshot["prism_blocks_total"], + "blocks_reversed_total": ledger_snapshot["blocks_reversed_total"], + "blocks_inactive_total": ledger_snapshot["blocks_inactive_total"], "total_mined_bits": ledger_snapshot["total_mined_bits"], "expected_time_to_block_seconds": expected_time_to_block_seconds( hashrate_ths=str(ledger_snapshot["hashrate_ths"]["h3"]), @@ -432,9 +680,17 @@ def pool_summary(coordinator: Any) -> dict[str, object]: } -def blocks(coordinator: Any, *, page: int, limit: int) -> dict[str, object]: +def blocks(coordinator: Any, *, page: int, limit: int, chain_state: str = "active") -> dict[str, object]: generated_at = utc_now_iso() - payload = coordinator.ledger.dashboard_blocks(page=page, limit=limit) + if chain_state == "active": + # The default keeps today's read-model call and today's schema tag + # exactly: no chain_state argument reaches the ledger, so the default + # response stays byte-compatible with pre-filter consumers. + payload = coordinator.ledger.dashboard_blocks(page=page, limit=limit) + schema = "prism.dashboard.blocks.v1" + else: + payload = coordinator.ledger.dashboard_blocks(page=page, limit=limit, chain_state=chain_state) + schema = "prism.dashboard.blocks.v2" for row in payload["rows"]: row.setdefault("bits", "00000000") if row.get("bits") is None: @@ -443,7 +699,7 @@ def blocks(coordinator: Any, *, page: int, limit: int) -> dict[str, object]: if row.get("network_difficulty") is None: row["network_difficulty"] = "0" return { - "schema": "prism.dashboard.blocks.v1", + "schema": schema, "generated_at": generated_at, "pagination": payload["pagination"], "rows": payload["rows"], @@ -497,13 +753,21 @@ def reward_leaderboard( } -def hashrate_series(coordinator: Any, *, subject: str, range_id: str, bucket: str) -> dict[str, object]: +def hashrate_series( + coordinator: Any, + *, + subject: str, + range_id: str, + bucket: str, + view: str | None = None, +) -> dict[str, object]: if range_id not in {"1w", "1m", "6m", "all"}: raise PublicApiError(400, "invalid_range", "range must be one of 1w, 1m, 6m, all") if bucket == "auto": bucket = auto_bucket(range_id) if bucket not in {"5m", "1h", "1d"}: raise PublicApiError(400, "invalid_bucket", "bucket must be one of auto, 5m, 1h, 1d") + allowed_hashrate_bucket(range_id, bucket) if subject == "pool": subject_type = "pool" subject_id = None @@ -512,13 +776,34 @@ def hashrate_series(coordinator: Any, *, subject: str, range_id: str, bucket: st subject_id = subject.removeprefix("miner:") else: raise PublicApiError(400, "invalid_subject", "subject must be pool or miner:{recipient_id}") - generated_at = utc_now_iso() + # An absent view keeps the v1 response; only `both` opts into the dual-rate + # v2 contract. Anything else is rejected rather than silently + # served as v1, so a client that mistypes the view cannot mistake the + # legacy smoothed rate for the raw one it asked for. + if view is not None and view != "both": + raise PublicApiError(400, "invalid_view", "view must be both or omitted") + dual_rate = view == "both" + now = datetime.now(timezone.utc) + generated_at = iso_datetime(now) + generated_at_epoch = int(now.timestamp()) bucket_seconds = HASHRATE_SERIES_BUCKET_SECONDS[bucket] window_seconds = public_hashrate_smoothing_seconds() range_seconds = HASHRATE_SERIES_RANGE_SECONDS[range_id] lookback_seconds = 0 min_epoch: int | None = None range_anchor_epoch: int | None = None + if range_seconds is not None and ( + dual_rate or window_seconds // bucket_seconds >= 2 + ): + # V2 raw rates always divide by a full bucket, so trim a leading bucket + # that straddles the range boundary even when this coarse bucket has no + # trailing smoothing. V1 keeps its frozen no-smoothing behavior. + range_anchor_epoch = generated_at_epoch + min_epoch = hashrate_series_min_epoch( + range_anchor_epoch, + range_seconds, + bucket_seconds, + ) if window_seconds // bucket_seconds >= 2 and range_seconds is not None: # Fetch one full smoothing window of pre-range history so the first # in-range points average over real data instead of artificial zeros, @@ -526,8 +811,12 @@ def hashrate_series(coordinator: Any, *, subject: str, range_id: str, bucket: st # its range lower bound on the same epoch min_epoch derives from, so # the trim cannot disagree with the fetch under clock skew. lookback_seconds = (window_seconds // bucket_seconds) * bucket_seconds - range_anchor_epoch = int(datetime.now(timezone.utc).timestamp()) - min_epoch = hashrate_series_min_epoch(range_anchor_epoch, range_seconds, bucket_seconds) + range_anchor_epoch = generated_at_epoch + min_epoch = hashrate_series_min_epoch( + range_anchor_epoch, + range_seconds, + bucket_seconds, + ) points = coordinator.ledger.dashboard_hashrate_series( subject_type=subject_type, subject_id=subject_id, @@ -536,6 +825,27 @@ def hashrate_series(coordinator: Any, *, subject: str, range_id: str, bucket: st lookback_seconds=lookback_seconds, range_anchor_epoch=range_anchor_epoch, ) + subject_payload = {"type": subject_type, "id": subject_id} + if dual_rate: + smoothing = hashrate_series_smoothing(bucket_seconds=bucket_seconds, window_seconds=window_seconds) + return { + "schema": "prism.dashboard.hashrate-series.v2", + "generated_at": generated_at, + "subject": subject_payload, + "range": range_id, + "bucket": bucket, + "bucket_seconds": bucket_seconds, + "unit": "ths", + "rate_basis": "accepted_share_difficulty", + "smoothing": smoothing, + "points": dual_rate_hashrate_series_points( + points, + bucket_seconds=bucket_seconds, + window_seconds=window_seconds, + min_epoch=min_epoch, + generated_at_epoch=generated_at_epoch, + ), + } points = smooth_hashrate_series_points( points, bucket_seconds=bucket_seconds, @@ -545,7 +855,7 @@ def hashrate_series(coordinator: Any, *, subject: str, range_id: str, bucket: st return { "schema": "prism.dashboard.hashrate-series.v1", "generated_at": generated_at, - "subject": {"type": subject_type, "id": subject_id}, + "subject": subject_payload, "range": range_id, "bucket": bucket, "unit": "ths", @@ -553,6 +863,68 @@ def hashrate_series(coordinator: Any, *, subject: str, range_id: str, bucket: st } +def block_markers(coordinator: Any, *, range_id: str, bucket: str) -> dict[str, object]: + """Found-block markers pre-bucketed onto the hashrate chart's grid. + + The dashboard draws markers over the hashrate series, so every bucket + epoch here is floor(epoch(found_at) / bucket_seconds) * bucket_seconds -- + the identical arithmetic the hashrate buckets use -- and the bounded + ranges keep only fully covered buckets via the same + hashrate_series_min_epoch anchor the chart trims against. Paging + /public/v1/blocks cannot serve this: at hundreds of found blocks a day the + long timeframes would need hundreds of pages per chart render. + """ + if range_id not in {"1w", "1m", "6m", "all"}: + raise PublicApiError(400, "invalid_range", "range must be one of 1w, 1m, 6m, all") + if bucket == "auto": + bucket = auto_bucket(range_id) + if bucket not in {"5m", "1h", "1d"}: + raise PublicApiError(400, "invalid_bucket", "bucket must be one of auto, 5m, 1h, 1d") + # Markers land on top of the hashrate chart, so the hashrate series' + # static bucket-per-range vocabulary is the markers' vocabulary too. + allowed_hashrate_bucket(range_id, bucket) + now = datetime.now(timezone.utc) + generated_at = iso_datetime(now) + bucket_seconds = HASHRATE_SERIES_BUCKET_SECONDS[bucket] + range_seconds = HASHRATE_SERIES_RANGE_SECONDS[range_id] + # Anchor the range lower bound on this process's clock, exactly as + # hashrate_series does, so marker buckets and hashrate buckets share one + # grid even when the database clock disagrees. `all` has no lower bound. + range_anchor_epoch = int(now.timestamp()) if range_seconds is not None else None + payload = coordinator.ledger.dashboard_block_markers( + range_id=range_id, + bucket=bucket, + range_anchor_epoch=range_anchor_epoch, + ) + points: list[dict[str, object]] = [] + for row in payload["points"]: + block_count = int(row["block_count"]) + points.append( + { + "timestamp": row["timestamp"], + "block_count": block_count, + "blocks": [ + { + "height": int(block["height"]), + "hash": str(block["hash"]), + "found_at": str(block["found_at"]), + } + for block in row["blocks"] + ], + "truncated": block_count > BLOCK_MARKERS_MAX_BLOCKS_PER_BUCKET, + } + ) + return { + "schema": "prism.dashboard.block-markers.v1", + "generated_at": generated_at, + "range": range_id, + "bucket": bucket, + "bucket_seconds": bucket_seconds, + "total_blocks": int(payload["total_blocks"]), + "points": points, + } + + def mining_configuration(coordinator: Any) -> dict[str, object]: port = int(getattr(coordinator, "port", os.environ.get("PRISM_STRATUM_PORT", "3340"))) host = os.environ.get("PRISM_PUBLIC_STRATUM_HOST") or str(getattr(coordinator, "bind", "127.0.0.1")) @@ -865,6 +1237,17 @@ def settlement_artifacts(coordinator: Any, *, block_hash: str) -> dict[str, obje def direct_coinbase_settlement_payload(ledger: Any, *, block_hash: str) -> dict[str, object] | None: + read_model = getattr(ledger, "dashboard_direct_coinbase_settlement", None) + if callable(read_model): + try: + payload = read_model(block_hash=block_hash) + except RuntimeError as exc: + if audit_bundle_body_read_failed(exc): + return None + raise + return payload if isinstance(payload, dict) else None + # In-memory ledger only: audit_bundle() is a writer-lock read on the + # Postgres ledger, so a public settlement lookup must not reach it. getter = getattr(ledger, "audit_bundle", None) if not callable(getter): return None @@ -949,6 +1332,26 @@ def fanout(coordinator: Any, *, fanout_txid: str) -> dict[str, object]: def artifact(coordinator: Any, *, sha256: str) -> object: + document = getattr(coordinator.ledger, "dashboard_public_artifact_document", None) + if callable(document): + row = document(sha256=sha256) + if not isinstance(row, dict): + raise PublicApiError(404, "not_found", "unknown public PRISM artifact") + canonical_json = row.get("canonical_json") + if isinstance(canonical_json, str): + body = canonical_json.encode() + if hashlib.sha256(body).hexdigest() == sha256: + return RawJsonBody(body=body) + # Stored canonical text that does not hash to its content address + # is a store-integrity fault; fall through to the re-serialized + # response rather than serve bytes that contradict the address. + payload = row.get("payload") + if payload is None: + raise PublicApiError(404, "not_found", "unknown public PRISM artifact") + fallback_reason = row.get("canonical_fallback_reason") + if isinstance(fallback_reason, str) and fallback_reason: + return UncacheableJsonBody(payload=payload, reason=fallback_reason) + return payload getter = getattr(coordinator.ledger, "dashboard_public_artifact", None) if not callable(getter): raise PublicApiError(404, "not_found", "unknown public PRISM artifact") @@ -1147,6 +1550,34 @@ def network_summary(coordinator: Any) -> dict[str, object]: } +def network_hashrate_ths(coordinator: Any) -> str | None: + """Chain-derived estimate of the permissionless-lane network hashrate, TH/s. + + qbitd's getnetworkhashps answers in H/s over a trailing block window; the + permissionless lane is requested because that is the lane pool shares are + credited against, so consumers can divide a pool hashrate_ths window by + this value for an approximate share-of-network. Nullable and non-fatal by + contract: any RPC failure or non-numeric answer yields None, never an + error response -- pool-summary may only 503 over missing compact bits. + """ + try: + raw = coordinator.rpc.call( + "getnetworkhashps", + [NETWORK_HASHRATE_BLOCK_WINDOW, -1, NETWORK_HASHRATE_LANE], + ) + except Exception: + return None + if raw is None or isinstance(raw, bool): + return None + try: + hashes_per_second = Decimal(str(raw)) + except (ArithmeticError, ValueError, TypeError): + return None + if not hashes_per_second.is_finite() or hashes_per_second < 0: + return None + return decimal_string(hashes_per_second / TERAHASH) + + def first_present(*values: object) -> object | None: for value in values: if value is not None: @@ -1230,6 +1661,95 @@ def smooth_hashrate_series_points( if (bucket_count < 2 and min_epoch is None) or not points: return points effective_window_seconds = bucket_count * bucket_seconds + parsed_points = _parse_hashrate_series_points(points) + if bucket_count < 2: + return [point for epoch, _, point in parsed_points if min_epoch is None or epoch >= min_epoch] + smoothed: list[dict[str, object]] = [] + for epoch, _, point, window_total in _trailing_window_totals(parsed_points, effective_window_seconds): + if min_epoch is not None and epoch < min_epoch: + continue + smoothed.append( + { + **point, + "hashrate_ths": hashrate_ths_from_difficulty(window_total, effective_window_seconds), + } + ) + return smoothed + + +def hashrate_series_smoothing(*, bucket_seconds: int, window_seconds: int) -> dict[str, object]: + """Describe the smoothing a dual-rate series was produced with. + + Mirrors smooth_hashrate_series_points: a window shorter than two buckets + (including the smoothing kill switch at 0) leaves rates untouched, so the + method is reported as `none` and the effective window is the bucket + itself. Otherwise the trailing window is the whole number of buckets that + fit in the configured seconds. + """ + bucket_count = window_seconds // bucket_seconds if bucket_seconds > 0 else 0 + if bucket_count < 2: + return {"method": "none", "window_seconds": bucket_seconds} + return {"method": "trailing", "window_seconds": bucket_count * bucket_seconds} + + +def dual_rate_hashrate_series_points( + points: list[dict[str, object]], + *, + bucket_seconds: int, + window_seconds: int, + min_epoch: int | None = None, + generated_at_epoch: int, +) -> list[dict[str, object]]: + """Build v2 points carrying both the raw and the trailing-window rate. + + raw_hashrate_ths is the bucket's credited difficulty over the full bucket + duration, recomputed here rather than trusting the ledger's per-bucket + rate so the documented definition holds for every backend. + smoothed_hashrate_ths is the same trailing-window estimate the v1 series + serves as hashrate_ths; when smoothing is off or shorter than two buckets + it equals the raw rate. A bucket is complete once generated_at has reached + its end, so the newest bucket is normally still accumulating credit and + its raw rate under-reads the true rate. + + Pre-range lookback context and malformed points are handled exactly as in + smooth_hashrate_series_points. Points are returned ascending by timestamp. + """ + smoothing = hashrate_series_smoothing(bucket_seconds=bucket_seconds, window_seconds=window_seconds) + trailing = smoothing["method"] == "trailing" + effective_window_seconds = int(smoothing["window_seconds"]) + parsed_points = _parse_hashrate_series_points(points) + dual_rate: list[dict[str, object]] = [] + for epoch, difficulty, point, window_total in _trailing_window_totals(parsed_points, effective_window_seconds): + if min_epoch is not None and epoch < min_epoch: + continue + try: + accepted_share_count = int( + Decimal(str(point.get("accepted_share_count", 0))) + ) + except (ValueError, InvalidOperation, OverflowError): + accepted_share_count = 0 + raw_hashrate_ths = hashrate_ths_from_difficulty(difficulty, bucket_seconds) + dual_rate.append( + { + "timestamp": point["timestamp"], + "raw_hashrate_ths": raw_hashrate_ths, + "smoothed_hashrate_ths": ( + hashrate_ths_from_difficulty(window_total, effective_window_seconds) + if trailing + else raw_hashrate_ths + ), + "accepted_share_count": accepted_share_count, + "accepted_share_difficulty": str(difficulty), + "complete": generated_at_epoch >= epoch + bucket_seconds, + } + ) + return dual_rate + + +def _parse_hashrate_series_points( + points: list[dict[str, object]], +) -> list[tuple[int, int, dict[str, object]]]: + """Return (epoch, difficulty, point) ascending by epoch, dropping malformed points.""" parsed_points: list[tuple[int, int, dict[str, object]]] = [] for point in points: try: @@ -1240,28 +1760,28 @@ def smooth_hashrate_series_points( continue parsed_points.append((epoch, difficulty, point)) parsed_points.sort(key=lambda entry: entry[0]) - smoothed: list[dict[str, object]] = [] + return parsed_points + + +def _trailing_window_totals( + parsed_points: list[tuple[int, int, dict[str, object]]], + window_seconds: int, +) -> Iterator[tuple[int, int, dict[str, object], int]]: + """Yield each parsed point with the difficulty total of its trailing window. + + The window covers buckets with epoch in (epoch - window_seconds, epoch]; + buckets missing from the series contribute nothing, so gaps count as zero + difficulty. + """ window: deque[tuple[int, int]] = deque() window_total = 0 for epoch, difficulty, point in parsed_points: - if bucket_count < 2: - if min_epoch is None or epoch >= min_epoch: - smoothed.append(point) - continue window.append((epoch, difficulty)) window_total += difficulty - while window and window[0][0] <= epoch - effective_window_seconds: + while window and window[0][0] <= epoch - window_seconds: _, expired_difficulty = window.popleft() window_total -= expired_difficulty - if min_epoch is not None and epoch < min_epoch: - continue - smoothed.append( - { - **point, - "hashrate_ths": hashrate_ths_from_difficulty(window_total, effective_window_seconds), - } - ) - return smoothed + yield epoch, difficulty, point, window_total def pagination(page: int, limit: int, total_count: int) -> dict[str, int]: @@ -1325,6 +1845,27 @@ def auto_bucket(range_id: str) -> str: return "1d" +def allowed_hashrate_bucket(range_id: str, bucket: str) -> None: + """Reject a bucket outside the range's static vocabulary. + + Called after auto resolution, so ``bucket`` is always concrete; auto + resolves inside every allowed set and can never be rejected here. A range + absent from HASHRATE_SERIES_ALLOWED_BUCKETS passes -- range validity is + the caller's own check, and this helper must stay reusable by every + bucketed public series endpoint, so it speaks only of buckets and ranges. + """ + allowed = HASHRATE_SERIES_ALLOWED_BUCKETS.get(range_id) + if allowed is None or bucket in allowed: + return + ordered = sorted(allowed, key=HASHRATE_SERIES_BUCKET_SECONDS.__getitem__) + raise PublicApiError( + 400, + "invalid_bucket", + f"bucket {bucket} is not allowed for range {range_id}; " + f"allowed: {', '.join(ordered)}", + ) + + def qbit_gbt_rules(chain: str) -> list[str]: rules = ["segwit"] if "signet" in chain.strip().lower(): @@ -1518,6 +2059,11 @@ def percent_string(part: int | str | Decimal, total: int | str | Decimal) -> str def owed_balance_for_recipient(ledger: Any, recipient_id: str) -> int: + read_model = getattr(ledger, "dashboard_miner_owed_balance_bits", None) + if callable(read_model): + return int(read_model(recipient_id=recipient_id)) + # In-memory ledger only: current_owed_balances() is a writer-lock read on + # the Postgres ledger, and returns every recipient's balance to filter one. return sum( int(balance.get("balance_sats", 0)) for balance in ledger.current_owed_balances() diff --git a/lab/prism/public_read_service.py b/lab/prism/public_read_service.py new file mode 100644 index 000000000..d9a02eba1 --- /dev/null +++ b/lab/prism/public_read_service.py @@ -0,0 +1,1535 @@ +#!/usr/bin/env python3 +"""Standalone process serving the PRISM public ``/public/v1`` read surface. + +The coordinator used to serve this surface from its own audit/ops listener, +under the GIL that acknowledges shares and lands blocks, against the primary +Postgres the lease-holding writer commits through. Public read traffic scales +with public interest rather than with hashrate, so it is the one workload whose +volume the pool does not control -- and it sat on the money path. This process +takes it off that path. + +What this is *not*: a rewrite. ``lab.prism.public_api`` reaches its host through +a deliberately small surface -- ``.ledger``, ``.rpc``, and an environment-backed +``getattr`` for the advertised Stratum endpoint -- so the extraction is a new +host object plus a small HTTP shell. Routing, payloads, cache policy and error +shapes stay in ``public_api``; ``tests/test_prism_public_read_service.py`` pins +this process's responses against ``public_api.dispatch`` directly. + +Three properties this process guarantees that the coordinator's listener did +not need to: + +Read-only in the strong sense + The ledger is constructed without any ``PRISM_LEDGER_WRITER_*`` input and + never acquires a writer lease: every public read model runs through + ``PsqlShareLedger._run_read_json`` (a bounded read slot), never through the + writer gate. The in-memory ledger is refused outright -- it lacks the + ``dashboard_*`` read models, so ``public_api``'s duck-typed fallbacks would + quietly reintroduce writer-lock reads (``current_owed_balances``, + ``recipient_payout_history``) on the public surface, which is precisely the + contention this extraction exists to remove. + +Fail-closed configuration + ``public_api.mining_configuration`` falls back to + ``PRISM_STRATUM_PORT``/``127.0.0.1`` when its host exposes no ``port`` or + ``bind``. This process deliberately exposes neither, so that fallback is + live -- and left alone it would advertise ``127.0.0.1:3340`` to the world as + the pool's Stratum endpoint. ``PRISM_PUBLIC_STRATUM_URL`` is therefore + required at startup rather than defaulted. + +An honest staleness contract + Every extracted response states the age it is willing to serve + (``X-Prism-Staleness-Budget-Seconds``) alongside the age it actually has + (``Age``), and refuses with 503 rather than serve past it. The budgets live + in ``lab.prism.endpoint_registry`` and are derived from the caches beneath + each route; see that module's docstring. This follows + ``lab/prism/observability.py``, where the cached ``/metrics`` endpoint + already returns 503 rather than serve an unbounded-stale snapshot. + + The same contract is what a database outage is answered with (#164). While + the readiness probe says Postgres is unreachable, a database-backed route + serves its warm cache entry if it has one inside that budget -- labelled as + degraded, so the client is told the difference between "30s old because + that is the poll interval" and "30s old because this is the last answer + that exists" -- and refuses with 503 otherwise. It never calls the database + to refill and never extends the entry it serves, so the budget stays the + real bound: ordinary traffic cannot keep a stale dashboard alive past it. + +Entry point: ``python3 -m lab.prism.public_read_service``. +""" + +from __future__ import annotations + +import ipaddress +import json +import math +import os +import re +import shlex +import signal +import socket +import sys +import threading +import time +import urllib.parse +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer +from pathlib import Path +from typing import Any, Callable + +if not __package__: + sys.path.insert(0, str(Path(__file__).resolve().parents[2])) + +from lab.prism import endpoint_registry, public_api +from lab.prism.audit_artifacts import AuditArtifactConfig, AuditArtifactStore +from lab.prism.coordinator_config import ( + DEFAULT_POSTGRES_IDLE_IN_TRANSACTION_TIMEOUT_SECONDS, + DEFAULT_POSTGRES_TCP_KEEPALIVES_COUNT, + DEFAULT_POSTGRES_TCP_KEEPALIVES_IDLE_SECONDS, + DEFAULT_POSTGRES_TCP_KEEPALIVES_INTERVAL_SECONDS, + env, + env_int, + env_nonnegative_float, + env_positive_float, + env_positive_int, +) +from lab.prism.rpc import JsonRpc +from lab.prism.share_ledger import LedgerOperationTimeout, PsqlShareLedger + + +DEFAULT_PUBLIC_API_BIND = "0.0.0.0" +DEFAULT_PUBLIC_API_PORT = 3342 + +# Deadline for one origin computation's database work. Sized above the ~15s +# nginx proxy timeout in front of this tier: a statement that outlives the +# client's patience by a little may still land a cache entry the retry can +# hit, but one that runs for minutes after the client hung up only holds a +# read slot against every other public read. +DEFAULT_PUBLIC_READ_STATEMENT_TIMEOUT_SECONDS = 20 + +# Emitted on every extracted response: the age this route is willing to serve. +# The observed age travels beside it in the standard Age header. +STALENESS_BUDGET_HEADER = "X-Prism-Staleness-Budget-Seconds" + +# What that header says for /public/v1/artifacts/{sha256}, the one route with no +# budget. A number would be a lie in either direction; this names the contract. +UNBOUNDED_STALENESS_BUDGET = "unbounded" + +# Said on every response served, or refused, while the readiness probe reports +# the database unreachable. Age and STALENESS_BUDGET_HEADER already carry the +# numbers; these two carry the fact, which the numbers alone cannot. Without +# them an ordinary 30-second-old poll response and the last answer that still +# exists are the same response. +# +# Warning 110 is the registered "Response is Stale" warn-code (RFC 7234 5.5.1, +# retained by clients that still parse the header), with this service named as +# the agent that added it. The X- header is what an operator greps and what a +# dashboard frontend switches an outage banner on, because it is a single value +# rather than a warn-code line to parse. +DEGRADED_WARNING_HEADER = "Warning" +DEGRADED_WARNING = '110 qbit-prism "database unavailable; serving cached response"' +DATABASE_STATE_HEADER = "X-Prism-Database-State" +DATABASE_STATE_UNAVAILABLE = "unavailable" +ARTIFACT_CANONICAL_STATE_HEADER = "X-Prism-Artifact-Canonical-State" + +# How often the background prober re-checks Postgres, and the age at which its +# answer stops counting. Same shape as observability.py's health staleness: +# three refresh intervals, floored, so ordinary jitter is not an outage. +DEFAULT_READINESS_PROBE_INTERVAL_SECONDS = 5.0 +MINIMUM_READINESS_STALE_SECONDS = 15.0 + +# How long the standby's replication stream may be silent before its answers +# stop counting as current. Bounds the *stream*, not replay lag -- see +# ReplicaProbe. +DEFAULT_REPLICA_MAX_LAG_SECONDS = 60.0 + +# Observed replay lag, published on every replica-backed response. Advisory: +# the enforced bound is the heartbeat age, for the reason ReplicaProbe explains. +REPLICA_LAG_HEADER = "X-Prism-Replica-Lag-Seconds" + +# require: refuse replica-backed routes unless the backing server is a standby +# whose replication stream is live within the bound. off: serve whatever the +# DSN names, which is the behaviour of the extraction before a standby existed. +# off is the default so that merging this does not 503 a deployment that has +# not provisioned one yet; the shipped compose sets require. +REPLICA_MODE_REQUIRE = "require" +REPLICA_MODE_OFF = "off" +REPLICA_MODES = (REPLICA_MODE_REQUIRE, REPLICA_MODE_OFF) + +HEALTH_SCHEMA = "qbit.prism.public-read-health.v1" + + +class PublicReadConfigurationError(RuntimeError): + """Startup refused because the service would otherwise serve wrong facts.""" + + +class PublicReadCoordinator: + """The coordinator stand-in ``lab.prism.public_api`` reads through. + + ``public_api`` needs exactly ``.ledger`` (24 references) and ``.rpc`` + (getblockchaininfo / getblocktemplate / getnetworkinfo). It also probes + ``getattr(coordinator, "port")`` and ``getattr(coordinator, "bind")`` in + ``mining_configuration``, both of which already fall back to the + environment. + + Those two attributes are deliberately absent here. On the coordinator they + named the Stratum listener that process was itself running; this process + runs no Stratum listener, so any value it invented would be a guess + published to miners as connection instructions. Leaving them undefined + routes the lookup to the environment, and ``require_public_stratum_url()`` + makes the operator state the answer rather than inherit ``127.0.0.1``. + """ + + def __init__(self, *, ledger: Any, rpc: Any) -> None: + self.ledger = ledger + self.rpc = rpc + + +class ServiceMetrics: + """Counters for this process only, rendered as Prometheus text. + + Intentionally small: the coordinator keeps its own /metrics with the pool's + real operational signal. What is worth knowing here is whether the tier is + serving, whether its response cache is earning its keep, and whether the + staleness contract is actually refusing anything. + """ + + def __init__(self) -> None: + self._lock = threading.Lock() + self._requests_total = 0 + self._responses_by_status: dict[int, int] = {} + # "stale" is a served response: an expired entry inside its + # stale-while-revalidate window, answered immediately while one + # background refresh recomputes it. + self._cache_by_state: dict[str, int] = { + "hit": 0, + "miss": 0, + "bypass": 0, + "stale": 0, + } + self._staleness_refusals = 0 + self._replica_refusals = 0 + self._degraded_responses = 0 + self._database_outage_refusals = 0 + + def record_request(self) -> None: + with self._lock: + self._requests_total += 1 + + def record_response(self, status: int) -> None: + with self._lock: + self._responses_by_status[status] = ( + self._responses_by_status.get(status, 0) + 1 + ) + + def record_cache_state(self, cache_state: str) -> None: + key = cache_state.lower() + with self._lock: + if key in self._cache_by_state: + self._cache_by_state[key] += 1 + + def record_staleness_refusal(self) -> None: + with self._lock: + self._staleness_refusals += 1 + + def record_replica_refusal(self) -> None: + with self._lock: + self._replica_refusals += 1 + + def record_degraded_response(self) -> None: + with self._lock: + self._degraded_responses += 1 + + def record_database_outage_refusal(self) -> None: + with self._lock: + self._database_outage_refusals += 1 + + def render( + self, + *, + ready: bool, + readiness_age_seconds: float, + replica: dict[str, object] | None = None, + ) -> str: + with self._lock: + requests_total = self._requests_total + responses = dict(self._responses_by_status) + cache_states = dict(self._cache_by_state) + refusals = self._staleness_refusals + replica_refusals = self._replica_refusals + degraded_responses = self._degraded_responses + database_outage_refusals = self._database_outage_refusals + lines = [ + "# HELP qbit_prism_public_requests_total Public read requests accepted.", + "# TYPE qbit_prism_public_requests_total counter", + f"qbit_prism_public_requests_total {requests_total}", + "# HELP qbit_prism_public_responses_total Public read responses by status code.", + "# TYPE qbit_prism_public_responses_total counter", + ] + lines.extend( + f'qbit_prism_public_responses_total{{status="{status}"}} {count}' + for status, count in sorted(responses.items()) + ) + lines.extend( + [ + "# HELP qbit_prism_public_cache_total Response-cache outcomes by state.", + "# TYPE qbit_prism_public_cache_total counter", + ] + ) + lines.extend( + f'qbit_prism_public_cache_total{{state="{state}"}} {count}' + for state, count in sorted(cache_states.items()) + ) + lines.extend( + [ + "# HELP qbit_prism_public_staleness_refusals_total Responses refused for exceeding their staleness budget.", + "# TYPE qbit_prism_public_staleness_refusals_total counter", + f"qbit_prism_public_staleness_refusals_total {refusals}", + "# HELP qbit_prism_public_ledger_ready Whether the last Postgres readiness probe succeeded.", + "# TYPE qbit_prism_public_ledger_ready gauge", + f"qbit_prism_public_ledger_ready {1 if ready else 0}", + "# HELP qbit_prism_public_ledger_probe_age_seconds Age of the last readiness probe, or -1 before the first.", + "# TYPE qbit_prism_public_ledger_probe_age_seconds gauge", + f"qbit_prism_public_ledger_probe_age_seconds {readiness_age_seconds:.3f}", + "# HELP qbit_prism_public_replica_refusals_total Responses refused for failing the replica freshness contract.", + "# TYPE qbit_prism_public_replica_refusals_total counter", + f"qbit_prism_public_replica_refusals_total {replica_refusals}", + "# HELP qbit_prism_public_degraded_responses_total Cached responses served while the database was unreachable.", + "# TYPE qbit_prism_public_degraded_responses_total counter", + f"qbit_prism_public_degraded_responses_total {degraded_responses}", + "# HELP qbit_prism_public_database_outage_refusals_total Responses refused because the database was unreachable and no warm entry existed.", + "# TYPE qbit_prism_public_database_outage_refusals_total counter", + f"qbit_prism_public_database_outage_refusals_total {database_outage_refusals}", + ] + ) + if replica is not None: + # -1 rather than omission for the unknowns: a missing series and a + # series reading zero are the same alert, and neither is true. + lines.extend( + [ + "# HELP qbit_prism_public_replica_in_recovery Whether the backing server is a standby.", + "# TYPE qbit_prism_public_replica_in_recovery gauge", + f"qbit_prism_public_replica_in_recovery {1 if replica.get('in_recovery') else 0}", + "# HELP qbit_prism_public_replica_heartbeat_age_seconds Age of the newest walreceiver message, or -1 when disconnected. This is the enforced bound.", + "# TYPE qbit_prism_public_replica_heartbeat_age_seconds gauge", + f"qbit_prism_public_replica_heartbeat_age_seconds {_metric_number(replica.get('receiver_heartbeat_age_seconds'))}", + "# HELP qbit_prism_public_replica_replay_lag_seconds Wall clock behind the newest replayed commit, or -1. Advisory: grows on an idle primary.", + "# TYPE qbit_prism_public_replica_replay_lag_seconds gauge", + f"qbit_prism_public_replica_replay_lag_seconds {_metric_number(replica.get('replay_lag_seconds'))}", + "# HELP qbit_prism_public_replica_apply_backlog_bytes WAL received but not yet replayed, or -1.", + "# TYPE qbit_prism_public_replica_apply_backlog_bytes gauge", + f"qbit_prism_public_replica_apply_backlog_bytes {_metric_number(replica.get('apply_backlog_bytes'))}", + "# HELP qbit_prism_public_replica_max_lag_seconds The configured replication-stream silence bound.", + "# TYPE qbit_prism_public_replica_max_lag_seconds gauge", + f"qbit_prism_public_replica_max_lag_seconds {_metric_number(replica.get('max_lag_seconds'))}", + ] + ) + return "\n".join(lines) + "\n" + + +class ReplicaProbe: + """Is the standby this tier reads through still a standby, and still fed? + + Two refusals and one number, from one query per interval: + + * **Not in recovery.** The public tier exists to keep public read volume + off the primary the coordinator lands blocks through. A DSN that + resolves to a writable server silently undoes that, and the failure is + invisible in the responses -- they are correct, just served from the + wrong place. So it is refused rather than served. + * **A silent replication stream.** The liveness proof is the walreceiver + heartbeat age, *not* the transactional replay lag. Heartbeats flow every + ``wal_receiver_status_interval`` (10s by default) whether or not the + primary has anything to send, whereas replay lag grows with wall clock + on an idle primary -- so bounding replay lag would 503 a perfectly + healthy replica of a quiet pool every time the pool went quiet, which is + exactly when the dashboard is least likely to be wrong. + + Replay lag and apply backlog are still measured and published (header, + /healthz, /metrics); they are the number an operator wants, just not the + number a gate can be built on. + + Freshness ages locally, which is what makes this fail *closed*. The bound + is checked against the heartbeat age the last successful probe reported + **plus how long ago that probe ran**, so a probe that starts failing -- + or a standby that stops answering entirely -- ages its own last good + answer out of the bound instead of pinning it there forever. + + Runs as ReadinessProbe's probe callable rather than owning a thread: + readiness and replica freshness are the same question, and one probe means + one round trip per interval instead of two. + """ + + def __init__( + self, + status_probe: Callable[[], dict[str, object]], + *, + max_lag_seconds: float = DEFAULT_REPLICA_MAX_LAG_SECONDS, + monotonic: Callable[[], float] = time.monotonic, + ) -> None: + self._status_probe = status_probe + self._max_lag_seconds = float(max_lag_seconds) + self._monotonic = monotonic + self._lock = threading.Lock() + self._status: dict[str, object] | None = None + self._probed_at: float | None = None + self._last_error: str | None = None + + @property + def max_lag_seconds(self) -> float: + return self._max_lag_seconds + + def check(self) -> bool: + """One probe. True when the backing server is a live standby. + + Failure is recorded and re-raised so ReadinessProbe reports the real + error text rather than a bare "returned false"; the recorded snapshot + is deliberately left in place, to age out through freshness_error() + rather than vanish and read as warm-up. + """ + try: + status = self._status_probe() + except Exception as exc: # noqa: BLE001 - recorded, then re-raised + with self._lock: + self._last_error = f"{type(exc).__name__}: {exc}" + raise + with self._lock: + self._status = dict(status) + self._probed_at = self._monotonic() + self._last_error = None + return bool(status.get("in_recovery")) + + def view(self) -> dict[str, object]: + """What /healthz and /metrics report, including the aged numbers.""" + + with self._lock: + status = dict(self._status) if self._status is not None else None + probed_at = self._probed_at + last_error = self._last_error + payload: dict[str, object] = { + "probed": status is not None, + "max_lag_seconds": self._max_lag_seconds, + } + if last_error is not None: + payload["last_error"] = last_error + if status is None or probed_at is None: + return payload + snapshot_age = max(0.0, self._monotonic() - probed_at) + payload["probe_age_seconds"] = round(snapshot_age, 3) + payload["in_recovery"] = bool(status.get("in_recovery")) + heartbeat = _coerce_float(status.get("receiver_heartbeat_age_seconds")) + payload["receiver_heartbeat_age_seconds"] = ( + None if heartbeat is None else round(heartbeat + snapshot_age, 3) + ) + replay_lag = _coerce_float(status.get("replay_lag_seconds")) + payload["replay_lag_seconds"] = ( + None if replay_lag is None else round(replay_lag + snapshot_age, 3) + ) + backlog = _coerce_float(status.get("apply_backlog_bytes")) + payload["apply_backlog_bytes"] = None if backlog is None else int(backlog) + return payload + + def replay_lag_seconds(self) -> float | None: + """Observed replay lag for the response header, aged locally.""" + + value = self.view().get("replay_lag_seconds") + return None if value is None else float(value) + + def freshness_error(self) -> tuple[str, str] | None: + """(code, message) for the 503 a replica-backed route owes, or None.""" + + with self._lock: + status = dict(self._status) if self._status is not None else None + probed_at = self._probed_at + last_error = self._last_error + if status is None or probed_at is None: + message = "public read service is warming up" + if last_error is not None: + message = f"{message}: replica probe failed: {last_error}" + return "replica_unavailable", message + if not status.get("in_recovery"): + return ( + "replica_unavailable", + "public read service refuses to serve from a server that is " + "not in recovery; point PRISM_PUBLIC_DATABASE_URL at the read " + "replica, or set PRISM_PUBLIC_REPLICA_MODE=off to serve from " + "the primary deliberately", + ) + heartbeat = _coerce_float(status.get("receiver_heartbeat_age_seconds")) + if heartbeat is None: + return ( + "replica_unavailable", + "read replica replication stream is not connected; the " + "walreceiver heartbeat is absent", + ) + silent_for = heartbeat + max(0.0, self._monotonic() - probed_at) + if silent_for > self._max_lag_seconds: + detail = "" + if last_error is not None: + detail = f" (last probe failed: {last_error})" + return ( + "replica_unavailable", + f"read replica replication stream has been silent for " + f"{silent_for:.1f}s, beyond the configured bound " + f"{self._max_lag_seconds:.1f}s{detail}", + ) + return None + + +def _coerce_float(value: object) -> float | None: + """Postgres json numbers arrive as int/float/str depending on the driver.""" + + if value is None: + return None + try: + return float(value) # type: ignore[arg-type] + except (TypeError, ValueError): + return None + + +class ReadinessProbe: + """Background Postgres liveness, so /healthz never queries on the request thread. + + The compose healthcheck polls this forever. Probing inline would put a + database round trip -- and a read slot -- on every poll, and a stalled + database would hang the health request rather than answer it. Instead one + background thread runs the constant-select probe on an interval and + publishes the result; the request thread only reads two fields. + """ + + def __init__( + self, + probe: Callable[[], bool], + *, + interval_seconds: float = DEFAULT_READINESS_PROBE_INTERVAL_SECONDS, + monotonic: Callable[[], float] = time.monotonic, + ) -> None: + self._probe = probe + self._interval_seconds = max(0.1, float(interval_seconds)) + self._monotonic = monotonic + self._lock = threading.Lock() + self._ok = False + self._error: str | None = None + self._checked_at: float | None = None + self._stop = threading.Event() + self._thread: threading.Thread | None = None + + @property + def stale_after_seconds(self) -> float: + return max(3 * self._interval_seconds, MINIMUM_READINESS_STALE_SECONDS) + + def check_once(self) -> None: + try: + ok = bool(self._probe()) + error = None if ok else "readiness probe returned false" + except Exception as exc: # noqa: BLE001 - any failure means "not ready" + ok = False + error = f"{type(exc).__name__}: {exc}" + with self._lock: + self._ok = ok + self._error = error + self._checked_at = self._monotonic() + + def start(self) -> None: + # One synchronous probe first so a freshly started service reports its + # real state immediately instead of a warm-up 503 the healthcheck would + # have to wait out. + self.check_once() + self._thread = threading.Thread( + target=self._run, + name="prism-public-readiness", + daemon=True, + ) + self._thread.start() + + def _run(self) -> None: + while not self._stop.wait(self._interval_seconds): + self.check_once() + + def stop(self) -> None: + self._stop.set() + thread = self._thread + if thread is not None: + thread.join(timeout=5) + + def age_seconds(self) -> float: + with self._lock: + checked_at = self._checked_at + if checked_at is None: + return -1.0 + return max(0.0, self._monotonic() - checked_at) + + def snapshot(self) -> tuple[int, dict[str, object]]: + with self._lock: + ok = self._ok + error = self._error + checked_at = self._checked_at + if checked_at is None: + return 503, { + "ok": False, + "schema": HEALTH_SCHEMA, + "state": "starting", + "error": "readiness probe has not completed yet", + "probe_age_seconds": -1.0, + } + age_seconds = max(0.0, self._monotonic() - checked_at) + stale = age_seconds > self.stale_after_seconds + payload: dict[str, object] = { + "ok": bool(ok and not stale), + "schema": HEALTH_SCHEMA, + "state": "ready" if (ok and not stale) else "unready", + "probe_age_seconds": round(age_seconds, 3), + "probe_stale_after_seconds": self.stale_after_seconds, + } + if stale: + payload["error"] = "readiness probe is stale" + elif error is not None: + payload["error"] = error + return (200 if payload["ok"] else 503), payload + + +def path_reads_database(path: str) -> bool: + """Does this route's answer come out of the database? + + One predicate, two gates. The replica contract asks it to decide which + routes a silent standby may not answer; the outage gate asks it to decide + which routes a dead database may not answer. The exempt set is the same + both times, and for the same reasons, so it is derived once here rather + than stated twice. + + Derived from the endpoint registry rather than a hand-kept path list, so a + route added there is classified once. Two kinds of route are exempt: + + * those that touch no read slot at all -- today only + /public/v1/mining-configuration, assembled from environment. It has no + database answer to lose; + * content-addressed ones, which declare ``immutable_content``. A body that + hashes to the requested sha256 is correct at any age, which is why the + registry gives them no staleness budget either; refusing one for + replication lag, or for an outage, would contradict the unbounded budget + already published on it. + + An unclassified path is not database-backed here because dispatch() 404s it + before it can read anything. + """ + endpoint = endpoint_registry.endpoint_for_request_path(path) + if endpoint is None or endpoint.immutable_content: + return False + return endpoint_registry.LedgerAccess.READ_SLOT in endpoint.access + + +def path_occupies_read_slot(path: str) -> bool: + """Does serving this route's origin miss take a ledger read slot? + + Deliberately not ``path_reads_database``: that predicate exempts + content-addressed routes because an immutable body is correct at any age, + which is an answer about *staleness* -- replica lag and outages. The + statement deadline asks a different question: does a cold request occupy + one of the bounded Postgres read slots while it computes? A cold + ``/public/v1/artifacts/{sha256}`` does exactly that, for as long as the + lookup takes, so the deadline must cover it even though the staleness + gates never should. + """ + endpoint = endpoint_registry.endpoint_for_request_path(path) + if endpoint is None: + return False + return endpoint_registry.LedgerAccess.READ_SLOT in endpoint.access + + +def public_read_statement_timeout_seconds() -> int: + """The per-request database deadline, in seconds. 0 disables the bound. + + Read per request rather than captured at startup, so the knob behaves + like the cache TTL knobs beside it and a background cache refresh sees + the same value a foreground request would. + """ + return public_api.env_nonnegative_int( + "PRISM_PUBLIC_READ_STATEMENT_TIMEOUT_SECONDS", + DEFAULT_PUBLIC_READ_STATEMENT_TIMEOUT_SECONDS, + ) + + +def bounded_public_dispatch( + coordinator: Any, + path: str, + query: dict[str, list[str]], + *, + occupies_read_slot: bool, +) -> tuple[int, object]: + """One origin computation, deadline-bounded when it takes a read slot. + + Without a bound, a request the fronting proxy has already abandoned at + its ~15s timeout keeps its Postgres statement running to completion, + holding one of the bounded read slots (PRISM_POSTGRES_READ_CONCURRENCY) + against every other public read and competing with the share writer's + I/O for as long as the scan takes. The ledger's ``statement_timeout`` + scope arms a server-side ``SET LOCAL statement_timeout`` per statement + plus a matching local admission bound, so the whole request stops costing + anything shortly after its caller stops listening. + + The scope is the ledger's ``operation_timeout``, not ``statement_timeout``: + the latter hands every admission and SQL statement a fresh budget, so a + route that performs several sequential ledger reads (the miner detail + page does) could run many multiples of the configured limit after the + proxy hung up. ``operation_timeout`` is one deadline counting down across + the entire dispatch -- every statement still gets server-side + cancellation, armed with whatever remains of the request's budget. + + Duck-typed exactly like the coordinator's block-submitter scope: a ledger + without the scope runs unbounded, as it always has. On the ledger's + psql-subprocess fallback the server-side statement deadline is not armed; + admission waits still observe the bound, and no deadline at all is an + acceptable degradation there. Routes are wrapped by + ``path_occupies_read_slot`` -- registry READ_SLOT access, immutable or + not -- rather than by the staleness gates' ``path_reads_database``: a + cold content-addressed artifact lookup holds a read slot exactly like any + other cold read, even though its body, once found, is correct at any + age. + + A ledger-reported deadline becomes the public ``read_timeout`` 503. Its + error response is no-store and never cached, so one expensive miss + cannot poison the response cache with a refusal. + """ + if not occupies_read_slot: + return public_api.dispatch(coordinator, path, query) + timeout_seconds = public_read_statement_timeout_seconds() + operation_timeout = getattr(coordinator.ledger, "operation_timeout", None) + if timeout_seconds <= 0 or not callable(operation_timeout): + return public_api.dispatch(coordinator, path, query) + try: + with operation_timeout(float(timeout_seconds)): + return public_api.dispatch(coordinator, path, query) + except LedgerOperationTimeout as exc: + raise public_api.PublicApiError( + 503, + "read_timeout", + "the read timed out; try again shortly", + ) from exc + + +def staleness_budget_for_path(path: str) -> float | None: + """The budget for one concrete request path, or None when it has none. + + None means "never refuse this for age", which today is only + /public/v1/artifacts/{sha256} -- content-addressed and immutable, so a body + that hashes to the requested sha256 is correct however old it is. An + unclassified path also lands here, but dispatch() 404s it before any + staleness question arises. + """ + endpoint = endpoint_registry.endpoint_for_request_path(path) + if endpoint is None or endpoint.immutable_content: + return None + return endpoint.max_staleness_seconds + + +def staleness_budget_header_value(path: str) -> str | None: + """What X-Prism-Staleness-Budget-Seconds says for one path, or None to omit. + + Every classified route publishes a budget, including the immutable one: + silence there would be indistinguishable from a route that simply forgot, + and "unbounded" is the actual contract -- a content-addressed body is + correct at any age. A path that is not a route at all publishes nothing, + because there is no route whose budget it could be. + """ + endpoint = endpoint_registry.endpoint_for_request_path(path) + if endpoint is None: + return None + if endpoint.immutable_content: + return UNBOUNDED_STALENESS_BUDGET + if endpoint.max_staleness_seconds is None: + return None + return _format_budget(endpoint.max_staleness_seconds) + + +# The refusal body, built once so the handler can recognise it by identity. +# +# Reported under ``stale_read_model`` because ``public_api`` owns the public +# error enum and that is the entry whose meaning this is: no answer for this +# route exists inside the age it publishes. It is the same documented wire code +# (``upstream_unavailable``) the replica refusal already resolves to -- from a +# client's side the conditions are one condition -- and inventing an +# undocumented code instead would silently ship a 503 that says "internal +# error", because error_payload() rewrites codes it does not know. +DATABASE_UNAVAILABLE_PAYLOAD = public_api.error_payload( + "stale_read_model", + "public read service cannot reach the database and has no cached response " + "for this request inside its staleness budget", +) + + +def database_unavailable_response() -> tuple[int, dict[str, object]]: + """The 503 a database-backed route owes while the database is unreachable. + + Shaped as a ``compute`` for ``PublicResponseCache.get_or_compute`` because + that is how the warm/cold distinction gets made without a second cache API: + the cache answers a warm key from its entry and never calls this, and on a + cold, bypassed or expired key it calls this and stores nothing, because the + cache only stores 2xx. So the same call expresses "serve what you have" and + "refuse, and do not go and ask". + """ + + return 503, DATABASE_UNAVAILABLE_PAYLOAD + + +def staleness_error_payload(*, budget_seconds: float, age_seconds: int) -> dict[str, object]: + """A refusal in the existing public error schema, naming both numbers. + + Uses public_api.error_payload so the body is prism.dashboard.error.v1 like + every other public error; a bespoke schema here would make the one response + a client most needs to parse the one it cannot. + """ + return public_api.error_payload( + "stale_read_model", + ( + "cached response age " + f"{age_seconds}s exceeds the staleness budget of " + f"{budget_seconds:g}s for this endpoint" + ), + ) + + +class PublicReadService: + """One public read tier instance: what to read through, and what to remember. + + Holds the collaborators a handler needs -- the ``public_api`` host object, + the shared response cache, this process's counters, and the background + readiness probe -- so that binding a socket is a separate, trivial step. + Tests construct one of these around a fake coordinator and get the real + request path without a database, an RPC node, or a signal handler. + """ + + def __init__( + self, + coordinator: Any, + *, + response_cache: public_api.PublicResponseCache | None = None, + metrics: ServiceMetrics | None = None, + readiness: ReadinessProbe | None = None, + replica: ReplicaProbe | None = None, + ) -> None: + self.coordinator = coordinator + self.cache = ( + public_api.PublicResponseCache() if response_cache is None else response_cache + ) + self.metrics = ServiceMetrics() if metrics is None else metrics + self.readiness = readiness + # None when PRISM_PUBLIC_REPLICA_MODE=off: no replica contract is + # enforced and no replica facts are published, because there are none. + self.replica = replica + + +class BoundedPublicHTTPServer(ThreadingHTTPServer): + """Admit a fixed number of connections without a waiting worker queue.""" + + daemon_threads = True + + def __init__(self, address, handler, *, max_connections=64, timeout_seconds=10.0): + if not 1 <= max_connections <= 1024: + raise PublicReadConfigurationError("PRISM_PUBLIC_HTTP_MAX_CONNECTIONS must be 1-1024") + if not math.isfinite(timeout_seconds) or timeout_seconds <= 0: + raise PublicReadConfigurationError("PRISM_PUBLIC_HTTP_TIMEOUT_SECONDS must be finite and positive") + self.connection_timeout = timeout_seconds + self._connection_slots = threading.BoundedSemaphore(max_connections) + super().__init__(address, handler) + + def process_request(self, request, client_address): + if not self._connection_slots.acquire(blocking=False): + # Do not block the accept loop writing a refusal to an idle peer. + self.shutdown_request(request) + return + try: + super().process_request(request, client_address) + except BaseException: + self._connection_slots.release() + raise + + def process_request_thread(self, request, client_address): + try: + super().process_request_thread(request, client_address) + finally: + self._connection_slots.release() + + +def make_handler(service: PublicReadService) -> type[BaseHTTPRequestHandler]: + """Build the request handler for one public read service instance. + + Deliberately not ``AuditHttpFacade``: that class's cancel-startup and + bounded-serve-exit machinery encodes shutdown races specific to starting and + stopping a listener inside the coordinator's long-lived process. This one + binds, serves, and exits on a signal, so it needs none of it. + """ + + coordinator = service.coordinator + cache = service.cache + service_metrics = service.metrics + readiness = service.readiness + replica = service.replica + + class PublicReadHandler(BaseHTTPRequestHandler): + server_version = "PrismPublicRead/1" + protocol_version = "HTTP/1.1" + + def setup(self) -> None: + self.connection_timeout = getattr(self.server, "connection_timeout", 10.0) + self.request.settimeout(self.connection_timeout) + super().setup() + + def handle_one_request(self) -> None: + # An idle socket timeout alone allows a slow drip to retain a slot. + # One timer per admitted worker bounds the entire request line and + # header phase, then is retired before application dispatch. + self._headers_expired = threading.Event() + + def expire_headers() -> None: + self._headers_expired.set() + try: + self.connection.shutdown(socket.SHUT_RDWR) + except OSError: + pass + + self._header_timer = threading.Timer(self.connection_timeout, expire_headers) + self._header_timer.daemon = True + self._header_timer.start() + try: + super().handle_one_request() + finally: + self._finish_headers() + self.close_connection = True + + def _finish_headers(self) -> None: + timer = self._header_timer + if timer is not None: + timer.cancel() + timer.join() + # The callback captures this handler. Break that ownership + # cycle so completed connections retire without cyclic GC. + self._header_timer = None + + def parse_request(self) -> bool: + try: + parsed = super().parse_request() + finally: + self._finish_headers() + # The stdlib parser can accept EOF as the end of headers. A timer + # shutdown must not turn an incomplete request into application work. + return parsed and not self._headers_expired.is_set() + + def end_headers(self) -> None: + # One response per connection prevents idle keep-alive clients + # from reserving the bounded worker pool between requests. + self.send_header("Connection", "close") + super().end_headers() + + def do_GET(self) -> None: # noqa: N802 - BaseHTTPRequestHandler API + parsed = urllib.parse.urlparse(self.path) + # Same normalization the coordinator listener applied, so a + # trailing slash resolves to the same route it always did. + path = parsed.path.rstrip("/") or "/" + query = urllib.parse.parse_qs(parsed.query) + service_metrics.record_request() + try: + if path == "/healthz": + self.handle_healthz() + return + if path == "/metrics": + self.handle_metrics() + return + if path == "/public/v1" or path.startswith("/public/v1/"): + self.handle_public(path, query) + return + self.write_json(404, {"error": "unknown endpoint"}) + except public_api.PublicApiError as exc: + self.write_json( + exc.status, + public_api.error_payload(exc.code, exc.message), + headers=self.with_budget(path, public_api.public_error_headers()), + ) + except Exception: # noqa: BLE001 - never leak internals to the public + # Matches the coordinator listener's old behaviour for + # /public/v1: a generic 500 in the public error schema, with no + # exception text. ValueError was folded in there too -- it took + # the same branch -- so it needs no separate arm here. + self.write_json( + 500, + public_api.error_payload("internal_error", "internal server error"), + headers=self.with_budget(path, public_api.public_error_headers()), + ) + + def handle_public(self, path: str, query: dict[str, list[str]]) -> None: + reads_database = path_reads_database(path) + # Before the cache, not after. A replica outage does not make + # cached entries expire, so a post-cache check would keep answering + # 200 from the last good snapshot for as long as traffic kept the + # entry warm -- the precise failure this gate exists to prevent. + if replica is not None and reads_database: + refusal = replica.freshness_error() + if refusal is not None: + code, message = refusal + service_metrics.record_replica_refusal() + headers = self.with_replica_lag( + public_api.public_error_headers() + ) + self.write_json( + 503, + public_api.error_payload(code, message), + headers=self.with_budget(path, headers), + ) + return + # Also before the cache, and before any origin computation: while + # readiness says the database is unreachable, a database-backed + # route may still answer from what it already has, but it may not + # go and ask. Handing the cache a non-cacheable 503 instead of + # dispatch() is exactly that instruction -- a warm key returns its + # entry and never reaches this, a cold, bypassed or expired one + # gets the refusal and stores nothing. + degraded = ( + readiness is not None + and reads_database + and readiness.snapshot()[0] != 200 + ) + cache_policy = public_api.public_cache_policy(path) + budget_seconds = staleness_budget_for_path(path) + # The stale-while-revalidate window the policy advertises to CDNs + # also lets the in-process cache serve an expired entry while one + # background refresh recomputes it -- but the budget check below + # is unconditional, so the window is clamped to what the route's + # staleness budget leaves after the TTL. Without the clamp, a + # route whose ttl+swr exceeds its budget (the 5s routes: 5+30 + # against a 15s budget) would turn ages the blocking path used to + # recompute into 503 refusals. While the database is unreachable + # the window is withdrawn entirely: the outage contract serves + # only unexpired entries, and its refusal compute must not be + # deferred to a background thread nobody answers. + ttl_seconds = cache_policy.ttl_seconds + swr_seconds = cache_policy.stale_while_revalidate_seconds + if budget_seconds is not None: + ttl_seconds = max(0, min(ttl_seconds, int(budget_seconds))) + swr_seconds = max(0, min(swr_seconds, int(budget_seconds) - ttl_seconds)) + if degraded: + swr_seconds = 0 + # Use the same effective policy for origin storage and CDN headers. + # Otherwise downstream stale serving can outlive this route's budget. + cache_policy = public_api.PublicCachePolicy( + ttl_seconds=ttl_seconds, + stale_while_revalidate_seconds=swr_seconds, + immutable=cache_policy.immutable, + ) + status, payload, cache_state, age_seconds = cache.get_or_compute( + key=public_api.public_cache_key(path, query), + ttl_seconds=cache_policy.ttl_seconds, + stale_while_revalidate_seconds=swr_seconds, + compute=( + database_unavailable_response + if degraded + else lambda: bounded_public_dispatch( + coordinator, + path, + query, + occupies_read_slot=path_occupies_read_slot(path), + ) + ), + ) + service_metrics.record_cache_state(cache_state) + if payload is DATABASE_UNAVAILABLE_PAYLOAD: + # Nothing warm to serve. Recognised by identity rather than + # from this request's own `degraded`, because get_or_compute + # coalesces: a request that arrived while another was computing + # gets that one's result whatever its own view of readiness + # was. Keying on the request would ship this 503 with ordinary + # shared-cache headers to a waiter whose probe had recovered in + # between, and a CDN would then hold the refusal. + service_metrics.record_database_outage_refusal() + headers = self.with_replica_lag(public_api.public_error_headers()) + headers[DATABASE_STATE_HEADER] = DATABASE_STATE_UNAVAILABLE + self.write_json( + status, payload, headers=self.with_budget(path, headers) + ) + return + if isinstance(payload, public_api.UncacheableJsonBody): + # Legacy audit rows can predate canonical artifact storage. + # Their reconstructed JSON remains available for continuity, + # but it must never be cached under an immutable content URL. + headers = self.with_replica_lag(public_api.public_error_headers()) + headers[ARTIFACT_CANONICAL_STATE_HEADER] = payload.reason + self.write_json( + status, + payload.payload, + headers=self.with_budget(path, headers), + ) + return + if budget_seconds is not None and age_seconds > budget_seconds: + # Refuse rather than serve past the budget. A silently stale + # dashboard is worse than an honest one. + service_metrics.record_staleness_refusal() + headers = public_api.public_error_headers() + headers[STALENESS_BUDGET_HEADER] = _format_budget(budget_seconds) + headers["Age"] = str(max(0, age_seconds)) + if degraded: + # Stated, but deliberately not counted as an outage + # refusal: the budget check is unconditional, so this entry + # would have been refused with a healthy database too. The + # outage is why no fresher one replaced it, which is worth + # saying and is not the same claim. + headers[DATABASE_STATE_HEADER] = DATABASE_STATE_UNAVAILABLE + self.write_json( + 503, + staleness_error_payload( + budget_seconds=budget_seconds, + age_seconds=age_seconds, + ), + headers=self.with_replica_lag(headers), + ) + return + headers = public_api.public_cache_headers( + cache_policy, + cache_state=cache_state, + age_seconds=age_seconds, + ) + headers = self.with_replica_lag(headers) + if degraded and cache_state == "HIT": + # A cached body, inside the budget the same response publishes, + # served with the outage stated rather than implied. The + # shared-cache headers are deliberately left as they were: the + # CDN TTL is the route's own, which the registry derives at a + # third of the budget or less, so a downstream cache cannot + # carry this body past the bound stated on it. + # + # Conditioned on the hit, not on `degraded` alone: a coalescing + # waiter can receive an origin result computed by a request + # that still saw a healthy database, and labelling that fresh + # body "serving cached response" would be untrue. + service_metrics.record_degraded_response() + headers[DEGRADED_WARNING_HEADER] = DEGRADED_WARNING + headers[DATABASE_STATE_HEADER] = DATABASE_STATE_UNAVAILABLE + self.write_json(status, payload, headers=self.with_budget(path, headers)) + + def with_replica_lag(self, headers: dict[str, str]) -> dict[str, str]: + """Publish observed replay lag beside the enforced heartbeat bound. + + Advisory, and deliberately so: this is the number that answers "how + far behind is the data I just got", which the enforced bound does + not. Omitted rather than guessed when nothing has replayed yet. + """ + + if replica is None: + return headers + lag_seconds = replica.replay_lag_seconds() + if lag_seconds is not None: + headers[REPLICA_LAG_HEADER] = f"{lag_seconds:.3f}" + return headers + + def with_budget(self, path: str, headers: dict[str, str]) -> dict[str, str]: + """Attach the route's budget so callers can see it on every response.""" + + budget = staleness_budget_header_value(path) + if budget is not None: + headers[STALENESS_BUDGET_HEADER] = budget + return headers + + def handle_healthz(self) -> None: + if readiness is None: + payload: dict[str, object] = {"ok": True, "schema": HEALTH_SCHEMA} + status = 200 + else: + status, payload = readiness.snapshot() + if replica is not None: + # The replication facts are why a 503 here says what it says, + # so they travel with it rather than only in /metrics. + payload["replica"] = replica.view() + refusal = replica.freshness_error() + if refusal is not None: + payload["ok"] = False + payload["state"] = "unready" + # Overwrite rather than defer to the readiness probe's + # text: in require mode the replica check *is* the + # readiness probe, so a failed contract shows up there as + # the useless "readiness probe returned false". The + # probe's own exception, when there was one, is in the + # replica block beside this. + payload["error"] = refusal[1] + status = 503 + self.write_json(status, payload) + + def handle_metrics(self) -> None: + ready = True + age_seconds = -1.0 + if readiness is not None: + status, _ = readiness.snapshot() + ready = status == 200 + age_seconds = readiness.age_seconds() + body = service_metrics.render( + ready=ready, + readiness_age_seconds=age_seconds, + replica=None if replica is None else replica.view(), + ).encode() + service_metrics.record_response(200) + try: + self.send_response(200) + self.send_header("Content-Type", "text/plain; version=0.0.4") + self.send_header("Content-Length", str(len(body))) + self.send_header("Cache-Control", "no-store") + self.end_headers() + self.wfile.write(body) + except (BrokenPipeError, ConnectionResetError, TimeoutError): + return + + def write_json( + self, + status: int, + payload: object, + headers: dict[str, str] | None = None, + ) -> None: + # Byte-identical framing to the coordinator listener's write_json: + # sorted keys and a trailing newline, so the extracted bodies match + # the pre-extraction ones exactly. + if isinstance(payload, public_api.RawJsonBody): + # Content-addressed bytes (#154): any re-serialization (key + # order, separators, trailing newline) would break + # sha256(response body) == the advertised artifact hash. + # /public/v1/artifacts/{sha256} moved here, so this branch + # moved with it. + body = payload.body + else: + body = json.dumps(payload, sort_keys=True).encode() + b"\n" + service_metrics.record_response(status) + try: + self.send_response(status) + self.send_header("Content-Type", "application/json") + self.send_header("Content-Length", str(len(body))) + for key, value in (headers or {}).items(): + self.send_header(key, value) + self.end_headers() + self.wfile.write(body) + except (BrokenPipeError, ConnectionResetError, TimeoutError): + # A client with a short timeout hung up before the response was + # written; nothing to salvage. + return + + def log_message(self, format: str, *args: object) -> None: # noqa: A002 + # BaseHTTPRequestHandler logs every request to stderr by default. + # This tier's request volume is the whole reason it exists, so that + # default would be the loudest thing in the container's logs. + return + + return PublicReadHandler + + +def _metric_number(value: object) -> str: + """Render a possibly-absent replica number as a gauge value. + + -1 stands in for "not known", matching how the readiness probe age already + reports its own pre-first-probe state. + """ + + number = _coerce_float(value) + return "-1" if number is None else f"{number:.3f}" + + +def _format_budget(budget_seconds: float) -> str: + """Render a budget without a trailing ``.0`` on whole-second values.""" + + if float(budget_seconds).is_integer(): + return str(int(budget_seconds)) + return f"{budget_seconds:g}" + + +def require_public_stratum_url(environ: dict[str, str] | None = None) -> str: + """Refuse to start unless the advertised Stratum endpoint is stated. + + public_api.mining_configuration composes the advertised endpoint from + getattr(coordinator, "bind"/"port") with an environment fallback of + 127.0.0.1:3340. This process exposes neither attribute, on purpose, so + without PRISM_PUBLIC_STRATUM_URL the public mining-configuration endpoint + would confidently tell every miner to point its hashrate at the loopback + address of whatever machine read the page. + """ + source = os.environ if environ is None else environ + value = (source.get("PRISM_PUBLIC_STRATUM_URL") or "").strip() + if not value: + raise PublicReadConfigurationError( + "PRISM_PUBLIC_STRATUM_URL is required: the public read service runs " + "no Stratum listener, so it cannot infer the pool's endpoint, and " + "the fallback would advertise 127.0.0.1 to miners" + ) + validate_advertised_stratum_url(source["PRISM_PUBLIC_STRATUM_URL"], "PRISM_PUBLIC_STRATUM_URL") + highdiff_url = source.get("PRISM_PUBLIC_STRATUM_HIGHDIFF_URL") + if public_api.optional_tcp_port(source.get("PRISM_STRATUM_HIGHDIFF_PORT")) is not None and highdiff_url: + validate_advertised_stratum_url(highdiff_url, "PRISM_PUBLIC_STRATUM_HIGHDIFF_URL") + return value + + +def validate_advertised_stratum_url(value: str, setting: str) -> None: + try: + parsed = urllib.parse.urlsplit(value) + hostname = parsed.hostname or "" + if ":" in hostname: + ipaddress.IPv6Address(hostname) + valid_host = True + else: + ascii_host = hostname.encode("idna").decode("ascii").rstrip(".") + valid_host = bool(ascii_host) and len(ascii_host) <= 253 and all( + re.fullmatch(r"[A-Za-z0-9](?:[A-Za-z0-9-]{0,61}[A-Za-z0-9])?", label) + for label in ascii_host.split(".") + ) + valid = ( + valid_host and parsed.scheme in {"stratum+tcp", "stratum+ssl"} + and parsed.hostname and parsed.port is not None and 1 <= parsed.port <= 65535 + and parsed.username is None and parsed.password is None + and not parsed.path and not parsed.query and not parsed.fragment + and not any(c.isspace() or ord(c) < 32 for c in value) + ) + except (ValueError, UnicodeError): + valid = False + if not valid: + raise PublicReadConfigurationError( + f"{setting} must be a stratum+tcp or stratum+ssl URL " + "with a hostname and port (1-65535), without credentials, path, query or fragment" + ) + + +def build_ledger_from_env(environ: dict[str, str] | None = None) -> PsqlShareLedger: + """Construct the read-only Postgres ledger this service reads through. + + Two refusals rather than defaults: + + * The in-memory ledger is rejected. It implements none of the ``dashboard_*`` + read models, so public_api's duck-typed fallbacks would silently take the + writer-lock paths (current_owed_balances, recipient_payout_history, + ledger.all_shares) that this extraction exists to get off the public + surface -- a memory-ledger public tier would look fine and reintroduce the + exact contention it removed. + * No PRISM_LEDGER_WRITER_* input is read at all. This process must never + acquire a writer lease; leaving the writer identity at its default and + touching only read models keeps that structural rather than conventional. + """ + source = os.environ if environ is None else environ + if _env_flag(source.get("PRISM_ALLOW_MEMORY_LEDGER")): + raise PublicReadConfigurationError( + "PRISM_ALLOW_MEMORY_LEDGER is not supported by the public read " + "service: the in-memory ledger has no dashboard_* read models, so " + "public_api would fall back to writer-lock reads " + "(current_owed_balances, recipient_payout_history) on public routes" + ) + psql_command = source.get("PRISM_POSTGRES_PSQL_COMMAND", "") + database_url = source.get("PRISM_DATABASE_URL", "") + if not database_url and source.get("PRISM_PUBLIC_DATABASE_HOST"): + # Compose supplies separate literal components; an explicit URL wins. + quote = lambda value: urllib.parse.quote(value, safe="") + user = quote(source.get("PRISM_POSTGRES_USER", "qbit")) + password = quote(source.get("PRISM_POSTGRES_PASSWORD", "change-this")) + database = quote(source.get("PRISM_POSTGRES_DB", "qbit")) + host = source["PRISM_PUBLIC_DATABASE_HOST"] + database_url = f"postgresql://{user}:{password}@{host}:5432/{database}" + if not psql_command and database_url: + psql_command = f"psql {shlex.quote(database_url)}" + if not psql_command: + raise PublicReadConfigurationError( + "PRISM_DATABASE_URL or PRISM_POSTGRES_PSQL_COMMAND is required: the " + "public read service serves every route from Postgres" + ) + return PsqlShareLedger( + psql_command=psql_command, + database_url=database_url or None, + native_client_mode=env("PRISM_POSTGRES_NATIVE_CLIENT", "auto"), + # Constructing an ordinary ledger claims the single-writer lease and + # gives the object a real writer lock. read_only does neither: no lease + # is acquired, and the writer gate refuses to be taken at all, so a + # public route that reached a fenced read would fail loudly here rather + # than quietly serialize dashboard polls against block landing. + read_only=True, + # Never initialize or repair schema from the read tier. + initialize_schema=False, + postgres_idle_in_transaction_timeout_seconds=env_positive_float( + "PRISM_POSTGRES_IDLE_IN_TRANSACTION_TIMEOUT_SECONDS", + DEFAULT_POSTGRES_IDLE_IN_TRANSACTION_TIMEOUT_SECONDS, + ), + postgres_tcp_keepalives_idle_seconds=env_positive_int( + "PRISM_POSTGRES_TCP_KEEPALIVES_IDLE_SECONDS", + DEFAULT_POSTGRES_TCP_KEEPALIVES_IDLE_SECONDS, + ), + postgres_tcp_keepalives_interval_seconds=env_positive_int( + "PRISM_POSTGRES_TCP_KEEPALIVES_INTERVAL_SECONDS", + DEFAULT_POSTGRES_TCP_KEEPALIVES_INTERVAL_SECONDS, + ), + postgres_tcp_keepalives_count=env_positive_int( + "PRISM_POSTGRES_TCP_KEEPALIVES_COUNT", + DEFAULT_POSTGRES_TCP_KEEPALIVES_COUNT, + ), + read_concurrency=env_positive_int("PRISM_POSTGRES_READ_CONCURRENCY", 4), + # Read by the ledger itself, and the second cache the miner page's + # staleness budget accounts for; both processes construct one. + reward_window_cache_seconds=env_nonnegative_float( + "PRISM_PUBLIC_REWARD_WINDOW_CACHE_SECONDS", + 30.0, + ), + audit_artifact_store=build_audit_artifact_store(source), + pool_application_name="prism-public-read", + ) + + +def build_audit_artifact_store( + environ: dict[str, str] | None = None, +) -> AuditArtifactStore | None: + """Open the audit artifact root read-only, for artifact and settlement reads. + + /public/v1/artifacts/{sha256} reads immutable compressed canonical bundles + first. A clean miss can use a labelled, non-cacheable legacy body; damage + fails closed. The compose service mounts that volume :ro precisely so this + process cannot write to it, which also means it cannot create the publication + lock file a normal store opens O_RDWR at construction -- hence read_only. + """ + source = os.environ if environ is None else environ + audit_dir = (source.get("PRISM_AUDIT_DIR") or "").strip() + if not audit_dir: + return None + root = Path(audit_dir) + return AuditArtifactStore( + AuditArtifactConfig( + root=root, + evidence_path=root / "prism-live-stratum-evidence.json", + ), + read_only=True, + ) + + +def build_rpc_from_env(environ: dict[str, str] | None = None) -> JsonRpc: + """This service's own qbit RPC client, built the way the coordinator builds its own. + + Read-only calls only: public_api reaches RPC for getblockchaininfo, + getblocktemplate and getnetworkinfo, all of which describe the node rather + than change it. + """ + del environ # env() reads os.environ directly, as every other PRISM entry point does. + return JsonRpc( + host=env("QBIT_RPC_HOST"), + port=env_int("QBIT_RPC_PORT", 18452), + user=env("QBIT_RPC_USER"), + password=env("QBIT_RPC_PASSWORD"), + ) + + +def _env_flag(raw: str | None) -> bool: + if raw is None: + return False + return raw.strip().lower() in {"1", "true", "yes", "on"} + + +def resolve_replica_mode(environ: dict[str, str] | None = None) -> str: + """Read PRISM_PUBLIC_REPLICA_MODE, refusing anything it does not mean. + + An unrecognised value is a configuration error, not a reason to pick a + default: the two modes differ in whether this process will serve from the + coordinator's primary, so guessing gets that wrong silently in one + direction or the other. + """ + source = os.environ if environ is None else environ + raw = (source.get("PRISM_PUBLIC_REPLICA_MODE") or "").strip().lower() + if not raw: + return REPLICA_MODE_OFF + if raw not in REPLICA_MODES: + raise PublicReadConfigurationError( + "PRISM_PUBLIC_REPLICA_MODE must be one of " + f"{', '.join(REPLICA_MODES)} (got {raw!r})" + ) + return raw + + +def build_replica_probe( + ledger: Any, + environ: dict[str, str] | None = None, +) -> ReplicaProbe | None: + """The replica contract, or None when this deployment has no standby yet.""" + + if resolve_replica_mode(environ) != REPLICA_MODE_REQUIRE: + return None + return ReplicaProbe( + ledger.read_replica_status, + max_lag_seconds=env_positive_float( + "PRISM_PUBLIC_REPLICA_MAX_LAG_SECONDS", + DEFAULT_REPLICA_MAX_LAG_SECONDS, + ), + ) + + +def build_service(environ: dict[str, str] | None = None) -> tuple[ + PublicReadCoordinator, + ReadinessProbe, + ServiceMetrics, + ReplicaProbe | None, +]: + """Validate configuration and assemble the process's collaborators. + + Every refusal happens here, before a socket is bound, so a misconfigured + service fails to start rather than starting and serving wrong facts. + """ + require_public_stratum_url(environ) + ledger = build_ledger_from_env(environ) + rpc = build_rpc_from_env(environ) + coordinator = PublicReadCoordinator(ledger=ledger, rpc=rpc) + replica = build_replica_probe(ledger, environ) + readiness = ReadinessProbe( + # In require mode the replication probe *is* the readiness probe: it + # round-trips to the same database and additionally proves the server + # is the standby this tier is supposed to be reading. Running both + # would be two round trips to answer one question. + ledger.dashboard_readiness_probe if replica is None else replica.check, + interval_seconds=env_positive_float( + "PRISM_PUBLIC_READINESS_PROBE_INTERVAL_SECONDS", + DEFAULT_READINESS_PROBE_INTERVAL_SECONDS, + ), + ) + return coordinator, readiness, ServiceMetrics(), replica + + +def main(argv: list[str] | None = None) -> int: + del argv + bind = os.environ.get("PRISM_PUBLIC_API_BIND") or DEFAULT_PUBLIC_API_BIND + port = env_int("PRISM_PUBLIC_API_PORT", DEFAULT_PUBLIC_API_PORT) + try: + max_connections = env_positive_int("PRISM_PUBLIC_HTTP_MAX_CONNECTIONS", 64) + connection_timeout = env_positive_float("PRISM_PUBLIC_HTTP_TIMEOUT_SECONDS", 10.0) + coordinator, readiness, metrics, replica = build_service() + handler = make_handler(PublicReadService( + coordinator, metrics=metrics, readiness=readiness, replica=replica, + )) + server = BoundedPublicHTTPServer( + (bind, port), handler, max_connections=max_connections, + timeout_seconds=connection_timeout, + ) + except (PublicReadConfigurationError, ValueError) as exc: + print(f"prism-public-read: {exc}", file=sys.stderr, flush=True) + return 2 + + readiness.start() + + def request_shutdown(signum: int, _frame: object) -> None: + # shutdown() blocks until serve_forever() returns and must not be called + # from the serving thread, so hand it to a helper thread. + del signum + threading.Thread(target=server.shutdown, daemon=True).start() + + signal.signal(signal.SIGTERM, request_shutdown) + signal.signal(signal.SIGINT, request_shutdown) + + backing = ( + "any server the DSN names (replica mode off)" + if replica is None + else ( + "a standby with a replication stream no more than " + f"{replica.max_lag_seconds:g}s silent" + ) + ) + print( + f"prism-public-read: serving /public/v1 on {bind}:{port}, reading from " + f"{backing}", + file=sys.stderr, + flush=True, + ) + try: + server.serve_forever() + finally: + readiness.stop() + server.server_close() + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/lab/prism/recover_pending_blocks.py b/lab/prism/recover_pending_blocks.py new file mode 100644 index 000000000..f9a214ab7 --- /dev/null +++ b/lab/prism/recover_pending_blocks.py @@ -0,0 +1,476 @@ +"""Recover allowlisted active-chain candidates using normal PRISM accounting. + +Invoke in a fresh managed coordinator container with the normal coordinator +stopped. Without --apply this only reads metadata and qbitd chain identity. +Apply holds the existing writer lease and heartbeat, loads one payload at a +time, and never starts serve(), Stratum, ordinary replay, or job prewarming. +""" + +from __future__ import annotations + +import argparse +import hmac +import json +import signal +import threading +from collections.abc import Mapping +from dataclasses import dataclass, replace +from typing import Any, Callable + +from lab.auxpow.stratum_codec import header_hash_hex +from lab.prism.block_candidates import _BlockCandidateNodeSubmission +from lab.prism.candidate_spool_view import iter_string_text_chunks +from lab.prism.candidate_store import CANDIDATE_SCHEMA_CAPABILITY, candidate_schema_refusal +from lab.prism.coordinator_config import CoordinatorConfig, load_coordinator_config +from lab.prism.prism_coordinator import PrismCoordinator +from lab.prism.recovery_json import cooperative_json +from lab.prism.rpc import JsonRpc +from lab.prism.share_ledger import ( + PsqlShareLedger, + block_candidate_identity_sha256, + canonical_hex, + database_url_from_psql_command, +) + +MAX_RECOVERY_BLOCKS = 32 +SUMMARY_SCHEMA = "qbit.prism.pending-block-recovery.v1" + + +class RecoveryError(RuntimeError): + pass + + +@dataclass(frozen=True) +class RecoveryBlock: + block_hash: str + height: int + parent_hash: str + state: str + + +def refuse_lease_wait(_seconds: float) -> None: + raise RecoveryError( + "exclusive writer lease unavailable; stop the coordinator and wait " + "for its lease release before retrying" + ) + + +class RecoveryCoordinator(PrismCoordinator): + """Use the production assembly without unrelated startup/publication work.""" + + def make_ledger(self, **_kwargs: Any) -> PsqlShareLedger: + ledger = super().make_ledger(lease_retry_sleep=refuse_lease_wait) + if not isinstance(ledger, PsqlShareLedger): + raise RecoveryError("recovery requires the native PostgreSQL ledger") + self.ledger = ledger # Available for cleanup even if __init__ later fails. + return ledger + + def _upgrade_legacy_audit_evidence(self) -> None: + # The normal coordinator owns this unrelated startup migration. + pass + + def _submit_block_candidate_to_node(self, *_args: Any, **_kwargs: Any) -> Any: + # Active-chain accounting must not create a new node offer if a chain + # observation changes during finalization. The normal verifier and + # accounting fences still run; stop and preserve the pending intent. + raise RecoveryError("recovery refuses to submit a block to the node") + + +class RecoveryReader: + """Metadata and single-payload reads on a server-enforced read-only session.""" + + def __init__(self, database_url: str) -> None: + import psycopg + from psycopg.rows import dict_row + + self.connection = psycopg.connect( + database_url, + autocommit=True, + row_factory=dict_row, + connect_timeout=5, + ) + try: + self.connection.execute("SET default_transaction_read_only = on") + self.connection.execute("SET statement_timeout = '30s'") + self.connection.execute("SET lock_timeout = '5s'") + except BaseException: + self.connection.close() + raise + + def close(self) -> None: + self.connection.close() + + def require_candidate_schema(self) -> None: + """Refuse a database this release cannot finalize against (#255). + + The coordinator's ledger applies the same rule when --apply builds + it; checking here lets the read-only plan report an unmigrated + database before any writer lease is requested. + """ + presence = self.connection.execute( + """SELECT to_regclass('qbit_block_candidate_body') IS NOT NULL AS has_body_table, + to_regclass('qbit_prism_schema_capabilities') IS NOT NULL AS has_capabilities""" + ).fetchone() + declared = None + if presence["has_capabilities"]: + capability = self.connection.execute( + """SELECT capability_value FROM qbit_prism_schema_capabilities + WHERE capability = %s""", + (CANDIDATE_SCHEMA_CAPABILITY,), + ).fetchone() + if capability is not None: + declared = int(capability["capability_value"]) + refusal = candidate_schema_refusal(declared, bool(presence["has_body_table"])) + if refusal is not None: + raise RecoveryError(refusal) + + def metadata(self, block_hash: str) -> dict[str, Any] | None: + return self.connection.execute( + """SELECT outbox.block_hash, outbox.state, outbox.candidate_sha256, + pool.block_height, pool.parent_hash, pool.chain_state, + EXISTS (SELECT 1 FROM qbit_pool_audit_bundles audit + WHERE audit.block_hash = outbox.block_hash) AS has_audit + FROM qbit_block_candidate_outbox outbox + LEFT JOIN qbit_pool_blocks pool USING (block_hash) + WHERE outbox.block_hash = %s""", + (block_hash,), + ).fetchone() + + def candidate(self, block_hash: str) -> dict[str, Any]: + """One pending row's payload facts, for either storage version. + + Issue #255: a version-2 row carries no ``candidate`` jsonb. Its + body lives in ``qbit_block_candidate_body`` and is read back in + bounded pages by the ledger (``hydrate_block_candidate_intent``), + never through this read-only session. The columns below name the + version explicitly so a chunked candidate is decoded through that + route rather than mistaken for a missing or corrupt payload. + """ + row = self.connection.execute( + """SELECT outbox.candidate, outbox.candidate_sha256, + outbox.storage_version, outbox.body_id, outbox.replay_header, + body.byte_count, body.chunk_count, body.chunk_bytes, + body.share_count, body.state AS body_state + FROM qbit_block_candidate_outbox outbox + LEFT JOIN qbit_block_candidate_body body ON body.body_id = outbox.body_id + WHERE outbox.block_hash = %s AND outbox.state = 'pending'""", + (block_hash,), + ).fetchone() + if row is None: + raise RecoveryError(f"pending candidate disappeared: {block_hash}") + return row + + +def require_active_block(rpc: Any, block_hash: str) -> tuple[int, str]: + header = rpc.call("getblockheader", [block_hash]) + height = header.get("height") if isinstance(header, dict) else None + if isinstance(height, bool) or not isinstance(height, int) or height < 1: + raise RecoveryError(f"invalid chain header for {block_hash}") + if rpc.call("getblockhash", [height]) != block_hash: + raise RecoveryError(f"candidate is not on the active chain: {block_hash}") + parent = canonical_hex( + str(header.get("previousblockhash", "")), name="parent_hash", expected_bytes=32 + ) + return height, parent + + +def require_completed(row: dict[str, Any] | None, block: RecoveryBlock) -> None: + if not row or not ( + row["state"] == "submitted" + and row["chain_state"] == "confirmed" + and row["block_height"] == block.height + and row["parent_hash"] == block.parent_hash + and row["has_audit"] is True + ): + raise RecoveryError(f"accounting/outbox completion is not proven: {block.block_hash}") + + +def plan_recovery(reader: RecoveryReader, rpc: Any, hashes: list[str]) -> list[RecoveryBlock]: + """Validate the schema and entire allowlist before the first writer lease.""" + hashes = [canonical_hex(value, name="block_hash", expected_bytes=32) for value in hashes] + if not hashes or len(hashes) > MAX_RECOVERY_BLOCKS or len(set(hashes)) != len(hashes): + raise RecoveryError(f"specify 1–{MAX_RECOVERY_BLOCKS} distinct block hashes") + reader.require_candidate_schema() + blocks = [] + for block_hash in hashes: + row = reader.metadata(block_hash) + if not row or row["state"] not in {"pending", "submitted"}: + raise RecoveryError(f"candidate is missing or abandoned: {block_hash}") + height, parent = require_active_block(rpc, block_hash) + block = RecoveryBlock(block_hash, height, parent, row["state"]) + if block.state == "submitted": + require_completed(row, block) + parent_row = reader.metadata(parent) + if parent_row and parent_row["state"] == "pending" and parent not in hashes: + raise RecoveryError(f"pending parent must be included in the allowlist: {parent}") + blocks.append(block) + return sorted(blocks, key=lambda block: block.height) + + +def load_intent( + coordinator: PrismCoordinator, block: RecoveryBlock, row: dict[str, Any] +) -> Any: + """The intent mapping for one pending row, by storage version (#255). + + Version 1 is the legacy whole-jsonb payload the read-only session + already decoded (cooperatively, in this standalone process). Version 2 + is hydrated by the coordinator's ledger in bounded chunk pages into a + spool file, and its decoded facts are exposed through the same mapping + interface. Any other version is refused rather than skipped. + """ + storage_version = int(row.get("storage_version") or 1) + if storage_version == 1: + return row["candidate"] + if storage_version != 2: + raise RecoveryError( + f"unsupported candidate storage version {storage_version}: {block.block_hash}" + ) + if row.get("body_id") is None or row.get("body_state") != "sealed": + raise RecoveryError(f"chunked candidate body is not sealed: {block.block_hash}") + hydrate = getattr(coordinator.ledger, "hydrate_block_candidate_intent", None) + if not callable(hydrate): + raise RecoveryError("the ledger cannot hydrate chunked candidate bodies") + header_row = { + "block_hash": block.block_hash, + "storage_version": 2, + "candidate_sha256": str(row["candidate_sha256"]), + "header": row.get("replay_header") or {}, + "body": { + "body_id": row["body_id"], + "storage_version": 2, + "candidate_sha256": str(row["candidate_sha256"]), + "byte_count": row["byte_count"], + "chunk_count": row["chunk_count"], + "chunk_bytes": row["chunk_bytes"], + "share_count": row["share_count"], + "state": row["body_state"], + }, + } + return hydrate(header_row, cancelled=coordinator.stop_event.is_set) + + +def block_hex_prefix(block_hex: Any, length: int) -> str: + """The first ``length`` characters of a plain or streamed block hex. + + A hydrated version-2 body keeps a large ``block_hex`` as a streamed + view that refuses ``str()``; only the header prefix is needed here. + """ + prefix = "" + for chunk in iter_string_text_chunks(block_hex): + prefix += chunk + if len(prefix) >= length: + break + return prefix[:length] + + +def decode_candidate( + coordinator: PrismCoordinator, block: RecoveryBlock, row: dict[str, Any] +) -> Any: + intent = load_intent(coordinator, block, row) + if not isinstance(intent, Mapping) or intent.get("block_hash_hex") != block.block_hash: + raise RecoveryError("candidate payload does not match its outbox key") + digest = block_candidate_identity_sha256(intent) + if not hmac.compare_digest(digest, str(row["candidate_sha256"])): + raise RecoveryError(f"candidate identity digest mismatch: {block.block_hash}") + if ( + intent.get("expected_height") != block.height + or intent.get("parent_hash") != block.parent_hash + ): + raise RecoveryError(f"candidate does not match its active-chain header: {block.block_hash}") + header = bytes.fromhex(block_hex_prefix(intent["block_hex"], 160)) + if len(header) != 80 or header_hash_hex(header) != block.block_hash: + raise RecoveryError(f"candidate block bytes do not match its hash: {block.block_hash}") + return replace(coordinator.block_candidate_from_intent(intent), durable_replay=True) + + +def recover_block( + coordinator: PrismCoordinator, reader: RecoveryReader, block: RecoveryBlock +) -> None: + # Re-read after acquisition; the earlier plan is advisory. + row = reader.metadata(block.block_hash) + if row and row["state"] == "submitted": + require_completed(row, block) + require_active_block(coordinator.rpc, block.block_hash) + return + candidate = decode_candidate(coordinator, block, reader.candidate(block.block_hash)) + with coordinator._writer_operation("accepted_block_handling"): + if coordinator.stop_event.is_set(): + raise RecoveryError("recovery interrupted") + if require_active_block(coordinator.rpc, block.block_hash) != ( + block.height, + block.parent_hash, + ): + raise RecoveryError("chain identity changed since the recovery plan") + coordinator._require_fresh_ledger_lease_for_external_side_effect("block-recovery") + coordinator._mark_block_candidate_attempted(block.block_hash) + with coordinator._block_landing_ledger_statement_timeout_scope(block.block_hash): + accepted = coordinator.submit_block_candidate( + candidate, + node_submission=_BlockCandidateNodeSubmission(attempted=False), + ) + if not accepted: + raise RecoveryError( + f"normal finalization deferred or rejected {block.block_hash}; outbox retained" + ) + coordinator._finalize_block_candidate( + candidate, + block_hash=block.block_hash, + accepted=True, + error="", + outcome=coordinator._block_candidate_outcome, + ) + require_completed(reader.metadata(block.block_hash), block) + require_active_block(coordinator.rpc, block.block_hash) + + +def recovery_config(config: CoordinatorConfig) -> CoordinatorConfig: + return replace( + config, + ledger=replace( + config.ledger, + initialize_schema=False, + native_client_mode="on", + writer_session_token=None, + ), + stop_after_block=False, + max_blocks=2**31 - 1, + ) + + +def apply_recovery( + config: CoordinatorConfig, + reader: RecoveryReader, + blocks: list[RecoveryBlock], + *, + timeout_seconds: int, + report: Callable[[dict[str, Any]], None], +) -> None: + coordinator = RecoveryCoordinator.__new__(RecoveryCoordinator) + initialized = False + deadline = None + old_handlers = {} + try: + coordinator.__init__(recovery_config(config)) + initialized = True + if coordinator._start_ledger_lease_heartbeat() is None: + raise RecoveryError("the guarded lease heartbeat did not start") + + def interrupt(signum: int, _frame: Any) -> None: + coordinator.request_shutdown(signum) + raise RecoveryError("recovery interrupted; pending work remains replayable") + + for signum in (signal.SIGTERM, signal.SIGINT): + old_handlers[signum] = signal.signal(signum, interrupt) + deadline = threading.Timer( + timeout_seconds, + coordinator._watchdog_hard_exit, + args=("pending-block recovery deadline exceeded",), + ) + deadline.daemon = True + deadline.start() + coordinator.validate_live_chain_identity() + coordinator.validate_live_template_and_fee_policy() + coordinator.prism_payout_policy() + for block in blocks: + report({"event": "recovering", "block_hash": block.block_hash, "height": block.height}) + recover_block(coordinator, reader, block) + report({"event": "completed", "block_hash": block.block_hash, "height": block.height}) + finally: + # Keep the deadline armed through writer drainage. An uncertain write + # must not turn a hung cleanup into an indefinitely held lease. + try: + if initialized: + if not coordinator.shutdown(reason="block_recovery_finished"): + raise RecoveryError("writer drainage/lease release failed") + coordinator.drain_non_writer_components() + else: + ledger = getattr(coordinator, "ledger", None) + if ledger is not None: + ledger.release_writer_lease() + finally: + try: + ledger = getattr(coordinator, "ledger", None) + if ledger is not None: + ledger.close() + finally: + try: + store = coordinator.__dict__.get("_audit_artifact_store") + if store is not None: + store.close() + finally: + if deadline is not None: + deadline.cancel() + deadline.join(timeout=1) + for signum, handler in old_handlers.items(): + signal.signal(signum, handler) + + +def main(argv: list[str] | None = None) -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--block-hash", action="append", required=True, dest="hashes") + parser.add_argument( + "--apply", + action="store_true", + help="complete accounting; normal coordinator must be stopped", + ) + parser.add_argument( + "--timeout-seconds", + type=int, + default=600, + help="processing and cleanup deadline after construction (default: 600)", + ) + args = parser.parse_args(argv) + if not 1 <= args.timeout_seconds <= 3600: + parser.error("--timeout-seconds must be between 1 and 3600") + reader = None + + def report(event: dict[str, Any]) -> None: + print(json.dumps({"schema": SUMMARY_SCHEMA, **event}, sort_keys=True), flush=True) + + try: + with cooperative_json(): + config = load_coordinator_config() + database_url = config.ledger.database_url or database_url_from_psql_command( + config.ledger.psql_command + ) + if not database_url or config.ledger.allow_memory_ledger: + raise RecoveryError( + "recovery requires PRISM_DATABASE_URL and the native PostgreSQL client" + ) + reader = RecoveryReader(database_url) + rpc = JsonRpc( + host=config.rpc.host, + port=config.rpc.port, + user=config.rpc.user, + password=config.rpc.password, + ) + blocks = plan_recovery(reader, rpc, args.hashes) + report( + { + "event": "plan", + "apply": args.apply, + "blocks": [block.__dict__ for block in blocks], + } + ) + if args.apply: + apply_recovery( + config, reader, blocks, timeout_seconds=args.timeout_seconds, report=report + ) + report({"event": "success", "apply": args.apply}) + return 0 + except Exception as exc: + # Driver errors can embed a credential-bearing DSN or SQL payload. + report( + { + "event": "failed", + "error": str(exc) if isinstance(exc, RecoveryError) else type(exc).__name__, + } + ) + return 1 + finally: + if reader is not None: + reader.close() + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/lab/prism/recovery_json.py b/lab/prism/recovery_json.py new file mode 100644 index 000000000..f5886d3f5 --- /dev/null +++ b/lab/prism/recovery_json.py @@ -0,0 +1,38 @@ +"""Cooperative stdlib JSON for the standalone recovery process only. + +Python's fallback codecs yield the interpreter between records. They trade +throughput for heartbeat responsiveness; they are not a byte/memory bound. +Never install these process-global settings in an existing coordinator. +""" + +from __future__ import annotations + +import json +import json.decoder +import json.encoder +import json.scanner +from contextlib import contextmanager +from typing import Iterator + + +@contextmanager +def cooperative_json() -> Iterator[None]: + previous = ( + json.scanner.make_scanner, + json.decoder.scanstring, + json.encoder.c_make_encoder, + json._default_decoder, + ) + try: + json.scanner.make_scanner = json.scanner.py_make_scanner + json.decoder.scanstring = json.decoder.py_scanstring + json.encoder.c_make_encoder = None + json._default_decoder = json.JSONDecoder() + yield + finally: + ( + json.scanner.make_scanner, + json.decoder.scanstring, + json.encoder.c_make_encoder, + json._default_decoder, + ) = previous diff --git a/lab/prism/reorg_reconciler.py b/lab/prism/reorg_reconciler.py new file mode 100644 index 000000000..f521776d7 --- /dev/null +++ b/lab/prism/reorg_reconciler.py @@ -0,0 +1,1366 @@ +"""PRISM active-chain reconciliation and payout-publication ownership. + +The service owns same-tip flight admission, the per-tip trusted memo with its +lookup counters, the bounded reconcile-prefetch executor, and the core +reconcile state machine. The coordinator retains its serialized cross-owner +adapter (writer admission, payout-balance mutation lock, landed-preview +fail-closed check) and thin public/private delegates; flight coalescing runs +*before* that adapter so same-tip followers never queue behind writer +admission. Memo, lookup, and outcome state is guarded by the coordinator's +control-plane lock supplied through ``ReorgPorts.state_lock`` so tip +observation can evict memo entries atomically with its own state updates. +""" + +from __future__ import annotations + +from collections import OrderedDict +from concurrent.futures import Future, ThreadPoolExecutor +from contextlib import AbstractContextManager, contextmanager +from dataclasses import dataclass +import threading +import time +from typing import Any, Callable, Iterator, Mapping + +from lab.prism.accepted_preview_telemetry import ( + AcceptedPreviewTelemetry, + PRISM_REORG_RECONCILE_CALLERS, + RECONCILE_CALLER_JOB_BUILD, + RECONCILE_CALLER_LANDING, + RECONCILE_CALLER_OTHER, + RECONCILE_CALLER_POST_CONFIRM, + RECONCILE_CALLER_TIP_REFRESH, + RECONCILE_STEP_ADMISSION_WAIT, + RECONCILE_STEP_CANDIDATE_PREPARE, + RECONCILE_STEP_CHAIN_PROBE, + RECONCILE_STEP_MUTATIONS, + RECONCILE_STEP_PUBLISH, + RECONCILE_STEP_WATCH_QUERY, +) +from lab.prism.coordinator_config import ( + DEFAULT_PRISM_REORG_RECONCILE_CACHE_SECONDS, + TESTNET_QBIT_CHAINS, +) +from lab.prism.payout_state import PayoutStateCandidate, TemplateRefreshSuperseded + + +# Bounded number of tips whose trusted reconcile outcome is memoized. Reorg +# churn moves between a handful of competing tips; eight entries cover deep +# flip-flop churn while keeping eviction scans trivial. +PRISM_REORG_RECONCILE_MEMO_MAX_TIPS = 8 +# How long an unflagged same-tip caller waits for an in-flight reconcile pass +# before falling back to its own serialized pass. +DEFAULT_PRISM_RECONCILE_FLIGHT_WAIT_SECONDS = 30.0 +# Reconcile demand accounting: which caller asked, and what satisfied it. +# "overlap" is a tip-refresh join that reused the pass prefetched alongside +# its template fetch; "serial" is a full serialized pass of the caller's own. +PRISM_REORG_RECONCILE_LOOKUP_PATHS = ("tip_refresh", "job_build") +PRISM_REORG_RECONCILE_LOOKUP_SOURCES = ("memo_hit", "overlap", "serial") +# Bounded wait when joining the prefetched reconcile pass. Long enough for a +# healthy pass (DB round trips plus a few RPCs), far shorter than the +# publication-progress failure budget, so a stuck pass surfaces as the normal +# blocked-retry path instead of parking the poll loop's liveness heartbeat. +PRISM_RECONCILE_PREFETCH_JOIN_TIMEOUT_SECONDS = 20.0 +# Hard ceiling for the operator override: the join must stay comfortably +# below the template-refresh failure budget or the bound stops bounding. +PRISM_RECONCILE_PREFETCH_JOIN_TIMEOUT_CEILING_SECONDS = 60.0 +# How deeply a pool block must be buried before the stranded-prepared sweep +# will reject it. +# +# A prepared row is owned by the live submit/replay path while its outbox +# entry exists; the sweep exists only for rows that lost that owner, and it +# cannot tell the two apart from the ledger alone. Depth is the proxy. 100 +# blocks is far past any legitimate in-flight finalization — a landing that +# is still retrying, an ancestor replay working its way forward, a +# confirmation racing its own audit publication all resolve within a handful +# of blocks — while still healing a genuinely stranded row within roughly +# two hours of block time rather than leaving its payout entries, carry +# forward, and CTV fanout artifacts pinned immature indefinitely. +STRANDED_PREPARED_REJECT_MIN_DEPTH = 100 +# One pass's bound on the sweep. Each returned row costs a getblockhash and +# possibly a fenced ledger mutation, and the pass already walks the reorg +# watch set; the remainder is picked up by the next pass. +STRANDED_PREPARED_SWEEP_LIMIT = 64 + + +def _no_reconcile_progress(phase: str) -> None: + """Discard reconcile progress stamps for ports built without a recorder.""" + return None + + +class _ReconcileFlight: + """One in-flight reconcile pass shared by concurrent same-tip callers.""" + + __slots__ = ("event", "summary", "exception") + + def __init__(self) -> None: + self.event = threading.Event() + self.summary: dict[str, object] | None = None + self.exception: BaseException | None = None + + +@dataclass(frozen=True) +class ReorgPorts: + """Dynamic infrastructure required by active-chain reconciliation. + + Callables that reach documented coordinator monkeypatch seams + (``reconcile_with_admission``, ``reconcile_serialized``, ``ensure_tip``, + ``chain_view_untrusted``) resolve the coordinator method at call time so + per-instance patches intercept every internal path. + """ + + rpc_call: Callable[..., object] + ledger: Callable[[], Any] + ensure_job_cache_state: Callable[[], None] + state_lock: Callable[[], AbstractContextManager[object]] + source_tip: Callable[[], str | None] + reserve_external_tip: Callable[[str], None] + max_supersession_retries: Callable[[], int] + prepare_lock: Callable[[], AbstractContextManager[object]] + capture_source: Callable[[], tuple[int, int, str | None, str, float]] + prepared_candidate: Callable[..., PayoutStateCandidate] + captured_publication_required: Callable[ + [tuple[int, int, str | None, str, float]], + bool, + ] + block_publication: Callable[..., None] + publication_guard: Callable[[], AbstractContextManager[object]] + publish_candidate: Callable[[PayoutStateCandidate], int | None] + observe_preparation: Callable[[float], None] + chain_view_untrusted: Callable[[], bool] + reorg_proof_snapshot: Callable[[], tuple[str | None, int]] + flight_wait_seconds: Callable[[], float] + prefetch_join_timeout_seconds: Callable[[], float] + reconcile_with_admission: Callable[..., Mapping[str, object]] + reconcile_serialized: Callable[..., dict[str, object]] + ensure_tip: Callable[[str], bool] + # Liveness stamp for a reconcile pass running inline on a + # watchdog-monitored thread. An accepted-block landing calls + # ``ensure_tip`` on the block-work thread, and one pass is an unbounded + # chain walk -- a getblockhash plus up to two ledger statements per + # watched block -- so bracketing the call from outside would still leave + # a silence proportional to the number of watched blocks. The coordinator + # supplies its block-work phase recorder, which no-ops on every other + # thread; ports built without one get a sink so background reconciliation + # and focused tests are unchanged. + record_progress: Callable[[str], None] = _no_reconcile_progress + # Issue #224 attribution owner: the coordinator supplies + # ``ensure_accepted_preview_telemetry(self)`` so the reconcile pass, + # step and caller families land in the one owner the renderer reads. + # Ports built without one get a private owner so focused tests observe + # the same families; nothing recorded here changes flight, memo, + # admission, lock or publication behavior. + accepted_preview_telemetry: ( + Callable[[], AcceptedPreviewTelemetry] | None + ) = None + + +@dataclass(frozen=True) +class ReorgState: + """Copied bookkeeping/last-outcome snapshot for observers.""" + + inactive_block_count: int + reactivated_block_count: int + reconcile_skip_count: int + reconcile_error_count: int + matured_payout_count: int + last_tip_hash: str | None + last_trusted: bool + last_monotonic: float | None + + +def qbit_chain_view_untrusted( + rpc_call: Callable[..., object], + chain: str, +) -> bool: + """Return whether the node cannot provide a coherent validated tip.""" + + blockchain_info = rpc_call("getblockchaininfo") + if not isinstance(blockchain_info, dict): + raise RuntimeError("getblockchaininfo returned non-object") + public_chain = chain.lower() in { + "main", + "mainnet", + *TESTNET_QBIT_CHAINS, + } + if ( + blockchain_info.get("initialblockdownload") is not False + if public_chain + else bool(blockchain_info.get("initialblockdownload")) + ): + return True + blocks_raw = blockchain_info.get("blocks") + headers_raw = blockchain_info.get("headers") + if public_chain and (blocks_raw is None or headers_raw is None): + return True + if blocks_raw is not None and headers_raw is not None: + try: + blocks = int(blocks_raw) + headers = int(headers_raw) + if blocks < 0 or headers < 0 or headers != blocks: + return True + except (TypeError, ValueError) as exc: + raise RuntimeError("getblockchaininfo blocks/headers are not integers") from exc + return False + + +class ReorgReconcilerService: + """Own flights, trusted memo/lookup state, prefetch, and the core pass.""" + + def __init__( + self, + ports: ReorgPorts, + *, + enabled: bool = True, + cache_seconds: float = DEFAULT_PRISM_REORG_RECONCILE_CACHE_SECONDS, + inactive_block_count: int = 0, + reactivated_block_count: int = 0, + reconcile_skip_count: int = 0, + reconcile_error_count: int = 0, + matured_payout_count: int = 0, + last_tip_hash: str | None = None, + last_trusted: bool = False, + last_monotonic: float | None = None, + ) -> None: + self._ports = ports + self.enabled = bool(enabled) + self.cache_seconds = cache_seconds + self.inactive_block_count = int(inactive_block_count) + self.reactivated_block_count = int(reactivated_block_count) + self.reconcile_skip_count = int(reconcile_skip_count) + self.reconcile_error_count = int(reconcile_error_count) + self.matured_payout_count = int(matured_payout_count) + self.last_tip_hash = last_tip_hash + self.last_trusted = bool(last_trusted) + self.last_monotonic = last_monotonic + self._reconcile_flight_lock = threading.Lock() + # In-flight reconcile passes keyed by tip hash. Unflagged callers + # for a tip already being reconciled await that pass's summary + # instead of queueing a redundant serialized pass of their own. + self._reconcile_flights: dict[str, _ReconcileFlight] = {} + # Monotonic completion time of the last trusted reconcile pass, + # per tip hash, guarded by the coordinator state lock. Entries are + # unarmed only by an untrusted outcome for their own tip (or a + # reconcile error, which clears the whole map); see note_outcome. + self._reorg_reconcile_trusted_memo: OrderedDict[str, float] = ( + OrderedDict() + ) + self._reconcile_prefetch_executor_lock = threading.Lock() + # Single-worker lane that overlaps the tip-refresh reconcile pass + # with the template fetch. Only the refresh singleflight owner + # submits, so at most one prefetched pass is in flight; same-tip + # followers still coalesce through _reconcile_flights. + self._reconcile_prefetch_executor: ThreadPoolExecutor | None = None + self._reconcile_prefetch_executor_shutdown = False + # At most one outstanding prefetch, keyed by tip, guarded by + # _reconcile_prefetch_executor_lock. Failed refresh attempts + # (for example a template-RPC outage) reuse it instead of + # queueing another serialized pass per retry. + self._reconcile_prefetch_pending: ( + tuple[str, Future[bool], bool] | None + ) = None + # Reconcile demand by (caller path, satisfying source), guarded by + # the coordinator state lock. + self.reorg_reconcile_lookup_counts = { + (path, source): 0 + for path in PRISM_REORG_RECONCILE_LOOKUP_PATHS + for source in PRISM_REORG_RECONCILE_LOOKUP_SOURCES + } + # Issue #224 caller attribution. Entry points stamp the bounded + # caller on the calling thread (the landing's in-lock ensure_tip, + # the job-build ensure_current, the tip-refresh prefetch lane) and + # the core pass reads it there, so the coordinator's monkeypatch + # seams (reconcile_with_admission / reconcile_serialized / + # ensure_tip) keep their exact signatures. The same slot carries + # the stamp ``admission_wait`` is measured from. + self._reconcile_attribution = threading.local() + self._fallback_telemetry_lock = threading.Lock() + self._fallback_telemetry: AcceptedPreviewTelemetry | None = None + + @property + def accepted_preview_telemetry(self) -> AcceptedPreviewTelemetry: + """The shared #224 owner, or a private one for ports without it.""" + port = self._ports.accepted_preview_telemetry + if port is not None: + return port() + telemetry = self._fallback_telemetry + if telemetry is None: + with self._fallback_telemetry_lock: + telemetry = self._fallback_telemetry + if telemetry is None: + telemetry = AcceptedPreviewTelemetry() + self._fallback_telemetry = telemetry + return telemetry + + def _attributed_caller(self) -> str | None: + return getattr(self._reconcile_attribution, "caller", None) + + @contextmanager + def _attributed(self, caller: str | None) -> Iterator[None]: + """Stamp ``caller`` on this thread for the passes it runs. + + The outermost entry point wins: a tip-refresh prefetch that reaches + ``ensure_tip`` through the coordinator keeps ``tip_refresh``, and a + forced publication inside an attributed stretch keeps that stretch's + label. ``None`` stamps nothing. + """ + local = self._reconcile_attribution + previous = getattr(local, "caller", None) + if caller is None or previous is not None: + yield + return + if caller not in PRISM_REORG_RECONCILE_CALLERS: + raise ValueError(f"unknown reorg reconcile caller: {caller!r}") + local.caller = caller + try: + yield + finally: + local.caller = previous + + def _serialized_pass( + self, + caller: str | None, + **kwargs: object, + ) -> dict[str, object]: + """Run one pass through the serialized adapter, attributed and stamped. + + The stamp is what ``reconcile`` measures ``admission_wait`` from: the + writer admission and payout-balance mutation lock the coordinator + adapter takes before the core pass starts. It is cleared on every + exit so a pass refused before the core (a landed preview) cannot + leave a stale stamp for a later direct call on this thread. + """ + local = self._reconcile_attribution + with self._attributed(caller): + local.admission_started = time.monotonic() + try: + return self._ports.reconcile_serialized(**kwargs) + finally: + local.admission_started = None + + @staticmethod + def _candidate_intent(payout_changed: bool) -> dict[str, bool]: + """Preparation intent for a pass whose mutations may have moved balances. + + Reconciliation mutates pool-block, payout-entry and carry state but + never ``qbit_share_ledger`` (the verified invariant recorded in + ``lab.prism.accepted_preview_telemetry``), so the accepted-share + window at a fixed anchor stays exact after a maturation, an inactive + marking or a reactivation. A confirmed mutation therefore asks for a + live prior-balances reread, not the O(window) oracle rescan that + ``force_full_window_rescan`` still means everywhere else (#224). The + reread intent is forwarded only when it applies so preparation seams + that predate it keep their exact signature. + """ + intent: dict[str, bool] = {"force_full_window_rescan": False} + if payout_changed: + intent["force_prior_balances_read"] = True + return intent + + def snapshot(self) -> ReorgState: + with self._ports.state_lock(): + return ReorgState( + inactive_block_count=self.inactive_block_count, + reactivated_block_count=self.reactivated_block_count, + reconcile_skip_count=self.reconcile_skip_count, + reconcile_error_count=self.reconcile_error_count, + matured_payout_count=self.matured_payout_count, + last_tip_hash=self.last_tip_hash, + last_trusted=self.last_trusted, + last_monotonic=self.last_monotonic, + ) + + def reorg_reconcile_lookup_snapshot(self) -> dict[tuple[str, str], int]: + """Copied lookup counters for the metrics renderer.""" + with self._ports.state_lock(): + return dict(self.reorg_reconcile_lookup_counts) + + def note_outcome( + self, + tip_hash: str | None, + *, + trusted: bool, + clear_memo: bool = False, + evict_others: bool = False, + proof_epoch: int | None = None, + ) -> None: + """Record a reconcile outcome in the per-tip trusted memo. + + A trusted pass arms the memo for its own tip. An untrusted outcome + (superseded publication, untrusted chain view) unarms only its own + tip: it proves nothing about reconciliations already completed for + other tips, and unarming them globally forces every job build into a + redundant full pass. A pass that applied orphan/maturity row + mutations passes evict_others=True: cached proofs for other tips + were taken against pre-mutation rows and no longer hold, even if the + chain later flips back before any tip observation lands. A reconcile + error passes clear_memo=True; a partially applied ledger mutation + invalidates every cached outcome. ``proof_epoch`` carries the + tip-detection epoch the pass started its reads in: arming is refused + when the epoch moved during the pass, so a flip away and back can + never re-arm an entry with a proof from the closed epoch (the + latest-detected-hash guard alone cannot see the round trip). + """ + + self._ports.ensure_job_cache_state() + now = time.monotonic() + with self._ports.state_lock(): + self.last_tip_hash = tip_hash + self.last_trusted = trusted + self.last_monotonic = now + memo = self._reorg_reconcile_trusted_memo + if clear_memo: + memo.clear() + return + if evict_others: + for cached_tip in list(memo): + if cached_tip != tip_hash: + del memo[cached_tip] + if tip_hash is None: + return + if trusted: + latest_tip, detection_epoch = ( + self._ports.reorg_proof_snapshot() + ) + if latest_tip is not None and latest_tip != tip_hash: + # This pass finished for a tip that is no longer the + # newest detected one; its epoch is over. Arming would + # re-add an entry the newer observation already evicted + # and let a flip-back reuse a pre-flip outcome. + return + if proof_epoch is not None and proof_epoch != detection_epoch: + # The pass spanned a detection cycle; every memo + # consumer (refresh joins, initial-job and vardiff-idle + # builds) must see a full re-proof instead. + return + memo[tip_hash] = now + memo.move_to_end(tip_hash) + while len(memo) > PRISM_REORG_RECONCILE_MEMO_MAX_TIPS: + memo.popitem(last=False) + else: + memo.pop(tip_hash, None) + + def evict_memo_for_new_tip_locked(self, tip_hash: str) -> None: + """Drop trusted-reconcile entries for every tip except ``tip_hash``. + + Called under the coordinator state lock when a newer tip is + detected. A detected flip ends the epoch of every previously cached + outcome: if the chain later flips back to an earlier hash within the + cache TTL, pool-block chain state must be re-proven by a fresh pass, + not assumed from a pre-flip reconciliation. Detection is + observation-sequenced, so only genuinely newer observations evict. + """ + + self._ports.ensure_job_cache_state() + memo = self._reorg_reconcile_trusted_memo + for cached_tip in list(memo): + if cached_tip != tip_hash: + del memo[cached_tip] + + def ensure_current( + self, + *, + expected_tip_hash: str | None = None, + ) -> bool: + if not self.enabled and expected_tip_hash is None: + return True + current_tip = str(self._ports.rpc_call("getbestblockhash")) + if expected_tip_hash is not None and current_tip != expected_tip_hash: + raise TemplateRefreshSuperseded( + "qbit tip changed while prepared work was queued " + f"expected={expected_tip_hash} current={current_tip}" + ) + if not self.enabled: + return True + if self.memo_fresh(current_tip): + self.record_lookup("job_build", "memo_hit") + return True + self.record_lookup("job_build", "serial") + with self._attributed(RECONCILE_CALLER_JOB_BUILD): + return self._ports.ensure_tip(current_tip) + + def memo_fresh(self, tip_hash: str) -> bool: + """True when a trusted pass for ``tip_hash`` is inside the cache TTL + and the live chain view is still trusted. + + A fresh memo entry lets a caller reuse the completed pass instead of + queueing a redundant serialized one. The memo is per tip: an + untrusted outcome recorded for another tip never unarms this one. + The chain-view trust check is NOT cached: headers can run ahead of + the validated tip without the best block hash changing (an arriving + reorg), and job issuance must pause immediately, not a TTL later. + """ + ttl = self.cache_seconds + if ttl <= 0: + return False + self._ports.ensure_job_cache_state() + with self._ports.state_lock(): + reconciled_monotonic = self._reorg_reconcile_trusted_memo.get( + tip_hash + ) + return bool( + reconciled_monotonic is not None + and time.monotonic() - reconciled_monotonic <= ttl + and not self._ports.chain_view_untrusted() + ) + + def record_lookup(self, path: str, source: str) -> None: + if path not in PRISM_REORG_RECONCILE_LOOKUP_PATHS: + raise ValueError(f"unknown reorg reconcile lookup path: {path}") + if source not in PRISM_REORG_RECONCILE_LOOKUP_SOURCES: + raise ValueError(f"unknown reorg reconcile lookup source: {source}") + self._ports.ensure_job_cache_state() + with self._ports.state_lock(): + self.reorg_reconcile_lookup_counts[(path, source)] += 1 + + def prefetch_pass( + self, + tip_hash: str, + prove: bool = False, + ) -> bool: + """One prefetched reconcile, honoring the memo like the join does. + + A prefetch that queued behind a completed same-tip pass (abandoned + refresh attempts reuse the slot, but a replaced tip can leave one + queued) would otherwise re-run the full serialized pass for nothing. + A proving pass (the serial re-prove branches) bypasses the memo: + those branches exist precisely because the entry cannot be trusted. + """ + if not prove and self.memo_fresh(tip_hash): + return True + # Only the tip-refresh owner submits prefetches and re-proves. + with self._attributed(RECONCILE_CALLER_TIP_REFRESH): + return self._ports.ensure_tip(tip_hash) + + def submit_prefetch( + self, + tip_hash: str, + *, + prove: bool = False, + ) -> Future[bool] | None: + """Run one reconcile pass on the prefetch worker so it overlaps the + caller's template fetch. + + Returns ``None`` once shutdown has retired the executor; the caller + falls back to its serial pass. At most one prefetch is outstanding: + a refresh attempt that failed before its join (for example a + template-RPC outage) leaves its future in the slot, and the retry + reuses it for the same tip instead of queueing another serialized + pass behind the first. + """ + self._ports.ensure_job_cache_state() + stale_future: Future[bool] | None = None + future: Future[bool] | None = None + with self._reconcile_prefetch_executor_lock: + if self._reconcile_prefetch_executor_shutdown: + return None + pending = self._reconcile_prefetch_pending + if pending is not None: + pending_tip, pending_future, pending_proves = pending + if ( + not pending_future.done() + and pending_tip == tip_hash + and (pending_proves or not prove) + ): + # A proving pass satisfies both kinds of caller; a + # memo-honoring pass cannot satisfy a prove request and + # is replaced below like a tip change. + return pending_future + # Replaced tip or completed future: hand the old future off + # for disposal outside this lock -- cancellation runs done + # callbacks inline on this thread, and _clear_slot below + # re-takes the lock. + self._reconcile_prefetch_pending = None + if not pending_future.done(): + stale_future = pending_future + executor = self._reconcile_prefetch_executor + if executor is None: + executor = ThreadPoolExecutor( + max_workers=1, + thread_name_prefix="prism-reconcile-prefetch", + ) + self._reconcile_prefetch_executor = executor + try: + future = executor.submit( + self.prefetch_pass, tip_hash, prove + ) + self._reconcile_prefetch_pending = (tip_hash, future, prove) + except RuntimeError: + # Executor shutdown raced this submit; the serial path + # covers it (after the stale future is disposed below). + future = None + if stale_future is not None and not stale_future.cancel(): + # Already running for a replaced tip; let it finish detached. + # The slot holds the new tip, so at most one task ever waits + # behind the running one. + self.discard_stale_prefetch(stale_future) + if future is None: + return None + + def _clear_slot(done: Future[bool]) -> None: + with self._reconcile_prefetch_executor_lock: + pending_now = self._reconcile_prefetch_pending + if pending_now is not None and pending_now[1] is done: + self._reconcile_prefetch_pending = None + + # Registered outside the slot lock: a completed future runs the + # callback inline on this thread. + future.add_done_callback(_clear_slot) + return future + + @staticmethod + def discard_stale_prefetch(future: Future[bool]) -> None: + """Detach a prefetched pass whose tip was superseded during the + template fetch. + + The pass cannot be cancelled mid-ledger-mutation and already records + its own outcome/error accounting; the serial pass for the current + tip re-surfaces any condition that still applies. Observe its terminal + error without re-raising it into a callback frame that holds the future. + """ + + def _consume(done: Future[bool]) -> None: + if not done.cancelled(): + done.exception() + + future.add_done_callback(_consume) + + def join_prefetch_bounded(self, prefetch: Future[bool]) -> bool: + """Join a reconcile pass under the poll loop's bounded budget. + + On a genuine expiry the pass keeps running in its single prefetch + slot -- the paced retry re-joins the same future -- and the timeout + surfaces the normal blocked-retry path, keeping the poll loop's + liveness heartbeat fed while the pass catches up. The budget is + clamped below the loop's failure budget so a misconfigured override + cannot reinstate the park this bound exists to prevent. + """ + + join_timeout = min( + PRISM_RECONCILE_PREFETCH_JOIN_TIMEOUT_CEILING_SECONDS, + max( + 0.001, + float(self._ports.prefetch_join_timeout_seconds()), + ), + ) + try: + return prefetch.result(timeout=join_timeout) + except TimeoutError: + if prefetch.done(): + # The pass itself raised TimeoutError (socket.timeout is + # TimeoutError here): not a join expiry. Propagate silently + # so diagnosis points at the pass, not the join. + raise + print( + "prism coordinator: reconcile prefetch join exceeded " + f"{join_timeout:g}s; retrying refresh pass while it " + "completes", + flush=True, + ) + raise + + def snapshot_tip_bounded(self, tip_hash: str) -> bool: + """Run a snapshot-tip re-prove off-thread with the bounded join. + + The serial re-prove branches run the same crawling pass as the + overlapped prefetch; routing them through the prefetch slot gives + the poll loop one uniform bounded wait. Falls back to the direct + pass only when the prefetch executor has already been retired at + shutdown, whose exceptions the caller already maps to a clean exit. + """ + + prefetch = self.submit_prefetch(tip_hash, prove=True) + if prefetch is None: + with self._attributed(RECONCILE_CALLER_TIP_REFRESH): + return self._ports.ensure_tip(tip_hash) + return self.join_prefetch_bounded(prefetch) + + def shutdown_prefetch_executor(self) -> None: + self._ports.ensure_job_cache_state() + with self._reconcile_prefetch_executor_lock: + executor = self._reconcile_prefetch_executor + self._reconcile_prefetch_executor = None + self._reconcile_prefetch_executor_shutdown = True + self._reconcile_prefetch_pending = None + if executor is not None: + # A pass blocked on writer admission aborts via + # ShutdownInProgress on its own; never hold shutdown for it. + executor.shutdown(wait=False, cancel_futures=True) + + def ensure_tip( + self, + tip_hash: str, + *, + _coalesce_same_tip: bool = True, + _caller: str | None = None, + ) -> bool: + """Reconcile one tip, optionally bypassing same-tip flight reuse. + + Lock-owning accepted-block callers disable waiting for an existing + leader because it may itself be waiting for the payout-balance + mutation lock. Their own pass remains visible to ordinary followers. + ``_caller`` is the optional #224 attribution label; without one, the + flight bypass identifies the accepted-block landing (its only user) + and every other caller keeps whatever label its entry point stamped. + """ + if not self.enabled: + return True + caller = _caller + if caller is None and not _coalesce_same_tip: + caller = RECONCILE_CALLER_LANDING + with self._attributed(caller): + if _coalesce_same_tip: + summary = self._ports.reconcile_with_admission( + tip_hash=tip_hash + ) + else: + summary = self._ports.reconcile_with_admission( + tip_hash=tip_hash, + _wait_for_same_tip_flight=False, + ) + return not bool(summary.get("untrusted") or summary.get("superseded")) + + def reconcile_with_flights( + self, + *, + tip_hash: str | None = None, + _force_publish: bool = False, + _source_reserved: bool = False, + _wait_for_same_tip_flight: bool = True, + _caller: str | None = None, + ) -> dict[str, object]: + """Reconcile pool blocks, coalescing same-tip concurrent callers. + + Unflagged callers asking about a tip whose pass is already in flight + await that pass and share its summary instead of queueing another + full serialized pass. Callers carrying side-effect obligations (a + forced publication or an already-reserved source) and callers without + a tip always run their own pass. Lock-owning callers may disable + waiting for an existing flight while still registering as the visible + leader when no same-tip flight exists. ``_caller`` is the optional + #224 attribution label; a forced publication without one is the + post-confirm path (its only user). + """ + self._ports.ensure_job_cache_state() + caller = _caller + if caller is None and _force_publish: + caller = RECONCILE_CALLER_POST_CONFIRM + if tip_hash is None or _force_publish or _source_reserved: + return self._serialized_pass( + caller, + tip_hash=tip_hash, + _force_publish=_force_publish, + _source_reserved=_source_reserved, + ) + with self._reconcile_flight_lock: + flight = self._reconcile_flights.get(tip_hash) + leading = flight is None + if leading: + flight = _ReconcileFlight() + self._reconcile_flights[tip_hash] = flight + if not leading: + if not _wait_for_same_tip_flight: + return self._serialized_pass(caller, tip_hash=tip_hash) + wait_seconds = float(self._ports.flight_wait_seconds()) + if flight.event.wait(timeout=wait_seconds): + exception = flight.exception + if exception is not None: + raise exception + summary = flight.summary + assert summary is not None + # Followers get a copy: summaries are mutable dicts and the + # leader's caller already holds the original. + return dict(summary) + # Liveness backstop: the leader outlived the wait. Run our own + # pass; the writer lock still serializes the actual work. + return self._serialized_pass(caller, tip_hash=tip_hash) + try: + summary = self._serialized_pass(caller, tip_hash=tip_hash) + flight.summary = summary + return summary + except BaseException as exc: + flight.exception = exc + raise + finally: + with self._reconcile_flight_lock: + self._reconcile_flights.pop(tip_hash, None) + flight.event.set() + + def reconcile( + self, + *, + tip_hash: str | None = None, + force_publish: bool = False, + source_reserved: bool = False, + caller: str | None = None, + ) -> dict[str, object]: + """The core reconcile state machine (behind the serialized adapter). + + ``caller`` is the bounded #224 attribution label. When omitted it is + the label the entry point stamped on this thread, ``post_confirm`` + for a forced publication without one, and ``other`` for tools, + benchmarks and periodic passes. The pass family is recorded on every + exit -- success, untrusted skip, superseded budget, and the + error-after-possible-commit path -- and never changes the outcome. + """ + summary: dict[str, object] = { + "enabled": bool(self.enabled), + "untrusted": False, + "superseded": False, + "published_generation": None, + "watched_blocks": 0, + "inactive_blocks": 0, + "reactivated_blocks": 0, + "matured_payouts": 0, + "stranded_prepared_rejected": 0, + "stranded_prepared_canonical": 0, + } + if not self.enabled: + return summary + self._ports.ensure_job_cache_state() + if caller is None: + caller = self._attributed_caller() + if caller is None: + caller = ( + RECONCILE_CALLER_POST_CONFIRM + if force_publish + else RECONCILE_CALLER_OTHER + ) + if caller not in PRISM_REORG_RECONCILE_CALLERS: + raise ValueError(f"unknown reorg reconcile caller: {caller!r}") + telemetry = self.accepted_preview_telemetry + pass_started = time.monotonic() + local = self._reconcile_attribution + admission_started = getattr(local, "admission_started", None) + local.admission_started = None + if admission_started is not None: + # Stamped by _serialized_pass ahead of the coordinator adapter's + # writer admission and payout-balance mutation lock: the one + # stretch a tip-refresh caller spends behind a landing. + telemetry.observe_reconcile_step( + caller, + RECONCILE_STEP_ADMISSION_WAIT, + pass_started - admission_started, + ) + + def step(name: str) -> AbstractContextManager[None]: + return telemetry.reconcile_step(caller, name) + + try: + return self._reconcile_pass( + summary, + tip_hash=tip_hash, + force_publish=force_publish, + source_reserved=source_reserved, + step=step, + ) + finally: + telemetry.observe_reconcile_pass( + caller, + time.monotonic() - pass_started, + ) + + def _reconcile_pass( + self, + summary: dict[str, object], + *, + tip_hash: str | None, + force_publish: bool, + source_reserved: bool, + step: Callable[[str], AbstractContextManager[None]], + ) -> dict[str, object]: + """One attributed pass; ``step`` times each bounded reconcile step.""" + if not source_reserved and tip_hash is not None: + # Tip observation normally reserves this source before queueing + # reconciliation. Direct callers only need a new source when they + # are asking about a different tip; repeated reconciliation of the + # same tip must not supersede otherwise valid prepared work. + current_source_tip = self._ports.source_tip() + if current_source_tip != tip_hash: + self._ports.reserve_external_tip(tip_hash) + + inactive_blocks_total = 0 + reactivated_blocks_total = 0 + matured_payouts_total = 0 + stranded_prepared_rejected_total = 0 + stranded_prepared_canonical_total = 0 + supersession_retries = 0 + skip_recorded = False + max_supersession_retries = max( + 0, + int(self._ports.max_supersession_retries()), + ) + + proof_epoch = self._ports.reorg_proof_snapshot()[1] + + def finish(*, trusted: bool) -> dict[str, object]: + with self._ports.state_lock(): + self.inactive_block_count += inactive_blocks_total + self.reactivated_block_count += reactivated_blocks_total + self.matured_payout_count += matured_payouts_total + self.note_outcome( + tip_hash, + trusted=trusted, + # Row mutations invalidate proofs cached for other tips even + # when the mutating pass's tip was never observed (per-client + # callers reconcile straight off getbestblockhash). + evict_others=bool( + inactive_blocks_total + or reactivated_blocks_total + or matured_payouts_total + or stranded_prepared_rejected_total + ), + proof_epoch=proof_epoch, + ) + summary["inactive_blocks"] = inactive_blocks_total + summary["reactivated_blocks"] = reactivated_blocks_total + summary["matured_payouts"] = matured_payouts_total + summary["stranded_prepared_rejected"] = ( + stranded_prepared_rejected_total + ) + summary["stranded_prepared_canonical"] = ( + stranded_prepared_canonical_total + ) + return summary + + def retry_superseded_candidate() -> bool: + nonlocal supersession_retries, tip_hash + supersession_retries += 1 + if supersession_retries > max_supersession_retries: + summary["superseded"] = True + self._ports.block_publication() + return False + latest_tip = self._ports.source_tip() + tip_hash = latest_tip or tip_hash + return True + + while True: + candidate_to_publish: PayoutStateCandidate | None = None + error_candidate: PayoutStateCandidate | None = None + attempt_trusted = True + # The memo entry this attempt may arm must prove state for the + # epoch its reads happen in; a detection cycle during the pass + # (away, or away and back) refuses the arm in note_outcome. + proof_epoch = self._ports.reorg_proof_snapshot()[1] + try: + with self._ports.prepare_lock(): + prepared_started = time.monotonic() + captured_source = self._ports.capture_source() + payout_changed = False + payout_mutation_attempted = False + inactive_blocks = 0 + reactivated_blocks = 0 + matured_payouts = 0 + stranded_prepared_rejected = 0 + stranded_prepared_canonical = 0 + summary["untrusted"] = False + summary["watched_blocks"] = 0 + try: + if self._ports.chain_view_untrusted(): + if not skip_recorded: + with self._ports.state_lock(): + self.reconcile_skip_count += 1 + skip_recorded = True + summary["untrusted"] = True + attempt_trusted = False + if force_publish: + with step(RECONCILE_STEP_CANDIDATE_PREPARE): + candidate_to_publish = ( + self._ports.prepared_candidate( + captured_source + ) + ) + else: + # Every stamp below marks one completed round + # trip. A landing thread's watchdog budget is + # sized for a single statement, and this pass + # crosses an unbounded number of them, so the + # stamps have to follow the work itself rather + # than bracket the pass as a whole. + self._ports.record_progress( + "reorg-reconcile:tip-height" + ) + with step(RECONCILE_STEP_CHAIN_PROBE): + active_tip_height = int( + self._ports.rpc_call("getblockcount") + ) + ledger = self._ports.ledger() + watch_blocks = getattr( + ledger, + "reorg_watch_blocks", + None, + ) + if not callable(watch_blocks): + if ( + force_publish + or self._ports.captured_publication_required( + captured_source + ) + ): + with step(RECONCILE_STEP_CANDIDATE_PREPARE): + candidate_to_publish = ( + self._ports.prepared_candidate( + captured_source, + **self._candidate_intent( + payout_changed + ), + ) + ) + else: + self._ports.record_progress( + "reorg-reconcile:watch-blocks" + ) + with step(RECONCILE_STEP_WATCH_QUERY): + rows = watch_blocks( + active_tip_height=active_tip_height + ) + summary["watched_blocks"] = len(rows) + + for row in rows: + # Per row, not per pass: the row count is + # bounded only by how many pool blocks the + # reorg window holds, and each iteration + # can spend a getblockhash plus a ledger + # mutation. + self._ports.record_progress( + "reorg-reconcile:watch-block" + ) + block_height = int(row["block_height"]) + block_hash = str(row["block_hash"]).lower() + chain_state = str(row.get("chain_state", "")) + if block_height > active_tip_height: + if chain_state == "confirmed": + payout_mutation_attempted = True + with step(RECONCILE_STEP_MUTATIONS): + inactive = ( + ledger.mark_pool_block_inactive( + block_hash=block_hash, + active_tip_height=active_tip_height, + ) + ) + inactive_count = int( + inactive.get("inactive_count", 0) + ) + inactive_blocks += inactive_count + payout_changed = ( + payout_changed + or bool(inactive_count) + ) + continue + with step(RECONCILE_STEP_CHAIN_PROBE): + active_hash = str( + self._ports.rpc_call( + "getblockhash", + [block_height], + ) + ).lower() + on_active_chain = active_hash == block_hash + if ( + on_active_chain + and chain_state == "inactive" + ): + payout_mutation_attempted = True + with step(RECONCILE_STEP_MUTATIONS): + with self._ports.publication_guard(): + reactivated = ledger.reactivate_pool_block( + block_hash=block_hash, + active_tip_height=active_tip_height, + ) + reactivated_count = int( + reactivated.get( + "reactivated_count", + 0, + ) + ) + reactivated_blocks += reactivated_count + payout_changed = ( + payout_changed + or bool(reactivated_count) + ) + elif ( + not on_active_chain + and chain_state == "confirmed" + ): + payout_mutation_attempted = True + with step(RECONCILE_STEP_MUTATIONS): + inactive = ( + ledger.mark_pool_block_inactive( + block_hash=block_hash, + active_tip_height=active_tip_height, + ) + ) + inactive_count = int( + inactive.get("inactive_count", 0) + ) + inactive_blocks += inactive_count + payout_changed = ( + payout_changed + or bool(inactive_count) + ) + + # Rows the watch loop above structurally + # cannot see: it selects confirmed/inactive + # only, because a prepared row belongs to the + # live submit/replay path that holds its + # outbox entry. When that entry is gone the + # row has no owner left, and an orphaned one + # pins its payout entries, carry forward, and + # CTV fanout artifacts immature forever. + stranded_prepared = getattr( + ledger, + "stranded_prepared_blocks", + None, + ) + if callable(stranded_prepared): + self._ports.record_progress( + "reorg-reconcile:stranded-prepared-blocks" + ) + # The sweep's row read is the same kind + # of watch-set query as the watch loop's. + with step(RECONCILE_STEP_WATCH_QUERY): + stranded_rows = stranded_prepared( + active_tip_height=active_tip_height, + min_depth=( + STRANDED_PREPARED_REJECT_MIN_DEPTH + ), + limit=STRANDED_PREPARED_SWEEP_LIMIT, + ) + for row in stranded_rows: + # Per row for the same reason as the + # watch loop: a getblockhash plus a + # fenced ledger mutation each. + self._ports.record_progress( + "reorg-reconcile:stranded-prepared" + ) + block_height = int(row["block_height"]) + block_hash = str( + row["block_hash"] + ).lower() + with step(RECONCILE_STEP_CHAIN_PROBE): + active_hash = str( + self._ports.rpc_call( + "getblockhash", + [block_height], + ) + ).lower() + if active_hash == block_hash: + # Canonical but never confirmed: + # the confirm path owns audit + # publication sequencing, so this + # sweep must never take the row + # from under it. Report loudly and + # leave it alone. + stranded_prepared_canonical += 1 + print( + "prism coordinator: stranded " + "prepared pool block is " + "canonical and needs operator " + "review " + f"hash={block_hash} " + f"height={block_height} " + "depth=" + f"{active_tip_height - block_height}", + flush=True, + ) + continue + payout_mutation_attempted = True + with step(RECONCILE_STEP_MUTATIONS): + rejected = ledger.reject_prepared_block( + block_hash=block_hash, + active_tip_height=active_tip_height, + ) + # The fenced ledger function cascades + # payout entries, carry forward, and + # fanout artifacts, and returns 0 when + # the row already moved under us. + rejected_count = int( + rejected.get("rejected_count", 0) + ) + stranded_prepared_rejected += ( + rejected_count + ) + payout_changed = ( + payout_changed + or bool(rejected_count) + ) + if rejected_count: + print( + "prism coordinator: rejected " + "orphaned stranded prepared " + "pool block " + f"hash={block_hash} " + f"height={block_height} " + "depth=" + f"{active_tip_height - block_height}", + flush=True, + ) + + mark_mature = getattr( + ledger, + "mark_mature_pool_payouts", + None, + ) + if callable(mark_mature): + payout_mutation_attempted = True + self._ports.record_progress( + "reorg-reconcile:mature-payouts" + ) + with step(RECONCILE_STEP_MUTATIONS): + matured = mark_mature( + active_tip_height=active_tip_height + ) + matured_payouts = int( + matured.get("matured_count", 0) + ) + payout_changed = ( + payout_changed + or bool(matured_payouts) + ) + + inactive_blocks_total += inactive_blocks + reactivated_blocks_total += reactivated_blocks + matured_payouts_total += matured_payouts + stranded_prepared_rejected_total += ( + stranded_prepared_rejected + ) + stranded_prepared_canonical_total += ( + stranded_prepared_canonical + ) + if ( + payout_changed + or force_publish + or self._ports.captured_publication_required( + captured_source + ) + ): + # Candidate preparation embeds the ledger + # snapshot artifact; a pass that will not + # publish must not pay for one only to + # discard it. A confirmed payout mutation + # asks for a live prior-balances reread + # over the still-exact share window, not + # a full rescan (#224; see + # _candidate_intent). + self._ports.record_progress( + "reorg-reconcile:prepare-candidate" + ) + with step(RECONCILE_STEP_CANDIDATE_PREPARE): + candidate_to_publish = ( + self._ports.prepared_candidate( + captured_source, + **self._candidate_intent( + payout_changed + ), + ) + ) + except Exception: + inactive_blocks_total += inactive_blocks + reactivated_blocks_total += reactivated_blocks + matured_payouts_total += matured_payouts + stranded_prepared_rejected_total += ( + stranded_prepared_rejected + ) + stranded_prepared_canonical_total += ( + stranded_prepared_canonical + ) + # Durable partial mutations close admission before the + # preparation lock is released. Publication drains old + # socket sends afterward without blocking new ledger + # preparation or snapshot acquisition. + if payout_mutation_attempted: + # A mutator can commit server-side and still raise + # locally when its response is lost. The observed + # row counts are then unavailable, so conservatively + # force the same ledger re-read as a confirmed + # mutation. Read-only failures never reach this flag. + payout_changed = True + if payout_changed: + # Fail closed on the uncertain path: the forced + # full rescan is unchanged here and necessarily + # rereads the current prior balances, so a lost + # mutator response can never republish stale + # balances. + with step(RECONCILE_STEP_CANDIDATE_PREPARE): + error_candidate = ( + self._ports.prepared_candidate( + captured_source, + force_full_window_rescan=True, + ) + ) + self._ports.block_publication(force=True) + with self._ports.state_lock(): + self.inactive_block_count += ( + inactive_blocks_total + ) + self.reactivated_block_count += ( + reactivated_blocks_total + ) + self.matured_payout_count += matured_payouts_total + self.reconcile_error_count += 1 + # A pass that errored mid-mutation invalidates every + # cached outcome, not just its own tip's. + self.note_outcome( + tip_hash, + trusted=False, + clear_memo=True, + ) + raise + finally: + self._ports.observe_preparation( + max(0.0, time.monotonic() - prepared_started) + ) + + if candidate_to_publish is not None: + # Atomically fence cache/build/delivery admission before + # releasing the ledger snapshot lock. The potentially + # slow drain then happens in publication() below. + self._ports.block_publication(force=True) + except Exception: + if error_candidate is not None: + with step(RECONCILE_STEP_PUBLISH): + published_after_error = self._ports.publish_candidate( + error_candidate + ) + if published_after_error is None: + self._ports.block_publication() + raise + + if candidate_to_publish is not None: + self._ports.record_progress("reorg-reconcile:publish") + with step(RECONCILE_STEP_PUBLISH): + published = self._ports.publish_candidate( + candidate_to_publish + ) + if published is None: + # Preserve durable counts and retry iteratively against the + # newest source. The explicit budget prevents tip churn + # from monopolizing preparation indefinitely; the fence + # stays closed between attempts. + if retry_superseded_candidate(): + continue + return finish(trusted=False) + summary["published_generation"] = published + return finish(trusted=attempt_trusted) + + +class ReorgCompatibilityField: + """Route retained coordinator fields to their single service owner.""" + + def __init__(self, name: str, default: object) -> None: + self.name = name + self.default = default + self.backing = f"_reorg_compat_{name}" + + def __get__(self, instance: Any, owner: type[Any]) -> object: + if instance is None: + return self + service = instance.__dict__.get("_reorg_reconciler_service") + if service is not None: + return getattr(service, self.name) + return instance.__dict__.get(self.backing, self.default) + + def __set__(self, instance: Any, value: object) -> None: + service = instance.__dict__.get("_reorg_reconciler_service") + if service is not None: + setattr(service, self.name, value) + return + instance.__dict__[self.backing] = value diff --git a/lab/prism/rpc.py b/lab/prism/rpc.py new file mode 100644 index 000000000..31455a557 --- /dev/null +++ b/lab/prism/rpc.py @@ -0,0 +1,241 @@ +"""Thread-local JSON-RPC client used by PRISM processes.""" + +from __future__ import annotations + +import base64 +import http.client +import json +import socket +import threading +import time +import urllib.parse +from typing import Any + + +# The default deadline for every JsonRpc.call without an explicit timeout, +# including the fence-guarded CTV broadcast RPCs. The ledger's own-write +# deferral margin is derived from the guarded deadlines, so route changes +# through this constant rather than the call signature's literal. +DEFAULT_QBIT_RPC_CALL_TIMEOUT_SECONDS = 10.0 + +_QBIT_RPC_NO_TRANSPORT_RETRY_METHODS = frozenset( + { + "getnewaddress", + "sendrawtransaction", + "signrawtransactionwithwallet", + "submitblock", + "submitpackage", + } +) + + +class JsonRpc: + def __init__(self, *, host: str, port: int, user: str, password: str): + self.host = host + self.port = port + self.url = f"http://{host}:{port}" + credentials = f"{user}:{password}".encode() + self.auth = f"Basic {base64.b64encode(credentials).decode()}" + # Keep-alive connections, one per calling thread. qbitd is called on + # the hot share/block paths (a fresh getaddrinfo + TCP connect per call + # was ~seconds of overhead under load); reusing the connection removes + # that. threading.local keeps each thread's HTTPConnection private, so + # concurrent callers never share a non-thread-safe connection. + self._connections = threading.local() + + def _acquire_connection(self, timeout: float) -> http.client.HTTPConnection: + conn = getattr(self._connections, "conn", None) + if conn is None: + conn = http.client.HTTPConnection(self.host, self.port, timeout=timeout) + # Never let http.client resurrect a severed connection: with + # auto_open, request() silently reconnects when the deadline + # watchdog's close() lands between the pre-send check and the + # send, and that implicit fresh socket lives until the next + # watchdog sweep — long enough for a short mutating POST to + # reach qbitd after a caller released its ordering locks. With + # auto_open off a cleared socket makes send() raise + # NotConnected instead; call() connects explicitly under the + # watchdog and rechecks the deadline before any byte goes out. + conn.auto_open = 0 + self._connections.conn = conn + else: + # Reuse: refresh the deadline for this call on the live socket. + conn.timeout = timeout + if conn.sock is not None: + conn.sock.settimeout(timeout) + return conn + + def _drop_connection(self) -> None: + conn = getattr(self._connections, "conn", None) + if conn is not None: + try: + conn.close() + except Exception: + pass + self._connections.conn = None + + def call( + self, + method: str, + params: list[object] | None = None, + *, + wallet: str | None = None, + timeout: float = DEFAULT_QBIT_RPC_CALL_TIMEOUT_SECONDS, + ) -> Any: + streamed_block = ( + params[0] if method == "submitblock" and params and len(params) == 1 + and callable(getattr(params[0], "iter_byte_chunks", None)) else None + ) + body = None if streamed_block is not None else json.dumps( + { + "jsonrpc": "1.0", + "id": method, + "method": method, + "params": params or [], + } + ).encode() + path = "/" + if wallet is not None: + path = f"/wallet/{urllib.parse.quote(wallet, safe='')}" + headers = { + "Authorization": self.auth, + "Content-Type": "application/json", + "User-Agent": "qbit-prism-coordinator/0.1", + } + if streamed_block is not None: + prefix = b'{"jsonrpc":"1.0","id":"submitblock","method":"submitblock","params":[' + suffix = b']}' + headers["Content-Length"] = str(len(prefix) + streamed_block.byte_length + len(suffix)) + # Read-only calls retry once with a fresh connection after a transport + # error, normally an idle keep-alive that qbitd closed. Mutating calls + # never retry inside this method: their writer-lease fence applies to + # the outer call, so an invisible second POST could otherwise run after + # that lease was lost. Durable block/CTV workflows retry later as a new + # fully fenced operation and reconcile an uncertain first result. + last_exc: Exception | None = None + attempt_count = ( + 1 if method in _QBIT_RPC_NO_TRANSPORT_RETRY_METHODS else 2 + ) + deadline = time.monotonic() + max(0.001, float(timeout)) + for attempt in range(attempt_count): + remaining = deadline - time.monotonic() + if remaining <= 0: + if last_exc is not None: + raise last_exc + raise TimeoutError(f"qbit RPC {method} timed out") + conn = self._acquire_connection(remaining) + # The socket timeout bounds each individual socket operation, + # not the call: a response body arriving one packet per + # interval extends the call arbitrarily past its nominal + # deadline. Lease authority margins are sized from this + # timeout as a wall-clock bound on fence-guarded effects, so + # enforce it end-to-end: past the deadline a watchdog severs + # the connection's socket — repeatedly, because a stalled name + # resolution has no socket yet and the one connect() creates + # afterwards must not carry the RPC onward — until the attempt + # observes the abort as a transport error. For mutating calls + # that is the same uncertain outcome a socket timeout already + # produces, reconciled by durable replay as a new fully fenced + # operation. + watchdog_fired = threading.Event() + attempt_finished = threading.Event() + + # The allowance is the attempt's exact remaining wall clock, no + # grace: callers like the submitblock hard-deadline wrapper + # abandon the call and release ordering locks (the payout + # landing fence) the instant their deadline passes, so a byte + # transmitted in any grace window would race work those callers + # already unblocked. + def _enforce_deadline( + doomed: http.client.HTTPConnection = conn, + fired: threading.Event = watchdog_fired, + finished: threading.Event = attempt_finished, + allowance: float = remaining, + ) -> None: + if finished.wait(allowance): + return + fired.set() + while True: + sock = getattr(doomed, "sock", None) + if sock is not None: + try: + sock.shutdown(socket.SHUT_RDWR) + except OSError: + pass + try: + doomed.close() + except Exception: + pass + if finished.wait(0.1): + return + + watchdog = threading.Thread( + target=_enforce_deadline, + name="qbit-rpc-deadline", + daemon=True, + ) + watchdog.start() + try: + # Establish the connection (name resolution + TCP) under + # the watchdog and re-check the deadline before any request + # byte is sent. Left to conn.request(), a name resolution + # returning after the deadline resumes straight into + # sendall(), and a short mutating POST can reach qbitd + # between the watchdog's periodic sweeps; the explicit + # check makes the late-connect case lose deterministically. + # The check is against the wall clock, not just the + # watchdog flag: the flag lags the deadline by thread + # scheduling, and a caller that abandoned this call at its + # deadline may already have released ordering locks that + # a post-deadline send would race. + if getattr(conn, "sock", None) is None: + conn_connect = getattr(conn, "connect", None) + if conn_connect is not None: + conn_connect() + if watchdog_fired.is_set() or time.monotonic() >= deadline: + raise TimeoutError(f"qbit RPC {method} timed out") + def request_chunks(): + yield prefix + for chunk in streamed_block.iter_byte_chunks(): + if watchdog_fired.is_set() or time.monotonic() >= deadline: + raise TimeoutError(f"qbit RPC {method} timed out") + yield chunk + yield suffix + + conn.request("POST", path, body=request_chunks() if streamed_block is not None else body, headers=headers) + response = conn.getresponse() + data = response.read() # drain so the connection can be reused + except (http.client.HTTPException, OSError) as exc: + self._drop_connection() + if watchdog_fired.is_set(): + raise TimeoutError(f"qbit RPC {method} timed out") from exc + last_exc = exc + if attempt + 1 < attempt_count: + continue + raise + except BaseException: + # A streamed source can fail after sending a request prefix. + # That connection cannot carry a later RPC safely. + self._drop_connection() + raise + finally: + attempt_finished.set() + if response.status != 200: + # Non-200 bodies may hold a JSON-RPC error (qbitd returns the + # error object with a 500 for some methods); surface it as the + # same RuntimeError text callers already match on (e.g. the + # "-32601 / Method not found" blockwait-unsupported probe). + self._drop_connection() + detail = data.decode("utf-8", "replace") + try: + error = json.loads(detail).get("error") + except Exception: + error = None + if error is not None: + raise RuntimeError(f"qbit RPC {method} failed: {error}") + raise RuntimeError(f"qbit RPC {method} HTTP {response.status}: {detail[:200]}") + payload = json.loads(data) + if payload["error"] is not None: + raise RuntimeError(f"qbit RPC {method} failed: {payload['error']}") + return payload["result"] + raise last_exc if last_exc is not None else RuntimeError("qbit RPC call failed") diff --git a/lab/prism/run_ctv_broadcaster_daemon.py b/lab/prism/run_ctv_broadcaster_daemon.py index c8cd26340..642f930f1 100644 --- a/lab/prism/run_ctv_broadcaster_daemon.py +++ b/lab/prism/run_ctv_broadcaster_daemon.py @@ -19,8 +19,25 @@ CtvFanoutDaemonResult, MAX_CTV_FANOUT_BROADCASTER_CHUNK_SIZE, ) +from lab.prism.coordinator_config import ( + DEFAULT_LEASE_ACQUIRE_ATTEMPTS, + DEFAULT_LEASE_ACQUIRE_LOCK_TIMEOUT_SECONDS, + DEFAULT_POSTGRES_IDLE_IN_TRANSACTION_TIMEOUT_SECONDS, + DEFAULT_POSTGRES_TCP_KEEPALIVES_COUNT, + DEFAULT_POSTGRES_TCP_KEEPALIVES_IDLE_SECONDS, + DEFAULT_POSTGRES_TCP_KEEPALIVES_INTERVAL_SECONDS, + env, + env_bool, + env_int, + env_positive_float, +) + +# The module-level env_positive_int below keeps its required-missing +# semantics for the daemon's own knobs; the coordinator_config variant is +# aliased for the ledger knobs so both scripts read them identically. +from lab.prism.coordinator_config import env_positive_int as config_env_positive_int +from lab.prism.rpc import JsonRpc from lab.prism.share_ledger import PsqlShareLedger, SingleWriterShareLedger -from lab.prism.prism_coordinator import JsonRpc, env, env_bool, env_int, env_positive_float def env_positive_int(name: str, default: int | None = None) -> int: @@ -80,6 +97,30 @@ def make_ledger_from_env() -> SingleWriterShareLedger | PsqlShareLedger: writer_session_token=os.environ.get("PRISM_LEDGER_WRITER_SESSION_TOKEN"), initialize_schema=env("PRISM_POSTGRES_INIT_SCHEMA", "0") in {"1", "true", "yes"}, lease_ttl_seconds=env_positive_float("PRISM_LEDGER_LEASE_TTL_SECONDS", 60.0), + lease_acquire_lock_timeout_seconds=env_positive_float( + "PRISM_LEDGER_LEASE_ACQUIRE_LOCK_TIMEOUT_SECONDS", + DEFAULT_LEASE_ACQUIRE_LOCK_TIMEOUT_SECONDS, + ), + lease_acquire_attempts=config_env_positive_int( + "PRISM_LEDGER_LEASE_ACQUIRE_ATTEMPTS", + DEFAULT_LEASE_ACQUIRE_ATTEMPTS, + ), + postgres_idle_in_transaction_timeout_seconds=env_positive_float( + "PRISM_POSTGRES_IDLE_IN_TRANSACTION_TIMEOUT_SECONDS", + DEFAULT_POSTGRES_IDLE_IN_TRANSACTION_TIMEOUT_SECONDS, + ), + postgres_tcp_keepalives_idle_seconds=config_env_positive_int( + "PRISM_POSTGRES_TCP_KEEPALIVES_IDLE_SECONDS", + DEFAULT_POSTGRES_TCP_KEEPALIVES_IDLE_SECONDS, + ), + postgres_tcp_keepalives_interval_seconds=config_env_positive_int( + "PRISM_POSTGRES_TCP_KEEPALIVES_INTERVAL_SECONDS", + DEFAULT_POSTGRES_TCP_KEEPALIVES_INTERVAL_SECONDS, + ), + postgres_tcp_keepalives_count=config_env_positive_int( + "PRISM_POSTGRES_TCP_KEEPALIVES_COUNT", + DEFAULT_POSTGRES_TCP_KEEPALIVES_COUNT, + ), ) diff --git a/lab/prism/share_json_stream.py b/lab/prism/share_json_stream.py new file mode 100644 index 000000000..b90664e09 --- /dev/null +++ b/lab/prism/share_json_stream.py @@ -0,0 +1,363 @@ +#!/usr/bin/env python3 +"""Bounded JSON encoding and digesting for PRISM share arrays. + +Why this exists: a payout window at union-mainnet size (roughly 210k +accepted shares; 400k at the stress size) used to pass through single +``json.dumps``/``json.loads`` calls -- the daemon prepare request, the +audit-builder compact payload, the digest of a freshly read snapshot, the +lazy parse of a daemon-prepared window -- each one a C call that holds the +GIL for hundreds of milliseconds while the writer-lease monitor thread waits +to run (#236). Every helper here produces or consumes exactly the bytes the +historical whole-array call did, while bounding any one C call to a batch +of records, so the interpreter can switch threads between batches. + +Byte identity is the contract, not a nicety: canonical share bytes are +hashed into ``share_snapshot_sha256`` and mirrored by the Rust builder, so +the streamed encoding must reproduce CPython's ``json.dumps`` output exactly +(``ensure_ascii`` escapes, non-BMP surrogate pairs, sorted keys where +canonical, arbitrary-precision integers). The batch form relies on one fact +about CPython's encoder that the tests pin: ``json.dumps(items)[1:-1]`` is +the items' individual encodings joined by ``,``, so a batch of any size +encodes to the same bytes as the whole array minus its brackets. + +This module is a leaf: standard library only, imported by the ledger, the +bundle compiler and the payout-state service alike. +""" + +from __future__ import annotations + +import hashlib +import json +from collections.abc import Collection, Iterable, Iterator, Mapping, Sequence +from typing import Any, Callable + +# Records encoded per ``json.dumps`` call. 512 matches the payout window's +# page size, so a page encodes in exactly one bounded call. +SHARE_JSON_BATCH_RECORDS = 512 +# Target size of one yielded chunk. Batches are coalesced up to this many +# characters (bytes: every chunk is ASCII) before being handed to a +# transport, so a pipe write or spool write moves tens of KiB at a time +# instead of one record. +SHARE_JSON_CHUNK_BYTES = 64 * 1024 + +_COMPACT_SEPARATORS = (",", ":") +# Entries dropped per ``del`` slice by release_share_list_incrementally. +RELEASE_BATCH_RECORDS = 2048 + + +def _batch_encoder( + *, + sort_keys: bool, + default: Callable[[Any], Any] | None, +) -> Callable[[list[Any]], str]: + """One ``json.dumps`` over a batch, minus the array brackets.""" + + def encode(batch: list[Any]) -> str: + return json.dumps( + batch, + sort_keys=sort_keys, + separators=_COMPACT_SEPARATORS, + default=default, + )[1:-1] + + return encode + + +class _ChunkCoalescer: + """Accumulates text pieces and releases them in bounded chunks.""" + + __slots__ = ("_chunk_chars", "_pending", "_pending_chars") + + def __init__(self, chunk_chars: int) -> None: + self._chunk_chars = int(chunk_chars) + self._pending: list[str] = [] + self._pending_chars = 0 + + def push(self, text: str) -> str | None: + """Take one piece; return a chunk once the target size is reached.""" + if not text: + return None + self._pending.append(text) + self._pending_chars += len(text) + if self._pending_chars < self._chunk_chars: + return None + return self.flush() + + def flush(self) -> str | None: + if not self._pending: + return None + chunk = ( + self._pending[0] + if len(self._pending) == 1 + else "".join(self._pending) + ) + self._pending = [] + self._pending_chars = 0 + return chunk + + +def iter_json_array_text_chunks( + items: Iterable[Any], + *, + sort_keys: bool = False, + default: Callable[[Any], Any] | None = None, + batch_records: int = SHARE_JSON_BATCH_RECORDS, + chunk_chars: int = SHARE_JSON_CHUNK_BYTES, +) -> Iterator[str]: + """The items' encodings joined by ``,`` -- no brackets -- in bounded chunks. + + Concatenating every yielded chunk gives exactly + ``json.dumps(list(items), separators=(",", ":"), sort_keys=sort_keys, + default=default)[1:-1]``. No chunk ever splits an item, so a consumer + may frame chunk boundaries however it likes (a spool file, a pipe, a + digest). Each ``json.dumps`` call covers at most ``batch_records`` items. + """ + if batch_records <= 0: + raise ValueError("batch_records must be positive") + encode = _batch_encoder(sort_keys=sort_keys, default=default) + coalescer = _ChunkCoalescer(chunk_chars) + batch: list[Any] = [] + first_batch = True + for item in items: + batch.append(item) + if len(batch) < batch_records: + continue + text = encode(batch) + batch = [] + if not first_batch: + text = "," + text + first_batch = False + chunk = coalescer.push(text) + if chunk is not None: + yield chunk + if batch: + text = encode(batch) + if not first_batch: + text = "," + text + chunk = coalescer.push(text) + if chunk is not None: + yield chunk + chunk = coalescer.flush() + if chunk is not None: + yield chunk + + +def iter_json_array_byte_chunks( + items: Iterable[Any], + *, + sort_keys: bool = False, + default: Callable[[Any], Any] | None = None, + batch_records: int = SHARE_JSON_BATCH_RECORDS, + chunk_chars: int = SHARE_JSON_CHUNK_BYTES, +) -> Iterator[bytes]: + """:func:`iter_json_array_text_chunks`, UTF-8 encoded chunk by chunk.""" + for chunk in iter_json_array_text_chunks( + items, + sort_keys=sort_keys, + default=default, + batch_records=batch_records, + chunk_chars=chunk_chars, + ): + yield chunk.encode("utf-8") + + +def iter_json_object_text_chunks( + fields: Mapping[str, Any], + *, + array_keys: Collection[str], + sort_keys: bool = False, + default: Callable[[Any], Any] | None = None, + batch_records: int = SHARE_JSON_BATCH_RECORDS, + chunk_chars: int = SHARE_JSON_CHUNK_BYTES, +) -> Iterator[str]: + """One JSON object in bounded chunks, streaming its large array members. + + Concatenating every yielded chunk gives exactly ``json.dumps(fields, + separators=(",", ":"), sort_keys=sort_keys, default=default)``. Members + named in ``array_keys`` whose values are lists or tuples are encoded + item-batch by item-batch through :func:`iter_json_array_text_chunks`; + every other member is one ``json.dumps`` call, which is bounded by that + member's own size (found-block facts, balances, policy). A mapping with a + non-string key is encoded whole so CPython's own key coercion rules keep + applying; no share payload has such keys. + """ + keys = list(fields) + if any(not isinstance(key, str) for key in keys): + yield json.dumps( + fields, + sort_keys=sort_keys, + separators=_COMPACT_SEPARATORS, + default=default, + ) + return + if sort_keys: + keys.sort() + coalescer = _ChunkCoalescer(chunk_chars) + chunk = coalescer.push("{") + if chunk is not None: + yield chunk + first = True + for key in keys: + value = fields[key] + prefix = ("" if first else ",") + json.dumps(key) + ":" + first = False + if key in array_keys and isinstance(value, (list, tuple)): + chunk = coalescer.push(prefix + "[") + if chunk is not None: + yield chunk + for piece in iter_json_array_text_chunks( + value, + sort_keys=sort_keys, + default=default, + batch_records=batch_records, + chunk_chars=chunk_chars, + ): + chunk = coalescer.push(piece) + if chunk is not None: + yield chunk + chunk = coalescer.push("]") + else: + chunk = coalescer.push( + prefix + + json.dumps( + value, + sort_keys=sort_keys, + separators=_COMPACT_SEPARATORS, + default=default, + ) + ) + if chunk is not None: + yield chunk + chunk = coalescer.push("}") + if chunk is not None: + yield chunk + chunk = coalescer.flush() + if chunk is not None: + yield chunk + + +def iter_canonical_share_item_chunks( + shares: Iterable[Any], + *, + batch_records: int = SHARE_JSON_BATCH_RECORDS, + chunk_chars: int = SHARE_JSON_CHUNK_BYTES, +) -> Iterator[bytes]: + """Canonical share items (sorted keys, ``default=str``) in bounded chunks. + + Joined, the chunks are byte-identical to the historical + ``b",".join(json.dumps(share, sort_keys=True, separators=(",", ":"), + default=str).encode() for share in shares)`` -- the payout window page + encoding, the digest input framed by ``[``/``]``, and the daemon's + ``canonical_items`` stream. + """ + return iter_json_array_byte_chunks( + shares, + sort_keys=True, + default=str, + batch_records=batch_records, + chunk_chars=chunk_chars, + ) + + +def canonical_share_items_bytes(shares: Iterable[Any]) -> bytes: + """The complete canonical items stream for one bounded batch of shares. + + For a window page (at most the batch size) this is one ``json.dumps`` + call; callers holding a whole window must stream the chunks instead. + """ + return b"".join(iter_canonical_share_item_chunks(shares)) + + +def canonical_share_array_sha256(shares: Iterable[Any]) -> str: + """``sha256(json.dumps(list(shares), sort_keys=True, ...))`` streamed. + + Prefers a ``canonical_json_sha256`` method when the sequence carries one + (the paged window view, the daemon mirror), so an existing digest is + reused rather than recomputed; otherwise the array framing and every + record batch feed one running SHA-256 without allocating a whole + serialized array. + """ + existing = getattr(shares, "canonical_json_sha256", None) + if callable(existing): + return str(existing()) + digest = hashlib.sha256() + digest.update(b"[") + for chunk in iter_canonical_share_item_chunks(shares): + digest.update(chunk) + digest.update(b"]") + return digest.hexdigest() + + +class ShareArrayJsonSequence(Sequence): + """A plain share list with the paged view's ``canonical_json_sha256``. + + The coordinator's digest hooks (``canonical_json_sha256`` in the job + bundle, payout-state and coordinator modules, and the test-patchable + overrides wired through them) already prefer a sequence's own digest + method over serializing the value whole. Wrapping a freshly converted + list in this view routes it through that same preference, so the one + O(window) ``json.dumps`` those hooks would otherwise make becomes the + streamed digest -- without changing the hooks or what they are called + with elsewhere. + """ + + __slots__ = ("_shares",) + + def __init__(self, shares: Sequence[Any]) -> None: + self._shares = shares + + def __len__(self) -> int: + return len(self._shares) + + def __iter__(self) -> Iterator[Any]: + return iter(self._shares) + + def __getitem__(self, index: int | slice) -> Any: + return self._shares[index] + + def canonical_json_sha256(self) -> str: + return canonical_share_array_sha256(self._shares) + + +def share_array_json_view(shares: Sequence[Any]) -> Any: + """``shares`` itself when it can digest itself, else a streaming view.""" + if callable(getattr(shares, "canonical_json_sha256", None)): + return shares + return ShareArrayJsonSequence(shares) + + +def release_share_list_incrementally( + values: Any, + *, + batch_records: int = RELEASE_BATCH_RECORDS, +) -> None: + """Empty a finished per-share list in bounded slices. + + Dropping a list of a few hundred thousand records or dicts frees every + element in one refcount cascade, which CPython runs as a single C call + under the GIL -- measured at ~90-215 ms for 210k-400k parsed shares in + the #236 probe, on the same order as the JSON calls this module bounds. + Deleting from the tail in slices lets the interpreter switch threads + between batches. Only a list can be emptied in place; anything else is + left to ordinary garbage collection. + """ + if not isinstance(values, list): + return + batch_records = max(1, int(batch_records)) + while values: + del values[-batch_records:] + + +__all__ = [ + "SHARE_JSON_BATCH_RECORDS", + "SHARE_JSON_CHUNK_BYTES", + "ShareArrayJsonSequence", + "canonical_share_array_sha256", + "canonical_share_items_bytes", + "iter_canonical_share_item_chunks", + "iter_json_array_byte_chunks", + "iter_json_array_text_chunks", + "iter_json_object_text_chunks", + "release_share_list_incrementally", + "share_array_json_view", +] diff --git a/lab/prism/share_ledger.py b/lab/prism/share_ledger.py index 396dd4d03..648fc42d9 100644 --- a/lab/prism/share_ledger.py +++ b/lab/prism/share_ledger.py @@ -4,26 +4,125 @@ from __future__ import annotations import json +import codecs import copy import hashlib -import hmac +import logging import os import math import shlex import subprocess +import tempfile import time import traceback import uuid -from collections.abc import Sequence +from collections.abc import Iterable, Mapping, Sequence from contextlib import contextmanager from dataclasses import dataclass, field, replace from datetime import datetime, timedelta, timezone from decimal import Decimal from pathlib import Path from threading import BoundedSemaphore, Lock, Thread, local -from typing import Any, Callable, Iterator +from typing import Any, Callable, ClassVar, Iterator, Protocol, runtime_checkable + +from lab.prism.candidate_codec import ( + CANDIDATE_BODY_STORAGE_VERSION, + INDEX_SPAN_ROWS, + LEGACY_CANDIDATE_STORAGE_VERSION, + CandidateBodyIntegrityError, + CandidateCodecError, + PreparedCandidateIntent, + SpoolCandidateBody, + SpoolFieldIndex, + prepare_candidate_intent, + prepared_intent_from_spool, + replay_header_from_fields, +) +from lab.prism.candidate_spool_view import is_spool_view, materialize_spool_views +from lab.prism.candidate_store import ( + BODY_READ_MAX_CHUNKS, + DEFAULT_SPOOL_RESERVATION_BYTES, + HEADER_PAGE_MAX_BYTES, + HEADER_PAGE_MAX_ROWS, + JANITOR_CHUNKS_PER_STEP, + LEGACY_HELPER_MEMORY_BYTES, + LEGACY_HELPER_TIMEOUT_SECONDS, + STALE_STAGING_SECONDS, + CandidateBodyHydrator, + CandidateBodyRef, + CandidateBodyUnavailable, + CandidateHeaderPage, + CandidateStorageError, + IncompatibleCandidateSchema, + LegacyCandidateHelper, + LegacyTransport, + SpoolAdmission, + body_chunk_sql, + body_index_rows_sql, + body_manifest_sql, + body_page_sql, + body_pages_sql, + body_spans_sql, + candidate_schema_refusal, + header_page_sql, + parse_body_page, + parse_manifest_row, + new_body_id, + reap_retired_chunks_sql, + retire_orphan_bodies_sql, + retire_staging_body_sql, + schema_capability_sql, + seal_body_sql, + stage_body_sql, +) +from lab.prism.audit_artifacts import ( + AuditArtifactConfig, + AuditArtifactStore, + CanonicalAuditBundleCorrupt, + canonical_audit_bundle_bytes, +) +from lab.prism.audit_bundle_view import ( + CanonicalAuditBundleView, + MappedSequence, + RawJsonDocument, + RawJsonRecord, + StreamedJsonString, + iter_json_chunks, + materialize_json, + streamed_sha256_json_hex, +) +from lab.prism.share_json_stream import ( + canonical_share_items_bytes, + iter_json_object_text_chunks, +) +from lab.prism.writer_lease_timing import ( # noqa: F401 - compatibility re-export + DEFAULT_WRITER_LEASE_ADOPTION_SILENCE_SECONDS, + WRITER_LEASE_GUARD_STATEMENT_TIMEOUT_SECONDS, + WRITER_LEASE_VERIFICATION_MAX_STATEMENTS, +) + + +LOGGER = logging.getLogger(__name__) -from lab.prism.prism_tools import prism_tool_command + +def _text_stream_contains(chunks: Iterable[str], marker: str) -> bool: + """``marker in "".join(chunks)`` without joining the chunks.""" + keep = max(0, len(marker) - 1) + tail = "" + for chunk in chunks: + window = tail + chunk + if marker in window: + return True + tail = window[-keep:] if keep else "" + return False + + +def _default_bundle_canonicalizer() -> Callable[[dict[str, Any]], bytes]: + # Resolved at call time: bundle_compiler pulls in coordinator_config, + # whose CTV imports reach back into this module during initialization. + from lab.prism.bundle_compiler import canonical_bundle_bytes + + return canonical_bundle_bytes AUDIT_BODY_REF_SCHEMA = "qbit.prism.audit-body-ref.v1" AUDIT_BUNDLE_V2_SCHEMA = "qbit.prism.audit-bundle.v2" @@ -33,11 +132,67 @@ DEFAULT_CTV_BROADCAST_ATTEMPT_DETAIL_LIMIT = 20 DEFAULT_CTV_BROADCAST_RETRY_BACKOFF_SECONDS = 300 DEFAULT_INCREMENTAL_SHARE_WINDOW_PAGE_SIZE = 512 +# Startup writer-lease acquisition runs each statement under this per-attempt +# lock/statement deadline and gives up after this many timed-out attempts (see +# _run_lease_acquisition_json). 5 attempts x 5s per attempt (~25s) is sized to +# outlast a typical orphan reap rather than to guarantee one: the 15s +# idle_in_transaction_session_timeout below runs on the blocking backend's own +# clock, started when its transaction went idle and not when the successor +# began retrying, so the two are not directly comparable. In the common case +# the reap lands inside the budget and a later attempt acquires the lease with +# no operator action; a budget that runs out reaches the fatal error, which is +# still the point — a bounded visible failure instead of an unbounded wait. +DEFAULT_LEASE_ACQUIRE_ATTEMPTS = 5 +DEFAULT_LEASE_ACQUIRE_LOCK_TIMEOUT_SECONDS = 5.0 +# Session guards every coordinator Postgres session carries (see +# PostgresSessionGuards). idle_in_transaction_session_timeout makes the server +# abort a transaction whose client vanished between statements — the failure +# that otherwise leaves the qbit_ledger_writer_lease row lock held until TCP +# keepalive teardown (hours at OS defaults). The server-side tcp_keepalives_* +# GUCs are the backstop for backends idle *outside* a transaction, where the +# idle-in-transaction timer does not apply: on a TCP connection whose platform +# implements them, the defaults bound the server's teardown of a socket toward +# a vanished client at 30 + 3x10 = 60 seconds. Unix-socket connections ignore +# them, leaving the idle-in-transaction timeout as the guard there. +DEFAULT_POSTGRES_IDLE_IN_TRANSACTION_TIMEOUT_SECONDS = 15.0 +# Records decoded per batch by the row-result read the payout-window snapshot +# and delta run through (issue #236). Both production backends turn one +# SELECT into one JSON value per row and convert those rows this many at a +# time, rechecking the caller's deadline and stamping liveness between +# batches, so no single decode call grows with the window. The figure bounds +# the size of each C-level decode call and the gap between deadline checks, +# not the result: the complete snapshot is still returned as one list, and +# the driver still buffers the raw rows. +PAYOUT_WINDOW_ROW_BATCH_SIZE = 512 +# read_replica_status() runs on the public read service's background probe +# thread on a 5s cadence by default; a probe that outlives its own interval +# tells the freshness gate nothing it does not already know from the previous +# answer's age, so bound it well inside one interval. +DEFAULT_READ_REPLICA_PROBE_TIMEOUT_SECONDS = 3.0 +DEFAULT_POSTGRES_TCP_KEEPALIVES_COUNT = 3 +DEFAULT_POSTGRES_TCP_KEEPALIVES_IDLE_SECONDS = 30 +DEFAULT_POSTGRES_TCP_KEEPALIVES_INTERVAL_SECONDS = 10 # Only the coordinator assigns this prefix, and PsqlShareLedger preserves it # only after acquiring the writer/epoch advisory guard. Other ledger users and # psql-only deployments retain ordinary TTL fencing and are never treated as # fast-adoptable merely because they share an identity with a replacement. WRITER_LEASE_HEARTBEAT_SESSION_PREFIX = "heartbeat-v1:" +# The third arm of _try_acquire_writer_lease's COALESCE. Both of the first two +# arms are empty at once in exactly one reachable case: the first-ever +# concurrent acquisition against an empty lease table, where the loser's +# ON CONFLICT DO UPDATE affects no rows and the holder SELECT still reads the +# statement snapshot it took before the winner committed. That is a retry +# signal local to this one statement -- a fresh statement snapshot sees the +# committed row -- so the statement names it instead of evaluating to SQL NULL +# and reaching the generic no-JSON parser error. +WRITER_LEASE_ACQUIRE_RETRY_KEY = "lease_snapshot_retry" +WRITER_LEASE_ACQUIRE_RETRY_SUBJECT = "qbit ledger writer lease" +# Attempts spent taking a fresh statement snapshot before the race is treated +# as something other than the transient it is. Two attempts suffice under +# READ COMMITTED (the second statement's snapshot postdates the winner's +# commit); the third covers a re-raced row and bounds a caller that pinned an +# older snapshot for the whole transaction, which no retry can advance. +WRITER_LEASE_ACQUIRE_RETRY_ATTEMPTS = 3 class WriterLeaseRenewalDeferred(RuntimeError): @@ -57,14 +212,11 @@ class WriterLeaseRenewalDeferred(RuntimeError): """ -# Statements verify_writer_lease_guard_session may lawfully run inside one -# guarded slot: the verification statement plus its single attribution -# recheck. Callers that budget the verification's execution wall-clock must -# cover this many server-side statement timeouts, or a lawful recheck under -# moderate database latency is killed by the caller's deadline instead of -# rescuing the coordinator. -WRITER_LEASE_VERIFICATION_MAX_STATEMENTS = 2 -DEFAULT_WRITER_LEASE_ADOPTION_SILENCE_SECONDS = 1.0 +# WRITER_LEASE_VERIFICATION_MAX_STATEMENTS and +# DEFAULT_WRITER_LEASE_ADOPTION_SILENCE_SECONDS now live in +# lab.prism.writer_lease_timing, next to the rest of the heartbeat timing +# policy they are terms of, and are re-exported above so every existing +# import site keeps working. VALID_CREDIT_POLICIES = frozenset({"stale-grace"}) @@ -72,6 +224,40 @@ class LedgerOperationTimeout(TimeoutError): """A caller-scoped PostgreSQL deadline expired before work completed.""" +class ReadOnlyLedgerError(RuntimeError): + """A read-only ledger was asked for the writer lock.""" + + +class _RefusingWriterGate: + """Stands in for the writer lock on a read-only ledger. + + Every fenced statement and every writer-lock read in PsqlShareLedger -- + appends, block landing, settlement, and the O(recipients) reads like + current_owed_balances() and audit_bundle() -- reaches the lock through + ``_operation_gate(self._lock, "writer lock")``. Substituting a gate that + refuses to be acquired makes "this process never fences" a property of the + object rather than of which methods its callers happen to use, so a future + public route that reached a fenced read would fail loudly here instead of + quietly serializing dashboard traffic against the block-landing path. + """ + + __slots__ = () + + _MESSAGE = "ledger is read-only: the writer lock is not available" + + def acquire(self, timeout: float = -1.0) -> bool: + raise ReadOnlyLedgerError(self._MESSAGE) + + def release(self) -> None: + raise ReadOnlyLedgerError(self._MESSAGE) + + def __enter__(self) -> None: + raise ReadOnlyLedgerError(self._MESSAGE) + + def __exit__(self, *exc_info: object) -> None: + raise ReadOnlyLedgerError(self._MESSAGE) + + def _is_postgres_deadline_error(error: BaseException | str) -> bool: """Recognize backend cancellations caused by an armed caller deadline.""" message = str(error).casefold() @@ -99,10 +285,6 @@ def _is_postgres_deadline_error(error: BaseException | str) -> bool: ) -class _AuditShareSegmentConflict(RuntimeError): - """A share sequence is bound to more than one audit payload.""" - - def validate_credit_policy(credit_policy: str | None) -> str | None: if credit_policy is None: return None @@ -205,19 +387,13 @@ def from_records( records: tuple[AcceptedShareRecord, ...], ) -> _IncrementalShareWindowPage: prism_json_records = tuple(record.to_prism_json() for record in records) + # One bounded json.dumps per page (the page is at most one batch), + # byte-identical to encoding each record alone and joining with ",". return cls( records=records, total_difficulty=sum(int(record.share_difficulty) for record in records), prism_json_records=prism_json_records, - canonical_json_items=b",".join( - json.dumps( - record, - sort_keys=True, - separators=(",", ":"), - default=str, - ).encode() - for record in prism_json_records - ), + canonical_json_items=canonical_share_items_bytes(prism_json_records), ) @@ -360,6 +536,11 @@ def from_full_snapshot( def records(self) -> tuple[AcceptedShareRecord, ...]: return tuple(record for page in self.pages for record in page.records) + @property + def record_count(self) -> int: + """Retained record total; shared surface with DaemonShareWindowMirror.""" + return sum(len(page.records) for page in self.pages) + def json_records(self) -> IncrementalShareJsonSequence: return IncrementalShareJsonSequence( pages=self.pages, @@ -487,6 +668,429 @@ def advance( ) +class DaemonWindowMirrorDivergence(RuntimeError): + """The coordinator's byte mirror disagrees with what the daemon reported. + + Every mirror construction re-hashes the canonical items it holds against + the digest the daemon reported AND counts the records those bytes + actually contain against the count it reported, so a divergence between + the two implementations -- or a bug in the byte surgery itself -- becomes + a detected full-rescan instead of a silently wrong payout artifact. + Both checks are eager: nothing downstream may hold a mirror whose bytes + and metadata have not already been reconciled. + """ + + +# Bytes of one daemon canonical-items stream handed to the incremental UTF-8 +# decoder per call while the stream is walked. Read at call time so a test +# can shrink it and drive record and multi-byte boundaries across chunks. +CANONICAL_ITEMS_DECODE_CHUNK_BYTES = 64 * 1024 + + +def _walk_canonical_items( + fragment: bytes, + *, + chunk_bytes: int | None = None, + share_keys: bool = True, +) -> Iterator[dict[str, object]]: + """Yield each record of one canonical items span; return the trailing flag. + + The span is decoded in bounded chunks through an incremental UTF-8 + decoder and parsed one record at a time with ``raw_decode``, which + reports where each record's JSON ends without re-encoding anything, so + no single C call covers more than one chunk or one record whatever the + window size (#236): the count comes from the bytes themselves rather + than from a number the daemon declared, and the interpreter can switch + threads between records. A span that is not a run of complete records + separated by ``,`` -- a truncated record, a doubled or leading + separator, junk between records, invalid UTF-8 -- raises + DaemonWindowMirrorDivergence where it is reached, never a parse the + caller may retry. + + The generator's return value (``StopIteration.value``) is the trailing + flag, which distinguishes ``a,b`` from ``a,b,``: a dropped prefix + legitimately ends on a separator when records remain behind it, and a + complete stream never does. An empty span yields nothing and returns + False. + + ``share_keys`` makes every yielded record reuse one string object per + distinct key, the way a single whole-array ``json.loads`` shares keys + through its scanner memo; ``raw_decode`` clears that memo per call, and + without the hook a 400k-record parse would carry 13 private key strings + per record (about 60% more resident memory, and a proportionally longer + release). A walk that only counts records passes False and skips the + rebuild. + """ + total = len(fragment) + if not total: + return False + if chunk_bytes is None: + chunk_bytes = CANONICAL_ITEMS_DECODE_CHUNK_BYTES + chunk_bytes = max(1, int(chunk_bytes)) + view = memoryview(fragment) + utf8 = codecs.getincrementaldecoder("utf-8")() + if share_keys: + shared_keys: dict[str, str] = {} + + def rekey(record: dict[str, object]) -> dict[str, object]: + return { + shared_keys.setdefault(key, key): value + for key, value in record.items() + } + + decoder = json.JSONDecoder(object_hook=rekey) + else: + decoder = json.JSONDecoder() + offset = 0 + text = "" + position = 0 + + def refill(at_least: int = 0) -> bool: + """Decode the next chunk onto the unconsumed text; False once spent.""" + nonlocal offset, text, position + if offset >= total: + return False + end = min(offset + max(chunk_bytes, at_least), total) + try: + decoded = utf8.decode(view[offset:end], end >= total) + except UnicodeDecodeError as exc: + raise DaemonWindowMirrorDivergence( + "daemon window mirror items are not valid UTF-8" + ) from exc + offset = end + text = text[position:] + decoded + position = 0 + return True + + after_separator = False + while True: + while position >= len(text): + if not refill(): + # Spent where a record was required: only reachable right + # after a separator, so the span ended on one. (A non-empty + # span always decodes to at least one character, so the + # start never lands here.) + return after_separator + if text[position] != "{": + raise DaemonWindowMirrorDivergence( + "daemon window mirror items hold " + f"{text[position]!r} where a record was expected" + ) + while True: + try: + record, end = decoder.raw_decode(text, position) + except ValueError as exc: + # The record may merely be cut by the chunk boundary: pull + # more bytes -- at least as many as are already pending, so + # a genuinely malformed record is found in logarithmically + # many retries rather than one per chunk -- and retry from + # the same position. + if refill(len(text) - position): + continue + raise DaemonWindowMirrorDivergence( + f"daemon window mirror items hold a malformed record: {exc}" + ) from exc + break + after_separator = False + yield record + position = end + while position >= len(text): + if not refill(): + return False + if text[position] != ",": + raise DaemonWindowMirrorDivergence( + "daemon window mirror items hold " + f"{text[position]!r} where a record separator was expected" + ) + position += 1 + after_separator = True + + +def _canonical_items_layout(fragment: bytes) -> tuple[int, bool]: + """Record count and trailing-separator flag for one canonical items span. + + Walks the span record by record (see :func:`_walk_canonical_items`), + counting what the bytes actually hold and discarding each parsed record. + """ + walker = _walk_canonical_items(fragment, share_keys=False) + count = 0 + while True: + try: + next(walker) + except StopIteration as stop: + return count, bool(stop.value) + count += 1 + + +def _canonical_items_record_count(fragment: bytes) -> int: + """Record count of one complete canonical items stream.""" + count, trailing = _canonical_items_layout(fragment) + if trailing: + raise DaemonWindowMirrorDivergence( + "daemon window mirror items end on a record separator" + ) + return count + + +class DaemonShareJsonSequence(Sequence): + """Lazy, byte-backed twin of :class:`IncrementalShareJsonSequence`. + + When the daemon owns the payout-window fold, the coordinator holds only + the canonical items stream (every record's canonical JSON encoding joined + with ``,``) rather than materialized dicts. The routine build path needs + only the length, the digest, and this object's identity; the rare + consumers that genuinely need dicts (the found-block audit build, the + durable candidate intent, the one-shot builder fallback) force one + ``json.loads`` here, paying the parse exactly where the bytes are used. + Iteration order and the canonical digest are byte-identical to the paged + sequence by construction: both stream the same fragments in the same + order, and the digest framing is invariant to page layout. + + The count check below is a floor, not the contract: a sequence handed + out by :class:`DaemonShareWindowMirror` had its count reconciled with + its bytes at construction, so no consumer of a mirror can be the first + to learn of a divergence. + """ + + __slots__ = ("canonical_items", "record_count", "_parse_lock", "_parsed") + + def __init__(self, canonical_items: bytes, record_count: int) -> None: + self.canonical_items = bytes(canonical_items) + self.record_count = int(record_count) + self._parse_lock = Lock() + self._parsed: tuple[dict[str, object], ...] | None = None + + def __len__(self) -> int: + return self.record_count + + def _records(self) -> tuple[dict[str, object], ...]: + with self._parse_lock: + if self._parsed is None: + # Record at a time through the same strict walker that + # reconciled the mirror's count at construction: one + # ``raw_decode`` per record and one bounded UTF-8 decode per + # chunk, never a whole-window ``json.loads`` (#236). The + # parsed tuple is published only after every record parsed + # and the count matched, so a failure leaves nothing cached. + walker = _walk_canonical_items(self.canonical_items) + parsed: list[dict[str, object]] = [] + while True: + try: + parsed.append(next(walker)) + except StopIteration as stop: + trailing = bool(stop.value) + break + if trailing: + raise DaemonWindowMirrorDivergence( + "daemon window mirror items end on a record separator" + ) + if len(parsed) != self.record_count: + raise DaemonWindowMirrorDivergence( + "daemon window mirror parsed " + f"{len(parsed)} records where {self.record_count}" + " were declared" + ) + self._parsed = tuple(parsed) + return self._parsed + + def __iter__(self) -> Iterator[dict[str, object]]: + return iter(self._records()) + + def __getitem__( + self, + index: int | slice, + ) -> dict[str, object] | tuple[dict[str, object], ...]: + return self._records()[index] + + def canonical_json_sha256(self) -> str: + digest = hashlib.sha256() + digest.update(b"[") + digest.update(self.canonical_items) + digest.update(b"]") + return digest.hexdigest() + + +@dataclass(frozen=True) +class DaemonShareWindowMirror: + """Coordinator-held opaque mirror of one daemon-prepared payout window. + + Exposes the same identity surface as :class:`IncrementalShareWindow` + (anchor, weight, page size, ``record_count``) so the payout-state cache + policy code reads either interchangeably, while the window contents stay + pre-encoded bytes the daemon produced. Advancing applies the daemon's + reported byte surgery -- drop a prefix, append a suffix -- and every + construction verifies the resulting stream hashes to the daemon's digest + before anything downstream may consume it. + """ + + anchor_job_issued_at_ms: int + window_weight: int + page_size: int + record_count: int + canonical_items: bytes = field(repr=False) + share_snapshot_sha256: str + + @staticmethod + def _verified_items_digest(canonical_items: bytes, declared_digest: str) -> None: + digest = hashlib.sha256() + digest.update(b"[") + digest.update(canonical_items) + digest.update(b"]") + if digest.hexdigest() != declared_digest: + raise DaemonWindowMirrorDivergence( + "daemon window mirror bytes do not hash to the daemon's digest" + ) + + @staticmethod + def _verified_record_count(counted: int, declared_count: int) -> None: + """Pin the declared count to one counted from the mirror's own bytes. + + The digest above pins the bytes but says nothing about the count the + daemon reported alongside them, and the count is the one divergence + that would otherwise surface only when some consumer first parsed the + stream -- arbitrarily far from the materialization that produced it, + and after a partial request may already be on the daemon's stdin. + Counting eagerly here puts it beside the digest check, where every + construction pays it once and no consumer can be surprised. + """ + if counted != int(declared_count): + raise DaemonWindowMirrorDivergence( + f"daemon window mirror holds {counted} records where " + f"{int(declared_count)} were declared" + ) + + @classmethod + def from_full_items( + cls, + *, + anchor_job_issued_at_ms: int, + window_weight: int, + page_size: int, + record_count: int, + canonical_items: bytes, + share_snapshot_sha256: str, + ) -> DaemonShareWindowMirror: + cls._verified_items_digest(canonical_items, share_snapshot_sha256) + cls._verified_record_count( + _canonical_items_record_count(canonical_items), + record_count, + ) + return cls( + anchor_job_issued_at_ms=int(anchor_job_issued_at_ms), + window_weight=int(window_weight), + page_size=int(page_size), + record_count=int(record_count), + canonical_items=bytes(canonical_items), + share_snapshot_sha256=share_snapshot_sha256, + ) + + def _advanced_record_count( + self, + retained_drop_bytes: int, + appended_items: bytes, + ) -> int: + """Count an advance's records from the surgery, not the whole stream. + + Only the dropped prefix and the appended suffix are walked, so this + costs what the byte surgery itself costs rather than what a rescan of + the retained window would. The already-verified count of this mirror + supplies the retained middle, and the separator placement at both + seams is checked: a drop or an append landing inside a record would + otherwise let a miscount masquerade as an aligned edit. + """ + dropped, dropped_trailing = _canonical_items_layout( + self.canonical_items[:retained_drop_bytes] + ) + retained_items = self.canonical_items[retained_drop_bytes:] + if retained_items: + if dropped and not dropped_trailing: + raise DaemonWindowMirrorDivergence( + "daemon window advance dropped a partial record" + ) + elif dropped_trailing: + raise DaemonWindowMirrorDivergence( + "daemon window advance dropped every record but kept a" + " separator" + ) + retained = int(self.record_count) - dropped + if retained < 0: + raise DaemonWindowMirrorDivergence( + "daemon window advance dropped more records than the mirror" + " holds" + ) + if not appended_items: + return retained + if retained_items: + # The retained span already ends on a record, so the suffix must + # carry the separator that rejoins the two. + if appended_items[:1] != b",": + raise DaemonWindowMirrorDivergence( + "daemon window advance appended items without a record" + " separator" + ) + appended = _canonical_items_record_count(appended_items[1:]) + else: + appended = _canonical_items_record_count(appended_items) + if not appended: + # A suffix that carried only a separator would leave the stream + # ending on one, which no complete window ever does. + raise DaemonWindowMirrorDivergence( + "daemon window advance appended a separator with no record" + ) + return retained + appended + + def advanced( + self, + *, + anchor_job_issued_at_ms: int, + record_count: int, + retained_drop_bytes: int, + appended_items: bytes, + share_snapshot_sha256: str, + ) -> DaemonShareWindowMirror: + retained_drop_bytes = int(retained_drop_bytes) + if retained_drop_bytes < 0 or retained_drop_bytes > len(self.canonical_items): + raise DaemonWindowMirrorDivergence( + "daemon window advance dropped more bytes than the mirror holds" + ) + if retained_drop_bytes == 0 and not appended_items: + # Anchor-only advance: the stream is byte-identical, so a digest + # string comparison replaces the copy and the re-hash, and the + # count must be unchanged for the same reason. + canonical_items = self.canonical_items + if share_snapshot_sha256 != self.share_snapshot_sha256: + raise DaemonWindowMirrorDivergence( + "daemon window advance changed the digest without bytes" + ) + self._verified_record_count(self.record_count, record_count) + else: + canonical_items = ( + self.canonical_items[retained_drop_bytes:] + bytes(appended_items) + ) + self._verified_items_digest(canonical_items, share_snapshot_sha256) + self._verified_record_count( + self._advanced_record_count( + retained_drop_bytes, + bytes(appended_items), + ), + record_count, + ) + return DaemonShareWindowMirror( + anchor_job_issued_at_ms=int(anchor_job_issued_at_ms), + window_weight=self.window_weight, + page_size=self.page_size, + record_count=int(record_count), + canonical_items=canonical_items, + share_snapshot_sha256=share_snapshot_sha256, + ) + + def json_records(self) -> DaemonShareJsonSequence: + return DaemonShareJsonSequence( + canonical_items=self.canonical_items, + record_count=self.record_count, + ) + + @dataclass(frozen=True) class BlockCandidateIntentPersistResult: inserted: bool @@ -496,6 +1100,69 @@ def __bool__(self) -> bool: return self.inserted +def _block_candidate_cursor_parts(cursor: object) -> tuple[object, str]: + """Split one opaque pending-candidate cursor into its ordering parts. + + The cursor is whatever ``pending_block_candidate_rows`` handed back on a + prior row and travels through the caller (and, for the Postgres backend, + through JSON) untouched, so it is validated on the way back in rather + than trusted. The shape is deliberately the two ordering columns and + nothing else: ordering by the creation stamp alone cannot resume, because + equal stamps would either re-emit or skip their peers. + """ + if isinstance(cursor, str) or not isinstance(cursor, Sequence): + raise ValueError("pending block candidate cursor is not a two-element list") + parts = list(cursor) + if len(parts) != 2: + raise ValueError("pending block candidate cursor is not a two-element list") + created_at, block_hash = parts + if not isinstance(block_hash, str) or not block_hash: + raise ValueError("pending block candidate cursor has no block hash") + return created_at, block_hash + + +def _normalized_block_candidate_hash_set( + block_hashes: Sequence[str], +) -> tuple[str, ...]: + """Normalize a caller-supplied block-hash set for a batch outbox write. + + Both backends key the outbox on the lowercase hash, so the same + lowercasing the single-hash terminal updates apply per call is applied + once here to the whole set. Duplicates collapse and the result is sorted + so a given input set always produces one identical target set -- + including the generated SQL text, which a caller must be able to reason + about without knowing the caller's iteration order. + """ + return tuple(sorted({str(block_hash).lower() for block_hash in block_hashes})) + + +def _memory_block_candidate_row_key(row: dict[str, Any]) -> tuple[float, str]: + """Total-order key for one in-memory pending outbox row.""" + return (float(row.get("created_monotonic", 0.0)), str(row["block_hash"])) + + +def _memory_block_candidate_cursor_key(cursor: object) -> tuple[float, str]: + """Rebuild the in-memory ordering key from a returned cursor.""" + created_monotonic, block_hash = _block_candidate_cursor_parts(cursor) + if isinstance(created_monotonic, bool) or not isinstance( + created_monotonic, (int, float) + ): + raise ValueError("pending block candidate cursor has no creation stamp") + return (float(created_monotonic), block_hash) + + +class ShareReplayConflict(RuntimeError): + """A recovery row reused a share ID with a different durable payload.""" + + +@dataclass(frozen=True) +class ShareReplayResult: + """Typed result for one legacy recovery-journal append.""" + + disposition: str + record: AcceptedShareRecord + + class SingleWriterShareLedger: """Assigns canonical share_seq values and returns immutable snapshots. @@ -504,6 +1171,10 @@ class SingleWriterShareLedger: while moving storage to `qbit_share_ledger`. """ + # Bodies live in memory here, so a replay header row is hydrated at + # registration; the PostgreSQL ledger defers hydration to dequeue (#255). + candidate_hydration_deferred: ClassVar[bool] = False + def __init__( self, *, @@ -529,6 +1200,10 @@ def __init__( self._ctv_fanout_sets: dict[str, dict[str, Any]] = {} self._ctv_fanout_statuses: dict[str, dict[str, Any]] = {} self._ctv_fanout_attempts: dict[str, list[dict[str, Any]]] = {} + self._audit_publication_sequences: dict[str, int | None] = {} + self._next_audit_publication_sequence = 1 + self._inactive_audit_publications: set[str] = set() + self._memory_pool_blocks: dict[str, tuple[int, str, str]] = {} self._lock = Lock() def append(self, pending: PendingShare) -> AcceptedShareRecord: @@ -564,6 +1239,49 @@ def append(self, pending: PendingShare) -> AcceptedShareRecord: self._next_share_seq += 1 return record + def append_recovered_share(self, pending: PendingShare) -> ShareReplayResult: + """Append one recovery row with an explicit exact/conflict outcome.""" + if pending.share_difficulty <= 0: + raise ValueError("share_difficulty must be positive") + if pending.network_difficulty <= 0: + raise ValueError("network_difficulty must be positive") + credit_policy = validate_credit_policy(pending.credit_policy) + with self._lock: + existing = self._shares_by_id.get(pending.share_id) + if existing is not None: + if not self._pending_matches_record( + pending, + existing, + credit_policy=credit_policy, + ): + raise ShareReplayConflict( + f"recovered share payload conflicts with {pending.share_id}" + ) + return ShareReplayResult( + "exact_existing", + replace(existing, newly_inserted=False), + ) + record = AcceptedShareRecord( + share_seq=self._next_share_seq, + share_id=pending.share_id, + miner_id=pending.miner_id, + order_key=pending.order_key, + p2mr_program_hex=pending.p2mr_program_hex, + share_difficulty=pending.share_difficulty, + network_difficulty=pending.network_difficulty, + template_height=pending.template_height, + job_id=pending.job_id, + job_issued_at_ms=pending.job_issued_at_ms, + accepted_at_ms=pending.accepted_at_ms, + ntime=pending.ntime, + credit_policy=credit_policy, + ) + self._shares.append(record) + self._share_ids.add(pending.share_id) + self._shares_by_id[pending.share_id] = record + self._next_share_seq += 1 + return ShareReplayResult("inserted", replace(record)) + @staticmethod def _pending_matches_record( pending: PendingShare, @@ -708,6 +1426,8 @@ def persist_block_candidate_intent( self._block_candidate_outbox[block_hash] = { "block_hash": block_hash, "share_id": None, + # Stored as handed in: a PreparedCandidateIntent keeps its + # immutable share sequence, never a list copy (#255). "candidate": candidate, "candidate_sha256": candidate_sha256, "state": "pending", @@ -726,9 +1446,122 @@ def pending_block_candidates(self, *, limit: int = 32) -> list[dict[str, Any]]: for row in self.pending_block_candidate_rows(limit=limit) ] - def pending_block_candidate_rows(self, *, limit: int = 32) -> list[dict[str, Any]]: - """Return pending payloads together with their authoritative row keys.""" + def pending_block_candidate_headers( + self, + *, + limit: int = HEADER_PAGE_MAX_ROWS, + after_cursor: object | None = None, + max_bytes: int = HEADER_PAGE_MAX_BYTES, + ) -> CandidateHeaderPage: + """Metadata-first page over the in-memory outbox (#255). + + Same total order, cursor and configured row limit as + ``pending_block_candidate_rows`` (so the query shape and the + short-page exhaustion proof match what a caller configured), but + each row carries only the bounded typed replay header. There is no + transport here, so the byte cap that bounds the PostgreSQL page's + native decode does not apply: an in-memory page is row-bounded only + and never reports a byte truncation. + """ + limit = max(1, min(int(limit), HEADER_PAGE_MAX_ROWS)) + if after_cursor is None: + fetched = self.pending_block_candidate_rows(limit=limit) + else: + fetched = self.pending_block_candidate_rows( + limit=limit, + after_cursor=after_cursor, + ) + rows: list[dict[str, Any]] = [] + truncated_by_bytes = False + for row in fetched[:limit]: + candidate = row["candidate"] + header = replay_header_from_fields(candidate) + header_bytes = len(json.dumps(header, separators=(",", ":"), default=str)) + with self._lock: + stored = self._block_candidate_outbox.get(str(row["block_hash"])) + rows.append( + { + "block_hash": str(row["block_hash"]), + "storage_version": ( + CANDIDATE_BODY_STORAGE_VERSION + if isinstance(candidate, PreparedCandidateIntent) + else LEGACY_CANDIDATE_STORAGE_VERSION + ), + "candidate_sha256": ( + str(stored["candidate_sha256"]) if stored is not None else "" + ), + "header": header, + "header_bytes": header_bytes, + "body": None, + "pool_block_exists": bool(row["pool_block_exists"]), + "cursor": row["cursor"], + } + ) + return CandidateHeaderPage( + rows=tuple(rows), + next_cursor=rows[-1]["cursor"] if rows else after_cursor, + exhausted=len(fetched) < limit and not truncated_by_bytes, + fetched=len(fetched), + truncated_by_bytes=truncated_by_bytes, + ) + + def hydrate_block_candidate_intent( + self, + row: dict[str, Any], + *, + cancelled: Callable[[], bool] | None = None, + ) -> Any: + """Return the stored intent mapping for one still-pending header row.""" + block_hash = str(row["block_hash"]).lower() + with self._lock: + stored = self._block_candidate_outbox.get(block_hash) + if stored is None or stored["state"] != "pending" or stored["candidate"] is None: + raise CandidateBodyUnavailable( + f"block candidate {block_hash} is no longer pending" + ) + return stored["candidate"] + + def pending_block_candidate_rows( + self, + *, + limit: int = 32, + after_cursor: object | None = None, + ) -> list[dict[str, Any]]: + """Return pending payloads together with their authoritative row keys. + + Rows carry an opaque ``cursor`` the caller passes back verbatim as + ``after_cursor`` to resume strictly after that row. The order is the + total order ``(created, block_hash)``, so a page shorter than + ``limit`` proves no further pending row existed at query time and a + backlog of any size enumerates completely in bounded pages. The + block-hash tiebreak is what makes it a *total* order: creation + stamps collide (``time.monotonic`` here, one transaction's + ``clock_timestamp`` in Postgres), and a cursor on a colliding stamp + alone would either replay or skip its peers. + + Each row also carries ``pool_block_exists``: whether a durable + ``qbit_pool_blocks`` row exists for that hash, i.e. whether the + candidate ever reached ``persist_accepted_block``. It is answered + inside this page read -- one bounded existence probe per returned row + -- because the alternative is one round trip per row, and a page is + read precisely when the backlog is large. The fact is advisory by the + time the caller holds it; the terminal batch update re-checks it under + the writer fence. + """ + after = ( + None + if after_cursor is None + else _memory_block_candidate_cursor_key(after_cursor) + ) with self._lock: + ordered = sorted( + ( + (_memory_block_candidate_row_key(row), row) + for row in self._block_candidate_outbox.values() + if row["state"] == "pending" + ), + key=lambda entry: entry[0], + ) return [ { "block_hash": str(row["block_hash"]), @@ -737,9 +1570,13 @@ def pending_block_candidate_rows(self, *, limit: int = 32) -> list[dict[str, Any if isinstance(row["candidate"], dict) else row["candidate"] ), + "pool_block_exists": ( + str(row["block_hash"]) in self._memory_pool_blocks + ), + "cursor": list(key), } - for row in self._block_candidate_outbox.values() - if row["state"] == "pending" + for key, row in ordered + if after is None or key > after ][:limit] def block_candidate_pending_metrics(self) -> dict[str, int | float]: @@ -795,6 +1632,46 @@ def _finish_block_candidate(self, *, block_hash: str, state: str, error: str | N row["candidate"] = None return True + def mark_block_candidates_abandoned( + self, + *, + block_hashes: Sequence[str], + error: str, + ) -> tuple[str, ...]: + """Terminally abandon a caller-supplied page of pending rows at once. + + Mirrors ``mark_block_candidate_abandoned`` for a set: only rows whose + current state is exactly ``pending`` transition, and the return value + is the normalized hashes this call actually transitioned rather than a + count, so the caller can restrict any follow-up cleanup to the rows it + won. Already-terminal and missing hashes are neither returned nor + mutated, and an empty set is a no-op. + + A hash that owns a durable pool-block row is also left alone: that row + only exists for a candidate that was offered and landed, so it is not + a superseded sibling regardless of what the caller observed earlier. + + The whole page runs under one lock acquisition so the returned set is + the outcome of a single atomic decision, matching the Postgres backend + where the same page is one fenced statement. + """ + targets = _normalized_block_candidate_hash_set(block_hashes) + if not targets: + return () + abandoned: list[str] = [] + with self._lock: + for block_hash in targets: + row = self._block_candidate_outbox.get(block_hash) + if row is None or row["state"] != "pending": + continue + if block_hash in self._memory_pool_blocks: + continue + row["state"] = "abandoned" + row["last_error"] = error + row["candidate"] = None + abandoned.append(block_hash) + return tuple(abandoned) + def snapshot_at_job_issue( self, anchor_job_issued_at_ms: int, @@ -901,13 +1778,18 @@ def persist_ctv_fanout_manifest_set( self, *, block_hash: str, - manifest_set: dict[str, Any], + manifest_set: Mapping[str, Any], manifest_set_sha256: str, ) -> dict[str, int | str]: - payload = ctv_fanout_recovery_payload( - block_hash=block_hash, - manifest_set=manifest_set, - manifest_set_sha256=manifest_set_sha256, + # The in-memory ledger holds its state as plain objects by design; + # a streamed payload (lazy manifests from a bounded artifact view) + # is materialized here, in this test-and-parity backend only. + payload = materialize_json( + ctv_fanout_recovery_payload( + block_hash=block_hash, + manifest_set=manifest_set, + manifest_set_sha256=manifest_set_sha256, + ) ) with self._lock: existing = self._ctv_fanout_sets.get(block_hash) @@ -993,20 +1875,47 @@ def dashboard_pending_fanout_rows(self, *, page: int, limit: int) -> dict[str, o } def dashboard_public_artifact(self, *, sha256: str) -> dict[str, object] | None: + document = self.dashboard_public_artifact_document(sha256=sha256) + if document is None: + return None + return document.get("payload") + + def dashboard_public_artifact_document(self, *, sha256: str) -> dict[str, object] | None: + """Artifact payload plus, for manifest kinds, its canonical text. + + canonical_json is the exact serialized text the artifact's sha256 was + computed over, persisted at record time. Audit bundles are hashed + over Rust struct-order bytes that are not stored, so their + canonical_json is None and they keep the re-serialized response. + """ with self._lock: for payload in self._ctv_fanout_sets.values(): if payload.get("audit_bundle_sha256") == sha256: audit_bundle = payload.get("audit_bundle") - return copy.deepcopy(audit_bundle) if isinstance(audit_bundle, dict) else None + if not isinstance(audit_bundle, dict): + return None + return {"payload": copy.deepcopy(audit_bundle), "canonical_json": None} if payload.get("manifest_set_sha256") == sha256: manifest_set = payload.get("manifest_set") - return copy.deepcopy(manifest_set) if isinstance(manifest_set, dict) else None + if not isinstance(manifest_set, dict): + return None + manifest_set_json = payload.get("manifest_set_json") + return { + "payload": copy.deepcopy(manifest_set), + "canonical_json": manifest_set_json if isinstance(manifest_set_json, str) else None, + } for artifact in payload.get("artifacts", []): if not isinstance(artifact, dict): continue if artifact.get("manifest_sha256") == sha256: manifest = artifact.get("manifest") - return copy.deepcopy(manifest) if isinstance(manifest, dict) else None + if not isinstance(manifest, dict): + return None + manifest_json = artifact.get("manifest_json") + return { + "payload": copy.deepcopy(manifest), + "canonical_json": manifest_json if isinstance(manifest_json, str) else None, + } return None def update_ctv_fanout_status(self, *, fanout_txid: str, settlement_status: str) -> dict[str, int | str]: @@ -1144,6 +2053,8 @@ def dashboard_pool_snapshot( ), "blocks_found_total": 0, "prism_blocks_total": 0, + "blocks_reversed_total": 0, + "blocks_inactive_total": 0, "total_mined_bits": 0, "latest_block": None, "reward_window": { @@ -1184,9 +2095,11 @@ def dashboard_miner_reward_window( "share_percent": share_percent, } - def dashboard_blocks(self, *, page: int, limit: int) -> dict[str, object]: + def dashboard_blocks(self, *, page: int, limit: int, chain_state: str = "active") -> dict[str, object]: from lab.prism import public_api + if chain_state not in public_api.BLOCKS_CHAIN_STATE_FILTERS: + raise ValueError("chain_state must be one of active, all, reversed") return {"pagination": public_api.pagination(page, limit, 0), "rows": []} def dashboard_miner_lifetime_earnings_bits(self, *, recipient_id: str) -> int: @@ -1441,6 +2354,19 @@ def dashboard_hashrate_series( for bucket_epoch, entry in sorted(buckets.items()) ] + def dashboard_block_markers( + self, + *, + range_id: str, + bucket: str, + range_anchor_epoch: int | None = None, + ) -> dict[str, object]: + # The in-memory backend records no found_at or solver detail for its + # pool blocks -- dashboard_blocks serves an empty page for the same + # reason -- so the marker series degrades to empty the way the other + # in-memory dashboard read models do. + return {"total_blocks": 0, "points": []} + def persist_accepted_block( self, *, @@ -1451,47 +2377,240 @@ def persist_accepted_block( audit_report: dict[str, Any], canonical_bundle_path: Path | None = None, ) -> dict[str, int | str]: + block_hash = canonical_hex(block_hash, name="block_hash", expected_bytes=32) + with self._lock: + previous = self._memory_pool_blocks.get(block_hash) + if previous is not None and previous[0] != int(block_height): + raise RuntimeError("memory pool block height conflicts") + if previous is None: + self._memory_pool_blocks[block_hash] = ( + int(block_height), + "prepared", + str(parent_hash), + ) + self._audit_publication_sequences.setdefault(block_hash, None) + share_count = len(self._shares) return { "backend": "memory", - "share_count": len(self), + "share_count": share_count, "block_count": 0, "payout_entry_count": 0, "carry_forward_count": 0, } def reverse_immature_block(self, *, block_hash: str, active_tip_height: int) -> dict[str, int | str]: + block_hash = canonical_hex(block_hash, name="block_hash", expected_bytes=32) + with self._lock: + block = self._memory_pool_blocks.get(block_hash) + if ( + block is not None + and block[1] in {"confirmed", "inactive"} + and int(active_tip_height) >= block[0] + 1000 + ): + raise RuntimeError( + f"refusing to reverse mature pool block {block_hash}" + ) + if block is None or block[1] not in { + "prepared", + "confirmed", + "inactive", + }: + count = 0 + else: + self._memory_pool_blocks[block_hash] = ( + block[0], + "reversed", + block[2], + ) + self._inactive_audit_publications.discard(block_hash) + count = 1 return { "backend": "memory", - "reversed_count": 0, + "reversed_count": count, } def reject_prepared_block(self, *, block_hash: str, active_tip_height: int) -> dict[str, int | str]: - return { - "backend": "memory", - "rejected_count": 0, - } + block_hash = canonical_hex(block_hash, name="block_hash", expected_bytes=32) + with self._lock: + block = self._memory_pool_blocks.get(block_hash) + if block is None or block[1] != "prepared": + count = 0 + else: + self._memory_pool_blocks[block_hash] = ( + block[0], + "rejected", + block[2], + ) + count = 1 + return { + "backend": "memory", + "rejected_count": count, + } def confirm_accepted_block(self, *, block_hash: str, active_tip_height: int) -> dict[str, int | str]: + block_hash = canonical_hex(block_hash, name="block_hash", expected_bytes=32) + with self._lock: + block = self._memory_pool_blocks.get(block_hash) + if ( + block is None + or block[0] != int(active_tip_height) + or block[1] not in {"prepared", "confirmed"} + ): + # Mirror the Postgres disposition split: a row already + # terminally disposed (reorg quarantine, rejection, or + # reversal) reports superseded (-1); 0 keeps meaning no row + # or a live row this confirmation does not match. + if block is not None and block[1] in { + "inactive", + "rejected", + "reversed", + }: + return {"backend": "memory", "confirmed_count": -1} + return {"backend": "memory", "confirmed_count": 0} + # Mirror the Postgres disposition split again: only a flip out of + # 'prepared' is a fresh confirmation (1). A row already confirmed + # at this height is an idempotent replay (2) that keeps the + # ordinal its flip allocated and burns none. + already_confirmed = block[1] == "confirmed" + publication_sequence = self._audit_publication_sequences.get(block_hash) + if publication_sequence is None: + publication_sequence = self._next_audit_publication_sequence + self._next_audit_publication_sequence += 1 + self._audit_publication_sequences[block_hash] = publication_sequence + self._memory_pool_blocks[block_hash] = ( + block[0], + "confirmed", + block[2], + ) return { "backend": "memory", - "confirmed_count": 1, + "confirmed_count": 2 if already_confirmed else 1, + "audit_publication_sequence": publication_sequence, } def reorg_watch_blocks(self, *, active_tip_height: int) -> list[dict[str, object]]: return [] + def stranded_prepared_blocks( + self, + *, + active_tip_height: int, + min_depth: int, + limit: int = 64, + ) -> list[dict[str, object]]: + """Return deeply buried rows still parked in the prepared state. + + ``reorg_watch_blocks`` deliberately watches only confirmed/inactive + rows, and a prepared row is normally resolved by the live + submit/replay path that owns it. A row whose outbox entry is gone + (quarantined, or completed by a process that died before confirming) + therefore has nothing left to re-examine it, and stays prepared + forever. This read finds those rows; the caller decides, against the + active chain, which ones are provably orphaned. + """ + with self._lock: + rows = [ + { + "block_hash": block_hash, + "block_height": int(block[0]), + "parent_hash": str(block[2]), + } + for block_hash, block in self._memory_pool_blocks.items() + # The memory backend derives maturity_state from chain_state + # (see pool_block_state): 'prepared' is always 'immature'. + if block[1] == "prepared" + and int(block[0]) <= int(active_tip_height) - int(min_depth) + ] + rows.sort(key=lambda row: (int(row["block_height"]), str(row["block_hash"]))) + return rows[: max(0, int(limit))] + def mark_pool_block_inactive(self, *, block_hash: str, active_tip_height: int) -> dict[str, int | str]: + block_hash = canonical_hex(block_hash, name="block_hash", expected_bytes=32) + with self._lock: + block = self._memory_pool_blocks.get(block_hash) + if block is None or block[1] != "confirmed": + count = 0 + else: + self._inactive_audit_publications.add(block_hash) + self._memory_pool_blocks[block_hash] = ( + block[0], + "inactive", + block[2], + ) + count = 1 return { "backend": "memory", - "inactive_count": 0, + "inactive_count": count, } def reactivate_pool_block(self, *, block_hash: str, active_tip_height: int) -> dict[str, int | str]: + block_hash = canonical_hex(block_hash, name="block_hash", expected_bytes=32) + with self._lock: + block = self._memory_pool_blocks.get(block_hash) + if ( + block is None + or block[0] > int(active_tip_height) + or block[1] != "inactive" + or block_hash not in self._inactive_audit_publications + ): + count = 0 + sequence = None + else: + sequence = self._audit_publication_sequences.get(block_hash) + if sequence is None: + raise RuntimeError( + "inactive pool block has no audit publication sequence" + ) + self._inactive_audit_publications.remove(block_hash) + self._memory_pool_blocks[block_hash] = ( + block[0], + "confirmed", + block[2], + ) + count = 1 return { "backend": "memory", - "reactivated_count": 0, + "reactivated_count": count, + **( + {"audit_publication_sequence": int(sequence)} + if sequence is not None + else {} + ), } + def pool_block_state(self, *, block_hash: str) -> dict[str, object] | None: + block_hash = canonical_hex(block_hash, name="block_hash", expected_bytes=32) + with self._lock: + block = self._memory_pool_blocks.get(block_hash) + if block is None: + return None + publication_sequence = self._audit_publication_sequences.get(block_hash) + return { + "block_hash": block_hash, + "block_height": block[0], + "parent_hash": block[2], + "chain_state": block[1], + "maturity_state": ( + "reversed" + if block[1] in {"rejected", "reversed"} + else "immature" + ), + "audit_publication_sequence": publication_sequence, + } + + def audit_publication_sequence_floor(self) -> int: + """Return the newest ordinal attached to any durable pool-block row.""" + + with self._lock: + return max( + ( + sequence + for sequence in self._audit_publication_sequences.values() + if sequence is not None + ), + default=0, + ) + def mark_mature_pool_payouts(self, *, active_tip_height: int) -> dict[str, int | str]: return { "backend": "memory", @@ -1636,6 +2755,172 @@ def _writer_lease_advisory_lock_key(writer_id: str, writer_epoch: int) -> int: return int.from_bytes(digest[:8], "big", signed=True) +@runtime_checkable +class LedgerSqlPort(Protocol): + """The pooled statement-execution seam `PsqlShareLedger` writes through. + + `_NativePostgresClient` is the production implementation. Naming the + contract lets a test substitute a whole PostgreSQL model at construction + time (see `sql_backend_factory`) instead of reassigning bound methods on + a live ledger, which is what the lease and landing concurrency tests + need: statement timing, tuple-lock waits and transaction lifetime are + properties of this seam, not of any single method. + + Two statement shapes cross it. ``run_json`` runs a statement that + evaluates to one JSON value (every mutation, lease statement and + aggregate read). ``run_json_rows`` runs a read-only statement that + yields one JSON value per row and hands the rows back in bounded + batches (issue #236): the payout-window snapshot and delta, whose + single ``json_agg`` value used to be decoded in one GIL-held call the + size of the whole window. + """ + + def run_json( + self, + sql: str, + *, + retry_safe: bool = False, + timeout_seconds: float | None = None, + on_statement_start: Callable[[], None] | None = None, + ) -> Any: ... + + def run_json_rows( + self, + sql: str, + *, + retry_safe: bool = False, + timeout_seconds: float | None = None, + on_statement_start: Callable[[], None] | None = None, + row_converter: Callable[[Any], Any] | None = None, + batch_size: int = PAYOUT_WINDOW_ROW_BATCH_SIZE, + on_batch: Callable[[], None] | None = None, + ) -> list[Any]: ... + + def run_script(self, sql: str) -> None: ... + + def close(self) -> None: ... + + +@runtime_checkable +class LeaseGuardPort(Protocol): + """The dedicated writer-lease guard session seam. + + `_NativePostgresLeaseGuard` is the production implementation. Unlike + `LedgerSqlPort` this session is never transparently replaced: losing it + loses the advisory lock and must fence the owning coordinator, so the + heartbeat's liveness proof depends on the session's identity surviving. + A substitute must honour the same contract, including the serialized + query slot that `on_query_start` marks, the per-statement round trip + `on_statement_end` marks, the per-statement parsed result + `on_statement_result` hands back (issue #227: how a statement's own + server-side execution time reaches the caller), and the in-slot + `followup` statement the attribution recheck relies on. + """ + + def try_acquire(self) -> bool: ... + + @property + def held(self) -> bool: ... + + def run_json( + self, + sql: str, + *, + on_query_start: Callable[[], None] | None = None, + on_statement_end: Callable[[], None] | None = None, + on_statement_result: Callable[[Any], None] | None = None, + followup: Callable[[Any], str | None] | None = None, + ) -> Any: ... + + def close(self) -> None: ... + + +@dataclass(frozen=True) +class PostgresSessionGuards: + """Session-level GUCs this ledger sets on every connection it opens. + + Two guarantees ride the one libpq ``options`` carrier, because a setting + delivered at connect time covers every statement the session can run — + including the autocommit ones — rather than being re-asserted per + transaction by code that a future caller might bypass. + + The first is orphan reaping. A coordinator client that disappears without + an RST — network partition, SIGSTOP, VM pause — can leave its backend idle + in transaction, still holding every row lock the transaction took. The + writer-lease landing CTE row-locks the qbit_ledger_writer_lease singleton, + so one such orphan blocks a successor's startup lease upsert until the + kernel's TCP keepalive teardown (hours at OS defaults). These GUCs make the + *server* resolve that on its own: idle_in_transaction_session_timeout + aborts the orphaned transaction and releases its locks, and the + server-side tcp_keepalives_* settings (all PGC_USERSET, settable per + session) bound how long the server keeps a dead socket alive for backends + the idle-in-transaction timer cannot cover. + + The second is read-only enforcement. ``read_only`` adds + default_transaction_read_only=on, so PostgreSQL refuses the write itself + for a ledger built with ``read_only=True``. The refusing writer gate is + the in-process half of that promise and remains defense in depth, but it + only covers paths that take the gate; the GUC covers every statement the + session can issue, wherever the ledger points. Without it the guarantee + came from the deployment topology — a hot standby refusing writes — which + is absent whenever the read tier is aimed at a writable primary. + + Constructed once in PsqlShareLedger.__init__ and shared by all three + connection paths — the pooled native client, the dedicated lease-guard + session, and the psql subprocess backend. The orphan coverage is real but + bounded: the tcp_keepalives_* settings are silently ignored on + Unix-socket connections and on platforms that do not implement them, so + idle_in_transaction_session_timeout is the guard that always applies; + and these are session settings, so they reach only the sessions *this* + coordinator opens. A foreign session already holding the lease row — + another deployment's coordinator, an operator's psql — is bounded only + by the caller-side acquisition deadline in _run_lease_acquisition_json. + """ + + idle_in_transaction_timeout_seconds: float + tcp_keepalives_idle_seconds: int + tcp_keepalives_interval_seconds: int + tcp_keepalives_count: int + read_only: bool = False + + def options_fragment(self) -> str: + """Render the guards as libpq ``options`` / PGOPTIONS ``-c`` flags. + + The read-only setting goes last so that it wins over any earlier + duplicate, the same way the whole fragment is placed after an + operator's DSN-level options by _merged_session_options. + """ + idle_ms = max(1, int(self.idle_in_transaction_timeout_seconds * 1000)) + fragment = ( + f"-c idle_in_transaction_session_timeout={idle_ms}ms " + f"-c tcp_keepalives_idle={self.tcp_keepalives_idle_seconds} " + f"-c tcp_keepalives_interval={self.tcp_keepalives_interval_seconds} " + f"-c tcp_keepalives_count={self.tcp_keepalives_count}" + ) + if self.read_only: + fragment += " -c default_transaction_read_only=on" + return fragment + + +def _merged_session_options( + psycopg_module: Any, + conninfo: str, + *coordinator_fragments: str, +) -> str: + """Merge the coordinator's session options with the conninfo's own. + + Passing ``options`` as a connect kwarg replaces any ``options`` value the + operator embedded in the DSN outright, so that value must be read back + out of the conninfo and kept. The coordinator's fragments go last: libpq + applies ``-c`` settings left to right with the last duplicate winning, so + a DSN-level default can never silently disable the session guards. + """ + existing = psycopg_module.conninfo.conninfo_to_dict(conninfo).get("options") + fragments = [str(existing).strip()] if existing else [] + fragments.extend(coordinator_fragments) + return " ".join(fragment for fragment in fragments if fragment) + + class _NativePostgresClient: """Persistent pooled psycopg client for the share ledger. @@ -1656,18 +2941,26 @@ class _NativePostgresClient: (see ``verify_writer_lease_guard_session``). """ + # The clock ``run_json_rows`` budgets its local decoding against. A class + # attribute so a test built through ``__new__`` can drive the + # between-batch deadline with a virtual clock; ``run_json``'s + # single-value path is unchanged and keeps reading time.monotonic. + _monotonic: Callable[[], float] = staticmethod(time.monotonic) + def __init__( self, conninfo: str, *, pool_size: int, application_name: str | None = None, + session_guards: PostgresSessionGuards | None = None, ): import psycopg # deferred: the subprocess backend must work without it self._psycopg = psycopg self._conninfo = conninfo self._application_name = application_name + self._session_guards = session_guards self._pool_size = max(1, int(pool_size)) self._slots = BoundedSemaphore(self._pool_size) self._idle: list[Any] = [] @@ -1687,6 +2980,16 @@ def _connect(self, timeout_seconds: float | None = None) -> Any: kwargs["connect_timeout"] = max(1, math.ceil(timeout_seconds)) if self._application_name is not None: kwargs["application_name"] = self._application_name + session_guards = getattr(self, "_session_guards", None) + if session_guards is not None: + # Every pooled session carries the orphan-reaping guards, merged + # so an operator's DSN-level options value survives with the + # guard fragment last (last -c duplicate wins). + kwargs["options"] = _merged_session_options( + self._psycopg, + self._conninfo, + session_guards.options_fragment(), + ) return self._psycopg.connect(self._conninfo, **kwargs) @contextmanager @@ -1743,6 +3046,7 @@ def run_json( *, retry_safe: bool = False, timeout_seconds: float | None = None, + on_statement_start: Callable[[], None] | None = None, ) -> Any: """Run one JSON-returning statement. @@ -1773,6 +3077,8 @@ def run_json( ) with connection as conn: if deadline is None: + if on_statement_start is not None: + on_statement_start() row = conn.execute(sql).fetchone() else: remaining = deadline - time.monotonic() @@ -1780,6 +3086,8 @@ def run_json( raise LedgerOperationTimeout( "postgres statement deadline expired" ) + if on_statement_start is not None: + on_statement_start() timeout_ms = max(1, int(remaining * 1000)) # SET LOCAL confines both guards to this explicit # transaction, so pooled connections cannot leak a @@ -1802,6 +3110,156 @@ def run_json( raise RuntimeError(f"postgres query failed: {exc}") from exc raise AssertionError("unreachable") + def run_json_rows( + self, + sql: str, + *, + retry_safe: bool = False, + timeout_seconds: float | None = None, + on_statement_start: Callable[[], None] | None = None, + row_converter: Callable[[Any], Any] | None = None, + batch_size: int = PAYOUT_WINDOW_ROW_BATCH_SIZE, + on_batch: Callable[[], None] | None = None, + ) -> list[Any]: + """Run one read-only statement that yields one JSON value per row. + + The row-result counterpart of ``run_json`` (issue #236). ``run_json`` + returns a single JSON value, which for the payout-window snapshot + meant one ``json_agg`` over the whole window: the driver decoded + every record in one C call that never released the GIL, and the + lease monitor thread went unscheduled for the duration. Here the + statement projects one JSON object per row and the result is + consumed with ``fetchmany`` in ``batch_size`` slices, so the driver + decodes at most one slice per call and the interpreter can switch + threads between slices. + + Same connection borrowing, same deadline transaction shape (``SET + LOCAL statement_timeout`` / ``lock_timeout`` when a deadline is + armed), same ``OperationalError`` translation and the same + once-only retry for ``retry_safe`` statements as ``run_json``. Two + properties are specific to this path: + + * One SELECT, one snapshot. The complete result is received from + the server before the first row is decoded, and the deadline + transaction commits before local decoding starts, so a long + decode never holds a server transaction open against + ``idle_in_transaction_session_timeout``. A concurrent append + cannot appear in part of a snapshot, and a retry after a + connection loss discards every partially converted row and + re-executes the complete statement inside the original deadline. + * The deadline covers decoding. It is rechecked before every + batch, so a budget that expires while rows are still being + converted raises ``LedgerOperationTimeout`` and publishes + nothing, rather than returning late with a complete list. + + ``row_converter`` runs on each row's JSON value inside the batch + loop (the ledger passes its record constructor), so one batch's + intermediate dicts are released before the next is fetched. + ``on_batch`` fires after each converted batch; the ledger uses it to + recheck its own operation deadline and stamp liveness. The batch + size bounds each decode call and the interval between checks, not + client memory: psycopg's client-side cursor still buffers the raw + result, exactly as it did under ``run_json``. + """ + attempts = 2 if retry_safe else 1 + batch_size = max(1, int(batch_size)) + deadline = ( + None + if timeout_seconds is None + else self._monotonic() + max(0.0, timeout_seconds) + ) + for attempt in range(attempts): + try: + remaining = ( + None + if deadline is None + else max(0.0, deadline - self._monotonic()) + ) + if remaining is not None and remaining <= 0: + raise LedgerOperationTimeout("postgres statement deadline expired") + connection = ( + self.connection() + if remaining is None + else self.connection(timeout_seconds=remaining) + ) + with connection as conn: + if deadline is None: + if on_statement_start is not None: + on_statement_start() + cursor = conn.execute(sql) + else: + remaining = deadline - self._monotonic() + if remaining <= 0: + raise LedgerOperationTimeout( + "postgres statement deadline expired" + ) + if on_statement_start is not None: + on_statement_start() + timeout_ms = max(1, int(remaining * 1000)) + # Same SET LOCAL scoping as run_json. The block ends, + # and the server transaction commits, before any + # row is decoded: the client-side cursor already + # holds the complete result, and a long decode must + # not keep a server transaction open. + with conn.transaction(): + conn.execute( + f"SET LOCAL statement_timeout = '{timeout_ms}ms'" + ) + conn.execute( + f"SET LOCAL lock_timeout = '{timeout_ms}ms'" + ) + cursor = conn.execute(sql) + try: + return self._consume_json_rows( + cursor, + deadline=deadline, + row_converter=row_converter, + batch_size=batch_size, + on_batch=on_batch, + ) + finally: + cursor.close() + except self._psycopg.OperationalError as exc: + if timeout_seconds is not None and _is_postgres_deadline_error(exc): + raise LedgerOperationTimeout( + f"postgres operation exceeded {timeout_seconds:g}s" + ) from exc + if attempt + 1 >= attempts: + raise RuntimeError(f"postgres query failed: {exc}") from exc + raise AssertionError("unreachable") + + def _consume_json_rows( + self, + cursor: Any, + *, + deadline: float | None, + row_converter: Callable[[Any], Any] | None, + batch_size: int, + on_batch: Callable[[], None] | None, + ) -> list[Any]: + """Convert a buffered row result in bounded batches. + + A NULL row is rejected the way ``run_json`` rejects a NULL value; + a text row (a ``::text``-cast projection) is decoded here so both + column types produce the same Python objects. + """ + results: list[Any] = [] + while True: + if deadline is not None and self._monotonic() >= deadline: + raise LedgerOperationTimeout( + "postgres statement deadline expired while decoding rows" + ) + batch = cursor.fetchmany(batch_size) + if not batch: + return results + for row in batch: + value = parse_single_json_value(row[0] if row else None) + results.append( + value if row_converter is None else row_converter(value) + ) + if on_batch is not None: + on_batch() + def run_script(self, sql: str) -> None: """Run a multi-statement script (schema initialization).""" with self.connection() as conn: @@ -1826,14 +3284,40 @@ class _NativePostgresLeaseGuard: coordinator. The lease heartbeat runs on this same isolated session. """ - def __init__(self, conninfo: str, advisory_lock_key: int): + def __init__( + self, + conninfo: str, + advisory_lock_key: int, + *, + session_guards: PostgresSessionGuards | None = None, + ): import psycopg # deferred: psql-only users retain TTL fencing + # The short statement_timeout stays: the guard session must never + # queue behind a fenced write (see verify_writer_lease_guard_session). + # It is also the dominant term in the heartbeat's staleness envelope + # (WriterLeaseHeartbeatPolicy.max_healthy_server_gap_seconds), so the + # value is taken from the timing policy rather than written twice. + # The session guards ride the same options string so this dedicated + # connection carries the same orphan bounds as every pooled one. + # Merged with any operator DSN-level options, coordinator fragments + # last so they win. + statement_timeout_ms = int( + round(WRITER_LEASE_GUARD_STATEMENT_TIMEOUT_SECONDS * 1000) + ) + options = f"-c statement_timeout={statement_timeout_ms}" + if session_guards is not None: + options = _merged_session_options( + psycopg, + conninfo, + options, + session_guards.options_fragment(), + ) self._connection = psycopg.connect( conninfo, autocommit=True, connect_timeout=2, - options="-c statement_timeout=500", + options=options, ) self._advisory_lock_key = advisory_lock_key self._query_lock = Lock() @@ -1856,6 +3340,8 @@ def run_json( sql: str, *, on_query_start: Callable[[], None] | None = None, + on_statement_end: Callable[[], None] | None = None, + on_statement_result: Callable[[Any], None] | None = None, followup: Callable[[Any], str | None] | None = None, ) -> Any: with self._query_lock: @@ -1869,6 +3355,19 @@ def run_json( raise RuntimeError("postgres writer lease guard is not held") row = self._connection.execute(sql).fetchone() result = parse_single_json_value(row[0] if row else None) + # Every completed round trip is server-proven liveness, and the + # boundary the caller's phase attribution charges guard SQL + # against. It fires after the result is parsed so a malformed + # response is not counted as a healthy round trip. + # ``on_statement_result`` then hands the same round trip's parsed + # result to the caller, which is how a statement that reports + # its own server-side execution time (issue #227) gets that + # figure back per statement without a second round trip. The + # guard stays agnostic about the result's shape. + if on_statement_end is not None: + on_statement_end() + if on_statement_result is not None: + on_statement_result(result) # A followup runs inside this same serialized slot: no second # queue wait behind other guard callers can be charged to the # caller's execution budget. Each execute on this autocommit @@ -1881,6 +3380,10 @@ def run_json( break row = self._connection.execute(next_sql).fetchone() result = parse_single_json_value(row[0] if row else None) + if on_statement_end is not None: + on_statement_end() + if on_statement_result is not None: + on_statement_result(result) return result def close(self) -> None: @@ -1895,6 +3398,39 @@ def close(self) -> None: pass +# The JSON key both guard statements use to report their own server-side +# execution time (issue #227): ``clock_timestamp() - statement_timestamp()`` +# evaluated by the statement itself, so it costs no extra round trip and +# takes no lock. It is timing metadata only; nothing about liveness, +# identity, or the server-proven edge is read from it. +GUARD_SERVER_EXECUTION_SECONDS_KEY = "server_execution_seconds" +GUARD_SERVER_EXECUTION_SECONDS_SQL = ( + "extract(epoch FROM clock_timestamp() - statement_timestamp())" +) + + +def guard_server_execution_seconds(result: object) -> float | None: + """Server-reported execution seconds of one guard round trip, if any. + + ``None`` for a result that does not carry the key (an older statement + shape, a test fake) or carries something that is not a finite + non-negative number; callers then keep their conservative client-side + attribution rather than trusting a malformed figure. + """ + if not isinstance(result, dict): + return None + value = result.get(GUARD_SERVER_EXECUTION_SECONDS_KEY) + if value is None or isinstance(value, bool): + return None + try: + seconds = float(value) + except (TypeError, ValueError): + return None + if not math.isfinite(seconds) or seconds < 0.0: + return None + return seconds + + def parse_single_json_value(value: object) -> Any: """Normalize a one-row/one-column JSON query result. @@ -1909,6 +3445,28 @@ def parse_single_json_value(value: object) -> Any: return value +# The per-row JSON projection both payout-window reads return (issue #236): +# one object per accepted share, decoded row by row by either backend, rather +# than aggregated server-side with json_agg into a single value the client +# then had to decode in one call the size of the window. Same keys and value +# expressions the aggregate carried, so a record parses exactly as before. +_ACCEPTED_SHARE_JSON_ROW_SQL = """json_build_object( + 'share_seq', share_seq, + 'share_id', share_id, + 'miner_id', miner_id, + 'order_key', payout_order_key, + 'p2mr_program_hex', encode(p2mr_program, 'hex'), + 'share_difficulty', share_difficulty::text, + 'network_difficulty', network_difficulty::text, + 'template_height', template_height, + 'job_id', job_id, + 'job_issued_at_ms', round(extract(epoch FROM job_issued_at) * 1000)::bigint, + 'accepted_at_ms', round(extract(epoch FROM accepted_at) * 1000)::bigint, + 'ntime', ntime, + 'credit_policy', credit_policy +)""" + + class PsqlShareLedger: """Postgres-backed implementation of the coordinator share-ledger API. @@ -1919,6 +3477,22 @@ class PsqlShareLedger: durable_payout_state = True + # The lease lifecycle's clock, as a class attribute so an instance built + # without __init__ (several tests exercise one statement that way) still + # reads a working clock. __init__ overrides it per instance with whatever + # was injected. Declaring it here rather than resolving it through getattr + # at each call site means a rename fails loudly at the assignment instead + # of silently reverting every scenario to wall-clock time. + _monotonic: Callable[[], float] = staticmethod(time.monotonic) + + # Guards the one-time creation of the per-instance read-timing state for + # ledgers built through __new__ (several focused tests exercise a single + # statement that way). A class attribute, so the check-then-set inside + # _ensure_ledger_read_timings cannot let two threads each publish their + # own dict and lose one of them; __init__ builds the state directly and + # never reaches it. + _ledger_read_timings_bootstrap: ClassVar[Lock] = Lock() + @staticmethod def _resolve_lease_authority_margin_seconds( lease_ttl_seconds: float, @@ -1976,21 +3550,50 @@ def __init__( initialize_schema: bool = False, schema_path: Path | None = None, lease_retry_sleep: Callable[[float], None] | None = None, + monotonic: Callable[[], float] | None = None, + pool_application_name: str | None = None, + sql_backend_factory: Callable[..., LedgerSqlPort | None] | None = None, + lease_guard_factory: Callable[..., LeaseGuardPort | None] | None = None, lease_retry_max_sleep_seconds: float = 15.0, lease_ttl_seconds: float = 60.0, lease_authority_margin_seconds: float | None = None, lease_adoption_silence_seconds: float = DEFAULT_WRITER_LEASE_ADOPTION_SILENCE_SECONDS, + lease_acquire_lock_timeout_seconds: float = DEFAULT_LEASE_ACQUIRE_LOCK_TIMEOUT_SECONDS, + lease_acquire_attempts: int = DEFAULT_LEASE_ACQUIRE_ATTEMPTS, + postgres_idle_in_transaction_timeout_seconds: float = DEFAULT_POSTGRES_IDLE_IN_TRANSACTION_TIMEOUT_SECONDS, + postgres_tcp_keepalives_idle_seconds: int = DEFAULT_POSTGRES_TCP_KEEPALIVES_IDLE_SECONDS, + postgres_tcp_keepalives_interval_seconds: int = DEFAULT_POSTGRES_TCP_KEEPALIVES_INTERVAL_SECONDS, + postgres_tcp_keepalives_count: int = DEFAULT_POSTGRES_TCP_KEEPALIVES_COUNT, + read_only: bool = False, read_concurrency: int = 4, accepted_stats_cache_seconds: float = 60.0, reward_window_cache_seconds: float = 30.0, audit_body_dir: str | Path | None = None, audit_bundle_canonicalizer: Callable[[dict[str, Any]], bytes] | None = None, audit_share_segment_size: int = 0, + audit_artifact_store: AuditArtifactStore | None = None, ctv_broadcast_attempt_detail_limit: int = DEFAULT_CTV_BROADCAST_ATTEMPT_DETAIL_LIMIT, ctv_broadcast_retry_backoff_seconds: int = DEFAULT_CTV_BROADCAST_RETRY_BACKOFF_SECONDS, + candidate_storage_version: int = CANDIDATE_BODY_STORAGE_VERSION, + candidate_spool_dir: str | Path | None = None, + candidate_spool_reservation_bytes: int = DEFAULT_SPOOL_RESERVATION_BYTES, + candidate_schema_path: Path | None = None, + legacy_candidate_helper_timeout_seconds: float = LEGACY_HELPER_TIMEOUT_SECONDS, + legacy_candidate_helper_memory_bytes: int = LEGACY_HELPER_MEMORY_BYTES, ): if writer_epoch < 0: raise ValueError("writer_epoch must be >= 0") + candidate_storage_version = int(candidate_storage_version) + if candidate_storage_version not in ( + LEGACY_CANDIDATE_STORAGE_VERSION, + CANDIDATE_BODY_STORAGE_VERSION, + ): + raise ValueError( + f"unsupported candidate storage version {candidate_storage_version}" + ) + read_only = bool(read_only) + if read_only and initialize_schema: + raise ValueError("a read-only ledger cannot initialize the schema") accepted_stats_cache_seconds = float(accepted_stats_cache_seconds) if not math.isfinite(accepted_stats_cache_seconds) or accepted_stats_cache_seconds < 0: raise ValueError("accepted_stats_cache_seconds must be finite and non-negative") @@ -2015,6 +3618,40 @@ def __init__( or lease_adoption_silence_seconds <= 0 ): raise ValueError("lease_adoption_silence_seconds must be finite and positive") + # Validated here, not only in load_config: run_ctv_broadcaster_daemon + # and backfill_ctv_fanouts construct this class directly and must not + # be able to disarm the lease-acquisition bound or the session guards + # with a zero or negative value. + lease_acquire_lock_timeout_seconds = float(lease_acquire_lock_timeout_seconds) + if ( + not math.isfinite(lease_acquire_lock_timeout_seconds) + or lease_acquire_lock_timeout_seconds <= 0 + ): + raise ValueError("lease_acquire_lock_timeout_seconds must be finite and positive") + lease_acquire_attempts = int(lease_acquire_attempts) + if lease_acquire_attempts <= 0: + raise ValueError("lease_acquire_attempts must be positive") + postgres_idle_in_transaction_timeout_seconds = float( + postgres_idle_in_transaction_timeout_seconds + ) + if ( + not math.isfinite(postgres_idle_in_transaction_timeout_seconds) + or postgres_idle_in_transaction_timeout_seconds <= 0 + ): + raise ValueError( + "postgres_idle_in_transaction_timeout_seconds must be finite and positive" + ) + postgres_tcp_keepalives_idle_seconds = int(postgres_tcp_keepalives_idle_seconds) + if postgres_tcp_keepalives_idle_seconds <= 0: + raise ValueError("postgres_tcp_keepalives_idle_seconds must be positive") + postgres_tcp_keepalives_interval_seconds = int( + postgres_tcp_keepalives_interval_seconds + ) + if postgres_tcp_keepalives_interval_seconds <= 0: + raise ValueError("postgres_tcp_keepalives_interval_seconds must be positive") + postgres_tcp_keepalives_count = int(postgres_tcp_keepalives_count) + if postgres_tcp_keepalives_count <= 0: + raise ValueError("postgres_tcp_keepalives_count must be positive") read_concurrency = int(read_concurrency) if read_concurrency <= 0: raise ValueError("read_concurrency must be positive") @@ -2030,7 +3667,9 @@ def __init__( # than a competing expiry claim. Unique per ledger instance: a # predecessor or replacement process never shares it, so their # in-flight writes still fence this session out. - self._pool_application_name = f"qbit-prism-writer-{uuid.uuid4().hex}" + self._pool_application_name = ( + pool_application_name or f"qbit-prism-writer-{uuid.uuid4().hex}" + ) self._lease_ttl_seconds = lease_ttl_seconds # SQL fragment for the writer-lease expiry. The lease is refreshed on # every append (the dominant liveness signal during active mining), so a @@ -2047,19 +3686,96 @@ def __init__( f"make_interval(secs => {lease_authority_margin_seconds})" ) self._lease_retry_sleep = lease_retry_sleep or time.sleep + # The lease lifecycle's only clock. Every interval this process + # measures itself — adoption silence from guard acquisition, caller + # deadlines, lease refresh age — reads through here, so a test can + # supply a virtual clock and drive an interleaving by advancing it + # rather than by sleeping and hoping. Production passes None and gets + # time.monotonic. + # Resolved at construction, not at each call: an instance built the + # ordinary way binds whichever clock is in force now, while the class + # attribute above still covers instances built without __init__. + self._monotonic = monotonic or time.monotonic + self._sql_backend_factory = sql_backend_factory + self._lease_guard_factory = lease_guard_factory self._lease_retry_max_sleep_seconds = lease_retry_max_sleep_seconds self._lease_retry_min_sleep_seconds = min(0.25, self._lease_retry_max_sleep_seconds) self._lease_adoption_silence_seconds = lease_adoption_silence_seconds + self._lease_acquire_lock_timeout_seconds = lease_acquire_lock_timeout_seconds + self._lease_acquire_attempts = lease_acquire_attempts + # One immutable guard set shared by all three connection paths (the + # pooled native client, the dedicated lease-guard session, and the + # psql subprocess backend), so a session this coordinator opens is + # disowned by the server instead of holding the lease row on after + # its client vanishes. See PostgresSessionGuards for what the guards + # do and do not reach. + self._session_guards = PostgresSessionGuards( + idle_in_transaction_timeout_seconds=postgres_idle_in_transaction_timeout_seconds, + tcp_keepalives_idle_seconds=postgres_tcp_keepalives_idle_seconds, + tcp_keepalives_interval_seconds=postgres_tcp_keepalives_interval_seconds, + tcp_keepalives_count=postgres_tcp_keepalives_count, + # Fail closed at connect time rather than transaction by + # transaction: a read-only ledger tells the server it is read-only + # once, and every connection it can create -- pool slots, read + # slots, the psql subprocess, autocommit statements included -- + # inherits the refusal. + read_only=read_only, + ) self._operation_timeout_local = local() self._statement_timeout_local = local() - self._lock = Lock() + # Created here, not lazily in operation_progress: that scope's + # check-then-set is not atomic, so two block-work threads entering + # their first-ever scope on a fresh ledger can each build a local() + # and have one assignment win. The loser's hook would then live on an + # orphaned object and its admission wait would silently fall back to + # the heartbeat-silent path -- the exact failure the hook exists to + # remove. The scope keeps its lazy fallback for ledgers built through + # __new__ in focused tests; the production path must not race. + self._operation_progress_local = local() + self._read_only = read_only + # A read-only ledger never holds the writer lock, so it is not given + # one: see _RefusingWriterGate. Read-slot traffic is unaffected. + self._lock = _RefusingWriterGate() if read_only else Lock() self._read_semaphore = BoundedSemaphore(read_concurrency) - self._audit_body_dir = Path(audit_body_dir) if audit_body_dir else None - self._audit_bundle_canonicalizer = audit_bundle_canonicalizer + # Issue #255: chunked candidate bodies. Staging and hydration take + # this one-slot gate rather than the writer lock, because neither + # touches the lease row and both move body-sized data. + self._candidate_storage_version = candidate_storage_version + self._candidate_body_gate = BoundedSemaphore(1) + self._candidate_spool = SpoolAdmission( + candidate_spool_dir, + limit_bytes=int(candidate_spool_reservation_bytes), + ) + self._candidate_schema_path = candidate_schema_path + self._legacy_candidate_helper = LegacyCandidateHelper( + LegacyTransport( + psql_command=psql_command, + database_url=database_url or database_url_from_psql_command(self._command), + ), + memory_limit_bytes=int(legacy_candidate_helper_memory_bytes), + timeout_seconds=float(legacy_candidate_helper_timeout_seconds), + ) audit_share_segment_size = int(audit_share_segment_size) if audit_share_segment_size < 0: raise ValueError("audit_share_segment_size must be non-negative") - self._audit_share_segment_size = audit_share_segment_size + if audit_artifact_store is None and audit_body_dir is not None: + body_root = Path(audit_body_dir) + audit_artifact_store = AuditArtifactStore( + AuditArtifactConfig( + root=body_root, + evidence_path=body_root / "prism-live-stratum-evidence.json", + share_segment_size=audit_share_segment_size, + ), + # Legacy direct-ledger construction is an explicit adapter. + # Coordinator production wiring injects the shared A1 store. + canonicalizer=( + audit_bundle_canonicalizer or _default_bundle_canonicalizer() + ), + ) + self._audit_artifact_store = audit_artifact_store + self._audit_bundle_canonicalizer = ( + audit_bundle_canonicalizer or _default_bundle_canonicalizer() + ) ctv_broadcast_attempt_detail_limit = int(ctv_broadcast_attempt_detail_limit) if ctv_broadcast_attempt_detail_limit < 0: raise ValueError("ctv_broadcast_attempt_detail_limit must be non-negative") @@ -2084,18 +3800,39 @@ def __init__( self._prior_balances_reads_total = 0 self._prior_balances_read_last_seconds = 0.0 self._prior_balances_read_max_seconds = 0.0 + # Attribution for read-slot operations: local admission and server + # execution are recorded separately (see _run_attributed_read_json). + # Built here so the production path never reaches the __new__ + # bootstrap in _ensure_ledger_read_timings; the dict is published + # before the lock that guards it, so a reader that observes the lock + # observes the dict too. + self._ledger_read_timings: dict[str, dict[str, float | int]] = {} + self._ledger_read_timings_lock = Lock() self._native = self._make_native_client( native_client_mode, database_url, read_concurrency=read_concurrency, ) - self._writer_lease_guard: _NativePostgresLeaseGuard | None = None + self._writer_lease_guard: LeaseGuardPort | None = None try: - self._initialize_writer_lease_guard(database_url) - if initialize_schema: - path = schema_path or Path(__file__).resolve().parents[2] / "crates/qbit-prism/sql/001_share_ledger.sql" - self._run_script(path.read_text(encoding="utf-8")) - self._ensure_writer_lease() + if not read_only: + # Constructing an ordinary ledger claims the single-writer + # lease. A read-only ledger must not, or a second process + # opening one would contend with -- and could adopt -- the + # lease the coordinator lands blocks under. + self._initialize_writer_lease_guard(database_url) + if initialize_schema: + path = schema_path or Path(__file__).resolve().parents[2] / "crates/qbit-prism/sql/001_share_ledger.sql" + self._run_script(path.read_text(encoding="utf-8")) + # Additive #255 migration, applied after the base schema + # on every start; every statement in it is idempotent. + candidate_path = ( + candidate_schema_path + or Path(__file__).resolve().parents[2] + / "crates/qbit-prism/sql/002_candidate_bodies.sql" + ) + self._run_script(candidate_path.read_text(encoding="utf-8")) + self._ensure_writer_lease() except BaseException: self.close() raise @@ -2106,7 +3843,7 @@ def _make_native_client( database_url: str | None, *, read_concurrency: int, - ) -> _NativePostgresClient | None: + ) -> LedgerSqlPort | None: mode = (native_client_mode or "auto").strip().lower() if mode in {"0", "false", "no", "off", "psql"}: return None @@ -2121,35 +3858,65 @@ def _make_native_client( "postgres:// DSN inside PRISM_POSTGRES_PSQL_COMMAND" ) return None + # One pooled connection per concurrent reader plus one for the + # serialized write path (the coordinator's share writer thread). + pool_size = read_concurrency + 1 + if self._sql_backend_factory is not None: + # An injected backend is authoritative: it stands in for the + # whole server, so psycopg's availability is irrelevant and a + # None return means the same thing it does below (fall back to + # the psql subprocess path). + return self._sql_backend_factory( + conninfo, + pool_size=pool_size, + application_name=self._pool_application_name, + ) try: - # One pooled connection per concurrent reader plus one for the - # serialized write path (the coordinator's share writer thread). return _NativePostgresClient( conninfo, - pool_size=read_concurrency + 1, + pool_size=pool_size, application_name=self._pool_application_name, + session_guards=getattr(self, "_session_guards", None), ) except ImportError: if required: raise ValueError( "PRISM_POSTGRES_NATIVE_CLIENT=1 requires the psycopg package" ) from None - # Silent fallback: which execution backend is active is reported - # by the owning daemon's startup line via execution_backend. + # The daemon's startup line reports the active execution backend, + # but this silent capability loss is worth its own line too: the + # psql fallback applies the schema through a subprocess whose + # atomicity contract every operator relies on. + print( + "prism ledger native PostgreSQL client unavailable: " + "psycopg import failed; falling back to the psql subprocess " + "backend", + flush=True, + ) return None def _make_writer_lease_guard( self, database_url: str | None, - ) -> _NativePostgresLeaseGuard | None: + ) -> LeaseGuardPort | None: if self._native is None: return None conninfo = database_url or database_url_from_psql_command(self._command) if conninfo is None: return None + advisory_lock_key = _writer_lease_advisory_lock_key( + self._writer_id, + self._writer_epoch, + ) + if self._lease_guard_factory is not None: + return self._lease_guard_factory( + conninfo, + advisory_lock_key=advisory_lock_key, + ) return _NativePostgresLeaseGuard( conninfo, - _writer_lease_advisory_lock_key(self._writer_id, self._writer_epoch), + advisory_lock_key, + session_guards=getattr(self, "_session_guards", None), ) def _initialize_writer_lease_guard(self, database_url: str | None) -> None: @@ -2184,7 +3951,7 @@ def _initialize_writer_lease_guard(self, database_url: str | None) -> None: # failure budget; counting from acquisition guarantees it # that time even when its lease row is already stale because # a long fenced transaction withheld updated_at refreshes. - self._writer_lease_guard_acquired_monotonic = time.monotonic() + self._writer_lease_guard_acquired_monotonic = self._monotonic() return guard.close() if not warned: @@ -2241,6 +4008,62 @@ def writer_lease_last_refresh_monotonic(self) -> float | None: def backend_name(self) -> str: return "postgres-psql" + def read_replica_status( + self, + *, + timeout_seconds: float = DEFAULT_READ_REPLICA_PROBE_TIMEOUT_SECONDS, + ) -> dict[str, object]: + """Replication-state probe backing the public read staleness contract. + + Returns: + + - ``in_recovery`` -- true only on a hot standby; + - ``replay_lag_seconds`` -- wall clock minus the newest replayed + transaction's commit time (None before any replay). This is an + informational staleness *indicator*, not a freshness proof: on an + idle primary it grows with wall time even though the replica is + fully caught up; + - ``receiver_heartbeat_age_seconds`` -- wall clock minus the newest + message from the primary's WAL sender (None when the walreceiver + is not connected). Heartbeats flow every + ``wal_receiver_status_interval`` (default 10s) even when the + primary is idle, so this is the replica-side liveness proof the + public read service enforces; + - ``apply_backlog_bytes`` -- WAL bytes received but not yet replayed. + + The extracted public read service (issue #145) polls this to enforce + its bounded-staleness contract and to refuse serving from a writable + primary. Reads run through the ordinary read pool, so the probe never + touches the writer lease. + + Bounded by default: this runs on the service's background probe + thread, and an unbounded probe against a wedged standby would leave + the freshness gate holding its last answer indefinitely rather than + ageing out into a refusal. + """ + sql = """ +SELECT json_build_object( + 'in_recovery', pg_is_in_recovery(), + 'replay_lag_seconds', CASE + WHEN pg_last_xact_replay_timestamp() IS NULL THEN NULL + ELSE extract(epoch FROM (clock_timestamp() - pg_last_xact_replay_timestamp())) + END, + 'receiver_heartbeat_age_seconds', CASE + WHEN (SELECT last_msg_receipt_time FROM pg_stat_wal_receiver) IS NULL THEN NULL + ELSE extract(epoch FROM (clock_timestamp() - (SELECT last_msg_receipt_time FROM pg_stat_wal_receiver))) + END, + 'apply_backlog_bytes', CASE + WHEN pg_last_wal_receive_lsn() IS NULL OR pg_last_wal_replay_lsn() IS NULL THEN NULL + ELSE pg_wal_lsn_diff(pg_last_wal_receive_lsn(), pg_last_wal_replay_lsn()) + END +); +""" + with self.operation_timeout(timeout_seconds): + row = self._run_read_json(sql) + if not isinstance(row, dict): + raise RuntimeError("read replica status probe did not return an object") + return row + @contextmanager def operation_timeout(self, timeout_seconds: float) -> Iterator[None]: """Bound PostgreSQL and local admission for the current thread. @@ -2257,7 +4080,7 @@ def operation_timeout(self, timeout_seconds: float) -> Iterator[None]: timeout_local = local() self._operation_timeout_local = timeout_local previous = getattr(timeout_local, "deadline", None) - deadline = time.monotonic() + timeout_seconds + deadline = self._monotonic() + timeout_seconds timeout_local.deadline = ( deadline if previous is None else min(float(previous), deadline) ) @@ -2305,6 +4128,102 @@ def statement_timeout(self, timeout_seconds: float) -> Iterator[None]: else: timeout_local.timeout_seconds = previous + @contextmanager + def operation_progress( + self, + on_progress: Callable[[], None], + *, + slice_seconds: float, + ) -> Iterator[None]: + """Stamp caller liveness while a ledger admission wait is blocked. + + A landing-class caller runs its ledger step directly on a + watchdog-monitored block-work thread. Waiting for the writer lock or + the read semaphore is not database work: no statement has been sent, + so neither ``statement_timeout`` nor any server-side cancellation + bounds it, and the ledger has nothing to report until admission + succeeds. Without this hook the whole admission budget is + heartbeat-silent, and a coordinator that is merely queued behind + another writer is hard-exited by its own watchdog mid-landing -- + which loses the escalation state the retry depended on and restarts + the same doomed cycle. + + The hook fires between acquire slices, so it never runs while this + thread holds the gate, and it never replaces the caller's deadline: + ``_operation_gate`` still raises at the deadline + ``_remaining_operation_timeout`` reports. An exception from + ``on_progress`` propagates to the caller; a liveness stamp that + cannot be taken is a real failure, not something to swallow inside a + lock wait. + + Nesting merges the way ``operation_timeout``/``statement_timeout`` + merge: the effective slice is the minimum of the enclosing slices, so + an inner scope can only tighten the stamp cadence, never widen it. A + nested scope carrying a larger slice would otherwise lengthen the + heartbeat gaps its enclosing caller had already sized against its own + monitor. The callback itself does replace -- the innermost caller is + the one whose liveness is at stake -- and the outer pair is restored + on exit. + """ + slice_seconds = float(slice_seconds) + if not math.isfinite(slice_seconds) or slice_seconds <= 0: + raise ValueError("operation progress slice must be finite and positive") + progress_local = getattr(self, "_operation_progress_local", None) + if progress_local is None: + progress_local = local() + self._operation_progress_local = progress_local + previous = getattr(progress_local, "hook", None) + progress_local.hook = ( + (on_progress, slice_seconds) + if previous is None + else (on_progress, min(float(previous[1]), slice_seconds)) + ) + try: + yield + finally: + if previous is None: + try: + del progress_local.hook + except AttributeError: + pass + else: + progress_local.hook = previous + + def _operation_progress_hook(self) -> tuple[Callable[[], None], float] | None: + progress_local = getattr(self, "_operation_progress_local", None) + if progress_local is None: + return None + return getattr(progress_local, "hook", None) + + def _note_operation_progress(self) -> None: + """Report liveness between the statements of one gate-holding step. + + ``_acquire_operation_gate`` stamps only while a caller is *waiting* + for a gate, which is all a single-statement operation needs: once the + gate opens the caller is inside one server-side statement, and its + liveness monitor is sized for exactly that. An operation that issues + a second statement without releasing the gate breaks that sizing. + No admission slice runs between the two -- the gate is already held -- + so the monitor sees one unbroken silence of two statement budgets + where it was promised one. That is indistinguishable from a wedged + operation, and a watchdog with any tolerance below twice the budget + hard-exits a coordinator that is in fact making normal progress + (issue #125). Reporting here restores the contract: the first + statement's full round trip has completed, which is precisely the + evidence a liveness monitor watches for, while a genuinely stuck + statement still produces no report at all. + + With no hook installed this does nothing; an ordinary ledger caller + has no monitor to satisfy. An exception from the hook propagates, + matching ``_acquire_operation_gate``: a liveness stamp that cannot be + taken is a real failure, not something to swallow mid-operation. + """ + hook = self._operation_progress_hook() + if hook is None: + return + on_progress, _slice_seconds = hook + on_progress() + def _remaining_operation_timeout(self) -> float | None: timeout_local = getattr(self, "_operation_timeout_local", None) deadline = ( @@ -2328,24 +4247,56 @@ def _remaining_operation_timeout(self) -> float | None: if statement_timeout_seconds is None else float(statement_timeout_seconds) ) - remaining = float(deadline) - time.monotonic() + remaining = float(deadline) - self._monotonic() if remaining <= 0: raise LedgerOperationTimeout("postgres operation deadline expired") if statement_timeout_seconds is not None: remaining = min(remaining, float(statement_timeout_seconds)) return remaining - @contextmanager - def _operation_gate(self, gate: Any, name: str) -> Iterator[None]: - """Acquire a ledger lock/semaphore within the caller's deadline.""" + def _acquire_operation_gate(self, gate: Any, name: str) -> None: + """Wait for one ledger gate inside the caller's remaining deadline. + + With no progress hook installed this is a single blocking acquire, as + it has always been: an ordinary caller has no liveness monitor to + satisfy and gains nothing from waking up. With a hook installed the + same total wait is served in slices so the caller can stamp its + heartbeat between them (see ``operation_progress`` for why an + admission wait would otherwise be silent). Slicing never widens the + wait: the deadline derived from ``_remaining_operation_timeout`` stays + authoritative and still produces the same timeout error. + """ remaining = self._remaining_operation_timeout() - acquired = ( - gate.acquire() - if remaining is None - else gate.acquire(timeout=max(0.0, remaining)) - ) - if not acquired: - raise LedgerOperationTimeout(f"timed out waiting for postgres {name}") + hook = self._operation_progress_hook() + if hook is None: + acquired = ( + gate.acquire() + if remaining is None + else gate.acquire(timeout=max(0.0, remaining)) + ) + if not acquired: + raise LedgerOperationTimeout(f"timed out waiting for postgres {name}") + return + on_progress, slice_seconds = hook + deadline = ( + None if remaining is None else self._monotonic() + max(0.0, remaining) + ) + while True: + wait_seconds = slice_seconds + if deadline is not None: + wait_seconds = min(wait_seconds, deadline - self._monotonic()) + # An expired budget still gets one non-blocking attempt, so a + # zero or negative remaining fails exactly where it always did. + if gate.acquire(timeout=max(0.0, wait_seconds)): + return + if deadline is not None and self._monotonic() >= deadline: + raise LedgerOperationTimeout(f"timed out waiting for postgres {name}") + on_progress() + + @contextmanager + def _operation_gate(self, gate: Any, name: str) -> Iterator[None]: + """Acquire a ledger lock/semaphore within the caller's deadline.""" + self._acquire_operation_gate(gate, name) try: yield finally: @@ -2467,19 +4418,41 @@ def append(self, pending: PendingShare) -> AcceptedShareRecord: ) return record - def append_batch( + def _append_batch_with_replay_outcomes( self, - entries: list[tuple[PendingShare, dict[str, Any] | None]], - ) -> list[AcceptedShareRecord]: + entries: list[tuple[PendingShare, Any]], + ) -> list[ShareReplayResult]: """Commit accepted shares and optional block intents in one transaction. Replaying the exact same payload is idempotent. Reusing a share ID or - block hash with different content fails the whole batch. Postgres - assigns the share sequence and makes every row visible before this - method returns, which is the coordinator's Stratum ACK boundary. + block hash with different content fails the whole batch — a share-ID + payload mismatch raises the typed :class:`ShareReplayConflict`. + Postgres assigns the share sequence and makes every row visible before + this method returns, which is the coordinator's Stratum ACK boundary. + Each entry's outcome reports whether its row was ``inserted`` by this + statement or was an ``exact_existing`` durable duplicate. + + A batch with no candidate keeps the historical statement. A batch + carrying candidates takes the chunked-body route (#255) unless the + ledger was constructed in legacy storage mode, in which case the + candidate is materialized and written as the historical jsonb. """ if not entries: return [] + if any(candidate is not None for _, candidate in entries): + if self._candidate_storage_version_value() == CANDIDATE_BODY_STORAGE_VERSION: + return self._append_batch_with_replay_outcomes_v2(entries) + entries = [ + (pending, self._legacy_candidate_document(candidate)) + for pending, candidate in entries + ] + return self._append_batch_with_replay_outcomes_v1(entries) + + def _append_batch_with_replay_outcomes_v1( + self, + entries: list[tuple[PendingShare, dict[str, Any] | None]], + ) -> list[ShareReplayResult]: + """The historical whole-jsonb batch statement (storage version 1).""" payloads: list[dict[str, Any]] = [] share_ids: set[str] = set() block_hashes: set[str] = set() @@ -2669,6 +4642,7 @@ def append_batch( WHEN EXISTS (SELECT 1 FROM share_mismatch) THEN json_build_object( 'error', 'duplicate share_id payload mismatch', + 'error_kind', 'share_replay_conflict', 'share_ids', (SELECT json_agg(share_id ORDER BY share_id) FROM share_mismatch) ) WHEN EXISTS (SELECT 1 FROM candidate_mismatch) THEN @@ -2704,6 +4678,8 @@ def append_batch( with self._operation_gate(self._lock, "writer lock"): result = self._run_json(sql) if "error" in result: + if result.get("error_kind") == "share_replay_conflict": + raise ShareReplayConflict(str(result["error"])) raise RuntimeError(str(result["error"])) records = result.get("records") if not isinstance(records, list) or len(records) != len(entries): @@ -2719,221 +4695,1611 @@ def append_batch( record, new_miner=bool(payload.get("new_miner", False)), ) - return parsed + return [ + ShareReplayResult( + ( + "inserted" + if bool(payload.get("newly_inserted", True)) + else "exact_existing" + ), + record, + ) + for payload, record in zip(records, parsed, strict=True) + ] - def persist_block_candidate_intent( - self, - candidate: dict[str, Any], - ) -> BlockCandidateIntentPersistResult: - """Persist candidate work that is not yet eligible for share credit.""" - block_hash = str(candidate.get("block_hash_hex", "")).lower() - if not block_hash: - raise ValueError("block candidate is missing block_hash_hex") - candidate = {**candidate, "block_hash_hex": block_hash} - candidate_sha256 = block_candidate_identity_sha256(candidate) - sql = f""" -WITH durability AS ( - SELECT set_config('synchronous_commit', 'on', true) -), -lease AS ( - UPDATE qbit_ledger_writer_lease - SET lease_expires_at = clock_timestamp() + {self._lease_interval_sql}, - updated_at = clock_timestamp() - FROM durability - WHERE singleton - AND writer_id = {self._text_literal(self._writer_id)} - AND writer_epoch = {int(self._writer_epoch)} - AND writer_session_token = {self._text_literal(self._writer_session_token)} - RETURNING writer_id -), -existing AS ( - SELECT candidate_sha256, state - FROM qbit_block_candidate_outbox - WHERE block_hash = {self._text_literal(block_hash)} -), -inserted AS ( - INSERT INTO qbit_block_candidate_outbox ( - block_hash, share_id, candidate, candidate_sha256 - ) - SELECT - {self._text_literal(block_hash)}, NULL, - {self._jsonb_literal(candidate)}, {self._text_literal(candidate_sha256)} - FROM lease - WHERE NOT EXISTS (SELECT 1 FROM existing) - ON CONFLICT (block_hash) DO NOTHING - RETURNING block_hash -) -SELECT CASE - WHEN NOT EXISTS (SELECT 1 FROM lease) THEN - json_build_object('error', 'writer lease is not active') - WHEN EXISTS ( - SELECT 1 FROM existing - WHERE candidate_sha256 <> {self._text_literal(candidate_sha256)} - ) THEN - json_build_object('error', 'block candidate payload mismatch') - ELSE - json_build_object( - 'inserted', (SELECT count(*) FROM inserted), - 'state', COALESCE((SELECT state FROM existing), 'pending') - ) -END; -""" - result = self._run_fenced_json(sql) - if "error" in result: - raise RuntimeError(str(result["error"])) - return BlockCandidateIntentPersistResult( - inserted=int(result.get("inserted", 0)) > 0, - state=str(result.get("state", "pending")), + # -- chunked candidate bodies (#255) ----------------------------------- + + def _candidate_storage_version_value(self) -> int: + return int( + getattr(self, "_candidate_storage_version", CANDIDATE_BODY_STORAGE_VERSION) ) - def pending_block_candidates(self, *, limit: int = 32) -> list[dict[str, Any]]: - return [ - row["candidate"] - for row in self.pending_block_candidate_rows(limit=limit) - ] + def _ensure_candidate_store_state(self) -> None: + """Backfill #255 state for ledgers built without ``__init__``.""" + if not hasattr(self, "_candidate_body_gate"): + self._candidate_body_gate = BoundedSemaphore(1) + if not hasattr(self, "_candidate_spool"): + self._candidate_spool = SpoolAdmission() + if not hasattr(self, "_legacy_candidate_helper"): + command = getattr(self, "_command", None) or [] + self._legacy_candidate_helper = LegacyCandidateHelper( + LegacyTransport( + psql_command=" ".join(shlex.quote(part) for part in command) or None, + database_url=database_url_from_psql_command(list(command)), + ) + ) - def pending_block_candidate_rows(self, *, limit: int = 32) -> list[dict[str, Any]]: - """Return pending payloads together with their authoritative row keys.""" - if limit <= 0: - return [] - sql = f""" -SELECT COALESCE( - json_agg( - json_build_object('block_hash', block_hash, 'candidate', candidate) - ORDER BY created_at, block_hash - ), - '[]'::json -) -FROM ( - SELECT candidate, created_at, block_hash - FROM qbit_block_candidate_outbox - WHERE state = 'pending' - ORDER BY created_at, block_hash - LIMIT {int(limit)} -) pending; -""" - with self._operation_gate(self._lock, "writer lock"): - return list(self._run_retry_safe_read_json(sql)) + @staticmethod + def _legacy_candidate_document(candidate: Any) -> Any: + """Materialize an intent for the legacy whole-jsonb route only. + + Storage version 1 is the compatibility/rollback mode; its statement + inlines the whole document, so the share sequence is copied here + and nowhere else. A candidate-only intent keeps no share member, + and a hydrated replay (the rollback floor re-persisting a version-2 + row) has its streamed views decoded, because this route writes + plain JSON. + """ + if candidate is None or not isinstance(candidate, PreparedCandidateIntent): + return candidate + document = materialize_spool_views(candidate.facts) + if candidate.has_shares: + document["shares_json"] = [ + materialize_spool_views(share) if is_spool_view(share) else share + for share in candidate.shares + ] + return document - def block_candidate_pending_metrics(self) -> dict[str, int | float]: - """Return aggregate pending ages using the existing outbox index.""" - sql = """ + def _writer_identity_payload(self) -> dict[str, Any]: + return { + "writer_id": self._writer_id, + "writer_epoch": int(self._writer_epoch), + "writer_session_token": self._writer_session_token, + } + + def _observe_candidate_outbox_row(self, block_hash: str) -> dict[str, Any] | None: + """One read-slot snapshot of an outbox row's publication facts. + + This is the version the precomparison binds to: ``state``, + ``share_id``, ``candidate_sha256``, ``storage_version`` and + ``body_id`` are re-asserted by the fenced statement, so any change + between this read and the commit fails that statement closed and + the caller redoes the comparison. + """ + sql = f""" SELECT json_build_object( - 'pending_count', count(*), - 'oldest_pending_age_seconds', COALESCE( - EXTRACT(EPOCH FROM clock_timestamp() - min(created_at)), - 0 + 'found', EXISTS ( + SELECT 1 FROM qbit_block_candidate_outbox WHERE block_hash = {self._text_literal(block_hash)} ), - 'oldest_unattempted_age_seconds', COALESCE( - EXTRACT( - EPOCH FROM clock_timestamp() - - min(created_at) FILTER (WHERE attempt_count = 0) - ), - 0 + 'state', (SELECT state FROM qbit_block_candidate_outbox WHERE block_hash = {self._text_literal(block_hash)}), + 'share_id', (SELECT share_id FROM qbit_block_candidate_outbox WHERE block_hash = {self._text_literal(block_hash)}), + 'candidate_sha256', (SELECT candidate_sha256 FROM qbit_block_candidate_outbox WHERE block_hash = {self._text_literal(block_hash)}), + 'storage_version', (SELECT storage_version FROM qbit_block_candidate_outbox WHERE block_hash = {self._text_literal(block_hash)}), + 'body_id', (SELECT body_id FROM qbit_block_candidate_outbox WHERE block_hash = {self._text_literal(block_hash)}), + 'body_state', ( + SELECT body.state + FROM qbit_block_candidate_body body + JOIN qbit_block_candidate_outbox outbox ON outbox.body_id = body.body_id + WHERE outbox.block_hash = {self._text_literal(block_hash)} ) -) -FROM qbit_block_candidate_outbox -WHERE state = 'pending'; +); """ - with self._operation_gate(self._lock, "writer lock"): - metrics = self._run_retry_safe_read_json(sql) + observed = self._run_attributed_read_json( + sql, + operation="observe_block_candidate_outbox_row", + gate=self._read_semaphore, + gate_name="read slot", + ) + if not isinstance(observed, dict) or not observed.get("found"): + return None return { - "pending_count": int(metrics.get("pending_count", 0)), - "oldest_pending_age_seconds": float( - metrics.get("oldest_pending_age_seconds", 0.0) + "state": str(observed.get("state")), + "share_id": ( + None if observed.get("share_id") is None else str(observed["share_id"]) ), - "oldest_unattempted_age_seconds": float( - metrics.get("oldest_unattempted_age_seconds", 0.0) + "candidate_sha256": str(observed.get("candidate_sha256")), + "storage_version": int(observed.get("storage_version") or 1), + "body_id": ( + None if observed.get("body_id") is None else str(observed["body_id"]) + ), + "body_state": ( + None if observed.get("body_state") is None else str(observed["body_state"]) ), } - def mark_block_candidate_attempted(self, *, block_hash: str) -> bool: - """Fence and count entry into a candidate processing attempt.""" - sql = f""" -WITH lease AS ( - UPDATE qbit_ledger_writer_lease - SET lease_expires_at = clock_timestamp() + {self._lease_interval_sql}, - updated_at = clock_timestamp() - WHERE singleton - AND writer_id = {self._text_literal(self._writer_id)} - AND writer_epoch = {int(self._writer_epoch)} - AND writer_session_token = {self._text_literal(self._writer_session_token)} - RETURNING writer_id -), -updated AS ( - UPDATE qbit_block_candidate_outbox - SET attempt_count = attempt_count + 1, - updated_at = clock_timestamp() - FROM lease - WHERE block_hash = {self._text_literal(block_hash.lower())} - AND state = 'pending' - RETURNING block_hash -) -SELECT CASE - WHEN NOT EXISTS (SELECT 1 FROM lease) THEN - json_build_object('error', 'writer lease is not active') - ELSE - json_build_object('updated', (SELECT count(*) FROM updated)) -END; + def verify_candidate_schema(self) -> dict[str, Any]: + """Refuse a database whose candidate storage is newer than this code. + + Reads what the schema declares (``qbit_prism_schema_capabilities``) + and whether the chunked-body tables exist. A declared storage + version above the one this process understands raises + :class:`IncompatibleCandidateSchema`; so does a database without + the 002 migration, whichever storage version is configured (replay + and the terminal outbox statements use its columns), before any + candidate is staged. Cached after the first successful check; the + coordinator also calls it at boot. + """ + cached = getattr(self, "_candidate_schema_verified", None) + if cached is not None: + return cached + presence = self._run_json( + """ +SELECT json_build_object( + 'has_body_table', to_regclass('qbit_block_candidate_body') IS NOT NULL, + 'has_capabilities', to_regclass('qbit_prism_schema_capabilities') IS NOT NULL +); """ - result = self._run_fenced_json(sql) - if "error" in result: - raise RuntimeError(str(result["error"])) - return int(result.get("updated", 0)) > 0 + ) + has_body_table = bool(isinstance(presence, dict) and presence.get("has_body_table")) + declared: int | None = None + if isinstance(presence, dict) and presence.get("has_capabilities"): + capability = self._run_json(schema_capability_sql()) + if isinstance(capability, dict) and capability.get("declared") is not None: + declared = int(capability["declared"]) + refusal = candidate_schema_refusal(declared, has_body_table) + if refusal is not None: + raise IncompatibleCandidateSchema(refusal) + verdict = {"declared": declared, "has_body_table": has_body_table} + self._candidate_schema_verified = verdict + return verdict + + def stage_candidate_body(self, prepared: PreparedCandidateIntent) -> str: + """Upload and seal one immutable body; returns its ``body_id``. + + Runs under the one-slot body gate, never the writer lock: staging + touches no lease row and grants no credit. One statement per chunk + (each bounded by the chunk size), index rows in batches of at most + 256 pages / 64 spans, then one seal statement that locks the + manifest, proves completeness from the rows and re-hashes every + chunk on the server. A failure retires the staging manifest so the + janitor reclaims it. The schema-capability refusal runs at + coordinator boot (``verify_candidate_schema``), not here. + """ + self._ensure_candidate_store_state() + manifest = prepared.manifest + body_id = new_body_id() + identity = self._writer_identity_payload() + with self._operation_gate(self._candidate_body_gate, "candidate body slot"): + staged = self._run_json( + stage_body_sql( + { + **manifest.to_json(), + "body_id": body_id, + "block_hash": prepared.block_hash, + **identity, + }, + jsonb=self._jsonb_literal, + ) + ) + if not isinstance(staged, dict) or staged.get("state") != "staging": + raise CandidateStorageError( + f"candidate body staging was not created: {staged!r}" + ) + try: + pending_spans: list[dict[str, Any]] = [] + + def flush_index(spans: list[dict[str, Any]], pages: list[list[Any]]) -> None: + if not spans and not pages: + return + outcome = self._run_json( + body_index_rows_sql( + {"body_id": body_id, **identity, "spans": spans, "pages": pages}, + jsonb=self._jsonb_literal, + ) + ) + if not isinstance(outcome, dict) or not outcome.get("owned"): + raise CandidateStorageError( + "candidate body staging lost its producer session" + ) - def mark_block_candidate_submitted(self, *, block_hash: str) -> bool: - return self._finish_block_candidate(block_hash=block_hash, state="submitted", error=None) + def upload(chunk: Any) -> None: + outcome = self._run_json( + body_chunk_sql( + { + "body_id": body_id, + "ordinal": chunk.ordinal, + "chunk_sha256": chunk.sha256, + "length": len(chunk.data), + **identity, + }, + chunk.data, + jsonb=self._jsonb_literal, + ) + ) + if not isinstance(outcome, dict) or not outcome.get("owned"): + raise CandidateStorageError( + "candidate body staging lost its producer session" + ) - def mark_block_candidate_abandoned(self, *, block_hash: str, error: str) -> bool: - return self._finish_block_candidate(block_hash=block_hash, state="abandoned", error=error) + def on_span(span: Any) -> None: + pending_spans.append(span.to_json()) + if len(pending_spans) >= INDEX_SPAN_ROWS: + flush_index(list(pending_spans), []) + pending_spans.clear() + + def on_page(batch: Any) -> None: + flush_index( + [], + [ + [entry.field, entry.ordinal, entry.offset, entry.record_index] + for entry in batch + ], + ) - def _finish_block_candidate(self, *, block_hash: str, state: str, error: str | None) -> bool: - if state not in {"submitted", "abandoned"}: - raise ValueError("invalid block candidate terminal state") - sql = f""" -WITH lease AS ( - UPDATE qbit_ledger_writer_lease - SET lease_expires_at = clock_timestamp() + {self._lease_interval_sql}, - updated_at = clock_timestamp() - WHERE singleton - AND writer_id = {self._text_literal(self._writer_id)} - AND writer_epoch = {int(self._writer_epoch)} - AND writer_session_token = {self._text_literal(self._writer_session_token)} - RETURNING writer_id -), -updated AS ( - UPDATE qbit_block_candidate_outbox - SET state = {self._text_literal(state)}, - last_error = {self._text_literal(error) if error is not None else 'NULL'}, - updated_at = clock_timestamp(), - completed_at = clock_timestamp(), - candidate = NULL - FROM lease - WHERE block_hash = {self._text_literal(block_hash.lower())} - AND state = 'pending' - RETURNING block_hash -) -SELECT CASE - WHEN NOT EXISTS (SELECT 1 FROM lease) THEN - json_build_object('error', 'writer lease is not active') - ELSE - json_build_object('updated', (SELECT count(*) FROM updated)) -END; -""" - result = self._run_fenced_json(sql) - if "error" in result: - raise RuntimeError(str(result["error"])) - return int(result.get("updated", 0)) > 0 + prepared.body.write_chunks(upload, on_span=on_span, on_page=on_page) + flush_index(list(pending_spans), []) + pending_spans.clear() + sealed = self._run_json( + seal_body_sql( + { + "body_id": body_id, + "candidate_sha256": manifest.candidate_sha256, + **identity, + }, + jsonb=self._jsonb_literal, + ) + ) + if not isinstance(sealed, dict) or int(sealed.get("sealed", 0)) != 1: + raise CandidateStorageError( + f"candidate body could not be sealed: {sealed!r}" + ) + except BaseException: + self._retire_staging_body(body_id) + raise + return body_id - def snapshot_at_job_issue( + def _retire_staging_body(self, body_id: str) -> None: + """Best-effort: hand a failed staging body to the janitor.""" + try: + self._run_json(retire_staging_body_sql(body_id, text=self._text_literal)) + except Exception: # noqa: BLE001 - the janitor retires stale staging anyway + pass + + def compare_candidate_body( self, - anchor_job_issued_at_ms: int, - *, - window_weight: int | None = None, - ) -> list[AcceptedShareRecord]: - anchor = ( - f"to_timestamp(({int(anchor_job_issued_at_ms)}::double precision / 1000.0))" - ) + body_id: str, + prepared: PreparedCandidateIntent, + ) -> bool: + """The historical content comparator for a stored chunked body. + + Historically an existing row was accepted only when its identity + digest matched *and* its normalized jsonb equalled the new intent's. + The caller has already matched the digest; this is the second + check, kept as a comparison of content rather than of any digest: + + 1. the actual stored bytes are read back one bounded page at a time + (at most three chunks per statement) and compared with our + re-encoded bytes chunk by chunk, holding one page at a time; + 2. if the bytes differ, the normalized jsonb-equivalence fallback + runs in the isolated helper (numeric and key-order differences + that jsonb treats as equal), exactly the equality PostgreSQL + applied to the two jsonb documents before. + + A forced digest collision between distinct documents is therefore + rejected here, and a jsonb-equivalent retry with a different + encoding still passes. The result is bound to the body version the + fenced statement re-asserts (compare-and-swap). + """ + self._ensure_candidate_store_state() + manifest_row = self.read_candidate_body_manifest(body_id) + try: + manifest, state, _referenced = parse_manifest_row(manifest_row) + except (CandidateBodyUnavailable, CandidateBodyIntegrityError): + return False + if state != "sealed": + return False + bytes_equal = ( + manifest.byte_count == prepared.manifest.byte_count + and manifest.chunk_count == prepared.manifest.chunk_count + ) + if bytes_equal: + + class _Mismatch(Exception): + pass + + page: dict[int, bytes] = {} + + def consumer(chunk: Any) -> None: + if chunk.ordinal not in page: + page.clear() + page_state, _referenced, stored = parse_body_page( + self.read_candidate_body_page(body_id, chunk.ordinal, BODY_READ_MAX_CHUNKS), + max_chunks=BODY_READ_MAX_CHUNKS, + ) + if page_state != "sealed": + raise _Mismatch() + for entry in stored: + page[entry.ordinal] = entry.data + stored_bytes = page.get(chunk.ordinal) + if stored_bytes is None or stored_bytes != chunk.data: + raise _Mismatch() + + try: + prepared.body.write_chunks(consumer) + except _Mismatch: + bytes_equal = False + if bytes_equal: + return True + return bool( + self._legacy_candidate_helper.compare_stored_body( + body_id, prepared.block_hash, prepared.body + ) + ) + + def read_candidate_body_manifest(self, body_id: str) -> Any: + return self._run_attributed_read_json( + body_manifest_sql(body_id, text=self._text_literal), + operation="read_block_candidate_body_manifest", + gate=self._read_semaphore, + gate_name="read slot", + ) + + def read_candidate_body_spans(self, body_id: str, after_field: str | None) -> Any: + return self._run_attributed_read_json( + body_spans_sql(body_id, after_field, text=self._text_literal), + operation="read_block_candidate_body_spans", + gate=self._read_semaphore, + gate_name="read slot", + ) + + def read_candidate_body_pages(self, body_id: str, field_name: str, from_ordinal: int) -> Any: + return self._run_attributed_read_json( + body_pages_sql(body_id, field_name, from_ordinal, text=self._text_literal), + operation="read_block_candidate_body_pages", + gate=self._read_semaphore, + gate_name="read slot", + ) + + def read_candidate_body_page(self, body_id: str, from_ordinal: int, max_chunks: int) -> Any: + return self._run_attributed_read_json( + body_page_sql(body_id, from_ordinal, max_chunks, text=self._text_literal), + operation="read_block_candidate_body_page", + gate=self._read_semaphore, + gate_name="read slot", + ) + + def _bind_candidate_publication( + self, + prepared: PreparedCandidateIntent, + share_id: str, + ) -> dict[str, Any]: + """Precompare and stage outside the lease-row critical section. + + Returns the observed version the fenced statement must find + unchanged. A missing row means the body is staged and sealed now; an + existing row is compared exactly -- digest, share linkage, and for a + pending row the full body (server chunk re-hash for a chunked body, + the isolated compatibility helper for a legacy jsonb body). Terminal + rows compare on digest and linkage only, exactly as the historical + statement did once their payload had been deleted. + """ + observed = self._observe_candidate_outbox_row(prepared.block_hash) + if observed is None: + return { + "found": False, + "state": None, + "share_id": None, + "storage_version": None, + "body_id": self.stage_candidate_body(prepared), + } + if observed["candidate_sha256"] != prepared.candidate_sha256: + raise RuntimeError("block candidate payload mismatch") + if observed["share_id"] is not None and observed["share_id"] != share_id: + raise RuntimeError("block candidate payload mismatch") + if observed["state"] == "pending": + if observed["storage_version"] == CANDIDATE_BODY_STORAGE_VERSION: + if observed["body_id"] is None or not self.compare_candidate_body( + observed["body_id"], prepared + ): + raise RuntimeError("block candidate payload mismatch") + else: + self._ensure_candidate_store_state() + if not self._legacy_candidate_helper.compare( + prepared.block_hash, prepared.body + ): + raise RuntimeError("block candidate payload mismatch") + return { + "found": True, + "state": observed["state"], + "share_id": observed["share_id"], + "storage_version": observed["storage_version"], + "body_id": observed["body_id"], + } + + def _append_batch_with_replay_outcomes_v2( + self, + entries: list[tuple[PendingShare, Any]], + ) -> list[ShareReplayResult]: + """Chunked-body batch: stage outside the fence, publish inside it. + + Same lease fence, durability setting, share comparator, ordering + and ACK boundary as the historical statement. The candidate half + differs only in *where* body-sized work happens: precomparison and + staging run before the writer gate is taken, and the fenced + statement re-asserts the observed row version (compare-and-swap) + and requires a sealed manifest carrying the same digest before it + publishes a reference. If the version moved, nothing is committed + and the whole bind/publish is redone under the same call. + """ + prepared_entries: list[tuple[PendingShare, PreparedCandidateIntent | None]] = [] + share_ids: set[str] = set() + block_hashes: set[str] = set() + for pending, candidate in entries: + if pending.share_difficulty <= 0: + raise ValueError("share_difficulty must be positive") + if pending.network_difficulty <= 0: + raise ValueError("network_difficulty must be positive") + if pending.share_id in share_ids: + raise ValueError("duplicate share_id in append batch") + share_ids.add(pending.share_id) + prepared: PreparedCandidateIntent | None = None + if candidate is not None: + prepared = prepare_candidate_intent(candidate) + if prepared.block_hash in block_hashes: + raise ValueError("duplicate block candidate in append batch") + block_hashes.add(prepared.block_hash) + prepared_entries.append((pending, prepared)) + attempts = 0 + while True: + attempts += 1 + payloads: list[dict[str, Any]] = [] + for pending, prepared in prepared_entries: + candidate_payload: dict[str, Any] | None = None + if prepared is not None: + candidate_payload = { + "block_hash_hex": prepared.block_hash, + "candidate_sha256": prepared.candidate_sha256, + "header": prepared.replay_header(), + "expected": self._bind_candidate_publication( + prepared, pending.share_id + ), + } + candidate_payload["body_id"] = candidate_payload["expected"]["body_id"] + payloads.append( + { + "share": { + **pending.__dict__, + "credit_policy": validate_credit_policy(pending.credit_policy), + }, + "candidate": candidate_payload, + } + ) + outcome = self._publish_candidate_batch(payloads, len(prepared_entries)) + if outcome is not None: + return outcome + if attempts >= 3: + raise RuntimeError( + "block candidate outbox changed repeatedly during publication" + ) + + def _publish_candidate_batch( + self, + payloads: list[dict[str, Any]], + expected_count: int, + ) -> list[ShareReplayResult] | None: + payload = {"entries": payloads, **self._writer_identity_payload()} + sql = f""" +WITH input AS ( + SELECT + {self._jsonb_literal(payload)} AS root, + set_config('synchronous_commit', 'on', true) AS durability +), +payload AS ( + SELECT + item->'share' AS data, + NULLIF(item->'candidate', 'null'::jsonb) AS candidate, + ordinality + FROM input, + jsonb_array_elements(root->'entries') WITH ORDINALITY AS rows(item, ordinality) +), +lease AS ( + UPDATE qbit_ledger_writer_lease + SET lease_expires_at = clock_timestamp() + {self._lease_interval_sql}, + updated_at = clock_timestamp() + FROM input + WHERE qbit_ledger_writer_lease.singleton + AND qbit_ledger_writer_lease.writer_id = root->>'writer_id' + AND qbit_ledger_writer_lease.writer_epoch = (root->>'writer_epoch')::bigint + AND qbit_ledger_writer_lease.writer_session_token = root->>'writer_session_token' + RETURNING qbit_ledger_writer_lease.writer_id +), +share_mismatch AS ( + SELECT data->>'share_id' AS share_id + FROM payload + JOIN qbit_share_ledger ledger ON ledger.share_id = data->>'share_id' + WHERE ledger.miner_id IS DISTINCT FROM data->>'miner_id' + OR ledger.payout_order_key IS DISTINCT FROM data->>'order_key' + OR ledger.p2mr_program IS DISTINCT FROM decode(data->>'p2mr_program_hex', 'hex') + OR ledger.share_difficulty IS DISTINCT FROM (data->>'share_difficulty')::numeric + OR ledger.network_difficulty IS DISTINCT FROM (data->>'network_difficulty')::numeric + OR ledger.template_height IS DISTINCT FROM (data->>'template_height')::bigint + OR ledger.job_id IS DISTINCT FROM data->>'job_id' + OR ledger.job_issued_at IS DISTINCT FROM to_timestamp((data->>'job_issued_at_ms')::double precision / 1000.0) + OR ledger.ntime IS DISTINCT FROM (data->>'ntime')::bigint + OR ledger.credit_policy IS DISTINCT FROM data->>'credit_policy' +), +candidate_mismatch AS ( + -- Compare-and-swap against the version the precomparison observed. + SELECT payload.candidate->>'block_hash_hex' AS block_hash + FROM payload + LEFT JOIN qbit_block_candidate_outbox outbox + ON outbox.block_hash = payload.candidate->>'block_hash_hex' + WHERE payload.candidate IS NOT NULL + AND ( + (outbox.block_hash IS NOT NULL) IS DISTINCT FROM (payload.candidate->'expected'->>'found')::boolean + OR ( + outbox.block_hash IS NOT NULL + AND ( + outbox.state IS DISTINCT FROM payload.candidate->'expected'->>'state' + OR outbox.share_id IS DISTINCT FROM payload.candidate->'expected'->>'share_id' + OR outbox.candidate_sha256 IS DISTINCT FROM payload.candidate->>'candidate_sha256' + OR outbox.storage_version IS DISTINCT FROM (payload.candidate->'expected'->>'storage_version')::integer + OR outbox.body_id IS DISTINCT FROM payload.candidate->'expected'->>'body_id' + OR (outbox.share_id IS NOT NULL AND outbox.share_id IS DISTINCT FROM payload.data->>'share_id') + ) + ) + ) +), +sealed_bodies AS ( + -- The bodies this statement may reference, locked FOR SHARE so an + -- orphan retirement (FOR UPDATE) serializes against this publication + -- and whichever side loses the lock re-evaluates and fails closed. + SELECT body.body_id + FROM qbit_block_candidate_body body + WHERE body.body_id IN ( + SELECT payload.candidate->>'body_id' + FROM payload + WHERE payload.candidate IS NOT NULL + AND NOT (payload.candidate->'expected'->>'found')::boolean + ) + AND body.state = 'sealed' + AND EXISTS ( + SELECT 1 FROM payload + WHERE payload.candidate->>'body_id' = body.body_id + AND body.candidate_sha256 = payload.candidate->>'candidate_sha256' + AND body.block_hash = payload.candidate->>'block_hash_hex' + ) + FOR SHARE +), +candidate_unsealed AS ( + -- A new reference may point only at a complete sealed body with the + -- same digest and block hash. + SELECT payload.candidate->>'block_hash_hex' AS block_hash + FROM payload + WHERE payload.candidate IS NOT NULL + AND NOT (payload.candidate->'expected'->>'found')::boolean + AND NOT EXISTS ( + SELECT 1 FROM sealed_bodies + WHERE sealed_bodies.body_id = payload.candidate->>'body_id' + ) +), +candidate_states AS ( + SELECT + payload.ordinality, + CASE + WHEN payload.candidate IS NULL THEN NULL + ELSE COALESCE(outbox.state, 'pending') + END AS candidate_outbox_state + FROM payload + LEFT JOIN qbit_block_candidate_outbox outbox + ON outbox.block_hash = payload.candidate->>'block_hash_hex' +), +batch_ok AS ( + SELECT 1 AS ok + WHERE EXISTS (SELECT 1 FROM lease) + AND NOT EXISTS (SELECT 1 FROM share_mismatch) + AND NOT EXISTS (SELECT 1 FROM candidate_mismatch) + AND NOT EXISTS (SELECT 1 FROM candidate_unsealed) +), +inserted_shares AS ( + INSERT INTO qbit_share_ledger ( + share_id, miner_id, payout_order_key, p2mr_program, + share_difficulty, network_difficulty, template_height, job_id, + job_issued_at, ntime, accepted_at, credit_policy, accepted, + writer_id, writer_epoch + ) + SELECT + data->>'share_id', data->>'miner_id', data->>'order_key', + decode(data->>'p2mr_program_hex', 'hex'), + (data->>'share_difficulty')::numeric, + (data->>'network_difficulty')::numeric, + (data->>'template_height')::bigint, data->>'job_id', + to_timestamp((data->>'job_issued_at_ms')::double precision / 1000.0), + (data->>'ntime')::bigint, + to_timestamp((data->>'accepted_at_ms')::double precision / 1000.0), + data->>'credit_policy', true, root->>'writer_id', + (root->>'writer_epoch')::bigint + FROM payload, input, batch_ok + WHERE NOT EXISTS ( + SELECT 1 FROM qbit_share_ledger existing + WHERE existing.share_id = payload.data->>'share_id' + ) + ORDER BY payload.ordinality + ON CONFLICT (share_id) DO NOTHING + RETURNING qbit_share_ledger.* +), +inserted_candidates AS ( + INSERT INTO qbit_block_candidate_outbox ( + block_hash, share_id, candidate, candidate_sha256, + storage_version, body_id, replay_header, parent_hash, expected_height + ) + SELECT + payload.candidate->>'block_hash_hex', payload.data->>'share_id', + NULL, payload.candidate->>'candidate_sha256', + {CANDIDATE_BODY_STORAGE_VERSION}, payload.candidate->>'body_id', + payload.candidate->'header', + payload.candidate->'header'->>'parent_hash', + NULLIF(payload.candidate->'header'->>'expected_height', '')::bigint + FROM payload, batch_ok + WHERE payload.candidate IS NOT NULL + AND NOT (payload.candidate->'expected'->>'found')::boolean + ON CONFLICT (block_hash) DO UPDATE + SET share_id = EXCLUDED.share_id, + updated_at = clock_timestamp() + WHERE qbit_block_candidate_outbox.share_id IS NULL + AND qbit_block_candidate_outbox.state = 'pending' + AND qbit_block_candidate_outbox.candidate_sha256 = EXCLUDED.candidate_sha256 + RETURNING block_hash +), +linked_candidates AS ( + -- Credit-on-accept: the standalone intent already published this body; + -- the share joins the same identity without re-uploading anything. + UPDATE qbit_block_candidate_outbox + SET share_id = payload.data->>'share_id', + updated_at = clock_timestamp() + FROM payload, batch_ok + WHERE payload.candidate IS NOT NULL + AND (payload.candidate->'expected'->>'found')::boolean + AND qbit_block_candidate_outbox.block_hash = payload.candidate->>'block_hash_hex' + AND qbit_block_candidate_outbox.share_id IS NULL + AND qbit_block_candidate_outbox.state = 'pending' + AND qbit_block_candidate_outbox.candidate_sha256 = payload.candidate->>'candidate_sha256' + RETURNING qbit_block_candidate_outbox.block_hash +), +records AS ( + SELECT + ledger.*, payload.ordinality, false AS newly_inserted, + false AS new_miner, candidate_states.candidate_outbox_state + FROM payload + JOIN qbit_share_ledger ledger ON ledger.share_id = payload.data->>'share_id' + JOIN candidate_states ON candidate_states.ordinality = payload.ordinality + UNION ALL + SELECT + inserted_shares.*, payload.ordinality, true AS newly_inserted, + NOT EXISTS ( + SELECT 1 + FROM qbit_share_ledger existing_miner + WHERE existing_miner.accepted + AND existing_miner.miner_id = inserted_shares.miner_id + ) + AND NOT EXISTS ( + SELECT 1 + FROM inserted_shares earlier_insert + WHERE earlier_insert.miner_id = inserted_shares.miner_id + AND earlier_insert.share_seq < inserted_shares.share_seq + ) AS new_miner, + candidate_states.candidate_outbox_state + FROM inserted_shares + JOIN payload ON payload.data->>'share_id' = inserted_shares.share_id + JOIN candidate_states ON candidate_states.ordinality = payload.ordinality +) +SELECT CASE + WHEN NOT EXISTS (SELECT 1 FROM lease) THEN + json_build_object('error', 'writer lease is not active') + WHEN EXISTS (SELECT 1 FROM share_mismatch) THEN + json_build_object( + 'error', 'duplicate share_id payload mismatch', + 'error_kind', 'share_replay_conflict', + 'share_ids', (SELECT json_agg(share_id ORDER BY share_id) FROM share_mismatch) + ) + WHEN EXISTS (SELECT 1 FROM candidate_mismatch) THEN + json_build_object( + 'error', 'block candidate outbox changed during publication', + 'error_kind', 'candidate_version_changed', + 'block_hashes', (SELECT json_agg(block_hash ORDER BY block_hash) FROM candidate_mismatch) + ) + WHEN EXISTS (SELECT 1 FROM candidate_unsealed) THEN + json_build_object( + 'error', 'block candidate body is not sealed', + 'error_kind', 'candidate_body_unsealed', + 'block_hashes', (SELECT json_agg(block_hash ORDER BY block_hash) FROM candidate_unsealed) + ) + ELSE json_build_object( + 'records', ( + SELECT json_agg(json_build_object( + 'share_seq', records.share_seq, + 'share_id', records.share_id, + 'miner_id', records.miner_id, + 'order_key', records.payout_order_key, + 'p2mr_program_hex', encode(records.p2mr_program, 'hex'), + 'share_difficulty', records.share_difficulty::text, + 'network_difficulty', records.network_difficulty::text, + 'template_height', records.template_height, + 'job_id', records.job_id, + 'job_issued_at_ms', round(extract(epoch FROM records.job_issued_at) * 1000)::bigint, + 'accepted_at_ms', round(extract(epoch FROM records.accepted_at) * 1000)::bigint, + 'ntime', records.ntime, + 'credit_policy', records.credit_policy, + 'newly_inserted', records.newly_inserted, + 'new_miner', records.new_miner, + 'candidate_outbox_state', records.candidate_outbox_state + ) ORDER BY records.ordinality) + FROM records + ) + ) +END; +""" + with self._operation_gate(self._lock, "writer lock"): + result = self._run_json(sql) + if "error" in result: + if result.get("error_kind") == "share_replay_conflict": + raise ShareReplayConflict(str(result["error"])) + if result.get("error_kind") in {"candidate_version_changed", "candidate_body_unsealed"}: + return None + raise RuntimeError(str(result["error"])) + records = result.get("records") + if not isinstance(records, list) or len(records) != expected_count: + raise RuntimeError("Postgres share batch returned an incomplete result") + parsed = [self._record_from_json(record) for record in records] + committed = sorted( + zip(records, parsed, strict=True), + key=lambda item: item[1].share_seq, + ) + for record_payload, record in committed: + if bool(record_payload.get("newly_inserted", True)): + self._note_appended_share( + record, + new_miner=bool(record_payload.get("new_miner", False)), + ) + return [ + ShareReplayResult( + ( + "inserted" + if bool(record_payload.get("newly_inserted", True)) + else "exact_existing" + ), + record, + ) + for record_payload, record in zip(records, parsed, strict=True) + ] + + def append_batch( + self, + entries: list[tuple[PendingShare, Any]], + ) -> list[AcceptedShareRecord]: + return [ + outcome.record + for outcome in self._append_batch_with_replay_outcomes(entries) + ] + + def append_recovered_share(self, pending: PendingShare) -> ShareReplayResult: + """Use the exact batch comparator for one typed recovery outcome.""" + outcomes = self._append_batch_with_replay_outcomes([(pending, None)]) + if len(outcomes) != 1: + raise RuntimeError("Postgres recovery append returned an incomplete result") + return outcomes[0] + + def persist_block_candidate_intent( + self, + candidate: Any, + ) -> BlockCandidateIntentPersistResult: + """Persist candidate work that is not yet eligible for share credit. + + Historically only the identity digest is compared against an + existing row, and that is preserved. The chunked route stages and + seals the body outside the writer gate first; the fenced insert then + publishes a reference only when a sealed manifest with the same + digest and block hash exists, and leaves a colliding row alone. + """ + if not isinstance(candidate, PreparedCandidateIntent): + if not isinstance(candidate, dict): + raise TypeError("block candidate intent must be an object") + if not str(candidate.get("block_hash_hex", "")).lower(): + raise ValueError("block candidate is missing block_hash_hex") + if self._candidate_storage_version_value() != CANDIDATE_BODY_STORAGE_VERSION: + return self._persist_block_candidate_intent_v1( + self._legacy_candidate_document(candidate) + ) + try: + prepared = prepare_candidate_intent(candidate) + except CandidateCodecError as exc: + raise ValueError(str(exc)) from exc + block_hash = prepared.block_hash + candidate_sha256 = prepared.candidate_sha256 + observed = self._observe_candidate_outbox_row(block_hash) + if observed is not None: + if observed["candidate_sha256"] != candidate_sha256: + raise RuntimeError("block candidate payload mismatch") + return BlockCandidateIntentPersistResult( + inserted=False, + state=observed["state"], + ) + body_id = self.stage_candidate_body(prepared) + header = prepared.replay_header() + sql = f""" +WITH durability AS ( + SELECT set_config('synchronous_commit', 'on', true) +), +lease AS ( + UPDATE qbit_ledger_writer_lease + SET lease_expires_at = clock_timestamp() + {self._lease_interval_sql}, + updated_at = clock_timestamp() + FROM durability + WHERE singleton + AND writer_id = {self._text_literal(self._writer_id)} + AND writer_epoch = {int(self._writer_epoch)} + AND writer_session_token = {self._text_literal(self._writer_session_token)} + RETURNING writer_id +), +existing AS ( + SELECT candidate_sha256, state + FROM qbit_block_candidate_outbox + WHERE block_hash = {self._text_literal(block_hash)} +), +sealed_body AS ( + -- Locked FOR SHARE: an orphan retirement's FOR UPDATE serializes + -- against this publication, and the loser re-evaluates and fails closed. + SELECT body_id + FROM qbit_block_candidate_body + WHERE body_id = {self._text_literal(body_id)} + AND state = 'sealed' + AND candidate_sha256 = {self._text_literal(candidate_sha256)} + AND block_hash = {self._text_literal(block_hash)} + FOR SHARE +), +inserted AS ( + INSERT INTO qbit_block_candidate_outbox ( + block_hash, share_id, candidate, candidate_sha256, + storage_version, body_id, replay_header, parent_hash, expected_height + ) + SELECT + {self._text_literal(block_hash)}, NULL, NULL, + {self._text_literal(candidate_sha256)}, + {CANDIDATE_BODY_STORAGE_VERSION}, sealed_body.body_id, + {self._jsonb_literal(header)}, + {self._jsonb_literal(header)}->>'parent_hash', + NULLIF({self._jsonb_literal(header)}->>'expected_height', '')::bigint + FROM lease, sealed_body + WHERE NOT EXISTS (SELECT 1 FROM existing) + ON CONFLICT (block_hash) DO NOTHING + RETURNING block_hash +) +SELECT CASE + WHEN NOT EXISTS (SELECT 1 FROM lease) THEN + json_build_object('error', 'writer lease is not active') + WHEN EXISTS ( + SELECT 1 FROM existing + WHERE candidate_sha256 <> {self._text_literal(candidate_sha256)} + ) THEN + json_build_object('error', 'block candidate payload mismatch') + WHEN NOT EXISTS (SELECT 1 FROM existing) AND NOT EXISTS (SELECT 1 FROM sealed_body) THEN + json_build_object('error', 'block candidate body is not sealed') + ELSE + json_build_object( + 'inserted', (SELECT count(*) FROM inserted), + 'state', COALESCE((SELECT state FROM existing), 'pending') + ) +END; +""" + result = self._run_fenced_json(sql) + if "error" in result: + raise RuntimeError(str(result["error"])) + return BlockCandidateIntentPersistResult( + inserted=int(result.get("inserted", 0)) > 0, + state=str(result.get("state", "pending")), + ) + + def pending_block_candidate_headers( + self, + *, + limit: int = HEADER_PAGE_MAX_ROWS, + after_cursor: object | None = None, + max_bytes: int = HEADER_PAGE_MAX_BYTES, + ) -> CandidateHeaderPage: + """Metadata-first pending page (#255): typed headers, explicit exhaustion. + + Same keyset, order, cursor precision and read-slot gating as + ``pending_block_candidate_rows``; what differs is what one row + carries (a fixed projection of the replay header plus the body + reference, never the candidate) and what the page proves: the + statement fetches one row beyond the row cap, so ``exhausted`` is + answered by the outbox, and a page cut short by the byte cap is + reported as such rather than mistaken for the end. + """ + limit = max(1, min(int(limit), HEADER_PAGE_MAX_ROWS)) + cursor_parts: tuple[str, str] | None = None + if after_cursor is not None: + created_at_text, cursor_block_hash = _block_candidate_cursor_parts(after_cursor) + if not isinstance(created_at_text, str): + raise ValueError("pending block candidate cursor has no creation stamp") + cursor_parts = (created_at_text, cursor_block_hash) + sql = header_page_sql(limit, cursor_parts, max_bytes, text=self._text_literal) + read_gate = getattr(self, "_read_semaphore", None) + if read_gate is None: + # A ledger built without __init__ (focused tests) has no read + # slot; the statement itself is what those tests observe. + result = self._run_retry_safe_read_json(sql) + else: + result = self._run_attributed_read_json( + sql, + operation="pending_block_candidate_headers", + gate=read_gate, + gate_name="read slot", + ) + if not isinstance(result, dict) or not isinstance(result.get("rows"), list): + raise RuntimeError("pending block candidate header page is malformed") + rows: list[dict[str, Any]] = [] + for row in result["rows"]: + if not isinstance(row, dict) or "pool_block_exists" not in row: + raise RuntimeError( + "pending block candidate header row is missing pool block existence" + ) + row["pool_block_exists"] = bool(row["pool_block_exists"]) + row["storage_version"] = int(row.get("storage_version") or 1) + rows.append(row) + fetched = int(result.get("fetched", len(rows))) + returned = int(result.get("returned", len(rows))) + truncated_by_bytes = returned < min(fetched, limit) + return CandidateHeaderPage( + rows=tuple(rows), + next_cursor=rows[-1]["cursor"] if rows else after_cursor, + exhausted=fetched <= limit and not truncated_by_bytes, + fetched=fetched, + truncated_by_bytes=truncated_by_bytes, + ) + + def hydrate_block_candidate_intent( + self, + row: dict[str, Any], + *, + cancelled: Callable[[], bool] | None = None, + ) -> PreparedCandidateIntent: + """Materialize one header row's body to a spool and return the intent. + + Storage version 2 reads bounded chunk pages through the read slot, + verifying manifest state, referencing outbox row and every chunk + digest as it goes. Storage version 1 delegates the unavoidable + whole-document decode to the isolated compatibility helper, which + writes the spool itself. Neither path holds the writer gate. + """ + self._ensure_candidate_store_state() + block_hash = str(row["block_hash"]).lower() + header = row.get("header") if isinstance(row.get("header"), dict) else {} + pending_share = header.get("pending_share") if isinstance(header, dict) else None + accepted_at_present = bool(header.get("accepted_at_present")) if isinstance(header, dict) else False + accepted_at_ms = ( + pending_share.get("accepted_at_ms") + if isinstance(pending_share, dict) + else None + ) + storage_version = int(row.get("storage_version") or 1) + with self._operation_gate(self._candidate_body_gate, "candidate body slot"): + if storage_version == CANDIDATE_BODY_STORAGE_VERSION: + body_row = row.get("body") + if not isinstance(body_row, dict): + raise CandidateBodyUnavailable( + f"block candidate {block_hash} carries no body reference" + ) + ref = CandidateBodyRef.from_row(body_row) + hydrator = CandidateBodyHydrator( + read_manifest=self.read_candidate_body_manifest, + read_spans=self.read_candidate_body_spans, + read_pages=self.read_candidate_body_pages, + read_page=self.read_candidate_body_page, + admission=self._candidate_spool, + cancelled=cancelled, + ) + return hydrator.hydrate( + ref, + block_hash=block_hash, + accepted_at_present=accepted_at_present, + accepted_at_ms=accepted_at_ms, + ) + if storage_version != LEGACY_CANDIDATE_STORAGE_VERSION: + raise CandidateStorageError( + f"unsupported candidate storage version {storage_version}" + ) + helper = self._legacy_candidate_helper + if cancelled is not None: + helper = helper.with_cancellation(cancelled) + admission = self._candidate_spool.snapshot() + available = admission["limit_bytes"] - admission["reserved_bytes"] + if available <= 0: + raise CandidateStorageError("legacy candidate spool reservation exhausted") + release = self._candidate_spool.reserve(available) + paths: tuple[str, ...] = () + body = None + try: + paths = self._candidate_spool.new_spool_paths(block_hash) + path, index_path = paths + converted = helper.convert(block_hash, path, index_path, spool_limit_bytes=available) + actual_bytes = converted.manifest.byte_count + converted.manifest.page_count * 16 + release() + release = self._candidate_spool.reserve(actual_bytes) + index = SpoolFieldIndex.open_written(index_path, converted.spans) + if not converted.identity_matches_row: + print( + "prism ledger: legacy block candidate identity re-encoded with a " + f"different digest hash={block_hash} row={converted.row_candidate_sha256} " + f"body={converted.manifest.candidate_sha256}", + flush=True, + ) + body = SpoolCandidateBody(path, converted.manifest, index, release=release) + return prepared_intent_from_spool( + body, + accepted_at_present=converted.accepted_at_present, + accepted_at_ms=converted.accepted_at_ms, + ) + except BaseException: + if body is not None: + body.close() + else: + for path in paths: + try: + os.unlink(path) + except FileNotFoundError: + pass + release() + raise + + def retire_orphan_candidate_bodies( + self, + *, + stale_staging_seconds: float = STALE_STAGING_SECONDS, + ) -> tuple[str, ...]: + """Tiny fenced compare-and-swap: one orphaned body becomes ``retired``. + + Under the writer fence so a deposed session retires nothing, but it + touches one manifest row and no chunk. ``retired`` is permanent and + publication references only sealed bodies, so nothing revives it. + """ + result = self._run_fenced_json( + retire_orphan_bodies_sql( + { + **self._writer_identity_payload(), + "stale_staging_seconds": float(stale_staging_seconds), + }, + jsonb=self._jsonb_literal, + ) + ) + if not isinstance(result, dict): + raise RuntimeError("candidate body orphan retirement returned no result") + if "error" in result: + raise RuntimeError(str(result["error"])) + return tuple(str(value) for value in result.get("retired", ())) + + def reap_retired_candidate_chunks( + self, + *, + max_chunks: int = JANITOR_CHUNKS_PER_STEP, + ) -> dict[str, Any]: + """One lease-free bounded deletion of a retired body's chunks. + + No lease row is read or written: reclaiming a permanently retired + body's bytes needs no fence, and a deposed session doing it harms + nobody. Takes the body gate so it never competes with a staging or + hydration for the same connection budget. + """ + self._ensure_candidate_store_state() + with self._operation_gate(self._candidate_body_gate, "candidate body slot"): + result = self._run_json( + reap_retired_chunks_sql( + {"max_chunks": max(1, int(max_chunks))}, + jsonb=self._jsonb_literal, + ) + ) + if not isinstance(result, dict): + raise RuntimeError("candidate body janitor returned no result") + remaining = result.get("remaining") if isinstance(result.get("remaining"), dict) else {} + return { + "body_id": result.get("body_id"), + "deleted_chunks": int(result.get("deleted_chunks", 0)), + "deleted_pages": int(result.get("deleted_pages", 0)), + "deleted_spans": int(result.get("deleted_spans", 0)), + "deleted_bodies": int(result.get("deleted_bodies", 0)), + # True while this body still has work for a later step. + "pending": result.get("body_id") is not None + and int(result.get("deleted_bodies", 0)) == 0, + "remaining": { + "chunks": bool(remaining.get("chunks")), + "pages": bool(remaining.get("pages")), + "spans": bool(remaining.get("spans")), + }, + } + + def reap_retired_candidate_bodies( + self, + *, + max_chunks: int = JANITOR_CHUNKS_PER_STEP, + stale_staging_seconds: float = STALE_STAGING_SECONDS, + ) -> dict[str, Any]: + """One janitor step: retire one orphan (fenced), then reap one page. + + Two statements by design (#255 review): the fenced transition is a + single-row compare-and-swap, and the body-sized deletion that + follows never holds the lease row. + """ + retired = self.retire_orphan_candidate_bodies( + stale_staging_seconds=stale_staging_seconds + ) + outcome = self.reap_retired_candidate_chunks(max_chunks=max_chunks) + outcome["retired"] = retired + return outcome + + def candidate_spool_snapshot(self) -> dict[str, int]: + self._ensure_candidate_store_state() + return self._candidate_spool.snapshot() + + def _persist_block_candidate_intent_v1( + self, + candidate: dict[str, Any], + ) -> BlockCandidateIntentPersistResult: + """The historical whole-jsonb standalone persist (storage version 1).""" + block_hash = str(candidate.get("block_hash_hex", "")).lower() + if not block_hash: + raise ValueError("block candidate is missing block_hash_hex") + candidate = {**candidate, "block_hash_hex": block_hash} + candidate_sha256 = block_candidate_identity_sha256(candidate) + sql = f""" +WITH durability AS ( + SELECT set_config('synchronous_commit', 'on', true) +), +lease AS ( + UPDATE qbit_ledger_writer_lease + SET lease_expires_at = clock_timestamp() + {self._lease_interval_sql}, + updated_at = clock_timestamp() + FROM durability + WHERE singleton + AND writer_id = {self._text_literal(self._writer_id)} + AND writer_epoch = {int(self._writer_epoch)} + AND writer_session_token = {self._text_literal(self._writer_session_token)} + RETURNING writer_id +), +existing AS ( + SELECT candidate_sha256, state + FROM qbit_block_candidate_outbox + WHERE block_hash = {self._text_literal(block_hash)} +), +inserted AS ( + INSERT INTO qbit_block_candidate_outbox ( + block_hash, share_id, candidate, candidate_sha256 + ) + SELECT + {self._text_literal(block_hash)}, NULL, + {self._jsonb_literal(candidate)}, {self._text_literal(candidate_sha256)} + FROM lease + WHERE NOT EXISTS (SELECT 1 FROM existing) + ON CONFLICT (block_hash) DO NOTHING + RETURNING block_hash +) +SELECT CASE + WHEN NOT EXISTS (SELECT 1 FROM lease) THEN + json_build_object('error', 'writer lease is not active') + WHEN EXISTS ( + SELECT 1 FROM existing + WHERE candidate_sha256 <> {self._text_literal(candidate_sha256)} + ) THEN + json_build_object('error', 'block candidate payload mismatch') + ELSE + json_build_object( + 'inserted', (SELECT count(*) FROM inserted), + 'state', COALESCE((SELECT state FROM existing), 'pending') + ) +END; +""" + result = self._run_fenced_json(sql) + if "error" in result: + raise RuntimeError(str(result["error"])) + return BlockCandidateIntentPersistResult( + inserted=int(result.get("inserted", 0)) > 0, + state=str(result.get("state", "pending")), + ) + + def pending_block_candidates(self, *, limit: int = 32) -> list[dict[str, Any]]: + return [ + row["candidate"] + for row in self.pending_block_candidate_rows(limit=limit) + ] + + def pending_block_candidate_rows( + self, + *, + limit: int = 32, + after_cursor: object | None = None, + ) -> list[dict[str, Any]]: + """Compatibility view with owned lazy payloads, never aggregate JSON. + + New replay uses header descriptors directly. Callers of this older + interface still receive its four fields; each payload is hydrated + separately through the same bounded reader. Fill byte-truncated header + pages until the requested row limit or explicit exhaustion. + """ + if (getattr(self._run_json, "__func__", None) is not PsqlShareLedger._run_json + or getattr(self._run_sql, "__func__", None) is not PsqlShareLedger._run_sql): + return self._legacy_pending_block_candidate_rows(limit=limit, after_cursor=after_cursor) + limit = max(1, min(int(limit), HEADER_PAGE_MAX_ROWS)) + rows: list[dict[str, Any]] = [] + cursor = after_cursor + while len(rows) < limit: + page = self.pending_block_candidate_headers(limit=limit - len(rows), after_cursor=cursor) + for row in page.rows: + rows.append({ + "block_hash": row["block_hash"], + "candidate": self.hydrate_block_candidate_intent(row), + "pool_block_exists": row["pool_block_exists"], + "cursor": row["cursor"], + }) + if page.exhausted: + break + if not page.rows or page.next_cursor == cursor: + raise CandidateStorageError("candidate header enumeration did not advance") + cursor = page.next_cursor + return rows + + def _legacy_pending_block_candidate_rows( + self, + *, + limit: int = 32, + after_cursor: object | None = None, + ) -> list[dict[str, Any]]: + """Return pending payloads together with their authoritative row keys. + + Rows carry an opaque ``cursor`` the caller passes back verbatim as + ``after_cursor`` to resume strictly after that row, so a backlog + larger than one window enumerates completely in bounded pages + instead of forcing an ever-wider single query. The keyset predicate + and the ordering both stay on ``(created_at, block_hash)``, which is + exactly the partial index + ``qbit_block_candidate_outbox_pending_idx``: every page is one + bounded index range scan regardless of how far in the backlog it + starts. + + The cursor stamp is rendered at microsecond precision with an + explicit UTC marker because ``created_at`` is a ``timestamptz`` whose + stored resolution is microseconds: a second-precision stamp (the + format the public API endpoints use) would truncate, and the + resulting predicate would re-emit or skip whole sub-second groups. + + Each row also carries ``pool_block_exists``: whether a durable + ``qbit_pool_blocks`` row exists for that hash, i.e. whether the + candidate ever reached ``persist_accepted_block``. It is answered + inside this page read -- one bounded existence probe per returned row + -- because the alternative is one round trip per row, and a page is + read precisely when the backlog is large. The fact is advisory by the + time the caller holds it; the terminal batch update re-checks it under + the writer fence. + + **Gate.** This is one read-only statement, and it takes the bounded + read slot rather than the global writer lock (issue #211). Waiting for + writer admission bought this page nothing: the query already crosses + no in-process state, and every fact it returns is advisory the instant + the statement commits -- a candidate can land, or be terminalized by + another path, between the snapshot and anything the caller does with + it. What holding the lock did buy was a convoy. During accepted-block + accounting the enumeration spent most or all of its bounded budget + queued behind an unrelated long write, and on ``union-mainnet`` that + cost the fast-call budget outright while PostgreSQL was idle: the + outer call reported ``exceeded 5s`` with only ~0.4-1.3s of the inner + statement deadline consumed, and accepted candidates converged in + 79-186s with ``qbit_prism_accepted_parent_unresolved_oldest_seconds`` + reaching ~101s. + + Nothing downstream weakens as a result, because nothing downstream + ever trusted this snapshot: + + * ``mark_block_candidates_abandoned`` re-asks ``qbit_pool_blocks`` + *inside* the fenced ``UPDATE``, under the writer-id/epoch/ + session-token lease predicate, and returns the exact hash set it + transitioned -- so a row that acquired a pool block after this read + is silently absent instead of abandoned. + * ``mark_block_candidate_attempted`` and ``_finish_block_candidate`` + carry the same lease fence and additionally require + ``state = 'pending'``, so a row another path terminalized between + the snapshot and the write transitions nobody. + * The node-offer path re-reads ``pool_block_state`` and the chain at + dequeue rather than reusing a page fact + (``_skip_superseded_block_candidate_at_dequeue``). + + Capacity is unchanged too: the read semaphore admits + ``read_concurrency`` callers and the pooled client holds + ``read_concurrency + 1`` connections, so moving this statement from + the writer-lock class to the read-slot class still cannot demand more + connections than the pool has, and it opens no connection and starts + no thread of its own. + """ + if limit <= 0: + return [] + after_predicate = "" + if after_cursor is not None: + created_at_text, cursor_block_hash = _block_candidate_cursor_parts( + after_cursor + ) + if not isinstance(created_at_text, str): + raise ValueError( + "pending block candidate cursor has no creation stamp" + ) + after_predicate = ( + "\n AND (created_at, block_hash) > " + f"({self._text_literal(created_at_text)}::timestamptz, " + f"{self._text_literal(cursor_block_hash)})" + ) + sql = f""" +SELECT COALESCE( + json_agg( + json_build_object( + 'block_hash', pending.block_hash, + 'candidate', pending.candidate, + 'storage_version', pending.storage_version, + 'candidate_sha256', pending.candidate_sha256, + 'pool_block_exists', EXISTS ( + SELECT 1 + FROM qbit_pool_blocks pool + WHERE pool.block_hash = pending.block_hash + ), + 'cursor', json_build_array( + pending.cursor_created_at, + pending.block_hash + ) + ) + ORDER BY pending.created_at, pending.block_hash + ), + '[]'::json +) +FROM ( + SELECT + candidate, + storage_version, + candidate_sha256, + created_at, + to_char( + created_at AT TIME ZONE 'UTC', + 'YYYY-MM-DD"T"HH24:MI:SS.US"Z"' + ) AS cursor_created_at, + block_hash + FROM qbit_block_candidate_outbox + WHERE state = 'pending'{after_predicate} + ORDER BY created_at, block_hash + LIMIT {int(limit)} +) pending; +""" + rows = list( + self._run_attributed_read_json( + sql, + operation="pending_block_candidate_rows", + gate=self._read_semaphore, + gate_name="read slot", + ) + ) + # A row whose existence fact did not arrive is not a row with no pool + # block: reading a missing key as false would hand a caller a false + # negative on the one fact that keeps an offered, landed candidate out + # of a terminal set. Fail the page instead. + for row in rows: + if not isinstance(row, dict) or "pool_block_exists" not in row: + raise RuntimeError( + "pending block candidate row is missing pool block existence" + ) + row["pool_block_exists"] = bool(row["pool_block_exists"]) + return rows + + def block_candidate_pending_metrics(self) -> dict[str, int | float]: + """Return aggregate pending ages using the existing outbox index.""" + sql = """ +SELECT json_build_object( + 'pending_count', count(*), + 'oldest_pending_age_seconds', COALESCE( + EXTRACT(EPOCH FROM clock_timestamp() - min(created_at)), + 0 + ), + 'oldest_unattempted_age_seconds', COALESCE( + EXTRACT( + EPOCH FROM clock_timestamp() + - min(created_at) FILTER (WHERE attempt_count = 0) + ), + 0 + ) +) +FROM qbit_block_candidate_outbox +WHERE state = 'pending'; +""" + with self._operation_gate(self._lock, "writer lock"): + metrics = self._run_retry_safe_read_json(sql) + return { + "pending_count": int(metrics.get("pending_count", 0)), + "oldest_pending_age_seconds": float( + metrics.get("oldest_pending_age_seconds", 0.0) + ), + "oldest_unattempted_age_seconds": float( + metrics.get("oldest_unattempted_age_seconds", 0.0) + ), + } + + def mark_block_candidate_attempted(self, *, block_hash: str) -> bool: + """Fence and count entry into a candidate processing attempt.""" + sql = f""" +WITH lease AS ( + UPDATE qbit_ledger_writer_lease + SET lease_expires_at = clock_timestamp() + {self._lease_interval_sql}, + updated_at = clock_timestamp() + WHERE singleton + AND writer_id = {self._text_literal(self._writer_id)} + AND writer_epoch = {int(self._writer_epoch)} + AND writer_session_token = {self._text_literal(self._writer_session_token)} + RETURNING writer_id +), +updated AS ( + UPDATE qbit_block_candidate_outbox + SET attempt_count = attempt_count + 1, + updated_at = clock_timestamp() + FROM lease + WHERE block_hash = {self._text_literal(block_hash.lower())} + AND state = 'pending' + RETURNING block_hash +) +SELECT CASE + WHEN NOT EXISTS (SELECT 1 FROM lease) THEN + json_build_object('error', 'writer lease is not active') + ELSE + json_build_object('updated', (SELECT count(*) FROM updated)) +END; +""" + result = self._run_fenced_json(sql) + if "error" in result: + raise RuntimeError(str(result["error"])) + return int(result.get("updated", 0)) > 0 + + def mark_block_candidate_submitted(self, *, block_hash: str) -> bool: + return self._finish_block_candidate(block_hash=block_hash, state="submitted", error=None) + + def mark_block_candidate_abandoned(self, *, block_hash: str, error: str) -> bool: + return self._finish_block_candidate(block_hash=block_hash, state="abandoned", error=error) + + def mark_block_candidates_abandoned( + self, + *, + block_hashes: Sequence[str], + error: str, + ) -> tuple[str, ...]: + """Abandon a caller-supplied page of pending rows in one fenced write. + + Same writer-id/epoch/session-token fence as the single-hash terminal + updates, and the same terminal column set, but the row predicate is + set-oriented: one ``block_hash = ANY(...)`` statement for the whole + page rather than one statement per hash. At the storm cardinalities + this exists for, the per-row form is the cost. + + ``RETURNING`` feeds the result, so the value is the exact set of + hashes this fenced statement transitioned -- not a count, and not the + requested set. Rows that were already terminal, that another writer + won, or that do not exist are silently absent from it, which is what + lets the caller confine follow-up cleanup to rows it actually won. + An empty request performs no query at all, so no degenerate + ``ANY(ARRAY[])`` statement is ever generated. + + The predicate re-checks ``qbit_pool_blocks`` rather than trusting the + ``pool_block_exists`` the caller read on a prior page: a candidate can + land between that read and this write, and a pool-block row is the + durable evidence that it did. Re-asking under the writer fence closes + that window inside the statement that does the transition, so a row + that acquired one is silently absent from the returned set instead of + being abandoned after it was won. + """ + targets = _normalized_block_candidate_hash_set(block_hashes) + if not targets: + return () + sql = f""" +WITH lease AS ( + UPDATE qbit_ledger_writer_lease + SET lease_expires_at = clock_timestamp() + {self._lease_interval_sql}, + updated_at = clock_timestamp() + WHERE singleton + AND writer_id = {self._text_literal(self._writer_id)} + AND writer_epoch = {int(self._writer_epoch)} + AND writer_session_token = {self._text_literal(self._writer_session_token)} + RETURNING writer_id +), +updated AS ( + UPDATE qbit_block_candidate_outbox + SET state = 'abandoned', + last_error = {self._text_literal(error)}, + updated_at = clock_timestamp(), + completed_at = clock_timestamp(), + candidate = NULL, + body_id = NULL, + retired_body_id = COALESCE(qbit_block_candidate_outbox.body_id, retired_body_id) + FROM lease + WHERE block_hash = ANY({self._text_array_literal(targets)}) + AND state = 'pending' + AND NOT EXISTS ( + SELECT 1 + FROM qbit_pool_blocks pool + WHERE pool.block_hash = qbit_block_candidate_outbox.block_hash + ) + RETURNING block_hash, retired_body_id +), +retired_bodies AS ( + -- Detach only: the chunks are reclaimed later by the bounded janitor. + UPDATE qbit_block_candidate_body + SET state = 'retired', retired_at = clock_timestamp() + WHERE body_id IN (SELECT retired_body_id FROM updated WHERE retired_body_id IS NOT NULL) + AND state <> 'retired' + RETURNING body_id +) +SELECT CASE + WHEN NOT EXISTS (SELECT 1 FROM lease) THEN + json_build_object('error', 'writer lease is not active') + ELSE + json_build_object( + 'abandoned', + COALESCE( + (SELECT json_agg(block_hash ORDER BY block_hash) FROM updated), + '[]'::json + ) + ) +END; +""" + result = self._run_fenced_json(sql) + if "error" in result: + raise RuntimeError(str(result["error"])) + return tuple(str(value) for value in result.get("abandoned", ())) + + def _finish_block_candidate(self, *, block_hash: str, state: str, error: str | None) -> bool: + if state not in {"submitted", "abandoned"}: + raise ValueError("invalid block candidate terminal state") + sql = f""" +WITH lease AS ( + UPDATE qbit_ledger_writer_lease + SET lease_expires_at = clock_timestamp() + {self._lease_interval_sql}, + updated_at = clock_timestamp() + WHERE singleton + AND writer_id = {self._text_literal(self._writer_id)} + AND writer_epoch = {int(self._writer_epoch)} + AND writer_session_token = {self._text_literal(self._writer_session_token)} + RETURNING writer_id +), +updated AS ( + UPDATE qbit_block_candidate_outbox + SET state = {self._text_literal(state)}, + last_error = {self._text_literal(error) if error is not None else 'NULL'}, + updated_at = clock_timestamp(), + completed_at = clock_timestamp(), + candidate = NULL, + body_id = NULL, + retired_body_id = COALESCE(qbit_block_candidate_outbox.body_id, retired_body_id) + FROM lease + WHERE block_hash = {self._text_literal(block_hash.lower())} + AND state = 'pending' + RETURNING block_hash, retired_body_id +), +retired_bodies AS ( + -- Detach only: the chunks are reclaimed later by the bounded janitor. + UPDATE qbit_block_candidate_body + SET state = 'retired', retired_at = clock_timestamp() + WHERE body_id IN (SELECT retired_body_id FROM updated WHERE retired_body_id IS NOT NULL) + AND state <> 'retired' + RETURNING body_id +) +SELECT CASE + WHEN NOT EXISTS (SELECT 1 FROM lease) THEN + json_build_object('error', 'writer lease is not active') + ELSE + json_build_object('updated', (SELECT count(*) FROM updated)) +END; +""" + result = self._run_fenced_json(sql) + if "error" in result: + raise RuntimeError(str(result["error"])) + return int(result.get("updated", 0)) > 0 + + def snapshot_at_job_issue( + self, + anchor_job_issued_at_ms: int, + *, + window_weight: int | None = None, + ) -> list[AcceptedShareRecord]: + anchor = ( + f"to_timestamp(({int(anchor_job_issued_at_ms)}::double precision / 1000.0))" + ) if window_weight is None: # Whole accepted history up to the anchor. Kept for callers that # want the full ledger (tools/tests); the coordinator passes a @@ -3028,31 +6394,33 @@ def snapshot_at_job_issue( FROM ranked WHERE cumulative_difficulty - share_difficulty < {int(window_weight)}::numeric )""" - sql = rows_cte + """ -SELECT COALESCE(json_agg(json_build_object( - 'share_seq', share_seq, - 'share_id', share_id, - 'miner_id', miner_id, - 'order_key', payout_order_key, - 'p2mr_program_hex', encode(p2mr_program, 'hex'), - 'share_difficulty', share_difficulty::text, - 'network_difficulty', network_difficulty::text, - 'template_height', template_height, - 'job_id', job_id, - 'job_issued_at_ms', round(extract(epoch FROM job_issued_at) * 1000)::bigint, - 'accepted_at_ms', round(extract(epoch FROM accepted_at) * 1000)::bigint, - 'ntime', ntime, - 'credit_policy', credit_policy -) ORDER BY share_seq ASC), '[]'::json) -FROM rows; + # One JSON object per row, in the same final order the json_agg + # projection imposed (#236). The selection above is untouched: only + # the outer projection changed, so the window's membership, the + # weighted crossing row and the ascending share_seq order are the + # same rows, decoded in bounded batches instead of one call. + sql = rows_cte + f""" +SELECT {_ACCEPTED_SHARE_JSON_ROW_SQL} +FROM rows +ORDER BY share_seq ASC; """ # Job construction is a retry-safe MVCC read. Use the independent read # pool so an accepted block's fenced bulk write cannot stall replacement # work behind the single-writer connection lock. - return [ - self._record_from_json(item) - for item in self._run_read_json(sql) - ] + # + # Attributed (#224) as ``payout_window_snapshot``: this fold is the + # payout-window read an accepted preview waits behind, and the alert + # could not say whether a slow one was queued for a read slot or slow + # inside PostgreSQL. The slot taken is the same slot ``_run_read_json`` + # takes, in the same order, under the same deadline; only the + # bookkeeping is new. + return self._run_attributed_read_json_rows( + sql, + operation="payout_window_snapshot", + gate=self._read_semaphore, + gate_name="read slot", + row_converter=self._record_from_json, + ) def snapshot_between_job_issues( self, @@ -3093,27 +6461,22 @@ def snapshot_between_job_issues( AND ledger.job_issued_at <= {anchor} AND ledger.accepted_at <= {previous_anchor} ) -SELECT COALESCE(json_agg(json_build_object( - 'share_seq', share_seq, - 'share_id', share_id, - 'miner_id', miner_id, - 'order_key', payout_order_key, - 'p2mr_program_hex', encode(p2mr_program, 'hex'), - 'share_difficulty', share_difficulty::text, - 'network_difficulty', network_difficulty::text, - 'template_height', template_height, - 'job_id', job_id, - 'job_issued_at_ms', round(extract(epoch FROM job_issued_at) * 1000)::bigint, - 'accepted_at_ms', round(extract(epoch FROM accepted_at) * 1000)::bigint, - 'ntime', ntime, - 'credit_policy', credit_policy -) ORDER BY share_seq ASC), '[]'::json) -FROM rows; +SELECT {_ACCEPTED_SHARE_JSON_ROW_SQL} +FROM rows +ORDER BY share_seq ASC; """ - return [ - self._record_from_json(item) - for item in self._run_read_json(sql) - ] + # Attributed (#224) as ``payout_window_delta``, apart from the whole + # snapshot above: an incremental advance and a full fold are different + # amounts of work, and folding them into one series would hide which + # of the two a landing actually paid for. Same read slot as before, + # and the same per-row projection as the snapshot (#236). + return self._run_attributed_read_json_rows( + sql, + operation="payout_window_delta", + gate=self._read_semaphore, + gate_name="read slot", + row_converter=self._record_from_json, + ) def all_shares(self) -> list[AcceptedShareRecord]: sql = """ @@ -3346,6 +6709,49 @@ def current_owed_balances(self) -> list[dict[str, object]]: balance["balance_sats"] = int(balance["balance_sats"]) return balances + def dashboard_readiness_probe(self) -> bool: + """Cheapest possible proof that Postgres is reachable through a read slot. + + The extracted public read tier's /healthz needs to answer "can I still + reach the database" without running any aggregate: a health check that + scans the share ledger turns a liveness probe into load, and the + compose healthcheck runs it on an interval forever. This is a constant + select -- it touches no PRISM table -- taken through the same bounded + read slot as every other public read, so a saturated read pool shows up + as an unhealthy service rather than as a probe that jumps the queue. + """ + payload = self._run_read_json("SELECT json_build_object('ok', true);") + return bool(isinstance(payload, dict) and payload.get("ok")) + + def dashboard_miner_owed_balance_bits(self, *, recipient_id: str) -> int: + """One recipient's owed balance, read without the writer lock. + + The same total as summing current_owed_balances() for this recipient, + but filtered in SQL and taken through the bounded read slot instead of + the writer gate. current_owed_balances() returns every recipient's + balance under the writer lock, so serving the most-polled public route + (/public/v1/miners/{recipient_id}) from it made a dashboard poll + serialize against the lease-holding writer. The predicates match that + method exactly -- rows from qbit_current_owed_balances() with + owed_balance_sats > 0, restricted to this miner_id -- so the integer + returned here is the integer the Python sum produced. + """ + if not recipient_id: + raise ValueError("recipient_id is required") + sql = f""" +SELECT json_build_object( + 'owed_balance_bits', + COALESCE(( + SELECT sum(owed_balance_sats) + FROM qbit_current_owed_balances() + WHERE owed_balance_sats > 0 + AND miner_id = {self._text_literal(recipient_id)} + ), 0)::text +); +""" + payload = self._run_read_json(sql) + return int(payload["owed_balance_bits"]) + def prior_balances_after_pool_block( self, *, @@ -3386,7 +6792,17 @@ def prior_balances_after_pool_block( ) ORDER BY payout_order_key, miner_id, encode(p2mr_program, 'hex')), '[]'::json) FROM balances; """ - balances = list(self._run_read_json(sql)) + # Attributed (#224). Still the read slot: this is a retry-safe MVCC + # read of already-confirmed rows, and the audit path that asks for it + # never needed writer serialization. + balances = list( + self._run_attributed_read_json( + sql, + operation="prior_balances_after_pool_block", + gate=self._read_semaphore, + gate_name="read slot", + ) + ) for balance in balances: balance["balance_sats"] = int(balance["balance_sats"]) return balances @@ -3401,9 +6817,28 @@ def current_prior_balances(self) -> list[dict[str, object]]: ) ORDER BY payout_order_key, miner_id, encode(p2mr_program, 'hex')), '[]'::json) FROM qbit_current_carry_forward_balances(); """ + # Attributed (#224) without changing the primitive. This reread stays + # on the *writer lock*, which is where the landing's prior-balances + # check has always taken it: it must observe the fenced state a payout + # mutation leaves behind, so it serializes against that mutation + # rather than running beside it on a read slot. The attribution is the + # only thing added, and it is exactly the split the alert lacked -- + # whether a slow reread was queued behind a landing or slow inside + # PostgreSQL. + # + # The legacy aggregate below is left as it was: one duration covering + # admission and execution together, recorded only on success. + # ``qbit_prism_prior_balances_*`` is an existing operator-facing + # series and a reader comparing it across this change must not find + # its meaning quietly redefined, so it duplicates part of the new + # per-operation sample on purpose. started = time.monotonic() - with self._operation_gate(self._lock, "writer lock"): - balances = self._run_retry_safe_read_json(sql) + balances = self._run_attributed_read_json( + sql, + operation="current_prior_balances", + gate=self._lock, + gate_name="writer lock", + ) self._note_prior_balances_read(max(0.0, time.monotonic() - started)) for balance in balances: balance["balance_sats"] = int(balance["balance_sats"]) @@ -3431,6 +6866,12 @@ def carry_forward_integrity_report(self) -> dict[str, object]: sql = "SELECT qbit_carry_forward_integrity_report();" with self._operation_gate(self._lock, "writer lock"): report = self._run_retry_safe_read_json(sql) + # The audit head is a second statement under the same held lock: + # the two reads must observe one another's rows, so the gate + # cannot be released between them. Report the first statement's + # completed round trip so the pair costs a monitor one budget of + # silence at a time (see _note_operation_progress). + self._note_operation_progress() audit_head = self._carry_forward_audit_head_locked() report["backend"] = "postgres-psql" report.update(audit_head) @@ -3489,6 +6930,28 @@ def _carry_forward_audit_head_locked(self) -> dict[str, object]: "audit_head_sha256": previous.hex() if rows else "00" * 32, } + def candidate_window_covers( + self, + shares: Iterable[Any], + *, + anchor_job_issued_at_ms: int, + network_difficulty: int, + ) -> bool: + """Check replay omissions without returning or retaining a full window. + + This coordinator-only read uses connection-private temporary storage; + it is deliberately unavailable to the enforced read-only API ledger. + """ + from lab.prism.candidate_window import postgres_window_covers + + if self._read_only: + raise ReadOnlyLedgerError("candidate window checks require temporary storage") + return postgres_window_covers( + self, shares, + anchor_job_issued_at_ms=anchor_job_issued_at_ms, + network_difficulty=network_difficulty, + ) + def audit_share_window( self, *, @@ -4068,7 +7531,57 @@ def dashboard_miner_worker_rows( "active_count": int(payload["active_count"]), } - def audit_bundle(self, *, block_hash: str) -> dict[str, object] | None: + def audit_bundle(self, *, block_hash: str) -> dict[str, object] | None: + sql = f""" +SELECT COALESCE( + ( + SELECT json_build_object( + 'block_hash', bundle.block_hash, + 'block_height', block.block_height, + 'payout_manifest_sha256', block.payout_manifest_sha256, + 'audit_bundle_sha256', bundle.audit_bundle_sha256, + 'coinbase_tx_hex', bundle.coinbase_tx_hex, + 'audit_bundle', bundle.audit_bundle, + 'body_uri', bundle.body_uri + ) + FROM qbit_pool_audit_bundles bundle + JOIN qbit_pool_blocks block + ON block.block_hash = bundle.block_hash + WHERE bundle.block_hash = {self._text_literal(block_hash)} + ), + 'null'::json +); +""" + with self._operation_gate(self._lock, "writer lock"): + row = self._run_retry_safe_read_json(sql) + return self._resolve_audit_bundle_row(row) + + def dashboard_direct_coinbase_settlement( + self, + *, + block_hash: str, + ) -> dict[str, object] | None: + """Public settlement facts for a direct-coinbase block, without fencing. + + /public/v1/blocks/{block_hash}/settlement-artifacts serves CTV blocks + from audit_ctv_fanout_manifest_set() (a read slot), but a + direct-coinbase block fell through to audit_bundle(), which reads the + same row under the writer lock. This is that read taken through the + bounded read slot instead, returning exactly the dict + public_api.direct_coinbase_settlement_payload() builds today. + + The row is resolved through _resolve_audit_bundle_row, so an + externalized body is read from body_uri on disk and sha256-verified + against audit_bundle_sha256 by the same helper the fenced path uses. + A body that is unretrievable, hash-mismatched, or not valid JSON is + reported as None rather than raised, matching what + public_api.audit_bundle_body_read_failed() already tolerates. + + Returns None when the block has no audit bundle, or when the bundle + settled in any mode other than direct_coinbase. + """ + from lab.prism import public_api + sql = f""" SELECT COALESCE( ( @@ -4077,7 +7590,6 @@ def audit_bundle(self, *, block_hash: str) -> dict[str, object] | None: 'block_height', block.block_height, 'payout_manifest_sha256', block.payout_manifest_sha256, 'audit_bundle_sha256', bundle.audit_bundle_sha256, - 'coinbase_tx_hex', bundle.coinbase_tx_hex, 'audit_bundle', bundle.audit_bundle, 'body_uri', bundle.body_uri ) @@ -4089,9 +7601,45 @@ def audit_bundle(self, *, block_hash: str) -> dict[str, object] | None: 'null'::json ); """ - with self._operation_gate(self._lock, "writer lock"): - row = self._run_retry_safe_read_json(sql) - return self._resolve_audit_bundle_row(row) + row = self._run_read_json(sql) + try: + resolved = self._resolve_audit_bundle_row(row) + except RuntimeError as exc: + if public_api.audit_bundle_body_read_failed(exc): + return None + raise + if not isinstance(resolved, dict): + return None + bundle = resolved.get("audit_bundle") + if not isinstance(bundle, dict): + return None + if public_api.audit_bundle_settlement_mode(bundle) != "direct_coinbase": + return None + return { + "block_hash": public_api.optional_hex_hash(resolved.get("block_hash")) + or block_hash, + "block_height": public_api.first_int( + resolved.get("block_height"), + public_api.audit_bundle_section_value( + bundle, "found_block", "block_height" + ), + public_api.audit_bundle_section_value( + bundle, "reward_manifest", "block_height" + ), + public_api.audit_bundle_section_value( + bundle, "ledger_window_attestation", "block_height" + ), + default=0, + ), + "settlement_mode": "direct_coinbase", + "audit_bundle_sha256": public_api.nullable_str( + resolved.get("audit_bundle_sha256") + ), + "payout_manifest_sha256": public_api.nullable_str( + resolved.get("payout_manifest_sha256") + ), + "artifacts": [], + } def audit_bundle_by_commitment(self, *, commitment_leaf_hex: str) -> dict[str, object] | None: leaf = self._text_literal(commitment_leaf_hex) @@ -4128,7 +7676,7 @@ def persist_ctv_fanout_manifest_set( self, *, block_hash: str, - manifest_set: dict[str, Any], + manifest_set: Mapping[str, Any], manifest_set_sha256: str, ) -> dict[str, int | str]: payload = { @@ -4141,11 +7689,7 @@ def persist_ctv_fanout_manifest_set( "writer_epoch": self._writer_epoch, "writer_session_token": self._writer_session_token, } - sql = f""" -WITH payload AS ( - SELECT {self._jsonb_literal(payload)} AS data -), -lease AS ( + sql_body = f"""lease AS ( UPDATE qbit_ledger_writer_lease SET lease_expires_at = clock_timestamp() + {self._lease_interval_sql}, updated_at = clock_timestamp() @@ -4342,7 +7886,7 @@ def persist_ctv_fanout_manifest_set( ) END; """ - result = self._run_fenced_json(sql) + result = self._run_candidate_payload_json(payload, sql_body) if "error" in result: raise RuntimeError(str(result["error"])) return { @@ -4587,7 +8131,7 @@ def dashboard_pending_fanout_rows(self, *, page: int, limit: int) -> dict[str, o 'error', attempt.error ) ORDER BY attempt.attempt_seq ASC) FROM qbit_ctv_fanout_broadcast_attempts attempt - WHERE attempt.fanout_txid = fanout_txid + WHERE attempt.fanout_txid = page_rows.fanout_txid ), '[]'::json ) @@ -4603,36 +8147,63 @@ def dashboard_pending_fanout_rows(self, *, page: int, limit: int) -> dict[str, o } def dashboard_public_artifact(self, *, sha256: str) -> dict[str, object] | None: + document = self.dashboard_public_artifact_document(sha256=sha256) + if document is None: + return None + return document.get("payload") + + def dashboard_public_artifact_document(self, *, sha256: str) -> dict[str, object] | None: + """Artifact payload plus its stored canonical text when available. + + canonical_json is the manifest_set_json/manifest_json text persisted + at record time next to the JSONB copies — the exact byte sequence the + artifact's sha256 was computed over. Audit bundles use the immutable + compressed canonical artifact beside the external body. One statement + serves both database reads so a request touches each artifact table at + most once. + + Replica ordering is intentionally simple. A visible row plus a missing + canonical file is legacy history and falls back to the JSONB/external + body. A file that reached shared storage before this replica replays its + row remains invisible and returns no match until replay catches up. + """ sha256 = str(sha256).lower() lit = self._text_literal(sha256) sql = f""" WITH audit AS ( - SELECT audit_bundle, audit_bundle_sha256, body_uri + SELECT block_hash, audit_bundle, audit_bundle_sha256, body_uri FROM qbit_pool_audit_bundles WHERE audit_bundle_sha256 = {lit} ORDER BY created_at DESC LIMIT 1 +), +set_row AS ( + SELECT manifest_set, manifest_set_json + FROM qbit_ctv_fanout_sets + WHERE manifest_set_sha256 = {lit} + ORDER BY created_at DESC + LIMIT 1 +), +artifact_row AS ( + SELECT manifest, manifest_json + FROM qbit_ctv_fanout_artifacts + WHERE manifest_sha256 = {lit} + ORDER BY updated_at DESC + LIMIT 1 ) SELECT json_build_object( + 'block_hash', (SELECT block_hash FROM audit), 'audit_bundle', (SELECT audit_bundle FROM audit), 'audit_bundle_sha256', (SELECT audit_bundle_sha256 FROM audit), 'body_uri', (SELECT body_uri FROM audit), 'has_audit_row', (SELECT count(*) FROM audit) > 0, 'fallback', COALESCE( - ( - SELECT manifest_set - FROM qbit_ctv_fanout_sets - WHERE manifest_set_sha256 = {lit} - ORDER BY created_at DESC - LIMIT 1 - ), - ( - SELECT manifest - FROM qbit_ctv_fanout_artifacts - WHERE manifest_sha256 = {lit} - ORDER BY updated_at DESC - LIMIT 1 - ) + (SELECT manifest_set FROM set_row), + (SELECT manifest FROM artifact_row) + ), + 'fallback_canonical', COALESCE( + (SELECT manifest_set_json FROM set_row), + (SELECT manifest_json FROM artifact_row) ) ); """ @@ -4640,20 +8211,64 @@ def dashboard_public_artifact(self, *, sha256: str) -> dict[str, object] | None: if not isinstance(row, dict): return None if row.get("has_audit_row"): + block_hash = row.get("block_hash") + if isinstance(block_hash, str): + canonical_bytes = self._stored_canonical_audit_bundle_bytes( + block_hash, + sha256, + ) + if canonical_bytes is not None: + try: + canonical_payload = json.loads(canonical_bytes) + canonical_json = canonical_bytes.decode("utf-8") + except (UnicodeDecodeError, json.JSONDecodeError, TypeError) as exc: + LOGGER.error( + "canonical audit bundle unavailable reason=corrupt " + "block_hash=%s audit_bundle_sha256=%s", + block_hash, + sha256, + ) + raise CanonicalAuditBundleCorrupt( + "canonical audit bundle is not valid UTF-8 JSON" + ) from exc + if not isinstance(canonical_payload, dict): + LOGGER.error( + "canonical audit bundle unavailable reason=corrupt " + "block_hash=%s audit_bundle_sha256=%s", + block_hash, + sha256, + ) + raise CanonicalAuditBundleCorrupt( + "canonical audit bundle JSON is not an object" + ) + return { + "payload": canonical_payload, + "canonical_json": canonical_json, + } body = row.get("audit_bundle") if body is None: body = self._read_external_body(row.get("body_uri"), expected_sha256=sha256) if body is not None: - return body + return { + "payload": body, + "canonical_json": None, + "canonical_fallback_reason": "missing", + } fallback = row.get("fallback") - return fallback if isinstance(fallback, dict) else None + if not isinstance(fallback, dict): + return None + fallback_canonical = row.get("fallback_canonical") + return { + "payload": fallback, + "canonical_json": fallback_canonical if isinstance(fallback_canonical, str) else None, + } def dashboard_public_artifact_exists(self, *, sha256: str) -> bool: sha256 = str(sha256).lower() lit = self._text_literal(sha256) sql = f""" WITH audit AS ( - SELECT audit_bundle, body_uri + SELECT block_hash, audit_bundle, audit_bundle_sha256, body_uri FROM qbit_pool_audit_bundles WHERE audit_bundle_sha256 = {lit} ORDER BY created_at DESC @@ -4661,6 +8276,8 @@ def dashboard_public_artifact_exists(self, *, sha256: str) -> bool: ) SELECT json_build_object( 'has_audit_row', (SELECT count(*) FROM audit) > 0, + 'block_hash', (SELECT block_hash FROM audit), + 'audit_bundle_sha256', (SELECT audit_bundle_sha256 FROM audit), 'audit_bundle_inline', (SELECT audit_bundle IS NOT NULL FROM audit), 'body_uri', (SELECT body_uri FROM audit), 'fallback_exists', @@ -4680,6 +8297,13 @@ def dashboard_public_artifact_exists(self, *, sha256: str) -> bool: if not isinstance(row, dict): return False if row.get("has_audit_row"): + block_hash = row.get("block_hash") + if ( + isinstance(block_hash, str) + and self._stored_canonical_audit_bundle_bytes(block_hash, sha256) + is not None + ): + return True if row.get("audit_bundle_inline"): return True body_uri = row.get("body_uri") @@ -5000,6 +8624,8 @@ def dashboard_pool_snapshot( 'participants_3h', (SELECT participants_3h FROM rollups), 'blocks_found_total', (SELECT count(*) FROM qbit_pool_blocks WHERE chain_state <> 'reversed'), 'prism_blocks_total', (SELECT count(*) FROM qbit_pool_blocks WHERE chain_state <> 'reversed'), + 'blocks_reversed_total', (SELECT count(*) FROM qbit_pool_blocks WHERE chain_state = 'reversed'), + 'blocks_inactive_total', (SELECT count(*) FROM qbit_pool_blocks WHERE chain_state = 'inactive'), 'total_mined_bits', COALESCE(( SELECT sum(carry.gross_amount_sats) FROM qbit_payout_carry_forward carry @@ -5035,6 +8661,8 @@ def dashboard_pool_snapshot( "participants_3h": int(row["participants_3h"]), "blocks_found_total": int(row["blocks_found_total"]), "prism_blocks_total": int(row["prism_blocks_total"]), + "blocks_reversed_total": int(row["blocks_reversed_total"]), + "blocks_inactive_total": int(row["blocks_inactive_total"]), "total_mined_bits": int(row["total_mined_bits"]), "latest_block": row["latest_block"], "reward_window": { @@ -5046,25 +8674,53 @@ def dashboard_pool_snapshot( }, } - def dashboard_blocks(self, *, page: int, limit: int) -> dict[str, object]: + def dashboard_blocks(self, *, page: int, limit: int, chain_state: str = "active") -> dict[str, object]: from lab.prism import public_api + # The default filter is exactly the pre-filter read (hide reversed + # rows), and the default emits exactly the pre-filter SQL and row + # shape so the v1 response stays byte-compatible. The non-default + # filters additionally surface chain_state and disconnected_at, which + # the public v2 block row reports. + state_predicates = { + "active": "{column} <> 'reversed'", + "all": "true", + "reversed": "{column} = 'reversed'", + } + if chain_state not in state_predicates: + raise ValueError("chain_state must be one of active, all, reversed") + total_predicate = state_predicates[chain_state].format(column="chain_state") + page_predicate = state_predicates[chain_state].format(column="block.chain_state") + include_state = chain_state != "active" + state_columns = "\n block.chain_state,\n block.disconnected_at," if include_state else "" + # disconnected_at is a reorg disconnect time in the public contract, + # null for every non-reversed row. The ledger also stamps the column + # on rejected rows (their maturity_state 'reversed' requires it), so + # it is masked to reversed chain states rather than serialized as + # recorded. + state_json_fields = ( + "\n 'chain_state', rows.chain_state," + "\n 'disconnected_at', CASE WHEN rows.chain_state = 'reversed'" + " THEN to_char(rows.disconnected_at AT TIME ZONE 'UTC', 'YYYY-MM-DD\"T\"HH24:MI:SS\"Z\"') END," + if include_state + else "" + ) offset = (page - 1) * limit explorer_prefix = os.environ.get("PRISM_PUBLIC_EXPLORER_BLOCK_URL_PREFIX") sql = f""" WITH total AS ( SELECT count(*) AS total_count FROM qbit_pool_blocks - WHERE chain_state <> 'reversed' + WHERE {total_predicate} ), page_blocks AS ( SELECT block.block_hash, block.block_height, - block.found_at, + block.found_at,{state_columns} block.payout_manifest_sha256 FROM qbit_pool_blocks block - WHERE block.chain_state <> 'reversed' + WHERE {page_predicate} ORDER BY block.block_height DESC, block.found_at DESC LIMIT {int(limit)} OFFSET {int(offset)} ), @@ -5072,7 +8728,7 @@ def dashboard_blocks(self, *, page: int, limit: int) -> dict[str, object]: SELECT block.block_hash, block.block_height, - block.found_at, + block.found_at,{state_columns} block.payout_manifest_sha256, COALESCE(bundle.found_block_network_difficulty::text, bundle.audit_bundle#>>'{{found_block,network_difficulty}}') AS audit_network_difficulty, COALESCE(bundle.found_block_bits, bundle.audit_bundle#>>'{{found_block,bits}}') AS audit_bits, @@ -5113,7 +8769,7 @@ def dashboard_blocks(self, *, page: int, limit: int) -> dict[str, object]: END, 'coinbase_value_bits', COALESCE(rows.audit_coinbase_value_sats::bigint, 0), 'audit_bundle_sha256', rows.audit_bundle_sha256, - 'payout_manifest_sha256', rows.payout_manifest_sha256, + 'payout_manifest_sha256', rows.payout_manifest_sha256,{state_json_fields} 'explorer_url', null ) ORDER BY rows.block_height DESC, rows.found_at DESC) FROM rows @@ -5464,6 +9120,37 @@ def dashboard_hashrate_series( lookback_seconds: int = 0, range_anchor_epoch: int | None = None, ) -> list[dict[str, object]]: + """Serve the per-bucket credited hashrate series in O(buckets). + + The series is answered from the incremental rollup tables plus a + live tail over the shares the maintenance watermark has not folded + in yet, so its cost tracks the number of buckets in range rather + than the number of shares. The rollup path must stay byte-identical + to the historical raw ``qbit_share_ledger`` aggregation for the same + underlying shares -- same buckets, counts, difficulty strings, and + ordering -- which pins three invariants: + + - The watermark, the rollup rows, and the tail scan are read inside + one statement, so they share one snapshot. Reading the watermark + in a separate statement would let a concurrent maintenance pass + advance it in between and double-count the shares it folded in. + - A range lower bound that is not bucket-aligned makes the raw scan + emit a partial leading bucket (only the shares at or after the + bound), and the bucket containing the statement's upper bound can + hold a rollup-folded share whose coordinator-assigned accepted_at + runs ahead of the database clock -- a share the raw scan's + ``accepted_at <= ended_at`` excludes until the clocks catch up. + Neither partial end can be served from full rollup buckets, so + both are re-aggregated from the ledger over the already-rolled-up + sequence range; each scan is bounded by one bucket span of shares + regardless of the requested range. + - An absent progress row gates the rollup and boundary branches off + and widens the tail to the whole ledger (``share_seq > -1``), so + the merged output degrades to exactly the raw scan; a database + missing the rollup tables entirely runs the raw statement + unchanged. Both keep pre-migration and mid-backfill states + correct without operator action. + """ from lab.prism import public_api bucket_seconds = {"5m": 300, "1h": 3600, "1d": 86400}[bucket] @@ -5475,6 +9162,7 @@ def dashboard_hashrate_series( "all": None, }[range_id] range_filter = "" + range_start_sql: str | None = None if range_interval is not None: # Anchor the range lower bound on the caller's clock when provided # so it agrees with the caller's min_epoch trim even when the @@ -5484,15 +9172,17 @@ def dashboard_hashrate_series( if range_anchor_epoch is not None else "bounds.ended_at" ) - range_filter = f"AND ledger.accepted_at >= {range_anchor_sql} - interval '{range_interval}'" + range_start_sql = f"{range_anchor_sql} - interval '{range_interval}'" if lookback_seconds > 0: # Pre-range context requested by the smoother; the caller trims # these buckets from the response after windowing. - range_filter += f" - interval '{int(lookback_seconds)} seconds'" + range_start_sql += f" - interval '{int(lookback_seconds)} seconds'" + range_filter = f"AND ledger.accepted_at >= {range_start_sql}" subject_filter = "" if subject_type == "miner": subject_filter = f"AND ledger.miner_id = {self._text_literal(str(subject_id))}" - sql = f""" + if not self._hashrate_rollup_schema_present(): + sql = f""" WITH bounds AS ( SELECT clock_timestamp() AS ended_at ), @@ -5514,6 +9204,137 @@ def dashboard_hashrate_series( 'accepted_share_difficulty', accepted_share_difficulty::text ) ORDER BY bucket_epoch ASC), '[]'::json) FROM bucketed; +""" + rows = self._run_read_json(sql) + return [ + { + "timestamp": row["timestamp"], + "hashrate_ths": public_api.hashrate_ths_from_difficulty(row["accepted_share_difficulty"], bucket_seconds), + "accepted_share_count": int(row["accepted_share_count"]), + "accepted_share_difficulty": str(row["accepted_share_difficulty"]), + } + for row in rows + ] + if subject_type == "miner": + rollup_table = "qbit_hashrate_rollup_miner" + rollup_subject_filter = ( + f"\n AND rollup.miner_id = {self._text_literal(str(subject_id))}" + ) + else: + rollup_table = "qbit_hashrate_rollup_pool" + rollup_subject_filter = "" + # The bucket containing bounds.ended_at is never served from its + # rollup row: maintenance folds by sequence, so a coordinator clock + # running ahead of the database clock can put a share into that + # bucket's rollup that the raw scan's accepted_at <= ended_at would + # exclude. Like the partial lower-bound bucket, it is re-aggregated + # from the already-rolled-up sequence range -- one bucket span of + # shares, whatever the requested range. + if range_start_sql is not None: + range_buckets_cte = f"""range_buckets AS ( + SELECT + range_bounds.range_started_at, + (ceil(extract(epoch FROM range_bounds.range_started_at) / {int(bucket_seconds)}))::bigint * {int(bucket_seconds)} AS first_full_bucket_epoch + FROM (SELECT {range_start_sql} AS range_started_at FROM bounds) range_bounds +), +""" + rollup_lower_bound = ( + "\n AND rollup.bucket_epoch >= (SELECT first_full_bucket_epoch FROM range_buckets)" + ) + boundary_window = """( + ledger.accepted_at < to_timestamp((SELECT first_full_bucket_epoch FROM range_buckets)) + OR ledger.accepted_at >= to_timestamp((SELECT bucket_epoch FROM current_bucket)) + )""" + boundary_range_filter = ( + "\n AND ledger.accepted_at >= (SELECT range_started_at FROM range_buckets)" + ) + tail_range_filter = ( + "\n AND ledger.accepted_at >= (SELECT range_started_at FROM range_buckets)" + ) + else: + range_buckets_cte = "" + rollup_lower_bound = "" + boundary_window = ( + "ledger.accepted_at >= to_timestamp((SELECT bucket_epoch FROM current_bucket))" + ) + boundary_range_filter = "" + tail_range_filter = "" + boundary_cte = f"""boundary AS ( + SELECT + floor(extract(epoch FROM ledger.accepted_at) / {int(bucket_seconds)})::bigint * {int(bucket_seconds)} AS bucket_epoch, + count(*) AS accepted_share_count, + sum(ledger.share_difficulty) AS accepted_share_difficulty + FROM qbit_share_ledger ledger, bounds + WHERE (SELECT rollups_ready FROM watermark) + AND ledger.accepted + AND ledger.share_seq <= (SELECT last_share_seq FROM watermark) + AND ledger.accepted_at <= bounds.ended_at + AND {boundary_window}{boundary_range_filter} + {subject_filter} + GROUP BY bucket_epoch +), +""" + boundary_union = """ UNION ALL + SELECT bucket_epoch, accepted_share_count, accepted_share_difficulty FROM boundary +""" + sql = f""" +WITH bounds AS ( + SELECT clock_timestamp() AS ended_at +), +progress AS ( + SELECT last_share_seq + FROM qbit_hashrate_rollup_progress + WHERE singleton +), +watermark AS ( + SELECT + COALESCE((SELECT last_share_seq FROM progress), -1) AS last_share_seq, + EXISTS (SELECT 1 FROM progress) AS rollups_ready +), +current_bucket AS ( + SELECT floor(extract(epoch FROM bounds.ended_at) / {int(bucket_seconds)})::bigint * {int(bucket_seconds)} AS bucket_epoch + FROM bounds +), +{range_buckets_cte}rolled AS ( + SELECT + rollup.bucket_epoch, + rollup.accepted_share_count, + rollup.accepted_share_difficulty + FROM {rollup_table} rollup, bounds + WHERE (SELECT rollups_ready FROM watermark) + AND rollup.grain_seconds = {int(bucket_seconds)} + AND rollup.bucket_epoch < (SELECT bucket_epoch FROM current_bucket){rollup_lower_bound}{rollup_subject_filter} +), +{boundary_cte}tail AS ( + SELECT + floor(extract(epoch FROM ledger.accepted_at) / {int(bucket_seconds)})::bigint * {int(bucket_seconds)} AS bucket_epoch, + count(*) AS accepted_share_count, + sum(ledger.share_difficulty) AS accepted_share_difficulty + FROM qbit_share_ledger ledger, bounds + WHERE ledger.accepted + AND ledger.share_seq > (SELECT last_share_seq FROM watermark) + AND ledger.accepted_at <= bounds.ended_at{tail_range_filter} + {subject_filter} + GROUP BY bucket_epoch +), +merged AS ( + SELECT + parts.bucket_epoch, + sum(parts.accepted_share_count)::bigint AS accepted_share_count, + sum(parts.accepted_share_difficulty) AS accepted_share_difficulty + FROM ( + SELECT bucket_epoch, accepted_share_count, accepted_share_difficulty FROM rolled +{boundary_union} UNION ALL + SELECT bucket_epoch, accepted_share_count, accepted_share_difficulty FROM tail + ) parts + GROUP BY parts.bucket_epoch +) +SELECT COALESCE(json_agg(json_build_object( + 'timestamp', to_char(to_timestamp(bucket_epoch) AT TIME ZONE 'UTC', 'YYYY-MM-DD"T"HH24:MI:SS"Z"'), + 'accepted_share_count', accepted_share_count, + 'accepted_share_difficulty', accepted_share_difficulty::text +) ORDER BY bucket_epoch ASC), '[]'::json) +FROM merged; """ rows = self._run_read_json(sql) return [ @@ -5526,443 +9347,413 @@ def dashboard_hashrate_series( for row in rows ] - def _externalize_audit_body( - self, - block_hash: str, - audit_bundle_sha256: str, - final_bundle: dict[str, Any], - ) -> str | None: - """Write the audit-bundle body to the external store and return its path. - - Returns None when no body store is configured, in which case the caller - keeps the body inline in Postgres (legacy behavior). Externalizing the - body is what stops the per-block audit_bundle JSONB from growing with the - full accepted-share history. + def _hashrate_rollup_schema_present(self) -> bool: + """Report whether the incremental hashrate rollup tables exist. + + The serving statement cannot even parse against a database that + predates the rollup DDL, so table existence has to be settled before + choosing a statement. A positive answer is cached for the life of + this instance: the rollup tables are only ever created (by the + writer's idempotent schema apply), never dropped. A negative answer + is deliberately not cached, so a read tier pointed at a database + whose writer applies the DDL mid-flight picks up the rollup path + without a restart. """ - if self._audit_body_dir is None: - return None - block_hash = canonical_hex(block_hash, name="block_hash", expected_bytes=32) - audit_bundle_sha256 = canonical_hex( - str(audit_bundle_sha256), - name="audit_bundle_sha256", - expected_bytes=32, + if getattr(self, "_hashrate_rollup_schema_ready", False): + return True + row = self._run_read_json( + """ +SELECT json_build_object( + 'rollup_schema_ready', + to_regclass('qbit_hashrate_rollup_progress') IS NOT NULL + AND to_regclass('qbit_hashrate_rollup_pool') IS NOT NULL + AND to_regclass('qbit_hashrate_rollup_miner') IS NOT NULL +); +""" ) - body_bytes = self._canonical_audit_body_bytes_for_sha(final_bundle, audit_bundle_sha256) - return self._write_external_audit_body(block_hash, audit_bundle_sha256, body_bytes) - - def _canonical_audit_body_bytes_for_sha( - self, - final_bundle: dict[str, Any], - audit_bundle_sha256: str, - ) -> bytes: - body_bytes = self._canonical_audit_bundle_bytes(final_bundle) - actual_sha256 = sha256_bytes_hex(body_bytes) - if actual_sha256 != str(audit_bundle_sha256).lower(): + ready = bool(row["rollup_schema_ready"]) + if ready: + self._hashrate_rollup_schema_ready = True + return ready + + def advance_hashrate_rollups(self, *, batch_limit: int) -> dict[str, object]: + """Fold the next watermarked share batch into the hashrate rollups. + + One statement, one transaction. ``qbit_share_ledger`` rows are + immutable and ``share_seq`` is append-only, so scanning strictly + above the stored watermark in sequence order folds every share into + its (grain, bucket) rows exactly once -- a late-clocked share still + lands in its correct ``accepted_at`` bucket, and no re-aggregation + window is needed. Rejected rows are read only to advance the + watermark. The first pass seeds the progress row itself and starts + from sequence 0, which is also how a grown ledger backfills. + + The watermark advance is a guarded upsert: it only applies while the + progress row still holds the value this statement read, and the + rollup upserts are gated on that advance having won. A concurrent + advance therefore leaves this pass writing nothing at all -- the + guarded upsert matches no row and both rollup inserts see an empty + gate -- and the caller raises instead of double-counting, because + two live maintenance passes mean the single-writer invariant is + already broken. + + The whole pass is additionally fenced on the writer lease, exactly + like every other mutation of ledger-derived state: the statement + renews the exact ``(writer_id, writer_epoch, writer_session_token)`` + lease tuple and the advance applies only when that renewal matched, + so a coordinator whose lease expired or was taken over cannot keep + mutating the rollups on the strength of its process-local lock + alone. A fenced-out pass writes nothing and raises. + """ + batch_limit = int(batch_limit) + if batch_limit <= 0: + raise ValueError("hashrate rollup batch limit must be positive") + sql = f""" +WITH lease AS ( + UPDATE qbit_ledger_writer_lease + SET lease_expires_at = clock_timestamp() + {self._lease_interval_sql}, + updated_at = clock_timestamp() + WHERE singleton + AND writer_id = {self._text_literal(self._writer_id)} + AND writer_epoch = {int(self._writer_epoch)} + AND writer_session_token = {self._text_literal(self._writer_session_token)} + RETURNING writer_id +), +progress AS ( + SELECT COALESCE(( + SELECT last_share_seq + FROM qbit_hashrate_rollup_progress + WHERE singleton + ), 0) AS last_share_seq +), +batch AS ( + SELECT + ledger.share_seq, + ledger.accepted, + ledger.accepted_at, + ledger.miner_id, + ledger.share_difficulty + FROM qbit_share_ledger ledger + WHERE ledger.share_seq > (SELECT last_share_seq FROM progress) + ORDER BY ledger.share_seq ASC + LIMIT {batch_limit} +), +batch_stats AS ( + SELECT + count(*) AS scanned, + COALESCE(max(batch.share_seq), (SELECT last_share_seq FROM progress)) AS next_share_seq + FROM batch +), +advance AS ( + INSERT INTO qbit_hashrate_rollup_progress (singleton, last_share_seq) + SELECT true, (SELECT next_share_seq FROM batch_stats) + WHERE EXISTS (SELECT 1 FROM lease) + ON CONFLICT (singleton) DO UPDATE + SET last_share_seq = EXCLUDED.last_share_seq, + updated_at = clock_timestamp() + WHERE qbit_hashrate_rollup_progress.last_share_seq = (SELECT last_share_seq FROM progress) + RETURNING last_share_seq +), +grains AS ( + SELECT grain_seconds + FROM (VALUES (300), (3600), (86400)) AS grain(grain_seconds) +), +pool_rollup AS ( + INSERT INTO qbit_hashrate_rollup_pool ( + grain_seconds, + bucket_epoch, + accepted_share_count, + accepted_share_difficulty + ) + SELECT + grains.grain_seconds, + floor(extract(epoch FROM batch.accepted_at) / grains.grain_seconds)::bigint * grains.grain_seconds AS bucket_epoch, + count(*) AS accepted_share_count, + sum(batch.share_difficulty) AS accepted_share_difficulty + FROM batch, grains + WHERE batch.accepted + AND EXISTS (SELECT 1 FROM advance) + GROUP BY grains.grain_seconds, bucket_epoch + ON CONFLICT (grain_seconds, bucket_epoch) DO UPDATE + SET accepted_share_count = qbit_hashrate_rollup_pool.accepted_share_count + + EXCLUDED.accepted_share_count, + accepted_share_difficulty = qbit_hashrate_rollup_pool.accepted_share_difficulty + + EXCLUDED.accepted_share_difficulty + RETURNING 1 +), +miner_rollup AS ( + INSERT INTO qbit_hashrate_rollup_miner ( + grain_seconds, + bucket_epoch, + miner_id, + accepted_share_count, + accepted_share_difficulty + ) + SELECT + grains.grain_seconds, + floor(extract(epoch FROM batch.accepted_at) / grains.grain_seconds)::bigint * grains.grain_seconds AS bucket_epoch, + batch.miner_id, + count(*) AS accepted_share_count, + sum(batch.share_difficulty) AS accepted_share_difficulty + FROM batch, grains + WHERE batch.accepted + AND EXISTS (SELECT 1 FROM advance) + GROUP BY grains.grain_seconds, bucket_epoch, batch.miner_id + ON CONFLICT (grain_seconds, bucket_epoch, miner_id) DO UPDATE + SET accepted_share_count = qbit_hashrate_rollup_miner.accepted_share_count + + EXCLUDED.accepted_share_count, + accepted_share_difficulty = qbit_hashrate_rollup_miner.accepted_share_difficulty + + EXCLUDED.accepted_share_difficulty + RETURNING 1 +) +SELECT CASE + WHEN NOT EXISTS (SELECT 1 FROM lease) THEN + json_build_object('error', 'writer lease is not active') + ELSE + json_build_object( + 'scanned', (SELECT scanned FROM batch_stats), + 'last_share_seq', (SELECT next_share_seq FROM batch_stats), + 'advanced', (SELECT count(*) FROM advance), + 'caught_up', (SELECT scanned FROM batch_stats) < {batch_limit} + ) +END; +""" + result = self._run_fenced_json(sql) + if "error" in result: + raise RuntimeError(str(result["error"])) + if int(result["advanced"]) != 1: raise RuntimeError( - "audit bundle sha256 mismatch: " - f"expected {str(audit_bundle_sha256).lower()}, got {actual_sha256}" + "hashrate rollup watermark advanced concurrently; " + "this pass wrote nothing (writer-fencing bug)" ) - return body_bytes - - def _external_audit_storage_bytes( - self, - *, - block_hash: str, - audit_bundle_sha256: str, - final_bundle: dict[str, Any], - canonical_body_bytes: bytes, - ) -> bytes: - v2_bundle = self._audit_bundle_v2( - block_hash=block_hash, - audit_bundle_sha256=audit_bundle_sha256, - final_bundle=final_bundle, - ) - if v2_bundle is not None: - return self._storage_json_bytes(v2_bundle) - body_ref = self._audit_body_ref( - block_hash=block_hash, - audit_bundle_sha256=audit_bundle_sha256, - final_bundle=final_bundle, - ) - if body_ref is None: - return canonical_body_bytes - return self._storage_json_bytes(body_ref) - - def _audit_body_ref( - self, - *, - block_hash: str, - audit_bundle_sha256: str, - final_bundle: dict[str, Any], - ) -> dict[str, Any] | None: - if self._audit_body_dir is None or self._audit_share_segment_size <= 0: - return None - shares = final_bundle.get("shares") - if not isinstance(shares, list) or not shares: - return None - share_parts = self._audit_share_parts(shares) - if share_parts is None or not any(part.get("kind") == "segment" for part in share_parts): - return None - shares_key_index = list(final_bundle).index("shares") - # Persistence only reads this immutable build result. A shallow outer - # mapping avoids recursively copying both the top-level shares and the - # reward-manifest share window merely to remove one key. - bundle_without_shares = { - key: value for key, value in final_bundle.items() if key != "shares" - } return { - "schema": AUDIT_BODY_REF_SCHEMA, - "block_hash": block_hash, - "audit_bundle_sha256": audit_bundle_sha256, - "audit_bundle_schema": str(final_bundle.get("schema") or ""), - "share_count": len(shares), - "share_segment_size": self._audit_share_segment_size, - "shares_key_index": shares_key_index, - "bundle_without_shares": bundle_without_shares, - "share_parts": share_parts, + "scanned": int(result["scanned"]), + "last_share_seq": int(result["last_share_seq"]), + "caught_up": bool(result["caught_up"]), } - def _audit_bundle_v2( + def dashboard_block_markers( self, *, - block_hash: str, - audit_bundle_sha256: str, - final_bundle: dict[str, Any], - ) -> dict[str, Any] | None: - if self._audit_body_dir is None or self._audit_share_segment_size <= 0: - return None - shares = final_bundle.get("shares") - if not isinstance(shares, list) or not shares: - return None - share_parts = self._audit_share_range_parts(shares) - if share_parts is None: - return None - shares_key_index = list(final_bundle).index("shares") - bundle_without_shares = { - key: value for key, value in final_bundle.items() if key != "shares" - } - reward_manifest = final_bundle.get("reward_manifest") - proof: dict[str, Any] = { - "schema": AUDIT_WINDOW_COMPLETENESS_PROOF_SCHEMA, - "share_segment_size": self._audit_share_segment_size, - "first_share_seq": int(shares[0]["share_seq"]), - "last_share_seq": int(shares[-1]["share_seq"]), - "share_count": len(shares), - "share_parts_digest_hex": sha256_bytes_hex( - self._storage_json_bytes({"share_parts": share_parts}) - ), - "share_parts": share_parts, - } - if isinstance(reward_manifest, dict): - for key in ( - "anchor_job_issued_at_ms", - "anchor_share_seq", - "newest_share_seq", - "oldest_share_seq", - "included_share_count", - "requested_window_weight", - "counted_window_weight", - "share_slice_digest_hex", - ): - if key in reward_manifest: - proof[key] = reward_manifest[key] - return { - "schema": AUDIT_BUNDLE_V2_SCHEMA, - "block_hash": block_hash, - "audit_bundle_sha256": audit_bundle_sha256, - "logical_audit_bundle_schema": str(final_bundle.get("schema") or ""), - "share_count": len(shares), - "shares_key_index": shares_key_index, - "bundle_without_shares": bundle_without_shares, - "share_window_proof": proof, - } - - def _audit_share_parts(self, shares: list[Any]) -> list[dict[str, Any]] | None: - share_seqs: list[int] = [] - for share in shares: - if not isinstance(share, dict): - return None - try: - share_seq = int(share["share_seq"]) - except (KeyError, TypeError, ValueError): - return None - share_seqs.append(share_seq) - if any(current + 1 != nxt for current, nxt in zip(share_seqs, share_seqs[1:])): - return None - - parts: list[dict[str, Any]] = [] - index = 0 - segment_size = self._audit_share_segment_size - while index < len(shares): - first_seq = share_seqs[index] - segment_start = ((first_seq - 1) // segment_size) * segment_size + 1 - segment_end = segment_start + segment_size - 1 - end = index - while end < len(shares) and share_seqs[end] <= segment_end: - end += 1 - chunk = shares[index:end] - chunk_seqs = share_seqs[index:end] - if len(chunk) == segment_size and chunk_seqs[0] == segment_start and chunk_seqs[-1] == segment_end: - segment_uri, segment_sha256 = self._write_audit_share_segment( - first_share_seq=segment_start, - last_share_seq=segment_end, - shares=chunk, - ) - parts.append( - { - "kind": "segment", - "first_share_seq": segment_start, - "last_share_seq": segment_end, - "share_count": len(chunk), - "sha256": segment_sha256, - "body_uri": segment_uri, - } - ) - else: - parts.append( - { - "kind": "inline", - "first_share_seq": chunk_seqs[0], - "last_share_seq": chunk_seqs[-1], - "share_count": len(chunk), - "shares": chunk, - } - ) - index = end - return parts - - def _audit_share_range_parts(self, shares: list[Any]) -> list[dict[str, Any]] | None: - share_seqs: list[int] = [] - for share in shares: - if not isinstance(share, dict): - return None - try: - share_seq = int(share["share_seq"]) - except (KeyError, TypeError, ValueError): - return None - share_seqs.append(share_seq) - if any(current + 1 != nxt for current, nxt in zip(share_seqs, share_seqs[1:])): - return None + range_id: str, + bucket: str, + range_anchor_epoch: int | None = None, + ) -> dict[str, object]: + from lab.prism import public_api - parts: list[dict[str, Any]] = [] - index = 0 - segment_size = self._audit_share_segment_size - while index < len(shares): - first_seq = share_seqs[index] - segment_start = ((first_seq - 1) // segment_size) * segment_size + 1 - segment_end = segment_start + segment_size - 1 - end = index - while end < len(shares) and share_seqs[end] <= segment_end: - end += 1 - chunk = shares[index:end] - chunk_seqs = share_seqs[index:end] - segment_uri, range_sha256 = self._write_audit_share_segment_range( - segment_first_share_seq=segment_start, - segment_last_share_seq=segment_end, - first_share_seq=chunk_seqs[0], - last_share_seq=chunk_seqs[-1], - shares=chunk, + bucket_seconds = {"5m": 300, "1h": 3600, "1d": 86400}[bucket] + range_seconds = { + "1w": 7 * 86400, + "1m": 30 * 86400, + "6m": 180 * 86400, + "all": None, + }[range_id] + range_filter = "" + if range_seconds is not None: + # Anchor the range lower bound on the caller's clock when provided + # so marker buckets sit on exactly the grid the hashrate series is + # trimmed against, even when the database clock disagrees with the + # application clock. Only fully covered buckets are kept: a bucket + # straddling the range start is dropped whole, the same boundary + # hashrate_series_min_epoch draws for the chart's points. `all` + # carries no lower bound. + anchor_epoch = ( + int(range_anchor_epoch) + if range_anchor_epoch is not None + else int(datetime.now(timezone.utc).timestamp()) ) - parts.append( - { - "kind": "segment_range", - "segment_first_share_seq": segment_start, - "segment_last_share_seq": segment_end, - "first_share_seq": chunk_seqs[0], - "last_share_seq": chunk_seqs[-1], - "share_count": len(chunk), - "range_sha256": range_sha256, - "body_uri": segment_uri, - } + min_epoch = public_api.hashrate_series_min_epoch( + anchor_epoch, + range_seconds, + bucket_seconds, ) - index = end - return parts - - def _audit_share_segment_payload(self, *, first_share_seq: int, last_share_seq: int, shares: list[Any]) -> dict[str, Any]: + range_filter = f"AND block.found_at >= to_timestamp({int(min_epoch)})" + # qbit_pool_blocks holds ~10^4 rows, so one grouped scan is cheap and + # no index beyond the primary key is needed. Reversed blocks are + # excluded by the same predicate dashboard_blocks applies. + sql = f""" +WITH in_range AS ( + SELECT + floor(extract(epoch FROM block.found_at) / {int(bucket_seconds)})::bigint * {int(bucket_seconds)} AS bucket_epoch, + block.block_hash, + block.block_height, + block.found_at + FROM qbit_pool_blocks block + WHERE block.chain_state <> 'reversed' + AND block.found_at <= clock_timestamp() + {range_filter} +), +ranked AS ( + SELECT + in_range.*, + row_number() OVER ( + PARTITION BY bucket_epoch + ORDER BY found_at DESC, block_height DESC + ) AS bucket_rank + FROM in_range +), +points AS ( + SELECT + bucket_epoch, + count(*) AS block_count, + json_agg(json_build_object( + 'height', block_height, + 'hash', block_hash, + 'found_at', to_char(found_at AT TIME ZONE 'UTC', 'YYYY-MM-DD"T"HH24:MI:SS"Z"') + ) ORDER BY found_at DESC, block_height DESC) + FILTER (WHERE bucket_rank <= {int(public_api.BLOCK_MARKERS_MAX_BLOCKS_PER_BUCKET)}) AS blocks + FROM ranked + GROUP BY bucket_epoch +) +SELECT json_build_object( + 'total_blocks', COALESCE((SELECT sum(block_count)::bigint FROM points), 0), + 'points', COALESCE(( + SELECT json_agg(json_build_object( + 'timestamp', to_char(to_timestamp(bucket_epoch) AT TIME ZONE 'UTC', 'YYYY-MM-DD"T"HH24:MI:SS"Z"'), + 'block_count', block_count, + 'blocks', blocks + ) ORDER BY bucket_epoch ASC) + FROM points + ), '[]'::json) +); +""" + payload = self._run_read_json(sql) return { - "schema": AUDIT_SHARE_SEGMENT_SCHEMA, - "first_share_seq": first_share_seq, - "last_share_seq": last_share_seq, - "share_count": len(shares), - "shares": shares, + "total_blocks": int(payload["total_blocks"]), + "points": payload["points"], } - def _write_audit_share_segment( - self, - *, - first_share_seq: int, - last_share_seq: int, - shares: list[Any], - ) -> tuple[str, str]: - if self._audit_body_dir is None: + def _audit_store(self) -> AuditArtifactStore: + store = getattr(self, "_audit_artifact_store", None) + if store is None: raise RuntimeError("audit body store is not configured") - segment = self._audit_share_segment_payload( - first_share_seq=first_share_seq, - last_share_seq=last_share_seq, - shares=shares, - ) - segment_bytes = self._storage_json_bytes(segment) - segment_sha256 = sha256_bytes_hex(segment_bytes) - segment_path = self._audit_body_dir.resolve() / ( - f"prism-audit-share-segment-{first_share_seq}-{last_share_seq}-{segment_sha256}.json" - ) - if segment_path.exists(): - if not self._file_matches_bytes(segment_path, segment_bytes): - raise RuntimeError(f"existing audit share segment does not match payload at {segment_path}") - else: - self._write_bytes_atomically(segment_path, segment_bytes) - return str(segment_path), segment_sha256 + return store - def _write_audit_share_segment_range( + def _stored_canonical_audit_bundle_bytes( self, - *, - segment_first_share_seq: int, - segment_last_share_seq: int, - first_share_seq: int, - last_share_seq: int, - shares: list[Any], - ) -> tuple[str, str]: - if self._audit_body_dir is None: - raise RuntimeError("audit body store is not configured") - if not shares: - raise RuntimeError("audit share segment range cannot be empty") - segment_path = self._audit_body_dir.resolve() / ( - f"prism-audit-share-segment-slot-{segment_first_share_seq}-{segment_last_share_seq}.json" - ) - range_payload = self._audit_share_segment_payload( - first_share_seq=first_share_seq, - last_share_seq=last_share_seq, - shares=shares, - ) - # Encode the incoming range once. Completed 10k slots dominate normal - # block persistence; byte equality lets them return without parsing, - # merging, deep-copying, or serializing the existing slot tree. - range_bytes = self._storage_json_bytes(range_payload) - range_sha256 = sha256_bytes_hex(range_bytes) - if segment_path.exists() and self._file_matches_bytes(segment_path, range_bytes): - return str(segment_path), range_sha256 - - merged_shares = shares - existing_bytes: bytes | None = None - if segment_path.exists(): - try: - existing_bytes = segment_path.read_bytes() - existing = json.loads(existing_bytes) - except (UnicodeDecodeError, json.JSONDecodeError) as exc: - raise RuntimeError(f"existing audit share segment is not valid JSON at {segment_path}") from exc - if not isinstance(existing, dict) or existing.get("schema") != AUDIT_SHARE_SEGMENT_SCHEMA: - raise RuntimeError(f"existing audit share segment has invalid schema at {segment_path}") - existing_shares = existing.get("shares") - if not isinstance(existing_shares, list): - raise RuntimeError(f"existing audit share segment has no shares at {segment_path}") - merged_shares = self._merge_audit_share_ranges( - existing_shares, - shares, - segment_path=segment_path, + block_hash: str, + audit_bundle_sha256: str, + ) -> bytes | None: + """Return a verified stored canonical bundle, or permit missing fallback. + + Missing artifacts are expected for history predating issue #158. + Present-but-corrupt or unreadable artifacts fail closed instead of + silently serving a non-byte-exact legacy reconstruction. + """ + store = getattr(self, "_audit_artifact_store", None) + reader = getattr(store, "read_canonical_audit_bundle", None) + if not callable(reader): + LOGGER.warning( + "canonical audit bundle unavailable reason=missing " + "block_hash=%s audit_bundle_sha256=%s", + block_hash, + audit_bundle_sha256, ) - if merged_shares is None: - # Keep prior body_uri references valid at the stable slot path, - # while the new bundle uses an immutable incoming-only segment. - fresh_uri, fresh_sha256 = self._write_audit_share_segment( - first_share_seq=first_share_seq, - last_share_seq=last_share_seq, - shares=shares, - ) - if fresh_sha256 != range_sha256: - raise RuntimeError("audit share segment range hash changed during fallback") - try: - self._quarantine_audit_share_segment( - segment_path, - expected_bytes=existing_bytes, - ) - except OSError: - # The stable slot already preserves the old range, and the - # immutable file preserves the incoming one. A quarantine - # snapshot failure must not wedge block finalization. - pass - return fresh_uri, range_sha256 - segment_first = int(merged_shares[0]["share_seq"]) - segment_last = int(merged_shares[-1]["share_seq"]) - if ( - segment_first == first_share_seq - and segment_last == last_share_seq - and len(merged_shares) == len(shares) - ): - segment_bytes = range_bytes - else: - segment = self._audit_share_segment_payload( - first_share_seq=segment_first, - last_share_seq=segment_last, - shares=merged_shares, + return None + try: + canonical_bytes = reader(block_hash, audit_bundle_sha256) + except CanonicalAuditBundleCorrupt: + LOGGER.error( + "canonical audit bundle unavailable reason=corrupt " + "block_hash=%s audit_bundle_sha256=%s", + block_hash, + audit_bundle_sha256, ) - segment_bytes = self._storage_json_bytes(segment) - if existing_bytes != segment_bytes: - self._write_bytes_atomically(segment_path, segment_bytes) - return str(segment_path), range_sha256 - - def _merge_audit_share_ranges( - self, - existing_shares: list[Any], - incoming_shares: list[Any], - *, - segment_path: Path, - ) -> list[Any] | None: - if not existing_shares: - return list(incoming_shares) - existing_by_seq = self._audit_shares_by_seq(existing_shares, segment_path=segment_path) - incoming_by_seq = self._audit_shares_by_seq(incoming_shares, segment_path=segment_path) - for share_seq, incoming in incoming_by_seq.items(): - existing = existing_by_seq.get(share_seq) - if existing is not None and existing != incoming: - raise _AuditShareSegmentConflict( - f"existing audit share segment conflicts at share_seq {share_seq} in {segment_path}" - ) - merged_by_seq = {**existing_by_seq, **incoming_by_seq} - ordered_seqs = sorted(merged_by_seq) - if any(current + 1 != nxt for current, nxt in zip(ordered_seqs, ordered_seqs[1:])): - # A skipped finalize can leave an otherwise valid slot behind the - # incoming range. Repair that hole from the append-only ledger. - gap_before, gap_after = next( - (current, nxt) - for current, nxt in zip(ordered_seqs, ordered_seqs[1:]) - if current + 1 != nxt + raise + except (OSError, RuntimeError, TypeError, ValueError): + LOGGER.exception( + "canonical audit bundle unavailable reason=read_error " + "block_hash=%s audit_bundle_sha256=%s", + block_hash, + audit_bundle_sha256, ) - missing_first = gap_before + 1 - missing_last = gap_after - 1 - try: - durable_shares = self._load_audit_share_ledger_range( - first_share_seq=missing_first, - last_share_seq=missing_last, - ) - except _AuditShareSegmentConflict: - raise - except Exception: - # This read is a recovery aid; its failure must not turn the - # original slot gap into another permanent finalize failure. - durable_shares = None - if durable_shares is not None: - try: - durable_by_seq = self._audit_shares_by_seq( - durable_shares, - segment_path=segment_path, - require_contiguous=False, - ) - except _AuditShareSegmentConflict: - raise - except RuntimeError: - durable_by_seq = {} - if sorted(durable_by_seq) == list(range(missing_first, missing_last + 1)): - merged_by_seq.update(durable_by_seq) - ordered_seqs = sorted(merged_by_seq) - if not any( - current + 1 != nxt - for current, nxt in zip(ordered_seqs, ordered_seqs[1:]) - ): - return [merged_by_seq[share_seq] for share_seq in ordered_seqs] - # The incoming range is independently complete and remains usable. - # The caller preserves this slot and gives that range a fresh URI. + raise + if canonical_bytes is None: + LOGGER.warning( + "canonical audit bundle unavailable reason=missing " + "block_hash=%s audit_bundle_sha256=%s", + block_hash, + audit_bundle_sha256, + ) + return None + if not isinstance(canonical_bytes, bytes): + LOGGER.error( + "canonical audit bundle unavailable reason=read_error " + "block_hash=%s audit_bundle_sha256=%s", + block_hash, + audit_bundle_sha256, + ) + raise TypeError("canonical audit bundle reader returned non-bytes") + return canonical_bytes + + def _audit_reader(self, body_uri: object) -> AuditArtifactStore: + del body_uri + store = getattr(self, "_audit_artifact_store", None) + if store is not None: + return store + # Compatibility-only dashboard resolvers historically initialized a + # read-only PsqlShareLedger subclass with just `_audit_body_dir`. Keep + # that explicit adapter working while routing every read through A1. + legacy_root = getattr(self, "_audit_body_dir", None) + if legacy_root is not None: + root = Path(legacy_root) + store = AuditArtifactStore( + AuditArtifactConfig( + root=root, + evidence_path=root / "prism-live-stratum-evidence.json", + ), + canonicalizer=( + getattr(self, "_audit_bundle_canonicalizer", None) + or _default_bundle_canonicalizer() + ), + ) + self._audit_artifact_store = store + return store + raise RuntimeError( + "audit bundle body is not retrievable: audit body store is not configured" + ) + + def _externalize_audit_body(self, block_hash: str, audit_bundle_sha256: str, final_bundle: Mapping[str, Any]) -> str | None: + if self._audit_artifact_store is None: return None - return [merged_by_seq[share_seq] for share_seq in ordered_seqs] + return self._audit_store().externalize_audit_body(block_hash, audit_bundle_sha256, final_bundle) + + def _canonical_audit_body_bytes_for_sha(self, final_bundle: Mapping[str, Any], audit_bundle_sha256: str) -> bytes: + return self._audit_store().canonical_audit_body_bytes_for_sha(final_bundle, audit_bundle_sha256) + + def _audit_body_ref(self, **kwargs: Any) -> dict[str, Any] | None: + return self._audit_store().audit_body_ref(**kwargs) + + def _audit_bundle_v2(self, **kwargs: Any) -> dict[str, Any] | None: + kwargs.setdefault("load_missing_range", self._load_audit_share_ledger_range) + return self._audit_store().audit_bundle_v2(**kwargs) + + def _audit_share_parts(self, shares: list[Any]) -> list[dict[str, Any]] | None: + return self._audit_store().audit_share_parts(shares) + + def _audit_share_range_parts(self, shares: list[Any]) -> list[dict[str, Any]] | None: + return self._audit_store().audit_share_range_parts( + shares, + load_missing_range=self._load_audit_share_ledger_range, + ) + + def _audit_share_segment_payload(self, **kwargs: Any) -> dict[str, Any]: + return self._audit_store().audit_share_segment_payload(**kwargs) + + def _write_audit_share_segment(self, **kwargs: Any) -> tuple[str, str]: + return self._audit_store().write_audit_share_segment(**kwargs) + + def _write_audit_share_segment_range(self, **kwargs: Any) -> tuple[str, str]: + kwargs.setdefault("load_missing_range", self._load_audit_share_ledger_range) + return self._audit_store().write_audit_share_segment_range(**kwargs) + + def _merge_audit_share_ranges(self, existing_shares: list[Any], incoming_shares: list[Any], *, segment_path: Path) -> list[Any] | None: + return self._audit_store().merge_audit_share_ranges( + existing_shares, + incoming_shares, + segment_path=segment_path, + load_missing_range=self._load_audit_share_ledger_range, + ) def _load_audit_share_ledger_range( self, @@ -5995,202 +9786,33 @@ def _load_audit_share_ledger_range( raise RuntimeError("audit share ledger backfill did not return a list") return [self._record_from_json(row).to_prism_json() for row in rows] - def _quarantine_audit_share_segment( - self, - segment_path: Path, - *, - expected_bytes: bytes, - ) -> Path: - for existing_path in segment_path.parent.glob(f"{segment_path.name}.conflict-*"): - if self._file_matches_bytes(existing_path, expected_bytes): - return existing_path - timestamp = time.time_ns() - quarantine_path = segment_path.with_name(f"{segment_path.name}.conflict-{timestamp}") - while quarantine_path.exists(): - timestamp += 1 - quarantine_path = segment_path.with_name(f"{segment_path.name}.conflict-{timestamp}") - # Snapshot rather than move: previously finalized bundles retain this - # stable slot URI, so it must remain readable across crashes and retries. - self._write_bytes_atomically(quarantine_path, expected_bytes) - return quarantine_path - - def _audit_shares_by_seq( - self, - shares: list[Any], - *, - segment_path: Path, - require_contiguous: bool = True, - ) -> dict[int, Any]: - by_seq: dict[int, Any] = {} - for share in shares: - if not isinstance(share, dict): - raise RuntimeError(f"audit share segment has invalid share payload at {segment_path}") - try: - share_seq = int(share["share_seq"]) - except (KeyError, TypeError, ValueError) as exc: - raise RuntimeError(f"audit share segment has invalid share_seq at {segment_path}") from exc - existing = by_seq.get(share_seq) - if existing is not None and existing != share: - raise _AuditShareSegmentConflict( - f"audit share segment has duplicate conflicting share_seq {share_seq} at {segment_path}" - ) - by_seq[share_seq] = share - ordered = sorted(by_seq) - if require_contiguous and any(current + 1 != nxt for current, nxt in zip(ordered, ordered[1:])): - raise RuntimeError(f"audit share segment has non-contiguous share_seq values at {segment_path}") - return by_seq + def _audit_shares_by_seq(self, shares: list[Any], *, segment_path: Path, require_contiguous: bool = True) -> dict[int, Any]: + return self._audit_store().audit_shares_by_seq( + shares, + segment_path=segment_path, + require_contiguous=require_contiguous, + ) def _storage_json_bytes(self, payload: dict[str, Any]) -> bytes: - return json.dumps(payload, separators=(",", ":")).encode("utf-8") - - @staticmethod - def _file_matches_bytes(path: Path, expected: bytes) -> bool: - try: - if path.stat().st_size != len(expected): - return False - offset = 0 - view = memoryview(expected) - with path.open("rb") as handle: - while offset < len(expected): - chunk = handle.read(min(1024 * 1024, len(expected) - offset)) - if not chunk or chunk != view[offset : offset + len(chunk)]: - return False - offset += len(chunk) - return not handle.read(1) - except OSError: - return False - - def _write_bytes_atomically(self, path: Path, payload: bytes) -> None: - path.parent.mkdir(parents=True, exist_ok=True) - tmp_path = path.with_name(f".{path.name}.{uuid.uuid4().hex}.tmp") - try: - with tmp_path.open("xb") as handle: - handle.write(payload) - handle.flush() - os.fsync(handle.fileno()) - tmp_path.replace(path) - finally: - try: - tmp_path.unlink() - except FileNotFoundError: - pass - - def _write_json_atomically(self, path: Path, payload: dict[str, Any]) -> None: - """Stream compact JSON through the existing fsync-and-rename boundary.""" - path.parent.mkdir(parents=True, exist_ok=True) - tmp_path = path.with_name(f".{path.name}.{uuid.uuid4().hex}.tmp") - encoder = json.JSONEncoder(separators=(",", ":")) - try: - with tmp_path.open("xb") as handle: - for chunk in encoder.iterencode(payload): - handle.write(chunk.encode("utf-8")) - handle.flush() - os.fsync(handle.fileno()) - tmp_path.replace(path) - finally: - try: - tmp_path.unlink() - except FileNotFoundError: - pass - - def _file_matches_json_payload(self, path: Path, payload: dict[str, Any]) -> bool: - """Compare compact JSON exactly without materializing expected bytes.""" - expected_digest = hashlib.sha256() - expected_length = 0 - encoder = json.JSONEncoder(separators=(",", ":")) - for chunk in encoder.iterencode(payload): - encoded = chunk.encode("utf-8") - expected_digest.update(encoded) - expected_length += len(encoded) - try: - return ( - path.stat().st_size == expected_length - and hmac.compare_digest( - self._file_sha256_hex(path), - expected_digest.hexdigest(), - ) - ) - except OSError: - return False - - def _copy_file_atomically(self, path: Path, source: Path) -> None: - path.parent.mkdir(parents=True, exist_ok=True) - tmp_path = path.with_name(f".{path.name}.{uuid.uuid4().hex}.tmp") - try: - with source.open("rb") as input_handle, tmp_path.open("xb") as output_handle: - while chunk := input_handle.read(1024 * 1024): - output_handle.write(chunk) - output_handle.flush() - os.fsync(output_handle.fileno()) - tmp_path.replace(path) - finally: - try: - tmp_path.unlink() - except FileNotFoundError: - pass + return self._audit_store().storage_json_bytes(payload) @staticmethod def _file_sha256_hex(path: Path) -> str: - digest = hashlib.sha256() - with path.open("rb") as handle: - while chunk := handle.read(1024 * 1024): - digest.update(chunk) - return digest.hexdigest() - - def _write_external_audit_body( - self, - block_hash: str, - audit_bundle_sha256: str, - body_bytes: bytes, - ) -> str | None: - if self._audit_body_dir is None: - return None - self._audit_body_dir.mkdir(parents=True, exist_ok=True) - body_path = self._audit_body_path(block_hash, audit_bundle_sha256) - if body_path.exists(): - existing = body_path.read_bytes() - if existing != body_bytes: - raise RuntimeError(f"existing audit bundle body does not match payload at {body_path}") - return str(body_path) - self._write_bytes_atomically(body_path, body_bytes) - return str(body_path) + return AuditArtifactStore.file_sha256_hex(path) def _canonical_audit_bundle_bytes(self, final_bundle: dict[str, Any]) -> bytes: - if self._audit_bundle_canonicalizer is not None: - canonical = self._audit_bundle_canonicalizer(final_bundle) - return canonical.encode() if isinstance(canonical, str) else bytes(canonical) - completed = subprocess.run( - prism_tool_command("qbit-prism-audit-canonicalize") - + [ - "--input", - "-", - ], - input=json.dumps(final_bundle).encode(), - stdout=subprocess.PIPE, - stderr=subprocess.PIPE, - check=False, - ) - if completed.returncode != 0: - stderr = completed.stderr.decode(errors="replace").strip() - raise RuntimeError(f"qbit-prism-audit-canonicalize failed: {stderr}") - return completed.stdout + if self._audit_artifact_store is None: + return canonical_audit_bundle_bytes( + final_bundle, + self._audit_bundle_canonicalizer, + ) + return self._audit_store().canonical_audit_bundle_bytes(final_bundle) def _audit_body_path(self, block_hash: str, audit_bundle_sha256: str) -> Path: - if self._audit_body_dir is None: - raise RuntimeError("audit body store is not configured") - root = self._audit_body_dir.resolve() - body_path = root / f"prism-audit-bundle-body-{block_hash}-{audit_bundle_sha256}.json" - return self._resolve_audit_body_path(body_path) + return self._audit_store().body_path(block_hash, audit_bundle_sha256) def _resolve_audit_body_path(self, body_uri: object) -> Path: - body_path = Path(str(body_uri)).expanduser().resolve() - if self._audit_body_dir is not None: - root = self._audit_body_dir.resolve() - try: - body_path.relative_to(root) - except ValueError as exc: - raise RuntimeError(f"audit bundle body path escapes audit body store: {body_uri}") from exc - return body_path + return self._audit_store().resolve_owned_path(body_uri) def _external_audit_body_write_plan(self, payload: dict[str, Any]) -> str | None: """Refresh the writer lease and decide whether this persist may write a body. @@ -6200,7 +9822,7 @@ def _external_audit_body_write_plan(self, payload: dict[str, Any]) -> str | None writers from creating artifacts by requiring the DB lease to be current before any filesystem side effect. """ - if self._audit_body_dir is None: + if self._audit_artifact_store is None: return None sql = f""" WITH payload AS ( @@ -6272,450 +9894,213 @@ def _external_audit_body_write_plan(self, payload: dict[str, Any]) -> str | None return None return str(self._audit_body_path(payload["block_hash"], payload["audit_bundle_sha256"])) - def _audit_body_byte_len( - self, - body_uri: object | None, - final_bundle: dict[str, Any], - canonical_bundle_path: Path | None = None, - ) -> int: - if body_uri: - return self._resolve_audit_body_path(body_uri).stat().st_size - if canonical_bundle_path is not None: - return canonical_bundle_path.stat().st_size - return len(self._canonical_audit_bundle_bytes(final_bundle)) - - def _prepare_external_audit_body( + def preflight_canonical_bundle_publication( self, - payload: dict[str, Any], - final_bundle: dict[str, Any], *, - canonical_bundle_path: Path | None = None, - ) -> str | None: - if self._audit_body_dir is None: - return None + block_hash: str, + audit_bundle_sha256: str, + ) -> dict[str, object]: + """Refresh the writer lease and confirm one block/digest row identity. + + The standalone canonical-bundle backfill calls this immediately before + each publication. It is the same fence `_external_audit_body_write_plan` + applies to the live path: a writer whose lease has lapsed must not keep + creating artifacts under the audit root, and the digest it is about to + publish must be the one the ledger row already advertises. Reads the + audit tables only; the sole mutation is the writer's own lease + heartbeat, so no schema changes. + + Returns the row's `body_uri` (None for legacy inline rows) and whether + an inline body is still present, which is what the backfill needs to + choose its byte source. Raises when the lease is not active or the + block/digest identity does not match. + """ + + block_hash = canonical_hex(str(block_hash), name="block_hash", expected_bytes=32) + digest = canonical_hex( + str(audit_bundle_sha256), + name="audit_bundle_sha256", + expected_bytes=32, + ) payload = { - **payload, - "block_hash": canonical_hex(str(payload["block_hash"]), name="block_hash", expected_bytes=32), - "audit_bundle_sha256": canonical_hex( - str(payload["audit_bundle_sha256"]), - name="audit_bundle_sha256", - expected_bytes=32, - ), + "block_hash": block_hash, + "audit_bundle_sha256": digest, + "writer_id": self._writer_id, + "writer_epoch": self._writer_epoch, + "writer_session_token": self._writer_session_token, + } + sql = f""" +WITH payload AS ( + SELECT {self._jsonb_literal(payload)} AS data +), +lease AS ( + UPDATE qbit_ledger_writer_lease + SET lease_expires_at = clock_timestamp() + {self._lease_interval_sql}, + updated_at = clock_timestamp() + FROM payload + WHERE qbit_ledger_writer_lease.singleton + AND qbit_ledger_writer_lease.writer_id = data->>'writer_id' + AND qbit_ledger_writer_lease.writer_epoch = (data->>'writer_epoch')::bigint + AND qbit_ledger_writer_lease.writer_session_token = data->>'writer_session_token' + RETURNING qbit_ledger_writer_lease.writer_id +), +bundle AS ( + SELECT + block_hash, + audit_bundle_sha256, + body_uri, + audit_bundle IS NOT NULL AS has_inline_bundle + FROM qbit_pool_audit_bundles + WHERE block_hash = (SELECT data->>'block_hash' FROM payload) +), +matching_bundle AS ( + SELECT bundle.* + FROM bundle, payload + WHERE bundle.audit_bundle_sha256 = data->>'audit_bundle_sha256' +) +SELECT CASE + WHEN (SELECT count(*) FROM lease) = 0 THEN + json_build_object('error', 'writer lease is not active') + WHEN (SELECT count(*) FROM bundle) = 0 THEN + json_build_object('error', 'audit bundle row does not exist') + WHEN (SELECT count(*) FROM matching_bundle) = 0 THEN + json_build_object('error', 'audit bundle digest does not match the stored row') + ELSE + json_build_object( + 'block_hash', (SELECT block_hash FROM matching_bundle LIMIT 1), + 'audit_bundle_sha256', (SELECT audit_bundle_sha256 FROM matching_bundle LIMIT 1), + 'body_uri', (SELECT body_uri FROM matching_bundle LIMIT 1), + 'has_inline_bundle', (SELECT has_inline_bundle FROM matching_bundle LIMIT 1) + ) +END; +""" + result = self._run_fenced_json(sql) + if not isinstance(result, dict): + raise RuntimeError("canonical bundle preflight returned no row") + if "error" in result: + raise RuntimeError(str(result["error"])) + body_uri = result.get("body_uri") + return { + "block_hash": block_hash, + "audit_bundle_sha256": digest, + "body_uri": str(body_uri) if body_uri else None, + "has_inline_bundle": bool(result.get("has_inline_bundle")), } - expected_sha256 = str(payload["audit_bundle_sha256"]) - body_bytes: bytes | None = None - if canonical_bundle_path is not None: - canonical_bundle_path = canonical_bundle_path.resolve() - try: - actual_sha256 = self._file_sha256_hex(canonical_bundle_path) - except OSError as exc: - raise RuntimeError( - f"canonical audit bundle is not retrievable at {canonical_bundle_path}: {exc}" - ) from exc - if actual_sha256 != expected_sha256: - raise RuntimeError( - "audit bundle sha256 mismatch: " - f"expected {expected_sha256}, got {actual_sha256}" - ) - else: - body_bytes = self._canonical_audit_body_bytes_for_sha( - final_bundle, - expected_sha256, - ) - body_uri = self._external_audit_body_write_plan(payload) - if body_uri is None: - return None - body_path = self._resolve_audit_body_path(body_uri) - storage_payload = self._audit_bundle_v2( - block_hash=str(payload["block_hash"]), - audit_bundle_sha256=expected_sha256, - final_bundle=final_bundle, - ) - if storage_payload is None: - storage_payload = self._audit_body_ref( - block_hash=str(payload["block_hash"]), - audit_bundle_sha256=expected_sha256, - final_bundle=final_bundle, - ) - if body_path.exists(): - if storage_payload is not None and self._file_matches_json_payload( - body_path, storage_payload - ): - return str(body_path) - if ( - storage_payload is None - and canonical_bundle_path is not None - and self._file_sha256_hex(body_path) == expected_sha256 - ): - return str(body_path) - # Preserve compatibility with bodies written by an older storage - # layout. This expensive reconstruction is only the mismatch path; - # same-version crash retries take the bounded exact-match path. - if not self._external_body_matches_sha(body_path, expected_sha256): - raise RuntimeError(f"existing audit bundle body does not match payload at {body_path}") - return str(body_path) - canonical_body_path = self._audit_body_path( - str(payload["block_hash"]), - str(payload["audit_bundle_sha256"]), - ) - if body_path != canonical_body_path: - raise RuntimeError( - "existing audit bundle body pointer does not match canonical external path: " - f"{body_uri}" - ) - if storage_payload is not None: - self._write_json_atomically(body_path, storage_payload) - elif canonical_bundle_path is not None: - self._copy_file_atomically(body_path, canonical_bundle_path) - else: - assert body_bytes is not None - self._write_bytes_atomically(body_path, body_bytes) - return str(body_path) - def _read_external_body( + def canonical_bundle_bytes_for_backfill( self, + *, + block_hash: str, + audit_bundle_sha256: str, body_uri: object, + ) -> bytes: + """Recover verified canonical bytes for an already published body. + + Binds the ledger's append-only share loader to the store helper so the + standalone backfill can repair share segments that have left the disk + without reaching into ledger internals. Every recovered range is checked + against the digest the body already commits to, and the assembled bundle + against the advertised bundle digest. + """ + + return self._audit_store().canonical_bundle_bytes_from_external_body( + block_hash=block_hash, + audit_bundle_sha256=audit_bundle_sha256, + body_uri=body_uri, + load_missing_range=self._load_audit_share_ledger_range, + ) + + def publish_canonical_bundle_bytes( + self, *, - expected_sha256: object | None = None, - ) -> dict[str, object] | None: - if not body_uri: + block_hash: str, + audit_bundle_sha256: str, + canonical_bytes: bytes, + ) -> str: + """Publish canonical bytes under the store's ownership checks. + + Callers must run `preflight_canonical_bundle_publication` immediately + before this so the writer lease is current when the artifact lands. + """ + + return str( + self._audit_store().write_canonical_audit_bundle( + block_hash, + audit_bundle_sha256, + canonical_bytes, + ) + ) + + def _audit_body_byte_len(self, body_uri: object | None, final_bundle: Mapping[str, Any], canonical_bundle_path: Path | None = None) -> int: + if self._audit_artifact_store is None: + if isinstance(final_bundle, CanonicalAuditBundleView): + # The artifact's own length: no canonicalizer round trip and + # no whole-body copy for the inline compatibility lane. + final_bundle.verify_identity() + return final_bundle.byte_length + return len(self._canonical_audit_bundle_bytes(final_bundle)) + return self._audit_store().audit_body_byte_len(body_uri, final_bundle, canonical_bundle_path) + + def _prepare_external_audit_body(self, payload: dict[str, Any], final_bundle: Mapping[str, Any], *, canonical_bundle_path: Path | None = None) -> str | None: + if self._audit_artifact_store is None: return None - try: - body_path = self._resolve_audit_body_path(body_uri) - body_bytes = body_path.read_bytes() - except OSError as exc: - raise RuntimeError( - f"audit bundle body is not retrievable at {body_uri}: {exc}" - ) from exc - try: - body = json.loads(body_bytes.decode("utf-8")) - except (UnicodeDecodeError, json.JSONDecodeError) as exc: - raise RuntimeError(f"audit bundle body is not valid JSON at {body_uri}: {exc}") from exc - if isinstance(body, dict) and body.get("schema") == AUDIT_BODY_REF_SCHEMA: - return self._resolve_audit_body_ref(body, expected_sha256=expected_sha256, body_uri=body_uri) - if isinstance(body, dict) and body.get("schema") == AUDIT_BUNDLE_V2_SCHEMA: - return self._resolve_audit_bundle_v2(body, expected_sha256=expected_sha256, body_uri=body_uri) - if expected_sha256: - expected = str(expected_sha256).lower() - actual = sha256_bytes_hex(body_bytes) - if actual != expected: - raise RuntimeError( - f"audit bundle body hash mismatch at {body_uri}: expected {expected}, got {actual}" - ) - return body + normalized = { + **payload, + "block_hash": canonical_hex(str(payload["block_hash"]), name="block_hash", expected_bytes=32), + "audit_bundle_sha256": canonical_hex(str(payload["audit_bundle_sha256"]), name="audit_bundle_sha256", expected_bytes=32), + } + if canonical_bundle_path is not None: + self._audit_store().validate_canonical_source( + canonical_bundle_path, + str(normalized["audit_bundle_sha256"]), + final_bundle, + ) + body_uri = self._external_audit_body_write_plan(normalized) + return self._audit_store().prepare_external_audit_body( + normalized, + final_bundle, + body_uri=body_uri, + canonical_bundle_path=canonical_bundle_path, + load_missing_range=self._load_audit_share_ledger_range, + ) + + def _read_external_body(self, body_uri: object, *, expected_sha256: object | None = None) -> dict[str, object] | None: + return self._audit_reader(body_uri).read_external_body(body_uri, expected_sha256=expected_sha256) def _external_body_matches_sha(self, body_path: Path, expected_sha256: str) -> bool: - try: - self._read_external_body(str(body_path), expected_sha256=expected_sha256) - except RuntimeError: - return False - return True + return self._audit_reader(body_path).external_body_matches_sha(body_path, expected_sha256) def _external_body_available_for_sha(self, body_uri: object, expected_sha256: str) -> bool: try: - body_path = self._resolve_audit_body_path(body_uri) - body_bytes = body_path.read_bytes() - except (OSError, RuntimeError): + reader = self._audit_reader(body_uri) + except RuntimeError: return False - try: - body = json.loads(body_bytes.decode("utf-8")) - except (UnicodeDecodeError, json.JSONDecodeError): - body = None - if isinstance(body, dict) and body.get("schema") == AUDIT_BODY_REF_SCHEMA: - if str(body.get("audit_bundle_sha256") or "").lower() != expected_sha256: - return False - bundle_without_shares = body.get("bundle_without_shares") - share_parts = body.get("share_parts") - if not isinstance(bundle_without_shares, dict) or not isinstance(share_parts, list): - return False - expected_share_count = int(body.get("share_count") or 0) - actual_share_count = 0 - for part in share_parts: - if not isinstance(part, dict): - return False - kind = part.get("kind") - if kind == "segment": - if not self._audit_share_segment_available(part, parent_body_uri=body_uri): - return False - elif kind in {"segment_range", "segment_prefix"}: - if not self._audit_share_segment_available(part, parent_body_uri=body_uri): - return False - elif kind == "inline": - inline_shares = part.get("shares") - if not isinstance(inline_shares, list) or len(inline_shares) != int(part.get("share_count") or 0): - return False - else: - return False - actual_share_count += int(part.get("share_count") or 0) - return actual_share_count == expected_share_count - if isinstance(body, dict) and body.get("schema") == AUDIT_BUNDLE_V2_SCHEMA: - try: - self._resolve_audit_bundle_v2(body, expected_sha256=expected_sha256, body_uri=body_uri) - except RuntimeError: - return False - return True - return sha256_bytes_hex(body_bytes) == expected_sha256 + return reader.external_body_available_for_sha(body_uri, expected_sha256) def _audit_share_segment_available(self, part: dict[str, Any], *, parent_body_uri: object) -> bool: try: - self._read_audit_share_segment(part, parent_body_uri=parent_body_uri) - except RuntimeError: + self._audit_store().read_audit_share_segment(part, parent_body_uri=parent_body_uri) + except (OSError, RuntimeError, TypeError, ValueError): return False return True - def _resolve_audit_body_ref( - self, - body_ref: dict[str, Any], - *, - expected_sha256: object | None, - body_uri: object, - ) -> dict[str, object]: - expected = str(expected_sha256).lower() if expected_sha256 else None - declared_sha256 = str(body_ref.get("audit_bundle_sha256") or "").lower() - if expected and declared_sha256 != expected: - raise RuntimeError( - f"audit bundle body hash mismatch at {body_uri}: expected {expected}, got {declared_sha256}" - ) - bundle_without_shares = body_ref.get("bundle_without_shares") - if not isinstance(bundle_without_shares, dict): - raise RuntimeError(f"audit bundle body is not valid JSON at {body_uri}: missing bundle_without_shares") - share_parts = body_ref.get("share_parts") - if not isinstance(share_parts, list): - raise RuntimeError(f"audit bundle body is not valid JSON at {body_uri}: missing share_parts") - shares: list[Any] = [] - for part in share_parts: - if not isinstance(part, dict): - raise RuntimeError(f"audit bundle body is not valid JSON at {body_uri}: invalid share part") - kind = part.get("kind") - if kind == "segment": - shares.extend(self._read_audit_share_segment(part, parent_body_uri=body_uri)) - elif kind in {"segment_range", "segment_prefix"}: - shares.extend(self._read_audit_share_segment(part, parent_body_uri=body_uri)) - elif kind == "inline": - inline_shares = part.get("shares") - if not isinstance(inline_shares, list): - raise RuntimeError(f"audit bundle body is not valid JSON at {body_uri}: invalid inline shares") - if len(inline_shares) != int(part.get("share_count") or 0): - raise RuntimeError(f"audit bundle body is not valid JSON at {body_uri}: inline share count mismatch") - # The body was freshly parsed for this request; transfer its - # share objects into the reconstructed bundle without cloning. - shares.extend(inline_shares) - else: - raise RuntimeError(f"audit bundle body is not valid JSON at {body_uri}: invalid share part kind") - expected_share_count = int(body_ref.get("share_count") or 0) - if len(shares) != expected_share_count: - raise RuntimeError( - f"audit bundle body is not valid JSON at {body_uri}: expected " - f"{expected_share_count} shares, reconstructed {len(shares)}" - ) - shares_key_index_raw = body_ref.get("shares_key_index") - shares_key_index = len(bundle_without_shares) if shares_key_index_raw is None else int(shares_key_index_raw) - bundle: dict[str, object] = {} - shares_inserted = False - for index, (key, value) in enumerate(bundle_without_shares.items()): - if index == shares_key_index: - bundle["shares"] = shares - shares_inserted = True - bundle[str(key)] = value - if not shares_inserted: - bundle["shares"] = shares - if expected: - actual = sha256_bytes_hex(self._canonical_audit_bundle_bytes(bundle)) - if actual != expected: - raise RuntimeError( - f"audit bundle body hash mismatch at {body_uri}: expected {expected}, got {actual}" - ) - return bundle + def _resolve_audit_body_ref(self, body_ref: dict[str, Any], *, expected_sha256: object | None, body_uri: object) -> dict[str, object]: + return self._audit_store().resolve_audit_body_ref(body_ref, expected_sha256=expected_sha256, body_uri=body_uri) - def _resolve_audit_bundle_v2( - self, - body: dict[str, Any], - *, - expected_sha256: object | None, - body_uri: object, - ) -> dict[str, object]: - expected = str(expected_sha256).lower() if expected_sha256 else None - declared_sha256 = str(body.get("audit_bundle_sha256") or "").lower() - if expected and declared_sha256 != expected: - raise RuntimeError( - f"audit bundle body hash mismatch at {body_uri}: expected {expected}, got {declared_sha256}" - ) - bundle_without_shares = body.get("bundle_without_shares") - if not isinstance(bundle_without_shares, dict): - raise RuntimeError(f"audit bundle body is not valid JSON at {body_uri}: missing bundle_without_shares") - proof = body.get("share_window_proof") - if not isinstance(proof, dict) or proof.get("schema") != AUDIT_WINDOW_COMPLETENESS_PROOF_SCHEMA: - raise RuntimeError(f"audit bundle body is not valid JSON at {body_uri}: missing share_window_proof") - share_parts = proof.get("share_parts") - if not isinstance(share_parts, list): - raise RuntimeError(f"audit bundle body is not valid JSON at {body_uri}: missing share_parts") - expected_parts_digest = str(proof.get("share_parts_digest_hex") or "").lower() - if expected_parts_digest: - actual_parts_digest = sha256_bytes_hex(self._storage_json_bytes({"share_parts": share_parts})) - if actual_parts_digest != expected_parts_digest: - raise RuntimeError( - f"audit bundle body is not valid JSON at {body_uri}: share_parts_digest_hex mismatch" - ) - shares: list[Any] = [] - for part in share_parts: - if not isinstance(part, dict): - raise RuntimeError(f"audit bundle body is not valid JSON at {body_uri}: invalid share part") - shares.extend(self._read_audit_share_segment(part, parent_body_uri=body_uri)) - expected_share_count = int(body.get("share_count") or proof.get("share_count") or 0) - if len(shares) != expected_share_count: - raise RuntimeError( - f"audit bundle body is not valid JSON at {body_uri}: expected " - f"{expected_share_count} shares, reconstructed {len(shares)}" - ) - if shares: - first_share_seq = int(shares[0].get("share_seq")) if isinstance(shares[0], dict) else None - last_share_seq = int(shares[-1].get("share_seq")) if isinstance(shares[-1], dict) else None - if int(proof.get("first_share_seq") or 0) != first_share_seq: - raise RuntimeError(f"audit bundle body is not valid JSON at {body_uri}: proof first_share_seq mismatch") - if int(proof.get("last_share_seq") or 0) != last_share_seq: - raise RuntimeError(f"audit bundle body is not valid JSON at {body_uri}: proof last_share_seq mismatch") - reward_manifest = bundle_without_shares.get("reward_manifest") - proof_share_digest = str(proof.get("share_slice_digest_hex") or "") - if proof_share_digest and isinstance(reward_manifest, dict): - reward_share_digest = str(reward_manifest.get("share_slice_digest_hex") or "") - if not proof_share_digest.lower() == reward_share_digest.lower(): - raise RuntimeError(f"audit bundle body is not valid JSON at {body_uri}: proof share digest mismatch") - shares_key_index_raw = body.get("shares_key_index") - shares_key_index = len(bundle_without_shares) if shares_key_index_raw is None else int(shares_key_index_raw) - bundle: dict[str, object] = {} - shares_inserted = False - for index, (key, value) in enumerate(bundle_without_shares.items()): - if index == shares_key_index: - bundle["shares"] = shares - shares_inserted = True - bundle[str(key)] = value - if not shares_inserted: - bundle["shares"] = shares - actual = sha256_bytes_hex(self._canonical_audit_bundle_bytes(bundle)) - if declared_sha256 and actual != declared_sha256: - raise RuntimeError( - f"audit bundle body hash mismatch at {body_uri}: expected {declared_sha256}, got {actual}" - ) - return bundle + def _resolve_audit_bundle_v2(self, body: dict[str, Any], *, expected_sha256: object | None, body_uri: object) -> dict[str, object]: + return self._audit_store().resolve_audit_bundle_v2(body, expected_sha256=expected_sha256, body_uri=body_uri) def _read_audit_share_segment(self, part: dict[str, Any], *, parent_body_uri: object) -> list[Any]: - body_uri = part.get("body_uri") - kind = str(part.get("kind") or "") - try: - body_path = self._resolve_audit_body_path(body_uri) - segment_bytes = body_path.read_bytes() - except OSError as exc: - raise RuntimeError( - f"audit bundle body is not retrievable at {parent_body_uri}: share segment {body_uri}: {exc}" - ) from exc - expected_sha256 = str(part.get("sha256") or "").lower() - if kind == "segment" and sha256_bytes_hex(segment_bytes) != expected_sha256: - raise RuntimeError( - f"audit bundle body hash mismatch at {parent_body_uri}: " - f"share segment {body_uri} expected {expected_sha256}, got {sha256_bytes_hex(segment_bytes)}" - ) - try: - segment = json.loads(segment_bytes.decode("utf-8")) - except (UnicodeDecodeError, json.JSONDecodeError) as exc: - raise RuntimeError( - f"audit bundle body is not valid JSON at {parent_body_uri}: share segment {body_uri}: {exc}" - ) from exc - if not isinstance(segment, dict) or segment.get("schema") != AUDIT_SHARE_SEGMENT_SCHEMA: - raise RuntimeError( - f"audit bundle body is not valid JSON at {parent_body_uri}: invalid share segment {body_uri}" - ) - shares = segment.get("shares") - if not isinstance(shares, list): - raise RuntimeError( - f"audit bundle body is not valid JSON at {parent_body_uri}: share segment {body_uri} has no shares" - ) - expected_count = int(part.get("share_count") or 0) - first_share_seq = int(part.get("first_share_seq") or 0) - last_share_seq = int(part.get("last_share_seq") or 0) - selected_shares = self._select_audit_share_segment_range( + return self._audit_store().read_audit_share_segment(part, parent_body_uri=parent_body_uri) + + def _select_audit_share_segment_range(self, shares: list[Any], *, first_share_seq: int, last_share_seq: int, parent_body_uri: object, body_uri: object) -> list[Any]: + return self._audit_store().select_audit_share_segment_range( shares, first_share_seq=first_share_seq, last_share_seq=last_share_seq, parent_body_uri=parent_body_uri, body_uri=body_uri, ) - if len(selected_shares) != expected_count: - raise RuntimeError( - f"audit bundle body is not valid JSON at {parent_body_uri}: share segment {body_uri} " - f"expected {expected_count} shares, found {len(selected_shares)}" - ) - if kind == "segment_range": - expected_range_sha256 = str(part.get("range_sha256") or "").lower() - actual_range_sha256 = sha256_bytes_hex( - self._storage_json_bytes( - self._audit_share_segment_payload( - first_share_seq=first_share_seq, - last_share_seq=last_share_seq, - shares=selected_shares, - ) - ) - ) - if actual_range_sha256 != expected_range_sha256: - raise RuntimeError( - f"audit bundle body hash mismatch at {parent_body_uri}: " - f"share segment range {body_uri} expected {expected_range_sha256}, got {actual_range_sha256}" - ) - elif kind == "segment_prefix": - expected_prefix_sha256 = str(part.get("prefix_sha256") or "").lower() - actual_prefix_sha256 = sha256_bytes_hex( - self._storage_json_bytes( - self._audit_share_segment_payload( - first_share_seq=first_share_seq, - last_share_seq=last_share_seq, - shares=selected_shares, - ) - ) - ) - if actual_prefix_sha256 != expected_prefix_sha256: - raise RuntimeError( - f"audit bundle body hash mismatch at {parent_body_uri}: " - f"share segment prefix {body_uri} expected {expected_prefix_sha256}, got {actual_prefix_sha256}" - ) - elif kind != "segment": - raise RuntimeError(f"audit bundle body is not valid JSON at {parent_body_uri}: invalid share part kind") - # selected_shares references a freshly parsed, request-local segment. - return selected_shares - - def _select_audit_share_segment_range( - self, - shares: list[Any], - *, - first_share_seq: int, - last_share_seq: int, - parent_body_uri: object, - body_uri: object, - ) -> list[Any]: - selected: list[Any] = [] - previous_seq: int | None = None - for share in shares: - if not isinstance(share, dict): - raise RuntimeError( - f"audit bundle body is not valid JSON at {parent_body_uri}: share segment {body_uri} has invalid share" - ) - share_seq = int(share.get("share_seq") or 0) - if previous_seq is not None and previous_seq + 1 != share_seq: - raise RuntimeError( - f"audit bundle body is not valid JSON at {parent_body_uri}: share segment {body_uri} is not contiguous" - ) - previous_seq = share_seq - if first_share_seq <= share_seq <= last_share_seq: - selected.append(share) - if selected: - if int(selected[0].get("share_seq") or 0) != first_share_seq: - selected = [] - elif int(selected[-1].get("share_seq") or 0) != last_share_seq: - selected = [] - if not selected and first_share_seq <= last_share_seq: - raise RuntimeError( - f"audit bundle body is not valid JSON at {parent_body_uri}: " - f"share segment {body_uri} does not contain requested range" - ) - return selected def _resolve_audit_bundle_row(self, row: object) -> dict[str, object] | None: """Return an audit-bundle row with its body resolved inline. @@ -6739,16 +10124,60 @@ def _resolve_audit_bundle_row(self, row: object) -> dict[str, object] | None: result["audit_bundle"] = body return result + @staticmethod + def _accepted_block_payload_literal_pieces(payload: Mapping[str, Any]) -> Iterator[str]: + """The accepted-block payload as a dollar-quoted JSONB literal, streamed. + + Equivalent to ``_jsonb_literal(payload)`` for a payload of plain + values, but built from bounded chunks: recipient-scaled members + (the payout accounts) and transaction-scaled members (the leaf + arrays) stream from their lazy views, and on the legacy inline lane + (no audit body store configured) the verified canonical artifact is + spliced verbatim from its descriptor rather than decoded and + re-encoded. The dollar-quote tag is chosen with the same collision + rule as ``_jsonb_literal``, checked by streaming. The statement text + itself still scales with those members: this method bounds the + Python object graph and the number of copies, not the size of the + one statement the schema requires (#255). + """ + + def chunks() -> Iterator[str]: + return iter_json_chunks(payload) + + tag = "qbit_prism_json" + for _attempt in range(64): + if not _text_stream_contains(chunks(), f"${tag}$"): + break + tag += "_x" + else: + raise RuntimeError("accepted-block payload cannot be dollar-quoted") + yield f"${tag}$" + yield from chunks() + yield f"${tag}$::jsonb" + def persist_accepted_block( self, *, block_hash: str, block_height: int, parent_hash: str, - final_bundle: dict[str, Any], + final_bundle: Mapping[str, Any], audit_report: dict[str, Any], canonical_bundle_path: Path | None = None, ) -> dict[str, int | str]: + """Persist one accepted block, its audit bundle and payout rows. + + ``final_bundle`` is either a dictionary (compatibility builders and + the memory ledger's callers) or the bundle compiler's bounded view + of the canonical artifact. Only header members are read here; the + external body publication streams the artifact and the row carries + the body pointer, so no window-sized structure enters the statement + on the production (externalized) path. What the statement still + carries per accepted block, bounded by recipient and transaction + cardinality rather than by the share window: the payout accounts + array (one row per recipient), the audit-commitment leaves and the + witness merkle leaves (one per block transaction). + """ manifest = final_bundle["signed_coinbase_manifest"]["manifest"] found_block = final_bundle.get("found_block") or {} audit_bundle_sha256 = canonical_hex( @@ -6780,26 +10209,35 @@ def persist_accepted_block( final_bundle, canonical_bundle_path, ) + # A bounded artifact view (the canonical build's result) has no + # dictionary form to embed; the legacy inline lane splices its + # canonical bytes into the statement literal instead (#255). + inline_bundle: object + if body_uri is not None: + inline_bundle = None + elif isinstance(final_bundle, CanonicalAuditBundleView): + final_bundle.verify_identity() + inline_bundle = RawJsonDocument(final_bundle.source) + else: + inline_bundle = final_bundle payload = { **payload, # Externalized rows store the body in body_uri and NULL here; legacy # rows (no body store configured) keep the inline body. - "audit_bundle": None if body_uri is not None else final_bundle, + "audit_bundle": inline_bundle, "body_uri": body_uri, "audit_body_byte_len": audit_body_byte_len, "schema_version": str(final_bundle.get("schema") or "qbit.prism.audit-bundle.v1"), "found_block_network_difficulty": found_block.get("network_difficulty"), "found_block_bits": found_block.get("bits"), "found_block_coinbase_value_sats": found_block.get("coinbase_value_sats"), + # The leaf arrays and the payout accounts stay lazy on a bounded + # artifact view; the statement literal streams them below. "audit_commitment_leaves_hex": final_bundle.get("audit_commitment_leaves_hex"), "witness_merkle_leaves_hex": final_bundle.get("witness_merkle_leaves_hex"), "accounts": final_bundle["payout_policy_manifest"]["accounts"], } - sql = f""" -WITH payload AS ( - SELECT {self._jsonb_literal(payload)} AS data -), -lease AS ( + sql_body = f"""lease AS ( UPDATE qbit_ledger_writer_lease SET lease_expires_at = clock_timestamp() + {self._lease_interval_sql}, updated_at = clock_timestamp() @@ -7084,7 +10522,7 @@ def persist_accepted_block( ) END; """ - result = self._run_fenced_json(sql) + result = self._run_candidate_payload_json(payload, sql_body) if "error" in result: raise RuntimeError(str(result["error"])) return { @@ -7158,13 +10596,59 @@ def confirm_accepted_block(self, *, block_hash: str, active_tip_height: int) -> ) ); """ - result = self._run_fenced_json(sql) - if "error" in result: - raise RuntimeError(str(result["error"])) - return { + with self._operation_gate(self._lock, "writer lock"): + result = self._run_json(sql) + if "error" in result: + raise RuntimeError(str(result["error"])) + confirmed_count = int(result["confirmed_count"]) + publication_sequence: object | None = None + if confirmed_count in {1, 2}: + # Both confirming dispositions leave a confirmed row carrying + # an ordinal: 1 allocated one in the statement above, and 2 -- + # the idempotent replay -- returns the one its original flip + # allocated. The caller needs the ordinal either way to + # address this block's audit publication, so read it for + # both. Non-confirming dispositions (0, -1) have no row to + # read and stay a single statement. + # + # A data-modifying PL/pgSQL function runs under the statement's + # command snapshot, so a join in that same statement cannot see + # the freshly assigned ordinal. Read it in the next statement + # while retaining the ledger writer lock. + # + # Holding the lock across both statements is what makes this + # step's heartbeat-silent span two statement budgets rather + # than one: no admission slice runs in between, so a landing + # caller's liveness monitor gets nothing until the second + # statement returns. Report the completed first round trip + # before starting the second (see _note_operation_progress). + self._note_operation_progress() + state = self._run_retry_safe_read_json( + f""" +SELECT json_build_object( + 'audit_publication_sequence', ( + SELECT audit_publication_sequence + FROM qbit_pool_blocks + WHERE block_hash = {self._text_literal(block_hash)} + AND block_height = {int(active_tip_height)} + AND chain_state = 'confirmed' + AND maturity_state <> 'reversed' + ) +); +""" + ) + publication_sequence = state.get("audit_publication_sequence") + if publication_sequence is None: + raise RuntimeError( + "confirmed pool block has no audit publication sequence" + ) + response: dict[str, int | str] = { "backend": str(result["backend"]), - "confirmed_count": int(result["confirmed_count"]), + "confirmed_count": confirmed_count, } + if publication_sequence is not None: + response["audit_publication_sequence"] = int(publication_sequence) + return response def pool_block_state(self, *, block_hash: str) -> dict[str, object] | None: block_hash = canonical_hex(block_hash, name="block_hash", expected_bytes=32) @@ -7176,7 +10660,8 @@ def pool_block_state(self, *, block_hash: str) -> dict[str, object] | None: 'block_height', block_height, 'parent_hash', parent_hash, 'chain_state', chain_state, - 'maturity_state', maturity_state + 'maturity_state', maturity_state, + 'audit_publication_sequence', audit_publication_sequence ) FROM qbit_pool_blocks WHERE block_hash = {self._text_literal(block_hash)} @@ -7195,8 +10680,29 @@ def pool_block_state(self, *, block_hash: str) -> dict[str, object] | None: state["block_height"] = int(state["block_height"]) return state + def audit_publication_sequence_floor(self) -> int: + """Return MAX durable pool-block ordinal, excluding sequence gaps.""" + + sql = """ +SELECT json_build_object( + 'audit_publication_sequence_floor', + COALESCE(MAX(audit_publication_sequence), 0) +) +FROM qbit_pool_blocks; +""" + with self._operation_gate(self._lock, "writer lock"): + result = self._run_retry_safe_read_json(sql) + if not isinstance(result, dict): + raise RuntimeError( + "audit publication sequence floor query returned non-object JSON" + ) + value = result.get("audit_publication_sequence_floor") + if isinstance(value, bool) or not isinstance(value, int) or value < 0: + raise RuntimeError("audit publication sequence floor is invalid") + return value + def reorg_watch_blocks(self, *, active_tip_height: int) -> list[dict[str, object]]: - sql = f""" + sql = """ SELECT COALESCE(json_agg(json_build_object( 'block_hash', block_hash, 'block_height', block_height, @@ -7208,6 +10714,54 @@ def reorg_watch_blocks(self, *, active_tip_height: int) -> list[dict[str, object WHERE chain_state IN ('confirmed', 'inactive') AND maturity_state = 'immature' ; +""" + with self._operation_gate(self._lock, "writer lock"): + rows = self._run_retry_safe_read_json(sql) + for row in rows: + row["block_height"] = int(row["block_height"]) + return rows + + def stranded_prepared_blocks( + self, + *, + active_tip_height: int, + min_depth: int, + limit: int = 64, + ) -> list[dict[str, object]]: + """Return deeply buried rows still parked in the prepared state. + + ``reorg_watch_blocks`` deliberately watches only confirmed/inactive + rows, and a prepared row is normally resolved by the live + submit/replay path that owns it through its outbox entry. A row + whose outbox entry is gone (quarantined, or completed by a process + that died before confirming) therefore has nothing left to + re-examine it, and stays prepared forever — holding immature payout + entries, carry-forward, and CTV fanout artifacts open with it. This + read finds those rows; the caller decides, against the active chain, + which ones are provably orphaned. + + The predicate leads with ``maturity_state`` and ``block_height`` so + it rides ``qbit_pool_blocks_maturity_idx``, and the depth floor + keeps the scan to rows the caller could actually act on. + """ + if limit <= 0: + return [] + depth_ceiling = int(active_tip_height) - int(min_depth) + sql = f""" +SELECT COALESCE(json_agg(json_build_object( + 'block_hash', block_hash, + 'block_height', block_height, + 'parent_hash', parent_hash +) ORDER BY block_height ASC, block_hash ASC), '[]'::json) +FROM ( + SELECT block_hash, block_height, parent_hash + FROM qbit_pool_blocks + WHERE maturity_state = 'immature' + AND block_height <= {depth_ceiling} + AND chain_state = 'prepared' + ORDER BY block_height ASC, block_hash ASC + LIMIT {int(limit)} +) stranded; """ with self._operation_gate(self._lock, "writer lock"): rows = self._run_retry_safe_read_json(sql) @@ -7251,13 +10805,46 @@ def reactivate_pool_block(self, *, block_hash: str, active_tip_height: int) -> d ) ); """ - result = self._run_fenced_json(sql) - if "error" in result: - raise RuntimeError(str(result["error"])) - return { + with self._operation_gate(self._lock, "writer lock"): + result = self._run_json(sql) + if "error" in result: + raise RuntimeError(str(result["error"])) + reactivated_count = int(result["reactivated_count"]) + publication_sequence: object | None = None + if reactivated_count == 1: + # Same two-statements-under-one-gate shape as + # confirm_accepted_block: the mutating statement's command + # snapshot cannot see the ordinal it just assigned, so the + # read runs next while the writer lock is still held. Report + # the completed first statement so a landing caller's monitor + # is not asked to sit through both budgets in silence. + self._note_operation_progress() + state = self._run_retry_safe_read_json( + f""" +SELECT json_build_object( + 'audit_publication_sequence', ( + SELECT audit_publication_sequence + FROM qbit_pool_blocks + WHERE block_hash = {self._text_literal(block_hash)} + AND block_height <= {int(active_tip_height)} + AND chain_state = 'confirmed' + AND maturity_state = 'immature' + ) +); +""" + ) + publication_sequence = state.get("audit_publication_sequence") + if publication_sequence is None: + raise RuntimeError( + "reactivated pool block has no audit publication sequence" + ) + response: dict[str, int | str] = { "backend": str(result["backend"]), - "reactivated_count": int(result["reactivated_count"]), + "reactivated_count": reactivated_count, } + if publication_sequence is not None: + response["audit_publication_sequence"] = int(publication_sequence) + return response def mark_mature_pool_payouts(self, *, active_tip_height: int) -> dict[str, int | str]: sql = f""" @@ -7294,30 +10881,517 @@ def __len__(self) -> int: with self._operation_gate(self._lock, "writer lock"): return int(self._run_retry_safe_read_json(sql)["count"]) + def _run_candidate_payload_json(self, payload: Mapping[str, Any], sql_body: str) -> Any: + def pieces() -> Iterator[str]: + yield "\nWITH payload AS (\n SELECT " + yield from self._accepted_block_payload_literal_pieces(payload) + yield " AS data\n),\n" + yield sql_body + + # Existing in-memory SQL test/embedding adapters explicitly replace + # execution; preserve their string-based seam. Real backends spool. + if (getattr(self._run_fenced_json, "__func__", None) is not PsqlShareLedger._run_fenced_json + or getattr(self._run_json, "__func__", None) is not PsqlShareLedger._run_json + or getattr(self._run_sql, "__func__", None) is not PsqlShareLedger._run_sql + or (getattr(self, "_native", None) is not None + and not isinstance(self._native, _NativePostgresClient))): + return self._run_fenced_json("".join(pieces())) + from lab.prism.statement_spool import run_fenced_statement + return run_fenced_statement(self, pieces()) + def _run_fenced_json(self, sql: str) -> Any: with self._operation_gate(self._lock, "writer lock"): return self._run_json(sql) - def _run_read_json(self, sql: str) -> Any: - with self._operation_gate(self._read_semaphore, "read slot"): - return self._run_retry_safe_read_json(sql) + def _run_read_json(self, sql: str) -> Any: + with self._operation_gate(self._read_semaphore, "read slot"): + return self._run_retry_safe_read_json(sql) + + def _run_attributed_read_json( + self, + sql: str, + *, + operation: str, + gate: Any, + gate_name: str, + ) -> Any: + """Run one gated single-value read, timing admission apart from execution. + + See ``_run_attributed_read`` for the bookkeeping; this is that helper + with ``_run_retry_safe_read_json`` as the statement. + """ + return self._run_attributed_read( + sql, + operation=operation, + gate=gate, + gate_name=gate_name, + execute=self._run_retry_safe_read_json, + ) + + def _run_attributed_read_json_rows( + self, + sql: str, + *, + operation: str, + gate: Any, + gate_name: str, + row_converter: Callable[[Any], Any] | None = None, + ) -> list[Any]: + """Run one gated row-result read, timing admission apart from execution. + + The row-result twin of ``_run_attributed_read_json`` (issue #236): + the same gate, taken and released the same way, the same admission + and execution samples under the same ``operation`` name, with + ``_run_retry_safe_read_json_rows`` as the statement. Execution time + still covers the whole statement including local decoding, exactly + as the single-value read's did when its decoding was one call. + """ + + def execute( + statement: str, + *, + on_statement_start: Callable[[], None] | None = None, + ) -> list[Any]: + return self._run_retry_safe_read_json_rows( + statement, + row_converter=row_converter, + on_statement_start=on_statement_start, + ) + + return self._run_attributed_read( + sql, + operation=operation, + gate=gate, + gate_name=gate_name, + execute=execute, + ) + + def _run_attributed_read( + self, + sql: str, + *, + operation: str, + gate: Any, + gate_name: str, + execute: Callable[..., Any], + ) -> Any: + """Run one gated read query, timing admission apart from execution. + + Identical to wrapping ``_operation_gate(gate, gate_name)`` around + ``execute`` (``_run_retry_safe_read_json`` or its row-result twin) + in what it acquires and what it executes -- the caller's own + admission primitive, the same retry-safe statement, no extra + connection and no extra thread -- and different only in what it + records. + + The gate is the caller's and stays the caller's. A read that takes the + bounded read semaphore keeps taking it; a read that takes the writer + lock keeps taking it. Attribution is a measurement, and a measurement + that quietly moved a statement between admission classes would change + which writes it serializes against -- the one property the rest of the + ledger reasons about. ``gate_name`` is the same string + ``_operation_gate`` would have been given, so an admission that + expires still names the primitive it was waiting for. + + ``operation`` is written at each call site as a bare string literal + taken from ``PRISM_LEDGER_READ_OPERATIONS`` in + ``lab.prism.accepted_preview_telemetry``, and deliberately not as an + imported constant: ``tests.test_prism_accepted_preview_telemetry`` + harvests these literals out of this file's source to fail the moment + a call site attributes a read under a name the closed vocabulary does + not carry. Importing the names instead would leave that check with + nothing to read. A name that escaped anyway is still bounded -- + ``fold_ledger_read_stats`` folds it into ``other`` before rendering, + so a drifted call site costs an attribution, never a new series. + + That record is the point. Issue #211 was diagnosed against an outer + call reporting ``replay-outbox-query exceeded 5s`` while the inner + PostgreSQL deadline still had seconds left and the server showed zero + blocked backends: the budget had gone to coordinator-local admission, + and nothing on ``/metrics`` said so. One duration covering both halves + cannot answer "database or convoy?", so the halves are counted + separately here and exported per operation. + + Execution time is measured across the whole statement including the + tail a server-cancelled statement spends returning, so a deadline that + expires inside PostgreSQL is attributed to PostgreSQL (cancel lag + included) rather than to the gate. + + Every exit path records, so a timed-out read is counted rather than + lost, and the gate is released in ``finally`` exactly as + ``_operation_gate`` releases it. + """ + gate_started = self._monotonic() + try: + self._acquire_operation_gate(gate, gate_name) + except BaseException as exc: + # Admission itself expired: no statement was ever sent, so there + # is no execution sample to record and the call counts as a gate + # timeout rather than a database one. + self._note_ledger_read_timing( + operation, + gate_wait_seconds=max(0.0, self._monotonic() - gate_started), + execute_seconds=None, + timed_out=isinstance(exc, TimeoutError), + ) + raise + gate_wait_seconds = max(0.0, self._monotonic() - gate_started) + execute_started: float | None = None + + def on_statement_start() -> None: + nonlocal execute_started + # A retry-safe native read may dispatch more than once after an + # ambiguous connection loss. Execution attribution begins at the + # first dispatch and includes the retry tail rather than resetting + # the clock for each attempt. + if execute_started is None: + execute_started = self._monotonic() + + timed_out = False + try: + return execute( + sql, + on_statement_start=on_statement_start, + ) + except BaseException as exc: + timed_out = isinstance(exc, TimeoutError) + raise + finally: + execute_seconds = ( + None + if execute_started is None + else max(0.0, self._monotonic() - execute_started) + ) + # Released before the record is taken: a bookkeeping failure must + # never leak the gate it was measuring. + gate.release() + self._note_ledger_read_timing( + operation, + gate_wait_seconds=gate_wait_seconds, + execute_seconds=execute_seconds, + timed_out=timed_out, + ) + + def _ensure_ledger_read_timings(self) -> Lock: + """Return the lock guarding this instance's read-timing record.""" + stats_lock = getattr(self, "_ledger_read_timings_lock", None) + if stats_lock is not None: + return stats_lock + with PsqlShareLedger._ledger_read_timings_bootstrap: + stats_lock = getattr(self, "_ledger_read_timings_lock", None) + if stats_lock is None: + self._ledger_read_timings = {} + stats_lock = Lock() + # Published last, so a caller that observes the lock also + # observes the dict the lock guards. + self._ledger_read_timings_lock = stats_lock + return stats_lock + + def _note_ledger_read_timing( + self, + operation: str, + *, + gate_wait_seconds: float, + execute_seconds: float | None, + timed_out: bool, + ) -> None: + stats_lock = self._ensure_ledger_read_timings() + gate_wait_seconds = max(0.0, float(gate_wait_seconds)) + with stats_lock: + stats = self._ledger_read_timings.setdefault( + operation, + { + "calls_total": 0, + "gate_wait_seconds_total": 0.0, + "gate_wait_seconds_max": 0.0, + "gate_timeouts_total": 0, + "execute_seconds_total": 0.0, + "execute_seconds_max": 0.0, + "execute_timeouts_total": 0, + }, + ) + stats["calls_total"] = int(stats["calls_total"]) + 1 + stats["gate_wait_seconds_total"] = ( + float(stats["gate_wait_seconds_total"]) + gate_wait_seconds + ) + stats["gate_wait_seconds_max"] = max( + float(stats["gate_wait_seconds_max"]), gate_wait_seconds + ) + if execute_seconds is None: + if timed_out: + stats["gate_timeouts_total"] = ( + int(stats["gate_timeouts_total"]) + 1 + ) + return + execute_seconds = max(0.0, float(execute_seconds)) + stats["execute_seconds_total"] = ( + float(stats["execute_seconds_total"]) + execute_seconds + ) + stats["execute_seconds_max"] = max( + float(stats["execute_seconds_max"]), execute_seconds + ) + if timed_out: + stats["execute_timeouts_total"] = ( + int(stats["execute_timeouts_total"]) + 1 + ) + + def ledger_read_gate_stats(self) -> dict[str, dict[str, float | int]]: + """Read-slot admission wait against SQL execution, by operation. + + The series this feeds exist so the *next* budget exhaustion is + attributable without a live debugging session: a rising + ``gate_wait_seconds_max`` is coordinator-local contention, a rising + ``execute_seconds_max`` is PostgreSQL. See + ``_run_attributed_read_json``. + """ + stats_lock = self._ensure_ledger_read_timings() + with stats_lock: + return { + operation: dict(stats) + for operation, stats in self._ledger_read_timings.items() + } + + def _run_retry_safe_read_json( + self, + sql: str, + *, + on_statement_start: Callable[[], None] | None = None, + ) -> Any: + native = getattr(self, "_native", None) + if native is not None: + timeout_seconds = self._remaining_operation_timeout() + run_kwargs: dict[str, Any] = {"retry_safe": True} + if on_statement_start is not None: + # The native client borrows or creates its connection before + # firing this signal, and rechecks the deadline afterward. + # Connection setup expiry is therefore local admission, not + # statement execution that never happened. + run_kwargs["on_statement_start"] = on_statement_start + if timeout_seconds is None: + return native.run_json(sql, **run_kwargs) + run_kwargs["timeout_seconds"] = timeout_seconds + return native.run_json(sql, **run_kwargs) + run_json = self._run_json + if getattr(run_json, "__func__", None) is PsqlShareLedger._run_json: + return run_json(sql, on_statement_start=on_statement_start) + # Test and embedding subclasses have historically overridden this + # private seam with the one-argument signature. Preserve that + # compatibility while treating entry into their replacement as the + # only observable statement-start boundary they expose. + if on_statement_start is not None: + on_statement_start() + return run_json(sql) + + # Batch size for the row-result read path; an instance attribute so a + # test can shrink it to exercise batch boundaries with a handful of rows. + _json_row_batch_size: int = PAYOUT_WINDOW_ROW_BATCH_SIZE + + # Replay header rows describe bodies that live in storage; the block + # submitter hydrates them one at a time at dequeue, never during the + # startup enumeration (#255). + candidate_hydration_deferred: ClassVar[bool] = True + + def _run_retry_safe_read_json_rows( + self, + sql: str, + *, + row_converter: Callable[[Any], Any] | None = None, + on_statement_start: Callable[[], None] | None = None, + ) -> list[Any]: + """Run one read-only statement that yields one JSON value per row. + + The row-result counterpart of ``_run_retry_safe_read_json`` (issue + #236), and the only way the payout-window snapshot and delta reach a + backend. The statement is one SELECT; each backend decodes its rows + in ``_json_row_batch_size`` batches and, between batches, rechecks + the caller's absolute operation deadline and stamps liveness + (``_note_json_row_batch``), so a window of any size is decoded in + bounded calls and a deadline that expires mid-decode fails the read + rather than returning late. + + Backend selection mirrors ``_run_retry_safe_read_json`` seam for + seam, so no backend keeps a whole-window aggregate by accident: + + * the native client runs ``run_json_rows`` (retry-safe, under the + same remaining deadline, with the same statement-start signal); + * a subclass that overrides the private ``_run_json`` seam stands in + for the whole server and answers a row-result statement with the + list of row values -- the same Python shape ``json_agg`` decoded + to, which is what the existing fakes already return -- converted + here in batches; + * a subclass that overrides the private ``_run_sql`` seam returns + psql's text, one JSON value per line, parsed line by line; + * the shipped subprocess backend spools psql's output and parses it + line by line (``_run_psql_json_rows``). + """ + native = getattr(self, "_native", None) + if native is not None: + timeout_seconds = self._remaining_operation_timeout() + run_kwargs: dict[str, Any] = { + "retry_safe": True, + "row_converter": row_converter, + "batch_size": self._json_row_batch_size, + "on_batch": self._note_json_row_batch, + } + if on_statement_start is not None: + run_kwargs["on_statement_start"] = on_statement_start + if timeout_seconds is not None: + run_kwargs["timeout_seconds"] = timeout_seconds + return native.run_json_rows(sql, **run_kwargs) + run_json = self._run_json + if getattr(run_json, "__func__", None) is not PsqlShareLedger._run_json: + timeout_seconds = self._remaining_operation_timeout() + deadline = ( + None + if timeout_seconds is None + else self._monotonic() + timeout_seconds + ) + if on_statement_start is not None: + on_statement_start() + rows = run_json(sql) + if not isinstance(rows, list): + raise RuntimeError( + "row-result statement seam returned a non-list value: " + f"{type(rows).__name__}" + ) + return self._convert_json_rows( + rows, + row_converter=row_converter, + deadline=deadline, + ) + run_sql = self._run_sql + if getattr(run_sql, "__func__", None) is not PsqlShareLedger._run_sql: + timeout_seconds = self._remaining_operation_timeout() + deadline = ( + None + if timeout_seconds is None + else self._monotonic() + timeout_seconds + ) + if on_statement_start is not None: + on_statement_start() + output = run_sql(sql) + return self._decode_json_lines( + output.splitlines(), + row_converter=row_converter, + deadline=deadline, + ) + return self._run_psql_json_rows( + sql, + row_converter=row_converter, + on_statement_start=on_statement_start, + ) + + def _note_json_row_batch(self) -> None: + """Between two decode batches: recheck the deadline, stamp liveness. - def _run_retry_safe_read_json(self, sql: str) -> Any: - native = getattr(self, "_native", None) - if native is not None: - timeout_seconds = self._remaining_operation_timeout() - if timeout_seconds is None: - return native.run_json(sql, retry_safe=True) - return native.run_json( - sql, - retry_safe=True, - timeout_seconds=timeout_seconds, + The native client's ``on_batch`` hook. ``_remaining_operation_timeout`` + raises once the caller's absolute operation deadline has passed, so + a decode that outlives its budget fails at the next batch boundary + instead of completing late; the progress stamp is the same one a + multi-statement gate body makes between statements, and tells a + liveness monitor the read is advancing rather than wedged. + """ + self._remaining_operation_timeout() + self._note_operation_progress() + + def _check_json_row_deadline(self, deadline: float | None) -> None: + """Raise once the statement deadline or the operation deadline has passed. + + Called before local decoding starts, between batches, and before a + result is published -- including after a trailing partial batch and + for a result smaller than one batch -- so a read never returns + successfully after its budget merely because the last batch was + short. ``deadline`` is the per-statement absolute deadline computed + at dispatch from the same remaining budget the statement ran under; + ``_remaining_operation_timeout`` covers the caller's absolute + operation deadline, which is the same instant in production and can + differ only under a test's virtual clock. + """ + if deadline is not None and self._monotonic() >= deadline: + raise LedgerOperationTimeout( + "statement deadline expired while decoding rows" ) - return self._run_json(sql) + self._remaining_operation_timeout() + + def _convert_json_rows( + self, + rows: Sequence[Any], + *, + row_converter: Callable[[Any], Any] | None, + deadline: float | None, + ) -> list[Any]: + """Convert already-decoded row values in bounded batches. + + Deadline-checked before the first batch, between batches (where + liveness is also stamped) and before the result is returned. + """ + batch_size = max(1, int(self._json_row_batch_size)) + results: list[Any] = [] + self._check_json_row_deadline(deadline) + for start in range(0, len(rows), batch_size): + if start: + self._check_json_row_deadline(deadline) + self._note_operation_progress() + batch = rows[start : start + batch_size] + if row_converter is None: + results.extend(batch) + else: + results.extend(row_converter(value) for value in batch) + self._check_json_row_deadline(deadline) + return results + + def _decode_json_lines( + self, + lines: Iterable[bytes | str], + *, + row_converter: Callable[[Any], Any] | None, + deadline: float | None, + ) -> list[Any]: + """Parse one JSON value per line, in bounded batches. + + Blank lines are skipped: psql prints nothing at all for an empty + result and a trailing newline otherwise. Every other line must be + one complete JSON value -- with ``--tuples-only --no-align`` a row is + one line, and PostgreSQL's JSON output escapes control characters, + so no row spans lines -- and the values are never reassembled into + an array for a single ``json.loads``. The statement deadline computed + at dispatch and the caller's operation deadline are both checked + before decoding starts, between batches (where liveness is stamped) + and before the result is returned, so a trailing partial batch or a + result shorter than one batch cannot publish after the budget; the + native client keeps the same cadence. + """ + batch_size = max(1, int(self._json_row_batch_size)) + results: list[Any] = [] + in_batch = 0 + line_number = 0 + self._check_json_row_deadline(deadline) + for raw_line in lines: + line_number += 1 + line = raw_line.strip() + if not line: + continue + try: + if isinstance(line, bytes): + line = line.decode("utf-8") + value = json.loads(line) + except ValueError as exc: + raise RuntimeError( + "psql row-result statement returned malformed JSON on " + f"output line {line_number}: {exc}" + ) from exc + results.append(value if row_converter is None else row_converter(value)) + in_batch += 1 + if in_batch >= batch_size: + in_batch = 0 + self._check_json_row_deadline(deadline) + self._note_operation_progress() + self._check_json_row_deadline(deadline) + return results def _ensure_writer_lease(self) -> None: while True: - acquire_started_monotonic = time.monotonic() + acquire_started_monotonic = self._monotonic() result = self._try_acquire_writer_lease() if result.get("acquired"): self._writer_lease_last_refresh_monotonic = ( @@ -7326,7 +11400,7 @@ def _ensure_writer_lease(self) -> None: return if self._can_adopt_writer_lease(result): observed_session = str(result["writer_session_token"]) - adoption_started_monotonic = time.monotonic() + adoption_started_monotonic = self._monotonic() adoption = self._try_adopt_writer_lease(result) if adoption.get("acquired"): self._writer_lease_last_refresh_monotonic = ( @@ -7368,7 +11442,85 @@ def _ensure_writer_lease(self) -> None: ) self._lease_retry_sleep(sleep_seconds) + def _run_lease_acquisition_json(self, sql: str, description: str) -> Any: + """Run one lease-acquisition statement under a bounded lock deadline. + + The startup lease upsert and the adoption CAS both row-lock the + qbit_ledger_writer_lease singleton. Unbounded, either statement + queues behind whatever transaction already holds that row — in the + worst case an orphaned idle-in-transaction backend of a vanished + predecessor — inside PsqlShareLedger.__init__, before the + coordinator's watchdog arms, for as long as the kernel keeps the + dead peer's socket alive. Each attempt therefore runs under the + configured lock/statement deadline (statement_timeout arms both + SET LOCAL statement_timeout and lock_timeout on the native backend, + and the PGOPTIONS equivalents plus a subprocess timeout on psql). + + The retry budget (5 attempts x 5s by default) is sized to outlast a + typical orphan reap rather than to guarantee one: the session guards' + idle_in_transaction_session_timeout runs from the moment the blocking + transaction went idle, not from the moment this process started + retrying, so an orphan frequently clears partway through the budget + and a later attempt lands the lease with no operator action. A + statement still hitting its deadline on every attempt becomes the + fatal RuntimeError, which __init__'s close-and-reraise turns into a + visible process exit the supervisor can restart, instead of a silent + multi-hour hang. + + A deadline expiry names the lock conflict first because it is by far + the most common cause, but it is not proof of one: a connect timeout, + an exhausted pool slot, and a healthy-but-overloaded server all + surface as the same LedgerOperationTimeout. Both the per-attempt line + and the fatal error therefore quote the underlying exception and the + fatal error chains it, so the operator diagnoses from the cause rather + than from this layer's guess. + """ + attempts = self._lease_acquire_attempts + lock_timeout_seconds = self._lease_acquire_lock_timeout_seconds + cause: LedgerOperationTimeout | None = None + for attempt in range(1, attempts + 1): + try: + with self.statement_timeout(lock_timeout_seconds): + return self._run_json(sql) + except LedgerOperationTimeout as exc: + cause = exc + print( + f"prism ledger {description}: attempt {attempt}/{attempts} " + f"did not complete within its {lock_timeout_seconds:g}s " + "deadline (commonly the qbit_ledger_writer_lease row is " + "lock-blocked by another transaction, but an unreachable " + f"or overloaded server produces the same signal): {exc}", + flush=True, + ) + if attempt < attempts: + self._lease_retry_sleep(self._lease_retry_min_sleep_seconds) + raise RuntimeError( + f"qbit ledger {description} did not complete within its " + f"{lock_timeout_seconds:g}s deadline on any of {attempts} attempts " + "(commonly the qbit_ledger_writer_lease row is lock-blocked by " + "another transaction, but an unreachable or overloaded server " + f"produces the same signal): {cause}; the coordinator is exiting " + "rather than blocking startup indefinitely" + ) from cause + def _try_acquire_writer_lease(self) -> dict[str, Any]: + """Claim, renew, or observe the writer lease in one statement. + + The statement is total: every outcome is a JSON object, so nothing + here can reach ``parse_single_json_value``'s NULL branch and surface a + raw driver error out of ``PsqlShareLedger.__init__``. Two arms are the + ordinary ones -- this identity took the lease, or someone else holds + it -- and the third exists because both can be empty at once during + the first-ever concurrent acquisition (see + WRITER_LEASE_ACQUIRE_RETRY_KEY). + + That third arm is handled here rather than by the caller because the + remedy is local to this statement: it carries no holder to wait on or + adopt, and only re-running gets the fresh READ COMMITTED snapshot in + which the winner's committed row is visible. The retry therefore + converges to one of the two ordinary arms, and ``_ensure_writer_lease`` + never sees the sentinel. + """ payload = { "writer_id": self._writer_id, "writer_epoch": self._writer_epoch, @@ -7436,13 +11588,37 @@ def _try_acquire_writer_lease(self) -> dict[str, Any]: ) FROM qbit_ledger_writer_lease WHERE singleton + ), + json_build_object( + 'acquired', false, + '{WRITER_LEASE_ACQUIRE_RETRY_KEY}', true, + 'lease', '{WRITER_LEASE_ACQUIRE_RETRY_SUBJECT}', + 'retry_reason', + 'the {WRITER_LEASE_ACQUIRE_RETRY_SUBJECT} row was committed by a concurrent first acquisition after this statement snapshot' ) ); """ - result = self._run_json(sql) - if not isinstance(result, dict): - raise RuntimeError("psql writer lease query returned non-object JSON") - return result + for attempt in range(1, WRITER_LEASE_ACQUIRE_RETRY_ATTEMPTS + 1): + result = self._run_lease_acquisition_json( + sql, + "writer lease acquisition", + ) + if not isinstance(result, dict): + raise RuntimeError("psql writer lease query returned non-object JSON") + if not result.get(WRITER_LEASE_ACQUIRE_RETRY_KEY): + return result + print( + "prism ledger writer lease acquisition raced a concurrent first " + f"acquisition (attempt {attempt}/{WRITER_LEASE_ACQUIRE_RETRY_ATTEMPTS}); " + "retrying on a fresh statement snapshot", + flush=True, + ) + raise RuntimeError( + f"{WRITER_LEASE_ACQUIRE_RETRY_SUBJECT} acquisition still could not see " + "the lease row committed by a concurrent first acquisition after " + f"{WRITER_LEASE_ACQUIRE_RETRY_ATTEMPTS} fresh statement snapshots; " + "the coordinator is exiting rather than starting without the lease" + ) def _can_adopt_writer_lease(self, result: dict[str, Any]) -> bool: wait_seconds = self._writer_lease_adoption_wait_seconds(result) @@ -7503,7 +11679,7 @@ def _writer_lease_adoption_wait_seconds( ) guard_held_seconds = max( 0.0, - time.monotonic() - guard_acquired_monotonic, + self._monotonic() - guard_acquired_monotonic, ) guard_wait_seconds = max( 0.0, @@ -7580,7 +11756,7 @@ def _try_adopt_writer_lease(self, observed: dict[str, Any]) -> dict[str, Any]: ) ); """ - result = self._run_json(sql) + result = self._run_lease_acquisition_json(sql, "writer lease adoption") if not isinstance(result, dict): raise RuntimeError("psql writer lease adoption query returned non-object JSON") return result @@ -7608,11 +11784,188 @@ def renew_writer_lease(self) -> dict[str, int | str]: """ return self._renew_writer_lease_with(self._run_fenced_json) + def prove_writer_lease_guard_session( + self, + *, + on_query_start: Callable[[], None] | None = None, + on_statement_end: Callable[[], None] | None = None, + on_statement_server_seconds: Callable[[float | None], None] | None = None, + ) -> dict[str, Any]: + """Prove ownership on one cheap read-only statement; never renew. + + The frequent half of the split issue #212 asked for. Ownership and + TTL renewal are two different questions asked at two very different + rates: the heartbeat must prove *ownership* several times inside one + adoption-silence window, but the lease TTL is 60s and only needs a + writer-side refresh well before it lapses (every fenced write + refreshes it too). Paying for the renewal question on every beat is + what made the frequent statement expensive: ``FOR NO KEY UPDATE SKIP + LOCKED`` on the hot lease tuple, a ``pg_stat_activity`` scan to + attribute the tuple's locker, and — in the ambiguous case — a second + statement. Under a rapid-block burst that statement approached the + guard's 500ms statement timeout, which is exactly how a healthy + coordinator ran out of server-proven envelope. + + This statement asks only the ownership question, and asks it with + two ``EXISTS`` reads: + + * PostgreSQL still shows *this* backend holding the writer/epoch + advisory lock, and + * the last committed lease row still names this exact + ``(writer_id, writer_epoch, writer_session_token)``. + + Those are the same two conditions + :meth:`verify_writer_lease_guard_session` raises on, evaluated the + same way, so the exact-session guarantee is unchanged: a guard + session that died, or an identity that was fenced out, still raises + here and still hard-exits the coordinator. What is *not* asked is + anything that can block or that needs live backend state, so the + statement takes no row lock, never queues behind a fenced write, and + never needs an attribution recheck — it is one round trip, always. + + The lease-expiry guarantee is preserved by escalation rather than by + renewal. The result reports ``lease_renewal_due`` whenever the + committed row's remaining validity has fallen to the own-write + authority margin (which ``_resolve_lease_authority_margin_seconds`` + keeps at or above half the TTL, and strictly below it), and + ``lease_expired`` when it has lapsed outright — an expired row is + always also renewal-due. The heartbeat answers either flag by + running the full renewing verification immediately, on the same + beat. So the cheap proof short-circuits only while the lease is + comfortably valid, which is precisely the window in which the full + verification had nothing to decide; the moment renewal actually + matters — including for the whole of a long own fenced write that + keeps skipping renewals — every beat runs the same fail-closed + verification it ran before this split existed. + + Never call this in place of the verification before an external + side effect. A proof is liveness and identity, not authority: it + deliberately does not renew, so it cannot distinguish a row this + writer's own in-flight write will refresh from one it will roll + back. :meth:`require_fresh_lease_for_external_side_effect` keeps + using the full verification for that reason. + + ``on_query_start`` fires once the guarded session's serialized query + slot is acquired and ``on_statement_end`` once the round trip + returns, so a caller can attribute queue wait and server time + separately. + + ``on_statement_server_seconds`` receives, for the same round trip, + the execution time the statement measured on the server + (``clock_timestamp() - statement_timestamp()``, returned as a + column of the statement itself, so it costs no extra round trip + and takes no lock), or ``None`` when the answer did not carry one. + Issue #227: ``on_statement_end`` is a client-side instant, so a GIL + stall between PostgreSQL answering and this thread resuming is + indistinguishable from database time without it. It is timing + metadata only — the server-proven edge is still the send edge. + """ + if not self.writer_lease_fast_adoption_capable: + raise RuntimeError("writer session is not heartbeat-capable") + guard = self._writer_lease_guard + if guard is None: + raise RuntimeError("postgres writer lease guard is not held") + payload = { + "writer_id": self._writer_id, + "writer_epoch": self._writer_epoch, + "writer_session_token": self._writer_session_token, + } + lock_key = _writer_lease_advisory_lock_key( + self._writer_id, + self._writer_epoch, + ) + lock_classid = (lock_key >> 32) & 0xFFFFFFFF + lock_objid = lock_key & 0xFFFFFFFF + sql = f""" +WITH payload AS ( + SELECT {self._jsonb_literal(payload)} AS data +) +SELECT json_build_object( + 'backend', 'postgres-psql', + 'guard_advisory_lock_held', EXISTS ( + SELECT 1 + FROM pg_locks + WHERE locktype = 'advisory' + AND granted + AND pid = pg_backend_pid() + AND classid = {lock_classid}::oid + AND objid = {lock_objid}::oid + AND objsubid = 1 + ), + 'writer_session_token_current', EXISTS ( + SELECT 1 + FROM qbit_ledger_writer_lease, payload + WHERE qbit_ledger_writer_lease.singleton + AND qbit_ledger_writer_lease.writer_id = data->>'writer_id' + AND qbit_ledger_writer_lease.writer_epoch = (data->>'writer_epoch')::bigint + AND qbit_ledger_writer_lease.writer_session_token = data->>'writer_session_token' + ), + 'lease_expired', EXISTS ( + SELECT 1 + FROM qbit_ledger_writer_lease, payload + WHERE qbit_ledger_writer_lease.singleton + AND qbit_ledger_writer_lease.writer_id = data->>'writer_id' + AND qbit_ledger_writer_lease.writer_epoch = (data->>'writer_epoch')::bigint + AND qbit_ledger_writer_lease.writer_session_token = data->>'writer_session_token' + AND qbit_ledger_writer_lease.lease_expires_at <= clock_timestamp() + ), + 'lease_renewal_due', EXISTS ( + SELECT 1 + FROM qbit_ledger_writer_lease, payload + WHERE qbit_ledger_writer_lease.singleton + AND qbit_ledger_writer_lease.writer_id = data->>'writer_id' + AND qbit_ledger_writer_lease.writer_epoch = (data->>'writer_epoch')::bigint + AND qbit_ledger_writer_lease.writer_session_token = data->>'writer_session_token' + AND qbit_ledger_writer_lease.lease_expires_at + <= clock_timestamp() + {self._lease_authority_margin_sql} + ), + '{GUARD_SERVER_EXECUTION_SECONDS_KEY}', {GUARD_SERVER_EXECUTION_SECONDS_SQL} +); +""" + run_kwargs: dict[str, Callable[..., None]] = {} + if on_query_start is not None: + run_kwargs["on_query_start"] = on_query_start + if on_statement_end is not None: + run_kwargs["on_statement_end"] = on_statement_end + if on_statement_server_seconds is not None: + run_kwargs["on_statement_result"] = ( + lambda statement_result: on_statement_server_seconds( + guard_server_execution_seconds(statement_result) + ) + ) + result = guard.run_json(sql, **run_kwargs) + if not isinstance(result, dict): + raise RuntimeError( + "psql writer lease guard proof returned non-object JSON" + ) + if not result.get("guard_advisory_lock_held"): + raise RuntimeError( + "postgres writer lease guard advisory lock is no longer held" + ) + if not result.get("writer_session_token_current"): + raise RuntimeError("writer lease is not active") + # An expired row is renewal-due by construction, but state it + # explicitly: a caller must never be able to read a stale committed + # row as "proved and nothing to do". + renewal_due = bool( + result.get("lease_renewal_due") or result.get("lease_expired") + ) + return { + "backend": str(result["backend"]), + "verified_count": 1, + "renewed_count": 0, + "proof_only": True, + "lease_expired": bool(result.get("lease_expired")), + "lease_renewal_due": renewal_due, + } + def verify_writer_lease_guard_session( self, *, on_query_start: Callable[[], None] | None = None, on_statement_progress: Callable[[], None] | None = None, + on_statement_end: Callable[[], None] | None = None, + on_statement_server_seconds: Callable[[float | None], None] | None = None, ) -> dict[str, int | str]: """Prove the guard session live; renew the TTL only without waiting. @@ -7730,6 +12083,23 @@ def verify_writer_lease_guard_session( stamp progress here so a lawful two-statement verification is not mistaken for a wedged heartbeat, while a genuinely stuck statement still produces no progress at all. + + ``on_statement_end`` is the general form of the same signal: it + fires after *every* completed round trip, including the last one, + which is what a caller attributing guard SQL time per phase needs. + A caller that supplies it does not also need + ``on_statement_progress``. + + ``on_statement_server_seconds`` fires alongside ``on_statement_end`` + for every completed round trip with the execution time that + statement measured on the server (``clock_timestamp() - + statement_timestamp()``, one more column of the same statement — + no extra round trip, no lock), or ``None`` when the answer did not + carry one. Issue #227: the end mark is a client-side instant, so + without the server's own figure a GIL stall between the answer + leaving PostgreSQL and this thread resuming is booked as database + time. Timing metadata only: the server-proven edge is still the + conservative send edge, never a server clock reading. """ if not self.writer_lease_fast_adoption_capable: raise RuntimeError("writer session is not heartbeat-capable") @@ -7818,7 +12188,8 @@ def verify_writer_lease_guard_session( AND pg_stat_activity.application_name = data->>'pool_application_name' AND pg_stat_activity.backend_xid IS NOT NULL AND pg_stat_activity.backend_xid = qbit_ledger_writer_lease.xmax - ) + ), + '{GUARD_SERVER_EXECUTION_SECONDS_KEY}', {GUARD_SERVER_EXECUTION_SECONDS_SQL} ); """ attribution_rechecks_left = WRITER_LEASE_VERIFICATION_MAX_STATEMENTS - 1 @@ -7846,14 +12217,18 @@ def attribution_recheck(result: Any) -> str | None: return sql return None - if on_query_start is None: - result = guard.run_json(sql, followup=attribution_recheck) - else: - result = guard.run_json( - sql, - on_query_start=on_query_start, - followup=attribution_recheck, + run_kwargs: dict[str, Any] = {"followup": attribution_recheck} + if on_query_start is not None: + run_kwargs["on_query_start"] = on_query_start + if on_statement_end is not None: + run_kwargs["on_statement_end"] = on_statement_end + if on_statement_server_seconds is not None: + run_kwargs["on_statement_result"] = ( + lambda statement_result: on_statement_server_seconds( + guard_server_execution_seconds(statement_result) + ) ) + result = guard.run_json(sql, **run_kwargs) if not isinstance(result, dict): raise RuntimeError( "psql writer lease guard verification returned non-object JSON" @@ -8032,14 +12407,36 @@ def _run_fresh_connection_json(self, sql: str) -> Any: raise RuntimeError("psql query returned no JSON") return json.loads(output.splitlines()[-1]) - def _run_json(self, sql: str) -> Any: + def _run_json( + self, + sql: str, + *, + on_statement_start: Callable[[], None] | None = None, + ) -> Any: native = getattr(self, "_native", None) if native is not None: timeout_seconds = self._remaining_operation_timeout() + run_kwargs: dict[str, Any] = {} + if on_statement_start is not None: + run_kwargs["on_statement_start"] = on_statement_start if timeout_seconds is None: - return native.run_json(sql) - return native.run_json(sql, timeout_seconds=timeout_seconds) - output = self._run_sql(sql).strip() + return native.run_json(sql, **run_kwargs) + run_kwargs["timeout_seconds"] = timeout_seconds + return native.run_json(sql, **run_kwargs) + run_sql = self._run_sql + if getattr(run_sql, "__func__", None) is PsqlShareLedger._run_sql: + output = run_sql( + sql, + on_statement_start=on_statement_start, + ).strip() + else: + # The A1 gate and embedders historically override this private + # seam with the one-argument signature. Their replacement owns + # the full execution boundary, so entry is the only statement- + # start signal the base class can expose without breaking them. + if on_statement_start is not None: + on_statement_start() + output = run_sql(sql).strip() if not output: raise RuntimeError("psql query returned no JSON") return json.loads(output.splitlines()[-1]) @@ -8051,10 +12448,151 @@ def _run_script(self, sql: str) -> None: return self._run_sql(sql) - def _run_sql(self, sql: str) -> str: + def _run_sql( + self, + sql: str, + *, + on_statement_start: Callable[[], None] | None = None, + ) -> str: + cmd, run_kwargs, timeout_seconds = self._psql_invocation() + # All local deadline validation is complete. Only now does this + # invocation count as execution: an expiry raised above never starts + # psql and must remain attributed to coordinator-local admission. + if on_statement_start is not None: + on_statement_start() + try: + completed = subprocess.run( + cmd, + input=sql, + text=True, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + check=False, + **run_kwargs, + ) + except subprocess.TimeoutExpired as exc: + raise LedgerOperationTimeout( + f"psql operation exceeded {timeout_seconds:g}s" + ) from exc + self._check_psql_exit(completed.returncode, completed.stderr, timeout_seconds) + return completed.stdout + + def _run_psql_json_rows( + self, + sql: str, + *, + row_converter: Callable[[Any], Any] | None, + on_statement_start: Callable[[], None] | None, + ) -> list[Any]: + """Row-result counterpart of ``_run_sql`` for the subprocess backend. + + The identical psql invocation (``_psql_invocation``: same flags, + session guards, statement/lock deadline, connect timeout and + subprocess timeout), with two differences (issue #236). psql's + stdout goes to an unnamed temporary file instead of being captured + into one string, so no whole-window text ever becomes a single + Python object; and the rows are read back from that spool one line + at a time and decoded in bounded batches (``_decode_json_lines``). + + The spool is validated before a single row is read from it. The exit + status and stderr are translated exactly as ``_run_sql`` translates + them, a subprocess timeout kills and reaps the child before raising + ``LedgerOperationTimeout``, and the temporary file closes on every + exit path, so no record is ever published from a psql run that did + not complete successfully. + """ + cmd, run_kwargs, timeout_seconds = self._psql_invocation() + decode_deadline = ( + None + if timeout_seconds is None + else self._monotonic() + timeout_seconds + ) + # Same boundary as _run_sql: deadline validation is done, execution + # starts here. + if on_statement_start is not None: + on_statement_start() + with tempfile.TemporaryFile(mode="w+b") as spool: + process = subprocess.Popen( + cmd, + stdin=subprocess.PIPE, + stdout=spool, + stderr=subprocess.PIPE, + env=run_kwargs.get("env"), + ) + try: + _, stderr_bytes = process.communicate( + input=sql.encode("utf-8"), + timeout=run_kwargs.get("timeout"), + ) + except subprocess.TimeoutExpired as exc: + process.kill() + process.communicate() + raise LedgerOperationTimeout( + f"psql operation exceeded {timeout_seconds:g}s" + ) from exc + except BaseException: + process.kill() + process.wait() + raise + self._check_psql_exit( + process.returncode, + stderr_bytes.decode("utf-8", errors="replace"), + timeout_seconds, + ) + spool.seek(0) + return self._decode_json_lines( + spool, + row_converter=row_converter, + deadline=decode_deadline, + ) + + @staticmethod + def _check_psql_exit( + returncode: int, + stderr: str, + timeout_seconds: float | None, + ) -> None: + """Translate a psql exit status the way every subprocess call does.""" + if returncode == 0: + return + stderr = stderr.strip() + if timeout_seconds is not None and _is_postgres_deadline_error(stderr): + raise LedgerOperationTimeout( + f"psql operation exceeded {timeout_seconds:g}s" + ) + raise RuntimeError( + "psql command failed " + f"(exit {returncode}): {stderr}" + ) + + def _psql_invocation(self) -> tuple[list[str], dict[str, Any], float | None]: + """Build the psql argv, subprocess keywords and deadline for one statement. + + Shared by the single-value ``_run_sql`` and the row-result + ``_run_psql_json_rows`` so the two cannot drift: the same + ``--single-transaction`` / ``ON_ERROR_STOP`` invocation, the same + session guards on PGOPTIONS, and the same per-statement deadline + (statement_timeout / lock_timeout, PGCONNECT_TIMEOUT and the + subprocess timeout) derived from ``_remaining_operation_timeout``. + A deadline that has already expired raises here, before any process + is spawned, so it stays attributed to coordinator-local admission. + """ cmd = [ *self._command, "--no-psqlrc", + # One transaction per invocation. For the multi-statement schema + # script this is belt-and-braces alongside the BEGIN/COMMIT + # wrapper inside the script itself (psql tolerates the nested + # BEGIN with a warning): without atomicity, a failure or + # interruption mid-script can commit trigger definitions while + # later statements -- including the carry-forward summary seed -- + # never run, and a live writer mutating carry state in that gap + # leaves a permanently partial summary. For single-statement + # queries this is semantically identical to autocommit. The SQL + # sent here contains no \connect and no commands that cannot run + # inside a transaction block; --no-psqlrc means no ON_ERROR_ + # ROLLBACK can be injected from a psqlrc either. + "--single-transaction", "--set", "ON_ERROR_STOP=1", "--set", @@ -8065,49 +12603,39 @@ def _run_sql(self, sql: str) -> str: ] timeout_seconds = self._remaining_operation_timeout() run_kwargs: dict[str, Any] = {} + # The session guards ride every psql invocation, deadline or not: an + # orphaned idle-in-transaction backend is exactly the failure that + # occurs when no deadline-scoped work is running, so building + # PGOPTIONS only for deadline-bearing statements would leave the + # unbounded sessions — the dangerous ones — unguarded. A deadline, + # when armed, appends its per-statement bounds after the guards. + session_guards = getattr(self, "_session_guards", None) + option_fragments: list[str] = [] + if session_guards is not None: + option_fragments.append(session_guards.options_fragment()) if timeout_seconds is not None: timeout_ms = max(1, int(timeout_seconds * 1000)) - subprocess_env = dict(os.environ) - existing_options = subprocess_env.get("PGOPTIONS", "").strip() - timeout_options = ( + option_fragments.append( f"-c statement_timeout={timeout_ms}ms " f"-c lock_timeout={timeout_ms}ms" ) + if option_fragments: + subprocess_env = dict(os.environ) + # Operator-supplied PGOPTIONS stay, ahead of the coordinator's + # fragments so a later -c duplicate resolves in our favor. + existing_options = subprocess_env.get("PGOPTIONS", "").strip() subprocess_env["PGOPTIONS"] = " ".join( - option for option in (existing_options, timeout_options) if option + option + for option in (existing_options, *option_fragments) + if option ) - subprocess_env["PGCONNECT_TIMEOUT"] = str( + run_kwargs["env"] = subprocess_env + if timeout_seconds is not None: + run_kwargs["env"]["PGCONNECT_TIMEOUT"] = str( max(1, math.ceil(timeout_seconds)) ) - run_kwargs = { - "env": subprocess_env, - "timeout": timeout_seconds, - } - try: - completed = subprocess.run( - cmd, - input=sql, - text=True, - stdout=subprocess.PIPE, - stderr=subprocess.PIPE, - check=False, - **run_kwargs, - ) - except subprocess.TimeoutExpired as exc: - raise LedgerOperationTimeout( - f"psql operation exceeded {timeout_seconds:g}s" - ) from exc - if completed.returncode != 0: - stderr = completed.stderr.strip() - if timeout_seconds is not None and _is_postgres_deadline_error(stderr): - raise LedgerOperationTimeout( - f"psql operation exceeded {timeout_seconds:g}s" - ) - raise RuntimeError( - "psql command failed " - f"(exit {completed.returncode}): {stderr}" - ) - return completed.stdout + run_kwargs["timeout"] = timeout_seconds + return cmd, run_kwargs, timeout_seconds @staticmethod def _record_from_json(payload: dict[str, Any]) -> AcceptedShareRecord: @@ -8149,6 +12677,18 @@ def _jsonb_literal(payload: object) -> str: def _text_literal(value: str) -> str: return "'" + value.replace("'", "''") + "'" + @classmethod + def _text_array_literal(cls, values: Sequence[str]) -> str: + """Render a non-empty text set as one array literal for ``= ANY``. + + Each element goes through the same quoting the scalar literals use. + The cast is explicit because an empty ``ARRAY[]`` has no inferable + element type; callers never build one, and the cast keeps that a + parse-time guarantee rather than a convention. + """ + elements = ", ".join(cls._text_literal(value) for value in values) + return f"ARRAY[{elements}]::text[]" + CTV_FANOUT_STATUSES = { "awaiting_maturity", @@ -8260,14 +12800,26 @@ def ctv_fanout_recovery_payload( expected_bytes=32, ) manifests_raw = manifest_set.get("manifests") - if not isinstance(manifests_raw, list) or not manifests_raw: + streamed = _is_lazy_manifest_sequence(manifests_raw) + if not streamed and (not isinstance(manifests_raw, list) or not manifests_raw): + raise ValueError("manifest_set.manifests must be a non-empty array") + if streamed and not len(manifests_raw): raise ValueError("manifest_set.manifests must be a non-empty array") - manifests = sorted( - (require_mapping(manifest, "manifest") for manifest in manifests_raw), - key=lambda item: int(require_mapping(item.get("precommitment"), "precommitment")["chunk_index"]), - ) - first_precommitment = require_mapping(manifests[0].get("precommitment"), "precommitment") + if streamed: + # A bounded artifact view keeps the manifests on disk. They are + # walked once here in their canonical (chunk-ordered) sequence for + # validation and again, on demand, when the statement literal is + # streamed; the whole set is never sorted or copied (#255). + manifests: Sequence[Any] = MappedSequence(manifests_raw, _manifest_record) + first_manifest = manifests[0] + else: + manifests = sorted( + (require_mapping(manifest, "manifest") for manifest in manifests_raw), + key=lambda item: int(require_mapping(item.get("precommitment"), "precommitment")["chunk_index"]), + ) + first_manifest = manifests[0] + first_precommitment = require_mapping(first_manifest.get("precommitment"), "precommitment") block_height_value = manifest_set.get("block_height", first_precommitment.get("block_height")) block_height = int(block_height_value) if block_height_value is not None else None fanout_count = int(manifest_set.get("fanout_count", len(manifests))) @@ -8277,19 +12829,18 @@ def ctv_fanout_recovery_payload( if settlement_mode not in {"hybrid_coinbase_ctv_fanout", "ctv_fanout"}: raise ValueError("manifest_set.settlement_mode must be a CTV settlement mode") parent_coinbase_txid = canonical_hex( - str(manifest_set.get("parent_coinbase_txid", manifests[0].get("parent_coinbase_txid", ""))), + str(manifest_set.get("parent_coinbase_txid", first_manifest.get("parent_coinbase_txid", ""))), name="parent_coinbase_txid", expected_bytes=32, ) parent_coinbase_tx_hex = canonical_hex( - str(manifests[0].get("parent_coinbase_tx_hex", "")), + str(first_manifest.get("parent_coinbase_tx_hex", "")), name="parent_coinbase_tx_hex", ) fanout_output_sum_sats = int(manifest_set.get("fanout_output_sum_sats", 0)) covenant_output_value_sats = int(manifest_set.get("covenant_output_value_sats", 0)) - artifacts: list[dict[str, Any]] = [] - for expected_index, manifest in enumerate(manifests): + def artifact_for(expected_index: int, manifest: Mapping[str, Any]) -> dict[str, Any]: precommitment = require_mapping(manifest.get("precommitment"), "precommitment") precommitment_block_height = precommitment.get("block_height") if block_height is not None and precommitment_block_height is not None and int(precommitment_block_height) != block_height: @@ -8319,9 +12870,22 @@ def ctv_fanout_recovery_payload( name="fanout_txid", expected_bytes=32, ), - "manifest_json": canonical_json_text(manifest), - "manifest": copy.deepcopy(manifest), - "manifest_sha256": sha256_json_hex(manifest), + # One manifest is bounded by one fanout transaction (its + # recipients are capped per transaction by the settlement + # configuration), so its canonical text and digest are + # per-manifest allocations; a decoded record is used directly + # instead of being deep-copied. + "manifest_json": ( + canonical_json_text(manifest) + if isinstance(manifest, dict) and not streamed + else "".join(iter_json_chunks(manifest, sort_keys=True)) + ), + "manifest": copy.deepcopy(manifest) if not streamed else manifest, + "manifest_sha256": ( + sha256_json_hex(manifest) + if isinstance(manifest, dict) and not streamed + else streamed_sha256_json_hex(manifest) + ), "precommitment_sha256": canonical_hex( str(manifest["precommitment_sha256_hex"]), name="precommitment_sha256_hex", @@ -8352,25 +12916,47 @@ def ctv_fanout_recovery_payload( } if block_height is not None: artifact["block_height"] = block_height - artifacts.append(artifact) + return artifact - if sum(int(artifact["fanout_output_sum_sats"]) for artifact in artifacts) != fanout_output_sum_sats: + # One validating walk; the sums never hold the artifacts. + output_sum = 0 + covenant_sum = 0 + for expected_index, manifest in enumerate(manifests): + artifact = artifact_for(expected_index, manifest) + output_sum += int(artifact["fanout_output_sum_sats"]) + covenant_sum += int(artifact["covenant_output_value_sats"]) + if output_sum != fanout_output_sum_sats: raise ValueError("CTV fanout output sum mismatch") - if sum(int(artifact["covenant_output_value_sats"]) for artifact in artifacts) != covenant_output_value_sats: + if covenant_sum != covenant_output_value_sats: raise ValueError("CTV covenant output value sum mismatch") + artifacts: Sequence[Any] + if streamed: + artifacts = MappedSequence( + _EnumeratedSequence(manifests), + lambda item: artifact_for(item[0], item[1]), + ) + manifest_set_json: object = StreamedJsonString( + lambda: iter_json_chunks(manifest_set, sort_keys=True) + ) + manifest_set_member: object = manifest_set + else: + artifacts = [artifact_for(index, manifest) for index, manifest in enumerate(manifests)] + manifest_set_json = canonical_json_text(manifest_set) + manifest_set_member = copy.deepcopy(manifest_set) + payload = { "schema": "qbit.prism.ctv-fanout-recovery.v1", "block_hash": block_hash, "manifest_set_sha256": manifest_set_sha256, - "manifest_set_json": canonical_json_text(manifest_set), + "manifest_set_json": manifest_set_json, "settlement_mode": settlement_mode, "parent_coinbase_txid": parent_coinbase_txid, "parent_coinbase_tx_hex": parent_coinbase_tx_hex, "fanout_count": fanout_count, "fanout_output_sum_sats": fanout_output_sum_sats, "covenant_output_value_sats": covenant_output_value_sats, - "manifest_set": copy.deepcopy(manifest_set), + "manifest_set": manifest_set_member, "artifacts": artifacts, } if block_height is not None: @@ -8388,10 +12974,57 @@ def ctv_fanout_recovery_payload( return payload +def _is_lazy_manifest_sequence(value: object) -> bool: + """A replayable, non-list manifest sequence (a bounded view member).""" + return isinstance(value, Sequence) and not isinstance( + value, + (list, tuple, str, bytes, bytearray), + ) + + +def _manifest_record(manifest: object) -> Mapping[str, Any]: + """One manifest as a mapping; an isolated record is decoded in-process. + + A manifest is bounded by one fanout transaction, whose recipients are + capped per transaction by the settlement configuration, so it never + reaches the isolated-decode limit in practice; the fallback keeps an + unexpected one usable rather than refusing it. + """ + if isinstance(manifest, RawJsonRecord): + manifest = materialize_json(manifest) + if not isinstance(manifest, Mapping): + raise ValueError("manifest must be an object") + return manifest + + +class _EnumeratedSequence(Sequence): + """``enumerate`` as a replayable sequence of ``(index, item)`` pairs.""" + + __slots__ = ("_base",) + + def __init__(self, base: Sequence[Any]) -> None: + self._base = base + + def __len__(self) -> int: + return len(self._base) + + def __iter__(self) -> Iterator[tuple[int, Any]]: + for index, item in enumerate(self._base): + yield index, item + + def __getitem__(self, index: int | slice) -> Any: + if isinstance(index, slice): + start, stop, step = index.indices(len(self._base)) + return [(position, self._base[position]) for position in range(start, stop, step)] + if index < 0: + index += len(self._base) + return index, self._base[index] + + def require_mapping(value: object, name: str) -> dict[str, Any]: - if not isinstance(value, dict): + if not isinstance(value, Mapping): raise ValueError(f"{name} must be an object") - return value + return value # type: ignore[return-value] def sha256_json_hex(payload: object) -> str: @@ -8417,8 +13050,37 @@ def block_candidate_identity(candidate: dict[str, Any]) -> dict[str, Any]: return candidate -def block_candidate_identity_sha256(candidate: dict[str, Any]) -> str: - return sha256_json_hex(block_candidate_identity(candidate)) +def block_candidate_identity_sha256(candidate: Any) -> str: + """``sha256_json_hex`` of the candidate identity, streamed. + + The candidate carries the whole payout window under ``shares_json``, so + one ``json.dumps`` over it is exactly the whole-window C call the + writer-lease monitor must never wait behind (#236). The digest is + byte-identical to ``sha256_json_hex(block_candidate_identity(candidate))``; + the share array feeds it batch by batch. + + A :class:`PreparedCandidateIntent` already carries this digest (its + body *is* the identity JSON), and a mapping whose share sequence is not + a plain list (a page-backed window, a daemon mirror, a spool sequence) + is digested through the same bounded codec rather than materialized. + """ + if isinstance(candidate, PreparedCandidateIntent): + return prepare_candidate_intent(candidate).candidate_sha256 + if isinstance(candidate, dict) and not isinstance( + candidate.get("shares_json"), (list, tuple) + ) and "shares_json" in candidate: + return prepare_candidate_intent(candidate).candidate_sha256 + identity = block_candidate_identity(candidate) + if not isinstance(identity, dict): + return sha256_json_hex(identity) + digest = hashlib.sha256() + for chunk in iter_json_object_text_chunks( + identity, + array_keys=("shares_json",), + sort_keys=True, + ): + digest.update(chunk.encode("utf-8")) + return digest.hexdigest() def sha256_bytes_hex(payload: bytes) -> str: diff --git a/lab/prism/share_submission.py b/lab/prism/share_submission.py new file mode 100644 index 000000000..57911d6b6 --- /dev/null +++ b/lab/prism/share_submission.py @@ -0,0 +1,799 @@ +"""Pure submit classification and the narrow PRISM submission orchestrator.""" + +from __future__ import annotations + +from collections import OrderedDict +from dataclasses import dataclass +import threading +from typing import Any, Callable, Iterable, Mapping, NoReturn + +from lab.prism.block_candidates import PrismBlockCandidate +from lab.prism.job_bundle import PRISM_JOB_BUILD_SECONDS_BUCKETS +from lab.prism.job_delivery import ( + EvictedJobEntry, + PRISM_CREDIT_POLICY_STALE_GRACE, + PrismJobContext, +) +from lab.prism.share_ledger import PendingShare +from lab.prism.stratum_session import ClientState + + +PRISM_REJECTION_STALE_JOB = "stale-job" +PRISM_REJECTION_DUPLICATE_SHARE = "duplicate-share" +PRISM_REJECTION_LOW_DIFFICULTY = "low-difficulty" +PRISM_REJECTION_MALFORMED_SUBMIT = "malformed-submit" +PRISM_REJECTION_UNAUTHORIZED_WORKER = "unauthorized-worker" +PRISM_REJECTION_UNKNOWN_JOB = "unknown-job" +PRISM_REJECTION_INVALID_EXTRANONCE = "invalid-extranonce" +PRISM_REJECTION_INVALID_NTIME_OR_NONCE = "invalid-ntime-or-nonce" +PRISM_REJECTION_BACKEND_RPC_UNAVAILABLE = "backend-rpc-unavailable" +PRISM_REJECTION_POOL_CLOSED = "pool-closed" +DEFAULT_RECENT_SHARE_CAPACITY = 50_000 +# Read-to-ack latency labels for mining.submit. Accepted shares wait for the +# group commit before their ack; rejected shares (measured when the reject +# decision is made) skip it, so the pair separates commit pressure from +# thread-scheduling/GIL pressure as connection count grows. +PRISM_SHARE_ACK_RESULTS = ("accepted", "rejected") + + +def empty_share_ack_histograms() -> dict[str, dict[str, Any]]: + """Zeroed per-result share-ack histograms in the rendered shape.""" + + return { + result: { + "buckets": {bucket: 0 for bucket in PRISM_JOB_BUILD_SECONDS_BUCKETS}, + "sum": 0.0, + "count": 0, + } + for result in PRISM_SHARE_ACK_RESULTS + } + + +class RecentShareIndex: + """Thread-safe insertion-ordered duplicate window with bounded memory.""" + + def __init__( + self, + *, + capacity: int = DEFAULT_RECENT_SHARE_CAPACITY, + initial: Iterable[tuple[str, str]] = (), + ) -> None: + if capacity < 1: + raise ValueError("recent share capacity must be positive") + self.capacity = int(capacity) + self._lock = threading.Lock() + self._entries: OrderedDict[tuple[str, str], None] = OrderedDict() + self.replace(initial) + + def reserve(self, share_key: tuple[str, str]) -> bool: + with self._lock: + if share_key in self._entries: + return False + self._entries[share_key] = None + while len(self._entries) > self.capacity: + self._entries.popitem(last=False) + return True + + def release(self, share_key: tuple[str, str]) -> None: + with self._lock: + self._entries.pop(share_key, None) + + def replace(self, entries: Iterable[tuple[str, str]]) -> None: + with self._lock: + self._entries.clear() + for share_key in entries: + self._entries[share_key] = None + while len(self._entries) > self.capacity: + self._entries.popitem(last=False) + + def snapshot(self) -> tuple[tuple[str, str], ...]: + with self._lock: + return tuple(self._entries) + + +class RecentShareCompatibilityField: + """Temporary coordinator view over submission-owned duplicate state.""" + + def __get__(self, instance: Any, owner: type[Any]) -> Any: + if instance is None: + return self + service = instance.__dict__.get("_share_submission_service") + if service is not None: + return set(service.recent_shares.snapshot()) + return instance.__dict__.get("recent_share_keys", set()) + + def __set__(self, instance: Any, value: Iterable[tuple[str, str]]) -> None: + service = instance.__dict__.get("_share_submission_service") + if service is not None: + service.recent_shares.replace(value) + else: + instance.__dict__["recent_share_keys"] = set(value) + + +class BlockSolvesDroppedCompatibilityField: + """Owner-routing coordinator view over submission-owned drop accounting. + + Current callers read ``coordinator.block_solves_dropped_counts`` directly, + so the historical attribute stays readable and writable while the + submission owner keeps the single mutable copy. Reads return a copied + snapshot once the owner exists; pre-owner values live in the instance + dict until service construction adopts them, and an unset field keeps the + pre-extraction ``getattr``-with-default semantics by raising + AttributeError. + """ + + def __get__(self, instance: Any, owner: type[Any]) -> Any: + if instance is None: + return self + service = instance.__dict__.get("_share_submission_service") + if service is not None: + return service.dropped_solves_snapshot() + try: + return instance.__dict__["block_solves_dropped_counts"] + except KeyError: + raise AttributeError("block_solves_dropped_counts") from None + + def __set__(self, instance: Any, value: Mapping[str, int]) -> None: + service = instance.__dict__.get("_share_submission_service") + if service is not None: + service.replace_dropped_solves(value) + else: + instance.__dict__["block_solves_dropped_counts"] = dict(value) + + +@dataclass(frozen=True) +class SubmitRequest: + worker_name: str + job_id: str + extranonce2_hex: str + ntime_hex: str + nonce_hex: str + version_bits_hex: str | None + + +@dataclass(frozen=True) +class SubmitRejected(ValueError): + code: int + reason: str + message: str + + def __str__(self) -> str: + return self.message + + +@dataclass(frozen=True) +class SubmitContextInput: + active_context: PrismJobContext | None + retained_entry: EvictedJobEntry | None + current_tip: str + stale_grace_eligible: bool + # A retained entry resumed by a different connection than the one the job + # was delivered to. Cross-connection resumes are same-tip only; the + # stale-grace window anchors on the submitting connection's delivery + # state, which says nothing about work another (dead) connection + # delivered. + cross_connection: bool = False + + +@dataclass(frozen=True) +class SubmitContextDecision: + context: PrismJobContext + current_tip: str + source: str + credit_policy: str | None + retained_entry: EvictedJobEntry | None + cross_connection: bool = False + + +@dataclass(frozen=True) +class SubmitControlSnapshot: + """One bounded capture of coordinator-owned submit admission state.""" + + pool_open: bool + active_context: PrismJobContext | None + published_tip: str | None + + +@dataclass(frozen=True) +class SubmitWorkDecision: + share_key: tuple[str, str] + block_worthy: bool + credit_share_on_accept: bool + route: str + + +@dataclass(frozen=True) +class ShareSubmissionPorts: + reject: Callable[[SubmitRejected, str | None], NoReturn] + control_snapshot: Callable[ + [ClientState, str], + SubmitControlSnapshot, + ] + note_submitted: Callable[[str, ClientState], None] + retained_entry: Callable[[ClientState, str], EvictedJobEntry | None] + live_tip: Callable[[], str] + stale_grace_eligible: Callable[[ClientState, PrismJobContext, str], bool] + assemble: Callable[[ClientState, PrismJobContext, SubmitRequest, int], Any] + pending_share: Callable[ + [PrismJobContext, Any, str, str | None], + PendingShare, + ] + # Returns the durable candidate_outbox_state when the accepted share + # carried a block-candidate intent whose outbox row already exists + # (submitted/abandoned short-circuits the enqueue), else None. + append_share: Callable[ + [ + ClientState, + PrismJobContext, + Any, + PendingShare, + str | None, + dict[str, Any] | None, + ], + str | None, + ] + note_retained_submit: Callable[[str | None, bool], None] + note_collection_candidate: Callable[[PrismJobContext, Any], None] + # Returns the prepared intent (#255): a mapping over the historical + # keys whose share sequence is the job's immutable window, not a copy. + candidate_intent: Callable[[PrismBlockCandidate], Any] + finish_pending_commit: Callable[[PendingShare], None] + record_terminal_outcome: Callable[[str, bool], None] + submit_synchronous_candidate: Callable[ + [ + PrismBlockCandidate, + tuple[str, str], + str, + EvictedJobEntry | None, + str | None, + ], + bool, + ] + enqueue_candidate: Callable[[PrismBlockCandidate], bool] + log: Callable[[str], None] + log_exception: Callable[[], None] + + +def parse_submit_request(params: list[object]) -> SubmitRequest: + """Parse immutable wire input without touching coordinator state.""" + + if len(params) < 5: + raise SubmitRejected( + 20, + PRISM_REJECTION_MALFORMED_SUBMIT, + "submit params are incomplete", + ) + worker_name, job_id, extranonce2_hex, ntime_hex, nonce_hex = ( + str(item) for item in params[:5] + ) + return SubmitRequest( + worker_name=worker_name, + job_id=job_id, + extranonce2_hex=extranonce2_hex, + ntime_hex=ntime_hex, + nonce_hex=nonce_hex, + version_bits_hex=str(params[5]) if len(params) > 5 else None, + ) + + +def validate_submit_request( + request: SubmitRequest, + *, + authorized_username: str, + pool_open: bool, + extranonce2_size: int, +) -> None: + """Validate request identity and fixed-width fields in protocol order. + + A closed pool rejects before any share accounting: post-close submits + must not inflate global/per-worker submitted totals (the stale-percent + denominator) or vardiff windows they can never contribute to. + """ + + if request.worker_name != authorized_username: + raise SubmitRejected( + 20, + PRISM_REJECTION_UNAUTHORIZED_WORKER, + "submit username does not match authorized username", + ) + if not pool_open: + raise SubmitRejected( + 21, + PRISM_REJECTION_POOL_CLOSED, + "pool is no longer accepting shares", + ) + if len(request.extranonce2_hex) != extranonce2_size * 2: + raise SubmitRejected( + 20, + PRISM_REJECTION_INVALID_EXTRANONCE, + "unexpected extranonce2 size", + ) + if len(request.ntime_hex) != 8 or len(request.nonce_hex) != 8: + raise SubmitRejected( + 20, + PRISM_REJECTION_INVALID_NTIME_OR_NONCE, + "ntime and nonce must be 4-byte hex strings", + ) + + +def classify_submit_context(value: SubmitContextInput) -> SubmitContextDecision: + """Choose current, retained, or stale-grace work from captured facts.""" + + context = value.active_context + source = "active" + retained_entry: EvictedJobEntry | None = None + if context is None: + retained_entry = value.retained_entry + if retained_entry is None: + raise SubmitRejected(21, PRISM_REJECTION_UNKNOWN_JOB, "stale job") + context = retained_entry.context + source = "retained" + parent_hash = str(context.template["previousblockhash"]) + if parent_hash == value.current_tip: + return SubmitContextDecision( + context=context, + current_tip=value.current_tip, + source=source, + credit_policy=None, + retained_entry=retained_entry, + cross_connection=value.cross_connection, + ) + if value.cross_connection: + # Cross-connection resumes are same-tip only. A tip that moves + # between the graveyard lookup and this classification must not + # fall through to stale grace: that window anchors on the + # submitting connection's delivery state, which says nothing + # about work another (dead) connection delivered. + raise SubmitRejected(21, PRISM_REJECTION_STALE_JOB, "stale job") + if not value.stale_grace_eligible: + raise SubmitRejected(21, PRISM_REJECTION_STALE_JOB, "stale job") + return SubmitContextDecision( + context=context, + current_tip=value.current_tip, + source=source, + credit_policy=PRISM_CREDIT_POLICY_STALE_GRACE, + retained_entry=retained_entry, + cross_connection=value.cross_connection, + ) + + +def classify_submit_work( + context: PrismJobContext, + submission: Any, + *, + credit_policy: str | None, +) -> SubmitWorkDecision: + """Classify a proof as an ordinary share or one of two block routes. + + A floor-bearing listener holds the advertised share target above the + qbit network target while network difficulty sits below the floor, so a + submission can solve a block yet miss the share target. Never discard a + block over share bookkeeping: reject as low-difficulty only when the + hash is not block-worthy. Stale-grace work is never block-worthy: its + parent is no longer the chain tip. + """ + + block_worthy = bool(submission.block_pass) and ( + credit_policy != PRISM_CREDIT_POLICY_STALE_GRACE + ) + if not submission.share_pass and not block_worthy: + raise SubmitRejected( + 23, + PRISM_REJECTION_LOW_DIFFICULTY, + "low difficulty share", + ) + if not block_worthy: + route = "share" + elif submission.share_pass: + route = "async_block" + else: + route = "synchronous_block" + return SubmitWorkDecision( + share_key=(context.worker.username, submission.header_hex), + block_worthy=block_worthy, + credit_share_on_accept=route == "synchronous_block", + route=route, + ) + + +class ShareSubmissionService: + """Apply one pure submit decision through the coordinator's narrow ports. + + Beyond the hot path, the service owns the share-ACK latency histograms + (observed by the session owner through the coordinator's + ``_observe_share_ack_seconds`` routing seam) and the stale-grace + ``block_solves_dropped_counts`` accounting. Prometheus rendering stays in + the coordinator through PR 79 and reads this owner's copied snapshots. + """ + + def __init__( + self, + ports: ShareSubmissionPorts, + *, + extranonce2_size: int, + recent_shares: RecentShareIndex | None = None, + ) -> None: + self.ports = ports + self.extranonce2_size = int(extranonce2_size) + self.recent_shares = recent_shares or RecentShareIndex() + self._metrics_lock = threading.Lock() + self.share_ack_histograms = empty_share_ack_histograms() + self.block_solves_dropped_counts: dict[str, int] = {"stale_grace": 0} + + def observe_share_ack_seconds( + self, + result: str, + elapsed_seconds: float, + ) -> None: + if result not in PRISM_SHARE_ACK_RESULTS: + raise ValueError(f"unknown share ack result: {result}") + with self._metrics_lock: + histogram = self.share_ack_histograms[result] + histogram["count"] = int(histogram["count"]) + 1 + histogram["sum"] = float(histogram["sum"]) + max( + 0.0, elapsed_seconds + ) + buckets = histogram["buckets"] + assert isinstance(buckets, dict) + for bucket in PRISM_JOB_BUILD_SECONDS_BUCKETS: + if elapsed_seconds <= bucket: + buckets[bucket] = int(buckets.get(bucket, 0)) + 1 + + def share_ack_snapshot(self) -> dict[str, dict[str, Any]]: + """Copy the per-result share-ack histograms under the owner lock.""" + + with self._metrics_lock: + return { + result: { + "buckets": dict(histogram["buckets"]), + "sum": float(histogram["sum"]), + "count": int(histogram["count"]), + } + for result, histogram in self.share_ack_histograms.items() + } + + def replace_share_ack_histograms( + self, + histograms: Mapping[str, Mapping[str, Any]], + ) -> None: + """Adopt legacy pre-owner histogram state at service construction.""" + + with self._metrics_lock: + self.share_ack_histograms = { + result: { + "buckets": dict(histogram["buckets"]), + "sum": float(histogram["sum"]), + "count": int(histogram["count"]), + } + for result, histogram in histograms.items() + } + + def dropped_solves_snapshot(self) -> dict[str, int]: + """Copy the bounded-policy dropped block-solve counters.""" + + with self._metrics_lock: + return dict(self.block_solves_dropped_counts) + + def replace_dropped_solves(self, counts: Mapping[str, int]) -> None: + with self._metrics_lock: + self.block_solves_dropped_counts = { + str(reason): int(count) for reason, count in counts.items() + } + + def _count_dropped_block_solve(self, reason: str) -> None: + with self._metrics_lock: + self.block_solves_dropped_counts[reason] = ( + int(self.block_solves_dropped_counts.get(reason, 0)) + 1 + ) + + def _reject(self, rejected: SubmitRejected, *, worker: str | None) -> NoReturn: + self.ports.reject(rejected, worker) + + def _context_decision( + self, + client: ClientState, + request: SubmitRequest, + control: SubmitControlSnapshot, + ) -> SubmitContextDecision: + active = control.active_context + retained = ( + self.ports.retained_entry(client, request.job_id) + if active is None + else None + ) + if active is None and retained is None: + # Reject before any live-tip fallback: an unknown job needs no + # RPC to classify. + try: + return classify_submit_context( + SubmitContextInput( + active_context=None, + retained_entry=None, + current_tip="", + stale_grace_eligible=False, + ) + ) + except SubmitRejected as rejected: + self._reject(rejected, worker=request.worker_name) + current_tip = ( + control.published_tip + if control.published_tip is not None + else self.ports.live_tip() + ) + # Share classification (normal and stale-grace alike) is deliberately + # point-in-time against this single tip read: a tip that advances + # between here and the ledger append does not retroactively invalidate + # the share, exactly as a normal current-tip share stays credited when + # the tip moves during processing. Re-checking would add an RPC per + # share during post-block bursts only to reject valid work over + # processing latency. Block submission is different (chain state): + # submit_block_candidate re-checks the tip under lock before + # submitblock, and stale-grace shares never reach it. + context = active if active is not None else retained.context + cross_connection = ( + retained is not None + and retained.connection_id != client.connection_id + ) + stale_eligible = False + if ( + not cross_connection + and str(context.template["previousblockhash"]) != current_tip + ): + # A cross-connection retained entry never consults the window: it + # is same-tip only, so the mismatch rejects as stale below + # without any parent-tip RPC (matching evicted_submit_context). + try: + stale_eligible = self.ports.stale_grace_eligible( + client, + context, + current_tip, + ) + except Exception: + self.ports.log( + "prism coordinator: failed to classify evicted submit context" + if retained is not None + else "prism coordinator: failed to classify stale-grace parent tip" + ) + self.ports.log_exception() + self._reject( + SubmitRejected( + 20, + PRISM_REJECTION_BACKEND_RPC_UNAVAILABLE, + "failed to classify stale-grace parent tip", + ), + worker=request.worker_name, + ) + try: + return classify_submit_context( + SubmitContextInput( + active_context=active, + retained_entry=retained, + current_tip=current_tip, + stale_grace_eligible=stale_eligible, + cross_connection=cross_connection, + ) + ) + except SubmitRejected as rejected: + self._reject(rejected, worker=request.worker_name) + + def handle(self, client: ClientState, params: list[object]) -> bool: + try: + request = parse_submit_request(params) + except SubmitRejected as rejected: + self._reject(rejected, worker=client.username or None) + # Capture pool admission, active-job membership, and published-tip + # authority together. All accounting, RPC fallback, hashing, and + # persistence remain outside the coordinator lock behind this port. + control = self.ports.control_snapshot(client, request.job_id) + try: + validate_submit_request( + request, + authorized_username=client.username, + pool_open=control.pool_open, + extranonce2_size=self.extranonce2_size, + ) + except SubmitRejected as rejected: + worker = ( + client.username or None + if rejected.reason == PRISM_REJECTION_UNAUTHORIZED_WORKER + else request.worker_name + ) + self._reject(rejected, worker=worker) + + # Count submitted shares once, after the format checks, so the + # per-worker counter and the aggregate submitted total cover the same + # population; malformed extranonce/ntime submits are recorded only as + # rejections, not submits. + self.ports.note_submitted(request.worker_name, client) + decision = self._context_decision(client, request, control) + submit_version_mask = client.version_mask + if decision.cross_connection: + # In-flight work from the dead connection was rolled under the + # mask negotiated there; the replacement's own mask (often still + # 0 before mining.configure) must not judge those version bits. + submit_version_mask = int( + getattr(decision.context, "version_mask", 0) + ) + try: + submission = self.ports.assemble( + client, + decision.context, + request, + submit_version_mask, + ) + except ValueError as error: + self._reject( + SubmitRejected( + 20, + PRISM_REJECTION_MALFORMED_SUBMIT, + f"malformed submit: {error}", + ), + worker=request.worker_name, + ) + # A retained job keeps its original worker even if the connection is + # later re-authorized. Deduplication must use that immutable identity: + # otherwise the same header can be replayed under each new username. + share_key = (decision.context.worker.username, submission.header_hex) + if not self.recent_shares.reserve(share_key): + self._reject( + SubmitRejected( + 22, + PRISM_REJECTION_DUPLICATE_SHARE, + "duplicate share", + ), + worker=request.worker_name, + ) + if ( + submission.block_pass + and decision.credit_policy == PRISM_CREDIT_POLICY_STALE_GRACE + ): + # Count the dropped solve before low-difficulty classification: + # the drop is real whether the stale-grace share passes its share + # target (credited below) or misses it (rejected below). + self._count_dropped_block_solve("stale_grace") + self.ports.log( + "prism coordinator: stale-grace block solve dropped " + f"hash={submission.block_hash_hex} " + f"parent={decision.context.template['previousblockhash']}" + ) + try: + work = classify_submit_work( + decision.context, + submission, + credit_policy=decision.credit_policy, + ) + except SubmitRejected as rejected: + self._reject(rejected, worker=request.worker_name) + + if work.block_worthy and decision.context.collection_only: + # Collection-mode jobs are block-worthy too: their signed + # bootstrap manifest already commits the whole coinbase to the + # submitting worker, so the solve settles solver-pays-all. + self.ports.note_collection_candidate(decision.context, submission) + pending_share = self.ports.pending_share( + decision.context, + submission, + request.ntime_hex, + decision.credit_policy, + ) + if work.route == "share": + try: + self.ports.append_share( + client, + decision.context, + submission, + pending_share, + decision.credit_policy, + None, + ) + if decision.retained_entry is not None: + self.ports.note_retained_submit( + decision.credit_policy, + decision.cross_connection, + ) + except BaseException: + self.recent_shares.release(work.share_key) + raise + return False + + candidate = PrismBlockCandidate( + context=decision.context, + submission=submission, + # The mined coinbase embeds the extranonce1 the job was stamped + # with. A cross-connection resume submits through a client whose + # own extranonce1 differs from the retained job's, and the audit + # bundle suffix must match the coinbase actually in the block, + # so the job's value is authoritative whenever it carries one. + extranonce1_hex=str( + getattr(decision.context.job, "extranonce1_hex", None) + or client.extranonce1_hex + ), + extranonce2_hex=request.extranonce2_hex, + pending_share=pending_share, + client=client, + credit_share_on_accept=work.credit_share_on_accept, + ) + if work.route == "synchronous_block": + return self._submit_synchronous( + candidate, + work=work, + request=request, + decision=decision, + ) + return self._submit_asynchronous( + candidate, + work=work, + client=client, + decision=decision, + submission=submission, + ) + + def _submit_synchronous( + self, + candidate: PrismBlockCandidate, + *, + work: SubmitWorkDecision, + request: SubmitRequest, + decision: SubmitContextDecision, + ) -> bool: + # The hash solved a block but missed the assigned share target + # (possible only while the listener floor sits above network + # difficulty). It is a valid share ONLY if the block lands, so it + # lands synchronously behind this port: the durable-outbox boundary, + # the terminal-state short circuit, and the low-difficulty rejection + # for a candidate that does not land all run coordinator-side where + # the block-submitter machinery lives. + return self.ports.submit_synchronous_candidate( + candidate, + work.share_key, + request.worker_name, + decision.retained_entry, + decision.credit_policy, + ) + + def _submit_asynchronous( + self, + candidate: PrismBlockCandidate, + *, + work: SubmitWorkDecision, + client: ClientState, + decision: SubmitContextDecision, + submission: Any, + ) -> bool: + # A block-worthy submission that met the share target is a valid + # share regardless of the block's fate: credit it now, acknowledge + # the miner immediately, and land the block from the dedicated + # submitter thread (ckpool/btcpool/StratumV2 semantics). An orphaned + # candidate keeps its share credit. + try: + candidate_intent = self.ports.candidate_intent(candidate) + durable_candidate_state = self.ports.append_share( + client, + decision.context, + submission, + candidate.pending_share, + decision.credit_policy, + candidate_intent, + ) + if decision.retained_entry is not None: + self.ports.note_retained_submit( + decision.credit_policy, + decision.cross_connection, + ) + except BaseException: + # Idempotent with the append's own release; also covers an intent + # serialization failure before the append started. + self.ports.finish_pending_commit(candidate.pending_share) + self.recent_shares.release(work.share_key) + raise + if durable_candidate_state in {"submitted", "abandoned"}: + # A process restart clears the in-memory disposition cache, but + # the existing outbox row remains authoritative. Join its + # terminal result instead of enqueueing a new node offer. + self.ports.record_terminal_outcome( + submission.block_hash_hex, + durable_candidate_state == "submitted", + ) + return False + self.ports.enqueue_candidate(candidate) + return False diff --git a/lab/prism/share_writer.py b/lab/prism/share_writer.py new file mode 100644 index 000000000..964f9c37e --- /dev/null +++ b/lab/prism/share_writer.py @@ -0,0 +1,831 @@ +#!/usr/bin/env python3 +"""Durable accepted-share writer, pending-commit floor, and legacy recovery. + +This module owns the S3 domain: the bounded group-commit queue and writer +loop, the pending-share snapshot-anchor floor, disk recovery of acked shares +the ledger could not persist, and startup replay of that recovery journal. + +It never imports ``prism_coordinator``. The ledger handle, shutdown +controller, watchdog heartbeats, the P1 append/anchor fencing entrypoints, +and live configuration attributes are reached through the +:class:`ShareWriterRuntime` typed port, resolved at call time so the +historical coordinator monkeypatch seams (including the instance-level facade +patches used by the current test suite) keep intercepting exactly as before +the extraction. Replay semantics deliberately keep the current tree's +behavior; the typed exact/conflict recovery consumption from the share +ledger arrives with a later chunk through the ``replay_recovered_shares`` +seam below. +""" + +from __future__ import annotations + +import dataclasses +from dataclasses import dataclass, field +import json +import os +import queue +import threading +import time +import traceback +from typing import Any, Protocol + +from lab.prism.share_ledger import PendingShare, ShareReplayConflict +from lab.prism.stratum_session import StratumError + + +MAX_PENDING_SHARE_APPENDS = 4_096 +# After this long on the pending-commit floor a share is assumed wedged (not +# normal group-commit latency) and the anchor clamp logs it once. +PENDING_SHARE_COMMIT_WARN_SECONDS = 30.0 +_WRITER_EXIT_COMPONENTS = frozenset( + {"share_submission", "share_persistence", "accepted_block_handling"} +) + + +class ShareWriterError(RuntimeError): + """A persistence operation failed before it could be acknowledged.""" + + +class ShareWriterQueueFull(ShareWriterError): + """The bounded share group-commit queue rejected an entry.""" + + +@dataclass +class PendingShareAppend: + """A share waiting for the ledger group-commit writer. + + The client thread does not count or acknowledge this share until + ``committed`` is set successfully. A block candidate intent, when + present, is inserted in the same transaction as the share. + """ + + pending_share: PendingShare + username: str + job_id: str + block_hash_hex: str + collection_only: bool + credit_policy: str | None + # The prepared candidate intent (#255), a mapping whose share sequence + # is the job's immutable window. Released by the writer the moment the + # entry reaches its terminal outcome so an idle writer local or a + # finished batch never pins a payout window. + candidate_intent: Any | None = None + committed: threading.Event = field(default_factory=threading.Event) + record: Any | None = None + error: BaseException | None = None + writer_token: "_WriterOperationToken | None" = None + + +class ShareWriterRuntime(Protocol): + """Typed port over the coordinator, resolved at call time. + + Every member is looked up on the live coordinator object when used, so + instance monkeypatches (``server._finish_pending_share_commit = ...``, + ``server._append_share_batch = ...`` and friends) and coordinator-owned + live state (the ledger handle, the P1 append/anchor fencing) keep working + exactly as before the extraction. Owner facades route back into this + service; that round trip is deliberate. The legacy S3 field names + (``share_append_queue`` through ``_pending_share_commit_floor``) also + resolve here -- coordinator class descriptors route them to this + service's single mutable copy. + """ + + # Cross-domain objects and live configuration attributes. + hot_path_log_enabled: Any + ledger: Any + stop_event: Any + + def _append_share_batch(self, *args: Any, **kwargs: Any) -> Any: ... + + def _append_share_entry(self, *args: Any, **kwargs: Any) -> Any: ... + + def _ensure_share_hot_path_state(self, *args: Any, **kwargs: Any) -> Any: ... + + def _ensure_shutdown_controller(self, *args: Any, **kwargs: Any) -> Any: ... + + def _finish_pending_share_commit(self, *args: Any, **kwargs: Any) -> Any: ... + + def _landing_fence_for_predating_append(self, *args: Any, **kwargs: Any) -> Any: ... + + def _record_heartbeat(self, *args: Any, **kwargs: Any) -> Any: ... + + def _record_late_visible_payout_append(self, *args: Any, **kwargs: Any) -> Any: ... + + def _recover_share_to_disk(self, *args: Any, **kwargs: Any) -> Any: ... + + def _retire_payout_windows_for_late_append(self, *args: Any, **kwargs: Any) -> Any: ... + + def accepted_share_difficulty(self, *args: Any, **kwargs: Any) -> Any: ... + + def enqueue_share_append(self, *args: Any, **kwargs: Any) -> Any: ... + + def note_vardiff_accepted_share(self, *args: Any, **kwargs: Any) -> Any: ... + + def note_worker_accepted_share(self, *args: Any, **kwargs: Any) -> Any: ... + + +class ShareWriter: + """Sole owner of S3 group commit, the pending floor, and recovery.""" + + def __init__( + self, + runtime: ShareWriterRuntime, + *, + internal_error_reason: str, + now_ms: Any, + ) -> None: + self._runtime = runtime + # The rejection-reason id remains a coordinator registry constant and + # the wall-clock stamp resolves the coordinator module's ``now_ms`` + # global at call time (tests patch it there); both are injected so + # this leaf module never imports prism_coordinator. + self._internal_error_reason = internal_error_reason + self._now_ms = now_ms + # Accepted shares drain through a bounded group-commit writer. A + # submitting client waits on its entry's completion event, making the + # database commit the acknowledgement boundary without paying one + # process/transaction round trip per share during bursts. + self.share_append_queue: queue.Queue[PendingShareAppend] = queue.Queue( + maxsize=MAX_PENDING_SHARE_APPENDS + ) + self.share_commit_batch_size = 64 + self.share_commit_linger_seconds = 0.005 + self.share_commit_timeout_seconds = 15.0 + self.share_writer_active = False + self.share_append_failure_count = 0 + # Retain the historical recovery-file reader for clean upgrades from a + # release that could acknowledge before Postgres commit. New shares + # are never written here: an unavailable ledger produces no success + # acknowledgement and an exact retry is idempotent. + self.share_recovery_path = None + self.share_recovery_lock = threading.Lock() + self.shares_recovered_to_disk = 0 + self.shares_replayed = 0 + self.share_replay_conflicts = 0 + self._pending_share_commit_lock = threading.Lock() + # Shares whose accepted_at_ms has been assigned but whose ledger + # row has not reached a terminal outcome in this process. Job and + # payout-artifact snapshot anchors clamp below every entry so a + # bundle's eligible-share set stays exactly reproducible from the + # durable ledger after those rows commit. + self._pending_share_commit_floor: dict[int, list[object]] = {} + + # -- pending-commit floor ---------------------------------------------- + + def finish_pending_share(self, pending_share: PendingShare) -> None: + """Drop a share from the snapshot anchor floor. + + Called once the share's ledger row reached a terminal outcome in this + process: durably committed, rejected back to the miner, recovered to + the on-disk replay file, or its block candidate terminally abandoned. + Idempotent, and a no-op for shares this process never registered + (intent/journal replays re-create PendingShare objects from JSON). + """ + with self._pending_share_commit_lock: + self._pending_share_commit_floor.pop(id(pending_share), None) + + def finish_pending_attempt(self, pending_share: PendingShare) -> None: + """Release one stamped submission attempt's floor authority. + + Layer-original seam: at this layer the floor keeps the current + tree's single object-identity holder per stamped share, so attempt + release and candidate release act on the same entry. A later chunk + introduces independent attempt/durable-candidate/actor holders behind + these two adapters without touching their callers. + """ + self.finish_pending_share(pending_share) + + def finish_pending_candidate(self, pending_share: PendingShare) -> None: + """Release the durable block-candidate actor's floor authority. + + See :meth:`finish_pending_attempt` for the staged holder split. + """ + self.finish_pending_share(pending_share) + + def adopt_pending_share(self, pending_share: PendingShare) -> None: + """Register an externally reconstructed share on the pending floor. + + Durable block-candidate replay decodes a credit-bearing PendingShare + whose accepted stamp predates this process. Adopting it here clamps + job/payout snapshot anchors below that stamp before the replayed + candidate becomes visible to issuance, exactly as if the share had + been stamped live. Released through the ordinary + :meth:`finish_pending_candidate` seam once the candidate reaches a + terminal outcome. + """ + with self._pending_share_commit_lock: + self._pending_share_commit_floor[id(pending_share)] = [ + pending_share, + time.monotonic(), + False, + ] + + def snapshot_anchor_ms(self, issued_at_ms: int) -> int: + """Clamp a share-snapshot anchor below every coverable share stamp. + + The reward-window contract lets an auditor replay + qbit_audit_share_window(anchor) against the durable ledger and expect + exactly the shares the published bundle counted. Two hazards would + violate that. A share whose accepted_at_ms is already assigned but + whose row has not committed yet (group-commit queue, in-flight batch, + or a block-candidate credit linked after landing) is invisible to the + MVCC snapshot now but joins later replays at any anchor at or above + its accepted_at_ms. And a share stamped right after this clamp + returns can land in the same millisecond as the clamp instant -- + never protected by the pending floor -- so an anchor equal to that + instant would cover it too (the window predicate is + anchor-inclusive). Anchoring strictly below every pending share and + strictly below the clamp-time millisecond keeps the issued snapshot + reproducible without making job builds wait behind the writer + connection: any stamp assigned after this call is at or above the + clamp instant, hence above the anchor. + """ + stale_share_ids: list[str] = [] + floor_ms: int | None = None + now_monotonic = time.monotonic() + with self._pending_share_commit_lock: + for entry in self._pending_share_commit_floor.values(): + share = entry[0] + accepted_at_ms = int(share.accepted_at_ms) + if floor_ms is None or accepted_at_ms < floor_ms: + floor_ms = accepted_at_ms + if ( + not entry[2] + and now_monotonic - float(entry[1]) + > PENDING_SHARE_COMMIT_WARN_SECONDS + ): + entry[2] = True + stale_share_ids.append(str(share.share_id)) + for share_id in stale_share_ids: + # A long-held floor entry is a wedged writer or a leaked release + # path, not normal group-commit latency. The share-commit liveness + # watchdog owns recovery; this log makes the anchor clamp visible. + print( + "prism coordinator: pending share commit is holding the job " + f"snapshot anchor floor share_id={share_id}", + flush=True, + ) + if floor_ms is None: + return issued_at_ms - 1 + return min(issued_at_ms - 1, floor_ms - 1) + + def pending_share_from_submission( + self, + *, + context: Any, + submission: Any, + ntime_hex: str, + credit_policy: str | None = None, + ) -> PendingShare: + runtime = self._runtime + share_difficulty = runtime.accepted_share_difficulty(context) + # Assign accepted_at_ms and register the commit-floor entry under one + # lock hold: a snapshot anchored between assignment and registration + # could otherwise anchor at or above this share and miss its later + # commit. Released via _finish_pending_share_commit. + with self._pending_share_commit_lock: + pending = PendingShare( + share_id=f"{context.worker.username}:{submission.block_hash_hex}", + miner_id=context.worker.payout_address, + order_key=context.worker.payout_address, + p2mr_program_hex=context.worker.p2mr_program_hex, + share_difficulty=share_difficulty, + network_difficulty=max(1, int(context.found_block["network_difficulty"])), + template_height=int(context.template["height"]) - 1, + job_id=context.job.job_id, + job_issued_at_ms=context.issued_at_ms, + accepted_at_ms=self._now_ms(), + ntime=int(ntime_hex, 16), + credit_policy=credit_policy, + ) + self._pending_share_commit_floor[id(pending)] = [ + pending, + time.monotonic(), + False, + ] + return pending + + # -- append paths ------------------------------------------------------ + + def append_accepted_share( + self, + client: Any, + context: Any, + submission: Any, + pending_share: PendingShare, + *, + credit_policy: str | None = None, + candidate_intent: dict[str, Any] | None = None, + ) -> str | None: + runtime = self._runtime + entry = PendingShareAppend( + pending_share=pending_share, + username=context.worker.username, + job_id=context.job.job_id, + block_hash_hex=submission.block_hash_hex, + collection_only=bool(context.collection_only), + credit_policy=credit_policy, + candidate_intent=candidate_intent, + ) + try: + if getattr(self, "share_writer_active", False): + runtime.enqueue_share_append(entry, wait=True) + else: + runtime._append_share_entry(entry) + finally: + # The append reached a terminal outcome for this process: durably + # committed, or surfaced an error the miner will retry with a fresh + # share. Either way the stamped share no longer holds the snapshot + # anchor floor. Idempotent with the group-commit writer's release. + runtime._finish_pending_share_commit(pending_share) + record = entry.record + # Exact ledger replays return the original record. Their durable share + # must not increment process-local worker/vardiff counters again. + if record is None or bool(getattr(record, "newly_inserted", True)): + runtime.note_worker_accepted_share(context.worker.username, credit_policy) + runtime.note_vardiff_accepted_share(client, context.job) + if candidate_intent is None or record is None: + return None + candidate_state = getattr(record, "candidate_outbox_state", None) + return str(candidate_state) if candidate_state is not None else None + + def enqueue_share_append( + self, entry: PendingShareAppend, *, wait: bool = False + ) -> None: + runtime = self._runtime + queue_obj = getattr(self, "share_append_queue", None) + if queue_obj is None: + queue_obj = queue.Queue(maxsize=MAX_PENDING_SHARE_APPENDS) + self.share_append_queue = queue_obj + if entry.writer_token is None: + entry.writer_token = runtime._ensure_shutdown_controller().reserve_writer( + "share_persistence" + ) + try: + if wait: + queue_obj.put( + entry, + timeout=getattr(self, "share_commit_timeout_seconds", 15.0), + ) + else: + queue_obj.put_nowait(entry) + except queue.Full: + entry.writer_token.finish() + entry.writer_token = None + raise StratumError( + 20, + "share ledger commit queue is full", + reason=self._internal_error_reason, + ) + if not wait: + return + # Once admitted, wait for a definite transaction outcome. A local + # timeout is ambiguous because Postgres may commit immediately after + # it; the liveness watchdog owns recovery from a wedged writer. + entry.committed.wait() + if entry.error is not None: + raise StratumError( + 20, + f"share ledger commit failed: {entry.error}", + reason=self._internal_error_reason, + ) + + def share_append_loop(self) -> None: + runtime = self._runtime + while True: + runtime._record_heartbeat("share_writer") + queue_obj = getattr(self, "share_append_queue", None) + if queue_obj is None: + queue_obj = queue.Queue(maxsize=MAX_PENDING_SHARE_APPENDS) + self.share_append_queue = queue_obj + stopping = runtime.stop_event.is_set() + try: + entry = queue_obj.get(timeout=0.2 if stopping else 1.0) + except queue.Empty: + controller = runtime._ensure_shutdown_controller() + if ( + stopping + and controller.writer_admission_closed() + and not controller.has_active_writer(set(_WRITER_EXIT_COMPONENTS)) + ): + return + continue + batch = [entry] + batch_size = max(1, int(getattr(self, "share_commit_batch_size", 64))) + deadline = time.monotonic() + max( + 0.0, float(getattr(self, "share_commit_linger_seconds", 0.005)) + ) + if entry.candidate_intent is not None: + deadline = time.monotonic() + while len(batch) < batch_size: + remaining = deadline - time.monotonic() + if remaining <= 0: + break + try: + next_entry = queue_obj.get(timeout=remaining) + batch.append(next_entry) + if next_entry.candidate_intent is not None: + break + except queue.Empty: + break + runtime._append_share_batch(batch) + + def append_share_batch(self, batch: list[PendingShareAppend]) -> bool: + """Commit a writer batch, then release every waiting submitter.""" + runtime = self._runtime + try: + invalidations: list[tuple[PendingShare, int]] = [] + # A batch holding a replay-shaped row commits under the landing + # fence: the durable append and its epoch bump must land on the + # same side of a landing's fence-guarded submit, or the landing + # can verify the pre-bump epoch after the row is already durable + # (see _landing_fence_for_predating_append). + with runtime._landing_fence_for_predating_append( + [entry.pending_share for entry in batch] + ) as fence_owned: + append_batch = getattr(runtime.ledger, "append_batch", None) + if callable(append_batch): + records = append_batch( + [(entry.pending_share, entry.candidate_intent) for entry in batch] + ) + else: + # Compatibility for lightweight test/tool ledgers. Production's + # Postgres ledger always supplies the atomic batch method. + records = [runtime.ledger.append(entry.pending_share) for entry in batch] + if len(records) != len(batch): + raise RuntimeError("share ledger returned an incomplete commit batch") + for entry, record in zip(batch, records, strict=True): + entry.record = record + invalidation_epoch = runtime._record_late_visible_payout_append( + entry.pending_share, + landing_fence_owned=fence_owned, + ) + if invalidation_epoch is not None: + invalidations.append( + (entry.pending_share, invalidation_epoch) + ) + for pending_share, invalidation_epoch in invalidations: + runtime._retire_payout_windows_for_late_append( + pending_share, invalidation_epoch + ) + if getattr(runtime, "hot_path_log_enabled", False): + for entry in batch: + record = entry.record + print( + "prism coordinator: accepted share " + f"seq={record.share_seq} miner={entry.username} job={entry.job_id} " + f"hash={entry.block_hash_hex} collection={entry.collection_only} " + f"credit_policy={entry.credit_policy or 'normal'}", + flush=True, + ) + return True + except Exception as exc: + runtime._ensure_share_hot_path_state() + with runtime._share_accounting_lock: + self.share_append_failure_count = ( + int(getattr(self, "share_append_failure_count", 0)) + len(batch) + ) + for entry in batch: + entry.error = exc + print( + f"prism coordinator: share ledger group commit failed count={len(batch)}", + flush=True, + ) + traceback.print_exc() + return False + finally: + for entry in batch: + runtime._finish_pending_share_commit(entry.pending_share) + # Terminal for this entry either way (committed, or failed + # and reported to the miner): drop the intent's hold on its + # window before waking the submitter. + entry.candidate_intent = None + entry.committed.set() + if entry.writer_token is not None: + entry.writer_token.finish() + entry.writer_token = None + + def recover_share_to_disk(self, entry: PendingShareAppend, reason: str) -> None: + """Durably capture an acked share the writer could not persist. + + Appends the canonical pending-share JSON to the recovery file (fsynced) + so a ledger outage or shutdown never silently loses a share the miner + was told was accepted; replayed on the next start. Best-effort: if even + the recovery write fails, log loudly rather than raise on the writer. + """ + path = getattr(self, "share_recovery_path", None) + if path is None: + print( + "prism coordinator: WOULD LOSE acked share (no recovery path) " + f"share_id={entry.pending_share.share_id} reason={reason}", + flush=True, + ) + return + try: + payload = json.dumps(dataclasses.asdict(entry.pending_share), separators=(",", ":")) + except Exception: + payload = None + with getattr(self, "share_recovery_lock", threading.Lock()): + try: + path.parent.mkdir(parents=True, exist_ok=True) + if payload is None: + raise ValueError("pending share is not serializable") + with open(path, "a", encoding="utf-8") as handle: + handle.write(payload + "\n") + handle.flush() + os.fsync(handle.fileno()) + self.shares_recovered_to_disk = ( + int(getattr(self, "shares_recovered_to_disk", 0)) + 1 + ) + print( + "prism coordinator: recovered unpersisted acked share to disk " + f"share_id={entry.pending_share.share_id} reason={reason}", + flush=True, + ) + except Exception: + print( + "prism coordinator: FAILED to recover acked share to disk; " + f"share may be lost share_id={entry.pending_share.share_id} reason={reason}", + flush=True, + ) + traceback.print_exc() + + def replay_recovered_shares(self) -> int: + """Replay any recovery-file shares into the ledger at startup. + + Ledgers with typed recovery support report each row's disposition + through ``share_ledger.append_recovered_share``: an ``inserted`` row + counts as replayed, an ``exact_existing`` row is an already-committed + duplicate from an earlier partial replay (skipped, not + double-counted). A :class:`ShareReplayConflict` is keyed to one + share_id, so the conflicting row is quarantined -- counted, logged, + and preserved in the retained journal -- while the remaining rows + keep replaying instead of stranding behind it; an unknown disposition + conservatively stops and keeps the journal file for inspection. Only + a clean pass may remove the journal, so a transient failure here + never drops shares. Embedder ledgers without ``append_recovered_share`` keep the + historical ``append`` path whose duplicate-share_id error is treated + as an exact replay. Writer admission stays with the coordinator + facade's ``ledger_writer_operation`` decorator. + """ + runtime = self._runtime + path = getattr(self, "share_recovery_path", None) + if path is None or not path.exists(): + return 0 + try: + lines = [line for line in path.read_text(encoding="utf-8").splitlines() if line.strip()] + except Exception: + print("prism coordinator: could not read share recovery file", flush=True) + traceback.print_exc() + return 0 + # Parse line-by-line and skip any single unparseable line rather than + # aborting the whole replay: a crash mid-append can leave the last line + # torn, and one torn line must not block the intact shares before it. + pendings: list[PendingShare] = [] + parse_failed = False + for line in lines: + try: + pendings.append(PendingShare(**json.loads(line))) + except Exception: + parse_failed = True + print("prism coordinator: skipping an unparseable recovered share line", flush=True) + traceback.print_exc() + # Replay in acceptance order. A share recovered out of FIFO order (a + # ledger flap during the shutdown drain, or an overflow-recovered newest + # share) otherwise sorts by file order; ordering by accepted_at_ms lands + # each share with a share_seq consistent with when it was accepted, so + # the reward window stays correctly ordered. + pendings.sort(key=lambda pending: pending.accepted_at_ms) + replayed = 0 + skipped_duplicates = 0 + replay_conflicts = 0 + for pending in pendings: + append_recovered = getattr( + runtime.ledger, "append_recovered_share", None + ) + try: + # A recovered row reconstructs its pre-crash timestamps, so + # it routinely predates a live anchor; each row commits under + # the landing fence with its epoch bump so a concurrent + # landing can never verify a pre-bump epoch against an + # already-durable replayed share. The window retirement runs + # after the fence releases, per row, so replay never holds + # the fence across the prepare-lock wait. + invalidation_epoch: int | None = None + with runtime._landing_fence_for_predating_append( + [pending] + ) as fence_owned: + if callable(append_recovered): + result = append_recovered(pending) + disposition = getattr(result, "disposition", None) + if disposition == "exact_existing": + # Already committed by an earlier (partial) + # replay; treat it as done so a retry never + # stops on the first committed row and strands + # every share after it. The durable row already + # bumped visibility on its original commit, so + # no new epoch bump happens here. + skipped_duplicates += 1 + continue + if disposition != "inserted": + print( + "prism coordinator: recovery ledger returned an " + f"unsupported replay disposition {disposition!r}; " + "keeping the file", + flush=True, + ) + self.shares_replayed = ( + int(getattr(self, "shares_replayed", 0)) + replayed + ) + return replayed + else: + runtime.ledger.append(pending) + invalidation_epoch = runtime._record_late_visible_payout_append( + pending, + landing_fence_owned=fence_owned, + ) + if invalidation_epoch is not None: + runtime._retire_payout_windows_for_late_append( + pending, invalidation_epoch + ) + replayed += 1 + except ShareReplayConflict: + # The durable row disagrees with the journal payload. The + # conflict is keyed to this one share_id and says nothing + # about the rows after it, so quarantine the row -- count it + # and keep the file for inspection -- and continue replaying + # the rest of the journal instead of stranding valid shares + # behind it. Exact duplicates replay idempotently, so a + # re-run of the retained file never double-credits. + replay_conflicts += 1 + self.share_replay_conflicts = ( + int(getattr(self, "share_replay_conflicts", 0)) + 1 + ) + print( + "prism coordinator: recovered share conflicts with durable " + "payload; quarantining the row and keeping the file " + f"share_id={pending.share_id}", + flush=True, + ) + continue + except Exception as exc: + if not callable(append_recovered) and "duplicate share_id" in str(exc): + # Historical string-matched idempotence for embedder + # ledgers without typed replay support. + skipped_duplicates += 1 + continue + print("prism coordinator: failed to replay a recovered share; keeping the file", flush=True) + traceback.print_exc() + self.shares_replayed = int(getattr(self, "shares_replayed", 0)) + replayed + return replayed + if skipped_duplicates: + print( + f"prism coordinator: skipped {skipped_duplicates} already-committed " + "recovered share(s) during replay", + flush=True, + ) + if replay_conflicts: + print( + f"prism coordinator: quarantined {replay_conflicts} conflicting " + "recovered share(s) during replay; keeping the file", + flush=True, + ) + if parse_failed or replay_conflicts: + # Keep the file (with its intact-but-already-replayed lines, which + # the ledger dedups on a re-run) so the torn or conflicting line is + # preserved for inspection rather than silently discarded. + self.shares_replayed = int(getattr(self, "shares_replayed", 0)) + replayed + if replayed: + print(f"prism coordinator: replayed {replayed} recovered share(s) into the ledger", flush=True) + return replayed + try: + path.unlink() + except FileNotFoundError: + pass + self.shares_replayed = int(getattr(self, "shares_replayed", 0)) + replayed + if replayed: + print(f"prism coordinator: replayed {replayed} recovered share(s) into the ledger", flush=True) + return replayed + + def append_share_entry( + self, entry: PendingShareAppend, *, retry_until_stopped: bool = False + ) -> bool: + """Synchronously append one accepted share. + + On the writer thread a transient ledger failure retries with capped + backoff so ordering is preserved and nothing is silently lost; the + synchronous path (no writer) propagates the exception exactly as the + pre-async code did. + + Returns True when the share was persisted to the ledger, or False when + it was recovered to disk instead (ledger still down at shutdown). The + caller uses that to keep the shutdown drain in order. + """ + runtime = self._runtime + backoff_seconds = 0.5 + invalidation_epoch: int | None = None + while True: + try: + # A replay-shaped row commits under the landing fence with + # its epoch bump (see _landing_fence_for_predating_append). + # The fence wraps one attempt only: the backoff sleep below + # runs outside it, so a retrying ledger outage never holds + # up a landing. + with runtime._landing_fence_for_predating_append( + [entry.pending_share] + ) as fence_owned: + append_batch = getattr(runtime.ledger, "append_batch", None) + if callable(append_batch): + record = append_batch( + [(entry.pending_share, entry.candidate_intent)] + )[0] + else: + record = runtime.ledger.append(entry.pending_share) + entry.record = record + invalidation_epoch = runtime._record_late_visible_payout_append( + entry.pending_share, + landing_fence_owned=fence_owned, + ) + break + except Exception: + if not retry_until_stopped: + raise + runtime._ensure_share_hot_path_state() + with runtime._share_accounting_lock: + self.share_append_failure_count = ( + int(getattr(self, "share_append_failure_count", 0)) + 1 + ) + print( + "prism coordinator: ledger share append failed; retrying " + f"share_id={entry.pending_share.share_id}", + flush=True, + ) + traceback.print_exc() + if runtime.stop_event.wait(backoff_seconds): + # Shutting down mid-outage: do not silently drop this + # already-acked, already-counted share -- recover it to + # disk for replay on the next start. + runtime._recover_share_to_disk(entry, "ledger unavailable at shutdown") + entry.candidate_intent = None + return False + backoff_seconds = min(backoff_seconds * 2, 5.0) + runtime._record_heartbeat("share_writer") + entry.candidate_intent = None + if invalidation_epoch is not None: + runtime._retire_payout_windows_for_late_append( + entry.pending_share, invalidation_epoch + ) + if getattr(runtime, "hot_path_log_enabled", False): + print( + "prism coordinator: accepted share " + f"seq={record.share_seq} miner={entry.username} job={entry.job_id} " + f"hash={entry.block_hash_hex} collection={entry.collection_only} " + f"credit_policy={entry.credit_policy or 'normal'}", + flush=True, + ) + entry.committed.set() + return True + + +class ShareWriterCompatibilityField: + """Route one legacy S3 coordinator field to its owner service. + + The descriptor keeps the historical attribute name readable and writable + on the coordinator while :class:`ShareWriter` owns the single mutable + copy, mirroring the S1/S2/J1/G1 extraction pattern. Direct test + assignment through the legacy name is thereby adopted by the owner. + """ + + def __init__(self, attribute: str) -> None: + self._attribute = attribute + + def __get__( + self, + instance: Any, + owner: type | None = None, + ) -> Any: + if instance is None: + return self + return getattr( + instance._ensure_share_writer_service(), + self._attribute, + ) + + def __set__(self, instance: Any, value: Any) -> None: + setattr( + instance._ensure_share_writer_service(), + self._attribute, + value, + ) + + +__all__ = [ + "MAX_PENDING_SHARE_APPENDS", + "PENDING_SHARE_COMMIT_WARN_SECONDS", + "PendingShareAppend", + "ShareWriter", + "ShareWriterCompatibilityField", + "ShareWriterError", + "ShareWriterQueueFull", + "ShareWriterRuntime", +] diff --git a/lab/prism/statement_spool.py b/lab/prism/statement_spool.py new file mode 100644 index 000000000..b66cf9062 --- /dev/null +++ b/lab/prism/statement_spool.py @@ -0,0 +1,105 @@ +"""Execute streamed finalization statements with bounded coordinator memory. + +The existing atomic fenced SQL is unchanged. psql owns its unavoidable full +statement buffer in an isolated process; the coordinator passes an open file, +never the complete SQL string, and reads only a bounded result after success. +""" + +from __future__ import annotations + +import os +import subprocess +import sys +import tempfile +import time +from collections.abc import Iterable +from typing import Any + + +STATEMENT_CHUNK_CHARS = 4096 +STATEMENT_SPOOL_BYTES = 4 * 1024 * 1024 * 1024 +STATEMENT_HELPER_MEMORY_BYTES = 4 * 1024 * 1024 * 1024 +STATEMENT_RESULT_BYTES = 64 * 1024 +STATEMENT_HELPER_TIMEOUT_SECONDS = 600.0 +_EXEC_LIMITED = """ +import os, resource, sys +resource.setrlimit(resource.RLIMIT_AS, (int(sys.argv[1]), int(sys.argv[1]))) +resource.setrlimit(resource.RLIMIT_FSIZE, (int(sys.argv[2]), int(sys.argv[2]))) +os.execvp(sys.argv[3], sys.argv[3:]) +""" + + +def run_fenced_statement(ledger: Any, pieces: Iterable[str]) -> Any: + from lab.prism.audit_bundle_view import ArtifactResourcePressure, RECORD_HELPER_ADMISSION + from lab.prism.share_ledger import LedgerOperationTimeout, parse_single_json_value + + with tempfile.TemporaryFile() as statement, tempfile.TemporaryFile() as result, tempfile.TemporaryFile() as errors: + size = 0 + try: + for piece in pieces: + for offset in range(0, len(piece), STATEMENT_CHUNK_CHARS): + ledger._remaining_operation_timeout() + chunk = piece[offset:offset + STATEMENT_CHUNK_CHARS].encode("utf-8") + size += len(chunk) + if size > STATEMENT_SPOOL_BYTES: + raise ArtifactResourcePressure("finalization statement spool reservation exhausted") + statement.write(chunk) + except OSError as exc: + raise ArtifactResourcePressure("finalization statement spool unavailable") from exc + statement.seek(0) + deadline = time.monotonic() + STATEMENT_HELPER_TIMEOUT_SECONDS + + def check() -> None: + ledger._remaining_operation_timeout() + if time.monotonic() >= deadline: + raise LedgerOperationTimeout("finalization statement helper exceeded its deadline") + + with ledger._operation_gate(ledger._lock, "writer lock"), RECORD_HELPER_ADMISSION.hold(check): + command, kwargs, timeout = ledger._psql_invocation() + environment = dict(kwargs.get("env", os.environ)) + native = getattr(ledger, "_native", None) + if native is not None: + # Preserve the exact native DSN, including private schemas, + # then append the same guards and deadlines as ordinary psql. + from psycopg.conninfo import conninfo_to_dict, make_conninfo + + connection = conninfo_to_dict(native._conninfo) + guards = getattr(ledger, "_session_guards", None) + fragments = [connection.get("options", "")] + if guards is not None: + fragments.append(guards.options_fragment()) + if timeout is not None: + timeout_ms = max(1, int(timeout * 1000)) + fragments.append(f"-c statement_timeout={timeout_ms}ms -c lock_timeout={timeout_ms}ms") + connection["options"] = " ".join(filter(None, fragments)) + connection["application_name"] = ledger._pool_application_name + password = connection.pop("password", None) + if password is not None: + environment["PGPASSWORD"] = password + command = ["psql", "--dbname", make_conninfo( + **connection, + ), + *command[len(ledger._command):]] + if timeout is not None: + deadline = min(deadline, time.monotonic() + timeout) + process = subprocess.Popen( + [sys.executable, "-c", _EXEC_LIMITED, + str(STATEMENT_HELPER_MEMORY_BYTES), str(STATEMENT_RESULT_BYTES), *command], + stdin=statement, stdout=result, stderr=errors, env=environment, + ) + try: + while process.poll() is None: + check() + time.sleep(0.05) + finally: + if process.poll() is None: + process.kill() + process.wait() + errors.seek(0) + diagnostic = errors.read(STATEMENT_RESULT_BYTES).decode("utf-8", "replace") + ledger._check_psql_exit(process.returncode, diagnostic, timeout) + result.seek(0) + output = result.read(STATEMENT_RESULT_BYTES + 1) + if len(output) > STATEMENT_RESULT_BYTES: + raise RuntimeError("finalization statement result exceeds its byte limit") + return parse_single_json_value(output.decode("utf-8")) diff --git a/lab/prism/stratum_session.py b/lab/prism/stratum_session.py new file mode 100644 index 000000000..8b71c6f04 --- /dev/null +++ b/lab/prism/stratum_session.py @@ -0,0 +1,1583 @@ +#!/usr/bin/env python3 +"""PRISM Stratum listener and session lifecycle. + +This module owns the S1 domain: miner-facing connection admission, protocol +dispatch, worker resolution, payout-address validation, and the live-session +registry (membership, connection generations, handler-thread accounting, and +exact per-connection delivery proof). + +It never imports ``prism_coordinator``. Job delivery, share submission, +progress health, shutdown state, and live configuration attributes are +reached through the :class:`SessionRuntime` typed port, resolved at call +time so the historical coordinator monkeypatch seams (including the +instance-level facade patches used by the current test suite) keep +intercepting exactly as before the extraction. Share-ACK timing state +remains coordinator-owned at this layer (it moves with the PR 77 submission +owner); the session code stamps ``request_received_monotonic`` and reports +the response boundary through the runtime seam. +""" + +from __future__ import annotations + +from collections import OrderedDict +from contextlib import ExitStack +from dataclasses import dataclass, field +from decimal import Decimal +import errno +import json +import socket +import struct +import threading +import time +import traceback +from types import MappingProxyType +from typing import Any, Callable, Mapping, Protocol + +from lab.auxpow import stratum_codec, vardiff +from lab.prism import direct_stratum +from lab.prism.coordinator_config import ( + DEFAULT_PRISM_PAYOUT_ADDRESS_CACHE_MAX_ENTRIES, + DEFAULT_PRISM_PAYOUT_ADDRESS_CACHE_TTL_SECONDS, + DEFAULT_PRISM_STRATUM_ACCEPT_RESOURCE_EXHAUSTION_BACKOFF_SECONDS, + DEFAULT_PRISM_STRATUM_MAX_CONNECTIONS, + DEFAULT_PRISM_STRATUM_MAX_CONNECTIONS_PER_USERNAME, + StratumListenerProfile, + default_prism_username_fallback_address, +) +from lab.prism.progress_health import DeliveryProof, WorkGeneration +from lab.prism.template_artifacts import qbit_template_fingerprint + + +_VARDIFF_LOCK_INITIALIZATION_LOCK = threading.Lock() + + +@dataclass(frozen=True) +class WorkerIdentity: + username: str + payout_address: str + worker_name: str | None + script_pubkey_hex: str + p2mr_program_hex: str + + +@dataclass(eq=False) +class ClientState: + sock: socket.socket + address: tuple[str, int] + connection_id: int + extranonce1_hex: str + subscribed: bool = False + authorized: bool = False + authorization_generation: int = 0 + difficulty_generation: int = 0 + authorized_monotonic: float | None = None + username: str = "" + worker: WorkerIdentity | None = None + version_mask: int = 0 + active_job: object | None = None + # active_job is registered before the potentially blocking socket write. + # These delivery-proof fields advance only after send_job_update succeeds. + _progress_delivered_context: object | None = None + _progress_delivered_template_fingerprint: str | None = None + _progress_delivered_template_generation: int = 0 + _progress_delivered_payout_generation: int = -1 + _progress_delivered_monotonic: float | None = None + # Admission ordering only. Delivery proof remains in the progress fields + # above and advances exclusively after the socket write succeeds. + _tip_refresh_admitted_epoch_sequence: int = 0 + listener_name: str = "default" + # Pristine difficulty policy of the accepting listener; never mutated. + listener_vardiff_config: vardiff.VardiffConfig | None = None + # Floor below which stamped jobs never advertise, copied from the + # accepting listener profile. Zero (default listener) keeps the network + # cap authoritative. + minimum_advertised_difficulty: Decimal = Decimal("0") + # Per-client specialization of the listener policy (password d=/md= or + # mining.suggest_difficulty); recomputed from the pristine base on every + # request so repeat applications cannot compound. + vardiff_config: vardiff.VardiffConfig | None = None + requested_difficulty: Decimal | None = None + requested_min_difficulty: Decimal | None = None + suggested_difficulty: Decimal | None = None + share_difficulty: Decimal = Decimal("1") + pending_share_difficulty: Decimal | None = None + vardiff_window_started_monotonic: float = field(default_factory=time.monotonic) + vardiff_window_accepted: int = 0 + vardiff_window_submitted: int = 0 + vardiff_window_work: Decimal = Decimal("0") + vardiff_difficulty_estimate: Decimal | None = None + # Per-connection evidence that this session's difficulty is real: set on + # the first accepted share of the connection and never reset within it. + # Reconnect retention reads it to decide whether re-recording an + # unchanged difficulty may refresh its TTL; a session that resumed a + # retained value and submitted nothing must let that value keep ageing. + vardiff_accepted_any: bool = False + # Most recent accepted-share evidence. Durable reconnect retention uses + # the exact stamped difficulty plus its wall-clock evidence time so a + # later explicit difficulty request cannot accidentally be persisted as + # safe merely because this connection accepted an older, easier share. + vardiff_last_accepted_difficulty: Decimal | None = None + vardiff_last_accepted_wall_ms: int | None = None + # Fast-arrival initial convergence. True until this connection's first + # share-driven retarget actually commits; while set, an accepted share + # may trigger a retarget before the configured interval and that + # retarget may take the larger initial step bound. Cleared exactly once, + # at the paired-send commit point, so a build/send failure cannot spend + # the relaxation without the miner ever seeing the new difficulty, and + # so every later retarget is back on the ordinary configured bound. + vardiff_initial_convergence_pending: bool = True + # The statistical gate is evaluated once at the first sample that meets + # both its share-count and elapsed-time floors. A failed delivery may + # re-arm it, but an ordinary on-cadence window is never tested after every + # subsequent share (which would compound the false-positive probability). + vardiff_initial_convergence_evaluated: bool = False + # Connection-scoped arrival evidence for the bounded high-difficulty + # arrival histograms: when this session's vardiff accounting began, how + # many accepted shares it has produced since, and whether its one + # crossing of the high-diff threshold has already been observed. + vardiff_session_started_monotonic: float = field(default_factory=time.monotonic) + vardiff_session_accepted: int = 0 + vardiff_high_diff_arrival_recorded: bool = False + # Owns all per-client difficulty policy, estimates, and Vardiff windows. + # Share handlers release it before acquiring job_update_lock or the + # coordinator lock; job delivery may acquire it while job_update_lock is + # held. Keeping this lock per connection prevents one miner's accounting + # or slow socket from convoying tip publication for the whole coordinator. + vardiff_lock: threading.RLock = field(default_factory=threading.RLock) + active_job_ids: set[str] = field(default_factory=set) + post_accept_refresh_block: tuple[int, str] | None = None + # (job previousblockhash, monotonic) of the FIRST job this connection was + # sent for that tip. Anchors the per-connection stale-grace window: a + # prior-tip share is in flight until shortly after this connection + # received replacement work, however long the refresh pass took to reach + # it. See stale_grace_deadline_open. + tip_work_delivered: tuple[str, float] | None = None + # Protected by the coordinator lock. Disconnect retirement sets this before + # waiting for any per-client job update so queued work can reject the client. + closing: bool = False + # Stamped by the handler thread when a request line arrives, before JSON + # parsing. Only that thread reads it; mining.submit uses it to observe + # read-to-ack latency (the share-ingest saturation instrument). + request_received_monotonic: float | None = None + # Serializes every job build/register/send transition for this connection. + # The coordinator lock may be acquired while this lock is held, never in + # the reverse order. RLock permits authorize/retarget helpers to call the + # common maybe_send_job path while retaining the same serialization scope. + job_update_lock: threading.RLock = field(default_factory=threading.RLock) + send_lock: threading.Lock = field(default_factory=threading.Lock) + handler_thread_registered: bool = False + + def send(self, payload: dict[str, object]) -> None: + data = json.dumps(payload).encode() + b"\n" + with self.send_lock: + self.sock.sendall(data) + + def send_batch( + self, + payloads: list[dict[str, object]], + *, + preserialized: bytes | None = None, + ) -> None: + # Tests and embedders may replace ``send`` with an in-memory recorder; + # retain that seam while production sockets write the whole difficulty + # + notify pair under one send lock with no response interleaving. + # ``preserialized`` must be the exact serialization of ``payloads``; + # callers with a shared precomposed fragment pass it so a fleet-wide + # wave does not re-serialize identical coinbase parts per client. + if "send" in self.__dict__: + for payload in payloads: + self.send(payload) + return + data = ( + preserialized + if preserialized is not None + else b"".join( + json.dumps(payload).encode() + b"\n" for payload in payloads + ) + ) + with self.send_lock: + self.sock.sendall(data) + + def close(self) -> None: + try: + self.sock.shutdown(socket.SHUT_RDWR) + except OSError: + pass + try: + self.sock.close() + except OSError: + pass + + +def client_vardiff_lock(client: ClientState) -> threading.RLock: + """Return the per-client vardiff lock, including lightweight embedders. + + Production ClientState instances always have this field. The lazy + fallback preserves focused tests and embedders that construct a + ClientState with __new__ and populate only the attributes they use. + """ + lock = getattr(client, "vardiff_lock", None) + if lock is not None: + return lock + with _VARDIFF_LOCK_INITIALIZATION_LOCK: + lock = getattr(client, "vardiff_lock", None) + if lock is None: + lock = threading.RLock() + client.vardiff_lock = lock + return lock + + +class StratumError(RuntimeError): + def __init__( + self, + code: int, + message: str, + *, + reason: str | None = None, + disconnect: bool = False, + ): + super().__init__(message) + self.code = code + self.message = message + self.reason = reason + self.disconnect = disconnect + + +def parse_stratum_password_options(password: str) -> tuple[Decimal | None, Decimal | None]: + """Extract the pool-side d=N / md=N difficulty convention from a password. + + Unknown tokens and malformed values are ignored: miners routinely send + junk passwords ("x") and rejecting them would break every such rig. + Returns (requested_difficulty, requested_min_difficulty). + """ + + requested: Decimal | None = None + requested_min: Decimal | None = None + for token in password.split(","): + key, separator, raw_value = token.strip().partition("=") + if not separator: + continue + key = key.strip().lower() + if key not in {"d", "md"}: + continue + try: + value = Decimal(raw_value.strip()) + except Exception: + continue + if not value.is_finite() or value <= 0: + continue + if key == "d": + requested = value + else: + requested_min = value + return requested, requested_min + + +def parse_worker_username(username: str) -> tuple[str, str | None]: + payout_address, worker_name = split_worker_username(username) + if not payout_address: + raise StratumError(20, "username base is empty") + return payout_address, worker_name + + +def split_worker_username(username: str) -> tuple[str, str | None]: + payout_address, separator, worker_name = username.partition(".") + return payout_address, worker_name if separator else None + + +def result_payload(request_id: object, result: object) -> dict[str, object]: + return {"id": request_id, "result": result, "error": None} + + +def error_payload( + request_id: object, + code: int, + message: str, + *, + reason: str | None = None, +) -> dict[str, object]: + data = {"reason_id": reason} if reason is not None else None + return {"id": request_id, "result": None, "error": [code, message, data]} + + +def difficulty_payload(difficulty: Decimal) -> dict[str, object]: + return { + "id": None, + "method": "mining.set_difficulty", + "params": [float(difficulty)], + } + + +def job_payload(job: direct_stratum.DirectQbitStratumJob) -> dict[str, object]: + return { + "id": None, + "method": "mining.notify", + "params": [ + job.job_id, + job.prevhash, + job.coinb1, + job.coinb2, + list(job.merkle_branch), + job.version, + job.nbits, + job.ntime, + job.clean_jobs, + ], + } + + +def client_can_receive_jobs(client: ClientState) -> bool: + return ( + not getattr(client, "closing", False) + and client.subscribed + and client.authorized + and client.worker is not None + ) + + +def stratum_accept_heartbeat_names( + profiles: list[StratumListenerProfile] | tuple[StratumListenerProfile, ...] | None, +) -> tuple[str, ...]: + if not profiles: + return ("stratum_accept",) + return tuple(profile.heartbeat_name for profile in profiles) + + +def apply_stratum_send_timeout(sock: socket.socket, timeout_seconds: float) -> None: + """Bound blocking sends to miners without touching receive semantics. + + Job refreshes use a bounded executor, but an unresponsive peer whose + TCP buffer is full must still release its worker eventually. + SO_SNDTIMEO turns that into an OSError, which the refresh path treats + as a dead client without failing delivery to other miners. + A plain socket timeout is not usable here: it would also apply to + recv, disconnecting idle-but-healthy miners. + """ + if timeout_seconds <= 0: + return + seconds = int(timeout_seconds) + microseconds = int((timeout_seconds - seconds) * 1_000_000) + try: + sock.setsockopt( + socket.SOL_SOCKET, + socket.SO_SNDTIMEO, + struct.pack("ll", seconds, microseconds), + ) + except (AttributeError, OSError, struct.error): + # Platform without SO_SNDTIMEO support: keep legacy blocking sends. + return + + +@dataclass(frozen=True) +class DeliveredSessionContext: + context: object + delivered_monotonic: float + + +@dataclass(frozen=True) +class EligibleSession: + connection_id: int + delivered: DeliveredSessionContext | None + + +class SessionRegistry: + """Atomic owner of live membership and connection-scoped session facts. + + Callers hold the shared coordinator control-plane lock around the + ``*_locked`` helpers exactly as the pre-extraction coordinator did; the + registry itself adds no second lock. Compatibility containers seeded by + ``__new__`` embedders are adopted, never copied, so test-held references + remain live. + """ + + def __init__( + self, + *, + clients: object | None = None, + connection_generation: int = 0, + rejection_counts: dict[str, int] | None = None, + ) -> None: + self.clients = clients if clients is not None else set() + self.connection_generation = max( + int(connection_generation), + max( + (int(client.connection_id) for client in self.clients), # type: ignore[union-attr] + default=0, + ), + ) + self.rejection_counts = ( + rejection_counts + if rejection_counts is not None + else {"global": 0, "username": 0} + ) + self.peak_active_connections = len(self.clients) # type: ignore[arg-type] + self.handler_thread_count = sum( + int(getattr(client, "handler_thread_registered", False)) + for client in self.clients + ) + self._delivered_by_connection: dict[int, DeliveredSessionContext] = {} + + def adopt_clients(self, clients: object) -> None: + """Adopt a compatibility replacement without changing its order/type.""" + self.clients = clients + live_ids = { + int(getattr(client, "connection_id", 0)) for client in clients # type: ignore[union-attr] + } + self._delivered_by_connection = { + connection_id: delivered + for connection_id, delivered in self._delivered_by_connection.items() + if connection_id in live_ids + } + self.peak_active_connections = max( + self.peak_active_connections, + len(clients), # type: ignore[arg-type] + ) + self.connection_generation = max( + self.connection_generation, + max( + ( + int(getattr(client, "connection_id", 0)) + for client in clients # type: ignore[union-attr] + ), + default=0, + ), + ) + self.handler_thread_count = sum( + int(getattr(client, "handler_thread_registered", False)) + for client in clients # type: ignore[union-attr] + ) + + def note_rejection_locked(self, scope: str) -> int: + count = int(self.rejection_counts.get(scope, 0)) + 1 + self.rejection_counts[scope] = count + return count + + def begin_retirement_locked(self, client: ClientState) -> bool: + """Claim retirement ownership; only the first claimer cleans up.""" + if getattr(client, "closing", False) and client not in self.clients: + return False + client.closing = True + self.clients.discard(client) + self._delivered_by_connection.pop(client.connection_id, None) + return True + + def record_delivery_locked( + self, + client: ClientState, + context: object, + delivered_monotonic: float, + ) -> None: + """Commit exact delivery proof while the shared lock is held.""" + self._delivered_by_connection[client.connection_id] = ( + DeliveredSessionContext(context, delivered_monotonic) + ) + + def eligible_snapshot(self) -> Mapping[int, EligibleSession]: + """Return an immutable exact client_can_receive_jobs population. + + Caller holds the shared lock; the compatibility mirrors on the + client remain the staged-split fallback for embedder-seeded state. + """ + captured: dict[int, EligibleSession] = {} + for client in self.clients: + if not client_can_receive_jobs(client): + continue + delivered = self._delivered_by_connection.get(client.connection_id) + if ( + delivered is None + and getattr(client, "_progress_delivered_context", None) is not None + ): + delivered = DeliveredSessionContext( + client._progress_delivered_context, + float(client._progress_delivered_monotonic or 0.0), + ) + captured[client.connection_id] = EligibleSession( + connection_id=client.connection_id, + delivered=delivered, + ) + return MappingProxyType(captured) + + +@dataclass +class _P2mrAddressValidationFlight: + event: threading.Event = field(default_factory=threading.Event) + result: tuple[str, str] | None = None + error: BaseException | None = None + waiters: int = 0 + + +class P2mrAddressValidator: + """Bounded LRU and singleflight wrapper around validateaddress RPC.""" + + def __init__( + self, + *, + rpc_call: Callable[[str, list[object]], object], + max_entries: Callable[[], int], + ttl_seconds: Callable[[], float], + cache_lock: threading.Lock | None = None, + cache: OrderedDict[str, tuple[float, tuple[str, str]]] | None = None, + inflight: dict[str, _P2mrAddressValidationFlight] | None = None, + ) -> None: + self.rpc_call = rpc_call + self.max_entries = max_entries + self.ttl_seconds = ttl_seconds + self.cache_lock = cache_lock if cache_lock is not None else threading.Lock() + self.cache = cache if cache is not None else OrderedDict() + self.inflight = inflight if inflight is not None else {} + + def validate(self, address: str, *, label: str) -> tuple[str, str]: + with self.cache_lock: + cached = self.cache.get(address) + if cached is not None: + expires_monotonic, cached_result = cached + if expires_monotonic > time.monotonic(): + self.cache.move_to_end(address) + return cached_result + self.cache.pop(address, None) + pending = self.inflight.get(address) + is_leader = pending is None + if pending is None: + pending = _P2mrAddressValidationFlight() + self.inflight[address] = pending + else: + pending.waiters += 1 + + if not is_leader: + pending.event.wait() + if pending.result is not None: + return pending.result + if pending.error is not None: + self._raise_shared_error(pending.error) + raise RuntimeError("payout address validation completed without a result") + + try: + validation = self.rpc_call("validateaddress", [address]) + if not isinstance(validation, dict) or not validation.get("isvalid"): + raise StratumError(20, f"{label} is not a valid qbit address: {address}") + script = str(validation.get("scriptPubKey") or "") + if not script.startswith("5220") or len(script) != 68: + raise StratumError(20, f"{label} does not resolve to a P2MR script: {address}") + result = (script, script[4:]) + with self.cache_lock: + max_entries = int(self.max_entries()) + ttl_seconds = float(self.ttl_seconds()) + if max_entries > 0 and ttl_seconds > 0: + self.cache[address] = ( + time.monotonic() + ttl_seconds, + result, + ) + self.cache.move_to_end(address) + while len(self.cache) > max_entries: + self.cache.popitem(last=False) + pending.result = result + return result + except BaseException as exc: + with self.cache_lock: + pending.error = exc + raise + finally: + with self.cache_lock: + if self.inflight.get(address) is pending: + self.inflight.pop(address, None) + pending.event.set() + # Release the leader frame's reference after the waiters are woken + # (#251). On failure the stored error's traceback already holds + # this frame; keeping the flight named here would close + # flight -> error -> traceback -> frame -> flight and pin the + # failed RPC frame and its locals until cyclic GC. Waiters keep + # the flight alive through their own frames until each has read + # the recorded result or error. + del pending + + @staticmethod + def _raise_shared_error(error: BaseException) -> None: + if isinstance(error, StratumError): + raise StratumError( + error.code, + error.message, + reason=error.reason, + disconnect=error.disconnect, + ) from error + raise RuntimeError(str(error)) from error + + +class SessionRuntime(Protocol): + """Typed port over the coordinator, resolved at call time. + + Every member is looked up on the live coordinator object when used, so + instance monkeypatches (``server.send_result = ...``, + ``server.validate_p2mr_address = ...`` and friends) and coordinator-owned + live configuration attributes keep working exactly as before the + extraction. Owner facades route back into this service; that round trip + is deliberate: it preserves the current white-box patch surface until + later stack layers repoint those tests. The legacy S1 field names + (``clients``, ``connection_counter``, the connection counters, and the + payout-address cache trio) also resolve here -- coordinator class + descriptors route them to this service's single mutable copy. + """ + + # Cross-domain objects and live configuration attributes. + _pool_ready_latched: Any + clients: Any + extranonce2_size: Any + handler_thread_count: Any + latest_detected_tip: Any + listener_profiles: Any + lock: Any + pending_initial_jobs: Any + rpc: Any + stop_event: Any + stratum_max_pending_initial_jobs: Any + version_mask: Any + + def _cancel_pending_initial_job_locked(self, *args: Any, **kwargs: Any) -> Any: ... + + def _client_has_current_tip_job_locked(self, *args: Any, **kwargs: Any) -> Any: ... + + def _client_vardiff_lock(self, *args: Any, **kwargs: Any) -> Any: ... + + def _ensure_initial_job_state(self, *args: Any, **kwargs: Any) -> Any: ... + + def _ensure_job_cache_state(self, *args: Any, **kwargs: Any) -> Any: ... + + def _ensure_job_delivery_service(self, *args: Any, **kwargs: Any) -> Any: ... + + def _ensure_progress_health_service(self, *args: Any, **kwargs: Any) -> Any: ... + + def _ensure_shutdown_controller(self, *args: Any, **kwargs: Any) -> Any: ... + + def _handle_request(self, *args: Any, **kwargs: Any) -> Any: ... + + def _note_collection_identity_available(self, *args: Any, **kwargs: Any) -> Any: ... + + def _observe_share_ack_seconds(self, *args: Any, **kwargs: Any) -> Any: ... + + def _progress_note_refresh_activity(self, *args: Any, **kwargs: Any) -> Any: ... + + def _progress_reconcile_pending(self, *args: Any, **kwargs: Any) -> Any: ... + + def _record_heartbeat(self, *args: Any, **kwargs: Any) -> Any: ... + + def _record_stratum_resource_exhaustion(self, *args: Any, **kwargs: Any) -> Any: ... + + def _record_tip_refresh_epoch_coverage(self, *args: Any, **kwargs: Any) -> Any: ... + + def _retain_current_collection_refresh_if_unrepresented( + self, *args: Any, **kwargs: Any + ) -> Any: ... + + def _tip_refresh_epoch_coverage_reached_locked( + self, *args: Any, **kwargs: Any + ) -> Any: ... + + def _wait_after_stratum_resource_failure(self, *args: Any, **kwargs: Any) -> Any: ... + + def apply_client_difficulty_requests(self, *args: Any, **kwargs: Any) -> Any: ... + + def apply_stratum_send_timeout(self, *args: Any, **kwargs: Any) -> Any: ... + + def advertise_client_difficulty(self, *args: Any, **kwargs: Any) -> Any: ... + + def client_startup_difficulty(self, *args: Any, **kwargs: Any) -> Any: ... + + def disconnect_client(self, *args: Any, **kwargs: Any) -> Any: ... + + def handle_client(self, *args: Any, **kwargs: Any) -> Any: ... + + def handle_configure(self, *args: Any, **kwargs: Any) -> Any: ... + + def handle_submit(self, *args: Any, **kwargs: Any) -> Any: ... + + def handle_suggest_difficulty(self, *args: Any, **kwargs: Any) -> Any: ... + + def note_vardiff_resume_overridden(self, *args: Any, **kwargs: Any) -> Any: ... + + def record_session_difficulty(self, *args: Any, **kwargs: Any) -> Any: ... + + def refresh_jobs_after_pending_accepted_block( + self, *args: Any, **kwargs: Any + ) -> Any: ... + + def request_initial_job_delivery(self, *args: Any, **kwargs: Any) -> Any: ... + + def reserve_client_username(self, *args: Any, **kwargs: Any) -> Any: ... + + def resume_client_difficulty(self, *args: Any, **kwargs: Any) -> Any: ... + + def send_error(self, *args: Any, **kwargs: Any) -> Any: ... + + def send_result(self, *args: Any, **kwargs: Any) -> Any: ... + + def validate_p2mr_address(self, *args: Any, **kwargs: Any) -> Any: ... + + +class StratumSessionService: + """Sole owner of S1 session admission, dispatch, and registry proof.""" + + def __init__( + self, + runtime: SessionRuntime, + *, + shutdown_error: type[BaseException], + pool_closed_reason: str, + ) -> None: + self._runtime = runtime + # The shutdown exception type remains coordinator-owned until the + # shutdown controller is extracted; inject it so this leaf module + # never imports prism_coordinator. + self._shutdown_error = shutdown_error + self.pool_closed_reason = pool_closed_reason + self.registry = SessionRegistry() + self.accept_resource_exhaustion_count = 0 + self.connection_setup_failure_count = 0 + self.address_validator = P2mrAddressValidator( + rpc_call=lambda method, params: runtime.rpc.call(method, params), + max_entries=lambda: int( + getattr( + runtime, + "payout_address_cache_max_entries", + DEFAULT_PRISM_PAYOUT_ADDRESS_CACHE_MAX_ENTRIES, + ) + ), + ttl_seconds=lambda: float( + getattr( + runtime, + "payout_address_cache_ttl_seconds", + DEFAULT_PRISM_PAYOUT_ADDRESS_CACHE_TTL_SECONDS, + ) + ), + ) + + # -- legacy coordinator field names routed by class descriptors -------- + + @property + def clients(self) -> object: + return self.registry.clients + + @clients.setter + def clients(self, value: object) -> None: + self.registry.adopt_clients(value) + + @property + def connection_counter(self) -> int: + return self.registry.connection_generation + + @connection_counter.setter + def connection_counter(self, value: int) -> None: + self.registry.connection_generation = int(value) + + @property + def connection_limit_rejection_counts(self) -> dict[str, int]: + return self.registry.rejection_counts + + @connection_limit_rejection_counts.setter + def connection_limit_rejection_counts(self, value: dict[str, int]) -> None: + self.registry.rejection_counts = value + + @property + def peak_active_connection_count(self) -> int: + return self.registry.peak_active_connections + + @peak_active_connection_count.setter + def peak_active_connection_count(self, value: int) -> None: + self.registry.peak_active_connections = int(value) + + @property + def handler_thread_count(self) -> int: + return self.registry.handler_thread_count + + @handler_thread_count.setter + def handler_thread_count(self, value: int) -> None: + self.registry.handler_thread_count = int(value) + + @property + def _p2mr_address_cache_lock(self) -> threading.Lock: + return self.address_validator.cache_lock + + @_p2mr_address_cache_lock.setter + def _p2mr_address_cache_lock(self, value: threading.Lock) -> None: + self.address_validator.cache_lock = value + + @property + def _p2mr_address_cache(self) -> OrderedDict[str, tuple[float, tuple[str, str]]]: + return self.address_validator.cache + + @_p2mr_address_cache.setter + def _p2mr_address_cache( + self, value: OrderedDict[str, tuple[float, tuple[str, str]]] + ) -> None: + self.address_validator.cache = value + + @property + def _p2mr_address_validation_inflight( + self, + ) -> dict[str, _P2mrAddressValidationFlight]: + return self.address_validator.inflight + + @_p2mr_address_validation_inflight.setter + def _p2mr_address_validation_inflight( + self, value: dict[str, _P2mrAddressValidationFlight] + ) -> None: + self.address_validator.inflight = value + + # -- listener factory -------------------------------------------------- + + @staticmethod + def open_stratum_listeners( + listener_stack: ExitStack, + profiles: list[StratumListenerProfile] | tuple[StratumListenerProfile, ...], + *, + backlog: int, + retry_seconds: float, + stop_event: threading.Event | None, + ) -> list[tuple[socket.socket, StratumListenerProfile]] | None: + """Bind and listen on every stratum listener profile. + + Called before the slow parts of startup (qbit readiness, policy + validation, block-work recovery) so miners reconnecting through a + restart park in the kernel accept backlog instead of getting + connection refused, which sends firmware into reconnect backoff or + failover and costs hashrate. bind() retries EADDRINUSE for a bounded + window because a predecessor process may still hold the port while + draining its shutdown. Returns None when shutdown is requested during + the retry, so startup can abort gracefully. + """ + listeners: list[tuple[socket.socket, StratumListenerProfile]] = [] + for profile in profiles: + server = listener_stack.enter_context( + socket.socket(socket.AF_INET, socket.SOCK_STREAM) + ) + server.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) + bind_deadline = time.monotonic() + retry_seconds + warned = False + while True: + try: + server.bind((profile.bind, profile.port)) + break + except OSError as exc: + if exc.errno != errno.EADDRINUSE or time.monotonic() >= bind_deadline: + raise + if stop_event is not None and stop_event.is_set(): + # Shutting down mid-startup: stop contending for a + # port this process will never serve. + print( + f"prism coordinator: shutdown requested while waiting " + f"to bind {profile.bind}:{profile.port}; aborting startup", + flush=True, + ) + return None + if not warned: + print( + f"prism coordinator: {profile.name} listener port " + f"{profile.bind}:{profile.port} is busy; retrying bind " + f"for up to {retry_seconds:g}s", + flush=True, + ) + warned = True + time.sleep(0.1) + server.listen(backlog) + server.settimeout(1) + listeners.append((server, profile)) + return listeners + + # -- accept/admission -------------------------------------------------- + + def accept_loop( + self, server: socket.socket, profile: StratumListenerProfile + ) -> None: + runtime = self._runtime + registry = self.registry + while not runtime.stop_event.is_set(): + runtime._record_heartbeat(profile.heartbeat_name) + try: + sock, address = server.accept() + except socket.timeout: + continue + except OSError as exc: + # The listener socket is torn down by serve()'s ExitStack on + # shutdown while secondary accept threads may still be blocked + # in accept(). Descriptor exhaustion is recoverable: keep the + # accept loop alive and refresh its watchdog heartbeat while + # waiting for client/RPC descriptors to drain. + if runtime.stop_event.is_set(): + return + if exc.errno in {errno.EMFILE, errno.ENFILE}: + runtime._record_stratum_resource_exhaustion( + listener_name=profile.name, + location="accept", + error_number=exc.errno, + ) + runtime._wait_after_stratum_resource_failure(profile.heartbeat_name) + continue + raise + + if ( + runtime.stop_event.is_set() + or runtime._ensure_shutdown_controller().phase != "running" + ): + try: + sock.close() + except OSError: + pass + return + + with runtime.lock: + if ( + runtime.stop_event.is_set() + or runtime._ensure_shutdown_controller().phase != "running" + ): + try: + sock.close() + except OSError: + pass + return + max_connections = int( + getattr( + runtime, + "stratum_max_connections", + DEFAULT_PRISM_STRATUM_MAX_CONNECTIONS, + ) + ) + if max_connections > 0 and len(registry.clients) >= max_connections: + rejection_count = registry.note_rejection_locked("global") + client = None + else: + registry.connection_generation += 1 + connection_id = registry.connection_generation + client = ClientState( + sock=sock, + address=address, + connection_id=connection_id, + extranonce1_hex=f"{connection_id & 0xFFFFFFFF:08x}", + listener_name=profile.name, + listener_vardiff_config=profile.vardiff_config, + minimum_advertised_difficulty=profile.minimum_advertised_difficulty, + share_difficulty=runtime.client_startup_difficulty(profile), + ) + registry.clients.add(client) + runtime._ensure_initial_job_state() + registry.peak_active_connections = max( + registry.peak_active_connections, + len(registry.clients), + ) + if client is None: + try: + sock.close() + except OSError: + pass + if rejection_count == 1 or rejection_count % 100 == 0: + print( + "prism coordinator: rejected stratum connection at global limit " + f"limit={max_connections} count={rejection_count}", + flush=True, + ) + continue + try: + sock.settimeout(None) + runtime.apply_stratum_send_timeout(sock) + thread = threading.Thread( + target=runtime.handle_client, args=(client,), daemon=True + ) + with runtime.lock: + client.handler_thread_registered = True + registry.handler_thread_count += 1 + thread.start() + except (OSError, RuntimeError) as exc: + # Admission is atomic with the global count. Undo it if socket + # setup or thread creation fails before a handler owns cleanup, + # then keep this listener alive for the next connection. + try: + with runtime.lock: + if client.handler_thread_registered: + client.handler_thread_registered = False + registry.handler_thread_count = max( + 0, + registry.handler_thread_count - 1, + ) + runtime.disconnect_client(client) + except Exception: + print( + "prism coordinator: failed to fully close rejected stratum client " + f"address={address}", + flush=True, + ) + traceback.print_exc() + with runtime.lock: + self.connection_setup_failure_count = int( + getattr(self, "connection_setup_failure_count", 0) + ) + 1 + setup_failure_count = self.connection_setup_failure_count + if isinstance(exc, OSError) and exc.errno in {errno.EMFILE, errno.ENFILE}: + runtime._record_stratum_resource_exhaustion( + listener_name=profile.name, + location="connection-setup", + error_number=exc.errno, + ) + if setup_failure_count == 1 or setup_failure_count % 100 == 0: + print( + "prism coordinator: stratum connection setup failed; backing off " + f"listener={profile.name} address={address} " + f"error={exc!r} count={setup_failure_count}", + flush=True, + ) + runtime._wait_after_stratum_resource_failure(profile.heartbeat_name) + continue + + def record_stratum_resource_exhaustion( + self, + *, + listener_name: str, + location: str, + error_number: int | None, + ) -> int: + runtime = self._runtime + with runtime.lock: + self.accept_resource_exhaustion_count = int( + getattr(self, "accept_resource_exhaustion_count", 0) + ) + 1 + exhaustion_count = self.accept_resource_exhaustion_count + if exhaustion_count == 1 or exhaustion_count % 100 == 0: + print( + "prism coordinator: stratum resource exhaustion " + f"listener={listener_name} location={location} errno={error_number} " + f"count={exhaustion_count}", + flush=True, + ) + return exhaustion_count + + def wait_after_stratum_resource_failure(self, heartbeat_name: str) -> None: + runtime = self._runtime + backoff_seconds = getattr( + runtime, + "stratum_accept_resource_exhaustion_backoff_seconds", + DEFAULT_PRISM_STRATUM_ACCEPT_RESOURCE_EXHAUSTION_BACKOFF_SECONDS, + ) + remaining_seconds = max(0.0, float(backoff_seconds)) + watchdog_timeout_seconds = max( + 0.001, + float(getattr(runtime, "watchdog_timeout_seconds", 120.0)), + ) + heartbeat_interval_seconds = max( + 0.001, + min(1.0, watchdog_timeout_seconds / 2.0), + ) + deadline = time.monotonic() + remaining_seconds + while not runtime.stop_event.is_set(): + runtime._record_heartbeat(heartbeat_name) + remaining_seconds = deadline - time.monotonic() + if remaining_seconds <= 0: + return + if runtime.stop_event.wait( + min(remaining_seconds, heartbeat_interval_seconds) + ): + return + + def reserve_client_username( + self, client: ClientState, worker: WorkerIdentity + ) -> bool: + """Atomically reserve an exact Stratum username for one connection.""" + runtime = self._runtime + registry = self.registry + with runtime.lock: + limit = int( + getattr( + runtime, + "stratum_max_connections_per_username", + DEFAULT_PRISM_STRATUM_MAX_CONNECTIONS_PER_USERNAME, + ) + ) + active_for_username = sum( + 1 + for other in registry.clients + if ( + other is not client + and other.worker is not None + and other.username == worker.username + ) + ) + if limit > 0 and active_for_username >= limit: + rejection_count = registry.note_rejection_locked("username") + if rejection_count == 1 or rejection_count % 100 == 0: + print( + "prism coordinator: rejected stratum authorization at username limit " + f"username={worker.username!r} limit={limit} count={rejection_count}", + flush=True, + ) + return False + # Commit the replacement only after capacity validation. A failed + # reauthorization therefore leaves the prior live identity intact. + client.worker = worker + client.username = worker.username + return True + + # -- session lifecycle ------------------------------------------------- + + def handle_client(self, client: ClientState) -> None: + runtime = self._runtime + registry = self.registry + reader = None + try: + reader = client.sock.makefile("r", encoding="utf-8", newline="\n") + for line in reader: + if runtime.stop_event.is_set(): + break + line = line.strip() + if not line: + continue + client.request_received_monotonic = time.monotonic() + request_id: object = None + request: object = None + try: + request = json.loads(line) + if not isinstance(request, dict): + raise StratumError(20, "request must be an object") + request_id = request.get("id") + self.handle_request(client, request) + except json.JSONDecodeError as exc: + runtime.send_error(client, request_id, 20, f"invalid JSON: {exc.msg}") + except StratumError as exc: + runtime.send_error( + client, request_id, exc.code, exc.message, reason=exc.reason + ) + received_monotonic = client.request_received_monotonic + if ( + received_monotonic is not None + and isinstance(request, dict) + and request.get("method") == "mining.submit" + ): + # Symmetric with the accepted-path observation after + # send_result: both sides include the response write, + # so the accepted-vs-rejected comparison separates + # commit pressure from transport/thread saturation. + runtime._observe_share_ack_seconds( + "rejected", + time.monotonic() - received_monotonic, + ) + if exc.disconnect: + break + except Exception: + print( + f"prism coordinator: client thread failed address={client.address}", + flush=True, + ) + traceback.print_exc() + break + except (OSError, ValueError) as exc: + if isinstance(exc, OSError) and exc.errno in {errno.EMFILE, errno.ENFILE}: + runtime._record_stratum_resource_exhaustion( + listener_name=client.listener_name, + location="client-reader", + error_number=exc.errno, + ) + print( + "prism coordinator: stratum client socket failed " + f"address={client.address} error={exc!r}", + flush=True, + ) + finally: + try: + if reader is not None: + reader.close() + except (OSError, ValueError): + pass + finally: + runtime.disconnect_client(client) + with runtime.lock: + runtime._ensure_initial_job_state() + if getattr(client, "handler_thread_registered", False): + client.handler_thread_registered = False + registry.handler_thread_count = max( + 0, + registry.handler_thread_count - 1, + ) + + def disconnect_client(self, client: ClientState) -> None: + # Retire admission and fanout eligibility without waiting behind job + # delivery. Only the first caller owns socket close and final cleanup. + runtime = self._runtime + registry = self.registry + with runtime.lock: + # The timeout sweeper marks closing while leaving membership in + # place as its atomic handoff token. Whichever disconnect caller + # first removes that membership owns socket close and cleanup. + if not registry.begin_retirement_locked(client): + return + runtime._cancel_pending_initial_job_locked(client, count=True) + + # Retirement was just claimed, so this runs once per connection and + # outside the coordinator lock: retain the session's converged + # difficulty for a reconnect resume. Reached through the runtime seam + # so monkeypatched coordinators keep intercepting; a retention + # failure must never break disconnect cleanup. + record_session_difficulty = getattr( + runtime, "record_session_difficulty", None + ) + if record_session_difficulty is not None: + try: + record_session_difficulty(client) + except Exception: + print( + "prism coordinator: session difficulty retention failed " + f"address={client.address}", + flush=True, + ) + traceback.print_exc() + + # Do not take send_lock here: shutdown must interrupt an in-flight + # sendall as well as the handler's blocking reader. + try: + client.close() + finally: + # Every mixed lock path uses job_update_lock -> coordinator lock. + # Retirement above holds neither while this potentially waits. + with client.job_update_lock: + runtime._ensure_job_delivery_service().cleanup_disconnected_client( + client + ) + runtime._retain_current_collection_refresh_if_unrepresented() + + # -- protocol dispatch ------------------------------------------------- + + def handle_request(self, client: ClientState, request: dict[str, object]) -> None: + """Dispatch one request, translating shutdown races to Stratum errors.""" + try: + self._runtime._handle_request(client, request) + except self._shutdown_error as exc: + # A request can pass the initial shutdown check immediately before + # writer admission closes. Preserve the normal protocol response + # instead of surfacing a generic client-thread failure. + raise StratumError( + 20, + "coordinator is shutting down", + reason=self.pool_closed_reason, + disconnect=True, + ) from exc + + def _handle_request(self, client: ClientState, request: dict[str, object]) -> None: + runtime = self._runtime + if ( + runtime.stop_event.is_set() + or runtime._ensure_shutdown_controller().phase != "running" + ): + raise StratumError( + 20, + "coordinator is shutting down", + reason=self.pool_closed_reason, + disconnect=True, + ) + method = request.get("method") + params = request.get("params", []) + request_id = request.get("id") + if not isinstance(method, str): + raise StratumError(20, "missing method") + if not isinstance(params, list): + raise StratumError(20, "params must be an array") + + if method == "mining.configure": + runtime.handle_configure(client, request_id, params) + return + if method == "mining.subscribe": + with client.job_update_lock: + client.subscribed = True + runtime.send_result( + client, + request_id, + [[], client.extranonce1_hex, runtime.extranonce2_size], + ) + runtime._note_collection_identity_available(client) + needs_initial_job = client.authorized + if needs_initial_job: + runtime.request_initial_job_delivery(client) + return + if method == "mining.authorize": + username = str(params[0]) if params else "" + password = str(params[1]) if len(params) > 1 and params[1] is not None else "" + # Address validation may use RPC; it is unrelated to client job + # state and therefore stays outside the job-update lock. + worker = runtime.resolve_worker(username) + with client.job_update_lock: + was_authorized = client.authorized + if was_authorized: + with runtime.lock: + runtime._ensure_initial_job_state() + if ( + client not in runtime.pending_initial_jobs + and len(runtime.pending_initial_jobs) + >= runtime.stratum_max_pending_initial_jobs + and runtime._client_has_current_tip_job_locked(client) + ): + # Reject before mutating identity or difficulty. A + # working session keeps its current authorization + # when there is no live first-job slot for the + # superseding generation. + raise StratumError( + 20, + "initial job delivery capacity unavailable", + disconnect=False, + ) + if not runtime.reserve_client_username(client, worker): + raise StratumError( + 20, + "too many connections for username", + # A new connection has no useful session to preserve. A + # live miner re-authorizing to a full username does: keep + # its prior worker/session active after returning the + # capacity error. + disconnect=not client.authorized, + ) + # The password is authoritative for password-derived options: a + # re-authorize without d=/md= clears any prior override (a stored + # suggest_difficulty still applies via the request resolution). + with runtime._client_vardiff_lock(client): + # Only the FIRST authorize of a connection may resume the + # worker's retained difficulty (a live session already + # holds a converged value), and it must land before the + # password options: an explicit d= computes its target + # from requested_difficulty and outranks the resume, + # while an md=-only password clamps the just-resumed + # value through the share_difficulty fallback. + resumed: Decimal | None = None + if not was_authorized: + resumed = runtime.resume_client_difficulty(client) + client.requested_difficulty, client.requested_min_difficulty = ( + parse_stratum_password_options(password) + ) + target = runtime.apply_client_difficulty_requests(client) + if target is not None: + current = client.pending_share_difficulty or client.share_difficulty + if resumed is not None and target != resumed: + # The resume applied a value and this authorize + # immediately replaced it, so it never stuck. + # resumed/clamped stay attempt counters; this + # subtracts the ones an explicit difficulty + # request (d=/md=, or a suggestion sent before + # authorize) overrode. + runtime.note_vardiff_resume_overridden() + if target != current: + if not was_authorized: + client.share_difficulty = target + client.pending_share_difficulty = None + else: + client.pending_share_difficulty = target + client.difficulty_generation = int( + getattr(client, "difficulty_generation", 0) + ) + 1 + client.authorization_generation = int( + getattr(client, "authorization_generation", 0) + ) + 1 + client.authorized = True + client.authorized_monotonic = time.monotonic() + runtime.send_result(client, request_id, True) + runtime._note_collection_identity_available(client) + # Exactly one coalesced delivery represents this authorization, + # including a password-derived difficulty change. + runtime.request_initial_job_delivery(client) + return + if method == "mining.extranonce.subscribe": + runtime.send_result(client, request_id, True) + return + if method == "mining.suggest_difficulty": + runtime.handle_suggest_difficulty(client, request_id, params) + return + if method == "mining.submit": + received_monotonic = getattr( + client, + "request_received_monotonic", + None, + ) + accepted_and_closed = runtime.handle_submit(client, params) + try: + runtime.send_result(client, request_id, True) + if received_monotonic is not None: + # Rejects are observed symmetrically in handle_client + # after their error response write. + runtime._observe_share_ack_seconds( + "accepted", + time.monotonic() - received_monotonic, + ) + finally: + runtime.refresh_jobs_after_pending_accepted_block(client) + if accepted_and_closed: + client.close() + return + raise StratumError(20, f"unsupported method {method}") + + def handle_suggest_difficulty( + self, client: ClientState, request_id: object, params: list[object] + ) -> None: + runtime = self._runtime + with client.job_update_lock: + suggested: Decimal | None = None + if params: + try: + suggested = Decimal(str(params[0])) + except Exception: + suggested = None + if suggested is not None and (not suggested.is_finite() or suggested <= 0): + suggested = None + if suggested is not None: + with runtime._client_vardiff_lock(client): + client.suggested_difficulty = suggested + target = runtime.apply_client_difficulty_requests(client) + if target is not None: + runtime.advertise_client_difficulty(client, target) + runtime.send_result(client, request_id, True) + + def handle_configure( + self, client: ClientState, request_id: object, params: list[object] + ) -> None: + runtime = self._runtime + extensions = params[0] if params else [] + extension_params = params[1] if len(params) > 1 and isinstance(params[1], dict) else {} + result: dict[str, object] = {} + if isinstance(extensions, list): + for extension in extensions: + if extension == "version-rolling": + miner_mask = 0xFFFFFFFF + if "version-rolling.mask" in extension_params: + miner_mask = stratum_codec.parse_mask_hex( + extension_params["version-rolling.mask"], + field_name="version-rolling.mask", + ) + client.version_mask = runtime.version_mask & miner_mask + result["version-rolling"] = client.version_mask != 0 + result["version-rolling.mask"] = stratum_codec.format_mask_hex(client.version_mask) + else: + result[str(extension)] = False + runtime.send_result(client, request_id, result) + + # -- worker resolution ------------------------------------------------- + + def resolve_worker(self, username: str) -> WorkerIdentity: + runtime = self._runtime + payout_address, worker_name = split_worker_username(username) + try: + if not payout_address: + raise StratumError(20, "username base is empty") + script, p2mr_program_hex = runtime.validate_p2mr_address( + payout_address, label="username base" + ) + except StratumError as username_error: + fallback_address = getattr( + runtime, + "username_fallback_address", + default_prism_username_fallback_address(), + ) + if fallback_address is None: + raise username_error + print( + f"prism coordinator: username {username!r} cannot be used as a payout " + f"({username_error.message}); using fallback payout {fallback_address}", + flush=True, + ) + payout_address = fallback_address + script, p2mr_program_hex = runtime.validate_p2mr_address( + fallback_address, + label="PRISM_USERNAME_FALLBACK_ADDRESS", + ) + return WorkerIdentity( + username=username, + payout_address=payout_address, + worker_name=worker_name, + script_pubkey_hex=script, + p2mr_program_hex=p2mr_program_hex, + ) + + # -- registry proof ---------------------------------------------------- + + def record_successful_delivery( + self, + client: ClientState, + context: object, + delivered_monotonic: float, + ) -> None: + """Commit exact registry delivery proof, then report to G1. + + Proof-before-health ordering is a lost-fix detector: within the one + shared critical section the S1 registry proof (and its staged-split + compatibility mirrors on the client object) commits first, and only + then does the progress-health owner observe the delivery. + """ + runtime = self._runtime + runtime._ensure_job_cache_state() + service = runtime._ensure_progress_health_service() + fingerprint = getattr(context, "template_fingerprint", None) + if fingerprint is None: + fingerprint = qbit_template_fingerprint(context.template) + payout_generation = int(getattr(context, "payout_state_generation", 0)) + delivered_tip = str(context.template.get("previousblockhash", "")) + recorded = False + coverage_reached: list[tuple[str, float]] = [] + with runtime.lock: + self.registry.record_delivery_locked( + client, context, delivered_monotonic + ) + client._progress_delivered_context = context + client._progress_delivered_template_fingerprint = fingerprint + client._progress_delivered_template_generation = int( + getattr(context, "template_generation", 0) + ) + client._progress_delivered_payout_generation = payout_generation + client._progress_delivered_monotonic = delivered_monotonic + ready_mode_required = bool( + getattr(runtime, "_pool_ready_latched", False) + ) + latest_detected = getattr(runtime, "latest_detected_tip", None) + recorded = service.record_delivery( + DeliveryProof( + connection_id=int(getattr(client, "connection_id", 0)), + delivered_work=WorkGeneration( + template_generation=int( + getattr(context, "template_generation", 0) + ), + template_fingerprint=fingerprint, + payout_generation=payout_generation, + ), + collection_only=bool( + getattr(context, "collection_only", False) + ), + delivered_monotonic=delivered_monotonic, + ), + ready_mode_required, + matches_latest_tip=bool( + latest_detected is None + or latest_detected[0] == delivered_tip + ), + ) + coverage_reached = ( + runtime._tip_refresh_epoch_coverage_reached_locked( + client, + context, + delivered_monotonic, + ) + ) + if recorded: + runtime._progress_note_refresh_activity(delivered_monotonic) + runtime._progress_reconcile_pending(now=delivered_monotonic) + runtime._record_tip_refresh_epoch_coverage(coverage_reached) + + +__all__ = [ + "ClientState", + "DeliveredSessionContext", + "EligibleSession", + "P2mrAddressValidator", + "SessionRegistry", + "SessionRuntime", + "StratumError", + "StratumSessionService", + "WorkerIdentity", + "_P2mrAddressValidationFlight", + "apply_stratum_send_timeout", + "client_can_receive_jobs", + "client_vardiff_lock", + "difficulty_payload", + "error_payload", + "job_payload", + "parse_stratum_password_options", + "parse_worker_username", + "result_payload", + "split_worker_username", + "stratum_accept_heartbeat_names", +] diff --git a/lab/prism/template_artifacts.py b/lab/prism/template_artifacts.py new file mode 100644 index 000000000..26c5dc04d --- /dev/null +++ b/lab/prism/template_artifacts.py @@ -0,0 +1,605 @@ +#!/usr/bin/env python3 +"""Immutable qbit template observations and their derived artifacts. + +The repository owns the coordinator's template-artifact cache and observation +generations. It does not import ``prism_coordinator``: every cross-domain +fact (tip publication, refresh scheduling, RPC access, cache-event metrics) +is reached through the :class:`TemplateArtifactRuntime` port, resolved at +call time so the historical coordinator monkeypatch seams keep working. +""" + +from __future__ import annotations + +import copy +from collections import OrderedDict +from dataclasses import dataclass, field +import hashlib +import json +import threading +import time +from typing import Any, Callable, Mapping, Protocol, Sequence + +from lab.prism import direct_stratum +from lab.prism.coordinator_config import DEFAULT_PRISM_BLOCKPOLL_SECONDS +# The refresh/publication exception family is owned by the payout-state +# module (its final owner); re-imported here so every existing import path +# keeps working. +from lab.prism.payout_state import ( + PayoutStatePublicationBlocked, # noqa: F401 - compatibility re-export + TemplateRefreshBlocked, # noqa: F401 - compatibility re-export + TemplateRefreshSuperseded, +) + + +PRISM_TEMPLATE_FINGERPRINT_VOLATILE_KEYS = frozenset( + { + # qbit can legitimately advance these without making already issued + # jobs stale. Rebuilding every miner job for clock-only changes would + # turn the poller into continuous audit-bundle churn. + "curtime", + "longpollid", + "mintime", + } +) + + +def qbit_template_fingerprint(template: dict[str, Any]) -> str: + stable_template = { + key: value + for key, value in template.items() + if key not in PRISM_TEMPLATE_FINGERPRINT_VOLATILE_KEYS + } + encoded = json.dumps( + stable_template, + sort_keys=True, + separators=(",", ":"), + default=str, + ).encode() + return hashlib.sha256(encoded).hexdigest() + + +def qbit_gbt_rules(chain: str) -> list[str]: + rules = ["segwit"] + if chain.strip().lower() == "signet": + rules.append("signet") + return rules + + +def target_from_compact(bits_hex: str) -> int: + return direct_stratum.target_from_compact_hex(bits_hex) + + +def scaled_target_difficulty(target: int) -> int: + if target <= 0: + raise ValueError("target must be positive") + pow_limit_target = target_from_compact("207fffff") + return max(1, (pow_limit_target * 1_000_000) // target) + + +def scaled_network_difficulty(bits_hex: str) -> int: + template_target = target_from_compact(bits_hex) + return scaled_target_difficulty(template_target) + + +class FrozenJsonDict(dict[str, Any]): + """JSON-compatible mapping that fails closed on mutation.""" + + __slots__ = () + + @staticmethod + def _immutable(*_args: object, **_kwargs: object) -> None: + raise TypeError("template artifact JSON is immutable") + + __setitem__ = _immutable + __delitem__ = _immutable + clear = _immutable + pop = _immutable + popitem = _immutable + setdefault = _immutable + update = _immutable + __ior__ = _immutable + + def __copy__(self) -> "FrozenJsonDict": + return self + + def __deepcopy__(self, _memo: dict[int, object]) -> "FrozenJsonDict": + return self + + +class FrozenJsonList(list[Any]): + """JSON-compatible sequence that fails closed on mutation.""" + + __slots__ = () + + @staticmethod + def _immutable(*_args: object, **_kwargs: object) -> None: + raise TypeError("template artifact JSON is immutable") + + __setitem__ = _immutable + __delitem__ = _immutable + __iadd__ = _immutable + __imul__ = _immutable + append = _immutable + clear = _immutable + extend = _immutable + insert = _immutable + pop = _immutable + remove = _immutable + reverse = _immutable + sort = _immutable + + def __copy__(self) -> "FrozenJsonList": + return self + + def __deepcopy__(self, _memo: dict[int, object]) -> "FrozenJsonList": + return self + + +def freeze_json(value: Any) -> Any: + """Detach and recursively freeze a JSON-like value without changing shape.""" + if isinstance(value, (FrozenJsonDict, FrozenJsonList)): + return value + if isinstance(value, Mapping): + frozen_dict = FrozenJsonDict() + dict.update( + frozen_dict, + ((str(key), freeze_json(item)) for key, item in value.items()), + ) + return frozen_dict + if isinstance(value, (list, tuple)): + frozen_list = FrozenJsonList() + list.extend(frozen_list, (freeze_json(item) for item in value)) + return frozen_list + return value + + +def freeze_json_rows(rows: Sequence[Mapping[str, Any]]) -> FrozenJsonList: + frozen = freeze_json(rows) + if not isinstance(frozen, FrozenJsonList): + raise TypeError("JSON rows must be a sequence") + return frozen + + +@dataclass(frozen=True) +class CachedTemplateArtifacts: + """Template plus everything derivable from it alone, shared by all clients. + + Derived fields are keyed by the template fingerprint: a refetch whose + fingerprint matches (only clock fields moved) reuses the previously + computed transaction hexes and witness merkle leaves instead of re-hashing + the full template. Generation records observation-start order so a slow, + older fetch cannot supersede a newer observation merely by finishing last. + """ + + template: dict[str, Any] + fingerprint: str + previousblockhash: str + transaction_hexes: tuple[str, ...] + witness_merkle_leaves_hex: tuple[str, ...] + network_difficulty: int + fetched_monotonic: float + generation: int = 0 + + +@dataclass(frozen=True) +class QbitTipTemplateSnapshot: + bestblockhash: str + previousblockhash: str + template_fingerprint: str + template_generation: int = 0 + # The observation owns this exact artifact object. It deliberately does + # not participate in snapshot equality: callers compare the stable + # identity fields above, while refresh preparation consumes the exact + # template and derivations that were observed even if the mutable cache's + # current pointer is replaced concurrently. + template_artifacts: CachedTemplateArtifacts | None = field( + default=None, + compare=False, + repr=False, + ) + + +class TemplateArtifactRuntime(Protocol): + """Typed port over the coordinator, resolved at call time. + + The repository reaches every cross-domain fact through this object so + instance-level monkeypatches on the coordinator (the historical test + seams) keep intercepting exactly as before the extraction. + """ + + rpc: Any + lock: Any + + def _ensure_job_cache_state(self) -> None: ... + + def _ensure_tip_refresh_state(self) -> None: ... + + def _job_build_phases(self) -> dict[str, float]: ... + + def _record_job_cache_event(self, kind: str, *, hit: bool) -> None: ... + + def _newest_observed_tip_locked(self) -> str | None: ... + + def _published_tip_authoritative_locked(self, now: float) -> bool: ... + + def _cancel_obsolete_job_builds( + self, + reason: str, + *, + keep_published_snapshot: bool = False, + ) -> None: ... + + def observe_tip_for_refresh(self, tip_hash: str) -> object: ... + + def _schedule_tip_refresh_retry(self) -> None: ... + + def store_template_artifacts( + self, + template: dict[str, Any], + *, + generation: int | None = None, + ) -> CachedTemplateArtifacts | None: ... + + def current_template_artifacts(self) -> CachedTemplateArtifacts: ... + + +class TemplateArtifactRepository: + """Sole owner of the template-artifact cache and observation generations. + + The repository deliberately shares the job-bundle cache lock: template + currency and job-bundle admission were historically guarded by one + ``_job_cache_lock`` and remaining coordinator domains still take that + lock around combined reads. + """ + + def __init__( + self, + runtime: TemplateArtifactRuntime, + *, + lock: threading.Lock, + ) -> None: + self._runtime = runtime + self._job_cache_lock = lock + self._template_artifacts: CachedTemplateArtifacts | None = None + self._template_artifact_generation = 0 + # Whether the generation counter was ever set explicitly (test + # embedders seed it through the coordinator compatibility field). + self._generation_explicit = False + + # -- compatibility adoption ------------------------------------------- + + def adopt_template_artifacts( + self, + artifacts: CachedTemplateArtifacts | None, + ) -> None: + """Adopt a directly assigned artifact object (legacy test seeding). + + Mirrors the historical first-touch initializer: a test that plants + ``_template_artifacts`` before touching the generation counter also + seeded the counter from the planted artifacts. + """ + self._template_artifacts = artifacts + if artifacts is not None and not self._generation_explicit: + self._template_artifact_generation = max( + self._template_artifact_generation, + int(getattr(artifacts, "generation", 0)), + ) + + def adopt_template_artifact_generation(self, generation: int) -> None: + self._template_artifact_generation = generation + self._generation_explicit = True + + # -- owner methods (bodies moved verbatim from the coordinator) -------- + + def reserve_generation(self) -> int: + """Reserve template ordering when a fetch starts, not when it finishes.""" + runtime = self._runtime + runtime._ensure_job_cache_state() + with self._job_cache_lock: + self._template_artifact_generation += 1 + return self._template_artifact_generation + + def derive( + self, + template: dict[str, Any], + *, + generation: int, + ) -> CachedTemplateArtifacts: + runtime = self._runtime + # Detach the observation from mutable RPC/test caller state. The + # dataclass then owns this exact tree for the lifetime of its snapshot. + template = copy.deepcopy(template) + fingerprint = qbit_template_fingerprint(template) + with self._job_cache_lock: + previous = self._template_artifacts + if previous is not None and previous.fingerprint == fingerprint: + return CachedTemplateArtifacts( + template=template, + fingerprint=fingerprint, + previousblockhash=str(template.get("previousblockhash", "")), + transaction_hexes=previous.transaction_hexes, + witness_merkle_leaves_hex=previous.witness_merkle_leaves_hex, + network_difficulty=previous.network_difficulty, + fetched_monotonic=time.monotonic(), + generation=generation, + ) + phases = runtime._job_build_phases() + started = time.monotonic() + transaction_hexes = direct_stratum.transaction_hexes_from_template(template) + witness_leaves = tuple(direct_stratum.witness_merkle_leaves_hex(transaction_hexes)) + network_difficulty = scaled_network_difficulty(str(template["bits"])) + phases["merkle"] = phases.get("merkle", 0.0) + (time.monotonic() - started) + return CachedTemplateArtifacts( + template=template, + fingerprint=fingerprint, + previousblockhash=str(template.get("previousblockhash", "")), + transaction_hexes=transaction_hexes, + witness_merkle_leaves_hex=witness_leaves, + network_difficulty=network_difficulty, + fetched_monotonic=time.monotonic(), + generation=generation, + ) + + def store_artifacts(self, artifacts: CachedTemplateArtifacts) -> bool: + runtime = self._runtime + changed = False + with self._job_cache_lock: + previous = self._template_artifacts + if previous is not None and artifacts.generation < previous.generation: + return False + self._template_artifacts = artifacts + if previous is not None and previous.fingerprint != artifacts.fingerprint: + changed = True + # Retain the published snapshot's bundles alongside the new + # fingerprint: until the replacement tip is published, direct + # issuance still serves published-snapshot work, and pruning + # it here would force those paths to defer for the whole + # detected-but-unpublished window. Publication moves the + # snapshot, so the next fingerprint change drops the old + # entries. + keep_fingerprints = {artifacts.fingerprint} + with runtime.lock: + published_snapshot = getattr(runtime, "tip_template_snapshot", None) + if published_snapshot is not None: + keep_fingerprints.add(published_snapshot.template_fingerprint) + runtime._job_bundle_cache = OrderedDict( + (key, entry) + for key, entry in runtime._job_bundle_cache.items() + if entry.template_fingerprint in keep_fingerprints + ) + if changed: + runtime._cancel_obsolete_job_builds( + "template fingerprint superseded", + keep_published_snapshot=True, + ) + return True + + def store( + self, + template: dict[str, Any], + *, + generation: int | None = None, + ) -> CachedTemplateArtifacts | None: + """Best-effort cache fill from an already-fetched template (blockpoll). + + Returns None instead of raising so a template the derivation cannot + digest degrades to the legacy per-build fetch path rather than failing + the poll. The returned artifacts describe this exact observation even + if a newer observation already won the cache-write race; blockpoll then + detects the mismatch before fanout. + """ + runtime = self._runtime + runtime._ensure_job_cache_state() + if generation is None: + generation = runtime._reserve_template_artifact_generation() + try: + artifacts = runtime._derive_template_artifacts( + template, + generation=generation, + ) + except Exception: + return None + runtime._store_template_artifacts(artifacts) + return artifacts + + def issuance(self) -> CachedTemplateArtifacts: + """Template artifacts for direct (non-refresh) job issuance. + + While a detected replacement tip is still unpublished, share + classification remains anchored to the published tip. Direct issuance + paths (initial delivery retries, Vardiff retargets, reauthorization) + must therefore keep handing out published-snapshot work; issuing + detected-tip work early would have every one of its shares rejected + as stale until the refresh publishes. Once the published authority + lapses (divergence lease expired), issuance falls through to the live + template exactly like submit classification falls back to the live + RPC read. + """ + runtime = self._runtime + runtime._ensure_tip_refresh_state() + with runtime.lock: + published = getattr(runtime, "current_tip_first_seen", None) + latest_detected = getattr(runtime, "latest_detected_tip", None) + published_snapshot = getattr(runtime, "tip_template_snapshot", None) + pinned = bool( + published is not None + and latest_detected is not None + and latest_detected[0] != published[0] + and published_snapshot is not None + and published_snapshot.bestblockhash == published[0] + and published_snapshot.template_artifacts is not None + and runtime._published_tip_authoritative_locked(time.monotonic()) + ) + if pinned: + assert published_snapshot is not None + assert published_snapshot.template_artifacts is not None + return published_snapshot.template_artifacts + artifacts = runtime.current_template_artifacts() + # The fetch above may itself be the first observation of a newer tip + # (a template-cache miss racing ahead of blockpoll/blockwait). The + # published tip still owns share classification, so serve its + # snapshot; the recorded detection has already armed the refresh. + with runtime.lock: + published = getattr(runtime, "current_tip_first_seen", None) + published_snapshot = getattr(runtime, "tip_template_snapshot", None) + repinned = bool( + published is not None + and artifacts.previousblockhash != published[0] + and published_snapshot is not None + and published_snapshot.bestblockhash == published[0] + and published_snapshot.template_artifacts is not None + and runtime._published_tip_authoritative_locked(time.monotonic()) + ) + if repinned: + assert published_snapshot is not None + assert published_snapshot.template_artifacts is not None + return published_snapshot.template_artifacts + return artifacts + + def current(self) -> CachedTemplateArtifacts: + """Return fresh template artifacts, fetching a template on cache miss.""" + runtime = self._runtime + runtime._ensure_job_cache_state() + ttl = getattr(runtime, "template_cache_seconds", DEFAULT_PRISM_BLOCKPOLL_SECONDS) + now = time.monotonic() + with self._job_cache_lock: + cached = self._template_artifacts + with runtime.lock: + observed_tip = runtime._newest_observed_tip_locked() + cached_tip_current = ( + observed_tip is None + or cached is None + or cached.previousblockhash == observed_tip + ) + if ( + cached is not None + and cached_tip_current + and ttl > 0 + and now - cached.fetched_monotonic <= ttl + ): + runtime._record_job_cache_event("template", hit=True) + return cached + runtime._record_job_cache_event("template", hit=False) + generation = runtime._reserve_template_artifact_generation() + phases = runtime._job_build_phases() + started = time.monotonic() + template = runtime.rpc.call( + "getblocktemplate", + [{"rules": qbit_gbt_rules(getattr(runtime, "qbit_chain", "regtest"))}], + ) + if not isinstance(template, dict): + raise RuntimeError("getblocktemplate returned non-object") + phases["template"] = phases.get("template", 0.0) + (time.monotonic() - started) + artifacts = runtime._derive_template_artifacts( + template, + generation=generation, + ) + if runtime._store_template_artifacts(artifacts): + # A direct fetch can be the first reader to see qbit advance. + # Record it as a detection like every other live-tip observation, + # or the buildability gates would treat the newer parent as + # unknown while pinned issuance still waits on blockpoll. + runtime.observe_tip_for_refresh(artifacts.previousblockhash) + return artifacts + # A later fetch completed first. Build from that current observation, + # never from the stale response that lost the cache-write race. + with self._job_cache_lock: + current = self._template_artifacts + if current is None: + raise RuntimeError("newer template artifacts disappeared after cache race") + runtime.observe_tip_for_refresh(current.previousblockhash) + return current + + def artifacts_are_current(self, artifacts: CachedTemplateArtifacts) -> bool: + runtime = self._runtime + runtime._ensure_job_cache_state() + with self._job_cache_lock: + current = self._template_artifacts + return ( + current is artifacts + or ( + current is not None + and current.fingerprint == artifacts.fingerprint + and current.generation == artifacts.generation + ) + ) + + def fetch_coherent_snapshot(self) -> QbitTipTemplateSnapshot: + runtime = self._runtime + # Reserve ordering before either RPC: a fetch that started on an older + # view must not become "newer" merely because its template arrived last. + generation = runtime._reserve_template_artifact_generation() + template = runtime.rpc.call( + "getblocktemplate", + [{"rules": qbit_gbt_rules(getattr(runtime, "qbit_chain", "regtest"))}], + ) + if not isinstance(template, dict): + raise RuntimeError("getblocktemplate returned non-object") + previousblockhash = str(template.get("previousblockhash", "") or "") + if not previousblockhash: + raise RuntimeError("getblocktemplate omitted previousblockhash") + # The template parent is the tip this work actually extends. Validate + # it after fetching the template so a tip transition between these RPCs + # cannot produce an old bestblockhash paired with newer work. Reject a + # template that was superseded before it can enter the shared cache or + # drive tip observation/graveyard pruning. + bestblockhash = str(runtime.rpc.call("getbestblockhash")) + if bestblockhash != previousblockhash: + # Record the discovery before failing: the retry-spacing gate + # releases on a newer DETECTED tip, and without blockwait this + # mismatch is the only place the new tip becomes known. An + # unrecorded discovery would hold the immediate new-tip retry + # for the full failure holdoff. + runtime.observe_tip_for_refresh(bestblockhash) + runtime._schedule_tip_refresh_retry() + raise TemplateRefreshSuperseded( + "qbit tip changed while fetching block template " + f"template_parent={previousblockhash} current={bestblockhash}" + ) + # The poll already paid for this template; seed the job-build cache so + # client job builds triggered by the refresh below reuse it instead of + # refetching one template per client. + artifacts = runtime.store_template_artifacts( + template, + generation=generation, + ) + if artifacts is not None: + return QbitTipTemplateSnapshot( + bestblockhash=bestblockhash, + previousblockhash=artifacts.previousblockhash, + template_fingerprint=artifacts.fingerprint, + template_generation=artifacts.generation, + template_artifacts=artifacts, + ) + raise TemplateRefreshBlocked( + "unable to derive exact artifacts for observed qbit template" + ) + + def current_artifacts(self) -> CachedTemplateArtifacts | None: + with self._job_cache_lock: + return self._template_artifacts + + +# Callable alias used by embedders that need a repository-compatible signature. +TemplateArtifactFetch = Callable[[], QbitTipTemplateSnapshot] + + +__all__ = [ + "CachedTemplateArtifacts", + "FrozenJsonDict", + "FrozenJsonList", + "PRISM_TEMPLATE_FINGERPRINT_VOLATILE_KEYS", + "PayoutStatePublicationBlocked", + "QbitTipTemplateSnapshot", + "TemplateArtifactRepository", + "TemplateArtifactRuntime", + "TemplateRefreshBlocked", + "TemplateRefreshSuperseded", + "freeze_json", + "freeze_json_rows", + "qbit_gbt_rules", + "qbit_template_fingerprint", + "scaled_network_difficulty", + "scaled_target_difficulty", + "target_from_compact", +] diff --git a/lab/prism/tip_refresh.py b/lab/prism/tip_refresh.py new file mode 100644 index 000000000..575532926 --- /dev/null +++ b/lab/prism/tip_refresh.py @@ -0,0 +1,4092 @@ +#!/usr/bin/env python3 +"""PRISM tip observation, publication, and bounded refresh ownership. + +The service owns the R1 domain: tip observation/publication state, the +latest-wins refresh waves and their epoch convergence authority (#101), +refresh pending/retry signaling, failure holdoff pacing, fanout delivery +coordination, blockpoll/blockwait loops, retained collection refreshes, and +the tip-refresh metric state. + +It never imports ``prism_coordinator``. Every cross-domain fact -- payout +state, job-bundle preparation, session membership, delivery submission, +reorg reconciliation, progress health, watchdog heartbeats, live +configuration attributes -- is reached through the :class:`TipRefreshRuntime` +typed port, resolved at call time so the historical coordinator monkeypatch +seams (including the instance-level facade patches used by the current test +suite) keep intercepting exactly as before the extraction. Reconcile +flights/memo/prefetch and share-ack timing remain coordinator-owned at this +layer (they belong to later stack owners); R1 reaches them only through +call-time runtime ports. +""" + +from __future__ import annotations + +from concurrent.futures import ( + FIRST_COMPLETED, + CancelledError as FuturesCancelledError, + Future, + ThreadPoolExecutor, + wait, +) +from dataclasses import dataclass, field +from decimal import Decimal, InvalidOperation +import os +import random +import threading +import time +import traceback +from typing import Any, Protocol + +from lab.prism.bounded_executor import _BoundedPriorityExecutor +from lab.prism.coordinator_config import ( + DEFAULT_PRISM_BLOCKPOLL_SECONDS, + DEFAULT_PRISM_BLOCKWAIT_TIMEOUT_SECONDS, + DEFAULT_PRISM_REORG_RECONCILE_CACHE_SECONDS, + DEFAULT_PRISM_SUBMIT_TIP_MAX_AGE_SECONDS, + DEFAULT_PRISM_TEMPLATE_MAX_AGE_SECONDS, + DEFAULT_PRISM_TIP_REFRESH_FAILURE_HOLDOFF_SECONDS, +) +from lab.prism.job_bundle import CachedJobBundle, JobBuildKey, JobBuildSuperseded +from lab.prism.template_artifacts import ( + CachedTemplateArtifacts, + PayoutStatePublicationBlocked, + QbitTipTemplateSnapshot, + TemplateRefreshBlocked, + TemplateRefreshSuperseded, + qbit_template_fingerprint, +) + + +# Owner-local copies of still-coordinator-visible constants; compatibility +# duplicates by design (the coordinator's import source for the admission +# poll interval is job_delivery.py, and the payout supersession-retry +# default remains a coordinator/P1 definition at this layer). +PRISM_TIP_REFRESH_ADMISSION_POLL_SECONDS = 0.05 +DEFAULT_PRISM_PAYOUT_RECONCILE_SUPERSESSION_RETRIES = 8 +# Fraction of the holdoff added as random jitter so coordinators sharing one +# qbitd do not phase-lock their blocked-refresh re-attempts. +PRISM_TIP_REFRESH_FAILURE_HOLDOFF_JITTER_FRACTION = 0.25 +PRISM_TIP_REFRESH_REENTRY_BACKOFF_SECONDS = 0.05 +PRISM_TIP_REFRESH_WAVE_PASS_BUDGET = 64 +PRISM_TIP_REFRESH_SECONDS_BUCKETS = ( + 0.01, + 0.025, + 0.05, + 0.1, + 0.25, + 0.5, + 1.0, + 2.5, + 5.0, + 10.0, + 30.0, +) +PRISM_TIP_REFRESH_BUILD_PHASES = ( + "reorg_reconcile", + "ledger_snapshot", + "payout_state_derivation", + "ctv_manifest_construction", + "coinbase_bundle_construction", + "signing_verification", + "serialization_copy", + "singleflight_wait", +) +PRISM_TIP_REFRESH_RESULTS = ("sent", "skipped", "disconnected", "failed") +PRISM_TIP_REFRESH_CANCELLATION_STAGES = ( + "executor_queue", + "client_lock", + "payout_gate", +) +PRISM_TIP_REFRESH_WAVE_OUTCOMES = ( + "completed", + "fanout_superseded", + "build_superseded", + "payout_blocked", + "trust_blocked", + "shutdown", + "error", +) +PRISM_TIP_REFRESH_COVERAGE_TARGETS = ( + ("50", Decimal("0.50")), + ("95", Decimal("0.95")), + ("99", Decimal("0.99")), +) + + +@dataclass(frozen=True) +class RetainedCollectionRefresh: + """Current immutable preparation waiting for a collection identity.""" + + snapshot: QbitTipTemplateSnapshot + observation_sequence: int + payout_state_generation: int + + +@dataclass(frozen=True) +class RefreshResult: + result: str + delivered_monotonic: float | None = None + + +@dataclass(frozen=True, eq=False) +class TipRefreshValidationToken: + """Immutable proof that one prepared refresh passed its expensive guard.""" + + tip_hash: str + template_fingerprint: str + template_generation: int + payout_state_generation: int + observation_sequence: int + build_key: JobBuildKey + snapshot: QbitTipTemplateSnapshot = field(repr=False) + epoch_sequence: int = 0 + + +@dataclass +class _TipRefreshEpochCoverage: + """Client cohort snapshot used to observe fixed-cardinality coverage latency.""" + + sequence: int + tip_hash: str + payout_state_generation: int + started_monotonic: float + client_weights: dict[int, Decimal] + total_weight: Decimal + delivered_weight: Decimal = Decimal(0) + delivered_clients: set[int] = field(default_factory=set) + recorded_targets: set[str] = field(default_factory=set) + + +class FanoutCancellation: + """Cancel a fanout without racing already-admitted deliveries. + + ``cancel`` closes admission without waiting, so workers can call it while + holding a client lock. The fanout coordinator calls ``set`` outside client + locks to wait for deliveries that already passed the final gate. + """ + + def __init__(self) -> None: + self._condition = threading.Condition() + self._cancelling = False + self._active_deliveries = 0 + + def is_set(self) -> bool: + with self._condition: + return self._cancelling + + def begin_delivery(self) -> bool: + with self._condition: + if self._cancelling: + return False + self._active_deliveries += 1 + return True + + def end_delivery(self) -> None: + with self._condition: + if self._active_deliveries <= 0: + raise RuntimeError("fanout delivery gate released without admission") + self._active_deliveries -= 1 + if self._active_deliveries == 0: + self._condition.notify_all() + + def cancel(self) -> None: + with self._condition: + self._cancelling = True + + def set(self) -> None: + self.cancel() + with self._condition: + while self._active_deliveries: + self._condition.wait() + + +class _TipRefreshFanoutSuperseded(TemplateRefreshSuperseded): + """A published fanout lost to a newer tip-refresh epoch.""" + + +class _TipRefreshTrustBlocked(TemplateRefreshBlocked): + """Refresh work stopped because the live chain view was not trusted.""" + + +class TipRefreshRuntime(Protocol): + """Typed port over the coordinator, resolved at call time. + + Every member is looked up on the live coordinator object when used, so + instance monkeypatches (``server.maybe_send_job = ...`` and friends) and + coordinator-owned live configuration attributes keep working exactly as + before the extraction. Owner facades route back into this service; that + round trip is deliberate: it is what preserves the current white-box + patch surface until later stack layers repoint those tests. The legacy + R1 field names also resolve here -- coordinator class descriptors route + them to this service's single mutable copy. + """ + + # Cross-domain objects and live configuration attributes. + _job_cache_lock: Any + _payout_artifact_executor_lock: Any + _payout_ledger_append_invalidation_epoch: Any + _payout_state_delivery_gate: Any + _payout_state_generation: Any + _payout_state_publication_blocked: Any + _payout_state_source: Any + _pool_ready_latched: Any + _prepared_ready_bundle: Any + _prepared_ready_snapshot: Any + _progress_has_published_work: Any + _progress_health_lock: Any + _progress_publication_divergence_since_monotonic: Any + _progress_published_payout_generation: Any + _progress_published_template_fingerprint: Any + _progress_refresh_signal_pending: Any + _published_payout_state: Any + _serve_builder_metrics_lock: Any + _template_artifacts: Any + blockpoll_seconds: Any + blockwait_timeout_seconds: Any + clients: Any + job_build_failure_count: Any + job_preparation_pending: Any + lock: Any + payout_artifact_event_counts: Any + payout_reconcile_supersession_retries: Any + reorg_reconcile_cache_seconds: Any + reorg_reconciler_enabled: Any + rpc: Any + serve_builder_counts: Any + serve_builder_window_cache_counts: Any + stop_event: Any + submit_tip_max_age_seconds: Any + template_cache_seconds: Any + template_refresh_failure_exit_seconds: Any + tip_refresh_epoch_fanout: Any + tip_refresh_failure_holdoff_seconds: Any + tip_refresh_max_workers: Any + watchdog_timeout_seconds: Any + + def _artifacts_buildable_locked(self, *args: Any, **kwargs: Any) -> Any: ... + + def _begin_job_build_priority_preparation(self, *args: Any, **kwargs: Any) -> Any: ... + + def _blockwait_unsupported(self, *args: Any, **kwargs: Any) -> Any: ... + + def _cancel_obsolete_job_builds(self, *args: Any, **kwargs: Any) -> Any: ... + + def _cancel_obsolete_job_bundle_builds(self, *args: Any, **kwargs: Any) -> Any: ... + + def _clear_coordination_blocked_streak(self, *args: Any, **kwargs: Any) -> Any: ... + + def _discard_stale_reconcile_prefetch(self, *args: Any, **kwargs: Any) -> Any: ... + + def _ensure_job_cache_state(self, *args: Any, **kwargs: Any) -> Any: ... + + def _ensure_tip_refresh_state(self, *args: Any, **kwargs: Any) -> Any: ... + + def _evict_reorg_reconcile_memo_for_new_tip_locked(self, *args: Any, **kwargs: Any) -> Any: ... + + def _finish_job_build_priority_preparation(self, *args: Any, **kwargs: Any) -> Any: ... + + def _join_reconcile_prefetch_bounded(self, *args: Any, **kwargs: Any) -> Any: ... + + def _newest_observed_tip_locked(self, *args: Any, **kwargs: Any) -> Any: ... + + def _note_tip_observation_for_candidates(self, *args: Any, **kwargs: Any) -> Any: ... + + def _progress_bundle_build_finished(self, *args: Any, **kwargs: Any) -> Any: ... + + def _progress_bundle_build_started(self, *args: Any, **kwargs: Any) -> Any: ... + + def _progress_note_refresh_pending(self, *args: Any, **kwargs: Any) -> Any: ... + + def _progress_refresh_finished(self, *args: Any, **kwargs: Any) -> Any: ... + + def _progress_refresh_started(self, *args: Any, **kwargs: Any) -> Any: ... + + def _reconcile_snapshot_tip_bounded(self, *args: Any, **kwargs: Any) -> Any: ... + + def _record_coordination_blocked_refresh(self, *args: Any, **kwargs: Any) -> Any: ... + + def _record_heartbeat(self, *args: Any, **kwargs: Any) -> Any: ... + + def _record_job_cache_event(self, *args: Any, **kwargs: Any) -> Any: ... + + def _record_progress_publication(self, *args: Any, **kwargs: Any) -> Any: ... + + def _record_progress_tip_poll(self, *args: Any, **kwargs: Any) -> Any: ... + + def _record_reorg_reconcile_lookup(self, *args: Any, **kwargs: Any) -> Any: ... + + def _record_startup_phase_once(self, *args: Any, **kwargs: Any) -> Any: ... + + def _remove_watchdog_heartbeat(self, *args: Any, **kwargs: Any) -> Any: ... + + def _reorg_reconcile_memo_fresh(self, *args: Any, **kwargs: Any) -> Any: ... + + def _submit_delivery_task(self, *args: Any, **kwargs: Any) -> Any: ... + + def _submit_reconcile_prefetch(self, *args: Any, **kwargs: Any) -> Any: ... + + def _submit_stale_check_tip_locked(self, *args: Any, **kwargs: Any) -> Any: ... + + def delivery_queue_limit(self, *args: Any, **kwargs: Any) -> Any: ... + + def disconnect_client(self, *args: Any, **kwargs: Any) -> Any: ... + + def ensure_reorg_reconciled_for_current_tip(self, *args: Any, **kwargs: Any) -> Any: ... + + def ensure_reorg_reconciled_for_tip(self, *args: Any, **kwargs: Any) -> Any: ... + + def fetch_qbit_tip_template_snapshot(self, *args: Any, **kwargs: Any) -> Any: ... + + def observe_job_build_elapsed(self, *args: Any, **kwargs: Any) -> Any: ... + + def pool_readiness_latched(self, *args: Any, **kwargs: Any) -> Any: ... + + def qbit_chain_view_untrusted(self, *args: Any, **kwargs: Any) -> Any: ... + + def retire_share_window_spool(self, *args: Any, **kwargs: Any) -> Any: ... + + def shutdown_job_build_executor(self, *args: Any, **kwargs: Any) -> Any: ... + + def shutdown_payout_artifact_executor(self, *args: Any, **kwargs: Any) -> Any: ... + + def shutdown_reconcile_prefetch_executor(self, *args: Any, **kwargs: Any) -> Any: ... + + def shutdown_serve_builder(self, *args: Any, **kwargs: Any) -> Any: ... + + def send_prepared_job(self, *args: Any, **kwargs: Any) -> Any: ... + + def shared_job_bundle(self, *args: Any, **kwargs: Any) -> Any: ... + + +class TipRefreshService: + """Sole owner of R1 tip observation, refresh scheduling, and metrics.""" + + def __init__( + self, + runtime: TipRefreshRuntime, + *, + shutdown_error: type[BaseException], + job_build_failed_error: type[BaseException], + delivery_priority_initial: int, + delivery_priority_new_tip: int, + delivery_priority_same_tip: int, + ) -> None: + self._runtime = runtime + # Coordinator-owned exception types and S2 delivery priorities are + # injected so this leaf module never imports prism_coordinator (or + # job_delivery, which sits above it in the owner layering). + self._shutdown_error = shutdown_error + self._job_build_failed_error = job_build_failed_error + self._delivery_priority_initial = delivery_priority_initial + self._delivery_priority_new_tip = delivery_priority_new_tip + self._delivery_priority_same_tip = delivery_priority_same_tip + self._tip_refresh_lock = threading.Lock() + self._tip_refresh_singleflight_lock = threading.Lock() + self._tip_refresh_executor_lock = threading.Lock() + self._tip_refresh_executor: _BoundedPriorityExecutor | None = None + self._tip_refresh_executor_shutdown = False + self._tip_refresh_metrics_lock = threading.Lock() + self.tip_refresh_histograms = { + name: { + "buckets": {bucket: 0 for bucket in PRISM_TIP_REFRESH_SECONDS_BUCKETS}, + "sum": 0.0, + "count": 0, + } + for name in ( + "refresh", + "bundle_build", + "first_delivery", + "last_delivery", + "fanout_wave", + ) + } + self.tip_refresh_build_phase_histograms = { + phase: { + "buckets": { + bucket: 0 for bucket in PRISM_TIP_REFRESH_SECONDS_BUCKETS + }, + "sum": 0.0, + "count": 0, + } + for phase in PRISM_TIP_REFRESH_BUILD_PHASES + } + self.tip_refresh_client_counts = { + result: 0 for result in PRISM_TIP_REFRESH_RESULTS + } + self.tip_refresh_cancellation_counts = { + stage: 0 for stage in PRISM_TIP_REFRESH_CANCELLATION_STAGES + } + self.tip_refresh_wave_outcome_counts = { + outcome: 0 for outcome in PRISM_TIP_REFRESH_WAVE_OUTCOMES + } + self.tip_refresh_coverage_histograms = { + target: { + "buckets": { + bucket: 0 for bucket in PRISM_TIP_REFRESH_SECONDS_BUCKETS + }, + "sum": 0.0, + "count": 0, + } + for target, _ratio in PRISM_TIP_REFRESH_COVERAGE_TARGETS + } + self.tip_refresh_inflight = 0 + self.tip_refresh_build_inflight = 0 + self.tip_refresh_build_queue_depth = 0 + self.tip_refresh_singleflight_hits = 0 + self.tip_refresh_superseded_results = 0 + self.tip_refresh_worker_failures = 0 + self.tip_refresh_worker_restarts = 0 + self.tip_refresh_ipc_bytes = {"input": 0, "output": 0} + self._tip_refresh_pending_event = threading.Event() + self._tip_refresh_pending_counter = 0 + self._tip_refresh_pending_token: int | None = None + self._tip_refresh_retry = threading.Event() + self._tip_refresh_retry_counter = 0 + self._tip_refresh_retry_consumed = 0 + self._tip_refresh_failure_holdoff_until: float | None = None + self._tip_refresh_failure_tip: str | None = None + self._active_tip_refresh: tuple[ + TipRefreshValidationToken, + FanoutCancellation, + ] | None = None + self._tip_refresh_epoch_sequence = 0 + self._tip_refresh_epoch_tip_hash: str | None = None + self._tip_refresh_epoch_payout_generation = 0 + self._tip_refresh_epoch_coverage: _TipRefreshEpochCoverage | None = None + self._published_tip_refresh_epoch_identity: tuple[ + int, + str, + int, + str, + int, + ] | None = None + self._retained_collection_refresh: RetainedCollectionRefresh | None = None + # The twelve historical plain coordinator fields (plus the + # tip-detection epoch) are deliberately NOT initialized here: they + # come into existence on first write exactly as the pre-extraction + # lazy fields did, so ``hasattr``/``getattr``-with-default legacy + # semantics (and the coordinator descriptor shims that route the old + # names to this service) stay byte-for-byte compatible: + # current_tip_first_seen, current_tip_parent, + # current_tip_observation_sequence, current_tip_observed_monotonic, + # tip_template_snapshot, latest_detected_tip, + # tip_refresh_divergence_started_monotonic, tip_observation_sequence, + # last_successful_template_refresh_monotonic, + # template_refresh_failure_started_monotonic, tip_refresh_job_count, + # post_accept_refresh_failure_count, tip_detection_epoch. + + def _retain_collection_refresh( + self, + snapshot: QbitTipTemplateSnapshot, + observation_sequence: int, + payout_state_generation: int, + ) -> None: + """Retain reusable current work until an eligible identity appears.""" + runtime = self._runtime + retained = RetainedCollectionRefresh( + snapshot=snapshot, + observation_sequence=observation_sequence, + payout_state_generation=payout_state_generation, + ) + should_log = False + with runtime.lock: + if not runtime._tip_refresh_snapshot_current_locked( + snapshot, + observation_sequence, + ): + return + if any( + runtime.client_can_receive_jobs(client) + for client in runtime.clients + ): + return + previous = self._retained_collection_refresh + self._retained_collection_refresh = retained + should_log = previous is None or ( + previous.snapshot.bestblockhash != snapshot.bestblockhash + or previous.payout_state_generation != payout_state_generation + ) + if should_log: + print( + "prism coordinator: collection refresh retained while no " + "authorized worker identity is available", + flush=True, + ) + + def _retained_collection_artifacts(self) -> CachedTemplateArtifacts | None: + """Return retained artifacts while their published work stays current. + + A same-tip poll advances its observation sequence before atomically + replacing ``tip_template_snapshot``. The published snapshot remains + reusable on both sides of that handoff even if the retained marker has + not yet been updated; a new tip or payout generation still invalidates + it immediately. + """ + runtime = self._runtime + runtime._ensure_job_cache_state() + runtime._ensure_tip_refresh_state() + with runtime._job_cache_lock: + payout_state_generation = runtime._payout_state_generation + with runtime.lock: + retained = self._retained_collection_refresh + if retained is None: + return None + if retained.payout_state_generation != payout_state_generation: + return None + current_tip = getattr(self, "current_tip_first_seen", None) + published_snapshot = self.tip_template_snapshot + if ( + published_snapshot is None + or current_tip is None + or current_tip[0] != published_snapshot.bestblockhash + ): + return None + return runtime._tip_refresh_artifacts(published_snapshot) + + def _retain_current_collection_refresh_if_unrepresented(self) -> None: + """Keep the last published collection work when the fleet empties.""" + runtime = self._runtime + runtime._ensure_tip_refresh_state() + if getattr(runtime, "_pool_ready_latched", False): + return + with runtime.lock: + if any( + runtime.client_can_receive_jobs(client) + for client in runtime.clients + ): + return + snapshot = self.tip_template_snapshot + observation_sequence = int( + getattr(self, "current_tip_observation_sequence", 0) + ) + if snapshot is None: + return + runtime._ensure_job_cache_state() + with runtime._job_cache_lock: + payout_state_generation = runtime._payout_state_generation + runtime._retain_collection_refresh( + snapshot, + observation_sequence, + payout_state_generation, + ) + + def _note_collection_identity_available(self, client: ClientState) -> None: + """Wake a retained collection refresh as soon as a client is eligible.""" + runtime = self._runtime + if not runtime.client_can_receive_jobs(client): + return + if runtime._retained_collection_artifacts() is None: + return + runtime._mark_tip_refresh_pending(client.connection_id) + runtime._schedule_tip_refresh_retry() + + def _consume_retained_collection_refresh( + self, + context: PrismJobContext, + ) -> None: + """Consume retention only after its collection work was delivered.""" + runtime = self._runtime + if not context.collection_only: + return + with runtime.lock: + retained = self._retained_collection_refresh + published_snapshot = self.tip_template_snapshot + artifacts = ( + published_snapshot.template_artifacts + if published_snapshot is not None + else None + ) + if ( + retained is not None + and retained.payout_state_generation + == context.payout_state_generation + and artifacts is not None + and context.template is artifacts.template + and context.template_fingerprint == artifacts.fingerprint + and context.template_generation == artifacts.generation + ): + self._retained_collection_refresh = None + + def _tip_refresh_pending(self) -> bool: + runtime = self._runtime + runtime._ensure_tip_refresh_state() + return self._tip_refresh_pending_event.is_set() + + def _mark_tip_refresh_pending(self, _observation: object) -> int: + runtime = self._runtime + runtime._ensure_tip_refresh_state() + with runtime.lock: + self._tip_refresh_pending_counter += 1 + token = self._tip_refresh_pending_counter + self._tip_refresh_pending_token = token + self._tip_refresh_pending_event.set() + return token + + def _claim_tip_refresh_pending(self) -> int | None: + """Snapshot pending work without replacing a newer producer's token.""" + runtime = self._runtime + runtime._ensure_tip_refresh_state() + with runtime.lock: + if not self._tip_refresh_pending_event.is_set(): + return None + return self._tip_refresh_pending_token + + def _mark_tip_refresh_pending_for_poll( + self, + owned_token: int | None, + _observation: object, + ) -> int | None: + """Mark poll-owned work only while no newer producer has superseded it.""" + runtime = self._runtime + runtime._ensure_tip_refresh_state() + with runtime.lock: + if self._tip_refresh_pending_token != owned_token: + return owned_token + if owned_token is not None: + self._tip_refresh_pending_event.set() + return owned_token + self._tip_refresh_pending_counter += 1 + token = self._tip_refresh_pending_counter + self._tip_refresh_pending_token = token + self._tip_refresh_pending_event.set() + return token + + def _clear_tip_refresh_pending(self, token: int) -> None: + runtime = self._runtime + runtime._ensure_tip_refresh_state() + with runtime.lock: + if self._tip_refresh_pending_token == token: + self._tip_refresh_pending_token = None + self._tip_refresh_pending_event.clear() + + def _clear_tip_refresh_pending_for_completed_refresh( + self, + snapshot: QbitTipTemplateSnapshot, + observation_sequence: int, + payout_state_generation: int, + pending_signal_token: int | None = None, + ) -> bool: + """Atomically acknowledge pending work handled by a completed poll.""" + runtime = self._runtime + runtime._ensure_job_cache_state() + with runtime._payout_state_delivery_gate.delivery_cancelable( + lambda: runtime._payout_state_generation != payout_state_generation, + generation=payout_state_generation, + priority=True, + ) as admission: + if not admission: + return False + with runtime._job_cache_lock: + payout_state_current = ( + runtime._payout_state_generation == payout_state_generation + ) + with runtime.lock: + refresh_current = runtime._tip_refresh_snapshot_current_locked( + snapshot, + observation_sequence, + ) + pending_owned = ( + self._tip_refresh_pending_token == pending_signal_token + ) + if ( + not payout_state_current + or not refresh_current + or not pending_owned + ): + return False + self._tip_refresh_pending_token = None + self._tip_refresh_pending_event.clear() + with runtime._progress_health_lock: + published_current = bool( + runtime._progress_has_published_work + and runtime._progress_published_template_fingerprint + == snapshot.template_fingerprint + and runtime._progress_published_payout_generation + == payout_state_generation + ) + if published_current: + # The completion guard above proves this coherent + # snapshot still represents the latest detected and + # published tip. A transient A -> B -> A observation + # therefore closes its publication-divergence epoch + # even when existing A work needed no new delivery. + runtime._progress_refresh_signal_pending = False + runtime._progress_publication_divergence_since_monotonic = None + return True + + def _schedule_tip_refresh_retry(self) -> None: + runtime = self._runtime + runtime._ensure_tip_refresh_state() + # Pair the Event with a monotonic generation so a producer cannot set + # it between a waiter's wake and clear and lose the newest retry. The + # event remains the blocking primitive; the generation is the durable + # coalesced work marker. + with runtime.lock: + self._tip_refresh_retry_counter += 1 + self._tip_refresh_retry.set() + + def _consume_tip_refresh_retry(self) -> bool: + """Consume all retry signals visible at one atomic wake boundary.""" + runtime = self._runtime + runtime._ensure_tip_refresh_state() + with runtime.lock: + generation = self._tip_refresh_retry_counter + if generation == self._tip_refresh_retry_consumed: + return False + self._tip_refresh_retry_consumed = generation + self._tip_refresh_retry.clear() + return True + + def _note_tip_refresh_attempt_failed( + self, + observed_tip: str | None = None, + ) -> None: + """Stamp a failed refresh pass so the poller spaces its re-attempt. + + ``observed_tip`` is the tip the failed pass worked against; a pass + that failed before learning one stamps the current observation so an + RPC outage with a static tip still gets spaced. + """ + runtime = self._runtime + runtime._ensure_tip_refresh_state() + holdoff = float( + getattr( + runtime, + "tip_refresh_failure_holdoff_seconds", + DEFAULT_PRISM_TIP_REFRESH_FAILURE_HOLDOFF_SECONDS, + ) + ) + if holdoff <= 0: + return + holdoff += random.uniform( + 0.0, + holdoff * PRISM_TIP_REFRESH_FAILURE_HOLDOFF_JITTER_FRACTION, + ) + with runtime.lock: + if observed_tip is None: + observed_tip = runtime._newest_observed_tip_locked() + self._tip_refresh_failure_tip = observed_tip + self._tip_refresh_failure_holdoff_until = time.monotonic() + holdoff + + def _tip_refresh_failure_holdoff_remaining(self) -> float: + """Seconds the poller must still wait before re-running a failed pass. + + Zero as soon as the newest known tip differs from the one the failed + pass worked against: spacing throttles re-attempts against an + unchanged, blocked or churning view, never the reaction to a + genuinely new tip (including one detected but not yet published). + """ + runtime = self._runtime + runtime._ensure_tip_refresh_state() + with runtime.lock: + deadline = self._tip_refresh_failure_holdoff_until + failed_tip = self._tip_refresh_failure_tip + current_hash = runtime._newest_observed_tip_locked() + if deadline is None: + return 0.0 + if current_hash != failed_tip: + return 0.0 + return max(0.0, deadline - time.monotonic()) + + def _observe_tip_refresh_seconds(self, name: str, elapsed_seconds: float) -> None: + runtime = self._runtime + runtime._ensure_tip_refresh_state() + with self._tip_refresh_metrics_lock: + histogram = self.tip_refresh_histograms[name] + histogram["count"] = int(histogram["count"]) + 1 + histogram["sum"] = float(histogram["sum"]) + elapsed_seconds + buckets = histogram["buckets"] + assert isinstance(buckets, dict) + for bucket in PRISM_TIP_REFRESH_SECONDS_BUCKETS: + if elapsed_seconds <= bucket: + buckets[bucket] = int(buckets.get(bucket, 0)) + 1 + + def _observe_tip_refresh_build_phase( + self, + phase: str, + elapsed_seconds: float, + ) -> None: + runtime = self._runtime + if phase not in PRISM_TIP_REFRESH_BUILD_PHASES: + raise ValueError(f"unknown tip refresh build phase: {phase}") + runtime._ensure_tip_refresh_state() + with self._tip_refresh_metrics_lock: + histogram = self.tip_refresh_build_phase_histograms[phase] + histogram["count"] = int(histogram["count"]) + 1 + histogram["sum"] = float(histogram["sum"]) + max( + 0.0, elapsed_seconds + ) + buckets = histogram["buckets"] + assert isinstance(buckets, dict) + for bucket in PRISM_TIP_REFRESH_SECONDS_BUCKETS: + if elapsed_seconds <= bucket: + buckets[bucket] = int(buckets.get(bucket, 0)) + 1 + + def _record_tip_refresh_ipc_bytes(self, direction: str, byte_count: int) -> None: + runtime = self._runtime + if direction not in {"input", "output"}: + raise ValueError(f"unknown tip refresh IPC direction: {direction}") + runtime._ensure_tip_refresh_state() + with self._tip_refresh_metrics_lock: + self.tip_refresh_ipc_bytes[direction] += max(0, int(byte_count)) + + def _record_tip_refresh_client_result(self, result: str) -> None: + runtime = self._runtime + if result not in PRISM_TIP_REFRESH_RESULTS: + raise ValueError(f"unknown tip refresh result: {result}") + runtime._ensure_tip_refresh_state() + with self._tip_refresh_metrics_lock: + self.tip_refresh_client_counts[result] += 1 + + def _record_tip_refresh_cancellation(self, stage: str) -> None: + runtime = self._runtime + if stage not in PRISM_TIP_REFRESH_CANCELLATION_STAGES: + raise ValueError(f"unknown tip refresh cancellation stage: {stage}") + runtime._ensure_tip_refresh_state() + with self._tip_refresh_metrics_lock: + self.tip_refresh_cancellation_counts[stage] += 1 + + def _record_tip_refresh_wave_outcome(self, outcome: str) -> None: + runtime = self._runtime + runtime._ensure_tip_refresh_state() + if outcome not in PRISM_TIP_REFRESH_WAVE_OUTCOMES: + raise ValueError(f"unknown tip refresh wave outcome: {outcome}") + with self._tip_refresh_metrics_lock: + self.tip_refresh_wave_outcome_counts[outcome] = ( + int(self.tip_refresh_wave_outcome_counts.get(outcome, 0)) + 1 + ) + + def _tip_refresh_future_started(self) -> None: + runtime = self._runtime + runtime._ensure_tip_refresh_state() + with self._tip_refresh_metrics_lock: + self.tip_refresh_inflight += 1 + + def _tip_refresh_future_finished(self, _future: Future[RefreshResult]) -> None: + runtime = self._runtime + runtime._ensure_tip_refresh_state() + with self._tip_refresh_metrics_lock: + self.tip_refresh_inflight = max(0, self.tip_refresh_inflight - 1) + + def tip_refresh_executor(self) -> _BoundedPriorityExecutor: + runtime = self._runtime + runtime._ensure_tip_refresh_state() + with self._tip_refresh_executor_lock: + if self._tip_refresh_executor_shutdown: + raise RuntimeError("tip refresh executor is shut down") + executor = self._tip_refresh_executor + if executor is None: + executor = _BoundedPriorityExecutor( + max_workers=runtime.tip_refresh_max_workers, + max_queue_size=runtime.delivery_queue_limit(), + thread_name_prefix="prism-tip-refresh-delivery", + ) + self._tip_refresh_executor = executor + return executor + + def shutdown_tip_refresh_executor(self) -> None: + runtime = self._runtime + runtime._ensure_tip_refresh_state() + with self._tip_refresh_executor_lock: + executor = self._tip_refresh_executor + self._tip_refresh_executor = None + self._tip_refresh_executor_shutdown = True + if executor is not None: + # Stop queued publication work before waiting on reconnect workers. + executor.shutdown(wait=False, cancel_futures=True) + runtime.shutdown_initial_job_executor() + if executor is not None: + # Running workers may already hold client/job state or be inside a + # socket send. Drain them before serve returns and the writer lease + # is released; queued workers are cancelled without starting. + executor.shutdown(wait=True) + runtime.shutdown_job_build_executor() + runtime.shutdown_payout_artifact_executor() + runtime.shutdown_reconcile_prefetch_executor() + runtime.retire_share_window_spool() + runtime.shutdown_serve_builder() + + def publication_progress_failure_expired(self, now: float) -> bool: + """Bound sustained detected/current publication divergence. + + Retry-loop heartbeats prove only that the driver is scheduled. This + dedicated deadline is independent of client-delivery health and is + cleared only by current work publication/delivery, so old delivery + backlog cannot make a later legitimate divergence expire immediately. + """ + runtime = self._runtime + budget = float( + getattr( + runtime, + "template_refresh_failure_exit_seconds", + DEFAULT_PRISM_TEMPLATE_MAX_AGE_SECONDS, + ) + ) + if budget <= 0: + return False + runtime._ensure_job_cache_state() + with runtime._progress_health_lock: + divergence_since = ( + runtime._progress_publication_divergence_since_monotonic + ) + return bool( + divergence_since is not None + and now - divergence_since >= budget + ) + + def _cancel_active_tip_refresh_for_shutdown(self) -> None: + runtime = self._runtime + runtime._ensure_tip_refresh_state() + with runtime.lock: + active = self._active_tip_refresh + if active is not None: + active[1].cancel() + self._tip_refresh_retry.clear() + + def _wait_for_blockpoll_trigger(self) -> bool: + """Wait for the normal interval or an immediate coalesced retry.""" + runtime = self._runtime + remaining = float(runtime.blockpoll_seconds) + while remaining > 0: + if runtime.stop_event.is_set(): + return False + # After a failed pass, leave retry signals unconsumed until the + # spacing window closes. Re-checked per slice: a newly observed + # tip zeroes the holdoff immediately, and signals arriving during + # the hold stay coalesced for the attempt that eventually runs. + holdoff = runtime._tip_refresh_failure_holdoff_remaining() + if holdoff <= 0 and runtime._consume_tip_refresh_retry(): + return not runtime.stop_event.is_set() + wait_seconds = min(remaining, 0.25) + if holdoff > 0: + # The retry event may already be set; pace on the stop event + # so the spacing window cannot be spun through instantly. + # Short slices keep a newly detected tip's release prompt. + # Beat per slice: a deliberately held poller is idle, not + # hung, and must not trip the watchdog when the configured + # holdoff exceeds its timeout. + runtime._record_heartbeat("qbit_blockpoll") + wait_seconds = min(wait_seconds, holdoff, 0.05) + runtime.stop_event.wait(wait_seconds) + else: + self._tip_refresh_retry.wait(wait_seconds) + remaining -= wait_seconds + # Interval-driven attempts respect the same spacing so a sub-holdoff + # blockpoll interval cannot re-run a failed pass early. + while not runtime.stop_event.is_set(): + holdoff = runtime._tip_refresh_failure_holdoff_remaining() + if holdoff <= 0: + break + runtime._record_heartbeat("qbit_blockpoll") + runtime.stop_event.wait(min(holdoff, 0.05)) + runtime._consume_tip_refresh_retry() + return not runtime.stop_event.is_set() + + def blockpoll_loop(self) -> None: + runtime = self._runtime + runtime._ensure_tip_refresh_state() + while runtime._wait_for_blockpoll_trigger(): + # A superseding observation or post-fanout tip change wakes this + # fallback loop immediately. The event coalesces repeated signals; + # ordinary same-tip polling retains its configured interval. + # Heartbeat at the top of each iteration: reaching here proves the + # loop is alive. A transient qbit RPC error still loops and beats; a + # hung RPC call never returns, so the beat goes stale and the + # watchdog restarts the process. + runtime._record_heartbeat("qbit_blockpoll") + try: + refreshed = runtime.poll_qbit_tip_template_once() + if refreshed: + print( + f"prism coordinator: refreshed {refreshed} client job(s) after qbit tip/template change", + flush=True, + ) + except self._shutdown_error: + # Admission can close after the loop condition but before a + # nested reconciliation enters the writer gate. That is an + # intentional shutdown stop, not a template-health failure. + return + except (TemplateRefreshSuperseded, PayoutStatePublicationBlocked) as exc: + # Coordination-blocked attempts remain outside the ordinary + # template failure budget. Their separate continuous-streak + # budget is recorded by poll_qbit_tip_template_once and + # enforced by the mandatory publication-progress watchdog. + # The retry is already scheduled by the raise site. + print( + f"prism coordinator: tip/template refresh superseded; retrying: {exc}", + flush=True, + ) + except Exception: + print("prism coordinator: qbit tip/template poll failed", flush=True) + traceback.print_exc() + if runtime.template_refresh_failure_expired(time.monotonic()): + print( + "prism coordinator: template refresh failure budget exhausted; " + "exiting non-zero so the restart policy recovers the process", + flush=True, + ) + os._exit(1) + + def template_refresh_failure_expired(self, now: float) -> bool: + runtime = self._runtime + budget = float( + getattr( + runtime, + "template_refresh_failure_exit_seconds", + DEFAULT_PRISM_TEMPLATE_MAX_AGE_SECONDS, + ) + ) + if budget <= 0: + return False + failure_started = getattr(self, "template_refresh_failure_started_monotonic", None) + return failure_started is not None and now - failure_started >= budget + + def _record_template_refresh_failure(self, now: float) -> None: + runtime = self._runtime + budget = float( + getattr( + runtime, + "template_refresh_failure_exit_seconds", + DEFAULT_PRISM_TEMPLATE_MAX_AGE_SECONDS, + ) + ) + if ( + budget > 0 + and getattr(self, "template_refresh_failure_started_monotonic", None) is None + ): + self.template_refresh_failure_started_monotonic = now + + def blockwait_once(self, known_tip: str) -> str: + """One waitfornewblock round: returns the tip after the wait. + + qbitd returns as soon as its tip differs from ``known_tip`` (or after + the server-side timeout, echoing the current tip), so a tip observed + between our last poll and this call is reported immediately rather + than being missed for a cycle. + """ + runtime = self._runtime + timeout_seconds = getattr( + runtime, + "blockwait_timeout_seconds", + DEFAULT_PRISM_BLOCKWAIT_TIMEOUT_SECONDS, + ) + watchdog_timeout = float(getattr(runtime, "watchdog_timeout_seconds", 120.0)) + max_rpc_timeout = max(1.0, watchdog_timeout * 0.8) + timeout_seconds = min(float(timeout_seconds), max(1.0, max_rpc_timeout - 1.0)) + result = runtime.rpc.call( + "waitfornewblock", + [max(1, int(timeout_seconds * 1000)), known_tip], + timeout=timeout_seconds + 10.0, + ) + if isinstance(result, dict): + new_tip = str(result.get("hash", "") or "") + if new_tip: + return new_tip + return known_tip + + def blockwait_loop(self) -> None: + """Push-style tip detection alongside the interval poller. + + Stale rejects are dominated by the window between a block connecting + and miners receiving fresh work; the poller alone leaves up to a full + PRISM_BLOCKPOLL_SECONDS of that window. This loop parks inside + waitfornewblock and triggers the same refresh path within milliseconds + of a new tip. The poller stays on as the fallback and still owns + same-tip template refreshes, which waitfornewblock does not signal. + Disabled cleanly when qbitd does not support the RPC. + """ + runtime = self._runtime + known_tip: str | None = None + while not runtime.stop_event.is_set(): + runtime._record_heartbeat("qbit_blockwait") + try: + if known_tip is None: + known_tip = str(runtime.rpc.call("getbestblockhash")) + runtime.observe_tip_for_refresh(known_tip) + new_tip = runtime.blockwait_once(known_tip) + if new_tip == known_tip: + if runtime.stop_event.wait(0.25): + return + continue + # Advance the wait cursor before observation/logging. If either + # operation raises, the next wait must not rediscover the same + # already-seen transition and create a notification storm. + known_tip = new_tip + # Detection can supersede/cancel obsolete heavy work, but only + # the blockpoll single-flight owner may fetch, build, and + # publish replacement work. + try: + runtime.observe_tip_for_refresh(new_tip) + finally: + # Preserve the wake even if observation-side cancellation + # or bookkeeping raises after known_tip advanced. + runtime._schedule_tip_refresh_retry() + print( + f"prism coordinator: blockwait saw new tip {new_tip}; " + "single-flight refresh scheduled", + flush=True, + ) + except Exception as exc: + if known_tip is not None and runtime._blockwait_unsupported(exc): + print( + "prism coordinator: waitfornewblock unavailable on this qbitd; " + "tip detection falls back to blockpoll only", + flush=True, + ) + runtime._remove_watchdog_heartbeat("qbit_blockwait") + return + print("prism coordinator: blockwait pass failed", flush=True) + traceback.print_exc() + if runtime.stop_event.wait(min(5.0, runtime.blockpoll_seconds)): + return + + def tip_refresh_metrics_lines(self) -> list[str]: + runtime = self._runtime + runtime._ensure_tip_refresh_state() + with self._tip_refresh_executor_lock: + executor_workers = ( + runtime.tip_refresh_max_workers + if self._tip_refresh_executor is not None + else 0 + ) + with self._tip_refresh_metrics_lock: + histograms = { + name: { + "buckets": dict(histogram["buckets"]), + "sum": float(histogram["sum"]), + "count": int(histogram["count"]), + } + for name, histogram in self.tip_refresh_histograms.items() + } + phase_histograms = { + phase: { + "buckets": dict(histogram["buckets"]), + "sum": float(histogram["sum"]), + "count": int(histogram["count"]), + } + for phase, histogram in self.tip_refresh_build_phase_histograms.items() + } + client_counts = dict(self.tip_refresh_client_counts) + cancellation_counts = dict(self.tip_refresh_cancellation_counts) + wave_outcome_counts = dict(self.tip_refresh_wave_outcome_counts) + coverage_histograms = { + target: { + "buckets": dict(histogram["buckets"]), + "sum": float(histogram["sum"]), + "count": int(histogram["count"]), + } + for target, histogram in ( + self.tip_refresh_coverage_histograms.items() + ) + } + inflight = self.tip_refresh_inflight + build_inflight = self.tip_refresh_build_inflight + build_queue_depth = self.tip_refresh_build_queue_depth + singleflight_hits = self.tip_refresh_singleflight_hits + superseded_results = self.tip_refresh_superseded_results + worker_failures = self.tip_refresh_worker_failures + worker_restarts = self.tip_refresh_worker_restarts + ipc_bytes = dict(self.tip_refresh_ipc_bytes) + + metric_names = { + "refresh": "qbit_prism_tip_refresh_seconds", + "bundle_build": "qbit_prism_tip_refresh_bundle_build_seconds", + "first_delivery": "qbit_prism_tip_refresh_first_delivery_seconds", + "last_delivery": "qbit_prism_tip_refresh_last_delivery_seconds", + "fanout_wave": "qbit_prism_tip_refresh_fanout_wave_seconds", + } + descriptions = { + "refresh": "Full qbit tip/template refresh pass wall time.", + "bundle_build": "Shared ready-pool refresh bundle preparation wall time.", + "first_delivery": "Tip observation to first successful client delivery.", + "last_delivery": "Tip observation to last successful client delivery.", + "fanout_wave": "First task submission to last successful delivery of one completed prepared fanout wave.", + } + lines: list[str] = [] + for name, metric_name in metric_names.items(): + histogram = histograms[name] + buckets = histogram["buckets"] + assert isinstance(buckets, dict) + lines.extend( + [ + f"# HELP {metric_name} {descriptions[name]}", + f"# TYPE {metric_name} histogram", + *[ + f'{metric_name}_bucket{{le="{bucket:g}"}} {int(buckets.get(bucket, 0))}' + for bucket in PRISM_TIP_REFRESH_SECONDS_BUCKETS + ], + f'{metric_name}_bucket{{le="+Inf"}} {histogram["count"]}', + f'{metric_name}_sum {float(histogram["sum"]):.6f}', + f'{metric_name}_count {histogram["count"]}', + ] + ) + phase_metric_name = "qbit_prism_tip_refresh_bundle_phase_seconds" + lines.extend( + [ + "# HELP qbit_prism_tip_refresh_bundle_phase_seconds Shared bundle-build phase wall time.", + "# TYPE qbit_prism_tip_refresh_bundle_phase_seconds histogram", + ] + ) + for phase in PRISM_TIP_REFRESH_BUILD_PHASES: + histogram = phase_histograms[phase] + buckets = histogram["buckets"] + assert isinstance(buckets, dict) + lines.extend( + [ + *[ + f'{phase_metric_name}_bucket{{phase="{phase}",le="{bucket:g}"}} {int(buckets.get(bucket, 0))}' + for bucket in PRISM_TIP_REFRESH_SECONDS_BUCKETS + ], + f'{phase_metric_name}_bucket{{phase="{phase}",le="+Inf"}} {histogram["count"]}', + f'{phase_metric_name}_sum{{phase="{phase}"}} {float(histogram["sum"]):.6f}', + f'{phase_metric_name}_count{{phase="{phase}"}} {histogram["count"]}', + ] + ) + lines.extend( + [ + "# HELP qbit_prism_tip_refresh_wave_outcomes_total Singleflight refresh waves by terminal or recovered supersession outcome.", + "# TYPE qbit_prism_tip_refresh_wave_outcomes_total counter", + *[ + f'qbit_prism_tip_refresh_wave_outcomes_total{{outcome="{outcome}"}} {int(wave_outcome_counts.get(outcome, 0))}' + for outcome in PRISM_TIP_REFRESH_WAVE_OUTCOMES + ], + "# HELP qbit_prism_tip_refresh_clients_total Client outcomes from tip/template refresh passes.", + "# TYPE qbit_prism_tip_refresh_clients_total counter", + *[ + f'qbit_prism_tip_refresh_clients_total{{result="{result}"}} {int(client_counts.get(result, 0))}' + for result in PRISM_TIP_REFRESH_RESULTS + ], + "# HELP qbit_prism_tip_refresh_cancellations_total Obsolete prepared refresh tasks canceled before delivery admission.", + "# TYPE qbit_prism_tip_refresh_cancellations_total counter", + *[ + f'qbit_prism_tip_refresh_cancellations_total{{stage="{stage}"}} {int(cancellation_counts.get(stage, 0))}' + for stage in PRISM_TIP_REFRESH_CANCELLATION_STAGES + ], + "# HELP qbit_prism_tip_refresh_inflight Prepared refresh client tasks currently queued or running.", + "# TYPE qbit_prism_tip_refresh_inflight gauge", + f"qbit_prism_tip_refresh_inflight {inflight}", + "# HELP qbit_prism_tip_refresh_executor_workers Configured persistent refresh executor workers, or zero before creation.", + "# TYPE qbit_prism_tip_refresh_executor_workers gauge", + f"qbit_prism_tip_refresh_executor_workers {executor_workers}", + "# HELP qbit_prism_tip_refresh_bundle_inflight Shared bundle builds currently running.", + "# TYPE qbit_prism_tip_refresh_bundle_inflight gauge", + f"qbit_prism_tip_refresh_bundle_inflight {build_inflight}", + "# HELP qbit_prism_tip_refresh_bundle_queue_depth Shared bundle callers waiting on bounded build admission or an identical single-flight.", + "# TYPE qbit_prism_tip_refresh_bundle_queue_depth gauge", + f"qbit_prism_tip_refresh_bundle_queue_depth {build_queue_depth}", + "# HELP qbit_prism_tip_refresh_bundle_singleflight_hits_total Shared bundle callers coalesced behind an identical build.", + "# TYPE qbit_prism_tip_refresh_bundle_singleflight_hits_total counter", + f"qbit_prism_tip_refresh_bundle_singleflight_hits_total {singleflight_hits}", + "# HELP qbit_prism_tip_refresh_bundle_superseded_results_total Completed or canceled shared bundles discarded after supersession.", + "# TYPE qbit_prism_tip_refresh_bundle_superseded_results_total counter", + f"qbit_prism_tip_refresh_bundle_superseded_results_total {superseded_results}", + "# HELP qbit_prism_tip_refresh_builder_worker_failures_total Audit-builder subprocess failures.", + "# TYPE qbit_prism_tip_refresh_builder_worker_failures_total counter", + f"qbit_prism_tip_refresh_builder_worker_failures_total {worker_failures}", + "# HELP qbit_prism_tip_refresh_builder_worker_restarts_total Long-lived builder worker restarts (persistent --serve daemon respawns after a crash or supersession).", + "# TYPE qbit_prism_tip_refresh_builder_worker_restarts_total counter", + f"qbit_prism_tip_refresh_builder_worker_restarts_total {worker_restarts}", + "# HELP qbit_prism_tip_refresh_builder_ipc_bytes_total Bytes copied across audit-builder subprocess IPC.", + "# TYPE qbit_prism_tip_refresh_builder_ipc_bytes_total counter", + *[ + f'qbit_prism_tip_refresh_builder_ipc_bytes_total{{direction="{direction}"}} {int(ipc_bytes.get(direction, 0))}' + for direction in ("input", "output") + ], + ] + ) + coverage_metric_name = "qbit_prism_tip_refresh_hashrate_coverage_seconds" + lines.extend( + [ + "# HELP qbit_prism_tip_refresh_hashrate_coverage_seconds Time from epoch detection until a fixed share of the snapshotted hashrate proxy received current work.", + "# TYPE qbit_prism_tip_refresh_hashrate_coverage_seconds histogram", + ] + ) + for target, _ratio in PRISM_TIP_REFRESH_COVERAGE_TARGETS: + histogram = coverage_histograms[target] + buckets = histogram["buckets"] + assert isinstance(buckets, dict) + lines.extend( + [ + *[ + f'{coverage_metric_name}_bucket{{coverage="{target}",le="{bucket:g}"}} {int(buckets.get(bucket, 0))}' + for bucket in PRISM_TIP_REFRESH_SECONDS_BUCKETS + ], + f'{coverage_metric_name}_bucket{{coverage="{target}",le="+Inf"}} {histogram["count"]}', + f'{coverage_metric_name}_sum{{coverage="{target}"}} {float(histogram["sum"]):.6f}', + f'{coverage_metric_name}_count{{coverage="{target}"}} {histogram["count"]}', + ] + ) + runtime._ensure_job_cache_state() + with runtime._serve_builder_metrics_lock: + serve_counts = dict(runtime.serve_builder_counts) + serve_window_counts = dict(runtime.serve_builder_window_cache_counts) + with runtime._payout_artifact_executor_lock: + payout_artifact_events = dict(runtime.payout_artifact_event_counts) + lines.extend( + [ + "# HELP qbit_prism_serve_builder_events_total Persistent audit-builder daemon lifecycle and request outcomes.", + "# TYPE qbit_prism_serve_builder_events_total counter", + *[ + f'qbit_prism_serve_builder_events_total{{event="{event}"}} {int(serve_counts.get(event, 0))}' + for event in ( + "requests", + "fallbacks", + "spawns", + "window_uploads", + "window_prepares", + "window_prepare_contended", + "window_prepare_out_of_range", + ) + ], + "# HELP qbit_prism_payout_artifact_events_total Payout-ledger-artifact lifecycle outcomes (build, install, reuse pacing).", + "# TYPE qbit_prism_payout_artifact_events_total counter", + *[ + f'qbit_prism_payout_artifact_events_total{{event="{event}"}} {int(payout_artifact_events.get(event, 0))}' + for event in ( + "built", + "build_aborted", + "debounced", + "incremental", + "full_rescan", + "self_check_match", + "self_check_mismatch", + "self_check_failed", + "balance_check_mismatch", + "found_block_cached", + "installed", + "refreshed", + "already_current", + "discarded", + "born_expired", + "rearm_scheduled", + "served_reuse", + "probe_rejected_ceiling", + "probe_past_floor", + "window_mirror_divergence", + ) + ], + "# HELP qbit_prism_serve_builder_window_cache_total Daemon parsed share-window cache outcomes.", + "# TYPE qbit_prism_serve_builder_window_cache_total counter", + *[ + f'qbit_prism_serve_builder_window_cache_total{{result="{result}"}} {int(serve_window_counts.get(counter_key, 0))}' + for result, counter_key in ( + ("hit", "hits"), + ("miss", "misses"), + ) + ], + ] + ) + return lines + + def _tip_refresh_artifacts( + self, + snapshot: QbitTipTemplateSnapshot, + ) -> CachedTemplateArtifacts: + artifacts = snapshot.template_artifacts + if ( + artifacts is None + or artifacts.fingerprint != snapshot.template_fingerprint + or artifacts.previousblockhash != snapshot.previousblockhash + or artifacts.generation != snapshot.template_generation + or snapshot.bestblockhash != snapshot.previousblockhash + or qbit_template_fingerprint(artifacts.template) != artifacts.fingerprint + or str(artifacts.template.get("previousblockhash", "")) + != artifacts.previousblockhash + ): + raise TemplateRefreshBlocked( + "tip/template snapshot does not own matching exact artifacts" + ) + return artifacts + + def prepare_tip_refresh_bundle( + self, + snapshot: QbitTipTemplateSnapshot, + *, + priority_requested_monotonic: float | None = None, + ) -> CachedJobBundle: + """Build ready-pool work from immutable shared inputs only. + + Client selection belongs exclusively to fanout. In particular, a + connection disappearing before or during this build cannot affect the + signed payout bundle or its cache lifetime. + """ + runtime = self._runtime + runtime._ensure_job_cache_state() + runtime._ensure_tip_refresh_state() + artifacts = runtime._tip_refresh_artifacts(snapshot) + build_started = time.monotonic() + progress_build_token = runtime._progress_bundle_build_started() + priority_token, priority_requested_monotonic = ( + runtime._begin_job_build_priority_preparation( + priority_requested_monotonic + ) + ) + try: + max_payout_retries = max( + 0, + int( + getattr( + runtime, + "payout_reconcile_supersession_retries", + DEFAULT_PRISM_PAYOUT_RECONCILE_SUPERSESSION_RETRIES, + ) + ), + ) + for attempt in range(max_payout_retries + 1): + with runtime._job_cache_lock: + payout_generation_before_build = ( + runtime._payout_state_generation + ) + try: + bundle = runtime.shared_job_bundle( + artifacts, + mode="ready", + retry_superseded=False, + publication_critical=True, + request_source="tip_refresh", + priority_requested_monotonic=( + priority_requested_monotonic + ), + ) + break + except JobBuildSuperseded: + if getattr(runtime, "tip_refresh_epoch_fanout", False): + raise + with runtime._job_cache_lock: + payout_generation_after_build = ( + runtime._payout_state_generation + ) + payout_publication_blocked = ( + runtime._payout_state_publication_blocked + ) + with runtime.lock: + artifacts_buildable = runtime._artifacts_buildable_locked( + artifacts + ) + if ( + attempt >= max_payout_retries + or payout_publication_blocked + or not artifacts_buildable + or payout_generation_after_build + == payout_generation_before_build + ): + raise + # Coalesce a completed payout generation into this same + # tip owner. Every abandoned attempt remains fenced by its + # immutable generation; only the latest coherent retry can + # reach final publication. + continue + else: # pragma: no cover - range always runs at least once + raise TemplateRefreshBlocked( + "payout generation did not stabilize during preparation" + ) + except TemplateRefreshBlocked: + raise + except Exception as exc: + with runtime.lock: + runtime.job_build_failure_count += 1 + raise TemplateRefreshBlocked("prepared refresh bundle build failed") from exc + finally: + runtime._finish_job_build_priority_preparation(priority_token) + runtime._progress_bundle_build_finished(progress_build_token) + runtime._observe_tip_refresh_seconds( + "bundle_build", + time.monotonic() - build_started, + ) + # Revalidate only the snapshot-owned object. A concurrent cache fill is + # unrelated to this refresh and cannot replace its exact artifacts. + artifacts = runtime._tip_refresh_artifacts(snapshot) + if ( + bundle.template is not artifacts.template + or bundle.template_fingerprint != artifacts.fingerprint + or bundle.template_generation != artifacts.generation + or str(bundle.template.get("previousblockhash", "")) + != artifacts.previousblockhash + ): + raise TemplateRefreshBlocked( + "prepared refresh bundle does not match exact template artifacts" + ) + if bundle.collection_only: + raise TemplateRefreshBlocked( + "ready-pool prepared refresh unexpectedly produced a collection bundle" + ) + return bundle + + def prewarm_current_tip_ready_bundle(self) -> CachedJobBundle | None: + """Publish one exact current-tip ready bundle before Stratum accepts.""" + runtime = self._runtime + runtime._ensure_job_cache_state() + with runtime._job_cache_lock: + runtime.job_preparation_pending = True + try: + observation_sequence = runtime._reserve_tip_observation_sequence() + snapshot = runtime.fetch_qbit_tip_template_snapshot() + runtime._record_progress_tip_poll(snapshot) + try: + reconciled = runtime.ensure_reorg_reconciled_for_tip( + snapshot.bestblockhash + ) + except Exception as exc: + raise TemplateRefreshBlocked( + "startup reorg reconciliation failed before job preparation" + ) from exc + if not reconciled: + raise TemplateRefreshBlocked( + "startup chain view remained untrusted during job preparation" + ) + + ready = runtime.pool_readiness_latched() + bundle: CachedJobBundle | None = None + if ready: + progress_build_token = runtime._progress_bundle_build_started() + try: + bundle = runtime.shared_job_bundle( + runtime._tip_refresh_artifacts(snapshot), + None, + publication_critical=True, + request_source="tip_refresh", + ) + finally: + runtime._progress_bundle_build_finished(progress_build_token) + if bundle.collection_only: + raise TemplateRefreshBlocked( + "startup ready preparation produced collection work" + ) + if bundle.payout_state_generation != int( + getattr(runtime, "_payout_state_generation", 0) + ): + raise TemplateRefreshSuperseded( + "payout state changed during startup job preparation" + ) + + if str(runtime.rpc.call("getbestblockhash")) != snapshot.bestblockhash: + raise TemplateRefreshSuperseded( + "qbit tip changed during startup job preparation" + ) + if not runtime.observe_tip_first_seen( + snapshot.bestblockhash, + observation_sequence=observation_sequence, + publish_refresh_observation=True, + published_snapshot=snapshot, + ): + raise TemplateRefreshSuperseded( + "startup job preparation was superseded before publication" + ) + with runtime._job_cache_lock: + runtime._prepared_ready_snapshot = snapshot if bundle is not None else None + runtime._prepared_ready_bundle = bundle + runtime._record_progress_publication( + snapshot, + ( + bundle.payout_state_generation + if bundle is not None + else int(getattr(runtime, "_payout_state_generation", 0)) + ), + ) + self.last_successful_template_refresh_monotonic = time.monotonic() + runtime._record_progress_tip_poll(snapshot) + return bundle + finally: + with runtime._job_cache_lock: + runtime.job_preparation_pending = False + + def prewarm_startup_jobs(self) -> CachedJobBundle | None: + """Best-effort startup prewarm; transient blocking defers to blockpoll.""" + runtime = self._runtime + try: + return runtime.prewarm_current_tip_ready_bundle() + except TemplateRefreshBlocked as exc: + # Startup prewarming is an optimization. A transient reconciliation, + # payout-generation, or tip race must not prevent Stratum listeners + # from opening; blockpoll and the bounded initial-job queue retry it. + runtime._schedule_tip_refresh_retry() + print( + "prism coordinator: startup job preparation deferred " + f"reason={exc}", + flush=True, + ) + return None + + def _tip_refresh_token_current_locked( + self, + token: TipRefreshValidationToken, + bundle: CachedJobBundle, + snapshot: QbitTipTemplateSnapshot, + ) -> bool: + runtime = self._runtime + return bool( + runtime._tip_refresh_token_prepublication_current_locked( + token, + bundle, + snapshot, + ) + and runtime._tip_refresh_snapshot_current_locked( + snapshot, + token.observation_sequence, + ) + ) + + def _tip_refresh_token_prepublication_current_locked( + self, + token: TipRefreshValidationToken, + bundle: CachedJobBundle, + snapshot: QbitTipTemplateSnapshot, + ) -> bool: + runtime = self._runtime + published = getattr(runtime, "_published_payout_state", None) + return bool( + token.snapshot is snapshot + and token.tip_hash == snapshot.bestblockhash + and token.template_fingerprint == snapshot.template_fingerprint + and token.template_generation == snapshot.template_generation + and bundle.template_fingerprint == token.template_fingerprint + and bundle.template_generation == token.template_generation + and bundle.payout_state_generation == token.payout_state_generation + and bundle.build_key is token.build_key + and token.build_key.best_tip_hash == snapshot.bestblockhash + and token.build_key.previous_block_hash == snapshot.previousblockhash + and token.build_key.template_fingerprint + == snapshot.template_fingerprint + and token.build_key.template_generation == snapshot.template_generation + and token.build_key.payout_state_generation + == token.payout_state_generation + and token.build_key.payout_append_invalidation_epoch + == int( + getattr( + runtime, + "_payout_ledger_append_invalidation_epoch", + 0, + ) + ) + and token.payout_state_generation + == int(getattr(runtime, "_payout_state_generation", 0)) + and published is not None + and published.artifact is not None + and token.build_key.payout_artifact_sha256 + == published.artifact.prior_balances_sha256 + and snapshot.template_artifacts is not None + and bundle.template is snapshot.template_artifacts.template + and not runtime._detected_tip_supersedes_locked( + snapshot.bestblockhash, + token.observation_sequence, + ) + and ( + not getattr(runtime, "tip_refresh_epoch_fanout", False) + or ( + token.epoch_sequence + == int(getattr(self, "_tip_refresh_epoch_sequence", 0)) + and token.tip_hash + == getattr(self, "_tip_refresh_epoch_tip_hash", None) + and token.payout_state_generation + == int( + getattr( + self, + "_tip_refresh_epoch_payout_generation", + 0, + ) + ) + ) + ) + ) + + def _tip_refresh_snapshot_current_locked( + self, + snapshot: QbitTipTemplateSnapshot, + observation_sequence: int, + ) -> bool: + runtime = self._runtime + current_tip = getattr(self, "current_tip_first_seen", None) + return bool( + self.tip_template_snapshot is snapshot + and current_tip is not None + and current_tip[0] == snapshot.bestblockhash + and int(getattr(self, "current_tip_observation_sequence", 0)) + == observation_sequence + and not runtime._detected_tip_supersedes_locked( + snapshot.bestblockhash, + observation_sequence, + ) + ) + + def _validate_prepared_tip_refresh( + self, + bundle: CachedJobBundle, + snapshot: QbitTipTemplateSnapshot, + observation_sequence: int, + ) -> TipRefreshValidationToken: + """Validate prepared work before publishing submit authority.""" + runtime = self._runtime + artifacts = runtime._tip_refresh_artifacts(snapshot) + if ( + bundle.template is not artifacts.template + or bundle.template_fingerprint != artifacts.fingerprint + or bundle.template_generation != artifacts.generation + or bundle.build_key is None + or bundle.build_key.best_tip_hash != snapshot.bestblockhash + or bundle.build_key.previous_block_hash != snapshot.previousblockhash + or bundle.build_key.template_fingerprint != artifacts.fingerprint + or bundle.build_key.template_generation != artifacts.generation + or bundle.build_key.mode != "ready" + ): + raise TemplateRefreshBlocked( + "prepared refresh bundle changed before final validation" + ) + try: + current_tip = str(runtime.rpc.call("getbestblockhash")) + except Exception as exc: + runtime._schedule_tip_refresh_retry() + raise TemplateRefreshBlocked( + "qbit tip validation failed before prepared fanout" + ) from exc + if current_tip != snapshot.bestblockhash: + runtime._schedule_tip_refresh_retry() + raise TemplateRefreshSuperseded( + "qbit tip changed before prepared fanout " + f"expected={snapshot.bestblockhash} current={current_tip}" + ) + try: + chain_view_untrusted = bool( + getattr(runtime, "reorg_reconciler_enabled", True) + and runtime.qbit_chain_view_untrusted() + ) + except Exception as exc: + runtime._schedule_tip_refresh_retry() + raise _TipRefreshTrustBlocked( + "qbit chain trust check failed before prepared fanout" + ) from exc + if chain_view_untrusted: + runtime._schedule_tip_refresh_retry() + raise _TipRefreshTrustBlocked( + "qbit chain view became untrusted before prepared fanout" + ) + with runtime.lock: + epoch_sequence = ( + int(getattr(self, "_tip_refresh_epoch_sequence", 0)) + if getattr(runtime, "tip_refresh_epoch_fanout", False) + else 0 + ) + token = TipRefreshValidationToken( + tip_hash=snapshot.bestblockhash, + template_fingerprint=artifacts.fingerprint, + template_generation=artifacts.generation, + payout_state_generation=bundle.payout_state_generation, + observation_sequence=observation_sequence, + epoch_sequence=epoch_sequence, + build_key=bundle.build_key, + snapshot=snapshot, + ) + if not runtime._tip_refresh_token_prepublication_current_locked( + token, + bundle, + snapshot, + ): + runtime._schedule_tip_refresh_retry() + raise TemplateRefreshSuperseded( + "prepared refresh was superseded before tip publication" + ) + return token + + def _activate_tip_refresh( + self, + token: TipRefreshValidationToken, + bundle: CachedJobBundle, + snapshot: QbitTipTemplateSnapshot, + cancel_event: FanoutCancellation, + ) -> None: + runtime = self._runtime + with runtime.lock: + if not runtime._tip_refresh_token_current_locked(token, bundle, snapshot): + runtime._schedule_tip_refresh_retry() + raise TemplateRefreshSuperseded( + "prepared refresh was superseded before cancellation registration" + ) + active = self._active_tip_refresh + if active is not None: + active[1].cancel() + self._active_tip_refresh = (token, cancel_event) + + def _publish_prepared_tip_refresh( + self, + token: TipRefreshValidationToken, + bundle: CachedJobBundle, + snapshot: QbitTipTemplateSnapshot, + *, + parent_hash: str | None, + ) -> FanoutCancellation: + """Atomically publish prepared work and register its cancellation token.""" + runtime = self._runtime + now = time.monotonic() + cancel_event = FanoutCancellation() + # Admit a synchronization-only reader of this payout generation. Do + # not mark it delivered: the gate's priority reservation belongs to + # the first current-tip socket delivery, not this publication fence. + with runtime._payout_state_delivery_gate.delivery_cancelable( + lambda: False, + generation=token.payout_state_generation, + priority=True, + ) as payout_admitted: + if not payout_admitted: + runtime._schedule_tip_refresh_retry() + raise TemplateRefreshSuperseded( + "prepared refresh was superseded before atomic publication" + ) + # Match payout publication's cache -> coordinator lock order. A + # payout fence can close admission after this reader entered, so + # recheck its blocked marker inside the same atomic section. + with runtime._job_cache_lock: + with runtime.lock: + current_sequence = int( + getattr(self, "current_tip_observation_sequence", 0) + ) + if ( + runtime._payout_state_publication_blocked + or current_sequence > token.observation_sequence + or not runtime._tip_refresh_token_prepublication_current_locked( + token, + bundle, + snapshot, + ) + ): + runtime._schedule_tip_refresh_retry() + raise TemplateRefreshSuperseded( + "prepared refresh was superseded before atomic publication" + ) + + first_seen = getattr(self, "current_tip_first_seen", None) + tip_changed = ( + first_seen is not None and first_seen[0] != token.tip_hash + ) + flip_stamp = ( + now + if tip_changed + else first_seen[1] + if first_seen is not None + else None + ) + self.current_tip_first_seen = ( + token.tip_hash, + flip_stamp, + ) + self.current_tip_observation_sequence = token.observation_sequence + self.current_tip_observed_monotonic = now + if parent_hash is not None: + self.current_tip_parent = (token.tip_hash, parent_hash) + else: + # The parent lookup is best-effort cleanup metadata. A + # transient RPC failure during a same-tip republication + # must not wipe the still-valid cached parent; only a + # parent belonging to a different tip is stale here. + prior_parent = getattr(self, "current_tip_parent", None) + if prior_parent is None or prior_parent[0] != token.tip_hash: + self.current_tip_parent = None + self.tip_template_snapshot = snapshot + runtime._record_startup_phase_once("first_tip_template_published") + self.tip_refresh_divergence_started_monotonic = None + if tip_changed: + # Retained collection work, the retained ready bundle, + # and graveyard classification all belong to the + # previously published tip. + self._retained_collection_refresh = None + runtime._prepared_ready_bundle = None + runtime._prepared_ready_snapshot = None + runtime.prune_evicted_job_graveyard(now=now, force=True) + + if not runtime._tip_refresh_token_current_locked( + token, + bundle, + snapshot, + ): + # The payout admission and both coordinator locks + # stabilize every field used by this predicate. + raise TemplateRefreshBlocked( + "prepared refresh publication did not produce a current token" + ) + runtime._publish_tip_refresh_epoch_identity_locked(snapshot) + active = self._active_tip_refresh + if active is not None: + active[1].cancel() + self._active_tip_refresh = (token, cancel_event) + return cancel_event + + def _clear_active_tip_refresh( + self, + token: TipRefreshValidationToken, + cancel_event: FanoutCancellation, + ) -> None: + runtime = self._runtime + with runtime.lock: + active = self._active_tip_refresh + if active is not None and active[0] is token and active[1] is cancel_event: + self._active_tip_refresh = None + + def _prepared_tip_refresh_obsolete( + self, + validation_token: TipRefreshValidationToken, + bundle: CachedJobBundle, + snapshot: QbitTipTemplateSnapshot, + cancel_event: FanoutCancellation | None, + ) -> bool: + runtime = self._runtime + if runtime.stop_event.is_set() or ( + cancel_event is not None and cancel_event.is_set() + ): + return True + with runtime.lock: + current = runtime._tip_refresh_token_current_locked( + validation_token, + bundle, + snapshot, + ) + if not current and cancel_event is not None: + cancel_event.cancel() + return not current + + def _fanout_prepared_tip_refresh( + self, + clients: list[ClientState], + bundle: CachedJobBundle, + snapshot: QbitTipTemplateSnapshot, + *, + observation_sequence: int | None = None, + validation_token: TipRefreshValidationToken | None = None, + preactivated_cancel_event: FanoutCancellation | None = None, + executor: ThreadPoolExecutor | None = None, + expected_active_jobs: dict[ClientState, PrismJobContext | None] | None = None, + heartbeat_name: str, + ) -> tuple[int, float | None, float | None, int]: + runtime = self._runtime + executor = executor or runtime.tip_refresh_executor() + cancel_event = preactivated_cancel_event or FanoutCancellation() + if observation_sequence is None: + with runtime.lock: + observation_sequence = int( + getattr(self, "current_tip_observation_sequence", 0) + ) + if validation_token is None: + validation_token = runtime._validate_prepared_tip_refresh( + bundle, + snapshot, + observation_sequence, + ) + if preactivated_cancel_event is None: + runtime._activate_tip_refresh( + validation_token, + bundle, + snapshot, + cancel_event, + ) + futures: dict[Future[RefreshResult], ClientState] = {} + submitted_at: dict[Future[RefreshResult], float] = {} + queued_cancellations: set[Future[RefreshResult]] = set() + with runtime.lock: + if expected_active_jobs is None: + expected_active_jobs = { + client: client.active_job + for client in clients + } + # Vardiff drives every client toward the same share interval, so + # the difficulty a client sustains is proportional to its hashrate. + # Snapshot the proxies in the same locked pass as the job snapshot. + hashrate_proxies = { + client: ( + client.vardiff_difficulty_estimate or client.share_difficulty + ) + for client in clients + } + # Bounded submission admits at most max_inflight tasks at a time, so + # queue priority alone cannot lift one client over a fleet wave that + # has not been submitted yet. Order admission itself: stale-job + # clients by descending hashrate, job-less clients last. A stale fast + # client burns its full rate on old-tip work every second of refresh + # lag, while a job-less client burns nothing while it waits regardless + # of its configured difficulty (it also keeps its initial-delivery + # queue priority once admitted). + clients = sorted( + clients, + key=lambda ordered: ( + expected_active_jobs.get(ordered) is not None, + hashrate_proxies.get(ordered, Decimal(0)), + ), + reverse=True, + ) + clients_iter = iter(clients) + max_inflight = max(1, int(runtime.tip_refresh_max_workers)) + + def record_queued_cancellation(future: Future[RefreshResult]) -> None: + if future in queued_cancellations: + return + queued_cancellations.add(future) + elapsed = max(0.0, time.monotonic() - submitted_at[future]) + runtime.observe_job_build_elapsed(elapsed, {"executor_queue": elapsed}) + runtime._record_tip_refresh_cancellation("executor_queue") + + def cancel_pending_futures(pending: set[Future[RefreshResult]]) -> None: + cancel_event.cancel() + for future in tuple(pending): + if future.cancel(): + record_queued_cancellation(future) + runtime._record_tip_refresh_client_result("skipped") + pending.discard(future) + + def submit_available(pending: set[Future[RefreshResult]]) -> None: + while ( + len(pending) < max_inflight + and not runtime.stop_event.is_set() + and not cancel_event.is_set() + ): + with runtime.lock: + token_current = runtime._tip_refresh_token_current_locked( + validation_token, + bundle, + snapshot, + ) + if not token_current: + cancel_event.cancel() + return + try: + client = next(clients_iter) + except StopIteration: + return + submitted = time.monotonic() + active_job = expected_active_jobs.get(client) + if active_job is None: + priority = self._delivery_priority_initial + elif runtime.client_tip_changed_for_snapshot(client, snapshot): + priority = self._delivery_priority_new_tip + else: + priority = self._delivery_priority_same_tip + future = runtime._submit_delivery_task( + executor, + runtime.send_prepared_job, + client, + bundle, + snapshot, + validation_token, + client.connection_id, + expected_active_jobs.get(client), + cancel_event, + submitted, + priority=priority, + ) + runtime._tip_refresh_future_started() + future.add_done_callback(runtime._tip_refresh_future_finished) + futures[future] = client + submitted_at[future] = submitted + pending.add(future) + + pending: set[Future[RefreshResult]] = set() + try: + sent = 0 + failed = 0 + first_delivery: float | None = None + last_delivery: float | None = None + invalidation: TemplateRefreshBlocked | None = None + last_live_trust_check = time.monotonic() + try: + submit_available(pending) + except RuntimeError: + cancel_pending_futures(pending) + cancel_event.set() + if pending: + wait(pending) + if not runtime.stop_event.is_set(): + runtime._schedule_tip_refresh_retry() + raise + while pending: + runtime._record_heartbeat(heartbeat_name) + if runtime.stop_event.is_set() or cancel_event.is_set(): + cancel_pending_futures(pending) + done, pending = wait( + pending, + timeout=PRISM_TIP_REFRESH_ADMISSION_POLL_SECONDS, + return_when=FIRST_COMPLETED, + ) + for future in done: + client = futures[future] + if future.cancelled(): + if future not in queued_cancellations: + record_queued_cancellation(future) + runtime._record_tip_refresh_client_result("skipped") + continue + try: + result = future.result() + except OSError: + runtime._record_tip_refresh_client_result("disconnected") + runtime.disconnect_client(client) + continue + except TemplateRefreshBlocked as exc: + runtime._record_tip_refresh_client_result("skipped") + invalidation = exc + cancel_pending_futures(pending) + continue + except Exception: + failed += 1 + runtime._record_tip_refresh_client_result("failed") + with runtime.lock: + runtime.job_build_failure_count += 1 + print( + "prism coordinator: prepared job fanout failed " + f"connection={client.connection_id} username={client.username}", + flush=True, + ) + traceback.print_exc() + continue + runtime._record_tip_refresh_client_result(result.result) + if result.result == "sent": + sent += 1 + delivered = result.delivered_monotonic + if delivered is not None: + first_delivery = ( + delivered + if first_delivery is None + else min(first_delivery, delivered) + ) + last_delivery = ( + delivered + if last_delivery is None + else max(last_delivery, delivered) + ) + if ( + pending + and invalidation is None + and not runtime.stop_event.is_set() + and time.monotonic() - last_live_trust_check >= 1.0 + ): + # Validation tokens keep queued per-client deliveries + # RPC-free, but they cannot observe headers advancing + # ahead of blocks while the best-block hash stays fixed. + # Recheck the live chain view from the fanout driver about + # once per second and cancel every delivery still queued + # if the view becomes untrusted. + try: + trusted = runtime.ensure_reorg_reconciled_for_current_tip( + expected_tip_hash=snapshot.bestblockhash, + ) + if not trusted: + raise _TipRefreshTrustBlocked( + "qbit chain view became untrusted during prepared fanout" + ) + last_live_trust_check = time.monotonic() + except self._shutdown_error: + # Admission can close after the stop-event check above + # but before reconciliation enters its writer scope. + # Preserve the intentional shutdown signal so the + # poller cannot consume its template-failure budget. + cancel_pending_futures(pending) + raise + except TemplateRefreshBlocked as exc: + invalidation = exc + except Exception as exc: + invalidation = _TipRefreshTrustBlocked( + "qbit chain trust check failed during prepared fanout" + ) + invalidation.__cause__ = exc + if invalidation is not None: + cancel_pending_futures(pending) + if invalidation is None: + try: + submit_available(pending) + except RuntimeError: + cancel_pending_futures(pending) + cancel_event.set() + if pending: + wait(pending) + if not runtime.stop_event.is_set(): + runtime._schedule_tip_refresh_retry() + raise + if invalidation is not None: + cancel_event.set() + runtime._schedule_tip_refresh_retry() + raise invalidation + with runtime.lock: + token_current = runtime._tip_refresh_token_current_locked( + validation_token, + bundle, + snapshot, + ) + if not token_current: + runtime._schedule_tip_refresh_retry() + raise _TipRefreshFanoutSuperseded( + "prepared refresh was superseded during fanout; immediate retry scheduled" + ) + try: + post_fanout_tip = str(runtime.rpc.call("getbestblockhash")) + except Exception as exc: + runtime._schedule_tip_refresh_retry() + raise TemplateRefreshBlocked( + "qbit tip validation failed after prepared fanout; " + "immediate retry scheduled" + ) from exc + if post_fanout_tip != snapshot.bestblockhash: + cancel_event.set() + runtime._schedule_tip_refresh_retry() + raise _TipRefreshFanoutSuperseded( + "qbit tip changed during prepared fanout; immediate retry scheduled " + f"expected={snapshot.bestblockhash} current={post_fanout_tip}" + ) + try: + post_fanout_untrusted = bool( + getattr(runtime, "reorg_reconciler_enabled", True) + and runtime.qbit_chain_view_untrusted() + ) + except Exception as exc: + cancel_event.set() + runtime._schedule_tip_refresh_retry() + raise _TipRefreshTrustBlocked( + "qbit chain trust check failed after prepared fanout; " + "immediate retry scheduled" + ) from exc + if post_fanout_untrusted: + cancel_event.set() + runtime._schedule_tip_refresh_retry() + raise _TipRefreshTrustBlocked( + "qbit chain view became untrusted during prepared fanout; " + "immediate retry scheduled" + ) + with runtime.lock: + token_current = runtime._tip_refresh_token_current_locked( + validation_token, + bundle, + snapshot, + ) + if not token_current: + cancel_event.set() + runtime._schedule_tip_refresh_retry() + raise _TipRefreshFanoutSuperseded( + "prepared refresh payout state changed during post-fanout " + "validation; immediate retry scheduled" + ) + if last_delivery is not None and submitted_at: + # Wall-clock span of the wave itself (first task submission + # to last successful delivery), independent of the reconcile + # and bundle-build stages the first/last_delivery histograms + # include. At fleet scale this is the dominant staleness term + # bounded by the delivery worker pool. + runtime._observe_tip_refresh_seconds( + "fanout_wave", + max(0.0, last_delivery - min(submitted_at.values())), + ) + return sent, first_delivery, last_delivery, failed + finally: + runtime._clear_active_tip_refresh(validation_token, cancel_event) + + def poll_qbit_tip_template_once( + self, + *, + heartbeat_name: str = "qbit_blockpoll", + ) -> int: + """Drive one latest-wins refresh wave as its singleflight owner.""" + runtime = self._runtime + runtime._ensure_tip_refresh_state() + runtime._ensure_job_cache_state() + refresh_started = time.monotonic() + observed_best_tip: str | None = None + try: + observation_sequence = runtime._reserve_tip_observation_sequence() + observed_best_tip = str(runtime.rpc.call("getbestblockhash")) + if not runtime.observe_tip_for_refresh( + observed_best_tip, + observation_sequence=observation_sequence, + mark_pending=False, + ): + runtime._schedule_tip_refresh_retry() + raise TemplateRefreshSuperseded( + "tip/template poll was superseded before template fetch" + ) + except TemplateRefreshSuperseded: + if not getattr(runtime, "tip_refresh_epoch_fanout", False): + runtime._record_coordination_blocked_refresh(time.monotonic()) + runtime._note_tip_refresh_attempt_failed(observed_best_tip) + raise + except Exception: + runtime._clear_coordination_blocked_streak() + runtime._record_template_refresh_failure(time.monotonic()) + runtime._note_tip_refresh_attempt_failed(observed_best_tip) + raise + assert observed_best_tip is not None + if not self._tip_refresh_singleflight_lock.acquire(blocking=False): + # A losing observer still publishes newer detection to cancellation + # state, but it does not own (or count) a refresh wave. + runtime.observe_tip_for_refresh( + observed_best_tip, + observation_sequence=observation_sequence, + mark_pending=True, + ) + runtime._schedule_tip_refresh_retry() + return 0 + + total_refreshed = 0 + saw_build_supersession = False + saw_fanout_supersession = False + outcome = "error" + first_pass = True + completed_passes = 0 + try: + while True: + try: + total_refreshed += runtime._poll_qbit_tip_template_pass_once( + heartbeat_name=heartbeat_name, + refresh_started=refresh_started, + observation_sequence=( + observation_sequence if first_pass else None + ), + observed_best_tip=( + observed_best_tip if first_pass else None + ), + ) + first_pass = False + completed_passes += 1 + except PayoutStatePublicationBlocked: + outcome = "payout_blocked" + raise + except _TipRefreshTrustBlocked: + outcome = "trust_blocked" + raise + except JobBuildSuperseded: + saw_build_supersession = True + first_pass = False + completed_passes += 1 + if runtime._tip_refresh_wave_reenters(completed_passes): + continue + outcome = runtime._interrupted_wave_outcome( + "build_superseded" + ) + raise + except TemplateRefreshSuperseded: + saw_fanout_supersession = True + first_pass = False + completed_passes += 1 + if runtime._tip_refresh_wave_reenters(completed_passes): + continue + outcome = runtime._interrupted_wave_outcome( + "fanout_superseded" + ) + raise + except Exception: + outcome = "error" + raise + + if ( + getattr(runtime, "tip_refresh_epoch_fanout", False) + and not runtime._tip_refresh_epoch_fixpoint_reached() + ): + saw_fanout_supersession = True + if runtime._tip_refresh_wave_reenters(completed_passes): + continue + outcome = runtime._interrupted_wave_outcome( + "fanout_superseded" + ) + return total_refreshed + outcome = ( + "build_superseded" + if saw_build_supersession + else "fanout_superseded" + if saw_fanout_supersession + else "completed" + ) + return total_refreshed + finally: + try: + runtime._record_tip_refresh_wave_outcome(outcome) + wave_elapsed = time.monotonic() - refresh_started + runtime._observe_tip_refresh_seconds( + "refresh", + wave_elapsed, + ) + if wave_elapsed > 1.0: + # One line per slow wave (~1/min at current cadence). + # Reconstructing wave latency by pairing blockwait lines + # with "refreshed N client job(s)" lines cannot + # attribute time or see superseded passes; this can. + print( + "prism coordinator: tip refresh wave " + f"outcome={outcome} refreshed={total_refreshed} " + f"passes={completed_passes} " + f"elapsed={wave_elapsed:.3f}s", + flush=True, + ) + finally: + self._tip_refresh_singleflight_lock.release() + + def _interrupted_wave_outcome(self, superseded_outcome: str) -> str: + """Classify a wave that declined re-entry. + + With epoch fanout active, a wave that would have re-entered but for + shutdown was terminated by shutdown, not by the supersession it + recovered; reporting it superseded would pollute the terminal + supersession counts that rollout evaluation watches. Legacy waves + never re-enter, so their supersession outcome stands. + """ + runtime = self._runtime + if ( + getattr(runtime, "tip_refresh_epoch_fanout", False) + and runtime.stop_event.is_set() + ): + return "shutdown" + return superseded_outcome + + def _tip_refresh_wave_reenters(self, completed_passes: int) -> bool: + """Gate owner-wave re-entry so tip churn cannot spin at RPC speed. + + Every re-entry pass costs template and reconcile RPC work against + qbitd exactly when block cadence is fastest. The inter-pass backoff + bounds that poll rate, and a wave that cannot converge within its + pass budget hands the remainder to the scheduled retry, whose fresh + wave restarts from the newest observed state. + """ + runtime = self._runtime + if not getattr(runtime, "tip_refresh_epoch_fanout", False): + return False + if runtime.stop_event.is_set(): + return False + if completed_passes >= PRISM_TIP_REFRESH_WAVE_PASS_BUDGET: + runtime._schedule_tip_refresh_retry() + return False + return not runtime.stop_event.wait( + PRISM_TIP_REFRESH_REENTRY_BACKOFF_SECONDS + ) + + def _poll_qbit_tip_template_pass_once( + self, + *, + heartbeat_name: str, + refresh_started: float, + observation_sequence: int | None = None, + observed_best_tip: str | None = None, + ) -> int: + runtime = self._runtime + runtime._ensure_tip_refresh_state() + runtime._ensure_job_cache_state() + publication_lock_acquired = False + progress_refresh_active = False + pending_signal_token: int | None = None + try: + observation_prevalidated = ( + observation_sequence is not None + and observed_best_tip is not None + ) + if observation_sequence is None: + observation_sequence = runtime._reserve_tip_observation_sequence() + # The interval poller has no push notification to mark priority for + # it. Probe the cheap best-tip RPC before fetching and deriving the + # template so CTV maintenance can yield as soon as a changed tip is + # observed, rather than after reconciliation or bundle preparation. + if observed_best_tip is None: + observed_best_tip = str(runtime.rpc.call("getbestblockhash")) + if ( + not observation_prevalidated + and not runtime.observe_tip_for_refresh( + observed_best_tip, + observation_sequence=observation_sequence, + mark_pending=False, + ) + ): + runtime._schedule_tip_refresh_retry() + raise TemplateRefreshSuperseded( + "tip/template poll was superseded before template fetch" + ) + pending_signal_token = runtime._claim_tip_refresh_pending() + with runtime.lock: + poll_start_clients = tuple( + client + for client in runtime.clients + if runtime.client_can_receive_jobs(client) + ) + with runtime.lock: + current_tip = getattr(self, "current_tip_first_seen", None) + if current_tip is not None and current_tip[0] != observed_best_tip: + pending_signal_token = runtime._mark_tip_refresh_pending_for_poll( + pending_signal_token, + observation_sequence, + ) + # Captured before reconciliation can begin: the pass may run on + # the prefetch worker while this thread fetches the template, so + # the mutation bracket must open before either starts. + payout_generation_before_reconciliation = int( + getattr(runtime, "_payout_state_generation", 0) + ) + # Reconciliation is keyed by tip hash and the probe above already + # observed the best hash, so a memo miss can run its full pass on + # the prefetch worker while this thread fetches and derives the + # template. Whether the pass can be skipped is decided at the + # join below against the memo's state AT THAT TIME -- a tip that + # flips away and back during the fetch evicts the entry so + # post-flip pool-block state is re-proven -- and publication + # still re-proves chain trust in _validate_prepared_tip_refresh + # before any fanout. With the memo disabled (TTL 0) there is no + # way to validate an overlapped pass across the fetch window, so + # the poll keeps its historical serial pass. + reconciler_enabled = bool( + getattr(runtime, "reorg_reconciler_enabled", True) + ) + reconcile_memo_enabled = reconciler_enabled and ( + float( + getattr( + runtime, + "reorg_reconcile_cache_seconds", + DEFAULT_PRISM_REORG_RECONCILE_CACHE_SECONDS, + ) + ) + > 0 + ) + reconcile_prefetch: Future[bool] | None = None + reconcile_probe_started = time.monotonic() + reconcile_detection_epoch = int( + getattr(self, "tip_detection_epoch", 0) + ) + if reconcile_memo_enabled and not runtime._reorg_reconcile_memo_fresh( + observed_best_tip + ): + reconcile_prefetch = runtime._submit_reconcile_prefetch( + observed_best_tip + ) + reconcile_probe_seconds = time.monotonic() - reconcile_probe_started + snapshot = self._reuse_current_tip_template_snapshot(observed_best_tip) + if snapshot is None: + runtime._record_job_cache_event("template", hit=False) + snapshot = runtime.fetch_qbit_tip_template_snapshot() + if not runtime.observe_tip_for_refresh( + snapshot.bestblockhash, + observation_sequence=observation_sequence, + mark_pending=False, + ): + runtime._schedule_tip_refresh_retry() + raise TemplateRefreshSuperseded( + "tip/template poll was superseded during template fetch" + ) + with runtime.lock: + published_after_fetch = getattr( + self, + "current_tip_first_seen", + None, + ) + if ( + published_after_fetch is not None + and published_after_fetch[0] != snapshot.bestblockhash + ): + pending_signal_token = runtime._mark_tip_refresh_pending_for_poll( + pending_signal_token, + observation_sequence, + ) + runtime._record_progress_tip_poll(snapshot) + runtime._progress_refresh_started() + progress_refresh_active = True + runtime.pool_readiness_latched() + with runtime.lock: + previous_snapshot = self.tip_template_snapshot + # Generation orders concurrent observations but is not itself + # a template change. Repeated observations of identical work + # must not trigger a clean fanout on every poll. + snapshot_changed = previous_snapshot is not None and ( + previous_snapshot.bestblockhash != snapshot.bestblockhash + or previous_snapshot.previousblockhash != snapshot.previousblockhash + or previous_snapshot.template_fingerprint + != snapshot.template_fingerprint + ) + if snapshot_changed: + clients = [ + client + for client in runtime.clients + if runtime.client_can_receive_jobs(client) + ] + else: + clients = [ + client + for client in runtime.clients + if runtime.client_can_receive_jobs(client) + and runtime.client_needs_tip_template_refresh(client, snapshot) + ] + # Capture the exact job each client had when this refresh pass + # selected it. A Vardiff/authorize path may install intervening + # work while the shared bundle is prepared or while its task + # waits in the executor queue. Artifact generations let the + # task replace stale intervening work while preserving work + # produced from a template stored after this snapshot. + expected_active_jobs = { + client: client.active_job + for client in clients + } + + if clients and snapshot_changed: + pending_signal_token = runtime._mark_tip_refresh_pending_for_poll( + pending_signal_token, + observation_sequence, + ) + + refreshed = 0 + build_failures = 0 + first_delivery: float | None = None + last_delivery: float | None = None + runtime._raise_if_tip_refresh_superseded( + snapshot, + observation_sequence, + ) + reconcile_source = "serial" + reconcile_join_started = time.monotonic() + try: + if ( + reconcile_memo_enabled + and int(getattr(self, "tip_detection_epoch", 0)) + == reconcile_detection_epoch + and runtime._reorg_reconcile_memo_fresh( + snapshot.bestblockhash + ) + ): + # Fresh at join time AND no detection interleaved the + # fetch. Both are required: a flip away and back evicts + # the entry, but a pass whose execution straddled the + # flip can re-arm it afterwards (the latest detected + # hash matches again), and its proof belongs to the + # closed epoch. Any epoch movement lands in the serial + # re-prove branches below. + reconcile_source = ( + "overlap" if reconcile_prefetch is not None else "memo_hit" + ) + if reconcile_prefetch is not None: + runtime._discard_stale_reconcile_prefetch( + reconcile_prefetch + ) + reconcile_prefetch = None + reorg_reconciled = True + elif ( + reconcile_prefetch is not None + and snapshot.bestblockhash == observed_best_tip + ): + reconcile_source = "overlap" + reorg_reconciled = runtime._join_reconcile_prefetch_bounded( + reconcile_prefetch + ) + if reorg_reconciled and ( + int(getattr(self, "tip_detection_epoch", 0)) + != reconcile_detection_epoch + ): + # A detection interleaved the fetch (a flip away, or + # away and back to this same hash): cached proofs + # were evicted and the overlapped pass may have run + # in the closed epoch. Re-prove off-thread with the + # same bounded join -- the crawl that slows the + # overlap join slows this re-prove identically. + reconcile_source = "serial" + reorg_reconciled = runtime._reconcile_snapshot_tip_bounded( + snapshot.bestblockhash + ) + # A trust flip after the pass completed (headers running + # ahead with no detection) is deliberately NOT re-checked + # here: prepared fanout re-proves the live chain view in + # _validate_prepared_tip_refresh, and sequential issuance + # re-proves it per client in + # ensure_reorg_reconciled_for_current_tip, whose trust + # check is never cached. A second live check here would + # break the one-trust-validation-per-refresh economy. + else: + if reconcile_prefetch is not None: + # The tip moved between the probe and the template + # fetch; the prefetched pass proved a superseded + # hash. Reconcile the snapshot tip off-thread with + # the same bounded join: the crawl that slows the + # overlap join reaches this branch through exactly + # the churn that moves tips mid-fetch, and the poll + # loop must never park on it here either. + runtime._discard_stale_reconcile_prefetch( + reconcile_prefetch + ) + reconcile_prefetch = None + reorg_reconciled = runtime._reconcile_snapshot_tip_bounded( + snapshot.bestblockhash + ) + except (self._shutdown_error, FuturesCancelledError): + # Shutdown may close writer admission (or cancel the queued + # prefetch) after this refresh has fetched a snapshot. Leave + # the refresh incomplete and let the controlled shutdown + # proceed without consuming the template failure budget or + # taking the hard-exit path. + return 0 + except (TemplateRefreshSuperseded, PayoutStatePublicationBlocked): + raise + except Exception as exc: + raise _TipRefreshTrustBlocked( + "qbit reorg reconciliation failed before refresh preparation" + ) from exc + finally: + # The effective serial reconcile cost of this refresh: the + # memo probe plus whatever wait the template fetch did not + # absorb. Observe failures too so a slow reconcile that + # blocks the refresh still shows up in the histogram. + runtime._observe_tip_refresh_build_phase( + "reorg_reconcile", + reconcile_probe_seconds + + (time.monotonic() - reconcile_join_started), + ) + if reconciler_enabled: + runtime._record_reorg_reconcile_lookup( + "tip_refresh", + reconcile_source, + ) + if not reorg_reconciled: + raise _TipRefreshTrustBlocked( + "qbit chain view remained untrusted after reorg reconciliation" + ) + payout_generation_after_reconciliation = int( + getattr(runtime, "_payout_state_generation", 0) + ) + if ( + payout_generation_after_reconciliation + != payout_generation_before_reconciliation + ): + # A same-tip reconciliation can invalidate signed payout state + # even when no client needed template work at initial + # selection. Reselect after the ledger mutation so every old- + # generation job is replaced from the post-reorg snapshot. + with runtime.lock: + clients = [ + client + for client in runtime.clients + if runtime.client_can_receive_jobs(client) + and runtime.client_needs_tip_template_refresh(client, snapshot) + ] + expected_active_jobs = { + client: client.active_job + for client in clients + } + # Reconciliation itself minted the payout-pending token and + # this pass has deliberately reselected clients from that new + # generation. Adopt exactly that token; completion still checks + # both token ownership and payout/detected-tip currentness, so + # a later producer cannot be cleared accidentally. + pending_signal_token = runtime._claim_tip_refresh_pending() + with runtime.lock: + selected_clients_list = list(clients) + selected_client_set = set(clients) + for client in poll_start_clients: + if client in selected_client_set: + continue + if snapshot_changed or runtime.client_needs_tip_template_refresh( + client, + snapshot, + ): + selected_clients_list.append(client) + selected_client_set.add(client) + expected_active_jobs[client] = client.active_job + selected_clients = tuple(selected_clients_list) + # Prepared-mode selection must cover every client this pass can + # end up serving. Revalidation only filters selected_clients, so + # deriving from the narrower initial list could publish authority + # through the sequential path with no shared bundle while + # poll-start targets still need prepared ready work. + use_prepared_fanout = bool( + selected_clients + and getattr(runtime, "_pool_ready_latched", False) + ) + ready_mode = bool(getattr(runtime, "_pool_ready_latched", False)) + bundle: CachedJobBundle | None = None + validation_token: TipRefreshValidationToken | None = None + preactivated_cancel_event: FanoutCancellation | None = None + prepared_executor: ThreadPoolExecutor | None = None + if use_prepared_fanout: + runtime._raise_if_tip_refresh_superseded( + snapshot, + observation_sequence, + ) + try: + bundle = runtime.prepare_tip_refresh_bundle( + snapshot, + priority_requested_monotonic=refresh_started, + ) + except PayoutStatePublicationBlocked: + for _client in selected_clients: + runtime._record_tip_refresh_client_result("skipped") + runtime._schedule_tip_refresh_retry() + raise + except TemplateRefreshBlocked: + for _client in selected_clients: + runtime._record_tip_refresh_client_result("failed") + raise + if ( + bundle.payout_state_generation + != payout_generation_after_reconciliation + ): + # Request construction can observe a payout generation + # published after reconciliation and still return a fully + # coherent newest-generation bundle. Adopt it only while + # it remains the current immutable payout pointer, then + # reselect the complete fleet so no old-generation client + # is omitted merely because selection preceded the build. + with runtime._job_cache_lock: + current_payout_generation = ( + runtime._payout_state_generation + ) + current_payout_artifact = ( + runtime._published_payout_state.artifact + ) + if ( + bundle.payout_state_generation + != current_payout_generation + or bundle.build_key is None + or current_payout_artifact is None + or bundle.build_key.payout_artifact_sha256 + != current_payout_artifact.prior_balances_sha256 + ): + runtime._schedule_tip_refresh_retry() + raise TemplateRefreshSuperseded( + "payout state changed after refresh preparation; " + "immediate retry scheduled" + ) + payout_generation_after_reconciliation = ( + current_payout_generation + ) + with runtime.lock: + selected_clients = tuple( + client + for client in runtime.clients + if runtime.client_can_receive_jobs(client) + and runtime.client_needs_tip_template_refresh( + client, + snapshot, + ) + ) + expected_active_jobs = { + client: client.active_job + for client in selected_clients + } + pending_signal_token = runtime._claim_tip_refresh_pending() + + runtime._raise_if_tip_refresh_superseded( + snapshot, + observation_sequence, + ) + # Construction is complete. Acquire the short publication lane + # only for final validation, snapshot publication, and current + # client selection. Other observations and replacement builds can + # progress while an obsolete builder is still unwinding. + while not self._tip_refresh_lock.acquire( + timeout=PRISM_TIP_REFRESH_ADMISSION_POLL_SECONDS + ): + runtime._record_heartbeat(heartbeat_name) + if runtime.stop_event.is_set(): + return 0 + runtime._probe_tip_while_refresh_waiting() + runtime._raise_if_tip_refresh_superseded( + snapshot, + observation_sequence, + ) + publication_lock_acquired = True + with runtime._job_cache_lock: + current_payout_generation = runtime._payout_state_generation + current_payout_artifact = runtime._published_payout_state.artifact + if ( + current_payout_generation + != payout_generation_after_reconciliation + ): + runtime._schedule_tip_refresh_retry() + raise TemplateRefreshSuperseded( + "payout state changed before refresh publication; " + "immediate retry scheduled" + ) + if ( + bundle is not None + and ( + current_payout_artifact is None + or bundle.build_key is None + or bundle.build_key.payout_artifact_sha256 + != current_payout_artifact.prior_balances_sha256 + ) + ): + runtime._schedule_tip_refresh_retry() + raise TemplateRefreshBlocked( + "complete build key changed before refresh publication" + ) + + # A ready-pool pass must build and validate its immutable shared + # bundle before publishing submit authority. Otherwise a cache, + # derivation, or final chain-validation failure can invalidate + # retained work without a deliverable replacement. Sequential / + # collection work has no shared preparation stage, so it commits + # here immediately before its worker-specific builds. + if use_prepared_fanout: + assert bundle is not None + # Acquire infrastructure before publishing authority. Once the + # active token is installed, every subsequent fanout exit is + # covered by its cancellation/cleanup finally block. + prepared_executor = runtime.tip_refresh_executor() + # Parent metadata is cleanup-only, but fetch it before the + # final live/trust guard so validation remains the last RPC + # boundary before atomic publication. + try: + prepared_parent_hash = runtime._fetch_tip_parent_hash( + snapshot.bestblockhash + ) + except Exception: + prepared_parent_hash = None + validation_token = runtime._validate_prepared_tip_refresh( + bundle, + snapshot, + observation_sequence, + ) + preactivated_cancel_event = runtime._publish_prepared_tip_refresh( + validation_token, + bundle, + snapshot, + parent_hash=prepared_parent_hash, + ) + else: + if not runtime.observe_tip_first_seen( + snapshot.bestblockhash, + observation_sequence=observation_sequence, + publish_refresh_observation=True, + published_snapshot=snapshot, + ): + raise TemplateRefreshSuperseded( + "tip/template poll was superseded by a newer tip observation" + ) + runtime.prune_evicted_job_graveyard(force=False) + with runtime.lock: + current_tip = getattr(self, "current_tip_first_seen", None) + if ( + current_tip is None + or current_tip[0] != snapshot.bestblockhash + or int(getattr(self, "current_tip_observation_sequence", 0)) + != observation_sequence + ): + raise TemplateRefreshSuperseded( + "tip/template poll was superseded before snapshot publication" + ) + if self.tip_template_snapshot is not snapshot: + raise TemplateRefreshBlocked( + "tip/template snapshot was not atomically published" + ) + + dropped_client_results: list[str] = [] + with runtime.lock: + # Revalidate the originally selected targets at publication. + # New connections keep their own pending/retained wake; this + # pass must not consume work that appeared after construction. + # Preserve the pre-build expected job pointer so an intervening + # authorize/Vardiff delivery is never overwritten. + current_clients: list[ClientState] = [] + for client in selected_clients: + if client not in runtime.clients: + dropped_client_results.append("disconnected") + elif not runtime.client_can_receive_jobs(client): + dropped_client_results.append("skipped") + elif not runtime.client_needs_tip_template_refresh(client, snapshot): + dropped_client_results.append("skipped") + else: + current_clients.append(client) + clients = current_clients + + for result in dropped_client_results: + runtime._record_tip_refresh_client_result(result) + + if bundle is not None and not bundle.collection_only: + with runtime._job_cache_lock: + if ( + bundle.payout_state_generation + == runtime._payout_state_generation + ): + runtime._prepared_ready_snapshot = snapshot + runtime._prepared_ready_bundle = bundle + + if ( + not use_prepared_fanout + and clients + and getattr(runtime, "_pool_ready_latched", False) + ): + # Publication already committed through the sequential path, + # so a ready-pool target that appeared after build selection + # has no shared bundle to receive. Leave the pending marker + # armed and let the immediate retry build for it. + runtime._mark_tip_refresh_pending(observation_sequence) + runtime._schedule_tip_refresh_retry() + raise TemplateRefreshBlocked( + "current clients appeared after build selection; retry scheduled" + ) + self._tip_refresh_lock.release() + publication_lock_acquired = False + + with runtime.lock: + progress_eligible_client = any( + runtime.client_can_receive_jobs(client) + for client in runtime.clients + ) + if use_prepared_fanout or not progress_eligible_client: + # Ready-mode work was prepared above. With no eligible clients, + # publishing the reconciled snapshot is sufficient; collection + # work with a live identity is not published until its worker- + # specific bundle is actually delivered below. + runtime._record_progress_publication( + snapshot, + payout_generation_after_reconciliation, + ) + + if not ready_mode: + with runtime.lock: + eligible_collection_client = any( + runtime.client_can_receive_jobs(client) + for client in runtime.clients + ) + if not eligible_collection_client: + runtime._retain_collection_refresh( + snapshot, + observation_sequence, + payout_generation_after_reconciliation, + ) + + if use_prepared_fanout: + assert bundle is not None + ( + refreshed, + first_delivery, + last_delivery, + build_failures, + ) = runtime._fanout_prepared_tip_refresh( + clients, + bundle, + snapshot, + observation_sequence=observation_sequence, + validation_token=validation_token, + preactivated_cancel_event=preactivated_cancel_event, + executor=prepared_executor, + expected_active_jobs=expected_active_jobs, + heartbeat_name=heartbeat_name, + ) + else: + for client in clients: + if runtime.stop_event.is_set(): + break + # Collection bundles are worker-specific, so build and + # validate each selected target independently. + runtime._record_heartbeat(heartbeat_name) + with runtime.lock: + target_connected = client in runtime.clients + target_eligible = ( + target_connected + and runtime.client_can_receive_jobs(client) + ) + if not target_connected: + runtime._record_tip_refresh_client_result("disconnected") + continue + if not target_eligible: + runtime._record_tip_refresh_client_result("skipped") + continue + try: + if runtime.maybe_send_job( + client, + clean_jobs=runtime.client_tip_changed_for_snapshot(client, snapshot), + raise_on_reorg_failure=True, + raise_on_build_failure=True, + tip_refresh_snapshot=snapshot, + tip_refresh_observation_sequence=observation_sequence, + ): + delivered = time.monotonic() + refreshed += 1 + first_delivery = ( + delivered + if first_delivery is None + else min(first_delivery, delivered) + ) + last_delivery = ( + delivered + if last_delivery is None + else max(last_delivery, delivered) + ) + runtime._record_tip_refresh_client_result("sent") + else: + runtime._record_tip_refresh_client_result("skipped") + except self._job_build_failed_error: + build_failures += 1 + runtime._record_tip_refresh_client_result("failed") + except OSError: + runtime._record_tip_refresh_client_result("disconnected") + runtime.disconnect_client(client) + + if not ready_mode: + with runtime.lock: + eligible_collection_client = any( + runtime.client_can_receive_jobs(client) + for client in runtime.clients + ) + if not eligible_collection_client: + runtime._retain_collection_refresh( + snapshot, + observation_sequence, + payout_generation_after_reconciliation, + ) + runtime._record_progress_publication( + snapshot, + payout_generation_after_reconciliation, + ) + + if clients: + try: + post_fanout_tip = str(runtime.rpc.call("getbestblockhash")) + except Exception as exc: + runtime._schedule_tip_refresh_retry() + raise TemplateRefreshBlocked( + "qbit tip validation failed after sequential refresh; " + "immediate retry scheduled" + ) from exc + if post_fanout_tip != snapshot.bestblockhash: + runtime._schedule_tip_refresh_retry() + raise TemplateRefreshSuperseded( + "qbit tip changed during sequential refresh; " + "immediate retry scheduled " + f"expected={snapshot.bestblockhash} current={post_fanout_tip}" + ) + if int(getattr(runtime, "_payout_state_generation", 0)) != ( + payout_generation_after_reconciliation + ): + runtime._schedule_tip_refresh_retry() + raise TemplateRefreshSuperseded( + "payout state changed during sequential refresh; " + "immediate retry scheduled" + ) + + if refreshed == 0 and build_failures: + raise TemplateRefreshBlocked( + f"job builds failed for {build_failures} client(s); no refreshed work was issued" + ) + if refreshed: + with runtime.lock: + self.tip_refresh_job_count += refreshed + assert first_delivery is not None and last_delivery is not None + runtime._observe_tip_refresh_seconds( + "first_delivery", + first_delivery - refresh_started, + ) + runtime._observe_tip_refresh_seconds( + "last_delivery", + last_delivery - refresh_started, + ) + if not runtime._clear_tip_refresh_pending_for_completed_refresh( + snapshot, + observation_sequence, + payout_generation_after_reconciliation, + pending_signal_token, + ): + # A newer tip or payout mutation won after the last delivery + # guard. Preserve its pending token and retry immediately. + pending_signal_token = None + runtime._schedule_tip_refresh_retry() + raise TemplateRefreshSuperseded( + "tip or payout state changed before refresh completion; " + "immediate retry scheduled" + ) + self.last_successful_template_refresh_monotonic = time.monotonic() + self.template_refresh_failure_started_monotonic = None + runtime._clear_coordination_blocked_streak() + self._clear_tip_refresh_failure_holdoff() + # A completed pass reconfirms that the coherent snapshot remained + # current through publication and fanout. Refresh the liveness + # stamp so a legitimately long, progressing pass does not become + # stale the instant its active marker is cleared. + runtime._record_progress_tip_poll(snapshot) + return refreshed + except TemplateRefreshSuperseded: + # Coordination-blocked refreshes -- a superseded tip, a pending + # payout publication fence, a refresh raced by payout mutation -- + # are churn between healthy components, not qbitd unhealthiness. + # They do not arm the ordinary failure budget, but their oldest + # continuous streak has its own longer restart deadline. Re-raise + # so callers still schedule their immediate retry. Plain + # TemplateRefreshBlocked stays budgeted below: it also wraps + # genuine failures (job builds failing, malformed template + # artifacts, untrusted chain views) whose persistence must still + # take the ordinary restart path. + # The retry-spacing stamp still applies: churn against an + # unchanged tip re-arms the poller no faster than the holdoff, + # while a genuinely newer tip zeroes it immediately. + if not getattr(runtime, "tip_refresh_epoch_fanout", False): + runtime._record_coordination_blocked_refresh(time.monotonic()) + runtime._note_tip_refresh_attempt_failed(observed_best_tip) + raise + except PayoutStatePublicationBlocked: + runtime._record_coordination_blocked_refresh(time.monotonic()) + runtime._note_tip_refresh_attempt_failed(observed_best_tip) + raise + except Exception: + runtime._clear_coordination_blocked_streak() + runtime._record_template_refresh_failure(time.monotonic()) + runtime._note_tip_refresh_attempt_failed(observed_best_tip) + raise + finally: + if publication_lock_acquired: + self._tip_refresh_lock.release() + if progress_refresh_active: + runtime._progress_refresh_finished() + + def _probe_tip_while_refresh_waiting(self) -> None: + """Detect a changed live tip without entering the heavy refresh lane.""" + runtime = self._runtime + observation_sequence = runtime._reserve_tip_observation_sequence() + try: + observed_tip = str(runtime.rpc.call("getbestblockhash")) + except Exception: + # The owning refresh still has to unwind or complete. Preserve its + # pending state and let the next bounded lock wait probe again. + return + runtime.observe_tip_for_refresh( + observed_tip, + observation_sequence=observation_sequence, + ) + + def _detected_tip_supersedes_locked( + self, + tip_hash: str, + observation_sequence: int, + ) -> bool: + latest = getattr(self, "latest_detected_tip", None) + return bool( + latest is not None + and latest[0] != tip_hash + and latest[1] > observation_sequence + ) + + def _raise_if_tip_refresh_superseded( + self, + snapshot: QbitTipTemplateSnapshot, + observation_sequence: int, + ) -> None: + """Stop obsolete work before entering another expensive phase.""" + runtime = self._runtime + with runtime.lock: + superseded = runtime._detected_tip_supersedes_locked( + snapshot.bestblockhash, + observation_sequence, + ) + if superseded: + runtime._schedule_tip_refresh_retry() + raise TemplateRefreshSuperseded( + "tip/template poll was superseded by a newer tip observation " + "before refresh preparation" + ) + + def _reserve_tip_observation_sequence(self) -> int: + runtime = self._runtime + with runtime.lock: + sequence = int(getattr(self, "tip_observation_sequence", 0)) + 1 + self.tip_observation_sequence = sequence + return sequence + + def observe_tip_for_refresh( + self, + tip_hash: str, + *, + observation_sequence: int | None = None, + mark_pending: bool = True, + ) -> bool: + """Record tip detection without publishing share-validation authority. + + A waiter outside ``_tip_refresh_lock`` must be able to cancel obsolete + bundle construction and fanout promptly. It must not update + ``current_tip_first_seen``: that value invalidates old jobs, so the + winning refresh publishes it only after replacement work is prepared + and validated. + """ + runtime = self._runtime + if observation_sequence is None: + observation_sequence = runtime._reserve_tip_observation_sequence() + runtime._ensure_job_cache_state() + runtime._ensure_tip_refresh_state() + # Before any sequencing decision: an own-candidate hash arriving as + # the tip is acceptance evidence (blockwait can see it before -- or + # instead of -- the direct submitblock ack) and must be registered + # even when this observation loses the sequence race below. + runtime._note_tip_observation_for_candidates(tip_hash) + now = time.monotonic() + active_to_cancel: FanoutCancellation | None = None + should_mark_pending = False + with runtime.lock: + latest = getattr(self, "latest_detected_tip", None) + if latest is not None and observation_sequence < latest[1]: + return latest[0] == tip_hash + published = getattr(self, "current_tip_first_seen", None) + prior_detected_hash = ( + latest[0] + if latest is not None + else published[0] + if published is not None + else None + ) + detection_changed = ( + prior_detected_hash is not None + and prior_detected_hash != tip_hash + ) + if detection_changed: + runtime._evict_reorg_reconcile_memo_for_new_tip_locked(tip_hash) + # Bumped exactly when cached reconcile proofs are dropped: + # a refresh's join compares this epoch to tell whether any + # flip (away, or away and back) interleaved its template + # fetch and invalidated a pass that ran concurrently. + self.tip_detection_epoch = ( + int(getattr(self, "tip_detection_epoch", 0)) + 1 + ) + if ( + detection_changed + and runtime._payout_state_source[1] != tip_hash + ): + # Supersede an in-progress immutable payout candidate as soon + # as the newer tip is detected, without publishing that tip as + # submit authority before replacement work is ready. + source_generation = runtime._payout_state_source[0] + 1 + runtime._payout_state_source = ( + source_generation, + tip_hash, + "external_tip", + now, + ) + if latest is None or detection_changed: + runtime._mint_tip_refresh_epoch_locked( + tip_hash=tip_hash, + payout_state_generation=int( + getattr(runtime, "_payout_state_generation", 0) + ), + started_monotonic=now, + ) + self.latest_detected_tip = (tip_hash, observation_sequence) + replacement_needed = published is None or published[0] != tip_hash + if published is not None and published[0] == tip_hash: + # A live observation has returned to (or reconfirmed) the + # published generation. This closes any unpublished divergence + # epoch without changing the published snapshot itself. + self.current_tip_observed_monotonic = now + self.tip_refresh_divergence_started_monotonic = None + elif ( + published is not None + and getattr( + self, + "tip_refresh_divergence_started_monotonic", + None, + ) + is None + ): + # Anchor the lease to the first departure from the published + # tip. Further B -> C observations deliberately do not renew it. + self.tip_refresh_divergence_started_monotonic = now + pending_already = bool( + getattr(self, "_tip_refresh_pending_event", None) + and self._tip_refresh_pending_event.is_set() + ) + active = getattr(self, "_active_tip_refresh", None) + if ( + active is not None + and active[0].tip_hash != tip_hash + and active[0].observation_sequence < observation_sequence + ): + active_to_cancel = active[1] + should_mark_pending = bool( + mark_pending + and ( + detection_changed + or (replacement_needed and not pending_already) + ) + ) + + if active_to_cancel is not None: + active_to_cancel.cancel() + if detection_changed: + runtime._progress_note_refresh_pending(now) + # In-flight constructions for older parents can no longer win; + # stop them at detection so replacement preparation starts + # immediately, well before publication moves submit authority. + runtime._cancel_obsolete_job_bundle_builds(current_tip=tip_hash) + runtime._cancel_obsolete_job_builds("chain tip superseded") + if should_mark_pending: + runtime._mark_tip_refresh_pending(observation_sequence) + runtime._schedule_tip_refresh_retry() + return True + + def observe_tip_first_seen( + self, + tip_hash: str, + *, + observation_sequence: int | None = None, + publish_refresh_observation: bool = False, + published_snapshot: QbitTipTemplateSnapshot | None = None, + ) -> bool: + """Publish a tip (and, when supplied, its coherent template snapshot). + + Detection and publication are deliberately separate. Callers doing a + prepared refresh must finish bundle construction and final validation + before invoking this method. + """ + runtime = self._runtime + if ( + published_snapshot is not None + and published_snapshot.bestblockhash != tip_hash + ): + raise ValueError("published snapshot does not match tip hash") + if observation_sequence is None: + observation_sequence = runtime._reserve_tip_observation_sequence() + if not runtime.observe_tip_for_refresh( + tip_hash, + observation_sequence=observation_sequence, + mark_pending=False, + ): + return False + now = time.monotonic() + with runtime.lock: + current_sequence = int( + getattr(self, "current_tip_observation_sequence", 0) + ) + if ( + observation_sequence < current_sequence + or runtime._detected_tip_supersedes_locked( + tip_hash, + observation_sequence, + ) + ): + return False + first_seen = getattr(self, "current_tip_first_seen", None) + if first_seen is not None and first_seen[0] == tip_hash: + # A same-tip re-observation proves the tip view is live; the + # freshness stamp bounds submit_stale_check_tip reuse. + self.current_tip_observed_monotonic = now + active = getattr(self, "_active_tip_refresh", None) + # A routine blockwait/poll observation of the same hash carries + # no newer template. While that hash is actively fanning out, + # do not invalidate its token merely by advancing the global + # observation sequence. The next real refresh observation can + # advance it after the active fanout clears. + if publish_refresh_observation and ( + active is None or active[0].tip_hash != tip_hash + ): + self.current_tip_observation_sequence = observation_sequence + if published_snapshot is not None: + self.tip_template_snapshot = published_snapshot + runtime._publish_tip_refresh_epoch_identity_locked( + published_snapshot + ) + self.tip_refresh_divergence_started_monotonic = None + return True + + # Fetch optional cleanup metadata before the atomic publication. A slow + # parent RPC must not create a window where submits see the new tip but + # the winning refresh has not yet published its coherent snapshot. + try: + parent_hash = runtime._fetch_tip_parent_hash(tip_hash) + except Exception: + parent_hash = None + + with runtime.lock: + current_sequence = int( + getattr(self, "current_tip_observation_sequence", 0) + ) + if ( + observation_sequence < current_sequence + or runtime._detected_tip_supersedes_locked( + tip_hash, + observation_sequence, + ) + ): + return False + first_seen = getattr(self, "current_tip_first_seen", None) + if first_seen is not None and first_seen[0] == tip_hash: + self.current_tip_observed_monotonic = now + active = getattr(self, "_active_tip_refresh", None) + if publish_refresh_observation and ( + active is None or active[0].tip_hash != tip_hash + ): + self.current_tip_observation_sequence = observation_sequence + if published_snapshot is not None: + self.tip_template_snapshot = published_snapshot + runtime._publish_tip_refresh_epoch_identity_locked( + published_snapshot + ) + self.tip_refresh_divergence_started_monotonic = None + return True + + tip_changed = first_seen is not None + # The first tip this process publishes is a startup baseline, not + # a tip flip: a None stamp keeps stale grace closed. + self.current_tip_first_seen = ( + tip_hash, + now if tip_changed else None, + ) + self.current_tip_observation_sequence = observation_sequence + self.current_tip_observed_monotonic = now + if published_snapshot is not None: + self.tip_template_snapshot = published_snapshot + runtime._publish_tip_refresh_epoch_identity_locked( + published_snapshot + ) + self.tip_refresh_divergence_started_monotonic = None + # Retained collection work is reusable throughout detection and + # preparation, but never after authority moves to a different tip. + self._retained_collection_refresh = None + if parent_hash is None: + self.current_tip_parent = None + else: + self.current_tip_parent = (tip_hash, parent_hash) + # Reclassify formerly same-tip entries immediately. On mainnet the + # zero stale-grace TTL removes them in this pass; on other chains + # the actual chain parent removes multi-tip-behind entries while + # the independently configured grace lifetime protects one-back. + runtime.prune_evicted_job_graveyard(now=now, force=True) + if tip_changed: + # A retained ready bundle belongs to the previously published tip + # and can never satisfy the consumer's published-snapshot identity + # check again; release it as soon as authority moves. + with runtime._job_cache_lock: + runtime._prepared_ready_bundle = None + runtime._prepared_ready_snapshot = None + return True + + def reorg_proof_snapshot(self) -> tuple[str | None, int]: + """Atomic (latest detected tip hash, detection epoch) reorg proof. + + The reorg reconciler remains coordinator-owned at this layer; its + later owner consumes this snapshot instead of reading the two fields + separately, so a detection between the reads can never pair a new tip + with a stale epoch. + """ + runtime = self._runtime + with runtime.lock: + latest = getattr(self, "latest_detected_tip", None) + return ( + latest[0] if latest is not None else None, + int(getattr(self, "tip_detection_epoch", 0)), + ) + + def _fetch_tip_parent_hash(self, tip_hash: str) -> str | None: + runtime = self._runtime + block = runtime.rpc.call("getblock", [tip_hash]) + if not isinstance(block, dict): + return None + parent = str(block.get("previousblockhash", "") or "") + if not parent: + return None + return parent + + def current_tip_parent_hash(self, tip_hash: str) -> str | None: + runtime = self._runtime + with runtime.lock: + cached = getattr(self, "current_tip_parent", None) + if cached is not None and cached[0] == tip_hash: + return cached[1] + first_seen = getattr(self, "current_tip_first_seen", None) + observed_sequence = ( + int(getattr(self, "current_tip_observation_sequence", 0)) + if first_seen is not None and first_seen[0] == tip_hash + else None + ) + parent = runtime._fetch_tip_parent_hash(tip_hash) + if parent is None: + return None + with runtime.lock: + current = getattr(self, "current_tip_first_seen", None) + if ( + observed_sequence is not None + and current is not None + and current[0] == tip_hash + and int(getattr(self, "current_tip_observation_sequence", 0)) + == observed_sequence + ): + self.current_tip_parent = (tip_hash, parent) + return parent + + def submit_stale_check_tip(self) -> str: + """Best-known chain tip for per-share submit classification. + + Prefers the tip for which the refresh path already published coherent + work (reconfirmed at least every PRISM_BLOCKPOLL_SECONDS while healthy) + so mining.submit never blocks on a getbestblockhash RPC per share. This + also removes the submit-races-ahead-of-the-refresh failure mode: a + submit-path RPC can observe a new tip seconds before jobs refresh, and + with PRISM_STRATUM_STALE_GRACE_SECONDS=0 (then pinned on mainnet) that + rejected every in-flight share on the old tip. Classifying against the + published tip keeps shares valid until the coordinator has prepared, + validated, and published the flip, and it is the same tip source the + stale-grace window and evicted-job classification are anchored to. + + During a detected-but-unpublished replacement, the published tip stays + authoritative beyond the ordinary freshness age so a large healthy + build cannot recreate the reject outage. That extension is bounded by + PRISM_TEMPLATE_REFRESH_FAILURE_EXIT_SECONDS and is anchored to the + first unpublished divergence; failed refreshes therefore still fall + back to the live RPC instead of accepting frozen work indefinitely. + """ + runtime = self._runtime + with runtime.lock: + published_tip = runtime._submit_stale_check_tip_locked(time.monotonic()) + if published_tip is not None: + return published_tip + return str(runtime.rpc.call("getbestblockhash")) + + def _published_tip_authoritative_locked(self, now: float) -> bool: + """True while the published tip still owns share classification. + + Either the ordinary freshness window (reconfirmed within + PRISM_SUBMIT_TIP_MAX_AGE_SECONDS) or the bounded detected-but- + unpublished replacement lease is open. Job issuance uses the same + predicate so work handed to miners is never classified against a + different tip than the one it was issued for. + """ + runtime = self._runtime + max_age = float( + getattr( + runtime, + "submit_tip_max_age_seconds", + DEFAULT_PRISM_SUBMIT_TIP_MAX_AGE_SECONDS, + ) + ) + if max_age <= 0: + return False + observed = getattr(self, "current_tip_first_seen", None) + if observed is None: + return False + observed_at = getattr(self, "current_tip_observed_monotonic", None) + if observed_at is not None and now - observed_at <= max_age: + return True + latest_detected = getattr(self, "latest_detected_tip", None) + divergence_started = getattr( + self, + "tip_refresh_divergence_started_monotonic", + None, + ) + divergence_budget = float( + getattr( + runtime, + "template_refresh_failure_exit_seconds", + DEFAULT_PRISM_TEMPLATE_MAX_AGE_SECONDS, + ) + ) + return bool( + latest_detected is not None + and latest_detected[0] != observed[0] + and divergence_started is not None + and divergence_budget > 0 + and now - divergence_started <= divergence_budget + ) + + def refresh_jobs_after_pending_accepted_block( + self, + client: ClientState, + *, + heartbeat_name: str = "qbit_blockpoll", + ) -> int: + runtime = self._runtime + with runtime.lock: + block = client.post_accept_refresh_block + client.post_accept_refresh_block = None + if block is None: + return 0 + block_height, block_hash = block + return runtime.refresh_jobs_after_accepted_block( + block_height=block_height, + block_hash=block_hash, + heartbeat_name=heartbeat_name, + ) + + def refresh_jobs_after_accepted_block( + self, *, block_height: int, block_hash: str, heartbeat_name: str = "qbit_blockpoll" + ) -> int: + runtime = self._runtime + try: + runtime._record_heartbeat(heartbeat_name) + observation_sequence = runtime._reserve_tip_observation_sequence() + observed_tip = str(runtime.rpc.call("getbestblockhash")) + if not runtime.observe_tip_for_refresh( + observed_tip, + observation_sequence=observation_sequence, + ): + raise TemplateRefreshSuperseded( + "post-accept tip observation was superseded" + ) + except TemplateRefreshSuperseded: + # A newer observation won before this notification could publish + # its tip. The shared driver is woken below; this is coordination + # churn, not a post-accept refresh failure. + return 0 + except Exception: + with runtime.lock: + self.post_accept_refresh_failure_count += 1 + print( + "prism coordinator: post-accept clean job refresh failed after direct PRISM block " + f"height={block_height} hash={block_hash}", + flush=True, + ) + traceback.print_exc() + return 0 + finally: + # Accepted-block payout/template changes can require a same-tip + # rebuild. Wake the driver even when the immediate best-tip RPC or + # observation fails; the owner will refetch a coherent snapshot. + runtime._schedule_tip_refresh_retry() + print( + "prism coordinator: scheduled single-flight job refresh after " + f"direct PRISM block height={block_height} hash={block_hash} " + f"observed_tip={observed_tip}", + flush=True, + ) + return 0 + + @staticmethod + def _tip_refresh_hashrate_proxy(client: ClientState) -> Decimal: + proxy = ( + getattr(client, "vardiff_difficulty_estimate", None) + or getattr(client, "share_difficulty", Decimal(0)) + ) + try: + resolved = Decimal(proxy) + except (InvalidOperation, TypeError, ValueError): + return Decimal(0) + return resolved if resolved.is_finite() and resolved > 0 else Decimal(0) + + def _payout_epoch_tip_hash_locked(self) -> str | None: + """Label a payout-minted epoch with the current refresh target tip. + + A payout publication advances the payout generation of the target + the fleet is already converging on; it must not move or invent the + tip. Candidate source hashes are unusable here: an accepted-block + preview stores the accepted block's own hash, which is not a + template parent and can trail an already-observed newer tip, so + labeling the epoch with it strands token currency, identity + publication, and the fixpoint until a later tip observation + remints. + """ + current_epoch_tip = getattr(self, "_tip_refresh_epoch_tip_hash", None) + if current_epoch_tip is not None: + return str(current_epoch_tip) + latest_detected = getattr(self, "latest_detected_tip", None) + if latest_detected is not None: + return str(latest_detected[0]) + published_tip = getattr(self, "current_tip_first_seen", None) + if published_tip is not None: + return str(published_tip[0]) + return None + + def _mint_tip_refresh_epoch_locked( + self, + *, + tip_hash: str, + payout_state_generation: int, + started_monotonic: float, + ) -> int: + """Mint the next target epoch while the coordinator lock is held.""" + runtime = self._runtime + if not getattr(runtime, "tip_refresh_epoch_fanout", False): + return 0 + sequence = int(self._tip_refresh_epoch_sequence) + 1 + self._tip_refresh_epoch_sequence = sequence + self._tip_refresh_epoch_tip_hash = tip_hash + self._tip_refresh_epoch_payout_generation = int(payout_state_generation) + client_weights = { + int(client.connection_id): runtime._tip_refresh_hashrate_proxy(client) + for client in runtime.clients + if runtime.client_can_receive_jobs(client) + } + total_weight = sum( + ( + weight + for weight in client_weights.values() + if weight > 0 + ), + Decimal(0), + ) + self._tip_refresh_epoch_coverage = _TipRefreshEpochCoverage( + sequence=sequence, + tip_hash=tip_hash, + payout_state_generation=int(payout_state_generation), + started_monotonic=float(started_monotonic), + client_weights=client_weights, + total_weight=total_weight, + ) + if total_weight <= 0: + tracker = self._tip_refresh_epoch_coverage + with self._tip_refresh_metrics_lock: + for target, _ratio in PRISM_TIP_REFRESH_COVERAGE_TARGETS: + tracker.recorded_targets.add(target) + histogram = self.tip_refresh_coverage_histograms[target] + histogram["count"] = int(histogram["count"]) + 1 + buckets = histogram["buckets"] + assert isinstance(buckets, dict) + for bucket in PRISM_TIP_REFRESH_SECONDS_BUCKETS: + buckets[bucket] = int(buckets.get(bucket, 0)) + 1 + return sequence + + def _tip_refresh_epoch_for_bundle_locked( + self, + cached: CachedJobBundle, + ) -> int: + runtime = self._runtime + if not getattr(runtime, "tip_refresh_epoch_fanout", False): + return 0 + identity = getattr( + self, + "_published_tip_refresh_epoch_identity", + None, + ) + if identity is None: + return 0 + ( + sequence, + tip_hash, + payout_generation, + template_fingerprint, + template_generation, + ) = identity + parent_hash = str(cached.template.get("previousblockhash", "")) + if ( + parent_hash != tip_hash + or int(cached.payout_state_generation) + != payout_generation + or cached.template_fingerprint != template_fingerprint + or int(cached.template_generation) + != template_generation + ): + return 0 + return sequence + + def _publish_tip_refresh_epoch_identity_locked( + self, + snapshot: QbitTipTemplateSnapshot, + ) -> None: + runtime = self._runtime + if not getattr(runtime, "tip_refresh_epoch_fanout", False): + return + sequence = int(getattr(self, "_tip_refresh_epoch_sequence", 0)) + tip_hash = getattr(self, "_tip_refresh_epoch_tip_hash", None) + payout_generation = int( + getattr(self, "_tip_refresh_epoch_payout_generation", 0) + ) + if ( + sequence <= 0 + or tip_hash != snapshot.bestblockhash + or payout_generation + != int(getattr(runtime, "_payout_state_generation", 0)) + ): + return + self._published_tip_refresh_epoch_identity = ( + sequence, + tip_hash, + payout_generation, + snapshot.template_fingerprint, + snapshot.template_generation, + ) + + def _admit_client_tip_refresh_epoch_locked( + self, + client: ClientState, + epoch_sequence: int, + ) -> bool: + """Fence per-client registration so epochs cannot regress on the wire.""" + runtime = self._runtime + if not getattr(runtime, "tip_refresh_epoch_fanout", False): + return True + if runtime._client_tip_refresh_epoch_blocked_locked(client, epoch_sequence): + return False + admitted_epoch = int( + getattr(client, "_tip_refresh_admitted_epoch_sequence", 0) + ) + client._tip_refresh_admitted_epoch_sequence = max( + int(epoch_sequence), + admitted_epoch, + ) + return True + + @staticmethod + def _client_tip_refresh_epoch_blocked_locked( + client: ClientState, + epoch_sequence: int, + ) -> bool: + active_epoch = int( + getattr( + getattr(client, "active_job", None), + "tip_refresh_epoch_sequence", + 0, + ) + ) + admitted_epoch = int( + getattr(client, "_tip_refresh_admitted_epoch_sequence", 0) + ) + return int(epoch_sequence) < max(active_epoch, admitted_epoch) + + def _tip_refresh_epoch_coverage_reached_locked( + self, + client: ClientState, + context: PrismJobContext, + delivered_monotonic: float, + ) -> list[tuple[str, float]]: + """Advance the active cohort while the coordinator lock is held.""" + runtime = self._runtime + if not getattr(runtime, "tip_refresh_epoch_fanout", False): + return [] + reached: list[tuple[str, float]] = [] + tracker = self._tip_refresh_epoch_coverage + connection_id = int(getattr(client, "connection_id", 0)) + if ( + tracker is None + or tracker.sequence != int(self._tip_refresh_epoch_sequence) + or tracker.total_weight <= 0 + or connection_id in tracker.delivered_clients + ): + return reached + weight = tracker.client_weights.get(connection_id) + if weight is None: + return reached + if ( + int(getattr(context, "connection_id", 0)) != connection_id + or int(getattr(context, "tip_refresh_epoch_sequence", 0)) + != tracker.sequence + or str(context.template.get("previousblockhash", "")) + != tracker.tip_hash + or int(getattr(context, "payout_state_generation", 0)) + != tracker.payout_state_generation + ): + return reached + tracker.delivered_clients.add(connection_id) + if weight <= 0: + return reached + tracker.delivered_weight += weight + coverage = tracker.delivered_weight / tracker.total_weight + elapsed = max( + 0.0, + float(delivered_monotonic) - tracker.started_monotonic, + ) + for target, ratio in PRISM_TIP_REFRESH_COVERAGE_TARGETS: + if target not in tracker.recorded_targets and coverage >= ratio: + tracker.recorded_targets.add(target) + reached.append((target, elapsed)) + return reached + + def _record_tip_refresh_epoch_coverage( + self, + reached: list[tuple[str, float]], + ) -> None: + """Observe fixed cohort thresholds without per-connection series.""" + if not reached: + return + with self._tip_refresh_metrics_lock: + for target, elapsed in reached: + histogram = self.tip_refresh_coverage_histograms[target] + histogram["count"] = int(histogram["count"]) + 1 + histogram["sum"] = float(histogram["sum"]) + elapsed + buckets = histogram["buckets"] + assert isinstance(buckets, dict) + for bucket in PRISM_TIP_REFRESH_SECONDS_BUCKETS: + if elapsed <= bucket: + buckets[bucket] = int(buckets.get(bucket, 0)) + 1 + + def _tip_refresh_epoch_fixpoint_reached(self) -> bool: + """Return whether every eligible connection has newest delivered work.""" + runtime = self._runtime + if not getattr(runtime, "tip_refresh_epoch_fanout", False): + return True + with runtime.lock: + sequence = int(getattr(self, "_tip_refresh_epoch_sequence", 0)) + tip_hash = getattr(self, "_tip_refresh_epoch_tip_hash", None) + payout_generation = int( + getattr(self, "_tip_refresh_epoch_payout_generation", 0) + ) + snapshot = getattr(self, "tip_template_snapshot", None) + ready_mode = bool(getattr(runtime, "_pool_ready_latched", False)) + for client in runtime.clients: + if not runtime.client_can_receive_jobs(client): + continue + context = getattr(client, "_progress_delivered_context", None) + if ( + context is None + or int(getattr(context, "connection_id", 0)) + != int(client.connection_id) + ): + return False + context_epoch = int( + getattr(context, "tip_refresh_epoch_sequence", 0) + ) + if context_epoch > sequence: + continue + if context_epoch < sequence: + return False + if str(context.template.get("previousblockhash", "")) != tip_hash: + return False + if ( + int(getattr(context, "payout_state_generation", 0)) + != payout_generation + ): + return False + if ready_mode and bool(getattr(context, "collection_only", False)): + return False + if ( + snapshot is not None + and snapshot.bestblockhash == tip_hash + and ( + getattr(context, "template_fingerprint", None) + != snapshot.template_fingerprint + ) + ): + return False + # Every eligible connection holds newest delivered work, so any + # refresh retry armed before this verification is satisfied. + # Consume it inside the same lock hold that proved convergence: + # signals armed afterwards stay armed, and blockpoll no longer + # re-runs a full wave against work a recovered supersession + # already finished. + self._tip_refresh_retry_consumed = self._tip_refresh_retry_counter + self._tip_refresh_retry.clear() + return True + + def _clear_tip_refresh_failure_holdoff(self) -> None: + runtime = self._runtime + runtime._ensure_tip_refresh_state() + with runtime.lock: + self._tip_refresh_failure_holdoff_until = None + self._tip_refresh_failure_tip = None + + def _reuse_current_tip_template_snapshot( + self, + observed_best_tip: str, + ) -> QbitTipTemplateSnapshot | None: + """Rebuild a snapshot from cached artifacts while their tip holds. + + Honors the PRISM_TEMPLATE_CACHE_SECONDS window that per-client job + builds already use: within it, a poll pass whose observed best tip + still equals the cached template's parent issues no getblocktemplate. + Rapid failed-refresh re-attempts then cost one cheap best-hash probe + each, while a changed tip or an expired window falls through to the + full fetch, so same-tip template rotation still lands on the normal + cadence. + """ + runtime = self._runtime + runtime._ensure_job_cache_state() + ttl = float( + getattr(runtime, "template_cache_seconds", DEFAULT_PRISM_BLOCKPOLL_SECONDS) + ) + if ttl <= 0: + return None + with runtime._job_cache_lock: + cached = runtime._template_artifacts + if cached is None: + return None + if time.monotonic() - cached.fetched_monotonic > ttl: + return None + if str(observed_best_tip) != cached.previousblockhash: + return None + runtime._record_job_cache_event("template", hit=True) + return QbitTipTemplateSnapshot( + bestblockhash=cached.previousblockhash, + previousblockhash=cached.previousblockhash, + template_fingerprint=cached.fingerprint, + template_generation=cached.generation, + template_artifacts=cached, + ) + + +__all__ = [ + "DEFAULT_PRISM_PAYOUT_RECONCILE_SUPERSESSION_RETRIES", + "FanoutCancellation", + "PRISM_TIP_REFRESH_ADMISSION_POLL_SECONDS", + "PRISM_TIP_REFRESH_BUILD_PHASES", + "PRISM_TIP_REFRESH_CANCELLATION_STAGES", + "PRISM_TIP_REFRESH_COVERAGE_TARGETS", + "PRISM_TIP_REFRESH_FAILURE_HOLDOFF_JITTER_FRACTION", + "PRISM_TIP_REFRESH_REENTRY_BACKOFF_SECONDS", + "PRISM_TIP_REFRESH_RESULTS", + "PRISM_TIP_REFRESH_SECONDS_BUCKETS", + "PRISM_TIP_REFRESH_WAVE_OUTCOMES", + "PRISM_TIP_REFRESH_WAVE_PASS_BUDGET", + "RefreshResult", + "RetainedCollectionRefresh", + "TipRefreshRuntime", + "TipRefreshService", + "TipRefreshValidationToken", +] diff --git a/lab/prism/vardiff_service.py b/lab/prism/vardiff_service.py new file mode 100644 index 000000000..627d9d6ce --- /dev/null +++ b/lab/prism/vardiff_service.py @@ -0,0 +1,2538 @@ +"""PRISM vardiff windows, retarget delivery, and bounded idle work.""" + +from __future__ import annotations + +from collections import OrderedDict +from concurrent.futures import Future, ThreadPoolExecutor +from dataclasses import dataclass +from decimal import Decimal +import threading +import time +import traceback +from typing import Any, Callable, Iterable, Protocol + +from lab.auxpow import vardiff +from lab.prism.coordinator_config import ( + DEFAULT_PRISM_VARDIFF_RESUME_MAX_ENTRIES, + DEFAULT_PRISM_VARDIFF_RESUME_MAX_START_FACTOR, + DEFAULT_PRISM_VARDIFF_RESUME_TTL_SECONDS, + StratumListenerProfile, +) +from lab.prism.job_bundle import CachedJobBundle, JobBuildSuperseded +from lab.prism.job_delivery import PrismJobContext +from lab.prism.stratum_session import ( + ClientState, + WorkerIdentity, + client_vardiff_lock, +) +from lab.prism.worker_difficulty_store import ( + MemoryWorkerDifficultyStore, + PostgresWorkerDifficultyStore, + WorkerDifficultyStorePort, +) + + +PRISM_VARDIFF_IDLE_RETARGET_MAX_WORKERS = 2 +MAX_PENDING_VARDIFF_IDLE_RETARGETS = 8 +# Hard bound on the durable persistence lane. Entries coalesce by worker +# identity, so this bounds DISTINCT pending workers, not the event rate. +MAX_PENDING_VARDIFF_DURABLE_WRITES = 256 +PRISM_VARDIFF_DURABLE_PRUNE_INTERVAL_SECONDS = 300.0 +# TTL pruning deletes in bounded batches rather than one unbounded statement, +# so a large inherited backlog cannot produce a single delete that outruns the +# ledger's operation deadline and then never converges. Several batches run per +# pass; whatever is left waits for the next interval. +PRISM_VARDIFF_DURABLE_PRUNE_BATCH = 1024 +PRISM_VARDIFF_DURABLE_PRUNE_MAX_BATCHES = 8 +# How long the lane's worker parks between wake-ups. Work arrives by event; +# this only bounds how long a stop request waits for the loop to notice. +PRISM_VARDIFF_DURABLE_IDLE_WAKE_SECONDS = 1.0 +# Mirrors the coordinator's PRISM_JOB_BUILD_SECONDS_BUCKETS values; owned +# here so metric formatting does not import the coordinator module. +PRISM_VARDIFF_IDLE_SECONDS_BUCKETS = ( + 0.01, + 0.025, + 0.05, + 0.1, + 0.25, + 0.5, + 1.0, + 2.5, + 5.0, + 10.0, + 30.0, +) +PRISM_VARDIFF_IDLE_SKIP_REASONS = ( + "busy", + "disconnected", + "not_idle", + "cache_miss", + "queue_full", + "superseded", +) +# Bounded outcomes for one retarget taken under the fast-arrival initial +# convergence policy; also the metric label order for +# qbit_prism_vardiff_initial_retargets_total. Their sum is the attempt count +# published alongside them. +PRISM_VARDIFF_INITIAL_RETARGET_OUTCOMES = ( + "applied", # committed together with its paired job + "suppressed", # the computed step landed inside the retarget tolerance + "superseded", # difficulty or job state moved before the paired send + "failed", # the build or send raised; speculative state was restored +) +# Seconds and accepted-share buckets for the high-difficulty arrival +# histograms. Fixed bucket sets, no per-worker labels: the whole point is to +# see how long a rental-scale session spends below the threshold it should +# be credited at, fleet-wide. +PRISM_VARDIFF_HIGH_DIFF_ARRIVAL_SECONDS_BUCKETS = ( + 1.0, + 2.5, + 5.0, + 10.0, + 30.0, + 60.0, + 120.0, + 300.0, + 600.0, +) +PRISM_VARDIFF_HIGH_DIFF_ARRIVAL_SHARES_BUCKETS = ( + 1, + 2, + 4, + 8, + 16, + 32, + 64, + 128, + 256, + 512, +) +# Bounded outcomes for one reconnect difficulty resume attempt; also the +# metric label order for qbit_prism_vardiff_resume_total. +PRISM_VARDIFF_RESUME_OUTCOMES = ( + "resumed", # retained value adopted unchanged + "clamped", # retained value adopted after being pulled into bounds + "overridden", # adopted, then superseded by an explicit difficulty request + "expired", # entry existed, TTL had passed + "miss", # no entry for this key + "rejected", # entry present but unusable (non-finite / non-positive) + "disabled", # retention off, or vardiff disabled for this client +) +# Bounded outcomes for one durable worker-difficulty write. The first two are +# evidence-carrying upserts; the next two are the lower-only correction, whose +# "unchanged" means there was no higher stored row left to correct. +PRISM_VARDIFF_DURABLE_WRITE_OUTCOMES = ( + "applied", + "stale", + "lowered", + "unchanged", + "failed", + "dropped", +) + + +@dataclass(frozen=True) +class _PendingDurableWrite: + """One queued durable worker-difficulty write, keyed by worker identity. + + ``downward_only`` selects the atomic lower-only correction, which carries + no evidence timestamp: it reduces an existing row without touching + ``evidence_at``, so it can neither refresh the TTL of a value no share + proved nor refuse a later share-backed write whose evidence predates it. + """ + + key: tuple[str, str] + downward_only: bool + difficulty: Decimal + evidence_at_ms: int | None + now_ms: int + # A coalesced evidence write may also carry the minimum correction that + # must run if the store rejects that evidence as stale. Keeping this + # branch separate is load-bearing: folding it into ``difficulty`` loses + # the correction when a newer row already stands in the store. + stale_downward_difficulty: Decimal | None = None + stale_downward_now_ms: int | None = None + + @property + def has_downward_correction(self) -> bool: + return self.downward_only or self.stale_downward_difficulty is not None + + def _stale_downward(self) -> tuple[Decimal, int] | None: + if self.downward_only: + return (self.difficulty, self.now_ms) + if self.stale_downward_difficulty is None: + return None + return ( + self.stale_downward_difficulty, + int( + self.now_ms + if self.stale_downward_now_ms is None + else self.stale_downward_now_ms + ), + ) + + @staticmethod + def _minimum_downward( + earlier: tuple[Decimal, int] | None, + later: tuple[Decimal, int] | None, + ) -> tuple[Decimal, int] | None: + """Collapse lower-only writes without inventing an update timestamp. + + The first occurrence of the minimum wins: a later equal or higher + correction is a no-op after the earlier minimum has landed. + """ + if earlier is None: + return later + if later is None or earlier[0] <= later[0]: + return earlier + return later + + @classmethod + def _evidence_write( + cls, + source: "_PendingDurableWrite", + *, + difficulty: Decimal | None = None, + now_ms: int | None = None, + stale_downward: tuple[Decimal, int] | None = None, + ) -> "_PendingDurableWrite": + return cls( + key=source.key, + downward_only=False, + difficulty=source.difficulty if difficulty is None else difficulty, + evidence_at_ms=source.evidence_at_ms, + now_ms=source.now_ms if now_ms is None else now_ms, + stale_downward_difficulty=( + None if stale_downward is None else stale_downward[0] + ), + stale_downward_now_ms=( + None if stale_downward is None else stale_downward[1] + ), + ) + + def merged_with(self, later: "_PendingDurableWrite") -> "_PendingDurableWrite": + """Collapse this entry and a later one for the same worker. + + Evidence order, not queue order, selects the upsert that can stand. + Lower-only corrections are retained as a stale-upsert fallback when + an evidence write follows them, because a newer row already in the + store can reject every queued upsert while still needing the lower. + This is a compact representation of both possible store branches: + evidence accepted, or all queued evidence rejected as stale. + """ + if self.key != later.key: + raise ValueError("durable writes for different workers cannot merge") + + if later.downward_only: + if self.downward_only: + minimum = self._minimum_downward( + self._stale_downward(), + later._stale_downward(), + ) + assert minimum is not None + return _PendingDurableWrite( + key=self.key, + downward_only=True, + difficulty=minimum[0], + evidence_at_ms=None, + now_ms=minimum[1], + ) + stale_downward = self._minimum_downward( + self._stale_downward(), + later._stale_downward(), + ) + difficulty = self.difficulty + now_ms = self.now_ms + if later.difficulty < difficulty: + difficulty = later.difficulty + now_ms = later.now_ms + return self._evidence_write( + self, + difficulty=difficulty, + now_ms=now_ms, + stale_downward=stale_downward, + ) + + if self.downward_only: + stale_downward = self._minimum_downward( + self._stale_downward(), + later._stale_downward(), + ) + return self._evidence_write( + later, + stale_downward=stale_downward, + ) + + self_evidence = int(self.evidence_at_ms or 0) + later_evidence = int(later.evidence_at_ms or 0) + if later_evidence >= self_evidence: + stale_downward = self._minimum_downward( + self._stale_downward(), + later._stale_downward(), + ) + return self._evidence_write( + later, + stale_downward=stale_downward, + ) + + # ``later`` carries older evidence and is rejected whenever ``self`` + # applies. If ``later`` already represents a coalesced sequence, only + # its stale-upsert lower branch remains observable after ``self``. + later_stale_downward = later._stale_downward() + stale_downward = self._minimum_downward( + self._stale_downward(), + later_stale_downward, + ) + difficulty = self.difficulty + now_ms = self.now_ms + if ( + later_stale_downward is not None + and later_stale_downward[0] < difficulty + ): + difficulty, now_ms = later_stale_downward + return self._evidence_write( + self, + difficulty=difficulty, + now_ms=now_ms, + stale_downward=stale_downward, + ) + + +@dataclass(frozen=True) +class RetainedSessionDifficulty: + difficulty: Decimal + recorded_monotonic: float + + +def session_difficulty_key(client: ClientState) -> tuple[str, str] | None: + """The retention identity for one session: (listener name, exact username). + + Scoping by lane keeps the two lanes' resume policies independent: a + reconnect to the same port (the reconnect-storm case) always hits, while + a lane switch is simply a miss that behaves exactly like today. ``None`` + means the connection has no reserved worker yet, so nothing is retained. + """ + worker = getattr(client, "worker", None) + if worker is None: + return None + return (str(getattr(client, "listener_name", "default")), worker.username) + + +class SessionDifficultyStore: + """Bounded, TTL'd, in-process map of last-converged share difficulty. + + Keyed by :func:`session_difficulty_key`. The internal lock is a leaf + lock: nothing here calls back into the coordinator or client locks while + holding it. Retention is disabled entirely (``record`` never stores, + ``lookup`` always answers ``(None, "disabled")``) when ``max_entries`` or + ``ttl_seconds`` is non-positive. + + In-memory only: the store does not survive a coordinator restart, + including the liveness watchdog's terminal ``exit_process(1)`` in + ``background_services.py``. The mass-reconnect wave right after such a + restart therefore finds an empty store and every session starts at lane + start, unsmoothed by retention. + """ + + def __init__(self, *, max_entries: int, ttl_seconds: float) -> None: + self.max_entries = int(max_entries) + self.ttl_seconds = float(ttl_seconds) + self._lock = threading.Lock() + self._entries: OrderedDict[tuple[str, str], RetainedSessionDifficulty] = ( + OrderedDict() + ) + self.record_count = 0 + self.hit_count = 0 + self.expired_count = 0 + self.miss_count = 0 + self.evicted_count = 0 + + @property + def enabled(self) -> bool: + return self.max_entries > 0 and self.ttl_seconds > 0 + + def record( + self, + key: tuple[str, str], + difficulty: Decimal, + *, + now: float, + share_backed: bool, + ) -> None: + """Retain ``difficulty`` for ``key``, ageing the entry honestly. + + The stored difficulty is always updated, but ``recorded_monotonic`` + (what the TTL measures) is refreshed only when the value was + genuinely re-validated: a first record, a value that actually moved, + or an unchanged value backed by accepted shares. Re-recording an + unchanged value with no share evidence -- a session that resumed a + retained difficulty, submitted nothing and disconnected -- keeps the + original timestamp, so a reconnect loop shorter than the TTL can no + longer keep a stale value alive forever. LRU recency is independent + of the TTL and still moves on every record, exactly as on ``lookup``. + """ + if not self.enabled: + return + if ( + not isinstance(difficulty, Decimal) + or not difficulty.is_finite() + or difficulty <= 0 + ): + return + with self._lock: + existing = self._entries.get(key) + if ( + existing is None + or difficulty != existing.difficulty + or share_backed + ): + recorded_monotonic = float(now) + else: + recorded_monotonic = existing.recorded_monotonic + self._entries[key] = RetainedSessionDifficulty( + difficulty=difficulty, + recorded_monotonic=recorded_monotonic, + ) + self._entries.move_to_end(key) + self.record_count += 1 + while len(self._entries) > self.max_entries: + self._entries.popitem(last=False) + self.evicted_count += 1 + + def lookup( + self, + key: tuple[str, str], + *, + now: float, + ) -> tuple[Decimal | None, str]: + if not self.enabled: + return None, "disabled" + with self._lock: + retained = self._entries.get(key) + if retained is None: + self.miss_count += 1 + return None, "miss" + if now - retained.recorded_monotonic > self.ttl_seconds: + self._entries.pop(key, None) + self.expired_count += 1 + return None, "expired" + # A hit refreshes LRU recency only. The TTL measures the age of + # the retained value, not of the last access, so + # recorded_monotonic stays untouched; and the entry survives the + # hit because a miner may re-authorize. + self._entries.move_to_end(key) + self.hit_count += 1 + return retained.difficulty, "hit" + + def preload( + self, + entries: Iterable[ + tuple[tuple[str, str], Decimal, float] + ], + ) -> int: + """Seed bounded retained values without counting them as live writes. + + ``recorded_monotonic`` is reconstructed by the durable integration + layer from each wall-clock evidence timestamp. Callers provide oldest + first so the resulting LRU order still has the newest evidence at the + end. Invalid rows are ignored defensively; the durable stores already + validate them on write and parse. + """ + if not self.enabled: + return 0 + loaded = 0 + with self._lock: + for key, difficulty, recorded_monotonic in entries: + if ( + not isinstance(difficulty, Decimal) + or not difficulty.is_finite() + or difficulty <= 0 + ): + continue + self._entries[key] = RetainedSessionDifficulty( + difficulty=difficulty, + recorded_monotonic=float(recorded_monotonic), + ) + self._entries.move_to_end(key) + loaded += 1 + while len(self._entries) > self.max_entries: + self._entries.popitem(last=False) + self.evicted_count += 1 + return loaded + + def prune(self, *, now: float) -> int: + if not self.enabled: + return 0 + with self._lock: + expired_keys = [ + key + for key, retained in self._entries.items() + if now - retained.recorded_monotonic > self.ttl_seconds + ] + for key in expired_keys: + del self._entries[key] + self.expired_count += len(expired_keys) + return len(expired_keys) + + def snapshot(self) -> dict[str, int]: + with self._lock: + return { + "entries": len(self._entries), + "records": self.record_count, + "hits": self.hit_count, + "expired": self.expired_count, + "misses": self.miss_count, + "evicted": self.evicted_count, + } + + def __len__(self) -> int: + with self._lock: + return len(self._entries) + + +@dataclass(frozen=True) +class IdleRetargetRequest: + """Immutable idle-window identity captured by one bounded sweep.""" + + client: ClientState + connection_id: int + worker: WorkerIdentity + active_job: PrismJobContext + window_started_monotonic: float + current_difficulty: Decimal + elapsed_seconds: Decimal + + +class VardiffRuntime(Protocol): + """Coordinator operations needed at vardiff's ownership boundary. + + Adjacent reconcile, retained-job, tip-epoch, payout-fence, and heartbeat + state stays with its owning modules; the service reaches all of it + through these call-time coordinator seams so monkeypatched instances + keep intercepting. + """ + + lock: Any + clients: set[ClientState] + stop_event: threading.Event + vardiff_config: vardiff.VardiffConfig + share_difficulty: Decimal + vardiff_idle_sweep_seconds: float + vardiff_resume_enabled: bool + vardiff_resume_ttl_seconds: float + vardiff_resume_max_entries: int + vardiff_resume_max_start_factor: Decimal + + def client_can_receive_jobs(self, client: ClientState) -> bool: ... + + def _record_heartbeat(self, name: str) -> None: ... + + def _record_vardiff_idle_skip(self, reason: str) -> None: ... + + def _cached_idle_job_bundle(self, client: ClientState) -> CachedJobBundle | None: ... + + def _build_idle_job_bundle( + self, + request: IdleRetargetRequest, + ) -> CachedJobBundle: ... + + def _idle_bundle_current_locked( + self, + client: ClientState, + bundle: CachedJobBundle, + *, + allow_uncached: bool = False, + ) -> bool: ... + + def pool_readiness_latched(self) -> bool: ... + + def ensure_reorg_reconciled_for_current_tip(self) -> bool: ... + + def disconnect_client(self, client: ClientState) -> None: ... + + def _maybe_send_job_locked(self, client: ClientState, **kwargs: object) -> bool: ... + + def maybe_send_job(self, client: ClientState, *, clean_jobs: bool) -> bool: ... + + def retarget_client(self, client: ClientState, **kwargs: object) -> bool: ... + + +class VardiffCompatibilityField: + """Temporary coordinator view over state owned by ``VardiffService``.""" + + def __init__(self, name: str, default: Callable[[], Any]) -> None: + self.name = name + self.default = default + + def __get__(self, instance: Any, owner: type[Any]) -> Any: + if instance is None: + return self + init_lock = instance.__dict__.setdefault( + "_vardiff_service_init_lock", + threading.RLock(), + ) + with init_lock: + service = instance.__dict__.get("_vardiff_service") + if service is not None: + return getattr(service, self.name) + if self.name not in instance.__dict__: + instance.__dict__[self.name] = self.default() + return instance.__dict__[self.name] + + def __set__(self, instance: Any, value: Any) -> None: + init_lock = instance.__dict__.setdefault( + "_vardiff_service_init_lock", + threading.RLock(), + ) + with init_lock: + service = instance.__dict__.get("_vardiff_service") + if service is not None: + setattr(service, self.name, value) + else: + instance.__dict__[self.name] = value + + +VARDIFF_COMPATIBILITY_FIELDS = ( + "idle_retarget_count", + "_vardiff_idle_lock", + "_vardiff_idle_executor", + "_vardiff_idle_executor_shutdown", + "_vardiff_idle_pending", + "vardiff_idle_queue_depth", + "vardiff_idle_inflight", + "vardiff_idle_clients_inspected", + "vardiff_idle_skip_counts", + "vardiff_idle_task_failures", + "vardiff_idle_sweep_histogram", + "vardiff_idle_task_histogram", +) + + +def _new_bucket_histogram(buckets: tuple[float, ...]) -> dict[str, Any]: + return { + "buckets": {bucket: 0 for bucket in buckets}, + "sum": 0.0, + "count": 0, + } + + +def _new_histogram() -> dict[str, Any]: + return _new_bucket_histogram(PRISM_VARDIFF_IDLE_SECONDS_BUCKETS) + + +def _observe_bucket_histogram( + histogram: dict[str, Any], + buckets: tuple[float, ...], + value: float, +) -> None: + """Record one observation. Caller holds the histogram's owning lock.""" + histogram["count"] = int(histogram["count"]) + 1 + histogram["sum"] = float(histogram["sum"]) + float(value) + counts = histogram["buckets"] + for bucket in buckets: + if value <= bucket: + counts[bucket] = int(counts.get(bucket, 0)) + 1 + + +def _histogram_lines( + metric_name: str, + description: str, + histogram: dict[str, Any], + buckets: tuple[float, ...], +) -> list[str]: + return [ + f"# HELP {metric_name} {description}", + f"# TYPE {metric_name} histogram", + *[ + f'{metric_name}_bucket{{le="{bucket:g}"}} {histogram["buckets"].get(bucket, 0)}' + for bucket in buckets + ], + f'{metric_name}_bucket{{le="+Inf"}} {histogram["count"]}', + f'{metric_name}_sum {float(histogram["sum"]):.6f}', + f'{metric_name}_count {histogram["count"]}', + ] + + +def _worker_difficulty_store(runtime: VardiffRuntime) -> WorkerDifficultyStorePort: + """Resolve one process-wide durable-store adapter for the runtime. + + Focused embedders may inject ``worker_difficulty_store`` directly. In + production the adapter is attached to the ledger so recreating the + vardiff service in-process does not discard the memory reference store, + while PostgreSQL remains the cross-process source of truth. + """ + injected = getattr(runtime, "worker_difficulty_store", None) + if injected is not None: + return injected + ledger = getattr(runtime, "ledger", None) + owner = ledger if ledger is not None else runtime + existing = getattr(owner, "_worker_difficulty_store", None) + if existing is not None: + return existing + if ( + ledger is not None + and getattr(ledger, "backend_name", "") == "postgres-psql" + ): + store: WorkerDifficultyStorePort = ( + PostgresWorkerDifficultyStore.from_share_ledger( + ledger, + timeout_seconds=float( + getattr(runtime, "share_commit_timeout_seconds", 15.0) + ), + ) + ) + else: + store = MemoryWorkerDifficultyStore() + setattr(owner, "_worker_difficulty_store", store) + return store + + +class VardiffService: + """Own vardiff client windows and its bounded idle-retarget machinery.""" + + def __init__(self, runtime: VardiffRuntime) -> None: + self.runtime = runtime + self.idle_retarget_count = 0 + self._vardiff_idle_lock = threading.Lock() + self._vardiff_idle_executor: ThreadPoolExecutor | None = None + self._vardiff_idle_executor_shutdown = False + self._vardiff_idle_pending: set[tuple[ClientState, int]] = set() + self.vardiff_idle_queue_depth = 0 + self.vardiff_idle_inflight = 0 + self.vardiff_idle_clients_inspected = 0 + self.vardiff_idle_skip_counts = { + reason: 0 for reason in PRISM_VARDIFF_IDLE_SKIP_REASONS + } + self.vardiff_idle_task_failures = 0 + self.vardiff_idle_sweep_histogram = _new_histogram() + self.vardiff_idle_task_histogram = _new_histogram() + # Reconnect difficulty retention. Sized once from the runtime's + # loaded configuration (getattr keeps lightweight embedder runtimes + # working); vardiff_resume_enabled=False behaves exactly as + # max_entries=0, so the store never records and always answers + # "disabled". + resume_enabled = bool(getattr(runtime, "vardiff_resume_enabled", True)) + self.session_difficulty_store = SessionDifficultyStore( + max_entries=( + int( + getattr( + runtime, + "vardiff_resume_max_entries", + DEFAULT_PRISM_VARDIFF_RESUME_MAX_ENTRIES, + ) + ) + if resume_enabled + else 0 + ), + ttl_seconds=float( + getattr( + runtime, + "vardiff_resume_ttl_seconds", + DEFAULT_PRISM_VARDIFF_RESUME_TTL_SECONDS, + ) + ), + ) + # Leaf lock for the convergence counters below (resume outcomes and + # per-lane accepted shares); never call out of this module while + # holding it, and never take it under the store's lock. + self._vardiff_convergence_lock = threading.Lock() + self.vardiff_resume_outcome_counts = { + outcome: 0 for outcome in PRISM_VARDIFF_RESUME_OUTCOMES + } + self.vardiff_lane_accepted_counts: dict[str, int] = {} + self.vardiff_initial_retarget_attempts = 0 + self.vardiff_initial_retarget_outcome_counts = { + outcome: 0 for outcome in PRISM_VARDIFF_INITIAL_RETARGET_OUTCOMES + } + self.vardiff_high_diff_arrival_seconds_histogram = _new_bucket_histogram( + PRISM_VARDIFF_HIGH_DIFF_ARRIVAL_SECONDS_BUCKETS + ) + self.vardiff_high_diff_arrival_shares_histogram = _new_bucket_histogram( + PRISM_VARDIFF_HIGH_DIFF_ARRIVAL_SHARES_BUCKETS + ) + self.worker_difficulty_store = _worker_difficulty_store(runtime) + self.vardiff_durable_preload_records = 0 + self.vardiff_durable_preload_failures = 0 + self.vardiff_durable_prune_failures = 0 + self.vardiff_durable_pruned_records = 0 + self.vardiff_durable_coalesced = 0 + self.vardiff_durable_downward_dropped = 0 + self.vardiff_durable_write_outcome_counts = { + outcome: 0 for outcome in PRISM_VARDIFF_DURABLE_WRITE_OUTCOMES + } + # Durable writes leave the Stratum/client lock path through this + # bounded, coalescing, single-worker lane. Nothing here reads client + # state or waits on a client lock: every payload is snapshotted by the + # caller, so a slow database can never reach back into share + # accounting or job delivery. The queue is capped by DISTINCT worker + # identity, and safety-critical downward corrections outrank optional + # resume hints for both eviction and shutdown drain order. + self._vardiff_durable_lock = threading.Lock() + self._vardiff_durable_pending: OrderedDict[ + tuple[str, str], _PendingDurableWrite + ] = OrderedDict() + self._vardiff_durable_thread: threading.Thread | None = None + self._vardiff_durable_wake = threading.Event() + self._vardiff_durable_stopping = False + self._vardiff_durable_prune_requested = False + self._vardiff_durable_inflight = 0 + self._vardiff_durable_operation_timeout_seconds = float( + getattr(runtime, "share_commit_timeout_seconds", 15.0) + ) + self._vardiff_durable_next_prune_monotonic = ( + time.monotonic() + PRISM_VARDIFF_DURABLE_PRUNE_INTERVAL_SECONDS + ) + self._preload_worker_difficulties() + + def _preload_worker_difficulties(self) -> None: + """Hydrate the authorization-time map with one bounded durable read.""" + store = self.session_difficulty_store + if not store.enabled: + return + now_ms = max(0, int(time.time() * 1000)) + ttl_ms = max(0, int(store.ttl_seconds * 1000)) + cutoff_ms = max(0, now_ms - ttl_ms) + try: + recent = self.worker_difficulty_store.load_recent( + evidence_after_ms=cutoff_ms, + limit=store.max_entries, + ) + except Exception: + self.vardiff_durable_preload_failures += 1 + return + now_monotonic = time.monotonic() + # load_recent is newest-first; preload oldest-first so LRU recency + # ends in the same order as evidence freshness. + entries = [ + ( + (record.listener, record.worker_username), + record.difficulty, + now_monotonic + - max(0.0, (now_ms - record.evidence_at_ms) / 1000.0), + ) + for record in reversed(recent) + ] + self.vardiff_durable_preload_records = store.preload(entries) + # Retention is an optimization over the safe cold-start path, so a + # prune failure must not prevent mining or discard the records already + # loaded into the process-local map; the durable lane's own scheduler + # retries it on the ordinary interval. + self.vardiff_durable_pruned_records = self._prune_expired_rows(cutoff_ms) + + def client_config(self, client: ClientState) -> vardiff.VardiffConfig: + """The difficulty policy for one client: its per-client specialization + if any, else its listener profile, else the default listener's config + (clients created without one: tests, legacy callers).""" + with client_vardiff_lock(client): + return ( + client.vardiff_config + or client.listener_vardiff_config + or self.runtime.vardiff_config + ) + + def startup_difficulty( + self, + profile: StratumListenerProfile | None = None, + ) -> Decimal: + config = ( + profile.vardiff_config + if profile is not None + else self.runtime.vardiff_config + ) + fixed_difficulty = ( + profile.share_difficulty + if profile is not None + else self.runtime.share_difficulty + ) + if not config.enabled: + return fixed_difficulty + return vardiff.clamp( + config.startup_difficulty, + config.min_difficulty, + config.max_difficulty, + ) + + @staticmethod + def desired_difficulty(client: ClientState) -> Decimal: + # pending_share_difficulty is set by vardiff retargets and by explicit + # difficulty requests (d=/suggest_difficulty); either way it applies to + # the next stamped job regardless of whether vardiff is enabled. + with client_vardiff_lock(client): + return client.pending_share_difficulty or client.share_difficulty + + def minimum_advertised_difficulty(self, client: ClientState) -> Decimal: + """The difficulty stamped jobs never advertise below for this client. + + Zero everywhere except floor-bearing listeners (the high-diff port), + where the effective policy floor governs: the listener minimum, raised + by any md= specialization. The floor overrides the network-difficulty + cap because the listener's marketplace contract is checked against the + first advertised difficulty, even while qbit network difficulty sits + below the floor. + """ + with client_vardiff_lock(client): + if client.minimum_advertised_difficulty <= 0: + return Decimal("0") + config = ( + client.vardiff_config + or client.listener_vardiff_config + or self.runtime.vardiff_config + ) + return max(client.minimum_advertised_difficulty, config.min_difficulty) + + def note_submitted(self, client: ClientState) -> None: + with client_vardiff_lock(client): + config = ( + client.vardiff_config + or client.listener_vardiff_config + or self.runtime.vardiff_config + ) + if not config.enabled: + return + client.vardiff_window_submitted += 1 + + def note_accepted(self, client: ClientState, share_difficulty: Decimal) -> None: + now = time.monotonic() + # Lane share-rate accounting must stay observable even where vardiff + # is off, so count before the enabled early-out below. + lane = str(getattr(client, "listener_name", "default")) + with self._vardiff_convergence_lock: + self.vardiff_lane_accepted_counts[lane] = ( + self.vardiff_lane_accepted_counts.get(lane, 0) + 1 + ) + with client_vardiff_lock(client): + # Per-connection evidence for reconnect retention, set before the + # enabled early-out so it holds where vardiff is disabled too. + # Never reset within a connection: one accepted share is enough to + # prove the difficulty this session is running at is real, which + # is what lets disconnect refresh the retained value's TTL. + client.vardiff_accepted_any = True + client.vardiff_last_accepted_difficulty = share_difficulty + client.vardiff_last_accepted_wall_ms = max( + 0, + int(time.time() * 1000), + ) + config = ( + client.vardiff_config + or client.listener_vardiff_config + or self.runtime.vardiff_config + ) + if not config.enabled: + return + client.vardiff_window_accepted += 1 + client.vardiff_window_work += share_difficulty + client.vardiff_session_accepted = int( + getattr(client, "vardiff_session_accepted", 0) + ) + 1 + elapsed_seconds = Decimal( + str(max(0.001, now - client.vardiff_window_started_monotonic)) + ) + current_difficulty = ( + client.pending_share_difficulty or client.share_difficulty + ) + # Fast arrival: a session that has not yet had a retarget commit + # may cross the ordinary interval early, but only on decisive + # evidence. Both the early trigger and the larger step bound come + # from the same predicate, so a session can never take the wider + # step on evidence that was not good enough to trigger it. + initial_evaluation = bool( + getattr(client, "vardiff_initial_convergence_pending", True) + and not getattr( + client, + "vardiff_initial_convergence_evaluated", + False, + ) + and client.vardiff_window_accepted + >= config.initial_min_accepted_shares + and elapsed_seconds >= config.initial_min_elapsed_seconds + ) + if initial_evaluation: + client.vardiff_initial_convergence_evaluated = True + initial_convergence = initial_evaluation and vardiff.initial_convergence_ready( + current_difficulty=current_difficulty, + accepted_shares=client.vardiff_window_accepted, + accepted_difficulty=client.vardiff_window_work, + elapsed_seconds=elapsed_seconds, + config=config, + ) + if ( + elapsed_seconds < config.retarget_interval_seconds + and not initial_convergence + ): + return + accepted_shares = client.vardiff_window_accepted + submitted_shares = client.vardiff_window_submitted + accepted_difficulty = client.vardiff_window_work + client.vardiff_window_started_monotonic = now + client.vardiff_window_accepted = 0 + client.vardiff_window_submitted = 0 + client.vardiff_window_work = Decimal("0") + applied = False + try: + applied = bool( + self.runtime.retarget_client( + client, + current_difficulty=current_difficulty, + accepted_shares=accepted_shares, + submitted_shares=submitted_shares, + accepted_difficulty=accepted_difficulty, + elapsed_seconds=elapsed_seconds, + initial_convergence=initial_convergence, + ) + ) + finally: + if initial_convergence and not applied: + with client_vardiff_lock(client): + if getattr( + client, + "vardiff_initial_convergence_pending", + True, + ): + client.vardiff_initial_convergence_evaluated = False + + def _count_resume_outcome(self, outcome: str) -> None: + if outcome not in PRISM_VARDIFF_RESUME_OUTCOMES: + raise ValueError(f"unknown vardiff resume outcome: {outcome}") + with self._vardiff_convergence_lock: + self.vardiff_resume_outcome_counts[outcome] += 1 + + def _count_initial_retarget_attempt(self) -> None: + with self._vardiff_convergence_lock: + self.vardiff_initial_retarget_attempts += 1 + + def _count_initial_retarget_outcome(self, outcome: str) -> None: + if outcome not in PRISM_VARDIFF_INITIAL_RETARGET_OUTCOMES: + raise ValueError(f"unknown vardiff initial retarget outcome: {outcome}") + with self._vardiff_convergence_lock: + self.vardiff_initial_retarget_outcome_counts[outcome] += 1 + + def observe_high_diff_arrival( + self, + client: ClientState, + *, + config: vardiff.VardiffConfig, + previous_difficulty: Decimal, + next_difficulty: Decimal, + ) -> bool: + """Observe this connection's first vardiff crossing of the high-diff + threshold, in seconds and in accepted shares. + + Observation only -- the threshold never gates or clamps a retarget. + Recorded at most once per connection, and only for a retarget that + carried the session from below the threshold to at or above it, so a + lane that already starts above it (the high-diff listener) never + contributes and a step-down followed by a re-crossing does not + double count. Returns whether an observation was recorded. + + The client's vardiff lock is released before the counter lock is + taken: this module never nests the two in either direction. + """ + threshold = getattr( + config, + "high_diff_arrival_threshold", + Decimal("0"), + ) + if ( + not threshold.is_finite() + or threshold <= 0 + or previous_difficulty >= threshold + or next_difficulty < threshold + ): + return False + with client_vardiff_lock(client): + if getattr(client, "vardiff_high_diff_arrival_recorded", False): + return False + client.vardiff_high_diff_arrival_recorded = True + started = getattr(client, "vardiff_session_started_monotonic", None) + accepted = int(getattr(client, "vardiff_session_accepted", 0)) + with self._vardiff_convergence_lock: + if started is not None: + _observe_bucket_histogram( + self.vardiff_high_diff_arrival_seconds_histogram, + PRISM_VARDIFF_HIGH_DIFF_ARRIVAL_SECONDS_BUCKETS, + max(0.0, time.monotonic() - float(started)), + ) + _observe_bucket_histogram( + self.vardiff_high_diff_arrival_shares_histogram, + PRISM_VARDIFF_HIGH_DIFF_ARRIVAL_SHARES_BUCKETS, + float(accepted), + ) + return True + + def record_session_difficulty( + self, + client: ClientState, + difficulty: Decimal | None = None, + *, + share_backed: bool, + previous_difficulty: Decimal | None = None, + initial_convergence: bool = False, + ) -> None: + """Retain a session's converged difficulty for reconnect resume. + + Callers pass the just-committed retarget difficulty explicitly; the + disconnect seam omits it and the client's DELIVERED difficulty is + read under its vardiff lock instead. pending_share_difficulty is + deliberately not consulted here: it is advertised with a future job, + so a disconnect racing an in-flight retarget would otherwise retain a + value the miner was never given. Nothing is lost -- a retarget whose + paired send completed records its own value at that commit point. + + ``share_backed`` says whether this difficulty is backed by accepted + shares; the process-local store uses it to decide whether re-recording + an unchanged value may refresh its TTL. Durable writes are stricter: + a committed retarget may carry forward the share evidence that drove + its estimate, while a disconnect is durable only when the last + accepted share was stamped at the exact delivered difficulty. This + prevents an unproven explicit step-up from surviving a restart. + + A committed retarget that LOWERS the difficulty is durable even with + no accepted shares behind it, but it goes out as an atomic lower-only + correction rather than a write of new evidence: leaving a stale higher + row standing would let a restart resurrect a difficulty the live + session had already abandoned, while re-stamping evidence_at would let + the correction suppress later genuine share-backed evidence. + """ + key = session_difficulty_key(client) + if key is None: + return + if not self.session_difficulty_store.enabled: + return + committed_retarget = difficulty is not None + with client_vardiff_lock(client): + if difficulty is None: + difficulty = client.share_difficulty + evidence_difficulty = getattr( + client, + "vardiff_last_accepted_difficulty", + None, + ) + evidence_at_ms = getattr( + client, + "vardiff_last_accepted_wall_ms", + None, + ) + self.session_difficulty_store.record( + key, + difficulty, + now=time.monotonic(), + share_backed=share_backed, + ) + now_ms = max(0, int(time.time() * 1000)) + downward_commit = bool( + committed_retarget + and previous_difficulty is not None + and difficulty < previous_difficulty + ) + durably_share_backed = bool( + share_backed + and evidence_at_ms is not None + and ( + (committed_retarget and not initial_convergence) + or evidence_difficulty == difficulty + ) + ) + # A share-backed write already carries this exact difficulty plus the + # evidence that proves it, so it subsumes the downward correction; the + # lower-only path is for the moves no accepted share backs, above all + # the idle step-down. + if durably_share_backed: + self._enqueue_durable_write( + _PendingDurableWrite( + key=key, + downward_only=False, + difficulty=difficulty, + evidence_at_ms=int(evidence_at_ms), + now_ms=now_ms, + ) + ) + elif downward_commit: + self._enqueue_durable_write( + _PendingDurableWrite( + key=key, + downward_only=True, + difficulty=difficulty, + evidence_at_ms=None, + now_ms=now_ms, + ) + ) + + def _count_durable_write(self, outcome: str) -> None: + if outcome not in PRISM_VARDIFF_DURABLE_WRITE_OUTCOMES: + raise ValueError(f"unknown vardiff durable write outcome: {outcome}") + with self._vardiff_convergence_lock: + self.vardiff_durable_write_outcome_counts[outcome] += 1 + + # -- bounded coalescing persistence lane ------------------------------- + + def _enqueue_durable_write(self, pending: _PendingDurableWrite) -> None: + """Hand one durable write to the lane. Never touches a client lock. + + The payload is fully snapshotted by the caller, so nothing here reads + client state and nothing waits on ``job_update_lock``: a slow database + can never reach back into share accounting or job delivery. + + Entries coalesce by worker identity, which is what keeps the queue + bounded by DISTINCT workers rather than by event rate. The merge is + order-equivalent to applying both writes in sequence -- see + :meth:`_PendingDurableWrite.merged_with`. + + After the lane is closed a step-down is applied on the calling thread + rather than queued, because the shutdown drain has already run and a + queued entry would simply be stranded. + """ + lost: _PendingDurableWrite | None = None + apply_inline = False + with self._vardiff_durable_lock: + if self._vardiff_durable_stopping: + # The lane is closed: its drain has already run, so queueing + # here would strand the entry. An optional resume hint is + # dropped, but a step-down is safety critical -- losing it + # leaves a restart resuming an abandoned difficulty -- so it + # is applied on this thread instead, bounded by the store's + # own per-statement operation deadline. + if pending.has_downward_correction: + apply_inline = True + else: + lost = pending + else: + existing = self._vardiff_durable_pending.get(pending.key) + if existing is not None: + self._vardiff_durable_pending[pending.key] = ( + existing.merged_with(pending) + ) + self.vardiff_durable_coalesced += 1 + else: + lost = self._evict_for_locked(pending) + # _evict_for_locked returns the incoming entry itself when + # it is the one refused, in which case it must not also be + # queued -- that would push the lane past its bound. + if lost is not pending: + self._vardiff_durable_pending[pending.key] = pending + self._ensure_durable_worker_locked() + if apply_inline: + # Outside the lane lock: _apply_durable_write reaches the store. + self._apply_durable_write(pending) + return + if lost is not None: + if lost.has_downward_correction: + self._count_downward_drop(lost) + else: + self._count_durable_write("dropped") + self._vardiff_durable_wake.set() + + def _evict_for_locked( + self, + incoming: _PendingDurableWrite, + ) -> _PendingDurableWrite | None: + """Make room for ``incoming`` under the hard queue bound. + + Caller holds ``_vardiff_durable_lock``. Optional resume hints are + evicted before safety-critical downward corrections, and the incoming + entry itself is refused only when every queued entry is already a + downward correction -- which needs MAX_PENDING distinct workers + stepping down while the database is unavailable. + """ + if len(self._vardiff_durable_pending) < MAX_PENDING_VARDIFF_DURABLE_WRITES: + return None + for key, queued in self._vardiff_durable_pending.items(): + if not queued.has_downward_correction: + del self._vardiff_durable_pending[key] + return queued + if not incoming.has_downward_correction: + return incoming + oldest_key = next(iter(self._vardiff_durable_pending)) + return self._vardiff_durable_pending.pop(oldest_key) + + def _count_downward_drop(self, pending: _PendingDurableWrite) -> None: + """Record -- loudly -- a downward correction the lane could not keep. + + This is the one durable-write loss that can leave a restart resuming a + difficulty the live session abandoned, so it gets its own counter and + a log line instead of folding into the generic drop count. + """ + self._count_durable_write("dropped") + with self._vardiff_convergence_lock: + self.vardiff_durable_downward_dropped += 1 + downward = pending._stale_downward() + assert downward is not None + print( + "prism coordinator: durable vardiff step-down dropped " + f"listener={pending.key[0]} worker={pending.key[1]} " + f"difficulty={downward[0]} " + "(reconnect may resume a higher difficulty until its TTL expires)", + flush=True, + ) + + def _ensure_durable_worker_locked(self) -> None: + """Start the lane's single worker thread. Caller holds the lane lock.""" + if self._vardiff_durable_thread is not None: + return + if self._vardiff_durable_stopping: + return + thread = threading.Thread( + target=self._durable_worker_loop, + name="prism-vardiff-durable", + daemon=True, + ) + self._vardiff_durable_thread = thread + thread.start() + + def start_durable_prune_scheduler(self) -> None: + """Keep TTL pruning alive even when idle vardiff retargets are off. + + Production calls this once before accepting Stratum connections. The + existing durable lane then owns both write traffic and the periodic + prune clock, without adding database work to an accept/client thread. + Disabled retention still creates no worker. + """ + if not self.session_difficulty_store.enabled: + return + with self._vardiff_durable_lock: + self._ensure_durable_worker_locked() + + def _durable_worker_loop(self) -> None: + while True: + self._vardiff_durable_wake.wait( + PRISM_VARDIFF_DURABLE_IDLE_WAKE_SECONDS + ) + self._vardiff_durable_wake.clear() + with self._vardiff_durable_lock: + stopping = self._vardiff_durable_stopping + if not stopping: + # The lane owns its own clock. In particular, this must not + # depend on vardiff_idle_sweep_seconds: operators may disable + # idle retargets without disabling durable-row retention. + self.request_durable_prune_if_due() + self._drain_durable_pending( + deadline=None if not stopping else ( + time.monotonic() + self._vardiff_durable_operation_timeout_seconds + ), + final=stopping, + ) + self._prune_durable_worker_difficulties_if_requested() + if stopping: + return + + def _next_pending_locked(self) -> _PendingDurableWrite | None: + """Pop the next entry, safety-critical corrections first. + + Caller holds ``_vardiff_durable_lock``. Downward-first ordering is + what lets a deadline-bounded shutdown drain finish the writes that + matter even when optional resume hints are still queued behind them. + """ + chosen_key: tuple[str, str] | None = None + for key, queued in self._vardiff_durable_pending.items(): + if queued.has_downward_correction: + chosen_key = key + break + if chosen_key is None: + chosen_key = next(iter(self._vardiff_durable_pending), None) + if chosen_key is None: + return None + return self._vardiff_durable_pending.pop(chosen_key) + + def _drain_durable_pending( + self, + *, + deadline: float | None, + final: bool, + ) -> None: + """Apply queued writes until the queue empties or ``deadline`` passes. + + ``deadline`` bounds the whole drain with the same budget the ledger + gives one operation; each individual statement is separately bounded + by the store's own operation timeout. On a final drain anything still + queued when the budget runs out is accounted for explicitly. + """ + while True: + if deadline is not None and time.monotonic() >= deadline: + break + with self._vardiff_durable_lock: + pending = self._next_pending_locked() + # Counted as in-flight while the lane lock is held, so a + # concurrent flush cannot observe an empty queue between the + # pop and the write landing. + self._vardiff_durable_inflight += 1 if pending is not None else 0 + if pending is None: + return + try: + self._apply_durable_write(pending) + finally: + with self._vardiff_durable_lock: + self._vardiff_durable_inflight -= 1 + if not final: + return + with self._vardiff_durable_lock: + abandoned = list(self._vardiff_durable_pending.values()) + self._vardiff_durable_pending.clear() + for pending in abandoned: + if pending.has_downward_correction: + self._count_downward_drop(pending) + else: + self._count_durable_write("dropped") + + def _apply_durable_write(self, pending: _PendingDurableWrite) -> None: + if pending.downward_only: + try: + lowered = self.worker_difficulty_store.apply_downward( + listener=pending.key[0], + worker_username=pending.key[1], + difficulty=pending.difficulty, + now_ms=pending.now_ms, + ) + outcome = "lowered" if lowered.applied else "unchanged" + except Exception: + outcome = "failed" + self._count_durable_write(outcome) + return + + try: + result = self.worker_difficulty_store.upsert( + listener=pending.key[0], + worker_username=pending.key[1], + difficulty=pending.difficulty, + evidence_at_ms=int(pending.evidence_at_ms or 0), + now_ms=pending.now_ms, + ) + except Exception: + self._count_durable_write("failed") + return + self._count_durable_write("applied" if result.applied else "stale") + stale_downward = pending._stale_downward() + if result.applied or stale_downward is None: + return + try: + lowered = self.worker_difficulty_store.apply_downward( + listener=pending.key[0], + worker_username=pending.key[1], + difficulty=stale_downward[0], + now_ms=stale_downward[1], + ) + outcome = "lowered" if lowered.applied else "unchanged" + except Exception: + outcome = "failed" + self._count_durable_write(outcome) + + def request_durable_prune_if_due(self) -> bool: + """Arm a TTL prune when one is due, without doing database work here. + + Called from the durable worker's own scheduler, so pruning is driven + by the clock rather than by successful write traffic or the optional + idle-retarget sweep. A lane whose writes all fail, or a process that + inherits rows and writes none of its own, still prunes. Returns + whether a prune was armed. + """ + if not self.session_difficulty_store.enabled: + return False + now_monotonic = time.monotonic() + with self._vardiff_durable_lock: + if now_monotonic < self._vardiff_durable_next_prune_monotonic: + return False + self._vardiff_durable_next_prune_monotonic = ( + now_monotonic + PRISM_VARDIFF_DURABLE_PRUNE_INTERVAL_SECONDS + ) + if self._vardiff_durable_stopping: + return False + self._vardiff_durable_prune_requested = True + self._ensure_durable_worker_locked() + self._vardiff_durable_wake.set() + return True + + def _prune_expired_rows(self, cutoff_ms: int) -> int: + """Delete expired rows in bounded batches; never one open-ended DELETE. + + Every statement carries a finite ``limit`` so it stays inside the + ledger's operation deadline, and batching continues only while a full + batch comes back, so an inherited backlog drains over successive + intervals instead of stalling on a delete it can never finish. + """ + pruned = 0 + for _ in range(PRISM_VARDIFF_DURABLE_PRUNE_MAX_BATCHES): + try: + deleted = self.worker_difficulty_store.prune( + evidence_cutoff_ms=cutoff_ms, + limit=PRISM_VARDIFF_DURABLE_PRUNE_BATCH, + ) + except Exception: + with self._vardiff_convergence_lock: + self.vardiff_durable_prune_failures += 1 + break + pruned += int(deleted) + if int(deleted) < PRISM_VARDIFF_DURABLE_PRUNE_BATCH: + break + return pruned + + def _prune_durable_worker_difficulties_if_requested(self) -> None: + with self._vardiff_durable_lock: + if not self._vardiff_durable_prune_requested: + return + self._vardiff_durable_prune_requested = False + now_ms = max(0, int(time.time() * 1000)) + cutoff_ms = max( + 0, + now_ms - int(self.session_difficulty_store.ttl_seconds * 1000), + ) + pruned = self._prune_expired_rows(cutoff_ms) + if not pruned: + return + with self._vardiff_convergence_lock: + self.vardiff_durable_pruned_records += pruned + + def durable_pending_depth(self) -> int: + with self._vardiff_durable_lock: + return len(self._vardiff_durable_pending) + + def durable_coalesced_count(self) -> int: + """Coalesced-entry count, read under the lane lock that maintains it.""" + with self._vardiff_durable_lock: + return int(self.vardiff_durable_coalesced) + + def flush_durable_writes(self, *, timeout: float | None = None) -> bool: + """Wait until the lane has nothing queued or in flight. + + No production caller needs this -- durable retention is deliberately + fire-and-forget so mining never waits on it. It exists so tests and + the shutdown drain can observe the lane deterministically. Returns + whether the lane went quiet within the budget. + """ + budget = ( + self._vardiff_durable_operation_timeout_seconds + if timeout is None + else float(timeout) + ) + deadline = time.monotonic() + max(0.0, budget) + self._vardiff_durable_wake.set() + while True: + with self._vardiff_durable_lock: + quiet = ( + not self._vardiff_durable_pending + and self._vardiff_durable_inflight == 0 + ) + worker_running = self._vardiff_durable_thread is not None + if quiet: + return True + if not worker_running: + # Nothing will drain this; do it on the caller's thread rather + # than spin until the budget expires. + self._drain_durable_pending(deadline=deadline, final=False) + continue + if time.monotonic() >= deadline: + return False + time.sleep(0.001) + + def shutdown_durable_executor(self) -> None: + """Close the lane, draining safety-critical corrections first. + + Queued work is drained, never cancelled: the whole point of the + downward correction is that losing it leaves a restart resuming an + abandoned difficulty. The drain is bounded by the ledger's operation + timeout so shutdown cannot stall on an unreachable database, and + anything the budget cannot cover is counted and logged. + """ + with self._vardiff_durable_lock: + already_stopped = self._vardiff_durable_stopping + self._vardiff_durable_stopping = True + thread = self._vardiff_durable_thread + self._vardiff_durable_thread = None + if already_stopped and thread is None: + return + deadline = ( + time.monotonic() + self._vardiff_durable_operation_timeout_seconds + ) + self._vardiff_durable_wake.set() + if thread is not None: + thread.join(timeout=max(0.0, deadline - time.monotonic())) + # The worker owns the drain while it lives; this covers a lane whose + # worker never started, already exited, or ran out of budget. + self._drain_durable_pending(deadline=deadline, final=True) + + def note_resume_overridden(self) -> None: + """Count a resumed difficulty that an explicit request superseded. + + resumed/clamped stay attempt counters, so + ``resumed + clamped - overridden`` is the number of resumes that + actually stuck. The caller is the authorize path: it is the only + place that knows both the value the resume applied and the target a + password ``d=``/``md=`` or a pending suggestion resolved to. + """ + self._count_resume_outcome("overridden") + + def resume_client_difficulty(self, client: ClientState) -> Decimal | None: + """The difficulty a reconnecting worker should resume at, or None. + + A retained value is only adopted inside the lane's plausibility + bounds: the ceiling is min(lane max, lane start * resume factor), so + a stale or absurd entry can never be adopted as-is. A retained value + BELOW the lane start is adopted unchanged on purpose -- vardiff + targets a share rate, so a session at its converged difficulty is on + target by definition, and cold-starting it higher would give the pool + fewer shares, not more. + """ + with client_vardiff_lock(client): + config = ( + client.vardiff_config + or client.listener_vardiff_config + or self.runtime.vardiff_config + ) + if not config.enabled: + self._count_resume_outcome("disabled") + return None + key = session_difficulty_key(client) + if key is None: + return None + retained, outcome = self.session_difficulty_store.lookup( + key, + now=time.monotonic(), + ) + if retained is None: + self._count_resume_outcome(outcome) + return None + if not retained.is_finite() or retained <= 0: + self._count_resume_outcome("rejected") + return None + factor = getattr( + self.runtime, + "vardiff_resume_max_start_factor", + DEFAULT_PRISM_VARDIFF_RESUME_MAX_START_FACTOR, + ) + ceiling = min(config.max_difficulty, config.startup_difficulty * factor) + floor = config.min_difficulty + resumed = vardiff.clamp(retained, floor, max(floor, ceiling)) + self._count_resume_outcome( + "resumed" if resumed == retained else "clamped" + ) + return resumed + + def apply_resumed_difficulty(self, client: ClientState) -> Decimal | None: + """Adopt the retained difficulty onto a freshly authorizing client.""" + with client_vardiff_lock(client): + resumed = self.resume_client_difficulty(client) + if resumed is None: + return None + current = client.pending_share_difficulty or client.share_difficulty + if resumed != current: + client.share_difficulty = resumed + client.pending_share_difficulty = None + client.difficulty_generation = int( + getattr(client, "difficulty_generation", 0) + ) + 1 + return resumed + + def convergence_snapshot(self) -> dict[str, object]: + """Copied convergence/retention counters plus the at-ceiling census. + + Lock order in this codebase is client vardiff lock -> coordinator + lock, never the reverse; membership is therefore snapshotted under + the coordinator lock alone, released, and only then is each client's + vardiff lock taken to read its effective policy and difficulty. + + The at-ceiling census is therefore O(connections) lock acquisitions + per call, bounded by the deployed 4096-connection cap. That cost is + acceptable because this runs on the background metrics refresher and + /metrics serves its cached payload, and because each vardiff lock is + held for a few field reads only -- no client vardiff lock is ever + held across a socket write, so a slow miner cannot convoy the census. + """ + with self.runtime.lock: + clients = tuple(self.runtime.clients) + sessions_at_max = 0 + for client in clients: + with client_vardiff_lock(client): + config = ( + client.vardiff_config + or client.listener_vardiff_config + or self.runtime.vardiff_config + ) + if not config.enabled: + continue + current = ( + client.pending_share_difficulty or client.share_difficulty + ) + if current >= config.max_difficulty: + sessions_at_max += 1 + with self._vardiff_convergence_lock: + lane_accepted = dict(self.vardiff_lane_accepted_counts) + resume_outcomes = dict(self.vardiff_resume_outcome_counts) + durable_writes = dict(self.vardiff_durable_write_outcome_counts) + durable_pruned = int(self.vardiff_durable_pruned_records) + durable_prune_failures = int(self.vardiff_durable_prune_failures) + durable_downward_dropped = int(self.vardiff_durable_downward_dropped) + # Prune first so the retained-sessions gauge counts only entries a + # reconnect could still adopt. + self.session_difficulty_store.prune(now=time.monotonic()) + store = self.session_difficulty_store.snapshot() + return { + "sessions_at_max_difficulty": sessions_at_max, + "lane_accepted_shares": lane_accepted, + "resume_outcomes": resume_outcomes, + "retained_sessions": int(store["entries"]), + "store": store, + "durable_store": { + "preloaded": self.vardiff_durable_preload_records, + "preload_failures": self.vardiff_durable_preload_failures, + "pruned": durable_pruned, + "prune_failures": durable_prune_failures, + "pending": self.durable_pending_depth(), + "coalesced": self.durable_coalesced_count(), + "downward_dropped": durable_downward_dropped, + "writes": durable_writes, + }, + } + + def record_idle_skip(self, reason: str) -> None: + if reason not in PRISM_VARDIFF_IDLE_SKIP_REASONS: + raise ValueError(f"unknown vardiff idle skip reason: {reason}") + with self._vardiff_idle_lock: + self.vardiff_idle_skip_counts[reason] += 1 + + def observe_idle_seconds(self, name: str, elapsed_seconds: float) -> None: + if name not in {"sweep", "task"}: + raise ValueError(f"unknown vardiff idle histogram: {name}") + with self._vardiff_idle_lock: + histogram = getattr(self, f"vardiff_idle_{name}_histogram") + histogram["count"] = int(histogram["count"]) + 1 + histogram["sum"] = float(histogram["sum"]) + elapsed_seconds + buckets = histogram["buckets"] + for bucket in PRISM_VARDIFF_IDLE_SECONDS_BUCKETS: + if elapsed_seconds <= bucket: + buckets[bucket] = int(buckets.get(bucket, 0)) + 1 + + def idle_tip_diverged_locked(self) -> bool: + """Whether detected tip work still lacks published submit authority.""" + published = getattr(self.runtime, "current_tip_first_seen", None) + latest_detected = getattr(self.runtime, "latest_detected_tip", None) + return bool( + latest_detected is not None + and (published is None or latest_detected[0] != published[0]) + ) + + def request_skip_reason( + self, + request: IdleRetargetRequest, + ) -> str | None: + client = request.client + # Take the per-client lock before coordinator admission. A share can + # delay this client's idle retarget, but it can never make the retarget + # hold the coordinator lock while waiting and convoy tip publication. + with client_vardiff_lock(client): + with self.runtime.lock: + if self.idle_tip_diverged_locked(): + return "superseded" + if ( + client not in self.runtime.clients + or getattr(client, "closing", False) + or not self.runtime.client_can_receive_jobs(client) + ): + return "disconnected" + if ( + client.connection_id != request.connection_id + or client.worker != request.worker + or client.active_job is not request.active_job + or (client.pending_share_difficulty or client.share_difficulty) + != request.current_difficulty + ): + return "superseded" + if ( + client.vardiff_window_started_monotonic + != request.window_started_monotonic + or client.vardiff_window_accepted != 0 + or client.vardiff_window_submitted != 0 + ): + return "not_idle" + return None + + def _request_pending(self, request: IdleRetargetRequest) -> bool: + with self._vardiff_idle_lock: + return ( + request.client, + request.connection_id, + ) in self._vardiff_idle_pending + + def _finish_idle_task( + self, + key: tuple[ClientState, int], + queued_monotonic: float, + *, + started: bool, + ) -> None: + with self._vardiff_idle_lock: + if key not in self._vardiff_idle_pending: + return + self._vardiff_idle_pending.discard(key) + if started: + self.vardiff_idle_inflight = max(0, self.vardiff_idle_inflight - 1) + else: + self.vardiff_idle_queue_depth = max( + 0, + self.vardiff_idle_queue_depth - 1, + ) + self.observe_idle_seconds( + "task", + max(0.0, time.monotonic() - queued_monotonic), + ) + + def _run_idle_task( + self, + request: IdleRetargetRequest, + bundle: CachedJobBundle | None, + queued_monotonic: float, + ) -> None: + key = (request.client, request.connection_id) + with self._vardiff_idle_lock: + if key not in self._vardiff_idle_pending: + return + self.vardiff_idle_queue_depth = max( + 0, + self.vardiff_idle_queue_depth - 1, + ) + self.vardiff_idle_inflight += 1 + client = request.client + delivery_attempted = False + try: + reason = self.request_skip_reason(request) + if reason is not None: + self.runtime._record_vardiff_idle_skip(reason) + return + # Readiness may have crossed in the ledger after the sweep's + # cache-only snapshot. Refresh it on this bounded worker so a + # cached collection bundle cannot be delivered after the pool is + # ready for normal payout work. + self.runtime.pool_readiness_latched() + # Canonicalize the sweep's cache-only snapshot on the dedicated + # worker. shared_job_bundle() selects the current payout-artifact + # key and rebinds a ready heavy bundle to the latest same-tip + # template observation; a miss may build here, never on the sweep. + bundle = self.runtime._build_idle_job_bundle(request) + reason = self.request_skip_reason(request) + if reason is not None: + self.runtime._record_vardiff_idle_skip(reason) + return + # Prepared bundles bypass _maybe_send_job_locked's normal build + # admission, so preserve its live reorg/headers/IBD trust guard on + # the dedicated worker before taking the client lock or sending. + if not self.runtime.ensure_reorg_reconciled_for_current_tip(): + self.runtime._record_vardiff_idle_skip("superseded") + return + if not client.job_update_lock.acquire(blocking=False): + self.runtime._record_vardiff_idle_skip("busy") + return + try: + reason = self.request_skip_reason(request) + if reason is not None: + self.runtime._record_vardiff_idle_skip(reason) + return + with self.runtime._job_cache_lock: + bundle_current = self.runtime._idle_bundle_current_locked( + client, + bundle, + allow_uncached=True, + ) + if not bundle_current: + self.runtime._record_vardiff_idle_skip("superseded") + return + # Everything above this point is coordinator preparation. An + # OSError there belongs to qbit RPC/ledger I/O, not the miner + # socket. Only retire the connection after entering the paired + # client delivery path below. + delivery_attempted = True + retargeted = self.retarget_locked( + client, + current_difficulty=request.current_difficulty, + accepted_shares=0, + submitted_shares=0, + accepted_difficulty=Decimal("0"), + elapsed_seconds=request.elapsed_seconds, + require_idle=True, + prepared_bundle=bundle, + expected_connection_id=request.connection_id, + expected_worker=request.worker, + expected_active_job=request.active_job, + expected_window_started=request.window_started_monotonic, + prepared_bundle_allow_uncached=True, + ) + finally: + client.job_update_lock.release() + if retargeted: + with self.runtime.lock: + self.idle_retarget_count = int( + getattr(self, "idle_retarget_count", 0) + ) + 1 + return + reason = self.request_skip_reason(request) + if reason is not None: + self.runtime._record_vardiff_idle_skip(reason) + return + with self.runtime._job_cache_lock: + bundle_current = self.runtime._idle_bundle_current_locked( + client, + bundle, + allow_uncached=True, + ) + if not bundle_current: + self.runtime._record_vardiff_idle_skip("superseded") + except JobBuildSuperseded: + self.runtime._record_vardiff_idle_skip("superseded") + except OSError: + with self._vardiff_idle_lock: + self.vardiff_idle_task_failures += 1 + if delivery_attempted: + self.runtime.disconnect_client(client) + return + print( + "prism coordinator: idle vardiff retarget preparation failed; " + "keeping client connected", + flush=True, + ) + traceback.print_exc() + except Exception: + with self._vardiff_idle_lock: + self.vardiff_idle_task_failures += 1 + print("prism coordinator: idle vardiff retarget task failed", flush=True) + traceback.print_exc() + + def _enqueue_idle( + self, + request: IdleRetargetRequest, + bundle: CachedJobBundle | None, + ) -> str | None: + key = (request.client, request.connection_id) + queued_monotonic = time.monotonic() + with self._vardiff_idle_lock: + if ( + self._vardiff_idle_executor_shutdown + or key in self._vardiff_idle_pending + ): + return "superseded" + if len(self._vardiff_idle_pending) >= MAX_PENDING_VARDIFF_IDLE_RETARGETS: + return "queue_full" + executor = self._vardiff_idle_executor + if executor is None: + executor = ThreadPoolExecutor( + max_workers=PRISM_VARDIFF_IDLE_RETARGET_MAX_WORKERS, + thread_name_prefix="prism-vardiff-idle", + ) + self._vardiff_idle_executor = executor + self._vardiff_idle_pending.add(key) + self.vardiff_idle_queue_depth += 1 + try: + future = executor.submit( + self._run_idle_task, + request, + bundle, + queued_monotonic, + ) + except RuntimeError: + self._vardiff_idle_pending.discard(key) + self.vardiff_idle_queue_depth = max( + 0, + self.vardiff_idle_queue_depth - 1, + ) + return "queue_full" + + def finish_task(completed: Future[None]) -> None: + self._finish_idle_task( + key, + queued_monotonic, + started=not completed.cancelled(), + ) + + future.add_done_callback(finish_task) + return None + + def shutdown_idle_executor(self) -> None: + with self._vardiff_idle_lock: + executor = self._vardiff_idle_executor + self._vardiff_idle_executor = None + self._vardiff_idle_executor_shutdown = True + if executor is not None: + executor.shutdown(wait=True, cancel_futures=True) + self.shutdown_durable_executor() + + def idle_sweep_loop(self) -> None: + while not self.runtime.stop_event.wait( + self.runtime.vardiff_idle_sweep_seconds + ): + self.runtime._record_heartbeat("vardiff_idle_sweep") + try: + queued = self.idle_sweep_once() + if queued: + print( + "prism coordinator: idle vardiff sweep queued " + f"{queued} client(s)", + flush=True, + ) + except Exception: + print("prism coordinator: idle vardiff sweep failed", flush=True) + traceback.print_exc() + + def idle_sweep_once(self) -> int: + sweep_started = time.monotonic() + now = time.monotonic() + queued = 0 + try: + with self.runtime.lock: + clients = tuple(self.runtime.clients) + with self._vardiff_idle_lock: + self.vardiff_idle_clients_inspected += len(clients) + for client in clients: + self.runtime._record_heartbeat("vardiff_idle_sweep") + with client_vardiff_lock(client), self.runtime.lock: + if self.idle_tip_diverged_locked(): + reason = "superseded" + request = None + elif ( + client not in self.runtime.clients + or not self.runtime.client_can_receive_jobs(client) + ): + reason = "disconnected" + request = None + else: + active_job = client.active_job + worker = client.worker + config = ( + client.vardiff_config + or client.listener_vardiff_config + or self.runtime.vardiff_config + ) + if not config.enabled: + continue + if active_job is None or worker is None: + reason = "superseded" + request = None + else: + elapsed = Decimal( + str( + max( + 0.001, + now + - client.vardiff_window_started_monotonic, + ) + ) + ) + if ( + elapsed < config.retarget_interval_seconds + or client.vardiff_window_accepted != 0 + or client.vardiff_window_submitted != 0 + ): + reason = "not_idle" + request = None + else: + reason = None + request = IdleRetargetRequest( + client=client, + connection_id=client.connection_id, + worker=worker, + active_job=active_job, + window_started_monotonic=( + client.vardiff_window_started_monotonic + ), + current_difficulty=( + client.pending_share_difficulty + or client.share_difficulty + ), + elapsed_seconds=elapsed, + ) + if reason is not None: + self.runtime._record_vardiff_idle_skip(reason) + continue + assert request is not None + if self._request_pending(request): + self.runtime._record_vardiff_idle_skip("superseded") + continue + if not client.job_update_lock.acquire(blocking=False): + self.runtime._record_vardiff_idle_skip("busy") + continue + try: + reason = self.request_skip_reason(request) + finally: + client.job_update_lock.release() + if reason is not None: + self.runtime._record_vardiff_idle_skip(reason) + continue + bundle = self.runtime._cached_idle_job_bundle(client) + if bundle is None: + # The sweep itself stays cache-only. A missing/expired + # bundle is rebuilt only by the dedicated bounded worker, + # so the client still makes eventual vardiff progress. + self.runtime._record_vardiff_idle_skip("cache_miss") + reason = self._enqueue_idle(request, bundle) + if reason is not None: + self.runtime._record_vardiff_idle_skip(reason) + continue + queued += 1 + return queued + finally: + self.runtime._record_heartbeat("vardiff_idle_sweep") + self.observe_idle_seconds( + "sweep", + max(0.0, time.monotonic() - sweep_started), + ) + + def retarget( + self, + client: ClientState, + *, + current_difficulty: Decimal, + accepted_shares: int, + submitted_shares: int, + accepted_difficulty: Decimal, + elapsed_seconds: Decimal, + require_idle: bool = False, + prepared_bundle: CachedJobBundle | None = None, + expected_connection_id: int | None = None, + expected_worker: WorkerIdentity | None = None, + expected_active_job: PrismJobContext | None = None, + expected_window_started: float | None = None, + initial_convergence: bool = False, + ) -> bool: + acquired = client.job_update_lock.acquire(blocking=not require_idle) + if not acquired: + return False + try: + if require_idle and prepared_bundle is None: + prepared_bundle = self.runtime._cached_idle_job_bundle(client) + if prepared_bundle is None: + return False + return self.retarget_locked( + client, + current_difficulty=current_difficulty, + accepted_shares=accepted_shares, + submitted_shares=submitted_shares, + accepted_difficulty=accepted_difficulty, + elapsed_seconds=elapsed_seconds, + require_idle=require_idle, + prepared_bundle=prepared_bundle, + expected_connection_id=expected_connection_id, + expected_worker=expected_worker, + expected_active_job=expected_active_job, + expected_window_started=expected_window_started, + initial_convergence=initial_convergence, + ) + finally: + client.job_update_lock.release() + + def retarget_locked( + self, + client: ClientState, + *, + current_difficulty: Decimal, + accepted_shares: int, + submitted_shares: int, + accepted_difficulty: Decimal, + elapsed_seconds: Decimal, + require_idle: bool = False, + prepared_bundle: CachedJobBundle | None = None, + expected_connection_id: int | None = None, + expected_worker: WorkerIdentity | None = None, + expected_active_job: PrismJobContext | None = None, + expected_window_started: float | None = None, + prepared_bundle_allow_uncached: bool = False, + initial_convergence: bool = False, + ) -> bool: + config = self.client_config(client) + if not config.enabled: + return False + # Idle step-downs carry no accepted shares, so they can never be the + # decisive evidence the initial relaxation is granted for. + with client_vardiff_lock(client): + initial_convergence = bool( + initial_convergence + and not require_idle + and getattr( + client, + "vardiff_initial_convergence_pending", + True, + ) + ) + if initial_convergence: + self._count_initial_retarget_attempt() + if require_idle: + if prepared_bundle is None: + return False + with client_vardiff_lock(client), self.runtime.lock: + if expected_connection_id is None: + expected_connection_id = client.connection_id + if expected_worker is None: + expected_worker = client.worker + if expected_active_job is None: + expected_active_job = client.active_job + if expected_window_started is None: + expected_window_started = client.vardiff_window_started_monotonic + if ( + client not in self.runtime.clients + or getattr(client, "closing", False) + or not self.runtime.client_can_receive_jobs(client) + or client.connection_id != expected_connection_id + or client.worker != expected_worker + or client.active_job is not expected_active_job + or client.vardiff_window_started_monotonic + != expected_window_started + or client.vardiff_window_accepted != 0 + or client.vardiff_window_submitted != 0 + ): + return False + observed_difficulty = vardiff.observed_difficulty( + accepted_difficulty=accepted_difficulty, + elapsed_seconds=elapsed_seconds, + target_share_interval_seconds=config.target_share_interval_seconds, + ) + with client_vardiff_lock(client): + previous_estimate = client.vardiff_difficulty_estimate + if observed_difficulty is None: + difficulty_estimate = None + with client_vardiff_lock(client): + client.vardiff_difficulty_estimate = None + else: + difficulty_estimate = vardiff.smooth_difficulty_estimate( + observed=observed_difficulty, + previous=previous_estimate, + config=config, + ) + with client_vardiff_lock(client): + client.vardiff_difficulty_estimate = difficulty_estimate + next_difficulty = vardiff.calculate_next_difficulty( + current_difficulty=current_difficulty, + accepted_shares=accepted_shares, + elapsed_seconds=elapsed_seconds, + config=config, + accepted_difficulty=accepted_difficulty, + difficulty_estimate=difficulty_estimate, + initial_convergence=initial_convergence, + ) + if not vardiff.should_retarget( + current_difficulty, + next_difficulty, + config.retarget_tolerance, + ): + if initial_convergence: + self._count_initial_retarget_outcome("suppressed") + return False + idle_window_state: tuple[float, int, int, Decimal] | None = None + idle_window_reset_at: float | None = None + difficulty_superseded = False + prior_pending: Decimal | None = None + with client_vardiff_lock(client), self.runtime.lock: + previous_difficulty = ( + client.pending_share_difficulty or client.share_difficulty + ) + if previous_difficulty != current_difficulty: + # An explicit d=/md= request (or another retarget) moved this + # client while the step was being computed. It keeps its + # precedence: this retarget yields rather than overriding it. + difficulty_superseded = True + else: + if require_idle and ( + client not in self.runtime.clients + or getattr(client, "closing", False) + or not self.runtime.client_can_receive_jobs(client) + or client.connection_id != expected_connection_id + or client.worker != expected_worker + or client.active_job is not expected_active_job + or client.vardiff_window_started_monotonic + != expected_window_started + or client.vardiff_window_accepted != 0 + or client.vardiff_window_submitted != 0 + ): + # A share landed since the idle snapshot; the accept path + # owns this window. Abort the speculative step-down rather + # than overriding a client that just resumed submitting. + return False + if require_idle: + idle_window_state = ( + client.vardiff_window_started_monotonic, + client.vardiff_window_accepted, + client.vardiff_window_submitted, + client.vardiff_window_work, + ) + prior_pending = client.pending_share_difficulty + client.pending_share_difficulty = next_difficulty + if difficulty_superseded: + if initial_convergence: + self._count_initial_retarget_outcome("superseded") + return False + # Advertise the new difficulty only with its corresponding job. Idle + # retargets stamp an already-cached bundle; normal share-driven + # retargets retain the existing build path. Either path sends the pair + # together or restores the prior pending difficulty/window state. + def idle_commit_guard() -> bool: + nonlocal idle_window_reset_at + if not require_idle: + return True + # _maybe_send_job_locked holds vardiff_lock before entering the + # coordinator commit section, so this guard never waits under + # the coordinator lock. + if ( + self.runtime.stop_event.is_set() + or client not in self.runtime.clients + or getattr(client, "closing", False) + or not self.runtime.client_can_receive_jobs(client) + or client.connection_id != expected_connection_id + or client.worker != expected_worker + or client.active_job is not expected_active_job + or client.vardiff_window_started_monotonic + != expected_window_started + or client.vardiff_window_accepted != 0 + or client.vardiff_window_submitted != 0 + or client.pending_share_difficulty != next_difficulty + ): + return False + idle_window_reset_at = time.monotonic() + client.vardiff_window_started_monotonic = idle_window_reset_at + client.vardiff_window_accepted = 0 + client.vardiff_window_submitted = 0 + client.vardiff_window_work = Decimal("0") + return True + + def restore_speculative_retarget() -> None: + with client_vardiff_lock(client), self.runtime.lock: + if client.pending_share_difficulty == next_difficulty: + client.pending_share_difficulty = prior_pending + self.restore_idle_window_state( + client, + idle_window_state, + idle_window_reset_at, + ) + + # A same-tip retarget only needs to flush in-flight work on a + # step-down: firmware that applies mining.set_difficulty retroactively + # would otherwise submit sub-target shares against the old job. On a + # step-up the old job stays valid at its own stamped share_target, so + # keeping it avoids discarding the miner's in-flight work. + clean_jobs = next_difficulty < current_difficulty + try: + if require_idle: + sent = self.runtime._maybe_send_job_locked( + client, + clean_jobs=clean_jobs, + raise_on_build_failure=True, + prepared_bundle=prepared_bundle, + commit_guard=idle_commit_guard, + commit_guard_lock=client_vardiff_lock(client), + prepared_bundle_allow_uncached=( + prepared_bundle_allow_uncached + ), + ) + else: + sent = bool( + client.authorized + and client.subscribed + and not self.runtime.stop_event.is_set() + and self.runtime.maybe_send_job(client, clean_jobs=clean_jobs) + ) + # A completed paired send is the commit point. Shutdown may race + # immediately afterward, but it cannot make already-delivered work + # speculative again. + if sent: + if not require_idle: + # Commit point for the fast-arrival relaxation: it is + # spent only once a share-driven retarget has actually + # reached the miner, so a skipped build or a failed send + # cannot consume it silently. Every later retarget on + # this connection is back on the ordinary step bound. + with client_vardiff_lock(client): + client.vardiff_initial_convergence_pending = False + if initial_convergence: + self._count_initial_retarget_outcome("applied") + self.observe_high_diff_arrival( + client, + config=config, + previous_difficulty=current_difficulty, + next_difficulty=next_difficulty, + ) + # Retain the committed difficulty here, not only on + # disconnect, so sessions that die without a clean + # disconnect still resume at their converged value. An idle + # step-down carries no accepted shares, but it also moves the + # value, so the store refreshes its TTL on the changed-value + # branch instead. + self.record_session_difficulty( + client, + next_difficulty, + share_backed=accepted_shares > 0, + previous_difficulty=current_difficulty, + initial_convergence=initial_convergence, + ) + return True + except Exception: + # Cached stamping can surface _JobBuildFailed before delivery, and + # socket errors can surface during the paired send. Both must undo + # every speculative client mutation before the task reports failure. + restore_speculative_retarget() + if initial_convergence: + self._count_initial_retarget_outcome("failed") + raise + restore_speculative_retarget() + if initial_convergence: + self._count_initial_retarget_outcome("superseded") + return False + + @staticmethod + def restore_idle_window_state( + client: ClientState, + idle_window_state: tuple[float, int, int, Decimal] | None, + idle_window_reset_at: float | None, + ) -> None: + """Un-restart the idle vardiff window after a step-down that never + reached the miner (skipped build/send), so the next sweep can retry + immediately instead of waiting out another full retarget interval. + Caller must hold the client's vardiff_lock. No-op unless this retarget did the reset + and nothing else has restarted the window since.""" + if idle_window_reset_at is None or idle_window_state is None: + return + if ( + client.vardiff_window_started_monotonic == idle_window_reset_at + and client.vardiff_window_accepted == 0 + and client.vardiff_window_submitted == 0 + and client.vardiff_window_work == 0 + ): + ( + client.vardiff_window_started_monotonic, + client.vardiff_window_accepted, + client.vardiff_window_submitted, + client.vardiff_window_work, + ) = idle_window_state + + def metrics_lines(self) -> list[str]: + with self._vardiff_idle_lock: + sweep = { + "buckets": dict(self.vardiff_idle_sweep_histogram["buckets"]), + "sum": float(self.vardiff_idle_sweep_histogram["sum"]), + "count": int(self.vardiff_idle_sweep_histogram["count"]), + } + task = { + "buckets": dict(self.vardiff_idle_task_histogram["buckets"]), + "sum": float(self.vardiff_idle_task_histogram["sum"]), + "count": int(self.vardiff_idle_task_histogram["count"]), + } + inspected = self.vardiff_idle_clients_inspected + skip_counts = dict(self.vardiff_idle_skip_counts) + queue_depth = self.vardiff_idle_queue_depth + inflight = self.vardiff_idle_inflight + failures = self.vardiff_idle_task_failures + with self._vardiff_convergence_lock: + initial_attempts = self.vardiff_initial_retarget_attempts + initial_outcomes = dict(self.vardiff_initial_retarget_outcome_counts) + durable_writes = dict(self.vardiff_durable_write_outcome_counts) + durable_pruned = int(self.vardiff_durable_pruned_records) + durable_prune_failures = int(self.vardiff_durable_prune_failures) + durable_downward_dropped = int(self.vardiff_durable_downward_dropped) + durable_backend = ( + "postgres" + if isinstance( + self.worker_difficulty_store, + PostgresWorkerDifficultyStore, + ) + else "memory" + ) + arrival_seconds = { + "buckets": dict( + self.vardiff_high_diff_arrival_seconds_histogram["buckets"] + ), + "sum": float(self.vardiff_high_diff_arrival_seconds_histogram["sum"]), + "count": int(self.vardiff_high_diff_arrival_seconds_histogram["count"]), + } + arrival_shares = { + "buckets": dict( + self.vardiff_high_diff_arrival_shares_histogram["buckets"] + ), + "sum": float(self.vardiff_high_diff_arrival_shares_histogram["sum"]), + "count": int(self.vardiff_high_diff_arrival_shares_histogram["count"]), + } + + lines = [ + "# HELP qbit_prism_vardiff_idle_clients_inspected_total Clients inspected by bounded vardiff idle sweeps.", + "# TYPE qbit_prism_vardiff_idle_clients_inspected_total counter", + f"qbit_prism_vardiff_idle_clients_inspected_total {inspected}", + "# HELP qbit_prism_vardiff_idle_skips_total Idle retargets skipped by a bounded reason.", + "# TYPE qbit_prism_vardiff_idle_skips_total counter", + *[ + f'qbit_prism_vardiff_idle_skips_total{{reason="{reason}"}} {int(skip_counts.get(reason, 0))}' + for reason in PRISM_VARDIFF_IDLE_SKIP_REASONS + ], + "# HELP qbit_prism_vardiff_idle_queue_depth Cache-only idle retarget tasks waiting for a dedicated worker.", + "# TYPE qbit_prism_vardiff_idle_queue_depth gauge", + f"qbit_prism_vardiff_idle_queue_depth {queue_depth}", + "# HELP qbit_prism_vardiff_idle_inflight Cache-only idle retarget tasks currently running.", + "# TYPE qbit_prism_vardiff_idle_inflight gauge", + f"qbit_prism_vardiff_idle_inflight {inflight}", + "# HELP qbit_prism_vardiff_idle_task_failures_total Idle retarget tasks that failed during cached delivery.", + "# TYPE qbit_prism_vardiff_idle_task_failures_total counter", + f"qbit_prism_vardiff_idle_task_failures_total {failures}", + "# HELP qbit_prism_vardiff_initial_retarget_attempts_total Retargets attempted under the fast-arrival initial convergence policy: a connection whose first retarget has not yet committed produced enough accepted shares, over enough elapsed time, to show an observed difficulty far above the one it is stamped at. Equals the sum of qbit_prism_vardiff_initial_retargets_total.", + "# TYPE qbit_prism_vardiff_initial_retarget_attempts_total counter", + f"qbit_prism_vardiff_initial_retarget_attempts_total {int(initial_attempts)}", + "# HELP qbit_prism_vardiff_initial_retargets_total Fast-arrival initial retargets by outcome; only applied reached the miner as a paired difficulty and job, and only applied spends the connection's one initial relaxation.", + "# TYPE qbit_prism_vardiff_initial_retargets_total counter", + *[ + f'qbit_prism_vardiff_initial_retargets_total{{outcome="{outcome}"}} {int(initial_outcomes.get(outcome, 0))}' + for outcome in PRISM_VARDIFF_INITIAL_RETARGET_OUTCOMES + ], + "# HELP qbit_prism_vardiff_durable_preloaded Worker difficulties restored into the bounded authorization-time cache during service initialization.", + "# TYPE qbit_prism_vardiff_durable_preloaded gauge", + f"qbit_prism_vardiff_durable_preloaded {self.vardiff_durable_preload_records}", + "# HELP qbit_prism_vardiff_durable_preload_failures_total Durable startup preload failures; mining safely falls back to lane startup difficulty.", + "# TYPE qbit_prism_vardiff_durable_preload_failures_total counter", + f"qbit_prism_vardiff_durable_preload_failures_total {self.vardiff_durable_preload_failures}", + "# HELP qbit_prism_vardiff_durable_pruned_total Worker-difficulty rows deleted as expired, across the startup prune and every periodic TTL prune since.", + "# TYPE qbit_prism_vardiff_durable_pruned_total counter", + f"qbit_prism_vardiff_durable_pruned_total {durable_pruned}", + "# HELP qbit_prism_vardiff_durable_prune_failures_total TTL prune attempts that raised; rows stay until the next interval retries.", + "# TYPE qbit_prism_vardiff_durable_prune_failures_total counter", + f"qbit_prism_vardiff_durable_prune_failures_total {durable_prune_failures}", + "# HELP qbit_prism_vardiff_durable_pending Worker-difficulty writes queued on the bounded persistence lane right now.", + "# TYPE qbit_prism_vardiff_durable_pending gauge", + f"qbit_prism_vardiff_durable_pending {self.durable_pending_depth()}", + "# HELP qbit_prism_vardiff_durable_coalesced_total Queued writes absorbed into an existing entry for the same worker; the lane is bounded by distinct workers, not by event rate.", + "# TYPE qbit_prism_vardiff_durable_coalesced_total counter", + f"qbit_prism_vardiff_durable_coalesced_total {self.durable_coalesced_count()}", + "# HELP qbit_prism_vardiff_durable_downward_dropped_total Safe step-down corrections the lane could not persist. Non-zero means a reconnect may resume a difficulty the live session had abandoned, until its TTL expires; alert on it.", + "# TYPE qbit_prism_vardiff_durable_downward_dropped_total counter", + f"qbit_prism_vardiff_durable_downward_dropped_total {durable_downward_dropped}", + "# HELP qbit_prism_vardiff_durable_backend Active worker-difficulty retention backend.", + "# TYPE qbit_prism_vardiff_durable_backend gauge", + f'qbit_prism_vardiff_durable_backend{{backend="{durable_backend}"}} 1', + "# HELP qbit_prism_vardiff_durable_writes_total Share-backed worker-difficulty persistence attempts by bounded outcome.", + "# TYPE qbit_prism_vardiff_durable_writes_total counter", + *[ + f'qbit_prism_vardiff_durable_writes_total{{outcome="{outcome}"}} {int(durable_writes.get(outcome, 0))}' + for outcome in PRISM_VARDIFF_DURABLE_WRITE_OUTCOMES + ], + ] + lines.extend( + _histogram_lines( + "qbit_prism_vardiff_high_diff_arrival_seconds", + "Seconds from the start of a session's vardiff accounting until a retarget first carried it to the configured high-difficulty arrival threshold. Observed once per connection, and only for sessions that started below the threshold, so a lane already starting above it never contributes.", + arrival_seconds, + PRISM_VARDIFF_HIGH_DIFF_ARRIVAL_SECONDS_BUCKETS, + ) + ) + lines.extend( + _histogram_lines( + "qbit_prism_vardiff_high_diff_arrival_shares", + "Accepted shares a session produced before a retarget first carried it to the configured high-difficulty arrival threshold. Shares credited at the pre-arrival stamped target are exactly the under-credited ones, so this is the share-denominated cost of the climb.", + arrival_shares, + PRISM_VARDIFF_HIGH_DIFF_ARRIVAL_SHARES_BUCKETS, + ) + ) + for metric_name, description, histogram in ( + ( + "qbit_prism_vardiff_idle_sweep_seconds", + "Wall time of one bounded vardiff idle sweep.", + sweep, + ), + ( + "qbit_prism_vardiff_idle_retarget_task_seconds", + "Queue plus execution latency for cache-only idle retarget tasks.", + task, + ), + ): + lines.extend( + _histogram_lines( + metric_name, + description, + histogram, + PRISM_VARDIFF_IDLE_SECONDS_BUCKETS, + ) + ) + return lines diff --git a/lab/prism/worker_difficulty_store.py b/lab/prism/worker_difficulty_store.py new file mode 100644 index 000000000..d01f5b315 --- /dev/null +++ b/lab/prism/worker_difficulty_store.py @@ -0,0 +1,664 @@ +"""Durable retention of delivered, safe per-worker vardiff difficulties. + +Storage seam only. The vardiff service keeps its in-process +``SessionDifficultyStore`` as the authorization-time source; this module gives +a later integration layer matching in-memory and PostgreSQL back ends for +retaining the last delivered, safe wire difficulty across coordinator +restarts. The caller — not this module — enforces share-backed upward moves: +the API carries the delivered decimal value and the evidence timestamp +explicitly so safety is never inferred from the username alone. Exact +usernames are public identities, not authentication secrets. + +Contract shared by both implementations: + +- One row per ``(listener, worker_username)``: the listener lane name plus the + exact Stratum username, matched byte-for-byte with no normalization. +- ``difficulty`` is the delivered decimal wire difficulty, retained exactly + (no integer or float conversion anywhere on the round trip). +- ``evidence_at_ms`` is when share-backed evidence last validated the value; + ``updated_at_ms`` is the last write of any kind. Both are caller-supplied + wall-clock milliseconds, so the two back ends stay deterministic and + comparable. +- ``upsert`` is atomic with deterministic last-write/evidence semantics: a + write whose evidence timestamp is older than the stored one is refused + outright, equal-or-newer evidence replaces the row (last write wins on + ties). Older evidence can therefore never displace newer evidence. +- ``apply_downward`` is the atomic lower-only correction: it reduces an + existing row to a smaller difficulty and leaves ``evidence_at`` untouched, + so a step-down the live session already made can never suppress later + genuine share-backed upward evidence, and can never refresh the TTL of a + value no share proved. A missing row, or one already at or below the + requested difficulty, is a no-op — never an insert, because there is no + stored value for a restart to resurrect. +- ``load_recent`` returns a bounded newest-first set whose evidence is + strictly newer than the cutoff, deterministically ordered for preload: + evidence timestamp descending, ties broken by (listener, worker_username) + in byte order (the key columns collate as "C", so both back ends agree). +- ``prune`` deletes rows whose evidence is at or before the cutoff — exactly + the complement of ``load_recent`` — entirely inside the store (an indexed + range scan in PostgreSQL), never via an unbounded application-side sweep. + +Deliberately absent: any per-key point read. Reconnect authorization must +never wait on a synchronous database round trip, so the only read is the +bounded batch preload an integration layer feeds into its in-process store. +""" + +from __future__ import annotations + +from dataclasses import dataclass +from decimal import Decimal +from threading import Lock +from typing import Any, Callable, Protocol, runtime_checkable + + +# Byte-identical to the qbit_worker_difficulty DDL block in +# crates/qbit-prism/sql/001_share_ledger.sql (which is the canonical, +# transactional apply path); a contract test pins the match. Both statements +# are idempotent, so ensure_schema() may be re-run freely and the monolithic +# file may be reapplied over an existing table. +WORKER_DIFFICULTY_SCHEMA_SQL = """\ +CREATE TABLE IF NOT EXISTS qbit_worker_difficulty ( + listener text COLLATE "C" NOT NULL CHECK (listener <> ''), + worker_username text COLLATE "C" NOT NULL CHECK (worker_username <> ''), + difficulty numeric NOT NULL CHECK ( + difficulty > 0 AND difficulty < 'Infinity'::numeric + ), + evidence_at timestamptz NOT NULL, + updated_at timestamptz NOT NULL DEFAULT clock_timestamp(), + PRIMARY KEY (listener, worker_username) +); + +CREATE INDEX IF NOT EXISTS qbit_worker_difficulty_evidence_idx + ON qbit_worker_difficulty (evidence_at DESC, listener, worker_username); +""" + +# Wall-clock bounds for every millisecond timestamp and cutoff this module +# accepts: non-negative (evidence predating the epoch is meaningless) and at +# most 9999-12-31T23:59:59.999Z, comfortably inside both PostgreSQL +# timestamptz range and exact float64 millisecond arithmetic. +MAX_TIMESTAMP_MS = 253_402_300_799_999 + +# Difficulty magnitude bounds enforced identically by both back ends, so the +# in-memory store cannot accept a value the PostgreSQL numeric apply would +# reject (or vice versa). Far beyond any real Stratum wire difficulty, far +# inside PostgreSQL numeric limits. +MAX_DIFFICULTY_DIGITS = 1_000 +MAX_DIFFICULTY_ADJUSTED_EXPONENT = 999 + + +@dataclass(frozen=True) +class WorkerDifficultyRecord: + """One retained per-worker difficulty row, identical in both back ends.""" + + listener: str + worker_username: str + difficulty: Decimal + evidence_at_ms: int + updated_at_ms: int + + +@dataclass(frozen=True) +class WorkerDifficultyUpsertResult: + """Outcome of one upsert. + + ``applied`` is False when the write carried evidence strictly older than + the stored row and was refused. ``stored`` is the row now standing for the + key; it can be None only on the PostgreSQL back end, in the narrow race + where a refused write's follow-up read finds no snapshot-visible row. + """ + + applied: bool + stored: WorkerDifficultyRecord | None + + +@dataclass(frozen=True) +class WorkerDifficultyLowerResult: + """Outcome of one lower-only correction. + + ``applied`` is True only when a stored row was actually reduced. A key with + no stored row, or one already at or below the requested difficulty, is a + no-op: there is nothing a restart could resurrect. ``stored`` is the row now + standing for the key, or None when no row exists (or, on the PostgreSQL + back end, in the narrow race where a no-op's follow-up read finds no + snapshot-visible row). + """ + + applied: bool + stored: WorkerDifficultyRecord | None + + +@runtime_checkable +class WorkerDifficultyStorePort(Protocol): + """The exact surface an integration layer may depend on. + + Four operations, none of which is a per-key synchronous read. + """ + + def upsert( + self, + *, + listener: str, + worker_username: str, + difficulty: Decimal, + evidence_at_ms: int, + now_ms: int, + ) -> WorkerDifficultyUpsertResult: ... + + def apply_downward( + self, + *, + listener: str, + worker_username: str, + difficulty: Decimal, + now_ms: int, + ) -> WorkerDifficultyLowerResult: ... + + def load_recent( + self, + *, + evidence_after_ms: int, + limit: int, + ) -> list[WorkerDifficultyRecord]: ... + + def prune( + self, + *, + evidence_cutoff_ms: int, + limit: int | None = None, + ) -> int: ... + + +def _validated_identity(value: str, name: str) -> str: + if not isinstance(value, str): + raise ValueError(f"{name} must be a string") + if value == "": + raise ValueError(f"{name} must not be empty") + if "\x00" in value: + # PostgreSQL text cannot hold NUL; refusing it here keeps the + # in-memory back end from accepting a key the durable one rejects. + raise ValueError(f"{name} must not contain NUL") + return value + + +def _validated_difficulty(difficulty: Decimal) -> Decimal: + if isinstance(difficulty, bool) or not isinstance(difficulty, Decimal): + raise ValueError("difficulty must be a Decimal (no float or int coercion)") + if not difficulty.is_finite() or difficulty <= 0: + raise ValueError("difficulty must be a positive finite Decimal") + parts = difficulty.as_tuple() + if len(parts.digits) > MAX_DIFFICULTY_DIGITS: + raise ValueError( + f"difficulty exceeds {MAX_DIFFICULTY_DIGITS} significant digits" + ) + if abs(difficulty.adjusted()) > MAX_DIFFICULTY_ADJUSTED_EXPONENT: + raise ValueError( + "difficulty magnitude exceeds " + f"1E±{MAX_DIFFICULTY_ADJUSTED_EXPONENT}" + ) + return difficulty + + +def _validated_timestamp_ms(value: int, name: str) -> int: + if isinstance(value, bool) or not isinstance(value, int): + raise ValueError(f"{name} must be an integer millisecond timestamp") + if value < 0 or value > MAX_TIMESTAMP_MS: + raise ValueError(f"{name} must be within [0, {MAX_TIMESTAMP_MS}]") + return value + + +def _validated_limit(limit: int) -> int: + if isinstance(limit, bool) or not isinstance(limit, int): + raise ValueError("limit must be an integer") + if limit <= 0: + raise ValueError("limit must be positive") + return limit + + +class MemoryWorkerDifficultyStore: + """In-memory reference implementation of the storage contract. + + Semantically identical to :class:`PostgresWorkerDifficultyStore` (the + parity gate drives both in lockstep). The lock is a leaf lock: nothing + here calls out of this module while holding it. + """ + + def __init__(self) -> None: + self._lock = Lock() + self._entries: dict[tuple[str, str], WorkerDifficultyRecord] = {} + + def upsert( + self, + *, + listener: str, + worker_username: str, + difficulty: Decimal, + evidence_at_ms: int, + now_ms: int, + ) -> WorkerDifficultyUpsertResult: + listener = _validated_identity(listener, "listener") + worker_username = _validated_identity(worker_username, "worker_username") + difficulty = _validated_difficulty(difficulty) + evidence_at_ms = _validated_timestamp_ms(evidence_at_ms, "evidence_at_ms") + now_ms = _validated_timestamp_ms(now_ms, "now_ms") + key = (listener, worker_username) + with self._lock: + existing = self._entries.get(key) + if existing is not None and evidence_at_ms < existing.evidence_at_ms: + return WorkerDifficultyUpsertResult(applied=False, stored=existing) + record = WorkerDifficultyRecord( + listener=listener, + worker_username=worker_username, + difficulty=difficulty, + evidence_at_ms=evidence_at_ms, + updated_at_ms=now_ms, + ) + self._entries[key] = record + return WorkerDifficultyUpsertResult(applied=True, stored=record) + + def apply_downward( + self, + *, + listener: str, + worker_username: str, + difficulty: Decimal, + now_ms: int, + ) -> WorkerDifficultyLowerResult: + listener = _validated_identity(listener, "listener") + worker_username = _validated_identity(worker_username, "worker_username") + difficulty = _validated_difficulty(difficulty) + now_ms = _validated_timestamp_ms(now_ms, "now_ms") + key = (listener, worker_username) + with self._lock: + existing = self._entries.get(key) + if existing is None or existing.difficulty <= difficulty: + return WorkerDifficultyLowerResult(applied=False, stored=existing) + # evidence_at is deliberately carried forward unchanged: this + # correction is not new evidence, and re-stamping it would both + # extend the TTL of an unproven value and let it refuse a later + # share-backed write whose evidence predates the step-down. + record = WorkerDifficultyRecord( + listener=existing.listener, + worker_username=existing.worker_username, + difficulty=difficulty, + evidence_at_ms=existing.evidence_at_ms, + updated_at_ms=now_ms, + ) + self._entries[key] = record + return WorkerDifficultyLowerResult(applied=True, stored=record) + + def load_recent( + self, + *, + evidence_after_ms: int, + limit: int, + ) -> list[WorkerDifficultyRecord]: + evidence_after_ms = _validated_timestamp_ms( + evidence_after_ms, "evidence_after_ms" + ) + limit = _validated_limit(limit) + with self._lock: + recent = [ + record + for record in self._entries.values() + if record.evidence_at_ms > evidence_after_ms + ] + recent.sort( + key=lambda record: ( + -record.evidence_at_ms, + record.listener, + record.worker_username, + ) + ) + return recent[:limit] + + def prune( + self, + *, + evidence_cutoff_ms: int, + limit: int | None = None, + ) -> int: + evidence_cutoff_ms = _validated_timestamp_ms( + evidence_cutoff_ms, "evidence_cutoff_ms" + ) + if limit is not None: + limit = _validated_limit(limit) + with self._lock: + doomed = [ + key + for key, record in self._entries.items() + if record.evidence_at_ms <= evidence_cutoff_ms + ] + if limit is not None: + doomed.sort( + key=lambda key: ( + self._entries[key].evidence_at_ms, + key[0], + key[1], + ) + ) + doomed = doomed[:limit] + for key in doomed: + del self._entries[key] + return len(doomed) + + def __len__(self) -> int: + with self._lock: + return len(self._entries) + + +def _text_literal(value: str) -> str: + return "'" + value.replace("'", "''") + "'" + + +def _difficulty_literal(difficulty: Decimal) -> str: + # str() of a validated finite Decimal contains only [0-9.E+-]; the quoted + # cast keeps the rendering injection-proof regardless. + return f"'{difficulty}'::numeric" + + +def _timestamp_literal(timestamp_ms: int) -> str: + # Same ms-to-timestamptz idiom as share_ledger.py. Validated bounds keep + # the double-precision division exact to well below the microsecond + # resolution timestamptz stores, so ms values round-trip exactly. + return f"to_timestamp(({timestamp_ms}::double precision / 1000.0))" + + +_RECORD_JSON_FIELDS = """\ +'listener', {alias}.listener, + 'worker_username', {alias}.worker_username, + 'difficulty', {alias}.difficulty::text, + 'evidence_at_ms', round(extract(epoch FROM {alias}.evidence_at) * 1000)::bigint, + 'updated_at_ms', round(extract(epoch FROM {alias}.updated_at) * 1000)::bigint""" + + +def _record_from_json(payload: dict[str, Any]) -> WorkerDifficultyRecord: + return WorkerDifficultyRecord( + listener=str(payload["listener"]), + worker_username=str(payload["worker_username"]), + # difficulty travels as text inside the JSON payload precisely so it + # never passes through a float. + difficulty=Decimal(str(payload["difficulty"])), + evidence_at_ms=int(payload["evidence_at_ms"]), + updated_at_ms=int(payload["updated_at_ms"]), + ) + + +class PostgresWorkerDifficultyStore: + """PostgreSQL implementation over an injected statement runner. + + ``run_json`` executes one self-contained SQL statement and returns its + single JSON value — the contract ``PsqlShareLedger._run_json`` and + ``LedgerSqlPort.run_json`` already honour, so the store rides the + coordinator's existing pooled connections without opening its own. + ``run_script`` (optional) executes a multi-statement script and is only + needed for :meth:`ensure_schema`; the canonical schema path remains the + monolithic ``001_share_ledger.sql`` apply, which creates the same table. + """ + + def __init__( + self, + run_json: Callable[[str], Any], + *, + run_script: Callable[[str], None] | None = None, + ) -> None: + self._run_json = run_json + self._run_script = run_script + + @classmethod + def from_share_ledger( + cls, + ledger: Any, + *, + timeout_seconds: float, + ) -> "PostgresWorkerDifficultyStore": + """Build over a bounded, admission-gated ``PsqlShareLedger`` seam. + + Worker-difficulty traffic is not writer-lease state, so it uses one + of the ledger's bounded read/concurrent statement slots. The statement + is still a mutation and is therefore never retried after an ambiguous + connection failure. + """ + + def run_json(sql: str) -> Any: + with ledger.operation_timeout(timeout_seconds): + with ledger._operation_gate( + ledger._read_semaphore, + "worker-difficulty slot", + ): + return ledger._run_json(sql) + + return cls(run_json, run_script=ledger._run_script) + + def ensure_schema(self) -> None: + """Idempotently apply this store's DDL (subset of the monolithic file).""" + if self._run_script is None: + raise RuntimeError( + "ensure_schema requires a run_script callable; the monolithic " + "001_share_ledger.sql apply also creates this table" + ) + self._run_script(WORKER_DIFFICULTY_SCHEMA_SQL) + + def upsert( + self, + *, + listener: str, + worker_username: str, + difficulty: Decimal, + evidence_at_ms: int, + now_ms: int, + ) -> WorkerDifficultyUpsertResult: + listener = _validated_identity(listener, "listener") + worker_username = _validated_identity(worker_username, "worker_username") + difficulty = _validated_difficulty(difficulty) + evidence_at_ms = _validated_timestamp_ms(evidence_at_ms, "evidence_at_ms") + now_ms = _validated_timestamp_ms(now_ms, "now_ms") + listener_literal = _text_literal(listener) + username_literal = _text_literal(worker_username) + applied_fields = _RECORD_JSON_FIELDS.format(alias="applied") + current_fields = _RECORD_JSON_FIELDS.format(alias="current") + # One atomic statement: the ON CONFLICT WHERE guard is what makes + # older evidence unable to displace newer evidence, and equal-or-newer + # evidence a deterministic last-write-wins replacement. The refused + # branch reads the standing row from the statement snapshot instead + # (a data-modifying CTE's effects are invisible to the outer query, + # so the applied branch must come from RETURNING). + payload = self._run_json( + f""" +WITH applied AS ( + INSERT INTO qbit_worker_difficulty AS existing ( + listener, + worker_username, + difficulty, + evidence_at, + updated_at + ) + VALUES ( + {listener_literal}, + {username_literal}, + {_difficulty_literal(difficulty)}, + {_timestamp_literal(evidence_at_ms)}, + {_timestamp_literal(now_ms)} + ) + ON CONFLICT (listener, worker_username) DO UPDATE + SET difficulty = EXCLUDED.difficulty, + evidence_at = EXCLUDED.evidence_at, + updated_at = EXCLUDED.updated_at + WHERE EXCLUDED.evidence_at >= existing.evidence_at + RETURNING listener, worker_username, difficulty, evidence_at, updated_at +) +SELECT json_build_object( + 'applied', EXISTS (SELECT 1 FROM applied), + 'stored', COALESCE( + (SELECT json_build_object( + {applied_fields} + ) FROM applied), + (SELECT json_build_object( + {current_fields} + ) + FROM qbit_worker_difficulty current + WHERE current.listener = {listener_literal} + AND current.worker_username = {username_literal}) + ) +); +""" + ) + stored = payload.get("stored") + return WorkerDifficultyUpsertResult( + applied=bool(payload["applied"]), + stored=None if stored is None else _record_from_json(stored), + ) + + def apply_downward( + self, + *, + listener: str, + worker_username: str, + difficulty: Decimal, + now_ms: int, + ) -> WorkerDifficultyLowerResult: + listener = _validated_identity(listener, "listener") + worker_username = _validated_identity(worker_username, "worker_username") + difficulty = _validated_difficulty(difficulty) + now_ms = _validated_timestamp_ms(now_ms, "now_ms") + listener_literal = _text_literal(listener) + username_literal = _text_literal(worker_username) + difficulty_literal = _difficulty_literal(difficulty) + lowered_fields = _RECORD_JSON_FIELDS.format(alias="lowered") + current_fields = _RECORD_JSON_FIELDS.format(alias="current") + # One atomic statement. The row-level UPDATE takes its own lock, so a + # concurrent upsert either precedes it (and is then compared against) + # or follows it (and wins on evidence); evidence_at is never written + # here, which is what keeps a step-down from suppressing later + # share-backed evidence. The no-op branch reads the standing row from + # the statement snapshot for the same reason upsert does. + payload = self._run_json( + f""" +WITH lowered AS ( + UPDATE qbit_worker_difficulty + SET difficulty = {difficulty_literal}, + updated_at = {_timestamp_literal(now_ms)} + WHERE listener = {listener_literal} + AND worker_username = {username_literal} + AND difficulty > {difficulty_literal} + RETURNING listener, worker_username, difficulty, evidence_at, updated_at +) +SELECT json_build_object( + 'applied', EXISTS (SELECT 1 FROM lowered), + 'stored', COALESCE( + (SELECT json_build_object( + {lowered_fields} + ) FROM lowered), + (SELECT json_build_object( + {current_fields} + ) + FROM qbit_worker_difficulty current + WHERE current.listener = {listener_literal} + AND current.worker_username = {username_literal}) + ) +); +""" + ) + stored = payload.get("stored") + return WorkerDifficultyLowerResult( + applied=bool(payload["applied"]), + stored=None if stored is None else _record_from_json(stored), + ) + + def load_recent( + self, + *, + evidence_after_ms: int, + limit: int, + ) -> list[WorkerDifficultyRecord]: + evidence_after_ms = _validated_timestamp_ms( + evidence_after_ms, "evidence_after_ms" + ) + limit = _validated_limit(limit) + recent_fields = _RECORD_JSON_FIELDS.format(alias="recent") + # The inner ORDER BY + LIMIT walk qbit_worker_difficulty_evidence_idx + # directly; the aggregate re-states the same ordering because + # json_agg offers no ordering guarantee of its own. + payload = self._run_json( + f""" +SELECT COALESCE( + ( + SELECT json_agg( + json_build_object( + {recent_fields} + ) + ORDER BY recent.evidence_at DESC, recent.listener, recent.worker_username + ) + FROM ( + SELECT listener, worker_username, difficulty, evidence_at, updated_at + FROM qbit_worker_difficulty + WHERE evidence_at > {_timestamp_literal(evidence_after_ms)} + ORDER BY evidence_at DESC, listener, worker_username + LIMIT {limit} + ) recent + ), + '[]'::json +); +""" + ) + return [_record_from_json(entry) for entry in payload] + + def prune( + self, + *, + evidence_cutoff_ms: int, + limit: int | None = None, + ) -> int: + evidence_cutoff_ms = _validated_timestamp_ms( + evidence_cutoff_ms, "evidence_cutoff_ms" + ) + cutoff_literal = _timestamp_literal(evidence_cutoff_ms) + if limit is None: + payload = self._run_json( + f""" +WITH deleted AS ( + DELETE FROM qbit_worker_difficulty + WHERE evidence_at <= {cutoff_literal} + RETURNING 1 +) +SELECT json_build_object('deleted', (SELECT count(*) FROM deleted)); +""" + ) + return int(payload["deleted"]) + limit = _validated_limit(limit) + # Oldest evidence first with the same tie-break the memory store + # applies, so a bounded prune removes a deterministic set. + payload = self._run_json( + f""" +WITH doomed AS ( + SELECT listener, worker_username + FROM qbit_worker_difficulty + WHERE evidence_at <= {cutoff_literal} + ORDER BY evidence_at ASC, listener, worker_username + LIMIT {limit} +), +deleted AS ( + DELETE FROM qbit_worker_difficulty target + USING doomed + WHERE target.listener = doomed.listener + AND target.worker_username = doomed.worker_username + AND target.evidence_at <= {cutoff_literal} + RETURNING 1 +) +SELECT json_build_object('deleted', (SELECT count(*) FROM deleted)); +""" + ) + return int(payload["deleted"]) + + +__all__ = [ + "MAX_DIFFICULTY_ADJUSTED_EXPONENT", + "MAX_DIFFICULTY_DIGITS", + "MAX_TIMESTAMP_MS", + "MemoryWorkerDifficultyStore", + "PostgresWorkerDifficultyStore", + "WORKER_DIFFICULTY_SCHEMA_SQL", + "WorkerDifficultyLowerResult", + "WorkerDifficultyRecord", + "WorkerDifficultyStorePort", + "WorkerDifficultyUpsertResult", +] diff --git a/lab/prism/writer_lease_timing.py b/lab/prism/writer_lease_timing.py new file mode 100644 index 000000000..3a69e9db3 --- /dev/null +++ b/lab/prism/writer_lease_timing.py @@ -0,0 +1,859 @@ +"""One validated timing policy for the PRISM writer-lease heartbeat. + +Why this module exists +---------------------- +The heartbeat's five timing knobs — adoption silence, heartbeat interval, +failure budget, monitor interval, exit margin — are not five independent +tunables. They encode one safety argument: + + a coordinator that has lost its guarded session must be *gone* before + any replacement becomes eligible to compare-and-swap the lease row. + +Historically the numbers were derived from the adoption silence alone +(``silence / 4`` and ``silence * 0.75``) and the coupling was re-derived by +hand at each call site, with only a printed warning when an override broke +it. Issue #212 is the failure mode that produces: the derived envelope was +sized for a heartbeat's *idle interval* but not for the *tail* of a lawful +verification, so ordinary rapid-block verification latency exhausted the +server-proven envelope and hard-exited healthy coordinators (69 restarts on +``union-mainnet``, activity ages 0.54-0.78s, server-proven ages 0.76-0.91s +against a 0.75s budget and a 0.80s cap). + +The fix is to state the whole inequality once, in terms of phases that are +actually measurable in production, and to reject a configuration that +breaks it instead of printing and continuing. + +The phase budgets +----------------- +Three inputs are physical properties of the deployment rather than free +choices: + +``guard_statement_timeout_seconds`` + The server-side ``statement_timeout`` on the dedicated guard session + (:class:`~lab.prism.share_ledger._NativePostgresLeaseGuard`). It is a + hard bound: no single guarded round trip can take longer, because + PostgreSQL cancels it. + +``heartbeat_interval_seconds`` + The idle gap the heartbeat waits between proofs. + +``scheduler_slack_seconds`` + The process-side scheduling allowance: thread wake-up and GIL + contention behind a rapid-block burst. It is applied in *both* + inequalities below, because it describes the process rather than one + thread — once to the heartbeat's stamping path, and once to the + monitor's own poll. Budgeting a stall for the heartbeat while + assuming the monitor wakes punctually is the mistake that made the + first cut of this policy unsafe. + +From those, the largest gap a *healthy* coordinator can leave between two +completed server round trips is + + max_healthy_server_gap = interval + guard_statement_timeout + slack + +and nothing shorter is a safe staleness bound. The argument is that at +every instant either a guarded statement is in flight — and it must answer +or be cancelled within the statement timeout — or none is, in which case +the heartbeat starts one within the remaining interval. Queueing behind a +concurrent external-effect fence does not extend the gap: the fence runs on +the same guard session and stamps at each of its own round trips. + +The safety inequality +--------------------- +The monitor measures staleness from completed round trips (client-side +marks can postdate a silent session death, so they cannot carry the +adoption guarantee). Write ``p`` for the newest server-proven edge and +``t_d`` for the instant the guarded session died; the conservative send-edge +stamping below guarantees ``p <= t_d``. The monitor polls every +``monitor_interval`` and may itself be ``scheduler_slack`` late, so it +observes ``age >= cap`` no later than + + p + cap + monitor_interval + scheduler_slack + +and the hard exit is then budgeted ``exit_margin``. A successor cannot +begin its silence window before the guard session is released, so it is +eligible no earlier than ``t_d + adoption_silence >= p + adoption_silence``. +Exit-before-adoption is therefore + + cap + exit_envelope <= adoption_silence (1) + where exit_envelope = exit_margin + 2 * monitor_interval + + scheduler_slack + heartbeat_interval < failure_budget (2) + max_healthy_server_gap <= cap (3) + +The second ``monitor_interval`` in the envelope is what makes (1) strict: +only one of the two is spent on poll granularity, so the worst-case exit +completes one whole poll interval *before* the adoption edge rather than +on it. + +(1) and (2) are *safety*: violating them means a lost coordinator may still +be alive when a replacement adopts, or that one idle wait alone exhausts +the liveness budget. They are rejected at startup. (3) is *stability*: +violating it does not permit two writers, it just guarantees that ordinary +tail latency will eventually hard-exit a healthy coordinator — exactly +issue #212. It is reported as an advisory, because a deliberately tiny +test or lab policy is allowed to trade stability away, and refusing to +start would be worse than restarting. + +What (1) actually guarantees +---------------------------- +(1) is a statement about a *budget*, not about the process. Write ``L`` +for the monitor's real lateness on the poll that first observes the stale +edge. That poll happens no later than ``p + cap + monitor_interval + L`` +and the exit completes ``exit_margin`` after it, so exit-before-adoption +holds for that beat if and only if + + L < adoption_silence - cap - monitor_interval - exit_margin + = scheduler_slack + monitor_interval (4) + +(``max_guaranteed_monitor_lateness_seconds`` below; the identity on the +right holds whenever the cap is the silence minus the exit envelope, which +it is for every policy that is not floored at the failure budget). Three +regimes follow: + +``L <= scheduler_slack`` + the case (1) budgets: the exit completes one whole monitor poll before + the adoption edge. +``scheduler_slack < L < scheduler_slack + monitor_interval`` + the strictness reserve is consumed: the exit still completes before + the adoption edge, but by less than one poll. +``L >= scheduler_slack + monitor_interval`` + exit-before-adoption is **not guaranteed** for that beat; the process + may still be alive up to ``L - (scheduler_slack + monitor_interval)`` + after the adoption edge. + +Production exceeded the assumption (issue #227) +----------------------------------------------- +Three days after this policy shipped, ``union-mainnet`` (bootstrap pin +``9c41894``, coordinator up 45h, zero restarts, no heartbeat exit) recorded +a monitor wake delay of **0.648s** — above the 0.50s slack and above the +0.55s bound (4). For the duration of that stall the sum on the left of +(1), evaluated with the real lateness, was ``1.30 + 0.05 + 0.648 + 0.10 = +2.098 > 2.00``. No guard loss coincided, so nothing happened; but the +assumption the proof rests on was violated in production. + +The stall was in-process, not on the host (32 cores 93% idle, PSI near +zero, no cgroup throttling). The window held a 5.1s +``reconcile_invalidation`` full rescan of a 52k-share payout window, and +the phase attribution of the same period showed why the old telemetry +could not see it: the worst ``guard_sql`` was 0.714s against a +server-enforced 0.50s statement timeout while ``scheduler_delay`` never +exceeded 22 microseconds in 645k attempts. ``guard_sql`` was measured +client-side, so a GIL stall between PostgreSQL answering and the heartbeat +thread resuming was booked as database time. + +Attribution that survives GIL stalls +------------------------------------ +Each guard statement now returns its own server-side execution time +(``clock_timestamp() - statement_timestamp()``, computed and returned by +the statement itself, so no extra round trip and no extra lock). The +attempt accumulator splits the client-measured statement span into +``guard_sql`` (server execution, when the statement reports it) and +``guard_client_resume`` (the remainder of the round trip: network transit +plus the time this process took to resume after the answer arrived). The +residual ``scheduler_delay = total - slot_wait - guard_sql`` therefore +absorbs every process-side delay, inside or outside a round trip; +``guard_client_resume`` is the part of it that fell inside one. A +statement that reports no server time keeps the old, deliberately +conservative attribution (the whole span is ``guard_sql``). The +server-proven edge is untouched: it is still the conservative send edge, +never a server-reported timestamp, because the ``p <= t_d`` step above +depends on it. + +The response to lateness beyond slack (the decision) +---------------------------------------------------- +Two responses were on the table. (a) Treat lateness beyond slack as +envelope consumed for that beat and tighten the cap for that beat. (b) +Accept the residual and rely on the fences. + +(a) is refused, on the arithmetic rather than on taste. On the late wake +``w`` the monitor can only exit *now*, completing at ``w + exit_margin``; +that bound does not move whether the threshold it compares the age +against is ``cap`` or ``cap - (L - slack)``. Tightening changes behaviour +only for server-proven ages in ``[cap - (L - slack), cap)`` — a band that a +healthy coordinator occupies whenever a slow statement coincides with the +stall (with the observed 0.648s the tightened threshold is 1.152s, below +the 1.25s a healthy gap may reach). So (a) hard-exits healthy +coordinators — issue #212's failure mode — and buys nothing against the +breach it is reacting to. + +(b) is safe because the un-covered interval permits no mutation. Three +observations: + +* A successor cannot begin its silence until the predecessor's *guard + session* is gone (``PsqlShareLedger._writer_lease_adoption_wait_seconds`` + cannot even be reached while the advisory guard is held). A stalled but + live process is therefore never adopted over; the hazard is the compound + event of a guard-session death *and* a monitor lateness beyond (4) in the + same window. +* During a GIL stall no Python thread runs, so nothing escapes *during* the + stall. What could escape is work done in the interval between the + adoption edge and the (late) exit, at most ``L - (slack + + monitor_interval)`` long. +* Every ledger write is fenced at the row: share appends and every landing + statement open with the exact-session ``lease AS (UPDATE + qbit_ledger_writer_lease ... WHERE writer_session_token = ...)`` CTE and + join it, and adoption rewrites ``writer_session_token``, so a stale + writer's statement matches zero lease rows and reports "writer lease is + not active". Every external effect (submitblock, CTV fanout, wallet) + runs behind ``require_fresh_lease_for_external_side_effect``, a + synchronous exact-session verification on the guard session that raises + — and hard-exits — the moment the committed row names another session. + The fence-to-RPC residual is the documented preflight residual between + independent systems and is not widened by the monitor's lateness. The + heartbeat's own next proof fails the same way. + +So the shipped response is (b), with the breach made loud instead of +invisible: the monitor's lateness feeds a fixed-bucket histogram, three +threshold counters (0.5x, 0.8x and 1.0x slack), a rolling-window maximum +and the age of the lifetime record; a wake beyond (4) increments an +exit-guarantee breach counter, records the worst overrun, and emits one +structured warning naming the breach; a wake beyond half the slack takes a +rate-limited stack sample of the running threads. The operator action is +in ``docs/prism-ledger-ops.md``. The durable fix — an exit path that does +not depend on the interpreter being scheduled — is an out-of-process +watchdog fed by the server-proven edge, scoped in issue #130. + +The shipped defaults +-------------------- + guard_statement_timeout = 0.50 (server-enforced, unchanged) + heartbeat_interval = 0.25 + scheduler_slack = 0.50 (~3x the worst delay observed in the + union-mainnet burst behind #212; + exceeded once since, by a 0.648s + in-process stall, see #227 above) + max_healthy_server_gap = 0.25 + 0.50 + 0.50 = 1.25 + failure_budget = 1.25 (the same bound; one number, not two) + monitor_interval = 0.05 + exit_margin = 0.10 + exit_envelope = 0.10 + 2 x 0.05 + 0.50 = 0.70 + adoption_silence = 2.00 (> 1.25 + 0.70 = 1.95) + => server_proven_cap = 2.00 - 0.70 = 1.30 + => stability surplus = 1.30 - 1.25 = 0.05 + => max_guaranteed_monitor_lateness = 0.50 + 0.05 = 0.55 + +Checking (1) at the shipped numbers, with a monitor exactly as late as the +policy budgets: + + 1.30 + 0.05 + 0.50 + 0.10 = 1.95 < 2.00 + +so the old coordinator is gone 0.05s — one monitor poll — before the +successor may compare-and-swap, *provided* the monitor's real lateness is +within the 0.55s bound (4). That proviso is an assumption about the +process, not a property of the policy; production has exceeded it once. + +Genuine writer failover therefore costs 2.0s of adoption silence rather +than 1.0s. That is the price of the guard session's own 0.5s statement +timeout: the monitor can only trust completed round trips, so the envelope +must contain one whole statement plus one idle interval plus process +scheduling — and then, separately, room for the monitor itself to be that +late in noticing. The stability surplus is deliberately thin (0.05s): +inequality (1) is the safety bound and comes first, while (3) is already a +worst-case-on-worst-case (a statement running its whole server-side +timeout *and* the process stalling for its whole slack allowance in the +same beat). Production's worst observed server-proven age during the +issue #212 burst was 0.91s, well under the 1.30s cap, and the ownership +proof that now runs on most beats is a single non-blocking read. It is half of the 4.0s downstream mitigation this replaces, +and unlike that mitigation it is derived rather than dialled in. + +Lowering it further requires shrinking a phase, not the envelope: a shorter +statement timeout for the frequent ownership proof, or a shorter heartbeat +interval. The proof/renewal split (see +``PsqlShareLedger.prove_writer_lease_guard_session``) already removed the +*typical* cost — the frequent statement no longer scans +``pg_stat_activity`` or touches the lease tuple's row lock — but the +*bound* is still the guard session's statement timeout, and the envelope +must be sized for the bound. +""" + +from __future__ import annotations + +from dataclasses import dataclass +import math +import time +from typing import Callable + +# The server-side statement_timeout carried by the dedicated writer-lease +# guard session. It is the hard ceiling on one guarded round trip, so it is +# also the dominant term in the heartbeat's staleness envelope. Declared here +# rather than inline in the connection options so the policy below and the +# connection that enforces it cannot drift apart. +WRITER_LEASE_GUARD_STATEMENT_TIMEOUT_SECONDS = 0.5 + +# Statements verify_writer_lease_guard_session may lawfully run inside one +# guarded slot: the verification statement plus its single attribution +# recheck. Callers that budget the verification's execution wall-clock must +# cover this many server-side statement timeouts, or a lawful recheck under +# moderate database latency is killed by the caller's deadline instead of +# rescuing the coordinator. +WRITER_LEASE_VERIFICATION_MAX_STATEMENTS = 2 + +# Process-side delay the envelope must absorb between PostgreSQL answering +# and the monitor observing that answer: heartbeat thread wake-up, GIL +# contention behind a rapid-block burst, result handling, and the monitor's +# own late wake. Sized at roughly three times the worst delay observed in the +# union-mainnet burst that produced issue #212. Issue #227 recorded a 0.648s +# monitor wake delay on union-mainnet, above this allowance; the module +# docstring states what is and is not guaranteed when that happens. +WRITER_LEASE_HEARTBEAT_SCHEDULER_SLACK_SECONDS = 0.5 + +WRITER_LEASE_HEARTBEAT_INTERVAL_SECONDS = 0.25 +WRITER_LEASE_HEARTBEAT_MONITOR_SECONDS = 0.05 +WRITER_LEASE_HEARTBEAT_EXIT_MARGIN_SECONDS = 0.1 + +# Derived, not chosen: the largest gap between completed guard round trips a +# healthy coordinator can produce (see the module docstring). +WRITER_LEASE_HEARTBEAT_FAILURE_SECONDS = ( + WRITER_LEASE_HEARTBEAT_INTERVAL_SECONDS + + WRITER_LEASE_GUARD_STATEMENT_TIMEOUT_SECONDS + + WRITER_LEASE_HEARTBEAT_SCHEDULER_SLACK_SECONDS +) + +# The smallest adoption silence that satisfies inequality (1) is just above +# 1.95s with the budgets above. 2.0s is the next round value and leaves 0.05s +# of stability surplus above the healthy-gap bound. +DEFAULT_WRITER_LEASE_ADOPTION_SILENCE_SECONDS = 2.0 + +# The monitor wake-delay lateness observed on union-mainnet at 2026-09-04 +# (issue #227), the production evidence that the slack allowance is an +# assumption rather than a bound. Kept as a named constant so the tests that +# judge the policy against it cannot drift from the docstring. +OBSERVED_MONITOR_WAKE_DELAY_SECONDS_227 = 0.648 + +# Heartbeat modes, and the phase names the attribution and metrics use. +# Fixed vocabularies: metric label cardinality must not grow with traffic. +LEASE_HEARTBEAT_MODE_PROOF = "proof" +LEASE_HEARTBEAT_MODE_RENEW = "renew" +LEASE_HEARTBEAT_MODE_FENCE = "fence" +LEASE_HEARTBEAT_MODES = ( + LEASE_HEARTBEAT_MODE_PROOF, + LEASE_HEARTBEAT_MODE_RENEW, + LEASE_HEARTBEAT_MODE_FENCE, +) + +# guard_slot_wait + guard_sql + scheduler_delay = total. guard_client_resume +# is the part of scheduler_delay that fell inside a round trip (the answer +# had left PostgreSQL but this process had not resumed); it is reported +# separately because it is exactly the delay issue #227's client-side +# attribution could not see. +LEASE_HEARTBEAT_PHASES = ( + "guard_slot_wait", + "guard_sql", + "guard_client_resume", + "scheduler_delay", + "total", +) + +# The policy terms the metrics surface exports, in derivation order. Closed +# set: adding a term is a deliberate schema change, not a runtime accident. +LEASE_HEARTBEAT_POLICY_TERMS = ( + "adoption_silence", + "heartbeat_interval", + "failure_budget", + "monitor_interval", + "exit_margin", + "guard_statement_timeout", + "scheduler_slack", + "exit_envelope", + "server_proven_cap", + "max_healthy_server_gap", + "stability_surplus", + "max_guaranteed_monitor_lateness", +) + +LEASE_HEARTBEAT_OUTCOME_PROVEN = "proven" +LEASE_HEARTBEAT_OUTCOME_RENEWED = "renewed" +LEASE_HEARTBEAT_OUTCOME_RENEWAL_DUE = "renewal_due" +LEASE_HEARTBEAT_OUTCOME_DEFERRED = "deferred" +LEASE_HEARTBEAT_OUTCOME_FAILED = "failed" +LEASE_HEARTBEAT_OUTCOMES = ( + LEASE_HEARTBEAT_OUTCOME_PROVEN, + LEASE_HEARTBEAT_OUTCOME_RENEWED, + LEASE_HEARTBEAT_OUTCOME_RENEWAL_DUE, + LEASE_HEARTBEAT_OUTCOME_DEFERRED, + LEASE_HEARTBEAT_OUTCOME_FAILED, +) + +# Monitor wake-delay telemetry (issue #227). Every set below is closed, so +# the rendered cardinality is fixed: the histogram has these buckets and +# +Inf, the late-wake counters carry exactly these slack fractions, and the +# rolling window is a fixed number of slices. +# +# The buckets straddle the shipped policy's decision points: 0.25 (half the +# slack, the stall-probe trigger), 0.4 (0.8x slack), 0.5 (the slack itself), +# and 0.75 / 1.0 (the region the 0.648s production record fell in). +LEASE_MONITOR_WAKE_DELAY_BUCKETS = ( + 0.001, + 0.005, + 0.01, + 0.025, + 0.05, + 0.1, + 0.25, + 0.4, + 0.5, + 0.75, + 1.0, + 2.0, +) +# Label values are the fraction text, so a PromQL author can read +# `slack_fraction="1.0"` as "wakes at least a whole scheduler_slack late". +LEASE_MONITOR_LATE_WAKE_SLACK_FRACTIONS = ("0.5", "0.8", "1.0") +# The rolling-window maximum falls back once a stall ages out of the window, +# unlike the lifetime gauge. Thirty 10s slices give a 5-minute window, the +# same horizon the qbit-tools alert's changes[5m] clause reasons over. +LEASE_MONITOR_WAKE_DELAY_WINDOW_SECONDS = 300.0 +LEASE_MONITOR_WAKE_DELAY_WINDOW_SLICES = 30 +# The stall probe samples the running threads' stacks when a wake is at +# least this fraction of the slack late. Rate-limited so it can never turn a +# stall into a stall storm: at most MAX_SAMPLES per WINDOW, the rest are +# counted as suppressed. +LEASE_MONITOR_STALL_PROBE_TRIGGER_SLACK_FRACTION = 0.5 +LEASE_MONITOR_STALL_PROBE_MAX_SAMPLES_PER_WINDOW = 3 +LEASE_MONITOR_STALL_PROBE_WINDOW_SECONDS = 60.0 + + +class WriterLeaseHeartbeatPolicyError(ValueError): + """An unsafe heartbeat timing policy, refused at startup.""" + + +@dataclass(frozen=True, slots=True) +class WriterLeaseHeartbeatPolicy: + """The five coupled heartbeat timings plus the phases that justify them. + + Constructed from live configuration at startup and from the ledger's + own adoption-silence value, so an operator override reaches the same + inequality the compiled defaults satisfy. Never mutated: a policy is + resolved once per heartbeat start and once per monitor loop. + """ + + adoption_silence_seconds: float + heartbeat_interval_seconds: float + failure_budget_seconds: float + monitor_interval_seconds: float + exit_margin_seconds: float + guard_statement_timeout_seconds: float = ( + WRITER_LEASE_GUARD_STATEMENT_TIMEOUT_SECONDS + ) + scheduler_slack_seconds: float = ( + WRITER_LEASE_HEARTBEAT_SCHEDULER_SLACK_SECONDS + ) + + @property + def exit_envelope_seconds(self) -> float: + """Wall-clock the monitor needs to notice and finish a hard exit. + + Four terms, and leaving any of them out is a split-brain window: + + ``exit_margin`` + the hard exit's own budget once the decision is taken. + one ``monitor_interval`` + poll granularity. The monitor can be a full period past the + instant the cap actually elapsed before it looks. + ``scheduler_slack`` + the monitor thread's own budgeted lateness. The monitor is a + Python thread in the same process, behind the same GIL, as the + heartbeat whose stalls this policy explicitly budgets for; it + cannot be assumed to wake on time when the heartbeat is not. + Omitting this term is what made the first cut of this policy + unsafe: with the shipped budgets a maximally late monitor + observed the stale edge 0.45s *after* the successor's adoption + edge, so a stall the policy said was acceptable produced two + live writers. + one further ``monitor_interval`` + the strictness reserve. Without it the worst-case exit lands + exactly on the adoption edge rather than before it, and the + guarantee is "not after" instead of "before". + + Inequality (1) requires all of this to fit inside the adoption + silence alongside the staleness cap. + """ + return ( + self.exit_margin_seconds + + 2.0 * self.monitor_interval_seconds + + self.scheduler_slack_seconds + ) + + @property + def server_proven_cap_seconds(self) -> float: + """Staleness cap the monitor enforces on completed round trips. + + The whole remaining silence budget after the exit envelope, floored + at the failure budget so a deliberately tiny (advised) silence + degrades to the plain liveness bound rather than killing lawful + single-statement verifications. + """ + return max( + self.failure_budget_seconds, + self.adoption_silence_seconds - self.exit_envelope_seconds, + ) + + @property + def max_healthy_server_gap_seconds(self) -> float: + """Largest gap between round trips a healthy coordinator produces.""" + return ( + self.heartbeat_interval_seconds + + self.guard_statement_timeout_seconds + + self.scheduler_slack_seconds + ) + + @property + def stability_surplus_seconds(self) -> float: + """Headroom the staleness cap keeps above the healthy-gap bound.""" + return ( + self.server_proven_cap_seconds + - self.max_healthy_server_gap_seconds + ) + + @property + def max_guaranteed_monitor_lateness_seconds(self) -> float: + """Largest real monitor lateness for which exit precedes adoption. + + Inequality (4) of the module docstring. The monitor that first + observes the stale edge does so no later than ``p + cap + + monitor_interval + L`` and exits ``exit_margin`` later; the + successor is eligible no earlier than ``p + adoption_silence``. So + the exit is guaranteed to precede adoption exactly when ``L`` is + below this value, which equals ``scheduler_slack + + monitor_interval`` whenever the cap is not floored at the failure + budget. A wake later than this is an exit-guarantee breach for + that beat, whether or not a guard loss coincided. + """ + return max( + 0.0, + self.adoption_silence_seconds + - self.server_proven_cap_seconds + - self.monitor_interval_seconds + - self.exit_margin_seconds, + ) + + def exit_guarantee_overrun_seconds(self, monitor_lateness_seconds: float) -> float: + """How far one monitor wake pushed the worst-case exit past adoption. + + Zero while the wake is within + :attr:`max_guaranteed_monitor_lateness_seconds`; otherwise the time + by which the process could still have been alive after the + successor's adoption edge on that beat, had its guard session died. + """ + return max( + 0.0, + float(monitor_lateness_seconds) + - self.max_guaranteed_monitor_lateness_seconds, + ) + + def violations(self) -> tuple[str, ...]: + """Reasons this policy cannot guarantee exit before adoption.""" + reasons: list[str] = [] + for name in ( + "adoption_silence_seconds", + "heartbeat_interval_seconds", + "failure_budget_seconds", + "monitor_interval_seconds", + "exit_margin_seconds", + "guard_statement_timeout_seconds", + "scheduler_slack_seconds", + ): + value = float(getattr(self, name)) + if not math.isfinite(value) or value < 0.0: + reasons.append(f"{name} must be finite and non-negative, got {value!r}") + if reasons: + return tuple(reasons) + if self.heartbeat_interval_seconds <= 0.0: + reasons.append("heartbeat_interval_seconds must be positive") + if self.monitor_interval_seconds <= 0.0: + reasons.append("monitor_interval_seconds must be positive") + if self.adoption_silence_seconds <= 0.0: + reasons.append("adoption_silence_seconds must be positive") + if self.failure_budget_seconds <= self.heartbeat_interval_seconds: + reasons.append( + "failure budget " + f"{self.failure_budget_seconds:g}s must exceed the heartbeat " + f"interval {self.heartbeat_interval_seconds:g}s, or one idle " + "wait alone exhausts the liveness budget" + ) + if ( + self.adoption_silence_seconds + <= self.failure_budget_seconds + self.exit_envelope_seconds + ): + reasons.append( + "adoption silence " + f"{self.adoption_silence_seconds:g}s must exceed the failure " + f"budget {self.failure_budget_seconds:g}s plus the exit " + f"envelope {self.exit_envelope_seconds:g}s (exit margin " + f"{self.exit_margin_seconds:g}s + 2 x monitor interval " + f"{self.monitor_interval_seconds:g}s + scheduler slack " + f"{self.scheduler_slack_seconds:g}s), or a coordinator that " + "lost its guarded session may still be live when a " + "replacement becomes adoption-eligible" + ) + return tuple(reasons) + + def advisories(self) -> tuple[str, ...]: + """Reasons this policy is safe but will false-exit under tail latency.""" + if self.violations(): + return () + if self.stability_surplus_seconds >= 0.0: + return () + return ( + "server-proven staleness cap " + f"{self.server_proven_cap_seconds:g}s is below the largest gap a " + "healthy coordinator can leave between completed guard round " + f"trips ({self.max_healthy_server_gap_seconds:g}s = heartbeat " + f"interval {self.heartbeat_interval_seconds:g}s + guard statement " + f"timeout {self.guard_statement_timeout_seconds:g}s + scheduler " + f"slack {self.scheduler_slack_seconds:g}s); ordinary verification " + "tail latency will hard-exit a healthy coordinator (issue #212). " + "Raise the adoption silence or lower the heartbeat interval", + ) + + def validate(self) -> None: + """Raise :class:`WriterLeaseHeartbeatPolicyError` when unsafe.""" + reasons = self.violations() + if reasons: + raise WriterLeaseHeartbeatPolicyError( + "writer lease heartbeat timing policy is unsafe: " + + "; ".join(reasons) + ) + + def describe(self) -> str: + """One line naming every term of the inequality, for logs and exits.""" + return ( + f"adoption_silence={self.adoption_silence_seconds:g}s " + f"interval={self.heartbeat_interval_seconds:g}s " + f"failure_budget={self.failure_budget_seconds:g}s " + f"monitor={self.monitor_interval_seconds:g}s " + f"exit_margin={self.exit_margin_seconds:g}s " + f"guard_statement_timeout={self.guard_statement_timeout_seconds:g}s " + f"scheduler_slack={self.scheduler_slack_seconds:g}s " + f"exit_envelope={self.exit_envelope_seconds:g}s " + f"=> server_proven_cap={self.server_proven_cap_seconds:g}s " + f"healthy_gap={self.max_healthy_server_gap_seconds:g}s " + f"surplus={self.stability_surplus_seconds:g}s " + f"max_monitor_lateness={self.max_guaranteed_monitor_lateness_seconds:g}s" + ) + + +DEFAULT_WRITER_LEASE_HEARTBEAT_POLICY = WriterLeaseHeartbeatPolicy( + adoption_silence_seconds=DEFAULT_WRITER_LEASE_ADOPTION_SILENCE_SECONDS, + heartbeat_interval_seconds=WRITER_LEASE_HEARTBEAT_INTERVAL_SECONDS, + failure_budget_seconds=WRITER_LEASE_HEARTBEAT_FAILURE_SECONDS, + monitor_interval_seconds=WRITER_LEASE_HEARTBEAT_MONITOR_SECONDS, + exit_margin_seconds=WRITER_LEASE_HEARTBEAT_EXIT_MARGIN_SECONDS, +) +# The compiled defaults must satisfy the inequality they document. A future +# edit that breaks it fails at import rather than in production. +DEFAULT_WRITER_LEASE_HEARTBEAT_POLICY.validate() +assert not DEFAULT_WRITER_LEASE_HEARTBEAT_POLICY.advisories() + + +@dataclass(frozen=True, slots=True) +class WriterLeaseVerificationPhases: + """Where one guard verification's wall-clock actually went. + + The operator question issue #212 could not answer was "which phase + consumed the safety envelope?". Every heartbeat attempt now finishes + with this breakdown, the monitor's hard exit quotes the last one, and + the metrics surface exposes it with fixed cardinality. + + ``slot_wait + guard_sql + scheduler_delay == total``. + ``client_resume`` is the part of ``scheduler_delay`` that fell inside a + statement's round trip (issue #227): PostgreSQL had answered, or the + answer was in transit, and this process had not yet resumed. It is + only non-zero when the statement reported its own server-side + execution time (``server_reported``); otherwise ``guard_sql`` is the + whole client-measured span, as it was before #227, which is the + conservative reading. + """ + + mode: str + outcome: str + slot_wait_seconds: float + guard_sql_seconds: float + statement_count: int + scheduler_delay_seconds: float + total_seconds: float + client_resume_seconds: float = 0.0 + server_reported: bool = False + + def summary(self) -> str: + return ( + f"mode={self.mode} outcome={self.outcome} " + f"slot_wait={self.slot_wait_seconds:.3f}s " + f"guard_sql={self.guard_sql_seconds:.3f}s" + f"({self.statement_count} stmt" + f"{', server-timed' if self.server_reported else ''}) " + f"client_resume={self.client_resume_seconds:.3f}s " + f"scheduler={self.scheduler_delay_seconds:.3f}s " + f"total={self.total_seconds:.3f}s" + ) + + def phase_seconds(self) -> dict[str, float]: + """Fixed-key phase map; keys are :data:`LEASE_HEARTBEAT_PHASES`.""" + return { + "guard_slot_wait": self.slot_wait_seconds, + "guard_sql": self.guard_sql_seconds, + "guard_client_resume": self.client_resume_seconds, + "scheduler_delay": self.scheduler_delay_seconds, + "total": self.total_seconds, + } + + +UNATTRIBUTED_PHASES = WriterLeaseVerificationPhases( + mode="none", + outcome="none", + slot_wait_seconds=0.0, + guard_sql_seconds=0.0, + statement_count=0, + scheduler_delay_seconds=0.0, + total_seconds=0.0, +) + + +class WriterLeaseVerificationAttempt: + """Monotonic phase accumulator for one guard verification attempt. + + Written only by the thread running the attempt and read only after + :meth:`finish` publishes an immutable snapshot, so it needs no lock. + Marks are best effort: a verifier that reports no slot acquisition and + no statement completion still yields a truthful total, with the whole + duration landing in ``scheduler_delay`` — which is itself the signal + that the attempt never reached PostgreSQL. + """ + + __slots__ = ( + "mode", + "_monotonic", + "_started", + "_slot_acquired", + "_last_statement_end", + "_statement_count", + "_pending_send", + "_proven_edge", + "_server_seconds", + "_server_reported", + ) + + def __init__( + self, + mode: str, + *, + monotonic: Callable[[], float] | None = None, + ) -> None: + self.mode = mode + # Resolved at construction, not bound as a default argument: tests + # that drive the lease lifecycle patch ``time.monotonic`` on the + # module object, and a default captured at import time would keep + # reading the real clock behind their backs. + self._monotonic = monotonic or time.monotonic + self._started = self._monotonic() + self._slot_acquired: float | None = None + self._last_statement_end: float | None = None + self._statement_count = 0 + # The send edge of the statement currently in flight, and the + # conservative "PostgreSQL was alive no earlier than this" edge of + # the newest completed round trip. Both default to the attempt's + # start so a verifier that reports no marks still yields an edge + # that cannot overstate freshness. + self._pending_send = self._started + self._proven_edge = self._started + # Server-reported execution time, summed over the statements that + # reported one (issue #227). Kept apart from the client marks: it + # refines the attribution and never touches the proven edge. + self._server_seconds = 0.0 + self._server_reported = False + + @property + def started_monotonic(self) -> float: + return self._started + + @property + def proven_edge_monotonic(self) -> float: + """Conservative server-proven edge of the newest completed round trip.""" + return self._proven_edge + + def slot_acquired(self) -> None: + """The guard session's serialized query slot was handed over. + + This is also the send edge of the attempt's first statement: the + guard issues it immediately after handing over the slot. + """ + if self._slot_acquired is None: + self._slot_acquired = self._monotonic() + self._pending_send = self._slot_acquired + + def statement_completed(self) -> float: + """One guarded statement's round trip returned a result. + + Returns the conservative server-proven edge for that round trip — + when the statement was sent, not when the answer was handled — and + arms the next statement's send edge, since a followup leaves for + PostgreSQL as soon as this result is in hand. + """ + now = self._monotonic() + self._proven_edge = self._pending_send + self._pending_send = now + self._last_statement_end = now + self._statement_count += 1 + return self._proven_edge + + def statement_server_seconds(self, seconds: float | None) -> None: + """One guarded statement reported its own server-side execution time. + + ``seconds`` is ``clock_timestamp() - statement_timestamp()`` as + evaluated by the statement itself, so it excludes the answer's + transit and everything this process did after receipt. ``None`` + (a statement that does not report, or a malformed value) leaves + the attribution on its conservative client-side reading. Never + consulted for the server-proven edge: a server clock reading is + not a monotonic instant in this process, and the exit-ordering + proof needs the send edge. + """ + if seconds is None: + return + try: + value = float(seconds) + except (TypeError, ValueError): + return + if not math.isfinite(value) or value < 0.0: + return + self._server_seconds += value + self._server_reported = True + + def finish(self, outcome: str) -> WriterLeaseVerificationPhases: + finished = self._monotonic() + total = max(0.0, finished - self._started) + slot_wait = ( + max(0.0, self._slot_acquired - self._started) + if self._slot_acquired is not None + else 0.0 + ) + # Statement 1 spans [slot acquisition, first response]; statement k + # spans [response k-1, response k]. Without server-reported timing + # the sub-millisecond client gap between them is charged to SQL, + # which keeps the attribution conservative rather than flattering + # the database. With it, guard_sql is the server's own execution + # time (clamped to the span it must physically fit inside) and the + # rest of the span is client resume: transit plus this process's + # delay in handling an answer PostgreSQL had already given, which + # is the GIL stall issue #227 could not see. + if self._last_statement_end is not None and self._slot_acquired is not None: + statement_span = max(0.0, self._last_statement_end - self._slot_acquired) + else: + statement_span = 0.0 + if self._server_reported: + guard_sql = min(statement_span, self._server_seconds) + client_resume = statement_span - guard_sql + else: + guard_sql = statement_span + client_resume = 0.0 + return WriterLeaseVerificationPhases( + mode=self.mode, + outcome=outcome, + slot_wait_seconds=slot_wait, + guard_sql_seconds=guard_sql, + statement_count=self._statement_count, + scheduler_delay_seconds=max(0.0, total - slot_wait - guard_sql), + total_seconds=total, + client_resume_seconds=client_resume, + server_reported=self._server_reported, + ) diff --git a/scripts/check-env.sh b/scripts/check-env.sh index 570fd2fb3..8de1c3f95 100644 --- a/scripts/check-env.sh +++ b/scripts/check-env.sh @@ -27,6 +27,7 @@ ENV_QBIT_DATA_SOURCE="${QBIT_DATA_SOURCE:-}" ENV_BITCOIN_DATA_SOURCE="${BITCOIN_DATA_SOURCE:-}" ENV_PRISM_POSTGRES_DATA_SOURCE="${PRISM_POSTGRES_DATA_SOURCE:-}" ENV_PRISM_POSTGRES_WAL_SOURCE="${PRISM_POSTGRES_WAL_SOURCE:-}" +ENV_PRISM_POSTGRES_REPLICA_DATA_SOURCE="${PRISM_POSTGRES_REPLICA_DATA_SOURCE:-}" ENV_PRISM_AUDIT_DATA_SOURCE="${PRISM_AUDIT_DATA_SOURCE:-}" ENV_QBIT_CHAIN="${QBIT_CHAIN:-}" ENV_QBIT_CHAIN_FLAG="${QBIT_CHAIN_FLAG:-}" @@ -165,6 +166,7 @@ for name in \ BITCOIN_DATA_SOURCE \ PRISM_POSTGRES_DATA_SOURCE \ PRISM_POSTGRES_WAL_SOURCE \ + PRISM_POSTGRES_REPLICA_DATA_SOURCE \ PRISM_AUDIT_DATA_SOURCE; do override_name="ENV_${name}" if [[ -n "${!override_name}" ]]; then @@ -595,22 +597,40 @@ check_absolute_storage_source() { check_distinct_storage_sources() { local names=("$@") - local left right - local left_name right_name - local left_value right_value - - for ((left = 0; left < ${#names[@]}; left++)); do - left_name="${names[left]}" - left_value="${!left_name}" - left_value="${left_value%/}" - for ((right = left + 1; right < ${#names[@]}; right++)); do - right_name="${names[right]}" - right_value="${!right_name}" - right_value="${right_value%/}" - [[ "${left_value}" != "${right_value}" ]] || \ - fail "release provenance storage sources ${left_name} and ${right_name} must be distinct" - done + local args=() name output + command -v python3 >/dev/null 2>&1 || fail "python3 is required to validate storage paths" + for name in "${names[@]}"; do + args+=("${name}" "${!name}") done + if ! output="$(python3 - "${args[@]}" <<'PYTHON' +from pathlib import Path +import sys + +sources = [] +try: + for name, raw in zip(sys.argv[1::2], sys.argv[2::2]): + path = Path(raw).resolve() + identities = [] + for parent in (path, *path.parents): + try: + stat = parent.stat() + identities.append((stat.st_dev, stat.st_ino)) + except FileNotFoundError: + identities.append(None) + for other_name, other_path, other_ids in sources: + if (path == other_path or path in other_path.parents + or other_path in path.parents + or identities[0] is not None and identities[0] in other_ids + or other_ids[0] is not None and other_ids[0] in identities): + sys.exit(f"release provenance storage sources {other_name} and {name} " + "must be distinct and non-overlapping (including filesystem aliases)") + sources.append((name, path, identities)) +except (OSError, RuntimeError): + sys.exit(f"cannot safely resolve release provenance storage source {name}") +PYTHON + )"; then + fail "${output}" + fi } check_production_deployment_inputs() { @@ -632,10 +652,12 @@ check_production_deployment_inputs() { check_digest_image PRISM_POSTGRES_IMAGE check_absolute_storage_source PRISM_POSTGRES_DATA_SOURCE check_absolute_storage_source PRISM_POSTGRES_WAL_SOURCE + check_absolute_storage_source PRISM_POSTGRES_REPLICA_DATA_SOURCE check_absolute_storage_source PRISM_AUDIT_DATA_SOURCE storage_names+=( PRISM_POSTGRES_DATA_SOURCE PRISM_POSTGRES_WAL_SOURCE + PRISM_POSTGRES_REPLICA_DATA_SOURCE PRISM_AUDIT_DATA_SOURCE ) fi @@ -744,9 +766,6 @@ check_production_gate() { fail "production mode rejects ${name}=1" fi done - if [[ "${QBIT_CHAIN:-regtest}" == "mainnet" ]]; then - [[ "${PRISM_STRATUM_STALE_GRACE_SECONDS:-3}" == "0" ]] || fail "mainnet requires PRISM_STRATUM_STALE_GRACE_SECONDS=0" - fi [[ -n "${PRISM_DATABASE_URL:-}" || -n "${PRISM_POSTGRES_PSQL_COMMAND:-}" ]] || fail "production mode requires PRISM_DATABASE_URL or PRISM_POSTGRES_PSQL_COMMAND" [[ "${PRISM_POSTGRES_PASSWORD:-}" != "change-this" ]] || fail "production mode requires a non-default PRISM_POSTGRES_PASSWORD" [[ "${PRISM_DATABASE_URL:-}" != *"change-this"* ]] || fail "production mode requires a non-default PRISM_DATABASE_URL" @@ -900,6 +919,35 @@ check_bitcoin_chain_selection() { esac } +check_prism_stale_grace() { + mining_lane_enabled prism || return 0 + + if ! command -v python3 >/dev/null 2>&1; then + # Production already requires python3 (check_prism_production_difficulty); + # lab bring-up does not, so skip rather than add a prerequisite there. + if production_mode_enabled; then + fail "python3 is required to validate PRISM_STRATUM_STALE_GRACE_SECONDS" + fi + printf 'doctor: python3 not found; skipping PRISM_STRATUM_STALE_GRACE_SECONDS validation\n' + return 0 + fi + # Mirror the coordinator's env_nonnegative_float(): the runtime's float + # syntax and range, finite and non-negative. + if ! python3 - "${PRISM_STRATUM_STALE_GRACE_SECONDS:-3}" >/dev/null 2>&1 <<'PY' +import math +import sys + +try: + value = float(sys.argv[1]) +except ValueError: + raise SystemExit(1) +raise SystemExit(0 if math.isfinite(value) and value >= 0 else 1) +PY + then + fail "PRISM_STRATUM_STALE_GRACE_SECONDS must be a finite non-negative number" + fi +} + check_ctv_fee_config() { mining_lane_enabled prism || return 0 is_true_env "${PRISM_CTV_SETTLEMENT_ENABLED:-0}" || return 0 @@ -965,6 +1013,7 @@ if mining_lane_enabled auxpow; then check_bool_env BITCOIN_DNSSEED check_bool_env BITCOIN_DISCOVER fi +check_prism_stale_grace check_ctv_fee_config check_production_gate diff --git a/scripts/prism-self-check.py b/scripts/prism-self-check.py index 30e1ca5b5..9fae29259 100755 --- a/scripts/prism-self-check.py +++ b/scripts/prism-self-check.py @@ -6,6 +6,7 @@ import argparse import base64 import json +import math import os import shutil import socket @@ -492,22 +493,16 @@ def static_checks(env: dict[str, str], reporter: Reporter) -> None: except ValueError as exc: reporter.fail("mining.share_diff", str(exc)) + # Match the coordinator's env_nonnegative_float(): the runtime's float + # syntax and range, finite and non-negative (fractional seconds included). stale_grace = env_value(env, "PRISM_STRATUM_STALE_GRACE_SECONDS", "3").strip() - if qbit_chain == "mainnet" and stale_grace != "0": - reporter.fail( - "mining.stale_grace", - "mainnet requires PRISM_STRATUM_STALE_GRACE_SECONDS=0", - hint="Enable stale-credit grace only after proving verifier compatibility for the deployed release.", - ) - else: - try: - stale_grace_value = int(stale_grace) - if stale_grace_value < 0: - raise ValueError - except ValueError: - reporter.fail("mining.stale_grace", "PRISM_STRATUM_STALE_GRACE_SECONDS must be a non-negative integer") - else: - reporter.pass_("mining.stale_grace", f"{stale_grace_value} seconds") + try: + stale_grace_value = float(stale_grace) + if not math.isfinite(stale_grace_value) or stale_grace_value < 0: + raise ValueError("stale grace must be a finite non-negative number") + reporter.pass_("mining.stale_grace", f"{stale_grace_value:g} seconds") + except ValueError: + reporter.fail("mining.stale_grace", "PRISM_STRATUM_STALE_GRACE_SECONDS must be a finite non-negative number") if prod and bitcoin_chain != "regtest": qbit_miner_address = env_value(env, "QBIT_MINER_ADDRESS", "auto").strip() diff --git a/scripts/run_python_test_shard.py b/scripts/run_python_test_shard.py new file mode 100644 index 000000000..08b64246f --- /dev/null +++ b/scripts/run_python_test_shard.py @@ -0,0 +1,57 @@ +#!/usr/bin/env python3 +"""Run one deterministic shard of the standard Python unittest discovery suite. + +Discovery sorts module paths; distribute its top-level suites round-robin so +module/class fixtures stay together and new test modules join CI automatically. +""" + +from __future__ import annotations + +import argparse +import os +from pathlib import Path +import sys +import unittest + + +ROOT = Path(__file__).resolve().parents[1] + + +def select_shard( + suite: unittest.TestSuite, shard_index: int, shard_count: int +) -> unittest.TestSuite: + if shard_count < 1 or not 0 <= shard_index < shard_count: + raise ValueError("require shard-count > 0 and 0 <= shard-index < shard-count") + modules = [module for module in suite if module.countTestCases()] + selected = unittest.TestSuite(modules[shard_index::shard_count]) + if selected.countTestCases() == 0: + raise ValueError("shard contains no tests; reduce shard-count") + return selected + + +def main(argv: list[str] | None = None) -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--shard-index", type=int, required=True) + parser.add_argument("--shard-count", type=int, required=True) + args = parser.parse_args(argv) + + # Match `python -m unittest discover -s tests -p 'test_*.py'` even when + # invoked by absolute script path from another working directory. + os.chdir(ROOT) + sys.path.insert(0, str(ROOT)) + suite = unittest.TestLoader().discover("tests", pattern="test_*.py") + try: + selected = select_shard(suite, args.shard_index, args.shard_count) + except ValueError as error: + parser.error(str(error)) + print( + f"Python test shard {args.shard_index}/{args.shard_count}: " + f"{selected.countTestCases()} of {suite.countTestCases()} tests", + flush=True, + ) + result = unittest.TextTestRunner(verbosity=2).run(selected) + return 0 if result.wasSuccessful() else 1 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/test/test-prism-postgres-candidate-storage.sh b/test/test-prism-postgres-candidate-storage.sh new file mode 100755 index 000000000..33d21f5d2 --- /dev/null +++ b/test/test-prism-postgres-candidate-storage.sh @@ -0,0 +1,65 @@ +#!/usr/bin/env bash +# Real-PostgreSQL gate for chunked candidate bodies (issue #255). Provisions a +# disposable postgres container and drives tests/prism_postgres_candidate_gate.py +# through the psql subprocess backend and, when the host python has psycopg, +# the native pooled client too. +set -euo pipefail + +ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +POSTGRES_IMAGE="${QBIT_PRISM_POSTGRES_IMAGE:-postgres:16-alpine}" +POSTGRES_CONTAINER="${QBIT_PRISM_POSTGRES_CONTAINER:-qbit-prism-candidate-pg-$$}" + +require_executable() { + command -v "$1" >/dev/null 2>&1 || { + echo "missing required executable: $1" >&2 + exit 1 + } +} + +cleanup() { + docker rm -f "${POSTGRES_CONTAINER}" >/dev/null 2>&1 || true +} +trap cleanup EXIT + +require_executable docker +require_executable python3 + +docker rm -f "${POSTGRES_CONTAINER}" >/dev/null 2>&1 || true +docker run \ + --rm \ + --detach \ + --name "${POSTGRES_CONTAINER}" \ + -p 127.0.0.1:0:5432 \ + -e POSTGRES_USER=qbit \ + -e POSTGRES_PASSWORD=qbit \ + -e POSTGRES_DB=qbit \ + "${POSTGRES_IMAGE}" >/dev/null + +deadline=$((SECONDS + 60)) +until docker exec "${POSTGRES_CONTAINER}" pg_isready -h 127.0.0.1 -U qbit -d qbit >/dev/null 2>&1; do + if [[ "${SECONDS}" -ge "${deadline}" ]]; then + echo "timed out waiting for PRISM Postgres container" >&2 + docker logs "${POSTGRES_CONTAINER}" >&2 || true + exit 1 + fi + sleep 1 +done + +HOST_PORT="$(docker port "${POSTGRES_CONTAINER}" 5432/tcp | head -n 1 | awk -F: '{print $NF}')" +DATABASE_URL="" +if [[ -n "${HOST_PORT}" ]]; then + DATABASE_URL="postgresql://qbit:qbit@127.0.0.1:${HOST_PORT}/qbit" +fi + +( + cd "${ROOT_DIR}" + PRISM_TEST_PSQL_COMMAND="docker exec -i ${POSTGRES_CONTAINER} psql -U qbit -d qbit" \ + PRISM_TEST_DATABASE_URL="${DATABASE_URL}" \ + python3 -m tests.prism_postgres_candidate_gate + if [[ -n "${DATABASE_URL}" ]] && python3 -c 'import psycopg' >/dev/null 2>&1; then + PRISM_TEST_DATABASE_URL="${DATABASE_URL}" \ + python3 -m unittest -v tests.test_prism_candidate_storage_races + fi +) + +echo "test-prism-postgres-candidate-storage: PASS" diff --git a/test/test-prism-postgres-ledger.sh b/test/test-prism-postgres-ledger.sh index 403c84175..a38573a7e 100644 --- a/test/test-prism-postgres-ledger.sh +++ b/test/test-prism-postgres-ledger.sh @@ -54,9 +54,30 @@ else PSQL_COMMAND="docker exec -i ${POSTGRES_CONTAINER} psql -U qbit -d qbit" fi +# Every PsqlShareLedger session GUC rides PGOPTIONS on the psql path, and +# "docker exec" does not forward the caller's environment, so the container +# form silently drops them. The read-only session assertion at the end needs +# PGOPTIONS to actually arrive, so it runs through a command that asks docker +# to pass the variable through. An external psql inherits the environment +# already and needs no help. +if [[ -n "${EXTERNAL_PSQL}" ]]; then + PSQL_COMMAND_WITH_ENV="${PSQL_COMMAND}" +else + PSQL_COMMAND_WITH_ENV="docker exec -i -e PGOPTIONS ${POSTGRES_CONTAINER} psql -U qbit -d qbit" +fi + +if [[ -n "${EXTERNAL_PSQL}" ]]; then + GATE_IMAGE="external-postgres" + GATE_IMAGE_DIGEST="not-applicable" +else + GATE_IMAGE="$(docker inspect --format '{{.Config.Image}}' "${POSTGRES_CONTAINER}")" + GATE_IMAGE_DIGEST="$(docker inspect --format '{{.Image}}' "${POSTGRES_CONTAINER}")" +fi + ( cd "${ROOT_DIR}" PRISM_PSQL_COMMAND="${PSQL_COMMAND}" \ + PRISM_PSQL_COMMAND_WITH_ENV="${PSQL_COMMAND_WITH_ENV}" \ python3 <<'PY' from __future__ import annotations @@ -67,7 +88,13 @@ import os import tempfile from pathlib import Path -from lab.prism.share_ledger import PendingShare, PsqlShareLedger +from lab.prism.share_ledger import ( + PendingShare, + PsqlShareLedger, + ShareReplayConflict, + SingleWriterShareLedger, +) +from tests.prism_window_rows_reference import assert_rows_match_aggregate def pending( @@ -139,6 +166,34 @@ SELECT json_build_object( assert_equal(int(comparison["drift_count"]), 0, message + " (drift count)") +def share_ledger_identity(runner: PsqlShareLedger) -> dict[str, object]: + """The exact identity of qbit_share_ledger, for the #224 invariant. + + Every column of every row in sequence order, digested together with + each tuple's ctid and xmin -- so an UPDATE that rewrote a row to the + same values still changes it -- plus the sequence allocator, so an + INSERT that rolled back still changes it. + """ + return runner._run_json( + """ +SELECT json_build_object( + 'row_count', (SELECT count(*) FROM qbit_share_ledger), + 'digest', ( + SELECT md5(COALESCE(string_agg( + ctid::text || '|' || xmin::text || '|' || ledger::text, + E'\\n' ORDER BY share_seq + ), '')) + FROM qbit_share_ledger AS ledger + ), + 'sequence', ( + SELECT json_build_object('last_value', last_value, 'is_called', is_called) + FROM qbit_share_ledger_share_seq_seq + ) +); +""" + ) + + def force_expired_idle_lease(runner: PsqlShareLedger) -> None: runner._run_sql( """ @@ -150,7 +205,10 @@ SET updated_at = clock_timestamp() - interval '6 minutes', psql = os.environ["PRISM_PSQL_COMMAND"] +psql_with_env = os.environ["PRISM_PSQL_COMMAND_WITH_ENV"] ledger = PsqlShareLedger( + # This gate mutates legacy JSONB fixtures; the storage gate covers v2. + candidate_storage_version=1, psql_command=psql, writer_id="writer-a", writer_epoch=1, @@ -190,12 +248,53 @@ candidate_intent = { candidate_row = ledger.append_batch([(batch_c, candidate_intent)])[0] assert_equal(candidate_row.share_seq, 3, "candidate share sequence") assert_equal(ledger.append_batch([(batch_c, candidate_intent)])[0].share_seq, 3, "exact replay") +recovery_exact = ledger.append_recovered_share(batch_c) +assert_equal(recovery_exact.disposition, "exact_existing", "typed recovery exact replay") +assert_equal(recovery_exact.record.share_seq, 3, "typed recovery exact replay keeps sequence") +try: + ledger.append_recovered_share( + PendingShare(**{**batch_c.__dict__, "ntime": batch_c.ntime + 1}) + ) +except ShareReplayConflict: + pass +else: + raise SystemExit("typed recovery payload conflict was not rejected") assert_equal(ledger.pending_block_candidates(), [candidate_intent], "pending candidate replay") +pending_rows = ledger.pending_block_candidate_rows() assert_equal( - ledger.pending_block_candidate_rows(), - [{"block_hash": "ab" * 32, "candidate": candidate_intent}], + [ + {key: value for key, value in row.items() if key != "cursor"} + for row in pending_rows + ], + [ + { + "block_hash": "ab" * 32, + "candidate": candidate_intent, + "pool_block_exists": False, + } + ], "pending candidate replay retains authoritative outbox key", ) +# Startup enumeration pages this read with a keyset cursor, so the cursor +# has to round-trip through the server exactly: a truncated stamp would +# make a page re-emit or skip its equal-timestamp peers. +assert_equal( + ledger.pending_block_candidate_rows(after_cursor=pending_rows[0]["cursor"]), + [], + "pending candidate cursor resumes strictly after its own row", +) +cursor_stamp, cursor_hash = pending_rows[0]["cursor"] +assert_equal(cursor_hash, "ab" * 32, "pending candidate cursor carries its row key") +assert_equal( + ledger._run_json( + "SELECT json_build_object('matched', (" + "SELECT count(*) FROM qbit_block_candidate_outbox " + f"WHERE created_at = '{cursor_stamp}'::timestamptz " + f"AND block_hash = '{cursor_hash}'))" + )["matched"], + 1, + "pending candidate cursor stamp round-trips at full precision", +) assert ledger.mark_block_candidate_submitted(block_hash="ab" * 32) assert_equal(ledger.pending_block_candidates(), [], "submitted candidate leaves pending set") assert_equal(ledger.append_batch([(batch_c, candidate_intent)])[0].share_seq, 3, "compact exact replay") @@ -210,6 +309,14 @@ assert_equal(ledger.pending_block_candidates(), [candidate_only], "candidate-onl assert_equal(ledger.append_batch([(batch_d, candidate_only)])[0].share_seq, 4, "linked solver credit") assert ledger.mark_block_candidate_submitted(block_hash="cd" * 32) assert_equal(ledger.pending_block_candidates(), [], "linked candidate completion") +batch_recovered = pending(94, job_ms=908, accepted_ms=909, share_id="batch-recovered") +recovery_inserted = ledger.append_recovered_share(batch_recovered) +assert_equal(recovery_inserted.disposition, "inserted", "typed recovery inserts a new row") +assert_equal( + recovery_inserted.record.share_id, + "batch-recovered", + "typed recovery returns the inserted record", +) poison_hash = "ef" * 32 poison_candidate = { **candidate_intent, @@ -233,6 +340,160 @@ assert ledger.mark_block_candidate_abandoned( error="invalid durable candidate intent", ) assert_equal(ledger.pending_block_candidates(), [], "poison row quarantined by durable key") + +# The bulk terminal update replaces N single-hash abandonments on the storm +# path, so its fence, its exactly-pending predicate, its pool-block veto and +# its RETURNING-fed report all have to hold against the real table -- where +# state, completed_at and candidate are checked together in SQL and a +# half-applied terminal row would be rejected outright. +bulk_pending_a = "1a" * 32 +bulk_pending_b = "1b" * 32 +bulk_submitted = "1c" * 32 +bulk_landed = "1d" * 32 +bulk_missing = "99" * 32 +for bulk_hash in (bulk_pending_a, bulk_pending_b, bulk_submitted, bulk_landed): + assert ledger.persist_block_candidate_intent( + {**candidate_intent, "block_hash_hex": bulk_hash} + ) +assert ledger.mark_block_candidate_submitted(block_hash=bulk_submitted) +ledger._run_sql( + "INSERT INTO qbit_pool_blocks " + "(block_hash, block_height, parent_hash, coinbase_txid, payout_manifest_sha256) " + f"VALUES ('{bulk_landed}', 11, '{'00' * 32}', '{'11' * 32}', '{'22' * 32}');" +) +# The page read answers the landed-block question for every row it returns, +# so the caller never pays one round trip per pending hash. +assert_equal( + { + row["block_hash"]: row["pool_block_exists"] + for row in ledger.pending_block_candidate_rows(limit=32) + }, + {bulk_pending_a: False, bulk_pending_b: False, bulk_landed: True}, + "pending page reports authoritative pool-block existence", +) +assert_equal( + ledger.mark_block_candidates_abandoned(block_hashes=[], error="unused"), + (), + "empty bulk abandon page is a no-op", +) +bulk_abandoned = ledger.mark_block_candidates_abandoned( + block_hashes=[ + bulk_pending_b, + bulk_pending_a.upper(), + bulk_pending_a, + bulk_submitted, + bulk_landed, + bulk_missing, + ], + error="superseded by decided height", +) +assert_equal( + list(bulk_abandoned), + [bulk_pending_a, bulk_pending_b], + "bulk abandon reports exactly the pending rows it won", +) +assert_equal( + ledger.pending_block_candidates(), + [{**candidate_intent, "block_hash_hex": bulk_landed}], + "bulk abandon leaves the landed candidate pending", +) +bulk_rows = ledger._run_json( + "SELECT json_build_object('rows', COALESCE(json_agg(json_build_object(" + "'block_hash', block_hash, 'state', state, 'last_error', last_error, " + "'candidate_cleared', candidate IS NULL, " + "'completed', completed_at IS NOT NULL) ORDER BY block_hash), '[]'::json)) " + "FROM qbit_block_candidate_outbox WHERE block_hash = ANY(ARRAY[" + f"'{bulk_pending_a}', '{bulk_pending_b}', '{bulk_submitted}', '{bulk_landed}'" + "]::text[]);" +)["rows"] +assert_equal( + bulk_rows, + [ + { + "block_hash": bulk_pending_a, + "state": "abandoned", + "last_error": "superseded by decided height", + "candidate_cleared": True, + "completed": True, + }, + { + "block_hash": bulk_pending_b, + "state": "abandoned", + "last_error": "superseded by decided height", + "candidate_cleared": True, + "completed": True, + }, + { + "block_hash": bulk_submitted, + "state": "submitted", + "last_error": None, + "candidate_cleared": True, + "completed": True, + }, + { + "block_hash": bulk_landed, + "state": "pending", + "last_error": None, + "candidate_cleared": False, + "completed": False, + }, + ], + "bulk abandon writes the terminal column set and spares every other row", +) +assert_equal( + ledger.mark_block_candidates_abandoned( + block_hashes=[bulk_pending_a, bulk_pending_b], + error="second attempt", + ), + (), + "already-terminal rows are not re-won", +) +ledger._run_sql(f"DELETE FROM qbit_pool_blocks WHERE block_hash = '{bulk_landed}';") +# The stale-page case below asserts an exact page and terminal set. Isolate it +# from the pending row intentionally retained by the bulk-abandon case above. +ledger._run_sql("DELETE FROM qbit_block_candidate_outbox;") + +# Issue #211 removed the writer gate from the page read, which makes the page +# advisory by construction: a candidate can land between the snapshot and +# whatever the caller does with it. The case above inserted the pool block +# first, so its page was never stale. This one reads the page *before* the +# landing, so the set handed to the terminal write genuinely disagrees with +# the durable state -- and the veto has to come from the fenced UPDATE's own +# re-check rather than from the fact the caller carried. +stale_pending = "2a" * 32 +stale_landed = "2b" * 32 +for stale_hash in (stale_pending, stale_landed): + assert ledger.persist_block_candidate_intent( + {**candidate_intent, "block_hash_hex": stale_hash} + ) +stale_page = ledger.pending_block_candidate_rows(limit=32) +assert_equal( + {row["block_hash"]: row["pool_block_exists"] for row in stale_page}, + {stale_pending: False, stale_landed: False}, + "a page read before the landing reports no pool block for either row", +) +ledger._run_sql( + "INSERT INTO qbit_pool_blocks " + "(block_hash, block_height, parent_hash, coinbase_txid, payout_manifest_sha256) " + f"VALUES ('{stale_landed}', 12, '{'00' * 32}', '{'33' * 32}', '{'44' * 32}');" +) +assert_equal( + list( + ledger.mark_block_candidates_abandoned( + block_hashes=[str(row["block_hash"]) for row in stale_page], + error="acted on a stale page", + ) + ), + [stale_pending], + "a candidate that landed after the page read is refused by the fence", +) +assert_equal( + [row["block_hash"] for row in ledger.pending_block_candidate_rows(limit=32)], + [stale_landed], + "the landed candidate survived a stale page that named it", +) +ledger._run_sql(f"DELETE FROM qbit_pool_blocks WHERE block_hash = '{stale_landed}';") + ledger._run_sql( """ DELETE FROM qbit_block_candidate_outbox; @@ -258,6 +519,8 @@ WHERE table_name = 'qbit_ctv_fanout_artifacts' assert_equal(legacy_anchor_nullable, "NO", "old schema simulation makes anchor_vout not nullable") ledger.release_writer_lease() ledger = PsqlShareLedger( + # This gate mutates legacy JSONB fixtures; the storage gate covers v2. + candidate_storage_version=1, psql_command=psql, writer_id="writer-a", writer_epoch=1, @@ -685,6 +948,86 @@ assert_equal( ["submitted"], "CTV broadcast attempt is journaled", ) +# A second pending fanout on the same block, with its own attempts, proves the +# public pending-fanout page scopes broadcast_attempts to each row's fanout. +# The persisted manifest set is immutable, so the sibling is a row copy; it is +# removed again so later block-level checks see the original single fanout. +replacement._run_sql( + """ +INSERT INTO qbit_ctv_fanout_artifacts ( + fanout_txid, + block_hash, + manifest_set_sha256, + manifest_json, + manifest, + manifest_sha256, + precommitment_sha256, + ctv_hash, + commitment_witness_leaf_hex, + chunk_index, + chunk_count, + parent_coinbase_txid, + parent_coinbase_vout, + fanout_tx_template_hex, + fanout_tx_hex, + anchor_vout, + covenant_output_value_sats, + fanout_output_sum_sats, + settlement_status +) +SELECT + '""" + "17" * 32 + """', + block_hash, + manifest_set_sha256, + manifest_json, + manifest, + manifest_sha256, + precommitment_sha256, + ctv_hash, + commitment_witness_leaf_hex, + 1, + 2, + parent_coinbase_txid, + parent_coinbase_vout, + fanout_tx_template_hex, + fanout_tx_hex, + anchor_vout, + covenant_output_value_sats, + fanout_output_sum_sats, + 'broadcastable' +FROM qbit_ctv_fanout_artifacts +WHERE fanout_txid = '""" + "12" * 32 + """'; +""" +) +for sibling_package_txid in ("18" * 32, "19" * 32): + replacement.record_ctv_fanout_broadcast_attempt( + fanout_txid="17" * 32, + attempt_status="submitted", + package_tx_hexes=["02"], + package_txids=[sibling_package_txid], + submit_result={"accepted": True}, + ) +pending_fanout_page = replacement.dashboard_pending_fanout_rows(page=1, limit=15) +assert_equal( + [ + [ + row["fanout_txid"], + [attempt["package_txids"] for attempt in row["broadcast_attempts"]], + ] + for row in pending_fanout_page["rows"] + ], + [ + ["12" * 32, [["16" * 32]]], + ["17" * 32, [["18" * 32], ["19" * 32]]], + ], + "pending fanout rows carry only their own broadcast attempts, in order", +) +sibling_attempt_seqs = [attempt["attempt_seq"] for attempt in pending_fanout_page["rows"][1]["broadcast_attempts"]] +assert_equal(sibling_attempt_seqs, sorted(sibling_attempt_seqs), "pending fanout broadcast attempts ascend by attempt_seq") +replacement._run_sql( + "DELETE FROM qbit_ctv_fanout_broadcast_attempts WHERE fanout_txid = '" + "17" * 32 + "';\n" + "DELETE FROM qbit_ctv_fanout_artifacts WHERE fanout_txid = '" + "17" * 32 + "';" +) duplicate_persist = replacement.persist_accepted_block( block_hash="44" * 32, block_height=7, @@ -774,8 +1117,59 @@ WHERE miner_id LIKE 'miner-unanchored-%'; """ ) assert_equal(unanchored_balances, [], "unanchored carry rows require a confirmed pool block") +unknown_confirmation = replacement.confirm_accepted_block( + block_hash="ff" * 32, + active_tip_height=7, +) +assert_equal( + unknown_confirmation, + {"backend": "postgres-psql", "confirmed_count": 0}, + "unknown block confirmation exposes no publication ordinal", +) +wrong_height_confirmation = replacement.confirm_accepted_block( + block_hash="44" * 32, + active_tip_height=8, +) +assert_equal( + wrong_height_confirmation, + {"backend": "postgres-psql", "confirmed_count": 0}, + "wrong-height confirmation exposes no publication ordinal", +) confirmed = replacement.confirm_accepted_block(block_hash="44" * 32, active_tip_height=7) -assert_equal(confirmed["confirmed_count"], 1, "confirmed block count") +assert_equal(confirmed["confirmed_count"], 1, "fresh confirmation reports the flip disposition") +first_publication_sequence = int(confirmed["audit_publication_sequence"]) +if first_publication_sequence <= 0: + raise SystemExit("first confirmed block received a non-positive publication ordinal") +allocator_after_flip = replacement._run_json( + "SELECT json_build_object('last_value', last_value, 'is_called', is_called) " + "FROM qbit_audit_publication_sequence_seq;" +) +confirmed_replay = replacement.confirm_accepted_block( + block_hash="44" * 32, + active_tip_height=7, +) +# The idempotent arm is the durable discriminator this contract exists for: +# the UPDATE matched nothing, so the row keeps the ordinal its flip +# allocated and the caller can tell a replay from a genuinely new +# confirmation that has to publish payout state (issue #61). +assert_equal( + confirmed_replay["confirmed_count"], + 2, + "exact confirmation replay reports the idempotent disposition", +) +assert_equal( + int(confirmed_replay["audit_publication_sequence"]), + first_publication_sequence, + "exact confirmation replay preserves publication ordinal", +) +assert_equal( + replacement._run_json( + "SELECT json_build_object('last_value', last_value, 'is_called', is_called) " + "FROM qbit_audit_publication_sequence_seq;" + ), + allocator_after_flip, + "exact confirmation replay burns no publication ordinal", +) assert_equal( replacement.dashboard_miner_pending_maturity_bits(recipient_id="miner-b"), 49500, @@ -799,14 +1193,16 @@ INSERT INTO qbit_pool_blocks ( parent_hash, coinbase_txid, payout_manifest_sha256, - chain_state + chain_state, + audit_publication_sequence ) VALUES ( '""" + alias_block_hash + """', 72, '""" + "44" * 32 + """', '""" + "46" * 32 + """', '""" + "47" * 32 + """', - 'confirmed' + 'confirmed', + nextval('qbit_audit_publication_sequence_seq') ); INSERT INTO qbit_payout_carry_forward ( @@ -887,6 +1283,56 @@ WHERE block_hash = '""" + alias_block_hash + """'; """ ) +# Reorg visibility on the public block read model, against a real reversed +# row: the default view keeps hiding it, chain_state=reversed surfaces it +# with its disconnect time, and every pagination total describes exactly its +# own filtered set. +default_blocks = replacement.dashboard_blocks(page=1, limit=50) +all_blocks = replacement.dashboard_blocks(page=1, limit=50, chain_state="all") +reversed_blocks = replacement.dashboard_blocks(page=1, limit=50, chain_state="reversed") +if any(row["hash"] == alias_block_hash for row in default_blocks["rows"]): + raise SystemExit("default dashboard blocks view leaked a reversed block") +if any("chain_state" in row or "disconnected_at" in row for row in default_blocks["rows"]): + raise SystemExit("default dashboard blocks view leaked chain-state fields") +assert_equal( + all_blocks["pagination"]["total_count"], + default_blocks["pagination"]["total_count"] + reversed_blocks["pagination"]["total_count"], + "chain_state filters partition the block history", +) +reversed_rows_by_hash = {row["hash"]: row for row in reversed_blocks["rows"]} +if alias_block_hash not in reversed_rows_by_hash: + raise SystemExit("chain_state=reversed did not surface the reversed block") +reversed_row = reversed_rows_by_hash[alias_block_hash] +assert_equal(reversed_row["chain_state"], "reversed", "reversed block reports its chain state") +if not str(reversed_row["disconnected_at"] or "").endswith("Z"): + raise SystemExit("reversed block disconnected_at is not a UTC timestamp") +for row in all_blocks["rows"]: + if "chain_state" not in row or "disconnected_at" not in row: + raise SystemExit("chain_state=all rows must carry the state fields") + if (row["chain_state"] == "reversed") != (row["disconnected_at"] is not None): + raise SystemExit("disconnected_at must be set exactly for reversed rows") +paged_reversed = replacement.dashboard_blocks(page=1, limit=1, chain_state="reversed") +assert_equal( + paged_reversed["pagination"]["total_count"], + reversed_blocks["pagination"]["total_count"], + "reversed pagination total is limit-independent", +) +assert_equal(len(paged_reversed["rows"]), 1, "reversed page honours its limit") +reorg_snapshot = replacement.dashboard_pool_snapshot( + current_network_difficulty="1000", + generated_at="2026-01-01T00:00:00Z", +) +assert_equal( + reorg_snapshot["blocks_reversed_total"], + reversed_blocks["pagination"]["total_count"], + "pool snapshot reversed counter matches the reversed block view", +) +assert_equal( + reorg_snapshot["blocks_found_total"], + default_blocks["pagination"]["total_count"], + "blocks_found_total keeps excluding reversed blocks", +) + zero_net_bundle = copy.deepcopy(bundle) zero_net_bundle["signed_coinbase_manifest"]["manifest"]["payout_count"] = 1 zero_net_bundle["payout_policy_manifest"]["accounts"] = [ @@ -911,7 +1357,13 @@ replacement.persist_accepted_block( audit_report=zero_net_report, ) force_expired_idle_lease(replacement) -replacement.confirm_accepted_block(block_hash="45" * 32, active_tip_height=8) +second_confirmation = replacement.confirm_accepted_block( + block_hash="45" * 32, + active_tip_height=8, +) +second_publication_sequence = int(second_confirmation["audit_publication_sequence"]) +if second_publication_sequence == first_publication_sequence: + raise SystemExit("distinct confirmed blocks shared a publication ordinal") zero_net_balances = replacement._run_json( """ SELECT COALESCE(json_agg(json_build_object( @@ -933,8 +1385,45 @@ replacement.persist_accepted_block( final_bundle=zero_net_bundle, audit_report=zero_net_report, ) +# The stranded-prepared sweep finds exactly the rows the reorg watch read +# structurally cannot: prepared and buried past the reject depth floor. +# Nothing else re-examines them once their outbox row is gone. +assert_equal( + replacement.stranded_prepared_blocks( + active_tip_height=9 + 100, + min_depth=100, + ), + [{"block_hash": "46" * 32, "block_height": 9, "parent_hash": "45" * 32}], + "stranded prepared sweep finds a buried prepared row", +) +assert_equal( + replacement.stranded_prepared_blocks( + active_tip_height=9 + 99, + min_depth=100, + ), + [], + "stranded prepared sweep leaves rows inside the depth floor alone", +) +assert_equal( + replacement.stranded_prepared_blocks( + active_tip_height=9 + 100, + min_depth=100, + limit=0, + ), + [], + "stranded prepared sweep honours its page bound", +) rejected_count = replacement.reject_prepared_block(block_hash="46" * 32, active_tip_height=8)["rejected_count"] assert_equal(rejected_count, 3, "reject prepared block/payout/carry row count") +# The fenced rejection is what retires the row from the sweep. +assert_equal( + replacement.stranded_prepared_blocks( + active_tip_height=9 + 100, + min_depth=100, + ), + [], + "rejected block leaves the stranded prepared sweep", +) rejected_state = replacement._run_json( """ SELECT json_build_object( @@ -950,6 +1439,28 @@ assert_equal( {"chain_state": "rejected", "maturity_state": "reversed"}, "rejected prepared block state", ) +# The reject flip stamps disconnected_at (its maturity_state 'reversed' +# requires it), but the public contract defines disconnected_at as a reorg +# disconnect time that is null for every non-reversed row -- a rejected +# block must not read as a reorg casualty. +rejected_all_view = replacement.dashboard_blocks(page=1, limit=50, chain_state="all") +rejected_public_row = next( + (row for row in rejected_all_view["rows"] if row["hash"] == "46" * 32), + None, +) +if rejected_public_row is None: + raise SystemExit("chain_state=all did not surface the rejected block") +assert_equal(rejected_public_row["chain_state"], "rejected", "rejected block reports its chain state") +if rejected_public_row["disconnected_at"] is not None: + raise SystemExit("rejected block must not report a reorg disconnect time") +assert_equal( + replacement.confirm_accepted_block( + block_hash="46" * 32, + active_tip_height=9, + ), + {"backend": "postgres-psql", "confirmed_count": -1}, + "rejected block confirmation reports the superseded disposition", +) try: replacement._run_json("SELECT json_build_object('count', qbit_reverse_immature_pool_block('" + "44" * 32 + "', 7));") @@ -961,15 +1472,62 @@ else: inactive_count = replacement.mark_pool_block_inactive(block_hash="44" * 32, active_tip_height=7)["inactive_count"] assert_equal(inactive_count, 1, "inactive block quarantine count") +assert_equal( + replacement.mark_pool_block_inactive( + block_hash="44" * 32, + active_tip_height=7, + )["inactive_count"], + 0, + "repeated inactive transition is count-zero", +) +assert_equal( + replacement.confirm_accepted_block( + block_hash="44" * 32, + active_tip_height=7, + ), + {"backend": "postgres-psql", "confirmed_count": -1}, + "inactive block confirmation reports the superseded disposition", +) assert_equal(replacement.current_owed_balances(), [], "inactive owed balances are excluded") -reactivated_count = replacement.reactivate_pool_block(block_hash="44" * 32, active_tip_height=7)["reactivated_count"] -assert_equal(reactivated_count, 1, "inactive block reactivation count") +reactivated = replacement.reactivate_pool_block(block_hash="44" * 32, active_tip_height=7) +assert_equal(reactivated["reactivated_count"], 1, "inactive block reactivation count") +reactivated_publication_sequence = int(reactivated["audit_publication_sequence"]) +assert_equal( + reactivated_publication_sequence, + first_publication_sequence, + "reactivated block preserves its published ordinal", +) +reactivated_confirmation = replacement.confirm_accepted_block( + block_hash="44" * 32, + active_tip_height=7, +) +# Reactivation restores 'confirmed' without returning the row to 'prepared', +# so a confirmation arriving after it is an idempotent replay too -- and it +# still addresses the ordinal the original flip allocated. +assert_equal( + reactivated_confirmation["confirmed_count"], + 2, + "confirmation after reactivation reports the idempotent disposition", +) +assert_equal( + int(reactivated_confirmation["audit_publication_sequence"]), + first_publication_sequence, + "confirmation after reactivation preserves the published ordinal", +) if not replacement.current_owed_balances(): raise SystemExit("reactivated block did not restore owed balances") inactive_count = replacement.mark_pool_block_inactive(block_hash="44" * 32, active_tip_height=7)["inactive_count"] assert_equal(inactive_count, 1, "inactive block can be quarantined again before final reversal") reversed_count = replacement.reverse_immature_block(block_hash="44" * 32, active_tip_height=7)["reversed_count"] assert_equal(reversed_count, 6, "reverse immature block/payout/carry/fanout row count") +assert_equal( + replacement.confirm_accepted_block( + block_hash="44" * 32, + active_tip_height=7, + ), + {"backend": "postgres-psql", "confirmed_count": -1}, + "reversed block confirmation reports the superseded disposition", +) assert_equal(replacement.current_owed_balances(), [], "reversed owed balances are excluded") replacement.persist_accepted_block( @@ -980,20 +1538,52 @@ replacement.persist_accepted_block( audit_report=report, ) replacement.confirm_accepted_block(block_hash="66" * 32, active_tip_height=8) + +# Issue #224: the reconciliation-owned mutations below (inactive quarantine, +# the maturation sweep, reactivation) move pool-block, payout-entry and carry +# state, and the coordinator relies on their never writing qbit_share_ledger +# to answer a confirmed mutation with a prior-balances reread instead of a +# full window rescan. Capture the share ledger's exact identity here and +# require it back after every transition, while the owed balance proves each +# transition actually happened. +share_ledger_identity_before = share_ledger_identity(replacement) + + +def assert_share_ledger_untouched(transition: str) -> None: + assert_equal( + share_ledger_identity(replacement), + share_ledger_identity_before, + f"qbit_share_ledger identity after {transition} (#224)", + ) + + +def owed_balances() -> list[tuple[str, int]]: + return [(row["recipient_id"], row["balance_sats"]) for row in replacement.current_owed_balances()] + + +assert_equal(owed_balances(), [("miner-a", 1000)], "confirmed block 66 owes its accrued balance") inactive_count = replacement.mark_pool_block_inactive(block_hash="66" * 32, active_tip_height=8)["inactive_count"] assert_equal(inactive_count, 1, "inactive block 66 quarantine count") +assert_equal(owed_balances(), [], "inactive block 66 owes nothing") +assert_share_ledger_untouched("inactive quarantine") matured_while_inactive = replacement._run_json("SELECT json_build_object('count', qbit_mark_mature_pool_payouts(1008));")["count"] assert_equal(matured_while_inactive, 0, "mature payout sweep ignores inactive blocks") +assert_share_ledger_untouched("maturation sweep over an inactive block") inactive_count = replacement.mark_pool_block_inactive(block_hash="66" * 32, active_tip_height=1008)["inactive_count"] assert_equal(inactive_count, 0, "height-mature inactive block quarantine is idempotent") reactivated_count = replacement.reactivate_pool_block(block_hash="66" * 32, active_tip_height=1008)["reactivated_count"] assert_equal(reactivated_count, 1, "height-mature inactive block reactivation count") +assert_equal(owed_balances(), [("miner-a", 1000)], "reactivated block 66 owes its accrued balance again") +assert_share_ledger_untouched("reactivation") inactive_count = replacement.mark_pool_block_inactive(block_hash="66" * 32, active_tip_height=1008)["inactive_count"] assert_equal(inactive_count, 1, "height-mature immature confirmed block can be quarantined") +assert_share_ledger_untouched("second inactive quarantine") reactivated_count = replacement.reactivate_pool_block(block_hash="66" * 32, active_tip_height=1008)["reactivated_count"] assert_equal(reactivated_count, 1, "height-mature re-quarantined block reactivation count") +assert_share_ledger_untouched("second reactivation") matured_count = replacement._run_json("SELECT json_build_object('count', qbit_mark_mature_pool_payouts(1008));")["count"] assert_equal(matured_count, 2, "mature payout entry count") +assert_share_ledger_untouched("maturation") carry_states = replacement._run_json( """ SELECT COALESCE(json_agg(DISTINCT maturity_state ORDER BY maturity_state), '[]'::json) @@ -1429,6 +2019,129 @@ WHERE block_hash = '""" + external_block_hash + """'; "externalized conflicting duplicate does not write an orphan body file", ) +external_successor.release_writer_lease() + +# A pre-ordinal writer confirms with a plain chain_state UPDATE that never +# mentions the publication ordinal. The BEFORE INSERT OR UPDATE trigger must +# assign the next ordinal for that write so the validated CHECK constraint +# stays satisfiable under a code-only rollback, while an ordinal-aware +# confirmation that assigns nextval() explicitly stays untouched. +legacy_confirm_hash = "e9" * 32 +replacement._run_sql( + """ +INSERT INTO qbit_pool_blocks ( + block_hash, + block_height, + parent_hash, + coinbase_txid, + payout_manifest_sha256, + chain_state +) VALUES ( + '""" + legacy_confirm_hash + """', + 95, + '""" + "ab" * 32 + """', + '""" + "4a" * 32 + """', + '""" + "4b" * 32 + """', + 'prepared' +); +""" +) +legacy_before = replacement._run_json( + """ +SELECT json_build_object( + 'ordinal', ( + SELECT audit_publication_sequence + FROM qbit_pool_blocks + WHERE block_hash = '""" + legacy_confirm_hash + """' + ), + 'max_ordinal', ( + SELECT COALESCE(MAX(audit_publication_sequence), 0) + FROM qbit_pool_blocks + ) +); +""" +) +assert_equal(legacy_before["ordinal"], None, "prepared legacy block carries no ordinal") +replacement._run_sql( + """ +UPDATE qbit_pool_blocks +SET chain_state = 'confirmed' +WHERE block_hash = '""" + legacy_confirm_hash + """'; +""" +) +legacy_after = replacement._run_json( + """ +SELECT json_build_object( + 'ordinal', ( + SELECT audit_publication_sequence + FROM qbit_pool_blocks + WHERE block_hash = '""" + legacy_confirm_hash + """' + ), + 'duplicate_ordinals', ( + SELECT count(*) + FROM ( + SELECT audit_publication_sequence + FROM qbit_pool_blocks + WHERE audit_publication_sequence IS NOT NULL + GROUP BY audit_publication_sequence + HAVING count(*) > 1 + ) AS duplicated + ) +); +""" +) +if legacy_after["ordinal"] is None: + raise SystemExit("legacy ordinal-less confirmation was not assigned an ordinal") +legacy_ordinal = int(legacy_after["ordinal"]) +if legacy_ordinal <= int(legacy_before["max_ordinal"]): + raise SystemExit("legacy confirmation ordinal did not extend publication order") +assert_equal( + legacy_after["duplicate_ordinals"], + 0, + "publication ordinals stay unique after a legacy confirmation", +) +explicit_confirm_hash = "e8" * 32 +replacement._run_sql( + """ +INSERT INTO qbit_pool_blocks ( + block_hash, + block_height, + parent_hash, + coinbase_txid, + payout_manifest_sha256, + chain_state +) VALUES ( + '""" + explicit_confirm_hash + """', + 96, + '""" + legacy_confirm_hash + """', + '""" + "4c" * 32 + """', + '""" + "4d" * 32 + """', + 'prepared' +); + +UPDATE qbit_pool_blocks +SET chain_state = 'confirmed', + audit_publication_sequence = nextval('qbit_audit_publication_sequence_seq') +WHERE block_hash = '""" + explicit_confirm_hash + """'; +""" +) +explicit_after = replacement._run_json( + """ +SELECT json_build_object( + 'ordinal', ( + SELECT audit_publication_sequence + FROM qbit_pool_blocks + WHERE block_hash = '""" + explicit_confirm_hash + """' + ) +); +""" +) +assert_equal( + int(explicit_after["ordinal"]), + legacy_ordinal + 1, + "explicit ordinal-aware confirmation continues the same allocation order", +) + # Everything above runs on fixtures far smaller than one 4096-row cutoff page, # so the recursive page step of the bounded window readers never executes. # Seed enough unit-difficulty shares that the paged cutoff walk must cross page @@ -1517,6 +2230,113 @@ for network_difficulty, expected_len in [(512, 4096), (1024, 8192), (1025, 8200) f"multi-page audit window crossing row at network difficulty {network_difficulty}", ) -print("prism postgres ledger PASS shares=14 lease=replay startup-retry persist-fence sql-window maturity=reorg carry-replay integrity multipage-window=9000") +# Issue #236: both payout-window reads now return one JSON object per row, +# decoded by the psql backend line by line from a spooled result instead of +# one json_agg value. The selection did not change, so the pre-#236 aggregate +# statement is the oracle: same rows, same order, same field values, across +# the 9000-row fixture (17 batches of 512, a boundary inside the crossing +# page), every exact weighted cutoff above, the unbounded full history, an +# empty window, and the delta's disjoint eligibility branches. +assert_equal(replacement.execution_backend, "psql-subprocess", "oracle check runs on the psql backend") +for weight, expected_len in [(4095, 4095), (4096, 4096), (4097, 4097), (8192, 8192), (9000, 9000), (12000, 9000)]: + assert_rows_match_aggregate( + replacement, + lambda: replacement.snapshot_at_job_issue(bulk_anchor_ms, window_weight=weight), + label=f"per-row bounded snapshot at window weight {weight}", + expected_len=expected_len, + ) +assert_rows_match_aggregate( + replacement, + lambda: replacement.snapshot_at_job_issue(bulk_anchor_ms), + label="per-row unbounded snapshot", + expected_len=9000, +) +assert_rows_match_aggregate( + replacement, + lambda: replacement.snapshot_at_job_issue(0, window_weight=64), + label="per-row empty window", + expected_len=0, +) +assert_rows_match_aggregate( + replacement, + lambda: replacement.snapshot_at_job_issue(1_700_000_000_001, window_weight=1), + label="per-row single-row window", + expected_len=1, +) +assert_rows_match_aggregate( + replacement, + lambda: replacement.snapshot_between_job_issues(1_700_000_004_000, 1_700_000_006_000), + label="per-row delta inside the bulk fixture", + expected_len=2000, +) +assert_rows_match_aggregate( + replacement, + lambda: replacement.snapshot_between_job_issues(0, bulk_anchor_ms), + label="per-row delta over the whole history", +) + +# A read-only ledger against this writable primary. Everything above proved +# the database accepts writes, which is exactly what makes it the right target: +# a standby would refuse these writes whoever asked, so it can never show that +# read_only=True is what refused them. +read_only_ledger = PsqlShareLedger( + psql_command=psql_with_env, + writer_id="public-read", + writer_epoch=1, + read_only=True, +) + +# release_writer_lease_fresh_connection issues an UPDATE over a one-shot psql +# connection, taking neither the writer gate nor the pool. The in-process gate +# cannot refuse it, so the server must. +try: + read_only_ledger.release_writer_lease_fresh_connection() +except RuntimeError as exc: + if "read-only transaction" not in str(exc): + raise +else: + raise SystemExit("read-only ledger wrote to a writable primary") + +# The read tier still reads: the session is read-only, not unusable. This goes +# through a read slot, which is the gate every public route takes; all_shares() +# and the other O(n) reads take the writer gate and are refused in-process, +# which is the pre-existing half of the promise. +assert_equal( + read_only_ledger.accepted_share_stats()["accepted_share_count"] > 0, + True, + "read-only ledger still serves reads", +) +read_only_ledger.close() + +# The same statement, the same identity, the same database, differing only in +# read_only: it lands. So the refusal above is the read-only session and not +# the statement, the schema, or the state of the lease row. +control_writer = PsqlShareLedger( + psql_command=psql_with_env, + writer_id="public-read", + writer_epoch=1, +) +assert_equal( + control_writer.release_writer_lease_fresh_connection(), + True, + "writable ledger of the same identity still releases its own lease", +) +control_writer.close() + +print("prism postgres ledger PASS shares=14 lease=replay startup-retry persist-fence sql-window bulk-abandon=fenced maturity=reorg carry-replay integrity multipage-window=9000 window-rows=oracle read-only-session share-ledger-identity=inactive+reactivate+mature") PY + + PRISM_PSQL_COMMAND="${PSQL_COMMAND}" \ + python3 -m tests.prism_postgres_a1_gate + + PRISM_PSQL_COMMAND="${PSQL_COMMAND}" \ + python3 -m tests.prism_postgres_a1_migration_gate + + PRISM_PSQL_COMMAND="${PSQL_COMMAND}" \ + python3 -m tests.prism_postgres_a1_revert_gate + + PRISM_PSQL_COMMAND="${PSQL_COMMAND}" \ + QBIT_PRISM_GATE_IMAGE="${GATE_IMAGE}" \ + QBIT_PRISM_GATE_IMAGE_DIGEST="${GATE_IMAGE_DIGEST}" \ + python3 -m tests.prism_postgres_a1_process_gate ) diff --git a/test/test-prism-postgres-native-ledger.sh b/test/test-prism-postgres-native-ledger.sh index 8bdac55bf..48edeb324 100755 --- a/test/test-prism-postgres-native-ledger.sh +++ b/test/test-prism-postgres-native-ledger.sh @@ -77,13 +77,20 @@ done cd "${ROOT_DIR}" PRISM_TEST_DATABASE_URL="${DATABASE_URL}" \ PRISM_TEST_PSQL_COMMAND="docker exec -i ${POSTGRES_CONTAINER} psql -U qbit -d qbit" \ + PRISM_TEST_PSQL_COMMAND_WITH_ENV="docker exec -i -e PGOPTIONS ${POSTGRES_CONTAINER} psql -U qbit -d qbit" \ python3 <<'PY' from __future__ import annotations import os import threading +import time -from lab.prism.share_ledger import PendingShare, PsqlShareLedger +from lab.prism.share_ledger import ( + LedgerOperationTimeout, + PendingShare, + PsqlShareLedger, +) +from tests.prism_window_rows_reference import assert_rows_match_aggregate def pending( @@ -114,6 +121,10 @@ def assert_equal(actual: object, expected: object, message: str) -> None: database_url = os.environ["PRISM_TEST_DATABASE_URL"] psql_command = os.environ["PRISM_TEST_PSQL_COMMAND"] +# Session GUCs reach a psql subprocess through PGOPTIONS, and "docker exec" +# does not forward the caller's environment, so the read-only assertions below +# use a command that asks docker to pass the variable through. +psql_command_with_env = os.environ["PRISM_TEST_PSQL_COMMAND_WITH_ENV"] ledger = PsqlShareLedger( psql_command=psql_command, @@ -224,9 +235,131 @@ ledger._accepted_stats_cache_seconds = 60.0 snapshot = ledger.snapshot_at_job_issue(1_700_000_002_000) assert_equal(len(snapshot), share_count, "snapshot returns all committed shares") + metrics = ledger.metrics() assert_equal(metrics["shares"], share_count, "metrics share count from cached stats") +# Issue #211: the replay enumeration is a read and must not queue on the +# coordinator-local writer gate. This is the production call shape -- the real +# statement, over the real pooled psycopg client, against a real server -- run +# while an accounting-shaped writer holds that gate for longer than the whole +# fast-call budget. Before the fix the page spent that budget on admission and +# never reached PostgreSQL. +candidate_hash = "ab" * 32 +second_candidate_hash = "cd" * 32 +candidate_intent = { + "schema": "qbit.prism.block-candidate-intent.v1", + "block_hash_hex": candidate_hash, + "block_hex": "00", +} +assert ledger.persist_block_candidate_intent(candidate_intent) +assert ledger.persist_block_candidate_intent( + {**candidate_intent, "block_hash_hex": second_candidate_hash} +) + +FAST_CALL_BUDGET_SECONDS = 1.0 +gate_taken = threading.Event() +release_gate = threading.Event() +holder_error: list[BaseException] = [] + + +def hold_writer_gate() -> None: + try: + with ledger._operation_gate(ledger._lock, "writer lock"): + gate_taken.set() + release_gate.wait(timeout=60) + except BaseException as exc: # noqa: BLE001 - surfaced below + holder_error.append(exc) + gate_taken.set() + + +holder = threading.Thread(target=hold_writer_gate, daemon=True) +holder.start() +if not gate_taken.wait(timeout=30): + raise SystemExit("the writer gate holder never acquired the gate") +if holder_error: + raise holder_error[0] + +try: + started = time.monotonic() + with ledger.operation_timeout(FAST_CALL_BUDGET_SECONDS): + page = ledger.pending_block_candidate_rows(limit=1) + enumeration_seconds = time.monotonic() - started + + # It completed, inside its own budget, with the gate still held. + assert_equal( + [row["block_hash"] for row in page], + [candidate_hash], + "pending page enumerated while the writer gate was held", + ) + assert_equal( + [row["pool_block_exists"] for row in page], + [False], + "pending page carried the landed-block fact from the same snapshot", + ) + if enumeration_seconds >= FAST_CALL_BUDGET_SECONDS: + raise SystemExit( + "pending page took " + f"{enumeration_seconds:.3f}s of a {FAST_CALL_BUDGET_SECONDS:g}s budget" + ) + + # Pagination stays exact across the same held gate: the cursor resumes + # strictly after its own row and the short page proves the end. + with ledger.operation_timeout(FAST_CALL_BUDGET_SECONDS): + second_page = ledger.pending_block_candidate_rows( + limit=1, + after_cursor=page[0]["cursor"], + ) + assert_equal( + [row["block_hash"] for row in second_page], + [second_candidate_hash], + "pending page cursor resumed strictly after its own row", + ) + with ledger.operation_timeout(FAST_CALL_BUDGET_SECONDS): + assert_equal( + ledger.pending_block_candidate_rows( + limit=1, + after_cursor=second_page[0]["cursor"], + ), + [], + "a cursor past every pending row proves the walk complete", + ) + + # The control. The same gate, the same budget, asked for writer admission + # instead: it times out, which is what the enumeration used to do. + try: + with ledger.operation_timeout(FAST_CALL_BUDGET_SECONDS): + ledger._acquire_operation_gate(ledger._lock, "writer lock") + except LedgerOperationTimeout as exc: + if "writer lock" not in str(exc): + raise + else: + ledger._lock.release() + raise SystemExit("the writer gate was not actually held") +finally: + release_gate.set() + holder.join(timeout=30) +if holder.is_alive(): + raise SystemExit("the writer gate holder never released the gate") +if holder_error: + raise holder_error[0] + +# The attribution the next budget exhaustion will be read from: no time on +# local admission, real time in PostgreSQL, and neither timeout counter armed. +read_gate_stats = ledger.ledger_read_gate_stats()["pending_block_candidate_headers"] +assert_equal(int(read_gate_stats["calls_total"]), 3, "read-slot calls counted") +assert_equal(int(read_gate_stats["gate_timeouts_total"]), 0, "no admission expiry") +assert_equal(int(read_gate_stats["execute_timeouts_total"]), 0, "no statement expiry") +if float(read_gate_stats["execute_seconds_total"]) <= 0.0: + raise SystemExit("pending page recorded no PostgreSQL execution time") +if float(read_gate_stats["gate_wait_seconds_max"]) >= FAST_CALL_BUDGET_SECONDS: + raise SystemExit( + "pending page charged " + f"{read_gate_stats['gate_wait_seconds_max']}s to local admission" + ) + +ledger._run_sql("DELETE FROM qbit_block_candidate_outbox;") + try: PsqlShareLedger( psql_command=psql_command, @@ -241,6 +374,129 @@ except RuntimeError as exc: else: raise SystemExit("second writer stole an unexpired lease over the native client") +# Issue #236: the pooled client consumes the payout-window reads one JSON +# object per row through fetchmany, in bounded batches, from one buffered +# SELECT. A 3000-row fixture crosses several 512-row batch boundaries; the +# pre-#236 json_agg statement over the same CTE is the oracle for every +# variant, exactly as on the psql backend. +ledger._run_script( + """ +INSERT INTO qbit_share_ledger ( + share_id, miner_id, payout_order_key, p2mr_program, + share_difficulty, network_difficulty, template_height, job_id, + job_issued_at, ntime, accepted_at, accepted, writer_id, writer_epoch +) +SELECT + 'bulk-' || g, + 'miner-' || (g % 7), + lpad((g % 7)::text, 4, '0'), + decode(md5(g::text) || md5((g + 7)::text), 'hex'), + 1 + (g % 3), + 1000, + 10, + 'job-bulk', + to_timestamp(1700001000) + (g * interval '1 millisecond'), + 1700001000, + to_timestamp(1700001000) + (g * interval '1 millisecond'), + TRUE, + 'writer-native', + 1 +FROM generate_series(1, 3000) AS g; +ANALYZE qbit_share_ledger; +""" +) +bulk_anchor_ms = 1_700_001_010_000 +assert_equal(ledger.execution_backend, "psycopg-pool", "oracle check runs on the pooled client") +for weight in (1, 512, 1023, 1024, 1025, 3000, 6000, 10**9): + assert_rows_match_aggregate( + ledger, + lambda: ledger.snapshot_at_job_issue(bulk_anchor_ms, window_weight=weight), + label=f"native per-row bounded snapshot at window weight {weight}", + ) +full_history = assert_rows_match_aggregate( + ledger, + lambda: ledger.snapshot_at_job_issue(bulk_anchor_ms), + label="native per-row unbounded snapshot", + expected_len=share_count + 3000, +) +assert_rows_match_aggregate( + ledger, + lambda: ledger.snapshot_at_job_issue(0, window_weight=64), + label="native per-row empty window", + expected_len=0, +) +assert_rows_match_aggregate( + ledger, + lambda: ledger.snapshot_between_job_issues(1_700_001_001_000, 1_700_001_002_000), + label="native per-row delta inside the bulk fixture", + expected_len=1000, +) +with ledger.operation_timeout(30.0): + assert_rows_match_aggregate( + ledger, + lambda: ledger.snapshot_at_job_issue(bulk_anchor_ms, window_weight=2048), + label="native per-row bounded snapshot under an armed deadline", + ) + +# One SELECT, one MVCC snapshot: a share appended while the rows are still +# being converted (between the first and second 512-row batches) is not in +# any part of that snapshot, and is in the next one. The between-batch hook +# is the ledger's own; a second pooled connection lands the append. +late_share = pending(share_count + 5000, share_id="appended-mid-decode") +late_share = PendingShare( + **{ + **late_share.__dict__, + "job_issued_at_ms": 1_700_001_005_000, + "accepted_at_ms": 1_700_001_005_000, + } +) +appended_during: list[object] = [] +original_hook = ledger._note_json_row_batch + + +def append_on_first_batch() -> None: + if not appended_during: + appended_during.append(ledger.append(late_share)) + original_hook() + + +ledger._note_json_row_batch = append_on_first_batch # type: ignore[method-assign] +try: + during = ledger.snapshot_at_job_issue(bulk_anchor_ms) +finally: + del ledger._note_json_row_batch +assert_equal(len(appended_during), 1, "the append landed during row conversion") +assert_equal( + [record.share_id for record in during], + [record.share_id for record in full_history], + "a share appended mid-decode is absent from the whole in-flight snapshot", +) +after = ledger.snapshot_at_job_issue(bulk_anchor_ms) +assert_equal(len(after), len(full_history) + 1, "the next snapshot includes the appended share") +assert_equal(after[-1].share_id, "appended-mid-decode", "the appended share is the newest row") +assert_equal( + ledger.ledger_read_gate_stats()["payout_window_snapshot"]["execute_timeouts_total"], + 0, + "no window read timed out", +) +print("prism postgres native ledger: OK window-rows=oracle mvcc=single-select", flush=True) + +# Restore the 32-share state the fallback and read-only checks below assert +# on: the bulk fixture and the mid-decode append were this section's only +# writes, and their sequence numbers sit above the contiguous 1..32 range. +ledger._run_script( + """ +DELETE FROM qbit_share_ledger +WHERE share_id LIKE 'bulk-%' OR share_id = 'appended-mid-decode'; +ANALYZE qbit_share_ledger; +""" +) +assert_equal( + len(ledger.snapshot_at_job_issue(bulk_anchor_ms)), + share_count, + "window fixture rows removed again", +) + released = ledger.release_writer_lease() assert_equal(released, True, "writer lease released") ledger.close() @@ -264,8 +520,86 @@ assert_equal( fallback.release_writer_lease() fallback.close() -print("prism postgres native ledger: OK") +# A read-only ledger against this writable primary. The database has accepted +# every write above, which is what makes it the right target: a standby would +# refuse these writes whoever asked, so it could never show that read_only=True +# is what refused them. +read_only_ledger = PsqlShareLedger( + psql_command=psql_command_with_env, + database_url=database_url, + native_client_mode="1", + writer_id="public-read", + writer_epoch=1, + read_only=True, +) +assert_equal( + read_only_ledger.execution_backend, + "psycopg-pool", + "read-only ledger uses the pooled native client", +) + +# One instance, two kinds of connection. The pooled psycopg session runs the +# writer-lease upsert -- production's own gate-free write, reached here +# directly because a read-only ledger deliberately never runs it at startup. +try: + read_only_ledger._try_acquire_writer_lease() +except Exception as exc: + if "read-only transaction" not in str(exc): + raise +else: + raise SystemExit("read-only pooled session wrote to a writable primary") + +# ...and the one-shot psql connection runs the lease release, which takes +# neither the writer gate nor the pool, so nothing in-process can refuse it. +try: + read_only_ledger.release_writer_lease_fresh_connection() +except RuntimeError as exc: + if "read-only transaction" not in str(exc): + raise +else: + raise SystemExit("read-only psql connection wrote to a writable primary") + +# Reads still work: the session is read-only, not unusable. A read slot is the +# gate every public route takes; the O(n) reads take the writer gate and are +# refused in-process, which is the pre-existing half of the promise. +assert_equal( + read_only_ledger.accepted_share_stats()["accepted_share_count"], + share_count, + "read-only ledger still serves reads over the pool", +) +read_only_ledger.close() + +# The same two statements, the same identity, differing only in read_only. +control_writer = PsqlShareLedger( + psql_command=psql_command_with_env, + database_url=database_url, + native_client_mode="1", + writer_id="public-read", + writer_epoch=1, +) +assert_equal( + control_writer._try_acquire_writer_lease()["acquired"], + True, + "writable pooled session still acquires the lease", +) +assert_equal( + control_writer.release_writer_lease_fresh_connection(), + True, + "writable psql connection still releases the lease", +) +control_writer.close() + +print("prism postgres native ledger: OK read-only-session") PY ) +( + cd "${ROOT_DIR}" + PRISM_RECOVERY_TEST_DATABASE_URL="${DATABASE_URL}" \ + python3 -m unittest tests.test_prism_pending_block_recovery.NativeRecoveryTests -v + PRISM_TEST_DATABASE_URL="${DATABASE_URL}" \ + PRISM_TEST_PSQL_COMMAND_WITH_ENV="docker exec -i -e PGOPTIONS ${POSTGRES_CONTAINER} psql -U qbit -d qbit" \ + python3 -m unittest tests.test_prism_candidate_window tests.test_prism_statement_spool -v +) + echo "test-prism-postgres-native-ledger: PASS" diff --git a/test/test-prism-postgres-scale.sh b/test/test-prism-postgres-scale.sh index 46a8812a0..8ac089af6 100755 --- a/test/test-prism-postgres-scale.sh +++ b/test/test-prism-postgres-scale.sh @@ -151,7 +151,7 @@ ALTER TABLE qbit_payout_carry_forward DISABLE TRIGGER qbit_payout_carry_forward_ INSERT INTO qbit_pool_blocks ( block_hash, block_height, parent_hash, coinbase_txid, - payout_manifest_sha256, chain_state + payout_manifest_sha256, chain_state, audit_publication_sequence ) SELECT 'scale-block-' || lpad(gs::text, 8, '0'), @@ -159,7 +159,8 @@ SELECT 'scale-block-' || lpad((gs - 1)::text, 8, '0'), md5('coinbase' || gs::text), md5('manifest' || gs::text), - 'confirmed' + 'confirmed', + pg_catalog.nextval('qbit_audit_publication_sequence_seq') FROM generate_series(1, {BLOCKS}) AS gs ON CONFLICT (block_hash) DO NOTHING; diff --git a/test/test-prism-postgres-seed-guard.sh b/test/test-prism-postgres-seed-guard.sh new file mode 100755 index 000000000..3911b812b --- /dev/null +++ b/test/test-prism-postgres-seed-guard.sh @@ -0,0 +1,606 @@ +#!/usr/bin/env bash +# Regression test for the carry-forward summary seed guard (#124). +# +# Reproduces the poisoning mechanism against a real Postgres: a per-statement +# autocommit schema apply commits the carry-forward summary sync triggers +# before the seeding block at the end of the file runs, a still-running writer +# confirms a block in that gap, and the summary ends up holding only the +# post-trigger delta. The old emptiness-based guard then treats that partial +# summary as already seeded and locks the damage in permanently. The hardened +# guard compares the summary against the carry history it summarizes and +# repairs it on the next apply, on both the psql-subprocess and the native +# psycopg apply paths. A second scenario proves the other half of the fix: +# a failing schema apply commits nothing on either path (the script wraps +# itself in one BEGIN/COMMIT transaction, and the psql backend additionally +# passes --single-transaction). +# +# Requires docker and a host python3 with psycopg installed +# (python3 -m pip install 'psycopg[binary]'). +set -euo pipefail + +ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +POSTGRES_IMAGE="${QBIT_PRISM_POSTGRES_IMAGE:-postgres:16-alpine}" +POSTGRES_CONTAINER="${QBIT_PRISM_POSTGRES_CONTAINER:-qbit-prism-seed-guard-pg-$$}" + +require_executable() { + command -v "$1" >/dev/null 2>&1 || { + echo "missing required executable: $1" >&2 + exit 1 + } +} + +cleanup() { + docker rm -f "${POSTGRES_CONTAINER}" >/dev/null 2>&1 || true +} +trap cleanup EXIT + +require_executable docker +require_executable python3 + +# Both apply paths are exercised, so the native psycopg client is mandatory +# here rather than opportunistic. +python3 -c 'import psycopg' 2>/dev/null || { + echo "host python3 lacks psycopg; install with: python3 -m pip install 'psycopg[binary]'" >&2 + exit 1 +} + +docker rm -f "${POSTGRES_CONTAINER}" >/dev/null 2>&1 || true +docker run \ + --rm \ + --detach \ + --name "${POSTGRES_CONTAINER}" \ + -p 127.0.0.1:0:5432 \ + -e POSTGRES_USER=qbit \ + -e POSTGRES_PASSWORD=qbit \ + -e POSTGRES_DB=qbit \ + "${POSTGRES_IMAGE}" >/dev/null + +# The official image runs initdb against a temporary server that binds no TCP +# address, so a socket-only `pg_isready -U qbit -d qbit` reports ready during +# bootstrap and races the restart into the durable server. Gate on the TCP +# listener instead, and confirm with a real query rather than trusting the +# liveness probe alone. +deadline=$((SECONDS + 60)) +until docker exec "${POSTGRES_CONTAINER}" \ + pg_isready -h 127.0.0.1 -U qbit -d qbit >/dev/null 2>&1 \ + && echo 'SELECT 1;' | docker exec -i "${POSTGRES_CONTAINER}" \ + psql --no-psqlrc --set ON_ERROR_STOP=1 -U qbit -d qbit >/dev/null 2>&1; do + if [[ "${SECONDS}" -ge "${deadline}" ]]; then + echo "timed out waiting for PRISM Postgres container" >&2 + docker logs "${POSTGRES_CONTAINER}" >&2 || true + exit 1 + fi + sleep 1 +done + +HOST_PORT="$(docker port "${POSTGRES_CONTAINER}" 5432/tcp | head -n 1 | awk -F: '{print $NF}')" +if [[ -z "${HOST_PORT}" ]]; then + echo "unable to resolve published Postgres port" >&2 + exit 1 +fi +DATABASE_URL="postgresql://qbit:qbit@127.0.0.1:${HOST_PORT}/qbit" + +# The gate above ran inside the container; the native apply path connects over +# the published port from the host, so prove that reaches the same server. +deadline=$((SECONDS + 60)) +until python3 - "$DATABASE_URL" <<'PY' >/dev/null 2>&1 +import sys +import psycopg + +with psycopg.connect(sys.argv[1], connect_timeout=3) as conn: + conn.execute("SELECT 1") +PY +do + if [[ "${SECONDS}" -ge "${deadline}" ]]; then + echo "timed out waiting for published PRISM Postgres port" >&2 + docker logs "${POSTGRES_CONTAINER}" >&2 || true + exit 1 + fi + sleep 1 +done + +( + cd "${ROOT_DIR}" + PRISM_TEST_ROOT_DIR="${ROOT_DIR}" \ + PRISM_TEST_POSTGRES_CONTAINER="${POSTGRES_CONTAINER}" \ + PRISM_TEST_POSTGRES_PORT="${HOST_PORT}" \ + python3 <<'PY' +from __future__ import annotations + +import json +import os +import subprocess +import sys +import tempfile +from pathlib import Path + +ROOT_DIR = os.environ["PRISM_TEST_ROOT_DIR"] +sys.path.insert(0, ROOT_DIR) + +from lab.prism.share_ledger import PsqlShareLedger + +CONTAINER = os.environ["PRISM_TEST_POSTGRES_CONTAINER"] +HOST_PORT = os.environ["PRISM_TEST_POSTGRES_PORT"] +SCHEMA_PATH = Path(ROOT_DIR) / "crates" / "qbit-prism" / "sql" / "001_share_ledger.sql" + +# The hardened guard's first comment line. Splitting the schema here yields a +# head that installs every table, function and carry-forward summary sync +# trigger but no seeding block at all -- exactly the state a non-atomic apply +# leaves behind when it is interrupted between the triggers and the seed. +SEED_GUARD_MARKER = ( + "-- Seed or repair the summary from carry history whenever the summary" +) + +# The pre-#124 guard, verbatim. It seeds only when the summary is completely +# empty, so a partially populated summary reads as "already seeded". +OLD_SEED_GUARD = """ +DO $$ +BEGIN + IF NOT EXISTS (SELECT 1 FROM qbit_payout_carry_forward_current) + AND EXISTS ( + SELECT 1 + FROM qbit_payout_carry_forward carry + JOIN qbit_pool_blocks block + ON block.block_hash = carry.block_hash + WHERE carry.maturity_state <> 'reversed' + AND block.chain_state = 'confirmed' + AND block.maturity_state <> 'reversed' + ) + THEN + PERFORM qbit_rebuild_carry_forward_current_balances(); + END IF; +END +$$; +""" + +CONFIRMED_BLOCK_HASH = "aa" * 32 +PREPARED_BLOCK_HASH = "ee" * 32 + + +def assert_equal(actual: object, expected: object, message: str) -> None: + if actual != expected: + raise SystemExit(f"{message}: expected {expected!r}, got {actual!r}") + + +def psql_command(database: str) -> str: + return f"docker exec -i {CONTAINER} psql -U qbit -d {database}" + + +def database_url(database: str) -> str: + return f"postgresql://qbit:qbit@127.0.0.1:{HOST_PORT}/{database}" + + +def _psql(sql: str, *, database: str, extra_args: list[str]) -> str: + completed = subprocess.run( + [ + "docker", + "exec", + "-i", + CONTAINER, + "psql", + "--no-psqlrc", + "--set", + "ON_ERROR_STOP=1", + *extra_args, + "-U", + "qbit", + "-d", + database, + ], + input=sql, + text=True, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + check=False, + ) + if completed.returncode != 0: + raise SystemExit( + f"psql on {database} failed (exit {completed.returncode}): " + f"{completed.stderr.strip()}" + ) + return completed.stdout + + +def psql_autocommit_apply(sql: str, *, database: str = "qbit") -> None: + """Apply SQL with no --single-transaction: one commit per statement. + + This is the operator error the old ops doc allowed, and the shape that + poisons the summary seed. + """ + _psql(sql, database=database, extra_args=[]) + + +def psql_json(sql: str, *, database: str = "qbit") -> object: + output = _psql( + sql, + database=database, + extra_args=["--tuples-only", "--no-align", "--quiet"], + ).strip() + if not output: + raise SystemExit(f"psql query on {database} returned no JSON") + return json.loads(output.splitlines()[-1]) + + +def summary_rows(*, database: str = "qbit") -> object: + return psql_json( + """ +SELECT COALESCE(json_agg(json_build_object( + 'miner_id', miner_id, + 'balance_sats', balance_sats::text, + 'active_row_count', active_row_count +) ORDER BY miner_id), '[]'::json) +FROM qbit_payout_carry_forward_current; +""", + database=database, + ) + + +def reported_balances(*, database: str = "qbit") -> object: + return psql_json( + """ +SELECT COALESCE(json_agg(json_build_object( + 'miner_id', miner_id, + 'balance_sats', balance_sats::text +) ORDER BY miner_id), '[]'::json) +FROM qbit_current_carry_forward_balances(); +""", + database=database, + ) + + +POISONED_SUMMARY = [ + {"miner_id": "miner-b", "balance_sats": "500", "active_row_count": 1} +] +REPAIRED_SUMMARY = [ + {"miner_id": "miner-a", "balance_sats": "1000", "active_row_count": 1}, + {"miner_id": "miner-b", "balance_sats": "500", "active_row_count": 1}, +] +REPAIRED_BALANCES = [ + {"miner_id": "miner-a", "balance_sats": "1000"}, + {"miner_id": "miner-b", "balance_sats": "500"}, +] + + +def assert_repaired(ledger: PsqlShareLedger, label: str) -> None: + assert_equal(summary_rows(), REPAIRED_SUMMARY, f"{label}: summary rebuilt from history") + assert_equal( + reported_balances(), + REPAIRED_BALANCES, + f"{label}: reported balances no longer under-report the pre-upgrade carry", + ) + assert_equal( + ledger._run_json( + """ +SELECT json_build_object( + 'drift_rows', (SELECT count(*) FROM qbit_carry_forward_current_drift()), + 'balance_rows', (SELECT count(*) FROM qbit_current_carry_forward_balances()) +); +""" + ), + {"drift_rows": 0, "balance_rows": 2}, + f"{label}: summary agrees with the recomputation it summarizes", + ) + + +# -------------------------------------------------------------------------- +# Scenario A: poison via a non-atomic apply, old guard locks it in, a new +# apply repairs it on both execution backends. +# -------------------------------------------------------------------------- + +schema_text = SCHEMA_PATH.read_text(encoding="utf-8") +schema_lines = schema_text.splitlines(keepends=True) +marker_indexes = [ + index + for index, line in enumerate(schema_lines) + if line.rstrip("\n") == SEED_GUARD_MARKER +] +assert_equal(len(marker_indexes), 1, "seed guard marker comment appears exactly once") +marker_index = marker_indexes[0] +schema_head = "".join(schema_lines[:marker_index]) +schema_tail = "".join(schema_lines[marker_index:]) +if "CREATE TRIGGER qbit_pool_blocks_carry_forward_current_sync" not in schema_head: + raise SystemExit("schema head is missing the carry-forward summary sync triggers") +if "CREATE OR REPLACE FUNCTION qbit_carry_forward_current_drift" not in schema_head: + raise SystemExit("schema head is missing the drift function the guard depends on") +if "qbit_rebuild_carry_forward_current_balances" not in schema_tail: + raise SystemExit("schema tail is missing the seed/repair guard") + +# The script wraps itself in one BEGIN/COMMIT transaction (that wrapper is +# half of the #124 fix). The interrupted-apply reproduction must strip the +# leading BEGIN so the head really does autocommit statement by statement, +# exactly like the pre-fix schema file did under a plain `psql -f`. +head_lines = schema_head.splitlines(keepends=True) +begin_indexes = [ + index for index, line in enumerate(head_lines) if line.rstrip("\n") == "BEGIN;" +] +assert_equal(len(begin_indexes), 1, "schema head contains exactly one leading BEGIN") +unwrapped_head = "".join(head_lines[begin_indexes[0] + 1 :]) + +# 1. The interrupted operator apply: every statement commits on its own and +# the run stops before the seeding block at the end of the file. +psql_autocommit_apply(unwrapped_head) + +# 2. Pre-upgrade carry history that never flowed through the summary triggers. +# session_replication_role = replica suppresses them for this session the +# way history written before they existed never reached them. It also +# suppresses the BEFORE trigger that assigns the publication ordinal, so +# the confirmed row carries one explicitly to satisfy the CHECK constraint. +psql_autocommit_apply( + """ +SET session_replication_role = replica; + +INSERT INTO qbit_pool_blocks ( + block_hash, + block_height, + parent_hash, + coinbase_txid, + payout_manifest_sha256, + chain_state, + audit_publication_sequence +) VALUES ( + '""" + + CONFIRMED_BLOCK_HASH + + """', + 100, + '""" + + "00" * 32 + + """', + '""" + + "ab" * 32 + + """', + '""" + + "ac" * 32 + + """', + 'confirmed', + nextval('qbit_audit_publication_sequence_seq') +); + +INSERT INTO qbit_pool_blocks ( + block_hash, + block_height, + parent_hash, + coinbase_txid, + payout_manifest_sha256, + chain_state +) VALUES ( + '""" + + PREPARED_BLOCK_HASH + + """', + 101, + '""" + + CONFIRMED_BLOCK_HASH + + """', + '""" + + "ef" * 32 + + """', + '""" + + "e0" * 32 + + """', + 'prepared' +); + +INSERT INTO qbit_payout_carry_forward ( + block_height, + block_hash, + miner_id, + payout_order_key, + p2mr_program, + gross_amount_sats, + prior_balance_sats, + candidate_balance_sats, + onchain_amount_sats, + carry_forward_balance_sats, + action, + maturity_state +) VALUES + ( + 100, + '""" + + CONFIRMED_BLOCK_HASH + + """', + 'miner-a', + 'a', + decode('11' || repeat('00', 31), 'hex'), + 1000, + 0, + 1000, + 0, + 1000, + 'accrued', + 'immature' + ), + ( + 101, + '""" + + PREPARED_BLOCK_HASH + + """', + 'miner-b', + 'b', + decode('22' || repeat('00', 31), 'hex'), + 500, + 0, + 500, + 0, + 500, + 'accrued', + 'immature' + ); + +SET session_replication_role = origin; +""" +) +assert_equal( + summary_rows(), + [], + "pre-upgrade carry history left the summary empty", +) + +# 3. The gap mutation: the still-running writer confirms the prepared block +# while the apply sits between the committed triggers and the seed. The +# summary receives that delta and nothing else. +psql_autocommit_apply( + "UPDATE qbit_pool_blocks SET chain_state = 'confirmed' " + "WHERE block_hash = '" + PREPARED_BLOCK_HASH + "';" +) +assert_equal( + summary_rows(), + POISONED_SUMMARY, + "the gap mutation leaves a summary holding only the post-trigger delta", +) +assert_equal( + reported_balances(), + [{"miner_id": "miner-b", "balance_sats": "500"}], + "the poisoned summary silently omits every pre-upgrade carry balance", +) + +# 4. The pre-#124 guard sees a non-empty summary and calls the poison seeded. +psql_autocommit_apply(OLD_SEED_GUARD) +assert_equal( + summary_rows(), + POISONED_SUMMARY, + "the old emptiness guard locks the poisoned summary in", +) + +# 5. Repair through the psql-subprocess apply path. +psql_ledger = PsqlShareLedger( + psql_command=psql_command("qbit"), + native_client_mode="0", + writer_id="seed-guard-psql", + writer_epoch=1, + initialize_schema=True, +) +assert_equal( + psql_ledger.execution_backend, "psql-subprocess", "psql apply path selected" +) +assert_repaired(psql_ledger, "psql apply path") +psql_ledger.release_writer_lease() +psql_ledger.close() + +# 6. Re-poison the summary the same way and repair through the native apply +# path, which wraps the script in the simple-query-protocol transaction +# rather than passing --single-transaction. +psql_autocommit_apply( + "DELETE FROM qbit_payout_carry_forward_current WHERE miner_id = 'miner-a';" +) +assert_equal( + summary_rows(), + POISONED_SUMMARY, + "re-poisoned summary before the native apply", +) +native_ledger = PsqlShareLedger( + psql_command=psql_command("qbit"), + database_url=database_url("qbit"), + native_client_mode="1", + writer_id="seed-guard-native", + writer_epoch=1, + initialize_schema=True, +) +assert_equal( + native_ledger.execution_backend, "psycopg-pool", "native apply path selected" +) +assert_repaired(native_ledger, "native apply path") +native_ledger.release_writer_lease() +native_ledger.close() + +# 7. The guard's drift-only branch: a summary whose active row-count total +# still matches the carry history but whose balances were corrupted. +# Row counts alone cannot catch this; the drift comparison must. +psql_autocommit_apply( + "UPDATE qbit_payout_carry_forward_current " + "SET balance_sats = balance_sats + 1 WHERE miner_id = 'miner-a';" +) +assert_equal( + reported_balances(), + [ + {"miner_id": "miner-a", "balance_sats": "1001"}, + {"miner_id": "miner-b", "balance_sats": "500"}, + ], + "balance-only corruption is visible in the reported balances", +) +drift_ledger = PsqlShareLedger( + psql_command=psql_command("qbit"), + native_client_mode="0", + writer_id="seed-guard-drift", + writer_epoch=1, + initialize_schema=True, +) +assert_repaired(drift_ledger, "drift-only corruption repair") +drift_ledger.release_writer_lease() +drift_ledger.close() + +# -------------------------------------------------------------------------- +# Scenario B: a failing apply commits nothing on either path. The failure is +# injected just before the script's own final COMMIT so it lands inside the +# apply's transaction: without atomicity (a pre-fix autocommit apply, or a +# client that neither honors the wrapper nor passes --single-transaction) +# every earlier statement would already be durable when the failure lands, +# which is exactly what opens the poisoning gap. +# -------------------------------------------------------------------------- + +ATOMIC_DATABASES = ( + ("qbit_psql_atomic", "0", "seed-guard-atomic-psql", False), + ("qbit_native_atomic", "1", "seed-guard-atomic-native", True), +) + +for atomic_database, _mode, _writer_id, _needs_url in ATOMIC_DATABASES: + # CREATE DATABASE cannot run inside a transaction block, so it goes + # through the maintenance command rather than the ledger. + psql_autocommit_apply(f"CREATE DATABASE {atomic_database};") + +schema_body_lines = schema_text.splitlines(keepends=True) +commit_indexes = [ + index + for index, line in enumerate(schema_body_lines) + if line.rstrip("\n") == "COMMIT;" +] +assert_equal( + len(commit_indexes), + 1, + "schema script contains exactly one top-level COMMIT", +) +failing_schema_text = ( + "".join(schema_body_lines[: commit_indexes[0]]) + + "SELECT 1/0;\n" + + "".join(schema_body_lines[commit_indexes[0] :]) +) + +with tempfile.TemporaryDirectory() as temporary_directory: + failing_schema_path = Path(temporary_directory) / "001_share_ledger_failing.sql" + failing_schema_path.write_text(failing_schema_text, encoding="utf-8") + for atomic_database, mode, writer_id, needs_url in ATOMIC_DATABASES: + failure: BaseException | None = None + try: + PsqlShareLedger( + psql_command=psql_command(atomic_database), + database_url=database_url(atomic_database) if needs_url else None, + native_client_mode=mode, + writer_id=writer_id, + writer_epoch=1, + initialize_schema=True, + schema_path=failing_schema_path, + ) + except Exception as exc: # noqa: BLE001 - the failure mode is the assertion + failure = exc + if failure is None: + raise SystemExit( + f"{atomic_database}: a failing schema apply unexpectedly succeeded" + ) + assert_equal( + psql_json( + """ +SELECT json_build_object( + 'pool_blocks', to_regclass('qbit_pool_blocks')::text, + 'carry_forward_current', to_regclass('qbit_payout_carry_forward_current')::text +); +""", + database=atomic_database, + ), + {"pool_blocks": None, "carry_forward_current": None}, + f"{atomic_database}: the failed apply committed nothing", + ) + +print("prism postgres seed guard: OK") +PY +) + +echo "test-prism-postgres-seed-guard: PASS" diff --git a/test/test-prism-public-read-replica.sh b/test/test-prism-public-read-replica.sh new file mode 100755 index 000000000..b49364a33 --- /dev/null +++ b/test/test-prism-public-read-replica.sh @@ -0,0 +1,89 @@ +#!/usr/bin/env bash +set -euo pipefail + +ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +POSTGRES_IMAGE="${QBIT_PRISM_POSTGRES_IMAGE:-postgres:16-alpine}" +NETWORK="qbit-prism-pubrep-net-$$" +PRIMARY="qbit-prism-pubrep-primary-$$" +REPLICA="qbit-prism-pubrep-replica-$$" + +require_executable() { + command -v "$1" >/dev/null 2>&1 || { + echo "missing required executable: $1" >&2 + exit 1 + } +} + +cleanup() { + docker rm -f "${PRIMARY}" "${REPLICA}" >/dev/null 2>&1 || true + docker network rm "${NETWORK}" >/dev/null 2>&1 || true +} +trap cleanup EXIT + +require_executable python3 +require_executable docker + +docker network create "${NETWORK}" >/dev/null + +# The primary runs the same mounted pg_hba.conf the compose service uses, so +# the replication rules (and the local trust the healthcheck depends on) are +# the production-shaped ones. +docker run \ + --rm \ + --detach \ + --name "${PRIMARY}" \ + --network "${NETWORK}" \ + -e POSTGRES_USER=qbit \ + -e POSTGRES_PASSWORD=qbit \ + -e POSTGRES_DB=qbit \ + -v "${ROOT_DIR}/config/prism-postgres/pg_hba.conf:/etc/postgresql/pg_hba.conf:ro" \ + "${POSTGRES_IMAGE}" \ + postgres -c hba_file=/etc/postgresql/pg_hba.conf >/dev/null + +deadline=$((SECONDS + 60)) +until docker exec "${PRIMARY}" pg_isready -h 127.0.0.1 -p 5432 -U qbit -d qbit >/dev/null 2>&1 \ + && sleep 1 \ + && docker exec "${PRIMARY}" pg_isready -h 127.0.0.1 -p 5432 -U qbit -d qbit >/dev/null 2>&1; do + if [ "${SECONDS}" -ge "${deadline}" ]; then + echo "timed out waiting for the primary Postgres container" >&2 + docker logs "${PRIMARY}" >&2 || true + exit 1 + fi + sleep 1 +done + +# The standby reuses the compose entrypoint: pg_basebackup with slot creation, +# then the stock image entrypoint. The schema lands later, through streaming. +docker run \ + --rm \ + --detach \ + --name "${REPLICA}" \ + --network "${NETWORK}" \ + -e POSTGRES_USER=qbit \ + -e POSTGRES_PASSWORD=qbit \ + -e POSTGRES_DB=qbit \ + -e PRISM_POSTGRES_PRIMARY_HOST="${PRIMARY}" \ + -e PRISM_POSTGRES_PRIMARY_PORT=5432 \ + -e PRISM_POSTGRES_REPLICATION_SLOT=prism_public_replica_gate \ + -v "${ROOT_DIR}/config/prism-postgres/replica-entrypoint.sh:/usr/local/bin/prism-replica-entrypoint.sh:ro" \ + --entrypoint /usr/local/bin/prism-replica-entrypoint.sh \ + "${POSTGRES_IMAGE}" >/dev/null + +deadline=$((SECONDS + 120)) +until docker exec "${REPLICA}" pg_isready -h 127.0.0.1 -p 5432 -U qbit -d qbit >/dev/null 2>&1 \ + && [ "$(docker exec "${REPLICA}" psql -U qbit -d qbit -tAc 'SELECT pg_is_in_recovery()' 2>/dev/null | tr -d '[:space:]')" = "t" ]; do + if [ "${SECONDS}" -ge "${deadline}" ]; then + echo "timed out waiting for the replica standby container" >&2 + docker logs "${REPLICA}" >&2 || true + exit 1 + fi + sleep 1 +done + +( + cd "${ROOT_DIR}" + GATE_PRIMARY_PSQL_COMMAND="docker exec -i ${PRIMARY} psql -U qbit -d qbit" \ + GATE_REPLICA_PSQL_COMMAND="docker exec -i ${REPLICA} psql -U qbit -d qbit" \ + PYTHONPATH="${ROOT_DIR}" \ + python3 tests/prism_public_read_replica_gate.py +) diff --git a/test/test-prism-stratum-regtest-live.sh b/test/test-prism-stratum-regtest-live.sh index 4fffce84d..da78da09c 100755 --- a/test/test-prism-stratum-regtest-live.sh +++ b/test/test-prism-stratum-regtest-live.sh @@ -26,6 +26,10 @@ COORDINATOR_LOG="${DATADIR}/prism-coordinator.log" MINER_TIMEOUT_SECONDS="${QBIT_PRISM_LIVE_MINER_TIMEOUT_SECONDS:-90}" AUDIT_API_ENABLED="${QBIT_PRISM_LIVE_AUDIT_API:-0}" POSTGRES_ENABLED="${QBIT_PRISM_LIVE_POSTGRES:-0}" +NATIVE_POSTGRES="${QBIT_PRISM_LIVE_NATIVE_POSTGRES:-0}" +PRIMED_WINDOW="${QBIT_PRISM_LIVE_PRIMED_WINDOW:-0}" +RESTART_REPLAY="${QBIT_PRISM_LIVE_RESTART_REPLAY:-0}" +CRASH_MARKER="${DATADIR}/candidate-crash-marker.json" POSTGRES_IMAGE="${QBIT_PRISM_POSTGRES_IMAGE:-postgres:16-alpine}" POSTGRES_CONTAINER="${QBIT_PRISM_POSTGRES_CONTAINER:-qbit-prism-pg-$$}" MANIFEST_SIGNING_SEED_HEX="${QBIT_PRISM_MANIFEST_SIGNING_SEED_HEX:-4242424242424242424242424242424242424242424242424242424242424242}" @@ -53,6 +57,14 @@ if [[ "${TARGET_BLOCKS}" -gt 1 && "${AUDIT_API_ENABLED}" != "1" ]]; then echo "QBIT_PRISM_LIVE_BLOCKS>1 requires QBIT_PRISM_LIVE_AUDIT_API=1" >&2 exit 1 fi +if [[ "${RESTART_REPLAY}" == "1" && ( "${PRIMED_WINDOW}" != "1" || "${NATIVE_POSTGRES}" != "1" || "${POSTGRES_ENABLED}" != "1" ) ]]; then + echo "restart replay requires primed-window and native-Postgres fixture modes" >&2 + exit 1 +fi +if [[ "${RESTART_REPLAY}" == "1" && -n "${EXTERNAL_PSQL}" && -z "${PRISM_DATABASE_URL:-}" ]]; then + echo "restart replay with external Postgres requires PRISM_DATABASE_URL" >&2 + exit 1 +fi if [[ "${POWER_LAW_ENABLED}" == "1" && "${MINER_COUNT}" -lt 6 ]]; then echo "QBIT_PRISM_LIVE_POWER_LAW=1 requires at least 6 miners" >&2 exit 1 @@ -95,7 +107,12 @@ if [[ "${POSTGRES_ENABLED}" == "1" ]]; then done else docker rm -f "${POSTGRES_CONTAINER}" >/dev/null 2>&1 || true + postgres_port_args=() + if [[ "${NATIVE_POSTGRES}" == "1" ]]; then + postgres_port_args=(-p 127.0.0.1::5432) + fi docker run \ + "${postgres_port_args[@]}" \ --rm \ --detach \ --name "${POSTGRES_CONTAINER}" \ @@ -109,7 +126,14 @@ if [[ "${POSTGRES_ENABLED}" == "1" ]]; then fi fi +node_fixture_args=() +if [[ "${PRIMED_WINDOW}" == "1" ]]; then + # Keep ASERT near its anchor while mining enough blocks to separate the + # network target from the low Stratum share target. This is regtest only. + node_fixture_args=(-mocktime=1738714602) +fi "${QBITD_BIN}" \ + "${node_fixture_args[@]}" \ -regtest \ -asert \ -p2mronly=1 \ @@ -128,21 +152,31 @@ fi -datadir="${DATADIR}" >/dev/null wait_for_qbit_rpc_state ready 60 -qbit_rpc createwallet "${WALLET_NAME}" >/dev/null - miner_usernames=() -for miner_index in $(seq 1 "${MINER_COUNT}"); do - miner_usernames+=("$(qbit_rpc -rpcwallet="${WALLET_NAME}" getnewaddress "" p2mr)") -done - -# A virgin regtest chain reports initialblockdownload=true until its first -# block, and the coordinator's chain-trust gate blocks job issuance during -# IBD, so a fresh chain could never mine its own first block through the -# pool. Pre-mine one block to exit IBD; the PRISM ledger stays empty, so the -# fresh-ledger (collection-mode) path is still exercised by the miners. -qbit_rpc generatetoaddress 1 "${miner_usernames[0]}" >/dev/null +if [[ "${PRIMED_WINDOW}" == "1" ]]; then + # Wallet-free deterministic P2MR programs are enough to verify the mined + # coinbase. The extra identity keeps difficulty probes out of miner state. + for miner_index in $(seq 1 "$((MINER_COUNT + 1))"); do + miner_usernames+=("$(python3 - "${miner_index}" "${QBIT_SRC_DIR}" <<'PYADDR' +import sys +sys.path.insert(0, sys.argv[2] + "/test/functional") +from test_framework.address import program_to_witness +print(program_to_witness(2, int(sys.argv[1]).to_bytes(32, "big"), main=False)) +PYADDR +)") + done + qbit_rpc generatetoaddress 1000 "${miner_usernames[0]}" >/dev/null +else + qbit_rpc createwallet "${WALLET_NAME}" >/dev/null + for _miner_index in $(seq 1 "$((MINER_COUNT + 1))"); do + miner_usernames+=("$(qbit_rpc -rpcwallet="${WALLET_NAME}" getnewaddress "" p2mr)") + done + # Exit IBD without adding any PRISM ledger rows. + qbit_rpc generatetoaddress 1 "${miner_usernames[0]}" >/dev/null +fi before_height="$(qbit_rpc getblockcount)" +start_coordinator() { ( cd "${ROOT_DIR}" export QBIT_RPC_HOST=127.0.0.1 @@ -154,12 +188,19 @@ before_height="$(qbit_rpc getblockcount)" export PRISM_STRATUM_HIGHDIFF_PORT="${HIGHDIFF_PORT}" export PRISM_STRATUM_HIGHDIFF_START_DIFF="${HIGHDIFF_START_DIFF}" export PRISM_STRATUM_HIGHDIFF_MIN_DIFF="${HIGHDIFF_START_DIFF}" + export PRISM_HOT_PATH_LOG=1 export PRISM_MIN_READY_MINERS="${MINER_COUNT}" export PRISM_EVIDENCE_PATH="${EVIDENCE_PATH}" export PRISM_AUDIT_DIR="${DATADIR}" export PRISM_MANIFEST_SIGNING_SEED_HEX="${MANIFEST_SIGNING_SEED_HEX}" export PRISM_LEDGER_ATTESTATION_SIGNING_SEED_HEX="${LEDGER_ATTESTATION_SIGNING_SEED_HEX}" export PRISM_ALLOW_BUNDLE_EMBEDDED_LEDGER_KEY=1 + if [[ "${PRIMED_WINDOW}" == "1" ]]; then + # The template intentionally uses historical node mocktime. Ledger and + # coordinator clocks stay real, and all lease budgets stay unchanged. + PRISM_TEMPLATE_MAX_AGE_SECONDS="$(python3 -c 'import time; print(max(120, int(time.time()) - 1738714602 + 120))')" + export PRISM_TEMPLATE_MAX_AGE_SECONDS + fi export PRISM_STRATUM_SHARE_DIFF=0.000000001 export PRISM_STOP_AFTER_BLOCK=1 if [[ "${AUDIT_API_ENABLED}" == "1" ]]; then @@ -173,12 +214,24 @@ before_height="$(qbit_rpc getblockcount)" if [[ "${POSTGRES_ENABLED}" == "1" ]]; then export PRISM_POSTGRES_PSQL_COMMAND="${PSQL_COMMAND}" export PRISM_POSTGRES_INIT_SCHEMA=1 + if [[ "${NATIVE_POSTGRES}" == "1" ]]; then + if [[ -z "${EXTERNAL_PSQL}" ]]; then + postgres_port="$(docker port "${POSTGRES_CONTAINER}" 5432/tcp | head -1 | sed 's/.*://')" + export PRISM_DATABASE_URL="postgresql://qbit:qbit@127.0.0.1:${postgres_port}/qbit" + fi + export PRISM_POSTGRES_NATIVE_CLIENT=1 + fi else export PRISM_ALLOW_MEMORY_LEDGER=1 fi - python3 lab/prism/prism_coordinator.py -) >"${COORDINATOR_LOG}" 2>&1 & + if [[ "${RESTART_REPLAY}" == "1" && ! -f "${CRASH_MARKER}" ]]; then + exec python3 tests/prism_regtest_crash_coordinator.py "${CRASH_MARKER}" + fi + exec python3 lab/prism/prism_coordinator.py +) >>"${COORDINATOR_LOG}" 2>&1 & coordinator_pid="$!" +} +start_coordinator STRATUM_PORT="${STRATUM_PORT}" python3 <<'PY' import os @@ -273,13 +326,23 @@ PY # explicit md= minimum and the high-diff wire floor still hold above it. On a # fresh regtest chain the network difficulty sits below every configured # value, so the first probe expects the network clamp. -network_difficulty="$(qbit_rpc getdifficulty)" -probe_first_difficulty "${STRATUM_PORT}" "${miner_usernames[0]}" "x" "${network_difficulty}" +network_difficulty="$(qbit_rpc getblocktemplate '{"rules":["segwit"]}' | python3 -c ' +import json, sys +bits = int(json.load(sys.stdin)["bits"], 16) +size, mantissa = bits >> 24, bits & 0x7fffff +target = mantissa << (8 * (size - 3)) if size > 3 else mantissa >> (8 * (3 - size)) +print(float(0xffff * 2**208 / target)) +')" +start_difficulty="${network_difficulty}" +if [[ "${PRIMED_WINDOW}" == "1" ]]; then + start_difficulty=0.000000001 +fi +probe_first_difficulty "${STRATUM_PORT}" "${miner_usernames[0]}" "x" "${start_difficulty}" probe_first_difficulty "${HIGHDIFF_PORT}" "${miner_usernames[0]}" "x" "${HIGHDIFF_START_DIFF}" # Password d=/md= bound vardiff but do not lift the first advertised job # above the network clamp; only the high-diff listener floor is a wire # guarantee that holds above it. -probe_first_difficulty "${STRATUM_PORT}" "${miner_usernames[0]}" "d=0.5,md=0.25" "${network_difficulty}" +probe_first_difficulty "${STRATUM_PORT}" "${miner_usernames[$MINER_COUNT]}" "d=0.5,md=0.25" "${network_difficulty}" if [[ "${AUDIT_API_ENABLED}" == "1" ]]; then AUDIT_PORT="${AUDIT_PORT}" python3 <<'PY' @@ -304,6 +367,11 @@ PY fi fi +seed_miner_script=lab/miner-sim/miner_sim.py +if [[ "${PRIMED_WINDOW}" == "1" ]]; then + seed_miner_script=tests/prism_regtest_share_miner.py +fi + for miner_index in $(seq 1 "${MINER_COUNT}"); do miner_username="${miner_usernames[$((miner_index - 1))]}" ( @@ -319,7 +387,7 @@ for miner_index in $(seq 1 "${MINER_COUNT}"); do QBIT_RPC_PORT="${RPC_PORT}" \ QBIT_RPC_USER="${RPC_USER}" \ QBIT_RPC_PASSWORD="${RPC_PASSWORD}" \ - python3 lab/miner-sim/miner_sim.py + python3 "${seed_miner_script}" ) >"${DATADIR}/miner-${miner_index}.log" 2>&1 & miner_pids+=("$!") done @@ -355,6 +423,34 @@ while [[ ! -f "${EVIDENCE_PATH}" && "${SECONDS}" -lt "${deadline}" ]]; do sleep 1 done +if [[ "${RESTART_REPLAY}" == "1" ]]; then + crash_deadline=$((SECONDS + MINER_TIMEOUT_SECONDS)) + while kill -0 "${coordinator_pid}" >/dev/null 2>&1; do + if [[ "${SECONDS}" -ge "${crash_deadline}" ]]; then + echo "candidate crash fixture did not reach the durable publication boundary" >&2 + cat "${COORDINATOR_LOG}" >&2 + exit 1 + fi + sleep 0.1 + done + crash_status=0 + wait "${coordinator_pid}" || crash_status=$? + coordinator_pid="" + if [[ "${crash_status}" != "75" || ! -f "${CRASH_MARKER}" ]]; then + echo "unexpected coordinator exit in crash fixture: ${crash_status}" >&2 + cat "${COORDINATOR_LOG}" >&2 + exit 1 + fi + # No miners remain to resubmit or solve a replacement. The successor must + # discover and finish exactly the candidate committed by its predecessor. + for pid in "${miner_pids[@]}"; do + kill "${pid}" >/dev/null 2>&1 || true + wait "${pid}" >/dev/null 2>&1 || true + done + miner_pids=() + start_coordinator +fi + deadline=$((SECONDS + MINER_TIMEOUT_SECONDS)) while [[ ! -f "${EVIDENCE_PATH}" && "${SECONDS}" -lt "${deadline}" ]]; do if ! kill -0 "${coordinator_pid}" >/dev/null 2>&1; then @@ -392,6 +488,7 @@ for pid in "${miner_pids[@]}"; do done if [[ "${AUDIT_API_ENABLED}" == "1" ]]; then + RESTART_REPLAY="${RESTART_REPLAY}" CRASH_MARKER="${CRASH_MARKER}" \ EVIDENCE_PATH="${EVIDENCE_PATH}" \ DATADIR="${DATADIR}" \ AUDIT_PORT="${AUDIT_PORT}" \ @@ -438,13 +535,19 @@ owed = get_json("/owed-balances") if owed["schema"] != "qbit.prism.owed-balances.v1": raise SystemExit("unexpected owed-balances schema") -metrics = get_text("/metrics") -for expected in ( +import time +expected_metrics = ( f"qbit_prism_accepted_shares_total {evidence['accepted_share_count']}", "qbit_prism_blocks_accepted_total 1", -): - if expected not in metrics: - raise SystemExit(f"missing metrics line: {expected}") +) +metrics_deadline = time.monotonic() + 20 +while True: + metrics = get_text("/metrics") + if all(expected in metrics for expected in expected_metrics): + break + if time.monotonic() >= metrics_deadline: + raise SystemExit(f"metrics cache did not publish expected counters: {expected_metrics}") + time.sleep(0.25) if os.environ["POSTGRES_ENABLED"] == "1": import shlex @@ -468,6 +571,27 @@ if os.environ["POSTGRES_ENABLED"] == "1": ).strip() return json.loads(raw.splitlines()[-1]) + if os.environ.get("RESTART_REPLAY") == "1": + with open(os.environ["CRASH_MARKER"], encoding="utf-8") as handle: + crashed = json.load(handle) + if evidence["block_hash"] != crashed["block_hash"]: + raise SystemExit("restart finalized a different candidate") + replayed = psql_json(""" + SELECT json_build_object( + 'state', o.state, 'digest', o.candidate_sha256, + 'share_id', o.share_id, 'share_count', + (SELECT count(*) FROM qbit_share_ledger s WHERE s.share_id = o.share_id), + 'storage_version', COALESCE((to_jsonb(o)->>'storage_version')::integer, 1), + 'body_id', to_jsonb(o)->>'body_id' + ) FROM qbit_block_candidate_outbox o + WHERE o.block_hash = '%s' + """ % crashed["block_hash"]) + if replayed != {"state": "submitted", "digest": crashed["candidate_sha256"], + "share_id": crashed["share_id"], "share_count": 1, + "storage_version": crashed["storage_version"], "body_id": None}: + raise SystemExit(f"restart candidate identity or atomicity mismatch: {replayed}") + print("native candidate crash/restart replay PASS", flush=True) + counts = psql_json( """ SELECT json_build_object( diff --git a/tests/fixtures/fake_serve_builder.py b/tests/fixtures/fake_serve_builder.py index 05cc91d62..f3163e65f 100644 --- a/tests/fixtures/fake_serve_builder.py +++ b/tests/fixtures/fake_serve_builder.py @@ -6,11 +6,32 @@ without it it echoes the one-shot stdin payload, so the same command list serves both the daemon path and its transparent one-shot fallback. FAKE_SERVE_BUILDER_MODE selects daemon misbehavior for anomaly tests. + +prepare_window requests are served by the real Python fold +(``IncrementalShareWindow``), so mirror digests verify exactly and the +window-pipeline tests exercise the coordinator against byte-faithful daemon +behavior without a Rust build; the Rust implementation itself is proven by +the parity oracle's ``rust-daemon`` adapter. Two more real-daemon contracts +are mirrored so the coordinator sees them without a Rust build: the +declared integer widths (a prepare request carrying a value outside +``RUST_INTEGER_DOMAIN`` is answered ``out_of_range``, never folded and +never treated as malformed), and the stable ``rejection`` category beside +every ``fold_invalid``/``fallback`` error. Additional modes fault-inject +the prepare protocol: ``prepare-forget-windows`` answers every advance with +``needs_full``, ``prepare-fallback`` answers every advance with ``fallback``, +``prepare-error`` answers prepare requests with a generic error, +``crash-during-prepare`` dies on the first prepare request, and +``prepare-slow`` sleeps FAKE_SERVE_BUILDER_PREPARE_DELAY_SECONDS before every +prepare response, so a test can make the daemon exchange itself outlast a +budget that was consumed waiting for the daemon's lock. ``prepare-miscount`` +answers with byte-exact windows whose declared record_count is one too many. """ import json import os import sys +import time +from pathlib import Path def one_shot() -> None: @@ -19,9 +40,273 @@ def one_shot() -> None: sys.stdout.flush() +def _fold_windows(): + """Import the shipped fold lazily; the repo root is not on script paths.""" + repo_root = Path(__file__).resolve().parents[2] + if str(repo_root) not in sys.path: + sys.path.insert(0, str(repo_root)) + from lab.prism.share_ledger import ( # noqa: PLC0415 - script-local import + AcceptedShareRecord, + IncrementalShareWindow, + IncrementalWindowFallback, + ) + + return AcceptedShareRecord, IncrementalShareWindow, IncrementalWindowFallback + + +def _declared_domain(): + """The real daemon's declared widths and rejection vocabulary, lazily. + + Taken from the parity oracle so the fake's ``out_of_range`` and + ``rejection`` answers are classified by the same tables the harness + holds the Rust daemon to, rather than by a third copy. + """ + repo_root = Path(__file__).resolve().parents[2] + if str(repo_root) not in sys.path: + sys.path.insert(0, str(repo_root)) + from tests.window_pipeline_parity import ( # noqa: PLC0415 - script-local import + RUST_INTEGER_DOMAIN, + rejection_category_for_message, + ) + + return RUST_INTEGER_DOMAIN, rejection_category_for_message + + +def _out_of_range(request: dict, domain) -> tuple[str, str] | None: + """The first prepare_window integer outside the declared widths. + + Mirrors the real daemon's classifier: the top-level fields first, then + each record's integer fields in declaration order; returns the field + path and the width it missed. + """ + checks = [ + ("window_weight", request.get("window_weight"), domain.window_weight), + ("page_size", request.get("page_size"), domain.page_size), + ( + "anchor_job_issued_at_ms", + request.get("anchor_job_issued_at_ms"), + domain.timestamps_ms, + ), + ] + for index, record in enumerate(request.get("records", [])): + for name in ( + "share_seq", + "share_difficulty", + "network_difficulty", + "template_height", + "job_issued_at_ms", + "accepted_at_ms", + "ntime", + ): + checks.append( + ( + f"records[{index}].{name}", + record.get(name), + domain.record_field_width(name), + ) + ) + for field, value, width in checks: + if isinstance(value, int) and not width.admits(value): + return field, width.name + return None + + +def _write_out_of_range(field: str, width: str) -> None: + _write_response( + { + "ok": False, + "request": "prepare_window", + "out_of_range": True, + "field": field, + "width": width, + "error": f"prepare_window {field} is outside the declared {width}", + } + ) + + +def _write_response(payload: dict, raw_section: bytes | None = None) -> None: + sys.stdout.write(json.dumps(payload) + "\n") + sys.stdout.flush() + if raw_section is not None: + sys.stdout.buffer.write(raw_section + b"\n") + sys.stdout.buffer.flush() + + +def _canonical_items(window) -> bytes: + return b",".join( + page.canonical_json_items + for page in window.pages + if page.canonical_json_items + ) + + +def _handle_prepare_window(request: dict, prepared: dict, mode: str) -> None: + if mode == "crash-during-prepare": + sys.exit(1) + if mode == "prepare-slow": + time.sleep( + float(os.environ.get("FAKE_SERVE_BUILDER_PREPARE_DELAY_SECONDS", "0")) + ) + if mode == "prepare-error": + _write_response( + { + "ok": False, + "request": "prepare_window", + "error": "injected prepare failure", + } + ) + return + miscount = 1 if mode == "prepare-miscount" else 0 + domain, category_for_message = _declared_domain() + out_of_range = _out_of_range(request, domain) + if out_of_range is not None: + # Like the real daemon: declined before anything is folded, state + # untouched, and the coordinator folds this window in-process. + _write_out_of_range(*out_of_range) + return + record_type, window_type, fallback_type = _fold_windows() + records = [ + record_type( + **{ + key: value + for key, value in record_json.items() + if value is not None or key == "credit_policy" + } + ) + for record_json in request.get("records", []) + ] + epoch = int(request.get("append_invalidation_epoch", 0)) + anchor = int(request["anchor_job_issued_at_ms"]) + prepare_mode = request.get("mode") + if prepare_mode == "full": + try: + window = window_type.from_full_snapshot( + records, + anchor_job_issued_at_ms=anchor, + window_weight=int(request["window_weight"]), + page_size=int(request.get("page_size", 512)), + ) + except ValueError as error: + _write_response( + { + "ok": False, + "request": "prepare_window", + "fold_invalid": True, + "rejection": category_for_message(str(error)), + "error": str(error), + } + ) + return + digest = window.json_records().canonical_json_sha256() + items = _canonical_items(window) + prepared.clear() + prepared[digest] = (window, epoch) + _write_response( + { + "ok": True, + "request": "prepare_window", + "share_snapshot_sha256": digest, + "record_count": window.record_count + miscount, + "added_rows": 0, + "expired_rows": 0, + "touched_pages": 0, + "window_items_len": len(items), + }, + items, + ) + return + if prepare_mode != "advance": + _write_response( + { + "ok": False, + "request": "prepare_window", + "error": f"unsupported prepare_window mode: {prepare_mode}", + } + ) + return + base_digest = str(request.get("base_digest", "")) + held = prepared.get(base_digest) + # Addressed by digest alone, like the real daemon: the epoch recorded + # beside the window is diagnostic, never an eviction rule, so a base the + # coordinator retagged still advances in place. + if mode == "prepare-forget-windows" or held is None: + _write_response( + { + "ok": False, + "request": "prepare_window", + "needs_full": True, + "error": f"prepared window {base_digest} is not held", + } + ) + return + if mode == "prepare-fallback": + _write_response( + { + "ok": False, + "request": "prepare_window", + "fallback": True, + "rejection": "delta_not_append", + "error": "injected advance invariant failure", + } + ) + return + base_window = held[0] + old_items = _canonical_items(base_window) + try: + advanced, stats = base_window.advance( + records, + anchor_job_issued_at_ms=anchor, + ) + except fallback_type as error: + _write_response( + { + "ok": False, + "request": "prepare_window", + "fallback": True, + "rejection": category_for_message(str(error)), + "error": str(error), + } + ) + return + base_epoch = held[1] + digest = advanced.json_records().canonical_json_sha256() + new_items = _canonical_items(advanced) + # Byte surgery from the old stream to the new: expiry only drops a + # prefix and appends only extend the tail, so the retained old suffix is + # a prefix of the new stream. Scanning for the smallest such drop keeps + # the fake honest against the fold it just ran. + drop = 0 + while drop <= len(old_items): + retained = old_items[drop:] + if new_items[: len(retained)] == retained: + break + drop += 1 + appended = new_items[len(old_items) - drop :] + prepared.clear() + prepared[digest] = (advanced, epoch) + _write_response( + { + "ok": True, + "request": "prepare_window", + "share_snapshot_sha256": digest, + "record_count": advanced.record_count + miscount, + "added_rows": stats.added_rows, + "expired_rows": stats.expired_rows, + "touched_pages": stats.touched_pages, + "retained_drop_bytes": drop, + "appended_items_len": len(appended), + "base_append_invalidation_epoch": base_epoch, + "append_invalidation_epoch": epoch, + }, + appended, + ) + + def serve() -> None: mode = os.environ.get("FAKE_SERVE_BUILDER_MODE", "ok") - protocol = 99 if mode == "protocol-mismatch" else 1 + protocol = int(os.environ.get("FAKE_SERVE_BUILDER_PROTOCOL", "2")) + if mode == "protocol-mismatch": + protocol = 99 sys.stdout.write( json.dumps( { @@ -43,6 +328,7 @@ def serve() -> None: sys.stdin.read() sys.exit(0) cache: dict[str, dict[str, object]] = {} + prepared: dict[str, tuple[object, int]] = {} hits = 0 misses = 0 for line in sys.stdin: @@ -50,6 +336,9 @@ def serve() -> None: if not line: continue request = json.loads(line) + if request.get("request") == "prepare_window": + _handle_prepare_window(request, prepared, mode) + continue key = request["window_key"]["share_snapshot_sha256"] had_window = bool(request.get("compact_shares")) if had_window: @@ -67,6 +356,9 @@ def serve() -> None: # Promote like the real daemon: a hit entry moves to # most-recent position before any later eviction. cache[key] = cache.pop(key) + elif key in prepared: + # The unified cache: a prepared window serves builds directly. + hits += 1 else: sys.stdout.write( json.dumps( @@ -86,7 +378,7 @@ def serve() -> None: "found_block": request["found_block"], "signed_coinbase_manifest": {"manifest": {}}, "payout_policy_manifest": {"accounts": []}, - "window": cache[key], + "window": cache.get(key, {"prepared": True}), "request_had_window": had_window, "transport": "serve", }, @@ -99,7 +391,7 @@ def serve() -> None: "hit": not had_window, "hits": hits, "misses": misses, - "entries": len(cache), + "entries": len(cache) + len(prepared), }, } sys.stdout.write(json.dumps(response) + "\n") diff --git a/tests/fixtures/mainnet-compose.env b/tests/fixtures/mainnet-compose.env index 8ed057d3d..c52d2d944 100644 --- a/tests/fixtures/mainnet-compose.env +++ b/tests/fixtures/mainnet-compose.env @@ -13,6 +13,7 @@ BITCOIN_DATA_SOURCE=/srv/qbit-mining-bootstrap/mainnet/bitcoin PRISM_POSTGRES_DATA_SOURCE=/srv/qbit-mining-bootstrap/mainnet/postgres/data PRISM_POSTGRES_WAL_SOURCE=/srv/qbit-mining-bootstrap/mainnet/postgres/wal PRISM_AUDIT_DATA_SOURCE=/srv/qbit-mining-bootstrap/mainnet/prism/audit +PRISM_POSTGRES_REPLICA_DATA_SOURCE=/srv/qbit-mining-bootstrap/mainnet/postgres-replica/data QBIT_PROVIDER=source QBIT_PRODUCTION=1 @@ -73,10 +74,14 @@ PRISM_STRATUM_HIGHDIFF_PORT_HOST=0.0.0.0:4334 PRISM_TEMPLATE_REFRESH_FAILURE_EXIT_SECONDS=120 PRISM_COORDINATION_BLOCKED_EXIT_SECONDS=900 PRISM_AUDIT_BIND=127.0.0.1 +PRISM_PUBLIC_REPLICA_MODE=require +PRISM_PUBLIC_REPLICA_MAX_LAG_SECONDS=60 +PRISM_POSTGRES_REPLICATION_SLOT=prism_public_replica PRISM_POSTGRES_USER=render-prism PRISM_POSTGRES_PASSWORD=synthetic-postgres-password-1ae478 PRISM_POSTGRES_DB=render_prism PRISM_DATABASE_URL=postgresql://render-prism:synthetic-postgres-password-1ae478@prism-postgres:5432/render_prism +PRISM_PUBLIC_DATABASE_URL=postgresql://render-prism:synthetic-postgres-password-1ae478@prism-postgres-replica:5432/render_prism PRISM_LEDGER_WRITER_ID=render-writer PRISM_LEDGER_WRITER_EPOCH=1 PRISM_LEDGER_WRITER_SESSION_TOKEN= @@ -89,7 +94,7 @@ PRISM_CTV_FANOUT_FEE_PREMIUM_BPS=12000 PRISM_CTV_BROADCASTER_ENABLED=1 PRISM_CTV_BROADCASTER_WALLET= PRISM_CTV_BROADCASTER_FEE_BITS=0 -PRISM_STRATUM_STALE_GRACE_SECONDS=0 +PRISM_STRATUM_STALE_GRACE_SECONDS=3 PRISM_ALLOW_MEMORY_LEDGER=0 PRISM_ALLOW_TEST_SIGNING_SEEDS=0 PRISM_ALLOW_BUNDLE_EMBEDDED_LEDGER_KEY=0 diff --git a/tests/fixtures/window_pipeline_parity/reference.json b/tests/fixtures/window_pipeline_parity/reference.json new file mode 100644 index 000000000..7125ccb84 --- /dev/null +++ b/tests/fixtures/window_pipeline_parity/reference.json @@ -0,0 +1,2666 @@ +{ + "cases": { + "advance-delta-exceeds-window": { + "advance_stats": [ + { + "added_rows": 5, + "expired_rows": 6, + "touched_pages": 1 + } + ], + "canonical_bytes_len": 661, + "canonical_bytes_sha256": "0cc864c698bb697517a352608a30edcee9d658b9f120613c8f93e27a5e80f56f", + "canonical_digest": "0cc864c698bb697517a352608a30edcee9d658b9f120613c8f93e27a5e80f56f", + "input_sha256": "84bb3dd25b8c14a9f51526bed4b7dbd171e8028e8f7f353d4bc512eabf24e1b9", + "pinned_literals": { + "canonical_bytes": "[{\"accepted_at_ms\":110001,\"job_id\":\"job-7\",\"job_issued_at_ms\":110001,\"miner_id\":\"miner-7\",\"network_difficulty\":1000007,\"ntime\":1700000007,\"order_key\":\"07:miner-7\",\"p2mr_program_hex\":\"0707070707070707070707070707070707070707070707070707070707070707\",\"share_difficulty\":5,\"share_id\":\"share-7\",\"share_seq\":7,\"template_height\":800000},{\"accepted_at_ms\":110001,\"job_id\":\"job-8\",\"job_issued_at_ms\":110001,\"miner_id\":\"miner-8\",\"network_difficulty\":1000008,\"ntime\":1700000008,\"order_key\":\"08:miner-8\",\"p2mr_program_hex\":\"0808080808080808080808080808080808080808080808080808080808080808\",\"share_difficulty\":5,\"share_id\":\"share-8\",\"share_seq\":8,\"template_height\":800000}]", + "input": { + "advances": [ + { + "anchor_job_issued_at_ms": 110001, + "records": [ + { + "accepted_at_ms": 110001, + "credit_policy": null, + "job_id": "job-4", + "job_issued_at_ms": 110001, + "miner_id": "miner-4", + "network_difficulty": 1000004, + "ntime": 1700000004, + "order_key": "04:miner-4", + "p2mr_program_hex": "0404040404040404040404040404040404040404040404040404040404040404", + "share_difficulty": 5, + "share_id": "share-4", + "share_seq": 4, + "template_height": 800000 + }, + { + "accepted_at_ms": 110001, + "credit_policy": null, + "job_id": "job-5", + "job_issued_at_ms": 110001, + "miner_id": "miner-5", + "network_difficulty": 1000005, + "ntime": 1700000005, + "order_key": "05:miner-5", + "p2mr_program_hex": "0505050505050505050505050505050505050505050505050505050505050505", + "share_difficulty": 5, + "share_id": "share-5", + "share_seq": 5, + "template_height": 800000 + }, + { + "accepted_at_ms": 110001, + "credit_policy": null, + "job_id": "job-6", + "job_issued_at_ms": 110001, + "miner_id": "miner-6", + "network_difficulty": 1000006, + "ntime": 1700000006, + "order_key": "06:miner-6", + "p2mr_program_hex": "0606060606060606060606060606060606060606060606060606060606060606", + "share_difficulty": 5, + "share_id": "share-6", + "share_seq": 6, + "template_height": 800000 + }, + { + "accepted_at_ms": 110001, + "credit_policy": null, + "job_id": "job-7", + "job_issued_at_ms": 110001, + "miner_id": "miner-7", + "network_difficulty": 1000007, + "ntime": 1700000007, + "order_key": "07:miner-7", + "p2mr_program_hex": "0707070707070707070707070707070707070707070707070707070707070707", + "share_difficulty": 5, + "share_id": "share-7", + "share_seq": 7, + "template_height": 800000 + }, + { + "accepted_at_ms": 110001, + "credit_policy": null, + "job_id": "job-8", + "job_issued_at_ms": 110001, + "miner_id": "miner-8", + "network_difficulty": 1000008, + "ntime": 1700000008, + "order_key": "08:miner-8", + "p2mr_program_hex": "0808080808080808080808080808080808080808080808080808080808080808", + "share_difficulty": 5, + "share_id": "share-8", + "share_seq": 8, + "template_height": 800000 + } + ] + } + ], + "name": "advance-delta-exceeds-window", + "page_size": 2, + "schema": "qbit-prism-window-pipeline-parity-input/v1", + "snapshot": { + "anchor_job_issued_at_ms": 110000, + "records": [ + { + "accepted_at_ms": 109999, + "credit_policy": null, + "job_id": "job-1", + "job_issued_at_ms": 109999, + "miner_id": "miner-1", + "network_difficulty": 1000001, + "ntime": 1700000001, + "order_key": "01:miner-1", + "p2mr_program_hex": "0101010101010101010101010101010101010101010101010101010101010101", + "share_difficulty": 3, + "share_id": "share-1", + "share_seq": 1, + "template_height": 800000 + }, + { + "accepted_at_ms": 109998, + "credit_policy": null, + "job_id": "job-2", + "job_issued_at_ms": 109998, + "miner_id": "miner-2", + "network_difficulty": 1000002, + "ntime": 1700000002, + "order_key": "02:miner-2", + "p2mr_program_hex": "0202020202020202020202020202020202020202020202020202020202020202", + "share_difficulty": 3, + "share_id": "share-2", + "share_seq": 2, + "template_height": 800000 + }, + { + "accepted_at_ms": 109997, + "credit_policy": null, + "job_id": "job-3", + "job_issued_at_ms": 109997, + "miner_id": "miner-3", + "network_difficulty": 1000003, + "ntime": 1700000003, + "order_key": "03:miner-3", + "p2mr_program_hex": "0303030303030303030303030303030303030303030303030303030303030303", + "share_difficulty": 3, + "share_id": "share-3", + "share_seq": 3, + "template_height": 800000 + } + ] + }, + "window_weight": 8 + }, + "record_jsons": [ + "{\"accepted_at_ms\":110001,\"job_id\":\"job-7\",\"job_issued_at_ms\":110001,\"miner_id\":\"miner-7\",\"network_difficulty\":1000007,\"ntime\":1700000007,\"order_key\":\"07:miner-7\",\"p2mr_program_hex\":\"0707070707070707070707070707070707070707070707070707070707070707\",\"share_difficulty\":5,\"share_id\":\"share-7\",\"share_seq\":7,\"template_height\":800000}", + "{\"accepted_at_ms\":110001,\"job_id\":\"job-8\",\"job_issued_at_ms\":110001,\"miner_id\":\"miner-8\",\"network_difficulty\":1000008,\"ntime\":1700000008,\"order_key\":\"08:miner-8\",\"p2mr_program_hex\":\"0808080808080808080808080808080808080808080808080808080808080808\",\"share_difficulty\":5,\"share_id\":\"share-8\",\"share_seq\":8,\"template_height\":800000}" + ], + "spool_tail": ",\"compact_share_identities\":[[\"miner-7\",\"07:miner-7\",\"0707070707070707070707070707070707070707070707070707070707070707\"],[\"miner-8\",\"08:miner-8\",\"0808080808080808080808080808080808080808080808080808080808080808\"]],\"compact_shares\":[[7,\"share-7\",0,5,110001,110001,null],[8,\"share-8\",1,5,110001,110001,null]]}" + }, + "record_count": 2, + "record_stream_len": 659, + "record_stream_sha256": "80c489e0067b94669cf48e92672fa3d3657fdeb40b4a8dbb0e5c96b182690d2c", + "spool_tail_len": 307, + "spool_tail_sha256": "beb3a0ce842b773d3f8cf4fad44a94f0bb53c951c5aec45b813d16f32b8b885b", + "why": "weight 8, page_size 2, three retained records of difficulty 3 in pages [1,2],[3], then one advance of five records of difficulty 5 (25 alone outweighs the window): the delta fills page [3,4] and allocates [5,6],[7,8]; expiry must run across the appended pages too, expiring every old page and three of the five new rows to retain [7,8] at total 10. A backend that expires only among pre-existing pages -- the natural shape of a drop-prefix / append-suffix protocol -- retains four. touched_pages is 1: the page appended into counts although this same advance then expires it wholesale" + }, + "advance-empty-delta": { + "advance_stats": [ + { + "added_rows": 0, + "expired_rows": 0, + "touched_pages": 0 + } + ], + "canonical_bytes_len": 331, + "canonical_bytes_sha256": "010442bcb4f76254e6bae75a121c8b7d1f26d2341997c9f0dea2f1ca6455cfa6", + "canonical_digest": "010442bcb4f76254e6bae75a121c8b7d1f26d2341997c9f0dea2f1ca6455cfa6", + "input_sha256": "6937a8d8455d4f6045bb3f4fa753d51958ab25c464bacbd5ee0adfd053a99fb3", + "pinned_literals": { + "canonical_bytes": "[{\"accepted_at_ms\":140000,\"job_id\":\"job-1\",\"job_issued_at_ms\":140000,\"miner_id\":\"miner-1\",\"network_difficulty\":1000001,\"ntime\":1700000001,\"order_key\":\"01:miner-1\",\"p2mr_program_hex\":\"0101010101010101010101010101010101010101010101010101010101010101\",\"share_difficulty\":5,\"share_id\":\"share-1\",\"share_seq\":1,\"template_height\":800000}]", + "input": { + "advances": [ + { + "anchor_job_issued_at_ms": 140001, + "records": [] + } + ], + "name": "advance-empty-delta", + "page_size": 512, + "schema": "qbit-prism-window-pipeline-parity-input/v1", + "snapshot": { + "anchor_job_issued_at_ms": 140000, + "records": [ + { + "accepted_at_ms": 140000, + "credit_policy": null, + "job_id": "job-1", + "job_issued_at_ms": 140000, + "miner_id": "miner-1", + "network_difficulty": 1000001, + "ntime": 1700000001, + "order_key": "01:miner-1", + "p2mr_program_hex": "0101010101010101010101010101010101010101010101010101010101010101", + "share_difficulty": 5, + "share_id": "share-1", + "share_seq": 1, + "template_height": 800000 + } + ] + }, + "window_weight": 10 + }, + "record_jsons": [ + "{\"accepted_at_ms\":140000,\"job_id\":\"job-1\",\"job_issued_at_ms\":140000,\"miner_id\":\"miner-1\",\"network_difficulty\":1000001,\"ntime\":1700000001,\"order_key\":\"01:miner-1\",\"p2mr_program_hex\":\"0101010101010101010101010101010101010101010101010101010101010101\",\"share_difficulty\":5,\"share_id\":\"share-1\",\"share_seq\":1,\"template_height\":800000}" + ], + "spool_tail": ",\"compact_share_identities\":[[\"miner-1\",\"01:miner-1\",\"0101010101010101010101010101010101010101010101010101010101010101\"]],\"compact_shares\":[[1,\"share-1\",0,5,140000,140000,null]]}" + }, + "record_count": 1, + "record_stream_len": 329, + "record_stream_sha256": "deef1605e8fb67008b48a9b3a787e9d07e0dbab1ffd88cf699c028d70c16f50b", + "spool_tail_len": 178, + "spool_tail_sha256": "e11c6455b641f8b9e99c2b1d52cdbaf989c3eb83c820b1a59645044b48a7a52d", + "why": "one retained record advanced to a later anchor with an empty delta: bytes identical to the snapshot, stats all zero; a backend that allocates an empty page and frames a trailing separator, or re-runs expiry with a stale total, diverges" + }, + "advance-equal-anchor": { + "advance_stats": [ + { + "added_rows": 0, + "expired_rows": 0, + "touched_pages": 0 + } + ], + "canonical_bytes_len": 331, + "canonical_bytes_sha256": "b2f069c0dcfae436b255938de6b299a4e35b3b3258f73783dda61fe29e57cd3e", + "canonical_digest": "b2f069c0dcfae436b255938de6b299a4e35b3b3258f73783dda61fe29e57cd3e", + "input_sha256": "20932ec79dc21e0cd8670b17a5b4b076fe640405b3aaad153cdf3c26b1f4fe67", + "pinned_literals": { + "canonical_bytes": "[{\"accepted_at_ms\":150000,\"job_id\":\"job-1\",\"job_issued_at_ms\":150000,\"miner_id\":\"miner-1\",\"network_difficulty\":1000001,\"ntime\":1700000001,\"order_key\":\"01:miner-1\",\"p2mr_program_hex\":\"0101010101010101010101010101010101010101010101010101010101010101\",\"share_difficulty\":5,\"share_id\":\"share-1\",\"share_seq\":1,\"template_height\":800000}]", + "input": { + "advances": [ + { + "anchor_job_issued_at_ms": 150000, + "records": [] + } + ], + "name": "advance-equal-anchor", + "page_size": 512, + "schema": "qbit-prism-window-pipeline-parity-input/v1", + "snapshot": { + "anchor_job_issued_at_ms": 150000, + "records": [ + { + "accepted_at_ms": 150000, + "credit_policy": null, + "job_id": "job-1", + "job_issued_at_ms": 150000, + "miner_id": "miner-1", + "network_difficulty": 1000001, + "ntime": 1700000001, + "order_key": "01:miner-1", + "p2mr_program_hex": "0101010101010101010101010101010101010101010101010101010101010101", + "share_difficulty": 5, + "share_id": "share-1", + "share_seq": 1, + "template_height": 800000 + } + ] + }, + "window_weight": 10 + }, + "record_jsons": [ + "{\"accepted_at_ms\":150000,\"job_id\":\"job-1\",\"job_issued_at_ms\":150000,\"miner_id\":\"miner-1\",\"network_difficulty\":1000001,\"ntime\":1700000001,\"order_key\":\"01:miner-1\",\"p2mr_program_hex\":\"0101010101010101010101010101010101010101010101010101010101010101\",\"share_difficulty\":5,\"share_id\":\"share-1\",\"share_seq\":1,\"template_height\":800000}" + ], + "spool_tail": ",\"compact_share_identities\":[[\"miner-1\",\"01:miner-1\",\"0101010101010101010101010101010101010101010101010101010101010101\"]],\"compact_shares\":[[1,\"share-1\",0,5,150000,150000,null]]}" + }, + "record_count": 1, + "record_stream_len": 329, + "record_stream_sha256": "d2f9755789ca9e3c8790eec9be44850885cafe20993bb961216bcb6553e85176", + "spool_tail_len": 178, + "spool_tail_sha256": "ff997843fa1b6b680e4fcc1bb762711184b8c256d2b178efd769297b5d680cf5", + "why": "an advance at the same anchor as the snapshot: the regression check is strictly <, so this is accepted (only an empty delta can be, since no record is both newly eligible and eligible at the same anchor); a backend rejecting on <= diverges" + }, + "advance-exact-fit-expiry": { + "advance_stats": [ + { + "added_rows": 1, + "expired_rows": 1, + "touched_pages": 1 + } + ], + "canonical_bytes_len": 987, + "canonical_bytes_sha256": "cb191fc6612c0f74e533a6a66f001e0c2cba0df34e86e5f4c360644db90c97a2", + "canonical_digest": "cb191fc6612c0f74e533a6a66f001e0c2cba0df34e86e5f4c360644db90c97a2", + "input_sha256": "6a4d7f49036980b4ddc901598d76cb08fcc3fe0b9469a944fbafd420240d8de5", + "pinned_literals": { + "canonical_bytes": "[{\"accepted_at_ms\":99998,\"job_id\":\"job-2\",\"job_issued_at_ms\":99998,\"miner_id\":\"miner-2\",\"network_difficulty\":1000002,\"ntime\":1700000002,\"order_key\":\"02:miner-2\",\"p2mr_program_hex\":\"0202020202020202020202020202020202020202020202020202020202020202\",\"share_difficulty\":4,\"share_id\":\"share-2\",\"share_seq\":2,\"template_height\":800000},{\"accepted_at_ms\":99997,\"job_id\":\"job-3\",\"job_issued_at_ms\":99997,\"miner_id\":\"miner-3\",\"network_difficulty\":1000003,\"ntime\":1700000003,\"order_key\":\"03:miner-3\",\"p2mr_program_hex\":\"0303030303030303030303030303030303030303030303030303030303030303\",\"share_difficulty\":4,\"share_id\":\"share-3\",\"share_seq\":3,\"template_height\":800000},{\"accepted_at_ms\":100001,\"job_id\":\"job-4\",\"job_issued_at_ms\":100001,\"miner_id\":\"miner-4\",\"network_difficulty\":1000004,\"ntime\":1700000004,\"order_key\":\"04:miner-4\",\"p2mr_program_hex\":\"0404040404040404040404040404040404040404040404040404040404040404\",\"share_difficulty\":2,\"share_id\":\"share-4\",\"share_seq\":4,\"template_height\":800000}]", + "input": { + "advances": [ + { + "anchor_job_issued_at_ms": 100001, + "records": [ + { + "accepted_at_ms": 100001, + "credit_policy": null, + "job_id": "job-4", + "job_issued_at_ms": 100001, + "miner_id": "miner-4", + "network_difficulty": 1000004, + "ntime": 1700000004, + "order_key": "04:miner-4", + "p2mr_program_hex": "0404040404040404040404040404040404040404040404040404040404040404", + "share_difficulty": 2, + "share_id": "share-4", + "share_seq": 4, + "template_height": 800000 + } + ] + } + ], + "name": "advance-exact-fit-expiry", + "page_size": 512, + "schema": "qbit-prism-window-pipeline-parity-input/v1", + "snapshot": { + "anchor_job_issued_at_ms": 100000, + "records": [ + { + "accepted_at_ms": 99999, + "credit_policy": null, + "job_id": "job-1", + "job_issued_at_ms": 99999, + "miner_id": "miner-1", + "network_difficulty": 1000001, + "ntime": 1700000001, + "order_key": "01:miner-1", + "p2mr_program_hex": "0101010101010101010101010101010101010101010101010101010101010101", + "share_difficulty": 4, + "share_id": "share-1", + "share_seq": 1, + "template_height": 800000 + }, + { + "accepted_at_ms": 99998, + "credit_policy": null, + "job_id": "job-2", + "job_issued_at_ms": 99998, + "miner_id": "miner-2", + "network_difficulty": 1000002, + "ntime": 1700000002, + "order_key": "02:miner-2", + "p2mr_program_hex": "0202020202020202020202020202020202020202020202020202020202020202", + "share_difficulty": 4, + "share_id": "share-2", + "share_seq": 2, + "template_height": 800000 + }, + { + "accepted_at_ms": 99997, + "credit_policy": null, + "job_id": "job-3", + "job_issued_at_ms": 99997, + "miner_id": "miner-3", + "network_difficulty": 1000003, + "ntime": 1700000003, + "order_key": "03:miner-3", + "p2mr_program_hex": "0303030303030303030303030303030303030303030303030303030303030303", + "share_difficulty": 4, + "share_id": "share-3", + "share_seq": 3, + "template_height": 800000 + } + ] + }, + "window_weight": 10 + }, + "record_jsons": [ + "{\"accepted_at_ms\":99998,\"job_id\":\"job-2\",\"job_issued_at_ms\":99998,\"miner_id\":\"miner-2\",\"network_difficulty\":1000002,\"ntime\":1700000002,\"order_key\":\"02:miner-2\",\"p2mr_program_hex\":\"0202020202020202020202020202020202020202020202020202020202020202\",\"share_difficulty\":4,\"share_id\":\"share-2\",\"share_seq\":2,\"template_height\":800000}", + "{\"accepted_at_ms\":99997,\"job_id\":\"job-3\",\"job_issued_at_ms\":99997,\"miner_id\":\"miner-3\",\"network_difficulty\":1000003,\"ntime\":1700000003,\"order_key\":\"03:miner-3\",\"p2mr_program_hex\":\"0303030303030303030303030303030303030303030303030303030303030303\",\"share_difficulty\":4,\"share_id\":\"share-3\",\"share_seq\":3,\"template_height\":800000}", + "{\"accepted_at_ms\":100001,\"job_id\":\"job-4\",\"job_issued_at_ms\":100001,\"miner_id\":\"miner-4\",\"network_difficulty\":1000004,\"ntime\":1700000004,\"order_key\":\"04:miner-4\",\"p2mr_program_hex\":\"0404040404040404040404040404040404040404040404040404040404040404\",\"share_difficulty\":2,\"share_id\":\"share-4\",\"share_seq\":4,\"template_height\":800000}" + ], + "spool_tail": ",\"compact_share_identities\":[[\"miner-2\",\"02:miner-2\",\"0202020202020202020202020202020202020202020202020202020202020202\"],[\"miner-3\",\"03:miner-3\",\"0303030303030303030303030303030303030303030303030303030303030303\"],[\"miner-4\",\"04:miner-4\",\"0404040404040404040404040404040404040404040404040404040404040404\"]],\"compact_shares\":[[2,\"share-2\",0,4,99998,99998,null],[3,\"share-3\",1,4,99997,99997,null],[4,\"share-4\",2,2,100001,100001,null]]}" + }, + "record_count": 3, + "record_stream_len": 985, + "record_stream_sha256": "1ca31fa84e5a820e0a3fb71016aa5b69c32d5df603421c4879e08acaac8dff21", + "spool_tail_len": 432, + "spool_tail_sha256": "85429297dbbea1cee910e0e3d09cf95a74533bdad9a18c5887e2d25e8a872809", + "why": "three records of difficulty 4 at weight 10 (all retained, total 12) plus one delta of difficulty 2: 14-4 = 10 >= 10 expires the head row exactly at the weight, retaining seqs 2..4 at total 10; an advance-side expiry using > instead of >= retains all four. The crossing-row-exact-fit sibling pins the snapshot-side twin of this comparison; the advance loops are separate code in any re-implementation" + }, + "advance-from-empty-window": { + "advance_stats": [ + { + "added_rows": 1, + "expired_rows": 0, + "touched_pages": 0 + } + ], + "canonical_bytes_len": 331, + "canonical_bytes_sha256": "40eabe0f5fc237d3e35dc1f79b0bb689f4d64f11707eafeb246e186f9adc2071", + "canonical_digest": "40eabe0f5fc237d3e35dc1f79b0bb689f4d64f11707eafeb246e186f9adc2071", + "input_sha256": "b5ae4c119bfd12cf7dbbe74c960e194230000ede28c16f97ac0b24c1d2363029", + "pinned_literals": { + "canonical_bytes": "[{\"accepted_at_ms\":130001,\"job_id\":\"job-1\",\"job_issued_at_ms\":130001,\"miner_id\":\"miner-1\",\"network_difficulty\":1000001,\"ntime\":1700000001,\"order_key\":\"01:miner-1\",\"p2mr_program_hex\":\"0101010101010101010101010101010101010101010101010101010101010101\",\"share_difficulty\":5,\"share_id\":\"share-1\",\"share_seq\":1,\"template_height\":800000}]", + "input": { + "advances": [ + { + "anchor_job_issued_at_ms": 130001, + "records": [ + { + "accepted_at_ms": 130001, + "credit_policy": null, + "job_id": "job-1", + "job_issued_at_ms": 130001, + "miner_id": "miner-1", + "network_difficulty": 1000001, + "ntime": 1700000001, + "order_key": "01:miner-1", + "p2mr_program_hex": "0101010101010101010101010101010101010101010101010101010101010101", + "share_difficulty": 5, + "share_id": "share-1", + "share_seq": 1, + "template_height": 800000 + } + ] + } + ], + "name": "advance-from-empty-window", + "page_size": 512, + "schema": "qbit-prism-window-pipeline-parity-input/v1", + "snapshot": { + "anchor_job_issued_at_ms": 130000, + "records": [] + }, + "window_weight": 10 + }, + "record_jsons": [ + "{\"accepted_at_ms\":130001,\"job_id\":\"job-1\",\"job_issued_at_ms\":130001,\"miner_id\":\"miner-1\",\"network_difficulty\":1000001,\"ntime\":1700000001,\"order_key\":\"01:miner-1\",\"p2mr_program_hex\":\"0101010101010101010101010101010101010101010101010101010101010101\",\"share_difficulty\":5,\"share_id\":\"share-1\",\"share_seq\":1,\"template_height\":800000}" + ], + "spool_tail": ",\"compact_share_identities\":[[\"miner-1\",\"01:miner-1\",\"0101010101010101010101010101010101010101010101010101010101010101\"]],\"compact_shares\":[[1,\"share-1\",0,5,130001,130001,null]]}" + }, + "record_count": 1, + "record_stream_len": 329, + "record_stream_sha256": "b0bd339465f085c23ebee2c2e242229ca8181cfd29b9a7b0ab0e44716316f492", + "spool_tail_len": 178, + "spool_tail_sha256": "c846293f15491bfccf89339164bf7e0530d915f35e3040f7eb37141937534908", + "why": "an empty window (no pages, no last page to append into) advanced with a one-record delta: the record lands in a freshly allocated page, nothing expires, touched_pages is 0; a backend that unwraps the last page fails here" + }, + "advance-partial-expiry-in-appended-page": { + "advance_stats": [ + { + "added_rows": 5, + "expired_rows": 7, + "touched_pages": 1 + } + ], + "canonical_bytes_len": 331, + "canonical_bytes_sha256": "02d82a152350657e5bb80aebad11f017152b73b1df93460872814ae9ebf95e89", + "canonical_digest": "02d82a152350657e5bb80aebad11f017152b73b1df93460872814ae9ebf95e89", + "input_sha256": "08cc4439f89d75dc729909cfeae7e0ad2126249b7dcfc7c6c9302acd3fce42ba", + "pinned_literals": { + "canonical_bytes": "[{\"accepted_at_ms\":120001,\"job_id\":\"job-8\",\"job_issued_at_ms\":120001,\"miner_id\":\"miner-8\",\"network_difficulty\":1000008,\"ntime\":1700000008,\"order_key\":\"08:miner-8\",\"p2mr_program_hex\":\"0808080808080808080808080808080808080808080808080808080808080808\",\"share_difficulty\":9,\"share_id\":\"share-8\",\"share_seq\":8,\"template_height\":800000}]", + "input": { + "advances": [ + { + "anchor_job_issued_at_ms": 120001, + "records": [ + { + "accepted_at_ms": 120001, + "credit_policy": null, + "job_id": "job-4", + "job_issued_at_ms": 120001, + "miner_id": "miner-4", + "network_difficulty": 1000004, + "ntime": 1700000004, + "order_key": "04:miner-4", + "p2mr_program_hex": "0404040404040404040404040404040404040404040404040404040404040404", + "share_difficulty": 5, + "share_id": "share-4", + "share_seq": 4, + "template_height": 800000 + }, + { + "accepted_at_ms": 120001, + "credit_policy": null, + "job_id": "job-5", + "job_issued_at_ms": 120001, + "miner_id": "miner-5", + "network_difficulty": 1000005, + "ntime": 1700000005, + "order_key": "05:miner-5", + "p2mr_program_hex": "0505050505050505050505050505050505050505050505050505050505050505", + "share_difficulty": 5, + "share_id": "share-5", + "share_seq": 5, + "template_height": 800000 + }, + { + "accepted_at_ms": 120001, + "credit_policy": null, + "job_id": "job-6", + "job_issued_at_ms": 120001, + "miner_id": "miner-6", + "network_difficulty": 1000006, + "ntime": 1700000006, + "order_key": "06:miner-6", + "p2mr_program_hex": "0606060606060606060606060606060606060606060606060606060606060606", + "share_difficulty": 5, + "share_id": "share-6", + "share_seq": 6, + "template_height": 800000 + }, + { + "accepted_at_ms": 120001, + "credit_policy": null, + "job_id": "job-7", + "job_issued_at_ms": 120001, + "miner_id": "miner-7", + "network_difficulty": 1000007, + "ntime": 1700000007, + "order_key": "07:miner-7", + "p2mr_program_hex": "0707070707070707070707070707070707070707070707070707070707070707", + "share_difficulty": 2, + "share_id": "share-7", + "share_seq": 7, + "template_height": 800000 + }, + { + "accepted_at_ms": 120001, + "credit_policy": null, + "job_id": "job-8", + "job_issued_at_ms": 120001, + "miner_id": "miner-8", + "network_difficulty": 1000008, + "ntime": 1700000008, + "order_key": "08:miner-8", + "p2mr_program_hex": "0808080808080808080808080808080808080808080808080808080808080808", + "share_difficulty": 9, + "share_id": "share-8", + "share_seq": 8, + "template_height": 800000 + } + ] + } + ], + "name": "advance-partial-expiry-in-appended-page", + "page_size": 2, + "schema": "qbit-prism-window-pipeline-parity-input/v1", + "snapshot": { + "anchor_job_issued_at_ms": 120000, + "records": [ + { + "accepted_at_ms": 119999, + "credit_policy": null, + "job_id": "job-1", + "job_issued_at_ms": 119999, + "miner_id": "miner-1", + "network_difficulty": 1000001, + "ntime": 1700000001, + "order_key": "01:miner-1", + "p2mr_program_hex": "0101010101010101010101010101010101010101010101010101010101010101", + "share_difficulty": 3, + "share_id": "share-1", + "share_seq": 1, + "template_height": 800000 + }, + { + "accepted_at_ms": 119998, + "credit_policy": null, + "job_id": "job-2", + "job_issued_at_ms": 119998, + "miner_id": "miner-2", + "network_difficulty": 1000002, + "ntime": 1700000002, + "order_key": "02:miner-2", + "p2mr_program_hex": "0202020202020202020202020202020202020202020202020202020202020202", + "share_difficulty": 3, + "share_id": "share-2", + "share_seq": 2, + "template_height": 800000 + }, + { + "accepted_at_ms": 119997, + "credit_policy": null, + "job_id": "job-3", + "job_issued_at_ms": 119997, + "miner_id": "miner-3", + "network_difficulty": 1000003, + "ntime": 1700000003, + "order_key": "03:miner-3", + "p2mr_program_hex": "0303030303030303030303030303030303030303030303030303030303030303", + "share_difficulty": 3, + "share_id": "share-3", + "share_seq": 3, + "template_height": 800000 + } + ] + }, + "window_weight": 8 + }, + "record_jsons": [ + "{\"accepted_at_ms\":120001,\"job_id\":\"job-8\",\"job_issued_at_ms\":120001,\"miner_id\":\"miner-8\",\"network_difficulty\":1000008,\"ntime\":1700000008,\"order_key\":\"08:miner-8\",\"p2mr_program_hex\":\"0808080808080808080808080808080808080808080808080808080808080808\",\"share_difficulty\":9,\"share_id\":\"share-8\",\"share_seq\":8,\"template_height\":800000}" + ], + "spool_tail": ",\"compact_share_identities\":[[\"miner-8\",\"08:miner-8\",\"0808080808080808080808080808080808080808080808080808080808080808\"]],\"compact_shares\":[[8,\"share-8\",0,9,120001,120001,null]]}" + }, + "record_count": 1, + "record_stream_len": 329, + "record_stream_sha256": "578adf08e99c120f8d9191cfdc0de6c43295ddc947adf012344a0419d4651bb3", + "spool_tail_len": 178, + "spool_tail_sha256": "6591c845f4d4a43859699cfe5cf7a20fda1515067f019bab55b76e59c6b5fc3e", + "why": "as advance-delta-exceeds-window but with delta difficulties (5,5,5,2,9): whole-page expiry stops at the freshly allocated page [7,8] (total 11) and the partial loop then expires seq 7 inside it, retaining [8] at total 9; touched_pages stays 1 because a freshly allocated append page never counts even when partial expiry lands in it" + }, + "bulk-seeded": { + "advance_stats": [ + { + "added_rows": 700, + "expired_rows": 650, + "touched_pages": 2 + } + ], + "canonical_bytes_len": 939133, + "canonical_bytes_sha256": "2c4aeccace2af085e7463f59cf2f9dc67c5e66a3b0a1d9dcb355e504cb581daa", + "canonical_digest": "2c4aeccace2af085e7463f59cf2f9dc67c5e66a3b0a1d9dcb355e504cb581daa", + "input_sha256": "f436c4580c2735ee9bbd21f12d1684a3bcb4cfd3cb3282481a72828913070dd4", + "record_count": 2709, + "record_stream_len": 939131, + "record_stream_sha256": "fac69fead46482a2343f3db7bc30b3d449b628cf98b10af0606993e89f6264b5", + "spool_tail_len": 373230, + "spool_tail_sha256": "7a1afc09c5895dbe71bd49f1be6a6cdf1ba46b95ca53da5db5393572fa85863e", + "why": "4000 seeded snapshot records (roughly a tenth ineligible) plus a 700-record advance: representative page-fragment concatenation, identity deduplication at natural factory collisions, and head expiry at scale" + }, + "credit-policy-identity-reuse": { + "advance_stats": [], + "canonical_bytes_len": 2029, + "canonical_bytes_sha256": "7e5624d438a160d8e94f4d4eef63fd543de01fa9c3f1317c8a653f55aa2e2aa2", + "canonical_digest": "7e5624d438a160d8e94f4d4eef63fd543de01fa9c3f1317c8a653f55aa2e2aa2", + "input_sha256": "65e8ff26cc1b1fa565ce047d035c4b89dc359d651a00f6bba1eee75c061cc1ee", + "pinned_literals": { + "canonical_bytes": "[{\"accepted_at_ms\":49999,\"job_id\":\"job-1\",\"job_issued_at_ms\":49999,\"miner_id\":\"miner-1\",\"network_difficulty\":1000001,\"ntime\":1700000001,\"order_key\":\"01:miner-1\",\"p2mr_program_hex\":\"0101010101010101010101010101010101010101010101010101010101010101\",\"share_difficulty\":1,\"share_id\":\"share-1\",\"share_seq\":1,\"template_height\":800000},{\"accepted_at_ms\":49998,\"credit_policy\":\"stale-grace\",\"job_id\":\"job-2\",\"job_issued_at_ms\":49998,\"miner_id\":\"miner-2\",\"network_difficulty\":1000002,\"ntime\":1700000002,\"order_key\":\"02:miner-2\",\"p2mr_program_hex\":\"0202020202020202020202020202020202020202020202020202020202020202\",\"share_difficulty\":2,\"share_id\":\"share-2\",\"share_seq\":2,\"template_height\":800000},{\"accepted_at_ms\":49997,\"job_id\":\"job-3\",\"job_issued_at_ms\":49997,\"miner_id\":\"miner-3\",\"network_difficulty\":1000003,\"ntime\":1700000003,\"order_key\":\"03:miner-3\",\"p2mr_program_hex\":\"0303030303030303030303030303030303030303030303030303030303030303\",\"share_difficulty\":3,\"share_id\":\"share-3\",\"share_seq\":3,\"template_height\":800000},{\"accepted_at_ms\":49996,\"job_id\":\"job-4\",\"job_issued_at_ms\":49996,\"miner_id\":\"miner-1\",\"network_difficulty\":1000004,\"ntime\":1700000004,\"order_key\":\"01:miner-1\",\"p2mr_program_hex\":\"0101010101010101010101010101010101010101010101010101010101010101\",\"share_difficulty\":4,\"share_id\":\"share-4\",\"share_seq\":4,\"template_height\":800000},{\"accepted_at_ms\":49995,\"credit_policy\":\"stale-grace\",\"job_id\":\"job-5\",\"job_issued_at_ms\":49995,\"miner_id\":\"miner-5\",\"network_difficulty\":1000005,\"ntime\":1700000005,\"order_key\":\"05:miner-5\",\"p2mr_program_hex\":\"0505050505050505050505050505050505050505050505050505050505050505\",\"share_difficulty\":5,\"share_id\":\"share-5\",\"share_seq\":5,\"template_height\":800000},{\"accepted_at_ms\":49994,\"job_id\":\"job-6\",\"job_issued_at_ms\":49994,\"miner_id\":\"miner-2\",\"network_difficulty\":1000006,\"ntime\":1700000006,\"order_key\":\"02:miner-2\",\"p2mr_program_hex\":\"0202020202020202020202020202020202020202020202020202020202020202\",\"share_difficulty\":6,\"share_id\":\"share-6\",\"share_seq\":6,\"template_height\":800000}]", + "input": { + "advances": [], + "name": "credit-policy-identity-reuse", + "page_size": 512, + "schema": "qbit-prism-window-pipeline-parity-input/v1", + "snapshot": { + "anchor_job_issued_at_ms": 50000, + "records": [ + { + "accepted_at_ms": 49999, + "credit_policy": null, + "job_id": "job-1", + "job_issued_at_ms": 49999, + "miner_id": "miner-1", + "network_difficulty": 1000001, + "ntime": 1700000001, + "order_key": "01:miner-1", + "p2mr_program_hex": "0101010101010101010101010101010101010101010101010101010101010101", + "share_difficulty": 1, + "share_id": "share-1", + "share_seq": 1, + "template_height": 800000 + }, + { + "accepted_at_ms": 49998, + "credit_policy": "stale-grace", + "job_id": "job-2", + "job_issued_at_ms": 49998, + "miner_id": "miner-2", + "network_difficulty": 1000002, + "ntime": 1700000002, + "order_key": "02:miner-2", + "p2mr_program_hex": "0202020202020202020202020202020202020202020202020202020202020202", + "share_difficulty": 2, + "share_id": "share-2", + "share_seq": 2, + "template_height": 800000 + }, + { + "accepted_at_ms": 49997, + "credit_policy": null, + "job_id": "job-3", + "job_issued_at_ms": 49997, + "miner_id": "miner-3", + "network_difficulty": 1000003, + "ntime": 1700000003, + "order_key": "03:miner-3", + "p2mr_program_hex": "0303030303030303030303030303030303030303030303030303030303030303", + "share_difficulty": 3, + "share_id": "share-3", + "share_seq": 3, + "template_height": 800000 + }, + { + "accepted_at_ms": 49996, + "credit_policy": null, + "job_id": "job-4", + "job_issued_at_ms": 49996, + "miner_id": "miner-1", + "network_difficulty": 1000004, + "ntime": 1700000004, + "order_key": "01:miner-1", + "p2mr_program_hex": "0101010101010101010101010101010101010101010101010101010101010101", + "share_difficulty": 4, + "share_id": "share-4", + "share_seq": 4, + "template_height": 800000 + }, + { + "accepted_at_ms": 49995, + "credit_policy": "stale-grace", + "job_id": "job-5", + "job_issued_at_ms": 49995, + "miner_id": "miner-5", + "network_difficulty": 1000005, + "ntime": 1700000005, + "order_key": "05:miner-5", + "p2mr_program_hex": "0505050505050505050505050505050505050505050505050505050505050505", + "share_difficulty": 5, + "share_id": "share-5", + "share_seq": 5, + "template_height": 800000 + }, + { + "accepted_at_ms": 49994, + "credit_policy": null, + "job_id": "job-6", + "job_issued_at_ms": 49994, + "miner_id": "miner-2", + "network_difficulty": 1000006, + "ntime": 1700000006, + "order_key": "02:miner-2", + "p2mr_program_hex": "0202020202020202020202020202020202020202020202020202020202020202", + "share_difficulty": 6, + "share_id": "share-6", + "share_seq": 6, + "template_height": 800000 + } + ] + }, + "window_weight": 1000000 + }, + "record_jsons": [ + "{\"accepted_at_ms\":49999,\"job_id\":\"job-1\",\"job_issued_at_ms\":49999,\"miner_id\":\"miner-1\",\"network_difficulty\":1000001,\"ntime\":1700000001,\"order_key\":\"01:miner-1\",\"p2mr_program_hex\":\"0101010101010101010101010101010101010101010101010101010101010101\",\"share_difficulty\":1,\"share_id\":\"share-1\",\"share_seq\":1,\"template_height\":800000}", + "{\"accepted_at_ms\":49998,\"credit_policy\":\"stale-grace\",\"job_id\":\"job-2\",\"job_issued_at_ms\":49998,\"miner_id\":\"miner-2\",\"network_difficulty\":1000002,\"ntime\":1700000002,\"order_key\":\"02:miner-2\",\"p2mr_program_hex\":\"0202020202020202020202020202020202020202020202020202020202020202\",\"share_difficulty\":2,\"share_id\":\"share-2\",\"share_seq\":2,\"template_height\":800000}", + "{\"accepted_at_ms\":49997,\"job_id\":\"job-3\",\"job_issued_at_ms\":49997,\"miner_id\":\"miner-3\",\"network_difficulty\":1000003,\"ntime\":1700000003,\"order_key\":\"03:miner-3\",\"p2mr_program_hex\":\"0303030303030303030303030303030303030303030303030303030303030303\",\"share_difficulty\":3,\"share_id\":\"share-3\",\"share_seq\":3,\"template_height\":800000}", + "{\"accepted_at_ms\":49996,\"job_id\":\"job-4\",\"job_issued_at_ms\":49996,\"miner_id\":\"miner-1\",\"network_difficulty\":1000004,\"ntime\":1700000004,\"order_key\":\"01:miner-1\",\"p2mr_program_hex\":\"0101010101010101010101010101010101010101010101010101010101010101\",\"share_difficulty\":4,\"share_id\":\"share-4\",\"share_seq\":4,\"template_height\":800000}", + "{\"accepted_at_ms\":49995,\"credit_policy\":\"stale-grace\",\"job_id\":\"job-5\",\"job_issued_at_ms\":49995,\"miner_id\":\"miner-5\",\"network_difficulty\":1000005,\"ntime\":1700000005,\"order_key\":\"05:miner-5\",\"p2mr_program_hex\":\"0505050505050505050505050505050505050505050505050505050505050505\",\"share_difficulty\":5,\"share_id\":\"share-5\",\"share_seq\":5,\"template_height\":800000}", + "{\"accepted_at_ms\":49994,\"job_id\":\"job-6\",\"job_issued_at_ms\":49994,\"miner_id\":\"miner-2\",\"network_difficulty\":1000006,\"ntime\":1700000006,\"order_key\":\"02:miner-2\",\"p2mr_program_hex\":\"0202020202020202020202020202020202020202020202020202020202020202\",\"share_difficulty\":6,\"share_id\":\"share-6\",\"share_seq\":6,\"template_height\":800000}" + ], + "spool_tail": ",\"compact_share_identities\":[[\"miner-1\",\"01:miner-1\",\"0101010101010101010101010101010101010101010101010101010101010101\"],[\"miner-2\",\"02:miner-2\",\"0202020202020202020202020202020202020202020202020202020202020202\"],[\"miner-3\",\"03:miner-3\",\"0303030303030303030303030303030303030303030303030303030303030303\"],[\"miner-5\",\"05:miner-5\",\"0505050505050505050505050505050505050505050505050505050505050505\"]],\"compact_shares\":[[1,\"share-1\",0,1,49999,49999,null],[2,\"share-2\",1,2,49998,49998,\"stale-grace\"],[3,\"share-3\",2,3,49997,49997,null],[4,\"share-4\",0,4,49996,49996,null],[5,\"share-5\",3,5,49995,49995,\"stale-grace\"],[6,\"share-6\",1,6,49994,49994,null]]}" + }, + "record_count": 6, + "record_stream_len": 2027, + "record_stream_sha256": "709ebd145614ea5e0c475c6374f7f9f95ca62fc461e22277b963bb28dbe4ed2f", + "spool_tail_len": 645, + "spool_tail_sha256": "4fcf4416fbf9a0ae882427db6ddc46f1b98275e8504aee4d13f7003fb880cdcb", + "why": "duplicate (miner_id, order_key, p2mr_program_hex) triples pin identity deduplication and identity_index reuse in the compact spool fragments, alongside mixed credit_policy values" + }, + "crossing-row-exact-fit": { + "advance_stats": [], + "canonical_bytes_len": 655, + "canonical_bytes_sha256": "0d423ee61de555cd1c231958351531f4902b29320817f010ea2ac5bd5b2baf05", + "canonical_digest": "0d423ee61de555cd1c231958351531f4902b29320817f010ea2ac5bd5b2baf05", + "input_sha256": "44b5c58495f43d8ad51331e162bd5b45f95ae2798dee5c4c359d9ebbb4e34e69", + "pinned_literals": { + "canonical_bytes": "[{\"accepted_at_ms\":9999,\"job_id\":\"job-2\",\"job_issued_at_ms\":9998,\"miner_id\":\"miner-2\",\"network_difficulty\":1000002,\"ntime\":1700000002,\"order_key\":\"02:miner-2\",\"p2mr_program_hex\":\"0202020202020202020202020202020202020202020202020202020202020202\",\"share_difficulty\":3,\"share_id\":\"share-2\",\"share_seq\":2,\"template_height\":800000},{\"accepted_at_ms\":10000,\"job_id\":\"job-3\",\"job_issued_at_ms\":10000,\"miner_id\":\"miner-3\",\"network_difficulty\":1000003,\"ntime\":1700000003,\"order_key\":\"03:miner-3\",\"p2mr_program_hex\":\"0303030303030303030303030303030303030303030303030303030303030303\",\"share_difficulty\":7,\"share_id\":\"share-3\",\"share_seq\":3,\"template_height\":800000}]", + "input": { + "advances": [], + "name": "crossing-row-exact-fit", + "page_size": 512, + "schema": "qbit-prism-window-pipeline-parity-input/v1", + "snapshot": { + "anchor_job_issued_at_ms": 10000, + "records": [ + { + "accepted_at_ms": 9997, + "credit_policy": null, + "job_id": "job-1", + "job_issued_at_ms": 9996, + "miner_id": "miner-1", + "network_difficulty": 1000001, + "ntime": 1700000001, + "order_key": "01:miner-1", + "p2mr_program_hex": "0101010101010101010101010101010101010101010101010101010101010101", + "share_difficulty": 2, + "share_id": "share-1", + "share_seq": 1, + "template_height": 800000 + }, + { + "accepted_at_ms": 9999, + "credit_policy": null, + "job_id": "job-2", + "job_issued_at_ms": 9998, + "miner_id": "miner-2", + "network_difficulty": 1000002, + "ntime": 1700000002, + "order_key": "02:miner-2", + "p2mr_program_hex": "0202020202020202020202020202020202020202020202020202020202020202", + "share_difficulty": 3, + "share_id": "share-2", + "share_seq": 2, + "template_height": 800000 + }, + { + "accepted_at_ms": 10000, + "credit_policy": null, + "job_id": "job-3", + "job_issued_at_ms": 10000, + "miner_id": "miner-3", + "network_difficulty": 1000003, + "ntime": 1700000003, + "order_key": "03:miner-3", + "p2mr_program_hex": "0303030303030303030303030303030303030303030303030303030303030303", + "share_difficulty": 7, + "share_id": "share-3", + "share_seq": 3, + "template_height": 800000 + } + ] + }, + "window_weight": 10 + }, + "record_jsons": [ + "{\"accepted_at_ms\":9999,\"job_id\":\"job-2\",\"job_issued_at_ms\":9998,\"miner_id\":\"miner-2\",\"network_difficulty\":1000002,\"ntime\":1700000002,\"order_key\":\"02:miner-2\",\"p2mr_program_hex\":\"0202020202020202020202020202020202020202020202020202020202020202\",\"share_difficulty\":3,\"share_id\":\"share-2\",\"share_seq\":2,\"template_height\":800000}", + "{\"accepted_at_ms\":10000,\"job_id\":\"job-3\",\"job_issued_at_ms\":10000,\"miner_id\":\"miner-3\",\"network_difficulty\":1000003,\"ntime\":1700000003,\"order_key\":\"03:miner-3\",\"p2mr_program_hex\":\"0303030303030303030303030303030303030303030303030303030303030303\",\"share_difficulty\":7,\"share_id\":\"share-3\",\"share_seq\":3,\"template_height\":800000}" + ], + "spool_tail": ",\"compact_share_identities\":[[\"miner-2\",\"02:miner-2\",\"0202020202020202020202020202020202020202020202020202020202020202\"],[\"miner-3\",\"03:miner-3\",\"0303030303030303030303030303030303030303030303030303030303030303\"]],\"compact_shares\":[[2,\"share-2\",0,3,9998,9999,null],[3,\"share-3\",1,7,10000,10000,null]]}" + }, + "record_count": 2, + "record_stream_len": 653, + "record_stream_sha256": "3fafb775c16d25984e47b0e560e4c7078f56efd1c56267789d46eef58477c0c1", + "spool_tail_len": 301, + "spool_tail_sha256": "c9b02e86e6d19e03a08394fbcff7232bf4d217b5771bd1eab933c87dd11c73d3", + "why": "difficulties (2,3,7) at weight 10: cumulative weight lands exactly on window_weight, so seq 1 is dropped and there is no crossing row; the pair with crossing-row-retained separates the >= cutoff from a wrong > cutoff" + }, + "crossing-row-retained": { + "advance_stats": [], + "canonical_bytes_len": 685, + "canonical_bytes_sha256": "17881e4d4e7775184971a94384e10dd071d409a5ba80f52dfdcfb6311e89fa4c", + "canonical_digest": "17881e4d4e7775184971a94384e10dd071d409a5ba80f52dfdcfb6311e89fa4c", + "input_sha256": "392a73c5aa97bf5c9442bbc7b49327c90fd4ec5cb8d92bc7839ac66d8eca0542", + "pinned_literals": { + "canonical_bytes": "[{\"accepted_at_ms\":9999,\"credit_policy\":\"stale-grace\",\"job_id\":\"job-2\",\"job_issued_at_ms\":9998,\"miner_id\":\"miner-2\",\"network_difficulty\":1000002,\"ntime\":1700000002,\"order_key\":\"02:miner-2\",\"p2mr_program_hex\":\"0202020202020202020202020202020202020202020202020202020202020202\",\"share_difficulty\":3,\"share_id\":\"share-2\",\"share_seq\":2,\"template_height\":800000},{\"accepted_at_ms\":10000,\"job_id\":\"job-3\",\"job_issued_at_ms\":10000,\"miner_id\":\"miner-3\",\"network_difficulty\":1000003,\"ntime\":1700000003,\"order_key\":\"03:miner-3\",\"p2mr_program_hex\":\"0303030303030303030303030303030303030303030303030303030303030303\",\"share_difficulty\":8,\"share_id\":\"share-3\",\"share_seq\":3,\"template_height\":800000}]", + "input": { + "advances": [], + "name": "crossing-row-retained", + "page_size": 512, + "schema": "qbit-prism-window-pipeline-parity-input/v1", + "snapshot": { + "anchor_job_issued_at_ms": 10000, + "records": [ + { + "accepted_at_ms": 9997, + "credit_policy": null, + "job_id": "job-1", + "job_issued_at_ms": 9996, + "miner_id": "miner-1", + "network_difficulty": 1000001, + "ntime": 1700000001, + "order_key": "01:miner-1", + "p2mr_program_hex": "0101010101010101010101010101010101010101010101010101010101010101", + "share_difficulty": 2, + "share_id": "share-1", + "share_seq": 1, + "template_height": 800000 + }, + { + "accepted_at_ms": 9999, + "credit_policy": "stale-grace", + "job_id": "job-2", + "job_issued_at_ms": 9998, + "miner_id": "miner-2", + "network_difficulty": 1000002, + "ntime": 1700000002, + "order_key": "02:miner-2", + "p2mr_program_hex": "0202020202020202020202020202020202020202020202020202020202020202", + "share_difficulty": 3, + "share_id": "share-2", + "share_seq": 2, + "template_height": 800000 + }, + { + "accepted_at_ms": 10000, + "credit_policy": null, + "job_id": "job-3", + "job_issued_at_ms": 10000, + "miner_id": "miner-3", + "network_difficulty": 1000003, + "ntime": 1700000003, + "order_key": "03:miner-3", + "p2mr_program_hex": "0303030303030303030303030303030303030303030303030303030303030303", + "share_difficulty": 8, + "share_id": "share-3", + "share_seq": 3, + "template_height": 800000 + } + ] + }, + "window_weight": 10 + }, + "record_jsons": [ + "{\"accepted_at_ms\":9999,\"credit_policy\":\"stale-grace\",\"job_id\":\"job-2\",\"job_issued_at_ms\":9998,\"miner_id\":\"miner-2\",\"network_difficulty\":1000002,\"ntime\":1700000002,\"order_key\":\"02:miner-2\",\"p2mr_program_hex\":\"0202020202020202020202020202020202020202020202020202020202020202\",\"share_difficulty\":3,\"share_id\":\"share-2\",\"share_seq\":2,\"template_height\":800000}", + "{\"accepted_at_ms\":10000,\"job_id\":\"job-3\",\"job_issued_at_ms\":10000,\"miner_id\":\"miner-3\",\"network_difficulty\":1000003,\"ntime\":1700000003,\"order_key\":\"03:miner-3\",\"p2mr_program_hex\":\"0303030303030303030303030303030303030303030303030303030303030303\",\"share_difficulty\":8,\"share_id\":\"share-3\",\"share_seq\":3,\"template_height\":800000}" + ], + "spool_tail": ",\"compact_share_identities\":[[\"miner-2\",\"02:miner-2\",\"0202020202020202020202020202020202020202020202020202020202020202\"],[\"miner-3\",\"03:miner-3\",\"0303030303030303030303030303030303030303030303030303030303030303\"]],\"compact_shares\":[[2,\"share-2\",0,3,9998,9999,\"stale-grace\"],[3,\"share-3\",1,8,10000,10000,null]]}" + }, + "record_count": 2, + "record_stream_len": 683, + "record_stream_sha256": "6552baf143848e555111c3c2292ca58f7440233d15266965b04578e098786d02", + "spool_tail_len": 310, + "spool_tail_sha256": "114620560516025d5d0b95a889fe5da45812e0d9fbe64a606e441e733d1c92ca", + "why": "difficulties (2,3,8) at weight 10: the final whole share crossing window_weight (seq 2) is deliberately retained, total 11 > 10; an implementation that drops it diverges here" + }, + "difficulty-beyond-u128": { + "advance_stats": [], + "canonical_bytes_len": 1243, + "canonical_bytes_sha256": "36bb2a87535eb0a5093fda2e617aeed9a44083fdc6f62ad494f013247a74b9fb", + "canonical_digest": "36bb2a87535eb0a5093fda2e617aeed9a44083fdc6f62ad494f013247a74b9fb", + "input_sha256": "62e0de88259429293f6a7d9906ef244d86f4db04f4273b350efb35707c92f6e8", + "pinned_literals": { + "canonical_bytes": "[{\"accepted_at_ms\":1754999999998,\"job_id\":\"job-2\",\"job_issued_at_ms\":1754999999998,\"miner_id\":\"miner-2\",\"network_difficulty\":999999999999999999999999999999999999999999999999999999999999999999999999999999,\"ntime\":1700000002,\"order_key\":\"02:miner-2\",\"p2mr_program_hex\":\"0202020202020202020202020202020202020202020202020202020202020202\",\"share_difficulty\":340282366920938463463374607431768211457,\"share_id\":\"share-2\",\"share_seq\":2,\"template_height\":800000},{\"accepted_at_ms\":1754999999999,\"job_id\":\"job-3\",\"job_issued_at_ms\":1754999999999,\"miner_id\":\"miner-3\",\"network_difficulty\":340282366920938463463374607431768211456,\"ntime\":1700000003,\"order_key\":\"03:miner-3\",\"p2mr_program_hex\":\"0303030303030303030303030303030303030303030303030303030303030303\",\"share_difficulty\":18446744073709551615,\"share_id\":\"share-3\",\"share_seq\":3,\"template_height\":800000},{\"accepted_at_ms\":1755000000000,\"job_id\":\"job-4\",\"job_issued_at_ms\":1755000000000,\"miner_id\":\"miner-4\",\"network_difficulty\":340282366920938463463374607431768211455,\"ntime\":1700000004,\"order_key\":\"04:miner-4\",\"p2mr_program_hex\":\"0404040404040404040404040404040404040404040404040404040404040404\",\"share_difficulty\":9223372036854775808,\"share_id\":\"share-4\",\"share_seq\":4,\"template_height\":800000}]", + "input": { + "advances": [], + "name": "difficulty-beyond-u128", + "page_size": 512, + "schema": "qbit-prism-window-pipeline-parity-input/v1", + "snapshot": { + "anchor_job_issued_at_ms": 1755000000000, + "records": [ + { + "accepted_at_ms": 1754999999997, + "credit_policy": null, + "job_id": "job-1", + "job_issued_at_ms": 1754999999997, + "miner_id": "miner-1", + "network_difficulty": 340282366920938463463374607431768211455, + "ntime": 1700000001, + "order_key": "01:miner-1", + "p2mr_program_hex": "0101010101010101010101010101010101010101010101010101010101010101", + "share_difficulty": 170141183460469231731687303715884105728, + "share_id": "share-1", + "share_seq": 1, + "template_height": 800000 + }, + { + "accepted_at_ms": 1754999999998, + "credit_policy": null, + "job_id": "job-2", + "job_issued_at_ms": 1754999999998, + "miner_id": "miner-2", + "network_difficulty": 999999999999999999999999999999999999999999999999999999999999999999999999999999, + "ntime": 1700000002, + "order_key": "02:miner-2", + "p2mr_program_hex": "0202020202020202020202020202020202020202020202020202020202020202", + "share_difficulty": 340282366920938463463374607431768211457, + "share_id": "share-2", + "share_seq": 2, + "template_height": 800000 + }, + { + "accepted_at_ms": 1754999999999, + "credit_policy": null, + "job_id": "job-3", + "job_issued_at_ms": 1754999999999, + "miner_id": "miner-3", + "network_difficulty": 340282366920938463463374607431768211456, + "ntime": 1700000003, + "order_key": "03:miner-3", + "p2mr_program_hex": "0303030303030303030303030303030303030303030303030303030303030303", + "share_difficulty": 18446744073709551615, + "share_id": "share-3", + "share_seq": 3, + "template_height": 800000 + }, + { + "accepted_at_ms": 1755000000000, + "credit_policy": null, + "job_id": "job-4", + "job_issued_at_ms": 1755000000000, + "miner_id": "miner-4", + "network_difficulty": 340282366920938463463374607431768211455, + "ntime": 1700000004, + "order_key": "04:miner-4", + "p2mr_program_hex": "0404040404040404040404040404040404040404040404040404040404040404", + "share_difficulty": 9223372036854775808, + "share_id": "share-4", + "share_seq": 4, + "template_height": 800000 + } + ] + }, + "window_weight": 340282366920938463481821351505477763072 + }, + "record_jsons": [ + "{\"accepted_at_ms\":1754999999998,\"job_id\":\"job-2\",\"job_issued_at_ms\":1754999999998,\"miner_id\":\"miner-2\",\"network_difficulty\":999999999999999999999999999999999999999999999999999999999999999999999999999999,\"ntime\":1700000002,\"order_key\":\"02:miner-2\",\"p2mr_program_hex\":\"0202020202020202020202020202020202020202020202020202020202020202\",\"share_difficulty\":340282366920938463463374607431768211457,\"share_id\":\"share-2\",\"share_seq\":2,\"template_height\":800000}", + "{\"accepted_at_ms\":1754999999999,\"job_id\":\"job-3\",\"job_issued_at_ms\":1754999999999,\"miner_id\":\"miner-3\",\"network_difficulty\":340282366920938463463374607431768211456,\"ntime\":1700000003,\"order_key\":\"03:miner-3\",\"p2mr_program_hex\":\"0303030303030303030303030303030303030303030303030303030303030303\",\"share_difficulty\":18446744073709551615,\"share_id\":\"share-3\",\"share_seq\":3,\"template_height\":800000}", + "{\"accepted_at_ms\":1755000000000,\"job_id\":\"job-4\",\"job_issued_at_ms\":1755000000000,\"miner_id\":\"miner-4\",\"network_difficulty\":340282366920938463463374607431768211455,\"ntime\":1700000004,\"order_key\":\"04:miner-4\",\"p2mr_program_hex\":\"0404040404040404040404040404040404040404040404040404040404040404\",\"share_difficulty\":9223372036854775808,\"share_id\":\"share-4\",\"share_seq\":4,\"template_height\":800000}" + ], + "spool_tail": ",\"compact_share_identities\":[[\"miner-2\",\"02:miner-2\",\"0202020202020202020202020202020202020202020202020202020202020202\"],[\"miner-3\",\"03:miner-3\",\"0303030303030303030303030303030303030303030303030303030303030303\"],[\"miner-4\",\"04:miner-4\",\"0404040404040404040404040404040404040404040404040404040404040404\"]],\"compact_shares\":[[2,\"share-2\",0,340282366920938463463374607431768211457,1754999999998,1754999999998,null],[3,\"share-3\",1,18446744073709551615,1754999999999,1754999999999,null],[4,\"share-4\",2,9223372036854775808,1755000000000,1755000000000,null]]}" + }, + "record_count": 3, + "record_stream_len": 1241, + "record_stream_sha256": "cda1575a1a6915da092fda79a5bef3389926a3c593331766fb38490fdf7c1f8c", + "spool_tail_len": 553, + "spool_tail_sha256": "ee82fbf5450d73af30b9bbb98da5d25a99ff9e7a24327189a9d683c8453404f7", + "why": "numeric(78,0) width taken literally: share_difficulty 2^128+1 as the retained crossing row, network_difficulty at 2^128-1, 2^128 and 10^78-1, window_weight and the retained total above 2^128 -- outside the declared u128 widths, so a backend declaring them answers Unsupported here while the frozen bytes stay Python's; one that clamps at u128::MAX, saturating or not, renders different digits and fails -- see the module docstring" + }, + "eligibility-filtering": { + "advance_stats": [], + "canonical_bytes_len": 657, + "canonical_bytes_sha256": "40fc09d08bc2a883bff421b8b81694558cdc95d2814b03dab42f85d032391cf6", + "canonical_digest": "40fc09d08bc2a883bff421b8b81694558cdc95d2814b03dab42f85d032391cf6", + "input_sha256": "232dd3419eb6543d104a7e2a7752ee4dd854545ebc11abe2346d767c41b6276c", + "pinned_literals": { + "canonical_bytes": "[{\"accepted_at_ms\":29999,\"job_id\":\"job-1\",\"job_issued_at_ms\":29998,\"miner_id\":\"miner-1\",\"network_difficulty\":1000001,\"ntime\":1700000001,\"order_key\":\"01:miner-1\",\"p2mr_program_hex\":\"0101010101010101010101010101010101010101010101010101010101010101\",\"share_difficulty\":4,\"share_id\":\"share-1\",\"share_seq\":1,\"template_height\":800000},{\"accepted_at_ms\":30000,\"job_id\":\"job-5\",\"job_issued_at_ms\":30000,\"miner_id\":\"miner-5\",\"network_difficulty\":1000005,\"ntime\":1700000005,\"order_key\":\"05:miner-5\",\"p2mr_program_hex\":\"0505050505050505050505050505050505050505050505050505050505050505\",\"share_difficulty\":8,\"share_id\":\"share-5\",\"share_seq\":5,\"template_height\":800000}]", + "input": { + "advances": [], + "name": "eligibility-filtering", + "page_size": 512, + "schema": "qbit-prism-window-pipeline-parity-input/v1", + "snapshot": { + "anchor_job_issued_at_ms": 30000, + "records": [ + { + "accepted_at_ms": 29999, + "credit_policy": null, + "job_id": "job-1", + "job_issued_at_ms": 29998, + "miner_id": "miner-1", + "network_difficulty": 1000001, + "ntime": 1700000001, + "order_key": "01:miner-1", + "p2mr_program_hex": "0101010101010101010101010101010101010101010101010101010101010101", + "share_difficulty": 4, + "share_id": "share-1", + "share_seq": 1, + "template_height": 800000 + }, + { + "accepted_at_ms": 29999, + "credit_policy": null, + "job_id": "job-2", + "job_issued_at_ms": 30001, + "miner_id": "miner-2", + "network_difficulty": 1000002, + "ntime": 1700000002, + "order_key": "02:miner-2", + "p2mr_program_hex": "0202020202020202020202020202020202020202020202020202020202020202", + "share_difficulty": 5, + "share_id": "share-2", + "share_seq": 2, + "template_height": 800000 + }, + { + "accepted_at_ms": 30001, + "credit_policy": null, + "job_id": "job-3", + "job_issued_at_ms": 29999, + "miner_id": "miner-3", + "network_difficulty": 1000003, + "ntime": 1700000003, + "order_key": "03:miner-3", + "p2mr_program_hex": "0303030303030303030303030303030303030303030303030303030303030303", + "share_difficulty": 6, + "share_id": "share-3", + "share_seq": 3, + "template_height": 800000 + }, + { + "accepted_at_ms": 30003, + "credit_policy": null, + "job_id": "job-4", + "job_issued_at_ms": 30002, + "miner_id": "miner-4", + "network_difficulty": 1000004, + "ntime": 1700000004, + "order_key": "04:miner-4", + "p2mr_program_hex": "0404040404040404040404040404040404040404040404040404040404040404", + "share_difficulty": 7, + "share_id": "share-4", + "share_seq": 4, + "template_height": 800000 + }, + { + "accepted_at_ms": 30000, + "credit_policy": null, + "job_id": "job-5", + "job_issued_at_ms": 30000, + "miner_id": "miner-5", + "network_difficulty": 1000005, + "ntime": 1700000005, + "order_key": "05:miner-5", + "p2mr_program_hex": "0505050505050505050505050505050505050505050505050505050505050505", + "share_difficulty": 8, + "share_id": "share-5", + "share_seq": 5, + "template_height": 800000 + } + ] + }, + "window_weight": 100 + }, + "record_jsons": [ + "{\"accepted_at_ms\":29999,\"job_id\":\"job-1\",\"job_issued_at_ms\":29998,\"miner_id\":\"miner-1\",\"network_difficulty\":1000001,\"ntime\":1700000001,\"order_key\":\"01:miner-1\",\"p2mr_program_hex\":\"0101010101010101010101010101010101010101010101010101010101010101\",\"share_difficulty\":4,\"share_id\":\"share-1\",\"share_seq\":1,\"template_height\":800000}", + "{\"accepted_at_ms\":30000,\"job_id\":\"job-5\",\"job_issued_at_ms\":30000,\"miner_id\":\"miner-5\",\"network_difficulty\":1000005,\"ntime\":1700000005,\"order_key\":\"05:miner-5\",\"p2mr_program_hex\":\"0505050505050505050505050505050505050505050505050505050505050505\",\"share_difficulty\":8,\"share_id\":\"share-5\",\"share_seq\":5,\"template_height\":800000}" + ], + "spool_tail": ",\"compact_share_identities\":[[\"miner-1\",\"01:miner-1\",\"0101010101010101010101010101010101010101010101010101010101010101\"],[\"miner-5\",\"05:miner-5\",\"0505050505050505050505050505050505050505050505050505050505050505\"]],\"compact_shares\":[[1,\"share-1\",0,4,29998,29999,null],[5,\"share-5\",1,8,30000,30000,null]]}" + }, + "record_count": 2, + "record_stream_len": 655, + "record_stream_sha256": "5b48e3140c8cb56e843859395108e1af18deadf4c2b337ff046f1250b8e1b404", + "spool_tail_len": 303, + "spool_tail_sha256": "6dba2c368162d4562aed7e66ef0e0e23cedb3134b8d77c36ca7243190baea561", + "why": "one record excluded by job_issued_at_ms alone, one by accepted_at_ms alone, one by both; seq 5 sits exactly on the anchor, pinning the inclusive <= comparisons" + }, + "empty-window": { + "advance_stats": [], + "canonical_bytes_len": 2, + "canonical_bytes_sha256": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945", + "canonical_digest": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945", + "input_sha256": "3783bf5efe7c0a3cde10ef43cbc331d1263f807e073d19c08752c4607ef59580", + "pinned_literals": { + "canonical_bytes": "[]", + "input": { + "advances": [], + "name": "empty-window", + "page_size": 512, + "schema": "qbit-prism-window-pipeline-parity-input/v1", + "snapshot": { + "anchor_job_issued_at_ms": 1000, + "records": [] + }, + "window_weight": 10 + }, + "record_jsons": [], + "spool_tail": ",\"compact_share_identities\":[],\"compact_shares\":[]}" + }, + "record_count": 0, + "record_stream_len": 0, + "record_stream_sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", + "spool_tail_len": 51, + "spool_tail_sha256": "0a724963f1ff7d7033eefd78473cb2132190352e0443e14db01ff3d89ae01bcf", + "why": "zero eligible records: [] framing, digest of [], empty compact arrays" + }, + "incremental-two-advances": { + "advance_stats": [ + { + "added_rows": 2, + "expired_rows": 4, + "touched_pages": 1 + }, + { + "added_rows": 4, + "expired_rows": 2, + "touched_pages": 1 + } + ], + "canonical_bytes_len": 2046, + "canonical_bytes_sha256": "2e05cae0e6203ecda5384cbb91768cd0812f0cd79e812ccecfe77775442dc151", + "canonical_digest": "2e05cae0e6203ecda5384cbb91768cd0812f0cd79e812ccecfe77775442dc151", + "input_sha256": "d7f951bbdc28813c5c5eacfe8aa2669db04ab173183a6f655a51f47ce42d145f", + "pinned_literals": { + "canonical_bytes": "[{\"accepted_at_ms\":79999,\"job_id\":\"job-9\",\"job_issued_at_ms\":80001,\"miner_id\":\"miner-9\",\"network_difficulty\":1000009,\"ntime\":1700000009,\"order_key\":\"09:miner-9\",\"p2mr_program_hex\":\"0909090909090909090909090909090909090909090909090909090909090909\",\"share_difficulty\":2,\"share_id\":\"share-9\",\"share_seq\":9,\"template_height\":800000},{\"accepted_at_ms\":80001,\"job_id\":\"job-10\",\"job_issued_at_ms\":79990,\"miner_id\":\"miner-10\",\"network_difficulty\":1000010,\"ntime\":1700000010,\"order_key\":\"10:miner-10\",\"p2mr_program_hex\":\"0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a\",\"share_difficulty\":4,\"share_id\":\"share-10\",\"share_seq\":10,\"template_height\":800000},{\"accepted_at_ms\":80002,\"job_id\":\"job-11\",\"job_issued_at_ms\":80005,\"miner_id\":\"miner-0\",\"network_difficulty\":1000011,\"ntime\":1700000011,\"order_key\":\"00:miner-0\",\"p2mr_program_hex\":\"0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b\",\"share_difficulty\":1,\"share_id\":\"share-11\",\"share_seq\":11,\"template_height\":800000},{\"accepted_at_ms\":79995,\"credit_policy\":\"stale-grace\",\"job_id\":\"job-12\",\"job_issued_at_ms\":80010,\"miner_id\":\"miner-1\",\"network_difficulty\":1000012,\"ntime\":1700000012,\"order_key\":\"01:miner-1\",\"p2mr_program_hex\":\"0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c\",\"share_difficulty\":1,\"share_id\":\"share-12\",\"share_seq\":12,\"template_height\":800000},{\"accepted_at_ms\":80010,\"job_id\":\"job-13\",\"job_issued_at_ms\":80000,\"miner_id\":\"miner-2\",\"network_difficulty\":1000013,\"ntime\":1700000013,\"order_key\":\"02:miner-2\",\"p2mr_program_hex\":\"0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d\",\"share_difficulty\":1,\"share_id\":\"share-13\",\"share_seq\":13,\"template_height\":800000},{\"accepted_at_ms\":80009,\"credit_policy\":\"stale-grace\",\"job_id\":\"job-14\",\"job_issued_at_ms\":80009,\"miner_id\":\"miner-3\",\"network_difficulty\":1000014,\"ntime\":1700000014,\"order_key\":\"03:miner-3\",\"p2mr_program_hex\":\"0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e\",\"share_difficulty\":1,\"share_id\":\"share-14\",\"share_seq\":14,\"template_height\":800000}]", + "input": { + "advances": [ + { + "anchor_job_issued_at_ms": 80001, + "records": [ + { + "accepted_at_ms": 79999, + "credit_policy": null, + "job_id": "job-9", + "job_issued_at_ms": 80001, + "miner_id": "miner-9", + "network_difficulty": 1000009, + "ntime": 1700000009, + "order_key": "09:miner-9", + "p2mr_program_hex": "0909090909090909090909090909090909090909090909090909090909090909", + "share_difficulty": 2, + "share_id": "share-9", + "share_seq": 9, + "template_height": 800000 + }, + { + "accepted_at_ms": 80001, + "credit_policy": null, + "job_id": "job-10", + "job_issued_at_ms": 79990, + "miner_id": "miner-10", + "network_difficulty": 1000010, + "ntime": 1700000010, + "order_key": "10:miner-10", + "p2mr_program_hex": "0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a", + "share_difficulty": 4, + "share_id": "share-10", + "share_seq": 10, + "template_height": 800000 + } + ] + }, + { + "anchor_job_issued_at_ms": 80010, + "records": [ + { + "accepted_at_ms": 80002, + "credit_policy": null, + "job_id": "job-11", + "job_issued_at_ms": 80005, + "miner_id": "miner-0", + "network_difficulty": 1000011, + "ntime": 1700000011, + "order_key": "00:miner-0", + "p2mr_program_hex": "0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b", + "share_difficulty": 1, + "share_id": "share-11", + "share_seq": 11, + "template_height": 800000 + }, + { + "accepted_at_ms": 79995, + "credit_policy": "stale-grace", + "job_id": "job-12", + "job_issued_at_ms": 80010, + "miner_id": "miner-1", + "network_difficulty": 1000012, + "ntime": 1700000012, + "order_key": "01:miner-1", + "p2mr_program_hex": "0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c", + "share_difficulty": 1, + "share_id": "share-12", + "share_seq": 12, + "template_height": 800000 + }, + { + "accepted_at_ms": 80010, + "credit_policy": null, + "job_id": "job-13", + "job_issued_at_ms": 80000, + "miner_id": "miner-2", + "network_difficulty": 1000013, + "ntime": 1700000013, + "order_key": "02:miner-2", + "p2mr_program_hex": "0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d", + "share_difficulty": 1, + "share_id": "share-13", + "share_seq": 13, + "template_height": 800000 + }, + { + "accepted_at_ms": 80009, + "credit_policy": "stale-grace", + "job_id": "job-14", + "job_issued_at_ms": 80009, + "miner_id": "miner-3", + "network_difficulty": 1000014, + "ntime": 1700000014, + "order_key": "03:miner-3", + "p2mr_program_hex": "0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e", + "share_difficulty": 1, + "share_id": "share-14", + "share_seq": 14, + "template_height": 800000 + } + ] + } + ], + "name": "incremental-two-advances", + "page_size": 3, + "schema": "qbit-prism-window-pipeline-parity-input/v1", + "snapshot": { + "anchor_job_issued_at_ms": 80000, + "records": [ + { + "accepted_at_ms": 79999, + "credit_policy": null, + "job_id": "job-1", + "job_issued_at_ms": 79999, + "miner_id": "miner-1", + "network_difficulty": 1000001, + "ntime": 1700000001, + "order_key": "01:miner-1", + "p2mr_program_hex": "0101010101010101010101010101010101010101010101010101010101010101", + "share_difficulty": 5, + "share_id": "share-1", + "share_seq": 1, + "template_height": 800000 + }, + { + "accepted_at_ms": 79998, + "credit_policy": null, + "job_id": "job-2", + "job_issued_at_ms": 79998, + "miner_id": "miner-2", + "network_difficulty": 1000002, + "ntime": 1700000002, + "order_key": "02:miner-2", + "p2mr_program_hex": "0202020202020202020202020202020202020202020202020202020202020202", + "share_difficulty": 1, + "share_id": "share-2", + "share_seq": 2, + "template_height": 800000 + }, + { + "accepted_at_ms": 79997, + "credit_policy": null, + "job_id": "job-3", + "job_issued_at_ms": 79997, + "miner_id": "miner-3", + "network_difficulty": 1000003, + "ntime": 1700000003, + "order_key": "03:miner-3", + "p2mr_program_hex": "0303030303030303030303030303030303030303030303030303030303030303", + "share_difficulty": 1, + "share_id": "share-3", + "share_seq": 3, + "template_height": 800000 + }, + { + "accepted_at_ms": 79996, + "credit_policy": null, + "job_id": "job-4", + "job_issued_at_ms": 79996, + "miner_id": "miner-4", + "network_difficulty": 1000004, + "ntime": 1700000004, + "order_key": "04:miner-4", + "p2mr_program_hex": "0404040404040404040404040404040404040404040404040404040404040404", + "share_difficulty": 2, + "share_id": "share-4", + "share_seq": 4, + "template_height": 800000 + }, + { + "accepted_at_ms": 79995, + "credit_policy": null, + "job_id": "job-5", + "job_issued_at_ms": 79995, + "miner_id": "miner-5", + "network_difficulty": 1000005, + "ntime": 1700000005, + "order_key": "05:miner-5", + "p2mr_program_hex": "0505050505050505050505050505050505050505050505050505050505050505", + "share_difficulty": 1, + "share_id": "share-5", + "share_seq": 5, + "template_height": 800000 + }, + { + "accepted_at_ms": 79994, + "credit_policy": "stale-grace", + "job_id": "job-6", + "job_issued_at_ms": 79994, + "miner_id": "miner-6", + "network_difficulty": 1000006, + "ntime": 1700000006, + "order_key": "06:miner-6", + "p2mr_program_hex": "0606060606060606060606060606060606060606060606060606060606060606", + "share_difficulty": 3, + "share_id": "share-6", + "share_seq": 6, + "template_height": 800000 + }, + { + "accepted_at_ms": 79993, + "credit_policy": null, + "job_id": "job-7", + "job_issued_at_ms": 79993, + "miner_id": "miner-7", + "network_difficulty": 1000007, + "ntime": 1700000007, + "order_key": "07:miner-7", + "p2mr_program_hex": "0707070707070707070707070707070707070707070707070707070707070707", + "share_difficulty": 1, + "share_id": "share-7", + "share_seq": 7, + "template_height": 800000 + }, + { + "accepted_at_ms": 1000000000, + "credit_policy": null, + "job_id": "job-8", + "job_issued_at_ms": 1000000000, + "miner_id": "miner-8", + "network_difficulty": 1000008, + "ntime": 1700000008, + "order_key": "08:miner-8", + "p2mr_program_hex": "0808080808080808080808080808080808080808080808080808080808080808", + "share_difficulty": 100, + "share_id": "share-8", + "share_seq": 8, + "template_height": 800000 + } + ] + }, + "window_weight": 9 + }, + "record_jsons": [ + "{\"accepted_at_ms\":79999,\"job_id\":\"job-9\",\"job_issued_at_ms\":80001,\"miner_id\":\"miner-9\",\"network_difficulty\":1000009,\"ntime\":1700000009,\"order_key\":\"09:miner-9\",\"p2mr_program_hex\":\"0909090909090909090909090909090909090909090909090909090909090909\",\"share_difficulty\":2,\"share_id\":\"share-9\",\"share_seq\":9,\"template_height\":800000}", + "{\"accepted_at_ms\":80001,\"job_id\":\"job-10\",\"job_issued_at_ms\":79990,\"miner_id\":\"miner-10\",\"network_difficulty\":1000010,\"ntime\":1700000010,\"order_key\":\"10:miner-10\",\"p2mr_program_hex\":\"0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a\",\"share_difficulty\":4,\"share_id\":\"share-10\",\"share_seq\":10,\"template_height\":800000}", + "{\"accepted_at_ms\":80002,\"job_id\":\"job-11\",\"job_issued_at_ms\":80005,\"miner_id\":\"miner-0\",\"network_difficulty\":1000011,\"ntime\":1700000011,\"order_key\":\"00:miner-0\",\"p2mr_program_hex\":\"0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b\",\"share_difficulty\":1,\"share_id\":\"share-11\",\"share_seq\":11,\"template_height\":800000}", + "{\"accepted_at_ms\":79995,\"credit_policy\":\"stale-grace\",\"job_id\":\"job-12\",\"job_issued_at_ms\":80010,\"miner_id\":\"miner-1\",\"network_difficulty\":1000012,\"ntime\":1700000012,\"order_key\":\"01:miner-1\",\"p2mr_program_hex\":\"0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c\",\"share_difficulty\":1,\"share_id\":\"share-12\",\"share_seq\":12,\"template_height\":800000}", + "{\"accepted_at_ms\":80010,\"job_id\":\"job-13\",\"job_issued_at_ms\":80000,\"miner_id\":\"miner-2\",\"network_difficulty\":1000013,\"ntime\":1700000013,\"order_key\":\"02:miner-2\",\"p2mr_program_hex\":\"0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d\",\"share_difficulty\":1,\"share_id\":\"share-13\",\"share_seq\":13,\"template_height\":800000}", + "{\"accepted_at_ms\":80009,\"credit_policy\":\"stale-grace\",\"job_id\":\"job-14\",\"job_issued_at_ms\":80009,\"miner_id\":\"miner-3\",\"network_difficulty\":1000014,\"ntime\":1700000014,\"order_key\":\"03:miner-3\",\"p2mr_program_hex\":\"0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e\",\"share_difficulty\":1,\"share_id\":\"share-14\",\"share_seq\":14,\"template_height\":800000}" + ], + "spool_tail": ",\"compact_share_identities\":[[\"miner-9\",\"09:miner-9\",\"0909090909090909090909090909090909090909090909090909090909090909\"],[\"miner-10\",\"10:miner-10\",\"0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a\"],[\"miner-0\",\"00:miner-0\",\"0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b\"],[\"miner-1\",\"01:miner-1\",\"0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c\"],[\"miner-2\",\"02:miner-2\",\"0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d\"],[\"miner-3\",\"03:miner-3\",\"0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e\"]],\"compact_shares\":[[9,\"share-9\",0,2,80001,79999,null],[10,\"share-10\",1,4,79990,80001,null],[11,\"share-11\",2,1,80005,80002,null],[12,\"share-12\",3,1,80010,79995,\"stale-grace\"],[13,\"share-13\",4,1,80000,80010,null],[14,\"share-14\",5,1,80009,80009,\"stale-grace\"]]}" + }, + "record_count": 6, + "record_stream_len": 2044, + "record_stream_sha256": "1aa361b6c09e4dbfcd2c766feef74fd601eec7b1cd6704d92f6412314cbf931f", + "spool_tail_len": 841, + "spool_tail_sha256": "fa839327371ffe159490e7e0792afde689722061068c67a6c76efcdf729d5a4f", + "why": "two advance() steps over a page_size-3 window: append into a partially filled page, spill into new pages, expire a whole head page and partially expire the next; the folded bytes must equal a full rebuild over the union" + }, + "multi-page-interior-cutoff": { + "advance_stats": [], + "canonical_bytes_len": 584180, + "canonical_bytes_sha256": "60792c716201a7035ed376c7742ee51349228bb4bac3d048188183c05aa94703", + "canonical_digest": "60792c716201a7035ed376c7742ee51349228bb4bac3d048188183c05aa94703", + "input_sha256": "7c5587ca344cabfa4898152e10878aa0888e1b1a690d5bd9335a06fc9cba46fa", + "record_count": 1700, + "record_stream_len": 584178, + "record_stream_sha256": "7d7c4eddd8245ce6620c8d101cc46878eb31bd9628080a39085a9dfebd839a47", + "spool_tail_len": 232993, + "spool_tail_sha256": "b262f2eb2f5444fe1bfa548555954b858b1c6dff9228aa13a00cc0cfb73aefa7", + "why": "1800 unsorted input records with the retention cutoff landing inside a page interior: pins multi-page fragment concatenation and a crossing row far from page edges" + }, + "non-ascii-strings": { + "advance_stats": [], + "canonical_bytes_len": 1504, + "canonical_bytes_sha256": "39d380d908dc40a48e25574d70fe37fa49964b6cbb961ec7e488169a76f12e00", + "canonical_digest": "39d380d908dc40a48e25574d70fe37fa49964b6cbb961ec7e488169a76f12e00", + "input_sha256": "071a0cb4c4c2d288f2880b145214274f0318f46950307b823a054d593fea1a58", + "pinned_literals": { + "canonical_bytes": "[{\"accepted_at_ms\":59998,\"job_id\":\"job-1\",\"job_issued_at_ms\":59997,\"miner_id\":\"min\\u00e9r-\\u00e9\",\"network_difficulty\":1000001,\"ntime\":1700000001,\"order_key\":\"00:min\\u00e9r-\\u00e9\",\"p2mr_program_hex\":\"0101010101010101010101010101010101010101010101010101010101010101\",\"share_difficulty\":3,\"share_id\":\"min\\u00e9r-\\u00e9.work\\u00e9r:0a1b\",\"share_seq\":1,\"template_height\":800000},{\"accepted_at_ms\":59999,\"credit_policy\":\"stale-grace\",\"job_id\":\"job-\\t\\n\\u0001\",\"job_issued_at_ms\":59998,\"miner_id\":\"miner-2\",\"network_difficulty\":1000002,\"ntime\":1700000002,\"order_key\":\"02:miner-2\",\"p2mr_program_hex\":\"0202020202020202020202020202020202020202020202020202020202020202\",\"share_difficulty\":4,\"share_id\":\"\\ud83d\\ude80-share-\\\"quoted\\\"\\\\back\\\\slash\",\"share_seq\":2,\"template_height\":800000},{\"accepted_at_ms\":60000,\"job_id\":\"job-3\",\"job_issued_at_ms\":59999,\"miner_id\":\"miner-3\",\"network_difficulty\":1000003,\"ntime\":1700000003,\"order_key\":\"03:miner-3\",\"p2mr_program_hex\":\"0303030303030303030303030303030303030303030303030303030303030303\",\"share_difficulty\":5,\"share_id\":\"share-3\",\"share_seq\":3,\"template_height\":800000},{\"accepted_at_ms\":60000,\"job_id\":\"job-4\",\"job_issued_at_ms\":59999,\"miner_id\":\"MINER-A/worker 1\\u007f\",\"network_difficulty\":1000004,\"ntime\":1700000004,\"order_key\":\"04:MINER-A/worker 1\\u007f\",\"p2mr_program_hex\":\"0404040404040404040404040404040404040404040404040404040404040404\",\"share_difficulty\":6,\"share_id\":\"MINER-A/worker 1:\\u007f\\r\\b\\f\\u001f\\u2028\\u2029\",\"share_seq\":4,\"template_height\":800000}]", + "input": { + "advances": [], + "name": "non-ascii-strings", + "page_size": 512, + "schema": "qbit-prism-window-pipeline-parity-input/v1", + "snapshot": { + "anchor_job_issued_at_ms": 60000, + "records": [ + { + "accepted_at_ms": 59998, + "credit_policy": null, + "job_id": "job-1", + "job_issued_at_ms": 59997, + "miner_id": "min\u00e9r-\u00e9", + "network_difficulty": 1000001, + "ntime": 1700000001, + "order_key": "00:min\u00e9r-\u00e9", + "p2mr_program_hex": "0101010101010101010101010101010101010101010101010101010101010101", + "share_difficulty": 3, + "share_id": "min\u00e9r-\u00e9.work\u00e9r:0a1b", + "share_seq": 1, + "template_height": 800000 + }, + { + "accepted_at_ms": 59999, + "credit_policy": "stale-grace", + "job_id": "job-\t\n\u0001", + "job_issued_at_ms": 59998, + "miner_id": "miner-2", + "network_difficulty": 1000002, + "ntime": 1700000002, + "order_key": "02:miner-2", + "p2mr_program_hex": "0202020202020202020202020202020202020202020202020202020202020202", + "share_difficulty": 4, + "share_id": "\ud83d\ude80-share-\"quoted\"\\back\\slash", + "share_seq": 2, + "template_height": 800000 + }, + { + "accepted_at_ms": 60000, + "credit_policy": null, + "job_id": "job-3", + "job_issued_at_ms": 59999, + "miner_id": "miner-3", + "network_difficulty": 1000003, + "ntime": 1700000003, + "order_key": "03:miner-3", + "p2mr_program_hex": "0303030303030303030303030303030303030303030303030303030303030303", + "share_difficulty": 5, + "share_id": "share-3", + "share_seq": 3, + "template_height": 800000 + }, + { + "accepted_at_ms": 60000, + "credit_policy": null, + "job_id": "job-4", + "job_issued_at_ms": 59999, + "miner_id": "MINER-A/worker 1\u007f", + "network_difficulty": 1000004, + "ntime": 1700000004, + "order_key": "04:MINER-A/worker 1\u007f", + "p2mr_program_hex": "0404040404040404040404040404040404040404040404040404040404040404", + "share_difficulty": 6, + "share_id": "MINER-A/worker 1:\u007f\r\b\f\u001f\u2028\u2029", + "share_seq": 4, + "template_height": 800000 + } + ] + }, + "window_weight": 1000000 + }, + "record_jsons": [ + "{\"accepted_at_ms\":59998,\"job_id\":\"job-1\",\"job_issued_at_ms\":59997,\"miner_id\":\"min\\u00e9r-\\u00e9\",\"network_difficulty\":1000001,\"ntime\":1700000001,\"order_key\":\"00:min\\u00e9r-\\u00e9\",\"p2mr_program_hex\":\"0101010101010101010101010101010101010101010101010101010101010101\",\"share_difficulty\":3,\"share_id\":\"min\\u00e9r-\\u00e9.work\\u00e9r:0a1b\",\"share_seq\":1,\"template_height\":800000}", + "{\"accepted_at_ms\":59999,\"credit_policy\":\"stale-grace\",\"job_id\":\"job-\\t\\n\\u0001\",\"job_issued_at_ms\":59998,\"miner_id\":\"miner-2\",\"network_difficulty\":1000002,\"ntime\":1700000002,\"order_key\":\"02:miner-2\",\"p2mr_program_hex\":\"0202020202020202020202020202020202020202020202020202020202020202\",\"share_difficulty\":4,\"share_id\":\"\\ud83d\\ude80-share-\\\"quoted\\\"\\\\back\\\\slash\",\"share_seq\":2,\"template_height\":800000}", + "{\"accepted_at_ms\":60000,\"job_id\":\"job-3\",\"job_issued_at_ms\":59999,\"miner_id\":\"miner-3\",\"network_difficulty\":1000003,\"ntime\":1700000003,\"order_key\":\"03:miner-3\",\"p2mr_program_hex\":\"0303030303030303030303030303030303030303030303030303030303030303\",\"share_difficulty\":5,\"share_id\":\"share-3\",\"share_seq\":3,\"template_height\":800000}", + "{\"accepted_at_ms\":60000,\"job_id\":\"job-4\",\"job_issued_at_ms\":59999,\"miner_id\":\"MINER-A/worker 1\\u007f\",\"network_difficulty\":1000004,\"ntime\":1700000004,\"order_key\":\"04:MINER-A/worker 1\\u007f\",\"p2mr_program_hex\":\"0404040404040404040404040404040404040404040404040404040404040404\",\"share_difficulty\":6,\"share_id\":\"MINER-A/worker 1:\\u007f\\r\\b\\f\\u001f\\u2028\\u2029\",\"share_seq\":4,\"template_height\":800000}" + ], + "spool_tail": ",\"compact_share_identities\":[[\"min\\u00e9r-\\u00e9\",\"00:min\\u00e9r-\\u00e9\",\"0101010101010101010101010101010101010101010101010101010101010101\"],[\"miner-2\",\"02:miner-2\",\"0202020202020202020202020202020202020202020202020202020202020202\"],[\"miner-3\",\"03:miner-3\",\"0303030303030303030303030303030303030303030303030303030303030303\"],[\"MINER-A/worker 1\\u007f\",\"04:MINER-A/worker 1\\u007f\",\"0404040404040404040404040404040404040404040404040404040404040404\"]],\"compact_shares\":[[1,\"min\\u00e9r-\\u00e9.work\\u00e9r:0a1b\",0,3,59997,59998,null],[2,\"\\ud83d\\ude80-share-\\\"quoted\\\"\\\\back\\\\slash\",1,4,59998,59999,\"stale-grace\"],[3,\"share-3\",2,5,59999,60000,null],[4,\"MINER-A/worker 1:\\u007f\\r\\b\\f\\u001f\\u2028\\u2029\",3,6,59999,60000,null]]}" + }, + "record_count": 4, + "record_stream_len": 1502, + "record_stream_sha256": "0f27aad124592ab0a5ec57303cd1a728978d886e06151c5ac4414470e514c2d8", + "spool_tail_len": 718, + "spool_tail_sha256": "fc574b1010ad468cfe56dd2259a7773cdb56904507ff6dc71cbdd1ca39c46092", + "why": "string fields carrying non-ASCII, a non-BMP code point, quotes, backslashes, control characters, DEL, solidus, CR/BS/FF, U+001F, U+2028/U+2029, space and uppercase: pins the ensure_ascii escape rules -- surrogate pairs, \\u007f for DEL, raw solidus, short escapes, and pass-through of printable ASCII" + }, + "ntime-beyond-u32": { + "advance_stats": [], + "canonical_bytes_len": 1072, + "canonical_bytes_sha256": "55a52eff2234207345c1fbbbf81d7b33abba263e3810b2a07cecbf4532d5bf4b", + "canonical_digest": "55a52eff2234207345c1fbbbf81d7b33abba263e3810b2a07cecbf4532d5bf4b", + "input_sha256": "2efc000d6c25b0dc049db474d21e4965095b7160a52debb91347dcf75e54e52f", + "pinned_literals": { + "canonical_bytes": "[{\"accepted_at_ms\":1754999999999,\"job_id\":\"job-1\",\"job_issued_at_ms\":1754999999998,\"miner_id\":\"miner-1\",\"network_difficulty\":1000001,\"ntime\":4294967296,\"order_key\":\"01:miner-1\",\"p2mr_program_hex\":\"0101010101010101010101010101010101010101010101010101010101010101\",\"share_difficulty\":3,\"share_id\":\"share-1\",\"share_seq\":1,\"template_height\":800000},{\"accepted_at_ms\":1754999999999,\"credit_policy\":\"stale-grace\",\"job_id\":\"job-2\",\"job_issued_at_ms\":1754999999999,\"miner_id\":\"miner-2\",\"network_difficulty\":1000002,\"ntime\":9223372036854775807,\"order_key\":\"02:miner-2\",\"p2mr_program_hex\":\"0202020202020202020202020202020202020202020202020202020202020202\",\"share_difficulty\":4,\"share_id\":\"share-2\",\"share_seq\":2,\"template_height\":800000},{\"accepted_at_ms\":1755000000000,\"job_id\":\"job-3\",\"job_issued_at_ms\":1755000000000,\"miner_id\":\"miner-3\",\"network_difficulty\":1000003,\"ntime\":1700000003,\"order_key\":\"03:miner-3\",\"p2mr_program_hex\":\"0303030303030303030303030303030303030303030303030303030303030303\",\"share_difficulty\":5,\"share_id\":\"share-3\",\"share_seq\":3,\"template_height\":800000}]", + "input": { + "advances": [], + "name": "ntime-beyond-u32", + "page_size": 512, + "schema": "qbit-prism-window-pipeline-parity-input/v1", + "snapshot": { + "anchor_job_issued_at_ms": 1755000000000, + "records": [ + { + "accepted_at_ms": 1754999999999, + "credit_policy": null, + "job_id": "job-1", + "job_issued_at_ms": 1754999999998, + "miner_id": "miner-1", + "network_difficulty": 1000001, + "ntime": 4294967296, + "order_key": "01:miner-1", + "p2mr_program_hex": "0101010101010101010101010101010101010101010101010101010101010101", + "share_difficulty": 3, + "share_id": "share-1", + "share_seq": 1, + "template_height": 800000 + }, + { + "accepted_at_ms": 1754999999999, + "credit_policy": "stale-grace", + "job_id": "job-2", + "job_issued_at_ms": 1754999999999, + "miner_id": "miner-2", + "network_difficulty": 1000002, + "ntime": 9223372036854775807, + "order_key": "02:miner-2", + "p2mr_program_hex": "0202020202020202020202020202020202020202020202020202020202020202", + "share_difficulty": 4, + "share_id": "share-2", + "share_seq": 2, + "template_height": 800000 + }, + { + "accepted_at_ms": 1755000000000, + "credit_policy": null, + "job_id": "job-3", + "job_issued_at_ms": 1755000000000, + "miner_id": "miner-3", + "network_difficulty": 1000003, + "ntime": 1700000003, + "order_key": "03:miner-3", + "p2mr_program_hex": "0303030303030303030303030303030303030303030303030303030303030303", + "share_difficulty": 5, + "share_id": "share-3", + "share_seq": 3, + "template_height": 800000 + } + ] + }, + "window_weight": 1000 + }, + "record_jsons": [ + "{\"accepted_at_ms\":1754999999999,\"job_id\":\"job-1\",\"job_issued_at_ms\":1754999999998,\"miner_id\":\"miner-1\",\"network_difficulty\":1000001,\"ntime\":4294967296,\"order_key\":\"01:miner-1\",\"p2mr_program_hex\":\"0101010101010101010101010101010101010101010101010101010101010101\",\"share_difficulty\":3,\"share_id\":\"share-1\",\"share_seq\":1,\"template_height\":800000}", + "{\"accepted_at_ms\":1754999999999,\"credit_policy\":\"stale-grace\",\"job_id\":\"job-2\",\"job_issued_at_ms\":1754999999999,\"miner_id\":\"miner-2\",\"network_difficulty\":1000002,\"ntime\":9223372036854775807,\"order_key\":\"02:miner-2\",\"p2mr_program_hex\":\"0202020202020202020202020202020202020202020202020202020202020202\",\"share_difficulty\":4,\"share_id\":\"share-2\",\"share_seq\":2,\"template_height\":800000}", + "{\"accepted_at_ms\":1755000000000,\"job_id\":\"job-3\",\"job_issued_at_ms\":1755000000000,\"miner_id\":\"miner-3\",\"network_difficulty\":1000003,\"ntime\":1700000003,\"order_key\":\"03:miner-3\",\"p2mr_program_hex\":\"0303030303030303030303030303030303030303030303030303030303030303\",\"share_difficulty\":5,\"share_id\":\"share-3\",\"share_seq\":3,\"template_height\":800000}" + ], + "spool_tail": ",\"compact_share_identities\":[[\"miner-1\",\"01:miner-1\",\"0101010101010101010101010101010101010101010101010101010101010101\"],[\"miner-2\",\"02:miner-2\",\"0202020202020202020202020202020202020202020202020202020202020202\"],[\"miner-3\",\"03:miner-3\",\"0303030303030303030303030303030303030303030303030303030303030303\"]],\"compact_shares\":[[1,\"share-1\",0,3,1754999999998,1754999999999,null],[2,\"share-2\",1,4,1754999999999,1754999999999,\"stale-grace\"],[3,\"share-3\",2,5,1755000000000,1755000000000,null]]}" + }, + "record_count": 3, + "record_stream_len": 1070, + "record_stream_sha256": "51be88f333f400b0c0297a079106b8e18e391e6e09b00dbcff999bbf20e9ed9c", + "spool_tail_len": 487, + "spool_tail_sha256": "fab4c8c3d72ed0f0dbf06bbf587608bbf81b0f86da24a4abf76d3dccb7e10b04", + "why": "ntime at 2^32 and 2^63-1 (bigint's maximum) on otherwise ordinary records at a realistic anchor: outside the declared u32 width alone, so a backend declaring it answers Unsupported here while the frozen bytes stay Python's full-digit rendering; one that truncates ntime to 32 bits renders different digits and fails" + }, + "page-boundary-511": { + "advance_stats": [], + "canonical_bytes_len": 174438, + "canonical_bytes_sha256": "58feda454b217b56f517e1d3a45656097c808691f5d36b8be709a1b8bc0c91e6", + "canonical_digest": "58feda454b217b56f517e1d3a45656097c808691f5d36b8be709a1b8bc0c91e6", + "input_sha256": "2a0911a1571970273d508db9f099064a92a3f750af949375d30df2e45e1c0810", + "record_count": 511, + "record_stream_len": 174436, + "record_stream_sha256": "9d8c3a86d59206d39acede0996dcf2bc188f5c289a2b92b91a232326fd5350a8", + "spool_tail_len": 69043, + "spool_tail_sha256": "94a123b56f8cbf7be963e8cac75a2a5ccf0d7f5dcc4189b95968ae17e6e537d6", + "why": "exactly 511 retained records around the default page size 512: pins page splitting and the digest's page-fragment framing at the boundary" + }, + "page-boundary-512": { + "advance_stats": [], + "canonical_bytes_len": 174535, + "canonical_bytes_sha256": "a6a006c0226496981bfc70cbbae5238d07f270c719a1a0a1987224c60c644f0d", + "canonical_digest": "a6a006c0226496981bfc70cbbae5238d07f270c719a1a0a1987224c60c644f0d", + "input_sha256": "bee70ffe8279e9b3b22fb39a101fb296600b17520855623c2e490d9e123d9387", + "record_count": 512, + "record_stream_len": 174533, + "record_stream_sha256": "f3e63c2bd2e64e6d783596dbf9041cd811cad2782ff35e88d20b6032ceda50f2", + "spool_tail_len": 69107, + "spool_tail_sha256": "4b7581d08b9356ec17b2d0a877529a95412ad60a130f722548eb4a9e190cf70e", + "why": "exactly 512 retained records around the default page size 512: pins page splitting and the digest's page-fragment framing at the boundary" + }, + "page-boundary-513": { + "advance_stats": [], + "canonical_bytes_len": 175164, + "canonical_bytes_sha256": "dfb7169f9f41dff0e8bb04e146f7c276ad385e3d5129b5a64994c132f1507d78", + "canonical_digest": "dfb7169f9f41dff0e8bb04e146f7c276ad385e3d5129b5a64994c132f1507d78", + "input_sha256": "66a66e502b4ffe31be1abbf6aa42d59dfd2cd0e492d4afd4cc7fb98f72fc500d", + "record_count": 513, + "record_stream_len": 175162, + "record_stream_sha256": "101ee825632e35a14dcccf24250193ee7fb7906b9c817c52ec3250ec99a7765b", + "spool_tail_len": 69346, + "spool_tail_sha256": "691beb8a990c68061b35b0dfdb3f0add802440a4cd7014d0429099b89a225d57", + "why": "exactly 513 retained records around the default page size 512: pins page splitting and the digest's page-fragment framing at the boundary" + }, + "reject-anchor-regression": { + "input_sha256": "79c29d1318069550430ff1d38a91cda2e3b7321c6e622e37542a4b66985e1008", + "pinned_literals": { + "input": { + "advances": [ + { + "anchor_job_issued_at_ms": 199999, + "records": [] + } + ], + "name": "reject-anchor-regression", + "page_size": 512, + "schema": "qbit-prism-window-pipeline-parity-input/v1", + "snapshot": { + "anchor_job_issued_at_ms": 200000, + "records": [ + { + "accepted_at_ms": 199999, + "credit_policy": null, + "job_id": "job-1", + "job_issued_at_ms": 199999, + "miner_id": "miner-1", + "network_difficulty": 1000001, + "ntime": 1700000001, + "order_key": "01:miner-1", + "p2mr_program_hex": "0101010101010101010101010101010101010101010101010101010101010101", + "share_difficulty": 1, + "share_id": "share-1", + "share_seq": 1, + "template_height": 800000 + }, + { + "accepted_at_ms": 199997, + "credit_policy": null, + "job_id": "job-3", + "job_issued_at_ms": 199997, + "miner_id": "miner-3", + "network_difficulty": 1000003, + "ntime": 1700000003, + "order_key": "03:miner-3", + "p2mr_program_hex": "0303030303030303030303030303030303030303030303030303030303030303", + "share_difficulty": 1, + "share_id": "share-3", + "share_seq": 3, + "template_height": 800000 + } + ] + }, + "window_weight": 100 + } + }, + "rejected": "anchor_regression", + "why": "an advance whose anchor is one ms earlier than the window's, with an empty delta: the anchor check fires before any record check" + }, + "reject-delta-ineligible-accepted-only": { + "input_sha256": "1466801b610efdb6838c82e2292da33221aae0c53b9da9387d61cfa1b6d46250", + "pinned_literals": { + "input": { + "advances": [ + { + "anchor_job_issued_at_ms": 200001, + "records": [ + { + "accepted_at_ms": 200002, + "credit_policy": null, + "job_id": "job-4", + "job_issued_at_ms": 200001, + "miner_id": "miner-4", + "network_difficulty": 1000004, + "ntime": 1700000004, + "order_key": "04:miner-4", + "p2mr_program_hex": "0404040404040404040404040404040404040404040404040404040404040404", + "share_difficulty": 1, + "share_id": "share-4", + "share_seq": 4, + "template_height": 800000 + } + ] + } + ], + "name": "reject-delta-ineligible-accepted-only", + "page_size": 512, + "schema": "qbit-prism-window-pipeline-parity-input/v1", + "snapshot": { + "anchor_job_issued_at_ms": 200000, + "records": [ + { + "accepted_at_ms": 199999, + "credit_policy": null, + "job_id": "job-1", + "job_issued_at_ms": 199999, + "miner_id": "miner-1", + "network_difficulty": 1000001, + "ntime": 1700000001, + "order_key": "01:miner-1", + "p2mr_program_hex": "0101010101010101010101010101010101010101010101010101010101010101", + "share_difficulty": 1, + "share_id": "share-1", + "share_seq": 1, + "template_height": 800000 + }, + { + "accepted_at_ms": 199997, + "credit_policy": null, + "job_id": "job-3", + "job_issued_at_ms": 199997, + "miner_id": "miner-3", + "network_difficulty": 1000003, + "ntime": 1700000003, + "order_key": "03:miner-3", + "p2mr_program_hex": "0303030303030303030303030303030303030303030303030303030303030303", + "share_difficulty": 1, + "share_id": "share-3", + "share_seq": 3, + "template_height": 800000 + } + ] + }, + "window_weight": 100 + } + }, + "rejected": "delta_ineligible_at_anchor", + "why": "a delta record whose accepted_at_ms is one ms past the new anchor while job_issued_at_ms is on it: ineligible by accepted alone" + }, + "reject-delta-ineligible-job-only": { + "input_sha256": "017a713d758ec1e2adbe9911db9a0faba24fef644f374861fa28c37b3f4ec3ee", + "pinned_literals": { + "input": { + "advances": [ + { + "anchor_job_issued_at_ms": 200001, + "records": [ + { + "accepted_at_ms": 200001, + "credit_policy": null, + "job_id": "job-4", + "job_issued_at_ms": 200002, + "miner_id": "miner-4", + "network_difficulty": 1000004, + "ntime": 1700000004, + "order_key": "04:miner-4", + "p2mr_program_hex": "0404040404040404040404040404040404040404040404040404040404040404", + "share_difficulty": 1, + "share_id": "share-4", + "share_seq": 4, + "template_height": 800000 + } + ] + } + ], + "name": "reject-delta-ineligible-job-only", + "page_size": 512, + "schema": "qbit-prism-window-pipeline-parity-input/v1", + "snapshot": { + "anchor_job_issued_at_ms": 200000, + "records": [ + { + "accepted_at_ms": 199999, + "credit_policy": null, + "job_id": "job-1", + "job_issued_at_ms": 199999, + "miner_id": "miner-1", + "network_difficulty": 1000001, + "ntime": 1700000001, + "order_key": "01:miner-1", + "p2mr_program_hex": "0101010101010101010101010101010101010101010101010101010101010101", + "share_difficulty": 1, + "share_id": "share-1", + "share_seq": 1, + "template_height": 800000 + }, + { + "accepted_at_ms": 199997, + "credit_policy": null, + "job_id": "job-3", + "job_issued_at_ms": 199997, + "miner_id": "miner-3", + "network_difficulty": 1000003, + "ntime": 1700000003, + "order_key": "03:miner-3", + "p2mr_program_hex": "0303030303030303030303030303030303030303030303030303030303030303", + "share_difficulty": 1, + "share_id": "share-3", + "share_seq": 3, + "template_height": 800000 + } + ] + }, + "window_weight": 100 + } + }, + "rejected": "delta_ineligible_at_anchor", + "why": "a delta record whose job_issued_at_ms is one ms past the new anchor while accepted_at_ms is on it: ineligible by job alone" + }, + "reject-delta-non-positive-difficulty": { + "input_sha256": "24bf4cb0be0e324fad50c0414e0517044cc9dd3972cf35bd9cbbae34b43d9ea5", + "pinned_literals": { + "input": { + "advances": [ + { + "anchor_job_issued_at_ms": 200001, + "records": [ + { + "accepted_at_ms": 200001, + "credit_policy": null, + "job_id": "job-4", + "job_issued_at_ms": 200001, + "miner_id": "miner-4", + "network_difficulty": 1000004, + "ntime": 1700000004, + "order_key": "04:miner-4", + "p2mr_program_hex": "0404040404040404040404040404040404040404040404040404040404040404", + "share_difficulty": 0, + "share_id": "share-4", + "share_seq": 4, + "template_height": 800000 + } + ] + } + ], + "name": "reject-delta-non-positive-difficulty", + "page_size": 512, + "schema": "qbit-prism-window-pipeline-parity-input/v1", + "snapshot": { + "anchor_job_issued_at_ms": 200000, + "records": [ + { + "accepted_at_ms": 199999, + "credit_policy": null, + "job_id": "job-1", + "job_issued_at_ms": 199999, + "miner_id": "miner-1", + "network_difficulty": 1000001, + "ntime": 1700000001, + "order_key": "01:miner-1", + "p2mr_program_hex": "0101010101010101010101010101010101010101010101010101010101010101", + "share_difficulty": 1, + "share_id": "share-1", + "share_seq": 1, + "template_height": 800000 + }, + { + "accepted_at_ms": 199997, + "credit_policy": null, + "job_id": "job-3", + "job_issued_at_ms": 199997, + "miner_id": "miner-3", + "network_difficulty": 1000003, + "ntime": 1700000003, + "order_key": "03:miner-3", + "p2mr_program_hex": "0303030303030303030303030303030303030303030303030303030303030303", + "share_difficulty": 1, + "share_id": "share-3", + "share_seq": 3, + "template_height": 800000 + } + ] + }, + "window_weight": 100 + } + }, + "rejected": "delta_non_positive_difficulty", + "why": "a newly eligible delta record of difficulty 0; the difficulty check is the first per-record check" + }, + "reject-delta-not-append": { + "input_sha256": "eee699914831120eb22a95466c164c6d0d09d910e9f66d1b7ca658a4f5ca4ecc", + "pinned_literals": { + "input": { + "advances": [ + { + "anchor_job_issued_at_ms": 200001, + "records": [ + { + "accepted_at_ms": 200001, + "credit_policy": null, + "job_id": "job-2", + "job_issued_at_ms": 200001, + "miner_id": "miner-2", + "network_difficulty": 1000002, + "ntime": 1700000002, + "order_key": "02:miner-2", + "p2mr_program_hex": "0202020202020202020202020202020202020202020202020202020202020202", + "share_difficulty": 1, + "share_id": "share-2", + "share_seq": 2, + "template_height": 800000 + } + ] + } + ], + "name": "reject-delta-not-append", + "page_size": 512, + "schema": "qbit-prism-window-pipeline-parity-input/v1", + "snapshot": { + "anchor_job_issued_at_ms": 200000, + "records": [ + { + "accepted_at_ms": 199999, + "credit_policy": null, + "job_id": "job-1", + "job_issued_at_ms": 199999, + "miner_id": "miner-1", + "network_difficulty": 1000001, + "ntime": 1700000001, + "order_key": "01:miner-1", + "p2mr_program_hex": "0101010101010101010101010101010101010101010101010101010101010101", + "share_difficulty": 1, + "share_id": "share-1", + "share_seq": 1, + "template_height": 800000 + }, + { + "accepted_at_ms": 199997, + "credit_policy": null, + "job_id": "job-3", + "job_issued_at_ms": 199997, + "miner_id": "miner-3", + "network_difficulty": 1000003, + "ntime": 1700000003, + "order_key": "03:miner-3", + "p2mr_program_hex": "0303030303030303030303030303030303030303030303030303030303030303", + "share_difficulty": 1, + "share_id": "share-3", + "share_seq": 3, + "template_height": 800000 + } + ] + }, + "window_weight": 100 + } + }, + "rejected": "delta_not_append", + "why": "a newly eligible record with seq 2, below the window's last seq 3: the late-visible row that cannot be folded as an append and in production forces the full rescan" + }, + "reject-delta-order-not-increasing": { + "input_sha256": "0218fd5360e854577c6a3a8450b7c20c0178c0d01edb9449bea18efc3d9388b4", + "pinned_literals": { + "input": { + "advances": [ + { + "anchor_job_issued_at_ms": 200001, + "records": [ + { + "accepted_at_ms": 200001, + "credit_policy": null, + "job_id": "job-5", + "job_issued_at_ms": 200001, + "miner_id": "miner-5", + "network_difficulty": 1000005, + "ntime": 1700000005, + "order_key": "05:miner-5", + "p2mr_program_hex": "0505050505050505050505050505050505050505050505050505050505050505", + "share_difficulty": 1, + "share_id": "share-5", + "share_seq": 5, + "template_height": 800000 + }, + { + "accepted_at_ms": 200001, + "credit_policy": null, + "job_id": "job-4", + "job_issued_at_ms": 200001, + "miner_id": "miner-4", + "network_difficulty": 1000004, + "ntime": 1700000004, + "order_key": "04:miner-4", + "p2mr_program_hex": "0404040404040404040404040404040404040404040404040404040404040404", + "share_difficulty": 1, + "share_id": "share-4", + "share_seq": 4, + "template_height": 800000 + } + ] + } + ], + "name": "reject-delta-order-not-increasing", + "page_size": 512, + "schema": "qbit-prism-window-pipeline-parity-input/v1", + "snapshot": { + "anchor_job_issued_at_ms": 200000, + "records": [ + { + "accepted_at_ms": 199999, + "credit_policy": null, + "job_id": "job-1", + "job_issued_at_ms": 199999, + "miner_id": "miner-1", + "network_difficulty": 1000001, + "ntime": 1700000001, + "order_key": "01:miner-1", + "p2mr_program_hex": "0101010101010101010101010101010101010101010101010101010101010101", + "share_difficulty": 1, + "share_id": "share-1", + "share_seq": 1, + "template_height": 800000 + }, + { + "accepted_at_ms": 199997, + "credit_policy": null, + "job_id": "job-3", + "job_issued_at_ms": 199997, + "miner_id": "miner-3", + "network_difficulty": 1000003, + "ntime": 1700000003, + "order_key": "03:miner-3", + "p2mr_program_hex": "0303030303030303030303030303030303030303030303030303030303030303", + "share_difficulty": 1, + "share_id": "share-3", + "share_seq": 3, + "template_height": 800000 + } + ] + }, + "window_weight": 100 + } + }, + "rejected": "delta_order_not_increasing", + "why": "two newly eligible records delivered as seqs 5 then 4, both above the window's last seq so only the intra-delta order check can fire" + }, + "reject-delta-repeats-retained-share": { + "input_sha256": "0e7a23e673ace185119f9df85494287656a5cb746b02657e7698a29f8e37ae42", + "pinned_literals": { + "input": { + "advances": [ + { + "anchor_job_issued_at_ms": 200001, + "records": [ + { + "accepted_at_ms": 199997, + "credit_policy": null, + "job_id": "job-3", + "job_issued_at_ms": 199997, + "miner_id": "miner-3", + "network_difficulty": 1000003, + "ntime": 1700000003, + "order_key": "03:miner-3", + "p2mr_program_hex": "0303030303030303030303030303030303030303030303030303030303030303", + "share_difficulty": 1, + "share_id": "share-3", + "share_seq": 3, + "template_height": 800000 + } + ] + } + ], + "name": "reject-delta-repeats-retained-share", + "page_size": 512, + "schema": "qbit-prism-window-pipeline-parity-input/v1", + "snapshot": { + "anchor_job_issued_at_ms": 200000, + "records": [ + { + "accepted_at_ms": 199999, + "credit_policy": null, + "job_id": "job-1", + "job_issued_at_ms": 199999, + "miner_id": "miner-1", + "network_difficulty": 1000001, + "ntime": 1700000001, + "order_key": "01:miner-1", + "p2mr_program_hex": "0101010101010101010101010101010101010101010101010101010101010101", + "share_difficulty": 1, + "share_id": "share-1", + "share_seq": 1, + "template_height": 800000 + }, + { + "accepted_at_ms": 199997, + "credit_policy": null, + "job_id": "job-3", + "job_issued_at_ms": 199997, + "miner_id": "miner-3", + "network_difficulty": 1000003, + "ntime": 1700000003, + "order_key": "03:miner-3", + "p2mr_program_hex": "0303030303030303030303030303030303030303030303030303030303030303", + "share_difficulty": 1, + "share_id": "share-3", + "share_seq": 3, + "template_height": 800000 + } + ] + }, + "window_weight": 100 + } + }, + "rejected": "delta_repeats_eligible_share", + "why": "the delta re-delivers seq 3, a share eligible at the previous anchor and still inside the window: refused as a repeat before the append-order check could see its seq" + }, + "reject-duplicate-share-id": { + "input_sha256": "3c2c075f2ec645d9e5160321ce90fa930b7548ae8ac81408533ffa4af3df6dde", + "pinned_literals": { + "input": { + "advances": [], + "name": "reject-duplicate-share-id", + "page_size": 512, + "schema": "qbit-prism-window-pipeline-parity-input/v1", + "snapshot": { + "anchor_job_issued_at_ms": 200000, + "records": [ + { + "accepted_at_ms": 199999, + "credit_policy": null, + "job_id": "job-1", + "job_issued_at_ms": 199999, + "miner_id": "miner-1", + "network_difficulty": 1000001, + "ntime": 1700000001, + "order_key": "01:miner-1", + "p2mr_program_hex": "0101010101010101010101010101010101010101010101010101010101010101", + "share_difficulty": 1, + "share_id": "share-1", + "share_seq": 1, + "template_height": 800000 + }, + { + "accepted_at_ms": 199998, + "credit_policy": null, + "job_id": "job-2", + "job_issued_at_ms": 199998, + "miner_id": "miner-2", + "network_difficulty": 1000002, + "ntime": 1700000002, + "order_key": "02:miner-2", + "p2mr_program_hex": "0202020202020202020202020202020202020202020202020202020202020202", + "share_difficulty": 1, + "share_id": "share-1", + "share_seq": 2, + "template_height": 800000 + } + ] + }, + "window_weight": 100 + } + }, + "rejected": "duplicate_share_id", + "why": "seqs 1 and 2 carry the same share_id; distinct seqs so only the share_id check can fire" + }, + "reject-duplicate-share-seq": { + "input_sha256": "564b7fe128b14dfcc7aaf8e87c9e4fa6286debc7c5b54c3e6323f1cf1eef5c34", + "pinned_literals": { + "input": { + "advances": [], + "name": "reject-duplicate-share-seq", + "page_size": 512, + "schema": "qbit-prism-window-pipeline-parity-input/v1", + "snapshot": { + "anchor_job_issued_at_ms": 200000, + "records": [ + { + "accepted_at_ms": 199999, + "credit_policy": null, + "job_id": "job-1", + "job_issued_at_ms": 199999, + "miner_id": "miner-1", + "network_difficulty": 1000001, + "ntime": 1700000001, + "order_key": "01:miner-1", + "p2mr_program_hex": "0101010101010101010101010101010101010101010101010101010101010101", + "share_difficulty": 1, + "share_id": "share-1", + "share_seq": 1, + "template_height": 800000 + }, + { + "accepted_at_ms": 199999, + "credit_policy": null, + "job_id": "job-1", + "job_issued_at_ms": 199999, + "miner_id": "miner-1", + "network_difficulty": 1000001, + "ntime": 1700000001, + "order_key": "01:miner-1", + "p2mr_program_hex": "0101010101010101010101010101010101010101010101010101010101010101", + "share_difficulty": 2, + "share_id": "share-1-again", + "share_seq": 1, + "template_height": 800000 + } + ] + }, + "window_weight": 100 + } + }, + "rejected": "duplicate_share_seq", + "why": "two eligible records share share_seq 1 under distinct share_ids; the sorted-order duplicate check fires before the share_id one" + }, + "reject-non-positive-difficulty": { + "input_sha256": "a86dc8aff25e7fe2439cda3142f792d11b06165d616ab3f77108c5f8fba90181", + "pinned_literals": { + "input": { + "advances": [], + "name": "reject-non-positive-difficulty", + "page_size": 512, + "schema": "qbit-prism-window-pipeline-parity-input/v1", + "snapshot": { + "anchor_job_issued_at_ms": 200000, + "records": [ + { + "accepted_at_ms": 199999, + "credit_policy": null, + "job_id": "job-1", + "job_issued_at_ms": 199999, + "miner_id": "miner-1", + "network_difficulty": 1000001, + "ntime": 1700000001, + "order_key": "01:miner-1", + "p2mr_program_hex": "0101010101010101010101010101010101010101010101010101010101010101", + "share_difficulty": 0, + "share_id": "share-1", + "share_seq": 1, + "template_height": 800000 + } + ] + }, + "window_weight": 100 + } + }, + "rejected": "non_positive_difficulty", + "why": "one eligible record with share_difficulty 0 (the check runs on eligible records only, so it must be eligible to be refused)" + }, + "reject-non-positive-page-size": { + "input_sha256": "de58627985157085da42c12bfe30ff83c6d1002ca70609936b5f23447fbf0528", + "pinned_literals": { + "input": { + "advances": [], + "name": "reject-non-positive-page-size", + "page_size": 0, + "schema": "qbit-prism-window-pipeline-parity-input/v1", + "snapshot": { + "anchor_job_issued_at_ms": 200000, + "records": [ + { + "accepted_at_ms": 199999, + "credit_policy": null, + "job_id": "job-1", + "job_issued_at_ms": 199999, + "miner_id": "miner-1", + "network_difficulty": 1000001, + "ntime": 1700000001, + "order_key": "01:miner-1", + "p2mr_program_hex": "0101010101010101010101010101010101010101010101010101010101010101", + "share_difficulty": 1, + "share_id": "share-1", + "share_seq": 1, + "template_height": 800000 + } + ] + }, + "window_weight": 100 + } + }, + "rejected": "non_positive_page_size", + "why": "page_size 0 with one ordinary eligible record; refused before any record is examined" + }, + "reject-non-positive-window-weight": { + "input_sha256": "e9c66266b5c0d0363f4af32cbbf5b886468ae03a9041433a41223705d8454929", + "pinned_literals": { + "input": { + "advances": [], + "name": "reject-non-positive-window-weight", + "page_size": 512, + "schema": "qbit-prism-window-pipeline-parity-input/v1", + "snapshot": { + "anchor_job_issued_at_ms": 200000, + "records": [ + { + "accepted_at_ms": 199999, + "credit_policy": null, + "job_id": "job-1", + "job_issued_at_ms": 199999, + "miner_id": "miner-1", + "network_difficulty": 1000001, + "ntime": 1700000001, + "order_key": "01:miner-1", + "p2mr_program_hex": "0101010101010101010101010101010101010101010101010101010101010101", + "share_difficulty": 1, + "share_id": "share-1", + "share_seq": 1, + "template_height": 800000 + } + ] + }, + "window_weight": 0 + } + }, + "rejected": "non_positive_window_weight", + "why": "window_weight 0 with one ordinary eligible record; refused before any record is examined" + }, + "single-record": { + "advance_stats": [], + "canonical_bytes_len": 327, + "canonical_bytes_sha256": "0c0f79989d847dff32c1ec70990ed03c73fb567155eb6fda1576809587116d31", + "canonical_digest": "0c0f79989d847dff32c1ec70990ed03c73fb567155eb6fda1576809587116d31", + "input_sha256": "f58f51dae8af68af210445807e536d84376c0b3a11b47695893913d69c8cf320", + "pinned_literals": { + "canonical_bytes": "[{\"accepted_at_ms\":2000,\"job_id\":\"job-1\",\"job_issued_at_ms\":1999,\"miner_id\":\"miner-1\",\"network_difficulty\":1000001,\"ntime\":1700000001,\"order_key\":\"01:miner-1\",\"p2mr_program_hex\":\"0101010101010101010101010101010101010101010101010101010101010101\",\"share_difficulty\":5,\"share_id\":\"share-1\",\"share_seq\":1,\"template_height\":800000}]", + "input": { + "advances": [], + "name": "single-record", + "page_size": 512, + "schema": "qbit-prism-window-pipeline-parity-input/v1", + "snapshot": { + "anchor_job_issued_at_ms": 2000, + "records": [ + { + "accepted_at_ms": 2000, + "credit_policy": null, + "job_id": "job-1", + "job_issued_at_ms": 1999, + "miner_id": "miner-1", + "network_difficulty": 1000001, + "ntime": 1700000001, + "order_key": "01:miner-1", + "p2mr_program_hex": "0101010101010101010101010101010101010101010101010101010101010101", + "share_difficulty": 5, + "share_id": "share-1", + "share_seq": 1, + "template_height": 800000 + } + ] + }, + "window_weight": 10 + }, + "record_jsons": [ + "{\"accepted_at_ms\":2000,\"job_id\":\"job-1\",\"job_issued_at_ms\":1999,\"miner_id\":\"miner-1\",\"network_difficulty\":1000001,\"ntime\":1700000001,\"order_key\":\"01:miner-1\",\"p2mr_program_hex\":\"0101010101010101010101010101010101010101010101010101010101010101\",\"share_difficulty\":5,\"share_id\":\"share-1\",\"share_seq\":1,\"template_height\":800000}" + ], + "spool_tail": ",\"compact_share_identities\":[[\"miner-1\",\"01:miner-1\",\"0101010101010101010101010101010101010101010101010101010101010101\"]],\"compact_shares\":[[1,\"share-1\",0,5,1999,2000,null]]}" + }, + "record_count": 1, + "record_stream_len": 325, + "record_stream_sha256": "b25c06b504068a2ba0ad150679c26e1648f9dcd0869a3294a20a50196e71145f", + "spool_tail_len": 174, + "spool_tail_sha256": "69673fe5321dfefc026ccedcd8f4ebbd38c562dadf1412dbe872db6225e75601", + "why": "one record with credit_policy None: to_prism_json omits the key while the compact share tuple carries an explicit null -- the null-vs-absent divergence surface" + }, + "unsorted-input": { + "advance_stats": [], + "canonical_bytes_len": 3098, + "canonical_bytes_sha256": "e4fbb63bef1cf2d8ab54afc77b766af863bdf2e8c56baaf21f82ab50443b3eb9", + "canonical_digest": "e4fbb63bef1cf2d8ab54afc77b766af863bdf2e8c56baaf21f82ab50443b3eb9", + "input_sha256": "27a0c45f9c641dafec411ba5a5df29f42957a34b43ce1355b3cb0300f3b6b19f", + "pinned_literals": { + "canonical_bytes": "[{\"accepted_at_ms\":39988,\"credit_policy\":\"stale-grace\",\"job_id\":\"job-9\",\"job_issued_at_ms\":39975,\"miner_id\":\"miner-9\",\"network_difficulty\":1000009,\"ntime\":1700000009,\"order_key\":\"09:miner-9\",\"p2mr_program_hex\":\"0909090909090909090909090909090909090909090909090909090909090909\",\"share_difficulty\":19,\"share_id\":\"share-9\",\"share_seq\":9,\"template_height\":800000},{\"accepted_at_ms\":39984,\"credit_policy\":\"stale-grace\",\"job_id\":\"job-19\",\"job_issued_at_ms\":39989,\"miner_id\":\"miner-8\",\"network_difficulty\":1000002,\"ntime\":1700000019,\"order_key\":\"08:miner-8\",\"p2mr_program_hex\":\"1313131313131313131313131313131313131313131313131313131313131313\",\"share_difficulty\":1,\"share_id\":\"share-19\",\"share_seq\":19,\"template_height\":800000},{\"accepted_at_ms\":39987,\"credit_policy\":\"stale-grace\",\"job_id\":\"job-20\",\"job_issued_at_ms\":39981,\"miner_id\":\"miner-9\",\"network_difficulty\":1000003,\"ntime\":1700000020,\"order_key\":\"09:miner-9\",\"p2mr_program_hex\":\"1414141414141414141414141414141414141414141414141414141414141414\",\"share_difficulty\":2,\"share_id\":\"share-20\",\"share_seq\":20,\"template_height\":800000},{\"accepted_at_ms\":39971,\"credit_policy\":\"stale-grace\",\"job_id\":\"job-24\",\"job_issued_at_ms\":39986,\"miner_id\":\"miner-2\",\"network_difficulty\":1000007,\"ntime\":1700000024,\"order_key\":\"02:miner-2\",\"p2mr_program_hex\":\"1818181818181818181818181818181818181818181818181818181818181818\",\"share_difficulty\":3,\"share_id\":\"share-24\",\"share_seq\":24,\"template_height\":800000},{\"accepted_at_ms\":39976,\"job_id\":\"job-28\",\"job_issued_at_ms\":40000,\"miner_id\":\"miner-6\",\"network_difficulty\":1000011,\"ntime\":1700000028,\"order_key\":\"06:miner-6\",\"p2mr_program_hex\":\"1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c\",\"share_difficulty\":1,\"share_id\":\"share-28\",\"share_seq\":28,\"template_height\":800000},{\"accepted_at_ms\":39992,\"job_id\":\"job-84\",\"job_issued_at_ms\":39971,\"miner_id\":\"miner-7\",\"network_difficulty\":1000016,\"ntime\":1700000084,\"order_key\":\"07:miner-7\",\"p2mr_program_hex\":\"5454545454545454545454545454545454545454545454545454545454545454\",\"share_difficulty\":1,\"share_id\":\"share-84\",\"share_seq\":84,\"template_height\":800002},{\"accepted_at_ms\":39980,\"job_id\":\"job-88\",\"job_issued_at_ms\":39978,\"miner_id\":\"miner-0\",\"network_difficulty\":1000003,\"ntime\":1700000088,\"order_key\":\"00:miner-0\",\"p2mr_program_hex\":\"5858585858585858585858585858585858585858585858585858585858585858\",\"share_difficulty\":7,\"share_id\":\"share-88\",\"share_seq\":88,\"template_height\":800002},{\"accepted_at_ms\":39984,\"job_id\":\"job-92\",\"job_issued_at_ms\":39974,\"miner_id\":\"miner-4\",\"network_difficulty\":1000007,\"ntime\":1700000092,\"order_key\":\"04:miner-4\",\"p2mr_program_hex\":\"5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c\",\"share_difficulty\":3,\"share_id\":\"share-92\",\"share_seq\":92,\"template_height\":800002},{\"accepted_at_ms\":39988,\"job_id\":\"job-94\",\"job_issued_at_ms\":40000,\"miner_id\":\"miner-6\",\"network_difficulty\":1000009,\"ntime\":1700000094,\"order_key\":\"06:miner-6\",\"p2mr_program_hex\":\"5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e\",\"share_difficulty\":1,\"share_id\":\"share-94\",\"share_seq\":94,\"template_height\":800002}]", + "input": { + "advances": [], + "name": "unsorted-input", + "page_size": 512, + "schema": "qbit-prism-window-pipeline-parity-input/v1", + "snapshot": { + "anchor_job_issued_at_ms": 40000, + "records": [ + { + "accepted_at_ms": 39984, + "credit_policy": null, + "job_id": "job-92", + "job_issued_at_ms": 39974, + "miner_id": "miner-4", + "network_difficulty": 1000007, + "ntime": 1700000092, + "order_key": "04:miner-4", + "p2mr_program_hex": "5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c", + "share_difficulty": 3, + "share_id": "share-92", + "share_seq": 92, + "template_height": 800002 + }, + { + "accepted_at_ms": 39971, + "credit_policy": "stale-grace", + "job_id": "job-24", + "job_issued_at_ms": 39986, + "miner_id": "miner-2", + "network_difficulty": 1000007, + "ntime": 1700000024, + "order_key": "02:miner-2", + "p2mr_program_hex": "1818181818181818181818181818181818181818181818181818181818181818", + "share_difficulty": 3, + "share_id": "share-24", + "share_seq": 24, + "template_height": 800000 + }, + { + "accepted_at_ms": 39976, + "credit_policy": null, + "job_id": "job-28", + "job_issued_at_ms": 40000, + "miner_id": "miner-6", + "network_difficulty": 1000011, + "ntime": 1700000028, + "order_key": "06:miner-6", + "p2mr_program_hex": "1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c", + "share_difficulty": 1, + "share_id": "share-28", + "share_seq": 28, + "template_height": 800000 + }, + { + "accepted_at_ms": 39988, + "credit_policy": null, + "job_id": "job-94", + "job_issued_at_ms": 40000, + "miner_id": "miner-6", + "network_difficulty": 1000009, + "ntime": 1700000094, + "order_key": "06:miner-6", + "p2mr_program_hex": "5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e", + "share_difficulty": 1, + "share_id": "share-94", + "share_seq": 94, + "template_height": 800002 + }, + { + "accepted_at_ms": 39987, + "credit_policy": "stale-grace", + "job_id": "job-20", + "job_issued_at_ms": 39981, + "miner_id": "miner-9", + "network_difficulty": 1000003, + "ntime": 1700000020, + "order_key": "09:miner-9", + "p2mr_program_hex": "1414141414141414141414141414141414141414141414141414141414141414", + "share_difficulty": 2, + "share_id": "share-20", + "share_seq": 20, + "template_height": 800000 + }, + { + "accepted_at_ms": 39988, + "credit_policy": "stale-grace", + "job_id": "job-9", + "job_issued_at_ms": 39975, + "miner_id": "miner-9", + "network_difficulty": 1000009, + "ntime": 1700000009, + "order_key": "09:miner-9", + "p2mr_program_hex": "0909090909090909090909090909090909090909090909090909090909090909", + "share_difficulty": 19, + "share_id": "share-9", + "share_seq": 9, + "template_height": 800000 + }, + { + "accepted_at_ms": 39980, + "credit_policy": null, + "job_id": "job-88", + "job_issued_at_ms": 39978, + "miner_id": "miner-0", + "network_difficulty": 1000003, + "ntime": 1700000088, + "order_key": "00:miner-0", + "p2mr_program_hex": "5858585858585858585858585858585858585858585858585858585858585858", + "share_difficulty": 7, + "share_id": "share-88", + "share_seq": 88, + "template_height": 800002 + }, + { + "accepted_at_ms": 39992, + "credit_policy": null, + "job_id": "job-84", + "job_issued_at_ms": 39971, + "miner_id": "miner-7", + "network_difficulty": 1000016, + "ntime": 1700000084, + "order_key": "07:miner-7", + "p2mr_program_hex": "5454545454545454545454545454545454545454545454545454545454545454", + "share_difficulty": 1, + "share_id": "share-84", + "share_seq": 84, + "template_height": 800002 + }, + { + "accepted_at_ms": 39984, + "credit_policy": "stale-grace", + "job_id": "job-19", + "job_issued_at_ms": 39989, + "miner_id": "miner-8", + "network_difficulty": 1000002, + "ntime": 1700000019, + "order_key": "08:miner-8", + "p2mr_program_hex": "1313131313131313131313131313131313131313131313131313131313131313", + "share_difficulty": 1, + "share_id": "share-19", + "share_seq": 19, + "template_height": 800000 + } + ] + }, + "window_weight": 1000000 + }, + "record_jsons": [ + "{\"accepted_at_ms\":39988,\"credit_policy\":\"stale-grace\",\"job_id\":\"job-9\",\"job_issued_at_ms\":39975,\"miner_id\":\"miner-9\",\"network_difficulty\":1000009,\"ntime\":1700000009,\"order_key\":\"09:miner-9\",\"p2mr_program_hex\":\"0909090909090909090909090909090909090909090909090909090909090909\",\"share_difficulty\":19,\"share_id\":\"share-9\",\"share_seq\":9,\"template_height\":800000}", + "{\"accepted_at_ms\":39984,\"credit_policy\":\"stale-grace\",\"job_id\":\"job-19\",\"job_issued_at_ms\":39989,\"miner_id\":\"miner-8\",\"network_difficulty\":1000002,\"ntime\":1700000019,\"order_key\":\"08:miner-8\",\"p2mr_program_hex\":\"1313131313131313131313131313131313131313131313131313131313131313\",\"share_difficulty\":1,\"share_id\":\"share-19\",\"share_seq\":19,\"template_height\":800000}", + "{\"accepted_at_ms\":39987,\"credit_policy\":\"stale-grace\",\"job_id\":\"job-20\",\"job_issued_at_ms\":39981,\"miner_id\":\"miner-9\",\"network_difficulty\":1000003,\"ntime\":1700000020,\"order_key\":\"09:miner-9\",\"p2mr_program_hex\":\"1414141414141414141414141414141414141414141414141414141414141414\",\"share_difficulty\":2,\"share_id\":\"share-20\",\"share_seq\":20,\"template_height\":800000}", + "{\"accepted_at_ms\":39971,\"credit_policy\":\"stale-grace\",\"job_id\":\"job-24\",\"job_issued_at_ms\":39986,\"miner_id\":\"miner-2\",\"network_difficulty\":1000007,\"ntime\":1700000024,\"order_key\":\"02:miner-2\",\"p2mr_program_hex\":\"1818181818181818181818181818181818181818181818181818181818181818\",\"share_difficulty\":3,\"share_id\":\"share-24\",\"share_seq\":24,\"template_height\":800000}", + "{\"accepted_at_ms\":39976,\"job_id\":\"job-28\",\"job_issued_at_ms\":40000,\"miner_id\":\"miner-6\",\"network_difficulty\":1000011,\"ntime\":1700000028,\"order_key\":\"06:miner-6\",\"p2mr_program_hex\":\"1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c\",\"share_difficulty\":1,\"share_id\":\"share-28\",\"share_seq\":28,\"template_height\":800000}", + "{\"accepted_at_ms\":39992,\"job_id\":\"job-84\",\"job_issued_at_ms\":39971,\"miner_id\":\"miner-7\",\"network_difficulty\":1000016,\"ntime\":1700000084,\"order_key\":\"07:miner-7\",\"p2mr_program_hex\":\"5454545454545454545454545454545454545454545454545454545454545454\",\"share_difficulty\":1,\"share_id\":\"share-84\",\"share_seq\":84,\"template_height\":800002}", + "{\"accepted_at_ms\":39980,\"job_id\":\"job-88\",\"job_issued_at_ms\":39978,\"miner_id\":\"miner-0\",\"network_difficulty\":1000003,\"ntime\":1700000088,\"order_key\":\"00:miner-0\",\"p2mr_program_hex\":\"5858585858585858585858585858585858585858585858585858585858585858\",\"share_difficulty\":7,\"share_id\":\"share-88\",\"share_seq\":88,\"template_height\":800002}", + "{\"accepted_at_ms\":39984,\"job_id\":\"job-92\",\"job_issued_at_ms\":39974,\"miner_id\":\"miner-4\",\"network_difficulty\":1000007,\"ntime\":1700000092,\"order_key\":\"04:miner-4\",\"p2mr_program_hex\":\"5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c\",\"share_difficulty\":3,\"share_id\":\"share-92\",\"share_seq\":92,\"template_height\":800002}", + "{\"accepted_at_ms\":39988,\"job_id\":\"job-94\",\"job_issued_at_ms\":40000,\"miner_id\":\"miner-6\",\"network_difficulty\":1000009,\"ntime\":1700000094,\"order_key\":\"06:miner-6\",\"p2mr_program_hex\":\"5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e\",\"share_difficulty\":1,\"share_id\":\"share-94\",\"share_seq\":94,\"template_height\":800002}" + ], + "spool_tail": ",\"compact_share_identities\":[[\"miner-9\",\"09:miner-9\",\"0909090909090909090909090909090909090909090909090909090909090909\"],[\"miner-8\",\"08:miner-8\",\"1313131313131313131313131313131313131313131313131313131313131313\"],[\"miner-9\",\"09:miner-9\",\"1414141414141414141414141414141414141414141414141414141414141414\"],[\"miner-2\",\"02:miner-2\",\"1818181818181818181818181818181818181818181818181818181818181818\"],[\"miner-6\",\"06:miner-6\",\"1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c\"],[\"miner-7\",\"07:miner-7\",\"5454545454545454545454545454545454545454545454545454545454545454\"],[\"miner-0\",\"00:miner-0\",\"5858585858585858585858585858585858585858585858585858585858585858\"],[\"miner-4\",\"04:miner-4\",\"5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c\"],[\"miner-6\",\"06:miner-6\",\"5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e\"]],\"compact_shares\":[[9,\"share-9\",0,19,39975,39988,\"stale-grace\"],[19,\"share-19\",1,1,39989,39984,\"stale-grace\"],[20,\"share-20\",2,2,39981,39987,\"stale-grace\"],[24,\"share-24\",3,3,39986,39971,\"stale-grace\"],[28,\"share-28\",4,1,40000,39976,null],[84,\"share-84\",5,1,39971,39992,null],[88,\"share-88\",6,7,39978,39980,null],[92,\"share-92\",7,3,39974,39984,null],[94,\"share-94\",8,1,40000,39988,null]]}" + }, + "record_count": 9, + "record_stream_len": 3096, + "record_stream_sha256": "ed085c4435070aab77acb77c5bddc5839c9ecae4c31d2a5fad10e7878062d4e1", + "spool_tail_len": 1245, + "spool_tail_sha256": "05651dc0dd4b4e151dec5ea8c5f1fa1c499ce513538b812ccdeff6b5e6f9e653", + "why": "snapshot input arrives in non-ascending share_seq order with non-contiguous seqs; the fold's sorted() is load-bearing and the output ordering is pinned" + }, + "wide-integers": { + "advance_stats": [], + "canonical_bytes_len": 1660, + "canonical_bytes_sha256": "50468ec22c9a2b4dff65693602bd128ea4753aa3fb7cf5156b79718a8dea8c69", + "canonical_digest": "50468ec22c9a2b4dff65693602bd128ea4753aa3fb7cf5156b79718a8dea8c69", + "input_sha256": "d861fbfa3e6df7f245d6565297cca3e6ac14a39b6c2b425960ef845e9466773f", + "pinned_literals": { + "canonical_bytes": "[{\"accepted_at_ms\":1754999999998,\"job_id\":\"job-1\",\"job_issued_at_ms\":1754999999997,\"miner_id\":\"miner-1\",\"network_difficulty\":340282366920938463463374607431768211455,\"ntime\":4294967295,\"order_key\":\"01:miner-1\",\"p2mr_program_hex\":\"0101010101010101010101010101010101010101010101010101010101010101\",\"share_difficulty\":170141183460469231731687303715884105731,\"share_id\":\"share-1\",\"share_seq\":1,\"template_height\":4611686018427387904},{\"accepted_at_ms\":1754999999999,\"job_id\":\"job-2147483648\",\"job_issued_at_ms\":1754999999998,\"miner_id\":\"miner-2\",\"network_difficulty\":18446744073709551617,\"ntime\":2147483648,\"order_key\":\"02:miner-2\",\"p2mr_program_hex\":\"0000000000000000000000000000000000000000000000000000000000000000\",\"share_difficulty\":18446744073709551617,\"share_id\":\"share-2147483648\",\"share_seq\":2147483648,\"template_height\":2147483649},{\"accepted_at_ms\":1754999999999,\"job_id\":\"job-4294967301\",\"job_issued_at_ms\":1754999999999,\"miner_id\":\"miner-9\",\"network_difficulty\":18446744073709551615,\"ntime\":0,\"order_key\":\"09:miner-9\",\"p2mr_program_hex\":\"0505050505050505050505050505050505050505050505050505050505050505\",\"share_difficulty\":9223372036854775808,\"share_id\":\"share-4294967301\",\"share_seq\":4294967301,\"template_height\":0},{\"accepted_at_ms\":1755000000000,\"job_id\":\"job-18446744073709551615\",\"job_issued_at_ms\":1755000000000,\"miner_id\":\"miner-4\",\"network_difficulty\":9007199254740993,\"ntime\":2147483649,\"order_key\":\"04:miner-4\",\"p2mr_program_hex\":\"ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff\",\"share_difficulty\":9007199254740993,\"share_id\":\"share-18446744073709551615\",\"share_seq\":18446744073709551615,\"template_height\":576460752304223487}]", + "input": { + "advances": [], + "name": "wide-integers", + "page_size": 512, + "schema": "qbit-prism-window-pipeline-parity-input/v1", + "snapshot": { + "anchor_job_issued_at_ms": 1755000000000, + "records": [ + { + "accepted_at_ms": 1754999999998, + "credit_policy": null, + "job_id": "job-1", + "job_issued_at_ms": 1754999999997, + "miner_id": "miner-1", + "network_difficulty": 340282366920938463463374607431768211455, + "ntime": 4294967295, + "order_key": "01:miner-1", + "p2mr_program_hex": "0101010101010101010101010101010101010101010101010101010101010101", + "share_difficulty": 170141183460469231731687303715884105731, + "share_id": "share-1", + "share_seq": 1, + "template_height": 4611686018427387904 + }, + { + "accepted_at_ms": 1754999999999, + "credit_policy": null, + "job_id": "job-2147483648", + "job_issued_at_ms": 1754999999998, + "miner_id": "miner-2", + "network_difficulty": 18446744073709551617, + "ntime": 2147483648, + "order_key": "02:miner-2", + "p2mr_program_hex": "0000000000000000000000000000000000000000000000000000000000000000", + "share_difficulty": 18446744073709551617, + "share_id": "share-2147483648", + "share_seq": 2147483648, + "template_height": 2147483649 + }, + { + "accepted_at_ms": 1754999999999, + "credit_policy": null, + "job_id": "job-4294967301", + "job_issued_at_ms": 1754999999999, + "miner_id": "miner-9", + "network_difficulty": 18446744073709551615, + "ntime": 0, + "order_key": "09:miner-9", + "p2mr_program_hex": "0505050505050505050505050505050505050505050505050505050505050505", + "share_difficulty": 9223372036854775808, + "share_id": "share-4294967301", + "share_seq": 4294967301, + "template_height": 0 + }, + { + "accepted_at_ms": 1755000000000, + "credit_policy": null, + "job_id": "job-18446744073709551615", + "job_issued_at_ms": 1755000000000, + "miner_id": "miner-4", + "network_difficulty": 9007199254740993, + "ntime": 2147483649, + "order_key": "04:miner-4", + "p2mr_program_hex": "ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff", + "share_difficulty": 9007199254740993, + "share_id": "share-18446744073709551615", + "share_seq": 18446744073709551615, + "template_height": 576460752304223487 + }, + { + "accepted_at_ms": 1755000000000, + "credit_policy": null, + "job_id": "job-7", + "job_issued_at_ms": 1755000000001, + "miner_id": "miner-7", + "network_difficulty": 1000007, + "ntime": 1700000007, + "order_key": "07:miner-7", + "p2mr_program_hex": "0707070707070707070707070707070707070707070707070707070707070707", + "share_difficulty": 1, + "share_id": "share-7", + "share_seq": 7, + "template_height": 800000 + } + ] + }, + "window_weight": 170141183460469231759357419826448433156 + }, + "record_jsons": [ + "{\"accepted_at_ms\":1754999999998,\"job_id\":\"job-1\",\"job_issued_at_ms\":1754999999997,\"miner_id\":\"miner-1\",\"network_difficulty\":340282366920938463463374607431768211455,\"ntime\":4294967295,\"order_key\":\"01:miner-1\",\"p2mr_program_hex\":\"0101010101010101010101010101010101010101010101010101010101010101\",\"share_difficulty\":170141183460469231731687303715884105731,\"share_id\":\"share-1\",\"share_seq\":1,\"template_height\":4611686018427387904}", + "{\"accepted_at_ms\":1754999999999,\"job_id\":\"job-2147483648\",\"job_issued_at_ms\":1754999999998,\"miner_id\":\"miner-2\",\"network_difficulty\":18446744073709551617,\"ntime\":2147483648,\"order_key\":\"02:miner-2\",\"p2mr_program_hex\":\"0000000000000000000000000000000000000000000000000000000000000000\",\"share_difficulty\":18446744073709551617,\"share_id\":\"share-2147483648\",\"share_seq\":2147483648,\"template_height\":2147483649}", + "{\"accepted_at_ms\":1754999999999,\"job_id\":\"job-4294967301\",\"job_issued_at_ms\":1754999999999,\"miner_id\":\"miner-9\",\"network_difficulty\":18446744073709551615,\"ntime\":0,\"order_key\":\"09:miner-9\",\"p2mr_program_hex\":\"0505050505050505050505050505050505050505050505050505050505050505\",\"share_difficulty\":9223372036854775808,\"share_id\":\"share-4294967301\",\"share_seq\":4294967301,\"template_height\":0}", + "{\"accepted_at_ms\":1755000000000,\"job_id\":\"job-18446744073709551615\",\"job_issued_at_ms\":1755000000000,\"miner_id\":\"miner-4\",\"network_difficulty\":9007199254740993,\"ntime\":2147483649,\"order_key\":\"04:miner-4\",\"p2mr_program_hex\":\"ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff\",\"share_difficulty\":9007199254740993,\"share_id\":\"share-18446744073709551615\",\"share_seq\":18446744073709551615,\"template_height\":576460752304223487}" + ], + "spool_tail": ",\"compact_share_identities\":[[\"miner-1\",\"01:miner-1\",\"0101010101010101010101010101010101010101010101010101010101010101\"],[\"miner-2\",\"02:miner-2\",\"0000000000000000000000000000000000000000000000000000000000000000\"],[\"miner-9\",\"09:miner-9\",\"0505050505050505050505050505050505050505050505050505050505050505\"],[\"miner-4\",\"04:miner-4\",\"ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff\"]],\"compact_shares\":[[1,\"share-1\",0,170141183460469231731687303715884105731,1754999999997,1754999999998,null],[2147483648,\"share-2147483648\",1,18446744073709551617,1754999999998,1754999999999,null],[4294967301,\"share-4294967301\",2,9223372036854775808,1754999999999,1754999999999,null],[18446744073709551615,\"share-18446744073709551615\",3,9007199254740993,1755000000000,1755000000000,null]]}" + }, + "record_count": 4, + "record_stream_len": 1658, + "record_stream_sha256": "609bd0feb644a22cea67903b6ad17e9900b39b06fbb753fd63d65e0eb752b639", + "spool_tail_len": 785, + "spool_tail_sha256": "450b2e639e7e75125ebaf35575cc45c35774a9ab19eed4bd2aaa6e6fe79c2c54", + "why": "every integer field above 2^32 at a realistic epoch-ms anchor, all inside the declared Rust widths: share_difficulty across 2^53+1, 2^63, 2^64+1 and 2^127+3, network_difficulty up to 2^128-1, template_height 2^31+1, 2^62 and near 2^59, ntime 2^31 to 2^32-1, share_seq up to 2^64-1, a zero template_height/ntime, and window_weight 2^127+2^64+2^63+4 so the retention cutoff depends on exact arithmetic above 2^64; pins full-digit rendering and non-wrapping, non-saturating, non-float accumulation, with full parity enforced on every backend" + } + }, + "default_page_size": 512, + "master_seed": "0xb17e5eed", + "regenerate_with": "python3 -m tests.window_pipeline_parity regenerate", + "schema": "qbit-prism-window-pipeline-parity-reference/v2" +} diff --git a/tests/perf/candidate_storage.py b/tests/perf/candidate_storage.py new file mode 100644 index 000000000..e62f5ae02 --- /dev/null +++ b/tests/perf/candidate_storage.py @@ -0,0 +1,162 @@ +"""Disposable native candidate storage/ownership probe, not a mined-block fixture. + +Runs actual prepare, append, header replay, hydration, exact retry and cleanup +with a real writer-session proof actor. Creates and drops a private schema. +Run serially in Docker; ARM timings do not establish a production root cause. +""" +from __future__ import annotations + +import argparse +from collections.abc import Sequence +import gc +import hashlib +import json +import platform +import sys +import threading +import time +import uuid +import weakref +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).resolve().parents[2])) + +import psycopg +from psycopg.conninfo import make_conninfo +from lab.prism.candidate_codec import prepare_candidate_intent +from lab.prism.share_ledger import PsqlShareLedger, WRITER_LEASE_HEARTBEAT_SESSION_PREFIX +from tests.perf.candidate_window_membership import _gc_callback, measure +from tests.prism_postgres_candidate_gate import intent, pending_share_for, share + + +class Shares(Sequence): + def __init__(self, count): + self.count = count + + def __len__(self): + return self.count + + def __getitem__(self, index): + if not 0 <= index < self.count: + raise IndexError(index) + row = share(index) + row["miner_id"] += "x" * 240 + return row + + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument("--database-url", required=True) + parser.add_argument("--shares", type=int, default=375000) + parser.add_argument("--candidates", type=int, default=3) + parser.add_argument("--generations", type=int, default=1) + parser.add_argument("--gc", choices=("on", "off"), default="off") + args = parser.parse_args() + if min(args.shares, args.candidates, args.generations) < 1: + parser.error("counts must be positive") + schema = "candidate_perf_" + uuid.uuid4().hex + with psycopg.connect(args.database_url, autocommit=True) as admin: + admin.execute(f'CREATE SCHEMA "{schema}"') + url = make_conninfo(args.database_url, options=f"-csearch_path={schema}") + ledger = None + stop = threading.Event() + proof_thread = None + proofs, wakes, errors = [], [], [] + was_enabled = gc.isenabled() + gc.callbacks.append(_gc_callback) + try: + ledger = PsqlShareLedger( + psql_command="psql", database_url=url, native_client_mode="native", + writer_id="candidate-storage-probe", initialize_schema=True, + writer_session_token=WRITER_LEASE_HEARTBEAT_SESSION_PREFIX + uuid.uuid4().hex, + ) + (gc.enable if args.gc == "on" else gc.disable)() + + def heartbeat(): + due = time.monotonic() + while not stop.is_set(): + started = time.monotonic() + wakes.append(max(0, started - due)) + try: + ledger.prove_writer_lease_guard_session() + proofs.append(time.monotonic() - started) + except Exception as exc: + errors.append(str(exc)) + due = time.monotonic() + 0.05 + stop.wait(max(0, due - time.monotonic())) + + proof_thread = threading.Thread(target=heartbeat) + proof_thread.start() + + def phase(name, function): + before = len(proofs), len(wakes), len(errors) + value = measure(name, function) + print(json.dumps({ + "phase": name, "proof_count": len(proofs) - before[0], + "max_proof_seconds": max(proofs[before[0]:], default=0), + "max_heartbeat_wake_lateness_seconds": max(wakes[before[1]:], default=0), + "proof_errors": errors[before[2]:], + "spool": ledger._candidate_spool.snapshot(), + }), flush=True) + return value + + print(json.dumps({"python": sys.version, "architecture": platform.machine(), + "shares": args.shares, "candidates": args.candidates, + "gc": args.gc, "generations": args.generations}), flush=True) + source = Shares(args.shares) + for index in range(args.candidates): + block_hash = hashlib.sha256(f"{schema}-{index}".encode()).hexdigest() + fields = intent(block_hash, 0, credit=False) + fields["shares_json"] = source + prepared = phase(f"prepare-{index}", lambda: prepare_candidate_intent(fields)) + print(json.dumps({"candidate": index, "bytes": prepared.manifest.byte_count, + "chunks": prepared.manifest.chunk_count, + "sha256": prepared.candidate_sha256}), flush=True) + pending = pending_share_for(fields) + phase(f"append-{index}", lambda: ledger.append_batch([(pending, prepared)])) + phase(f"exact-retry-{index}", lambda: ledger.append_batch([(pending, prepared)])) + del prepared, pending, fields + page = phase("replay-headers", lambda: ledger.pending_block_candidate_headers(limit=args.candidates)) + assert page.exhausted and len(page.rows) == args.candidates + for generation in range(args.generations): + for index, row in enumerate(page.rows): + hydrated = phase(f"hydrate-{generation}-{index}", lambda: ledger.hydrate_block_candidate_intent(row)) + sequence = hydrated["shares_json"] + def walk(): + count = 0 + for count, record in enumerate(sequence, 1): + assert record["share_id"] == source[count - 1]["share_id"] + assert count == args.shares + phase(f"walk-{generation}-{index}", walk) + observed = weakref.ref(hydrated.body) + phase(f"close-{generation}-{index}", hydrated.body.close) + del sequence, hydrated + assert observed() is None, "body survives refcount retirement" + assert ledger._candidate_spool.snapshot()["reserved_bytes"] == 0 + for row in page.rows: + phase("terminalize", lambda: ledger.mark_block_candidate_submitted(block_hash=row["block_hash"])) + + def drain(): + steps = 0 + while ledger.reap_retired_candidate_bodies()["body_id"] is not None: + steps += 1 + if steps > 100000: + raise AssertionError("janitor did not drain") + return steps + phase("janitor", drain) + assert not errors, errors + print(json.dumps({"result": "PASS", "proof_errors": errors}), flush=True) + finally: + stop.set() + if proof_thread is not None: + proof_thread.join() + gc.callbacks.remove(_gc_callback) + (gc.enable if was_enabled else gc.disable)() + if ledger is not None: + ledger.release_writer_lease() + ledger.close() + admin.execute(f'DROP SCHEMA "{schema}" CASCADE') + + +if __name__ == "__main__": + main() diff --git a/tests/perf/candidate_window_membership.py b/tests/perf/candidate_window_membership.py new file mode 100644 index 000000000..30c25a0e6 --- /dev/null +++ b/tests/perf/candidate_window_membership.py @@ -0,0 +1,223 @@ +"""Disposable PostgreSQL comparison of bounded candidate membership and the old aggregate. + +Example (after applying the ledger schema to a disposable database): + python tests/perf/candidate_window_membership.py --database-url "$TEST_DATABASE_URL" \ + --shares 375000 --gc off --prepare-disposable-fixture + +This replaces share rows in the supplied disposable database. It is a membership +and codec experiment, not a valid mined-block fixture or production timing proof. +The lease-guard actor is real; it records failures rather than terminating the +harness on a late wake. It reports scheduling delay separately from round-trip +duration; a short round trip alone does not imply a timely wake. Compare these +measurements with the integrated lease and +regtest gates before deployment. Run measurements serially in Docker Python3.14.7. +""" +import argparse +import gc +import json +import os +import platform +import sys +import threading +import time +import uuid +from pathlib import Path + +# Executable directly from a repository checkout. +sys.path.insert(0, str(Path(__file__).resolve().parents[2])) + +import psycopg +from lab.prism.share_ledger import PsqlShareLedger, WRITER_LEASE_HEARTBEAT_SESSION_PREFIX + +GC_PAUSES = {"active": {}, "max": {0: 0.0, 1: 0.0, 2: 0.0}, "count": {0: 0, 1: 0, 2: 0}} + + +def _gc_callback(phase: str, info: dict) -> None: + gen = int(info.get("generation", 0)) + now = time.monotonic() + if phase == "start": + GC_PAUSES["active"][gen] = now + return + start = GC_PAUSES["active"].pop(gen, None) + if start is None: + return + pause = now - start + GC_PAUSES["max"][gen] = max(GC_PAUSES["max"][gen], pause) + GC_PAUSES["count"][gen] += 1 + + +def _reset_gc_pauses() -> None: + GC_PAUSES["active"].clear() + for gen in (0, 1, 2): + GC_PAUSES["max"][gen] = 0.0 + GC_PAUSES["count"][gen] = 0 + + +def _rss() -> dict[str, int]: + out = {} + try: + with open("/proc/self/status") as handle: + for line in handle: + if line.startswith(("VmRSS:", "VmHWM:")): + key, value = line.split(":", 1) + out[key] = int(value.strip().split()[0]) // 1024 + except OSError: + pass + return out + + +def measure(label: str, function, *, note: str | None = None): + stop, ready = threading.Event(), threading.Event() + samples: list[float] = [] + + def observer() -> None: + ready.set() + while not stop.is_set(): + start = time.monotonic() + stop.wait(0.005) + samples.append(max(0.0, time.monotonic() - start - 0.005)) + + thread = threading.Thread(target=observer, name="observer") + thread.start() + ready.wait() + time.sleep(0.025) + _reset_gc_pauses() + before = [s["collections"] for s in gc.get_stats()] + start = time.monotonic() + try: + result = function() + elapsed = time.monotonic() - start + time.sleep(0.025) + finally: + stop.set() + thread.join() + record = dict( + operation=label, + wall_seconds=round(elapsed, 6), + max_observer_lateness_seconds=round(max(samples), 6), + wakes_over_100ms=sum(x >= 0.1 for x in samples), + wakes_over_400ms=sum(x >= 0.4 for x in samples), + gc_collection_delta=[s["collections"] - n for s, n in zip(gc.get_stats(), before)], + gc_max_pause_seconds={g: round(GC_PAUSES["max"][g], 6) for g in (0, 1, 2)}, + gc_pause_count={g: GC_PAUSES["count"][g] for g in (0, 1, 2)}, + rss_mb=_rss(), + ) + if note: + record["note"] = note + print(json.dumps(record), flush=True) + return result + + +def main() -> None: + parser = argparse.ArgumentParser() + parser.add_argument("--shares", type=int, default=375000) + parser.add_argument("--gc", choices=("off", "on"), default="off") + parser.add_argument("--database-url", required=True) + parser.add_argument("--prepare-disposable-fixture", action="store_true", required=True) + args = parser.parse_args() + if args.shares < 1: + parser.error("--shares must be positive") + + with psycopg.connect(args.database_url, autocommit=True) as connection: + connection.execute("TRUNCATE qbit_share_ledger CASCADE") + connection.execute(""" + INSERT INTO qbit_share_ledger ( + share_id, miner_id, payout_order_key, p2mr_program, + share_difficulty, network_difficulty, template_height, job_id, + job_issued_at, ntime, accepted_at, credit_policy, accepted, + writer_id, writer_epoch + ) SELECT + 'share-' || n, 'miner', 'miner', decode(repeat('aa', 32), 'hex'), + 1, 1000000, 1, 'job', to_timestamp(0), 0, to_timestamp(1), + NULL, true, 'fixture', 1 + FROM generate_series(0, %s) AS n + """, (args.shares - 1,)) + ledger = PsqlShareLedger( + psql_command="psql", database_url=args.database_url, + native_client_mode="native", writer_id="candidate-window-probe", + writer_session_token=WRITER_LEASE_HEARTBEAT_SESSION_PREFIX + uuid.uuid4().hex, + ) + + class Rows: + def __iter__(self): + for index in range(args.shares): + yield {"share_id": f"share-{index}"} + + stop = threading.Event() + heartbeats: list[float] = [] + heartbeat_wakes: list[float] = [] + failures: list[str] = [] + + def heartbeat() -> None: + due = time.monotonic() + while not stop.is_set(): + started = time.monotonic() + heartbeat_wakes.append(max(0.0, started - due)) + try: + ledger.prove_writer_lease_guard_session() + heartbeats.append(time.monotonic() - started) + except Exception as exc: + failures.append(str(exc)) + due = time.monotonic() + 0.05 + stop.wait(max(0.0, due - time.monotonic())) + + def report_proofs(phase: str, before: tuple[int, int, int]) -> None: + proof_start, wake_start, error_start = before + print(json.dumps({ + "phase": phase, + "proof_count": len(heartbeats) - proof_start, + "proof_errors": failures[error_start:], + "max_proof_seconds": max(heartbeats[proof_start:], default=0), + "max_heartbeat_wake_lateness_seconds": max(heartbeat_wakes[wake_start:], default=0), + }), flush=True) + + thread = threading.Thread(target=heartbeat, name="lease-proof") + print(json.dumps({ + "python": sys.version, "machine": platform.machine(), + "shares": args.shares, "gc": args.gc, + "psycopg": psycopg.__version__, + "libpq": psycopg.pq.version(), + "switch_interval_seconds": sys.getswitchinterval(), + "gc_thresholds": gc.get_threshold(), + "allocator": {name: os.environ.get(name) for name in ( + "MALLOC_ARENA_MAX", "PYTHONMALLOC", "GLIBC_TUNABLES", + )}, + }), flush=True) + gc.collect() + if args.gc == "off": + gc.disable() + gc.callbacks.append(_gc_callback) + thread.start() + try: + before_proofs = len(heartbeats), len(heartbeat_wakes), len(failures) + result = measure("bounded-window-membership", lambda: ledger.candidate_window_covers( + Rows(), anchor_job_issued_at_ms=2000, network_difficulty=1000000, + )) + assert result is True + report_proofs("bounded-window-membership", before_proofs) + + before_proofs = len(heartbeats), len(heartbeat_wakes), len(failures) + rows = measure("baseline-audit-window-aggregate", lambda: ledger.audit_share_window( + anchor_job_issued_at_ms=2000, network_difficulty=1000000, + )) + assert len(rows) == args.shares + report_proofs("baseline-audit-window-aggregate", before_proofs) + measure("baseline-reference-count-retirement", rows.clear) + finally: + stop.set() + thread.join() + gc.callbacks.remove(_gc_callback) + try: + ledger.release_writer_lease() + finally: + ledger.close() + print(json.dumps({ + "heartbeat_count": len(heartbeats), + "heartbeat_max_seconds": max(heartbeats, default=0), + "heartbeat_max_wake_lateness_seconds": max(heartbeat_wakes, default=0), + "heartbeat_errors": failures, + }), flush=True) + + +if __name__ == "__main__": + main() diff --git a/tests/perf/canonical_audit_retirement.py b/tests/perf/canonical_audit_retirement.py new file mode 100644 index 000000000..e544dfff9 --- /dev/null +++ b/tests/perf/canonical_audit_retirement.py @@ -0,0 +1,172 @@ +"""Isolated audit codec/ownership experiment; this is not a mined-block fixture. + +Run serially in Docker with the coordinator's Python and allocator settings. +Each artifact has 375,000 shares by default (about 232 MB). The ordinary +phases run without allocation tracing; --trace measures allocations separately +because tracing substantially changes elapsed time and scheduling. +""" +from __future__ import annotations + +import argparse +import gc +import hashlib +import json +import os +from pathlib import Path +import platform +import shutil +import sys +import tempfile +import threading +import time +import tracemalloc +import weakref + +sys.path.insert(0, str(Path(__file__).resolve().parents[2])) + +from lab.prism.audit_bundle_view import CanonicalAuditBundleView, iter_json_byte_chunks +from tests.test_prism_audit_bundle_view import make_share + + +def fixture(path: Path, count: int) -> tuple[int, str]: + digest = hashlib.sha256() + with path.open("wb") as output: + def emit(value: str) -> None: + encoded = value.encode("utf-8") + output.write(encoded) + digest.update(encoded) + + emit('{"schema":"qbit.prism.audit-bundle.v1","shares":[') + for index in range(count): + row = make_share(index) + row["share_id"] += "x" * 147 + row["credit_policy"] = None + emit(("," if index else "") + json.dumps(row, separators=(",", ":"))) + emit('],"found_block":{"block_height":10},"prior_balances":[],') + emit('"reward_manifest":{"shares":[') + for index in range(count): + emit(("," if index else "") + json.dumps( + {"share_seq": index + 1, "counted_difficulty": 5}, separators=(",", ":"), + )) + emit('],"entitlements":[]},"payout_policy_manifest":{"accounts":[]},') + emit('"signed_coinbase_manifest":{"manifest":{"coinbase_tx_hex":"00"}}}') + return path.stat().st_size, digest.hexdigest() + + +def memory() -> dict[str, int]: + result = {} + for line in Path("/proc/self/status").read_text().splitlines(): + key, _, value = line.partition(":") + if key in ("VmRSS", "VmHWM"): + result[key + "_bytes"] = int(value.strip().split()[0]) * 1024 + return result + + +def measure(label: str, operation, *, trace: bool): + stop, ready = threading.Event(), threading.Event() + wakes, pauses = [], [] + gc_started = {} + + def gc_event(phase, info): + generation = info["generation"] + if phase == "start": + gc_started[generation] = time.monotonic() + else: + started = gc_started.pop(generation, None) + if started is not None: + pauses.append((generation, time.monotonic() - started)) + + def observer(): + ready.set() + while not stop.is_set(): + started = time.monotonic() + stop.wait(0.005) + wakes.append(max(0.0, time.monotonic() - started - 0.005)) + + thread = threading.Thread(target=observer) + thread.start() + ready.wait() + time.sleep(0.025) + gc.callbacks.append(gc_event) + if trace: + tracemalloc.start() + try: + started = time.monotonic() + result = operation() + elapsed = time.monotonic() - started + peak = tracemalloc.get_traced_memory()[1] if trace else None + time.sleep(0.025) + finally: + stop.set() + thread.join() + gc.callbacks.remove(gc_event) + if trace: + tracemalloc.stop() + ordered = sorted(wakes) + print(json.dumps({ + "phase": label, "seconds": elapsed, + "observer_max_seconds": max(wakes, default=0), + "observer_p99_seconds": ordered[int((len(ordered) - 1) * 0.99)], + "wakes_over_100ms": sum(value >= 0.1 for value in wakes), + "gc_pause_count": len(pauses), + "gc_max_pause_seconds": max((value for _, value in pauses), default=0), + "peak_traced_bytes": peak, **memory(), + }), flush=True) + return result + + +def main() -> None: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--shares", type=int, default=375000) + parser.add_argument("--candidates", type=int, default=1) + parser.add_argument("--gc", choices=("off", "on"), default="off") + parser.add_argument("--trace", action="store_true") + parser.add_argument("--baseline", action="store_true") + args = parser.parse_args() + if args.shares < 1 or args.candidates < 1: + parser.error("share and candidate counts must be positive") + print(json.dumps({ + "python": sys.version, "machine": platform.machine(), + "gc": args.gc, "trace": args.trace, "candidates": args.candidates, + "switch_interval": sys.getswitchinterval(), "gc_thresholds": gc.get_threshold(), + "allocator": {name: os.environ.get(name) for name in ( + "MALLOC_ARENA_MAX", "PYTHONMALLOC", "GLIBC_TUNABLES", + )}, + }), flush=True) + with tempfile.TemporaryDirectory(prefix="audit-codec-probe-") as directory: + path = Path(directory) / "audit.json" + size, digest = fixture(path, args.shares) + print(json.dumps({"shares": args.shares, "artifact_bytes": size, "sha256": digest}), flush=True) + gc.collect() + if args.gc == "off": + gc.disable() + for index in range(args.candidates): + owner = {} + owner["view"] = measure(f"scan-{index}", lambda: CanonicalAuditBundleView.scan_path(path), trace=args.trace) + assert owner["view"].sha256_hex == digest + view_ref = weakref.ref(owner["view"]) + source_ref = weakref.ref(owner["view"].source) + count = measure(f"lazy-walk-{index}", lambda: sum(1 for _ in owner["view"]["shares"]), trace=args.trace) + assert count == args.shares + encoded = measure(f"encode-{index}", lambda: sum(len(chunk) for chunk in iter_json_byte_chunks(owner["view"])), trace=args.trace) + assert encoded == size + measure(f"retire-{index}", lambda: owner.pop("view").close(), trace=args.trace) + assert view_ref() is None and source_ref() is None + if args.baseline: + aggregate = Path(directory) / "aggregate.json" + with aggregate.open("wb") as destination: + destination.write(b"[") + for index in range(args.candidates): + if index: + destination.write(b",") + with path.open("rb") as source: + shutil.copyfileobj(source, destination, length=256 * 1024) + destination.write(b"]") + rows = measure("whole-decode-control", lambda: json.loads(aggregate.read_bytes()), trace=args.trace) + assert all(len(row["shares"]) == args.shares for row in rows) + measure("whole-reference-count-retirement", rows.clear, trace=args.trace) + measure("cyclic-gc-control", gc.collect, trace=args.trace) + + +if __name__ == "__main__": + main() diff --git a/tests/perf/serialize_block_share_cost.md b/tests/perf/serialize_block_share_cost.md new file mode 100644 index 000000000..1ff59ac9c --- /dev/null +++ b/tests/perf/serialize_block_share_cost.md @@ -0,0 +1,493 @@ +# Per-share CPU cost of `serialize_block` across template sizes + +Closes the "largest open number" named at the end of section 1 of the issue #143 findings +comment: `serialize_block` is O(block bytes) per accepted share, and the composite it measured +came from a 0-transaction template. + +Driver: `tests/perf/serialize_block_share_cost.py`. No database, no network, no coordinator, +standard library only. `python3 tests/perf/serialize_block_share_cost.py` prints these tables; +`--json` emits the same data structured. + +**Sections 1-6 measure the code as it stood at `413f51e`**, before the fix that shipped with +this file; no `lab/` change was in the tree while they were taken. **Section 7 is the after-fix +delta**, measured against the same driver on the same machine. Read 1-6 as the diagnosis and 7 +as the outcome. + +**Headline.** The block-serialization term is linear in block bytes at **2.43 µs of interpreter +CPU per KB of block**, with an essentially zero intercept (R² = 0.99996 over eight points from +158 B to the 2,000,000 B consensus ceiling). At a 1 MB template an accepted share costs +**~2,470 µs** of interpreter CPU, of which **~2,440 µs (98.8%)** is block serialization, and +single-core share capacity is **~405 shares/s**. At the 2 MB consensus ceiling it is +**~5,096 µs / ~5,090 µs (99.9%) / ~196 shares/s**. + +--- + +## 1. Machine, clock, and anchors + +### Machine + +| | | +|---|---| +| CPU | Apple M5 Max, 18 logical cores (12 performance / 6 efficiency) | +| Platform | macOS 26.4, arm64 | +| Python | 3.14.3 (main, Feb 3 2026, 15:32:20) [Clang 17.0.0 (clang-1700.6.3.2)], CPython | +| GIL | standard build (`Py_GIL_DISABLED` = False), GIL enabled at runtime | +| Idle? | **No.** Load average ranged 5.2–7.7 across the ten runs (the driver itself contributes ~1.0). See §6. | + +### Clock + +**`time.thread_time_ns()`** is the headline clock — interpreter CPU, not wall clock. Reasons: + +- the driver is strictly single-threaded, so per-thread CPU *is* the interpreter CPU this path + burns; +- it excludes time the OS deschedules us, which matters because the host was not idle and + `perf_counter` would charge that to the measurement; +- on this platform its resolution is 42 ns against `process_time`'s 1 µs + (`clock_gettime(CLOCK_THREAD_CPUTIME_ID)` vs `clock_gettime(CLOCK_PROCESS_CPUTIME_ID)`). + +`process_time` and `perf_counter` were captured over the same batches and **do not diverge**: +at the 0-tx point the composite is 9.22 µs thread CPU / 9.22 µs process CPU / 9.25 µs wall; at +2 MB it is 5,093.1 / 5,092.8 / 5,098.5 µs. Wall exceeds thread CPU by 0.3% and 0.1% +respectively, so there is no hidden kernel or I/O time even at multi-megabyte allocation +sizes, and every number below can be read as either. + +### Statistic + +Min-of-7 repetitions, matching the findings comment. Each repetition is an auto-sized batch of +calls (target 50 ms per repetition) reported per call, so sub-microsecond primitives stay clear +of clock granularity. The driver's default is min-of-7; every headline number below is the +**median of 7 independent default runs**. A **control** column reports the median of 3 runs at +`--reps 25 --min-batch-seconds 0.02`, used only to show that min-of-7 is not biased — it is +not, the two agree within the run-to-run spread (§6). + +### Anchors + +| anchor | published (#143) | this machine, min-of-7 | run range | control (min-of-25) | ratio mine/published | +|---|---|---|---|---|---| +| primitives floor, per accepted share | 12.4 µs | **10.85 µs** | 10.50–11.65 | 10.41 | **0.875** | +| 0-tx-template composite `assemble_submission` | 9.0 µs | **9.22 µs** | 8.93–9.87 | 8.92 | **1.025** | +| composite ÷ primitives inflation ratio | 1.266 (implied by 15.7 ÷ 12.4) | 1.095 | 1.059–1.129 | 1.108 | 0.865 | + +The 0-tx composite anchor — the one the findings comment specifies unambiguously — reproduces +at **1.02× published**. This machine is, for this path, the same speed as the machine the +findings comment ran on, so its absolute numbers carry over almost unchanged. + +The primitives floor anchor is **a reconstruction, not a replay**: the findings comment reports +"12.4 µs (primitives, min-of-7)" but does not enumerate which primitives it summed. The set +below was reconstructed from the accept path and lands 12.5% under the published figure, which +is consistent with the original set including per-share work this one omits (§6). + +Primitive breakdown at the 0-transaction baseline, min-of-7, median of 7 runs: + +| primitive | group | µs | note | +|---|---|---|---| +| `validate_hex(extranonce2)` | assemble | 0.114 | | +| `apply_version_bits` | assemble | 0.161 | no version rolling on this share | +| `list(job.merkle_branch)` | assemble | 0.034 | 0 branch levels | +| `assemble_coinbase` | assemble | 0.582 | | +| `compute_merkle_root_from_branch_hex` | assemble | 0.466 | 0 fold levels | +| `serialize_header_from_stratum_fields` | assemble | 3.353 | largest single primitive on the path | +| `full_coinbase_hex` concat | assemble | 0.069 | | +| `strip_witness_transaction(coinbase)` + compare | assemble | 1.267 | | +| `header_hash_int` | assemble | 0.495 | | +| `header_hash_hex` | assemble | 0.482 | | +| `coinbase_without_witness.hex()` | assemble | 0.049 | | +| `header.hex()` | assemble | 0.052 | | +| `serialize_block(...).hex()` | assemble | 0.653 | **0 tx, 158 block bytes — the term, at the baseline** | +| `DirectQbitSubmission(...)` | assemble | 0.640 | | +| `json.loads(mining.submit line)` | accept_path | 0.669 | | +| `parse_submit_request` | accept_path | 0.668 | | +| `validate_submit_request` | accept_path | 0.079 | | +| `RecentShareIndex.reserve` | accept_path | 0.332 | fresh key per call, includes its own lock | +| `json.dumps(submit ack)` | accept_path | 0.712 | | +| **sum, assemble group** | | **8.43** | | +| **sum, accept-path group** | | **2.45** | | +| **primitives floor** | | **10.85** | | +| *(memo)* harness floor, `lambda: None` | harness | 0.014 | included in every row above | + +--- + +## 2. Axis 1 — total block bytes + +The ceiling is tree-derived, not guessed: `crates/qbit-pool-builder/src/lib.rs:16-17` sets +`MAX_BLOCK_WEIGHT = 2_000_000` and comments it as qbit `consensus.h` with +`WITNESS_SCALE_FACTOR = 1`, so weight units are bytes; `compose.yaml:114` pins +`QBIT_EXPECTED_MAX_BLOCK_WEIGHT: 2000000`; and `docker/ckpool/qbit-ckpool-preflight.py:494-544` +hard-fails unless that expectation and live `getblocktemplate.weightlimit` are both exactly +2000000. Max block = **2,000,000 bytes**. The 0-transaction baseline is the repo's own +(`lab/prism/job_build_benchmark.py:208-217`, `"transactions": []`). + +**How each point was produced.** Real `DirectQbitStratumJob`s built through +`direct_stratum.make_job_from_builder_manifest` (route 1). Transactions are synthetic +1-in/1-out non-witness transactions, uniform size, padded in the output scriptPubKey, sized so +the total lands on the target byte count exactly; each is structurally valid and round-trips +through `strip_witness_transaction`, which is what `merkle_branch_for_coinbase` runs on every +one of them at job build. Route 1 was chosen over a hand-frozen job because merkle-branch depth +is itself a per-share cost inside `assemble_submission` — a frozen job with an empty branch +would understate the composite at large transaction counts. The coinbase is the repo +benchmark's synthetic one, 77 bytes. Block bytes = 80 header + transaction-count compact size + +coinbase + transactions; the "block bytes" column is the count actually produced, verified +against `len(submission.block_hex) // 2`. + +Transaction size is held at ~499 B on this axis (a plausible average), so transaction count +scales with block bytes; Axis 2 separates the two. + +| block bytes | tx count | tx size | merkle levels | composite `assemble_submission` µs/share | `serialize_block(...).hex()` µs/share | term % of composite | +|---:|---:|---:|---:|---:|---:|---:| +| 158 | 0 | — | 0 | 9.2 | 0.7 | 7.1% | +| 50,000 | 100 | 498 B | 7 | 137.2 | 122.0 | 88.9% | +| 100,000 | 200 | 499 B | 8 | 258.2 | 243.6 | 94.3% | +| 250,000 | 500 | 499 B | 9 | 623.0 | 615.7 | 98.8% | +| 500,000 | 1,000 | 499 B | 10 | 1,242.3 | 1,213.9 | 97.7% | +| 1,000,000 | 2,000 | 499 B | 11 | 2,469.2 | 2,438.5 | 98.8% | +| 1,500,000 | 3,000 | 499 B | 12 | 3,802.5 | 3,731.6 | 98.1% | +| **2,000,000** (ceiling) | 4,000 | 499 B | 12 | **5,093.1** | **5,090.0** | **99.9%** | + +Control run (min-of-25, median of 3), for the same points: + +| block bytes | composite µs | term µs | composite − term µs | +|---:|---:|---:|---:| +| 158 | 8.9 | 0.6 | 8.3 | +| 50,000 | 134.2 | 119.7 | 14.5 | +| 100,000 | 255.3 | 240.2 | 15.0 | +| 250,000 | 618.4 | 602.0 | 16.4 | +| 500,000 | 1,228.3 | 1,210.3 | 18.0 | +| 1,000,000 | 2,444.7 | 2,421.4 | 23.4 | +| 1,500,000 | 3,676.7 | 3,668.9 | 7.9 | +| 2,000,000 | 4,889.5 | 4,848.2 | 41.3 | + +**The law.** Least squares of the term against block bytes: **2.4318 µs/KB**, intercept +−2.6 µs, **R² = 0.99996** (control). On the min-of-7 medians: 2.5254 µs/KB, intercept +−22.7 µs, R² = 0.9996. The curve is clean; no extra points were needed. + +**Everything that is not serialization is a small constant.** The `composite − term` column is +the rest of `assemble_submission`: 8.3 µs at the 0-tx baseline, rising to ~18 µs at 10 merkle +levels — roughly **0.9 µs per merkle branch level** (each level is a `validate_hash_hex` + +`bytes.fromhex` + 64-byte `double_sha256`). The 1.5 MB (7.9 µs) and 2 MB (41.3 µs) entries are +noise: at those magnitudes a 1% error on either measurement is ±40 µs, which swamps a ~20 µs +quantity. Read that column as "8–20 µs, growing logarithmically with transaction count", not as +the printed values. + +**Split of the term.** `serialize_block` (produces the bytes) vs the caller's `.hex()` +re-encode, control run: + +| block bytes | `serialize_block` µs | `.hex()` re-encode µs | sum | term measured | +|---:|---:|---:|---:|---:| +| 158 | 0.5 | 0.1 | 0.6 | 0.6 | +| 100,000 | 193.7 | 46.1 | 239.8 | 240.2 | +| 500,000 | 978.4 | 229.9 | 1,208.3 | 1,210.3 | +| 1,000,000 | 1,953.9 | 459.6 | 2,413.5 | 2,421.4 | +| 2,000,000 | 3,925.0 | 922.3 | 4,847.3 | 4,848.2 | + +The caller's `.hex()` is a flat ~19% of the term at every size. + +--- + +## 3. Axis 2 — transaction count at fixed total bytes + +Total block bytes held at exactly 1,000,000 while transaction count varies 500×. Per-byte work +is therefore constant across the rows and the difference is the per-transaction constant. + +| tx count | block bytes | tx size | merkle levels | composite µs/share | term µs/share | term % of composite | +|---:|---:|---:|---:|---:|---:|---:| +| 10 | 1,000,000 | 99,984 B | 4 | 1,793.8 | 1,786.6 | 99.6% | +| 100 | 1,000,000 | 9,998 B | 7 | 1,881.7 | 1,880.8 | 100.0% | +| 500 | 1,000,000 | 1,999 B | 9 | 2,040.8 | 2,067.9 | 101.3% | +| 1,000 | 1,000,000 | 999 B | 10 | 2,265.4 | 2,213.5 | 97.7% | +| 2,000 | 1,000,000 | 499 B | 11 | 2,566.0 | 2,526.7 | 98.5% | +| 5,000 | 1,000,000 | 199 B | 13 | 3,106.3 | 3,078.9 | 99.1% | + +(Percentages slightly over 100% are measurement noise — the composite and the term are timed in +separate batches, and at ~2 ms a 1–2% error exceeds the ~20 µs that genuinely separates them.) + +**Per-transaction constant: ~0.25 µs/tx.** + +| derivation | min-of-7 | control (min-of-25) | +|---|---:|---:| +| least-squares slope over all six points | 0.250 µs/tx | 0.240 µs/tx | +| endpoint slope, 10 → 5,000 tx | 0.261 µs/tx | 0.248 µs/tx | +| R² of the linear fit | — | 0.95–0.97 | + +R² of 0.95–0.97 rather than 0.999 reflects that the relationship is not perfectly linear — +part of the growth is the merkle branch deepening from 4 to 13 levels, which is logarithmic in +count and sits in the composite, not the term. Take **0.25 µs/tx ± 0.02** as the constant. + +At 4,000 transactions (a 2 MB block at 500 B/tx) that constant is ~1.0 ms/share — **20% of the +5.1 ms total** — and it is the part that a byte-count-only model would miss. + +--- + +## 4. Decomposition of the term + +`serialize_block` calls `stratum_codec.validate_hex` on the coinbase and on every transaction. +`validate_hex` builds an f-string field name, length-checks, runs `bytes.fromhex` **and +discards the result**, then returns `value.lower()` — a full fresh copy of the hex string, and +unconditionally so: `str.lower()` on already-lowercase ASCII still allocates (verified: +`s.lower() is s` → `False`). The caller then `bytes.fromhex`-decodes that copy for real, joins, +and `.hex()`-re-encodes the whole block. + +All rows are control-run (min-of-25) medians. Rows sum to the measured term. + +**At the 2,000,000-byte ceiling (4,000 tx), term = 4,848.2 µs** + +| part | µs | % of term | +|---|---:|---:| +| (a) `validate_hex`'s `bytes.fromhex` probe (decoded, discarded) | 1,009.3 | 20.8% | +| (a′) `validate_hex`'s `value.lower()` copy | 1,098.4 | 22.7% | +| (a″) `validate_hex` call + per-tx f-string field name | 544.1 | 11.2% | +| (b) the real `bytes.fromhex` decode + list build | 1,061.3 | 21.9% | +| (c) `b"".join` concatenation | 58.2 | 1.2% | +| (d) the caller's `.hex()` re-encode | 932.1 | 19.2% | +| residual (generator step, `extend`, `compact_size`, header concat) | 144.8 | 3.0% | + +**At 1,000,000 bytes (2,000 tx), term = 2,421.4 µs** + +| part | µs | % of term | +|---|---:|---:| +| (a) `validate_hex`'s `bytes.fromhex` probe | 501.6 | 20.7% | +| (a′) `validate_hex`'s `value.lower()` copy | 548.8 | 22.7% | +| (a″) `validate_hex` call + f-string | 270.9 | 11.2% | +| (b) the real `bytes.fromhex` decode + list build | 517.5 | 21.4% | +| (c) `b"".join` | 28.5 | 1.2% | +| (d) the caller's `.hex()` re-encode | 461.0 | 19.0% | +| residual | 93.1 | 3.8% | + +**At the 158-byte baseline (0 tx), term = 0.63 µs** + +| part | µs | % of term | +|---|---:|---:| +| (a) probe | 0.077 | 12.2% | +| (a′) `.lower()` copy | 0.087 | 13.8% | +| (a″) call + f-string | 0.111 | 17.6% | +| (b) real decode + list build | 0.091 | 14.5% | +| (c) `b"".join` | 0.022 | 3.5% | +| (d) `.hex()` re-encode | 0.060 | 9.6% | +| residual | 0.182 | 28.9% | + +At the baseline the term is call overhead; at MB scale it is buffer work, in the proportions +above. + +### Counterfactual floors + +Measured on the same material, in the same run. These are *floors for alternative code shapes*, +recorded so the reviewer can see what is reducible and what is memcpy. **No recommendation is +implied or made here.** + +| variant | 2 MB | 1 MB | 158 B | +|---|---:|---:|---:| +| as shipped (`serialize_block(...).hex()`) | 4,848.2 µs (100%) | 2,421.4 µs (100%) | 0.63 µs (100%) | +| A: one decode, no `validate_hex`, bytes returned, no caller re-encode | 1,154.2 µs (**23.8%**) | 564.6 µs (**23.3%**) | 0.133 µs (21.1%) | +| B: transactions already held as bytes, bytes returned | 86.8 µs (**1.8%**) | 41.5 µs (**1.7%**) | 0.057 µs (9.0%) | +| memo: `validate_hex` full calls (a + a′ + a″) | 2,651.8 µs (54.7%) | 1,321.3 µs (54.6%) | 0.275 µs (43.6%) | + +Reading of those three numbers, as measurement: `validate_hex` accounts for **55% of the term** +at MB scale. The irreducible memcpy — joining bytes that already exist — is **1.8%**. Between +them sits the hex representation itself: keeping hex inputs but decoding once and returning +bytes still costs ~24% of the current term, because two full hex↔bytes conversions (one in, one +out) remain. The cost is not "irreducibly the memcpy"; it is dominated by redundant validation +and by hex being the carrying format. + +--- + +## 5. Capacity restatement + +The findings comment's single-core capacity was **64k–81k shares/s** at the 0-tx point, against +the deployed design point of ~4,096 connections at `PRISM_STRATUM_VARDIFF_TARGET_SECONDS=15`, +i.e. **~273 shares/s** steady state. + +"Accept-path µs/share" below is this machine's composite at that template size plus the +measured accept-path primitives (2.45 µs) inflated by the composite/primitives ratio (1.095) — +the same construction the findings comment used to turn 12.4 µs into 15.7 µs. It gives 11.9 µs +at the 0-tx point, against the comment's 15.7 µs. "Published-scale shares/s" rescales the +comment's own 64k–81k band by the relative collapse measured here, so it can be dropped +straight into that report without a machine correction. + +| block bytes | tx count | accept-path µs/share (this machine) | shares/s (this machine) | published-scale shares/s | % of one core at ~273 shares/s | +|---:|---:|---:|---:|---:|---:| +| 158 (0-tx baseline) | 0 | 11.9 | 84,015 | 64,000–81,000 | **0.33%** | +| 50,000 | 100 | 139.9 | 7,147 | 5,444–6,891 | 3.82% | +| 100,000 | 200 | 260.9 | 3,833 | 2,920–3,695 | 7.12% | +| 250,000 | 500 | 625.7 | 1,598 | 1,218–1,541 | 17.09% | +| 500,000 | 1,000 | 1,245.0 | 803 | 612–774 | 34.00% | +| 1,000,000 | 2,000 | 2,471.9 | 405 | 308–390 | **67.50%** | +| 1,500,000 | 3,000 | 3,805.2 | 263 | 200–253 | 103.91% | +| **2,000,000** (ceiling) | 4,000 | 5,095.7 | 196 | 149–189 | **139.15%** | + +The design point's ~273 shares/s is a **non-problem at 0 transactions** (0.33% of one core, +which is what "230–300× headroom" meant) and is **at or past one core somewhere between 1.0 and +1.5 MB of template**: 67.5% of a core at 1 MB, 104% at 1.5 MB, 139% at the 2 MB ceiling. The +crossover — where the accept path alone needs a full core to keep up with the design point — is +at **~1.45 MB of block** on this machine (3,662 µs/share), interpolated between the measured +1.0 MB and 1.5 MB points. + +--- + +## 6. Reading, noise, and limits + +### What the numbers show + +1. **The term is linear in block bytes at 2.43 µs/KB with a zero intercept.** One number + predicts the whole curve. Multiply template size in KB by 2.43 to get microseconds of + interpreter CPU per accepted share, on this machine, at this Python. + +2. **Above ~100 KB of template, the accepted-share path *is* block serialization.** The term is + 89% of the composite at 50 KB and ≥94% everywhere above 100 KB. Everything the findings + comment measured at the 0-tx point — the header assembly, the two `double_sha256`s, the + coinbase strip, the JSON — collapses to a rounding error: a fixed 8–20 µs against a term + that reaches 5,090 µs. + +3. **The 0-tx anchor understated the real per-share cost by two to three orders of magnitude.** + On this machine's own scale, 9.22 µs → 2,469 µs at 1 MB is 268×; → 5,093 µs at 2 MB is + 552×. The findings comment's "~230–300× headroom over the design point" was computed at + the 0-tx point, and the headroom it + describes is consumed almost exactly by moving to a full template. + +4. **There are two independent scaling terms, and both matter.** Per byte: 2.43 µs/KB. Per + transaction: 0.25 µs/tx. At 4,000 transactions the per-transaction term alone is ~1.0 ms, a + fifth of the 2 MB total. A model keyed only on block size would miss it. + +5. **Fifty-five percent of the term is `validate_hex`**, split roughly evenly between a decode + that is thrown away and a `.lower()` copy that allocates even when the string is already + lowercase. The irreducible join of already-decoded bytes is 1.8%. + +6. **At the deployed design point the term crosses one core inside the consensus range.** ~273 + shares/s costs 0.33% of a core at 0 tx and 139% at the 2 MB ceiling; the crossover is at + ~1.45 MB. + +### What is noisy + +- **The host was not idle.** Load average 5.2–7.7 across all ten runs (~1.0 of which is the + driver). `thread_time` excludes descheduled time, but it cannot exclude DVFS or + memory-bandwidth contention from other processes — CPU time is still charged while running + slower. This is the dominant error source. +- **Run-to-run spread at min-of-7 is 4–12%** at MB scale, 2–7% below 100 KB. Seven independent + runs were taken and the median reported for that reason. +- **`composite − term` is not resolvable at MB scale.** It is a genuine 8–20 µs quantity sitting + inside two ~5 ms measurements taken in separate batches; individual runs produce values from + −176 µs to +49 µs at 2 MB. Only the control run (min-of-25) resolves it consistently + positive. Do not read that column point by point. +- **Min-of-7 is not biased**, checked directly: three control runs at min-of-25 with 20 ms + batches land within 1–4% of the min-of-7 medians at every point, always on the low side, as + a longer min-search should. An independent stopwatch cross-check (arithmetic mean, not min, + over 200 calls at 1 MB, outside the harness) gives 2,525 µs against the harness's 2,438–2,469 + µs — a mean sitting ~3% above a min is exactly right. + +### What I could not measure, and why + +- **The primitives-floor anchor is a reconstruction.** The findings comment does not enumerate + the primitives behind its 12.4 µs. The set here — 14 primitives inside `assemble_submission` + plus 5 on the accept path around it — sums to 10.85 µs, 12.5% under published. Plausible + omissions: the ten process-global lock acquisitions the comment counts, vardiff accounting, + `pending_share_from_submission` and its ledger row, and socket read/write framing. The + precisely-specified anchor (the 0-tx composite) reproduces at 1.02×, so I would treat the + machine as matched and this gap as a difference in what was counted, not in how fast the + machine is. Every sweep number in this report is from this machine in the same set of runs, + so shapes and deltas are internally consistent regardless. +- **Coinbase size was not swept.** All points use the repo benchmark's 77-byte synthetic + coinbase. A production PRISM coinbase with many payout outputs is larger, and it enters the + term exactly as a transaction does (so, 2.43 µs/KB) *and* is parsed once more per share by + `strip_witness_transaction` in pure Python — a term measured here only at 77 bytes + (1.27 µs). For a KB-scale coinbase that second cost is unmeasured and is not covered by the + 2.43 µs/KB law. The driver has a `coinbase_outputs` parameter that would support the sweep; + it was not run. +- **Real transaction size distribution.** Transactions here are uniform-size, 1-in/1-out, + witness-free. `serialize_block` cost depends only on hex length and transaction count, so + this is faithful **for the term under measurement**; it is not faithful for anything that + parses transaction structure. Real templates have a size distribution and real witness data; + apply 2.43 µs/KB to actual bytes and 0.25 µs/tx to actual count rather than assuming uniform + transactions. +- **Threading behaviour at these buffer sizes — the most important gap.** The findings comment + established that this path is GIL-bound *at 0-tx buffer sizes*, and separately found the GIL + release threshold brackets 1–2 KiB with a 1 MiB positive control reaching 7.89 cores across 8 + threads. Every buffer in this sweep above the baseline is far past that threshold. Whether + CPython's `bytes.fromhex`, `str.lower`, `bytes.hex` and `b"".join` release the GIL at + megabyte sizes decides whether this term is 5 ms of *serialized* interpreter time per share + or 5 ms of work that partly parallelises — and it could cut either way for the sharding + argument in #143. I did not measure it: the brief scopes this to single-threaded per-share + cost, and I am not willing to assert a GIL-release claim from reading alone. **It should be + measured before these numbers are used to argue anything about concurrency.** +- **Downstream cost of `block_hex`.** `DirectQbitSubmission` carries the full hex block string + for **every** accepted share, block-worthy or not — 4 MB of live string at a 2 MB template. + I measured the CPU to produce it, not the allocator/GC pressure or peak RSS from holding it + across the rest of the accept path. That is a separate and unmeasured cost. +- **Everything outside `assemble_submission` at large template sizes.** The accept-path + primitives were measured at the 0-tx baseline only. They are template-size-independent as far + as I can tell from the code, but I did not re-measure them per point; the capacity table adds + a constant 2.68 µs at every size on that assumption, which is stated rather than verified. + +### Re-running + +``` +python3 tests/perf/serialize_block_share_cost.py # tables +python3 tests/perf/serialize_block_share_cost.py --json # same data, structured +python3 tests/perf/serialize_block_share_cost.py --reps 25 --min-batch-seconds 0.02 # control +``` + +One default run takes ~55 s. For an after-fix delta, compare the Axis 1 `serialize_term_us` +column and the §4 decomposition; the `--json` output carries `thread_cpu_us`, `process_cpu_us` +and `wall_us` for every measurement plus the batch size and repetition count behind it. + +**No verdict on whether to change anything is offered here** — that call belongs to the +reviewer. + +--- + +## 7. After the fix — measured delta + +`assemble_submission` now serializes the block only when `block_pass` is set, so an ordinary +accepted share never builds it. Same driver, same machine, same session; three `before` and three +`after` runs taken **alternating** so any machine drift falls on both sides equally. Medians of +three. `before` is `413f51e` unmodified; `after` is that tree plus the one-line gate. + +The driver's jobs use `bits = "1b00ffff"` (difficulty 65,536), so its submissions are ordinary +shares — `block_pass` is false — which is exactly the path under test. + +| block bytes | tx | composite before µs | composite after µs | speedup | +|---:|---:|---:|---:|---:| +| 158 | 0 | 9.0 | 8.3 | **1×** | +| 50,000 | 100 | 134.5 | 13.1 | **10×** | +| 100,000 | 200 | 255.6 | 13.8 | **19×** | +| 250,000 | 500 | 620.8 | 14.5 | **43×** | +| 500,000 | 1,000 | 1,235.2 | 15.1 | **82×** | +| 1,000,000 | 2,000 | 2,447.4 | 15.9 | **154×** | +| 1,500,000 | 3,000 | 3,677.0 | 16.5 | **222×** | +| 2,000,000 | 4,000 | 4,888.6 | 16.5 | **296×** | + +**The after column is flat.** It rises 8.3 → 16.5 µs across a 12,600× range of block size, and +that residual is not serialization: it is the per-share merkle-branch fold, ~0.9 µs per level, +deepening from 0 to 12 levels as transaction count grows. Block bytes no longer enter the +share path at all. + +Single-core share capacity, computed as in §5 (composite plus the 2.69 µs accept-path constant): + +| block bytes | shares/s before | shares/s after | % of one core at ~273 shares/s, before | after | +|---:|---:|---:|---:|---:| +| 158 | 85,889 | 91,261 | 0.32% | **0.30%** | +| 50,000 | 7,291 | 63,416 | 3.74% | **0.43%** | +| 100,000 | 3,872 | 60,714 | 7.05% | **0.45%** | +| 250,000 | 1,604 | 58,171 | 17.02% | **0.47%** | +| 500,000 | 808 | 56,189 | 33.79% | **0.49%** | +| 1,000,000 | 408 | 53,912 | 66.89% | **0.51%** | +| 1,500,000 | 272 | 51,995 | 100.45% | **0.53%** | +| 2,000,000 | 204 | 52,060 | 133.53% | **0.52%** | + +The deployed design point (~4,096 connections at `PRISM_STRATUM_VARDIFF_TARGET_SECONDS=15`, +~273 shares/s) went from **crossing one full core inside the consensus range** — 66.9% of a core +at 1 MB, 133.5% at the 2 MB ceiling — to **0.30-0.53% of a core at every template size**. The +"~230-300× headroom" of the #143 findings comment was true only at 0 transactions; it is now +true everywhere. + +Anchor check on this run set: the 0-tx composite measured **8.96 µs** against the findings +comment's published 9.0 µs, a ratio of **0.995** — an independent reproduction of the +anchor in §1, taken in a separate session from the sweep above. + +Load average was 4.7-5.9 across all six runs. The alternating order is what makes +the delta trustworthy under that load, not the absolute values. + +### What this does not change + +The `serialize_block` implementation is untouched, so the §4 decomposition still describes what +a block landing pays: `validate_hex` is still 55% of the term, still a discarded `bytes.fromhex` +probe plus a `value.lower()` copy. That cost now falls once per found block instead of once per +share, which is why it was left alone here. The GIL gap named in §6 is likewise still open, and +still matters to #143's sharding argument — it is just no longer on the share path. diff --git a/tests/perf/serialize_block_share_cost.py b/tests/perf/serialize_block_share_cost.py new file mode 100644 index 000000000..b4a9d1342 --- /dev/null +++ b/tests/perf/serialize_block_share_cost.py @@ -0,0 +1,1338 @@ +#!/usr/bin/env python3 +"""Per-share interpreter-CPU cost of ``direct_stratum.serialize_block``. + +Measures the block-serialization term on the PRISM share-accept path across +template size, so the "``serialize_block`` is O(block bytes) per accepted +share" gap left open by the issue #143 findings comment can be closed with a +number instead of an estimate. + +Methodology mirrors section 1 of that comment so the results are comparable: + +* every timing is **min-of-7 repetitions**, each repetition an auto-sized + batch of calls, reported per call; +* the headline clock is **interpreter CPU**, not wall clock -- see + ``CLOCK_RATIONALE`` below; +* single-threaded: the findings comment established this path is GIL-bound + and does not scale with threads, so what is wanted is the per-share cost, + not concurrency behaviour. + +Three measurement groups: + +1. **Anchors.** The comment's two published numbers -- a 12.4 us primitives + floor per accepted share and a 9.0 us 0-transaction-template composite + ``assemble_submission`` -- reproduced on the host running this script, so + every sweep number below can be read against the same scale. +2. **Axis 1**, total block bytes from the 0-transaction baseline to the qbit + consensus ceiling of 2,000,000 bytes (``MAX_BLOCK_WEIGHT`` with + ``WITNESS_SCALE_FACTOR = 1``, so weight units are bytes). +3. **Axis 2**, transaction count at fixed total block bytes, which separates + the per-transaction constant from the per-byte memcpy/hex cost. + +Plus a decomposition of the serialization term into its constituent buffer +operations, and a restatement of single-core share capacity at each point. + +Self-contained and re-runnable: no database, no network, no coordinator, no +third-party packages. ``python3 tests/perf/serialize_block_share_cost.py`` +prints the tables; ``--json`` emits the same data as JSON. + +Nothing under ``lab/`` is imported for anything but read-only measurement. +""" + +from __future__ import annotations + +import argparse +import json +import os +import platform +import sys +import sysconfig +import time +from dataclasses import dataclass +from decimal import Decimal +from pathlib import Path +from typing import Any, Callable, Sequence + +if not __package__: + sys.path.insert(0, str(Path(__file__).resolve().parents[2])) + +from lab.auxpow import stratum_codec +from lab.prism import direct_stratum +from lab.prism.share_submission import ( + RecentShareIndex, + parse_submit_request, + validate_submit_request, +) + + +CLOCK_RATIONALE = ( + "time.thread_time_ns() is the headline clock: the driver is single " + "threaded, so per-thread CPU is exactly the interpreter CPU this path " + "burns, it excludes time the OS deschedules us (which perf_counter would " + "charge us for on a non-idle host), and on this platform its resolution " + "is far finer than process_time's. process_time and perf_counter are " + "captured over the same batch and reported alongside so any divergence " + "-- e.g. kernel time for multi-megabyte allocations -- is visible." +) + +# Published anchors from the issue #143 findings comment, section 1. +PUBLISHED_PRIMITIVES_FLOOR_US = 12.4 +PUBLISHED_ZERO_TX_COMPOSITE_US = 9.0 +# Same comment: "~15.7 us for the composite submission path (primitives-ratio +# estimate)" and "64k-81k shares/s of single-core capacity". +PUBLISHED_FULL_PATH_ESTIMATE_US = 15.7 +PUBLISHED_CAPACITY_LOW_PER_S = 64_000 +PUBLISHED_CAPACITY_HIGH_PER_S = 81_000 +# Deployed design point named by the issue: ~4,096 connections at +# PRISM_STRATUM_VARDIFF_TARGET_SECONDS=15 -> 4096/15 shares/s steady state. +DESIGN_POINT_SHARES_PER_S = 4096 / 15 + +# qbit consensus ceiling. crates/qbit-pool-builder/src/lib.rs:16-17 sets +# MAX_BLOCK_WEIGHT = 2_000_000 with WITNESS_SCALE_FACTOR = 1, so weight units +# are bytes; compose.yaml and the ckpool preflight both hard-pin the same +# 2000000 against live getblocktemplate.weightlimit. +MAX_BLOCK_BYTES = 2_000_000 + +EXTRANONCE1_HEX = "deadbeef" +EXTRANONCE2_SIZE = 8 +NTIME_HEX = "68a4c1a0" +NONCE_HEX = "12345678" + +# Header (80) + the transaction-count compact size + the coinbase. +HEADER_BYTES = 80 + +AXIS1_TARGET_BLOCK_BYTES = (0, 50_000, 100_000, 250_000, 500_000, 1_000_000, 1_500_000, 2_000_000) +AXIS1_NOMINAL_TX_BYTES = 500 +AXIS2_TOTAL_BLOCK_BYTES = 1_000_000 +AXIS2_TX_COUNTS = (10, 100, 500, 1_000, 2_000, 5_000) + + +# -------------------------------------------------------------------------- +# timing harness +# -------------------------------------------------------------------------- + + +@dataclass(frozen=True) +class Timing: + """Per-call cost from the best of ``reps`` batched repetitions.""" + + thread_cpu_us: float + process_cpu_us: float + wall_us: float + batch: int + reps: int + + def as_json(self) -> dict[str, Any]: + return { + "thread_cpu_us": round(self.thread_cpu_us, 6), + "process_cpu_us": round(self.process_cpu_us, 6), + "wall_us": round(self.wall_us, 6), + "batch": self.batch, + "reps": self.reps, + } + + +def measure( + fn: Callable[[], Any], + *, + reps: int = 7, + min_batch_seconds: float = 0.05, + max_batch: int = 1 << 22, +) -> Timing: + """Min-of-``reps`` batched timing of ``fn``, reported per call. + + The batch is auto-sized so one repetition spans at least + ``min_batch_seconds``, keeping sub-microsecond primitives well clear of + clock granularity. The minimum is taken on thread CPU; the process-CPU and + wall figures reported are the ones from that same winning repetition, so + the three are internally consistent. + """ + + batch = 1 + while True: + start = time.perf_counter_ns() + for _ in range(batch): + fn() + elapsed = time.perf_counter_ns() - start + if elapsed >= min_batch_seconds * 1e9 or batch >= max_batch: + break + growth = max(2, min(16, int(min_batch_seconds * 1e9 / max(elapsed, 1)) + 1)) + batch = min(max_batch, batch * growth) + + best: tuple[float, float, float] | None = None + for _ in range(reps): + thread_start = time.thread_time_ns() + process_start = time.process_time_ns() + wall_start = time.perf_counter_ns() + for _ in range(batch): + fn() + wall_end = time.perf_counter_ns() + process_end = time.process_time_ns() + thread_end = time.thread_time_ns() + candidate = ( + (thread_end - thread_start) / batch / 1000.0, + (process_end - process_start) / batch / 1000.0, + (wall_end - wall_start) / batch / 1000.0, + ) + if best is None or candidate[0] < best[0]: + best = candidate + + assert best is not None + return Timing(best[0], best[1], best[2], batch=batch, reps=reps) + + +# -------------------------------------------------------------------------- +# synthetic template material +# -------------------------------------------------------------------------- + + +def _compact_size_hex(value: int) -> str: + return direct_stratum.compact_size(value).hex() + + +def coinbase_transaction_hex(placeholder_hex: str, *, outputs: int = 1) -> str: + """A structurally valid non-witness coinbase ending in the placeholder. + + Same shape as ``lab/prism/job_build_benchmark.py``'s + ``synthetic_manifest_coinbase_hex``: a height push followed by the + extranonce placeholder at the tail of the coinbase scriptSig, which is + where ``split_coinbase_extranonce`` requires it. + """ + + script_sig = "03aabbcc" + placeholder_hex + output = (50_00000000).to_bytes(8, "little").hex() + "0151" + return ( + "01000000" + + "01" + + "00" * 32 + + "ffffffff" + + _compact_size_hex(len(script_sig) // 2) + + script_sig + + "ffffffff" + + _compact_size_hex(outputs) + + output * outputs + + "00000000" + ) + + +def synthetic_transaction_hex(size_bytes: int, seed: int) -> str: + """A non-witness 1-in/1-out transaction of exactly ``size_bytes`` bytes. + + Padding lives in the output scriptPubKey, with 0-3 slack bytes available + in the input scriptSig so every target size is reachable across compact + size width boundaries. ``seed`` varies the prevout so txids differ and the + merkle branch is not degenerate. + + Structurally valid enough for the code under measurement: it round-trips + through ``strip_witness_transaction``, which is what the job build runs on + every template transaction. + """ + + for cs_width in (1, 3, 5): + for script_sig_len in range(0, 4): + script_pubkey_len = size_bytes - 59 - cs_width - script_sig_len + if script_pubkey_len < 0: + continue + if len(direct_stratum.compact_size(script_pubkey_len)) != cs_width: + continue + prevout = (seed & 0xFFFFFFFF).to_bytes(4, "little").hex() + "11" * 28 + return ( + "02000000" + + "01" + + prevout + + "03000000" + + _compact_size_hex(script_sig_len) + + "ab" * script_sig_len + + "ffffffff" + + "01" + + (1234).to_bytes(8, "little").hex() + + _compact_size_hex(script_pubkey_len) + + "51" * script_pubkey_len + + "00000000" + ) + raise ValueError(f"no transaction encoding reaches {size_bytes} bytes") + + +def base_template() -> dict[str, Any]: + """The repo's own 0-transaction baseline template. + + Mirrors ``lab/prism/job_build_benchmark.py``'s ``base_template``; the + transaction list is supplied separately to + ``make_job_from_builder_manifest``. + """ + + return { + "height": 1000, + "previousblockhash": "11" * 32, + "bits": "1b00ffff", + "version": 0x20000000, + "curtime": 1_755_000_000, + "coinbasevalue": 50_00000000, + "transactions": [], + } + + +@dataclass +class Scenario: + """One template size, built as a real job the accept path would receive.""" + + label: str + job: direct_stratum.DirectQbitStratumJob + transaction_count: int + block_bytes: int + coinbase_bytes: int + transaction_bytes: int + merkle_branch_levels: int + construction: str + + +def build_scenario( + *, + label: str, + transaction_count: int, + target_block_bytes: int | None, + coinbase_outputs: int = 1, + construction: str = "", +) -> Scenario: + """Build a real ``DirectQbitStratumJob`` of the requested size. + + Route 1 of the two acceptable routes: transactions go through + ``make_job_from_builder_manifest``, so ``merkle_branch_for_coinbase`` runs + ``strip_witness_transaction`` over every one of them and the resulting job + carries a realistic merkle branch. That matters because branch depth is + itself a per-share cost inside ``assemble_submission``; a hand-frozen job + with an empty branch would understate the composite at large tx counts. + """ + + placeholder_hex = EXTRANONCE1_HEX + "00" * EXTRANONCE2_SIZE + coinbase_hex = coinbase_transaction_hex(placeholder_hex, outputs=coinbase_outputs) + coinbase_bytes = len(coinbase_hex) // 2 + + if transaction_count == 0: + transaction_hexes: tuple[str, ...] = () + else: + count_prefix = len(direct_stratum.compact_size(transaction_count + 1)) + assert target_block_bytes is not None + payload = target_block_bytes - HEADER_BYTES - count_prefix - coinbase_bytes + base_size, remainder = divmod(payload, transaction_count) + if base_size < 60: + raise ValueError( + f"{transaction_count} transactions cannot fit in " + f"{target_block_bytes} bytes (min 60 bytes each)" + ) + transaction_hexes = tuple( + synthetic_transaction_hex(base_size + (1 if index < remainder else 0), index) + for index in range(transaction_count) + ) + + job = direct_stratum.make_job_from_builder_manifest( + job_id=f"perf-{label}", + template=base_template(), + manifest={"coinbase_tx_hex": coinbase_hex}, + extranonce1_hex=EXTRANONCE1_HEX, + extranonce2_size=EXTRANONCE2_SIZE, + desired_share_difficulty=Decimal("1"), + transaction_hexes=transaction_hexes, + ) + + transaction_bytes = sum(len(tx) // 2 for tx in transaction_hexes) + block_bytes = ( + HEADER_BYTES + + len(direct_stratum.compact_size(transaction_count + 1)) + + coinbase_bytes + + transaction_bytes + ) + return Scenario( + label=label, + job=job, + transaction_count=transaction_count, + block_bytes=block_bytes, + coinbase_bytes=coinbase_bytes, + transaction_bytes=transaction_bytes, + merkle_branch_levels=len(job.merkle_branch), + construction=construction, + ) + + +# -------------------------------------------------------------------------- +# the measured terms +# -------------------------------------------------------------------------- + + +@dataclass +class ScenarioTiming: + scenario: Scenario + composite: Timing + serialize_term: Timing + serialize_block_only: Timing + hex_encode_only: Timing + + # False once the share path stops serializing the block: the term is then + # measured in isolation and is no longer a component of the composite, so + # a ratio between them is not a share of anything. Set from the assembled + # submission rather than inferred from the numbers, because at the 0-tx + # point the isolated term is still smaller than the composite and the + # ratio would look plausible while meaning nothing. + composite_includes_term: bool = True + + @property + def term_share_percent(self) -> float: + if not self.composite_includes_term: + return float("nan") + if self.composite.thread_cpu_us <= 0: + return float("nan") + return 100.0 * self.serialize_term.thread_cpu_us / self.composite.thread_cpu_us + + @property + def composite_less_term_us(self) -> float: + if not self.composite_includes_term: + return float("nan") + return self.composite.thread_cpu_us - self.serialize_term.thread_cpu_us + + +def submission_inputs(job: direct_stratum.DirectQbitStratumJob) -> dict[str, Any]: + """Recompute the intermediates ``assemble_submission`` derives per share.""" + + extranonce2_hex = "00" * EXTRANONCE2_SIZE + version_hex = stratum_codec.apply_version_bits(job.version, None, 0) + coinbase_without_witness, header = stratum_codec.assemble_header_from_notify_submit( + coinb1_hex=job.coinb1, + extranonce1_hex=job.extranonce1_hex, + extranonce2_hex=extranonce2_hex, + coinb2_hex=job.coinb2, + merkle_branch_hex=list(job.merkle_branch), + version_hex=version_hex, + prevhash_hex=job.prevhash, + ntime_hex=NTIME_HEX, + nbits_hex=job.nbits, + nonce_hex=NONCE_HEX, + ) + full_coinbase_hex = ( + job.full_coinbase_prefix + + job.extranonce1_hex + + extranonce2_hex + + job.full_coinbase_suffix + ) + return { + "extranonce2_hex": extranonce2_hex, + "version_hex": version_hex, + "coinbase_without_witness": coinbase_without_witness, + "header": header, + "full_coinbase_hex": full_coinbase_hex, + } + + +def time_scenario(scenario: Scenario, *, reps: int, min_batch_seconds: float) -> ScenarioTiming: + job = scenario.job + inputs = submission_inputs(job) + header = inputs["header"] + full_coinbase_hex = inputs["full_coinbase_hex"] + extranonce2_hex = inputs["extranonce2_hex"] + transaction_hexes = job.transaction_hexes + serialize_block = direct_stratum.serialize_block + assemble = direct_stratum.assemble_submission + block_bytes = serialize_block(header, full_coinbase_hex, transaction_hexes) + + kwargs = {"reps": reps, "min_batch_seconds": min_batch_seconds} + # Does the composite still contain the term on the tree under measurement? + # The driver's jobs use difficulty-65,536 bits, so their submissions are + # ordinary shares -- which is the path worth measuring, and also the path + # that stops serializing the block once the share-path gate is in. + composite_includes_term = bool( + assemble( + job, + extranonce2_hex=extranonce2_hex, + ntime_hex=NTIME_HEX, + nonce_hex=NONCE_HEX, + ).block_hex + ) + return ScenarioTiming( + scenario=scenario, + composite_includes_term=composite_includes_term, + # The whole per-share call as the coordinator invokes it. + composite=measure( + lambda: assemble( + job, + extranonce2_hex=extranonce2_hex, + ntime_hex=NTIME_HEX, + nonce_hex=NONCE_HEX, + ), + **kwargs, + ), + # direct_stratum.py:500 verbatim, the term under measurement. + serialize_term=measure( + lambda: serialize_block(header, full_coinbase_hex, transaction_hexes).hex(), + **kwargs, + ), + serialize_block_only=measure( + lambda: serialize_block(header, full_coinbase_hex, transaction_hexes), + **kwargs, + ), + hex_encode_only=measure(block_bytes.hex, **kwargs), + ) + + +def decompose_term(scenario: Scenario, *, reps: int, min_batch_seconds: float) -> dict[str, Timing]: + """Split the serialization term into the buffer operations it performs. + + ``serialize_block`` runs ``validate_hex`` (which builds an f-string field + name, length-checks, ``bytes.fromhex``-probes and discards, then returns + ``value.lower()`` -- a full fresh copy of the hex string) over the + coinbase and every transaction, then ``bytes.fromhex`` decodes the + validated result for real, joins, and the caller re-encodes with + ``.hex()``. Each of those is timed here over the same material. + """ + + inputs = submission_inputs(scenario.job) + hexes: tuple[str, ...] = (inputs["full_coinbase_hex"],) + scenario.job.transaction_hexes + decoded = [bytes.fromhex(value) for value in hexes] + joined = b"".join(decoded) + validate_hex = stratum_codec.validate_hex + from_hex = bytes.fromhex + kwargs = {"reps": reps, "min_batch_seconds": min_batch_seconds} + + def fromhex_probe() -> None: + for value in hexes: + from_hex(value) + + def lower_copy() -> None: + for value in hexes: + value.lower() + + def validate_hex_calls() -> None: + # Full validate_hex as serialize_block calls it, f-string field name + # included -- the f-string is built per transaction even though it is + # only ever used on the raising path. + for index, value in enumerate(hexes): + validate_hex(value, field_name=f"transaction_hexes[{index}]") + + def real_decode() -> None: + # The list/generator build that produces the transaction bytes. + [from_hex(value) for value in hexes] + + def join_only() -> None: + b"".join(decoded) + + def hex_encode() -> None: + joined.hex() + + def loop_floor() -> None: + for _value in hexes: + pass + + header = inputs["header"] + count_prefix = direct_stratum.compact_size(len(hexes)) + + def floor_single_decode_bytes_out() -> None: + # Counterfactual A: hex inputs kept, but decoded once (no validate_hex + # probe, no .lower() copy) and the block returned as bytes with no + # caller-side .hex() re-encode. + header + count_prefix + b"".join([from_hex(value) for value in hexes]) + + def floor_bytes_in_bytes_out() -> None: + # Counterfactual B: transactions already held as bytes. Pure memcpy + # floor for producing the same block. + header + count_prefix + b"".join(decoded) + + return { + "validate_hex_fromhex_probe": measure(fromhex_probe, **kwargs), + "validate_hex_lower_copy": measure(lower_copy, **kwargs), + "validate_hex_full_calls": measure(validate_hex_calls, **kwargs), + "real_fromhex_decode": measure(real_decode, **kwargs), + "join_concatenation": measure(join_only, **kwargs), + "caller_hex_reencode": measure(hex_encode, **kwargs), + "bare_iteration_floor": measure(loop_floor, **kwargs), + "floor_single_decode_bytes_out": measure(floor_single_decode_bytes_out, **kwargs), + "floor_bytes_in_bytes_out": measure(floor_bytes_in_bytes_out, **kwargs), + } + + +# -------------------------------------------------------------------------- +# anchors +# -------------------------------------------------------------------------- + + +@dataclass +class PrimitiveResult: + name: str + group: str + timing: Timing + note: str = "" + + +def measure_primitives( + scenario: Scenario, *, reps: int, min_batch_seconds: float +) -> list[PrimitiveResult]: + """Time every per-accepted-share primitive on the accept path in isolation. + + The findings comment reports a 12.4 us "primitives, min-of-7" floor but + does not enumerate its primitive set, so this is a reconstruction from the + code path rather than a replay of the original list. Two groups: + ``assemble`` covers the primitives ``assemble_submission`` composes, and + ``accept_path`` covers the per-accepted-share work around it that the + coordinator does for every share (wire decode, request parse/validate, + duplicate reservation, ack encode). + """ + + job = scenario.job + inputs = submission_inputs(job) + header: bytes = inputs["header"] + coinbase_without_witness: bytes = inputs["coinbase_without_witness"] + full_coinbase_hex: str = inputs["full_coinbase_hex"] + extranonce2_hex: str = inputs["extranonce2_hex"] + version_hex: str = inputs["version_hex"] + merkle_branch = list(job.merkle_branch) + transaction_hexes = job.transaction_hexes + + validate_hex = stratum_codec.validate_hex + apply_version_bits = stratum_codec.apply_version_bits + assemble_coinbase = stratum_codec.assemble_coinbase + merkle_from_branch = stratum_codec.compute_merkle_root_from_branch_hex + serialize_header = stratum_codec.serialize_header_from_stratum_fields + strip_witness = direct_stratum.strip_witness_transaction + header_hash_int = stratum_codec.header_hash_int + header_hash_hex = stratum_codec.header_hash_hex + serialize_block = direct_stratum.serialize_block + submission_cls = direct_stratum.DirectQbitSubmission + + coinbase_bytes = assemble_coinbase( + job.coinb1, job.extranonce1_hex, extranonce2_hex, job.coinb2 + ) + merkle_root = merkle_from_branch(coinbase_bytes, merkle_branch) + block_hex = serialize_block(header, full_coinbase_hex, transaction_hexes).hex() + header_hex = header.hex() + + username = "qbit1qperfworker.rig0" + submit_line = json.dumps( + { + "id": 4171, + "method": "mining.submit", + "params": [username, job.job_id, extranonce2_hex, NTIME_HEX, NONCE_HEX], + } + ) + submit_params: list[object] = [ + username, + job.job_id, + extranonce2_hex, + NTIME_HEX, + NONCE_HEX, + ] + request = parse_submit_request(submit_params) + ack = {"id": 4171, "result": True, "error": None} + recent_shares = RecentShareIndex(capacity=50_000) + duplicate_counter = [0] + + def reserve_fresh() -> None: + # A fresh key every call: reserve() on a repeated key short-circuits, + # so replaying one key would measure the duplicate path, not the + # accepted-share path. + duplicate_counter[0] += 1 + recent_shares.reserve((username, f"{header_hex}{duplicate_counter[0]}")) + + kwargs = {"reps": reps, "min_batch_seconds": min_batch_seconds} + specs: list[tuple[str, str, Callable[[], Any], str]] = [ + ( + "validate_hex(extranonce2)", + "assemble", + lambda: validate_hex(extranonce2_hex, field_name="extranonce2"), + "", + ), + ( + "apply_version_bits", + "assemble", + lambda: apply_version_bits(job.version, None, 0), + "no version rolling on this share", + ), + ( + "list(job.merkle_branch)", + "assemble", + lambda: list(job.merkle_branch), + f"{len(merkle_branch)} branch levels", + ), + ( + "assemble_coinbase", + "assemble", + lambda: assemble_coinbase( + job.coinb1, job.extranonce1_hex, extranonce2_hex, job.coinb2 + ), + "", + ), + ( + "compute_merkle_root_from_branch_hex", + "assemble", + lambda: merkle_from_branch(coinbase_bytes, merkle_branch), + f"{len(merkle_branch)} double_sha256 fold levels", + ), + ( + "serialize_header_from_stratum_fields", + "assemble", + lambda: serialize_header( + version_hex=version_hex, + prevhash_hex=job.prevhash, + merkle_root_serialized=merkle_root, + ntime_hex=NTIME_HEX, + nbits_hex=job.nbits, + nonce_hex=NONCE_HEX, + ), + "", + ), + ( + "full_coinbase_hex concat", + "assemble", + lambda: job.full_coinbase_prefix + + job.extranonce1_hex + + extranonce2_hex + + job.full_coinbase_suffix, + "", + ), + ( + "strip_witness_transaction(coinbase) + compare", + "assemble", + lambda: strip_witness(full_coinbase_hex) != coinbase_without_witness, + "", + ), + ("header_hash_int", "assemble", lambda: header_hash_int(header), ""), + ("header_hash_hex", "assemble", lambda: header_hash_hex(header), ""), + ( + "coinbase_without_witness.hex()", + "assemble", + coinbase_without_witness.hex, + "", + ), + ("header.hex()", "assemble", header.hex, ""), + ( + "serialize_block(...).hex()", + "assemble", + lambda: serialize_block(header, full_coinbase_hex, transaction_hexes).hex(), + f"{scenario.transaction_count} tx, {scenario.block_bytes} block bytes", + ), + ( + "DirectQbitSubmission(...)", + "assemble", + lambda: submission_cls( + coinbase_tx_hex=full_coinbase_hex, + coinbase_txid_preimage_hex=header_hex, + header_hex=header_hex, + block_hex=block_hex, + block_hash_hex=header_hex, + block_hash_int=1, + share_pass=True, + block_pass=False, + applied_version_hex=version_hex, + ), + "", + ), + ( + "json.loads(mining.submit line)", + "accept_path", + lambda: json.loads(submit_line), + "", + ), + ( + "parse_submit_request", + "accept_path", + lambda: parse_submit_request(submit_params), + "", + ), + ( + "validate_submit_request", + "accept_path", + lambda: validate_submit_request( + request, + authorized_username=username, + pool_open=True, + extranonce2_size=EXTRANONCE2_SIZE, + ), + "", + ), + ( + "RecentShareIndex.reserve", + "accept_path", + reserve_fresh, + "fresh key per call, includes its own lock", + ), + ("json.dumps(submit ack)", "accept_path", lambda: json.dumps(ack), ""), + ] + + results = [ + PrimitiveResult(name=name, group=group, timing=measure(fn, **kwargs), note=note) + for name, group, fn, note in specs + ] + results.append( + PrimitiveResult( + name="harness floor (lambda: None)", + group="harness", + timing=measure(lambda: None, **kwargs), + note="per-call cost of the measurement loop itself, included in every row above", + ) + ) + return results + + +# -------------------------------------------------------------------------- +# environment +# -------------------------------------------------------------------------- + + +def describe_environment() -> dict[str, Any]: + try: + load = os.getloadavg() + except (OSError, AttributeError): + load = None + gil_disabled = bool(sysconfig.get_config_var("Py_GIL_DISABLED")) + try: + gil_enabled: bool | None = sys._is_gil_enabled() # type: ignore[attr-defined] + except AttributeError: + gil_enabled = None + cpu_brand = platform.processor() + if sys.platform == "darwin": + # platform.processor() is just "arm" on Apple silicon; the sysctl name + # is the informative one and needs no third-party package. + try: + import subprocess + + brand = subprocess.run( + ["sysctl", "-n", "machdep.cpu.brand_string"], + capture_output=True, + text=True, + timeout=5, + ).stdout.strip() + if brand: + cpu_brand = brand + except Exception: # pragma: no cover - informational only + pass + return { + "cpu": cpu_brand, + "machine": platform.machine(), + "platform": platform.platform(), + "cpu_count": os.cpu_count(), + "python_version": sys.version, + "python_implementation": platform.python_implementation(), + "gil_disabled_build": gil_disabled, + "gil_enabled_at_runtime": gil_enabled, + "loadavg_1_5_15": load, + "clock_rationale": CLOCK_RATIONALE, + "clocks": { + name: { + "implementation": info.implementation, + "resolution_s": info.resolution, + } + for name, info in ( + (name, time.get_clock_info(name)) + for name in ("thread_time", "process_time", "perf_counter") + ) + }, + } + + +# -------------------------------------------------------------------------- +# reporting +# -------------------------------------------------------------------------- + + +def _term_scope_note(rows: list[dict[str, Any]]) -> str | None: + """Explain an ``n/a`` term column instead of leaving the reader to guess.""" + + if all(row.get("composite_includes_term", True) for row in rows): + return None + return ( + "`term % of composite` and `composite - term` read **n/a**: on this " + "tree `assemble_submission` serializes the block only for a hash that " + "solved one, so an ordinary share's composite does not contain the " + "term and a ratio between the two is not a share of anything. The " + "term column is still the isolated cost of serializing a block of " + "that size, which is what a block landing pays. For an after-fix " + "delta compare `serialize_term_us` and the composite separately." + ) + + +def _json_number(value: float) -> float | None: + return None if value != value else value + + +def _fmt(value: float | None, digits: int = 2) -> str: + # None/NaN marks a quantity that is undefined on this tree rather than + # merely unmeasured -- printing "nan" invites the reader to treat it as a + # number. Undefined cells render as "n/a" and carry no unit suffix. + if value is None or value != value: + return "n/a" + return f"{value:,.{digits}f}" + + +def _fmt_percent(value: float | None, digits: int = 1) -> str: + text = _fmt(value, digits) + return text if text == "n/a" else f"{text}%" + + +def render_report(results: dict[str, Any]) -> str: + env = results["environment"] + out: list[str] = [] + add = out.append + + add("# serialize_block per-share CPU cost") + add("") + add("## Machine") + add("") + add(f"- CPU: {env['cpu']} ({env['cpu_count']} logical cores)") + add(f"- Platform: {env['platform']}") + add(f"- Python: {env['python_version'].splitlines()[0]} ({env['python_implementation']})") + add( + f"- GIL: build Py_GIL_DISABLED={env['gil_disabled_build']}, " + f"runtime enabled={env['gil_enabled_at_runtime']}" + ) + load = env["loadavg_1_5_15"] + if load: + add(f"- Load average at start (1/5/15 min): {load[0]:.2f} / {load[1]:.2f} / {load[2]:.2f}") + add(f"- Load average at end: {results['loadavg_end']}") + add(f"- Clock: {env['clock_rationale']}") + for name, info in env["clocks"].items(): + add(f" - `{name}`: {info['implementation']}, resolution {info['resolution_s']:.3g} s") + add("") + + anchors = results["anchors"] + add("## Anchors against the issue #143 findings comment") + add("") + add("| anchor | published | this machine | ratio (mine/published) |") + add("|---|---|---|---|") + add( + f"| primitives floor, per accepted share | {PUBLISHED_PRIMITIVES_FLOOR_US} us | " + f"{_fmt(anchors['primitives_floor_us'])} us | " + f"{_fmt(anchors['primitives_floor_ratio'], 3)} |" + ) + add( + f"| 0-tx-template composite `assemble_submission` | {PUBLISHED_ZERO_TX_COMPOSITE_US} us | " + f"{_fmt(anchors['zero_tx_composite_us'])} us | " + f"{_fmt(anchors['zero_tx_composite_ratio'], 3)} |" + ) + add( + f"| composite / primitives inflation ratio | " + f"{_fmt(PUBLISHED_FULL_PATH_ESTIMATE_US / PUBLISHED_PRIMITIVES_FLOOR_US, 3)} (implied " + f"by 15.7/12.4) | {_fmt(anchors['composite_over_assemble_primitives'], 3)} | " + f"{_fmt(anchors['composite_over_assemble_primitives'] / (PUBLISHED_FULL_PATH_ESTIMATE_US / PUBLISHED_PRIMITIVES_FLOOR_US), 3)} |" + ) + add("") + add("Primitive breakdown at the 0-transaction baseline (min-of-7, interpreter CPU):") + add("") + add("| primitive | group | thread CPU us | note |") + add("|---|---|---|---|") + for row in anchors["primitives"]: + add( + f"| `{row['name']}` | {row['group']} | {_fmt(row['timing']['thread_cpu_us'], 3)} | " + f"{row['note']} |" + ) + add( + f"| **sum (assemble group)** | assemble | " + f"**{_fmt(anchors['assemble_primitives_us'], 3)}** | |" + ) + add( + f"| **sum (accept-path group)** | accept_path | " + f"**{_fmt(anchors['accept_path_primitives_us'], 3)}** | |" + ) + add( + f"| **primitives floor (both groups)** | | " + f"**{_fmt(anchors['primitives_floor_us'], 3)}** | |" + ) + add("") + + add("## Axis 1 - total block bytes") + add("") + add( + "| block bytes | tx count | tx size (B) | merkle levels | composite " + "`assemble_submission` us/share | `serialize_block(...).hex()` us/share | " + "term % of composite | composite - term us | composite wall us |" + ) + add("|---|---|---|---|---|---|---|---|---|") + for row in results["axis1"]: + add( + f"| {row['block_bytes']:,} | {row['transaction_count']:,} | " + f"{row['nominal_tx_bytes']} | {row['merkle_branch_levels']} | " + f"{_fmt(row['composite_us'])} | {_fmt(row['serialize_term_us'])} | " + f"{_fmt_percent(row['term_percent'])} | {_fmt(row['composite_less_term_us'])} | " + f"{_fmt(row['composite_wall_us'])} |" + ) + add("") + axis1_note = _term_scope_note(results["axis1"]) + if axis1_note: + add(axis1_note) + add("") + add("Term split: `serialize_block` (bytes out) vs the caller's `.hex()` re-encode.") + add("") + add("| block bytes | `serialize_block` us | `.hex()` re-encode us | sum us | term measured us |") + add("|---|---|---|---|---|") + for row in results["axis1"]: + add( + f"| {row['block_bytes']:,} | {_fmt(row['serialize_block_only_us'])} | " + f"{_fmt(row['hex_encode_only_us'])} | " + f"{_fmt(row['serialize_block_only_us'] + row['hex_encode_only_us'])} | " + f"{_fmt(row['serialize_term_us'])} |" + ) + add("") + + add("## Axis 2 - transaction count at fixed total block bytes") + add("") + add(f"Total block bytes held at ~{AXIS2_TOTAL_BLOCK_BYTES:,}.") + add("") + add( + "| tx count | block bytes | tx size (B) | merkle levels | composite us/share | " + "term us/share | term % of composite |" + ) + add("|---|---|---|---|---|---|---|") + for row in results["axis2"]: + add( + f"| {row['transaction_count']:,} | {row['block_bytes']:,} | " + f"{row['nominal_tx_bytes']} | {row['merkle_branch_levels']} | " + f"{_fmt(row['composite_us'])} | {_fmt(row['serialize_term_us'])} | " + f"{_fmt_percent(row['term_percent'])} |" + ) + add("") + axis2_note = _term_scope_note(results["axis2"]) + if axis2_note: + add(axis2_note) + add("") + fit = results["axis2_fit"] + add( + f"Least-squares fit of the term against transaction count at fixed bytes: " + f"**{_fmt(fit['per_tx_us'], 4)} us/tx** plus a " + f"{_fmt(fit['intercept_us'])} us fixed term (R^2 = {_fmt(fit['r_squared'], 4)}). " + f"Endpoint slope over {fit['endpoint_span'][0]}->{fit['endpoint_span'][1]} tx: " + f"**{_fmt(fit['endpoint_per_tx_us'], 4)} us/tx**." + ) + add("") + + add("## Decomposition of the term") + add("") + for block_label, parts in results["decomposition"].items(): + add(f"### {block_label}") + add("") + add("| part | thread CPU us | % of term |") + add("|---|---|---|") + term = parts["term_us"] + for name, value in parts["parts_us"].items(): + share = 100.0 * value / term if term else float("nan") + add(f"| {name} | {_fmt(value, 3)} | {_fmt(share, 1)}% |") + add(f"| **measured term** | **{_fmt(term, 3)}** | 100% |") + add(f"| residual (term - accounted parts) | {_fmt(parts['residual_us'], 3)} | " + f"{_fmt(100.0 * parts['residual_us'] / term if term else float('nan'), 1)}% |") + add("") + add("Memoranda for the same material (not part of the sum above):") + add("") + add("| memo | thread CPU us | % of term |") + add("|---|---|---|") + for label, value in ( + ( + "`validate_hex` full calls incl. per-tx f-string field name", + parts["validate_hex_full_calls_us"], + ), + ( + "counterfactual A: one decode, bytes out, no re-encode", + parts["floor_single_decode_bytes_out_us"], + ), + ( + "counterfactual B: transactions already bytes, bytes out", + parts["floor_bytes_in_bytes_out_us"], + ), + ("bare iteration over the same list", parts["bare_iteration_floor_us"]), + ): + add( + f"| {label} | {_fmt(value, 3)} | " + f"{_fmt(100.0 * value / term if term else float('nan'), 1)}% |" + ) + add("") + + add("## Capacity") + add("") + add( + "Single-core capacity restated at each Axis 1 point. `mine` is this machine's " + "measured accept-path budget; `published-scale` rescales the findings comment's " + "64k-81k band by the same relative collapse, so it is directly comparable to the " + "number already in the report." + ) + add("") + add( + "| block bytes | accept-path us/share (mine) | shares/s (mine) | " + "published-scale shares/s | % of one core at ~273 shares/s |" + ) + add("|---|---|---|---|---|") + for row in results["capacity"]: + add( + f"| {row['block_bytes']:,} | {_fmt(row['accept_path_us'])} | " + f"{row['shares_per_second_mine']:,.0f} | " + f"{row['published_scale_low']:,.0f}-{row['published_scale_high']:,.0f} | " + f"{_fmt(row['design_point_core_percent'], 2)}% |" + ) + add("") + return "\n".join(out) + + +# -------------------------------------------------------------------------- +# driver +# -------------------------------------------------------------------------- + + +def _linear_fit(xs: Sequence[float], ys: Sequence[float]) -> tuple[float, float, float]: + n = len(xs) + mean_x = sum(xs) / n + mean_y = sum(ys) / n + sxx = sum((x - mean_x) ** 2 for x in xs) + sxy = sum((x - mean_x) * (y - mean_y) for x, y in zip(xs, ys)) + slope = sxy / sxx if sxx else float("nan") + intercept = mean_y - slope * mean_x + ss_tot = sum((y - mean_y) ** 2 for y in ys) + ss_res = sum((y - (intercept + slope * x)) ** 2 for x, y in zip(xs, ys)) + r_squared = 1.0 - ss_res / ss_tot if ss_tot else float("nan") + return slope, intercept, r_squared + + +def run(args: argparse.Namespace) -> dict[str, Any]: + environment = describe_environment() + reps = args.reps + min_batch_seconds = args.min_batch_seconds + kwargs = {"reps": reps, "min_batch_seconds": min_batch_seconds} + + baseline = build_scenario( + label="0tx", + transaction_count=0, + target_block_bytes=None, + construction="repo baseline: job_build_benchmark.base_template() with transactions=[]", + ) + + primitives = measure_primitives(baseline, **kwargs) + assemble_primitives_us = sum( + p.timing.thread_cpu_us for p in primitives if p.group == "assemble" + ) + accept_path_primitives_us = sum( + p.timing.thread_cpu_us for p in primitives if p.group == "accept_path" + ) + primitives_floor_us = assemble_primitives_us + accept_path_primitives_us + + axis1_scenarios: list[Scenario] = [baseline] + for target in AXIS1_TARGET_BLOCK_BYTES: + if target == 0: + continue + payload_estimate = target - HEADER_BYTES - 3 - baseline.coinbase_bytes + count = max(1, round(payload_estimate / AXIS1_NOMINAL_TX_BYTES)) + axis1_scenarios.append( + build_scenario( + label=f"{target}B", + transaction_count=count, + target_block_bytes=target, + construction=( + f"{count} synthetic 1-in/1-out non-witness transactions of " + f"~{AXIS1_NOMINAL_TX_BYTES} B, sized to hit {target} total block bytes exactly" + ), + ) + ) + + axis1_timings = [ + time_scenario(scenario, **kwargs) for scenario in axis1_scenarios + ] + baseline_timing = axis1_timings[0] + + axis2_scenarios = [ + build_scenario( + label=f"1MB-{count}tx", + transaction_count=count, + target_block_bytes=AXIS2_TOTAL_BLOCK_BYTES, + construction=( + f"{count} equal-sized synthetic transactions filling " + f"{AXIS2_TOTAL_BLOCK_BYTES} total block bytes" + ), + ) + for count in AXIS2_TX_COUNTS + ] + axis2_timings = [time_scenario(scenario, **kwargs) for scenario in axis2_scenarios] + + slope, intercept, r_squared = _linear_fit( + [float(t.scenario.transaction_count) for t in axis2_timings], + [t.serialize_term.thread_cpu_us for t in axis2_timings], + ) + first, last = axis2_timings[0], axis2_timings[-1] + endpoint_slope = ( + last.serialize_term.thread_cpu_us - first.serialize_term.thread_cpu_us + ) / (last.scenario.transaction_count - first.scenario.transaction_count) + + decomposition: dict[str, Any] = {} + for timing in axis1_timings: + scenario = timing.scenario + if scenario.transaction_count and scenario.block_bytes not in ( + 1_000_000, + 2_000_000, + ): + continue + parts = decompose_term(scenario, **kwargs) + accounted = { + "validate_hex: bytes.fromhex probe (discarded)": parts[ + "validate_hex_fromhex_probe" + ].thread_cpu_us, + "validate_hex: value.lower() copy": parts["validate_hex_lower_copy"].thread_cpu_us, + "real bytes.fromhex decode + list build": parts["real_fromhex_decode"].thread_cpu_us, + "b''.join concatenation": parts["join_concatenation"].thread_cpu_us, + "caller .hex() re-encode": parts["caller_hex_reencode"].thread_cpu_us, + } + term_us = timing.serialize_term.thread_cpu_us + decomposition[ + f"{scenario.block_bytes:,} block bytes / {scenario.transaction_count:,} tx" + ] = { + "term_us": term_us, + "parts_us": accounted, + "residual_us": term_us - sum(accounted.values()), + "validate_hex_full_calls_us": parts["validate_hex_full_calls"].thread_cpu_us, + "bare_iteration_floor_us": parts["bare_iteration_floor"].thread_cpu_us, + "floor_single_decode_bytes_out_us": parts[ + "floor_single_decode_bytes_out" + ].thread_cpu_us, + "floor_bytes_in_bytes_out_us": parts["floor_bytes_in_bytes_out"].thread_cpu_us, + "raw": {name: timing_.as_json() for name, timing_ in parts.items()}, + } + + # Accept-path budget: the composite at this template size plus the + # per-share accept-path primitives measured around it. The accept-path + # primitives are inflated by the same composite/primitives ratio the + # findings comment used to turn its 12.4 us floor into 15.7 us. + ratio = ( + baseline_timing.composite.thread_cpu_us / assemble_primitives_us + if assemble_primitives_us + else float("nan") + ) + baseline_accept_path_us = ( + baseline_timing.composite.thread_cpu_us + accept_path_primitives_us * ratio + ) + capacity: list[dict[str, Any]] = [] + for timing in axis1_timings: + accept_path_us = timing.composite.thread_cpu_us + accept_path_primitives_us * ratio + collapse = baseline_accept_path_us / accept_path_us if accept_path_us else float("nan") + capacity.append( + { + "block_bytes": timing.scenario.block_bytes, + "transaction_count": timing.scenario.transaction_count, + "accept_path_us": accept_path_us, + "shares_per_second_mine": 1e6 / accept_path_us, + "relative_to_baseline": collapse, + "published_scale_low": PUBLISHED_CAPACITY_LOW_PER_S * collapse, + "published_scale_high": PUBLISHED_CAPACITY_HIGH_PER_S * collapse, + "design_point_core_percent": ( + 100.0 * DESIGN_POINT_SHARES_PER_S * accept_path_us / 1e6 + ), + } + ) + + def scenario_json(timing: ScenarioTiming) -> dict[str, Any]: + scenario = timing.scenario + nominal = ( + scenario.transaction_bytes // scenario.transaction_count + if scenario.transaction_count + else 0 + ) + return { + "label": scenario.label, + "block_bytes": scenario.block_bytes, + "transaction_count": scenario.transaction_count, + "transaction_bytes": scenario.transaction_bytes, + "coinbase_bytes": scenario.coinbase_bytes, + "nominal_tx_bytes": nominal, + "merkle_branch_levels": scenario.merkle_branch_levels, + "construction": scenario.construction, + "composite_us": timing.composite.thread_cpu_us, + "composite_wall_us": timing.composite.wall_us, + "composite_process_us": timing.composite.process_cpu_us, + "serialize_term_us": timing.serialize_term.thread_cpu_us, + "serialize_term_wall_us": timing.serialize_term.wall_us, + "serialize_block_only_us": timing.serialize_block_only.thread_cpu_us, + "hex_encode_only_us": timing.hex_encode_only.thread_cpu_us, + # null rather than NaN: NaN is not valid strict JSON, and the + # quantity is undefined -- not merely unmeasured -- once the share + # path no longer serializes the block. + "composite_includes_term": timing.composite_includes_term, + "term_percent": _json_number(timing.term_share_percent), + "composite_less_term_us": _json_number(timing.composite_less_term_us), + "raw": { + "composite": timing.composite.as_json(), + "serialize_term": timing.serialize_term.as_json(), + "serialize_block_only": timing.serialize_block_only.as_json(), + "hex_encode_only": timing.hex_encode_only.as_json(), + }, + } + + try: + loadavg_end = "%.2f / %.2f / %.2f" % os.getloadavg() + except (OSError, AttributeError): + loadavg_end = "unavailable" + + return { + "environment": environment, + "loadavg_end": loadavg_end, + "methodology": { + "repetitions": reps, + "statistic": "min-of-N over auto-sized batches, reported per call", + "min_batch_seconds": min_batch_seconds, + "clock": "time.thread_time_ns (headline); process_time_ns and perf_counter_ns alongside", + "threads": 1, + "job_construction": ( + "route 1: real DirectQbitStratumJob via " + "direct_stratum.make_job_from_builder_manifest, so " + "merkle_branch_for_coinbase runs strip_witness_transaction over every " + "transaction and the job carries a realistic merkle branch" + ), + "consensus_ceiling_bytes": MAX_BLOCK_BYTES, + }, + "anchors": { + "published_primitives_floor_us": PUBLISHED_PRIMITIVES_FLOOR_US, + "published_zero_tx_composite_us": PUBLISHED_ZERO_TX_COMPOSITE_US, + "primitives_floor_us": primitives_floor_us, + "primitives_floor_ratio": primitives_floor_us / PUBLISHED_PRIMITIVES_FLOOR_US, + "assemble_primitives_us": assemble_primitives_us, + "accept_path_primitives_us": accept_path_primitives_us, + "zero_tx_composite_us": baseline_timing.composite.thread_cpu_us, + "zero_tx_composite_ratio": ( + baseline_timing.composite.thread_cpu_us / PUBLISHED_ZERO_TX_COMPOSITE_US + ), + "composite_over_assemble_primitives": ratio, + "primitives": [ + { + "name": p.name, + "group": p.group, + "note": p.note, + "timing": p.timing.as_json(), + } + for p in primitives + ], + }, + "axis1": [scenario_json(t) for t in axis1_timings], + "axis2": [scenario_json(t) for t in axis2_timings], + "axis2_fit": { + "per_tx_us": slope, + "intercept_us": intercept, + "r_squared": r_squared, + "endpoint_per_tx_us": endpoint_slope, + "endpoint_span": [first.scenario.transaction_count, last.scenario.transaction_count], + "fixed_total_block_bytes": AXIS2_TOTAL_BLOCK_BYTES, + }, + "decomposition": decomposition, + "capacity": capacity, + "design_point_shares_per_second": DESIGN_POINT_SHARES_PER_S, + } + + +def main(argv: list[str] | None = None) -> int: + parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) + parser.add_argument("--json", action="store_true", help="emit the full result set as JSON") + parser.add_argument("--reps", type=int, default=7, help="repetitions per timing (default 7)") + parser.add_argument( + "--min-batch-seconds", + type=float, + default=0.05, + help="target duration of one timing repetition (default 0.05)", + ) + args = parser.parse_args(argv) + + results = run(args) + if args.json: + json.dump(results, sys.stdout, indent=2, default=str) + sys.stdout.write("\n") + else: + sys.stdout.write(render_report(results)) + sys.stdout.write("\n") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tests/perf/stratum_session_concurrency_load.md b/tests/perf/stratum_session_concurrency_load.md new file mode 100644 index 000000000..a55230620 --- /dev/null +++ b/tests/perf/stratum_session_concurrency_load.md @@ -0,0 +1,245 @@ +# Real-thread concurrency load on the PRISM Stratum session path + +Rebuilds the on-demand load driver issue #143 asked for: what the coordinator actually does +when many miners are connected at once, measured with **real OS threads, real sockets and a +real wall clock**. + +Driver: `tests/perf/stratum_session_concurrency_load.py`. On demand only — the filename does +not begin with `test_`, so unittest discovery never collects it, and no test imports it. +`--json` captures a run and `--render PATH` replays a capture without measuring. + +## Why this is a separate instrument from the baton harness + +`tests/prism_concurrency_harness.py` exists to make one interleaving reproducible, and it +replaces the scheduler, the clock and the lock to do it. Those are exactly the three things +this question is about, so the two instruments are deliberately kept apart and neither is +built on the other. The harness answers "is this ordering possible?"; this driver answers +"what happens on a real host under real load?". + +## What is under test + +The shipped session stack, driven through a real `PrismCoordinator`: + +| service | reached through | +|---|---| +| `StratumSessionService` | `coordinator.accept_loop` / `handle_client` | +| `VardiffService` | difficulty assignment, retarget, and reconnect resume via the real submit path | +| `ShareSubmissionService` | `mining.submit` handling end to end | + +Nothing under `lab/` is modified, monkeypatched for behaviour, or reimplemented. The +coordinator is assembled by the repository's own `prism_coordinator_test_support.coordinator` +scaffold — the same one every unit test uses to get a coordinator without a live qbitd — with +the shipped `_ObservedRLock` installed in place of the scaffold's plain `RLock` (a plain lock +reports no contention at all), a real listening socket, a real accept loop on its own thread, +and real per-connection handler threads spawned by the shipped session service. + +Two collaborators are synthesized, and only these two: the node JSON-RPC (this measures the +coordinator, not qbitd) and the audit-bundle subprocess (the repository's existing counting +fake). The `validateaddress` answer is synthesized while the shipped `P2mrAddressValidator`, +its LRU and its singleflight all still run. + +## Modes + +| mode | question | +|---|---| +| `paced` | The deployed design point: one share per connection per `--share-interval-seconds` (default 15, the vardiff target). What does steady state cost? | +| `saturating` | Every connection submits as fast as it is answered. Where is the ceiling, and can sessions still be *established* while the share path is busy? | +| `herd` | Drop every session at once and reconnect. The headline is a **per-session re-establishment census**, not throughput. | + +Herd mode reports the exact number of sessions that never came back inside the window. A +reconnect storm that settles at high shares/s while a third of the pool never gets work back +is a failure, and a throughput-only report would call it a success. + +## Ledger modes + +Every output identifies the ledger mode, because a shares/s figure means something entirely +different with and without a durable write behind it. + +* `--ledger memory` — the shipped `SingleWriterShareLedger`. Isolates interpreter cost and + **omits all database wait**. +* `--ledger postgres` — the real `PsqlShareLedger`, selected through the repository's + real-server gate (`QBIT_PRISM_EXTERNAL_PSQL_COMMAND`, `PRISM_POSTGRES_PSQL_COMMAND` or + `PRISM_DATABASE_URL`, in that order). When the gate is unset the driver exits non-zero and + prints the exact commands to start a server and re-run. + +## Difficulty behaviour + +Difficulty is not permanently pinned, because pinning it removes the retarget path from the +measurement and makes reconnect behaviour meaningless. + +* `resume` (default) — `vardiff_resume_enabled`, the shipped default: a reconnecting session + may adopt its retained difficulty. The run reports `vardiff_resume_outcome_counts` so the + resume path is visible rather than assumed. +* `climb` — resume disabled; every reconnect retargets from the floor. +* `pinned` — `min == max`. Isolation only, and the report says it is not representative. + +The floor difficulty is deliberately below the `2**256-1` clamp in `difficulty_target`, so the +first candidate nonce always clears the share target and nonce search costs one hash per share. +The driver still searches with the shipped `assemble_header_from_notify_submit` and +`header_hash_int` — a real miner searches, so this one does too — and reports the hash count so +that cost stays visible. + +## Controls and the health abort + +Before any conclusion is drawn, the driver measures whether this host can support one: + +* **positive control** — 1 MiB `hashlib.sha256` across `--control-threads`, which releases the + GIL and should scale; +* **negative control** — a small-buffer pure-Python loop, which cannot, and must pin near 1.00; +* **load-average guard**, **descriptor guard** (each connection costs two descriptors; a run + that silently hits the limit reads as coordinator backpressure when it is really the + harness), **memory guard**, and a **wall-clock floor**. + +If any fails, the run reports the abort and **no settle, load, herd or lock numbers at all**. +A number measured on a host that cannot show parallelism is not a weaker result, it is a +different quantity. `--selftest` asserts this: the abort report must contain none of +`SETTLE:`, `LOAD:`, `HERD:` or `COORDINATOR LOCK`. + +## Lock hold attribution + +Production deliberately does not instrument hold *duration* — `_ObservedRLock`'s docstring +explains that a clock read on every acquire and release is exactly the fast-path cost that +class exists to avoid, and that a re-entrant lock needs per-thread depth tracking to avoid +double counting. Both objections are correct, and **this PR changes no production code**, so +the attribution lives in the driver and is paid for only while measuring. Re-entrant +acquisitions are tracked per thread by depth and only the outermost hold is timed. +`--no-lock-attribution` removes the wrapper so its own overhead can be quantified on the same +host instead of argued about. Acquisition and contention counts always come from the shipped +`contention_snapshot()`, never recomputed here. + +--- + +## Recorded baseline: N = 1024 herd census + +One run, recorded to pin what the driver reports. **This is evidence, not a threshold**, and +nothing asserts against it. + +### Machine + +| | | +|---|---| +| CPU | Apple M5 Max, 18 logical cores | +| Python | 3.14.3, GIL enabled | +| Load average (start / end) | 6.61 / 6.59 (1-minute), ≈0.37 per core | +| Peak RSS | 175 MB | +| Descriptor limit | 1,048,576 (ample for 1,024 connections) | + +### Controls + +| control | cores-used across 8 threads | +|---|---| +| positive: 1 MiB `hashlib.sha256` | **7.72** | +| negative: pure-Python loop | **0.97** | + +The host demonstrably shows parallelism, and the negative control pins as the model requires, +so the numbers below are about the coordinator. + +### Run + +`--mode herd --connections 1024 --ledger memory --settle-seconds 50 --herd-return-seconds 50 +--measure-seconds 30 --share-interval-seconds 15` + +### Settle + +| | | +|---|---| +| requested / established | 1,024 / **1,024** | +| never established | **0** | +| wall | 1.49 s | +| establish latency p50 / p95 / max | 663 ms / 1,276 ms / 1,385 ms | + +### Herd + +| | | +|---|---| +| dropped together | 1,024 in 3.45 s | +| re-established | **691 / 1,024** | +| **never re-established** | **333** (within the 50 s window) | +| failure kind | `TimeoutError` × 333 | +| return latency p50 / p95 / max | 724 ms / 9,041 ms / 9,288 ms | +| vardiff resume outcomes | `resumed: 691`, `rejected: 0`, `expired: 0` | +| rejections | **none** | +| watchdog misses | **none** | + +### Load after the herd (paced, 691 surviving sessions) + +| | | +|---|---| +| wall | 34.35 s | +| cores-used (GIL probe discounted) | 0.03 of 18 | +| accepted / rejected shares | 1,382 / 0 | +| shares/s | 40.2 | +| CPU µs/share | 725 | +| ack latency p50 / p95 / p99 / max | 19.1 ms / 43.9 ms / 56.5 ms / 73.8 ms | + +### Coordinator lock + +| | whole run | herd window | +|---|---|---| +| acquisitions | **49,106** | 21,757 | +| contentions | **1** (0.0%) | 0 (0.0%) | + +Longest holders during the herd: + +| call site | holds | total s | mean µs | +|---|---|---|---| +| `prism_coordinator._progress_eligible_client_counts` | 691 | 0.108 | 157.0 | +| `job_delivery._deliver_initial_bundle` | 691 | 0.071 | 103.2 | +| `stratum_session.reserve_client_username` | 691 | 0.058 | 84.5 | +| `job_delivery.cleanup_disconnected_client` | 1,024 | 0.023 | 22.9 | +| `stratum_session.accept_loop` | 1,382 | 0.016 | 11.7 | +| `job_delivery.schedule_initial_job` | 691 | 0.007 | 10.7 | + +### GIL-wait probe + +Median stretch **0.98×** against the idle baseline — no measurable GIL wait at 40 shares/s. +That is expected at this rate and is not evidence that the path is parallel; the saturating +mode is where that question gets asked. + +--- + +## How this compares to the original #143 calibration + +The historical figures pin what these numbers *mean*. They are **not** host-independent +thresholds, and this run neither confirms nor refutes them as such. + +| quantity | #143 calibration | this run | +|---|---|---| +| saturating N=1024 settle | 0/5 settled; only 889–944/1024 established within 50 s | 1,024/1024 established in 1.49 s (herd run, memory ledger, 0-transaction template) | +| herd sessions never re-established | 660–768 / 1024 | **333 / 1024** | +| rejections / watchdog during herd | zero / none | zero / none | +| coordinator-lock acquisitions when settled | ~48,000 | **49,106** | +| lock contention during herd | 30–82% | **0.0% (1 of 49,106)** | +| longest holders | `note_tip_work_delivered`, `accept_loop`, `schedule_initial_job` | `_progress_eligible_client_counts`, `_deliver_initial_bundle`, `reserve_client_username`, then `accept_loop`, `schedule_initial_job` | + +**Two of these agree closely and one does not, and the disagreement is stated rather than +smoothed over.** The acquisition count reproduces almost exactly, and the qualitative herd +finding reproduces: a large minority of sessions never return, with *zero* rejections and +*no* watchdog miss — the failure is silent, which is the part that matters. The contention +percentage did **not** reproduce on this host. Candidate explanations, none of them verified +here: this host has 18 cores and a 0.37 per-core load, the herd drop spreads over 3.45 s +rather than arriving instantaneously, and this run used the in-memory ledger, so no lock +holder ever waits on a database. Establishing which of those accounts for the gap needs runs +on the original rig and is out of scope for this instrument's landing. + +## Reading, noise and limits + +* **Driver overhead is included.** The miner threads, their sockets and the lock-attribution + wrapper run in this process and compete for these cores. This measures a host running the + coordinator *and* its load, not a coordinator alone. +* **Absolute values are host-dependent.** Core count, load average and the Python build change + every number here. Compare runs on one host, not across hosts. +* **In-memory results omit database wait.** A `--ledger postgres` smoke on the same host moved + ack p50 from ≈19 ms to ≈230 ms at N=8. Only the postgres mode exercises the durable path. +* **This is evidence, not a CI threshold.** Nothing here should be asserted against in an + automated test. +* **The GIL-wait probe charges itself.** It is a busy loop that burns most of a core for the + whole window, and `time.process_time()` sums every thread. Its own CPU is measured with + per-thread time, sampled at exactly the load window's boundaries, and subtracted; the + undiscounted whole-process figure is printed beside the adjusted one rather than hidden. +* **`never re-established` is a window statement.** It counts sessions that did not complete + subscribe → authorize → first job inside `--herd-return-seconds`. A session that returns at + 51 s counts as a failure here and may not be one operationally. +* **"Established" means work was delivered**, not merely that TCP was accepted. A session that + is accepted but never receives a job cannot mine, so counting it as established would hide + exactly the failure this census exists to find. diff --git a/tests/perf/stratum_session_concurrency_load.py b/tests/perf/stratum_session_concurrency_load.py new file mode 100644 index 000000000..f0515610d --- /dev/null +++ b/tests/perf/stratum_session_concurrency_load.py @@ -0,0 +1,2028 @@ +#!/usr/bin/env python3 +"""Real-thread concurrency load driver for the PRISM Stratum session path. + +Issue #143 asked what the coordinator actually does when many miners are +connected at once. The deterministic baton harness in +``tests/prism_concurrency_harness.py`` cannot answer that: it exists to make +one interleaving reproducible, and it replaces the scheduler, the clock and +the lock to do it. Those are exactly the three things this question is about. +This driver is the other instrument -- **real OS threads, real sockets, real +wall clock, no baton** -- and the two are deliberately kept separate. + +What is under test is the shipped session stack, driven through a real +``PrismCoordinator``: + +=============================== ============================================ +service reached through +=============================== ============================================ +``StratumSessionService`` ``coordinator.accept_loop`` / ``handle_client`` +``VardiffService`` difficulty assignment, retarget and reconnect + resume, via the real submit path +``ShareSubmissionService`` ``mining.submit`` handling end to end +=============================== ============================================ + +Nothing in ``lab/`` is modified, monkeypatched for behaviour, or +reimplemented. The node RPC is a fake (this measures the coordinator, not +qbitd) and the audit-bundle subprocess is the repository's existing counting +fake; every other object on the path is the shipped one. + +Three modes, which answer different questions: + +``paced`` + The deployed design point: each connection submits one share every + ``--share-interval-seconds`` (default 15, matching + ``PRISM_STRATUM_VARDIFF_TARGET_SECONDS``). This is a steady-state + measurement -- what the coordinator costs when miners behave. +``saturating`` + Every connection submits as fast as it is answered. This finds the + ceiling and, more usefully, shows whether sessions can even be + *established* while the share path is busy. +``herd`` + All sessions are dropped at once and immediately reconnect. This is the + reconnect storm. The headline is **not** throughput: it is a per-session + re-establishment census, including the exact number that never came back + inside the window. + +Metrics, per run: cores-used, shares/s, CPU microseconds per share, ack +latency p50/p95/p99/max, coordinator-lock acquisitions and contentions with +hold attribution by call site, watchdog misses, rejections by reason, and a +fixed-work GIL-wait probe. + +``--json`` captures a run; ``--render PATH`` re-renders a captured run without +measuring, so a review never needs to put two load generators on the same +cores to get both output formats. + +READ THE OUTPUT THIS WAY (the human report repeats all of this): + +* **Driver overhead is included.** The miner threads, their sockets and the + lock-attribution wrapper all run in this process and on these cores. This + measures a host running the coordinator *and* its load, not a coordinator + alone. +* **Absolute values are host-dependent.** Core count, load average and the + Python build change every number here. +* **In-memory ledger results omit database wait.** ``--ledger memory`` exists + to isolate interpreter cost. Only ``--ledger postgres`` includes the real + write path. +* **This is evidence, not a CI threshold.** Nothing here should be asserted + against in an automated test. + +On demand only. The filename does not begin with ``test_``, so unittest +discovery never collects it, and it is never imported by the suite. +""" + +from __future__ import annotations + +import argparse +import hashlib +import json +import os +import platform +import socket +import statistics +import sys +import sysconfig +import threading +import time +import traceback +from dataclasses import dataclass, field +from decimal import Decimal +from pathlib import Path +from typing import Any, Callable, Sequence + +if not __package__: + sys.path.insert(0, str(Path(__file__).resolve().parents[2])) + +from lab.auxpow import stratum_codec, vardiff +from lab.prism import direct_stratum +from lab.prism.prism_coordinator import StratumListenerProfile, _ObservedRLock +from lab.prism.share_ledger import SingleWriterShareLedger +from tests import prism_coordinator_test_support as support + + +# -------------------------------------------------------------------------- +# constants and disclosures +# -------------------------------------------------------------------------- + +#: The deployed design point named by issue #143: one share per connection per +#: vardiff target interval. +DEFAULT_SHARE_INTERVAL_SECONDS = 15.0 + +#: Historical calibration from the original #143 run. These pin what the +#: numbers *mean*; they are explicitly NOT host-independent thresholds and +#: nothing in this driver asserts against them. +CALIBRATION = { + "saturating_n1024_settled": "0/5 settled; only 889-944/1024 established within a 50s settle", + "herd_n1024_unreestablished": "660-768/1024 sessions never re-established, with zero rejections and no watchdog miss", + "lock_contention": "0/~48k settled acquisitions rose to 30-82% contended during herd", + "long_holders": "note_tip_work_delivered, accept_loop, schedule_initial_job", +} + +INTERPRETATION_NOTES = ( + "Driver overhead is included: the miner threads, their sockets and the " + "lock-attribution wrapper run in this process and compete for these " + "cores. This is a host running the coordinator and its load, not a " + "coordinator alone.", + "Absolute values are host-dependent. Core count, load average and the " + "Python build change every number here; compare runs on one host, not " + "across hosts.", + "An in-memory ledger omits database wait entirely. --ledger memory " + "isolates interpreter cost; only --ledger postgres exercises the real " + "durable write path.", + "This is evidence, not a CI threshold. Do not assert against these " + "numbers in an automated test.", +) + +#: A share target below this is reached by essentially every nonce, so the +#: driver spends no measurable CPU searching for one. difficulty_target() +#: clamps at 2**256-1, and DIFF1_TARGET / 1e-12 is past that clamp. +FLOOR_DIFFICULTY = Decimal("1e-12") + +CLOCK_RATIONALE = ( + "cores-used = process CPU / wall, both measured across the same window: " + "time.process_time() sums CPU over every thread in this process, so " + "dividing by time.perf_counter() elapsed gives cores actually kept busy. " + "1.00 means one core's worth of work regardless of how many threads ran, " + "which is the GIL-bound signature; values above 1.00 mean real overlap. " + "Wall clock is the headline for latency because a miner waits in wall " + "time, not in CPU time." +) + + +class DriverAbort(RuntimeError): + """The run could not establish a trustworthy measurement environment.""" + + +class LedgerUnavailable(RuntimeError): + """A requested ledger backend is not reachable on this host.""" + + +# -------------------------------------------------------------------------- +# environment +# -------------------------------------------------------------------------- + + +def _cpu_brand() -> str: + brand = platform.processor() + if sys.platform == "darwin": + try: + import subprocess + + probed = subprocess.run( + ["sysctl", "-n", "machdep.cpu.brand_string"], + capture_output=True, + text=True, + timeout=5, + ).stdout.strip() + if probed: + brand = probed + except Exception: # pragma: no cover - informational only + pass + return brand + + +def _loadavg() -> tuple[float, float, float] | None: + try: + return os.getloadavg() + except (OSError, AttributeError): + return None + + +def _peak_rss_mb() -> float | None: + try: + import resource + except ImportError: # pragma: no cover - non-POSIX + return None + usage = resource.getrusage(resource.RUSAGE_SELF).ru_maxrss + # Linux reports kilobytes, macOS reports bytes. + return usage / (1024 * 1024) if sys.platform == "darwin" else usage / 1024 + + +def raise_descriptor_limit(wanted: int) -> dict[str, Any]: + """Raise RLIMIT_NOFILE toward ``wanted``, reporting what was achieved. + + Each connection costs two descriptors in this process (the miner's socket + and the coordinator's accepted socket), so an N=1024 run needs well over + 2,048. A run that silently hits the limit reads as coordinator + backpressure when it is really the harness, so the achieved limit is + always reported and checked against the connection count. + """ + detail: dict[str, Any] = {"requested": wanted} + try: + import resource + except ImportError: # pragma: no cover - non-POSIX + detail["status"] = "unavailable" + return detail + soft, hard = resource.getrlimit(resource.RLIMIT_NOFILE) + detail["soft_before"] = soft + detail["hard"] = hard + target = min(wanted, hard) if hard != resource.RLIM_INFINITY else wanted + if soft < target: + try: + resource.setrlimit(resource.RLIMIT_NOFILE, (target, hard)) + except (ValueError, OSError) as exc: + detail["status"] = f"raise-failed: {exc}" + detail["soft_after"] = soft + return detail + detail["soft_after"] = resource.getrlimit(resource.RLIMIT_NOFILE)[0] + detail["status"] = "ok" + return detail + + +def describe_environment() -> dict[str, Any]: + try: + gil_enabled: bool | None = sys._is_gil_enabled() # type: ignore[attr-defined] + except AttributeError: + gil_enabled = None + return { + "cpu": _cpu_brand(), + "machine": platform.machine(), + "platform": platform.platform(), + "cpu_count": os.cpu_count(), + "python_version": sys.version, + "python_implementation": platform.python_implementation(), + "gil_disabled_build": bool(sysconfig.get_config_var("Py_GIL_DISABLED")), + "gil_enabled_at_runtime": gil_enabled, + "loadavg_1_5_15": _loadavg(), + "clock_rationale": CLOCK_RATIONALE, + "clocks": { + name: { + "implementation": time.get_clock_info(name).implementation, + "resolution_s": time.get_clock_info(name).resolution, + } + for name in ("process_time", "perf_counter", "monotonic") + }, + } + + +# -------------------------------------------------------------------------- +# statistics +# -------------------------------------------------------------------------- + + +def percentile(values: Sequence[float], fraction: float) -> float: + """Nearest-rank percentile. Empty input is undefined, not zero.""" + if not values: + return float("nan") + ordered = sorted(values) + if fraction <= 0: + return ordered[0] + rank = max(1, min(len(ordered), int(round(fraction * len(ordered) + 0.5)))) + return ordered[rank - 1] + + +def latency_summary(values: Sequence[float]) -> dict[str, Any]: + if not values: + return {"count": 0, "p50_ms": None, "p95_ms": None, "p99_ms": None, "max_ms": None, "mean_ms": None} + return { + "count": len(values), + "p50_ms": percentile(values, 0.50) * 1000.0, + "p95_ms": percentile(values, 0.95) * 1000.0, + "p99_ms": percentile(values, 0.99) * 1000.0, + "max_ms": max(values) * 1000.0, + "mean_ms": statistics.fmean(values) * 1000.0, + } + + +# -------------------------------------------------------------------------- +# coordinator-lock hold attribution +# -------------------------------------------------------------------------- + + +class AttributingLock: + """Wrap the shipped ``_ObservedRLock`` and attribute holds to call sites. + + Production deliberately does not instrument hold *duration*: the class + docstring in ``prism_coordinator`` explains that a clock read on every + acquire and release is exactly the fast-path cost that lock exists to + avoid, and that a re-entrant lock needs per-thread depth tracking to + avoid double counting nested holds. Both objections are correct, and this + PR changes no production code -- so the attribution lives here, in the + driver, and is paid for only while measuring. + + Two consequences the report states rather than hides: + + * This wrapper adds a clock read and a frame inspection per acquisition. + That cost lands inside the measurement. ``--no-lock-attribution`` + removes it, so the overhead can be quantified on the same host instead + of argued about. + * Re-entrant acquisitions are tracked per thread by depth, and only the + outermost hold is timed. A nested acquire is counted as an acquisition + but contributes no second hold interval, so hold seconds cannot exceed + wall time per thread. + + The delegate's own ``contention_snapshot`` remains the source of truth for + acquisitions and contentions; this class never recomputes them. + """ + + def __init__(self, delegate: _ObservedRLock, *, attribute: bool = True) -> None: + self._delegate = delegate + self._attribute = attribute + self._state = threading.local() + self._stats_lock = threading.Lock() + self.holds: dict[str, dict[str, float]] = {} + + # -- call-site identification ----------------------------------------- + + @staticmethod + def _call_site(depth: int) -> str: + """Name the shipped function that took the lock. + + Frames inside this module are skipped so the attribution names a + coordinator call site (``accept_loop``, ``note_tip_work_delivered``, + ``schedule_initial_job``, ...) rather than the wrapper. + """ + frame = sys._getframe(depth) + this_file = __file__ + while frame is not None and frame.f_code.co_filename == this_file: + frame = frame.f_back + if frame is None: + return "" + module = Path(frame.f_code.co_filename).stem + return f"{module}.{frame.f_code.co_name}" + + def _enter_hold(self, site_depth: int) -> None: + depth = getattr(self._state, "depth", 0) + self._state.depth = depth + 1 + if depth == 0: + self._state.site = self._call_site(site_depth + 1) + self._state.started = time.perf_counter() + + def _exit_hold(self) -> None: + depth = getattr(self._state, "depth", 0) + if depth <= 0: + return + self._state.depth = depth - 1 + if depth != 1: + return + held = time.perf_counter() - getattr(self._state, "started", time.perf_counter()) + site = getattr(self._state, "site", "") + with self._stats_lock: + entry = self.holds.get(site) + if entry is None: + entry = {"holds": 0.0, "seconds": 0.0, "max_seconds": 0.0} + self.holds[site] = entry + entry["holds"] += 1 + entry["seconds"] += held + entry["max_seconds"] = max(entry["max_seconds"], held) + + # -- lock protocol ----------------------------------------------------- + + def acquire(self, blocking: bool = True, timeout: float = -1) -> bool: + acquired = self._delegate.acquire(blocking, timeout) + if acquired and self._attribute: + self._enter_hold(1) + return acquired + + def release(self) -> None: + if self._attribute: + self._exit_hold() + self._delegate.release() + + def __enter__(self) -> "AttributingLock": + self._delegate.acquire() + if self._attribute: + self._enter_hold(1) + return self + + def __exit__(self, *_args: object) -> None: + self.release() + + def _is_owned(self) -> bool: + return self._delegate._is_owned() + + def contention_snapshot(self) -> tuple[int, int, float, float]: + return self._delegate.contention_snapshot() + + # -- reporting --------------------------------------------------------- + + def snapshot(self) -> dict[str, Any]: + acquisitions, contentions, wait_sum, wait_max = self._delegate.contention_snapshot() + with self._stats_lock: + sites = [ + { + "site": site, + "holds": int(entry["holds"]), + "hold_seconds": entry["seconds"], + "max_hold_ms": entry["max_seconds"] * 1000.0, + "mean_hold_us": (entry["seconds"] / entry["holds"]) * 1e6 if entry["holds"] else None, + } + for site, entry in self.holds.items() + ] + sites.sort(key=lambda row: row["hold_seconds"], reverse=True) + return { + "acquisitions": acquisitions, + "contentions": contentions, + "contended_percent": (contentions / acquisitions * 100.0) if acquisitions else None, + "wait_seconds_sum": wait_sum, + "wait_seconds_max": wait_max, + "attribution_enabled": self._attribute, + "sites": sites, + } + + +def diff_lock_snapshot(before: dict[str, Any], after: dict[str, Any]) -> dict[str, Any]: + """Attribute only what happened between two snapshots. + + Startup and settle take the lock too; charging their acquisitions to the + measurement window would understate the contended percentage during it. + """ + acquisitions = after["acquisitions"] - before["acquisitions"] + contentions = after["contentions"] - before["contentions"] + prior = {row["site"]: row for row in before["sites"]} + sites = [] + for row in after["sites"]: + was = prior.get(row["site"]) + holds = row["holds"] - (was["holds"] if was else 0) + seconds = row["hold_seconds"] - (was["hold_seconds"] if was else 0.0) + if holds <= 0 and seconds <= 0: + continue + sites.append( + { + "site": row["site"], + "holds": holds, + "hold_seconds": seconds, + "max_hold_ms": row["max_hold_ms"], + "mean_hold_us": (seconds / holds) * 1e6 if holds else None, + } + ) + sites.sort(key=lambda row: row["hold_seconds"], reverse=True) + return { + "acquisitions": acquisitions, + "contentions": contentions, + "contended_percent": (contentions / acquisitions * 100.0) if acquisitions else None, + "wait_seconds_sum": after["wait_seconds_sum"] - before["wait_seconds_sum"], + "wait_seconds_max": after["wait_seconds_max"], + "attribution_enabled": after["attribution_enabled"], + "sites": sites, + } + + +# -------------------------------------------------------------------------- +# controls and probes +# -------------------------------------------------------------------------- + + +def _sha256_work(payload: bytes, rounds: int) -> None: + for _ in range(rounds): + hashlib.sha256(payload).digest() + + +def _python_work(iterations: int) -> int: + total = 0 + for index in range(iterations): + total = (total + index * 7919) & 0xFFFFFFFF + return total + + +def _measure_threaded(work: Callable[[], None], threads: int) -> dict[str, float]: + started = threading.Barrier(threads + 1) + workers = [threading.Thread(target=lambda: (started.wait(), work())) for _ in range(threads)] + for worker in workers: + worker.start() + started.wait() + cpu0, wall0 = time.process_time(), time.perf_counter() + for worker in workers: + worker.join() + cpu = time.process_time() - cpu0 + wall = time.perf_counter() - wall0 + return {"cores_used": cpu / wall if wall > 0 else float("nan"), "wall_s": wall, "cpu_s": cpu} + + +def run_controls(*, threads: int, min_wall_seconds: float) -> dict[str, Any]: + """Positive and negative controls, grown until they clear the wall floor. + + The positive control is a 1 MiB ``hashlib.sha256`` loop, which releases + the GIL and therefore scales; the negative control is a small-buffer + pure-Python loop, which cannot. Together they say whether this host is + capable of showing parallelism at all right now -- a host already + saturated by other work reports a low positive control, and every stage + number measured beside it would be a statement about the host rather than + about the coordinator. + + Thread start/join is charged against the measurement, so a configuration + that finishes too fast reads low for reasons that have nothing to do with + the GIL. Both controls are therefore grown until they clear + ``min_wall_seconds``. + """ + payload = b"\x5a" * (1024 * 1024) + rounds = 64 + while True: + positive = _measure_threaded(lambda: _sha256_work(payload, rounds), threads) + if positive["wall_s"] >= min_wall_seconds or rounds >= 1 << 20: + break + rounds = max(rounds * 2, int(rounds * min_wall_seconds / max(positive["wall_s"], 1e-6)) + 1) + iterations = 200_000 + while True: + negative = _measure_threaded(lambda: _python_work(iterations), threads) + if negative["wall_s"] >= min_wall_seconds or iterations >= 1 << 30: + break + iterations = max(iterations * 2, int(iterations * min_wall_seconds / max(negative["wall_s"], 1e-6)) + 1) + return { + "threads": threads, + "positive_sha256_1mib": positive | {"rounds_per_thread": rounds}, + "negative_pure_python": negative | {"iterations_per_thread": iterations}, + "min_wall_seconds": min_wall_seconds, + } + + +class GilWaitProbe: + """Fixed-work probe: how long does a known unit of Python take under load? + + A single thread repeatedly runs an identical pure-Python work unit. The + unit never changes, so any growth in its wall time is time the thread + spent not holding the GIL. Calibrated on an idle process first, the ratio + ``loaded / idle`` is a direct read of GIL pressure -- and unlike + cores-used it does not need the load to be uniform, because each sample + is independently comparable to the calibration. + """ + + def __init__(self, iterations: int = 40_000) -> None: + self.iterations = iterations + self.samples: list[float] = [] + self.baseline_s: float | None = None + # The probe is a busy loop: it burns most of a core for the whole + # measurement window, and time.process_time() sums every thread. Left + # unaccounted it would land in cores-used and CPU-per-share as if the + # coordinator had spent it. The probe therefore charges itself, using + # per-thread CPU, and the load metrics subtract it. + self.cpu_seconds = 0.0 + self._stop = threading.Event() + self._thread: threading.Thread | None = None + + def calibrate(self, repeats: int = 25) -> float: + timings = [] + for _ in range(repeats): + started = time.perf_counter() + _python_work(self.iterations) + timings.append(time.perf_counter() - started) + self.baseline_s = min(timings) + return self.baseline_s + + def _loop(self) -> None: + cpu0 = time.thread_time() + while not self._stop.is_set(): + started = time.perf_counter() + _python_work(self.iterations) + self.samples.append(time.perf_counter() - started) + # Published every iteration, not just at exit: the load window is + # only part of the probe's lifetime, so callers sample this at the + # window boundaries and subtract the difference. Charging the + # probe's whole run against one window over-subtracts and can + # drive the coordinator's measured CPU to zero. + self.cpu_seconds = time.thread_time() - cpu0 + + def start(self) -> None: + self.samples.clear() + self._stop.clear() + self._thread = threading.Thread(target=self._loop, name="gil-wait-probe", daemon=True) + self._thread.start() + + def stop(self) -> None: + self._stop.set() + if self._thread is not None: + self._thread.join(timeout=10.0) + + def result(self) -> dict[str, Any]: + if not self.samples or self.baseline_s is None: + return {"samples": 0, "baseline_ms": None, "note": "probe did not run"} + median = statistics.median(self.samples) + return { + "samples": len(self.samples), + "iterations_per_sample": self.iterations, + "baseline_ms": self.baseline_s * 1000.0, + "median_ms": median * 1000.0, + "p95_ms": percentile(self.samples, 0.95) * 1000.0, + "max_ms": max(self.samples) * 1000.0, + "stretch_median": median / self.baseline_s, + "stretch_p95": percentile(self.samples, 0.95) / self.baseline_s, + "gil_wait_fraction_median": max(0.0, 1.0 - self.baseline_s / median), + "probe_cpu_seconds": self.cpu_seconds, + } + + +# -------------------------------------------------------------------------- +# ledger selection +# -------------------------------------------------------------------------- + +#: The repository's real-server gate, in the order test/test-prism-postgres-*.sh +#: consult it. QBIT_PRISM_EXTERNAL_PSQL_COMMAND points at an already-running +#: server; the PRISM_* names are the coordinator's own runtime contract. +POSTGRES_GATE_ENV = ( + "QBIT_PRISM_EXTERNAL_PSQL_COMMAND", + "PRISM_POSTGRES_PSQL_COMMAND", + "PRISM_DATABASE_URL", +) + + +def postgres_command_from_gate() -> tuple[str, str] | None: + """Return ``(env_name, psql_command)`` if the real-server gate is set.""" + for name in POSTGRES_GATE_ENV: + value = os.environ.get(name, "").strip() + if not value: + continue + if name == "PRISM_DATABASE_URL": + import shlex + + return name, f"psql {shlex.quote(value)}" + return name, value + return None + + +def build_ledger(mode: str) -> tuple[Any, dict[str, Any]]: + """Build the requested ledger, or explain exactly what is missing. + + Every output identifies the ledger mode, because a shares/s figure means + something entirely different with and without a durable write behind it. + """ + if mode == "memory": + ledger = SingleWriterShareLedger() + return ledger, { + "mode": "memory", + "backend_name": ledger.backend_name, + "includes_database_wait": False, + "detail": "shipped in-memory ledger; isolates interpreter cost and omits all database wait", + } + gate = postgres_command_from_gate() + if gate is None: + raise LedgerUnavailable( + "PostgreSQL mode requires the repository's real-server gate. None of " + + ", ".join(POSTGRES_GATE_ENV) + + " is set. Start a server the way test/test-prism-postgres-ledger.sh does:\n" + " docker run --rm -d --name qbit-prism-perf-postgres " + "-e POSTGRES_USER=qbit -e POSTGRES_PASSWORD=qbit -e POSTGRES_DB=qbit postgres:16-alpine\n" + "then re-run with:\n" + " QBIT_PRISM_EXTERNAL_PSQL_COMMAND='docker exec -i qbit-prism-perf-postgres psql -U qbit -d qbit'" + ) + env_name, psql_command = gate + from lab.prism.share_ledger import ( + WRITER_LEASE_HEARTBEAT_SESSION_PREFIX, + PsqlShareLedger, + ) + import uuid + + ledger = PsqlShareLedger( + psql_command=psql_command, + database_url=os.environ.get("PRISM_DATABASE_URL") or None, + writer_id="prism-perf-driver", + writer_epoch=1, + writer_session_token=f"{WRITER_LEASE_HEARTBEAT_SESSION_PREFIX}{uuid.uuid4().hex}", + initialize_schema=True, + ) + return ledger, { + "mode": "postgres", + "backend_name": ledger.backend_name, + "includes_database_wait": True, + "gate_env": env_name, + "detail": "real PsqlShareLedger through the repository's real-server gate; includes durable write wait", + } + + +# -------------------------------------------------------------------------- +# template sizing +# -------------------------------------------------------------------------- + + +def synthetic_transaction_hex(index: int, payload_bytes: int) -> str: + """A structurally valid non-witness transaction of roughly ``payload_bytes``. + + Template size is an input to this driver because merkle-branch depth and + per-job work both grow with it, and a 0-transaction template flatters the + job-build path in a way no deployed pool would see. + """ + script = (f"{index:08x}" * max(1, payload_bytes // 4))[: max(2, payload_bytes * 2)] + if len(script) % 2: + script += "0" + script_bytes = bytes.fromhex(script) + output_script = "51" + return ( + "01000000" + + "01" + + f"{index + 1:064x}" + + "00000000" + + direct_stratum.compact_size(len(script_bytes)).hex() + + script + + "ffffffff" + + "01" + + (10_000).to_bytes(8, "little").hex() + + direct_stratum.compact_size(len(bytes.fromhex(output_script))).hex() + + output_script + + "00000000" + ) + + +def build_template(transactions: int, transaction_bytes: int) -> dict[str, object]: + template = support.base_template() + template["transactions"] = [ + {"data": synthetic_transaction_hex(index, transaction_bytes)} + for index in range(transactions) + ] + return template + + +# -------------------------------------------------------------------------- +# the rig: one real coordinator behind one real listening socket +# -------------------------------------------------------------------------- + + +@dataclass +class DriverConfig: + connections: int = 64 + mode: str = "paced" + share_interval_seconds: float = DEFAULT_SHARE_INTERVAL_SECONDS + settle_seconds: float = 10.0 + measure_seconds: float = 15.0 + herd_return_seconds: float = 50.0 + ledger_mode: str = "memory" + difficulty_mode: str = "resume" + template_transactions: int = 0 + template_transaction_bytes: int = 250 + lock_attribution: bool = True + control_threads: int = 8 + control_floor_fraction: float = 0.5 + control_min_wall_seconds: float = 0.5 + min_wall_seconds: float = 5.0 + max_rss_mb: float = 8192.0 + max_loadavg_per_core: float = 2.0 + watchdog_timeout_seconds: float = 30.0 + connect_timeout_seconds: float = 20.0 + skip_controls: bool = False + + +class Rig: + """A real ``PrismCoordinator`` on a real socket, with real accept threads. + + The coordinator is assembled by the repository's own + ``prism_coordinator_test_support.coordinator`` scaffold, which is how + every unit test in this tree obtains one without a live qbitd. The + difference here is what is layered on top: the shipped ``_ObservedRLock`` + (the scaffold installs a plain RLock, which would report no contention at + all), a real listening socket, a real accept loop on its own thread, and + real per-connection handler threads spawned by the shipped session + service. + """ + + def __init__(self, config: DriverConfig) -> None: + self.config = config + self.template = build_template( + config.template_transactions, config.template_transaction_bytes + ) + self.server, self.rpc = support.coordinator(template=self.template) + support.install_fake_bundle_builder(self.server) + self.ledger, self.ledger_detail = build_ledger(config.ledger_mode) + self.server.ledger = self.ledger + + base_lock = _ObservedRLock() + self.lock = AttributingLock(base_lock, attribute=config.lock_attribution) + self.server.lock = self.lock + + self._install_rpc_shim() + self._install_difficulty_policy() + + self.server.watchdog_timeout_seconds = config.watchdog_timeout_seconds + self.server.stratum_max_connections = 0 # unbounded; admission is what we measure + self.server.vardiff_resume_enabled = config.difficulty_mode == "resume" + + self.listener_socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + self.listener_socket.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) + self.listener_socket.bind(("127.0.0.1", 0)) + self.listener_socket.listen(max(128, min(config.connections * 2, 4096))) + self.listener_socket.settimeout(0.25) + self.port = self.listener_socket.getsockname()[1] + self.profile = StratumListenerProfile( + name="default", + bind="127.0.0.1", + port=self.port, + vardiff_config=self.server.vardiff_config, + share_difficulty=self.server.share_difficulty, + minimum_advertised_difficulty=FLOOR_DIFFICULTY, + heartbeat_name="stratum_accept", + ) + self.accept_errors: list[str] = [] + self.watchdog_misses: dict[str, int] = {} + self._accept_thread: threading.Thread | None = None + self._watchdog_thread: threading.Thread | None = None + self._watchdog_stop = threading.Event() + + # -- collaborators ----------------------------------------------------- + + def _install_rpc_shim(self) -> None: + """Answer the one RPC the session path needs that FakeRpc lacks. + + ``P2mrAddressValidator`` calls ``validateaddress`` for every distinct + payout identity. The shipped validator, its LRU and its singleflight + all still run; only the node answer is synthesized. + """ + inner = self.rpc.call + + def call(method: str, params: list[object] | None = None) -> object: + if method == "validateaddress": + address = str((params or [""])[0]) + digest = hashlib.sha256(address.encode()).hexdigest()[:64] + return {"isvalid": True, "scriptPubKey": "5220" + digest} + return inner(method, params) + + self.rpc.call = call # type: ignore[method-assign] + + def _install_difficulty_policy(self) -> None: + """Set the vardiff band this run measures under. + + ``pinned`` clamps min == max so no nonce search is ever needed; it + isolates coordinator cost but is explicitly not representative, and + the report says so. ``resume`` and ``climb`` both leave a real band + open so ``VardiffService`` retargets for real -- they differ in + whether a reconnecting session may adopt its retained difficulty. + """ + config = self.config + floor = FLOOR_DIFFICULTY + ceiling = floor if config.difficulty_mode == "pinned" else Decimal("1e-6") + self.server.vardiff_config = vardiff.VardiffConfig( + enabled=True, + target_share_interval_seconds=Decimal(str(config.share_interval_seconds)), + min_difficulty=floor, + max_difficulty=ceiling, + retarget_interval_seconds=Decimal("90"), + max_step_factor=Decimal("4"), + startup_difficulty=floor, + max_step_down_factor=Decimal("4"), + ewma_alpha=Decimal("0.4"), + retarget_tolerance=Decimal("0.25"), + ) + self.server.share_difficulty = floor + + # -- lifecycle --------------------------------------------------------- + + def start(self) -> None: + def accept() -> None: + try: + self.server.accept_loop(self.listener_socket, self.profile) + except BaseException: # noqa: BLE001 - recorded, not swallowed + self.accept_errors.append(traceback.format_exc()) + + self._accept_thread = threading.Thread(target=accept, name="prism-accept", daemon=True) + self._accept_thread.start() + + def watch() -> None: + while not self._watchdog_stop.wait(0.5): + for name in self.server._overdue_heartbeats(time.monotonic()): + self.watchdog_misses[name] = self.watchdog_misses.get(name, 0) + 1 + + self._watchdog_thread = threading.Thread(target=watch, name="prism-watchdog-probe", daemon=True) + self._watchdog_thread.start() + + def stop(self) -> None: + self._watchdog_stop.set() + self.server.stop_event.set() + if self._watchdog_thread is not None: + self._watchdog_thread.join(timeout=3.0) + if self._accept_thread is not None: + self._accept_thread.join(timeout=5.0) + try: + self.listener_socket.close() + except OSError: + pass + for client in list(getattr(self.server, "clients", []) or []): + try: + client.sock.close() + except OSError: + pass + + # -- observation ------------------------------------------------------- + + def rejection_counts(self) -> dict[str, int]: + counts = { + reason: int(count) + for reason, count in dict(getattr(self.server, "rejection_counts_by_reason", {})).items() + if count + } + limits = dict(getattr(self.server, "connection_limit_rejection_counts", {}) or {}) + for name, count in limits.items(): + if count: + counts[f"connection-limit:{name}"] = int(count) + return counts + + def vardiff_resume_outcomes(self) -> dict[str, int]: + service = getattr(self.server, "_vardiff_service", None) + counts = getattr(service, "vardiff_resume_outcome_counts", None) + return {name: int(count) for name, count in dict(counts or {}).items()} + + +# -------------------------------------------------------------------------- +# the miner side: one real socket and one real thread per connection +# -------------------------------------------------------------------------- + + +@dataclass +class SessionOutcome: + """What one connection did, kept per session for the herd census.""" + + index: int + established: bool = False + establish_seconds: float | None = None + failure: str | None = None + accepted: int = 0 + rejected: int = 0 + reject_reasons: dict[str, int] = field(default_factory=dict) + ack_latencies: list[float] = field(default_factory=list) + nonce_hashes: int = 0 + + +class MinerSession: + """One Stratum client: real socket, real handshake, real submits. + + Deliberately hand-rolled rather than reusing ``tests/stratum_client.py``: + that client is a diagnostic CLI with per-message reporting, and its + formatting cost would land inside every ack latency measured here. + """ + + def __init__(self, index: int, port: int, config: DriverConfig) -> None: + self.index = index + self.port = port + self.config = config + self.username = f"{support.PAYOUT_ADDRESS}.perf{index}" + self.sock: socket.socket | None = None + self.stream: Any = None + self.extranonce1 = "" + self.extranonce2_size = 8 + self.difficulty = FLOOR_DIFFICULTY + self.job: list[Any] | None = None + self.outcome = SessionOutcome(index=index) + self._request_id = 0 + self._nonce = (index * 0x9E3779B1) & 0xFFFFFFFF + + # -- wire -------------------------------------------------------------- + + def _send(self, message: dict[str, Any]) -> None: + assert self.stream is not None + self.stream.write(json.dumps(message).encode() + b"\n") + self.stream.flush() + + def _read(self) -> dict[str, Any] | None: + assert self.stream is not None + line = self.stream.readline() + if not line: + return None + return json.loads(line) + + def _absorb(self, message: dict[str, Any]) -> None: + """Apply a server notification. Both are load-bearing for submits.""" + method = message.get("method") + if method == "mining.set_difficulty": + params = message.get("params") or [None] + if params[0] is not None: + self.difficulty = Decimal(str(params[0])) + elif method == "mining.notify": + self.job = list(message.get("params") or []) + + def _await_response(self, request_id: int, deadline: float) -> dict[str, Any]: + while True: + if time.monotonic() > deadline: + raise TimeoutError(f"no response to request {request_id}") + message = self._read() + if message is None: + raise ConnectionError("server closed the connection") + if message.get("id") == request_id: + return message + self._absorb(message) + + def _next_id(self) -> int: + self._request_id += 1 + return self._request_id + + # -- lifecycle --------------------------------------------------------- + + def connect(self) -> None: + self.sock = socket.create_connection( + ("127.0.0.1", self.port), timeout=self.config.connect_timeout_seconds + ) + self.sock.setsockopt(socket.IPPROTO_TCP, socket.TCP_NODELAY, 1) + self.sock.settimeout(self.config.connect_timeout_seconds) + self.stream = self.sock.makefile("rwb") + + def establish(self) -> None: + """Subscribe, authorize and wait for the first job. + + "Established" means the coordinator delivered work -- not merely that + the TCP connection was accepted. A session that is accepted but never + receives a job cannot mine, so counting it as established would hide + exactly the failure the herd census exists to find. + """ + started = time.monotonic() + deadline = started + self.config.connect_timeout_seconds + self.connect() + + request_id = self._next_id() + self._send({"id": request_id, "method": "mining.subscribe", "params": ["prism-perf-driver/1"]}) + response = self._await_response(request_id, deadline) + result = response.get("result") or [] + if len(result) >= 3: + self.extranonce1 = str(result[1]) + self.extranonce2_size = int(result[2]) + + request_id = self._next_id() + self._send({"id": request_id, "method": "mining.authorize", "params": [self.username, "x"]}) + response = self._await_response(request_id, deadline) + if response.get("result") is not True: + raise ConnectionError(f"authorize refused: {response.get('error')}") + + while self.job is None: + if time.monotonic() > deadline: + raise TimeoutError("no job delivered") + message = self._read() + if message is None: + raise ConnectionError("server closed before delivering work") + self._absorb(message) + + self.outcome.established = True + self.outcome.establish_seconds = time.monotonic() - started + + def close(self) -> None: + for closer in (self.stream, self.sock): + try: + if closer is not None: + closer.close() + except OSError: + pass + self.stream = None + self.sock = None + + def drop(self) -> None: + """Abandon the connection the way a herd does: close, no goodbye.""" + self.job = None + self.outcome.established = False + self.close() + + # -- work -------------------------------------------------------------- + + def _find_nonce(self) -> tuple[str, str]: + """Return an ``(extranonce2, nonce)`` that clears the share target. + + A real miner searches, so this driver searches too, using the shipped + header assembly and hash. At the floor difficulty the target is the + 2**256-1 clamp and the first candidate always passes, which is the + point: nonce search is not what this instrument is trying to measure, + and the hash count is reported so its cost stays visible. + """ + assert self.job is not None + job_id, prevhash, coinb1, coinb2, branch, version, nbits, ntime = self.job[:8] + target = direct_stratum.difficulty_target(self.difficulty) + extranonce2 = f"{self.index & ((1 << (self.extranonce2_size * 8)) - 1):0{self.extranonce2_size * 2}x}" + for _ in range(1 << 20): + self._nonce = (self._nonce + 1) & 0xFFFFFFFF + nonce_hex = f"{self._nonce:08x}" + _coinbase, header = stratum_codec.assemble_header_from_notify_submit( + coinb1_hex=coinb1, + extranonce1_hex=self.extranonce1, + extranonce2_hex=extranonce2, + coinb2_hex=coinb2, + merkle_branch_hex=list(branch), + version_hex=version, + prevhash_hex=prevhash, + ntime_hex=ntime, + nbits_hex=nbits, + nonce_hex=nonce_hex, + ) + self.outcome.nonce_hashes += 1 + if stratum_codec.header_hash_int(header) <= target: + return extranonce2, nonce_hex + raise RuntimeError("no nonce cleared the share target") + + def submit_share(self) -> None: + """Submit one share and time the acknowledgement.""" + assert self.job is not None + job_id = self.job[0] + ntime = self.job[7] + extranonce2, nonce_hex = self._find_nonce() + request_id = self._next_id() + started = time.perf_counter() + self._send( + { + "id": request_id, + "method": "mining.submit", + "params": [self.username, job_id, extranonce2, ntime, nonce_hex], + } + ) + response = self._await_response(request_id, time.monotonic() + self.config.connect_timeout_seconds) + self.outcome.ack_latencies.append(time.perf_counter() - started) + if response.get("result") is True: + self.outcome.accepted += 1 + return + self.outcome.rejected += 1 + error = response.get("error") or [] + reason = "unknown" + if isinstance(error, list) and len(error) >= 3 and isinstance(error[2], dict): + reason = str(error[2].get("reason_id", "unknown")) + elif isinstance(error, list) and len(error) >= 2: + reason = str(error[1]) + self.outcome.reject_reasons[reason] = self.outcome.reject_reasons.get(reason, 0) + 1 + + +# -------------------------------------------------------------------------- +# run orchestration +# -------------------------------------------------------------------------- + + +def _establish_all( + sessions: Sequence[MinerSession], *, window_seconds: float +) -> dict[str, Any]: + """Bring every session up concurrently and census the result. + + One thread per session, started together, is the point: a serial ramp + would never produce the admission contention this driver exists to + measure. + """ + started = threading.Barrier(len(sessions) + 1) + deadline_wall = time.monotonic() + window_seconds + + def bring_up(session: MinerSession) -> None: + started.wait() + try: + session.establish() + except BaseException as exc: # noqa: BLE001 - the census wants the reason + session.outcome.established = False + session.outcome.failure = f"{type(exc).__name__}: {exc}" + session.close() + + threads = [ + threading.Thread(target=bring_up, args=(session,), name=f"miner-{session.index}", daemon=True) + for session in sessions + ] + for thread in threads: + thread.start() + began = time.perf_counter() + started.wait() + for thread in threads: + thread.join(timeout=max(0.0, deadline_wall - time.monotonic()) + 5.0) + elapsed = time.perf_counter() - began + + established = [s for s in sessions if s.outcome.established] + failures: dict[str, int] = {} + for session in sessions: + if session.outcome.failure: + key = session.outcome.failure.split(":")[0] + failures[key] = failures.get(key, 0) + 1 + times = [s.outcome.establish_seconds for s in established if s.outcome.establish_seconds is not None] + return { + "requested": len(sessions), + "established": len(established), + "not_established": len(sessions) - len(established), + "window_seconds": window_seconds, + "wall_seconds": elapsed, + "failure_kinds": failures, + "establish_latency": latency_summary(times), + } + + +def _run_share_load( + sessions: Sequence[MinerSession], + *, + mode: str, + duration_seconds: float, + share_interval_seconds: float, + probe: "GilWaitProbe | None" = None, +) -> dict[str, Any]: + """Drive shares for ``duration_seconds`` and return window totals.""" + live = [s for s in sessions if s.outcome.established] + if not live: + return {"threads": 0, "note": "no established session could submit"} + stop = threading.Event() + barrier = threading.Barrier(len(live) + 1) + errors: dict[str, int] = {} + errors_lock = threading.Lock() + + def pump(session: MinerSession) -> None: + barrier.wait() + # Stagger paced miners across the interval so they do not all submit + # on the same tick, which would manufacture a convoy that no real + # pool sees. + if mode == "paced": + time.sleep(share_interval_seconds * (session.index % 100) / 100.0) + while not stop.is_set(): + try: + session.submit_share() + except BaseException as exc: # noqa: BLE001 - counted, not swallowed + with errors_lock: + key = type(exc).__name__ + errors[key] = errors.get(key, 0) + 1 + session.outcome.failure = session.outcome.failure or f"{type(exc).__name__}: {exc}" + return + if mode == "paced" and stop.wait(share_interval_seconds): + return + + threads = [ + threading.Thread(target=pump, args=(session,), name=f"pump-{session.index}", daemon=True) + for session in live + ] + for thread in threads: + thread.start() + barrier.wait() + # The probe is sampled on exactly these two lines, so its discount covers + # the measured window and nothing else. Sampling it around this call + # instead would also charge thread creation, which can exceed the window's + # own CPU and zero out the result. + probe_cpu0 = probe.cpu_seconds if probe is not None else 0.0 + cpu0, wall0 = time.process_time(), time.perf_counter() + time.sleep(duration_seconds) + stop.set() + for thread in threads: + thread.join(timeout=30.0) + cpu = time.process_time() - cpu0 + wall = time.perf_counter() - wall0 + probe_cpu = (probe.cpu_seconds - probe_cpu0) if probe is not None else 0.0 + + accepted = sum(s.outcome.accepted for s in live) + rejected = sum(s.outcome.rejected for s in live) + latencies: list[float] = [] + for session in live: + latencies.extend(session.outcome.ack_latencies) + reject_reasons: dict[str, int] = {} + for session in live: + for reason, count in session.outcome.reject_reasons.items(): + reject_reasons[reason] = reject_reasons.get(reason, 0) + count + return { + "threads": len(live), + "wall_seconds": wall, + "process_cpu_seconds": cpu, + "cores_used": cpu / wall if wall > 0 else float("nan"), + "accepted_shares": accepted, + "rejected_shares": rejected, + "shares_per_second": accepted / wall if wall > 0 else float("nan"), + "cpu_us_per_share": (cpu * 1e6 / accepted) if accepted else None, + "ack_latency": latency_summary(latencies), + "reject_reasons": reject_reasons, + "pump_errors": errors, + "nonce_hashes": sum(s.outcome.nonce_hashes for s in live), + "probe_cpu_seconds_window": probe_cpu, + } + + +def _run_herd(rig: Rig, sessions: Sequence[MinerSession], config: DriverConfig) -> dict[str, Any]: + """Drop every session at once, reconnect, and census what came back. + + The headline is the census, not throughput. A reconnect storm that + settles at high shares/s while a third of the pool never gets work back + is a failure, and a throughput-only report would call it a success. + """ + before_lock = rig.lock.snapshot() + live_before = sum(1 for s in sessions if s.outcome.established) + dropped_at = time.perf_counter() + for session in sessions: + session.drop() + drop_seconds = time.perf_counter() - dropped_at + + # Give the coordinator a moment to observe the disconnects before the + # herd returns; a reconnect that races its own teardown measures cleanup, + # not admission. + time.sleep(0.5) + + returning = [MinerSession(s.index, rig.port, config) for s in sessions] + census = _establish_all(returning, window_seconds=config.herd_return_seconds) + after_lock = rig.lock.snapshot() + + per_session = [ + { + "index": session.index, + "established": session.outcome.established, + "establish_seconds": session.outcome.establish_seconds, + "failure": session.outcome.failure, + } + for session in returning + ] + return { + "dropped": live_before, + "drop_seconds": drop_seconds, + "return_window_seconds": config.herd_return_seconds, + "census": census, + "re_established": census["established"], + "never_re_established": census["not_established"], + "lock": diff_lock_snapshot(before_lock, after_lock), + "vardiff_resume_outcomes": rig.vardiff_resume_outcomes(), + "per_session": per_session, + "sessions": returning, + } + + +def _health_abort(config: DriverConfig, controls: dict[str, Any] | None) -> dict[str, Any] | None: + """Decide whether this host can support a trustworthy measurement. + + Everything below is a statement about the *environment*, checked before + any conclusion is drawn from it. If one fails the run reports the abort + and no stage or herd numbers at all -- a number measured on a host that + cannot show parallelism is not a weaker result, it is a different + quantity. + """ + load = _loadavg() + cores = os.cpu_count() or 1 + if load is not None and load[0] / cores > config.max_loadavg_per_core: + return { + "reason": "load-average", + "detail": ( + f"1-minute load average {load[0]:.2f} over {cores} cores is " + f"{load[0] / cores:.2f} per core, above the " + f"{config.max_loadavg_per_core:.2f} guard. Another workload owns " + "this host; every number here would be about that workload." + ), + } + if controls is None: + return None + positive = controls["positive_sha256_1mib"] + negative = controls["negative_pure_python"] + floor = config.control_floor_fraction * controls["threads"] + if positive["cores_used"] < floor: + return { + "reason": "control-floor", + "detail": ( + f"the 1 MiB sha256 positive control reached only " + f"{positive['cores_used']:.2f} cores across {controls['threads']} " + f"threads, below the {floor:.2f} floor " + f"({config.control_floor_fraction:.0%} of thread count). This host " + "cannot currently demonstrate parallelism, so a low cores-used " + "reading from the coordinator would not be evidence about the " + "coordinator." + ), + } + if negative["cores_used"] > 1.5: + return { + "reason": "negative-control", + "detail": ( + f"the pure-Python negative control reached " + f"{negative['cores_used']:.2f} cores, which should pin near 1.00 " + "under a GIL. The measurement model does not describe this " + "interpreter." + ), + } + return None + + +def _discount_probe_cpu(load: dict[str, Any] | None, probe_cpu_seconds: float) -> None: + """Re-derive the load metrics with the probe's own CPU removed. + + The raw figures are kept beside the adjusted ones rather than replaced: a + reader who wants to know what this whole process cost should be able to + see it, and silently rewriting a measured number is how a driver starts + lying. The adjusted values are the headline because the probe is + measurement apparatus, not coordinator work. + """ + if not load or not load.get("threads"): + return + wall = float(load.get("wall_seconds") or 0.0) + raw_cpu = float(load.get("process_cpu_seconds") or 0.0) + load["probe_cpu_seconds"] = probe_cpu_seconds + load["process_cpu_seconds_raw"] = raw_cpu + load["cores_used_raw"] = load.get("cores_used") + adjusted = raw_cpu - probe_cpu_seconds + if adjusted < 0.0: + # Never silently report zero work: a negative discount means the + # probe accounting is wrong, and that is a defect in this driver + # rather than a coordinator that used no CPU. + load["probe_discount_error"] = ( + f"probe CPU {probe_cpu_seconds:.3f}s exceeded window CPU " + f"{raw_cpu:.3f}s; reporting the undiscounted figures" + ) + load["probe_cpu_seconds"] = probe_cpu_seconds + return + adjusted = max(0.0, adjusted) + load["process_cpu_seconds"] = adjusted + load["cores_used"] = adjusted / wall if wall > 0 else float("nan") + accepted = int(load.get("accepted_shares") or 0) + load["cpu_us_per_share"] = (adjusted * 1e6 / accepted) if accepted else None + + +def run(config: DriverConfig) -> dict[str, Any]: + """Execute one measurement and return the full captured result.""" + environment = describe_environment() + descriptors = raise_descriptor_limit(max(4096, config.connections * 4)) + results: dict[str, Any] = { + "environment": environment, + "descriptors": descriptors, + "config": { + "connections": config.connections, + "mode": config.mode, + "share_interval_seconds": config.share_interval_seconds, + "settle_seconds": config.settle_seconds, + "measure_seconds": config.measure_seconds, + "herd_return_seconds": config.herd_return_seconds, + "ledger_mode": config.ledger_mode, + "difficulty_mode": config.difficulty_mode, + "template_transactions": config.template_transactions, + "template_transaction_bytes": config.template_transaction_bytes, + "lock_attribution": config.lock_attribution, + "control_floor_fraction": config.control_floor_fraction, + "min_wall_seconds": config.min_wall_seconds, + }, + "calibration": CALIBRATION, + "interpretation": list(INTERPRETATION_NOTES), + "aborted": None, + } + + controls = ( + None + if config.skip_controls + else run_controls( + threads=config.control_threads, min_wall_seconds=config.control_min_wall_seconds + ) + ) + results["controls"] = controls + + abort = _health_abort(config, controls) + if abort is not None: + results["aborted"] = abort + results["loadavg_end"] = _loadavg() + results["peak_rss_mb"] = _peak_rss_mb() + return results + + soft = descriptors.get("soft_after") or 0 + if soft and soft < config.connections * 2 + 64: + results["aborted"] = { + "reason": "descriptor-limit", + "detail": ( + f"RLIMIT_NOFILE is {soft}, below the {config.connections * 2 + 64} " + f"this run needs for {config.connections} connections. Sessions " + "would fail to establish for harness reasons and read as " + "coordinator backpressure." + ), + } + results["loadavg_end"] = _loadavg() + results["peak_rss_mb"] = _peak_rss_mb() + return results + + probe = GilWaitProbe() + probe.calibrate() + + rig = Rig(config) + results["ledger"] = rig.ledger_detail + rig.start() + sessions = [MinerSession(index, rig.port, config) for index in range(config.connections)] + try: + settle = _establish_all(sessions, window_seconds=max(config.settle_seconds, 1.0)) + results["settle"] = settle + + lock_before = rig.lock.snapshot() + probe.start() + if config.mode == "herd": + herd = _run_herd(rig, sessions, config) + returning = herd.pop("sessions") + load = _run_share_load( + returning, + mode="paced", + duration_seconds=config.measure_seconds, + share_interval_seconds=config.share_interval_seconds, + probe=probe, + ) + results["herd"] = herd + results["load"] = load + else: + results["load"] = _run_share_load( + sessions, + mode=config.mode, + duration_seconds=config.measure_seconds, + share_interval_seconds=config.share_interval_seconds, + probe=probe, + ) + probe.stop() + lock_after = rig.lock.snapshot() + + _discount_probe_cpu( + results.get("load"), + float((results.get("load") or {}).get("probe_cpu_seconds_window") or 0.0), + ) + results["gil_wait_probe"] = probe.result() + results["lock"] = diff_lock_snapshot(lock_before, lock_after) + results["lock_total"] = lock_after + results["rejections"] = rig.rejection_counts() + results["watchdog_misses"] = dict(rig.watchdog_misses) + results["accept_errors"] = rig.accept_errors[:3] + results["vardiff_resume_outcomes"] = rig.vardiff_resume_outcomes() + finally: + probe.stop() + for session in sessions: + session.close() + rig.stop() + + peak_rss = _peak_rss_mb() + results["peak_rss_mb"] = peak_rss + results["loadavg_end"] = _loadavg() + if peak_rss is not None and peak_rss > config.max_rss_mb: + results["memory_guard"] = { + "exceeded": True, + "peak_rss_mb": peak_rss, + "limit_mb": config.max_rss_mb, + "detail": "peak RSS passed the guard; results may include swap wait", + } + else: + results["memory_guard"] = {"exceeded": False, "peak_rss_mb": peak_rss, "limit_mb": config.max_rss_mb} + + measured_wall = float(results.get("load", {}).get("wall_seconds") or 0.0) + if measured_wall and measured_wall < config.min_wall_seconds: + results["wall_floor_warning"] = ( + f"measurement window was {measured_wall:.2f}s, below the " + f"{config.min_wall_seconds:.2f}s floor; thread start/join is charged " + "against it and cores-used reads low for that reason alone" + ) + return results + + +# -------------------------------------------------------------------------- +# reporting +# -------------------------------------------------------------------------- + + +def _fmt(value: Any, digits: int = 2, suffix: str = "") -> str: + if value is None: + return "n/a" + if isinstance(value, float) and value != value: + return "n/a" + if isinstance(value, (int, float)): + return f"{value:,.{digits}f}{suffix}" + return str(value) + + +def _rule(char: str = "=") -> str: + return char * 78 + + +def render_report(results: dict[str, Any]) -> str: + out: list[str] = [] + add = out.append + env = results["environment"] + config = results["config"] + + add(_rule()) + add("PRISM Stratum session concurrency load (real threads, real sockets)") + add(_rule()) + add("") + add(f"CPU {env['cpu']} x{env['cpu_count']} logical") + add(f"Platform {env['platform']}") + add(f"Python {env['python_version'].splitlines()[0]}") + add(f"GIL build Py_GIL_DISABLED={env['gil_disabled_build']}, runtime enabled={env['gil_enabled_at_runtime']}") + load = env["loadavg_1_5_15"] + if load: + add(f"Load at start {load[0]:.2f} / {load[1]:.2f} / {load[2]:.2f}") + if results.get("loadavg_end"): + end = results["loadavg_end"] + add(f"Load at end {end[0]:.2f} / {end[1]:.2f} / {end[2]:.2f}") + add(f"Peak RSS {_fmt(results.get('peak_rss_mb'), 1, ' MB')}") + descriptors = results.get("descriptors", {}) + add(f"Descriptors soft={descriptors.get('soft_after')} (was {descriptors.get('soft_before')}), status={descriptors.get('status')}") + add("") + + ledger = results.get("ledger") + add("RUN") + add(f" mode {config['mode']}") + add(f" connections {config['connections']:,}") + if ledger: + add(f" LEDGER MODE {ledger['mode']} ({ledger['backend_name']}) -- database wait included: {ledger['includes_database_wait']}") + else: + add(f" LEDGER MODE {config['ledger_mode']} (not constructed; run aborted before the rig was built)") + add(f" difficulty mode {config['difficulty_mode']}") + add(f" share interval {config['share_interval_seconds']:g}s") + add(f" template {config['template_transactions']:,} transactions x {config['template_transaction_bytes']:,} bytes") + add(f" settle / measure {config['settle_seconds']:g}s / {config['measure_seconds']:g}s") + add(f" lock attribution {'on' if config['lock_attribution'] else 'off'}") + add("") + add(f"Clock {env['clock_rationale']}") + add("") + + controls = results.get("controls") + add(_rule("-")) + add("CONTROLS (this host, this process, immediately before the run)") + add(_rule("-")) + if controls is None: + add(" skipped (--skip-controls): no evidence this host can show parallelism") + else: + positive = controls["positive_sha256_1mib"] + negative = controls["negative_pure_python"] + add(f" positive: 1 MiB sha256 x{controls['threads']} threads {positive['cores_used']:.2f} cores over {positive['wall_s']:.2f}s") + add(f" negative: pure-Python x{controls['threads']} threads {negative['cores_used']:.2f} cores over {negative['wall_s']:.2f}s") + add(f" floor required: {config['control_floor_fraction'] * controls['threads']:.2f} cores") + add("") + + if results.get("aborted"): + abort = results["aborted"] + add(_rule()) + add(f"ABORTED: {abort['reason']}") + add(_rule()) + add("") + for line in _wrap(abort["detail"]): + add(f" {line}") + add("") + add(" No stage, load or herd conclusions are reported: the run could not") + add(" establish a trustworthy measurement environment, so there is nothing") + add(" here that would be evidence about the coordinator.") + add("") + return "\n".join(out) + + settle = results.get("settle") + if settle: + add(_rule("-")) + add("SETTLE: session establishment") + add(_rule("-")) + add(f" requested {settle['requested']:,}") + add(f" established {settle['established']:,}") + add(f" NOT established {settle['not_established']:,}") + add(f" wall {settle['wall_seconds']:.2f}s (window {settle['window_seconds']:g}s)") + latency = settle["establish_latency"] + if latency["count"]: + add(f" establish latency p50 {_fmt(latency['p50_ms'])}ms p95 {_fmt(latency['p95_ms'])}ms p99 {_fmt(latency['p99_ms'])}ms max {_fmt(latency['max_ms'])}ms") + if settle["failure_kinds"]: + add(f" failure kinds {settle['failure_kinds']}") + add("") + + herd = results.get("herd") + if herd: + add(_rule("-")) + add("HERD: reconnect census (the headline, not throughput)") + add(_rule("-")) + census = herd["census"] + add(f" dropped together {herd['dropped']:,} in {herd['drop_seconds'] * 1000:.0f}ms") + add(f" re-established {herd['re_established']:,} / {census['requested']:,}") + add(f" NEVER re-established {herd['never_re_established']:,} (within {herd['return_window_seconds']:g}s)") + latency = census["establish_latency"] + if latency["count"]: + add(f" return latency p50 {_fmt(latency['p50_ms'])}ms p95 {_fmt(latency['p95_ms'])}ms p99 {_fmt(latency['p99_ms'])}ms max {_fmt(latency['max_ms'])}ms") + if census["failure_kinds"]: + add(f" failure kinds {census['failure_kinds']}") + if herd.get("vardiff_resume_outcomes"): + add(f" vardiff resume {herd['vardiff_resume_outcomes']}") + herd_lock = herd.get("lock") or {} + add(f" lock during herd {herd_lock.get('acquisitions', 0):,} acquisitions, {_fmt(herd_lock.get('contended_percent'), 1)}% contended") + add(f" lock wait total / max {_fmt(herd_lock.get('wait_seconds_sum'), 3)}s / {_fmt((herd_lock.get('wait_seconds_max') or 0) * 1000)}ms") + for line in _render_hold_sites(herd_lock, indent=" "): + add(line) + add("") + + load_result = results.get("load") or {} + if load_result.get("threads"): + add(_rule("-")) + add(f"LOAD: {config['mode']} share submission") + add(_rule("-")) + add(f" submitting sessions {load_result['threads']:,}") + add(f" wall {load_result['wall_seconds']:.2f}s") + add(f" cores-used {_fmt(load_result['cores_used'])} (of {env['cpu_count']} logical, GIL probe discounted)") + if load_result.get("cores_used_raw") is not None: + add( + f" whole process {_fmt(load_result['cores_used_raw'])} " + f"including {_fmt(load_result.get('probe_cpu_seconds'), 2, 's')} of GIL-probe CPU" + ) + add(f" accepted shares {load_result['accepted_shares']:,}") + add(f" rejected shares {load_result['rejected_shares']:,}") + add(f" shares/s {_fmt(load_result['shares_per_second'])}") + add(f" CPU us/share {_fmt(load_result['cpu_us_per_share'])} (GIL probe discounted)") + ack = load_result["ack_latency"] + add(f" ack latency p50 {_fmt(ack['p50_ms'])}ms p95 {_fmt(ack['p95_ms'])}ms p99 {_fmt(ack['p99_ms'])}ms max {_fmt(ack['max_ms'])}ms") + add(f" nonce hashes {load_result['nonce_hashes']:,} (driver-side search cost, included above)") + if load_result["reject_reasons"]: + add(f" reject reasons {load_result['reject_reasons']}") + if load_result["pump_errors"]: + add(f" submit errors {load_result['pump_errors']}") + add("") + elif load_result: + add(_rule("-")) + add(f"LOAD: {config['mode']} share submission") + add(_rule("-")) + add(f" {load_result.get('note', 'no load ran')}") + add("") + + lock = results.get("lock") + if lock: + add(_rule("-")) + add("COORDINATOR LOCK (measurement window only)") + add(_rule("-")) + add(f" acquisitions {lock['acquisitions']:,}") + add(f" contentions {lock['contentions']:,} ({_fmt(lock['contended_percent'], 1)}%)") + add(f" wait total / max {_fmt(lock['wait_seconds_sum'], 3)}s / {_fmt(lock['wait_seconds_max'] * 1000)}ms") + for line in _render_hold_sites(lock, indent=""): + add(line) + add("") + total = results.get("lock_total") + if total: + add(" Whole run, including settle and any herd -- this is where the") + add(" establishment path (accept_loop, schedule_initial_job) shows up:") + add(f" acquisitions {total['acquisitions']:,}") + add(f" contentions {total['contentions']:,} ({_fmt(total['contended_percent'], 1)}%)") + for line in _render_hold_sites(total, indent=" "): + add(line) + add("") + + probe = results.get("gil_wait_probe") + if probe and probe.get("samples"): + add(_rule("-")) + add("GIL-WAIT PROBE (fixed work unit, one dedicated thread)") + add(_rule("-")) + add(f" samples {probe['samples']:,} x {probe['iterations_per_sample']:,} iterations") + add(f" idle baseline {_fmt(probe['baseline_ms'], 3)}ms") + add(f" under load median {_fmt(probe['median_ms'], 3)}ms p95 {_fmt(probe['p95_ms'], 3)}ms max {_fmt(probe['max_ms'], 3)}ms") + add(f" stretch median {_fmt(probe['stretch_median'])}x p95 {_fmt(probe['stretch_p95'])}x") + add(f" GIL wait (median) {_fmt(probe['gil_wait_fraction_median'] * 100, 1)}% of the unit's wall time") + add("") + + add(_rule("-")) + add("HEALTH") + add(_rule("-")) + add(f" watchdog misses {results.get('watchdog_misses') or 'none'}") + add(f" rejections {results.get('rejections') or 'none'}") + if results.get("vardiff_resume_outcomes"): + add(f" vardiff resume {results['vardiff_resume_outcomes']}") + guard = results.get("memory_guard") or {} + add(f" peak RSS {_fmt(guard.get('peak_rss_mb'), 1, ' MB')} (guard {_fmt(guard.get('limit_mb'), 0, ' MB')}){' EXCEEDED' if guard.get('exceeded') else ''}") + if results.get("wall_floor_warning"): + add(f" wall floor {results['wall_floor_warning']}") + if results.get("accept_errors"): + add(f" accept errors {len(results['accept_errors'])} (first shown in JSON)") + add("") + + add(_rule("-")) + add("HOW TO READ THIS") + add(_rule("-")) + for note in results.get("interpretation", INTERPRETATION_NOTES): + for index, line in enumerate(_wrap(note)): + add(f" {'* ' if index == 0 else ' '}{line}") + add("") + add(" Historical calibration from the original #143 run, which pins what") + add(" these numbers mean and is NOT a threshold for this host:") + for key, value in results.get("calibration", CALIBRATION).items(): + for index, line in enumerate(_wrap(f"{key}: {value}", width=68)): + add(f" {'- ' if index == 0 else ' '}{line}") + add("") + return "\n".join(out) + + +def _render_hold_sites(lock: dict[str, Any], *, indent: str = "") -> list[str]: + """Render the per-call-site hold table, or say why there isn't one.""" + if not lock: + return [] + if not lock.get("attribution_enabled", True): + return [f"{indent} hold attribution off (--no-lock-attribution)"] + sites = lock.get("sites") or [] + if not sites: + return [f"{indent} hold attribution no holds recorded in this window"] + lines = [ + "", + f"{indent} hold attribution by call site (top 8 by total hold):", + f"{indent} {'call site':<54}{'holds':>9}{'total s':>10}{'mean us':>10}{'max ms':>9}", + ] + for row in sites[:8]: + lines.append( + f"{indent} {row['site'][:53]:<54}{row['holds']:>9,}" + f"{row['hold_seconds']:>10.3f}{_fmt(row['mean_hold_us'], 1):>10}" + f"{_fmt(row['max_hold_ms'], 1):>9}" + ) + return lines + + +def _wrap(text: str, width: int = 72) -> list[str]: + words = text.split() + lines: list[str] = [] + current: list[str] = [] + for word in words: + if current and sum(len(w) + 1 for w in current) + len(word) > width: + lines.append(" ".join(current)) + current = [word] + else: + current.append(word) + if current: + lines.append(" ".join(current)) + return lines or [""] + + +# -------------------------------------------------------------------------- +# self-test +# -------------------------------------------------------------------------- + + +def _selftest(argv_json_path: str | None = None) -> int: + """Smoke the driver itself at a size that fits any host. + + Not a unit test and not collected by discovery: this proves the + instrument runs, captures, replays, and refuses to report conclusions + after an abort. It is the check to run before trusting a real sweep. + """ + failures: list[str] = [] + + def check(name: str, condition: bool, detail: str = "") -> None: + status = "PASS" if condition else "FAIL" + print(f" [{status}] {name}{'' if condition else f' -- {detail}'}", flush=True) + if not condition: + failures.append(name) + + print("selftest: discovery safety", flush=True) + check( + "filename is not collected by unittest discovery", + not Path(__file__).name.startswith("test"), + f"{Path(__file__).name} would match the default test*.py pattern", + ) + + base = dict( + connections=4, + settle_seconds=10.0, + measure_seconds=2.0, + herd_return_seconds=10.0, + min_wall_seconds=0.0, + control_threads=2, + control_min_wall_seconds=0.2, + control_floor_fraction=0.0, + max_loadavg_per_core=1e9, + connect_timeout_seconds=20.0, + ) + + captured: dict[str, Any] | None = None + for mode in ("paced", "saturating", "herd"): + print(f"selftest: {mode} mode at N=4", flush=True) + config = DriverConfig(mode=mode, share_interval_seconds=0.5, **base) + try: + result = run(config) + except BaseException as exc: # noqa: BLE001 - the smoke wants the reason + check(f"{mode} run completes", False, f"{type(exc).__name__}: {exc}") + continue + check(f"{mode} run completes", True) + check(f"{mode} was not aborted", not result.get("aborted"), str(result.get("aborted"))) + check(f"{mode} established every session", result["settle"]["established"] == 4, str(result["settle"])) + check(f"{mode} identifies the ledger mode", bool(result.get("ledger", {}).get("mode"))) + check(f"{mode} recorded lock acquisitions", (result.get("lock") or {}).get("acquisitions", 0) > 0) + if mode == "herd": + check("herd reports a per-session census", len(result["herd"]["per_session"]) == 4) + check( + "herd reports the exact count that did not return", + isinstance(result["herd"]["never_re_established"], int), + ) + else: + check(f"{mode} accepted shares", result["load"]["accepted_shares"] > 0, str(result["load"])) + check(f"{mode} renders", bool(render_report(result))) + captured = result + + print("selftest: JSON capture and render replay", flush=True) + if captured is not None: + path = argv_json_path or os.path.join( + os.environ.get("TMPDIR", "/tmp"), "prism_session_load_selftest.json" + ) + with open(path, "w", encoding="utf-8") as handle: + json.dump(captured, handle, indent=2, default=str) + with open(path, encoding="utf-8") as handle: + replayed = json.load(handle) + first = render_report(replayed) + second = render_report(json.loads(json.dumps(replayed, default=str))) + check("render replay is byte-identical across reloads", first == second) + check("render replay names the ledger mode", "LEDGER MODE" in first) + print(f" capture written to {path}", flush=True) + + print("selftest: control-floor abort", flush=True) + abort_config = DriverConfig( + connections=2, + measure_seconds=1.0, + control_threads=2, + control_min_wall_seconds=0.2, + # Arithmetically unreachable rather than merely demanding: 2 threads + # cannot exceed 2.00 cores-used, so a floor of 5x thread count can + # never be met on any host. A floor that a fast idle machine could + # occasionally clear would make this smoke pass or fail with load. + control_floor_fraction=5.0, + max_loadavg_per_core=1e9, + ) + aborted = run(abort_config) + check("aborts when the control floor is not met", bool(aborted.get("aborted")), str(aborted.get("aborted"))) + if aborted.get("aborted"): + check("abort reason is the control floor", aborted["aborted"]["reason"] == "control-floor") + report = render_report(aborted) + check("abort report says ABORTED", "ABORTED" in report) + for forbidden in ("SETTLE:", "LOAD:", "HERD:", "COORDINATOR LOCK"): + check(f"abort report omits {forbidden!r}", forbidden not in report) + + print("", flush=True) + if failures: + print(f"selftest FAILED: {len(failures)} check(s): {failures}", flush=True) + return 1 + print("selftest PASSED", flush=True) + return 0 + + +# -------------------------------------------------------------------------- +# entry point +# -------------------------------------------------------------------------- + + +def build_parser() -> argparse.ArgumentParser: + parser = argparse.ArgumentParser( + description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter + ) + parser.add_argument("--json", action="store_true", help="emit the full result set as JSON") + parser.add_argument( + "--render", + metavar="PATH", + default=None, + help=( + "render a previously captured --json file instead of measuring. " + "Running the sweep twice to get both formats would put two load " + "generators on the same cores and corrupt both." + ), + ) + parser.add_argument("--selftest", action="store_true", help="smoke the driver itself and exit") + parser.add_argument("--connections", type=int, default=64, help="connection count (default 64; supports 1024+)") + parser.add_argument( + "--mode", + choices=("paced", "saturating", "herd"), + default="paced", + help="paced: one share per interval per connection (the design point); " + "saturating: submit as fast as answered; herd: drop all and census the return", + ) + parser.add_argument( + "--share-interval-seconds", + type=float, + default=DEFAULT_SHARE_INTERVAL_SECONDS, + help=f"paced submit interval (default {DEFAULT_SHARE_INTERVAL_SECONDS:g}, the vardiff target)", + ) + parser.add_argument("--settle-seconds", type=float, default=50.0, help="session establishment window (default 50)") + parser.add_argument("--measure-seconds", type=float, default=30.0, help="measurement window (default 30)") + parser.add_argument("--herd-return-seconds", type=float, default=50.0, help="herd re-establishment window (default 50)") + parser.add_argument( + "--ledger", + dest="ledger_mode", + choices=("memory", "postgres"), + default="memory", + help="memory isolates interpreter cost; postgres uses the repository's real-server gate", + ) + parser.add_argument( + "--difficulty-mode", + choices=("resume", "climb", "pinned"), + default="resume", + help="resume: reconnects may adopt their retained difficulty (shipped default); " + "climb: every reconnect retargets from the floor; " + "pinned: min==max, isolation only and not representative", + ) + parser.add_argument("--template-transactions", type=int, default=0, help="synthetic transactions in the template (default 0)") + parser.add_argument("--template-transaction-bytes", type=int, default=250, help="approximate bytes per synthetic transaction") + parser.add_argument( + "--no-lock-attribution", + dest="lock_attribution", + action="store_false", + help="skip per-call-site hold timing, so its own overhead can be quantified", + ) + parser.add_argument("--control-threads", type=int, default=8, help="threads for the positive/negative controls") + parser.add_argument( + "--control-floor-fraction", + type=float, + default=0.5, + help="the positive control must reach this fraction of control-thread count in cores-used, else abort", + ) + parser.add_argument("--control-min-wall-seconds", type=float, default=0.5, help="wall floor per control measurement") + parser.add_argument("--min-wall-seconds", type=float, default=5.0, help="warn below this measurement wall clock") + parser.add_argument("--max-rss-mb", type=float, default=8192.0, help="peak RSS guard") + parser.add_argument( + "--max-loadavg-per-core", + type=float, + default=2.0, + help="abort if the 1-minute load average per core exceeds this before measuring", + ) + parser.add_argument("--connect-timeout-seconds", type=float, default=20.0, help="per-connection socket timeout") + parser.add_argument("--skip-controls", action="store_true", help="skip controls (removes the parallelism evidence)") + return parser + + +def main(argv: list[str] | None = None) -> int: + args = build_parser().parse_args(argv) + + if args.selftest: + return _selftest() + + if args.render: + with open(args.render, encoding="utf-8") as handle: + sys.stdout.write(render_report(json.load(handle))) + sys.stdout.write("\n") + return 0 + + config = DriverConfig( + connections=args.connections, + mode=args.mode, + share_interval_seconds=args.share_interval_seconds, + settle_seconds=args.settle_seconds, + measure_seconds=args.measure_seconds, + herd_return_seconds=args.herd_return_seconds, + ledger_mode=args.ledger_mode, + difficulty_mode=args.difficulty_mode, + template_transactions=args.template_transactions, + template_transaction_bytes=args.template_transaction_bytes, + lock_attribution=args.lock_attribution, + control_threads=args.control_threads, + control_floor_fraction=args.control_floor_fraction, + control_min_wall_seconds=args.control_min_wall_seconds, + min_wall_seconds=args.min_wall_seconds, + max_rss_mb=args.max_rss_mb, + max_loadavg_per_core=args.max_loadavg_per_core, + connect_timeout_seconds=args.connect_timeout_seconds, + skip_controls=args.skip_controls, + ) + try: + results = run(config) + except LedgerUnavailable as exc: + sys.stderr.write(f"{exc}\n") + return 2 + + if args.json: + json.dump(results, sys.stdout, indent=2, default=str) + sys.stdout.write("\n") + else: + sys.stdout.write(render_report(results)) + sys.stdout.write("\n") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tests/perf/window_builder_pipe_latency.md b/tests/perf/window_builder_pipe_latency.md new file mode 100644 index 000000000..ca2d2f2bb --- /dev/null +++ b/tests/perf/window_builder_pipe_latency.md @@ -0,0 +1,270 @@ +# Builder pipe transport latency: fixed polling sleeps vs readiness waits (#236) + +Issue #236's cold window materializations spent 47.5-48.8 s of 54-56 s inside +`daemon_prepare`, a timer that covers lock admission, request encoding, both +pipe directions, the Rust fold and response handling. The shipped transport +helpers in `lab/prism/bundle_compiler.py` slept a fixed 20 ms after every +EAGAIN -- once per drained pipe capacity -- so a multi-megabyte exchange could +pay far more in idle sleeps than in copying. This report measures that on Linux +against the real daemon binary, before and after replacing the sleeps with +readiness waits bounded by the deadline and the existing 20 ms +cancellation-check cadence. + +Driver: `tests/perf/window_builder_pipe_latency.py`. Standard library only; no +database, network or coordinator. It loads the baseline transport module +straight from a git revision and runs it beside the working tree in one +process, against the same peer subprocesses and the same daemon binary. + +This is an **on-demand instrument, not a test**: it asserts no thresholds and +is not named `test_*`, so discovery never runs it (#160). + +**Measured at** working tree on `e59bda34016f6dc0a6bc39f9b879b8df64f51f60` +(baseline = that commit's `lab/prism/bundle_compiler.py`, patched = the same +file with the readiness-wait change of this PR). Everything else, including +the Rust daemon, is identical between the two variants. + +--- + +## Headline + +**The fixed sleeps were the transport cost, and they are gone.** On this host, +a cold 210k-share `prepare_window` through the shipped +`BundleCompiler.prepare_payout_window` drops from **39.8-43.8 s to 12.2-13.8 s**. +The 90.4 MiB request write alone drops from **26.9-27.6 s to 0.13-0.15 s** +(1,327-1,362 twenty-millisecond sleeps replaced by 5,813-6,705 readiness +waits that each return as soon as the daemon drains the pipe). The bytes, +digests and canonical item bytes are identical across variants. + +**What remains after the change is mostly not the coordinator's pipe loops.** +At 210k the patched round trip is 10.7-12.0 s of `response_wait` -- elapsed +wall time from the moment the pipe accepted the last request byte until the +envelope line was received, which contains the daemon's scheduling and +processing (parsing what it had not yet consumed, the fold, canonical +serialization) together with response readiness and receipt; it is **not** a +measured Rust CPU figure -- plus 1.0-1.2 s of unattributed coordinator time +before the write (request encoding and everything else outside the wrapped +calls) and 0.26-0.39 s reading the response section. Separating the residual +needs Rust-side profiling; this report does not attribute time inside the +daemon. + +Raw pipe throughput of the helpers rises from **3-6 MiB/s to 30-215 MiB/s** +(1-16 MiB payloads), with the number of blocked waits essentially unchanged -- +each is still one wait per 64 KiB pipe capacity -- because each wait now ends +on readiness rather than on a timer. + +**Not established here:** the production improvement. This host is not the +production container (CPython 3.12.3, no Docker image, no PostgreSQL ledger +read in the loop), the measured path is the compiler's daemon exchange only +(no ledger read, record conversion, mirror validation or Stratum job build), +the host was shared with the other two #236 implementation workers during +the runs, and the incident's `daemon_prepare` was measured on different +hardware with production data. The recovery-time gate (first usable Stratum +work within 20 s at 210k) remains to be measured in the dedicated environment. + +--- + +## 1. Machine and method + +| | | +|---|---| +| Platform | Linux 6.8.0-106-generic x86_64, glibc 2.39 (KVM guest, 8 vCPU) | +| Python | CPython 3.12.3 | +| Pipe capacity | 65,536 bytes (`F_GETPIPE_SZ` on a fresh pipe) | +| `os.splice` | available | +| Load average at start | 0.38 / 0.49 / 0.52 | +| Daemon | `target/release/qbit-prism-build-audit-bundle` built from this tree (`cargo build --release -p qbit-prism`) | + +Commands, in this order. The host was **not** idle: the PR 1 and PR 2 +implementation workers for #236 were active on the same machine during both +runs, so absolute times carry that interference (the variants alternate per +repetition, so it lands on both). + +``` +python3 tests/perf/window_builder_pipe_latency.py \ + --baseline-rev e59bda34016f6dc0a6bc39f9b879b8df64f51f60 \ + --sizes-mib 1,4,16 --reps 5 --skip-daemon --json raw.json +python3 tests/perf/window_builder_pipe_latency.py \ + --baseline-rev e59bda34016f6dc0a6bc39f9b879b8df64f51f60 \ + --skip-raw --daemon-sizes 52000,210000,400000 --daemon-reps 2 --json daemon.json +``` + +The second command was stopped after the first 400k repetition of each variant +(see section 3); its JSON was therefore not written, and the daemon table +below is transcribed from the per-repetition log lines. + +**Raw transport.** Each helper is called exactly as the daemon transport calls +it, on a `_ServeBuilderClient` wrapping a Python peer subprocess: the peer +writes N bytes to its stdout in 64 KiB blocks (reads: `read_exact`, +`read_line`), or reads its stdin to EOF in 64 KiB reads (writes: `write`, +`splice` from a temporary spool file). Timing starts after a go byte so peer +start-up is excluded. The peer reports a SHA-256 of what it sent or received +and every sample is checked byte-exact (`exact` column). Variants alternate per +repetition so host drift lands on both. + +**Blocked waits.** For the patched module, the count of `_PipeReadinessWaiter.wait` +calls; for the baseline, the count of `time.sleep` calls (its loops resolve +`time.sleep` at call time, and every peer is a subprocess, so nothing else in +the process sleeps during a sample). + +**Real daemon.** One cold `prepare_window` `full` round trip through the +shipped `BundleCompiler.prepare_payout_window` per repetition, on a freshly +spawned daemon, with records shaped like the ledger's rows (production +`username:block_hash_hex` share ids, 64 miners × 8 rigs, 16384 share +difficulty, window weight = total difficulty so every record is retained). +Phases come from wrapping the compiler's own transport methods: + +| phase | contains | +|---|---| +| `spawn` | daemon spawn + handshake line read | +| `request_encode` | total minus every wrapped phase: unattributed coordinator time outside the transport calls, dominated by but not isolated to the whole-request `json.dumps` | +| `request_write` | `_serve_builder_write` of the request line: elapsed until the pipe accepted every byte (not proof the daemon consumed them; up to one pipe capacity may still be in flight) | +| `response_wait` | `_serve_builder_read_line` of the envelope: elapsed from write completion to envelope receipt -- daemon scheduling and processing (remaining parse, fold, canonical serialization) plus response readiness and receipt; wall time, not measured Rust CPU | +| `response_read` | `_serve_builder_read_exact` of the canonical items section and its newline: elapsed receipt of bytes the daemon produces after the envelope | + +Request sizes (`json.dumps` of the same request, measured separately): +52k → 22.3 MiB, 210k → 90.4 MiB. + +--- + +## 2. Raw transport (5 reps, peer chunk 64 KiB, peer pause 0) + +| helper | bytes | variant | median ms | min ms | max ms | median waits | MiB/s | exact | +|---|---:|---|---:|---:|---:|---:|---:|---| +| read_exact | 1,048,576 | baseline | 204.81 | 184.20 | 206.04 | 10 | 4.9 | yes | +| read_exact | 1,048,576 | patched | 30.60 | 30.31 | 33.43 | 16 | 32.7 | yes | +| read_exact | 4,194,304 | baseline | 701.37 | 676.85 | 785.10 | 34 | 5.7 | yes | +| read_exact | 4,194,304 | patched | 54.50 | 51.13 | 58.89 | 63 | 73.4 | yes | +| read_exact | 16,777,216 | baseline | 4040.83 | 2982.84 | 4300.37 | 199 | 4.0 | yes | +| read_exact | 16,777,216 | patched | 127.70 | 123.48 | 140.50 | 256 | 125.3 | yes | +| read_line | 1,048,576 | baseline | 184.48 | 183.79 | 224.74 | 9 | 5.4 | yes | +| read_line | 1,048,576 | patched | 31.77 | 29.30 | 32.83 | 17 | 31.5 | yes | +| read_line | 4,194,304 | baseline | 647.92 | 628.02 | 707.43 | 31 | 6.2 | yes | +| read_line | 4,194,304 | patched | 55.20 | 52.84 | 61.62 | 65 | 72.5 | yes | +| read_line | 16,777,216 | baseline | 990.90 | 858.37 | 1096.40 | 36 | 16.1 | yes | +| read_line | 16,777,216 | patched | 175.04 | 163.06 | 206.02 | 256 | 91.4 | yes | +| write | 1,048,576 | baseline | 323.77 | 303.52 | 324.60 | 16 | 3.1 | yes | +| write | 1,048,576 | patched | 25.99 | 24.99 | 27.01 | 16 | 38.5 | yes | +| write | 4,194,304 | baseline | 1274.77 | 1235.51 | 1278.18 | 63 | 3.1 | yes | +| write | 4,194,304 | patched | 36.32 | 35.04 | 39.85 | 64 | 110.1 | yes | +| write | 16,777,216 | baseline | 5048.17 | 5035.91 | 5061.10 | 249 | 3.2 | yes | +| write | 16,777,216 | patched | 77.21 | 75.11 | 83.05 | 255 | 207.2 | yes | +| splice | 1,048,576 | baseline | 322.88 | 302.60 | 323.78 | 16 | 3.1 | yes | +| splice | 1,048,576 | patched | 27.43 | 26.28 | 37.18 | 16 | 36.5 | yes | +| splice | 4,194,304 | baseline | 1291.49 | 1212.55 | 1292.21 | 64 | 3.1 | yes | +| splice | 4,194,304 | patched | 37.11 | 35.95 | 43.09 | 64 | 107.8 | yes | +| splice | 16,777,216 | baseline | 5151.02 | 5128.69 | 5180.66 | 255 | 3.1 | yes | +| splice | 16,777,216 | patched | 74.04 | 72.86 | 82.28 | 256 | 216.1 | yes | + +Reading the table: + +- **Writes and splices are the cleanest case.** The baseline pays exactly one + 20 ms sleep per pipe capacity (16 per MiB → ~320 ms/MiB → 3.1 MiB/s at every + size). The patched helper takes the same number of waits but each ends when + the peer drains the pipe. +- **Reads show fewer baseline sleeps than pipe fills** (10 per MiB rather than + 16) because the peer keeps writing while the coordinator sleeps, so some + wakeups find more than one block. The patched helper wakes once per block. + The baseline's 16 MiB `read_line` (36 sleeps, 16 MiB/s) is the peer + outrunning a sleeping reader, not the reader being fast; the same reader + sleeps 199 times on `read_exact`. +- The patched numbers still show one wait per 64 KiB. That ping-pong per pipe + capacity is the remaining transport cost (roughly 0.5 ms per block on this + guest, mostly wakeup latency of two processes handing a full pipe back and + forth). Raising the pipe capacity (`F_SETPIPE_SZ`) is the obvious follow-up + lever; it is **not** part of this change and was not measured. + +--- + +## 3. Real daemon: cold `prepare_window` (per repetition) + +Time in seconds; the number in parentheses is blocked waits in that phase +(20 ms sleeps for the baseline, readiness waits for the patched module). + +| records | variant | rep | status | total | spawn | encode | write (waits) | wait (waits) | read (waits) | +|---:|---|---:|---|---:|---:|---:|---:|---:|---:| +| 52,000 | baseline | 1 | prepared | 8.204 | 0.021 | 0.263 | 6.639 (328) | 1.191 (59) | 0.091 (2) | +| 52,000 | patched | 1 | prepared | 1.341 | 0.002 | 0.253 | 0.029 (1695) | 0.998 (50) | 0.059 (2) | +| 52,000 | baseline | 2 | prepared | 8.478 | 0.021 | 0.273 | 7.197 (356) | 0.910 (45) | 0.077 (1) | +| 52,000 | patched | 2 | prepared | 1.336 | 0.002 | 0.253 | 0.038 (1628) | 0.997 (50) | 0.047 (1) | +| 210,000 | baseline | 1 | prepared | 43.823 | 0.021 | 1.010 | 26.894 (1327) | 15.167 (744) | 0.731 (19) | +| 210,000 | patched | 1 | prepared | 13.760 | 0.002 | 1.241 | 0.133 (6705) | 11.997 (590) | 0.388 (7) | +| 210,000 | baseline | 2 | prepared | 39.777 | 0.021 | 1.182 | 27.581 (1362) | 10.737 (530) | 0.255 (1) | +| 210,000 | patched | 2 | prepared | 12.150 | 0.002 | 1.028 | 0.151 (5813) | 10.715 (529) | 0.255 (5) | +| 400,000 | baseline | 1 | fell back | 80.808 | 0.021 | 1.907 | 54.911 (2712) | 23.970 (1174) | — | +| 400,000 | patched | 1 | fell back | 22.856 | 0.006 | 2.298 | 0.363 (12079) | 20.188 (985) | — | + +Parity: at 52k and 210k both variants returned `prepared` with the same +`share_snapshot_sha256` and the same canonical item bytes in every repetition +(the driver checks both). + +Reading the table: + +- **`request_write` is where the baseline's time went**: 6.6-7.2 s of 8.2-8.5 s + at 52k, 26.9-27.6 s of 39.8-43.8 s at 210k, 54.9 s of 80.8 s at 400k. The + patched write takes 29-38 ms, 133-151 ms and 363 ms respectively. The phase + ends when the pipe has accepted the last byte, so up to one pipe capacity + of the request may still be unread by the daemon at that point and lands in + `response_wait` instead. The readiness waits are numerous (the daemon reads + its stdin in small chunks, so the pipe drains and refills often) but each + returns as soon as it can progress. +- **`response_wait` is the largest residual and is not addressed by this + PR**: 0.9-1.2 s at 52k, 10.7-15.2 s at 210k, 20-24 s at 400k, in both + variants. It is elapsed time between the pipe accepting the last request + byte and the envelope's arrival, so it contains whatever request bytes the + daemon had not yet consumed, the daemon's own processing, and the response + becoming readable; it is not a Rust CPU measurement and its run-to-run + spread (10.7 vs 15.2 s at 210k) is not attributed here. The baseline's + sleeps inside it are the coordinator polling an idle pipe every 20 ms while + the daemon works. Rust-side profiling is needed to split it. +- **`request_encode` (1.0-1.2 s at 210k)** is coordinator time outside the + wrapped transport calls, derived by subtraction; it is dominated by the + whole-request `json.dumps` that PR 2 targets but is not an isolated + measurement of it. +- **Cold spawn + handshake** drops from 21 ms to 2 ms: the baseline's handshake + read slept one full 20 ms slice before the daemon's first line arrived. + +**The 400k point is incomplete and its cause is unresolved.** In both +variants `prepare_payout_window` returned `None` after `response_wait` (a +daemon anomaly: the daemon was retired and a caller would fold in-process); +`response_read` never ran. The compiler discards the anomaly text and the +daemon printed nothing to stderr, so the cause was not diagnosed, and the +observation that both variants fell back does not establish that they failed +for the same reason or that the transport is uninvolved. It should be +captured (the envelope's `error`, or the exception the transport raised) +before 400k is used as a release-gate size. The run was stopped after this +first repetition per variant to bound the experiment; the second repetition +and the JSON capture were not produced. + +--- + +## 4. Relation to the incident numbers + +The incident's cold `daemon_prepare` was 47.5-48.8 s at roughly 210k shares +on the production host. On this host the baseline transport spends 26.9-27.6 s +of a 39.8-43.8 s cold 210k prepare sleeping in `request_write`; that fraction +is consistent with the incident's `daemon_prepare` being dominated by transport +sleeps rather than Rust work, but this report does not claim the production +split -- different hardware, different pipe behavior under load, and the +production path also carries the ledger read and record conversion the driver +does not include. + +What this change does establish: after it, the coordinator's pipe loops +contribute well under a second at 210k on Linux; the residual is +`response_wait` (daemon-side processing plus response receipt, not split +here) and the unattributed pre-write coordinator time PR 2 targets; and every +byte, digest and canonical item stream is unchanged. + +--- + +## 5. What is not validated here + +- The production container image and CPython 3.14 (this host runs 3.12.3 and + Docker was not used). +- The end-to-end 210k cold path (ledger read → conversion → daemon → + mirror → first usable Stratum job) and the 20 s recovery target. +- Lease-monitor lateness under this workload; the transport change removes + sleeps from the builder thread and does not touch the lease thread. +- The pipe-capacity follow-up (`F_SETPIPE_SZ`) and the Rust-side profiling + needed to split the 10-12 s `response_wait` at 210k. +- The 400k fallback above. +- Measurements on a host without concurrent unrelated load. diff --git a/tests/perf/window_builder_pipe_latency.py b/tests/perf/window_builder_pipe_latency.py new file mode 100644 index 000000000..d6c0ade6c --- /dev/null +++ b/tests/perf/window_builder_pipe_latency.py @@ -0,0 +1,951 @@ +#!/usr/bin/env python3 +"""Builder pipe transport latency: fixed polling sleeps vs readiness waits. + +Issue #236's cold window builds spent 47.5-48.8 s of a 54-56 s materialization +inside ``daemon_prepare``. That timer covers lock admission, request encoding, +the pipe transport in both directions, the Rust fold, and response handling; +it does not say how much of it is transport. The shipped transport helpers in +``lab/prism/bundle_compiler.py`` slept a fixed 20 ms after every EAGAIN, once +per drained pipe capacity, so the suspicion was that a multi-megabyte +exchange paid far more in idle sleeps than in copying. A synthetic macOS probe +(146 ms vs 2 ms for a 4 MiB read) motivated the change; it is not production +evidence, and this driver exists to produce the Linux numbers. + +Two sections, both running the **shipped callables** of two revisions of the +transport module side by side in one process: + +1. **Raw transport** over real pipes against a Python peer subprocess: + ``_serve_builder_read_exact``, ``_serve_builder_read_line``, + ``_serve_builder_write`` and (Linux) ``_serve_builder_splice_spool``. + Reported per helper and payload size: wall seconds (median / min over + ``--reps``), throughput, **blocked waits** (readiness waits for the + patched module, 20 ms sleeps for the baseline), bytes, and a byte-exact + digest check against the peer. +2. **Real daemon** (``qbit-prism-build-audit-bundle --serve``, prebuilt via + ``--tool-bin-dir`` / ``PRISM_TOOL_BIN_DIR``): one cold ``prepare_window`` + ``full`` round trip through ``BundleCompiler.prepare_payout_window`` at + each ``--daemon-sizes`` window, attributed by wrapping the compiler's own + transport methods: + + =================== ======================================================= + phase what the wall time contains + =================== ======================================================= + ``spawn`` daemon spawn + protocol handshake (line read) + ``request_encode`` total minus every wrapped phase: unattributed + coordinator time outside the transport calls, + dominated by (not isolated to) the whole-request + ``json.dumps`` PR 2 targets + ``request_write`` ``_serve_builder_write`` of the request line: elapsed + until the pipe accepted every byte -- not proof the + daemon consumed them; up to one pipe capacity may + still be unread + ``response_wait`` ``_serve_builder_read_line`` of the envelope: elapsed + from write completion to envelope receipt -- daemon + scheduling and processing (remaining parse, fold, + canonical serialization) plus response readiness and + receipt; wall time, NOT measured Rust CPU. Splitting + it needs Rust-side profiling. + ``response_read`` ``_serve_builder_read_exact`` of the raw canonical + items section + its newline: elapsed receipt of the + bytes the daemon produces after the envelope + =================== ======================================================= + + Both revisions must return the same ``share_snapshot_sha256`` and the same + canonical item bytes; a mismatch is reported as a correctness failure. + +The baseline module is extracted with ``git show REV:lab/prism/bundle_compiler.py`` +and imported under a private name; it resolves its own imports from the +current tree, which is fine for the transport code because the helpers only +depend on the module-level constants beside them. Wait counting hooks the +patched module's ``_PipeReadinessWaiter`` and the baseline's ``time.sleep``; +every peer is a subprocess so nothing else in the process sleeps during a +measurement. + +Self-contained: no database, no network, no coordinator, standard library +only. ``python3 tests/perf/window_builder_pipe_latency.py --baseline-rev +e59bda34016f6dc0a6bc39f9b879b8df64f51f60`` prints the tables; ``--json FILE`` +also captures the structured results. + +This is an **on-demand instrument, not a test**: it asserts no thresholds and +is deliberately not named ``test_*`` so the discovery run never executes it. +Nothing under ``lab/`` is modified. +""" + +from __future__ import annotations + +import argparse +import contextlib +import fcntl +import hashlib +import importlib.util +import json +import os +import platform +import statistics +import subprocess +import sys +import tempfile +import threading +import time +from dataclasses import asdict, dataclass, field +from pathlib import Path +from types import ModuleType +from typing import Any, Callable, Iterator + +if not __package__: + sys.path.insert(0, str(Path(__file__).resolve().parents[2])) + +from lab.prism.share_ledger import ( # noqa: E402 - path fix-up above + DEFAULT_INCREMENTAL_SHARE_WINDOW_PAGE_SIZE, + AcceptedShareRecord, +) + + +REPO_ROOT = Path(__file__).resolve().parents[2] +COMPILER_RELATIVE_PATH = "lab/prism/bundle_compiler.py" +DEFAULT_TOOL_BIN_DIR = REPO_ROOT / "target" / "release" +DAEMON_BINARY = "qbit-prism-build-audit-bundle" +FIXED_NOW_MS = 1_760_000_000_000 +F_GETPIPE_SZ = 1032 + + +# --------------------------------------------------------------------------- +# module loading and wait counting +# --------------------------------------------------------------------------- + + +def load_compiler(label: str, rev: str | None) -> tuple[ModuleType, str]: + """The transport module for one revision (None = the working tree).""" + if rev is None: + import lab.prism.bundle_compiler as module + + head = subprocess.run( + ["git", "-C", str(REPO_ROOT), "rev-parse", "HEAD"], + check=False, + stdout=subprocess.PIPE, + stderr=subprocess.DEVNULL, + text=True, + ).stdout.strip() + dirty = subprocess.run( + ["git", "-C", str(REPO_ROOT), "status", "--porcelain", COMPILER_RELATIVE_PATH], + check=False, + stdout=subprocess.PIPE, + stderr=subprocess.DEVNULL, + text=True, + ).stdout.strip() + return module, f"{head or 'unknown'}{' (working tree, modified)' if dirty else ''}" + resolved = subprocess.run( + ["git", "-C", str(REPO_ROOT), "rev-parse", rev], + check=True, + stdout=subprocess.PIPE, + text=True, + ).stdout.strip() + source = subprocess.run( + ["git", "-C", str(REPO_ROOT), "show", f"{resolved}:{COMPILER_RELATIVE_PATH}"], + check=True, + stdout=subprocess.PIPE, + ).stdout + directory = tempfile.mkdtemp(prefix="prism-236-transport-") + path = Path(directory) / f"bundle_compiler_{label}.py" + path.write_bytes(source) + name = f"prism_bundle_compiler_{label}" + spec = importlib.util.spec_from_file_location(name, path) + assert spec is not None and spec.loader is not None + module = importlib.util.module_from_spec(spec) + sys.modules[name] = module + spec.loader.exec_module(module) + return module, resolved + + +class WaitCounter: + def __init__(self) -> None: + self.count = 0 + + +@contextlib.contextmanager +def counting_blocked_waits(module: ModuleType) -> Iterator[WaitCounter]: + """Count blocked waits taken by ``module``'s transport loops. + + The patched module waits through ``_PipeReadinessWaiter.wait``; the + baseline sleeps through ``time.sleep``, which its loops resolve at call + time, so a process-global hook counts exactly those sleeps as long as + nothing else in the process sleeps meanwhile (every peer is a + subprocess). + """ + counter = WaitCounter() + waiter_class = getattr(module, "_PipeReadinessWaiter", None) + if waiter_class is not None: + + class Counting(waiter_class): # type: ignore[misc,valid-type] + def wait(self, deadline: float) -> bool: + counter.count += 1 + return super().wait(deadline) + + module._PipeReadinessWaiter = Counting + try: + yield counter + finally: + module._PipeReadinessWaiter = waiter_class + return + real_sleep = time.sleep + + def sleep(seconds: float) -> None: + counter.count += 1 + real_sleep(seconds) + + time.sleep = sleep + try: + yield counter + finally: + time.sleep = real_sleep + + +class BenchRuntime: + """The BundleCompilerRuntime port, reduced to counters and no-ops.""" + + signing_seed_hex = "42" * 32 + ledger_attestation_signing_seed_hex = "43" * 32 + + def __init__(self, timeout_seconds: float) -> None: + self.bundle_build_timeout_seconds = timeout_seconds + self._job_build_scheduler_lock = threading.Lock() + self._tip_refresh_metrics_lock = threading.Lock() + self.job_build_worker_counts = { + "starts": 0, + "restarts": 0, + "crashes": 0, + "terminations": 0, + } + self._job_build_worker_restart_pending = False + self.tip_refresh_worker_restarts = 0 + self.tip_refresh_worker_failures = 0 + self._phases: dict[str, float] = {} + + def _ensure_job_cache_state(self) -> None: + pass + + def _ensure_tip_refresh_state(self) -> None: + pass + + def _job_build_checkpoint(self, phase: str, cancellation: Any) -> None: + pass + + def _job_build_phases(self) -> dict[str, float]: + return self._phases + + def prism_payout_policy(self) -> dict[str, object]: + return {} + + def prism_ctv_settlement_config(self, **_kwargs: Any) -> None: + return None + + def _observe_tip_refresh_build_phase(self, name: str, elapsed: float) -> None: + pass + + def _record_tip_refresh_ipc_bytes(self, direction: str, byte_count: int) -> None: + pass + + +class _Control: + """Never instantiated; only its type is consulted.""" + + +class _Serialization: + def __init__(self) -> None: + self.spool_failures = 0 + + def mark_spool_failed(self) -> None: + self.spool_failures += 1 + + +def make_compiler(module: ModuleType, runtime: BenchRuntime, tool_bin_dir: Path | None) -> Any: + tool_command = None + if tool_bin_dir is not None: + + def tool_command(bin_name: str) -> list[str]: + return [str(tool_bin_dir / bin_name)] + + return module.BundleCompiler( + runtime, + superseded_error=RuntimeError, + cancellation_error_types=(RuntimeError,), + build_control_type=_Control, + tool_command=tool_command, + ) + + +# --------------------------------------------------------------------------- +# section 1: raw transport against a peer subprocess +# --------------------------------------------------------------------------- + + +# Writes `total` bytes to stdout in `chunk`-sized blocks (a single line when +# the mode is "line"), pausing `pause` seconds after each block, after a go +# byte arrives on stdin; the digest of what was written goes to stderr. +PEER_WRITE_STDOUT = r""" +import hashlib, os, sys, time +total = int(sys.argv[1]); chunk = int(sys.argv[2]); pause = float(sys.argv[3]) +line = sys.argv[4] == "line" +os.read(0, 1) +digest = hashlib.sha256() +remaining = total +while remaining: + n = min(chunk, remaining) + block = os.urandom(n) + if line: + block = block.replace(b"\n", b"\x00") + digest.update(block) + view = memoryview(block) + while view: + written = os.write(1, view) + view = view[written:] + remaining -= n + if pause: + time.sleep(pause) +if line: + os.write(1, b"\n") +sys.stderr.write(digest.hexdigest() + "\n") +sys.stderr.flush() +""" + +# Reads stdin to EOF in `chunk`-sized reads, pausing `pause` seconds after +# each, then reports " " on stdout. +PEER_READ_STDIN = r""" +import hashlib, os, sys, time +chunk = int(sys.argv[1]); pause = float(sys.argv[2]) +digest = hashlib.sha256(); total = 0 +while True: + data = os.read(0, chunk) + if not data: + break + digest.update(data); total += len(data) + if pause: + time.sleep(pause) +sys.stdout.write(f"{total} {digest.hexdigest()}\n") +sys.stdout.flush() +""" + + +@dataclass +class TransportSample: + seconds: float + waits: int + bytes: int + ok: bool + + +def _far_deadline() -> float: + return time.monotonic() + 600.0 + + +def run_read_exact(module: ModuleType, size: int, peer_chunk: int, peer_pause: float) -> TransportSample: + peer = subprocess.Popen( + [sys.executable, "-c", PEER_WRITE_STDOUT, str(size), str(peer_chunk), str(peer_pause), "raw"], + stdin=subprocess.PIPE, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + ) + assert peer.stdin is not None and peer.stdout is not None + os.set_blocking(peer.stdout.fileno(), False) + client = module._ServeBuilderClient(process=peer) + compiler = make_compiler(module, BenchRuntime(600.0), None) + with counting_blocked_waits(module) as counter: + peer.stdin.write(b"g") + peer.stdin.flush() + started = time.perf_counter() + data = compiler._serve_builder_read_exact(client, size, _far_deadline(), None, None) + elapsed = time.perf_counter() - started + ok = len(data) == size and hashlib.sha256(data).hexdigest() == _writer_digest(peer) + return TransportSample(elapsed, counter.count, size, ok) + + +def run_read_line(module: ModuleType, size: int, peer_chunk: int, peer_pause: float) -> TransportSample: + peer = subprocess.Popen( + [sys.executable, "-c", PEER_WRITE_STDOUT, str(size), str(peer_chunk), str(peer_pause), "line"], + stdin=subprocess.PIPE, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + ) + assert peer.stdin is not None and peer.stdout is not None + os.set_blocking(peer.stdout.fileno(), False) + client = module._ServeBuilderClient(process=peer) + compiler = make_compiler(module, BenchRuntime(600.0), None) + with counting_blocked_waits(module) as counter: + peer.stdin.write(b"g") + peer.stdin.flush() + started = time.perf_counter() + line = compiler._serve_builder_read_line(client, _far_deadline(), None, None) + elapsed = time.perf_counter() - started + ok = len(line) == size and hashlib.sha256(line).hexdigest() == _writer_digest(peer) + return TransportSample(elapsed, counter.count, size + 1, ok) + + +def _spawn_reader(peer_chunk: int, peer_pause: float) -> subprocess.Popen[bytes]: + return subprocess.Popen( + [sys.executable, "-c", PEER_READ_STDIN, str(peer_chunk), str(peer_pause)], + stdin=subprocess.PIPE, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + ) + + +def _reader_result(peer: subprocess.Popen[bytes]) -> tuple[int, str]: + """The reader peer's " " report, after its stdin closed.""" + assert peer.stdin is not None and peer.stdout is not None and peer.stderr is not None + peer.stdin.close() + stdout = peer.stdout.read() + stderr = peer.stderr.read() + peer.wait(timeout=120) + if peer.returncode != 0: + raise RuntimeError(f"reader peer failed: {stderr.decode(errors='replace')}") + total, digest = stdout.decode().split() + return int(total), digest + + +def _writer_digest(peer: subprocess.Popen[bytes]) -> str: + """The writer peer's digest line, after its payload was fully read.""" + assert peer.stdin is not None and peer.stderr is not None + peer.stdin.close() + stderr = peer.stderr.read() + peer.wait(timeout=120) + if peer.returncode != 0: + raise RuntimeError(f"writer peer failed: {stderr.decode(errors='replace')}") + return stderr.decode().strip() + + +def run_write(module: ModuleType, size: int, peer_chunk: int, peer_pause: float) -> TransportSample: + payload = os.urandom(size) + expected = hashlib.sha256(payload).hexdigest() + peer = _spawn_reader(peer_chunk, peer_pause) + assert peer.stdin is not None + os.set_blocking(peer.stdin.fileno(), False) + client = module._ServeBuilderClient(process=peer) + compiler = make_compiler(module, BenchRuntime(600.0), None) + with counting_blocked_waits(module) as counter: + started = time.perf_counter() + written = compiler._serve_builder_write(client, payload, _far_deadline(), None, None) + elapsed = time.perf_counter() - started + total, digest = _reader_result(peer) + ok = written == size and total == size and digest == expected + return TransportSample(elapsed, counter.count, size, ok) + + +def run_splice(module: ModuleType, size: int, peer_chunk: int, peer_pause: float) -> TransportSample: + payload = os.urandom(size) + expected = hashlib.sha256(payload).hexdigest() + spool = tempfile.TemporaryFile() + spool.write(payload) + spool.flush() + peer = _spawn_reader(peer_chunk, peer_pause) + assert peer.stdin is not None + os.set_blocking(peer.stdin.fileno(), False) + client = module._ServeBuilderClient(process=peer) + compiler = make_compiler(module, BenchRuntime(600.0), None) + serialization = _Serialization() + try: + with counting_blocked_waits(module) as counter: + started = time.perf_counter() + moved = compiler._serve_builder_splice_spool( + client, serialization, spool, size, _far_deadline(), None, None + ) + elapsed = time.perf_counter() - started + finally: + spool.close() + total, digest = _reader_result(peer) + ok = moved == size and total == size and digest == expected and serialization.spool_failures == 0 + return TransportSample(elapsed, counter.count, size, ok) + + +RAW_HELPERS: dict[str, Callable[[ModuleType, int, int, float], TransportSample]] = { + "read_exact": run_read_exact, + "read_line": run_read_line, + "write": run_write, +} +if hasattr(os, "splice"): + RAW_HELPERS["splice"] = run_splice + + +@dataclass +class RawResult: + helper: str + variant: str + bytes: int + reps: int + median_seconds: float + min_seconds: float + max_seconds: float + median_waits: float + mib_per_second: float + ok: bool + samples: list[dict[str, Any]] = field(default_factory=list) + + +def run_raw_section( + variants: list[tuple[str, ModuleType]], + sizes: list[int], + reps: int, + peer_chunk: int, + peer_pause: float, + log: Callable[[str], None], +) -> list[RawResult]: + results: list[RawResult] = [] + for helper, runner in RAW_HELPERS.items(): + for size in sizes: + samples: dict[str, list[TransportSample]] = {label: [] for label, _ in variants} + # Alternate variants per repetition so host drift lands on both. + for rep in range(reps): + for label, module in variants: + sample = runner(module, size, peer_chunk, peer_pause) + samples[label].append(sample) + log( + f" {helper:10s} {size / (1 << 20):8.2f} MiB {label:9s} rep {rep + 1}:" + f" {sample.seconds * 1000:9.2f} ms, {sample.waits:6d} waits," + f" {'ok' if sample.ok else 'MISMATCH'}" + ) + for label, _module in variants: + seconds = [s.seconds for s in samples[label]] + median = statistics.median(seconds) + results.append( + RawResult( + helper=helper, + variant=label, + bytes=size, + reps=reps, + median_seconds=median, + min_seconds=min(seconds), + max_seconds=max(seconds), + median_waits=statistics.median(s.waits for s in samples[label]), + mib_per_second=(size / (1 << 20)) / median if median > 0 else float("inf"), + ok=all(s.ok for s in samples[label]), + samples=[asdict(s) for s in samples[label]], + ) + ) + return results + + +# --------------------------------------------------------------------------- +# section 2: the real daemon's cold prepare_window round trip +# --------------------------------------------------------------------------- + + +def build_records(count: int, *, miners: int, rigs_per_miner: int) -> list[AcceptedShareRecord]: + """Records shaped like the ledger's rows (production share_id form).""" + programs = [hashlib.sha256(f"bench-miner-{index}".encode()).hexdigest() for index in range(miners)] + records: list[AcceptedShareRecord] = [] + for index in range(count): + miner = index % miners + block_hash = hashlib.sha256(f"{index}".encode()).hexdigest() + records.append( + AcceptedShareRecord( + share_seq=index + 1, + share_id=f"bench-miner-{miner}.rig{index % rigs_per_miner}:{block_hash}", + miner_id=f"bench-miner-{miner}", + order_key=f"bench-miner-{miner:06d}", + p2mr_program_hex=programs[miner], + share_difficulty=16384, + network_difficulty=226646186, + template_height=9, + job_id=f"bench-job-{index + 1}", + job_issued_at_ms=FIXED_NOW_MS - (count - index) * 1_000, + accepted_at_ms=FIXED_NOW_MS - (count - index) * 1_000, + ntime=FIXED_NOW_MS // 1000 - (count - index), + ) + ) + return records + + +@dataclass +class PhaseSample: + seconds: float = 0.0 + bytes: int = 0 + waits: int = 0 + calls: int = 0 + + +@dataclass +class DaemonSample: + variant: str + records: int + status: str + total_seconds: float + phases: dict[str, PhaseSample] + request_bytes: int + response_bytes: int + digest: str | None + items_sha256: str | None + record_count: int + + +def run_daemon_prepare( + module: ModuleType, + label: str, + records_json: list[dict[str, object]], + *, + anchor_job_issued_at_ms: int, + window_weight: int, + tool_bin_dir: Path, + timeout_seconds: float, +) -> DaemonSample: + runtime = BenchRuntime(timeout_seconds) + compiler = make_compiler(module, runtime, tool_bin_dir) + phases: dict[str, PhaseSample] = { + name: PhaseSample() + for name in ("spawn", "request_encode", "request_write", "response_wait", "response_read") + } + counter_holder: list[WaitCounter] = [] + state = {"in_spawn": False} + + def timed(name: str, method: Callable[..., Any], byte_count: Callable[[tuple, Any], int]) -> Callable[..., Any]: + def wrapper(*args: Any, **kwargs: Any) -> Any: + if state["in_spawn"] and name != "spawn": + # The handshake line read runs inside the spawn wrapper and + # is already attributed to spawn. + return method(*args, **kwargs) + counter = counter_holder[0] + before = counter.count + started = time.perf_counter() + try: + result = method(*args, **kwargs) + finally: + phase = phases[name] + phase.seconds += time.perf_counter() - started + phase.waits += counter.count - before + phase.calls += 1 + phase.bytes += byte_count(args, result) + return result + + return wrapper + + def spawn_wrapper(*args: Any, **kwargs: Any) -> Any: + state["in_spawn"] = True + try: + return timed_spawn(*args, **kwargs) + finally: + state["in_spawn"] = False + + timed_spawn = timed("spawn", compiler._spawn_serve_builder_locked, lambda args, result: 0) + compiler._spawn_serve_builder_locked = spawn_wrapper + compiler._serve_builder_write = timed( + "request_write", compiler._serve_builder_write, lambda args, result: int(result) + ) + compiler._serve_builder_read_line = timed( + "response_wait", compiler._serve_builder_read_line, lambda args, result: len(result) + 1 + ) + compiler._serve_builder_read_exact = timed( + "response_read", compiler._serve_builder_read_exact, lambda args, result: len(result) + ) + outcome = None + error: str | None = None + with counting_blocked_waits(module) as counter: + counter_holder.append(counter) + started = time.perf_counter() + try: + outcome = compiler.prepare_payout_window( + mode="full", + records_json=records_json, + anchor_job_issued_at_ms=anchor_job_issued_at_ms, + append_invalidation_epoch=0, + window_weight=window_weight, + page_size=DEFAULT_INCREMENTAL_SHARE_WINDOW_PAGE_SIZE, + ) + except Exception as exc: # noqa: BLE001 - reported, not raised + error = f"{type(exc).__name__}: {exc}" + total = time.perf_counter() - started + compiler.shutdown_serve_builder() + measured = sum( + phases[name].seconds for name in ("spawn", "request_write", "response_wait", "response_read") + ) + phases["request_encode"].seconds = max(0.0, total - measured) + phases["request_encode"].calls = 1 + if error is not None: + status = f"error: {error}" + elif outcome is None: + status = "none (daemon anomaly, fell back)" + else: + status = str(outcome.status) + items = getattr(outcome, "window_items", None) if outcome is not None else None + return DaemonSample( + variant=label, + records=len(records_json), + status=status, + total_seconds=total, + phases=phases, + request_bytes=phases["request_write"].bytes, + response_bytes=phases["response_wait"].bytes + phases["response_read"].bytes, + digest=getattr(outcome, "share_snapshot_sha256", None) if outcome is not None else None, + items_sha256=hashlib.sha256(items).hexdigest() if items is not None else None, + record_count=int(getattr(outcome, "record_count", 0) or 0) if outcome is not None else 0, + ) + + +def run_daemon_section( + variants: list[tuple[str, ModuleType]], + sizes: list[int], + reps: int, + *, + miners: int, + rigs_per_miner: int, + tool_bin_dir: Path, + timeout_seconds: float, + log: Callable[[str], None], +) -> list[dict[str, Any]]: + results: list[dict[str, Any]] = [] + for size in sizes: + log(f" building {size} records ...") + records = build_records(size, miners=miners, rigs_per_miner=rigs_per_miner) + anchor = int(records[-1].job_issued_at_ms) + weight = sum(int(record.share_difficulty) for record in records) + records_json = [record.to_prism_json() for record in records] + samples: dict[str, list[DaemonSample]] = {label: [] for label, _ in variants} + for rep in range(reps): + for label, module in variants: + sample = run_daemon_prepare( + module, + label, + records_json, + anchor_job_issued_at_ms=anchor, + window_weight=weight, + tool_bin_dir=tool_bin_dir, + timeout_seconds=timeout_seconds, + ) + samples[label].append(sample) + phase_text = ", ".join( + f"{name} {phase.seconds:.3f}s/{phase.waits}w" + for name, phase in sample.phases.items() + ) + log( + f" daemon {size:7d} {label:9s} rep {rep + 1}: {sample.status}," + f" total {sample.total_seconds:.3f}s [{phase_text}]" + ) + digests = {s.digest for label in samples for s in samples[label]} + items = {s.items_sha256 for label in samples for s in samples[label]} + parity = len(digests) == 1 and None not in digests and len(items) == 1 and None not in items + for label, _module in variants: + runs = samples[label] + by_total = sorted(runs, key=lambda s: s.total_seconds) + median_run = by_total[len(by_total) // 2] + results.append( + { + "records": size, + "variant": label, + "reps": reps, + "statuses": [s.status for s in runs], + "parity_across_variants": parity, + "digest": median_run.digest, + "record_count": median_run.record_count, + "request_bytes": median_run.request_bytes, + "response_bytes": median_run.response_bytes, + "median_total_seconds": statistics.median(s.total_seconds for s in runs), + "min_total_seconds": min(s.total_seconds for s in runs), + "median_run_phases": { + name: asdict(phase) for name, phase in median_run.phases.items() + }, + "runs": [ + { + "status": s.status, + "total_seconds": s.total_seconds, + "phases": {name: asdict(phase) for name, phase in s.phases.items()}, + } + for s in runs + ], + } + ) + return results + + +# --------------------------------------------------------------------------- +# reporting +# --------------------------------------------------------------------------- + + +def machine_info() -> dict[str, Any]: + read_end, write_end = os.pipe() + try: + pipe_capacity = fcntl.fcntl(read_end, F_GETPIPE_SZ) + except OSError: + pipe_capacity = None + finally: + os.close(read_end) + os.close(write_end) + try: + load = os.getloadavg() + except OSError: + load = None + return { + "platform": platform.platform(), + "python": sys.version.split()[0], + "implementation": platform.python_implementation(), + "cpu_count": os.cpu_count(), + "pipe_capacity_bytes": pipe_capacity, + "load_average": load, + "splice": hasattr(os, "splice"), + } + + +def render_raw(results: list[RawResult]) -> str: + lines = [ + "| helper | bytes | variant | median ms | min ms | max ms | median waits | MiB/s | exact |", + "|---|---:|---|---:|---:|---:|---:|---:|---|", + ] + for result in results: + lines.append( + f"| {result.helper} | {result.bytes:,} | {result.variant} |" + f" {result.median_seconds * 1000:.2f} | {result.min_seconds * 1000:.2f} |" + f" {result.max_seconds * 1000:.2f} | {result.median_waits:.0f} |" + f" {result.mib_per_second:.1f} | {'yes' if result.ok else 'NO'} |" + ) + return "\n".join(lines) + + +def render_daemon(results: list[dict[str, Any]]) -> str: + lines = [ + "| records | variant | status | total s | spawn s | encode s | write s (waits) | wait s (waits) | read s (waits) | req MiB | resp MiB | parity |", + "|---:|---|---|---:|---:|---:|---:|---:|---:|---:|---:|---|", + ] + for result in results: + phases = result["median_run_phases"] + + def cell(name: str) -> str: + phase = phases[name] + return f"{phase['seconds']:.3f} ({phase['waits']})" + + lines.append( + f"| {result['records']:,} | {result['variant']} | {result['statuses'][-1]} |" + f" {result['median_total_seconds']:.3f} | {phases['spawn']['seconds']:.3f} |" + f" {phases['request_encode']['seconds']:.3f} | {cell('request_write')} |" + f" {cell('response_wait')} | {cell('response_read')} |" + f" {result['request_bytes'] / (1 << 20):.1f} | {result['response_bytes'] / (1 << 20):.1f} |" + f" {'yes' if result['parity_across_variants'] else 'NO'} |" + ) + return "\n".join(lines) + + +def parse_sizes(text: str) -> list[int]: + return [int(float(item)) for item in text.split(",") if item.strip()] + + +def main(argv: list[str] | None = None) -> int: + parser = argparse.ArgumentParser( + description=__doc__, + formatter_class=argparse.RawDescriptionHelpFormatter, + ) + parser.add_argument( + "--baseline-rev", + default=None, + help="git revision whose lab/prism/bundle_compiler.py is measured beside the working tree", + ) + parser.add_argument( + "--sizes-mib", + default="1,4,16", + help="raw transport payload sizes in MiB (default 1,4,16)", + ) + parser.add_argument("--reps", type=int, default=5, help="raw transport repetitions (default 5)") + parser.add_argument( + "--peer-chunk", + type=int, + default=1 << 16, + help="peer subprocess write/read chunk in bytes (default 65536)", + ) + parser.add_argument( + "--peer-pause", + type=float, + default=0.0, + help="seconds the peer pauses after each chunk (default 0: pipe-bound)", + ) + parser.add_argument( + "--daemon-sizes", + default="52000,210000", + help="window sizes for the real-daemon cold prepare (default 52000,210000)", + ) + parser.add_argument("--daemon-reps", type=int, default=2, help="cold prepares per size (default 2)") + parser.add_argument("--miners", type=int, default=64, help="distinct miners in the window (default 64)") + parser.add_argument("--rigs-per-miner", type=int, default=8, help="rig suffixes per miner (default 8)") + parser.add_argument( + "--tool-bin-dir", + default=os.environ.get("PRISM_TOOL_BIN_DIR") or str(DEFAULT_TOOL_BIN_DIR), + help="directory holding a prebuilt qbit-prism-build-audit-bundle (default $PRISM_TOOL_BIN_DIR or target/release)", + ) + parser.add_argument("--daemon-timeout", type=float, default=900.0, help="per-prepare budget in seconds") + parser.add_argument("--skip-raw", action="store_true", help="skip the raw transport section") + parser.add_argument("--skip-daemon", action="store_true", help="skip the real-daemon section") + parser.add_argument("--json", default=None, help="write structured results to this file") + parser.add_argument("--quiet", action="store_true", help="suppress per-repetition progress lines") + args = parser.parse_args(argv) + + def log(text: str) -> None: + if not args.quiet: + print(text, flush=True) + + variants: list[tuple[str, ModuleType]] = [] + revisions: dict[str, str] = {} + if args.baseline_rev: + module, resolved = load_compiler("baseline", args.baseline_rev) + variants.append(("baseline", module)) + revisions["baseline"] = resolved + module, resolved = load_compiler("patched", None) + variants.append(("patched", module)) + revisions["patched"] = resolved + + info = machine_info() + print("# window_builder_pipe_latency") + print() + for key, value in info.items(): + print(f"- {key}: {value}") + for label, resolved in revisions.items(): + print(f"- {label}: {resolved}") + print() + + output: dict[str, Any] = { + "machine": info, + "revisions": revisions, + "arguments": vars(args), + } + + if not args.skip_raw: + sizes = [int(size * (1 << 20)) for size in (float(s) for s in args.sizes_mib.split(","))] + print(f"## Raw transport (peer chunk {args.peer_chunk} B, peer pause {args.peer_pause} s, {args.reps} reps)") + print() + raw = run_raw_section(variants, sizes, args.reps, args.peer_chunk, args.peer_pause, log) + print() + print(render_raw(raw)) + print() + output["raw"] = [asdict(result) for result in raw] + + if not args.skip_daemon: + tool_bin_dir = Path(args.tool_bin_dir) + binary = tool_bin_dir / DAEMON_BINARY + if not (binary.is_file() and os.access(binary, os.X_OK)): + message = ( + f"real-daemon section skipped: {binary} is not an executable;" + " build it with `cargo build --release -p qbit-prism --bin" + f" {DAEMON_BINARY}` or point --tool-bin-dir / PRISM_TOOL_BIN_DIR at a build" + ) + print(f"## Real daemon\n\n{message}\n") + output["daemon_skipped"] = message + else: + sizes = parse_sizes(args.daemon_sizes) + print(f"## Real daemon cold prepare_window ({binary}, {args.daemon_reps} reps)") + print() + daemon = run_daemon_section( + variants, + sizes, + args.daemon_reps, + miners=args.miners, + rigs_per_miner=args.rigs_per_miner, + tool_bin_dir=tool_bin_dir, + timeout_seconds=args.daemon_timeout, + log=log, + ) + print() + print(render_daemon(daemon)) + print() + output["daemon"] = daemon + + if args.json: + Path(args.json).write_text(json.dumps(output, indent=2, default=str) + "\n") + print(f"wrote {args.json}") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/tests/perf/window_ledger_decode_latency.md b/tests/perf/window_ledger_decode_latency.md new file mode 100644 index 000000000..3dbdfc110 --- /dev/null +++ b/tests/perf/window_ledger_decode_latency.md @@ -0,0 +1,219 @@ +# Payout-window database decode latency (#236, PR 1) + +Companion to `tests/perf/window_ledger_decode_latency.py`. Measures the +database/decode phase of the coordinator's payout-window reads -- +`snapshot_at_job_issue` (bounded and unbounded) and +`snapshot_between_job_issues` -- before and after the per-row decoding change +in PR 1 of the #236 plan, on both ledger backends, at the incident window size +(210k shares) and the stress size (400k), with a wake-lateness monitor thread +beside every call. + +**Scope of the claim.** This is the ledger read only: admission, statement +execution, result receipt, row decoding and record conversion, returned as +the same `list[AcceptedShareRecord]`. It says nothing about the Rust window +builder, the serialization paths (PR 2), the pipe transport (PR 3), the lease +heartbeat itself, or the 24-hour production soak. The monitor thread is a +proxy for the lease monitor's wake lateness -- it measures whether another +Python thread can get the interpreter while the read runs -- and its numbers +are *scheduling symptoms*, not a proof of which C call held the GIL. The GC +probe in section 4 is the only attribution in this document that separates +one cause from another. + +**Provenance.** Measured 2026-09-06 on `alexdevbox1`, a **shared host**: +three Claude agent sessions for the #236 lanes plus other long-lived +sessions were resident, and the 1-minute load average sat between 1.5 and +3.8 (8 CPUs) during the runs, most of it this benchmark and its PostgreSQL +backend. Nothing here was measured in isolation. Baseline and patched +passes ran back to back in the same environment on the same fixture; treat +wall-time differences below ~10% as noise and read the lateness columns as +the signal. + +- Baseline: `lab.prism.share_ledger` from a detached checkout of + `e59bda34016f6dc0a6bc39f9b879b8df64f51f60` (the plan's baseline and the + branch tip when this lane started). +- Patched: the working tree of this PR before commit (the JSON therefore + also records `e59bda3` as the tree's HEAD). The only change made after the + run was the final-partial-batch deadline recheck (two extra clock reads + per read), which cannot move these figures. +- Python 3.12.3 (system), psycopg 3.2.13 (`psycopg[binary]`) in a scratch + venv for the native backend, `/usr/bin/psql` 16 for the subprocess + backend, PostgreSQL 16.15 over a Unix socket on a local cluster + (`port 55436`), not the deployment's container image or CPython 3.14. +- Fixture: generated, production-shaped rows (`username.rig:block_hash_hex` + share ids, 62-character payout addresses as `miner_id`/`order_key`, + 32-byte P2MR programs, `share_difficulty` 16384, `network_difficulty` + 226646186, one job id per 64 shares, `stale-grace` on ~1% of rows, eight + miners with the scale gate's skew). `--reps 3`, one warm-up per read + shape. Command lines are in the driver's docstring; the two JSON captures + and this comparison were produced by `--json` and `--compare`. + +## 1. Headline + +**The whole-window decode call is gone from both backends, and the worst +monitor wake lateness during a read drops by roughly 3-4x at both sizes.** +No wake-up later than 500 ms was observed with the patch, against three per +read (one per rep, i.e. every rep) at the baseline. At 210k shares the +patched worst case is 176-260 ms; at 400k it is 342-484 ms. + +**The residual is CPython's cyclic GC, not the decode.** With automatic GC +disabled around the same patched read, the worst wake-up is 11-12 ms +(section 4). The remaining 100-500 ms stalls are generation-2 collections +triggered by allocating the 210k-400k record objects the read returns; +they exist at the baseline too, underneath its single decode call. Fixing +them is a process-wide GC policy decision outside this lane (section 6). + +Server-side time did not regress: every per-row statement is as fast or +faster than its `json_agg` predecessor (the delta halves), with the same +index scans and the aggregate node replaced by a sort. Peak RSS of the +benchmark process fell by ~43%, from 1.0 GB to 0.57 GB at 210k and from +1.87 GB to 1.05 GB at 400k, because no whole-window JSON text and no +whole-window list of dicts exist at once. Wall time is mixed: the psql +backend and the delta read get faster (10-25%), while the native bounded and +unbounded reads get 8-11% slower and burn ~30% more caller-thread CPU, +because psycopg now runs its JSON loader once per row through Python instead +of one C decode of the aggregate. That trade is deliberate: the extra CPU is +spread across 400+ GIL-switchable batches instead of one uninterruptible +call. + +## 2. Results + +### 210,000 shares + +| backend | read | records | wall median (ms) base -> patch | wall max (ms) base -> patch | monitor max lateness (ms) base -> patch | wake-ups > 100 ms base -> patch | wake-ups > 250 ms base -> patch | wake-ups > 500 ms base -> patch | +|---|---|---:|---:|---:|---:|---:|---:|---:| +| native | bounded | 210,000 | 6207 -> 6695 | 6470 -> 6761 | 949 -> 255 | 15 -> 8 | 3 -> 1 | 3 -> 0 | +| native | delta | 210,000 | 5536 -> 4598 | 5576 -> 4643 | 830 -> 260 | 15 -> 7 | 3 -> 1 | 3 -> 0 | +| native | unbounded | 210,000 | 5357 -> 5607 | 5522 -> 5794 | 827 -> 222 | 14 -> 7 | 3 -> 0 | 3 -> 0 | +| psql | bounded | 210,000 | 6985 -> 6161 | 7248 -> 6337 | 852 -> 204 | 17 -> 6 | 3 -> 0 | 3 -> 0 | +| psql | delta | 210,000 | 6032 -> 4460 | 6500 -> 4491 | 752 -> 200 | 17 -> 6 | 3 -> 0 | 3 -> 0 | +| psql | unbounded | 210,000 | 6448 -> 5315 | 6570 -> 5627 | 794 -> 176 | 18 -> 6 | 3 -> 0 | 3 -> 0 | + +| read | server planning+execution (ms) base -> patch | statement uses json_agg base -> patch | +|---|---:|---| +| bounded | 2859 -> 2708 | True -> False | +| delta | 2179 -> 884 | True -> False | +| unbounded | 2058 -> 1807 | True -> False | + +| backend | read | caller-thread CPU median (ms) base -> patch | process RSS high-water (MB) base -> patch | +|---|---|---:|---:| +| native | bounded | 3122 -> 4054 | 994 -> 571 | +| native | delta | 3047 -> 3832 | 1001 -> 571 | +| native | unbounded | 3039 -> 4017 | 1000 -> 571 | +| psql | bounded | 3439 -> 3088 | 1001 -> 571 | +| psql | delta | 3267 -> 3024 | 1001 -> 571 | +| psql | unbounded | 3446 -> 2956 | 1001 -> 571 | + +### 400,000 shares + +| backend | read | records | wall median (ms) base -> patch | wall max (ms) base -> patch | monitor max lateness (ms) base -> patch | wake-ups > 100 ms base -> patch | wake-ups > 250 ms base -> patch | wake-ups > 500 ms base -> patch | +|---|---|---:|---:|---:|---:|---:|---:|---:| +| native | bounded | 400,000 | 11892 -> 13156 | 12249 -> 15200 | 1643 -> 484 | 19 -> 9 | 11 -> 9 | 3 -> 0 | +| native | delta | 400,000 | 10403 -> 8844 | 10710 -> 9017 | 1586 -> 429 | 19 -> 8 | 10 -> 8 | 3 -> 0 | +| native | unbounded | 400,000 | 10210 -> 11108 | 10331 -> 11451 | 1516 -> 465 | 20 -> 9 | 10 -> 9 | 3 -> 0 | +| psql | bounded | 400,000 | 13039 -> 12403 | 13050 -> 12539 | 1500 -> 380 | 26 -> 7 | 13 -> 7 | 3 -> 0 | +| psql | delta | 400,000 | 11514 -> 8587 | 11820 -> 8928 | 1420 -> 342 | 25 -> 7 | 10 -> 6 | 3 -> 0 | +| psql | unbounded | 400,000 | 11277 -> 10734 | 11452 -> 10885 | 1479 -> 406 | 26 -> 7 | 13 -> 6 | 3 -> 0 | + +| read | server planning+execution (ms) base -> patch | statement uses json_agg base -> patch | +|---|---:|---| +| bounded | 5555 -> 5179 | True -> False | +| delta | 4167 -> 1766 | True -> False | +| unbounded | 4120 -> 3364 | True -> False | + +| backend | read | caller-thread CPU median (ms) base -> patch | process RSS high-water (MB) base -> patch | +|---|---|---:|---:| +| native | bounded | 5809 -> 7934 | 1856 -> 1043 | +| native | delta | 5806 -> 7259 | 1865 -> 1047 | +| native | unbounded | 5664 -> 7779 | 1864 -> 1044 | +| psql | bounded | 6288 -> 5987 | 1865 -> 1046 | +| psql | delta | 5999 -> 5865 | 1865 -> 1046 | +| psql | unbounded | 6055 -> 6128 | 1865 -> 1046 | + +Columns: *wall* is the call as the coordinator sees it (admission included; +the gate was uncontended); *monitor max lateness* is the worst single +wake-up of the 10 ms sleeper across the three reps; the *wake-ups* columns +count wake-ups later than the threshold summed over the three reps (each rep +takes ~450-1300 samples); *server* is `EXPLAIN (ANALYZE)` planning plus +execution of the statement the ledger actually sent, so *wall - server* is +the client-side share (transfer, decode, conversion); *caller-thread CPU* is +`time.thread_time()` of the reading thread; *RSS high-water* is `VmHWM` of +the benchmark process after the read (monotonic across a run, so the 400k +figure includes the 210k pass). + +## 3. Query plans + +Captured from the statements the ledger sent (`statement_sha256_prefix` in +the JSON identifies each). The selection CTEs are byte-identical between +baseline and patch; only the outer projection changed. + +- *bounded*: `Aggregate -> Recursive Union -> ... Index Scan[qbit_share_ledger_pkey] (LIMIT 4096 pages)` + became `Sort -> Recursive Union -> ...`. Same 52 (210k) / 98 (400k) page + steps, same index. 2859 -> 2708 ms at 210k, 5555 -> 5179 ms at 400k. +- *unbounded*: `Aggregate -> Index Scan[pkey]` became `Index Scan[pkey]` + (the ORDER BY is satisfied by the index). 2058 -> 1807 ms, 4120 -> 3364 ms. +- *delta*: `Aggregate -> Gather Merge -> Sort -> Append(...accepted_recent_idx...)` + became `Gather Merge -> Sort -> Result -> Append(...)`, same parallel + branches. 2179 -> 884 ms, 4167 -> 1766 ms: the aggregate previously + serialized the whole result on the leader. + +## 4. Attributing the residual stalls + +`gc_probe.py` (kept in the session scratchpad, not committed; it reuses the +driver's fixture and monitor) ran the patched native bounded read at 210k +three ways, two reps each, recording every collection through +`gc.callbacks`: + +| GC mode during the read | wall (ms) | monitor max lateness (ms) | collections | gen-2 collections | worst single collection (ms) | GC total (ms) | +|---|---:|---:|---:|---:|---:|---:| +| default | 6185, 6120 | 108, 114 | 412 | 4 | 111, 118 | 457, 431 | +| `gc.freeze()` before the read | 6372, 5994 | 104, 92 | 412 | 4 | 103, 98 | 387, 374 | +| `gc.disable()` during the read | 5458, 5664 | 11, 12 | 2 | 1 | 26, 17 | 26, 17 | + +Every wake-up later than 100 ms in the default rows coincides with a gen-2 +collection of the same duration; with collection disabled the same code +path, same fixture and same batch size shows nothing above 12 ms and runs +~10% faster. `gc.freeze()` helps little because the objects being traversed +are the freshly built records themselves, not the pre-existing heap. The +benchmark's higher patched maxima (255 ms at 210k, 484 ms at 400k) are the +same mechanism in a process that had already accumulated more live objects +(both backends, both sizes, three reps each) and on a busier host; the +count of gen-2 collections per read grows with the number of live tracked +objects, and each one traverses all of them. + +This is measurement, not a fix. Changing collection policy is process-wide +and affects every other allocation-heavy path in the coordinator (PR 2's +serialization included), so it is recorded here as the residual for the +lease-latency gate rather than changed in this lane. + +## 5. What this does and does not establish + +- Established: neither production backend builds a whole-window JSON value + for these two reads; individual decode calls are bounded (512 rows); the + worst observed wake lateness during the read fell 3-4x; server time and + peak RSS fell; results are byte-identical to the `json_agg` oracle on both + backends (the PostgreSQL gates). +- Not established: the 250 ms engineering target at 400k (worst observed + 342-484 ms, all GC); any figure on the deployment image, CPython 3.14 or + psycopg pinned by the deployment; the lease heartbeat's own behaviour; + first-usable-work time; anything about the Rust builder or the + serialization/transport lanes; the 24-hour soak. The dedicated-environment + release gates in the plan remain to be run with all three PRs applied. +- Caveats: shared host (above); three reps; the native backend's fixed + ~30% extra caller-thread CPU is a real cost the coordinator pays on the + reading thread; the psql backend still spools the whole result to a + temporary file (page cache, not heap) and psycopg still buffers the raw + result in libpq, exactly as before. + +## 6. Follow-ups suggested by the data + +1. Decide GC policy at the coordinator level with all three lanes in view: + candidates are `gc.freeze()` after startup plus raised thresholds, or + suspending automatic collection around the bulk window read/build with an + explicit collection at a point where the lease monitor is not waiting. + Measure with the same monitor; the probe above is the template. +2. If native-backend CPU matters more than the psql path, register a + psycopg loader for the `json` OID that returns the raw text and decode + with `json.loads` directly, removing one Python-level indirection per row. +3. Re-run this driver in the dedicated production-image environment with + the plan's 210k/400k gates, alongside the real heartbeat, once PRs 2 and + 3 land. diff --git a/tests/perf/window_ledger_decode_latency.py b/tests/perf/window_ledger_decode_latency.py new file mode 100644 index 000000000..617e04e40 --- /dev/null +++ b/tests/perf/window_ledger_decode_latency.py @@ -0,0 +1,593 @@ +#!/usr/bin/env python3 +"""Payout-window database decode latency and monitor responsiveness (#236). + +The 2026-09 union-mainnet incident (issue #236) attributes coordinator lease +exits to the coordinator's own scheduling stalls while a ~210k-share payout +window was being decoded: ``snapshot_at_job_issue`` returned one ``json_agg`` +value the size of the whole window, and decoding it was one C call that never +released the GIL, so the lease monitor thread went unscheduled for as long as +the decode took. PR 1 of the fix plan turns both payout-window reads into one +JSON object per row, decoded in bounded batches on both database backends. + +This driver measures that database/decode phase alone -- ledger admission, +statement execution, result receipt, row decoding and record conversion -- +for the three read shapes the coordinator issues, on both backends, at the +incident window size and the stress size, with a monitor thread beside it: + +================= ========================================================== +read call +================= ========================================================== +``bounded`` ``snapshot_at_job_issue(anchor, window_weight=)`` +``unbounded`` ``snapshot_at_job_issue(anchor)`` (full history) +``delta`` ``snapshot_between_job_issues(prev, anchor)`` (all rows) +================= ========================================================== + +Per call it records wall time, the calling thread's CPU time, whole-process +CPU time, RSS before/after/high-water, the record count, and the ledger's +own admission/execution attribution. Beside every call a **monitor thread** +sleeps 10 ms in a loop and records how late each wake-up was: the maximum +lateness, and how many wake-ups were more than 100, 250 and 500 ms late. It +is a proxy for the lease monitor's wake lateness, not the lease monitor +itself: the real heartbeat needs a running coordinator and a guard session, +and the quantity that decides whether it can run is exactly this one -- +whether another Python thread can get the interpreter while the read is in +progress. Server-side time is measured separately with ``EXPLAIN (ANALYZE)`` +of the statement the ledger actually sent, so the client-side share of the +wall time (transfer + decode + convert) is attributable as ``wall - server`` +without labelling any of it as PostgreSQL time. + +**Baseline versus patch is the same script on two checkouts**: ``--lab-root`` +selects the tree ``lab.prism.share_ledger`` is imported from, and the result +records that tree's commit. The statements are captured from whatever that +checkout sends (``json_agg`` at the baseline, one object per row after), so +the server-side comparison is of the real statements, not a re-typed copy. + +The fixture is generated, production-shaped and disposable: ``username.rig: +block_hash_hex`` share ids (``share_writer.py``'s form), 62-character payout +addresses as miner id and order key, 32-byte P2MR programs, the live-host +difficulty values ``window_pipeline_gil_scaling.py`` uses, a distinct job id +per 64 shares, a ``stale-grace`` credit policy on ~1% of rows, and miner skew +in the shape ``test-prism-postgres-scale.sh`` seeds. Nothing is exported from +a production ledger. Point it at a disposable database; it truncates the +share ledger between sizes. + +This is an **on-demand instrument, not a test**: it asserts no thresholds and +is not named ``test_*``, so the discovery run never executes it. Example:: + + python3 tests/perf/window_ledger_decode_latency.py \\ + --psql-command "psql -p 55436 -d qbit_bench" \\ + --database-url "postgresql://ubuntu@/qbit_bench?host=/var/run/postgresql&port=55436" \\ + --shares 210000 400000 --reps 3 --json patched.json + python3 tests/perf/window_ledger_decode_latency.py ... --lab-root /path/to/baseline-checkout --json baseline.json +""" + +from __future__ import annotations + +import argparse +import hashlib +import json +import os +import platform +import resource +import statistics +import subprocess +import sys +import threading +import time +from pathlib import Path +from typing import Any, Callable + +REPO_ROOT = Path(__file__).resolve().parents[2] + + +def _install_lab_root(lab_root: Path) -> None: + """Make ``lab`` import from the chosen checkout, before anything imports it.""" + for name in list(sys.modules): + if name == "lab" or name.startswith("lab."): + raise SystemExit("lab.* was imported before --lab-root was applied") + sys.path.insert(0, str(lab_root)) + + +def _git_head(root: Path) -> str | None: + try: + return subprocess.run( + ["git", "-C", str(root), "rev-parse", "HEAD"], + capture_output=True, + text=True, + check=True, + ).stdout.strip() + except (OSError, subprocess.CalledProcessError): + return None + + +# -------------------------------------------------------------------------- +# fixture +# -------------------------------------------------------------------------- + +SHARE_DIFFICULTY = 16384 +NETWORK_DIFFICULTY = 226646186 +EPOCH_SECONDS = 1_700_000_000 +SHARE_SPACING_MS = 250 +ACCEPT_LAG_MS = 400 + + +def seed_sql(share_rows: int, miners: int) -> str: + miners = max(4, int(miners)) + return f""" +TRUNCATE qbit_share_ledger CASCADE; +ALTER SEQUENCE qbit_share_ledger_share_seq_seq RESTART WITH 1; +INSERT INTO qbit_share_ledger ( + share_id, miner_id, payout_order_key, p2mr_program, + share_difficulty, network_difficulty, template_height, job_id, + job_issued_at, ntime, accepted_at, accepted, writer_id, writer_epoch, + credit_policy +) +SELECT + 'bench-miner-' || miner_idx || '.rig' || (g % 512) || ':' + || encode(sha256(('bench-share:' || g)::bytea), 'hex'), + 'qbit1' || substr(encode(sha256(('bench-miner:' || miner_idx)::bytea), 'hex'), 1, 58), + 'qbit1' || substr(encode(sha256(('bench-miner:' || miner_idx)::bytea), 'hex'), 1, 58), + sha256(('bench-program:' || miner_idx)::bytea), + {SHARE_DIFFICULTY}, + {NETWORK_DIFFICULTY}, + 1000000 + (g / 64), + 'bench-job-' || lpad((g / 64)::text, 12, '0'), + to_timestamp({EPOCH_SECONDS}) + (g * interval '{SHARE_SPACING_MS} milliseconds'), + {EPOCH_SECONDS} + (g / 4), + to_timestamp({EPOCH_SECONDS}) + (g * interval '{SHARE_SPACING_MS} milliseconds') + + interval '{ACCEPT_LAG_MS} milliseconds', + TRUE, + 'bench-seed', + 1, + CASE WHEN g % 97 = 0 THEN 'stale-grace' ELSE NULL END +FROM ( + SELECT + gs AS g, + CASE + WHEN mod(gs, 10) < 3 THEN 1 + WHEN mod(gs, 10) < 5 THEN 2 + WHEN mod(gs, 10) < 6 THEN 3 + ELSE 4 + mod(gs, {miners} - 3) + END AS miner_idx + FROM generate_series(1, {int(share_rows)}) AS gs +) AS seeded; +ANALYZE qbit_share_ledger; +""" + + +def anchors_for(share_rows: int) -> tuple[int, int, int]: + """(anchor_ms, previous_anchor_ms, window_weight covering every row).""" + last_accepted_ms = ( + EPOCH_SECONDS * 1000 + share_rows * SHARE_SPACING_MS + ACCEPT_LAG_MS + ) + anchor_ms = last_accepted_ms + 10_000 + previous_anchor_ms = EPOCH_SECONDS * 1000 - 1 + return anchor_ms, previous_anchor_ms, share_rows * SHARE_DIFFICULTY + + +# -------------------------------------------------------------------------- +# measurement +# -------------------------------------------------------------------------- + + +class WakeMonitor(threading.Thread): + """Sleep 10 ms in a loop and record how late each wake-up was.""" + + def __init__(self, interval: float = 0.010) -> None: + super().__init__(daemon=True, name="wake-monitor") + self.interval = interval + self._stop_event = threading.Event() + self.samples = 0 + self.max_late = 0.0 + self.late_100 = 0 + self.late_250 = 0 + self.late_500 = 0 + self.late_total = 0.0 + + def run(self) -> None: + while not self._stop_event.is_set(): + started = time.perf_counter() + time.sleep(self.interval) + late = max(0.0, time.perf_counter() - started - self.interval) + self.samples += 1 + self.late_total += late + if late > self.max_late: + self.max_late = late + if late > 0.100: + self.late_100 += 1 + if late > 0.250: + self.late_250 += 1 + if late > 0.500: + self.late_500 += 1 + + def stop(self) -> dict[str, Any]: + self._stop_event.set() + self.join(timeout=5) + return { + "samples": self.samples, + "max_late_ms": round(self.max_late * 1000, 1), + "mean_late_ms": round( + (self.late_total / self.samples * 1000) if self.samples else 0.0, 2 + ), + "late_over_100ms": self.late_100, + "late_over_250ms": self.late_250, + "late_over_500ms": self.late_500, + } + + +def _rss_kb() -> tuple[int, int]: + rss = hwm = 0 + try: + for line in Path("/proc/self/status").read_text().splitlines(): + if line.startswith("VmRSS:"): + rss = int(line.split()[1]) + elif line.startswith("VmHWM:"): + hwm = int(line.split()[1]) + except OSError: + pass + return rss, hwm + + +def capture_statement(ledger: Any, call: Callable[[], Any]) -> tuple[str, Any]: + """Run ``call`` and return the statement the ledger sent for it. + + Works on both checkouts: the row-result read helper after #236, the + single-value read helper before it. + """ + statements: list[str] = [] + for name in ("_run_retry_safe_read_json_rows", "_run_retry_safe_read_json"): + original = getattr(ledger, name, None) + if original is None: + continue + + def recording(sql: str, *args: Any, _original=original, **kwargs: Any) -> Any: + statements.append(sql) + return _original(sql, *args, **kwargs) + + setattr(ledger, name, recording) + try: + result = call() + finally: + delattr(ledger, name) + if len(statements) != 1: + raise SystemExit(f"expected one statement, captured {len(statements)}") + return statements[0], result + raise SystemExit("no read helper to capture on this ledger") + + +def timed_call(ledger: Any, call: Callable[[], Any]) -> dict[str, Any]: + stats_before = ledger.ledger_read_gate_stats() + rss_before, _ = _rss_kb() + monitor = WakeMonitor() + monitor.start() + time.sleep(0.05) # let the monitor establish its cadence + process_cpu = time.process_time() + thread_cpu = time.thread_time() + started = time.perf_counter() + result = call() + wall = time.perf_counter() - started + thread_cpu = time.thread_time() - thread_cpu + process_cpu = time.process_time() - process_cpu + monitor_stats = monitor.stop() + rss_after, hwm_after = _rss_kb() + stats_after = ledger.ledger_read_gate_stats() + attribution: dict[str, float] = {} + for operation, after in stats_after.items(): + before = stats_before.get(operation, {}) + delta = float(after.get("execute_seconds_total", 0.0)) - float( + before.get("execute_seconds_total", 0.0) + ) + gate_delta = float(after.get("gate_wait_seconds_total", 0.0)) - float( + before.get("gate_wait_seconds_total", 0.0) + ) + if delta or gate_delta: + attribution[operation] = { + "execute_ms": round(delta * 1000, 1), + "gate_wait_ms": round(gate_delta * 1000, 1), + } + return { + "records": len(result), + "wall_ms": round(wall * 1000, 1), + "thread_cpu_ms": round(thread_cpu * 1000, 1), + "process_cpu_ms": round(process_cpu * 1000, 1), + "rss_before_mb": round(rss_before / 1024, 1), + "rss_after_mb": round(rss_after / 1024, 1), + "rss_hwm_mb": round(hwm_after / 1024, 1), + "monitor": monitor_stats, + "ledger_attribution": attribution, + } + + +def explain(psql_ledger: Any, sql: str) -> dict[str, Any]: + """Server-side planning + execution time and the plan's node summary.""" + output = psql_ledger._run_sql("EXPLAIN (ANALYZE, FORMAT JSON) " + sql) + plan = json.loads(output)[0] + nodes: list[str] = [] + + def walk(node: dict[str, Any], depth: int) -> None: + label = node.get("Node Type", "?") + if node.get("Strategy"): + label += f"/{node['Strategy']}" + if node.get("Index Name"): + label += f"[{node['Index Name']}]" + nodes.append(f"{' ' * depth}{label} rows={node.get('Actual Rows')}") + for child in node.get("Plans", []) or []: + walk(child, depth + 1) + + walk(plan["Plan"], 0) + return { + "planning_ms": round(float(plan.get("Planning Time", 0.0)), 1), + "execution_ms": round(float(plan["Execution Time"]), 1), + "plan": nodes, + } + + +# -------------------------------------------------------------------------- +# driver +# -------------------------------------------------------------------------- + + +def run(args: argparse.Namespace) -> dict[str, Any]: + lab_root = Path(args.lab_root).resolve() + _install_lab_root(lab_root) + from lab.prism.share_ledger import PsqlShareLedger # noqa: E402 + + results: dict[str, Any] = { + "lab_root": str(lab_root), + "lab_commit": _git_head(lab_root), + "driver_commit": _git_head(REPO_ROOT), + "python": platform.python_version(), + "platform": platform.platform(), + "cpu_count": os.cpu_count(), + "switch_interval_ms": sys.getswitchinterval() * 1000, + "miners": args.miners, + "reps": args.reps, + "backends": list(args.backends), + "sizes": [], + } + sizes_for_json = results["sizes"] + + seed_ledger = PsqlShareLedger( + psql_command=args.psql_command, + native_client_mode="psql", + writer_id="bench-seed", + writer_epoch=1, + initialize_schema=True, + ) + try: + server_version = seed_ledger._run_json( + "SELECT json_build_object('v', version());" + )["v"] + finally: + pass + results["postgres"] = server_version + + for share_rows in args.shares: + print(f"== {share_rows} shares: seeding", flush=True) + seed_started = time.perf_counter() + seed_ledger._run_script(seed_sql(share_rows, args.miners)) + seed_seconds = time.perf_counter() - seed_started + count = int( + seed_ledger._run_json( + "SELECT json_build_object('n', count(*)) FROM qbit_share_ledger WHERE accepted;" + )["n"] + ) + if count != share_rows: + raise SystemExit(f"seeded {count} rows, expected {share_rows}") + anchor_ms, previous_anchor_ms, window_weight = anchors_for(share_rows) + size_result: dict[str, Any] = { + "shares": share_rows, + "seed_seconds": round(seed_seconds, 1), + "anchor_ms": anchor_ms, + "window_weight": window_weight, + "server": {}, + "backends": {}, + } + sizes_for_json.append(size_result) + + read_shapes: dict[str, Callable[[Any], Callable[[], Any]]] = { + "bounded": lambda ledger: lambda: ledger.snapshot_at_job_issue( + anchor_ms, window_weight=window_weight + ), + "unbounded": lambda ledger: lambda: ledger.snapshot_at_job_issue(anchor_ms), + "delta": lambda ledger: lambda: ledger.snapshot_between_job_issues( + previous_anchor_ms, anchor_ms + ), + } + + for backend in args.backends: + if backend == "native": + ledger = PsqlShareLedger( + psql_command=args.psql_command, + database_url=args.database_url, + native_client_mode="1", + writer_id="bench-read", + writer_epoch=1, + read_only=True, + ) + else: + ledger = PsqlShareLedger( + psql_command=args.psql_command, + native_client_mode="psql", + writer_id="bench-read", + writer_epoch=1, + read_only=True, + ) + backend_result: dict[str, Any] = { + "execution_backend": ledger.execution_backend, + "reads": {}, + } + size_result["backends"][backend] = backend_result + try: + for shape, make_call in read_shapes.items(): + call = make_call(ledger) + # One warm-up per shape captures the statement (and warms + # the buffer cache); measured reps follow. + statement, warm = capture_statement(ledger, call) + if shape not in size_result["server"] and args.explain: + size_result["server"][shape] = { + "statement_uses_json_agg": "json_agg" in statement, + "statement_sha256_prefix": hashlib.sha256( + statement.encode() + ).hexdigest()[:12], + **explain(seed_ledger, statement), + } + reps = [timed_call(ledger, call) for _ in range(args.reps)] + walls = [rep["wall_ms"] for rep in reps] + backend_result["reads"][shape] = { + "records": len(warm), + "wall_ms_median": round(statistics.median(walls), 1), + "wall_ms_max": round(max(walls), 1), + "monitor_max_late_ms_max": max( + rep["monitor"]["max_late_ms"] for rep in reps + ), + "monitor_late_over_100ms_total": sum( + rep["monitor"]["late_over_100ms"] for rep in reps + ), + "monitor_late_over_250ms_total": sum( + rep["monitor"]["late_over_250ms"] for rep in reps + ), + "monitor_late_over_500ms_total": sum( + rep["monitor"]["late_over_500ms"] for rep in reps + ), + "reps": reps, + } + print( + f" {backend:>6} {shape:>9}: records={len(warm)} " + f"wall median={statistics.median(walls):.0f}ms " + f"max={max(walls):.0f}ms " + f"monitor max-late={backend_result['reads'][shape]['monitor_max_late_ms_max']:.0f}ms " + f">100ms={backend_result['reads'][shape]['monitor_late_over_100ms_total']} " + f">250ms={backend_result['reads'][shape]['monitor_late_over_250ms_total']}", + flush=True, + ) + finally: + ledger.close() + for shape, server in size_result["server"].items(): + print( + f" server {shape:>9}: planning={server['planning_ms']}ms " + f"execution={server['execution_ms']}ms json_agg={server['statement_uses_json_agg']}", + flush=True, + ) + seed_ledger.close() + results["peak_rss_mb"] = round(resource.getrusage(resource.RUSAGE_SELF).ru_maxrss / 1024, 1) + return results + + +def render_comparison(baseline: dict[str, Any], patched: dict[str, Any]) -> str: + """Markdown tables comparing two result files, size by size.""" + lines: list[str] = [] + lines.append( + f"Baseline `{(baseline.get('lab_commit') or '?')[:12]}` vs patched " + f"`{(patched.get('lab_commit') or '?')[:12]}` (driver " + f"`{(patched.get('driver_commit') or '?')[:12]}`); reps={patched.get('reps')}; " + f"miners={patched.get('miners')}; {patched.get('postgres', '?')}; " + f"Python {patched.get('python')}; {patched.get('cpu_count')} CPUs." + ) + lines.append("") + by_size_base = {size["shares"]: size for size in baseline["sizes"]} + for size in patched["sizes"]: + shares = size["shares"] + base = by_size_base.get(shares) + lines.append(f"### {shares:,} shares") + lines.append("") + lines.append( + "| backend | read | records | wall median (ms) base -> patch | wall max (ms) base -> patch " + "| monitor max lateness (ms) base -> patch | wake-ups > 100 ms base -> patch " + "| wake-ups > 250 ms base -> patch | wake-ups > 500 ms base -> patch |" + ) + lines.append("|---|---|---:|---:|---:|---:|---:|---:|---:|") + for backend, patched_backend in size["backends"].items(): + base_backend = (base or {}).get("backends", {}).get(backend, {}) + for shape, after in patched_backend["reads"].items(): + before = base_backend.get("reads", {}).get(shape) + + def pair(key: str, fmt: str = "{:.0f}") -> str: + if before is None: + return "n/a -> " + fmt.format(after[key]) + return fmt.format(before[key]) + " -> " + fmt.format(after[key]) + + lines.append( + f"| {backend} | {shape} | {after['records']:,} | {pair('wall_ms_median')} " + f"| {pair('wall_ms_max')} | {pair('monitor_max_late_ms_max')} " + f"| {pair('monitor_late_over_100ms_total', '{}')} " + f"| {pair('monitor_late_over_250ms_total', '{}')} " + f"| {pair('monitor_late_over_500ms_total', '{}')} |" + ) + lines.append("") + lines.append("| read | server planning+execution (ms) base -> patch | statement uses json_agg base -> patch |") + lines.append("|---|---:|---|") + for shape, after in size.get("server", {}).items(): + before = (base or {}).get("server", {}).get(shape) + after_ms = after["planning_ms"] + after["execution_ms"] + if before is None: + lines.append(f"| {shape} | n/a -> {after_ms:.0f} | n/a -> {after['statement_uses_json_agg']} |") + else: + before_ms = before["planning_ms"] + before["execution_ms"] + lines.append( + f"| {shape} | {before_ms:.0f} -> {after_ms:.0f} " + f"| {before['statement_uses_json_agg']} -> {after['statement_uses_json_agg']} |" + ) + lines.append("") + rss_rows = [] + for backend, patched_backend in size["backends"].items(): + base_backend = (base or {}).get("backends", {}).get(backend, {}) + for shape, after in patched_backend["reads"].items(): + before = base_backend.get("reads", {}).get(shape) + after_rss = max(rep["rss_hwm_mb"] for rep in after["reps"]) + after_cpu = statistics.median(rep["thread_cpu_ms"] for rep in after["reps"]) + if before is None: + rss_rows.append(f"| {backend} | {shape} | n/a -> {after_cpu:.0f} | n/a -> {after_rss:.0f} |") + else: + before_rss = max(rep["rss_hwm_mb"] for rep in before["reps"]) + before_cpu = statistics.median(rep["thread_cpu_ms"] for rep in before["reps"]) + rss_rows.append( + f"| {backend} | {shape} | {before_cpu:.0f} -> {after_cpu:.0f} " + f"| {before_rss:.0f} -> {after_rss:.0f} |" + ) + lines.append("| backend | read | caller-thread CPU median (ms) base -> patch | process RSS high-water (MB) base -> patch |") + lines.append("|---|---|---:|---:|") + lines.extend(rss_rows) + lines.append("") + return "\n".join(lines) + + +def main(argv: list[str] | None = None) -> int: + parser = argparse.ArgumentParser( + description=__doc__, + formatter_class=argparse.RawDescriptionHelpFormatter, + ) + parser.add_argument( + "--compare", + nargs=2, + metavar=("BASELINE_JSON", "PATCHED_JSON"), + help="render a markdown comparison of two --json results and exit (no database needed)", + ) + parser.add_argument("--psql-command", help="psql invocation for the disposable database") + parser.add_argument("--database-url", help="DSN for the native (psycopg) backend") + parser.add_argument("--shares", type=int, nargs="+", default=[210_000, 400_000]) + parser.add_argument("--miners", type=int, default=8, help="distinct miners (skewed like the scale gate)") + parser.add_argument("--reps", type=int, default=3) + parser.add_argument("--backends", nargs="+", default=["native", "psql"], choices=["native", "psql"]) + parser.add_argument("--lab-root", default=str(REPO_ROOT), help="checkout to import lab.prism.share_ledger from") + parser.add_argument("--no-explain", dest="explain", action="store_false") + parser.add_argument("--json", help="write structured results here") + args = parser.parse_args(argv) + if args.compare: + baseline = json.loads(Path(args.compare[0]).read_text()) + patched = json.loads(Path(args.compare[1]).read_text()) + print(render_comparison(baseline, patched)) + return 0 + if not args.psql_command: + parser.error("--psql-command is required unless --compare is given") + if "native" in args.backends and not args.database_url: + parser.error("--database-url is required for the native backend") + results = run(args) + if args.json: + Path(args.json).write_text(json.dumps(results, indent=2, sort_keys=True)) + print(f"wrote {args.json}") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/tests/perf/window_paging_allocation.md b/tests/perf/window_paging_allocation.md new file mode 100644 index 000000000..07e41d259 --- /dev/null +++ b/tests/perf/window_paging_allocation.md @@ -0,0 +1,460 @@ +# Rust payout-window paging allocation: builder RSS and timing at 210k and 400k shares (#240) + +Issue #240 (incident #236): `PayoutWindow::from_full_snapshot` in +`crates/qbit-prism/src/window.rs` paged the retained records with +`remaining.split_off(page_size)` in a loop and stored the head as each page. +`Vec::split_off` keeps the original vector's allocation, so page *k* retained +backing capacity for every record still unpaged when it was cut (N, N−P, N−2P, +…) and the tail was copied again on every iteration: O(N²/P) retained memory +and copying. + +What that cost had been measured at, before this PR: a **prior synthetic +reproduction on this development host** (`tests/perf/window_pipeline_gil_scaling.md` +§9, same fixture and page size as below, single run) recorded 9,613 MiB peak +builder RSS and a 12.2 s response for a 210k-share preparation, and 15,623 MiB +followed by SIGKILL at 400k. Production (#236) separately observed roughly +9.69 GiB builder RSS at its ~210k-share window; those production figures are +cited from the incident, not reproduced here. + +This report measures the fix in this PR (every retained record moves exactly +once into a page allocated for exactly the records it holds; the `advance` +delta loop, which had the same `split_off` shape, is paged the same way) +against the real `qbit-prism-build-audit-bundle --serve` daemon, before and +after, on this host and inside a locally built copy of the production +coordinator image, with every experiment bounded so it cannot exhaust the +host. + +Driver: `tests/perf/window_paging_allocation.py`. Standard library plus +read-only imports from `lab/`; no database, network or coordinator. It is an +**on-demand instrument, not a test**: it asserts no thresholds and is not +named `test_*` (#160); its bounded safety behaviour (exchange deadlines, +child reaping) is covered by `tests/test_window_paging_allocation_harness.py`. +Every number a reviewer needs is in this file; the structured JSON captures +behind the tables (`--json` output of the runs below) are retained in the +worker's workspace outside the repository and are not committed. + +--- + +## Headline + +**The allocation defect is gone and paging is linear in the retained count.** +Peak resident set of the daemon across a cold full preparation, two advances, +a re-center (second full preparation held beside the first) and a further +advance: + +| retained shares | old paging (baseline binary) | fixed paging (this PR) | fixed, single window only | +|---:|---:|---:|---:| +| 52,000 | **1,028 MiB**, full response 1.05 s | **163 MiB**, 0.33 s | 110 MiB | +| 105,000 | **3,884 MiB**, 3.78 s | **325 MiB**, 0.63 s | 219 MiB | +| 210,000 | 9,613 MiB, 12.2 s *(prior synthetic measurement, §5)* | **601 MiB**, 1.29 s | 435 MiB | +| 400,000 | 15,623 MiB then SIGKILL *(prior synthetic measurement, §5)* | **1,142 MiB**, 2.60 s | 826 MiB | + +Old-paging memory per 1k shares doubles when the window doubles (19.8 → +37.0 MiB per 1k from 52k to 105k), consistent with the quadratic record-vector +capacity model (§4). Fixed-paging memory is flat at **2.9–3.1 MiB per +1k shares** from 52k to 400k. The 400k preparation completes in 2.6 s with a +clean daemon exit; no daemon fell back, died, timed out or declined. + +**The same binary built inside the production coordinator image gives the +same numbers** (§7): 601 MiB / 1.46 s at 210k and 1,142 MiB / 2.60 s at 400k +under CPython 3.14.7 with `MALLOC_ARENA_MAX=2`. + +**Every digest agreed.** All 40 phase outcomes (host: two binaries × sizes × +five phases; image: one binary × two sizes × five phases) returned +`prepared`; every returned digest re-hashed from the daemon's own canonical +bytes (with the coordinator's drop-prefix/append-suffix mirror surgery +replayed for advances), and every one matched the shipped Python fold run on +the same inputs as an independent oracle — under CPython 3.12 on the host and +CPython 3.14 in the image. The frozen parity corpus passed against the +rebuilt binary (§6). + +**Transport and Rust processing are separated by measurement, not +inference:** the fixed daemon reports its own JSON parse, fold and canonical +serialization times in the `prepare_window` envelope (an additive `metrics` +field the coordinator ignores). At 210k the 1.29 s response wait is 0.27 s +parse + 0.38 s fold + 0.63 s canonical digest/items; the blocking-pipe +transport is 0.15 s to write the 91 MiB request and 0.12 s to read the 91 MiB +response. The paging fold is now the *smallest* of the three daemon terms. + +--- + +## 1. Source, binaries and verification + +| | | +|---|---| +| Branch | `fix/prism-240-rust-window-paging-20260906`, base `0bc7fa6c023082df81d23137ebabf4ff96f24683` (`2.x.x` with #237/#238/#239 merged) | +| Rust fix commit | `527b3acb5e25148ec6e6a71141818ddbaa06866a` — the only commit on this branch touching `crates/` | +| Fixed binary | `target/release/qbit-prism-build-audit-bundle`, `cargo build --release --locked -p qbit-prism --bin qbit-prism-build-audit-bundle`, sha256 `46faaa9547c39960441be84fc85268789d5d55f99a0ab70a214b643ca10293ce`, 2,241,216 bytes | +| Verification of the fixed binary | built from a tree whose `crates/` content is byte-identical to `527b3ac`; re-running the same build command after the commit was a cargo fingerprint no-op (nothing recompiled, same sha256). The two touched sources at `527b3ac` hash to `d1529288d117af52…` (`window.rs`) and `68ab080a746e3d18…` (`qbit-prism-build-audit-bundle.rs`), and the copies inside the image (§7) hash identically | +| Baseline binary | same command from the unmodified base `0bc7fa6`, sha256 `e7f5ca0cccf31b6a3045a3b898ef53b0e2857d7cf276cadf0530147e3c298e2d`, 2,244,296 bytes | +| Harness | the `tests/perf/window_paging_allocation.py` committed beside this report, byte for byte (the host sweep recorded its tree as `527b3ac` plus the then-uncommitted harness, report and harness tests that the next commit on this branch adds; no `crates/` change) | +| Toolchain (host) | cargo/rustc 1.97.1 (c980f4866 2026-06-30), stable x86_64-unknown-linux-gnu | +| Host | Linux 6.8.0-106-generic x86_64, glibc 2.39, KVM guest, 8 vCPU, 23,464 MiB RAM, no swap; shared development host | +| Available memory | 17,436 MiB at the start of the host sweep; recorded before every daemon run (16,425–17,281 MiB) | +| Load (1 m) | 0.78 at start, 1.13–1.69 before the larger runs (other worktrees on the host were quiet but not idle) | +| Python (driver + oracle, host) | CPython 3.12.3, `MALLOC_ARENA_MAX` unset | +| Page size | 512 (`DEFAULT_INCREMENTAL_SHARE_WINDOW_PAGE_SIZE`) | +| Bounding | every daemon under `RLIMIT_AS` = 6,144 MiB; a run is skipped unless `MemAvailable` ≥ 8,192 MiB; one daemon at a time; every exchange (handshake included) under a 600 s deadline that kills and reaps the daemon | + +### Fixture + +`build_records` from `tests/perf/window_pipeline_gil_scaling.py`, the fixture +the #236 probe and the prior synthetic baseline used: production-shaped +`share_id` (`bench-miner-N.rigM:<64 hex>`), 200 identities, difficulty +16,384, one share per second, all eligible at the anchor. `window_weight` is +the fixture's total difficulty so every record is retained. Request sizes: +22.5 MiB (52k), 45.4 MiB (105k), 91.0 MiB (210k), 173.6 MiB (400k). + +### Phases, per binary and size, on one daemon + +| phase | request | what it exercises | +|---|---|---| +| `full` | `mode=full`, all records | the cold fold: filter, sort, validate, cutoff, **paging** | +| `advance_small` | `mode=advance`, 16 appended records | the boundary-page merge | +| `advance_large` | `mode=advance`, 1,043 appended records | the delta paging loop (two full pages plus a partial, same `split_off` shape as the fold) | +| `recenter` | `mode=full`, same records at ¾ of the weight | what the coordinator's self-check re-center sends while the previous window is still held: a second prepared window in the two-entry cache | +| `recenter_advance` | `mode=advance` against the re-centered digest, 16 records | advance from the re-centered generation | + +### What the columns mean + +- **write** — elapsed until the blocking pipe accepted the last request byte + (the daemon has consumed all but at most one pipe capacity). Transport. +- **wait** — from then until the envelope line arrived. Contains the daemon's + remaining parse, the fold or advance, canonical serialization and response + readiness. +- **read** — the raw canonical-items section after the envelope. Transport. +- **parse / fold / serialize** — the daemon's own `metrics`: `serde_json` + parse of the request line; `from_full_snapshot` or `advance`; canonical + digest plus (for `full`) the items stream. Measured inside the process with + `Instant`, so this is Rust processing, not pipe time. +- **residual (approx.)** — wait − (parse + fold + serialize). The client's + wall interval and the daemon's internal timers are independent intervals on + either side of a pipe (the daemon may still be reading and parsing when the + client clock starts; the envelope write and scheduling are on neither + timer), so this is an approximation of scheduling plus envelope handling, + **not** a measured transport term. It is clamped at zero and blank when a + binary reports no metrics. +- **RSS after** — `VmRSS` from `/proc//status` right after the phase. + **peak so far / peak** — `VmHWM`, the kernel's lifetime high-water mark, + read before the daemon is reaped (not a sampling artefact; the 20 ms + sampler's maximum agreed within 22 MiB in every run). +- **self-check** — `sha256("[" + items + "]")` of the daemon's bytes equals + its `share_snapshot_sha256`; for advances the coordinator's mirror surgery + (drop `retained_drop_bytes`, append the returned suffix) is replayed first. +- **oracle** — the digest equals `IncrementalShareWindow` (the shipped Python + fold, `lab/prism/share_ledger.py`) on the same records, weight, anchor and + page size, including its `advance` for the advance phases. + +--- + +## 2. Host results + +All peak figures below are observed high-water marks before the daemon exits, +including the clean runs. They are lower bounds on the final lifetime peak: +a child can allocate between a live `/proc` read and normal exit or delivery +of `SIGKILL`. The harness now renders `≥` for every such observation, including +watchdog pre-kill reads, with the source recorded separately. The tables retain +the captured numeric observations; they do not establish a lifetime memory +ceiling. Structural page-capacity tests establish the allocation bound. + +### Summary per daemon (observed before exit) + +| binary | shares | peak RSS MiB (VmHWM) | peak VSZ MiB | MiB per 1k shares | full wait s | full fold s | daemon exit | available MiB before | +|---|---:|---:|---:|---:|---:|---:|---|---:| +| baseline | 52,000 | 1,028 | 1,042 | 19.76 | 1.05 | – | clean (0) | 17,281 | +| fixed | 52,000 | 163 | 177 | 3.14 | 0.33 | 0.085 | clean (0) | 16,903 | +| baseline | 105,000 | 3,884 | 3,910 | 36.99 | 3.78 | – | clean (0) | 16,861 | +| fixed | 105,000 | 325 | 351 | 3.10 | 0.63 | 0.184 | clean (0) | 16,823 | +| fixed | 210,000 | 601 | 640 | 2.86 | 1.29 | 0.377 | clean (0) | 16,510 | +| fixed | 400,000 | 1,142 | 1,226 | 2.85 | 2.60 | 0.779 | clean (0) | 16,425 | + +The baseline binary has no `metrics` field (blank columns); its wait is +whole-daemon time. + +### Every phase + +| binary | shares | phase | records | request MiB | write s | wait s | read s | parse s | fold s | serialize s | residual s (approx.) | RSS after MiB | peak so far MiB | status | self-check | oracle | +|---|---:|---|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---|---|---| +| baseline | 52,000 | full | 52,000 | 22.5 | 0.03 | 1.05 | 0.02 | – | – | – | – | 623 | 668 | prepared | yes | yes | +| baseline | 52,000 | advance_small | 16 | 0.0 | 0.00 | 0.14 | 0.00 | – | – | – | – | 625 | 668 | prepared | yes | yes | +| baseline | 52,000 | advance_large | 1,043 | 0.5 | 0.00 | 0.14 | 0.00 | – | – | – | – | 616 | 668 | prepared | yes | yes | +| baseline | 52,000 | recenter | 52,000 | 22.5 | 0.04 | 0.71 | 0.02 | – | – | – | – | 983 | 1,028 | prepared | yes | yes | +| baseline | 52,000 | recenter_advance | 16 | 0.0 | 0.00 | 0.15 | 0.00 | – | – | – | – | 715 | 1,028 | prepared | yes | yes | +| fixed | 52,000 | full | 52,000 | 22.5 | 0.05 | 0.33 | 0.03 | 0.085 | 0.085 | 0.155 | 0.003 | 65 | 110 | prepared | yes | yes | +| fixed | 52,000 | advance_small | 16 | 0.0 | 0.00 | 0.14 | 0.00 | 0.000 | 0.001 | 0.141 | 0.001 | 66 | 110 | prepared | yes | yes | +| fixed | 52,000 | advance_large | 1,043 | 0.5 | 0.00 | 0.15 | 0.00 | 0.002 | 0.005 | 0.140 | 0.000 | 69 | 110 | prepared | yes | yes | +| fixed | 52,000 | recenter | 52,000 | 22.5 | 0.05 | 0.24 | 0.01 | 0.058 | 0.063 | 0.114 | 0.004 | 141 | 163 | prepared | yes | yes | +| fixed | 52,000 | recenter_advance | 16 | 0.0 | 0.00 | 0.12 | 0.00 | 0.000 | 0.001 | 0.103 | 0.012 | 141 | 163 | prepared | yes | yes | +| baseline | 105,000 | full | 105,000 | 45.4 | 0.06 | 3.78 | 0.06 | – | – | – | – | 2,417 | 2,508 | prepared | yes | yes | +| baseline | 105,000 | advance_small | 16 | 0.0 | 0.00 | 0.29 | 0.00 | – | – | – | – | 2,418 | 2,508 | prepared | yes | yes | +| baseline | 105,000 | advance_large | 1,043 | 0.5 | 0.00 | 0.29 | 0.00 | – | – | – | – | 2,399 | 2,508 | prepared | yes | yes | +| baseline | 105,000 | recenter | 105,000 | 45.4 | 0.10 | 2.36 | 0.04 | – | – | – | – | 3,760 | 3,884 | prepared | yes | yes | +| baseline | 105,000 | recenter_advance | 16 | 0.0 | 0.00 | 0.37 | 0.00 | – | – | – | – | 2,091 | 3,884 | prepared | yes | yes | +| fixed | 105,000 | full | 105,000 | 45.4 | 0.06 | 0.63 | 0.06 | 0.137 | 0.184 | 0.308 | 0.005 | 128 | 219 | prepared | yes | yes | +| fixed | 105,000 | advance_small | 16 | 0.0 | 0.00 | 0.27 | 0.00 | 0.000 | 0.002 | 0.273 | 0.000 | 129 | 219 | prepared | yes | yes | +| fixed | 105,000 | advance_large | 1,043 | 0.5 | 0.00 | 0.29 | 0.00 | 0.002 | 0.004 | 0.281 | 0.000 | 132 | 219 | prepared | yes | yes | +| fixed | 105,000 | recenter | 105,000 | 45.4 | 0.08 | 0.53 | 0.04 | 0.138 | 0.139 | 0.252 | 0.006 | 246 | 325 | prepared | yes | yes | +| fixed | 105,000 | recenter_advance | 16 | 0.0 | 0.00 | 0.25 | 0.00 | 0.000 | 0.001 | 0.221 | 0.026 | 246 | 325 | prepared | yes | yes | +| fixed | 210,000 | full | 210,000 | 91.0 | 0.15 | 1.29 | 0.12 | 0.270 | 0.377 | 0.634 | 0.012 | 253 | 435 | prepared | yes | yes | +| fixed | 210,000 | advance_small | 16 | 0.0 | 0.00 | 0.55 | 0.00 | 0.000 | 0.003 | 0.548 | 0.000 | 255 | 435 | prepared | yes | yes | +| fixed | 210,000 | advance_large | 1,043 | 0.5 | 0.00 | 0.58 | 0.00 | 0.002 | 0.006 | 0.575 | 0.000 | 258 | 435 | prepared | yes | yes | +| fixed | 210,000 | recenter | 210,000 | 91.0 | 0.17 | 1.04 | 0.11 | 0.281 | 0.282 | 0.465 | 0.013 | 442 | 601 | prepared | yes | yes | +| fixed | 210,000 | recenter_advance | 16 | 0.0 | 0.00 | 0.48 | 0.00 | 0.000 | 0.002 | 0.419 | 0.054 | 443 | 601 | prepared | yes | yes | +| fixed | 400,000 | full | 400,000 | 173.6 | 0.28 | 2.60 | 0.29 | 0.574 | 0.779 | 1.231 | 0.020 | 479 | 826 | prepared | yes | yes | +| fixed | 400,000 | advance_small | 16 | 0.0 | 0.00 | 1.10 | 0.00 | 0.000 | 0.011 | 1.093 | 0.000 | 483 | 826 | prepared | yes | yes | +| fixed | 400,000 | advance_large | 1,043 | 0.5 | 0.00 | 1.08 | 0.00 | 0.001 | 0.007 | 1.067 | 0.001 | 486 | 826 | prepared | yes | yes | +| fixed | 400,000 | recenter | 400,000 | 173.6 | 0.26 | 2.13 | 0.17 | 0.578 | 0.611 | 0.918 | 0.021 | 838 | 1,142 | prepared | yes | yes | +| fixed | 400,000 | recenter_advance | 16 | 0.0 | 0.00 | 1.03 | 0.00 | 0.000 | 0.004 | 0.858 | 0.168 | 840 | 1,142 | prepared | yes | yes | + +Advance stats returned by both binaries at every size: `advance_small` added +16 / expired 16 / touched 2 pages; `advance_large` added 1,043 / expired 1,043 +/ touched 2; the re-center retained exactly ¾ of the records (39,000; 78,750; +157,500; 300,000), matching the oracle's counts. An earlier host sweep with +the pre-hardening harness (same binaries) gave the same peak figures when +rounded to MiB and comparable timings. + +### The Python oracle beside it (host, CPython 3.12.3) + +| shares | full fold s | re-center fold s | records (full / re-center) | full digest | +|---:|---:|---:|---:|---| +| 52,000 | 0.41 | 0.31 | 52,000 / 39,000 | `cff331dccb93a7ff…` | +| 105,000 | 0.82 | 0.65 | 105,000 / 78,750 | `cfdf4a0ed72673b7…` | +| 210,000 | 1.58 | 1.26 | 210,000 / 157,500 | `e38ca96a869cee23…` | +| 400,000 | 3.27 | 2.73 | 400,000 / 300,000 | `30b53c3312705f2c…` | + +For scale only: the Rust fold at 210k is 0.38 s against 1.58 s in-process +Python, and the Python fold holds the GIL (`window_pipeline_gil_scaling.md`). + +--- + +## 3. Where the daemon's time goes now + +Fixed binary, `full` phase, host: + +| shares | write (transport) | parse | fold | serialize | residual (approx.) | read (transport) | Rust processing / round trip | +|---:|---:|---:|---:|---:|---:|---:|---:| +| 52,000 | 0.05 | 0.09 | 0.09 | 0.16 | 0.00 | 0.03 | 0.33 / 0.41 s | +| 105,000 | 0.06 | 0.14 | 0.18 | 0.31 | 0.01 | 0.06 | 0.63 / 0.75 s | +| 210,000 | 0.15 | 0.27 | 0.38 | 0.63 | 0.01 | 0.12 | 1.28 / 1.56 s | +| 400,000 | 0.28 | 0.57 | 0.78 | 1.23 | 0.02 | 0.29 | 2.58 / 3.17 s | + +Between 52k and 400k every measured term grows in proportion to the window +(a 7.7× window gives 6.8× parse, 9.2× fold, 7.9× serialize). The claim this +PR makes is about **paging and page materialization**: they are now linear +in the retained count. The fold as a whole still sorts the eligible records +(`sort_by_key`, O(N log N)) and validates them in one pass; the parse and the +canonical serialization are proportional to the request and response bytes. +After the fix the fold is ~29% of daemon time; canonical serialization +(per-record fragment concatenation plus a SHA-256 over 91–174 MiB) is ~49% and +the request parse ~21%. The blocking-pipe transport measured here is under +0.3 s per direction at 400k; the coordinator's own transport after #237 is +readiness-driven and was measured in `window_builder_pipe_latency.md`. + +Two residual daemon costs are visible and are **not** addressed by this PR: + +- **Every advance re-hashes the whole window.** `serialize` for a 16-record + advance is 0.55 s at 210k and 1.09 s at 400k, because `advance` recomputes + `canonical_digest_hex()` over every retained fragment to name the new + generation. The advance itself (`fold`) is 3–11 ms. This is daemon-side + latency per window generation, off the coordinator's GIL, and proportional + to the window; an incremental digest would be separate work. +- **The request parse** (`serde_json` into `Vec`) is 0.27 s at + 210k. It is proportional to the request's own bytes; a more compact wire + encoding would be a protocol change outside this lane. + +On the old binary's time: at 105k the baseline answers in 3.78 s against the +fixed 0.63 s. No profiler was attached, so the 3.15 s difference is not +attributed to any one operation by measurement; the inference is that the +repeated tail copying and allocation the old loop performed (about 10.8 M +record moves of 224 bytes each, and the same volume of allocation and +release, §4) accounts for it, since the fold's other work is unchanged +between the two binaries. + +--- + +## 4. Why it was quadratic, and why the numbers say the paging fix is complete + +`AcceptedShare` is 224 bytes (`size_of`, x86_64). With N retained records and +P = 512, the old loop made ⌈N/P⌉ pages whose record vectors kept capacities +N, N−P, N−2P, …; aggregate capacity Σ(N−kP) records × 224 B: + +| N | pages | old aggregate capacity | old record-vector storage (MiB) | measured RSS after `full` (baseline) | new record-vector storage (MiB) | +|---:|---:|---:|---:|---:|---:| +| 52,000 | 102 | 2,666,688 records | 569.7 | 623 MiB | 11.1 | +| 105,000 | 206 | 10,819,120 | 2,311.2 | 2,417 MiB | 22.4 | +| 210,000 | 411 | 43,171,440 | 9,222.4 | 9,613 MiB peak (prior synthetic measurement, §5) | 44.9 | +| 400,000 | 782 | 156,450,048 | 33,421.3 | SIGKILL at 15,623 MiB before completion (prior synthetic measurement) | 85.4 | + +The capacity model counts only record-vector backing storage, in MiB +(bytes / 2**20). RSS also includes heap-owned record strings, canonical +fragments, input/output buffers, and allocator overhead, so it is not an +exact RSS prediction. The smaller completed baseline runs and the structural +capacity tests establish the quadratic paging defect without requiring a +leak or a second generation. The 400k run was killed before allocating the +full window: its observed RSS is a lower bound on that run's memory demand, +not a completed measurement validating the 33,421.3 MiB storage estimate. + +The fragment vectors were never affected (`collect` from an exact-size +iterator). After the fix, record-vector storage and the other window buffers +are proportional to the window; the measured 2.9–3.1 MiB per 1k shares from +52k to 400k is consistent with that bound. + +The structural regression tests in `window.rs` +(`full_snapshot_paging_keeps_aggregate_capacity_linear_across_sizes`, +`full_snapshot_paging_after_the_cutoff_allocates_only_retained_records`, +`advance_shares_untouched_pages_and_pages_large_deltas_linearly`, +`digest_items_and_payout_totals_are_independent_of_page_size`, +`empty_snapshots_and_page_size_validation`) assert exactly this bound — no +page vector's capacity above `page_size`, aggregate at most one page's worth +per page — across page sizes 1, 3, 7 and 512 at empty input, one record, one +short of a page, exact single and multiple boundaries, one past a boundary +and long partial tails, plus the cutoff, unsorted input, page-size and weight +validation, immutable page sharing across an advance and a full-rebuild +digest match. Run against the unmodified base implementation (same tests +appended to `0bc7fa6`'s `window.rs`), four of them fail at the capacity +assertion, e.g. *"page 0 of 3 retains capacity 234 for 100 records +(page_size 100)"*; the other 17 window tests pass on both. + +--- + +## 5. The 210k/400k baseline is a prior synthetic measurement, cited not repeated + +The old binary was measured here only at 52k and 105k, sizes whose predicted +footprint (§4) fits under the 6 GiB address-space limit with margin on a +shared host. The 210k and 400k old-paging figures in the headline are the +single-run blocking-pipe measurements recorded in +`tests/perf/window_pipeline_gil_scaling.md` §9 on the same host, same fixture +and same page size, made while investigating #236 and before this PR: peak +RSS 9,613 MiB with a 12.2 s response at 210k; 15,623 MiB then SIGKILL after +~24 s at 400k, with ~15 GB available. They are synthetic development-host +measurements, not production observations. Re-running the 400k case would +have risked exhausting the host again. The completed smaller runs and the +structural capacity tests suffice to reproduce the defect; the killed 400k +run does not measure its full allocation demand. + +--- + +## 6. Correctness gates run with the rebuilt binary + +- `cargo test --locked -p qbit-prism --lib window`: 21 tests, ok (16 + pre-existing plus the five above). +- `cargo test --locked --workspace --all-targets` (the CI gate): 241 tests + across the workspace, ok. +- `QBIT_WINDOW_PIPELINE_PARITY_ADAPTER=rust-daemon PRISM_TOOL_BIN_DIR=$PWD/target/release python3 -m tests.window_pipeline_parity_gate` + against the fixed binary: 7 tests, ok — 35 full preparations and 16 + `advance()` calls across 15 frozen cases, covering canonical bytes, digests, + spool tails, advance stats, every rejection category and the declared + integer domain. +- `python3 -m unittest tests.test_window_pipeline_parity_gate tests.test_prism_window_pipeline_rust tests.test_prism_payout_window_daemon_recenter`: + 43 tests, ok (coordinator-side contracts, including the #207 re-center + behaviour, against the fake daemon; these do not spawn the binary). +- `python3 -m unittest tests.test_window_paging_allocation_harness`: 5 + tests, ok — the driver's exchange deadline kills and reaps a daemon that + never handshakes, one that stalls after reading the request, and one that + never reads it (request writer blocked on a full pipe), with pipes and the + stderr capture closed on every path, and the run recorded as a timeout. +- This driver: 40/40 phases prepared, 40/40 self-consistent, 40/40 oracle + matches (host and image), the two host binaries agreeing with each other + and with Python at 52k and 105k. + +--- + +## 7. Production coordinator image, built and measured locally + +`lab/prism/Dockerfile` (the coordinator's production image recipe: +`python:3.14-slim-trixie`, Debian's rustc/cargo, `cargo build --locked +--release -p qbit-prism --bins`, `MALLOC_ARENA_MAX=2`) was built on this host +from a disposable context holding only `Cargo.toml`, `Cargo.lock`, `crates/`, +`lab/auxpow/` and `lab/prism/` copied from the tree at `527b3ac`: + +| | | +|---|---| +| Image | `prism-240-local:test`, id `f36022841b35`, 1.36 GB on disk; never pushed | +| Interpreter / toolchain inside | CPython 3.14.7; rustc 1.85.0 (4d91de4e4 2025-02-17), cargo 1.85.0; Debian GNU/Linux 13 (trixie), glibc 2.41 | +| Source verification | `/app/crates/qbit-prism/src/window.rs` sha256 `d1529288d117af52…` and `/app/crates/qbit-prism/src/bin/qbit-prism-build-audit-bundle.rs` `68ab080a746e3d18…` — identical to `git show 527b3ac:` | +| Image binary | `/app/target/release/qbit-prism-build-audit-bundle` sha256 `50fda6ac0f8e44450545f3b16592b96622ab0c7bab4c7b80fed827d32eabab33`, 2,143,032 bytes (different compiler, so a different binary from the host build of the same source) | +| Run | `docker run --rm --memory 8g -v $PWD/tests:/app/tests:ro … prism-240-local:test python3 tests/perf/window_paging_allocation.py --daemon-binary /app/target/release --sizes 210000,400000`; driver and oracle under the image's CPython 3.14.7 with `MALLOC_ARENA_MAX=2`; daemon under the same `RLIMIT_AS` and deadline as on the host | + +| binary | shares | peak RSS MiB (VmHWM) | peak VSZ MiB | MiB per 1k shares | full wait s | full fold s | daemon exit | available MiB before | +|---|---:|---:|---:|---:|---:|---:|---|---:| +| image | 210,000 | 601 | 640 | 2.86 | 1.46 | 0.473 | clean (0) | 16,682 | +| image | 400,000 | 1,142 | 1,226 | 2.85 | 2.60 | 0.847 | clean (0) | 16,429 | + +| binary | shares | phase | records | request MiB | write s | wait s | read s | parse s | fold s | serialize s | residual s (approx.) | RSS after MiB | peak so far MiB | status | self-check | oracle | +|---|---:|---|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---|---|---| +| image | 210,000 | full | 210,000 | 91.0 | 0.17 | 1.46 | 0.11 | 0.334 | 0.473 | 0.646 | 0.012 | 253 | 435 | prepared | yes | yes | +| image | 210,000 | advance_small | 16 | 0.0 | 0.00 | 0.60 | 0.00 | 0.000 | 0.004 | 0.595 | 0.000 | 255 | 435 | prepared | yes | yes | +| image | 210,000 | advance_large | 1,043 | 0.5 | 0.00 | 0.56 | 0.00 | 0.001 | 0.006 | 0.556 | 0.000 | 259 | 435 | prepared | yes | yes | +| image | 210,000 | recenter | 210,000 | 91.0 | 0.16 | 1.17 | 0.09 | 0.282 | 0.391 | 0.480 | 0.013 | 442 | 601 | prepared | yes | yes | +| image | 210,000 | recenter_advance | 16 | 0.0 | 0.00 | 0.49 | 0.00 | 0.000 | 0.002 | 0.430 | 0.059 | 444 | 601 | prepared | yes | yes | +| image | 400,000 | full | 400,000 | 173.6 | 0.23 | 2.60 | 0.30 | 0.529 | 0.847 | 1.202 | 0.021 | 479 | 826 | prepared | yes | yes | +| image | 400,000 | advance_small | 16 | 0.0 | 0.00 | 1.05 | 0.00 | 0.000 | 0.005 | 1.040 | 0.000 | 483 | 826 | prepared | yes | yes | +| image | 400,000 | advance_large | 1,043 | 0.5 | 0.00 | 1.09 | 0.00 | 0.001 | 0.006 | 1.078 | 0.000 | 486 | 826 | prepared | yes | yes | +| image | 400,000 | recenter | 400,000 | 173.6 | 0.31 | 2.12 | 0.16 | 0.517 | 0.679 | 0.900 | 0.021 | 838 | 1,142 | prepared | yes | yes | +| image | 400,000 | recenter_advance | 16 | 0.0 | 0.00 | 0.90 | 0.00 | 0.000 | 0.004 | 0.784 | 0.117 | 840 | 1,142 | prepared | yes | yes | + +The image's peak and after-phase resident sizes are identical to the host +build's at both sizes — the daemon is single-threaded and its footprint is +page vectors and fragment buffers, so neither the arena policy nor the +compiler moved it. Its fold is 0.1 s slower at 210k (rustc 1.85 codegen +against 1.97 on the host), still under half a second. The CPython 3.14.7 +oracle produced the same digests as CPython 3.12.3 on the host +(`e38ca96a869cee23…` at 210k, `30b53c3312705f2c…` at 400k), and the image's +in-process fold at 210k took 2.11 s against 1.58 s on the host interpreter. + +This is the production image's interpreter, allocator setting and binary +exercising the daemon protocol; it is **not** a coordinator run (§8). + +--- + +## 8. Limits + +- Single run per configuration on a shared KVM development host. Timings are + wall-clock with load 0.8–1.7; the memory figures are kernel high-water + marks and are not sensitive to load. The earlier pre-hardening host sweep + reproduced every peak figure when rounded to MiB. +- Synthetic fixture: uniform difficulty, one share per second, 200 + identities. Production windows have mixed difficulties and credit policies; + neither changes the paging arithmetic, which depends only on record count + and page size. +- Not a coordinator run, on the host or in the image: no ledger read, record + conversion, mirror validation, lease heartbeat, Stratum job build or + restart loop. The #236 gates — cold-start time to first usable Stratum + work at the production window with the heartbeat running, builder RSS in + the deployed container, and the testnet/mainnet soak — remain operator + gates and are not claimed here. +- The image was built on this host from the branch's sources; it is not the + registry image the deployment pulls, though it is the same Dockerfile. +- `RLIMIT_AS` bounds virtual size, which for this daemon tracked resident + size within 1–8% (peak VSZ column); it is a safety net, not a measurement. + +--- + +## Re-running + +``` +cargo build --release --locked -p qbit-prism --bin qbit-prism-build-audit-bundle +# an old binary for the baseline rows, built from the base commit into its own target dir: +git worktree add /tmp/base 0bc7fa6c023082df81d23137ebabf4ff96f24683 && \ + (cd /tmp/base && CARGO_TARGET_DIR=/tmp/base-target cargo build --release --locked -p qbit-prism --bin qbit-prism-build-audit-bundle) + +python3 tests/perf/window_paging_allocation.py \ + --daemon-binary target/release/qbit-prism-build-audit-bundle \ + --baseline-binary /tmp/base-target/release/qbit-prism-build-audit-bundle \ + --sizes 52000,105000,210000,400000 --baseline-sizes 52000,105000 \ + --json window_paging_allocation.json + +python3 tests/perf/window_paging_allocation.py --sizes 210000 --skip-oracle # fixed binary only, quick + +# inside a locally built coordinator image (context: Cargo.toml, Cargo.lock, crates/, lab/auxpow/, lab/prism/): +docker build -f lab/prism/Dockerfile -t prism-240-local:test +docker run --rm --memory 8g -v "$PWD/tests:/app/tests:ro" -v "$PWD/out:/out" prism-240-local:test \ + python3 tests/perf/window_paging_allocation.py --daemon-binary /app/target/release --daemon-label image \ + --sizes 210000,400000 --stderr-dir /tmp/stderr --json /out/container.json +``` + +Check `MemAvailable` first; the driver refuses a run below the limit plus +margin (`--daemon-memory-limit-mb`, `--memory-margin-mb`) and records the +skip, and kills a daemon that stalls past `--exchange-timeout`. Never point +`--baseline-binary` at sizes above ~105k on a host without tens of gigabytes +to spare. diff --git a/tests/perf/window_paging_allocation.py b/tests/perf/window_paging_allocation.py new file mode 100644 index 000000000..8c3106c9c --- /dev/null +++ b/tests/perf/window_paging_allocation.py @@ -0,0 +1,1236 @@ +#!/usr/bin/env python3 +"""Rust payout-window paging allocation: builder RSS and timing (#240). + +``PayoutWindow::from_full_snapshot`` (``crates/qbit-prism/src/window.rs``) +paged the retained records with ``remaining.split_off(page_size)`` in a +loop. ``Vec::split_off`` keeps the original vector's allocation, so every +512-record page retained a backing allocation sized for every record still +unpaged when it was cut, and the tail was copied again on each iteration: +O(N^2/P) retained memory and copying. A prior synthetic reproduction for +#236 measured 9,613 MiB builder RSS and a 12 s response at 210k shares, and +a SIGKILL at 400k. This driver measures that path against the real +``--serve`` daemon binary, before and after the fix, with every experiment +bounded so it cannot exhaust the host. + +Per binary and window size it runs one daemon through the coordinator's own +``prepare_window`` protocol over blocking pipes: + +===================== ====================================================== +phase what it exercises +===================== ====================================================== +``full`` cold full-snapshot preparation of the whole fixture +``advance_small`` ``advance`` with a handful of appended records +``advance_large`` ``advance`` with a multi-page delta (the delta paging + loop had the same ``split_off`` shape) +``recenter`` a second full preparation on the same daemon at a + smaller weight -- what the coordinator's self-check + re-center sends while the previous window is held +``recenter_advance`` ``advance`` against the re-centered digest +===================== ====================================================== + +and reports, per phase, wall time split into ``request_write`` (until the +pipe accepted the last request byte), ``response_wait`` (from then until the +envelope line arrived) and ``response_read`` (the raw canonical-items +section), the daemon's own ``metrics`` (JSON parse, fold/advance, canonical +serialization -- Rust processing measured inside the process, so transport +and processing are separated rather than inferred), the daemon's resident +set after the phase, and the observed peak with source provenance. All peak +observations precede process exit and are labelled lower bounds on the final +lifetime peak, including kernel ``VmHWM`` reads. The derived +``residual`` column is ``response_wait`` minus the daemon's three timers; the +client's wall interval and the daemon's internal timers are independent +intervals on either side of a pipe, so it is an approximation of scheduling +and envelope handling, not a measured transport term. Every digest the +daemon returns is checked against its own items bytes, the coordinator's +mirror surgery (drop prefix, append suffix) is replayed and re-hashed, and +unless ``--skip-oracle`` is given the shipped Python fold +(``IncrementalShareWindow``) is run on the same inputs as an independent +oracle for every digest. + +Bounding: the daemon runs under ``RLIMIT_AS`` (``--daemon-memory-limit-mb``, +default 6144), so a quadratic binary aborts on allocation failure instead of +taking the host down; a size is skipped, and the skip recorded, when +``MemAvailable`` is below the limit plus ``--memory-margin-mb``; and every +exchange -- the handshake included -- runs under a wall-clock deadline +(``--exchange-timeout``) after which the daemon is killed, reaped and the +phase recorded as a failure, so a stalled daemon cannot hang the run. The +known-quadratic baseline should only be run at sizes whose expected footprint +fits the limit (52k and 105k here); the 210k/400k baseline numbers are on +record in ``window_pipeline_gil_scaling.md`` section 9 and are not repeated. + +Fixture: ``build_records`` from ``window_pipeline_gil_scaling.py`` -- the +same production-shaped ``share_id`` fixture (200 identities, difficulty +16384) the #236 probe and the cited baseline used -- with ``window_weight`` +equal to the fixture's total difficulty so every record is retained. + +Self-contained: standard library plus read-only imports from ``lab/``; no +database, network or coordinator. Example:: + + cargo build --release --locked -p qbit-prism --bin qbit-prism-build-audit-bundle + python3 tests/perf/window_paging_allocation.py \ + --daemon-binary target/release/qbit-prism-build-audit-bundle \ + --baseline-binary /path/to/old/qbit-prism-build-audit-bundle \ + --sizes 52000,105000,210000,400000 --baseline-sizes 52000,105000 \ + --json window_paging_allocation.json + +This is an **on-demand instrument, not a test**: it asserts no thresholds +and is deliberately not named ``test_*`` so discovery never runs it (#160). +Nothing under ``lab/`` is modified. + +Memory measurements use MiB (2**20 bytes). Existing ``_mb`` result keys and +``*-mb`` CLI option names are retained and use the same binary units. +""" + +from __future__ import annotations + +import argparse +import hashlib +import json +import os +import platform +import resource +import subprocess +import sys +import threading +import time +from dataclasses import asdict, dataclass, field +from pathlib import Path +from typing import Any, Callable + +if not __package__: + sys.path.insert(0, str(Path(__file__).resolve().parents[2])) + +from lab.prism.bundle_compiler import ( # noqa: E402 - path fix-up above + PRISM_SERVE_BUILDER_PROTOCOL_VERSION, +) +from lab.prism.share_ledger import ( # noqa: E402 + DEFAULT_INCREMENTAL_SHARE_WINDOW_PAGE_SIZE, + AcceptedShareRecord, + IncrementalShareWindow, +) +from tests.perf.window_pipeline_gil_scaling import ( # noqa: E402 + benchmark_miner_programs, + build_records, +) + + +REPO_ROOT = Path(__file__).resolve().parents[2] +DAEMON_BINARY = "qbit-prism-build-audit-bundle" +DEFAULT_TOOL_BIN_DIR = REPO_ROOT / "target" / "release" +DEFAULT_SIZES = "52000,105000,210000,400000" +DEFAULT_BASELINE_SIZES = "52000,105000" +DEFAULT_MINERS = 200 +DEFAULT_SMALL_DELTA = 16 +DEFAULT_LARGE_DELTA = 2 * DEFAULT_INCREMENTAL_SHARE_WINDOW_PAGE_SIZE + 19 +WRITE_CHUNK = 1 << 20 +DEFAULT_EXCHANGE_TIMEOUT = 600.0 +DEFAULT_SHUTDOWN_TIMEOUT = 30.0 +PHASES = ("full", "advance_small", "advance_large", "recenter", "recenter_advance") + + +# --------------------------------------------------------------------------- +# host observation +# --------------------------------------------------------------------------- + + +def read_meminfo_mb(key: str) -> float | None: + try: + with open("/proc/meminfo", encoding="ascii") as handle: + for line in handle: + if line.startswith(key + ":"): + return int(line.split()[1]) / 1024.0 + except OSError: + return None + return None + + +def read_proc_status_mb(pid: int) -> dict[str, float]: + """``VmRSS``/``VmHWM``/``VmSize``/``VmPeak`` of a live process, in MiB.""" + out: dict[str, float] = {} + try: + with open(f"/proc/{pid}/status", encoding="ascii") as handle: + for line in handle: + key, _, rest = line.partition(":") + if key in ("VmRSS", "VmHWM", "VmSize", "VmPeak"): + out[key] = int(rest.split()[0]) / 1024.0 + except (OSError, ValueError): + pass + return out + + +def machine_info() -> dict[str, Any]: + try: + load = os.getloadavg() + except OSError: + load = None + return { + "platform": platform.platform(), + "python": sys.version.split()[0], + "implementation": platform.python_implementation(), + "cpu_count": os.cpu_count(), + "mem_total_mb": read_meminfo_mb("MemTotal"), + "mem_available_mb_at_start": read_meminfo_mb("MemAvailable"), + "load_average_at_start": load, + "malloc_arena_max": os.environ.get("MALLOC_ARENA_MAX"), + } + + +def git_describe(path: Path) -> dict[str, Any]: + """The source revision this driver runs from, and whether it is clean.""" + def run(*args: str) -> str | None: + try: + return subprocess.run( + ["git", *args], cwd=path, capture_output=True, text=True, check=True + ).stdout.strip() + except (OSError, subprocess.CalledProcessError): + return None + + return { + "commit": run("rev-parse", "HEAD"), + "branch": run("rev-parse", "--abbrev-ref", "HEAD"), + "dirty": bool(run("status", "--porcelain", "--untracked-files=no")), + } + + +def binary_identity(binary: Path) -> dict[str, Any]: + digest = hashlib.sha256() + with open(binary, "rb") as handle: + for chunk in iter(lambda: handle.read(1 << 20), b""): + digest.update(chunk) + stat = binary.stat() + return { + "path": str(binary), + "sha256": digest.hexdigest(), + "size_bytes": stat.st_size, + "mtime": time.strftime("%Y-%m-%dT%H:%M:%S", time.localtime(stat.st_mtime)), + } + + +# --------------------------------------------------------------------------- +# RSS sampling +# --------------------------------------------------------------------------- + + +class RssSampler: + """Samples ``VmRSS`` (and the running ``VmHWM``) of one pid on a thread. + + Kernel high-water marks and timeline maxima are observations made while + the child can still allocate, so both are lower bounds on its final + lifetime peak (see :func:`resolve_peak_rss`). + """ + + def __init__(self, pid: int, interval: float) -> None: + self.pid = pid + self.interval = interval + self.samples: list[tuple[float, float]] = [] + self.max_sampled_mb = 0.0 + self.max_sampled_hwm_mb = 0.0 + self._stop = threading.Event() + self._thread = threading.Thread(target=self._run, name="rss-sampler", daemon=True) + + def _run(self) -> None: + while not self._stop.is_set(): + status = read_proc_status_mb(self.pid) + rss = status.get("VmRSS") + if rss is not None: + self.samples.append((time.perf_counter(), rss)) + self.max_sampled_mb = max(self.max_sampled_mb, rss) + hwm = status.get("VmHWM") + if hwm is not None: + self.max_sampled_hwm_mb = max(self.max_sampled_hwm_mb, hwm) + self._stop.wait(self.interval) + + def __enter__(self) -> RssSampler: + self._thread.start() + return self + + def __exit__(self, *_exc: object) -> None: + self._stop.set() + self._thread.join() + + +# --------------------------------------------------------------------------- +# peak provenance +# --------------------------------------------------------------------------- + + +PEAK_SOURCE_LABELS = { + "vmhwm": "lower bound (kernel VmHWM at close)", + "vmhwm_before_kill": "lower bound (kernel VmHWM before watchdog kill)", + "lower_bound": "lower bound (highest earlier VmHWM/RSS observation)", + "unavailable": "unavailable", +} + + +def resolve_peak_rss(outcome: dict[str, Any], observations: list[float | None]) -> dict[str, Any]: + """Settle ``peak_rss_mb`` and its provenance in a daemon outcome. + + Keep the numeric observation and its source. A live kernel ``VmHWM`` + read at close or before SIGKILL is a lower bound: the daemon can still + allocate before it exits or the signal is delivered. Earlier phase and + sampler maxima are also lower bounds. None of these observations proves + the final lifetime peak, even if shutdown later exits cleanly. + """ + if outcome.get("peak_rss_mb") is not None: + outcome.setdefault("peak_rss_source", "vmhwm") + return outcome + candidates = [value for value in observations if value] + if candidates: + outcome["peak_rss_mb"] = max(candidates) + outcome["peak_rss_source"] = "lower_bound" + else: + outcome["peak_rss_mb"] = None + outcome["peak_rss_source"] = "unavailable" + return outcome + + +def format_peak(value: float | None, source: str | None, digits: int = 0) -> str: + """Render a peak figure, prefixing a lower bound with ``≥``.""" + if value is None: + return "-" + text = f"{value:.{digits}f}" + return f"≥ {text}" + + +def peak_source_label(source: str | None) -> str: + return PEAK_SOURCE_LABELS.get(source or "unavailable", str(source)) + + +# --------------------------------------------------------------------------- +# the daemon exchange +# --------------------------------------------------------------------------- + + +@dataclass +class Exchange: + request_bytes: int + request_write_seconds: float + response_wait_seconds: float + response_read_seconds: float + envelope: dict[str, Any] | None + payload: bytes | None + error: str | None = None + + +class DaemonTimeout(RuntimeError): + """An exchange (or the handshake) exceeded its wall-clock deadline.""" + + +class Daemon: + """One ``--serve`` daemon over blocking pipes, memory- and time-bounded. + + Every exchange, the handshake included, runs under ``exchange_timeout``: + a watchdog kills the daemon when it expires, which unblocks the pending + pipe read (EOF) and the pending write (EPIPE), and the exchange is + reported as a timeout failure. ``close`` is idempotent and always reaps + the child and closes every pipe and the stderr capture; the constructor + calls it before re-raising if the handshake fails. + """ + + def __init__( + self, + binary: Path, + *, + memory_limit_mb: int | None, + stderr_path: Path, + exchange_timeout: float = DEFAULT_EXCHANGE_TIMEOUT, + shutdown_timeout: float = DEFAULT_SHUTDOWN_TIMEOUT, + ) -> None: + self.binary = binary + self.stderr_path = stderr_path + self.exchange_timeout = exchange_timeout + self.shutdown_timeout = shutdown_timeout + self.exit_code: int | None = None + self.timed_out: str | None = None + self.status_at_kill: dict[str, float] = {} + self._outcome: dict[str, Any] | None = None + self._lock = threading.Lock() + limit_bytes = None if not memory_limit_mb else memory_limit_mb * 1024 * 1024 + + def preexec() -> None: + if limit_bytes is not None: + resource.setrlimit(resource.RLIMIT_AS, (limit_bytes, limit_bytes)) + + self._stderr = open(stderr_path, "wb") + try: + self.process = subprocess.Popen( + [ + str(binary), + "--serve", + "--signing-key-seed-hex", + "42" * 32, + "--ledger-signing-key-seed-hex", + "43" * 32, + ], + stdin=subprocess.PIPE, + stdout=subprocess.PIPE, + stderr=self._stderr, + preexec_fn=preexec, + ) + except Exception: + self._stderr.close() + raise + try: + assert self.process.stdout is not None + with self._watchdog("handshake"): + handshake_line = self.process.stdout.readline() + if self.timed_out: + raise DaemonTimeout( + f"{binary} sent no handshake within {exchange_timeout:g}s; killed" + ) + if not handshake_line: + raise RuntimeError(f"{binary} produced no handshake (see {stderr_path})") + handshake = json.loads(handshake_line) + expected = { + "event": "handshake", + "tool": DAEMON_BINARY, + "protocol": PRISM_SERVE_BUILDER_PROTOCOL_VERSION, + } + announced = {key: handshake.get(key) for key in expected} + if announced != expected: + raise RuntimeError(f"daemon announced {announced!r}, expected {expected!r}") + except BaseException as error: + outcome = self.close() + if isinstance(error, Exception): + # Let a caller (or a test) see how the child was reaped. + error.daemon_outcome = outcome # type: ignore[attr-defined] + raise + + @property + def pid(self) -> int: + return self.process.pid + + def alive(self) -> bool: + return self.process.poll() is None + + def status_mb(self) -> dict[str, float]: + return read_proc_status_mb(self.process.pid) + + def _kill_on_timeout(self, label: str) -> None: + with self._lock: + if self.timed_out is None: + self.timed_out = label + # The high-water mark is lost once the child is reaped, so + # sample it before the kill. + self.status_at_kill = self.status_mb() + if self.process.poll() is None: + self.process.kill() + + class _Watchdog: + def __init__(self, daemon: Daemon, label: str) -> None: + self.timer = threading.Timer(daemon.exchange_timeout, daemon._kill_on_timeout, (label,)) + self.timer.daemon = True + + def __enter__(self) -> Daemon._Watchdog: + self.timer.start() + return self + + def __exit__(self, *_exc: object) -> None: + self.timer.cancel() + + def _watchdog(self, label: str) -> Daemon._Watchdog: + return Daemon._Watchdog(self, label) + + def exchange(self, request: bytes, label: str = "exchange") -> Exchange: + """Write one request line, read the envelope and its raw section. + + Bounded by ``exchange_timeout`` end to end; a timeout kills the daemon + and is reported in ``Exchange.error``. + """ + process = self.process + assert process.stdin is not None and process.stdout is not None + if self.timed_out or process.poll() is not None: + return Exchange(len(request), 0.0, 0.0, 0.0, None, None, error="daemon is not running") + stdin_fd = process.stdin.fileno() + stdout = process.stdout + write_done: list[float] = [] + write_error: list[str] = [] + + def writer() -> None: + view = memoryview(request) + offset = 0 + try: + while offset < len(view): + offset += os.write(stdin_fd, view[offset : offset + WRITE_CHUNK]) + except OSError as exc: + write_error.append(f"{type(exc).__name__}: {exc}") + finally: + write_done.append(time.perf_counter()) + + def timeout_error(detail: str) -> str: + return f"{label} exceeded {self.exchange_timeout:g}s; daemon killed ({detail})" + + with self._watchdog(label): + started = time.perf_counter() + thread = threading.Thread(target=writer, name="request-writer", daemon=True) + thread.start() + envelope_line = stdout.readline() + envelope_at = time.perf_counter() + # The write either finished, failed with EPIPE once the daemon + # died, or is stuck behind a daemon that stopped reading, which + # the watchdog kill also unblocks; the join is bounded regardless. + thread.join(timeout=max(1.0, self.exchange_timeout)) + if thread.is_alive(): + write_done.append(time.perf_counter()) + write_error.append("request writer still blocked after the exchange") + write_seconds = write_done[0] - started + wait_seconds = envelope_at - write_done[0] + if not envelope_line: + detail = "closed stdout without an envelope" + ( + f"; write: {write_error[0]}" if write_error else "" + ) + error = timeout_error(detail) if self.timed_out else f"daemon {detail}" + return Exchange(len(request), write_seconds, wait_seconds, 0.0, None, None, error=error) + envelope = json.loads(envelope_line) + payload: bytes | None = None + read_seconds = 0.0 + if envelope.get("ok") is True: + length_key = "window_items_len" if "window_items_len" in envelope else "appended_items_len" + length = int(envelope[length_key]) + chunks: list[bytes] = [] + remaining = length + while remaining > 0: + chunk = stdout.read(min(remaining, 1 << 22)) + if not chunk: + break + chunks.append(chunk) + remaining -= len(chunk) + newline = stdout.read(1) + read_seconds = time.perf_counter() - envelope_at + payload = b"".join(chunks) + if remaining or newline != b"\n": + detail = f"short raw section: {len(payload)} of {length} bytes" + error = timeout_error(detail) if self.timed_out else detail + return Exchange(len(request), write_seconds, wait_seconds, read_seconds, envelope, payload, error=error) + return Exchange(len(request), write_seconds, wait_seconds, read_seconds, envelope, payload) + + def close(self) -> dict[str, Any]: + """Reap the daemon and close every handle; idempotent. + + Closing stdin asks a healthy daemon to exit; one that has not exited + within ``shutdown_timeout`` is killed. ``VmHWM`` is read before the + child is reaped (it is gone from ``/proc`` afterwards); the outcome's + ``peak_rss_source`` records whether the figure is that live read + (``vmhwm``), the watchdog's read before a kill + (``vmhwm_before_kill``), or ``unavailable`` because the daemon had + already exited -- :func:`resolve_peak_rss` may then substitute a + labelled lower bound. All live reads are lower bounds too: memory + may still grow before normal exit or before SIGKILL is delivered. + """ + if self._outcome is not None: + return self._outcome + process = self.process + final_status = self.status_mb() if process.poll() is None else {} + if process.stdin is not None: + try: + process.stdin.close() + except OSError: + pass + try: + self.exit_code = process.wait(timeout=self.shutdown_timeout) + except subprocess.TimeoutExpired: + process.kill() + self.exit_code = process.wait(timeout=10.0) + if process.stdout is not None: + process.stdout.close() + self._stderr.close() + stderr_tail = "" + try: + stderr_tail = self.stderr_path.read_text(errors="replace")[-2000:] + except OSError: + pass + if final_status.get("VmHWM") is not None: + peak, peak_source = final_status["VmHWM"], "vmhwm" + elif self.status_at_kill.get("VmHWM") is not None: + peak, peak_source = self.status_at_kill["VmHWM"], "vmhwm_before_kill" + else: + peak, peak_source = None, "unavailable" + peak_vsize = final_status.get("VmPeak", self.status_at_kill.get("VmPeak")) + self._outcome = { + "pid": process.pid, + "exit_code": self.exit_code, + "signal": -self.exit_code if self.exit_code is not None and self.exit_code < 0 else None, + "timed_out": self.timed_out, + "peak_rss_mb": peak, + "peak_rss_source": peak_source, + "peak_vsize_mb": peak_vsize, + "stderr_tail": stderr_tail, + } + return self._outcome + + +# --------------------------------------------------------------------------- +# fixture and requests +# --------------------------------------------------------------------------- + + +def delta_records(last: AcceptedShareRecord, count: int, *, miners: int, offset: int) -> list[AcceptedShareRecord]: + """``count`` appended records after ``last``, shaped like the probe's delta.""" + programs = benchmark_miner_programs(miners) + out: list[AcceptedShareRecord] = [] + for index in range(count): + seq = int(last.share_seq) + offset + index + 1 + miner = seq % miners + out.append( + AcceptedShareRecord( + share_seq=seq, + share_id=f"bench-miner-{miner}.rig{seq % 512}:{'ab' * 32}", + miner_id=f"bench-miner-{miner}", + order_key=f"bench-miner-{miner:06d}", + p2mr_program_hex=programs[miner], + share_difficulty=int(last.share_difficulty), + network_difficulty=int(last.network_difficulty), + template_height=int(last.template_height), + job_id=f"bench-job-{seq}", + job_issued_at_ms=int(last.job_issued_at_ms) + offset + index + 1, + accepted_at_ms=int(last.accepted_at_ms) + offset + index + 1, + ntime=int(last.ntime) + offset + index + 1, + ) + ) + return out + + +def encode_request(fields: dict[str, Any]) -> bytes: + """The bytes the coordinator sends: one compact JSON line.""" + return json.dumps(fields, separators=(",", ":")).encode("ascii") + b"\n" + + +def full_request(records_json: list[dict[str, object]], *, anchor: int, weight: int, page_size: int) -> bytes: + return encode_request( + { + "request": "prepare_window", + "mode": "full", + "append_invalidation_epoch": 0, + "anchor_job_issued_at_ms": anchor, + "records": records_json, + "window_weight": weight, + "page_size": page_size, + } + ) + + +def advance_request(records_json: list[dict[str, object]], *, anchor: int, base_digest: str) -> bytes: + return encode_request( + { + "request": "prepare_window", + "mode": "advance", + "append_invalidation_epoch": 0, + "anchor_job_issued_at_ms": anchor, + "records": records_json, + "base_digest": base_digest, + } + ) + + +def items_digest(items: bytes) -> str: + return hashlib.sha256(b"[" + items + b"]").hexdigest() + + +@dataclass +class Fixture: + size: int + page_size: int + records: list[AcceptedShareRecord] + records_json: list[dict[str, object]] + anchor: int + weight: int + recenter_weight: int + small_delta: list[AcceptedShareRecord] + small_anchor: int + large_delta: list[AcceptedShareRecord] + large_anchor: int + recenter_delta: list[AcceptedShareRecord] + recenter_delta_anchor: int + build_seconds: float + + +def build_fixture(size: int, *, miners: int, page_size: int, small: int, large: int) -> Fixture: + started = time.perf_counter() + records = build_records(size, miners=miners, share_id_shape="production") + anchor = int(records[-1].job_issued_at_ms) + weight = sum(int(record.share_difficulty) for record in records) + last = records[-1] + small_delta = delta_records(last, small, miners=miners, offset=0) + small_anchor = int(last.job_issued_at_ms) + small + 1 + large_delta = delta_records(last, large, miners=miners, offset=small + 1) + large_anchor = int(last.job_issued_at_ms) + small + 1 + large + 1 + # The re-center starts again from the original snapshot at a smaller + # live weight; its advance appends after the original last record. + recenter_delta = delta_records(last, small, miners=miners, offset=0) + return Fixture( + size=size, + page_size=page_size, + records=records, + records_json=[record.to_prism_json() for record in records], + anchor=anchor, + weight=weight, + recenter_weight=(weight * 3) // 4, + small_delta=small_delta, + small_anchor=small_anchor, + large_delta=large_delta, + large_anchor=large_anchor, + recenter_delta=recenter_delta, + recenter_delta_anchor=small_anchor, + build_seconds=time.perf_counter() - started, + ) + + +# --------------------------------------------------------------------------- +# the Python oracle +# --------------------------------------------------------------------------- + + +def python_oracle(fixture: Fixture) -> dict[str, Any]: + """Every expected digest from the shipped in-process fold.""" + timings: dict[str, float] = {} + digests: dict[str, str] = {} + counts: dict[str, int] = {} + + def timed(name: str, run: Callable[[], IncrementalShareWindow]) -> IncrementalShareWindow: + started = time.perf_counter() + window = run() + timings[name] = time.perf_counter() - started + digests[name] = window.json_records().canonical_json_sha256() + counts[name] = int(window.record_count) + return window + + full = timed( + "full", + lambda: IncrementalShareWindow.from_full_snapshot( + fixture.records, + anchor_job_issued_at_ms=fixture.anchor, + window_weight=fixture.weight, + page_size=fixture.page_size, + ), + ) + small = timed( + "advance_small", + lambda: full.advance(fixture.small_delta, anchor_job_issued_at_ms=fixture.small_anchor)[0], + ) + timed( + "advance_large", + lambda: small.advance(fixture.large_delta, anchor_job_issued_at_ms=fixture.large_anchor)[0], + ) + recentered = timed( + "recenter", + lambda: IncrementalShareWindow.from_full_snapshot( + fixture.records, + anchor_job_issued_at_ms=fixture.anchor, + window_weight=fixture.recenter_weight, + page_size=fixture.page_size, + ), + ) + timed( + "recenter_advance", + lambda: recentered.advance( + fixture.recenter_delta, anchor_job_issued_at_ms=fixture.recenter_delta_anchor + )[0], + ) + return {"digests": digests, "record_counts": counts, "seconds": timings} + + +# --------------------------------------------------------------------------- +# one daemon run +# --------------------------------------------------------------------------- + + +@dataclass +class PhaseResult: + phase: str + records_sent: int + request_bytes: int + request_write_seconds: float + response_wait_seconds: float + response_read_seconds: float + daemon_metrics: dict[str, float] = field(default_factory=dict) + status: str = "" + digest: str | None = None + record_count: int | None = None + items_bytes: int | None = None + self_consistent: bool | None = None + oracle_match: bool | None = None + rss_after_mb: float | None = None + hwm_after_mb: float | None = None + stats: dict[str, int] = field(default_factory=dict) + error: str | None = None + + @property + def residual_wait_seconds(self) -> float | None: + """``response_wait`` minus the daemon's own timers -- an approximation. + + The client's wall interval (from the last request byte accepted to + the envelope line received) and the daemon's ``Instant`` timers are + independent intervals measured on either side of a pipe: the daemon + may still be reading and parsing while the client clock starts, and + the envelope write and scheduling are on neither timer. So this is + roughly scheduling plus envelope handling, not a measured transport + term, and it is clamped at zero. None when the daemon reported no + metrics. + """ + if not self.daemon_metrics: + return None + return max(0.0, self.response_wait_seconds - sum(self.daemon_metrics.values())) + + +@dataclass +class RunResult: + variant: str + binary: str + size: int + page_size: int + mem_available_before_mb: float | None + load_before: tuple[float, float, float] | None + memory_limit_mb: int | None + phases: list[PhaseResult] + outcome: dict[str, Any] + max_sampled_rss_mb: float + rss_timeline: list[tuple[float, float]] + skipped: str | None = None + error: str | None = None + + +def run_variant( + label: str, + binary: Path, + fixture: Fixture, + *, + oracle: dict[str, Any] | None, + memory_limit_mb: int | None, + memory_margin_mb: int, + sample_interval: float, + exchange_timeout: float, + stderr_dir: Path, + log: Callable[[str], None], +) -> RunResult: + available = read_meminfo_mb("MemAvailable") + try: + load = os.getloadavg() + except OSError: + load = None + required = (memory_limit_mb or 0) + memory_margin_mb + if available is not None and available < required: + reason = f"MemAvailable {available:.0f} MiB below required {required} MiB" + log(f" {label} @ {fixture.size}: skipped, {reason}") + return RunResult( + label, str(binary), fixture.size, fixture.page_size, available, load, + memory_limit_mb, [], {}, 0.0, [], skipped=reason, + ) + + stderr_path = stderr_dir / f"{label}-{fixture.size}.stderr" + try: + daemon = Daemon( + binary, + memory_limit_mb=memory_limit_mb, + stderr_path=stderr_path, + exchange_timeout=exchange_timeout, + ) + except Exception as exc: # noqa: BLE001 - reported per run, not raised + error = f"daemon failed to start: {type(exc).__name__}: {exc}" + log(f" {label} @ {fixture.size}: {error}") + return RunResult( + label, str(binary), fixture.size, fixture.page_size, available, load, + memory_limit_mb, [], getattr(exc, "daemon_outcome", {}), 0.0, [], error=error, + ) + phases: list[PhaseResult] = [] + expected = (oracle or {}).get("digests", {}) + + def run_phase(name: str, request: bytes, records_sent: int, *, base_items: bytes | None) -> tuple[PhaseResult, bytes | None]: + log(f" {label} @ {fixture.size}: {name} ({records_sent} records, {len(request) / 2**20:.1f} MiB request)") + exchange = daemon.exchange(request, label=name) + status_after = daemon.status_mb() if daemon.alive() else {} + result = PhaseResult( + phase=name, + records_sent=records_sent, + request_bytes=exchange.request_bytes, + request_write_seconds=exchange.request_write_seconds, + response_wait_seconds=exchange.response_wait_seconds, + response_read_seconds=exchange.response_read_seconds, + rss_after_mb=status_after.get("VmRSS"), + hwm_after_mb=status_after.get("VmHWM"), + error=exchange.error, + ) + envelope = exchange.envelope + if envelope is None: + result.status = "timeout" if daemon.timed_out else "no_response" + return result, None + metrics = envelope.get("metrics") + if isinstance(metrics, dict): + result.daemon_metrics = {key: float(value) for key, value in metrics.items()} + if envelope.get("ok") is not True: + for key in ("needs_full", "fallback", "fold_invalid", "out_of_range"): + if envelope.get(key): + result.status = key + break + else: + result.status = "error" + result.error = str(envelope.get("error")) + return result, None + if exchange.error: + result.status = "timeout" if daemon.timed_out else "short_response" + return result, None + result.status = "prepared" + result.digest = str(envelope["share_snapshot_sha256"]) + result.record_count = int(envelope["record_count"]) + payload = exchange.payload or b"" + if "window_items_len" in envelope: + items = payload + else: + assert base_items is not None + items = base_items[int(envelope["retained_drop_bytes"]) :] + payload + result.stats = { + key: int(envelope[key]) for key in ("added_rows", "expired_rows", "touched_pages") + } + result.items_bytes = len(items) + result.self_consistent = items_digest(items) == result.digest + if name in expected: + result.oracle_match = expected[name] == result.digest + return result, items + + run_error: str | None = None + max_sampled = 0.0 + max_sampled_hwm = 0.0 + timeline: list[tuple[float, float]] = [] + try: + with RssSampler(daemon.pid, sample_interval) as sampler: + try: + full, items = run_phase( + "full", + full_request(fixture.records_json, anchor=fixture.anchor, weight=fixture.weight, page_size=fixture.page_size), + fixture.size, + base_items=None, + ) + phases.append(full) + if full.status == "prepared" and items is not None and daemon.alive(): + small, items_small = run_phase( + "advance_small", + advance_request([r.to_prism_json() for r in fixture.small_delta], anchor=fixture.small_anchor, base_digest=full.digest or ""), + len(fixture.small_delta), + base_items=items, + ) + phases.append(small) + if small.status == "prepared" and items_small is not None and daemon.alive(): + large, _ = run_phase( + "advance_large", + advance_request([r.to_prism_json() for r in fixture.large_delta], anchor=fixture.large_anchor, base_digest=small.digest or ""), + len(fixture.large_delta), + base_items=items_small, + ) + phases.append(large) + if daemon.alive() and full.status == "prepared": + recenter, items_recenter = run_phase( + "recenter", + full_request(fixture.records_json, anchor=fixture.anchor, weight=fixture.recenter_weight, page_size=fixture.page_size), + fixture.size, + base_items=None, + ) + phases.append(recenter) + if recenter.status == "prepared" and items_recenter is not None and daemon.alive(): + recenter_advance, _ = run_phase( + "recenter_advance", + advance_request([r.to_prism_json() for r in fixture.recenter_delta], anchor=fixture.recenter_delta_anchor, base_digest=recenter.digest or ""), + len(fixture.recenter_delta), + base_items=items_recenter, + ) + phases.append(recenter_advance) + except Exception as exc: # noqa: BLE001 - reported per run, not raised + run_error = f"{type(exc).__name__}: {exc}" + log(f" {label} @ {fixture.size}: aborted: {run_error}") + finally: + max_sampled = sampler.max_sampled_mb + max_sampled_hwm = sampler.max_sampled_hwm_mb + timeline = sampler.samples + finally: + # Always reaps the child and closes every pipe and the stderr + # capture, whatever happened above. + outcome = daemon.close() + # A daemon that exited on its own took its VmHWM with it; what remains + # is a labelled lower bound from the observations made while it lived. + resolve_peak_rss( + outcome, + [phase.hwm_after_mb for phase in phases] + [max_sampled_hwm, max_sampled], + ) + return RunResult( + label, + str(binary), + fixture.size, + fixture.page_size, + available, + load, + memory_limit_mb, + phases, + outcome, + max_sampled, + timeline, + error=run_error, + ) + + +# --------------------------------------------------------------------------- +# rendering +# --------------------------------------------------------------------------- + + +def _fmt(value: float | None, digits: int = 2) -> str: + return "-" if value is None else f"{value:.{digits}f}" + + +def _flag(value: bool | None) -> str: + if value is None: + return "n/a" + return "yes" if value else "NO" + + +def render_runs(runs: list[RunResult]) -> str: + lines = [ + "| binary | shares | phase | records | request MiB | write s | wait s | read s | parse s | fold s | serialize s | residual s (approx.) | RSS after MiB | status | self-check | oracle |", + "|---|---:|---|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---|---|---|", + ] + for run in runs: + if run.skipped or (run.error and not run.phases): + lines.append(f"| {run.variant} | {run.size:,} | {'skipped' if run.skipped else 'failed'} | | | | | | | | | | | {run.skipped or run.error} | | |") + continue + for phase in run.phases: + metrics = phase.daemon_metrics + lines.append( + f"| {run.variant} | {run.size:,} | {phase.phase} | {phase.records_sent:,} |" + f" {phase.request_bytes / 2**20:.1f} | {_fmt(phase.request_write_seconds)} |" + f" {_fmt(phase.response_wait_seconds)} | {_fmt(phase.response_read_seconds)} |" + f" {_fmt(metrics.get('input_deserialization_seconds'), 3)} |" + f" {_fmt(metrics.get('fold_seconds'), 3)} |" + f" {_fmt(metrics.get('output_serialization_seconds'), 3)} |" + f" {_fmt(phase.residual_wait_seconds, 3)} |" + f" {_fmt(phase.rss_after_mb, 0)} | {phase.status}" + f"{' (' + phase.error + ')' if phase.error else ''} |" + f" {_flag(phase.self_consistent)} | {_flag(phase.oracle_match)} |" + ) + return "\n".join(lines) + + +def render_summary(runs: list[RunResult]) -> str: + lines = [ + "| binary | shares | peak RSS MiB | peak source | peak VSZ MiB | MiB per 1k shares | full wait s | full fold s | daemon exit | available MiB before |", + "|---|---:|---:|---|---:|---:|---:|---:|---|---:|", + ] + for run in runs: + if run.skipped: + lines.append(f"| {run.variant} | {run.size:,} | skipped: {run.skipped} | | | | | | | {_fmt(run.mem_available_before_mb, 0)} |") + continue + full = next((phase for phase in run.phases if phase.phase == "full"), None) + peak = run.outcome.get("peak_rss_mb") + peak_source = run.outcome.get("peak_rss_source") + exit_code = run.outcome.get("exit_code") + signal = run.outcome.get("signal") + ended = "clean (0)" if exit_code == 0 else (f"signal {signal}" if signal else f"exit {exit_code}") + if exit_code is None: + ended = "unavailable" + if run.outcome.get("timed_out"): + ended += f", killed on {run.outcome['timed_out']} timeout" + if run.error and not run.phases: + ended += f", failed: {run.error}" + lines.append( + f"| {run.variant} | {run.size:,} | {format_peak(peak, peak_source)} | {peak_source_label(peak_source)} |" + f" {format_peak(run.outcome.get('peak_vsize_mb'), peak_source)} |" + f" {format_peak(None if peak is None else peak / (run.size / 1000.0), peak_source, 2)} |" + f" {_fmt(full.response_wait_seconds if full else None)} |" + f" {_fmt(full.daemon_metrics.get('fold_seconds') if full else None, 3)} |" + f" {ended} | {_fmt(run.mem_available_before_mb, 0)} |" + ) + return "\n".join(lines) + + +def parse_sizes(text: str) -> list[int]: + return [int(float(item)) for item in text.split(",") if item.strip()] + + +def resolve_binary(text: str | None, *, default: Path | None) -> Path | None: + if text: + candidate = Path(text) + elif default is not None: + candidate = default + else: + return None + if candidate.is_dir(): + candidate = candidate / DAEMON_BINARY + if not (candidate.is_file() and os.access(candidate, os.X_OK)): + raise SystemExit(f"{candidate} is not an executable daemon binary") + return candidate.resolve() + + +def main(argv: list[str] | None = None) -> int: + parser = argparse.ArgumentParser( + description=__doc__, + formatter_class=argparse.RawDescriptionHelpFormatter, + ) + parser.add_argument( + "--daemon-binary", + default=os.environ.get("PRISM_TOOL_BIN_DIR") or str(DEFAULT_TOOL_BIN_DIR), + help="the fixed qbit-prism-build-audit-bundle (a file, or a directory holding it; default $PRISM_TOOL_BIN_DIR or target/release)", + ) + parser.add_argument("--daemon-label", default="fixed", help="row label for --daemon-binary (default fixed)") + parser.add_argument("--baseline-binary", default=None, help="an unfixed daemon binary to measure at --baseline-sizes only") + parser.add_argument("--baseline-label", default="baseline", help="row label for --baseline-binary") + parser.add_argument("--sizes", default=DEFAULT_SIZES, help=f"window sizes for the fixed binary (default {DEFAULT_SIZES})") + parser.add_argument("--baseline-sizes", default=DEFAULT_BASELINE_SIZES, help=f"window sizes for the baseline binary (default {DEFAULT_BASELINE_SIZES}; keep these inside the memory limit)") + parser.add_argument("--page-size", type=int, default=DEFAULT_INCREMENTAL_SHARE_WINDOW_PAGE_SIZE) + parser.add_argument("--miners", type=int, default=DEFAULT_MINERS, help=f"distinct identities in the fixture (default {DEFAULT_MINERS})") + parser.add_argument("--small-delta", type=int, default=DEFAULT_SMALL_DELTA, help=f"records in the small advance (default {DEFAULT_SMALL_DELTA})") + parser.add_argument("--large-delta", type=int, default=DEFAULT_LARGE_DELTA, help=f"records in the multi-page advance (default {DEFAULT_LARGE_DELTA})") + parser.add_argument("--daemon-memory-limit-mb", type=int, default=6144, help="RLIMIT_AS for every daemon in MiB; 0 disables (default 6144)") + parser.add_argument("--memory-margin-mb", type=int, default=2048, help="MemAvailable headroom required above the limit before a run (default 2048)") + parser.add_argument("--sample-interval", type=float, default=0.02, help="RSS sampling interval in seconds (default 0.02)") + parser.add_argument("--exchange-timeout", type=float, default=DEFAULT_EXCHANGE_TIMEOUT, help=f"wall-clock deadline per exchange, handshake included; the daemon is killed and the phase recorded as a timeout (default {DEFAULT_EXCHANGE_TIMEOUT:g})") + parser.add_argument("--skip-oracle", action="store_true", help="do not run the Python fold as a digest oracle") + parser.add_argument("--stderr-dir", default=None, help="directory for daemon stderr captures (default: a temporary directory)") + parser.add_argument("--json", default=None, help="write structured results to this file") + parser.add_argument("--quiet", action="store_true") + args = parser.parse_args(argv) + + def log(text: str) -> None: + if not args.quiet: + print(text, flush=True) + + fixed = resolve_binary(args.daemon_binary, default=DEFAULT_TOOL_BIN_DIR) + baseline = resolve_binary(args.baseline_binary, default=None) + assert fixed is not None + memory_limit = args.daemon_memory_limit_mb or None + stderr_dir = Path(args.stderr_dir) if args.stderr_dir else Path( + os.environ.get("TMPDIR", "/tmp") + ) / f"window_paging_allocation-{os.getpid()}" + stderr_dir.mkdir(parents=True, exist_ok=True) + + info = machine_info() + source = git_describe(REPO_ROOT) + binaries = {args.daemon_label: binary_identity(fixed)} + if baseline is not None: + binaries[args.baseline_label] = binary_identity(baseline) + + print("# window_paging_allocation") + print() + for key, value in info.items(): + print(f"- {key}: {value}") + print(f"- source: {source}") + for label, identity in binaries.items(): + print(f"- {label}: {identity['path']} (sha256 {identity['sha256'][:16]}..., {identity['mtime']})") + print(f"- page_size: {args.page_size}; miners: {args.miners}; deltas: {args.small_delta} / {args.large_delta}") + print(f"- daemon RLIMIT_AS: {memory_limit or 'none'} MiB; required MemAvailable: {(memory_limit or 0) + args.memory_margin_mb} MiB; exchange timeout: {args.exchange_timeout:g} s") + print() + + plan: list[tuple[str, Path, int]] = [] + sizes = parse_sizes(args.sizes) + baseline_sizes = parse_sizes(args.baseline_sizes) if baseline is not None else [] + for size in sorted(set(sizes) | set(baseline_sizes)): + if baseline is not None and size in baseline_sizes: + plan.append((args.baseline_label, baseline, size)) + if size in sizes: + plan.append((args.daemon_label, fixed, size)) + + runs: list[RunResult] = [] + oracles: dict[int, dict[str, Any]] = {} + fixtures_seconds: dict[int, float] = {} + current_size: int | None = None + fixture: Fixture | None = None + oracle: dict[str, Any] | None = None + + def skip_parent_work_if_low_memory(label: str, binary: Path, size: int, stage: str) -> bool: + available = read_meminfo_mb("MemAvailable") + required = (memory_limit or 0) + args.memory_margin_mb + if available is None or available >= required: + return False + reason = f"MemAvailable {available:.0f} MiB below required {required} MiB before {stage}" + log(f" {label} @ {size}: skipped, {reason}") + runs.append(RunResult( + label, str(binary), size, args.page_size, available, None, + memory_limit, [], {}, 0.0, [], skipped=reason, + )) + return True + + for label, binary, size in plan: + if size != current_size: + # Release the preceding fixture before checking headroom for + # the next parent-side allocation. The daemon admission check + # alone is too late to protect fixture/oracle construction. + fixture = None + oracle = None + if skip_parent_work_if_low_memory(label, binary, size, "fixture construction"): + continue + log(f" building {size:,}-record fixture ...") + fixture = build_fixture(size, miners=args.miners, page_size=args.page_size, small=args.small_delta, large=args.large_delta) + fixtures_seconds[size] = fixture.build_seconds + if not args.skip_oracle: + if skip_parent_work_if_low_memory(label, binary, size, "Python oracle"): + fixture = None + continue + log(f" running the Python fold oracle at {size:,} ...") + oracle = python_oracle(fixture) + oracles[size] = oracle + current_size = size + assert fixture is not None + runs.append( + run_variant( + label, + binary, + fixture, + oracle=oracle, + memory_limit_mb=memory_limit, + memory_margin_mb=args.memory_margin_mb, + sample_interval=args.sample_interval, + exchange_timeout=args.exchange_timeout, + stderr_dir=stderr_dir, + log=log, + ) + ) + last = runs[-1] + if not last.skipped and last.phases: + log( + f" -> peak RSS {format_peak(last.outcome.get('peak_rss_mb'), last.outcome.get('peak_rss_source'))} MiB" + f" ({peak_source_label(last.outcome.get('peak_rss_source'))}), exit {last.outcome.get('exit_code')}," + f" phases {[(p.phase, p.status) for p in last.phases]}" + ) + + print("## Summary per daemon") + print() + print(render_summary(runs)) + print() + print("## Phases") + print() + print(render_runs(runs)) + print() + if oracles: + print("## Python oracle (in-process fold on the same inputs)") + print() + print("| shares | phase | seconds | records | digest |") + print("|---:|---|---:|---:|---|") + for size, oracle in sorted(oracles.items()): + for phase in PHASES: + print(f"| {size:,} | {phase} | {oracle['seconds'][phase]:.2f} | {oracle['record_counts'][phase]:,} | {oracle['digests'][phase][:16]}... |") + print() + + if args.json: + output = { + "machine": info, + "source": source, + "binaries": binaries, + "arguments": vars(args), + "fixture_build_seconds": fixtures_seconds, + "oracles": oracles, + "runs": [ + { + **{key: value for key, value in asdict(run).items() if key != "rss_timeline"}, + "phases": [ + {**asdict(phase), "residual_wait_seconds": phase.residual_wait_seconds} + for phase in run.phases + ], + "rss_timeline_mb": [ + [round(t - run.rss_timeline[0][0], 3), round(rss, 1)] for t, rss in run.rss_timeline + ] if run.rss_timeline else [], + } + for run in runs + ], + } + Path(args.json).write_text(json.dumps(output, indent=2, default=str) + "\n") + print(f"wrote {args.json}") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/tests/perf/window_pipeline_gil_scaling.md b/tests/perf/window_pipeline_gil_scaling.md new file mode 100644 index 000000000..e8fa46a63 --- /dev/null +++ b/tests/perf/window_pipeline_gil_scaling.md @@ -0,0 +1,704 @@ +# GIL scaling of the PRISM payout-window materialization pipeline + +Closes the Known gap #162 recorded on #143: *"GIL behaviour at megabyte buffer sizes is +unmeasured."* Issue #131 attributes ~135 ms per window materialization to **GIL-held** +Python at today's 21,868-share window (~555 ms at 100k, ~2.2 s at 400k). That attribution +was asserted for this pipeline, never measured on it — and it matters, because #143 §1 +bracketed CPython's GIL-release threshold at **1–2 KiB** while every buffer here is +megabytes. + +Driver: `tests/perf/window_pipeline_gil_scaling.py`. No database, no network, no +coordinator, standard library only. `python3 tests/perf/window_pipeline_gil_scaling.py` +runs the sweep and prints these tables; `--json` captures the same data structured, and +`--render FILE` re-prints the tables from a captured JSON. + +This is an **on-demand instrument, not a test.** It asserts no thresholds and is +deliberately not named `test_*` so the discovery run never executes it (#160: a threshold +assertion on a shared runner is a flaky test in waiting). Nothing under `lab/` was +modified; it is imported read-only. + +**Measured at `4c8ef4c`**, not the `2aa63cd` this task's contract names — the `2.x.x` tip +advanced by one commit during launch. That commit ("Make schema applies atomic…") touches +only `PsqlShareLedger`'s psql subprocess backend; `lab/prism/bundle_compiler.py` is +byte-identical between the two revisions and the `share_ledger.py` delta does not reach any +callable measured here. Every line number below was re-derived at `4c8ef4c`. + +--- + +## Headline + +**The premise weakens substantially. It does not fail, and it does not hold as stated.** + +- The **canonical-JSON digest is already fully parallel** — 7.57 cores across 8 threads, + statistically indistinguishable from the 1 MiB `hashlib.sha256` positive control (7.23) + measured beside it. That row is *not* GIL time. Lead 1 is confirmed. +- The **fold is genuinely GIL-held** — 1.02 cores flat from 1→8 threads, at every window + size. Lead 2 is confirmed, and the paging half (`json.dumps` per record) is 84% of it. +- The **spool term is mixed, but the mix does not rescue it**: 90–96% of it is + `json.dumps` at 1.02 cores, and only the `os.write` half (1.0 ms of 37.3 ms) runs + parallel at 7.01 cores. Lead 3 is confirmed in structure and refuted in significance — + splitting the term buys ~1%. + +Applying the measured per-stage release fractions to #131's own published milliseconds, +**23–25% of the profiled time is already running in parallel** (32 ms of 135 ms; 141 of +555; 537 of 2,158), essentially all of it the digest. The remaining ~75% is real GIL time. + +That percentage is a weighted average carrying #131's own stage proportions, which this +host does not reproduce — weighted by *this* machine's costs the parallel share is 8.5%. +The per-stage verdicts transfer; the aggregate percentage should not be quoted without +§4's caveat. + +Two attribution defects in #131's table, both material and both stated with the evidence in +§5: the four rows **double-count** ~5 ms because record→JSON runs *inside* the fold, and the +spool row's **byte annotations are canonical-JSON sizes, not spool sizes** — the real spool +payload is 5.5× smaller at every window size (1.49 MB, not 8 MB, today). + +--- + +## 1. Machine, metric, and controls + +### Machine + +| | | +|---|---| +| CPU | Intel Core (Haswell, no TSX), 8 vCPU — KVM guest, 1 core/socket × 8 sockets, no SMT | +| SHA acceleration | **none** (`sha_ni` absent), so `hashlib.sha256` is software | +| Platform | Linux 6.8.0-106-generic, x86_64, glibc 2.39 | +| Python | 3.12.3 (main, Jun 19 2026) [GCC 13.3.0], CPython | +| GIL | standard build (`Py_GIL_DISABLED` = False) | +| Memory | 22 GB total, ~16 GB available; peak RSS during the sweep **7.6 GB** (400k × 8 threads) | +| Load | 0.27 at start, 2.10 at end; per-table ranges reported beside each table | +| Page size | 512 records (`DEFAULT_INCREMENTAL_SHARE_WINDOW_PAGE_SIZE`) | + +### Metric + +**cores-used = `time.process_time_ns()` / `time.perf_counter_ns()`**, with the stage +running in N threads over N **independent inputs**. `process_time` sums CPU across every +thread in the process, which is exactly what separates "N threads made progress together" +from "N threads took turns under the GIL". 1.00 flat from 1→8 means fully GIL-held; +scaling toward N means the stage releases it. + +`thread_time` — the headline clock in the sibling instrument +`tests/perf/serialize_block_share_cost.py` — is deliberately *not* used here: it is +per-thread and structurally cannot see the concurrency that is the whole question. + +### Statistic and alternation + +Median of **3 repetitions**, taken **alternating** (outer loop repetition, inner loop +configuration) rather than in blocks, so host drift lands on every configuration equally +instead of on whichever ran last. Run-to-run spread was very tight — every stage's +min-to-max across the 3 reps sits within 0.01 cores except `spool_write` at 400k +(6.71–7.53) and the digest at 100k (7.25–7.57). + +### The wall-clock floor — a real trap, measured not assumed + +Thread create/join is charged to wall clock but contributes almost no process CPU, so a +measurement whose wall time is close to the thread-start cost reads **low** on cores-used +regardless of GIL behaviour. On this rig, the *same* 1 MiB sha256 positive control reads: + +| wall | cores-used, 8 threads | +|---|---| +| 0.25 s | 6.43 | +| 0.26 s | 5.33 | +| 0.90 s | 7.42 | +| 3.71 s | 7.42 | + +This bias is **one-sided**: it pushes every stage toward 1.00, i.e. toward falsely +confirming "GIL-held". The driver therefore grows every configuration until its wall clock +clears `--min-wall-seconds` (default 1.0 s). My first-cut probe of this pipeline, before the +floor was enforced, reported the positive control at 6.43 — had I reported stage numbers +from that rig, the digest's parallelism would have been understated. + +### Controls + +Both controls run **in the same process, at the same thread counts, in the same +alternation** as the stages. + +| control | N=1 | N=2 | N=4 | N=8 | +|---|---|---|---|---| +| **positive** — 1 MiB `hashlib.sha256` | 1.00 | 1.98 | 3.94 | **7.23** | +| **negative** — 20,000-iteration pure-Python loop | 1.00 | 1.00 | 1.01 | **1.02** | + +Load average (1 m) during the controls: 0.71–1.00. + +The positive control reaches **7.23 of #143's published 7.89 — a ratio of 0.916** — on an +8-vCPU KVM guest whose 8th core also carries the main thread and the OS. The negative +control pins at 1.02. **The rig reproduces #143 §1 and the stage numbers below are +reportable.** + +The driver enforces this as a gate, not a footnote: `--control-floor-fraction` (default +0.75 of the nominal thread count — 6.0 cores on an 8-thread sweep) aborts the run and +withholds every stage number if the positive control does not reach it. That +gate fired for real during development — an early version built each thread's control +buffer with `bytes(n)` instead of `bytes([n])`, giving every thread a differently sized +buffer; the control collapsed to 3.71 and the run refused to report stages. + +**Cores-used is normalized against the positive control measured on this host at the same +thread count**, not against the nominal thread count: an 8-vCPU guest cannot reach 8.00 even +on perfectly parallel work, so dividing by 8 would understate every stage. The reported +`released` fraction is `(cores − 1) / (control − 1)`, clamped to [0, 1]. + +--- + +## 2. Cores-used: stage × thread count × window size + +Indented rows are sub-terms of the row above, not additional profile rows. `CPU ms` is +per-call interpreter CPU at N=1. `released` is at the highest thread count. + +### 21,868 shares (today's window) + +canonical JSON 8.34 MB (381 B/record, 43 pages, **189 KiB/page**) · spool payload 1.49 MB +(68 B/record) · input 35.8 MB/thread + +| stage | N=1 | N=2 | N=4 | N=8 | CPU ms | released | +|---|---|---|---|---|---|---| +| `fold` | 1.00 | 1.00 | 1.01 | **1.02** | 220.8 | 0% | +|   `fold_pages` | 1.00 | 1.00 | 1.01 | 1.02 | 186.1 | 0% | +| `digest` | 1.00 | 1.97 | 3.91 | **7.57** | 22.8 | **100%** | +| `to_prism_json` | 1.00 | 1.00 | 1.01 | **1.02** | 15.3 | 0% | +| `spool_acquire` | 1.00 | 1.03 | 1.05 | **1.06** | 37.3 | 1% | +|   `spool_compact` | 1.00 | 1.00 | 1.01 | 1.02 | 33.6 | 0% | +|   `spool_encode` | 1.00 | 1.00 | 1.01 | 1.02 | 0.1 | 0% | +|   `spool_write` | 1.00 | 1.98 | 3.85 | **7.01** | 1.0 | 96% | +| *positive control* | 1.00 | 1.98 | 3.94 | 7.23 | | | +| *negative control* | 1.00 | 1.00 | 1.01 | 1.02 | | | + +Load average (1 m) during this table: 1.32–1.62. + +### 100,000 shares + +canonical JSON 38.27 MB (383 B/record, 196 pages, 191 KiB/page) · spool payload 6.88 MB +(69 B/record) · input 87.3 MB/thread + +| stage | N=1 | N=2 | N=4 | N=8 | CPU ms | released | +|---|---|---|---|---|---|---| +| `fold` | 1.00 | 1.01 | 1.01 | **1.02** | 1,067.6 | 0% | +|   `fold_pages` | 1.00 | 1.01 | 1.01 | 1.02 | 896.0 | 0% | +| `digest` | 1.00 | 1.99 | 3.95 | **7.41** | 101.2 | **100%** | +| `to_prism_json` | 1.00 | 1.00 | 1.01 | **1.02** | 89.5 | 0% | +| `spool_acquire` | 1.00 | 1.02 | 1.04 | **1.05** | 176.1 | 1% | +|   `spool_compact` | 1.00 | 1.00 | 1.01 | 1.02 | 157.0 | 0% | +|   `spool_encode` | 1.00 | 1.01 | 1.01 | 1.02 | 0.9 | 0% | +|   `spool_write` | 1.00 | 1.98 | 3.94 | **7.14** | 5.4 | 99% | +| *positive control* | 1.00 | 1.98 | 3.94 | 7.23 | | | +| *negative control* | 1.00 | 1.00 | 1.01 | 1.02 | | | + +Load average (1 m) during this table: 1.40–1.82. + +### 400,000 shares — **N=1 and N=8 only** + +Swept at the endpoints only, as the task permits: eight independent 400k inputs cost +373 MB each and the sweep already peaked at 7.6 GB RSS. N=2 and N=4 were not measured at +this size. Given 1.02 at N=2 and N=4 at both smaller sizes and 1.02 at N=8 here, no +interior behaviour is in question — but it is not measured, and I am not claiming it. + +canonical JSON 154.07 MB (385 B/record, 782 pages, 192 KiB/page) · spool payload 28.18 MB +(70 B/record) · input 373.4 MB/thread + +| stage | N=1 | N=8 | CPU ms | released | +|---|---|---|---|---| +| `fold` | 1.00 | **1.02** | 4,260.4 | 0% | +|   `fold_pages` | 1.00 | 1.02 | 3,727.0 | 0% | +| `digest` | 1.00 | **7.64** | 408.3 | **100%** | +| `to_prism_json` | 1.00 | **1.02** | 414.8 | 0% | +| `spool_acquire` | 1.00 | **1.04** | 710.1 | 1% | +|   `spool_compact` | 1.00 | 1.01 | 679.7 | 0% | +|   `spool_encode` | 1.00 | 1.02 | 5.5 | 0% | +|   `spool_write` | 1.00 | **6.98** | 23.8 | 96% | +| *positive control* | 1.00 | 7.23 | | | +| *negative control* | 1.00 | 1.02 | | | + +Load average (1 m) during this table: 1.11–1.66. + +**The verdicts are flat in window size.** Nothing crosses a threshold between 8 MB and +154 MB of canonical JSON: the digest is parallel at all three sizes, the fold is GIL-held at +all three. + +--- + +## 3. Stage-internal decomposition + +### The fold splits into a cheap prefix and an expensive paging half + +`IncrementalShareWindow.from_full_snapshot` (`lab/prism/share_ledger.py:364-430`) is a +`sorted()` plus a per-record validation loop, then `_IncrementalShareWindowPage.from_records` +(`:274-293`) per page. + +| sub-term | 21,868 | 100k | 400k | cores @ N=8 | +|---|---|---|---|---| +| `fold` total | 220.8 ms | 1,067.6 ms | 4,260.4 ms | 1.02 | +| `fold_pages` (paging) | 186.1 ms | 896.0 ms | 3,727.0 ms | 1.02 | +| prefix (`sorted` + validation), by difference | 34.7 ms | 171.6 ms | 533.3 ms | — | + +The paging half is **84–87%** of the fold and is where `to_prism_json`, the per-record +`json.dumps`, and `b",".join` live. Both halves are GIL-held; there is no parallel +sub-term to recover here. + +### The spool splits into `json.dumps` and `os.write` — and `os.write` is 3% + +`_ShareWindowSerialization.acquire_spooled_tail` (`lab/prism/bundle_compiler.py:250-293`) +calls `compact_fragments` (`:222-248`), encodes both fragments to UTF-8, then writes them to +a `tempfile.TemporaryFile`. Separated by driving each shipped sub-step against pre-built +inputs held constant, so each measurement isolates one primitive: + +| sub-term | what it is | 21,868 | share of term | cores @ N=8 | verdict | +|---|---|---|---|---|---| +| `spool_compact` | `_compact_share_payload` + 2 × `json.dumps` | 33.6 ms | 90% | 1.02 | **GIL-held** | +| `spool_encode` | `str.encode("utf-8")` ×2 | 0.1 ms | 0.3% | 1.02 | GIL-held | +| `spool_write` | `TemporaryFile` write/flush/seek | 1.0 ms | 2.7% | **7.01** | **GIL-released** | +| residual | instance construction, lock, teardown | 2.6 ms | 7% | — | — | +| **`spool_acquire`** | **all of the above** | **37.3 ms** | 100% | **1.06** | **GIL-held** | + +At 400k the same split holds: `spool_compact` 679.7 ms (96% of the 710.1 ms term) at 1.01 +cores, `spool_write` 23.8 ms (3.3%) at 6.98 cores. + +**Lead 3 is structurally right and practically wrong.** The term *is* mixed, and `os.write` +*does* release the GIL exactly as predicted — but it is 3% of the term. Splitting it changes +the spool row's parallel fraction from 0% to about 1%. + +--- + +## 4. Splitting #131's profile into GIL-held and GIL-released ms + +**Method for the split.** For each profile row, `released_fraction = (cores_used − 1) / +(positive_control_cores − 1)` measured at 8 threads on this host and clamped to [0, 1], then +`parallel_ms = published_ms × released_fraction` and `held_ms = published_ms − parallel_ms`. +The fraction — not the millisecond — is the portable quantity; see the caveat below, which +is important. + +| profile row | #131 ms | released | **GIL-held ms** | **parallel ms** | +|---|---|---|---|---| +| **21,868 shares** | | | | | +| Window fold | 71 | 0% | **71** | 0 | +| Canonical-JSON digest | 31 | 100% | **0** | **31** | +| Spool serialization | 28 | 1% | **28** | 0 | +| Record→JSON conversion | 5 | 0% | **5** | 0 | +| **total** | **135** | | **103** | **32 (23%)** | +| **100,000 shares** | | | | | +| Window fold | 271 | 0% | **270** | 1 | +| Canonical-JSON digest | 139 | 100% | **0** | **139** | +| Spool serialization | 122 | 1% | **121** | 1 | +| Record→JSON conversion | 23 | 0% | **23** | 0 | +| **total** | **555** | | **414** | **141 (25%)** | +| **400,000 shares** | | | | | +| Window fold | 1,057 | 0% | **1,054** | 3 | +| Canonical-JSON digest | 530 | 100% | **0** | **530** | +| Spool serialization | 476 | 1% | **473** | 3 | +| Record→JSON conversion | 95 | 0% | **95** | 0 | +| **total** | **2,158** | | **1,621** | **537 (25%)** | + +### The caveat that governs how much of that 23–25% transfers + +**The percentage depends on #131's host balance, which this host does not reproduce.** The +per-stage *verdicts* are host-independent and robust. The *aggregate percentage* is not, +because it is a weighted average and the weights are #131's. + +On #131's workstation the fold:digest ratio is 71:31 = **2.3**. On this VM it is +220.8:22.8 = **9.7** — this host is disproportionately slow at pure-Python bytecode relative +to software SHA-256. Running the identical split on *this host's own* milliseconds gives: + +| | this host, 21,868 shares | +|---|---| +| GIL-held | 257.0 ms | +| parallel | 23.9 ms | +| **parallel share** | **8.5%** | + +So: "the digest is fully parallel" is a measured fact that transfers. "23% of the 135 ms is +already parallel" is that fact re-weighted by #131's published proportions, and lands +anywhere between ~8% (this host's balance) and ~23% (#131's balance) depending on whose +CPU you weight it with. Both are reported; neither should be quoted without the other. + +--- + +## 5. Attribution — what I charged where, and two defects in the profile + +### Which callable I charged to each row + +| #131 row | callable driven | location at `4c8ef4c` | +|---|---|---| +| Window fold | `IncrementalShareWindow.from_full_snapshot` | `lab/prism/share_ledger.py:364-430` | +| Canonical-JSON digest | `IncrementalShareJsonSequence.canonical_json_sha256` | `lab/prism/share_ledger.py:333-345` | +| Spool serialization | `_ShareWindowSerialization.acquire_spooled_tail` (cold, fresh instance per call) | `lab/prism/bundle_compiler.py:250-293` | +| Record→JSON conversion | `AcceptedShareRecord.to_prism_json` over every record | `lab/prism/share_ledger.py:212-229` | + +Each is the shipped callable, driven directly — nothing is reimplemented or stubbed. The +spool stage constructs a **fresh** `_ShareWindowSerialization` per call because +`acquire_spooled_tail` memoizes; a reused instance would measure a dict lookup after the +first call. Its descriptor is closed through the shipped `retire_spool` / +`release_spooled_tail` teardown rather than left to the GC. + +### Defect 1 — the four rows double-count record→JSON conversion + +`_IncrementalShareWindowPage.from_records` opens with +`prism_json_records = tuple(record.to_prism_json() for record in records)` +(`lab/prism/share_ledger.py:279`). The per-record JSON conversion therefore happens +**inside** the fold. Measured on this host at 21,868 shares, `to_prism_json` over all +records is 15.3 ms and sits within `fold_pages`' 186.1 ms — it is 8% of the paging half, and +11% at 400k. + +Summing all four of #131's rows to a "Total GIL-held" therefore counts that work twice. On +#131's own numbers the overstatement is the full 5 ms row at 21,868 (135 → ~130), 23 ms at +100k, and 95 ms at 400k. It does not change any verdict — the term is GIL-held either way — +but the totals are ~4% high, and the "Record→JSON conversion" row is not an independent term +that could be removed or migrated separately from the fold. + +### Defect 2 — the spool row's byte annotations are canonical-JSON sizes, not spool sizes + +#131 annotates the spool row "28 ms (8 MB)", "122 ms (37 MB)", "476 ms (151 MB)". Measured: + +| window | #131's spool-row bytes | **actual spool payload** | **canonical JSON** | canonical ÷ #131 | +|---|---|---|---|---| +| 21,868 | 8 MB | **1.49 MB** | 8.34 MB | **1.043** | +| 100,000 | 37 MB | **6.88 MB** | 38.27 MB | **1.034** | +| 400,000 | 151 MB | **28.18 MB** | 154.07 MB | **1.020** | + +#131's figures match the **canonical-JSON** size to within 2–4% at all three points, and +overstate the **actual spool payload** by 5.5× at all three. The spool writes the +*compact* form — `_compact_share_payload` (`lab/prism/bundle_compiler.py:123-153`) +deduplicates the (`miner_id`, `order_key`, `p2mr_program_hex`) identity triple into an index +and emits positional tuples — which is 68–70 B/record against canonical JSON's +381–385 B/record. + +This does not change the GIL verdict (1.49 MB is still ~750× the 1–2 KiB release threshold), +but it matters for anyone sizing this work off #131's table: the premise's own framing — +"the spool is 8 MB today and 151 MB at 400k shares" — is off by 5.5×, and the multi-megabyte +buffer whose GIL behaviour was in question is the *digest's* input, not the spool's. + +### Sensitivity: a realistic `share_id` moves the bytes, not the verdicts + +Re-run at 21,868 shares with `--share-id-shape production`, which swaps the benchmark's +`bench-share-N` for the `username:block_hash_hex` form `lab/prism/share_writer.py:284` +actually builds (2 repetitions, N=1 and N=8, same controls: positive 7.23, negative 1.02): + +| | benchmark shape | production shape | +|---|---|---| +| canonical JSON | 8.34 MB (381 B/record) | **9.84 MB** (450 B/record) | +| spool payload | 1.49 MB (68 B/record) | **2.98 MB** (136 B/record) | +| `fold` @ N=8 | 1.02 | **1.02** | +| `digest` @ N=8 | 7.57 | **7.43** | +| `spool_acquire` @ N=8 | 1.06 | **1.08** | +| `spool_write` @ N=8 | 7.01 | **7.31** | + +The spool payload doubles — the `share_id` is per-share and is not deduplicated by the +identity index — and every GIL verdict is unchanged. Note this also *widens* the gap in +Defect 2 rather than closing it: with a realistic `share_id` the spool payload is 2.98 MB +against #131's 8 MB, and the canonical JSON overshoots to 9.84 MB. The benchmark shape is +the closer match to #131's byte annotations, which is consistent with #131 having profiled a +benchmark-shaped window. + +### Not a defect, but worth stating + +I could not reproduce #131's *absolute* milliseconds and did not try to: this VM is ~3.1× +slower than that workstation on the fold and ~0.74× as slow on the digest. The instrument +measures concurrency behaviour, which is what was unmeasured; #131's per-stage costs on its +own host are not in dispute here. + +--- + +## 6. My reading + +**The premise weakens substantially. It does not fail.** + +Stated plainly, because a negative result is the useful outcome here: **the +canonical-JSON digest is not GIL-held, and #131 counts it as GIL-held.** At 8 threads it +runs at 7.57 cores against a 7.23-core positive control measured beside it — it is, to the +resolution this host offers, perfectly parallel, and it is the second-largest row in the +profile at every window size. Lead 1 was right. `hashlib` releases the GIL around +189 KiB-per-`update()` page buffers exactly as #143 §1's 1–2 KiB threshold predicts, and +the paged design of `canonical_json_sha256` — feeding pre-encoded per-page buffers into one +`sha256` — is what makes that possible. + +But the majority of the profile survives the measurement intact. The fold is GIL-held at +1.02 cores at every size, and it is the largest row. The spool is GIL-held at 1.06, and the +`os.write` escape hatch that lead 3 predicted is real but worth 3% of the term. Between +them, fold + spool are 99 ms of the 135 ms, and they are genuinely serialized. + +So the honest summary is: **~75% of #131's ~135 ms is real GIL time; ~25% of it (the digest) +was never GIL time and should not have been counted.** On this host's own cost balance the +parallel share is smaller — 8.5% — because the fold dominates here far more than it does on +#131's workstation; the fraction is weight-dependent, the per-stage verdicts are not. + +One thing I want to flag as a judgment rather than a measurement: the profile's *shape* is +now different from what it looks like in #131. Three of the four rows are GIL-held and one +is not, and the one that is not is the one whose "megabyte buffer" framing motivated the +doubt in the first place. That is worth knowing before anyone reasons further from that +table. I am not drawing a conclusion about #131 itself, or about any migration — out of +scope for this task, and the next question (what the GIL-held remainder would cost to +change) is not one these numbers answer. + +### What is noisy, and what is not + +- **Not noisy.** Every verdict here is separated from its alternative by a factor of ~7. + Run-to-run spread across 3 alternating repetitions is ≤0.01 cores for every GIL-held stage + and ≤0.1 for the parallel ones. Nothing is close to a threshold. +- **The 100% figures are clamped.** The digest measured 7.57 / 7.41 / 7.64 against a + 7.23-core control, i.e. slightly *above* the ceiling; the released fraction saturates at + 1.0. Read it as "indistinguishable from fully parallel", not as a claim of exactly 100%. + The digest and the control are both `hashlib.sha256`, so this is unsurprising — the digest + does less per-byte Python work than the control's loop overhead. +- **Load was not zero.** 1.1–1.8 during the stage tables, most of which is the driver + itself. The controls were re-measured in the same process under the same conditions and + bracket the stages, so drift is visible rather than assumed. + +--- + +## 7. Limits — stated as gates, not guesses + +- **This is a developer KVM VM, not production.** 8 vCPUs, no SMT, **no SHA-NI**. A + production host with SHA extensions would make the digest cheaper in absolute ms without + changing its GIL behaviour; the parallel *fraction* of the profile would fall, not rise. +- **Nobody on this task has production access.** Every number here is synthetic-input, + single-host. Nothing about real production window sizes, real miner counts, real + concurrency, or real contention with the rest of the coordinator is measured or implied. +- **Records are synthetic**, shaped like `lab/prism/job_build_benchmark.py`'s defaults + (`--shares 21868 --miners 2`), whose values encode a live-host measurement. Two known + distortions: `--miners 2` makes identity deduplication maximally effective, and the + benchmark's `bench-share-N` `share_id` is far shorter than the real + `username:block_hash_hex` form built at `lab/prism/share_writer.py:284` (the ledger's + `length(share_id) >= 65` index predicate corroborates the length). `--share-id-shape + production` re-runs the sweep with a realistic `share_id` — measured, and it moves the byte + accounting without moving any verdict (§5). +- **400k was swept at N=1 and N=8 only** — N=2 and N=4 are unmeasured at that size. +- **`cores-used` measures GIL release, not goodness.** A stage reading 1.02 is serialized; + that is all it says. It says nothing about whether that stage should change, or how. +- The **positive control lands at 0.916× #143's published 7.89** on this smaller host. All + released-fractions are normalized against the control measured here, not against 8.00. + +--- + +## 8. Re-running + +``` +python3 tests/perf/window_pipeline_gil_scaling.py # full sweep +python3 tests/perf/window_pipeline_gil_scaling.py --json > run.json # capture +python3 tests/perf/window_pipeline_gil_scaling.py --render run.json # re-print +python3 tests/perf/window_pipeline_gil_scaling.py --sizes 21868 --threads 1,8 --reps 1 +``` + +**Run it once.** Two concurrent sweeps contend for the same cores and corrupt both — which +is why `--render` exists rather than a second measuring pass for the text format. + +Useful flags: `--min-wall-seconds` (the amortization floor; lowering it biases every stage +toward 1.00), `--control-floor-fraction` (abort threshold for the positive control, as a +fraction of the nominal thread count so the gate stays valid at any `--threads`), `--reps`, +`--miners`, `--share-id-shape`, `--large-size-threads`. The driver skips a window size +outright, with a printed reason, if eight independent inputs would exceed 80% of available +memory. + +The full sweep takes ~35 minutes and peaks at 7.6 GB RSS on this host. + +--- + +## 9. #236 follow-up: bounded serialization and the monitor-lateness probe + +**Context.** Issue #236's incident thread samples pointed at whole-window JSON encoding and +decoding on the coordinator's payout-window paths (~210k shares) while the writer-lease +monitor thread was late by 0.6–1.05 s. PR 2 of the #236 plan bounds every such call: the +daemon `prepare_window` request streams in record batches, the audit-builder compact tail is +encoded batch by batch into bounded chunks (spool, in-memory fallback and one-shot alike), +plain share lists digest through a streamed SHA-256, the daemon mirror's lazy parse walks one +record at a time through a chunked UTF-8 decoder, the found-block candidate identity digest +streams its `shares_json`, the one-shot canonical payload streams its share array, and the +transient per-share lists on the daemon prepare paths are released in bounded slices. All of +it is byte-identical to the historical output (pinned by the parity tests in +`test_prism_incremental_payout_window`, `test_prism_job_builder`, +`test_prism_window_pipeline_rust` and `test_prism_share_ledger`; the Rust `rust-daemon` +parity gate is unchanged and passes). + +**Instrument.** `python3 tests/perf/window_pipeline_gil_scaling.py --latency-probe +[--daemon-binary PATH]` runs each phase once per window size in the main thread while a +monitor thread wakes every `WRITER_LEASE_HEARTBEAT_MONITOR_SECONDS` (50 ms) and records +`wake - due`. Every bounded phase is paired with the historical whole-window call it +replaced, on the same input in the same process, and `gc.callbacks` attributes cyclic-GC +pauses to the phase they landed in. Two controls bracket the host: `idle_control` (main +thread asleep) is the monitor's own jitter and `busy_python_control` (pure-Python loop) is +the interpreter's switch interval. See `LATENCY_PROBE_RATIONALE`. + +**Read this before the numbers.** Every figure below was taken on `alexdevbox1`, a shared +8-vCPU KVM guest on which two other #236 implementation agents were running their own tests +and benchmarks at the same time (load average 1.7–3.5 at the start of the runs). Nothing here +is an isolated or production measurement. Monitor lateness is what a 50 ms-cadence thread +observed, whatever the cause; it is **not** by itself a proof of GIL attribution. The +attribution argument is structural and comparative: a historical row's lateness equals that +row's own wall time to within a few percent, while the bounded row over the same bytes, in the +same process and under the same host load, stays at the controls' single-digit milliseconds. +That pattern is what a GIL-held C call produces and what host contention does not. One run +per phase (`--probe-reps 1`); a few percent of run-to-run drift is visible between the three +captures cited below. + +### Final capture (bounded and historical phases, production share_id, 200 identities) + +`max late`/`p99 late` in ms; `>250ms`/`>500ms` count monitor wakes later than the plan's +strict target and the configured scheduler slack; `gc full` counts generation-2 +collections in the phase and `gc max` is the longest single collection pause. `residual` +rows isolate one deallocation cascade (a `del` of a whole parsed tuple or window), measured +on its own. + +#### 210,000 shares (210,000 retained) -- canonical items 95.5 MB, compact tail 29.5 MB + +| phase | kind | wall ms | wakes | max late ms | p99 late ms | >250ms | >500ms | gc full | gc max ms | +|---|---|---:|---:|---:|---:|---:|---:|---:|---:| +| idle_control | control | 1000 | 19 | 0.2 | 0.2 | 0 | 0 | 0 | 0.0 | +| busy_python_control | control | 1000 | 18 | 5.3 | 5.3 | 0 | 0 | 0 | 0.0 | +| record_conversion | bounded | 243 | 4 | 5.2 | 5.2 | 0 | 0 | 0 | 0.1 | +| fold_in_process | bounded | 1757 | 31 | 9.5 | 9.5 | 0 | 0 | 0 | 23.9 | +| fold_in_process_keep | bounded | 1704 | 29 | 20.7 | 20.7 | 0 | 0 | 0 | 23.1 | +| fold_release | residual | 38 | 0 | n/a | n/a | 0 | 0 | 0 | 0.0 | +| canonical_encode_stream | bounded | 993 | 17 | 7.4 | 7.4 | 0 | 0 | 0 | 0.0 | +| canonical_encode_whole | historical | 1131 | 1 | **1057.6** | 1057.6 | 1 | 1 | 0 | 0.0 | +| array_digest_stream | bounded | 1273 | 24 | 4.5 | 4.5 | 0 | 0 | 0 | 0.0 | +| array_digest_whole | historical | 1355 | 6 | **1030.4** | 1030.4 | 1 | 1 | 0 | 0.0 | +| compact_tail_stream | bounded | 425 | 7 | 6.0 | 6.0 | 0 | 0 | 0 | 0.1 | +| compact_tail_whole | historical | 427 | 3 | **249.6** | 249.6 | 0 | 0 | 0 | 0.2 | +| spool_write_stream | bounded | 427 | 7 | 6.8 | 6.8 | 0 | 0 | 0 | 0.1 | +| prepare_request_stream | bounded | 735 | 13 | 8.2 | 8.2 | 0 | 0 | 0 | 0.0 | +| prepare_request_whole | historical | 882 | 2 | **722.0** | 722.0 | 1 | 1 | 0 | 0.0 | +| mirror_validate | bounded | 1210 | 22 | 6.0 | 6.0 | 0 | 0 | 0 | 0.0 | +| mirror_parse_stream | bounded | 1408 | 25 | 5.7 | 5.7 | 0 | 0 | 0 | 0.4 | +| mirror_parse_release | residual | 66 | 1 | 16.5 | 16.5 | 0 | 0 | 0 | 0.0 | +| mirror_parse_whole | historical | 949 | 3 | **573.2** | 573.2 | 1 | 1 | 0 | 14.2 | +| mirror_parse_whole_release | residual | 57 | 1 | 7.3 | 7.3 | 0 | 0 | 0 | 0.0 | +| candidate_identity_stream | bounded | 1261 | 24 | 2.1 | 2.1 | 0 | 0 | 0 | 0.0 | +| candidate_identity_whole | historical | 1420 | 6 | **1020.3** | 1020.3 | 1 | 1 | 0 | 0.0 | +| oneshot_payload_stream | bounded | 740 | 13 | 7.8 | 7.8 | 0 | 0 | 0 | 0.0 | +| oneshot_payload_whole | historical | 771 | 1 | **714.7** | 714.7 | 1 | 1 | 0 | 0.0 | + +#### 400,000 shares (400,000 retained) -- canonical items 182.1 MB, compact tail 56.2 MB + +| phase | kind | wall ms | wakes | max late ms | p99 late ms | >250ms | >500ms | gc full | gc max ms | +|---|---|---:|---:|---:|---:|---:|---:|---:|---:| +| idle_control | control | 1000 | 19 | 0.2 | 0.2 | 0 | 0 | 0 | 0.0 | +| busy_python_control | control | 1000 | 18 | 7.7 | 7.7 | 0 | 0 | 0 | 0.0 | +| record_conversion | bounded | 294 | 5 | 5.2 | 5.2 | 0 | 0 | 0 | 0.1 | +| fold_in_process | bounded | 3239 | 56 | 46.8 | 40.5 | 0 | 0 | 0 | 49.5 | +| fold_in_process_keep | bounded | 3537 | 62 | 38.1 | 17.7 | 0 | 0 | 0 | 60.5 | +| fold_release | residual | 80 | 1 | 29.9 | 29.9 | 0 | 0 | 0 | 0.0 | +| canonical_encode_stream | bounded | 1951 | 34 | 8.4 | 8.4 | 0 | 0 | 0 | 0.0 | +| canonical_encode_whole | historical | 2384 | 2 | **2040.0** | 2040.0 | 1 | 1 | 0 | 0.0 | +| array_digest_stream | bounded | 2400 | 46 | 4.3 | 4.3 | 0 | 0 | 0 | 0.0 | +| array_digest_whole | historical | 2650 | 11 | **1942.9** | 1942.9 | 1 | 1 | 0 | 0.0 | +| compact_tail_stream | bounded | 709 | 12 | 5.8 | 5.8 | 0 | 0 | 0 | 0.1 | +| compact_tail_whole | historical | 737 | 5 | **438.2** | 438.2 | 1 | 0 | 0 | 0.2 | +| spool_write_stream | bounded | 760 | 13 | 5.8 | 5.8 | 0 | 0 | 0 | 0.1 | +| prepare_request_stream | bounded | 1452 | 25 | 7.6 | 7.6 | 0 | 0 | 0 | 0.0 | +| prepare_request_whole | historical | 1797 | 2 | **1488.9** | 1488.9 | 1 | 1 | 0 | 0.0 | +| mirror_validate | bounded | 2214 | 40 | 7.4 | 7.4 | 0 | 0 | 0 | 0.0 | +| mirror_parse_stream | bounded | 2718 | 49 | 9.4 | 9.4 | 0 | 0 | 0 | 0.4 | +| mirror_parse_release | residual | 113 | 1 | 62.9 | 62.9 | 0 | 0 | 0 | 0.0 | +| mirror_parse_whole | historical | 1813 | 3 | **1180.9** | 1180.9 | 2 | 1 | 0 | 21.8 | +| mirror_parse_whole_release | residual | 105 | 1 | 55.3 | 55.3 | 0 | 0 | 0 | 0.0 | +| candidate_identity_stream | bounded | 2496 | 48 | 4.8 | 4.8 | 0 | 0 | 0 | 0.0 | +| candidate_identity_whole | historical | 2704 | 11 | **1986.5** | 1986.5 | 1 | 1 | 0 | 0.0 | +| oneshot_payload_stream | bounded | 1419 | 25 | 6.9 | 6.9 | 0 | 0 | 0 | 0.0 | +| oneshot_payload_whole | historical | 1510 | 1 | **1449.9** | 1449.9 | 1 | 1 | 0 | 0.0 | + + +### Reading + +- **Every historical whole-window call delays the monitor by about its own duration**: + 0.24–1.06 s at 210k and 0.5–2.2 s at 400k, past the 500 ms scheduler slack in all but one + case. Those are the calls the incident's thread samples caught, and every one of them + is gone from the coordinator's paths. +- **Every bounded replacement stays at the controls' level** (≤ 10 ms) at both sizes, with + the fold as the one exception at 17–46 ms: its `gc max` column shows that is a cyclic-GC + pause (generation 0/1 collections over the hundreds of thousands of new records and + dicts), not a JSON call. +- **The remaining single stretches are deallocation cascades, not parsing.** Before the + key-sharing change, an attribution capture on the same fixture had the streamed mirror + parse itself late by at most 5.5 ms (210k) and 8.2 ms (400k) while dropping the parsed + tuple afterwards took 140 ms wall / 90 ms late and 264 ms / 214 ms respectively: CPython + freeing a few hundred thousand dicts and their private key strings in one refcount + cascade. Two mitigations went into this PR from that measurement. The walker now shares + one string object per key across records (`object_hook`), exactly as a whole-array + `json.loads` does through its scanner memo, which returns the parsed representation to + the whole-array footprint (105 MB vs 169 MB retained at 100k in a `tracemalloc` check) + and brings the release down to the `mirror_parse_release` rows above (66 ms / 17 ms late + at 210k, 113 ms / 63 ms at 400k, on par with `json.loads`' own release). And the + transient `records`/`records_json` lists on the daemon prepare paths are emptied in + 2,048-entry slices (`release_share_list_incrementally`) instead of one cascade. What + remains unbounded is the release of a long-lived parsed daemon sequence or in-process + window when the artifact rotates (`fold_release`, `mirror_parse_release`); both are + measured above and stay under the strict target at both sizes. +- **CPU cost is unchanged or lower.** Batched `json.dumps` over 512 records costs less than + one call over the whole array (the canonical encode, digest, request and candidate rows), + because the batch strings stay cache-resident. The streamed mirror parse costs ~30% more + than one `json.loads` (`raw_decode` per record); that path only runs for found-block + consumers. + +### The daemon round trip (first capture, `probe-236.json`, `--daemon-binary` set) + +| size | phase | wall ms | Python CPU ms | monitor max late | write | read_line | read_exact | +|---|---|---:|---:|---:|---:|---:|---:| +| 210k | daemon_prepare_full | 39,832 | 1,547 | 62.5 ms | 24,570 | 13,833 | 415 | +| 210k | daemon_prepare_advance (16 records) | 571 | 4 | 1.1 ms | 0 | 571 | 0 | +| 400k | daemon_prepare_full | 60,683 | 2,404 | 62.5 ms | 38,416 | 20,346 | — (daemon exited) | + +The round trip is transport- and daemon-bound, not encode-bound: at 210k the streamed +request encode is 0.83 s of Python CPU, but the shipped write helper needs 24.6 s to move +95.5 MB because it sleeps 20 ms whenever the 64 KiB pipe is full (PR 3's lane), and +`read_line` then waits 13.8 s for the daemon. The monitor stayed within 63 ms throughout, +because the writer spends its time asleep. Writing the same request through blocking pipes +outside the coordinator (`repro_400k_daemon.py`, same fixture) took 1.0 s and the daemon +answered 12.2 s later, so ~12 s of the 210k cold preparation is daemon-side work — +consistent with the incident's 47.5–48.8 s `daemon_prepare` being transport plus daemon +time rather than Python encoding. At 400k the daemon exited mid-request; see next. + +### Rust builder memory at the incident size (outside this PR's lane; reported for #236) + +Sending the 210k prepare request to the prebuilt `qbit-prism-build-audit-bundle --serve` +through blocking pipes: **peak daemon RSS 9,613 MB**, response after 12.2 s. At 400k: +**peak RSS 15,623 MB, then SIGKILL** (exit −9, no stderr) 24 s in, on a host with ~15 GB +available. The incident's "roughly 9.7 GiB builder RSS" is therefore reproducible from the +window size alone and is not a leak: `PayoutWindow::from_full_snapshot` +(`crates/qbit-prism/src/window.rs`) pages the retained records with +`remaining.split_off(page_size)` in a loop, and `Vec::split_off` leaves the original +vector's capacity unchanged, so each 512-record page keeps a backing allocation sized for +every record still remaining when it was cut — capacity 210k, then 209.5k, … — and the +tail is copied on every iteration. Summed over 410 pages that is ~9.5 GB of `AcceptedShare` +capacity (~220 B each) and gigabytes of copying. + +A scratch experiment (applied, built into a separate target directory, measured, then +reverted; nothing under `crates/` is part of this PR) replaced the loop with + +```rust +let mut records = retained.into_iter(); +loop { + let page: Vec = records.by_ref().take(page_size).collect(); + if page.is_empty() { + break; + } + pages.push(Rc::new(WindowPage::from_records(page))); +} +``` + +and measured, same fixture and same host: **210k: 422 MB peak RSS, response 1.4 s after +the request** (was 9,613 MB / 12.2 s); **400k: 826 MB, 2.8 s** (was SIGKILL at 15.6 GB), +with digests identical to the shipped daemon's. That is the first thing to land for cold-start +recovery, in a Rust-crate PR gated by `tests.window_pipeline_parity_gate` (`rust-daemon` +adapter); with it, the 210k round trip becomes transport-bound outright. + +### Limits + +- Shared host, single run per phase, synthetic fixture (`build_records`, production + share_id shape, 200 identities, difficulty 16384). Not production, not isolated, no + 24-hour soak, no Docker or production-image run, no concurrent Stratum submissions. +- Monitor lateness is observed scheduling on this host. The GIL reading rests on the + historical-vs-bounded contrast under identical conditions and on the GC attribution, not + on a profiler. +- The daemon phases include the shipped 20 ms polling transport; they are reported to + attribute, not to claim, and PR 3 changes those helpers. +- Nothing here measures first-usable-Stratum-work time or the combined release gates; the + `#236` recovery target remains unmeasured. + +### Re-running + +``` +python3 tests/perf/window_pipeline_gil_scaling.py --latency-probe # 210k + 400k +python3 tests/perf/window_pipeline_gil_scaling.py --latency-probe --json > probe.json # capture +python3 tests/perf/window_pipeline_gil_scaling.py --render probe.json # re-print +python3 tests/perf/window_pipeline_gil_scaling.py --latency-probe --probe-sizes 52000,210000 \ + --probe-reps 2 --daemon-binary target/release/qbit-prism-build-audit-bundle +``` + +Run it alone if the point is the lateness numbers; other load on the host lands in the +`idle_control` row first, which is the row to compare against. diff --git a/tests/perf/window_pipeline_gil_scaling.py b/tests/perf/window_pipeline_gil_scaling.py new file mode 100644 index 000000000..e5d23ab27 --- /dev/null +++ b/tests/perf/window_pipeline_gil_scaling.py @@ -0,0 +1,2390 @@ +#!/usr/bin/env python3 +"""GIL scaling of the PRISM payout-window materialization pipeline. + +Issue #131 profiles one window materialization as ~135 ms of **GIL-held** +Python at today's 21,868-share window (~555 ms at 100k, ~2.2 s at 400k), split +across four terms: window fold, canonical-JSON digest, spool serialization, +and record->JSON conversion. "GIL-held" was *asserted* for this pipeline and +never measured on it -- the status comment #162 left on #143 names exactly +this gap: "GIL behaviour at megabyte buffer sizes is unmeasured." + +It matters because #143 section 1 bracketed CPython's GIL-release threshold at +**1-2 KiB** and reached **7.89 cores across 8 threads** on a 1 MiB +``hashlib.sha256`` positive control. Every buffer in this pipeline is far past +that threshold. A term built out of GIL-releasing primitives is already +running in parallel and is not evidence for a migration. + +This driver answers, per stage: **how much of the profiled time is actually +GIL-held, and how much is already parallel?** + +Method mirrors #143 section 1 so the numbers are comparable: + +* **cores-used = (process CPU time) / (wall time)** with the stage running in + N = 1, 2, 4, 8 threads, each thread driving its **own independent input**. + 1.00 flat from 1->8 threads means fully GIL-held; scaling toward N means the + stage releases the GIL. +* a **1 MiB ``hashlib.sha256`` positive control** and a **small-buffer + pure-Python negative control** are measured in the same process, at the same + thread counts, in the same alternation as the stages, and are reported beside + every table. The positive control sets this host's achievable ceiling; the + negative control must pin at ~1.00. +* configurations are visited **alternating** (outer loop = repetition, inner + loop = configuration) rather than in blocks, so host drift lands on every + configuration equally rather than on whichever ran last. + +**Thread-start amortization is load-bearing and was measured, not assumed.** +On this rig the same 1 MiB sha256 positive control reads 5.33 cores at 0.26 s +of wall and 7.42 cores at 0.90 s -- the shortfall is thread create/join +overhead charged against too short a measurement, not a GIL effect. Every +configuration is therefore grown until its wall clock clears +``--min-wall-seconds`` (default 1.0 s). Skipping this understates cores-used +on *every* stage, which would have manufactured a false confirmation of the +"GIL-held" premise. See ``AMORTIZATION_RATIONALE``. + +Stages drive the **shipped** callables -- nothing is reimplemented: + +=========================== ================================================== +stage callable +=========================== ================================================== +``fold`` ``IncrementalShareWindow.from_full_snapshot`` +``fold_pages`` ``_IncrementalShareWindowPage.from_records`` (all) +``digest`` ``IncrementalShareJsonSequence.canonical_json_sha256`` +``to_prism_json`` ``AcceptedShareRecord.to_prism_json`` (all records) +``spool_acquire`` ``_ShareWindowSerialization.acquire_spooled_tail`` +``spool_compact`` ``_ShareWindowSerialization.compact_tail_chunks`` +``spool_encode`` ``str.encode("utf-8")`` of both fragments +``spool_write`` ``TemporaryFile`` write/flush/seek of the payload +=========================== ================================================== + +``fold_pages``, ``spool_compact``, ``spool_encode`` and ``spool_write`` are +sub-terms, reported to decompose the two stages that are internally mixed; +they are not additional profile rows. + +Records are shaped like ``lab/prism/job_build_benchmark.py``'s defaults +(``--shares 21868 --miners 2``), whose values encode a live-host measurement. +``--share-id-shape production`` re-runs the byte accounting with the +``username:block_hash_hex`` share_id that ``lab/prism/share_writer.py`` builds, +because the benchmark's short synthetic ``share_id`` materially changes payload +bytes (but not, as measured, any GIL verdict). + +Self-contained and re-runnable: no database, no network, no coordinator, no +third-party packages. ``python3 tests/perf/window_pipeline_gil_scaling.py`` +prints the tables; ``--json`` emits the same data structured. + +This is an **on-demand instrument, not a test**: it asserts no thresholds and +is deliberately not named ``test_*`` so the discovery run never executes it +(#160 -- a threshold assertion on a shared runner is a flaky test in waiting). +Nothing under ``lab/`` is imported for anything but read-only measurement. + +``--latency-probe`` (#236) is a second mode over the same fixture: instead of +cores-used across threads it runs each serialization phase once at the +incident's window sizes (210k and 400k shares by default) in the main thread +while a monitor thread wakes on the writer-lease monitor's cadence +(``WRITER_LEASE_HEARTBEAT_MONITOR_SECONDS``) and records how late every wake +was. A GIL-held C call delays the monitor for its whole duration; a Python +loop of bounded C calls lets the interpreter switch at its normal interval. +Each bounded phase is paired with the historical whole-window call it +replaced, and ``--daemon-binary`` adds the real Rust builder's prepare +round trip split into request write, response wait and response read. See +``LATENCY_PROBE_RATIONALE``. +""" + +from __future__ import annotations + +import argparse +import contextlib +import gc +import hashlib +import json +import os +import platform +import sys +import sysconfig +import tempfile +import threading +import time +from dataclasses import dataclass +from pathlib import Path +from typing import Any, Callable, Iterator, Sequence + +if not __package__: + sys.path.insert(0, str(Path(__file__).resolve().parents[2])) + +from lab.prism.bundle_compiler import ( + BundleCompiler, + _compact_share_payload, + _compact_share_tail_chunks, + _iter_prepare_window_request_chunks, + _ShareWindowSerialization, +) +from lab.prism.share_json_stream import ( + canonical_share_array_sha256, + iter_canonical_share_item_chunks, + iter_json_object_text_chunks, +) +from lab.prism.share_ledger import ( + DEFAULT_INCREMENTAL_SHARE_WINDOW_PAGE_SIZE, + AcceptedShareRecord, + DaemonShareJsonSequence, + DaemonShareWindowMirror, + IncrementalShareWindow, + _IncrementalShareWindowPage, + block_candidate_identity, + block_candidate_identity_sha256, + sha256_json_hex, +) +from lab.prism.writer_lease_timing import ( + WRITER_LEASE_HEARTBEAT_MONITOR_SECONDS, + WRITER_LEASE_HEARTBEAT_SCHEDULER_SLACK_SECONDS, +) + + +CLOCK_RATIONALE = ( + "cores-used = process CPU / wall. time.process_time_ns() is the numerator " + "because it sums CPU across every thread in the process, which is exactly " + "the quantity that separates 'N threads made progress together' from 'N " + "threads took turns under the GIL'. time.perf_counter_ns() is the " + "denominator. thread_time is deliberately NOT used here: it is per-thread " + "and cannot see the concurrency that is the whole question." +) + +AMORTIZATION_RATIONALE = ( + "Thread create/join is charged to wall clock but contributes little " + "process CPU, so a measurement whose wall is close to the thread-start " + "cost reads LOW on cores-used regardless of GIL behaviour. Measured on " + "this rig with the 1 MiB sha256 positive control: 5.33 cores at 0.26 s " + "wall, 6.43 at 0.25 s, 7.42 at 0.90 s, 7.42 at 3.71 s. Every " + "configuration is therefore grown until wall >= --min-wall-seconds. " + "Without this the bias is one-sided: it pushes every stage toward 1.00, " + "i.e. toward falsely confirming 'GIL-held'." +) + +# --------------------------------------------------------------------------- +# published anchors +# --------------------------------------------------------------------------- + +# #143 section 1: 1 MiB hashlib.sha256, 8 threads. +PUBLISHED_POSITIVE_CONTROL_CORES = 7.89 +# #143 section 1 bracketed CPython's GIL-release threshold here. +PUBLISHED_RELEASE_THRESHOLD_BYTES = (1024, 2048) + +# #131's profile table, per materialization, developer workstation. Keyed by +# window size; values are milliseconds attributed to each term. +PUBLISHED_PROFILE_MS: dict[int, dict[str, float]] = { + 21_868: { + "fold": 71.0, + "digest": 31.0, + "spool_acquire": 28.0, + "to_prism_json": 5.0, + }, + 100_000: { + "fold": 271.0, + "digest": 139.0, + "spool_acquire": 122.0, + "to_prism_json": 23.0, + }, + 400_000: { + "fold": 1_057.0, + "digest": 530.0, + "spool_acquire": 476.0, + "to_prism_json": 95.0, + }, +} + +# #131 annotates the spool row with a byte size at each window size. +PUBLISHED_SPOOL_ROW_BYTES: dict[int, int] = { + 21_868: 8_000_000, + 100_000: 37_000_000, + 400_000: 151_000_000, +} + +# job_build_benchmark.py defaults: --shares 21868, --miners 2. +BENCHMARK_SHARES = 21_868 +BENCHMARK_MINERS = 2 + +DEFAULT_SIZES = (21_868, 100_000, 400_000) +DEFAULT_THREADS = (1, 2, 4, 8) +# Above this window size an 8-thread sweep holds ~0.7 GB of independent input +# per thread; the contract for this measurement allows the largest size to be +# reported at the endpoints only. Overridable with --large-size-threads. +LARGE_SIZE_BYTES_THRESHOLD = 200_000 +LARGE_SIZE_THREADS = (1, 8) + +POSITIVE_CONTROL_BYTES = 1 << 20 +NEGATIVE_CONTROL_ITERATIONS = 20_000 + +# Fixed epoch so runs are reproducible; the pipeline only compares these. +FIXED_NOW_MS = 1_760_000_000_000 + + +# --------------------------------------------------------------------------- +# measurement harness +# --------------------------------------------------------------------------- + + +@dataclass(frozen=True) +class CoresResult: + """One (stage, thread-count) measurement.""" + + threads: int + cores_used: float + wall_seconds: float + process_cpu_seconds: float + iterations: int + cpu_ms_per_call: float + wall_ms_per_call: float + loadavg_1: float | None + grew: int + + def as_json(self) -> dict[str, Any]: + return { + "threads": self.threads, + "cores_used": round(self.cores_used, 4), + "wall_seconds": round(self.wall_seconds, 4), + "process_cpu_seconds": round(self.process_cpu_seconds, 4), + "iterations": self.iterations, + "cpu_ms_per_call": round(self.cpu_ms_per_call, 4), + "wall_ms_per_call": round(self.wall_ms_per_call, 4), + "loadavg_1": self.loadavg_1, + "grew": self.grew, + } + + +def _loadavg_1() -> float | None: + try: + return os.getloadavg()[0] + except (OSError, AttributeError): # pragma: no cover - platform dependent + return None + + +def measure_cores( + make_callable: Callable[[int], Callable[[], Any]], + threads: int, + *, + min_wall_seconds: float, + max_wall_seconds: float, + max_growths: int = 4, +) -> CoresResult: + """Cores-used for ``threads`` copies of a stage on independent inputs. + + ``make_callable(i)`` returns the zero-argument callable thread ``i`` will + drive; each index must own its own input so the only thing shared between + threads is the interpreter itself. + + The iteration count is calibrated from one untimed warm call and then + **grown until the wall clock clears ``min_wall_seconds``** -- see + ``AMORTIZATION_RATIONALE``; an under-length measurement is biased toward + 1.00 and would read as "GIL-held" whatever the truth is. ``max_wall_seconds`` + bounds the opposite end: a fully GIL-held stage serializes, so its wall grows + ~linearly in ``threads`` and would otherwise run for minutes at 400k shares. + """ + + callables = [make_callable(index) for index in range(threads)] + + # Warm one copy (imports, allocator arenas, branch predictors) and use it + # to size the batch. The warm call is not part of any reported number. + warm_start = time.perf_counter() + callables[0]() + per_call = max(time.perf_counter() - warm_start, 1e-9) + + iterations = max(1, int(min_wall_seconds / per_call)) + serialized_wall = per_call * iterations * threads + if serialized_wall > max_wall_seconds: + iterations = max(1, int(max_wall_seconds / (per_call * threads))) + + grew = 0 + while True: + gate = threading.Barrier(threads + 1) + + def worker(fn: Callable[[], Any], count: int) -> None: + gate.wait() + for _ in range(count): + fn() + + workers = [ + threading.Thread(target=worker, args=(fn, iterations), daemon=True) + for fn in callables + ] + for thread in workers: + thread.start() + # Threads exist and are parked on the barrier before the clocks start, + # so thread *creation* is outside the measured window. Start/join + # scheduling still is not, which is what min_wall_seconds covers. + gate.wait() + load = _loadavg_1() + cpu_start = time.process_time_ns() + wall_start = time.perf_counter_ns() + for thread in workers: + thread.join() + wall_ns = time.perf_counter_ns() - wall_start + cpu_ns = time.process_time_ns() - cpu_start + + wall_seconds = wall_ns / 1e9 + if wall_seconds >= min_wall_seconds or grew >= max_growths: + break + # Under-length: grow toward the floor and re-measure from scratch. + # Wall scales with the iteration count whether the stage is serialized + # or parallel, so the projection needs no thread-count factor. + scale = max(2, int(min_wall_seconds / max(wall_seconds, 1e-6)) + 1) + if wall_seconds * scale > max_wall_seconds: + break + iterations = max(iterations + 1, iterations * scale) + grew += 1 + + calls = iterations * threads + return CoresResult( + threads=threads, + cores_used=cpu_ns / wall_ns if wall_ns else float("nan"), + wall_seconds=wall_ns / 1e9, + process_cpu_seconds=cpu_ns / 1e9, + iterations=iterations, + cpu_ms_per_call=cpu_ns / 1e6 / calls, + wall_ms_per_call=wall_ns / 1e6 / iterations, + loadavg_1=load, + grew=grew, + ) + + +# --------------------------------------------------------------------------- +# controls +# --------------------------------------------------------------------------- + + +def positive_control(index: int) -> Callable[[], Any]: + """#143's control: 1 MiB hashlib.sha256, which must scale with threads. + + Each thread hashes a distinct buffer so nothing is shared but the + interpreter. + """ + # bytes([n]) -- a one-element list. bytes(n) would build n zero bytes and + # give every thread a differently sized buffer, which desynchronizes them + # and reads as a GIL effect it is not. + buffer = bytes([(index * 37 + 11) % 251]) * POSITIVE_CONTROL_BYTES + + def run() -> bytes: + return hashlib.sha256(buffer).digest() + + return run + + +def negative_control(index: int) -> Callable[[], Any]: + """Small-buffer pure-Python CPU, which must pin at ~1.00 cores.""" + seed = index + 1 + + def run() -> int: + total = 0 + for value in range(NEGATIVE_CONTROL_ITERATIONS): + total += value * seed + return total + + return run + + +# --------------------------------------------------------------------------- +# inputs +# --------------------------------------------------------------------------- + + +def benchmark_miner_programs(count: int) -> tuple[str, ...]: + """Same construction as ``lab/prism/job_build_benchmark.py``.""" + return tuple( + hashlib.sha256(f"bench-miner-{index}".encode()).hexdigest() + for index in range(count) + ) + + +def build_records( + count: int, + *, + miners: int = BENCHMARK_MINERS, + share_id_shape: str = "benchmark", + salt: int = 0, +) -> list[AcceptedShareRecord]: + """Records shaped like ``job_build_benchmark.BenchLedger``'s window. + + ``share_id_shape="production"`` swaps the benchmark's short synthetic + ``share_id`` for the ``username:block_hash_hex`` form that + ``lab/prism/share_writer.py`` actually builds (the ledger schema's + ``length(share_id) >= 65`` index predicate corroborates the length). Only + the byte accounting moves; the GIL verdicts do not. + """ + + programs = benchmark_miner_programs(miners) + records: list[AcceptedShareRecord] = [] + for index in range(count): + miner = index % miners + if share_id_shape == "production": + block_hash = hashlib.sha256( + f"{salt}:{index}".encode() + ).hexdigest() + share_id = f"bench-miner-{miner}.rig{index % 512}:{block_hash}" + else: + share_id = f"bench-share-{index + 1}" + records.append( + AcceptedShareRecord( + share_seq=index + 1, + share_id=share_id, + miner_id=f"bench-miner-{miner}", + order_key=f"bench-miner-{miner:06d}", + p2mr_program_hex=programs[miner], + share_difficulty=16384, + network_difficulty=226646186, + template_height=9, + job_id=f"bench-job-{index + 1}", + job_issued_at_ms=FIXED_NOW_MS - (count - index) * 1_000, + accepted_at_ms=FIXED_NOW_MS - (count - index) * 1_000, + ntime=FIXED_NOW_MS // 1000 - (count - index), + ) + ) + return records + + +@dataclass +class StageInputs: + """One thread's private copy of every input the stages need.""" + + records: list[AcceptedShareRecord] + anchor_job_issued_at_ms: int + window_weight: int + window: IncrementalShareWindow + page_records: tuple[tuple[AcceptedShareRecord, ...], ...] + json_sequence: Any + shares: list[dict[str, object]] + snapshot_sha256: str + identities_json: str + compact_shares_json: str + identities_bytes: bytes + compact_shares_bytes: bytes + canonical_json_bytes: int + spool_bytes: int + + +def build_inputs( + count: int, + *, + miners: int, + share_id_shape: str, + salt: int, +) -> StageInputs: + records = build_records( + count, miners=miners, share_id_shape=share_id_shape, salt=salt + ) + anchor = int(records[-1].job_issued_at_ms) + # Retain every record: the window-weight cutoff is not what is under test, + # and a partial retention would silently shrink the measured window. + weight = sum(int(record.share_difficulty) for record in records) + window = IncrementalShareWindow.from_full_snapshot( + records, + anchor_job_issued_at_ms=anchor, + window_weight=weight, + ) + sequence = window.json_records() + shares = list(sequence) + snapshot = sequence.canonical_json_sha256() + identities, compact_shares = _compact_share_payload(shares) + identities_json = json.dumps(identities, separators=(",", ":")) + compact_shares_json = json.dumps(compact_shares, separators=(",", ":")) + identities_bytes = identities_json.encode("utf-8") + compact_shares_bytes = compact_shares_json.encode("utf-8") + canonical_bytes = sum(len(page.canonical_json_items) for page in window.pages) + # Exactly the byte layout acquire_spooled_tail writes. + spool_bytes = ( + len(b',"compact_share_identities":') + + len(identities_bytes) + + len(b',"compact_shares":') + + len(compact_shares_bytes) + + len(b"}") + ) + return StageInputs( + records=records, + anchor_job_issued_at_ms=anchor, + window_weight=weight, + window=window, + page_records=tuple(page.records for page in window.pages), + json_sequence=sequence, + shares=shares, + snapshot_sha256=snapshot, + identities_json=identities_json, + compact_shares_json=compact_shares_json, + identities_bytes=identities_bytes, + compact_shares_bytes=compact_shares_bytes, + canonical_json_bytes=canonical_bytes, + spool_bytes=spool_bytes, + ) + + +# --------------------------------------------------------------------------- +# stages +# --------------------------------------------------------------------------- + + +@dataclass(frozen=True) +class Stage: + key: str + callable_name: str + profile_row: str | None + parent: str | None + note: str + factory: Callable[[StageInputs], Callable[[], Any]] + + +def _fold(inputs: StageInputs) -> Callable[[], Any]: + records = inputs.records + anchor = inputs.anchor_job_issued_at_ms + weight = inputs.window_weight + + def run() -> IncrementalShareWindow: + return IncrementalShareWindow.from_full_snapshot( + records, + anchor_job_issued_at_ms=anchor, + window_weight=weight, + ) + + return run + + +def _fold_pages(inputs: StageInputs) -> Callable[[], Any]: + page_records = inputs.page_records + + def run() -> tuple[Any, ...]: + return tuple( + _IncrementalShareWindowPage.from_records(records) + for records in page_records + ) + + return run + + +def _digest(inputs: StageInputs) -> Callable[[], Any]: + sequence = inputs.json_sequence + + def run() -> str: + return sequence.canonical_json_sha256() + + return run + + +def _to_prism_json(inputs: StageInputs) -> Callable[[], Any]: + records = inputs.records + + def run() -> list[dict[str, object]]: + return [record.to_prism_json() for record in records] + + return run + + +def _new_serialization(inputs: StageInputs) -> _ShareWindowSerialization: + return _ShareWindowSerialization( + key=(inputs.snapshot_sha256, len(inputs.records), inputs.window_weight), + share_count=len(inputs.records), + share_snapshot_sha256=inputs.snapshot_sha256, + ) + + +def _spool_acquire(inputs: StageInputs) -> Callable[[], Any]: + shares = inputs.shares + + def run() -> None: + # A fresh instance every call: acquire_spooled_tail memoizes, so a + # reused instance would measure a dict lookup after the first call. + serialization = _new_serialization(inputs) + serialization.acquire_spooled_tail(shares) + # Retire then release so the leased descriptor is closed by the + # shipped teardown path rather than left to the GC. + serialization.retire_spool() + serialization.release_spooled_tail() + + return run + + +def _spool_compact(inputs: StageInputs) -> Callable[[], Any]: + shares = inputs.shares + + def run() -> tuple[str, ...]: + return _new_serialization(inputs).compact_tail_chunks(shares) + + return run + + +def _spool_encode(inputs: StageInputs) -> Callable[[], Any]: + identities_json = inputs.identities_json + compact_shares_json = inputs.compact_shares_json + + def run() -> tuple[bytes, bytes]: + return ( + identities_json.encode("utf-8"), + compact_shares_json.encode("utf-8"), + ) + + return run + + +def _spool_write(inputs: StageInputs) -> Callable[[], Any]: + identities_bytes = inputs.identities_bytes + compact_shares_bytes = inputs.compact_shares_bytes + + def run() -> None: + spool = tempfile.TemporaryFile() + try: + spool.write(b',"compact_share_identities":') + spool.write(identities_bytes) + spool.write(b',"compact_shares":') + spool.write(compact_shares_bytes) + spool.write(b"}") + spool.flush() + spool.seek(0, os.SEEK_END) + finally: + spool.close() + + return run + + +STAGES: tuple[Stage, ...] = ( + Stage( + "fold", + "IncrementalShareWindow.from_full_snapshot", + "Window fold", + None, + "sorted() + per-record validation + paging; paging calls to_prism_json " + "and json.dumps per record", + _fold, + ), + Stage( + "fold_pages", + "_IncrementalShareWindowPage.from_records (every page)", + None, + "fold", + "the paging half of the fold: to_prism_json, json.dumps, b','.join", + _fold_pages, + ), + Stage( + "digest", + "IncrementalShareJsonSequence.canonical_json_sha256", + "Canonical-JSON digest", + None, + "one hashlib.sha256 fed pre-encoded per-page buffers", + _digest, + ), + Stage( + "to_prism_json", + "AcceptedShareRecord.to_prism_json (every record)", + "Record->JSON conversion", + None, + "measured standalone; note the fold already calls it internally", + _to_prism_json, + ), + Stage( + "spool_acquire", + "_ShareWindowSerialization.acquire_spooled_tail (cold)", + "Spool serialization", + None, + "compact_fragments + utf-8 encode + TemporaryFile writes", + _spool_acquire, + ), + Stage( + "spool_compact", + "_ShareWindowSerialization.compact_tail_chunks (cold)", + None, + "spool_acquire", + "identity dedup + batched json.dumps into bounded chunks (#236)", + _spool_compact, + ), + Stage( + "spool_encode", + 'str.encode("utf-8") of both fragments', + None, + "spool_acquire", + "str -> bytes for the two fragment strings", + _spool_encode, + ), + Stage( + "spool_write", + "TemporaryFile write/flush/seek of the spool payload", + None, + "spool_acquire", + "the os.write half of the spool term", + _spool_write, + ), +) + +STAGES_BY_KEY = {stage.key: stage for stage in STAGES} +PROFILE_ROW_STAGES = tuple(stage for stage in STAGES if stage.profile_row) + + +# --------------------------------------------------------------------------- +# environment +# --------------------------------------------------------------------------- + + +def _peak_rss_mb() -> float | None: + try: + import resource + + peak = resource.getrusage(resource.RUSAGE_SELF).ru_maxrss + except Exception: # pragma: no cover - platform dependent + return None + # Linux reports KiB, macOS bytes. + return peak / 1024.0 if sys.platform != "darwin" else peak / (1024.0 * 1024.0) + + +def _current_rss_mb() -> float | None: + """Resident set size *right now*. + + Deliberately not ru_maxrss: that is a lifetime high-water mark that never + falls, so after the smaller window sizes have run it would over-estimate + the live footprint by several hundred MB and skip the large size for a + shortage that does not exist. + """ + try: + with open("/proc/self/statm", encoding="ascii") as handle: + resident_pages = int(handle.read().split()[1]) + except (OSError, IndexError, ValueError): # pragma: no cover - platform + return None + return resident_pages * os.sysconf("SC_PAGE_SIZE") / (1024.0 * 1024.0) + + +def _available_memory_mb() -> float | None: + try: + with open("/proc/meminfo", encoding="ascii") as handle: + for line in handle: + if line.startswith("MemAvailable:"): + return int(line.split()[1]) / 1024.0 + except OSError: # pragma: no cover - platform dependent + return None + return None + + +def describe_environment() -> dict[str, Any]: + try: + load = os.getloadavg() + except (OSError, AttributeError): # pragma: no cover - platform dependent + load = None + try: + gil_enabled: bool | None = sys._is_gil_enabled() # type: ignore[attr-defined] + except AttributeError: + gil_enabled = None + physical = None + try: + with open("/proc/cpuinfo", encoding="ascii") as handle: + text = handle.read() + cores = { + line.split(":", 1)[1].strip() + for line in text.splitlines() + if line.startswith("cpu cores") + } + if cores: + physical = sorted(cores)[0] + except OSError: # pragma: no cover - platform dependent + pass + return { + "cpu": platform.processor() or platform.machine(), + "machine": platform.machine(), + "platform": platform.platform(), + "cpu_count": os.cpu_count(), + "cpu_cores_per_socket": physical, + "python_version": sys.version, + "python_implementation": platform.python_implementation(), + "gil_disabled_build": bool(sysconfig.get_config_var("Py_GIL_DISABLED")), + "gil_enabled_at_runtime": gil_enabled, + "loadavg_1_5_15": load, + "available_memory_mb": _available_memory_mb(), + "page_size_records": DEFAULT_INCREMENTAL_SHARE_WINDOW_PAGE_SIZE, + "clock_rationale": CLOCK_RATIONALE, + "amortization_rationale": AMORTIZATION_RATIONALE, + } + + +# --------------------------------------------------------------------------- +# analysis +# --------------------------------------------------------------------------- + + +def released_fraction(cores: float, ceiling: float) -> float | None: + """Fraction of a stage's CPU that ran with the GIL released. + + Normalized against the positive control measured **on this host at the + same thread count**, not against the nominal thread count: an 8-vCPU guest + cannot reach 8.00 even on a perfectly parallel workload, so dividing by 8 + would understate every stage. 1.00 cores maps to 0.0, the control's + ceiling maps to 1.0. Returns None when the ceiling offers no headroom. + """ + headroom = ceiling - 1.0 + if headroom <= 0.05: + return None + return max(0.0, min(1.0, (cores - 1.0) / headroom)) + + +def _fmt(value: float | None, digits: int = 2) -> str: + if value is None: + return "n/a" + if value != value: # NaN + return "n/a" + return f"{value:.{digits}f}" + + +def _fmt_mb(value: float | None) -> str: + return "n/a" if value is None else f"{value / 1e6:.2f}" + + +def _median(values: Sequence[float]) -> float: + ordered = sorted(values) + count = len(ordered) + if not count: + return float("nan") + mid = count // 2 + if count % 2: + return ordered[mid] + return (ordered[mid - 1] + ordered[mid]) / 2.0 + + +# --------------------------------------------------------------------------- +# runner +# --------------------------------------------------------------------------- + + +def thread_counts_for(size: int, threads: Sequence[int], large: Sequence[int]) -> tuple[int, ...]: + if size > LARGE_SIZE_BYTES_THRESHOLD: + return tuple(t for t in threads if t in set(large)) + return tuple(threads) + + +def run(args: argparse.Namespace) -> dict[str, Any]: + sizes = tuple(args.sizes) + threads = tuple(args.threads) + large_threads = tuple(args.large_size_threads) + reps = args.reps + + environment = describe_environment() + results: dict[str, Any] = { + "environment": environment, + "methodology": { + "metric": "cores-used = process CPU / wall", + "repetitions": reps, + "statistic": "median of N alternating repetitions", + "alternation": ( + "outer loop repetition, inner loop configuration, so host " + "drift lands on every configuration equally" + ), + "min_wall_seconds": args.min_wall_seconds, + "max_wall_seconds": args.max_wall_seconds, + "independent_inputs_per_thread": True, + "share_id_shape": args.share_id_shape, + "miners": args.miners, + "page_size": DEFAULT_INCREMENTAL_SHARE_WINDOW_PAGE_SIZE, + "published_positive_control_cores": PUBLISHED_POSITIVE_CONTROL_CORES, + }, + "sizes": [], + "controls": {}, + "peak_rss_mb": None, + "aborted": None, + } + + control_specs = ( + ("positive", positive_control, "1 MiB hashlib.sha256"), + ("negative", negative_control, f"{NEGATIVE_CONTROL_ITERATIONS}-iteration pure-Python loop"), + ) + + # ---- controls, alternating across thread counts and repetitions -------- + control_samples: dict[str, dict[int, list[CoresResult]]] = { + name: {count: [] for count in threads} for name, _, _ in control_specs + } + for _ in range(reps): + for count in threads: + for name, factory, _ in control_specs: + control_samples[name][count].append( + measure_cores( + factory, + count, + min_wall_seconds=args.min_wall_seconds, + max_wall_seconds=args.max_wall_seconds, + ) + ) + + ceilings: dict[int, float] = {} + for name, _, description in control_specs: + per_thread = [] + for count in threads: + samples = control_samples[name][count] + cores = _median([s.cores_used for s in samples]) + if name == "positive": + ceilings[count] = cores + per_thread.append( + { + "threads": count, + "cores_used_median": cores, + "cores_used_min": min(s.cores_used for s in samples), + "cores_used_max": max(s.cores_used for s in samples), + "wall_seconds_median": _median([s.wall_seconds for s in samples]), + "loadavg_1_median": _median( + [s.loadavg_1 for s in samples if s.loadavg_1 is not None] + ) + if any(s.loadavg_1 is not None for s in samples) + else None, + "samples": [s.as_json() for s in samples], + } + ) + results["controls"][name] = { + "description": description, + "by_threads": per_thread, + } + + positive_at_max = ceilings.get(max(threads)) + results["controls"]["positive_control_reproduces"] = ( + None + if positive_at_max is None + else positive_at_max / PUBLISHED_POSITIVE_CONTROL_CORES + ) + + # The contract for this measurement: if the positive control does not + # reproduce, the rig is wrong and the stage numbers must not be reported. + # The floor is a fraction of the *nominal* thread count rather than an + # absolute core count, so the gate stays meaningful when the sweep is run + # at fewer threads: a 4-thread sweep can never reach 6 cores, and an + # absolute floor would abort a perfectly healthy rig. #143 published + # 7.89/8 = 0.986 of nominal. + control_floor = args.control_floor_fraction * max(threads) + results["methodology"]["control_floor_cores"] = control_floor + results["methodology"]["control_floor_fraction"] = args.control_floor_fraction + if positive_at_max is not None and positive_at_max < control_floor: + results["aborted"] = ( + f"positive control reached {positive_at_max:.2f} cores at " + f"{max(threads)} threads, below the floor of " + f"{control_floor:.2f} (= {args.control_floor_fraction:g} x " + f"{max(threads)} threads; #143 published " + f"{PUBLISHED_POSITIVE_CONTROL_CORES} at 8). The rig is not " + f"measuring parallelism correctly; stage numbers are withheld." + ) + results["peak_rss_mb"] = _peak_rss_mb() + return results + + # ---- stages ----------------------------------------------------------- + for size in sizes: + size_threads = thread_counts_for(size, threads, large_threads) + if not size_threads: + continue + max_threads = max(size_threads) + + rss_before = _current_rss_mb() or 0.0 + probe = build_inputs( + size, + miners=args.miners, + share_id_shape=args.share_id_shape, + salt=0, + ) + rss_after = _current_rss_mb() or 0.0 + available = _available_memory_mb() + # Footprint of exactly one thread's inputs, measured rather than + # guessed, with 1.6x headroom for the transient window a running fold + # allocates before the previous one is collected. + per_input_mb = max(rss_after - rss_before, 1.0) + estimate_mb = per_input_mb * max_threads * 1.6 + if available is not None and estimate_mb > available * 0.8: + results["sizes"].append( + { + "size": size, + "skipped": ( + f"estimated {estimate_mb:.0f} MB for {max_threads} " + f"independent inputs ({per_input_mb:.0f} MB each) " + f"exceeds 80% of {available:.0f} MB available" + ), + } + ) + del probe + continue + + inputs = [probe] + [ + build_inputs( + size, + miners=args.miners, + share_id_shape=args.share_id_shape, + salt=index, + ) + for index in range(1, max_threads) + ] + + samples: dict[str, dict[int, list[CoresResult]]] = { + stage.key: {count: [] for count in size_threads} for stage in STAGES + } + for _ in range(reps): + for count in size_threads: + for stage in STAGES: + samples[stage.key][count].append( + measure_cores( + # noqa is for `inputs`: it is bound at the top of + # this loop body and freed by `del inputs` at the + # end of it, which is enough for Ruff to read the + # closure as unbound. measure_cores joins every + # thread before returning, so no invocation of this + # lambda outlives the binding. + lambda index, stage=stage: stage.factory(inputs[index]), # noqa: F821 + count, + min_wall_seconds=args.min_wall_seconds, + max_wall_seconds=args.max_wall_seconds, + ) + ) + + stage_rows = [] + for stage in STAGES: + by_threads = [] + for count in size_threads: + stage_samples = samples[stage.key][count] + cores = _median([s.cores_used for s in stage_samples]) + by_threads.append( + { + "threads": count, + "cores_used_median": cores, + "cores_used_min": min(s.cores_used for s in stage_samples), + "cores_used_max": max(s.cores_used for s in stage_samples), + "cpu_ms_per_call_median": _median( + [s.cpu_ms_per_call for s in stage_samples] + ), + "wall_ms_per_call_median": _median( + [s.wall_ms_per_call for s in stage_samples] + ), + "released_fraction": released_fraction( + cores, ceilings.get(count, float("nan")) + ) + if count > 1 + else None, + "loadavg_1_median": _median( + [s.loadavg_1 for s in stage_samples if s.loadavg_1 is not None] + ) + if any(s.loadavg_1 is not None for s in stage_samples) + else None, + "samples": [s.as_json() for s in stage_samples], + } + ) + stage_rows.append( + { + "key": stage.key, + "callable": stage.callable_name, + "profile_row": stage.profile_row, + "parent": stage.parent, + "note": stage.note, + "by_threads": by_threads, + } + ) + + results["sizes"].append( + { + "size": size, + "thread_counts": list(size_threads), + "input_mb_per_thread": round(per_input_mb, 1), + "bytes": { + "canonical_json_bytes": probe.canonical_json_bytes, + "spool_payload_bytes": probe.spool_bytes, + "canonical_bytes_per_record": probe.canonical_json_bytes / size, + "spool_bytes_per_record": probe.spool_bytes / size, + "page_count": len(probe.window.pages), + "canonical_bytes_per_page": ( + probe.canonical_json_bytes / max(len(probe.window.pages), 1) + ), + "published_spool_row_bytes": PUBLISHED_SPOOL_ROW_BYTES.get(size), + }, + "stages": stage_rows, + } + ) + + del inputs + del probe + gc.collect() + + results["peak_rss_mb"] = _peak_rss_mb() + try: + results["loadavg_end"] = list(os.getloadavg()) + except (OSError, AttributeError): # pragma: no cover - platform dependent + results["loadavg_end"] = None + return results + + +# --------------------------------------------------------------------------- +# reporting +# --------------------------------------------------------------------------- + + +def render_report(results: dict[str, Any]) -> str: + out: list[str] = [] + env = results["environment"] + method = results["methodology"] + + out.append("=" * 78) + out.append("GIL scaling of the PRISM payout-window materialization pipeline") + out.append("=" * 78) + out.append("") + out.append(f"CPU {env['cpu']} x{env['cpu_count']} logical") + out.append(f"Platform {env['platform']}") + out.append(f"Python {env['python_version'].splitlines()[0]}") + out.append( + f"GIL build Py_GIL_DISABLED={env['gil_disabled_build']}, " + f"runtime enabled={env['gil_enabled_at_runtime']}" + ) + load = env["loadavg_1_5_15"] + out.append( + "Load at start " + + ("n/a" if load is None else " / ".join(f"{v:.2f}" for v in load)) + ) + out.append(f"Page size {env['page_size_records']} records") + out.append( + f"Statistic median of {method['repetitions']} alternating repetitions; " + f"wall floor {method['min_wall_seconds']}s" + ) + out.append("") + out.append("Metric: " + method["metric"]) + out.append("") + + # ---- controls ---------------------------------------------------------- + out.append("-" * 78) + out.append("CONTROLS (measured in this process, same alternation as the stages)") + out.append("-" * 78) + out.append("") + header = f"{'control':<34}" + "".join( + f"{'N=' + str(row['threads']):>10}" + for row in results["controls"]["positive"]["by_threads"] + ) + out.append(header) + out.append("-" * len(header)) + for name in ("positive", "negative"): + control = results["controls"][name] + label = f"{name}: {control['description']}" + out.append( + f"{label[:33]:<34}" + + "".join( + f"{_fmt(row['cores_used_median']):>10}" + for row in control["by_threads"] + ) + ) + out.append("") + loads = [ + row["loadavg_1_median"] + for row in results["controls"]["positive"]["by_threads"] + if row["loadavg_1_median"] is not None + ] + if loads: + out.append(f"load average (1m) during controls: {min(loads):.2f}-{max(loads):.2f}") + reproduces = results["controls"].get("positive_control_reproduces") + top_threads = results["controls"]["positive"]["by_threads"][-1]["threads"] + if reproduces is not None and top_threads == 8: + out.append( + f"positive control vs #143's published {PUBLISHED_POSITIVE_CONTROL_CORES} " + f"cores at 8 threads: ratio {reproduces:.3f}" + ) + elif reproduces is not None: + # #143's anchor is an 8-thread number; comparing a shorter sweep's top + # thread count against it would read as a failure to reproduce when it + # is only a different thread count. + out.append( + f"positive control top sweep point is {top_threads} threads; " + f"#143's {PUBLISHED_POSITIVE_CONTROL_CORES}-core anchor is an " + f"8-thread figure and is not comparable here" + ) + out.append("") + + if results.get("aborted"): + out.append("!" * 78) + out.append("ABORTED: " + results["aborted"]) + out.append("!" * 78) + return "\n".join(out) + + # ---- per size ---------------------------------------------------------- + for entry in results["sizes"]: + size = entry["size"] + out.append("=" * 78) + out.append(f"WINDOW SIZE: {size:,} shares") + out.append("=" * 78) + if entry.get("skipped"): + out.append(f" SKIPPED: {entry['skipped']}") + out.append("") + continue + + counts = entry["thread_counts"] + byte_info = entry["bytes"] + out.append("") + out.append( + f" canonical JSON {_fmt_mb(byte_info['canonical_json_bytes'])} MB " + f"({byte_info['canonical_bytes_per_record']:.0f} B/record, " + f"{byte_info['page_count']} pages, " + f"{byte_info['canonical_bytes_per_page'] / 1024:.0f} KiB/page)" + ) + published_bytes = byte_info.get("published_spool_row_bytes") + out.append( + f" spool payload {_fmt_mb(byte_info['spool_payload_bytes'])} MB " + f"({byte_info['spool_bytes_per_record']:.0f} B/record)" + + ( + f" [#131 spool row says {published_bytes / 1e6:.0f} MB]" + if published_bytes + else "" + ) + ) + out.append("") + + head = f" {'stage':<30}" + "".join(f"{'N=' + str(c):>9}" for c in counts) + head += f"{'CPU ms':>10}{'released':>10}" + out.append(head) + out.append(" " + "-" * (len(head) - 2)) + for stage_row in entry["stages"]: + stage = STAGES_BY_KEY[stage_row["key"]] + label = (" " if stage.parent else "") + stage.key + cells = "" + for count in counts: + match = next( + r for r in stage_row["by_threads"] if r["threads"] == count + ) + cells += f"{_fmt(match['cores_used_median']):>9}" + single = next( + r for r in stage_row["by_threads"] if r["threads"] == counts[0] + ) + top = next( + r for r in stage_row["by_threads"] if r["threads"] == counts[-1] + ) + released = top["released_fraction"] + out.append( + f" {label:<30}{cells}" + f"{single['cpu_ms_per_call_median']:>10.1f}" + f"{(_fmt(released * 100, 0) + '%') if released is not None else 'n/a':>10}" + ) + out.append("") + loads = [ + r["loadavg_1_median"] + for s in entry["stages"] + for r in s["by_threads"] + if r["loadavg_1_median"] is not None + ] + if loads: + out.append(f" load average (1m) during this table: {min(loads):.2f}-{max(loads):.2f}") + out.append("") + + # ---- split of the published profile -------------------------------- + published = PUBLISHED_PROFILE_MS.get(size) + if published: + out.append(f" #131 profile split for {size:,} shares") + out.append( + f" {'profile row':<26}{'#131 ms':>9}{'released':>10}" + f"{'GIL-held':>10}{'parallel':>10}" + ) + out.append(" " + "-" * 65) + held_total = 0.0 + parallel_total = 0.0 + for stage in PROFILE_ROW_STAGES: + stage_row = next( + s for s in entry["stages"] if s["key"] == stage.key + ) + top = next( + r for r in stage_row["by_threads"] if r["threads"] == counts[-1] + ) + fraction = top["released_fraction"] or 0.0 + ms = published[stage.key] + parallel_ms = ms * fraction + held_ms = ms - parallel_ms + held_total += held_ms + parallel_total += parallel_ms + out.append( + f" {stage.profile_row:<26}{ms:>9.0f}" + f"{fraction * 100:>9.0f}%{held_ms:>10.0f}{parallel_ms:>10.0f}" + ) + total = held_total + parallel_total + out.append(" " + "-" * 65) + out.append( + f" {'TOTAL':<26}{total:>9.0f}{'':>10}" + f"{held_total:>10.0f}{parallel_total:>10.0f}" + ) + if total: + out.append( + f" => {parallel_total / total * 100:.0f}% of the profiled " + f"time is already running in parallel" + ) + out.append("") + + peak = results.get("peak_rss_mb") + if peak: + out.append(f"peak RSS: {peak:.0f} MB") + end = results.get("loadavg_end") + if end: + out.append("load average at end: " + " / ".join(f"{v:.2f}" for v in end)) + return "\n".join(out) + + +# --------------------------------------------------------------------------- +# #236 monitor-lateness probe +# --------------------------------------------------------------------------- + +LATENCY_PROBE_RATIONALE = ( + "The lease monitor thread must wake on its cadence while the payout " + "window is being serialized in another thread of the same process. A " + "single C call (json.dumps/json.loads over a whole 200k-share window) " + "holds the GIL until it returns, so the monitor's wake is late by the " + "call's duration; a Python loop of bounded C calls lets the interpreter " + "switch threads at its normal interval. The probe therefore measures, " + "for each phase, the maximum and p99 lateness of a thread that sleeps " + "on WRITER_LEASE_HEARTBEAT_MONITOR_SECONDS and records wake - due. This " + "is scheduling evidence on one host, not a proof of hard real-time " + "behaviour, and not a production measurement." +) + +PROBE_DEFAULT_SIZES = (210_000, 400_000) +# Distinct payout identities in the probe fixture: the compact payload's +# identity table and the fold's per-identity work depend on it. +PROBE_DEFAULT_MINERS = 200 +# The plan's stricter engineering target for monitor wake lateness, giving +# headroom against the configured scheduler slack. +PROBE_STRICT_LATENESS_SECONDS = 0.25 +PROBE_CONTROL_SECONDS = 1.0 +# Records appended by the advance step of the daemon round trip. +PROBE_ADVANCE_RECORDS = 16 + + +class MonitorLatenessProbe: + """A thread waking on the lease monitor's cadence, recording lateness. + + Each wake is scheduled ``interval`` after the previous wake (a late + wake does not owe a burst of catch-up wakes, exactly like a loop that + sleeps for its interval), and its lateness is ``wake - due``. Samples + are attributed to whichever phase the main thread has declared current; + wakes between phases are discarded. + """ + + def __init__(self, interval: float) -> None: + self._interval = float(interval) + self._lock = threading.Lock() + self._phase: str | None = None + self._samples: dict[str, list[float]] = {} + # Cyclic-GC pauses attributed to the current phase: (generation, + # seconds) per collection, from gc.callbacks. A full collection over + # a large live heap is itself one GIL-held stretch, and is the first + # suspect whenever a bounded phase still shows a late wake. + self._gc_pauses: dict[str, list[tuple[int, float]]] = {} + self._gc_started: float | None = None + self._stop = threading.Event() + self._thread = threading.Thread( + target=self._run, + name="lease-monitor-lateness-probe", + daemon=True, + ) + + def start(self) -> None: + gc.callbacks.append(self._gc_callback) + self._thread.start() + + def stop(self) -> None: + self._stop.set() + self._thread.join() + with contextlib.suppress(ValueError): + gc.callbacks.remove(self._gc_callback) + + def _gc_callback(self, event: str, info: dict[str, Any]) -> None: + # Runs on whichever thread triggered the collection, under the GIL. + if event == "start": + self._gc_started = time.perf_counter() + return + started = self._gc_started + self._gc_started = None + if started is None: + return + pause = time.perf_counter() - started + with self._lock: + phase = self._phase + if phase is not None: + self._gc_pauses[phase].append((int(info.get("generation", -1)), pause)) + + def _run(self) -> None: + due = time.monotonic() + self._interval + while not self._stop.is_set(): + delay = due - time.monotonic() + if delay > 0: + time.sleep(delay) + now = time.monotonic() + late = now - due + with self._lock: + phase = self._phase + if phase is not None: + self._samples[phase].append(late) + due = now + self._interval + + @contextlib.contextmanager + def phase(self, name: str) -> Iterator[None]: + with self._lock: + self._phase = name + self._samples[name] = [] + self._gc_pauses[name] = [] + try: + yield + finally: + with self._lock: + self._phase = None + + def summary(self, name: str) -> dict[str, Any]: + with self._lock: + samples = list(self._samples.get(name, ())) + pauses = list(self._gc_pauses.get(name, ())) + gc_summary = { + "gc_collections": len(pauses), + "gc_full_collections": sum(1 for gen, _ in pauses if gen >= 2), + "gc_max_pause_ms": max((pause for _, pause in pauses), default=0.0) * 1e3, + "gc_total_pause_ms": sum(pause for _, pause in pauses) * 1e3, + } + if not samples: + return { + "wakes": 0, + "max_late_ms": None, + "p99_late_ms": None, + "mean_late_ms": None, + "over_strict": 0, + "over_slack": 0, + **gc_summary, + } + ordered = sorted(samples) + p99_index = min(len(ordered) - 1, int(round(0.99 * (len(ordered) - 1)))) + return { + "wakes": len(samples), + "max_late_ms": ordered[-1] * 1e3, + "p99_late_ms": ordered[p99_index] * 1e3, + "mean_late_ms": sum(samples) / len(samples) * 1e3, + "over_strict": sum( + 1 for late in samples if late > PROBE_STRICT_LATENESS_SECONDS + ), + "over_slack": sum( + 1 + for late in samples + if late > WRITER_LEASE_HEARTBEAT_SCHEDULER_SLACK_SECONDS + ), + **gc_summary, + } + + +@dataclass(frozen=True) +class ProbePhaseResult: + key: str + kind: str + note: str + wall_seconds: float + cpu_seconds: float + bytes: int | None + lateness: dict[str, Any] + detail: dict[str, float] | None = None + + def as_json(self) -> dict[str, Any]: + return { + "key": self.key, + "kind": self.kind, + "note": self.note, + "wall_ms": round(self.wall_seconds * 1e3, 3), + "cpu_ms": round(self.cpu_seconds * 1e3, 3), + "bytes": self.bytes, + "lateness": { + key: (round(value, 3) if isinstance(value, float) else value) + for key, value in self.lateness.items() + }, + "detail_ms": ( + {key: round(value * 1e3, 3) for key, value in self.detail.items()} + if self.detail + else None + ), + } + + +class _ProbeRuntime: + """The slice of the coordinator runtime port prepare_payout_window uses.""" + + def __init__(self) -> None: + self.signing_seed_hex = "42" * 32 + self.ledger_attestation_signing_seed_hex = "43" * 32 + self.bundle_build_timeout_seconds = 600.0 + self._job_build_scheduler_lock = threading.Lock() + self.job_build_worker_counts = { + "starts": 0, + "restarts": 0, + "crashes": 0, + "terminations": 0, + } + self._job_build_worker_restart_pending = False + self._tip_refresh_metrics_lock = threading.Lock() + self.tip_refresh_worker_restarts = 0 + + def _ensure_job_cache_state(self) -> None: + return None + + +class _ProbeBuildControl: + """Placeholder build-control type; the probe never registers one.""" + + +def _count_bytes(chunks: Iterator[bytes | str]) -> int: + total = 0 + for chunk in chunks: + total += len(chunk) + return total + + +def _probe_candidate(shares: list[dict[str, object]]) -> dict[str, Any]: + """A durable block-candidate intent shaped like block_candidate_intent.""" + return { + "schema": "qbit.prism.block-candidate-intent.v1", + "block_hash_hex": "ab" * 32, + "block_hex": "00" * 256, + "coinbase_tx_hex": "01" * 128, + "parent_hash": "cd" * 32, + "expected_height": 800_001, + "template": { + "previousblockhash": "cd" * 32, + "height": 800_001, + "coinbasevalue": 50_00000000, + }, + "shares_json": shares, + "prior_balances": [], + "found_block": { + "block_height": 800_001, + "coinbase_value_sats": 50_00000000, + "network_difficulty": 226646186, + "anchor_job_issued_at_ms": FIXED_NOW_MS, + }, + "prospective_prior_balances": None, + "witness_merkle_leaves_hex": [], + "pending_share": {"share_id": "s", "accepted_at_ms": FIXED_NOW_MS}, + "username": "bench-miner-0", + } + + +def _instrument_transport( + compiler: BundleCompiler, + timers: dict[str, float], +) -> None: + """Time the compiler's transport helpers on this instance only.""" + for name in ( + "_serve_builder_write", + "_serve_builder_read_line", + "_serve_builder_read_exact", + ): + original = getattr(compiler, name) + + def wrapper( + *args: Any, + _original: Callable[..., Any] = original, + _name: str = name, + **kwargs: Any, + ) -> Any: + started = time.perf_counter() + try: + return _original(*args, **kwargs) + finally: + timers[_name] = timers.get(_name, 0.0) + ( + time.perf_counter() - started + ) + + setattr(compiler, name, wrapper) + + +def run_latency_probe(args: argparse.Namespace) -> dict[str, Any]: + daemon_binary = args.daemon_binary + if daemon_binary is not None: + daemon_path = Path(daemon_binary) + if not (daemon_path.is_file() and os.access(daemon_path, os.X_OK)): + raise SystemExit(f"--daemon-binary {daemon_binary} is not executable") + results: dict[str, Any] = { + "environment": describe_environment(), + "latency_probe": { + "monitor_interval_seconds": WRITER_LEASE_HEARTBEAT_MONITOR_SECONDS, + "scheduler_slack_seconds": WRITER_LEASE_HEARTBEAT_SCHEDULER_SLACK_SECONDS, + "strict_lateness_seconds": PROBE_STRICT_LATENESS_SECONDS, + "miners": args.probe_miners, + "share_id_shape": "production", + "daemon_binary": daemon_binary, + "rationale": LATENCY_PROBE_RATIONALE, + "sizes": [], + }, + } + probe = MonitorLatenessProbe(WRITER_LEASE_HEARTBEAT_MONITOR_SECONDS) + probe.start() + try: + for size in args.probe_sizes: + results["latency_probe"]["sizes"].append( + _probe_one_size( + size, + probe=probe, + miners=args.probe_miners, + daemon_binary=daemon_binary, + reps=args.probe_reps, + ) + ) + gc.collect() + finally: + probe.stop() + results["peak_rss_mb"] = _peak_rss_mb() + return results + + +def _probe_one_size( + size: int, + *, + probe: MonitorLatenessProbe, + miners: int, + daemon_binary: str | None, + reps: int, +) -> dict[str, Any]: + records = build_records(size, miners=miners, share_id_shape="production") + anchor = int(records[-1].job_issued_at_ms) + weight = sum(int(record.share_difficulty) for record in records) + state: dict[str, Any] = {} + phases: list[ProbePhaseResult] = [] + + def measure( + key: str, + kind: str, + note: str, + run: Callable[[], int | None], + *, + detail: dict[str, float] | None = None, + ) -> None: + best: ProbePhaseResult | None = None + for _ in range(max(1, reps)): + gc.collect() + with probe.phase(key): + cpu_started = time.process_time() + started = time.perf_counter() + byte_count = run() + wall = time.perf_counter() - started + cpu = time.process_time() - cpu_started + lateness = probe.summary(key) + result = ProbePhaseResult( + key=key, + kind=kind, + note=note, + wall_seconds=wall, + cpu_seconds=cpu, + bytes=byte_count, + lateness=lateness, + detail=dict(detail) if detail else None, + ) + # Keep the repetition with the worst lateness: the question is + # whether the monitor can be late, not how fast the best run was. + if best is None or ( + (result.lateness["max_late_ms"] or 0.0) + > (best.lateness["max_late_ms"] or 0.0) + ): + best = result + assert best is not None + phases.append(best) + + # ---- controls --------------------------------------------------------- + measure( + "idle_control", + "control", + "main thread sleeps; the monitor's own wake jitter on this host", + lambda: time.sleep(PROBE_CONTROL_SECONDS), + ) + + def busy_python() -> None: + deadline = time.perf_counter() + PROBE_CONTROL_SECONDS + total = 0 + while time.perf_counter() < deadline: + for value in range(10_000): + total += value + state["busy_total"] = total + + measure( + "busy_python_control", + "control", + "pure-Python loop; lateness is the interpreter's switch interval", + busy_python, + ) + + # ---- conversion and fold --------------------------------------------- + def convert() -> None: + state["shares"] = [record.to_prism_json() for record in records] + + measure( + "record_conversion", + "bounded", + "AcceptedShareRecord.to_prism_json per record (Python loop)", + convert, + ) + shares: list[dict[str, object]] = state["shares"] + + def fold() -> None: + state["window"] = IncrementalShareWindow.from_full_snapshot( + records, + anchor_job_issued_at_ms=anchor, + window_weight=weight, + ) + + measure( + "fold_in_process", + "bounded", + "IncrementalShareWindow.from_full_snapshot: the in-process fallback " + "fold, one bounded json.dumps per 512-record page", + fold, + ) + window: IncrementalShareWindow = state["window"] + sequence = window.json_records() + items = b",".join( + page.canonical_json_items for page in window.pages if page.canonical_json_items + ) + digest = sequence.canonical_json_sha256() + count = window.record_count + + def fold_again_and_release() -> None: + # A second fold whose result is dropped at once: the release of a + # whole window's pages (records, dicts, bytes) is what a full rescan + # or rotation pays when the previous window goes away. + second = IncrementalShareWindow.from_full_snapshot( + records, + anchor_job_issued_at_ms=anchor, + window_weight=weight, + ) + state["second_window"] = second + + measure( + "fold_in_process_keep", + "bounded", + "from_full_snapshot with the result retained (no release inside)", + fold_again_and_release, + ) + + def release_second_window() -> None: + del state["second_window"] + + measure( + "fold_release", + "residual", + "del of a whole in-process window (pages, records, dicts, bytes)", + release_second_window, + ) + + # ---- canonical encoding and digest ----------------------------------- + measure( + "canonical_encode_stream", + "bounded", + "iter_canonical_share_item_chunks over the share dicts", + lambda: _count_bytes(iter_canonical_share_item_chunks(shares)), + ) + measure( + "canonical_encode_whole", + "historical", + "json.dumps(shares, sort_keys=True, ...) in one call", + lambda: len( + json.dumps( + shares, sort_keys=True, separators=(",", ":"), default=str + ).encode("utf-8") + ), + ) + measure( + "array_digest_stream", + "bounded", + "canonical_share_array_sha256 over a plain list (legacy digest path)", + lambda: (canonical_share_array_sha256(shares), None)[1], + ) + measure( + "array_digest_whole", + "historical", + "sha256(json.dumps(shares, sort_keys=True, ...)) in one call", + lambda: ( + hashlib.sha256( + json.dumps( + shares, sort_keys=True, separators=(",", ":"), default=str + ).encode("utf-8") + ).hexdigest(), + None, + )[1], + ) + + # ---- compact payload and spool --------------------------------------- + measure( + "compact_tail_stream", + "bounded", + "_compact_share_tail_chunks: identities + compact rows in batches", + lambda: _count_bytes(iter(_compact_share_tail_chunks(shares))), + ) + + def compact_whole() -> int: + identities, compact_shares = _compact_share_payload(shares) + return len(json.dumps(identities, separators=(",", ":"))) + len( + json.dumps(compact_shares, separators=(",", ":")) + ) + + measure( + "compact_tail_whole", + "historical", + "_compact_share_payload + two whole json.dumps (old compact_fragments)", + compact_whole, + ) + + def spool_cold() -> int: + serialization = _ShareWindowSerialization( + key=(digest, count, weight), + share_count=count, + share_snapshot_sha256=digest, + ) + lease = serialization.acquire_spooled_tail(shares) + try: + return int(lease[1]) if lease is not None else None + finally: + serialization.retire_spool() + if lease is not None: + serialization.release_spooled_tail() + + measure( + "spool_write_stream", + "bounded", + "_ShareWindowSerialization.acquire_spooled_tail cold (encode + write)", + spool_cold, + ) + + # ---- daemon prepare request ------------------------------------------ + request_fields: dict[str, object] = { + "request": "prepare_window", + "mode": "full", + "append_invalidation_epoch": 0, + "anchor_job_issued_at_ms": anchor, + "records": shares, + "window_weight": weight, + "page_size": DEFAULT_INCREMENTAL_SHARE_WINDOW_PAGE_SIZE, + } + measure( + "prepare_request_stream", + "bounded", + "_iter_prepare_window_request_chunks (envelope + record batches)", + lambda: _count_bytes(_iter_prepare_window_request_chunks(request_fields)), + ) + measure( + "prepare_request_whole", + "historical", + "json.dumps(request_fields) in one call (old prepare_payout_window)", + lambda: len( + json.dumps(request_fields, separators=(",", ":")).encode("utf-8") + ), + ) + + # ---- mirror validation and lazy parse -------------------------------- + measure( + "mirror_validate", + "bounded", + "DaemonShareWindowMirror.from_full_items: digest + record walk", + lambda: ( + DaemonShareWindowMirror.from_full_items( + anchor_job_issued_at_ms=anchor, + window_weight=weight, + page_size=DEFAULT_INCREMENTAL_SHARE_WINDOW_PAGE_SIZE, + record_count=count, + canonical_items=items, + share_snapshot_sha256=digest, + ), + len(items), + )[1], + ) + def parse_stream() -> int: + state["parsed_stream"] = DaemonShareJsonSequence(items, count)._records() + return len(items) + + measure( + "mirror_parse_stream", + "bounded", + "DaemonShareJsonSequence._records: raw_decode per record, chunked decode", + parse_stream, + ) + + def release_parsed_stream() -> None: + # Dropping the parsed tuple frees every dict and string it holds in + # one refcount cascade -- a single C call, measured on its own. + del state["parsed_stream"] + + measure( + "mirror_parse_release", + "residual", + "del of the parsed record tuple (one deallocation cascade)", + release_parsed_stream, + ) + + def parse_whole() -> int: + state["parsed_whole"] = json.loads(b"[" + items + b"]") + return len(items) + + measure( + "mirror_parse_whole", + "historical", + 'json.loads(b"[" + items + b"]") in one call (old _records)', + parse_whole, + ) + + def release_parsed_whole() -> None: + del state["parsed_whole"] + + measure( + "mirror_parse_whole_release", + "residual", + "del of json.loads' record list (one deallocation cascade)", + release_parsed_whole, + ) + + # ---- durable candidate identity -------------------------------------- + candidate = _probe_candidate(shares) + measure( + "candidate_identity_stream", + "bounded", + "block_candidate_identity_sha256 (shares_json streamed)", + lambda: (block_candidate_identity_sha256(candidate), None)[1], + ) + measure( + "candidate_identity_whole", + "historical", + "sha256_json_hex(block_candidate_identity(candidate)) in one call", + lambda: (sha256_json_hex(block_candidate_identity(candidate)), None)[1], + ) + + # ---- one-shot canonical build input ---------------------------------- + payload = { + "found_block": candidate["found_block"], + "prior_balances": [], + "payout_policy": {"policy": "day-one"}, + "coinbase_script_sig_suffix_hex": "00", + "witness_merkle_leaves_hex": [], + "shares": shares, + } + measure( + "oneshot_payload_stream", + "bounded", + "iter_json_object_text_chunks(payload, array_keys=('shares',))", + lambda: _count_bytes( + iter_json_object_text_chunks(payload, array_keys=("shares",)) + ), + ) + measure( + "oneshot_payload_whole", + "historical", + "json.dumps(payload) in one call (the canonical build's share array)", + lambda: len(json.dumps(payload, separators=(",", ":"))), + ) + + # ---- real daemon round trip ------------------------------------------ + daemon: dict[str, Any] | None = None + if daemon_binary is not None: + daemon = _probe_daemon( + probe=probe, + measure=measure, + records=records, + shares=shares, + anchor=anchor, + weight=weight, + miners=miners, + daemon_binary=daemon_binary, + ) + + return { + "size": size, + "record_count": count, + "bytes": { + "canonical_items": len(items), + "compact_tail": sum( + len(chunk) for chunk in _compact_share_tail_chunks(shares) + ), + }, + "phases": [phase.as_json() for phase in phases], + "daemon": daemon, + } + + +def _probe_daemon( + *, + probe: MonitorLatenessProbe, + measure: Callable[..., None], + records: list[AcceptedShareRecord], + shares: list[dict[str, object]], + anchor: int, + weight: int, + miners: int, + daemon_binary: str, +) -> dict[str, Any]: + """The real --serve builder's prepare_window round trip, attributed.""" + runtime = _ProbeRuntime() + compiler = BundleCompiler( + runtime, # type: ignore[arg-type] + superseded_error=RuntimeError, + cancellation_error_types=(), + build_control_type=_ProbeBuildControl, + tool_command=lambda _name: [daemon_binary], + ) + timers: dict[str, float] = {} + _instrument_transport(compiler, timers) + outcomes: dict[str, Any] = {} + try: + # Spawn and handshake outside the measured phases, with a + # one-record window that does not collide with the real one. + warm = compiler.prepare_payout_window( + mode="full", + records_json=[records[0].to_prism_json()], + anchor_job_issued_at_ms=anchor, + append_invalidation_epoch=0, + window_weight=int(records[0].share_difficulty), + page_size=DEFAULT_INCREMENTAL_SHARE_WINDOW_PAGE_SIZE, + ) + outcomes["warm_status"] = getattr(warm, "status", None) + timers.clear() + + def full() -> int: + outcome = compiler.prepare_payout_window( + mode="full", + records_json=shares, + anchor_job_issued_at_ms=anchor, + append_invalidation_epoch=0, + window_weight=weight, + page_size=DEFAULT_INCREMENTAL_SHARE_WINDOW_PAGE_SIZE, + ) + outcomes["full"] = outcome + return len(outcome.window_items or b"") if outcome is not None else None + + measure( + "daemon_prepare_full", + "daemon", + "BundleCompiler.prepare_payout_window(mode='full'): streamed " + "request, Rust fold, raw response", + full, + detail=timers, + ) + full_detail = dict(timers) + timers.clear() + full_outcome = outcomes.get("full") + status = getattr(full_outcome, "status", None) + mirror_bytes = None + if status == "prepared": + measure( + "daemon_mirror_validate", + "bounded", + "DaemonShareWindowMirror.from_full_items on the daemon's bytes", + lambda: len( + DaemonShareWindowMirror.from_full_items( + anchor_job_issued_at_ms=anchor, + window_weight=weight, + page_size=DEFAULT_INCREMENTAL_SHARE_WINDOW_PAGE_SIZE, + record_count=full_outcome.record_count, + canonical_items=full_outcome.window_items or b"", + share_snapshot_sha256=full_outcome.share_snapshot_sha256 or "", + ).canonical_items + ), + ) + mirror_bytes = len(full_outcome.window_items or b"") + programs = benchmark_miner_programs(miners) + last = records[-1] + delta = [] + for index in range(PROBE_ADVANCE_RECORDS): + seq = int(last.share_seq) + index + 1 + miner = seq % miners + delta.append( + AcceptedShareRecord( + share_seq=seq, + share_id=f"bench-miner-{miner}.rig{seq % 512}:{'ab' * 32}", + miner_id=f"bench-miner-{miner}", + order_key=f"bench-miner-{miner:06d}", + p2mr_program_hex=programs[miner], + share_difficulty=int(last.share_difficulty), + network_difficulty=int(last.network_difficulty), + template_height=int(last.template_height), + job_id=f"bench-job-{seq}", + job_issued_at_ms=int(last.job_issued_at_ms) + index + 1, + accepted_at_ms=int(last.accepted_at_ms) + index + 1, + ntime=int(last.ntime) + index + 1, + ).to_prism_json() + ) + new_anchor = int(last.job_issued_at_ms) + PROBE_ADVANCE_RECORDS + 1 + + def advance() -> int: + outcome = compiler.prepare_payout_window( + mode="advance", + records_json=delta, + anchor_job_issued_at_ms=new_anchor, + append_invalidation_epoch=0, + base_digest=full_outcome.share_snapshot_sha256, + ) + outcomes["advance"] = outcome + return len(outcome.appended_items) if outcome is not None else None + + measure( + "daemon_prepare_advance", + "daemon", + f"prepare_payout_window(mode='advance') with " + f"{PROBE_ADVANCE_RECORDS} appended records", + advance, + detail=timers, + ) + finally: + compiler.shutdown_serve_builder() + return { + "binary": daemon_binary, + "warm_status": outcomes.get("warm_status"), + "full_status": getattr(outcomes.get("full"), "status", None), + "full_error": getattr(outcomes.get("full"), "error", None), + "full_transport_ms": { + key: round(value * 1e3, 3) for key, value in full_detail.items() + }, + "advance_status": getattr(outcomes.get("advance"), "status", None), + "advance_stats": ( + { + "added_rows": outcomes["advance"].added_rows, + "expired_rows": outcomes["advance"].expired_rows, + "touched_pages": outcomes["advance"].touched_pages, + } + if getattr(outcomes.get("advance"), "status", None) == "prepared" + else None + ), + "mirror_bytes": mirror_bytes, + "worker_counts": dict(runtime.job_build_worker_counts), + "serve_counts": dict(compiler.serve_builder_counts), + } + + +def render_latency_probe(results: dict[str, Any]) -> str: + out: list[str] = [] + env = results["environment"] + probe = results["latency_probe"] + out.append("=" * 78) + out.append("#236 monitor-lateness probe: PRISM payout-window serialization phases") + out.append("=" * 78) + out.append("") + out.append(f"CPU {env['cpu']} x{env['cpu_count']} logical") + out.append(f"Platform {env['platform']}") + out.append(f"Python {env['python_version'].splitlines()[0]}") + load = env["loadavg_1_5_15"] + out.append( + "Load at start " + + ("n/a" if load is None else " / ".join(f"{v:.2f}" for v in load)) + ) + out.append( + f"Monitor cadence {probe['monitor_interval_seconds'] * 1e3:.0f} ms " + f"(WRITER_LEASE_HEARTBEAT_MONITOR_SECONDS); scheduler slack " + f"{probe['scheduler_slack_seconds'] * 1e3:.0f} ms; strict target " + f"{probe['strict_lateness_seconds'] * 1e3:.0f} ms" + ) + out.append( + f"Fixture production share_id shape, {probe['miners']} identities" + ) + out.append( + "Daemon " + + (probe["daemon_binary"] or "not measured (pass --daemon-binary)") + ) + out.append("") + out.append("Metric: per phase, wall time of the phase in the main thread and the") + out.append("lateness (wake - due) of a monitor thread sleeping on the lease") + out.append("monitor's cadence. 'historical' rows are the whole-window calls the") + out.append("'bounded' rows replaced; both run the same input in the same process.") + out.append("'gc full' counts generation-2 cyclic collections during the phase and") + out.append("'gc max' is the longest single collection pause (ms), from gc.callbacks.") + out.append("Lateness is what the monitor thread observed on this host, whatever the") + out.append("cause (a GIL-held C call, a GC pause, or other load on a shared host).") + out.append("") + for entry in probe["sizes"]: + out.append("=" * 78) + out.append( + f"WINDOW SIZE: {entry['size']:,} shares " + f"({entry['record_count']:,} retained)" + ) + out.append("=" * 78) + out.append( + f" canonical items {entry['bytes']['canonical_items'] / 1e6:.1f} MB, " + f"compact tail {entry['bytes']['compact_tail'] / 1e6:.1f} MB" + ) + out.append("") + head = ( + f" {'phase':<28}{'kind':<11}{'wall ms':>9}{'cpu ms':>9}{'MB':>8}" + f"{'wakes':>7}{'max late':>10}{'p99 late':>10}{'>250ms':>8}{'>500ms':>8}" + f"{'gc full':>8}{'gc max':>8}" + ) + out.append(head) + out.append(" " + "-" * (len(head) - 2)) + for phase in entry["phases"]: + late = phase["lateness"] + out.append( + f" {phase['key']:<28}{phase['kind']:<11}" + f"{phase['wall_ms']:>9.1f}{phase['cpu_ms']:>9.1f}" + f"{(phase['bytes'] or 0) / 1e6 if phase['bytes'] else 0:>8.1f}" + f"{late['wakes']:>7}" + f"{_fmt(late['max_late_ms'], 1):>10}" + f"{_fmt(late['p99_late_ms'], 1):>10}" + f"{late['over_strict']:>8}{late['over_slack']:>8}" + f"{late.get('gc_full_collections', 0):>8}" + f"{_fmt(late.get('gc_max_pause_ms'), 1):>8}" + ) + if phase.get("detail_ms"): + detail = ", ".join( + f"{key.removeprefix('_serve_builder_')}={value:.1f} ms" + for key, value in phase["detail_ms"].items() + ) + out.append(f" {'':<28}transport: {detail}") + out.append("") + daemon = entry.get("daemon") + if daemon: + out.append( + f" daemon: full={daemon['full_status']} advance={daemon['advance_status']}" + f" mirror_bytes={daemon['mirror_bytes']}" + f" advance_stats={daemon['advance_stats']}" + f" worker_counts={daemon['worker_counts']}" + ) + if daemon.get("full_error"): + out.append(f" daemon error: {daemon['full_error']}") + out.append("") + peak = results.get("peak_rss_mb") + if peak: + out.append(f"peak RSS: {peak:.0f} MB") + return "\n".join(out) + + +# --------------------------------------------------------------------------- +# CLI +# --------------------------------------------------------------------------- + + +def _int_list(text: str) -> list[int]: + return [int(part) for part in text.replace(",", " ").split()] + + +def main(argv: list[str] | None = None) -> int: + parser = argparse.ArgumentParser( + description=__doc__, + formatter_class=argparse.RawDescriptionHelpFormatter, + ) + parser.add_argument("--json", action="store_true", help="emit results as JSON") + parser.add_argument( + "--render", + metavar="PATH", + default=None, + help=( + "render the tables from a previously saved --json file instead of " + "measuring. Running the sweep twice to get both output formats " + "would put two 8-thread processes on the same cores and corrupt " + "both, so capture --json once and render from it." + ), + ) + parser.add_argument( + "--sizes", + type=_int_list, + default=list(DEFAULT_SIZES), + help="window sizes in shares (default: 21868,100000,400000)", + ) + parser.add_argument( + "--threads", + type=_int_list, + default=list(DEFAULT_THREADS), + help="thread counts to sweep (default: 1,2,4,8)", + ) + parser.add_argument( + "--large-size-threads", + type=_int_list, + default=list(LARGE_SIZE_THREADS), + help=( + f"thread counts used for sizes above {LARGE_SIZE_BYTES_THRESHOLD:,} " + f"shares, where 8 independent inputs cost ~0.7 GB each " + f"(default: 1,8)" + ), + ) + parser.add_argument("--reps", type=int, default=3, help="repetitions (default 3)") + parser.add_argument( + "--min-wall-seconds", + type=float, + default=1.0, + help=( + "wall-clock floor per configuration; below this, thread start/join " + "biases cores-used downward (default 1.0)" + ), + ) + parser.add_argument( + "--max-wall-seconds", + type=float, + default=12.0, + help=( + "bound on a single configuration's wall clock; a fully GIL-held " + "stage serializes, so its wall grows with thread count (default 12)" + ), + ) + parser.add_argument( + "--control-floor-fraction", + type=float, + default=0.75, + help=( + "the positive control must reach this fraction of the highest " + "thread count in cores-used, else the run aborts without " + "reporting stages (default 0.75, i.e. 6.0 cores on an 8-thread " + "sweep; #143 published 0.986)" + ), + ) + parser.add_argument( + "--miners", + type=int, + default=BENCHMARK_MINERS, + help=f"distinct payout identities (default {BENCHMARK_MINERS})", + ) + parser.add_argument( + "--share-id-shape", + choices=("benchmark", "production"), + default="benchmark", + help=( + "benchmark: job_build_benchmark's short synthetic share_id; " + "production: the username:block_hash_hex form share_writer builds" + ), + ) + parser.add_argument( + "--latency-probe", + action="store_true", + help=( + "#236 mode: run each serialization phase once per size in the " + "main thread while a monitor thread on the lease monitor's " + "cadence records its wake lateness; see LATENCY_PROBE_RATIONALE" + ), + ) + parser.add_argument( + "--probe-sizes", + type=_int_list, + default=list(PROBE_DEFAULT_SIZES), + help="--latency-probe window sizes in shares (default: 210000,400000)", + ) + parser.add_argument( + "--probe-reps", + type=int, + default=1, + help=( + "--latency-probe repetitions per phase; the repetition with the " + "worst monitor lateness is reported (default 1)" + ), + ) + parser.add_argument( + "--probe-miners", + type=int, + default=PROBE_DEFAULT_MINERS, + help=f"--latency-probe distinct payout identities (default {PROBE_DEFAULT_MINERS})", + ) + parser.add_argument( + "--daemon-binary", + default=None, + help=( + "--latency-probe: path to a prebuilt qbit-prism-build-audit-bundle; " + "adds the real --serve daemon's prepare round trip" + ), + ) + args = parser.parse_args(argv) + + if args.render: + with open(args.render, encoding="utf-8") as handle: + loaded = json.load(handle) + sys.stdout.write( + render_latency_probe(loaded) + if "latency_probe" in loaded + else render_report(loaded) + ) + sys.stdout.write("\n") + return 0 + + if args.latency_probe: + results = run_latency_probe(args) + if args.json: + json.dump(results, sys.stdout, indent=2, default=str) + else: + sys.stdout.write(render_latency_probe(results)) + sys.stdout.write("\n") + return 0 + + results = run(args) + if args.json: + json.dump(results, sys.stdout, indent=2, default=str) + sys.stdout.write("\n") + else: + sys.stdout.write(render_report(results)) + sys.stdout.write("\n") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tests/prism_candidate_storm.py b/tests/prism_candidate_storm.py new file mode 100644 index 000000000..2de773d60 --- /dev/null +++ b/tests/prism_candidate_storm.py @@ -0,0 +1,2508 @@ +#!/usr/bin/env python3 +"""Candidate-storm characterization instrument for PRISM. + +The 2026-08-20 testnet4 incident produced 3,120 durable block candidates, +well above both the 32-entry live queue and the 1,024-row replay-enumeration +page. This instrument reproduces that cardinality against the shipped +in-memory ledger and coordinator paths, then reports the component counts on +both sides of a process restart. + +The ownership counts are deliberately first-class output. Live admission +marks a candidate outstanding *before* attempting the bounded queue, so the +marker covers coalesced wakeups as well as queued ones. Restart adoption +marks every restored row replay-inflight before it is processed. Those +markers mean "this process still owns eventual disposition", not "qbitd has +already been offered this candidate". Any maintenance selector that treats +either whole set as active node-offer evidence will exclude the whole storm. +A wakeup parked in the retry holder is the same kind of marker for the same +reason: ``_retain_block_candidate_for_retry`` also runs when the fast-lane +reservation declines, before any submitblock. So is a claimed disposition +lease: ``submit_next`` takes it on the dequeued hash *before* the terminal, +capacity and fast-lane checks, each of which releases it again having +offered nothing. So, less obviously, is a recorded terminal outcome: when +pool capacity is already closed the deferred split builds a node submission +with ``attempted=False`` and lets accounting terminalize the durable row +without calling qbitd once, and the durable ``attempt_count`` is marked on +that same zero-RPC path. The rig therefore reports the attested offers +(``node_offer_evidence``) apart from the conservative must-not-abandon union +(``selector_evidence``) that a selector owes the #183 contract, so neither a +retry holder, nor a held lease, nor a terminal seal is ever mistaken for a +node call. + +This is a component instrument, not a performance threshold. It uses the +real coordinator admission, durable codec, pagination, payout barriers, and +``SingleWriterShareLedger``; it does not include PostgreSQL wait or qbitd. +Issues #187 and #185 can extend the same rig with wall-clock restart/drain +assertions and component-memory gauges without rebuilding the storm fixture. + +The decided-height extension models the instant after the storm height is +settled: ``decide_height`` makes one candidate the active block at that +height (``DecidedHeightRpc``), ``ownership`` reads every candidate's +ownership and node-offer-evidence facts from the shipped state so a +maintenance selector can be evaluated against the real population, +``perturb`` installs one selector-evidence shape at a time through the +shipped entry points, and ``drain_per_row`` drives the shipped per-row +disposition path in the production split -- ``submit_next`` with +``defer_accounting=True`` handing every node offer to the real accounting +queue and task runner -- reporting the exact hash sets it terminalized and +what they cost. The per-row drain is the oracle any set-oriented selector +has to agree with; the instrument does not contain a selector itself, and +suppresses the shipped #183 collapse for the duration of the drain so the +oracle stays the per-row path even where the drain re-enumerates. + +Driving both lanes from one thread is what makes the drain deterministic, +and it costs nothing in fidelity: the two lanes are already serialized per +hash by the disposition lease, and the drain stamps the accounting-thread +identity the shipped loop stamps, so retry pacing records a not-before +deadline exactly as it does on the real accounting thread. Splitting the +drive this way also matters for correctness, not just fidelity: +``submit_next`` claims the disposition lease *blocking* unless accounting +is deferred, and ``_restore_replayed_candidate_acceptance_evidence`` runs +only inside the accounting task. + +The cleanup-backlog extension (issue #198) measures the one collapse state +with no durable replay source: the in-memory retry record a won row leaves +behind when its post-write cleanup fails. ``CleanupFaultInjector`` breaks +exactly one cleanup dependency (``CLEANUP_FAULTS``) at the shipped seam +while counting every seam per hash, and ``measure_cleanup_backlog`` drives +the shipped collapse walk under that fault, reports the backlog it produces +-- depth, oldest age, retained pending-share holders, terminal-outcome pins, +deep and traced bytes per record, and the rows the admission bound +preserved -- holds the fault for a bounded run of failing retry passes, +heals it, and then proves recovery: the backlog drains through the shipped +retry pass at a measured throughput, admission resumes, every collapsed +hash runs each cleanup seam exactly once more (no double cleanup), no +terminal row is replay-adopted or re-offered to the node, and the decided +winner keeps its pending-share floor authority. +""" + +from __future__ import annotations + +import argparse +import gc +from collections.abc import Mapping +import json +import linecache +import os +import platform +import queue +import random +import subprocess +import sys +import threading +import time +import tracemalloc +from contextlib import ExitStack, nullcontext, redirect_stderr, redirect_stdout +from dataclasses import asdict, dataclass +from io import StringIO +from pathlib import Path +from types import SimpleNamespace +from typing import Any, ContextManager, Iterable, Sequence + +if not __package__: + sys.path.insert(0, str(Path(__file__).resolve().parents[1])) + +from lab.prism.block_candidates import ( # noqa: E402 + BLOCK_CANDIDATE_COLLAPSE_CLEANUP_PAYOUT_STEP, + BLOCK_CANDIDATE_COLLAPSE_CLEANUP_STEPS, + COLLAPSE_DIFFICULTY_SCALE, + COLLAPSE_POW_LIMIT_BITS, + MAX_BLOCK_REPLAY_ENUMERATION_ROWS, + MAX_PENDING_BLOCK_CANDIDATES, + _BlockCandidateNodeSubmission, +) +from lab.prism.coordinator_config import ( # noqa: E402 + DEFAULT_BLOCK_CANDIDATE_CLEANUP_RETRY_BACKLOG_MAX, +) +from lab.prism.process_telemetry import ( # noqa: E402 + MallocTrimmer, + ProcessHeapTelemetry, + read_anonymous_map_shape, + read_malloc_arena_count, + read_malloc_info_summary, + read_resident_memory_bytes, +) +from lab.prism.share_ledger import ( # noqa: E402 + PendingShare, + SingleWriterShareLedger, +) +from lab.prism.template_artifacts import scaled_network_difficulty # noqa: E402 +from tests.prism_vardiff_test_support import ( # noqa: E402 + FakeRpc, + block_candidate, + submit_coordinator, +) + + +OBSERVED_TESTNET_CANDIDATE_STORM = 3_120 +STORM_PARENT_HASH = "11" * 32 + +# The selector-evidence shapes record_abandoned and submit_next consult, +# each installable on one seeded candidate through CandidateStormRig.perturb. +PERTURBATIONS = ( + "lease_held", + "retained_success", + "tip_observed", + "retry_slot", + "prepared_pool_block", + "terminal", +) + +# Issue #198: every in-memory dependency one won row's terminal cleanup runs +# through, each injectable as a fault through CleanupFaultInjector. The +# first seven are the shipped cleanup steps, named exactly as the retry +# registry names what a hash still owes; ``floor-index`` is the page-scope +# pending-share holder scan that precedes them, whose failure aborts the +# whole apply's cleanup and is the path the retry re-indexes for. +CLEANUP_FAULTS = tuple(BLOCK_CANDIDATE_COLLAPSE_CLEANUP_STEPS) + ("floor-index",) +# Install the seam counters without breaking anything: the clean baseline +# every marginal per-record figure is taken against. +CLEANUP_FAULT_NONE = "none" +# seam -> (owner, attribute, successful calls one collapsed hash is owed). +# The payout withdrawal is two calls (invalidate, then drop the tombstone); +# the floor release is once per retained holder and the holder index once +# per page, so neither has a per-hash expectation. +_CLEANUP_SEAMS: dict[str, tuple[str, str, int | None]] = { + BLOCK_CANDIDATE_COLLAPSE_CLEANUP_PAYOUT_STEP: ( + "server", + "_clear_accepted_block_payout_preview", + 2, + ), + "finalize-retry": ("service", "finalize_retries", 1), + "retry-state": ("server", "_clear_block_candidate_retry_state", 1), + "outstanding-and-tip-observation": ( + "server", + "_discard_outstanding_block_candidate", + 1, + ), + "terminal-outcome": ("server", "_record_block_candidate_terminal_outcome", 1), + "abandonment-accounting": ( + "server", + "_record_committed_block_candidate_abandonment", + 1, + ), + "pending-share-floor": ("server", "_finish_pending_share_commit", None), + "floor-index": ("service", "_collapsed_candidate_floor_holders", None), +} +assert set(_CLEANUP_SEAMS) == set(CLEANUP_FAULTS) + + +@dataclass(frozen=True) +class CandidateStormSnapshot: + """Bounded component gauges after one phase of the storm.""" + + durable_pending: int + live_queue_capacity: int + live_queued: int + live_coalesced: int + outstanding_marked: int + outstanding_covers_all_durable: bool + replay_queued: int + replay_inflight_marked: int + replay_inflight_covers_all_durable: bool + accepted_parent_previews: int + replay_enumeration_owed: bool + # Candidates the #183 must-not-abandon contract covers, i.e. carrying + # any evidence that a node offer happened, is in flight, or cannot be + # ruled out (see CandidateOwnership.selector_evidence). The + # outstanding and replay-inflight markers above are deliberately NOT + # part of it. + selector_evidence_marked: int = 0 + # How that union splits: rows something actually attests a node offer + # for, and the three widenings that attest nothing -- a wakeup parked in + # a retry holder, a claimed disposition lease, and a recorded terminal + # outcome. None is by itself evidence qbitd was ever called; production + # reaches all three without a submitblock. They can coincide on one row, + # so they are counted separately rather than folded into one residue. + node_offer_evidence_marked: int = 0 + unoffered_retry_marked: int = 0 + unoffered_lease_marked: int = 0 + unoffered_terminal_marked: int = 0 + + +@dataclass(frozen=True) +class CandidateOwnership: + """Ownership and evidence facts for one durable candidate at one instant. + + Every field is read from the shipped coordinator/ledger state, so a + maintenance selector can be evaluated against the real population + instead of a hand-built corpus. ``outstanding`` and ``replay_inflight`` + are process-ownership markers; the fields folded into + ``node_offer_evidence`` are the ones that mean a node offer happened, + is happening, or left acceptance evidence behind. ``retry_held``, + ``disposition_held`` and ``terminal_outcome`` are none of those: they + are a third category, covered by the conservative ``selector_evidence`` + union without attesting an offer. + """ + + block_hash: str + expected_height: int + parent_hash: str + network_difficulty: float | None + outbox_state: str | None + attempt_count: int + live_queued: bool + replay_queued: bool + outstanding: bool + replay_inflight: bool + preview_barrier: bool + disposition_held: bool + retry_held: bool + node_acceptance_retained: bool + tip_observed: bool + accounted_accepted: bool + terminal_outcome: bool | None + pool_block_chain_state: str | None + + @property + def node_offer_evidence(self) -> bool: + """Whether anything attests that qbitd was offered this hash. + + Every fact here is written by the offer or downstream of it: a + retained submission records an offer that already returned, a tip + observation and an accounted acceptance are readings of an accepted + offer, and a pool-block row is what an accepted offer's payout + preparation left behind. ``retry_held``, ``disposition_held`` and + ``terminal_outcome`` are deliberately absent; see + :attr:`retry_held_without_offer`, :attr:`lease_held_without_offer` + and :attr:`terminal_without_offer`. + """ + return bool( + self.node_acceptance_retained + or self.tip_observed + or self.accounted_accepted + or self.pool_block_chain_state is not None + ) + + @property + def retry_held_without_offer(self) -> bool: + """A wakeup parked in a retry holder with no offer attested for it. + + ``_retain_block_candidate_for_retry`` also runs on paths that never + reached qbitd -- most plainly when ``_reserve_block_fast_lane_slot`` + declines and ``submit_next`` parks the candidate *before* any + submitblock. Retry retention on its own therefore says only "this + process still owes this row a disposition", exactly like + ``outstanding`` and ``replay_inflight``, and reading it as an offer + would credit the rig with a node call that never happened. + """ + return bool(self.retry_held) and not self.node_offer_evidence + + @property + def lease_held_without_offer(self) -> bool: + """A claimed disposition lease with no offer attested for it. + + ``submit_next`` claims the lease on the dequeued hash *before* it + consults the terminal outcome, before the accounted/capacity close, + and before ``_reserve_block_fast_lane_slot``. Each of those paths + releases the lease again without ever reaching submitblock, and the + deferred split hands the still-held lease to accounting even when the + node submission is ``attempted=False``. The registry this field is + read from is weaker still: ``_claim_block_candidate_disposition`` + registers a user on the flight *before* it takes the flight's lock, + so the key is present for a blocked waiter as well as for the holder. + A held lease therefore says only "some pass owns this row's eventual + disposition", exactly like ``outstanding`` and ``replay_inflight``, + and reading it as an offer would credit the rig with a node call that + never happened. + """ + return bool(self.disposition_held) and not self.node_offer_evidence + + @property + def terminal_without_offer(self) -> bool: + """A recorded terminal outcome with no offer attested for it. + + ``_record_block_candidate_terminal_outcome`` means "this process + finished deciding this row", not "this process offered it". The + deferred submitter split reaches it having called nothing: when pool + capacity is already closed -- ``accepted_block_count`` has reached + ``max_blocks``, or ``stop_after_block`` is set and one block landed + -- ``submit_next`` builds a ``_BlockCandidateNodeSubmission( + attempted=False)`` and hands it straight to accounting, which + terminalizes the durable outbox row without a single ``submitblock`` + (the already-accounted and finalize-only replay paths do the same). + + ``attempt_count`` is no substitute for the missing RPC either: + ``submit_writer`` calls ``_mark_block_candidate_attempted`` *before* + the offer branch, and the ledger documents the mark as "admission to + a real processing phase", so the durable row records an attempt on + exactly the same zero-RPC path. + + Excluding the terminal registry from :attr:`node_offer_evidence` + costs that gauge no attestation it legitimately had: finalization + adds an accepted hash to ``_accounted_accepted_block_hashes`` and + persists its pool-block row before the terminal outcome is recorded, + so a candidate that really was offered and accepted still attests it + in its own right. + """ + return self.terminal_outcome is not None and not self.node_offer_evidence + + @property + def selector_evidence(self) -> bool: + """The conservative must-not-abandon union the #183 contract needs. + + This is :attr:`node_offer_evidence` widened by ``retry_held``, by + ``disposition_held``, and by a recorded ``terminal_outcome``. Each + widening covers an ambiguity rather than an attestation: a parked + wakeup may equally have come from a path that *did* offer and is + retrying its tail, a claimed lease may equally span an offer already + in flight, and a terminal outcome may equally be the seal on an offer + that returned. Nothing the selector can read tells the two apart, so + it has to leave the row to the per-row path either way -- and a row + whose lease it cannot claim, or whose disposition this process has + already fixed, is not one it could dispose of anyway. Because the + widening is deliberately conservative, a true value here is not a + claim that qbitd was offered the hash -- read + :attr:`node_offer_evidence` for that, and + :attr:`retry_held_without_offer` / :attr:`lease_held_without_offer` / + :attr:`terminal_without_offer` for the rows the widenings add. + """ + return ( + self.node_offer_evidence + or bool(self.retry_held) + or bool(self.disposition_held) + or self.terminal_outcome is not None + ) + + +@dataclass(frozen=True) +class PerRowDrainReport: + """What the shipped per-row disposition path did, and cost, over a storm. + + Every field reports *one* invocation of :meth:`CandidateStormRig.drain_per_row`. + ``abandoned_hashes`` is the oracle a set-oriented selector has to agree + with: the exact set of durable rows that call terminalized as abandoned, + not a count, and not every abandoned row standing in the rig. It and + ``submitted_hashes`` are transitions -- a row already terminal when the + call started is left out -- so successive bounded drains (``max_rounds``) + partition the work between them instead of each claiming the whole + history. ``pending_hashes`` is the complementary residue: the rows still + awaiting disposition when the call returned, which is a state rather than + a transition because nothing ever transitions *into* pending. + + ``deferred_hashes`` and ``lease_blocked_hashes`` are the oracle's + counterweight -- rows the drain reached and the shipped path deliberately + refused to terminalize -- so a selector that claims one of them can be + caught rather than silently believed. + + ``rounds`` is the number of node offers the call actually performed -- + the ``submitblock`` delta in the rig's own RPC log -- not the number of + wakeups it consumed. ``submit_next_block_candidate`` returns True on + several paths that never reach qbitd: a disposition lease held elsewhere + parks the wakeup in ``_block_disposition_waiting_retries``, a same-hash + duplicate whose disposition already landed is dropped, a declined + fast-lane reservation retains the candidate for retry, and the + capacity-closed split hands accounting an ``attempted=False`` submission + that terminalizes the durable row without calling qbitd once. Counting + those would make ``rounds`` disagree with this report's own ``rpc_calls``, + ``accounting_tasks`` and ``ledger_attempt_marks``, and would let a + ``max_rounds`` budget be spent without a single offer being made. + """ + + rounds: int + accounting_tasks: int + replay_enumerations: int + abandoned_hashes: frozenset[str] + submitted_hashes: frozenset[str] + pending_hashes: frozenset[str] + withheld_hashes: frozenset[str] + lease_blocked_hashes: frozenset[str] + deferred_hashes: frozenset[str] + rpc_calls: dict[str, int] + ledger_attempt_marks: int + wall_seconds: float + + @property + def abandoned_rows(self) -> int: + return len(self.abandoned_hashes) + + @property + def submitted_rows(self) -> int: + return len(self.submitted_hashes) + + @property + def pending_rows(self) -> int: + return len(self.pending_hashes) + + +class _FaultableRegistry(dict): + """``finalize_retries`` with a breakable ``pop``. + + The finalize-retry cleanup step reaches no coordinator seam -- it pops + the service's own registry under the lock -- so the fault has to live + on the container. Membership reads stay plain dict reads. + """ + + def __init__(self, injector: "CleanupFaultInjector", seam: str, contents: Any) -> None: + super().__init__(contents) + self._injector = injector + self._seam = seam + + def pop(self, key: Any, *default: Any) -> Any: # type: ignore[override] + self._injector._guard(self._seam) + result = super().pop(key, *default) + self._injector._count(self._seam, str(key).lower()) + return result + + +class CleanupFaultInjector: + """Break one cleanup dependency until healed; count every seam meanwhile. + + Every seam the terminal cleanup runs through is wrapped so a *successful* + call is counted per hash (per holder for the floor release, per page for + the holder index), and exactly one of them -- ``fault`` -- raises while + ``on`` is true. Nothing else is replaced: the collapse, the fence, the + retry registry and the retry pass all stay as shipped, so the backlog + this produces is the backlog production would produce under the same + fault. ``CLEANUP_FAULT_NONE`` installs the counters alone, which is the + clean baseline the marginal memory figure is taken against. + + The counters are what make recovery provable rather than believed: after + healing, every collapsed hash must show exactly the calls one terminal + cleanup is owed at every seam -- one more than that is a double cleanup, + one fewer is a step the retry never finished -- and a floor holder must + be released at most once by identity. + """ + + def __init__(self, server: Any, fault: str) -> None: + if fault != CLEANUP_FAULT_NONE and fault not in CLEANUP_FAULTS: + raise ValueError(f"unknown cleanup fault: {fault}") + self.server = server + self.service = server._ensure_block_candidate_service() + self.fault = fault + self.on = fault != CLEANUP_FAULT_NONE + self.failed_calls = 0 + # seam -> key -> successful calls. + self.calls: dict[str, dict[str, int]] = {seam: {} for seam in _CLEANUP_SEAMS} + # Floor releases by holder identity. + self.floor_releases: dict[int, int] = {} + self._originals: dict[str, tuple[Any, str, Any]] = {} + + def install(self) -> None: + if self._originals: + raise RuntimeError("cleanup faults are already installed") + for seam, (owner_name, attribute, _expected) in _CLEANUP_SEAMS.items(): + owner = self.server if owner_name == "server" else self.service + original = getattr(owner, attribute) + self._originals[seam] = (owner, attribute, original) + if seam == "finalize-retry": + setattr(owner, attribute, _FaultableRegistry(self, seam, original)) + continue + setattr(owner, attribute, self._wrap(seam, original)) + + def uninstall(self) -> None: + for seam, (owner, attribute, original) in self._originals.items(): + if seam == "finalize-retry": + # Hand the live contents back to the shipped container. + current = getattr(owner, attribute) + original.clear() + original.update(current) + setattr(owner, attribute, original) + self._originals.clear() + + def heal(self) -> None: + self.on = False + + def _wrap(self, seam: str, original: Any) -> Any: + def wrapped(*args: Any, **kwargs: Any) -> Any: + self._guard(seam) + result = original(*args, **kwargs) + self._count(seam, self._key(seam, args, kwargs)) + return result + + return wrapped + + def _guard(self, seam: str) -> None: + if self.on and seam == self.fault: + self.failed_calls += 1 + raise RuntimeError(f"injected cleanup fault: {seam}") + + def _key(self, seam: str, args: tuple[Any, ...], kwargs: dict[str, Any]) -> str: + if seam == "pending-share-floor": + share = args[0] if args else kwargs["pending_share"] + self.floor_releases[id(share)] = self.floor_releases.get(id(share), 0) + 1 + # The rig stamps ``share_id`` as ``miner:``. + return str(getattr(share, "share_id", "")).rsplit(":", 1)[-1].lower() + if seam == "floor-index": + return "page" + value = args[0] if args else kwargs.get("block_hash", "") + return str(value).lower() + + def _count(self, seam: str, key: str) -> None: + counts = self.calls[seam] + counts[key] = counts.get(key, 0) + 1 + + def snapshot_calls(self) -> dict[str, dict[str, int]]: + return {seam: dict(counts) for seam, counts in self.calls.items()} + + def _expected(self, seam: str, expected: int) -> int: + # An aborted apply -- the holder index raising -- has already run the + # payout withdrawal for the whole page before it stops, and the + # retry then re-runs every step because the abort proved nothing + # about which ones completed. That documented repeat of an + # idempotent step is the shipped abort contract, not a double + # cleanup, so the expectation carries it. + if self.fault == "floor-index" and seam == BLOCK_CANDIDATE_COLLAPSE_CLEANUP_PAYOUT_STEP: + return 2 * expected + return expected + + def excess_calls(self, hashes: Iterable[str]) -> dict[str, int]: + """Per seam, how many of ``hashes`` were cleaned up more than once.""" + wanted = {str(value).lower() for value in hashes} + return { + seam: sum( + 1 + for key in wanted + if self.calls[seam].get(key, 0) > self._expected(seam, expected) + ) + for seam, (_owner, _attribute, expected) in _CLEANUP_SEAMS.items() + if expected is not None + } + + def calls_since( + self, + earlier: dict[str, dict[str, int]], + hashes: Iterable[str], + ) -> int: + """Successful seam calls for ``hashes`` made after ``earlier`` was taken.""" + wanted = {str(value).lower() for value in hashes} + return sum( + counts.get(key, 0) - earlier.get(seam, {}).get(key, 0) + for seam, counts in self.calls.items() + for key in wanted + ) + + def missing_calls(self, hashes: Iterable[str]) -> dict[str, int]: + """Per seam, how many of ``hashes`` still lack their one cleanup.""" + wanted = {str(value).lower() for value in hashes} + return { + seam: sum(1 for key in wanted if self.calls[seam].get(key, 0) < expected) + for seam, (_owner, _attribute, expected) in _CLEANUP_SEAMS.items() + if expected is not None + } + + @property + def floor_double_releases(self) -> int: + return sum(1 for count in self.floor_releases.values() if count > 1) + + +def _deep_size(value: Any, seen: set[int]) -> int: + """Bytes reachable from ``value`` through containers and instance dicts. + + Each object is charged once per ``seen`` set, so a string or holder + shared by two records is counted for the first and free for the second + -- the same accounting the process pays. + """ + if id(value) in seen: + return 0 + seen.add(id(value)) + size = sys.getsizeof(value) + if isinstance(value, dict): + for key, item in value.items(): + size += _deep_size(key, seen) + _deep_size(item, seen) + elif isinstance(value, (tuple, list, set, frozenset)): + for item in value: + size += _deep_size(item, seen) + elif hasattr(value, "__dict__"): + size += _deep_size(vars(value), seen) + slots = getattr(type(value), "__slots__", ()) + if isinstance(slots, str): + slots = (slots,) + for name in slots: + if hasattr(value, name): + size += _deep_size(getattr(value, name), seen) + return size + + +@dataclass(frozen=True) +class CleanupBacklogReport: + """One faulted collapse walk, its backlog, and the recovery that drained it. + + Every figure is from one :meth:`CandidateStormRig.measure_cleanup_backlog` + call. The memory figures are two independent readings of the same + state: ``registry_bytes`` walks the shipped registry with + ``sys.getsizeof`` (records, their step sets, hash keys, and the exact + holder objects they retain, each object charged once), while + ``traced_walk_bytes`` is ``tracemalloc``'s change in live traced bytes + across the faulted walk -- which also carries everything else the walk + allocated (fences, counters, the adopted remainder), so the per-record + marginal figure is taken against a ``CLEANUP_FAULT_NONE`` walk of the + same storm by :func:`cleanup_backlog_marginal_bytes_per_record`. + + The recovery block is the contract this measurement exists to prove: + ``excess_cleanup_calls`` and ``floor_double_releases`` must be zero (no + double cleanup), ``missing_cleanup_calls`` must be zero (every owed + step ran), ``terminal_rows_adopted`` must be zero (no replay adoption of + a terminal row), ``node_offers`` and ``drain_rounds`` must be zero (no + re-offer), and ``winner_floor_holder_retained`` must hold when the rig + seeded credit shares (no loss of payout-floor authority). + """ + + fault: str + view: str + candidates: int + credit_shares: bool + backlog_max: int + # -- the faulted walk -------------------------------------------------- + walk_seconds: float + collapsed_rows: int + deferred_records: int + preserved_rows: int + backpressure_engagements: int + backpressure_active: bool + pending_share_holders: int + terminal_outcome_pins: int + oldest_age_seconds: float + registry_bytes: int + holder_bytes: int + registry_bytes_per_record: float + holder_bytes_per_holder: float + traced_walk_bytes: int + traced_owner_bytes: int + traced_peak_bytes: int + # -- sustained failure ------------------------------------------------- + sustained_passes: int + sustained_failed_passes: int + depth_after_sustained: int + holders_after_sustained: int + pins_after_sustained: int + # -- recovery ---------------------------------------------------------- + recovery_seconds: float + recovery_passes: int + recovered_records: int + retry_records_per_second: float + depth_after_recovery: int + second_walk_seconds: float + collapsed_rows_after_recovery: int + excess_cleanup_calls: dict[str, int] + missing_cleanup_calls: dict[str, int] + post_recovery_cleanup_calls: int + floor_double_releases: int + terminal_rows_adopted: int + node_offers: int + drain_rounds: int + pending_rows_after_drain: int + floor_holders_before: int + floor_holders_after: int + winner_floor_holder_retained: bool + + @property + def recovered_cleanly(self) -> bool: + return ( + self.depth_after_recovery == 0 + and not any(self.excess_cleanup_calls.values()) + and not any(self.missing_cleanup_calls.values()) + and self.post_recovery_cleanup_calls == 0 + and self.floor_double_releases == 0 + and self.terminal_rows_adopted == 0 + and self.node_offers == 0 + and self.drain_rounds == 0 + ) + + +def cleanup_backlog_marginal_bytes_per_record( + faulted: CleanupBacklogReport, + clean: CleanupBacklogReport, + *, + owner_only: bool = False, +) -> float: + """Traced bytes one retry record cost over a clean walk of the same storm. + + ``owner_only`` restricts both readings to allocations made by the + block-candidate owner module, which is where the record, its step set + and its registry entry are allocated. + """ + if faulted.deferred_records <= 0: + return 0.0 + if owner_only: + return (faulted.traced_owner_bytes - clean.traced_owner_bytes) / faulted.deferred_records + return (faulted.traced_walk_bytes - clean.traced_walk_bytes) / faulted.deferred_records + + +def _warm_traceback_caches() -> None: + """Load the source lines a faulted cleanup's tracebacks will print. + + ``traceback.print_exc`` fills ``linecache`` with the whole source of + every frame's file the first time it runs, hundreds of kilobytes that + would otherwise land inside the first faulted walk's traced delta. + """ + import lab.prism.block_candidates as block_candidates + + for path in (block_candidates.__file__, __file__): + linecache.getlines(path) + + +def _owner_traced_bytes(snapshot: Any) -> int: + """Live traced bytes allocated by the block-candidate owner module.""" + import lab.prism.block_candidates as block_candidates + + filtered = snapshot.filter_traces( + (tracemalloc.Filter(True, block_candidates.__file__),) + ) + return sum(statistic.size for statistic in filtered.statistics("filename")) + + +class DecidedHeightRpc(FakeRpc): + """A chain whose best tip is one storm candidate at the storm height. + + Models the instant after a height is decided: ``getbestblockhash`` and + ``getblockhash(height)`` name the winner, the winner's header is active + with one confirmation, every other candidate header is unknown, and a + further ``submitblock`` answers ``inconclusive`` (a valid block that did + not become the tip). Calls are counted per method so a drain's RPC cost + is measurable; the counter is thread-safe because the shipped fast lane + runs ``submitblock`` on a bounded worker thread. + """ + + def __init__( + self, + *, + winner_hash: str, + height: int, + bits: str = COLLAPSE_POW_LIMIT_BITS, + ) -> None: + self.winner_hash = winner_hash.lower() + self.height = int(height) + # The decided block's compact bits as qbitd would report them in + # getblockheader. Work is stated in bits, not in the raw difficulty + # float, because a candidate row carries scaled difficulty + # (raw * COLLAPSE_DIFFICULTY_SCALE) and only a bits-derived value is + # in the same units. The default is the qbit powLimit, at or below + # every sibling in the storm, so the decided block never reads as + # the weaker chain. + self.bits = str(bits).lower() + self.difficulty = ( + scaled_network_difficulty(self.bits) / COLLAPSE_DIFFICULTY_SCALE + ) + self.calls: dict[str, int] = {} + self._lock = threading.Lock() + + def _count(self, method: str) -> None: + with self._lock: + self.calls[method] = self.calls.get(method, 0) + 1 + + def call(self, method: str, params: list[object] | None = None) -> object: + self._count(method) + if method == "getbestblockhash": + return self.winner_hash + if method == "getblockcount": + return self.height + if method == "getblockhash": + height = int((params or [0])[0]) + if height == self.height: + return self.winner_hash + raise RuntimeError(f"qbit RPC getblockhash failed: -8 unknown height {height}") + if method == "getblockheader": + block_hash = str((params or [""])[0]).lower() + if block_hash == self.winner_hash: + return { + "hash": self.winner_hash, + "height": self.height, + "confirmations": 1, + "bits": self.bits, + "difficulty": self.difficulty, + } + raise RuntimeError("qbit RPC getblockheader failed: -5 Block not found") + if method == "submitblock": + return "inconclusive" + return super().call(method, params) + + +class CandidateStormRig: + """Seed one same-parent storm and observe live admission plus restart.""" + + def __init__( + self, + *, + candidates: int = OBSERVED_TESTNET_CANDIDATE_STORM, + queue_depth: int = MAX_PENDING_BLOCK_CANDIDATES, + quiet: bool = True, + credit_shares: bool = False, + backlog_max: int | None = None, + ) -> None: + if candidates <= 0: + raise ValueError("candidates must be positive") + if queue_depth <= 0: + raise ValueError("queue_depth must be positive") + if backlog_max is not None and backlog_max <= 0: + raise ValueError("backlog_max must be positive") + self.candidates = candidates + self.queue_depth = queue_depth + self.quiet = quiet + # Issue #198. ``credit_shares`` seeds every candidate as credit-bearing + # and registers its pending share on the writer's floor, so the + # collapse cleanup has real floor authority to release and the + # decided winner has real authority to keep; the restart view adopts + # holders through the shipped decode exactly as production does. + # ``backlog_max`` pins the cleanup-retry admission bound on every + # coordinator the rig owns (the shipped default otherwise). + self.credit_shares = bool(credit_shares) + self.backlog_max = backlog_max + self.ledger = SingleWriterShareLedger() + self.live_server: Any | None = None + self.restarted_server: Any | None = None + self.block_hashes: list[str] = [] + self.decided_rpc: DecidedHeightRpc | None = None + + def _coordinator(self) -> tuple[Any, Any]: + server, state, _recording = submit_coordinator(tip=STORM_PARENT_HASH) + server.max_blocks = self.candidates + 1 + server.stop_after_block = False + server.block_candidate_queue = queue.Queue(maxsize=self.queue_depth) + server.ledger = self.ledger + # A shipped tunable, set to zero so a parked or retained retry neither + # sleeps on the submitter lane nor parks behind a not-before deadline. + # This is a component oracle, not a pacing test: every wakeup must be + # re-offerable in the same round it was parked. + server.block_candidate_retry_initial_seconds = 0.0 + if self.backlog_max is not None: + server.block_candidate_cleanup_retry_backlog_max = int(self.backlog_max) + if self.decided_rpc is not None: + server.rpc = self.decided_rpc + return server, state + + def _silence(self) -> ContextManager[Any]: + if not self.quiet: + return nullcontext() + return redirect_stdout(StringIO()) + + def _silence_all(self) -> ContextManager[Any]: + """Silence both streams: a faulted cleanup prints tracebacks to stderr.""" + if not self.quiet: + return nullcontext() + stack = ExitStack() + stack.enter_context(redirect_stdout(StringIO())) + stack.enter_context(redirect_stderr(StringIO())) + return stack + + def _candidate(self, server: Any, state: Any, index: int) -> tuple[Any, Any]: + block_hash = f"{index:064x}" + pending = PendingShare( + share_id=f"miner-a:{block_hash}", + miner_id="miner-a", + order_key="miner-a", + p2mr_program_hex="11" * 32, + share_difficulty=1, + network_difficulty=1, + template_height=9, + job_id="job-1", + job_issued_at_ms=1, + accepted_at_ms=index, + ntime=1, + ) + candidate = block_candidate( + server, + state, + SimpleNamespace( + coinbase_tx_hex="00", + block_hash_hex=block_hash, + block_hex="00", + share_pass=True, + block_pass=True, + ), + pending_share=pending, + credit_share_on_accept=self.credit_shares, + ) + return candidate, (pending, server.block_candidate_intent(candidate)) + + def seed_live(self) -> CandidateStormSnapshot: + """Persist the storm, then admit every live candidate wakeup.""" + + if self.live_server is not None: + raise RuntimeError("live storm has already been seeded") + server, state = self._coordinator() + candidates = [] + durable_records = [] + for index in range(1, self.candidates + 1): + candidate, durable_record = self._candidate(server, state, index) + candidates.append(candidate) + durable_records.append(durable_record) + self.block_hashes = [ + str(candidate.submission.block_hash_hex).lower() + for candidate in candidates + ] + self.ledger.append_batch(durable_records) + if self.credit_shares: + # The live stamp path registers a credit-bearing share on the + # floor before its candidate is admitted; the rig appends the + # batch directly, so it registers the same holder here. + writer = server._ensure_share_writer_service() + for candidate in candidates: + writer.adopt_pending_share(candidate.pending_share) + with self._silence(): + for candidate in candidates: + server.enqueue_block_candidate(candidate) + self.live_server = server + return self.snapshot(server) + + def restart_and_enumerate(self) -> CandidateStormSnapshot: + """Create a fresh coordinator and enumerate every durable row.""" + + if self.live_server is None: + raise RuntimeError("seed_live must run before restart") + if self.restarted_server is not None: + raise RuntimeError("storm has already been restarted") + server, _state = self._coordinator() + # The restart view exists to hand a set-oriented selector the full + # replay-adopted population, so the shipped #183 collapse is + # suppressed for this enumeration exactly as it is for the per-row + # drain. Without that, deciding the height before the restart would + # let the enumeration dispose of the storm itself and leave the + # instrument with nothing to characterize. + server._ensure_block_candidate_service()._collapse_superseded_block_candidates = ( + lambda durable_rows, **_kwargs: durable_rows + ) + server._note_block_replay_enumeration_owed() + try: + with self._silence(): + restored = server.replay_pending_block_candidates() + finally: + server._ensure_block_candidate_service().__dict__.pop( + "_collapse_superseded_block_candidates", + None, + ) + if restored != self.candidates: + raise AssertionError( + f"restart restored {restored} of {self.candidates} candidates" + ) + self.restarted_server = server + return self.snapshot(server) + + def snapshot(self, server: Any) -> CandidateStormSnapshot: + """Capture fixed-cardinality gauges without retaining payload copies.""" + + service = server._ensure_block_candidate_service() + service._ensure_block_replay_state() + durable_rows = self.ledger.pending_block_candidate_rows( + limit=self.candidates + 1 + ) + durable_hashes = { + str(row["block_hash"]).lower() + for row in durable_rows + } + outstanding = set(service._outstanding_block_candidate_hashes) + replay_inflight = set(service._block_replay_inflight_hashes) + ownership = self.ownership(server) + return CandidateStormSnapshot( + durable_pending=len(durable_hashes), + live_queue_capacity=service.candidate_queue.maxsize, + live_queued=service.candidate_queue.qsize(), + live_coalesced=int(service.wakeups_coalesced), + outstanding_marked=len(outstanding), + outstanding_covers_all_durable=durable_hashes <= outstanding, + replay_queued=service._block_replay_candidate_queue.qsize(), + replay_inflight_marked=len(replay_inflight), + replay_inflight_covers_all_durable=durable_hashes <= replay_inflight, + accepted_parent_previews=len(server._accepted_block_payout_previews), + replay_enumeration_owed=server._block_replay_enumeration_owed(), + selector_evidence_marked=sum( + 1 for record in ownership if record.selector_evidence + ), + node_offer_evidence_marked=sum( + 1 for record in ownership if record.node_offer_evidence + ), + unoffered_retry_marked=sum( + 1 for record in ownership if record.retry_held_without_offer + ), + unoffered_lease_marked=sum( + 1 for record in ownership if record.lease_held_without_offer + ), + unoffered_terminal_marked=sum( + 1 for record in ownership if record.terminal_without_offer + ), + ) + + # -- decided-height extension ------------------------------------------ + + @property + def storm_height(self) -> int: + """The template height every storm candidate was built for.""" + server = self.live_server or self.restarted_server + if server is None: + raise RuntimeError("seed_live must run before the height is known") + return int(server.jobs["job-1"].template["height"]) + + def decide_height(self, *, winner_index: int | None = None) -> str: + """Make one storm candidate the active block at the storm height. + + Installs :class:`DecidedHeightRpc` on every coordinator this rig + owns (and on ones created later), so both the live and the restart + views observe the same decided chain. The default winner is the + LAST seeded candidate: it is a coalesced wakeup in the live view (it + never entered the bounded queue) and the last row of the restart + replay order, so a per-row drain reaches every sibling before it + would reach the winner. + """ + if not self.block_hashes: + raise RuntimeError("seed_live must run before a height can be decided") + if winner_index is None: + winner_index = self.candidates + if not 1 <= winner_index <= self.candidates: + raise ValueError("winner_index must name a seeded candidate") + winner_hash = self.block_hashes[winner_index - 1] + self.decided_rpc = DecidedHeightRpc( + winner_hash=winner_hash, + height=self.storm_height, + ) + for server in (self.live_server, self.restarted_server): + if server is not None: + server.rpc = self.decided_rpc + return winner_hash + + def ownership(self, server: Any) -> list[CandidateOwnership]: + """Read every seeded candidate's ownership/evidence facts at once.""" + service = server._ensure_block_candidate_service() + service._ensure_block_replay_state() + service._ensure_block_candidate_disposition_state() + outbox = self.ledger._block_candidate_outbox + live_queued = { + str(item.submission.block_hash_hex).lower() + for item in list(service.candidate_queue.queue) + } + replay_queued = { + str(item.submission.block_hash_hex).lower() + for item in list(service._block_replay_candidate_queue.queue) + } + with server.lock: + outstanding = set(service._outstanding_block_candidate_hashes) + replay_inflight = set(service._block_replay_inflight_hashes) + tip_observed = set(service._tip_observed_accepted_block_hashes) + retained = set( + getattr(service, "_block_candidate_retained_node_submissions", {}) + ) + accounted = set(server._accounted_accepted_block_hashes) + terminal = dict(service._block_candidate_terminal_outcomes) + retry_held: set[str] = set() + for holder in ( + service.retry_candidate, + getattr(service, "_block_accounting_deferred_retry_candidate", None), + ): + if holder is not None: + retry_held.add(str(holder.submission.block_hash_hex).lower()) + retry_held.update(service._block_disposition_waiting_retries) + retry_held.update(service.finalize_retries) + with service._block_candidate_disposition_registry_lock: + disposition_held = set(service._block_candidate_disposition_flights) + with server._accepted_block_payout_preview_condition: + previews = set(server._accepted_block_payout_previews) + records = [] + for block_hash in self.block_hashes: + row = outbox.get(block_hash) + intent = row["candidate"] if row is not None else None + pool_block = self.ledger.pool_block_state(block_hash=block_hash) + records.append( + CandidateOwnership( + block_hash=block_hash, + expected_height=( + int(intent["expected_height"]) + if isinstance(intent, Mapping) + else self.storm_height + ), + parent_hash=( + str(intent["parent_hash"]).lower() + if isinstance(intent, Mapping) + else STORM_PARENT_HASH + ), + network_difficulty=( + float(intent["found_block"]["network_difficulty"]) + if isinstance(intent, Mapping) + and isinstance(intent.get("found_block"), dict) + and intent["found_block"].get("network_difficulty") is not None + else None + ), + outbox_state=str(row["state"]) if row is not None else None, + attempt_count=int(row["attempt_count"]) if row is not None else 0, + live_queued=block_hash in live_queued, + replay_queued=block_hash in replay_queued, + outstanding=block_hash in outstanding, + replay_inflight=block_hash in replay_inflight, + preview_barrier=block_hash in previews, + disposition_held=block_hash in disposition_held, + retry_held=block_hash in retry_held, + node_acceptance_retained=block_hash in retained, + tip_observed=block_hash in tip_observed, + accounted_accepted=block_hash in accounted, + terminal_outcome=terminal.get(block_hash), + pool_block_chain_state=( + str(pool_block["chain_state"]) + if pool_block is not None + else None + ), + ) + ) + return records + + # -- perturbation ------------------------------------------------------- + + def _take_queued_candidate(self, server: Any, block_hash: str) -> Any: + """Remove one queued wakeup from whichever lane currently holds it.""" + service = server._ensure_block_candidate_service() + service._ensure_block_replay_state() + for queue_obj in ( + service.candidate_queue, + service._block_replay_candidate_queue, + ): + for item in list(queue_obj.queue): + if str(item.submission.block_hash_hex).lower() == block_hash: + # Single-threaded instrument: removing in place from the + # backing deque keeps both queue order and qsize() exact. + queue_obj.queue.remove(item) + return item + raise KeyError(f"no queued wakeup for {block_hash}") + + def perturb(self, server: Any, kind: str, block_hash: str) -> Any: + """Install one selector-evidence shape on a seeded candidate. + + Each kind reproduces one of the facts ``submit_next`` and + ``record_abandoned`` consult, through the shipped entry point rather + than a hand-written marker, and each shows up in :meth:`ownership` as + ``selector_evidence``. Three kinds show up there *without* + ``node_offer_evidence``, because none reaches qbitd: ``retry_slot`` + moves an already-queued, never-offered wakeup into the retry holder, + which is exactly what the shipped path does when the fast-lane + reservation declines; ``lease_held`` claims the disposition lease + by itself, which is all ``submit_next`` holds while it runs the + terminal, capacity and fast-lane checks that precede any submitblock; + and ``terminal`` records the disposition seal that the capacity-closed + path installs with ``attempted=False`` and no submitblock at all. + Returns a callable that undoes the perturbation where undoing is + meaningful (the held disposition lease), otherwise ``None``. + """ + service = server._ensure_block_candidate_service() + service._ensure_block_replay_state() + service._ensure_block_candidate_disposition_state() + key = str(block_hash).lower() + if kind == "lease_held": + lease = server._claim_block_candidate_disposition(key, blocking=False) + if lease is None: + raise RuntimeError(f"disposition lease already held for {key}") + return lambda: server._release_block_candidate_disposition(lease) + if kind == "retained_success": + service._stash_retained_block_candidate_node_submission( + key, + _BlockCandidateNodeSubmission(attempted=True, result=None), + ) + return None + if kind == "tip_observed": + # Only an outstanding hash can register a tip observation, and + # replay adoption alone never makes one outstanding -- the node + # offer does (see test_replay_adoption_alone_drops_a_tip_observation). + # Model "offered, then seen as the tip" through both shipped entry + # points so the shape holds in the restart view too. + server._register_outstanding_block_candidate(key) + server._note_tip_observation_for_candidates(key) + return None + if kind == "retry_slot": + candidate = self._take_queued_candidate(server, key) + with server.lock: + service.retry_candidate = candidate + return None + if kind == "prepared_pool_block": + self.ledger.persist_accepted_block( + block_hash=key, + block_height=self.storm_height, + parent_hash=STORM_PARENT_HASH, + final_bundle={}, + audit_report={}, + ) + return None + if kind == "terminal": + service._record_block_candidate_terminal_outcome(key, accepted=False) + return None + raise ValueError(f"unknown perturbation: {kind}") + + # -- per-row drain ------------------------------------------------------ + + # A head that neither terminalizes a row nor parks on a held lease is + # being deferred by the shipped path (acceptance evidence). Offer it + # once more, then withhold it: re-offering forever is what the shipped + # loop does behind its backoff, and this drain has no backoff. + _DRAIN_STALL_LIMIT = 2 + + def _dequeue_head(self, server: Any) -> tuple[str | None, str | None]: + """Peek the hash and lane ``submit_next`` would take next, in order.""" + service = server._ensure_block_candidate_service() + service._ensure_block_replay_state() + service._ensure_block_candidate_disposition_state() + with server.lock: + retry = service.retry_candidate + if retry is not None and service._block_candidate_retry_ready_locked( + retry + ): + return str(retry.submission.block_hash_hex).lower(), "retry" + for lane, queue_obj in ( + ("live", service.candidate_queue), + ("replay", service._block_replay_candidate_queue), + ): + items = list(queue_obj.queue) + if items: + return str(items[0].submission.block_hash_hex).lower(), lane + with server.lock: + waiting = service._block_disposition_waiting_retries + ready = [ + key + for key in waiting + if service._block_candidate_retry_ready_locked(waiting[key]) + ] + if ready: + return ( + min( + ready, + key=lambda key: int( + waiting[key].context.template["height"] + ), + ), + "waiting", + ) + return None, None + + def _withhold_head( + self, + server: Any, + block_hash: str, + lane: str, + ) -> Any | None: + """Lift one withheld hash out of the lane that would dequeue it next. + + Returns the exact wakeup object removed, so the drain can put it back + before it returns. Discarding it instead would strand the row for + good: replay adoption keeps the hash in + ``_block_replay_inflight_hashes`` until a terminal outcome is + recorded, and ``_enqueue_replayed_block_candidate`` drops any + re-adoption of an in-flight hash as a duplicate, so no later + enumeration can rebuild the wakeup this call threw away. + """ + service = server._ensure_block_candidate_service() + if lane == "retry": + with server.lock: + candidate = service.retry_candidate + service.retry_candidate = None + return candidate + if lane == "live": + return service.candidate_queue.get_nowait() + if lane == "replay": + return service._block_replay_candidate_queue.get_nowait() + if lane == "waiting": + with server.lock: + return service._block_disposition_waiting_retries.pop( + block_hash, + None, + ) + return None + + def _restore_withheld_wakeup( + self, + server: Any, + block_hash: str, + lane: str, + wakeup: Any, + ) -> bool: + """Put one lifted wakeup back where it was lifted from. + + Returns False when an equivalent wakeup already stands for the hash, + which makes this object a dropped same-hash duplicate for the caller + to release. + + Queue lanes are restored by pushing onto the front of the backing + deque rather than through ``put_nowait``. The wakeup was the head + when it was lifted and everything ahead of it has since been + consumed, so the front is its exact position; and because + ``get_nowait`` never touched ``unfinished_tasks``, putting it back in + place leaves that counter matching the item's original ``put`` + instead of double-counting it. This is the same in-place idiom + :meth:`_take_queued_candidate` uses, for the same reason. + """ + service = server._ensure_block_candidate_service() + service._ensure_block_replay_state() + service._ensure_block_candidate_disposition_state() + if lane in ("retry", "waiting"): + with server.lock: + if lane == "retry" and service.retry_candidate is None: + service.retry_candidate = wakeup + return True + # Either this is a waiting-registry entry, or the single + # retry head slot has been taken since. The shipped merge + # parks a displaced hash in the waiting registry for exactly + # this reason, so park it there rather than drop it. + waiting = service._block_disposition_waiting_retries + if block_hash in waiting: + return False + waiting[block_hash] = wakeup + return True + queue_obj = ( + service._block_replay_candidate_queue + if lane == "replay" + else service.candidate_queue + ) + queue_obj.queue.appendleft(wakeup) + return True + + def _next_drainable( + self, + server: Any, + withheld: set[str], + held: list[tuple[str, str, Any]] | None = None, + ) -> tuple[str | None, str | None]: + """Return the next hash to drive, lifting withheld ones out on the way. + + Every wakeup lifted out is appended to ``held`` in lift order so the + drain can restore it; see :meth:`_restore_withheld_wakeup`. + """ + while True: + head, lane = self._dequeue_head(server) + if head is None or head not in withheld: + return head, lane + wakeup = self._withhold_head(server, head, str(lane)) + if wakeup is not None and held is not None: + held.append((str(lane), head, wakeup)) + + @staticmethod + def _node_offers_made(rpc: Any) -> int: + """Total ``submitblock`` calls the rig's RPC log has counted so far. + + :class:`DecidedHeightRpc` counts per method under a lock, and the + shipped fast lane runs ``submitblock`` on a bounded worker thread + whose completion ``submit_next`` waits for, so a delta taken across + one ``submit_next`` names exactly the offers that call performed. + + The counter is required rather than optional: a drain that could not + read it would report zero offers forever, which would silently make + ``max_rounds`` unbounded instead of stopping the drain. Every drain + runs against a decided height, so the counter is always installed. + """ + calls = getattr(rpc, "calls", None) + if not isinstance(calls, dict): + raise TypeError( + "drain_per_row needs the per-method RPC counter installed by " + "decide_height; rounds and max_rounds are measured from its " + "submitblock count" + ) + return int(calls.get("submitblock", 0)) + + def _run_block_accounting_tasks(self, server: Any) -> int: + """Run the shipped accounting lane to quiescence, on this thread. + + Mirrors ``BlockCandidateService.block_accounting_loop``: the + definitive-acceptance lane first while its dispatch quota lasts, then + the primary queue, then the unbounded spillover, then the + invalid-intent quarantine, with the loop's own retain-on-failure + behaviour. The quota is read from the service rather than assumed so + a scenario that pins it also pins what this drain does. + """ + service = server._ensure_block_candidate_service() + service._ensure_block_accounting_state() + # block_accounting_loop stamps this on entry, and + # _pace_block_candidate_retry reads it to record a not-before deadline + # instead of sleeping while the lease and writer admission are held. + service._block_accounting_thread_ident = threading.get_ident() + ran = 0 + accepted_run = 0 + while True: + source_queue = None + if accepted_run < service._block_accounting_accepted_dispatch_quota(): + try: + _priority, _sequence, task = ( + service._block_accounting_accepted_queue.get_nowait() + ) + source_queue = service._block_accounting_accepted_queue + accepted_run += 1 + except queue.Empty: + pass + if source_queue is None: + try: + _priority, _sequence, task = ( + service._block_accounting_queue.get_nowait() + ) + source_queue = service._block_accounting_queue + except queue.Empty: + try: + _priority, _sequence, task = ( + service._block_accounting_overflow_queue.get_nowait() + ) + source_queue = service._block_accounting_overflow_queue + except queue.Empty: + pass + if source_queue is not None: + accepted_run = 0 + if source_queue is None: + try: + _priority, _sequence, task = ( + service._block_accounting_accepted_queue.get_nowait() + ) + source_queue = service._block_accounting_accepted_queue + accepted_run = 0 + except queue.Empty: + while server._run_one_invalid_block_candidate_quarantine(): + pass + return ran + try: + server._run_block_accounting_task(task) + except Exception: + server._retain_block_candidate_for_retry(task.candidate) + finally: + source_queue.task_done() + ran += 1 + + def drain_per_row( + self, + server: Any, + *, + stop_before_hash: str | None = None, + preserve_hashes: Iterable[str] | None = None, + max_rounds: int | None = None, + dequeue_skip: bool = False, + ) -> PerRowDrainReport: + """Drive the shipped per-row path in the production submitter split. + + Each round runs ``submit_next_block_candidate(defer_accounting=True)`` + -- the call ``BlockCandidateService.run`` makes -- and then drains the + real accounting queue through ``_run_block_accounting_task``, so the + node offer, the fast-lane reservation, the replayed-candidate + acceptance-evidence restore and the durable finalization all run on + their shipped code paths. When both lanes are empty the outbox replay + query recovers coalesced wakeups, exactly as the loop does. + + ``defer_accounting=True`` is not a detail: with accounting inline + ``submit_next`` claims each disposition lease *blocking*, so a single + candidate whose lease is held elsewhere stops the drain forever, and + ``_restore_replayed_candidate_acceptance_evidence`` -- which is what + keeps a replayed candidate with a prepared pool-block row from being + abandoned -- never runs at all. + + ``stop_before_hash`` ends the drain when that hash reaches the head + (used for the decided winner, whose finalization tail is outside this + instrument). ``preserve_hashes`` instead withholds hashes for the + whole drain: they are lifted out of the dequeue order without being + offered or disposed, so their durable row and evidence markers stay + exactly as the caller arranged them while every other row drains. + Any head the shipped path parks on a held lease, or repeatedly + declines to terminalize, is withheld the same way and reported, so + the drain is bounded no matter what evidence the caller installed. + + Withholding is scoped to the invocation, so every wakeup this call + lifted out is put back before it returns, on every exit -- exhausted, + ``max_rounds``, ``stop_before_hash``, or an exception. Restoring is + not a courtesy: a withheld row keeps its ownership markers, and a + hash marked replay-inflight is one no enumeration will re-adopt, so + a discarded wakeup leaves the row pending with nothing left to + dispose of it and no later drain able to reach it. + + ``max_rounds`` stops the drain after that many *node offers* -- the + ``submitblock`` calls this invocation actually made, measured across + each ``submit_next`` from the rig's own RPC log -- so a caller can + step the oracle one offered row at a time. It is deliberately not + the count of consumed wakeups: ``submit_next`` also returns True for + a row it parked on a lease held elsewhere, a same-hash duplicate it + dropped, a fast-lane reservation it declined, and the capacity-closed + ``attempted=False`` split that terminalizes a durable row without + calling qbitd, and spending the budget on those would stop a bounded + drain before it performed the offers it promised. Boundedness for + those rows comes from the withholding above, not from the budget. + The report then covers only the rows *this* call moved: it is + compared against the durable state captured on entry, so a second + bounded call never re-reports the first one's abandonment. + + ``dequeue_skip`` is off by default, and off is what keeps this the + per-row oracle. Issue #181 item 2 added a second set-oriented + disposition inside ``submit_next`` itself -- the dequeue-time stale + sibling skip -- which would otherwise dispose of rows this drain is + meant to measure the per-row path against, exactly as #183's + replay-adoption collapse would. Suppressed the same way and for the + same reason. Turning it on drives the shipped skip instead, so the + same report shape (abandoned hashes, per-method RPC counts, attempt + marks) can be compared against the oracle the default produces. + """ + service = server._ensure_block_candidate_service() + service._ensure_block_candidate_disposition_state() + # Issue #183 landed a set-oriented collapse inside ``replay_pending``. + # This drain exists to be the *per-row* oracle that selector is + # measured against, and it re-enumerates the outbox whenever both + # lanes empty, so the collapse is suppressed for its duration: + # otherwise the enumeration would dispose of rows itself and the + # comparison would be the selector against the selector. + service._collapse_superseded_block_candidates = ( + lambda durable_rows, **_kwargs: durable_rows + ) + if not dequeue_skip: + # Issue #181 item 2's dequeue-time skip is a second shipped + # disposition inside ``submit_next``. The oracle is the per-row + # path, so it is suppressed exactly as the #183 collapse is. + service._skip_superseded_block_candidate_at_dequeue = ( + lambda _candidate, **_kwargs: False + ) + rpc = server.rpc + calls_before = dict(getattr(rpc, "calls", {})) + outbox = self.ledger._block_candidate_outbox + attempts_before = sum(int(row["attempt_count"]) for row in outbox.values()) + # The durable state of every row on entry. A row is only ever + # finished out of "pending" once, so comparing against this is what + # makes the report name this call's transitions rather than every + # terminal row the rig has accumulated across earlier drains. + states_before = { + str(block_hash).lower(): str(row["state"]) + for block_hash, row in outbox.items() + } + withheld = {str(item).lower() for item in (preserve_hashes or ())} + caller_withheld = frozenset(withheld) + stop_hash = None if stop_before_hash is None else str(stop_before_hash).lower() + lease_blocked: set[str] = set() + deferred: set[str] = set() + stalls: dict[str, int] = {} + # Node offers this call performed. The rig's RPC log is the only + # authoritative source: ``submit_next``'s boolean is True for + # no-offer paths too (see PerRowDrainReport). + rounds = 0 + accounting_tasks = 0 + enumerations = 0 + # Every wakeup withholding lifted out, in lift order, so the finally + # below can put each one back exactly where it came from. + held_wakeups: list[tuple[str, str, Any]] = [] + started = time.monotonic() + try: + with self._silence(): + while True: + if max_rounds is not None and rounds >= max_rounds: + break + head, _lane = self._next_drainable(server, withheld, held_wakeups) + if head is None: + enumerations += 1 + if server.replay_pending_block_candidates() == 0: + break + if self._next_drainable( + server, + withheld, + held_wakeups, + )[0] is None: + # The enumeration restored only withheld rows; the + # shipped loop would re-adopt and re-park them for as + # long as their evidence stands. + break + continue + if stop_hash is not None and head == stop_hash: + break + with server.lock: + terminal_before = len(service._block_candidate_terminal_outcomes) + offers_before = self._node_offers_made(rpc) + server.submit_next_block_candidate(defer_accounting=True) + # submit_next waits for its own submitblock worker before + # returning, so this delta is complete and is either 0 or + # 1 for the row just driven. + rounds += self._node_offers_made(rpc) - offers_before + accounting_tasks += self._run_block_accounting_tasks(server) + with server.lock: + progressed = ( + len(service._block_candidate_terminal_outcomes) + != terminal_before + ) + parked = head in service._block_disposition_waiting_retries + if progressed: + stalls.pop(head, None) + continue + if parked: + # submit_next could not claim this hash's lease without + # blocking and parked the wakeup, exactly as the shipped + # loop does. Withhold it instead of spinning behind a + # lease this drain does not own. + withheld.add(head) + lease_blocked.add(head) + continue + stalls[head] = stalls.get(head, 0) + 1 + if stalls[head] >= self._DRAIN_STALL_LIMIT: + withheld.add(head) + deferred.add(head) + finally: + try: + with self._silence(): + # Newest lift first: a mid-drain enumeration can re-adopt a + # row this call already lifted, and the re-adopted object is + # the one the shipped markers now describe. Restoring + # newest-first also rebuilds the original lane order, + # because each lifted wakeup goes back to its lane's front. + restored: set[str] = set() + for lane, key, wakeup in reversed(held_wakeups): + if key in restored or not self._restore_withheld_wakeup( + server, + key, + lane, + wakeup, + ): + # A wakeup for this hash already stands, so this + # object is a dropped same-hash duplicate and must + # release the credit floor it carries -- exactly + # what the shipped duplicate paths do. + service._release_dropped_duplicate_candidate_floor( + wakeup + ) + continue + restored.add(key) + finally: + service.__dict__.pop( + "_collapse_superseded_block_candidates", + None, + ) + service.__dict__.pop( + "_skip_superseded_block_candidate_at_dequeue", + None, + ) + terminalized: dict[str, set[str]] = {"submitted": set(), "abandoned": set()} + still_pending: set[str] = set() + for block_hash, row in outbox.items(): + key = str(block_hash).lower() + state = str(row["state"]) + if state == "pending": + still_pending.add(key) + elif states_before.get(key) != state: + terminalized.setdefault(state, set()).add(key) + calls_after = dict(getattr(rpc, "calls", {})) + return PerRowDrainReport( + rounds=rounds, + accounting_tasks=accounting_tasks, + replay_enumerations=enumerations, + abandoned_hashes=frozenset(terminalized["abandoned"]), + submitted_hashes=frozenset(terminalized["submitted"]), + pending_hashes=frozenset(still_pending), + withheld_hashes=( + caller_withheld + | frozenset(lease_blocked) + | frozenset(deferred) + ), + lease_blocked_hashes=frozenset(lease_blocked), + deferred_hashes=frozenset(deferred), + rpc_calls={ + method: calls_after.get(method, 0) - calls_before.get(method, 0) + for method in sorted(set(calls_before) | set(calls_after)) + }, + ledger_attempt_marks=( + sum(int(row["attempt_count"]) for row in outbox.values()) + - attempts_before + ), + wall_seconds=time.monotonic() - started, + ) + + # -- cleanup-retry backlog (issue #198) --------------------------------- + + def _pending_hashes(self) -> set[str]: + return { + str(row["block_hash"]).lower() + for row in self.ledger.pending_block_candidate_rows( + limit=self.candidates + 1 + ) + } + + def _registry_bytes(self, server: Any) -> tuple[int, int, int]: + """(registry bytes, holder bytes within them, retained holders). + + Read under the coordinator lock like every other registry reader. + Holders are charged inside the registry total first, then measured + again on their own so the per-holder figure can be reported apart + from the per-record one. + """ + service = server._ensure_block_candidate_service() + with server.lock: + registry = service._block_candidate_collapse_cleanup_retries + total = _deep_size(registry, set()) + holders: dict[int, Any] = {} + for record in registry.values(): + for share in record.shares: + holders[id(share)] = share + holder_seen: set[int] = set() + holder_bytes = sum(_deep_size(share, holder_seen) for share in holders.values()) + return total, holder_bytes, len(holders) + + def _collapse_walk(self, server: Any) -> tuple[set[str], float, int]: + """Run the shipped enumeration walk; return (rows it won, seconds, adoptions).""" + before = self._pending_hashes() + started = time.monotonic() + with self._silence_all(): + server._note_block_replay_enumeration_owed() + adopted = int(server.replay_pending_block_candidates()) + elapsed = time.monotonic() - started + return before - self._pending_hashes(), elapsed, adopted + + def _drain_backpressured_replay_page( + self, + server: Any, + *, + stop_before_hash: str, + ) -> set[str]: + """Drive the bounded spill page through the shipped dequeue collapse.""" + service = server._ensure_block_candidate_service() + if not service._block_replay_backpressure_drain_pending: + return set() + terminalized: set[str] = set() + replay_queue = service._block_replay_candidate_queue + # The queue cannot grow in this synchronous rig. Bound the driver by + # the captured page size so a failed disposition cannot spin. + rounds = replay_queue.qsize() + with self._silence_all(): + for _ in range(rounds): + if replay_queue.empty(): + break + candidate = replay_queue.queue[0] + block_hash = str(candidate.submission.block_hash_hex).lower() + if block_hash == stop_before_hash: + break + row = self.ledger._block_candidate_outbox[block_hash] + before = str(row["state"]) + if not server.submit_next_block_candidate(defer_accounting=True): + break + self._run_block_accounting_tasks(server) + after = str(row["state"]) + if before == "pending" and after != "pending": + terminalized.add(block_hash) + return terminalized + + def measure_cleanup_backlog( + self, + fault: str, + *, + view: str = "restart", + sustained_passes: int = 64, + trace_memory: bool = True, + ) -> CleanupBacklogReport: + """Drive the shipped collapse under one cleanup fault, then recover it. + + Phases, each on the shipped code path: + + 1. Seed the storm (and restart, for the restart view), decide the + height, and install ``fault`` at its seam. + 2. Run the enumeration walk. Every row the fenced write wins whose + cleanup the fault breaks becomes one retry record; rows the + admission bound declines stay durable and are counted as + preserved. The registry is measured in place. + 3. With the fault still on, run up to ``sustained_passes`` retry + passes: each must fail, re-register its record, and leave depth, + holders and pins unchanged -- the backlog is never shed. + 4. Heal the fault and drain the backlog through the shipped retry + pass until it reports nothing due, timing the drain. + 5. Drain the one replay page adopted at backpressure through the + shipped dequeue collapse, run the walk again so admission can + resume over whatever remains durable, then drive ``drain_per_row`` + over every queued wakeup to prove no terminal row reaches the + node. + + ``view`` selects the coordinator: ``live`` holds the storm's first + ``queue_depth`` wakeups in the bounded queue with the rest coalesced, + so only that many holders can be retained; ``restart`` holds every + row in the replay queue, so every record retains a holder -- the + heavier and more instructive case. + """ + if fault != CLEANUP_FAULT_NONE and fault not in CLEANUP_FAULTS: + raise ValueError(f"unknown cleanup fault: {fault}") + if view not in ("live", "restart"): + raise ValueError("view must be 'live' or 'restart'") + if self.live_server is None: + self.seed_live() + if view == "restart" and self.restarted_server is None: + self.restart_and_enumerate() + server = self.restarted_server if view == "restart" else self.live_server + assert server is not None + winner = ( + self.decide_height() + if self.decided_rpc is None + else self.decided_rpc.winner_hash + ) + service = server._ensure_block_candidate_service() + service._ensure_block_replay_state() + service._ensure_block_candidate_disposition_state() + writer = server._ensure_share_writer_service() + floor_before = len(writer._pending_share_commit_floor) + injector = CleanupFaultInjector(server, fault) + injector.install() + traced_walk = -1 + traced_owner = -1 + traced_peak = -1 + try: + counts_before = service.block_candidate_collapse_snapshot() + with server.lock: + inflight_before = set(service._block_replay_inflight_hashes) + if trace_memory: + _warm_traceback_caches() + gc.collect() + tracemalloc.start() + tracemalloc.reset_peak() + traced_before = tracemalloc.get_traced_memory()[0] + owner_before = _owner_traced_bytes(tracemalloc.take_snapshot()) + collapsed, walk_seconds, _adopted = self._collapse_walk(server) + if trace_memory: + gc.collect() + traced_after, traced_peak_after = tracemalloc.get_traced_memory() + owner_after = _owner_traced_bytes(tracemalloc.take_snapshot()) + tracemalloc.stop() + traced_walk = traced_after - traced_before + traced_owner = owner_after - owner_before + traced_peak = traced_peak_after - traced_before + counts_after = service.block_candidate_collapse_snapshot() + snapshot = service.collapsed_candidate_cleanup_backlog_snapshot() + registry_bytes, holder_bytes, holders = self._registry_bytes(server) + records = int(snapshot["depth"]) + # -- sustained failure ------------------------------------------ + sustained = 0 + failed_before = counts_after["cleanup_retry_failed"] + with self._silence_all(): + for _ in range(min(int(sustained_passes), records)): + if not service._run_one_collapsed_block_candidate_cleanup_retry(): + break + sustained += 1 + sustained_failed = ( + service.block_candidate_collapse_snapshot()["cleanup_retry_failed"] + - failed_before + ) + after_sustained = service.collapsed_candidate_cleanup_backlog_snapshot() + # -- recovery --------------------------------------------------- + injector.heal() + recovered_before = service.block_candidate_collapse_snapshot()[ + "cleanup_recovered" + ] + passes = 0 + started = time.monotonic() + with self._silence_all(): + while service._run_one_collapsed_block_candidate_cleanup_retry(): + passes += 1 + recovery_seconds = time.monotonic() - started + recovered = ( + service.block_candidate_collapse_snapshot()["cleanup_recovered"] + - recovered_before + ) + after_recovery = service.collapsed_candidate_cleanup_backlog_snapshot() + recovered_calls = injector.snapshot_calls() + resumed_at_dequeue = self._drain_backpressured_replay_page( + server, + stop_before_hash=winner, + ) + collapsed_after, second_walk_seconds, _adopted = self._collapse_walk(server) + with server.lock: + inflight_after = set(service._block_replay_inflight_hashes) + every_collapsed = collapsed | resumed_at_dequeue | collapsed_after + excess = injector.excess_calls(every_collapsed) + missing = injector.missing_calls(every_collapsed) + terminal_adopted = len((inflight_after - inflight_before) & every_collapsed) + drain = self.drain_per_row(server, stop_before_hash=winner) + # A hash whose record was discharged must never be cleaned up + # again: not by the second walk, not by the queued duplicates + # the drain drops. + post_recovery_calls = injector.calls_since(recovered_calls, collapsed) + finally: + if trace_memory and tracemalloc.is_tracing(): + tracemalloc.stop() + injector.uninstall() + # The winner's holder is the one the shipped path still owns: in the + # restart view the object adopted at restart, in the live view the + # object the first walk's recovery enumeration adopted (the seeded + # live wakeup coalesced and was never queued). Looked up after the + # walks for exactly that reason. + winner_share = next( + ( + item.pending_share + for queue_obj in ( + service.candidate_queue, + service._block_replay_candidate_queue, + ) + for item in list(queue_obj.queue) + if str(item.submission.block_hash_hex).lower() == winner + ), + None, + ) + floor_after = len(writer._pending_share_commit_floor) + return CleanupBacklogReport( + fault=fault, + view=view, + candidates=self.candidates, + credit_shares=self.credit_shares, + backlog_max=int(snapshot["backlog_max"]), + walk_seconds=walk_seconds, + collapsed_rows=len(collapsed), + deferred_records=records, + preserved_rows=( + counts_after["backlog_deferred"] - counts_before["backlog_deferred"] + ), + backpressure_engagements=int(snapshot["backpressure_engagements"]), + backpressure_active=bool(snapshot["backpressure_active"]), + pending_share_holders=int(snapshot["pending_share_holders"]), + terminal_outcome_pins=int(snapshot["terminal_outcome_pins"]), + oldest_age_seconds=float(snapshot["oldest_age_seconds"]), + registry_bytes=registry_bytes, + holder_bytes=holder_bytes, + registry_bytes_per_record=(registry_bytes / records) if records else 0.0, + holder_bytes_per_holder=(holder_bytes / holders) if holders else 0.0, + traced_walk_bytes=traced_walk, + traced_owner_bytes=traced_owner, + traced_peak_bytes=traced_peak, + sustained_passes=sustained, + sustained_failed_passes=int(sustained_failed), + depth_after_sustained=int(after_sustained["depth"]), + holders_after_sustained=int(after_sustained["pending_share_holders"]), + pins_after_sustained=int(after_sustained["terminal_outcome_pins"]), + recovery_seconds=recovery_seconds, + recovery_passes=passes, + recovered_records=int(recovered), + retry_records_per_second=( + (recovered / recovery_seconds) if recovery_seconds > 0 else 0.0 + ), + depth_after_recovery=int(after_recovery["depth"]), + second_walk_seconds=second_walk_seconds, + collapsed_rows_after_recovery=len( + resumed_at_dequeue | collapsed_after + ), + excess_cleanup_calls=excess, + missing_cleanup_calls=missing, + post_recovery_cleanup_calls=post_recovery_calls, + floor_double_releases=injector.floor_double_releases, + terminal_rows_adopted=terminal_adopted, + node_offers=int(self.decided_rpc.calls.get("submitblock", 0)), + drain_rounds=drain.rounds, + pending_rows_after_drain=drain.pending_rows, + floor_holders_before=floor_before, + floor_holders_after=floor_after, + winner_floor_holder_retained=( + winner_share is not None + and id(winner_share) in writer._pending_share_commit_floor + ), + ) + + +# -- heap and allocator readings (issue #226) ------------------------------ +# +# #185 measured the coordinator at 125 MiB before a storm and 613 MiB after +# the drain, and #226 found the mainnet resident set growing 390 MB/h with a +# memory map shaped like glibc per-thread arenas. These readings put the same +# instrument around the storm so both can be re-run on any host: the resident +# set, the interpreter's live-block count, glibc's arena/in-use/free bytes and +# arena count, and the anonymous-map band histogram, at every phase boundary, +# then after the storm's objects are released and collected, then after one +# malloc_trim. Every number is host-specific -- absolutes vary by up to an +# order of magnitude across hosts -- and the instrument reports the platform +# beside them; what transfers is the mechanism: does the live-block count +# return, does the free-bytes reading grow, does the trim return it. + +ARENA_PROBE_MIN_BYTES = 4 * 1024 +# Under glibc's initial 128 KiB mmap threshold, so the buffers come from the +# arenas rather than from direct mmap; over pymalloc's 512-byte ceiling, so +# they come from glibc at all. +ARENA_PROBE_MAX_BYTES = 120 * 1024 +ARENA_PROBE_LIVE_BUFFERS = 64 +# A worker's wait at the barrier, and half the join ceiling. 64 threads x +# 2,000 rounds take about a second on a development host; a slow host has +# an order of magnitude of headroom before the probe reports a failure. +ARENA_PROBE_BARRIER_TIMEOUT_SECONDS = 60.0 + + +@dataclass(frozen=True) +class HeapReading: + """One phase boundary's resident-set and allocator readings.""" + + phase: str + elapsed_seconds: float + resident_bytes: int + allocated_blocks: int + threads: int + malloc_info_available: bool + malloc_arena_bytes: int + malloc_in_use_bytes: int + malloc_free_bytes: int + malloc_mmapped_bytes: int + malloc_arena_count: int + # Free bytes split by where they sit: interior bins, which malloc_trim + # returns in every arena, and per-thread heap tops, which it cannot. + malloc_free_top_bytes: int + malloc_free_bin_bytes: int + anonymous_regions: int + anonymous_bytes: int + anonymous_regions_4mib_to_64mib: int + heap_segment_bytes: int + + +def take_heap_reading( + phase: str, + telemetry: ProcessHeapTelemetry, + started_monotonic: float, +) -> HeapReading: + sample = telemetry.sample() + shape = read_anonymous_map_shape() + malloc_info = read_malloc_info_summary() + return HeapReading( + phase=phase, + elapsed_seconds=round(time.monotonic() - started_monotonic, 3), + resident_bytes=read_resident_memory_bytes(), + allocated_blocks=sample.allocated_blocks, + threads=sample.threads, + malloc_info_available=sample.malloc_info_available, + malloc_arena_bytes=sample.malloc_arena_bytes, + malloc_in_use_bytes=sample.malloc_in_use_bytes, + malloc_free_bytes=sample.malloc_free_bytes, + malloc_mmapped_bytes=sample.malloc_mmapped_bytes, + malloc_arena_count=malloc_info.arenas, + malloc_free_top_bytes=malloc_info.top_bytes, + malloc_free_bin_bytes=malloc_info.bin_bytes, + anonymous_regions=shape.regions, + anonymous_bytes=shape.total_bytes, + anonymous_regions_4mib_to_64mib=shape.band_regions.get("4mib_to_64mib", -1), + heap_segment_bytes=shape.heap_segment_bytes, + ) + + +def allocator_environment() -> dict[str, str]: + """The glibc and CPython allocator variables this process started with.""" + return { + name: value + for name, value in sorted(os.environ.items()) + if name.startswith("MALLOC_") or name in ("GLIBC_TUNABLES", "PYTHONMALLOC") + } + + +def run_arena_probe( + threads: int, + rounds: int, + *, + seed: int = 226, + telemetry: ProcessHeapTelemetry | None = None, + allocate: Any = bytes, +) -> dict[str, Any]: + """Reproduce the per-thread arena shape and report what it costs. + + ``threads`` Python threads each allocate ``rounds`` medium buffers + (4-120 KiB: over pymalloc's ceiling, under glibc's initial mmap + threshold) into a small ring so a bounded number stay live at once, then + drop the ring and exit. glibc assigns a thread its own arena on its first + allocation (up to ``MALLOC_ARENA_MAX``) only while no exited thread's + arena is waiting on the free list, so the threads meet at a barrier + before any exits -- the coordinator's 64 threads are long-lived, and + that is the shape under test. The arena count after the probe is the + mechanism: it should approach ``threads + 1`` at glibc's default and stop + at the configured cap otherwise. The rings' freed buffers stay on each + arena's free lists after the threads exit -- arenas outlive threads -- + which is the retained-free-bytes reading the trim that follows is judged + by. ``allocate`` is the buffer constructor (``bytes``); a test injects a + failing one to prove a broken worker cannot wedge the probe. + """ + telemetry = telemetry if telemetry is not None else ProcessHeapTelemetry() + before = telemetry.sample() + resident_before = read_resident_memory_bytes() + arenas_before = read_malloc_arena_count() + started = time.monotonic() + # Bounded: a worker that fails breaks the barrier for everyone, a worker + # that stalls trips the barrier timeout, and the join below has its own + # ceiling, so one bad thread cannot wedge the instrument. Every such + # event is reported in ``failures`` rather than raised mid-probe. + all_allocated = threading.Barrier(threads) + failures: list[str] = [] + + def worker(index: int) -> None: + rng = random.Random(seed + index) + ring: list[bytes | None] = [None] * ARENA_PROBE_LIVE_BUFFERS + try: + for round_index in range(rounds): + size = rng.randint(ARENA_PROBE_MIN_BYTES, ARENA_PROBE_MAX_BYTES) + ring[round_index % ARENA_PROBE_LIVE_BUFFERS] = allocate(size) + all_allocated.wait(timeout=ARENA_PROBE_BARRIER_TIMEOUT_SECONDS) + except threading.BrokenBarrierError: + failures.append(f"worker {index}: barrier broken or timed out") + except Exception as exc: + all_allocated.abort() + failures.append(f"worker {index}: {exc!r}") + finally: + ring.clear() + + workers = [ + threading.Thread(target=worker, args=(index,), name=f"arena-probe-{index}") + for index in range(threads) + ] + for thread in workers: + thread.start() + join_deadline = time.monotonic() + ARENA_PROBE_BARRIER_TIMEOUT_SECONDS * 2 + for thread in workers: + thread.join(timeout=max(0.0, join_deadline - time.monotonic())) + if thread.is_alive(): + all_allocated.abort() + failures.append(f"{thread.name}: did not finish before the join ceiling") + seconds = time.monotonic() - started + after = telemetry.sample() + return { + "threads": threads, + "rounds": rounds, + "buffer_bytes": [ARENA_PROBE_MIN_BYTES, ARENA_PROBE_MAX_BYTES], + "live_buffers_per_thread": ARENA_PROBE_LIVE_BUFFERS, + "failures": failures, + "seconds": round(seconds, 6), + "arena_count_before": arenas_before, + "arena_count_after": read_malloc_arena_count(), + "resident_before": resident_before, + "resident_after": read_resident_memory_bytes(), + "malloc_info_available": before.malloc_info_available and after.malloc_info_available, + "malloc_arena_before": before.malloc_arena_bytes, + "malloc_arena_after": after.malloc_arena_bytes, + "malloc_free_before": before.malloc_free_bytes, + "malloc_free_after": after.malloc_free_bytes, + "malloc_in_use_before": before.malloc_in_use_bytes, + "malloc_in_use_after": after.malloc_in_use_bytes, + } + + +def allocator_experiment_child_args(argv: Sequence[str]) -> list[str]: + """The child invocation: this run's arguments minus the experiment options. + + ``--verbose`` is dropped too: a child's coordinator logs would precede + the JSON document on the same stdout the parent parses, and the parse + would fail only after the whole storm had run. + """ + child: list[str] = [] + skip_next = False + for token in argv: + if skip_next: + skip_next = False + continue + if token in ("--allocator-experiment", "--allocator-mmap-threshold"): + skip_next = True + continue + if token.startswith("--allocator-experiment=") or token.startswith( + "--allocator-mmap-threshold=" + ): + continue + if token == "--verbose": + continue + child.append(token) + if "--heap-report" not in child: + child.append("--heap-report") + return child + + +def run_allocator_experiment( + settings: Sequence[str], + child_args: Sequence[str], + *, + mmap_threshold: int | None = None, +) -> list[dict[str, Any]]: + """Run this instrument once per ``MALLOC_ARENA_MAX`` setting, in a child each. + + glibc reads its environment once at process start, so every setting + needs its own process; ``default`` runs with the variable unset. The + children inherit everything else, and ``mmap_threshold`` (when given) + pins ``MALLOC_MMAP_THRESHOLD_`` in each of them, which also switches + glibc's dynamic threshold off. + """ + results: list[dict[str, Any]] = [] + for setting in settings: + env = dict(os.environ) + env.pop("MALLOC_ARENA_MAX", None) + if setting != "default": + env["MALLOC_ARENA_MAX"] = str(int(setting)) + if mmap_threshold is not None: + env["MALLOC_MMAP_THRESHOLD_"] = str(int(mmap_threshold)) + completed = subprocess.run( + [sys.executable, str(Path(__file__).resolve()), *child_args], + env=env, + capture_output=True, + text=True, + check=True, + ) + child = json.loads(completed.stdout) + decided = child.get("decided") or {} + results.append( + { + "malloc_arena_max": setting, + "malloc_mmap_threshold": mmap_threshold, + "heap": child["heap"], + "per_row_drain": decided.get("per_row_drain"), + } + ) + return results + + +def _run_storm_phases( + args: argparse.Namespace, + read_heap: Any, + phase_seconds: dict[str, float], +) -> dict[str, Any]: + """Seed, restart, optionally decide/drain, optionally fault; return the report. + + Everything the storm allocates is referenced only from this frame, so a + caller that wants the post-storm resident set reads it after this + returns. ``read_heap`` is called at every phase boundary and + ``phase_seconds`` receives each phase's wall-clock. + """ + rig = CandidateStormRig( + candidates=args.candidates, + queue_depth=args.queue_depth, + quiet=not args.verbose, + ) + phase_started = time.monotonic() + live = rig.seed_live() + phase_seconds["seed_live"] = round(time.monotonic() - phase_started, 6) + read_heap("after_seed_live") + phase_started = time.monotonic() + restarted = rig.restart_and_enumerate() + phase_seconds["restart_enumerate"] = round(time.monotonic() - phase_started, 6) + read_heap("after_restart_enumerate") + report: dict[str, Any] = { + "bounds": { + "live_queue": MAX_PENDING_BLOCK_CANDIDATES, + "replay_page": MAX_BLOCK_REPLAY_ENUMERATION_ROWS, + }, + "input": { + "candidates": args.candidates, + "queue_depth": args.queue_depth, + }, + "live": asdict(live), + "restart": asdict(restarted), + } + if args.decide or args.drain_per_row: + winner = rig.decide_height() + decided = rig.snapshot(rig.restarted_server) + report["decided"] = { + "winner_hash": winner, + "restart": asdict(decided), + } + read_heap("after_decide") + if args.drain_per_row: + phase_started = time.monotonic() + drain = rig.drain_per_row( + rig.restarted_server, + stop_before_hash=winner, + ) + phase_seconds["drain_per_row"] = round(time.monotonic() - phase_started, 6) + read_heap("after_drain") + # The abandoned/submitted hash sets are the programmatic output + # and are storm-sized; only the bounded sets are printed. + report["decided"]["per_row_drain"] = { + "rounds": drain.rounds, + "accounting_tasks": drain.accounting_tasks, + "replay_enumerations": drain.replay_enumerations, + "abandoned_rows": drain.abandoned_rows, + "submitted_rows": drain.submitted_rows, + "pending_rows": drain.pending_rows, + "pending_hashes": sorted(drain.pending_hashes), + "withheld_hashes": sorted(drain.withheld_hashes), + "lease_blocked_hashes": sorted(drain.lease_blocked_hashes), + "deferred_hashes": sorted(drain.deferred_hashes), + "rpc_calls": drain.rpc_calls, + "ledger_attempt_marks": drain.ledger_attempt_marks, + "wall_seconds": drain.wall_seconds, + } + if args.cleanup_fault: + faults: list[str] = [] + for fault in args.cleanup_fault: + if fault == "all": + faults.extend([CLEANUP_FAULT_NONE, *CLEANUP_FAULTS]) + else: + faults.append(fault) + runs: list[dict[str, Any]] = [] + clean: CleanupBacklogReport | None = None + for fault in dict.fromkeys(faults): + # A fresh storm per run: the backlog and its memory are only + # comparable across runs that started from the same state. + fault_rig = CandidateStormRig( + candidates=args.candidates, + queue_depth=args.queue_depth, + quiet=not args.verbose, + credit_shares=args.credit_shares, + backlog_max=args.backlog_max, + ) + measured = fault_rig.measure_cleanup_backlog( + fault, + view=args.cleanup_view, + trace_memory=not args.no_tracemalloc, + ) + if fault == CLEANUP_FAULT_NONE: + clean = measured + entry = asdict(measured) + entry["recovered_cleanly"] = measured.recovered_cleanly + entry["traced_marginal_bytes_per_record"] = ( + cleanup_backlog_marginal_bytes_per_record(measured, clean) + if clean is not None and fault != CLEANUP_FAULT_NONE + else None + ) + entry["traced_owner_marginal_bytes_per_record"] = ( + cleanup_backlog_marginal_bytes_per_record( + measured, + clean, + owner_only=True, + ) + if clean is not None and fault != CLEANUP_FAULT_NONE + else None + ) + runs.append(entry) + report["cleanup_backlog"] = { + "python": sys.version.split()[0], + "platform": sys.platform, + "runs": runs, + } + read_heap("after_cleanup_faults") + return report + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument( + "--candidates", + type=int, + default=OBSERVED_TESTNET_CANDIDATE_STORM, + ) + parser.add_argument( + "--queue-depth", + type=int, + default=MAX_PENDING_BLOCK_CANDIDATES, + ) + parser.add_argument( + "--verbose", + action="store_true", + help="show coordinator coalescing and replay logs before the JSON report", + ) + parser.add_argument( + "--decide", + action="store_true", + help="after restart enumeration, make the last candidate the active " + "block at the storm height and report ownership/evidence gauges", + ) + parser.add_argument( + "--drain-per-row", + action="store_true", + help="with --decide, also drive the shipped per-row disposition path " + "over every sibling on the restarted coordinator and report its cost", + ) + parser.add_argument( + "--cleanup-fault", + action="append", + choices=(CLEANUP_FAULT_NONE, "all", *CLEANUP_FAULTS), + help="issue #198: run the shipped collapse walk under this injected " + "cleanup fault on a fresh storm, measure the retry backlog, and prove " + "its recovery; repeatable, 'all' runs the clean baseline and every fault", + ) + parser.add_argument( + "--cleanup-view", + choices=("live", "restart"), + default="restart", + help="coordinator view the cleanup-fault runs use (default: restart)", + ) + parser.add_argument( + "--backlog-max", + type=int, + default=None, + help="pin the cleanup-retry admission bound for the cleanup-fault runs " + f"(default: the shipped {DEFAULT_BLOCK_CANDIDATE_CLEANUP_RETRY_BACKLOG_MAX})", + ) + parser.add_argument( + "--credit-shares", + action="store_true", + help="seed credit-bearing candidates so the cleanup-fault runs exercise " + "real pending-share floor authority", + ) + parser.add_argument( + "--no-tracemalloc", + action="store_true", + help="skip the tracemalloc reading in the cleanup-fault runs", + ) + parser.add_argument( + "--heap-report", + action="store_true", + help="issue #226: report resident set, glibc arena, and anonymous-map " + "readings at every phase, after the storm is released and collected, " + "and after one malloc_trim (host-specific numbers; the platform is " + "reported beside them)", + ) + parser.add_argument( + "--arena-probe-threads", + type=int, + default=0, + help="with --heap-report, run this many allocating threads after the " + "storm to reproduce the per-thread arena shape (0 skips the probe)", + ) + parser.add_argument( + "--arena-probe-rounds", + type=int, + default=2_000, + help="medium-buffer allocations per arena-probe thread (default 2000)", + ) + parser.add_argument( + "--allocator-experiment", + default=None, + help="comma-separated MALLOC_ARENA_MAX settings ('default', '2', '4'); " + "runs this instrument once per setting in a child process with " + "--heap-report and reports every child's heap section side by side", + ) + parser.add_argument( + "--allocator-mmap-threshold", + type=int, + default=None, + help="with --allocator-experiment, also pin MALLOC_MMAP_THRESHOLD_ in " + "every child (which switches glibc's dynamic threshold off)", + ) + args = parser.parse_args() + if args.allocator_experiment: + settings = [ + token.strip() + for token in args.allocator_experiment.split(",") + if token.strip() + ] + runs = run_allocator_experiment( + settings, + allocator_experiment_child_args(sys.argv[1:]), + mmap_threshold=args.allocator_mmap_threshold, + ) + print( + json.dumps( + { + "allocator_experiment": { + "python": sys.version.split()[0], + "platform": sys.platform, + "libc": " ".join(platform.libc_ver()), + "machine": platform.machine(), + "cpu_count": os.cpu_count(), + "runs": runs, + } + }, + indent=2, + sort_keys=True, + ) + ) + return 0 + telemetry = ProcessHeapTelemetry() if args.heap_report else None + readings: list[HeapReading] = [] + phase_seconds: dict[str, float] = {} + instrument_started = time.monotonic() + + def read_heap(phase: str) -> None: + if telemetry is not None: + readings.append(take_heap_reading(phase, telemetry, instrument_started)) + + read_heap("start") + # The storm runs in its own frame so that, when it returns, nothing in + # this function still references the rig, the drain, or a fault run. + report = _run_storm_phases(args, read_heap, phase_seconds) + if telemetry is not None: + probe: dict[str, Any] | None = None + if args.arena_probe_threads > 0: + probe = run_arena_probe( + args.arena_probe_threads, + args.arena_probe_rounds, + telemetry=telemetry, + ) + read_heap("after_arena_probe") + # #185's question: once the storm's own objects are gone, does the + # process return to its pre-storm size? Every reference to the storm + # died with _run_storm_phases' frame; collect, and read. The gap + # between "after_release_gc" and "start" that survives is what the + # interpreter still holds (allocated_blocks) plus what glibc has not + # returned (malloc_free_bytes and the resident set); the trim that + # follows shows how much of the latter one malloc_trim recovers. + gc.collect() + read_heap("after_release_gc") + trim = MallocTrimmer(telemetry, log=lambda _line: None).trim_once("storm") + read_heap("after_malloc_trim") + report["heap"] = { + "python": sys.version.split()[0], + "platform": sys.platform, + "libc": " ".join(platform.libc_ver()), + "machine": platform.machine(), + "cpu_count": os.cpu_count(), + "allocator_env": allocator_environment(), + "phase_seconds": phase_seconds, + "readings": [asdict(reading) for reading in readings], + "arena_probe": probe, + "malloc_trim": trim.as_dict() if trim is not None else None, + } + print(json.dumps(report, indent=2, sort_keys=True)) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tests/prism_concurrency_harness.py b/tests/prism_concurrency_harness.py new file mode 100644 index 000000000..f5dc91e8d --- /dev/null +++ b/tests/prism_concurrency_harness.py @@ -0,0 +1,4175 @@ +#!/usr/bin/env python3 +"""Deterministic concurrency harness for PRISM's stateful owners. + +Why this exists +--------------- +The concurrency defects this line keeps shipping (tip-refresh livelock, +publication starvation, lease heartbeat self-fencing, the found-block landing +livelock, the orphaned lease-tuple lock in #123) share a shape: two owners +interleave around a PostgreSQL row, and the bad schedule is rare enough that +running the coordinator and hoping never produces it. Reviews caught them; +tests could not express them. + +This module makes the schedule an input rather than an accident. Four pieces: + +``DeterministicScheduler`` + Real threads, one baton. Exactly one actor runs at a time, and it only + stops at named checkpoints the harness places at PostgreSQL statement + boundaries. The controlling test decides who runs next, so an + interleaving is *reproducible and order-stable*, not merely likely. The + scheduler records a trace of every checkpoint, which is what determinism + is asserted against. + +``VirtualClock`` + All time the lease lifecycle measures about itself — adoption silence + from advisory-guard acquisition, caller deadlines, TTL expiry, retry + sleeps — reads this clock. Nothing sleeps in wall-clock time. Tests + advance it explicitly, or call ``advance_to_next_deadline`` to jump to + the earliest pending timeout. + +``FakePostgres`` + An in-memory model of the tables PRISM's owners contend on, with the + PostgreSQL semantics those owners actually depend on: exclusive tuple + locks held for the lifetime of a transaction, ``FOR NO KEY UPDATE SKIP + LOCKED`` declining to queue, per-statement READ COMMITTED snapshots, + ``lock_timeout``/``statement_timeout``, session-scoped advisory locks, + and ``pg_stat_activity`` locker attribution against a snapshot-visible + ``xmax``. Explicit transactions are modelled as BEGIN / statement / + COMMIT with a checkpoint in between, because that gap is the whole of + #123. + +``HarnessLock`` / ``HarnessRLock`` / ``HarnessCondition`` + Process-local synchronisation the scheduler can see. A contended acquire + parks at a named checkpoint instead of wedging the baton, and a timed + wait measures the virtual clock. The lease scenarios never needed these: + they interleave two *processes* around one row, so no two actors ever + contended a lock inside one coordinator. The landing topology does — + the payout-balance serializer, the publication order guard and the + per-hash disposition lease are exactly what its two tails share. + +Substitution goes through ports, not monkeypatching +--------------------------------------------------- +``PsqlShareLedger`` takes ``sql_backend_factory``, ``lease_guard_factory``, +``monotonic`` and ``lease_retry_sleep``. The fakes here implement +``LedgerSqlPort`` and ``LeaseGuardPort``. No bound method is reassigned and +no ``time`` module is patched, so the code under test is the shipped code: +the real retry loop, the real CAS SQL, the real adoption arithmetic, the real +fail-closed branches. + +The synchronisation primitives are the one deliberate exception, and they are +a different kind of substitution: a harness lock preserves the semantics of +the lock it replaces exactly and changes only how a waiter blocks, so the +decision logic under test is untouched. Landing scenarios install them on the +coordinator's lock attributes at construction, the same way the coordinator +itself installs a ``threading.RLock`` there. + +What is modelled and what is not +-------------------------------- +Modelled: the writer-lease statements ``PsqlShareLedger`` emits — startup +acquisition, same-identity adoption, renewal, release, and the guard-session +verification probe — plus the landing statements it emits against +``qbit_pool_blocks`` and ``qbit_block_candidate_outbox``: prepared-row +persistence, the ``qbit_confirm_pool_block`` publication-ordinal allocator +and its read-back, the durable ordinal floor, pool-block state, prepared +rejection and the outbox terminal update. + +Not modelled: everything else. ``FakePostgres`` classifies each statement and +raises ``UnsupportedStatement`` on anything it does not recognise, including +the share-ledger statements, which belong to the follow-up state machines +named in issue #128. That is deliberate: a fake that silently answers an +unrecognised statement drifts away from production without anyone noticing, +so this one fails loudly instead. The classifier is anchored on fragments of +the production SQL, so a change to that SQL breaks classification rather than +quietly changing meaning. + +Where a modelled statement is a large CTE — ``persist_accepted_block`` writes +pool-block, bundle, payout-entry and carry-forward rows in one statement — +the model reproduces the effects the landing decisions read back, not the +whole statement. The classifier still pins the statement's identity on its +production fragments, so the day persist grows an effect a landing consults, +that fragment set is where the model is extended. +""" + +from __future__ import annotations + +import base64 +import contextlib +import hashlib +import io +import json +import os +import re +import sys +import threading +import uuid +from collections.abc import Callable +from dataclasses import dataclass, field, replace +from datetime import datetime, timedelta, timezone +from enum import Enum +from pathlib import Path +from typing import TYPE_CHECKING, Any + +if TYPE_CHECKING: + from typing import Self + +sys.path.insert(0, str(Path(__file__).resolve().parents[1])) + +from lab.prism.candidate_codec import ( # noqa: E402 + INDEX_PAGE_ENTRIES, + INDEX_SPAN_ROWS, + replay_header_from_fields, +) +from lab.prism.candidate_store import BODY_READ_MAX_CHUNKS # noqa: E402 +from lab.prism.share_ledger import ( # noqa: E402 + PAYOUT_WINDOW_ROW_BATCH_SIZE, + WRITER_LEASE_HEARTBEAT_SESSION_PREFIX, + LedgerOperationTimeout, + PsqlShareLedger, + parse_single_json_value, +) +from lab.prism.writer_lease_timing import ( # noqa: E402 + WRITER_LEASE_GUARD_STATEMENT_TIMEOUT_SECONDS, +) + +# Real seconds the controller will wait for an actor to hand the baton back +# before declaring the harness itself broken. It is a failure detector, not +# an ordering input: no schedule depends on it, so it cannot change what a +# run does. An actor that blocks on a resource the harness does not model (a +# real threading.Lock, a real socket) trips it instead of hanging the suite. +# It is not free of consequence — an actor that genuinely took this long to +# reach its next checkpoint would fail a run that ought to pass — but the +# margin against microseconds of work is enormous. The env override exists +# for heavily oversubscribed CI hosts where 20s of scheduler stall is a real +# false red; it widens the failure detector only and never affects ordering. +BATON_TIMEOUT_SECONDS = float( + os.environ.get("PRISM_HARNESS_BATON_TIMEOUT_SECONDS", "20.0") +) + +# Virtual epoch origin for clock_timestamp(). Fixed so timestamp text is +# byte-identical across runs, which matters: the adoption CAS compares +# updated_at as text. +CLOCK_ORIGIN = datetime(2026, 6, 26, 19, 49, 22, 233718, tzinfo=timezone.utc) + +# The guard session's server-side statement_timeout, taken from the shipped +# timing policy rather than repeated, so a change to the bound the heartbeat +# envelope is sized against also changes what this model enforces. +GUARD_STATEMENT_TIMEOUT_SECONDS = WRITER_LEASE_GUARD_STATEMENT_TIMEOUT_SECONDS + + +class HarnessError(AssertionError): + """The harness could not model what the code under test asked for.""" + + +class UnsupportedStatement(HarnessError): + """FakePostgres was handed SQL it does not model.""" + + +class SchedulerStall(HarnessError): + """An actor did not reach a checkpoint within the baton timeout.""" + + +class NotRunnable(HarnessError): + """The controller tried to step an actor that cannot make progress.""" + + +class _ActorAbort(BaseException): + """Unwinds a parked actor thread at harness teardown.""" + + +# -------------------------------------------------------------------------- +# Virtual clock +# -------------------------------------------------------------------------- + + +class VirtualClock: + """Monotonic and wall clock that only moves when a test says so.""" + + def __init__(self, *, monotonic_origin: float = 10_000.0) -> None: + self._monotonic = float(monotonic_origin) + self._origin_monotonic = float(monotonic_origin) + + def monotonic(self) -> float: + return self._monotonic + + def now(self) -> datetime: + """``clock_timestamp()``: wall time, advancing with the monotonic clock.""" + return CLOCK_ORIGIN + timedelta( + seconds=self._monotonic - self._origin_monotonic + ) + + def advance(self, seconds: float) -> float: + seconds = float(seconds) + if seconds < 0: + raise HarnessError("virtual time cannot move backwards") + self._monotonic += seconds + return self._monotonic + + def advance_to(self, monotonic_target: float) -> float: + if monotonic_target < self._monotonic: + raise HarnessError("virtual time cannot move backwards") + self._monotonic = float(monotonic_target) + return self._monotonic + + @staticmethod + def timestamp_text(moment: datetime) -> str: + """Render a timestamptz the way psql/psycopg hand it back as text.""" + return f"{moment:%Y-%m-%d %H:%M:%S}.{moment.microsecond:06d}+00" + + +# -------------------------------------------------------------------------- +# Scheduler +# -------------------------------------------------------------------------- + + +@dataclass +class Call: + """One unit of work handed to an actor.""" + + label: str + fn: Callable[[], Any] + done: bool = False + result: Any = None + error: BaseException | None = None + + def value(self) -> Any: + """Return the result, re-raising whatever the actor raised.""" + if not self.done: + raise HarnessError(f"call {self.label!r} has not finished") + if self.error is not None: + raise self.error + return self.result + + +class Actor: + """One logical thread of the system under test. + + An actor owns a work queue. It parks at the ``idle`` checkpoint between + items, so a scenario reads as a sequence of ``call``/``start`` steps + rather than as thread plumbing. + """ + + def __init__(self, scheduler: DeterministicScheduler, name: str) -> None: + self.scheduler = scheduler + self.name = name + self.queue: list[Call] = [] + self.calls: list[Call] = [] + self.stop: str | None = None + self.finished = False + self.parked_forever = False + self._ready: Callable[[], bool] | None = None + self._wake_at: float | None = None + self._blocked = False + self._thread = threading.Thread( + target=self._run, + name=f"harness-{name}", + daemon=True, + ) + self._started = False + + # -- controller side --------------------------------------------------- + + def submit(self, fn: Callable[[], Any], *, label: str | None = None) -> Call: + call = Call(label=label or getattr(fn, "__name__", "call"), fn=fn) + self.queue.append(call) + self.calls.append(call) + return call + + @property + def blocked(self) -> bool: + return self._blocked + + @property + def block_reason(self) -> str | None: + return self.stop if self._blocked else None + + @property + def wake_at(self) -> float | None: + return self._wake_at if self._blocked else None + + def runnable(self) -> bool: + if self.finished or self.parked_forever: + return False + if not self._blocked: + return True + if self._ready is not None and self._ready(): + return True + return ( + self._wake_at is not None + and self.scheduler.clock.monotonic() >= self._wake_at + ) + + # -- actor side -------------------------------------------------------- + + def _run(self) -> None: + try: + self.scheduler._await_baton(self) + while True: + while not self.queue: + # Blocked rather than checkpointed: an actor with nothing + # to do is not runnable, so it neither consumes a drain + # round nor masks the fact that the system has quiesced. + self.scheduler.block("idle", ready=lambda: bool(self.queue)) + call = self.queue.pop(0) + try: + call.result = call.fn() + except BaseException as exc: # recorded, not swallowed + if isinstance(exc, _ActorAbort): + raise + call.error = exc + finally: + call.done = True + self.scheduler.checkpoint(f"done:{call.label}") + except _ActorAbort: + pass + finally: + self.finished = True + self.stop = "finished" + self.scheduler._release_baton(self) + + +class DeterministicScheduler: + """Baton-passing scheduler: one runnable actor, chosen by the test.""" + + def __init__( + self, + clock: VirtualClock, + *, + baton_timeout_seconds: float = BATON_TIMEOUT_SECONDS, + ) -> None: + self.clock = clock + self.baton_timeout_seconds = float(baton_timeout_seconds) + self.actors: dict[str, Actor] = {} + self.trace: list[str] = [] + self._condition = threading.Condition() + self._current: Actor | None = None + self._local = threading.local() + self._closed = False + + # -- construction ------------------------------------------------------ + + def actor(self, name: str) -> Actor: + if name in self.actors: + raise HarnessError(f"actor {name!r} already exists") + actor = Actor(self, name) + self.actors[name] = actor + return actor + + # -- actor-side primitives -------------------------------------------- + + def current(self) -> Actor: + actor = getattr(self._local, "actor", None) + if actor is None: + raise HarnessError( + "this call must run on an actor thread; drive it through " + "Actor.submit/LeaseHarness.run rather than from the test body" + ) + return actor + + def checkpoint(self, label: str) -> None: + """Announce a stop and hand the baton back to the controller.""" + actor = self.current() + actor.stop = label + actor._blocked = False + actor._ready = None + actor._wake_at = None + self.trace.append(f"{actor.name}@{label}") + self._yield(actor) + + def block( + self, + label: str, + *, + ready: Callable[[], bool] | None = None, + wake_at: float | None = None, + ) -> None: + """Park at ``label`` until ``ready()`` holds or the clock reaches ``wake_at``. + + Returns once the controller schedules this actor again; the caller + re-checks its own condition, so a spurious resume is harmless. + """ + actor = self.current() + actor.stop = label + actor._blocked = True + actor._ready = ready + actor._wake_at = wake_at + self.trace.append(f"{actor.name}@{label}") + self._yield(actor) + actor._blocked = False + actor._ready = None + actor._wake_at = None + + def park_forever(self, label: str) -> None: + """Abandon this actor at ``label``; it never runs again. + + Models a client that vanished: the process is gone, but nothing it + held server-side has been released. + """ + actor = self.current() + actor.stop = label + actor.parked_forever = True + actor._blocked = True + actor._ready = lambda: False + actor._wake_at = None + self.trace.append(f"{actor.name}@{label}") + self._yield(actor) + raise HarnessError("a parked actor must never be resumed") + + # -- baton mechanics --------------------------------------------------- + + def _yield(self, actor: Actor) -> None: + with self._condition: + self._current = None + self._condition.notify_all() + while self._current is not actor: + if self._closed: + raise _ActorAbort() + self._condition.wait(0.05) + + def _await_baton(self, actor: Actor) -> None: + self._local.actor = actor + with self._condition: + while self._current is not actor: + if self._closed: + raise _ActorAbort() + self._condition.wait(0.05) + + def _release_baton(self, actor: Actor) -> None: + with self._condition: + if self._current is actor: + self._current = None + self._condition.notify_all() + + # -- controller-side stepping ----------------------------------------- + + def step(self, actor: Actor) -> str: + """Run ``actor`` until its next checkpoint. Returns the stop label.""" + if actor.finished: + raise NotRunnable(f"actor {actor.name!r} has finished") + if not actor.runnable(): + raise NotRunnable( + f"actor {actor.name!r} is blocked at {actor.stop!r} and cannot " + "make progress; advance the clock or let another actor commit" + ) + if not actor._started: + actor._started = True + actor._thread.start() + with self._condition: + self._current = actor + self._condition.notify_all() + waited = self._condition.wait_for( + lambda: self._current is not actor, + timeout=self.baton_timeout_seconds, + ) + if not waited: + raise SchedulerStall( + f"actor {actor.name!r} did not reach a checkpoint within " + f"{self.baton_timeout_seconds:g}s (last stop {actor.stop!r}). It is " + "probably blocked on a resource the harness does not model." + ) + return actor.stop or "finished" + + def run_until(self, actor: Actor, label: str, *, limit: int = 200) -> str: + """Step ``actor`` until it stops at ``label``.""" + for _ in range(limit): + stop = self.step(actor) + if stop == label: + return stop + raise HarnessError( + f"actor {actor.name!r} did not reach {label!r} within {limit} steps; " + f"last stop {actor.stop!r}" + ) + + def runnable_actors(self) -> list[Actor]: + return [actor for actor in self.actors.values() if actor.runnable()] + + def next_deadline(self) -> float | None: + """Earliest virtual monotonic time at which a blocked actor wakes.""" + deadlines = [ + actor.wake_at + for actor in self.actors.values() + if actor.blocked + and not actor.parked_forever + and actor.wake_at is not None + ] + return min(deadlines) if deadlines else None + + def close(self) -> None: + with self._condition: + self._closed = True + self._current = None + self._condition.notify_all() + for actor in self.actors.values(): + if actor._started: + actor._thread.join(timeout=2.0) + + +# -------------------------------------------------------------------------- +# Scheduler-aware synchronisation +# -------------------------------------------------------------------------- + + +class HarnessLock: + """A mutex the scheduler can see, drop-in for ``threading.Lock``. + + The code under test acquires and releases exactly as it would a real + lock; the only difference is what happens when the lock is already held. + A real lock would block the actor thread outside the baton, so the + controller would time out and report a stall on a wait that is in fact + legitimate. This one parks the waiter at ``lock:`` and lets the + controller schedule the holder, which is what makes a contended + process-local lock an *interleaving point* rather than a harness limit. + + Ownership is per actor, not per OS thread: actors are threads here, but + naming the actor is what lets the trace read as a schedule. + + Grants are FIFO. PostgreSQL's tuple-lock queue is modelled that way for + the same reason (see ``_await_lease_lock``): without a queue the model + could express a grant order that neither a real futex-backed lock nor + the scheduler would produce, and the schedule is an input here. + """ + + reentrant = False + + def __init__(self, scheduler: DeterministicScheduler, name: str) -> None: + self.scheduler = scheduler + self.name = name + self._owner: Actor | None = None + self._depth = 0 + self._waiters: list[Actor] = [] + + # -- observation ------------------------------------------------------- + + @property + def owner(self) -> Actor | None: + return self._owner + + def locked(self) -> bool: + return self._owner is not None + + def held_by_current(self) -> bool: + return self._owner is self.scheduler.current() + + # -- acquisition ------------------------------------------------------- + + def acquire(self, blocking: bool = True, timeout: float = -1) -> bool: + actor = self.scheduler.current() + if self._owner is actor: + if not self.reentrant: + raise HarnessError( + f"actor {actor.name!r} re-acquired non-reentrant lock " + f"{self.name!r}; production would deadlock here" + ) + self._depth += 1 + return True + deadline = ( + None + if timeout is None or timeout < 0 + else self.scheduler.clock.monotonic() + max(0.0, float(timeout)) + ) + + def granted() -> bool: + if self._owner is not None: + return False + return not self._waiters or self._waiters[0] is actor + + if granted(): + self._take(actor) + return True + if not blocking: + return False + self._waiters.append(actor) + try: + while not granted(): + if ( + deadline is not None + and self.scheduler.clock.monotonic() >= deadline + ): + return False + self.scheduler.block( + f"lock:{self.name}", + ready=granted, + wake_at=deadline, + ) + finally: + self._waiters.remove(actor) + self._take(actor) + return True + + def _take(self, actor: Actor) -> None: + self._owner = actor + self._depth = 1 + + def release(self) -> None: + actor = self.scheduler.current() + if self._owner is None: + raise HarnessError(f"lock {self.name!r} released while unheld") + if self._owner is not actor: + raise HarnessError( + f"lock {self.name!r} released by {actor.name!r} but held by " + f"{self._owner.name!r}" + ) + self._depth -= 1 + if self._depth == 0: + self._owner = None + + def __enter__(self) -> bool: + return self.acquire() + + def __exit__(self, *exc_info: object) -> None: + self.release() + + +class HarnessRLock(HarnessLock): + """``threading.RLock`` for actors: the owner may re-enter.""" + + reentrant = True + + +class HarnessSemaphore: + """``threading.BoundedSemaphore`` for actors. + + The ledger's read slots are a semaphore, and ``_operation_gate`` treats + it exactly like the writer lock — same ``acquire(timeout=)`` contract, + same ``LedgerOperationTimeout`` on expiry. Modelling it means a scenario + can exhaust read concurrency on purpose rather than discovering that the + harness quietly never blocks there. + """ + + def __init__( + self, + scheduler: DeterministicScheduler, + name: str, + *, + value: int = 1, + ) -> None: + self.scheduler = scheduler + self.name = name + self.initial = int(value) + self._value = int(value) + self._waiters: list[Actor] = [] + + @property + def value(self) -> int: + return self._value + + def acquire(self, blocking: bool = True, timeout: float = -1) -> bool: + actor = self.scheduler.current() + deadline = ( + None + if timeout is None or timeout < 0 + else self.scheduler.clock.monotonic() + max(0.0, float(timeout)) + ) + + def granted() -> bool: + if self._value <= 0: + return False + return not self._waiters or self._waiters[0] is actor + + if granted(): + self._value -= 1 + return True + if not blocking: + return False + self._waiters.append(actor) + try: + while not granted(): + if ( + deadline is not None + and self.scheduler.clock.monotonic() >= deadline + ): + return False + self.scheduler.block( + f"semaphore:{self.name}", + ready=granted, + wake_at=deadline, + ) + finally: + self._waiters.remove(actor) + self._value -= 1 + return True + + def release(self) -> None: + if self._value >= self.initial: + raise HarnessError(f"semaphore {self.name!r} released too many times") + self._value += 1 + + def __enter__(self) -> bool: + return self.acquire() + + def __exit__(self, *exc_info: object) -> None: + self.release() + + +class HarnessCondition: + """``threading.Condition`` over a harness lock and the virtual clock. + + ``wait`` releases the lock, parks, and re-acquires before returning, so a + predicate loop written against a real condition behaves identically. A + timed wait measures the virtual clock, which is what lets a scenario put + a heartbeat-slice wait (``_await_unfenced_appends_predating_anchor``, the + ledger admission slices) under a watchdog without anything sleeping. + """ + + def __init__( + self, + lock: HarnessLock | None = None, + *, + scheduler: DeterministicScheduler | None = None, + name: str = "condition", + ) -> None: + if lock is None: + if scheduler is None: + raise HarnessError("a condition needs a lock or a scheduler") + lock = HarnessRLock(scheduler, name) + self._lock = lock + self.scheduler = lock.scheduler + self.name = name + # Monotonic notify counter. A waiter records the value it parked on, + # so a notify that arrives while the controller has the baton is + # never lost the way a missed edge would be. + self._generation = 0 + + def __enter__(self) -> bool: + return self._lock.acquire() + + def __exit__(self, *exc_info: object) -> None: + self._lock.release() + + def acquire(self, *args: Any, **kwargs: Any) -> bool: + return self._lock.acquire(*args, **kwargs) + + def release(self) -> None: + self._lock.release() + + def wait(self, timeout: float | None = None) -> bool: + if not self._lock.held_by_current(): + raise HarnessError( + f"cannot wait on condition {self.name!r} without holding its lock" + ) + parked_generation = self._generation + depth = self._lock._depth + self._lock._depth = 1 + self._lock.release() + clock = self.scheduler.clock + wake_at = ( + None if timeout is None else clock.monotonic() + max(0.0, float(timeout)) + ) + notified = False + while True: + if self._generation != parked_generation: + notified = True + break + if wake_at is not None and clock.monotonic() >= wake_at: + break + self.scheduler.block( + f"cond:{self.name}", + ready=lambda: self._generation != parked_generation, + wake_at=wake_at, + ) + self._lock.acquire() + self._lock._depth = depth + return notified + + def wait_for( + self, + predicate: Callable[[], bool], + timeout: float | None = None, + ) -> bool: + clock = self.scheduler.clock + deadline = ( + None if timeout is None else clock.monotonic() + max(0.0, float(timeout)) + ) + while not predicate(): + if deadline is None: + self.wait() + continue + remaining = deadline - clock.monotonic() + if remaining <= 0: + break + self.wait(remaining) + return predicate() + + def notify(self, n: int = 1) -> None: + self.notify_all() + + def notify_all(self) -> None: + self._generation += 1 + + +class HarnessEvent: + """``threading.Event`` for actors, with a clock-backed ``wait``.""" + + def __init__(self, scheduler: DeterministicScheduler, name: str = "event") -> None: + self.scheduler = scheduler + self.name = name + self._set = False + + def is_set(self) -> bool: + return self._set + + def set(self) -> None: + self._set = True + + def clear(self) -> None: + self._set = False + + def wait(self, timeout: float | None = None) -> bool: + clock = self.scheduler.clock + wake_at = ( + None if timeout is None else clock.monotonic() + max(0.0, float(timeout)) + ) + while not self._set: + if wake_at is not None and clock.monotonic() >= wake_at: + break + self.scheduler.block( + f"event:{self.name}", + ready=lambda: self._set, + wake_at=wake_at, + ) + return self._set + + +# -------------------------------------------------------------------------- +# Statement classification +# -------------------------------------------------------------------------- + + +class LeaseOp(str, Enum): + ACQUIRE = "acquire" + ADOPT = "adopt" + RENEW = "renew" + RELEASE = "release" + VERIFY = "verify" + # The cheap read-only ownership proof the heartbeat runs on most beats + # (issue #212). Deliberately a distinct trace step from VERIFY: a + # scenario asserting the split has to be able to see which question the + # coordinator actually asked PostgreSQL on each beat. + PROVE = "prove" + + +class LandingOp(str, Enum): + """Statements the block-candidate landing tails emit. + + Names follow the ``PsqlShareLedger`` method that emits them rather than + the SQL verb, because a scenario reads the trace as landing steps. + """ + + PERSIST_BLOCK = "persist_block" + CONFIRM_BLOCK = "confirm_block" + CONFIRMED_SEQUENCE = "confirmed_sequence" + PUBLICATION_FLOOR = "publication_floor" + POOL_BLOCK_STATE = "pool_block_state" + REJECT_PREPARED = "reject_prepared" + OUTBOX_RECORD = "outbox_record" + OUTBOX_ATTEMPT = "outbox_attempt" + OUTBOX_FINISH = "outbox_finish" + OUTBOX_PENDING_PAGE = "outbox_pending_page" + OUTBOX_BATCH_ABANDON = "outbox_batch_abandon" + # Issue #255: chunked candidate bodies. Reads first, then the staging + # writes (no lease CTE), then the fenced orphan retirement and the + # lease-free chunk reaping. + OUTBOX_OBSERVE = "outbox_observe" + OUTBOX_HEADER_PAGE = "outbox_header_page" + BODY_MANIFEST = "body_manifest" + BODY_SPANS = "body_spans" + BODY_PAGES = "body_pages" + BODY_PAGE = "body_page" + BODY_STAGE = "body_stage" + BODY_CHUNK = "body_chunk" + BODY_INDEX = "body_index" + BODY_SEAL = "body_seal" + BODY_RETIRE_STAGING = "body_retire_staging" + BODY_RETIRE_ORPHANS = "body_retire_orphans" + BODY_REAP = "body_reap" + SCHEMA_PRESENCE = "schema_presence" + SCHEMA_CAPABILITY = "schema_capability" + ALL_SHARES = "all_shares" + PRIOR_BALANCES = "prior_balances" + PRIOR_BALANCES_AS_OF = "prior_balances_as_of" + SHARE_STATS = "share_stats" + + +StatementKind = LeaseOp | LandingOp + + +@dataclass(frozen=True) +class Statement: + kind: StatementKind + payload: dict[str, Any] + lease_ttl_seconds: float | None + authority_margin_seconds: float | None + advisory_lock: tuple[int, int] | None + sql: str + + @property + def is_landing(self) -> bool: + return isinstance(self.kind, LandingOp) + + +_PAYLOAD_RE = re.compile( + r"\$(qbit_prism_json(?:_x)*)\$(.*?)\$\1\$::jsonb", + re.DOTALL, +) +_INTERVAL_RE = re.compile(r"make_interval\(secs => ([0-9eE\.\+\-]+)\)") +_CLASSID_RE = re.compile(r"classid = (\d+)::oid") +_OBJID_RE = re.compile(r"objid = (\d+)::oid") + +# Each entry is (op, required fragments). Fragments are lifted verbatim from +# the production SQL in lab/prism/share_ledger.py, so a change there breaks +# classification loudly instead of silently changing what the fake models. +_SIGNATURES: tuple[tuple[LeaseOp, tuple[str, ...]], ...] = ( + ( + LeaseOp.PROVE, + ( + "'guard_advisory_lock_held'", + "'writer_session_token_current'", + "'lease_renewal_due'", + ), + ), + ( + LeaseOp.VERIFY, + ( + "FOR NO KEY UPDATE SKIP LOCKED", + "'guard_advisory_lock_held'", + "'lease_renewed_count'", + "'lease_locked_by_this_process'", + ), + ), + ( + LeaseOp.ACQUIRE, + ( + "INSERT INTO qbit_ledger_writer_lease", + "ON CONFLICT (singleton) DO UPDATE", + "qbit_ledger_writer_lease.lease_expires_at <= clock_timestamp()", + ), + ), + ( + LeaseOp.ADOPT, + ( + "UPDATE qbit_ledger_writer_lease", + "data->>'observed_writer_session_token'", + "data->>'observed_lease_updated_at'", + ), + ), + ( + LeaseOp.RELEASE, + ( + "UPDATE qbit_ledger_writer_lease", + "lease_expires_at = clock_timestamp() - interval '1 second'", + "json_build_object('released'", + ), + ), + ( + LeaseOp.RENEW, + ( + "UPDATE qbit_ledger_writer_lease", + "'renewed_count', (SELECT count(*) FROM lease)", + ), + ), +) + +# The landing statements, same discipline: every fragment is lifted verbatim +# from lab/prism/share_ledger.py. Ordered most specific first, because two +# pool-block reads differ only by which columns they project. +_LANDING_SIGNATURES: tuple[tuple[LandingOp, tuple[str, ...]], ...] = ( + ( + LandingOp.PERSIST_BLOCK, + ( + "INSERT INTO qbit_pool_blocks (", + "existing_block AS (", + "INSERT INTO qbit_pool_audit_bundles (", + ), + ), + ( + LandingOp.CONFIRM_BLOCK, + ("'confirmed_count', qbit_confirm_pool_block(",), + ), + ( + LandingOp.REJECT_PREPARED, + ("'rejected_count', qbit_reject_prepared_pool_block(",), + ), + ( + LandingOp.POOL_BLOCK_STATE, + ( + "'state', (", + "'maturity_state', maturity_state,", + "FROM qbit_pool_blocks", + ), + ), + ( + LandingOp.CONFIRMED_SEQUENCE, + ( + "'audit_publication_sequence', (", + "FROM qbit_pool_blocks", + "AND chain_state = 'confirmed'", + ), + ), + ( + LandingOp.PUBLICATION_FLOOR, + ( + "'audit_publication_sequence_floor',", + "COALESCE(MAX(audit_publication_sequence), 0)", + ), + ), + ( + # Issue #255: the metadata-first page. Listed before the legacy page + # because it carries every fragment of that page plus its own. + LandingOp.OUTBOX_HEADER_PAGE, + ("'header_bytes', octet_length(page.header::text)",), + ), + ( + LandingOp.OUTBOX_PENDING_PAGE, + ( + # The read-only replay enumeration (issue #211). Its fragments + # name the three facts one page must answer from one snapshot: + # the row, whether the candidate already landed, and the keyset + # cursor the next page resumes from. + "'pool_block_exists', EXISTS (", + "FROM qbit_pool_blocks pool", + "FROM qbit_block_candidate_outbox", + "WHERE state = 'pending'", + ), + ), + (LandingOp.OUTBOX_OBSERVE, ("'body_state', (",)), + (LandingOp.SCHEMA_PRESENCE, ("to_regclass('qbit_block_candidate_body')",)), + (LandingOp.SCHEMA_CAPABILITY, ("FROM qbit_prism_schema_capabilities",)), + (LandingOp.BODY_STAGE, ("INSERT INTO qbit_block_candidate_body (",)), + (LandingOp.BODY_CHUNK, ("INSERT INTO qbit_block_candidate_body_chunk (",)), + (LandingOp.BODY_INDEX, ("INSERT INTO qbit_block_candidate_body_span (",)), + (LandingOp.BODY_SEAL, ("SET state = 'sealed', sealed_at = clock_timestamp()",)), + ( + LandingOp.BODY_RETIRE_STAGING, + ("SET state = 'retired', retired_at = clock_timestamp()", "AND state = 'staging'"), + ), + ( + LandingOp.BODY_RETIRE_ORPHANS, + ("SET state = 'retired', retired_at = clock_timestamp()", "FOR UPDATE OF body"), + ), + (LandingOp.BODY_REAP, ("deleted_chunks AS (",)), + (LandingOp.BODY_MANIFEST, ("'manifest', (", "'referenced', EXISTS (")), + (LandingOp.BODY_SPANS, ("'spans', (",)), + (LandingOp.BODY_PAGES, ("'pages', (",)), + (LandingOp.BODY_PAGE, ("encode(page.chunk, 'base64')",)), + ( + LandingOp.OUTBOX_RECORD, + ( + "INSERT INTO qbit_block_candidate_outbox (", + "'block candidate payload mismatch'", + # Both fragments above also occur in the share-submission append, + # which opens the same outbox insert and raises the same mismatch + # error, so on their own they name a share append as the outbox + # state machine. Only `persist_block_candidate_intent` inserts the + # candidate with no share to attach it to and therefore leaves a + # colliding row alone; the share append claims that row with + # `ON CONFLICT (block_hash) DO UPDATE`. That difference in conflict + # action is the discriminator, and it is load-bearing rather than + # incidental: it is the SQL-level statement of which caller owns + # the row. + "ON CONFLICT (block_hash) DO NOTHING", + ), + ), + ( + LandingOp.OUTBOX_BATCH_ABANDON, + ( + # The page-oriented terminal write (issue #196), listed before + # OUTBOX_FINISH because it satisfies that signature too: the + # discriminators are the set predicate and the pool-block + # re-check, which is the clause that makes acting on a stale + # enumeration safe. + "UPDATE qbit_block_candidate_outbox", + "SET state = 'abandoned',", + "WHERE block_hash = ANY(ARRAY[", + "FROM qbit_pool_blocks pool", + ), + ), + ( + LandingOp.OUTBOX_FINISH, + ( + "UPDATE qbit_block_candidate_outbox", + "completed_at = clock_timestamp(),", + "candidate = NULL", + ), + ), + ( + LandingOp.OUTBOX_ATTEMPT, + ( + "UPDATE qbit_block_candidate_outbox", + "SET attempt_count = attempt_count + 1,", + ), + ), + ( + LandingOp.ALL_SHARES, + ( + "'share_seq', share_seq,", + "FROM qbit_share_ledger", + "WHERE accepted;", + ), + ), + ( + LandingOp.SHARE_STATS, + ( + "'accepted_share_count', count(*),", + "FROM qbit_share_ledger", + ), + ), + ( + LandingOp.PRIOR_BALANCES_AS_OF, + ( + "'recipient_id', miner_id,", + "FROM qbit_payout_carry_forward carry", + "AND block.block_height <= target.block_height", + ), + ), + ( + LandingOp.PRIOR_BALANCES, + ( + "'recipient_id', miner_id,", + "FROM qbit_current_carry_forward_balances();", + ), + ), +) + +# Statements that belong to the state machines issue #128 still defers. +# Recognised only so the failure names the right follow-up instead of reading +# as an unexplained harness gap. +_DEFERRED_SIGNATURES: tuple[tuple[str, str], ...] = ( + ("INSERT INTO qbit_share_ledger", "share submission and dedupe"), + ("qbit_ctv_fanouts", "CTV fanout broadcasting"), +) + +# Landing statement arguments. The ledger inlines them as SQL literals, so +# the model reads them back out of the statement rather than being told. +_POOL_FUNCTION_RE = re.compile( + r"qbit_(?:confirm|reject_prepared)_pool_block\(\s*" + r"'([0-9a-f]+)',\s*(-?\d+),\s*" + r"'((?:[^']|'')*)',\s*(\d+),\s*'((?:[^']|'')*)'", + re.DOTALL, +) +# The lease CTE every outbox statement opens with, which inlines the writer +# identity rather than carrying it in a jsonb payload. +_LEASE_IDENTITY_RE = re.compile( + r"writer_id = '((?:[^']|'')*)'\s*" + r"AND writer_epoch = (\d+)\s*" + r"AND writer_session_token = '((?:[^']|'')*)'", + re.DOTALL, +) +_POOL_BLOCK_HASH_RE = re.compile(r"WHERE block_hash = '([0-9a-f]+)'") +# The pending page's own two arguments: the bounded window and the optional +# keyset cursor. Both are inlined as literals by pending_block_candidate_rows. +_PENDING_PAGE_LIMIT_RE = re.compile(r"LIMIT (\d+)\n\) pending;") +_PENDING_PAGE_CURSOR_RE = re.compile( + r"AND \(created_at, block_hash\) > " + r"\('((?:[^']|'')*)'::timestamptz, '((?:[^']|'')*)'\)" +) +_OUTBOX_STATE_RE = re.compile(r"SET state = '(submitted|abandoned)'") +_OUTBOX_BATCH_HASHES_RE = re.compile( + r"WHERE block_hash = ANY\(ARRAY\[(.*?)\]::text\[\]\)", + re.DOTALL, +) +_OUTBOX_ERROR_RE = re.compile(r"last_error = (NULL|'(?:[^']|'')*')") +_OUTBOX_SHA_RE = re.compile(r"candidate_sha256 <> '([0-9a-f]+)'") +# Issue #255 literals: the body id a statement addresses, the chunk bytes a +# chunk upload inlines, and the arguments of the index/page reads. +_BODY_ID_RE = re.compile(r"body_id = '([0-9a-f]{32})'") +_CHUNK_HEX_RE = re.compile(r"decode\(\$qbit_prism_chunk\$([0-9a-f]*)\$qbit_prism_chunk\$, 'hex'\)") +_SPANS_AFTER_RE = re.compile(r"AND field > '((?:[^']|'')*)'") +_PAGES_FIELD_RE = re.compile(r"AND field = '((?:[^']|'')*)'\s*AND page_ordinal >= (\d+)") +_BODY_PAGE_ARGS_RE = re.compile(r"AND ordinal >= (\d+)\s*ORDER BY ordinal\s*LIMIT (\d+)") +_HEADER_PAGE_LIMIT_RE = re.compile(r"LIMIT (\d+)\n\),\nmeasured AS") +_HEADER_PAGE_BYTES_RE = re.compile(r"running_bytes <= (\d+)") + + +#: Terminal-arm keys that name a lease column. Nothing is visible to report a +#: holder from -- the arm exists precisely for that case -- so they answer +#: null. +_ACQUIRE_TERMINAL_NULL_KEYS = frozenset( + { + "writer_id", + "writer_epoch", + "writer_session_token", + "lease_expires_at", + "lease_updated_at", + "lease_age_seconds", + "lease_wait_seconds", + } +) + +#: Terminal-arm keys whose value is a SQL literal, taken from the statement +#: rather than restated here. ``acquired`` is false on this arm; +#: ``lease_snapshot_retry`` is the flag ``_try_acquire_writer_lease`` retries +#: on; ``lease`` and ``retry_reason`` carry the operator-facing subject and +#: explanation that replace the old "postgres query returned no JSON". +#: +#: Reading the value out of the SQL matters more than it looks. An earlier +#: draft keyed on the *name* -- anything containing "retry" answered true -- +#: which got ``lease_snapshot_retry`` right by luck and would have answered +#: ``retry_reason`` with a boolean where production emits a sentence. The +#: model now answers what the statement says, so it cannot drift from the +#: wording the ledger PR chose. +_ACQUIRE_TERMINAL_LITERAL_KEYS = frozenset( + { + "acquired", + "lease_snapshot_retry", + "lease", + "retry_reason", + } +) + + +def _split_sql_arguments(body: str) -> list[str]: + """Split a call's argument text on its top-level commas. + + Parenthesised sub-expressions and single-quoted literals are held + together, because both occur in this statement: the holder arm nests + ``GREATEST(0, EXTRACT(...))``, and the terminal arm's retry reason is a + sentence that must survive as one argument. + """ + parts: list[str] = [] + current: list[str] = [] + depth = 0 + quoted = False + index = 0 + while index < len(body): + char = body[index] + if quoted: + current.append(char) + if char == "'": + if index + 1 < len(body) and body[index + 1] == "'": + current.append("'") + index += 1 + else: + quoted = False + elif char == "'": + quoted = True + current.append(char) + elif char in "([": + depth += 1 + current.append(char) + elif char in ")]": + depth -= 1 + current.append(char) + elif char == "," and depth == 0: + parts.append("".join(current).strip()) + current = [] + else: + current.append(char) + index += 1 + remainder = "".join(current).strip() + if remainder: + parts.append(remainder) + return parts + + +def _balanced_call_body(text: str, open_index: int) -> str: + """Return the argument text of a call whose ``(`` is at ``open_index``.""" + depth = 0 + for index in range(open_index, len(text)): + char = text[index] + if char == "(": + depth += 1 + elif char == ")": + depth -= 1 + if depth == 0: + return text[open_index + 1 : index] + raise UnsupportedStatement("acquire statement has unbalanced parentheses") + + +def acquire_result_arms(sql: str) -> int: + """Count the result arms of the acquire statement's final COALESCE. + + The acquire statement ends in ``SELECT COALESCE(, , ...)``. At + this baseline there are exactly two: the ``RETURNING`` row of the upsert + (acquired) and the committed holder row (not acquired). Both can be empty + at once -- two coordinators racing the very first acquire -- and the + statement then evaluates to SQL NULL, which the ports turn into + production's "postgres query returned no JSON". + + Issue #140 adds an explicit not-acquired/retry arm so that race reports a + lease outcome instead of what reads as a driver fault. That is a change to + the shipped statement, so this model keys off the statement rather than + off a version flag or a local constant: a statement carrying a terminal + arm cannot evaluate to NULL, and continuing to model it as if it could + would let a scenario assert an outcome the server can no longer produce -- + the exact class of harness lie this suite exists to prevent. + + Counting arms rather than matching a chosen literal is deliberate: the arm + has to exist for the fix to work, whatever its author names its keys. + """ + _head, separator, tail = sql.partition("SELECT COALESCE(") + if not separator: + raise UnsupportedStatement( + "acquire statement does not end in the SELECT COALESCE this model " + "recognises; its result shape changed and the lease evaluator has " + "to be re-read against it rather than assumed:\n" + sql.strip()[:400] + ) + arms = tail.count("'acquired',") + if arms < 2: + raise UnsupportedStatement( + f"acquire statement's COALESCE names {arms} result arm(s); this " + "model knows the two-arm shape (acquired / holder) and the " + "three-or-more-arm shape issue #140 adds. Fewer than two is a " + "shape it has never seen." + ) + return arms + + +def acquire_terminal_arm_fields(sql: str) -> tuple[tuple[str, str], ...]: + """Return ``(key, raw value text)`` for the last ``json_build_object`` arm. + + ``json_build_object`` takes alternating key and value arguments, so the + pairing is the call's own structure rather than a convention this model + imposes. + """ + _head, _separator, tail = sql.partition("SELECT COALESCE(") + marker = "json_build_object(" + index = tail.rfind(marker) + if index < 0: + raise UnsupportedStatement("acquire statement's terminal arm builds no JSON object") + body = _balanced_call_body(tail, index + len(marker) - 1) + arguments = _split_sql_arguments(body) + if len(arguments) % 2: + raise UnsupportedStatement( + "the acquire statement's terminal arm names an odd number of " + f"json_build_object arguments ({len(arguments)}); it cannot be read " + "as key/value pairs" + ) + fields: list[tuple[str, str]] = [] + for key_text, value_text in zip(arguments[::2], arguments[1::2], strict=True): + if len(key_text) < 2 or not key_text.startswith("'") or not key_text.endswith("'"): + raise UnsupportedStatement( + f"the acquire statement's terminal arm names {key_text!r} where a " + "quoted key was expected" + ) + fields.append((_unquote(key_text[1:-1]), value_text)) + return tuple(fields) + + +def acquire_terminal_arm_keys(sql: str) -> tuple[str, ...]: + """Return the keys named by the last ``json_build_object`` arm.""" + return tuple(key for key, _value in acquire_terminal_arm_fields(sql)) + + +def classify(sql: str) -> Statement: + """Map one production statement onto the operation it performs.""" + for op, fragments in _SIGNATURES: + if all(fragment in sql for fragment in fragments): + if op is LeaseOp.PROVE: + # The proof renews nothing, so its only interval literal is + # the renewal-due (own-write authority) margin. + lease_ttl_seconds = None + authority_margin_seconds = _extract_interval(sql, first=True) + else: + lease_ttl_seconds = _extract_interval(sql, first=True) + authority_margin_seconds = ( + _extract_interval(sql, first=False) + if op is LeaseOp.VERIFY + else None + ) + return Statement( + kind=op, + payload=_extract_payload(sql), + lease_ttl_seconds=lease_ttl_seconds, + authority_margin_seconds=authority_margin_seconds, + advisory_lock=_extract_advisory_lock(sql), + sql=sql, + ) + for landing_op, fragments in _LANDING_SIGNATURES: + if all(fragment in sql for fragment in fragments): + return Statement( + kind=landing_op, + payload=_landing_payload(landing_op, sql), + lease_ttl_seconds=_extract_interval(sql, first=True), + authority_margin_seconds=None, + advisory_lock=None, + sql=sql, + ) + for marker, state_machine in _DEFERRED_SIGNATURES: + if marker in sql: + raise UnsupportedStatement( + f"{marker} belongs to the {state_machine} state machine, which " + "issue #128 defers to a follow-up PR; this harness models the " + "writer-lease lifecycle and block candidate landing" + ) + raise UnsupportedStatement( + "FakePostgres does not model this statement. Either the production " + "SQL changed shape (update _SIGNATURES or _LANDING_SIGNATURES) or the " + "scenario reached a state machine this harness does not cover:\n" + f"{sql.strip()[:400]}" + ) + + +def _unquote(literal: str) -> str: + """Undo ``PsqlShareLedger._text_literal`` quoting.""" + return literal.replace("''", "'") + + +def _lease_identity(op: LandingOp, sql: str) -> dict[str, Any]: + match = _LEASE_IDENTITY_RE.search(sql) + if match is None: + raise UnsupportedStatement( + f"{op.value} statement carried no writer lease identity" + ) + return { + "writer_id": _unquote(match.group(1)), + "writer_epoch": int(match.group(2)), + "writer_session_token": _unquote(match.group(3)), + } + + +def _landing_payload(op: LandingOp, sql: str) -> dict[str, Any]: + """Recover the arguments a landing statement inlined as SQL literals.""" + if op is LandingOp.PERSIST_BLOCK: + return _extract_payload(sql) + if op in {LandingOp.CONFIRM_BLOCK, LandingOp.REJECT_PREPARED}: + match = _POOL_FUNCTION_RE.search(sql) + if match is None: + raise UnsupportedStatement( + f"{op.value} statement did not name a block hash and height" + ) + return { + "block_hash": match.group(1), + "active_tip_height": int(match.group(2)), + "writer_id": _unquote(match.group(3)), + "writer_epoch": int(match.group(4)), + "writer_session_token": _unquote(match.group(5)), + } + if op in {LandingOp.POOL_BLOCK_STATE, LandingOp.CONFIRMED_SEQUENCE}: + match = _POOL_BLOCK_HASH_RE.search(sql) + if match is None: + raise UnsupportedStatement(f"{op.value} statement named no block hash") + return {"block_hash": match.group(1)} + if op is LandingOp.OUTBOX_RECORD: + hash_match = _POOL_BLOCK_HASH_RE.search(sql) + sha_match = _OUTBOX_SHA_RE.search(sql) + if hash_match is None or sha_match is None: + raise UnsupportedStatement( + "outbox record statement named no block hash or payload digest" + ) + body_match = _BODY_ID_RE.search(sql) + header_match = _PAYLOAD_RE.search(sql) + return { + "block_hash": hash_match.group(1), + "candidate_sha256": sha_match.group(1), + "body_id": None if body_match is None else body_match.group(1), + "header": ( + None if header_match is None else json.loads(header_match.group(2)) + ), + **_lease_identity(op, sql), + } + if op in { + LandingOp.BODY_STAGE, + LandingOp.BODY_INDEX, + LandingOp.BODY_SEAL, + LandingOp.BODY_RETIRE_ORPHANS, + LandingOp.BODY_REAP, + }: + return _extract_payload(sql) + if op is LandingOp.BODY_CHUNK: + chunk_match = _CHUNK_HEX_RE.search(sql) + if chunk_match is None: + raise UnsupportedStatement("chunk upload statement carried no chunk bytes") + return {**_extract_payload(sql), "chunk": bytes.fromhex(chunk_match.group(1))} + if op in { + LandingOp.OUTBOX_OBSERVE, + }: + hash_match = _POOL_BLOCK_HASH_RE.search(sql) + if hash_match is None: + raise UnsupportedStatement(f"{op.value} statement named no block hash") + return {"block_hash": hash_match.group(1)} + if op in {LandingOp.BODY_MANIFEST, LandingOp.BODY_RETIRE_STAGING}: + body_match = _BODY_ID_RE.search(sql) + if body_match is None: + raise UnsupportedStatement(f"{op.value} statement named no body id") + return {"body_id": body_match.group(1)} + if op is LandingOp.BODY_SPANS: + body_match = _BODY_ID_RE.search(sql) + after_match = _SPANS_AFTER_RE.search(sql) + if body_match is None: + raise UnsupportedStatement("span page statement named no body id") + return { + "body_id": body_match.group(1), + "after_field": None if after_match is None else _unquote(after_match.group(1)), + } + if op is LandingOp.BODY_PAGES: + body_match = _BODY_ID_RE.search(sql) + args_match = _PAGES_FIELD_RE.search(sql) + if body_match is None or args_match is None: + raise UnsupportedStatement("page-index statement named no body, field or ordinal") + return { + "body_id": body_match.group(1), + "field": _unquote(args_match.group(1)), + "from_ordinal": int(args_match.group(2)), + } + if op is LandingOp.BODY_PAGE: + body_match = _BODY_ID_RE.search(sql) + args_match = _BODY_PAGE_ARGS_RE.search(sql) + if body_match is None or args_match is None: + raise UnsupportedStatement("chunk page statement named no body, ordinal or limit") + return { + "body_id": body_match.group(1), + "from_ordinal": int(args_match.group(1)), + "max_chunks": int(args_match.group(2)), + } + if op is LandingOp.OUTBOX_HEADER_PAGE: + limit_match = _HEADER_PAGE_LIMIT_RE.search(sql) + bytes_match = _HEADER_PAGE_BYTES_RE.search(sql) + if limit_match is None or bytes_match is None: + raise UnsupportedStatement("header page statement carried no bounded LIMIT or byte cap") + cursor_match = _PENDING_PAGE_CURSOR_RE.search(sql) + return { + # The statement fetches one row beyond the cap it returns. + "limit": int(limit_match.group(1)) - 1, + "max_bytes": int(bytes_match.group(1)), + "after_cursor": ( + None + if cursor_match is None + else (_unquote(cursor_match.group(1)), _unquote(cursor_match.group(2))) + ), + } + if op in {LandingOp.SCHEMA_PRESENCE, LandingOp.SCHEMA_CAPABILITY}: + return {} + if op is LandingOp.OUTBOX_BATCH_ABANDON: + hashes_match = _OUTBOX_BATCH_HASHES_RE.search(sql) + error_match = _OUTBOX_ERROR_RE.search(sql) + if hashes_match is None: + raise UnsupportedStatement( + "outbox batch abandon statement named no block hash set" + ) + raw_error = None if error_match is None else error_match.group(1) + return { + "block_hashes": tuple( + _unquote(element.strip()[1:-1]) + for element in _split_sql_arguments(hashes_match.group(1)) + ), + "last_error": ( + None + if raw_error in (None, "NULL") + else str(raw_error)[1:-1].replace("''", "'") + ), + **_lease_identity(op, sql), + } + if op is LandingOp.OUTBOX_PENDING_PAGE: + limit_match = _PENDING_PAGE_LIMIT_RE.search(sql) + if limit_match is None: + raise UnsupportedStatement( + "pending candidate page statement carried no bounded LIMIT" + ) + cursor_match = _PENDING_PAGE_CURSOR_RE.search(sql) + return { + "limit": int(limit_match.group(1)), + "after_cursor": ( + None + if cursor_match is None + else ( + _unquote(cursor_match.group(1)), + _unquote(cursor_match.group(2)), + ) + ), + } + if op in {LandingOp.OUTBOX_ATTEMPT, LandingOp.OUTBOX_FINISH}: + hash_match = _POOL_BLOCK_HASH_RE.search(sql) + if hash_match is None: + raise UnsupportedStatement(f"{op.value} statement named no block hash") + payload: dict[str, Any] = { + "block_hash": hash_match.group(1), + **_lease_identity(op, sql), + } + if op is LandingOp.OUTBOX_FINISH: + state_match = _OUTBOX_STATE_RE.search(sql) + if state_match is None: + raise UnsupportedStatement( + "outbox finish statement named no terminal state" + ) + payload["state"] = state_match.group(1) + error_match = _OUTBOX_ERROR_RE.search(sql) + raw_error = None if error_match is None else error_match.group(1) + payload["last_error"] = ( + None + if raw_error in (None, "NULL") + else str(raw_error)[1:-1].replace("''", "'") + ) + return payload + return {} + + +def _extract_payload(sql: str) -> dict[str, Any]: + match = _PAYLOAD_RE.search(sql) + if match is None: + raise UnsupportedStatement("statement carried no jsonb payload") + payload = json.loads(match.group(2)) + if not isinstance(payload, dict): + raise UnsupportedStatement("statement payload was not an object") + return payload + + +def _extract_interval(sql: str, *, first: bool) -> float | None: + matches = _INTERVAL_RE.findall(sql) + if not matches: + return None + if first: + return float(matches[0]) + return float(matches[1]) if len(matches) > 1 else None + + +def _extract_advisory_lock(sql: str) -> tuple[int, int] | None: + classid = _CLASSID_RE.search(sql) + objid = _OBJID_RE.search(sql) + if classid is None or objid is None: + return None + return (int(classid.group(1)), int(objid.group(1))) + + +def advisory_lock_pair(key: int) -> tuple[int, int]: + """Split a 64-bit advisory key the way pg_locks reports it.""" + return ((key >> 32) & 0xFFFFFFFF, key & 0xFFFFFFFF) + + +# -------------------------------------------------------------------------- +# The PostgreSQL model +# -------------------------------------------------------------------------- + + +@dataclass +class LeaseRow: + writer_id: str + writer_epoch: int + writer_session_token: str + lease_expires_at: datetime + updated_at: datetime + # The locking transaction's xid, exactly as PostgreSQL stamps xmax on a + # row-locked tuple. Visible to a snapshot taken while the lock is held. + xmax: str | None = None + + +@dataclass +class PoolBlockRow: + """One ``qbit_pool_blocks`` row. + + ``audit_publication_sequence`` is the durable publication ordinal the + audit store gates on. It is assigned once, by the confirmation that first + moves the row to ``confirmed``, and survives every later disposition — + which is why an exact replay must not burn a second ordinal and why the + floor is a MAX over this column rather than a counter. + """ + + block_hash: str + block_height: int + parent_hash: str + chain_state: str = "prepared" + maturity_state: str = "immature" + audit_publication_sequence: int | None = None + + +@dataclass +class OutboxRow: + """One ``qbit_block_candidate_outbox`` row: the durable replay source.""" + + block_hash: str + candidate_sha256: str + # ``created_at DEFAULT clock_timestamp()``. The pending page orders and + # keysets on ``(created_at, block_hash)``, so the model has to carry the + # stamp the real column would: an ordering that came from insertion order + # would answer the cursor questions the enumeration actually asks with a + # property production does not have. + created_at: datetime = CLOCK_ORIGIN + state: str = "pending" + attempt_count: int = 0 + last_error: str | None = None + # Issue #255: a version-2 row references a sealed body and carries the + # bounded replay header; terminalization detaches the body. + storage_version: int = 1 + body_id: str | None = None + retired_body_id: str | None = None + header: dict[str, Any] | None = None + + +@dataclass +class BodyRow: + """One ``qbit_block_candidate_body`` manifest with its parts (#255). + + Chunks, spans and pages live on the manifest here for simplicity; the + model enforces the same rules the database triggers do: parts are + accepted only while ``staging``, never updated, and deleted only once + ``retired``. + """ + + body_id: str + block_hash: str + candidate_sha256: str + byte_count: int + chunk_count: int + chunk_bytes: int + share_count: int + shares_offset: int + shares_end: int + span_count: int + page_count: int + writer_id: str + writer_epoch: int + writer_session_token: str + created_at: datetime = CLOCK_ORIGIN + state: str = "staging" + sealed_at: datetime | None = None + retired_at: datetime | None = None + chunks: dict[int, tuple[bytes, str]] = field(default_factory=dict) + spans: dict[str, dict[str, Any]] = field(default_factory=dict) + pages: dict[tuple[str, int], tuple[int, int]] = field(default_factory=dict) + + def copy(self) -> "BodyRow": + return replace( + self, + chunks=dict(self.chunks), + spans={key: dict(value) for key, value in self.spans.items()}, + pages=dict(self.pages), + ) + + +@dataclass +class Transaction: + xid: str + backend: Backend + explicit: bool + holds_lease_lock: bool = False + staged_lease: LeaseRow | None = None + # Landing writes are staged for the same reason lease writes are: a + # deadline-scoped statement sends COMMIT as a separate message, so there + # is a window in which this transaction can see its own pool-block row + # and nobody else can. #133 lives inside exactly such a window, so a + # model that published the write at statement time could hide it. + staged_pool_blocks: dict[str, PoolBlockRow] = field(default_factory=dict) + staged_outbox: dict[str, OutboxRow] = field(default_factory=dict) + staged_bodies: dict[str, BodyRow] = field(default_factory=dict) + deleted_bodies: set[str] = field(default_factory=set) + orphaned: bool = False + + +@dataclass +class Backend: + """One server-side session.""" + + pid: str + application_name: str + server: FakePostgres + transaction: Transaction | None = None + closed: bool = False + advisory_locks: set[tuple[int, int]] = field(default_factory=set) + # Guard sessions carry a server-side statement_timeout; pool sessions + # take their deadline from the caller instead. + statement_timeout_seconds: float | None = None + + @property + def backend_xid(self) -> str | None: + if self.closed or self.transaction is None: + return None + return self.transaction.xid + + +class LockTimeout(RuntimeError): + """SQLSTATE 55P03 / 57014 as the ledger's callers observe them.""" + + +class FakePostgres: + """In-memory model of the writer-lease table and its locking behaviour.""" + + def __init__(self, scheduler: DeterministicScheduler, clock: VirtualClock) -> None: + self.scheduler = scheduler + self.clock = clock + self.lease: LeaseRow | None = None + self.statements: list[Statement] = [] + # Optional per-statement server time, in virtual seconds. Scenarios + # that model database latency (the rapid-block tail behind issue + # #212) install a callable here; the default None keeps every + # existing scenario's statements instantaneous. + self.statement_latency_seconds: ( + Callable[[Statement], float] | None + ) = None + # Server time charged to the statement currently being evaluated; + # the guard statements report it as server_execution_seconds. + self._last_statement_server_seconds = 0.0 + # Landing tables. Insertion-ordered dicts, because the trace and every + # assertion made against them has to be order-stable across runs. + self.pool_blocks: dict[str, PoolBlockRow] = {} + self.outbox: dict[str, OutboxRow] = {} + # Issue #255: chunked candidate bodies, keyed by body id. + self.bodies: dict[str, BodyRow] = {} + self.shares: list[dict[str, Any]] = [] + self.carry_forward_balances: list[dict[str, Any]] = [] + self._lease_lock_holder: Transaction | None = None + self._lease_lock_waiters: list[Transaction] = [] + self._backends: list[Backend] = [] + self._advisory_owners: dict[tuple[int, int], Backend] = {} + self._next_pid = 1 + self._next_xid = 1 + # qbit_audit_publication_sequence_seq. A sequence, not MAX + 1: the + # allocator burns a value per fresh confirmation and never reuses + # one, which is why the floor read below is a separate MAX. + self._audit_publication_sequence = 0 + + # -- sessions ---------------------------------------------------------- + + def connect( + self, + *, + application_name: str, + statement_timeout_seconds: float | None = None, + ) -> Backend: + backend = Backend( + pid=f"pid-{self._next_pid}", + application_name=application_name, + server=self, + statement_timeout_seconds=statement_timeout_seconds, + ) + self._next_pid += 1 + self._backends.append(backend) + return backend + + def disconnect(self, backend: Backend) -> None: + """Close a session cleanly: PostgreSQL rolls back and releases everything.""" + if backend.closed: + return + backend.closed = True + if backend.transaction is not None: + self._rollback(backend.transaction) + for key in list(backend.advisory_locks): + if self._advisory_owners.get(key) is backend: + del self._advisory_owners[key] + backend.advisory_locks.clear() + + def try_advisory_lock(self, backend: Backend, key: int) -> bool: + pair = advisory_lock_pair(key) + owner = self._advisory_owners.get(pair) + if owner is not None and not owner.closed: + return owner is backend + self._advisory_owners[pair] = backend + backend.advisory_locks.add(pair) + return True + + def live_backends(self) -> list[Backend]: + return [backend for backend in self._backends if not backend.closed] + + # -- transactions ------------------------------------------------------ + + def _begin(self, backend: Backend, *, explicit: bool) -> Transaction: + if backend.transaction is not None: + raise HarnessError( + f"backend {backend.pid} already has an open transaction" + ) + transaction = Transaction( + xid=f"xid-{self._next_xid}", + backend=backend, + explicit=explicit, + ) + self._next_xid += 1 + backend.transaction = transaction + return transaction + + def _commit(self, transaction: Transaction) -> None: + if transaction.staged_lease is not None: + self.lease = replace(transaction.staged_lease, xmax=None) + transaction.staged_lease = None + self.pool_blocks.update(transaction.staged_pool_blocks) + self.outbox.update(transaction.staged_outbox) + self.bodies.update(transaction.staged_bodies) + for body_id in transaction.deleted_bodies: + self.bodies.pop(body_id, None) + transaction.staged_pool_blocks.clear() + transaction.staged_outbox.clear() + transaction.staged_bodies.clear() + transaction.deleted_bodies.clear() + self._release_lease_lock(transaction) + transaction.backend.transaction = None + + def _rollback(self, transaction: Transaction) -> None: + transaction.staged_lease = None + # The publication ordinal is deliberately not rolled back: nextval is + # non-transactional, so an aborted confirmation burns its value and + # leaves a gap. The floor read is a MAX precisely so a gap is + # harmless. + transaction.staged_pool_blocks.clear() + transaction.staged_outbox.clear() + transaction.staged_bodies.clear() + transaction.deleted_bodies.clear() + self._release_lease_lock(transaction) + transaction.backend.transaction = None + + def _release_lease_lock(self, transaction: Transaction) -> None: + if self._lease_lock_holder is transaction: + self._lease_lock_holder = None + if self.lease is not None and self.lease.xmax == transaction.xid: + self.lease = replace(self.lease, xmax=None) + + def orphan(self, transaction: Transaction) -> None: + """The client vanished without RST: the transaction stays open. + + PostgreSQL finished executing the statement and is waiting for a + COMMIT that will never arrive, so it keeps the tuple lock. + + This model does not implement a reaper. Production now sets + ``idle_in_transaction_session_timeout`` + (``PRISM_POSTGRES_IDLE_IN_TRANSACTION_TIMEOUT_SECONDS``, 15s by + default) on every coordinator session, so a real orphan of *this* + deployment's making is aborted and its locks released. An orphan here + is therefore the worst case rather than the only case: the foreign + session the guard cannot reach, or the window before it fires. + Scenarios that want the reap must model it explicitly; nothing in the + harness ends an orphan on its own. + """ + transaction.orphaned = True + + @property + def lease_lock_holder(self) -> Transaction | None: + return self._lease_lock_holder + + def lease_lock_held_by_other(self, transaction: Transaction | None) -> bool: + holder = self._lease_lock_holder + return holder is not None and holder is not transaction + + # -- statement execution ---------------------------------------------- + + def execute( + self, + backend: Backend, + sql: str, + *, + transaction: Transaction | None, + timeout_seconds: float | None, + tag: str, + ) -> Any: + """Run one statement, stopping at every interleaving point it offers.""" + if backend.closed: + raise RuntimeError("connection is closed") + statement = classify(sql) + self.statements.append(statement) + autocommit = transaction is None + if autocommit: + transaction = self._begin(backend, explicit=False) + + self.scheduler.checkpoint(f"{tag}.begin:{statement.kind.value}") + + # A caller deadline arrives as SET LOCAL lock_timeout; the guard + # session's bound is a connection-level statement_timeout. PostgreSQL + # reports them with different messages and SQLSTATEs, and the whole + # point of the SKIP LOCKED heartbeat was that the guard's bound is the + # fatal one, so the model must not blur them. + effective_timeout = timeout_seconds + timeout_kind = "lock" + session_bound = backend.statement_timeout_seconds + if session_bound is not None and ( + effective_timeout is None or session_bound <= effective_timeout + ): + effective_timeout = session_bound + timeout_kind = "statement" + try: + # The modelled server time is also what the guard statements + # report about themselves (issue #227): production computes + # clock_timestamp() - statement_timestamp() inside the + # statement, and here that is exactly the latency just charged. + self._last_statement_server_seconds = self._charge_execution_latency( + statement, + effective_timeout, + timeout_kind, + ) + result = self._evaluate( + statement, + transaction, + effective_timeout, + tag, + timeout_kind, + ) + except BaseException: + if autocommit: + self._rollback(transaction) + raise + + self.scheduler.checkpoint(f"{tag}.done:{statement.kind.value}") + if autocommit: + self._commit(transaction) + return result + + def _charge_execution_latency( + self, + statement: Statement, + timeout_seconds: float | None, + timeout_kind: str, + ) -> float: + """Spend modelled server time for this statement, honouring its bound. + + Returns the seconds charged, which the guard statements report back + as their own ``server_execution_seconds`` (issue #227). + + Lock waits were already modelled (``_await_lease_lock``); this is the + other half — a statement that is simply *slow*, which is what the + rapid-block bursts behind issue #212 produce and what the heartbeat's + staleness envelope has to absorb. Advancing the virtual clock inside + the statement is exact under the baton scheduler: no other actor is + running, so the time is charged to this statement alone and every + other actor's deadline sees it on its next turn. + + A statement whose modelled cost exceeds the session bound is + cancelled at the bound, exactly as PostgreSQL's statement_timeout + cancels it — the guard session's 0.5s ceiling is the term the + heartbeat policy is sized against, so a scenario must not be able to + model a guarded round trip that quietly outlives it. + """ + latency_for = self.statement_latency_seconds + if latency_for is None: + return 0.0 + latency = float(latency_for(statement) or 0.0) + if latency <= 0.0: + return 0.0 + if timeout_seconds is not None and latency > timeout_seconds: + self.clock.advance(max(0.0, timeout_seconds)) + raise LockTimeout( + f"canceling statement due to {timeout_kind} timeout" + ) + self.clock.advance(latency) + return latency + + def _evaluate( + self, + statement: Statement, + transaction: Transaction, + timeout_seconds: float | None, + tag: str, + timeout_kind: str = "lock", + ) -> Any: + if statement.is_landing: + return self._evaluate_landing( + statement, + transaction, + timeout_seconds, + tag, + timeout_kind, + ) + if statement.kind is LeaseOp.VERIFY: + return self._evaluate_verify(statement, transaction, tag) + if statement.kind is LeaseOp.PROVE: + return self._evaluate_prove(statement, transaction) + + # READ COMMITTED takes one snapshot per command, *before* any lock + # wait. This is the distinction that makes a queued statement report + # stale data: after the wait, only the conflicting tuple is re-read + # (EvalPlanQual) to re-evaluate the write predicate, while every other + # scan in the statement — including the arm that builds the whole + # not-acquired result — still reads `snapshot`. A model that re-read + # committed state everywhere would report a fresh row and a small + # lease_age_seconds where the real server reports the pre-wait row and + # an age that has silently absorbed the entire wait. + snapshot = self._visible_lease(transaction) + + if statement.kind is LeaseOp.ACQUIRE: + # INSERT ... ON CONFLICT DO UPDATE locks the conflicting tuple + # before evaluating its WHERE, so it queues even when the + # predicate would exclude the row. + self._await_lease_lock( + statement, + transaction, + timeout_seconds, + tag, + timeout_kind, + ) + return self._evaluate_acquire(statement, transaction, snapshot) + + # A plain UPDATE evaluates its WHERE against the snapshot version + # first and never locks a row the predicate already excludes: it + # returns zero rows immediately rather than queueing. Getting this + # wrong matters to any fencing argument, where "the predecessor is + # blocked" and "the predecessor already knows it lost" are different + # states. + evaluators = { + LeaseOp.ADOPT: (self._adopt_matches, self._evaluate_adopt), + LeaseOp.RENEW: (self._renew_matches, self._evaluate_renew), + LeaseOp.RELEASE: (self._release_matches, self._evaluate_release), + } + try: + matches, evaluate = evaluators[statement.kind] + except KeyError: + raise UnsupportedStatement( + f"unhandled lease operation {statement.kind}" + ) from None + if snapshot is None or not matches(statement, snapshot): + return evaluate(statement, transaction, None) + self._await_lease_lock( + statement, + transaction, + timeout_seconds, + tag, + timeout_kind, + ) + # EvalPlanQual: the qual is re-checked against the version this + # statement just locked, which a concurrent commit may have moved. + current = self._visible_lease(transaction) + if current is None or not matches(statement, current): + return evaluate(statement, transaction, None) + return evaluate(statement, transaction, current) + + def _await_lease_lock( + self, + statement: Statement, + transaction: Transaction, + timeout_seconds: float | None, + tag: str, + timeout_kind: str = "lock", + ) -> None: + # Waiting is not conditional on a *committed* row existing. Two + # concurrent INSERTs conflict on the singleton unique index, and the + # second one queues behind the first exactly as an ON CONFLICT + # DO UPDATE queues behind an uncommitted update. + deadline = ( + None + if timeout_seconds is None + else self.clock.monotonic() + max(0.0, timeout_seconds) + ) + + def granted() -> bool: + # PostgreSQL serializes tuple-lock waiters through a heavyweight + # lock and grants in arrival order, so a later arrival cannot + # overtake a queued waiter. Without the queue the model could + # express a grant order the server never produces — which matters + # precisely because the schedule is an input here. + if self.lease_lock_held_by_other(transaction): + return False + return ( + not self._lease_lock_waiters + or self._lease_lock_waiters[0] is transaction + ) + + if granted(): + self._take_lease_lock(transaction) + return + self._lease_lock_waiters.append(transaction) + try: + while not granted(): + if deadline is not None and self.clock.monotonic() >= deadline: + raise LockTimeout( + f"canceling statement due to {timeout_kind} timeout\n" + "CONTEXT: while updating tuple (0,1) in relation " + '"qbit_ledger_writer_lease"' + ) + self.scheduler.block( + f"{tag}.lockwait:{statement.kind.value}", + ready=granted, + wake_at=deadline, + ) + finally: + self._lease_lock_waiters.remove(transaction) + self._take_lease_lock(transaction) + + def _take_lease_lock(self, transaction: Transaction) -> None: + if self.lease_lock_held_by_other(transaction): + raise HarnessError( + "the lease tuple lock was taken while another transaction " + "held it; the model must queue, never steal" + ) + self._lease_lock_holder = transaction + transaction.holds_lease_lock = True + if self.lease is not None: + self.lease = replace(self.lease, xmax=transaction.xid) + + def _visible_lease(self, transaction: Transaction) -> LeaseRow | None: + """READ COMMITTED: the statement sees committed state plus its own writes.""" + if transaction.staged_lease is not None: + return transaction.staged_lease + return self.lease + + def _stage(self, transaction: Transaction, row: LeaseRow) -> None: + transaction.staged_lease = row + if not transaction.explicit: + return + # An explicit transaction's write is invisible to everyone else until + # COMMIT, but the tuple lock is already taken; reflect the locker in + # the committed row's xmax so pg_stat_activity attribution works. + if self.lease is not None: + self.lease = replace(self.lease, xmax=transaction.xid) + + # -- individual operations -------------------------------------------- + + def _evaluate_acquire( + self, + statement: Statement, + transaction: Transaction, + snapshot: LeaseRow | None, + ) -> dict[str, Any] | None: + """INSERT ... ON CONFLICT DO UPDATE, with EPQ on the locked tuple. + + Returns None when the statement evaluates to SQL NULL, which the + ports turn into production's "postgres query returned no JSON" the + same way `parse_single_json_value` does. That is a reachable + production outcome, not a harness edge case: when two coordinators + race the very first acquire, the loser's DO UPDATE affects zero rows + and its COALESCE fallback reads a snapshot in which no row exists + yet, so the whole statement returns NULL. + """ + payload = statement.payload + now = self.clock.now() + ttl = self._ttl(statement) + acquired = { + "acquired": True, + "writer_id": payload["writer_id"], + "writer_epoch": int(payload["writer_epoch"]), + "writer_session_token": payload["writer_session_token"], + } + + def claim() -> None: + self._stage( + transaction, + LeaseRow( + writer_id=str(payload["writer_id"]), + writer_epoch=int(payload["writer_epoch"]), + writer_session_token=str(payload["writer_session_token"]), + lease_expires_at=now + timedelta(seconds=ttl), + updated_at=now, + ), + ) + + # The conflict is resolved against the tuple this statement locked, + # not against its snapshot. + current = self._visible_lease(transaction) + if current is None: + claim() + return acquired + if self._is_exact_session(current, payload) or ( + current.lease_expires_at <= now + ): + claim() + return acquired + if snapshot is None: + return self._unobserved_not_acquired(statement) + return self._observed(snapshot, acquired=False) + + def _unobserved_not_acquired(self, statement: Statement) -> dict[str, Any] | None: + """Answer a race whose loser can see neither the upsert nor a holder. + + Which answer is correct is a property of the shipped statement, not of + this harness, so it is read off the statement every time. The two-arm + shape evaluates to SQL NULL here; the shape issue #140 adds carries a + terminal arm and therefore cannot. + """ + if acquire_result_arms(statement.sql) == 2: + return None + payload: dict[str, Any] = {} + for key, value_text in acquire_terminal_arm_fields(statement.sql): + payload[key] = self._terminal_arm_value(key, value_text) + if "acquired" not in payload: + raise UnsupportedStatement( + "the acquire statement's terminal arm does not name 'acquired'; " + "the lease evaluator cannot report an outcome from it" + ) + return payload + + @staticmethod + def _terminal_arm_value(key: str, value_text: str) -> Any: + """Answer one terminal-arm key from what the statement says it emits. + + The key set stays an allowlist -- an unknown key is refused even when + its value is a perfectly readable literal, because a key this model + has never seen is a shape nobody has reasoned about. What is *not* + guessed is the value: for the literal keys it is parsed out of the + SQL, so the model reports the wording the ledger PR chose rather than + a restatement of it that can drift. + """ + text = value_text.strip() + if key in _ACQUIRE_TERMINAL_LITERAL_KEYS: + lowered = text.lower() + if lowered == "true": + return True + if lowered == "false": + return False + if lowered == "null": + return None + if len(text) >= 2 and text.startswith("'") and text.endswith("'"): + return _unquote(text[1:-1]) + raise UnsupportedStatement( + f"the acquire statement's terminal arm gives {key!r} the " + f"non-literal value {text!r}; this model can only report a " + "value the statement states outright" + ) + if key in _ACQUIRE_TERMINAL_NULL_KEYS: + # A lease column with no visible row behind it. + return None + raise UnsupportedStatement( + f"the acquire statement's terminal arm names {key!r}, which this " + "model does not know how to answer. Teach it that key rather than " + "letting a scenario assert against a guess." + ) + + @staticmethod + def _adopt_matches(statement: Statement, row: LeaseRow) -> bool: + payload = statement.payload + return ( + row.writer_id == payload["writer_id"] + and row.writer_epoch == int(payload["writer_epoch"]) + and row.writer_session_token + == payload["observed_writer_session_token"] + and VirtualClock.timestamp_text(row.updated_at) + == payload["observed_lease_updated_at"] + ) + + @staticmethod + def _renew_matches(statement: Statement, row: LeaseRow) -> bool: + return FakePostgres._is_exact_session(row, statement.payload) + + _release_matches = _renew_matches + + def _evaluate_adopt( + self, + statement: Statement, + transaction: Transaction, + matched: LeaseRow | None, + ) -> dict[str, Any] | None: + payload = statement.payload + now = self.clock.now() + ttl = self._ttl(statement) + # `lease_expires_at > clock_timestamp()` is evaluated with the rest of + # the predicate but depends on live time rather than on the row + # version, so it is checked here rather than in _adopt_matches. + if matched is not None and matched.lease_expires_at > now: + self._stage( + transaction, + replace( + matched, + writer_session_token=str(payload["writer_session_token"]), + lease_expires_at=now + timedelta(seconds=ttl), + updated_at=now, + xmax=None, + ), + ) + return { + "acquired": True, + "adopted": True, + "writer_id": payload["writer_id"], + "writer_epoch": int(payload["writer_epoch"]), + "writer_session_token": payload["writer_session_token"], + } + # The CAS affected no rows, so the COALESCE fallback reports whatever + # this statement's snapshot holds. + snapshot = self._visible_lease(transaction) + if snapshot is None: + return None + observed = self._observed(snapshot, acquired=False) + observed["adopted"] = False + return observed + + def _evaluate_renew( + self, + statement: Statement, + transaction: Transaction, + matched: LeaseRow | None, + ) -> dict[str, Any]: + now = self.clock.now() + ttl = self._ttl(statement) + if matched is None: + return {"error": "writer lease is not active"} + self._stage( + transaction, + replace( + matched, + lease_expires_at=now + timedelta(seconds=ttl), + updated_at=now, + xmax=None, + ), + ) + return {"backend": "postgres-psql", "renewed_count": 1} + + def _evaluate_release( + self, + statement: Statement, + transaction: Transaction, + matched: LeaseRow | None, + ) -> dict[str, Any]: + now = self.clock.now() + if matched is None: + return {"released": 0} + self._stage( + transaction, + replace( + matched, + lease_expires_at=now - timedelta(seconds=1), + updated_at=now, + xmax=None, + ), + ) + return {"released": 1} + + def _evaluate_prove( + self, + statement: Statement, + transaction: Transaction, + ) -> dict[str, Any]: + """The read-only ownership proof: no row lock, no locker attribution. + + Everything it reads comes from one MVCC snapshot plus this backend's + own advisory-lock state, so unlike VERIFY there is no + snapshot-versus-live-state window and no interleaving point inside + the statement. That is the whole point of the split: the frequent + statement cannot queue, cannot misattribute, and cannot need a + second round trip. + """ + payload = statement.payload + now = self.clock.now() + margin = statement.authority_margin_seconds or 0.0 + snapshot = self._visible_lease(transaction) + exact = snapshot is not None and self._is_exact_session(snapshot, payload) + guard_backend = transaction.backend + advisory_held = ( + statement.advisory_lock is not None + and statement.advisory_lock in guard_backend.advisory_locks + and self._advisory_owners.get(statement.advisory_lock) is guard_backend + and not guard_backend.closed + ) + return { + "backend": "postgres-psql", + "guard_advisory_lock_held": bool(advisory_held), + "writer_session_token_current": bool(exact), + "lease_expired": bool( + exact and snapshot.lease_expires_at <= now + ), + "lease_renewal_due": bool( + exact + and snapshot.lease_expires_at <= now + timedelta(seconds=margin) + ), + "server_execution_seconds": float( + self._last_statement_server_seconds + ), + } + + def _evaluate_verify( + self, + statement: Statement, + transaction: Transaction, + tag: str, + ) -> dict[str, Any]: + payload = statement.payload + now = self.clock.now() + ttl = self._ttl(statement) + margin = statement.authority_margin_seconds or 0.0 + # One MVCC snapshot for every lease-row read in this statement. + snapshot = self._visible_lease(transaction) + exact = snapshot is not None and self._is_exact_session(snapshot, payload) + + # `renewable ... FOR NO KEY UPDATE SKIP LOCKED` never queues. + renewed = 0 + if exact and not self.lease_lock_held_by_other(transaction): + assert snapshot is not None + self._take_lease_lock(transaction) + self._stage( + transaction, + replace( + snapshot, + lease_expires_at=now + timedelta(seconds=ttl), + updated_at=now, + xmax=None, + ), + ) + renewed = 1 + + # The documented misfire window: the statement's row reads come from + # `snapshot`, while pg_stat_activity is read live. A checkpoint here + # lets a scenario commit an own fenced write between the two. + self.scheduler.checkpoint(f"{tag}.snapshot:verify") + + guard_backend = transaction.backend + advisory_held = ( + statement.advisory_lock is not None + and statement.advisory_lock in guard_backend.advisory_locks + and self._advisory_owners.get(statement.advisory_lock) is guard_backend + and not guard_backend.closed + ) + locked_by_this_process = any( + backend.application_name == payload.get("pool_application_name") + and backend.backend_xid is not None + and snapshot is not None + and backend.backend_xid == snapshot.xmax + for backend in self.live_backends() + ) + return { + "backend": "postgres-psql", + "guard_advisory_lock_held": bool(advisory_held), + "writer_session_token_current": bool(exact), + "lease_renewed_count": renewed, + "lease_expired": bool(exact and snapshot.lease_expires_at <= now), + "lease_expiring_within_authority_margin": bool( + exact + and snapshot.lease_expires_at <= now + timedelta(seconds=margin) + ), + "lease_locked_by_this_process": bool(locked_by_this_process), + "server_execution_seconds": float( + self._last_statement_server_seconds + ), + } + + # -- block candidate landing ------------------------------------------- + + def _visible_pool_block( + self, + transaction: Transaction, + block_hash: str, + ) -> PoolBlockRow | None: + """READ COMMITTED: committed rows plus this transaction's own writes.""" + staged = transaction.staged_pool_blocks.get(block_hash) + if staged is not None: + return staged + return self.pool_blocks.get(block_hash) + + def _visible_pool_blocks( + self, + transaction: Transaction, + ) -> dict[str, PoolBlockRow]: + merged = dict(self.pool_blocks) + merged.update(transaction.staged_pool_blocks) + return merged + + def _stage_pool_block( + self, + transaction: Transaction, + row: PoolBlockRow, + ) -> PoolBlockRow: + staged = replace(row) + transaction.staged_pool_blocks[staged.block_hash] = staged + return staged + + def _visible_outbox_row( + self, + transaction: Transaction, + block_hash: str, + ) -> OutboxRow | None: + staged = transaction.staged_outbox.get(block_hash) + if staged is not None: + return staged + return self.outbox.get(block_hash) + + def _visible_body(self, transaction: Transaction, body_id: str) -> BodyRow | None: + if body_id in transaction.deleted_bodies: + return None + staged = transaction.staged_bodies.get(body_id) + if staged is not None: + return staged + return self.bodies.get(body_id) + + def _stage_body(self, transaction: Transaction, body: BodyRow) -> BodyRow: + staged = transaction.staged_bodies.get(body.body_id) + if staged is None: + staged = body.copy() + transaction.staged_bodies[body.body_id] = staged + return staged + + def _body_referenced(self, transaction: Transaction, body_id: str, *, pending_only: bool) -> bool: + visible = dict(self.outbox) + visible.update(transaction.staged_outbox) + return any( + row.body_id == body_id and (not pending_only or row.state == "pending") + for row in visible.values() + ) + + def _renew_lease_for_landing( + self, + statement: Statement, + transaction: Transaction, + timeout_seconds: float | None, + tag: str, + timeout_kind: str, + ) -> bool: + """The ``lease AS (UPDATE ...)`` CTE every landing statement opens with. + + Every landing statement fences itself on the writer lease and renews + it in the same round trip. Modelling that is not decoration: a + deposed writer's landing statement fails here, which is the only + thing standing between a fenced-out process and a durable pool-block + write. The identity comes out of the statement itself — inlined as + literals by the ledger — so the model never has to be told which + writer a session belongs to. + + The renewal is an ordinary UPDATE on the lease tuple, so it obeys the + same rules as every other one: a predicate the snapshot already + excludes returns zero rows without queueing, and a matching predicate + takes the tuple lock and waits behind whoever holds it. A landing + stalled behind an orphaned lease transaction is a real outage shape, + and the model has to be able to express it. + """ + payload = statement.payload + writer_id = payload.get("writer_id") + writer_epoch = payload.get("writer_epoch") + session_token = payload.get("writer_session_token") + if writer_id is None or session_token is None: + # A statement whose identity the classifier could not recover is + # a model gap, not a lease failure; fail loudly rather than + # inventing a fence outcome. + raise UnsupportedStatement( + f"{statement.kind.value} statement carried no writer identity" + ) + identity = { + "writer_id": writer_id, + "writer_epoch": writer_epoch, + "writer_session_token": session_token, + } + + def matches(row: LeaseRow | None) -> bool: + return row is not None and self._is_exact_session(row, identity) + + if not matches(self._visible_lease(transaction)): + return False + self._await_lease_lock( + statement, + transaction, + timeout_seconds, + tag, + timeout_kind, + ) + current = self._visible_lease(transaction) + if not matches(current): + return False + assert current is not None + now = self.clock.now() + ttl = statement.lease_ttl_seconds + if ttl is not None: + self._stage( + transaction, + replace( + current, + lease_expires_at=now + timedelta(seconds=ttl), + updated_at=now, + xmax=None, + ), + ) + return True + + def _evaluate_landing( + self, + statement: Statement, + transaction: Transaction, + timeout_seconds: float | None, + tag: str, + timeout_kind: str, + ) -> Any: + kind = statement.kind + assert isinstance(kind, LandingOp) + payload = statement.payload + # Reads that open no lease CTE at all. + if kind is LandingOp.PUBLICATION_FLOOR: + return { + "audit_publication_sequence_floor": max( + [ + row.audit_publication_sequence or 0 + for row in self._visible_pool_blocks(transaction).values() + ], + default=0, + ) + } + if kind is LandingOp.ALL_SHARES: + return [dict(share) for share in self.shares] + if kind is LandingOp.SHARE_STATS: + # The evidence annotations, not an accounting input: the audit + # envelope carries them and the publication replay path reuses + # the originally durable values rather than re-reading. + return { + "accepted_share_count": len(self.shares), + "distinct_miner_count": len( + {str(share.get("miner_id")) for share in self.shares} + ), + "max_share_seq": max( + (int(share.get("share_seq", 0)) for share in self.shares), + default=0, + ), + } + if kind in {LandingOp.PRIOR_BALANCES, LandingOp.PRIOR_BALANCES_AS_OF}: + # The durable carry-forward view. A landing reads it to prove the + # payout base under its coinbase has not moved since the job was + # issued, so a scenario that wants to fail that check moves this + # list rather than reaching into the coordinator. The as-of + # variant restricts the same view to blocks at or below one + # confirmed height; with no payout entries modelled the two + # coincide, and a scenario that needs them to differ is asking + # for the settlement machine, which #128 lists separately. + return [dict(balance) for balance in self.carry_forward_balances] + if kind is LandingOp.POOL_BLOCK_STATE: + row = self._visible_pool_block(transaction, str(payload["block_hash"])) + return { + "state": None + if row is None + else { + "block_hash": row.block_hash, + "block_height": row.block_height, + "parent_hash": row.parent_hash, + "chain_state": row.chain_state, + "maturity_state": row.maturity_state, + "audit_publication_sequence": row.audit_publication_sequence, + } + } + if kind is LandingOp.OUTBOX_PENDING_PAGE: + # Deliberately grouped with the reads above, before + # _renew_lease_for_landing: the production statement opens no + # lease CTE, so a model that renewed here would report a + # lease-touching read the ledger does not perform -- and would + # hide the very property issue #211 turns on, that enumeration + # neither takes the writer gate nor touches the lease row. + return self._pending_candidate_page(transaction, payload) + if kind is LandingOp.OUTBOX_HEADER_PAGE: + return self._header_page(transaction, payload) + if kind is LandingOp.OUTBOX_OBSERVE: + row = self._visible_outbox_row(transaction, str(payload["block_hash"])) + if row is None: + return {"found": False} + body = None if row.body_id is None else self._visible_body(transaction, row.body_id) + return { + "found": True, + "state": row.state, + "share_id": None, + "candidate_sha256": row.candidate_sha256, + "storage_version": row.storage_version, + "body_id": row.body_id, + "body_state": None if body is None else body.state, + } + if kind is LandingOp.SCHEMA_PRESENCE: + return {"has_body_table": True, "has_capabilities": True} + if kind is LandingOp.SCHEMA_CAPABILITY: + return {"declared": 2} + if kind in { + LandingOp.BODY_MANIFEST, + LandingOp.BODY_SPANS, + LandingOp.BODY_PAGES, + LandingOp.BODY_PAGE, + }: + return self._body_read(kind, transaction, payload) + if kind in { + LandingOp.BODY_STAGE, + LandingOp.BODY_CHUNK, + LandingOp.BODY_INDEX, + LandingOp.BODY_SEAL, + LandingOp.BODY_RETIRE_STAGING, + LandingOp.BODY_REAP, + }: + # Staging, sealing and reaping open no lease CTE: they are private + # preparation (or reclamation of a permanently retired body) and + # a model that renewed here would misreport a lease touch. + return self._body_write(kind, transaction, payload) + if kind is LandingOp.CONFIRMED_SEQUENCE: + row = self._visible_pool_block(transaction, str(payload["block_hash"])) + confirmed = ( + row is not None + and row.chain_state == "confirmed" + and row.maturity_state != "reversed" + ) + return { + "audit_publication_sequence": ( + row.audit_publication_sequence if confirmed else None + ) + } + + lease_held = self._renew_lease_for_landing( + statement, + transaction, + timeout_seconds, + tag, + timeout_kind, + ) + if kind in {LandingOp.CONFIRM_BLOCK, LandingOp.REJECT_PREPARED}: + if not lease_held: + # The PL/pgSQL function RAISEs rather than returning a value, + # so the caller sees an error, not a zero count. + raise RuntimeError("writer lease is not active") + if kind is LandingOp.CONFIRM_BLOCK: + return { + "backend": "postgres-psql", + "confirmed_count": self._confirm_pool_block( + transaction, + str(payload["block_hash"]), + int(payload["active_tip_height"]), + ), + } + return { + "backend": "postgres-psql", + "rejected_count": self._reject_prepared_pool_block( + transaction, + str(payload["block_hash"]), + ), + } + + if not lease_held: + return {"error": "writer lease is not active"} + if kind is LandingOp.PERSIST_BLOCK: + return self._persist_pool_block(statement, transaction) + if kind is LandingOp.OUTBOX_RECORD: + return self._record_outbox_row(statement, transaction) + if kind is LandingOp.BODY_RETIRE_ORPHANS: + return self._retire_orphan_bodies(transaction, payload) + if kind is LandingOp.OUTBOX_BATCH_ABANDON: + # The pool-block clause lives *inside* this fenced UPDATE for the + # reason issue #211 depends on: the caller's page fact is advisory + # by the time it gets here, and a candidate that landed in between + # must be absent from the returned set rather than abandoned. + pool_blocks = self._visible_pool_blocks(transaction) + abandoned: list[str] = [] + for block_hash in payload["block_hashes"]: + row = self._visible_outbox_row(transaction, str(block_hash)) + if row is None or row.state != "pending": + continue + if row.block_hash in pool_blocks: + continue + staged = replace(row) + staged.state = "abandoned" + staged.last_error = payload.get("last_error") + self._detach_body(transaction, staged) + transaction.staged_outbox[staged.block_hash] = staged + abandoned.append(staged.block_hash) + return {"abandoned": sorted(abandoned)} + if kind in {LandingOp.OUTBOX_ATTEMPT, LandingOp.OUTBOX_FINISH}: + row = self._visible_outbox_row(transaction, str(payload["block_hash"])) + if row is None or row.state != "pending": + return {"updated": 0} + staged = replace(row) + if kind is LandingOp.OUTBOX_ATTEMPT: + staged.attempt_count += 1 + else: + staged.state = str(payload["state"]) + staged.last_error = payload.get("last_error") + self._detach_body(transaction, staged) + transaction.staged_outbox[staged.block_hash] = staged + return {"updated": 1} + raise UnsupportedStatement(f"unhandled landing operation {kind}") + + # -- chunked candidate bodies (#255) ------------------------------------ + + def _detach_body(self, transaction: Transaction, row: OutboxRow) -> None: + """The terminal write detaches and retires the body; chunks stay.""" + if row.body_id is None: + return + body = self._visible_body(transaction, row.body_id) + row.retired_body_id = row.body_id + row.body_id = None + if body is not None and body.state != "retired": + staged = self._stage_body(transaction, body) + staged.state = "retired" + staged.retired_at = self.clock.now() + + def _body_owner_matches(self, body: BodyRow, payload: dict[str, Any]) -> bool: + return ( + body.state == "staging" + and body.writer_id == payload.get("writer_id") + and body.writer_epoch == int(payload.get("writer_epoch", -1)) + and body.writer_session_token == payload.get("writer_session_token") + ) + + def _body_write( + self, + kind: LandingOp, + transaction: Transaction, + payload: dict[str, Any], + ) -> Any: + if kind is LandingOp.BODY_STAGE: + body_id = str(payload["body_id"]) + existing = self._visible_body(transaction, body_id) + if existing is not None: + return {"staged": 0, "state": existing.state} + transaction.staged_bodies[body_id] = BodyRow( + body_id=body_id, + block_hash=str(payload["block_hash"]), + candidate_sha256=str(payload["candidate_sha256"]), + byte_count=int(payload["byte_count"]), + chunk_count=int(payload["chunk_count"]), + chunk_bytes=int(payload["chunk_bytes"]), + share_count=int(payload["share_count"]), + shares_offset=int(payload["shares_offset"]), + shares_end=int(payload["shares_end"]), + span_count=int(payload.get("span_count", 0)), + page_count=int(payload.get("page_count", 0)), + writer_id=str(payload["writer_id"]), + writer_epoch=int(payload["writer_epoch"]), + writer_session_token=str(payload["writer_session_token"]), + created_at=self.clock.now(), + ) + return {"staged": 1, "state": "staging"} + if kind in {LandingOp.BODY_CHUNK, LandingOp.BODY_INDEX}: + body = self._visible_body(transaction, str(payload["body_id"])) + if body is None or not self._body_owner_matches(body, payload): + return {"owned": False, "inserted": 0, "spans": 0, "pages": 0} + staged = self._stage_body(transaction, body) + if kind is LandingOp.BODY_CHUNK: + ordinal = int(payload["ordinal"]) + if ordinal in staged.chunks: + return {"owned": True, "inserted": 0} + staged.chunks[ordinal] = (bytes(payload["chunk"]), str(payload["chunk_sha256"])) + return {"owned": True, "inserted": 1} + spans = 0 + pages = 0 + for span in payload.get("spans") or []: + if span["field"] not in staged.spans: + staged.spans[str(span["field"])] = dict(span) + spans += 1 + for page in payload.get("pages") or []: + key = (str(page[0]), int(page[1])) + if key not in staged.pages: + staged.pages[key] = (int(page[2]), int(page[3])) + pages += 1 + return {"owned": True, "spans": spans, "pages": pages} + if kind is LandingOp.BODY_SEAL: + body = self._visible_body(transaction, str(payload["body_id"])) + if body is None: + return {"sealed": 0, "state": None} + ordinals = sorted(body.chunks) + byte_count = sum(len(data) for data, _ in body.chunks.values()) + digests_ok = all( + hashlib.sha256(data).hexdigest() == digest for data, digest in body.chunks.values() + ) + lengths_ok = all( + len(body.chunks[ordinal][0]) == body.chunk_bytes or ordinal == body.chunk_count - 1 + for ordinal in ordinals + ) + complete = ( + self._body_owner_matches(body, payload) + and body.candidate_sha256 == payload.get("candidate_sha256") + and len(ordinals) == body.chunk_count + and byte_count == body.byte_count + and (body.chunk_count == 0 or ordinals[0] == 0) + and (ordinals[-1] if ordinals else -1) == body.chunk_count - 1 + and digests_ok + and lengths_ok + and len(body.spans) == body.span_count + and len(body.pages) == body.page_count + ) + if not complete: + return { + "sealed": 0, + "state": body.state, + "observed_chunk_count": len(ordinals), + "observed_byte_count": byte_count, + "observed_span_count": len(body.spans), + "observed_page_count": len(body.pages), + "digests_ok": digests_ok, + "lengths_ok": lengths_ok, + } + staged = self._stage_body(transaction, body) + staged.state = "sealed" + staged.sealed_at = self.clock.now() + return {"sealed": 1, "state": "staging"} + if kind is LandingOp.BODY_RETIRE_STAGING: + body = self._visible_body(transaction, str(payload["body_id"])) + if body is None or body.state != "staging": + return {"retired": 0} + staged = self._stage_body(transaction, body) + staged.state = "retired" + staged.retired_at = self.clock.now() + return {"retired": 1} + if kind is LandingOp.BODY_REAP: + visible = { + body_id: body + for body_id, body in {**self.bodies, **transaction.staged_bodies}.items() + if body_id not in transaction.deleted_bodies + } + retired = sorted( + (body for body in visible.values() if body.state == "retired"), + key=lambda body: (body.retired_at or CLOCK_ORIGIN, body.body_id), + ) + if not retired: + return {"body_id": None, "deleted_chunks": 0, "deleted_bodies": 0} + target = self._stage_body(transaction, retired[0]) + max_chunks = max(1, int(payload.get("max_chunks", 1))) + # The production statement's snapshot decides which table this + # step works on: chunks first, then pages once no chunk was + # visible, then spans, then the manifest on a later step. + before = { + "chunks": bool(target.chunks), + "pages": bool(target.pages), + "spans": bool(target.spans), + } + chunk_victims = sorted(target.chunks)[:max_chunks] + for ordinal in chunk_victims: + del target.chunks[ordinal] + page_victims = [] if before["chunks"] else sorted(target.pages)[:max_chunks] + for key in page_victims: + del target.pages[key] + span_victims = [] if before["chunks"] or before["pages"] else sorted(target.spans)[:max_chunks] + for key in span_victims: + del target.spans[key] + deleted_bodies = 0 + if not any(before.values()): + transaction.deleted_bodies.add(target.body_id) + transaction.staged_bodies.pop(target.body_id, None) + deleted_bodies = 1 + return { + "body_id": target.body_id, + "deleted_chunks": len(chunk_victims), + "deleted_pages": len(page_victims), + "deleted_spans": len(span_victims), + "deleted_bodies": deleted_bodies, + "remaining": before, + } + raise UnsupportedStatement(f"unhandled body write {kind}") + + def _retire_orphan_bodies( + self, + transaction: Transaction, + payload: dict[str, Any], + ) -> Any: + stale = timedelta(seconds=float(payload.get("stale_staging_seconds", 0.0))) + now = self.clock.now() + visible = { + body_id: body + for body_id, body in {**self.bodies, **transaction.staged_bodies}.items() + if body_id not in transaction.deleted_bodies + } + candidates = [] + for body in sorted(visible.values(), key=lambda body: (body.created_at, body.body_id)): + if self._body_referenced(transaction, body.body_id, pending_only=False): + continue + foreign = not ( + body.writer_id == payload.get("writer_id") + and body.writer_session_token == payload.get("writer_session_token") + ) + if body.state == "staging" and foreign and body.created_at < now - stale: + candidates.append(body) + elif body.state == "sealed" and body.sealed_at is not None and body.sealed_at < now - stale: + candidates.append(body) + retired: list[str] = [] + for body in candidates[:1]: + # The manifest trigger's invariant: a referenced body is never + # retired by orphan cleanup. + if self._body_referenced(transaction, body.body_id, pending_only=False): + raise RuntimeError( + f"candidate body {body.body_id} is referenced by a pending outbox row" + ) + staged = self._stage_body(transaction, body) + staged.state = "retired" + staged.retired_at = now + retired.append(body.body_id) + return {"retired": retired} + + def _body_read( + self, + kind: LandingOp, + transaction: Transaction, + payload: dict[str, Any], + ) -> Any: + body = self._visible_body(transaction, str(payload["body_id"])) + if kind is LandingOp.BODY_MANIFEST: + if body is None: + return {"found": False, "manifest": None} + return { + "found": True, + "manifest": { + "body_id": body.body_id, + "storage_version": 2, + "block_hash": body.block_hash, + "candidate_sha256": body.candidate_sha256, + "byte_count": body.byte_count, + "chunk_count": body.chunk_count, + "chunk_bytes": body.chunk_bytes, + "share_count": body.share_count, + "shares_offset": body.shares_offset, + "shares_end": body.shares_end, + "span_count": body.span_count, + "page_count": body.page_count, + "state": body.state, + "referenced": self._body_referenced(transaction, body.body_id, pending_only=True), + }, + } + state = None if body is None else body.state + if kind is LandingOp.BODY_SPANS: + after = payload.get("after_field") + spans = [] if body is None else sorted(body.spans.values(), key=lambda span: span["field"]) + spans = [span for span in spans if after is None or span["field"] > after][:INDEX_SPAN_ROWS] + return {"state": state, "spans": spans} + if kind is LandingOp.BODY_PAGES: + field_name = str(payload["field"]) + from_ordinal = int(payload["from_ordinal"]) + entries = [] if body is None else sorted( + ( + [ordinal, offset, record_index] + for (name, ordinal), (offset, record_index) in body.pages.items() + if name == field_name and ordinal >= from_ordinal + ), + ) + return {"state": state, "pages": entries[:INDEX_PAGE_ENTRIES]} + if kind is LandingOp.BODY_PAGE: + from_ordinal = int(payload["from_ordinal"]) + max_chunks = max(1, min(int(payload["max_chunks"]), BODY_READ_MAX_CHUNKS)) + chunks = [] if body is None else [ + { + "ordinal": ordinal, + "sha256": body.chunks[ordinal][1], + # PostgreSQL wraps base64 at 76 characters with newlines. + "base64": base64.encodebytes(body.chunks[ordinal][0]).decode("ascii").rstrip("\n"), + } + for ordinal in sorted(body.chunks) + if ordinal >= from_ordinal + ][:max_chunks] + return { + "state": state, + "referenced": body is not None + and self._body_referenced(transaction, body.body_id, pending_only=True), + "chunks": chunks, + } + raise UnsupportedStatement(f"unhandled body read {kind}") + + def _header_page(self, transaction: Transaction, payload: dict[str, Any]) -> Any: + """The metadata-first page (#255): bounded headers, explicit exhaustion.""" + visible = dict(self.outbox) + visible.update(transaction.staged_outbox) + pool_blocks = self._visible_pool_blocks(transaction) + after = payload.get("after_cursor") + limit = max(1, int(payload["limit"])) + max_bytes = max(1, int(payload["max_bytes"])) + rows = sorted( + (row for row in visible.values() if row.state == "pending"), + key=lambda row: (row.created_at, row.block_hash), + ) + fetched: list[OutboxRow] = [] + for row in rows: + cursor_stamp = self._pending_cursor_stamp(row.created_at) + if after is not None and (cursor_stamp, row.block_hash) <= (str(after[0]), str(after[1])): + continue + fetched.append(row) + if len(fetched) > limit: + break + page: list[dict[str, Any]] = [] + running = 0 + for row in fetched[:limit]: + header = replay_header_from_fields(row.header or {"block_hash_hex": row.block_hash}) + header_bytes = len(json.dumps(header, separators=(",", ":"))) + if page and running + header_bytes > max_bytes: + break + running += header_bytes + body = None if row.body_id is None else self._visible_body(transaction, row.body_id) + page.append( + { + "block_hash": row.block_hash, + "storage_version": row.storage_version, + "candidate_sha256": row.candidate_sha256, + "header": header, + "header_bytes": header_bytes, + "body": None if body is None else { + "body_id": body.body_id, + "storage_version": 2, + "candidate_sha256": body.candidate_sha256, + "byte_count": body.byte_count, + "chunk_count": body.chunk_count, + "chunk_bytes": body.chunk_bytes, + "share_count": body.share_count, + "state": body.state, + }, + "pool_block_exists": row.block_hash in pool_blocks, + "cursor": [self._pending_cursor_stamp(row.created_at), row.block_hash], + } + ) + return {"fetched": len(fetched), "returned": len(page), "rows": page} + + def _confirm_pool_block( + self, + transaction: Transaction, + block_hash: str, + active_tip_height: int, + ) -> int: + """``qbit_confirm_pool_block``: allocate the publication ordinal. + + The ordinal comes from a sequence, not from ``MAX + 1``. That + distinction is the whole of the allocator's contract: an exact replay + of an already-confirmed row matches zero rows in the UPDATE and so + burns nothing, while a terminally disposed row reports the superseded + disposition (-1) instead of a plain miss. The durable floor read is a + separate MAX, which is what lets a *later* ordinal publish first and + leave an earlier one behind the floor — the state #133 turns on. + """ + row = self._visible_pool_block(transaction, block_hash) + if ( + row is not None + and row.block_height == active_tip_height + and row.chain_state == "prepared" + and row.maturity_state == "immature" + ): + self._audit_publication_sequence += 1 + staged = self._stage_pool_block(transaction, row) + staged.chain_state = "confirmed" + staged.audit_publication_sequence = self._audit_publication_sequence + return 1 + if ( + row is not None + and row.block_height == active_tip_height + and row.chain_state == "confirmed" + and row.maturity_state != "reversed" + ): + return 1 + if row is not None and ( + row.chain_state in {"inactive", "rejected", "reversed"} + or row.maturity_state == "reversed" + ): + return -1 + return 0 + + def _reject_prepared_pool_block( + self, + transaction: Transaction, + block_hash: str, + ) -> int: + row = self._visible_pool_block(transaction, block_hash) + if ( + row is None + or row.chain_state != "prepared" + or row.maturity_state != "immature" + ): + return 0 + staged = self._stage_pool_block(transaction, row) + staged.chain_state = "rejected" + staged.maturity_state = "reversed" + return 1 + + def _persist_pool_block( + self, + statement: Statement, + transaction: Transaction, + ) -> dict[str, Any]: + """``persist_accepted_block``, reduced to what a landing reads back. + + The production statement writes the pool block, its audit bundle, the + payout entries and the carry-forward deltas in one CTE. The landing + decisions downstream read the prepared row (through + ``pool_block_state`` and the confirmation) and the returned counts; + the payout rows are the reorg reconciler's and the settlement + machine's business, neither of which this harness drives yet. The + counts below are therefore reported from what is modelled, and the + row shape stays honest: a re-persist of an existing block reports the + existing row rather than inserting a second one. + """ + payload = statement.payload + block_hash = str(payload["block_hash"]) + existing = self._visible_pool_block(transaction, block_hash) + if existing is None: + transaction.staged_pool_blocks[block_hash] = PoolBlockRow( + block_hash=block_hash, + block_height=int(payload["block_height"]), + parent_hash=str(payload["parent_hash"]), + ) + elif existing.block_height != int(payload["block_height"]): + raise RuntimeError("pool block height conflicts") + accounts = payload.get("accounts") or [] + return { + "backend": "postgres-psql", + "share_count": len(self.shares), + "block_count": 1, + "bundle_count": 1, + "payout_entry_count": len(accounts), + "carry_forward_count": 0, + "onchain_output_count": 0, + } + + def _pending_candidate_page( + self, + transaction: Transaction, + payload: dict[str, Any], + ) -> list[dict[str, Any]]: + """One snapshot answering row, landed-block fact and cursor together. + + The three facts come from one visible set here for the same reason + they come from one statement in production: a caller that read the + rows and their pool-block facts at different instants could see a + candidate as pending *and* unlanded when it was neither. + + The order and the keyset predicate are ``(created_at, block_hash)``, + the total order the production index provides. The block-hash + tiebreak is not decoration -- creation stamps collide under one + transaction's ``clock_timestamp()``, and a cursor on the stamp alone + would replay or skip a whole colliding group. + """ + visible = dict(self.outbox) + visible.update(transaction.staged_outbox) + pool_blocks = self._visible_pool_blocks(transaction) + after = payload.get("after_cursor") + rows = sorted( + ( + row + for row in visible.values() + if row.state == "pending" + ), + key=lambda row: (row.created_at, row.block_hash), + ) + page: list[dict[str, Any]] = [] + for row in rows: + cursor_stamp = self._pending_cursor_stamp(row.created_at) + if after is not None and (cursor_stamp, row.block_hash) <= ( + str(after[0]), + str(after[1]), + ): + continue + page.append( + { + "block_hash": row.block_hash, + "candidate": {"block_hash_hex": row.block_hash}, + "storage_version": row.storage_version, + "candidate_sha256": row.candidate_sha256, + "pool_block_exists": row.block_hash in pool_blocks, + "cursor": [cursor_stamp, row.block_hash], + } + ) + if len(page) >= int(payload["limit"]): + break + return page + + @staticmethod + def _pending_cursor_stamp(moment: datetime) -> str: + """``to_char(created_at AT TIME ZONE 'UTC', ...US"Z"')``, exactly. + + Microsecond precision with an explicit UTC marker, because the cursor + is compared as text and a truncated stamp would re-emit or skip whole + sub-second groups. + """ + return ( + f"{moment.astimezone(timezone.utc):%Y-%m-%dT%H:%M:%S}" + f".{moment.microsecond:06d}Z" + ) + + def _record_outbox_row( + self, + statement: Statement, + transaction: Transaction, + ) -> dict[str, Any]: + payload = statement.payload + block_hash = str(payload["block_hash"]) + digest = str(payload["candidate_sha256"]) + existing = self._visible_outbox_row(transaction, block_hash) + if existing is None: + body_id = payload.get("body_id") + if body_id is not None: + # A version-2 reference may point only at a sealed body with + # the same digest and block hash (#255). + body = self._visible_body(transaction, str(body_id)) + if ( + body is None + or body.state != "sealed" + or body.candidate_sha256 != digest + or body.block_hash != block_hash + ): + return {"error": "block candidate body is not sealed"} + transaction.staged_outbox[block_hash] = OutboxRow( + block_hash=block_hash, + candidate_sha256=digest, + created_at=self.clock.now(), + storage_version=1 if body_id is None else 2, + body_id=None if body_id is None else str(body_id), + header=payload.get("header"), + ) + return {"inserted": 1, "state": "pending"} + if existing.candidate_sha256 != digest: + return {"error": "block candidate payload mismatch"} + return {"inserted": 0, "state": existing.state} + + # -- helpers ----------------------------------------------------------- + + @staticmethod + def _is_exact_session(row: LeaseRow, payload: dict[str, Any]) -> bool: + return ( + row.writer_id == payload["writer_id"] + and row.writer_epoch == int(payload["writer_epoch"]) + and row.writer_session_token == payload["writer_session_token"] + ) + + @staticmethod + def _ttl(statement: Statement) -> float: + if statement.lease_ttl_seconds is None: + raise UnsupportedStatement( + f"{statement.kind.value} statement carried no make_interval TTL" + ) + return statement.lease_ttl_seconds + + def _observed(self, row: LeaseRow, *, acquired: bool) -> dict[str, Any]: + now = self.clock.now() + return { + "acquired": acquired, + "writer_id": row.writer_id, + "writer_epoch": row.writer_epoch, + "writer_session_token": row.writer_session_token, + "lease_expires_at": VirtualClock.timestamp_text(row.lease_expires_at), + "lease_updated_at": VirtualClock.timestamp_text(row.updated_at), + "lease_age_seconds": max( + 0.0, (now - row.updated_at).total_seconds() + ), + "lease_wait_seconds": max( + 0.0, (row.lease_expires_at - now).total_seconds() + ), + } + + +# -------------------------------------------------------------------------- +# Ports +# -------------------------------------------------------------------------- + + +class FakeSqlBackend: + """``LedgerSqlPort`` over ``FakePostgres``. + + Mirrors ``_NativePostgresClient.run_json``'s transaction shape exactly, + because that shape is what #123 turns on: with no caller deadline the + statement runs in autocommit and commits with it, while a deadline-scoped + statement runs BEGIN / SET LOCAL / statement / COMMIT, leaving a window in + which the server holds the tuple lock and is waiting for a COMMIT the + client may never send. + """ + + def __init__( + self, + server: FakePostgres, + *, + pool_size: int, + application_name: str, + tag: str, + ) -> None: + self.server = server + self._pool_size = max(1, int(pool_size)) + self.application_name = application_name + self.tag = tag + self._free: list[Backend] = [ + server.connect(application_name=application_name) + for _ in range(self._pool_size) + ] + self._in_use: list[Backend] = [] + self.closed = False + + @property + def pool_size(self) -> int: + return self._pool_size + + def _acquire(self, timeout_seconds: float | None = None) -> Backend: + deadline = ( + None + if timeout_seconds is None + else self.server.clock.monotonic() + max(0.0, timeout_seconds) + ) + while not self._free: + if deadline is not None and self.server.clock.monotonic() >= deadline: + # _NativePostgresClient.connection raises exactly this rather + # than waiting past the caller's budget. + raise LedgerOperationTimeout( + "timed out waiting for a postgres pool slot" + ) + self.server.scheduler.block( + f"{self.tag}.pool:exhausted", + ready=lambda: bool(self._free), + wake_at=deadline, + ) + backend = self._free.pop(0) + self._in_use.append(backend) + return backend + + def _release(self, backend: Backend) -> None: + if backend in self._in_use: + self._in_use.remove(backend) + if not backend.closed and backend.transaction is None: + self._free.append(backend) + + def run_json( + self, + sql: str, + *, + retry_safe: bool = False, + timeout_seconds: float | None = None, + on_statement_start: Callable[[], None] | None = None, + ) -> Any: + if self.closed: + raise RuntimeError("connection pool is closed") + backend = self._acquire(timeout_seconds) + transaction: Transaction | None = None + try: + if on_statement_start is not None: + on_statement_start() + if timeout_seconds is not None: + # `with conn.transaction():` — SET LOCAL statement_timeout and + # lock_timeout, run the statement, then send COMMIT. + transaction = self.server._begin(backend, explicit=True) + result = self.server.execute( + backend, + sql, + transaction=transaction, + timeout_seconds=timeout_seconds, + tag=self.tag, + ) + if transaction is not None: + self.server.scheduler.checkpoint(f"{self.tag}.precommit") + self.server._commit(transaction) + # Production's own NULL handling, called rather than re-described: + # a statement that evaluates to SQL NULL raises here, exactly as + # it does for a real client. + return parse_single_json_value(result) + except LockTimeout as exc: + if transaction is not None and backend.transaction is transaction: + self.server._rollback(transaction) + if timeout_seconds is None: + raise + # Both shipped `LedgerSqlPort` implementations translate a + # cancellation caused by the caller's own deadline into + # `LedgerOperationTimeout`, and only when that deadline was armed: + # `_NativePostgresClient.run_json` gates on + # `timeout_seconds is not None and _is_postgres_deadline_error`, + # and the psql backend applies the same test to stderr. That + # translation is the contract callers retry against, so a model of + # this seam that skipped it would let a caller-side retry loop look + # dead here while working in production. Nothing exercised it + # before a deadline reached a pooled lease statement. + raise LedgerOperationTimeout( + f"postgres operation exceeded {timeout_seconds:g}s" + ) from exc + except BaseException: + if transaction is not None and backend.transaction is transaction: + self.server._rollback(transaction) + raise + finally: + self._release(backend) + + def run_json_rows( + self, + sql: str, + *, + retry_safe: bool = False, + timeout_seconds: float | None = None, + on_statement_start: Callable[[], None] | None = None, + row_converter: Callable[[Any], Any] | None = None, + batch_size: int = PAYOUT_WINDOW_ROW_BATCH_SIZE, + on_batch: Callable[[], None] | None = None, + ) -> list[Any]: + """``LedgerSqlPort.run_json_rows`` over the same modelled server. + + The row-result statements (#236: the payout-window snapshot and + delta) take the identical connection, transaction and deadline + shape as ``run_json``; only the result differs, and the model + answers one with the list of row values. ``FakePostgres`` models no + such statement today, so ``classify`` rejects the SQL with + ``UnsupportedStatement`` exactly as it rejects any other statement + outside the lease vocabulary. Kept here so the fake satisfies the + port it stands in for, and so a scenario that does reach the + payout-window read fails on the model's vocabulary rather than on + a missing method. + """ + rows = self.run_json( + sql, + retry_safe=retry_safe, + timeout_seconds=timeout_seconds, + on_statement_start=on_statement_start, + ) + if not isinstance(rows, list): + raise UnsupportedStatement( + "row-result statement must be modelled as a list of row values" + ) + batch_size = max(1, int(batch_size)) + results: list[Any] = [] + for start in range(0, len(rows), batch_size): + if start and on_batch is not None: + on_batch() + batch = rows[start : start + batch_size] + results.extend( + batch if row_converter is None else [row_converter(v) for v in batch] + ) + return results + + def run_script(self, sql: str) -> None: + raise UnsupportedStatement( + "schema initialization is not modelled; construct the ledger with " + "initialize_schema=False" + ) + + def close(self) -> None: + self.closed = True + for backend in [*self._free, *self._in_use]: + if backend.transaction is not None and backend.transaction.orphaned: + # A vanished client's backend outlives the object that owned + # it; closing the pool locally does not reach the server. + continue + self.server.disconnect(backend) + self._free.clear() + + +class FakeLeaseGuard: + """``LeaseGuardPort`` over ``FakePostgres``. + + The guard session is autocommit, carries a server-side + ``statement_timeout``, and serializes callers through one query slot. + That slot is modelled with a scheduler-aware lock so a second caller + parks at a named checkpoint instead of deadlocking the baton. + """ + + def __init__( + self, + server: FakePostgres, + *, + advisory_lock_key: int, + application_name: str, + tag: str, + ) -> None: + self.server = server + self.advisory_lock_key = advisory_lock_key + self.tag = tag + self.backend = server.connect( + application_name=application_name, + statement_timeout_seconds=GUARD_STATEMENT_TIMEOUT_SECONDS, + ) + self._held = False + self._closed = False + self._slot_owner: Actor | None = None + + def try_acquire(self) -> bool: + self._held = self.server.try_advisory_lock( + self.backend, + self.advisory_lock_key, + ) + return self._held + + @property + def held(self) -> bool: + return self._held and not self._closed and not self.backend.closed + + def run_json( + self, + sql: str, + *, + on_query_start: Callable[[], None] | None = None, + on_statement_end: Callable[[], None] | None = None, + on_statement_result: Callable[[Any], None] | None = None, + followup: Callable[[Any], str | None] | None = None, + ) -> Any: + actor = self.server.scheduler.current() + while self._slot_owner is not None and self._slot_owner is not actor: + self.server.scheduler.block( + f"{self.tag}.slot:wait", + ready=lambda: self._slot_owner is None, + ) + self._slot_owner = actor + try: + if on_query_start is not None: + on_query_start() + if not self.held: + raise RuntimeError("postgres writer lease guard is not held") + result = parse_single_json_value( + self.server.execute( + self.backend, + sql, + transaction=None, + timeout_seconds=None, + tag=self.tag, + ) + ) + # Same order as the native guard: the client-side end mark, + # then the parsed result (which carries the statement's own + # server execution time, issue #227). + if on_statement_end is not None: + on_statement_end() + if on_statement_result is not None: + on_statement_result(result) + while followup is not None: + next_sql = followup(result) + if next_sql is None: + break + result = parse_single_json_value( + self.server.execute( + self.backend, + next_sql, + transaction=None, + timeout_seconds=None, + tag=self.tag, + ) + ) + if on_statement_end is not None: + on_statement_end() + if on_statement_result is not None: + on_statement_result(result) + return result + finally: + self._slot_owner = None + + def close(self) -> None: + if self._closed: + return + self._closed = True + self._held = False + self.server.disconnect(self.backend) + + +# -------------------------------------------------------------------------- +# Harness front end +# -------------------------------------------------------------------------- + + +class Coordinator: + """One PRISM coordinator process: an actor plus the ledger it builds.""" + + def __init__( + self, + harness: LeaseHarness, + name: str, + *, + writer_id: str, + writer_epoch: int, + session_token: str, + ledger_kwargs: dict[str, Any], + ) -> None: + self.harness = harness + self.name = name + self.writer_id = writer_id + self.writer_epoch = writer_epoch + self.session_token = session_token + self.actor = harness.scheduler.actor(name) + self._ledger: PsqlShareLedger | None = None + self._ledger_kwargs = ledger_kwargs + self.sql_backend: FakeSqlBackend | None = None + self.guard: FakeLeaseGuard | None = None + + # -- construction ------------------------------------------------------ + + def _sql_backend_factory( + self, + conninfo: str, + *, + pool_size: int, + application_name: str, + ) -> FakeSqlBackend: + self.sql_backend = FakeSqlBackend( + self.harness.server, + pool_size=pool_size, + application_name=application_name, + tag=self.name, + ) + return self.sql_backend + + def _lease_guard_factory( + self, + conninfo: str, + *, + advisory_lock_key: int, + ) -> FakeLeaseGuard: + self.guard = FakeLeaseGuard( + self.harness.server, + advisory_lock_key=advisory_lock_key, + application_name=f"{self.name}-guard", + tag=f"{self.name}.guard", + ) + return self.guard + + def _build(self) -> PsqlShareLedger: + ledger = PsqlShareLedger( + psql_command="psql postgres://harness/qbit", + database_url="postgres://harness/qbit", + writer_id=self.writer_id, + writer_epoch=self.writer_epoch, + writer_session_token=self.session_token, + initialize_schema=False, + monotonic=self.harness.clock.monotonic, + lease_retry_sleep=self.harness.sleep, + sql_backend_factory=self._sql_backend_factory, + lease_guard_factory=self._lease_guard_factory, + **self._ledger_kwargs, + ) + self._ledger = ledger + return ledger + + def start(self) -> Call: + """Queue this coordinator's startup (construction acquires the lease).""" + return self.actor.submit(self._build, label="startup") + + @property + def ledger(self) -> PsqlShareLedger: + if self._ledger is None: + raise HarnessError( + f"coordinator {self.name!r} has not finished startup yet" + ) + return self._ledger + + @property + def started(self) -> bool: + return self._ledger is not None + + # -- driving ----------------------------------------------------------- + + def submit(self, fn: Callable[[], Any], *, label: str | None = None) -> Call: + return self.actor.submit(fn, label=label) + + def call(self, fn: Callable[[], Any], *, label: str | None = None) -> Any: + """Run ``fn`` on this actor to completion and return its result.""" + call = self.actor.submit(fn, label=label) + self.harness.run_until(self, f"done:{call.label}") + return call.value() + + def vanish(self) -> None: + """Abandon this coordinator where it stands, without releasing anything. + + A partition, SIGSTOP or VM pause: the process stops responding and no + RST reaches the server. + + What survives server-side depends on the transaction shape, and the + distinction is the whole of #123 rather than a detail. A + deadline-scoped statement runs inside an explicit transaction whose + COMMIT is a separate client message, so the server is left holding an + open transaction and its locks indefinitely. A plain autocommit + statement has no such message: PostgreSQL finishes executing it and + commits it on its own, and the vanished client merely never reads the + answer. Modelling both as orphans would let a scenario construct the + outage shape on a code path that provably cannot produce it — #123 is + explicit that the shape did not exist before the deadline plumbing + arrived. + """ + actor = self.actor + if not actor.blocked and actor.stop is None: + raise HarnessError(f"coordinator {self.name!r} has not run yet") + for backend in [ + *(self.sql_backend._in_use if self.sql_backend else []), + ]: + transaction = backend.transaction + if transaction is None: + continue + if transaction.explicit: + self.harness.server.orphan(transaction) + else: + self.harness.server._commit(transaction) + actor.parked_forever = True + actor._blocked = True + actor._ready = lambda: False + actor._wake_at = None + self.harness.scheduler.trace.append(f"{self.name}@vanished") + + +class HarnessBase: + """What every state machine's harness owns, whichever one it drives. + + Issue #128 names four owners and this covers the second of them, so the + boundary matters more than it would for a one-off: the clock, the + scheduler, the PostgreSQL model, the stepping vocabulary and the output + capture are the same for landing as for the lease lifecycle, and will be + the same again for tip refresh and share submission. What differs is only + what a harness *builds* — coordinators here, a pool with two landing + tails there — so that is all a subclass supplies. + + ``run_scenario`` and ``assert_deterministic`` are typed against this + class rather than against any one harness, which is what stops the + determinism check from being re-derived per state machine. + """ + + def __init__( + self, + *, + capture_output: bool = True, + baton_timeout_seconds: float = BATON_TIMEOUT_SECONDS, + ) -> None: + self.clock = VirtualClock() + self.scheduler = DeterministicScheduler( + self.clock, + baton_timeout_seconds=baton_timeout_seconds, + ) + self.server = FakePostgres(self.scheduler, self.clock) + self.sleeps: list[tuple[str, float]] = [] + self._output = io.StringIO() + self._errors = io.StringIO() + self._redirects: list[contextlib.AbstractContextManager[Any]] = [] + if capture_output: + # Kept apart rather than merged. The coordinator reserves stdout + # for a strict JSON-lines protocol in worker processes and routes + # audit-path diagnostics to stderr, so which stream a message + # arrives on is itself an assertion a scenario may want to make. + self._redirects = [ + contextlib.redirect_stdout(self._output), + contextlib.redirect_stderr(self._errors), + ] + for redirect in self._redirects: + redirect.__enter__() + + # -- lifecycle --------------------------------------------------------- + + def __enter__(self) -> Self: + return self + + def __exit__(self, *exc_info: object) -> None: + self.close() + + def close(self) -> None: + self.scheduler.close() + for redirect in reversed(self._redirects): + redirect.__exit__(None, None, None) + self._redirects = [] + + @property + def output(self) -> str: + return self._output.getvalue() + + @property + def errors(self) -> str: + return self._errors.getvalue() + + @property + def trace(self) -> list[str]: + return list(self.scheduler.trace) + + # -- synchronisation --------------------------------------------------- + + def lock(self, name: str) -> HarnessLock: + return HarnessLock(self.scheduler, name) + + def rlock(self, name: str) -> HarnessRLock: + return HarnessRLock(self.scheduler, name) + + def condition(self, name: str) -> HarnessCondition: + return HarnessCondition(self.rlock(name), name=name) + + def event(self, name: str) -> HarnessEvent: + return HarnessEvent(self.scheduler, name) + + def semaphore(self, name: str, value: int) -> HarnessSemaphore: + return HarnessSemaphore(self.scheduler, name, value=value) + + # -- clock ------------------------------------------------------------- + + def sleep(self, seconds: float) -> None: + """Park until the virtual clock reaches the wake time.""" + actor = self.scheduler.current() + self.sleeps.append((actor.name, float(seconds))) + wake_at = self.clock.monotonic() + float(seconds) + while self.clock.monotonic() < wake_at: + self.scheduler.block(f"sleep:{seconds:g}", wake_at=wake_at) + + def advance(self, seconds: float) -> float: + return self.clock.advance(seconds) + + def advance_to_next_deadline(self) -> float | None: + """Jump to the earliest pending sleep or timeout. + + Returns the new virtual monotonic time, or None when no actor is + waiting on time at all — which is itself a finding: an actor blocked + with no deadline can only be released by another actor, and if none + can run, nothing bounds the wait. + """ + deadline = self.scheduler.next_deadline() + if deadline is None: + return None + return self.clock.advance_to(max(deadline, self.clock.monotonic())) + + # -- stepping ---------------------------------------------------------- + + def _actor(self, who: Any) -> Actor: + if isinstance(who, Actor): + return who + actor = getattr(who, "actor", None) + if isinstance(actor, Actor): + return actor + raise HarnessError(f"{who!r} is not an actor and does not own one") + + def step(self, who: Any) -> str: + return self.scheduler.step(self._actor(who)) + + def run_until(self, who: Any, label: str, *, limit: int = 200) -> str: + return self.scheduler.run_until(self._actor(who), label, limit=limit) + + def run_until_blocked(self, who: Any, label: str, *, limit: int = 200) -> str: + """Step until the actor parks at ``label`` and is genuinely stuck there.""" + actor = self._actor(who) + stop = self.scheduler.run_until(actor, label, limit=limit) + if not actor.blocked: + raise HarnessError( + f"actor {actor.name!r} reached {label!r} but is not blocked" + ) + return stop + + def drain(self, order: list[Any] | None = None, *, limit: int = 500) -> None: + """Step runnable actors in a fixed order until none can progress. + + Order is explicit and never depends on thread timing, so a drained + run is as reproducible as a hand-written schedule. + """ + actors = ( + [self._actor(who) for who in order] + if order is not None + else list(self.scheduler.actors.values()) + ) + for _ in range(limit): + progressed = False + for actor in actors: + if actor.runnable(): + self.scheduler.step(actor) + progressed = True + if not progressed: + return + raise HarnessError(f"drain did not quiesce within {limit} rounds") + + # -- observation ------------------------------------------------------- + + def statement_kinds(self) -> list[str]: + return [statement.kind.value for statement in self.server.statements] + + +class LeaseHarness(HarnessBase): + """Front end: build coordinators, step them, advance time, read the trace.""" + + def __init__( + self, + *, + lease_ttl_seconds: float = 60.0, + capture_output: bool = True, + baton_timeout_seconds: float = BATON_TIMEOUT_SECONDS, + **default_ledger_kwargs: Any, + ) -> None: + super().__init__( + capture_output=capture_output, + baton_timeout_seconds=baton_timeout_seconds, + ) + self.coordinators: dict[str, Coordinator] = {} + self._default_ledger_kwargs = { + "lease_ttl_seconds": lease_ttl_seconds, + **default_ledger_kwargs, + } + self._session_counter = 0 + + # -- construction ------------------------------------------------------ + + def coordinator( + self, + name: str, + *, + writer_id: str = "prism-coordinator", + writer_epoch: int = 1, + session_token: str | None = None, + heartbeat_capable: bool = True, + **ledger_kwargs: Any, + ) -> Coordinator: + self._session_counter += 1 + if session_token is None: + suffix = f"{name}-{self._session_counter}" + session_token = ( + f"{WRITER_LEASE_HEARTBEAT_SESSION_PREFIX}{suffix}" + if heartbeat_capable + else uuid.uuid5(uuid.NAMESPACE_OID, suffix).hex + ) + coordinator = Coordinator( + self, + name, + writer_id=writer_id, + writer_epoch=writer_epoch, + session_token=session_token, + ledger_kwargs={**self._default_ledger_kwargs, **ledger_kwargs}, + ) + self.coordinators[name] = coordinator + return coordinator + + # -- observation ------------------------------------------------------- + + def lease_row(self) -> LeaseRow | None: + return self.server.lease + + def lease_holder_session(self) -> str | None: + row = self.server.lease + return None if row is None else row.writer_session_token + + +# -------------------------------------------------------------------------- +# Determinism checking +# -------------------------------------------------------------------------- + + +@dataclass(frozen=True) +class ScenarioRun: + """What one execution of a scenario produced.""" + + trace: tuple[str, ...] + outcome: Any + + def digest(self) -> str: + return json.dumps( + {"trace": list(self.trace), "outcome": self.outcome}, + sort_keys=True, + default=repr, + ) + + +def run_scenario( + scenario: Callable[[Any], Any], + *, + harness_factory: Callable[..., HarnessBase] = LeaseHarness, + **harness_kwargs: Any, +) -> ScenarioRun: + """Execute ``scenario`` against a fresh harness and record what happened. + + ``harness_factory`` names which state machine's harness to build. It + defaults to the lease lifecycle only because that is what landed first; + nothing here knows anything about leases, and a scenario for any owner + deriving from ``HarnessBase`` runs unchanged. + """ + harness = harness_factory(**harness_kwargs) + try: + outcome = scenario(harness) + return ScenarioRun(trace=tuple(harness.trace), outcome=outcome) + finally: + harness.close() + + +def assert_deterministic( + test: Any, + scenario: Callable[[Any], Any], + *, + repeats: int = 25, + harness_factory: Callable[..., HarnessBase] = LeaseHarness, + **harness_kwargs: Any, +) -> ScenarioRun: + """Run ``scenario`` repeatedly and require an identical trace and outcome. + + A concurrency test that is merely usually right is worse than no test, so + every scenario in this suite is required to prove order-stability rather + than to assert it once. The checkpoint trace is the stronger half of the + check: two runs can agree on the answer while disagreeing about the + schedule that produced it, and that is exactly the flakiness this harness + exists to rule out. + """ + if repeats < 2: + raise HarnessError("determinism needs at least two runs to mean anything") + runs = [ + run_scenario( + scenario, + harness_factory=harness_factory, + **harness_kwargs, + ) + for _ in range(repeats) + ] + first = runs[0] + for index, run in enumerate(runs[1:], start=2): + if run.trace != first.trace: + test.fail( + f"run {index} produced a different schedule than run 1.\n" + f"run 1: {list(first.trace)}\n" + f"run {index}: {list(run.trace)}" + ) + if run.digest() != first.digest(): + test.fail( + f"run {index} produced a different outcome than run 1.\n" + f"run 1: {first.outcome!r}\n" + f"run {index}: {run.outcome!r}" + ) + return first + + +__all__ = [ + "Actor", + "Backend", + "Call", + "Coordinator", + "DeterministicScheduler", + "FakeLeaseGuard", + "FakePostgres", + "FakeSqlBackend", + "HarnessBase", + "HarnessCondition", + "HarnessError", + "HarnessEvent", + "HarnessLock", + "HarnessRLock", + "HarnessSemaphore", + "LandingOp", + "LeaseHarness", + "LeaseOp", + "LeaseRow", + "LockTimeout", + "NotRunnable", + "OutboxRow", + "PoolBlockRow", + "ScenarioRun", + "SchedulerStall", + "Statement", + "StatementKind", + "Transaction", + "UnsupportedStatement", + "VirtualClock", + "acquire_result_arms", + "acquire_terminal_arm_fields", + "acquire_terminal_arm_keys", + "advisory_lock_pair", + "assert_deterministic", + "classify", + "run_scenario", +] diff --git a/tests/prism_coordinator_test_support.py b/tests/prism_coordinator_test_support.py new file mode 100644 index 000000000..4a6cb9660 --- /dev/null +++ b/tests/prism_coordinator_test_support.py @@ -0,0 +1,500 @@ +#!/usr/bin/env python3 +"""Shared no-environment fixtures for coordinator job and delivery tests.""" +# ruff: noqa: F401 + +from __future__ import annotations + +import contextlib +import io +import os +import queue +import socket +import subprocess +import sys +import threading +import time +import unittest +from pathlib import Path +from concurrent.futures import Future +from contextlib import contextmanager +from dataclasses import dataclass, replace as dataclass_replace +from decimal import Decimal +from types import SimpleNamespace +from unittest.mock import patch + +from lab.auxpow import vardiff +from lab.prism import direct_stratum +from lab.prism import prism_coordinator as prism_coordinator_module +from lab.prism.coordinator_shutdown import ShutdownInProgress +from lab.prism.prism_coordinator import ( + ClientState, + JobBuildAdmissionDeadlineExceeded, + JobBuildSuperseded, + MAX_PRISM_JOB_BUNDLE_CACHE_ENTRIES, + PRISM_JOB_EXTRANONCE1_PLACEHOLDER_HEX, + PRISM_REJECTION_REASON_IDS, + PendingShareAppend, + PayoutLedgerArtifact, + PrismCoordinator, + TemplateRefreshBlocked, + WorkerIdentity, + _compact_share_payload, + canonical_json_sha256, + canonical_json_text, + now_ms, + qbit_template_fingerprint, +) +from lab.prism.coordinator_config import default_prism_coinbase_tag_hex +from lab.prism.payout_state import ( + PayoutStatePublicationBlocked as _PayoutStatePublicationBlocked, +) +from lab.prism.share_ledger import ( + LedgerOperationTimeout, + PendingShare, + SingleWriterShareLedger, +) + + +PAYOUT_ADDRESS = "tq1z70ukpvs96kye6jmgvl3nttevtkrq8uu89snkpm6m8gwqukw8u5dsz32kwa" + + +EXTRANONCE2_SIZE = 8 + + +@dataclass(frozen=True) +class FakeShare: + miner_id: str + share_seq: int + + def to_prism_json(self) -> dict[str, object]: + return {"share_seq": self.share_seq, "miner_id": self.miner_id} + + +class FakeLedger: + backend_name = "fake" + + def __init__(self, miners: list[str] | None = None) -> None: + self.miners = miners if miners is not None else ["miner-a", "miner-b", "miner-c"] + self.snapshot_calls = 0 + self.stats_calls = 0 + + def accepted_share_stats(self) -> dict[str, int]: + self.stats_calls += 1 + return { + "accepted_share_count": len(self.miners), + "distinct_miner_count": len(set(self.miners)), + } + + def all_shares(self) -> list[FakeShare]: + raise AssertionError("all_shares must not be called when accepted_share_stats exists") + + def snapshot_at_job_issue(self, anchor_job_issued_at_ms: int, *, window_weight: int | None = None) -> list[FakeShare]: + self.snapshot_calls += 1 + return [FakeShare(miner_id=miner, share_seq=seq + 1) for seq, miner in enumerate(self.miners)] + + def current_prior_balances(self) -> list[dict[str, object]]: + return [] + + def metrics(self) -> dict[str, int]: + return {"blocks": 0, "owed_accounts": 0} + + +class ReadyLedgerWithEmptyFirstSnapshot(FakeLedger): + def __init__(self) -> None: + super().__init__(miners=["miner-a", "miner-b", "miner-c"]) + + def snapshot_at_job_issue(self, anchor_job_issued_at_ms: int, *, window_weight: int | None = None) -> list[FakeShare]: + self.snapshot_calls += 1 + if self.snapshot_calls == 1: + return [] + return [FakeShare(miner_id=miner, share_seq=seq + 1) for seq, miner in enumerate(self.miners)] + + +class FakeRpc: + def __init__(self, template: dict[str, object], tip: str) -> None: + self.template = template + self.tip = tip + self.blockchain_info: dict[str, object] = { + "initialblockdownload": False, + "blocks": 100, + "headers": 100, + } + self.calls: list[str] = [] + + def call(self, method: str, params: list[object] | None = None) -> object: + self.calls.append(method) + if method == "getblocktemplate": + return dict(self.template) + if method == "getbestblockhash": + return self.tip + if method == "getblockchaininfo": + return dict(self.blockchain_info) + if method == "getblockcount": + return int(self.blockchain_info["blocks"]) + raise AssertionError(f"unexpected RPC {method}") + + def count(self, method: str) -> int: + return sum(1 for name in self.calls if name == method) + + +def synthetic_manifest_coinbase_hex(suffix_hex: str) -> str: + """A structurally valid non-witness coinbase whose scriptSig ends with the + extranonce placeholder suffix, as the audit bundle builder produces.""" + height_push = "03aabbcc" + script_sig = height_push + suffix_hex + script_sig_bytes = bytes.fromhex(script_sig) + output = (50_00000000).to_bytes(8, "little").hex() + "0151" + return ( + "01000000" + + "01" + + "00" * 32 + + "ffffffff" + + direct_stratum.compact_size(len(script_sig_bytes)).hex() + + script_sig + + "ffffffff" + + "01" + + output + + "00000000" + ) + + +def base_template(height: int = 10, prevhash: str = "11" * 32) -> dict[str, object]: + # Realistic (non-regtest) bits: the network target must be harder than the + # vardiff range for per-client share targets to differ, as on testnet4. + return { + "height": height, + "previousblockhash": prevhash, + "bits": "1b00ffff", + "version": 0x20000000, + "curtime": 1_700_000_000, + "coinbasevalue": 50_00000000, + "transactions": [], + } + + +def durable_candidate_row(index: int) -> dict[str, object]: + """A minimal valid outbox row whose intent survives block_candidate_from_intent.""" + block_hash = f"{index + 1:064x}" + return { + "block_hash": block_hash, + "candidate": { + "schema": "qbit.prism.block-candidate-intent.v1", + "block_hash_hex": block_hash, + "block_hex": "00", + "coinbase_tx_hex": "00", + "parent_hash": "11" * 32, + "expected_height": 10, + "template": { + "previousblockhash": "11" * 32, + "height": 10, + "coinbasevalue": 50_00000000, + }, + "shares_json": [], + "prior_balances": [], + "found_block": {}, + "prospective_prior_balances": None, + "witness_merkle_leaves_hex": [], + "extranonce1_hex": "00000001", + "extranonce2_hex": "00", + "username": "miner-a", + "pending_share": { + "share_id": f"miner-a:{block_hash}", + "miner_id": "miner-a", + "order_key": "miner-a", + "p2mr_program_hex": "22" * 32, + "share_difficulty": 1, + "network_difficulty": 1, + "template_height": 10, + "job_id": "job-1", + "job_issued_at_ms": 1, + "accepted_at_ms": 1, + "ntime": 1, + }, + "credit_share_on_accept": False, + "collection_only": False, + }, + } + + +def worker(payout: str = PAYOUT_ADDRESS, username: str | None = None) -> WorkerIdentity: + return WorkerIdentity( + username=username or payout, + payout_address=payout, + worker_name=None, + script_pubkey_hex="5220" + "22" * 32, + p2mr_program_hex="22" * 32, + ) + + +def client(connection_id: int, identity: WorkerIdentity | None = None) -> ClientState: + state = ClientState.__new__(ClientState) + state.sock = None + state.address = ("127.0.0.1", 40_000 + connection_id) + state.connection_id = connection_id + state.extranonce1_hex = f"{connection_id:08x}" + state.subscribed = True + state.authorized = True + identity = identity or worker() + state.username = identity.username + state.worker = identity + state.version_mask = 0 + state.active_job = None + state.share_difficulty = Decimal("1") + state.pending_share_difficulty = None + state.active_job_ids = set() + state.post_accept_refresh_block = None + state.tip_work_delivered = None + state.closing = False + state.job_update_lock = threading.RLock() + state.send_lock = threading.Lock() + return state + + +class ObservedRLock: + """RLock test double that exposes a contending acquire without sleeps.""" + + def __init__(self) -> None: + self._lock = threading.RLock() + self.observe_acquires = False + self.acquire_attempted = threading.Event() + + def acquire(self, blocking: bool = True, timeout: float = -1) -> bool: + if self.observe_acquires: + self.acquire_attempted.set() + return self._lock.acquire(blocking, timeout) + + def release(self) -> None: + self._lock.release() + + def __enter__(self) -> ObservedRLock: + self.acquire() + return self + + def __exit__(self, *_args: object) -> None: + self.release() + + +def coordinator(*, ledger: object | None = None, template: dict[str, object] | None = None) -> tuple[PrismCoordinator, FakeRpc]: + server = PrismCoordinator.__new__(PrismCoordinator) + template = template or base_template() + rpc = FakeRpc(template, tip=str(template["previousblockhash"])) + server.rpc = rpc + server.qbit_chain = "regtest" + server.lock = threading.RLock() + server.stop_event = threading.Event() + server.clients = set() + server.jobs = {} + server.job_counter = 0 + server.connection_counter = 0 + server.accepted_block_count = 0 + server.max_blocks = 1_000 + server.started_monotonic = time.monotonic() + server.submitted_share_count = 0 + server.stale_share_count = 0 + server.duplicate_share_count = 0 + server.low_difficulty_share_count = 0 + server.rejection_counts_by_reason = {reason: 0 for reason in PRISM_REJECTION_REASON_IDS} + server.job_build_failure_count = 0 + server.tip_refresh_job_count = 0 + server.post_accept_refresh_failure_count = 0 + server.reorg_reconciler_enabled = False + server.reorg_inactive_block_count = 0 + server.reorg_reactivated_block_count = 0 + server.reorg_reconcile_skip_count = 0 + server.reorg_reconcile_error_count = 0 + server.matured_payout_count = 0 + server.last_reorg_reconciled_tip_hash = None + server.last_reorg_reconciled_trusted = False + server.last_reorg_reconciled_monotonic = None + server.latest_evidence = None + server.latest_coinbase_size_bytes = None + server.tip_template_snapshot = None + server.extranonce2_size = EXTRANONCE2_SIZE + server.coinbase_tag_hex = default_prism_coinbase_tag_hex() + server.share_difficulty = Decimal("1") + server.vardiff_config = vardiff.VardiffConfig( + enabled=True, + target_share_interval_seconds=Decimal("15"), + min_difficulty=Decimal("0.000000001"), + max_difficulty=Decimal("1024"), + retarget_interval_seconds=Decimal("90"), + max_step_factor=Decimal("4"), + startup_difficulty=Decimal("1"), + max_step_down_factor=Decimal("4"), + ewma_alpha=Decimal("0.4"), + retarget_tolerance=Decimal("0.25"), + ) + server.default_share_weight = 1 + server.share_weights_by_username = {} + server.min_ready_miners = 3 + server.ledger = ledger if ledger is not None else FakeLedger() + server.blockpoll_seconds = 2.0 + # Failed-refresh spacing is opt-in per test: its holdoff waits on real + # time, which deadlocks tests that freeze time.monotonic around failing + # polls. Pacing behavior is covered by test_prism_refresh_retry_pacing. + server.tip_refresh_failure_holdoff_seconds = 0.0 + server.job_bundle_cache_seconds = 10.0 + server.template_cache_seconds = 2.0 + server.reorg_reconcile_cache_seconds = 5.0 + server.health_refresh_seconds = 5.0 + server.stratum_send_timeout_seconds = 20.0 + server._ensure_job_cache_state() + return server, rpc + + +def install_fake_bundle_builder(server: PrismCoordinator) -> dict[str, object]: + """Replace the audit bundle subprocess with a counting fake whose manifest + coinbase embeds exactly the suffix the coordinator asked for.""" + recorded: dict[str, object] = {"calls": 0, "suffixes": []} + + def fake_build_audit_bundle(**kwargs: object) -> dict[str, object]: + recorded["calls"] = int(recorded["calls"]) + 1 + suffix_hex = str(kwargs["coinbase_script_sig_suffix_hex"]) + recorded["suffixes"].append(suffix_hex) + recorded["last_kwargs"] = kwargs + return { + "found_block": dict(kwargs["found_block"]), + "payout_policy_manifest": {"accounts": []}, + "signed_coinbase_manifest": { + "manifest": { + "coinbase_tx_hex": synthetic_manifest_coinbase_hex(suffix_hex), + } + }, + } + + server.build_audit_bundle = fake_build_audit_bundle # type: ignore[method-assign] + server._ensure_bundle_compiler().build_audit_bundle = ( # type: ignore[method-assign] + fake_build_audit_bundle + ) + return recorded + + +def stamped_pending_share(accepted_at_ms: int) -> PendingShare: + return PendingShare( + share_id=f"miner-a:{accepted_at_ms}", + miner_id="miner-a", + order_key="miner-a", + p2mr_program_hex="22" * 32, + share_difficulty=1, + network_difficulty=1, + template_height=9, + job_id="job-1", + job_issued_at_ms=accepted_at_ms - 1, + accepted_at_ms=accepted_at_ms, + ntime=1_700_000_000, + ) + + +class AnchorRecordingLedger(FakeLedger): + def __init__(self) -> None: + super().__init__() + self.anchors: list[int] = [] + + def snapshot_at_job_issue( + self, anchor_job_issued_at_ms: int, *, window_weight: int | None = None + ) -> list[FakeShare]: + self.anchors.append(int(anchor_job_issued_at_ms)) + return super().snapshot_at_job_issue( + anchor_job_issued_at_ms, window_weight=window_weight + ) + + +class IncrementalRecordingLedger(SingleWriterShareLedger): + def __init__(self) -> None: + super().__init__() + self.full_snapshot_calls = 0 + self.delta_snapshot_calls = 0 + + def snapshot_at_job_issue( + self, + anchor_job_issued_at_ms: int, + *, + window_weight: int | None = None, + ) -> list[object]: + self.full_snapshot_calls += 1 + return super().snapshot_at_job_issue( + anchor_job_issued_at_ms, + window_weight=window_weight, + ) + + def snapshot_between_job_issues( + self, + previous_anchor_job_issued_at_ms: int, + anchor_job_issued_at_ms: int, + ) -> list[object]: + self.delta_snapshot_calls += 1 + return super().snapshot_between_job_issues( + previous_anchor_job_issued_at_ms, + anchor_job_issued_at_ms, + ) + + +def append_incremental_share( + ledger: SingleWriterShareLedger, + *, + share_seq: int, + accepted_at_ms: int, +) -> None: + ledger.append( + PendingShare( + share_id=f"incremental-{share_seq}", + miner_id=f"miner-{share_seq % 3}", + order_key=f"miner-{share_seq % 3}", + p2mr_program_hex=f"{share_seq % 256:02x}" * 32, + share_difficulty=1, + network_difficulty=1, + template_height=9, + job_id=f"job-{share_seq}", + job_issued_at_ms=accepted_at_ms - 1, + accepted_at_ms=accepted_at_ms, + ntime=1_700_000_000 + share_seq, + ) + ) + + +def mark_progress_healthy(server: PrismCoordinator) -> None: + snapshot = server.fetch_qbit_tip_template_snapshot() + server._record_progress_tip_poll(snapshot) + server._record_progress_publication( + snapshot, + int(getattr(server, "_payout_state_generation", 0)), + ) + + +def spool_share(seq: int) -> dict[str, object]: + return { + "share_seq": seq, + "share_id": f"share-{seq}", + "miner_id": "miner-a", + "order_key": "miner-a", + "p2mr_program_hex": "22" * 32, + "share_difficulty": 1, + "job_issued_at_ms": 1_700_000_000_000 + seq, + "accepted_at_ms": 1_700_000_000_100 + seq, + "credit_policy": None, + } + + +ECHO_BUILDER_COMMAND = [ + sys.executable, + "-c", + "import json,sys; json.dump({'received': json.load(sys.stdin)}, sys.stdout)", +] + + +FAKE_SERVE_BUILDER_COMMAND = [ + sys.executable, + str(Path(__file__).resolve().parent / "fixtures" / "fake_serve_builder.py"), +] + + +def build_job_cache_test_coordinator( + *, + ledger: object | None = None, + template: dict[str, object] | None = None, +) -> tuple[PrismCoordinator, FakeRpc]: + """Build a coordinator fixture without production init or worker startup.""" + return coordinator(ledger=ledger, template=template) + + +__all__ = [name for name in globals() if not name.startswith("__")] diff --git a/tests/prism_landing_harness.py b/tests/prism_landing_harness.py new file mode 100644 index 000000000..5d12fb4f3 --- /dev/null +++ b/tests/prism_landing_harness.py @@ -0,0 +1,951 @@ +#!/usr/bin/env python3 +"""Deterministic harness for PRISM's block candidate landing state machine. + +Issue #128 ranks landing first by incident history, and its shape is the +reason: a found block is real money, and the landing is the one owner where +two threads of the *same* process reach the same durable rows. The lease +harness in ``prism_concurrency_harness`` interleaves two coordinators around +one row; this one interleaves the two tails one coordinator lands on. + +The two tails +------------- +``lab/prism/block_candidates.py`` runs landings down two routes: + +``_submit_synchronous_block_candidate`` + The miner connection thread. A share that is block-worthy at the + assigned target lands inline on the client's own thread, which is + normal while the listener floor sits above network difficulty. + +``block_accounting_loop`` / ``_run_block_accounting_task`` + The dedicated accounting actor, which lands everything else. + +The disposition lease that serialises them is **per hash** +(``block_candidates.py``), so it provides no mutual exclusion between two +*distinct* found blocks. That is the topology #133 turns on, and expressing +it is what this harness exists for. + +What is real here and what is modelled +-------------------------------------- +Real: ``BlockCandidateService``, ``BlockFinalizationService``, +``AuditArtifactStore`` over a genuine audit root, ``PsqlShareLedger`` +emitting its shipped SQL, and the coordinator's own landing arithmetic — +budgets, watchdog ceiling, epoch fences, publication ordinal handling. + +Modelled: PostgreSQL (``FakePostgres``, extended with ``qbit_pool_blocks`` +and ``qbit_block_candidate_outbox``), qbitd (``LandingRpc``), the audit +bundle build and its verification, and every lock the two tails share — +installed as harness locks so a contended acquire is an interleaving point +rather than a wedged baton. + +The audit root is a real directory on purpose. What #133 loses is a *file* +that is never written, and no in-memory double can fail to write it. +""" + +from __future__ import annotations + +import json +import shutil +import sys +import tempfile +import threading +from collections.abc import Callable +from dataclasses import dataclass +from pathlib import Path +from types import SimpleNamespace +from typing import Any + +sys.path.insert(0, str(Path(__file__).resolve().parents[1])) + +from tests.prism_concurrency_harness import ( # noqa: E402 + BATON_TIMEOUT_SECONDS, + Actor, + Call, + FakeLeaseGuard, + FakePostgres, + FakeSqlBackend, + HarnessBase, + HarnessCondition, + HarnessError, + PoolBlockRow, +) +from lab.prism.block_candidates import ( # noqa: E402 + PrismBlockCandidate, + _BlockCandidateAccountingTask, + _BlockCandidateNodeSubmission, +) +from lab.prism.share_ledger import ( # noqa: E402 + WRITER_LEASE_HEARTBEAT_SESSION_PREFIX, + PendingShare, + PsqlShareLedger, +) + +PARENT_HASH = "00" * 32 +DEFAULT_HEIGHT = 10 + + +# -------------------------------------------------------------------------- +# qbitd +# -------------------------------------------------------------------------- + + +class LandingRpc: + """The node, as the landing path sees it. + + ``submitblock`` accepts by default and the chain tip follows the last + accepted block, because a landing that is *rejected* never reaches the + interleavings this harness is for. A scenario overrides ``submit_result`` + to make one landing fail. + """ + + def __init__(self, *, tip: str, height: int) -> None: + self.tip = tip + self.height = height + self.calls: list[tuple[str, tuple[Any, ...]]] = [] + self.submitted: list[str] = [] + self.submit_result: Callable[[str], object] | None = None + self.blocks_by_height: dict[int, str] = {} + self._registered: dict[str, tuple[str, int]] = {} + + def register(self, *, block_hex: str, block_hash: str, height: int) -> None: + """Teach the node which block a submitted payload is.""" + self._registered[block_hex] = (block_hash, int(height)) + + def call(self, method: str, params: list[object] | None = None) -> object: + args = tuple(params or ()) + self.calls.append((method, args)) + if method == "getbestblockhash": + return self.tip + if method == "getblockcount": + return self.height + if method == "getblockhash": + requested = int(args[0]) + return self.blocks_by_height.get(requested, self.tip) + if method == "submitblock": + return self._submit(str(args[0]) if args else "") + if method == "getblockchaininfo": + return {"initialblockdownload": False} + if method == "getnetworkinfo": + return {"connections": 4} + if method == "getblockheader": + raise RuntimeError("qbit RPC getblockheader failed: -5 Block not found") + raise RuntimeError(f"unmodelled qbit RPC method {method}") + + def _submit(self, block_hex: str) -> object: + self.submitted.append(block_hex) + result = None if self.submit_result is None else self.submit_result(block_hex) + if result in (None, "duplicate"): + known = self._registered.get(block_hex) + if known is not None: + self.accept(block_hash=known[0], height=known[1]) + return result + + def accept(self, *, block_hash: str, height: int) -> None: + """Make ``block_hash`` the active block at ``height``.""" + self.blocks_by_height[height] = block_hash + self.tip = block_hash + self.height = height + + +# -------------------------------------------------------------------------- +# Candidates +# -------------------------------------------------------------------------- + + +@dataclass(frozen=True) +class LandingCandidate: + """One found block, plus everything a scenario needs to talk about it.""" + + block_hash: str + block_height: int + job_id: str + candidate: PrismBlockCandidate + + @property + def envelope_name(self) -> str: + return f"{self.block_height}-{self.block_hash}" + + +# -------------------------------------------------------------------------- +# The harness +# -------------------------------------------------------------------------- + + +class LandingHarness(HarnessBase): + """Front end: build found blocks, land them on either tail, read the trace.""" + + def __init__( + self, + *, + parent_hash: str = PARENT_HASH, + block_height: int = DEFAULT_HEIGHT, + lease_ttl_seconds: float = 60.0, + watchdog_timeout_seconds: float = 120.0, + block_landing_db_timeout_seconds: float = 30.0, + read_concurrency: int = 4, + capture_output: bool = True, + baton_timeout_seconds: float = BATON_TIMEOUT_SECONDS, + ) -> None: + super().__init__( + capture_output=capture_output, + baton_timeout_seconds=baton_timeout_seconds, + ) + self.parent_hash = parent_hash + self.block_height = int(block_height) + self.lease_ttl_seconds = float(lease_ttl_seconds) + self.read_concurrency = int(read_concurrency) + self._audit_root = Path(tempfile.mkdtemp(prefix="prism-landing-harness-")) + self._job_counter = 0 + self.sql_backend: FakeSqlBackend | None = None + self.guard: FakeLeaseGuard | None = None + self.session_token = f"{WRITER_LEASE_HEARTBEAT_SESSION_PREFIX}landing" + # Which external-side-effect components this run refuses authority + # for, and which ones actually asked. A scenario sets the first to + # drive the degraded branch; the second is evidence the gate ran. + self.withheld_lease_components: set[str] | None = None + self.lease_fence_components: list[str] = [] + # Production phase names a scenario wants to stop at (see + # _install_phase_breakpoints). + self.breakpoints: set[str] = set() + + self.rpc = LandingRpc(tip=parent_hash, height=self.block_height - 1) + self.pool = self._build_pool( + watchdog_timeout_seconds=watchdog_timeout_seconds, + block_landing_db_timeout_seconds=block_landing_db_timeout_seconds, + ) + + # The tails. Named for the production threads they stand in for, so a + # checkpoint trace reads as "which tail was running". + self.client = self.scheduler.actor("client") + self.accounting = self.scheduler.actor("accounting") + # Startup is its own actor: the ledger constructor acquires the + # writer lease, which is a modelled statement and so has to run on an + # actor thread like everything else. + self.startup = self.scheduler.actor("startup") + # The share writer's side of the epoch machinery. A landing's fences + # exist to order against it, so it has to be a third schedulable + # thread rather than something the test body does between steps. + self.appender = self.scheduler.actor("appender") + + # -- lifecycle --------------------------------------------------------- + + def close(self) -> None: + super().close() + store = self.pool.__dict__.get("_audit_artifact_store") + if store is not None: + # Teardown runs on the controller thread, which is not an actor, + # so the store's locks go back to ordinary ones before its own + # close path takes them. The scheduler is already shut down by + # then; nothing is left to schedule. + store._lock = threading.RLock() + store._publication_lifecycle_lock = threading.RLock() + store.close() + shutil.rmtree(self._audit_root, ignore_errors=True) + + # -- construction ------------------------------------------------------ + + def _build_pool( + self, + *, + watchdog_timeout_seconds: float, + block_landing_db_timeout_seconds: float, + ) -> Any: + # Imported here rather than at module scope: the support module pulls + # in the whole coordinator surface, and only a landing scenario needs + # it. + from tests.prism_vardiff_test_support import submit_coordinator + + pool, client_state, _ledger = submit_coordinator(tip=self.parent_hash) + self.client_state = client_state + pool.max_blocks = 1_000 + pool.stop_after_block = False + pool.rpc = self.rpc + pool.watchdog_timeout_seconds = watchdog_timeout_seconds + pool.block_landing_db_timeout_seconds = block_landing_db_timeout_seconds + pool.block_submitter_db_timeout_seconds = block_landing_db_timeout_seconds + pool.audit_dir = self._audit_root / "audit" + pool.evidence_path = self._audit_root / "state" / "evidence.json" + pool.ledger_writer_public_key_hex = "aa" * 32 + # The bundle build and its verification are the audit machine, not + # the landing machine. #128 scopes this harness to landing, so both + # are answered from a fixed report: a landing that turns on bundle + # content would be testing the wrong owner. + pool.build_audit_bundle = lambda **_kwargs: _verified_block_bundle() + pool.verify_bundle = ( + lambda *_args, expected_block_height=DEFAULT_HEIGHT, **_kwargs: ( + _verified_audit_report(block_height=expected_block_height) + ) + ) + # Reconciliation walks the chain through the node one block at a + # time. It is the reorg reconciler's state machine, and #128 lists it + # separately; a landing only needs its verdict. + pool.ensure_reorg_reconciled_for_tip = lambda _tip, **_kwargs: True + pool.reorg_reconciler_enabled = False + self._install_harness_locks(pool) + self._install_inline_outbox_worker(pool) + self._install_inline_external_lease_fence(pool) + self._install_phase_breakpoints(pool) + return pool + + def _install_phase_breakpoints(self, pool: Any) -> None: + """Let a scenario stop a tail at a phase the landing already names. + + SQL statement boundaries are the harness's default interleaving + points, and for the lease lifecycle they were enough: every state + transition there *is* a statement. Landing is not like that. Its + interesting boundaries are the gaps *between* statements — the + confirm→publish window is bounded on one side by a durable + confirmation and on the other by a lock the publish step has not + taken yet, and no statement runs at the instant that matters. + + The landing already names those boundaries for its own watchdog, in + ``_record_block_submitter_phase`` and + ``_record_block_candidate_progress``. Reusing that vocabulary means a + scenario says where it wants to interleave in the code's own words — + ``progress:evidence-write`` is the top of the publish step — instead + of counting statements and hoping the count survives a refactor. An + armed phase that never fires is a loud failure, because the scenario + then waits at a checkpoint that never arrives. + """ + record_progress = pool._record_block_candidate_progress + record_phase = pool._record_block_submitter_phase + + def progress(phase: str = "accounting-progress") -> None: + record_progress(phase) + self._maybe_break(f"progress:{phase}") + + def submitter_phase(phase: str) -> None: + record_phase(phase) + self._maybe_break(f"phase:{phase}") + + pool._record_block_candidate_progress = progress + pool._record_block_submitter_phase = submitter_phase + + def _maybe_break(self, label: str) -> None: + if label in self.breakpoints: + self.scheduler.checkpoint(label) + + def break_at(self, *labels: str) -> None: + """Arm one or more phase breakpoints for every tail.""" + self.breakpoints.update(labels) + + def _install_inline_outbox_worker(self, pool: Any) -> None: + """Run the durable outbox calls on the caller's own actor. + + This is the harness's one substantive model boundary, so it is worth + being exact about. + + The *landing-class* ledger steps — persist, confirm, the ordinal + read-back, the pool-block reads — already run inline on the + block-work thread in production, so they arrive here unchanged, with + their real budgets, their real admission gates and their real + statements. The **outbox** marks do not: they run on a spawned + bounded worker thread (``_run_block_submitter_ledger_call``) so a + driver that ignores its statement deadline cannot wedge the + submitter, and the caller waits on a ``threading.Event`` in slices. + + A baton scheduler cannot schedule that worker. It is created inside + the wrapper with no factory seam, and the caller blocks on a real + event while holding the baton, so the worker could only ever run + unscheduled — which would put its statements outside the schedule + the harness exists to control. Running the operation inline instead + keeps everything the landing state machine depends on (the operation + itself, its budget scope, its error propagation and the per-class + latency metric) and drops only the liveness wrapper around it. + + What that costs is honest to state: this harness cannot express a + *stuck* outbox driver, so the restart escalations in + ``_maybe_restart_for_stuck_block_call`` are out of its reach. Those + are covered example-wise in the block-candidate suite. Giving the + harness the wrapper too needs a thread-factory seam in production, + which belongs to whichever issue wants that interleaving. + """ + service = pool._ensure_block_candidate_service() + + def run_inline( + key: tuple[object, ...], + phase: str, + operation: Callable[[], Any], + *, + timeout_seconds: float | None = None, + call_class: str = "fast", + ) -> Any: + budget = ( + service._block_submitter_db_timeout() + if timeout_seconds is None + else max(0.001, float(timeout_seconds)) + ) + started = self.clock.monotonic() + + def observe(timed_out: bool) -> None: + pool._record_block_ledger_call( + call_class=call_class, + budget_seconds=budget, + duration_seconds=max(0.0, self.clock.monotonic() - started), + timed_out=timed_out, + ) + + try: + with service._block_submitter_ledger_timeout_scope(budget): + result = operation() + except BaseException as exc: + observe(isinstance(exc, TimeoutError)) + raise + observe(False) + return result + + pool._run_block_submitter_ledger_call = run_inline + + def _install_inline_external_lease_fence(self, pool: Any) -> None: + """Prove the guarded session on the caller's own actor. + + ``_require_fresh_ledger_lease_for_external_side_effect`` gates two + landing steps that reach outside PostgreSQL: the fallback lane's + ``submitblock``, and the restore of a superseded audit envelope. Both + gates matter here — the second is the authority check #133's fix + depends on — so the harness keeps them, and keeps the real + verification statement behind them. + + What it cannot keep is the shape of the wait. The production fence + runs the verification on a spawned thread and joins it in wall-clock + time, so that it can hard-exit a process whose guard session has + genuinely stopped answering. The baton cannot schedule that thread + (see ``_install_inline_outbox_worker``), and a verification that runs + unscheduled would take its statement outside the schedule. Running it + inline preserves every outcome the landing branches on: the + verification's own SQL, the deferred-renewal refusal, and a refusal + that withholds restore authority. + + Out of reach, and named rather than implied: the fence's *timeout* + arm — a guard session that answers too slowly, or not at all. + """ + from lab.prism.background_services import guard_session_verifier + from lab.prism.coordinator_shutdown import ShutdownInProgress + from lab.prism.share_ledger import WriterLeaseRenewalDeferred + + def require_fresh_lease(component: str) -> None: + if self.withheld_lease_components is not None and ( + component in self.withheld_lease_components + ): + raise ShutdownInProgress( + f"writer lease guard verification failed before {component}" + ) + ledger = pool.ledger + if not bool(getattr(ledger, "writer_lease_guard_required", False)): + return + verify = guard_session_verifier(ledger) + if verify is None: + raise ShutdownInProgress( + f"writer lease guard verification is unavailable before {component}" + ) + self.lease_fence_components.append(component) + try: + result = verify() + except BaseException as exc: + raise ShutdownInProgress( + f"writer lease guard verification failed before {component}" + ) from exc + if isinstance(result, dict) and result.get( + "renewal_deferred_to_own_write" + ): + raise WriterLeaseRenewalDeferred( + f"withholding {component}: writer lease renewal is deferred " + "behind this coordinator's own in-flight fenced write" + ) + + pool._require_fresh_ledger_lease_for_external_side_effect = ( + require_fresh_lease + ) + + def _install_harness_locks(self, pool: Any) -> None: + """Replace every lock the two tails share with a schedulable one. + + Semantics are preserved exactly — a mutex stays a mutex, a + reentrant lock stays reentrant, the condition keeps its lock — and + only the blocking mechanism changes. Nothing in the code under test + can tell the difference except that a contended wait now parks at a + named checkpoint instead of outside the baton. + + This list is deliberately explicit rather than a sweep over + ``threading`` objects: a lock that appears in a later slice and is + not named here will stall the scheduler at a checkpoint that says + exactly which wait was unmodelled, which is the failure mode worth + having. + """ + pool._ensure_job_cache_state() + pool.lock = self.rlock("coordinator") + pool._job_cache_lock = self.lock("job-cache") + pool._payout_balance_mutation_lock = self.rlock("payout-balance-mutation") + pool._payout_append_landing_fence_lock = self.lock("payout-append-landing-fence") + pool._payout_state_prepare_lock = self.rlock("payout-state-prepare") + # Rebuilt rather than reused: the condition is defined over the job + # cache lock, and it has just been replaced. + pool._payout_unfenced_append_drained = HarnessCondition( + pool._job_cache_lock, + name="unfenced-append-drained", + ) + pool._accepted_block_payout_preview_condition = self.condition( + "accepted-block-payout-preview" + ) + pool._tip_refresh_lock = self.lock("tip-refresh") + + def _sql_backend_factory( + self, + conninfo: str, + *, + pool_size: int, + application_name: str, + ) -> FakeSqlBackend: + self.sql_backend = FakeSqlBackend( + self.server, + pool_size=pool_size, + application_name=application_name, + tag="ledger", + ) + return self.sql_backend + + def _lease_guard_factory( + self, + conninfo: str, + *, + advisory_lock_key: int, + ) -> FakeLeaseGuard: + self.guard = FakeLeaseGuard( + self.server, + advisory_lock_key=advisory_lock_key, + application_name="landing-guard", + tag="ledger.guard", + ) + return self.guard + + def _build_ledger(self) -> PsqlShareLedger: + ledger = PsqlShareLedger( + psql_command="psql postgres://harness/qbit", + database_url="postgres://harness/qbit", + writer_id="prism-coordinator", + writer_epoch=1, + writer_session_token=self.session_token, + initialize_schema=False, + lease_ttl_seconds=self.lease_ttl_seconds, + read_concurrency=self.read_concurrency, + monotonic=self.clock.monotonic, + lease_retry_sleep=self.sleep, + sql_backend_factory=self._sql_backend_factory, + lease_guard_factory=self._lease_guard_factory, + # Canonicalization is a Rust binary that validates audit bundle + # schema. It is the audit machine, not the landing machine, and + # the bundle here is a fixed stand-in; the constructor already + # takes the seam. + audit_bundle_canonicalizer=_canonicalize_bundle, + ) + # The ledger's own admission gates. Both tails queue on them, and + # #125 is precisely about how long a caller waits here, so they have + # to be visible to the scheduler and to the virtual clock. + ledger._lock = self.lock("ledger-writer") + ledger._read_semaphore = self.semaphore( + "ledger-read-slot", + self.read_concurrency, + ) + # The accepted-share-stats single-flight. Both tails read the stats + # while building their evidence, so the second one queues here behind + # the first — which is exactly the confirm→publish gap #133 needs to + # be *open*, not blocked. + ledger._stats_refresh_lock = self.lock("ledger-stats-refresh") + ledger._stats_lock = self.lock("ledger-stats") + return ledger + + def boot(self) -> PsqlShareLedger: + """Acquire the writer lease and attach the ledger to the pool. + + Runs on its own actor because lease acquisition is modelled SQL. + Returns once the lease is held, so a scenario that only cares about + landing never has to step through startup. + """ + call = self.startup.submit(self._build_ledger, label="ledger-startup") + self.run_until(self.startup, "done:ledger-startup") + ledger = call.value() + self.pool.ledger = ledger + self._install_audit_store_locks() + return ledger + + def _install_audit_store_locks(self) -> None: + """Make the publication order guard schedulable. + + The guard is what serialises confirm against publish inside one + landing, and both tails take it. #133 is a *released*-guard defect — + the interleaving happens in the gap where neither tail holds it — so + the point of installing a harness lock here is the opposite of + enabling the bug: it lets a scenario prove the guard is genuinely + released in that gap, by having the other tail take it and proceed. + """ + store = self.pool._ensure_audit_artifact_store() + store._lock = self.rlock("audit-store") + store._publication_lifecycle_lock = self.rlock("audit-publication-lifecycle") + + # -- found blocks ------------------------------------------------------ + + def found_block( + self, + block_hash: str, + *, + height: int | None = None, + parent_hash: str | None = None, + anchor_job_issued_at_ms: int = 12_000, + append_epoch: int | None = None, + collection_only: bool = False, + ) -> LandingCandidate: + """Register a job and mint the found-block candidate it produced. + + ``append_epoch`` stamps the candidate's build key with a payout + append-invalidation epoch. It defaults to the live epoch, which is + what an ordinary job carries; a scenario that wants the landing to + see a *moved* epoch bumps the live one afterwards rather than lying + about what the job was built from. + """ + self._job_counter += 1 + job_id = f"job-{self._job_counter}" + block_height = self.block_height if height is None else int(height) + parent = self.parent_hash if parent_hash is None else str(parent_hash) + if append_epoch is None: + append_epoch = int(self.pool._payout_ledger_append_invalidation_epoch) + context = SimpleNamespace( + job=SimpleNamespace( + job_id=job_id, + share_target=(1 << 256) - 1, + share_difficulty=1, + transaction_hexes=(), + ), + template={ + "previousblockhash": parent, + "height": block_height, + "coinbasevalue": 50_00000000, + }, + found_block={ + "network_difficulty": 1, + "anchor_job_issued_at_ms": int(anchor_job_issued_at_ms), + }, + issued_at_ms=int(anchor_job_issued_at_ms), + collection_only=collection_only, + worker=self.client_state.worker, + shares_json=[], + prior_balances=[], + payout_append_invalidation_epoch=int(append_epoch), + ) + self.pool.jobs[job_id] = context + self.client_state.active_job_ids.add(job_id) + block_hex = f"block:{block_hash}" + self.rpc.register( + block_hex=block_hex, + block_hash=block_hash, + height=block_height, + ) + submission = SimpleNamespace( + coinbase_tx_hex="c0ffee", + block_hash_hex=block_hash, + block_hex=block_hex, + share_pass=False, + block_pass=True, + ) + candidate = PrismBlockCandidate( + context=context, + submission=submission, + extranonce1_hex=self.client_state.extranonce1_hex, + extranonce2_hex="00" * 8, + # A real PendingShare, not a stand-in: the durable outbox intent + # encodes it, so the row the landing replays from is only as + # honest as this is. + pending_share=PendingShare( + share_id=f"miner-a:{block_hash}", + miner_id="miner-a", + order_key="miner-a", + p2mr_program_hex="11" * 32, + share_difficulty=1, + network_difficulty=1, + template_height=block_height - 1, + job_id=job_id, + job_issued_at_ms=int(anchor_job_issued_at_ms), + accepted_at_ms=int(anchor_job_issued_at_ms), + ntime=1_700_000_000, + ), + client=self.client_state, + credit_share_on_accept=False, + ) + return LandingCandidate( + block_hash=block_hash, + block_height=block_height, + job_id=job_id, + candidate=candidate, + ) + + # -- driving the tails ------------------------------------------------- + + def _persist_intent(self, found: LandingCandidate) -> None: + """Write the durable outbox row this candidate replays from. + + Production writes it at the pre-submit boundary, before either tail + runs, and refuses to hold a retry slot until it is durable. The tails + below start after that boundary, so each one writes its own row first + — otherwise the terminal outbox update at the end of a landing would + find nothing to mark and the durable replay source would be missing + from every scenario. + """ + persist_intent = getattr(self.pool.ledger, "persist_block_candidate_intent", None) + if not callable(persist_intent): + return + persist_intent(self.pool.block_candidate_intent(found.candidate)) + + def land_on_client_tail(self, found: LandingCandidate) -> Call: + """Queue ``found`` down the synchronous miner-connection-thread route.""" + + def run() -> bool: + self._persist_intent(found) + return self.pool._submit_synchronous_block_candidate(found.candidate) + + return self.client.submit( + run, + label=f"client-land:{found.block_hash[:4]}", + ) + + def land_on_accounting_tail( + self, + found: LandingCandidate, + *, + node_submission: _BlockCandidateNodeSubmission | None = None, + ) -> Call: + """Queue ``found`` down the accounting-actor route. + + The accounting actor's task carries the fast lane's node-offer + evidence, so the default here is an unattempted offer: the tail then + performs the submit itself, which is the shape both #133 and #126 + need. + """ + submission = node_submission or _BlockCandidateNodeSubmission(attempted=False) + + def run() -> None: + self._persist_intent(found) + # The task the accounting loop dequeues, built the way + # ``submit_next`` builds it: the disposition lease for this hash + # is already claimed, and the loop hands it back afterwards. The + # loop itself is a queue drain, so the harness supplies the task + # rather than the queue. + lease = self.pool._claim_block_candidate_disposition( + found.block_hash, + blocking=True, + ) + if lease is None: + raise HarnessError( + f"disposition for {found.block_hash} was already claimed" + ) + self.pool._run_block_accounting_task( + _BlockCandidateAccountingTask( + candidate=found.candidate, + node_submission=submission, + disposition_lease=lease, + ) + ) + + return self.accounting.submit( + run, + label=f"accounting-land:{found.block_hash[:4]}", + ) + + # -- the append side --------------------------------------------------- + + def publish_window_anchor(self, anchor_ms: int) -> None: + """Expose a newer published job window's anchor to the append side. + + A seedless build hands its declared anchor to this watermark at the + publication fence, so jobs still serving that window keep an anchor + visible to the append-side ``predates`` checks after the build's own + exposure retires. It is the cheapest way for a scenario to say "a + newer window than this candidate's is live", which is the precondition + for an append that invalidates *something* without invalidating the + candidate under test. + """ + anchor = int(anchor_ms) + current = getattr(self.pool, "_payout_published_job_window_anchor_ms", None) + self.pool._payout_published_job_window_anchor_ms = max(anchor, current or 0) + + def append_late_visible_share( + self, + *, + stamp_ms: int, + share_id: str | None = None, + accepted_at_ms: int | None = None, + ) -> Call: + """Make one share durable and let it advance the invalidation epoch. + + Queued on its own actor, because the append side is a third thread in + production (the share writer) and because the bump takes the landing + fence, which a landing may be holding. ``stamp_ms`` sets the job + issue stamp and, unless ``accepted_at_ms`` says otherwise, the + acceptance stamp too. With both stamps equal it is exactly the + quantity the append-side ``predates`` test compares against an + anchor: the row invalidates a window whose declared anchor is at or + after it, and no other. A distinct ``accepted_at_ms`` stages a + straddling row -- issued at or before an anchor, accepted after -- + which predates only anchors at or after *both* stamps, and whose + recorded bump stamp is their max. + + Returns the queued call; its value is the advanced epoch, or None when + no live anchor predated the row. + """ + stamp = int(stamp_ms) + accepted = stamp if accepted_at_ms is None else int(accepted_at_ms) + share = PendingShare( + share_id=share_id or f"appender:{stamp}", + miner_id="miner-b", + order_key="miner-b", + p2mr_program_hex="22" * 32, + share_difficulty=1, + network_difficulty=1, + template_height=self.block_height - 1, + job_id="append-job", + job_issued_at_ms=stamp, + accepted_at_ms=accepted, + ntime=1_700_000_000, + ) + + def run() -> int | None: + self.server.shares.append( + {"share_seq": len(self.server.shares) + 1, "miner_id": share.miner_id} + ) + return self.pool._record_late_visible_payout_append(share) + + label = f"append:{stamp}" if accepted == stamp else f"append:{stamp}:{accepted}" + return self.appender.submit(run, label=label) + + def live_append_epoch(self) -> int: + return int(self.pool._payout_ledger_append_invalidation_epoch) + + # -- observation ------------------------------------------------------- + + @property + def audit_root(self) -> Path: + return self._audit_root + + def live_envelope_path(self, found: LandingCandidate) -> Path: + store = self.pool._ensure_audit_artifact_store() + return store.live_envelope_path( + block_height=found.block_height, + block_hash=found.block_hash, + ) + + def envelope_written(self, found: LandingCandidate) -> bool: + """Whether this block's published evidence pointer exists on disk. + + This is #133's whole assertion. The block is paid and audited + either way; what the defect destroyed was the file. + """ + return self.live_envelope_path(found).exists() + + def pool_block(self, block_hash: str) -> PoolBlockRow | None: + return self.server.pool_blocks.get(block_hash) + + def publication_sequence(self, found: LandingCandidate) -> int | None: + row = self.pool_block(found.block_hash) + return None if row is None else row.audit_publication_sequence + + def publication_floor(self) -> int: + return max( + [ + row.audit_publication_sequence or 0 + for row in self.server.pool_blocks.values() + ], + default=0, + ) + + def outbox_state(self, found: LandingCandidate) -> str | None: + row = self.server.outbox.get(found.block_hash) + return None if row is None else row.state + + def landing_summary(self) -> dict[str, Any]: + """A compact, order-stable digest for ``assert_deterministic``. + + Deliberately not the whole world: it names the durable facts a + landing scenario asserts about, so two runs that agree here and on + the checkpoint trace agree on everything a scenario claims. + """ + return { + "pool_blocks": [ + { + "block_hash": row.block_hash, + "block_height": row.block_height, + "chain_state": row.chain_state, + "maturity_state": row.maturity_state, + "audit_publication_sequence": row.audit_publication_sequence, + } + for row in self.server.pool_blocks.values() + ], + "outbox": [ + { + "block_hash": row.block_hash, + "state": row.state, + "attempt_count": row.attempt_count, + "last_error": row.last_error, + } + for row in self.server.outbox.values() + ], + "statements": self.statement_kinds(), + "submitted_blocks": list(self.rpc.submitted), + "envelopes": sorted( + path.name + for path in (self._audit_root / "audit").rglob("*.json") + if path.is_file() + ), + } + + +# -------------------------------------------------------------------------- +# Fixed audit answers +# -------------------------------------------------------------------------- + + +def _canonicalize_bundle(bundle: dict[str, Any]) -> bytes: + return json.dumps(bundle, sort_keys=True, separators=(",", ":")).encode("utf-8") + + +def _verified_block_bundle(coinbase_tx_hex: str = "c0ffee") -> dict[str, object]: + return { + "found_block": {"coinbase_value_sats": 50_00000000}, + "ledger_window_attestation": {"signature": {"public_key_hex": "aa" * 32}}, + "payout_policy_manifest": {"accounts": []}, + "signed_coinbase_manifest": { + "manifest": { + "coinbase_tx_hex": coinbase_tx_hex, + "payout_count": 1, + } + }, + } + + +def _verified_audit_report( + coinbase_tx_hex: str = "c0ffee", + *, + block_height: int = DEFAULT_HEIGHT, +) -> dict[str, object]: + return { + "schema": "qbit.prism.audit-verification-report.v1", + "block_height": int(block_height), + "coinbase_value_sats": 50_00000000, + "reward_manifest_sha256_hex": "44" * 32, + "payout_policy_manifest_sha256_hex": "55" * 32, + "prism_audit_commitment_leaf_hex": "66" * 32, + "audit_commitment_root_hex": "77" * 32, + "coinbase_txid": "11" * 32, + "coinbase_wtxid": "88" * 32, + "coinbase_manifest_sha256_hex": "22" * 32, + "audit_bundle_sha256_hex": "33" * 32, + "coinbase_tx_hex": coinbase_tx_hex, + "min_output_sats": 1, + "onchain_output_count": 0, + "accrued_account_count": 0, + } + + +__all__ = [ + "DEFAULT_HEIGHT", + "PARENT_HASH", + "LandingCandidate", + "LandingHarness", + "LandingRpc", +] diff --git a/tests/prism_postgres_a1_gate.py b/tests/prism_postgres_a1_gate.py new file mode 100644 index 000000000..d40c7caa6 --- /dev/null +++ b/tests/prism_postgres_a1_gate.py @@ -0,0 +1,2306 @@ +"""Non-discovered PostgreSQL/A1 integration gate. + +This helper is invoked by ``test/test-prism-postgres-ledger.sh``. It lives +outside unittest discovery because every scenario requires an explicitly +provisioned PostgreSQL target. +""" + +from __future__ import annotations + +import atexit +import hashlib +import json +import os +import re +import shlex +import signal +import subprocess +import tempfile +import threading +import time +from datetime import datetime, timezone +from typing import Any + +from lab.prism import public_api +from lab.prism.share_ledger import PsqlShareLedger, SingleWriterShareLedger + + +PSQL_TIMEOUT_SECONDS = 30.0 +PSQL_OUTPUT_LIMIT_BYTES = 1 << 20 +SCHEMA_PATTERN = re.compile(r"qbit_a1_[a-z0-9_]+") +BASE_PSQL_COMMAND = os.environ.get("PRISM_PSQL_COMMAND", "") +if not BASE_PSQL_COMMAND: + raise SystemExit("PRISM_PSQL_COMMAND is required") +BASE_PSQL_ARGV = shlex.split(BASE_PSQL_COMMAND) +if not BASE_PSQL_ARGV: + raise SystemExit("PRISM_PSQL_COMMAND is empty") + +RUN_TOKEN = os.urandom(16).hex() +RUN_MARKER = f"qbit-a1-test:{RUN_TOKEN}" +OWNED_SCHEMAS: list[tuple[str, str]] = [] +ACTIVE_CHILDREN: set[subprocess.Popen[str]] = set() +ACTIVE_CHILDREN_LOCK = threading.Lock() + + +def fake_bundle_bytes(payload: dict[str, object]) -> bytes: + return json.dumps(payload, separators=(",", ":")).encode() + + +PERSIST_BUNDLE: dict[str, object] = { + "signed_coinbase_manifest": {"manifest": {"payout_count": 1}}, + "payout_policy_manifest": { + "accounts": [ + { + "recipient_id": "a1-parity-miner", + "order_key": "a1-parity-order", + "p2mr_program_hex": "42" * 32, + "gross_amount_sats": 1000, + "prior_balance_sats": 0, + "candidate_balance_sats": 1000, + "onchain_amount_sats": 0, + "carry_forward_balance_sats": 1000, + "action": "accrued", + } + ] + }, +} +PERSIST_BUNDLE_SHA256 = hashlib.sha256( + fake_bundle_bytes(PERSIST_BUNDLE) +).hexdigest() +PERSIST_REPORT: dict[str, object] = { + "coinbase_txid": "20" * 32, + "coinbase_manifest_sha256_hex": "30" * 32, + "audit_bundle_sha256_hex": PERSIST_BUNDLE_SHA256, + "coinbase_tx_hex": "00", +} + + +class GateFailure(RuntimeError): + pass + + +def assert_equal(actual: object, expected: object, message: str) -> None: + if actual != expected: + raise GateFailure(f"{message}: expected {expected!r}, got {actual!r}") + + +def _terminate_and_reap(process: subprocess.Popen[str]) -> None: + if process.poll() is not None: + process.wait() + return + try: + os.killpg(process.pid, signal.SIGTERM) + except ProcessLookupError: + pass + try: + process.wait(timeout=2.0) + return + except subprocess.TimeoutExpired: + pass + try: + os.killpg(process.pid, signal.SIGKILL) + except ProcessLookupError: + pass + process.wait(timeout=2.0) + + +def cleanup_active_children() -> None: + with ACTIVE_CHILDREN_LOCK: + children = list(ACTIVE_CHILDREN) + for process in children: + _terminate_and_reap(process) + for stream in (process.stdin, process.stdout, process.stderr): + if stream is not None and not stream.closed: + stream.close() + with ACTIVE_CHILDREN_LOCK: + ACTIVE_CHILDREN.discard(process) + + +def run_psql(sql: str, *, schema: str | None = None) -> str: + if schema is not None and SCHEMA_PATTERN.fullmatch(schema) is None: + raise GateFailure(f"invalid scoped schema: {schema!r}") + scoped_sql = sql + if schema is not None: + scoped_sql = ( + "SET statement_timeout = '20s';\n" + "SET lock_timeout = '10s';\n" + f'SET search_path TO "{schema}", pg_catalog;\n' + + sql + ) + command = [ + *BASE_PSQL_ARGV, + "--no-psqlrc", + "--set", + "ON_ERROR_STOP=1", + "--tuples-only", + "--no-align", + "--quiet", + ] + with ( + tempfile.TemporaryFile(mode="w+t", encoding="utf-8") as stdout_file, + tempfile.TemporaryFile(mode="w+t", encoding="utf-8") as stderr_file, + ): + process = subprocess.Popen( + command, + stdin=subprocess.PIPE, + stdout=stdout_file, + stderr=stderr_file, + text=True, + start_new_session=True, + ) + with ACTIVE_CHILDREN_LOCK: + ACTIVE_CHILDREN.add(process) + try: + process.communicate( + scoped_sql, + timeout=PSQL_TIMEOUT_SECONDS, + ) + except BaseException: + _terminate_and_reap(process) + raise + finally: + if process.poll() is None: + _terminate_and_reap(process) + if process.stdin is not None and not process.stdin.closed: + process.stdin.close() + with ACTIVE_CHILDREN_LOCK: + ACTIVE_CHILDREN.discard(process) + stdout_file.seek(0) + stderr_file.seek(0) + stdout = stdout_file.read(PSQL_OUTPUT_LIMIT_BYTES + 1) + stderr = stderr_file.read(PSQL_OUTPUT_LIMIT_BYTES + 1) + if ( + len(stdout.encode("utf-8")) > PSQL_OUTPUT_LIMIT_BYTES + or len(stderr.encode("utf-8")) > PSQL_OUTPUT_LIMIT_BYTES + ): + raise GateFailure("psql output exceeded 1 MiB limit") + if process.returncode != 0: + raise GateFailure( + f"psql failed with exit {process.returncode}: {stderr.strip()}" + ) + return stdout + + +def run_json(sql: str, *, schema: str | None = None) -> Any: + output = run_psql(sql, schema=schema).strip() + if not output: + raise GateFailure("psql query returned no JSON") + return json.loads(output.splitlines()[-1]) + + +def _schema_registered(schema: str) -> bool: + return any(name == schema for name, _marker in OWNED_SCHEMAS) + + +def create_owned_schema(label: str) -> str: + if re.fullmatch(r"[a-z0-9_]+", label) is None: + raise GateFailure(f"invalid schema label: {label!r}") + schema = f"qbit_a1_{label}_{os.getpid()}_{os.urandom(5).hex()}" + if SCHEMA_PATTERN.fullmatch(schema) is None or len(schema) > 63: + raise GateFailure(f"invalid generated schema: {schema!r}") + # Register authority before the transaction starts. If this call is + # interrupted, run_psql kills/reaps its child before atexit examines the + # durable marker. + OWNED_SCHEMAS.append((schema, RUN_MARKER)) + run_psql( + f""" +BEGIN; +CREATE SCHEMA "{schema}"; +COMMENT ON SCHEMA "{schema}" IS '{RUN_MARKER}'; +COMMIT; +""" + ) + return schema + + +def cleanup_owned_schemas() -> None: + while OWNED_SCHEMAS: + schema, expected_marker = OWNED_SCHEMAS[-1] + if SCHEMA_PATTERN.fullmatch(schema) is None: + raise GateFailure(f"refusing to drop invalid schema: {schema!r}") + marker = run_json( + f""" +SELECT json_build_object( + 'marker', ( + SELECT obj_description(oid, 'pg_namespace') + FROM pg_namespace + WHERE nspname = '{schema}' + ) +); +""" + )["marker"] + if marker is None: + OWNED_SCHEMAS.pop() + continue + if marker != expected_marker: + raise GateFailure( + f"refusing to drop schema with wrong marker: {schema!r}" + ) + run_psql(f'DROP SCHEMA "{schema}" CASCADE;') + OWNED_SCHEMAS.pop() + + +def _signal_exit(signum: int, _frame: object) -> None: + # Raising first lets the active run_psql frame kill and reap its exact + # process group. Registered atexit cleanup runs only after that unwind. + raise SystemExit(128 + signum) + + +signal.signal(signal.SIGTERM, _signal_exit) +signal.signal(signal.SIGINT, _signal_exit) +atexit.register(cleanup_owned_schemas) +atexit.register(cleanup_active_children) + + +class ScopedPsqlLedger(PsqlShareLedger): + def __init__(self, *, test_schema: str, **kwargs: object) -> None: + if not _schema_registered(test_schema): + raise GateFailure("ledger requires a registered owned schema") + self._test_schema = test_schema + kwargs["psql_command"] = BASE_PSQL_COMMAND + kwargs["native_client_mode"] = "psql" + super().__init__(**kwargs) # type: ignore[arg-type] + + def _run_sql(self, sql: str) -> str: + if not _schema_registered(self._test_schema): + raise GateFailure("ledger schema authority was revoked") + return run_psql(sql, schema=self._test_schema) + + +def public_sentinel() -> dict[str, object]: + return run_json( + """ +SELECT json_build_object( + 'pool_oid', 'public.qbit_pool_blocks'::regclass::oid, + 'pool_rows', ( + SELECT md5(COALESCE( + string_agg(row_to_json(row_value)::text, ',' ORDER BY block_hash), + '' + )) + FROM public.qbit_pool_blocks row_value + ), + 'sequence_oid', + 'public.qbit_audit_publication_sequence_seq'::regclass::oid, + 'sequence_definition', ( + SELECT json_build_object( + 'type_oid', sequence.seqtypid, + 'start', sequence.seqstart, + 'increment', sequence.seqincrement, + 'max', sequence.seqmax, + 'min', sequence.seqmin, + 'cache', sequence.seqcache, + 'cycle', sequence.seqcycle, + 'persistence', relation.relpersistence, + 'owner', relation.relowner + ) + FROM pg_catalog.pg_sequence sequence + JOIN pg_catalog.pg_class relation + ON relation.oid = sequence.seqrelid + WHERE sequence.seqrelid = + 'public.qbit_audit_publication_sequence_seq'::regclass + ), + 'sequence_state', ( + SELECT row_to_json(state) + FROM ( + SELECT last_value, is_called + FROM public.qbit_audit_publication_sequence_seq + ) state + ), + 'index_oid', + 'public.qbit_pool_blocks_audit_publication_sequence_idx'::regclass::oid, + 'index_filenode', pg_relation_filenode( + 'public.qbit_pool_blocks_audit_publication_sequence_idx'::regclass + ), + 'index_definition', pg_get_indexdef( + 'public.qbit_pool_blocks_audit_publication_sequence_idx'::regclass + ), + 'index_catalog', ( + SELECT json_build_object( + 'kind', relation.relkind, + 'persistence', relation.relpersistence, + 'owner', relation.relowner, + 'same_owner_as_table', relation.relowner = ( + SELECT pool_blocks.relowner + FROM pg_catalog.pg_class pool_blocks + JOIN pg_catalog.pg_namespace pool_namespace + ON pool_namespace.oid = pool_blocks.relnamespace + WHERE pool_namespace.nspname = 'public' + AND pool_blocks.relname = 'qbit_pool_blocks' + ), + 'unique', index_definition.indisunique, + 'valid', index_definition.indisvalid, + 'ready', index_definition.indisready, + 'live', index_definition.indislive, + 'immediate', index_definition.indimmediate, + 'primary', index_definition.indisprimary, + 'exclusion', index_definition.indisexclusion, + 'clustered', index_definition.indisclustered, + 'replica_identity', index_definition.indisreplident, + 'nulls_not_distinct', index_definition.indnullsnotdistinct, + 'key_count', index_definition.indnkeyatts, + 'attribute_count', index_definition.indnatts, + 'keys', index_definition.indkey::text, + 'classes', index_definition.indclass::text, + 'collations', index_definition.indcollation::text, + 'options', index_definition.indoption::text, + 'expressions', index_definition.indexprs::text, + 'predicate', index_definition.indpred::text + ) + FROM pg_catalog.pg_index index_definition + JOIN pg_catalog.pg_class relation + ON relation.oid = index_definition.indexrelid + WHERE index_definition.indexrelid = + 'public.qbit_pool_blocks_audit_publication_sequence_idx'::regclass + ), + 'constraint', ( + SELECT json_build_object( + 'oid', oid, + 'conbin', conbin::text, + 'definition', pg_get_constraintdef(oid, true), + 'validated', convalidated, + 'name', conname, + 'type', contype, + 'keys', conkey::text, + 'local', conislocal, + 'inherited_count', coninhcount, + 'no_inherit', connoinherit, + 'deferrable', condeferrable, + 'deferred', condeferred + ) + FROM pg_constraint + WHERE conrelid = 'public.qbit_pool_blocks'::regclass + AND conname = + 'qbit_pool_blocks_audit_publication_sequence_check' + ) +); +""" + ) + + +def marker_schema_count() -> int: + value = run_json( + f""" +SELECT json_build_object( + 'count', count(*) +) +FROM pg_namespace +WHERE obj_description(oid, 'pg_namespace') = '{RUN_MARKER}'; +""" + )["count"] + return int(value) + + +def allocator_state(ledger: ScopedPsqlLedger) -> dict[str, object]: + return ledger._run_json( + """ +SELECT json_build_object( + 'last_value', last_value, + 'is_called', is_called +) +FROM qbit_audit_publication_sequence_seq; +""" + ) + + +def seed_prepared_direct( + postgres: ScopedPsqlLedger, + memory: SingleWriterShareLedger, + *, + block_hash: str, + block_height: int, +) -> None: + postgres._run_sql( + f""" +INSERT INTO qbit_pool_blocks ( + block_hash, + block_height, + parent_hash, + coinbase_txid, + payout_manifest_sha256, + chain_state, + maturity_state +) VALUES ( + '{block_hash}', + {block_height}, + '{'10' * 32}', + '{'20' * 32}', + '{'30' * 32}', + 'prepared', + 'immature' +) +ON CONFLICT (block_hash) DO NOTHING; +""" + ) + memory.persist_accepted_block( + block_hash=block_hash, + block_height=block_height, + parent_hash="10" * 32, + final_bundle={}, + audit_report={}, + ) + + +def normalized_response(payload: dict[str, int | str]) -> dict[str, int | str]: + return {**payload, "backend": "ledger"} + + +def assert_states_equal( + postgres: ScopedPsqlLedger, + memory: SingleWriterShareLedger, + block_hashes: list[str], + message: str, +) -> None: + for block_hash in block_hashes: + assert_equal( + postgres.pool_block_state(block_hash=block_hash), + memory.pool_block_state(block_hash=block_hash), + f"{message} state {block_hash}", + ) + assert_equal( + postgres.audit_publication_sequence_floor(), + memory.audit_publication_sequence_floor(), + f"{message} durable floor", + ) + + +def call_both( + postgres: ScopedPsqlLedger, + memory: SingleWriterShareLedger, + method: str, + *, + block_hash: str, + active_tip_height: int, + known_hashes: list[str], + message: str, +) -> dict[str, int | str]: + postgres_result = getattr(postgres, method)( + block_hash=block_hash, + active_tip_height=active_tip_height, + ) + memory_result = getattr(memory, method)( + block_hash=block_hash, + active_tip_height=active_tip_height, + ) + assert_equal( + normalized_response(postgres_result), + normalized_response(memory_result), + f"{message} response", + ) + assert_states_equal(postgres, memory, known_hashes, message) + return postgres_result + + +def test_exact_transition_parity() -> None: + schema = create_owned_schema("parity") + postgres = ScopedPsqlLedger( + test_schema=schema, + writer_id="a1-parity", + writer_epoch=1, + initialize_schema=True, + audit_bundle_canonicalizer=fake_bundle_bytes, + ) + memory = SingleWriterShareLedger() + try: + assert_equal( + postgres._run_json( + "SELECT json_build_object('schema', current_schema());" + )["schema"], + schema, + "parity current schema", + ) + assert_equal(allocator_state(postgres), {"last_value": 1, "is_called": False}, "parity fresh allocator") + block_a = "a1" * 32 + block_b = "b1" * 32 + block_c = "c1" * 32 + known = [block_a] + seed_prepared_direct(postgres, memory, block_hash=block_a, block_height=10) + seed_prepared_direct(postgres, memory, block_hash=block_a, block_height=10) + assert_states_equal(postgres, memory, known, "duplicate direct prepared seed") + + api_hash = "d1" * 32 + expected_inline_body_bytes = fake_bundle_bytes(PERSIST_BUNDLE) + expected_inline_body_byte_len = len(expected_inline_body_bytes) + postgres_first_persist = postgres.persist_accepted_block( + block_hash=api_hash, + block_height=30, + parent_hash="10" * 32, + final_bundle=PERSIST_BUNDLE, + audit_report=PERSIST_REPORT, + ) + memory_first_persist = memory.persist_accepted_block( + block_hash=api_hash, + block_height=30, + parent_hash="10" * 32, + final_bundle=PERSIST_BUNDLE, + audit_report=PERSIST_REPORT, + ) + assert_equal( + postgres_first_persist, + { + "backend": "postgres-psql", + "share_count": 0, + "block_count": 1, + "bundle_count": 1, + "payout_entry_count": 1, + "carry_forward_count": 1, + "onchain_output_count": 1, + "audit_bundle_sha256": PERSIST_BUNDLE_SHA256, + "body_uri": "", + "audit_body_byte_len": expected_inline_body_byte_len, + }, + "PostgreSQL first persist API response", + ) + persisted_inline_body = postgres._run_json( + f""" +SELECT json_build_object( + 'audit_bundle_present', audit_bundle IS NOT NULL, + 'body_uri_is_null', body_uri IS NULL, + 'audit_body_byte_len', audit_body_byte_len, + 'audit_bundle_sha256', audit_bundle_sha256, + 'audit_bundle', audit_bundle +) +FROM qbit_pool_audit_bundles +WHERE block_hash = '{api_hash}'; +""" + ) + assert_equal( + persisted_inline_body, + { + "audit_bundle_present": True, + "body_uri_is_null": True, + "audit_body_byte_len": expected_inline_body_byte_len, + "audit_bundle_sha256": hashlib.sha256( + expected_inline_body_bytes + ).hexdigest(), + "audit_bundle": PERSIST_BUNDLE, + }, + "PostgreSQL inline audit body row", + ) + assert_equal( + memory_first_persist, + { + "backend": "memory", + "share_count": 0, + "block_count": 0, + "payout_entry_count": 0, + "carry_forward_count": 0, + }, + "memory first persist API response", + ) + postgres_duplicate_persist = postgres.persist_accepted_block( + block_hash=api_hash, + block_height=30, + parent_hash="10" * 32, + final_bundle=PERSIST_BUNDLE, + audit_report=PERSIST_REPORT, + ) + memory_duplicate_persist = memory.persist_accepted_block( + block_hash=api_hash, + block_height=30, + parent_hash="10" * 32, + final_bundle=PERSIST_BUNDLE, + audit_report=PERSIST_REPORT, + ) + assert_equal( + postgres_duplicate_persist, + postgres_first_persist, + "PostgreSQL duplicate persist exact response", + ) + assert_equal( + memory_duplicate_persist, + memory_first_persist, + "memory duplicate persist exact response", + ) + known.append(api_hash) + assert_states_equal(postgres, memory, known, "public persist API parity") + + # The confirm disposition matrix. Every value is asserted on both + # backends through call_both, which compares the full responses, so + # the memory ledger stays a parity implementation of the SQL function: + # 1 fresh flip out of 'prepared', allocates a publication ordinal + # 2 idempotent replay of an already-confirmed row, allocates none + # 0 no row, or a live row this confirmation does not match + # -1 terminally disposed before the confirmation landed + # 1 and 2 are the split issue #61 needs: only a fresh confirmation has + # payout state to publish. + before = allocator_state(postgres) + unmatched = call_both(postgres, memory, "confirm_accepted_block", block_hash=block_a, active_tip_height=9, known_hashes=known, message="wrong-height confirmation") + assert_equal(unmatched["confirmed_count"], 0, "wrong-height confirmation disposition") + assert_equal(allocator_state(postgres), before, "wrong-height confirmation allocator immobility") + confirmed = call_both(postgres, memory, "confirm_accepted_block", block_hash=block_a, active_tip_height=10, known_hashes=known, message="first confirmation") + assert_equal(confirmed["confirmed_count"], 1, "first confirmation disposition") + assert_equal(confirmed["audit_publication_sequence"], 1, "first event ordinal") + before = allocator_state(postgres) + replay = call_both(postgres, memory, "confirm_accepted_block", block_hash=block_a, active_tip_height=10, known_hashes=known, message="exact confirmation replay") + assert_equal(replay["confirmed_count"], 2, "exact confirmation replay disposition") + assert_equal(replay["audit_publication_sequence"], 1, "replay ordinal") + for replay_round in range(2, 4): + replay = call_both( + postgres, + memory, + "confirm_accepted_block", + block_hash=block_a, + active_tip_height=10, + known_hashes=known, + message=f"exact confirmation replay round {replay_round}", + ) + assert_equal( + replay["confirmed_count"], + 2, + f"replay round {replay_round} disposition", + ) + assert_equal( + replay["audit_publication_sequence"], + 1, + f"replay round {replay_round} ordinal", + ) + assert_equal(allocator_state(postgres), before, "exact replay allocator immobility") + before = allocator_state(postgres) + call_both(postgres, memory, "reactivate_pool_block", block_hash=block_a, active_tip_height=10, known_hashes=known, message="confirmed count-zero reactivation") + assert_equal(allocator_state(postgres), before, "count-zero reactivation allocator immobility") + call_both(postgres, memory, "mark_pool_block_inactive", block_hash=block_a, active_tip_height=10, known_hashes=known, message="inactive transition") + before = allocator_state(postgres) + inactive_confirmation = call_both( + postgres, + memory, + "confirm_accepted_block", + block_hash=block_a, + active_tip_height=10, + known_hashes=known, + message="inactive confirmation", + ) + assert_equal( + inactive_confirmation["confirmed_count"], + -1, + "inactive confirmation disposition", + ) + assert_equal( + allocator_state(postgres), + before, + "inactive confirmation allocator immobility", + ) + call_both(postgres, memory, "reactivate_pool_block", block_hash=block_a, active_tip_height=9, known_hashes=known, message="wrong-height reactivation") + assert_equal(allocator_state(postgres), before, "wrong-height reactivation allocator immobility") + postgres.release_writer_lease() + postgres.close() + postgres = ScopedPsqlLedger( + test_schema=schema, + writer_id="a1-parity-reactivation-restart", + writer_epoch=1, + audit_bundle_canonicalizer=fake_bundle_bytes, + ) + assert_states_equal( + postgres, + memory, + known, + "inactive restart serializer", + ) + assert_equal( + allocator_state(postgres), + before, + "inactive restart allocator preservation", + ) + reactivated = call_both(postgres, memory, "reactivate_pool_block", block_hash=block_a, active_tip_height=10, known_hashes=known, message="reactivation") + assert_equal(reactivated["audit_publication_sequence"], 1, "reactivation preserves published ordinal") + assert_equal(allocator_state(postgres), before, "reactivation allocator immobility") + before = allocator_state(postgres) + reactivated_confirmation = call_both( + postgres, + memory, + "confirm_accepted_block", + block_hash=block_a, + active_tip_height=10, + known_hashes=known, + message="reactivated confirmation replay", + ) + # Reactivation restores 'confirmed' without returning the row to + # 'prepared', so a confirmation after it is an idempotent replay too. + assert_equal( + reactivated_confirmation["confirmed_count"], + 2, + "reactivated confirmation replay disposition", + ) + assert_equal( + reactivated_confirmation["audit_publication_sequence"], + 1, + "reactivated confirmation replay ordinal", + ) + call_both(postgres, memory, "reactivate_pool_block", block_hash=block_a, active_tip_height=10, known_hashes=known, message="reactivation replay") + assert_equal(allocator_state(postgres), before, "reactivation replay allocator immobility") + call_both(postgres, memory, "mark_pool_block_inactive", block_hash=block_a, active_tip_height=10, known_hashes=known, message="second inactive transition") + call_both(postgres, memory, "reverse_immature_block", block_hash=block_a, active_tip_height=10, known_hashes=known, message="terminal reversal") + before = allocator_state(postgres) + postgres.release_writer_lease() + postgres.close() + postgres = ScopedPsqlLedger( + test_schema=schema, + writer_id="a1-parity-reversed-restart", + writer_epoch=1, + audit_bundle_canonicalizer=fake_bundle_bytes, + ) + assert_states_equal(postgres, memory, known, "reversed restart serializer") + assert_equal( + allocator_state(postgres), + before, + "reversed restart allocator preservation", + ) + reversed_confirmation = call_both(postgres, memory, "confirm_accepted_block", block_hash=block_a, active_tip_height=10, known_hashes=known, message="reversed confirmation") + assert_equal( + reversed_confirmation["confirmed_count"], + -1, + "reversed confirmation disposition", + ) + call_both(postgres, memory, "reactivate_pool_block", block_hash=block_a, active_tip_height=10, known_hashes=known, message="reversed reactivation") + assert_equal(allocator_state(postgres), before, "terminal transition allocator immobility") + + known.append(block_b) + seed_prepared_direct(postgres, memory, block_hash=block_b, block_height=1) + lower = call_both(postgres, memory, "confirm_accepted_block", block_hash=block_b, active_tip_height=1, known_hashes=known, message="later lower-height confirmation") + assert_equal(lower["audit_publication_sequence"], 2, "event order dominates block height") + + known.append(block_c) + seed_prepared_direct(postgres, memory, block_hash=block_c, block_height=20) + before = allocator_state(postgres) + call_both(postgres, memory, "reject_prepared_block", block_hash=block_c, active_tip_height=20, known_hashes=known, message="prepared rejection") + call_both(postgres, memory, "reject_prepared_block", block_hash=block_c, active_tip_height=20, known_hashes=known, message="rejection replay") + assert_equal(allocator_state(postgres), before, "rejection allocator immobility") + block_d = "e1" * 32 + known.append(block_d) + seed_prepared_direct(postgres, memory, block_hash=block_d, block_height=0) + final_confirmation = call_both( + postgres, + memory, + "confirm_accepted_block", + block_hash=block_d, + active_tip_height=0, + known_hashes=known, + message="post-replay fresh confirmation", + ) + assert_equal( + final_confirmation["confirmed_count"], + 1, + "post-replay fresh confirmation disposition", + ) + assert_equal( + final_confirmation["audit_publication_sequence"], + 3, + "post-replay exact next ordinal", + ) + assert_equal(postgres.audit_publication_sequence_floor(), 3, "parity final durable floor") + finally: + postgres.release_writer_lease() + postgres.close() + + +LEGACY_POOL_BLOCKS_SQL = """ +CREATE TABLE qbit_pool_blocks ( + block_hash text PRIMARY KEY, + block_height bigint NOT NULL CHECK (block_height >= 0), + parent_hash text NOT NULL, + coinbase_txid text NOT NULL, + payout_manifest_sha256 text NOT NULL, + found_at timestamptz NOT NULL DEFAULT clock_timestamp(), + chain_state text NOT NULL DEFAULT 'prepared' + CHECK (chain_state IN ('prepared', 'confirmed', 'inactive', 'rejected', 'reversed')), + maturity_state text NOT NULL DEFAULT 'immature' + CHECK (maturity_state IN ('immature', 'mature', 'reversed')), + matured_at timestamptz, + disconnected_at timestamptz, + CHECK ((maturity_state = 'mature') = (matured_at IS NOT NULL)), + CHECK ((maturity_state = 'reversed') = (disconnected_at IS NOT NULL)) +); +""" + + +def assert_empty_serializer_case(*, legacy: bool) -> None: + label = "empty_legacy" if legacy else "empty_fresh" + schema = create_owned_schema(label) + if legacy: + run_psql(LEGACY_POOL_BLOCKS_SQL, schema=schema) + first = ScopedPsqlLedger( + test_schema=schema, + writer_id=f"a1-{label}-first", + writer_epoch=1, + initialize_schema=True, + ) + try: + assert_equal(first.audit_publication_sequence_floor(), 0, f"{label} floor") + assert_equal(allocator_state(first), {"last_value": 1, "is_called": False}, f"{label} allocator") + finally: + first.release_writer_lease() + first.close() + second = ScopedPsqlLedger( + test_schema=schema, + writer_id=f"a1-{label}-second", + writer_epoch=1, + initialize_schema=True, + ) + try: + assert_equal(second.audit_publication_sequence_floor(), 0, f"{label} rerun floor") + assert_equal(allocator_state(second), {"last_value": 1, "is_called": False}, f"{label} rerun allocator") + block_hash = hashlib.sha256(label.encode()).hexdigest() + memory = SingleWriterShareLedger() + before = allocator_state(second) + for method, operation in ( + ("confirm_accepted_block", "confirmation"), + ("reactivate_pool_block", "reactivation"), + ("mark_pool_block_inactive", "inactive transition"), + ("reverse_immature_block", "reversal"), + ("reject_prepared_block", "rejection"), + ): + call_both( + second, + memory, + method, + block_hash=block_hash, + active_tip_height=1, + known_hashes=[block_hash], + message=f"{label} missing {operation}", + ) + assert_equal( + second.pool_block_state(block_hash=block_hash), + None, + f"{label} missing state serializer", + ) + assert_equal( + allocator_state(second), + before, + f"{label} missing-row allocator immobility", + ) + seed_prepared_direct(second, memory, block_hash=block_hash, block_height=1) + seed_prepared_direct(second, memory, block_hash=block_hash, block_height=1) + assert_states_equal( + second, + memory, + [block_hash], + f"{label} nullable prepared serializer", + ) + assert_equal( + second.pool_block_state(block_hash=block_hash)[ + "audit_publication_sequence" + ], + None, + f"{label} explicit prepared nullable ordinal", + ) + before = allocator_state(second) + call_both( + second, + memory, + "confirm_accepted_block", + block_hash=block_hash, + active_tip_height=0, + known_hashes=[block_hash], + message=f"{label} wrong-height confirmation", + ) + call_both( + second, + memory, + "reactivate_pool_block", + block_hash=block_hash, + active_tip_height=1, + known_hashes=[block_hash], + message=f"{label} prepared count-zero reactivation", + ) + assert_equal( + allocator_state(second), + before, + f"{label} nullable zero-op allocator immobility", + ) + result = call_both( + second, + memory, + "confirm_accepted_block", + block_hash=block_hash, + active_tip_height=1, + known_hashes=[block_hash], + message=f"{label} first confirmation", + ) + assert_equal(result["audit_publication_sequence"], 1, f"{label} first ordinal") + assert_equal(second.audit_publication_sequence_floor(), 1, f"{label} confirmed floor") + finally: + second.release_writer_lease() + second.close() + + +def transition_binding_state_sql( + *, + target_schema: str, + decoy_schema: str, +) -> str: + return f""" +SELECT json_build_object( + 'target_sequence', ( + SELECT json_build_object('last_value', last_value, 'is_called', is_called) + FROM "{target_schema}".qbit_audit_publication_sequence_seq + ), + 'decoy_sequence', ( + SELECT json_build_object('last_value', last_value, 'is_called', is_called) + FROM {decoy_schema}.qbit_audit_publication_sequence_seq + ), + 'target_rows', ( + SELECT json_object_agg( + block_hash, + json_build_object( + 'chain_state', chain_state, + 'audit_publication_sequence', audit_publication_sequence + ) + ) + FROM "{target_schema}".qbit_pool_blocks + WHERE block_hash IN ('{'71' * 32}', '{'72' * 32}') + ), + 'decoy_rows', ( + SELECT json_object_agg( + block_hash, + json_build_object( + 'chain_state', chain_state, + 'audit_publication_sequence', audit_publication_sequence + ) + ) + FROM {decoy_schema}.qbit_pool_blocks + ), + 'target_lease_changed', ( + SELECT before.value <> + row_to_json(current_lease)::text + FROM a1_target_lease_before before, + "{target_schema}".qbit_ledger_writer_lease current_lease + ), + 'target_lease', ( + SELECT json_build_object( + 'writer_id', writer_id, + 'writer_epoch', writer_epoch, + 'writer_session_token', writer_session_token, + 'active', lease_expires_at > clock_timestamp(), + 'expiry_after_update', lease_expires_at > updated_at + ) + FROM "{target_schema}".qbit_ledger_writer_lease + ), + 'decoy_lease_unchanged', ( + SELECT before.value = row_to_json(current_lease)::text + FROM a1_decoy_lease_before before, + {decoy_schema}.qbit_ledger_writer_lease current_lease + ), + 'decoy_lease', ( + SELECT json_build_object( + 'writer_id', writer_id, + 'writer_epoch', writer_epoch, + 'writer_session_token', writer_session_token + ) + FROM {decoy_schema}.qbit_ledger_writer_lease + ), + 'proconfig', ( + SELECT json_object_agg(procedure.proname, procedure.proconfig[1]) + FROM pg_catalog.pg_proc procedure + JOIN pg_catalog.pg_namespace namespace + ON namespace.oid = procedure.pronamespace + WHERE procedure.oid IN ( + pg_catalog.to_regprocedure( + '"{target_schema}".qbit_confirm_pool_block(' + 'text,bigint,text,bigint,text,interval)' + ), + pg_catalog.to_regprocedure( + '"{target_schema}".qbit_reactivate_pool_block(' + 'text,bigint,text,bigint,text,interval)' + ) + ) + ) +); +""" + + +def seed_transition_binding_target(ledger: ScopedPsqlLedger) -> None: + ledger._run_sql( + f""" +INSERT INTO qbit_pool_blocks ( + block_hash, + audit_publication_sequence, + block_height, + parent_hash, + coinbase_txid, + payout_manifest_sha256, + chain_state, + maturity_state +) VALUES + ('{'71' * 32}', NULL, 10, '{'10' * 32}', '{'20' * 32}', + '{'30' * 32}', 'prepared', 'immature'), + ('{'72' * 32}', NULL, 20, '{'10' * 32}', '{'20' * 32}', + '{'30' * 32}', 'prepared', 'immature'); +""" + ) + confirmed = ledger.confirm_accepted_block( + block_hash="72" * 32, + active_tip_height=20, + ) + assert_equal( + confirmed["audit_publication_sequence"], + 1, + "binding seed confirmation ordinal", + ) + assert_equal( + ledger.mark_pool_block_inactive( + block_hash="72" * 32, + active_tip_height=20, + )["inactive_count"], + 1, + "binding seed inactive transition", + ) + + +def decoy_objects_sql(*, temporary: bool) -> str: + temporary_keyword = "TEMPORARY " if temporary else "" + return f""" +CREATE {temporary_keyword}TABLE qbit_ledger_writer_lease ( + singleton boolean PRIMARY KEY, + writer_id text NOT NULL, + writer_epoch bigint NOT NULL, + writer_session_token text NOT NULL, + lease_expires_at timestamptz NOT NULL, + updated_at timestamptz NOT NULL +); +CREATE {temporary_keyword}TABLE qbit_pool_blocks ( + block_hash text PRIMARY KEY, + audit_publication_sequence bigint, + block_height bigint NOT NULL, + chain_state text NOT NULL, + maturity_state text NOT NULL +); +CREATE {temporary_keyword}SEQUENCE qbit_audit_publication_sequence_seq + AS bigint START WITH 700; +INSERT INTO qbit_ledger_writer_lease VALUES ( + true, + 'a1-binding', + 1, + 'a1-binding-token', + '2040-01-01T00:00:00Z', + '2000-01-01T00:00:00Z' +); +INSERT INTO qbit_pool_blocks VALUES + ('{'71' * 32}', NULL, 10, 'prepared', 'immature'), + ('{'72' * 32}', 700, 20, 'inactive', 'immature'); +""" + + +def assert_transition_binding_case(*, temporary: bool) -> None: + label = "binding_temp" if temporary else "binding_ordinary" + target_schema = create_owned_schema(label) + target = ScopedPsqlLedger( + test_schema=target_schema, + writer_id="a1-binding", + writer_epoch=1, + writer_session_token="a1-binding-token", + initialize_schema=True, + ) + try: + seed_transition_binding_target(target) + if temporary: + decoy_schema = "pg_temp" + caller_setup = decoy_objects_sql(temporary=True) + caller_path = f'pg_temp, "{target_schema}", pg_catalog' + else: + ordinary_decoy = create_owned_schema("binding_decoy") + run_psql(decoy_objects_sql(temporary=False), schema=ordinary_decoy) + decoy_schema = f'"{ordinary_decoy}"' + caller_setup = "" + caller_path = f'"{ordinary_decoy}", "{target_schema}", pg_catalog' + expected_config = f"search_path=pg_catalog, {target_schema}, pg_temp" + result = run_json( + f""" +SET statement_timeout = '20s'; +{caller_setup} +SET search_path TO {caller_path}; +CREATE TEMPORARY TABLE a1_target_lease_before (value text NOT NULL); +INSERT INTO a1_target_lease_before +SELECT row_to_json(target_lease)::text +FROM "{target_schema}".qbit_ledger_writer_lease target_lease; +CREATE TEMPORARY TABLE a1_decoy_lease_before (value text NOT NULL); +INSERT INTO a1_decoy_lease_before +SELECT row_to_json(decoy_lease)::text +FROM {decoy_schema}.qbit_ledger_writer_lease decoy_lease; +SELECT "{target_schema}".qbit_confirm_pool_block( + '{'71' * 32}', 10, 'a1-binding', 1, 'a1-binding-token', interval '5 minutes' +); +SELECT "{target_schema}".qbit_reactivate_pool_block( + '{'72' * 32}', 20, 'a1-binding', 1, 'a1-binding-token', interval '5 minutes' +); +{transition_binding_state_sql(target_schema=target_schema, decoy_schema=decoy_schema)} +""" + ) + assert_equal( + result["proconfig"], + { + "qbit_confirm_pool_block": expected_config, + "qbit_reactivate_pool_block": expected_config, + }, + f"{label} exact transition proconfig", + ) + assert_equal( + result["target_sequence"], + {"last_value": 2, "is_called": True}, + f"{label} target allocator", + ) + assert_equal( + result["decoy_sequence"], + {"last_value": 700, "is_called": False}, + f"{label} decoy allocator preservation", + ) + assert_equal( + result["target_rows"], + { + "71" * 32: { + "chain_state": "confirmed", + "audit_publication_sequence": 2, + }, + "72" * 32: { + "chain_state": "confirmed", + "audit_publication_sequence": 1, + }, + }, + f"{label} target row transitions", + ) + assert_equal( + result["decoy_rows"], + { + "71" * 32: { + "chain_state": "prepared", + "audit_publication_sequence": None, + }, + "72" * 32: { + "chain_state": "inactive", + "audit_publication_sequence": 700, + }, + }, + f"{label} decoy row preservation", + ) + assert_equal( + result["target_lease_changed"], + True, + f"{label} target lease renewal", + ) + assert_equal( + result["target_lease"], + { + "writer_id": "a1-binding", + "writer_epoch": 1, + "writer_session_token": "a1-binding-token", + "active": True, + "expiry_after_update": True, + }, + f"{label} target lease identity", + ) + assert_equal( + result["decoy_lease_unchanged"], + True, + f"{label} decoy lease byte preservation", + ) + assert_equal( + result["decoy_lease"], + { + "writer_id": "a1-binding", + "writer_epoch": 1, + "writer_session_token": "a1-binding-token", + }, + f"{label} decoy lease identity", + ) + finally: + target.release_writer_lease() + target.close() + + +def test_durable_floor_ignores_allocator_gaps() -> None: + schema = create_owned_schema("durable_floor") + ledger = ScopedPsqlLedger( + test_schema=schema, + writer_id="a1-durable-floor", + writer_epoch=1, + initialize_schema=True, + ) + memory_seed = SingleWriterShareLedger() + block_a = "61" * 32 + block_b = "62" * 32 + rejected = "63" * 32 + try: + assert_equal(ledger.audit_publication_sequence_floor(), 0, "empty floor") + seed_prepared_direct( + ledger, + memory_seed, + block_hash=block_a, + block_height=10, + ) + assert_equal( + ledger.pool_block_state(block_hash=block_a)[ + "audit_publication_sequence" + ], + None, + "prepared floor row is nullable", + ) + committed_gap = ledger._run_json( + """ +SELECT json_build_object( + 'value', nextval('qbit_audit_publication_sequence_seq') +); +""" + )["value"] + assert_equal(committed_gap, 1, "committed unattached allocator gap") + assert_equal(ledger.audit_publication_sequence_floor(), 0, "gap floor") + ledger._run_sql( + """ +BEGIN; +SELECT nextval('qbit_audit_publication_sequence_seq'); +ROLLBACK; +""" + ) + assert_equal( + allocator_state(ledger), + {"last_value": 2, "is_called": True}, + "rolled-back allocator gap persists", + ) + assert_equal( + ledger.audit_publication_sequence_floor(), + 0, + "rolled-back gap is not authority", + ) + confirmed = ledger.confirm_accepted_block( + block_hash=block_a, + active_tip_height=10, + ) + assert_equal( + confirmed["audit_publication_sequence"], + 3, + "confirmation follows both allocator gaps", + ) + before = allocator_state(ledger) + for _round in range(3): + replay = ledger.confirm_accepted_block( + block_hash=block_a, + active_tip_height=10, + ) + assert_equal( + replay["confirmed_count"], + 2, + "gap-vector confirmation replay disposition", + ) + assert_equal( + replay["audit_publication_sequence"], + 3, + "gap-vector confirmation replay", + ) + assert_equal(allocator_state(ledger), before, "gap replay immobility") + ledger.mark_pool_block_inactive( + block_hash=block_a, + active_tip_height=10, + ) + assert_equal(ledger.audit_publication_sequence_floor(), 3, "inactive floor") + reactivated = ledger.reactivate_pool_block( + block_hash=block_a, + active_tip_height=10, + ) + assert_equal( + reactivated["audit_publication_sequence"], + 3, + "reactivation preserves ordinal after gaps", + ) + assert_equal( + allocator_state(ledger), + before, + "gap-vector reactivation allocator immobility", + ) + before = allocator_state(ledger) + assert_equal( + ledger.reactivate_pool_block( + block_hash=block_a, + active_tip_height=10, + ), + {"backend": "postgres-psql", "reactivated_count": 0}, + "gap-vector reactivation replay", + ) + assert_equal(allocator_state(ledger), before, "reactivation replay immobility") + ledger.mark_pool_block_inactive( + block_hash=block_a, + active_tip_height=10, + ) + ledger.reverse_immature_block( + block_hash=block_a, + active_tip_height=10, + ) + assert_equal(ledger.audit_publication_sequence_floor(), 3, "reversed floor") + seed_prepared_direct( + ledger, + memory_seed, + block_hash=rejected, + block_height=20, + ) + before = allocator_state(ledger) + assert_equal( + ledger.reject_prepared_block( + block_hash=rejected, + active_tip_height=20, + )["rejected_count"], + 1, + "rejected nullable floor row", + ) + assert_equal(allocator_state(ledger), before, "rejection allocator immobility") + assert_equal(ledger.audit_publication_sequence_floor(), 3, "rejected floor") + assert_equal( + ledger._run_json( + """ +SELECT json_build_object( + 'value', nextval('qbit_audit_publication_sequence_seq') +); +""" + )["value"], + 4, + "second unattached allocator gap", + ) + seed_prepared_direct( + ledger, + memory_seed, + block_hash=block_b, + block_height=1, + ) + assert_equal( + ledger.confirm_accepted_block( + block_hash=block_b, + active_tip_height=1, + )["audit_publication_sequence"], + 5, + "post-gap event ordinal", + ) + ledger._run_sql( + f""" +INSERT INTO qbit_pool_blocks ( + block_hash, audit_publication_sequence, block_height, parent_hash, + coinbase_txid, payout_manifest_sha256, chain_state, maturity_state +) VALUES + ('{'64' * 32}', 9000000000, 30, '{'10' * 32}', '{'20' * 32}', + '{'30' * 32}', 'prepared', 'immature'), + ('{'65' * 32}', 9000000001, 31, '{'10' * 32}', '{'20' * 32}', + '{'30' * 32}', 'rejected', 'immature'); +""" + ) + assert_equal( + ledger.audit_publication_sequence_floor(), + 9_000_000_001, + "partial-state bigint durable floor", + ) + finally: + ledger.release_writer_lease() + ledger.close() + restart = ScopedPsqlLedger( + test_schema=schema, + writer_id="a1-durable-floor-restart", + writer_epoch=1, + initialize_schema=True, + ) + try: + assert_equal( + restart.audit_publication_sequence_floor(), + 9_000_000_001, + "restart bigint durable floor", + ) + assert_equal( + restart.pool_block_state(block_hash=block_a), + { + "block_hash": block_a, + "block_height": 10, + "parent_hash": "10" * 32, + "chain_state": "reversed", + "maturity_state": "reversed", + "audit_publication_sequence": 3, + }, + "restart exact reversed serializer", + ) + assert_equal( + restart.pool_block_state(block_hash=rejected)[ + "audit_publication_sequence" + ], + None, + "restart exact rejected nullable serializer", + ) + finally: + restart.release_writer_lease() + restart.close() + + +HASHRATE_SERIES_SUBJECTS: list[tuple[str, str | None]] = [ + ("pool", None), + ("miner", "m1"), + ("miner", "m2"), + # An unknown miner keeps the empty-series ('[]') path in the sweep. + ("miner", "m3"), +] +# (range_id, bucket, lookback_seconds); the internal "window" range id rides +# the same statement as the public ranges and must stay working. +HASHRATE_SERIES_RANGES: list[tuple[str, str, int]] = [ + ("all", "5m", 0), + ("all", "1h", 0), + ("all", "1d", 0), + ("1w", "5m", 900), + ("1w", "1h", 0), + ("1m", "1h", 3600), + ("6m", "1d", 0), + ("window", "5m", 900), +] + + +def seed_hashrate_shares( + ledger: ScopedPsqlLedger, + shares: list[dict[str, object]], +) -> None: + """Insert shares in list order so share_seq mirrors the list index + 1.""" + values = [] + for index, share in enumerate(shares, start=1): + accepted = bool(share.get("accepted", True)) + values.append( + "(" + f"'hashrate-share-{index}', " + f"'{share['miner_id']}', " + f"'{index:04d}', " + f"decode('{index:02x}' || repeat('00', 31), 'hex'), " + f"{int(share['share_difficulty'])}, " # type: ignore[call-overload] + "1000, 10, " + f"'hashrate-job-{index}', " + f"to_timestamp({int(share['accepted_epoch']) - 1}), " # type: ignore[call-overload] + f"{1_700_000_000 + index}, " + f"to_timestamp({int(share['accepted_epoch'])}), " # type: ignore[call-overload] + f"{'true' if accepted else 'false'}, " + f"{'NULL' if accepted else chr(39) + 'low-difficulty' + chr(39)}, " + "'a1-hashrate', 1" + ")" + ) + ledger._run_sql( + "INSERT INTO qbit_share_ledger (\n" + " share_id, miner_id, payout_order_key, p2mr_program,\n" + " share_difficulty, network_difficulty, template_height, job_id,\n" + " job_issued_at, ntime, accepted_at, accepted, reject_reason,\n" + " writer_id, writer_epoch\n" + ") VALUES\n " + + ",\n ".join(values) + + ";" + ) + + +def hashrate_series_matrix( + ledger: ScopedPsqlLedger, + *, + anchor: int, + forced_raw: bool, +) -> dict[tuple[str, str | None, str, str], list[dict[str, object]]]: + """Fetch every subject x range x bucket combination in one sweep. + + ``forced_raw`` shadows the schema probe so the historical raw-scan + statement runs against the same database; its output is the parity + ground truth every rollup-backed sweep is compared against byte for + byte. + """ + if forced_raw: + ledger.__dict__["_hashrate_rollup_schema_present"] = lambda: False + try: + results = {} + for subject_type, subject_id in HASHRATE_SERIES_SUBJECTS: + for range_id, bucket, lookback_seconds in HASHRATE_SERIES_RANGES: + results[(subject_type, subject_id, range_id, bucket)] = ( + ledger.dashboard_hashrate_series( + subject_type=subject_type, + subject_id=subject_id, + range_id=range_id, + bucket=bucket, + lookback_seconds=lookback_seconds, + range_anchor_epoch=None if range_id == "all" else anchor, + ) + ) + return results + finally: + ledger.__dict__.pop("_hashrate_rollup_schema_present", None) + + +def assert_hashrate_matrices_equal( + actual: dict[tuple[str, str | None, str, str], list[dict[str, object]]], + expected: dict[tuple[str, str | None, str, str], list[dict[str, object]]], + message: str, +) -> None: + assert_equal(sorted(actual), sorted(expected), f"{message} combination set") + for key in expected: + assert_equal(actual[key], expected[key], f"{message} {key}") + + +def hashrate_rollup_progress_row(ledger: ScopedPsqlLedger) -> object: + return ledger._run_json( + """ +SELECT json_build_object( + 'last_share_seq', ( + SELECT last_share_seq FROM qbit_hashrate_rollup_progress WHERE singleton + ) +); +""" + )["last_share_seq"] + + +def hashrate_rollup_snapshot(ledger: ScopedPsqlLedger) -> dict[str, object]: + return ledger._run_json( + """ +SELECT json_build_object( + 'pool', ( + SELECT COALESCE(json_agg(json_build_object( + 'grain', grain_seconds, + 'bucket', bucket_epoch, + 'count', accepted_share_count, + 'difficulty', accepted_share_difficulty::text + ) ORDER BY grain_seconds, bucket_epoch), '[]'::json) + FROM qbit_hashrate_rollup_pool + ), + 'miner', ( + SELECT COALESCE(json_agg(json_build_object( + 'grain', grain_seconds, + 'bucket', bucket_epoch, + 'miner', miner_id, + 'count', accepted_share_count, + 'difficulty', accepted_share_difficulty::text + ) ORDER BY grain_seconds, bucket_epoch, miner_id), '[]'::json) + FROM qbit_hashrate_rollup_miner + ) +); +""" + ) + + +def test_hashrate_rollup_series_parity() -> None: + """Rollup-served hashrate series must be byte-identical to the raw scan. + + Covers the acceptance bar for every serving state: before any rollup + pass (absent progress row), mid-backfill (watermark inside history, so + the live tail must cover the un-rolled remainder), fully caught up, + after the progress row is deleted with stale rollup rows left behind, + and with the rollup tables missing entirely (the pre-migration raw + statement). The range anchor is deliberately not bucket-aligned so the + raw scan emits a partial bucket straddling the range lower bound, which + the rollup path must reproduce from the ledger rather than from full + rollup buckets. A future-dated share (a coordinator clock running ahead + of the database clock) is folded into the rollups and must stay excluded + from every serving state, exactly as the raw scan's + accepted_at <= ended_at excludes it. + """ + schema = create_owned_schema("hashrate") + ledger = ScopedPsqlLedger( + test_schema=schema, + writer_id="a1-hashrate", + writer_epoch=1, + initialize_schema=True, + ) + try: + anchor = int(time.time()) + anchor -= anchor % 300 + anchor += 7 # unaligned modulo every grain, so boundary buckets exist + lookback_1w = 900 + range_start_1w = anchor - 7 * 86400 - lookback_1w + bucket_base = anchor - 7 - 86400 # aligned 5m bucket, one day back + seed_hashrate_shares( + ledger, + [ + # seq 1-6: one aligned 5m bucket split across two maintenance + # passes below, so the same rollup rows must accumulate. + {"miner_id": "m1", "accepted_epoch": bucket_base + 10, "share_difficulty": 100}, + {"miner_id": "m2", "accepted_epoch": bucket_base + 20, "share_difficulty": 200}, + {"miner_id": "m1", "accepted_epoch": bucket_base + 30, "share_difficulty": 300}, + {"miner_id": "m1", "accepted_epoch": bucket_base + 40, "share_difficulty": 400}, + {"miner_id": "m2", "accepted_epoch": bucket_base + 50, "share_difficulty": 500}, + {"miner_id": "m1", "accepted_epoch": bucket_base + 60, "share_difficulty": 600}, + # seq 7: rejected rows advance the watermark without counting. + {"miner_id": "m1", "accepted_epoch": bucket_base + 70, "share_difficulty": 700, "accepted": False}, + # seq 8: outside the 1w window, inside 1m/6m/all. + {"miner_id": "m2", "accepted_epoch": anchor - 9 * 86400, "share_difficulty": 800}, + # seq 9-12: around the 1w lower bound. seq 9 sits inside the + # straddling 5m bucket but before the bound, so it must stay + # excluded; 10-12 form the raw scan's partial leading bucket. + {"miner_id": "m1", "accepted_epoch": range_start_1w - 5, "share_difficulty": 900}, + {"miner_id": "m1", "accepted_epoch": range_start_1w, "share_difficulty": 1000}, + {"miner_id": "m2", "accepted_epoch": range_start_1w + 10, "share_difficulty": 1100}, + {"miner_id": "m1", "accepted_epoch": range_start_1w + 290, "share_difficulty": 1200}, + # seq 13-14: recent tail-of-history shares. + {"miner_id": "m2", "accepted_epoch": anchor - 50, "share_difficulty": 1300, "accepted": False}, + {"miner_id": "m1", "accepted_epoch": anchor - 30, "share_difficulty": 1400}, + # seq 15: future-dated relative to the database clock (a + # coordinator clock running ahead). It is folded into the + # rollups like any other share, and every serving state must + # exclude it exactly as the raw scan's + # accepted_at <= ended_at does. + {"miner_id": "m2", "accepted_epoch": anchor + 3600, "share_difficulty": 1500}, + ], + ) + raw_truth = hashrate_series_matrix(ledger, anchor=anchor, forced_raw=True) + straddle_epoch = range_start_1w - range_start_1w % 300 + straddle_stamp = datetime.fromtimestamp( + straddle_epoch, timezone.utc + ).strftime("%Y-%m-%dT%H:%M:%SZ") + partial_bucket = [ + point + for point in raw_truth[("pool", None, "1w", "5m")] + if point["timestamp"] == straddle_stamp + ] + assert_equal( + [ + (point["accepted_share_count"], point["accepted_share_difficulty"]) + for point in partial_bucket + ], + [(3, "3300")], + "raw ground truth contains the partial range-boundary bucket", + ) + + assert_hashrate_matrices_equal( + hashrate_series_matrix(ledger, anchor=anchor, forced_raw=False), + raw_truth, + "absent progress row serves the raw scan", + ) + + assert_equal( + ledger.advance_hashrate_rollups(batch_limit=5), + {"scanned": 5, "last_share_seq": 5, "caught_up": False}, + "first rollup pass bootstraps the watermark", + ) + assert_hashrate_matrices_equal( + hashrate_series_matrix(ledger, anchor=anchor, forced_raw=False), + raw_truth, + "mid-backfill rollup + tail merge", + ) + + assert_equal( + ledger.advance_hashrate_rollups(batch_limit=4), + {"scanned": 4, "last_share_seq": 9, "caught_up": False}, + "second rollup pass advances the watermark", + ) + # seq 1-5 landed in the first pass and seq 6 in the second, all in + # the same aligned 5m bucket: the upsert must accumulate, and the + # rejected seq 7 must advance the watermark without counting. + assert_equal( + ledger._run_json( + f""" +SELECT json_build_object( + 'count', accepted_share_count, + 'difficulty', accepted_share_difficulty::text +) +FROM qbit_hashrate_rollup_pool +WHERE grain_seconds = 300 AND bucket_epoch = {bucket_base}; +""" + ), + {"count": 6, "difficulty": "2100"}, + "adjacent passes accumulate into the same pool bucket", + ) + assert_equal( + ledger._run_json( + f""" +SELECT json_build_object( + 'count', accepted_share_count, + 'difficulty', accepted_share_difficulty::text +) +FROM qbit_hashrate_rollup_miner +WHERE grain_seconds = 300 AND bucket_epoch = {bucket_base} AND miner_id = 'm1'; +""" + ), + {"count": 4, "difficulty": "1400"}, + "adjacent passes accumulate into the same miner bucket", + ) + + assert_equal( + ledger.advance_hashrate_rollups(batch_limit=50000), + {"scanned": 6, "last_share_seq": 15, "caught_up": True}, + "final rollup pass reports caught up", + ) + assert_hashrate_matrices_equal( + hashrate_series_matrix(ledger, anchor=anchor, forced_raw=False), + raw_truth, + "caught-up rollup serving", + ) + assert_equal( + hashrate_rollup_progress_row(ledger), + 15, + "watermark covers rejected rows", + ) + assert_equal( + ledger._run_json( + """ +SELECT json_build_object( + 'count', ( + SELECT COALESCE(sum(accepted_share_count), 0) + FROM qbit_hashrate_rollup_pool + WHERE grain_seconds = 300 + ) +); +""" + )["count"], + 13, + "rejected shares never reach the rollup rows; the future-dated" + " share does", + ) + assert_equal( + ledger.advance_hashrate_rollups(batch_limit=50000), + {"scanned": 0, "last_share_seq": 15, "caught_up": True}, + "caught-up pass is an idempotent no-op", + ) + + # Deleting only the progress row leaves fully populated rollup + # tables behind; the gate on the watermark must exclude them and the + # widened tail alone must reproduce the raw scan -- if the gating + # were broken every bucket would double. + ledger._run_sql("DELETE FROM qbit_hashrate_rollup_progress;") + assert_hashrate_matrices_equal( + hashrate_series_matrix(ledger, anchor=anchor, forced_raw=False), + raw_truth, + "deleted progress row falls back to the raw scan", + ) + + # A database that predates the rollup DDL cannot even parse the + # rollup statement; the probe must route to the unchanged raw + # statement. + ledger._run_sql( + "DROP TABLE qbit_hashrate_rollup_miner;\n" + "DROP TABLE qbit_hashrate_rollup_pool;\n" + "DROP TABLE qbit_hashrate_rollup_progress;" + ) + ledger.__dict__.pop("_hashrate_rollup_schema_ready", None) + assert_hashrate_matrices_equal( + hashrate_series_matrix(ledger, anchor=anchor, forced_raw=False), + raw_truth, + "missing rollup tables run the raw statement", + ) + finally: + ledger.release_writer_lease() + ledger.close() + + +def test_hashrate_rollup_watermark_guard() -> None: + """A concurrent watermark advance must fail the pass without writes. + + The guard can only trip when the progress row changes between the + statement's snapshot and its guarded upsert, so the race is staged for + real: a second session updates the row inside an open transaction, the + ledger pass is started and provably blocks on that row lock, and only + then does the second session commit. The blocked pass must resume, + lose the guard, write no rollup rows at all, and raise. + """ + schema = create_owned_schema("hashrate_guard") + ledger = ScopedPsqlLedger( + test_schema=schema, + writer_id="a1-hashrate-guard", + writer_epoch=1, + initialize_schema=True, + ) + locker: subprocess.Popen[str] | None = None + try: + now_epoch = int(time.time()) + seed_hashrate_shares( + ledger, + [ + {"miner_id": "m1", "accepted_epoch": now_epoch - 500, "share_difficulty": 100}, + {"miner_id": "m2", "accepted_epoch": now_epoch - 400, "share_difficulty": 200}, + {"miner_id": "m1", "accepted_epoch": now_epoch - 300, "share_difficulty": 300}, + {"miner_id": "m2", "accepted_epoch": now_epoch - 200, "share_difficulty": 400}, + ], + ) + assert_equal( + ledger.advance_hashrate_rollups(batch_limit=100), + {"scanned": 4, "last_share_seq": 4, "caught_up": True}, + "guard scenario bootstrap pass", + ) + before = hashrate_rollup_snapshot(ledger) + # Give the raced pass real work so a broken guard would visibly + # double-count rather than trivially writing nothing. + seed_hashrate_shares_offset = [ + {"miner_id": "m1", "accepted_epoch": now_epoch - 100, "share_difficulty": 500}, + {"miner_id": "m2", "accepted_epoch": now_epoch - 90, "share_difficulty": 600}, + ] + ledger._run_sql( + "INSERT INTO qbit_share_ledger (\n" + " share_id, miner_id, payout_order_key, p2mr_program,\n" + " share_difficulty, network_difficulty, template_height, job_id,\n" + " job_issued_at, ntime, accepted_at, accepted, reject_reason,\n" + " writer_id, writer_epoch\n" + ") VALUES\n" + + ",\n".join( + "(" + f"'hashrate-guard-share-{index}', " + f"'{share['miner_id']}', " + f"'9{index:03d}', " + f"decode('9{index:01d}' || repeat('00', 31), 'hex'), " + f"{share['share_difficulty']}, " + "1000, 10, " + f"'hashrate-guard-job-{index}', " + f"to_timestamp({int(share['accepted_epoch']) - 1}), " # type: ignore[call-overload] + f"{1_700_009_000 + index}, " + f"to_timestamp({int(share['accepted_epoch'])}), " # type: ignore[call-overload] + "true, NULL, 'a1-hashrate-guard', 1)" + for index, share in enumerate(seed_hashrate_shares_offset, start=1) + ) + + ";" + ) + + locker = subprocess.Popen( + [ + *BASE_PSQL_ARGV, + "--no-psqlrc", + "--set", + "ON_ERROR_STOP=1", + "--quiet", + ], + stdin=subprocess.PIPE, + stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL, + text=True, + start_new_session=True, + ) + with ACTIVE_CHILDREN_LOCK: + ACTIVE_CHILDREN.add(locker) + assert locker.stdin is not None + locker.stdin.write( + f'SET search_path TO "{schema}", pg_catalog;\n' + "BEGIN;\n" + "UPDATE qbit_hashrate_rollup_progress\n" + "SET last_share_seq = last_share_seq + 100,\n" + " updated_at = clock_timestamp()\n" + "WHERE singleton;\n" + ) + locker.stdin.flush() + deadline = time.monotonic() + 20.0 + while True: + holding = run_json( + """ +SELECT json_build_object('holding', ( + SELECT count(*) + FROM pg_stat_activity + WHERE state = 'idle in transaction' + AND query LIKE '%last_share_seq = last_share_seq + 100%' +)); +""" + )["holding"] + if int(holding) > 0: + break + if time.monotonic() > deadline: + raise GateFailure("watermark lock holder never reached its open transaction") + time.sleep(0.1) + + outcome: dict[str, object] = {} + + def run_raced_advance() -> None: + try: + outcome["result"] = ledger.advance_hashrate_rollups(batch_limit=100) + except BaseException as error: # noqa: BLE001 - reported below + outcome["error"] = error + + raced = threading.Thread(target=run_raced_advance, daemon=True) + raced.start() + deadline = time.monotonic() + 20.0 + while True: + waiting = run_json( + """ +SELECT json_build_object('waiting', ( + SELECT count(*) + FROM pg_stat_activity + WHERE state = 'active' + AND wait_event_type = 'Lock' + AND pid <> pg_backend_pid() + AND query LIKE '%qbit_hashrate_rollup_progress%' +)); +""" + )["waiting"] + if int(waiting) > 0: + break + if time.monotonic() > deadline: + raise GateFailure("raced rollup advance never blocked on the progress row") + time.sleep(0.1) + locker.stdin.write("COMMIT;\n") + locker.stdin.flush() + locker.stdin.close() + raced.join(timeout=25.0) + if raced.is_alive(): + raise GateFailure("raced rollup advance did not finish after the commit") + error = outcome.get("error") + if not isinstance(error, RuntimeError) or "advanced concurrently" not in str(error): + raise GateFailure( + f"raced rollup advance outcome was {outcome!r}; expected the loud watermark failure" + ) + assert_equal( + hashrate_rollup_snapshot(ledger), + before, + "lost watermark race writes no rollup rows", + ) + assert_equal( + hashrate_rollup_progress_row(ledger), + 104, + "only the concurrent watermark advance landed", + ) + finally: + if locker is not None: + _terminate_and_reap(locker) + if locker.stdin is not None and not locker.stdin.closed: + locker.stdin.close() + with ACTIVE_CHILDREN_LOCK: + ACTIVE_CHILDREN.discard(locker) + ledger.release_writer_lease() + ledger.close() + + +def test_hashrate_rollup_difficulty_overflow_headroom() -> None: + """Bucket totals may exceed one share's numeric(78, 0) precision. + + Individually valid 78-digit share difficulties land in one bucket, so + the total needs 79 digits. The aggregate columns are unconstrained + numeric precisely for this: a constrained column would make the + maintenance upsert overflow on every retry and wedge the watermark. + Both the insert fold and the accumulate (ON CONFLICT) fold must carry + the oversized total exactly, matching the raw scan's unconstrained + sum(numeric). The dashboard's TH/s shaping is not exercised here: its + Decimal rendering cannot represent such magnitudes on the raw path + either, and this scenario pins the storage layer, not that shared + shaping. + """ + schema = create_owned_schema("hashrate_overflow") + ledger = ScopedPsqlLedger( + test_schema=schema, + writer_id="a1-hashrate-overflow", + writer_epoch=1, + initialize_schema=True, + ) + try: + anchor = int(time.time()) + # All three shares inside one 5m bucket, so every grain folds them + # into a single row whose total needs 79 digits. + bucket_epoch = anchor // 300 * 300 - 600 + huge = 6 * 10**77 + seed_hashrate_shares( + ledger, + [ + {"miner_id": "m1", "accepted_epoch": bucket_epoch + 10, "share_difficulty": huge}, + {"miner_id": "m1", "accepted_epoch": bucket_epoch + 20, "share_difficulty": huge}, + {"miner_id": "m1", "accepted_epoch": bucket_epoch + 30, "share_difficulty": huge}, + ], + ) + # First pass exercises the oversized insert fold... + assert_equal( + ledger.advance_hashrate_rollups(batch_limit=2), + {"scanned": 2, "last_share_seq": 2, "caught_up": False}, + "overflow insert fold advances", + ) + total = str(2 * huge) + snapshot = hashrate_rollup_snapshot(ledger) + assert_equal( + [(row["grain"], row["count"], row["difficulty"]) for row in snapshot["pool"]], + [(300, 2, total), (3600, 2, total), (86400, 2, total)], + "pool rollup carries the 79-digit total", + ) + assert_equal( + [(row["grain"], row["miner"], row["difficulty"]) for row in snapshot["miner"]], + [(300, "m1", total), (3600, "m1", total), (86400, "m1", total)], + "miner rollup carries the 79-digit total", + ) + # ...and the second pass exercises the oversized accumulate fold, + # the exact statement a constrained column would fail on every retry. + assert_equal( + ledger.advance_hashrate_rollups(batch_limit=100), + {"scanned": 1, "last_share_seq": 3, "caught_up": True}, + "overflow accumulate fold advances", + ) + accumulated = str(3 * huge) + snapshot = hashrate_rollup_snapshot(ledger) + assert_equal( + [(row["grain"], row["count"], row["difficulty"]) for row in snapshot["pool"]], + [(300, 3, accumulated), (3600, 3, accumulated), (86400, 3, accumulated)], + "pool rollup accumulates past one share's precision", + ) + assert_equal( + [(row["grain"], row["miner"], row["difficulty"]) for row in snapshot["miner"]], + [(300, "m1", accumulated), (3600, "m1", accumulated), (86400, "m1", accumulated)], + "miner rollup accumulates past one share's precision", + ) + finally: + ledger.release_writer_lease() + ledger.close() + + +def test_hashrate_rollup_lease_fence() -> None: + """A fenced-out coordinator's rollup advance writes nothing and raises. + + The maintenance statement renews the exact writer lease tuple and gates + the watermark advance on that renewal matching, so a coordinator whose + lease another session took over must fail loudly without touching the + watermark or either rollup table -- its process-local writer lock alone + does not authorize mutating ledger-derived state. + """ + schema = create_owned_schema("hashrate_lease") + ledger = ScopedPsqlLedger( + test_schema=schema, + writer_id="a1-hashrate-lease", + writer_epoch=1, + initialize_schema=True, + ) + try: + now_epoch = int(time.time()) + seed_hashrate_shares( + ledger, + [ + {"miner_id": "m1", "accepted_epoch": now_epoch - 500, "share_difficulty": 100}, + {"miner_id": "m2", "accepted_epoch": now_epoch - 400, "share_difficulty": 200}, + {"miner_id": "m1", "accepted_epoch": now_epoch - 300, "share_difficulty": 300}, + ], + ) + # Bootstrap with one share left behind the watermark so a broken + # fence would visibly fold it rather than trivially writing nothing. + assert_equal( + ledger.advance_hashrate_rollups(batch_limit=2), + {"scanned": 2, "last_share_seq": 2, "caught_up": False}, + "lease scenario bootstrap pass", + ) + before_rows = hashrate_rollup_snapshot(ledger) + original_lease = ledger._run_json( + """ +SELECT json_build_object( + 'writer_id', writer_id, + 'writer_epoch', writer_epoch, + 'writer_session_token', writer_session_token +) +FROM qbit_ledger_writer_lease +WHERE singleton; +""" + ) + ledger._run_sql( + "UPDATE qbit_ledger_writer_lease\n" + "SET writer_id = 'a1-hashrate-usurper',\n" + " writer_epoch = 2,\n" + " writer_session_token = 'a1-hashrate-usurper-session',\n" + " lease_expires_at = clock_timestamp() + interval '60 seconds',\n" + " updated_at = clock_timestamp()\n" + "WHERE singleton;" + ) + try: + ledger.advance_hashrate_rollups(batch_limit=100) + except RuntimeError as error: + if "writer lease is not active" not in str(error): + raise GateFailure( + f"fenced-out advance raised {error!r}; expected the lease refusal" + ) from error + else: + raise GateFailure("fenced-out advance did not raise") + assert_equal( + hashrate_rollup_progress_row(ledger), + 2, + "fenced-out advance leaves the watermark alone", + ) + assert_equal( + hashrate_rollup_snapshot(ledger), + before_rows, + "fenced-out advance writes no rollup rows", + ) + # Hand the lease back so release and cleanup run as the owner. + ledger._run_sql( + "UPDATE qbit_ledger_writer_lease\n" + f"SET writer_id = '{original_lease['writer_id']}',\n" + f" writer_epoch = {int(original_lease['writer_epoch'])},\n" + f" writer_session_token = '{original_lease['writer_session_token']}',\n" + " lease_expires_at = clock_timestamp() + interval '60 seconds',\n" + " updated_at = clock_timestamp()\n" + "WHERE singleton;" + ) + assert_equal( + ledger.advance_hashrate_rollups(batch_limit=100), + {"scanned": 1, "last_share_seq": 3, "caught_up": True}, + "restored lease folds the held-back share", + ) + finally: + ledger.release_writer_lease() + ledger.close() + + +def _marker_iso(epoch: int) -> str: + return datetime.fromtimestamp(epoch, timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ") + + +def test_dashboard_block_markers_read_model() -> None: + """The block-marker SQL over real qbit_pool_blocks rows. + + Seeds found blocks across two 1h buckets plus one block just below the 1w + range's lower bound, one reversed block, and one future-dated block, then + asserts the read model's counts, per-bucket top-3 ordering (found_at + DESC, height DESC), range filtering against the caller-provided anchor, + the unbounded ``all`` range, and the upper cutoff at the statement clock + that keeps a future-dated block off the chart -- the same bound the + hashrate series applies. + """ + schema = create_owned_schema("block_markers") + ledger = ScopedPsqlLedger( + test_schema=schema, + writer_id="a1-block-markers", + writer_epoch=1, + initialize_schema=True, + ) + anchor_epoch = 1_750_000_000 + bucket_a = public_api.hashrate_series_min_epoch(anchor_epoch, 7 * 86400, 3600) + bucket_b = anchor_epoch // 3600 * 3600 - 7200 + hash_in_a = "71" * 32 + hash_below_range = "72" * 32 + hashes_in_b = {201: "73" * 32, 202: "74" * 32, 203: "75" * 32, 204: "76" * 32} + hash_reversed = "77" * 32 + hash_future = "78" * 32 + found_in_a = bucket_a + 10 + found_in_b = {201: bucket_b + 100, 202: bucket_b + 200, 203: bucket_b + 300, 204: bucket_b + 300} + try: + rows = [ + (hash_in_a, 100, found_in_a, "confirmed"), + # One second below the range lower bound: dropped from 1w with the + # whole straddling bucket, present in the unbounded all range. + (hash_below_range, 99, bucket_a - 1, "confirmed"), + (hashes_in_b[201], 201, found_in_b[201], "confirmed"), + (hashes_in_b[202], 202, found_in_b[202], "confirmed"), + # Two blocks tied on found_at so the height tiebreak is observable. + (hashes_in_b[203], 203, found_in_b[203], "confirmed"), + (hashes_in_b[204], 204, found_in_b[204], "confirmed"), + # Future-dated relative to the database clock (a clock jump or a + # bad import): capped out by found_at <= clock_timestamp() so it + # can never render beyond the chart's current endpoint. + (hash_future, 300, int(time.time()) + 3600, "confirmed"), + ] + values = ",\n ".join( + f"('{block_hash}', {height}, '{'10' * 32}', '{'20' * 32}', " + f"'{'30' * 32}', to_timestamp({epoch}), '{chain_state}', 'immature', NULL)" + for block_hash, height, epoch, chain_state in rows + ) + ledger._run_sql( + f""" +INSERT INTO qbit_pool_blocks ( + block_hash, block_height, parent_hash, coinbase_txid, + payout_manifest_sha256, found_at, chain_state, maturity_state, + disconnected_at +) VALUES + {values}, + ('{hash_reversed}', 205, '{'10' * 32}', '{'20' * 32}', '{'30' * 32}', + to_timestamp({bucket_b + 400}), 'reversed', 'reversed', + to_timestamp({bucket_b + 500})); +""" + ) + + week = ledger.dashboard_block_markers( + range_id="1w", + bucket="1h", + range_anchor_epoch=anchor_epoch, + ) + assert_equal( + week, + { + "total_blocks": 5, + "points": [ + { + "timestamp": _marker_iso(bucket_a), + "block_count": 1, + "blocks": [ + { + "height": 100, + "hash": hash_in_a, + "found_at": _marker_iso(found_in_a), + } + ], + }, + { + "timestamp": _marker_iso(bucket_b), + "block_count": 4, + "blocks": [ + { + "height": 204, + "hash": hashes_in_b[204], + "found_at": _marker_iso(found_in_b[204]), + }, + { + "height": 203, + "hash": hashes_in_b[203], + "found_at": _marker_iso(found_in_b[203]), + }, + { + "height": 202, + "hash": hashes_in_b[202], + "found_at": _marker_iso(found_in_b[202]), + }, + ], + }, + ], + }, + "1w block markers", + ) + + everything = ledger.dashboard_block_markers( + range_id="all", + bucket="1d", + range_anchor_epoch=None, + ) + assert_equal(everything["total_blocks"], 6, "all-range total excludes reversed only") + assert_equal( + sum(int(point["block_count"]) for point in everything["points"]), + 6, + "all-range bucket counts sum to the total", + ) + listed_hashes = { + block["hash"] + for point in everything["points"] + for block in point["blocks"] + } + if hash_below_range not in listed_hashes: + raise GateFailure("all range must include the below-1w-range block") + if hash_reversed in listed_hashes: + raise GateFailure("reversed block leaked into the marker series") + if hash_future in listed_hashes: + raise GateFailure("future-dated block leaked into the marker series") + epochs = [ + int( + datetime.strptime(str(point["timestamp"]), "%Y-%m-%dT%H:%M:%SZ") + .replace(tzinfo=timezone.utc) + .timestamp() + ) + for point in everything["points"] + ] + assert_equal(epochs, sorted(epochs), "all-range points ascend") + for epoch in epochs: + assert_equal(epoch % 86400, 0, "all-range 1d bucket alignment") + finally: + ledger.release_writer_lease() + ledger.close() + + +def main() -> None: + public_before = public_sentinel() + failure: BaseException | None = None + try: + test_exact_transition_parity() + assert_empty_serializer_case(legacy=False) + assert_empty_serializer_case(legacy=True) + assert_transition_binding_case(temporary=False) + assert_transition_binding_case(temporary=True) + test_durable_floor_ignores_allocator_gaps() + test_hashrate_rollup_series_parity() + test_hashrate_rollup_watermark_guard() + test_hashrate_rollup_difficulty_overflow_headroom() + test_hashrate_rollup_lease_fence() + test_dashboard_block_markers_read_model() + except BaseException as error: + failure = error + try: + cleanup_active_children() + cleanup_owned_schemas() + assert_equal(marker_schema_count(), 0, "owned schema marker cleanup") + assert_equal(public_sentinel(), public_before, "public sentinel preservation") + except BaseException as cleanup_error: + if failure is None: + raise + raise GateFailure( + f"scenario failed with {failure!r}; cleanup also failed with " + f"{cleanup_error!r}" + ) from cleanup_error + else: + atexit.unregister(cleanup_active_children) + atexit.unregister(cleanup_owned_schemas) + if failure is not None: + raise failure + print( + "prism postgres A1 gate PASS " + "exact-transition-parity empty-fresh empty-legacy " + "ordinary-decoy-binding temporary-decoy-binding durable-floor-gaps " + "hashrate-rollup-parity hashrate-rollup-guard " + "hashrate-rollup-overflow hashrate-rollup-lease-fence " + "block-markers" + ) + + +if __name__ == "__main__": + main() diff --git a/tests/prism_postgres_a1_migration_gate.py b/tests/prism_postgres_a1_migration_gate.py new file mode 100644 index 000000000..949e75f8b --- /dev/null +++ b/tests/prism_postgres_a1_migration_gate.py @@ -0,0 +1,1782 @@ +"""Non-discovered PostgreSQL ordinal migration integration gate.""" + +from __future__ import annotations + +import hashlib +import subprocess +import tempfile +import time +from pathlib import Path +from typing import Any + +from tests import prism_postgres_a1_gate as support +from tests.prism_postgres_a1_gate import ( + ACTIVE_CHILDREN, + ACTIVE_CHILDREN_LOCK, + BASE_PSQL_ARGV, + GateFailure, + LEGACY_POOL_BLOCKS_SQL, + PSQL_OUTPUT_LIMIT_BYTES, + PSQL_TIMEOUT_SECONDS, + RUN_TOKEN, + SCHEMA_PATTERN, + ScopedPsqlLedger, + _schema_registered, + _terminate_and_reap, + assert_equal, + create_owned_schema, + run_json, + run_psql, +) + +MIGRATION_HASH_A = "81" * 32 +MIGRATION_HASH_B = "82" * 32 +MIGRATION_HASH_C = "83" * 32 +MIGRATION_HASH_OTHER = "84" * 32 +BIGINT_MAX = 9_223_372_036_854_775_807 + + +def add_ordinal_column(schema: str) -> None: + run_psql( + "ALTER TABLE qbit_pool_blocks " + "ADD COLUMN audit_publication_sequence bigint;", + schema=schema, + ) + + +def create_ordinal_sequence( + schema: str, + *, + last_value: int = 1, + is_called: bool = False, +) -> None: + run_psql( + f""" +CREATE SEQUENCE qbit_audit_publication_sequence_seq AS bigint; +SELECT setval( + 'qbit_audit_publication_sequence_seq'::regclass, + {last_value}, + {'true' if is_called else 'false'} +); +""", + schema=schema, + ) + + +def seed_migration_rows( + schema: str, + *, + ordinals: tuple[int | None, int | None, int | None] | None, + include_other_states: bool = True, +) -> None: + ordinal_column = ( + ", audit_publication_sequence" if ordinals is not None else "" + ) + ordinal_values = ( + ["NULL" if value is None else str(value) for value in ordinals] + if ordinals is not None + else [None, None, None] + ) + + def row( + block_hash: str, + found_at: str, + ordinal: str | None, + state: str, + maturity: str = "immature", + ) -> str: + ordinal_sql = f", {ordinal}" if ordinal is not None else "" + disconnected = ( + ", '2020-01-02T00:00:00Z'" if maturity == "reversed" else ", NULL" + ) + return ( + f"('{block_hash}', 10, '{'10' * 32}', '{'20' * 32}', " + f"'{'30' * 32}', '{found_at}', '{state}', '{maturity}', " + f"NULL{disconnected}{ordinal_sql})" + ) + + rows = [ + # Insertion order intentionally differs from deterministic migration + # order. A/B share a timestamp and sort by block hash. + row( + MIGRATION_HASH_C, + "2020-01-01T00:00:02Z", + ordinal_values[2], + "confirmed", + ), + row( + MIGRATION_HASH_B, + "2020-01-01T00:00:01Z", + ordinal_values[1], + "confirmed", + ), + row( + MIGRATION_HASH_A, + "2020-01-01T00:00:01Z", + ordinal_values[0], + "confirmed", + ), + ] + if include_other_states: + rows.extend( + [ + row( + MIGRATION_HASH_OTHER, + "2020-01-01T00:00:03Z", + "NULL" if ordinals is not None else None, + "prepared", + ), + row( + "85" * 32, + "2020-01-01T00:00:04Z", + "NULL" if ordinals is not None else None, + "inactive", + ), + row( + "86" * 32, + "2020-01-01T00:00:05Z", + "NULL" if ordinals is not None else None, + "rejected", + "reversed", + ), + row( + "87" * 32, + "2020-01-01T00:00:06Z", + "NULL" if ordinals is not None else None, + "reversed", + "reversed", + ), + ] + ) + run_psql( + f""" +INSERT INTO qbit_pool_blocks ( + block_hash, + block_height, + parent_hash, + coinbase_txid, + payout_manifest_sha256, + found_at, + chain_state, + maturity_state, + matured_at, + disconnected_at + {ordinal_column} +) VALUES +{', '.join(rows)}; +""", + schema=schema, + ) + + +def migration_snapshot(schema: str) -> dict[str, object]: + if SCHEMA_PATTERN.fullmatch(schema) is None: + raise GateFailure(f"invalid migration snapshot schema: {schema!r}") + snapshot = run_json( + f""" +SELECT json_build_object( + 'rows', ( + SELECT COALESCE( + json_agg(to_jsonb(block) ORDER BY block_hash), + '[]'::json + ) + FROM "{schema}".qbit_pool_blocks block + ), + 'column', ( + SELECT json_build_object( + 'attnum', attribute.attnum, + 'type_oid', attribute.atttypid::pg_catalog.int8, + 'nullable', NOT attribute.attnotnull, + 'has_default', attribute.atthasdef, + 'identity', attribute.attidentity, + 'generated', attribute.attgenerated, + 'collation', attribute.attcollation::pg_catalog.int8 + ) + FROM pg_catalog.pg_attribute attribute + JOIN pg_catalog.pg_class relation + ON relation.oid = attribute.attrelid + JOIN pg_catalog.pg_namespace namespace + ON namespace.oid = relation.relnamespace + WHERE namespace.nspname = '{schema}' + AND relation.relname = 'qbit_pool_blocks' + AND attribute.attname = 'audit_publication_sequence' + AND attribute.attnum > 0 + AND NOT attribute.attisdropped + ), + 'sequence_catalog', ( + SELECT json_build_object( + 'oid', relation.oid, + 'kind', relation.relkind, + 'persistence', relation.relpersistence, + 'owner', relation.relowner, + 'type_oid', sequence.seqtypid::pg_catalog.int8, + 'start', sequence.seqstart, + 'increment', sequence.seqincrement, + 'max', sequence.seqmax, + 'min', sequence.seqmin, + 'cache', sequence.seqcache, + 'cycle', sequence.seqcycle, + 'owned_dependencies', ( + SELECT count(*) + FROM pg_catalog.pg_depend dependency + WHERE dependency.classid = 'pg_catalog.pg_class'::regclass + AND dependency.objid = relation.oid + AND dependency.refclassid = 'pg_catalog.pg_class'::regclass + AND dependency.refobjsubid > 0 + AND dependency.deptype IN ('a', 'i') + ), + 'same_owner_as_table', relation.relowner = ( + SELECT pool_blocks.relowner + FROM pg_catalog.pg_class pool_blocks + JOIN pg_catalog.pg_namespace pool_namespace + ON pool_namespace.oid = pool_blocks.relnamespace + WHERE pool_namespace.nspname = '{schema}' + AND pool_blocks.relname = 'qbit_pool_blocks' + ) + ) + FROM pg_catalog.pg_class relation + JOIN pg_catalog.pg_namespace namespace + ON namespace.oid = relation.relnamespace + LEFT JOIN pg_catalog.pg_sequence sequence + ON sequence.seqrelid = relation.oid + WHERE namespace.nspname = '{schema}' + AND relation.relname = 'qbit_audit_publication_sequence_seq' + ), + 'indexes', ( + SELECT COALESCE(json_agg(index_row ORDER BY index_row.name), '[]'::json) + FROM ( + SELECT + index_relation.relname AS name, + index_relation.oid, + pg_catalog.pg_relation_filenode(index_relation.oid) AS filenode, + index_relation.relkind AS kind, + index_relation.relpersistence AS persistence, + index_relation.relowner AS owner, + pg_catalog.pg_get_indexdef(index_relation.oid) AS definition, + index_definition.indisunique AS unique, + index_definition.indisvalid AS valid, + index_definition.indisready AS ready, + index_definition.indislive AS live, + index_definition.indimmediate AS immediate, + index_definition.indisprimary AS primary, + index_definition.indisexclusion AS exclusion, + index_definition.indisclustered AS clustered, + index_definition.indisreplident AS replica_identity, + index_definition.indnullsnotdistinct AS nulls_not_distinct, + index_definition.indnkeyatts AS key_count, + index_definition.indnatts AS attribute_count, + index_definition.indkey::text AS keys, + index_definition.indclass::text AS classes, + index_definition.indcollation::text AS collations, + index_definition.indoption::text AS options, + index_definition.indexprs::text AS expressions, + index_definition.indpred::text AS predicate + FROM pg_catalog.pg_index index_definition + JOIN pg_catalog.pg_class index_relation + ON index_relation.oid = index_definition.indexrelid + JOIN pg_catalog.pg_class table_relation + ON table_relation.oid = index_definition.indrelid + JOIN pg_catalog.pg_namespace namespace + ON namespace.oid = table_relation.relnamespace + WHERE namespace.nspname = '{schema}' + AND table_relation.relname = 'qbit_pool_blocks' + AND EXISTS ( + SELECT 1 + FROM unnest(index_definition.indkey) AS key(attnum) + JOIN pg_catalog.pg_attribute attribute + ON attribute.attrelid = table_relation.oid + AND attribute.attnum = key.attnum + WHERE attribute.attname = 'audit_publication_sequence' + ) + ) index_row + ), + 'named_index_relation', ( + SELECT json_build_object( + 'oid', relation.oid, + 'kind', relation.relkind, + 'persistence', relation.relpersistence, + 'owner', relation.relowner, + 'filenode', pg_catalog.pg_relation_filenode(relation.oid) + ) + FROM pg_catalog.pg_class relation + JOIN pg_catalog.pg_namespace namespace + ON namespace.oid = relation.relnamespace + WHERE namespace.nspname = '{schema}' + AND relation.relname = + 'qbit_pool_blocks_audit_publication_sequence_idx' + ), + 'constraints', ( + SELECT COALESCE( + json_agg(constraint_row ORDER BY constraint_row.name), + '[]'::json + ) + FROM ( + SELECT + constraint_definition.conname AS name, + constraint_definition.oid, + constraint_definition.contype AS type, + constraint_definition.conbin::text AS expression_tree, + pg_catalog.pg_get_constraintdef( + constraint_definition.oid, + true + ) AS definition, + constraint_definition.convalidated AS validated, + constraint_definition.conkey::text AS keys, + constraint_definition.conislocal AS local, + constraint_definition.coninhcount AS inherited_count, + constraint_definition.connoinherit AS no_inherit, + constraint_definition.condeferrable AS deferrable, + constraint_definition.condeferred AS deferred + FROM pg_catalog.pg_constraint constraint_definition + JOIN pg_catalog.pg_class table_relation + ON table_relation.oid = constraint_definition.conrelid + JOIN pg_catalog.pg_namespace namespace + ON namespace.oid = table_relation.relnamespace + WHERE namespace.nspname = '{schema}' + AND table_relation.relname = 'qbit_pool_blocks' + AND ( + constraint_definition.conname = + 'qbit_pool_blocks_audit_publication_sequence_check' + OR pg_catalog.pg_get_constraintdef( + constraint_definition.oid, + true + ) LIKE '%audit_publication_sequence%' + ) + ) constraint_row + ) +); +""" + ) + sequence_catalog = snapshot["sequence_catalog"] + if sequence_catalog is None or sequence_catalog["kind"] != "S": + snapshot["sequence_state"] = None + else: + snapshot["sequence_state"] = run_json( + f""" +SELECT json_build_object('last_value', last_value, 'is_called', is_called) +FROM "{schema}".qbit_audit_publication_sequence_seq; +""" + ) + return snapshot + + +def assert_advisory_lock_available(message: str) -> None: + result = run_json( + """ +WITH acquired AS ( + SELECT pg_catalog.pg_try_advisory_lock( + pg_catalog.hashtext('qbit_audit_publication_sequence_migration') + ) AS ok +) +SELECT json_build_object( + 'acquired', (SELECT ok FROM acquired), + 'released', CASE + WHEN (SELECT ok FROM acquired) THEN pg_catalog.pg_advisory_unlock( + pg_catalog.hashtext('qbit_audit_publication_sequence_migration') + ) + ELSE false + END +); +""" + ) + assert_equal( + result, + {"acquired": True, "released": True}, + message, + ) + + +def initialize_migration_schema( + schema: str, + *, + writer_id: str, +) -> ScopedPsqlLedger: + return ScopedPsqlLedger( + test_schema=schema, + writer_id=writer_id, + writer_epoch=1, + initialize_schema=True, + ) + + +def assert_valid_migration_case( + *, + schema: str, + label: str, + expected_ordinals: dict[str, int | None], + expected_next_ordinal: int | None, + expected_sequence_state: dict[str, object] | None = None, +) -> dict[str, object]: + if not _schema_registered(schema): + raise GateFailure(f"{label} requires an explicit registered schema") + ledger = initialize_migration_schema(schema, writer_id=f"a1-{label}-first") + try: + first = migration_snapshot(schema) + rows = { + str(row["block_hash"]): row.get("audit_publication_sequence") + for row in first["rows"] # type: ignore[union-attr] + } + for block_hash, expected in expected_ordinals.items(): + assert_equal(rows[block_hash], expected, f"{label} ordinal {block_hash}") + if expected_sequence_state is not None: + assert_equal( + first["sequence_state"], + expected_sequence_state, + f"{label} exact allocator state", + ) + assert_equal(len(first["indexes"]), 1, f"{label} canonical index count") + assert_equal(len(first["constraints"]), 1, f"{label} canonical constraint count") + assert_equal( + first["column"], + { + "attnum": first["column"]["attnum"], # type: ignore[index] + "type_oid": 20, + "nullable": True, + "has_default": False, + "identity": "", + "generated": "", + "collation": 0, + }, + f"{label} exact ordinal column catalog", + ) + sequence_catalog = first["sequence_catalog"] + assert sequence_catalog is not None + assert_equal( + { + key: sequence_catalog[key] # type: ignore[index] + for key in ( + "kind", + "persistence", + "type_oid", + "start", + "increment", + "max", + "min", + "cache", + "cycle", + "owned_dependencies", + "same_owner_as_table", + ) + }, + { + "kind": "S", + "persistence": "p", + "type_oid": 20, + "start": 1, + "increment": 1, + "max": BIGINT_MAX, + "min": 1, + "cache": 1, + "cycle": False, + "owned_dependencies": 0, + "same_owner_as_table": True, + }, + f"{label} exact ordinal sequence catalog", + ) + index = first["indexes"][0] # type: ignore[index] + assert_equal( + { + key: index[key] + for key in ( + "name", + "kind", + "persistence", + "unique", + "valid", + "ready", + "live", + "immediate", + "primary", + "exclusion", + "clustered", + "replica_identity", + "nulls_not_distinct", + "key_count", + "attribute_count", + "collations", + "options", + "expressions", + "predicate", + ) + }, + { + "name": "qbit_pool_blocks_audit_publication_sequence_idx", + "kind": "i", + "persistence": "p", + "unique": True, + "valid": True, + "ready": True, + "live": True, + "immediate": True, + "primary": False, + "exclusion": False, + "clustered": False, + "replica_identity": False, + "nulls_not_distinct": False, + "key_count": 1, + "attribute_count": 1, + "collations": "0", + "options": "0", + "expressions": None, + "predicate": None, + }, + f"{label} exact ordinal index catalog", + ) + assert_equal( + index["definition"], + "CREATE UNIQUE INDEX " + "qbit_pool_blocks_audit_publication_sequence_idx ON " + f"{schema}.qbit_pool_blocks USING btree " + "(audit_publication_sequence)", + f"{label} exact ordinal index definition", + ) + constraint = first["constraints"][0] # type: ignore[index] + assert_equal( + { + key: constraint[key] + for key in ( + "name", + "type", + "definition", + "validated", + "local", + "inherited_count", + "no_inherit", + "deferrable", + "deferred", + ) + }, + { + "name": "qbit_pool_blocks_audit_publication_sequence_check", + "type": "c", + "definition": "CHECK ((audit_publication_sequence IS NULL OR " + "audit_publication_sequence > 0) AND (chain_state <> " + "'confirmed'::text OR audit_publication_sequence IS NOT NULL))", + "validated": True, + "local": True, + "inherited_count": 0, + "no_inherit": False, + "deferrable": False, + "deferred": False, + }, + f"{label} exact ordinal constraint catalog", + ) + finally: + ledger.release_writer_lease() + ledger.close() + for rerun_round in (1, 2): + rerun = initialize_migration_schema( + schema, + writer_id=f"a1-{label}-rerun-{rerun_round}", + ) + try: + assert_equal( + migration_snapshot(schema), + first, + f"{label} idempotent catalog rerun {rerun_round}", + ) + finally: + rerun.release_writer_lease() + rerun.close() + legacy_inactive_hash = "85" * 32 + legacy_inactive_sequence = expected_ordinals.get(legacy_inactive_hash) + if legacy_inactive_sequence is not None: + reactivator = ScopedPsqlLedger( + test_schema=schema, + writer_id=f"a1-{label}-legacy-reactivation", + writer_epoch=1, + ) + try: + reactivated = reactivator.reactivate_pool_block( + block_hash=legacy_inactive_hash, + active_tip_height=10, + ) + assert_equal( + reactivated["audit_publication_sequence"], + legacy_inactive_sequence, + f"{label} legacy inactive reactivation preserves backfilled ordinal", + ) + finally: + reactivator.release_writer_lease() + reactivator.close() + if expected_next_ordinal is not None: + final = ScopedPsqlLedger( + test_schema=schema, + writer_id=f"a1-{label}-next", + writer_epoch=1, + ) + try: + block_hash = hashlib.sha256(f"{label}-next".encode()).hexdigest() + final._run_sql( + f""" +INSERT INTO qbit_pool_blocks ( + block_hash, block_height, parent_hash, coinbase_txid, + payout_manifest_sha256, chain_state, maturity_state +) VALUES ( + '{block_hash}', 99, '{'10' * 32}', '{'20' * 32}', + '{'30' * 32}', 'prepared', 'immature' +); +""" + ) + confirmation = final.confirm_accepted_block( + block_hash=block_hash, + active_tip_height=99, + ) + assert_equal( + confirmation["audit_publication_sequence"], + expected_next_ordinal, + f"{label} next production ordinal", + ) + finally: + final.release_writer_lease() + final.close() + assert_advisory_lock_available(f"{label} advisory lock released") + return first + + +def insert_migration_row( + schema: str, + *, + block_hash: str, + ordinal: int | None, + found_at: str, + chain_state: str = "confirmed", +) -> None: + ordinal_sql = "NULL" if ordinal is None else str(ordinal) + run_psql( + f""" +INSERT INTO qbit_pool_blocks ( + block_hash, + audit_publication_sequence, + block_height, + parent_hash, + coinbase_txid, + payout_manifest_sha256, + found_at, + chain_state, + maturity_state +) VALUES ( + '{block_hash}', + {ordinal_sql}, + 10, + '{'10' * 32}', + '{'20' * 32}', + '{'30' * 32}', + '{found_at}', + '{chain_state}', + 'immature' +); +""", + schema=schema, + ) + + +def assert_initializer_rejects_unchanged( + schema: str, + *, + label: str, + error_fragment: str, +) -> dict[str, object]: + before = migration_snapshot(schema) + try: + initialize_migration_schema(schema, writer_id=f"a1-{label}-reject") + except GateFailure as error: + if error_fragment not in str(error): + raise GateFailure( + f"{label} wrong failure: expected {error_fragment!r}, got {error!r}" + ) from error + else: + raise GateFailure(f"{label} initializer unexpectedly succeeded") + assert_equal( + migration_snapshot(schema), + before, + f"{label} transactional rollback snapshot", + ) + assert_advisory_lock_available(f"{label} rejecting advisory lock released") + return before + + +def create_canonical_ordinal_constraint(schema: str, *, not_valid: bool = False) -> None: + run_psql( + """ +ALTER TABLE qbit_pool_blocks +ADD CONSTRAINT qbit_pool_blocks_audit_publication_sequence_check +CHECK ( + (audit_publication_sequence IS NULL OR audit_publication_sequence > 0) + AND (chain_state <> 'confirmed' OR audit_publication_sequence IS NOT NULL) +) +""" + + (" NOT VALID;" if not_valid else ";"), + schema=schema, + ) + + +def test_m0_m11_migration_matrix() -> None: + other_null_expected = { + MIGRATION_HASH_OTHER: None, + "85" * 32: 4, + "86" * 32: None, + "87" * 32: None, + } + common_null_expected = { + MIGRATION_HASH_A: 1, + MIGRATION_HASH_B: 2, + MIGRATION_HASH_C: 3, + **other_null_expected, + } + + m0 = create_owned_schema("m0") + run_psql(LEGACY_POOL_BLOCKS_SQL, schema=m0) + seed_migration_rows(m0, ordinals=None) + assert_valid_migration_case( + schema=m0, + label="m0", + expected_ordinals=common_null_expected, + expected_next_ordinal=5, + ) + + m1 = create_owned_schema("m1") + run_psql(LEGACY_POOL_BLOCKS_SQL, schema=m1) + add_ordinal_column(m1) + seed_migration_rows(m1, ordinals=(None, None, None)) + assert_valid_migration_case( + schema=m1, + label="m1", + expected_ordinals=common_null_expected, + expected_next_ordinal=5, + ) + + m2 = create_owned_schema("m2") + run_psql(LEGACY_POOL_BLOCKS_SQL, schema=m2) + create_ordinal_sequence(m2) + seed_migration_rows(m2, ordinals=None) + assert_valid_migration_case( + schema=m2, + label="m2", + expected_ordinals=common_null_expected, + expected_next_ordinal=5, + ) + + m3 = create_owned_schema("m3") + run_psql(LEGACY_POOL_BLOCKS_SQL, schema=m3) + add_ordinal_column(m3) + create_ordinal_sequence(m3) + seed_migration_rows(m3, ordinals=(None, None, None)) + assert_valid_migration_case( + schema=m3, + label="m3", + expected_ordinals=common_null_expected, + expected_next_ordinal=5, + ) + + m4 = create_owned_schema("m4") + run_psql(LEGACY_POOL_BLOCKS_SQL, schema=m4) + add_ordinal_column(m4) + create_ordinal_sequence(m4) + seed_migration_rows(m4, ordinals=(None, None, None)) + run_psql( + "CREATE UNIQUE INDEX " + "qbit_pool_blocks_audit_publication_sequence_idx " + "ON qbit_pool_blocks (audit_publication_sequence);", + schema=m4, + ) + m4_before = migration_snapshot(m4) + m4_first = assert_valid_migration_case( + schema=m4, + label="m4", + expected_ordinals=common_null_expected, + expected_next_ordinal=5, + ) + assert_equal( + m4_first["indexes"], + m4_before["indexes"], + "M4 first migration preserves correct index OID/filenode/catalog", + ) + assert_equal( + m4_first["named_index_relation"], + m4_before["named_index_relation"], + "M4 first migration preserves correct named index relation", + ) + + m5 = create_owned_schema("m5") + run_psql(LEGACY_POOL_BLOCKS_SQL, schema=m5) + add_ordinal_column(m5) + create_ordinal_sequence(m5, last_value=3, is_called=True) + seed_migration_rows(m5, ordinals=(1, 2, 3)) + create_canonical_ordinal_constraint(m5) + m5_before = migration_snapshot(m5) + m5_first = assert_valid_migration_case( + schema=m5, + label="m5", + expected_ordinals={ + MIGRATION_HASH_A: 1, + MIGRATION_HASH_B: 2, + MIGRATION_HASH_C: 3, + **other_null_expected, + }, + expected_next_ordinal=5, + ) + assert_equal( + m5_first["constraints"], + m5_before["constraints"], + "M5 first migration preserves correct constraint OID/conbin/catalog", + ) + + m6 = create_owned_schema("m6") + run_psql(LEGACY_POOL_BLOCKS_SQL, schema=m6) + add_ordinal_column(m6) + create_ordinal_sequence(m6) + seed_migration_rows(m6, ordinals=(9002, None, None)) + assert_valid_migration_case( + schema=m6, + label="m6", + expected_ordinals={ + MIGRATION_HASH_A: 9002, + MIGRATION_HASH_B: 9003, + MIGRATION_HASH_C: 9004, + MIGRATION_HASH_OTHER: None, + "85" * 32: 9005, + "86" * 32: None, + "87" * 32: None, + }, + expected_next_ordinal=9006, + expected_sequence_state={"last_value": 9005, "is_called": True}, + ) + + for label, called, inactive_ordinal, expected_next in ( + ("m7a", True, 12, 13), + ("m8a", False, 11, 12), + ): + schema = create_owned_schema(label) + run_psql(LEGACY_POOL_BLOCKS_SQL, schema=schema) + add_ordinal_column(schema) + create_ordinal_sequence(schema, last_value=11, is_called=called) + seed_migration_rows(schema, ordinals=(5, 6, 7)) + assert_valid_migration_case( + schema=schema, + label=label, + expected_ordinals={ + MIGRATION_HASH_A: 5, + MIGRATION_HASH_B: 6, + MIGRATION_HASH_C: 7, + MIGRATION_HASH_OTHER: None, + "85" * 32: inactive_ordinal, + "86" * 32: None, + "87" * 32: None, + }, + expected_next_ordinal=expected_next, + expected_sequence_state={ + "last_value": inactive_ordinal, + "is_called": True, + }, + ) + + for label, called, expected, inactive_ordinal, expected_next in ( + ("m7b", True, (12, 13, 14), 15, 16), + ("m8b", False, (11, 12, 13), 14, 15), + ): + schema = create_owned_schema(label) + run_psql(LEGACY_POOL_BLOCKS_SQL, schema=schema) + add_ordinal_column(schema) + create_ordinal_sequence(schema, last_value=11, is_called=called) + seed_migration_rows(schema, ordinals=(None, None, None)) + run_psql( + f""" +UPDATE qbit_pool_blocks +SET audit_publication_sequence = 7 +WHERE block_hash = '{MIGRATION_HASH_OTHER}'; +""", + schema=schema, + ) + assert_valid_migration_case( + schema=schema, + label=label, + expected_ordinals={ + MIGRATION_HASH_A: expected[0], + MIGRATION_HASH_B: expected[1], + MIGRATION_HASH_C: expected[2], + MIGRATION_HASH_OTHER: 7, + "85" * 32: inactive_ordinal, + "86" * 32: None, + "87" * 32: None, + }, + expected_next_ordinal=expected_next, + expected_sequence_state={ + "last_value": inactive_ordinal, + "is_called": True, + }, + ) + + m9 = create_owned_schema("m9") + run_psql(LEGACY_POOL_BLOCKS_SQL, schema=m9) + add_ordinal_column(m9) + seed_migration_rows(m9, ordinals=(0, None, None)) + assert_initializer_rejects_unchanged( + m9, + label="m9", + error_fragment="invalid non-positive audit publication sequence", + ) + + m10 = create_owned_schema("m10") + run_psql(LEGACY_POOL_BLOCKS_SQL, schema=m10) + add_ordinal_column(m10) + seed_migration_rows(m10, ordinals=(4, 4, None)) + assert_initializer_rejects_unchanged( + m10, + label="m10", + error_fragment="duplicate audit publication sequence", + ) + + m11 = create_owned_schema("m11") + run_psql(LEGACY_POOL_BLOCKS_SQL, schema=m11) + create_ordinal_sequence(m11, last_value=19, is_called=True) + seed_migration_rows(m11, ordinals=None) + run_psql( + """ +ALTER TABLE qbit_pool_blocks +ADD CONSTRAINT qbit_pool_blocks_audit_publication_sequence_check +CHECK (block_height < 1000); +""", + schema=m11, + ) + assert_initializer_rejects_unchanged( + m11, + label="m11_wrong_constraint", + error_fragment="invalid audit publication sequence constraint definition", + ) + run_psql( + "ALTER TABLE qbit_pool_blocks DROP CONSTRAINT " + "qbit_pool_blocks_audit_publication_sequence_check;", + schema=m11, + ) + assert_valid_migration_case( + schema=m11, + label="m11", + expected_ordinals={ + MIGRATION_HASH_A: 20, + MIGRATION_HASH_B: 21, + MIGRATION_HASH_C: 22, + "85" * 32: 23, + }, + expected_next_ordinal=24, + ) + + m11_alt = create_owned_schema("m11_alt") + run_psql(LEGACY_POOL_BLOCKS_SQL, schema=m11_alt) + add_ordinal_column(m11_alt) + create_ordinal_sequence(m11_alt, last_value=3, is_called=True) + seed_migration_rows(m11_alt, ordinals=(1, 2, 3)) + run_psql( + "CREATE UNIQUE INDEX qbit_audit_publication_sequence_alternate " + "ON qbit_pool_blocks (audit_publication_sequence);", + schema=m11_alt, + ) + assert_initializer_rejects_unchanged( + m11_alt, + label="m11_alternate_index", + error_fragment="duplicate audit publication sequence index definition", + ) + run_psql( + "DROP INDEX qbit_audit_publication_sequence_alternate;", + schema=m11_alt, + ) + assert_valid_migration_case( + schema=m11_alt, + label="m11_alt", + expected_ordinals={ + MIGRATION_HASH_A: 1, + MIGRATION_HASH_B: 2, + MIGRATION_HASH_C: 3, + "85" * 32: 4, + }, + expected_next_ordinal=5, + ) + + m11_named = create_owned_schema("m11_named") + run_psql(LEGACY_POOL_BLOCKS_SQL, schema=m11_named) + add_ordinal_column(m11_named) + create_ordinal_sequence(m11_named, last_value=3, is_called=True) + seed_migration_rows(m11_named, ordinals=(1, 2, 3)) + run_psql( + "CREATE INDEX qbit_pool_blocks_audit_publication_sequence_idx " + "ON qbit_pool_blocks (audit_publication_sequence);", + schema=m11_named, + ) + assert_initializer_rejects_unchanged( + m11_named, + label="m11_wrong_named_index", + error_fragment="invalid audit publication sequence index definition", + ) + run_psql( + "DROP INDEX qbit_pool_blocks_audit_publication_sequence_idx;", + schema=m11_named, + ) + assert_valid_migration_case( + schema=m11_named, + label="m11_named", + expected_ordinals={ + MIGRATION_HASH_A: 1, + MIGRATION_HASH_B: 2, + MIGRATION_HASH_C: 3, + "85" * 32: 4, + }, + expected_next_ordinal=5, + ) + + m11_sequence = create_owned_schema("m11_sequence") + run_psql(LEGACY_POOL_BLOCKS_SQL, schema=m11_sequence) + add_ordinal_column(m11_sequence) + run_psql( + "CREATE SEQUENCE qbit_audit_publication_sequence_seq AS integer;", + schema=m11_sequence, + ) + seed_migration_rows(m11_sequence, ordinals=(None, None, None)) + assert_initializer_rejects_unchanged( + m11_sequence, + label="m11_wrong_sequence", + error_fragment="invalid audit publication sequence definition", + ) + run_psql( + "DROP SEQUENCE qbit_audit_publication_sequence_seq;", + schema=m11_sequence, + ) + create_ordinal_sequence(m11_sequence) + assert_valid_migration_case( + schema=m11_sequence, + label="m11_sequence", + expected_ordinals=common_null_expected, + expected_next_ordinal=5, + ) + + m11_column = create_owned_schema("m11_column") + run_psql(LEGACY_POOL_BLOCKS_SQL, schema=m11_column) + run_psql( + "ALTER TABLE qbit_pool_blocks " + "ADD COLUMN audit_publication_sequence integer;", + schema=m11_column, + ) + seed_migration_rows(m11_column, ordinals=(None, None, None)) + assert_initializer_rejects_unchanged( + m11_column, + label="m11_wrong_column", + error_fragment="invalid audit publication sequence column definition", + ) + run_psql( + "ALTER TABLE qbit_pool_blocks ALTER COLUMN " + "audit_publication_sequence TYPE bigint;", + schema=m11_column, + ) + assert_valid_migration_case( + schema=m11_column, + label="m11_column", + expected_ordinals=common_null_expected, + expected_next_ordinal=5, + ) + + m11_not_valid = create_owned_schema("m11_not_valid") + run_psql(LEGACY_POOL_BLOCKS_SQL, schema=m11_not_valid) + add_ordinal_column(m11_not_valid) + create_ordinal_sequence(m11_not_valid, last_value=3, is_called=True) + seed_migration_rows(m11_not_valid, ordinals=(1, 2, 3)) + create_canonical_ordinal_constraint(m11_not_valid, not_valid=True) + before_not_valid = migration_snapshot(m11_not_valid) + not_valid_constraint = before_not_valid["constraints"][0] # type: ignore[index] + assert_equal( + not_valid_constraint["validated"], + False, + "M11 preexisting canonical constraint starts NOT VALID", + ) + normalized = assert_valid_migration_case( + schema=m11_not_valid, + label="m11_not_valid", + expected_ordinals={ + MIGRATION_HASH_A: 1, + MIGRATION_HASH_B: 2, + MIGRATION_HASH_C: 3, + **other_null_expected, + }, + expected_next_ordinal=5, + ) + assert_equal( + normalized["constraints"][0]["oid"], # type: ignore[index] + not_valid_constraint["oid"], + "M11 NOT VALID normalization preserves constraint OID", + ) + + m11_alt_constraint = create_owned_schema("m11_alt_constraint") + run_psql(LEGACY_POOL_BLOCKS_SQL, schema=m11_alt_constraint) + add_ordinal_column(m11_alt_constraint) + create_ordinal_sequence(m11_alt_constraint, last_value=3, is_called=True) + seed_migration_rows(m11_alt_constraint, ordinals=(1, 2, 3)) + run_psql( + """ +ALTER TABLE qbit_pool_blocks +ADD CONSTRAINT qbit_audit_publication_sequence_alternate_check +CHECK ( + (audit_publication_sequence IS NULL OR audit_publication_sequence > 0) + AND (chain_state <> 'confirmed' OR audit_publication_sequence IS NOT NULL) +); +""", + schema=m11_alt_constraint, + ) + assert_initializer_rejects_unchanged( + m11_alt_constraint, + label="m11_alternate_constraint", + error_fragment="duplicate audit publication sequence constraint definition", + ) + run_psql( + "ALTER TABLE qbit_pool_blocks DROP CONSTRAINT " + "qbit_audit_publication_sequence_alternate_check;", + schema=m11_alt_constraint, + ) + assert_valid_migration_case( + schema=m11_alt_constraint, + label="m11_alt_constraint", + expected_ordinals={ + MIGRATION_HASH_A: 1, + MIGRATION_HASH_B: 2, + MIGRATION_HASH_C: 3, + **other_null_expected, + }, + expected_next_ordinal=5, + ) + + m11_relkind = create_owned_schema("m11_relkind") + run_psql(LEGACY_POOL_BLOCKS_SQL, schema=m11_relkind) + add_ordinal_column(m11_relkind) + create_ordinal_sequence(m11_relkind, last_value=3, is_called=True) + seed_migration_rows(m11_relkind, ordinals=(1, 2, 3)) + run_psql( + "CREATE TABLE qbit_pool_blocks_audit_publication_sequence_idx " + "(sentinel integer);", + schema=m11_relkind, + ) + assert_initializer_rejects_unchanged( + m11_relkind, + label="m11_wrong_index_relkind", + error_fragment="invalid audit publication sequence index definition", + ) + run_psql( + "DROP TABLE qbit_pool_blocks_audit_publication_sequence_idx;", + schema=m11_relkind, + ) + assert_valid_migration_case( + schema=m11_relkind, + label="m11_relkind", + expected_ordinals={ + MIGRATION_HASH_A: 1, + MIGRATION_HASH_B: 2, + MIGRATION_HASH_C: 3, + **other_null_expected, + }, + expected_next_ordinal=5, + ) + + +def prepare_ordinal_migration_schema( + label: str, + *, + sequence_last: int, + sequence_called: bool, +) -> str: + schema = create_owned_schema(label) + run_psql(LEGACY_POOL_BLOCKS_SQL, schema=schema) + add_ordinal_column(schema) + create_ordinal_sequence( + schema, + last_value=sequence_last, + is_called=sequence_called, + ) + return schema + + +def assert_next_allocation_exhausted(schema: str, *, label: str) -> None: + block_hash = hashlib.sha256(f"{label}-exhausted".encode()).hexdigest() + run_psql( + f""" +INSERT INTO qbit_pool_blocks ( + block_hash, block_height, parent_hash, coinbase_txid, + payout_manifest_sha256, chain_state, maturity_state +) VALUES ( + '{block_hash}', 101, '{'10' * 32}', '{'20' * 32}', + '{'30' * 32}', 'prepared', 'immature' +); +""", + schema=schema, + ) + ledger = ScopedPsqlLedger( + test_schema=schema, + writer_id=f"a1-{label}-exhausted", + writer_epoch=1, + ) + try: + before = migration_snapshot(schema) + try: + ledger.confirm_accepted_block( + block_hash=block_hash, + active_tip_height=101, + ) + except GateFailure as error: + if "reached maximum value" not in str(error): + raise GateFailure( + f"{label} wrong allocator exhaustion: {error!r}" + ) from error + else: + raise GateFailure(f"{label} exhausted allocator unexpectedly advanced") + assert_equal( + migration_snapshot(schema), + before, + f"{label} exhaustion leaves rows/catalog/allocator unchanged", + ) + finally: + ledger.release_writer_lease() + ledger.close() + + +def test_migration_bigint_boundaries() -> None: + max_called = prepare_ordinal_migration_schema( + "bound_max_called", + sequence_last=BIGINT_MAX, + sequence_called=True, + ) + for offset, block_hash in zip( + (-2, -1, 0), + (MIGRATION_HASH_A, MIGRATION_HASH_B, MIGRATION_HASH_C), + strict=True, + ): + insert_migration_row( + max_called, + block_hash=block_hash, + ordinal=BIGINT_MAX + offset, + found_at=f"2020-01-01T00:00:0{offset + 3}Z", + ) + assert_valid_migration_case( + schema=max_called, + label="bound_max_called", + expected_ordinals={ + MIGRATION_HASH_A: BIGINT_MAX - 2, + MIGRATION_HASH_B: BIGINT_MAX - 1, + MIGRATION_HASH_C: BIGINT_MAX, + }, + expected_next_ordinal=None, + expected_sequence_state={"last_value": BIGINT_MAX, "is_called": True}, + ) + assert_next_allocation_exhausted( + max_called, + label="bound_max_called", + ) + + max_uncalled_one = prepare_ordinal_migration_schema( + "bound_max_one", + sequence_last=BIGINT_MAX, + sequence_called=False, + ) + insert_migration_row( + max_uncalled_one, + block_hash=MIGRATION_HASH_A, + ordinal=None, + found_at="2020-01-01T00:00:01Z", + ) + assert_valid_migration_case( + schema=max_uncalled_one, + label="bound_max_one", + expected_ordinals={MIGRATION_HASH_A: BIGINT_MAX}, + expected_next_ordinal=None, + expected_sequence_state={"last_value": BIGINT_MAX, "is_called": True}, + ) + assert_next_allocation_exhausted( + max_uncalled_one, + label="bound_max_one", + ) + + max_uncalled_two = prepare_ordinal_migration_schema( + "bound_max_two", + sequence_last=BIGINT_MAX, + sequence_called=False, + ) + for block_hash, found_at in ( + (MIGRATION_HASH_A, "2020-01-01T00:00:01Z"), + (MIGRATION_HASH_B, "2020-01-01T00:00:02Z"), + ): + insert_migration_row( + max_uncalled_two, + block_hash=block_hash, + ordinal=None, + found_at=found_at, + ) + assert_initializer_rejects_unchanged( + max_uncalled_two, + label="bound_max_two", + error_fragment="audit publication sequence exhausted", + ) + + durable_max_pending = prepare_ordinal_migration_schema( + "bound_durable_max", + sequence_last=1, + sequence_called=False, + ) + insert_migration_row( + durable_max_pending, + block_hash=MIGRATION_HASH_A, + ordinal=BIGINT_MAX, + found_at="2020-01-01T00:00:01Z", + ) + insert_migration_row( + durable_max_pending, + block_hash=MIGRATION_HASH_B, + ordinal=None, + found_at="2020-01-01T00:00:02Z", + ) + assert_initializer_rejects_unchanged( + durable_max_pending, + label="bound_durable_max", + error_fragment="audit publication sequence exhausted", + ) + + near_max_three = prepare_ordinal_migration_schema( + "bound_near_three", + sequence_last=BIGINT_MAX - 2, + sequence_called=False, + ) + for block_hash, second in ( + (MIGRATION_HASH_C, 2), + (MIGRATION_HASH_B, 1), + (MIGRATION_HASH_A, 1), + ): + insert_migration_row( + near_max_three, + block_hash=block_hash, + ordinal=None, + found_at=f"2020-01-01T00:00:0{second}Z", + ) + assert_valid_migration_case( + schema=near_max_three, + label="bound_near_three", + expected_ordinals={ + MIGRATION_HASH_A: BIGINT_MAX - 2, + MIGRATION_HASH_B: BIGINT_MAX - 1, + MIGRATION_HASH_C: BIGINT_MAX, + }, + expected_next_ordinal=None, + expected_sequence_state={"last_value": BIGINT_MAX, "is_called": True}, + ) + + near_max_four = prepare_ordinal_migration_schema( + "bound_near_four", + sequence_last=BIGINT_MAX - 2, + sequence_called=False, + ) + for index, block_hash in enumerate( + (MIGRATION_HASH_A, MIGRATION_HASH_B, MIGRATION_HASH_C, "88" * 32), + start=1, + ): + insert_migration_row( + near_max_four, + block_hash=block_hash, + ordinal=None, + found_at=f"2020-01-01T00:00:0{index}Z", + ) + assert_initializer_rejects_unchanged( + near_max_four, + label="bound_near_four", + error_fragment="audit publication sequence exhausted", + ) + + +def test_invalid_sequence_and_column_definitions() -> None: + sequence_definitions = { + "seq_start": "CREATE SEQUENCE qbit_audit_publication_sequence_seq AS bigint START WITH 2", + "seq_increment": "CREATE SEQUENCE qbit_audit_publication_sequence_seq AS bigint INCREMENT BY 2", + "seq_min": "CREATE SEQUENCE qbit_audit_publication_sequence_seq AS bigint MINVALUE 2 START WITH 2", + "seq_max": "CREATE SEQUENCE qbit_audit_publication_sequence_seq AS bigint MAXVALUE 100", + "seq_cache": "CREATE SEQUENCE qbit_audit_publication_sequence_seq AS bigint CACHE 2", + "seq_cycle": "CREATE SEQUENCE qbit_audit_publication_sequence_seq AS bigint CYCLE", + "seq_unlogged": "CREATE UNLOGGED SEQUENCE qbit_audit_publication_sequence_seq AS bigint", + } + for label, definition in sequence_definitions.items(): + schema = create_owned_schema(label) + run_psql(LEGACY_POOL_BLOCKS_SQL, schema=schema) + add_ordinal_column(schema) + run_psql(definition + ";", schema=schema) + seed_migration_rows( + schema, + ordinals=(1, 2, 3), + include_other_states=False, + ) + assert_initializer_rejects_unchanged( + schema, + label=label, + error_fragment="invalid audit publication sequence definition", + ) + + owned = create_owned_schema("seq_owned") + run_psql(LEGACY_POOL_BLOCKS_SQL, schema=owned) + add_ordinal_column(owned) + create_ordinal_sequence(owned) + run_psql( + "ALTER SEQUENCE qbit_audit_publication_sequence_seq " + "OWNED BY qbit_pool_blocks.audit_publication_sequence;", + schema=owned, + ) + seed_migration_rows(owned, ordinals=(1, 2, 3)) + assert_initializer_rejects_unchanged( + owned, + label="seq_owned", + error_fragment="invalid audit publication sequence definition", + ) + + wrong_kind = create_owned_schema("seq_wrong_kind") + run_psql(LEGACY_POOL_BLOCKS_SQL, schema=wrong_kind) + add_ordinal_column(wrong_kind) + run_psql( + "CREATE TABLE qbit_audit_publication_sequence_seq (sentinel integer);", + schema=wrong_kind, + ) + seed_migration_rows(wrong_kind, ordinals=(1, 2, 3)) + assert_initializer_rejects_unchanged( + wrong_kind, + label="seq_wrong_kind", + error_fragment="invalid audit publication sequence definition", + ) + + column_definitions = { + "column_not_null": "bigint NOT NULL", + "column_default": "bigint DEFAULT 1", + "column_identity": "bigint GENERATED BY DEFAULT AS IDENTITY", + } + for label, definition in column_definitions.items(): + schema = create_owned_schema(label) + run_psql(LEGACY_POOL_BLOCKS_SQL, schema=schema) + run_psql( + "ALTER TABLE qbit_pool_blocks ADD COLUMN " + f"audit_publication_sequence {definition};", + schema=schema, + ) + seed_migration_rows( + schema, + ordinals=(1, 2, 3), + include_other_states=False, + ) + assert_initializer_rejects_unchanged( + schema, + label=label, + error_fragment="invalid audit publication sequence column definition", + ) + + generated = create_owned_schema("column_generated") + run_psql(LEGACY_POOL_BLOCKS_SQL, schema=generated) + seed_migration_rows(generated, ordinals=None) + run_psql( + "ALTER TABLE qbit_pool_blocks ADD COLUMN " + "audit_publication_sequence bigint " + "GENERATED ALWAYS AS (block_height + 1) STORED;", + schema=generated, + ) + assert_initializer_rejects_unchanged( + generated, + label="column_generated", + error_fragment="invalid audit publication sequence column definition", + ) + # Owner mismatch needs CREATE/SET ROLE authority that the supported + # external-psql mode does not promise. Production still validates exact + # same-owner identity, and every portable sequence property is exercised. + + +def _psql_process_command() -> list[str]: + return [ + *BASE_PSQL_ARGV, + "--no-psqlrc", + "--set", + "ON_ERROR_STOP=1", + "--tuples-only", + "--no-align", + "--quiet", + ] + + +def _register_process(process: subprocess.Popen[str]) -> None: + with ACTIVE_CHILDREN_LOCK: + ACTIVE_CHILDREN.add(process) + + +def _forget_process(process: subprocess.Popen[str]) -> None: + with ACTIVE_CHILDREN_LOCK: + ACTIVE_CHILDREN.discard(process) + + +def _wait_for_advisory_holder( + process: subprocess.Popen[str], + *, + application_name: str, + deadline: float, +) -> None: + while time.monotonic() < deadline: + state = run_json( + f""" +SELECT json_build_object( + 'ready', EXISTS ( + SELECT 1 + FROM pg_catalog.pg_stat_activity activity + JOIN pg_catalog.pg_locks lock_state + ON lock_state.pid = activity.pid + WHERE activity.application_name = '{application_name}' + AND lock_state.locktype = 'advisory' + AND lock_state.granted + ) +); +""" + ) + if state["ready"]: + return + if process.poll() is not None: + break + time.sleep(0.05) + raise GateFailure( + f"advisory holder {application_name!r} not observed; exit={process.poll()}" + ) + + +def _wait_file_process( + process: subprocess.Popen[str], + *, + stdout_file: Any, + stderr_file: Any, + expected_success: bool, + error_fragment: str | None, +) -> None: + try: + process.wait(timeout=PSQL_TIMEOUT_SECONDS) + except subprocess.TimeoutExpired as error: + _terminate_and_reap(process) + raise GateFailure("tagged psql worker timed out") from error + finally: + _forget_process(process) + stdout_file.seek(0) + stderr_file.seek(0) + stdout = stdout_file.read(PSQL_OUTPUT_LIMIT_BYTES + 1) + stderr = stderr_file.read(PSQL_OUTPUT_LIMIT_BYTES + 1) + if ( + len(stdout.encode("utf-8")) > PSQL_OUTPUT_LIMIT_BYTES + or len(stderr.encode("utf-8")) > PSQL_OUTPUT_LIMIT_BYTES + ): + raise GateFailure("tagged psql worker output exceeded 1 MiB") + if expected_success and process.returncode != 0: + raise GateFailure( + f"tagged psql worker failed with {process.returncode}: {stderr.strip()}" + ) + if not expected_success: + if process.returncode == 0: + raise GateFailure("rejecting tagged psql worker unexpectedly succeeded") + if error_fragment is not None and error_fragment not in stderr: + raise GateFailure( + f"tagged psql worker wrong failure: {stderr.strip()}" + ) + + +def test_migration_advisory_waiter() -> None: + schema_sql = ( + Path(__file__).resolve().parents[1] + / "crates/qbit-prism/sql/001_share_ledger.sql" + ).read_text(encoding="utf-8") + for outcome, rejecting in (("commit", False), ("rollback", True)): + schema = create_owned_schema(f"migration_wait_{outcome}") + rejection_before: dict[str, object] | None = None + if rejecting: + run_psql(LEGACY_POOL_BLOCKS_SQL, schema=schema) + add_ordinal_column(schema) + seed_migration_rows(schema, ordinals=(0, None, None)) + rejection_before = migration_snapshot(schema) + # PostgreSQL truncates application_name to NAMEDATALEN - 1 bytes. + # Keep the tags below that limit so catalog coordination uses the + # exact values the clients set. + holder_tag = f"a1_mig_holder_{RUN_TOKEN}_{outcome}" + waiter_tag = f"a1_mig_waiter_{RUN_TOKEN}_{outcome}" + holder_input = tempfile.TemporaryFile(mode="w+t", encoding="utf-8") + holder_stderr = tempfile.TemporaryFile(mode="w+t", encoding="utf-8") + waiter_input = tempfile.TemporaryFile(mode="w+t", encoding="utf-8") + waiter_stdout = tempfile.TemporaryFile(mode="w+t", encoding="utf-8") + waiter_stderr = tempfile.TemporaryFile(mode="w+t", encoding="utf-8") + holder: subprocess.Popen[str] | None = None + waiter: subprocess.Popen[str] | None = None + try: + holder_input.write( + f""" +SET application_name = '{holder_tag}'; +BEGIN; +SELECT pg_catalog.pg_advisory_xact_lock( + pg_catalog.hashtext('qbit_audit_publication_sequence_migration') +); +DO $qbit_a1_wait$ +DECLARE + waiter_observed boolean := false; +BEGIN + FOR attempt IN 1..600 LOOP + PERFORM pg_catalog.pg_stat_clear_snapshot(); + SELECT EXISTS ( + SELECT 1 + FROM pg_catalog.pg_stat_activity waiter + JOIN pg_catalog.pg_locks lock_state + ON lock_state.pid = waiter.pid + WHERE waiter.application_name = '{waiter_tag}' + AND lock_state.locktype = 'advisory' + AND NOT lock_state.granted + AND pg_catalog.pg_backend_pid() = ANY( + pg_catalog.pg_blocking_pids(waiter.pid) + ) + ) + INTO waiter_observed; + EXIT WHEN waiter_observed; + PERFORM pg_catalog.pg_sleep(0.05); + END LOOP; + IF NOT waiter_observed THEN + RAISE EXCEPTION 'migration advisory waiter not observed'; + END IF; +END; +$qbit_a1_wait$; +{outcome.upper()}; +""" + ) + holder_input.seek(0) + holder = subprocess.Popen( + _psql_process_command(), + stdin=holder_input, + stdout=subprocess.DEVNULL, + stderr=holder_stderr, + text=True, + bufsize=1, + start_new_session=True, + ) + _register_process(holder) + _wait_for_advisory_holder( + holder, + application_name=holder_tag, + deadline=time.monotonic() + PSQL_TIMEOUT_SECONDS, + ) + + waiter_input.write( + f""" +SET application_name = '{waiter_tag}'; +SET statement_timeout = '20s'; +SET lock_timeout = '20s'; +SET search_path TO "{schema}", pg_catalog; +{schema_sql} +""" + ) + waiter_input.seek(0) + waiter = subprocess.Popen( + _psql_process_command(), + stdin=waiter_input, + stdout=waiter_stdout, + stderr=waiter_stderr, + text=True, + start_new_session=True, + ) + _register_process(waiter) + try: + holder.wait(timeout=PSQL_TIMEOUT_SECONDS) + except subprocess.TimeoutExpired as error: + _terminate_and_reap(holder) + raise GateFailure("migration advisory holder timed out") from error + finally: + _forget_process(holder) + if holder.returncode != 0: + holder_stderr.seek(0) + raise GateFailure( + "migration advisory holder failed: " + + holder_stderr.read(PSQL_OUTPUT_LIMIT_BYTES + 1).strip() + ) + + _wait_file_process( + waiter, + stdout_file=waiter_stdout, + stderr_file=waiter_stderr, + expected_success=not rejecting, + error_fragment=( + "invalid non-positive audit publication sequence" + if rejecting + else None + ), + ) + if rejecting: + assert rejection_before is not None + assert_equal( + migration_snapshot(schema), + rejection_before, + "rejecting advisory waiter exact rollback", + ) + else: + success_snapshot = migration_snapshot(schema) + assert_equal( + success_snapshot["sequence_state"], + {"last_value": 1, "is_called": False}, + "successful advisory waiter empty allocator", + ) + assert_advisory_lock_available( + f"migration {outcome} advisory lock no leak" + ) + remaining = run_json( + f""" +SELECT json_build_object('count', count(*)) +FROM pg_catalog.pg_stat_activity +WHERE application_name IN ('{holder_tag}', '{waiter_tag}'); +""" + )["count"] + assert_equal( + remaining, + 0, + f"migration {outcome} tagged backend cleanup", + ) + finally: + for process in (waiter, holder): + if process is not None: + if process.poll() is None: + _terminate_and_reap(process) + _forget_process(process) + for stream in (process.stdin, process.stdout, process.stderr): + if stream is not None and not stream.closed: + stream.close() + for stream in ( + holder_input, + holder_stderr, + waiter_input, + waiter_stdout, + waiter_stderr, + ): + if not stream.closed: + stream.close() + + +def main() -> None: + public_before = support.public_sentinel() + failure: BaseException | None = None + try: + test_m0_m11_migration_matrix() + test_migration_bigint_boundaries() + test_invalid_sequence_and_column_definitions() + test_migration_advisory_waiter() + except BaseException as error: + failure = error + try: + support.cleanup_active_children() + support.cleanup_owned_schemas() + support.assert_equal(support.marker_schema_count(), 0, "migration marker cleanup") + support.assert_equal(support.public_sentinel(), public_before, "migration public preservation") + except BaseException as cleanup_error: + if failure is None: + raise + raise GateFailure( + f"migration scenario failed with {failure!r}; cleanup also failed with {cleanup_error!r}" + ) from cleanup_error + else: + support.atexit.unregister(support.cleanup_active_children) + support.atexit.unregister(support.cleanup_owned_schemas) + if failure is not None: + raise failure + print( + "prism postgres A1 migration gate PASS " + "M0-M11 bigint-bounds invalid-definitions migration-advisory-waiter" + ) + + +if __name__ == "__main__": + main() diff --git a/tests/prism_postgres_a1_process_gate.py b/tests/prism_postgres_a1_process_gate.py new file mode 100644 index 000000000..9ff853653 --- /dev/null +++ b/tests/prism_postgres_a1_process_gate.py @@ -0,0 +1,2624 @@ +"""Non-discovered PostgreSQL/A1 cross-process integration gate.""" + +from __future__ import annotations + +from contextlib import ExitStack +import fcntl +import hashlib +import json +import os +from pathlib import Path +import re +import select +import subprocess +import sys +import tempfile +import time +import traceback +from typing import Any + +from lab.prism import audit_artifacts as audit_artifacts_module +from lab.prism.audit_artifacts import ( + AuditArtifactConfig, + AuditArtifactStore, + AuditPublicationIdentity, +) +from lab.prism.share_ledger import PsqlShareLedger +from tests import prism_postgres_a1_gate as support +from tests import prism_postgres_a1_migration_gate as migration + + +_PROCESS_GATE_MODULE = "tests.prism_postgres_a1_process_gate" +_C2_DIGEST = "11" * 32 + + +class _WorkerScopedPsqlLedger(PsqlShareLedger): + """Child-process ledger that consumes, but never owns, the parent schema.""" + + def __init__( + self, + *, + test_schema: str, + application_name: str, + **kwargs: object, + ) -> None: + if support.SCHEMA_PATTERN.fullmatch(test_schema) is None: + raise support.GateFailure(f"invalid worker schema: {test_schema!r}") + if ( + re.fullmatch(r"qbit_a1_c[23]_[a-z0-9_]+", application_name) is None + or len(application_name) > 63 + ): + raise support.GateFailure( + f"invalid process worker application name: {application_name!r}" + ) + self._worker_schema = test_schema + self._application_name = application_name + kwargs["psql_command"] = support.BASE_PSQL_COMMAND + kwargs["native_client_mode"] = "psql" + super().__init__(**kwargs) # type: ignore[arg-type] + + def _run_sql(self, sql: str) -> str: + return support.run_psql( + f"SET application_name = '{self._application_name}';\n" + sql, + schema=self._worker_schema, + ) + + +def _c2_report(*, block_height: int) -> dict[str, object]: + return { + "schema": "qbit.prism.audit-verification-report.v1", + "block_height": block_height, + "audit_bundle_sha256_hex": _C2_DIGEST, + "reward_manifest_sha256_hex": "44" * 32, + "payout_policy_manifest_sha256_hex": "55" * 32, + "prism_audit_commitment_leaf_hex": "66" * 32, + "audit_commitment_root_hex": "77" * 32, + "coinbase_txid": "22" * 32, + "coinbase_wtxid": "88" * 32, + "coinbase_manifest_sha256_hex": "33" * 32, + "coinbase_tx_hex": "00", + "coinbase_value_sats": 1, + "min_output_sats": 1, + "onchain_output_count": 0, + "accrued_account_count": 0, + } + + +def _c2_store(root: Path, evidence_path: Path) -> AuditArtifactStore: + return AuditArtifactStore( + AuditArtifactConfig( + root=root, + evidence_path=evidence_path, + live_bundle_retention=1, + candidate_retention_seconds=60, + share_segment_size=0, + ) + ) + + +def _c2_publish( + store: AuditArtifactStore, + *, + identity: AuditPublicationIdentity, + publication_floor_sequence: int, + created_at: str, +) -> object: + report = _c2_report(block_height=identity.block_height) + verification_identity = store.build_verification_identity( + trust_source="configured", + trusted_writer_public_key_hex="44" * 32, + literal_sha256=_C2_DIGEST, + literal_byte_len=123, + report=report, + ) + return store.publish_success( + identity=identity, + publication_floor_sequence=publication_floor_sequence, + report=report, + persistence={ + "audit_bundle_sha256": _C2_DIGEST, + "body_uri": "", + }, + evidence={ + "accepted_share_count": 0, + "distinct_miner_count": 0, + }, + verification_identity=verification_identity, + created_at=created_at, + ) + + +def _c2_authority(store: AuditArtifactStore) -> dict[str, list[int]]: + root_value = os.fstat(store._root_fd) + lock_value = os.fstat(store._publication_lock_fd) + return { + "root": [root_value.st_dev, root_value.st_ino], + "lock": [lock_value.st_dev, lock_value.st_ino], + } + + +def _c2_path_snapshot(path: Path) -> dict[str, object]: + try: + value = path.lstat() + except FileNotFoundError: + return {"exists": False} + payload: dict[str, object] = { + "exists": True, + "device": value.st_dev, + "inode": value.st_ino, + "mode": value.st_mode, + "size": value.st_size, + "mtime_ns": value.st_mtime_ns, + } + if path.is_file() and not path.is_symlink(): + payload["sha256"] = hashlib.sha256(path.read_bytes()).hexdigest() + return payload + + +def _c2_filesystem_snapshot( + *, + root: Path, + evidence_path: Path, +) -> dict[str, object]: + return { + "root": _c2_path_snapshot(root), + "entries": { + path.name: _c2_path_snapshot(path) + for path in sorted(root.iterdir(), key=lambda value: value.name) + }, + "evidence": _c2_path_snapshot(evidence_path), + } + + +def _c2_database_snapshot( + ledger: PsqlShareLedger, + *, + block_hashes: tuple[str, ...], +) -> dict[str, object]: + quoted = ", ".join(f"'{block_hash}'" for block_hash in block_hashes) + return ledger._run_json( + f""" +SELECT json_build_object( + 'rows', COALESCE(( + SELECT json_agg(json_build_object( + 'block_hash', block_hash, + 'block_height', block_height, + 'chain_state', chain_state, + 'maturity_state', maturity_state, + 'audit_publication_sequence', audit_publication_sequence + ) ORDER BY block_hash) + FROM qbit_pool_blocks + WHERE block_hash IN ({quoted}) + ), '[]'::json), + 'floor', COALESCE(( + SELECT MAX(audit_publication_sequence) + FROM qbit_pool_blocks + ), 0), + 'allocator', ( + SELECT json_build_object( + 'last_value', last_value, + 'is_called', is_called + ) + FROM qbit_audit_publication_sequence_seq + ) +); +""" + ) + + +def _emit_worker_event(event: str, **payload: object) -> None: + print( + json.dumps( + {"event": event, **payload}, + separators=(",", ":"), + sort_keys=True, + ), + flush=True, + ) + + +def _read_worker_command(expected: str) -> dict[str, object]: + line = sys.stdin.readline(support.PSQL_OUTPUT_LIMIT_BYTES + 2) + if not line: + raise support.GateFailure(f"worker command {expected!r} was not received") + if len(line.encode("utf-8")) > support.PSQL_OUTPUT_LIMIT_BYTES: + raise support.GateFailure("worker command exceeded 1 MiB") + try: + payload = json.loads(line) + except json.JSONDecodeError as error: + raise support.GateFailure("worker command is not JSON") from error + if not isinstance(payload, dict) or payload.get("command") != expected: + raise support.GateFailure( + f"expected worker command {expected!r}, got {payload!r}" + ) + return payload + + +class _JsonWorker: + def __init__(self, config_path: Path) -> None: + self._stderr = tempfile.TemporaryFile(mode="w+t", encoding="utf-8") + self._captured = 0 + self._event_buffer = bytearray() + self.process = subprocess.Popen( + [ + sys.executable, + "-m", + _PROCESS_GATE_MODULE, + "--c2-worker", + str(config_path), + ], + stdin=subprocess.PIPE, + stdout=subprocess.PIPE, + stderr=self._stderr, + text=True, + bufsize=1, + start_new_session=True, + ) + migration._register_process(self.process) + + def _stderr_text(self) -> str: + raw = os.pread( + self._stderr.fileno(), + support.PSQL_OUTPUT_LIMIT_BYTES + 1, + 0, + ) + if len(raw) > support.PSQL_OUTPUT_LIMIT_BYTES: + return "worker stderr exceeded 1 MiB" + return raw.decode("utf-8", errors="replace").strip() + + def read_event(self, expected: str) -> dict[str, object]: + if self.process.stdout is None: + raise support.GateFailure("JSON worker has no stdout") + deadline = time.monotonic() + support.PSQL_TIMEOUT_SECONDS + while time.monotonic() < deadline: + newline = self._event_buffer.find(b"\n") + if newline >= 0: + raw_line = bytes(self._event_buffer[:newline]) + del self._event_buffer[: newline + 1] + try: + line = raw_line.decode("utf-8") + except UnicodeDecodeError as error: + raise support.GateFailure( + "JSON worker emitted non-UTF-8 protocol bytes" + ) from error + try: + payload = json.loads(line) + except json.JSONDecodeError as error: + raise support.GateFailure( + f"JSON worker emitted invalid protocol line: {line!r}" + ) from error + if not isinstance(payload, dict): + raise support.GateFailure("JSON worker event is not an object") + if payload.get("event") == "error": + raise support.GateFailure( + f"JSON worker failed: {payload.get('message')}; " + f"stderr={self._stderr_text()}" + ) + if payload.get("event") != expected: + raise support.GateFailure( + f"expected JSON worker event {expected!r}, got {payload!r}" + ) + return payload + remaining = max(0.0, deadline - time.monotonic()) + ready, _, _ = select.select([self.process.stdout], [], [], remaining) + if not ready: + break + chunk = os.read(self.process.stdout.fileno(), 64 * 1024) + if not chunk: + break + self._captured += len(chunk) + if self._captured > support.PSQL_OUTPUT_LIMIT_BYTES: + raise support.GateFailure("JSON worker stdout exceeded 1 MiB") + self._event_buffer.extend(chunk) + if ( + b"\n" not in self._event_buffer + and len(self._event_buffer) > support.PSQL_OUTPUT_LIMIT_BYTES + ): + raise support.GateFailure( + "JSON worker event line exceeded 1 MiB" + ) + raise support.GateFailure( + f"JSON worker event {expected!r} timed out; " + f"exit={self.process.poll()} stderr={self._stderr_text()}" + ) + + def send(self, command: str) -> None: + if self.process.stdin is None or self.process.stdin.closed: + raise support.GateFailure("JSON worker has no writable stdin") + self.process.stdin.write( + json.dumps( + {"command": command}, + separators=(",", ":"), + sort_keys=True, + ) + + "\n" + ) + self.process.stdin.flush() + + def wait_success(self) -> None: + if self.process.stdin is not None and not self.process.stdin.closed: + self.process.stdin.close() + try: + self.process.wait(timeout=support.PSQL_TIMEOUT_SECONDS) + except subprocess.TimeoutExpired as error: + support._terminate_and_reap(self.process) + raise support.GateFailure("JSON worker timed out during exit") from error + finally: + migration._forget_process(self.process) + residual = bytes(self._event_buffer) + self._event_buffer.clear() + if self.process.stdout is not None: + while True: + chunk = os.read(self.process.stdout.fileno(), 64 * 1024) + if not chunk: + break + residual += chunk + self._captured += len(chunk) + if self._captured > support.PSQL_OUTPUT_LIMIT_BYTES: + raise support.GateFailure("JSON worker stdout exceeded 1 MiB") + stderr = self._stderr_text() + if self.process.returncode != 0: + raise support.GateFailure( + f"JSON worker exited {self.process.returncode}: {stderr}" + ) + if residual.strip(): + raise support.GateFailure( + "JSON worker emitted unexpected residual output: " + + residual.decode("utf-8", errors="replace").strip() + ) + if stderr: + raise support.GateFailure( + f"JSON worker emitted unexpected stderr: {stderr}" + ) + + def close(self) -> None: + if self.process.poll() is None: + support._terminate_and_reap(self.process) + migration._forget_process(self.process) + for stream in (self.process.stdin, self.process.stdout, self.process.stderr): + if stream is not None and not stream.closed: + stream.close() + if not self._stderr.closed: + self._stderr.close() + + +def _c2_worker_ledger(config: dict[str, object]) -> _WorkerScopedPsqlLedger: + writer = config.get("writer") + if not isinstance(writer, dict): + raise support.GateFailure("process worker writer configuration is invalid") + return _WorkerScopedPsqlLedger( + test_schema=str(config["schema"]), + application_name=str(config["application_name"]), + writer_id=str(writer["id"]), + writer_epoch=int(writer["epoch"]), + writer_session_token=str(writer["token"]), + initialize_schema=False, + ) + + +def _c2_worker_store(config: dict[str, object]) -> AuditArtifactStore: + return _c2_store( + Path(str(config["root"])), + Path(str(config["evidence_path"])), + ) + + +def _c2_worker_identity( + config: dict[str, object], + key: str, + *, + sequence: int, +) -> AuditPublicationIdentity: + value = config.get(key) + if not isinstance(value, dict): + raise support.GateFailure(f"process worker {key} identity is invalid") + return AuditPublicationIdentity( + sequence, + int(value["height"]), + str(value["hash"]), + ) + + +def _run_c2_a_worker(config: dict[str, object]) -> None: + ledger: _WorkerScopedPsqlLedger | None = None + store: AuditArtifactStore | None = None + try: + ledger = _c2_worker_ledger(config) + store = _c2_worker_store(config) + initial = _c2_worker_identity( + config, + "initial", + sequence=int(config["initial_sequence"]), + ) + with store.publication_order_guard(): + floor = ledger.audit_publication_sequence_floor() + _emit_worker_event( + "guard-acquired", + authority=_c2_authority(store), + floor=floor, + pid=os.getpid(), + ) + _read_worker_command("repair") + publication = _c2_publish( + store, + identity=initial, + publication_floor_sequence=floor, + created_at=f"c2-{config['variant']}-a-repair", + ) + _emit_worker_event( + "repaired", + published=publication.published, # type: ignore[attr-defined] + identity=publication.identity.to_json(), # type: ignore[attr-defined] + latest=store.latest_evidence(), + authority=_c2_authority(store), + ) + _emit_worker_event("guard-released", authority=_c2_authority(store)) + _read_worker_command("stale-check") + stale_hashes = tuple( + str(config[key]["hash"]) # type: ignore[index] + for key in ("stale_confirmation", "stale_reactivation") + ) + before = _c2_database_snapshot(ledger, block_hashes=stale_hashes) + errors: dict[str, str] = {} + stale_confirmation = config["stale_confirmation"] + stale_reactivation = config["stale_reactivation"] + assert isinstance(stale_confirmation, dict) + assert isinstance(stale_reactivation, dict) + operations = ( + ( + "confirmation", + ledger.confirm_accepted_block, + stale_confirmation, + ), + ( + "reactivation", + ledger.reactivate_pool_block, + stale_reactivation, + ), + ) + for label, operation, target in operations: + try: + operation( + block_hash=str(target["hash"]), + active_tip_height=int(target["height"]), + ) + except Exception as error: + message = str(error) + if "writer lease is not active" not in message: + raise support.GateFailure( + f"stale A {label} wrong failure: {message}" + ) from error + errors[label] = message + else: + raise support.GateFailure( + f"stale A {label} unexpectedly succeeded" + ) + after = _c2_database_snapshot(ledger, block_hashes=stale_hashes) + _emit_worker_event( + "stale-result", + errors=errors, + before=before, + after=after, + authority=_c2_authority(store), + ) + _read_worker_command("finish") + store.close() + store = None + ledger.close() + ledger = None + _emit_worker_event("closed") + finally: + if store is not None: + store.close() + if ledger is not None: + ledger.close() + + +def _run_c2_b_worker(config: dict[str, object]) -> None: + ledger: _WorkerScopedPsqlLedger | None = None + store: AuditArtifactStore | None = None + original_flock = audit_artifacts_module.fcntl.flock + try: + ledger = _c2_worker_ledger(config) + store = _c2_worker_store(config) + writer = config["writer"] + assert isinstance(writer, dict) + ledger._run_sql( + f""" +INSERT INTO qbit_a1_c2_attempts (variant, transition, writer_id) +VALUES ( + '{config['variant']}', + '{config['transition']}', + '{writer['id']}' +); +""" + ) + attempted = False + + def observed_flock(fd: int, operation: int) -> object: + nonlocal attempted + if ( + not attempted + and fd == store._publication_lock_fd # type: ignore[union-attr] + and operation & fcntl.LOCK_EX + ): + attempted = True + _emit_worker_event( + "flock-attempt", + authority=_c2_authority(store), # type: ignore[arg-type] + pid=os.getpid(), + ) + return original_flock(fd, operation) + + audit_artifacts_module.fcntl.flock = observed_flock + try: + with store.publication_order_guard(): + if str(config["transition"]) == "confirmation": + target = config["target"] + assert isinstance(target, dict) + result = ledger.confirm_accepted_block( + block_hash=str(target["hash"]), + active_tip_height=int(target["height"]), + ) + count = int(result["confirmed_count"]) + elif str(config["transition"]) == "reactivation": + target = config["target"] + assert isinstance(target, dict) + result = ledger.reactivate_pool_block( + block_hash=str(target["hash"]), + active_tip_height=int(target["height"]), + ) + count = int(result["reactivated_count"]) + else: + raise support.GateFailure("invalid C2 B transition") + support.assert_equal(count, 1, "C2 B transition count") + sequence = int(result["audit_publication_sequence"]) + identity = _c2_worker_identity( + config, + "target", + sequence=sequence, + ) + floor = ledger.audit_publication_sequence_floor() + publication = _c2_publish( + store, + identity=identity, + publication_floor_sequence=floor, + created_at=f"c2-{config['variant']}-b-publication", + ) + _emit_worker_event( + "transition-published", + transition_result=result, + floor=floor, + published=publication.published, # type: ignore[attr-defined] + identity=identity.to_json(), + latest=store.latest_evidence(), + authority=_c2_authority(store), + ) + finally: + audit_artifacts_module.fcntl.flock = original_flock + if not attempted: + raise support.GateFailure("C2 B did not attempt the publication flock") + _read_worker_command("finish") + released = ledger.release_writer_lease() + support.assert_equal(released, True, "C2 B writer lease release") + store.close() + store = None + ledger.close() + ledger = None + _emit_worker_event("closed", lease_released=released) + finally: + audit_artifacts_module.fcntl.flock = original_flock + if store is not None: + store.close() + if ledger is not None: + try: + ledger.release_writer_lease() + except BaseException: + pass + ledger.close() + + +def _c3_stale_publication_attempt( + *, + ledger: _WorkerScopedPsqlLedger, + store: AuditArtifactStore, + config: dict[str, object], +) -> tuple[int, dict[str, object]]: + floor = ledger.audit_publication_sequence_floor() + identity = _c2_worker_identity( + config, + "initial", + sequence=int(config["initial_sequence"]), + ) + try: + publication = _c2_publish( + store, + identity=identity, + publication_floor_sequence=floor, + created_at=f"c3-{config['variant']}-late-a", + ) + except RuntimeError as error: + message = str(error) + if "behind" not in message: + raise support.GateFailure( + f"C3 stale A publication wrong failure: {message}" + ) from error + outcome = {"kind": "behind-error", "message": message} + else: + if publication.published: + raise support.GateFailure("C3 stale A publication regressed evidence") + outcome = { + "kind": "returned", + "published": publication.published, + "identity": publication.identity.to_json(), + } + return floor, outcome + + +def _c3_stale_transition_check( + ledger: _WorkerScopedPsqlLedger, + config: dict[str, object], +) -> dict[str, object]: + stale_hashes = tuple( + str(config[key]["hash"]) # type: ignore[index] + for key in ("stale_confirmation", "stale_reactivation") + ) + before = _c2_database_snapshot(ledger, block_hashes=stale_hashes) + errors: dict[str, str] = {} + stale_confirmation = config["stale_confirmation"] + stale_reactivation = config["stale_reactivation"] + assert isinstance(stale_confirmation, dict) + assert isinstance(stale_reactivation, dict) + operations = ( + ("confirmation", ledger.confirm_accepted_block, stale_confirmation), + ("reactivation", ledger.reactivate_pool_block, stale_reactivation), + ) + for label, operation, target in operations: + try: + operation( + block_hash=str(target["hash"]), + active_tip_height=int(target["height"]), + ) + except Exception as error: + message = str(error) + if "writer lease is not active" not in message: + raise support.GateFailure( + f"C3 stale A {label} wrong failure: {message}" + ) from error + errors[label] = message + else: + raise support.GateFailure(f"C3 stale A {label} unexpectedly succeeded") + after = _c2_database_snapshot(ledger, block_hashes=stale_hashes) + return {"errors": errors, "before": before, "after": after} + + +def _run_c3_primary_a_worker(config: dict[str, object]) -> None: + ledger: _WorkerScopedPsqlLedger | None = None + store: AuditArtifactStore | None = None + try: + ledger = _c2_worker_ledger(config) + store = _c2_worker_store(config) + _emit_worker_event( + "parked", + authority=_c2_authority(store), + pid=os.getpid(), + guard="outside", + ) + _read_worker_command("attempt") + with store.publication_order_guard(): + floor, outcome = _c3_stale_publication_attempt( + ledger=ledger, + store=store, + config=config, + ) + _emit_worker_event( + "stale-publication", + authority=_c2_authority(store), + floor=floor, + outcome=outcome, + latest=store.latest_evidence(), + ) + transition_check = _c3_stale_transition_check(ledger, config) + retention: list[dict[str, int]] = [] + for live_retention in (0, 1): + store.reconfigure(live_bundle_retention=live_retention) + result = store.prune_best_effort() + retention.append( + { + "retention": live_retention, + "live_removed": result.live_removed, + "candidate_removed": result.candidate_removed, + "errors": result.errors, + } + ) + _emit_worker_event( + "post-check", + authority=_c2_authority(store), + transition_check=transition_check, + retention=retention, + latest=store.latest_evidence(), + ) + _read_worker_command("finish") + store.close() + store = None + ledger.close() + ledger = None + _emit_worker_event("closed") + finally: + if store is not None: + store.close() + if ledger is not None: + ledger.close() + + +def _run_c3_late_a_worker(config: dict[str, object]) -> None: + ledger: _WorkerScopedPsqlLedger | None = None + store: AuditArtifactStore | None = None + try: + ledger = _c2_worker_ledger(config) + store = _c2_worker_store(config) + initial = config["initial"] + assert isinstance(initial, dict) + with store.publication_order_guard(): + result = ledger.confirm_accepted_block( + block_hash=str(initial["hash"]), + active_tip_height=int(initial["height"]), + ) + sequence = int(result["audit_publication_sequence"]) + support.assert_equal(sequence, 1, "C3 late A confirmation ordinal N") + floor = ledger.audit_publication_sequence_floor() + support.assert_equal(floor, sequence, "C3 late A confirmed floor N") + _emit_worker_event( + "confirmed", + authority=_c2_authority(store), + pid=os.getpid(), + identity=_c2_worker_identity( + config, + "initial", + sequence=sequence, + ).to_json(), + floor=floor, + ) + _emit_worker_event( + "parked", + authority=_c2_authority(store), + pid=os.getpid(), + guard="outside", + ) + _read_worker_command("attempt") + with store.publication_order_guard(): + floor, outcome = _c3_stale_publication_attempt( + ledger=ledger, + store=store, + config=config, + ) + _emit_worker_event( + "stale-publication", + authority=_c2_authority(store), + floor=floor, + outcome=outcome, + latest=store.latest_evidence(), + ) + _read_worker_command("finish") + store.close() + store = None + ledger.close() + ledger = None + _emit_worker_event("closed") + finally: + if store is not None: + store.close() + if ledger is not None: + ledger.close() + + +def _run_c3_b_worker(config: dict[str, object]) -> None: + ledger: _WorkerScopedPsqlLedger | None = None + store: AuditArtifactStore | None = None + original_flock = audit_artifacts_module.fcntl.flock + try: + ledger = _c2_worker_ledger(config) + store = _c2_worker_store(config) + writer = config["writer"] + assert isinstance(writer, dict) + ledger._run_sql( + f""" +INSERT INTO qbit_a1_c3_attempts (variant, writer_id) +VALUES ('{config['variant']}', '{writer['id']}'); +""" + ) + attempted = False + + def observed_flock(fd: int, operation: int) -> object: + nonlocal attempted + if ( + not attempted + and fd == store._publication_lock_fd # type: ignore[union-attr] + and operation & fcntl.LOCK_EX + ): + attempted = True + _emit_worker_event( + "flock-attempt", + authority=_c2_authority(store), # type: ignore[arg-type] + pid=os.getpid(), + ) + return original_flock(fd, operation) + + audit_artifacts_module.fcntl.flock = observed_flock + try: + with store.publication_order_guard(): + target = config["target"] + assert isinstance(target, dict) + result = ledger.confirm_accepted_block( + block_hash=str(target["hash"]), + active_tip_height=int(target["height"]), + ) + support.assert_equal( + result["confirmed_count"], + 1, + "C3 B confirmation count", + ) + sequence = int(result["audit_publication_sequence"]) + identity = _c2_worker_identity( + config, + "target", + sequence=sequence, + ) + floor = ledger.audit_publication_sequence_floor() + publication = _c2_publish( + store, + identity=identity, + publication_floor_sequence=floor, + created_at=f"c3-{config['variant']}-b-publication", + ) + _emit_worker_event( + "transition-published", + authority=_c2_authority(store), + transition_result=result, + floor=floor, + identity=identity.to_json(), + published=publication.published, + latest=store.latest_evidence(), + ) + finally: + audit_artifacts_module.fcntl.flock = original_flock + if not attempted: + raise support.GateFailure("C3 B did not attempt the publication flock") + _read_worker_command("finish") + released = ledger.release_writer_lease() + support.assert_equal(released, True, "C3 B writer lease release") + store.close() + store = None + ledger.close() + ledger = None + _emit_worker_event("closed", lease_released=released) + finally: + audit_artifacts_module.fcntl.flock = original_flock + if store is not None: + store.close() + if ledger is not None: + try: + ledger.release_writer_lease() + except BaseException: + pass + ledger.close() + + +def _run_c2_worker(config_path: Path) -> int: + try: + raw = config_path.read_bytes() + if len(raw) > support.PSQL_OUTPUT_LIMIT_BYTES: + raise support.GateFailure("process worker configuration exceeded 1 MiB") + config = json.loads(raw) + if not isinstance(config, dict): + raise support.GateFailure("process worker configuration is not an object") + role = config.get("role") + if role == "a": + _run_c2_a_worker(config) + elif role == "b": + _run_c2_b_worker(config) + elif role == "c3-primary-a": + _run_c3_primary_a_worker(config) + elif role == "c3-late-a": + _run_c3_late_a_worker(config) + elif role == "c3-b": + _run_c3_b_worker(config) + else: + raise support.GateFailure(f"invalid C2 worker role: {role!r}") + except BaseException as error: + _emit_worker_event( + "error", + error_type=type(error).__name__, + message=str(error), + ) + traceback.print_exc(file=sys.stderr) + return 1 + return 0 + + +def _start_file_worker( + sql: str, +) -> tuple[subprocess.Popen[str], Any, Any, Any]: + input_file = tempfile.TemporaryFile(mode="w+t", encoding="utf-8") + stdout_file = tempfile.TemporaryFile(mode="w+t", encoding="utf-8") + stderr_file = tempfile.TemporaryFile(mode="w+t", encoding="utf-8") + input_file.write(sql) + input_file.seek(0) + process = subprocess.Popen( + migration._psql_process_command(), + stdin=input_file, + stdout=stdout_file, + stderr=stderr_file, + text=True, + start_new_session=True, + ) + migration._register_process(process) + return process, input_file, stdout_file, stderr_file + + +def _close_file_worker( + worker: tuple[subprocess.Popen[str], Any, Any, Any] | None, +) -> None: + if worker is None: + return + process, input_file, stdout_file, stderr_file = worker + if process.poll() is None: + support._terminate_and_reap(process) + migration._forget_process(process) + for stream in ( + process.stdin, + process.stdout, + process.stderr, + input_file, + stdout_file, + stderr_file, + ): + if stream is not None and not stream.closed: + stream.close() + + +def _wait_success( + worker: tuple[subprocess.Popen[str], Any, Any, Any], +) -> None: + process, _input_file, stdout_file, stderr_file = worker + migration._wait_file_process( + process, + stdout_file=stdout_file, + stderr_file=stderr_file, + expected_success=True, + error_fragment=None, + ) + + +def _wait_holder_success( + holder: subprocess.Popen[str], + *, + stderr_file: Any, + label: str, +) -> None: + try: + holder.wait(timeout=support.PSQL_TIMEOUT_SECONDS) + except subprocess.TimeoutExpired as error: + support._terminate_and_reap(holder) + raise support.GateFailure(f"{label} timed out") from error + finally: + migration._forget_process(holder) + stdout = "" + if holder.stdout is not None: + stdout = holder.stdout.read(support.PSQL_OUTPUT_LIMIT_BYTES + 1) + stderr_file.seek(0) + stderr = stderr_file.read(support.PSQL_OUTPUT_LIMIT_BYTES + 1) + if ( + len(stdout.encode("utf-8")) > support.PSQL_OUTPUT_LIMIT_BYTES + or len(stderr.encode("utf-8")) > support.PSQL_OUTPUT_LIMIT_BYTES + ): + raise support.GateFailure(f"{label} output exceeded 1 MiB") + if holder.returncode != 0: + diagnostics = stderr.strip() or stdout.strip() or "no diagnostic output" + raise support.GateFailure( + f"{label} failed with {holder.returncode}: {diagnostics}" + ) + + +def _prepared_rows_sql(block_hashes: tuple[str, ...]) -> str: + values = ",\n".join( + ( + f"('{block_hash}', {height}, '{'10' * 32}', '{'20' * 32}', " + f"'{'30' * 32}', 'prepared', 'immature')" + ) + for height, block_hash in enumerate(block_hashes, start=10) + ) + return f""" +INSERT INTO qbit_pool_blocks ( + block_hash, block_height, parent_hash, coinbase_txid, + payout_manifest_sha256, chain_state, maturity_state +) VALUES +{values}; +""" + + +def _confirmation_worker_sql( + *, + schema: str, + application_name: str, + round_name: str, + block_hash: str, + height: int, + writer_id: str, + writer_epoch: int, + writer_token: str, + gate_a: int, + gate_b: int, +) -> str: + return f""" +SET application_name = '{application_name}'; +SET statement_timeout = '20s'; +SET lock_timeout = '20s'; +SET search_path TO "{schema}", pg_catalog; +BEGIN; +SELECT pg_catalog.pg_advisory_xact_lock_shared({gate_a}, {gate_b}); +DO $qbit_a1_confirm$ +DECLARE + confirmed_count integer; +BEGIN + SELECT "{schema}".qbit_confirm_pool_block( + '{block_hash}', {height}, '{writer_id}', {writer_epoch}, + '{writer_token}', interval '5 minutes' + ) + INTO confirmed_count; + IF confirmed_count <> 1 THEN + RAISE EXCEPTION 'expected one confirmed pool block, got %', confirmed_count; + END IF; +END; +$qbit_a1_confirm$; +INSERT INTO "{schema}".qbit_a1_confirmation_results ( + round_name, + block_hash, + audit_publication_sequence +) +SELECT + '{round_name}', + block_hash, + audit_publication_sequence +FROM "{schema}".qbit_pool_blocks +WHERE block_hash = '{block_hash}' + AND chain_state = 'confirmed' + AND audit_publication_sequence IS NOT NULL; +COMMIT; +""" + + +def _assert_no_tagged_backends(tags: tuple[str, ...], message: str) -> None: + quoted = ", ".join(f"'{tag}'" for tag in tags) + count = int( + support.run_json( + f""" +SELECT json_build_object('count', count(*)) +FROM pg_catalog.pg_stat_activity +WHERE application_name IN ({quoted}); +""" + )["count"] + ) + support.assert_equal(count, 0, message) + + +def test_database_observed_confirmation_order() -> None: + for round_index, reverse_launch in enumerate((False, True), start=1): + round_name = f"c1-round-{round_index}" + schema = support.create_owned_schema(f"c1_round_{round_index}") + writer_id = f"a1-c1-writer-{round_index}" + writer_token = f"a1-c1-token-{round_index}" + ledger = support.ScopedPsqlLedger( + test_schema=schema, + writer_id=writer_id, + writer_epoch=1, + writer_session_token=writer_token, + initialize_schema=True, + ) + block_hashes = ("91" * 32, "92" * 32) + heights = {block_hashes[0]: 10, block_hashes[1]: 11} + gate_a = 41_000 + round_index + gate_b = 51_000 + round_index + holder_tag = f"qbit_a1_c1_holder_{support.RUN_TOKEN}_{round_index}" + worker_tags = ( + f"qbit_a1_c1_worker_a_{support.RUN_TOKEN}_{round_index}", + f"qbit_a1_c1_worker_b_{support.RUN_TOKEN}_{round_index}", + ) + holder_input = tempfile.TemporaryFile(mode="w+t", encoding="utf-8") + holder_stderr = tempfile.TemporaryFile(mode="w+t", encoding="utf-8") + holder: subprocess.Popen[str] | None = None + workers: list[tuple[subprocess.Popen[str], Any, Any, Any]] = [] + try: + ledger._run_sql( + """ +CREATE TABLE qbit_a1_confirmation_results ( + completion_id bigserial PRIMARY KEY, + round_name text NOT NULL, + block_hash text NOT NULL UNIQUE, + audit_publication_sequence bigint NOT NULL +); +""" + + _prepared_rows_sql(block_hashes) + ) + holder_input.write( + f""" +SET application_name = '{holder_tag}'; +BEGIN; +SELECT pg_catalog.pg_advisory_xact_lock({gate_a}, {gate_b}); +DO $qbit_a1_wait$ +DECLARE + both_waiting boolean := false; +BEGIN + FOR attempt IN 1..600 LOOP + PERFORM pg_catalog.pg_stat_clear_snapshot(); + SELECT count(DISTINCT worker.pid) = 2 + INTO both_waiting + FROM pg_catalog.pg_stat_activity worker + WHERE worker.application_name IN ( + '{worker_tags[0]}', + '{worker_tags[1]}' + ) + AND pg_catalog.pg_backend_pid() = ANY( + pg_catalog.pg_blocking_pids(worker.pid) + ) + AND EXISTS ( + SELECT 1 + FROM pg_catalog.pg_locks lock_state + WHERE lock_state.pid = worker.pid + AND lock_state.locktype = 'advisory' + AND NOT lock_state.granted + ); + EXIT WHEN both_waiting; + PERFORM pg_catalog.pg_sleep(0.05); + END LOOP; + IF NOT both_waiting THEN + RAISE EXCEPTION 'two confirmation advisory waiters were not observed'; + END IF; +END; +$qbit_a1_wait$; +COMMIT; +""" + ) + holder_input.seek(0) + holder = subprocess.Popen( + migration._psql_process_command(), + stdin=holder_input, + stdout=subprocess.DEVNULL, + stderr=holder_stderr, + text=True, + bufsize=1, + start_new_session=True, + ) + migration._register_process(holder) + migration._wait_for_advisory_holder( + holder, + application_name=holder_tag, + deadline=time.monotonic() + support.PSQL_TIMEOUT_SECONDS, + ) + launch = [0, 1] + if reverse_launch: + launch.reverse() + for worker_index in launch: + block_hash = block_hashes[worker_index] + workers.append( + _start_file_worker( + _confirmation_worker_sql( + schema=schema, + application_name=worker_tags[worker_index], + round_name=round_name, + block_hash=block_hash, + height=heights[block_hash], + writer_id=writer_id, + writer_epoch=1, + writer_token=writer_token, + gate_a=gate_a, + gate_b=gate_b, + ) + ) + ) + _wait_holder_success( + holder, + stderr_file=holder_stderr, + label="C1 holder", + ) + for worker in workers: + _wait_success(worker) + results = ledger._run_json( + f""" +SELECT json_build_object( + 'rows', COALESCE(json_agg(json_build_object( + 'completion_id', completion_id, + 'block_hash', block_hash, + 'audit_publication_sequence', audit_publication_sequence + ) ORDER BY completion_id), '[]'::json) +) +FROM qbit_a1_confirmation_results +WHERE round_name = '{round_name}'; +""" + )["rows"] + support.assert_equal(len(results), 2, f"{round_name} completion count") + support.assert_equal( + [row["completion_id"] for row in results], + [1, 2], + f"{round_name} exact completion identifiers", + ) + support.assert_equal( + [row["audit_publication_sequence"] for row in results], + [1, 2], + f"{round_name} completion order matches ordinal order", + ) + support.assert_equal( + {row["block_hash"] for row in results}, + set(block_hashes), + f"{round_name} distinct completed blocks", + ) + before_replay = support.allocator_state(ledger) + for _replay_round in range(3): + for block_hash in block_hashes: + state = ledger.pool_block_state(block_hash=block_hash) + replay = ledger.confirm_accepted_block( + block_hash=block_hash, + active_tip_height=heights[block_hash], + ) + support.assert_equal( + replay["audit_publication_sequence"], + state["audit_publication_sequence"], # type: ignore[index] + f"{round_name} exact replay ordinal", + ) + support.assert_equal( + support.allocator_state(ledger), + before_replay, + f"{round_name} replay allocator immobility", + ) + fresh_hash = "93" * 32 + ledger._run_sql(_prepared_rows_sql((fresh_hash,))) + support.assert_equal( + ledger.confirm_accepted_block( + block_hash=fresh_hash, + active_tip_height=10, + )["audit_publication_sequence"], + 3, + f"{round_name} exact next ordinal", + ) + _assert_no_tagged_backends( + (holder_tag, *worker_tags), + f"{round_name} tagged backend cleanup", + ) + finally: + for worker in workers: + _close_file_worker(worker) + if holder is not None: + if holder.poll() is None: + support._terminate_and_reap(holder) + migration._forget_process(holder) + for stream in (holder.stdin, holder.stdout, holder.stderr): + if stream is not None and not stream.closed: + stream.close() + if not holder_stderr.closed: + holder_stderr.close() + if not holder_input.closed: + holder_input.close() + ledger.release_writer_lease() + ledger.close() + + +def _c2_seed_rows( + ledger: support.ScopedPsqlLedger, + *, + initial: dict[str, object], + target: dict[str, object], + stale_confirmation: dict[str, object], + stale_reactivation: dict[str, object], +) -> None: + rows = ( + (initial, "prepared"), + (target, "prepared"), + (stale_confirmation, "prepared"), + (stale_reactivation, "inactive"), + ) + values = ",\n".join( + ( + f"('{row['hash']}', {int(row['height'])}, '{'10' * 32}', " + f"'{'20' * 32}', '{'30' * 32}', '{chain_state}', 'immature')" + ) + for row, chain_state in rows + ) + ledger._run_sql( + f""" +CREATE TABLE qbit_a1_c2_attempts ( + attempt_id bigserial PRIMARY KEY, + variant text NOT NULL, + transition text NOT NULL, + writer_id text NOT NULL, + attempted_at timestamptz NOT NULL DEFAULT clock_timestamp() +); + +INSERT INTO qbit_pool_blocks ( + block_hash, + block_height, + parent_hash, + coinbase_txid, + payout_manifest_sha256, + chain_state, + maturity_state +) VALUES +{values}; +""" + ) + + +def _c2_attempt_and_lease( + ledger: support.ScopedPsqlLedger, + *, + variant: str, +) -> dict[str, object]: + return ledger._run_json( + f""" +SELECT json_build_object( + 'attempts', COALESCE(( + SELECT json_agg(json_build_object( + 'variant', variant, + 'transition', transition, + 'writer_id', writer_id + ) ORDER BY attempt_id) + FROM qbit_a1_c2_attempts + WHERE variant = '{variant}' + ), '[]'::json), + 'lease', ( + SELECT json_build_object( + 'writer_id', writer_id, + 'writer_epoch', writer_epoch, + 'writer_session_token', writer_session_token + ) + FROM qbit_ledger_writer_lease + WHERE singleton + ) +); +""" + ) + + +def _c2_expire_writer( + ledger: support.ScopedPsqlLedger, + *, + writer_id: str, + writer_epoch: int, + writer_token: str, +) -> None: + expired = ledger._run_json( + f""" +WITH expired AS ( + UPDATE qbit_ledger_writer_lease + SET updated_at = clock_timestamp() - interval '6 minutes', + lease_expires_at = clock_timestamp() - interval '1 minute' + WHERE singleton + AND writer_id = '{writer_id}' + AND writer_epoch = {writer_epoch} + AND writer_session_token = '{writer_token}' + RETURNING writer_id +) +SELECT json_build_object('count', count(*)) +FROM expired; +""" + )["count"] + support.assert_equal(expired, 1, "C2 exact A lease expiry") + + +def _c2_write_config(path: Path, config: dict[str, object]) -> None: + path.write_text( + json.dumps(config, separators=(",", ":"), sort_keys=True), + encoding="utf-8", + ) + + +def _c2_expected_authority(root: Path) -> dict[str, list[int]]: + root_value = root.lstat() + lock_value = (root / ".prism-audit-publication.lock").lstat() + return { + "root": [root_value.st_dev, root_value.st_ino], + "lock": [lock_value.st_dev, lock_value.st_ino], + } + + +def _assert_c2_final_database( + snapshot: dict[str, object], + *, + initial: dict[str, object], + target: dict[str, object], + stale_confirmation: dict[str, object], + stale_reactivation: dict[str, object], + initial_sequence: int, + target_sequence: int, + variant: str, +) -> None: + rows = snapshot["rows"] + assert isinstance(rows, list) + by_hash = {str(row["block_hash"]): row for row in rows} + support.assert_equal( + by_hash[str(initial["hash"])], + { + "block_hash": initial["hash"], + "block_height": initial["height"], + "chain_state": "confirmed", + "maturity_state": "immature", + "audit_publication_sequence": initial_sequence, + }, + f"C2 {variant} initial exact row", + ) + support.assert_equal( + by_hash[str(target["hash"])], + { + "block_hash": target["hash"], + "block_height": target["height"], + "chain_state": "confirmed", + "maturity_state": "immature", + "audit_publication_sequence": target_sequence, + }, + f"C2 {variant} B exact row", + ) + support.assert_equal( + by_hash[str(stale_confirmation["hash"])], + { + "block_hash": stale_confirmation["hash"], + "block_height": stale_confirmation["height"], + "chain_state": "prepared", + "maturity_state": "immature", + "audit_publication_sequence": None, + }, + f"C2 {variant} stale prepared C row", + ) + support.assert_equal( + by_hash[str(stale_reactivation["hash"])], + { + "block_hash": stale_reactivation["hash"], + "block_height": stale_reactivation["height"], + "chain_state": "inactive", + "maturity_state": "immature", + "audit_publication_sequence": None, + }, + f"C2 {variant} stale inactive R row", + ) + support.assert_equal( + snapshot["floor"], + target_sequence, + f"C2 {variant} final floor", + ) + support.assert_equal( + snapshot["allocator"], + {"last_value": target_sequence, "is_called": True}, + f"C2 {variant} exact allocator", + ) + ordinals = sorted( + int(row["audit_publication_sequence"]) + for row in rows + if row["audit_publication_sequence"] is not None + ) + support.assert_equal( + len(ordinals), + len(set(ordinals)), + f"C2 {variant} unique durable ordinals", + ) + + +def _test_c2_a_wins(transition: str) -> None: + if transition not in {"confirmation", "reactivation"}: + raise support.GateFailure(f"invalid C2 transition: {transition!r}") + variant_index = 1 if transition == "confirmation" else 2 + variant = transition + schema = support.create_owned_schema(f"c2_{variant}") + writer_a = { + "id": f"a1-c2-{variant}-writer-a", + "epoch": 1, + "token": f"a1-c2-{variant}-token-a", + } + writer_b = { + "id": f"a1-c2-{variant}-writer-b", + "epoch": 2, + "token": f"a1-c2-{variant}-token-b", + } + initial = {"hash": f"a{variant_index}" * 32, "height": 10} + target = {"hash": f"b{variant_index}" * 32, "height": 20} + stale_confirmation = {"hash": f"c{variant_index}" * 32, "height": 30} + stale_reactivation = {"hash": f"d{variant_index}" * 32, "height": 31} + all_hashes = tuple( + str(value["hash"]) + for value in ( + initial, + target, + stale_confirmation, + stale_reactivation, + ) + ) + ledger = support.ScopedPsqlLedger( + test_schema=schema, + writer_id=str(writer_a["id"]), + writer_epoch=int(writer_a["epoch"]), + writer_session_token=str(writer_a["token"]), + initialize_schema=True, + ) + worker_a: _JsonWorker | None = None + worker_b: _JsonWorker | None = None + setup_store: AuditArtifactStore | None = None + try: + _c2_seed_rows( + ledger, + initial=initial, + target=target, + stale_confirmation=stale_confirmation, + stale_reactivation=stale_reactivation, + ) + initial_sequence = 1 + if transition == "reactivation": + target_prior = ledger.confirm_accepted_block( + block_hash=str(target["hash"]), + active_tip_height=int(target["height"]), + ) + support.assert_equal( + target_prior["audit_publication_sequence"], + 1, + "C2 reactivation target prior confirmation ordinal", + ) + support.assert_equal( + ledger.mark_pool_block_inactive( + block_hash=str(target["hash"]), + active_tip_height=int(target["height"]), + )["inactive_count"], + 1, + "C2 reactivation target prior inactive transition", + ) + initial_sequence = 2 + initial_result = ledger.confirm_accepted_block( + block_hash=str(initial["hash"]), + active_tip_height=int(initial["height"]), + ) + support.assert_equal( + initial_result["audit_publication_sequence"], + initial_sequence, + f"C2 {variant} initial ordinal", + ) + support.assert_equal( + ledger.audit_publication_sequence_floor(), + initial_sequence, + f"C2 {variant} initial floor", + ) + support.assert_equal( + support.allocator_state(ledger), + {"last_value": initial_sequence, "is_called": True}, + f"C2 {variant} initial allocator", + ) + target_sequence = initial_sequence + 1 + with tempfile.TemporaryDirectory() as tmp, ExitStack() as temp_cleanup: + base = Path(tmp) + root = base / "audit" + evidence_path = base / "state" / "evidence.json" + setup_store = _c2_store(root, evidence_path) + temp_cleanup.callback(setup_store.close) + initial_identity = AuditPublicationIdentity( + initial_sequence, + int(initial["height"]), + str(initial["hash"]), + ) + with setup_store.publication_order_guard(): + initial_publication = _c2_publish( + setup_store, + identity=initial_identity, + publication_floor_sequence=initial_sequence, + created_at=f"c2-{variant}-initial", + ) + support.assert_equal( + initial_publication.published, # type: ignore[attr-defined] + True, + f"C2 {variant} initial publication", + ) + initial_envelope = setup_store.live_envelope_path( + block_height=int(initial["height"]), + block_hash=str(initial["hash"]), + ) + setup_store.close() + setup_store = None + evidence_path.write_bytes(b"{c2-damaged-evidence") + expected_authority = _c2_expected_authority(root) + common_config: dict[str, object] = { + "schema": schema, + "root": str(root), + "evidence_path": str(evidence_path), + "variant": variant, + "transition": transition, + "initial_sequence": initial_sequence, + "initial": initial, + "target": target, + "stale_confirmation": stale_confirmation, + "stale_reactivation": stale_reactivation, + } + config_a = base / "worker-a.json" + config_b = base / "worker-b.json" + tag_suffix = f"{variant[0]}_{support.RUN_TOKEN[:8]}" + tag_a = f"qbit_a1_c2_a_{tag_suffix}" + tag_b = f"qbit_a1_c2_b_{tag_suffix}" + _c2_write_config( + config_a, + { + **common_config, + "role": "a", + "writer": writer_a, + "application_name": tag_a, + }, + ) + _c2_write_config( + config_b, + { + **common_config, + "role": "b", + "writer": writer_b, + "application_name": tag_b, + }, + ) + worker_a = _JsonWorker(config_a) + temp_cleanup.callback(worker_a.close) + a_guard = worker_a.read_event("guard-acquired") + support.assert_equal( + a_guard["authority"], + expected_authority, + f"C2 {variant} A exact root/lock inode", + ) + support.assert_equal( + a_guard["floor"], + initial_sequence, + f"C2 {variant} A floor N", + ) + blocked_filesystem = _c2_filesystem_snapshot( + root=root, + evidence_path=evidence_path, + ) + blocked_database = _c2_database_snapshot( + ledger, + block_hashes=all_hashes, + ) + _c2_expire_writer( + ledger, + writer_id=str(writer_a["id"]), + writer_epoch=int(writer_a["epoch"]), + writer_token=str(writer_a["token"]), + ) + worker_b = _JsonWorker(config_b) + temp_cleanup.callback(worker_b.close) + b_attempt = worker_b.read_event("flock-attempt") + support.assert_equal( + b_attempt["authority"], + expected_authority, + f"C2 {variant} B exact root/lock inode", + ) + support.assert_equal( + worker_b.process.poll(), + None, + f"C2 {variant} B remains alive while flock-blocked", + ) + child_pids = {int(a_guard["pid"]), int(b_attempt["pid"])} + support.assert_equal( + len(child_pids), + 2, + f"C2 {variant} distinct A/B OS processes", + ) + if os.getpid() in child_pids: + raise support.GateFailure( + f"C2 {variant} worker reused the parent process" + ) + support.assert_equal( + _c2_attempt_and_lease(ledger, variant=variant), + { + "attempts": [ + { + "variant": variant, + "transition": transition, + "writer_id": writer_b["id"], + } + ], + "lease": { + "writer_id": writer_b["id"], + "writer_epoch": writer_b["epoch"], + "writer_session_token": writer_b["token"], + }, + }, + f"C2 {variant} DB-visible B attempt and replacement lease", + ) + support.assert_equal( + _c2_database_snapshot(ledger, block_hashes=all_hashes), + blocked_database, + f"C2 {variant} B row/floor/allocator unchanged while blocked", + ) + support.assert_equal( + _c2_filesystem_snapshot(root=root, evidence_path=evidence_path), + blocked_filesystem, + f"C2 {variant} filesystem unchanged while B blocked", + ) + worker_a.send("repair") + repaired = worker_a.read_event("repaired") + support.assert_equal( + repaired["published"], + True, + f"C2 {variant} A exact repair published", + ) + support.assert_equal( + repaired["identity"], + initial_identity.to_json(), + f"C2 {variant} A repair identity N", + ) + support.assert_equal( + repaired["authority"], + expected_authority, + f"C2 {variant} A repair authority", + ) + released = worker_a.read_event("guard-released") + support.assert_equal( + released["authority"], + expected_authority, + f"C2 {variant} A guard release authority", + ) + b_done = worker_b.read_event("transition-published") + expected_b_identity = AuditPublicationIdentity( + target_sequence, + int(target["height"]), + str(target["hash"]), + ) + support.assert_equal( + b_done["identity"], + expected_b_identity.to_json(), + f"C2 {variant} B identity N+1", + ) + support.assert_equal( + b_done["floor"], + target_sequence, + f"C2 {variant} B fresh floor", + ) + support.assert_equal( + b_done["published"], + True, + f"C2 {variant} B publication", + ) + support.assert_equal( + b_done["authority"], + expected_authority, + f"C2 {variant} B publication authority", + ) + final_database = _c2_database_snapshot( + ledger, + block_hashes=all_hashes, + ) + _assert_c2_final_database( + final_database, + initial=initial, + target=target, + stale_confirmation=stale_confirmation, + stale_reactivation=stale_reactivation, + initial_sequence=initial_sequence, + target_sequence=target_sequence, + variant=variant, + ) + latest = json.loads(evidence_path.read_text(encoding="utf-8")) + support.assert_equal( + latest["audit_publication_identity"], + expected_b_identity.to_json(), + f"C2 {variant} final evidence B/N+1", + ) + support.assert_equal( + latest["block_hash"], + target["hash"], + f"C2 {variant} final evidence B hash", + ) + target_envelope = root / ( + f"prism-live-audit-bundle-{target['height']}-{target['hash']}.json" + ) + support.assert_equal( + initial_envelope.exists(), + False, + f"C2 {variant} retention removes unpinned A envelope", + ) + support.assert_equal( + target_envelope.exists(), + True, + f"C2 {variant} retention keeps B envelope", + ) + support.assert_equal( + sorted(path.name for path in root.iterdir()), + [ + ".prism-audit-publication.lock", + target_envelope.name, + ], + f"C2 {variant} exact retained root entries", + ) + before_stale_database = _c2_database_snapshot( + ledger, + block_hashes=all_hashes, + ) + before_stale_filesystem = _c2_filesystem_snapshot( + root=root, + evidence_path=evidence_path, + ) + worker_a.send("stale-check") + stale_result = worker_a.read_event("stale-result") + stale_errors = stale_result["errors"] + assert isinstance(stale_errors, dict) + support.assert_equal( + set(stale_errors), + {"confirmation", "reactivation"}, + f"C2 {variant} exact stale A failure labels", + ) + for label in ("confirmation", "reactivation"): + if "writer lease is not active" not in str(stale_errors[label]): + raise support.GateFailure( + f"C2 {variant} stale A {label} wrong error: " + f"{stale_errors[label]!r}" + ) + support.assert_equal( + stale_result["before"], + stale_result["after"], + f"C2 {variant} child-observed stale transition immobility", + ) + support.assert_equal( + stale_result["authority"], + expected_authority, + f"C2 {variant} stale A authority", + ) + support.assert_equal( + _c2_database_snapshot(ledger, block_hashes=all_hashes), + before_stale_database, + f"C2 {variant} stale A row/floor/allocator immobility", + ) + support.assert_equal( + _c2_filesystem_snapshot(root=root, evidence_path=evidence_path), + before_stale_filesystem, + f"C2 {variant} stale A filesystem immobility", + ) + worker_a.send("finish") + worker_a.read_event("closed") + worker_a.wait_success() + worker_b.send("finish") + b_closed = worker_b.read_event("closed") + support.assert_equal( + b_closed["lease_released"], + True, + f"C2 {variant} B lease released", + ) + worker_b.wait_success() + _assert_no_tagged_backends( + (tag_a, tag_b), + f"C2 {variant} tagged child backend cleanup", + ) + finally: + if setup_store is not None: + setup_store.close() + if worker_a is not None: + worker_a.close() + if worker_b is not None: + worker_b.close() + ledger.close() + + +def test_c2_a_wins_confirmation_publication() -> None: + _test_c2_a_wins("confirmation") + + +def _c3_seed_rows( + ledger: support.ScopedPsqlLedger, + *, + initial: dict[str, object], + target: dict[str, object], + stale_confirmation: dict[str, object], + stale_reactivation: dict[str, object], +) -> None: + rows = ( + (initial, "prepared"), + (target, "prepared"), + (stale_confirmation, "prepared"), + (stale_reactivation, "inactive"), + ) + values = ",\n".join( + ( + f"('{row['hash']}', {int(row['height'])}, '{'10' * 32}', " + f"'{'20' * 32}', '{'30' * 32}', '{chain_state}', 'immature')" + ) + for row, chain_state in rows + ) + ledger._run_sql( + f""" +CREATE TABLE qbit_a1_c3_attempts ( + attempt_id bigserial PRIMARY KEY, + variant text NOT NULL, + writer_id text NOT NULL, + attempted_at timestamptz NOT NULL DEFAULT clock_timestamp() +); + +INSERT INTO qbit_pool_blocks ( + block_hash, + block_height, + parent_hash, + coinbase_txid, + payout_manifest_sha256, + chain_state, + maturity_state +) VALUES +{values}; +""" + ) + + +def _c3_attempt_and_lease( + ledger: support.ScopedPsqlLedger, + *, + variant: str, +) -> dict[str, object]: + return ledger._run_json( + f""" +SELECT json_build_object( + 'attempts', COALESCE(( + SELECT json_agg(json_build_object( + 'variant', variant, + 'writer_id', writer_id + ) ORDER BY attempt_id) + FROM qbit_a1_c3_attempts + WHERE variant = '{variant}' + ), '[]'::json), + 'lease', ( + SELECT json_build_object( + 'writer_id', writer_id, + 'writer_epoch', writer_epoch, + 'writer_session_token', writer_session_token + ) + FROM qbit_ledger_writer_lease + WHERE singleton + ) +); +""" + ) + + +def _c3_expected_database( + *, + initial: dict[str, object], + target: dict[str, object], + stale_confirmation: dict[str, object], + stale_reactivation: dict[str, object], +) -> dict[str, object]: + return { + "rows": [ + { + "block_hash": initial["hash"], + "block_height": initial["height"], + "chain_state": "confirmed", + "maturity_state": "immature", + "audit_publication_sequence": 1, + }, + { + "block_hash": target["hash"], + "block_height": target["height"], + "chain_state": "confirmed", + "maturity_state": "immature", + "audit_publication_sequence": 2, + }, + { + "block_hash": stale_confirmation["hash"], + "block_height": stale_confirmation["height"], + "chain_state": "prepared", + "maturity_state": "immature", + "audit_publication_sequence": None, + }, + { + "block_hash": stale_reactivation["hash"], + "block_height": stale_reactivation["height"], + "chain_state": "inactive", + "maturity_state": "immature", + "audit_publication_sequence": None, + }, + ], + "floor": 2, + "allocator": {"last_value": 2, "is_called": True}, + } + + +def _assert_c3_stale_outcome(outcome: object, *, label: str) -> None: + if not isinstance(outcome, dict): + raise support.GateFailure(f"{label} outcome is not an object") + if outcome.get("kind") == "returned": + support.assert_equal( + outcome.get("published"), + False, + f"{label} returns published false", + ) + return + if outcome.get("kind") == "behind-error" and "behind" in str( + outcome.get("message") + ): + return + raise support.GateFailure(f"{label} unexpected outcome: {outcome!r}") + + +def _test_c3_interleaving(variant: str) -> None: + if variant not in {"primary", "late"}: + raise support.GateFailure(f"invalid C3 variant: {variant!r}") + variant_index = 3 if variant == "primary" else 4 + schema = support.create_owned_schema(f"c3_{variant}") + writer_a = { + "id": f"a1-c3-{variant}-writer-a", + "epoch": 1, + "token": f"a1-c3-{variant}-token-a", + } + writer_b = { + "id": f"a1-c3-{variant}-writer-b", + "epoch": 2, + "token": f"a1-c3-{variant}-token-b", + } + initial = {"hash": f"a{variant_index}" * 32, "height": 10} + target = {"hash": f"b{variant_index}" * 32, "height": 20} + stale_confirmation = {"hash": f"c{variant_index}" * 32, "height": 30} + stale_reactivation = {"hash": f"d{variant_index}" * 32, "height": 31} + all_hashes = tuple( + str(value["hash"]) + for value in ( + initial, + target, + stale_confirmation, + stale_reactivation, + ) + ) + ledger = support.ScopedPsqlLedger( + test_schema=schema, + writer_id=str(writer_a["id"]), + writer_epoch=int(writer_a["epoch"]), + writer_session_token=str(writer_a["token"]), + initialize_schema=True, + ) + worker_a: _JsonWorker | None = None + worker_b: _JsonWorker | None = None + setup_store: AuditArtifactStore | None = None + try: + _c3_seed_rows( + ledger, + initial=initial, + target=target, + stale_confirmation=stale_confirmation, + stale_reactivation=stale_reactivation, + ) + if variant == "primary": + initial_result = ledger.confirm_accepted_block( + block_hash=str(initial["hash"]), + active_tip_height=int(initial["height"]), + ) + support.assert_equal( + initial_result["audit_publication_sequence"], + 1, + "C3 primary initial ordinal N", + ) + with tempfile.TemporaryDirectory() as tmp, ExitStack() as temp_cleanup: + base = Path(tmp) + root = base / "audit" + evidence_path = base / "state" / "evidence.json" + setup_store = _c2_store(root, evidence_path) + temp_cleanup.callback(setup_store.close) + initial_identity = AuditPublicationIdentity( + 1, + int(initial["height"]), + str(initial["hash"]), + ) + initial_envelope = setup_store.live_envelope_path( + block_height=int(initial["height"]), + block_hash=str(initial["hash"]), + ) + if variant == "primary": + with setup_store.publication_order_guard(): + initial_publication = _c2_publish( + setup_store, + identity=initial_identity, + publication_floor_sequence=1, + created_at="c3-primary-initial", + ) + support.assert_equal( + initial_publication.published, + True, + "C3 primary initial publication N", + ) + setup_store.close() + setup_store = None + if variant == "primary": + initial_envelope.unlink() + support.assert_equal( + evidence_path.exists(), + True, + "C3 primary damaged N retains evidence", + ) + support.assert_equal( + initial_envelope.exists(), + False, + f"C3 {variant} N envelope starts absent", + ) + expected_authority = _c2_expected_authority(root) + tag_suffix = f"{variant[0]}_{support.RUN_TOKEN[:8]}" + tag_a = f"qbit_a1_c3_a_{tag_suffix}" + tag_b = f"qbit_a1_c3_b_{tag_suffix}" + common_config: dict[str, object] = { + "schema": schema, + "root": str(root), + "evidence_path": str(evidence_path), + "variant": variant, + "transition": "confirmation", + "initial_sequence": 1, + "initial": initial, + "target": target, + "stale_confirmation": stale_confirmation, + "stale_reactivation": stale_reactivation, + } + config_a = base / "worker-a.json" + config_b = base / "worker-b.json" + _c2_write_config( + config_a, + { + **common_config, + "role": ( + "c3-primary-a" if variant == "primary" else "c3-late-a" + ), + "writer": writer_a, + "application_name": tag_a, + }, + ) + _c2_write_config( + config_b, + { + **common_config, + "role": "c3-b", + "writer": writer_b, + "application_name": tag_b, + }, + ) + worker_a = _JsonWorker(config_a) + temp_cleanup.callback(worker_a.close) + if variant == "late": + confirmed = worker_a.read_event("confirmed") + support.assert_equal( + confirmed["authority"], + expected_authority, + "C3 late A confirmation guard authority", + ) + support.assert_equal( + confirmed["identity"], + initial_identity.to_json(), + "C3 late A confirmed identity N", + ) + support.assert_equal( + confirmed["floor"], + 1, + "C3 late A confirmed floor N", + ) + parked = worker_a.read_event("parked") + support.assert_equal( + parked["authority"], + expected_authority, + f"C3 {variant} A exact root/lock inode", + ) + support.assert_equal( + parked["guard"], + "outside", + f"C3 {variant} A parks outside guard", + ) + if variant == "late": + support.assert_equal( + parked["pid"], + confirmed["pid"], + "C3 late A exits guard in same child", + ) + support.assert_equal( + worker_a.process.poll(), + None, + f"C3 {variant} parked A remains alive", + ) + initial_database = _c2_database_snapshot( + ledger, + block_hashes=all_hashes, + ) + support.assert_equal( + initial_database["floor"], + 1, + f"C3 {variant} durable floor N before replacement", + ) + support.assert_equal( + initial_database["allocator"], + {"last_value": 1, "is_called": True}, + f"C3 {variant} allocator N before replacement", + ) + support.assert_equal( + initial_database["rows"], + [ + { + "block_hash": initial["hash"], + "block_height": initial["height"], + "chain_state": "confirmed", + "maturity_state": "immature", + "audit_publication_sequence": 1, + }, + { + "block_hash": target["hash"], + "block_height": target["height"], + "chain_state": "prepared", + "maturity_state": "immature", + "audit_publication_sequence": None, + }, + { + "block_hash": stale_confirmation["hash"], + "block_height": stale_confirmation["height"], + "chain_state": "prepared", + "maturity_state": "immature", + "audit_publication_sequence": None, + }, + { + "block_hash": stale_reactivation["hash"], + "block_height": stale_reactivation["height"], + "chain_state": "inactive", + "maturity_state": "immature", + "audit_publication_sequence": None, + }, + ], + f"C3 {variant} exact rows before replacement", + ) + damaged_snapshot = _c2_filesystem_snapshot( + root=root, + evidence_path=evidence_path, + ) + if variant == "primary": + damaged_evidence = json.loads( + evidence_path.read_text(encoding="utf-8") + ) + support.assert_equal( + damaged_evidence["audit_publication_identity"], + initial_identity.to_json(), + "C3 primary damaged evidence still names N", + ) + _c2_expire_writer( + ledger, + writer_id=str(writer_a["id"]), + writer_epoch=int(writer_a["epoch"]), + writer_token=str(writer_a["token"]), + ) + worker_b = _JsonWorker(config_b) + temp_cleanup.callback(worker_b.close) + b_attempt = worker_b.read_event("flock-attempt") + support.assert_equal( + b_attempt["authority"], + expected_authority, + f"C3 {variant} B exact root/lock inode", + ) + child_pids = {int(parked["pid"]), int(b_attempt["pid"])} + support.assert_equal( + len(child_pids), + 2, + f"C3 {variant} distinct A/B OS processes", + ) + if os.getpid() in child_pids: + raise support.GateFailure( + f"C3 {variant} worker reused the parent process" + ) + b_done = worker_b.read_event("transition-published") + expected_b_identity = AuditPublicationIdentity( + 2, + int(target["height"]), + str(target["hash"]), + ) + support.assert_equal( + b_done["authority"], + expected_authority, + f"C3 {variant} B publication authority", + ) + support.assert_equal( + b_done["identity"], + expected_b_identity.to_json(), + f"C3 {variant} B identity N+1", + ) + support.assert_equal(b_done["floor"], 2, f"C3 {variant} B floor N+1") + support.assert_equal( + b_done["published"], + True, + f"C3 {variant} B publishes N+1", + ) + support.assert_equal( + _c3_attempt_and_lease(ledger, variant=variant), + { + "attempts": [ + {"variant": variant, "writer_id": writer_b["id"]} + ], + "lease": { + "writer_id": writer_b["id"], + "writer_epoch": writer_b["epoch"], + "writer_session_token": writer_b["token"], + }, + }, + f"C3 {variant} B attempt and active replacement lease", + ) + expected_database = _c3_expected_database( + initial=initial, + target=target, + stale_confirmation=stale_confirmation, + stale_reactivation=stale_reactivation, + ) + support.assert_equal( + _c2_database_snapshot(ledger, block_hashes=all_hashes), + expected_database, + f"C3 {variant} exact database after B", + ) + target_envelope = root / ( + f"prism-live-audit-bundle-{target['height']}-{target['hash']}.json" + ) + b_evidence_bytes = evidence_path.read_bytes() + b_envelope_bytes = target_envelope.read_bytes() + durable_b_evidence = json.loads(b_evidence_bytes) + durable_b_envelope = json.loads(b_envelope_bytes) + support.assert_equal( + durable_b_evidence["audit_publication_identity"], + expected_b_identity.to_json(), + f"C3 {variant} durable B evidence identity N+1", + ) + support.assert_equal( + durable_b_evidence["block_hash"], + target["hash"], + f"C3 {variant} durable B evidence hash", + ) + support.assert_equal( + { + "block_hash": durable_b_envelope["block_hash"], + "block_height": durable_b_envelope["block_height"], + "audit_bundle_sha256": durable_b_envelope[ + "audit_bundle_sha256" + ], + }, + { + "block_hash": target["hash"], + "block_height": target["height"], + "audit_bundle_sha256": _C2_DIGEST, + }, + f"C3 {variant} durable B envelope identity", + ) + b_evidence_stat = evidence_path.lstat() + b_envelope_stat = target_envelope.lstat() + b_filesystem = _c2_filesystem_snapshot( + root=root, + evidence_path=evidence_path, + ) + support.assert_equal( + sorted(path.name for path in root.iterdir()), + [".prism-audit-publication.lock", target_envelope.name], + f"C3 {variant} exact B root entries", + ) + support.assert_equal( + initial_envelope.exists(), + False, + f"C3 {variant} N remains absent after B", + ) + worker_a.send("attempt") + stale_publication = worker_a.read_event("stale-publication") + support.assert_equal( + stale_publication["authority"], + expected_authority, + f"C3 {variant} late A same lock inode", + ) + support.assert_equal( + stale_publication["floor"], + 2, + f"C3 {variant} late A reads fresh N+1 floor", + ) + _assert_c3_stale_outcome( + stale_publication["outcome"], + label=f"C3 {variant} stale N publication", + ) + stale_latest = stale_publication["latest"] + assert isinstance(stale_latest, dict) + support.assert_equal( + stale_latest["audit_publication_identity"], + expected_b_identity.to_json(), + f"C3 {variant} stale A reconciles B evidence", + ) + if variant == "primary": + post_check = worker_a.read_event("post-check") + transition_check = post_check["transition_check"] + assert isinstance(transition_check, dict) + errors = transition_check["errors"] + assert isinstance(errors, dict) + support.assert_equal( + set(errors), + {"confirmation", "reactivation"}, + "C3 primary exact stale lease-failure labels", + ) + for label in ("confirmation", "reactivation"): + if "writer lease is not active" not in str(errors[label]): + raise support.GateFailure( + f"C3 primary stale {label} wrong error: {errors[label]!r}" + ) + support.assert_equal( + transition_check["before"], + transition_check["after"], + "C3 primary child-observed stale transition immobility", + ) + support.assert_equal( + post_check["retention"], + [ + { + "retention": 0, + "live_removed": 0, + "candidate_removed": 0, + "errors": 0, + }, + { + "retention": 1, + "live_removed": 0, + "candidate_removed": 0, + "errors": 0, + }, + ], + "C3 primary stale A retention 0/1 pins B", + ) + support.assert_equal( + post_check["authority"], + expected_authority, + "C3 primary post-check authority", + ) + support.assert_equal( + _c2_database_snapshot(ledger, block_hashes=all_hashes), + expected_database, + f"C3 {variant} late A row/floor/allocator immobility", + ) + support.assert_equal( + _c3_attempt_and_lease(ledger, variant=variant)["lease"], + { + "writer_id": writer_b["id"], + "writer_epoch": writer_b["epoch"], + "writer_session_token": writer_b["token"], + }, + f"C3 {variant} B lease remains active through A checks", + ) + support.assert_equal( + _c2_filesystem_snapshot(root=root, evidence_path=evidence_path), + b_filesystem, + f"C3 {variant} exact B filesystem survives late A", + ) + support.assert_equal( + evidence_path.read_bytes(), + b_evidence_bytes, + f"C3 {variant} B evidence bytes unchanged", + ) + support.assert_equal( + target_envelope.read_bytes(), + b_envelope_bytes, + f"C3 {variant} B envelope bytes unchanged", + ) + evidence_after = evidence_path.lstat() + envelope_after = target_envelope.lstat() + support.assert_equal( + (evidence_after.st_dev, evidence_after.st_ino), + (b_evidence_stat.st_dev, b_evidence_stat.st_ino), + f"C3 {variant} B evidence inode unchanged", + ) + support.assert_equal( + (envelope_after.st_dev, envelope_after.st_ino), + (b_envelope_stat.st_dev, b_envelope_stat.st_ino), + f"C3 {variant} B envelope inode unchanged", + ) + support.assert_equal( + initial_envelope.exists(), + False, + f"C3 {variant} stale A never repairs N", + ) + if variant == "primary": + damaged_entries = damaged_snapshot["entries"] + assert isinstance(damaged_entries, dict) + support.assert_equal( + sorted(damaged_entries), + [".prism-audit-publication.lock"], + "C3 primary damaged snapshot has no N envelope", + ) + worker_a.send("finish") + worker_a.read_event("closed") + worker_a.wait_success() + worker_b.send("finish") + b_closed = worker_b.read_event("closed") + support.assert_equal( + b_closed["lease_released"], + True, + f"C3 {variant} B lease release", + ) + worker_b.wait_success() + _assert_no_tagged_backends( + (tag_a, tag_b), + f"C3 {variant} tagged child backend cleanup", + ) + with support.ACTIVE_CHILDREN_LOCK: + support.assert_equal( + len(support.ACTIVE_CHILDREN), + 0, + f"C3 {variant} active child registry cleanup", + ) + finally: + if setup_store is not None: + setup_store.close() + if worker_a is not None: + worker_a.close() + if worker_b is not None: + worker_b.close() + ledger.close() + + +def test_c3_b_wins_and_late_a_interleavings() -> None: + for variant in ("primary", "late"): + _test_c3_interleaving(variant) + + +def server_evidence() -> dict[str, object]: + evidence = support.run_json( + """ +SELECT json_build_object( + 'server_version', current_setting('server_version'), + 'server_version_num', current_setting('server_version_num') +); +""" + ) + configured_image = os.environ.get("QBIT_PRISM_GATE_IMAGE", "").strip() + provisioned_image_digest = os.environ.get( + "QBIT_PRISM_GATE_IMAGE_DIGEST", + "", + ).strip() + if not configured_image or configured_image.casefold() == "unreported": + raise support.GateFailure("configured PostgreSQL image evidence is required") + if ( + not provisioned_image_digest + or provisioned_image_digest.casefold() == "unreported" + ): + raise support.GateFailure( + "provisioned PostgreSQL image digest evidence is required" + ) + evidence["configured_image"] = configured_image + evidence["provisioned_image_digest"] = provisioned_image_digest + return evidence + + +def main() -> None: + public_before = support.public_sentinel() + failure: BaseException | None = None + try: + test_database_observed_confirmation_order() + test_c2_a_wins_confirmation_publication() + test_c3_b_wins_and_late_a_interleavings() + except BaseException as error: + failure = error + try: + support.cleanup_active_children() + support.cleanup_owned_schemas() + support.assert_equal( + support.marker_schema_count(), + 0, + "process gate marker cleanup", + ) + support.assert_equal( + support.public_sentinel(), + public_before, + "process gate public preservation", + ) + except BaseException as cleanup_error: + if failure is None: + raise + raise support.GateFailure( + f"process scenario failed with {failure!r}; cleanup also failed " + f"with {cleanup_error!r}" + ) from cleanup_error + else: + support.atexit.unregister(support.cleanup_active_children) + support.atexit.unregister(support.cleanup_owned_schemas) + if failure is not None: + raise failure + print("prism postgres A1 process gate evidence " + json.dumps(server_evidence())) + print( + "prism postgres A1 process gate PASS " + "C1-confirmation-order C2-A-wins-confirmation-publication " + "C3-B-wins-late-A" + ) + + +if __name__ == "__main__": + if len(sys.argv) == 3 and sys.argv[1] == "--c2-worker": + raise SystemExit(_run_c2_worker(Path(sys.argv[2]))) + if len(sys.argv) != 1: + raise SystemExit("unexpected process-gate arguments") + main() diff --git a/tests/prism_postgres_a1_revert_gate.py b/tests/prism_postgres_a1_revert_gate.py new file mode 100644 index 000000000..8a220d747 --- /dev/null +++ b/tests/prism_postgres_a1_revert_gate.py @@ -0,0 +1,548 @@ +"""Non-discovered PostgreSQL ordinal schema-revert integration gate. + +Round-trips crates/qbit-prism/sql/001_share_ledger_revert_audit_publication_sequence.sql: +forward-migrate a schema holding confirmed rows, revert it to its +pre-ordinal shape, confirm blocks pre-ordinal-style against the reverted +schema, then re-apply the forward migration and require a correct backfill +with no double-assigned ordinals. Invoked by +``test/test-prism-postgres-ledger.sh``. +""" + +from __future__ import annotations + +from pathlib import Path + +from tests import prism_postgres_a1_gate as support +from tests.prism_postgres_a1_gate import ( + GateFailure, + LEGACY_POOL_BLOCKS_SQL, + ScopedPsqlLedger, + assert_equal, + create_owned_schema, + run_json, + run_psql, +) + +REVERT_SCRIPT_PATH = ( + Path(__file__).resolve().parents[1] + / "crates/qbit-prism/sql/001_share_ledger_revert_audit_publication_sequence.sql" +) +REVERT_SCRIPT_SQL = REVERT_SCRIPT_PATH.read_text(encoding="utf-8") + +REVERT_HASH_A = "91" * 32 +REVERT_HASH_B = "92" * 32 +REVERT_HASH_C = "93" * 32 +REVERT_HASH_INACTIVE = "94" * 32 +REVERT_HASH_REVERSED = "95" * 32 +REVERT_HASH_PREPARED = "96" * 32 +REVERT_HASH_LEGACY_UPDATE = "97" * 32 +REVERT_HASH_POST_REAPPLY = "98" * 32 + + +def apply_revert(schema: str) -> None: + run_psql(REVERT_SCRIPT_SQL, schema=schema) + + +def seed_legacy_pool_blocks(schema: str) -> None: + run_psql(LEGACY_POOL_BLOCKS_SQL, schema=schema) + run_psql( + f""" +INSERT INTO qbit_pool_blocks ( + block_hash, block_height, parent_hash, coinbase_txid, + payout_manifest_sha256, found_at, chain_state, maturity_state, + matured_at, disconnected_at +) VALUES + -- Insertion order intentionally differs from deterministic backfill + -- order. A/B share a timestamp and sort by block hash. + ('{REVERT_HASH_C}', 10, '{'10' * 32}', '{'20' * 32}', '{'30' * 32}', + '2020-01-01T00:00:02Z', 'confirmed', 'immature', NULL, NULL), + ('{REVERT_HASH_B}', 10, '{'10' * 32}', '{'20' * 32}', '{'30' * 32}', + '2020-01-01T00:00:01Z', 'confirmed', 'immature', NULL, NULL), + ('{REVERT_HASH_A}', 10, '{'10' * 32}', '{'20' * 32}', '{'30' * 32}', + '2020-01-01T00:00:01Z', 'confirmed', 'immature', NULL, NULL), + ('{REVERT_HASH_INACTIVE}', 10, '{'10' * 32}', '{'20' * 32}', '{'30' * 32}', + '2020-01-01T00:00:03Z', 'inactive', 'immature', NULL, NULL), + ('{REVERT_HASH_REVERSED}', 10, '{'10' * 32}', '{'20' * 32}', '{'30' * 32}', + '2020-01-01T00:00:03Z', 'reversed', 'reversed', NULL, + '2020-01-02T00:00:00Z'), + ('{REVERT_HASH_PREPARED}', 10, '{'10' * 32}', '{'20' * 32}', '{'30' * 32}', + '2020-01-01T00:00:04Z', 'prepared', 'immature', NULL, NULL); +""", + schema=schema, + ) + + +def insert_prepared_block(schema: str, block_hash: str, found_at: str) -> None: + run_psql( + f""" +INSERT INTO qbit_pool_blocks ( + block_hash, block_height, parent_hash, coinbase_txid, + payout_manifest_sha256, found_at, chain_state, maturity_state +) VALUES ( + '{block_hash}', 10, '{'10' * 32}', '{'20' * 32}', '{'30' * 32}', + '{found_at}', 'prepared', 'immature' +); +""", + schema=schema, + ) + + +def ordinal_catalog(schema: str) -> dict[str, object]: + return run_json( + """ +SELECT json_build_object( + 'column_present', EXISTS ( + SELECT 1 + FROM pg_attribute + WHERE attrelid = to_regclass('qbit_pool_blocks') + AND attname = 'audit_publication_sequence' + AND attnum > 0 + AND NOT attisdropped + ), + 'sequence_present', + to_regclass('qbit_audit_publication_sequence_seq') IS NOT NULL, + 'index_present', + to_regclass('qbit_pool_blocks_audit_publication_sequence_idx') + IS NOT NULL, + 'constraint_present', EXISTS ( + SELECT 1 + FROM pg_constraint + WHERE conrelid = to_regclass('qbit_pool_blocks') + AND conname = 'qbit_pool_blocks_audit_publication_sequence_check' + ), + 'trigger_present', EXISTS ( + SELECT 1 + FROM pg_trigger + WHERE tgrelid = to_regclass('qbit_pool_blocks') + AND tgname = 'qbit_pool_blocks_assign_publication_ordinal' + AND NOT tgisinternal + ), + 'trigger_function_present', EXISTS ( + SELECT 1 + FROM pg_proc function_definition + JOIN pg_namespace namespace + ON namespace.oid = function_definition.pronamespace + WHERE namespace.nspname = current_schema() + AND function_definition.proname = + 'qbit_pool_blocks_assign_publication_ordinal' + ), + 'confirm_mentions_ordinal', COALESCE(( + SELECT function_definition.prosrc LIKE '%audit_publication_sequence%' + FROM pg_proc function_definition + JOIN pg_namespace namespace + ON namespace.oid = function_definition.pronamespace + WHERE namespace.nspname = current_schema() + AND function_definition.proname = 'qbit_confirm_pool_block' + ), false), + 'confirm_search_path_pinned', COALESCE(( + SELECT function_definition.proconfig IS NOT NULL + FROM pg_proc function_definition + JOIN pg_namespace namespace + ON namespace.oid = function_definition.pronamespace + WHERE namespace.nspname = current_schema() + AND function_definition.proname = 'qbit_confirm_pool_block' + ), false), + 'reactivate_search_path_pinned', COALESCE(( + SELECT function_definition.proconfig IS NOT NULL + FROM pg_proc function_definition + JOIN pg_namespace namespace + ON namespace.oid = function_definition.pronamespace + WHERE namespace.nspname = current_schema() + AND function_definition.proname = 'qbit_reactivate_pool_block' + ), false) +); +""", + schema=schema, + ) + + +MIGRATED_CATALOG = { + "column_present": True, + "sequence_present": True, + "index_present": True, + "constraint_present": True, + "trigger_present": True, + "trigger_function_present": True, + "confirm_mentions_ordinal": True, + "confirm_search_path_pinned": True, + "reactivate_search_path_pinned": True, +} +REVERTED_CATALOG = { + "column_present": False, + "sequence_present": False, + "index_present": False, + "constraint_present": False, + "trigger_present": False, + "trigger_function_present": False, + "confirm_mentions_ordinal": False, + "confirm_search_path_pinned": False, + "reactivate_search_path_pinned": False, +} + + +def ordinal_rows(schema: str) -> dict[str, int | None]: + rows = run_json( + """ +SELECT COALESCE(json_agg(json_build_object( + 'block_hash', block_hash, + 'audit_publication_sequence', audit_publication_sequence +) ORDER BY block_hash), '[]'::json) +FROM qbit_pool_blocks; +""", + schema=schema, + ) + return { + str(row["block_hash"]): row["audit_publication_sequence"] + for row in rows + } + + +def duplicate_ordinal_count(schema: str) -> int: + return int( + run_json( + """ +SELECT json_build_object('duplicates', count(*)) +FROM ( + SELECT audit_publication_sequence + FROM qbit_pool_blocks + WHERE audit_publication_sequence IS NOT NULL + GROUP BY audit_publication_sequence + HAVING count(*) > 1 +) duplicated; +""", + schema=schema, + )["duplicates"] + ) + + +def allocator_state(schema: str) -> dict[str, object]: + return run_json( + """ +SELECT json_build_object( + 'last_value', last_value, + 'is_called', is_called +) +FROM qbit_audit_publication_sequence_seq; +""", + schema=schema, + ) + + +def install_writer_lease(schema: str, *, writer_id: str, token: str) -> None: + run_psql( + f""" +DELETE FROM qbit_ledger_writer_lease; +INSERT INTO qbit_ledger_writer_lease ( + writer_id, writer_epoch, writer_session_token, lease_expires_at +) VALUES ( + '{writer_id}', 1, '{token}', clock_timestamp() + interval '5 minutes' +); +""", + schema=schema, + ) + + +def expire_writer_lease(schema: str) -> None: + run_psql( + """ +UPDATE qbit_ledger_writer_lease +SET updated_at = clock_timestamp() - interval '6 minutes', + lease_expires_at = clock_timestamp() - interval '1 minute'; +""", + schema=schema, + ) + + +def initialize_forward_schema(schema: str, *, writer_id: str) -> None: + ledger = ScopedPsqlLedger( + test_schema=schema, + writer_id=writer_id, + writer_epoch=1, + initialize_schema=True, + ) + try: + pass + finally: + ledger.release_writer_lease() + ledger.close() + + +def test_revert_round_trip() -> None: + schema = create_owned_schema("revert_round_trip") + seed_legacy_pool_blocks(schema) + initialize_forward_schema(schema, writer_id="a1-revert-forward") + + assert_equal( + ordinal_catalog(schema), MIGRATED_CATALOG, "round-trip migrated catalog" + ) + assert_equal( + ordinal_rows(schema), + { + REVERT_HASH_A: 1, + REVERT_HASH_B: 2, + REVERT_HASH_C: 3, + REVERT_HASH_INACTIVE: 4, + REVERT_HASH_REVERSED: None, + REVERT_HASH_PREPARED: None, + }, + "round-trip forward backfill ordinals", + ) + + apply_revert(schema) + assert_equal( + ordinal_catalog(schema), REVERTED_CATALOG, "round-trip reverted catalog" + ) + assert_equal( + run_json( + """ +SELECT json_build_object( + 'confirmed', ( + SELECT count(*) FROM qbit_pool_blocks WHERE chain_state = 'confirmed' + ), + 'total', (SELECT count(*) FROM qbit_pool_blocks) +); +""", + schema=schema, + ), + {"confirmed": 3, "total": 6}, + "revert keeps every pool-block row", + ) + + # Pre-ordinal code confirms through qbit_confirm_pool_block's plain + # chain_state UPDATE; the restored function must work with no ordinal + # machinery present. + install_writer_lease( + schema, writer_id="revert-legacy-writer", token="revert-legacy-session" + ) + confirm_sql = ( + "SELECT json_build_object('confirmed', qbit_confirm_pool_block(" + f"'{REVERT_HASH_PREPARED}', 10, 'revert-legacy-writer', 1, " + "'revert-legacy-session'));" + ) + assert_equal( + int(run_json(confirm_sql, schema=schema)["confirmed"]), + 1, + "restored confirm function confirms against reverted schema", + ) + assert_equal( + int(run_json(confirm_sql, schema=schema)["confirmed"]), + 1, + "restored confirm function replays an exact confirmation", + ) + # The rawest pre-ordinal shape: a direct chain_state UPDATE. + insert_prepared_block( + schema, REVERT_HASH_LEGACY_UPDATE, "2020-01-01T00:00:05Z" + ) + run_psql( + f""" +UPDATE qbit_pool_blocks +SET chain_state = 'confirmed' +WHERE block_hash = '{REVERT_HASH_LEGACY_UPDATE}'; +""", + schema=schema, + ) + assert_equal( + run_json( + f""" +SELECT json_build_object( + 'chain_state', ( + SELECT chain_state FROM qbit_pool_blocks + WHERE block_hash = '{REVERT_HASH_LEGACY_UPDATE}' + ) +); +""", + schema=schema, + )["chain_state"], + "confirmed", + "plain chain_state UPDATE confirms against reverted schema", + ) + + # Idempotent style: a second run of the revert is a no-op success. + apply_revert(schema) + assert_equal( + ordinal_catalog(schema), + REVERTED_CATALOG, + "second revert run is a no-op", + ) + + # Re-apply the forward migration: the backfill must renumber every + # confirmed and inactive row deterministically by (found_at, block_hash), + # including the two rows confirmed pre-ordinal-style after the revert. + expire_writer_lease(schema) + initialize_forward_schema(schema, writer_id="a1-revert-reapply") + assert_equal( + ordinal_catalog(schema), MIGRATED_CATALOG, "re-applied catalog" + ) + reapplied = ordinal_rows(schema) + assert_equal( + reapplied, + { + REVERT_HASH_A: 1, + REVERT_HASH_B: 2, + REVERT_HASH_C: 3, + REVERT_HASH_INACTIVE: 4, + REVERT_HASH_PREPARED: 5, + REVERT_HASH_REVERSED: None, + REVERT_HASH_LEGACY_UPDATE: 6, + }, + "re-applied forward backfill ordinals", + ) + assert_equal(duplicate_ordinal_count(schema), 0, "re-applied uniqueness") + assert_equal( + allocator_state(schema), + {"last_value": 6, "is_called": True}, + "re-applied allocator floor", + ) + + # The assignment trigger continues past the backfilled floor for a + # pre-ordinal-style confirmation and does not disturb existing ordinals. + insert_prepared_block( + schema, REVERT_HASH_POST_REAPPLY, "2020-01-01T00:00:06Z" + ) + run_psql( + f""" +UPDATE qbit_pool_blocks +SET chain_state = 'confirmed' +WHERE block_hash = '{REVERT_HASH_POST_REAPPLY}'; +""", + schema=schema, + ) + after_trigger = ordinal_rows(schema) + assert_equal( + after_trigger, + {**reapplied, REVERT_HASH_POST_REAPPLY: 7}, + "trigger assigns the next ordinal after re-apply", + ) + assert_equal(duplicate_ordinal_count(schema), 0, "post-trigger uniqueness") + + # A further schema apply must not re-assign or double-assign anything. + expire_writer_lease(schema) + initialize_forward_schema(schema, writer_id="a1-revert-stability") + assert_equal( + ordinal_rows(schema), + after_trigger, + "third schema apply leaves every ordinal unchanged", + ) + assert_equal( + allocator_state(schema), + {"last_value": 7, "is_called": True}, + "third schema apply leaves the allocator unchanged", + ) + + +def test_revert_refuses_unknown_dependents() -> None: + schema = create_owned_schema("revert_dependent") + seed_legacy_pool_blocks(schema) + initialize_forward_schema(schema, writer_id="a1-revert-dependent") + migrated_rows = ordinal_rows(schema) + run_psql( + """ +CREATE VIEW qbit_pool_blocks_ordinal_view AS +SELECT block_hash, audit_publication_sequence +FROM qbit_pool_blocks; +""", + schema=schema, + ) + try: + apply_revert(schema) + except GateFailure as failure: + message = str(failure) + if "audit_publication_sequence" not in message or "depend" not in message: + raise + else: + raise GateFailure( + "revert with a dependent view unexpectedly succeeded" + ) + # The failed revert must roll back completely: nothing half-reverted. + assert_equal( + ordinal_catalog(schema), + MIGRATED_CATALOG, + "failed revert leaves the migrated catalog intact", + ) + assert_equal( + ordinal_rows(schema), + migrated_rows, + "failed revert leaves every ordinal intact", + ) + run_psql("DROP VIEW qbit_pool_blocks_ordinal_view;", schema=schema) + apply_revert(schema) + assert_equal( + ordinal_catalog(schema), + REVERTED_CATALOG, + "revert succeeds once the dependent view is gone", + ) + + +def test_revert_noop_and_missing_table() -> None: + schema = create_owned_schema("revert_legacy_noop") + seed_legacy_pool_blocks(schema) + # Never migrated: the revert must be a no-op that still converges on the + # pre-ordinal shape (including the restored confirmation function). + apply_revert(schema) + assert_equal( + ordinal_catalog(schema), + REVERTED_CATALOG, + "revert of a never-migrated schema is a no-op", + ) + assert_equal( + run_json( + "SELECT json_build_object(" + "'total', (SELECT count(*) FROM qbit_pool_blocks));", + schema=schema, + )["total"], + 6, + "no-op revert keeps every legacy row", + ) + + empty_schema = create_owned_schema("revert_missing_table") + try: + apply_revert(empty_schema) + except GateFailure as failure: + if "missing qbit_pool_blocks in current schema" not in str(failure): + raise + else: + raise GateFailure( + "revert without qbit_pool_blocks unexpectedly succeeded" + ) + + +def main() -> None: + public_before = support.public_sentinel() + failure: BaseException | None = None + try: + test_revert_round_trip() + test_revert_refuses_unknown_dependents() + test_revert_noop_and_missing_table() + except BaseException as error: + failure = error + try: + support.cleanup_active_children() + support.cleanup_owned_schemas() + support.assert_equal( + support.marker_schema_count(), 0, "revert marker cleanup" + ) + support.assert_equal( + support.public_sentinel(), + public_before, + "revert public preservation", + ) + except BaseException as cleanup_error: + if failure is None: + raise + raise GateFailure( + f"revert scenario failed with {failure!r}; " + f"cleanup also failed with {cleanup_error!r}" + ) from cleanup_error + else: + support.atexit.unregister(support.cleanup_active_children) + support.atexit.unregister(support.cleanup_owned_schemas) + if failure is not None: + raise failure + print( + "prism postgres A1 revert gate PASS " + "round-trip legacy-confirm loud-failure noop-idempotent" + ) + + +if __name__ == "__main__": + main() diff --git a/tests/prism_postgres_candidate_gate.py b/tests/prism_postgres_candidate_gate.py new file mode 100644 index 000000000..2bcd8f0af --- /dev/null +++ b/tests/prism_postgres_candidate_gate.py @@ -0,0 +1,281 @@ +#!/usr/bin/env python3 +"""Real-PostgreSQL gate for chunked candidate bodies (issue #255). + +Runs against a disposable database through the transport the environment +offers: ``PRISM_TEST_PSQL_COMMAND`` (the psql subprocess backend) and, +when ``PRISM_TEST_DATABASE_URL`` is set and psycopg imports, the native +pooled client as well. Every check is a semantic gate on the shipped +``PsqlShareLedger``: the additive migration applies twice, both durable +write routes stage outside the writer gate and publish under the fence, +the metadata-first page answers exhaustion, hydration reproduces the +share array from bounded chunk pages, terminalization detaches the body, +the janitor reclaims it in bounded steps, the database's own triggers +refuse every mutation of a sealed body, and a body an outbox row +references cannot be retired. + +Usage: ``python3 -m tests.prism_postgres_candidate_gate``; the wrapper +``test/test-prism-postgres-candidate-storage.sh`` provisions the container. +""" + +from __future__ import annotations + +import hashlib +import json +import os +import shlex +import subprocess +import sys +import time +import uuid +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).resolve().parents[1])) + +from lab.prism.share_ledger import ( # noqa: E402 + PendingShare, + PsqlShareLedger, + block_candidate_identity_sha256, +) + +CHECKS: list[str] = [] + + +def check(condition: object, label: str) -> None: + if not condition: + raise SystemExit(f"prism postgres candidate gate: FAIL {label}") + CHECKS.append(label) + print(f"prism postgres candidate gate: ok {label}", flush=True) + + +def psql_json(command: str, sql: str) -> object: + completed = subprocess.run( + [*shlex.split(command), "--tuples-only", "--no-align", "--quiet"], + input=sql, + text=True, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + check=False, + ) + if completed.returncode != 0: + raise RuntimeError(completed.stderr.strip()) + output = completed.stdout.strip() + return json.loads(output.splitlines()[-1]) if output else None + + +def psql_expect_error(command: str, sql: str, fragment: str, label: str) -> None: + completed = subprocess.run( + [*shlex.split(command), "--quiet", "--set", "ON_ERROR_STOP=1"], + input=sql, + text=True, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + check=False, + ) + refused = completed.returncode != 0 and fragment in completed.stderr + if not refused: + print( + f"prism postgres candidate gate: unexpected outcome for {label}: " + f"exit={completed.returncode} stderr={completed.stderr.strip()[:400]!r} " + f"stdout={completed.stdout.strip()[:200]!r}", + flush=True, + ) + check(refused, f"{label} ({fragment})") + + +def share(index: int) -> dict[str, object]: + return { + "share_seq": index, + "share_id": f"miner-{index % 5}:é{index}", + "miner_id": f"m{index % 5}", + "order_key": "k", + "p2mr_program_hex": "ab" * 32, + "share_difficulty": 10**20 + index, + "network_difficulty": 1000, + "template_height": 9, + "job_id": "job-a", + "job_issued_at_ms": 1_700_000_000_000 + index, + "accepted_at_ms": 1_700_000_000_001 + index, + "ntime": 1_700_000_000, + "credit_policy": None, + } + + +def intent(block_hash: str, share_count: int, *, credit: bool) -> dict[str, object]: + pending = { + "share_id": f"miner-0:{block_hash}", + "miner_id": "m0", + "order_key": "m0", + "p2mr_program_hex": "ab" * 32, + "share_difficulty": 100, + "network_difficulty": 1000, + "template_height": 9, + "job_id": "job-a", + "job_issued_at_ms": 1_700_000_000_000, + "accepted_at_ms": 1_700_000_000_500, + "ntime": 1_700_000_000, + "credit_policy": None, + } + return { + "schema": "qbit.prism.block-candidate-intent.v1", + "block_hash_hex": block_hash, + "block_hex": "00" * 200, + "coinbase_tx_hex": "01", + "parent_hash": "11" * 32, + "expected_height": 10, + "template": {"previousblockhash": "11" * 32, "height": 10, "coinbasevalue": 5}, + "shares_json": [share(index) for index in range(share_count)], + "prior_balances": [{"recipient_id": f"r{index}", "balance_sats": index} for index in range(3000)], + "found_block": {"network_difficulty": 1000}, + "prospective_prior_balances": None, + "witness_merkle_leaves_hex": ["cc" * 32] * 5, + "extranonce1_hex": "00", + "extranonce2_hex": "01", + "username": "miner-0", + "pending_share": pending, + "credit_share_on_accept": credit, + "collection_only": False, + } + + +def pending_share_for(fields: dict[str, object]) -> PendingShare: + return PendingShare(**dict(fields["pending_share"])) # type: ignore[arg-type] + + +def run_gate(psql_command: str, database_url: str | None, *, native: bool) -> None: + label = "native" if native else "psql" + ledger = PsqlShareLedger( + psql_command=psql_command, + database_url=database_url if native else None, + native_client_mode="1" if native else "0", + writer_id=f"writer-{label}", + writer_epoch=1, + initialize_schema=True, + candidate_spool_dir=os.environ.get("PRISM_TEST_SPOOL_DIR"), + ) + try: + # The migration is idempotent: apply it a second time through psql. + root = Path(__file__).resolve().parents[1] + psql_json(psql_command, (root / "crates/qbit-prism/sql/002_candidate_bodies.sql").read_text() + "\nSELECT json_build_object('ok', true);") + check(ledger.verify_candidate_schema() == {"declared": 2, "has_body_table": True}, f"{label}: schema capability declared") + + block_a = hashlib.sha256(f"{label}-a-{uuid.uuid4().hex}".encode()).hexdigest() + fields = intent(block_a, 3000, credit=True) + oracle_digest = block_candidate_identity_sha256(fields) + started = time.monotonic() + result = ledger.persist_block_candidate_intent(fields) + check(result.inserted and result.state == "pending", f"{label}: standalone persist publishes a chunked body") + again = ledger.persist_block_candidate_intent(intent(block_a, 3000, credit=True)) + check(not again.inserted and again.state == "pending", f"{label}: exact standalone replay is idempotent") + try: + ledger.persist_block_candidate_intent({**intent(block_a, 3000, credit=True), "username": "other"}) + check(False, f"{label}: differing payload is refused") + except RuntimeError as exc: + check("payload mismatch" in str(exc), f"{label}: differing payload is refused") + print(f"prism postgres candidate gate: staged 3000-share body in {time.monotonic() - started:.2f}s", flush=True) + + row = psql_json( + psql_command, + f"SELECT json_build_object('storage_version', storage_version, 'candidate_sha256', candidate_sha256, 'body_id', body_id, 'state', state, 'candidate_is_null', candidate IS NULL) FROM qbit_block_candidate_outbox WHERE block_hash = '{block_a}';", + ) + check(row["storage_version"] == 2 and row["candidate_is_null"] and row["candidate_sha256"] == oracle_digest, f"{label}: outbox row is version 2 with the historical digest") + body_id = row["body_id"] + manifest = psql_json(psql_command, f"SELECT json_build_object('state', state, 'chunk_count', chunk_count, 'span_count', span_count, 'page_count', page_count) FROM qbit_block_candidate_body WHERE body_id = '{body_id}';") + check(manifest["state"] == "sealed" and manifest["chunk_count"] >= 1 and manifest["span_count"] >= 2 and manifest["page_count"] >= 2, f"{label}: manifest is sealed with spans and pages") + + page = ledger.pending_block_candidate_headers(limit=32) + check(len(page.rows) == 1 and page.exhausted and page.rows[0]["block_hash"] == block_a, f"{label}: header page enumerates the row and answers exhaustion") + header = page.rows[0]["header"] + check(header["parent_hash"] == "11" * 32 and header["expected_height"] == 10 and header["pending_share"]["accepted_at_ms"] == 1_700_000_000_500 and header["oversized"] is False, f"{label}: bounded header carries the typed facts") + check(page.rows[0]["body"]["state"] == "sealed" and page.rows[0]["body"]["body_id"] == body_id, f"{label}: header carries the sealed body reference") + + hydrated = ledger.hydrate_block_candidate_intent(page.rows[0]) + check(len(hydrated["shares_json"]) == 3000 and hydrated["shares_json"][2999] == share(2999) and hydrated["shares_json"][0] == share(0), f"{label}: hydration reproduces the share array") + check(hydrated["pending_share"]["accepted_at_ms"] == 1_700_000_000_500 and hydrated.candidate_sha256 == oracle_digest, f"{label}: hydration restores the stamp and identity") + check(list(hydrated["prior_balances"]) == fields["prior_balances"], f"{label}: large balances decode through the spool adapter") + spool_path = hydrated.body.path + hydrated.body.close() + check(not os.path.exists(spool_path), f"{label}: spool is unlinked on close") + + # Immutability at the database boundary. + psql_expect_error(psql_command, f"INSERT INTO qbit_block_candidate_body_chunk (body_id, ordinal, chunk, chunk_sha256) VALUES ('{body_id}', 9999, '\\x00', '{'0' * 64}');", "accepts no parts", f"{label}: chunk insert after seal is refused") + psql_expect_error(psql_command, f"UPDATE qbit_block_candidate_body_chunk SET chunk = '\\x00' WHERE body_id = '{body_id}' AND ordinal = 0;", "immutable", f"{label}: chunk update is refused") + psql_expect_error(psql_command, f"DELETE FROM qbit_block_candidate_body_chunk WHERE body_id = '{body_id}' AND ordinal = 0;", "cannot be deleted", f"{label}: sealed chunk delete is refused") + psql_expect_error(psql_command, f"UPDATE qbit_block_candidate_body SET byte_count = byte_count + 1 WHERE body_id = '{body_id}';", "is sealed", f"{label}: sealed manifest scalars are immutable") + psql_expect_error(psql_command, f"UPDATE qbit_block_candidate_body SET state = 'retired', retired_at = clock_timestamp() WHERE body_id = '{body_id}';", "referenced by a pending outbox row", f"{label}: a referenced body cannot be retired") + psql_expect_error(psql_command, f"INSERT INTO qbit_block_candidate_body_page (body_id, field, page_ordinal, body_offset, record_index) VALUES ('{body_id}', 'x', 99999, 0, 0);", "accepts no parts", f"{label}: page insert after seal is refused") + check(ledger.retire_orphan_candidate_bodies() == (), f"{label}: orphan retirement leaves the referenced body alone") + + # Credit-on-accept: the share append joins the published body without re-uploading. + pending = pending_share_for(fields) + records = ledger.append_batch([(pending, intent(block_a, 3000, credit=True))]) + check(len(records) == 1 and records[0].candidate_outbox_state == "pending", f"{label}: credit append links the existing body") + linked = psql_json(psql_command, f"SELECT json_build_object('share_id', share_id, 'body_id', body_id) FROM qbit_block_candidate_outbox WHERE block_hash = '{block_a}';") + check(linked["share_id"] == pending.share_id and linked["body_id"] == body_id, f"{label}: outbox row carries the share and the same body") + replay = ledger.append_batch([(pending, intent(block_a, 3000, credit=True))]) + check(replay[0].share_seq == records[0].share_seq, f"{label}: exact replay of the credit append is idempotent") + try: + ledger.append_batch([(pending, {**intent(block_a, 3000, credit=True), "username": "other"})]) + check(False, f"{label}: differing credit payload is refused") + except RuntimeError as exc: + check("payload mismatch" in str(exc), f"{label}: differing credit payload is refused") + + # Async route: a fresh share with a new candidate in one fenced statement. + block_b = hashlib.sha256(f"{label}-b-{uuid.uuid4().hex}".encode()).hexdigest() + fields_b = intent(block_b, 700, credit=False) + records_b = ledger.append_batch([(pending_share_for(fields_b), fields_b)]) + check(records_b[0].candidate_outbox_state == "pending", f"{label}: share append publishes a new chunked candidate") + page = ledger.pending_block_candidate_headers(limit=1) + check(len(page.rows) == 1 and not page.exhausted, f"{label}: a full page is not exhaustion") + page2 = ledger.pending_block_candidate_headers(limit=1, after_cursor=page.next_cursor) + check(len(page2.rows) == 1 and page2.exhausted, f"{label}: the cursor walks to the last row and proves exhaustion") + + # Terminalization detaches the body; the janitor reclaims it in bounded steps. + check(ledger.mark_block_candidate_submitted(block_hash=block_a), f"{label}: terminal write succeeds") + detached = psql_json(psql_command, f"SELECT json_build_object('body_id', body_id, 'retired_body_id', retired_body_id, 'state', (SELECT state FROM qbit_block_candidate_body WHERE body_id = '{body_id}')) FROM qbit_block_candidate_outbox WHERE block_hash = '{block_a}';") + check(detached["body_id"] is None and detached["retired_body_id"] == body_id and detached["state"] == "retired", f"{label}: terminal write detaches and retires the body") + steps = 0 + reclaimed = False + while steps < 500: + outcome = ledger.reap_retired_candidate_bodies(max_chunks=2) + steps += 1 + if outcome["deleted_bodies"]: + reclaimed = True + break + if outcome["body_id"] is None: + break + remaining = psql_json(psql_command, f"SELECT json_build_object('body', EXISTS (SELECT 1 FROM qbit_block_candidate_body WHERE body_id = '{body_id}'), 'chunks', EXISTS (SELECT 1 FROM qbit_block_candidate_body_chunk WHERE body_id = '{body_id}'), 'pages', EXISTS (SELECT 1 FROM qbit_block_candidate_body_page WHERE body_id = '{body_id}'));") + check(reclaimed and not any(remaining.values()) and steps > 3, f"{label}: janitor reclaims the retired body in {steps} bounded steps") + + # A pre-#255 terminal write cannot touch a version-2 row (rollback floor). + psql_expect_error(psql_command, f"UPDATE qbit_block_candidate_outbox SET state = 'abandoned', completed_at = clock_timestamp(), candidate = NULL WHERE block_hash = '{block_b}';", "dual_format_check", f"{label}: legacy terminal write is refused by the schema") + check(ledger.mark_block_candidate_abandoned(block_hash=block_b, error="gate"), f"{label}: fenced abandon detaches the second body") + # Drain the second body too, then a step with nothing to do. + for _ in range(500): + if not ledger.reap_retired_candidate_bodies(max_chunks=64)["body_id"]: + break + check(ledger.reap_retired_candidate_bodies()["body_id"] is None, f"{label}: janitor idles with nothing retired") + finally: + ledger.release_writer_lease() + ledger.close() + + +def main() -> int: + psql_command = os.environ["PRISM_TEST_PSQL_COMMAND"] + database_url = os.environ.get("PRISM_TEST_DATABASE_URL") + run_gate(psql_command, database_url, native=False) + native_available = False + if database_url: + try: + import psycopg # noqa: F401 + + native_available = True + except ImportError: + print("prism postgres candidate gate: psycopg unavailable; native transport skipped", flush=True) + if native_available: + run_gate(psql_command, database_url, native=True) + print(f"prism postgres candidate gate: PASS ({len(CHECKS)} checks)", flush=True) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tests/prism_public_read_replica_gate.py b/tests/prism_public_read_replica_gate.py new file mode 100644 index 000000000..6b4133e0c --- /dev/null +++ b/tests/prism_public_read_replica_gate.py @@ -0,0 +1,288 @@ +#!/usr/bin/env python3 +"""Non-discovered public read replica integration gate (issue #145). + +Invoked by ``test/test-prism-public-read-replica.sh``. Lives outside unittest +discovery because it requires an explicitly provisioned primary plus streaming +hot-standby replica. Proves, against real containers (the #134 pattern): + +- the extracted public read service serves a moved endpoint with data that + reached it by streaming replication, and publishes the observed lag; +- the replica contract is enforced rather than described: a replication stream + silent past the configured bound answers 503, and a backing server that is + not in recovery is refused outright; +- the contract is scoped by what a route actually reads -- the environment-only + mining-configuration route keeps serving 200 while replica-backed routes + refuse. + +The unit-level route split and the freshness rules under a fake clock live in +tests/test_prism_public_read_service.py; this gate proves the container-side +serving contract those tests model. +""" + +from __future__ import annotations + +import contextlib +import json +import os +import shlex +import subprocess +import tempfile +import threading +import time +import urllib.error +import urllib.request +from http.server import ThreadingHTTPServer +from pathlib import Path + +from lab.prism import public_read_service + +PRIMARY_PSQL_COMMAND = os.environ.get("GATE_PRIMARY_PSQL_COMMAND", "") +REPLICA_PSQL_COMMAND = os.environ.get("GATE_REPLICA_PSQL_COMMAND", "") +if not PRIMARY_PSQL_COMMAND or not REPLICA_PSQL_COMMAND: + raise SystemExit( + "GATE_PRIMARY_PSQL_COMMAND and GATE_REPLICA_PSQL_COMMAND are required" + ) + +BLOCK_HASH = "aa" * 32 + + +class GateFailure(RuntimeError): + pass + + +def assert_equal(actual: object, expected: object, message: str) -> None: + if actual != expected: + raise GateFailure(f"{message}: expected {expected!r}, got {actual!r}") + + +def http_get(url: str) -> tuple[int, object, dict[str, str]]: + try: + with urllib.request.urlopen(url, timeout=10) as response: + body = response.read() + return ( + response.status, + json.loads(body) if body else None, + {k.lower(): v for k, v in response.headers.items()}, + ) + except urllib.error.HTTPError as exc: + body = exc.read() + return ( + exc.code, + json.loads(body) if body else None, + {k.lower(): v for k, v in exc.headers.items()}, + ) + + +def http_get_text(url: str) -> str: + with urllib.request.urlopen(url, timeout=10) as response: + return response.read().decode() + + +def run_psql(command: str, sql: str) -> None: + process = subprocess.run( + shlex.split(command) + ["-v", "ON_ERROR_STOP=1", "-c", sql], + capture_output=True, + text=True, + timeout=60, + ) + if process.returncode != 0: + raise GateFailure( + f"psql failed ({process.returncode}): {process.stderr.strip()}" + ) + + +def psql_scalar(command: str, sql: str) -> str | None: + """Scalar result, or None when the query could not run. + + None rather than an exception because callers poll for state that is still + replicating: before the DDL replays, the query fails with "relation does + not exist", which is a "not yet", not a gate failure. + """ + process = subprocess.run( + shlex.split(command) + ["-tA", "-c", sql], + capture_output=True, + text=True, + timeout=60, + ) + if process.returncode != 0: + return None + return process.stdout.strip() + + +def wait_for_replica_row(deadline_seconds: float = 90.0) -> None: + """Poll the standby until the row arrives, tolerating the schema's absence. + + Both the schema and the row reach the standby by streaming replication -- + the shell wrapper starts the replica before this gate runs, so the standby + was cloned from a primary that had neither. Until the CREATE TABLE replays, + the count query fails rather than returning 0, so a failed query has to be + treated as "not yet" inside the deadline instead of aborting the gate. + """ + deadline = time.monotonic() + deadline_seconds + last_seen: str | None = None + while time.monotonic() < deadline: + last_seen = psql_scalar( + REPLICA_PSQL_COMMAND, + "SELECT count(*) FROM qbit_pool_blocks " + f"WHERE block_hash = '{BLOCK_HASH}'", + ) + if last_seen == "1": + return + time.sleep(0.5) + raise GateFailure( + f"replica did not replay the qbit_pool_blocks row within " + f"{deadline_seconds}s (last count query returned {last_seen!r})" + ) + + +GATE_ENVIRONMENT = { + "QBIT_RPC_HOST": "127.0.0.1", + "QBIT_RPC_PORT": "18452", + "QBIT_RPC_USER": "gate", + "QBIT_RPC_PASSWORD": "gate", + # Required at startup: this process runs no Stratum listener. + "PRISM_PUBLIC_STRATUM_URL": "stratum+tcp://gate.invalid:3340", + # psql transport, because the DSN is a `docker exec` into the container. + "PRISM_POSTGRES_NATIVE_CLIENT": "0", + "PRISM_DATABASE_URL": "", + "PRISM_PUBLIC_REPLICA_MODE": "require", +} + + +@contextlib.contextmanager +def serving(**overrides: str): + """Build the real service from the environment and serve it on a free port. + + Goes through build_service() rather than assembling the collaborators by + hand: the point of a container gate is that the process an operator starts + is the process under test, including every refusal build_service makes. + """ + audit_dir = Path(tempfile.mkdtemp(prefix="prism-public-read-gate-")) + environment = dict(GATE_ENVIRONMENT) + environment["PRISM_AUDIT_DIR"] = str(audit_dir) + environment["PRISM_POSTGRES_PSQL_COMMAND"] = REPLICA_PSQL_COMMAND + environment.update(overrides) + + previous = dict(os.environ) + os.environ.update(environment) + try: + coordinator, readiness, metrics, replica = public_read_service.build_service() + finally: + os.environ.clear() + os.environ.update(previous) + + readiness.start() + service = public_read_service.PublicReadService( + coordinator, + metrics=metrics, + readiness=readiness, + replica=replica, + ) + server = ThreadingHTTPServer( + ("127.0.0.1", 0), public_read_service.make_handler(service) + ) + server.daemon_threads = True + thread = threading.Thread(target=server.serve_forever, daemon=True) + thread.start() + try: + yield f"http://127.0.0.1:{server.server_address[1]}" + finally: + server.shutdown() + server.server_close() + readiness.stop() + thread.join(timeout=5) + + +def check_serves_replicated_data() -> None: + with serving() as base_url: + status, payload, _ = http_get(f"{base_url}/healthz") + assert_equal(status, 200, "health status") + assert_equal(payload["ok"], True, "health ok") + assert_equal(payload["replica"]["in_recovery"], True, "in_recovery") + if payload["replica"]["receiver_heartbeat_age_seconds"] is None: + raise GateFailure("health reported no walreceiver heartbeat") + + status, payload, headers = http_get(f"{base_url}/public/v1/blocks") + assert_equal(status, 200, "blocks status") + assert_equal(payload["schema"], "prism.dashboard.blocks.v1", "blocks schema") + assert_equal(payload["pagination"]["total_count"], 1, "blocks total_count") + assert_equal(payload["rows"][0]["hash"], BLOCK_HASH, "blocks row hash") + if public_read_service.REPLICA_LAG_HEADER.lower() not in headers: + raise GateFailure( + f"public response missing {public_read_service.REPLICA_LAG_HEADER}" + ) + + metrics = http_get_text(f"{base_url}/metrics") + for gauge in ( + "qbit_prism_public_replica_in_recovery 1", + "qbit_prism_public_replica_heartbeat_age_seconds", + "qbit_prism_public_replica_replay_lag_seconds", + "qbit_prism_public_replica_apply_backlog_bytes", + ): + if gauge not in metrics: + raise GateFailure(f"metrics missing {gauge!r}") + + +def check_refuses_past_the_bound() -> None: + """Any live heartbeat is older than this bound, so every probe fails it.""" + + with serving(PRISM_PUBLIC_REPLICA_MAX_LAG_SECONDS="0.0001") as base_url: + status, payload, _ = http_get(f"{base_url}/public/v1/blocks") + assert_equal(status, 503, "degraded blocks status") + assert_equal(payload["error"]["code"], "upstream_unavailable", "degraded code") + if "silent for" not in payload["error"]["message"]: + raise GateFailure(f"unexpected degraded message: {payload}") + + status, _, _ = http_get(f"{base_url}/healthz") + assert_equal(status, 503, "degraded health status") + + # Scoped by what the route reads: mining-configuration is assembled + # from the environment and touches no read slot, so the replica being + # stale is not a reason to refuse it. + status, payload, _ = http_get(f"{base_url}/public/v1/mining-configuration") + assert_equal(status, 200, "mining-configuration under a stale replica") + + +def check_refuses_a_writable_primary() -> None: + with serving(PRISM_POSTGRES_PSQL_COMMAND=PRIMARY_PSQL_COMMAND) as base_url: + status, payload, _ = http_get(f"{base_url}/public/v1/blocks") + assert_equal(status, 503, "primary refusal status") + if "not in recovery" not in payload["error"]["message"]: + raise GateFailure(f"unexpected primary refusal message: {payload}") + + status, _, _ = http_get(f"{base_url}/healthz") + assert_equal(status, 503, "primary refusal health status") + + +def main() -> int: + # The schema is created on the primary *after* the standby was cloned -- + # the shell wrapper brings the replica up before invoking this gate -- so + # both the DDL and the row below reach the standby by streaming, which is + # what wait_for_replica_row polls for. + run_psql( + PRIMARY_PSQL_COMMAND, + Path("crates/qbit-prism/sql/001_share_ledger.sql").read_text(encoding="utf-8"), + ) + run_psql( + PRIMARY_PSQL_COMMAND, + f""" +INSERT INTO qbit_pool_blocks + (block_hash, block_height, parent_hash, coinbase_txid, + payout_manifest_sha256, chain_state) +VALUES ('{BLOCK_HASH}', 101, '{"bb" * 32}', '{"cc" * 32}', + '{"dd" * 32}', 'confirmed') +ON CONFLICT (block_hash) DO NOTHING; +""", + ) + wait_for_replica_row() + + check_serves_replicated_data() + check_refuses_past_the_bound() + check_refuses_a_writable_primary() + + print("public read replica gate: OK") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tests/prism_regtest_crash_coordinator.py b/tests/prism_regtest_crash_coordinator.py new file mode 100644 index 000000000..d013491ce --- /dev/null +++ b/tests/prism_regtest_crash_coordinator.py @@ -0,0 +1,73 @@ +#!/usr/bin/env python3 +"""Crash once after durable candidate publication and before its first node offer. + +Only the disposable regtest harness launches this entry point. The successor +uses the normal coordinator entry point and discovers the candidate from SQL. +""" + +from __future__ import annotations + +import json +import os +from pathlib import Path +import sys + +sys.path.insert(0, str(Path(__file__).resolve().parents[1])) + +import psycopg + +from lab.prism.block_candidates import BlockCandidateService +from lab.prism.prism_coordinator import main + + +def crash_before_offer(self, candidate): + block_hash = candidate.submission.block_hash_hex.lower() + with psycopg.connect(os.environ["PRISM_DATABASE_URL"]) as connection: + row = connection.execute(""" + SELECT o.candidate_sha256, o.share_id, s.writer_epoch, + COALESCE((to_jsonb(o)->>'storage_version')::integer, 1), + to_jsonb(o)->>'body_id' + FROM qbit_block_candidate_outbox o + JOIN qbit_share_ledger s ON s.share_id = o.share_id + WHERE o.block_hash = %s AND o.state = 'pending' AND s.accepted + """, (block_hash,)).fetchone() + if row is None: + raise RuntimeError("crash fixture reached node offer before atomic share/outbox durability") + requested_version = os.environ.get("PRISM_CANDIDATE_STORAGE_VERSION") + if requested_version is not None and row[3] != int(requested_version): + raise RuntimeError("crash fixture persisted a different candidate storage version") + if row[3] == 2: + body = connection.execute(""" + SELECT b.state, b.candidate_sha256, b.chunk_count, b.byte_count, + count(c.ordinal), COALESCE(sum(octet_length(c.chunk)), 0), + min(c.ordinal), max(c.ordinal), + bool_and(sha256(c.chunk) = decode(c.chunk_sha256, 'hex')) + FROM qbit_block_candidate_body b + LEFT JOIN qbit_block_candidate_body_chunk c USING (body_id) + WHERE b.body_id = %s + GROUP BY b.body_id + """, (row[4],)).fetchone() + if body is None or not ( + body[0] == "sealed" and body[1] == row[0] + and body[2] == body[4] and body[3] == body[5] + and body[6] == 0 and body[7] == body[2] - 1 and body[8] is True + ): + raise RuntimeError("crash fixture found an incomplete published candidate body") + marker = Path(sys.argv[1]) + with marker.open("x", encoding="utf-8") as handle: + json.dump({ + "block_hash": block_hash, "candidate_sha256": row[0], + "share_id": row[1], "writer_epoch": row[2], + "storage_version": row[3], "body_id": row[4], + }, handle) + handle.flush() + os.fsync(handle.fileno()) + print(f"regtest intentional exit after durable candidate {block_hash}", flush=True) + os._exit(75) + + +if __name__ == "__main__": + if len(sys.argv) != 2: + raise SystemExit("requires the disposable crash-marker path") + BlockCandidateService._submit_block_candidate_to_node = crash_before_offer + raise SystemExit(main()) diff --git a/tests/prism_regtest_share_miner.py b/tests/prism_regtest_share_miner.py new file mode 100644 index 000000000..f9e37e353 --- /dev/null +++ b/tests/prism_regtest_share_miner.py @@ -0,0 +1,72 @@ +#!/usr/bin/env python3 +"""Seed a real regtest payout window with one acknowledged non-block share. + +The live harness subsequently runs the ordinary block-solving miner. Keeping +this fixture separate leaves the deployed miner simulator's behavior intact. +""" + +from __future__ import annotations + +import importlib.util +from pathlib import Path +import sys +import time + + +def load_miner(): + path = Path(__file__).resolve().parents[1] / "lab/miner-sim/miner_sim.py" + spec = importlib.util.spec_from_file_location("prism_fixture_miner", path) + assert spec is not None and spec.loader is not None + module = importlib.util.module_from_spec(spec) + sys.modules[spec.name] = module + spec.loader.exec_module(module) + return module + + +def solve_share(miner, job, extranonce1, extranonce2_size, share_target, deadline): + block_target = miner.target_from_compact(job.nbits) + if share_target <= block_target: + raise RuntimeError("primed regtest requires a share target easier than the block target") + for extranonce2 in range(1 << (8 * extranonce2_size)): + extranonce2_hex = f"{extranonce2:0{extranonce2_size * 2}x}" + _, raw = miner.assemble_header(job, extranonce1, extranonce2_hex, "00000000") + header = bytearray(raw) + for nonce in range(0x100000000): + if nonce % 4096 == 0 and time.time() >= deadline: + raise miner.RecoverableError("ordinary-share fixture timed out") + header[76:80] = nonce.to_bytes(4, "little") + value = int.from_bytes(miner.double_sha256(header), "little") + if block_target < value <= share_target: + return extranonce2_hex, f"{nonce:08x}" + raise RuntimeError("ordinary-share fixture exhausted the nonce space") + + +def main() -> int: + miner = load_miner() + username = miner.resolve_miner_username() + deadline = time.time() + miner.MINER_TIMEOUT_SECONDS + while time.time() < deadline: + client = miner.StratumClient(miner.STRATUM_HOST, miner.STRATUM_PORT, deadline) + try: + extranonce1, size, job, target, _ = miner.receive_handshake_and_job(client, username) + extranonce2, nonce = solve_share(miner, job, extranonce1, size, target, deadline) + request_id = client.send("mining.submit", [ + username, job.job_id, extranonce2, job.ntime, nonce, + ]) + while time.time() < deadline: + message = client.recv() + if message.get("id") != request_id: + continue + if message.get("result") is True: + print("ordinary share durably accepted", flush=True) + return 0 + raise miner.RecoverableError(str(message.get("error") or "share rejected")) + except miner.RecoverableError as exc: + print(f"ordinary-share fixture retry: {exc}", flush=True) + finally: + client.close() + raise SystemExit("ordinary-share fixture timed out without a durable acknowledgment") + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tests/prism_vardiff_test_support.py b/tests/prism_vardiff_test_support.py new file mode 100644 index 000000000..806a191fe --- /dev/null +++ b/tests/prism_vardiff_test_support.py @@ -0,0 +1,1053 @@ +#!/usr/bin/env python3 +# ruff: noqa: F401 +"""Shared no-environment fixtures for coordinator domain tests.""" + +from __future__ import annotations + +import errno +import hashlib +import inspect +import json +import os +import queue +import socket +import subprocess +import tempfile +import contextlib +import threading +import time +import unittest +from concurrent.futures import Future +from dataclasses import replace as dataclass_replace +from decimal import Decimal +from pathlib import Path +from types import SimpleNamespace +from unittest.mock import patch + +from lab.auxpow import vardiff +from lab.prism import direct_stratum +from lab.prism.share_ledger import ( + LedgerOperationTimeout, + PendingShare, + PsqlShareLedger, + ShareReplayConflict, + ShareReplayResult, + SingleWriterShareLedger, + WRITER_LEASE_HEARTBEAT_SESSION_PREFIX, +) +from lab.prism.prism_coordinator import ( + CachedJobBundle, + CachedTemplateArtifacts, + ClientState, + MAX_PENDING_SHARE_APPENDS, + PRISM_CREDIT_POLICY_STALE_GRACE, + PRISM_REJECTION_BACKEND_RPC_UNAVAILABLE, + PRISM_REJECTION_BLOCK_ACCEPT_PENDING, + PRISM_REJECTION_CANDIDATE_AUDIT_MISMATCH, + PRISM_REJECTION_DUPLICATE_SHARE, + PRISM_REJECTION_INVALID_NTIME_OR_NONCE, + PRISM_REJECTION_LEDGER_CONFIRMATION_FAILED, + PRISM_REJECTION_LEDGER_CONFIRMATION_SUPERSEDED, + PRISM_REJECTION_LOW_DIFFICULTY, + PendingShareAppend, + PayoutStateArtifact, + PrismBlockCandidate, + PRISM_REJECTION_POOL_CLOSED, + PRISM_REJECTION_REASON_IDS, + PRISM_REJECTION_STALE_JOB, + PRISM_REJECTION_SUBMITBLOCK_REJECTED, + PRISM_REJECTION_UNAUTHORIZED_WORKER, + PRISM_REJECTION_UNKNOWN_JOB, + PRISM_WORKER_METRICS_OVERFLOW_LABEL, + QbitTipTemplateSnapshot, + StratumError, + StratumListenerProfile, + JobBuildSuperseded, + TemplateRefreshBlocked, + TemplateRefreshSuperseded, + PrismCoordinator, + ShutdownInProgress, + WorkerIdentity, + WriterLeaseRenewalDeferred, + _FanoutCancellation, + _ObservedRLock, + _JobBuildCancellation, + qbit_template_fingerprint, + qbit_gbt_rules, + scaled_target_difficulty, + target_from_compact, +) +from lab.prism.reorg_reconciler import _ReconcileFlight +from lab.prism.coordinator_config import ( + DEFAULT_TESTNET_USERNAME_FALLBACK_ADDRESS, + default_prism_coinbase_tag_hex, + default_prism_username_fallback_address, + env_positive_float, + load_prism_highdiff_listener, + load_prism_vardiff_config, + validate_prism_production_gate, + validate_same_tip_job_retention_limits, +) +from lab.prism.job_bundle import JobBuildCancelled +from lab.prism.job_delivery import MAX_ACTIVE_PRISM_JOBS_PER_CLIENT +from lab.prism.payout_state import ( + PayoutStatePublicationBlocked as _PayoutStatePublicationBlocked, +) +from lab.prism.stratum_session import parse_stratum_password_options + + +PAYOUT_ADDRESS = "tq1z70ukpvs96kye6jmgvl3nttevtkrq8uu89snkpm6m8gwqukw8u5dsz32kwa" + + +def fake_audit_bundle_popen( + captured: dict[str, object], + *, + output_text: str = '{"ok":true}', + returncode: int = 0, + stderr_text: str = "", +) -> type: + class FakeStdin: + def __init__(self) -> None: + self.parts: list[str] = [] + + def write(self, value: str) -> int: + self.parts.append(value) + return len(value) + + def close(self) -> None: + return None + + class FakePopen: + def __init__(self, cmd: list[str], **kwargs: object) -> None: + captured["cmd"] = cmd + self.stdin = FakeStdin() + self.stdout = kwargs["stdout"] + self.stderr = kwargs["stderr"] + + def wait(self, timeout: float | None = None) -> int: + captured["timeout"] = timeout + captured["payload"] = json.loads("".join(self.stdin.parts)) + self.stdout.write(output_text) + self.stderr.write(stderr_text) + return returncode + + return FakePopen + + +def tx_output(value_sats: int, script_hex: str) -> str: + return value_sats.to_bytes(8, "little").hex() + direct_stratum.compact_size(len(bytes.fromhex(script_hex))).hex() + script_hex + + +def synthetic_witness_transaction(seed: str) -> str: + script = seed * 3 + witness_item = seed * 5 + return ( + "01000000" + + "0001" + + "01" + + (seed * 32) + + "00000000" + + direct_stratum.compact_size(len(bytes.fromhex(script))).hex() + + script + + "ffffffff" + + "01" + + tx_output(1, "51") + + "01" + + direct_stratum.compact_size(len(bytes.fromhex(witness_item))).hex() + + witness_item + + "00000000" + ) + + +class FakeJob: + def __init__(self, difficulty: Decimal) -> None: + self.share_difficulty = difficulty + + +class FakeLedger: + backend_name = "fake" + + def __init__(self, shares: int = 0, prior_balances: list[dict[str, object]] | None = None) -> None: + self.shares = shares + self.prior_balances = prior_balances or [] + + def all_shares(self) -> list[object]: + return [object()] * self.shares + + def current_prior_balances(self) -> list[dict[str, object]]: + return [dict(balance) for balance in self.prior_balances] + + def metrics(self) -> dict[str, int]: + return {"blocks": 2, "owed_accounts": 3} + + def reorg_watch_blocks(self, *, active_tip_height: int) -> list[dict[str, object]]: + return [] + + def mark_pool_block_inactive(self, *, block_hash: str, active_tip_height: int) -> dict[str, object]: + return {"backend": "fake", "inactive_count": 0} + + def reject_prepared_block(self, *, block_hash: str, active_tip_height: int) -> dict[str, object]: + return {"backend": "fake", "rejected_count": 0} + + def reactivate_pool_block(self, *, block_hash: str, active_tip_height: int) -> dict[str, object]: + return {"backend": "fake", "reactivated_count": 0} + + def mark_mature_pool_payouts(self, *, active_tip_height: int) -> dict[str, object]: + return {"backend": "fake", "matured_count": 0} + + +class RecordingLedger(FakeLedger): + def __init__(self) -> None: + super().__init__(shares=0) + self.pending: list[object] = [] + self.persisted: list[dict[str, object]] = [] + self.confirmed: list[dict[str, object]] = [] + self.reversed: list[dict[str, object]] = [] + self.rejected: list[dict[str, object]] = [] + self.submit_seen = False + self._audit_publication_sequences: dict[str, int] = {} + + def append(self, pending: object) -> object: + self.pending.append(pending) + self.shares += 1 + return SimpleNamespace(share_seq=self.shares, miner_id=getattr(pending, "miner_id", "miner-a")) + + def append_recovered_share(self, pending: PendingShare) -> ShareReplayResult: + for index, existing in enumerate(self.pending, start=1): + if getattr(existing, "share_id", None) != pending.share_id: + continue + if existing != pending: + raise ShareReplayConflict(pending.share_id) + return ShareReplayResult( + "exact_existing", + SimpleNamespace(share_seq=index, miner_id=pending.miner_id), + ) + return ShareReplayResult("inserted", self.append(pending)) + + def persist_accepted_block(self, **kwargs: object) -> dict[str, object]: + self.persisted.append({**kwargs, "submit_seen_at_persist": self.submit_seen}) + return { + "backend": "fake", + "share_count": self.shares, + "block_count": 1, + "bundle_count": 1, + "payout_entry_count": 0, + "carry_forward_count": 0, + "onchain_output_count": 1, + } + + def reverse_immature_block(self, **kwargs: object) -> dict[str, object]: + self.reversed.append(kwargs) + return {"backend": "fake", "reversed_count": 1} + + def reject_prepared_block(self, **kwargs: object) -> dict[str, object]: + self.rejected.append(kwargs) + return {"backend": "fake", "rejected_count": 1} + + def confirm_accepted_block(self, **kwargs: object) -> dict[str, object]: + self.confirmed.append({**kwargs, "submit_seen_at_confirm": self.submit_seen}) + block_hash = str(kwargs.get("block_hash") or "") + sequence = self._audit_publication_sequences.get(block_hash) + already_confirmed = sequence is not None + if sequence is None: + sequence = len(self._audit_publication_sequences) + 1 + self._audit_publication_sequences[block_hash] = sequence + return { + "backend": "fake", + # Mirror the production disposition split: 1 is the fresh flip + # that allocates the ordinal, 2 the idempotent replay of a row + # this writer already confirmed, which allocates none. + "confirmed_count": 2 if already_confirmed else 1, + "audit_publication_sequence": sequence, + } + + def audit_publication_sequence_floor(self) -> int: + return max(self._audit_publication_sequences.values(), default=0) + + def all_shares(self) -> list[object]: + return [ + SimpleNamespace(miner_id=getattr(pending, "miner_id", "miner-a")) + for pending in self.pending + ] + + +class FakeRpc: + def call(self, method: str, params: list[object] | None = None) -> object: + if method == "getblockchaininfo": + return {"initialblockdownload": False} + if method == "getnetworkinfo": + return {"connections": 4} + raise RuntimeError(method) + + +class FeeEstimateRpc(FakeRpc): + def __init__(self, estimate: object) -> None: + self.estimate = estimate + self.calls: list[tuple[str, list[object] | None]] = [] + + def call(self, method: str, params: list[object] | None = None) -> object: + self.calls.append((method, params)) + if method == "estimatesmartfee": + return self.estimate + return super().call(method, params) + + +class TemplateRpc(FakeRpc): + def __init__(self, template: object) -> None: + self.template = template + self.calls: list[tuple[str, list[object] | None]] = [] + + def call(self, method: str, params: list[object] | None = None) -> object: + self.calls.append((method, params)) + if method == "getblocktemplate": + return self.template + return super().call(method, params) + + +class AddressValidationRpc(FakeRpc): + def __init__( + self, + *, + valid_address: str = PAYOUT_ADDRESS, + script_byte: str = "11", + p2mr: bool = True, + ) -> None: + self.valid_address = valid_address + self.script_byte = script_byte + self.p2mr = p2mr + self.validated: list[str] = [] + + def call(self, method: str, params: list[object] | None = None) -> object: + if method == "validateaddress": + address = str((params or [""])[0]) + self.validated.append(address) + script = "5220" + self.script_byte * 32 if self.p2mr else "51" + return {"isvalid": address == self.valid_address, "scriptPubKey": script} + return super().call(method, params) + + +AddressRpc = AddressValidationRpc + + +class TipRpc(FakeRpc): + def __init__(self, tip: str) -> None: + self.tip = tip + + def call(self, method: str, params: list[object] | None = None) -> object: + if method == "getbestblockhash": + return self.tip + if method == "getblockheader": + raise RuntimeError("qbit RPC getblockheader failed: -5 Block not found") + return super().call(method, params) + + +class RejectingSubmitTipRpc(TipRpc): + def call(self, method: str, params: list[object] | None = None) -> object: + if method == "submitblock": + return "bad-prevblk" + return super().call(method, params) + + +class ParentTipRpc(TipRpc): + def __init__(self, *, tip: str, parent: str) -> None: + super().__init__(tip) + self.parent = parent + self.submitblock_calls = 0 + + def call(self, method: str, params: list[object] | None = None) -> object: + if method == "getblock": + self.assert_tip_param(params) + return {"hash": self.tip, "previousblockhash": self.parent} + if method == "submitblock": + self.submitblock_calls += 1 + return None + return super().call(method, params) + + def assert_tip_param(self, params: list[object] | None) -> None: + if not params or str(params[0]) != self.tip: + raise AssertionError(f"expected getblock current tip {self.tip}, got {params!r}") + + +class UnsupportedBlockwaitRpc(TipRpc): + def call( + self, + method: str, + params: list[object] | None = None, + *, + timeout: float | None = None, + ) -> object: + if method == "waitfornewblock": + raise RuntimeError("Method not found") + return super().call(method, params) + + +class TipTemplateRpc(FakeRpc): + def __init__(self, *, tip: str, template: dict[str, object]) -> None: + self.tip = tip + self.template = template + self.calls: list[tuple[str, list[object] | None]] = [] + + def call(self, method: str, params: list[object] | None = None) -> object: + self.calls.append((method, params)) + if method == "getbestblockhash": + return self.tip + if method == "getblocktemplate": + return self.template + return super().call(method, params) + + +class ReorgRpc(TipTemplateRpc): + def __init__( + self, + *, + tip: str, + template: dict[str, object], + height: int, + block_hashes: dict[int, str], + initialblockdownload: bool = False, + headers: int | None = None, + ) -> None: + super().__init__(tip=tip, template=template) + self.height = height + self.block_hashes = block_hashes + self.initialblockdownload = initialblockdownload + self.headers = headers if headers is not None else height + + def call(self, method: str, params: list[object] | None = None) -> object: + if method == "getblockchaininfo": + return { + "initialblockdownload": self.initialblockdownload, + "blocks": self.height, + "headers": self.headers, + } + if method == "getblockcount": + return self.height + if method == "getblockhash": + height = int((params or [0])[0]) + try: + return self.block_hashes[height] + except KeyError as exc: + raise RuntimeError(f"unknown height {height}") from exc + return super().call(method, params) + + +class ReorgLedger(FakeLedger): + def __init__(self, rows: list[dict[str, object]]) -> None: + super().__init__(shares=0) + self.rows = [dict(row) for row in rows] + self.events: list[object] = [] + + def reorg_watch_blocks(self, *, active_tip_height: int) -> list[dict[str, object]]: + self.events.append(("watch", active_tip_height)) + return [dict(row) for row in self.rows] + + def mark_pool_block_inactive(self, *, block_hash: str, active_tip_height: int) -> dict[str, object]: + self.events.append(("inactive", block_hash, active_tip_height)) + for row in self.rows: + if str(row.get("block_hash", "")).lower() == block_hash.lower(): + row["chain_state"] = "inactive" + return {"backend": "fake", "inactive_count": 1} + return {"backend": "fake", "inactive_count": 0} + + def reactivate_pool_block(self, *, block_hash: str, active_tip_height: int) -> dict[str, object]: + self.events.append(("reactivate", block_hash, active_tip_height)) + for row in self.rows: + if str(row.get("block_hash", "")).lower() == block_hash.lower(): + row["chain_state"] = "confirmed" + return {"backend": "fake", "reactivated_count": 1} + return {"backend": "fake", "reactivated_count": 0} + + def mark_mature_pool_payouts(self, *, active_tip_height: int) -> dict[str, object]: + self.events.append(("mature", active_tip_height)) + return {"backend": "fake", "matured_count": 0} + + +class SubmitRpc(FakeRpc): + def __init__( + self, + *, + tip: str, + block_hash: str, + submit_result: object = None, + ledger: RecordingLedger | None = None, + ) -> None: + self.tip = tip + self.block_hash = block_hash + self.submit_result = submit_result + self.ledger = ledger + self.height = 9 + self.submitted = False + + def call(self, method: str, params: list[object] | None = None) -> object: + if method == "getbestblockhash": + return self.tip + if method == "getblockcount": + return self.height + if method == "submitblock": + self.submitted = True + if self.ledger is not None: + self.ledger.submit_seen = True + if self.submit_result is None: + self.height += 1 + return self.submit_result + if method == "getblockhash": + return self.block_hash + return super().call(method, params) + + +class SubmitAcceptingTemplateRpc(FakeRpc): + def __init__( + self, + *, + old_tip: str, + block_hash: str, + fail_template_after_submit: bool = False, + ledger: RecordingLedger | None = None, + ) -> None: + self.old_tip = old_tip + self.block_hash = block_hash + self.fail_template_after_submit = fail_template_after_submit + self.ledger = ledger + self.height = 9 + self.submitted = False + + def call(self, method: str, params: list[object] | None = None) -> object: + if method == "getbestblockhash": + return self.block_hash if self.submitted else self.old_tip + if method == "getblockcount": + return self.height + if method == "submitblock": + self.submitted = True + self.height += 1 + if self.ledger is not None: + self.ledger.submit_seen = True + return None + if method == "getblockhash": + return self.block_hash + if method == "getblocktemplate": + if self.submitted and self.fail_template_after_submit: + raise RuntimeError("transient getblocktemplate failure after submitblock") + previousblockhash = self.block_hash if self.submitted else self.old_tip + return gbt_template(previousblockhash, height=self.height + 1) + return super().call(method, params) + + +def client() -> ClientState: + state = ClientState(sock=object(), address=("127.0.0.1", 1), connection_id=1, extranonce1_hex="00000001") + state.subscribed = True + state.authorized = True + return state + + +def gbt_template( + previousblockhash: str, + *, + height: int = 10, + coinbasevalue: int = 50_00000000, + curtime: int = 1_700_000_000, + transactions: list[str] | None = None, +) -> dict[str, object]: + return { + "previousblockhash": previousblockhash, + "version": 0x20000000, + "bits": "207fffff", + "curtime": curtime, + "height": height, + "coinbasevalue": coinbasevalue, + "transactions": [{"data": tx_hex} for tx_hex in transactions or []], + } + + +def worker_identity(username: str = "miner-a") -> WorkerIdentity: + return WorkerIdentity( + username=username, + payout_address=username, + worker_name=None, + script_pubkey_hex="5220" + "11" * 32, + p2mr_program_hex="11" * 32, + ) + + +def stratum_job( + job_id: str, + *, + difficulty: Decimal = Decimal("1"), + clean_jobs: bool = True, +) -> SimpleNamespace: + return SimpleNamespace( + job_id=job_id, + share_difficulty=difficulty, + share_target=target_from_compact("207fffff"), + prevhash="00" * 32, + coinb1="", + coinb2="", + merkle_branch=(), + version="20000000", + nbits="207fffff", + ntime="6553f100", + clean_jobs=clean_jobs, + transaction_hexes=(), + ) + + +def prism_context( + job_id: str, + previousblockhash: str, + *, + worker: WorkerIdentity | None = None, + difficulty: Decimal = Decimal("1"), + clean_jobs: bool = True, +) -> SimpleNamespace: + return SimpleNamespace( + job=stratum_job(job_id, difficulty=difficulty, clean_jobs=clean_jobs), + template=gbt_template(previousblockhash), + found_block={"network_difficulty": 1}, + issued_at_ms=12345, + collection_only=False, + worker=worker or worker_identity(), + shares_json=[], + prior_balances=[], + ) + + +def prepare_idle_client( + server: PrismCoordinator, + state: ClientState, + *, + connection_id: int = 1, + difficulty: Decimal = Decimal("16"), + tip: str = "00" * 32, +) -> None: + if not hasattr(server, "jobs"): + server.jobs = {} + state.connection_id = connection_id + state.extranonce1_hex = f"{connection_id:08x}" + state.username = f"miner-{connection_id}" + state.worker = worker_identity(state.username) + job_id = f"job-{connection_id}" + state.active_job = prism_context( + job_id, + tip, + worker=state.worker, + difficulty=difficulty, + ) + state.active_job_ids = {job_id} + state.share_difficulty = difficulty + state.pending_share_difficulty = None + state.vardiff_window_started_monotonic = time.monotonic() - 2 + state.vardiff_window_accepted = 0 + state.vardiff_window_submitted = 0 + state.vardiff_window_work = Decimal("0") + server.clients.add(state) + server.jobs[job_id] = state.active_job + + +def install_idle_job_cache( + server: PrismCoordinator, + *, + tip: str = "00" * 32, +) -> CachedJobBundle: + server._pool_ready_latched = True + server.job_bundle_cache_seconds = 60.0 + server.job_counter = 0 + server.share_weights_by_username = {} + server.default_share_weight = 1 + server._ensure_job_cache_state() + template = gbt_template(tip) + fingerprint = qbit_template_fingerprint(template) + artifacts = CachedTemplateArtifacts( + template=template, + fingerprint=fingerprint, + previousblockhash=tip, + transaction_hexes=(), + witness_merkle_leaves_hex=(), + network_difficulty=1, + fetched_monotonic=time.monotonic(), + generation=1, + ) + key = server._job_bundle_key( + artifacts, + mode="ready", + payout_state_generation=server._payout_state_generation, + payout_artifact_generation=0, + worker=None, + ) + payout_artifact_sha256 = "aa" * 32 + qbit_target = direct_stratum.difficulty_target(Decimal("1024")) + base_job = direct_stratum.DirectQbitStratumJob( + job_id="prism-template-base", + previousblockhash_display=tip, + prevhash=tip, + coinb1="", + coinb2="", + full_coinbase_prefix="", + full_coinbase_suffix="", + merkle_branch=(), + transaction_hexes=(), + version="20000000", + nbits="207fffff", + ntime="6553f100", + qbit_target=qbit_target, + share_target=qbit_target, + share_difficulty=Decimal("1"), + extranonce1_hex="ffffffff", + extranonce2_size=8, + clean_jobs=True, + ) + bundle = CachedJobBundle( + key=key, + template=template, + template_fingerprint=fingerprint, + coinbase_manifest={}, + shares_json=[], + prior_balances=[], + found_block={"network_difficulty": 1}, + collection_only=False, + issued_at_ms=12345, + base_job=base_job, + built_monotonic=time.monotonic(), + template_generation=1, + payout_state_generation=0, + build_key=SimpleNamespace( + payout_artifact_sha256=payout_artifact_sha256, + payout_append_invalidation_epoch=( + server._payout_ledger_append_invalidation_epoch + ), + ), + ) + with server._job_cache_lock: + server._template_artifacts = artifacts + server._published_payout_state = dataclass_replace( + server._published_payout_state, + artifact=PayoutStateArtifact( + generation=server._payout_state_generation, + source_generation=0, + prior_balances_json="[]", + prior_balances_sha256=payout_artifact_sha256, + prepared_monotonic=time.monotonic(), + ), + ) + server._job_bundle_cache.clear() + server._job_bundle_cache[bundle.key] = bundle + return bundle + + +def verified_block_bundle(coinbase_tx_hex: str = "c0ffee") -> dict[str, object]: + return { + "found_block": {"coinbase_value_sats": 50_00000000}, + "ledger_window_attestation": {"signature": {"public_key_hex": "aa" * 32}}, + "payout_policy_manifest": {"accounts": []}, + "signed_coinbase_manifest": { + "manifest": { + "coinbase_tx_hex": coinbase_tx_hex, + "payout_count": 1, + } + }, + } + + +def verified_audit_report(coinbase_tx_hex: str = "c0ffee") -> dict[str, object]: + return { + "schema": "qbit.prism.audit-verification-report.v1", + "block_height": 10, + "coinbase_value_sats": 50_00000000, + "reward_manifest_sha256_hex": "44" * 32, + "payout_policy_manifest_sha256_hex": "55" * 32, + "prism_audit_commitment_leaf_hex": "66" * 32, + "audit_commitment_root_hex": "77" * 32, + "coinbase_txid": "11" * 32, + "coinbase_wtxid": "88" * 32, + "coinbase_manifest_sha256_hex": "22" * 32, + "audit_bundle_sha256_hex": "33" * 32, + "coinbase_tx_hex": coinbase_tx_hex, + "min_output_sats": 1, + "onchain_output_count": 0, + "accrued_account_count": 0, + } + + +def coordinator() -> PrismCoordinator: + server = PrismCoordinator.__new__(PrismCoordinator) + server.vardiff_config = vardiff.VardiffConfig( + enabled=True, + target_share_interval_seconds=Decimal("15"), + min_difficulty=Decimal("0.000000001"), + max_difficulty=Decimal("1024"), + retarget_interval_seconds=Decimal("1"), + max_step_factor=Decimal("4"), + startup_difficulty=Decimal("0.000000001"), + max_step_down_factor=Decimal("4"), + ewma_alpha=Decimal("1"), + retarget_tolerance=Decimal("0"), + ) + server.share_difficulty = Decimal("0.000000001") + server.lock = threading.RLock() + server.stop_event = threading.Event() + server.clients = set() + server.submitted_share_count = 0 + server.stale_share_count = 0 + server.duplicate_share_count = 0 + server.low_difficulty_share_count = 0 + server.collection_block_submission_count = 0 + server._pool_ready_latched = False + server.grace_credited_share_count = 0 + server.idle_retarget_count = 0 + server.rejection_counts_by_reason = {reason: 0 for reason in PRISM_REJECTION_REASON_IDS} + server.worker_metrics_limit = 100 + server.worker_metrics_lock = threading.Lock() + server.worker_share_counts = {} + server.worker_rejection_counts = {} + server.evicted_job_graveyard = {} + server.block_candidate_queue = queue.Queue(maxsize=8) + server.block_candidates_dropped = 0 + server.block_candidate_abandoned_counts = {} + server.share_append_queue = queue.Queue(maxsize=8) + server.share_writer_active = False + server.share_append_failure_count = 0 + server.share_recovery_path = None + server.share_recovery_lock = threading.Lock() + server.shares_recovered_to_disk = 0 + server.shares_replayed = 0 + server.current_tip_first_seen = None + server.current_tip_parent = None + server.stale_grace_seconds = 3.0 + server.blockwait_enabled = True + server.blockwait_timeout_seconds = 5.0 + server.vardiff_idle_sweep_seconds = 15.0 + server.job_build_failure_count = 0 + server.tip_refresh_job_count = 0 + server.post_accept_refresh_failure_count = 0 + server.reorg_reconciler_enabled = False + server.reorg_inactive_block_count = 0 + server.reorg_reactivated_block_count = 0 + server.reorg_reconcile_skip_count = 0 + server.reorg_reconcile_error_count = 0 + server.matured_payout_count = 0 + server.last_reorg_reconciled_tip_hash = None + server.last_reorg_reconciled_trusted = False + server.accepted_block_count = 1 + server.started_monotonic = time.monotonic() - 10 + server.ledger = FakeLedger(shares=5) + server.latest_coinbase_size_bytes = 250 + server.rpc = FakeRpc() + server.qbit_chain = "regtest" + server.blockpoll_seconds = 2.0 + # Failed-refresh spacing is opt-in per test: its holdoff waits on real + # time, which deadlocks tests that freeze time.monotonic around failing + # polls. Pacing behavior is covered by test_prism_refresh_retry_pacing. + server.tip_refresh_failure_holdoff_seconds = 0.0 + server.ctv_broadcaster_enabled = False + server.ctv_broadcaster_wallet = None + server.ctv_broadcaster_fee_sats = 0 + server.ctv_broadcaster_limit = 100 + server.ctv_broadcaster_interval_seconds = 30.0 + server.ctv_fanout_broadcast_daemon = None + server._ctv_fanout_market_fee_rate_cache = {} + server.tip_template_snapshot = None + server._tip_refresh_lock = threading.Lock() + server.extranonce2_size = 8 + server.coinbase_tag_hex = default_prism_coinbase_tag_hex() + server.version_mask = direct_stratum.QBIT_VERSION_ROLLING_MASK + server.version_mask_selection = direct_stratum.VersionRollingMaskSelection( + direct_stratum.QBIT_VERSION_ROLLING_MASK, + "fallback", + "test", + ) + return server + + +def submit_coordinator(tip: str = "00" * 32) -> tuple[PrismCoordinator, ClientState, RecordingLedger]: + server = coordinator() + # This suite runs on a synthetic millisecond timeline (declared anchors + # like 12000, share stamps of 1-2) while _expose_inflight_scan_anchor + # prunes exposures older than a real-wall-clock ceiling. Every synthetic + # anchor is decades past that ceiling, so the next exposure (a landing + # publishing its declared anchor) silently pruned a test's standing + # anchor and turned epoch-bump assertions into thread races against the + # landing's own anchor retirement. Pin the ceiling out of reach so + # exposed anchors live exactly as long as each test intends. + server.payout_artifact_max_anchor_age_seconds = float("inf") + server.vardiff_config = SimpleNamespace(enabled=False) + server.rpc = TipRpc(tip) + server.jobs = {} + server.recent_share_keys = set() + server.accepted_block_count = 0 + server.max_blocks = 1 + server.stop_after_block = True + server.extranonce2_size = 8 + server.share_weights_by_username = {"miner-a": 7} + ledger = RecordingLedger() + server.ledger = ledger + worker = WorkerIdentity( + username="miner-a", + payout_address="miner-a", + worker_name=None, + script_pubkey_hex="5220" + "11" * 32, + p2mr_program_hex="11" * 32, + ) + context = SimpleNamespace( + job=SimpleNamespace( + job_id="job-1", + share_target=target_from_compact("207fffff"), + share_difficulty=Decimal("1"), + transaction_hexes=(), + ), + template={"previousblockhash": tip, "height": 10, "coinbasevalue": 50_00000000}, + found_block={"network_difficulty": 1}, + issued_at_ms=12345, + collection_only=False, + worker=worker, + shares_json=[], + prior_balances=[], + ) + state = client() + state.username = "miner-a" + state.worker = worker + state.active_job_ids = {"job-1"} + server.jobs["job-1"] = context + return server, state, ledger + + +def block_candidate( + server: PrismCoordinator, + state: ClientState, + submission: object, + *, + job_id: str = "job-1", + pending_share: object | None = None, + credit_share_on_accept: bool = False, +) -> PrismBlockCandidate: + return PrismBlockCandidate( + context=server.jobs[job_id], + submission=submission, + extranonce1_hex=state.extranonce1_hex, + extranonce2_hex="00" * 8, + pending_share=pending_share + or SimpleNamespace( + share_id="miner-a:" + submission.block_hash_hex, + job_issued_at_ms=0, + accepted_at_ms=0, + ), + client=state, + credit_share_on_accept=credit_share_on_accept, + ) + + +def highdiff_vardiff_config(**overrides: object) -> vardiff.VardiffConfig: + values: dict[str, object] = dict( + enabled=True, + target_share_interval_seconds=Decimal("15"), + min_difficulty=Decimal("500000"), + max_difficulty=Decimal("4294967296"), + retarget_interval_seconds=Decimal("1"), + max_step_factor=Decimal("4"), + startup_difficulty=Decimal("500000"), + max_step_down_factor=Decimal("4"), + ewma_alpha=Decimal("1"), + retarget_tolerance=Decimal("0"), + ) + values.update(overrides) + return vardiff.VardiffConfig(**values) # type: ignore[arg-type] + + +def clear_stratum_diff_env() -> None: + for name in [ + name + for name in os.environ + if name.startswith("PRISM_STRATUM_HIGHDIFF") or name.startswith("PRISM_STRATUM_VARDIFF") + ]: + os.environ.pop(name, None) + + +class LostAckSubmitRpc(FakeRpc): + """qbitd accepts submitted blocks while every submitblock ack is lost. + + Models the mainnet 2026-07-25 interleaving: the block connects (the tip + flips to the candidate hash, so blockwait reports it) but the RPC + response never reaches the coordinator, and the retry disposition can + then read a transient racing snapshot in which the tip has moved on and + the header probe cannot prove the candidate active. + """ + + def __init__(self, *, start_tip: str, hash_by_hex: dict[str, str]) -> None: + self.tip = start_tip + self.height = 9 + self.hash_by_hex = dict(hash_by_hex) + self.active: dict[str, int] = {} + self.racing_tip: str | None = None + self.getblockhash_override: str | None = None + self.lose_acks = True + self.submitblock_calls = 0 + + def call(self, method: str, params: list[object] | None = None) -> object: + if method == "getbestblockhash": + return self.racing_tip or self.tip + if method == "getblockcount": + return self.height + if method == "submitblock": + self.submitblock_calls += 1 + block_hash = self.hash_by_hex[str((params or [""])[0])] + if block_hash in self.active: + return "duplicate" + if self.racing_tip is not None: + return "bad-prevblk" + self.height += 1 + self.active[block_hash] = self.height + self.tip = block_hash + if self.lose_acks: + raise OSError("connection reset by peer before submitblock ack") + return None + if method == "getblockheader": + requested = str((params or [""])[0]).lower() + if self.racing_tip is None and requested in self.active: + block_height = self.active[requested] + return { + "height": block_height, + "confirmations": self.height - block_height + 1, + } + raise RuntimeError("qbit RPC getblockheader failed: -5 Block not found") + if method == "getblockhash": + if self.getblockhash_override is not None: + return self.getblockhash_override + requested_height = int((params or [0])[0]) + for block_hash, block_height in self.active.items(): + if block_height == requested_height: + return block_hash + raise RuntimeError(f"unknown height {requested_height}") + return super().call(method, params) + + +class AcceptanceProbeRpc(FakeRpc): + def __init__( + self, + *, + tip: str, + header: dict[str, object] | None = None, + fail: bool = False, + fail_tip: bool = False, + ) -> None: + self.tip = tip + self.header = header + self.fail = fail + self.fail_tip = fail_tip + + def call(self, method: str, params: list[object] | None = None) -> object: + if self.fail: + raise RuntimeError("qbit RPC unavailable") + if method == "getbestblockhash": + if self.fail_tip: + raise RuntimeError("qbit RPC getbestblockhash unavailable") + return self.tip + if method == "getblockheader": + if self.header is None: + raise RuntimeError("qbit RPC getblockheader failed: -5 Block not found") + return self.header + return super().call(method, params) + + +def build_vardiff_test_coordinator() -> PrismCoordinator: + """Build a coordinator fixture without production init or worker startup.""" + return coordinator() + + +__all__ = [name for name in globals() if not name.startswith("__")] diff --git a/tests/prism_window_rows_reference.py b/tests/prism_window_rows_reference.py new file mode 100644 index 000000000..a52a868e0 --- /dev/null +++ b/tests/prism_window_rows_reference.py @@ -0,0 +1,97 @@ +"""Reference check for the per-row payout-window reads (issue #236). + +The payout-window snapshot and delta project one JSON object per row and +decode it in bounded batches. Their selection CTEs did not change, so the +pre-#236 statement -- the same CTE with the rows folded into one ``json_agg`` +value -- is a byte-exact oracle for membership, order and field values. This +module rebuilds that oracle from the statement the ledger actually sends and +compares the two on a live PostgreSQL, whichever backend the ledger runs. + +Used by ``test/test-prism-postgres-ledger.sh`` (psql subprocess backend) and +``test/test-prism-postgres-native-ledger.sh`` (pooled psycopg backend). +""" + +from __future__ import annotations + +import re +from typing import Any, Callable + +from lab.prism.share_ledger import AcceptedShareRecord, PsqlShareLedger + +_ROW_PROJECTION = re.compile( + r"SELECT (?Pjson_build_object\(.*?\))\nFROM rows\nORDER BY share_seq ASC;", + re.S, +) + + +def aggregate_reference_sql(row_sql: str) -> str: + """Turn the per-row statement back into its pre-#236 ``json_agg`` form.""" + rewritten, count = _ROW_PROJECTION.subn( + lambda match: ( + "SELECT COALESCE(json_agg(" + + match.group("object") + + " ORDER BY share_seq ASC), '[]'::json)\nFROM rows;" + ), + row_sql, + ) + if count != 1: + raise AssertionError( + "expected exactly one per-row projection in the window statement, " + f"found {count}" + ) + if "json_agg" in row_sql: + raise AssertionError("the per-row statement still carries a json_agg") + return rewritten + + +def capture_window_sql( + ledger: PsqlShareLedger, + call: Callable[[], list[AcceptedShareRecord]], +) -> tuple[str, list[AcceptedShareRecord]]: + """Run ``call`` on ``ledger`` and return the row statement it sent.""" + statements: list[str] = [] + original = ledger._run_retry_safe_read_json_rows + + def recording(sql: str, **kwargs: Any) -> list[Any]: + statements.append(sql) + return original(sql, **kwargs) + + ledger._run_retry_safe_read_json_rows = recording # type: ignore[method-assign] + try: + records = call() + finally: + del ledger._run_retry_safe_read_json_rows + if len(statements) != 1: + raise AssertionError( + f"expected one row statement per window read, saw {len(statements)}" + ) + return statements[0], records + + +def assert_rows_match_aggregate( + ledger: PsqlShareLedger, + call: Callable[[], list[AcceptedShareRecord]], + *, + label: str, + expected_len: int | None = None, +) -> list[AcceptedShareRecord]: + """Assert the per-row read equals the ``json_agg`` oracle, record for record.""" + row_sql, records = capture_window_sql(ledger, call) + reference_sql = aggregate_reference_sql(row_sql) + reference = [ + ledger._record_from_json(item) for item in ledger._run_json(reference_sql) + ] + if records != reference: + detail = ( + f"rows={len(records)} reference={len(reference)}" + if len(records) != len(reference) + else next( + f"first mismatch at index {index}: {left!r} != {right!r}" + for index, (left, right) in enumerate(zip(records, reference)) + if left != right + ) + ) + raise SystemExit(f"{label}: per-row window read differs from json_agg oracle ({detail})") + if expected_len is not None and len(records) != expected_len: + raise SystemExit(f"{label}: expected {expected_len} records, got {len(records)}") + return records diff --git a/tests/prism_worker_difficulty_store_gate.py b/tests/prism_worker_difficulty_store_gate.py new file mode 100644 index 000000000..5498b7660 --- /dev/null +++ b/tests/prism_worker_difficulty_store_gate.py @@ -0,0 +1,427 @@ +"""Non-discovered PostgreSQL parity gate for the worker-difficulty store. + +Drives :class:`MemoryWorkerDifficultyStore` and +:class:`PostgresWorkerDifficultyStore` through the same operation sequence and +requires identical results after every step, then checks the PostgreSQL-only +properties (schema reapply safety, CHECK constraint enforcement, index-backed +plans). It lives outside unittest discovery because it needs an explicitly +provisioned PostgreSQL target; run it from the repository root: + + PRISM_PSQL_COMMAND="psql -p 5432 -d some_scratch_db" \\ + python3 -m tests.prism_worker_difficulty_store_gate + +The target database is written to (the monolithic ledger schema is applied +into the default schema); use a scratch database. +""" + +from __future__ import annotations + +import json +import os +import shlex +import subprocess +import sys +from decimal import Decimal +from pathlib import Path + +from lab.prism.worker_difficulty_store import ( + MemoryWorkerDifficultyStore, + PostgresWorkerDifficultyStore, + WorkerDifficultyRecord, +) + +ROOT = Path(__file__).resolve().parents[1] +LEDGER_SCHEMA = ROOT / "crates" / "qbit-prism" / "sql" / "001_share_ledger.sql" +PSQL_TIMEOUT_SECONDS = 60.0 + +BASE_PSQL_COMMAND = os.environ.get("PRISM_PSQL_COMMAND", "") +if not BASE_PSQL_COMMAND: + raise SystemExit("PRISM_PSQL_COMMAND is required") +BASE_PSQL_ARGV = shlex.split(BASE_PSQL_COMMAND) +if not BASE_PSQL_ARGV: + raise SystemExit("PRISM_PSQL_COMMAND is empty") + +BASE_MS = 1_756_000_000_000 + + +class GateFailure(RuntimeError): + pass + + +def assert_equal(actual: object, expected: object, message: str) -> None: + if actual != expected: + raise GateFailure(f"{message}: expected {expected!r}, got {actual!r}") + + +def assert_true(condition: bool, message: str) -> None: + if not condition: + raise GateFailure(message) + + +def run_psql(sql: str, *, expect_failure: bool = False) -> str: + command = [ + *BASE_PSQL_ARGV, + "--no-psqlrc", + "--set", + "ON_ERROR_STOP=1", + "--tuples-only", + "--no-align", + "--quiet", + ] + completed = subprocess.run( + command, + input=sql, + text=True, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + timeout=PSQL_TIMEOUT_SECONDS, + check=False, + ) + if expect_failure: + if completed.returncode == 0: + raise GateFailure(f"psql unexpectedly succeeded for: {sql.strip()[:200]}") + return completed.stderr + if completed.returncode != 0: + raise GateFailure( + f"psql failed with exit {completed.returncode}: {completed.stderr.strip()}" + ) + return completed.stdout + + +def run_json(sql: str) -> object: + output = run_psql(sql).strip() + if not output: + raise GateFailure(f"psql query returned no JSON: {sql.strip()[:200]}") + return json.loads(output.splitlines()[-1]) + + +def run_script(sql: str) -> None: + run_psql(sql) + + +def record_key(record: WorkerDifficultyRecord) -> tuple[str, str]: + return (record.listener, record.worker_username) + + +def check_parity( + memory: MemoryWorkerDifficultyStore, + postgres: PostgresWorkerDifficultyStore, + message: str, +) -> None: + """Full-table equality via the bounded load with a generous limit.""" + memory_rows = memory.load_recent(evidence_after_ms=0, limit=10_000) + postgres_rows = postgres.load_recent(evidence_after_ms=0, limit=10_000) + assert_equal(postgres_rows, memory_rows, message) + + +def main() -> None: + schema_sql = LEDGER_SCHEMA.read_text(encoding="utf-8") + + # Idempotent schema application: the monolithic file applies cleanly + # twice, and the focused module DDL reapplies over it a third time. + run_psql(schema_sql) + run_psql(schema_sql) + + memory = MemoryWorkerDifficultyStore() + postgres = PostgresWorkerDifficultyStore(run_json, run_script=run_script) + postgres.ensure_schema() + postgres.ensure_schema() + + # Clean slate in case the scratch database was reused. + run_psql("DELETE FROM qbit_worker_difficulty;") + + def upsert_both(**kwargs: object) -> None: + memory_outcome = memory.upsert(**kwargs) # type: ignore[arg-type] + postgres_outcome = postgres.upsert(**kwargs) # type: ignore[arg-type] + assert_equal( + postgres_outcome, + memory_outcome, + f"upsert outcome parity for {kwargs!r}", + ) + + # Insert across two lanes, same username in both; lookalike usernames in + # one lane; a quote-bearing username to exercise literal quoting. + upsert_both( + listener="stratum-public", + worker_username="miner.worker1", + difficulty=Decimal("16"), + evidence_at_ms=BASE_MS + 1_000, + now_ms=BASE_MS + 1_000, + ) + upsert_both( + listener="stratum-highdiff", + worker_username="miner.worker1", + difficulty=Decimal("65536"), + evidence_at_ms=BASE_MS + 2_000, + now_ms=BASE_MS + 2_000, + ) + upsert_both( + listener="stratum-public", + worker_username="Miner.worker1", + difficulty=Decimal("32"), + evidence_at_ms=BASE_MS + 3_000, + now_ms=BASE_MS + 3_000, + ) + upsert_both( + listener="stratum-public", + worker_username=" miner.worker1", + difficulty=Decimal("64"), + evidence_at_ms=BASE_MS + 4_000, + now_ms=BASE_MS + 4_000, + ) + upsert_both( + listener="stratum-public", + worker_username="o'brien.worker1", + difficulty=Decimal("128"), + evidence_at_ms=BASE_MS + 5_000, + now_ms=BASE_MS + 5_000, + ) + check_parity(memory, postgres, "parity after inserts") + + # Update with newer evidence applies; the lane sibling is untouched. + upsert_both( + listener="stratum-public", + worker_username="miner.worker1", + difficulty=Decimal("24"), + evidence_at_ms=BASE_MS + 6_000, + now_ms=BASE_MS + 6_500, + ) + # Older evidence is refused even with a later wall-clock write time. + upsert_both( + listener="stratum-public", + worker_username="miner.worker1", + difficulty=Decimal("1"), + evidence_at_ms=BASE_MS + 5_999, + now_ms=BASE_MS + 9_000, + ) + # Equal evidence is deterministic last-write-wins. + upsert_both( + listener="stratum-public", + worker_username="miner.worker1", + difficulty=Decimal("48"), + evidence_at_ms=BASE_MS + 6_000, + now_ms=BASE_MS + 9_500, + ) + check_parity(memory, postgres, "parity after evidence-guarded updates") + + def lower_both(**kwargs: object) -> None: + memory_outcome = memory.apply_downward(**kwargs) # type: ignore[arg-type] + postgres_outcome = postgres.apply_downward(**kwargs) # type: ignore[arg-type] + assert_equal( + postgres_outcome, + memory_outcome, + f"apply_downward outcome parity for {kwargs!r}", + ) + + # The lower-only correction: reduces a higher row, refuses to raise one, + # is a no-op on an equal value or a missing key, and never touches + # evidence_at -- so a later share-backed write whose evidence predates the + # correction still applies. + lower_both( + listener="stratum-public", + worker_username="miner.worker1", + difficulty=Decimal("8"), + now_ms=BASE_MS + 20_000, + ) + lower_both( + listener="stratum-public", + worker_username="miner.worker1", + difficulty=Decimal("4096"), + now_ms=BASE_MS + 20_100, + ) + lower_both( + listener="stratum-public", + worker_username="miner.worker1", + difficulty=Decimal("8.000"), + now_ms=BASE_MS + 20_200, + ) + lower_both( + listener="stratum-public", + worker_username="never.seen", + difficulty=Decimal("8"), + now_ms=BASE_MS + 20_300, + ) + check_parity(memory, postgres, "parity after lower-only corrections") + assert_true( + not any( + record.worker_username == "never.seen" + for record in postgres.load_recent(evidence_after_ms=0, limit=10_000) + ), + "apply_downward must never insert a row for an unknown worker", + ) + # Evidence older than the correction's wall clock, newer than the row's: + # the correction wrote no evidence, so this must still apply. + upsert_both( + listener="stratum-public", + worker_username="miner.worker1", + difficulty=Decimal("512"), + evidence_at_ms=BASE_MS + 6_001, + now_ms=BASE_MS + 21_000, + ) + check_parity( + memory, + postgres, + "a lower-only correction must not suppress later share evidence", + ) + + # Decimal exactness through the durable round trip. + exact_values = [ + Decimal("0.000030517578125"), # 2**-15 + Decimal("123456789012345678901234567890.5"), + Decimal("1E-12"), + Decimal("8192.000"), + ] + for index, difficulty in enumerate(exact_values): + upsert_both( + listener="decimal-lane", + worker_username=f"miner.exact{index}", + difficulty=difficulty, + evidence_at_ms=BASE_MS + 10_000 + index, + now_ms=BASE_MS + 10_000 + index, + ) + loaded = postgres.load_recent(evidence_after_ms=BASE_MS + 9_999, limit=10) + by_user = {entry.worker_username: entry.difficulty for entry in loaded} + for index, difficulty in enumerate(exact_values): + assert_equal( + by_user[f"miner.exact{index}"], + difficulty, + f"decimal round trip for {difficulty}", + ) + check_parity(memory, postgres, "parity after decimal round-trip rows") + + # Bounded newest-first load: limit and deterministic ordering pinned, + # cutoff strictly exclusive. Both stores must agree element-for-element. + for cutoff, limit in [ + (0, 3), + (BASE_MS + 4_000, 2), + (BASE_MS + 6_000, 10), + (BASE_MS + 10_003, 10), + ]: + assert_equal( + postgres.load_recent(evidence_after_ms=cutoff, limit=limit), + memory.load_recent(evidence_after_ms=cutoff, limit=limit), + f"bounded load parity (cutoff={cutoff}, limit={limit})", + ) + top = postgres.load_recent(evidence_after_ms=0, limit=3) + assert_equal( + [record_key(entry) for entry in top], + [ + ("decimal-lane", "miner.exact3"), + ("decimal-lane", "miner.exact2"), + ("decimal-lane", "miner.exact1"), + ], + "newest-first pinned ordering", + ) + boundary = postgres.load_recent(evidence_after_ms=BASE_MS + 10_003, limit=10) + assert_equal(boundary, [], "strict cutoff excludes the boundary row") + + # Bounded prune removes the deterministic oldest set; both back ends + # agree on the count and on every survivor. + memory_deleted = memory.prune(evidence_cutoff_ms=BASE_MS + 5_000, limit=2) + postgres_deleted = postgres.prune(evidence_cutoff_ms=BASE_MS + 5_000, limit=2) + assert_equal(postgres_deleted, memory_deleted, "bounded prune count parity") + assert_equal(postgres_deleted, 2, "bounded prune removes exactly the limit") + check_parity(memory, postgres, "parity after bounded prune") + + # Unbounded prune clears the rest of the old range store-side. + memory_deleted = memory.prune(evidence_cutoff_ms=BASE_MS + 6_000) + postgres_deleted = postgres.prune(evidence_cutoff_ms=BASE_MS + 6_000) + assert_equal(postgres_deleted, memory_deleted, "prune count parity") + check_parity(memory, postgres, "parity after prune") + + # A no-op prune below every remaining row deletes nothing. + assert_equal(postgres.prune(evidence_cutoff_ms=1), memory.prune(evidence_cutoff_ms=1), "no-op prune parity") + + # Ordering ties: several keys share one evidence timestamp, so the + # (listener, worker_username) byte-order tie-break itself is what both + # back ends must agree on (COLLATE "C" pins the PostgreSQL side to byte + # order regardless of the database's locale collation). + tie_usernames = ["zeta", "Alpha", " alpha", "alpha", "ALPHA.9"] + for tie_index, username in enumerate(tie_usernames): + upsert_both( + listener="tie-lane", + worker_username=username, + difficulty=Decimal(tie_index + 1), + evidence_at_ms=BASE_MS + 20_000, + now_ms=BASE_MS + 20_000, + ) + tie_memory = memory.load_recent(evidence_after_ms=BASE_MS + 19_999, limit=10) + tie_postgres = postgres.load_recent(evidence_after_ms=BASE_MS + 19_999, limit=10) + assert_equal(tie_postgres, tie_memory, "tie-break ordering parity") + assert_equal( + [entry.worker_username for entry in tie_postgres], + sorted(tie_usernames), + "tie-break is byte order", + ) + check_parity(memory, postgres, "parity with the tie group present") + + # Schema-level validation: the CHECK constraints refuse what the Python + # validation refuses, so a writer bypassing this module cannot poison the + # preload path with NaN/Infinity/non-positive difficulties or empty keys. + for bad_row in ( + "('lane', 'user-nan', 'NaN'::numeric, now(), now())", + "('lane', 'user-inf', 'Infinity'::numeric, now(), now())", + "('lane', 'user-zero', 0, now(), now())", + "('lane', 'user-negative', -5, now(), now())", + "('', 'user-empty-lane', 1, now(), now())", + "('lane', '', 1, now(), now())", + ): + stderr = run_psql( + "INSERT INTO qbit_worker_difficulty " + "(listener, worker_username, difficulty, evidence_at, updated_at) " + f"VALUES {bad_row};", + expect_failure=True, + ) + assert_true( + "violates check constraint" in stderr or "not-null constraint" in stderr, + f"unexpected rejection reason for {bad_row}: {stderr.strip()}", + ) + + # The bounded newest-first load plans through the evidence index rather + # than a sequential scan once the planner is told scans are expensive + # (the table is tiny here, so seqscan is otherwise legitimately cheaper). + plan = run_psql( + """ +BEGIN; +SET LOCAL enable_seqscan = off; +EXPLAIN (COSTS OFF) +SELECT listener, worker_username, difficulty +FROM qbit_worker_difficulty +WHERE evidence_at > now() - interval '1 hour' +ORDER BY evidence_at DESC, listener, worker_username +LIMIT 100; +ROLLBACK; +""" + ) + assert_true( + "qbit_worker_difficulty_evidence_idx" in plan, + f"bounded load did not use the evidence index:\n{plan}", + ) + assert_true("Sort" not in plan, f"bounded load required an explicit sort:\n{plan}") + + # Canonical share/payout tables are untouched by this gate's traffic. + counts = run_json( + """ +SELECT json_build_object( + 'shares', (SELECT count(*) FROM qbit_share_ledger), + 'blocks', (SELECT count(*) FROM qbit_pool_blocks), + 'payouts', (SELECT count(*) FROM qbit_pool_payout_entries), + 'carry', (SELECT count(*) FROM qbit_payout_carry_forward) +); +""" + ) + assert_equal( + counts, + {"shares": 0, "blocks": 0, "payouts": 0, "carry": 0}, + "canonical tables must stay empty", + ) + + print("prism worker-difficulty store gate: all checks passed") + + +if __name__ == "__main__": + try: + main() + except GateFailure as failure: + print(f"GATE FAILURE: {failure}", file=sys.stderr) + raise SystemExit(1) diff --git a/tests/test_auxpow_vardiff.py b/tests/test_auxpow_vardiff.py index 98c26ee91..1c8535a11 100644 --- a/tests/test_auxpow_vardiff.py +++ b/tests/test_auxpow_vardiff.py @@ -86,5 +86,175 @@ def test_hysteresis_suppresses_small_retargets(self) -> None: self.assertTrue(vardiff.should_retarget(Decimal("100"), Decimal("120"), Decimal("0.20"))) +class AuxPowVardiffInitialConvergenceTests(unittest.TestCase): + """Fast-arrival initial convergence math (#132).""" + + def config(self, **overrides: object) -> vardiff.VardiffConfig: + base = dict( + enabled=True, + target_share_interval_seconds=Decimal("15"), + min_difficulty=Decimal("16384"), + max_difficulty=Decimal("4294967296"), + retarget_interval_seconds=Decimal("300"), + max_step_factor=Decimal("4"), + startup_difficulty=Decimal("16384"), + max_step_down_factor=Decimal("4"), + ewma_alpha=Decimal("0.4"), + retarget_tolerance=Decimal("0.25"), + ) + base.update(overrides) + return vardiff.VardiffConfig(**base) # type: ignore[arg-type] + + def ready(self, **overrides: object) -> bool: + # The incident window: eight router-filtered shares stamped at the + # 16384 lane start, arriving in the ~1.83s a 9.4 PH/s rig needs to + # produce eight shares that clear a 500000 filter. + call = dict( + current_difficulty=Decimal("16384"), + accepted_shares=8, + accepted_difficulty=Decimal("16384") * 8, + elapsed_seconds=Decimal("1.828"), + config=self.config(), + ) + call.update(overrides) + return vardiff.initial_convergence_ready(**call) # type: ignore[arg-type] + + def test_router_filtered_burst_is_ready(self) -> None: + self.assertTrue(self.ready()) + + def test_too_few_shares_is_not_ready(self) -> None: + self.assertFalse( + self.ready( + accepted_shares=7, + accepted_difficulty=Decimal("16384") * 7, + ) + ) + + def test_near_zero_elapsed_window_is_not_ready(self) -> None: + # Without the elapsed floor this window divides real work by an + # almost-zero interval. note_accepted clamps elapsed at 1ms, and at + # that clamp eight lane-start shares observe ~1.97e9 -- a 120000x + # gap that the wider initial bound would happily act on. + self.assertFalse(self.ready(elapsed_seconds=Decimal("0.001"))) + self.assertGreater( + vardiff.observed_difficulty( + accepted_difficulty=Decimal("16384") * 8, + elapsed_seconds=Decimal("0.001"), + target_share_interval_seconds=Decimal("15"), + ), + Decimal("16384") * 100000, + ) + + def test_on_target_miner_is_not_ready(self) -> None: + # One accepted share per target interval: eight shares over eight + # intervals observes exactly the current difficulty, so the early + # path must not engage no matter how much evidence accumulates. + for shares in (8, 64, 512): + with self.subTest(shares=shares): + self.assertFalse( + self.ready( + accepted_shares=shares, + accepted_difficulty=Decimal("16384") * shares, + elapsed_seconds=Decimal("15") * shares, + ) + ) + + def test_gap_below_the_confidence_floor_is_not_ready(self) -> None: + # 3.9x observed against a 4x floor: inside the noise a small sample + # can manufacture, so it waits for the ordinary interval. + self.assertFalse( + self.ready( + accepted_difficulty=Decimal("16384") * 8, + elapsed_seconds=Decimal("15") * 8 / Decimal("3.9"), + ) + ) + self.assertTrue( + self.ready( + accepted_difficulty=Decimal("16384") * 8, + elapsed_seconds=Decimal("15") * 8 / Decimal("4.1"), + ) + ) + + def test_disabled_policy_is_never_ready(self) -> None: + self.assertFalse(self.ready(config=self.config(initial_convergence_enabled=False))) + self.assertFalse(self.ready(config=self.config(enabled=False))) + + def test_initial_step_crosses_the_router_difficulty_in_one_move(self) -> None: + next_difficulty = vardiff.calculate_next_difficulty( + current_difficulty=Decimal("16384"), + accepted_shares=8, + accepted_difficulty=Decimal("16384") * 8, + elapsed_seconds=Decimal("1.828"), + config=self.config(), + initial_convergence=True, + ) + + self.assertEqual(next_difficulty, Decimal("16384") * 64) + self.assertGreaterEqual(next_difficulty, Decimal("500000")) + + def test_same_window_without_the_initial_flag_keeps_the_ordinary_bound(self) -> None: + next_difficulty = vardiff.calculate_next_difficulty( + current_difficulty=Decimal("16384"), + accepted_shares=8, + accepted_difficulty=Decimal("16384") * 8, + elapsed_seconds=Decimal("1.828"), + config=self.config(), + ) + + self.assertEqual(next_difficulty, Decimal("16384") * 4) + self.assertLess(next_difficulty, Decimal("500000")) + + def test_initial_step_obeys_the_lane_ceiling(self) -> None: + next_difficulty = vardiff.calculate_next_difficulty( + current_difficulty=Decimal("16384"), + accepted_shares=8, + accepted_difficulty=Decimal("16384") * 8, + elapsed_seconds=Decimal("1.828"), + config=self.config(max_difficulty=Decimal("262144")), + initial_convergence=True, + ) + + self.assertEqual(next_difficulty, Decimal("262144")) + + def test_initial_step_obeys_the_high_diff_floor(self) -> None: + # The high-diff lane's floor is a wire guarantee, so even a step + # computed under the wider initial bound clamps up to it. + next_difficulty = vardiff.calculate_next_difficulty( + current_difficulty=Decimal("500000"), + accepted_shares=8, + accepted_difficulty=Decimal("500000") * 8, + elapsed_seconds=Decimal("1200"), + config=self.config( + min_difficulty=Decimal("500000"), + startup_difficulty=Decimal("500000"), + ), + initial_convergence=True, + ) + + self.assertEqual(next_difficulty, Decimal("500000")) + + def test_initial_flag_does_not_widen_the_step_down_bound(self) -> None: + next_difficulty = vardiff.calculate_next_difficulty( + current_difficulty=Decimal("1048576"), + accepted_shares=0, + elapsed_seconds=Decimal("300"), + config=self.config(), + initial_convergence=True, + ) + + self.assertEqual(next_difficulty, Decimal("1048576") / 4) + + def test_inverted_step_bounds_are_rejected(self) -> None: + with self.assertRaises(ValueError): + self.config( + max_step_factor=Decimal("128"), + initial_max_step_factor=Decimal("64"), + ) + with self.assertRaises(ValueError): + self.config(initial_min_accepted_shares=0) + with self.assertRaises(ValueError): + self.config(initial_min_elapsed_seconds=Decimal("0")) + + if __name__ == "__main__": unittest.main() diff --git a/tests/test_auxpow_vardiff_coordinator.py b/tests/test_auxpow_vardiff_coordinator.py index 786030ca2..bbfd0224e 100644 --- a/tests/test_auxpow_vardiff_coordinator.py +++ b/tests/test_auxpow_vardiff_coordinator.py @@ -658,6 +658,18 @@ def test_load_vardiff_config_defaults_match_documented_public_defaults(self) -> self.assertEqual(config.ewma_alpha, Decimal("0.4")) self.assertEqual(config.retarget_tolerance, Decimal("0.25")) self.assertEqual(config.startup_difficulty, Decimal("8192")) + self.assertFalse(config.initial_convergence_enabled) + + def test_load_vardiff_config_accepts_legacy_step_factor_above_prism_cap(self) -> None: + with patch.dict( + os.environ, + {"AUXPOW_STRATUM_VARDIFF_MAX_STEP_FACTOR": "128"}, + ): + config = coordinator.load_vardiff_config() + + self.assertEqual(config.max_step_factor, Decimal("128")) + self.assertEqual(config.initial_max_step_factor, Decimal("128")) + self.assertFalse(config.initial_convergence_enabled) def test_vardiff_minimum_sets_advertised_floor_even_below_qbit_difficulty(self) -> None: server = self.server() diff --git a/tests/test_check_env_production_gate.py b/tests/test_check_env_production_gate.py index f8152df5a..498c00960 100644 --- a/tests/test_check_env_production_gate.py +++ b/tests/test_check_env_production_gate.py @@ -3,6 +3,7 @@ from __future__ import annotations import os +import shlex import shutil import subprocess import tempfile @@ -13,6 +14,12 @@ ROOT_DIR = Path(__file__).resolve().parents[1] CHECK_ENV = ROOT_DIR / "scripts" / "check-env.sh" class CheckEnvProductionGateTests(unittest.TestCase): + def setUp(self) -> None: + root = Path(self.enterContext(tempfile.TemporaryDirectory())) + # Policy tests stop at a predictable Docker boundary. The runner's + # daemon may still be starting, or may not be installed at all. + self.docker_bin = self.write_fake_docker(root, exit_code=1) + def production_prism_env(self, root: Path) -> dict[str, str]: return { "MINING_LANES": "prism", @@ -30,6 +37,7 @@ def production_prism_env(self, root: Path) -> dict[str, str]: "PRISM_POSTGRES_IMAGE": "registry.example.invalid/postgres@sha256:" + "e" * 64, "PRISM_POSTGRES_DATA_SOURCE": str(root / "postgres-data"), "PRISM_POSTGRES_WAL_SOURCE": str(root / "postgres-wal"), + "PRISM_POSTGRES_REPLICA_DATA_SOURCE": str(root / "postgres-replica"), "PRISM_AUDIT_DATA_SOURCE": str(root / "prism-audit"), "PRISM_DATABASE_URL": "postgresql://example.invalid/qbit", "PRISM_POSTGRES_PASSWORD": "not-default", @@ -40,7 +48,7 @@ def production_prism_env(self, root: Path) -> dict[str, str]: "PRISM_LEDGER_WRITER_EPOCH": "7", "PRISM_AUDIT_DIR": "/var/lib/qbit/prism/audit", "PRISM_EVIDENCE_PATH": "/var/lib/qbit/prism/evidence.json", - "PRISM_STRATUM_STALE_GRACE_SECONDS": "0", + "PRISM_STRATUM_STALE_GRACE_SECONDS": "3", "PRISM_STRATUM_SHARE_DIFF": "1024", "PRISM_STRATUM_VARDIFF": "1", "PRISM_STRATUM_VARDIFF_TARGET_SECONDS": "15", @@ -93,11 +101,11 @@ def write_pinned_qbit_checkout(self, root: Path) -> tuple[Path, str]: ).strip() return checkout, commit - def write_fake_docker(self, root: Path) -> Path: + def write_fake_docker(self, root: Path, *, exit_code: int = 0) -> Path: fake_bin = root / "bin" fake_bin.mkdir() docker = fake_bin / "docker" - docker.write_text("#!/bin/sh\nexit 0\n", encoding="utf-8") + docker.write_text(f"#!/bin/sh\nexit {exit_code}\n", encoding="utf-8") docker.chmod(0o755) return fake_bin @@ -137,7 +145,7 @@ def production_testnet4_env(self, root: Path) -> dict[str, str]: "PRISM_LEDGER_WRITER_EPOCH": "7", "PRISM_AUDIT_DIR": "/var/lib/qbit/prism/audit", "PRISM_EVIDENCE_PATH": "/var/lib/qbit/prism/evidence.json", - "PRISM_STRATUM_STALE_GRACE_SECONDS": "0", + "PRISM_STRATUM_STALE_GRACE_SECONDS": "3", "PRISM_STRATUM_SHARE_DIFF": "1024", "PRISM_STRATUM_VARDIFF_MIN_DIFF": "1024", "PRISM_STRATUM_VARDIFF_START_DIFF": "4096", @@ -153,6 +161,9 @@ def run_check_env( ) -> subprocess.CompletedProcess[str]: env = os.environ.copy() env.pop("QBIT_GIT_COMMIT", None) + env["PATH"] = f"{self.docker_bin}:{env['PATH']}" + # Tests exercising successful Docker checks or a minimal PATH supply + # their own controlled tool fixtures through overrides. env.update(overrides) return subprocess.run( ["/bin/bash", str(script), *arguments], @@ -172,6 +183,9 @@ def isolated_check_env_root(self, directory: Path) -> tuple[Path, Path]: shutil.copyfile(CHECK_ENV, script) shutil.copyfile(ROOT_DIR / ".env.example", root / ".env.example") shutil.copyfile(ROOT_DIR / "config" / "upstream.env.example", root / "config" / "upstream.env") + entrypoint = Path("docker/qbit/qbit-entrypoint.sh") + (root / entrypoint).parent.mkdir(parents=True) + shutil.copyfile(ROOT_DIR / entrypoint, root / entrypoint) return root, script def test_deploy_env_file_is_loaded_as_the_final_config_layer(self) -> None: @@ -284,6 +298,7 @@ def test_non_auxpow_lane_ignores_unused_bitcoin_validation_settings(self) -> Non self.assertNotIn("BITCOIN_CHAIN must", result.stderr) self.assertNotIn("BITCOIN_DNSSEED must", result.stderr) self.assertNotIn("BITCOIN_DISCOVER must", result.stderr) + self.assertIn("docker daemon is not reachable", result.stderr) def test_production_mode_rejects_regtest_before_docker_check(self) -> None: result = self.run_check_env(QBIT_PRODUCTION="1") @@ -324,6 +339,8 @@ def test_nonproduction_skips_production_only_policy_checks(self) -> None: self.assertNotIn("rejects CKPOOL_PUBLIC_DIFF_POLICY=permissive", result.stderr) self.assertNotIn("rejects PRISM_ALLOW_MEMORY_LEDGER=1", result.stderr) self.assertNotIn("rejects PRISM_ALLOW_TEST_SIGNING_SEEDS=1", result.stderr) + self.assertNotEqual(result.returncode, 0) + self.assertIn("docker daemon is not reachable", result.stderr) def test_production_mainnet_prelaunch_accepts_explicit_authorization(self) -> None: result = self.run_check_env( @@ -730,6 +747,104 @@ def test_production_rejects_reused_state_paths_before_docker_check(self) -> None self.assertIn("PRISM_POSTGRES_DATA_SOURCE and PRISM_POSTGRES_WAL_SOURCE must be distinct", result.stderr) self.assertNotIn("docker is required", result.stderr) + def test_production_validates_replica_storage_before_docker(self) -> None: + key = "PRISM_POSTGRES_REPLICA_DATA_SOURCE" + with tempfile.TemporaryDirectory() as temp_dir: + root, script = self.isolated_check_env_root(Path(temp_dir)) + env = self.production_prism_env(root) + env[key] = "" + fake_bin = self.write_fake_docker(root) + docker_log = root / "docker-calls.log" + (fake_bin / "docker").write_text( + '#!/bin/sh\nprintf "%s\\n" "$*" >> "$DOCKER_CALL_LOG"\nexit 0\n', + encoding="utf-8", + ) + deploy_env = root / "deployment.env" + env.update({ + "PATH": f"{fake_bin}:{os.environ['PATH']}", + "DOCKER_CALL_LOG": str(docker_log), + "DEPLOY_ENV_FILE": str(deploy_env), + }) + cases = [ + (None, "absolute host path"), + ("", "absolute host path"), + ("replica-volume", "absolute host path"), + ("./replica-data", "absolute host path"), + ] + for other in ( + "QBIT_DATA_SOURCE", + "PRISM_POSTGRES_DATA_SOURCE", + "PRISM_POSTGRES_WAL_SOURCE", + "PRISM_AUDIT_DATA_SOURCE", + ): + cases.append((env[other] + "/", "must be distinct")) + for value, error in cases: + with self.subTest(value=value): + deploy_env.write_text( + "" if value is None else f"{key}={shlex.quote(value)}\n", + encoding="utf-8", + ) + result = self.run_check_env(script=script, cwd=root, **env) + self.assertNotEqual(result.returncode, 0) + self.assertIn(key, result.stderr) + self.assertIn(error, result.stderr) + self.assertFalse(docker_log.exists(), "validation called Docker") + + def test_replica_storage_rejects_overlap_and_aliases_before_docker(self) -> None: + key = "PRISM_POSTGRES_REPLICA_DATA_SOURCE" + with tempfile.TemporaryDirectory() as temp_dir: + root = Path(temp_dir) + env = self.production_prism_env(root) + primary = Path(env["PRISM_POSTGRES_DATA_SOURCE"]) + primary.mkdir(parents=True) + marker = primary / "PG_VERSION" + marker.write_text("16") + alias = root / "primary-alias" + alias.symlink_to(primary, target_is_directory=True) + cases = [ + str(primary.parent), str(primary / "replica"), + str(primary) + "/../" + primary.name + "//", + str(alias), str(alias / "new-replica"), + ] + for value in cases: + with self.subTest(value=value): + result = self.run_check_env(**{**env, key: value}) + self.assertNotEqual(result.returncode, 0) + self.assertIn("non-overlapping", result.stderr) + self.assertNotIn("docker daemon", result.stderr) + self.assertEqual(marker.read_text(), "16") + # Component boundaries matter: primary and primary-copy are siblings. + result = self.run_check_env(**{**env, key: str(primary) + "-copy"}) + self.assertNotIn("non-overlapping", result.stderr) + self.assertIn("docker daemon", result.stderr) + + def test_production_replica_storage_honors_configuration_precedence(self) -> None: + key = "PRISM_POSTGRES_REPLICA_DATA_SOURCE" + with tempfile.TemporaryDirectory() as temp_dir: + root, script = self.isolated_check_env_root(Path(temp_dir)) + checkout, commit = self.write_pinned_qbit_checkout(root) + fake_bin = self.write_fake_docker(root) + env = self.production_prism_env(root) + env.update({ + "PATH": f"{fake_bin}:{os.environ['PATH']}", + "QBIT_SRC_DIR": str(checkout), + "QBIT_SRC_DIR_OVERRIDE": str(checkout), + "QBIT_GIT_COMMIT": commit, + }) + deploy_env = root / "deployment.env" + env["DEPLOY_ENV_FILE"] = str(deploy_env) + for file_value, shell_value in ( + (str(root / "file-replica"), ""), + ("invalid-named-volume", str(root / "shell-replica")), + ): + with self.subTest(file_value=file_value, shell_value=shell_value): + deploy_env.write_text(f"{key}={shlex.quote(file_value)}\n", encoding="utf-8") + result = self.run_check_env( + script=script, cwd=root, **{**env, key: shell_value} + ) + self.assertEqual(result.returncode, 0, result.stderr) + self.assertIn(f"qbit source checkout verified at {commit}", result.stdout) + def test_qbit_mainnet_auxpow_requires_mainnet_parent(self) -> None: result = self.run_check_env( MINING_LANES="auxpow", @@ -920,7 +1035,7 @@ def test_production_auxpow_requires_explicit_payouts(self) -> None: "BITCOIN_EXPECTED_GENESIS_HASH": ( "000000000019d6689c085ae165831e934ff763ae46a2a6c172b3f1b60a8ce26f" ).upper(), - "PRISM_STRATUM_STALE_GRACE_SECONDS": "0", + "PRISM_STRATUM_STALE_GRACE_SECONDS": "3", } for qbit_address, bitcoin_address, expected_name in ( ("auto", "bc1explicit", "QBIT_MINER_ADDRESS"), @@ -956,7 +1071,7 @@ def test_production_git_provider_requires_exact_commit_pin(self) -> None: "PRISM_LEDGER_WRITER_EPOCH": "7", "PRISM_AUDIT_DIR": "/var/lib/qbit/prism/audit", "PRISM_EVIDENCE_PATH": "/var/lib/qbit/prism/evidence.json", - "PRISM_STRATUM_STALE_GRACE_SECONDS": "0", + "PRISM_STRATUM_STALE_GRACE_SECONDS": "3", "CKPOOL_MINDIFF": "1024", "CKPOOL_STARTDIFF": "65536", "CKPOOL_REQUIRE_P2MR_PAYOUT": "1", @@ -982,23 +1097,128 @@ def test_production_git_provider_requires_exact_commit_pin(self) -> None: ) self.assertNotIn("docker is required", result.stderr) - def test_mainnet_requires_zero_stale_grace(self) -> None: - result = self.run_check_env( - MINING_LANES="prism", - QBIT_CHAIN="mainnet", - QBIT_CHAIN_FLAG="-chain=main", - QBIT_EXPECTED_GENESIS_HASH="11" * 32, - QBIT_GIT_COMMIT="41" * 20, - PRISM_STRATUM_STALE_GRACE_SECONDS="3", - ) + def test_mainnet_accepts_bounded_stale_grace(self) -> None: + # Mainnet follows the same bounded-grace rule as every other chain: + # zero grace rejects every in-flight prior-tip share at each block, + # which miners read as pool failure. Run a complete mainnet production + # environment so a clean exit proves the gate accepts the window, + # rather than only that the old message is gone. + with tempfile.TemporaryDirectory() as temp_dir: + root = Path(temp_dir) + checkout, commit = self.write_pinned_qbit_checkout(root) + fake_bin = self.write_fake_docker(root) + common = self.production_prism_env(root) + common.update({ + "PATH": f"{fake_bin}:{os.environ['PATH']}", + "QBIT_CHAIN": "mainnet", + "QBIT_CHAIN_FLAG": "-chain=main", + "QBIT_EXPECTED_GENESIS_HASH": "11" * 32, + "QBIT_GIT_COMMIT": commit, + "QBIT_SRC_DIR": str(checkout), + "QBIT_SRC_DIR_OVERRIDE": str(checkout), + }) - self.assertNotEqual(result.returncode, 0) - self.assertIn("mainnet requires PRISM_STRATUM_STALE_GRACE_SECONDS=0", result.stderr) - self.assertNotIn("docker is required", result.stderr) + for grace in ("3", "0.5", "0"): + with self.subTest(grace=grace): + result = self.run_check_env( + **{**common, "PRISM_STRATUM_STALE_GRACE_SECONDS": grace} + ) + + self.assertEqual(result.returncode, 0, result.stderr) + self.assertNotIn("PRISM_STRATUM_STALE_GRACE_SECONDS", result.stderr) + + def test_prism_stale_grace_python3_requirement_matches_prior_prerequisites(self) -> None: + # python3 was already required in production (production difficulty), but + # never for lab bring-up. Validating stale grace must not change that: + # production still fails loudly, lab skips the check instead. + with tempfile.TemporaryDirectory() as temp_dir: + root = Path(temp_dir) + minimal_bin = root / "no-python-bin" + minimal_bin.mkdir() + for tool in ( + "bash", + "sh", + "cat", + "dirname", + "docker", + "env", + "git", + "rm", + "sed", + "tr", + "uname", + ): + resolved = ( + str(self.docker_bin / "docker") + if tool == "docker" + else shutil.which(tool) + ) + if resolved is not None: + (minimal_bin / tool).symlink_to(resolved) + self.assertIsNone( + shutil.which("python3", path=str(minimal_bin)), + "the minimal PATH must not expose python3", + ) + + lab = self.run_check_env( + PATH=str(minimal_bin), + MINING_LANES="prism", + PRISM_STRATUM_STALE_GRACE_SECONDS="not-a-number", + ) + self.assertNotIn("PRISM_STRATUM_STALE_GRACE_SECONDS", lab.stderr) + self.assertIn( + "python3 not found; skipping PRISM_STRATUM_STALE_GRACE_SECONDS validation", + lab.stdout, + ) + + production = self.run_check_env( + PATH=str(minimal_bin), + MINING_LANES="prism", + QBIT_PRODUCTION="1", + QBIT_CHAIN="signet", + QBIT_CHAIN_FLAG="-signet", + PRISM_STRATUM_STALE_GRACE_SECONDS="not-a-number", + ) + self.assertNotEqual(production.returncode, 0) + self.assertIn( + "python3 is required to validate PRISM_STRATUM_STALE_GRACE_SECONDS", + production.stderr, + ) + + def test_prism_rejects_invalid_stale_grace_before_docker_check(self) -> None: + # The exported value is what Compose hands the coordinator, so the + # doctor validates it (not the .env.example default) in every mode, + # with the coordinator's own float syntax and range. + for grace in ("-1", "not-a-number", "nan", "inf", "1" + "0" * 400): + with self.subTest(grace=grace): + result = self.run_check_env( + MINING_LANES="prism", + PRISM_STRATUM_STALE_GRACE_SECONDS=grace, + ) + + self.assertNotEqual(result.returncode, 0) + self.assertIn( + "PRISM_STRATUM_STALE_GRACE_SECONDS must be a finite non-negative number", + result.stderr, + ) + self.assertNotIn("docker is required", result.stderr) + + def test_prism_accepts_runtime_float_stale_grace_syntax(self) -> None: + for grace in ("0", "3", "0.5", "1e-1", ".5", "3."): + with self.subTest(grace=grace): + result = self.run_check_env( + MINING_LANES="prism", + PRISM_STRATUM_STALE_GRACE_SECONDS=grace, + ) + + self.assertNotIn("PRISM_STRATUM_STALE_GRACE_SECONDS", result.stderr) + + self.assertNotEqual(result.returncode, 0) + self.assertIn("docker daemon is not reachable", result.stderr) def test_non_mainnet_production_accepts_bounded_stale_grace(self) -> None: # A public-chain production pool may credit shares that raced a block - # within a bounded grace window; only mainnet pins the strict zero. + # within a bounded grace window. result = self.run_check_env( MINING_LANES="prism", QBIT_PRODUCTION="1", @@ -1215,7 +1435,7 @@ def test_production_requires_commit_pinned_mining_sources(self) -> None: "PRISM_LEDGER_WRITER_EPOCH": "7", "PRISM_AUDIT_DIR": "/var/lib/qbit/prism/audit", "PRISM_EVIDENCE_PATH": "/var/lib/qbit/prism/evidence.json", - "PRISM_STRATUM_STALE_GRACE_SECONDS": "0", + "PRISM_STRATUM_STALE_GRACE_SECONDS": "3", "CKPOOL_MINDIFF": "1024", "CKPOOL_STARTDIFF": "65536", } diff --git a/tests/test_ckpool_startup.py b/tests/test_ckpool_startup.py index 3bbfe54f0..e536a3067 100644 --- a/tests/test_ckpool_startup.py +++ b/tests/test_ckpool_startup.py @@ -18,7 +18,9 @@ START_CKPOOL = ROOT_DIR / "docker" / "ckpool" / "start-ckpool.sh" FAKE_QBIT_RPC = ROOT_DIR / "tests" / "fake_qbit_rpc.py" PREFLIGHT = ROOT_DIR / "docker" / "ckpool" / "qbit-ckpool-preflight.py" +VERSION_MASK = ROOT_DIR / "docker" / "ckpool" / "ckpool-version-mask.py" REGTEST_ADDRESS = "qbrt1staticqbitaddress" +VERSION_MASK_PROBE_MARKER = "start-ckpool-test: version mask helper invoked" def free_port() -> int: @@ -73,6 +75,8 @@ def run_start_ckpool_raw( self, tmpdir: Path, rpc: FakeRpcServer, + *, + real_version_mask: bool = False, **overrides: str, ) -> tuple[subprocess.CompletedProcess[str], Path]: config_file = tmpdir / "ckpool.conf" @@ -80,11 +84,23 @@ def run_start_ckpool_raw( bin_dir = tmpdir / "bin" bin_dir.mkdir() version_mask_helper = bin_dir / "ckpool-version-mask" - version_mask_helper.write_text( - "#!/usr/bin/env bash\n" - "printf '%s\\n' \"${CKPOOL_VERSION_MASK_TEST_VALUE:-1fffe000}\"\n", - encoding="utf-8", - ) + if real_version_mask: + shutil.copy(VERSION_MASK, version_mask_helper) + else: + # The stub records every invocation on stderr and in a marker file so + # tests can assert when the mask is resolved relative to the + # supervisor's readiness gate. + version_mask_helper.write_text( + "#!/usr/bin/env bash\n" + "if [[ \"${1:-}\" == \"--validate-config\" ]]; then\n" + " printf 'stub validate-config\\n' >&2\n" + " exit 0\n" + "fi\n" + f"printf '{VERSION_MASK_PROBE_MARKER}\\n' >&2\n" + "printf 'probe\\n' >> \"${CKPOOL_VERSION_MASK_PROBE_LOG}\"\n" + "printf '%s\\n' \"${CKPOOL_VERSION_MASK_TEST_VALUE:-1fffe000}\"\n", + encoding="utf-8", + ) version_mask_helper.chmod(0o755) preflight_helper = bin_dir / "qbit-ckpool-preflight" shutil.copy(PREFLIGHT, preflight_helper) @@ -105,6 +121,9 @@ def run_start_ckpool_raw( "CKPOOL_LOG_DIR": str(tmpdir / "logs"), "CKPOOL_STATE_DIR": str(state_dir), "QBIT_MINER_ADDRESS_FILE": str(state_dir / "miner-address.txt"), + "CKPOOL_VERSION_MASK_PROBE_LOG": str(tmpdir / "version-mask-probes.log"), + "CKPOOL_VERSION_MASK_PROBE_ATTEMPTS": "1", + "CKPOOL_VERSION_MASK_PROBE_RETRY_SECONDS": "0", } ) env.update(overrides) @@ -119,8 +138,17 @@ def run_start_ckpool_raw( ) return result, config_file - def run_start_ckpool(self, tmpdir: Path, rpc: FakeRpcServer, **overrides: str) -> dict[str, object]: - result, config_file = self.run_start_ckpool_raw(tmpdir, rpc, **overrides) + def run_start_ckpool( + self, + tmpdir: Path, + rpc: FakeRpcServer, + *, + real_version_mask: bool = False, + **overrides: str, + ) -> dict[str, object]: + result, config_file = self.run_start_ckpool_raw( + tmpdir, rpc, real_version_mask=real_version_mask, **overrides + ) self.assertEqual(result.returncode, 0, result.stderr) with config_file.open(encoding="utf-8") as handle: return json.load(handle) @@ -274,6 +302,8 @@ def test_compose_passes_supervisor_settings_to_ckpool_service(self) -> None: "CKPOOL_TEMPLATE_WATCHDOG_POLL_SECONDS": "5", "CKPOOL_TEMPLATE_FAILURE_EXIT_SECONDS": "120", "QBIT_EXPECTED_GENESIS_HASH": "", + "CKPOOL_VERSION_MASK_PROBE_ATTEMPTS": "3", + "CKPOOL_VERSION_MASK_PROBE_RETRY_SECONDS": "2", } for name, default in settings.items(): @@ -293,6 +323,8 @@ def test_production_mainnet_prelaunch_starts_with_explicit_authorization(self) - result, config_file = self.run_start_ckpool_raw( Path(tmp), rpc, + real_version_mask=True, + CKPOOL_VERSION_MASK_MODE="static", QBIT_CHAIN="mainnet", QBIT_PRODUCTION="1", QBIT_TOOLS_PRODUCTION="1", @@ -375,6 +407,176 @@ def test_invalid_numeric_knob_fails_before_config_write(self) -> None: self.assertIn("CKPOOL_STARTDIFF must be finite", result.stderr) self.assertFalse(config_file.exists()) + def test_version_mask_is_not_resolved_before_the_readiness_gate(self) -> None: + """A cold start must not freeze a mask chosen while qbit is unusable. + + The node here rejects getblocktemplate during initial block download, + which is exactly the window a container restart lands in. Startup has to + fail on the supervisor's template validation without ever having asked for a mask. + """ + with tempfile.TemporaryDirectory() as tmp, FakeRpcServer( + "--initialblockdownload", + "--reject-gbt-during-ibd", + ) as rpc: + tmpdir = Path(tmp) + result, config_file = self.run_start_ckpool_raw(tmpdir, rpc) + + self.assertNotEqual(result.returncode, 0) + self.assertNotIn(VERSION_MASK_PROBE_MARKER, result.stderr) + self.assertFalse((tmpdir / "version-mask-probes.log").exists()) + self.assertFalse(config_file.exists()) + + def test_version_mask_is_resolved_after_supervisor_checks_pass(self) -> None: + with tempfile.TemporaryDirectory() as tmp, FakeRpcServer() as rpc: + tmpdir = Path(tmp) + result, config_file = self.run_start_ckpool_raw(tmpdir, rpc) + + self.assertEqual(result.returncode, 0, result.stderr) + gate_index = result.stderr.index("initial supervisor checks: PASS") + probe_index = result.stderr.index(VERSION_MASK_PROBE_MARKER) + self.assertLess(gate_index, probe_index) + self.assertEqual( + (tmpdir / "version-mask-probes.log").read_text(encoding="utf-8").count("probe"), + 1, + ) + self.assertTrue(config_file.exists()) + + def test_cold_start_uses_the_mask_the_ready_node_advertises(self) -> None: + with tempfile.TemporaryDirectory() as tmp, FakeRpcServer( + "--versionrollingmask", "00ffe000" + ) as rpc: + config = self.run_start_ckpool( + Path(tmp), + rpc, + real_version_mask=True, + CKPOOL_VERSION_MASK="1fffe000", + CKPOOL_VERSION_MASK_MODE="dynamic", + ) + + self.assertEqual(config["version_mask"], "00ffe000") + + def test_cold_start_honours_an_advertised_zero_mask(self) -> None: + with tempfile.TemporaryDirectory() as tmp, FakeRpcServer( + "--versionrollingmask", "00000000" + ) as rpc: + config = self.run_start_ckpool( + Path(tmp), + rpc, + real_version_mask=True, + CKPOOL_VERSION_MASK="1fffe000", + CKPOOL_VERSION_MASK_MODE="dynamic", + ) + + self.assertEqual(config["version_mask"], "00000000") + + def test_cold_start_keeps_configured_mask_when_node_omits_the_field(self) -> None: + with tempfile.TemporaryDirectory() as tmp, FakeRpcServer( + "--versionrollingmask", "" + ) as rpc: + config = self.run_start_ckpool( + Path(tmp), + rpc, + real_version_mask=True, + CKPOOL_VERSION_MASK="0000007f", + CKPOOL_VERSION_MASK_MODE="dynamic", + ) + + self.assertEqual(config["version_mask"], "0000007f") + + def test_static_mode_renders_the_configured_mask_without_probing(self) -> None: + with tempfile.TemporaryDirectory() as tmp, FakeRpcServer( + "--versionrollingmask", "00ffe000" + ) as rpc: + config = self.run_start_ckpool( + Path(tmp), + rpc, + real_version_mask=True, + CKPOOL_VERSION_MASK="0x7f", + CKPOOL_VERSION_MASK_MODE="static", + ) + + self.assertEqual(config["version_mask"], "0000007f") + + def test_malformed_mask_configuration_fails_before_state_creation(self) -> None: + cases = { + "CKPOOL_VERSION_MASK": ("not-hex", "invalid fallback CKPOOL_VERSION_MASK"), + "CKPOOL_VERSION_MASK_MODE": ( + "sometimes", + "CKPOOL_VERSION_MASK_MODE must be one of", + ), + } + for name, (value, expected_error) in cases.items(): + with self.subTest(name=name), tempfile.TemporaryDirectory() as tmp, FakeRpcServer() as rpc: + tmpdir = Path(tmp) + result, config_file = self.run_start_ckpool_raw( + tmpdir, + rpc, + real_version_mask=True, + **{name: value}, + ) + + self.assertNotEqual(result.returncode, 0) + self.assertIn(expected_error, result.stderr) + self.assertNotIn("initial supervisor checks: PASS", result.stderr) + self.assertFalse(config_file.exists()) + self.assertFalse((tmpdir / "state").exists()) + + def test_mainnet_prelaunch_requires_static_mode_for_the_version_mask(self) -> None: + """Prelaunch keeps qbit in IBD, so dynamic resolution has to fail closed. + + Operators who want CKPool to start in that window say so explicitly with + CKPOOL_VERSION_MASK_MODE=static rather than getting a silent fallback. + """ + prelaunch = { + "QBIT_CHAIN": "mainnet", + "QBIT_PRODUCTION": "1", + "QBIT_TOOLS_PRODUCTION": "1", + "QBIT_RPC_PASSWORD": "not-default", + "QBIT_MINER_ADDRESS": "qb1staticqbitaddress", + "QBIT_EXPECTED_GENESIS_HASH": "00" * 32, + "CKPOOL_MINDIFF": "1024", + "CKPOOL_STARTDIFF": "65536", + "CKPOOL_NON_TEST_READINESS_GATE": "0", + "QBIT_MAINNET_LAUNCH_READINESS_CHECKS_ENABLED": "0", + "CKPOOL_REQUIRE_P2MR_PAYOUT": "1", + "CKPOOL_VERSION_MASK": "1fffe000", + } + + with tempfile.TemporaryDirectory() as tmp, FakeRpcServer( + "--chain", + "main", + "--initialblockdownload", + "--reject-gbt-during-ibd", + ) as rpc: + result, config_file = self.run_start_ckpool_raw( + Path(tmp), + rpc, + real_version_mask=True, + CKPOOL_VERSION_MASK_MODE="dynamic", + **prelaunch, + ) + + self.assertNotEqual(result.returncode, 0) + self.assertIn("initial supervisor checks: PASS", result.stderr) + self.assertIn("getblocktemplate probe failed", result.stderr) + self.assertFalse(config_file.exists()) + + with tempfile.TemporaryDirectory() as tmp, FakeRpcServer( + "--chain", + "main", + "--initialblockdownload", + "--reject-gbt-during-ibd", + ) as rpc: + config = self.run_start_ckpool( + Path(tmp), + rpc, + real_version_mask=True, + CKPOOL_VERSION_MASK_MODE="static", + **prelaunch, + ) + + self.assertEqual(config["version_mask"], "1fffe000") + if __name__ == "__main__": unittest.main() diff --git a/tests/test_ckpool_version_mask.py b/tests/test_ckpool_version_mask.py index 17c62dc6a..caa9475e9 100644 --- a/tests/test_ckpool_version_mask.py +++ b/tests/test_ckpool_version_mask.py @@ -3,9 +3,17 @@ from __future__ import annotations import importlib.util +import os +import http.client +import io +import subprocess import sys import unittest from pathlib import Path +from unittest import mock +from urllib import error + +from tests.test_ckpool_startup import FakeRpcServer, free_port SCRIPT_PATH = Path(__file__).parents[1] / "docker" / "ckpool" / "ckpool-version-mask.py" @@ -78,6 +86,385 @@ def test_signet_gbt_rules_include_signet(self) -> None: self.assertEqual(ckpool_version_mask.gbt_rules("signet"), ["segwit", "signet"]) self.assertEqual(ckpool_version_mask.gbt_rules("testnet4"), ["segwit"]) + def test_static_mode_uses_configured_mask_without_probing(self) -> None: + env = { + "CKPOOL_VERSION_MASK_MODE": "static", + "CKPOOL_VERSION_MASK": "0x1fffe000", + } + with mock.patch.dict(os.environ, env, clear=False), mock.patch.object( + ckpool_version_mask, "rpc_getblocktemplate" + ) as probe: + result = ckpool_version_mask.resolve_from_env() + + probe.assert_not_called() + self.assertEqual(result.selected_mask, "1fffe000") + self.assertEqual(result.source, "fallback") + self.assertEqual(result.detail, "static_mode") + + def test_static_mode_rejects_invalid_configured_mask(self) -> None: + env = { + "CKPOOL_VERSION_MASK_MODE": "static", + "CKPOOL_VERSION_MASK": "not-hex", + } + with mock.patch.dict(os.environ, env, clear=False): + with self.assertRaisesRegex(ValueError, "CKPOOL_VERSION_MASK"): + ckpool_version_mask.resolve_from_env() + + def test_dynamic_mode_fails_closed_on_permanent_probe_failure(self) -> None: + env = { + "CKPOOL_VERSION_MASK_MODE": "dynamic", + "CKPOOL_VERSION_MASK": "1fffe000", + "CKPOOL_VERSION_MASK_PROBE_ATTEMPTS": "2", + "CKPOOL_VERSION_MASK_PROBE_RETRY_SECONDS": "0", + "QBIT_RPC_USER": "qbitrpc", + "QBIT_RPC_PASSWORD": "secret", + } + sleeps: list[float] = [] + with mock.patch.dict(os.environ, env, clear=False), mock.patch.object( + ckpool_version_mask, + "rpc_getblocktemplate", + side_effect=error.URLError("connection refused"), + ) as probe: + with self.assertRaisesRegex( + ckpool_version_mask.ProbeError, "after 2 attempt" + ): + ckpool_version_mask.resolve_from_env(sleep=sleeps.append) + + self.assertEqual(probe.call_count, 2) + + def test_dynamic_mode_retries_transient_failure_then_succeeds(self) -> None: + env = { + "CKPOOL_VERSION_MASK_MODE": "dynamic", + "CKPOOL_VERSION_MASK": "0000007f", + "CKPOOL_VERSION_MASK_PROBE_ATTEMPTS": "3", + "CKPOOL_VERSION_MASK_PROBE_RETRY_SECONDS": "0.25", + "QBIT_RPC_USER": "qbitrpc", + "QBIT_RPC_PASSWORD": "secret", + } + attempts = [ + OSError("temporary failure in name resolution"), + RuntimeError({"code": -10, "message": "still warming up"}), + {"versionrollingmask": "1fffe000"}, + ] + + def probe(**_kwargs: object) -> dict[str, object]: + outcome = attempts.pop(0) + if isinstance(outcome, Exception): + raise outcome + return outcome + + sleeps: list[float] = [] + with mock.patch.dict(os.environ, env, clear=False), mock.patch.object( + ckpool_version_mask, "rpc_getblocktemplate", side_effect=probe + ): + result = ckpool_version_mask.resolve_from_env(sleep=sleeps.append) + + self.assertEqual(result.selected_mask, "1fffe000") + self.assertEqual(result.source, "qbit_getblocktemplate") + self.assertEqual(result.detail, "advertised") + self.assertEqual(sleeps, [0.25, 0.25]) + + def test_dynamic_mode_does_not_retry_missing_rpc_credentials(self) -> None: + env = { + "CKPOOL_VERSION_MASK_MODE": "dynamic", + "CKPOOL_VERSION_MASK": "1fffe000", + } + with mock.patch.dict(os.environ, env, clear=False), mock.patch.object( + ckpool_version_mask, "rpc_getblocktemplate" + ) as probe: + os.environ.pop("QBIT_RPC_USER", None) + os.environ.pop("QBIT_RPC_PASSWORD", None) + with self.assertRaisesRegex(ValueError, "QBIT_RPC_USER is required"): + ckpool_version_mask.resolve_from_env() + + probe.assert_not_called() + + def test_dynamic_mode_does_not_retry_invalid_advertised_mask(self) -> None: + env = { + "CKPOOL_VERSION_MASK_MODE": "dynamic", + "CKPOOL_VERSION_MASK": "1fffe000", + "CKPOOL_VERSION_MASK_PROBE_ATTEMPTS": "5", + "CKPOOL_VERSION_MASK_PROBE_RETRY_SECONDS": "0", + "QBIT_RPC_USER": "qbitrpc", + "QBIT_RPC_PASSWORD": "secret", + } + with mock.patch.dict(os.environ, env, clear=False), mock.patch.object( + ckpool_version_mask, + "rpc_getblocktemplate", + return_value={"versionrollingmask": "not-hex"}, + ) as probe: + with self.assertRaisesRegex( + ValueError, "invalid getblocktemplate.versionrollingmask" + ): + ckpool_version_mask.resolve_from_env() + + self.assertEqual(probe.call_count, 1) + + def test_dynamic_mode_keeps_zero_and_missing_field_after_successful_probe(self) -> None: + env = { + "CKPOOL_VERSION_MASK_MODE": "dynamic", + "CKPOOL_VERSION_MASK": "1fffe000", + "QBIT_RPC_USER": "qbitrpc", + "QBIT_RPC_PASSWORD": "secret", + } + with mock.patch.dict(os.environ, env, clear=False), mock.patch.object( + ckpool_version_mask, "rpc_getblocktemplate", + return_value={"versionrollingmask": "00000000"}, + ): + result = ckpool_version_mask.resolve_from_env() + self.assertEqual(result.selected_mask, "00000000") + self.assertEqual(result.detail, "disabled_by_zero_mask") + + with mock.patch.dict(os.environ, env, clear=False), mock.patch.object( + ckpool_version_mask, "rpc_getblocktemplate", return_value={} + ): + result = ckpool_version_mask.resolve_from_env() + + self.assertEqual(result.selected_mask, "1fffe000") + self.assertEqual(result.source, "fallback") + self.assertEqual(result.detail, "missing_versionrollingmask") + + def test_probe_settings_reject_invalid_or_unbounded_values_without_rpc(self) -> None: + cases = { + "CKPOOL_VERSION_MASK_RPC_TIMEOUT_SECONDS": ["soon", "0", "-1", "inf", "nan", "1000"], + "CKPOOL_VERSION_MASK_PROBE_ATTEMPTS": ["many", "0", "-1", "1.5", "1000000000"], + "CKPOOL_VERSION_MASK_PROBE_RETRY_SECONDS": ["later", "-1", "inf", "nan", "1000"], + } + for name, values in cases.items(): + for value in values: + with self.subTest(name=name, value=value), mock.patch.dict( + os.environ, {name: value}, clear=True + ), mock.patch.object(ckpool_version_mask, "rpc_getblocktemplate") as probe: + with self.assertRaises(ValueError): + ckpool_version_mask.validate_config() + probe.assert_not_called() + + def test_probe_settings_default_and_accept_zero_retry_delay(self) -> None: + with mock.patch.dict(os.environ, {}, clear=True): + self.assertEqual(ckpool_version_mask.probe_settings(), (5.0, 3, 2.0)) + with mock.patch.dict(os.environ, {"CKPOOL_VERSION_MASK_PROBE_RETRY_SECONDS": "0"}, clear=True): + self.assertEqual(ckpool_version_mask.probe_settings(), (5.0, 3, 0.0)) + + def test_incomplete_http_response_is_retried(self) -> None: + env = { + "QBIT_RPC_USER": "test-user", "QBIT_RPC_PASSWORD": "test-password", + "CKPOOL_VERSION_MASK_PROBE_RETRY_SECONDS": "0", + } + with mock.patch.dict(os.environ, env, clear=True), mock.patch.object( + ckpool_version_mask, "rpc_getblocktemplate", + side_effect=[http.client.IncompleteRead(b"partial"), {"versionrollingmask": "1fffe000"}], + ) as probe: + self.assertEqual(ckpool_version_mask.resolve_from_env().selected_mask, "1fffe000") + self.assertEqual(probe.call_count, 2) + + def test_non_object_rpc_response_fails_cleanly_and_auth_refusal_is_not_retried(self) -> None: + env = { + "QBIT_RPC_USER": "test-user", "QBIT_RPC_PASSWORD": "test-password", + "CKPOOL_VERSION_MASK_PROBE_ATTEMPTS": "2", + "CKPOOL_VERSION_MASK_PROBE_RETRY_SECONDS": "0", + } + with mock.patch.dict(os.environ, env, clear=True), mock.patch.object( + ckpool_version_mask.request, "urlopen", side_effect=lambda *_a, **_kw: io.BytesIO(b"[]") + ) as request_mock: + with self.assertRaisesRegex(ckpool_version_mask.ProbeError, "response was not an object"): + ckpool_version_mask.resolve_from_env() + self.assertEqual(request_mock.call_count, 2) + for status in (401, 403): + with self.subTest(status=status), mock.patch.dict(os.environ, env, clear=True), mock.patch.object( + ckpool_version_mask.request, "urlopen", + side_effect=error.HTTPError("http://qbitd", status, "refused", {}, None), + ) as request_mock: + with self.assertRaisesRegex(ValueError, "RPC authentication refused"): + ckpool_version_mask.resolve_from_env() + self.assertEqual(request_mock.call_count, 1) + + def test_mainnet_prelaunch_dynamic_mode_still_fails_closed(self) -> None: + """Explicit static mode, not a hidden fallback, is the prelaunch escape. + + The authorized mainnet prelaunch combination keeps the node in initial + block download on purpose, so getblocktemplate is unavailable. Dynamic + mode must refuse to invent a mask there; an operator who wants CKPool to + start anyway has to say so with CKPOOL_VERSION_MASK_MODE=static. + """ + env = { + "QBIT_CHAIN": "mainnet", + "QBIT_PRODUCTION": "1", + "QBIT_TOOLS_PRODUCTION": "1", + "CKPOOL_NON_TEST_READINESS_GATE": "0", + "QBIT_MAINNET_LAUNCH_READINESS_CHECKS_ENABLED": "0", + "CKPOOL_VERSION_MASK": "1fffe000", + "CKPOOL_VERSION_MASK_PROBE_ATTEMPTS": "1", + "CKPOOL_VERSION_MASK_PROBE_RETRY_SECONDS": "0", + "QBIT_RPC_USER": "qbitrpc", + "QBIT_RPC_PASSWORD": "secret", + } + gbt_unavailable = RuntimeError( + {"code": -10, "message": "qbit is in initial sync and waiting for blocks"} + ) + + with mock.patch.dict( + os.environ, {**env, "CKPOOL_VERSION_MASK_MODE": "dynamic"}, clear=False + ), mock.patch.object( + ckpool_version_mask, "rpc_getblocktemplate", side_effect=gbt_unavailable + ): + with self.assertRaises(ckpool_version_mask.ProbeError): + ckpool_version_mask.resolve_from_env() + + with mock.patch.dict( + os.environ, {**env, "CKPOOL_VERSION_MASK_MODE": "static"}, clear=False + ), mock.patch.object( + ckpool_version_mask, "rpc_getblocktemplate", side_effect=gbt_unavailable + ) as probe: + result = ckpool_version_mask.resolve_from_env() + + probe.assert_not_called() + self.assertEqual(result.selected_mask, "1fffe000") + self.assertEqual(result.detail, "static_mode") + + +class CkpoolVersionMaskCliTests(unittest.TestCase): + """Exercise the real helper the way startup and other consumers invoke it.""" + + def run_helper(self, *args: str, **overrides: str) -> subprocess.CompletedProcess[str]: + env = os.environ.copy() + for name in ( + "CKPOOL_VERSION_MASK", + "CKPOOL_VERSION_MASK_MODE", + "QBIT_RPC_USER", + "QBIT_RPC_PASSWORD", + "QBIT_CHAIN", + ): + env.pop(name, None) + env.update(overrides) + return subprocess.run( + [sys.executable, str(SCRIPT_PATH), *args], + env=env, + text=True, + capture_output=True, + timeout=30, + ) + + def test_static_mode_prints_mask_on_stdout_and_exits_zero(self) -> None: + result = self.run_helper( + CKPOOL_VERSION_MASK_MODE="static", + CKPOOL_VERSION_MASK="ff", + ) + + self.assertEqual(result.returncode, 0, result.stderr) + self.assertEqual(result.stdout.strip(), "000000ff") + self.assertIn("selected=000000ff source=fallback detail=static_mode", result.stderr) + + def test_dynamic_mode_unreachable_node_exits_nonzero_without_stdout(self) -> None: + result = self.run_helper( + CKPOOL_VERSION_MASK_MODE="dynamic", + CKPOOL_VERSION_MASK="1fffe000", + CKPOOL_VERSION_MASK_PROBE_ATTEMPTS="2", + CKPOOL_VERSION_MASK_PROBE_RETRY_SECONDS="0", + CKPOOL_VERSION_MASK_RPC_TIMEOUT_SECONDS="1", + QBIT_RPC_USER="qbitrpc", + QBIT_RPC_PASSWORD="secret", + QBIT_RPC_HOST="127.0.0.1", + QBIT_RPC_PORT=str(free_port()), + ) + + self.assertNotEqual(result.returncode, 0) + self.assertEqual(result.stdout.strip(), "") + self.assertIn("getblocktemplate probe failed after 2 attempt", result.stderr) + self.assertNotIn("1fffe000\n", result.stdout) + + def test_validate_config_performs_no_rpc_and_reports_configuration_errors(self) -> None: + ok = self.run_helper( + "--validate-config", + CKPOOL_VERSION_MASK_MODE="dynamic", + CKPOOL_VERSION_MASK="1fffe000", + QBIT_RPC_HOST="127.0.0.1", + QBIT_RPC_PORT=str(free_port()), + ) + + self.assertEqual(ok.returncode, 0, ok.stderr) + self.assertEqual(ok.stdout.strip(), "") + self.assertIn("config ok mode=dynamic configured=1fffe000", ok.stderr) + + bad_mask = self.run_helper( + "--validate-config", + CKPOOL_VERSION_MASK_MODE="dynamic", + CKPOOL_VERSION_MASK="not-hex", + ) + self.assertNotEqual(bad_mask.returncode, 0) + self.assertIn("invalid fallback CKPOOL_VERSION_MASK", bad_mask.stderr) + + bad_mode = self.run_helper( + "--validate-config", + CKPOOL_VERSION_MASK_MODE="sometimes", + CKPOOL_VERSION_MASK="1fffe000", + ) + self.assertNotEqual(bad_mode.returncode, 0) + self.assertIn("CKPOOL_VERSION_MASK_MODE must be one of", bad_mode.stderr) + + def test_dynamic_mode_resolves_advertised_mask_from_a_live_node(self) -> None: + with FakeRpcServer("--versionrollingmask", "00ffe000") as rpc: + result = self.run_helper( + CKPOOL_VERSION_MASK_MODE="dynamic", + CKPOOL_VERSION_MASK="1fffe000", + QBIT_RPC_USER="qbitrpc", + QBIT_RPC_PASSWORD="secret", + QBIT_RPC_HOST="127.0.0.1", + QBIT_RPC_PORT=str(rpc.port), + ) + + self.assertEqual(result.returncode, 0, result.stderr) + self.assertEqual(result.stdout.strip(), "00ffe000") + self.assertIn("source=qbit_getblocktemplate detail=advertised", result.stderr) + + def test_dynamic_mode_honours_a_live_zero_mask(self) -> None: + with FakeRpcServer("--versionrollingmask", "00000000") as rpc: + result = self.run_helper( + CKPOOL_VERSION_MASK_MODE="dynamic", + CKPOOL_VERSION_MASK="1fffe000", + QBIT_RPC_USER="qbitrpc", + QBIT_RPC_PASSWORD="secret", + QBIT_RPC_HOST="127.0.0.1", + QBIT_RPC_PORT=str(rpc.port), + ) + + self.assertEqual(result.returncode, 0, result.stderr) + self.assertEqual(result.stdout.strip(), "00000000") + self.assertIn("detail=disabled_by_zero_mask", result.stderr) + + def test_dynamic_mode_falls_back_only_when_a_live_node_omits_the_field(self) -> None: + with FakeRpcServer("--versionrollingmask", "") as rpc: + result = self.run_helper( + CKPOOL_VERSION_MASK_MODE="dynamic", + CKPOOL_VERSION_MASK="0000007f", + QBIT_RPC_USER="qbitrpc", + QBIT_RPC_PASSWORD="secret", + QBIT_RPC_HOST="127.0.0.1", + QBIT_RPC_PORT=str(rpc.port), + ) + + self.assertEqual(result.returncode, 0, result.stderr) + self.assertEqual(result.stdout.strip(), "0000007f") + self.assertIn("source=fallback detail=missing_versionrollingmask", result.stderr) + + def test_dynamic_mode_fails_closed_while_the_node_rejects_templates(self) -> None: + with FakeRpcServer( + "--initialblockdownload", "--reject-gbt-during-ibd" + ) as rpc: + result = self.run_helper( + CKPOOL_VERSION_MASK_MODE="dynamic", + CKPOOL_VERSION_MASK="1fffe000", + CKPOOL_VERSION_MASK_PROBE_ATTEMPTS="2", + CKPOOL_VERSION_MASK_PROBE_RETRY_SECONDS="0", + QBIT_RPC_USER="qbitrpc", + QBIT_RPC_PASSWORD="secret", + QBIT_RPC_HOST="127.0.0.1", + QBIT_RPC_PORT=str(rpc.port), + ) + + self.assertNotEqual(result.returncode, 0) + self.assertEqual(result.stdout.strip(), "") + self.assertIn("initial sync", result.stderr) + if __name__ == "__main__": unittest.main() diff --git a/tests/test_ctv_broadcaster.py b/tests/test_ctv_broadcaster.py index 7bc84c286..e62d2cdd2 100644 --- a/tests/test_ctv_broadcaster.py +++ b/tests/test_ctv_broadcaster.py @@ -13,7 +13,6 @@ CPFP_CHILD_TX_VERSION, REORGED, BroadcasterError, - BroadcastAttempt, CtvFanoutBroadcaster, FanoutArtifact, RpcError, diff --git a/tests/test_ctv_broadcaster_daemon.py b/tests/test_ctv_broadcaster_daemon.py index 254e6f71b..b0ea8c60b 100644 --- a/tests/test_ctv_broadcaster_daemon.py +++ b/tests/test_ctv_broadcaster_daemon.py @@ -7,7 +7,6 @@ from datetime import datetime, timedelta, timezone import threading import time -import urllib.request import unittest from unittest.mock import patch @@ -26,9 +25,14 @@ MAX_CTV_FANOUT_BROADCASTER_CHUNK_SIZE, artifact_from_status_row, ) -from lab.prism.run_ctv_broadcaster_daemon import env_positive_int, make_daemon_from_env +from lab.prism.backfill_ctv_fanouts import build_parser, ledger_from_args +from lab.prism.run_ctv_broadcaster_daemon import ( + env_positive_int, + make_daemon_from_env, + make_ledger_from_env, +) from lab.prism.prism_coordinator import JsonRpc -from lab.prism.share_ledger import SingleWriterShareLedger +from lab.prism.share_ledger import PsqlShareLedger, SingleWriterShareLedger def pending_row(fanout_txid: str = "aa" * 32) -> dict[str, object]: @@ -809,6 +813,62 @@ def test_make_daemon_from_env_requires_wallet_for_cpfp_fee(self) -> None: with self.assertRaisesRegex(SystemExit, "PRISM_CTV_BROADCASTER_WALLET"): make_daemon_from_env() + # The lease/session-guard env vars below tune caller-side lease + # acquisition and the server-side session guards (#137). These two + # operator-run scripts construct PsqlShareLedger directly, so the tests + # pin that operator tuning actually reaches the constructed ledger + # instead of silently falling back to the class defaults. Construction + # is kept offline by stubbing the two lease I/O steps at the end of + # PsqlShareLedger.__init__; everything asserted is set before them. + + LEASE_KNOB_ENV = { + "PRISM_LEDGER_LEASE_ACQUIRE_LOCK_TIMEOUT_SECONDS": "2.5", + "PRISM_LEDGER_LEASE_ACQUIRE_ATTEMPTS": "7", + "PRISM_POSTGRES_IDLE_IN_TRANSACTION_TIMEOUT_SECONDS": "33.5", + "PRISM_POSTGRES_TCP_KEEPALIVES_IDLE_SECONDS": "17", + "PRISM_POSTGRES_TCP_KEEPALIVES_INTERVAL_SECONDS": "4", + "PRISM_POSTGRES_TCP_KEEPALIVES_COUNT": "9", + } + + def assert_lease_knobs_applied(self, ledger: object) -> None: + self.assertIsInstance(ledger, PsqlShareLedger) + self.assertEqual(ledger._lease_acquire_lock_timeout_seconds, 2.5) + self.assertEqual(ledger._lease_acquire_attempts, 7) + guards = ledger._session_guards + self.assertEqual(guards.idle_in_transaction_timeout_seconds, 33.5) + self.assertEqual(guards.tcp_keepalives_idle_seconds, 17) + self.assertEqual(guards.tcp_keepalives_interval_seconds, 4) + self.assertEqual(guards.tcp_keepalives_count, 9) + + def test_make_ledger_from_env_honours_lease_and_session_guard_env(self) -> None: + environ = { + "PRISM_POSTGRES_PSQL_COMMAND": "psql postgresql://qbit@127.0.0.1:1/qbit", + **self.LEASE_KNOB_ENV, + } + with patch.dict("os.environ", environ, clear=True), patch.object( + PsqlShareLedger, "_initialize_writer_lease_guard", lambda self, database_url: None + ), patch.object(PsqlShareLedger, "_ensure_writer_lease", lambda self: None): + ledger = make_ledger_from_env() + try: + self.assert_lease_knobs_applied(ledger) + finally: + ledger.close() + + def test_backfill_construction_honours_lease_and_session_guard_env(self) -> None: + environ = { + "PRISM_POSTGRES_PSQL_COMMAND": "psql postgresql://qbit@127.0.0.1:1/qbit", + **self.LEASE_KNOB_ENV, + } + with patch.dict("os.environ", environ, clear=True), patch.object( + PsqlShareLedger, "_initialize_writer_lease_guard", lambda self, database_url: None + ), patch.object(PsqlShareLedger, "_ensure_writer_lease", lambda self: None): + args = build_parser().parse_args(["--no-init-schema"]) + ledger = ledger_from_args(args) + try: + self.assert_lease_knobs_applied(ledger) + finally: + ledger.close() + def test_json_rpc_wallet_call_uses_wallet_url(self) -> None: seen: list[tuple[str, str]] = [] diff --git a/tests/test_mainnet_compose_contract.py b/tests/test_mainnet_compose_contract.py index a369828ee..cf48dd466 100644 --- a/tests/test_mainnet_compose_contract.py +++ b/tests/test_mainnet_compose_contract.py @@ -20,7 +20,13 @@ "bitcoind", "auxpow-stratum", "prism-postgres", + # The standby the public read tier answers from, so public read volume + # reaches neither the coordinator's process nor its primary. + "prism-postgres-replica", "prism-coordinator", + # The /public/v1 read tier runs in its own process (issue #145) so public + # read volume cannot contend with share acknowledgement or block landing. + "prism-public-api", } @@ -198,6 +204,12 @@ def test_operator_state_uses_explicit_host_bind_mounts(self) -> None: ("prism-coordinator", "/var/lib/qbit-prism/audit"): ( "/srv/qbit-mining-bootstrap/mainnet/prism/audit" ), + ("prism-postgres-replica", "/var/lib/postgresql/data"): ( + "/srv/qbit-mining-bootstrap/mainnet/postgres-replica/data" + ), + ("prism-public-api", "/var/lib/qbit-prism/audit"): ( + "/srv/qbit-mining-bootstrap/mainnet/prism/audit" + ), } for (service, target), source in expected.items(): @@ -229,6 +241,46 @@ def test_postgres_wal_has_a_separate_storage_boundary(self) -> None: ) self.assertEqual(postgres["environment"]["POSTGRES_INITDB_WALDIR"], "/var/lib/postgresql/wal") + def test_public_read_tier_serves_from_the_standby(self) -> None: + # Issue #145: the public read routes answer from a replica, so public + # traffic reaches neither the coordinator's process nor the primary it + # lands blocks through. Rendered from the mainnet .env, which is where + # a deployment that names the primary in PRISM_DATABASE_URL would show + # up as the public tier silently reading it. + environment = self._environment("prism-public-api") + + self.assertIn("prism-postgres-replica", environment["PRISM_DATABASE_URL"]) + self.assertNotIn("@prism-postgres:", environment["PRISM_DATABASE_URL"]) + self.assertEqual(environment["PRISM_PUBLIC_REPLICA_MODE"], "require") + self.assertEqual(environment["PRISM_PUBLIC_REPLICA_MAX_LAG_SECONDS"], "60") + + # The coordinator keeps the primary. + coordinator = self._environment("prism-coordinator") + self.assertIn("@prism-postgres:", coordinator["PRISM_DATABASE_URL"]) + + def test_public_read_tier_waits_on_the_standby_not_the_primary(self) -> None: + depends_on = self.config["services"]["prism-public-api"]["depends_on"] + + self.assertEqual({"prism-postgres-replica"}, set(depends_on)) + self.assertEqual( + "service_healthy", depends_on["prism-postgres-replica"]["condition"] + ) + + def test_replica_storage_is_separate_from_the_primary(self) -> None: + # A standby sharing the primary's data directory is not a standby. + sources = { + service: { + volume["source"] + for volume in self.config["services"][service]["volumes"] + if isinstance(volume, dict) + } + for service in ("prism-postgres", "prism-postgres-replica") + } + + self.assertEqual( + set(), sources["prism-postgres"] & sources["prism-postgres-replica"] + ) + def test_ckpool_runtimes_receive_template_freshness_limit(self) -> None: environment = self._environment("ckpool") self.assertEqual(environment["CKPOOL_TEMPLATE_MAX_AGE_SECONDS"], "120") diff --git a/tests/test_makefile_lifecycle.py b/tests/test_makefile_lifecycle.py index cde8bc73f..ef7944fae 100644 --- a/tests/test_makefile_lifecycle.py +++ b/tests/test_makefile_lifecycle.py @@ -211,6 +211,7 @@ def run_target( bitcoin_node_extra_args: str = "", compose_config_fails: bool = False, confirm: bool = False, + prism_public_stratum_url: str | None = None, ) -> tuple[subprocess.CompletedProcess[str], list[str]]: with tempfile.TemporaryDirectory() as tmp: tmp_path = Path(tmp) @@ -229,6 +230,12 @@ def run_target( printf 'QBIT_CHAIN_FLAG=%s\\n' "$FAKE_QBIT_CHAIN_FLAG" printf 'BITCOIN_CHAIN=%s\\n' "$FAKE_BITCOIN_CHAIN" printf 'BITCOIN_CHAIN_FLAG=%s\\n' "$FAKE_BITCOIN_CHAIN_FLAG" + for key in PRISM_MANIFEST_SIGNING_SEED_HEX PRISM_LEDGER_ATTESTATION_SIGNING_SEED_HEX PRISM_LEDGER_WRITER_PUBLIC_KEY_HEX; do + printf '%s=%s\\n' "$key" "$FAKE_PRISM_SIGNING_KEY" + done + if [ "${FAKE_PRISM_PUBLIC_STRATUM_URL+x}" = x ]; then + printf 'PRISM_PUBLIC_STRATUM_URL=%s\\n' "$FAKE_PRISM_PUBLIC_STRATUM_URL" + fi exit 0 fi printf '%s\\n' "$*" >> "$FAKE_COMPOSE_CALL_LOG" @@ -254,8 +261,12 @@ def run_target( "BITCOIN_CHAIN_FLAG": bitcoin_chain_flag, "QBIT_NODE_EXTRA_ARG": qbit_node_extra_arg, "BITCOIN_NODE_EXTRA_ARGS": bitcoin_node_extra_args, + "FAKE_PRISM_SIGNING_KEY": "11" * 32, } ) + env.pop("FAKE_PRISM_PUBLIC_STRATUM_URL", None) + if prism_public_stratum_url is not None: + env["FAKE_PRISM_PUBLIC_STRATUM_URL"] = prism_public_stratum_url if confirm: env["PURGE_CONFIRM"] = "delete-all-local-mining-data" else: @@ -269,6 +280,10 @@ def run_target( target, f"COMPOSE={fake_compose}", f"COMPOSE_ALL_PROFILES={fake_compose}", + f"PRODUCTION_COMPOSE={fake_compose}", + # Source staging is covered separately; exercise the + # target's own preflight before any service can start. + "WITH_RESOLVED_QBIT=set -euo pipefail;", ], cwd=ROOT, env=env, @@ -279,6 +294,36 @@ def run_target( calls = call_log.read_text(encoding="utf-8").splitlines() if call_log.exists() else [] return result, calls + def test_prism_start_rejects_missing_or_blank_public_stratum_url(self) -> None: + for production in ("0", "1"): + for url in (None, "", " \t "): + with self.subTest(production=production, url=url): + result, calls = self.run_target( + "up-prism-pool", + qbit_production=production, + prism_public_stratum_url=url, + ) + self.assertNotEqual(result.returncode, 0) + self.assertIn("PRISM_PUBLIC_STRATUM_URL is required", result.stderr) + self.assertEqual(calls, []) + + def test_prism_start_with_public_stratum_url_launches_both_services(self) -> None: + for production, launch_flags in ( + ("0", "up --build"), + ("1", "up -d --no-build --pull never"), + ): + with self.subTest(production=production): + result, calls = self.run_target( + "up-prism-pool", + qbit_production=production, + prism_public_stratum_url="stratum+tcp://pool.example:3340", + ) + self.assertEqual(result.returncode, 0, result.stderr) + self.assertEqual( + calls, + [f"--profile prism {launch_flags} qbitd prism-postgres prism-coordinator prism-public-api"], + ) + def test_down_stops_all_profiles_without_deleting_volumes(self) -> None: result, calls = self.run_target("down") diff --git a/tests/test_mining_compose_profiles.py b/tests/test_mining_compose_profiles.py index 213c410f1..72f575bdd 100644 --- a/tests/test_mining_compose_profiles.py +++ b/tests/test_mining_compose_profiles.py @@ -4,11 +4,20 @@ import json import os +import shlex import shutil import subprocess +import tempfile import unittest +import urllib.parse +from contextlib import nullcontext from pathlib import Path from typing import Any +from unittest.mock import Mock, patch + +from lab.prism import public_api, public_read_service +from lab.prism.coordinator_config import CoordinatorConfig, load_coordinator_config +from lab.prism.prism_coordinator import JsonRpc, PrismCoordinator ROOT = Path(__file__).resolve().parents[1] @@ -33,10 +42,13 @@ def setUpClass(cls) -> None: raise unittest.SkipTest(f"docker compose is unavailable: {version.stderr.strip()}") cls.docker = docker - def render_profile(self, profile: str) -> dict[str, Any]: + def render_profile( + self, profile: str, overrides: dict[str, str] | None = None + ) -> dict[str, Any]: inherited_keys = ("PATH", "HOME", "DOCKER_HOST", "DOCKER_CONTEXT", "XDG_CONFIG_HOME") env = {key: os.environ[key] for key in inherited_keys if key in os.environ} env["QBIT_SRC_DIR"] = str(ROOT) + env.update(overrides or {}) completed = subprocess.run( [ self.docker, @@ -70,6 +82,254 @@ def render_profile(self, profile: str) -> dict[str, Any]: ) return json.loads(completed.stdout) + def test_replica_retry_controls_reach_the_container(self) -> None: + attempts = "PRISM_POSTGRES_REPLICA_BASEBACKUP_ATTEMPTS" + retry_seconds = "PRISM_POSTGRES_REPLICA_BASEBACKUP_RETRY_SECONDS" + for overrides, expected in ( + ({}, {attempts: "60", retry_seconds: "5"}), + ({attempts: "120", retry_seconds: "2"}, {attempts: "120", retry_seconds: "2"}), + ): + with self.subTest(overrides=overrides): + config = self.render_profile("prism", overrides) + env = config["services"]["prism-postgres-replica"]["environment"] + for key, value in expected.items(): + self.assertEqual(env.get(key), value, key) + + def test_public_psql_uses_replica_configuration(self) -> None: + primary_command = "psql --host=primary.internal --dbname=qbit" + replica_url = "postgresql://reader@replica.internal:5432/qbit" + replica_command = "psql --host=replica.internal --dbname=qbit --username=reader" + for public_command in ("", replica_command): + with self.subTest(public_command=public_command): + config = self.render_profile("prism", { + "PRISM_POSTGRES_NATIVE_CLIENT": "psql", + "PRISM_POSTGRES_PSQL_COMMAND": primary_command, + "PRISM_PUBLIC_DATABASE_URL": replica_url, + "PRISM_PUBLIC_PSQL_COMMAND": public_command, + }) + services = config["services"] + self.assertEqual( + services["prism-coordinator"]["environment"]["PRISM_POSTGRES_PSQL_COMMAND"], + primary_command, + ) + env = services["prism-public-api"]["environment"] + self.assertEqual(env["PRISM_DATABASE_URL"], replica_url) + env = {key: value for key, value in env.items() if value is not None} + with patch.dict(os.environ, env, clear=True), patch.object( + public_read_service, "build_audit_artifact_store", return_value=None + ): + ledger = public_read_service.build_ledger_from_env(env) + try: + expected = shlex.split(public_command) if public_command else ["psql", replica_url] + self.assertEqual(ledger._command, expected) + self.assertIsNone(ledger._native) + self.assertTrue(ledger._read_only) + finally: + ledger.close() + + def test_public_database_url_uses_configured_replica_credentials(self) -> None: + for user, password, database in ( + ("pool_reader", "synthetic-rotated-password", "pool_ledger"), + ("reader/@?#%: ü", "synthetic/@?#%:' ü", "ledger/@?#% ü"), + ): + credentials = { + "PRISM_POSTGRES_USER": user, + "PRISM_POSTGRES_PASSWORD": password, + "PRISM_POSTGRES_DB": database, + "PRISM_POSTGRES_NATIVE_CLIENT": "psql", + "PRISM_DATABASE_URL": "postgresql://writer@primary.internal/pool_ledger", + } + quote = lambda value: urllib.parse.quote(value, safe="") + derived_url = (f"postgresql://{quote(user)}:{quote(password)}" + f"@prism-postgres-replica:5432/{quote(database)}") + custom_url = "postgresql://reader@standby.internal/pool_ledger" + for overrides, expected in ( + (credentials, derived_url), + ({**credentials, "PRISM_PUBLIC_DATABASE_URL": ""}, derived_url), + ({**credentials, "PRISM_PUBLIC_DATABASE_URL": custom_url}, custom_url), + ): + with self.subTest(user=user, explicit=overrides.get("PRISM_PUBLIC_DATABASE_URL")): + services = self.render_profile("prism", overrides)["services"] + env = {k: v for k, v in services["prism-public-api"]["environment"].items() + if v is not None} + with patch.dict(os.environ, env, clear=True), patch.object( + public_read_service, "build_audit_artifact_store", return_value=None + ): + ledger = public_read_service.build_ledger_from_env(env) + try: + self.assertEqual(ledger._command, ["psql", expected]) + if expected == derived_url: + parsed = urllib.parse.urlsplit(ledger._command[1]) + self.assertEqual(urllib.parse.unquote(parsed.username), user) + self.assertEqual(urllib.parse.unquote(parsed.password), password) + self.assertEqual(urllib.parse.unquote(parsed.path[1:]), database) + finally: + ledger.close() + self.assertEqual( + services["prism-coordinator"]["environment"]["PRISM_DATABASE_URL"], + credentials["PRISM_DATABASE_URL"], + ) + + def test_public_http_limits_reach_the_running_server(self) -> None: + services = self.render_profile("prism", { + "PRISM_PUBLIC_HTTP_MAX_CONNECTIONS": "7", + "PRISM_PUBLIC_HTTP_TIMEOUT_SECONDS": "1.25", + "PRISM_PUBLIC_API_BIND": "127.0.0.1", + })["services"] + env = {k: v for k, v in services["prism-public-api"]["environment"].items() + if v is not None} + # The test listener uses an ephemeral port; Compose needs a real target. + env["PRISM_PUBLIC_API_PORT"] = "0" + observed = [] + + def inspect_server(server): + self.assertEqual(server.connection_timeout, 1.25) + for _ in range(7): + self.assertTrue(server._connection_slots.acquire(blocking=False)) + self.assertFalse(server._connection_slots.acquire(blocking=False)) + for _ in range(7): + server._connection_slots.release() + observed.append(server) + + with patch.dict(os.environ, env, clear=True), patch.object( + public_read_service, "build_service", + return_value=(object(), Mock(), public_read_service.ServiceMetrics(), None), + ), patch.object(public_read_service.signal, "signal"), patch.object( + public_read_service.BoundedPublicHTTPServer, "serve_forever", inspect_server, + ): + self.assertEqual(public_read_service.main(), 0) + self.assertEqual(len(observed), 1) + + def test_public_minimum_payout_preserves_pool_fallbacks(self) -> None: + for overrides, expected in ( + ({}, 0), + ({"PRISM_PAYOUT_MIN_OUTPUT_SATS": "500"}, 500), + ({"PRISM_PAYOUT_MIN_OUTPUT_BITS": "700", "PRISM_PAYOUT_MIN_OUTPUT_SATS": "500"}, 700), + ({"PRISM_PUBLIC_MINIMUM_PAYOUT_BITS": "900", "PRISM_PAYOUT_MIN_OUTPUT_BITS": "700"}, 900), + ({"PRISM_PUBLIC_MINIMUM_PAYOUT_BITS": "0", "PRISM_PAYOUT_MIN_OUTPUT_SATS": "500"}, 0), + ): + with self.subTest(overrides=overrides): + services = self.render_profile("prism", overrides)["services"] + env = services["prism-public-api"]["environment"] + env = {key: value for key, value in env.items() if value is not None} + with patch.dict(os.environ, env, clear=True): + self.assertEqual(public_api.public_minimum_payout_bits(), expected) + + def coordinator_config(self, overrides: dict[str, str]) -> CoordinatorConfig: + services = self.render_profile("prism", overrides)["services"] + env = services["prism-coordinator"]["environment"] + env = {key: value for key, value in env.items() if value is not None} + # Supply fixture signing material so configuration validation reaches + # the runtime knobs without starting a coordinator or opening storage. + env.update({ + "PRISM_MANIFEST_SIGNING_SEED_HEX": "11" * 32, + "PRISM_LEDGER_ATTESTATION_SIGNING_SEED_HEX": "22" * 32, + "PRISM_LEDGER_WRITER_PUBLIC_KEY_HEX": "33" * 32, + }) + return load_coordinator_config(env) + + def test_vardiff_resume_controls_reach_runtime_config(self) -> None: + for overrides, expected in ( + ({}, (True, 900, 8192, 1024)), + ({ + "PRISM_STRATUM_VARDIFF_RESUME": "0", + "PRISM_STRATUM_VARDIFF_RESUME_TTL_SECONDS": "42.5", + "PRISM_STRATUM_VARDIFF_RESUME_MAX_ENTRIES": "33", + "PRISM_STRATUM_VARDIFF_RESUME_MAX_START_FACTOR": "8", + }, (False, 42.5, 33, 8)), + ): + with self.subTest(overrides=overrides): + config = self.coordinator_config(overrides).stratum + self.assertEqual(( + config.vardiff_resume_enabled, + config.vardiff_resume_ttl_seconds, + config.vardiff_resume_max_entries, + config.vardiff_resume_max_start_factor, + ), expected) + + def test_rust_window_switch_reaches_runtime_config(self) -> None: + for overrides, expected in ( + ({}, False), + ({"PRISM_WINDOW_PIPELINE_RUST": "1"}, True), + ({"PRISM_WINDOW_PIPELINE_RUST": "0"}, False), + ): + with self.subTest(overrides=overrides): + config = self.coordinator_config(overrides) + self.assertEqual(config.jobs.window_pipeline_rust_enabled, expected) + with self.assertRaisesRegex(SystemExit, "PRISM_WINDOW_PIPELINE_RUST"): + self.coordinator_config({"PRISM_WINDOW_PIPELINE_RUST": "invalid"}) + + def test_hashrate_rollup_controls_reach_coordinator(self) -> None: + for overrides, expected in ( + ({}, (True, 15.0, 50000)), + ({ + "PRISM_HASHRATE_ROLLUP_ENABLED": "0", + "PRISM_HASHRATE_ROLLUP_INTERVAL_SECONDS": "42.5", + "PRISM_HASHRATE_ROLLUP_BATCH_SHARES": "1000", + }, (False, 42.5, 1000)), + ): + with self.subTest(overrides=overrides), tempfile.TemporaryDirectory() as root: + services = self.render_profile("prism", overrides)["services"] + env = services["prism-coordinator"]["environment"] + env = {key: value for key, value in env.items() if value is not None} + # The constructor reads these controls from the process + # environment. Keep its unrelated storage and RPC local. + config = load_coordinator_config({ + "QBIT_RPC_HOST": "qbit.example", + "QBIT_RPC_USER": "rpc-user", + "QBIT_RPC_PASSWORD": "rpc-password", + "PRISM_ALLOW_MEMORY_LEDGER": "1", + "PRISM_ALLOW_TEST_SIGNING_SEEDS": "1", + "PRISM_ALLOW_BUNDLE_EMBEDDED_LEDGER_KEY": "1", + "PRISM_AUDIT_DIR": root, + "PRISM_EVIDENCE_PATH": str(Path(root) / "evidence.json"), + }) + with patch.dict(os.environ, env, clear=True), patch.object( + JsonRpc, "call", side_effect=RuntimeError("offline") + ): + coordinator = PrismCoordinator(config) + self.assertEqual(( + coordinator.hashrate_rollup_enabled, + coordinator.hashrate_rollup_interval_seconds, + coordinator.hashrate_rollup_batch_shares, + ), expected) + + def test_public_read_deadline_reaches_origin_dispatch(self) -> None: + for overrides, expected in ( + ({}, 20), + ({"PRISM_PUBLIC_READ_STATEMENT_TIMEOUT_SECONDS": "7"}, 7), + ({"PRISM_PUBLIC_READ_STATEMENT_TIMEOUT_SECONDS": "0"}, 0), + ): + with self.subTest(overrides=overrides): + services = self.render_profile("prism", overrides)["services"] + env = services["prism-public-api"]["environment"] + env = {key: value for key, value in env.items() if value is not None} + coordinator = Mock() + timeout = coordinator.ledger.operation_timeout + timeout.return_value = nullcontext() + with patch.dict(os.environ, env, clear=True), patch.object( + public_api, "dispatch", return_value=(200, {"ok": True}) + ) as dispatch: + result = public_read_service.bounded_public_dispatch( + coordinator, "/public/v1/pool", {}, occupies_read_slot=True + ) + self.assertEqual(result, (200, {"ok": True})) + dispatch.assert_called_once_with(coordinator, "/public/v1/pool", {}) + if expected: + timeout.assert_called_once_with(float(expected)) + else: + timeout.assert_not_called() + + def test_candidate_cleanup_backlog_limit_reaches_runtime_config(self) -> None: + key = "PRISM_BLOCK_CANDIDATE_CLEANUP_RETRY_BACKLOG_MAX" + for overrides, expected in (({}, 4096), ({key: "128"}, 128), ({key: "8192"}, 8192)): + with self.subTest(overrides=overrides): + config = self.coordinator_config(overrides) + self.assertEqual(config.block.candidate_cleanup_retry_backlog_max, expected) + for invalid in ("0", "8193"): + with self.subTest(invalid=invalid), self.assertRaisesRegex(SystemExit, key): + self.coordinator_config({key: invalid}) + def test_each_mining_profile_has_an_exact_service_graph(self) -> None: expected = { "permissionless": {"qbitd", "ckpool", "permissionless-miner"}, @@ -82,7 +342,16 @@ def test_each_mining_profile_has_an_exact_service_graph(self) -> None: "auxpow-stratum", "auxpow-real-miner", }, - "prism": {"qbitd", "prism-postgres", "prism-coordinator"}, + # prism-public-api serves the extracted /public/v1 read tier + # in its own process (issue #145), against the standby that + # prism-postgres-replica streams from the coordinator's primary. + "prism": { + "qbitd", + "prism-postgres", + "prism-postgres-replica", + "prism-coordinator", + "prism-public-api", + }, } for profile, services in expected.items(): diff --git a/tests/test_prism_accepted_block_accounting.py b/tests/test_prism_accepted_block_accounting.py new file mode 100644 index 000000000..77b262cba --- /dev/null +++ b/tests/test_prism_accepted_block_accounting.py @@ -0,0 +1,656 @@ +#!/usr/bin/env python3 +"""Definitive-acceptance priority at the block-accounting handoff (#181). + +The 2026-08-21 testnet4 run left a tail of accepted-parent preview waits -- +the first two at only 0.32 PH/s -- behind stale same-height replay work that +the accounting lane served first. The task that carries the decided block +already knows it is decided: its ``_BlockCandidateNodeSubmission`` is an +attempted offer that raised nothing and returned nothing. + +These tests own that classification and the lane it selects: + +* the discriminator itself, including the two shapes that are easy to + misread as acceptance (a transport failure, which also leaves ``result`` + at ``None``, and a ``"duplicate"`` rejection); +* the issue's own counterexample -- three durable same-height replay tasks + queued before one definitively accepted task -- now draining accepted + first; +* the burst case, where the primary handoff queue is at its bound and + spillover has already begun. This is the case a priority *key* on the + primary queue cannot reach: once the overflow queue is non-empty every + later handoff joins it, and the loop drains the primary in full before it + ever looks at the overflow, so the accepted task would be served last + exactly when it matters most; +* the fairness bound that keeps the accepted lane from becoming a + starvation lane, stated as a bound rather than as an argument; and +* the spillover put rule's own anti-starvation property, which this change + leaves byte-for-byte alone and which is asserted here so it stays that + way. +""" + +from __future__ import annotations + +import itertools +import sys +import threading +import unittest +from contextlib import redirect_stdout +from io import StringIO +from pathlib import Path +from types import SimpleNamespace +from typing import Any, Callable + +if not __package__: + sys.path.insert(0, str(Path(__file__).resolve().parents[1])) + +from lab.prism.block_candidates import ( # noqa: E402 + BLOCK_ACCOUNTING_ACCEPTED_DISPATCH_QUOTA, + DEFAULT_BLOCK_ACCOUNTING_QUEUE_DEPTH, + _BlockCandidateAccountingTask, + _BlockCandidateNodeSubmission, + _is_definitive_node_acceptance, +) +from lab.prism.share_ledger import ( # noqa: E402 + PendingShare, + SingleWriterShareLedger, +) +from tests.prism_vardiff_test_support import ( # noqa: E402 + RejectingSubmitTipRpc, + block_candidate, + submit_coordinator, +) + + +# The parent every seeded job builds on, and the tip the chain moved to +# while the stale candidate sat in the durable outbox. +PARENT_TIP = "00" * 32 +MOVED_TIP = "77" * 32 + + +class AccountingLaneRig: + """Drive the shipped accounting loop over controllable lane traffic. + + Only the three sinks the loop reaches are replaced -- the accounting task + runner, the invalid-candidate quarantine drain and the deferred-cleanup + retry -- so the lane selection under test, the three queues, the handoff + routing and the dispatch quota all stay exactly as shipped. A real + :class:`_BlockCandidateAccountingTask` is still run through the shipped + ``_run_block_accounting_task``; only the synthetic placeholders are + counted and dropped. + + The loop runs on the calling thread, which makes dispatch order a plain + list rather than a race. It is bounded twice over: the quarantine stub + counts the loop's empty passes and stops it after a couple of them, and + a watchdog timer sets the stop event if the loop somehow never reaches + an empty pass. A starved lane therefore ends the test with a report + instead of an unbounded wait. + """ + + def __init__( + self, + *, + quota: int | None = None, + idle_passes_before_stop: int = 2, + ledger: Any = None, + rpc: Any = None, + ) -> None: + server, state, _recording = submit_coordinator() + server.max_blocks = 1 << 30 + server.stop_after_block = False + server.block_candidate_retry_initial_seconds = 0.0 + if quota is not None: + server.block_accounting_accepted_dispatch_quota = int(quota) + if ledger is not None: + server.ledger = ledger + if rpc is not None: + server.rpc = rpc + self.server = server + self.state = state + self.ledger = server.ledger + self.rpc = server.rpc + self.service = server._ensure_block_candidate_service() + self.service._ensure_block_accounting_state() + self.service._ensure_block_replay_state() + self.service._ensure_block_candidate_disposition_state() + # Dispatch order, one label per task the lane selected. + self.dispatched: list[str] = [] + # Called after each dispatch so a scenario can hand off more work + # mid-drain, which is what the submitter thread does in production. + self.on_dispatch: Callable[[Any], None] | None = None + self._shipped_run = self.service._run_block_accounting_task + self._idle_passes = 0 + self._idle_ceiling = int(idle_passes_before_stop) + self._labels = itertools.count() + server._run_block_accounting_task = self._run_task + server._run_one_invalid_block_candidate_quarantine = self._idle_pass + self.service._run_one_collapsed_block_candidate_cleanup_retry = ( + self._no_cleanup + ) + + # -- the replaced sinks ------------------------------------------------ + + def _run_task(self, task: Any) -> None: + self.dispatched.append(self.label(task)) + if isinstance(task, _BlockCandidateAccountingTask): + self._shipped_run(task) + if self.on_dispatch is not None: + self.on_dispatch(task) + + def _idle_pass(self) -> bool: + self._idle_passes += 1 + if self._idle_passes >= self._idle_ceiling: + self.server.stop_event.set() + return False + + @staticmethod + def _no_cleanup() -> bool: + return False + + # -- traffic ----------------------------------------------------------- + + @staticmethod + def label(task: Any) -> str: + return str( + getattr(task, "label", None) + or task.candidate.submission.block_hash_hex + ) + + def enqueue( + self, + label: str, + *, + accepted: bool, + height: int = 10, + ) -> Any: + """Hand one synthetic task to the shipped routing. + + The task carries only what ``_enqueue_block_accounting_task`` reads: + the template height it keys on, the hash its spill log names, and the + node submission whose classification chooses the lane. + """ + node_submission = ( + _BlockCandidateNodeSubmission(attempted=True) + if accepted + else _BlockCandidateNodeSubmission(attempted=False) + ) + task = SimpleNamespace( + label=label, + candidate=SimpleNamespace( + context=SimpleNamespace(template={"height": int(height)}), + submission=SimpleNamespace(block_hash_hex=label), + ), + node_submission=node_submission, + ) + # The spill log is part of the rule this scenario exercises; it is + # captured rather than printed so the suite stays readable. + with redirect_stdout(StringIO()): + assert self.server._enqueue_block_accounting_task(task) + return task + + def accepted_stream(self, *, until: int) -> Callable[[Any], None]: + """Keep the accepted lane non-empty for the whole drain. + + Every accepted dispatch immediately hands off another accepted task, + so the lane is never observed empty while stale work waits -- which + is the only condition under which the fairness bound says anything. + The stream stops after ``until`` total dispatches so the drain ends. + """ + + def rearm(task: Any) -> None: + if len(self.dispatched) >= until: + return + if _is_definitive_node_acceptance(task.node_submission): + self.enqueue( + f"accepted-{next(self._labels)}", + accepted=True, + ) + + return rearm + + # -- the drain --------------------------------------------------------- + + def drain(self, *, timeout: float = 15.0) -> list[str]: + watchdog = threading.Timer(timeout, self.server.stop_event.set) + watchdog.start() + try: + with redirect_stdout(StringIO()): + self.service.block_accounting_loop() + finally: + watchdog.cancel() + return list(self.dispatched) + + # -- queue inspection -------------------------------------------------- + + @property + def accepted_queue(self) -> Any: + return self.service._block_accounting_accepted_queue + + @property + def primary_queue(self) -> Any: + return self.service._block_accounting_queue + + @property + def overflow_queue(self) -> Any: + return self.service._block_accounting_overflow_queue + + +def durable_candidate( + rig: AccountingLaneRig, + block_hash: str, + *, + tag: str = "aa", +) -> Any: + """Seed one durable pending outbox row and return its candidate.""" + pending = PendingShare( + share_id=f"miner-a:{block_hash}", + miner_id="miner-a", + order_key="miner-a", + p2mr_program_hex="11" * 32, + share_difficulty=1, + network_difficulty=1, + template_height=9, + job_id="job-1", + job_issued_at_ms=1, + accepted_at_ms=1, + ntime=1, + ) + candidate = block_candidate( + rig.server, + rig.state, + SimpleNamespace( + coinbase_tx_hex="00", + block_hash_hex=block_hash, + block_hex=tag, + share_pass=True, + block_pass=True, + ), + pending_share=pending, + ) + rig.ledger.append_batch( + [(pending, rig.server.block_candidate_intent(candidate))] + ) + return candidate + + +def durable_accounting_task( + rig: AccountingLaneRig, + candidate: Any, +) -> _BlockCandidateAccountingTask: + """Queue the durable candidate the way the deferred handoff leaves it. + + ``attempted=False`` is the replay shape: the row was adopted from the + outbox, so no offer has been made for it yet and it is not accepted + evidence. The disposition lease travels with the task until the + accounting lane releases it. + """ + block_hash = str(candidate.submission.block_hash_hex) + lease = rig.server._claim_block_candidate_disposition( + block_hash, + blocking=False, + ) + assert lease is not None + task = _BlockCandidateAccountingTask( + candidate=candidate, + node_submission=_BlockCandidateNodeSubmission(attempted=False), + disposition_lease=lease, + ) + assert rig.server._enqueue_block_accounting_task(task) + return task + + +class DefinitiveNodeAcceptanceTests(unittest.TestCase): + """The discriminator, and the two sites that must never disagree.""" + + def test_only_an_attempted_silent_offer_is_definitive_acceptance( + self, + ) -> None: + self.assertTrue( + _is_definitive_node_acceptance( + _BlockCandidateNodeSubmission(attempted=True) + ) + ) + # A transport failure builds attempted=True with error set, which + # leaves result at its None default: a two-clause test that dropped + # `error is None` would read this failed offer as an acceptance. + self.assertFalse( + _is_definitive_node_acceptance( + _BlockCandidateNodeSubmission( + attempted=True, + error=RuntimeError("submitblock timed out"), + ) + ) + ) + # A rejection string is a result, so it is not silent acceptance. + self.assertFalse( + _is_definitive_node_acceptance( + _BlockCandidateNodeSubmission(attempted=True, result="duplicate") + ) + ) + # No offer has been made at all. + self.assertFalse( + _is_definitive_node_acceptance( + _BlockCandidateNodeSubmission(attempted=False) + ) + ) + self.assertFalse(_is_definitive_node_acceptance(None)) + + def test_the_handoff_routes_exactly_the_definitive_acceptances( + self, + ) -> None: + rig = AccountingLaneRig() + rig.enqueue("accepted", accepted=True) + self.assertEqual(rig.accepted_queue.qsize(), 1) + self.assertTrue(rig.primary_queue.empty()) + for label, node_submission in ( + ("errored", _BlockCandidateNodeSubmission( + attempted=True, + error=RuntimeError("boom"), + )), + ("duplicate", _BlockCandidateNodeSubmission( + attempted=True, + result="duplicate", + )), + ("unattempted", _BlockCandidateNodeSubmission(attempted=False)), + ): + task = SimpleNamespace( + label=label, + candidate=SimpleNamespace( + context=SimpleNamespace(template={"height": 10}), + submission=SimpleNamespace(block_hash_hex=label), + ), + node_submission=node_submission, + ) + self.assertTrue(rig.server._enqueue_block_accounting_task(task)) + # None of the three joined the accepted lane. + self.assertEqual(rig.accepted_queue.qsize(), 1) + self.assertEqual(rig.primary_queue.qsize(), 3) + + def test_the_retained_offer_stash_shares_the_one_predicate(self) -> None: + rig = AccountingLaneRig() + service = rig.service + + def retained() -> dict[str, Any]: + return dict( + getattr( + service, + "_block_candidate_retained_node_submissions", + {}, + ) + ) + + for block_hash, node_submission in ( + ("11" * 32, _BlockCandidateNodeSubmission( + attempted=True, + error=RuntimeError("boom"), + )), + ("22" * 32, _BlockCandidateNodeSubmission( + attempted=True, + result="duplicate", + )), + ("33" * 32, _BlockCandidateNodeSubmission(attempted=False)), + ("44" * 32, None), + ): + service._stash_retained_block_candidate_node_submission( + block_hash, + node_submission, + ) + self.assertEqual(retained(), {}) + definitive = _BlockCandidateNodeSubmission(attempted=True) + service._stash_retained_block_candidate_node_submission( + "55" * 32, + definitive, + ) + self.assertEqual(retained(), {"55" * 32: definitive}) + + +class AcceptedAccountingPriorityTests(unittest.TestCase): + """Dispatch order: the issue's counterexample and the burst it hides in.""" + + def test_accepted_evidence_drains_before_queued_same_height_replay( + self, + ) -> None: + """#181's BLOCKED integration review, now passing. + + Three durable same-height replay tasks are queued and a definitively + accepted task is queued afterward. The reported drain order was + replay 1, replay 2, replay 3, accepted -- every one of those replay + tasks costing ~6 RPCs, two ledger writes and a payout-balance + mutation before the decided block's accounting could start. + """ + rig = AccountingLaneRig() + for index in (1, 2, 3): + rig.enqueue(f"replay-{index}", accepted=False, height=10) + rig.enqueue("accepted", accepted=True, height=10) + self.assertEqual( + rig.drain(), + ["accepted", "replay-1", "replay-2", "replay-3"], + ) + + def test_accepted_evidence_overtakes_a_spilled_stale_backlog( + self, + ) -> None: + """The burst case, which a priority key cannot reach. + + The primary queue is at its bound and spillover has begun before the + accepted task is handed off. A key on the primary heap is inert + here: the accepted task would be *put* into the overflow queue by + the unchanged spillover rule, and the loop drains the bounded + primary in full before it looks at the overflow at all. Ordering by + lane rather than by key is what makes the accepted class + structurally un-spillable. + + The accepted task is deliberately given the highest height and the + last arrival sequence in the whole batch, so nothing about its key + can explain it draining first. + """ + rig = AccountingLaneRig() + depth = DEFAULT_BLOCK_ACCOUNTING_QUEUE_DEPTH + self.assertEqual(rig.primary_queue.maxsize, depth) + for index in range(depth): + rig.enqueue(f"stale-{index}", accepted=False, height=10) + self.assertTrue(rig.primary_queue.full()) + # The burst state itself: this handoff finds the primary full and + # spills, and the one after it joins the spill by the put rule. + rig.enqueue("spilled-0", accepted=False, height=10) + self.assertFalse(rig.overflow_queue.empty()) + rig.enqueue("spilled-1", accepted=False, height=10) + self.assertEqual(rig.overflow_queue.qsize(), 2) + rig.enqueue("accepted", accepted=True, height=99) + self.assertEqual(rig.accepted_queue.qsize(), 1) + + order = rig.drain() + self.assertEqual(order[0], "accepted") + self.assertEqual( + order, + ["accepted"] + + [f"stale-{index}" for index in range(depth)] + + ["spilled-0", "spilled-1"], + ) + + def test_a_newer_stale_handoff_never_jumps_an_older_spill_entry( + self, + ) -> None: + """The spillover put rule's anti-starvation property, unchanged. + + Membership is what this change moved, not the rule. Once the + overflow queue is non-empty, a later non-accepted handoff still + joins it rather than refilling the bounded primary in front of the + older spill entries -- even when the primary has since drained and + has room again, which is exactly when refilling would starve them. + """ + rig = AccountingLaneRig(quota=1) + rig.server.block_accounting_queue_depth = 1 + del rig.service._block_accounting_queue + rig.service._ensure_block_accounting_state() + self.assertEqual(rig.primary_queue.maxsize, 1) + + rig.enqueue("primary", accepted=False, height=10) + rig.enqueue("spilled-old", accepted=False, height=10) + self.assertEqual(rig.overflow_queue.qsize(), 1) + + def hand_off_newer(task: Any) -> None: + if AccountingLaneRig.label(task) != "primary": + return + # The primary queue has just been emptied by this dispatch, so + # it has room; the put rule must still send the newer handoff + # behind the older spill entry. + self.assertTrue(rig.primary_queue.empty()) + rig.enqueue("newer", accepted=False, height=10) + self.assertTrue(rig.primary_queue.empty()) + self.assertEqual(rig.overflow_queue.qsize(), 2) + + rig.on_dispatch = hand_off_newer + self.assertEqual(rig.drain(), ["primary", "spilled-old", "newer"]) + + +class AcceptedDispatchQuotaTests(unittest.TestCase): + """Bounded fairness (D3): stale replay is delayed, never starved.""" + + QUOTAS = (1, BLOCK_ACCOUNTING_ACCEPTED_DISPATCH_QUOTA) + + def test_the_default_quota_is_four(self) -> None: + self.assertEqual(BLOCK_ACCOUNTING_ACCEPTED_DISPATCH_QUOTA, 4) + + def test_a_stale_task_waits_behind_at_most_the_quota(self) -> None: + """The acceptance criterion, stated the way D3 states it. + + With a continuous stream of accepted tasks and at least one stale + task pending, the stale task is dispatched after at most ``quota`` + accepted services -- never behind an unbounded stream of them, + whatever the accepted arrival rate. + """ + for quota in self.QUOTAS: + with self.subTest(quota=quota): + rig = AccountingLaneRig(quota=quota) + rig.enqueue("stale", accepted=False, height=10) + rig.on_dispatch = rig.accepted_stream(until=8 * quota) + rig.enqueue("accepted-seed", accepted=True, height=10) + + order = rig.drain() + self.assertIn("stale", order) + waited_behind = order.index("stale") + self.assertLessEqual(waited_behind, quota) + # Every dispatch it waited behind was an accepted one, so + # the count above really is the accepted-service bound. + self.assertTrue( + all( + label.startswith("accepted") + for label in order[:waited_behind] + ), + order, + ) + # And the quota is a bound on delay, not a ban on accepted + # work: the stream kept running afterwards. + self.assertGreater(len(order) - waited_behind, 1) + + def test_the_quota_only_yields_to_work_that_exists(self) -> None: + """An exhausted quota with no stale work still dispatches accepted. + + The bound is defined against *available* non-accepted work. With + the stale lanes empty there is nothing to yield to, so an accepted + backlog keeps draining instead of stalling behind a spent counter. + """ + rig = AccountingLaneRig(quota=1) + for index in range(5): + rig.enqueue(f"accepted-{index}", accepted=True, height=10) + self.assertEqual( + rig.drain(), + [f"accepted-{index}" for index in range(5)], + ) + + def test_a_delayed_stale_task_still_reaches_terminal_disposition( + self, + ) -> None: + """Delay is the fix; starvation would be a different bug. + + A stale accounting task that never runs leaves its durable outbox + row pending, its offer-time accepted-block payout-preview barrier + armed and its disposition lease held. The stale task here is a real + one with a real pending outbox row whose parent tip has since moved, + run through the shipped ``_run_block_accounting_task``, while the + accepted lane is kept continuously non-empty around it. Delayed by + the quota, it still reaches its terminal abandonment. + """ + for quota in self.QUOTAS: + with self.subTest(quota=quota): + block_hash = "e7" * 32 + rig = AccountingLaneRig( + quota=quota, + ledger=SingleWriterShareLedger(), + rpc=RejectingSubmitTipRpc(MOVED_TIP), + ) + candidate = durable_candidate(rig, block_hash, tag="e7") + self.assertEqual( + rig.ledger._block_candidate_outbox[block_hash]["state"], + "pending", + ) + durable_accounting_task(rig, candidate) + rig.on_dispatch = rig.accepted_stream(until=8 * quota) + rig.enqueue("accepted-seed", accepted=True, height=10) + + order = rig.drain() + self.assertIn(block_hash, order) + self.assertLessEqual(order.index(block_hash), quota) + self.assertEqual( + rig.ledger._block_candidate_outbox[block_hash]["state"], + "abandoned", + ) + with rig.server.lock: + self.assertIn( + block_hash, + rig.service._block_candidate_terminal_outcomes, + ) + # The lease travelled with the task and the lane released it. + with rig.service._block_candidate_disposition_registry_lock: + self.assertNotIn( + block_hash, + rig.service._block_candidate_disposition_flights, + ) + + +class AcceptedLaneShapeTests(unittest.TestCase): + """The lane's own registration and unboundedness.""" + + def test_the_accepted_lane_is_unbounded(self) -> None: + """A maxsize here would need a spill target, and any queue-state + dependent spill is the inversion this lane removes. The offer has + already happened, so it may never be converted back into a + raw-submit retry; max-block admission bounds how many unresolved + real offers can exist at once. + """ + rig = AccountingLaneRig() + self.assertEqual(rig.accepted_queue.maxsize, 0) + self.assertEqual(rig.overflow_queue.maxsize, 0) + self.assertEqual( + rig.primary_queue.maxsize, + DEFAULT_BLOCK_ACCOUNTING_QUEUE_DEPTH, + ) + for index in range(4 * DEFAULT_BLOCK_ACCOUNTING_QUEUE_DEPTH): + rig.enqueue(f"accepted-{index}", accepted=True, height=10) + self.assertEqual( + rig.accepted_queue.qsize(), + 4 * DEFAULT_BLOCK_ACCOUNTING_QUEUE_DEPTH, + ) + self.assertTrue(rig.primary_queue.empty()) + self.assertTrue(rig.overflow_queue.empty()) + + def test_the_accepted_lane_is_reachable_through_the_coordinator( + self, + ) -> None: + """The state-field registration, which only a delegate-backed + coordinator exercises: the lane must be readable and writable under + its historical coordinator attribute name like the other two. + """ + rig = AccountingLaneRig() + self.assertIs( + rig.server._block_accounting_accepted_queue, + rig.service._block_accounting_accepted_queue, + ) + rig.server._block_accounting_accepted_queue.put_nowait( + (10, 0, SimpleNamespace()) + ) + self.assertEqual(rig.service._block_accounting_accepted_queue.qsize(), 1) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_prism_accepted_preview_publication_latency.py b/tests/test_prism_accepted_preview_publication_latency.py new file mode 100644 index 000000000..2a5aad979 --- /dev/null +++ b/tests/test_prism_accepted_preview_publication_latency.py @@ -0,0 +1,880 @@ +#!/usr/bin/env python3 +"""Acceptance-to-preview-publication latency, driven by accepted-block cadence. + +Issue #181 item 3. The 2026-08-21 testnet4 validation run had #190/#191/#196 +deployed and demonstrably active, and the accepted-parent tail still dominated: +25 accepted-parent preview wait timeouts, the first two at only 0.32 PH/s, +semantic current-work coverage repeatedly at 0.0, and the coordinator using +about 1.26 CPU cores at ~8.5 PH/s. That comment states the distinguishing +input directly: "The distinguishing input should be **accepted-block cadence**, +not just PH/s: this testnet difficulty turns the hashrate into rapid +consecutive accepted tips, which is what exercises the tail." + +So this instrument is parameterised on **accepted tips per minute**, never on +hashrate. It releases one definitively accepted winner per tip, each at the +next height and parented on the previous winner, with a fixed number of +same-height siblings behind it, and measures the shipped +``qbit_prism_accepted_block_preview_publication_seconds`` histogram end to end +through the real coordinator: the real bounded FIFO ``candidate_queue``, the +real submitter split (``submit_next`` with ``defer_accounting=True``, the call +``BlockCandidateService.run`` makes), the real bounded accounting handoff and +its unbounded spillover, the shipped ``block_accounting_loop`` on its own +thread, and the shipped payout-preview publication boundary. + +Why the node latency is modelled rather than left at zero +--------------------------------------------------------- +The quantity under test is a wall-clock queueing latency, and the cost that +fills the accounting lane is node round-trips: the #181 spike measured a stale +sibling at "1 ``submitblock`` + 3 ``getbestblockhash`` + 3 ``getblockheader`` + +2 ledger writes". Against an in-process fake those calls cost microseconds, so +a zero-latency rig reports a p95 near zero for *every* dispatch order and can +therefore distinguish nothing. ``rpc_latency_seconds`` restores the only cost +that makes ordering observable. It is an explicit, reported parameter of the +instrument: the tips-per-minute numbers below are cadences relative to *this* +modelled drain rate, and are not claims about testnet4's block interval. + +What is deliberately not modelled: PostgreSQL wait (the 2026-08-21 run had +Postgres at ~12% CPU and the host at ~1.26 cores, so the tail was not storage +or CPU bound), qbitd itself, and the child job-build side of the wait. This +measures the producer half of the 5 s child wait budget -- +``DEFAULT_ACCEPTED_BLOCK_PAYOUT_PREVIEW_WAIT_SECONDS`` -- which is the half +#181 is about. That budget is not tuned here and must not be. + +Relationship to the shipped instruments +--------------------------------------- +``tests/prism_candidate_storm.py`` is the per-row oracle for a *single* decided +height: it seeds one same-parent storm, and its per-row drain explicitly stops +before the winner because "the decided winner, whose finalization tail is +outside this instrument". The accepted tail *is* what this file measures, and +it needs consecutive heights over a wall clock, so the cadence driver lives +here rather than being bolted onto a single-height characterization rig. It +reuses that rig's proven fixtures directly -- ``SingleWriterShareLedger`` with +durable rows appended before admission, the ``PendingShare``/intent seeding +shape, and a per-method-counting RPC. +""" + +from __future__ import annotations + +import argparse +import json +import math +import sys +import tempfile +import threading +import time +import unittest +from contextlib import nullcontext, redirect_stdout +from dataclasses import asdict, dataclass, field +from io import StringIO +from pathlib import Path +from types import SimpleNamespace +from typing import Any, ContextManager + +if not __package__: + sys.path.insert(0, str(Path(__file__).resolve().parents[1])) + +from lab.prism.block_candidates import ( # noqa: E402 + COLLAPSE_DIFFICULTY_SCALE, + COLLAPSE_POW_LIMIT_BITS, + PRISM_ACCEPTED_BLOCK_PREVIEW_PUBLICATION_RESULTS, + PRISM_ACCEPTED_BLOCK_PREVIEW_PUBLICATION_SECONDS_BUCKETS, +) +from lab.prism.payout_state import ( # noqa: E402 + DEFAULT_ACCEPTED_BLOCK_PAYOUT_PREVIEW_WAIT_SECONDS, +) +from lab.prism.share_ledger import ( # noqa: E402 + PendingShare, + SingleWriterShareLedger, +) +from lab.prism.template_artifacts import scaled_network_difficulty # noqa: E402 +from tests.prism_vardiff_test_support import ( # noqa: E402 + FakeRpc, + block_candidate, + submit_coordinator, + verified_audit_report, + verified_block_bundle, +) + + +CADENCE_PARENT_HASH = "22" * 32 +CADENCE_BASE_HEIGHT = 10 +# One node round-trip. Sized from the #181 spike's per-row measurement (a stale +# sibling costs about six of these) so that a single stale accounting task +# costs roughly a fifth of a second, the order the 2026-08-21 run's 5-12 s +# build_superseded waves imply for a handful of them. +DEFAULT_CADENCE_RPC_LATENCY_SECONDS = 0.03 +# Same-height candidates released behind each accepted winner. The 2026-08-20 +# storm reached 3,120; #196 brought the durable peak to 158. Four per tip is +# well inside what #196 leaves for the per-row lane and keeps the instrument's +# own runtime bounded. +DEFAULT_CADENCE_SIBLINGS_PER_TIP = 4 +# The acceptance bar, restated from the 2026-08-21 comment: "p95 +# acceptance-to-preview publication remains below the 5 s child wait budget at +# the tested accepted-block cadence". +PREVIEW_PUBLICATION_P95_BUDGET_SECONDS = ( + DEFAULT_ACCEPTED_BLOCK_PAYOUT_PREVIEW_WAIT_SECONDS +) + + +def bucket_percentile( + buckets: dict[float, int], + count: int, + quantile: float, +) -> float: + """Smallest histogram boundary whose cumulative count covers ``quantile``. + + Prometheus buckets are cumulative, so a percentile read from them is a + bucket boundary, not an interpolated sample. Returning the boundary is the + honest answer: it means "at least ``quantile`` of observations completed + within this many seconds". ``inf`` means only the ``+Inf`` bucket covers + it, i.e. the percentile ran off the top of the tuple. + """ + if count <= 0: + return 0.0 + target = quantile * count + for boundary in sorted(buckets): + if int(buckets[boundary]) >= target: + return float(boundary) + return math.inf + + +@dataclass(frozen=True) +class CadenceReport: + """One cadence sweep's inputs and measured publication latency.""" + + tips: int + tips_per_minute: float + inter_acceptance_seconds: float + siblings_per_tip: int + rpc_latency_seconds: float + accepted_block_count: int + published_count: int + degraded_count: int + published_p95_seconds: float + published_p50_seconds: float + published_mean_seconds: float + slowest_bucket_seconds: float + durable_pending_rows: int + node_calls: dict[str, int] = field(default_factory=dict) + + @property + def observed_count(self) -> int: + return self.published_count + self.degraded_count + + +class CadenceChainRpc(FakeRpc): + """A node that accepts registered winners and calls every sibling stale. + + Each registered winner's ``submitblock`` returns ``None`` -- the definitive + acceptance the #181 discriminator keys on -- and occupies its height, which + moves the best tip. Every other ``submitblock`` answers ``inconclusive``: a + valid block that did not become the tip, which is what a same-height + sibling of an already-accepted winner gets. Calls are counted per method, + as ``DecidedHeightRpc`` counts them, and every call pays + ``latency_seconds`` so a node round-trip has the cost that makes lane + ordering observable in wall clock. + + ``getblockheader`` reports the occupant's compact ``bits`` (and the raw + ``difficulty`` float qbitd reports beside it) because #181 item 2's + dequeue-time skip reads clause 4b from them. Without ``bits`` the clause + has no work value to compare and every evaluation fails closed, so the + instrument would silently measure a coordinator whose skip never fires + -- and would still report a changed p95, because failing closed is not + free. The default is the qbit powLimit, exactly as ``DecidedHeightRpc`` + defaults, which is at or above the raw ``network_difficulty`` of 1 this + rig stamps its candidates with. + """ + + def __init__( + self, + *, + parent: str, + height: int, + latency_seconds: float = DEFAULT_CADENCE_RPC_LATENCY_SECONDS, + bits: str = COLLAPSE_POW_LIMIT_BITS, + ) -> None: + self.tip = str(parent).lower() + self.height = int(height) + self.bits = str(bits).lower() + self.latency_seconds = max(0.0, float(latency_seconds)) + self.active: dict[int, str] = {} + self.winners: dict[str, int] = {} + self.calls: dict[str, int] = {} + self._lock = threading.Lock() + + def register_winner(self, block_hash: str, height: int) -> None: + with self._lock: + self.winners[str(block_hash).lower()] = int(height) + + def call_count(self) -> dict[str, int]: + with self._lock: + return dict(self.calls) + + def call(self, method: str, params: list[object] | None = None) -> object: + with self._lock: + self.calls[method] = self.calls.get(method, 0) + 1 + if self.latency_seconds: + time.sleep(self.latency_seconds) + if method == "getbestblockhash": + with self._lock: + return self.tip + if method == "getblockcount": + with self._lock: + return self.height + if method == "submitblock": + # The candidate carries its own hash as its block bytes, so one + # fake node can tell the registered winner from its siblings. + block_hex = str((params or [""])[0]).lower() + with self._lock: + height = self.winners.get(block_hex) + if height is None: + return "inconclusive" + self.active[height] = block_hex + self.tip = block_hex + self.height = max(self.height, height) + return None + if method == "getblockhash": + height = int((params or [0])[0]) + with self._lock: + found = self.active.get(height) + if found is None: + raise RuntimeError( + f"qbit RPC getblockhash failed: -8 unknown height {height}" + ) + return found + if method == "getblockheader": + block_hash = str((params or [""])[0]).lower() + with self._lock: + heights = { + active: height for height, active in self.active.items() + } + height = heights.get(block_hash) + top = self.height + if height is None: + raise RuntimeError( + "qbit RPC getblockheader failed: -5 Block not found" + ) + return { + "hash": block_hash, + "height": height, + "confirmations": top - height + 1, + "bits": self.bits, + # Reported beside bits, as qbitd does, and a million times + # smaller than the scaled units a candidate row carries: a + # reader that took this float instead would make an + # equal-work occupant read as weaker. + "difficulty": ( + scaled_network_difficulty(self.bits) + / COLLAPSE_DIFFICULTY_SCALE + ), + } + return super().call(method, params) + + +class AcceptedTipCadenceRig: + """Release accepted tips at a fixed cadence and measure publication. + + The lanes run in the shipped production split: the submitter thread calls + ``submit_next`` with ``defer_accounting=True`` and hands every node offer + to the accounting lane, and ``block_accounting_loop`` drains that lane on + its own thread. Nothing about either lane's ordering is simulated here -- + the queues, the spillover rule and the dequeue discipline are the shipped + ones -- so what the histogram records is the shipped dispatch order. + """ + + def __init__( + self, + *, + tips: int, + tips_per_minute: float, + siblings_per_tip: int = DEFAULT_CADENCE_SIBLINGS_PER_TIP, + rpc_latency_seconds: float = DEFAULT_CADENCE_RPC_LATENCY_SECONDS, + quiet: bool = True, + ) -> None: + if tips <= 0: + raise ValueError("tips must be positive") + if tips_per_minute <= 0: + raise ValueError("tips_per_minute must be positive") + if siblings_per_tip < 0: + raise ValueError("siblings_per_tip must not be negative") + self.tips = int(tips) + self.tips_per_minute = float(tips_per_minute) + self.siblings_per_tip = int(siblings_per_tip) + self.rpc_latency_seconds = float(rpc_latency_seconds) + self.quiet = quiet + self.ledger = SingleWriterShareLedger() + self.rpc = CadenceChainRpc( + parent=CADENCE_PARENT_HASH, + height=CADENCE_BASE_HEIGHT - 1, + latency_seconds=self.rpc_latency_seconds, + ) + self._lane_failures: list[str] = [] + + @property + def inter_acceptance_seconds(self) -> float: + return 60.0 / self.tips_per_minute + + def _silence(self) -> ContextManager[Any]: + if not self.quiet: + return nullcontext() + return redirect_stdout(StringIO()) + + # -- fixture ----------------------------------------------------------- + + def _coordinator(self) -> tuple[Any, Any]: + server, state, _recording = submit_coordinator(tip=CADENCE_PARENT_HASH) + # Every released winner must be admissible: max-block admission is not + # what this instrument is measuring. + server.max_blocks = self.tips + 1 + server.stop_after_block = False + server.accepted_block_count = 0 + server.ledger = self.ledger + server.rpc = self.rpc + # A shipped tunable, set to zero so a wakeup parked by a transient + # failure is re-offerable in the same round rather than parking behind + # a not-before deadline that would show up as publication latency it + # did not cause. + server.block_candidate_retry_initial_seconds = 0.0 + server._ensure_job_cache_state() + server.build_audit_bundle = lambda **_kwargs: verified_block_bundle() + server.verify_bundle = self._verify_bundle + return server, state + + @staticmethod + def _verify_bundle( + *_args: Any, + expected_block_height: int | None = None, + **_kwargs: Any, + ) -> dict[str, object]: + # The shipped verifier report is height-checked against the candidate; + # consecutive tips mean consecutive heights, so the stub answers the + # height it was asked about rather than a fixed one. + report = verified_audit_report() + if expected_block_height is not None: + report["block_height"] = int(expected_block_height) + return report + + def _candidate( + self, + server: Any, + state: Any, + *, + block_hash: str, + height: int, + parent: str, + ordinal: int, + ) -> tuple[Any, tuple[Any, Any]]: + job_id = f"job-{height}-{ordinal}" + context = SimpleNamespace( + job=server.jobs["job-1"].job, + template={ + "previousblockhash": parent, + "height": height, + "coinbasevalue": 50_00000000, + }, + found_block={"network_difficulty": 1}, + issued_at_ms=12345, + collection_only=False, + worker=server.jobs["job-1"].worker, + shares_json=[], + prior_balances=[], + # The compact issued-job preview. Its presence is what makes the + # accepted tail publish a payout preview at all + # (block_finalization.py's issued_preview branch), which is the + # end of the interval under measurement. + prospective_prior_balances=(), + ) + server.jobs[job_id] = context + pending = PendingShare( + share_id=f"miner-a:{block_hash}", + miner_id="miner-a", + order_key="miner-a", + p2mr_program_hex="11" * 32, + share_difficulty=1, + network_difficulty=1, + template_height=height, + job_id=job_id, + job_issued_at_ms=1, + accepted_at_ms=ordinal, + ntime=1, + ) + submission = SimpleNamespace( + coinbase_tx_hex="c0ffee", + block_hash_hex=block_hash, + block_hex=block_hash, + share_pass=True, + block_pass=True, + ) + candidate = block_candidate( + server, + state, + submission, + job_id=job_id, + pending_share=pending, + ) + return candidate, (pending, server.block_candidate_intent(candidate)) + + @staticmethod + def _hash_for(tip_index: int, ordinal: int) -> str: + return f"{((tip_index + 1) << 8) | ordinal:064x}" + + # -- lanes ------------------------------------------------------------- + + def _submitter_loop(self, server: Any) -> None: + service = server._ensure_block_candidate_service() + # block_submit_loop stamps this on entry; the progress boundary stamps + # are gated to it. + service._block_submitter_thread_ident = threading.get_ident() + while not server.stop_event.is_set(): + try: + server.submit_next_block_candidate( + timeout=0.05, + defer_accounting=True, + ) + except Exception as exc: # noqa: BLE001 + # Reported, never swallowed: a submitter that dies silently + # would look like a cadence the instrument simply never + # reached. + self._lane_failures.append(f"submitter: {exc!r}") + return + + # -- driver ------------------------------------------------------------ + + def _build_tips( + self, + server: Any, + state: Any, + ) -> list[tuple[int, list[Any], list[Any]]]: + """Build every tip's candidates before either lane starts. + + Job contexts are registered in ``server.jobs``, which shipped paths + iterate. Building them all up front keeps the driver from mutating + that dictionary underneath a running lane; only the durable append and + the queue admission happen on the cadence, which is what the cadence + is supposed to control. + """ + built: list[tuple[int, list[Any], list[Any]]] = [] + for tip_index in range(self.tips): + height = CADENCE_BASE_HEIGHT + tip_index + parent = ( + CADENCE_PARENT_HASH + if tip_index == 0 + else self._hash_for(tip_index - 1, 0) + ) + candidates = [] + durable_records = [] + for ordinal in range(self.siblings_per_tip + 1): + candidate, durable_record = self._candidate( + server, + state, + block_hash=self._hash_for(tip_index, ordinal), + height=height, + parent=parent, + ordinal=ordinal, + ) + candidates.append(candidate) + durable_records.append(durable_record) + built.append((height, candidates, durable_records)) + return built + + def _release_tip( + self, + server: Any, + tip_index: int, + built: list[tuple[int, list[Any], list[Any]]], + ) -> None: + height, candidates, durable_records = built[tip_index] + # Registered before admission: the winner has to be acceptable the + # instant the submitter offers it. + self.rpc.register_winner(self._hash_for(tip_index, 0), height) + # Durable first, exactly as live admission does: the outbox row is + # what makes a coalesced wakeup replayable. Appending per tip rather + # than up front is also what keeps a replay enumeration from pulling + # a later tip's candidates forward and dissolving the cadence. + self.ledger.append_batch(durable_records) + for candidate in candidates: + server.enqueue_block_candidate(candidate) + + def _drive( + self, + server: Any, + built: list[tuple[int, list[Any], list[Any]]], + ) -> None: + interval = self.inter_acceptance_seconds + started = time.monotonic() + for tip_index in range(self.tips): + delay = (started + tip_index * interval) - time.monotonic() + if delay > 0: + time.sleep(delay) + self._release_tip(server, tip_index, built) + + def _await_publication(self, server: Any, *, timeout: float) -> None: + """Wait until every released winner has published, or time out. + + The deadline is generous on purpose: a cadence that overruns it is a + result, not a flake, and the report carries the shortfall so the test + can say what did not publish. + """ + service = server._ensure_block_candidate_service() + deadline = time.monotonic() + timeout + while time.monotonic() < deadline: + snapshot = service.accepted_block_preview_publication_snapshot() + observed = sum( + int(histogram["count"]) for histogram in snapshot.values() + ) + if observed >= self.tips: + return + time.sleep(0.05) + + def run(self) -> CadenceReport: + server, state = self._coordinator() + service = server._ensure_block_candidate_service() + service._ensure_block_accounting_state() + accounting = threading.Thread( + target=server.block_accounting_loop, + name="cadence-block-accounting", + daemon=True, + ) + submitter = threading.Thread( + target=self._submitter_loop, + args=(server,), + name="cadence-block-submitter", + daemon=True, + ) + with tempfile.TemporaryDirectory() as tempdir: + server.audit_dir = Path(tempdir) + server.evidence_path = Path(tempdir) / "evidence.json" + server.ledger_writer_public_key_hex = "aa" * 32 + with self._silence(): + built = self._build_tips(server, state) + accounting.start() + submitter.start() + try: + self._drive(server, built) + # The drain budget scales with the work released, not with + # the cadence: a saturated lane still has to be allowed to + # finish so its latencies land in the histogram instead of + # being truncated into a flattering p95. + self._await_publication( + server, + timeout=30.0 + + self.tips + * (self.siblings_per_tip + 1) + * 12.0 + * max(self.rpc_latency_seconds, 0.001), + ) + finally: + server.stop_event.set() + submitter.join(timeout=60.0) + accounting.join(timeout=60.0) + snapshot = service.accepted_block_preview_publication_snapshot() + pending = self.ledger.block_candidate_pending_metrics() + published = snapshot["published"] + degraded = snapshot["degraded"] + published_count = int(published["count"]) + return CadenceReport( + tips=self.tips, + tips_per_minute=self.tips_per_minute, + inter_acceptance_seconds=self.inter_acceptance_seconds, + siblings_per_tip=self.siblings_per_tip, + rpc_latency_seconds=self.rpc_latency_seconds, + accepted_block_count=int(server.accepted_block_count), + published_count=published_count, + degraded_count=int(degraded["count"]), + published_p95_seconds=bucket_percentile( + published["buckets"], published_count, 0.95 + ), + published_p50_seconds=bucket_percentile( + published["buckets"], published_count, 0.50 + ), + published_mean_seconds=( + float(published["sum"]) / published_count + if published_count + else 0.0 + ), + slowest_bucket_seconds=bucket_percentile( + published["buckets"], published_count, 1.0 + ), + durable_pending_rows=int(pending.get("pending_count", -1)), + node_calls=self.rpc.call_count(), + ) + + @property + def lane_failures(self) -> tuple[str, ...]: + return tuple(self._lane_failures) + + +def accepted_lane_is_present(server: Any) -> bool: + """Whether item 1's dedicated definitive-acceptance lane has landed. + + On the #181 base (``ce4a1f3``) every accounting handoff shares one bounded + primary queue and its unbounded spillover, and a definitively accepted task + is dispatched strictly behind whatever stale work is already queued. The + high-cadence assertion below is written against whichever of those two + worlds it is running in, so that this file is a truthful statement in both + and becomes the bar the moment the lane exists. + """ + service = server._ensure_block_candidate_service() + service._ensure_block_accounting_state() + return hasattr(service, "_block_accounting_accepted_queue") + + +class AcceptedPreviewPublicationHistogramTests(unittest.TestCase): + """The metric itself: labels, boundaries, and what closes an interval.""" + + def _service(self) -> Any: + server, _state, _ledger = submit_coordinator() + return server, server._ensure_block_candidate_service() + + def test_five_second_child_wait_budget_is_a_bucket_boundary(self) -> None: + # The acceptance criterion is a p95 below the 5 s child wait budget, + # and a Prometheus histogram can only answer that at a boundary it + # carries. The range must also extend past it, or a saturated tail + # folds into +Inf and stops being measurable. + self.assertIn( + PREVIEW_PUBLICATION_P95_BUDGET_SECONDS, + PRISM_ACCEPTED_BLOCK_PREVIEW_PUBLICATION_SECONDS_BUCKETS, + ) + self.assertGreater( + max(PRISM_ACCEPTED_BLOCK_PREVIEW_PUBLICATION_SECONDS_BUCKETS), + PREVIEW_PUBLICATION_P95_BUDGET_SECONDS, + ) + self.assertEqual( + tuple(sorted(PRISM_ACCEPTED_BLOCK_PREVIEW_PUBLICATION_SECONDS_BUCKETS)), + PRISM_ACCEPTED_BLOCK_PREVIEW_PUBLICATION_SECONDS_BUCKETS, + ) + self.assertEqual( + PRISM_ACCEPTED_BLOCK_PREVIEW_PUBLICATION_RESULTS, + ("published", "degraded"), + ) + + def test_publication_without_an_acceptance_stamp_observes_nothing( + self, + ) -> None: + # A replayed candidate confirmed by chain probe never made a + # definitive offer in this process. Reporting a zero for it would + # understate the very percentile the bar is read from. + server, service = self._service() + server._observe_accepted_block_preview_publication( + "ab" * 32, + result="published", + ) + snapshot = service.accepted_block_preview_publication_snapshot() + self.assertEqual(snapshot["published"]["count"], 0) + self.assertEqual(snapshot["degraded"]["count"], 0) + + def test_only_the_first_publication_of_a_hash_is_measured(self) -> None: + server, service = self._service() + service._note_accepted_block_preview_acceptance("AB" * 32) + server._observe_accepted_block_preview_publication( + "ab" * 32, + result="published", + ) + # A matching republication -- the early return with an existing + # published generation, or a deferred retry's republish -- is not a + # second interval. + server._observe_accepted_block_preview_publication( + "ab" * 32, + result="published", + ) + server._observe_accepted_block_preview_publication( + "ab" * 32, + result="degraded", + ) + # Nor does a second definitive offer of the same hash restart it. + service._note_accepted_block_preview_acceptance("ab" * 32) + server._observe_accepted_block_preview_publication( + "ab" * 32, + result="published", + ) + snapshot = service.accepted_block_preview_publication_snapshot() + self.assertEqual(snapshot["published"]["count"], 1) + self.assertEqual(snapshot["degraded"]["count"], 0) + + def test_degraded_publication_is_labelled_apart(self) -> None: + server, service = self._service() + service._note_accepted_block_preview_acceptance("cd" * 32) + server._observe_accepted_block_preview_publication( + "cd" * 32, + result="degraded", + ) + snapshot = service.accepted_block_preview_publication_snapshot() + self.assertEqual(snapshot["degraded"]["count"], 1) + self.assertEqual(snapshot["published"]["count"], 0) + + def test_acceptance_stamps_are_bounded(self) -> None: + from lab.prism.block_candidates import ( + MAX_ACCEPTED_BLOCK_PREVIEW_PUBLICATION_STAMPS, + ) + + _server, service = self._service() + for index in range(MAX_ACCEPTED_BLOCK_PREVIEW_PUBLICATION_STAMPS + 64): + service._note_accepted_block_preview_acceptance(f"{index:064x}") + self.assertLessEqual( + len(service._accepted_block_preview_acceptance_monotonic), + MAX_ACCEPTED_BLOCK_PREVIEW_PUBLICATION_STAMPS, + ) + + def test_failed_offer_is_not_a_definitive_acceptance(self) -> None: + # The submitblock error path also builds ``result=None``; a two-clause + # test would stamp a failed offer as an acceptance and then measure a + # publication that a later, successful offer produced. + from lab.prism.block_candidates import _BlockCandidateNodeSubmission + + failed = _BlockCandidateNodeSubmission( + attempted=True, + error=RuntimeError("submitblock failed"), + ) + self.assertIsNone(failed.result) + self.assertFalse( + failed.attempted + and failed.error is None + and failed.result is None + ) + + +class AcceptedTipCadenceRegressionTests(unittest.TestCase): + """The bar: p95 publication latency at a low and a high accepted cadence.""" + + # A cadence comfortably slower than the modelled per-tip accounting cost, + # so the lane drains between tips. This is the "normal forward progress" + # case from the 2026-08-21 acceptance criteria. + LOW_CADENCE_TIPS_PER_MINUTE = 40.0 + LOW_CADENCE_TIPS = 8 + # A cadence faster than the lane can drain, which is what the run's tip + # bursts produced. On the #181 base this is the inversion itself: the + # accepted task for height H+n sorts behind every stale sibling of heights + # H..H+n-1 already in the accounting queue. + HIGH_CADENCE_TIPS_PER_MINUTE = 150.0 + HIGH_CADENCE_TIPS = 20 + + def _report(self, **kwargs: Any) -> CadenceReport: + rig = AcceptedTipCadenceRig(**kwargs) + report = rig.run() + self.assertEqual( + rig.lane_failures, + (), + f"a coordinator lane died during the sweep: {rig.lane_failures}", + ) + self.assertEqual( + report.published_count + report.degraded_count, + report.tips, + "every released winner must close an acceptance-to-publication " + f"interval; report={asdict(report)}", + ) + return report + + def test_low_cadence_publishes_well_inside_the_child_wait_budget( + self, + ) -> None: + report = self._report( + tips=self.LOW_CADENCE_TIPS, + tips_per_minute=self.LOW_CADENCE_TIPS_PER_MINUTE, + ) + self.assertEqual(report.degraded_count, 0, asdict(report)) + self.assertLessEqual( + report.published_p95_seconds, + PREVIEW_PUBLICATION_P95_BUDGET_SECONDS, + "p95 acceptance-to-preview publication must stay below the 5 s " + f"child wait budget at a low accepted cadence; report={asdict(report)}", + ) + # "Comfortably": at a cadence the lane keeps up with, the accepted tail + # should not be spending a meaningful fraction of the budget at all. + self.assertLessEqual( + report.published_p95_seconds, + 2.0, + f"low-cadence p95 is not comfortable; report={asdict(report)}", + ) + + def test_high_cadence_exercises_the_accepted_parent_tail(self) -> None: + server, _state, _ledger = submit_coordinator() + lane_present = accepted_lane_is_present(server) + report = self._report( + tips=self.HIGH_CADENCE_TIPS, + tips_per_minute=self.HIGH_CADENCE_TIPS_PER_MINUTE, + ) + if lane_present: + # Item 1 has landed: this is the bar #181 closes against. + self.assertLessEqual( + report.published_p95_seconds, + PREVIEW_PUBLICATION_P95_BUDGET_SECONDS, + "p95 acceptance-to-preview publication must stay below the " + "5 s child wait budget at a high accepted-block cadence; " + f"report={asdict(report)}", + ) + return + # On the #181 base there is no accepted lane, and this assertion is + # the honest statement of what the instrument sees: the tail really is + # exercised. If this ever fails on the base, the instrument has + # stopped reproducing the failure mode and the bar above would be one + # items 1 and 2 could pass without fixing anything -- which is exactly + # the outcome the contract says to report rather than paper over. + self.assertGreater( + report.published_p95_seconds, + PREVIEW_PUBLICATION_P95_BUDGET_SECONDS, + "without a dedicated definitive-acceptance lane the high-cadence " + "sweep must drive p95 publication latency past the 5 s child " + "wait budget; if it no longer does, this instrument has stopped " + f"exercising the accepted-parent tail. report={asdict(report)}", + ) + + +def main() -> int: + parser = argparse.ArgumentParser( + description=( + "Report acceptance-to-preview-publication latency at one or more " + "accepted-block cadences." + ) + ) + parser.add_argument( + "--tips-per-minute", + type=float, + action="append", + default=None, + help="accepted tips per minute; repeatable (default: 40 and 150)", + ) + parser.add_argument("--tips", type=int, default=None) + parser.add_argument( + "--siblings-per-tip", + type=int, + default=DEFAULT_CADENCE_SIBLINGS_PER_TIP, + ) + parser.add_argument( + "--rpc-latency-seconds", + type=float, + default=DEFAULT_CADENCE_RPC_LATENCY_SECONDS, + ) + parser.add_argument("--verbose", action="store_true") + args = parser.parse_args() + cadences = args.tips_per_minute or [ + AcceptedTipCadenceRegressionTests.LOW_CADENCE_TIPS_PER_MINUTE, + AcceptedTipCadenceRegressionTests.HIGH_CADENCE_TIPS_PER_MINUTE, + ] + reports = [] + for tips_per_minute in cadences: + if args.tips is not None: + tips = args.tips + elif tips_per_minute >= 100: + tips = AcceptedTipCadenceRegressionTests.HIGH_CADENCE_TIPS + else: + tips = AcceptedTipCadenceRegressionTests.LOW_CADENCE_TIPS + rig = AcceptedTipCadenceRig( + tips=tips, + tips_per_minute=tips_per_minute, + siblings_per_tip=args.siblings_per_tip, + rpc_latency_seconds=args.rpc_latency_seconds, + quiet=not args.verbose, + ) + report = rig.run() + reports.append(asdict(report)) + if rig.lane_failures: + reports[-1]["lane_failures"] = list(rig.lane_failures) + print(json.dumps(reports, indent=2, sort_keys=True)) + return 0 + + +if __name__ == "__main__": + if len(sys.argv) > 1 and sys.argv[1] == "report": + del sys.argv[1] + raise SystemExit(main()) + unittest.main() diff --git a/tests/test_prism_accepted_preview_telemetry.py b/tests/test_prism_accepted_preview_telemetry.py new file mode 100644 index 000000000..0ff3c37e1 --- /dev/null +++ b/tests/test_prism_accepted_preview_telemetry.py @@ -0,0 +1,654 @@ +#!/usr/bin/env python3 +"""Issue #224 Wave 0: the shared accepted-preview telemetry contract. + +These tests pin the contract the three Wave 1 lanes build on: every +vocabulary is closed and label-safe, the rescan reasons mirror the +payout_state literals, the recorder refuses labels outside the contract, +normalizes the two runtime strings onto ``other``, keeps its cells +thread-safe, and the renderer exports every cell from zero without ever +carrying a block hash or height into a label. +""" + +from __future__ import annotations + +import ast +import json +import math +from pathlib import Path +import re +import threading +from types import SimpleNamespace +import unittest + +from lab.prism import accepted_preview_telemetry as contract +from lab.prism.accepted_preview_telemetry import ( + AcceptedPreviewPublicationDiagnostic, + AcceptedPreviewTelemetry, + PRISM_ACCEPTED_LANDING_PHASES, + PRISM_ACCEPTED_PREVIEW_DIAGNOSTICS_CAPACITY, + PRISM_LEDGER_READ_OPERATIONS, + PRISM_PAYOUT_WINDOW_FULL_RESCAN_PATHS, + PRISM_PAYOUT_WINDOW_FULL_RESCAN_REASONS, + PRISM_REORG_RECONCILE_CALLERS, + PRISM_REORG_RECONCILE_STEPS, + empty_duration_stats, + ensure_accepted_preview_telemetry, + fold_ledger_read_stats, + normalize_ledger_read_operation, + normalize_payout_window_full_rescan_reason, +) +from lab.prism.block_candidates import ( + PRISM_ACCEPTED_BLOCK_PREVIEW_PUBLICATION_RESULTS, +) +from lab.prism.metrics import MetricsRenderer +from lab.prism.reorg_reconciler import PRISM_REORG_RECONCILE_LOOKUP_PATHS + +REPO_ROOT = Path(__file__).resolve().parents[1] +LABEL_VALUE = re.compile(r"^[a-z][a-z0-9_]*$") +VOCABULARIES = { + "PRISM_ACCEPTED_LANDING_PHASES": PRISM_ACCEPTED_LANDING_PHASES, + "PRISM_REORG_RECONCILE_CALLERS": PRISM_REORG_RECONCILE_CALLERS, + "PRISM_REORG_RECONCILE_STEPS": PRISM_REORG_RECONCILE_STEPS, + "PRISM_PAYOUT_WINDOW_FULL_RESCAN_REASONS": PRISM_PAYOUT_WINDOW_FULL_RESCAN_REASONS, + "PRISM_PAYOUT_WINDOW_FULL_RESCAN_PATHS": PRISM_PAYOUT_WINDOW_FULL_RESCAN_PATHS, + "PRISM_LEDGER_READ_OPERATIONS": PRISM_LEDGER_READ_OPERATIONS, +} + + +def _string_constants(node: ast.AST) -> set[str]: + return { + child.value + for child in ast.walk(node) + if isinstance(child, ast.Constant) and isinstance(child.value, str) + } + + +def payout_state_full_rescan_reason_literals() -> set[str]: + """Every reason literal payout_state can hand to a full rescan. + + The source of truth keeps them as scattered literals rather than a + tuple (and that module is not this wave's to edit), so the contract + tuple is checked against the module text: assignments to + ``full_reason`` / ``check_reason`` / the cached-window invalidation + reason, the first argument of ``_DaemonWindowRebuildRequired``, and + the ``reason=`` keyword of ``_full_payout_window_materialization``. + """ + source = (REPO_ROOT / "lab" / "prism" / "payout_state.py").read_text() + found: set[str] = set() + for node in ast.walk(ast.parse(source)): + if isinstance(node, ast.Assign): + for target in node.targets: + if isinstance(target, ast.Name) and target.id in ( + "full_reason", + "check_reason", + ): + found |= _string_constants(node.value) + elif ( + isinstance(target, ast.Attribute) + and target.attr + == "_incremental_payout_artifact_window_invalidation_reason" + ): + found |= _string_constants(node.value) + elif isinstance(node, ast.Call): + func = node.func + if isinstance(func, ast.Name): + name = func.id + elif isinstance(func, ast.Attribute): + name = func.attr + else: + continue + if name == "_DaemonWindowRebuildRequired" and node.args: + found |= _string_constants(node.args[0]) + elif name == "_full_payout_window_materialization": + for keyword in node.keywords: + if keyword.arg == "reason": + found |= _string_constants(keyword.value) + return found + + +def _clock() -> tuple[list[float], "callable"]: + ticks: list[float] = [100.0] + + def monotonic() -> float: + return ticks[0] + + return ticks, monotonic + + +class VocabularyContractTests(unittest.TestCase): + def test_every_vocabulary_is_closed_unique_and_label_safe(self) -> None: + for name, vocabulary in VOCABULARIES.items(): + with self.subTest(vocabulary=name): + self.assertIsInstance(vocabulary, tuple) + self.assertGreater(len(vocabulary), 0) + self.assertEqual(len(vocabulary), len(set(vocabulary))) + for member in vocabulary: + self.assertRegex(member, LABEL_VALUE) + + def test_runtime_string_vocabularies_carry_the_other_bucket(self) -> None: + self.assertEqual(PRISM_PAYOUT_WINDOW_FULL_RESCAN_REASONS[-1], "other") + self.assertEqual(PRISM_LEDGER_READ_OPERATIONS[-1], "other") + self.assertIn("other", PRISM_REORG_RECONCILE_CALLERS) + # Programmer-chosen vocabularies have no fold-in bucket: a typo is + # a ValueError at the call site, not a silently merged series. + self.assertNotIn("other", PRISM_ACCEPTED_LANDING_PHASES) + self.assertNotIn("other", PRISM_REORG_RECONCILE_STEPS) + self.assertNotIn("other", PRISM_PAYOUT_WINDOW_FULL_RESCAN_PATHS) + + def test_landing_phases_follow_the_landing_order(self) -> None: + self.assertEqual( + PRISM_ACCEPTED_LANDING_PHASES, + ( + "lane_wait", + "balance_lock_wait", + "reconcile", + "prior_balances_check", + "chain_probe", + "preview_prepare", + "preview_publish", + ), + ) + + def test_reconcile_callers_join_the_existing_lookup_paths(self) -> None: + # The demand-accounting family already labels tip_refresh and + # job_build; the caller vocabulary spells them identically so the + # two families join on the label value. + for path in PRISM_REORG_RECONCILE_LOOKUP_PATHS: + self.assertIn(path, PRISM_REORG_RECONCILE_CALLERS) + self.assertEqual( + PRISM_REORG_RECONCILE_STEPS, + ( + "admission_wait", + "watch_query", + "chain_probe", + "mutations", + "candidate_prepare", + "publish", + ), + ) + + def test_full_rescan_reasons_mirror_payout_state_literals(self) -> None: + literals = payout_state_full_rescan_reason_literals() + # The collector must find the reasons the issue was diagnosed on; + # an empty or partial harvest would make the drift check vacuous. + for expected in ( + "reconcile_invalidation", + "cold_start", + "window_daemon_state_lost", + "periodic_self_check", + ): + self.assertIn(expected, literals) + self.assertEqual( + sorted(literals - set(PRISM_PAYOUT_WINDOW_FULL_RESCAN_REASONS)), + [], + "payout_state grew a full-rescan reason the contract does not carry", + ) + self.assertEqual( + sorted(set(PRISM_PAYOUT_WINDOW_FULL_RESCAN_REASONS) - literals), + ["other"], + "the contract carries a reason payout_state no longer produces", + ) + + def test_ledger_read_operations_cover_every_attributed_statement(self) -> None: + source = (REPO_ROOT / "lab" / "prism" / "share_ledger.py").read_text() + literals = set(re.findall(r'operation="([^"]+)"', source)) + self.assertIn("pending_block_candidate_rows", literals) + self.assertEqual( + sorted(literals - set(PRISM_LEDGER_READ_OPERATIONS)), + [], + "share_ledger attributes a read under a name outside the contract", + ) + + def test_normalizers_bound_runtime_strings(self) -> None: + self.assertEqual( + normalize_payout_window_full_rescan_reason("reconcile_invalidation"), + "reconcile_invalidation", + ) + for value in ("hash=" + "ab" * 32, "", None, 7, "Other"): + self.assertEqual(normalize_payout_window_full_rescan_reason(value), "other") + self.assertEqual( + normalize_ledger_read_operation("payout_window_snapshot"), + "payout_window_snapshot", + ) + self.assertEqual(normalize_ledger_read_operation("snapshot_at_job_issue"), "other") + + +class TelemetryRecorderTests(unittest.TestCase): + def test_empty_owner_snapshot_covers_every_closed_product_at_zero(self) -> None: + snapshot = AcceptedPreviewTelemetry().snapshot() + self.assertEqual( + set(snapshot["landing_phases"]), set(PRISM_ACCEPTED_LANDING_PHASES) + ) + self.assertEqual( + set(snapshot["reconcile_passes"]), set(PRISM_REORG_RECONCILE_CALLERS) + ) + self.assertEqual( + set(snapshot["reconcile_steps"]), + { + (caller, step) + for caller in PRISM_REORG_RECONCILE_CALLERS + for step in PRISM_REORG_RECONCILE_STEPS + }, + ) + self.assertEqual( + set(snapshot["full_rescans"]), + { + (reason, path) + for reason in PRISM_PAYOUT_WINDOW_FULL_RESCAN_REASONS + for path in PRISM_PAYOUT_WINDOW_FULL_RESCAN_PATHS + }, + ) + for family in ("landing_phases", "reconcile_passes", "reconcile_steps", "full_rescans"): + for stats in snapshot[family].values(): + self.assertEqual(stats, empty_duration_stats()) + self.assertEqual(snapshot["diagnostics_retained"], 0) + self.assertEqual(snapshot["diagnostics_recorded"], 0) + + def test_programmer_labels_outside_the_contract_raise(self) -> None: + telemetry = AcceptedPreviewTelemetry() + with self.assertRaises(ValueError): + telemetry.observe_landing_phase("reorg-reconcile", 0.1) + with self.assertRaises(ValueError): + telemetry.observe_reconcile_pass("replay", 0.1) + with self.assertRaises(ValueError): + telemetry.observe_reconcile_step("landing", "getblockhash", 0.1) + with self.assertRaises(ValueError): + telemetry.observe_reconcile_step("replay", "watch_query", 0.1) + with self.assertRaises(ValueError): + telemetry.observe_payout_window_full_rescan( + "reconcile_invalidation", "rust", 0.1 + ) + with self.assertRaises(ValueError): + with telemetry.landing_phase("submitblock-rpc"): + pass + with self.assertRaises(ValueError): + with telemetry.reconcile_step("landing", "sweep"): + pass + # Nothing was recorded by the refused calls. + snapshot = telemetry.snapshot() + for family in ("landing_phases", "reconcile_passes", "reconcile_steps", "full_rescans"): + for stats in snapshot[family].values(): + self.assertEqual(int(stats["count"]), 0) + + def test_runtime_rescan_reason_folds_into_other(self) -> None: + telemetry = AcceptedPreviewTelemetry() + telemetry.observe_payout_window_full_rescan("reconcile_invalidation", "daemon", 3.0) + telemetry.observe_payout_window_full_rescan("hash=" + "ab" * 32, "daemon", 1.0) + telemetry.observe_payout_window_full_rescan(None, "in_process", 2.0) + rescans = telemetry.snapshot()["full_rescans"] + self.assertEqual(rescans[("reconcile_invalidation", "daemon")]["count"], 1) + self.assertEqual(rescans[("other", "daemon")]["count"], 1) + self.assertEqual(rescans[("other", "in_process")]["count"], 1) + self.assertEqual( + {key for key, stats in rescans.items() if stats["count"]}, + {("reconcile_invalidation", "daemon"), ("other", "daemon"), ("other", "in_process")}, + ) + + def test_accumulates_sum_count_max_and_clamps_bad_durations(self) -> None: + telemetry = AcceptedPreviewTelemetry() + for seconds in (0.25, 1.5, 0.75): + telemetry.observe_landing_phase("reconcile", seconds) + for bad in (-1.0, math.nan, math.inf): + telemetry.observe_landing_phase("reconcile", bad) + stats = telemetry.snapshot()["landing_phases"]["reconcile"] + self.assertEqual(stats["count"], 6) + self.assertAlmostEqual(stats["sum"], 2.5) + self.assertEqual(stats["max"], 1.5) + self.assertIsInstance(stats["count"], int) + + def test_snapshot_is_a_copy(self) -> None: + telemetry = AcceptedPreviewTelemetry() + first = telemetry.snapshot() + first["landing_phases"]["reconcile"]["count"] = 99 + first["reconcile_steps"][("landing", "publish")]["sum"] = 99.0 + second = telemetry.snapshot() + self.assertEqual(second["landing_phases"]["reconcile"]["count"], 0) + self.assertEqual(second["reconcile_steps"][("landing", "publish")]["sum"], 0.0) + + def test_context_managers_record_elapsed_even_when_raising(self) -> None: + ticks, monotonic = _clock() + telemetry = AcceptedPreviewTelemetry(monotonic=monotonic) + with telemetry.landing_phase("balance_lock_wait"): + ticks[0] += 0.5 + with self.assertRaises(RuntimeError): + with telemetry.reconcile_step("landing", "mutations"): + ticks[0] += 0.25 + raise RuntimeError("fenced mutation failed") + snapshot = telemetry.snapshot() + self.assertEqual( + snapshot["landing_phases"]["balance_lock_wait"], + {"count": 1, "sum": 0.5, "max": 0.5}, + ) + self.assertEqual( + snapshot["reconcile_steps"][("landing", "mutations")], + {"count": 1, "sum": 0.25, "max": 0.25}, + ) + + def test_concurrent_observers_never_lose_a_sample(self) -> None: + telemetry = AcceptedPreviewTelemetry() + per_thread = 500 + threads = 8 + + def worker(index: int) -> None: + for _ in range(per_thread): + telemetry.observe_landing_phase("preview_publish", 0.001) + telemetry.observe_reconcile_step("job_build", "chain_probe", 0.002) + telemetry.observe_payout_window_full_rescan( + "cold_start" if index % 2 else "bogus", "in_process", 0.003 + ) + telemetry.record_publication_diagnostic( + AcceptedPreviewPublicationDiagnostic( + block_hash=f"{index:02x}" * 32, + block_height=index, + result="published", + acceptance_to_publication_seconds=0.5, + ) + ) + + pool = [threading.Thread(target=worker, args=(index,)) for index in range(threads)] + for thread in pool: + thread.start() + for thread in pool: + thread.join() + snapshot = telemetry.snapshot() + total = per_thread * threads + self.assertEqual(snapshot["landing_phases"]["preview_publish"]["count"], total) + self.assertEqual( + snapshot["reconcile_steps"][("job_build", "chain_probe")]["count"], total + ) + rescans = snapshot["full_rescans"] + self.assertEqual( + rescans[("cold_start", "in_process")]["count"] + + rescans[("other", "in_process")]["count"], + total, + ) + self.assertEqual(snapshot["diagnostics_recorded"], total) + self.assertEqual( + snapshot["diagnostics_retained"], PRISM_ACCEPTED_PREVIEW_DIAGNOSTICS_CAPACITY + ) + + def test_diagnostics_ring_is_bounded_oldest_first_and_stamped(self) -> None: + ticks, monotonic = _clock() + telemetry = AcceptedPreviewTelemetry(diagnostics_capacity=3, monotonic=monotonic) + for height in range(5): + ticks[0] += 1.0 + telemetry.record_publication_diagnostic( + AcceptedPreviewPublicationDiagnostic( + block_hash=f"{height:02x}" * 32, + block_height=height, + result="published", + acceptance_to_publication_seconds=float(height), + ) + ) + retained = telemetry.diagnostics_snapshot() + self.assertEqual([record.block_height for record in retained], [2, 3, 4]) + self.assertEqual([record.recorded_monotonic for record in retained], [103.0, 104.0, 105.0]) + snapshot = telemetry.snapshot() + self.assertEqual(snapshot["diagnostics_retained"], 3) + self.assertEqual(snapshot["diagnostics_recorded"], 5) + # A caller's own stamp is kept, and the retained record is what the + # recorder handed back. + stamped = telemetry.record_publication_diagnostic( + AcceptedPreviewPublicationDiagnostic( + block_hash="ff" * 32, + block_height=9, + result="degraded", + acceptance_to_publication_seconds=6.0, + recorded_monotonic=42.0, + ) + ) + self.assertEqual(stamped.recorded_monotonic, 42.0) + self.assertIs(telemetry.diagnostics_snapshot()[-1], stamped) + with self.assertRaises(TypeError): + telemetry.record_publication_diagnostic({"block_hash": "ab" * 32}) # type: ignore[arg-type] + with self.assertRaises(ValueError): + AcceptedPreviewTelemetry(diagnostics_capacity=0) + + def test_ensure_owner_attaches_once_and_is_shared(self) -> None: + owner = SimpleNamespace() + first = ensure_accepted_preview_telemetry(owner) + self.assertIs(ensure_accepted_preview_telemetry(owner), first) + self.assertIs(owner.__dict__["_accepted_preview_telemetry"], first) + # Recording through one handle is visible through the other: the + # renderer and every Wave 1 lane share one owner per coordinator. + first.observe_landing_phase("lane_wait", 0.1) + self.assertEqual( + ensure_accepted_preview_telemetry(owner).snapshot()["landing_phases"]["lane_wait"]["count"], + 1, + ) + self.assertIsNot(ensure_accepted_preview_telemetry(SimpleNamespace()), first) + with self.assertRaises(TypeError): + ensure_accepted_preview_telemetry(object()) + + def test_ensure_owner_is_race_free(self) -> None: + owner = SimpleNamespace() + seen: list[AcceptedPreviewTelemetry] = [] + seen_lock = threading.Lock() + barrier = threading.Barrier(16) + + def worker() -> None: + barrier.wait() + telemetry = ensure_accepted_preview_telemetry(owner) + with seen_lock: + seen.append(telemetry) + + pool = [threading.Thread(target=worker) for _ in range(16)] + for thread in pool: + thread.start() + for thread in pool: + thread.join() + self.assertEqual(len(seen), 16) + self.assertEqual(len({id(telemetry) for telemetry in seen}), 1) + + +class DiagnosticRecordTests(unittest.TestCase): + def test_normalizes_hash_and_bounds_every_field(self) -> None: + record = AcceptedPreviewPublicationDiagnostic( + block_hash=" " + "AB" * 32 + " ", + block_height="12", # type: ignore[arg-type] + result="published", + acceptance_to_publication_seconds=-3.0, + phase_seconds={"reconcile": 1.25, "preview_publish": math.nan}, + reconcile_caller="landing", + full_rescan_reason="not-a-contract-reason", + full_rescan_path="daemon", + full_rescan_seconds=math.inf, + ledger_gate_wait_seconds=0.5, + ledger_execute_seconds=-0.5, + ) + self.assertEqual(record.block_hash, "ab" * 32) + self.assertEqual(record.block_height, 12) + self.assertEqual(record.acceptance_to_publication_seconds, 0.0) + self.assertEqual(tuple(record.phase_seconds), PRISM_ACCEPTED_LANDING_PHASES) + self.assertEqual(record.phase_seconds["reconcile"], 1.25) + self.assertEqual(record.phase_seconds["preview_publish"], 0.0) + self.assertEqual(record.phase_seconds["lane_wait"], 0.0) + self.assertEqual(record.full_rescan_reason, "other") + self.assertEqual(record.full_rescan_seconds, 0.0) + self.assertEqual(record.ledger_execute_seconds, 0.0) + with self.assertRaises(TypeError): + record.phase_seconds["reconcile"] = 9.0 # type: ignore[index] + + def test_refuses_values_outside_the_vocabularies(self) -> None: + base = dict( + block_hash="ab" * 32, + block_height=1, + result="published", + acceptance_to_publication_seconds=1.0, + ) + for result in PRISM_ACCEPTED_BLOCK_PREVIEW_PUBLICATION_RESULTS: + AcceptedPreviewPublicationDiagnostic(**dict(base, result=result)) + with self.assertRaises(ValueError): + AcceptedPreviewPublicationDiagnostic(**dict(base, result="timeout")) + with self.assertRaises(ValueError): + AcceptedPreviewPublicationDiagnostic(**dict(base, block_hash=" ")) + with self.assertRaises(ValueError): + AcceptedPreviewPublicationDiagnostic( + **dict(base, phase_seconds={"submitblock-rpc": 0.1}) + ) + with self.assertRaises(ValueError): + AcceptedPreviewPublicationDiagnostic(**dict(base, reconcile_caller="replay")) + with self.assertRaises(ValueError): + AcceptedPreviewPublicationDiagnostic(**dict(base, full_rescan_path="rust")) + + def test_log_fields_are_flat_json_serializable_and_fixed(self) -> None: + record = AcceptedPreviewPublicationDiagnostic( + block_hash="cd" * 32, + block_height=7, + result="degraded", + acceptance_to_publication_seconds=4.123456789, + phase_seconds={"reconcile": 3.0}, + reconcile_caller="post_confirm", + full_rescan_reason="reconcile_invalidation", + full_rescan_path="daemon", + full_rescan_seconds=2.5, + ) + fields = record.log_fields() + json.dumps(fields, sort_keys=True) + self.assertEqual( + set(fields), + { + "block_hash", + "block_height", + "result", + "acceptance_to_publication_seconds", + "reconcile_caller", + "full_rescan_reason", + "full_rescan_path", + "full_rescan_seconds", + "ledger_gate_wait_seconds", + "ledger_execute_seconds", + *(f"phase_{phase}_seconds" for phase in PRISM_ACCEPTED_LANDING_PHASES), + }, + ) + self.assertEqual(fields["acceptance_to_publication_seconds"], 4.123457) + self.assertEqual(fields["phase_reconcile_seconds"], 3.0) + self.assertEqual(fields["phase_lane_wait_seconds"], 0.0) + summary = record.summary() + self.assertIn("hash=" + "cd" * 32, summary) + self.assertIn("rescan=reconcile_invalidation/daemon=2.500s", summary) + self.assertIn("reconcile=3.000s", summary) + + +class LedgerReadFoldTests(unittest.TestCase): + STATS = { + "calls_total": 2, + "gate_wait_seconds_total": 0.25, + "gate_wait_seconds_max": 0.2, + "gate_timeouts_total": 1, + "execute_seconds_total": 1.0, + "execute_seconds_max": 0.75, + "execute_timeouts_total": 0, + } + + def test_contract_operations_pass_through_as_copies(self) -> None: + source = { + "pending_block_candidate_rows": dict(self.STATS), + "payout_window_snapshot": dict(self.STATS, calls_total=5), + } + folded = fold_ledger_read_stats(source) + self.assertEqual(folded, source) + folded["payout_window_snapshot"]["calls_total"] = 99 + self.assertEqual(source["payout_window_snapshot"]["calls_total"], 5) + self.assertEqual(fold_ledger_read_stats({}), {}) + + def test_out_of_contract_operations_merge_into_other(self) -> None: + folded = fold_ledger_read_stats( + { + "snapshot_at_job_issue": dict(self.STATS), + "pending_block_candidate_rows": dict(self.STATS), + "replay_probe": dict(self.STATS, gate_wait_seconds_max=0.9, execute_timeouts_total=3), + } + ) + self.assertEqual(set(folded), {"pending_block_candidate_rows", "other"}) + other = folded["other"] + self.assertEqual(other["calls_total"], 4) + self.assertIsInstance(other["calls_total"], int) + self.assertAlmostEqual(other["gate_wait_seconds_total"], 0.5) + self.assertEqual(other["gate_wait_seconds_max"], 0.9) + self.assertEqual(other["execute_seconds_max"], 0.75) + self.assertEqual(other["gate_timeouts_total"], 2) + self.assertEqual(other["execute_timeouts_total"], 3) + + +class RendererContractTests(unittest.TestCase): + def _labels(self, line: str) -> dict[str, str]: + return dict(re.findall(r'(\w+)="([^"]*)"', line)) + + def test_empty_owner_renders_deterministically_at_zero(self) -> None: + port = SimpleNamespace() + renderer = MetricsRenderer(port) # type: ignore[arg-type] + first = renderer.accepted_preview_attribution_metrics_lines() + second = renderer.accepted_preview_attribution_metrics_lines() + self.assertEqual(first, second) + families = [ + line.split()[2] for line in first if line.startswith("# TYPE ") + ] + self.assertEqual( + families, + [ + "qbit_prism_accepted_block_landing_phase_seconds", + "qbit_prism_reorg_reconcile_pass_seconds", + "qbit_prism_reorg_reconcile_step_seconds", + "qbit_prism_payout_window_full_rescan_seconds", + ], + ) + samples = [line for line in first if not line.startswith("#")] + for line in samples: + value = line.rsplit(" ", 1)[1] + self.assertIn(value, ("0", "0.000000"), line) + # Sample lines are unique: no cell renders twice. + self.assertEqual(len(samples), len({line.split(" ")[0] for line in samples})) + + def test_rendered_labels_stay_inside_the_vocabularies(self) -> None: + port = SimpleNamespace() + telemetry = ensure_accepted_preview_telemetry(port) + block_hash = "ab" * 32 + telemetry.observe_landing_phase("chain_probe", 0.3) + telemetry.observe_reconcile_pass("other", 0.1) + telemetry.observe_reconcile_step("other", "publish", 0.05) + telemetry.observe_payout_window_full_rescan("hash=" + block_hash, "daemon", 2.0) + telemetry.record_publication_diagnostic( + AcceptedPreviewPublicationDiagnostic( + block_hash=block_hash, + block_height=4242, + result="published", + acceptance_to_publication_seconds=4.5, + ) + ) + lines = MetricsRenderer(port).accepted_preview_attribution_metrics_lines() # type: ignore[arg-type] + payload = "\n".join(lines) + self.assertNotIn(block_hash, payload) + self.assertNotIn("4242", payload) + allowed = { + "phase": set(PRISM_ACCEPTED_LANDING_PHASES), + "caller": set(PRISM_REORG_RECONCILE_CALLERS), + "step": set(PRISM_REORG_RECONCILE_STEPS), + "reason": set(PRISM_PAYOUT_WINDOW_FULL_RESCAN_REASONS), + "path": set(PRISM_PAYOUT_WINDOW_FULL_RESCAN_PATHS), + } + for line in lines: + if line.startswith("#"): + continue + labels = self._labels(line) + self.assertTrue(labels, line) + for name, value in labels.items(): + self.assertIn(value, allowed[name], line) + self.assertIn( + 'qbit_prism_payout_window_full_rescan_seconds_count{reason="other",path="daemon"} 1', + lines, + ) + self.assertIn( + 'qbit_prism_accepted_block_landing_phase_seconds_sum{phase="chain_probe"} 0.300000', + lines, + ) + + def test_public_surface_is_declared(self) -> None: + for name in contract.__all__: + self.assertTrue(hasattr(contract, name), name) + for name in VOCABULARIES: + self.assertIn(name, contract.__all__) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_prism_admission_lock.py b/tests/test_prism_admission_lock.py new file mode 100644 index 000000000..1214b181c --- /dev/null +++ b/tests/test_prism_admission_lock.py @@ -0,0 +1,651 @@ +#!/usr/bin/env python3 +"""Admission-path lock isolation for the reconnect herd (issue #159). + +The coordinator lock used to serialize three unrelated things that a +reconnect herd hits at once: per-connection stale-grace anchoring, first-job +queue bookkeeping, and an O(N) fleet coverage census run once per delivered +job. These tests pin each of them to its own owner, and pin the lock order +that keeps the split deadlock-free. +""" + +from __future__ import annotations + +import threading +import time +import unittest +from decimal import Decimal +from types import SimpleNamespace + +from lab.prism.prism_coordinator import ( + ClientState, + PendingInitialJob, + PrismCoordinator, + WorkerIdentity, +) + + +HERD = 1024 +TIP_A = "aa" * 32 +TIP_B = "bb" * 32 + + +class FakeSocket: + def __init__(self) -> None: + self.closed = False + + def settimeout(self, _timeout: object) -> None: + return + + def setsockopt(self, *_args: object) -> None: + return + + def shutdown(self, _how: object) -> None: + return + + def close(self) -> None: + self.closed = True + + +class TrackedLock: + """Plain lock that remembers its current owner thread.""" + + def __init__(self) -> None: + self._lock = threading.Lock() + self._owner: int | None = None + + def acquire(self, blocking: bool = True, timeout: float = -1) -> bool: + acquired = self._lock.acquire(blocking, timeout) + if acquired: + self._owner = threading.get_ident() + return acquired + + def release(self) -> None: + self._owner = None + self._lock.release() + + def __enter__(self) -> "TrackedLock": + self.acquire() + return self + + def __exit__(self, *_args: object) -> None: + self.release() + + def held_by_current_thread(self) -> bool: + return self._owner == threading.get_ident() + + +class CountingLock: + """Coordinator-lock double that counts acquisitions and pins lock order. + + Acquisitions are counted while the lock is held, so the count is exact + under any amount of contention. ``forbidden`` names a lock that must never + be held when this one is taken; a violation there is the reverse nesting + that would make the admission split deadlock-prone. + """ + + def __init__(self, forbidden: TrackedLock | None = None) -> None: + self._lock = threading.RLock() + self._forbidden = forbidden + self.acquisitions = 0 + self.order_violations = 0 + + def acquire(self, blocking: bool = True, timeout: float = -1) -> bool: + forbidden = self._forbidden + if forbidden is not None and forbidden.held_by_current_thread(): + self.order_violations += 1 + acquired = self._lock.acquire(blocking, timeout) + if acquired: + self.acquisitions += 1 + return acquired + + def release(self) -> None: + self._lock.release() + + def __enter__(self) -> "CountingLock": + self.acquire() + return self + + def __exit__(self, *_args: object) -> None: + self.release() + + +class AnchorRecordingClient(ClientState): + """ClientState that counts every commit of the stale-grace anchor.""" + + @property + def tip_work_delivered(self) -> tuple[str, float] | None: + return self.__dict__.get("_anchor") + + @tip_work_delivered.setter + def tip_work_delivered(self, value: tuple[str, float] | None) -> None: + self.__dict__["_anchor"] = value + if value is not None: + self.__dict__["_anchor_writes"] = ( + int(self.__dict__.get("_anchor_writes", 0)) + 1 + ) + + @property + def anchor_writes(self) -> int: + return int(self.__dict__.get("_anchor_writes", 0)) + + +def worker(name: str) -> WorkerIdentity: + return WorkerIdentity( + username=name, + payout_address=name, + worker_name=None, + script_pubkey_hex="5220" + "11" * 32, + p2mr_program_hex="11" * 32, + ) + + +def coordinator( + *, + connection_limit: int = HERD + 8, + pending_limit: int = HERD, +) -> PrismCoordinator: + server = PrismCoordinator.__new__(PrismCoordinator) + server.stop_event = threading.Event() + server.clients = set() + server.jobs = {} + server.connection_counter = 0 + server.job_counter = 0 + server.stratum_max_connections = connection_limit + server.stratum_max_connections_per_username = 0 + server.stratum_max_pending_initial_jobs = pending_limit + server.stratum_initial_job_timeout_seconds = 30.0 + server.mining_health_startup_grace_seconds = 30.0 + server.initial_job_max_workers = 1 + server.tip_refresh_max_workers = 1 + server.tip_template_snapshot = None + server.started_monotonic = 0.0 + server.submitted_share_count = 0 + server.rejection_counts_by_reason = {} + server._record_heartbeat = lambda _name: None # type: ignore[method-assign] + server.apply_stratum_send_timeout = lambda _sock: None # type: ignore[method-assign] + server.client_startup_difficulty = lambda _profile: Decimal("1") # type: ignore[method-assign] + # The S2 admission lock is a private implementation detail; the lock-order + # assertions below are the reason this test reaches for it. + admission = TrackedLock() + server.lock = CountingLock(forbidden=admission) + server._ensure_job_cache_state() + server._ensure_tip_refresh_state() + server._ensure_initial_job_state() + service = server._ensure_job_delivery_service() + service._initial_job_admission_lock = admission + return server + + +def client( + server: PrismCoordinator, + connection_id: int, + *, + with_job: bool = False, + recording: bool = False, +) -> ClientState: + factory = AnchorRecordingClient if recording else ClientState + state = factory( + sock=FakeSocket(), + address=("127.0.0.1", 40_000 + connection_id), + connection_id=connection_id, + extranonce1_hex=f"{connection_id:08x}", + ) + state.subscribed = True + state.authorized = True + state.authorization_generation = 1 + state.worker = worker(f"miner-{connection_id}") + state.username = state.worker.username + if with_job: + state.active_job = SimpleNamespace( + template={"previousblockhash": TIP_A}, + payout_state_generation=0, + ) + server.clients.add(state) + return state + + +def run_concurrently(targets: list, *, timeout: float = 60.0) -> None: + """Start every target behind one barrier and join them all.""" + barrier = threading.Barrier(len(targets), timeout=timeout) + + def entry(target) -> None: # type: ignore[no-untyped-def] + barrier.wait() + target() + + threads = [threading.Thread(target=entry, args=(target,)) for target in targets] + for thread in threads: + thread.start() + for thread in threads: + thread.join(timeout) + for thread in threads: + assert not thread.is_alive(), "worker thread did not finish" + + +class DeliveryAnchorIsolationTests(unittest.TestCase): + def test_contended_coordinator_lock_cannot_delay_a_delivery_anchor(self) -> None: + server = coordinator() + blocked = client(server, 1) + delivering = client(server, 2) + held = threading.Event() + release = threading.Event() + + def hold_coordinator_lock() -> None: + with server.lock: + held.set() + release.wait(30) + + holder = threading.Thread(target=hold_coordinator_lock) + holder.start() + try: + self.assertTrue(held.wait(10)) + before = server.lock.acquisitions + started = time.monotonic() + server.note_tip_work_delivered(delivering, TIP_A) + elapsed = time.monotonic() - started + finally: + release.set() + holder.join(10) + + # A different connection's first delivery completed while the whole + # control plane was blocked: it neither took nor waited for the + # coordinator lock. + self.assertEqual(server.lock.acquisitions, before) + self.assertLess(elapsed, 5.0) + self.assertIsNotNone(delivering.tip_work_delivered) + assert delivering.tip_work_delivered is not None + self.assertEqual(delivering.tip_work_delivered[0], TIP_A) + self.assertIsNone(blocked.tip_work_delivered) + self.assertEqual(server.lock.order_violations, 0) + + def test_same_tip_refresh_race_cannot_slide_an_existing_anchor(self) -> None: + server = coordinator() + state = client(server, 1) + + server.note_tip_work_delivered(state, TIP_A) + first = state.tip_work_delivered + self.assertIsNotNone(first) + + # However many same-tip refreshes race, the stale-grace anchor stays + # on the first delivery for that tip. + run_concurrently( + [lambda: server.note_tip_work_delivered(state, TIP_A) for _ in range(16)] + ) + self.assertEqual(state.tip_work_delivered, first) + + # A new tip is a new anchor. + server.note_tip_work_delivered(state, TIP_B) + anchored = state.tip_work_delivered + assert anchored is not None and first is not None + self.assertEqual(anchored[0], TIP_B) + self.assertGreaterEqual(anchored[1], first[1]) + + def test_reconnect_herd_first_deliveries_anchor_each_connection_once(self) -> None: + server = coordinator() + clients = [ + client(server, index + 1, recording=True) for index in range(HERD) + ] + before = server.lock.acquisitions + + def deliver(state: ClientState) -> None: + server.note_tip_work_delivered(state, TIP_A) + + # Four threads race the first delivery of every connection at once. + run_concurrently( + [ + (lambda state=state: deliver(state)) + for state in clients + for _ in range(4) + ] + ) + + self.assertEqual(server.lock.acquisitions, before) + self.assertEqual(server.lock.order_violations, 0) + for state in clients: + anchor = state.tip_work_delivered + self.assertIsNotNone(anchor) + assert anchor is not None + self.assertEqual(anchor[0], TIP_A) + self.assertEqual(state.anchor_writes, 1) + + +class InitialJobAdmissionLockTests(unittest.TestCase): + def test_queue_bookkeeping_never_reaches_the_coordinator_lock(self) -> None: + server = coordinator(pending_limit=HERD) + started = threading.Event() + release = threading.Event() + + def blocked(request: PendingInitialJob) -> bool: + started.set() + release.wait(30) + return not server._initial_request_cancelled(request) + + server._run_initial_job = blocked # type: ignore[method-assign] + clients = [client(server, index + 1) for index in range(HERD)] + try: + # Fresh admission asks the coordinator exactly one constant-time + # question -- does this connection already hold current-tip work -- + # and nothing else. + before = server.lock.acquisitions + for state in clients: + self.assertTrue(server.schedule_initial_job(state)) + self.assertTrue(started.wait(10)) + self.assertEqual(server.lock.acquisitions - before, HERD) + self.assertEqual(len(server.pending_initial_jobs), HERD) + + # Coalescing a repeat request is pure queue bookkeeping. + before = server.lock.acquisitions + for state in clients: + self.assertTrue(server.schedule_initial_job(state)) + self.assertEqual(server.lock.acquisitions, before) + self.assertEqual(server.initial_job_coalesced_count, HERD) + self.assertEqual(len(server.pending_initial_jobs), HERD) + + # So is releasing every one of them again, including the physical + # queue reclamation its cancellation callbacks perform. + before = server.lock.acquisitions + for state in clients: + server.cancel_initial_job_delivery(state) + self.assertEqual(server.lock.acquisitions, before) + self.assertEqual(len(server.pending_initial_jobs), 0) + self.assertEqual(server.initial_job_cancelled_count, HERD) + self.assertEqual( + server.initial_job_queue_capacity_reclaimed_count, + HERD - 1, + ) + self.assertEqual(server.lock.order_violations, 0) + finally: + release.set() + server.shutdown_initial_job_executor() + + def test_queue_bookkeeping_runs_while_the_coordinator_lock_is_held(self) -> None: + server = coordinator(pending_limit=8) + submitted: list[PendingInitialJob] = [] + server._submit_initial_job_request = ( # type: ignore[method-assign] + lambda request: bool(submitted.append(request)) or True + ) + state = client(server, 1) + self.assertTrue(server.schedule_initial_job(state)) + self.assertEqual(len(submitted), 1) + + held = threading.Event() + release = threading.Event() + + def hold_coordinator_lock() -> None: + with server.lock: + held.set() + release.wait(30) + + finished = threading.Event() + + def bookkeep() -> None: + # Coalescing and cancellation are admission-domain only, so a + # blocked control plane cannot stall a reconnecting miner's queue + # bookkeeping. + server.schedule_initial_job(state) + server.cancel_initial_job_delivery(state) + finished.set() + + holder = threading.Thread(target=hold_coordinator_lock) + bookkeeper = threading.Thread(target=bookkeep) + holder.start() + try: + self.assertTrue(held.wait(10)) + bookkeeper.start() + self.assertTrue( + finished.wait(5), + "queue bookkeeping waited on the contended coordinator lock", + ) + # Still held: the bookkeeping above ran through, not after, the + # control plane being blocked. + self.assertFalse(release.is_set()) + self.assertEqual(server.initial_job_coalesced_count, 1) + self.assertEqual(len(server.pending_initial_jobs), 0) + self.assertEqual(server.initial_job_cancelled_count, 1) + finally: + release.set() + holder.join(10) + bookkeeper.join(10) + self.assertEqual(server.lock.order_violations, 0) + + def test_concurrent_admission_enforces_the_exact_pending_capacity(self) -> None: + capacity = 64 + server = coordinator(pending_limit=capacity) + submitted: list[PendingInitialJob] = [] + submit_lock = threading.Lock() + + def record(request: PendingInitialJob) -> bool: + with submit_lock: + submitted.append(request) + return True + + server._submit_initial_job_request = record # type: ignore[method-assign] + clients = [client(server, index + 1) for index in range(HERD)] + admitted: list[bool] = [False] * HERD + + def admit(index: int) -> None: + admitted[index] = bool(server.schedule_initial_job(clients[index])) + + run_concurrently( + [(lambda index=index: admit(index)) for index in range(HERD)] + ) + + self.assertEqual(sum(admitted), capacity) + self.assertEqual(len(server.pending_initial_jobs), capacity) + self.assertEqual(len(submitted), capacity) + self.assertEqual(server.initial_job_queue_rejection_count, HERD - capacity) + # Every rejected connection was retired exactly once, and every + # admitted one still owns its slot. + self.assertEqual(len(server.clients), capacity) + for index, state in enumerate(clients): + self.assertEqual(server.pending_initial_jobs.get(state) is not None, + admitted[index]) + self.assertEqual(state.sock.closed, not admitted[index]) + self.assertEqual(server.lock.order_violations, 0) + + +class PendingQueueCompatibilitySurfaceTests(unittest.TestCase): + """The coordinator-facing queue surface is owned by the admission lock.""" + + def test_capacity_read_is_taken_under_the_admission_lock(self) -> None: + server = coordinator(pending_limit=8) + server._submit_initial_job_request = lambda _request: True # type: ignore[method-assign] + state = client(server, 1) + self.assertTrue(server.schedule_initial_job(state)) + + admission = server._ensure_job_delivery_service()._initial_job_admission_lock + started = threading.Event() + finished = threading.Event() + observed: list[object] = [] + + def capacity_read() -> None: + started.set() + observed.append(len(server.pending_initial_jobs)) + observed.append(state in server.pending_initial_jobs) + finished.set() + + reader = threading.Thread(target=capacity_read) + self.assertTrue(admission.acquire(timeout=5)) + try: + reader.start() + self.assertTrue(started.wait(5)) + # The external capacity read is the one the session owner's + # reauthorization pre-check performs. It must serialize with + # admission rather than run beside it. + self.assertFalse( + finished.wait(0.5), + "capacity read completed without the admission lock", + ) + finally: + admission.release() + reader.join(10) + + self.assertTrue(finished.is_set()) + self.assertEqual(observed, [1, True]) + + def test_external_reads_cannot_observe_a_partly_swept_queue(self) -> None: + cohort = 64 + server = coordinator(pending_limit=HERD) + clients = [client(server, index + 1) for index in range(cohort)] + observed: set[int] = set() + failures: list[BaseException] = [] + stop = threading.Event() + empty_observed = threading.Event() + cohort_observed = threading.Event() + + def reader() -> None: + try: + while not stop.is_set(): + size = len(server.pending_initial_jobs) + observed.add(size) + if size == 0: + empty_observed.set() + elif size == cohort: + cohort_observed.set() + except BaseException as exc: # pragma: no cover - failure path + failures.append(exc) + + watcher = threading.Thread(target=reader) + watcher.start() + try: + self.assertTrue( + empty_observed.wait(5), + "reader did not observe the empty queue before seeding", + ) + for _ in range(24): + expired = time.monotonic() - 1.0 + for state in clients: + state.closing = False + server.clients.add(state) + server.pending_initial_jobs = { + state: PendingInitialJob( + client=state, + connection_id=state.connection_id, + authorization_generation=1, + difficulty_generation=0, + worker=state.worker, + requested_monotonic=expired, + deadline_monotonic=expired, + ) + for state in clients + } + self.assertTrue( + cohort_observed.wait(5), + "reader did not observe the fully seeded queue", + ) + self.assertEqual(server.sweep_initial_job_timeouts(), cohort) + finally: + stop.set() + watcher.join(10) + + self.assertEqual(failures, []) + # The sweep releases the whole cohort inside one critical section and + # seeding installs it inside another, so an external reader sees the + # queue wholly pending or wholly released -- never mid-cancellation. + self.assertTrue( + observed.issubset({0, cohort}), + f"external read observed a partly swept queue: {sorted(observed)}", + ) + self.assertIn(cohort, observed) + self.assertIn(0, observed) + self.assertEqual(server.lock.order_violations, 0) + + def test_surface_is_strictly_immutable(self) -> None: + server = coordinator(pending_limit=4) + state = client(server, 1) + view = server.pending_initial_jobs + + # No write reaches admission state through the compatibility surface: + # every mutator is absent, not merely guarded. + for mutator in ( + "__setitem__", + "__delitem__", + "pop", + "popitem", + "clear", + "update", + "setdefault", + ): + self.assertFalse( + hasattr(view, mutator), + f"compatibility surface exposes {mutator}", + ) + request = PendingInitialJob( + client=state, + connection_id=state.connection_id, + authorization_generation=1, + difficulty_generation=0, + worker=state.worker, + requested_monotonic=0.0, + deadline_monotonic=None, + ) + with self.assertRaises(TypeError): + view[state] = request + with self.assertRaises(TypeError): + del view[state] + + # Seeding goes through the setter, which adopts the mapping's contents + # under the admission lock instead of exposing a mutable map. + server.pending_initial_jobs = {state: request} + self.assertEqual(len(server.pending_initial_jobs), 1) + self.assertIs(server.pending_initial_jobs[state], request) + + # The owner's mutable map is private and is never the object handed to + # a compatibility reader. + service = server._ensure_job_delivery_service() + self.assertIsNot(view, service._pending_initial_jobs) + self.assertIsInstance(service._pending_initial_jobs, dict) + + +class CoverageCensusOwnershipTests(unittest.TestCase): + def test_observability_refresh_not_delivery_resets_the_coverage_failure( + self, + ) -> None: + server = coordinator(connection_limit=4) + state = client(server, 1, with_job=True) + server.current_tip_first_seen = (TIP_A, 100.0) + observability = server._ensure_observability_service() + observability.set_delivery_failure_started_monotonic_for_test(50.0) + + # Delivery anchors this connection and does nothing else: no fleet + # census, no coordinator lock, no health-policy write. + before = server.lock.acquisitions + server.note_tip_work_delivered(state, TIP_A) + self.assertEqual(server.lock.acquisitions, before) + self.assertEqual( + observability.state().mining_delivery_failure_started_monotonic, + 50.0, + ) + + # The refresh census -- the one that already walks authorized clients + # and current-work coverage -- performs the reset. + inputs = observability.port.mining_delivery_inputs(120.0) + + self.assertEqual(inputs.authorized_connections, 1) + self.assertEqual(inputs.clients_with_current_tip_jobs, 1) + self.assertIsNone( + observability.state().mining_delivery_failure_started_monotonic + ) + + def test_refresh_below_the_coverage_threshold_leaves_the_failure_running( + self, + ) -> None: + server = coordinator(connection_limit=HERD) + covered = [client(server, index + 1, with_job=True) for index in range(90)] + for index in range(10): + client(server, 1_000 + index) + server.current_tip_first_seen = (TIP_A, 100.0) + observability = server._ensure_observability_service() + observability.set_delivery_failure_started_monotonic_for_test(50.0) + + inputs = observability.port.mining_delivery_inputs(120.0) + + self.assertEqual(inputs.authorized_connections, 100) + self.assertEqual(inputs.clients_with_current_tip_jobs, len(covered)) + self.assertEqual( + observability.state().mining_delivery_failure_started_monotonic, + 50.0, + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_prism_allocator_experiment.py b/tests/test_prism_allocator_experiment.py new file mode 100644 index 000000000..5d9712a45 --- /dev/null +++ b/tests/test_prism_allocator_experiment.py @@ -0,0 +1,256 @@ +#!/usr/bin/env python3 +"""Issue #226 part 2: the storm instrument's heap and allocator readings. + +These pin the mechanism the allocator experiment relies on, not any byte +count: that a phase reading carries every field the soak runbook reads, +that the arena probe multiplies glibc arenas at the default and stops at a +configured cap, that the experiment driver strips its own options before +re-invoking the instrument, and that ``--heap-report`` produces the section +the runbook compares across settings. +""" + +from __future__ import annotations + +import json +import os +import subprocess +import sys +import time +import unittest +from dataclasses import fields +from pathlib import Path + +from lab.prism.process_telemetry import ProcessHeapTelemetry, read_malloc_arena_count +from tests.prism_candidate_storm import ( + ARENA_PROBE_LIVE_BUFFERS, + ARENA_PROBE_MAX_BYTES, + ARENA_PROBE_MIN_BYTES, + HeapReading, + allocator_environment, + allocator_experiment_child_args, + run_arena_probe, + take_heap_reading, +) + +ROOT = Path(__file__).resolve().parents[1] +STORM = ROOT / "tests" / "prism_candidate_storm.py" + +HEAP_READING_FIELDS = ( + "phase", + "elapsed_seconds", + "resident_bytes", + "allocated_blocks", + "threads", + "malloc_info_available", + "malloc_arena_bytes", + "malloc_in_use_bytes", + "malloc_free_bytes", + "malloc_mmapped_bytes", + "malloc_arena_count", + "malloc_free_top_bytes", + "malloc_free_bin_bytes", + "anonymous_regions", + "anonymous_bytes", + "anonymous_regions_4mib_to_64mib", + "heap_segment_bytes", +) + + +class HeapReadingTests(unittest.TestCase): + def test_reading_carries_every_field_the_runbook_reads(self) -> None: + self.assertEqual(tuple(field.name for field in fields(HeapReading)), HEAP_READING_FIELDS) + reading = take_heap_reading("probe", ProcessHeapTelemetry(), 0.0) + self.assertEqual(reading.phase, "probe") + self.assertGreaterEqual(reading.elapsed_seconds, 0.0) + self.assertGreater(reading.allocated_blocks, 0) + self.assertGreaterEqual(reading.threads, 1) + if Path("/proc/self/statm").exists(): + self.assertGreater(reading.resident_bytes, 0) + self.assertGreaterEqual(reading.anonymous_regions, 1) + self.assertGreaterEqual(reading.anonymous_regions_4mib_to_64mib, 0) + if reading.malloc_info_available: + self.assertGreaterEqual(reading.malloc_arena_count, 1) + self.assertGreaterEqual(reading.malloc_arena_bytes, reading.malloc_in_use_bytes) + + def test_allocator_environment_reports_only_allocator_variables(self) -> None: + with unittest.mock.patch.dict( + os.environ, + {"MALLOC_ARENA_MAX": "2", "PYTHONMALLOC": "malloc", "PRISM_UNRELATED": "x"}, + ): + reported = allocator_environment() + self.assertEqual(reported.get("MALLOC_ARENA_MAX"), "2") + self.assertEqual(reported.get("PYTHONMALLOC"), "malloc") + self.assertNotIn("PRISM_UNRELATED", reported) + + +class ArenaProbeTests(unittest.TestCase): + def test_probe_buffers_land_in_glibc_arenas(self) -> None: + """Over pymalloc's ceiling, under glibc's initial mmap threshold.""" + self.assertGreater(ARENA_PROBE_MIN_BYTES, 512) + self.assertLess(ARENA_PROBE_MAX_BYTES, 128 * 1024) + self.assertGreater(ARENA_PROBE_LIVE_BUFFERS, 0) + + def test_probe_multiplies_arenas_at_default_and_respects_the_cap(self) -> None: + """glibc gives concurrent threads their own arenas until MALLOC_ARENA_MAX.""" + if read_malloc_arena_count() < 0: + self.skipTest("malloc_info unavailable on this platform") + script = ( + "import sys, json\n" + f"sys.path.insert(0, {str(ROOT)!r})\n" + "from tests.prism_candidate_storm import run_arena_probe\n" + "print(json.dumps(run_arena_probe(12, 40)))\n" + ) + uncapped = subprocess.run( + [sys.executable, "-c", script], + env={k: v for k, v in os.environ.items() if k != "MALLOC_ARENA_MAX"}, + capture_output=True, + text=True, + check=True, + ) + capped = subprocess.run( + [sys.executable, "-c", script], + env={**os.environ, "MALLOC_ARENA_MAX": "3"}, + capture_output=True, + text=True, + check=True, + ) + free_run = json.loads(uncapped.stdout.strip().splitlines()[-1]) + cap_run = json.loads(capped.stdout.strip().splitlines()[-1]) + self.assertEqual(free_run["threads"], 12) + self.assertEqual(free_run["rounds"], 40) + self.assertGreater(free_run["arena_count_after"], free_run["arena_count_before"]) + self.assertGreater(free_run["arena_count_after"], 3) + self.assertLessEqual(cap_run["arena_count_after"], 3) + for run in (free_run, cap_run): + self.assertEqual(run["failures"], []) + self.assertGreaterEqual(run["seconds"], 0.0) + self.assertTrue(run["malloc_info_available"]) + # Freed rings stay on the arenas' free lists: the mechanism the + # trim is later judged by. + self.assertGreaterEqual(run["malloc_free_after"], run["malloc_free_before"]) + # In-process, with the cap of this process (whatever it is), the + # probe still returns the full record. + record = run_arena_probe(2, 5) + self.assertEqual(set(record), set(free_run)) + self.assertEqual(record["failures"], []) + + def test_probe_reports_a_failing_worker_instead_of_wedging(self) -> None: + """One worker that raises breaks the barrier for the rest; the probe returns.""" + calls = {"count": 0} + + def failing_allocate(size: int) -> bytes: + calls["count"] += 1 + if calls["count"] == 1: + raise RuntimeError("synthetic allocation failure") + return bytes(size) + + started = time.monotonic() + record = run_arena_probe(4, 3, allocate=failing_allocate) + self.assertLess(time.monotonic() - started, 30.0) + self.assertEqual(record["threads"], 4) + self.assertTrue(record["failures"], record) + self.assertTrue( + any("synthetic allocation failure" in failure for failure in record["failures"]), + record["failures"], + ) + + +class ExperimentDriverTests(unittest.TestCase): + def test_child_args_strip_the_experiment_options_and_add_heap_report(self) -> None: + self.assertEqual( + allocator_experiment_child_args( + ["--decide", "--allocator-experiment", "2,4", "--drain-per-row"] + ), + ["--decide", "--drain-per-row", "--heap-report"], + ) + self.assertEqual( + allocator_experiment_child_args( + [ + "--allocator-experiment=default", + "--allocator-mmap-threshold=131072", + "--heap-report", + "--candidates", + "8", + ] + ), + ["--heap-report", "--candidates", "8"], + ) + self.assertEqual( + allocator_experiment_child_args(["--allocator-mmap-threshold", "131072"]), + ["--heap-report"], + ) + # --verbose never reaches a child: its logs would precede the JSON + # the parent parses and the parse would fail after the whole storm. + self.assertEqual( + allocator_experiment_child_args(["--verbose", "--decide", "--allocator-experiment", "2"]), + ["--decide", "--heap-report"], + ) + + def test_heap_report_section_is_produced_end_to_end(self) -> None: + """A small storm with --heap-report carries the readings the runbook compares.""" + completed = subprocess.run( + [ + sys.executable, + str(STORM), + "--candidates", + "40", + "--queue-depth", + "8", + "--heap-report", + "--arena-probe-threads", + "2", + "--arena-probe-rounds", + "10", + ], + cwd=ROOT, + capture_output=True, + text=True, + check=True, + ) + report = json.loads(completed.stdout) + heap = report["heap"] + self.assertEqual( + set(heap), + { + "python", + "platform", + "libc", + "machine", + "cpu_count", + "allocator_env", + "phase_seconds", + "readings", + "arena_probe", + "malloc_trim", + }, + ) + phases = [reading["phase"] for reading in heap["readings"]] + self.assertEqual( + phases, + [ + "start", + "after_seed_live", + "after_restart_enumerate", + "after_arena_probe", + "after_release_gc", + "after_malloc_trim", + ], + ) + for reading in heap["readings"]: + self.assertEqual(set(reading), set(HEAP_READING_FIELDS)) + self.assertEqual(set(heap["phase_seconds"]), {"seed_live", "restart_enumerate"}) + self.assertEqual(heap["arena_probe"]["threads"], 2) + self.assertEqual(heap["arena_probe"]["failures"], []) + if heap["readings"][0]["malloc_info_available"]: + trim = heap["malloc_trim"] + self.assertIsNotNone(trim) + self.assertEqual(trim["in_use_delta"], 0) + self.assertLessEqual(trim["free_delta"], 0) + # The storm sections the instrument already produced are untouched. + self.assertEqual(report["input"], {"candidates": 40, "queue_depth": 8}) + self.assertIn("live", report) + self.assertIn("restart", report) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_prism_ancestor_redrive.py b/tests/test_prism_ancestor_redrive.py new file mode 100644 index 000000000..32a9c7d18 --- /dev/null +++ b/tests/test_prism_ancestor_redrive.py @@ -0,0 +1,809 @@ +#!/usr/bin/env python3 +"""In-process targeted ancestor re-drive for stuck payout transitions (#190). + +The production wedge these tests reproduce (2026-08-24 smoke test): a found +block's finalization deferred on an ancestor whose accepted payout +transition was armed and landed but whose preview never published. The +retained-retry loop could only re-check the transition -- 37 cycles of +``ledger-confirmation-failed`` over 906 seconds -- while the durable-replay +pass, the one path that provably resolves the ancestor (startup replay +resolved the same state in ~19s after the watchdog restart), was +short-circuited by the very retry candidate it needed to unblock. + +The fix under test: a per-ancestor streak of finalization deferrals arms a +targeted re-drive; the next ``replay_pending`` pass bypasses the +retry/live/replay-queue short-circuits, re-enumerates the durable outbox +through the same adoption startup replay uses (same pagination, same +landing-class budget), and either re-queues the ancestor's pending row for +ordinary replay finalization or -- when the completed enumeration proves no +pending row exists, nothing in-process owns the hash, the block is durably +confirmed, and a fresh chain probe proves it active on the node's chain -- +clears the stale transition. The attempt cap bounds the mechanism per +ancestor (each attempt needs a fresh deferral streak); on exhaustion +deferrals fall back to the pre-#190 behavior and the publication-progress +watchdog remains the backstop. +""" +# ruff: noqa: E402 + +from __future__ import annotations + +import dataclasses +import sys +import time +import unittest +from pathlib import Path +from types import SimpleNamespace +from unittest.mock import patch + +if not __package__: + sys.path.insert(0, str(Path(__file__).resolve().parents[1])) + +from lab.prism.share_ledger import PendingShare, SingleWriterShareLedger +from tests.prism_coordinator_test_support import ( + coordinator as light_coordinator, +) +from tests.prism_vardiff_test_support import block_candidate, submit_coordinator + + +ANCESTOR_HASH = "aa" * 32 +CHILD_HASH = "cc" * 32 +PARENT_TIP = "00" * 32 + + +def _pending_share(block_hash: str, stamp: int) -> PendingShare: + return PendingShare( + share_id=f"miner-a:{block_hash}", + miner_id="miner-a", + order_key="miner-a", + p2mr_program_hex="11" * 32, + share_difficulty=1, + network_difficulty=1, + template_height=9, + job_id="job-1", + job_issued_at_ms=1, + accepted_at_ms=stamp, + ntime=1, + ) + + +def _wedged_coordinator(): + """The incident shape: retained child, armed+landed ancestor, no preview. + + The ancestor's durable outbox row stays pending (its replay source), its + in-memory transition is armed and landed with the preview never + published, and the child found on top of it is retained in the retry + slot -- exactly the state the 2026-08-24 run wedged in. + """ + server, state, _recording = submit_coordinator() + ledger = SingleWriterShareLedger() + server.ledger = ledger + server.block_candidate_retry_initial_seconds = 0.0 + server.block_candidate_retry_max_seconds = 0.0 + ancestor = block_candidate( + server, + state, + SimpleNamespace( + coinbase_tx_hex="00", + block_hash_hex=ANCESTOR_HASH, + block_hex="00", + share_pass=False, + block_pass=True, + ), + pending_share=_pending_share(ANCESTOR_HASH, 1), + ) + ledger.persist_block_candidate_intent(server.block_candidate_intent(ancestor)) + server._begin_accepted_block_payout_preview(ANCESTOR_HASH, block_height=10) + server._mark_accepted_block_payout_landed(ANCESTOR_HASH, block_height=10) + child_context = SimpleNamespace( + job=SimpleNamespace( + job_id="job-2", + share_target=server.jobs["job-1"].job.share_target, + share_difficulty=server.jobs["job-1"].job.share_difficulty, + transaction_hexes=(), + ), + template={ + "previousblockhash": ANCESTOR_HASH, + "height": 11, + "coinbasevalue": 50_00000000, + }, + found_block={"network_difficulty": 1}, + issued_at_ms=12346, + collection_only=False, + worker=state.worker, + shares_json=[], + prior_balances=[], + ) + server.jobs["job-2"] = child_context + state.active_job_ids.add("job-2") + child = block_candidate( + server, + state, + SimpleNamespace( + coinbase_tx_hex="00", + block_hash_hex=CHILD_HASH, + block_hex="00", + share_pass=False, + block_pass=True, + ), + job_id="job-2", + pending_share=_pending_share(CHILD_HASH, 2), + ) + # The child's intent is durable too (persisted at admission in the + # incident), so a forced enumeration sees a pending row for a hash the + # retry holder still owns -- adoption must skip it. + ledger.persist_block_candidate_intent(server.block_candidate_intent(child)) + server._retain_block_candidate_for_retry(child) + return server, state, ledger, child + + +def _script_chain_probe(server, result): + """Pin the sweep's node-side probe; returns the recorded calls.""" + service = server._ensure_block_candidate_service() + calls: list[tuple[str, int | None]] = [] + + def probe(block_hash, *, expected_height=None): + calls.append((str(block_hash).lower(), expected_height)) + return result + + service._block_candidate_chain_probe = probe + return calls + + +class SweepLedger(SingleWriterShareLedger): + """Empty outbox plus a caller-scripted durable pool-block verdict.""" + + def __init__(self, block_state: dict[str, object] | None) -> None: + super().__init__() + self.scripted_block_state = block_state + self.pool_block_state_calls: list[str] = [] + + def pool_block_state(self, *, block_hash: str) -> dict[str, object] | None: + self.pool_block_state_calls.append(block_hash) + return self.scripted_block_state + + +class AncestorRedriveWedgeTests(unittest.TestCase): + """Issue #190 finding 1: the wedge, and the re-drive that resolves it.""" + + def _child_defers(self, server) -> bool: + return server._defer_for_pending_parent_payout_transition( + block_hash=CHILD_HASH, + parent_hash=ANCESTOR_HASH, + parent_height=10, + worker=None, + ) + + def test_deferral_streak_redrives_the_stuck_ancestor(self) -> None: + server, _state, _ledger, _child = _wedged_coordinator() + server.accepted_parent_redrive_defer_threshold = 3 + server.accepted_parent_redrive_attempt_max = 3 + service = server._ensure_block_candidate_service() + + # The pre-fix wedge: every retry cycle re-checks the transition and + # defers again, while the durable replay pass -- the only code that + # can resolve the ancestor -- short-circuits on the retained retry + # candidate it exists to unblock. + with patch("builtins.print"): + for _cycle in range(2): + self.assertTrue(self._child_defers(server)) + self.assertEqual(server.replay_pending_block_candidates(), 0) + self.assertFalse(service._ancestor_redrive_owed()) + + # Crossing the deferral threshold arms the targeted re-drive. + self.assertTrue(self._child_defers(server)) + self.assertTrue(service._ancestor_redrive_owed()) + self.assertEqual(int(server.accepted_parent_redrive_attempt_count), 1) + + # The armed pass bypasses the retry short-circuit and re-adopts the + # ancestor's pending row through the same durable-replay code + # startup replay runs. The child's pending row is durable too, but + # its live copy sits in the retry holder, so adoption must skip it + # rather than install a second in-process copy under the holder. + with patch("builtins.print"): + self.assertEqual(server.replay_pending_block_candidates(), 1) + self.assertFalse(service._ancestor_redrive_owed()) + with server.lock: + self.assertIn(ANCESTOR_HASH, server._block_replay_inflight_hashes) + self.assertNotIn(CHILD_HASH, server._block_replay_inflight_hashes) + + # The submitter's next pass runs the re-driven ancestor while the + # child's retry is parked in backoff. The finalization tail itself + # is owned by the block-finalization suites; the double here + # resolves the transition through the same production payout-state + # seam that tail uses on completion. + landed: list[str] = [] + + def finalize(candidate, **_kwargs) -> bool: + block_hash = str(candidate.submission.block_hash_hex).lower() + landed.append(block_hash) + server._clear_accepted_block_payout_preview(block_hash) + return True + + server.submit_block_candidate = finalize + # Post-accept job fanout is the tip-refresh owner's machinery, not + # this wedge's; the replay-reconstructed client is not a full + # ClientState in this fixture. + server.refresh_jobs_after_pending_accepted_block = ( + lambda *_args, **_kwargs: 0 + ) + server._block_candidate_retry_not_before = { + CHILD_HASH: time.monotonic() + 300.0 + } + with patch("builtins.print"): + self.assertTrue(server.submit_next_block_candidate()) + self.assertEqual(landed, [ANCESTOR_HASH]) + with server._accepted_block_payout_preview_condition: + self.assertNotIn( + ANCESTOR_HASH, server._accepted_block_payout_previews + ) + + # With the ancestor resolved, the retained child's next retry passes + # the fence and its finalization completes. + with patch("builtins.print"): + self.assertFalse(self._child_defers(server)) + self.assertEqual(int(server.accepted_parent_redrive_resolved_count), 1) + server._clear_block_candidate_retry_state(CHILD_HASH) + with patch("builtins.print"): + self.assertTrue(server.submit_next_block_candidate()) + self.assertEqual(landed, [ANCESTOR_HASH, CHILD_HASH]) + + def test_streak_below_threshold_never_arms_a_redrive(self) -> None: + server, _state, _ledger, _child = _wedged_coordinator() + server.accepted_parent_redrive_defer_threshold = 5 + service = server._ensure_block_candidate_service() + + with patch("builtins.print"): + for _cycle in range(4): + self.assertTrue(self._child_defers(server)) + self.assertEqual(server.replay_pending_block_candidates(), 0) + self.assertFalse(service._ancestor_redrive_owed()) + self.assertEqual(int(server.accepted_parent_redrive_attempt_count), 0) + + def test_redrive_respects_the_attempt_cap_and_falls_back(self) -> None: + server, _rpc = light_coordinator() + server.accepted_parent_redrive_defer_threshold = 1 + server.accepted_parent_redrive_attempt_max = 2 + service = server._ensure_block_candidate_service() + server._begin_accepted_block_payout_preview(ANCESTOR_HASH, block_height=9) + server._mark_accepted_block_payout_landed(ANCESTOR_HASH, block_height=9) + + def child_defers() -> bool: + return server._defer_for_pending_parent_payout_transition( + block_hash=CHILD_HASH, + parent_hash=ANCESTOR_HASH, + parent_height=9, + worker=None, + ) + + with patch("builtins.print"): + for expected_attempts in (1, 2): + self.assertTrue(child_defers()) + self.assertEqual( + int(server.accepted_parent_redrive_attempt_count), + expected_attempts, + ) + self.assertTrue(service._ancestor_redrive_owed()) + self.assertEqual( + service._consume_ancestor_redrive_requests(), + (ANCESTOR_HASH,), + ) + # The cap is exhausted: the wedge keeps deferring exactly as it + # did before the fix, no further pass is armed, and exhaustion + # is counted once for the ancestor. + for _cycle in range(3): + self.assertTrue(child_defers()) + self.assertFalse(service._ancestor_redrive_owed()) + self.assertEqual(int(server.accepted_parent_redrive_attempt_count), 2) + self.assertEqual(int(server.accepted_parent_redrive_exhausted_count), 1) + server._clear_accepted_block_payout_preview(ANCESTOR_HASH) + + def test_armed_request_does_not_double_arm_before_consumption(self) -> None: + server, _rpc = light_coordinator() + server.accepted_parent_redrive_defer_threshold = 1 + server.accepted_parent_redrive_attempt_max = 5 + service = server._ensure_block_candidate_service() + server._begin_accepted_block_payout_preview(ANCESTOR_HASH, block_height=9) + server._mark_accepted_block_payout_landed(ANCESTOR_HASH, block_height=9) + + with patch("builtins.print"): + for _cycle in range(3): + self.assertTrue( + server._defer_for_pending_parent_payout_transition( + block_hash=CHILD_HASH, + parent_hash=ANCESTOR_HASH, + parent_height=9, + worker=None, + ) + ) + # Deferrals while a pass is already armed count toward nothing new. + self.assertEqual(int(server.accepted_parent_redrive_attempt_count), 1) + self.assertEqual( + service._consume_ancestor_redrive_requests(), (ANCESTOR_HASH,) + ) + server._clear_accepted_block_payout_preview(ANCESTOR_HASH) + + +class AncestorRedriveSweepTests(unittest.TestCase): + """The no-pending-row wedge shape: converge with durable state, or stand + down for the watchdog.""" + + def _defer_once(self, server) -> None: + with patch("builtins.print"): + self.assertTrue( + server._defer_for_pending_parent_payout_transition( + block_hash=CHILD_HASH, + parent_hash=ANCESTOR_HASH, + parent_height=10, + worker=None, + ) + ) + + def _armed_sweep_coordinator(self, block_state, *, chain_probe=True): + server, _state, _recording = submit_coordinator() + ledger = SweepLedger(block_state) + server.ledger = ledger + server.accepted_parent_redrive_defer_threshold = 1 + probe_calls = _script_chain_probe(server, chain_probe) + server._begin_accepted_block_payout_preview( + ANCESTOR_HASH, block_height=10 + ) + server._mark_accepted_block_payout_landed(ANCESTOR_HASH, block_height=10) + return server, ledger, probe_calls + + def test_confirmed_stale_transition_is_cleared(self) -> None: + server, ledger, probe_calls = self._armed_sweep_coordinator( + { + "block_hash": ANCESTOR_HASH, + "block_height": 10, + "parent_hash": PARENT_TIP, + "chain_state": "confirmed", + "maturity_state": "immature", + } + ) + self._defer_once(server) + with patch("builtins.print"): + self.assertEqual(server.replay_pending_block_candidates(), 0) + self.assertEqual(ledger.pool_block_state_calls, [ANCESTOR_HASH]) + # The clear demanded the node-side proof at the transition's height. + self.assertEqual(probe_calls, [(ANCESTOR_HASH, 10)]) + with server._accepted_block_payout_preview_condition: + self.assertNotIn( + ANCESTOR_HASH, server._accepted_block_payout_previews + ) + self.assertNotIn( + ANCESTOR_HASH, + server._invalidated_accepted_block_payout_previews, + ) + with patch("builtins.print"): + self.assertFalse( + server._defer_for_pending_parent_payout_transition( + block_hash=CHILD_HASH, + parent_hash=ANCESTOR_HASH, + parent_height=10, + worker=None, + ) + ) + self.assertEqual(int(server.accepted_parent_redrive_resolved_count), 1) + + def test_unconfirmed_transition_is_left_for_the_watchdog(self) -> None: + server, ledger, _probe_calls = self._armed_sweep_coordinator(None) + self._defer_once(server) + with patch("builtins.print"): + self.assertEqual(server.replay_pending_block_candidates(), 0) + self.assertEqual(ledger.pool_block_state_calls, [ANCESTOR_HASH]) + with server._accepted_block_payout_preview_condition: + self.assertIn(ANCESTOR_HASH, server._accepted_block_payout_previews) + self._defer_once(server) + self.assertEqual(int(server.accepted_parent_redrive_resolved_count), 0) + + def test_reversed_pool_block_transition_is_left_alone(self) -> None: + server, ledger, _probe_calls = self._armed_sweep_coordinator( + { + "block_hash": ANCESTOR_HASH, + "block_height": 10, + "parent_hash": PARENT_TIP, + "chain_state": "confirmed", + "maturity_state": "reversed", + } + ) + self._defer_once(server) + with patch("builtins.print"): + self.assertEqual(server.replay_pending_block_candidates(), 0) + with server._accepted_block_payout_preview_condition: + self.assertIn(ANCESTOR_HASH, server._accepted_block_payout_previews) + + def test_transition_owned_by_an_in_process_lane_is_left_alone(self) -> None: + server, ledger, _probe_calls = self._armed_sweep_coordinator( + { + "block_hash": ANCESTOR_HASH, + "block_height": 10, + "parent_hash": PARENT_TIP, + "chain_state": "confirmed", + "maturity_state": "immature", + } + ) + service = server._ensure_block_candidate_service() + service._ensure_block_replay_state() + # Armed first, then adopted: deferrals against an already-inflight + # ancestor bank nothing (covered in the bookkeeping suite), so the + # adopted short-circuit is reached by arming before adoption. + self._defer_once(server) + with server.lock: + server._block_replay_inflight_hashes.add(ANCESTOR_HASH) + with patch("builtins.print"): + self.assertEqual(server.replay_pending_block_candidates(), 0) + # An inflight hash means ordinary replay finalization still owns the + # resolution; the sweep never consults the ledger for it. + self.assertEqual(ledger.pool_block_state_calls, []) + with server._accepted_block_payout_preview_condition: + self.assertIn(ANCESTOR_HASH, server._accepted_block_payout_previews) + + def test_unprovable_chain_activity_is_left_for_the_watchdog(self) -> None: + """A durably confirmed row alone is not proof against a fresh reorg. + + Reconciliation fails closed while a landed transition exists, so a + block qbitd has already reorged out can still read confirmed in the + durable row. The sweep must demand the node-side probe the landing + demands and stand down on anything short of a definite True. + """ + for probe_verdict in (None, False): + with self.subTest(probe_verdict=probe_verdict): + server, ledger, probe_calls = self._armed_sweep_coordinator( + { + "block_hash": ANCESTOR_HASH, + "block_height": 10, + "parent_hash": PARENT_TIP, + "chain_state": "confirmed", + "maturity_state": "immature", + }, + chain_probe=probe_verdict, + ) + self._defer_once(server) + with patch("builtins.print"): + self.assertEqual( + server.replay_pending_block_candidates(), 0 + ) + self.assertEqual( + ledger.pool_block_state_calls, [ANCESTOR_HASH] + ) + self.assertEqual(probe_calls, [(ANCESTOR_HASH, 10)]) + with server._accepted_block_payout_preview_condition: + self.assertIn( + ANCESTOR_HASH, server._accepted_block_payout_previews + ) + self.assertEqual( + int(server.accepted_parent_redrive_resolved_count), 0 + ) + + def test_retry_held_ancestor_is_left_to_its_owner(self) -> None: + """The in-process-ownership guard itself, reached with nothing adopted. + + The ancestor's own candidate sits in the retry holder while its + outbox row is already terminal, so the forced pass enumerates + nothing, the adopted short-circuit cannot fire, and the sweep runs. + The guard must see the held copy and stand down before the durable + read or the chain probe -- a guard that wrongly answered "not owned" + would clear the transition out from under the live lane and fails + every assertion below. + """ + server, state, _recording = submit_coordinator() + ledger = SweepLedger( + { + "block_hash": ANCESTOR_HASH, + "block_height": 10, + "parent_hash": PARENT_TIP, + "chain_state": "confirmed", + "maturity_state": "immature", + } + ) + server.ledger = ledger + server.accepted_parent_redrive_defer_threshold = 1 + probe_calls = _script_chain_probe(server, True) + ancestor = block_candidate( + server, + state, + SimpleNamespace( + coinbase_tx_hex="00", + block_hash_hex=ANCESTOR_HASH, + block_hex="00", + share_pass=False, + block_pass=True, + ), + pending_share=_pending_share(ANCESTOR_HASH, 1), + ) + server._begin_accepted_block_payout_preview( + ANCESTOR_HASH, block_height=10 + ) + server._mark_accepted_block_payout_landed(ANCESTOR_HASH, block_height=10) + server._retain_block_candidate_for_retry(ancestor) + self._defer_once(server) + with patch("builtins.print"): + self.assertEqual(server.replay_pending_block_candidates(), 0) + with server.lock: + self.assertNotIn( + ANCESTOR_HASH, server._block_replay_inflight_hashes + ) + self.assertEqual(ledger.pool_block_state_calls, []) + self.assertEqual(probe_calls, []) + with server._accepted_block_payout_preview_condition: + self.assertIn(ANCESTOR_HASH, server._accepted_block_payout_previews) + + +class AncestorRedriveBookkeepingTests(unittest.TestCase): + """The per-ancestor record's cap, streak, eviction, and counter rules.""" + + def _tracker(self, *, threshold: int, cap: int): + server, _rpc = light_coordinator() + server.accepted_parent_redrive_defer_threshold = threshold + server.accepted_parent_redrive_attempt_max = cap + return server, server._ensure_block_candidate_service() + + def test_each_attempt_needs_a_fresh_deferral_streak(self) -> None: + """Deferrals behind an armed request must not bank toward the next. + + Otherwise attempts 2..N arm on single deferrals right after + consumption and the cap exhausts before the first pass's adoption + had any chance to resolve the ancestor. + """ + server, service = self._tracker(threshold=3, cap=3) + with patch("builtins.print"): + for _cycle in range(3): + service.note_pending_parent_transition_deferral( + CHILD_HASH, ANCESTOR_HASH + ) + self.assertEqual(int(server.accepted_parent_redrive_attempt_count), 1) + # Deferrals while armed bank nothing. + for _cycle in range(4): + service.note_pending_parent_transition_deferral( + CHILD_HASH, ANCESTOR_HASH + ) + self.assertEqual( + service._consume_ancestor_redrive_requests(), (ANCESTOR_HASH,) + ) + # A fresh threshold of deferrals is required for attempt 2. + for _cycle in range(2): + service.note_pending_parent_transition_deferral( + CHILD_HASH, ANCESTOR_HASH + ) + self.assertEqual( + int(server.accepted_parent_redrive_attempt_count), 1 + ) + service.note_pending_parent_transition_deferral( + CHILD_HASH, ANCESTOR_HASH + ) + self.assertEqual(int(server.accepted_parent_redrive_attempt_count), 2) + + def test_zero_attempt_cap_disables_the_mechanism_silently(self) -> None: + """Cap 0 means off: no arming, no exhaustion signal, no bookkeeping. + + The exhausted counter is the alert-worthy pre-restart signal; an + installation that disabled the feature must not emit it. + """ + server, service = self._tracker(threshold=1, cap=0) + with patch("builtins.print"): + for _cycle in range(5): + service.note_pending_parent_transition_deferral( + CHILD_HASH, ANCESTOR_HASH + ) + self.assertFalse(service._ancestor_redrive_owed()) + self.assertEqual(int(server.accepted_parent_redrive_attempt_count), 0) + self.assertEqual(int(server.accepted_parent_redrive_exhausted_count), 0) + self.assertEqual(service._ancestor_redrive_records, {}) + self.assertEqual(service._ancestor_redrive_last_blocking, {}) + + def test_resolution_without_a_consumed_pass_is_not_counted(self) -> None: + """An armed-but-unconsumed request that resolves on its own is the + mechanism standing by, not succeeding; the resolved counter must not + inflate against attempts.""" + server, service = self._tracker(threshold=1, cap=3) + with patch("builtins.print"): + service.note_pending_parent_transition_deferral( + CHILD_HASH, ANCESTOR_HASH + ) + self.assertTrue(service._ancestor_redrive_owed()) + service.note_pending_parent_transition_resolved(CHILD_HASH) + self.assertEqual(int(server.accepted_parent_redrive_resolved_count), 0) + self.assertEqual(service._ancestor_redrive_records, {}) + self.assertEqual(service._ancestor_redrive_last_blocking, {}) + + def test_deferrals_while_the_ancestor_is_inflight_bank_nothing(self) -> None: + """An adopted ancestor is mid-resolution; a new pass would no-op. + + Burning armed attempts against an adoption already in progress + would exhaust the cap before slow-but-working replay finalization + completes; the streak must resume only once the flight ends. + """ + server, service = self._tracker(threshold=1, cap=3) + service._ensure_block_replay_state() + with server.lock: + server._block_replay_inflight_hashes.add(ANCESTOR_HASH) + with patch("builtins.print"): + for _cycle in range(3): + service.note_pending_parent_transition_deferral( + CHILD_HASH, ANCESTOR_HASH + ) + self.assertFalse(service._ancestor_redrive_owed()) + self.assertEqual(int(server.accepted_parent_redrive_attempt_count), 0) + with server.lock: + server._block_replay_inflight_hashes.discard(ANCESTOR_HASH) + with patch("builtins.print"): + service.note_pending_parent_transition_deferral( + CHILD_HASH, ANCESTOR_HASH + ) + self.assertTrue(service._ancestor_redrive_owed()) + self.assertEqual(int(server.accepted_parent_redrive_attempt_count), 1) + + def test_child_entry_eviction_never_cancels_an_ancestor_record(self) -> None: + """The two registries are different hash spaces; trimming one must + not reach into the other. + + Hash B is both a child (B deferred on A once) and an armed ancestor + (C's deferrals armed B's re-drive). Flooding the child-keyed index + past its bound evicts B's stale child entry; B's armed record must + survive that eviction untouched. + """ + server, service = self._tracker(threshold=3, cap=3) + ancestor_b = "bb" * 32 + with patch("builtins.print"): + service.note_pending_parent_transition_deferral( + ancestor_b, ANCESTOR_HASH + ) + for _cycle in range(3): + service.note_pending_parent_transition_deferral( + CHILD_HASH, ancestor_b + ) + self.assertEqual(int(server.accepted_parent_redrive_attempt_count), 1) + for index in range(70): + service.note_pending_parent_transition_deferral( + "%064x" % (0x1000 + index), ancestor_b + ) + with server.lock: + self.assertNotIn(ancestor_b, service._ancestor_redrive_last_blocking) + self.assertEqual( + service._consume_ancestor_redrive_requests(), (ancestor_b,) + ) + self.assertEqual(int(server.accepted_parent_redrive_attempt_count), 1) + + def test_eviction_never_drops_the_actively_wedged_ancestor(self) -> None: + """The lifetime cap must survive the pathological many-ancestor + stream the eviction bound exists for. + + An exhausted ancestor that keeps deferring re-inserts its whole + record every deferral, so a storm of distinct arming ancestors must + never evict it -- eviction would discard the exhaustion latch and + re-arm re-drives past the cap. + """ + server, service = self._tracker(threshold=1, cap=1) + with patch("builtins.print"): + service.note_pending_parent_transition_deferral( + CHILD_HASH, ANCESTOR_HASH + ) + self.assertEqual( + service._consume_ancestor_redrive_requests(), (ANCESTOR_HASH,) + ) + service.note_pending_parent_transition_deferral( + CHILD_HASH, ANCESTOR_HASH + ) + self.assertEqual( + int(server.accepted_parent_redrive_exhausted_count), 1 + ) + for index in range(70): + service.note_pending_parent_transition_deferral( + "%064x" % (0x2000 + index), + "%064x" % (0x3000 + index), + ) + # The wedge keeps deferring throughout the storm. + service.note_pending_parent_transition_deferral( + CHILD_HASH, ANCESTOR_HASH + ) + attempts_after_storm = int(server.accepted_parent_redrive_attempt_count) + service.note_pending_parent_transition_deferral( + CHILD_HASH, ANCESTOR_HASH + ) + self.assertEqual( + int(server.accepted_parent_redrive_attempt_count), + attempts_after_storm, + ) + self.assertEqual(int(server.accepted_parent_redrive_exhausted_count), 1) + with server.lock: + record = service._ancestor_redrive_records.get(ANCESTOR_HASH) + self.assertIsNotNone(record) + self.assertTrue(record.exhausted) + + +class AncestorRedriveRunLoopTests(unittest.TestCase): + """The submitter loop's fall-through: an armed re-drive must reach the + replay entrypoint even when immediate live work keeps succeeding -- + sustained live traffic taking the ``continue`` every pass is exactly how + the incident starved the one path that resolves a stuck ancestor.""" + + def _run_one_iteration(self, *, arm_redrive: bool): + server, _rpc = light_coordinator() + server.accepted_parent_redrive_defer_threshold = 1 + server.accepted_parent_redrive_attempt_max = 3 + service = server._ensure_block_candidate_service() + if arm_redrive: + with patch("builtins.print"): + service.note_pending_parent_transition_deferral( + CHILD_HASH, ANCESTOR_HASH + ) + self.assertTrue(service._ancestor_redrive_owed()) + server._start_block_accounting_thread = lambda: None + submits: list[float | None] = [] + + def submit_next(timeout=None, *, defer_accounting=False): + submits.append(timeout) + return True + + server.submit_next_block_candidate = submit_next + replays: list[bool] = [] + stop_states = iter([False, True]) + stop = SimpleNamespace(is_set=lambda: next(stop_states)) + service.ports = dataclasses.replace( + service.ports, + stop_event=lambda: stop, + replay_entrypoint=lambda: (replays.append(True), 0)[1], + ) + # Immediate work is available and keeps succeeding. + service.retry_candidate = object() + with patch("builtins.print"): + service.run() + return replays, submits + + def test_armed_redrive_falls_through_to_the_replay_entrypoint(self) -> None: + replays, submits = self._run_one_iteration(arm_redrive=True) + self.assertEqual(replays, [True]) + self.assertEqual(submits, [None, 1.0]) + + def test_without_a_redrive_immediate_success_short_circuits(self) -> None: + replays, submits = self._run_one_iteration(arm_redrive=False) + self.assertEqual(replays, []) + self.assertEqual(submits, [None]) + + +class AncestorRedriveForcedPassTests(unittest.TestCase): + """A forced re-drive pass runs the startup enumeration's code: keyset + pagination when the ledger supports it, on the landing-class budget.""" + + def test_forced_pass_paginates_on_the_landing_budget(self) -> None: + server, _state, _recording = submit_coordinator() + ledger = SweepLedger(None) + server.ledger = ledger + server.accepted_parent_redrive_defer_threshold = 1 + _script_chain_probe(server, True) + server._begin_accepted_block_payout_preview( + ANCESTOR_HASH, block_height=10 + ) + server._mark_accepted_block_payout_landed(ANCESTOR_HASH, block_height=10) + original_call = server._run_block_submitter_ledger_call + outbox_queries: list[tuple[object, object, object]] = [] + + def spying_call(key, label, fn, **kwargs): + if label == "replay-outbox-query": + outbox_queries.append( + (key, kwargs.get("call_class"), kwargs.get("timeout_seconds")) + ) + return original_call(key, label, fn, **kwargs) + + server._run_block_submitter_ledger_call = spying_call + with patch("builtins.print"): + self.assertTrue( + server._defer_for_pending_parent_payout_transition( + block_hash=CHILD_HASH, + parent_hash=ANCESTOR_HASH, + parent_height=10, + worker=None, + ) + ) + self.assertEqual(server.replay_pending_block_candidates(), 0) + self.assertEqual(len(outbox_queries), 1) + key, call_class, timeout_seconds = outbox_queries[0] + # The paginated fetch keys its dedupe on (label, limit, page, + # cursor); the legacy widening window keys on (label, limit) alone. + self.assertEqual(len(key), 4) + self.assertEqual(call_class, "landing") + self.assertIsNotNone(timeout_seconds) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_prism_audit_api.py b/tests/test_prism_audit_api.py index 1c63d4535..c8b24ca01 100644 --- a/tests/test_prism_audit_api.py +++ b/tests/test_prism_audit_api.py @@ -3,13 +3,15 @@ from __future__ import annotations import json +import tempfile import threading import unittest import urllib.error import urllib.request from http.server import ThreadingHTTPServer +from pathlib import Path -from lab.prism.prism_coordinator import make_audit_handler +from lab.prism.prism_coordinator import PrismCoordinator, make_audit_handler class FakeLedger: @@ -225,6 +227,22 @@ def cached_health_payload(self) -> tuple[int, dict[str, object]]: } +class CachedMetricsCoordinator(FakeCoordinator): + def cached_metrics_payload(self) -> tuple[int, str]: + return 200, "qbit_prism_cached_snapshot 1\n" + + def metrics_payload(self) -> str: + raise AssertionError("HTTP scrape reached the live metrics renderer") + + +class UninitializedCachedMetricsCoordinator(FakeCoordinator): + def cached_metrics_payload(self) -> tuple[int, str]: + return 503, "qbit_prism_metrics_snapshot_age_seconds -1\n" + + def metrics_payload(self) -> str: + return "qbit_prism_direct_compatibility 1\n" + + class PrismAuditApiTests(unittest.TestCase): def setUp(self) -> None: handler = make_audit_handler(FakeCoordinator()) # type: ignore[arg-type] @@ -272,6 +290,89 @@ def test_health_latest_owed_and_metrics_endpoints(self) -> None: self.assertIn("qbit_prism_ctv_fanouts_broadcastable 1", metrics) self.assertIn("qbit_prism_ctv_fanouts_failed 0", metrics) + def test_metrics_endpoint_reads_only_the_cached_complete_snapshot(self) -> None: + handler = make_audit_handler(CachedMetricsCoordinator()) # type: ignore[arg-type] + server = ThreadingHTTPServer(("127.0.0.1", 0), handler) + thread = threading.Thread(target=server.serve_forever, daemon=True) + thread.start() + try: + with urllib.request.urlopen( + f"http://127.0.0.1:{server.server_port}/metrics", + timeout=5, + ) as response: + self.assertEqual(response.status, 200) + self.assertEqual( + response.read().decode(), + "qbit_prism_cached_snapshot 1\n", + ) + finally: + server.shutdown() + server.server_close() + thread.join(timeout=5) + + def test_metrics_endpoint_falls_back_when_compatibility_cache_is_uninitialized( + self, + ) -> None: + handler = make_audit_handler( # type: ignore[arg-type] + UninitializedCachedMetricsCoordinator() + ) + server = ThreadingHTTPServer(("127.0.0.1", 0), handler) + thread = threading.Thread(target=server.serve_forever, daemon=True) + thread.start() + try: + with urllib.request.urlopen( + f"http://127.0.0.1:{server.server_port}/metrics", + timeout=5, + ) as response: + self.assertEqual(response.status, 200) + self.assertEqual( + response.read().decode(), + "qbit_prism_direct_compatibility 1\n", + ) + finally: + server.shutdown() + server.server_close() + thread.join(timeout=5) + + def test_coordinator_latest_audit_endpoint_preserves_none_and_snapshot_payload(self) -> None: + coordinator = PrismCoordinator.__new__(PrismCoordinator) + coordinator.latest_evidence = None + with tempfile.TemporaryDirectory() as audit_root: + coordinator.audit_dir = Path(audit_root) / "artifacts" + coordinator.evidence_path = Path(audit_root) / "evidence.json" + handler = make_audit_handler(coordinator) + server = ThreadingHTTPServer(("127.0.0.1", 0), handler) + thread = threading.Thread(target=server.serve_forever, daemon=True) + thread.start() + base_url = f"http://127.0.0.1:{server.server_port}" + try: + with self.assertRaises(urllib.error.HTTPError) as raised: + urllib.request.urlopen(base_url + "/audit/latest", timeout=5) + self.assertEqual(raised.exception.code, 404) + raised.exception.close() + + seed = {"schema": "test", "nested": {"value": 1}} + coordinator.latest_evidence = seed + seed["nested"]["value"] = 2 + with urllib.request.urlopen( + base_url + "/audit/latest", + timeout=5, + ) as response: + payload = json.loads(response.read()) + self.assertEqual(payload, {"schema": "test", "nested": {"value": 1}}) + payload["nested"]["value"] = 3 + self.assertEqual( + coordinator.latest_evidence_payload(), + {"schema": "test", "nested": {"value": 1}}, + ) + finally: + server.shutdown() + server.server_close() + thread.join(timeout=5) + store = coordinator.__dict__.get("_audit_artifact_store") + if store is not None: + store.close() + def test_unhealthy_progress_returns_http_503_and_bounded_reason(self) -> None: handler = make_audit_handler(UnhealthyCoordinator()) # type: ignore[arg-type] server = ThreadingHTTPServer(("127.0.0.1", 0), handler) diff --git a/tests/test_prism_audit_artifacts.py b/tests/test_prism_audit_artifacts.py new file mode 100644 index 000000000..72cf2a5f0 --- /dev/null +++ b/tests/test_prism_audit_artifacts.py @@ -0,0 +1,4564 @@ +#!/usr/bin/env python3 + +import gzip +import hashlib +import json +import os +from dataclasses import replace as dataclass_replace +from pathlib import Path +import select +import subprocess +import sys +import tempfile +import threading +import time +import unittest +from typing import Any +from unittest import mock + +from lab.prism.audit_artifacts import ( + AuditArtifactConfig, + AuditArtifactStore, + AuditPublicationIdentity, + CanonicalAuditBundleCorrupt, + LIVE_ENVELOPE_SCHEMA, + LIVE_EVIDENCE_SCHEMA, + OwnedCandidateArtifact, + _AuditShareSegmentConflict, + _FileIdentity, + canonical_audit_bundle_bytes, + gzip_canonical_bundle_bytes, +) +from lab.prism.bundle_compiler import canonical_bundle_bytes +from lab.prism.share_ledger import SingleWriterShareLedger + + +BLOCK_A = "aa" * 32 +BLOCK_B = "bb" * 32 +DIGEST = "11" * 32 + + +class _TestAuditArtifactStore(AuditArtifactStore): + """Keep publication fixtures terse while supplying a real verifier identity.""" + + def publish_success(self, **kwargs: object): # type: ignore[no-untyped-def] + report = kwargs.get("report") + identity = kwargs.get("identity") + if "publication_floor_sequence" not in kwargs: + if not isinstance(identity, AuditPublicationIdentity): + raise AssertionError("publication test identity is required") + kwargs["publication_floor_sequence"] = identity.sequence + if "verification_identity" not in kwargs: + if not isinstance(report, dict): + raise AssertionError("publication test report is required") + kwargs["verification_identity"] = self.build_verification_identity( + trust_source="configured", + trusted_writer_public_key_hex="44" * 32, + literal_sha256=DIGEST, + literal_byte_len=123, + report=report, + ) + with self.publication_order_guard(): + return super().publish_success(**kwargs) # type: ignore[arg-type] + + +class AuditArtifactStoreTest(unittest.TestCase): + def make_store(self, root: Path, **kwargs: object) -> AuditArtifactStore: + evidence_path = Path(kwargs.pop("evidence_path", root / "evidence.json")) + return _TestAuditArtifactStore( + AuditArtifactConfig( + root=root, + evidence_path=evidence_path, + live_bundle_retention=int(kwargs.pop("live_bundle_retention", 5)), + candidate_retention_seconds=int( + kwargs.pop("candidate_retention_seconds", 86_400) + ), + share_segment_size=int(kwargs.pop("share_segment_size", 0)), + verifier_timeout_seconds=float( + kwargs.pop("verifier_timeout_seconds", 60.0) + ), + ), + **kwargs, + ) + + @staticmethod + def transfer_candidate( + store: AuditArtifactStore, + candidate: OwnedCandidateArtifact, + ) -> None: + path = candidate.path + with path.open("rb") as handle: + store.adopt_compiler_candidate( + candidate, + path=path, + value=os.fstat(handle.fileno()), + ) + + @staticmethod + def report( + digest: str = DIGEST, + *, + block_height: int = 1, + ) -> dict[str, object]: + return { + "schema": "qbit.prism.audit-verification-report.v1", + "block_height": block_height, + "audit_bundle_sha256_hex": digest, + "reward_manifest_sha256_hex": "44" * 32, + "payout_policy_manifest_sha256_hex": "55" * 32, + "prism_audit_commitment_leaf_hex": "66" * 32, + "audit_commitment_root_hex": "77" * 32, + "coinbase_txid": "22" * 32, + "coinbase_wtxid": "88" * 32, + "coinbase_manifest_sha256_hex": "33" * 32, + "coinbase_tx_hex": "00", + "coinbase_value_sats": 1, + "min_output_sats": 1, + "onchain_output_count": 0, + "accrued_account_count": 0, + } + + @staticmethod + def persistence(digest: str = DIGEST) -> dict[str, object]: + return {"audit_bundle_sha256": digest, "body_uri": ""} + + def test_paths_reject_untrusted_hashes_and_stay_in_resolved_root(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) / "audit" + store = self.make_store(root) + candidate = store.issue_candidate(block_hash=BLOCK_A) + self.assertEqual(candidate.path.parent, root.resolve()) + self.assertEqual( + store.live_envelope_path(block_height=0, block_hash=BLOCK_A).parent, + root.resolve(), + ) + for invalid in ("a", "gg" * 32, "../" + BLOCK_A, BLOCK_A + "/x"): + with self.subTest(invalid=invalid), self.assertRaises(ValueError): + store.issue_candidate(block_hash=invalid) + + def test_publication_identity_and_live_path_reject_lossy_types(self) -> None: + invalid_identities = ( + (True, 1, BLOCK_A), + ("1", 1, BLOCK_A), + (-1, 1, BLOCK_A), + (1, False, BLOCK_A), + (1, "1", BLOCK_A), + (1, -1, BLOCK_A), + (1, 1, BLOCK_A.upper()), + ) + for sequence, height, block_hash in invalid_identities: + with self.subTest( + sequence=sequence, + height=height, + block_hash=block_hash, + ), self.assertRaises(ValueError): + AuditPublicationIdentity(sequence, height, block_hash) # type: ignore[arg-type] + with tempfile.TemporaryDirectory() as tmp: + store = self.make_store(Path(tmp)) + for height in (True, "1"): + with self.subTest(path_height=height), self.assertRaises(ValueError): + store.live_envelope_path( # type: ignore[arg-type] + block_height=height, + block_hash=BLOCK_A, + ) + identity = AuditPublicationIdentity(1, 1, BLOCK_A) + for floor in (True, "1", -1): + with self.subTest(publication_floor=floor), self.assertRaises( + ValueError + ): + store.publish_success( + identity=identity, + publication_floor_sequence=floor, # type: ignore[arg-type] + report=self.report(), + persistence=self.persistence(), + evidence={}, + created_at="now", + ) + with self.assertRaisesRegex(RuntimeError, "exceeds"): + store.publish_success( + identity=identity, + publication_floor_sequence=0, + report=self.report(), + persistence=self.persistence(), + evidence={}, + created_at="now", + ) + with self.assertRaisesRegex(RuntimeError, "behind"): + store.publish_success( + identity=identity, + publication_floor_sequence=2, + report=self.report(), + persistence=self.persistence(), + evidence={}, + created_at="now", + ) + + def test_live_publication_canonicalizes_persistence_digest_for_restart(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + store = self.make_store(root) + publication = store.publish_success( + identity=AuditPublicationIdentity(1, 1, BLOCK_A), + report=self.report(), + persistence=self.persistence(DIGEST.upper()), + evidence={}, + created_at="now", + ) + self.assertEqual( + publication.evidence["persistence"]["audit_bundle_sha256"], # type: ignore[index] + DIGEST, + ) + store.close() + restarted = self.make_store(root) + latest = restarted.latest_evidence() + self.assertIsNotNone(latest) + assert latest is not None + self.assertEqual(latest["persistence"]["audit_bundle_sha256"], DIGEST) + + def test_root_final_symlink_is_rejected_and_ancestor_is_resolved(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + base = Path(tmp) + real = base / "real" + real.mkdir() + ancestor = base / "ancestor" + ancestor.symlink_to(real, target_is_directory=True) + store = self.make_store(ancestor / "audit") + self.assertEqual(store.root, (real / "audit").resolve()) + target = base / "target" + target.mkdir() + final_link = base / "final-link" + final_link.symlink_to(target, target_is_directory=True) + with self.assertRaisesRegex(RuntimeError, "non-symlink"): + self.make_store(final_link) + + def test_root_directory_authority_swap_matrix_fails_closed_and_recovers(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + base = Path(tmp) + root = base / "audit" + evidence_path = base / "state" / "evidence.json" + store = self.make_store(root, evidence_path=evidence_path) + store.publish_success( + identity=AuditPublicationIdentity(1, 1, BLOCK_A), + report=self.report(), + persistence=self.persistence(), + evidence={}, + created_at="now", + ) + candidate = store.issue_candidate(block_hash=BLOCK_B) + store.write_compatibility_candidate(candidate, {"candidate": True}) + body = store.body_path(BLOCK_A, DIGEST) + store._write_immutable_bytes(body, b"body") + + pinned_root = base / "audit-pinned" + root.rename(pinned_root) + root.mkdir() + sentinel = root / "operator-sentinel" + sentinel.write_bytes(b"operator") + + self.assertIsNone(store.latest_evidence()) + self.assertEqual(store.publication_sequence_floor(), 0) + self.assertEqual(store.metrics_snapshot()["scan_error"], 1) + self.assertEqual(store.prune_best_effort().errors, 1) + with self.assertRaisesRegex(RuntimeError, "root identity"): + store._write_immutable_bytes(body, b"replacement") + with self.assertRaisesRegex(RuntimeError, "root identity"): + store._read_owned_regular_bytes(body) + with self.assertRaisesRegex(RuntimeError, "root identity"): + store.discard_candidate(candidate) + with self.assertRaisesRegex(RuntimeError, "root identity"): + store.publish_success( + identity=AuditPublicationIdentity(2, 2, BLOCK_B), + report=self.report(block_height=2), + persistence=self.persistence(), + evidence={}, + created_at="blocked", + ) + self.assertEqual(sentinel.read_bytes(), b"operator") + self.assertEqual(sorted(path.name for path in root.iterdir()), [sentinel.name]) + + replacement = base / "audit-replacement" + root.rename(replacement) + pinned_root.rename(root) + self.assertEqual(store.publication_sequence_floor(), 1) + self.assertEqual(store.latest_evidence()["block_hash"], BLOCK_A) # type: ignore[index] + self.assertEqual(store._read_owned_regular_bytes(body)[0], b"body") + store.discard_candidate(candidate) + self.assertFalse(candidate.path.exists()) + self.assertEqual((replacement / sentinel.name).read_bytes(), b"operator") + + def test_external_evidence_parent_swap_rolls_back_publication_and_recovers(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + base = Path(tmp) + root = base / "audit" + state = base / "state" + evidence_path = state / "evidence.json" + store = self.make_store(root, evidence_path=evidence_path) + store.publish_success( + identity=AuditPublicationIdentity(1, 1, BLOCK_A), + report=self.report(), + persistence=self.persistence(), + evidence={}, + created_at="now", + ) + original_write = store._write_mutable_json + pinned_state = base / "state-pinned" + replacement_state = base / "state-replacement" + + def swap_before_evidence( + path: Path, + *args: object, + **kwargs: object, + ) -> object: + if path == store.evidence_path: + state.rename(pinned_state) + state.mkdir() + (state / "operator-sentinel").write_bytes(b"operator") + return original_write(path, *args, **kwargs) + + new_envelope = store.live_envelope_path( + block_height=2, + block_hash=BLOCK_B, + ) + with mock.patch.object( + store, + "_write_mutable_json", + side_effect=swap_before_evidence, + ), self.assertRaisesRegex(RuntimeError, "evidence parent identity"): + store.publish_success( + identity=AuditPublicationIdentity(2, 2, BLOCK_B), + report=self.report(block_height=2), + persistence=self.persistence(), + evidence={}, + created_at="blocked", + ) + self.assertFalse(new_envelope.exists()) + self.assertEqual( + (state / "operator-sentinel").read_bytes(), + b"operator", + ) + self.assertIsNone(store.latest_evidence()) + self.assertEqual(store.publication_sequence_floor(), 0) + + state.rename(replacement_state) + pinned_state.rename(state) + self.assertEqual(store.publication_sequence_floor(), 1) + self.assertEqual(store.latest_evidence()["block_hash"], BLOCK_A) # type: ignore[index] + + def test_live_prune_rechecks_evidence_authority_before_removal(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + base = Path(tmp) + root = base / "audit" + state = base / "state" + pinned_state = base / "state-pinned" + store = self.make_store( + root, + evidence_path=state / "evidence.json", + live_bundle_retention=0, + ) + store.publish_success( + identity=AuditPublicationIdentity(1, 1, BLOCK_A), + report=self.report(), + persistence=self.persistence(), + evidence={}, + created_at="now", + ) + stale = store.live_envelope_path(block_height=2, block_hash=BLOCK_B) + stale.write_text("stale", encoding="utf-8") + real_unlink = store._unlink_scanned_owned + swapped = False + + def swap_after_scan( + path: Path, + identity: _FileIdentity, + *, + require_all_authorities: bool = False, + ) -> bool: + nonlocal swapped + if require_all_authorities and not swapped: + swapped = True + state.rename(pinned_state) + state.mkdir() + (state / "operator-sentinel").write_bytes(b"operator") + return real_unlink( + path, + identity, + require_all_authorities=require_all_authorities, + ) + + with mock.patch.object( + store, + "_unlink_scanned_owned", + side_effect=swap_after_scan, + ): + result = store.prune_best_effort() + self.assertTrue(swapped) + self.assertEqual(result.live_removed, 0) + self.assertGreaterEqual(result.errors, 1) + self.assertTrue(stale.exists()) + self.assertEqual( + (state / "operator-sentinel").read_bytes(), + b"operator", + ) + + def test_candidate_prune_rechecks_reservations_after_active_snapshot(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + store = self.make_store( + Path(tmp), + candidate_retention_seconds=0, + ) + scan_waiting = threading.Event() + release_scan = threading.Event() + result: list[object] = [] + errors: list[BaseException] = [] + real_listdir = os.listdir + + def pause_before_scan(fd: int) -> list[str]: + # Scans enumerate through a fresh directory description of + # the pinned root inode (issue #255), never the pinned + # descriptor itself; recognize the scan by that identity. + value = os.fstat(fd) + if (value.st_dev, value.st_ino) == store._root_identity: + scan_waiting.set() + if not release_scan.wait(5): + raise AssertionError("timed out waiting to release prune scan") + return real_listdir(fd) + + def prune() -> None: + try: + result.append(store.prune_best_effort()) + except BaseException as exc: # pragma: no cover - asserted below + errors.append(exc) + + with mock.patch( + "lab.prism.audit_artifacts.os.listdir", + side_effect=pause_before_scan, + ): + thread = threading.Thread(target=prune) + thread.start() + self.assertTrue(scan_waiting.wait(5)) + candidate = store.issue_candidate(block_hash=BLOCK_A) + store.write_compatibility_candidate(candidate, {"reserved": True}) + expected = candidate.path.read_bytes() + release_scan.set() + thread.join(timeout=5) + + self.assertFalse(errors) + self.assertFalse(thread.is_alive()) + self.assertEqual(len(result), 1) + self.assertEqual(result[0].candidate_removed, 0) # type: ignore[union-attr] + self.assertEqual(candidate.path.read_bytes(), expected) + store.discard_candidate(candidate) + + def test_observer_scans_fail_closed_on_midscan_authority_change(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + base = Path(tmp) + root = base / "audit" + store = self.make_store( + root, + candidate_retention_seconds=0, + live_bundle_retention=0, + ) + candidate = root / f"prism-live-audit-bundle-candidate-{BLOCK_A}.json" + candidate.write_bytes(b"candidate") + real_lstat = store._owned_lstat + + def run_with_swap(operation: object, label: str) -> object: + pinned = base / f"{label}-pinned" + replacement = base / f"{label}-replacement" + swapped = False + + def swap_then_reject(path: Path) -> os.stat_result: + nonlocal swapped + if not swapped: + swapped = True + root.rename(pinned) + root.mkdir() + (root / "operator-sentinel").write_bytes(b"operator") + raise RuntimeError("authority changed") + return real_lstat(path) + + with mock.patch.object( + store, + "_owned_lstat", + side_effect=swap_then_reject, + ): + result = operation() # type: ignore[operator] + self.assertTrue(swapped) + self.assertEqual( + (root / "operator-sentinel").read_bytes(), + b"operator", + ) + root.rename(replacement) + pinned.rename(root) + return result + + metrics = run_with_swap(store.metrics_snapshot, "metrics") + self.assertEqual(metrics["scan_error"], 1) # type: ignore[index] + self.assertTrue(candidate.exists()) + retained = run_with_swap(store.prune_best_effort, "prune") + self.assertGreaterEqual(retained.errors, 1) # type: ignore[union-attr] + self.assertEqual(retained.candidate_removed, 0) # type: ignore[union-attr] + self.assertTrue(candidate.exists()) + + def test_directory_authority_reconfigure_and_close_are_atomic_and_fd_clean(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + base = Path(tmp) + old_root = base / "old-root" + old_evidence = base / "old-state" / "evidence.json" + store = self.make_store(old_root, evidence_path=old_evidence) + old_root_fd = store._root_fd + old_publication_lock_fd = store._publication_lock_fd + old_evidence_fd = store._evidence_parent_fd + real_open = store._open_directory_authority + calls = 0 + prepared_root_fd: int | None = None + + def fail_second(path: Path) -> tuple[int, tuple[int, int]]: + nonlocal calls, prepared_root_fd + calls += 1 + if calls == 2: + raise OSError("second authority") + result = real_open(path) + prepared_root_fd = result[0] + return result + + new_root = base / "new-root" + new_evidence = base / "new-state" / "evidence.json" + with mock.patch.object( + store, + "_open_directory_authority", + side_effect=fail_second, + ), self.assertRaisesRegex(OSError, "second authority"): + store.reconfigure(root=new_root, evidence_path=new_evidence) + self.assertEqual(store.root, old_root.resolve()) + self.assertEqual(store.evidence_path, old_evidence.resolve()) + os.fstat(old_root_fd) + os.fstat(old_evidence_fd) + self.assertIsNotNone(prepared_root_fd) + assert prepared_root_fd is not None + with self.assertRaises(OSError): + os.fstat(prepared_root_fd) + + real_os_close = os.close + old_fds = { + old_root_fd, + old_publication_lock_fd, + old_evidence_fd, + } + + def close_then_report_error(fd: int) -> None: + real_os_close(fd) + if fd in old_fds: + raise OSError("close status unavailable") + + with mock.patch( + "lab.prism.audit_artifacts.os.close", + side_effect=close_then_report_error, + ): + store.reconfigure(root=new_root, evidence_path=new_evidence) + new_root_fd = store._root_fd + new_publication_lock_fd = store._publication_lock_fd + new_evidence_fd = store._evidence_parent_fd + for closed_fd in ( + old_root_fd, + old_publication_lock_fd, + old_evidence_fd, + ): + with self.assertRaises(OSError): + os.fstat(closed_fd) + store.close() + store.close() + self.assertIsNone(store.latest_evidence()) + self.assertEqual(store.publication_sequence_floor(), 0) + metrics = store.metrics_snapshot() + self.assertEqual(metrics["scan_error"], 1) + for kind in ( + "body", + "share_segment", + "live_bundle", + "candidate", + "other", + ): + self.assertEqual(metrics[kind], {"files": 0, "bytes": 0}) + retention = store.prune_best_effort() + self.assertEqual(retention.live_removed, 0) + self.assertEqual(retention.candidate_removed, 0) + self.assertEqual(retention.errors, 1) + with self.assertRaisesRegex(RuntimeError, "closed"): + store.issue_candidate(block_hash=BLOCK_A) + for closed_fd in ( + new_root_fd, + new_publication_lock_fd, + new_evidence_fd, + ): + with self.assertRaises(OSError): + os.fstat(closed_fd) + + def test_reconfigure_hides_transient_new_authority_state_from_readers(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + base = Path(tmp) + old_store = self.make_store( + base / "old-root", + evidence_path=base / "old-state" / "evidence.json", + ) + old_store.publish_success( + identity=AuditPublicationIdentity(1, 1, BLOCK_A), + report=self.report(), + persistence=self.persistence(), + evidence={}, + created_at="old", + ) + new_root = base / "new-root" + new_evidence = base / "new-state" / "evidence.json" + prepared = self.make_store(new_root, evidence_path=new_evidence) + prepared.publish_success( + identity=AuditPublicationIdentity(2, 2, BLOCK_B), + report=self.report(block_height=2), + persistence=self.persistence(), + evidence={}, + created_at="new", + ) + prepared.close() + + real_load = old_store._load_current_evidence_locked + load_entered = threading.Event() + release_load = threading.Event() + reader_started = threading.Event() + reader_values: list[dict[str, object] | None] = [] + errors: list[BaseException] = [] + + def blocked_load() -> None: + load_entered.set() + if not release_load.wait(5): + raise AssertionError("timed out waiting to release evidence load") + real_load() + + def reconfigure() -> None: + try: + old_store.reconfigure( + root=new_root, + evidence_path=new_evidence, + ) + except BaseException as exc: # pragma: no cover - asserted below + errors.append(exc) + + def read_latest() -> None: + reader_started.set() + try: + reader_values.append(old_store.latest_evidence()) + except BaseException as exc: # pragma: no cover - asserted below + errors.append(exc) + + with mock.patch.object( + old_store, + "_load_current_evidence_locked", + side_effect=blocked_load, + ): + reconfigure_thread = threading.Thread(target=reconfigure) + reconfigure_thread.start() + self.assertTrue(load_entered.wait(5)) + reader_thread = threading.Thread(target=read_latest) + reader_thread.start() + self.assertTrue(reader_started.wait(5)) + reader_thread.join(timeout=0.05) + self.assertTrue(reader_thread.is_alive()) + release_load.set() + reconfigure_thread.join(timeout=5) + reader_thread.join(timeout=5) + + self.assertFalse(errors) + self.assertFalse(reconfigure_thread.is_alive()) + self.assertFalse(reader_thread.is_alive()) + self.assertEqual(len(reader_values), 1) + assert reader_values[0] is not None + self.assertEqual(reader_values[0]["block_hash"], BLOCK_B) + self.assertEqual(old_store.publication_sequence_floor(), 2) + + def test_publication_guard_is_required_and_internal_file_is_hidden(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + store = AuditArtifactStore( + AuditArtifactConfig( + root=root, + evidence_path=root / "evidence.json", + ), + canonicalizer=canonical_bundle_bytes, + ) + identity = AuditPublicationIdentity(1, 1, BLOCK_A) + with self.assertRaisesRegex(RuntimeError, "guard is required"): + store.publish_success( + identity=identity, + publication_floor_sequence=1, + report=self.report(), + persistence=self.persistence(), + evidence={}, + verification_identity={}, + created_at="now", + ) + with self.assertRaisesRegex(RuntimeError, "guard is required"): + store.adopt_legacy_publication_identity( + identity, + publication_floor_sequence=1, + ) + self.assertFalse(store.evidence_path.exists()) + metrics = store.metrics_snapshot() + self.assertEqual(metrics["scan_error"], 0) + for kind in ( + "body", + "share_segment", + "live_bundle", + "candidate", + "other", + ): + self.assertEqual(metrics[kind], {"files": 0, "bytes": 0}) + + lock_path = root / ".prism-audit-publication.lock" + parked = root / ".prism-audit-publication.lock.parked" + lock_path.rename(parked) + lock_path.write_bytes(b"replacement") + with self.assertRaisesRegex(RuntimeError, "lock identity changed"): + with store.publication_order_guard(): + pass + self.assertEqual(store.prune_best_effort().errors, 1) + lock_path.unlink() + parked.rename(lock_path) + with store.publication_order_guard(): + with store.publication_order_guard(): + pass + old_authority = ( + store.root, + store.evidence_path, + store._root_fd, + store._publication_lock_fd, + store._evidence_parent_fd, + ) + next_root = root / "nested-reconfigure" + with self.assertRaisesRegex(RuntimeError, "inside publication guard"): + store.reconfigure( + root=next_root, + evidence_path=next_root / "evidence.json", + ) + self.assertEqual( + ( + store.root, + store.evidence_path, + store._root_fd, + store._publication_lock_fd, + store._evidence_parent_fd, + ), + old_authority, + ) + self.assertFalse(next_root.exists()) + for fd in old_authority[2:]: + os.fstat(fd) + + def test_close_waits_for_guard_and_is_rejected_inside_owned_guard(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + store = self.make_store(root) + guard_entered = threading.Event() + release_guard = threading.Event() + close_attempting = threading.Event() + close_finished = threading.Event() + + def hold_guard() -> None: + with store.publication_order_guard(): + guard_entered.set() + release_guard.wait(5) + + def close_store() -> None: + close_attempting.set() + store.close() + close_finished.set() + + guard_thread = threading.Thread(target=hold_guard) + guard_thread.start() + self.assertTrue(guard_entered.wait(5)) + close_thread = threading.Thread(target=close_store) + close_thread.start() + self.assertTrue(close_attempting.wait(5)) + self.assertFalse(close_finished.wait(0.05)) + release_guard.set() + guard_thread.join(timeout=5) + close_thread.join(timeout=5) + self.assertFalse(guard_thread.is_alive()) + self.assertFalse(close_thread.is_alive()) + self.assertTrue(close_finished.is_set()) + with self.assertRaisesRegex(RuntimeError, "closed"): + with store.publication_order_guard(): + pass + + second = self.make_store(root / "second") + old_fds = ( + second._root_fd, + second._publication_lock_fd, + second._evidence_parent_fd, + ) + with second.publication_order_guard(): + with self.assertRaisesRegex(RuntimeError, "inside publication guard"): + second.close() + for fd in old_fds: + os.fstat(fd) + second.close() + + def test_publication_lock_symlink_is_rejected_on_open_and_reconfigure(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + base = Path(tmp) + outside = base / "outside" + outside.write_bytes(b"outside") + bad_root = base / "bad-root" + bad_root.mkdir() + (bad_root / ".prism-audit-publication.lock").symlink_to(outside) + with self.assertRaisesRegex(RuntimeError, "cannot be opened safely"): + AuditArtifactStore( + AuditArtifactConfig( + root=bad_root, + evidence_path=bad_root / "evidence.json", + ) + ) + + good_root = base / "good-root" + store = self.make_store(good_root) + old_state = ( + store.root, + store._root_fd, + store._publication_lock_fd, + store._evidence_parent_fd, + ) + with self.assertRaisesRegex(RuntimeError, "cannot be opened safely"): + store.reconfigure(root=bad_root) + self.assertEqual(store.root, old_state[0]) + for fd in old_state[1:]: + os.fstat(fd) + self.assertEqual(outside.read_bytes(), b"outside") + + def test_two_store_and_subprocess_publication_guards_serialize(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + first = self.make_store(root) + second = self.make_store(root) + attempting = threading.Event() + entered = threading.Event() + release = threading.Event() + + def hold_second() -> None: + attempting.set() + with second.publication_order_guard(): + entered.set() + release.wait(5) + + with first.publication_order_guard(): + thread = threading.Thread(target=hold_second) + thread.start() + self.assertTrue(attempting.wait(5)) + self.assertFalse(entered.wait(0.05)) + with self.assertRaisesRegex(RuntimeError, "another store"): + with second.publication_order_guard(): + pass + self.assertTrue(entered.wait(5)) + release.set() + thread.join(timeout=5) + self.assertFalse(thread.is_alive()) + + script = """ +import sys +from pathlib import Path +from lab.prism.audit_artifacts import AuditArtifactConfig, AuditArtifactStore +root = Path(sys.argv[1]) +store = AuditArtifactStore(AuditArtifactConfig(root=root, evidence_path=root / 'evidence.json')) +with store.publication_order_guard(): + print('locked', flush=True) + sys.stdin.readline() +""" + child = subprocess.Popen( + [sys.executable, "-c", script, str(root)], + cwd=Path(__file__).resolve().parents[1], + stdin=subprocess.PIPE, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, + ) + try: + assert child.stdin is not None + assert child.stdout is not None + assert child.stderr is not None + ready, _writable, _exceptional = select.select( + [child.stdout], + [], + [], + 5, + ) + self.assertEqual(ready, [child.stdout], "child guard handshake timed out") + self.assertEqual(child.stdout.readline().strip(), "locked") + parent_attempting = threading.Event() + parent_entered = threading.Event() + + def enter_parent() -> None: + parent_attempting.set() + with first.publication_order_guard(): + parent_entered.set() + + parent_thread = threading.Thread(target=enter_parent) + parent_thread.start() + self.assertTrue(parent_attempting.wait(5)) + self.assertFalse(parent_entered.wait(0.05)) + child.stdin.write("\n") + child.stdin.flush() + child.wait(timeout=5) + parent_thread.join(timeout=5) + self.assertEqual(child.returncode, 0, child.stderr.read()) + self.assertTrue(parent_entered.is_set()) + self.assertFalse(parent_thread.is_alive()) + finally: + if child.poll() is None: + try: + assert child.stdin is not None + child.stdin.write("\n") + child.stdin.flush() + child.wait(timeout=2) + except (BrokenPipeError, subprocess.TimeoutExpired): + child.kill() + child.wait(timeout=2) + for stream in (child.stdin, child.stdout, child.stderr): + if stream is not None: + stream.close() + + def test_peer_publication_reconciles_stale_replay_repair_and_prune(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + stale = self.make_store(root, live_bundle_retention=2) + first = stale.publish_success( + identity=AuditPublicationIdentity(1, 1, BLOCK_A), + report=self.report(), + persistence=self.persistence(), + evidence={"confirmation": {"confirmed_count": 1}}, + created_at="first", + ) + peer = self.make_store(root, live_bundle_retention=2) + second = peer.publish_success( + identity=AuditPublicationIdentity(2, 2, BLOCK_B), + report=self.report(block_height=2), + persistence=self.persistence(), + evidence={"confirmation": {"confirmed_count": 1}}, + created_at="second", + ) + + replay = stale.publish_success( + identity=AuditPublicationIdentity(1, 1, BLOCK_A), + publication_floor_sequence=2, + report=self.report(), + persistence=self.persistence(), + evidence={"confirmation": {"confirmed_count": 1}}, + created_at="stale", + ) + self.assertFalse(replay.published) + self.assertEqual(stale.latest_evidence()["block_hash"], BLOCK_B) # type: ignore[index] + + stale._invalidated_legacy_identity = AuditPublicationIdentity( + 1, + 1, + BLOCK_A, + ) + with self.assertRaisesRegex(RuntimeError, "identity conflict"): + stale.publish_success( + identity=AuditPublicationIdentity(2, 2, BLOCK_A), + publication_floor_sequence=2, + report=self.report(block_height=2), + persistence=self.persistence(), + evidence={}, + created_at="conflict", + ) + stale.reconfigure(live_bundle_retention=0) + result = stale.prune_best_effort() + self.assertTrue(second.envelope_path.exists()) + self.assertFalse(first.envelope_path.exists()) + self.assertEqual(result.live_removed, 1) + + def test_floor_publication_fence_blocks_allocator_and_stale_inverse_repair( + self, + ) -> None: + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + store = self.make_store(root, live_bundle_retention=2) + ledger = SingleWriterShareLedger() + balance_lock = threading.RLock() + + def persist(block_hash: str, height: int) -> None: + ledger.persist_accepted_block( + block_hash=block_hash, + block_height=height, + parent_hash="00" * 32, + final_bundle={}, + audit_report={}, + ) + + persist(BLOCK_A, 1) + confirmed_a = ledger.confirm_accepted_block( + block_hash=BLOCK_A, + active_tip_height=1, + ) + identity_a = AuditPublicationIdentity( + int(confirmed_a["audit_publication_sequence"]), + 1, + BLOCK_A, + ) + floor_read = threading.Event() + release_publisher = threading.Event() + allocator_attempting = threading.Event() + allocator_allocated = threading.Event() + errors: list[BaseException] = [] + publications: list[object] = [] + + def publish_a() -> None: + try: + with balance_lock: + with store.publication_order_guard(): + floor = ledger.audit_publication_sequence_floor() + floor_read.set() + if not release_publisher.wait(5): + raise AssertionError("publisher release timed out") + publications.append( + store.publish_success( + identity=identity_a, + publication_floor_sequence=floor, + report=self.report(block_height=1), + persistence=self.persistence(), + evidence={"confirmation": confirmed_a}, + created_at="a", + ) + ) + except BaseException as exc: # pragma: no cover - asserted below + errors.append(exc) + + def allocate_and_publish_b() -> None: + try: + allocator_attempting.set() + with balance_lock: + with store.publication_order_guard(): + persist(BLOCK_B, 2) + confirmed_b = ledger.confirm_accepted_block( + block_hash=BLOCK_B, + active_tip_height=2, + ) + allocator_allocated.set() + identity_b = AuditPublicationIdentity( + int(confirmed_b["audit_publication_sequence"]), + 2, + BLOCK_B, + ) + publications.append( + store.publish_success( + identity=identity_b, + publication_floor_sequence=( + ledger.audit_publication_sequence_floor() + ), + report=self.report(block_height=2), + persistence=self.persistence(), + evidence={"confirmation": confirmed_b}, + created_at="b", + ) + ) + except BaseException as exc: # pragma: no cover - asserted below + errors.append(exc) + + publisher = threading.Thread(target=publish_a) + publisher.start() + allocator: threading.Thread | None = None + try: + self.assertTrue(floor_read.wait(5)) + allocator = threading.Thread(target=allocate_and_publish_b) + allocator.start() + self.assertTrue(allocator_attempting.wait(5)) + unexpectedly_acquired = balance_lock.acquire(blocking=False) + if unexpectedly_acquired: + balance_lock.release() + self.assertFalse(unexpectedly_acquired) + self.assertFalse(allocator_allocated.is_set()) + finally: + release_publisher.set() + publisher.join(timeout=5) + if allocator is not None: + allocator.join(timeout=5) + self.assertFalse(publisher.is_alive()) + assert allocator is not None + self.assertFalse(allocator.is_alive()) + self.assertFalse(errors) + self.assertTrue(allocator_allocated.is_set()) + self.assertEqual(len(publications), 2) + self.assertEqual(ledger.audit_publication_sequence_floor(), 2) + latest = store.latest_evidence() + assert latest is not None + self.assertEqual(latest["block_hash"], BLOCK_B) + + stale_envelope = store.live_envelope_path( + block_height=1, + block_hash=BLOCK_A, + ) + stale_envelope.unlink() + durable_before = store.evidence_path.read_bytes() + durable_identity = _FileIdentity.from_stat(store.evidence_path.stat()) + with balance_lock: + with store.publication_order_guard(): + stale = store.publish_success( + identity=identity_a, + publication_floor_sequence=( + ledger.audit_publication_sequence_floor() + ), + report=self.report(block_height=1), + persistence=self.persistence(), + evidence={"confirmation": confirmed_a}, + created_at="stale", + ) + self.assertFalse(stale.published) + self.assertFalse(stale_envelope.exists()) + self.assertEqual(store.evidence_path.read_bytes(), durable_before) + self.assertTrue(durable_identity.matches(store.evidence_path.stat())) + + def test_interleaved_distinct_landings_preserve_the_superseded_envelope( + self, + ) -> None: + """Two distinct blocks land inside one confirm->publish window. + + The synchronous client-thread tail confirms block A under the payout + balance lock and the publication order guard, then releases both before + it publishes. The accounting-actor tail lands block B inside that gap + and publishes first, so A reaches publication already superseded. A + must not become the current evidence reference, but its own height+hash + live envelope is its block's only published evidence and must be + written. + """ + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + store = self.make_store(root, live_bundle_retention=4) + ledger = SingleWriterShareLedger() + balance_lock = threading.RLock() + + def persist(block_hash: str, height: int) -> None: + ledger.persist_accepted_block( + block_hash=block_hash, + block_height=height, + parent_hash="00" * 32, + final_bundle={}, + audit_report={}, + ) + + persist(BLOCK_A, 1) + confirmed_a = ledger.confirm_accepted_block( + block_hash=BLOCK_A, + active_tip_height=1, + ) + sequence_a = int(confirmed_a["audit_publication_sequence"]) + identity_a = AuditPublicationIdentity(sequence_a, 1, BLOCK_A) + envelope_a = store.live_envelope_path(block_height=1, block_hash=BLOCK_A) + + confirm_gap_open = threading.Event() + resume_publish_a = threading.Event() + errors: list[BaseException] = [] + publications: dict[str, object] = {} + + def synchronous_tail_publishes_a() -> None: + try: + # The confirm scope releases the balance lock and the + # publication guard before the publish scope is entered. + # Holding either across the gap would make the interleaving + # unreachable and the regression invisible. + with balance_lock: + with store.publication_order_guard(): + if ledger.audit_publication_sequence_floor() != sequence_a: + raise AssertionError("confirm floor is unexpected") + confirm_gap_open.set() + if not resume_publish_a.wait(5): + raise AssertionError("publish release timed out") + with balance_lock: + with store.publication_order_guard(): + publications["a"] = store.publish_success( + identity=identity_a, + publication_floor_sequence=( + ledger.audit_publication_sequence_floor() + ), + report=self.report(block_height=1), + persistence=self.persistence(), + evidence={"confirmation": confirmed_a}, + created_at="a", + # The live writer finalizing its own landing, + # exactly as the block finalization path does. + restore_superseded_envelope=True, + ) + except BaseException as exc: # pragma: no cover - asserted below + errors.append(exc) + + def actor_tail_lands_b() -> None: + try: + with balance_lock: + with store.publication_order_guard(): + persist(BLOCK_B, 2) + confirmed_b = ledger.confirm_accepted_block( + block_hash=BLOCK_B, + active_tip_height=2, + ) + identity_b = AuditPublicationIdentity( + int(confirmed_b["audit_publication_sequence"]), + 2, + BLOCK_B, + ) + publications["b_identity"] = identity_b + publications["b"] = store.publish_success( + identity=identity_b, + publication_floor_sequence=( + ledger.audit_publication_sequence_floor() + ), + report=self.report(block_height=2), + persistence=self.persistence(), + evidence={"confirmation": confirmed_b}, + created_at="b", + ) + except BaseException as exc: # pragma: no cover - asserted below + errors.append(exc) + + synchronous = threading.Thread(target=synchronous_tail_publishes_a) + synchronous.start() + actor: threading.Thread | None = None + envelope_b_before = b"" + try: + self.assertTrue(confirm_gap_open.wait(5)) + actor = threading.Thread(target=actor_tail_lands_b) + actor.start() + actor.join(timeout=5) + self.assertFalse(actor.is_alive()) + self.assertFalse(errors) + b = publications["b"] + self.assertTrue(b.published) + self.assertEqual( + publications["b_identity"].sequence, + sequence_a + 1, + ) + self.assertEqual(ledger.audit_publication_sequence_floor(), sequence_a + 1) + envelope_b_before = b.envelope_path.read_bytes() + self.assertFalse(envelope_a.exists()) + finally: + resume_publish_a.set() + synchronous.join(timeout=5) + if actor is not None: + actor.join(timeout=5) + self.assertFalse(synchronous.is_alive()) + self.assertFalse(errors) + + a = publications["a"] + self.assertEqual(a.identity.sequence, sequence_a) + self.assertFalse(a.published) + self.assertTrue(a.envelope_path.exists()) + self.assertEqual(a.envelope_path, envelope_a) + landed_a = json.loads(envelope_a.read_text(encoding="utf-8")) + self.assertEqual(landed_a["block_hash"], BLOCK_A) + self.assertEqual(landed_a["block_height"], 1) + + latest = store.latest_evidence() + assert latest is not None + self.assertEqual(latest["block_hash"], BLOCK_B) + b = publications["b"] + self.assertTrue(b.envelope_path.exists()) + self.assertEqual(b.envelope_path.read_bytes(), envelope_b_before) + + def test_superseded_landing_converges_after_the_peer_publishes(self) -> None: + """The second interleaving shape: B is confirmed but not yet published. + + A publishes at its own ordinal against a floor the peer already + advanced, which is retained as retryable. Once B publishes, A's retry + lands its live envelope without disturbing the durable evidence + pointer. + """ + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + store = self.make_store(root, live_bundle_retention=4) + ledger = SingleWriterShareLedger() + + def persist(block_hash: str, height: int) -> None: + ledger.persist_accepted_block( + block_hash=block_hash, + block_height=height, + parent_hash="00" * 32, + final_bundle={}, + audit_report={}, + ) + + persist(BLOCK_A, 1) + confirmed_a = ledger.confirm_accepted_block( + block_hash=BLOCK_A, + active_tip_height=1, + ) + identity_a = AuditPublicationIdentity( + int(confirmed_a["audit_publication_sequence"]), + 1, + BLOCK_A, + ) + persist(BLOCK_B, 2) + confirmed_b = ledger.confirm_accepted_block( + block_hash=BLOCK_B, + active_tip_height=2, + ) + identity_b = AuditPublicationIdentity( + int(confirmed_b["audit_publication_sequence"]), + 2, + BLOCK_B, + ) + self.assertEqual(identity_b.sequence, identity_a.sequence + 1) + floor = ledger.audit_publication_sequence_floor() + self.assertEqual(floor, identity_b.sequence) + envelope_a = store.live_envelope_path(block_height=1, block_hash=BLOCK_A) + + with self.assertRaisesRegex( + RuntimeError, + "behind the durable ledger floor", + ): + store.publish_success( + identity=identity_a, + publication_floor_sequence=floor, + report=self.report(block_height=1), + persistence=self.persistence(), + evidence={"confirmation": confirmed_a}, + created_at="a-early", + ) + self.assertFalse(envelope_a.exists()) + self.assertIsNone(store.latest_evidence()) + + published_b = store.publish_success( + identity=identity_b, + publication_floor_sequence=floor, + report=self.report(block_height=2), + persistence=self.persistence(), + evidence={"confirmation": confirmed_b}, + created_at="b", + ) + self.assertTrue(published_b.published) + durable_before = store.evidence_path.read_bytes() + durable_identity = _FileIdentity.from_stat(store.evidence_path.stat()) + + retried_a = store.publish_success( + identity=identity_a, + publication_floor_sequence=( + ledger.audit_publication_sequence_floor() + ), + report=self.report(block_height=1), + persistence=self.persistence(), + evidence={"confirmation": confirmed_a}, + created_at="a-retry", + restore_superseded_envelope=True, + ) + self.assertFalse(retried_a.published) + self.assertTrue(envelope_a.exists()) + landed_a = json.loads(envelope_a.read_text(encoding="utf-8")) + self.assertEqual(landed_a["block_hash"], BLOCK_A) + self.assertEqual(landed_a["block_height"], 1) + latest = store.latest_evidence() + assert latest is not None + self.assertEqual(latest["block_hash"], BLOCK_B) + self.assertEqual(store.evidence_path.read_bytes(), durable_before) + self.assertTrue(durable_identity.matches(store.evidence_path.stat())) + + def test_reconfigure_busy_and_post_swap_failure_are_atomic_and_fd_clean(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + base = Path(tmp) + old_root = base / "old-root" + old_evidence = base / "old-state" / "evidence.json" + store = self.make_store(old_root, evidence_path=old_evidence) + new_root = base / "new-root" + blocker = self.make_store( + new_root, + evidence_path=base / "new-state" / "evidence.json", + ) + blocker_entered = threading.Event() + release_blocker = threading.Event() + + def hold_new_root() -> None: + with blocker.publication_order_guard(): + blocker_entered.set() + release_blocker.wait(5) + + blocker_thread = threading.Thread(target=hold_new_root) + blocker_thread.start() + self.assertTrue(blocker_entered.wait(5)) + old_fds = ( + store._root_fd, + store._publication_lock_fd, + store._evidence_parent_fd, + ) + with self.assertRaisesRegex(RuntimeError, "guard is busy"): + store.reconfigure( + root=new_root, + evidence_path=base / "new-state" / "evidence.json", + ) + self.assertEqual(store.root, old_root.resolve()) + for fd in old_fds: + os.fstat(fd) + release_blocker.set() + blocker_thread.join(timeout=5) + self.assertFalse(blocker_thread.is_alive()) + + target_root = base / "target-root" + target_evidence = base / "target-state" / "evidence.json" + prepared_fds: list[int] = [] + real_open_directory = store._open_directory_authority + real_open_lock = store._open_publication_lock_authority + + def record_directory(path: Path) -> tuple[int, tuple[int, int]]: + result = real_open_directory(path) + prepared_fds.append(result[0]) + return result + + def record_lock( + root: Path, + root_fd: int, + root_identity: tuple[int, int], + ) -> tuple[int, _FileIdentity]: + result = real_open_lock(root, root_fd, root_identity) + prepared_fds.append(result[0]) + return result + + with mock.patch.object( + store, + "_open_directory_authority", + side_effect=record_directory, + ), mock.patch.object( + store, + "_open_publication_lock_authority", + side_effect=record_lock, + ), mock.patch.object( + store, + "_reload_current_evidence_locked", + side_effect=RuntimeError("injected post-swap load failure"), + ), self.assertRaisesRegex(RuntimeError, "post-swap load"): + store.reconfigure( + root=target_root, + evidence_path=target_evidence, + ) + self.assertEqual(store.root, old_root.resolve()) + self.assertEqual(store.evidence_path, old_evidence.resolve()) + self.assertEqual( + ( + store._root_fd, + store._publication_lock_fd, + store._evidence_parent_fd, + ), + old_fds, + ) + for fd in old_fds: + os.fstat(fd) + for fd in prepared_fds: + with self.assertRaises(OSError): + os.fstat(fd) + + boundary_root = base / "boundary-root" + boundary_evidence = base / "boundary-state" / "evidence.json" + boundary_prepared_fds: list[int] = [] + explicit_old_validations = 0 + real_validate = store._validate_publication_lock_identity + + def record_boundary_directory( + path: Path, + ) -> tuple[int, tuple[int, int]]: + result = real_open_directory(path) + boundary_prepared_fds.append(result[0]) + return result + + def record_boundary_lock( + root: Path, + root_fd: int, + root_identity: tuple[int, int], + ) -> tuple[int, _FileIdentity]: + result = real_open_lock(root, root_fd, root_identity) + boundary_prepared_fds.append(result[0]) + return result + + def fail_final_old_validation(**kwargs: object) -> None: + nonlocal explicit_old_validations + if kwargs.get("root_fd") == old_fds[0]: + explicit_old_validations += 1 + if explicit_old_validations == 2: + raise RuntimeError("injected final old-authority loss") + real_validate(**kwargs) # type: ignore[arg-type] + + with mock.patch.object( + store, + "_open_directory_authority", + side_effect=record_boundary_directory, + ), mock.patch.object( + store, + "_open_publication_lock_authority", + side_effect=record_boundary_lock, + ), mock.patch.object( + store, + "_validate_publication_lock_identity", + side_effect=fail_final_old_validation, + ), self.assertRaisesRegex(RuntimeError, "final old-authority"): + store.reconfigure( + root=boundary_root, + evidence_path=boundary_evidence, + ) + self.assertEqual(store.root, old_root.resolve()) + self.assertEqual(store.evidence_path, old_evidence.resolve()) + self.assertEqual( + ( + store._root_fd, + store._publication_lock_fd, + store._evidence_parent_fd, + ), + old_fds, + ) + for fd in old_fds: + os.fstat(fd) + for fd in boundary_prepared_fds: + with self.assertRaises(OSError): + os.fstat(fd) + + store.reconfigure( + root=boundary_root, + evidence_path=boundary_evidence, + ) + self.assertEqual(store.root, boundary_root.resolve()) + self.assertEqual(store.evidence_path, boundary_evidence.resolve()) + for fd in old_fds: + with self.assertRaises(OSError): + os.fstat(fd) + + def test_strict_artifact_classification_rejects_lookalikes(self) -> None: + self.assertEqual( + AuditArtifactStore.artifact_kind( + f"prism-audit-bundle-body-{BLOCK_A}-{DIGEST}.json" + ), + "body", + ) + self.assertEqual( + AuditArtifactStore.artifact_kind( + f"prism-live-audit-bundle-2-{BLOCK_A}.json" + ), + "live_bundle", + ) + self.assertEqual( + AuditArtifactStore.artifact_kind( + f".prism-live-audit-bundle-candidate-{BLOCK_A}.json.tmp" + ), + "candidate", + ) + for name in ( + f"prism-live-audit-bundle-02-{BLOCK_A}.json", + f"prism-live-audit-bundle-2-{BLOCK_A}.json.bak", + "prism-live-audit-bundle-candidate-operator.json", + ): + self.assertEqual(AuditArtifactStore.artifact_kind(name), "other") + + def test_a1_only_consumes_an_injected_j1_canonical_capability(self) -> None: + with mock.patch( + "lab.prism.audit_artifacts.subprocess.run", + side_effect=AssertionError("A1 must not own canonicalizer subprocesses"), + ): + self.assertEqual( + canonical_audit_bundle_bytes({"x": 1}, lambda _value: b"typed"), + b"typed", + ) + with self.assertRaisesRegex(RuntimeError, "J1 canonical"): + canonical_audit_bundle_bytes({"x": 1}) + + completed = mock.Mock(returncode=0, stdout=b"rust-typed", stderr=b"") + with mock.patch( + "lab.prism.bundle_compiler.subprocess.run", + return_value=completed, + ) as run: + self.assertEqual(canonical_bundle_bytes({"x": "miner-é"}), b"rust-typed") + self.assertIn("qbit-prism-audit-canonicalize", " ".join(run.call_args.args[0])) + + def test_candidate_cleanup_removes_only_its_adopted_inode(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + store = self.make_store(Path(tmp)) + candidate = store.issue_candidate(block_hash=BLOCK_A) + candidate.path.write_bytes(b"first") + self.transfer_candidate(store, candidate) + candidate.path.unlink() + candidate.path.write_bytes(b"replacement") + store.discard_candidate(candidate) + self.assertEqual(candidate.path.read_bytes(), b"replacement") + + def test_unadopted_candidate_collision_is_preserved_and_released(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + store = self.make_store(root) + candidate = store.issue_candidate(block_hash=BLOCK_A) + candidate.path.write_bytes(b"competitor") + with self.assertRaises(FileExistsError): + store.write_compatibility_candidate(candidate, {"x": 1}) + self.assertEqual(candidate.path.read_bytes(), b"competitor") + store.reconfigure(root=root / "next") + + def test_unadopted_discard_never_deletes_a_created_path(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + store = self.make_store(Path(tmp)) + candidate = store.issue_candidate(block_hash=BLOCK_A) + candidate.path.write_bytes(b"not-transferred") + store.discard_candidate(candidate) + self.assertEqual(candidate.path.read_bytes(), b"not-transferred") + + def test_reconfigure_rejects_active_candidate(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + store = self.make_store(root / "a") + store.issue_candidate(block_hash=BLOCK_A) + with self.assertRaisesRegex(RuntimeError, "candidates are active"): + store.reconfigure(root=root / "b") + + def test_compatibility_candidate_failure_cleans_owned_temp(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + store = self.make_store(Path(tmp)) + candidate = store.issue_candidate(block_hash=BLOCK_A) + with mock.patch("os.fsync", side_effect=OSError("fsync")): + with self.assertRaises(OSError): + store.write_compatibility_candidate(candidate, {"x": 1}) + self.assertFalse(candidate.path.exists()) + + def test_verify_candidate_binds_literal_identity_and_report(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + store = self.make_store(Path(tmp)) + candidate = store.issue_candidate(block_hash=BLOCK_A) + candidate.path.write_bytes(b"canonical") + self.transfer_candidate(store, candidate) + digest = hashlib.sha256(b"canonical").hexdigest() + verified = store.verify_candidate( + candidate, + coinbase_tx_hex="00", + expected_coinbase_value_sats=1, + trusted_writer_public_key_hex="44" * 32, + verifier=lambda *_args, **_kwargs: self.report(digest), + ) + self.assertTrue(verified.canonical_copy_eligible) + self.assertEqual(verified.literal_sha256, digest) + self.assertEqual( + verified.verification_identity, + AuditArtifactStore.build_verification_identity( + trust_source="configured", + trusted_writer_public_key_hex="44" * 32, + literal_sha256=digest, + literal_byte_len=len(b"canonical"), + report=self.report(digest), + ), + ) + + def test_verifier_retry_identity_never_reuses_prior_attempt(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + store = self.make_store(Path(tmp)) + candidate = store.issue_candidate(block_hash=BLOCK_A) + candidate.path.write_bytes(b"canonical") + self.transfer_candidate(store, candidate) + digest = hashlib.sha256(b"canonical").hexdigest() + + with self.assertRaisesRegex(RuntimeError, "failed attempt"): + store.verify_candidate( + candidate, + coinbase_tx_hex="00", + expected_coinbase_value_sats=1, + trusted_writer_public_key_hex="44" * 32, + verifier=lambda *_args, **_kwargs: (_ for _ in ()).throw( + RuntimeError("failed attempt") + ), + ) + verified = store.verify_candidate( + candidate, + coinbase_tx_hex="00", + expected_coinbase_value_sats=1, + trusted_writer_public_key_hex="44" * 32, + verifier=lambda *_args, **_kwargs: self.report(digest), + ) + store.require_current_verified_candidate(verified, candidate) + + store.discard_candidate(candidate) + replacement = store.issue_candidate(block_hash=BLOCK_A) + replacement.path.write_bytes(b"canonical") + self.transfer_candidate(store, replacement) + with self.assertRaisesRegex(RuntimeError, "another candidate"): + store.require_current_verified_candidate(verified, replacement) + with self.assertRaisesRegex(RuntimeError, "later failure"): + store.verify_candidate( + replacement, + coinbase_tx_hex="00", + expected_coinbase_value_sats=1, + trusted_writer_public_key_hex="44" * 32, + verifier=lambda *_args, **_kwargs: (_ for _ in ()).throw( + RuntimeError("later failure") + ), + ) + + def test_verify_candidate_rejects_post_verifier_swap(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + store = self.make_store(Path(tmp)) + candidate = store.issue_candidate(block_hash=BLOCK_A) + candidate.path.write_bytes(b"canonical") + self.transfer_candidate(store, candidate) + + def verifier(_path: Path, *_args: object, **_kwargs: object) -> dict[str, object]: + candidate.path.unlink() + candidate.path.write_bytes(b"replacement") + return self.report(hashlib.sha256(b"replacement").hexdigest()) + + with self.assertRaisesRegex(RuntimeError, "changed during"): + store.verify_candidate( + candidate, + coinbase_tx_hex="00", + expected_coinbase_value_sats=1, + trusted_writer_public_key_hex="44" * 32, + verifier=verifier, + ) + + def test_verifier_report_requires_complete_coinbase_identity(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + for missing in ( + "audit_bundle_sha256_hex", + "reward_manifest_sha256_hex", + "payout_policy_manifest_sha256_hex", + "prism_audit_commitment_leaf_hex", + "audit_commitment_root_hex", + "coinbase_txid", + "coinbase_wtxid", + "coinbase_manifest_sha256_hex", + "coinbase_tx_hex", + "coinbase_value_sats", + "min_output_sats", + "onchain_output_count", + "accrued_account_count", + ): + with self.subTest(missing=missing): + store = self.make_store(Path(tmp) / missing) + candidate = store.issue_candidate(block_hash=BLOCK_A) + candidate.path.write_bytes(b"canonical") + self.transfer_candidate(store, candidate) + report = self.report(hashlib.sha256(b"canonical").hexdigest()) + report.pop(missing) + with self.assertRaises((RuntimeError, ValueError)): + store.verify_candidate( + candidate, + coinbase_tx_hex="00", + expected_coinbase_value_sats=1, + trusted_writer_public_key_hex="44" * 32, + verifier=lambda *_args, _report=report, **_kwargs: _report, + ) + store.discard_candidate(candidate) + + def test_verifier_report_schema_types_and_height_fail_closed(self) -> None: + cases = ( + ("schema", "wrong"), + ("block_height", "1"), + ("block_height", 2), + ("coinbase_value_sats", 1.5), + ("min_output_sats", True), + ("onchain_output_count", -1), + ("accrued_account_count", 1.5), + ) + for field, value in cases: + with self.subTest(field=field, value=value): + report = self.report() + report[field] = value + with self.assertRaises(RuntimeError): + AuditArtifactStore._validate_verifier_report( + report, + coinbase_tx_hex="00", + expected_coinbase_value_sats=1, + expected_block_height=1, + ) + + def test_trust_precedence_and_embedded_test_key_gate(self) -> None: + bundle = { + "ledger_window_attestation": { + "signature": {"public_key_hex": "55" * 32} + } + } + self.assertEqual( + AuditArtifactStore.trusted_writer_key( + "44" * 32, + bundle, + allow_embedded_test_key=True, + ), + "44" * 32, + ) + with self.assertRaisesRegex(RuntimeError, "configured"): + AuditArtifactStore.trusted_writer_key( + None, + bundle, + allow_embedded_test_key=False, + ) + self.assertEqual( + AuditArtifactStore.trusted_writer_key( + None, + bundle, + allow_embedded_test_key=True, + ), + "55" * 32, + ) + + def test_verifier_reads_unlinked_snapshot_during_candidate_aba(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + store = self.make_store(Path(tmp)) + candidate = store.issue_candidate(block_hash=BLOCK_A) + candidate.path.write_bytes(b"canonical") + self.transfer_candidate(store, candidate) + original = candidate.path.stat() + digest = hashlib.sha256(b"canonical").hexdigest() + + def verifier(path: Path, *_args: object, **_kwargs: object) -> dict[str, object]: + self.assertEqual(path.read_bytes(), b"canonical") + candidate.path.write_bytes(b"replacement") + candidate.path.write_bytes(b"canonical") + os.utime( + candidate.path, + ns=(original.st_atime_ns, original.st_mtime_ns), + ) + return self.report(digest) + + verified = store.verify_candidate( + candidate, + coinbase_tx_hex="00", + expected_coinbase_value_sats=1, + trusted_writer_public_key_hex="44" * 32, + expected_block_height=1, + verifier=verifier, + ) + self.assertEqual(verified.literal_sha256, digest) + + def test_verifier_subprocess_timeout_nonzero_malformed_and_oversize(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + bundle = root / "bundle.json" + bundle.write_text("{}", encoding="utf-8") + cases = ( + ( + [sys.executable, "-c", "import time; time.sleep(2)"], + "timed out", + 0.05, + ), + ([sys.executable, "-c", "import sys; sys.stderr.write('bad'); sys.exit(2)"], "failed: bad", 1.0), + ([sys.executable, "-c", "print('not-json')"], "invalid JSON", 1.0), + ([sys.executable, "-c", "import sys; sys.stdout.write('x' * 1100000)"], "output exceeded", 1.0), + ) + for command, message, timeout in cases: + with self.subTest(message=message): + store = self.make_store( + root / message.replace(" ", "-"), + verifier_timeout_seconds=timeout, + ) + with mock.patch( + "lab.prism.audit_artifacts.prism_tool_command", + return_value=command, + ), self.assertRaisesRegex(RuntimeError, message): + store.verify_bundle( + bundle, + "00", + "44" * 32, + expected_coinbase_value_sats=1, + ) + + def test_verifier_timeout_kills_descendants_that_inherit_output_pipes(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + bundle = root / "bundle.json" + marker = root / "descendant-survived" + bundle.write_text("{}", encoding="utf-8") + descendant = ( + "import pathlib,time; time.sleep(0.4); " + f"pathlib.Path({str(marker)!r}).write_text('survived')" + ) + parent = ( + "import subprocess,sys,time; " + f"subprocess.Popen([sys.executable,'-c',{descendant!r}]," + "stdout=sys.stdout,stderr=sys.stderr); time.sleep(5)" + ) + store = self.make_store( + root / "audit", + verifier_timeout_seconds=0.05, + ) + with mock.patch( + "lab.prism.audit_artifacts.prism_tool_command", + return_value=[sys.executable, "-c", parent], + ), self.assertRaisesRegex(RuntimeError, "timed out"): + store.verify_bundle( + bundle, + "00", + "44" * 32, + expected_coinbase_value_sats=1, + ) + time.sleep(0.5) + self.assertFalse(marker.exists()) + + def test_publication_uses_durable_ordinal_not_height(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + store = self.make_store(Path(tmp), live_bundle_retention=0) + newer = store.publish_success( + identity=AuditPublicationIdentity(8, 101, BLOCK_B), + report=self.report(block_height=101), + persistence=self.persistence(), + evidence={"audit_report": self.report(), "persistence": self.persistence()}, + created_at="now", + ) + stale = store.publish_success( + identity=AuditPublicationIdentity(7, 100, BLOCK_A), + report=self.report(block_height=100), + persistence=self.persistence(), + evidence={"audit_report": self.report(), "persistence": self.persistence()}, + created_at="later", + ) + self.assertTrue(newer.published) + self.assertFalse(stale.published) + self.assertEqual(store.latest_evidence()["block_hash"], BLOCK_B) # type: ignore[index] + deep_reorg = store.publish_success( + identity=AuditPublicationIdentity(9, 99, BLOCK_A), + report=self.report(block_height=99), + persistence=self.persistence(), + evidence={"audit_report": self.report(), "persistence": self.persistence()}, + created_at="latest", + ) + self.assertTrue(deep_reorg.published) + self.assertEqual(store.latest_evidence()["block_hash"], BLOCK_A) # type: ignore[index] + + def test_publication_and_startup_bind_report_height(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + store = self.make_store(root) + with self.assertRaisesRegex(RuntimeError, "block height"): + store.publish_success( + identity=AuditPublicationIdentity(1, 2, BLOCK_A), + report=self.report(block_height=1), + persistence=self.persistence(), + evidence={}, + created_at="now", + ) + store.publish_success( + identity=AuditPublicationIdentity(1, 2, BLOCK_A), + report=self.report(block_height=2), + persistence=self.persistence(), + evidence={}, + created_at="now", + ) + payload = json.loads(store.evidence_path.read_text(encoding="utf-8")) + payload["audit_report"]["block_height"] = 1 + store.evidence_path.write_text(json.dumps(payload), encoding="utf-8") + self.assertIsNone(self.make_store(root).latest_evidence()) + + def test_equal_ordinal_conflict_fails_closed(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + store = self.make_store(Path(tmp)) + store.publish_success( + identity=AuditPublicationIdentity(1, 1, BLOCK_A), + report=self.report(block_height=1), + persistence=self.persistence(), + evidence={"audit_report": self.report(), "persistence": self.persistence()}, + created_at="now", + ) + with self.assertRaisesRegex(RuntimeError, "conflict"): + store.publish_success( + identity=AuditPublicationIdentity(1, 1, BLOCK_B), + report=self.report(block_height=1), + persistence=self.persistence(), + evidence={"audit_report": self.report(), "persistence": self.persistence()}, + created_at="later", + ) + + def test_exact_replay_rejects_changed_coinbase_identity(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + store = self.make_store(Path(tmp)) + identity = AuditPublicationIdentity(1, 1, BLOCK_A) + store.publish_success( + identity=identity, + report=self.report(block_height=1), + persistence=self.persistence(), + evidence={}, + created_at="now", + ) + changed = self.report(block_height=1) + changed["coinbase_txid"] = "99" * 32 + with self.assertRaisesRegex(RuntimeError, "replay payload conflict"): + store.publish_success( + identity=identity, + report=changed, + persistence=self.persistence(), + evidence={}, + created_at="later", + ) + + def test_exact_replay_is_stable_but_changed_evidence_conflicts(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + store = self.make_store(Path(tmp)) + identity = AuditPublicationIdentity(3, 4, BLOCK_A) + first = store.publish_success( + identity=identity, + report=self.report(block_height=4), + persistence=self.persistence(), + evidence={"confirmation": {"confirmed_count": 1}}, + created_at="now", + ) + replay = store.publish_success( + identity=identity, + report=self.report(block_height=4), + persistence=self.persistence(), + evidence={"confirmation": {"confirmed_count": 1}}, + created_at="later", + ) + self.assertTrue(first.published) + self.assertFalse(replay.published) + advanced_floor_replay = store.publish_success( + identity=identity, + publication_floor_sequence=4, + report=self.report(block_height=4), + persistence=self.persistence(), + evidence={"confirmation": {"confirmed_count": 1}}, + created_at="later-still", + ) + self.assertFalse(advanced_floor_replay.published) + # The confirm disposition is observational, like the share + # counters above: the flip that published this evidence reported a + # fresh confirmation (1), and an outbox replay of the same durable + # row reports the idempotent arm (2) -- which is the whole point of + # that split (issue #61). Reuse the durable value rather than + # stranding the replay as a payload conflict. + idempotent_disposition_replay = store.publish_success( + identity=identity, + report=self.report(block_height=4), + persistence=self.persistence(), + evidence={"confirmation": {"confirmed_count": 2}}, + created_at="idempotent", + ) + self.assertFalse(idempotent_disposition_replay.published) + self.assertEqual( + idempotent_disposition_replay.evidence["confirmation"], + {"confirmed_count": 1}, + ) + # Only the disposition is observational. Everything else the + # confirmation carries -- the publication ordinal above all -- is + # block identity and still conflicts when it changes. + with self.assertRaisesRegex(RuntimeError, "replay payload conflict"): + store.publish_success( + identity=identity, + report=self.report(block_height=4), + persistence=self.persistence(), + evidence={ + "confirmation": { + "confirmed_count": 2, + "audit_publication_sequence": 9, + } + }, + created_at="later", + ) + # A changed evidence payload outside the confirmation is unchanged + # in its treatment. + with self.assertRaisesRegex(RuntimeError, "replay payload conflict"): + store.publish_success( + identity=identity, + report=self.report(block_height=4), + persistence=self.persistence(), + evidence={ + "confirmation": {"confirmed_count": 1}, + "job_share_count": 7, + }, + created_at="later", + ) + + def test_exact_replay_repairs_invalid_mutable_evidence_but_rejects_envelope_replacement( + self, + ) -> None: + identity = AuditPublicationIdentity(3, 4, BLOCK_A) + for missing in ("evidence", "envelope", "both"): + with self.subTest(missing=missing), tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + store = self.make_store(root) + first = store.publish_success( + identity=identity, + report=self.report(block_height=4), + persistence=self.persistence(), + evidence={"confirmation": {"confirmed_count": 1}}, + created_at="now", + ) + if missing in {"evidence", "both"}: + store.evidence_path.unlink() + if missing in {"envelope", "both"}: + first.envelope_path.unlink() + before_advanced_floor_attempt = ( + first.envelope_path.read_bytes() + if first.envelope_path.exists() + else None, + store.evidence_path.read_bytes() + if store.evidence_path.exists() + else None, + ) + with self.assertRaisesRegex(RuntimeError, "behind"): + store.publish_success( + identity=identity, + publication_floor_sequence=4, + report=self.report(block_height=4), + persistence=self.persistence(), + evidence={"confirmation": {"confirmed_count": 1}}, + created_at="advanced-floor", + ) + self.assertEqual( + ( + first.envelope_path.read_bytes() + if first.envelope_path.exists() + else None, + store.evidence_path.read_bytes() + if store.evidence_path.exists() + else None, + ), + before_advanced_floor_attempt, + ) + repaired = store.publish_success( + identity=identity, + report=self.report(block_height=4), + persistence=self.persistence(), + evidence={"confirmation": {"confirmed_count": 1}}, + created_at="later", + ) + self.assertTrue(repaired.published) + self.assertTrue(store.evidence_path.exists()) + self.assertTrue(first.envelope_path.exists()) + store.close() + restarted = self.make_store(root) + self.assertEqual( + restarted.latest_evidence()["block_hash"], # type: ignore[index] + BLOCK_A, + ) + + for replacement_bytes in ( + b"not-json", + b'{"operator":"replacement"}', + ): + for restart in (False, True): + with self.subTest( + evidence_replacement=replacement_bytes, + restart=restart, + ), tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + store = self.make_store(root) + first = store.publish_success( + identity=identity, + report=self.report(block_height=4), + persistence=self.persistence(), + evidence={"confirmation": {"confirmed_count": 1}}, + created_at="now", + ) + envelope_bytes = first.envelope_path.read_bytes() + store.evidence_path.write_bytes(replacement_bytes) + if restart: + store.close() + store = self.make_store(root) + self.assertIsNone(store.latest_evidence()) + with self.assertRaisesRegex(RuntimeError, "behind"): + store.publish_success( + identity=identity, + publication_floor_sequence=4, + report=self.report(block_height=4), + persistence=self.persistence(), + evidence={"confirmation": {"confirmed_count": 1}}, + created_at="advanced-floor", + ) + self.assertEqual( + store.evidence_path.read_bytes(), + replacement_bytes, + ) + repaired = store.publish_success( + identity=identity, + report=self.report(block_height=4), + persistence=self.persistence(), + evidence={"confirmation": {"confirmed_count": 1}}, + created_at="later", + ) + self.assertTrue(repaired.published) + self.assertEqual(first.envelope_path.read_bytes(), envelope_bytes) + self.assertEqual( + json.loads(store.evidence_path.read_text(encoding="utf-8"))[ + "block_hash" + ], + BLOCK_A, + ) + + for restart in (False, True): + with self.subTest( + envelope_replacement=True, + restart=restart, + ), tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + store = self.make_store(root) + first = store.publish_success( + identity=identity, + report=self.report(block_height=4), + persistence=self.persistence(), + evidence={"confirmation": {"confirmed_count": 1}}, + created_at="now", + ) + first.envelope_path.write_bytes(b'{"operator":"replacement"}') + replacement_bytes = first.envelope_path.read_bytes() + if restart: + store.close() + store = self.make_store(root) + self.assertIsNone(store.latest_evidence()) + with self.assertRaisesRegex(RuntimeError, "(conflicts|invalid)"): + store.publish_success( + identity=identity, + report=self.report(block_height=4), + persistence=self.persistence(), + evidence={"confirmation": {"confirmed_count": 1}}, + created_at="later", + ) + self.assertEqual(first.envelope_path.read_bytes(), replacement_bytes) + + def test_newer_publication_repairs_invalid_evidence_and_repoints_it(self) -> None: + for next_block_hash, next_height in ((BLOCK_A, 4), (BLOCK_B, 5)): + for restart in (False, True): + with self.subTest( + next_block_hash=next_block_hash, + restart=restart, + ), tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + store = self.make_store(root) + first = store.publish_success( + identity=AuditPublicationIdentity(3, 4, BLOCK_A), + report=self.report(block_height=4), + persistence=self.persistence(), + evidence={"confirmation": {"confirmed_count": 1}}, + created_at="now", + ) + first_envelope_bytes = first.envelope_path.read_bytes() + store.evidence_path.write_bytes(b'{"operator":"replacement"}') + if restart: + store.close() + store = self.make_store(root) + self.assertIsNone(store.latest_evidence()) + second = store.publish_success( + identity=AuditPublicationIdentity( + 4, + next_height, + next_block_hash, + ), + publication_floor_sequence=4, + report=self.report(block_height=next_height), + persistence=self.persistence(), + evidence={"confirmation": {"confirmed_count": 1}}, + created_at="later", + ) + self.assertTrue(second.published) + self.assertEqual( + first.envelope_path.read_bytes(), + first_envelope_bytes, + ) + self.assertTrue(second.envelope_path.exists()) + self.assertEqual( + json.loads(store.evidence_path.read_text(encoding="utf-8"))[ + "block_hash" + ], + next_block_hash, + ) + + def test_restart_repair_requires_the_fresh_durable_publication_floor(self) -> None: + identity = AuditPublicationIdentity(3, 4, BLOCK_A) + for damage in ("missing_envelope", "malformed_evidence", "both_missing"): + with self.subTest(damage=damage), tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + store = self.make_store(root) + first = store.publish_success( + identity=identity, + report=self.report(block_height=4), + persistence=self.persistence(), + evidence={"confirmation": {"confirmed_count": 1}}, + created_at="now", + ) + if damage in {"missing_envelope", "both_missing"}: + first.envelope_path.unlink() + if damage == "malformed_evidence": + store.evidence_path.write_bytes(b"not-json") + elif damage == "both_missing": + store.evidence_path.unlink() + store.close() + + restarted = self.make_store(root) + self.assertIsNone(restarted.latest_evidence()) + evidence_before = ( + restarted.evidence_path.read_bytes() + if restarted.evidence_path.exists() + else None + ) + with self.assertRaisesRegex(RuntimeError, "behind"): + restarted.publish_success( + identity=AuditPublicationIdentity(2, 5, BLOCK_B), + publication_floor_sequence=3, + report=self.report(block_height=5), + persistence=self.persistence(), + evidence={"confirmation": {"confirmed_count": 1}}, + created_at="stale", + ) + self.assertFalse( + restarted.live_envelope_path( + block_height=5, + block_hash=BLOCK_B, + ).exists() + ) + self.assertEqual( + restarted.evidence_path.read_bytes() + if restarted.evidence_path.exists() + else None, + evidence_before, + ) + repaired = restarted.publish_success( + identity=identity, + publication_floor_sequence=3, + report=self.report(block_height=4), + persistence=self.persistence(), + evidence={"confirmation": {"confirmed_count": 1}}, + created_at="repair", + ) + self.assertTrue(repaired.published) + self.assertTrue(first.envelope_path.exists()) + self.assertEqual( + json.loads(restarted.evidence_path.read_text(encoding="utf-8"))[ + "block_hash" + ], + BLOCK_A, + ) + + def test_failed_invalid_evidence_repair_preserves_bytes_and_revokes_stale_pin(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + store = self.make_store(Path(tmp)) + identity = AuditPublicationIdentity(3, 4, BLOCK_A) + first = store.publish_success( + identity=identity, + report=self.report(block_height=4), + persistence=self.persistence(), + evidence={"confirmation": {"confirmed_count": 1}}, + created_at="now", + ) + latest_before = store.latest_evidence() + envelope_before = first.envelope_path.read_bytes() + corrupt_evidence = b'{"operator":"replacement"}' + store.evidence_path.write_bytes(corrupt_evidence) + original_fsync = store._fsync_directory + failed = False + + def fail_evidence_parent(parent: Path) -> None: + nonlocal failed + if parent == store.evidence_path.parent and not failed: + failed = True + raise OSError("injected evidence durability failure") + original_fsync(parent) + + with mock.patch.object( + store, + "_fsync_directory", + side_effect=fail_evidence_parent, + ), self.assertRaisesRegex(OSError, "evidence durability"): + store.publish_success( + identity=identity, + publication_floor_sequence=3, + report=self.report(block_height=4), + persistence=self.persistence(), + evidence={"confirmation": {"confirmed_count": 1}}, + created_at="later", + ) + + self.assertTrue(failed) + self.assertEqual(store.evidence_path.read_bytes(), corrupt_evidence) + self.assertEqual(first.envelope_path.read_bytes(), envelope_before) + self.assertIsNotNone(latest_before) + self.assertIsNone(store.latest_evidence()) + self.assertIsNone(store._current_envelope) + self.assertIsNone(store._current_identity) + + def test_exact_replay_reuses_nonidentity_global_stats_after_restart(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + identity = AuditPublicationIdentity(3, 4, BLOCK_A) + store = self.make_store(root) + store.publish_success( + identity=identity, + report=self.report(block_height=4), + persistence={**self.persistence(), "share_count": 10}, + evidence={ + "confirmation": {"confirmed_count": 1}, + "accepted_share_count": 10, + "distinct_miner_count": 2, + "job_share_count": 3, + }, + created_at="now", + ) + restarted = self.make_store(root) + replay = restarted.publish_success( + identity=identity, + report=self.report(block_height=4), + persistence={**self.persistence(), "share_count": 99}, + evidence={ + "confirmation": {"confirmed_count": 1}, + "accepted_share_count": 99, + "distinct_miner_count": 20, + "job_share_count": 3, + }, + created_at="later", + ) + self.assertFalse(replay.published) + self.assertEqual(replay.evidence["accepted_share_count"], 10) + self.assertEqual(replay.evidence["distinct_miner_count"], 2) + self.assertEqual(replay.evidence["job_share_count"], 3) + self.assertEqual(replay.evidence["persistence"]["share_count"], 10) # type: ignore[index] + with self.assertRaisesRegex(RuntimeError, "replay payload conflict"): + restarted.publish_success( + identity=identity, + report=self.report(block_height=4), + persistence={**self.persistence(), "share_count": 100}, + evidence={ + "confirmation": {"confirmed_count": 1}, + "accepted_share_count": 100, + "distinct_miner_count": 21, + "job_share_count": 4, + }, + created_at="later", + ) + + def test_verification_identity_is_restart_stable_and_tamper_evident(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + store = self.make_store(root) + report = self.report(block_height=4) + verification = AuditArtifactStore.build_verification_identity( + trust_source="configured", + trusted_writer_public_key_hex="44" * 32, + literal_sha256="99" * 32, + literal_byte_len=456, + report=report, + ) + identity = AuditPublicationIdentity(3, 4, BLOCK_A) + store.publish_success( + identity=identity, + report=report, + persistence=self.persistence(), + evidence={}, + verification_identity=verification, + created_at="now", + ) + restarted = self.make_store(root) + latest = restarted.latest_evidence() + assert latest is not None + self.assertEqual(latest["audit_verification_identity"], verification) + replay = restarted.publish_success( + identity=identity, + report=report, + persistence=self.persistence(), + evidence={}, + verification_identity=verification, + created_at="later", + ) + self.assertFalse(replay.published) + + changed_report = dict(report) + changed_report["coinbase_wtxid"] = "aa" * 32 + variants = ( + ( + report, + AuditArtifactStore.build_verification_identity( + trust_source="configured", + trusted_writer_public_key_hex="55" * 32, + literal_sha256="99" * 32, + literal_byte_len=456, + report=report, + ), + ), + ( + report, + AuditArtifactStore.build_verification_identity( + trust_source="embedded_test_only", + trusted_writer_public_key_hex="44" * 32, + literal_sha256="99" * 32, + literal_byte_len=456, + report=report, + ), + ), + ( + report, + AuditArtifactStore.build_verification_identity( + trust_source="configured", + trusted_writer_public_key_hex="44" * 32, + literal_sha256="aa" * 32, + literal_byte_len=456, + report=report, + ), + ), + ( + report, + AuditArtifactStore.build_verification_identity( + trust_source="configured", + trusted_writer_public_key_hex="44" * 32, + literal_sha256="99" * 32, + literal_byte_len=457, + report=report, + ), + ), + ( + changed_report, + AuditArtifactStore.build_verification_identity( + trust_source="configured", + trusted_writer_public_key_hex="44" * 32, + literal_sha256="99" * 32, + literal_byte_len=456, + report=changed_report, + ), + ), + ) + for variant_report, changed in variants: + with self.subTest(changed=changed), self.assertRaisesRegex( + RuntimeError, + "replay payload conflict", + ): + restarted.publish_success( + identity=identity, + report=variant_report, + persistence=self.persistence(), + evidence={}, + verification_identity=changed, + created_at="later", + ) + + def test_startup_rejects_each_verification_identity_tamper(self) -> None: + cases = ( + ("trust_source", "embedded_test_only"), + ("ledger_writer_public_key_hex", "55" * 32), + ("literal_sha256_hex", "99" * 32), + ("literal_byte_len", 999), + ("identity_sha256_hex", "88" * 32), + ) + for field, value in cases: + with self.subTest(field=field), tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + store = self.make_store(root) + store.publish_success( + identity=AuditPublicationIdentity(1, 1, BLOCK_A), + report=self.report(), + persistence=self.persistence(), + evidence={}, + created_at="now", + ) + payload = json.loads(store.evidence_path.read_text(encoding="utf-8")) + payload["audit_verification_identity"][field] = value + store.evidence_path.write_text(json.dumps(payload), encoding="utf-8") + self.assertIsNone(self.make_store(root).latest_evidence()) + + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + store = self.make_store(root) + store.publish_success( + identity=AuditPublicationIdentity(1, 1, BLOCK_A), + report=self.report(), + persistence=self.persistence(), + evidence={}, + created_at="now", + ) + payload = json.loads(store.evidence_path.read_text(encoding="utf-8")) + payload["audit_verification_identity"]["report"]["coinbase_txid"] = ( + "99" * 32 + ) + store.evidence_path.write_text(json.dumps(payload), encoding="utf-8") + self.assertIsNone(self.make_store(root).latest_evidence()) + + def test_higher_ordinal_replaces_same_height_block(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + store = self.make_store(Path(tmp)) + store.publish_success( + identity=AuditPublicationIdentity(1, 10, BLOCK_A), + report=self.report(block_height=10), + persistence=self.persistence(), + evidence={}, + created_at="now", + ) + replacement = store.publish_success( + identity=AuditPublicationIdentity(2, 10, BLOCK_B), + report=self.report(block_height=10), + persistence=self.persistence(), + evidence={}, + created_at="later", + ) + self.assertTrue(replacement.published) + self.assertEqual(store.latest_evidence()["block_hash"], BLOCK_B) # type: ignore[index] + + def test_publication_rejects_body_for_wrong_block_or_digest(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + store = self.make_store(Path(tmp)) + identity = AuditPublicationIdentity(1, 1, BLOCK_A) + for body in ( + store.body_path(BLOCK_B, DIGEST), + store.body_path(BLOCK_A, "77" * 32), + ): + with self.subTest(body=body), self.assertRaisesRegex( + RuntimeError, + "body URI does not match", + ): + store.publish_success( + identity=identity, + report=self.report(block_height=1), + persistence={ + "audit_bundle_sha256": DIGEST, + "body_uri": str(body), + }, + evidence={}, + created_at="now", + ) + + def test_restart_preserves_publication_ordinal_fence(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + store = self.make_store(root) + store.publish_success( + identity=AuditPublicationIdentity(5, 5, BLOCK_B), + report=self.report(block_height=5), + persistence=self.persistence(), + evidence={}, + created_at="now", + ) + restarted = self.make_store(root) + stale = restarted.publish_success( + identity=AuditPublicationIdentity(4, 6, BLOCK_A), + report=self.report(), + persistence=self.persistence(), + evidence={}, + created_at="later", + ) + self.assertFalse(stale.published) + self.assertEqual(restarted.latest_evidence()["block_hash"], BLOCK_B) # type: ignore[index] + + def test_invalid_evidence_can_be_repaired_by_durable_publication(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + (root / "evidence.json").write_text("broken", encoding="utf-8") + store = self.make_store(root) + publication = store.publish_success( + identity=AuditPublicationIdentity(1, 1, BLOCK_A), + report=self.report(), + persistence=self.persistence(), + evidence={}, + created_at="now", + ) + self.assertTrue(publication.published) + self.assertEqual(self.make_store(root).latest_evidence()["block_hash"], BLOCK_A) # type: ignore[index] + + def test_startup_rejects_coinbase_tampered_evidence(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + store = self.make_store(root) + store.publish_success( + identity=AuditPublicationIdentity(1, 1, BLOCK_A), + report=self.report(), + persistence=self.persistence(), + evidence={}, + created_at="now", + ) + payload = json.loads(store.evidence_path.read_text(encoding="utf-8")) + payload["coinbase_txid"] = "99" * 32 + store.evidence_path.write_text(json.dumps(payload), encoding="utf-8") + self.assertIsNone(self.make_store(root).latest_evidence()) + + def test_startup_rejects_noncanonical_durable_identity_fields(self) -> None: + canonical_digest = "ab" * 32 + + def publish_fixture(root: Path, *, with_body: bool = False) -> AuditArtifactStore: + store = self.make_store(root) + persistence = self.persistence(canonical_digest) + if with_body: + persistence["body_uri"] = str( + store.body_path(BLOCK_A, canonical_digest) + ) + store.publish_success( + identity=AuditPublicationIdentity(1, 1, BLOCK_A), + report=self.report(canonical_digest), + persistence=persistence, + evidence={}, + created_at="now", + ) + return store + + cases = ( + ("height-string", lambda evidence, _envelope: evidence.__setitem__("block_height", "1")), + ("height-bool", lambda evidence, _envelope: evidence.__setitem__("block_height", True)), + ("sequence-string", lambda evidence, _envelope: evidence["audit_publication_identity"].__setitem__("sequence", "1")), + ("sequence-bool", lambda evidence, _envelope: evidence["audit_publication_identity"].__setitem__("sequence", True)), + ("coinbase-value-string", lambda evidence, _envelope: evidence.__setitem__("coinbase_value_sats", "1")), + ("block-hash-uppercase", lambda evidence, _envelope: evidence.__setitem__("block_hash", BLOCK_A.upper())), + ("persistence-digest-uppercase", lambda evidence, _envelope: evidence["persistence"].__setitem__("audit_bundle_sha256", canonical_digest.upper())), + ("report-hash-uppercase", lambda evidence, _envelope: evidence["audit_report"].__setitem__("audit_bundle_sha256_hex", canonical_digest.upper())), + ("envelope-height-string", lambda _evidence, envelope: envelope.__setitem__("block_height", "1")), + ("envelope-value-bool", lambda _evidence, envelope: envelope.__setitem__("coinbase_value_sats", True)), + ) + for name, mutate in cases: + with self.subTest(name=name), tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + store = publish_fixture(root) + evidence = json.loads(store.evidence_path.read_text(encoding="utf-8")) + envelope_path = Path(evidence["audit_bundle_path"]) + envelope = json.loads(envelope_path.read_text(encoding="utf-8")) + mutate(evidence, envelope) + store.evidence_path.write_text(json.dumps(evidence), encoding="utf-8") + envelope_path.write_text(json.dumps(envelope), encoding="utf-8") + self.assertIsNone(self.make_store(root).latest_evidence()) + + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + store = publish_fixture(root, with_body=True) + evidence = json.loads(store.evidence_path.read_text(encoding="utf-8")) + envelope_path = Path(evidence["audit_bundle_path"]) + envelope = json.loads(envelope_path.read_text(encoding="utf-8")) + (root / "sub").mkdir() + body_name = Path(evidence["persistence"]["body_uri"]).name + noncanonical = str(root / "sub" / ".." / body_name) + evidence["persistence"]["body_uri"] = noncanonical + envelope["body_uri"] = noncanonical + store.evidence_path.write_text(json.dumps(evidence), encoding="utf-8") + envelope_path.write_text(json.dumps(envelope), encoding="utf-8") + self.assertIsNone(self.make_store(root).latest_evidence()) + + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + store = publish_fixture(root) + evidence = json.loads(store.evidence_path.read_text(encoding="utf-8")) + original_cwd = Path.cwd() + try: + os.chdir(root) + evidence["audit_bundle_path"] = Path( + evidence["audit_bundle_path"] + ).name + store.evidence_path.write_text(json.dumps(evidence), encoding="utf-8") + self.assertIsNone(self.make_store(root).latest_evidence()) + finally: + os.chdir(original_cwd) + + def test_prune_failure_does_not_undo_durable_publication(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + store = self.make_store(Path(tmp)) + with mock.patch.object(store, "prune_best_effort", side_effect=OSError("prune")): + result = store.publish_success( + identity=AuditPublicationIdentity(1, 1, BLOCK_A), + report=self.report(block_height=1), + persistence=self.persistence(), + evidence={}, + created_at="now", + ) + self.assertTrue(result.published) + self.assertEqual(store.latest_evidence()["block_hash"], BLOCK_A) # type: ignore[index] + + def test_evidence_failure_preserves_old_reference_and_skips_prune(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + store = self.make_store(Path(tmp), live_bundle_retention=0) + store.publish_success( + identity=AuditPublicationIdentity(1, 1, BLOCK_A), + report=self.report(), + persistence=self.persistence(), + evidence={"audit_report": self.report(), "persistence": self.persistence()}, + created_at="old", + ) + old = store.evidence_path.read_bytes() + original = store._write_mutable_json + + def fail_evidence(path: Path, *args: object, **kwargs: object) -> object: + if path == store.evidence_path: + raise OSError("evidence") + return original(path, *args, **kwargs) + + with mock.patch.object(store, "_write_mutable_json", side_effect=fail_evidence): + with self.assertRaises(OSError): + store.publish_success( + identity=AuditPublicationIdentity(2, 2, BLOCK_B), + report=self.report(block_height=2), + persistence=self.persistence(), + evidence={"audit_report": self.report(), "persistence": self.persistence()}, + created_at="new", + ) + self.assertEqual(store.evidence_path.read_bytes(), old) + self.assertTrue( + store.live_envelope_path(block_height=1, block_hash=BLOCK_A).exists() + ) + + def test_publication_fsyncs_each_parent_in_commit_order(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + base = Path(tmp) + root = base / "artifacts" + evidence_path = base / "state" / "evidence.json" + store = self.make_store(root, evidence_path=evidence_path) + fsync_parents: list[Path] = [] + real_fsync = store._fsync_directory + + def record(parent: Path) -> None: + fsync_parents.append(parent) + real_fsync(parent) + + with mock.patch.object( + store, + "_fsync_directory", + side_effect=record, + ): + store.publish_success( + identity=AuditPublicationIdentity(1, 1, BLOCK_A), + report=self.report(), + persistence=self.persistence(), + evidence={}, + created_at="now", + ) + + self.assertGreaterEqual(len(fsync_parents), 2) + self.assertEqual(fsync_parents[:2], [root.resolve(), evidence_path.parent.resolve()]) + + def test_second_parent_fsync_failure_preserves_recoverable_state(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + base = Path(tmp) + root = base / "artifacts" + evidence_path = base / "state" / "evidence.json" + store = self.make_store(root, evidence_path=evidence_path) + fsync_parents: list[Path] = [] + real_fsync = store._fsync_directory + failed = False + + def fail_evidence_parent_once(parent: Path) -> None: + nonlocal failed + fsync_parents.append(parent) + if parent == evidence_path.parent.resolve() and not failed: + failed = True + raise OSError("evidence parent fsync") + real_fsync(parent) + + envelope = store.live_envelope_path( + block_height=1, + block_hash=BLOCK_A, + ) + with mock.patch.object( + store, + "_fsync_directory", + side_effect=fail_evidence_parent_once, + ), self.assertRaisesRegex(OSError, "evidence parent fsync"): + store.publish_success( + identity=AuditPublicationIdentity(1, 1, BLOCK_A), + report=self.report(), + persistence=self.persistence(), + evidence={}, + created_at="now", + ) + + self.assertEqual( + fsync_parents[:2], + [root.resolve(), evidence_path.parent.resolve()], + ) + self.assertFalse(envelope.exists()) + self.assertFalse(evidence_path.exists()) + restarted = self.make_store(root, evidence_path=evidence_path) + self.assertIsNone(restarted.latest_evidence()) + + def test_evidence_failure_preserves_competing_envelope_replacement(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + store = self.make_store(Path(tmp), live_bundle_retention=0) + store.publish_success( + identity=AuditPublicationIdentity(1, 1, BLOCK_A), + report=self.report(), + persistence=self.persistence(), + evidence={}, + created_at="old", + ) + competitor_path = store.live_envelope_path( + block_height=2, + block_hash=BLOCK_B, + ) + original = store._write_mutable_json + + def replace_then_fail( + path: Path, + *args: object, + **kwargs: object, + ) -> object: + if path == store.evidence_path: + competitor_path.unlink() + competitor_path.write_bytes(b"competitor") + raise OSError("evidence") + return original(path, *args, **kwargs) + + with mock.patch.object( + store, + "_write_mutable_json", + side_effect=replace_then_fail, + ), self.assertRaises(OSError): + store.publish_success( + identity=AuditPublicationIdentity(2, 2, BLOCK_B), + report=self.report(block_height=2), + persistence=self.persistence(), + evidence={}, + created_at="new", + ) + + self.assertEqual(competitor_path.read_bytes(), b"competitor") + self.assertEqual(store.latest_evidence()["block_hash"], BLOCK_A) # type: ignore[index] + + def test_directory_fsync_failure_rolls_back_previous_bytes(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + store = self.make_store(Path(tmp)) + path = store.root / "operator.json" + path.write_bytes(b"old") + real = store._fsync_directory + calls = 0 + + def fail_once(parent: Path) -> None: + nonlocal calls + calls += 1 + if calls == 1: + raise OSError("dir fsync") + real(parent) + + with mock.patch.object(store, "_fsync_directory", side_effect=fail_once): + with self.assertRaises(OSError): + store._write_mutable_bytes(path, b"new") + self.assertEqual(path.read_bytes(), b"old") + + def test_directory_fsync_rollback_preserves_competing_replacement(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + store = self.make_store(Path(tmp)) + path = store.root / "mutable.json" + path.write_bytes(b"old") + real = store._fsync_directory + calls = 0 + + def swap_then_fail(parent: Path) -> None: + nonlocal calls + calls += 1 + if calls == 1: + path.unlink() + path.write_bytes(b"competitor") + raise OSError("dir fsync") + real(parent) + + with mock.patch.object(store, "_fsync_directory", side_effect=swap_then_fail): + with self.assertRaises(OSError): + store._write_mutable_bytes(path, b"new") + self.assertEqual(path.read_bytes(), b"competitor") + + def test_identity_cleanup_never_moves_an_existing_replacement(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + store = self.make_store(root) + path = root / "owned.json" + path.write_bytes(b"owned-a") + owned = _FileIdentity.from_stat(path.stat()) + path.unlink() + path.write_bytes(b"competitor-b") + replacement = path.stat() + # Force the Linux ABA case even on filesystems that do not + # immediately reuse the unlinked inode in this test. + owned = dataclass_replace( + owned, + device=replacement.st_dev, + inode=replacement.st_ino, + ) + with mock.patch.object( + store, + "_owned_replace", + side_effect=AssertionError("replacement must not move"), + ): + self.assertFalse(store._remove_identity_safe(path, owned)) + + self.assertEqual(path.read_bytes(), b"competitor-b") + quarantines = list(root.glob(".owned.json.*.cleanup")) + self.assertEqual(quarantines, []) + + def test_identity_cleanup_never_relocates_nonregular_replacement(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + store = self.make_store(root) + path = root / "owned.json" + path.write_bytes(b"owned") + owned = _FileIdentity.from_stat(path.stat()) + path.unlink() + path.mkdir() + + self.assertFalse(store._remove_identity_safe(path, owned)) + self.assertTrue(path.is_dir()) + self.assertEqual(list(root.glob(".owned.json.*.cleanup")), []) + + path.rmdir() + target = root / "operator" + target.write_bytes(b"operator") + path.symlink_to(target) + self.assertFalse(store._remove_identity_safe(path, owned)) + self.assertTrue(path.is_symlink()) + self.assertEqual(path.read_bytes(), b"operator") + + def test_atomic_stage_faults_preserve_target_and_clean_owned_temps(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + store = self.make_store(Path(tmp)) + path = store.root / "mutable.json" + path.write_bytes(b"old") + with mock.patch("os.fsync", side_effect=OSError("file fsync")): + with self.assertRaises(OSError): + store._write_mutable_bytes(path, b"new") + self.assertEqual(path.read_bytes(), b"old") + self.assertEqual(list(store.root.glob(".*.tmp")), []) + + real_replace = os.replace + + def fail_publish( + source: object, + target: object, + **kwargs: object, + ) -> None: + target_path = Path(target) + if kwargs.get("dst_dir_fd") == store._root_fd: + target_path = store.root / target_path + if target_path == path: + raise OSError("replace") + real_replace(source, target, **kwargs) + + with mock.patch("os.replace", side_effect=fail_publish): + with self.assertRaises(OSError): + store._write_mutable_bytes(path, b"new") + self.assertEqual(path.read_bytes(), b"old") + self.assertEqual(list(store.root.glob(".*.tmp")), []) + + token = "66" * 16 + stage = store.root / f".{path.name}.{token}.tmp" + stage.write_bytes(b"preexisting") + with mock.patch( + "lab.prism.audit_artifacts.uuid.uuid4", + return_value=mock.Mock(hex=token), + ): + with self.assertRaises(FileExistsError): + store._write_mutable_bytes(path, b"new") + self.assertEqual(stage.read_bytes(), b"preexisting") + self.assertEqual(path.read_bytes(), b"old") + + def test_immutable_primitive_rejects_arbitrary_owned_root_targets(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + store = self.make_store(Path(tmp)) + target = store.root / "operator-selected.json" + with self.assertRaisesRegex(RuntimeError, "is not an owned body"): + store._write_immutable_bytes(target, b"payload") + self.assertFalse(target.exists()) + + def test_immutable_publish_fsync_failure_is_recoverable_and_idempotent(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + store = self.make_store(Path(tmp)) + target = store.body_path(BLOCK_A, DIGEST) + real_fsync = store._fsync_directory + calls = 0 + + def fail_once(parent: Path) -> None: + nonlocal calls + calls += 1 + if calls == 1: + raise OSError("dir fsync") + real_fsync(parent) + + with mock.patch.object( + store, + "_fsync_directory", + side_effect=fail_once, + ), self.assertRaisesRegex(OSError, "dir fsync"): + store._write_immutable_bytes(target, b"payload") + self.assertFalse(target.exists()) + self.assertEqual(list(store.root.glob(".*.tmp")), []) + + store._write_immutable_bytes(target, b"payload") + with mock.patch.object( + store, + "_fsync_directory", + wraps=real_fsync, + ) as fsync: + store._write_immutable_bytes(target, b"payload") + fsync.assert_called_once_with(target.parent) + with self.assertRaisesRegex(RuntimeError, "does not match"): + store._write_immutable_bytes(target, b"different") + self.assertEqual(target.read_bytes(), b"payload") + + def test_compatibility_none_override_is_stable_and_defensive(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + store = self.make_store(root) + store.publish_success( + identity=AuditPublicationIdentity(1, 1, BLOCK_A), + report=self.report(), + persistence=self.persistence(), + evidence={"nested": {"value": 1}}, + created_at="now", + ) + store.set_latest_evidence_for_compatibility(None) + self.assertIsNone(store.latest_evidence()) + seed = {"nested": {"value": 2}} + store.set_latest_evidence_for_compatibility(seed) + seed["nested"]["value"] = 3 + first = store.latest_evidence() + assert first is not None + self.assertEqual(first["nested"]["value"], 2) + first["nested"]["value"] = 4 + self.assertEqual(store.latest_evidence()["nested"]["value"], 2) # type: ignore[index] + + def test_compact_body_and_segment_headers_are_identity_bound(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + store = self.make_store( + Path(tmp), + canonicalizer=lambda value: json.dumps( + value, + separators=(",", ":"), + ).encode(), + share_segment_size=2, + ) + logical = {"schema": "example", "shares": [{"share_seq": 1}]} + digest = hashlib.sha256( + json.dumps(logical, separators=(",", ":")).encode() + ).hexdigest() + body_path = store.body_path(BLOCK_A, digest) + wrapper = { + "schema": "qbit.prism.audit-body-ref.v1", + "block_hash": BLOCK_B, + "audit_bundle_sha256": digest, + "audit_bundle_schema": "example", + "share_count": 1, + "share_segment_size": 2, + "shares_key_index": 1, + "bundle_without_shares": {"schema": "example"}, + "share_parts": [{ + "kind": "inline", + "first_share_seq": 1, + "last_share_seq": 1, + "share_count": 1, + "shares": [{"share_seq": 1}], + }], + } + body_path.write_bytes(store.storage_json_bytes(wrapper)) + with self.assertRaisesRegex(RuntimeError, "identity mismatch"): + store.read_external_body(str(body_path), expected_sha256=digest) + + segment = { + "schema": "qbit.prism.audit-share-segment.v1", + "first_share_seq": 1, + "last_share_seq": 2, + "share_count": 2, + "shares": [{"share_seq": 1}], + } + segment_bytes = store.storage_json_bytes(segment) + segment_digest = hashlib.sha256(segment_bytes).hexdigest() + segment_path = store.root / ( + f"prism-audit-share-segment-1-2-{segment_digest}.json" + ) + segment_path.write_bytes(segment_bytes) + with self.assertRaisesRegex(RuntimeError, "header mismatch"): + store.read_audit_share_segment( + { + "kind": "segment", + "first_share_seq": 1, + "last_share_seq": 2, + "share_count": 2, + "sha256": segment_digest, + "body_uri": str(segment_path), + }, + parent_body_uri=str(body_path), + ) + + def test_memory_ledger_reactivation_preserves_publication_ordinal(self) -> None: + ledger = SingleWriterShareLedger() + self.assertEqual(ledger.audit_publication_sequence_floor(), 0) + + def persist(block_hash: str, height: int) -> None: + ledger.persist_accepted_block( + block_hash=block_hash, + block_height=height, + parent_hash=BLOCK_B, + final_bundle={}, + audit_report={}, + ) + + persist(BLOCK_A, 1) + self.assertEqual( + ledger.pool_block_state(block_hash=BLOCK_A)["chain_state"], # type: ignore[index] + "prepared", + ) + first = ledger.confirm_accepted_block(block_hash=BLOCK_A, active_tip_height=1) + replay = ledger.confirm_accepted_block(block_hash=BLOCK_A, active_tip_height=1) + self.assertEqual(ledger.audit_publication_sequence_floor(), 1) + self.assertEqual( + first["audit_publication_sequence"], + replay["audit_publication_sequence"], + ) + persist(BLOCK_A, 1) + self.assertEqual( + ledger.pool_block_state(block_hash=BLOCK_A)["chain_state"], # type: ignore[index] + "confirmed", + ) + self.assertEqual( + ledger.mark_pool_block_inactive(block_hash=BLOCK_A, active_tip_height=2)[ + "inactive_count" + ], + 1, + ) + persist(BLOCK_A, 1) + inactive_state = ledger.pool_block_state(block_hash=BLOCK_A) + self.assertEqual(inactive_state["chain_state"], "inactive") # type: ignore[index] + self.assertEqual( + inactive_state["audit_publication_sequence"], # type: ignore[index] + first["audit_publication_sequence"], + ) + wrong_height = ledger.reactivate_pool_block( + block_hash=BLOCK_A, + active_tip_height=0, + ) + self.assertEqual(wrong_height["reactivated_count"], 0) + self.assertNotIn("audit_publication_sequence", wrong_height) + reactivated = ledger.reactivate_pool_block( + block_hash=BLOCK_A, + active_tip_height=1, + ) + self.assertEqual( + reactivated["audit_publication_sequence"], + first["audit_publication_sequence"], + ) + self.assertEqual(ledger.audit_publication_sequence_floor(), 1) + replay_reactivation = ledger.reactivate_pool_block( + block_hash=BLOCK_A, + active_tip_height=1, + ) + self.assertEqual(replay_reactivation["reactivated_count"], 0) + self.assertNotIn("audit_publication_sequence", replay_reactivation) + self.assertEqual( + ledger.mark_pool_block_inactive( + block_hash=BLOCK_A, + active_tip_height=2, + )["inactive_count"], + 1, + ) + self.assertEqual( + ledger.reverse_immature_block( + block_hash=BLOCK_A, + active_tip_height=2, + )["reversed_count"], + 1, + ) + persist(BLOCK_A, 1) + reversed_state = ledger.pool_block_state(block_hash=BLOCK_A) + self.assertEqual(reversed_state["chain_state"], "reversed") # type: ignore[index] + self.assertEqual(reversed_state["maturity_state"], "reversed") # type: ignore[index] + self.assertEqual( + reversed_state["audit_publication_sequence"], # type: ignore[index] + reactivated["audit_publication_sequence"], + ) + # A reversed row is terminally disposed: confirmation reports the + # superseded disposition (-1) instead of the unexplained 0. + self.assertEqual( + ledger.confirm_accepted_block( + block_hash=BLOCK_A, + active_tip_height=1, + ), + {"backend": "memory", "confirmed_count": -1}, + ) + self.assertEqual( + ledger.reactivate_pool_block( + block_hash=BLOCK_A, + active_tip_height=1, + ), + {"backend": "memory", "reactivated_count": 0}, + ) + persist(BLOCK_B, 2) + next_publication = ledger.confirm_accepted_block( + block_hash=BLOCK_B, + active_tip_height=2, + ) + self.assertEqual(next_publication["audit_publication_sequence"], 2) + self.assertEqual(ledger.audit_publication_sequence_floor(), 2) + with self.assertRaisesRegex(RuntimeError, "mature pool block"): + ledger.reverse_immature_block( + block_hash=BLOCK_B, + active_tip_height=1002, + ) + rejected_hash = "cc" * 32 + persist(rejected_hash, 3) + self.assertEqual( + ledger.reject_prepared_block( + block_hash=rejected_hash, + active_tip_height=3, + )["rejected_count"], + 1, + ) + self.assertEqual( + ledger.reject_prepared_block( + block_hash=rejected_hash, + active_tip_height=3, + )["rejected_count"], + 0, + ) + rejected_state = ledger.pool_block_state(block_hash=rejected_hash) + self.assertEqual(rejected_state["chain_state"], "rejected") # type: ignore[index] + self.assertEqual(rejected_state["maturity_state"], "reversed") # type: ignore[index] + + def test_memory_ledger_rejects_unknown_wrong_height_and_inactive_confirmation(self) -> None: + ledger = SingleWriterShareLedger() + self.assertEqual( + ledger.confirm_accepted_block( + block_hash=BLOCK_A, + active_tip_height=1, + )["confirmed_count"], + 0, + ) + ledger.persist_accepted_block( + block_hash=BLOCK_A, + block_height=1, + parent_hash=BLOCK_B, + final_bundle={}, + audit_report={}, + ) + self.assertEqual( + ledger.confirm_accepted_block( + block_hash=BLOCK_A, + active_tip_height=2, + )["confirmed_count"], + 0, + ) + ledger.confirm_accepted_block(block_hash=BLOCK_A, active_tip_height=1) + self.assertEqual( + ledger.mark_pool_block_inactive( + block_hash=BLOCK_A, + active_tip_height=2, + )["inactive_count"], + 1, + ) + self.assertEqual( + ledger.mark_pool_block_inactive( + block_hash=BLOCK_A, + active_tip_height=2, + )["inactive_count"], + 0, + ) + # An inactive row is terminally disposed for this confirmation + # attempt: it reports superseded (-1), not the unexplained 0. + inactive_confirmation = ledger.confirm_accepted_block( + block_hash=BLOCK_A, + active_tip_height=1, + ) + self.assertEqual(inactive_confirmation["confirmed_count"], -1) + self.assertNotIn("audit_publication_sequence", inactive_confirmation) + + def test_invalid_evidence_disables_live_prune_but_not_candidate_prune(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + (root / "evidence.json").write_text("not-json", encoding="utf-8") + live = root / f"prism-live-audit-bundle-1-{BLOCK_A}.json" + live.write_text("{}", encoding="utf-8") + candidate = root / f"prism-live-audit-bundle-candidate-{BLOCK_A}.json" + candidate.write_text("{}", encoding="utf-8") + store = self.make_store( + root, + live_bundle_retention=0, + candidate_retention_seconds=0, + ) + result = store.prune_best_effort() + self.assertTrue(live.exists()) + self.assertFalse(candidate.exists()) + self.assertEqual(result.live_removed, 0) + self.assertEqual(result.candidate_removed, 1) + + def test_legacy_evidence_requires_durable_identity_upgrade(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + store = self.make_store(root) + envelope = store.live_envelope_path(block_height=2, block_hash=BLOCK_B) + envelope.write_text( + json.dumps( + { + "schema": LIVE_ENVELOPE_SCHEMA, + "block_hash": BLOCK_B, + "block_height": 2, + "audit_bundle_sha256": DIGEST, + "body_uri": "", + "body_filename": None, + "coinbase_txid": "22" * 32, + "coinbase_manifest_sha256": "33" * 32, + "coinbase_tx_hex": "00", + "coinbase_value_sats": 1, + } + ), + encoding="utf-8", + ) + legacy = { + "schema": LIVE_EVIDENCE_SCHEMA, + "block_hash": BLOCK_B, + "block_height": 2, + "audit_bundle_path": str(envelope), + "audit_report": self.report(block_height=2), + "persistence": self.persistence(), + "coinbase_txid": "22" * 32, + "coinbase_manifest_sha256_hex": "33" * 32, + "coinbase_tx_hex": "00", + "coinbase_value_sats": 1, + } + store.evidence_path.write_text(json.dumps(legacy), encoding="utf-8") + restarted = self.make_store(root) + with self.assertRaisesRegex(RuntimeError, "validated publication identity"): + restarted.publish_success( + identity=AuditPublicationIdentity(10, 1, BLOCK_A), + report=self.report(), + persistence=self.persistence(), + evidence={"audit_report": self.report(), "persistence": self.persistence()}, + created_at="later", + ) + with restarted.publication_order_guard(): + restarted.adopt_legacy_publication_identity( + AuditPublicationIdentity(20, 2, BLOCK_B), + publication_floor_sequence=20, + ) + with self.assertRaisesRegex(RuntimeError, "never exact-replay"): + restarted.publish_success( + identity=AuditPublicationIdentity(10, 1, BLOCK_A), + report=self.report(block_height=1), + persistence=self.persistence(), + evidence={ + "audit_report": self.report(), + "persistence": self.persistence(), + }, + created_at="later", + ) + + # The disk marker never grants order or pin authority by itself. + second_process = self.make_store(root) + self.assertIsNone(second_process.latest_evidence()) + self.assertEqual(second_process.publication_sequence_floor(), 0) + self.assertEqual( + second_process.legacy_evidence_identity(), + AuditPublicationIdentity(20, 2, BLOCK_B), + ) + with second_process.publication_order_guard(): + second_process.adopt_legacy_publication_identity( + AuditPublicationIdentity(20, 2, BLOCK_B), + publication_floor_sequence=20, + ) + self.assertEqual(second_process.publication_sequence_floor(), 20) + self.assertEqual( + second_process.latest_evidence()["block_hash"], # type: ignore[index] + BLOCK_B, + ) + for stale_identity in ( + AuditPublicationIdentity(20, 2, BLOCK_B), + AuditPublicationIdentity(19, 3, BLOCK_A), + ): + with self.subTest(stale_identity=stale_identity), self.assertRaisesRegex( + RuntimeError, + "never exact-replay", + ): + second_process.publish_success( + identity=stale_identity, + report=self.report(block_height=stale_identity.block_height), + persistence=self.persistence(), + evidence={}, + created_at="stale", + ) + repair = second_process.publish_success( + identity=AuditPublicationIdentity(21, 3, BLOCK_A), + report=self.report(block_height=3), + persistence=self.persistence(), + evidence={}, + created_at="repair", + ) + self.assertTrue(repair.published) + + def test_legacy_proof_token_is_revoked_by_peer_inode_replacement(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + seed = self.make_store(root) + envelope = seed.live_envelope_path( + block_height=2, + block_hash=BLOCK_B, + ) + envelope.write_text( + json.dumps( + { + "schema": LIVE_ENVELOPE_SCHEMA, + "block_hash": BLOCK_B, + "block_height": 2, + "audit_bundle_sha256": DIGEST, + "body_uri": "", + "body_filename": None, + "coinbase_txid": "22" * 32, + "coinbase_manifest_sha256": "33" * 32, + "coinbase_tx_hex": "00", + "coinbase_value_sats": 1, + } + ), + encoding="utf-8", + ) + seed.evidence_path.write_text( + json.dumps( + { + "schema": LIVE_EVIDENCE_SCHEMA, + "block_hash": BLOCK_B, + "block_height": 2, + "audit_bundle_path": str(envelope), + "audit_report": self.report(block_height=2), + "persistence": self.persistence(), + "coinbase_txid": "22" * 32, + "coinbase_manifest_sha256_hex": "33" * 32, + "coinbase_tx_hex": "00", + "coinbase_value_sats": 1, + } + ), + encoding="utf-8", + ) + seed.close() + identity = AuditPublicationIdentity(20, 2, BLOCK_B) + proven = self.make_store(root) + with proven.publication_order_guard(): + proven.adopt_legacy_publication_identity( + identity, + publication_floor_sequence=20, + ) + self.assertIsNotNone(proven._legacy_proof_token) + self.assertEqual(proven.publication_sequence_floor(), 20) + original_evidence_inode = _FileIdentity.from_stat( + proven.evidence_path.stat() + ) + + peer = self.make_store(root) + with peer.publication_order_guard(): + peer.adopt_legacy_publication_identity( + identity, + publication_floor_sequence=20, + ) + self.assertFalse( + original_evidence_inode.matches(proven.evidence_path.stat()) + ) + + proven.prune_best_effort() + self.assertIsNone(proven._legacy_proof_token) + self.assertIsNone(proven.latest_evidence()) + self.assertEqual(proven.publication_sequence_floor(), 0) + self.assertEqual(proven.legacy_evidence_identity(), identity) + + def test_unprovable_legacy_evidence_is_repairable(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + store = self.make_store(Path(tmp)) + store._evidence_state = "legacy" + store._latest_evidence = {"block_hash": BLOCK_A, "block_height": 1} + store._current_identity = AuditPublicationIdentity(0, 1, BLOCK_A) + store.invalidate_unprovable_legacy_evidence() + repaired = store.publish_success( + identity=AuditPublicationIdentity(2, 2, BLOCK_B), + report=self.report(block_height=2), + persistence=self.persistence(), + evidence={}, + created_at="now", + ) + self.assertTrue(repaired.published) + + def test_disk_legacy_invalidation_is_sticky_until_durable_repair(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + store = self.make_store(root, live_bundle_retention=0) + envelope = store.live_envelope_path(block_height=2, block_hash=BLOCK_B) + envelope.write_text( + json.dumps( + { + "schema": LIVE_ENVELOPE_SCHEMA, + "block_hash": BLOCK_B, + "block_height": 2, + "audit_bundle_sha256": DIGEST, + "body_uri": "", + "body_filename": None, + "coinbase_txid": "22" * 32, + "coinbase_manifest_sha256": "33" * 32, + "coinbase_tx_hex": "00", + "coinbase_value_sats": 1, + } + ), + encoding="utf-8", + ) + legacy = { + "schema": LIVE_EVIDENCE_SCHEMA, + "block_hash": BLOCK_B, + "block_height": 2, + "audit_bundle_path": str(envelope), + "audit_report": self.report(block_height=2), + "persistence": self.persistence(), + "coinbase_txid": "22" * 32, + "coinbase_manifest_sha256_hex": "33" * 32, + "coinbase_tx_hex": "00", + "coinbase_value_sats": 1, + } + store.evidence_path.write_text(json.dumps(legacy), encoding="utf-8") + restarted = self.make_store(root, live_bundle_retention=0) + restarted.invalidate_unprovable_legacy_evidence() + self.assertIsNone(restarted.latest_evidence()) + self.assertEqual(restarted.prune_best_effort().live_removed, 0) + repaired = restarted.publish_success( + identity=AuditPublicationIdentity(21, 3, BLOCK_A), + report=self.report(block_height=3), + persistence=self.persistence(), + evidence={}, + created_at="repair", + ) + self.assertTrue(repaired.published) + self.assertEqual(restarted.latest_evidence()["block_hash"], BLOCK_A) # type: ignore[index] + + def test_retention_preserves_current_and_unowned_entries(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + store = self.make_store(root, live_bundle_retention=0) + publication = store.publish_success( + identity=AuditPublicationIdentity(1, 1, BLOCK_A), + report=self.report(), + persistence=self.persistence(), + evidence={"audit_report": self.report(), "persistence": self.persistence()}, + created_at="now", + ) + lookalike = root / f"prism-live-audit-bundle-1-{BLOCK_B}.json.bak" + lookalike.write_text("operator", encoding="utf-8") + link = root / f"prism-live-audit-bundle-2-{BLOCK_B}.json" + link.symlink_to(lookalike) + store.prune_best_effort() + self.assertTrue(publication.envelope_path.exists()) + self.assertTrue(lookalike.exists()) + self.assertTrue(link.is_symlink()) + + def test_retention_ties_pin_active_candidate_and_never_remove_bodies(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + store = self.make_store( + root, + live_bundle_retention=2, + candidate_retention_seconds=0, + ) + current = store.publish_success( + identity=AuditPublicationIdentity(1, 0, BLOCK_A), + report=self.report(block_height=0), + persistence=self.persistence(), + evidence={}, + created_at="current", + ) + first = store.live_envelope_path(block_height=1, block_hash=BLOCK_A) + second = store.live_envelope_path(block_height=2, block_hash=BLOCK_B) + first.write_text("{}", encoding="utf-8") + second.write_text("{}", encoding="utf-8") + os.utime(first, ns=(1, 1)) + os.utime(second, ns=(1, 1)) + active = store.issue_candidate(block_hash=BLOCK_A) + active.path.write_bytes(b"active") + self.transfer_candidate(store, active) + body = store.body_path(BLOCK_A, DIGEST) + body.write_bytes(b"body") + result = store.prune_best_effort() + self.assertEqual(result.live_removed, 1) + self.assertTrue(current.envelope_path.exists()) + self.assertTrue(active.path.exists()) + self.assertTrue(body.exists()) + store.discard_candidate(active) + + def test_body_segment_and_evidence_symlinks_fail_no_follow(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + store = self.make_store(root) + outside = root / "outside" + outside.write_text("{}", encoding="utf-8") + body = store.body_path(BLOCK_A, DIGEST) + body.symlink_to(outside) + with self.assertRaisesRegex(RuntimeError, "not retrievable"): + store.read_external_body(str(body), expected_sha256=DIGEST) + store.evidence_path.symlink_to(outside) + self.assertIsNone(self.make_store(root).latest_evidence()) + + def test_metrics_ignore_symlinks_and_classify_malformed_as_other(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + store = self.make_store(root) + body = root / f"prism-audit-bundle-body-{BLOCK_A}-{DIGEST}.json" + body.write_bytes(b"body") + malformed = root / "prism-audit-bundle-body-operator.json" + malformed.write_bytes(b"x") + (root / f"prism-live-audit-bundle-1-{BLOCK_A}.json").symlink_to(body) + metrics = store.metrics_snapshot() + self.assertEqual(metrics["body"], {"files": 1, "bytes": 4}) + self.assertEqual(metrics["other"], {"files": 1, "bytes": 1}) + + def test_mutable_share_slot_merge_preserves_prior_range(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + store = self.make_store(Path(tmp), share_segment_size=4) + uri, _digest = store.write_audit_share_segment_range( + segment_first_share_seq=1, + segment_last_share_seq=4, + first_share_seq=1, + last_share_seq=2, + shares=[{"share_seq": 1}, {"share_seq": 2}], + ) + store.write_audit_share_segment_range( + segment_first_share_seq=1, + segment_last_share_seq=4, + first_share_seq=3, + last_share_seq=4, + shares=[{"share_seq": 3}, {"share_seq": 4}], + ) + payload = json.loads(Path(uri).read_text(encoding="utf-8")) + self.assertEqual([row["share_seq"] for row in payload["shares"]], [1, 2, 3, 4]) + + def test_concurrent_mutable_share_slot_merge_is_serialized_and_lossless( + self, + ) -> None: + rows = [ + { + "share_seq": share_seq, + "worker": f"miner-{share_seq}", + "accepted": True, + } + for share_seq in range(1, 5) + ] + scenarios = ( + ("disjoint", rows[:2], rows[2:]), + ("identical_overlap", rows[:3], rows[1:]), + ) + + for scenario, first_shares, second_shares in scenarios: + with self.subTest(scenario=scenario), tempfile.TemporaryDirectory() as tmp: + store = self.make_store(Path(tmp), share_segment_size=4) + first_name = f"audit-slot-{scenario}-first" + second_name = f"audit-slot-{scenario}-second" + first_at_write = threading.Event() + release_first = threading.Event() + second_lock_attempt = threading.Event() + second_lock_acquired = threading.Event() + second_at_write = threading.Event() + write_call_lock = threading.Lock() + write_call_threads: list[str] = [] + results: dict[str, tuple[str, str]] = {} + errors: dict[str, BaseException] = {} + original_lock = store._lock + # The slot writer streams its chunks through the bounded + # mutable primitive (issue #255); gate that primitive. + original_write_mutable_chunks = store._write_mutable_chunks + + class ObservedLock: + def __enter__(self) -> "ObservedLock": + if threading.current_thread().name == second_name: + second_lock_attempt.set() + original_lock.acquire() + if threading.current_thread().name == second_name: + second_lock_acquired.set() + return self + + def __exit__( + self, + _exc_type: object, + _exc_value: object, + _traceback: object, + ) -> None: + original_lock.release() + + def gated_write(path: Path, chunks_factory: object) -> None: + thread_name = threading.current_thread().name + with write_call_lock: + write_call_threads.append(thread_name) + if thread_name == first_name: + first_at_write.set() + if not release_first.wait(timeout=5.0): + raise AssertionError("first share-slot writer was not released") + elif thread_name == second_name: + second_at_write.set() + original_write_mutable_chunks(path, chunks_factory) + + def write_range(label: str, shares: list[dict[str, object]]) -> None: + try: + results[label] = store.write_audit_share_segment_range( + segment_first_share_seq=1, + segment_last_share_seq=4, + first_share_seq=int(shares[0]["share_seq"]), + last_share_seq=int(shares[-1]["share_seq"]), + shares=shares, + ) + except BaseException as exc: + errors[label] = exc + + first_thread = threading.Thread( + target=write_range, + args=("first", first_shares), + name=first_name, + ) + second_thread = threading.Thread( + target=write_range, + args=("second", second_shares), + name=second_name, + ) + second_started = False + with mock.patch.object( + store, + "_lock", + ObservedLock(), + ), mock.patch.object( + store, + "_write_mutable_chunks", + side_effect=gated_write, + ): + first_thread.start() + try: + self.assertTrue( + first_at_write.wait(timeout=5.0), + "first writer did not reach the gated mutable write", + ) + second_thread.start() + second_started = True + self.assertTrue( + second_lock_attempt.wait(timeout=5.0), + "second writer did not attempt the store lock", + ) + self.assertTrue(first_thread.is_alive()) + self.assertTrue(second_thread.is_alive()) + self.assertFalse(second_lock_acquired.is_set()) + self.assertFalse(second_at_write.is_set()) + finally: + release_first.set() + first_thread.join(timeout=5.0) + if second_started: + second_thread.join(timeout=5.0) + + self.assertFalse(first_thread.is_alive()) + self.assertFalse(second_thread.is_alive()) + active_names = {thread.name for thread in threading.enumerate()} + self.assertNotIn(first_name, active_names) + self.assertNotIn(second_name, active_names) + self.assertEqual(errors, {}) + self.assertTrue(second_lock_acquired.is_set()) + self.assertTrue(second_at_write.is_set()) + self.assertEqual(write_call_threads, [first_name, second_name]) + + self.assertEqual(results["first"][0], results["second"][0]) + slot_path = Path(results["first"][0]) + expected_payload = store.audit_share_segment_payload( + first_share_seq=1, + last_share_seq=4, + shares=rows, + ) + expected_bytes = store.storage_json_bytes(expected_payload) + stored_bytes = slot_path.read_bytes() + self.assertEqual(stored_bytes, expected_bytes) + stored = json.loads(stored_bytes) + sequences = [int(row["share_seq"]) for row in stored["shares"]] + self.assertEqual(sequences, [1, 2, 3, 4]) + self.assertEqual(len(sequences), len(set(sequences))) + self.assertEqual(stored["shares"], rows) + + inputs = {"first": first_shares, "second": second_shares} + for label, shares in inputs.items(): + incoming_payload = store.audit_share_segment_payload( + first_share_seq=int(shares[0]["share_seq"]), + last_share_seq=int(shares[-1]["share_seq"]), + shares=shares, + ) + expected_digest = hashlib.sha256( + store.storage_json_bytes(incoming_payload) + ).hexdigest() + self.assertEqual(results[label][1], expected_digest) + + before_retry = slot_path.stat() + for label, shares in inputs.items(): + retry = store.write_audit_share_segment_range( + segment_first_share_seq=1, + segment_last_share_seq=4, + first_share_seq=int(shares[0]["share_seq"]), + last_share_seq=int(shares[-1]["share_seq"]), + shares=shares, + ) + self.assertEqual(retry, results[label]) + after_retry = slot_path.stat() + self.assertEqual(slot_path.read_bytes(), expected_bytes) + self.assertEqual( + (after_retry.st_dev, after_retry.st_ino, after_retry.st_mtime_ns), + (before_retry.st_dev, before_retry.st_ino, before_retry.st_mtime_ns), + ) + + conflicting_share = {**rows[1], "worker": "conflicting-miner"} + with self.assertRaisesRegex( + RuntimeError, + "conflicts at share_seq 2", + ): + store.write_audit_share_segment_range( + segment_first_share_seq=1, + segment_last_share_seq=4, + first_share_seq=2, + last_share_seq=2, + shares=[conflicting_share], + ) + after_conflict = slot_path.stat() + self.assertEqual(slot_path.read_bytes(), expected_bytes) + self.assertEqual( + ( + after_conflict.st_dev, + after_conflict.st_ino, + after_conflict.st_mtime_ns, + ), + (after_retry.st_dev, after_retry.st_ino, after_retry.st_mtime_ns), + ) + + +class AuditShareSegmentRecoveryTests(unittest.TestCase): + """Memo §4 recovery: store-owned merge/conflict/quarantine, ledger callback.""" + + def make_store(self, root: Path) -> AuditArtifactStore: + return AuditArtifactStore( + AuditArtifactConfig( + root=root, + evidence_path=root / "evidence.json", + share_segment_size=4, + ), + canonicalizer=lambda bundle: json.dumps( + bundle, + separators=(",", ":"), + ).encode("utf-8"), + ) + + @staticmethod + def rows(first: int, last: int) -> list[dict[str, object]]: + return [ + {"share_seq": share_seq, "worker": f"miner-{share_seq}"} + for share_seq in range(first, last + 1) + ] + + def write_range( + self, + store: AuditArtifactStore, + first: int, + last: int, + *, + loader: object = None, + shares: list[dict[str, object]] | None = None, + ) -> tuple[str, str]: + return store.write_audit_share_segment_range( + segment_first_share_seq=1, + segment_last_share_seq=4, + first_share_seq=first, + last_share_seq=last, + shares=self.rows(first, last) if shares is None else shares, + load_missing_range=loader, + ) + + def test_slot_gap_is_repaired_from_the_ledger_callback(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + store = self.make_store(Path(tmp)) + uri, _digest = self.write_range(store, 1, 2) + calls: list[tuple[int, int]] = [] + + def loader(*, first_share_seq: int, last_share_seq: int) -> list[dict[str, object]]: + calls.append((first_share_seq, last_share_seq)) + return self.rows(first_share_seq, last_share_seq) + + gap_uri, _range_digest = self.write_range(store, 4, 4, loader=loader) + self.assertEqual(calls, [(3, 3)]) + self.assertEqual(gap_uri, uri) + payload = json.loads(Path(uri).read_text(encoding="utf-8")) + self.assertEqual( + [row["share_seq"] for row in payload["shares"]], + [1, 2, 3, 4], + ) + + def test_unrepairable_gap_preserves_slot_and_quarantines(self) -> None: + for loader in ( + None, + lambda **_kwargs: (_ for _ in ()).throw(RuntimeError("backfill down")), + lambda **_kwargs: [{"share_seq": 3}, {"share_seq": 9}], + ): + with self.subTest(loader=loader), tempfile.TemporaryDirectory() as tmp: + store = self.make_store(Path(tmp)) + uri, _digest = self.write_range(store, 1, 2) + slot_bytes = Path(uri).read_bytes() + incoming = self.rows(4, 4) + expected_digest = hashlib.sha256( + store.storage_json_bytes( + store.audit_share_segment_payload( + first_share_seq=4, + last_share_seq=4, + shares=incoming, + ) + ) + ).hexdigest() + fresh_uri, range_digest = self.write_range( + store, + 4, + 4, + loader=loader, + shares=incoming, + ) + # The stable slot survives byte-for-byte; the incoming range + # gets an immutable content-addressed URI and finalization + # continues. + self.assertEqual(Path(uri).read_bytes(), slot_bytes) + self.assertNotEqual(fresh_uri, uri) + self.assertEqual(range_digest, expected_digest) + self.assertIn(expected_digest, Path(fresh_uri).name) + conflicts = [ + name + for name in os.listdir(tmp) + if ".conflict-" in name + ] + self.assertEqual(len(conflicts), 1) + self.assertEqual( + (Path(tmp) / conflicts[0]).read_bytes(), + slot_bytes, + ) + + def test_conflicting_payload_still_raises(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + store = self.make_store(Path(tmp)) + self.write_range(store, 1, 2) + conflicting = [{"share_seq": 2, "worker": "other-miner"}] + with self.assertRaisesRegex( + _AuditShareSegmentConflict, + "conflicts at share_seq 2", + ): + self.write_range( + store, + 2, + 2, + loader=lambda **kwargs: self.rows( + kwargs["first_share_seq"], + kwargs["last_share_seq"], + ), + shares=conflicting, + ) + + def test_quarantine_snapshot_is_idempotent_for_equal_bytes(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + store = self.make_store(Path(tmp)) + uri, _digest = self.write_range(store, 1, 2) + slot_bytes = Path(uri).read_bytes() + first = store.quarantine_audit_share_segment( + Path(uri), + expected_bytes=slot_bytes, + ) + second = store.quarantine_audit_share_segment( + Path(uri), + expected_bytes=slot_bytes, + ) + self.assertEqual(first, second) + self.assertEqual(first.read_bytes(), slot_bytes) + third = store.quarantine_audit_share_segment( + Path(uri), + expected_bytes=b"different-earlier-bytes", + ) + self.assertNotEqual(third, first) + + def test_range_parts_thread_the_recovery_callback(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + store = self.make_store(Path(tmp)) + self.write_range(store, 1, 2) + calls: list[tuple[int, int]] = [] + + def loader(*, first_share_seq: int, last_share_seq: int) -> list[dict[str, object]]: + calls.append((first_share_seq, last_share_seq)) + return self.rows(first_share_seq, last_share_seq) + + parts = store.audit_share_range_parts( + self.rows(4, 4), + load_missing_range=loader, + ) + self.assertEqual(calls, [(3, 3)]) + assert parts is not None + self.assertEqual(parts[0]["kind"], "segment_range") + self.assertEqual(parts[0]["first_share_seq"], 4) + + +class CanonicalAuditBundleTests(unittest.TestCase): + """Phase 2: the exact bytes behind a published digest stay retrievable.""" + + def make_store( + self, + root: Path, + *, + share_segment_size: int = 2, + counter: list[int] | None = None, + **kwargs: object, + ) -> AuditArtifactStore: + def canonicalizer(bundle: dict[str, object]) -> bytes: + if counter is not None: + counter.append(1) + return json.dumps(bundle, separators=(",", ":")).encode("utf-8") + + return AuditArtifactStore( + AuditArtifactConfig( + root=root, + evidence_path=root / "evidence.json", + share_segment_size=share_segment_size, + **kwargs, # type: ignore[arg-type] + ), + canonicalizer=canonicalizer, + ) + + @staticmethod + def bundle(share_count: int = 4) -> dict[str, object]: + return { + "schema": "example", + "shares": [ + {"share_seq": share_seq, "worker": f"miner-{share_seq}"} + for share_seq in range(1, share_count + 1) + ], + } + + @staticmethod + def canonical(bundle: dict[str, object]) -> bytes: + return json.dumps(bundle, separators=(",", ":")).encode("utf-8") + + def digest(self, bundle: dict[str, object]) -> str: + return hashlib.sha256(self.canonical(bundle)).hexdigest() + + def publish( + self, + store: AuditArtifactStore, + bundle: dict[str, object], + *, + block_hash: str = BLOCK_A, + ) -> tuple[str, str]: + digest = self.digest(bundle) + body_uri = str(store.body_path(block_hash, digest)) + published = store.prepare_external_audit_body( + {"block_hash": block_hash, "audit_bundle_sha256": digest}, + bundle, + body_uri=body_uri, + ) + assert published is not None + return published, digest + + def test_publication_verifies_the_digest_before_any_store_side_effect(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + store = self.make_store(root) + bundle = self.bundle() + wrong = "cc" * 32 + with self.assertRaisesRegex(RuntimeError, "sha256 mismatch"): + store.write_canonical_audit_bundle( + BLOCK_A, + wrong, + self.canonical(bundle), + ) + self.assertEqual( + list(root.glob("prism-audit-bundle-canonical-*")), + [], + ) + + def test_publication_is_deterministic_reproducible_and_idempotent(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + store = self.make_store(root) + bundle = self.bundle() + digest = self.digest(bundle) + path = store.write_canonical_audit_bundle( + BLOCK_A, + digest, + self.canonical(bundle), + ) + self.assertEqual( + path.name, + f"prism-audit-bundle-canonical-{BLOCK_A}-{digest}.json.gz", + ) + first = path.read_bytes() + # mtime=0 and no FNAME field keep the header a constant. + self.assertEqual(first[:10].hex(), "1f8b08000000000002ff") + self.assertEqual(gzip.decompress(first), self.canonical(bundle)) + self.assertEqual(first, gzip_canonical_bundle_bytes(self.canonical(bundle))) + before = path.stat() + repeat = store.write_canonical_audit_bundle( + BLOCK_A, + digest, + self.canonical(bundle), + ) + after = repeat.stat() + self.assertEqual(path.read_bytes(), first) + self.assertEqual((before.st_dev, before.st_ino), (after.st_dev, after.st_ino)) + + def test_republication_accepts_equivalent_gzip_from_another_compressor( + self, + ) -> None: + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + store = self.make_store(root) + bundle = self.bundle() + payload = self.canonical(bundle) + digest = self.digest(bundle) + path = store.write_canonical_audit_bundle(BLOCK_A, digest, payload) + + alternative = gzip.compress(payload, compresslevel=1, mtime=0) + self.assertNotEqual(alternative, path.read_bytes()) + self.assertEqual(gzip.decompress(alternative), payload) + path.write_bytes(alternative) + before = path.stat() + + repeat = store.write_canonical_audit_bundle(BLOCK_A, digest, payload) + + after = repeat.stat() + self.assertEqual(repeat.read_bytes(), alternative) + self.assertEqual((before.st_dev, before.st_ino), (after.st_dev, after.st_ino)) + + def test_conflicting_or_corrupt_artifact_fails_loudly(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + store = self.make_store(root) + bundle = self.bundle() + digest = self.digest(bundle) + path = store.canonical_bundle_path(BLOCK_A, digest) + path.write_bytes(gzip_canonical_bundle_bytes(b"{}")) + with self.assertRaisesRegex(RuntimeError, "does not match payload"): + store.write_canonical_audit_bundle( + BLOCK_A, + digest, + self.canonical(bundle), + ) + path.write_bytes(b"not gzip at all") + with self.assertRaisesRegex(RuntimeError, "does not match payload"): + store.write_canonical_audit_bundle( + BLOCK_A, + digest, + self.canonical(bundle), + ) + + def test_post_publication_verification_catches_a_bad_publish(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + store = self.make_store(root) + bundle = self.bundle() + digest = self.digest(bundle) + path = store.canonical_bundle_path(BLOCK_A, digest) + + def publish_other_bytes(target: Path, _payload: bytes) -> None: + Path(target).write_bytes(gzip_canonical_bundle_bytes(b'{"other":1}')) + + with mock.patch.object( + store, + "_write_immutable_bytes", + side_effect=publish_other_bytes, + ): + with self.assertRaises(CanonicalAuditBundleCorrupt): + store.write_canonical_audit_bundle( + BLOCK_A, + digest, + self.canonical(bundle), + ) + path.unlink() + with mock.patch.object(store, "_write_immutable_bytes"): + with self.assertRaisesRegex(RuntimeError, "absent after publication"): + store.write_canonical_audit_bundle( + BLOCK_A, + digest, + self.canonical(bundle), + ) + + def test_corruption_is_distinguishable_from_a_clean_miss(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + store = self.make_store(root) + bundle = self.bundle() + digest = self.digest(bundle) + self.assertIsNone(store.read_canonical_audit_bundle(BLOCK_A, digest)) + path = store.write_canonical_audit_bundle( + BLOCK_A, + digest, + self.canonical(bundle), + ) + self.assertEqual( + store.read_canonical_audit_bundle(BLOCK_A, digest), + self.canonical(bundle), + ) + # A miss for a different block stays a miss, not a failure. + self.assertIsNone(store.read_canonical_audit_bundle(BLOCK_B, digest)) + + path.write_bytes(path.read_bytes()[:12]) + with self.assertRaisesRegex( + CanonicalAuditBundleCorrupt, + "does not decompress", + ): + store.read_canonical_audit_bundle(BLOCK_A, digest) + + path.write_bytes(gzip_canonical_bundle_bytes(b'{"schema":"other"}')) + with self.assertRaisesRegex(CanonicalAuditBundleCorrupt, "hash mismatch"): + store.read_canonical_audit_bundle(BLOCK_A, digest) + + path.unlink() + outside = root / "outside.gz" + outside.write_bytes(gzip_canonical_bundle_bytes(self.canonical(bundle))) + path.symlink_to(outside) + with self.assertRaisesRegex( + CanonicalAuditBundleCorrupt, + "not retrievable", + ): + store.read_canonical_audit_bundle(BLOCK_A, digest) + + def test_root_replacement_revokes_canonical_publication_and_reads(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + base = Path(tmp) + root = base / "audit" + root.mkdir() + store = self.make_store(root) + bundle = self.bundle() + digest = self.digest(bundle) + store.write_canonical_audit_bundle( + BLOCK_A, + digest, + self.canonical(bundle), + ) + + pinned = base / "audit-pinned" + root.rename(pinned) + root.mkdir() + for operation in ( + lambda: store.canonical_bundle_path(BLOCK_A, digest), + lambda: store.write_canonical_audit_bundle( + BLOCK_A, + digest, + self.canonical(bundle), + ), + lambda: store.read_canonical_audit_bundle(BLOCK_A, digest), + ): + with self.assertRaisesRegex(RuntimeError, "root identity"): + operation() + self.assertEqual(list(root.iterdir()), []) + + replacement = base / "audit-replacement" + root.rename(replacement) + pinned.rename(root) + self.assertEqual( + store.read_canonical_audit_bundle(BLOCK_A, digest), + self.canonical(bundle), + ) + + def test_kind_metrics_and_pruning_preserve_canonical_bundles(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + store = self.make_store( + root, + live_bundle_retention=0, + candidate_retention_seconds=0, + ) + bundle = self.bundle() + digest = self.digest(bundle) + name = f"prism-audit-bundle-canonical-{BLOCK_A}-{digest}.json.gz" + self.assertEqual(AuditArtifactStore.artifact_kind(name), "canonical_bundle") + for invalid in ( + f"prism-audit-bundle-canonical-{BLOCK_A}-{digest}.json", + f"prism-audit-bundle-canonical-{BLOCK_A}.json.gz", + f"prism-audit-bundle-canonical-{BLOCK_A}-{digest.upper()}.json.gz", + ): + with self.subTest(invalid=invalid): + self.assertEqual(AuditArtifactStore.artifact_kind(invalid), "other") + + path = store.write_canonical_audit_bundle( + BLOCK_A, + digest, + self.canonical(bundle), + ) + metrics = store.metrics_snapshot() + self.assertEqual( + metrics["canonical_bundle"], + {"files": 1, "bytes": path.stat().st_size}, + ) + + # Retention only ever unlinks the two expendable kinds, and a + # canonical bundle is classified as neither. + self.assertNotIn( + AuditArtifactStore.artifact_kind(path.name), + {"live_bundle", "candidate"}, + ) + stale = store.issue_candidate(block_hash=BLOCK_B) + stale.path.write_bytes(b"stale") + store.release_candidate(stale) + os.utime(stale.path, ns=(1, 1)) + result = store.prune_best_effort() + self.assertEqual(result.errors, 0) + self.assertEqual(result.candidate_removed, 1) + self.assertFalse(stale.path.exists()) + self.assertTrue(path.exists()) + self.assertEqual( + store.read_canonical_audit_bundle(BLOCK_A, digest), + self.canonical(bundle), + ) + + def test_write_path_persists_the_canonical_bytes_it_already_holds(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + calls: list[int] = [] + store = self.make_store(root, counter=calls) + bundle = self.bundle() + body_uri, digest = self.publish(store, bundle) + self.assertEqual( + store.read_canonical_audit_bundle(BLOCK_A, digest), + self.canonical(bundle), + ) + # One serialization for the digest check, reused for publication; + # the canonical copy is never re-derived. + self.assertEqual(len(calls), 1) + self.assertTrue(Path(body_uri).exists()) + + # Re-persisting the same block is a no-op on both artifacts. + body_bytes = Path(body_uri).read_bytes() + canonical_path = store.canonical_bundle_path(BLOCK_A, digest) + canonical_bytes = canonical_path.read_bytes() + self.publish(store, bundle) + self.assertEqual(Path(body_uri).read_bytes(), body_bytes) + self.assertEqual(canonical_path.read_bytes(), canonical_bytes) + + def test_write_path_reuses_a_verified_canonical_candidate_verbatim(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + calls: list[int] = [] + store = self.make_store(root, counter=calls) + bundle = self.bundle() + digest = self.digest(bundle) + candidate = root / "candidate.json" + candidate.write_bytes(self.canonical(bundle)) + calls.clear() + store.prepare_external_audit_body( + {"block_hash": BLOCK_A, "audit_bundle_sha256": digest}, + bundle, + body_uri=str(store.body_path(BLOCK_A, digest)), + canonical_bundle_path=candidate, + ) + # The candidate bytes are taken as-is: nothing is reserialized. + self.assertEqual(calls, []) + self.assertEqual( + store.read_canonical_audit_bundle(BLOCK_A, digest), + self.canonical(bundle), + ) + + def test_backfill_helper_repairs_missing_segments_under_the_body_digest(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + store = self.make_store(root) + bundle = self.bundle() + body_uri, digest = self.publish(store, bundle) + for segment in root.glob("prism-audit-share-segment-*"): + segment.unlink() + requested: list[tuple[int, int]] = [] + + def loader( + *, + first_share_seq: int, + last_share_seq: int, + ) -> list[dict[str, object]]: + requested.append((first_share_seq, last_share_seq)) + return [ + {"share_seq": share_seq, "worker": f"miner-{share_seq}"} + for share_seq in range(first_share_seq, last_share_seq + 1) + ] + + recovered = store.canonical_bundle_bytes_from_external_body( + block_hash=BLOCK_A, + audit_bundle_sha256=digest, + body_uri=body_uri, + load_missing_range=loader, + ) + self.assertEqual(recovered, self.canonical(bundle)) + self.assertEqual(requested, [(1, 2), (3, 4)]) + + # Without a callback the loss is loud rather than silently patched. + with self.assertRaisesRegex(RuntimeError, "not retrievable"): + store.canonical_bundle_bytes_from_external_body( + block_hash=BLOCK_A, + audit_bundle_sha256=digest, + body_uri=body_uri, + ) + + def test_backfill_helper_rejects_unverified_database_fallback_bytes(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + store = self.make_store(root) + bundle = self.bundle() + body_uri, digest = self.publish(store, bundle) + for segment in root.glob("prism-audit-share-segment-*"): + segment.unlink() + + def tampered( + *, + first_share_seq: int, + last_share_seq: int, + ) -> list[dict[str, object]]: + return [ + {"share_seq": share_seq, "worker": "attacker"} + for share_seq in range(first_share_seq, last_share_seq + 1) + ] + + def short( + *, + first_share_seq: int, + last_share_seq: int, + ) -> list[dict[str, object]]: + return [{"share_seq": first_share_seq, "worker": f"miner-{first_share_seq}"}] + + def failing(*, first_share_seq: int, last_share_seq: int) -> list[Any]: + raise RuntimeError("ledger unavailable") + + for loader, expected in ( + (tampered, "hash mismatch"), + (short, "not exactly contiguous"), + (failing, "could not be recovered"), + ): + with self.subTest(loader=loader.__name__): + with self.assertRaisesRegex(RuntimeError, expected): + store.canonical_bundle_bytes_from_external_body( + block_hash=BLOCK_A, + audit_bundle_sha256=digest, + body_uri=body_uri, + load_missing_range=loader, + ) + self.assertEqual( + list(root.glob("prism-audit-share-segment-*")), + [], + ) + + def test_backfill_helper_binds_the_body_uri_to_the_advertised_identity(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + store = self.make_store(root) + bundle = self.bundle() + body_uri, digest = self.publish(store, bundle) + for block_hash, expected_digest, message in ( + (BLOCK_B, digest, "block hash does not match"), + (BLOCK_A, "cc" * 32, "digest does not match"), + ): + with self.subTest(block_hash=block_hash, digest=expected_digest): + with self.assertRaisesRegex(RuntimeError, message): + store.canonical_bundle_bytes_from_external_body( + block_hash=block_hash, + audit_bundle_sha256=expected_digest, + body_uri=body_uri, + ) + + def test_pre_v2_literal_body_is_taken_without_reserialization(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + calls: list[int] = [] + store = self.make_store(root, share_segment_size=0, counter=calls) + bundle = self.bundle() + body_uri, digest = self.publish(store, bundle) + # A pre-v2 body stores the canonical bundle verbatim. + self.assertEqual(Path(body_uri).read_bytes(), self.canonical(bundle)) + + calls.clear() + self.assertEqual( + store.literal_canonical_bundle_bytes( + body_uri, + expected_sha256=digest, + ), + self.canonical(bundle), + ) + self.assertEqual( + store.canonical_bundle_bytes_from_external_body( + block_hash=BLOCK_A, + audit_bundle_sha256=digest, + body_uri=body_uri, + ), + self.canonical(bundle), + ) + # Neither call parsed and reserialized the bundle. + self.assertEqual(calls, []) + + def test_literal_helper_declines_a_compact_body(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + store = self.make_store(root) + bundle = self.bundle() + body_uri, digest = self.publish(store, bundle) + self.assertIsNone( + store.literal_canonical_bundle_bytes( + body_uri, + expected_sha256=digest, + ) + ) + with self.assertRaisesRegex(RuntimeError, "digest does not match"): + store.literal_canonical_bundle_bytes( + body_uri, + expected_sha256="cc" * 32, + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_prism_audit_bundle_view.py b/tests/test_prism_audit_bundle_view.py new file mode 100644 index 000000000..3ef4b4c05 --- /dev/null +++ b/tests/test_prism_audit_bundle_view.py @@ -0,0 +1,925 @@ +#!/usr/bin/env python3 +"""Bounded canonical audit artifact views (issue #255). + +The view must reproduce ``json.load`` of the artifact member for member +while never holding a window-sized Python structure: the share arrays are +replayable lazy sequences, one oversized record only grows its own decode, +UTF-8 and escapes may straddle read windows, the descriptor is released +without cyclic garbage collection, and the streaming encoder reproduces the +compact ``json.dumps`` bytes exactly. +""" + +from __future__ import annotations + +import gc +import hashlib +import io +import json +import os +from pathlib import Path +import tempfile +import time +import tracemalloc +import unittest +from unittest import mock +import weakref + +from lab.prism.audit_bundle_view import ( + SCAN_CHUNK_BYTES, + ArtifactResourcePressure, + ArtifactSource, + CanonicalArtifactError, + CanonicalArtifactSyntaxError, + CanonicalAuditBundleView, + LazyRecordSequence, + LazyRecordSlice, + RawJsonDocument, + RawJsonRecord, + iter_json_byte_chunks, + iter_json_chunks, + json_chunks_sha256_and_size, + materialize_json, + spool_json_document, +) + + +def make_share(index: int, *, oversized_bytes: int = 0) -> dict[str, object]: + share: dict[str, object] = { + "share_seq": index + 1, + "share_id": f"share-{index + 1}", + "miner_id": "mïner-ü" if index % 7 == 0 else f"miner-{index % 5}", + "order_key": f"o{index % 5}", + "p2mr_program_hex": ("%02x" % (index % 5)) * 32, + "share_difficulty": 2**100 + index, + "network_difficulty": 2**127 + 1, + "template_height": 10, + "job_id": "job-1", + "job_issued_at_ms": 1_700_000_000_000 - 1000, + "accepted_at_ms": 1_700_000_000_000 - 500, + "ntime": 1_700_000_000, + } + if index % 3 == 0: + share["credit_policy"] = None + elif index % 3 == 1: + share["credit_policy"] = 'quote"back\\slash\n\ttab ☃' + if oversized_bytes: + share["share_id"] = "Z" * oversized_bytes + return share + + +def make_bundle(count: int, *, oversized_index: int | None = None, oversized_bytes: int = 0) -> dict[str, object]: + shares = [ + make_share( + index, + oversized_bytes=oversized_bytes if index == oversized_index else 0, + ) + for index in range(count) + ] + return { + "schema": "qbit.prism.audit-bundle.v1", + "shares": shares, + "found_block": {"block_height": 10, "network_difficulty": 2**90}, + "prior_balances": [], + "reward_manifest": { + "schema": "qbit.prism.reward-manifest.v1", + "included_share_count": count, + "shares": [ + {"share_seq": index + 1, "counted_difficulty": 5, "credit_policy": None} + for index in range(count) + ], + "entitlements": [{"recipient_id": "a", "weight": 1}], + }, + "payout_policy_manifest": { + "accounts": [{"recipient_id": "a", "nested": {"deep": [1, {"x": [1, 2]}]}}] + }, + "signed_coinbase_manifest": {"manifest": {"coinbase_tx_hex": "00"}}, + } + + +def write_document(root: Path, name: str, raw: bytes) -> Path: + path = root / name + path.write_bytes(raw) + return path + + +def compact(bundle: dict[str, object], *, ensure_ascii: bool = True, separators: tuple[str, str] = (",", ":")) -> bytes: + return json.dumps(bundle, separators=separators, ensure_ascii=ensure_ascii).encode("utf-8") + + +def scan(path: Path, **kwargs: object) -> CanonicalAuditBundleView: + return CanonicalAuditBundleView.scan_path(path, **kwargs) # type: ignore[arg-type] + + +class ViewParityTests(unittest.TestCase): + def test_view_reproduces_json_load_across_row_counts(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + for count in (0, 1, 255, 256, 257, 511, 512, 513, 1300): + for ensure_ascii, separators in ( + (True, (",", ":")), + (False, (", ", ": ")), + ): + with self.subTest(count=count, ascii=ensure_ascii): + bundle = make_bundle(count) + raw = compact(bundle, ensure_ascii=ensure_ascii, separators=separators) + path = write_document(root, f"bundle-{count}.json", raw) + view = scan(path, chunk_bytes=4096, stride=16) + try: + self.assertEqual(view.sha256_hex, hashlib.sha256(raw).hexdigest()) + self.assertEqual(view.byte_length, len(raw)) + self.assertEqual(list(view), list(bundle)) + self.assertEqual(view.key_index("shares"), 1) + self.assertIsInstance(view["shares"], LazyRecordSequence) + self.assertIsInstance(view["reward_manifest"]["shares"], LazyRecordSequence) + self.assertEqual(len(view["shares"]), count) + # Full logical equality, both operand orders. + self.assertTrue(view == bundle) + self.assertTrue(bundle == view) + self.assertFalse(view != bundle) + self.assertFalse(bundle != view) + # Replayable: two complete iterations agree. + self.assertEqual(list(view["shares"]), bundle["shares"]) + self.assertEqual(list(view["shares"]), bundle["shares"]) + self.assertEqual( + list(view["reward_manifest"]["shares"]), + bundle["reward_manifest"]["shares"], + ) + if count: + shares = view["shares"] + self.assertEqual(shares[0], bundle["shares"][0]) + self.assertEqual(shares[-1], bundle["shares"][-1]) + self.assertEqual(shares[count // 2], bundle["shares"][count // 2]) + self.assertIsInstance(shares[3:9], LazyRecordSlice) + self.assertEqual(shares[3:9], bundle["shares"][3:9]) + self.assertEqual(shares[::7], bundle["shares"][::7]) + self.assertEqual( + [list(batch) for batch in shares.batches(100)], + [bundle["shares"][i : i + 100] for i in range(0, count, 100)], + ) + with self.assertRaises(IndexError): + shares[count] + with self.assertRaises(IndexError): + shares[-count - 1] + finally: + view.close() + + def test_default_windows_and_strides_agree_with_small_ones(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + bundle = make_bundle(2000) + raw = compact(bundle, ensure_ascii=False) + path = write_document(Path(tmp), "bundle.json", raw) + default_view = scan(path) + tiny_view = scan(path, chunk_bytes=7, stride=3) + try: + self.assertEqual(default_view, bundle) + self.assertEqual(tiny_view, bundle) + for index in (0, 1, 2, 3, 999, 1998, 1999): + self.assertEqual(tiny_view["shares"][index], bundle["shares"][index]) + self.assertEqual(default_view["shares"][index], bundle["shares"][index]) + finally: + default_view.close() + tiny_view.close() + + def test_oversized_single_field_is_isolated_to_its_own_decode(self) -> None: + oversized = 3 * SCAN_CHUNK_BYTES + 17 + with tempfile.TemporaryDirectory() as tmp: + bundle = make_bundle(257, oversized_index=128, oversized_bytes=oversized) + raw = compact(bundle) + path = write_document(Path(tmp), "bundle.json", raw) + view = scan(path) + try: + self.assertEqual(view, bundle) + self.assertEqual(view["shares"][128], bundle["shares"][128]) + # Reading an ordinary record never pays for the oversized one. + tracemalloc.start() + try: + tracemalloc.reset_peak() + self.assertEqual(view["shares"][0], bundle["shares"][0]) + _current, peak = tracemalloc.get_traced_memory() + finally: + tracemalloc.stop() + self.assertLess(peak, oversized) + tracemalloc.start() + try: + tracemalloc.reset_peak() + self.assertEqual(view["shares"][128], bundle["shares"][128]) + _current, peak = tracemalloc.get_traced_memory() + finally: + tracemalloc.stop() + # One oversized record costs a small multiple of itself + # (decoded text plus the string), never the window. + self.assertLess(peak, 8 * oversized) + finally: + view.close() + + def test_null_and_absent_members_stay_distinct(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + bundle = make_bundle(6) + raw = compact(bundle) + path = write_document(Path(tmp), "bundle.json", raw) + view = scan(path) + try: + self.assertEqual(view, bundle) + absent = json.loads(raw) + del absent["shares"][0]["credit_policy"] + self.assertNotEqual(view, absent) + nulled = json.loads(raw) + nulled["shares"][2]["credit_policy"] = None + self.assertNotEqual(view, nulled) + self.assertEqual(view["shares"][0]["share_difficulty"], 2**100) + self.assertEqual(view["found_block"]["network_difficulty"], 2**90) + finally: + view.close() + + def test_equality_mismatches_are_detected_streaming(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + bundle = make_bundle(600) + raw = compact(bundle) + path = write_document(Path(tmp), "bundle.json", raw) + view = scan(path, chunk_bytes=1024) + try: + last_record = json.loads(raw) + last_record["shares"][-1]["share_seq"] = 999_999 + self.assertNotEqual(view, last_record) + self.assertNotEqual(last_record, view) + extra_key = {**json.loads(raw), "extra": 1} + self.assertNotEqual(view, extra_key) + missing_key = json.loads(raw) + del missing_key["prior_balances"] + self.assertNotEqual(view, missing_key) + counted = json.loads(raw) + counted["reward_manifest"]["shares"][300]["counted_difficulty"] = 6 + self.assertNotEqual(view, counted) + shorter = json.loads(raw) + shorter["shares"].pop() + self.assertNotEqual(view, shorter) + longer = json.loads(raw) + longer["shares"].append(make_share(600)) + self.assertNotEqual(view, longer) + self.assertNotEqual(view["shares"], "not a sequence") + self.assertEqual(view["shares"][10:20], list(view["shares"])[10:20]) + finally: + view.close() + + +class TokenBoundaryTests(unittest.TestCase): + """Number tokens cut by a read window must never decode as a prefix.""" + + DOCUMENTS = ( + {"x": 1e300, "shares": []}, + {"x": -1.5e-3, "shares": [1.25, -0.5e2, 3, 1e300, -7]}, + {"x": 12345678901234567890, "shares": [0, -0, 10, 100000000000000000000]}, + {"x": 0.000001, "y": 5e-324, "shares": [{"n": 1e+300}, {"n": -2.5E+10}]}, + {"a": True, "b": None, "c": False, "x": -7, "shares": ["1e+", "-", "1."]}, + ) + + def _raw(self, document: dict[str, object]) -> bytes: + text = json.dumps(document, separators=(",", ":")) + # Spell exponents the way serializers may: keep an explicit sign. + return text.replace("1e+300", "1e+300").encode() + + def test_tiny_window_sweep_matches_json_loads(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + for index, document in enumerate(self.DOCUMENTS): + raw = self._raw(document) + # Also exercise an explicit plus sign the stdlib serializer + # never emits but valid documents may carry. + variants = [raw, raw.replace(b"1e+300", b"1E+300").replace(b"-7", b"-7.0e+0")] + for variant_index, variant in enumerate(variants): + expected = json.loads(variant) + path = write_document(Path(tmp), f"num-{index}-{variant_index}.json", variant) + for chunk_bytes in (1, 2, 3, 4, 5, 7, 8, 16, 64): + with self.subTest(document=index, variant=variant_index, chunk_bytes=chunk_bytes): + view = scan(path, chunk_bytes=chunk_bytes, stride=2) + try: + self.assertEqual(view, expected) + self.assertEqual(list(view["shares"]), expected["shares"]) + self.assertEqual(view["shares"][0:2], expected["shares"][0:2]) + if expected["shares"]: + self.assertEqual(view["shares"][-1], expected["shares"][-1]) + finally: + view.close() + + def test_default_window_alignment_sweep(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + for pad in range(SCAN_CHUNK_BYTES - 40, SCAN_CHUNK_BYTES + 8): + raw = b'{"pad":"' + b"a" * pad + b'","x":1e+300,"y":-2.5e-7,"shares":[12,3.5e+1]}' + expected = json.loads(raw) + path = write_document(Path(tmp), "aligned.json", raw) + with self.subTest(pad=pad): + view = scan(path) + try: + self.assertEqual(view, expected) + self.assertEqual(view["shares"][1], 35.0) + finally: + view.close() + + def test_malformed_tokens_are_rejected_at_scan_time(self) -> None: + """Every rejection the standard decoder makes, the scan makes too.""" + with tempfile.TemporaryDirectory() as tmp: + for index, raw in enumerate( + ( + b'{"x":1e+,"shares":[]}', + b'{"x":1.,"shares":[]}', + b'{"x":01,"shares":[]}', + b'{"x":-,"shares":[]}', + b'{"x":1e+300', + b'{"shares":[1.]}', + b'{"shares":[1 2]}', + b'{"shares":[{"a":1,}]}', + b'{"shares":[{"a" 1}]}', + b'{"shares":["a\x01b"]}', + b'{"shares":["\\q"]}', + b'{"shares":["\\u12"]}', + b'{"shares":[{"a":[1,]}]}', + b'{"shares":[tru]}', + b'{"shares":[{"a":-}]}', + b'{"shares":[{"a":1}{"b":2}]}', + b'{"shares":[[1,2}]}', + b'{"shares":[{"a":1,"b":}]}', + b'{"shares":[{,}]}', + b'{"shares":[nul]}', + b'{"shares":[1]', + ) + ): + with self.subTest(document=raw): + self.assertRaises(ValueError, json.loads, raw) + path = write_document(Path(tmp), f"bad-{index}.json", raw) + for chunk_bytes in (1, 3, 8, 4096): + with self.subTest(document=raw, chunk_bytes=chunk_bytes): + with self.assertRaises(json.JSONDecodeError): + scan(path, chunk_bytes=chunk_bytes) + + def test_nested_and_escaped_records_survive_tiny_windows(self) -> None: + document = { + "shares": [ + {"a": [1, {"b": "é\n\"}\\", "c": ""}], "d": " x ", "e": [], "f": {}}, + {"g": [[[]]], "h": "😀", "i": -0.0, "j": [True, False, None]}, + "plain", + 12, + [], + ], + "reward_manifest": {"shares": [{"n": 1}, {"n": [2, {"m": "é"}]}]}, + } + raw = json.dumps(document, indent=2).encode() + with tempfile.TemporaryDirectory() as tmp: + path = write_document(Path(tmp), "nested.json", raw) + for chunk_bytes in (1, 2, 5, 13, 64, SCAN_CHUNK_BYTES): + with self.subTest(chunk_bytes=chunk_bytes): + view = scan(path, chunk_bytes=chunk_bytes, stride=2) + try: + self.assertEqual(view, document) + self.assertEqual(list(view["shares"]), document["shares"]) + self.assertEqual(view["shares"][1]["h"], "😀") + self.assertEqual(view["shares"][2:5], document["shares"][2:5]) + finally: + view.close() + + +class ViewRobustnessTests(unittest.TestCase): + def test_malformed_documents_fail_as_json_decode_errors_and_release_fd(self) -> None: + cases = [ + b"[1,2]", + b'{"a":1', + b'{"a":1}x', + b'{"shares":[1,2,}', + b'{"shares":[{"a":1}', + b'{"a":"\xff"}', + b'{"shares":[{"a":1}]}{', + b'{"shares":[{"a":1}],}', + b'{1:2}', + b"", + b'{"shares":[{"a":"unterminated}]}', + ] + with tempfile.TemporaryDirectory() as tmp: + for index, raw in enumerate(cases): + with self.subTest(document=raw): + path = write_document(Path(tmp), f"bad-{index}.json", raw) + fd = os.open(path, os.O_RDONLY) + with self.assertRaises(json.JSONDecodeError) as caught: + CanonicalAuditBundleView.scan(fd, chunk_bytes=4) + self.assertIsInstance(caught.exception, CanonicalArtifactSyntaxError) + self.assertIsInstance(caught.exception, CanonicalArtifactError) + with self.assertRaises(OSError): + os.fstat(fd) + + def test_not_a_regular_file_is_rejected_and_fd_closed(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + fd = os.open(tmp, os.O_RDONLY) + with self.assertRaises(CanonicalArtifactError): + ArtifactSource(fd) + with self.assertRaises(OSError): + os.fstat(fd) + + def test_identity_change_is_detected_by_lazy_reads(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + bundle = make_bundle(40) + raw = compact(bundle) + path = write_document(Path(tmp), "bundle.json", raw) + view = scan(path) + try: + header = view["found_block"] + with path.open("ab") as handle: + handle.write(b" ") + self.assertEqual(view["found_block"], header) + with self.assertRaises(CanonicalArtifactError) as caught: + list(view["shares"]) + self.assertNotIsInstance(caught.exception, json.JSONDecodeError) + with self.assertRaises(CanonicalArtifactError): + view.verify_identity() + with self.assertRaises(CanonicalArtifactError): + list(view.iter_bytes()) + finally: + view.close() + + def test_descriptor_is_released_without_cyclic_gc(self) -> None: + gc.disable() + try: + with tempfile.TemporaryDirectory() as tmp: + bundle = make_bundle(30) + raw = compact(bundle) + path = write_document(Path(tmp), "bundle.json", raw) + view = scan(path) + fd = view.source.fileno() + shares = view["shares"] + self.assertEqual(shares[3], bundle["shares"][3]) + ref = weakref.ref(view) + source_ref = weakref.ref(view.source) + del view + # The lazy sequence still owns the source; the descriptor + # stays open exactly as long as a legitimate owner exists. + self.assertIsNone(ref()) + self.assertIsNotNone(source_ref()) + self.assertEqual(shares[4], bundle["shares"][4]) + del shares + self.assertIsNone(source_ref()) + with self.assertRaises(OSError): + os.fstat(fd) + # Explicit close is idempotent and fails lazy reads closed. + view = scan(path) + lazy = view["shares"] + view.close() + view.close() + self.assertTrue(view.closed) + self.assertEqual(view["schema"], bundle["schema"]) + with self.assertRaises(CanonicalArtifactError): + list(lazy) + finally: + gc.enable() + + def test_scan_stops_within_one_read_when_cancelled(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + bundle = make_bundle(3000) + raw = compact(bundle) + path = write_document(Path(tmp), "bundle.json", raw) + calls: list[int] = [] + + class Cancelled(RuntimeError): + pass + + def checkpoint() -> None: + calls.append(1) + if len(calls) == 3: + raise Cancelled("cancelled") + + fd = os.open(path, os.O_RDONLY) + with self.assertRaises(Cancelled): + CanonicalAuditBundleView.scan(fd, chunk_bytes=8192, cancellation=checkpoint) + self.assertEqual(len(calls), 3) + with self.assertRaises(OSError): + os.fstat(fd) + + +class StreamingEncoderTests(unittest.TestCase): + def test_encoder_reproduces_compact_json_with_lazy_members(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + bundle = make_bundle(700) + raw = compact(bundle) + path = write_document(Path(tmp), "bundle.json", raw) + view = scan(path, chunk_bytes=2048, stride=8) + try: + storage = { + "schema": "example", + "bundle_without_shares": view.without("shares"), + "count": 700, + "parts": [ + {"kind": "inline", "shares": view["shares"][0:5]}, + {"kind": "inline", "shares": view["shares"][695:700]}, + ], + "plain": {"list": [1, 2.5, None, True, "ü"], "empty": [], "obj": {}}, + "lazy_only": view["shares"], + } + expected_storage = { + "schema": "example", + "bundle_without_shares": {k: v for k, v in bundle.items() if k != "shares"}, + "count": 700, + "parts": [ + {"kind": "inline", "shares": bundle["shares"][0:5]}, + {"kind": "inline", "shares": bundle["shares"][695:700]}, + ], + "plain": {"list": [1, 2.5, None, True, "ü"], "empty": [], "obj": {}}, + "lazy_only": bundle["shares"], + } + expected = json.dumps(expected_storage, separators=(",", ":")) + for batch_records, chunk_chars in ((1, 1), (7, 100), (256, 65536)): + chunks = list( + iter_json_chunks( + storage, + batch_records=batch_records, + chunk_chars=chunk_chars, + ) + ) + self.assertEqual("".join(chunks), expected) + self.assertTrue(all(chunk.isascii() for chunk in chunks)) + digest, size = json_chunks_sha256_and_size(iter_json_byte_chunks(storage)) + self.assertEqual(digest, hashlib.sha256(expected.encode()).hexdigest()) + self.assertEqual(size, len(expected)) + with self.assertRaises(TypeError): + json.dumps(storage) + finally: + view.close() + + def test_encoder_matches_json_dumps_for_plain_values(self) -> None: + values = [ + {}, + [], + {"a": [1, [2, [3, {"b": None}]]], "c": "d"}, + {"k": {1: "int key"}}, + [{"x": 1}, {"y": [1, 2]}, "s", 3], + "solo", + 12, + ] + for value in values: + with self.subTest(value=value): + self.assertEqual( + "".join(iter_json_chunks(value, batch_records=2, chunk_chars=3)), + json.dumps(value, separators=(",", ":")), + ) + + +def _write_synthetic_bundle(path: Path, count: int) -> tuple[bytes, int]: + """Write a count-share bundle by streaming, returning (sha256, size).""" + digest = hashlib.sha256() + size = 0 + with path.open("wb") as handle: + + def emit(text: str) -> None: + nonlocal size + data = text.encode("utf-8") + digest.update(data) + size += len(data) + handle.write(data) + + emit('{"schema":"qbit.prism.audit-bundle.v1","shares":[') + for index in range(count): + if index: + emit(",") + emit(json.dumps(make_share(index), separators=(",", ":"))) + emit('],"found_block":{"block_height":10,"network_difficulty":1000},"prior_balances":[],') + emit('"reward_manifest":{"schema":"qbit.prism.reward-manifest.v1","shares":[') + for index in range(count): + if index: + emit(",") + emit( + json.dumps( + {"share_seq": index + 1, "counted_difficulty": 5}, + separators=(",", ":"), + ) + ) + emit('],"entitlements":[]},"payout_policy_manifest":{"accounts":[]},') + emit('"signed_coinbase_manifest":{"manifest":{"coinbase_tx_hex":"00"}}}') + return digest.digest(), size + + +class StressWindowTests(unittest.TestCase): + """Window-sized artifacts; the 375k case runs when explicitly enabled.""" + + def _exercise(self, count: int, *, peak_limit: int) -> dict[str, float]: + with tempfile.TemporaryDirectory() as tmp: + path = Path(tmp) / "canonical.json" + expected_digest, size = _write_synthetic_bundle(path, count) + tracemalloc.start() + try: + tracemalloc.reset_peak() + started = time.monotonic() + view = scan(path) + scan_seconds = time.monotonic() - started + try: + self.assertEqual(view.sha256_hex, expected_digest.hex()) + self.assertEqual(view.byte_length, size) + self.assertEqual(len(view["shares"]), count) + self.assertEqual(len(view["reward_manifest"]["shares"]), count) + for index in (0, 1, count // 2, count - 2, count - 1): + self.assertEqual(view["shares"][index], make_share(index)) + started = time.monotonic() + walked = 0 + for record in view["shares"]: + walked += 1 + if walked % 50_000 == 0: + self.assertEqual(record["share_seq"], walked) + walk_seconds = time.monotonic() - started + self.assertEqual(walked, count) + encoded = 0 + started = time.monotonic() + for chunk in iter_json_byte_chunks({"shares": view["shares"]}): + encoded += len(chunk) + encode_seconds = time.monotonic() - started + _current, peak = tracemalloc.get_traced_memory() + finally: + view.close() + finally: + tracemalloc.stop() + self.assertGreater(encoded, 0) + self.assertLess(peak, peak_limit, f"peak traced memory {peak} exceeded {peak_limit}") + return { + "count": count, + "bytes": size, + "scan_seconds": scan_seconds, + "walk_seconds": walk_seconds, + "encode_seconds": encode_seconds, + "peak_traced_bytes": peak, + } + + def test_twenty_thousand_shares_stay_bounded(self) -> None: + # ~7.5 MB of JSON; peak traced memory stays a few read windows. + report = self._exercise(20_000, peak_limit=6 * 1024 * 1024) + self.assertGreater(report["bytes"], 5_000_000) + + @unittest.skipUnless( + os.environ.get("PRISM_BOUNDED_AUDIT_STRESS"), + "set PRISM_BOUNDED_AUDIT_STRESS=1 to run the 375k-share artifact case", + ) + def test_375k_shares_across_generations(self) -> None: + reports = [] + for generation in range(int(os.environ.get("PRISM_BOUNDED_AUDIT_GENERATIONS", "2"))): + reports.append(self._exercise(375_000, peak_limit=32 * 1024 * 1024)) + for report in reports: + print(f"bounded-audit-stress {json.dumps(report)}", flush=True) + + +class BoundedScanTests(unittest.TestCase): + """The scan holds at most about two read chunks whatever a value's size.""" + + def test_checkpoint_index_lives_on_disk_and_pressure_is_recoverable(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + bundle = make_bundle(3000) + raw = compact(bundle) + path = write_document(Path(tmp), "bundle.json", raw) + view = scan(path, stride=16) + try: + stats = view.scan_stats + # 3000 shares and 3000 counted shares at a stride of 16, one + # checkpoint per recipient-scaled array (entitlements, + # accounts) and none for the empty prior balances. + self.assertEqual(stats.checkpoint_entries, 2 * 188 + 2) + self.assertEqual(view["shares"][2999], bundle["shares"][2999]) + self.assertEqual(view["reward_manifest"]["shares"][1234], bundle["reward_manifest"]["shares"][1234]) + self.assertIsInstance(view["prior_balances"], LazyRecordSequence) + self.assertIsInstance(view["payout_policy_manifest"]["accounts"], LazyRecordSequence) + self.assertIsInstance(view["reward_manifest"]["entitlements"], LazyRecordSequence) + self.assertEqual(view, bundle) + finally: + view.close() + # Scratch pressure is neither a syntax error nor corruption: the + # artifact is intact and the caller retries later. + fd = os.open(path, os.O_RDONLY) + with mock.patch("lab.prism.audit_bundle_view.tempfile.TemporaryFile", side_effect=OSError("no temp")): + with self.assertRaises(ArtifactResourcePressure) as caught: + CanonicalAuditBundleView.scan(fd, stride=16) + self.assertNotIsInstance(caught.exception, CanonicalArtifactError) + self.assertNotIsInstance(caught.exception, json.JSONDecodeError) + with self.assertRaises(OSError): + os.fstat(fd) + # A closed view fails lazy reads closed rather than reading a + # released index or descriptor. + view = scan(path, stride=16) + lazy = view["shares"] + view.close() + with self.assertRaises(CanonicalArtifactError): + lazy[5] + + def test_oversized_records_are_normalized_by_the_isolated_helper(self) -> None: + limit = 8192 + oversized = 3 * limit + with tempfile.TemporaryDirectory() as tmp: + bundle = make_bundle(40, oversized_index=7, oversized_bytes=oversized) + bundle["shares"][8]["nested"] = {"deep": [1, {"x": "é\"}"}], "big": "Q" * (2 * limit)} + bundle["witness_merkle_leaves_hex"] = ["ab" * 32, "W" * (2 * limit)] + raw = compact(bundle, ensure_ascii=False) + path = write_document(Path(tmp), "bundle.json", raw) + with mock.patch("lab.prism.audit_bundle_view.RECORD_DECODE_SOFT_LIMIT_BYTES", limit): + view = scan(path, chunk_bytes=1024) + try: + records = list(view["shares"]) + self.assertEqual(view.scan_stats.isolated_records, 2) + isolated = records[7] + self.assertIsInstance(isolated, RawJsonRecord) + self.assertTrue(isolated.is_object) + self.assertEqual(isolated["share_seq"], 8) + self.assertEqual(isolated["miner_id"], bundle["shares"][7]["miner_id"]) + self.assertIn("share_id", isolated.omitted_members) + self.assertIn("share_id", isolated) + with self.assertRaises(CanonicalArtifactError): + isolated["share_id"] + with self.assertRaises(KeyError): + isolated["absent"] + self.assertEqual(list(isolated), list(bundle["shares"][7])) + # Equality streams the sorted encodings, both operand + # orders, against dictionaries and other records. + self.assertEqual(isolated, bundle["shares"][7]) + self.assertTrue(bundle["shares"][7] == isolated) + self.assertNotEqual(isolated, bundle["shares"][8]) + self.assertNotEqual(isolated, {**bundle["shares"][7], "extra": 1}) + self.assertEqual(records[8], bundle["shares"][8]) + self.assertEqual(view["shares"][7], view["shares"][7]) + self.assertNotEqual(view["shares"][7], view["shares"][8]) + self.assertEqual(view, bundle) + self.assertEqual(materialize_json(isolated), bundle["shares"][7]) + # The streaming encoder emits the record's own encoding + # verbatim, byte-identical to json.dumps of the value. + self.assertEqual( + "".join(iter_json_chunks({"shares": view["shares"][6:9]})), + json.dumps({"shares": bundle["shares"][6:9]}, separators=(",", ":")), + ) + self.assertEqual( + "".join(iter_json_chunks(isolated, sort_keys=True)), + json.dumps(bundle["shares"][7], sort_keys=True, separators=(",", ":")), + ) + # A non-object oversized value (a leaf string) is + # normalized the same way. + leaves = list(view["witness_merkle_leaves_hex"]) + self.assertEqual(leaves[0], "ab" * 32) + self.assertIsInstance(leaves[1], RawJsonRecord) + self.assertFalse(leaves[1].is_object) + self.assertEqual(leaves[1], "W" * (2 * limit)) + self.assertEqual(materialize_json(view["witness_merkle_leaves_hex"]), bundle["witness_merkle_leaves_hex"]) + finally: + view.close() + + def test_isolated_helper_failures_are_classified(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + raw = b'{"shares":[{"a":"' + b"x" * 20000 + b'"}]}' + path = write_document(Path(tmp), "bundle.json", raw) + with mock.patch("lab.prism.audit_bundle_view.RECORD_DECODE_SOFT_LIMIT_BYTES", 1024): + view = scan(path) + try: + self.assertEqual(view["shares"][0], {"a": "x" * 20000}) + with mock.patch( + "lab.prism.audit_bundle_view.subprocess.Popen", + side_effect=OSError("fork failed"), + ): + with self.assertRaises(ArtifactResourcePressure): + view["shares"][0] + real_temporary_file = tempfile.TemporaryFile + calls = {"count": 0} + + def failing_spool(*args: object, **kwargs: object) -> object: + calls["count"] += 1 + raise OSError("disk full") + + with mock.patch( + "lab.prism.audit_bundle_view.tempfile.TemporaryFile", + side_effect=failing_spool, + ): + with self.assertRaises(ArtifactResourcePressure): + view["shares"][0] + self.assertGreaterEqual(calls["count"], 1) + self.assertIs(tempfile.TemporaryFile, real_temporary_file) + finally: + view.close() + # The scan rejects a token-malformed record before any helper + # runs; the helper classifies malformed input the same way when + # it is handed such bytes directly, never as resource pressure. + bad = b'{"shares":[{"a":tru,"b":"' + b"y" * 20000 + b'"}]}' + bad_path = write_document(Path(tmp), "bad.json", bad) + with self.assertRaises(json.JSONDecodeError): + scan(bad_path) + from lab.prism.audit_bundle_view import normalize_record_isolated + + source = ArtifactSource(os.open(bad_path, os.O_RDONLY), path=bad_path) + try: + with self.assertRaises(json.JSONDecodeError) as caught: + normalize_record_isolated(source, 11, len(bad) - 2) + self.assertIsInstance(caught.exception, CanonicalArtifactSyntaxError) + finally: + source.close() + + def test_raw_document_and_spooled_documents_round_trip(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + bundle = make_bundle(12) + raw = compact(bundle, ensure_ascii=False) + path = write_document(Path(tmp), "bundle.json", raw) + view = scan(path) + try: + document = RawJsonDocument(view.source) + self.assertEqual(document.byte_length, len(raw)) + self.assertEqual(b"".join(document.iter_byte_chunks(chunk_bytes=7)), raw) + self.assertEqual("".join(document.iter_text_chunks(chunk_bytes=7)), raw.decode("utf-8")) + embedded = "".join(iter_json_chunks({"audit_bundle": document, "n": 1})) + self.assertEqual(json.loads(embedded), {"audit_bundle": bundle, "n": 1}) + self.assertEqual(materialize_json(view), bundle) + spooled = spool_json_document( + iter_json_byte_chunks({"share_parts": [{"kind": "x", "n": i} for i in range(700)]}), + lazy_paths=(("share_parts",),), + stride=8, + ) + try: + parts = spooled["share_parts"] + self.assertIsInstance(parts, LazyRecordSequence) + self.assertEqual(len(parts), 700) + self.assertEqual(parts[699], {"kind": "x", "n": 699}) + self.assertEqual(list(parts)[:3], [{"kind": "x", "n": i} for i in range(3)]) + finally: + spooled.close() + finally: + view.close() + + def test_oversized_values_are_skipped_without_being_held(self) -> None: + chunk = 4096 + oversized = 6 * SCAN_CHUNK_BYTES + 11 # far above the soft limit + with tempfile.TemporaryDirectory() as tmp: + bundle = make_bundle(300, oversized_index=150, oversized_bytes=oversized) + bundle["shares"][151]["nested"] = {"deep": [1, {"x": '"}]{[\\'}], "s": "a\\\"b"} + bundle["shares"][152]["credit_policy"] = "quote\"in\\side" + "é" * 3000 + bundle["reward_manifest"]["shares"][150]["blob"] = "Y" * (3 * chunk + 1) + bundle["witness_merkle_leaves_hex"] = ["ab" * 32] * 500 + raw = compact(bundle, ensure_ascii=False) + path = write_document(Path(tmp), "bundle.json", raw) + tracemalloc.start() + try: + tracemalloc.reset_peak() + view = scan(path, chunk_bytes=chunk) + _current, scan_peak = tracemalloc.get_traced_memory() + finally: + tracemalloc.stop() + try: + stats = view.scan_stats + # The oversized record was walked structurally: the largest + # decoded window stayed a couple of chunks, not the record. + self.assertLess(stats.window_high_water_chars, 4 * chunk) + self.assertGreaterEqual(stats.max_value_chars, oversized) + self.assertEqual(stats.oversized_values, 1) + self.assertLess(scan_peak, oversized) + self.assertEqual(view.sha256_hex, hashlib.sha256(raw).hexdigest()) + self.assertEqual(len(view["shares"]), 300) + self.assertEqual(len(view["witness_merkle_leaves_hex"]), 500) + # Consumers that need the record decode it whole; that + # decode is the remaining record-sized allocation. + self.assertEqual(view["shares"][150], bundle["shares"][150]) + self.assertEqual(view["shares"][151], bundle["shares"][151]) + self.assertEqual(view["shares"][152], bundle["shares"][152]) + self.assertEqual(view, bundle) + self.assertEqual(list(view["shares"][149:153]), bundle["shares"][149:153]) + finally: + view.close() + + def test_structural_walk_rejects_unterminated_values(self) -> None: + cases = [ + b'{"shares":[{"a":"' + b"x" * 5000, + b'{"shares":[{"a":[1,2}]}', + b'{"shares":[{"a":"\\' , + b'{"shares":[,]}', + b'{"shares":[1,]}', + b'{"shares":["abc]}', + ] + with tempfile.TemporaryDirectory() as tmp: + for index, raw in enumerate(cases): + with self.subTest(document=raw[:24]): + path = write_document(Path(tmp), f"bad-{index}.json", raw) + with self.assertRaises(json.JSONDecodeError): + scan(path, chunk_bytes=16) + + +class ArtifactSourceTests(unittest.TestCase): + def test_iter_bytes_and_sha256_cover_the_whole_file(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + raw = os.urandom(3 * SCAN_CHUNK_BYTES + 5) + path = write_document(Path(tmp), "blob.bin", raw) + source = ArtifactSource(os.open(path, os.O_RDONLY), path=path) + try: + self.assertEqual(b"".join(source.iter_bytes(chunk_bytes=1000)), raw) + self.assertEqual(source.sha256_hex(), hashlib.sha256(raw).hexdigest()) + self.assertEqual(source.pread(len(raw) - 3, 10), raw[-3:]) + self.assertEqual(source.pread(len(raw), 10), b"") + finally: + source.close() + self.assertTrue(source.closed) + with self.assertRaises(CanonicalArtifactError): + source.fileno() + + def test_chunk_reader_adapter_streams_sequentially(self) -> None: + from lab.prism.audit_artifacts import _ChunkReader + + with tempfile.TemporaryDirectory() as tmp: + raw = bytes(range(256)) * 700 + path = write_document(Path(tmp), "blob.bin", raw) + source = ArtifactSource(os.open(path, os.O_RDONLY), path=path) + try: + reader = io.BufferedReader(_ChunkReader(source), buffer_size=4096) + self.assertEqual(reader.read(), raw) + finally: + source.close() + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_prism_audit_http.py b/tests/test_prism_audit_http.py new file mode 100644 index 000000000..7b44aa45c --- /dev/null +++ b/tests/test_prism_audit_http.py @@ -0,0 +1,555 @@ +#!/usr/bin/env python3 +"""Direct routing and bounded lifecycle tests for the PRISM HTTP facade.""" + +from __future__ import annotations + +import json +import threading +import time +import unittest +from unittest.mock import patch +import urllib.error +import urllib.request + +from lab.prism.audit_http import ( + AuditHttpConfig, + AuditHttpFacade, + _BoundedThreadingHttpServer, +) +from lab.prism.observability import ( + METRICS_STALE_WARNING, + METRICS_STATE_FRESH, + METRICS_STATE_HEADER, + METRICS_STATE_STALE, + METRICS_STATE_UNAVAILABLE, + MetricsSnapshotResponse, +) + + +class FakeAuditHttpPort: + def __init__(self) -> None: + self.health_entered: threading.Event | None = None + self.health_release: threading.Event | None = None + self.health_calls = 0 + self.metrics_calls = 0 + self.metrics_response = MetricsSnapshotResponse( + status=200, + body="qbit_prism_cached_fixture 1\n", + state=METRICS_STATE_FRESH, + age_seconds=0, + ) + + def cached_health_payload(self) -> tuple[int, dict[str, object]]: + self.health_calls += 1 + if self.health_entered is not None: + self.health_entered.set() + if self.health_release is not None: + self.health_release.wait(2.0) + return 200, {"ok": True, "schema": "health-fixture"} + + def cached_metrics_payload(self) -> MetricsSnapshotResponse: + self.metrics_calls += 1 + return self.metrics_response + + def latest_evidence_payload(self) -> dict[str, object] | None: + return None + + def owed_balances_payload(self) -> dict[str, object]: + return {"balances": []} + + def carry_forward_integrity_payload(self) -> dict[str, object]: + return {"ok": True} + + def miner_status_payload(self, recipient_id: str) -> dict[str, object]: + return {"recipient_id": recipient_id} + + def public_payload( + self, + path: str, + query: dict[str, list[str]], + ) -> tuple[int, object]: + raise AssertionError((path, query)) + + +class AuditHttpFacadeTests(unittest.TestCase): + def test_start_failure_releases_lifecycle_lock_before_join(self) -> None: + class NeverReadyServer: + def __init__(self, address: tuple[str, int], _handler: object) -> None: + self.server_address = address + self.ready = threading.Event() + self.shutdown_requested = threading.Event() + self.closed = False + self.serve_committed = False + + def serve_unless_startup_cancelled(self, *, poll_interval: float) -> None: + del poll_interval + self.serve_committed = True + self.shutdown_requested.wait(2.0) + + def cancel_startup(self) -> bool: + return self.serve_committed + + def shutdown(self) -> None: + self.shutdown_requested.set() + + def server_close(self) -> None: + self.closed = True + + facade = AuditHttpFacade( + FakeAuditHttpPort(), # type: ignore[arg-type] + AuditHttpConfig( + "127.0.0.1", + 0, + join_timeout_seconds=0.5, + ), + ) + started = time.monotonic() + with patch( + "lab.prism.audit_http._BoundedThreadingHttpServer", + NeverReadyServer, + ), self.assertRaisesRegex(RuntimeError, "did not enter"): + facade.start() + + self.assertLess(time.monotonic() - started, 1.25) + state = facade.state() + self.assertEqual(state.lifecycle, "stopped") + self.assertFalse(state.thread_alive) + self.assertIsNone(state.bound_address) + + def test_start_timeout_cancels_thread_delayed_before_serve_forever(self) -> None: + release_thread = threading.Event() + serve_forever_called = threading.Event() + real_thread = threading.Thread + + class DelayedThread(real_thread): + def run(self) -> None: + release_thread.wait(2.0) + super().run() + + def join(self, timeout: float | None = None) -> None: + release_thread.set() + super().join(timeout) + + class TrackedServer: + instance: TrackedServer | None = None + + def __init__(self, address: tuple[str, int], _handler: object) -> None: + self.server_address = address + self.ready = threading.Event() + self.cancelled = False + self.closed = False + TrackedServer.instance = self + + def serve_unless_startup_cancelled(self, *, poll_interval: float) -> None: + del poll_interval + if not self.cancelled: + serve_forever_called.set() + + def cancel_startup(self) -> bool: + self.cancelled = True + return False + + def shutdown(self) -> None: + raise AssertionError("shutdown called before serve_forever") + + def server_close(self) -> None: + self.closed = True + + facade = AuditHttpFacade( + FakeAuditHttpPort(), # type: ignore[arg-type] + AuditHttpConfig("127.0.0.1", 0, join_timeout_seconds=0.5), + ) + with patch("lab.prism.audit_http.threading.Thread", DelayedThread), patch( + "lab.prism.audit_http._BoundedThreadingHttpServer", + TrackedServer, + ), self.assertRaisesRegex(RuntimeError, "did not enter"): + facade.start() + + self.assertFalse(serve_forever_called.is_set()) + assert TrackedServer.instance is not None + self.assertTrue(TrackedServer.instance.cancelled) + self.assertTrue(TrackedServer.instance.closed) + self.assertEqual(facade.state().lifecycle, "stopped") + + def test_configuration_bounds_listener_and_join_values(self) -> None: + with self.assertRaises(ValueError): + AuditHttpConfig("", 1) + with self.assertRaises(ValueError): + AuditHttpConfig("127.0.0.1", -1) + with self.assertRaises(ValueError): + AuditHttpConfig("127.0.0.1", 65_536) + with self.assertRaises(ValueError): + AuditHttpConfig("127.0.0.1", 1, join_timeout_seconds=-1) + + def test_start_serves_cached_payloads_and_stop_is_idempotent(self) -> None: + port = FakeAuditHttpPort() + facade = AuditHttpFacade( + port, # type: ignore[arg-type] + AuditHttpConfig("127.0.0.1", 0), + ) + state = facade.start() + self.assertEqual(state.lifecycle, "running") + self.assertTrue(state.thread_alive) + self.assertIsNotNone(state.bound_address) + assert state.bound_address is not None + base_url = f"http://127.0.0.1:{state.bound_address[1]}" + + with urllib.request.urlopen(base_url + "/healthz", timeout=2) as response: + self.assertEqual(response.status, 200) + self.assertEqual( + json.loads(response.read()), + {"ok": True, "schema": "health-fixture"}, + ) + self.assertTrue( + response.headers["Server"].startswith( + "QbitPrismAudit/0.1 Python/" + ) + ) + with urllib.request.urlopen(base_url + "/metrics", timeout=2) as response: + self.assertEqual(response.status, 200) + self.assertEqual( + response.headers["Content-Type"], + "text/plain; version=0.0.4", + ) + self.assertEqual(response.read(), b"qbit_prism_cached_fixture 1\n") + + self.assertEqual(port.health_calls, 1) + self.assertEqual(port.metrics_calls, 1) + self.assertTrue(facade.stop()) + self.assertTrue(facade.stop()) + stopped = facade.state() + self.assertEqual(stopped.lifecycle, "stopped") + self.assertFalse(stopped.thread_alive) + self.assertIsNone(stopped.bound_address) + + def test_metrics_publishes_freshness_in_response_metadata(self) -> None: + """A scraper must tell fresh from stale from unavailable (issue #184). + + None of these assertions parse the Prometheus body: the whole point of + the response metadata is that a consumer does not have to. + """ + + port = FakeAuditHttpPort() + facade = AuditHttpFacade( + port, # type: ignore[arg-type] + AuditHttpConfig(bind="127.0.0.1", port=0), + ) + self.addCleanup(facade.stop) + state = facade.start() + assert state.bound_address is not None + url = f"http://127.0.0.1:{state.bound_address[1]}/metrics" + + with urllib.request.urlopen(url, timeout=2) as response: + self.assertEqual(response.status, 200) + self.assertEqual(response.headers[METRICS_STATE_HEADER], "fresh") + self.assertEqual(response.headers["Age"], "0") + self.assertIsNone(response.headers["Warning"]) + self.assertEqual(response.headers["Cache-Control"], "no-store") + self.assertEqual(response.read(), b"qbit_prism_cached_fixture 1\n") + + # A stale complete payload is served, not refused: 200 is what keeps + # Prometheus from discarding the last document that still exists. + port.metrics_response = MetricsSnapshotResponse( + status=200, + body="qbit_prism_cached_fixture 1\n", + state=METRICS_STATE_STALE, + age_seconds=42, + ) + with urllib.request.urlopen(url, timeout=2) as response: + self.assertEqual(response.status, 200) + self.assertEqual(response.headers[METRICS_STATE_HEADER], "stale") + self.assertEqual(response.headers["Age"], "42") + self.assertEqual(response.headers["Warning"], METRICS_STALE_WARNING) + self.assertEqual( + response.headers["Content-Type"], + "text/plain; version=0.0.4", + ) + # The prior complete document, unchanged. + self.assertEqual(response.read(), b"qbit_prism_cached_fixture 1\n") + + # Warm-up still refuses, and says so out of band. No Age and no stale + # warning: there is no served payload for either to describe. + port.metrics_response = MetricsSnapshotResponse( + status=503, + body="qbit_prism_metrics_snapshot_available 0\n", + state=METRICS_STATE_UNAVAILABLE, + age_seconds=None, + ) + with self.assertRaises(urllib.error.HTTPError) as raised: + urllib.request.urlopen(url, timeout=2) + refusal = raised.exception + self.addCleanup(refusal.close) + self.assertEqual(refusal.status, 503) + self.assertEqual(refusal.headers[METRICS_STATE_HEADER], "unavailable") + self.assertIsNone(refusal.headers["Age"]) + self.assertIsNone(refusal.headers["Warning"]) + self.assertEqual( + refusal.read(), + b"qbit_prism_metrics_snapshot_available 0\n", + ) + self.assertEqual(port.metrics_calls, 3) + + def test_unexpected_serve_exit_closes_listener_before_restart(self) -> None: + class ExitServer: + instances: list[ExitServer] = [] + + def __init__(self, address: tuple[str, int], _handler: object) -> None: + if self.instances and not self.instances[-1].closed.is_set(): + raise OSError("previous listener is still open") + self.server_address = address + self.ready = threading.Event() + self.release = threading.Event() + self.closed = threading.Event() + self.instances.append(self) + + def serve_unless_startup_cancelled(self, *, poll_interval: float) -> None: + del poll_interval + self.ready.set() + self.release.wait(2.0) + + def cancel_startup(self) -> bool: + return True + + def shutdown(self) -> None: + self.release.set() + + def server_close(self) -> None: + self.closed.set() + + facade = AuditHttpFacade( + FakeAuditHttpPort(), # type: ignore[arg-type] + AuditHttpConfig("127.0.0.1", 0), + ) + with patch( + "lab.prism.audit_http._BoundedThreadingHttpServer", + ExitServer, + ): + self.assertEqual(facade.start().lifecycle, "running") + first = ExitServer.instances[0] + first.release.set() + self.assertTrue(first.closed.wait(1.0)) + deadline = time.monotonic() + 1.0 + while facade.state().bound_address is not None: + if time.monotonic() >= deadline: + self.fail("exited audit HTTP listener was not retired") + time.sleep(0.01) + + exited = facade.state() + self.assertEqual(exited.lifecycle, "exited") + self.assertFalse(exited.thread_alive) + self.assertEqual(facade.start().lifecycle, "running") + self.assertEqual(len(ExitServer.instances), 2) + self.assertTrue(facade.stop()) + + def test_stop_before_start_is_safe(self) -> None: + facade = AuditHttpFacade(FakeAuditHttpPort()) # type: ignore[arg-type] + + self.assertTrue(facade.stop()) + self.assertEqual(facade.state().lifecycle, "stopped") + + def test_active_request_thread_does_not_delay_listener_shutdown(self) -> None: + port = FakeAuditHttpPort() + port.health_entered = threading.Event() + port.health_release = threading.Event() + facade = AuditHttpFacade( + port, # type: ignore[arg-type] + AuditHttpConfig("127.0.0.1", 0), + ) + state = facade.start() + assert state.bound_address is not None + outcome: list[object] = [] + + def request() -> None: + try: + with urllib.request.urlopen( + f"http://127.0.0.1:{state.bound_address[1]}/healthz", + timeout=2, + ) as response: + outcome.append(response.read()) + except Exception as exc: + outcome.append(exc) + + request_thread = threading.Thread(target=request) + request_thread.start() + self.assertTrue(port.health_entered.wait(1.0)) + + self.assertTrue(facade.stop()) + self.assertFalse(port.health_release.is_set()) + + port.health_release.set() + request_thread.join(2.0) + self.assertFalse(request_thread.is_alive()) + self.assertEqual(outcome, [b'{"ok": true, "schema": "health-fixture"}\n']) + + def test_bind_failure_does_not_publish_a_thread_or_socket(self) -> None: + first = AuditHttpFacade( + FakeAuditHttpPort(), # type: ignore[arg-type] + AuditHttpConfig("127.0.0.1", 0), + ) + first_state = first.start() + assert first_state.bound_address is not None + second = AuditHttpFacade( + FakeAuditHttpPort(), # type: ignore[arg-type] + AuditHttpConfig("127.0.0.1", first_state.bound_address[1]), + ) + try: + with self.assertRaises(OSError): + second.start() + failed = second.state() + self.assertEqual(failed.lifecycle, "new") + self.assertFalse(failed.thread_alive) + self.assertIsNone(failed.bound_address) + finally: + self.assertTrue(second.stop()) + self.assertTrue(first.stop()) + + def test_stop_racing_startup_before_serve_commit_returns_within_bound( + self, + ) -> None: + """The documented old-layer gap: stop() used to call the blocking + BaseServer.shutdown() whenever the serve thread was alive, so a stop + racing startup before serve_forever() committed waited forever on a + loop that never starts. The bounded stop must return within its + configured budget and still release the real listener socket.""" + real_thread = threading.Thread + stalled_started = threading.Event() + release_stall = threading.Event() + + class StalledThread(real_thread): + def run(self) -> None: + stalled_started.set() + release_stall.wait(5.0) + super().run() + + class RecordingBounded(_BoundedThreadingHttpServer): + instances: list[_BoundedThreadingHttpServer] = [] + + def __init__(self, *args: object, **kwargs: object) -> None: + super().__init__(*args, **kwargs) # type: ignore[arg-type] + RecordingBounded.instances.append(self) + + facade = AuditHttpFacade( + FakeAuditHttpPort(), # type: ignore[arg-type] + AuditHttpConfig("127.0.0.1", 0, join_timeout_seconds=0.25), + ) + start_outcome: list[object] = [] + + def run_start() -> None: + try: + start_outcome.append(facade.start()) + except Exception as exc: + start_outcome.append(exc) + + starter = real_thread(target=run_start) + with patch("lab.prism.audit_http.threading.Thread", StalledThread), patch( + "lab.prism.audit_http._BoundedThreadingHttpServer", + RecordingBounded, + ): + starter.start() + self.assertTrue(stalled_started.wait(2.0)) + # The serve thread is alive but has not committed to + # serve_forever(); stop() must still return within its bound. + stop_started = time.monotonic() + stopped = facade.stop() + elapsed = time.monotonic() - stop_started + + self.assertLess(elapsed, 1.5) + self.assertFalse(stopped) + self.assertEqual(facade.state().lifecycle, "stop_timeout") + + release_stall.set() + starter.join(5.0) + self.assertFalse(starter.is_alive()) + self.assertEqual(len(start_outcome), 1) + self.assertIsInstance(start_outcome[0], RuntimeError) + self.assertEqual(len(RecordingBounded.instances), 1) + # The cancelled serve thread never entered serve_forever, and the + # listener socket is closed so a successor can bind the port. + self.assertEqual(RecordingBounded.instances[0].socket.fileno(), -1) + state = facade.state() + self.assertEqual(state.lifecycle, "stopped") + self.assertFalse(state.thread_alive) + self.assertIsNone(state.bound_address) + + def test_stop_reports_timeout_when_committed_loop_refuses_to_exit(self) -> None: + """A committed serve loop that ignores shutdown must not block stop() + past its budget; the facade reports stop_timeout (the state the + coordinator's audit_http_stop shutdown log is keyed on).""" + + class WedgedServer: + instance: WedgedServer | None = None + + def __init__(self, address: tuple[str, int], _handler: object) -> None: + self.server_address = address + self.ready = threading.Event() + self.release = threading.Event() + self.closed = False + WedgedServer.instance = self + + def serve_unless_startup_cancelled(self, *, poll_interval: float) -> None: + del poll_interval + self.ready.set() + self.release.wait(5.0) + + def cancel_startup(self) -> bool: + return True + + def shutdown(self) -> None: + # Deliberately ignore the request: the loop stays wedged. + return None + + def server_close(self) -> None: + self.closed = True + + facade = AuditHttpFacade( + FakeAuditHttpPort(), # type: ignore[arg-type] + AuditHttpConfig("127.0.0.1", 0, join_timeout_seconds=0.25), + ) + with patch( + "lab.prism.audit_http._BoundedThreadingHttpServer", + WedgedServer, + ): + self.assertEqual(facade.start().lifecycle, "running") + stop_started = time.monotonic() + stopped = facade.stop() + elapsed = time.monotonic() - stop_started + + self.assertFalse(stopped) + self.assertLess(elapsed, 1.5) + self.assertEqual(facade.state().lifecycle, "stop_timeout") + assert WedgedServer.instance is not None + self.assertTrue(WedgedServer.instance.closed) + WedgedServer.instance.release.set() + + def test_concurrent_stops_against_a_live_listener_all_return(self) -> None: + facade = AuditHttpFacade( + FakeAuditHttpPort(), # type: ignore[arg-type] + AuditHttpConfig("127.0.0.1", 0), + ) + self.assertEqual(facade.start().lifecycle, "running") + results: list[bool] = [] + errors: list[BaseException] = [] + + def run_stop() -> None: + try: + results.append(facade.stop()) + except BaseException as exc: + errors.append(exc) + + stoppers = [threading.Thread(target=run_stop) for _ in range(3)] + for stopper in stoppers: + stopper.start() + for stopper in stoppers: + stopper.join(5.0) + + self.assertTrue(all(not stopper.is_alive() for stopper in stoppers)) + self.assertEqual(errors, []) + self.assertEqual(len(results), 3) + self.assertTrue(all(results)) + state = facade.state() + self.assertEqual(state.lifecycle, "stopped") + self.assertFalse(state.thread_alive) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_prism_audit_root_scan.py b/tests/test_prism_audit_root_scan.py new file mode 100644 index 000000000..a5539358b --- /dev/null +++ b/tests/test_prism_audit_root_scan.py @@ -0,0 +1,144 @@ +#!/usr/bin/env python3 +"""Audit root enumeration through a fresh directory description. + +Some kernels and overlay filesystems serve a stale or truncated ``readdir`` +through a long-lived directory descriptor once the directory has been +mutated after its first enumeration. Observed on an aarch64 Docker host: a +conflict snapshot written moments earlier was absent from the next scan +through the pinned root descriptor, so quarantine deduplication created a +second copy and metrics and retention saw an empty root. The store now +reopens ``.`` relative to the pinned descriptor for every scan, keeping the +inode authority while never trusting the pinned description's own offset +or cache. +""" + +from __future__ import annotations + +import json +import os +from pathlib import Path +import tempfile +import unittest +from unittest import mock +from typing import Any + +from lab.prism.audit_artifacts import AuditArtifactConfig, AuditArtifactStore + + +BLOCK_HASH = "a1" * 32 + + +class FreshDirectoryScanTests(unittest.TestCase): + """Root enumeration never trusts the pinned descriptor's own readdir.""" + + def make_store(self, root: Path) -> AuditArtifactStore: + store = AuditArtifactStore( + AuditArtifactConfig( + root=root, + evidence_path=root / "evidence.json", + share_segment_size=4, + candidate_retention_seconds=0, + ), + canonicalizer=lambda bundle: json.dumps(bundle, separators=(",", ":")).encode(), + ) + self.addCleanup(store.close) + return store + + def test_scans_survive_a_stale_pinned_descriptor_enumeration(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + store = self.make_store(root) + real_listdir = os.listdir + stale_reads = 0 + + def stale_listdir(target: Any = None) -> list[str]: + nonlocal stale_reads + if isinstance(target, int) and target == store._root_fd: + # The defect this guards against: a long-lived directory + # description that stops reporting entries created after + # its first enumeration. + stale_reads += 1 + return [] + return real_listdir(target) + + rows = [{"share_seq": seq, "worker": f"miner-{seq}"} for seq in (1, 2)] + with mock.patch("lab.prism.audit_artifacts.os.listdir", side_effect=stale_listdir): + uri, _digest = store.write_audit_share_segment_range( + segment_first_share_seq=1, + segment_last_share_seq=4, + first_share_seq=1, + last_share_seq=2, + shares=rows, + ) + slot_bytes = Path(uri).read_bytes() + first = store.quarantine_audit_share_segment(Path(uri), expected_bytes=slot_bytes) + second = store.quarantine_audit_share_segment(Path(uri), expected_bytes=slot_bytes) + self.assertEqual(first, second) + self.assertEqual(len(list(root.glob("*.conflict-*"))), 1) + metrics = store.metrics_snapshot() + self.assertEqual(metrics["scan_error"], 0) + self.assertEqual(metrics["share_segment"]["files"], 1) + stale = store.issue_candidate(block_hash=BLOCK_HASH) + stale.path.write_bytes(b"stale") + store.release_candidate(stale) + os.utime(stale.path, ns=(1, 1)) + result = store.prune_best_effort() + self.assertEqual(result.errors, 0) + self.assertEqual(result.candidate_removed, 1) + self.assertFalse(stale.path.exists()) + self.assertEqual(stale_reads, 0) + + def test_scan_rejects_a_swapped_root_before_enumerating(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + base = Path(tmp) + root = base / "audit" + store = self.make_store(root) + (root / f"prism-audit-bundle-body-{BLOCK_HASH}-{'11' * 32}.json").write_bytes(b"x") + self.assertEqual(store.metrics_snapshot()["body"]["files"], 1) + root.rename(base / "pinned") + root.mkdir() + try: + metrics = store.metrics_snapshot() + self.assertEqual(metrics["scan_error"], 1) + with self.assertRaises(RuntimeError): + store._scan_root_names() + finally: + root.rmdir() + (base / "pinned").rename(root) + self.assertEqual(store.metrics_snapshot()["body"]["files"], 1) + + def test_scan_opens_and_closes_a_fresh_description_each_time(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + store = self.make_store(root) + (root / "prism-audit-bundle-body-operator.json").write_bytes(b"x") + opened: list[int] = [] + real_open = os.open + real_close = os.close + + def recording_open(*args: Any, **kwargs: Any) -> int: + fd = real_open(*args, **kwargs) + if kwargs.get("dir_fd") == store._root_fd and args[0] == ".": + opened.append(fd) + return fd + + closed: list[int] = [] + + def recording_close(fd: int) -> None: + closed.append(fd) + real_close(fd) + + with mock.patch("lab.prism.audit_artifacts.os.open", side_effect=recording_open), mock.patch( + "lab.prism.audit_artifacts.os.close", + side_effect=recording_close, + ): + names = store._scan_root_names() + store.metrics_snapshot() + self.assertIn("prism-audit-bundle-body-operator.json", names) + self.assertEqual(len(opened), 2) + self.assertTrue(all(fd in closed for fd in opened)) + self.assertNotIn(store._root_fd, closed) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_prism_backfill_audit_bundle_canonical.py b/tests/test_prism_backfill_audit_bundle_canonical.py new file mode 100644 index 000000000..12f8f50ed --- /dev/null +++ b/tests/test_prism_backfill_audit_bundle_canonical.py @@ -0,0 +1,1215 @@ +#!/usr/bin/env python3 + +import copy +import hashlib +import io +import json +import re +import tempfile +import unittest +from dataclasses import dataclass, field +from pathlib import Path +from typing import Any +from unittest import mock + +from lab.prism.audit_artifacts import AuditArtifactConfig, AuditArtifactStore +from lab.prism.backfill_audit_bundle_canonical import ( + LEDGER_PREFLIGHT_CANDIDATES, + AuditBundleLedgerAdapter, + BackfillWriterLeaseUnavailable, + CanonicalArtifactAdapter, + CanonicalBundleBackfill, + DEFAULT_BATCH_SIZE, + MissingCanonicalCapability, + audit_bundle_page_sql, + build_audit_store_from_env, + build_parser, + ledger_from_args, + main, +) +from lab.prism.share_ledger import PsqlShareLedger + + +AUDIT_SHARE_SEGMENT_SCHEMA = "qbit.prism.audit-share-segment.v1" + + +def sha256_hex(payload: bytes) -> str: + return hashlib.sha256(payload).hexdigest() + + +def canonical_bytes(bundle: dict[str, Any]) -> bytes: + """Stand-in for J1's canonicalizer: deterministic and order-independent.""" + + return json.dumps(bundle, sort_keys=True, separators=(",", ":")).encode("utf-8") + + +def digest_of(bundle: dict[str, Any]) -> str: + return sha256_hex(canonical_bytes(bundle)) + + +def block_hash(index: int) -> str: + return f"{index:064x}" + + +def share(seq: int) -> dict[str, Any]: + return { + "share_seq": seq, + "share_id": f"share-{seq}", + "miner_id": f"miner-{seq % 3}", + "share_difficulty": "1024", + } + + +def bundle_for(hash_hex: str, shares: list[dict[str, Any]]) -> dict[str, Any]: + return { + "schema": "qbit.prism.audit-bundle", + "block_hash": hash_hex, + "found_block": {"block_hash": hash_hex, "block_height": 4242}, + "shares": shares, + } + + +@dataclass +class ExternalBody: + """A compact/v2 external body as the artifact store would resolve it.""" + + bundle_without_shares: dict[str, Any] + shares_key: str + parts: list[dict[str, Any]] = field(default_factory=list) + + +class FakeArtifactStore: + """The assumed phase-2 canonical artifact surface, in memory.""" + + def __init__(self, journal: list[tuple[Any, ...]]) -> None: + self.journal = journal + self.canonical: dict[tuple[str, str], bytes] = {} + self.literal_bodies: dict[str, bytes] = {} + self.external_bodies: dict[str, ExternalBody] = {} + self.canonicalized: list[dict[str, Any]] = [] + self.write_failures: set[tuple[str, str]] = set() + + # -- phase-1 surface, unchanged ------------------------------------ + + def canonical_audit_bundle_bytes(self, bundle: dict[str, Any]) -> bytes: + self.canonicalized.append(copy.deepcopy(bundle)) + return canonical_bytes(bundle) + + def storage_json_bytes(self, payload: dict[str, Any]) -> bytes: + return json.dumps(payload, sort_keys=True, separators=(",", ":")).encode("utf-8") + + def audit_share_segment_payload( + self, + *, + first_share_seq: int, + last_share_seq: int, + shares: list[Any], + ) -> dict[str, Any]: + return { + "schema": AUDIT_SHARE_SEGMENT_SCHEMA, + "first_share_seq": first_share_seq, + "last_share_seq": last_share_seq, + "share_count": len(shares), + "shares": list(shares), + } + + # -- assumed phase-2 surface --------------------------------------- + + def read_canonical_audit_bundle(self, block_hash_hex: str, digest: str) -> bytes | None: + self.journal.append(("read_canonical", block_hash_hex, digest)) + return self.canonical.get((block_hash_hex, digest)) + + def write_canonical_audit_bundle( + self, + block_hash_hex: str, + digest: str, + payload: bytes, + ) -> None: + self.journal.append(("write_canonical", block_hash_hex, digest)) + if (block_hash_hex, digest) in self.write_failures: + raise RuntimeError("canonical artifact write refused") + self.canonical[(block_hash_hex, digest)] = bytes(payload) + + def read_literal_audit_body( + self, + body_uri: object, + *, + expected_sha256: str, + ) -> bytes | None: + self.journal.append(("read_literal", str(body_uri))) + payload = self.literal_bodies.get(str(body_uri)) + if payload is None: + return None + # A verified reader declines bytes that are not the advertised ones. + return payload if sha256_hex(payload) == expected_sha256 else None + + def reconstruct_external_audit_body( + self, + body_uri: object, + *, + expected_sha256: str, + load_missing_range: Any, + ) -> dict[str, Any]: + self.journal.append(("reconstruct", str(body_uri))) + body = self.external_bodies.get(str(body_uri)) + if body is None: + raise RuntimeError(f"no external audit body at {body_uri}") + shares: list[Any] = [] + for part in body.parts: + if part.get("on_disk", True): + shares.extend(copy.deepcopy(part["shares"])) + continue + declared = { + key: value + for key, value in part.items() + if key not in {"on_disk", "shares"} + } + shares.extend( + load_missing_range( + first_share_seq=part["first_share_seq"], + last_share_seq=part["last_share_seq"], + part=declared, + ) + ) + assembled = dict(body.bundle_without_shares) + assembled[body.shares_key] = shares + return assembled + + +class FakeLedger: + """The PostgreSQL surface the backfill depends on, in memory.""" + + def __init__( + self, + journal: list[tuple[Any, ...]], + rows: list[dict[str, Any]], + shares: dict[int, dict[str, Any]] | None = None, + ) -> None: + self.journal = journal + self.rows = {str(row["block_hash"]): row for row in rows} + self.shares = shares or {} + self.page_requests: list[tuple[str | None, int]] = [] + self.preflight_failures: set[tuple[str, str]] = set() + self.preflight_results: dict[tuple[str, str], Any] = {} + + @staticmethod + def _text_literal(value: str) -> str: + return "'" + str(value).replace("'", "''") + "'" + + def _run_read_json(self, sql: str) -> list[dict[str, Any]]: + if "FROM qbit_pool_audit_bundles" not in sql: + raise AssertionError("page query does not read the audit bundle table") + if "ORDER BY block_hash ASC" not in sql: + raise AssertionError("page query is not ordered by the primary key") + limit_match = re.search(r"LIMIT (\d+)", sql) + if limit_match is None: + raise AssertionError("page query is not bounded") + limit = int(limit_match.group(1)) + cursor_match = re.search(r"WHERE block_hash > '([^']*)'", sql) + start_after = cursor_match.group(1) if cursor_match else None + self.page_requests.append((start_after, limit)) + keys = sorted(self.rows) + if start_after is not None: + keys = [key for key in keys if key > start_after] + return [copy.deepcopy(self.rows[key]) for key in keys[:limit]] + + def load_audit_share_ledger_range( + self, + *, + first_share_seq: int, + last_share_seq: int, + ) -> list[dict[str, Any]]: + self.journal.append(("load_share_range", first_share_seq, last_share_seq)) + return [ + copy.deepcopy(self.shares[seq]) + for seq in range(first_share_seq, last_share_seq + 1) + if seq in self.shares + ] + + def preflight_audit_bundle_publication( + self, + *, + block_hash: str, + audit_bundle_sha256: str, + ) -> Any: + self.journal.append(("preflight", block_hash, audit_bundle_sha256)) + if (block_hash, audit_bundle_sha256) in self.preflight_failures: + raise RuntimeError("writer lease is not active") + return self.preflight_results.get( + (block_hash, audit_bundle_sha256), + {"confirmed": True}, + ) + + +def inline_row(index: int, *, digest: str | None = None) -> dict[str, Any]: + hash_hex = block_hash(index) + bundle = bundle_for(hash_hex, [share(index)]) + return { + "block_hash": hash_hex, + "audit_bundle_sha256": digest or digest_of(bundle), + "body_uri": None, + "audit_bundle": bundle, + } + + +class BackfillHarness: + """Wire fakes to the backfill under test and expose what each run did.""" + + def __init__( + self, + rows: list[dict[str, Any]], + shares: dict[int, dict[str, Any]] | None = None, + ) -> None: + self.journal: list[tuple[Any, ...]] = [] + self.store = FakeArtifactStore(self.journal) + self.ledger = FakeLedger(self.journal, rows, shares) + self.stderr = io.StringIO() + + def run(self, **kwargs: Any) -> dict[str, Any]: + backfill = CanonicalBundleBackfill( + artifacts=CanonicalArtifactAdapter(self.store), + ledger=AuditBundleLedgerAdapter(self.ledger), + stderr=self.stderr, + **kwargs, + ) + return backfill.run() + + def publication_events(self) -> list[tuple[Any, ...]]: + return [ + event + for event in self.journal + if event[0] in {"preflight", "write_canonical"} + ] + + def errors(self) -> str: + return self.stderr.getvalue() + + +class CanonicalBundleBackfillTest(unittest.TestCase): + def test_inline_history_publishes_through_the_real_phase_2_store(self) -> None: + row = inline_row(1) + journal: list[tuple[Any, ...]] = [] + ledger = FakeLedger(journal, [row]) + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + store = AuditArtifactStore( + AuditArtifactConfig( + root=root, + evidence_path=root / "evidence.json", + ), + canonicalizer=canonical_bytes, + ) + try: + result = CanonicalBundleBackfill( + artifacts=CanonicalArtifactAdapter(store), + ledger=AuditBundleLedgerAdapter(ledger), + ).run() + + self.assertEqual(result["written"], 1) + self.assertEqual( + store.read_canonical_audit_bundle( + str(row["block_hash"]), + str(row["audit_bundle_sha256"]), + ), + canonical_bytes(row["audit_bundle"]), + ) + self.assertEqual( + journal[-1], + ( + "preflight", + row["block_hash"], + row["audit_bundle_sha256"], + ), + ) + finally: + store.close() + + def test_first_run_publishes_every_shape_and_rerun_is_idempotent(self) -> None: + inline = inline_row(1) + + literal_hash = block_hash(2) + # Deliberately not what any canonicalizer here would emit: promotion + # must copy these bytes, not re-derive them. + literal_payload = b'{ "legacy": true, "block_hash": "' + literal_hash.encode() + b'" }\n' + literal = { + "block_hash": literal_hash, + "audit_bundle_sha256": sha256_hex(literal_payload), + "body_uri": f"prism-audit-bundle-body-{literal_hash}.json", + "audit_bundle": None, + } + + compact_hash = block_hash(3) + compact_bundle = bundle_for(compact_hash, [share(7), share(8)]) + compact = { + "block_hash": compact_hash, + "audit_bundle_sha256": digest_of(compact_bundle), + "body_uri": f"prism-audit-bundle-body-{compact_hash}.json", + "audit_bundle": None, + } + + harness = BackfillHarness([inline, literal, compact]) + harness.store.literal_bodies[str(literal["body_uri"])] = literal_payload + harness.store.external_bodies[str(compact["body_uri"])] = ExternalBody( + bundle_without_shares={ + key: value for key, value in compact_bundle.items() if key != "shares" + }, + shares_key="shares", + parts=[ + { + "kind": "segment", + "first_share_seq": 7, + "last_share_seq": 8, + "shares": [share(7), share(8)], + "on_disk": True, + } + ], + ) + + first = harness.run() + self.assertEqual(first["visited"], 3) + self.assertEqual(first["written"], 3) + self.assertEqual(first["already_present"], 0) + self.assertEqual(first["literal_freebie"], 1) + self.assertEqual(first["reconstructed"], 1) + self.assertEqual(first["inline_canonicalized"], 1) + self.assertEqual(first["mismatch"], 0) + self.assertEqual(first["errors"], 0) + self.assertEqual(first["exit_code"], 0) + self.assertEqual(first["last_checkpoint"], compact_hash) + self.assertEqual(harness.errors(), "") + + published = dict(harness.store.canonical) + self.assertEqual(len(published), 3) + for (_hash, digest), payload in published.items(): + self.assertEqual(sha256_hex(payload), digest) + + # An identical rerun against the same store republishes nothing. + rerun_journal_start = len(harness.journal) + second = harness.run() + self.assertEqual(second["visited"], 3) + self.assertEqual(second["already_present"], 3) + self.assertEqual(second["written"], 0) + self.assertEqual(second["mismatch"], 0) + self.assertEqual(second["errors"], 0) + self.assertEqual(second["exit_code"], 0) + self.assertEqual(dict(harness.store.canonical), published) + rerun_events = [ + event + for event in harness.journal[rerun_journal_start:] + if event[0] in {"preflight", "write_canonical"} + ] + self.assertEqual(rerun_events, []) + + def test_pre_v2_literal_body_is_promoted_without_reserialization(self) -> None: + hash_hex = block_hash(11) + # Whitespace and key order a re-serialization would normalize away. + literal_payload = b'{\n "shares": [],\n "block_hash": "' + hash_hex.encode() + b'"\n}\n' + digest = sha256_hex(literal_payload) + row = { + "block_hash": hash_hex, + "audit_bundle_sha256": digest, + "body_uri": f"prism-audit-bundle-body-{hash_hex}.json", + "audit_bundle": None, + } + harness = BackfillHarness([row]) + harness.store.literal_bodies[str(row["body_uri"])] = literal_payload + + summary = harness.run() + + self.assertEqual(summary["written"], 1) + self.assertEqual(summary["literal_freebie"], 1) + self.assertEqual(summary["reconstructed"], 0) + self.assertEqual(summary["inline_canonicalized"], 0) + self.assertEqual(harness.store.canonical[(hash_hex, digest)], literal_payload) + # The canonicalizer is the proof: promoting must not parse or re-emit. + self.assertEqual(harness.store.canonicalized, []) + self.assertNotIn("reconstruct", [event[0] for event in harness.journal]) + + def test_modern_reconstruction_reloads_absent_segment_from_database(self) -> None: + hash_hex = block_hash(21) + on_disk = [share(1), share(2)] + from_db = [share(3), share(4)] + full_bundle = bundle_for(hash_hex, on_disk + from_db) + digest = digest_of(full_bundle) + row = { + "block_hash": hash_hex, + "audit_bundle_sha256": digest, + "body_uri": f"prism-audit-bundle-body-{hash_hex}.json", + "audit_bundle": None, + } + harness = BackfillHarness( + [row], + shares={item["share_seq"]: item for item in from_db}, + ) + missing_part = { + "kind": "segment_range", + "first_share_seq": 3, + "last_share_seq": 4, + "range_sha256": sha256_hex( + harness.store.storage_json_bytes( + harness.store.audit_share_segment_payload( + first_share_seq=3, + last_share_seq=4, + shares=from_db, + ) + ) + ), + "on_disk": False, + } + harness.store.external_bodies[str(row["body_uri"])] = ExternalBody( + bundle_without_shares={ + key: value for key, value in full_bundle.items() if key != "shares" + }, + shares_key="shares", + parts=[ + { + "kind": "segment", + "first_share_seq": 1, + "last_share_seq": 2, + "shares": on_disk, + "on_disk": True, + }, + missing_part, + ], + ) + + summary = harness.run() + + self.assertEqual(summary["written"], 1) + self.assertEqual(summary["reconstructed"], 1) + self.assertEqual(summary["db_segment_reloads"], 1) + self.assertEqual(summary["literal_freebie"], 0) + self.assertEqual(summary["mismatch"], 0) + self.assertEqual(summary["errors"], 0) + self.assertIn(("load_share_range", 3, 4), harness.journal) + self.assertEqual( + harness.store.canonical[(hash_hex, digest)], + canonical_bytes(full_bundle), + ) + + def test_database_range_that_fails_its_segment_digest_publishes_nothing(self) -> None: + hash_hex = block_hash(22) + on_disk = [share(1)] + expected_db = [share(3), share(4)] + full_bundle = bundle_for(hash_hex, on_disk + expected_db) + digest = digest_of(full_bundle) + row = { + "block_hash": hash_hex, + "audit_bundle_sha256": digest, + "body_uri": f"prism-audit-bundle-body-{hash_hex}.json", + "audit_bundle": None, + } + tampered = [dict(expected_db[0], miner_id="someone-else"), expected_db[1]] + harness = BackfillHarness( + [row], + shares={item["share_seq"]: item for item in tampered}, + ) + harness.store.external_bodies[str(row["body_uri"])] = ExternalBody( + bundle_without_shares={ + key: value for key, value in full_bundle.items() if key != "shares" + }, + shares_key="shares", + parts=[ + { + "kind": "segment", + "first_share_seq": 1, + "last_share_seq": 1, + "shares": on_disk, + "on_disk": True, + }, + { + "kind": "segment_range", + "first_share_seq": 3, + "last_share_seq": 4, + "range_sha256": sha256_hex( + harness.store.storage_json_bytes( + harness.store.audit_share_segment_payload( + first_share_seq=3, + last_share_seq=4, + shares=expected_db, + ) + ) + ), + "on_disk": False, + }, + ], + ) + + summary = harness.run() + + self.assertEqual(summary["mismatch"], 1) + self.assertEqual(summary["written"], 0) + self.assertEqual(summary["db_segment_reloads"], 0) + self.assertEqual(summary["exit_code"], 1) + self.assertEqual(harness.store.canonical, {}) + self.assertIn("canonicalizes to", harness.errors()) + + def test_incomplete_database_range_is_refused(self) -> None: + hash_hex = block_hash(23) + full_bundle = bundle_for(hash_hex, [share(3), share(4)]) + row = { + "block_hash": hash_hex, + "audit_bundle_sha256": digest_of(full_bundle), + "body_uri": f"prism-audit-bundle-body-{hash_hex}.json", + "audit_bundle": None, + } + # The ledger can only serve half of the absent range. + harness = BackfillHarness([row], shares={3: share(3)}) + harness.store.external_bodies[str(row["body_uri"])] = ExternalBody( + bundle_without_shares={ + key: value for key, value in full_bundle.items() if key != "shares" + }, + shares_key="shares", + parts=[ + { + "kind": "segment_range", + "first_share_seq": 3, + "last_share_seq": 4, + "on_disk": False, + } + ], + ) + + summary = harness.run() + + self.assertEqual(summary["written"], 0) + self.assertEqual(summary["errors"], 1) + self.assertEqual(summary["exit_code"], 1) + self.assertEqual(harness.store.canonical, {}) + self.assertIn("incomplete or out of order", harness.errors()) + + def test_digest_mismatch_is_loud_and_publishes_nothing(self) -> None: + good = inline_row(31) + bad = inline_row(32, digest="ab" * 32) + harness = BackfillHarness([good, bad]) + + summary = harness.run() + + self.assertEqual(summary["visited"], 2) + self.assertEqual(summary["written"], 1) + self.assertEqual(summary["mismatch"], 1) + self.assertEqual(summary["exit_code"], 1) + self.assertEqual(list(harness.store.canonical), [(good["block_hash"], good["audit_bundle_sha256"])]) + + reported = harness.errors() + self.assertIn(str(bad["block_hash"]), reported) + self.assertIn("ab" * 32, reported) + self.assertIn("inline bundle canonicalizes to", reported) + # Never rewrite the advertised digest to whatever the bytes hash to. + self.assertNotIn(("write_canonical", bad["block_hash"], digest_of(bad["audit_bundle"])), harness.journal) + # The bad row still advances the checkpoint so a resume makes progress. + self.assertEqual(summary["last_checkpoint"], bad["block_hash"]) + self.assertEqual(len(summary["failed_rows"]), 1) + self.assertEqual( + summary["failed_rows"][0]["block_hash"], + bad["block_hash"], + ) + self.assertEqual( + summary["failed_rows"][0]["audit_bundle_sha256"], + bad["audit_bundle_sha256"], + ) + self.assertEqual(summary["failed_rows"][0]["kind"], "mismatch") + + def test_existing_canonical_artifact_with_wrong_bytes_is_reported(self) -> None: + row = inline_row(41) + harness = BackfillHarness([row]) + harness.store.canonical[ + (str(row["block_hash"]), str(row["audit_bundle_sha256"])) + ] = b"not the canonical bytes" + + summary = harness.run() + + self.assertEqual(summary["mismatch"], 1) + self.assertEqual(summary["written"], 0) + self.assertEqual(summary["already_present"], 0) + self.assertEqual(summary["exit_code"], 1) + self.assertIn("existing canonical artifact hashes to", harness.errors()) + # The corrupt artifact is left exactly as found, never overwritten. + self.assertEqual( + harness.store.canonical[ + (str(row["block_hash"]), str(row["audit_bundle_sha256"])) + ], + b"not the canonical bytes", + ) + + def test_dry_run_verifies_without_publishing_or_taking_the_lease(self) -> None: + rows = [inline_row(51), inline_row(52)] + harness = BackfillHarness(rows) + + summary = harness.run(dry_run=True) + + self.assertEqual(summary["visited"], 2) + self.assertEqual(summary["dry_run"], 2) + self.assertEqual(summary["written"], 0) + self.assertEqual(summary["mismatch"], 0) + self.assertEqual(summary["errors"], 0) + self.assertEqual(summary["exit_code"], 0) + self.assertTrue(summary["dry_run_mode"]) + self.assertEqual(harness.store.canonical, {}) + # No filesystem publication means no lease refresh either. + self.assertEqual(harness.publication_events(), []) + + def test_dry_run_still_reports_mismatches(self) -> None: + harness = BackfillHarness([inline_row(53, digest="cd" * 32)]) + + summary = harness.run(dry_run=True) + + self.assertEqual(summary["mismatch"], 1) + self.assertEqual(summary["dry_run"], 0) + self.assertEqual(summary["exit_code"], 1) + self.assertEqual(harness.publication_events(), []) + + def test_batches_page_in_resumable_primary_key_order(self) -> None: + indexes = [61, 62, 63, 64, 65] + rows = [inline_row(index) for index in indexes] + # Insertion order is shuffled: the ordering must come from the query. + harness = BackfillHarness([rows[3], rows[0], rows[4], rows[2], rows[1]]) + + summary = harness.run(batch_size=2) + + self.assertEqual(summary["visited"], 5) + self.assertEqual(summary["written"], 5) + self.assertEqual(summary["last_checkpoint"], block_hash(65)) + self.assertEqual( + harness.ledger.page_requests, + [(None, 2), (block_hash(62), 2), (block_hash(64), 2)], + ) + written = [ + event[1] for event in harness.journal if event[0] == "write_canonical" + ] + self.assertEqual(written, [block_hash(index) for index in indexes]) + + def test_limit_stops_early_and_start_after_resumes_from_the_checkpoint(self) -> None: + indexes = [71, 72, 73, 74, 75] + rows = [inline_row(index) for index in indexes] + + first = BackfillHarness(rows) + first_summary = first.run(batch_size=2, limit=3) + self.assertEqual(first_summary["visited"], 3) + self.assertEqual(first_summary["written"], 3) + self.assertEqual(first_summary["last_checkpoint"], block_hash(73)) + self.assertEqual(first_summary["limit"], 3) + self.assertEqual(first.ledger.page_requests, [(None, 2), (block_hash(72), 1)]) + + resumed = BackfillHarness(rows) + resumed_summary = resumed.run( + batch_size=2, + start_after=first_summary["last_checkpoint"], + ) + self.assertEqual(resumed_summary["visited"], 2) + self.assertEqual(resumed_summary["written"], 2) + self.assertEqual(resumed_summary["start_after"], block_hash(73)) + self.assertEqual(resumed_summary["last_checkpoint"], block_hash(75)) + self.assertEqual( + [event[1] for event in resumed.journal if event[0] == "write_canonical"], + [block_hash(74), block_hash(75)], + ) + self.assertEqual(resumed.ledger.page_requests[0], (block_hash(73), 2)) + + def test_resuming_past_the_last_row_is_a_clean_no_op(self) -> None: + harness = BackfillHarness([inline_row(81)]) + + summary = harness.run(start_after=block_hash(81)) + + self.assertEqual(summary["visited"], 0) + self.assertEqual(summary["written"], 0) + self.assertEqual(summary["exit_code"], 0) + self.assertEqual(summary["last_checkpoint"], block_hash(81)) + + def test_lease_preflight_runs_immediately_before_every_publication(self) -> None: + rows = [inline_row(91), inline_row(92)] + harness = BackfillHarness(rows) + + summary = harness.run() + + self.assertEqual(summary["written"], 2) + self.assertEqual( + harness.publication_events(), + [ + ("preflight", rows[0]["block_hash"], rows[0]["audit_bundle_sha256"]), + ("write_canonical", rows[0]["block_hash"], rows[0]["audit_bundle_sha256"]), + ("preflight", rows[1]["block_hash"], rows[1]["audit_bundle_sha256"]), + ("write_canonical", rows[1]["block_hash"], rows[1]["audit_bundle_sha256"]), + ], + ) + + def test_preflight_refusal_withholds_the_publication(self) -> None: + rows = [inline_row(101), inline_row(102)] + harness = BackfillHarness(rows) + harness.ledger.preflight_failures.add( + (str(rows[0]["block_hash"]), str(rows[0]["audit_bundle_sha256"])) + ) + + summary = harness.run() + + self.assertEqual(summary["written"], 1) + self.assertEqual(summary["errors"], 1) + self.assertEqual(summary["exit_code"], 1) + self.assertNotIn( + ("write_canonical", rows[0]["block_hash"], rows[0]["audit_bundle_sha256"]), + harness.journal, + ) + self.assertIn("writer lease is not active", harness.errors()) + # A refusal on one row does not stop the requested range. + self.assertIn( + ("write_canonical", rows[1]["block_hash"], rows[1]["audit_bundle_sha256"]), + harness.journal, + ) + + def test_preflight_error_result_withholds_the_publication(self) -> None: + row = inline_row(103) + harness = BackfillHarness([row]) + harness.ledger.preflight_results[ + (str(row["block_hash"]), str(row["audit_bundle_sha256"])) + ] = {"error": "existing audit bundle does not match payload"} + + summary = harness.run() + + self.assertEqual(summary["written"], 0) + self.assertEqual(summary["errors"], 1) + self.assertEqual(summary["exit_code"], 1) + self.assertEqual(harness.store.canonical, {}) + self.assertIn("existing audit bundle does not match payload", harness.errors()) + + def test_unconfirmed_preflight_withholds_the_publication(self) -> None: + row = inline_row(104) + harness = BackfillHarness([row]) + harness.ledger.preflight_results[ + (str(row["block_hash"]), str(row["audit_bundle_sha256"])) + ] = {"confirmed": False} + + summary = harness.run() + + self.assertEqual(summary["written"], 0) + self.assertEqual(summary["errors"], 1) + self.assertEqual(harness.store.canonical, {}) + + def test_row_without_any_recoverable_body_is_reported(self) -> None: + hash_hex = block_hash(111) + row = { + "block_hash": hash_hex, + "audit_bundle_sha256": digest_of(bundle_for(hash_hex, [])), + "body_uri": None, + "audit_bundle": None, + } + harness = BackfillHarness([row]) + + summary = harness.run() + + self.assertEqual(summary["errors"], 1) + self.assertEqual(summary["written"], 0) + self.assertEqual(summary["exit_code"], 1) + self.assertIn("neither an inline bundle nor an external body", harness.errors()) + + def test_non_canonical_row_identity_is_reported(self) -> None: + row = inline_row(112) + row["audit_bundle_sha256"] = "not-hex" + harness = BackfillHarness([row]) + + summary = harness.run() + + self.assertEqual(summary["errors"], 1) + self.assertEqual(summary["written"], 0) + self.assertEqual(summary["exit_code"], 1) + self.assertIn("row identity is not canonical", harness.errors()) + + def test_unreadable_external_body_falls_back_to_the_inline_bundle(self) -> None: + hash_hex = block_hash(121) + bundle = bundle_for(hash_hex, [share(5)]) + row = { + "block_hash": hash_hex, + "audit_bundle_sha256": digest_of(bundle), + # Registered nowhere: the literal reader declines and the + # reconstruction helper raises. + "body_uri": f"prism-audit-bundle-body-{hash_hex}.json", + "audit_bundle": bundle, + } + harness = BackfillHarness([row]) + + summary = harness.run() + + self.assertEqual(summary["written"], 1) + self.assertEqual(summary["inline_canonicalized"], 1) + self.assertEqual(summary["mismatch"], 0) + self.assertEqual(summary["errors"], 0) + self.assertEqual( + harness.store.canonical[(hash_hex, str(row["audit_bundle_sha256"]))], + canonical_bytes(bundle), + ) + + def test_store_write_failure_is_counted_and_scanning_continues(self) -> None: + rows = [inline_row(131), inline_row(132)] + harness = BackfillHarness(rows) + harness.store.write_failures.add( + (str(rows[0]["block_hash"]), str(rows[0]["audit_bundle_sha256"])) + ) + + summary = harness.run() + + self.assertEqual(summary["written"], 1) + self.assertEqual(summary["errors"], 1) + self.assertEqual(summary["exit_code"], 1) + self.assertEqual(summary["last_checkpoint"], rows[1]["block_hash"]) + + def test_summary_is_machine_readable_and_names_the_bound_interface(self) -> None: + harness = BackfillHarness([inline_row(141)]) + + summary = harness.run() + + for key in ( + "schema", + "visited", + "written", + "already_present", + "dry_run", + "literal_freebie", + "mismatch", + "errors", + "last_checkpoint", + "exit_code", + "failed_rows", + ): + self.assertIn(key, summary) + self.assertEqual( + summary["schema"], + "qbit.prism.audit-bundle-canonical-backfill.v1", + ) + # The summary must survive a round trip through JSON unchanged. + self.assertEqual(json.loads(json.dumps(summary, sort_keys=True)), summary) + self.assertEqual( + summary["interface"]["artifacts"]["read_canonical"], + "read_canonical_audit_bundle", + ) + self.assertEqual( + summary["interface"]["ledger"]["publication_preflight"], + "preflight_audit_bundle_publication", + ) + self.assertEqual( + summary["interface"]["ledger"]["preflight_digest_keyword"], + "audit_bundle_sha256", + ) + + +class WithoutAttributes: + """A build that simply does not have some of the assumed methods.""" + + def __init__(self, target: Any, *hidden: str) -> None: + self.__dict__["_target"] = target + self.__dict__["_hidden"] = frozenset(hidden) + + def __getattr__(self, name: str) -> Any: + if name in self.__dict__["_hidden"]: + raise AttributeError(name) + return getattr(self.__dict__["_target"], name) + + +class AdapterBindingTest(unittest.TestCase): + def test_phase_2_production_interfaces_bind_and_round_trip(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + store = AuditArtifactStore( + AuditArtifactConfig( + root=root, + evidence_path=root / "evidence.json", + ), + canonicalizer=canonical_bytes, + ) + try: + artifacts = CanonicalArtifactAdapter(store) + block = "ab" * 32 + payload = b'{"schema":"qbit.prism.audit-bundle.v1"}' + digest = sha256_hex(payload) + + artifacts.write_canonical(block, digest, payload) + + self.assertEqual(artifacts.read_canonical(block, digest), payload) + self.assertEqual( + artifacts.resolved_interface()["literal_body_reader"], + "literal_canonical_bundle_bytes", + ) + self.assertEqual( + artifacts.resolved_interface()["external_body_reconstruction"], + "canonical_bundle_bytes_from_external_body", + ) + finally: + store.close() + + ledger = AuditBundleLedgerAdapter(PsqlShareLedger.__new__(PsqlShareLedger)) + self.assertEqual( + ledger.resolved_interface()["publication_preflight"], + "preflight_canonical_bundle_publication", + ) + + def test_missing_canonical_capability_is_named(self) -> None: + journal: list[tuple[Any, ...]] = [] + store = WithoutAttributes( + FakeArtifactStore(journal), + "read_canonical_audit_bundle", + ) + with self.assertRaises(MissingCanonicalCapability) as caught: + CanonicalArtifactAdapter(store) + self.assertIn("read_canonical_audit_bundle", str(caught.exception)) + + def test_missing_ledger_preflight_is_named(self) -> None: + journal: list[tuple[Any, ...]] = [] + ledger = WithoutAttributes( + FakeLedger(journal, []), + *LEDGER_PREFLIGHT_CANDIDATES, + ) + with self.assertRaises(MissingCanonicalCapability) as caught: + AuditBundleLedgerAdapter(ledger) + self.assertIn("preflight", str(caught.exception)) + + def test_reconstruction_helper_without_callback_still_binds(self) -> None: + """A build without the phase-2 helper falls back to the phase-1 reader.""" + + journal: list[tuple[Any, ...]] = [] + target = FakeArtifactStore(journal) + target.read_external_body = ( # type: ignore[attr-defined] + lambda body_uri, *, expected_sha256: None + ) + store = WithoutAttributes(target, "reconstruct_external_audit_body") + adapter = CanonicalArtifactAdapter(store) + self.assertEqual( + adapter.resolved_interface()["external_body_reconstruction"], + "read_external_body", + ) + # The phase-1 reader takes no missing-range callback, so it must not be + # handed one; binding still succeeds and the row simply cannot recover + # an absent segment. + self.assertIsNone( + adapter.canonical_bytes_from_external_body( + "bb" * 32, + "body", + "aa" * 32, + lambda **_: [], + ) + ) + + +class PageQueryTest(unittest.TestCase): + def test_first_page_has_no_cursor_predicate(self) -> None: + sql = audit_bundle_page_sql(start_after=None, limit=25) + self.assertNotIn("WHERE", sql) + self.assertIn("ORDER BY block_hash ASC", sql) + self.assertIn("LIMIT 25", sql) + self.assertIn("FROM qbit_pool_audit_bundles", sql) + + def test_resumed_page_filters_and_escapes_the_cursor(self) -> None: + sql = audit_bundle_page_sql(start_after="a b'c", limit=5) + self.assertIn("WHERE block_hash > 'a b''c'", sql) + self.assertIn("LIMIT 5", sql) + + def test_page_limit_must_be_positive(self) -> None: + with self.assertRaises(ValueError): + audit_bundle_page_sql(start_after=None, limit=0) + + def test_page_query_selects_only_the_columns_the_backfill_reads(self) -> None: + sql = audit_bundle_page_sql(start_after=None, limit=1) + for column in ("block_hash", "audit_bundle_sha256", "body_uri", "audit_bundle"): + self.assertIn(column, sql) + # A backfill never writes the ledger. + for statement in ("INSERT", "UPDATE", "DELETE"): + self.assertNotIn(statement, sql.upper()) + + +class ConstructionTest(unittest.TestCase): + def _backfill(self, **kwargs: Any) -> CanonicalBundleBackfill: + journal: list[tuple[Any, ...]] = [] + return CanonicalBundleBackfill( + artifacts=CanonicalArtifactAdapter(FakeArtifactStore(journal)), + ledger=AuditBundleLedgerAdapter(FakeLedger(journal, [])), + **kwargs, + ) + + def test_batch_size_must_be_positive(self) -> None: + with self.assertRaises(ValueError): + self._backfill(batch_size=0) + + def test_limit_must_be_positive_when_given(self) -> None: + with self.assertRaises(ValueError): + self._backfill(limit=0) + + def test_dry_run_store_has_no_publication_authority(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + store = build_audit_store_from_env( + { + "PRISM_AUDIT_DIR": str(root), + "PRISM_EVIDENCE_PATH": str(root / "evidence.json"), + }, + read_only=True, + ) + try: + self.assertFalse((root / ".prism-audit-publication.lock").exists()) + payload = b'{"schema":"qbit.prism.audit-bundle.v1"}' + with self.assertRaisesRegex(RuntimeError, "read-only"): + store.write_canonical_audit_bundle( + "ab" * 32, + sha256_hex(payload), + payload, + ) + finally: + store.close() + + def test_dry_run_ledger_uses_read_only_database_sessions(self) -> None: + args = build_parser().parse_args( + [ + "--dry-run", + "--psql-command", + "psql postgresql://example.invalid/qbit", + ] + ) + with mock.patch( + "lab.prism.backfill_audit_bundle_canonical.PsqlShareLedger" + ) as constructor: + ledger_from_args(args, mock.sentinel.store) + + self.assertTrue(constructor.call_args.kwargs["read_only"]) + self.assertFalse(constructor.call_args.kwargs["initialize_schema"]) + + def test_publish_ledger_refuses_same_identity_lease_waits(self) -> None: + args = build_parser().parse_args( + ["--psql-command", "psql postgresql://example.invalid/qbit"] + ) + with mock.patch( + "lab.prism.backfill_audit_bundle_canonical.PsqlShareLedger" + ) as constructor: + ledger_from_args(args, mock.sentinel.store) + + refuse_wait = constructor.call_args.kwargs["lease_retry_sleep"] + with self.assertRaisesRegex( + BackfillWriterLeaseUnavailable, + "stop the coordinator", + ): + refuse_wait(60) + + +class CommandLifecycleTest(unittest.TestCase): + @staticmethod + def resources(journal: list[str]) -> tuple[mock.Mock, mock.Mock]: + store = mock.Mock() + ledger = mock.Mock() + ledger.release_writer_lease.side_effect = lambda: ( + journal.append("release_writer_lease") or True + ) + ledger.close.side_effect = lambda: journal.append("ledger_close") + store.close.side_effect = lambda: journal.append("store_close") + return store, ledger + + def run_main( + self, + *, + store: mock.Mock, + ledger: mock.Mock, + backfill: mock.Mock, + ) -> int: + with ( + mock.patch( + "lab.prism.backfill_audit_bundle_canonical.build_audit_store_from_env", + return_value=store, + ), + mock.patch( + "lab.prism.backfill_audit_bundle_canonical.CanonicalArtifactAdapter", + return_value=mock.sentinel.artifacts, + ), + mock.patch( + "lab.prism.backfill_audit_bundle_canonical.ledger_from_args", + return_value=ledger, + ), + mock.patch( + "lab.prism.backfill_audit_bundle_canonical.AuditBundleLedgerAdapter", + return_value=mock.sentinel.ledger_adapter, + ), + mock.patch( + "lab.prism.backfill_audit_bundle_canonical.CanonicalBundleBackfill", + return_value=backfill, + ), + mock.patch("sys.stdout", new=io.StringIO()), + ): + return main([]) + + def test_success_releases_lease_then_closes_ledger_and_store(self) -> None: + journal: list[str] = [] + store, ledger = self.resources(journal) + backfill = mock.Mock() + backfill.run.return_value = {"exit_code": 0} + + self.assertEqual( + self.run_main(store=store, ledger=ledger, backfill=backfill), + 0, + ) + self.assertEqual( + journal, + ["release_writer_lease", "ledger_close", "store_close"], + ) + + def test_failure_releases_lease_then_closes_ledger_and_store(self) -> None: + journal: list[str] = [] + store, ledger = self.resources(journal) + backfill = mock.Mock() + backfill.run.side_effect = RuntimeError("backfill failed") + + with self.assertRaisesRegex(RuntimeError, "backfill failed"): + self.run_main(store=store, ledger=ledger, backfill=backfill) + self.assertEqual( + journal, + ["release_writer_lease", "ledger_close", "store_close"], + ) + + def test_conflicting_writer_lease_fails_clearly_and_closes_store(self) -> None: + journal: list[str] = [] + store, _ledger = self.resources(journal) + with ( + mock.patch( + "lab.prism.backfill_audit_bundle_canonical.build_audit_store_from_env", + return_value=store, + ), + mock.patch( + "lab.prism.backfill_audit_bundle_canonical.CanonicalArtifactAdapter", + return_value=mock.sentinel.artifacts, + ), + mock.patch( + "lab.prism.backfill_audit_bundle_canonical.ledger_from_args", + side_effect=RuntimeError("lease held by prism-coordinator"), + ), + ): + with self.assertRaisesRegex(SystemExit, "stop the coordinator"): + main([]) + + self.assertEqual(journal, ["store_close"]) + + +class CommandLineTest(unittest.TestCase): + def test_checkpoint_batch_and_dry_run_options_exist(self) -> None: + args = build_parser().parse_args( + ["--start-after", block_hash(1), "--batch-size", "7", "--limit", "3", "--dry-run"] + ) + self.assertEqual(args.start_after, block_hash(1)) + self.assertEqual(args.batch_size, 7) + self.assertEqual(args.limit, 3) + self.assertTrue(args.dry_run) + + def test_defaults_publish_the_whole_table(self) -> None: + args = build_parser().parse_args([]) + self.assertIsNone(args.start_after) + self.assertIsNone(args.limit) + self.assertFalse(args.dry_run) + self.assertEqual(args.batch_size, DEFAULT_BATCH_SIZE) + + def test_established_postgres_options_are_available(self) -> None: + args = build_parser().parse_args([]) + for attribute in ( + "psql_command", + "writer_id", + "writer_epoch", + "writer_session_token", + "lease_ttl_seconds", + "lease_acquire_attempts", + "postgres_tcp_keepalives_count", + ): + self.assertTrue(hasattr(args, attribute), attribute) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_prism_background_services.py b/tests/test_prism_background_services.py new file mode 100644 index 000000000..37af1d5db --- /dev/null +++ b/tests/test_prism_background_services.py @@ -0,0 +1,1022 @@ +#!/usr/bin/env python3 + +from __future__ import annotations + +import threading +from types import SimpleNamespace +import unittest +from unittest.mock import patch + +from lab.prism.background_services import ( + BackgroundServiceRegistry, + BackgroundServiceSpec, + WatchdogPorts, + WatchdogService, +) +from lab.prism.prism_coordinator import PrismCoordinator + + +def make_watchdog_ports(**overrides: object) -> WatchdogPorts: + """Inert watchdog ports; individual tests override what they exercise.""" + ports: dict[str, object] = { + "wait_for_stop": lambda _timeout: True, + "interval_seconds": lambda: 1.0, + "fatal_exit_requested": lambda: False, + "publication_state": lambda _now: (None, 0.0, 1.0, 1.0), + "hard_exit": lambda _reason: None, + "liveness_enabled": lambda: True, + "overdue_heartbeats": lambda _now: [], + "liveness_timeout_seconds": lambda: 60.0, + "coordination_budget_seconds": lambda: 900.0, + "publication_budget_seconds": lambda: 30.0, + "ensure_job_cache_state": lambda: None, + "publication_failure_expired": lambda _now: False, + "publication_divergence_since": lambda: None, + "lease_release_timeout_seconds": lambda: 0.05, + "shutdown_controller": lambda: None, + "request_shutdown": lambda: None, + "release_ledger_lease": lambda _deadline: True, + "lease_failure_reason": lambda: None, + "exit_process": lambda _code: None, + "log": lambda _message: None, + } + ports.update(overrides) + return WatchdogPorts(**ports) # type: ignore[arg-type] + + +class WatchdogServiceTests(unittest.TestCase): + def test_publication_failure_exits_after_one_bounded_wait(self) -> None: + events: list[object] = [] + + def hard_exit(reason: str) -> None: + events.append(("hard_exit", reason)) + raise SystemExit(1) + + service = WatchdogService( + make_watchdog_ports( + wait_for_stop=lambda timeout: events.append(("wait", timeout)) + or False, + interval_seconds=lambda: 2.0, + publication_state=lambda _now: ("publication", 0.0, 900.0, 30.0), + hard_exit=hard_exit, + ) + ) + + with self.assertRaises(SystemExit): + service.run() + + self.assertEqual(events[0], ("wait", 2.0)) + self.assertEqual(events[1], ("hard_exit", "publication")) + self.assertIn( + "publication-progress watchdog firing", + str(service.failure_detail), + ) + + def test_coordination_classification_precedes_publication(self) -> None: + events: list[object] = [] + + def hard_exit(reason: str) -> None: + events.append(("hard_exit", reason)) + raise SystemExit(1) + + service = WatchdogService( + make_watchdog_ports( + wait_for_stop=lambda _timeout: False, + publication_state=lambda _now: ( + "coordination", + 12.5, + 10.0, + 30.0, + ), + hard_exit=hard_exit, + ) + ) + + with self.assertRaises(SystemExit): + service.run() + + self.assertEqual(events, [("hard_exit", "coordination")]) + self.assertIn("coordination-blocked", str(service.failure_detail)) + self.assertIn("streak_age=12.500s", str(service.failure_detail)) + + def test_requested_stop_does_not_sample_or_exit(self) -> None: + service = WatchdogService( + make_watchdog_ports( + wait_for_stop=lambda _timeout: True, + fatal_exit_requested=lambda: False, + publication_state=lambda _now: self.fail( + "stopped watchdog sampled publication state" + ), + hard_exit=lambda _reason: self.fail("stopped watchdog exited"), + exit_process=lambda _code: self.fail("stopped watchdog exited"), + log=lambda _message: self.fail("stopped watchdog logged"), + ) + ) + + service.run() + + def test_fatal_stop_request_exits_nonzero(self) -> None: + events: list[object] = [] + service = WatchdogService( + make_watchdog_ports( + wait_for_stop=lambda _timeout: True, + fatal_exit_requested=lambda: True, + exit_process=lambda code: events.append(("exit", code)), + log=lambda message: events.append(("log", message)), + ) + ) + + service.run() + + self.assertEqual(events[0][0], "log") + self.assertIn("fatal block-work restart requested", str(events[0][1])) + self.assertEqual(events[1], ("exit", 1)) + + def test_claimed_coordination_exit_is_terminal(self) -> None: + service = WatchdogService( + make_watchdog_ports(coordination_budget_seconds=lambda: 5.0) + ) + service.record_coordination_blocked_refresh(100.0) + state = service.publication_watchdog_state(106.0) + self.assertEqual(state[0], "coordination") + # A refresh completing after the claim cannot cancel the exit. + service.clear_coordination_blocked_streak() + self.assertGreater( + service.coordination_blocked_streak_age_seconds(107.0), + 0.0, + ) + + def test_publication_recheck_uses_divergence_snapshot(self) -> None: + # The cheap preflight fired, but the locked divergence recheck shows + # no divergence old enough: no exit may be claimed. + service = WatchdogService( + make_watchdog_ports( + publication_failure_expired=lambda _now: True, + publication_divergence_since=lambda: None, + ) + ) + state = service.publication_watchdog_state(100.0) + self.assertIsNone(state[0]) + divergent = WatchdogService( + make_watchdog_ports( + publication_budget_seconds=lambda: 30.0, + publication_failure_expired=lambda _now: True, + publication_divergence_since=lambda: 50.0, + ) + ) + self.assertEqual(divergent.publication_watchdog_state(100.0)[0], "publication") + + def test_hard_exit_bounds_release_and_exits_unconditionally(self) -> None: + events: list[object] = [] + controller = SimpleNamespace( + begin_shutdown=lambda label: events.append(("begin", label)) or True, + wait_for_writer_quiescence=lambda budget: (True, 0.0, []), + wait_for_lease_handling=lambda: True, + ) + service = WatchdogService( + make_watchdog_ports( + shutdown_controller=lambda: controller, + request_shutdown=lambda: events.append(("request", None)), + release_ledger_lease=lambda deadline: events.append( + ("release", True) + ) + or True, + exit_process=lambda code: events.append(("exit", code)), + log=lambda message: events.append(("log", message)), + lease_release_timeout_seconds=lambda: 0.2, + ) + ) + + service.hard_exit("liveness") + + self.assertEqual(events[0], ("request", None)) + self.assertEqual(events[1], ("begin", "watchdog_liveness")) + self.assertEqual(events[2], ("release", True)) + self.assertEqual(events[3][0], "log") + self.assertIn("lease_handled=True", str(events[3][1])) + self.assertEqual(events[-1], ("exit", 1)) + + def test_hard_exit_withholds_release_when_quiescence_fails(self) -> None: + events: list[object] = [] + controller = SimpleNamespace( + begin_shutdown=lambda _label: True, + wait_for_writer_quiescence=lambda _budget: (False, 0.05, ["writer"]), + ) + service = WatchdogService( + make_watchdog_ports( + shutdown_controller=lambda: controller, + release_ledger_lease=lambda _deadline: events.append( + ("release", None) + ) + or True, + exit_process=lambda code: events.append(("exit", code)), + log=lambda message: events.append(("log", message)), + ) + ) + + service.hard_exit("publication") + + self.assertNotIn(("release", None), events) + self.assertIn("lease_handled=False", str(events[0][1])) + self.assertEqual(events[-1], ("exit", 1)) + + def test_lease_heartbeat_diagnostic_uses_lease_failure_reason(self) -> None: + events: list[object] = [] + controller = SimpleNamespace( + begin_shutdown=lambda _label: True, + wait_for_writer_quiescence=lambda _budget: (True, 0.0, []), + ) + service = WatchdogService( + make_watchdog_ports( + shutdown_controller=lambda: controller, + lease_failure_reason=lambda: "lease heartbeat stopped", + exit_process=lambda code: events.append(("exit", code)), + log=lambda message: events.append(("log", message)), + ) + ) + + service.hard_exit("lease_heartbeat") + + self.assertIn("lease heartbeat stopped", str(events[0][1])) + self.assertEqual(events[-1], ("exit", 1)) + + +class DormantThread: + def __init__( + self, + *, + target: object, + name: str, + daemon: bool, + ) -> None: + self.target = target + self.name = name + self.daemon = daemon + self.start_count = 0 + self.join_timeouts: list[float | None] = [] + + def start(self) -> None: + self.start_count += 1 + + def join(self, timeout: float | None = None) -> None: + self.join_timeouts.append(timeout) + + +class FlakyThreadFactory: + def __init__(self) -> None: + self.attempts = 0 + self.threads: list[DormantThread] = [] + + def __call__(self, **kwargs: object) -> DormantThread: + factory = self + + class FlakyDormantThread(DormantThread): + def start(self) -> None: + factory.attempts += 1 + if factory.attempts == 1: + raise RuntimeError("thread start failed") + super().start() + + thread = FlakyDormantThread(**kwargs) # type: ignore[arg-type] + self.threads.append(thread) + return thread + + +class ContentionObservedLock: + """Lock that exposes an acquire attempt made while another caller owns it.""" + + def __init__(self) -> None: + self._lock = threading.Lock() + self.contended = threading.Event() + + def __enter__(self) -> "ContentionObservedLock": + if self._lock.locked(): + self.contended.set() + self._lock.acquire() + return self + + def __exit__(self, *_args: object) -> None: + self._lock.release() + + +def specification( + name: str, + *, + join_timeout: float = 1.0, + watchdog_monitored: bool = False, +) -> BackgroundServiceSpec: + return BackgroundServiceSpec( + name=name, + thread_name=f"prism-{name}", + target=lambda: None, + daemon=True, + join_timeout=join_timeout, + watchdog_monitored=watchdog_monitored, + ) + + +class BackgroundServiceRegistryTests(unittest.TestCase): + def test_named_start_is_idempotent_and_retains_exact_thread(self) -> None: + registry = BackgroundServiceRegistry( + [specification("poll", watchdog_monitored=True)], + thread_factory=DormantThread, # type: ignore[arg-type] + ) + + first = registry.start("poll") + second = registry.start("poll") + snapshot = registry.snapshot("poll") + + self.assertIs(first, second) + self.assertIs(snapshot.thread, first) + self.assertTrue(snapshot.started) + self.assertEqual(first.start_count, 1) # type: ignore[attr-defined] + self.assertEqual(first.name, "prism-poll") + self.assertTrue(first.daemon) + + def test_drain_threads_are_only_started_services_in_registration_order(self) -> None: + registry = BackgroundServiceRegistry( + [ + specification("poll", join_timeout=1.0), + specification("writer", join_timeout=5.0), + specification("optional", join_timeout=2.0), + ], + thread_factory=DormantThread, # type: ignore[arg-type] + ) + poll = registry.start("poll") + writer = registry.start("writer") + + self.assertEqual( + registry.threads_to_drain(), + ((poll, 1.0), (writer, 5.0)), + ) + + def test_watchdog_names_derive_from_the_same_service_records(self) -> None: + registry = BackgroundServiceRegistry( + [ + specification("poll", watchdog_monitored=True), + specification("health"), + specification("writer", watchdog_monitored=True), + ], + thread_factory=DormantThread, # type: ignore[arg-type] + ) + + registry.start("writer") + + self.assertEqual(registry.watchdog_service_names(), ("poll", "writer")) + self.assertEqual( + registry.watchdog_service_names(started_only=True), + ("writer",), + ) + + def test_duplicate_service_or_thread_names_are_rejected(self) -> None: + registry = BackgroundServiceRegistry([specification("poll")]) + + with self.assertRaisesRegex(ValueError, "already registered: poll"): + registry.register(specification("poll")) + with self.assertRaisesRegex(ValueError, "thread name is already registered"): + registry.register( + BackgroundServiceSpec( + name="other", + thread_name="prism-poll", + target=lambda: None, + daemon=True, + join_timeout=1.0, + watchdog_monitored=False, + ) + ) + + def test_start_failure_rolls_back_and_retry_runs_start_hook_once(self) -> None: + factory = FlakyThreadFactory() + registry = BackgroundServiceRegistry( + [specification("poll", watchdog_monitored=True)], + thread_factory=factory, + ) + started: list[str] = [] + + with self.assertRaisesRegex(RuntimeError, "thread start failed"): + registry.start( + "poll", + on_started=lambda service: started.append(service.name), + ) + + failed = registry.snapshot("poll") + self.assertFalse(failed.started) + self.assertIsNone(failed.thread) + self.assertEqual(started, []) + + thread = registry.start( + "poll", + on_started=lambda service: started.append(service.name), + ) + + self.assertEqual(factory.attempts, 2) + self.assertIs(thread, factory.threads[1]) + self.assertEqual(started, ["poll"]) + + def test_started_thread_retries_a_failed_start_hook_without_restarting(self) -> None: + registry = BackgroundServiceRegistry( + [specification("poll", watchdog_monitored=True)], + thread_factory=DormantThread, # type: ignore[arg-type] + ) + hook_attempts = 0 + + def flaky_hook(_service: BackgroundServiceSpec) -> None: + nonlocal hook_attempts + hook_attempts += 1 + if hook_attempts == 1: + raise RuntimeError("start hook failed") + + with self.assertRaisesRegex(RuntimeError, "start hook failed"): + registry.start("poll", on_started=flaky_hook) + + started = registry.snapshot("poll") + self.assertTrue(started.started) + self.assertIsNotNone(started.thread) + + retried = registry.start("poll", on_started=flaky_hook) + + self.assertIs(retried, started.thread) + self.assertEqual(retried.start_count, 1) # type: ignore[attr-defined] + self.assertEqual(hook_attempts, 2) + + def test_dynamic_registration_is_equivalent_or_fails_clearly(self) -> None: + def target() -> None: + return None + + registered = specification("dynamic") + registered = BackgroundServiceSpec( + name=registered.name, + thread_name=registered.thread_name, + target=target, + daemon=registered.daemon, + join_timeout=registered.join_timeout, + watchdog_monitored=registered.watchdog_monitored, + registration_identity=("dynamic", 1), + ) + registry = BackgroundServiceRegistry() + + self.assertTrue(registry.register_if_absent(registered)) + self.assertFalse( + registry.register_if_absent( + BackgroundServiceSpec( + name="dynamic", + thread_name="prism-dynamic", + target=lambda: None, + daemon=True, + join_timeout=1.0, + watchdog_monitored=False, + registration_identity=("dynamic", 1), + ) + ) + ) + with self.assertRaisesRegex(ValueError, "incompatible.*dynamic"): + registry.register_if_absent( + BackgroundServiceSpec( + name="dynamic", + thread_name="prism-dynamic", + target=target, + daemon=True, + join_timeout=1.0, + watchdog_monitored=False, + registration_identity=("dynamic", 2), + ) + ) + + def test_post_start_registration_keeps_registration_order_for_drain(self) -> None: + registry = BackgroundServiceRegistry( + [specification("first")], + thread_factory=DormantThread, # type: ignore[arg-type] + ) + first = registry.start("first") + registry.register(specification("second", join_timeout=2.0)) + registry.register(specification("third", join_timeout=3.0)) + third = registry.start("third") + second = registry.start("second") + + self.assertEqual( + registry.threads_to_drain(), + ((first, 1.0), (second, 2.0), (third, 3.0)), + ) + + +class CoordinatorBackgroundServiceIntegrationTests(unittest.TestCase): + @staticmethod + def coordinator_with_optional_services(enabled: bool) -> PrismCoordinator: + server = PrismCoordinator.__new__(PrismCoordinator) + server.blockwait_enabled = enabled + server.vardiff_idle_sweep_seconds = 1.0 if enabled else 0.0 + server.stratum_initial_job_timeout_seconds = 1.0 if enabled else 0.0 + server.ctv_broadcaster_enabled = enabled + server.watchdog_enabled = enabled + server.audit_bind = "127.0.0.1" if enabled else None + server.audit_port = 8080 if enabled else 0 + return server + + def test_every_process_service_is_registered_unconditionally(self) -> None: + # The registry declares every spec regardless of the live enable + # flags; serve() applies the conditional at each named start. A + # disabled coordinator therefore composes the identical registry. + disabled = self.coordinator_with_optional_services(False) + enabled = self.coordinator_with_optional_services(True) + + self.assertEqual( + disabled._make_background_service_registry().service_names(), + enabled._make_background_service_registry().service_names(), + ) + + def test_audit_http_starts_only_after_both_snapshot_refreshers(self) -> None: + # Upstream #120 ordering: both cached-snapshot refreshers are armed + # before the listener facade binds; the warm-up itself runs inside + # their background loops so the bind path never blocks on it. + server = PrismCoordinator.__new__(PrismCoordinator) + server.audit_bind = "127.0.0.1" + server.audit_port = 3341 + events: list[str] = [] + server.start_health_snapshot_refresher = lambda: events.append( # type: ignore[method-assign] + "health" + ) + server.start_metrics_snapshot_refresher = lambda: events.append( # type: ignore[method-assign] + "metrics" + ) + server._ensure_audit_http_facade = lambda: SimpleNamespace( # type: ignore[method-assign] + start=lambda: events.append("http") + ) + + with patch("builtins.print"): + server.start_audit_server() + + self.assertEqual(events, ["health", "metrics", "http"]) + + def test_process_service_specs_preserve_names_and_join_order(self) -> None: + server = self.coordinator_with_optional_services(True) + + registry = server._make_background_service_registry() + + # The watchdog spec is deliberately absent from the composed + # registry: serve() registers it inline at its start boundary so the + # publication-progress watchdog exists before synchronous startup + # prewarm/recovery work. + self.assertEqual( + registry.service_names(), + ( + "qbit_blockpoll", + "block_submitter", + "qbit_blockwait", + "vardiff_idle_sweep", + "initial_job_timeout_sweep", + "share_writer", + "ctv_fanout_broadcaster", + "health_snapshot_refresher", + "metrics_snapshot_refresher", + ), + ) + expected = { + "qbit_blockpoll": ("prism-qbit-block-poll", 1.0, True), + "block_submitter": ("prism-block-submitter", 1.0, True), + "qbit_blockwait": ("prism-qbit-block-wait", 1.0, True), + "vardiff_idle_sweep": ("prism-vardiff-idle-sweep", 1.0, True), + "initial_job_timeout_sweep": ("prism-initial-job-timeouts", 1.0, False), + "share_writer": ("prism-share-writer", 5.0, True), + "ctv_fanout_broadcaster": ( + "prism-ctv-fanout-broadcaster", + 1.0, + True, + ), + "health_snapshot_refresher": ( + "prism-health-snapshot-refresher", + 1.0, + False, + ), + "metrics_snapshot_refresher": ( + "prism-metrics-snapshot-refresher", + 1.0, + False, + ), + } + for name, properties in expected.items(): + service = registry.snapshot(name).specification + self.assertEqual( + (service.thread_name, service.join_timeout, service.watchdog_monitored), + properties, + ) + + def test_monitored_service_start_seeds_its_own_watchdog_key(self) -> None: + server = self.coordinator_with_optional_services(False) + server._heartbeats = {} + server._watchdog_pauses = {} + server._heartbeats_lock = threading.Lock() + server._background_services = BackgroundServiceRegistry( + [specification("tracked", watchdog_monitored=True)], + thread_factory=DormantThread, # type: ignore[arg-type] + ) + + server._start_background_service("tracked") + + self.assertEqual(tuple(server._heartbeats), ("tracked",)) + server._heartbeats["tracked"] = 1.0 + server._start_background_service("tracked") + self.assertEqual(server._heartbeats["tracked"], 1.0) + + def test_concurrent_wrapper_starts_seed_one_heartbeat_and_one_thread(self) -> None: + server = self.coordinator_with_optional_services(False) + server._heartbeats = {} + server._watchdog_pauses = {} + server._heartbeats_lock = threading.Lock() + server._background_services = BackgroundServiceRegistry( + [specification("tracked", watchdog_monitored=True)], + thread_factory=DormantThread, # type: ignore[arg-type] + ) + observed_registry_lock = ContentionObservedLock() + server._background_services._lock = observed_registry_lock # type: ignore[assignment] + heartbeat_entered = threading.Event() + release_heartbeat = threading.Event() + heartbeat_names: list[str] = [] + results: list[object] = [] + errors: list[BaseException] = [] + + def blocked_heartbeat(name: str) -> None: + heartbeat_names.append(name) + heartbeat_entered.set() + if not release_heartbeat.wait(5): + raise AssertionError("heartbeat test interleaving timed out") + + server._record_heartbeat = blocked_heartbeat # type: ignore[method-assign] + + def start() -> None: + try: + results.append(server._start_background_service("tracked")) + except BaseException as exc: + errors.append(exc) + + first = threading.Thread(target=start) + second = threading.Thread(target=start) + first.start() + self.assertTrue(heartbeat_entered.wait(5)) + second.start() + second_contended_inside_start = observed_registry_lock.contended.wait(5) + release_heartbeat.set() + first.join(5) + second.join(5) + + self.assertFalse(first.is_alive()) + self.assertFalse(second.is_alive()) + self.assertTrue(second_contended_inside_start) + self.assertEqual(errors, []) + self.assertEqual(heartbeat_names, ["tracked"]) + self.assertEqual(len(results), 2) + self.assertIs(results[0], results[1]) + self.assertEqual(results[0].start_count, 1) # type: ignore[union-attr] + + def test_wrapper_start_failure_rolls_back_heartbeat_and_retries_cleanly(self) -> None: + server = self.coordinator_with_optional_services(False) + server._heartbeats = {} + server._watchdog_pauses = {} + server._heartbeats_lock = threading.Lock() + factory = FlakyThreadFactory() + server._background_services = BackgroundServiceRegistry( + [specification("tracked", watchdog_monitored=True)], + thread_factory=factory, + ) + + with self.assertRaisesRegex(RuntimeError, "thread start failed"): + server._start_background_service("tracked") + + self.assertEqual(server._heartbeats, {}) + self.assertFalse(server._background_services.snapshot("tracked").started) + + thread = server._start_background_service("tracked") + + self.assertIs(thread, factory.threads[1]) + self.assertEqual(tuple(server._heartbeats), ("tracked",)) + self.assertTrue(server._background_services.snapshot("tracked").started) + + def test_concurrent_secondary_starts_register_and_start_once(self) -> None: + server = self.coordinator_with_optional_services(False) + server._heartbeats = {} + server._watchdog_pauses = {} + server._heartbeats_lock = threading.Lock() + server._background_services = BackgroundServiceRegistry( + thread_factory=DormantThread, # type: ignore[arg-type] + ) + profile = SimpleNamespace( + heartbeat_name="stratum_accept_highdiff", + name="highdiff", + ) + listener = SimpleNamespace() + registration_barrier = threading.Barrier(2) + original_register = server._background_services.register_if_absent + + def synchronized_register(service: BackgroundServiceSpec) -> bool: + registration_barrier.wait(timeout=5) + return original_register(service) + + server._background_services.register_if_absent = synchronized_register # type: ignore[method-assign] + heartbeat_names: list[str] = [] + server._record_heartbeat = heartbeat_names.append # type: ignore[method-assign] + results: list[object] = [] + errors: list[BaseException] = [] + + def start() -> None: + try: + results.append( + server._start_secondary_accept_service( # type: ignore[arg-type] + listener, + profile, + ) + ) + except BaseException as exc: + errors.append(exc) + + callers = [threading.Thread(target=start) for _ in range(2)] + for caller in callers: + caller.start() + for caller in callers: + caller.join(5) + + self.assertTrue(all(not caller.is_alive() for caller in callers)) + self.assertEqual(errors, []) + self.assertEqual(heartbeat_names, ["stratum_accept_highdiff"]) + self.assertEqual(len(results), 2) + self.assertIs(results[0], results[1]) + self.assertEqual(results[0].start_count, 1) # type: ignore[union-attr] + self.assertEqual( + server._background_services.service_names(), + ("stratum_accept_highdiff",), + ) + + def test_secondary_listener_is_named_monitored_and_bounded_for_drain(self) -> None: + server = self.coordinator_with_optional_services(False) + server._heartbeats = {} + server._watchdog_pauses = {} + server._heartbeats_lock = threading.Lock() + server._background_services = BackgroundServiceRegistry( + thread_factory=DormantThread, # type: ignore[arg-type] + ) + profile = SimpleNamespace( + heartbeat_name="stratum_accept_highdiff", + name="highdiff", + ) + + thread = server._start_secondary_accept_service( # type: ignore[arg-type] + SimpleNamespace(), + profile, + ) + + snapshot = server._background_services.snapshot("stratum_accept_highdiff") + self.assertEqual(thread.name, "prism-stratum-accept-highdiff") + self.assertTrue(snapshot.specification.watchdog_monitored) + self.assertEqual(snapshot.specification.join_timeout, 1.0) + self.assertEqual(tuple(server._heartbeats), ("stratum_accept_highdiff",)) + self.assertEqual( + server._background_services.threads_to_drain(), + ((thread, 1.0),), + ) + + def test_health_refresher_start_marks_latch_before_thread_dispatch(self) -> None: + # Upstream #120 contract: the running latch is set before the + # registry can start the refresher thread, and it is a one-shot + # latch (the loop never clears it), so a second start call returns + # without touching the registry again. + server = self.coordinator_with_optional_services(False) + server.stop_event = threading.Event() + server.stop_event.set() + latch_at_dispatch: list[bool] = [] + + class LatchObservingThread(DormantThread): + def start(self) -> None: + latch_at_dispatch.append( + server._ensure_observability_service() + .state() + .health_refresh_loop_running + ) + super().start() + + server._background_services = BackgroundServiceRegistry( + [specification("health_snapshot_refresher")], + thread_factory=LatchObservingThread, # type: ignore[arg-type] + ) + + server.start_health_snapshot_refresher() + + started = server._background_services.snapshot("health_snapshot_refresher") + self.assertEqual(latch_at_dispatch, [True]) + self.assertTrue(started.started) + self.assertTrue( + server._ensure_observability_service() + .state() + .health_refresh_loop_running + ) + + server.start_health_snapshot_refresher() + + self.assertEqual(started.thread.start_count, 1) # type: ignore[union-attr] + self.assertTrue( + server._ensure_observability_service() + .state() + .health_refresh_loop_running + ) + + +class ScriptedStopEvent: + """Stop signal whose transitions the test scripts explicitly.""" + + def __init__(self) -> None: + self.stopped = False + self.waits: list[float] = [] + + def is_set(self) -> bool: + return self.stopped + + def wait(self, timeout: float) -> bool: + self.waits.append(timeout) + return self.stopped + + +class ScriptedRollupLedger: + """Rollup-capable ledger stub yielding one scripted result per pass.""" + + def __init__(self, results: list[object], *, on_exhausted=None) -> None: + self.results = list(results) + self.calls: list[int] = [] + self.on_exhausted = on_exhausted + + def advance_hashrate_rollups(self, *, batch_limit: int) -> dict[str, object]: + self.calls.append(batch_limit) + result = self.results.pop(0) + if not self.results and self.on_exhausted is not None: + self.on_exhausted() + if isinstance(result, BaseException): + raise result + return result # type: ignore[return-value] + + +class HashrateRollupMaintenanceServiceTests(unittest.TestCase): + @staticmethod + def rollup_coordinator(ledger: object) -> PrismCoordinator: + server = PrismCoordinator.__new__(PrismCoordinator) + server.blockwait_enabled = False + server.vardiff_idle_sweep_seconds = 0.0 + server.stratum_initial_job_timeout_seconds = 0.0 + server.ctv_broadcaster_enabled = False + server.watchdog_enabled = False + server.audit_bind = None + server.audit_port = 0 + server.ledger = ledger + server.hashrate_rollup_enabled = True + server.hashrate_rollup_interval_seconds = 15.0 + server.hashrate_rollup_batch_shares = 50000 + return server + + def test_rollup_service_registers_only_for_rollup_capable_ledgers(self) -> None: + # The capability conditional is the one intentional exception to the + # registered-unconditionally rule: a memory ledger never grows + # advance_hashrate_rollups at runtime, so the service either exists + # for the life of the process or never will. + capable = self.rollup_coordinator(ScriptedRollupLedger([])) + registry = capable._make_background_service_registry() + self.assertIn("hashrate-rollup-maintenance", registry.service_names()) + service = registry.snapshot("hashrate-rollup-maintenance").specification + self.assertEqual( + (service.thread_name, service.join_timeout, service.watchdog_monitored), + ("prism-hashrate-rollup-maintenance", 1.0, False), + ) + + memory_backed = self.rollup_coordinator(SimpleNamespace()) + self.assertNotIn( + "hashrate-rollup-maintenance", + memory_backed._make_background_service_registry().service_names(), + ) + + def test_rollup_start_honours_enable_flag_and_ledger_capability(self) -> None: + started = self.rollup_coordinator(ScriptedRollupLedger([])) + started._heartbeats = {} + started._watchdog_pauses = {} + started._heartbeats_lock = threading.Lock() + started._background_services = BackgroundServiceRegistry( + [specification("hashrate-rollup-maintenance")], + thread_factory=DormantThread, # type: ignore[arg-type] + ) + with patch("builtins.print"): + started._start_hashrate_rollup_maintenance_if_enabled() + self.assertTrue( + started._background_services.snapshot("hashrate-rollup-maintenance").started + ) + + disabled = self.rollup_coordinator(ScriptedRollupLedger([])) + disabled.hashrate_rollup_enabled = False + disabled._background_services = BackgroundServiceRegistry( + [specification("hashrate-rollup-maintenance")], + thread_factory=DormantThread, # type: ignore[arg-type] + ) + disabled._start_hashrate_rollup_maintenance_if_enabled() + self.assertFalse( + disabled._background_services.snapshot("hashrate-rollup-maintenance").started + ) + + # A memory-backed coordinator has no registered service to start, so + # the guard must return before asking the registry for the name. + memory_backed = self.rollup_coordinator(SimpleNamespace()) + memory_backed._background_services = BackgroundServiceRegistry([]) + memory_backed._start_hashrate_rollup_maintenance_if_enabled() + + def test_rollup_loop_drains_backlog_then_sleeps_the_interval(self) -> None: + # Catch-up looping is the backfill mechanism: while the ledger + # reports more history behind the watermark the loop advances again + # immediately, and only a caught-up pass reaches the interval wait. + stop = ScriptedStopEvent() + ledger = ScriptedRollupLedger( + [ + {"scanned": 3, "last_share_seq": 3, "caught_up": False}, + {"scanned": 3, "last_share_seq": 6, "caught_up": False}, + {"scanned": 2, "last_share_seq": 8, "caught_up": True}, + ], + on_exhausted=lambda: setattr(stop, "stopped", True), + ) + server = self.rollup_coordinator(ledger) + server.stop_event = stop # type: ignore[assignment] + server.hashrate_rollup_batch_shares = 3 + + with patch("builtins.print"): + server.hashrate_rollup_maintenance_loop() + + self.assertEqual(ledger.calls, [3, 3, 3]) + self.assertEqual(stop.waits, [15.0]) + + def test_rollup_passes_run_inside_the_writer_admission(self) -> None: + """Each pass is a writer operation the shutdown controller counts. + + The maintenance statement renews the writer lease on exact identity, + which an orderly release keeps -- so a pass outside the admission + could slip past the stop check, renew the just-released lease, and + mutate rollups after shutdown reported the writer quiesced. Running + the ledger call inside _writer_operation makes quiescence wait for + an in-flight pass. + """ + stop = ScriptedStopEvent() + server = self.rollup_coordinator(None) + server.stop_event = stop # type: ignore[assignment] + controller = server._ensure_shutdown_controller() + observed: list[int] = [] + + class AdmissionObservingLedger(ScriptedRollupLedger): + def advance_hashrate_rollups(self, *, batch_limit: int) -> dict[str, object]: + observed.append(controller.active_writers.get("hashrate_rollup_maintenance", 0)) + return super().advance_hashrate_rollups(batch_limit=batch_limit) + + server.ledger = AdmissionObservingLedger( + [{"scanned": 0, "last_share_seq": 8, "caught_up": True}], + on_exhausted=lambda: setattr(stop, "stopped", True), + ) + + with patch("builtins.print"): + server.hashrate_rollup_maintenance_loop() + + self.assertEqual([1], observed) + self.assertEqual( + 0, controller.active_writers.get("hashrate_rollup_maintenance", 0) + ) + + def test_rollup_loop_stops_cleanly_when_admission_is_closed(self) -> None: + """A refused admission is the orderly shutdown, not a failure. + + The race this pins: shutdown begins after the loop's stop check but + before the pass enters the admission. The pass must be refused, and + the loop must exit without logging an error or running the ledger. + """ + stop = ScriptedStopEvent() + ledger = ScriptedRollupLedger([]) + server = self.rollup_coordinator(ledger) + server.stop_event = stop # type: ignore[assignment] + server._ensure_shutdown_controller().request_shutdown(None) + + with patch("builtins.print"), patch("traceback.print_exc") as print_exc: + server.hashrate_rollup_maintenance_loop() + + self.assertEqual([], ledger.calls) + self.assertEqual([], stop.waits) + self.assertEqual(0, print_exc.call_count) + + def test_rollup_loop_logs_the_error_and_retries_next_interval(self) -> None: + stop = ScriptedStopEvent() + ledger = ScriptedRollupLedger( + [ + RuntimeError("rollup pass failed"), + {"scanned": 0, "last_share_seq": 8, "caught_up": True}, + ], + on_exhausted=lambda: setattr(stop, "stopped", True), + ) + server = self.rollup_coordinator(ledger) + server.stop_event = stop # type: ignore[assignment] + + with patch("builtins.print"), patch("traceback.print_exc") as print_exc: + server.hashrate_rollup_maintenance_loop() + + # The failed pass reaches the interval wait instead of hot-looping, + # then the next pass succeeds and the scripted stop ends the loop. + self.assertEqual(ledger.calls, [50000, 50000]) + self.assertEqual(stop.waits, [15.0, 15.0]) + self.assertEqual(print_exc.call_count, 1) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_prism_block_candidate_collapse.py b/tests/test_prism_block_candidate_collapse.py new file mode 100644 index 000000000..0a9352fdf --- /dev/null +++ b/tests/test_prism_block_candidate_collapse.py @@ -0,0 +1,4851 @@ +#!/usr/bin/env python3 +"""Decided-height block-candidate collapse (issue #183). + +The 2026-08-20 testnet4 incident left 3,120 durable block candidates behind +one decided height. The shipped per-row disposition path terminalizes them +correctly but at one node offer, one disposition lease, and one durable +write each; ``tests/prism_candidate_storm.py`` measures that cost and, in +``drain_per_row``, is the oracle every set-oriented selector has to agree +with. + +These tests own the selector and its apply: predicate S clause by clause, +its fail-closed behaviour, the bounded chain reads, the non-blocking lease, +the immediate pre-write revalidation, the exact-returned-set cleanup and its +failure modes, the page partition both replay enumeration shapes must +perform, the bounded logs and fixed-label metrics, and the storm comparisons +against the shipped per-row oracle at 100 and 3,120 candidates in both the +live and the restart view. + +The safety property under test is one-directional and absolute: for every +comparison ``S - oracle_abandoned`` must be empty. A conservative +``oracle_abandoned - S`` delta is allowed and reported; an S-only hash is a +bug that would destroy a block the per-row path deliberately kept. +""" + +from __future__ import annotations + +import queue +import sys +import threading +import time +import unittest +from contextlib import redirect_stdout +from io import StringIO +from pathlib import Path +from types import SimpleNamespace +from typing import Any, Iterable + +if not __package__: + sys.path.insert(0, str(Path(__file__).resolve().parents[1])) + +from lab.prism.block_candidates import ( # noqa: E402 + BLOCK_CANDIDATE_COLLAPSE_CLEANUP_FLOOR_STEP, + BLOCK_CANDIDATE_COLLAPSE_CLEANUP_PAYOUT_STEP, + BLOCK_CANDIDATE_COLLAPSE_CLEANUP_STEPS, + BLOCK_CANDIDATE_COLLAPSE_LOG_FAILURES, + BLOCK_CANDIDATE_COLLAPSE_LOG_GROUPS, + BLOCK_CANDIDATE_COLLAPSE_LOG_SAMPLE_HASHES, + BLOCK_CANDIDATE_TERMINAL_OUTCOME_EVICTION_SCAN, + COLLAPSE_DIFFICULTY_SCALE, + COLLAPSE_POW_LIMIT_BITS, + DEFAULT_BLOCK_ACCOUNTING_CLEANUP_RETRY_WORK_ITEMS, + MAX_BLOCK_CANDIDATE_COLLAPSE_HEIGHT_PROBES, + MAX_BLOCK_CANDIDATE_TERMINAL_OUTCOMES, + MAX_BLOCK_REPLAY_ENUMERATION_ROWS, + MAX_PENDING_BLOCK_CANDIDATES, + PRISM_BLOCK_CANDIDATE_COLLAPSE_OUTCOMES, + PRISM_BLOCK_CANDIDATE_COLLAPSE_REASON, + PRISM_BLOCK_CANDIDATE_COLLAPSE_STALE_JOB_CLASS, + PRISM_STALE_JOB_ABANDON_CLASSES, + _BlockCandidateAccountingTask, + _BlockCandidateChainView, + _BlockCandidateNodeSubmission, + _collapse_scaled_difficulty, +) +from lab.prism.metrics import MetricsRenderer # noqa: E402 +from lab.prism.share_ledger import ( # noqa: E402 + PendingShare, + SingleWriterShareLedger, +) +from lab.prism.template_artifacts import scaled_network_difficulty # noqa: E402 +from lab.prism.share_submission import PRISM_REJECTION_STALE_JOB # noqa: E402 +from tests.prism_candidate_storm import ( # noqa: E402 + PERTURBATIONS, + STORM_PARENT_HASH, + CandidateStormRig, +) +from tests.prism_vardiff_test_support import ( # noqa: E402 + FakeRpc, + block_candidate, + submit_coordinator, +) + + +STORM_PARENT = STORM_PARENT_HASH +OTHER_PARENT = "22" * 32 +DECIDED_HEIGHT = 10 +DECIDED_WINNER = "dd" * 32 +# A candidate building on the current best tip: the block waiting to be +# offered, which predicate S must never select and whose submission must not +# sit behind a long enumeration walk. +LIVE_SOLVE_HASH = f"{0xFEED0001:064x}" +# Compact header bits, the only work fact these tests state. A candidate's +# durable ``found_block.network_difficulty`` is +# ``scaled_network_difficulty(template["bits"])`` -- the raw difficulty times +# COLLAPSE_DIFFICULTY_SCALE -- so a chain-side work value is only comparable +# with it once it is re-derived from the occupant's own bits. Spelling the +# fixture in bits is what makes "equal work" mean the same integer on both +# sides instead of two numbers a million apart. +MIN_WORK_BITS = COLLAPSE_POW_LIMIT_BITS +# A retargeted header: raw difficulty ~2.1e9, scaled ~2.1e15. +RETARGETED_BITS = "1d00ffff" +# Scaled work above 2**53, where widening the stored integer to float rounds +# it *up* and an equal-work occupant would read as strictly weaker. +HIGH_WORK_BITS = "1b0404cb" + + +def _hash(index: int) -> str: + return f"{index:064x}" + + +def _scaled(bits: str) -> int: + """The PRISM-scaled difficulty a header with ``bits`` carries.""" + return scaled_network_difficulty(bits) + + +def _raw(bits: str) -> float: + """The raw ``getblockheader.difficulty`` float a node reports for ``bits``. + + A million times smaller than the scaled units every candidate row is + stamped in; the fixture reports it alongside ``bits`` exactly so a + selector that read it again would fail the equal-work tests. + """ + return _scaled(bits) / COLLAPSE_DIFFICULTY_SCALE + + +class CollapseChainRpc(FakeRpc): + """A controllable decided-height chain view with per-method call counts. + + Every read predicate S makes is separately overridable -- absent, raising, + or returning an unusable value -- so each fail-closed branch can be driven + without reaching for a mock framework. + """ + + def __init__( + self, + *, + tip: str = DECIDED_WINNER, + tip_height: int = DECIDED_HEIGHT, + active: dict[int, str] | None = None, + bits: dict[str, str] | None = None, + ) -> None: + self.tip = tip.lower() + self.tip_height = int(tip_height) + self.active = { + int(height): str(value).lower() + for height, value in (active or {DECIDED_HEIGHT: DECIDED_WINNER}).items() + } + # Per-block compact bits; every unlisted block sits at the qbit + # powLimit, the weakest header the chain can carry. + self.bits = { + str(key).lower(): str(value).lower() + for key, value in (bits or {}).items() + } + self.default_bits = MIN_WORK_BITS + self.failures: set[str] = set() + self.results: dict[str, object] = {} + self.calls: list[tuple[str, tuple[object, ...]]] = [] + self.before_call: Any | None = None + + def call(self, method: str, params: list[object] | None = None) -> object: + self.calls.append((method, tuple(params or ()))) + if self.before_call is not None: + self.before_call(self, method, params) + if method in self.failures: + raise RuntimeError(f"qbit RPC {method} failed") + if method in self.results: + return self.results[method] + if method == "getbestblockhash": + return self.tip + if method == "getblockcount": + return self.tip_height + if method == "getblockhash": + height = int((params or [0])[0]) + if height in self.active: + return self.active[height] + raise RuntimeError( + f"qbit RPC getblockhash failed: -8 unknown height {height}" + ) + if method == "getblockheader": + block_hash = str((params or [""])[0]).lower() + if block_hash not in set(self.active.values()) | {self.tip}: + raise RuntimeError( + "qbit RPC getblockheader failed: -5 Block not found" + ) + bits = self.bits.get(block_hash, self.default_bits) + return { + "hash": block_hash, + "height": self.tip_height, + "confirmations": 1, + "bits": bits, + # qbitd reports both; ``difficulty`` is the raw float, a + # million times smaller than the units the candidate rows + # carry, and the selector must not read it. + "difficulty": _raw(bits), + } + return super().call(method, params) + + def counts(self) -> dict[str, int]: + counts: dict[str, int] = {} + for method, _params in self.calls: + counts[method] = counts.get(method, 0) + 1 + return counts + + +class CollapseLedger: + """A real in-memory ledger with controllable page reads and batch writes.""" + + def __init__(self, inner: Any | None = None) -> None: + self.inner = inner if inner is not None else SingleWriterShareLedger() + self.page_hook: Any | None = None + self.abandon_hook: Any | None = None + self.abandon_calls: list[tuple[tuple[str, ...], str]] = [] + + def __getattr__(self, name: str) -> Any: + return getattr(self.inner, name) + + def pending_block_candidate_rows( + self, + *, + limit: int = 32, + after_cursor: object | None = None, + ) -> list[dict[str, Any]]: + rows = self.inner.pending_block_candidate_rows( + limit=limit, + after_cursor=after_cursor, + ) + if self.page_hook is not None: + rows = self.page_hook(rows) + return rows + + def mark_block_candidates_abandoned( + self, + *, + block_hashes: Any, + error: str, + ) -> tuple[str, ...]: + requested = tuple(str(value).lower() for value in block_hashes) + self.abandon_calls.append((requested, error)) + if self.abandon_hook is not None: + return self.abandon_hook(requested, error) + return self.inner.mark_block_candidates_abandoned( + block_hashes=requested, + error=error, + ) + + +class NoBatchCollapseLedger(CollapseLedger): + """A compatibility ledger with no fenced batch abandonment at all.""" + + def __getattr__(self, name: str) -> Any: + if name == "mark_block_candidates_abandoned": + raise AttributeError(name) + return super().__getattr__(name) + + mark_block_candidates_abandoned = None # type: ignore[assignment] + + +class WindowCollapseLedger(CollapseLedger): + """A ledger whose page reader cannot paginate, forcing the legacy window.""" + + def pending_block_candidate_rows( # type: ignore[override] + self, + *, + limit: int = 32, + ) -> list[dict[str, Any]]: + rows = self.inner.pending_block_candidate_rows(limit=limit) + if self.page_hook is not None: + rows = self.page_hook(rows) + return rows + + +class CollapseFixture: + """One coordinator, ledger, and chain view seeded with durable rows.""" + + def __init__( + self, + *, + ledger: CollapseLedger | None = None, + rpc: CollapseChainRpc | None = None, + ) -> None: + server, state, _recording = submit_coordinator(tip=STORM_PARENT) + server.max_blocks = 1 << 30 + server.stop_after_block = False + server.block_candidate_queue = queue.Queue( + maxsize=MAX_PENDING_BLOCK_CANDIDATES + ) + server.block_candidate_retry_initial_seconds = 0.0 + self.ledger = ledger if ledger is not None else CollapseLedger() + server.ledger = self.ledger + self.rpc = rpc if rpc is not None else CollapseChainRpc() + server.rpc = self.rpc + self.server = server + self.state = state + self.service = server._ensure_block_candidate_service() + self.service._ensure_block_replay_state() + self.service._ensure_block_candidate_disposition_state() + self._jobs: set[str] = set() + + # -- seeding ----------------------------------------------------------- + + def job( + self, + *, + parent: str = STORM_PARENT, + height: int = DECIDED_HEIGHT, + network_difficulty: int = 1, + ) -> str: + job_id = f"job-{parent}-{height}-{network_difficulty}" + if job_id in self._jobs: + return job_id + base = self.server.jobs["job-1"] + self.server.jobs[job_id] = SimpleNamespace( + job=SimpleNamespace( + job_id=job_id, + share_target=base.job.share_target, + share_difficulty=base.job.share_difficulty, + transaction_hexes=(), + ), + template={ + "previousblockhash": parent, + "height": int(height), + "coinbasevalue": 50_00000000, + }, + found_block={"network_difficulty": int(network_difficulty)}, + issued_at_ms=12345, + collection_only=False, + worker=base.worker, + shares_json=[], + prior_balances=[], + ) + self._jobs.add(job_id) + return job_id + + def seed( + self, + hashes: list[str], + *, + parent: str = STORM_PARENT, + height: int = DECIDED_HEIGHT, + network_difficulty: int = 1, + credit_share_on_accept: bool = False, + ) -> list[Any]: + job_id = self.job( + parent=parent, + height=height, + network_difficulty=network_difficulty, + ) + candidates = [] + entries = [] + for block_hash in hashes: + pending = PendingShare( + share_id=f"miner-a:{block_hash}", + miner_id="miner-a", + order_key="miner-a", + p2mr_program_hex="11" * 32, + share_difficulty=1, + network_difficulty=int(network_difficulty), + template_height=int(height) - 1, + job_id=job_id, + job_issued_at_ms=1, + accepted_at_ms=1, + ntime=1, + ) + candidate = block_candidate( + self.server, + self.state, + SimpleNamespace( + coinbase_tx_hex="00", + block_hash_hex=block_hash, + block_hex="00", + share_pass=True, + block_pass=True, + ), + job_id=job_id, + pending_share=pending, + credit_share_on_accept=credit_share_on_accept, + ) + candidates.append(candidate) + entries.append((pending, self.server.block_candidate_intent(candidate))) + self.ledger.inner.append_batch(entries) + return candidates + + # -- driving ----------------------------------------------------------- + + def page(self, limit: int = 512) -> list[dict[str, Any]]: + return self.ledger.pending_block_candidate_rows(limit=limit) + + def collapse( + self, + rows: list[Any] | None = None, + *, + call_class: str = "fast", + ) -> tuple[list[Any], str]: + """Run one apply over a page and return (retained rows, log text).""" + if rows is None: + rows = self.page() + buffer = StringIO() + with redirect_stdout(buffer): + retained = self.service._collapse_superseded_block_candidates( + rows, + timeout_seconds=None, + call_class=call_class, + ) + return retained, buffer.getvalue() + + def replay(self, *, enumeration_owed: bool = True) -> tuple[int, str]: + if enumeration_owed: + self.server._note_block_replay_enumeration_owed() + buffer = StringIO() + with redirect_stdout(buffer): + queued = self.server.replay_pending_block_candidates() + return queued, buffer.getvalue() + + def retry_cleanup(self, *, force_due: bool = False) -> tuple[bool, str]: + """Run one deferred cleanup retry pass on the accounting lane's runner. + + ``force_due`` clears the per-hash backoff deadline so a second and + later attempt can be driven without sleeping; a freshly deferred + hash is due immediately and needs it not at all. + """ + service = self.service + if force_due: + with self.server.lock: + for record in ( + service._block_candidate_collapse_cleanup_retries.values() + ): + record.not_before_monotonic = 0.0 + buffer = StringIO() + with redirect_stdout(buffer): + ran = service._run_one_collapsed_block_candidate_cleanup_retry() + return ran, buffer.getvalue() + + # -- observation ------------------------------------------------------- + + def cleanup_backlog(self) -> dict[str, frozenset[str]]: + return self.service.collapsed_candidate_cleanup_backlog() + + def pending(self) -> set[str]: + return { + str(row["block_hash"]).lower() + for row in self.ledger.inner.pending_block_candidate_rows(limit=1 << 16) + } + + def counts(self) -> dict[str, int]: + return self.service.block_candidate_collapse_snapshot() + + def floor(self) -> dict[int, Any]: + writer = self.server._ensure_share_writer_service() + return dict(writer._pending_share_commit_floor) + + +def _selected(fixture: CollapseFixture, rows: list[Any] | None = None) -> set[str]: + """The hashes one apply actually removed from the durable outbox.""" + before = fixture.pending() + fixture.collapse(rows) + return before - fixture.pending() + + +class BlockCandidateCollapsePredicateTests(unittest.TestCase): + """Predicate S, clause by clause, against the shipped durable page.""" + + def test_decided_height_sibling_is_selected(self) -> None: + fixture = CollapseFixture() + fixture.seed([_hash(1), _hash(2)]) + self.assertEqual(_selected(fixture), {_hash(1), _hash(2)}) + self.assertEqual(fixture.counts()["selected"], 2) + self.assertEqual(fixture.counts()["abandoned"], 2) + + def test_ownership_markers_alone_do_not_exclude(self) -> None: + """Outstanding/replay-inflight mean owned, not offered (#183 note).""" + fixture = CollapseFixture() + fixture.seed([_hash(1)]) + fixture.server._register_outstanding_block_candidate(_hash(1)) + with fixture.server.lock: + fixture.service._block_replay_inflight_hashes.add(_hash(1)) + self.assertEqual(_selected(fixture), {_hash(1)}) + + def test_attempt_count_alone_does_not_exclude(self) -> None: + fixture = CollapseFixture() + fixture.seed([_hash(1)]) + fixture.ledger.inner.mark_block_candidate_attempted(block_hash=_hash(1)) + row = fixture.ledger.inner._block_candidate_outbox[_hash(1)] + self.assertEqual(int(row["attempt_count"]), 1) + self.assertEqual(_selected(fixture), {_hash(1)}) + + def test_each_evidence_source_excludes_its_hash(self) -> None: + target = _hash(1) + other = _hash(2) + + def flight(fixture: CollapseFixture) -> None: + lease = fixture.server._claim_block_candidate_disposition( + target, + blocking=False, + ) + self.assertIsNotNone(lease) + self.addCleanup( + fixture.server._release_block_candidate_disposition, + lease, + ) + + def retry_candidate(fixture: CollapseFixture) -> None: + candidate = fixture.seed([target])[0] + with fixture.server.lock: + fixture.service.retry_candidate = candidate + + def waiting_retry(fixture: CollapseFixture) -> None: + candidate = fixture.seed([target])[0] + with fixture.server.lock: + fixture.service._block_disposition_waiting_retries[target] = candidate + + def finalize_retry(fixture: CollapseFixture) -> None: + with fixture.server.lock: + fixture.service.finalize_retries[target] = (False, "boom") + + def deferred_retry(fixture: CollapseFixture) -> None: + candidate = fixture.seed([target])[0] + with fixture.server.lock: + fixture.service._block_accounting_deferred_retry_candidate = candidate + + def retained_submission(fixture: CollapseFixture) -> None: + fixture.service._stash_retained_block_candidate_node_submission( + target, + _BlockCandidateNodeSubmission(attempted=True, result=None), + ) + + def tip_observed(fixture: CollapseFixture) -> None: + fixture.server._register_outstanding_block_candidate(target) + fixture.server._note_tip_observation_for_candidates(target) + + def accounted(fixture: CollapseFixture) -> None: + fixture.server._ensure_job_cache_state() + with fixture.server.lock: + fixture.server._accounted_accepted_block_hashes.add(target) + + def terminal(fixture: CollapseFixture) -> None: + fixture.server._record_block_candidate_terminal_outcome( + target, + accepted=False, + ) + + sources = { + "_block_candidate_disposition_flights": flight, + "retry_candidate": retry_candidate, + "_block_disposition_waiting_retries": waiting_retry, + "finalize_retries": finalize_retry, + "_block_accounting_deferred_retry_candidate": deferred_retry, + "_block_candidate_retained_node_submissions": retained_submission, + "_tip_observed_accepted_block_hashes": tip_observed, + "_accounted_accepted_block_hashes": accounted, + "_block_candidate_terminal_outcomes": terminal, + } + for name, install in sources.items(): + with self.subTest(evidence=name): + fixture = CollapseFixture() + fixture.seed([target, other]) + with redirect_stdout(StringIO()): + install(fixture) + self.assertEqual(_selected(fixture), {other}) + + def test_pool_block_row_excludes_its_hash(self) -> None: + fixture = CollapseFixture() + fixture.seed([_hash(1), _hash(2)]) + fixture.ledger.inner.persist_accepted_block( + block_hash=_hash(1), + block_height=DECIDED_HEIGHT, + parent_hash=STORM_PARENT, + final_bundle={}, + audit_report={}, + ) + self.assertEqual(_selected(fixture), {_hash(2)}) + + def test_missing_pool_block_fact_fails_the_whole_page_closed(self) -> None: + fixture = CollapseFixture() + fixture.seed([_hash(1), _hash(2)]) + fixture.ledger.page_hook = lambda rows: [ + {key: value for key, value in row.items() if key != "pool_block_exists"} + if index == 0 + else row + for index, row in enumerate(rows) + ] + rows = fixture.page() + retained, _log = fixture.collapse(rows) + self.assertEqual(fixture.pending(), {_hash(1), _hash(2)}) + self.assertEqual(len(retained), len(rows)) + self.assertEqual(fixture.counts()["fail_closed"], 2) + self.assertEqual(fixture.counts()["selected"], 0) + + def test_a_page_without_the_pool_block_fact_spends_no_chain_read(self) -> None: + """A compatibility page reader must not cost a tip read per poll.""" + fixture = CollapseFixture() + fixture.seed([_hash(1), _hash(2)]) + fixture.ledger.page_hook = lambda rows: [ + {"block_hash": row["block_hash"], "candidate": row["candidate"]} + for row in rows + ] + fixture.collapse() + self.assertEqual(fixture.rpc.calls, []) + self.assertEqual(fixture.pending(), {_hash(1), _hash(2)}) + self.assertEqual(fixture.counts()["fail_closed"], 2) + + def test_malformed_pool_block_fact_fails_the_whole_page_closed(self) -> None: + fixture = CollapseFixture() + fixture.seed([_hash(1), _hash(2)]) + fixture.ledger.page_hook = lambda rows: [ + {**row, "pool_block_exists": "no"} if index == 1 else row + for index, row in enumerate(rows) + ] + fixture.collapse() + self.assertEqual(fixture.pending(), {_hash(1), _hash(2)}) + self.assertEqual(fixture.counts()["fail_closed"], 2) + + def test_parent_equal_to_best_tip_excludes_its_hash(self) -> None: + fixture = CollapseFixture( + rpc=CollapseChainRpc( + tip=OTHER_PARENT, + tip_height=DECIDED_HEIGHT, + active={DECIDED_HEIGHT: DECIDED_WINNER}, + ) + ) + fixture.seed([_hash(1)], parent=OTHER_PARENT) + fixture.seed([_hash(2)], parent=STORM_PARENT) + self.assertEqual(_selected(fixture), {_hash(2)}) + + def test_height_above_the_tip_excludes_its_hash(self) -> None: + fixture = CollapseFixture( + rpc=CollapseChainRpc( + tip=DECIDED_WINNER, + tip_height=DECIDED_HEIGHT, + active={DECIDED_HEIGHT: DECIDED_WINNER}, + ) + ) + fixture.seed([_hash(1)], height=DECIDED_HEIGHT + 1) + fixture.seed([_hash(2)], height=DECIDED_HEIGHT) + self.assertEqual(_selected(fixture), {_hash(2)}) + # A height above the tip must not even be looked up: getblockhash + # would fail there and fail the whole page closed. + self.assertEqual( + [params for method, params in fixture.rpc.calls if method == "getblockhash"], + [(DECIDED_HEIGHT,), (DECIDED_HEIGHT,)], + ) + + def test_candidate_that_is_the_active_block_is_never_selected(self) -> None: + fixture = CollapseFixture( + rpc=CollapseChainRpc( + tip=_hash(1), + tip_height=DECIDED_HEIGHT, + active={DECIDED_HEIGHT: _hash(1)}, + ) + ) + fixture.seed([_hash(1), _hash(2)]) + self.assertEqual(_selected(fixture), {_hash(2)}) + + def test_lower_work_occupant_never_destroys_a_heavier_sibling(self) -> None: + """Clause 4b: a testnet4 minimum-difficulty occupant is not a decision.""" + fixture = CollapseFixture( + rpc=CollapseChainRpc( + tip=DECIDED_WINNER, + tip_height=DECIDED_HEIGHT, + active={DECIDED_HEIGHT: DECIDED_WINNER}, + bits={DECIDED_WINNER: MIN_WORK_BITS}, + ) + ) + occupant = _scaled(MIN_WORK_BITS) + fixture.seed([_hash(1)], network_difficulty=occupant * 8) + fixture.seed([_hash(2)], network_difficulty=occupant) + self.assertEqual(_selected(fixture), {_hash(2)}) + + def test_equal_work_occupant_still_supersedes(self) -> None: + fixture = CollapseFixture( + rpc=CollapseChainRpc( + tip=DECIDED_WINNER, + tip_height=DECIDED_HEIGHT, + active={DECIDED_HEIGHT: DECIDED_WINNER}, + bits={DECIDED_WINNER: RETARGETED_BITS}, + ) + ) + fixture.seed([_hash(1)], network_difficulty=_scaled(RETARGETED_BITS)) + self.assertEqual(_selected(fixture), {_hash(1)}) + + def test_header_work_is_read_in_the_candidates_scaled_units(self) -> None: + """Regression: the row is scaled, ``getblockheader.difficulty`` is raw. + + A candidate stamped with ``scaled_network_difficulty`` and an + active sibling of exactly equal work must collapse. Reading the + header's raw ``difficulty`` float instead of re-deriving from its + bits makes the occupant look COLLAPSE_DIFFICULTY_SCALE times + weaker, so clause 4b/5 rejects it and a decided height never + collapses at all. + """ + for bits in (MIN_WORK_BITS, RETARGETED_BITS, HIGH_WORK_BITS): + with self.subTest(bits=bits): + fixture = CollapseFixture( + rpc=CollapseChainRpc( + tip=DECIDED_WINNER, + tip_height=DECIDED_HEIGHT, + active={DECIDED_HEIGHT: DECIDED_WINNER}, + bits={DECIDED_WINNER: bits}, + ) + ) + scaled = _scaled(bits) + # The stored fact really is COLLAPSE_DIFFICULTY_SCALE times + # the float the node reports for the very same header -- + # only approximately so, which is itself why bits and not + # the float are authoritative here. + self.assertAlmostEqual( + _raw(bits) * COLLAPSE_DIFFICULTY_SCALE / scaled, + 1.0, + places=9, + ) + fixture.seed([_hash(1)], network_difficulty=scaled) + self.assertEqual(_selected(fixture), {_hash(1)}) + + def test_candidate_heavier_than_the_occupant_is_preserved(self) -> None: + """One scaled unit of extra work is still a heavier sibling.""" + for bits in (RETARGETED_BITS, HIGH_WORK_BITS): + with self.subTest(bits=bits): + fixture = CollapseFixture( + rpc=CollapseChainRpc( + tip=DECIDED_WINNER, + tip_height=DECIDED_HEIGHT, + active={DECIDED_HEIGHT: DECIDED_WINNER}, + bits={DECIDED_WINNER: bits}, + ) + ) + scaled = _scaled(bits) + fixture.seed([_hash(1)], network_difficulty=scaled + 1) + fixture.seed([_hash(2)], network_difficulty=scaled) + self.assertEqual(_selected(fixture), {_hash(2)}) + self.assertEqual(fixture.pending(), {_hash(1)}) + + def test_collapse_reuses_the_production_scaled_difficulty_formula(self) -> None: + """The restated formula must not drift from ``template_artifacts``.""" + for bits in ( + MIN_WORK_BITS, + RETARGETED_BITS, + HIGH_WORK_BITS, + "1f00ffff", + "1e00ffff", + "1c7fffff", + ): + with self.subTest(bits=bits): + self.assertEqual( + _collapse_scaled_difficulty(bits), + scaled_network_difficulty(bits), + ) + + def test_unusable_header_bits_fails_the_page_closed(self) -> None: + for value in ( + None, + "", + "n/a", + "not-hex!", + "1d00ff", + "1d00ffff00", + "00000000", + "00000001", + 0x1D00FFFF, + True, + b"1d00ffff", + ): + with self.subTest(bits=value): + fixture = CollapseFixture() + fixture.seed([_hash(1), _hash(2)]) + fixture.rpc.results["getblockheader"] = { + "hash": DECIDED_WINNER, + "bits": value, + # A usable raw float is present and must not rescue the + # read: the scaled comparison has no raw fallback. + "difficulty": 1.0, + } + fixture.collapse() + self.assertEqual(fixture.pending(), {_hash(1), _hash(2)}) + self.assertEqual(fixture.counts()["fail_closed"], 2) + + def test_header_without_bits_fails_the_page_closed(self) -> None: + fixture = CollapseFixture() + fixture.seed([_hash(1), _hash(2)]) + fixture.rpc.results["getblockheader"] = { + "hash": DECIDED_WINNER, + "height": DECIDED_HEIGHT, + "confirmations": 1, + "difficulty": 1.0, + } + fixture.collapse() + self.assertEqual(fixture.pending(), {_hash(1), _hash(2)}) + self.assertEqual(fixture.counts()["fail_closed"], 2) + self.assertEqual(fixture.ledger.abandon_calls, []) + + def test_each_chain_read_failure_fails_the_page_closed(self) -> None: + for method in ( + "getbestblockhash", + "getblockcount", + "getblockhash", + "getblockheader", + ): + with self.subTest(method=method): + fixture = CollapseFixture() + fixture.seed([_hash(1), _hash(2)]) + fixture.rpc.failures.add(method) + retained, _log = fixture.collapse() + self.assertEqual(fixture.pending(), {_hash(1), _hash(2)}) + self.assertEqual(len(retained), 2) + self.assertEqual(fixture.counts()["fail_closed"], 2) + self.assertEqual(fixture.counts()["selected"], 0) + self.assertEqual(fixture.ledger.abandon_calls, []) + + def test_unknown_chain_read_results_fail_the_page_closed(self) -> None: + for method, value in ( + ("getbestblockhash", None), + ("getbestblockhash", "not-a-hash"), + ("getblockcount", "abc"), + ("getblockcount", True), + ("getblockhash", ""), + ("getblockheader", "not-an-object"), + ): + with self.subTest(method=method, value=value): + fixture = CollapseFixture() + fixture.seed([_hash(1)]) + fixture.rpc.results[method] = value + fixture.collapse() + self.assertEqual(fixture.pending(), {_hash(1)}) + self.assertGreaterEqual(fixture.counts()["fail_closed"], 1) + + def test_malformed_intent_fails_the_whole_page_closed(self) -> None: + fixture = CollapseFixture() + fixture.seed([_hash(1), _hash(2)]) + + def corrupt(rows: list[dict[str, Any]]) -> list[dict[str, Any]]: + patched = [] + for row in rows: + if str(row["block_hash"]).lower() == _hash(1): + intent = dict(row["candidate"]) + intent["expected_height"] = "not-a-height" + row = {**row, "candidate": intent} + patched.append(row) + return patched + + fixture.ledger.page_hook = corrupt + self.assertEqual(_selected(fixture), set()) + self.assertEqual(fixture.pending(), {_hash(1), _hash(2)}) + self.assertEqual(fixture.counts()["fail_closed"], 2) + self.assertEqual(fixture.ledger.abandon_calls, []) + + def test_intent_disagreeing_with_template_fails_the_page_closed(self) -> None: + fixture = CollapseFixture() + fixture.seed([_hash(1), _hash(2)]) + + def corrupt(rows: list[dict[str, Any]]) -> list[dict[str, Any]]: + patched = [] + for row in rows: + if str(row["block_hash"]).lower() == _hash(1): + intent = dict(row["candidate"]) + intent["template"] = { + **intent["template"], + "height": DECIDED_HEIGHT + 5, + } + row = {**row, "candidate": intent} + patched.append(row) + return patched + + fixture.ledger.page_hook = corrupt + self.assertEqual(_selected(fixture), set()) + self.assertEqual(fixture.pending(), {_hash(1), _hash(2)}) + self.assertEqual(fixture.counts()["fail_closed"], 2) + + def test_unusable_candidate_difficulty_fails_the_page_closed(self) -> None: + for value in (None, "n/a", float("nan"), True, float("inf"), 0, -1): + with self.subTest(difficulty=value): + fixture = CollapseFixture() + fixture.seed([_hash(1), _hash(2)]) + + def corrupt(rows: list[dict[str, Any]]) -> list[dict[str, Any]]: + intent = dict(rows[0]["candidate"]) + intent["found_block"] = { + **intent["found_block"], + "network_difficulty": value, + } + rows[0] = {**rows[0], "candidate": intent} + return rows + + fixture.ledger.page_hook = corrupt + self.assertEqual(_selected(fixture), set()) + self.assertEqual(fixture.pending(), {_hash(1), _hash(2)}) + self.assertEqual(fixture.counts()["fail_closed"], 2) + + def test_chain_is_read_once_per_page_and_once_per_distinct_height(self) -> None: + fixture = CollapseFixture( + rpc=CollapseChainRpc( + tip=DECIDED_WINNER, + tip_height=DECIDED_HEIGHT, + active={ + DECIDED_HEIGHT: DECIDED_WINNER, + DECIDED_HEIGHT - 1: "ee" * 32, + }, + ) + ) + fixture.seed([_hash(index) for index in range(1, 51)]) + fixture.seed( + [_hash(index) for index in range(51, 101)], + height=DECIDED_HEIGHT - 1, + ) + fixture.collapse() + counts = fixture.rpc.counts() + # One tip read for the page plus one for the pre-write revalidation; + # one active-block read per distinct height in each pass; the + # occupant header only in the selection pass, because revalidation + # found both occupants unchanged. + self.assertEqual(counts["getbestblockhash"], 2) + self.assertEqual(counts["getblockcount"], 1) + self.assertEqual(counts["getblockhash"], 4) + self.assertEqual(counts["getblockheader"], 2) + self.assertEqual(fixture.counts()["abandoned"], 100) + + +class BlockCandidateCollapseApplyTests(unittest.TestCase): + """Lease claiming, revalidation, the single write, and its result set.""" + + def test_lease_is_claimed_without_blocking(self) -> None: + fixture = CollapseFixture() + fixture.seed([_hash(1), _hash(2)]) + claims: list[bool] = [] + original = fixture.server._claim_block_candidate_disposition + + def record(block_hash: str, *, blocking: bool): + claims.append(blocking) + return original(block_hash, blocking=blocking) + + fixture.server._claim_block_candidate_disposition = record + fixture.collapse() + self.assertTrue(claims) + self.assertEqual(set(claims), {False}) + + def test_unclaimable_lease_is_skipped_not_waited_on(self) -> None: + fixture = CollapseFixture() + fixture.seed([_hash(1), _hash(2)]) + original = fixture.server._claim_block_candidate_disposition + + def refuse(block_hash: str, *, blocking: bool): + if block_hash == _hash(1): + return None + return original(block_hash, blocking=blocking) + + fixture.server._claim_block_candidate_disposition = refuse + self.assertEqual(_selected(fixture), {_hash(2)}) + self.assertEqual(fixture.counts()["lease_skipped"], 1) + self.assertEqual(fixture.ledger.abandon_calls[0][0], (_hash(2),)) + + def test_apply_leases_do_not_disqualify_their_own_rows(self) -> None: + """The apply's own flights must not read back as busy evidence.""" + fixture = CollapseFixture() + fixture.seed([_hash(1), _hash(2)]) + seen: list[frozenset[str]] = [] + original = fixture.service._block_candidate_collapse_evidence + + def record(hashes, *, ignore_leases=frozenset()): + seen.append(frozenset(ignore_leases)) + return original(hashes, ignore_leases=ignore_leases) + + fixture.service._block_candidate_collapse_evidence = record + self.assertEqual(_selected(fixture), {_hash(1), _hash(2)}) + self.assertEqual(seen[0], frozenset()) + self.assertEqual(seen[1], frozenset({_hash(1), _hash(2)})) + + def test_every_lease_is_released_after_the_apply(self) -> None: + fixture = CollapseFixture() + fixture.seed([_hash(1), _hash(2)]) + fixture.collapse() + with fixture.service._block_candidate_disposition_registry_lock: + self.assertEqual(fixture.service._block_candidate_disposition_flights, {}) + + def test_leases_are_released_when_the_write_fails(self) -> None: + fixture = CollapseFixture() + fixture.seed([_hash(1)]) + + def explode(hashes: tuple[str, ...], error: str): + raise RuntimeError("ledger down") + + fixture.ledger.abandon_hook = explode + fixture.collapse() + with fixture.service._block_candidate_disposition_registry_lock: + self.assertEqual(fixture.service._block_candidate_disposition_flights, {}) + self.assertEqual(fixture.pending(), {_hash(1)}) + self.assertEqual(fixture.counts()["fail_closed"], 1) + + def test_leases_are_released_when_cleanup_fails(self) -> None: + fixture = CollapseFixture() + fixture.seed([_hash(1)]) + fixture.server._clear_accepted_block_payout_preview = ( + lambda *args, **kwargs: (_ for _ in ()).throw(RuntimeError("nope")) + ) + fixture.collapse() + with fixture.service._block_candidate_disposition_registry_lock: + self.assertEqual(fixture.service._block_candidate_disposition_flights, {}) + self.assertEqual(fixture.counts()["cleanup_failed"], 1) + + def test_revalidation_drops_a_candidate_that_became_the_active_block(self) -> None: + """The immediate pre-write re-read closes the select-to-write gap.""" + fixture = CollapseFixture() + fixture.seed([_hash(1), _hash(2)]) + state = {"selected": False} + rpc = fixture.rpc + + def move_chain(chain: CollapseChainRpc, method: str, params) -> None: + if method != "getbestblockhash": + return + if state["selected"]: + # Second best-tip read: the revalidation. Candidate 1 was + # offered, accepted, and is now the active block. + chain.tip = _hash(1) + chain.active = {DECIDED_HEIGHT: _hash(1)} + state["selected"] = True + + rpc.before_call = move_chain + self.assertEqual(_selected(fixture), {_hash(2)}) + self.assertEqual(fixture.counts()["revalidation_dropped"], 1) + self.assertEqual(fixture.ledger.abandon_calls[0][0], (_hash(2),)) + + def test_revalidation_drops_a_candidate_whose_parent_became_the_tip(self) -> None: + fixture = CollapseFixture() + fixture.seed([_hash(1)], parent=STORM_PARENT) + fixture.seed([_hash(2)], parent=OTHER_PARENT) + state = {"seen": 0} + + def move_chain(chain: CollapseChainRpc, method: str, params) -> None: + if method != "getbestblockhash": + return + state["seen"] += 1 + if state["seen"] == 2: + chain.tip = OTHER_PARENT + + fixture.rpc.before_call = move_chain + self.assertEqual(_selected(fixture), {_hash(1)}) + self.assertEqual(fixture.counts()["revalidation_dropped"], 1) + + def test_revalidation_reads_a_changed_occupants_work(self) -> None: + """A lighter replacement occupant drops the row, in scaled units.""" + fixture = CollapseFixture() + fixture.seed([_hash(1)], network_difficulty=_scaled(RETARGETED_BITS)) + fixture.rpc.bits[DECIDED_WINNER] = RETARGETED_BITS + fixture.rpc.bits["ee" * 32] = MIN_WORK_BITS + state = {"seen": 0} + + def move_chain(chain: CollapseChainRpc, method: str, params) -> None: + if method != "getbestblockhash": + return + state["seen"] += 1 + if state["seen"] == 2: + chain.tip = "ee" * 32 + chain.active = {DECIDED_HEIGHT: "ee" * 32} + + fixture.rpc.before_call = move_chain + self.assertEqual(_selected(fixture), set()) + self.assertEqual(fixture.counts()["revalidation_dropped"], 1) + headers = [ + params for method, params in fixture.rpc.calls if method == "getblockheader" + ] + self.assertEqual(headers, [(DECIDED_WINNER,), ("ee" * 32,)]) + + def test_revalidation_keeps_an_equal_work_changed_occupant(self) -> None: + """The changed-occupant re-read normalizes exactly as selection does. + + The replacement occupant carries different bits from the one + selection read but exactly the candidate's scaled work, so the row + still collapses. A revalidation that compared the replacement's raw + header float against the row's scaled value would drop it instead. + """ + fixture = CollapseFixture() + fixture.seed([_hash(1)], network_difficulty=_scaled(HIGH_WORK_BITS)) + fixture.rpc.bits[DECIDED_WINNER] = HIGH_WORK_BITS + fixture.rpc.bits["ee" * 32] = HIGH_WORK_BITS + state = {"seen": 0} + + def move_chain(chain: CollapseChainRpc, method: str, params) -> None: + if method != "getbestblockhash": + return + state["seen"] += 1 + if state["seen"] == 2: + chain.active = {DECIDED_HEIGHT: "ee" * 32} + + fixture.rpc.before_call = move_chain + self.assertEqual(_selected(fixture), {_hash(1)}) + self.assertEqual(fixture.counts()["revalidation_dropped"], 0) + headers = [ + params for method, params in fixture.rpc.calls if method == "getblockheader" + ] + self.assertEqual(headers, [(DECIDED_WINNER,), ("ee" * 32,)]) + + def test_revalidation_of_a_changed_occupant_fails_closed_without_bits( + self, + ) -> None: + """The pre-write header re-read has no raw-difficulty fallback either.""" + fixture = CollapseFixture() + fixture.seed([_hash(1)], network_difficulty=_scaled(RETARGETED_BITS)) + fixture.rpc.bits[DECIDED_WINNER] = RETARGETED_BITS + state = {"seen": 0} + + def move_chain(chain: CollapseChainRpc, method: str, params) -> None: + if method != "getbestblockhash": + return + state["seen"] += 1 + if state["seen"] == 2: + chain.active = {DECIDED_HEIGHT: "ee" * 32} + chain.results["getblockheader"] = { + "hash": "ee" * 32, + "difficulty": _raw(RETARGETED_BITS), + } + + fixture.rpc.before_call = move_chain + fixture.collapse() + self.assertEqual(fixture.pending(), {_hash(1)}) + self.assertEqual(fixture.counts()["fail_closed"], 1) + self.assertEqual(fixture.ledger.abandon_calls, []) + + def test_revalidation_failure_fails_the_page_closed(self) -> None: + fixture = CollapseFixture() + fixture.seed([_hash(1), _hash(2)]) + state = {"seen": 0} + + def fail_second(chain: CollapseChainRpc, method: str, params) -> None: + if method != "getbestblockhash": + return + state["seen"] += 1 + if state["seen"] == 2: + chain.failures.add("getbestblockhash") + + fixture.rpc.before_call = fail_second + retained, _log = fixture.collapse() + self.assertEqual(fixture.pending(), {_hash(1), _hash(2)}) + self.assertEqual(len(retained), 2) + self.assertEqual(fixture.ledger.abandon_calls, []) + self.assertEqual(fixture.counts()["fail_closed"], 2) + + def test_one_batch_write_replaces_the_per_row_writes(self) -> None: + fixture = CollapseFixture() + fixture.seed([_hash(index) for index in range(1, 21)]) + fixture.collapse() + self.assertEqual(len(fixture.ledger.abandon_calls), 1) + requested, error = fixture.ledger.abandon_calls[0] + self.assertEqual(len(requested), 20) + self.assertIn("tip moved before submit", error) + self.assertIn(DECIDED_WINNER, error) + + def test_cleanup_follows_the_returned_set_not_the_request(self) -> None: + fixture = CollapseFixture() + fixture.seed([_hash(1), _hash(2), _hash(3)]) + service = fixture.service + + def partial(hashes: tuple[str, ...], error: str) -> tuple[str, ...]: + won = tuple(value for value in hashes if value != _hash(2)) + return fixture.ledger.inner.mark_block_candidates_abandoned( + block_hashes=won, + error=error, + ) + + fixture.ledger.abandon_hook = partial + retained, _log = fixture.collapse() + self.assertEqual(fixture.pending(), {_hash(2)}) + self.assertEqual(fixture.counts()["abandoned"], 2) + self.assertEqual(fixture.counts()["write_lost"], 1) + with fixture.server.lock: + terminal = dict(service._block_candidate_terminal_outcomes) + self.assertEqual(set(terminal), {_hash(1), _hash(3)}) + self.assertNotIn(_hash(2), terminal) + self.assertEqual( + {str(row["block_hash"]).lower() for row in retained}, + {_hash(2)}, + ) + + def test_a_hash_outside_the_request_is_never_cleaned_up(self) -> None: + fixture = CollapseFixture() + fixture.seed([_hash(1)]) + stranger = _hash(99) + + def stray(hashes: tuple[str, ...], error: str) -> tuple[str, ...]: + fixture.ledger.inner.mark_block_candidates_abandoned( + block_hashes=hashes, + error=error, + ) + return hashes + (stranger,) + + fixture.ledger.abandon_hook = stray + fixture.collapse() + with fixture.server.lock: + terminal = dict(fixture.service._block_candidate_terminal_outcomes) + self.assertEqual(set(terminal), {_hash(1)}) + + def test_an_empty_returned_set_preserves_the_whole_page(self) -> None: + fixture = CollapseFixture() + fixture.seed([_hash(1), _hash(2)]) + fixture.ledger.abandon_hook = lambda hashes, error: () + rows = fixture.page() + retained, _log = fixture.collapse(rows) + self.assertEqual(len(retained), 2) + self.assertEqual(fixture.counts()["abandoned"], 0) + self.assertEqual(fixture.counts()["write_lost"], 2) + with fixture.server.lock: + self.assertEqual(fixture.service._block_candidate_terminal_outcomes, {}) + + def test_a_non_hash_set_result_fails_the_page_closed(self) -> None: + """A count or a boolean says nothing about which rows were won.""" + for value in (True, 2, None, "01" * 32): + with self.subTest(returned=value): + fixture = CollapseFixture() + fixture.seed([_hash(1), _hash(2)]) + + def answer(hashes: tuple[str, ...], error: str, value=value): + fixture.ledger.inner.mark_block_candidates_abandoned( + block_hashes=hashes, + error=error, + ) + return value + + fixture.ledger.abandon_hook = answer + rows = fixture.page() + retained, _log = fixture.collapse(rows) + self.assertEqual(len(retained), 2) + self.assertEqual(fixture.counts()["abandoned"], 0) + self.assertEqual(fixture.counts()["fail_closed"], 2) + with fixture.server.lock: + self.assertEqual( + fixture.service._block_candidate_terminal_outcomes, + {}, + ) + + def test_no_write_is_issued_for_an_empty_selection(self) -> None: + fixture = CollapseFixture( + rpc=CollapseChainRpc(tip=STORM_PARENT, tip_height=DECIDED_HEIGHT) + ) + fixture.seed([_hash(1)]) + fixture.collapse() + self.assertEqual(fixture.ledger.abandon_calls, []) + self.assertEqual(fixture.pending(), {_hash(1)}) + + def test_a_ledger_without_the_batch_method_is_left_entirely_alone(self) -> None: + fixture = CollapseFixture(ledger=NoBatchCollapseLedger()) + fixture.seed([_hash(1)]) + rows = fixture.page() + retained, _log = fixture.collapse(rows) + self.assertEqual(retained, rows) + self.assertEqual(fixture.rpc.calls, []) + self.assertEqual(fixture.counts()["considered"], 0) + self.assertEqual(fixture.pending(), {_hash(1)}) + + +class BlockCandidateCollapseCleanupTests(unittest.TestCase): + """Cleanup list M: everything terminal per-row abandonment releases.""" + + def _fixture_with_state(self) -> tuple[CollapseFixture, str]: + fixture = CollapseFixture() + target = _hash(1) + fixture.seed([target]) + service = fixture.service + server = fixture.server + server._register_outstanding_block_candidate(target) + server._begin_accepted_block_payout_preview( + target, + block_height=DECIDED_HEIGHT, + ) + server._mark_accepted_block_payout_landed( + target, + block_height=DECIDED_HEIGHT, + ) + with server.lock: + service._block_fast_lane_reservations.add(target) + service._block_replay_inflight_hashes.add(target) + service.retry_delays[target] = 4.0 + # Both registries are created on first touch by the pacing and + # landing-timeout paths; seed them the way those paths would. + service._block_candidate_retry_not_before = {target: 1.0} + service._block_landing_timeout_counts = {target: 3} + return fixture, target + + def test_cleanup_releases_every_owned_state(self) -> None: + fixture, target = self._fixture_with_state() + service = fixture.service + server = fixture.server + with redirect_stdout(StringIO()): + fixture.collapse() + self.assertEqual(fixture.pending(), set()) + with server.lock: + self.assertIs(service._block_candidate_terminal_outcomes[target], False) + self.assertNotIn(target, service._block_fast_lane_reservations) + self.assertNotIn(target, service._block_replay_inflight_hashes) + self.assertNotIn(target, service._outstanding_block_candidate_hashes) + self.assertNotIn(target, service._tip_observed_accepted_block_hashes) + self.assertNotIn(target, service.retry_delays) + self.assertNotIn(target, service._block_candidate_retry_not_before) + self.assertNotIn(target, service._block_landing_timeout_counts) + self.assertNotIn(target, service.finalize_retries) + with server._accepted_block_payout_preview_condition: + self.assertNotIn(target, server._accepted_block_payout_previews) + self.assertNotIn( + target, + server._invalidated_accepted_block_payout_previews, + ) + self.assertEqual( + server.block_candidate_abandoned_counts, + {PRISM_BLOCK_CANDIDATE_COLLAPSE_REASON: 1}, + ) + self.assertEqual( + server.stale_job_abandon_counts[ + PRISM_BLOCK_CANDIDATE_COLLAPSE_STALE_JOB_CLASS + ], + 1, + ) + + def test_cleanup_leaves_no_payout_tombstone_behind(self) -> None: + fixture, target = self._fixture_with_state() + with redirect_stdout(StringIO()): + fixture.collapse() + with fixture.server._accepted_block_payout_preview_condition: + self.assertEqual( + fixture.server._invalidated_accepted_block_payout_previews, + {}, + ) + + def test_finalize_retry_state_is_removed_by_cleanup(self) -> None: + fixture = CollapseFixture() + target = _hash(1) + fixture.seed([target]) + with fixture.server.lock: + fixture.service.finalize_retries[target] = (False, "stale") + with redirect_stdout(StringIO()): + fixture.service._clean_up_collapsed_block_candidates((target,)) + with fixture.server.lock: + self.assertEqual(fixture.service.finalize_retries, {}) + + def test_abandonment_is_counted_exactly_once(self) -> None: + fixture = CollapseFixture() + target = _hash(1) + fixture.seed([target]) + with redirect_stdout(StringIO()): + fixture.service._clean_up_collapsed_block_candidates((target,)) + fixture.service._clean_up_collapsed_block_candidates((target,)) + self.assertEqual( + fixture.server.block_candidate_abandoned_counts, + {PRISM_BLOCK_CANDIDATE_COLLAPSE_REASON: 1}, + ) + self.assertEqual( + fixture.server.stale_job_abandon_counts[ + PRISM_BLOCK_CANDIDATE_COLLAPSE_STALE_JOB_CLASS + ], + 1, + ) + + def test_cleanup_is_idempotent(self) -> None: + fixture, target = self._fixture_with_state() + with redirect_stdout(StringIO()): + fixture.service._clean_up_collapsed_block_candidates((target,)) + fixture.service._clean_up_collapsed_block_candidates((target,)) + self.assertEqual(fixture.counts()["cleanup_failed"], 0) + + def test_one_failing_cleanup_does_not_strand_later_steps_or_rows(self) -> None: + fixture = CollapseFixture() + hashes = [_hash(1), _hash(2), _hash(3)] + candidates = fixture.seed(hashes, credit_share_on_accept=True) + writer = fixture.server._ensure_share_writer_service() + writer.adopt_pending_share(candidates[1].pending_share) + fixture.service.candidate_queue.put_nowait(candidates[1]) + original = fixture.server._clear_block_candidate_retry_state + + def explode(block_hash: str) -> None: + if block_hash == _hash(2): + raise RuntimeError("cleanup boom") + original(block_hash) + + fixture.server._clear_block_candidate_retry_state = explode + with redirect_stdout(StringIO()): + fixture.collapse() + self.assertEqual(fixture.pending(), set()) + with fixture.server.lock: + terminal = dict(fixture.service._block_candidate_terminal_outcomes) + self.assertEqual(set(terminal), set(hashes)) + self.assertNotIn(id(candidates[1].pending_share), fixture.floor()) + self.assertEqual(fixture.counts()["cleanup_failed"], 1) + self.assertEqual(fixture.counts()["abandoned"], 3) + + def test_systemic_cleanup_failure_logging_stays_bounded(self) -> None: + fixture = CollapseFixture() + hashes = [_hash(index) for index in range(1, 41)] + fixture.seed(hashes) + fixture.server._clear_block_candidate_retry_state = ( + lambda block_hash: (_ for _ in ()).throw(RuntimeError("boom")) + ) + _retained, log = fixture.collapse() + detailed = [ + line + for line in log.splitlines() + if "cleanup failed step=" in line + ] + self.assertEqual(len(detailed), BLOCK_CANDIDATE_COLLAPSE_LOG_FAILURES) + self.assertIn("cleanup failed rows=40 of 40", log) + self.assertEqual(fixture.counts()["cleanup_failed"], 40) + + def test_an_aborted_cleanup_still_partitions_the_terminal_rows(self) -> None: + """A terminal row must never be replay-adopted, cleanup or not.""" + fixture = CollapseFixture() + fixture.seed([_hash(1), _hash(2)]) + fixture.service._collapsed_candidate_floor_holders = ( + lambda abandoned: (_ for _ in ()).throw(RuntimeError("boom")) + ) + rows = fixture.page() + retained, log = fixture.collapse(rows) + self.assertEqual(retained, []) + self.assertEqual(fixture.pending(), set()) + self.assertIn("cleanup aborted rows=2", log) + self.assertEqual(fixture.counts()["cleanup_failed"], 2) + + def test_a_log_failure_does_not_double_count_cleanup_failures(self) -> None: + """cleanup_failed counts affected hashes, not diagnostic failures. + + A contained per-step failure belongs to the one hash it happened to, + while a later summary formatter fault does not change any hash's + cleanup state. Treating both as cleanup reported four affected hashes + for a three-row page even though only one cleanup actually failed. + """ + fixture = CollapseFixture() + hashes = [_hash(1), _hash(2), _hash(3)] + fixture.seed(hashes) + original = fixture.server._clear_block_candidate_retry_state + + def explode(block_hash: str) -> None: + if block_hash == _hash(2): + raise RuntimeError("cleanup boom") + original(block_hash) + + fixture.server._clear_block_candidate_retry_state = explode + # The grouped formatter runs only after every cleanup step. + fixture.service._log_collapsed_block_candidates = ( + lambda *args, **kwargs: (_ for _ in ()).throw(RuntimeError("log boom")) + ) + rows = fixture.page() + retained, log = fixture.collapse(rows) + counts = fixture.counts() + self.assertEqual(counts["abandoned"], 3) + self.assertEqual(counts["cleanup_failed"], 1) + self.assertLessEqual(counts["cleanup_failed"], counts["abandoned"]) + # Both faults are visible, every cleanup attempt still ran, and the + # terminal rows are still partitioned out of the page. + self.assertIn("cleanup failed rows=1 of 3", log) + self.assertIn("logging failed rows=3", log) + self.assertNotIn("cleanup aborted rows=3", log) + self.assertEqual(retained, []) + self.assertEqual(fixture.pending(), set()) + with fixture.server.lock: + self.assertEqual( + set(fixture.service._block_candidate_terminal_outcomes), + set(hashes), + ) + with fixture.service._block_candidate_disposition_registry_lock: + self.assertEqual(fixture.service._block_candidate_disposition_flights, {}) + + def test_a_queued_candidates_floor_holder_is_released(self) -> None: + fixture = CollapseFixture() + target = _hash(1) + candidate = fixture.seed([target], credit_share_on_accept=True)[0] + writer = fixture.server._ensure_share_writer_service() + writer.adopt_pending_share(candidate.pending_share) + fixture.service.candidate_queue.put_nowait(candidate) + self.assertIn(id(candidate.pending_share), fixture.floor()) + with redirect_stdout(StringIO()): + fixture.collapse() + self.assertNotIn(id(candidate.pending_share), fixture.floor()) + + def test_a_replay_queued_candidates_floor_holder_is_released(self) -> None: + fixture = CollapseFixture() + target = _hash(1) + candidate = fixture.seed([target], credit_share_on_accept=True)[0] + writer = fixture.server._ensure_share_writer_service() + writer.adopt_pending_share(candidate.pending_share) + fixture.service._block_replay_candidate_queue.put_nowait(candidate) + with redirect_stdout(StringIO()): + fixture.collapse() + self.assertNotIn(id(candidate.pending_share), fixture.floor()) + + def test_another_candidates_floor_holder_is_never_released(self) -> None: + fixture = CollapseFixture() + collapsed = fixture.seed([_hash(1)], credit_share_on_accept=True)[0] + # A same-hash duplicate object and an unrelated hash both keep their + # own identity-keyed holders; only the collapsed object's is dropped. + duplicate = fixture.seed( + [_hash(1)], + credit_share_on_accept=True, + )[0] + untouched = fixture.seed( + [_hash(2)], + parent=OTHER_PARENT, + credit_share_on_accept=True, + )[0] + writer = fixture.server._ensure_share_writer_service() + for candidate in (collapsed, duplicate, untouched): + writer.adopt_pending_share(candidate.pending_share) + fixture.service.candidate_queue.put_nowait(collapsed) + fixture.rpc.tip = OTHER_PARENT + with redirect_stdout(StringIO()): + fixture.collapse() + floor = fixture.floor() + self.assertNotIn(id(collapsed.pending_share), floor) + self.assertIn(id(duplicate.pending_share), floor) + self.assertIn(id(untouched.pending_share), floor) + + def test_no_floor_holder_is_invented_for_an_unadopted_row(self) -> None: + fixture = CollapseFixture() + fixture.seed([_hash(1)], credit_share_on_accept=True) + before = set(fixture.floor()) + with redirect_stdout(StringIO()): + fixture.collapse() + self.assertEqual(set(fixture.floor()), before) + + def test_cleanup_is_safe_for_an_already_terminal_hash(self) -> None: + fixture = CollapseFixture() + target = _hash(1) + fixture.seed([target]) + fixture.server._record_block_candidate_terminal_outcome( + target, + accepted=True, + ) + with redirect_stdout(StringIO()): + fixture.service._clean_up_collapsed_block_candidates((target,)) + with fixture.server.lock: + self.assertIs( + fixture.service._block_candidate_terminal_outcomes[target], + False, + ) + self.assertEqual(fixture.counts()["cleanup_failed"], 0) + + def test_collapse_reason_matches_the_per_row_stale_job_reason(self) -> None: + self.assertEqual( + PRISM_BLOCK_CANDIDATE_COLLAPSE_REASON, + PRISM_REJECTION_STALE_JOB, + ) + self.assertIn( + PRISM_BLOCK_CANDIDATE_COLLAPSE_STALE_JOB_CLASS, + PRISM_STALE_JOB_ABANDON_CLASSES, + ) + + +def _break_cleanup_step(target: Any, name: str, *, only: str | None = None) -> dict: + """Make one cleanup step raise until the returned switch is flipped.""" + original = getattr(target, name) + broken = {"on": True} + + def failing(block_hash: str, *args: Any, **kwargs: Any) -> Any: + if broken["on"] and (only is None or block_hash == only): + raise RuntimeError(f"{name} boom") + return original(block_hash, *args, **kwargs) + + setattr(target, name, failing) + return broken + + +class AccountingLaneRig: + """Drive the shipped accounting loop against controllable lane traffic. + + Only the two sinks the loop reaches through the coordinator are replaced + -- the ordinary accounting task runner and the invalid-candidate + quarantine drain -- so the loop body, its cleanup-retry cadence, and the + real deferred cleanup all stay exactly as shipped. ``mode`` chooses + which lane is kept permanently busy: + + ``accounting`` + Each finished task re-arms the primary handoff queue before it + returns, so the loop's next queue read can never see an empty lane. + ``quarantine`` + Both handoff queues stay empty and the quarantine drain always + reports an item, so the loop never reaches its idle branch either. + ``both`` + The two alternate: each quarantine item re-arms the accounting + queue, so one work item of each kind completes per pair. + ``idle`` + No traffic at all, so the loop falls straight through to the idle + branch that predates the cadence. + + In every busy mode the idle branch is unreachable by construction, which + is what makes these fixtures a starvation test rather than a race. + """ + + ACCOUNTING = "accounting" + QUARANTINE = "quarantine" + BOTH = "both" + IDLE = "idle" + + def __init__(self, fixture: CollapseFixture, mode: str) -> None: + self.fixture = fixture + self.server = fixture.server + self.service = fixture.service + self.mode = mode + self.accounting_items = 0 + self.quarantine_items = 0 + # (accounting items, quarantine items) completed at each offer. + self.offers: list[tuple[int, int]] = [] + self.attempts = 0 + self.log = "" + self.thread_alive = False + self._sequence = 0 + self.service._ensure_block_accounting_state() + self.service._ensure_block_replay_state() + self.server._run_block_accounting_task = self._run_task + self.server._run_one_invalid_block_candidate_quarantine = self._run_quarantine + shipped = self.service._run_one_collapsed_block_candidate_cleanup_retry + + def offer() -> bool: + self.offers.append((self.accounting_items, self.quarantine_items)) + ran = shipped() + if ran: + self.attempts += 1 + return ran + + self.service._run_one_collapsed_block_candidate_cleanup_retry = offer + if mode in (self.ACCOUNTING, self.BOTH): + self._arm_accounting() + + @property + def work_items(self) -> int: + return self.accounting_items + self.quarantine_items + + def _arm_accounting(self) -> None: + self._sequence += 1 + self.service._block_accounting_queue.put_nowait( + (0, self._sequence, SimpleNamespace()) + ) + + def _run_task(self, task: Any) -> None: + self.accounting_items += 1 + if self.mode == self.ACCOUNTING: + self._arm_accounting() + + def _run_quarantine(self) -> bool: + if self.mode not in (self.QUARANTINE, self.BOTH): + return False + self.quarantine_items += 1 + if self.mode == self.BOTH: + self._arm_accounting() + return True + + def run_until(self, predicate: Any, *, timeout: float = 10.0) -> bool: + """Run the lane until ``predicate`` holds or the deadline expires. + + Bounded on purpose: a starved retry must end the test with a report, + not with an unbounded wait. Counters are only read by callers after + the loop thread has been stopped and joined, so every assertion sees + one consistent snapshot. + """ + thread = threading.Thread( + target=self.service.block_accounting_loop, + daemon=True, + ) + buffer = StringIO() + with redirect_stdout(buffer): + thread.start() + try: + deadline = time.monotonic() + timeout + while not predicate() and time.monotonic() < deadline: + time.sleep(0.002) + finally: + self.server.stop_event.set() + thread.join(timeout=timeout) + self.log = buffer.getvalue() + self.thread_alive = thread.is_alive() + return predicate() + + +class BlockCandidateCollapseCleanupRetryTests(unittest.TestCase): + """A failed cleanup is the only state with no durable replay source. + + Once the fenced batch write returns a hash, that row is terminal: it is + partitioned out of the replay page and no later enumeration can hand it + back. A cleanup step that failed therefore has nothing left to retry + it, and whatever it did not tear down -- a payout preview or its + tombstone, a pending-share floor holder, an outstanding-hash marker -- + would stay installed for the process lifetime. These tests own the + bounded, idempotent retry that closes that gap, and the accounting of + it: the affected-hash series still counts hashes, never attempts. + """ + + @staticmethod + def _breaker(target: Any, name: str, *, only: str | None = None) -> dict: + """Make one cleanup step raise until the returned switch is flipped.""" + return _break_cleanup_step(target, name, only=only) + + # -- payout preview and tombstone -------------------------------------- + + def test_a_failed_payout_withdrawal_is_retried_until_it_recovers(self) -> None: + """The wait storm the collapse exists to end must not be re-armed.""" + fixture = CollapseFixture() + target = _hash(1) + fixture.seed([target]) + server = fixture.server + server._begin_accepted_block_payout_preview( + target, + block_height=DECIDED_HEIGHT, + ) + server._mark_accepted_block_payout_landed( + target, + block_height=DECIDED_HEIGHT, + ) + broken = self._breaker(server, "_clear_accepted_block_payout_preview") + with redirect_stdout(StringIO()): + fixture.collapse() + # The durable row is terminal and gone from every replay source + # while the landed barrier it armed is still installed: descendants + # keep waiting on a transition nothing else will ever clear. + self.assertEqual(fixture.pending(), set()) + self.assertTrue(server._accepted_block_payout_transition_landed(target)) + self.assertEqual(fixture.counts()["cleanup_failed"], 1) + self.assertEqual( + fixture.cleanup_backlog(), + {target: frozenset({BLOCK_CANDIDATE_COLLAPSE_CLEANUP_PAYOUT_STEP})}, + ) + broken["on"] = False + ran, log = fixture.retry_cleanup() + self.assertTrue(ran) + with server._accepted_block_payout_preview_condition: + self.assertNotIn(target, server._accepted_block_payout_previews) + self.assertNotIn( + target, + server._invalidated_accepted_block_payout_previews, + ) + self.assertEqual(fixture.cleanup_backlog(), {}) + self.assertIn("cleanup recovered", log) + counts = fixture.counts() + self.assertEqual(counts["cleanup_recovered"], 1) + # The affected-hash series counts the hash once, whatever it took. + self.assertEqual(counts["cleanup_failed"], 1) + + def test_a_failed_tombstone_drop_is_retried(self) -> None: + """A retained tombstone fails descendants closed just as hard.""" + fixture = CollapseFixture() + target = _hash(1) + fixture.seed([target]) + server = fixture.server + server._begin_accepted_block_payout_preview( + target, + block_height=DECIDED_HEIGHT, + ) + server._mark_accepted_block_payout_landed( + target, + block_height=DECIDED_HEIGHT, + ) + original = server._clear_accepted_block_payout_preview + broken = {"on": True} + + def clear(block_hash: str, *, invalidate_published: bool = False) -> None: + # The withdrawal lands and installs the tombstone; only the + # second half -- the drop -- fails. + if broken["on"] and not invalidate_published: + raise RuntimeError("tombstone drop boom") + original(block_hash, invalidate_published=invalidate_published) + + server._clear_accepted_block_payout_preview = clear + with redirect_stdout(StringIO()): + fixture.collapse() + with server._accepted_block_payout_preview_condition: + self.assertIn( + target, + server._invalidated_accepted_block_payout_previews, + ) + broken["on"] = False + self.assertTrue(fixture.retry_cleanup()[0]) + with server._accepted_block_payout_preview_condition: + self.assertEqual(server._invalidated_accepted_block_payout_previews, {}) + self.assertEqual(fixture.cleanup_backlog(), {}) + + # -- pending-share floor ----------------------------------------------- + + def test_a_failed_floor_release_survives_the_queue_it_was_read_from(self) -> None: + """The floor keys holders by identity, so the retry must carry them.""" + fixture = CollapseFixture() + target = _hash(1) + candidate = fixture.seed([target], credit_share_on_accept=True)[0] + server = fixture.server + server._ensure_share_writer_service().adopt_pending_share( + candidate.pending_share + ) + fixture.service.candidate_queue.put_nowait(candidate) + original = server._finish_pending_share_commit + broken = {"on": True} + + def finish(pending_share: Any) -> None: + if broken["on"]: + raise RuntimeError("floor release boom") + original(pending_share) + + server._finish_pending_share_commit = finish + with redirect_stdout(StringIO()): + fixture.collapse() + self.assertIn(id(candidate.pending_share), fixture.floor()) + self.assertEqual( + fixture.cleanup_backlog(), + {target: frozenset({BLOCK_CANDIDATE_COLLAPSE_CLEANUP_FLOOR_STEP})}, + ) + # The queue the holder was indexed from drains before the retry + # runs, so a fresh scan would find nothing to release. + self.assertIs(fixture.service.candidate_queue.get_nowait(), candidate) + self.assertEqual( + fixture.service._collapsed_candidate_floor_holders((target,)), + {}, + ) + broken["on"] = False + self.assertTrue(fixture.retry_cleanup()[0]) + self.assertNotIn(id(candidate.pending_share), fixture.floor()) + self.assertEqual(fixture.cleanup_backlog(), {}) + self.assertEqual(fixture.counts()["cleanup_recovered"], 1) + + def test_another_candidates_floor_holder_is_never_released_by_a_retry( + self, + ) -> None: + fixture = CollapseFixture() + collapsed = fixture.seed([_hash(1)], credit_share_on_accept=True)[0] + untouched = fixture.seed( + [_hash(2)], + parent=OTHER_PARENT, + credit_share_on_accept=True, + )[0] + server = fixture.server + writer = server._ensure_share_writer_service() + for candidate in (collapsed, untouched): + writer.adopt_pending_share(candidate.pending_share) + fixture.service.candidate_queue.put_nowait(collapsed) + fixture.rpc.tip = OTHER_PARENT + original = server._finish_pending_share_commit + broken = {"on": True} + + def finish(pending_share: Any) -> None: + if broken["on"]: + raise RuntimeError("floor release boom") + original(pending_share) + + server._finish_pending_share_commit = finish + with redirect_stdout(StringIO()): + fixture.collapse() + broken["on"] = False + self.assertTrue(fixture.retry_cleanup()[0]) + floor = fixture.floor() + self.assertNotIn(id(collapsed.pending_share), floor) + self.assertIn(id(untouched.pending_share), floor) + + # -- what a retry repeats ---------------------------------------------- + + def test_a_retry_repeats_only_the_steps_that_are_still_owed(self) -> None: + fixture = CollapseFixture() + target = _hash(1) + fixture.seed([target]) + server = fixture.server + discarded: list[str] = [] + original_discard = server._discard_outstanding_block_candidate + + def discard(block_hash: str) -> None: + discarded.append(block_hash) + original_discard(block_hash) + + server._discard_outstanding_block_candidate = discard + broken = self._breaker(server, "_clear_block_candidate_retry_state") + with redirect_stdout(StringIO()): + fixture.collapse() + self.assertEqual(discarded, [target]) + self.assertEqual( + fixture.cleanup_backlog(), + {target: frozenset({"retry-state"})}, + ) + broken["on"] = False + self.assertTrue(fixture.retry_cleanup()[0]) + # A step that already completed is never run a second time. + self.assertEqual(discarded, [target]) + self.assertEqual(fixture.cleanup_backlog(), {}) + + def test_an_aborted_cleanup_is_deferred_and_retried_in_full(self) -> None: + """An abort proves nothing, so every won hash owes every step again.""" + fixture = CollapseFixture() + target = _hash(1) + candidate = fixture.seed([target], credit_share_on_accept=True)[0] + server = fixture.server + server._ensure_share_writer_service().adopt_pending_share( + candidate.pending_share + ) + fixture.service.candidate_queue.put_nowait(candidate) + scan = fixture.service._collapsed_candidate_floor_holders + fixture.service._collapsed_candidate_floor_holders = ( + lambda abandoned: (_ for _ in ()).throw(RuntimeError("index boom")) + ) + retained, log = fixture.collapse() + self.assertEqual(retained, []) + self.assertEqual(fixture.pending(), set()) + self.assertIn("cleanup aborted rows=1", log) + self.assertEqual(fixture.counts()["cleanup_failed"], 1) + self.assertEqual( + fixture.cleanup_backlog(), + {target: frozenset(BLOCK_CANDIDATE_COLLAPSE_CLEANUP_STEPS)}, + ) + # The abort ran the withdrawal and then stopped: the floor holder + # and every terminal accounting step are still owed. + self.assertIn(id(candidate.pending_share), fixture.floor()) + self.assertIn("terminal-outcome", fixture.cleanup_backlog()[target]) + with server.lock: + # The apply published the terminal fence before cleanup, so the + # durably terminal row is already unofferable even though the + # step that normally publishes it never ran. + self.assertIs( + fixture.service._block_candidate_terminal_outcomes[target], + False, + ) + self.assertEqual(server.block_candidate_abandoned_counts, {}) + fixture.service._collapsed_candidate_floor_holders = scan + self.assertTrue(fixture.retry_cleanup()[0]) + # The retry re-indexes the holders the abort never got to read. + self.assertNotIn(id(candidate.pending_share), fixture.floor()) + with server.lock: + self.assertIs( + fixture.service._block_candidate_terminal_outcomes[target], + False, + ) + self.assertNotIn( + target, + fixture.service._outstanding_block_candidate_hashes, + ) + self.assertEqual( + server.block_candidate_abandoned_counts, + {PRISM_BLOCK_CANDIDATE_COLLAPSE_REASON: 1}, + ) + self.assertEqual(fixture.cleanup_backlog(), {}) + self.assertEqual(fixture.counts()["cleanup_recovered"], 1) + + def test_a_cleanup_retry_never_re_adopts_or_re_offers_the_row(self) -> None: + fixture = CollapseFixture() + target = _hash(1) + fixture.seed([target]) + server = fixture.server + broken = self._breaker(server, "_record_block_candidate_terminal_outcome") + with redirect_stdout(StringIO()): + fixture.collapse() + abandon_calls = len(fixture.ledger.abandon_calls) + rpc_calls = len(fixture.rpc.calls) + broken["on"] = False + self.assertTrue(fixture.retry_cleanup()[0]) + # No durable read or write, no node offer, no queued candidate. + self.assertEqual(fixture.pending(), set()) + self.assertEqual(len(fixture.ledger.abandon_calls), abandon_calls) + self.assertEqual(len(fixture.rpc.calls), rpc_calls) + self.assertTrue(fixture.service.candidate_queue.empty()) + self.assertTrue(fixture.service._block_replay_candidate_queue.empty()) + with server.lock: + self.assertEqual(fixture.service._block_replay_inflight_hashes, set()) + self.assertIsNone(fixture.service.retry_candidate) + self.assertIs( + fixture.service._block_candidate_terminal_outcomes[target], + False, + ) + + # -- bounds, pacing, and accounting ------------------------------------ + + def test_the_backlog_is_bounded_by_the_affected_terminal_hashes(self) -> None: + fixture = CollapseFixture() + hashes = [_hash(1), _hash(2), _hash(3)] + fixture.seed(hashes) + server = fixture.server + original = server._clear_block_candidate_retry_state + broken = {"on": True} + + def retry_state(block_hash: str) -> None: + if broken["on"] and block_hash != _hash(3): + raise RuntimeError("retry state boom") + original(block_hash) + + server._clear_block_candidate_retry_state = retry_state + with redirect_stdout(StringIO()): + fixture.collapse() + # Only the hashes whose cleanup actually failed are retained. + self.assertEqual(set(fixture.cleanup_backlog()), {_hash(1), _hash(2)}) + broken["on"] = False + # One hash per pass, and the lane idles once the backlog drains. + self.assertTrue(fixture.retry_cleanup()[0]) + self.assertEqual(len(fixture.cleanup_backlog()), 1) + self.assertTrue(fixture.retry_cleanup()[0]) + self.assertEqual(fixture.cleanup_backlog(), {}) + self.assertFalse(fixture.retry_cleanup()[0]) + counts = fixture.counts() + self.assertEqual(counts["cleanup_recovered"], 2) + self.assertEqual(counts["cleanup_failed"], 2) + self.assertLessEqual(counts["cleanup_recovered"], counts["cleanup_failed"]) + + def test_a_persistently_failing_retry_backs_off_without_recounting(self) -> None: + fixture = CollapseFixture() + target = _hash(1) + fixture.seed([target]) + fixture.service.retry_initial_seconds = 0.5 + fixture.service.retry_max_seconds = 2.0 + fixture.server._clear_block_candidate_retry_state = ( + lambda block_hash: (_ for _ in ()).throw(RuntimeError("boom")) + ) + with redirect_stdout(StringIO()): + fixture.collapse() + self.assertEqual(fixture.counts()["cleanup_failed"], 1) + for attempt, delay in ((1, 1.0), (2, 2.0), (3, 2.0)): + self.assertTrue(fixture.retry_cleanup(force_due=True)[0]) + self.assertEqual(fixture.counts()["cleanup_retry_failed"], attempt) + record = fixture.service._block_candidate_collapse_cleanup_retries[ + target + ] + self.assertEqual(record.attempts, attempt) + self.assertEqual(record.delay_seconds, delay) + # Attempts never inflate the affected-hash series. + counts = fixture.counts() + self.assertEqual(counts["cleanup_failed"], 1) + self.assertEqual(counts["cleanup_recovered"], 0) + self.assertEqual(set(fixture.cleanup_backlog()), {target}) + # A record parked behind its own backoff is not due. + self.assertFalse(fixture.retry_cleanup()[0]) + + def test_the_backlog_only_ever_holds_fixed_step_labels(self) -> None: + fixture = CollapseFixture() + hashes = [_hash(1), _hash(2)] + fixture.seed(hashes) + fixture.service._collapsed_candidate_floor_holders = ( + lambda abandoned: (_ for _ in ()).throw(RuntimeError("boom")) + ) + with redirect_stdout(StringIO()): + fixture.collapse() + backlog = fixture.cleanup_backlog() + self.assertEqual(set(backlog), set(hashes)) + for steps in backlog.values(): + self.assertLessEqual( + steps, + frozenset(BLOCK_CANDIDATE_COLLAPSE_CLEANUP_STEPS), + ) + # A step name outside the closed set is refused, not stored. + with redirect_stdout(StringIO()): + fixture.service._defer_collapsed_candidate_cleanup( + _hash(9), + ("not-a-cleanup-step",), + ) + self.assertNotIn(_hash(9), fixture.cleanup_backlog()) + self.assertEqual( + set(fixture.counts()), + set(PRISM_BLOCK_CANDIDATE_COLLAPSE_OUTCOMES), + ) + + def test_the_accounting_lane_drives_the_deferred_cleanup(self) -> None: + """The retry must reach a shipped loop, not just a direct call.""" + fixture = CollapseFixture() + target = _hash(1) + fixture.seed([target]) + server = fixture.server + broken = self._breaker(server, "_clear_block_candidate_retry_state") + with redirect_stdout(StringIO()): + fixture.collapse() + self.assertEqual(set(fixture.cleanup_backlog()), {target}) + broken["on"] = False + service = fixture.service + service._ensure_block_accounting_state() + thread = threading.Thread(target=service.block_accounting_loop, daemon=True) + with redirect_stdout(StringIO()): + thread.start() + try: + # Bounded: the lane either drains the backlog quickly or the + # deadline expires and the assertions below report it. + deadline = time.monotonic() + 5.0 + while fixture.cleanup_backlog() and time.monotonic() < deadline: + time.sleep(0.01) + finally: + server.stop_event.set() + thread.join(timeout=5.0) + self.assertFalse(thread.is_alive()) + self.assertEqual(fixture.cleanup_backlog(), {}) + self.assertEqual(fixture.counts()["cleanup_recovered"], 1) + + +class BlockCandidateCollapseCleanupCadenceTests(unittest.TestCase): + """The accounting lane must reach a due cleanup while it is busy. + + A collapsed row whose cleanup failed has no durable replay source, so + the accounting lane is its only driver. Driving it purely from that + lane's idle branch made the retry hostage to the lane going idle: + sustained ordinary accounting traffic, or a continuously replenished + invalid-candidate quarantine queue, starved it for as long as the + traffic lasted. These tests own the explicit work-item cadence that + closes the gap in both directions -- the cleanup reaches its record + within a bounded number of work items, and the cleanup backlog in turn + can spend no more than one attempt per cadence, so it cannot starve + accounting or quarantine in return. + """ + + def _deferred(self, fixture: CollapseFixture, hashes: list[str]) -> dict: + """Collapse ``hashes`` with one cleanup step broken; return the switch.""" + fixture.seed(hashes) + broken = _break_cleanup_step( + fixture.server, + "_clear_block_candidate_retry_state", + ) + with redirect_stdout(StringIO()): + fixture.collapse() + self.assertEqual(set(fixture.cleanup_backlog()), set(hashes)) + return broken + + def test_the_cadence_bound_is_the_shipped_default(self) -> None: + fixture = CollapseFixture() + self.assertEqual( + fixture.service._block_accounting_cleanup_retry_work_items(), + DEFAULT_BLOCK_ACCOUNTING_CLEANUP_RETRY_WORK_ITEMS, + ) + self.assertGreaterEqual(DEFAULT_BLOCK_ACCOUNTING_CLEANUP_RETRY_WORK_ITEMS, 1) + + def test_a_non_positive_cadence_offers_after_every_work_item(self) -> None: + """A misconfigured bound degrades to offering more, never never.""" + fixture = CollapseFixture() + fixture.server.block_accounting_cleanup_retry_work_items = 0 + self.assertEqual( + fixture.service._block_accounting_cleanup_retry_work_items(), + 1, + ) + broken = self._deferred(fixture, [_hash(1)]) + broken["on"] = False + rig = AccountingLaneRig(fixture, AccountingLaneRig.ACCOUNTING) + self.assertTrue(rig.run_until(lambda: not fixture.cleanup_backlog())) + self.assertLessEqual(rig.offers[0][0], 1) + + # -- starvation by ordinary accounting traffic ------------------------- + + def test_a_due_cleanup_runs_while_the_accounting_queues_stay_busy(self) -> None: + """The handoff queues never empty, so only the cadence can run it.""" + fixture = CollapseFixture() + target = _hash(1) + broken = self._deferred(fixture, [target]) + broken["on"] = False + bound = fixture.service._block_accounting_cleanup_retry_work_items() + rig = AccountingLaneRig(fixture, AccountingLaneRig.ACCOUNTING) + self.assertTrue( + rig.run_until(lambda: not fixture.cleanup_backlog()), + f"cleanup starved behind accounting traffic; log={rig.log}", + ) + self.assertFalse(rig.thread_alive) + self.assertEqual(fixture.cleanup_backlog(), {}) + self.assertEqual(fixture.counts()["cleanup_recovered"], 1) + # The lane was never idle: every offer came from the cadence, and + # the first one landed inside the documented bound. + self.assertGreater(rig.accounting_items, 0) + self.assertEqual(rig.quarantine_items, 0) + self.assertLessEqual(rig.offers[0][0], bound) + + # -- starvation by invalid-candidate quarantine traffic ---------------- + + def test_a_due_cleanup_runs_while_quarantine_is_replenished(self) -> None: + """Quarantine always reports work, so the idle branch never runs.""" + fixture = CollapseFixture() + target = _hash(1) + broken = self._deferred(fixture, [target]) + broken["on"] = False + bound = fixture.service._block_accounting_cleanup_retry_work_items() + rig = AccountingLaneRig(fixture, AccountingLaneRig.QUARANTINE) + self.assertTrue( + rig.run_until(lambda: not fixture.cleanup_backlog()), + f"cleanup starved behind quarantine traffic; log={rig.log}", + ) + self.assertFalse(rig.thread_alive) + self.assertEqual(fixture.cleanup_backlog(), {}) + self.assertEqual(fixture.counts()["cleanup_recovered"], 1) + self.assertEqual(rig.accounting_items, 0) + self.assertGreater(rig.quarantine_items, 0) + self.assertLessEqual(rig.offers[0][1], bound) + + # -- the cadence cannot be starved *by* the cleanup -------------------- + + def test_a_cleanup_backlog_spends_one_attempt_per_cadence(self) -> None: + """A permanently failing backlog must not take the lane over.""" + fixture = CollapseFixture() + hashes = [_hash(index) for index in range(1, 6)] + self._deferred(fixture, hashes) + # retry_initial_seconds is 0 in this fixture, so the doubled backoff + # stays 0 and every record in the backlog is due on every offer. + self.assertEqual(fixture.service.retry_initial_seconds, 0.0) + bound = fixture.service._block_accounting_cleanup_retry_work_items() + rig = AccountingLaneRig(fixture, AccountingLaneRig.BOTH) + self.assertTrue(rig.run_until(lambda: rig.work_items >= 20 * bound)) + self.assertFalse(rig.thread_alive) + # One offer per completed cadence, and one attempt per offer at most. + self.assertEqual(len(rig.offers), rig.work_items // bound) + self.assertLessEqual(rig.attempts, rig.work_items // bound) + self.assertEqual(fixture.counts()["cleanup_retry_failed"], rig.attempts) + # Both lanes kept draining throughout; neither was taken over. + self.assertGreaterEqual(rig.accounting_items, 5 * bound) + self.assertGreaterEqual(rig.quarantine_items, 5 * bound) + # Round-robin re-registration keeps the whole backlog owed. + self.assertEqual(set(fixture.cleanup_backlog()), set(hashes)) + self.assertEqual(fixture.counts()["cleanup_recovered"], 0) + + # -- the cadence offers, the record's own deadline decides ------------- + + def test_a_record_behind_its_backoff_is_offered_but_not_run(self) -> None: + fixture = CollapseFixture() + target = _hash(1) + self._deferred(fixture, [target]) + registry = fixture.service._block_candidate_collapse_cleanup_retries + with fixture.server.lock: + registry[target].not_before_monotonic = time.monotonic() + 3600.0 + bound = fixture.service._block_accounting_cleanup_retry_work_items() + rig = AccountingLaneRig(fixture, AccountingLaneRig.ACCOUNTING) + self.assertTrue(rig.run_until(lambda: len(rig.offers) >= 3)) + self.assertFalse(rig.thread_alive) + # Offered on cadence... + self.assertLessEqual(rig.offers[0][0], bound) + # ...and refused by the record's own deadline, not run early. + self.assertEqual(rig.attempts, 0) + self.assertEqual(registry[target].attempts, 0) + self.assertEqual(set(fixture.cleanup_backlog()), {target}) + counts = fixture.counts() + self.assertEqual(counts["cleanup_retry_failed"], 0) + self.assertEqual(counts["cleanup_recovered"], 0) + + # -- the pre-existing idle path is unchanged --------------------------- + + def test_a_truly_idle_lane_still_runs_the_cleanup_immediately(self) -> None: + """The idle branch must not wait for a cadence's worth of items.""" + fixture = CollapseFixture() + target = _hash(1) + broken = self._deferred(fixture, [target]) + broken["on"] = False + rig = AccountingLaneRig(fixture, AccountingLaneRig.IDLE) + self.assertTrue( + rig.run_until(lambda: not fixture.cleanup_backlog()), + f"idle lane did not finish the cleanup; log={rig.log}", + ) + self.assertFalse(rig.thread_alive) + self.assertEqual(fixture.cleanup_backlog(), {}) + self.assertEqual(fixture.counts()["cleanup_recovered"], 1) + # No work item of either kind was needed. + self.assertEqual(rig.accounting_items, 0) + self.assertEqual(rig.quarantine_items, 0) + self.assertEqual(rig.offers[0], (0, 0)) + + # -- regression proof for the cadence itself --------------------------- + + def test_without_the_cadence_both_lanes_starve_the_cleanup(self) -> None: + """Neutralising the cadence must break the two starvation tests. + + Raising the bound out of reach removes every cadence offer, which is + exactly what deleting the cadence calls does to the loop: in both + busy modes the idle branch is unreachable, so nothing is left to + drive the retry. If this test ever stops starving, the two tests + above have stopped proving the cadence. + """ + bound = DEFAULT_BLOCK_ACCOUNTING_CLEANUP_RETRY_WORK_ITEMS + for mode in (AccountingLaneRig.ACCOUNTING, AccountingLaneRig.QUARANTINE): + with self.subTest(mode=mode): + fixture = CollapseFixture() + target = _hash(1) + broken = self._deferred(fixture, [target]) + broken["on"] = False + fixture.server.block_accounting_cleanup_retry_work_items = 1 << 30 + rig = AccountingLaneRig(fixture, mode) + rig.run_until(lambda: rig.work_items >= 50 * bound) + self.assertFalse(rig.thread_alive) + self.assertGreaterEqual(rig.work_items, 50 * bound) + self.assertEqual(rig.offers, []) + self.assertEqual(rig.attempts, 0) + self.assertEqual(set(fixture.cleanup_backlog()), {target}) + self.assertEqual(fixture.counts()["cleanup_recovered"], 0) + + +class BlockCandidateCollapseSubmissionFenceTests(unittest.TestCase): + """A won row is fenced from the node by the write, not by its cleanup. + + The fenced batch write is the moment a row becomes durably terminal, but + the in-memory fence the rest of the submitter honours -- the recorded + terminal outcome -- used to be installed only by the ``terminal-outcome`` + cleanup step. A cleanup that failed at that step, or a page-level abort + that never reached it, therefore left a same-hash candidate in the live, + replay, retry, or waiting lane free to be dequeued the moment the apply + released its disposition lease: ``submit_next`` found no terminal + outcome, consulted nothing else, and offered qbitd a block whose durable + row was already abandoned. + + These tests own the invariant that closes it. The apply publishes the + false terminal outcome for the exact returned hash set while every won + hash's lease is still held and before any cleanup runs, so the fence is + up from the write onwards and no cleanup outcome -- success, contained + per-step failure, page-level abort, or a retry that keeps failing -- can + make a won row offerable again. + """ + + _breaker = staticmethod(BlockCandidateCollapseCleanupRetryTests._breaker) + + # -- harness ----------------------------------------------------------- + + @staticmethod + def _arm_node(fixture: CollapseFixture) -> None: + """Let a node offer succeed, so an escape is loud rather than lost. + + With ``submitblock`` erroring, a fence regression could be mistaken + for an ordinary transport failure; answering it makes the offer land + exactly as it would in production. + """ + fixture.rpc.results["submitblock"] = None + fixture.server.block_submit_rpc_timeout_seconds = 0.5 + + @staticmethod + def _offers(fixture: CollapseFixture) -> int: + return sum(1 for method, _ in fixture.rpc.calls if method == "submitblock") + + @staticmethod + def _queue( + fixture: CollapseFixture, + candidate: Any, + *, + replay: bool = False, + ) -> None: + service = fixture.service + service._ensure_block_replay_state() + if replay: + service._block_replay_candidate_queue.put_nowait(candidate) + else: + service.candidate_queue.put_nowait(candidate) + + @staticmethod + def _submit(fixture: CollapseFixture, *, defer_accounting: bool = True) -> bool: + """Drive the shipped queue-to-node path exactly as the loop does.""" + with redirect_stdout(StringIO()): + return fixture.server.submit_next_block_candidate( + defer_accounting=defer_accounting + ) + + def _seed(self, fixture: CollapseFixture, target: str) -> Any: + """One durable row whose credit-bearing candidate owns a floor holder.""" + candidate = fixture.seed([target], credit_share_on_accept=True)[0] + fixture.server._ensure_share_writer_service().adopt_pending_share( + candidate.pending_share + ) + self._arm_node(fixture) + return candidate + + def _seed_queued( + self, + fixture: CollapseFixture, + target: str, + *, + replay: bool = False, + ) -> Any: + candidate = self._seed(fixture, target) + self._queue(fixture, candidate, replay=replay) + return candidate + + def _assert_unofferable(self, fixture: CollapseFixture, target: str) -> None: + """Nothing was offered, nothing was resurrected, nothing re-adopted.""" + self.assertEqual(self._offers(fixture), 0) + self.assertEqual(fixture.pending(), set()) + self.assertEqual(len(fixture.ledger.abandon_calls), 1) + service = fixture.service + self.assertTrue(service.candidate_queue.empty()) + self.assertTrue(service._block_replay_candidate_queue.empty()) + with fixture.server.lock: + self.assertIsNone(service.retry_candidate) + self.assertNotIn(target, service._block_replay_inflight_hashes) + self.assertNotIn(target, service._block_disposition_waiting_retries) + + # -- the two cleanup outcomes that used to unfence a won row ----------- + + def test_a_failed_terminal_outcome_still_fences_the_queued_candidate( + self, + ) -> None: + """The P1: the step that publishes the fence is the step that failed.""" + fixture = CollapseFixture() + target = _hash(1) + candidate = self._seed_queued(fixture, target) + broken = self._breaker( + fixture.server, + "_record_block_candidate_terminal_outcome", + ) + with redirect_stdout(StringIO()): + fixture.collapse() + # Exactly the failure this fence exists for: the durable row is gone + # and its terminal cleanup is still owed, with the candidate object + # still sitting in the live queue. + self.assertEqual(fixture.pending(), set()) + self.assertEqual( + fixture.cleanup_backlog(), + {target: frozenset({"terminal-outcome"})}, + ) + self.assertFalse(fixture.service.candidate_queue.empty()) + # The submitter reaches the candidate before the accounting lane + # reaches the retry -- the ordering the incident actually produced. + self.assertTrue(self._submit(fixture)) + self._assert_unofferable(fixture, target) + # Only now does the retry finish the step that failed. + broken["on"] = False + self.assertTrue(fixture.retry_cleanup()[0]) + self.assertEqual(fixture.cleanup_backlog(), {}) + self.assertEqual(self._offers(fixture), 0) + self.assertNotIn(id(candidate.pending_share), fixture.floor()) + counts = fixture.counts() + self.assertEqual(counts["cleanup_failed"], 1) + self.assertEqual(counts["cleanup_recovered"], 1) + + def test_an_aborted_cleanup_still_fences_the_queued_candidate(self) -> None: + """A page-level abort never reaches the terminal-outcome step at all.""" + fixture = CollapseFixture() + target = _hash(1) + candidate = self._seed_queued(fixture, target) + scan = fixture.service._collapsed_candidate_floor_holders + fixture.service._collapsed_candidate_floor_holders = ( + lambda abandoned: (_ for _ in ()).throw(RuntimeError("index boom")) + ) + with redirect_stdout(StringIO()): + fixture.collapse() + self.assertEqual( + fixture.cleanup_backlog(), + {target: frozenset(BLOCK_CANDIDATE_COLLAPSE_CLEANUP_STEPS)}, + ) + # The abort ran nothing past the withdrawal, so the queued object + # still owns its identity-keyed floor holder. + self.assertIn(id(candidate.pending_share), fixture.floor()) + self.assertTrue(self._submit(fixture)) + self._assert_unofferable(fixture, target) + # The dropped duplicate released its own holder on the way out, which + # is what lets the retry re-scan the drained queues and lose nothing. + self.assertNotIn(id(candidate.pending_share), fixture.floor()) + fixture.service._collapsed_candidate_floor_holders = scan + self.assertTrue(fixture.retry_cleanup()[0]) + self.assertEqual(fixture.cleanup_backlog(), {}) + self.assertEqual(self._offers(fixture), 0) + + # -- lane ownership ---------------------------------------------------- + + def test_the_replay_lane_is_fenced_by_the_same_published_outcome(self) -> None: + """Durable replay dequeues through the same guard as a live solve.""" + fixture = CollapseFixture() + target = _hash(1) + self._seed_queued(fixture, target, replay=True) + service = fixture.service + with fixture.server.lock: + service._block_replay_inflight_hashes.add(target) + broken = self._breaker( + fixture.server, + "_record_block_candidate_terminal_outcome", + ) + with redirect_stdout(StringIO()): + fixture.collapse() + self.assertFalse(service._block_replay_candidate_queue.empty()) + self.assertTrue(self._submit(fixture)) + self.assertEqual(self._offers(fixture), 0) + self.assertTrue(service._block_replay_candidate_queue.empty()) + self.assertEqual(fixture.pending(), set()) + # A freshly decoded same-hash replay object is refused admission by + # the same published outcome rather than queued behind it. + sibling = fixture.seed([target], credit_share_on_accept=True)[0] + with redirect_stdout(StringIO()): + self.assertFalse(service._enqueue_replayed_block_candidate(sibling)) + self.assertTrue(service._block_replay_candidate_queue.empty()) + self.assertEqual(self._offers(fixture), 0) + broken["on"] = False + self.assertTrue(fixture.retry_cleanup()[0]) + self.assertEqual(fixture.cleanup_backlog(), {}) + + def test_the_waiting_retry_lane_is_fenced_when_the_lease_releases(self) -> None: + """A wakeup parked behind the apply's own lease must not be offered.""" + fixture = CollapseFixture() + target = _hash(1) + candidate = self._seed(fixture, target) + service = fixture.service + inner = fixture.ledger.inner.mark_block_candidates_abandoned + + def park_then_write(requested: tuple[str, ...], error: str) -> Any: + # A parked wakeup is collapse evidence, so the only ordering that + # produces one for a won hash is this: the submitter dequeued the + # hash, could not claim the apply's lease, and parked it after the + # pre-write revalidation re-read the evidence and before the + # fenced write landed. + with fixture.server.lock: + service._block_disposition_waiting_retries[target] = candidate + return inner(block_hashes=requested, error=error) + + fixture.ledger.abandon_hook = park_then_write + broken = self._breaker( + fixture.server, + "_record_block_candidate_terminal_outcome", + ) + with redirect_stdout(StringIO()): + fixture.collapse() + # The failed step is also the one that drops parked wakeups, so the + # candidate is still parked and will be dequeued from there. + with fixture.server.lock: + self.assertIn(target, service._block_disposition_waiting_retries) + self.assertTrue(self._submit(fixture)) + self._assert_unofferable(fixture, target) + broken["on"] = False + self.assertTrue(fixture.retry_cleanup()[0]) + self.assertEqual(fixture.cleanup_backlog(), {}) + + # -- the non-queue seams ------------------------------------------------ + + def test_the_direct_writer_seam_refuses_a_fenced_hash(self) -> None: + """The historical direct entrypoint takes the guard and answers False.""" + fixture = CollapseFixture() + target = _hash(1) + candidate = self._seed_queued(fixture, target) + fixture.service.candidate_queue.get_nowait() + broken = self._breaker( + fixture.server, + "_record_block_candidate_terminal_outcome", + ) + with redirect_stdout(StringIO()): + fixture.collapse() + with redirect_stdout(StringIO()): + landed = fixture.server._submit_next_block_candidate_writer(candidate) + self.assertFalse(landed) + self.assertEqual(self._offers(fixture), 0) + self.assertEqual(fixture.pending(), set()) + self.assertEqual(len(fixture.ledger.abandon_calls), 1) + broken["on"] = False + self.assertTrue(fixture.retry_cleanup()[0]) + + def test_the_synchronous_seam_refuses_a_fenced_hash(self) -> None: + """The miner-facing resubmit joins the same terminal disposition.""" + fixture = CollapseFixture() + target = _hash(1) + candidate = self._seed_queued(fixture, target) + fixture.service.candidate_queue.get_nowait() + broken = self._breaker( + fixture.server, + "_record_block_candidate_terminal_outcome", + ) + with redirect_stdout(StringIO()): + fixture.collapse() + with redirect_stdout(StringIO()): + landed = fixture.server._submit_synchronous_block_candidate(candidate) + self.assertFalse(landed) + self.assertEqual(self._offers(fixture), 0) + self.assertEqual(fixture.pending(), set()) + self.assertNotIn(id(candidate.pending_share), fixture.floor()) + broken["on"] = False + self.assertTrue(fixture.retry_cleanup()[0]) + + # -- durability of the fence across the retry series ------------------- + + def test_the_fence_survives_every_retry_failure_and_its_discharge(self) -> None: + """Repeated failures, then recovery, and never one node offer.""" + fixture = CollapseFixture() + target = _hash(1) + candidate = self._seed(fixture, target) + fixture.service.retry_initial_seconds = 0.5 + fixture.service.retry_max_seconds = 2.0 + broken = self._breaker( + fixture.server, + "_record_block_candidate_terminal_outcome", + ) + with redirect_stdout(StringIO()): + fixture.collapse() + for attempt in (1, 2, 3): + self.assertTrue(fixture.retry_cleanup(force_due=True)[0]) + self.assertEqual(fixture.counts()["cleanup_retry_failed"], attempt) + self.assertEqual(set(fixture.cleanup_backlog()), {target}) + # The fence is unaffected by the attempt that just failed: the + # same-hash wakeup is dropped, not offered, on every round. + self._queue(fixture, candidate) + self.assertTrue(self._submit(fixture)) + self._assert_unofferable(fixture, target) + broken["on"] = False + self.assertTrue(fixture.retry_cleanup(force_due=True)[0]) + self.assertEqual(fixture.cleanup_backlog(), {}) + # Discharging the backlog leaves the ordinary terminal fence behind, + # so the durably terminal row is still never re-offered. + self._queue(fixture, candidate) + self.assertTrue(self._submit(fixture)) + self._assert_unofferable(fixture, target) + with fixture.server.lock: + self.assertIs( + fixture.service._block_candidate_terminal_outcomes[target], + False, + ) + # Accounting stayed bounded by the affected hashes, and the label + # space is still the closed outcome set. + counts = fixture.counts() + self.assertEqual(counts["cleanup_failed"], 1) + self.assertEqual(counts["cleanup_recovered"], 1) + self.assertEqual(counts["cleanup_retry_failed"], 3) + self.assertLessEqual(counts["cleanup_recovered"], counts["cleanup_failed"]) + self.assertEqual( + set(counts), + set(PRISM_BLOCK_CANDIDATE_COLLAPSE_OUTCOMES), + ) + + def test_a_clean_collapse_fences_its_queued_candidate_too(self) -> None: + """The fence is a property of the write, not of a cleanup failure.""" + fixture = CollapseFixture() + target = _hash(1) + self._seed_queued(fixture, target) + with redirect_stdout(StringIO()): + fixture.collapse() + self.assertEqual(fixture.cleanup_backlog(), {}) + self.assertEqual(fixture.counts()["cleanup_failed"], 0) + self.assertTrue(self._submit(fixture)) + self._assert_unofferable(fixture, target) + + def test_a_preserved_row_is_never_fenced_by_another_pages_win(self) -> None: + """Only the exact returned hash set is published, never the request.""" + fixture = CollapseFixture() + won, lost = _hash(1), _hash(2) + fixture.seed([won, lost], credit_share_on_accept=True) + fixture.ledger.abandon_hook = lambda requested, error: (won,) + with redirect_stdout(StringIO()): + fixture.collapse() + service = fixture.service + with fixture.server.lock: + outcomes = dict(service._block_candidate_terminal_outcomes) + self.assertEqual(outcomes, {won: False}) + self.assertEqual(fixture.counts()["write_lost"], 1) + + +class _CountingRegistry(dict): + """A registry that counts membership probes and reports being walked. + + The page-bounded evidence read has to answer from probes alone; a + regression that goes back to copying a registry into a set shows up here + as a walk (optionally a hard failure) rather than as a wall-clock wobble + that a fast enough machine hides. + """ + + def __init__(self, *args: Any, **kwargs: Any) -> None: + super().__init__(*args, **kwargs) + self.probes = 0 + self.walks = 0 + self.forbid_walks = False + + def __contains__(self, key: object) -> bool: + self.probes += 1 + return super().__contains__(key) + + def __iter__(self) -> Any: + self.walks += 1 + if self.forbid_walks: + raise AssertionError("terminal outcome registry was walked") + return super().__iter__() + + +def _history_hashes(count: int, *, start: int = 1 << 20) -> list[str]: + """``count`` distinct hashes standing in for already-disposed blocks.""" + return [f"{index:064x}" for index in range(start, start + count)] + + +class BlockCandidateCollapseEvidenceCostTests(unittest.TestCase): + """One evidence probe per hash the caller is deciding, and no more. + + ``_block_candidate_terminal_outcomes`` is the one registry the collapse + evidence read touches that grows with every block the process has ever + disposed: the 2026-08-20 storm left 312,000 entries behind, and every + poll folded all of them into a set while holding ``coordinator.lock`` + -- about 124 ms per page, linear in history, with every unrelated share + ack queued behind the same lock. The selector never needed the union: + it asks one question, "is this page's hash already offered", for at most + a page of hashes at a time. + + These tests pin that shape with a registry that counts its probes and + refuses to be walked, so the cost claim is a deterministic assertion + about work performed rather than a timing measurement. + """ + + HISTORY = 200_000 + + @staticmethod + def _install_history( + fixture: CollapseFixture, + size: int, + *, + terminal: Iterable[str] = (), + ) -> _CountingRegistry: + registry = _CountingRegistry( + (block_hash, False) for block_hash in _history_hashes(size) + ) + for block_hash in terminal: + registry[block_hash] = False + with fixture.server.lock: + fixture.service._block_candidate_terminal_outcomes = registry + registry.probes = 0 + registry.walks = 0 + return registry + + def test_evidence_answers_a_page_without_walking_the_history(self) -> None: + fixture = CollapseFixture() + page = [_hash(1), _hash(2)] + fixture.seed(page) + registry = self._install_history( + fixture, + self.HISTORY, + terminal=[_hash(2)], + ) + registry.forbid_walks = True + evidence = fixture.service._block_candidate_collapse_evidence(page) + # Semantics unchanged: the historical terminal outcome still speaks + # for its hash, it is simply asked about rather than enumerated. + self.assertEqual(evidence, frozenset({_hash(2)})) + self.assertEqual(registry.walks, 0) + self.assertEqual(registry.probes, len(page)) + + def test_selection_work_does_not_grow_with_the_history(self) -> None: + """The same page costs the same probes at 1,000 and at 200,000.""" + page = [_hash(index) for index in range(1, 6)] + probes: dict[int, int] = {} + for size in (1_000, self.HISTORY): + with self.subTest(history=size): + fixture = CollapseFixture() + fixture.seed(page) + rows = fixture.page() + registry = self._install_history(fixture, size) + registry.forbid_walks = True + chain = _BlockCandidateChainView(fixture.service) + selected = fixture.service._select_superseded_block_candidates( + rows, + chain, + ) + self.assertEqual( + {row.block_hash for row in selected}, + set(page), + ) + self.assertEqual(registry.walks, 0) + probes[size] = registry.probes + self.assertEqual(probes[1_000], probes[self.HISTORY]) + self.assertEqual(probes[self.HISTORY], len(page)) + + def test_revalidation_probes_only_the_leased_subset(self) -> None: + """The pre-write re-read is bounded by the leases, not by the page.""" + fixture = CollapseFixture() + page = [_hash(index) for index in range(1, 6)] + fixture.seed(page) + chain = _BlockCandidateChainView(fixture.service) + selected = fixture.service._select_superseded_block_candidates( + fixture.page(), + chain, + ) + leased = selected[:2] + registry = self._install_history(fixture, 1_000) + registry.forbid_walks = True + qualified, _tip = fixture.service._revalidate_superseded_block_candidates( + leased, + chain, + ) + self.assertEqual( + {row.block_hash for row in qualified}, + {row.block_hash for row in leased}, + ) + self.assertEqual(registry.walks, 0) + self.assertEqual(registry.probes, len(leased)) + + def test_a_collapse_over_a_large_history_bounds_it(self) -> None: + """The apply still wins its page, and leaves the registry bounded.""" + fixture = CollapseFixture() + page = [_hash(index) for index in range(1, 6)] + fixture.seed(page) + with fixture.server.lock: + fixture.service._block_candidate_terminal_outcomes = { + block_hash: False + for block_hash in _history_hashes(self.HISTORY) + } + self.assertEqual(_selected(fixture), set(page)) + outcomes = fixture.service._block_candidate_terminal_outcomes + self.assertEqual(len(outcomes), MAX_BLOCK_CANDIDATE_TERMINAL_OUTCOMES) + for block_hash in page: + self.assertIs(outcomes[block_hash], False) + + +class BlockCandidateTerminalOutcomeBoundTests(unittest.TestCase): + """The terminal-outcome registry is bounded, and never unfences a copy. + + The same-hash disposition guard used to remember every outcome for the + life of the process. That is what made the collapse evidence read + expensive, but it is also unbounded memory in its own right, so the + registry is trimmed oldest-first once it passes + ``MAX_BLOCK_CANDIDATE_TERMINAL_OUTCOMES``. + + Eviction is the dangerous half. A forgotten outcome reads exactly like + an outcome that never happened, so a same-hash candidate still sitting + in the live, replay, retry, parked, or quarantine lane would find no + fence and offer qbitd a block whose durable row is already terminal -- + the very escape ``_publish_collapsed_candidate_terminal_fence`` exists + to close. Every lane that can still be holding such a copy therefore + pins its hash, and these tests own that pin, lane by lane. + """ + + CAP = MAX_BLOCK_CANDIDATE_TERMINAL_OUTCOMES + + # -- harness ----------------------------------------------------------- + + @staticmethod + def _fill(fixture: CollapseFixture, count: int) -> None: + """Accumulate historical outcomes behind whatever is already there.""" + with fixture.server.lock: + outcomes = fixture.service._block_candidate_terminal_outcomes + for block_hash in _history_hashes(count): + outcomes[block_hash] = False + + @staticmethod + def _record(fixture: CollapseFixture, count: int) -> None: + """Drive ``count`` real terminal outcomes, each one a trim occasion.""" + for block_hash in _history_hashes(count, start=1 << 40): + fixture.server._record_block_candidate_terminal_outcome( + block_hash, + accepted=False, + ) + + def _pressure(self, fixture: CollapseFixture) -> None: + """Push the registry past its bound through the shipped writer.""" + self._fill(fixture, self.CAP) + self._record(fixture, 64) + + # -- the bound --------------------------------------------------------- + + def test_the_registry_stops_growing_at_its_bound(self) -> None: + fixture = CollapseFixture() + self._record(fixture, self.CAP + 512) + outcomes = fixture.service._block_candidate_terminal_outcomes + self.assertEqual(len(outcomes), self.CAP) + history = _history_hashes(self.CAP + 512, start=1 << 40) + # Oldest-first: the first outcomes are gone, the newest are kept. + self.assertIsNone( + fixture.server._block_candidate_terminal_outcome(history[0]) + ) + self.assertIs( + fixture.server._block_candidate_terminal_outcome(history[-1]), + False, + ) + + def test_a_re_recorded_outcome_is_treated_as_the_newest(self) -> None: + """Reasserting an outcome must not leave it at the eviction front.""" + fixture = CollapseFixture() + target = _hash(1) + fixture.server._record_block_candidate_terminal_outcome( + target, + accepted=False, + ) + self._fill(fixture, 8) + fixture.server._record_block_candidate_terminal_outcome( + target, + accepted=False, + ) + outcomes = fixture.service._block_candidate_terminal_outcomes + self.assertEqual(list(outcomes)[-1], target) + + def test_pinned_entries_do_not_stall_the_trim(self) -> None: + """A run of pinned hashes longer than the scan window still drains.""" + fixture = CollapseFixture() + pinned = _history_hashes( + BLOCK_CANDIDATE_TERMINAL_OUTCOME_EVICTION_SCAN * 2, + start=1 << 30, + ) + for block_hash in pinned: + fixture.server._record_block_candidate_terminal_outcome( + block_hash, + accepted=False, + ) + with fixture.server.lock: + fixture.service._block_replay_inflight_hashes.update(pinned) + self._pressure(fixture) + outcomes = fixture.service._block_candidate_terminal_outcomes + self.assertEqual(len(outcomes), self.CAP) + for block_hash in pinned: + self.assertIn(block_hash, outcomes) + + def test_a_fence_page_larger_than_the_bound_keeps_its_publication( + self, + ) -> None: + """A storm-sized fence never evicts the outcomes it just published.""" + fixture = CollapseFixture() + self._fill(fixture, self.CAP) + page = tuple(_history_hashes(self.CAP + 100, start=1 << 50)) + with redirect_stdout(StringIO()): + fixture.service._publish_collapsed_candidate_terminal_fence(page) + outcomes = fixture.service._block_candidate_terminal_outcomes + for block_hash in page: + self.assertIs(outcomes[block_hash], False) + # Only history was dropped; the registry stays over its bound + # rather than unfencing a row the apply is still cleaning up. + self.assertEqual(len(outcomes), len(page)) + + def test_the_trim_is_skipped_rather_than_waiting_on_a_lane(self) -> None: + """Nothing is evicted against a lane that could not be read.""" + fixture = CollapseFixture() + self._fill(fixture, self.CAP + 16) + service = fixture.service + with service.candidate_queue.mutex: + with fixture.server.lock: + self.assertEqual( + service._bound_block_candidate_terminal_outcomes(), + 0, + ) + self.assertEqual( + len(service._block_candidate_terminal_outcomes), + self.CAP + 16, + ) + with fixture.server.lock: + self.assertEqual( + service._bound_block_candidate_terminal_outcomes(), + 16, + ) + + # -- the pins ---------------------------------------------------------- + + def test_every_lane_that_holds_a_copy_pins_its_outcome(self) -> None: + """Each lane the fence protects keeps its hash out of the eviction.""" + target = _hash(1) + + def outstanding(fixture: CollapseFixture, candidate: Any) -> None: + fixture.server._register_outstanding_block_candidate(target) + + def replay_inflight(fixture: CollapseFixture, candidate: Any) -> None: + with fixture.server.lock: + fixture.service._block_replay_inflight_hashes.add(target) + + def quarantined(fixture: CollapseFixture, candidate: Any) -> None: + with fixture.server.lock: + fixture.service._block_quarantine_hashes.add(target) + + def fast_lane(fixture: CollapseFixture, candidate: Any) -> None: + with fixture.server.lock: + fixture.service._block_fast_lane_reservations.add(target) + + def waiting_retry(fixture: CollapseFixture, candidate: Any) -> None: + with fixture.server.lock: + fixture.service._block_disposition_waiting_retries[target] = ( + candidate + ) + + def finalize_retry(fixture: CollapseFixture, candidate: Any) -> None: + with fixture.server.lock: + fixture.service.finalize_retries[target] = (False, "boom") + + def retained_submission(fixture: CollapseFixture, candidate: Any) -> None: + fixture.service._stash_retained_block_candidate_node_submission( + target, + _BlockCandidateNodeSubmission(attempted=True, result=None), + ) + + def cleanup_retry(fixture: CollapseFixture, candidate: Any) -> None: + fixture.service._defer_collapsed_candidate_cleanup( + target, + frozenset({"terminal-outcome"}), + ) + + def retry_holder(fixture: CollapseFixture, candidate: Any) -> None: + with fixture.server.lock: + fixture.service.retry_candidate = candidate + + def deferred_retry_holder( + fixture: CollapseFixture, + candidate: Any, + ) -> None: + with fixture.server.lock: + fixture.service._block_accounting_deferred_retry_candidate = ( + candidate + ) + + def live_queue(fixture: CollapseFixture, candidate: Any) -> None: + fixture.service.candidate_queue.put_nowait(candidate) + + def replay_queue(fixture: CollapseFixture, candidate: Any) -> None: + fixture.service._block_replay_candidate_queue.put_nowait(candidate) + + def disposition_flight(fixture: CollapseFixture, candidate: Any) -> None: + lease = fixture.server._claim_block_candidate_disposition( + target, + blocking=False, + ) + self.assertIsNotNone(lease) + + def dequeued_pin(fixture: CollapseFixture, candidate: Any) -> None: + # The instant between leaving a lane and owning a flight. + with fixture.server.lock: + fixture.service._pin_dequeued_block_candidate_locked(candidate) + + lanes = { + "_outstanding_block_candidate_hashes": outstanding, + "_block_replay_inflight_hashes": replay_inflight, + "_block_quarantine_hashes": quarantined, + "_block_fast_lane_reservations": fast_lane, + "_block_disposition_waiting_retries": waiting_retry, + "finalize_retries": finalize_retry, + "_block_candidate_retained_node_submissions": retained_submission, + "_block_candidate_collapse_cleanup_retries": cleanup_retry, + "retry_candidate": retry_holder, + "_block_accounting_deferred_retry_candidate": deferred_retry_holder, + "candidate_queue": live_queue, + "_block_replay_candidate_queue": replay_queue, + "_block_candidate_disposition_flights": disposition_flight, + "_block_candidate_dequeued_hashes": dequeued_pin, + } + for name, install in lanes.items(): + with self.subTest(lane=name): + fixture = CollapseFixture() + candidate = fixture.seed([target])[0] + # The outcome is recorded first, exactly as a terminal + # disposition does, and the lane is populated afterwards: + # recording clears the reservation, replay, and waiting + # markers for its own hash by design. + fixture.server._record_block_candidate_terminal_outcome( + target, + accepted=False, + ) + with redirect_stdout(StringIO()): + install(fixture, candidate) + self._pressure(fixture) + outcomes = fixture.service._block_candidate_terminal_outcomes + self.assertEqual(len(outcomes), self.CAP) + self.assertIs( + fixture.server._block_candidate_terminal_outcome(target), + False, + ) + + def test_a_queued_duplicate_keeps_its_fence_under_eviction_pressure( + self, + ) -> None: + """The escape the fence closes must survive the registry's bound. + + A won row's cleanup clears the ownership markers -- outstanding, + replay-inflight -- for the hash it just terminalized, so a same-hash + candidate still queued behind the apply is named by no registry at + all. Its fence would be the oldest evictable entry in the registry + and the dequeue would then find nothing. The queue itself is read, + so it does not. + """ + fixture = CollapseFixture() + queued, forgotten = _hash(1), _hash(2) + candidate = fixture.seed([queued], credit_share_on_accept=True)[0] + fixture.seed([forgotten]) + fixture.server._ensure_share_writer_service().adopt_pending_share( + candidate.pending_share + ) + # A working node offer, so an escape lands as a real submitblock + # rather than being lost in a transport error. + fixture.rpc.results["submitblock"] = None + fixture.server.block_submit_rpc_timeout_seconds = 0.5 + fixture.service.candidate_queue.put_nowait(candidate) + with redirect_stdout(StringIO()): + fixture.collapse() + self.assertEqual(fixture.pending(), set()) + self.assertEqual(fixture.cleanup_backlog(), {}) + with fixture.server.lock: + service = fixture.service + self.assertNotIn( + queued, + service._outstanding_block_candidate_hashes, + ) + self.assertNotIn(queued, service._block_replay_inflight_hashes) + self._pressure(fixture) + # The unheld hash ages out; the queued copy's fence does not. + self.assertIsNone( + fixture.server._block_candidate_terminal_outcome(forgotten) + ) + self.assertIs( + fixture.server._block_candidate_terminal_outcome(queued), + False, + ) + with redirect_stdout(StringIO()): + self.assertTrue( + fixture.server.submit_next_block_candidate(defer_accounting=True) + ) + offers = [ + method for method, _ in fixture.rpc.calls if method == "submitblock" + ] + self.assertEqual(offers, []) + self.assertTrue(fixture.service.candidate_queue.empty()) + + # -- the dequeue-to-disposition pin ------------------------------------ + + WAIT_SECONDS = 10.0 + + def _park_claim( + self, + fixture: CollapseFixture, + target: str, + ) -> tuple[threading.Event, threading.Event]: + """Park the submitter inside the old dequeue-to-disposition gap. + + Wraps the disposition claim so the pass that dequeued ``target`` is + held after it has left its lane and before any flight exists for + it -- exactly where ``submit_next``'s local variable used to be the + only thing holding the candidate. The park is deterministic: only + the test releases it, after it has applied eviction pressure and + looked at the registry. + """ + service = fixture.service + original = service._claim_block_candidate_disposition + in_gap = threading.Event() + release = threading.Event() + self.addCleanup(release.set) + + def park_then_claim(block_hash: str, *, blocking: bool) -> Any: + if block_hash == target: + in_gap.set() + if not release.wait(self.WAIT_SECONDS): + raise AssertionError("the parked claim was never released") + return original(block_hash, blocking=blocking) + + service._claim_block_candidate_disposition = park_then_claim + return in_gap, release + + @staticmethod + def _submit_in_background( + fixture: CollapseFixture, + ) -> tuple[threading.Thread, list[Any]]: + """Run one submitter pass on its own thread; collect its result.""" + results: list[Any] = [] + + def run() -> None: + try: + results.append( + fixture.server.submit_next_block_candidate( + defer_accounting=True + ) + ) + except BaseException as error: # reported to the test, not raised here + results.append(error) + + thread = threading.Thread(target=run, daemon=True) + thread.start() + return thread, results + + @staticmethod + def _dequeue_lanes_empty(fixture: CollapseFixture, target: str) -> bool: + """Whether no dequeue source still holds a copy of ``target``.""" + service = fixture.service + with fixture.server.lock: + return ( + service.retry_candidate is None + and service.candidate_queue.empty() + and service._block_replay_candidate_queue.empty() + and target not in service._block_disposition_waiting_retries + ) + + def test_a_dequeued_duplicate_keeps_its_fence_across_the_claim_gap( + self, + ) -> None: + """Eviction inside the dequeue-to-disposition gap finds the hash pinned. + + Every lane pins its hash while a candidate sits in it, and the + disposition flight pins it from the claim onward. Between the two + the candidate used to be held only by ``submit_next``'s local + variable: the retry holder cleared, the queue entry consumed, or + the waiting entry popped, and no flight installed yet. An eviction + in that window dropped the oldest unpinned outcome -- this + duplicate's fence -- and the pass then found nothing and offered a + durably terminal block to the node. + + The dequeue now moves the hash into the dequeued-hash pin under the + same lock hold that empties the lane, and hands it to the flight + (or back to the waiting registry) under the lock again, so there is + no instant at which nothing names it. The barrier parks the pass + in the old gap, applies real eviction pressure from the other side, + and proves the fence survives -- for each of the four sources a + pass can dequeue from. + """ + target = _hash(1) + + def retry_holder(fixture: CollapseFixture, candidate: Any) -> None: + with fixture.server.lock: + fixture.service.retry_candidate = candidate + + def live_queue(fixture: CollapseFixture, candidate: Any) -> None: + fixture.service.candidate_queue.put_nowait(candidate) + + def replay_queue(fixture: CollapseFixture, candidate: Any) -> None: + fixture.service._block_replay_candidate_queue.put_nowait(candidate) + + def waiting_retry(fixture: CollapseFixture, candidate: Any) -> None: + with fixture.server.lock: + fixture.service._block_disposition_waiting_retries[target] = ( + candidate + ) + + sources = { + "retry_candidate": retry_holder, + "candidate_queue": live_queue, + "_block_replay_candidate_queue": replay_queue, + "_block_disposition_waiting_retries": waiting_retry, + } + for name, install in sources.items(): + with self.subTest(source=name): + fixture = CollapseFixture() + service = fixture.service + candidate = fixture.seed([target])[0] + # A working node offer, so an escape would land as a real + # submitblock rather than being lost in a transport error. + fixture.rpc.results["submitblock"] = None + fixture.server.block_submit_rpc_timeout_seconds = 0.5 + # Recorded first, exactly as a terminal disposition does; + # the duplicate arrives in its lane afterwards. + fixture.server._record_block_candidate_terminal_outcome( + target, + accepted=False, + ) + install(fixture, candidate) + # The duplicate's fence is now the oldest entry and the + # registry sits one over its bound: the next recorded + # outcome evicts exactly the oldest unpinned hash. + self._fill(fixture, self.CAP) + in_gap, release = self._park_claim(fixture, target) + with redirect_stdout(StringIO()): + thread, results = self._submit_in_background(fixture) + self.assertTrue(in_gap.wait(self.WAIT_SECONDS)) + # This is the old gap: the lane is empty, no flight + # exists, and the candidate object is the submitter's + # local variable. + self.assertTrue(self._dequeue_lanes_empty(fixture, target)) + with service._block_candidate_disposition_registry_lock: + self.assertNotIn( + target, + service._block_candidate_disposition_flights, + ) + self._record(fixture, 64) + # The fence survived the pressure; history did not. + self.assertIs( + fixture.server._block_candidate_terminal_outcome(target), + False, + ) + self.assertEqual( + len(service._block_candidate_terminal_outcomes), + self.CAP, + ) + # ...because the dequeue itself is what names the hash. + with fixture.server.lock: + self.assertEqual( + service._block_candidate_dequeued_hashes, + {target: 1}, + ) + release.set() + thread.join(self.WAIT_SECONDS) + self.assertFalse(thread.is_alive()) + self.assertEqual(results, [True]) + offers = [ + method + for method, _ in fixture.rpc.calls + if method == "submitblock" + ] + self.assertEqual(offers, []) + # The duplicate was dropped against its fence: nothing holds + # it, no pin outlived the pass, and the fence still stands. + self.assertTrue(self._dequeue_lanes_empty(fixture, target)) + with fixture.server.lock: + self.assertEqual(service._block_candidate_dequeued_hashes, {}) + with service._block_candidate_disposition_registry_lock: + self.assertEqual(service._block_candidate_disposition_flights, {}) + self.assertIs( + fixture.server._block_candidate_terminal_outcome(target), + False, + ) + + def test_the_dequeued_pin_never_outlives_the_claim(self) -> None: + """Every way the claim comes back hands the hash on and unpins it. + + A pin that leaked would be a hash the eviction could never reclaim + -- unbounded growth by another name -- and a pin dropped early is + the window the pin exists to close. A lease hands the hash to the + installed flight (covered above); a miss parks the candidate in the + waiting registry in the same critical section; an exception leaves + nothing behind because the candidate object leaves with it; and a + parked retry that is not yet due is never dequeued, so it is never + pinned. + """ + target = _hash(1) + + with self.subTest(outcome="lease-miss"): + fixture = CollapseFixture() + service = fixture.service + candidate = fixture.seed([target])[0] + # Another same-hash pass spans the offer-to-finalize region. + lease = fixture.server._claim_block_candidate_disposition( + target, + blocking=False, + ) + self.assertIsNotNone(lease) + self.addCleanup( + fixture.server._release_block_candidate_disposition, + lease, + ) + service.candidate_queue.put_nowait(candidate) + with redirect_stdout(StringIO()): + self.assertTrue( + fixture.server.submit_next_block_candidate( + defer_accounting=True + ) + ) + with fixture.server.lock: + self.assertIs( + service._block_disposition_waiting_retries.get(target), + candidate, + ) + self.assertEqual(service._block_candidate_dequeued_hashes, {}) + self.assertTrue(service.candidate_queue.empty()) + + with self.subTest(outcome="claim-raises"): + fixture = CollapseFixture() + service = fixture.service + candidate = fixture.seed([target])[0] + + def explode(block_hash: str, *, blocking: bool) -> Any: + raise RuntimeError("claim exploded") + + service._claim_block_candidate_disposition = explode + service.candidate_queue.put_nowait(candidate) + with redirect_stdout(StringIO()): + with self.assertRaises(RuntimeError): + fixture.server.submit_next_block_candidate( + defer_accounting=True + ) + with fixture.server.lock: + self.assertEqual(service._block_candidate_dequeued_hashes, {}) + with service._block_candidate_disposition_registry_lock: + self.assertEqual(service._block_candidate_disposition_flights, {}) + self.assertTrue(self._dequeue_lanes_empty(fixture, target)) + + with self.subTest(outcome="retry-not-due"): + fixture = CollapseFixture() + service = fixture.service + candidate = fixture.seed([target])[0] + with fixture.server.lock: + service.retry_candidate = candidate + service._block_candidate_retry_not_before = { + target: time.monotonic() + 60.0 + } + with redirect_stdout(StringIO()): + self.assertFalse( + fixture.server.submit_next_block_candidate( + defer_accounting=True + ) + ) + with fixture.server.lock: + self.assertIs(service.retry_candidate, candidate) + self.assertEqual(service._block_candidate_dequeued_hashes, {}) + + # -- the accounting lane's exception handoff --------------------------- + + @staticmethod + def _accounting_task( + fixture: CollapseFixture, + target: str, + candidate: Any, + ) -> Any: + """One queued accounting task owning ``target``'s disposition lease. + + Built exactly as the submitter's deferred handoff leaves it: the + node offer is done and the lease travels with the task until the + accounting lane releases it. + """ + server = fixture.server + lease = server._claim_block_candidate_disposition(target, blocking=False) + assert lease is not None + task = _BlockCandidateAccountingTask( + candidate=candidate, + node_submission=_BlockCandidateNodeSubmission( + attempted=True, + result=None, + ), + disposition_lease=lease, + ) + assert server._enqueue_block_accounting_task(task) + return task + + def _drain_accounting_lane( + self, + fixture: CollapseFixture, + *, + before_stop: Any = None, + ) -> None: + """Run the shipped accounting loop until its queues are drained. + + Bounded: the lane either finishes the queued task quickly or the + deadline expires and the assertions that follow report it. All three + handoff lanes are watched, because a task built with definitive node + acceptance is routed to the accepted lane rather than the primary + queue and a wait that only watched the primary would stop the loop + before that task ever ran. + """ + service = fixture.service + server = fixture.server + service._ensure_block_accounting_state() + + def unfinished() -> int: + return ( + service._block_accounting_accepted_queue.unfinished_tasks + + service._block_accounting_queue.unfinished_tasks + + service._block_accounting_overflow_queue.unfinished_tasks + ) + + thread = threading.Thread(target=service.block_accounting_loop, daemon=True) + with redirect_stdout(StringIO()): + thread.start() + try: + if before_stop is not None: + before_stop() + deadline = time.monotonic() + self.WAIT_SECONDS + while unfinished() and time.monotonic() < deadline: + time.sleep(0.01) + finally: + server.stop_event.set() + thread.join(self.WAIT_SECONDS) + self.assertFalse(thread.is_alive()) + self.assertEqual(unfinished(), 0) + + def test_an_accounting_failure_hands_its_candidate_off_without_a_gap( + self, + ) -> None: + """The lane's unexpected-failure retry leaves no unnamed instant. + + ``_run_block_accounting_task`` owns the task's disposition lease and + releases it in its ``finally``; ``block_accounting_loop`` used to + retain the candidate only afterwards, in its own catch. Between + the release and that retain the candidate was the loop's local + variable and nothing named its hash -- the dequeue gap above in its + accounting-lane form. The lane now retains while it still holds + the disposition, which routes the candidate into the accounting + owner's deferred holder; the ``finally`` then releases the lease + and merges that holder into the retry slot under + ``coordinator.lock``, exactly as it always did for a writer-side + retain. The loop's catch sees the task already retained and does + not retain it again. + + The barrier parks the lane the instant its lease release returns, + applies eviction pressure, and proves the fence survives; afterwards + the candidate sits in the retry slot exactly once, with one retry + counted and its floor holder intact. + """ + target = _hash(1) + fixture = CollapseFixture() + service, server = fixture.service, fixture.server + candidate = fixture.seed([target], credit_share_on_accept=True)[0] + server._ensure_share_writer_service().adopt_pending_share( + candidate.pending_share + ) + self.assertEqual(len(fixture.floor()), 1) + # The fence is the oldest entry and the registry sits one over its + # bound: the next recorded outcome evicts the oldest unpinned hash. + server._record_block_candidate_terminal_outcome(target, accepted=False) + self._fill(fixture, self.CAP) + writer_calls = [0] + + def exploding_writer( + candidate_arg: Any, + *, + node_submission: Any, + disposition_held: bool, + ) -> bool: + writer_calls[0] += 1 + raise RuntimeError("accounting writer exploded") + + server._call_block_candidate_writer = exploding_writer + task = self._accounting_task(fixture, target, candidate) + # Park the lane the instant its lease release returns. + in_gap = threading.Event() + release = threading.Event() + self.addCleanup(release.set) + original_release = service._release_block_candidate_disposition + + def release_then_park(lease: Any) -> None: + original_release(lease) + if lease.block_hash == target: + in_gap.set() + if not release.wait(self.WAIT_SECONDS): + raise AssertionError("the parked accounting lane was never released") + + service._release_block_candidate_disposition = release_then_park + + def in_the_gap() -> None: + try: + self.assertTrue(in_gap.wait(self.WAIT_SECONDS)) + # The old gap: the lease is gone and the loop's catch has + # not run; the candidate is a local variable of the lane. + with service._block_candidate_disposition_registry_lock: + self.assertNotIn( + target, + service._block_candidate_disposition_flights, + ) + self._record(fixture, 64) + # The fence survived the pressure; history did not. + self.assertIs( + server._block_candidate_terminal_outcome(target), + False, + ) + self.assertEqual( + len(service._block_candidate_terminal_outcomes), + self.CAP, + ) + # ...because the lane retained into the deferred holder + # before it let go of the lease. + with server.lock: + self.assertIs( + service._block_accounting_deferred_retry_candidate, + candidate, + ) + finally: + # A failed assertion must not leave the lane parked for the + # whole ceiling before the test can report it. + release.set() + + self._drain_accounting_lane(fixture, before_stop=in_the_gap) + self.assertEqual(writer_calls[0], 1) + self.assertTrue(task.retained_for_retry.is_set()) + # Handed off exactly once: in the retry slot, the deferred holder + # cleared, one retry counted, one floor holder, nothing offered. + with server.lock: + self.assertIs(service.retry_candidate, candidate) + self.assertIsNone(service._block_accounting_deferred_retry_candidate) + self.assertFalse( + getattr(service, "_block_accounting_holds_disposition", False) + ) + self.assertEqual(service.retries, 1) + self.assertIn(target, service._outstanding_block_candidate_hashes) + self.assertEqual(len(fixture.floor()), 1) + with service._block_candidate_disposition_registry_lock: + self.assertEqual(service._block_candidate_disposition_flights, {}) + self.assertIs(server._block_candidate_terminal_outcome(target), False) + offers = [ + method for method, _ in fixture.rpc.calls if method == "submitblock" + ] + self.assertEqual(offers, []) + + def test_a_failure_ahead_of_the_accounting_lane_is_still_retained_once( + self, + ) -> None: + """The loop's catch still retains when the lane itself never ran. + + The lane's own exception path marks the task once it has retained. + A failure that fires before the lane can -- here the coordinator + delegate raising ahead of it -- leaves the mark unset, and the loop + retains the candidate exactly as it always did: once. + """ + target = _hash(1) + fixture = CollapseFixture() + service, server = fixture.service, fixture.server + candidate = fixture.seed([target])[0] + task = self._accounting_task(fixture, target, candidate) + # The lane never runs, so nothing releases the task's lease; the + # test owns it from here. + self.addCleanup( + server._release_block_candidate_disposition, + task.disposition_lease, + ) + + def delegate_explodes(task_arg: Any) -> None: + raise RuntimeError("delegate exploded") + + server._run_block_accounting_task = delegate_explodes + self._drain_accounting_lane(fixture) + self.assertFalse(task.retained_for_retry.is_set()) + with server.lock: + self.assertIs(service.retry_candidate, candidate) + self.assertIsNone( + getattr(service, "_block_accounting_deferred_retry_candidate", None) + ) + self.assertEqual(service.retries, 1) + + +class BlockCandidateAbandonmentDedupBoundTests(unittest.TestCase): + """The counted-abandonment dedup set retires with the fence it guards. + + ``_counted_block_candidate_abandonments`` exists because one hash can + reach the terminal accounting path more than once: a collapse whose + cleanup failed retries every owed step, and a finalize failure freezes a + false finalize-only disposition that replays. Abandonment metrics count + candidates, not attempts, so the key deduplicates them. + + Nothing ever retired those keys. One key per candidate the process had + ever abandoned is the same unbounded history the terminal-outcome + registry was just bounded away from, and a collapsed storm writes them + 3,120 at a time. The key is now retired inside the eviction pass, in + the same ``coordinator.lock`` critical section that has already proved + the hash unpinned by every live and cleanup-owing lane, and only for a + hash whose fence that pass actually drops. + + Retiring one early is the dangerous half: a lane still owed its + accounting would count its candidate a second time and the abandoned + series would exceed the candidates it describes. Both writer orderings + have to survive that -- the direct finalize counts before it publishes + its terminal outcome, the collapse publishes first and counts during + cleanup -- so these tests own the bound and the pins in both. + """ + + CAP = MAX_BLOCK_CANDIDATE_TERMINAL_OUTCOMES + REASON = PRISM_BLOCK_CANDIDATE_COLLAPSE_REASON + STALE_CLASS = PRISM_BLOCK_CANDIDATE_COLLAPSE_STALE_JOB_CLASS + STORM = 3_120 + STORMS = 4 + # Well clear of the fixture's own hashes and of the pressure histories. + STORM_START = 1 << 60 + + # -- harness ----------------------------------------------------------- + + def _count(self, fixture: CollapseFixture, block_hash: str) -> None: + """One committed abandonment, with the outcome the collapse builds.""" + fixture.server._record_committed_block_candidate_abandonment( + block_hash, + SimpleNamespace(reason=self.REASON, stale_job_class=self.STALE_CLASS), + ) + + @staticmethod + def _fence(fixture: CollapseFixture, block_hash: str) -> None: + fixture.server._record_block_candidate_terminal_outcome( + block_hash, + accepted=False, + ) + + def _abandon( + self, + fixture: CollapseFixture, + block_hash: str, + *, + terminal_first: bool, + ) -> None: + """Drive one terminal abandonment in one of the two shipped orders. + + ``terminal_first`` is the collapse: the fenced write publishes the + terminal outcome and the cleanup's accounting step follows it. The + other order is ``finalize``, which counts as soon as the durable + outbox mark returns and records the process-local outcome at its + tail. + """ + if terminal_first: + self._fence(fixture, block_hash) + self._count(fixture, block_hash) + else: + self._count(fixture, block_hash) + self._fence(fixture, block_hash) + + @staticmethod + def _counted(fixture: CollapseFixture) -> set[str]: + with fixture.server.lock: + return set(fixture.service._counted_block_candidate_abandonments) + + @staticmethod + def _outcomes(fixture: CollapseFixture) -> set[str]: + with fixture.server.lock: + return set(fixture.service._block_candidate_terminal_outcomes) + + def _pressure(self, fixture: CollapseFixture, *, round_index: int = 0) -> None: + """Push the registry past its bound through the shipped writer. + + The history is uncounted, so its own eviction retires nothing; it is + here purely to make the pass reach whatever the test is protecting. + Each round uses a disjoint range so a second round is real pressure + rather than a re-stamp of the first. + """ + offset = round_index * (1 << 18) + with fixture.server.lock: + outcomes = fixture.service._block_candidate_terminal_outcomes + for block_hash in _history_hashes(self.CAP, start=(1 << 20) + offset): + outcomes[block_hash] = False + for block_hash in _history_hashes(64, start=(1 << 40) + offset): + self._fence(fixture, block_hash) + + # -- the bound --------------------------------------------------------- + + def test_repeated_storms_do_not_grow_the_dedup_history(self) -> None: + """Storm after storm, the dedup set tracks the fences, not the past.""" + for terminal_first in (False, True): + with self.subTest(terminal_first=terminal_first): + fixture = CollapseFixture() + sizes: list[int] = [] + for index in range(self.STORMS): + start = self.STORM_START + index * self.STORM + for block_hash in _history_hashes(self.STORM, start=start): + self._abandon( + fixture, + block_hash, + terminal_first=terminal_first, + ) + sizes.append(len(self._counted(fixture))) + # It stops growing where the fences do, and holds there + # however many further storms arrive. + self.assertEqual(sizes[-1], self.CAP) + self.assertEqual(sizes[-1], sizes[-2]) + self.assertLessEqual(max(sizes), self.CAP) + self.assertEqual(self._counted(fixture), self._outcomes(fixture)) + # Bounding it cost no accuracy: every candidate counted once. + counts = fixture.server.block_candidate_abandoned_counts + self.assertEqual(counts[self.REASON], self.STORM * self.STORMS) + self.assertEqual( + fixture.server.stale_job_abandon_counts[self.STALE_CLASS], + self.STORM * self.STORMS, + ) + + def test_a_collapse_page_retires_its_keys_when_its_fences_age_out( + self, + ) -> None: + """The shipped apply, not a hand-rolled ordering, ends up bounded.""" + fixture = CollapseFixture() + page = [_hash(index) for index in range(1, 5)] + fixture.seed(page) + self.assertEqual(_selected(fixture), set(page)) + self.assertEqual(fixture.cleanup_backlog(), {}) + self.assertEqual( + fixture.server.block_candidate_abandoned_counts[self.REASON], + len(page), + ) + self.assertEqual(self._counted(fixture), set(page)) + self._pressure(fixture) + for block_hash in page: + self.assertIsNone( + fixture.server._block_candidate_terminal_outcome(block_hash) + ) + self.assertEqual(self._counted(fixture), set()) + + def test_the_key_retires_with_its_fence_and_not_before(self) -> None: + """A pass that evicts nothing retires nothing.""" + fixture = CollapseFixture() + target = _hash(1) + self._abandon(fixture, target, terminal_first=True) + with fixture.server.lock: + self.assertEqual( + fixture.service._bound_block_candidate_terminal_outcomes(), + 0, + ) + self.assertEqual(self._counted(fixture), {target}) + self._pressure(fixture) + self.assertIsNone(fixture.server._block_candidate_terminal_outcome(target)) + self.assertEqual(self._counted(fixture), set()) + + # -- the two orderings ------------------------------------------------- + + def test_both_writer_orderings_converge_on_the_same_state(self) -> None: + """Count-then-fence and fence-then-count are indistinguishable.""" + states = [] + for terminal_first in (False, True): + fixture = CollapseFixture() + history = _history_hashes(self.CAP + 64, start=self.STORM_START) + for block_hash in history: + self._abandon(fixture, block_hash, terminal_first=terminal_first) + states.append( + ( + self._counted(fixture), + self._outcomes(fixture), + dict(fixture.server.block_candidate_abandoned_counts), + dict(fixture.server.stale_job_abandon_counts), + ) + ) + self.assertEqual(states[0], states[1]) + counted, outcomes, counts, stale = states[0] + self.assertEqual(counted, outcomes) + self.assertEqual(len(counted), self.CAP) + self.assertEqual(counts[self.REASON], self.CAP + 64) + self.assertEqual(stale[self.STALE_CLASS], self.CAP + 64) + + def test_a_repeated_disposition_is_still_counted_once(self) -> None: + """The dedup the key exists for survives in both orderings.""" + for terminal_first in (False, True): + with self.subTest(terminal_first=terminal_first): + fixture = CollapseFixture() + target = _hash(1) + for _attempt in range(3): + self._abandon(fixture, target, terminal_first=terminal_first) + self.assertEqual( + fixture.server.block_candidate_abandoned_counts[self.REASON], + 1, + ) + self.assertEqual(self._counted(fixture), {target}) + + def test_a_false_finalize_failure_stays_counted_before_any_outcome( + self, + ) -> None: + """The ambiguous branch counts with no fence to retire it by. + + ``finalize`` freezes a false finalize-only disposition when the + durable abandonment mark raises: it counts the candidate and leaves + the hash in ``finalize_retries`` without ever recording a terminal + outcome. There is nothing for the eviction pass to drop, and the + registry that pins it is itself bounded, so the key simply waits for + the replay that publishes its outcome. + """ + fixture = CollapseFixture() + target = _hash(1) + self._count(fixture, target) + with fixture.server.lock: + fixture.service.finalize_retries[target] = (False, "boom") + self._pressure(fixture) + self.assertIsNone(fixture.server._block_candidate_terminal_outcome(target)) + self.assertEqual(self._counted(fixture), {target}) + # The paced finalize-only replay re-enters the same accounting call. + self._count(fixture, target) + self.assertEqual( + fixture.server.block_candidate_abandoned_counts[self.REASON], + 1, + ) + # Its retry succeeds: the tail clears the registry and publishes the + # outcome, and only then can the key age out with it. + with fixture.server.lock: + fixture.service.finalize_retries.pop(target, None) + self._fence(fixture, target) + self._pressure(fixture, round_index=1) + self.assertIsNone(fixture.server._block_candidate_terminal_outcome(target)) + self.assertEqual(self._counted(fixture), set()) + + # -- the pins ---------------------------------------------------------- + + def test_pressure_cannot_forget_a_pinned_hash_and_double_count_it( + self, + ) -> None: + """Every lane that can still run the accounting keeps its key.""" + target = _hash(1) + + def live_queue(fixture: CollapseFixture, candidate: Any) -> None: + fixture.service.candidate_queue.put_nowait(candidate) + + def replay_queue(fixture: CollapseFixture, candidate: Any) -> None: + fixture.service._block_replay_candidate_queue.put_nowait(candidate) + + def cleanup_owed(fixture: CollapseFixture, candidate: Any) -> None: + fixture.service._defer_collapsed_candidate_cleanup( + target, + frozenset({"abandonment-accounting"}), + ) + + def finalize_retry(fixture: CollapseFixture, candidate: Any) -> None: + with fixture.server.lock: + fixture.service.finalize_retries[target] = (False, "boom") + + def disposition_flight(fixture: CollapseFixture, candidate: Any) -> None: + lease = fixture.server._claim_block_candidate_disposition( + target, + blocking=False, + ) + self.assertIsNotNone(lease) + self.addCleanup( + fixture.server._release_block_candidate_disposition, + lease, + ) + + def outstanding(fixture: CollapseFixture, candidate: Any) -> None: + fixture.server._register_outstanding_block_candidate(target) + + def retry_holder(fixture: CollapseFixture, candidate: Any) -> None: + with fixture.server.lock: + fixture.service.retry_candidate = candidate + + lanes = { + "candidate_queue": live_queue, + "_block_replay_candidate_queue": replay_queue, + "_block_candidate_collapse_cleanup_retries": cleanup_owed, + "finalize_retries": finalize_retry, + "_block_candidate_disposition_flights": disposition_flight, + "_outstanding_block_candidate_hashes": outstanding, + "retry_candidate": retry_holder, + } + for terminal_first in (False, True): + for name, install in lanes.items(): + with self.subTest(lane=name, terminal_first=terminal_first): + fixture = CollapseFixture() + candidate = fixture.seed([target])[0] + # The outcome is recorded before the lane is populated: + # recording clears the reservation, replay, and waiting + # markers for its own hash by design. + self._abandon(fixture, target, terminal_first=terminal_first) + with redirect_stdout(StringIO()): + install(fixture, candidate) + self._pressure(fixture) + self.assertEqual( + len(fixture.service._block_candidate_terminal_outcomes), + self.CAP, + ) + self.assertIs( + fixture.server._block_candidate_terminal_outcome(target), + False, + ) + self.assertIn(target, self._counted(fixture)) + # The lane now runs the accounting it was still owed. + # The surviving key is the only thing standing between + # that and a second count of one candidate. + self._count(fixture, target) + self.assertEqual( + fixture.server.block_candidate_abandoned_counts[ + self.REASON + ], + 1, + ) + + +class BlockCandidateCollapseEnumerationTests(unittest.TestCase): + """Both replay enumeration shapes collapse before they adopt.""" + + def _adopted(self, fixture: CollapseFixture) -> set[str]: + service = fixture.service + service._ensure_block_replay_state() + with fixture.server.lock: + return set(service._block_replay_inflight_hashes) + + def test_keyset_pagination_collapses_then_adopts_the_remainder(self) -> None: + fixture = CollapseFixture() + fixture.seed([_hash(index) for index in range(1, 6)]) + fixture.seed([_hash(index) for index in range(6, 9)], parent=OTHER_PARENT) + fixture.rpc.tip = OTHER_PARENT + queued, log = fixture.replay(enumeration_owed=True) + self.assertIn("pending block candidate enumeration page=1", log) + self.assertEqual(fixture.pending(), {_hash(6), _hash(7), _hash(8)}) + self.assertEqual( + self._adopted(fixture), + {_hash(6), _hash(7), _hash(8)}, + ) + self.assertEqual(queued, 3) + + def test_legacy_window_collapses_then_adopts_the_remainder(self) -> None: + fixture = CollapseFixture() + fixture.seed([_hash(index) for index in range(1, 6)]) + fixture.seed([_hash(index) for index in range(6, 9)], parent=OTHER_PARENT) + fixture.rpc.tip = OTHER_PARENT + # Enumeration is not owed, so replay_pending takes the widening + # window even against a cursor-capable ledger. + queued, log = fixture.replay(enumeration_owed=False) + self.assertNotIn("enumeration page=", log) + self.assertEqual(fixture.pending(), {_hash(6), _hash(7), _hash(8)}) + self.assertEqual( + self._adopted(fixture), + {_hash(6), _hash(7), _hash(8)}, + ) + self.assertEqual(queued, 3) + + def test_cursorless_ledger_collapses_then_adopts_the_remainder(self) -> None: + fixture = CollapseFixture(ledger=WindowCollapseLedger()) + fixture.seed([_hash(index) for index in range(1, 6)]) + fixture.seed([_hash(index) for index in range(6, 9)], parent=OTHER_PARENT) + fixture.rpc.tip = OTHER_PARENT + queued, log = fixture.replay(enumeration_owed=True) + self.assertNotIn("enumeration page=", log) + self.assertEqual(fixture.pending(), {_hash(6), _hash(7), _hash(8)}) + self.assertEqual(self._adopted(fixture), {_hash(6), _hash(7), _hash(8)}) + self.assertEqual(queued, 3) + + def test_a_collapsed_row_is_never_adopted_or_re_armed(self) -> None: + """A won row's fetched payload is a stale copy of a terminal row.""" + for owed in (True, False): + with self.subTest(enumeration_owed=owed): + fixture = CollapseFixture() + fixture.seed([_hash(1), _hash(2)]) + decoded: list[str] = [] + original = fixture.server.block_candidate_from_intent + + def record(intent): + decoded.append(str(intent["block_hash_hex"]).lower()) + return original(intent) + + fixture.server.block_candidate_from_intent = record + queued, _log = fixture.replay(enumeration_owed=owed) + self.assertEqual(queued, 0) + self.assertEqual(decoded, []) + self.assertEqual(self._adopted(fixture), set()) + with fixture.server._accepted_block_payout_preview_condition: + self.assertEqual( + fixture.server._accepted_block_payout_previews, + {}, + ) + + def test_a_partly_returned_request_still_adopts_every_other_row(self) -> None: + for owed in (True, False): + with self.subTest(enumeration_owed=owed): + fixture = CollapseFixture() + fixture.seed([_hash(1), _hash(2), _hash(3)]) + + def partial(hashes: tuple[str, ...], error: str) -> tuple[str, ...]: + won = tuple(value for value in hashes if value != _hash(2)) + return fixture.ledger.inner.mark_block_candidates_abandoned( + block_hashes=won, + error=error, + ) + + fixture.ledger.abandon_hook = partial + queued, _log = fixture.replay(enumeration_owed=owed) + self.assertEqual(fixture.pending(), {_hash(2)}) + self.assertEqual(self._adopted(fixture), {_hash(2)}) + self.assertEqual(queued, 1) + + def test_a_fail_closed_page_adopts_every_row(self) -> None: + for owed in (True, False): + with self.subTest(enumeration_owed=owed): + fixture = CollapseFixture() + fixture.seed([_hash(1), _hash(2), _hash(3)]) + fixture.rpc.failures.add("getblockcount") + queued, _log = fixture.replay(enumeration_owed=owed) + self.assertEqual( + fixture.pending(), + {_hash(1), _hash(2), _hash(3)}, + ) + self.assertEqual( + self._adopted(fixture), + {_hash(1), _hash(2), _hash(3)}, + ) + self.assertEqual(queued, 3) + + def test_a_failed_write_adopts_every_row(self) -> None: + for owed in (True, False): + with self.subTest(enumeration_owed=owed): + fixture = CollapseFixture() + fixture.seed([_hash(1), _hash(2)]) + fixture.ledger.abandon_hook = lambda hashes, error: ( + (_ for _ in ()).throw(RuntimeError("ledger down")) + ) + queued, _log = fixture.replay(enumeration_owed=owed) + self.assertEqual(fixture.pending(), {_hash(1), _hash(2)}) + self.assertEqual(self._adopted(fixture), {_hash(1), _hash(2)}) + self.assertEqual(queued, 2) + + def test_a_timed_out_write_that_lands_late_preserves_then_converges(self) -> None: + """The caller's deadline expires while the batch write is still out. + + This drives the real submitter ledger-call seam rather than raising + from the ledger hook: the batch write runs on the bounded worker + ``_run_block_submitter_ledger_call`` spawns, the coordinator-side + wait expires first and raises ``BlockSubmitterDatabaseTimeout``, and + the worker completes the write afterwards. The apply learns nothing + from a call that never answered it, so the whole page fails closed + and stays adoptable, and the write that lands later is still fenced + by the durable pool-block fact rather than by what the selector read + a page earlier. Whatever that write transitions, no cleanup and no + payout authority may follow from a hash the call did not return + synchronously. + """ + wait_seconds = 10.0 + fixture = CollapseFixture() + abandoned_late = _hash(1) + landed_meanwhile = _hash(2) + fixture.seed([abandoned_late, landed_meanwhile], credit_share_on_accept=True) + # Short enough to keep the test quick, wide enough that the page + # read ahead of it is never the call that expires. + fixture.server.block_submit_db_timeout_seconds = 0.25 + started = threading.Event() + release = threading.Event() + self.addCleanup(release.set) + + def stall_then_write(hashes: tuple[str, ...], error: str) -> tuple[str, ...]: + # Deterministic: the write cannot land before the deadline + # because only the test releases it, and it is released only + # after the timed-out apply has returned. + started.set() + if not release.wait(wait_seconds): + raise AssertionError("the stalled ledger call was never released") + return fixture.ledger.inner.mark_block_candidates_abandoned( + block_hashes=hashes, + error=error, + ) + + fixture.ledger.abandon_hook = stall_then_write + queued, log = fixture.replay(enumeration_owed=False) + self.assertTrue(started.is_set()) + self.assertIn("ledger phase timed out phase=collapse-superseded", log) + # Failed closed: every row is preserved, adopted, and left to the + # per-row path exactly as an outright write failure leaves it. + self.assertEqual(queued, 2) + self.assertEqual(fixture.pending(), {abandoned_late, landed_meanwhile}) + self.assertEqual(self._adopted(fixture), {abandoned_late, landed_meanwhile}) + counts = fixture.counts() + self.assertEqual(counts["fail_closed"], 2) + self.assertEqual(counts["abandoned"], 0) + self.assertEqual(counts["write_lost"], 0) + self.assertEqual(counts["cleanup_failed"], 0) + self.assertEqual( + fixture.service.block_ledger_call_class_metrics()["fast"][ + "timeouts_total" + ], + 1, + ) + with fixture.service._block_candidate_disposition_registry_lock: + self.assertEqual(fixture.service._block_candidate_disposition_flights, {}) + adopted_floor = set(fixture.floor()) + self.assertEqual(len(adopted_floor), 2) + with fixture.service._block_submitter_ledger_calls_lock: + stalled = [ + call + for key, call in fixture.service._block_submitter_ledger_calls.items() + if key[0] == "collapse-superseded" + ] + self.assertEqual(len(stalled), 1) + # One of the two rows really did land while the call was out. Only + # the durable pool-block row proves it, and the fenced write re-asks + # for that fact itself. + fixture.ledger.inner.persist_accepted_block( + block_hash=landed_meanwhile, + block_height=DECIDED_HEIGHT, + parent_hash=STORM_PARENT, + final_bundle={}, + audit_report={}, + ) + release.set() + self.assertTrue(stalled[0].done.wait(wait_seconds)) + self.assertEqual(len(fixture.ledger.abandon_calls), 1) + self.assertEqual( + set(fixture.ledger.abandon_calls[0][0]), + {abandoned_late, landed_meanwhile}, + ) + self.assertEqual(set(stalled[0].result), {abandoned_late}) + self.assertEqual(fixture.pending(), {landed_meanwhile}) + # Nothing was cleaned up for the hash the call never returned to + # anybody: its in-memory state still belongs to the per-row path. + server = fixture.server + with server.lock: + self.assertEqual(fixture.service._block_candidate_terminal_outcomes, {}) + self.assertEqual(server.block_candidate_abandoned_counts, {}) + self.assertEqual(set(fixture.floor()), adopted_floor) + # A later enumeration is idempotent: the terminal row is gone from + # the outbox, the landed one is excluded by its pool-block row, and + # no second fenced write is issued. + second_queued, _second_log = fixture.replay(enumeration_owed=True) + self.assertEqual(second_queued, 0) + self.assertEqual(len(fixture.ledger.abandon_calls), 1) + self.assertEqual(fixture.pending(), {landed_meanwhile}) + self.assertEqual(self._adopted(fixture), {abandoned_late, landed_meanwhile}) + counts = fixture.counts() + self.assertEqual(counts["selected"], 2) + self.assertEqual(counts["abandoned"], 0) + self.assertEqual(counts["cleanup_failed"], 0) + # The barrier replay armed is still only a barrier: no landed + # transition and no published preview were invented for a row whose + # fate the coordinator never observed. + with server._accepted_block_payout_preview_condition: + previews = dict(server._accepted_block_payout_previews) + self.assertEqual( + dict(server._invalidated_accepted_block_payout_previews), + {}, + ) + self.assertEqual(set(previews), {abandoned_late, landed_meanwhile}) + self.assertFalse(previews[abandoned_late].landed) + self.assertIsNone(previews[abandoned_late].preview) + self.assertFalse( + server._accepted_block_payout_transition_landed(abandoned_late) + ) + + def test_repeated_late_collapse_writes_leave_no_ledger_calls(self) -> None: + """A page key that never recurs must still leave the registry. + + Each fenced write is keyed by the exact page of hashes it abandons, + so the write that lands after the caller's deadline is the very + thing that empties that page out of the outbox: nothing invokes that + key again to observe the completion and drop the entry. Several such + pages in a row must leave the registry back at empty, not one + completed call per page each pinning its whole hash tuple for the + life of the process. + """ + fixture = CollapseFixture() + # Wide enough that the page read ahead of the write is never the + # call that expires, short enough to keep the test quick. + fixture.server.block_submit_db_timeout_seconds = 0.25 + rounds = 4 + page_keys: list[tuple[object, ...]] = [] + for round_index in range(rounds): + fixture.seed( + [_hash(2 * round_index + 1), _hash(2 * round_index + 2)] + ) + release = threading.Event() + self.addCleanup(release.set) + + def stall_then_write( + hashes: tuple[str, ...], + error: str, + release: threading.Event = release, + ) -> tuple[str, ...]: + if not release.wait(10): + raise AssertionError( + "the stalled ledger call was never released" + ) + return fixture.ledger.inner.mark_block_candidates_abandoned( + block_hashes=hashes, + error=error, + ) + + fixture.ledger.abandon_hook = stall_then_write + _retained, log = fixture.collapse() + self.assertIn( + "ledger phase timed out phase=collapse-superseded", + log, + ) + with fixture.service._block_submitter_ledger_calls_lock: + stalled = [ + key + for key, call in ( + fixture.service._block_submitter_ledger_calls.items() + ) + if not call.done.is_set() + ] + self.assertEqual(len(stalled), 1) + page_keys.append(stalled[0]) + worker = next( + thread + for thread in threading.enumerate() + if thread.name == "prism-block-ledger-collapse-superseded" + ) + release.set() + worker.join(10) + self.assertFalse(worker.is_alive()) + # The write committed on the worker, and took its entry with it. + with fixture.service._block_submitter_ledger_calls_lock: + self.assertEqual( + dict(fixture.service._block_submitter_ledger_calls), + {}, + ) + + # Every page really was a distinct key, and every delayed write + # really did commit. + self.assertEqual(len(fixture.ledger.abandon_calls), rounds) + self.assertEqual(len(set(page_keys)), rounds) + self.assertEqual(fixture.pending(), set()) + + def test_pagination_still_completes_after_a_full_page_collapses(self) -> None: + """Collapse must not break the short-page completeness proof.""" + fixture = CollapseFixture() + total = MAX_BLOCK_REPLAY_ENUMERATION_ROWS + 7 + fixture.seed([_hash(index) for index in range(1, total + 1)]) + queued, log = fixture.replay(enumeration_owed=True) + self.assertEqual(fixture.pending(), set()) + self.assertEqual(queued, 0) + self.assertIn("enumeration page=2", log) + self.assertFalse(fixture.server._block_replay_enumeration_owed()) + self.assertEqual(len(fixture.ledger.abandon_calls), 2) + self.assertEqual( + [len(call[0]) for call in fixture.ledger.abandon_calls], + [MAX_BLOCK_REPLAY_ENUMERATION_ROWS, 7], + ) + + +class BlockCandidateCollapseHeightProbeBoundTests(unittest.TestCase): + """A height-diverse backlog must not turn collapse into a page-sized walk. + + Collapse reads the chain once per *distinct candidate height*. For the + same-height storm this path exists for that is a handful of round trips + however wide the page is. For a page whose rows sit at many different + heights it was one ``getblockhash`` plus one ``getblockheader`` per + height, sequentially, on the sole block-submitter thread: a full + 1,024-row page of distinct heights is ~2,048 node calls, and at a + production RPC deadline that is minutes to hours in which no live solve + reaches ``submitblock``. + + MAX_BLOCK_CANDIDATE_COLLAPSE_HEIGHT_PROBES caps the distinct heights one + enumeration walk reads. These tests own the four properties that make it + safe: the cap actually binds, an unprobed row is preserved rather than + terminalized and collapses on a later walk, the walk hands the submitter + back to a waiting live solve once its budget is gone, and none of that + disturbs the storm, which spends one probe for the whole walk however + many pages it spans. + """ + + # -- harness ----------------------------------------------------------- + + @staticmethod + def _diverse(heights: int) -> tuple[CollapseFixture, list[str]]: + """A backlog of one row at each of ``heights`` distinct decided heights. + + Every candidate height is occupied by a different heavier block and + the tip sits one height above the highest of them, so predicate S + would select every seeded row if the walk could afford to probe every + height. What the walk actually collapses is therefore exactly what + the probe budget bought. + """ + top = heights + 1 + occupied = { + height: f"{0xC0DE0000 + height:064x}" + for height in range(1, top) + } + occupied[top] = DECIDED_WINNER + fixture = CollapseFixture( + rpc=CollapseChainRpc( + tip=DECIDED_WINNER, + tip_height=top, + active=occupied, + ) + ) + seeded = [] + for height in range(1, top): + block_hash = _hash(height) + seeded.append(block_hash) + fixture.seed([block_hash], height=height) + return fixture, seeded + + @staticmethod + def _live_solve(fixture: CollapseFixture, *, height: int) -> Any: + """One durable candidate on the best tip, admitted to the live lane. + + Its parent is the tip, so clause 3 rejects it before any height is + read: it costs the walk no probe and is never a collapse candidate. + It is queued through the shipped admission path so the submitter sees + exactly what a fresh solve leaves behind. + """ + candidate = fixture.seed( + [LIVE_SOLVE_HASH], + parent=DECIDED_WINNER, + height=height, + )[0] + fixture.server.enqueue_block_candidate(candidate) + fixture.rpc.results["submitblock"] = None + fixture.server.block_submit_rpc_timeout_seconds = 0.5 + return candidate + + @staticmethod + def _offers(fixture: CollapseFixture) -> int: + return sum(1 for method, _ in fixture.rpc.calls if method == "submitblock") + + # -- the bound binds --------------------------------------------------- + + def test_a_height_diverse_page_stops_at_the_probe_bound(self) -> None: + """The P1: 1,024 rows at 1,024 heights, still a fixed handful of reads.""" + rows = MAX_BLOCK_REPLAY_ENUMERATION_ROWS + bound = MAX_BLOCK_CANDIDATE_COLLAPSE_HEIGHT_PROBES + fixture, _seeded = self._diverse(rows) + page = fixture.page(limit=rows) + self.assertEqual(len(page), rows) + retained, _log = fixture.collapse(page) + probed = { + params[0] + for method, params in fixture.rpc.calls + if method == "getblockhash" + } + self.assertEqual(len(probed), bound) + counts = fixture.rpc.counts() + # Selection probes the bound; the pre-write revalidation re-reads + # exactly the heights it selected and reuses their cached work, so + # the whole pass is a constant regardless of the page's row count. + self.assertEqual(counts["getblockhash"], 2 * bound) + self.assertEqual(counts["getblockheader"], bound) + self.assertEqual(counts["getbestblockhash"], 2) + self.assertEqual(counts["getblockcount"], 1) + self.assertEqual(fixture.counts()["abandoned"], bound) + self.assertEqual(fixture.counts()["height_deferred"], rows - bound) + # Every unprobed row is handed back, and handed back durable. + self.assertEqual(len(retained), rows - bound) + self.assertEqual( + {str(row["block_hash"]).lower() for row in retained}, + fixture.pending(), + ) + + def test_unprobed_rows_are_retained_and_later_passes_converge(self) -> None: + """Deferral is not loss: each pass buys the bound, and they add up.""" + bound = MAX_BLOCK_CANDIDATE_COLLAPSE_HEIGHT_PROBES + heights = 3 * bound + fixture, seeded = self._diverse(heights) + outstanding = set(seeded) + self.assertEqual(fixture.pending(), outstanding) + collapsed: set[str] = set() + passes = 0 + # Bounded rather than a while-loop: a pass that stopped making + # progress must fail this test, not spin in it. + for _attempt in range(heights + 1): + if not fixture.pending(): + break + before = fixture.pending() + retained, _log = fixture.collapse(fixture.page(limit=heights)) + after = fixture.pending() + passes += 1 + won = before - after + self.assertEqual(len(won), bound) + self.assertEqual(won & collapsed, set()) + self.assertEqual( + {str(row["block_hash"]).lower() for row in retained}, + after, + "a preserved row must still be the durable row it came from", + ) + collapsed |= won + self.assertEqual(collapsed, outstanding) + self.assertEqual(fixture.pending(), set()) + self.assertEqual(passes, heights // bound) + + # -- the walk hands the submitter back -------------------------------- + + def test_the_walk_yields_to_a_waiting_live_solve_between_pages(self) -> None: + """A diverse backlog cannot hold the only thread that reaches qbitd.""" + bound = MAX_BLOCK_CANDIDATE_COLLAPSE_HEIGHT_PROBES + page_rows = MAX_BLOCK_REPLAY_ENUMERATION_ROWS + fixture, _seeded = self._diverse(page_rows + 7) + # Seeded last, so its durable row sits on the page the walk yields + # before ever fetching. + self._live_solve(fixture, height=page_rows + 9) + queued, log = fixture.replay(enumeration_owed=True) + self.assertIn("enumeration page=1", log) + self.assertNotIn("enumeration page=2", log) + self.assertIn("yielded to a waiting live candidate", log) + counts = fixture.rpc.counts() + self.assertEqual(counts["getblockhash"], 2 * bound) + self.assertEqual(counts["getblockheader"], bound) + # An unproven enumeration stays owed: yielding must not be mistaken + # for the completeness proof that unblocks job builds. + self.assertTrue(fixture.server._block_replay_enumeration_owed()) + # Nothing was skipped -- the page it did fetch was adopted in full. + self.assertEqual(queued, page_rows - bound) + service = fixture.service + self.assertEqual(service.candidate_queue.qsize(), 1) + self.assertEqual( + service._block_replay_candidate_queue.qsize(), + page_rows - bound, + ) + # The scheduling contract itself: the very next turn of the submitter + # loop lands the live solve, ahead of the whole adopted backlog. + with redirect_stdout(StringIO()): + self.assertTrue( + fixture.server.submit_next_block_candidate(defer_accounting=True) + ) + self.assertEqual(self._offers(fixture), 1) + self.assertTrue(service.candidate_queue.empty()) + self.assertEqual( + service._block_replay_candidate_queue.qsize(), + page_rows - bound, + ) + + def test_a_walk_with_nothing_waiting_finishes_its_pages(self) -> None: + """The yield is conditional; without a live solve the walk completes.""" + bound = MAX_BLOCK_CANDIDATE_COLLAPSE_HEIGHT_PROBES + rows = MAX_BLOCK_REPLAY_ENUMERATION_ROWS + 7 + fixture, _seeded = self._diverse(rows) + queued, log = fixture.replay(enumeration_owed=True) + self.assertIn("enumeration page=2", log) + self.assertNotIn("yielded", log) + self.assertFalse(fixture.server._block_replay_enumeration_owed()) + self.assertEqual(queued, rows - bound) + # Page two spends no chain read at all. Every row in this backlog + # sits at a height of its own, so page two holds nothing the walk + # already read and a spent budget leaves it nothing to qualify -- + # even the two page-scope tip reads would have been round trips for + # a decided answer. + counts = fixture.rpc.counts() + self.assertEqual(counts["getbestblockhash"], 2) + self.assertEqual(counts["getblockcount"], 1) + self.assertEqual(counts["getblockhash"], 2 * bound) + self.assertEqual(fixture.counts()["height_deferred"], rows - bound) + + def test_a_same_height_storm_walk_is_unchanged(self) -> None: + """The shape collapse exists for pays one probe per page and never yields.""" + fixture = CollapseFixture() + storm = MAX_BLOCK_REPLAY_ENUMERATION_ROWS + 7 + fixture.seed([_hash(index) for index in range(1, storm + 1)]) + self._live_solve(fixture, height=DECIDED_HEIGHT + 1) + queued, log = fixture.replay(enumeration_owed=True) + # A live solve waits for the whole walk, but one decided height costs + # one probe per page, so the budget is never spent and the storm + # still collapses inside a single, complete enumeration. + self.assertNotIn("yielded", log) + self.assertIn("enumeration page=2", log) + self.assertFalse(fixture.server._block_replay_enumeration_owed()) + self.assertEqual(fixture.counts()["abandoned"], storm) + self.assertEqual(fixture.counts()["height_deferred"], 0) + self.assertEqual(fixture.pending(), {LIVE_SOLVE_HASH}) + self.assertEqual(queued, 1) + self.assertEqual( + [len(call[0]) for call in fixture.ledger.abandon_calls], + [MAX_BLOCK_REPLAY_ENUMERATION_ROWS, 7], + ) + + def test_a_storm_deeper_than_the_budget_collapses_in_one_walk(self) -> None: + """A storm spanning more pages than the walk has probes still collapses. + + The budget is per walk but the chain reads it pays for were once per + *page*: every page rebuilt its view, so one decided height was + charged again on every page and the walk ran out after + MAX_BLOCK_CANDIDATE_COLLAPSE_HEIGHT_PROBES pages. Past that point + collapse did nothing at all. Each remaining page was preserved + whole by the exhausted-budget early return and replay-adopted, or -- + as here, with a live candidate waiting -- never fetched at all, + because a spent budget is half of the yield condition. Either way a + backlog deeper than eight pages collapsed only 8,192 rows per walk, + which at the 312,000-row scale this path is sized for defeats it + outright. + + This walk is nine pages of one decided height, so the eight-page + boundary is genuinely crossed, and the whole storm has to go in one + enumeration off a single probe. + """ + bound = MAX_BLOCK_CANDIDATE_COLLAPSE_HEIGHT_PROBES + page_rows = MAX_BLOCK_REPLAY_ENUMERATION_ROWS + # One page past the bound, plus a short final page: nine fetched + # pages against eight affordable probes. + storm = bound * page_rows + 7 + fixture = CollapseFixture() + fixture.seed([_hash(index) for index in range(1, storm + 1)]) + # Seeded last, so it sits on the ninth page -- the page a walk that + # recharged per page never reaches, having yielded to this very + # candidate once its budget was gone. + self._live_solve(fixture, height=DECIDED_HEIGHT + 1) + queued, log = fixture.replay(enumeration_owed=True) + # One complete enumeration: every page fetched, none yielded, and the + # completeness proof the job-build gate waits on delivered. + self.assertIn(f"enumeration page={bound + 1}", log) + self.assertNotIn("yielded", log) + self.assertFalse(fixture.server._block_replay_enumeration_owed()) + # The whole storm collapsed, not the first eight pages of it. + self.assertEqual(fixture.counts()["abandoned"], storm) + self.assertEqual(fixture.counts()["height_deferred"], 0) + self.assertEqual(fixture.pending(), {LIVE_SOLVE_HASH}) + self.assertEqual( + [len(call[0]) for call in fixture.ledger.abandon_calls], + [page_rows] * bound + [7], + ) + # Not one collapsed row was replay-adopted: the single adoption is + # the live solve, which predicate S rejects on clause 3. + self.assertEqual(queued, 1) + service = fixture.service + self.assertEqual(service._block_replay_candidate_queue.qsize(), 1) + adopted = service._block_replay_candidate_queue.get_nowait() + self.assertEqual(adopted.submission.block_hash_hex.lower(), LIVE_SOLVE_HASH) + # The decided height is charged once for the whole walk. Selection + # reads it on page one and every later page answers from the walk's + # cache, so the occupant's header is read exactly once; the nine + # remaining height reads are the per-page pre-write revalidation, + # which keeps a budget -- and therefore a cache -- of its own by + # design and re-reads the chain under the held leases. + probes = [ + params[0] + for method, params in fixture.rpc.calls + if method == "getblockhash" + ] + self.assertEqual(set(probes), {DECIDED_HEIGHT}) + self.assertEqual(len(probes), 1 + (bound + 1)) + counts = fixture.rpc.counts() + self.assertEqual(counts["getblockheader"], 1) + # Page-scope reads stay per page: one selection tip pair and one + # revalidation tip read each. + self.assertEqual(counts["getbestblockhash"], 2 * (bound + 1)) + self.assertEqual(counts["getblockcount"], bound + 1) + + def test_cached_heights_keep_collapsing_after_the_budget_is_spent(self) -> None: + """A spent budget stops new heights, not the ones already read. + + The counter and the caches are shared by the walk, but the page pass + used to return the whole page the moment the counter hit zero. A + first page spread across all eight affordable heights therefore + disarmed collapse for the rest of the walk, even for later siblings + at exactly those heights -- rows the walk could have decided without + one new round trip, because their occupant was already in + ``probe_budget.active``. Every one of them was preserved and + replay-adopted instead, which is the same defeat the shared cache was + added to prevent, just one page later. + + Nine heights are seeded and only eight are ever affordable, so the + walk here has to draw the line in exactly the right place: reuse what + it read, refuse what it did not. + """ + bound = MAX_BLOCK_CANDIDATE_COLLAPSE_HEIGHT_PROBES + page_rows = MAX_BLOCK_REPLAY_ENUMERATION_ROWS + per_height = page_rows // bound + # The ninth height is the negative control: decided, occupied by a + # heavier block, and selected by every clause of S -- so the only + # thing that can defer it is the spent budget. + control_height = bound + 1 + top = control_height + 1 + occupied = { + height: f"{0xC0DE0000 + height:064x}" + for height in range(1, top) + } + occupied[top] = DECIDED_WINNER + fixture = CollapseFixture( + rpc=CollapseChainRpc( + tip=DECIDED_WINNER, + tip_height=top, + active=occupied, + ) + ) + index = 1 + + def seed(height: int, rows: int) -> list[str]: + nonlocal index + hashes = [_hash(value) for value in range(index, index + rows)] + index += rows + fixture.seed(hashes, height=height) + return hashes + + # Page one: a full page spread across all eight affordable heights, + # which spends the walk's budget exactly. + for height in range(1, bound + 1): + seed(height, per_height) + # Page two: siblings at those same eight heights -- free to decide -- + # and, last on the page, the control rows at the height the walk can + # no longer afford to read. + for height in range(1, bound + 1): + seed(height, per_height - 1) + control = set(seed(control_height, bound)) + # A short third page, still at cached heights, proves the reuse + # outlives the page that first ran into the bound and ends the walk. + for height in range(1, bound): + seed(height, 1) + cached_rows = page_rows + (bound * (per_height - 1)) + (bound - 1) + + queued, log = fixture.replay(enumeration_owed=True) + # One complete enumeration, all three pages walked. + self.assertIn("enumeration page=3", log) + self.assertNotIn("yielded", log) + self.assertFalse(fixture.server._block_replay_enumeration_owed()) + # Every sibling at a height this walk had already read collapsed, + # including the two pages that ran entirely on a spent counter. + self.assertEqual(fixture.counts()["abandoned"], cached_rows) + self.assertEqual( + [len(call[0]) for call in fixture.ledger.abandon_calls], + [page_rows, bound * (per_height - 1), bound - 1], + ) + # The control rows are the only deferral, and they are preserved + # durable rather than lost. + self.assertEqual(fixture.counts()["height_deferred"], bound) + self.assertEqual(fixture.pending(), control) + # Nor did the walk leak a collapsed sibling into replay adoption: the + # deferred controls are the entire adopted set. + self.assertEqual(queued, bound) + service = fixture.service + self.assertEqual(service._block_replay_candidate_queue.qsize(), bound) + adopted = set() + while not service._block_replay_candidate_queue.empty(): + candidate = service._block_replay_candidate_queue.get_nowait() + adopted.add(candidate.submission.block_hash_hex.lower()) + self.assertEqual(adopted, control) + # Not one new distinct height was read past the bound. The control + # height is never asked about at all -- an unprobed height is not a + # cheaper probe, it is no probe. + probes = [ + params[0] + for method, params in fixture.rpc.calls + if method == "getblockhash" + ] + self.assertEqual(set(probes), set(range(1, bound + 1))) + self.assertNotIn(control_height, probes) + # Selection reads the eight heights once for the whole walk; the rest + # are the per-page pre-write revalidation, which keeps its own budget + # and re-reads each retained height under the held leases. + self.assertEqual(len(probes), bound + (bound + bound + (bound - 1))) + counts = fixture.rpc.counts() + # One header per occupant for the walk: page two and page three both + # answer clause 4b from the work this walk already derived. + self.assertEqual(counts["getblockheader"], bound) + # Page-scope tip reads stay per page and stay fresh, on the pages a + # spent budget now lets through as much as on the first. + self.assertEqual(counts["getbestblockhash"], 2 * 3) + self.assertEqual(counts["getblockcount"], 3) + + +class BlockCandidateCollapseObservabilityTests(unittest.TestCase): + """Bounded logs and bounded, hash-free metrics.""" + + def test_the_bulk_path_does_not_log_one_line_per_candidate(self) -> None: + fixture = CollapseFixture() + fixture.seed([_hash(index) for index in range(1, 201)]) + _retained, log = fixture.collapse() + lines = [line for line in log.splitlines() if line.strip()] + self.assertLessEqual(len(lines), 4 + BLOCK_CANDIDATE_COLLAPSE_LOG_GROUPS) + self.assertNotIn("block candidate abandoned reason=", log) + summary = [line for line in lines if "collapsed superseded" in line] + self.assertEqual(len(summary), 1) + self.assertIn("considered=200", summary[0]) + self.assertIn("abandoned=200", summary[0]) + + def test_group_lines_and_hash_samples_stay_bounded(self) -> None: + fixture = CollapseFixture() + groups = BLOCK_CANDIDATE_COLLAPSE_LOG_GROUPS + 3 + index = 1 + for group in range(groups): + parent = f"{group:064x}" + fixture.seed( + [_hash(index + offset) for offset in range(5)], + parent=parent, + ) + index += 5 + _retained, log = fixture.collapse() + group_lines = [ + line for line in log.splitlines() if "collapsed candidate group " in line + ] + self.assertEqual(len(group_lines), BLOCK_CANDIDATE_COLLAPSE_LOG_GROUPS) + self.assertIn("collapsed candidate groups not shown count=3", log) + for line in group_lines: + sample = line.split("sample=", 1)[1].strip() + self.assertLessEqual( + len(sample.split(",")), + BLOCK_CANDIDATE_COLLAPSE_LOG_SAMPLE_HASHES, + ) + + def test_fail_closed_logging_is_rate_limited(self) -> None: + fixture = CollapseFixture() + fixture.seed([_hash(1)]) + fixture.rpc.failures.add("getblockcount") + _retained, first = fixture.collapse() + _retained, second = fixture.collapse() + self.assertIn("collapse failed", first) + self.assertNotIn("collapse failed", second) + self.assertEqual(fixture.counts()["fail_closed"], 2) + + def test_the_snapshot_carries_exactly_the_fixed_outcome_keys(self) -> None: + fixture = CollapseFixture() + self.assertEqual( + tuple(fixture.counts()), + PRISM_BLOCK_CANDIDATE_COLLAPSE_OUTCOMES, + ) + + def test_metrics_render_bounded_hash_free_collapse_series(self) -> None: + fixture = CollapseFixture() + fixture.seed([_hash(1), _hash(2)]) + with redirect_stdout(StringIO()): + fixture.collapse() + lines = MetricsRenderer(fixture.server).block_submitter_metrics_lines() + series = [ + line + for line in lines + if line.startswith("qbit_prism_block_candidate_collapse_total") + ] + self.assertEqual(len(series), len(PRISM_BLOCK_CANDIDATE_COLLAPSE_OUTCOMES)) + rendered = "\n".join(lines) + self.assertIn( + 'qbit_prism_block_candidate_collapse_total{outcome="abandoned"} 2', + rendered, + ) + self.assertIn( + 'qbit_prism_block_candidate_collapse_total{outcome="considered"} 2', + rendered, + ) + for forbidden in (_hash(1), _hash(2), STORM_PARENT, DECIDED_WINNER, "job-"): + self.assertNotIn(forbidden, rendered) + + def test_per_row_diagnostics_survive_for_preserved_candidates(self) -> None: + """A candidate the collapse declines still logs its own abandonment.""" + fixture = CollapseFixture() + preserved = _hash(1) + candidate = fixture.seed([preserved], parent=OTHER_PARENT)[0] + fixture.rpc.tip = OTHER_PARENT + fixture.collapse() + self.assertEqual(fixture.pending(), {preserved}) + buffer = StringIO() + with redirect_stdout(buffer): + fixture.service.record_abandoned( + PRISM_REJECTION_STALE_JOB, + "tip moved before submit: deadbeef", + block_hash=preserved, + worker="miner-a", + expected_height=DECIDED_HEIGHT, + stale_job_class="tip_moved", + ) + self.assertIn("block candidate abandoned reason=stale-job", buffer.getvalue()) + self.assertIs(candidate.credit_share_on_accept, False) + + +class BlockCandidateCollapseStormTests(unittest.TestCase): + """Compare the selector against the shipped per-row oracle at storm sizes.""" + + def _oracle(self, candidates: int, view: str) -> tuple[set[str], str]: + rig = CandidateStormRig(candidates=candidates) + rig.seed_live() + if view == "restart": + rig.restart_and_enumerate() + server = rig.restarted_server if view == "restart" else rig.live_server + # ``drain_per_row`` suppresses the collapse itself, so this stays + # the shipped per-row oracle without any patching here. + winner = rig.decide_height() + report = rig.drain_per_row(server, stop_before_hash=winner) + return set(report.abandoned_hashes), winner + + def _selector( + self, + candidates: int, + view: str, + *, + perturbation: str | None = None, + target: str | None = None, + ) -> tuple[set[str], Any, str]: + rig = CandidateStormRig(candidates=candidates) + rig.seed_live() + if view == "restart": + rig.restart_and_enumerate() + server = rig.restarted_server if view == "restart" else rig.live_server + winner = rig.decide_height() + if perturbation is not None: + with redirect_stdout(StringIO()): + rig.perturb(server, perturbation, str(target)) + before = { + str(row["block_hash"]).lower() + for row in rig.ledger.pending_block_candidate_rows(limit=candidates + 1) + } + server._note_block_replay_enumeration_owed() + with redirect_stdout(StringIO()): + server.replay_pending_block_candidates() + after = { + str(row["block_hash"]).lower() + for row in rig.ledger.pending_block_candidate_rows(limit=candidates + 1) + } + return before - after, server, winner + + def _compare(self, candidates: int, view: str) -> set[str]: + oracle, winner = self._oracle(candidates, view) + selected, server, winner_again = self._selector(candidates, view) + self.assertEqual(winner, winner_again) + self.assertEqual( + selected - oracle, + set(), + f"selector abandoned {len(selected - oracle)} hash(es) the shipped " + f"per-row path refused ({candidates} candidates, {view} view)", + ) + # The baseline storm is the exact-agreement case: every sibling of the + # decided block is superseded and carries no offer evidence. + self.assertEqual(oracle - selected, set()) + self.assertNotIn(winner, selected) + self.assertEqual( + server.block_candidate_abandoned_counts, + {PRISM_BLOCK_CANDIDATE_COLLAPSE_REASON: len(selected)}, + ) + return selected + + def test_efficacy_and_agreement_at_one_hundred_candidates(self) -> None: + for view in ("restart", "live"): + with self.subTest(view=view): + selected = self._compare(100, view) + # Every candidate but the decided winner; the storm's whole + # point is that the shipped path pays 99 node offers for it. + self.assertGreaterEqual(len(selected), 90) + self.assertEqual(len(selected), 99) + + def test_agreement_at_the_observed_testnet_storm(self) -> None: + for view in ("restart", "live"): + with self.subTest(view=view): + selected = self._compare(3_120, view) + self.assertEqual(len(selected), 3_119) + + def test_the_storm_costs_a_bounded_number_of_chain_reads(self) -> None: + selected, server, _winner = self._selector(3_120, "restart") + self.assertEqual(len(selected), 3_119) + pages = -(-3_120 // MAX_BLOCK_REPLAY_ENUMERATION_ROWS) + calls = server.rpc.calls + # Tip reads are page-scope: two per page (selection plus + # revalidation) and one tip-height read each. The decided height and + # its occupant are walk-scope: selection reads them once for the + # whole walk and every later page answers from the walk's cache, so + # the only further active-block reads are the one each page's + # pre-write revalidation makes under its own fresh budget. + self.assertEqual(calls["getbestblockhash"], 2 * pages) + self.assertEqual(calls["getblockcount"], pages) + self.assertEqual(calls["getblockhash"], 1 + pages) + self.assertEqual(calls["getblockheader"], 1) + self.assertNotIn("submitblock", calls) + + def test_no_perturbation_ever_produces_a_selector_only_hash(self) -> None: + candidates = 40 + target = _hash(7) + for view in ("restart", "live"): + for kind in PERTURBATIONS: + with self.subTest(view=view, perturbation=kind): + rig = CandidateStormRig(candidates=candidates) + rig.seed_live() + if view == "restart": + rig.restart_and_enumerate() + server = ( + rig.restarted_server + if view == "restart" + else rig.live_server + ) + winner = rig.decide_height() + with redirect_stdout(StringIO()): + undo = rig.perturb(server, kind, target) + report = rig.drain_per_row(server, stop_before_hash=winner) + oracle = set(report.abandoned_hashes) + if undo is not None: + undo() + selected, _server, _winner = self._selector( + candidates, + view, + perturbation=kind, + target=target, + ) + self.assertEqual( + selected - oracle, + set(), + f"{kind} produced a selector-only hash in the {view} view", + ) + self.assertNotIn(target, selected) + self.assertLessEqual(len(oracle - selected), 1) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_prism_coordinator_vardiff.py b/tests/test_prism_block_candidates.py similarity index 51% rename from tests/test_prism_coordinator_vardiff.py rename to tests/test_prism_block_candidates.py index de9a9c5c2..ef9f6b120 100644 --- a/tests/test_prism_coordinator_vardiff.py +++ b/tests/test_prism_block_candidates.py @@ -1,7163 +1,4852 @@ #!/usr/bin/env python3 +"""Focused PRISM coordinator block candidates tests.""" +# ruff: noqa: F403, F405 from __future__ import annotations -import errno -import hashlib -import inspect -import json -import os -import queue -import socket -import subprocess -import tempfile -import contextlib -import threading -import time +import dataclasses +import io import unittest -from concurrent.futures import Future -from dataclasses import replace as dataclass_replace -from decimal import Decimal -from pathlib import Path -from types import SimpleNamespace -from unittest.mock import patch - -from lab.auxpow import vardiff -from lab.prism import direct_stratum -from lab.prism.share_ledger import ( - LedgerOperationTimeout, - PendingShare, - PsqlShareLedger, - SingleWriterShareLedger, - WRITER_LEASE_HEARTBEAT_SESSION_PREFIX, +from unittest import mock +from tests.prism_vardiff_test_support import * +from lab.prism.coordinator_config import LifecycleConfig +from lab.prism.audit_artifacts import ( + AuditArtifactStore, + AuditPublicationIdentity, + RetentionResult, ) -from lab.prism.prism_coordinator import ( - CachedJobBundle, - CachedTemplateArtifacts, - ClientState, - DEFAULT_TESTNET_USERNAME_FALLBACK_ADDRESS, - MAX_ACTIVE_PRISM_JOBS_PER_CLIENT, - MAX_PENDING_SHARE_APPENDS, - PRISM_CREDIT_POLICY_STALE_GRACE, - PRISM_REJECTION_BACKEND_RPC_UNAVAILABLE, - PRISM_REJECTION_BLOCK_ACCEPT_PENDING, - PRISM_REJECTION_CANDIDATE_AUDIT_MISMATCH, - PRISM_REJECTION_DUPLICATE_SHARE, - PRISM_REJECTION_INVALID_NTIME_OR_NONCE, - PRISM_REJECTION_LEDGER_CONFIRMATION_FAILED, - PRISM_REJECTION_LOW_DIFFICULTY, - PendingShareAppend, - PayoutStateArtifact, - PrismBlockCandidate, - PRISM_REJECTION_POOL_CLOSED, - PRISM_REJECTION_REASON_IDS, - PRISM_REJECTION_STALE_JOB, - PRISM_REJECTION_SUBMITBLOCK_REJECTED, - PRISM_REJECTION_UNAUTHORIZED_WORKER, - PRISM_REJECTION_UNKNOWN_JOB, - PRISM_WORKER_METRICS_OVERFLOW_LABEL, - QbitTipTemplateSnapshot, - StratumError, - StratumListenerProfile, - JobBuildCancelled, - JobBuildSuperseded, - TemplateRefreshBlocked, - TemplateRefreshSuperseded, - PrismCoordinator, - ShutdownInProgress, - WorkerIdentity, - WriterLeaseRenewalDeferred, - _FanoutCancellation, - _ObservedRLock, - _PayoutStatePublicationBlocked, - _JobBuildCancellation, - _ReconcileFlight, - default_prism_coinbase_tag_hex, - default_prism_username_fallback_address, - load_prism_highdiff_listener, - load_prism_vardiff_config, - parse_stratum_password_options, - qbit_template_fingerprint, - qbit_gbt_rules, - env_positive_float, - scaled_target_difficulty, - target_from_compact, - validate_prism_production_gate, - validate_same_tip_job_retention_limits, +from lab.prism.block_candidates import ( + MAX_BLOCK_REPLAY_ENUMERATION_ROWS, + MAX_BLOCK_SUBMITTER_STUCK_CALL_WORKERS, + MAX_PENDING_BLOCK_CANDIDATES, + BlockCandidateAttemptResult, + BlockCandidateRunResult, + _BlockCandidateNodeSubmission, + _BlockSubmitterLedgerCall, + block_candidate_from_intent, + block_candidate_intent, ) +from tests.prism_coordinator_test_support import durable_candidate_row +from lab.prism.bundle_compiler import BundleCompiler +from lab.prism.prism_coordinator import PrismCoordinator -PAYOUT_ADDRESS = "tq1z70ukpvs96kye6jmgvl3nttevtkrq8uu89snkpm6m8gwqukw8u5dsz32kwa" +_compat_verified_audit_report = verified_audit_report -def fake_audit_bundle_popen( - captured: dict[str, object], - *, - output_text: str = '{"ok":true}', - returncode: int = 0, - stderr_text: str = "", -) -> type: - class FakeStdin: - def __init__(self) -> None: - self.parts: list[str] = [] - def write(self, value: str) -> int: - self.parts.append(value) - return len(value) - - def close(self) -> None: - return None - - class FakePopen: - def __init__(self, cmd: list[str], **kwargs: object) -> None: - captured["cmd"] = cmd - self.stdin = FakeStdin() - self.stdout = kwargs["stdout"] - self.stderr = kwargs["stderr"] - - def wait(self, timeout: float | None = None) -> int: - captured["timeout"] = timeout - captured["payload"] = json.loads("".join(self.stdin.parts)) - self.stdout.write(output_text) - self.stderr.write(stderr_text) - return returncode - - return FakePopen - - -def tx_output(value_sats: int, script_hex: str) -> str: - return value_sats.to_bytes(8, "little").hex() + direct_stratum.compact_size(len(bytes.fromhex(script_hex))).hex() + script_hex - - -def synthetic_witness_transaction(seed: str) -> str: - script = seed * 3 - witness_item = seed * 5 - return ( - "01000000" - + "0001" - + "01" - + (seed * 32) - + "00000000" - + direct_stratum.compact_size(len(bytes.fromhex(script))).hex() - + script - + "ffffffff" - + "01" - + tx_output(1, "51") - + "01" - + direct_stratum.compact_size(len(bytes.fromhex(witness_item))).hex() - + witness_item - + "00000000" - ) - - -class FakeJob: - def __init__(self, difficulty: Decimal) -> None: - self.share_difficulty = difficulty - - -class FakeLedger: - backend_name = "fake" - - def __init__(self, shares: int = 0, prior_balances: list[dict[str, object]] | None = None) -> None: - self.shares = shares - self.prior_balances = prior_balances or [] - - def all_shares(self) -> list[object]: - return [object()] * self.shares - - def current_prior_balances(self) -> list[dict[str, object]]: - return [dict(balance) for balance in self.prior_balances] - - def metrics(self) -> dict[str, int]: - return {"blocks": 2, "owed_accounts": 3} - - def reorg_watch_blocks(self, *, active_tip_height: int) -> list[dict[str, object]]: - return [] - - def mark_pool_block_inactive(self, *, block_hash: str, active_tip_height: int) -> dict[str, object]: - return {"backend": "fake", "inactive_count": 0} - - def reject_prepared_block(self, *, block_hash: str, active_tip_height: int) -> dict[str, object]: - return {"backend": "fake", "rejected_count": 0} - - def reactivate_pool_block(self, *, block_hash: str, active_tip_height: int) -> dict[str, object]: - return {"backend": "fake", "reactivated_count": 0} - - def mark_mature_pool_payouts(self, *, active_tip_height: int) -> dict[str, object]: - return {"backend": "fake", "matured_count": 0} - - -class RecordingLedger(FakeLedger): - def __init__(self) -> None: - super().__init__(shares=0) - self.pending: list[object] = [] - self.persisted: list[dict[str, object]] = [] - self.confirmed: list[dict[str, object]] = [] - self.reversed: list[dict[str, object]] = [] - self.rejected: list[dict[str, object]] = [] - self.submit_seen = False - - def append(self, pending: object) -> object: - self.pending.append(pending) - self.shares += 1 - return SimpleNamespace(share_seq=self.shares, miner_id=getattr(pending, "miner_id", "miner-a")) - - def persist_accepted_block(self, **kwargs: object) -> dict[str, object]: - self.persisted.append({**kwargs, "submit_seen_at_persist": self.submit_seen}) - return { - "backend": "fake", - "share_count": self.shares, - "block_count": 1, - "bundle_count": 1, - "payout_entry_count": 0, - "carry_forward_count": 0, - "onchain_output_count": 1, - } - - def reverse_immature_block(self, **kwargs: object) -> dict[str, object]: - self.reversed.append(kwargs) - return {"backend": "fake", "reversed_count": 1} - - def reject_prepared_block(self, **kwargs: object) -> dict[str, object]: - self.rejected.append(kwargs) - return {"backend": "fake", "rejected_count": 1} - - def confirm_accepted_block(self, **kwargs: object) -> dict[str, object]: - self.confirmed.append({**kwargs, "submit_seen_at_confirm": self.submit_seen}) - return {"backend": "fake", "confirmed_count": 1} - - def all_shares(self) -> list[object]: - return [ - SimpleNamespace(miner_id=getattr(pending, "miner_id", "miner-a")) - for pending in self.pending - ] - - -class FakeRpc: - def call(self, method: str, params: list[object] | None = None) -> object: - if method == "getblockchaininfo": - return {"initialblockdownload": False} - if method == "getnetworkinfo": - return {"connections": 4} - raise RuntimeError(method) - - -class FeeEstimateRpc(FakeRpc): - def __init__(self, estimate: object) -> None: - self.estimate = estimate - self.calls: list[tuple[str, list[object] | None]] = [] - - def call(self, method: str, params: list[object] | None = None) -> object: - self.calls.append((method, params)) - if method == "estimatesmartfee": - return self.estimate - return super().call(method, params) - - -class TemplateRpc(FakeRpc): - def __init__(self, template: object) -> None: - self.template = template - self.calls: list[tuple[str, list[object] | None]] = [] - - def call(self, method: str, params: list[object] | None = None) -> object: - self.calls.append((method, params)) - if method == "getblocktemplate": - return self.template - return super().call(method, params) - - -class AddressValidationRpc(FakeRpc): - def __init__( - self, - *, - valid_address: str = PAYOUT_ADDRESS, - script_byte: str = "11", - p2mr: bool = True, - ) -> None: - self.valid_address = valid_address - self.script_byte = script_byte - self.p2mr = p2mr - self.validated: list[str] = [] - - def call(self, method: str, params: list[object] | None = None) -> object: - if method == "validateaddress": - address = str((params or [""])[0]) - self.validated.append(address) - script = "5220" + self.script_byte * 32 if self.p2mr else "51" - return {"isvalid": address == self.valid_address, "scriptPubKey": script} - return super().call(method, params) - - -AddressRpc = AddressValidationRpc - - -class TipRpc(FakeRpc): - def __init__(self, tip: str) -> None: - self.tip = tip - - def call(self, method: str, params: list[object] | None = None) -> object: - if method == "getbestblockhash": - return self.tip - if method == "getblockheader": - raise RuntimeError("qbit RPC getblockheader failed: -5 Block not found") - return super().call(method, params) - - -class RejectingSubmitTipRpc(TipRpc): - def call(self, method: str, params: list[object] | None = None) -> object: - if method == "submitblock": - return "bad-prevblk" - return super().call(method, params) - - -class ParentTipRpc(TipRpc): - def __init__(self, *, tip: str, parent: str) -> None: - super().__init__(tip) - self.parent = parent - self.submitblock_calls = 0 - - def call(self, method: str, params: list[object] | None = None) -> object: - if method == "getblock": - self.assert_tip_param(params) - return {"hash": self.tip, "previousblockhash": self.parent} - if method == "submitblock": - self.submitblock_calls += 1 - return None - return super().call(method, params) - - def assert_tip_param(self, params: list[object] | None) -> None: - if not params or str(params[0]) != self.tip: - raise AssertionError(f"expected getblock current tip {self.tip}, got {params!r}") - - -class UnsupportedBlockwaitRpc(TipRpc): - def call( - self, - method: str, - params: list[object] | None = None, - *, - timeout: float | None = None, - ) -> object: - if method == "waitfornewblock": - raise RuntimeError("Method not found") - return super().call(method, params) - - -class TipTemplateRpc(FakeRpc): - def __init__(self, *, tip: str, template: dict[str, object]) -> None: - self.tip = tip - self.template = template - self.calls: list[tuple[str, list[object] | None]] = [] - - def call(self, method: str, params: list[object] | None = None) -> object: - self.calls.append((method, params)) - if method == "getbestblockhash": - return self.tip - if method == "getblocktemplate": - return self.template - return super().call(method, params) - - -class ReorgRpc(TipTemplateRpc): - def __init__( - self, - *, - tip: str, - template: dict[str, object], - height: int, - block_hashes: dict[int, str], - initialblockdownload: bool = False, - headers: int | None = None, - ) -> None: - super().__init__(tip=tip, template=template) - self.height = height - self.block_hashes = block_hashes - self.initialblockdownload = initialblockdownload - self.headers = headers if headers is not None else height - - def call(self, method: str, params: list[object] | None = None) -> object: - if method == "getblockchaininfo": - return { - "initialblockdownload": self.initialblockdownload, - "blocks": self.height, - "headers": self.headers, - } - if method == "getblockcount": - return self.height - if method == "getblockhash": - height = int((params or [0])[0]) - try: - return self.block_hashes[height] - except KeyError as exc: - raise RuntimeError(f"unknown height {height}") from exc - return super().call(method, params) - - -class ReorgLedger(FakeLedger): - def __init__(self, rows: list[dict[str, object]]) -> None: - super().__init__(shares=0) - self.rows = [dict(row) for row in rows] - self.events: list[object] = [] - - def reorg_watch_blocks(self, *, active_tip_height: int) -> list[dict[str, object]]: - self.events.append(("watch", active_tip_height)) - return [dict(row) for row in self.rows] - - def mark_pool_block_inactive(self, *, block_hash: str, active_tip_height: int) -> dict[str, object]: - self.events.append(("inactive", block_hash, active_tip_height)) - for row in self.rows: - if str(row.get("block_hash", "")).lower() == block_hash.lower(): - row["chain_state"] = "inactive" - return {"backend": "fake", "inactive_count": 1} - return {"backend": "fake", "inactive_count": 0} - - def reactivate_pool_block(self, *, block_hash: str, active_tip_height: int) -> dict[str, object]: - self.events.append(("reactivate", block_hash, active_tip_height)) - for row in self.rows: - if str(row.get("block_hash", "")).lower() == block_hash.lower(): - row["chain_state"] = "confirmed" - return {"backend": "fake", "reactivated_count": 1} - return {"backend": "fake", "reactivated_count": 0} - - def mark_mature_pool_payouts(self, *, active_tip_height: int) -> dict[str, object]: - self.events.append(("mature", active_tip_height)) - return {"backend": "fake", "matured_count": 0} - - -class SubmitRpc(FakeRpc): - def __init__( - self, - *, - tip: str, - block_hash: str, - submit_result: object = None, - ledger: RecordingLedger | None = None, - ) -> None: - self.tip = tip - self.block_hash = block_hash - self.submit_result = submit_result - self.ledger = ledger - self.height = 9 - self.submitted = False - - def call(self, method: str, params: list[object] | None = None) -> object: - if method == "getbestblockhash": - return self.tip - if method == "getblockcount": - return self.height - if method == "submitblock": - self.submitted = True - if self.ledger is not None: - self.ledger.submit_seen = True - if self.submit_result is None: - self.height += 1 - return self.submit_result - if method == "getblockhash": - return self.block_hash - return super().call(method, params) - - -class SubmitAcceptingTemplateRpc(FakeRpc): - def __init__( - self, - *, - old_tip: str, - block_hash: str, - fail_template_after_submit: bool = False, - ledger: RecordingLedger | None = None, - ) -> None: - self.old_tip = old_tip - self.block_hash = block_hash - self.fail_template_after_submit = fail_template_after_submit - self.ledger = ledger - self.height = 9 - self.submitted = False - - def call(self, method: str, params: list[object] | None = None) -> object: - if method == "getbestblockhash": - return self.block_hash if self.submitted else self.old_tip - if method == "getblockcount": - return self.height - if method == "submitblock": - self.submitted = True - self.height += 1 - if self.ledger is not None: - self.ledger.submit_seen = True - return None - if method == "getblockhash": - return self.block_hash - if method == "getblocktemplate": - if self.submitted and self.fail_template_after_submit: - raise RuntimeError("transient getblocktemplate failure after submitblock") - previousblockhash = self.block_hash if self.submitted else self.old_tip - return gbt_template(previousblockhash, height=self.height + 1) - return super().call(method, params) - - -def client() -> ClientState: - state = ClientState(sock=object(), address=("127.0.0.1", 1), connection_id=1, extranonce1_hex="00000001") - state.subscribed = True - state.authorized = True - return state - - -def gbt_template( - previousblockhash: str, - *, - height: int = 10, - coinbasevalue: int = 50_00000000, - curtime: int = 1_700_000_000, - transactions: list[str] | None = None, -) -> dict[str, object]: - return { - "previousblockhash": previousblockhash, - "version": 0x20000000, - "bits": "207fffff", - "curtime": curtime, - "height": height, - "coinbasevalue": coinbasevalue, - "transactions": [{"data": tx_hex} for tx_hex in transactions or []], - } - - -def worker_identity(username: str = "miner-a") -> WorkerIdentity: - return WorkerIdentity( - username=username, - payout_address=username, - worker_name=None, - script_pubkey_hex="5220" + "11" * 32, - p2mr_program_hex="11" * 32, - ) - - -def stratum_job( - job_id: str, - *, - difficulty: Decimal = Decimal("1"), - clean_jobs: bool = True, -) -> SimpleNamespace: - return SimpleNamespace( - job_id=job_id, - share_difficulty=difficulty, - share_target=target_from_compact("207fffff"), - prevhash="00" * 32, - coinb1="", - coinb2="", - merkle_branch=(), - version="20000000", - nbits="207fffff", - ntime="6553f100", - clean_jobs=clean_jobs, - transaction_hexes=(), - ) - - -def prism_context( - job_id: str, - previousblockhash: str, - *, - worker: WorkerIdentity | None = None, - difficulty: Decimal = Decimal("1"), - clean_jobs: bool = True, -) -> SimpleNamespace: - return SimpleNamespace( - job=stratum_job(job_id, difficulty=difficulty, clean_jobs=clean_jobs), - template=gbt_template(previousblockhash), - found_block={"network_difficulty": 1}, - issued_at_ms=12345, - collection_only=False, - worker=worker or worker_identity(), - shares_json=[], - prior_balances=[], - ) - - -def prepare_idle_client( +def configure_temporary_audit_root( + test_case: unittest.TestCase, server: PrismCoordinator, - state: ClientState, - *, - connection_id: int = 1, - difficulty: Decimal = Decimal("16"), - tip: str = "00" * 32, ) -> None: - if not hasattr(server, "jobs"): - server.jobs = {} - state.connection_id = connection_id - state.extranonce1_hex = f"{connection_id:08x}" - state.username = f"miner-{connection_id}" - state.worker = worker_identity(state.username) - job_id = f"job-{connection_id}" - state.active_job = prism_context( - job_id, - tip, - worker=state.worker, - difficulty=difficulty, - ) - state.active_job_ids = {job_id} - state.share_difficulty = difficulty - state.pending_share_difficulty = None - state.vardiff_window_started_monotonic = time.monotonic() - 2 - state.vardiff_window_accepted = 0 - state.vardiff_window_submitted = 0 - state.vardiff_window_work = Decimal("0") - server.clients.add(state) - server.jobs[job_id] = state.active_job - - -def install_idle_job_cache( - server: PrismCoordinator, - *, - tip: str = "00" * 32, -) -> CachedJobBundle: - server._pool_ready_latched = True - server.job_bundle_cache_seconds = 60.0 - server.job_counter = 0 - server.share_weights_by_username = {} - server.default_share_weight = 1 - server._ensure_job_cache_state() - template = gbt_template(tip) - fingerprint = qbit_template_fingerprint(template) - artifacts = CachedTemplateArtifacts( - template=template, - fingerprint=fingerprint, - previousblockhash=tip, - transaction_hexes=(), - witness_merkle_leaves_hex=(), - network_difficulty=1, - fetched_monotonic=time.monotonic(), - generation=1, - ) - key = server._job_bundle_key( - artifacts, - mode="ready", - payout_state_generation=server._payout_state_generation, - payout_artifact_generation=0, - worker=None, - ) - payout_artifact_sha256 = "aa" * 32 - qbit_target = direct_stratum.difficulty_target(Decimal("1024")) - base_job = direct_stratum.DirectQbitStratumJob( - job_id="prism-template-base", - previousblockhash_display=tip, - prevhash=tip, - coinb1="", - coinb2="", - full_coinbase_prefix="", - full_coinbase_suffix="", - merkle_branch=(), - transaction_hexes=(), - version="20000000", - nbits="207fffff", - ntime="6553f100", - qbit_target=qbit_target, - share_target=qbit_target, - share_difficulty=Decimal("1"), - extranonce1_hex="ffffffff", - extranonce2_size=8, - clean_jobs=True, - ) - bundle = CachedJobBundle( - key=key, - template=template, - template_fingerprint=fingerprint, - coinbase_manifest={}, - shares_json=[], - prior_balances=[], - found_block={"network_difficulty": 1}, - collection_only=False, - issued_at_ms=12345, - base_job=base_job, - built_monotonic=time.monotonic(), - template_generation=1, - payout_state_generation=0, - build_key=SimpleNamespace( - payout_artifact_sha256=payout_artifact_sha256, - payout_append_invalidation_epoch=( - server._payout_ledger_append_invalidation_epoch - ), - ), - ) - with server._job_cache_lock: - server._template_artifacts = artifacts - server._published_payout_state = dataclass_replace( - server._published_payout_state, - artifact=PayoutStateArtifact( - generation=server._payout_state_generation, - source_generation=0, - prior_balances_json="[]", - prior_balances_sha256=payout_artifact_sha256, - prepared_monotonic=time.monotonic(), - ), - ) - server._job_bundle_cache.clear() - server._job_bundle_cache[bundle.key] = bundle - return bundle - - -def verified_block_bundle(coinbase_tx_hex: str = "c0ffee") -> dict[str, object]: - return { - "found_block": {"coinbase_value_sats": 50_00000000}, - "ledger_window_attestation": {"signature": {"public_key_hex": "aa" * 32}}, - "payout_policy_manifest": {"accounts": []}, - "signed_coinbase_manifest": { - "manifest": { - "coinbase_tx_hex": coinbase_tx_hex, - "payout_count": 1, - } - }, - } + temporary = tempfile.TemporaryDirectory() + server.audit_dir = Path(temporary.name) / "audit" + server.evidence_path = Path(temporary.name) / "state" / "evidence.json" + def cleanup() -> None: + store = server.__dict__.get("_audit_artifact_store") + if isinstance(store, AuditArtifactStore): + store.close() + temporary.cleanup() -def verified_audit_report(coinbase_tx_hex: str = "c0ffee") -> dict[str, object]: - return { - "coinbase_txid": "11" * 32, - "coinbase_manifest_sha256_hex": "22" * 32, - "audit_bundle_sha256_hex": "33" * 32, - "coinbase_tx_hex": coinbase_tx_hex, - } + test_case.addCleanup(cleanup) -def coordinator() -> PrismCoordinator: - server = PrismCoordinator.__new__(PrismCoordinator) - server.vardiff_config = vardiff.VardiffConfig( - enabled=True, - target_share_interval_seconds=Decimal("15"), - min_difficulty=Decimal("0.000000001"), - max_difficulty=Decimal("1024"), - retarget_interval_seconds=Decimal("1"), - max_step_factor=Decimal("4"), - startup_difficulty=Decimal("0.000000001"), - max_step_down_factor=Decimal("4"), - ewma_alpha=Decimal("1"), - retarget_tolerance=Decimal("0"), - ) - server.share_difficulty = Decimal("0.000000001") - server.lock = threading.RLock() - server.stop_event = threading.Event() - server.clients = set() - server.submitted_share_count = 0 - server.stale_share_count = 0 - server.duplicate_share_count = 0 - server.low_difficulty_share_count = 0 - server.collection_block_submission_count = 0 - server._pool_ready_latched = False - server.grace_credited_share_count = 0 - server.idle_retarget_count = 0 - server.rejection_counts_by_reason = {reason: 0 for reason in PRISM_REJECTION_REASON_IDS} - server.worker_metrics_limit = 100 - server.worker_metrics_lock = threading.Lock() - server.worker_share_counts = {} - server.worker_rejection_counts = {} - server.evicted_job_graveyard = {} - server.block_candidate_queue = queue.Queue(maxsize=8) - server.block_candidates_dropped = 0 - server.block_candidate_abandoned_counts = {} - server.share_append_queue = queue.Queue(maxsize=8) - server.share_writer_active = False - server.share_append_failure_count = 0 - server.share_recovery_path = None - server.share_recovery_lock = threading.Lock() - server.shares_recovered_to_disk = 0 - server.shares_replayed = 0 - server.current_tip_first_seen = None - server.current_tip_parent = None - server.stale_grace_seconds = 3.0 - server.blockwait_enabled = True - server.blockwait_timeout_seconds = 5.0 - server.vardiff_idle_sweep_seconds = 15.0 - server.job_build_failure_count = 0 - server.tip_refresh_job_count = 0 - server.post_accept_refresh_failure_count = 0 - server.reorg_reconciler_enabled = False - server.reorg_inactive_block_count = 0 - server.reorg_reactivated_block_count = 0 - server.reorg_reconcile_skip_count = 0 - server.reorg_reconcile_error_count = 0 - server.matured_payout_count = 0 - server.last_reorg_reconciled_tip_hash = None - server.last_reorg_reconciled_trusted = False - server.accepted_block_count = 1 - server.started_monotonic = time.monotonic() - 10 - server.ledger = FakeLedger(shares=5) - server.latest_coinbase_size_bytes = 250 - server.rpc = FakeRpc() - server.qbit_chain = "regtest" - server.blockpoll_seconds = 2.0 - # Failed-refresh spacing is opt-in per test: its holdoff waits on real - # time, which deadlocks tests that freeze time.monotonic around failing - # polls. Pacing behavior is covered by test_prism_refresh_retry_pacing. - server.tip_refresh_failure_holdoff_seconds = 0.0 - server.ctv_broadcaster_enabled = False - server.ctv_broadcaster_wallet = None - server.ctv_broadcaster_fee_sats = 0 - server.ctv_broadcaster_limit = 100 - server.ctv_broadcaster_interval_seconds = 30.0 - server.ctv_fanout_broadcast_daemon = None - server._ctv_fanout_market_fee_rate_cache = {} - server.tip_template_snapshot = None - server._tip_refresh_lock = threading.Lock() - server.extranonce2_size = 8 - server.coinbase_tag_hex = default_prism_coinbase_tag_hex() - server.version_mask = direct_stratum.QBIT_VERSION_ROLLING_MASK - server.version_mask_selection = direct_stratum.VersionRollingMaskSelection( - direct_stratum.QBIT_VERSION_ROLLING_MASK, - "fallback", - "test", - ) - return server - - -def submit_coordinator(tip: str = "00" * 32) -> tuple[PrismCoordinator, ClientState, RecordingLedger]: - server = coordinator() - # This suite runs on a synthetic millisecond timeline (declared anchors - # like 12000, share stamps of 1-2) while _expose_inflight_scan_anchor - # prunes exposures older than a real-wall-clock ceiling. Every synthetic - # anchor is decades past that ceiling, so the next exposure (a landing - # publishing its declared anchor) silently pruned a test's standing - # anchor and turned epoch-bump assertions into thread races against the - # landing's own anchor retirement. Pin the ceiling out of reach so - # exposed anchors live exactly as long as each test intends. - server.payout_artifact_max_anchor_age_seconds = float("inf") - server.vardiff_config = SimpleNamespace(enabled=False) - server.rpc = TipRpc(tip) - server.jobs = {} - server.recent_share_keys = set() - server.accepted_block_count = 0 - server.max_blocks = 1 - server.stop_after_block = True - server.extranonce2_size = 8 - server.share_weights_by_username = {"miner-a": 7} - ledger = RecordingLedger() - server.ledger = ledger - worker = WorkerIdentity( - username="miner-a", - payout_address="miner-a", - worker_name=None, - script_pubkey_hex="5220" + "11" * 32, - p2mr_program_hex="11" * 32, - ) - context = SimpleNamespace( - job=SimpleNamespace( - job_id="job-1", - share_target=target_from_compact("207fffff"), - share_difficulty=Decimal("1"), - transaction_hexes=(), - ), - template={"previousblockhash": tip, "height": 10, "coinbasevalue": 50_00000000}, - found_block={"network_difficulty": 1}, - issued_at_ms=12345, - collection_only=False, - worker=worker, - shares_json=[], - prior_balances=[], - ) - state = client() - state.username = "miner-a" - state.worker = worker - state.active_job_ids = {"job-1"} - server.jobs["job-1"] = context - return server, state, ledger - - -def block_candidate( - server: PrismCoordinator, - state: ClientState, - submission: object, - *, - job_id: str = "job-1", - pending_share: object | None = None, - credit_share_on_accept: bool = False, -) -> PrismBlockCandidate: - return PrismBlockCandidate( - context=server.jobs[job_id], - submission=submission, - extranonce1_hex=state.extranonce1_hex, - extranonce2_hex="00" * 8, - pending_share=pending_share - or SimpleNamespace( - share_id="miner-a:" + submission.block_hash_hex, - job_issued_at_ms=0, - accepted_at_ms=0, - ), - client=state, - credit_share_on_accept=credit_share_on_accept, - ) +def verified_audit_report( + coinbase_tx_hex: str = "c0ffee", + block_height: int = 10, +) -> dict[str, object]: + report = _compat_verified_audit_report(coinbase_tx_hex) + report["schema"] = "qbit.prism.audit-verification-report.v1" + report["block_height"] = block_height + report["coinbase_value_sats"] = 50_00000000 + return report class PrismCoordinatorVardiffTests(unittest.TestCase): - def test_load_prism_vardiff_config_defaults_to_small_miner_vardiff(self) -> None: - names = [name for name in os.environ if name.startswith("PRISM_STRATUM_VARDIFF")] - with patch.dict(os.environ, {}, clear=False): - for name in names: - os.environ.pop(name, None) - config = load_prism_vardiff_config(Decimal("0.000000001")) - - self.assertTrue(config.enabled) - self.assertEqual(config.target_share_interval_seconds, Decimal("15")) - self.assertEqual(config.min_difficulty, Decimal("1E-9")) - self.assertEqual(config.startup_difficulty, Decimal("1E-9")) - self.assertEqual(config.max_step_factor, Decimal("4")) - self.assertEqual(config.max_step_down_factor, Decimal("4")) - - def test_vardiff_step_up_retarget_sends_new_difficulty_without_flush(self) -> None: - server = coordinator() - state = client() - state.share_difficulty = Decimal("1") - state.vardiff_window_started_monotonic = time.monotonic() - 2 - sent: dict[str, object] = {"jobs": 0} + def test_audit_store_lazy_adopts_compatibility_fields_before_and_after_construction(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + base = Path(tmp) + server = PrismCoordinator.__new__(PrismCoordinator) + server.audit_dir = base / "audit-a" + server.evidence_path = base / "state-a" / "evidence.json" + server.audit_live_bundle_retention = 3 + server.audit_candidate_retention_seconds = 7 + server.audit_share_segment_size = 11 + first = server._ensure_audit_artifact_store() + self.assertEqual(first.root, server.audit_dir.resolve()) + self.assertEqual(first.evidence_path, server.evidence_path.resolve()) + self.assertEqual(first.live_bundle_retention, 3) + self.assertEqual(first.candidate_retention_seconds, 7) + self.assertEqual(first.share_segment_size, 11) + + server.audit_dir = base / "audit-b" + server.evidence_path = base / "state-b" / "evidence.json" + server.audit_live_bundle_retention = 5 + server.audit_candidate_retention_seconds = 13 + server.audit_share_segment_size = 17 + second = server._ensure_audit_artifact_store() + self.assertIs(second, first) + self.assertEqual(second.root, server.audit_dir.resolve()) + self.assertEqual(second.evidence_path, server.evidence_path.resolve()) + self.assertEqual(second.live_bundle_retention, 5) + self.assertEqual(second.candidate_retention_seconds, 13) + self.assertEqual(second.share_segment_size, 17) + + candidate = second.issue_candidate(block_hash="aa" * 32) + server.audit_dir = base / "audit-c" + with self.assertRaisesRegex(RuntimeError, "candidates are active"): + server._ensure_audit_artifact_store() + self.assertIs(server.__dict__["_audit_artifact_store"], first) + self.assertEqual(first.root, (base / "audit-b").resolve()) + first.discard_candidate(candidate) + + def test_coordinator_latest_evidence_seed_is_stable_and_defensive_before_and_after_store(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + server = PrismCoordinator.__new__(PrismCoordinator) + seed = {"nested": {"value": 1}} + server.latest_evidence = seed + seed["nested"]["value"] = 2 + before = server.latest_evidence + self.assertEqual(before, {"nested": {"value": 1}}) + assert before is not None + before["nested"]["value"] = 3 + self.assertEqual(server.latest_evidence, {"nested": {"value": 1}}) + + root = Path(tmp) + server.audit_dir = root / "audit" + server.evidence_path = root / "state" / "evidence.json" + store = server._ensure_audit_artifact_store() + self.assertNotIn("_audit_latest_evidence_seed", server.__dict__) + self.assertEqual(server.latest_evidence_payload(), {"nested": {"value": 1}}) + snapshot = server.latest_evidence_payload() + assert snapshot is not None + snapshot["nested"]["value"] = 4 # type: ignore[index] + self.assertEqual(server.latest_evidence_payload(), {"nested": {"value": 1}}) + server.latest_evidence = None + self.assertIsNone(server.latest_evidence_payload()) + server.latest_evidence = {"after": {"value": 5}} + self.assertEqual(server.latest_evidence_payload(), {"after": {"value": 5}}) + self.assertIs(server._ensure_audit_artifact_store(), store) + + def test_coordinator_audit_static_and_instance_facades_preserve_contract(self) -> None: + server = PrismCoordinator.__new__(PrismCoordinator) + sentinel = mock.Mock() + sentinel.metrics_snapshot.return_value = {"scan_error": 0} + sentinel.prune_best_effort.return_value = RetentionResult(live_removed=1) + sentinel.verify_bundle.return_value = {"verified": True} + server._ensure_audit_artifact_store = lambda: sentinel # type: ignore[method-assign] - def fake_send_job(client: object, clean_jobs: bool) -> bool: - sent.update({"jobs": sent["jobs"] + 1, "clean": clean_jobs}) - return True + self.assertEqual( + PrismCoordinator.audit_artifact_kind( + f"prism-live-audit-bundle-1-{'aa' * 32}.json" + ), + "live_bundle", + ) + self.assertEqual(server.audit_artifact_metrics(), {"scan_error": 0}) + keep = Path("keep.json") + server.prune_audit_artifacts(keep_live_path=keep) + sentinel.prune_best_effort.assert_called_once_with(keep_live_path=keep) + result = PrismCoordinator.verify_bundle( + server, + Path("bundle.json"), + "00", + "11" * 32, + expected_coinbase_value_sats=5, + expected_block_height=6, + ) + self.assertEqual(result, {"verified": True}) + sentinel.verify_bundle.assert_called_once_with( + Path("bundle.json"), + "00", + "11" * 32, + expected_coinbase_value_sats=5, + expected_block_height=6, + ) + server.ledger_writer_public_key_hex = "22" * 32 + self.assertEqual( + server.trusted_ledger_writer_public_key_hex({}), + "22" * 32, + ) - server.maybe_send_job = fake_send_job # type: ignore[method-assign] + with tempfile.TemporaryDirectory() as tmp: + candidate = Path(tmp) / "candidate.json" + candidate.write_bytes(b"canonical") + digest = hashlib.sha256(b"canonical").hexdigest() + self.assertEqual( + PrismCoordinator.verified_canonical_bundle_path( + candidate, + {"audit_bundle_sha256_hex": digest.upper()}, + ), + candidate, + ) - server.note_vardiff_submitted_share(state) - server.note_vardiff_accepted_share(state, FakeJob(Decimal("1"))) # type: ignore[arg-type] + def test_candidate_verifier_override_is_resolved_after_store_construction(self) -> None: + server, state, ledger = submit_coordinator() + server.max_blocks = 10 + server.stop_after_block = False + with tempfile.TemporaryDirectory() as tmp: + server.audit_dir = Path(tmp) + server.evidence_path = Path(tmp) / "evidence.json" + server.ledger_writer_public_key_hex = "aa" * 32 + store = server._ensure_audit_artifact_store() + calls: list[str] = [] - # Difficulty is advertised by maybe_send_job alongside the job (gated on - # a successful build). A step-up must not flush the miner's in-flight - # work: old jobs validate at their own stamped share_target, so the - # retarget requests a single non-clean job. - self.assertEqual(state.pending_share_difficulty, Decimal("4")) - self.assertEqual(sent["jobs"], 1) - self.assertFalse(sent["clean"]) + def verifier_a(*_args: object, **_kwargs: object) -> dict[str, object]: + calls.append("a") + return verified_audit_report() - def test_vardiff_step_down_retarget_flushes_with_clean_job(self) -> None: - # Firmware that applies mining.set_difficulty retroactively would - # submit sub-target shares against the old job after a step-down, so - # the step-down keeps flushing in-flight work. - server = coordinator() - state = client() - state.share_difficulty = Decimal("16") - state.vardiff_window_started_monotonic = time.monotonic() - 65 - sent: dict[str, object] = {"jobs": 0} + def verifier_b(*_args: object, **_kwargs: object) -> dict[str, object]: + calls.append("b") + return verified_audit_report() - def fake_send_job(client: object, clean_jobs: bool) -> bool: - sent.update({"jobs": sent["jobs"] + 1, "clean": clean_jobs}) - return True + server.verify_bundle = verifier_a # type: ignore[method-assign] + server.verify_bundle = verifier_b # type: ignore[method-assign] + block_hash = "91" * 32 + server.rpc = SubmitRpc( + tip="00" * 32, + block_hash=block_hash, + ledger=ledger, + ) + server.build_audit_bundle = ( # type: ignore[method-assign] + lambda **_kwargs: verified_block_bundle() + ) + submission = SimpleNamespace( + coinbase_tx_hex="c0ffee", + block_hash_hex=block_hash, + block_hex="00", + ) + self.assertTrue( + server.submit_block_candidate( + block_candidate(server, state, submission) + ) + ) + self.assertIs(server._ensure_audit_artifact_store(), store) + self.assertEqual(calls, ["b"]) + + direct_server, direct_state, direct_ledger = submit_coordinator() + direct_server.max_blocks = 10 + direct_server.stop_after_block = False + direct_server.audit_dir = Path(tmp) / "direct-a1" + direct_server.evidence_path = Path(tmp) / "direct-a1-evidence.json" + direct_server.ledger_writer_public_key_hex = "aa" * 32 + direct_store = direct_server._ensure_audit_artifact_store() + direct_calls: list[str] = [] + + def direct_a1_verifier( + *_args: object, + **_kwargs: object, + ) -> dict[str, object]: + direct_calls.append("a1") + return verified_audit_report() - server.maybe_send_job = fake_send_job # type: ignore[method-assign] + direct_store.verify_bundle = direct_a1_verifier # type: ignore[method-assign] + self.assertNotIn("verify_bundle", direct_server.__dict__) + direct_hash = "93" * 32 + direct_server.rpc = SubmitRpc( + tip="00" * 32, + block_hash=direct_hash, + ledger=direct_ledger, + ) + direct_server.build_audit_bundle = ( # type: ignore[method-assign] + lambda **_kwargs: verified_block_bundle() + ) + direct_submission = SimpleNamespace( + coinbase_tx_hex="c0ffee", + block_hash_hex=direct_hash, + block_hex="00", + ) + self.assertTrue( + direct_server.submit_block_candidate( + block_candidate( + direct_server, + direct_state, + direct_submission, + ) + ) + ) + self.assertEqual(direct_calls, ["a1"]) - server.note_vardiff_submitted_share(state) - server.note_vardiff_accepted_share(state, FakeJob(Decimal("16"))) # type: ignore[arg-type] + def test_audit_publication_occurs_after_durable_confirm_and_before_success_tail(self) -> None: + server, state, ledger = submit_coordinator() + events: list[str] = [] + with tempfile.TemporaryDirectory() as tmp: + server.audit_dir = Path(tmp) + server.evidence_path = Path(tmp) / "evidence.json" + server.ledger_writer_public_key_hex = "aa" * 32 + store = server._ensure_audit_artifact_store() + payout_service = server._ensure_payout_state_service() + real_persist = ledger.persist_accepted_block + real_confirm = ledger.confirm_accepted_block + real_floor = ledger.audit_publication_sequence_floor + real_identity = server._audit_publication_identity + real_publish = store.publish_success + real_shutdown = server.request_shutdown - self.assertEqual(state.pending_share_difficulty, Decimal("4")) - self.assertEqual(sent["jobs"], 1) - self.assertTrue(sent["clean"]) + def verify(*_args: object, **_kwargs: object) -> dict[str, object]: + events.append("verify") + return verified_audit_report() - def test_step_up_retarget_keeps_old_job_submittable_at_old_difficulty(self) -> None: - tip = "00" * 32 - server = coordinator() - server.accepted_block_count = 0 - server.max_blocks = 1 - server.stop_after_block = True - server.jobs = {} - server.recent_share_keys = set() - server.share_weights_by_username = {} - ledger = RecordingLedger() - server.ledger = ledger - worker = worker_identity() - state = client() - state.username = worker.username - state.worker = worker - state.share_difficulty = Decimal("1") - state.vardiff_window_started_monotonic = time.monotonic() - 2 - sent: list[dict[str, object]] = [] - state.send = lambda payload: sent.append(payload) # type: ignore[method-assign] - server.clients = {state} - server.rpc = TipTemplateRpc(tip=tip, template=gbt_template(tip)) + def persist(**kwargs: object) -> dict[str, object]: + events.append("persist") + return real_persist(**kwargs) - old_context = prism_context("old-job", tip, worker=worker, difficulty=Decimal("1")) - state.active_job = old_context - state.active_job_ids = {"old-job"} - server.jobs["old-job"] = old_context + def confirm(**kwargs: object) -> dict[str, object]: + self.assertTrue( + payout_service._payout_balance_mutation_lock._is_owned(), # type: ignore[attr-defined] + ) + self.assertEqual(store._publication_guard_owner, threading.get_ident()) + events.append("confirm") + return real_confirm(**kwargs) - def build_fresh_job(client: ClientState, *, clean_jobs: bool) -> object: - return prism_context( - "fresh-job", - tip, - worker=worker, - difficulty=client.pending_share_difficulty or client.share_difficulty, - clean_jobs=clean_jobs, - ) + def publication_identity(**kwargs: object) -> AuditPublicationIdentity: + self.assertTrue( + payout_service._payout_balance_mutation_lock._is_owned(), # type: ignore[attr-defined] + ) + self.assertEqual(store._publication_guard_owner, threading.get_ident()) + events.append("publication_identity") + return real_identity(**kwargs) - server.build_job_for_client = build_fresh_job # type: ignore[method-assign] + def publication_floor() -> int: + self.assertTrue( + payout_service._payout_balance_mutation_lock._is_owned(), # type: ignore[attr-defined] + ) + self.assertEqual(store._publication_guard_owner, threading.get_ident()) + events.append("publication_floor") + return real_floor() - server.note_vardiff_submitted_share(state) - server.note_vardiff_accepted_share(state, FakeJob(Decimal("1"))) # type: ignore[arg-type] + def publish(**kwargs: object) -> object: + self.assertTrue( + payout_service._payout_balance_mutation_lock._is_owned(), # type: ignore[attr-defined] + ) + self.assertEqual(store._publication_guard_owner, threading.get_ident()) + events.append("publish_success") + return real_publish(**kwargs) + + def terminal_success() -> None: + events.append("terminal_success") + real_shutdown() + + server.verify_bundle = verify # type: ignore[method-assign] + ledger.persist_accepted_block = persist # type: ignore[method-assign] + ledger.confirm_accepted_block = confirm # type: ignore[method-assign] + ledger.audit_publication_sequence_floor = publication_floor # type: ignore[method-assign] + server._audit_publication_identity = publication_identity # type: ignore[method-assign] + store.publish_success = publish # type: ignore[method-assign] + server.request_shutdown = terminal_success # type: ignore[method-assign] + block_hash = "92" * 32 + server.rpc = SubmitRpc( + tip="00" * 32, + block_hash=block_hash, + ledger=ledger, + ) + server.build_audit_bundle = ( # type: ignore[method-assign] + lambda **_kwargs: verified_block_bundle() + ) + submission = SimpleNamespace( + coinbase_tx_hex="c0ffee", + block_hash_hex=block_hash, + block_hex="00", + ) + self.assertTrue( + server.submit_block_candidate( + block_candidate(server, state, submission) + ) + ) + self.assertEqual( + events, + [ + "verify", + "persist", + "confirm", + "publication_identity", + "publication_floor", + "publish_success", + "terminal_success", + ], + ) - # The 1 -> 4 step-up pairs the new difficulty with a non-clean job and - # keeps the old job registered and submittable. - self.assertEqual(state.share_difficulty, Decimal("4")) - self.assertIsNone(state.pending_share_difficulty) - self.assertEqual( - [payload["method"] for payload in sent], - ["mining.set_difficulty", "mining.notify"], - ) - self.assertEqual(sent[0]["params"], [4.0]) - self.assertEqual(sent[1]["params"][0], "fresh-job") - self.assertFalse(sent[1]["params"][8]) - self.assertIn("old-job", server.jobs) - self.assertIn("fresh-job", server.jobs) - self.assertEqual(state.active_job_ids, {"old-job", "fresh-job"}) + def test_audit_publication_failure_after_confirm_retries_same_ordinal_once(self) -> None: + server, state, ledger = submit_coordinator() + block_hash = "94" * 32 + with tempfile.TemporaryDirectory() as tmp: + server.audit_dir = Path(tmp) + server.evidence_path = Path(tmp) / "evidence.json" + server.ledger_writer_public_key_hex = "aa" * 32 + store = server._ensure_audit_artifact_store() + server.rpc = SubmitRpc( + tip="00" * 32, + block_hash=block_hash, + ledger=ledger, + ) + server.build_audit_bundle = ( # type: ignore[method-assign] + lambda **_kwargs: verified_block_bundle() + ) + server.verify_bundle = ( # type: ignore[method-assign] + lambda *_args, **_kwargs: verified_audit_report() + ) + real_confirm = ledger.confirm_accepted_block + real_publish = store.publish_success + real_shutdown = server.request_shutdown + confirmation_sequences: list[int] = [] + publication_sequences: list[int] = [] + terminal_successes: list[str] = [] + publication_attempts = 0 + + def confirm(**kwargs: object) -> dict[str, object]: + result = real_confirm(**kwargs) + result["audit_publication_sequence"] = 7 + confirmation_sequences.append(7) + return result - # In-flight work against the old job still lands, validated at the old - # job's own stamped difficulty: this share passes the diff-1 target it - # was mined against, not the diff-4 target advertised afterward. - def assemble_at_stamped_difficulty(job: object, **_kwargs: object) -> SimpleNamespace: - return SimpleNamespace( - header_hex="aa" * 80, - block_hash_hex="bb" * 32, - share_pass=job.share_difficulty <= Decimal("1"), - block_pass=False, + def publish(**kwargs: object) -> object: + nonlocal publication_attempts + publication_attempts += 1 + identity = kwargs["identity"] + assert isinstance(identity, AuditPublicationIdentity) + publication_sequences.append(identity.sequence) + if publication_attempts == 1: + raise RuntimeError("injected evidence publication failure") + return real_publish(**kwargs) + + def terminal_success() -> None: + terminal_successes.append("success") + real_shutdown() + + ledger.confirm_accepted_block = confirm # type: ignore[method-assign] + ledger.audit_publication_sequence_floor = lambda: 7 # type: ignore[method-assign] + store.publish_success = publish # type: ignore[method-assign] + server.request_shutdown = terminal_success # type: ignore[method-assign] + submission = SimpleNamespace( + coinbase_tx_hex="c0ffee", + block_hash_hex=block_hash, + block_hex="00", ) + candidate = block_candidate(server, state, submission) - with patch( - "lab.prism.prism_coordinator.direct_stratum.assemble_submission", - side_effect=assemble_at_stamped_difficulty, - ): - should_close = server.handle_submit( - state, - ["miner-a", "old-job", "00" * 8, "00000001", "00000002"], + with self.assertRaisesRegex( + RuntimeError, + "injected evidence publication failure", + ): + server.submit_block_candidate(candidate) + self.assertEqual(confirmation_sequences, [7]) + self.assertEqual(publication_sequences, [7]) + self.assertIsNone(store.latest_evidence()) + self.assertEqual(server.accepted_block_count, 0) + self.assertEqual(terminal_successes, []) + + assert isinstance(server.rpc, SubmitRpc) + server.rpc.tip = block_hash + self.assertTrue(server.submit_block_candidate(candidate)) + self.assertEqual(confirmation_sequences, [7, 7]) + self.assertEqual(publication_sequences, [7, 7]) + self.assertEqual(server.accepted_block_count, 1) + self.assertEqual(terminal_successes, ["success"]) + latest = store.latest_evidence() + self.assertIsNotNone(latest) + assert latest is not None + self.assertEqual( + latest["audit_publication_identity"]["sequence"], # type: ignore[index] + 7, ) - self.assertFalse(should_close) - self.assertEqual(len(ledger.pending), 1) - self.assertEqual(ledger.pending[0].job_id, "old-job") - self.assertEqual(server.low_difficulty_share_count, 0) - self.assertEqual(server.rejection_counts_by_reason[PRISM_REJECTION_LOW_DIFFICULTY], 0) - self.assertEqual(server.rejection_counts_by_reason[PRISM_REJECTION_UNKNOWN_JOB], 0) - self.assertEqual(server.rejection_counts_by_reason[PRISM_REJECTION_STALE_JOB], 0) - - def test_vardiff_retarget_build_failure_keeps_consistent_difficulty_and_job(self) -> None: - # If the job build is skipped during a retarget, the client must stay on its - # existing job at its existing difficulty -- never advertise a new difficulty - # for a job it never received. Otherwise its easier shares miss the old - # target, nothing is accepted, and (since retargets only fire on accepted - # shares) it cannot self-heal without reconnecting. - server = coordinator() - server.jobs = {"old-job": SimpleNamespace(job=SimpleNamespace(job_id="old-job"))} - server.extranonce2_size = 8 - state = client() - state.username = "miner-a" - state.worker = WorkerIdentity( - username="miner-a", - payout_address="miner-a", - worker_name=None, - script_pubkey_hex="5220" + "11" * 32, - p2mr_program_hex="11" * 32, + def test_publication_floor_failure_or_mismatch_never_publishes_evidence(self) -> None: + cases = ( + ("query_failure", RuntimeError("floor query failed"), "floor query failed"), + ("below_identity", 0, "exceeds"), + ("above_identity", 2, "behind"), + ("invalid_bool", True, "floor sequence"), ) - state.share_difficulty = Decimal("1") - state.active_job_ids = {"old-job"} - state.vardiff_window_started_monotonic = time.monotonic() - 2 - advertised: list[object] = [] - state.send = lambda payload: advertised.append(payload) # type: ignore[method-assign] - - def failing_build(client: ClientState, *, clean_jobs: bool) -> object: - raise ValueError("transient getblocktemplate failure") + for name, floor_result, error_pattern in cases: + with self.subTest(name=name), tempfile.TemporaryDirectory() as tmp: + server, state, ledger = submit_coordinator() + block_hash = "95" * 32 + server.audit_dir = Path(tmp) + server.evidence_path = Path(tmp) / "evidence.json" + server.ledger_writer_public_key_hex = "aa" * 32 + store = server._ensure_audit_artifact_store() + server.rpc = SubmitRpc( + tip="00" * 32, + block_hash=block_hash, + ledger=ledger, + ) + server.build_audit_bundle = ( # type: ignore[method-assign] + lambda **_kwargs: verified_block_bundle() + ) + server.verify_bundle = ( # type: ignore[method-assign] + lambda *_args, **_kwargs: verified_audit_report() + ) + publish_calls = 0 + real_publish = store.publish_success - server.build_job_for_client = failing_build # type: ignore[method-assign] + def publication_floor() -> int: + if isinstance(floor_result, BaseException): + raise floor_result + return floor_result # type: ignore[return-value] - server.note_vardiff_submitted_share(state) - server.note_vardiff_accepted_share(state, FakeJob(Decimal("1"))) # type: ignore[arg-type] + def publish(**kwargs: object) -> object: + nonlocal publish_calls + publish_calls += 1 + return real_publish(**kwargs) - self.assertEqual(server.job_build_failure_count, 1) - self.assertIsNone(state.pending_share_difficulty) # rolled back, not left at the new value - self.assertEqual(state.share_difficulty, Decimal("1")) # unchanged - self.assertEqual(state.active_job_ids, {"old-job"}) # old job retained, still submittable - self.assertEqual(set(server.jobs), {"old-job"}) - self.assertEqual(advertised, []) # no set_difficulty / notify advertised for the skipped build + ledger.audit_publication_sequence_floor = publication_floor # type: ignore[method-assign] + store.publish_success = publish # type: ignore[method-assign] + submission = SimpleNamespace( + coinbase_tx_hex="c0ffee", + block_hash_hex=block_hash, + block_hex="00", + ) + with self.assertRaisesRegex( + (RuntimeError, ValueError), + error_pattern, + ): + server.submit_block_candidate( + block_candidate(server, state, submission) + ) + self.assertEqual( + publish_calls, + 0 if name == "query_failure" else 1, + ) + self.assertIsNone(store.latest_evidence()) + self.assertEqual(server.accepted_block_count, 0) - def test_idle_vardiff_success_sends_paired_job_and_resets_window(self) -> None: - server = coordinator() - state = client() - prepare_idle_client(server, state) - install_idle_job_cache(server) - sent: list[dict[str, object]] = [] - delivered = threading.Event() - window_started = state.vardiff_window_started_monotonic + def test_make_ledger_preserves_single_a1_store_identity(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + server = PrismCoordinator.__new__(PrismCoordinator) + server.audit_dir = Path(tmp) / "audit-a" + server.evidence_path = Path(tmp) / "state-a" / "evidence.json" + store = server._ensure_audit_artifact_store() + ledger = SimpleNamespace() + with mock.patch.dict( + os.environ, + {"PRISM_POSTGRES_PSQL_COMMAND": "psql example"}, + clear=True, + ), mock.patch( + "lab.prism.prism_coordinator.PsqlShareLedger", + return_value=ledger, + ) as constructor: + self.assertIs(server.make_ledger(), ledger) + self.assertIs( + constructor.call_args.kwargs["audit_artifact_store"], + store, + ) + server.audit_dir = Path(tmp) / "audit-b" + server.evidence_path = Path(tmp) / "state-b" / "evidence.json" + self.assertIs(server._ensure_audit_artifact_store(), store) + self.assertEqual(store.root, server.audit_dir.resolve()) - def record(payload: dict[str, object]) -> None: - sent.append(payload) - if payload.get("method") == "mining.notify": - delivered.set() + memory_server = PrismCoordinator.__new__(PrismCoordinator) + with mock.patch.dict( + os.environ, + {"PRISM_ALLOW_MEMORY_LEDGER": "1"}, + clear=True, + ): + self.assertIsInstance( + memory_server.make_ledger(), + SingleWriterShareLedger, + ) + self.assertNotIn("_audit_artifact_store", memory_server.__dict__) - state.send = record # type: ignore[method-assign] + def test_legacy_evidence_upgrade_requires_exact_durable_pool_block_proof(self) -> None: + legacy_identity = AuditPublicationIdentity(0, 10, "aa" * 32) - self.assertEqual(server.vardiff_idle_sweep_once(), 1) - self.assertTrue(delivered.wait(timeout=1)) - server.shutdown_vardiff_idle_executor() + for maturity_state in ("immature", "mature"): + with self.subTest(valid_maturity=maturity_state): + server = coordinator() + store = mock.MagicMock() + store.legacy_evidence_identity.return_value = legacy_identity + server._ensure_audit_artifact_store = lambda: store # type: ignore[method-assign] + server.ledger = SimpleNamespace( + audit_publication_sequence_floor=lambda: 7, + pool_block_state=lambda **_kwargs: { + "block_hash": legacy_identity.block_hash, + "block_height": legacy_identity.block_height, + "chain_state": "confirmed", + "maturity_state": maturity_state, + "audit_publication_sequence": 7, + } + ) + server._upgrade_legacy_audit_evidence() + store.adopt_legacy_publication_identity.assert_called_once_with( + AuditPublicationIdentity(7, 10, legacy_identity.block_hash), + publication_floor_sequence=7, + ) + store.invalidate_unprovable_legacy_evidence.assert_not_called() - self.assertEqual(server.idle_retarget_count, 1) - self.assertEqual( - [payload["method"] for payload in sent], - ["mining.set_difficulty", "mining.notify"], - ) - self.assertEqual(sent[0]["params"], [4.0]) - self.assertTrue(sent[1]["params"][8]) - self.assertEqual(state.share_difficulty, Decimal("4")) - self.assertIsNone(state.pending_share_difficulty) - self.assertGreater(state.vardiff_window_started_monotonic, window_started) - self.assertEqual(state.vardiff_window_accepted, 0) - self.assertEqual(state.vardiff_window_submitted, 0) - self.assertEqual(state.vardiff_window_work, Decimal("0")) + invalid_states = ( + None, + { + "block_height": 10, + "chain_state": "confirmed", + "maturity_state": "immature", + "audit_publication_sequence": 7, + }, + { + "block_hash": legacy_identity.block_hash.upper(), + "block_height": 10, + "chain_state": "confirmed", + "maturity_state": "immature", + "audit_publication_sequence": 7, + }, + { + "block_hash": legacy_identity.block_hash, + "block_height": "10", + "chain_state": "confirmed", + "maturity_state": "immature", + "audit_publication_sequence": 7, + }, + { + "block_hash": legacy_identity.block_hash, + "block_height": True, + "chain_state": "confirmed", + "maturity_state": "immature", + "audit_publication_sequence": 7, + }, + { + "block_height": 10, + "chain_state": "inactive", + "maturity_state": "immature", + "audit_publication_sequence": 7, + }, + { + "block_height": 10, + "chain_state": "confirmed", + "maturity_state": "reversed", + "audit_publication_sequence": 7, + }, + { + "block_height": 10, + "chain_state": "confirmed", + "maturity_state": "unknown", + "audit_publication_sequence": 7, + }, + { + "block_height": 11, + "chain_state": "confirmed", + "maturity_state": "immature", + "audit_publication_sequence": 7, + }, + { + "block_height": 10, + "chain_state": "confirmed", + "maturity_state": "immature", + "audit_publication_sequence": True, + }, + ) + for state in invalid_states: + with self.subTest(invalid_state=state): + server = coordinator() + store = mock.MagicMock() + store.legacy_evidence_identity.return_value = legacy_identity + server._ensure_audit_artifact_store = lambda: store # type: ignore[method-assign] + server.ledger = SimpleNamespace( + pool_block_state=lambda **_kwargs: state + ) + server._upgrade_legacy_audit_evidence() + store.invalidate_unprovable_legacy_evidence.assert_called_once_with() + store.adopt_legacy_publication_identity.assert_not_called() - def test_idle_vardiff_shutdown_after_delivery_keeps_committed_window(self) -> None: server = coordinator() - state = client() - prepare_idle_client(server, state) - install_idle_job_cache(server) - sent: list[dict[str, object]] = [] - delivered = threading.Event() - window_started = state.vardiff_window_started_monotonic - - def stop_after_delivery(payload: dict[str, object]) -> None: - sent.append(payload) - if payload.get("method") == "mining.notify": - server.stop_event.set() - delivered.set() + store = mock.MagicMock() + store.legacy_evidence_identity.return_value = legacy_identity + server._ensure_audit_artifact_store = lambda: store # type: ignore[method-assign] + server.ledger = SingleWriterShareLedger() + server._upgrade_legacy_audit_evidence() + store.invalidate_unprovable_legacy_evidence.assert_called_once_with() - state.send = stop_after_delivery # type: ignore[method-assign] + def test_build_audit_bundle_passes_pool_fee_policy_to_cli_payload(self) -> None: + server = coordinator() + server.rpc = AddressRpc(valid_address="tq1fee", script_byte="99") + server.signing_seed_hex = "42" * 32 + server.ledger_attestation_signing_seed_hex = "43" * 32 + captured: dict[str, object] = {} - self.assertEqual(server.vardiff_idle_sweep_once(), 1) - self.assertTrue(delivered.wait(timeout=1)) - server.shutdown_vardiff_idle_executor() + with patch.dict( + os.environ, + { + "PRISM_POOL_FEE_ENABLED": "1", + "PRISM_POOL_FEE_BPS": "125", + "PRISM_POOL_FEE_ADDRESS": "tq1fee", + }, + clear=True, + ), patch( + "lab.prism.prism_coordinator.subprocess.Popen", + fake_audit_bundle_popen(captured), + ): + bundle = server.build_audit_bundle( + shares=[], + found_block={"block_height": 10, "coinbase_value_sats": 50_00000000}, + prior_balances=[], + coinbase_script_sig_suffix_hex="00", + ) + self.assertEqual(bundle, {"ok": True}) + self.assertEqual(captured["payload"]["payout_policy"]["pool_fee_policy"]["fee_bps"], 125) self.assertEqual( - [payload["method"] for payload in sent], - ["mining.set_difficulty", "mining.notify"], + captured["payload"]["payout_policy"]["pool_fee_policy"]["p2mr_program_hex"], + "99" * 32, ) - self.assertEqual(server.idle_retarget_count, 1) - self.assertEqual(state.share_difficulty, Decimal("4")) - self.assertIsNone(state.pending_share_difficulty) - self.assertGreater(state.vardiff_window_started_monotonic, window_started) - self.assertEqual(state.vardiff_window_accepted, 0) - self.assertEqual(state.vardiff_window_submitted, 0) - self.assertEqual(state.vardiff_window_work, Decimal("0")) - def test_idle_vardiff_sweep_skips_submitted_reject_storm_window(self) -> None: + def test_build_audit_bundle_passes_ctv_settlement_config_to_cli_payload(self) -> None: server = coordinator() - state = client() - state.worker = worker_identity() - state.active_job = prism_context("job-1", "00" * 32, worker=state.worker) - state.share_difficulty = Decimal("16") - state.vardiff_window_started_monotonic = time.monotonic() - 2 - state.vardiff_window_submitted = 3 - server.clients = {state} + server.signing_seed_hex = "42" * 32 + server.ledger_attestation_signing_seed_hex = "43" * 32 + captured: dict[str, object] = {} + + with patch.dict( + os.environ, + { + "PRISM_CTV_SETTLEMENT_ENABLED": "1", + "PRISM_DIRECT_COINBASE_PAYOUT_FLOOR_BITS": "10485760", + "PRISM_MAX_COINBASE_SETTLEMENT_OUTPUTS": "16", + "PRISM_MAX_DIRECT_COINBASE_OUTPUTS": "12", + "PRISM_MAX_CTV_FANOUT_RECIPIENTS_PER_TRANSACTION": "1000", + "PRISM_CTV_FANOUT_FEE_MARKET_RATE_BITS_PER_1000_WEIGHT": "25", + "PRISM_CTV_FANOUT_FEE_PREMIUM_BPS": "12000", + }, + clear=True, + ), patch( + "lab.prism.prism_coordinator.subprocess.Popen", + fake_audit_bundle_popen(captured), + ): + bundle = server.build_audit_bundle( + shares=[], + found_block={"block_height": 10, "coinbase_value_sats": 50_00000000}, + prior_balances=[], + coinbase_script_sig_suffix_hex="00", + ) - def fail_send_job(client: ClientState, *, clean_jobs: bool) -> bool: - raise AssertionError("reject-storm windows must not idle-retarget") + self.assertEqual(bundle, {"ok": True}) + self.assertEqual( + captured["payload"]["ctv_settlement"], + { + "direct_floor_sats": 10_485_760, + "config": { + "max_coinbase_settlement_outputs": 16, + "max_direct_coinbase_outputs": 12, + "max_fanout_recipients_per_transaction": 1000, + "reserved_coinbase_outputs": 0, + }, + "fanout_fee_rate_policy": { + "market_fee_rate_sats_per_1000_weight": 25, + "premium_bps": 12_000, + }, + }, + ) - server.maybe_send_job = fail_send_job # type: ignore[method-assign] + def test_build_audit_bundle_preserves_exact_canonical_output_file(self) -> None: + server = coordinator() + server.signing_seed_hex = "42" * 32 + server.ledger_attestation_signing_seed_hex = "43" * 32 + canonical_bytes = b'{"ok":true,"nested":[1,2]}' + captured: dict[str, object] = {} - retargeted = server.vardiff_idle_sweep_once() + with tempfile.TemporaryDirectory() as tmp, patch( + "lab.prism.prism_coordinator.subprocess.Popen", + fake_audit_bundle_popen( + captured, + output_text=canonical_bytes.decode("utf-8"), + ), + ): + output_path = Path(tmp) / "candidate.audit.json" + bundle = server.build_audit_bundle( + shares=[], + found_block={"block_height": 10, "coinbase_value_sats": 50_00000000}, + prior_balances=[], + coinbase_script_sig_suffix_hex="00", + canonical_output_path=output_path, + ) - self.assertEqual(retargeted, 0) - self.assertEqual(state.pending_share_difficulty, None) - self.assertEqual(state.vardiff_window_submitted, 3) + self.assertEqual(bundle, {"ok": True, "nested": [1, 2]}) + self.assertEqual(output_path.read_bytes(), canonical_bytes) + self.assertIn("--canonical-output", captured["cmd"]) + self.assertEqual(captured["payload"]["shares"], []) - def test_idle_vardiff_share_after_snapshot_is_not_stepped_down(self) -> None: + def test_build_audit_bundle_transfers_the_exact_open_output_inode(self) -> None: server = coordinator() - blockers = [client(), client()] - for connection_id, blocker in enumerate(blockers, start=1): - prepare_idle_client(server, blocker, connection_id=connection_id) - install_idle_job_cache(server) - workers_started = threading.Barrier(3) - release_workers = threading.Event() - - def block_delivery(payload: dict[str, object]) -> None: - if payload.get("method") == "mining.set_difficulty": - workers_started.wait(timeout=1) - release_workers.wait(timeout=1) - - for blocker in blockers: - blocker.send = block_delivery # type: ignore[method-assign] - - target = client() - target.connection_id = 3 - sent: list[dict[str, object]] = [] - target.send = sent.append # type: ignore[method-assign] - try: - self.assertEqual(server.vardiff_idle_sweep_once(), 2) - workers_started.wait(timeout=1) - prepare_idle_client(server, target, connection_id=3) - self.assertEqual(server.vardiff_idle_sweep_once(), 1) - - # The task is queued from an idle snapshot. A later submit changes - # that exact window before any worker can commit the step-down. - server.note_vardiff_submitted_share(target) - finally: - release_workers.set() - server.shutdown_vardiff_idle_executor() + server.signing_seed_hex = "42" * 32 + server.ledger_attestation_signing_seed_hex = "43" * 32 + captured: dict[str, object] = {} + adopted: list[tuple[Path, int, int]] = [] - self.assertEqual(sent, []) - self.assertIsNone(target.pending_share_difficulty) - self.assertEqual(target.share_difficulty, Decimal("16")) - self.assertEqual(target.vardiff_window_submitted, 1) - self.assertGreaterEqual(server.vardiff_idle_skip_counts["not_idle"], 1) + def adopt(path: Path, value: os.stat_result) -> None: + current = path.stat() + self.assertEqual((value.st_dev, value.st_ino), (current.st_dev, current.st_ino)) + adopted.append((path, value.st_dev, value.st_ino)) - def test_idle_vardiff_failure_restores_pending_and_idle_window(self) -> None: - server = coordinator() - state = client() - prepare_idle_client(server, state) - install_idle_job_cache(server) - window_state = ( - state.vardiff_window_started_monotonic, - state.vardiff_window_accepted, - state.vardiff_window_submitted, - state.vardiff_window_work, - ) - disconnected: list[ClientState] = [] - failure_finished = threading.Event() - - def failing_send(_payload: dict[str, object]) -> None: - self.assertEqual(state.pending_share_difficulty, Decimal("4")) - raise OSError("socket send failed") - - def fake_disconnect(client: ClientState) -> None: - disconnected.append(client) - server.clients.discard(client) - failure_finished.set() - - state.send = failing_send # type: ignore[method-assign] - server.disconnect_client = fake_disconnect # type: ignore[method-assign] - - self.assertEqual(server.vardiff_idle_sweep_once(), 1) - self.assertTrue(failure_finished.wait(timeout=1)) - server.shutdown_vardiff_idle_executor() - - self.assertEqual(disconnected, [state]) - self.assertIsNone(state.pending_share_difficulty) - self.assertEqual(state.share_difficulty, Decimal("16")) - self.assertEqual( - ( - state.vardiff_window_started_monotonic, - state.vardiff_window_accepted, - state.vardiff_window_submitted, - state.vardiff_window_work, - ), - window_state, - ) - self.assertEqual(server.vardiff_idle_task_failures, 1) + with tempfile.TemporaryDirectory() as tmp, patch( + "lab.prism.bundle_compiler.subprocess.Popen", + fake_audit_bundle_popen(captured), + ): + output_path = Path(tmp) / "candidate.audit.json" + server.build_audit_bundle( + shares=[], + found_block={"block_height": 10, "coinbase_value_sats": 50_00000000}, + prior_balances=[], + coinbase_script_sig_suffix_hex="00", + canonical_output_path=output_path, + canonical_output_adopter=adopt, + ) - def test_idle_vardiff_stamp_failure_restores_speculative_state(self) -> None: - server = coordinator() - state = client() - prepare_idle_client(server, state) - install_idle_job_cache(server) - window_state = ( - state.vardiff_window_started_monotonic, - state.vardiff_window_accepted, - state.vardiff_window_submitted, - state.vardiff_window_work, - ) + self.assertEqual(len(adopted), 1) + self.assertEqual(adopted[0][0], output_path) + self.assertEqual(adopted[0][1:], (output_path.stat().st_dev, output_path.stat().st_ino)) - def fail_stamp(*_args: object, **_kwargs: object) -> None: - self.assertEqual(state.pending_share_difficulty, Decimal("4")) - raise RuntimeError("cached job stamping failed") + def test_build_audit_bundle_pinned_parent_never_recreates_stale_path(self) -> None: + server = coordinator() + server.signing_seed_hex = "42" * 32 + server.ledger_attestation_signing_seed_hex = "43" * 32 + captured: dict[str, object] = {} - state.send = lambda payload: self.fail( # type: ignore[method-assign] - f"unexpected delivery: {payload}" - ) - server.stamp_job_for_client = fail_stamp # type: ignore[method-assign] + with tempfile.TemporaryDirectory() as tmp, patch( + "lab.prism.bundle_compiler.subprocess.Popen", + fake_audit_bundle_popen(captured), + ): + base = Path(tmp) + root = base / "audit" + root.mkdir() + pinned = base / "audit-pinned" + output_path = root / "candidate.audit.json" + parent_fd = os.open( + root, + os.O_RDONLY | getattr(os, "O_DIRECTORY", 0), + ) + root.rename(pinned) + + def reject_stale_authority( + path: Path, + value: os.stat_result, + ) -> None: + self.assertEqual(path, output_path) + self.assertFalse(root.exists()) + pinned_output = pinned / output_path.name + self.assertTrue(pinned_output.exists()) + self.assertEqual( + (value.st_dev, value.st_ino), + (pinned_output.stat().st_dev, pinned_output.stat().st_ino), + ) + raise RuntimeError("stale A1 authority") - self.assertEqual(server.vardiff_idle_sweep_once(), 1) - server.shutdown_vardiff_idle_executor() + try: + with self.assertRaisesRegex(RuntimeError, "stale A1 authority"): + server.build_audit_bundle( + shares=[], + found_block={ + "block_height": 10, + "coinbase_value_sats": 50_00000000, + }, + prior_balances=[], + coinbase_script_sig_suffix_hex="00", + canonical_output_path=output_path, + canonical_output_parent_fd=parent_fd, + canonical_output_adopter=reject_stale_authority, + ) + finally: + os.close(parent_fd) - self.assertIsNone(state.pending_share_difficulty) - self.assertEqual(state.share_difficulty, Decimal("16")) - self.assertEqual( - ( - state.vardiff_window_started_monotonic, - state.vardiff_window_accepted, - state.vardiff_window_submitted, - state.vardiff_window_work, - ), - window_state, - ) - self.assertEqual(server.job_build_failure_count, 1) - self.assertEqual(server.vardiff_idle_task_failures, 1) + self.assertFalse(root.exists()) + self.assertFalse((pinned / output_path.name).exists()) - def test_idle_cached_bundle_requires_live_reorg_trust(self) -> None: + def test_build_audit_bundle_adopter_failure_preserves_path_replacement(self) -> None: server = coordinator() - state = client() - prepare_idle_client(server, state) - install_idle_job_cache(server) - trust_checked = threading.Event() - sent: list[dict[str, object]] = [] - window_started = state.vardiff_window_started_monotonic + server.signing_seed_hex = "42" * 32 + server.ledger_attestation_signing_seed_hex = "43" * 32 + captured: dict[str, object] = {} - def reject_untrusted_tip() -> bool: - trust_checked.set() - return False + def replace_then_reject(path: Path, _value: os.stat_result) -> None: + path.unlink() + path.write_bytes(b"competitor") + raise RuntimeError("transfer rejected") - state.send = sent.append # type: ignore[method-assign] - server.ensure_reorg_reconciled_for_current_tip = ( # type: ignore[method-assign] - reject_untrusted_tip - ) + with tempfile.TemporaryDirectory() as tmp, patch( + "lab.prism.bundle_compiler.subprocess.Popen", + fake_audit_bundle_popen(captured), + ): + output_path = Path(tmp) / "candidate.audit.json" + with self.assertRaisesRegex(RuntimeError, "transfer rejected"): + server.build_audit_bundle( + shares=[], + found_block={"block_height": 10, "coinbase_value_sats": 50_00000000}, + prior_balances=[], + coinbase_script_sig_suffix_hex="00", + canonical_output_path=output_path, + canonical_output_adopter=replace_then_reject, + ) - self.assertEqual(server.vardiff_idle_sweep_once(), 1) - server.shutdown_vardiff_idle_executor() + self.assertEqual(output_path.read_bytes(), b"competitor") - self.assertTrue(trust_checked.is_set()) - self.assertEqual(sent, []) - self.assertEqual(server.idle_retarget_count, 0) - self.assertEqual(state.share_difficulty, Decimal("16")) - self.assertIsNone(state.pending_share_difficulty) - self.assertEqual(state.vardiff_window_started_monotonic, window_started) - self.assertGreaterEqual(server.vardiff_idle_skip_counts["superseded"], 1) + def test_builder_cleanup_never_moves_an_existing_replacement(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + path = root / "candidate.audit.json" + path.write_bytes(b"owned-a") + identity = path.stat() + path.unlink() + path.write_bytes(b"competitor-b") + with patch( + "os.replace", + side_effect=AssertionError("replacement must not move"), + ): + BundleCompiler._remove_created_output_if_same(path, identity) - def test_idle_retarget_defers_detected_payout_source_during_tip_divergence( - self, - ) -> None: - old_tip = "00" * 32 - new_tip = "11" * 32 - server = coordinator() - state = client() - prepare_idle_client(server, state, tip=old_tip) - install_idle_job_cache(server, tip=old_tip) - with server._job_cache_lock: - published_artifacts = server._template_artifacts - assert published_artifacts is not None - detected_template = gbt_template(new_tip, height=11) - detected_artifacts = CachedTemplateArtifacts( - template=detected_template, - fingerprint=qbit_template_fingerprint(detected_template), - previousblockhash=new_tip, - transaction_hexes=(), - witness_merkle_leaves_hex=(), - network_difficulty=1, - fetched_monotonic=time.monotonic(), - generation=2, - ) - now = time.monotonic() - server.current_tip_first_seen = (old_tip, now) - server.current_tip_observed_monotonic = now - server.latest_detected_tip = (new_tip, 2) - server.tip_refresh_divergence_started_monotonic = now - server.tip_template_snapshot = QbitTipTemplateSnapshot( - bestblockhash=old_tip, - previousblockhash=old_tip, - template_fingerprint=published_artifacts.fingerprint, - template_generation=published_artifacts.generation, - template_artifacts=published_artifacts, - ) - with server._job_cache_lock: - server._template_artifacts = detected_artifacts - server._published_payout_state = dataclass_replace( - server._published_payout_state, - source_tip_hash=new_tip, - ) - window_state = ( - state.vardiff_window_started_monotonic, - state.vardiff_window_accepted, - state.vardiff_window_submitted, - state.vardiff_window_work, - ) - sent: list[dict[str, object]] = [] + self.assertEqual(path.read_bytes(), b"competitor-b") + quarantines = list(root.glob(".candidate.audit.json.*.cleanup")) + self.assertEqual(quarantines, []) - server._build_idle_job_bundle = lambda _request: self.fail( # type: ignore[method-assign] - "idle divergence entered the shared build scheduler" - ) - state.send = sent.append # type: ignore[method-assign] + def test_builder_cleanup_never_relocates_nonregular_replacement(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + path = root / "candidate.audit.json" + path.write_bytes(b"owned") + identity = path.stat() + path.unlink() + path.mkdir() - self.assertEqual(server.vardiff_idle_sweep_once(), 0) - server.shutdown_vardiff_idle_executor() + BundleCompiler._remove_created_output_if_same(path, identity) - self.assertEqual(sent, []) - self.assertEqual(state.share_difficulty, Decimal("16")) - self.assertIsNone(state.pending_share_difficulty) - self.assertEqual( - ( - state.vardiff_window_started_monotonic, - state.vardiff_window_accepted, - state.vardiff_window_submitted, - state.vardiff_window_work, - ), - window_state, - ) - self.assertEqual(server.vardiff_idle_skip_counts["superseded"], 1) + self.assertTrue(path.is_dir()) + self.assertEqual( + list(root.glob(".candidate.audit.json.*.cleanup")), + [], + ) - def test_replacement_tip_build_survives_repeated_idle_sweeps(self) -> None: - old_tip = "00" * 32 - new_tip = "11" * 32 + path.rmdir() + target = root / "operator" + target.write_bytes(b"operator") + path.symlink_to(target) + BundleCompiler._remove_created_output_if_same(path, identity) + self.assertTrue(path.is_symlink()) + self.assertEqual(path.read_bytes(), b"operator") + + def test_build_audit_bundle_summary_only_requests_and_parses_job_summary(self) -> None: server = coordinator() - state = client() - prepare_idle_client(server, state, tip=old_tip) - install_idle_job_cache(server, tip=old_tip) - with server._job_cache_lock: - published_artifacts = server._template_artifacts - assert published_artifacts is not None - detected_template = gbt_template(new_tip, height=11) - detected_artifacts = CachedTemplateArtifacts( - template=detected_template, - fingerprint=qbit_template_fingerprint(detected_template), - previousblockhash=new_tip, - transaction_hexes=(), - witness_merkle_leaves_hex=(), - network_difficulty=1, - fetched_monotonic=time.monotonic(), - generation=2, - ) - now = time.monotonic() - server.current_tip_first_seen = (old_tip, now) - server.current_tip_observed_monotonic = now - server.latest_detected_tip = (new_tip, 2) - server.tip_refresh_divergence_started_monotonic = now - server.tip_template_snapshot = QbitTipTemplateSnapshot( - bestblockhash=old_tip, - previousblockhash=old_tip, - template_fingerprint=published_artifacts.fingerprint, - template_generation=published_artifacts.generation, - template_artifacts=published_artifacts, - ) - with server._job_cache_lock: - server._template_artifacts = detected_artifacts - server._ensure_job_cache_state() - replacement_cancellation = _JobBuildCancellation(timeout_seconds=60.0) - replacement_request = SimpleNamespace( - artifacts=detected_artifacts, - cancellation=replacement_cancellation, - ) - replacement_flight = SimpleNamespace(request=replacement_request) - with server._job_build_scheduler_lock: - server._job_build_active = replacement_flight - build_called = threading.Event() - window_started = state.vardiff_window_started_monotonic - - def unexpected_idle_build(_request: object) -> CachedJobBundle: - build_called.set() - raise AssertionError("idle sweep displaced the replacement build") - - server._build_idle_job_bundle = unexpected_idle_build # type: ignore[method-assign] - - for _sweep in range(4): - self.assertEqual(server.vardiff_idle_sweep_once(), 0) - - # Close the race where a worker passed its first divergence check just - # before detection. Scheduler admission must reject that idle request - # without superseding the active replacement build. - racing_idle_cancellation = _JobBuildCancellation(timeout_seconds=60.0) - racing_idle_request = SimpleNamespace( - idle_retarget=True, - cancellation=racing_idle_cancellation, - promise=Future(), - ) - racing_idle_promise = server._request_job_build(racing_idle_request) # type: ignore[arg-type] - with self.assertRaises(JobBuildCancelled): - racing_idle_promise.result() - - with server._job_build_scheduler_lock: - self.assertIs(server._job_build_active, replacement_flight) - self.assertFalse(replacement_cancellation.is_set()) - self.assertTrue(racing_idle_cancellation.is_set()) - self.assertFalse(build_called.is_set()) - self.assertEqual(state.vardiff_window_started_monotonic, window_started) - self.assertEqual(server.vardiff_idle_skip_counts["superseded"], 4) - with server._job_build_scheduler_lock: - server._job_build_active = None - server.shutdown_vardiff_idle_executor() - - def test_idle_shared_build_does_not_retry_scheduler_divergence_race( - self, - ) -> None: - old_tip = "00" * 32 - new_tip = "11" * 32 - server = coordinator() - state = client() - prepare_idle_client(server, state, tip=old_tip) - install_idle_job_cache(server, tip=old_tip) - with server._job_cache_lock: - old_artifacts = server._template_artifacts - server._job_bundle_cache.clear() - assert old_artifacts is not None - now = time.monotonic() - server.current_tip_first_seen = (old_tip, now) - server.current_tip_observed_monotonic = now - server.latest_detected_tip = (old_tip, 1) - server.tip_template_snapshot = QbitTipTemplateSnapshot( - bestblockhash=old_tip, - previousblockhash=old_tip, - template_fingerprint=old_artifacts.fingerprint, - template_generation=old_artifacts.generation, - template_artifacts=old_artifacts, - ) - server._ensure_job_cache_state() - replacement_cancellation = _JobBuildCancellation(timeout_seconds=60.0) - replacement_flight = SimpleNamespace( - request=SimpleNamespace( - artifacts=SimpleNamespace(previousblockhash=new_tip), - cancellation=replacement_cancellation, - ) - ) - with server._job_build_scheduler_lock: - server._job_build_active = replacement_flight - request_builds = 0 - admission_attempts = 0 - idle_cancellation: _JobBuildCancellation | None = None - original_request_job_build = server._request_job_build - - def make_idle_request( - _artifacts: CachedTemplateArtifacts, - _worker: WorkerIdentity | None, - **kwargs: object, - ) -> object: - nonlocal request_builds, idle_cancellation - request_builds += 1 - self.assertTrue(kwargs["idle_retarget"]) - idle_cancellation = _JobBuildCancellation(timeout_seconds=60.0) - return SimpleNamespace( - idle_retarget=True, - cancellation=idle_cancellation, - promise=Future(), - ) + server.signing_seed_hex = "42" * 32 + server.ledger_attestation_signing_seed_hex = "43" * 32 + summary = { + "found_block": { + "block_height": 10, + "coinbase_value_sats": 50_00000000, + }, + "signed_coinbase_manifest": { + "manifest": {"coinbase_tx_hex": "c0ffee"}, + "signature": {"signature_hex": "11" * 64}, + }, + } + captured: dict[str, object] = {} - def detect_before_admission(request: object) -> Future[CachedJobBundle]: - nonlocal admission_attempts - admission_attempts += 1 - with server.lock: - server.latest_detected_tip = (new_tip, 2) - server.tip_refresh_divergence_started_monotonic = time.monotonic() - return original_request_job_build(request) # type: ignore[arg-type] - - server._new_job_build_request = make_idle_request # type: ignore[method-assign] - server._request_job_build = detect_before_admission # type: ignore[method-assign] - server._schedule_tip_refresh_retry = lambda: None # type: ignore[method-assign] - assert state.worker is not None - - with self.assertRaises(JobBuildSuperseded): - server._build_idle_job_bundle(SimpleNamespace(worker=state.worker)) # type: ignore[arg-type] - - self.assertEqual(request_builds, 1) - self.assertEqual(admission_attempts, 1) - self.assertIs(server._template_artifacts, old_artifacts) - self.assertIsNotNone(idle_cancellation) - assert idle_cancellation is not None - self.assertTrue(idle_cancellation.is_set()) - self.assertFalse(replacement_cancellation.is_set()) - with server._job_build_scheduler_lock: - self.assertIs(server._job_build_active, replacement_flight) - server._job_build_active = None - server.shutdown_vardiff_idle_executor() - - def test_idle_cached_collection_bundle_refreshes_readiness(self) -> None: - server = coordinator() - state = client() - prepare_idle_client(server, state) - ready_bundle = install_idle_job_cache(server) - assert state.worker is not None - with server._job_cache_lock: - artifacts = server._template_artifacts - assert artifacts is not None - collection_key = server._job_bundle_key( - artifacts, - mode="collection", - payout_state_generation=server._payout_state_generation, - payout_artifact_generation=0, - worker=state.worker, - ) - collection_bundle = dataclass_replace( - ready_bundle, - key=collection_key, - collection_only=True, - collection_identity=( - state.worker.payout_address, - state.worker.p2mr_program_hex, - ), + with patch( + "lab.prism.prism_coordinator.subprocess.Popen", + fake_audit_bundle_popen( + captured, + output_text=json.dumps(summary, separators=(",", ":")), + ), + ), patch.object( + Path, + "open", + side_effect=AssertionError("summary-only build must not open an output path"), + ): + bundle = server.build_audit_bundle( + shares=[], + found_block={"block_height": 10, "coinbase_value_sats": 50_00000000}, + prior_balances=[], + coinbase_script_sig_suffix_hex="00", + summary_only=True, ) - server._job_bundle_cache.clear() - server._job_bundle_cache[collection_key] = collection_bundle - server._pool_ready_latched = False - server.min_ready_miners = 3 - server.accepted_share_stats = lambda: (3, 3) # type: ignore[method-assign] - rebuilt = threading.Event() - sent: list[dict[str, object]] = [] - - def build_ready(_request: object) -> CachedJobBundle: - rebuilt.set() - with server._job_cache_lock: - server._job_bundle_cache[ready_bundle.key] = ready_bundle - return ready_bundle - - state.send = sent.append # type: ignore[method-assign] - server._build_idle_job_bundle = build_ready # type: ignore[method-assign] - self.assertEqual(server.vardiff_idle_sweep_once(), 1) - server.shutdown_vardiff_idle_executor() - - self.assertTrue(server._pool_ready_latched) - self.assertTrue(rebuilt.is_set()) - self.assertEqual( - [payload["method"] for payload in sent], - ["mining.set_difficulty", "mining.notify"], - ) - self.assertIsNotNone(state.active_job) - self.assertFalse(state.active_job.collection_only) + self.assertEqual(bundle, summary) + self.assertEqual(set(bundle), {"found_block", "signed_coinbase_manifest"}) + self.assertIn("--job-summary-output", captured["cmd"]) + self.assertNotIn("--canonical-output", captured["cmd"]) - def test_idle_cached_ready_bundle_rebinds_same_tip_observation(self) -> None: + def test_build_audit_bundle_removes_partial_output_after_builder_failure(self) -> None: server = coordinator() - state = client() - prepare_idle_client(server, state) - cached = install_idle_job_cache(server) - updated_template = dict(cached.template) - updated_template["curtime"] = int(updated_template["curtime"]) + 30 - current = server.store_template_artifacts(updated_template, generation=2) - assert current is not None - delivered = threading.Event() - rebound = threading.Event() - sent: list[dict[str, object]] = [] + server.signing_seed_hex = "42" * 32 + server.ledger_attestation_signing_seed_hex = "43" * 32 - def bind_current_observation( - bundle: CachedJobBundle, - artifacts: CachedTemplateArtifacts, - ) -> CachedJobBundle: - rebound.set() - return dataclass_replace( - bundle, - template=artifacts.template, - base_job=dataclass_replace( - bundle.base_job, - ntime=f'{artifacts.template["curtime"]:08x}', + captured: dict[str, object] = {} + with tempfile.TemporaryDirectory() as tmp: + output_path = Path(tmp) / "candidate.audit.json" + with patch( + "lab.prism.prism_coordinator.subprocess.Popen", + fake_audit_bundle_popen( + captured, + output_text='{"partial":', + returncode=9, + stderr_text="synthetic builder failure", ), - template_generation=artifacts.generation, - ) - - def record(payload: dict[str, object]) -> None: - sent.append(payload) - if payload.get("method") == "mining.notify": - delivered.set() + ): + with self.assertRaisesRegex(RuntimeError, "synthetic builder failure"): + server.build_audit_bundle( + shares=[], + found_block={"block_height": 10, "coinbase_value_sats": 50_00000000}, + prior_balances=[], + coinbase_script_sig_suffix_hex="00", + canonical_output_path=output_path, + ) + self.assertFalse(output_path.exists()) + self.assertEqual(server.job_build_worker_counts["crashes"], 1) - state.send = record # type: ignore[method-assign] - server._bind_cached_bundle_to_artifacts = ( # type: ignore[method-assign] - bind_current_observation - ) + with patch( + "lab.prism.prism_coordinator.subprocess.Popen", + fake_audit_bundle_popen(captured), + ): + recovered = server.build_audit_bundle( + shares=[], + found_block={"block_height": 10, "coinbase_value_sats": 50_00000000}, + prior_balances=[], + coinbase_script_sig_suffix_hex="00", + ) - self.assertEqual(server.vardiff_idle_sweep_once(), 1) - self.assertTrue(delivered.wait(timeout=1)) - server.shutdown_vardiff_idle_executor() + self.assertEqual(recovered, {"ok": True}) + self.assertEqual(server.job_build_worker_counts["restarts"], 1) - self.assertIsNotNone(state.active_job) - self.assertTrue(rebound.is_set()) - self.assertIs(state.active_job.template, current.template) - self.assertEqual(state.active_job.template_generation, current.generation) - expected_ntime = f'{updated_template["curtime"]:08x}' - self.assertEqual(state.active_job.job.ntime, expected_ntime) - notify = next( - payload - for payload in sent - if payload.get("method") == "mining.notify" - ) - self.assertEqual(notify["params"][7], expected_ntime) - def test_repeated_idle_sweeps_do_not_enqueue_duplicate_connection_work(self) -> None: + def test_build_audit_bundle_removes_owned_output_after_parse_failure(self) -> None: server = coordinator() - state = client() - prepare_idle_client(server, state) - install_idle_job_cache(server) - delivery_started = threading.Event() - release_delivery = threading.Event() - sent: list[dict[str, object]] = [] - - def blocking_send(payload: dict[str, object]) -> None: - if payload.get("method") == "mining.set_difficulty": - delivery_started.set() - release_delivery.wait(timeout=1) - sent.append(payload) - - state.send = blocking_send # type: ignore[method-assign] - try: - self.assertEqual(server.vardiff_idle_sweep_once(), 1) - self.assertTrue(delivery_started.wait(timeout=1)) - self.assertEqual(server.vardiff_idle_sweep_once(), 0) - with server._vardiff_idle_lock: - self.assertEqual(len(server._vardiff_idle_pending), 1) - finally: - release_delivery.set() - server.shutdown_vardiff_idle_executor() + server.signing_seed_hex = "42" * 32 + server.ledger_attestation_signing_seed_hex = "43" * 32 + captured: dict[str, object] = {} - self.assertEqual( - [payload["method"] for payload in sent], - ["mining.set_difficulty", "mining.notify"], - ) - self.assertGreaterEqual( - server.vardiff_idle_skip_counts["superseded"] - + server.vardiff_idle_skip_counts["not_idle"], - 1, - ) + with tempfile.TemporaryDirectory() as tmp, patch( + "lab.prism.bundle_compiler.subprocess.Popen", + fake_audit_bundle_popen(captured, output_text='{"partial":'), + ): + output_path = Path(tmp) / "candidate.audit.json" + with self.assertRaises(json.JSONDecodeError): + server.build_audit_bundle( + shares=[], + found_block={"block_height": 10, "coinbase_value_sats": 50_00000000}, + prior_balances=[], + coinbase_script_sig_suffix_hex="00", + canonical_output_path=output_path, + ) + self.assertFalse(output_path.exists()) - def test_idle_sweep_skips_busy_client_lock_and_returns_promptly(self) -> None: + def test_build_audit_bundle_recovers_after_cancelled_worker_timeout(self) -> None: server = coordinator() - state = client() - prepare_idle_client(server, state) - install_idle_job_cache(server) - lock_held = threading.Event() - release_lock = threading.Event() + server.signing_seed_hex = "42" * 32 + server.ledger_attestation_signing_seed_hex = "43" * 32 + cancellation = _JobBuildCancellation(timeout_seconds=60) + process_calls = 0 - def hold_client_lock() -> None: - state.job_update_lock.acquire() - try: - lock_held.set() - release_lock.wait(timeout=1) - finally: - state.job_update_lock.release() + class FakeStdin: + def write(self, value: str) -> int: + return len(value) - holder = threading.Thread(target=hold_client_lock) - holder.start() - self.assertTrue(lock_held.wait(timeout=1)) - started = time.monotonic() - try: - self.assertEqual(server.vardiff_idle_sweep_once(), 0) - finally: - elapsed = time.monotonic() - started - release_lock.set() - holder.join(timeout=1) - server.shutdown_vardiff_idle_executor() + def close(self) -> None: + return None - self.assertLess(elapsed, 0.25) - self.assertEqual(server.vardiff_idle_skip_counts["busy"], 1) + class HungThenHealthyPopen: + def __init__(self, _cmd: list[str], **kwargs: object) -> None: + nonlocal process_calls + process_calls += 1 + self.healthy = process_calls == 2 + self.stdin = FakeStdin() + self.stdout = kwargs["stdout"] + self.stderr = kwargs["stderr"] + self.returncode: int | None = None + self.output_written = False - def test_stuck_bundle_builder_does_not_stale_idle_sweep_heartbeat(self) -> None: - server = coordinator() - state = client() - prepare_idle_client(server, state) - bundle = install_idle_job_cache(server) - with server._job_cache_lock: - server._job_bundle_cache.clear() - state.send = lambda _payload: None # type: ignore[method-assign] - build_started = threading.Event() - release_build = threading.Event() + def poll(self) -> int | None: + if self.returncode is not None: + return self.returncode + if self.healthy: + if not self.output_written: + self.stdout.write('{"ok":true}') # type: ignore[union-attr] + self.output_written = True + self.returncode = 0 + return 0 + cancellation.cancel("timeout") + return None - def blocked_build( - *_args: object, - **_kwargs: object, - ) -> CachedJobBundle: - build_started.set() - if not release_build.wait(timeout=1): - raise AssertionError("idle retarget bundle build was not released") - return bundle - - server.build_shared_job_bundle = blocked_build # type: ignore[method-assign] - server._record_heartbeat("vardiff_idle_sweep") - heartbeat_before = server._heartbeats["vardiff_idle_sweep"] - started = time.monotonic() - try: - self.assertEqual(server.vardiff_idle_sweep_once(), 1) - elapsed = time.monotonic() - started - self.assertTrue(build_started.wait(timeout=0.25)) - self.assertLess(elapsed, 0.25) - self.assertGreater( - server._heartbeats["vardiff_idle_sweep"], - heartbeat_before, - ) - finally: - release_build.set() - server.shutdown_vardiff_idle_executor() + def terminate(self) -> None: + self.returncode = -15 - self.assertEqual(server.vardiff_idle_skip_counts["cache_miss"], 1) + def kill(self) -> None: + self.returncode = -9 - def test_idle_sweep_cache_miss_builds_only_on_bounded_worker(self) -> None: - server = coordinator() - state = client() - prepare_idle_client(server, state) - bundle = install_idle_job_cache(server) - server.job_bundle_cache_seconds = 10.0 - expired_bundle = dataclass_replace( - bundle, - built_monotonic=time.monotonic() - 11.0, - ) - with server._job_cache_lock: - server._job_bundle_cache[bundle.key] = expired_bundle - build_started = threading.Event() - release_build = threading.Event() - build_thread_ids: list[int] = [] - sent: list[dict[str, object]] = [] + def wait(self, timeout: float | None = None) -> int: + del timeout + assert self.returncode is not None + return self.returncode - def blocked_build( - *_args: object, - **_kwargs: object, - ) -> CachedJobBundle: - build_thread_ids.append(threading.get_ident()) - build_started.set() - if not release_build.wait(timeout=1): - raise AssertionError("idle retarget bundle build was not released") - return bundle - - state.send = sent.append # type: ignore[method-assign] - server.build_shared_job_bundle = blocked_build # type: ignore[method-assign] - sweep_thread_id = threading.get_ident() - started = time.monotonic() - try: - self.assertEqual(server.vardiff_idle_sweep_once(), 1) - elapsed = time.monotonic() - started - self.assertTrue(build_started.wait(timeout=0.25)) - self.assertLess(elapsed, 0.25) - self.assertEqual(server.vardiff_idle_sweep_once(), 0) - self.assertEqual(len(build_thread_ids), 1) - self.assertNotEqual(build_thread_ids[0], sweep_thread_id) - with server._vardiff_idle_lock: - self.assertEqual(len(server._vardiff_idle_pending), 1) - finally: - release_build.set() - server.shutdown_vardiff_idle_executor() + build_kwargs = { + "shares": [], + "found_block": { + "block_height": 10, + "coinbase_value_sats": 50_00000000, + }, + "prior_balances": [], + "coinbase_script_sig_suffix_hex": "00", + } + with patch( + "lab.prism.prism_coordinator.subprocess.Popen", + HungThenHealthyPopen, + ): + with self.assertRaisesRegex(JobBuildCancelled, "timeout"): + server.build_audit_bundle( + **build_kwargs, + cancellation=cancellation, + ) + recovered = server.build_audit_bundle( + **build_kwargs, + cancellation=_JobBuildCancellation(timeout_seconds=60), + ) - self.assertEqual(server.vardiff_idle_skip_counts["cache_miss"], 1) - self.assertEqual( - [payload["method"] for payload in sent], - ["mining.set_difficulty", "mining.notify"], - ) - self.assertEqual(server.idle_retarget_count, 1) - self.assertIsNone(state.pending_share_difficulty) - self.assertEqual(state.share_difficulty, Decimal("4")) - self.assertEqual(state.vardiff_window_accepted, 0) - self.assertEqual(state.vardiff_window_submitted, 0) - self.assertEqual(state.vardiff_window_work, Decimal("0")) + self.assertEqual(recovered, {"ok": True}) + self.assertEqual(process_calls, 2) + self.assertEqual(server.job_build_worker_counts["terminations"], 1) + self.assertEqual(server.job_build_worker_counts["restarts"], 1) - def test_idle_retarget_delivers_fresh_bundle_when_cache_is_disabled( - self, - ) -> None: + def test_build_audit_bundle_does_not_unlink_preexisting_output_path(self) -> None: server = coordinator() - state = client() - prepare_idle_client(server, state) - bundle = install_idle_job_cache(server) - server.job_bundle_cache_seconds = 0.0 - with server._job_cache_lock: - server._job_bundle_cache.clear() - built = threading.Event() - sent: list[dict[str, object]] = [] - - def build_uncached(_request: object) -> CachedJobBundle: - built.set() - return bundle + server.signing_seed_hex = "42" * 32 + server.ledger_attestation_signing_seed_hex = "43" * 32 - server._build_idle_job_bundle = build_uncached # type: ignore[method-assign] - state.send = sent.append # type: ignore[method-assign] + with tempfile.TemporaryDirectory() as tmp, patch( + "lab.prism.prism_coordinator.subprocess.Popen" + ) as popen: + output_path = Path(tmp) / "preexisting-candidate.audit.json" + output_path.write_bytes(b"do-not-clobber") + with self.assertRaises(FileExistsError): + server.build_audit_bundle( + shares=[], + found_block={"block_height": 10, "coinbase_value_sats": 50_00000000}, + prior_balances=[], + coinbase_script_sig_suffix_hex="00", + canonical_output_path=output_path, + ) - self.assertEqual(server.vardiff_idle_sweep_once(), 1) - server.shutdown_vardiff_idle_executor() + self.assertEqual(output_path.read_bytes(), b"do-not-clobber") + popen.assert_not_called() - self.assertTrue(built.is_set()) - self.assertEqual(server.vardiff_idle_skip_counts["cache_miss"], 1) - self.assertEqual( - [payload["method"] for payload in sent], - ["mining.set_difficulty", "mining.notify"], - ) - self.assertEqual(server.idle_retarget_count, 1) - self.assertEqual(state.share_difficulty, Decimal("4")) - self.assertIsNone(state.pending_share_difficulty) - with server._job_cache_lock: - self.assertEqual(server._job_bundle_cache, {}) - - def test_idle_preparation_oserror_keeps_client_connected(self) -> None: - for failure_phase in ("bundle", "reorg"): - with self.subTest(failure_phase=failure_phase): - server = coordinator() - state = client() - prepare_idle_client(server, state) - install_idle_job_cache(server) - disconnected: list[ClientState] = [] - window_started = state.vardiff_window_started_monotonic - - def fail_bundle(_request: object) -> CachedJobBundle: - raise OSError("qbit RPC transport unavailable") - - def fail_reorg() -> bool: - raise OSError("qbit trust RPC transport unavailable") - - def unexpected_send(payload: dict[str, object]) -> None: - self.fail(f"unexpected idle delivery: {payload}") - - def record_disconnect(client_state: ClientState) -> None: - disconnected.append(client_state) - - if failure_phase == "bundle": - server._build_idle_job_bundle = ( # type: ignore[method-assign] - fail_bundle - ) - else: - server.ensure_reorg_reconciled_for_current_tip = ( # type: ignore[method-assign] - fail_reorg - ) - state.send = unexpected_send # type: ignore[method-assign] - server.disconnect_client = record_disconnect # type: ignore[method-assign] - - self.assertEqual(server.vardiff_idle_sweep_once(), 1) - server.shutdown_vardiff_idle_executor() - - self.assertEqual(disconnected, []) - self.assertIn(state, server.clients) - self.assertFalse(state.closing) - self.assertEqual(server.vardiff_idle_task_failures, 1) - self.assertIsNone(state.pending_share_difficulty) - self.assertEqual(state.share_difficulty, Decimal("16")) - self.assertEqual( - state.vardiff_window_started_monotonic, - window_started, - ) - - def test_disconnect_while_idle_retarget_pending_prevents_delivery(self) -> None: - server = coordinator() - blockers = [client(), client()] - for connection_id, blocker in enumerate(blockers, start=1): - prepare_idle_client(server, blocker, connection_id=connection_id) - install_idle_job_cache(server) - workers_started = threading.Barrier(3) - release_workers = threading.Event() - - def block_delivery(payload: dict[str, object]) -> None: - if payload.get("method") == "mining.set_difficulty": - workers_started.wait(timeout=1) - release_workers.wait(timeout=1) - - for blocker in blockers: - blocker.send = block_delivery # type: ignore[method-assign] - - target = client() - target_sent: list[dict[str, object]] = [] - target.send = target_sent.append # type: ignore[method-assign] - target.close = lambda: None # type: ignore[method-assign] - disconnected_skip = threading.Event() - record_skip = server._record_vardiff_idle_skip - - def record_and_signal(reason: str) -> None: - record_skip(reason) - if reason == "disconnected": - disconnected_skip.set() - - server._record_vardiff_idle_skip = record_and_signal # type: ignore[method-assign] - try: - self.assertEqual(server.vardiff_idle_sweep_once(), 2) - workers_started.wait(timeout=1) - prepare_idle_client(server, target, connection_id=3) - self.assertEqual(server.vardiff_idle_sweep_once(), 1) - server.disconnect_client(target) - finally: - release_workers.set() - try: - self.assertTrue(disconnected_skip.wait(timeout=1)) - finally: - server.shutdown_vardiff_idle_executor() - - self.assertEqual(target_sent, []) - self.assertTrue(target.closing) - self.assertNotIn(target, server.clients) - self.assertGreaterEqual(server.vardiff_idle_skip_counts["disconnected"], 1) - - def test_hundreds_of_busy_and_dead_clients_cannot_stall_idle_sweep(self) -> None: - server = coordinator() - - class BusyLock: - def acquire(self, blocking: bool = True) -> bool: - self.assert_nonblocking = blocking - return False - - def release(self) -> None: - raise AssertionError("unacquired busy lock released") - - for connection_id in range(1, 201): - state = client() - prepare_idle_client(server, state, connection_id=connection_id) - state.job_update_lock = BusyLock() # type: ignore[assignment] - for connection_id in range(201, 401): - state = client() - prepare_idle_client(server, state, connection_id=connection_id) - state.closing = True - - server.vardiff_idle_sweep_seconds = 0.5 - started = time.monotonic() - self.assertEqual(server.vardiff_idle_sweep_once(), 0) - elapsed = time.monotonic() - started - server.shutdown_vardiff_idle_executor() - - self.assertLess(elapsed, server.vardiff_idle_sweep_seconds) - self.assertEqual(server.vardiff_idle_clients_inspected, 400) - self.assertEqual(server.vardiff_idle_skip_counts["busy"], 200) - self.assertEqual(server.vardiff_idle_skip_counts["disconnected"], 200) - - def test_idle_retarget_queue_is_globally_bounded(self) -> None: - server = coordinator() - states = [client() for _ in range(9)] - for connection_id, state in enumerate(states, start=1): - prepare_idle_client(server, state, connection_id=connection_id) - install_idle_job_cache(server) - release_workers = threading.Event() - two_workers_started = threading.Event() - started_lock = threading.Lock() - started_count = 0 - - def block_delivery(payload: dict[str, object]) -> None: - nonlocal started_count - if payload.get("method") != "mining.set_difficulty": - return - with started_lock: - started_count += 1 - if started_count == 2: - two_workers_started.set() - release_workers.wait(timeout=1) - - for state in states: - state.send = block_delivery # type: ignore[method-assign] - - try: - self.assertEqual(server.vardiff_idle_sweep_once(), 8) - self.assertTrue(two_workers_started.wait(timeout=1)) - with server._vardiff_idle_lock: - self.assertEqual(len(server._vardiff_idle_pending), 8) - self.assertEqual(server.vardiff_idle_inflight, 2) - self.assertEqual(server.vardiff_idle_queue_depth, 6) - self.assertEqual(server.vardiff_idle_skip_counts["queue_full"], 1) - finally: - release_workers.set() - server.shutdown_vardiff_idle_executor() - - def test_maybe_send_job_isolates_build_failure_and_keeps_client_connected(self) -> None: - server = coordinator() - server.jobs = {} - server.extranonce2_size = 8 - state = client() - state.username = "miner-a" - state.worker = WorkerIdentity( - username="miner-a", - payout_address="miner-a", - worker_name=None, - script_pubkey_hex="5220" + "11" * 32, - p2mr_program_hex="11" * 32, + def test_block_submit_rejects_job_when_prior_balances_changed_before_persist(self) -> None: + server, state, _recording = submit_coordinator() + configure_temporary_audit_root(self, server) + ledger = SingleWriterShareLedger() + ledger.durable_payout_state = True # type: ignore[attr-defined] + server.ledger = ledger + server.jobs["job-1"].prior_balances = [ + { + "recipient_id": "miner-a", + "order_key": "miner-a", + "p2mr_program_hex": "11" * 32, + "balance_sats": 25, + } + ] + server.build_audit_bundle = lambda **_kwargs: (_ for _ in ()).throw( # type: ignore[method-assign] + AssertionError("audit bundle should not be rebuilt from stale prior balances") ) - sent: list[object] = [] - state.send = lambda payload: sent.append(payload) # type: ignore[method-assign] - - def boom(client: ClientState, *, clean_jobs: bool) -> None: - raise ValueError( - "full coinbase transaction does not end its coinbase scriptSig " - "with the extranonce placeholder" - ) - - server.build_job_for_client = boom # type: ignore[method-assign] - - # The bug: this used to propagate out of handle_client and drop the miner. - # It must now be swallowed so the client thread survives a single bad template. - server.maybe_send_job(state, clean_jobs=True) - - self.assertEqual(server.job_build_failure_count, 1) - self.assertEqual(state.active_job_ids, set()) - self.assertEqual(server.jobs, {}) - self.assertEqual(sent, []) # no difficulty / mining.notify pushed for the failed build - - # A subsequent good template still issues a job (skip, do not permanently break). - server.build_job_for_client = lambda client, *, clean_jobs: SimpleNamespace( # type: ignore[method-assign] - job=SimpleNamespace( - job_id="job-ok", - share_difficulty=Decimal("1"), - share_target=target_from_compact("207fffff"), - ), - template={"previousblockhash": "00" * 32}, - collection_only=False, + submission = SimpleNamespace( + coinbase_tx_hex="c0ffee", + block_hash_hex="ef" * 32, + block_hex="00", ) - server.send_difficulty = lambda client, job: None # type: ignore[method-assign] - server.send_job = lambda client, job: sent.append("notify") # type: ignore[method-assign] - server.apply_job_difficulty = lambda client, job: None # type: ignore[method-assign] - - server.maybe_send_job(state, clean_jobs=True) - - self.assertEqual(server.job_build_failure_count, 1) - self.assertEqual(state.active_job_ids, {"job-ok"}) - self.assertEqual(sent, ["notify"]) - - def test_maybe_send_job_does_not_swallow_send_failures_as_build_failures(self) -> None: - # Only the job build is isolated. A Stratum send failure (a dead socket) - # must propagate so handle_client disconnects and cleans up, rather than - # being miscounted as a build failure or leaving the client wedged. - server = coordinator() - server.jobs = {} - server.extranonce2_size = 8 - state = client() - state.username = "miner-a" - state.worker = WorkerIdentity( - username="miner-a", - payout_address="miner-a", - worker_name=None, - script_pubkey_hex="5220" + "11" * 32, - p2mr_program_hex="11" * 32, + pending = self._pending_append("balance-stale").pending_share + candidate = block_candidate( + server, + state, + submission, + pending_share=pending, ) - - server.build_job_for_client = lambda client, *, clean_jobs: SimpleNamespace( # type: ignore[method-assign] - job=SimpleNamespace( - job_id="job-dead", - share_difficulty=Decimal("1"), - share_target=target_from_compact("207fffff"), - ), - collection_only=False, + ledger.append_batch( + [(pending, server.block_candidate_intent(candidate))] ) - server.send_difficulty = lambda client, job: None # type: ignore[method-assign] - - def dead_socket(client: ClientState, job: object) -> None: - raise OSError("broken pipe") - - server.send_job = dead_socket # type: ignore[method-assign] - - with self.assertRaises(OSError): - server.maybe_send_job(state, clean_jobs=True) - - # The send failure is not a build failure, and handle_client (not us) owns - # the disconnect/cleanup of the registered job for the dead connection. - self.assertEqual(server.job_build_failure_count, 0) - - def test_metrics_include_issue_scope_operational_gauges(self) -> None: - server = coordinator() - server.submitted_share_count = 10 - server.stale_share_count = 2 - server.duplicate_share_count = 1 - server.low_difficulty_share_count = 3 - server.grace_credited_share_count = 6 - server.idle_retarget_count = 7 - server.rejection_counts_by_reason[PRISM_REJECTION_STALE_JOB] = 2 - server.rejection_counts_by_reason["duplicate-share"] = 1 - server.rejection_counts_by_reason["low-difficulty"] = 3 - server.tip_refresh_job_count = 4 - server.post_accept_refresh_failure_count = 5 - server.connection_limit_rejection_counts = {"global": 2, "username": 3} - server.accept_resource_exhaustion_count = 4 - server.connection_setup_failure_count = 5 - server._ensure_vardiff_idle_state() - with server._vardiff_idle_lock: - server.vardiff_idle_clients_inspected = 8 - server.vardiff_idle_skip_counts["busy"] = 2 - server.vardiff_idle_queue_depth = 1 - server.vardiff_idle_inflight = 2 - server.vardiff_idle_task_failures = 1 - server._observe_vardiff_idle_seconds("sweep", 0.005) - server._observe_vardiff_idle_seconds("task", 0.01) - - metrics = server.metrics_payload() + server.enqueue_block_candidate(candidate) - self.assertIn("qbit_prism_submitted_shares_total 10", metrics) - self.assertIn("qbit_prism_stale_shares_total 2", metrics) - self.assertIn("qbit_prism_duplicate_shares_total 1", metrics) - self.assertIn("qbit_prism_low_difficulty_shares_total 3", metrics) - self.assertIn("qbit_prism_grace_credited_shares_total 6", metrics) - self.assertIn("qbit_prism_stratum_active_connections 0", metrics) - self.assertIn("qbit_prism_stratum_connection_limit 384", metrics) - self.assertIn("qbit_prism_stratum_peak_active_connections 0", metrics) - self.assertIn("qbit_prism_stratum_pending_initial_jobs 0", metrics) - self.assertIn("qbit_prism_stratum_pending_initial_job_limit 128", metrics) - self.assertIn( - "qbit_prism_stratum_oldest_genuinely_pending_initial_job_seconds 0.0", - metrics, + self.assertTrue(server.submit_next_block_candidate()) + # The share was already accepted at submit time, so a lost block is a + # block-abandonment, not a stale share rejection. + self.assertEqual(server.stale_share_count, 0) + self.assertEqual(server.block_candidate_abandoned_counts[PRISM_REJECTION_STALE_JOB], 1) + self.assertEqual( + server.stale_job_abandon_counts, + {"tip_moved": 0, "balance_stale": 1, "append_epoch_stale": 0}, ) - self.assertIn( - "qbit_prism_stratum_current_tip_coverage_gap_seconds 0.0", - metrics, + outbox_row = ledger._block_candidate_outbox[submission.block_hash_hex] + self.assertEqual(outbox_row["state"], "abandoned") + self.assertEqual( + outbox_row["last_error"], + "prior balances changed since the job was issued", ) - self.assertIn("qbit_prism_stratum_current_tip_job_coverage 1.0", metrics) - self.assertIn("qbit_prism_stratum_handler_threads 0", metrics) - self.assertIn("qbit_prism_job_delivery_queue_depth 0", metrics) - self.assertIn("qbit_prism_job_delivery_active_workers 0", metrics) + metrics = server.metrics_payload() self.assertIn( - 'qbit_prism_stratum_connection_limit_rejections_total{scope="global"} 2', + 'qbit_prism_stale_job_abandons_total{class="tip_moved"} 0', metrics, ) self.assertIn( - 'qbit_prism_stratum_connection_limit_rejections_total{scope="username"} 3', + 'qbit_prism_stale_job_abandons_total{class="balance_stale"} 1', metrics, ) - self.assertIn("qbit_prism_stratum_accept_resource_exhaustions_total 4", metrics) - self.assertIn("qbit_prism_stratum_connection_setup_failures_total 5", metrics) - self.assertIn('qbit_prism_rejections_total{reason_id="stale-job"} 2', metrics) - self.assertIn('qbit_prism_rejections_total{reason_id="duplicate-share"} 1', metrics) - self.assertIn('qbit_prism_rejections_total{reason_id="low-difficulty"} 3', metrics) - self.assertIn("qbit_prism_tip_refresh_jobs_total 4", metrics) - self.assertIn("qbit_prism_post_accept_refresh_failures_total 5", metrics) - self.assertIn("qbit_prism_vardiff_idle_retargets_total 7", metrics) - self.assertIn("qbit_prism_vardiff_idle_clients_inspected_total 8", metrics) - self.assertIn('qbit_prism_vardiff_idle_skips_total{reason="busy"} 2', metrics) - self.assertIn("qbit_prism_vardiff_idle_queue_depth 1", metrics) - self.assertIn("qbit_prism_vardiff_idle_inflight 2", metrics) - self.assertIn("qbit_prism_vardiff_idle_task_failures_total 1", metrics) - self.assertIn("qbit_prism_vardiff_idle_sweep_seconds_count 1", metrics) - self.assertIn("qbit_prism_vardiff_idle_retarget_task_seconds_count 1", metrics) - self.assertIn("qbit_prism_stale_share_percent 20", metrics) - self.assertIn("qbit_prism_coinbase_weight_headroom_bytes 1999750", metrics) - self.assertIn("qbit_prism_vardiff_enabled 1", metrics) - self.assertIn("qbit_prism_qbitd_initial_block_download 0", metrics) - self.assertIn("qbit_prism_qbitd_peers 4", metrics) - - def test_hot_lock_and_pending_candidate_metrics_are_bounded(self) -> None: - server = coordinator() - server.lock = _ObservedRLock() - lock_wait_started = threading.Event() - - def contend() -> None: - lock_wait_started.set() - with server.lock: - pass - - with server.lock: - waiter = threading.Thread(target=contend) - waiter.start() - self.assertTrue(lock_wait_started.wait(1)) - time.sleep(0.02) - waiter.join(1) - self.assertFalse(waiter.is_alive()) - ledger = SingleWriterShareLedger() - block_hash = "bc" * 32 - ledger.persist_block_candidate_intent( + def test_block_submit_defers_descendant_until_active_ancestor_is_durable( + self, + ) -> None: + parent_hash = "ed" * 32 + ancestor_hash = "ee" * 32 + server, state, ledger = submit_coordinator(tip=parent_hash) + server.max_blocks = 10 + server.stop_after_block = False + ledger.durable_payout_state = True + preview = [ { - "schema": "qbit.prism.block-candidate-intent.v1", - "block_hash_hex": block_hash, - "block_hex": "00", + "recipient_id": "miner-a", + "order_key": "miner-a", + "p2mr_program_hex": "11" * 32, + "balance_sats": 25, } + ] + context = server.jobs["job-1"] + context.template["height"] = 12 + context.prior_balances = preview + original_rpc_call = server.rpc.call + submit_calls: list[str] = [] + + def active_ancestor_call( + method: str, + params: list[object] | None = None, + ) -> object: + if method == "getblockhash": + self.assertEqual(params, [10]) + return ancestor_hash + if method == "submitblock": + submit_calls.append(method) + return original_rpc_call(method, params) + + server.rpc.call = active_ancestor_call # type: ignore[method-assign] + server._begin_accepted_block_payout_preview( + ancestor_hash, + block_height=10, ) - server.ledger = ledger - before_attempt = "\n".join( - server.coordinator_lock_metrics_lines() - + server.block_submitter_metrics_lines() + server._publish_accepted_block_payout_preview(ancestor_hash, preview) + server.build_audit_bundle = lambda **_kwargs: (_ for _ in ()).throw( # type: ignore[method-assign] + AssertionError("descendant must wait for ancestor durability") ) - self.assertIn("qbit_prism_coordinator_lock_contentions_total 1", before_attempt) - self.assertIn("qbit_prism_block_candidates_pending 1", before_attempt) - self.assertRegex( - before_attempt, - r"qbit_prism_block_candidate_oldest_unattempted_seconds 0\.\d+", + submission = SimpleNamespace( + coinbase_tx_hex="c0ffee", + block_hash_hex="ef" * 32, + block_hex="00", ) - self.assertTrue(ledger.mark_block_candidate_attempted(block_hash=block_hash)) - after_attempt = "\n".join(server.block_submitter_metrics_lines()) - self.assertIn( - "qbit_prism_block_candidate_oldest_unattempted_seconds 0.000000", - after_attempt, + accepted = server.submit_block_candidate( + block_candidate(server, state, submission) ) - def test_metrics_include_bounded_worker_share_and_rejection_counters(self) -> None: - server = coordinator() - server.worker_metrics_limit = 1 - - server.note_worker_submitted_share("miner-a") - server.note_worker_accepted_share("miner-a", PRISM_CREDIT_POLICY_STALE_GRACE) - server.note_worker_submitted_share("miner-b") - server.record_rejection(PRISM_REJECTION_LOW_DIFFICULTY, worker="miner-b") - - metrics = server.metrics_payload() - - self.assertIn('qbit_prism_worker_submitted_shares_total{worker="miner-a"} 1', metrics) - self.assertIn('qbit_prism_worker_accepted_shares_total{worker="miner-a"} 1', metrics) - self.assertIn('qbit_prism_worker_grace_credited_shares_total{worker="miner-a"} 1', metrics) - self.assertIn('qbit_prism_worker_submitted_shares_total{worker="_other"} 1', metrics) - self.assertIn( - 'qbit_prism_worker_rejections_total{worker="_other",reason_id="low-difficulty"} 1', - metrics, + self.assertFalse(accepted) + # Node propagation is the fast lane: the durable descendant is offered + # to qbitd before payout/accounting notices the ancestor transition. + # Its accounting still defers until that ancestor is durable. + self.assertEqual(submit_calls, ["submitblock"]) + self.assertEqual(ledger.persisted, []) + self.assertEqual( + server._block_candidate_outcome.reason, + PRISM_REJECTION_LEDGER_CONFIRMATION_FAILED, ) - - def test_metrics_include_ctv_broadcaster_progress_and_pass_duration(self) -> None: - server = coordinator() - server._record_ctv_fanout_broadcaster_progress() - server._record_ctv_fanout_broadcaster_progress() - server.observe_ctv_fanout_broadcaster_pass(102.0) - server.observe_ctv_fanout_broadcaster_chunk( - SimpleNamespace(processed_count=1, elapsed_seconds=0.25) + self.assertNotIn( + PRISM_REJECTION_STALE_JOB, + server.block_candidate_abandoned_counts, ) - server._record_ctv_fanout_broadcaster_yield() + self.assertFalse(server.stop_event.is_set()) - metrics = server.metrics_payload() + def test_active_descendant_replay_stays_landed_until_ancestor_is_durable( + self, + ) -> None: + parent_hash = "ea" * 32 + ancestor_hash = "eb" * 32 + descendant_hash = "ec" * 32 + server, state, ledger = submit_coordinator(tip=parent_hash) + server.max_blocks = 10 + server.stop_after_block = False + ledger.durable_payout_state = True + preview = [ + { + "recipient_id": "miner-a", + "order_key": "miner-a", + "p2mr_program_hex": "11" * 32, + "balance_sats": 25, + } + ] + context = server.jobs["job-1"] + context.template["height"] = 12 + context.prior_balances = preview + original_rpc_call = server.rpc.call - self.assertIn("qbit_prism_ctv_fanout_broadcaster_processed_rows_total 2", metrics) - self.assertIn( - 'qbit_prism_ctv_fanout_broadcaster_pass_seconds_bucket{le="60"} 0', - metrics, + def active_descendant_call( + method: str, + params: list[object] | None = None, + ) -> object: + if method == "getbestblockhash": + return descendant_hash + if method == "getblockhash": + self.assertEqual(params, [10]) + return ancestor_hash + if method == "submitblock": + raise AssertionError("active descendant must not be resubmitted") + return original_rpc_call(method, params) + + server.rpc.call = active_descendant_call # type: ignore[method-assign] + server._begin_accepted_block_payout_preview( + ancestor_hash, + block_height=10, ) - self.assertIn( - 'qbit_prism_ctv_fanout_broadcaster_pass_seconds_bucket{le="120"} 1', - metrics, + server._publish_accepted_block_payout_preview(ancestor_hash, preview) + server.build_audit_bundle = lambda **_kwargs: (_ for _ in ()).throw( # type: ignore[method-assign] + AssertionError("active descendant must wait for ancestor durability") ) - self.assertIn("qbit_prism_ctv_fanout_broadcaster_pass_seconds_sum 102.000000", metrics) - self.assertIn("qbit_prism_ctv_fanout_broadcaster_pass_seconds_count 1", metrics) - self.assertIn( - "qbit_prism_ctv_fanout_broadcaster_tip_refresh_yields_total 1", - metrics, + submission = SimpleNamespace( + coinbase_tx_hex="c0ffee", + block_hash_hex=descendant_hash, + block_hex="00", ) - self.assertIn( - 'qbit_prism_ctv_fanout_broadcaster_chunk_seconds_bucket{le="0.25"} 1', - metrics, + + accepted = server.submit_block_candidate( + block_candidate(server, state, submission) ) - self.assertIn( - 'qbit_prism_ctv_fanout_broadcaster_chunk_rows_bucket{le="1"} 1', - metrics, - ) - - def test_zero_worker_metric_limit_uses_overflow_bucket(self) -> None: - server = coordinator() - server.worker_metrics_limit = 0 - - server.note_worker_submitted_share("miner-a") - - self.assertEqual(set(server.worker_share_counts), {"_other"}) - self.assertEqual(server.worker_share_counts["_other"]["submitted"], 1) - - def test_unauthorized_submit_does_not_admit_payload_worker_metric_label(self) -> None: - server, state, _ledger = submit_coordinator() - server.worker_metrics_limit = 1 - - with self.assertRaises(StratumError) as raised: - server.handle_submit( - state, - ["spoofed-miner", "job-1", "00" * 8, "00000001", "00000002"], - ) - self.assertEqual(raised.exception.reason, PRISM_REJECTION_UNAUTHORIZED_WORKER) - self.assertNotIn("spoofed-miner", server.worker_share_counts) - self.assertEqual(server.worker_share_counts["miner-a"]["submitted"], 0) + self.assertFalse(accepted) + self.assertEqual(ledger.persisted, []) self.assertEqual( - server.worker_rejection_counts[("miner-a", PRISM_REJECTION_UNAUTHORIZED_WORKER)], - 1, + server._block_candidate_outcome.reason, + PRISM_REJECTION_LEDGER_CONFIRMATION_FAILED, ) + transition = server._accepted_block_payout_previews[descendant_hash] + self.assertTrue(transition.landed) + self.assertIsNone(transition.preview) + self.assertFalse(server.stop_event.is_set()) - server.note_worker_submitted_share("miner-a") + def test_block_submit_reconciliation_error_is_structured_rejection(self) -> None: + tip = "f0" * 32 + server, state, _ledger = submit_coordinator(tip=tip) + server.reorg_reconciler_enabled = True + server.rpc = ReorgRpc( + tip=tip, + template=gbt_template(tip, height=10), + height=9, + block_hashes={9: tip}, + ) - self.assertNotIn(PRISM_WORKER_METRICS_OVERFLOW_LABEL, server.worker_share_counts) - self.assertEqual(server.worker_share_counts["miner-a"]["submitted"], 1) + class FailingSubmitReorgLedger(RecordingLedger): + def reorg_watch_blocks(self, *, active_tip_height: int) -> list[dict[str, object]]: + raise RuntimeError("ledger unavailable") - def test_metrics_include_audit_artifact_storage_gauges(self) -> None: - server = coordinator() - with tempfile.TemporaryDirectory() as tempdir: - server.audit_dir = Path(tempdir) - files = { - f"prism-audit-bundle-body-{'aa' * 32}-{'bb' * 32}.json": b"abc", - f"prism-audit-share-segment-1-1-{'cc' * 32}.json": b"defg", - f"prism-live-audit-bundle-1-{'dd' * 32}.json": b"hi", - f"prism-live-audit-bundle-candidate-{'ee' * 32}.json": b"j", - f".prism-live-audit-bundle-candidate-{'ff' * 32}.json.tmp": b"klmno", - "operator-note.txt": b"pqrstu", - } - for name, body in files.items(): - (Path(tempdir) / name).write_bytes(body) - - metrics = server.metrics_payload() - - self.assertIn('qbit_prism_audit_artifact_bytes{kind="body"} 3', metrics) - self.assertIn('qbit_prism_audit_artifact_files{kind="body"} 1', metrics) - self.assertIn('qbit_prism_audit_artifact_bytes{kind="share_segment"} 4', metrics) - self.assertIn('qbit_prism_audit_artifact_files{kind="share_segment"} 1', metrics) - self.assertIn('qbit_prism_audit_artifact_bytes{kind="live_bundle"} 2', metrics) - self.assertIn('qbit_prism_audit_artifact_files{kind="live_bundle"} 1', metrics) - self.assertIn('qbit_prism_audit_artifact_bytes{kind="candidate"} 6', metrics) - self.assertIn('qbit_prism_audit_artifact_files{kind="candidate"} 2', metrics) - self.assertIn('qbit_prism_audit_artifact_bytes{kind="other"} 6', metrics) - self.assertIn('qbit_prism_audit_artifact_files{kind="other"} 1', metrics) - self.assertIn("qbit_prism_audit_artifact_scan_error 0", metrics) - - def test_send_error_includes_canonical_reason_id_data(self) -> None: - server = coordinator() - sent: list[dict[str, object]] = [] - client = SimpleNamespace(send=lambda payload: sent.append(payload)) + ledger = FailingSubmitReorgLedger() + server.ledger = ledger + server.build_audit_bundle = lambda **_kwargs: (_ for _ in ()).throw( # type: ignore[method-assign] + AssertionError("audit bundle should not be rebuilt after reconcile failure") + ) + submission = SimpleNamespace( + coinbase_tx_hex="c0ffee", + block_hash_hex="f1" * 32, + block_hex="00", + ) - server.send_error(client, "submit-1", 21, "stale job", reason=PRISM_REJECTION_STALE_JOB) # type: ignore[arg-type] + with patch("builtins.print") as printed: + accepted = server.submit_block_candidate(block_candidate(server, state, submission)) + self.assertFalse(accepted) self.assertEqual( - sent, - [ - { - "id": "submit-1", - "result": None, - "error": [21, "stale job", {"reason_id": PRISM_REJECTION_STALE_JOB}], - } - ], + server.block_candidate_abandoned_counts.get(PRISM_REJECTION_BACKEND_RPC_UNAVAILABLE, 0), + 0, ) + messages = [str(call.args[0]) for call in printed.call_args_list if call.args] + self.assertTrue(any("block candidate deferred" in message for message in messages)) + self.assertFalse(any("block candidate abandoned" in message for message in messages)) + self.assertEqual(server.rejection_counts_by_reason[PRISM_REJECTION_BACKEND_RPC_UNAVAILABLE], 0) + self.assertEqual(ledger.persisted, []) + self.assertEqual(ledger.pending, []) - def test_scaled_target_difficulty_uses_pow_limit_units(self) -> None: - pow_limit = target_from_compact("207fffff") - - self.assertEqual(scaled_target_difficulty(pow_limit), 1_000_000) - self.assertEqual(scaled_target_difficulty(pow_limit // 4), 4_000_000) - - def test_qbit_gbt_rules_include_signet_rule_only_for_signet(self) -> None: - self.assertEqual(qbit_gbt_rules("regtest"), ["segwit"]) - self.assertEqual(qbit_gbt_rules("testnet4"), ["segwit"]) - self.assertEqual(qbit_gbt_rules("signet"), ["segwit", "signet"]) - - def test_qbit_template_fingerprint_ignores_clock_only_fields(self) -> None: - base = gbt_template("00" * 32, curtime=1) - base["longpollid"] = "10:0" - base["mintime"] = 1 - clock_only = gbt_template("00" * 32, curtime=2) - clock_only["longpollid"] = "10:1" - clock_only["mintime"] = 2 - changed_value = dict(clock_only) - changed_value["coinbasevalue"] = 49_99999999 + def test_address_worker_suffixes_share_one_payout_account(self) -> None: + server, state_a, ledger = submit_coordinator() + usernames = [f"{PAYOUT_ADDRESS}.rig-a", f"{PAYOUT_ADDRESS}.rig-b"] + submissions = [ + SimpleNamespace( + header_hex="aa" * 80, + block_hash_hex="bd" * 32, + share_pass=True, + block_pass=False, + ), + SimpleNamespace( + header_hex="bb" * 80, + block_hash_hex="be" * 32, + share_pass=True, + block_pass=False, + ), + ] + states = [state_a, client()] + states[1].active_job_ids = {"job-1"} + for state, username in zip(states, usernames, strict=True): + worker = WorkerIdentity( + username=username, + payout_address=PAYOUT_ADDRESS, + worker_name=username.rsplit(".", 1)[1], + script_pubkey_hex="5220" + "55" * 32, + p2mr_program_hex="55" * 32, + ) + state.username = username + state.worker = worker + state.subscribed = True + state.authorized = True + server.jobs["job-1"].worker = worker + with patch( + "lab.prism.prism_coordinator.direct_stratum.assemble_submission", + return_value=submissions.pop(0), + ): + server.handle_submit( + state, + [username, "job-1", "00" * 8, "00000001", "00000002"], + ) - self.assertEqual(qbit_template_fingerprint(base), qbit_template_fingerprint(clock_only)) - self.assertNotEqual(qbit_template_fingerprint(base), qbit_template_fingerprint(changed_value)) + self.assertEqual([pending.miner_id for pending in ledger.pending], [PAYOUT_ADDRESS, PAYOUT_ADDRESS]) + self.assertEqual([pending.order_key for pending in ledger.pending], [PAYOUT_ADDRESS, PAYOUT_ADDRESS]) - def test_resolve_version_mask_uses_gbt_versionrollingmask(self) -> None: - server = PrismCoordinator.__new__(PrismCoordinator) - rpc = TemplateRpc({"versionrollingmask": "1fffe000"}) - server.rpc = rpc - server.qbit_chain = "signet" + def _pending_append(self, tag: str, accepted_at_ms: int = 2) -> PendingShareAppend: + from lab.prism.share_ledger import PendingShare - selection = server.resolve_version_rolling_mask(0x000000FF) + return PendingShareAppend( + pending_share=PendingShare( + share_id=f"miner-a:{tag}", + miner_id="miner-a", + order_key="miner-a", + p2mr_program_hex="11" * 32, + share_difficulty=1, + network_difficulty=1, + template_height=10, + job_id="job-1", + job_issued_at_ms=1, + accepted_at_ms=accepted_at_ms, + ntime=1_700_000_000, + ), + username="miner-a", + job_id="job-1", + block_hash_hex=tag * 32, + collection_only=False, + credit_policy=None, + ) - self.assertEqual(selection.selected_mask, 0x1FFFE000) - self.assertEqual(selection.source, "qbit_getblocktemplate") - self.assertEqual(rpc.calls, [("getblocktemplate", [{"rules": ["segwit", "signet"]}])]) + def test_orphaned_block_candidate_keeps_share_credit(self) -> None: + # Option-A semantics: a share that met its target stays credited even + # when its block candidate loses the tip race in the submitter. + old_tip = "00" * 32 + new_tip = "11" * 32 + server, state, _recording = submit_coordinator(tip=old_tip) + configure_temporary_audit_root(self, server) + ledger = SingleWriterShareLedger() + server.ledger = ledger + submission = SimpleNamespace( + header_hex="aa" * 80, + block_hash_hex="cc" * 32, + block_hex="00", + share_pass=True, + block_pass=True, + ) - def test_resolve_version_mask_falls_back_only_when_gbt_missing_or_unavailable(self) -> None: - missing = PrismCoordinator.__new__(PrismCoordinator) - missing.rpc = TemplateRpc({}) - missing.qbit_chain = "regtest" + with patch( + "lab.prism.prism_coordinator.direct_stratum.assemble_submission", + return_value=submission, + ): + server.handle_submit( + state, + ["miner-a", "job-1", "00" * 8, "00000001", "00000002"], + ) - missing_selection = missing.resolve_version_rolling_mask(direct_stratum.QBIT_VERSION_ROLLING_MASK) + self.assertEqual(len(ledger.all_shares()), 1) + # The tip moves before the submitter drains the candidate. + server.rpc = RejectingSubmitTipRpc(new_tip) - self.assertEqual(missing_selection.selected_mask, direct_stratum.QBIT_VERSION_ROLLING_MASK) - self.assertEqual(missing_selection.source, "fallback") - self.assertEqual(missing_selection.detail, "missing_versionrollingmask") + self.assertTrue(server.submit_next_block_candidate()) - unavailable = PrismCoordinator.__new__(PrismCoordinator) - unavailable.rpc = FakeRpc() - unavailable.qbit_chain = "regtest" + self.assertEqual(server.accepted_block_count, 0) + # Counted as a block abandonment, not a share rejection. + self.assertEqual(server.block_candidate_abandoned_counts[PRISM_REJECTION_STALE_JOB], 1) + self.assertEqual( + server.stale_job_abandon_counts, + {"tip_moved": 1, "balance_stale": 0, "append_epoch_stale": 0}, + ) + self.assertEqual(server.stale_share_count, 0) + # The credited share survives the lost block race. + self.assertEqual(len(ledger.all_shares()), 1) + outbox_row = ledger._block_candidate_outbox[submission.block_hash_hex] + self.assertEqual(outbox_row["state"], "abandoned") + self.assertEqual( + outbox_row["last_error"], + f"tip moved before submit: {new_tip}", + ) + metrics = server.metrics_payload() + self.assertIn( + 'qbit_prism_stale_job_abandons_total{class="tip_moved"} 1', + metrics, + ) + self.assertIn( + 'qbit_prism_stale_job_abandons_total{class="balance_stale"} 0', + metrics, + ) - unavailable_selection = unavailable.resolve_version_rolling_mask(direct_stratum.QBIT_VERSION_ROLLING_MASK) + def test_block_candidate_queue_overflow_coalesces_wakeup_without_drop(self) -> None: + server, state, _ledger = submit_coordinator() + configure_temporary_audit_root(self, server) + server.block_candidate_queue = queue.Queue(maxsize=2) - self.assertEqual(unavailable_selection.selected_mask, direct_stratum.QBIT_VERSION_ROLLING_MASK) - self.assertEqual(unavailable_selection.source, "fallback") - self.assertTrue(unavailable_selection.detail.startswith("probe_error:")) + def candidate(tag: str) -> PrismBlockCandidate: + return block_candidate( + server, + state, + SimpleNamespace(block_hash_hex=tag * 32, share_pass=True, block_pass=True), + ) - def test_resolve_version_mask_disables_only_on_gbt_zero_mask(self) -> None: - server = PrismCoordinator.__new__(PrismCoordinator) - server.rpc = TemplateRpc({"versionrollingmask": "00000000"}) - server.qbit_chain = "regtest" + server.enqueue_block_candidate(candidate("aa")) + server.enqueue_block_candidate(candidate("bb")) + server.enqueue_block_candidate(candidate("cc")) - selection = server.resolve_version_rolling_mask(direct_stratum.QBIT_VERSION_ROLLING_MASK) + self.assertEqual(server.block_candidates_dropped, 0) + self.assertEqual(server.block_candidate_queue.qsize(), 2) + remaining = [ + server.block_candidate_queue.get_nowait().submission.block_hash_hex + for _ in range(2) + ] + # Existing wakeups remain ordered; the third candidate remains durable + # in the outbox and will be re-read after the queue drains. + self.assertEqual(remaining, ["aa" * 32, "bb" * 32]) + self.assertIn( + "qbit_prism_block_candidates_dropped_total 0", server.metrics_payload() + ) + self.assertIn( + "qbit_prism_block_candidate_wakeups_coalesced_total 1", + server.metrics_payload(), + ) - self.assertEqual(selection.selected_mask, 0) - self.assertEqual(selection.source, "qbit_getblocktemplate") - self.assertEqual(selection.detail, "disabled_by_zero_mask") + def test_b1_codec_owner_round_trips_without_coordinator_state(self) -> None: + server, state, _ledger = submit_coordinator() + candidate = block_candidate( + server, + state, + SimpleNamespace( + block_hash_hex="ca" * 32, + block_hex="00", + coinbase_tx_hex="11", + share_pass=True, + block_pass=True, + ), + pending_share=PendingShare( + share_id="codec-share", + miner_id="miner-a", + order_key="miner-a", + p2mr_program_hex="11" * 32, + share_difficulty=1, + network_difficulty=1, + template_height=10, + job_id="job-1", + job_issued_at_ms=1, + accepted_at_ms=2, + ntime=3, + ), + ) - def test_resolve_version_mask_rejects_invalid_gbt_mask(self) -> None: - server = PrismCoordinator.__new__(PrismCoordinator) - server.rpc = TemplateRpc({"versionrollingmask": "not-hex"}) - server.qbit_chain = "regtest" + intent = block_candidate_intent(candidate) + replayed = block_candidate_from_intent(intent) - with self.assertRaisesRegex(SystemExit, "invalid getblocktemplate.versionrollingmask"): - server.resolve_version_rolling_mask(direct_stratum.QBIT_VERSION_ROLLING_MASK) + self.assertEqual(replayed.submission.block_hash_hex, "ca" * 32) + self.assertEqual(replayed.context.template, candidate.context.template) + self.assertEqual(replayed.pending_share, candidate.pending_share) + # The module decode carries the restart sentinel: process-local + # append epochs are meaningless after a restart, so the landing + # fence must revalidate the recorded window instead. + self.assertEqual(replayed.context.payout_append_invalidation_epoch, -1) - def test_configure_negotiates_requested_mask_with_gbt_server_mask(self) -> None: - server = coordinator() - server.version_mask = 0x1FFFE000 - state = client() - captured: dict[str, object] = {} - server.send_result = lambda _client, request_id, result: captured.update( # type: ignore[method-assign] - {"request_id": request_id, "result": result} - ) + def test_b1_intent_refuses_a_submission_with_no_serialized_block(self) -> None: + """An unmaterialized block must never become a durable intent. - server.handle_configure( + ``assemble_submission`` builds ``block_hex`` only for a hash that + solved the block, which every candidate route requires. If that + coupling were ever broken the block is already lost; persisting the + empty value would lose it again after every restart, as a candidate + that can never be offered to the node. Fail on the client thread + instead, before the durable write. + """ + server, state, _ledger = submit_coordinator() + candidate = block_candidate( + server, state, - "configure-1", - [ - ["version-rolling"], - {"version-rolling.mask": "0000f000"}, - ], + direct_stratum.DirectQbitSubmission( + coinbase_tx_hex="11", + coinbase_txid_preimage_hex="", + header_hex="", + block_hex="", + block_hash_hex="cb" * 32, + block_hash_int=1, + share_pass=True, + block_pass=True, + applied_version_hex="", + ), ) - self.assertEqual(captured["request_id"], "configure-1") - self.assertEqual( - captured["result"], - { - "version-rolling": True, - "version-rolling.mask": "0000e000", - }, - ) - self.assertEqual(state.version_mask, 0x0000E000) + with self.assertRaisesRegex(ValueError, "carries no serialized block"): + block_candidate_intent(candidate) - def test_configure_disables_version_rolling_when_gbt_mask_is_zero(self) -> None: - server = coordinator() - server.version_mask = 0 - state = client() - captured: dict[str, object] = {} - server.send_result = lambda _client, request_id, result: captured.update( # type: ignore[method-assign] - {"request_id": request_id, "result": result} - ) + def test_b1_intent_still_tolerates_an_embedder_without_block_bytes(self) -> None: + """The invariant belongs to the real submission type, not to stubs. - server.handle_configure( + Some integrations retain no block bytes at all; the landing path + already guards them by attribute probe. Tightening the codec must not + turn those into hard failures. + """ + server, state, _ledger = submit_coordinator() + candidate = block_candidate( + server, state, - "configure-1", - [ - ["version-rolling"], - {"version-rolling.mask": "ffffffff"}, - ], + SimpleNamespace( + block_hash_hex="cc" * 32, + coinbase_tx_hex="11", + share_pass=True, + block_pass=True, + ), + pending_share=PendingShare( + share_id="embedder-share", + miner_id="miner-a", + order_key="miner-a", + p2mr_program_hex="11" * 32, + share_difficulty=1, + network_difficulty=1, + template_height=10, + job_id="job-1", + job_issued_at_ms=1, + accepted_at_ms=2, + ntime=3, + ), ) - self.assertEqual( - captured["result"], - { - "version-rolling": False, - "version-rolling.mask": "00000000", - }, - ) - self.assertEqual(state.version_mask, 0) + intent = block_candidate_intent(candidate) - def test_accepted_share_difficulty_uses_actual_target_unless_overridden(self) -> None: - server = coordinator() - worker = WorkerIdentity( - username="miner-a", - payout_address="miner-a", - worker_name=None, - script_pubkey_hex="5220" + "11" * 32, + self.assertEqual(intent["block_hex"], "") + + def test_b1_replayed_intent_re_encodes_without_tripping_the_guard(self) -> None: + """Decode erases duck-typing, so the guard must exempt a replay. + + An embedder that retains no block bytes writes ``""`` and is tolerated. + ``block_candidate_from_intent`` then rebuilds every replayed candidate + as a real ``DirectQbitSubmission``, so that tolerated value comes back + wearing the strict type. The only caller that re-encodes a replayed + candidate is the credit-on-accept append, which runs after the block + has landed -- raising there would fail a credit for a block already in + the chain. Re-persisting an intent the row already holds is not the + write the guard exists to stop. + """ + server, state, _ledger = submit_coordinator() + pending = PendingShare( + share_id="replay-share", + miner_id="miner-a", + order_key="miner-a", p2mr_program_hex="11" * 32, + share_difficulty=1, + network_difficulty=1, + template_height=10, + job_id="job-1", + job_issued_at_ms=1, + accepted_at_ms=2, + ntime=3, ) - context = SimpleNamespace( - worker=worker, - job=SimpleNamespace(share_target=target_from_compact("207fffff") // 2), + written = block_candidate_intent( + block_candidate( + server, + state, + SimpleNamespace( + block_hash_hex="cf" * 32, + coinbase_tx_hex="11", + share_pass=True, + block_pass=True, + ), + pending_share=pending, + credit_share_on_accept=True, + ) ) - server.share_weights_by_username = {} - - self.assertEqual(server.accepted_share_difficulty(context), 2_000_000) - - server.share_weights_by_username = {"miner-a": 7} - self.assertEqual(server.accepted_share_difficulty(context), 7) - - def test_resolve_worker_accepts_bare_payout_address(self) -> None: - server = PrismCoordinator.__new__(PrismCoordinator) - rpc = AddressValidationRpc(script_byte="22") - server.rpc = rpc - - worker = server.resolve_worker(PAYOUT_ADDRESS) + self.assertEqual(written["block_hex"], "") - self.assertEqual(rpc.validated, [PAYOUT_ADDRESS]) - self.assertEqual(worker.username, PAYOUT_ADDRESS) - self.assertEqual(worker.payout_address, PAYOUT_ADDRESS) - self.assertIsNone(worker.worker_name) - self.assertEqual(worker.p2mr_program_hex, "22" * 32) - - def test_resolve_worker_accepts_address_worker_username(self) -> None: - server = PrismCoordinator.__new__(PrismCoordinator) - rpc = AddressValidationRpc(script_byte="33") - server.rpc = rpc + replayed = dataclasses.replace( + block_candidate_from_intent(written), + durable_replay=True, + ) + # Decode really does hand back the strict type; without the replay + # exemption this is exactly what would trip the guard. + self.assertIsInstance( + replayed.submission, + direct_stratum.DirectQbitSubmission, + ) + self.assertEqual(replayed.submission.block_hex, "") - worker = server.resolve_worker(f"{PAYOUT_ADDRESS}.rig-a") + self.assertEqual(block_candidate_intent(replayed)["block_hex"], "") - self.assertEqual(rpc.validated, [PAYOUT_ADDRESS]) - self.assertEqual(worker.username, f"{PAYOUT_ADDRESS}.rig-a") - self.assertEqual(worker.payout_address, PAYOUT_ADDRESS) - self.assertEqual(worker.worker_name, "rig-a") - self.assertEqual(worker.p2mr_program_hex, "33" * 32) + def test_b1_replayed_intent_preserves_a_real_serialized_block(self) -> None: + """The ordinary replay: block bytes survive the round trip intact.""" + server, state, _ledger = submit_coordinator() + pending = PendingShare( + share_id="replay-real-share", + miner_id="miner-a", + order_key="miner-a", + p2mr_program_hex="11" * 32, + share_difficulty=1, + network_difficulty=1, + template_height=10, + job_id="job-1", + job_issued_at_ms=1, + accepted_at_ms=2, + ntime=3, + ) + written = block_candidate_intent( + block_candidate( + server, + state, + direct_stratum.DirectQbitSubmission( + coinbase_tx_hex="11", + coinbase_txid_preimage_hex="", + header_hex="", + block_hex="0badc0de", + block_hash_hex="da" * 32, + block_hash_int=1, + share_pass=True, + block_pass=True, + applied_version_hex="", + ), + pending_share=pending, + credit_share_on_accept=True, + ) + ) - def test_resolve_worker_caches_successful_address_validation(self) -> None: - server = PrismCoordinator.__new__(PrismCoordinator) - rpc = AddressValidationRpc(script_byte="33") - server.rpc = rpc + replayed = dataclasses.replace( + block_candidate_from_intent(written), + durable_replay=True, + ) - first = server.resolve_worker(f"{PAYOUT_ADDRESS}.rig-a") - second = server.resolve_worker(f"{PAYOUT_ADDRESS}.rig-b") + self.assertEqual(replayed.submission.block_hex, "0badc0de") + self.assertEqual(block_candidate_intent(replayed)["block_hex"], "0badc0de") - self.assertEqual(rpc.validated, [PAYOUT_ADDRESS]) - self.assertEqual(first.p2mr_program_hex, second.p2mr_program_hex) + def test_b1_service_owns_queue_and_structured_attempt_results(self) -> None: + server, state, _ledger = submit_coordinator() + service = server._ensure_block_candidate_service() + self.assertIs(server.block_candidate_queue, service.candidate_queue) + self.assertIsInstance(service.submit_next(), BlockCandidateRunResult) + self.assertFalse(service.submit_next().ran) - def test_payout_address_cache_evicts_least_recently_used_entry(self) -> None: - server = PrismCoordinator.__new__(PrismCoordinator) - server.payout_address_cache_max_entries = 2 - server.payout_address_cache_ttl_seconds = 60 + candidate = block_candidate( + server, + state, + SimpleNamespace( + block_hash_hex="cb" * 32, + share_pass=True, + block_pass=True, + ), + ) - class AnyAddressRpc: - def __init__(self) -> None: - self.validated: list[str] = [] + def reject(_candidate: PrismBlockCandidate) -> bool: + server._abandon_block_candidate( + PRISM_REJECTION_STALE_JOB, + "direct owner result", + block_hash="cb" * 32, + worker="miner-a", + ) + return False - def call(self, method: str, params: list[object] | None = None) -> object: - address = str((params or [""])[0]) - self.validated.append(address) - return {"isvalid": True, "scriptPubKey": "5220" + "33" * 32} + server.submit_block_candidate = reject # type: ignore[method-assign] + with patch("builtins.print"): + result = service.attempt(candidate) - rpc = AnyAddressRpc() - server.rpc = rpc + self.assertIsInstance(result, BlockCandidateAttemptResult) + self.assertFalse(result.accepted) + self.assertEqual(result.reason, PRISM_REJECTION_STALE_JOB) - server.validate_p2mr_address("address-a", label="test") - server.validate_p2mr_address("address-b", label="test") - server.validate_p2mr_address("address-a", label="test") - server.validate_p2mr_address("address-c", label="test") + def test_b1_service_resolves_replaced_stop_event_at_use_time(self) -> None: + server, _state, _ledger = submit_coordinator() + service = server._ensure_block_candidate_service() + replacement = threading.Event() + server.stop_event = replacement + replacement.set() - self.assertEqual(rpc.validated, ["address-a", "address-b", "address-c"]) - self.assertEqual(list(server._p2mr_address_cache), ["address-a", "address-c"]) + service.run() - server.validate_p2mr_address("address-b", label="test") - self.assertEqual(rpc.validated[-1], "address-b") - self.assertEqual(len(server._p2mr_address_cache), 2) + self.assertIs(service.ports.stop_event(), replacement) - def test_payout_address_cache_revalidates_expired_entry(self) -> None: - server = PrismCoordinator.__new__(PrismCoordinator) - server.payout_address_cache_max_entries = 2 - server.payout_address_cache_ttl_seconds = 5 - rpc = AddressValidationRpc(script_byte="33") - server.rpc = rpc - now = 100.0 + def test_b1_landing_tail_routes_through_the_named_submit_port(self) -> None: + # #127: both landing consumers hand the whole call shape to the named + # submit port instead of asking the runtime seam which tail is live. + server, state, _ledger = submit_coordinator() + service = server._ensure_block_candidate_service() + self.assertFalse(hasattr(server, "_is_production_block_submit")) + calls: list[tuple[str, object, bool]] = [] - with patch( - "lab.prism.prism_coordinator.time.monotonic", - side_effect=lambda: now, - ): - server.validate_p2mr_address(PAYOUT_ADDRESS, label="test") - now = 104.0 - server.validate_p2mr_address(PAYOUT_ADDRESS, label="test") - now = 106.0 - server.validate_p2mr_address(PAYOUT_ADDRESS, label="test") + def recording_port( + landing: PrismBlockCandidate, + *, + node_submission: object = None, + disposition_held: bool = False, + ) -> bool: + calls.append( + ( + str(landing.submission.block_hash_hex), + node_submission, + disposition_held, + ) + ) + return True - self.assertEqual(rpc.validated, [PAYOUT_ADDRESS, PAYOUT_ADDRESS]) + service.ports = dataclasses.replace( + service.ports, + submit_candidate=recording_port, + ) + queued = block_candidate( + server, + state, + SimpleNamespace( + coinbase_tx_hex="00", + block_hash_hex="d1" * 32, + block_hex="00", + share_pass=True, + block_pass=True, + ), + ) + server.enqueue_block_candidate(queued) + self.assertTrue(server.submit_next_block_candidate()) - def test_concurrent_worker_resolution_singleflights_address_validation(self) -> None: - server = PrismCoordinator.__new__(PrismCoordinator) - entered = threading.Event() - release = threading.Event() + synchronous = block_candidate( + server, + state, + SimpleNamespace( + coinbase_tx_hex="00", + block_hash_hex="d2" * 32, + block_hex="00", + share_pass=True, + block_pass=True, + ), + ) + self.assertTrue(server._submit_synchronous_block_candidate(synchronous)) - class BlockingAddressRpc(AddressValidationRpc): - def call(self, method: str, params: list[object] | None = None) -> object: - if method == "validateaddress": - entered.set() - if not release.wait(timeout=5): - raise TimeoutError("test did not release validateaddress") - return super().call(method, params) + self.assertEqual([call[0] for call in calls], ["d1" * 32, "d2" * 32]) + # Both consumers already own the same-hash disposition, and both pass + # the node submission they created rather than leaving it to the port. + self.assertEqual([call[2] for call in calls], [True, True]) + for _block_hash, node_submission, _held in calls: + self.assertIsInstance(node_submission, _BlockCandidateNodeSubmission) - rpc = BlockingAddressRpc(script_byte="33") - server.rpc = rpc - server._ensure_p2mr_address_cache_state() - workers: list[WorkerIdentity] = [] - errors: list[BaseException] = [] + def test_b1_submit_port_honours_replacement_installed_after_construction( + self, + ) -> None: + # The service is built before the double is installed, so the named + # port must resolve the coordinator's callable at call time. + server, state, _ledger = submit_coordinator() + service = server._ensure_block_candidate_service() + landed: list[str] = [] - def resolve(index: int) -> None: - try: - workers.append(server.resolve_worker(f"{PAYOUT_ADDRESS}.rig-{index}")) - except BaseException as exc: # pragma: no cover - surfaced below - errors.append(exc) + def double(landing: PrismBlockCandidate) -> bool: + landed.append(str(landing.submission.block_hash_hex)) + return True - threads = [threading.Thread(target=resolve, args=(index,)) for index in range(8)] - for thread in threads: - thread.start() - self.assertTrue(entered.wait(timeout=5)) - release.set() - for thread in threads: - thread.join(timeout=5) + server.submit_block_candidate = double # type: ignore[method-assign] + candidate = block_candidate( + server, + state, + SimpleNamespace( + coinbase_tx_hex="00", + block_hash_hex="d3" * 32, + block_hex="00", + share_pass=True, + block_pass=True, + ), + ) + server.enqueue_block_candidate(candidate) + self.assertTrue(server.submit_next_block_candidate()) - self.assertFalse(errors) - self.assertEqual(len(workers), 8) - self.assertEqual(rpc.validated, [PAYOUT_ADDRESS]) + self.assertEqual(landed, ["d3" * 32]) + self.assertIs(service, server._ensure_block_candidate_service()) - def test_concurrent_failed_worker_resolution_shares_singleflight_error(self) -> None: - server = PrismCoordinator.__new__(PrismCoordinator) - entered = threading.Event() - release = threading.Event() + def test_b1_submit_port_preserves_every_former_landing_call_shape(self) -> None: + server, state, _ledger = submit_coordinator() + port = server._ensure_block_candidate_service().ports.submit_candidate + candidate = block_candidate( + server, + state, + SimpleNamespace( + coinbase_tx_hex="00", + block_hash_hex="d4" * 32, + block_hex="00", + share_pass=True, + block_pass=True, + ), + ) + node_submission = _BlockCandidateNodeSubmission(attempted=False) - class FailingAddressRpc: - def __init__(self) -> None: - self.calls = 0 + # Shape 1: production entrypoint plus a held disposition runs the + # serialized inner tail, not the public entrypoint that would take + # that same-hash guard a second time. + serialized: list[object] = [] + server._submit_block_candidate_serialized = ( # type: ignore[method-assign] + lambda _landing, *, node_submission: ( + serialized.append(node_submission) or True + ) + ) + self.assertTrue( + port(candidate, node_submission=node_submission, disposition_held=True) + ) + self.assertEqual(serialized, [node_submission]) - def call(self, method: str, params: list[object] | None = None) -> object: - self.calls += 1 - entered.set() - if not release.wait(timeout=5): - raise TimeoutError("test did not release validateaddress") - raise RuntimeError("qbitd unavailable") + shapes: list[tuple[str, object]] = [] - rpc = FailingAddressRpc() - server.rpc = rpc - server._ensure_p2mr_address_cache_state() - errors: list[BaseException] = [] + def bare_double(_landing: PrismBlockCandidate) -> bool: + shapes.append(("bare", None)) + return True - def resolve() -> None: - try: - server.resolve_worker(PAYOUT_ADDRESS) - except BaseException as exc: - errors.append(exc) + # Shape 2: no node submission was created, so the historical bare + # entrypoint call stands -- including when the disposition is held, + # which is the writer path's middle arm. + server.submit_block_candidate = bare_double # type: ignore[method-assign] + self.assertTrue(port(candidate)) + self.assertTrue( + port(candidate, node_submission=None, disposition_held=True) + ) - threads = [threading.Thread(target=resolve) for _ in range(8)] - for thread in threads: - thread.start() - self.assertTrue(entered.wait(timeout=5)) - deadline = time.monotonic() + 5 - while time.monotonic() < deadline: - with server._p2mr_address_cache_lock: - pending = server._p2mr_address_validation_inflight[PAYOUT_ADDRESS] - if pending.waiters == 7: - break - time.sleep(0.001) - self.assertEqual(pending.waiters, 7) - release.set() - for thread in threads: - thread.join(timeout=5) + def sniffed_double( + _landing: PrismBlockCandidate, + *, + node_submission: object = None, + ) -> bool: + shapes.append(("node-submission", node_submission)) + return True - self.assertEqual(rpc.calls, 1) - self.assertEqual(len(errors), 8) - self.assertTrue(all("qbitd unavailable" in str(exc) for exc in errors)) + # Shape 3: an already-created node submission is passed through to a + # replacement whose signature can accept it. + server.submit_block_candidate = sniffed_double # type: ignore[method-assign] + self.assertTrue( + port(candidate, node_submission=node_submission, disposition_held=True) + ) + self.assertTrue(port(candidate, node_submission=node_submission)) - def test_resolve_worker_rejects_invalid_base_address_with_worker_suffix(self) -> None: - server = PrismCoordinator.__new__(PrismCoordinator) - rpc = AddressValidationRpc() - server.rpc = rpc - server.username_fallback_address = None + self.assertEqual( + shapes, + [ + ("bare", None), + ("bare", None), + ("node-submission", node_submission), + ("node-submission", node_submission), + ], + ) - with self.assertRaises(StratumError) as raised: - server.resolve_worker("not-a-qbit-address.rig-a") + def test_b1_production_landing_still_reaches_the_serialized_tail(self) -> None: + # #127 is structural hygiene: submit-double tests still do not run + # _submit_block_candidate_serialized. This pins the other side -- with + # no double installed, ordinary production operation still does. + old_tip = "00" * 32 + new_tip = "11" * 32 + block_hash = "cd" * 32 + server, state, _recording = submit_coordinator(tip=old_tip) + ledger = SingleWriterShareLedger() + server.ledger = ledger + pending = self._pending_append("production-serialized-tail").pending_share + candidate = block_candidate( + server, + state, + SimpleNamespace( + coinbase_tx_hex="00", + block_hash_hex=block_hash, + block_hex="00", + share_pass=True, + block_pass=True, + ), + pending_share=pending, + ) + ledger.append_batch([(pending, server.block_candidate_intent(candidate))]) + submitted: list[str] = [] + original_submitted = ledger.mark_block_candidate_submitted - self.assertEqual(raised.exception.code, 20) - self.assertEqual(rpc.validated, ["not-a-qbit-address"]) + def mark_submitted(*, block_hash: str) -> bool: + submitted.append(block_hash) + return original_submitted(block_hash=block_hash) - def test_resolve_worker_uses_configured_fallback_for_invalid_username(self) -> None: - fallback_address = "tq1fallback" - server = PrismCoordinator.__new__(PrismCoordinator) - rpc = AddressValidationRpc(valid_address=fallback_address, script_byte="44") - server.rpc = rpc - server.username_fallback_address = fallback_address + ledger.mark_block_candidate_submitted = mark_submitted # type: ignore[method-assign] + server._begin_accepted_block_payout_preview(block_hash, block_height=10) + server._mark_accepted_block_payout_landed(block_hash, block_height=10) + with server.lock: + server._accounted_accepted_block_hashes.add(block_hash) + server.rpc = TipRpc(new_tip) + serialized: list[object] = [] + real_serialized = server._submit_block_candidate_serialized - worker = server.resolve_worker("not-a-qbit-address.rig-a") + def recording_serialized( + landing: PrismBlockCandidate, + *, + node_submission: object, + ) -> bool: + serialized.append(node_submission) + return real_serialized(landing, node_submission=node_submission) - self.assertEqual(rpc.validated, ["not-a-qbit-address", fallback_address]) - self.assertEqual(worker.username, "not-a-qbit-address.rig-a") - self.assertEqual(worker.payout_address, fallback_address) - self.assertEqual(worker.worker_name, "rig-a") - self.assertEqual(worker.p2mr_program_hex, "44" * 32) + server._submit_block_candidate_serialized = ( # type: ignore[method-assign] + recording_serialized + ) + server.enqueue_block_candidate(candidate) - def test_resolve_worker_uses_testnet_default_fallback_for_invalid_username(self) -> None: - server = PrismCoordinator.__new__(PrismCoordinator) - rpc = AddressValidationRpc(valid_address=DEFAULT_TESTNET_USERNAME_FALLBACK_ADDRESS, script_byte="55") - server.rpc = rpc + self.assertTrue(server.submit_next_block_candidate()) - with patch.dict(os.environ, {"QBIT_CHAIN": "testnet4"}, clear=True): - worker = server.resolve_worker("not-a-qbit-address") + self.assertEqual(submitted, [block_hash]) + self.assertEqual(len(serialized), 1) + self.assertIsInstance(serialized[0], _BlockCandidateNodeSubmission) - self.assertEqual(rpc.validated, ["not-a-qbit-address", DEFAULT_TESTNET_USERNAME_FALLBACK_ADDRESS]) - self.assertEqual(worker.username, "not-a-qbit-address") - self.assertEqual(worker.payout_address, DEFAULT_TESTNET_USERNAME_FALLBACK_ADDRESS) - self.assertIsNone(worker.worker_name) - self.assertEqual(worker.p2mr_program_hex, "55" * 32) + def test_failed_replay_adoption_does_not_abort_remaining_rows(self) -> None: + server, state, _recording = submit_coordinator() + ledger = SingleWriterShareLedger() + server.ledger = ledger - def test_default_username_fallback_is_testnet_only_unless_configured(self) -> None: - with patch.dict(os.environ, {"QBIT_CHAIN": "testnet4"}, clear=True): - self.assertEqual(default_prism_username_fallback_address(), DEFAULT_TESTNET_USERNAME_FALLBACK_ADDRESS) - with patch.dict(os.environ, {"QBIT_CHAIN": "regtest"}, clear=True): - self.assertIsNone(default_prism_username_fallback_address()) - with patch.dict( - os.environ, - {"QBIT_CHAIN": "regtest", "PRISM_USERNAME_FALLBACK_ADDRESS": "qbrt1fallback"}, - clear=True, - ): - self.assertEqual(default_prism_username_fallback_address(), "qbrt1fallback") - - def test_prism_payout_policy_defaults_to_no_pool_fee(self) -> None: - server = coordinator() + def durable_candidate(tag: str, stamp: int) -> PrismBlockCandidate: + pending = PendingShare( + share_id=f"miner-a:{tag * 32}", + miner_id="miner-a", + order_key="miner-a", + p2mr_program_hex="11" * 32, + share_difficulty=1, + network_difficulty=1, + template_height=9, + job_id="job-1", + job_issued_at_ms=1, + accepted_at_ms=stamp, + ntime=1, + ) + value = dataclass_replace( + block_candidate( + server, + state, + SimpleNamespace( + coinbase_tx_hex="00", + block_hash_hex=tag * 32, + block_hex="00", + share_pass=False, + block_pass=True, + ), + pending_share=pending, + ), + credit_share_on_accept=True, + ) + ledger.append_batch([(pending, server.block_candidate_intent(value))]) + return value - with patch.dict(os.environ, {}, clear=True): - policy = server.prism_payout_policy() + first = durable_candidate("a5", 1) + second = durable_candidate("b5", 2) + service = server._ensure_block_candidate_service() + original_adopt = service.adopt_replayed_candidate - self.assertEqual( - policy, - { - "p2mr_spend_input_bytes": 3_680, - "target_feerate_sats_per_byte": 1, - "safety_multiplier": 4, - }, - ) + def fail_first(value: PrismBlockCandidate) -> None: + if value.submission.block_hash_hex == first.submission.block_hash_hex: + raise RuntimeError("share-writer adoption unavailable") + original_adopt(value) - def test_prism_payout_policy_allows_fixed_min_output_bits_override(self) -> None: - server = coordinator() + def unexpected_finish(_pending: PendingShare) -> None: + raise AssertionError("unadopted replay must not be finished") - with patch.dict(os.environ, {"PRISM_PAYOUT_MIN_OUTPUT_BITS": "10000"}, clear=True): - policy = server.prism_payout_policy() + service.adopt_replayed_candidate = fail_first # type: ignore[method-assign] + server._finish_pending_share_candidate = ( # type: ignore[method-assign] + unexpected_finish + ) + with patch("builtins.print"): + self.assertEqual(server.replay_pending_block_candidates(), 1) + replayed = server._block_replay_candidate_queue.get_nowait() self.assertEqual( - policy, - { - "p2mr_spend_input_bytes": 3_680, - "target_feerate_sats_per_byte": 1, - "safety_multiplier": 4, - "min_output_sats": 10_000, - }, + replayed.submission.block_hash_hex, + second.submission.block_hash_hex, + ) + # The unadopted row quarantines off the node-offer lane. Draining it + # terminalizes only that row, and its never-adopted floor holder is + # never finished. + with patch("builtins.print"): + self.assertTrue(server._run_one_invalid_block_candidate_quarantine()) + self.assertEqual( + [intent["block_hash_hex"] for intent in ledger.pending_block_candidates()], + [second.submission.block_hash_hex], ) - def test_prism_payout_policy_falls_back_to_legacy_min_output_sats_override(self) -> None: - server = coordinator() + def test_durable_block_candidates_replay_after_queue_drains(self) -> None: + server, state, _recording = submit_coordinator() + ledger = SingleWriterShareLedger() + server.ledger = ledger + server.block_candidate_queue = queue.Queue(maxsize=2) - with patch.dict(os.environ, {"PRISM_PAYOUT_MIN_OUTPUT_SATS": "10000"}, clear=True): - policy = server.prism_payout_policy() + for index, tag in enumerate(("aa", "bb", "cc"), start=1): + pending = PendingShare( + share_id=f"miner-a:{tag * 32}", + miner_id="miner-a", + order_key="miner-a", + p2mr_program_hex="11" * 32, + share_difficulty=1, + network_difficulty=1, + template_height=9, + job_id="job-1", + job_issued_at_ms=1, + accepted_at_ms=index, + ntime=1, + ) + submission = SimpleNamespace( + coinbase_tx_hex="00", + block_hash_hex=tag * 32, + block_hex="00", + share_pass=True, + block_pass=True, + ) + candidate = block_candidate( + server, state, submission, pending_share=pending + ) + ledger.append_batch([(pending, server.block_candidate_intent(candidate))]) - self.assertEqual(policy["min_output_sats"], 10_000) + self.assertEqual(server.replay_pending_block_candidates(), 3) + self.assertTrue(server.block_candidate_queue.empty()) + first = server._block_replay_candidate_queue.get_nowait() + second = server._block_replay_candidate_queue.get_nowait() + third = server._block_replay_candidate_queue.get_nowait() + self.assertEqual( + [ + first.submission.block_hash_hex, + second.submission.block_hash_hex, + third.submission.block_hash_hex, + ], + ["aa" * 32, "bb" * 32, "cc" * 32], + ) + ledger.mark_block_candidate_submitted(block_hash="aa" * 32) + ledger.mark_block_candidate_abandoned(block_hash="bb" * 32, error="stale") + self.assertEqual(third.pending_share.share_id, "miner-a:" + "cc" * 32) - def test_prism_payout_policy_min_output_bits_overrides_legacy_sats_override(self) -> None: - server = coordinator() + def test_candidate_intent_avoids_duplicate_template_transaction_bodies(self) -> None: + server, state, _ledger = submit_coordinator() + witness_tx = synthetic_witness_transaction("55") + server.jobs["job-1"].template["transactions"] = [{"data": witness_tx}] + server.jobs["job-1"].job.transaction_hexes = (witness_tx,) + pending = self._pending_append("ca").pending_share + candidate = block_candidate( + server, + state, + SimpleNamespace( + coinbase_tx_hex="00", + block_hash_hex="ca" * 32, + block_hex="00" + witness_tx, + share_pass=True, + block_pass=True, + ), + pending_share=pending, + ) - with patch.dict( - os.environ, - { - "PRISM_PAYOUT_MIN_OUTPUT_BITS": "11000", - "PRISM_PAYOUT_MIN_OUTPUT_SATS": "10000", - }, - clear=True, - ): - policy = server.prism_payout_policy() + intent = server.block_candidate_intent(candidate) - self.assertEqual(policy["min_output_sats"], 11_000) + self.assertEqual( + set(intent["template"]), + {"previousblockhash", "height", "coinbasevalue"}, + ) + self.assertNotIn("transaction_hexes", intent) + self.assertEqual( + intent["witness_merkle_leaves_hex"], + direct_stratum.witness_merkle_leaves_hex((witness_tx,)), + ) - def test_prism_coinbase_tag_defaults_to_prism(self) -> None: - with patch.dict(os.environ, {}, clear=True): - self.assertEqual(default_prism_coinbase_tag_hex(), "/PRISM/".encode("ascii").hex()) + def test_transient_candidate_failure_remains_pending_for_retry(self) -> None: + server, state, _recording = submit_coordinator() + configure_temporary_audit_root(self, server) + ledger = SingleWriterShareLedger() + server.ledger = ledger + pending = PendingShare( + share_id="miner-a:" + "aa" * 32, + miner_id="miner-a", + order_key="miner-a", + p2mr_program_hex="11" * 32, + share_difficulty=1, + network_difficulty=1, + template_height=9, + job_id="job-1", + job_issued_at_ms=1, + accepted_at_ms=2, + ntime=1, + ) + candidate = block_candidate( + server, + state, + SimpleNamespace( + coinbase_tx_hex="00", + block_hash_hex="aa" * 32, + block_hex="00", + share_pass=True, + block_pass=True, + ), + pending_share=pending, + ) + intent = server.block_candidate_intent(candidate) + ledger.append_batch([(pending, intent)]) + server.enqueue_block_candidate(candidate) + server.submit_block_candidate = ( # type: ignore[method-assign] + lambda _candidate: (_ for _ in ()).throw(RuntimeError("rpc unavailable")) + ) - def test_prism_coinbase_tag_is_configurable_and_can_be_disabled(self) -> None: - with patch.dict(os.environ, {"PRISM_COINBASE_TAG": "/CUSTOM/"}, clear=True): - self.assertEqual(default_prism_coinbase_tag_hex(), "/CUSTOM/".encode("ascii").hex()) - with patch.dict(os.environ, {"PRISM_COINBASE_TAG": ""}, clear=True): - self.assertEqual(default_prism_coinbase_tag_hex(), "") + self.assertTrue(server.submit_next_block_candidate()) - def test_prism_coinbase_tag_rejects_non_printable_or_long_values(self) -> None: - for tag, message in ( - ("PRISM\n", "printable ASCII"), - ("P" * 41, "at most 40 bytes"), - ("PRISM-π", "ASCII"), - ): - with self.subTest(tag=tag), patch.dict( - os.environ, {"PRISM_COINBASE_TAG": tag}, clear=True - ): - with self.assertRaisesRegex(SystemExit, message): - default_prism_coinbase_tag_hex() + self.assertEqual(ledger.pending_block_candidates(), [intent]) + self.assertEqual(server.block_candidate_abandoned_counts, {}) + self.assertIn( + "qbit_prism_block_candidate_retries_total 1", + server.metrics_payload(), + ) - def test_coinbase_script_sig_suffix_places_pool_tag_before_extranonce(self) -> None: - server = coordinator() - server.coinbase_tag_hex = "/PRISM/".encode("ascii").hex() + def test_retryable_parent_stays_ahead_of_queued_child(self) -> None: + server, state, _ledger = submit_coordinator() + server.block_candidate_retry_initial_seconds = 0 + server.block_candidate_retry_max_seconds = 0 + parent = block_candidate( + server, + state, + SimpleNamespace( + coinbase_tx_hex="00", + block_hash_hex="a1" * 32, + block_hex="00", + share_pass=True, + block_pass=True, + ), + ) + child = block_candidate( + server, + state, + SimpleNamespace( + coinbase_tx_hex="00", + block_hash_hex="b1" * 32, + block_hex="01", + share_pass=True, + block_pass=True, + ), + ) + attempts: list[str] = [] - suffix = server.coinbase_script_sig_suffix_hex("aabbccdd", "00" * 8) + def submit(candidate: PrismBlockCandidate) -> bool: + block_hash = str(candidate.submission.block_hash_hex) + attempts.append(block_hash) + if block_hash == "a1" * 32 and attempts.count(block_hash) == 1: + server._abandon_block_candidate( + PRISM_REJECTION_BACKEND_RPC_UNAVAILABLE, + "temporary parent finalization failure", + block_hash=block_hash, + worker="miner-a", + ) + return False + return True - self.assertEqual(suffix, "/PRISM/".encode("ascii").hex() + "aabbccdd" + "00" * 8) - self.assertTrue(suffix.endswith("aabbccdd" + "00" * 8)) + server.submit_block_candidate = submit # type: ignore[method-assign] + self.assertTrue(server.enqueue_block_candidate(parent)) + self.assertTrue(server.enqueue_block_candidate(child)) - def test_prism_payout_policy_allows_formula_overrides(self) -> None: - server = coordinator() + self.assertTrue(server.submit_next_block_candidate()) + self.assertIs(server._retry_block_candidate, parent) + self.assertEqual(server.block_candidate_queue.qsize(), 1) + self.assertTrue(server.submit_next_block_candidate()) + self.assertIsNone(getattr(server, "_retry_block_candidate", None)) + self.assertEqual(server.block_candidate_queue.qsize(), 1) + self.assertTrue(server.submit_next_block_candidate()) - with patch.dict( - os.environ, - { - "PRISM_PAYOUT_P2MR_SPEND_INPUT_BYTES": "2500", - "PRISM_PAYOUT_TARGET_FEERATE_BITS_PER_BYTE": "2", - "PRISM_PAYOUT_SAFETY_MULTIPLIER": "3", - }, - clear=True, - ): - policy = server.prism_payout_policy() + self.assertEqual(attempts, ["a1" * 32, "a1" * 32, "b1" * 32]) + self.assertTrue(server.block_candidate_queue.empty()) - self.assertEqual( - policy, - { - "p2mr_spend_input_bytes": 2_500, - "target_feerate_sats_per_byte": 2, - "safety_multiplier": 3, - }, + def test_candidate_retry_backoff_is_capped_and_cleared_on_success(self) -> None: + server, state, _recording = submit_coordinator() + ledger = SingleWriterShareLedger() + server.ledger = ledger + pending = self._pending_append("retry-success").pending_share + candidate = block_candidate( + server, + state, + SimpleNamespace( + coinbase_tx_hex="00", + block_hash_hex="a1" * 32, + block_hex="00", + share_pass=True, + block_pass=True, + ), + pending_share=pending, ) + ledger.append_batch([(pending, server.block_candidate_intent(candidate))]) + server.block_candidate_retry_initial_seconds = 0.1 + server.block_candidate_retry_max_seconds = 0.4 + attempts = 0 - def test_prism_payout_policy_formula_uses_legacy_feerate_alias(self) -> None: - server = coordinator() - - with patch.dict(os.environ, {"PRISM_PAYOUT_TARGET_FEERATE_SATS_PER_BYTE": "2"}, clear=True): - policy = server.prism_payout_policy() + def retry_then_succeed(_candidate: PrismBlockCandidate) -> bool: + nonlocal attempts + attempts += 1 + if attempts <= 4: + server._defer_block_candidate( + PRISM_REJECTION_BACKEND_RPC_UNAVAILABLE, + "temporary RPC outage", + worker="miner-a", + ) + return False + return True - self.assertEqual(policy["target_feerate_sats_per_byte"], 2) - - def test_prism_payout_policy_formula_bits_feerate_overrides_legacy_alias(self) -> None: - server = coordinator() - - with patch.dict( - os.environ, - { - "PRISM_PAYOUT_TARGET_FEERATE_BITS_PER_BYTE": "3", - "PRISM_PAYOUT_TARGET_FEERATE_SATS_PER_BYTE": "2", - }, - clear=True, + server.submit_block_candidate = retry_then_succeed # type: ignore[method-assign] + waits: list[float] = [] + with patch.object( + server.stop_event, + "wait", + side_effect=lambda delay: waits.append(delay) or False, ): - policy = server.prism_payout_policy() - - self.assertEqual(policy["target_feerate_sats_per_byte"], 3) - - def test_prism_payout_policy_rejects_invalid_floor_settings(self) -> None: - cases = [ - ({"PRISM_PAYOUT_MIN_OUTPUT_BITS": "0"}, "PRISM_PAYOUT_MIN_OUTPUT_BITS must be positive"), - ( - {"PRISM_PAYOUT_MIN_OUTPUT_BITS": "not-int"}, - "PRISM_PAYOUT_MIN_OUTPUT_BITS must be an integer", - ), - ({"PRISM_PAYOUT_MIN_OUTPUT_SATS": "0"}, "PRISM_PAYOUT_MIN_OUTPUT_SATS must be positive"), - ( - {"PRISM_PAYOUT_MIN_OUTPUT_SATS": "not-int"}, - "PRISM_PAYOUT_MIN_OUTPUT_SATS must be an integer", - ), - ( - {"PRISM_PAYOUT_SAFETY_MULTIPLIER": "0"}, - "PRISM_PAYOUT_SAFETY_MULTIPLIER must be positive", - ), - ( - {"PRISM_PAYOUT_TARGET_FEERATE_BITS_PER_BYTE": "0"}, - "PRISM_PAYOUT_TARGET_FEERATE_BITS_PER_BYTE must be positive", - ), - ] - for env_vars, expected in cases: - with self.subTest(env_vars=env_vars), patch.dict(os.environ, env_vars, clear=True): - server = coordinator() - with self.assertRaisesRegex(SystemExit, expected): - server.prism_payout_policy() - - def test_prism_pool_fee_address_config_validates_p2mr_address(self) -> None: - server = coordinator() - server.rpc = AddressRpc(valid_address="tq1fee", script_byte="88") + for _attempt in range(5): + server.enqueue_block_candidate(candidate) + self.assertTrue(server.submit_next_block_candidate()) - with patch.dict( - os.environ, - { - "PRISM_POOL_FEE_ENABLED": "1", - "PRISM_POOL_FEE_BPS": "125", - "PRISM_POOL_FEE_ADDRESS": "tq1fee", - }, - clear=True, + self.assertEqual(len(waits), 6) + for observed, expected in zip( + waits, + [0.1, 0.2, 0.25, 0.15, 0.25, 0.15], + strict=True, ): - policy = server.prism_payout_policy() + self.assertAlmostEqual(observed, expected) + self.assertNotIn(candidate.submission.block_hash_hex, server.block_candidate_retry_delays) + self.assertEqual(server.block_candidate_abandoned_counts, {}) + self.assertEqual(ledger.pending_block_candidates(), []) - self.assertEqual(server.rpc.validated, ["tq1fee"]) - self.assertEqual( - policy["pool_fee_policy"], - { - "fee_bps": 125, - "recipient_id": "tq1fee", - "order_key": "tq1fee", - "p2mr_program_hex": "88" * 32, - }, + def test_candidate_retry_state_is_cleared_on_terminal_abandonment(self) -> None: + server, state, _recording = submit_coordinator() + ledger = SingleWriterShareLedger() + server.ledger = ledger + pending = self._pending_append("retry-terminal").pending_share + candidate = block_candidate( + server, + state, + SimpleNamespace( + coinbase_tx_hex="00", + block_hash_hex="a2" * 32, + block_hex="00", + share_pass=True, + block_pass=True, + ), + pending_share=pending, ) + ledger.append_batch([(pending, server.block_candidate_intent(candidate))]) + server.block_candidate_retry_delays = {candidate.submission.block_hash_hex: 2.0} - def test_prism_pool_fee_enabled_allows_zero_bps_policy(self) -> None: - server = coordinator() - server.rpc = AddressRpc(valid_address="tq1fee", script_byte="66") + def terminal(_candidate: PrismBlockCandidate) -> bool: + block_hash = _candidate.submission.block_hash_hex + block_height = int(_candidate.context.template["height"]) + server._begin_accepted_block_payout_preview( + block_hash, + block_height=block_height, + ) + server._mark_accepted_block_payout_landed( + block_hash, + block_height=block_height, + ) + server._abandon_block_candidate( + PRISM_REJECTION_STALE_JOB, + "tip moved", + block_hash=block_hash, + worker="miner-a", + ) + return False - with patch.dict( - os.environ, - { - "PRISM_POOL_FEE_ENABLED": "1", - "PRISM_POOL_FEE_BPS": "0", - "PRISM_POOL_FEE_ADDRESS": "tq1fee", - }, - clear=True, - ): - policy = server.prism_payout_policy() + server.submit_block_candidate = terminal # type: ignore[method-assign] + server.enqueue_block_candidate(candidate) - self.assertEqual(policy["pool_fee_policy"]["fee_bps"], 0) - self.assertEqual(policy["pool_fee_policy"]["p2mr_program_hex"], "66" * 32) + self.assertTrue(server.submit_next_block_candidate()) - def test_prism_pool_fee_program_config_requires_recipient_identity(self) -> None: - server = coordinator() + self.assertNotIn(candidate.submission.block_hash_hex, server.block_candidate_retry_delays) + self.assertEqual(server.block_candidate_abandoned_counts[PRISM_REJECTION_STALE_JOB], 1) + self.assertEqual(ledger.pending_block_candidates(), []) + self.assertNotIn( + candidate.submission.block_hash_hex, + server._accepted_block_payout_previews, + ) + self.assertNotIn( + candidate.submission.block_hash_hex, + server._invalidated_accepted_block_payout_previews, + ) - with patch.dict( - os.environ, - { - "PRISM_POOL_FEE_ENABLED": "1", - "PRISM_POOL_FEE_BPS": "125", - "PRISM_POOL_FEE_P2MR_PROGRAM_HEX": "55" * 32, - }, - clear=True, - ): - with self.assertRaisesRegex(SystemExit, "PRISM_POOL_FEE_RECIPIENT_ID"): - server.prism_payout_policy() + def test_terminal_abandonment_keeps_tombstone_when_outbox_update_fails( + self, + ) -> None: + server, state, _recording = submit_coordinator() + ledger = SingleWriterShareLedger() + server.ledger = ledger + pending = self._pending_append("terminal-outbox-failure").pending_share + candidate = block_candidate( + server, + state, + SimpleNamespace( + coinbase_tx_hex="00", + block_hash_hex="a3" * 32, + block_hex="00", + share_pass=True, + block_pass=True, + ), + pending_share=pending, + ) + block_hash = candidate.submission.block_hash_hex + block_height = int(candidate.context.template["height"]) + ledger.append_batch([(pending, server.block_candidate_intent(candidate))]) - def test_prism_pool_fee_program_config_uses_explicit_order_key(self) -> None: - server = coordinator() + def terminal(_candidate: PrismBlockCandidate) -> bool: + server._begin_accepted_block_payout_preview( + block_hash, + block_height=block_height, + ) + server._mark_accepted_block_payout_landed( + block_hash, + block_height=block_height, + ) + server._abandon_block_candidate( + PRISM_REJECTION_STALE_JOB, + "tip moved", + block_hash=block_hash, + worker="miner-a", + ) + return False - with patch.dict( - os.environ, - { - "PRISM_POOL_FEE_ENABLED": "1", - "PRISM_POOL_FEE_BPS": "125", - "PRISM_POOL_FEE_P2MR_PROGRAM_HEX": "55" * 32, - "PRISM_POOL_FEE_RECIPIENT_ID": "pool-fee", - "PRISM_POOL_FEE_ORDER_KEY": "000-pool-fee", - }, - clear=True, + server.submit_block_candidate = terminal # type: ignore[method-assign] + server.enqueue_block_candidate(candidate) + with patch.object( + ledger, + "mark_block_candidate_abandoned", + side_effect=RuntimeError("postgres unavailable"), ): - policy = server.prism_payout_policy() + self.assertTrue(server.submit_next_block_candidate()) + self.assertNotIn(block_hash, server._accepted_block_payout_previews) + self.assertIn(block_hash, server._invalidated_accepted_block_payout_previews) self.assertEqual( - policy["pool_fee_policy"], - { - "fee_bps": 125, - "recipient_id": "pool-fee", - "order_key": "000-pool-fee", - "p2mr_program_hex": "55" * 32, - }, + [intent["block_hash_hex"] for intent in ledger.pending_block_candidates()], + [block_hash], ) - def test_prism_pool_fee_config_rejects_ambiguous_or_invalid_settings(self) -> None: - cases = [ - ( - {"PRISM_POOL_FEE_ENABLED": "1", "PRISM_POOL_FEE_ADDRESS": "tq1fee"}, - "PRISM_POOL_FEE_BPS", - ), - ( - { - "PRISM_POOL_FEE_ENABLED": "1", - "PRISM_POOL_FEE_BPS": "10001", - "PRISM_POOL_FEE_ADDRESS": "tq1fee", - }, - "between 0 and 10000", - ), - ( - { - "PRISM_POOL_FEE_ENABLED": "1", - "PRISM_POOL_FEE_BPS": "125", - "PRISM_POOL_FEE_ADDRESS": "tq1fee", - "PRISM_POOL_FEE_P2MR_PROGRAM_HEX": "55" * 32, - }, - "exactly one", + def test_finalize_failure_replays_with_candidate_backoff(self) -> None: + server, state, _recording = submit_coordinator() + configure_temporary_audit_root(self, server) + ledger = SingleWriterShareLedger() + server.ledger = ledger + pending = self._pending_append("f1").pending_share + candidate = block_candidate( + server, + state, + SimpleNamespace( + coinbase_tx_hex="00", + block_hash_hex="f1" * 32, + block_hex="00", + share_pass=True, + block_pass=True, ), - ] - for env_vars, expected in cases: - with self.subTest(env_vars=env_vars), patch.dict(os.environ, env_vars, clear=True): - server = coordinator() - server.rpc = AddressRpc(valid_address="tq1fee") - with self.assertRaisesRegex(SystemExit, expected): - server.prism_payout_policy() - - def test_prism_pool_fee_config_rejects_disabled_fee_settings(self) -> None: - cases = [ - {"PRISM_POOL_FEE_BPS": "125"}, - {"PRISM_POOL_FEE_ADDRESS": "tq1fee"}, - {"PRISM_POOL_FEE_P2MR_PROGRAM_HEX": "55" * 32}, - {"PRISM_POOL_FEE_RECIPIENT_ID": "pool-fee"}, - { - "PRISM_POOL_FEE_ENABLED": "0", - "PRISM_POOL_FEE_BPS": "125", - "PRISM_POOL_FEE_ADDRESS": "tq1fee", - }, - ] - for env_vars in cases: - with self.subTest(env_vars=env_vars), patch.dict(os.environ, env_vars, clear=True): - server = coordinator() - server.rpc = AddressRpc(valid_address="tq1fee") - with self.assertRaisesRegex(SystemExit, "PRISM_POOL_FEE_ENABLED=1"): - server.prism_payout_policy() + pending_share=pending, + ) + candidate_intent = server.block_candidate_intent(candidate) + ledger.append_batch([(pending, candidate_intent)]) + server.block_candidate_retry_initial_seconds = 0.1 + server.block_candidate_retry_max_seconds = 0.4 + # The block itself lands once; only the terminal outbox update fails. + # Replays must pace like any other candidate retry AND run finalize- + # only, never re-entering submission. + submit_calls = 0 - def test_prism_pool_fee_config_rejects_non_p2mr_fee_address(self) -> None: - server = coordinator() - server.rpc = AddressRpc(valid_address="tq1fee", p2mr=False) + def accepting_submit(_candidate: PrismBlockCandidate) -> bool: + nonlocal submit_calls + submit_calls += 1 + return True - with patch.dict( - os.environ, - { - "PRISM_POOL_FEE_ENABLED": "1", - "PRISM_POOL_FEE_BPS": "125", - "PRISM_POOL_FEE_ADDRESS": "tq1fee", - }, - clear=True, - ): - with self.assertRaisesRegex(SystemExit, "P2MR"): - server.prism_payout_policy() + server.submit_block_candidate = accepting_submit # type: ignore[method-assign] + original_finish = ledger.mark_block_candidate_submitted + original_mark_attempted = ledger.mark_block_candidate_attempted + finish_attempts = 0 + attempt_marks = 0 - def test_build_audit_bundle_passes_pool_fee_policy_to_cli_payload(self) -> None: - server = coordinator() - server.rpc = AddressRpc(valid_address="tq1fee", script_byte="99") - server.signing_seed_hex = "42" * 32 - server.ledger_attestation_signing_seed_hex = "43" * 32 - captured: dict[str, object] = {} + def mark_attempted(*, block_hash: str) -> bool: + nonlocal attempt_marks + attempt_marks += 1 + return original_mark_attempted(block_hash=block_hash) - with patch.dict( - os.environ, - { - "PRISM_POOL_FEE_ENABLED": "1", - "PRISM_POOL_FEE_BPS": "125", - "PRISM_POOL_FEE_ADDRESS": "tq1fee", - }, - clear=True, - ), patch( - "lab.prism.prism_coordinator.subprocess.Popen", - fake_audit_bundle_popen(captured), + def flaky_finish(*, block_hash: str) -> bool: + nonlocal finish_attempts + finish_attempts += 1 + if finish_attempts <= 4: + raise RuntimeError("ledger unavailable") + return original_finish(block_hash=block_hash) + + ledger.mark_block_candidate_attempted = mark_attempted # type: ignore[method-assign] + ledger.mark_block_candidate_submitted = flaky_finish # type: ignore[method-assign] + waits: list[float] = [] + with patch.object( + server.stop_event, + "wait", + side_effect=lambda delay: waits.append(delay) or False, ): - bundle = server.build_audit_bundle( - shares=[], - found_block={"block_height": 10, "coinbase_value_sats": 50_00000000}, - prior_balances=[], - coinbase_script_sig_suffix_hex="00", - ) + for _attempt in range(4): + server.enqueue_block_candidate(candidate) + self.assertTrue(server.submit_next_block_candidate()) + self.assertEqual(ledger.pending_block_candidates(), [candidate_intent]) + server.enqueue_block_candidate(candidate) + self.assertTrue(server.submit_next_block_candidate()) - self.assertEqual(bundle, {"ok": True}) - self.assertEqual(captured["payload"]["payout_policy"]["pool_fee_policy"]["fee_bps"], 125) - self.assertEqual( - captured["payload"]["payout_policy"]["pool_fee_policy"]["p2mr_program_hex"], - "99" * 32, + # The first accepted finalize failure returns unpaced so the caller + # can refresh the fleet immediately; the ladder starts at the first + # finalize-only replay. + self.assertEqual(len(waits), 4) + for observed, expected in zip( + waits, + [0.1, 0.2, 0.25, 0.15], + strict=True, + ): + self.assertAlmostEqual(observed, expected) + self.assertEqual(finish_attempts, 5) + self.assertEqual(attempt_marks, 1) + self.assertEqual(submit_calls, 1) + self.assertNotIn(candidate.submission.block_hash_hex, server.block_candidate_retry_delays) + self.assertEqual(server.block_candidate_abandoned_counts, {}) + self.assertEqual(ledger.pending_block_candidates(), []) + self.assertEqual(server._block_candidate_finalize_retries, {}) + self.assertIn( + "qbit_prism_block_candidate_retries_total 4", + server.metrics_payload(), ) - def test_prism_coinbase_output_policy_defaults_to_canonical_and_is_omitted(self) -> None: - server = coordinator() - server.rpc = AddressRpc(valid_address="tq1fee", script_byte="88") + def test_abandon_finalize_failure_counts_one_abandonment(self) -> None: + server, state, _recording = submit_coordinator() + ledger = SingleWriterShareLedger() + server.ledger = ledger + moved_tip = "11" * 32 + expected_error = f"tip moved before submit: {moved_tip}" + pending = self._pending_append("f2").pending_share + candidate = block_candidate( + server, + state, + SimpleNamespace( + coinbase_tx_hex="00", + block_hash_hex="f2" * 32, + block_hex="00", + share_pass=True, + block_pass=True, + ), + pending_share=pending, + ) + ledger.append_batch([(pending, server.block_candidate_intent(candidate))]) + server.block_candidate_retry_initial_seconds = 0.1 + server.block_candidate_retry_max_seconds = 0.4 + submit_calls = 0 - with patch.dict( - os.environ, - { - "PRISM_POOL_FEE_ENABLED": "1", - "PRISM_POOL_FEE_BPS": "125", - "PRISM_POOL_FEE_ADDRESS": "tq1fee", - }, - clear=True, - ): - policy = server.prism_payout_policy() + def terminal_submit(_candidate: PrismBlockCandidate) -> bool: + nonlocal submit_calls + submit_calls += 1 + server._abandon_block_candidate( + PRISM_REJECTION_STALE_JOB, + expected_error, + block_hash=_candidate.submission.block_hash_hex, + worker="miner-a", + stale_job_class="tip_moved", + ) + return False - self.assertNotIn("coinbase_output_policy", policy) + server.submit_block_candidate = terminal_submit # type: ignore[method-assign] + original_finish = ledger.mark_block_candidate_abandoned + finish_attempts = 0 - def test_prism_coinbase_output_policy_explicit_canonical_is_omitted(self) -> None: - server = coordinator() - server.rpc = AddressRpc(valid_address="tq1fee", script_byte="88") + def flaky_finish(*, block_hash: str, error: str) -> bool: + nonlocal finish_attempts + finish_attempts += 1 + if finish_attempts <= 2: + raise RuntimeError("ledger unavailable") + return original_finish(block_hash=block_hash, error=error) - with patch.dict( - os.environ, - { - "PRISM_COINBASE_OUTPUT_POLICY": "canonical", - "PRISM_POOL_FEE_ENABLED": "1", - "PRISM_POOL_FEE_BPS": "125", - "PRISM_POOL_FEE_ADDRESS": "tq1fee", - }, - clear=True, + ledger.mark_block_candidate_abandoned = flaky_finish # type: ignore[method-assign] + waits: list[float] = [] + with patch.object( + server.stop_event, + "wait", + side_effect=lambda delay: waits.append(delay) or False, ): - policy = server.prism_payout_policy() - - self.assertNotIn("coinbase_output_policy", policy) - - def test_prism_coinbase_output_policy_pool_fee_first_is_included(self) -> None: - server = coordinator() - server.rpc = AddressRpc(valid_address="tq1fee", script_byte="88") + for _attempt in range(3): + server.enqueue_block_candidate(candidate) + self.assertTrue(server.submit_next_block_candidate()) - with patch.dict( - os.environ, - { - "PRISM_COINBASE_OUTPUT_POLICY": "pool-fee-first", - "PRISM_POOL_FEE_ENABLED": "1", - "PRISM_POOL_FEE_BPS": "125", - "PRISM_POOL_FEE_ADDRESS": "tq1fee", - }, - clear=True, - ): - policy = server.prism_payout_policy() + self.assertEqual(waits, [0.1, 0.2]) + self.assertEqual(submit_calls, 1) + self.assertEqual(finish_attempts, 3) + # Finalize-only replays must not recount the terminal abandonment. + self.assertEqual( + server.block_candidate_abandoned_counts, + {PRISM_REJECTION_STALE_JOB: 1}, + ) + self.assertEqual( + server.stale_job_abandon_counts, + {"tip_moved": 1, "balance_stale": 0, "append_epoch_stale": 0}, + ) + self.assertEqual(ledger.pending_block_candidates(), []) + outbox_row = ledger._block_candidate_outbox[ + candidate.submission.block_hash_hex + ] + self.assertEqual(outbox_row["state"], "abandoned") + self.assertEqual(outbox_row["last_error"], expected_error) + self.assertEqual(server._block_candidate_finalize_retries, {}) - self.assertEqual(policy["coinbase_output_policy"], "pool-fee-first") - self.assertEqual(policy["pool_fee_policy"]["fee_bps"], 125) + def test_accepted_finalize_failure_still_triggers_post_accept_refresh(self) -> None: + server, state, _recording = submit_coordinator() + ledger = SingleWriterShareLedger() + server.ledger = ledger + pending = self._pending_append("f3").pending_share + candidate = block_candidate( + server, + state, + SimpleNamespace( + coinbase_tx_hex="00", + block_hash_hex="f3" * 32, + block_hex="00", + share_pass=True, + block_pass=True, + ), + pending_share=pending, + ) + ledger.append_batch([(pending, server.block_candidate_intent(candidate))]) + server.block_candidate_retry_initial_seconds = 0.1 + server.block_candidate_retry_max_seconds = 0.4 + server.submit_block_candidate = lambda _candidate: True # type: ignore[method-assign] - def test_prism_coinbase_output_policy_rejects_unknown_values(self) -> None: - for invalid in ("fee-first", "POOL-FEE-FIRST", "canonical "): - with self.subTest(invalid=invalid), patch.dict( - os.environ, - { - "PRISM_COINBASE_OUTPUT_POLICY": invalid, - "PRISM_POOL_FEE_ENABLED": "1", - "PRISM_POOL_FEE_BPS": "125", - "PRISM_POOL_FEE_ADDRESS": "tq1fee", - }, - clear=True, - ): - server = coordinator() - server.rpc = AddressRpc(valid_address="tq1fee") - with self.assertRaisesRegex( - SystemExit, - "PRISM_COINBASE_OUTPUT_POLICY must be one of: canonical, pool-fee-first", - ): - server.prism_payout_policy() + def failing_finish(*, block_hash: str) -> bool: + raise RuntimeError("ledger unavailable") - def test_prism_coinbase_output_policy_pool_fee_first_requires_pool_fee(self) -> None: - cases = [ - {"PRISM_COINBASE_OUTPUT_POLICY": "pool-fee-first"}, - { - "PRISM_COINBASE_OUTPUT_POLICY": "pool-fee-first", - "PRISM_POOL_FEE_ENABLED": "0", - }, - ] - for env_vars in cases: - with self.subTest(env_vars=env_vars), patch.dict(os.environ, env_vars, clear=True): - server = coordinator() - with self.assertRaisesRegex(SystemExit, "requires PRISM_POOL_FEE_ENABLED=1"): - server.prism_payout_policy() + ledger.mark_block_candidate_submitted = failing_finish # type: ignore[method-assign] + refreshed_clients: list[object] = [] + server.refresh_jobs_after_pending_accepted_block = ( # type: ignore[method-assign] + lambda client, **_kwargs: refreshed_clients.append(client) or 0 + ) + released_shares: list[object] = [] + original_release = server._finish_pending_share_commit - def test_build_audit_bundle_passes_coinbase_output_policy_to_cli_payload(self) -> None: - server = coordinator() - server.rpc = AddressRpc(valid_address="tq1fee", script_byte="99") - server.signing_seed_hex = "42" * 32 - server.ledger_attestation_signing_seed_hex = "43" * 32 - captured: dict[str, object] = {} + def recording_release(pending_share: object) -> None: + released_shares.append(pending_share) + original_release(pending_share) - with patch.dict( - os.environ, - { - "PRISM_COINBASE_OUTPUT_POLICY": "pool-fee-first", - "PRISM_POOL_FEE_ENABLED": "1", - "PRISM_POOL_FEE_BPS": "125", - "PRISM_POOL_FEE_ADDRESS": "tq1fee", - }, - clear=True, - ), patch( - "lab.prism.prism_coordinator.subprocess.Popen", - fake_audit_bundle_popen(captured), + server._finish_pending_share_commit = recording_release # type: ignore[method-assign] + waits: list[float] = [] + with patch.object( + server.stop_event, + "wait", + side_effect=lambda delay: waits.append(delay) or False, ): - server.build_audit_bundle( - shares=[], - found_block={"block_height": 10, "coinbase_value_sats": 50_00000000}, - prior_balances=[], - coinbase_script_sig_suffix_hex="00", - ) - - payout_policy = captured["payload"]["payout_policy"] - self.assertEqual(payout_policy["coinbase_output_policy"], "pool-fee-first") - self.assertEqual(payout_policy["pool_fee_policy"]["fee_bps"], 125) - - def test_build_audit_bundle_passes_ctv_settlement_config_to_cli_payload(self) -> None: - server = coordinator() - server.signing_seed_hex = "42" * 32 - server.ledger_attestation_signing_seed_hex = "43" * 32 - captured: dict[str, object] = {} + server.enqueue_block_candidate(candidate) + self.assertTrue(server.submit_next_block_candidate()) + # The block is active on-chain; the fleet refresh fires on the + # first finalize failure without waiting out a backoff, and the + # snapshot anchor floor is released despite the pending outbox + # mark. + self.assertEqual(refreshed_clients, [state]) + self.assertEqual(waits, []) + self.assertIn(pending, released_shares) + self.assertTrue(server.submit_next_block_candidate()) + self.assertEqual(refreshed_clients, [state]) + self.assertEqual(waits, [0.1]) - with patch.dict( - os.environ, - { - "PRISM_CTV_SETTLEMENT_ENABLED": "1", - "PRISM_DIRECT_COINBASE_PAYOUT_FLOOR_BITS": "10485760", - "PRISM_MAX_COINBASE_SETTLEMENT_OUTPUTS": "16", - "PRISM_MAX_DIRECT_COINBASE_OUTPUTS": "12", - "PRISM_MAX_CTV_FANOUT_RECIPIENTS_PER_TRANSACTION": "1000", - "PRISM_CTV_FANOUT_FEE_MARKET_RATE_BITS_PER_1000_WEIGHT": "25", - "PRISM_CTV_FANOUT_FEE_PREMIUM_BPS": "12000", - }, - clear=True, - ), patch( - "lab.prism.prism_coordinator.subprocess.Popen", - fake_audit_bundle_popen(captured), - ): - bundle = server.build_audit_bundle( - shares=[], - found_block={"block_height": 10, "coinbase_value_sats": 50_00000000}, - prior_balances=[], - coinbase_script_sig_suffix_hex="00", - ) + def test_invalid_durable_candidate_is_quarantined_by_outbox_row_key(self) -> None: + for payload_hash in (None, "ff" * 32): + with self.subTest(payload_hash=payload_hash): + server, _state, _recording = submit_coordinator() + configure_temporary_audit_root(self, server) + ledger = SingleWriterShareLedger() + server.ledger = ledger + durable_hash = "de" * 32 + invalid = { + "schema": "unsupported", + "block_hash_hex": durable_hash, + "block_hex": "00", + } + ledger.persist_block_candidate_intent(invalid) + stored = ledger._block_candidate_outbox[durable_hash]["candidate"] + if payload_hash is None: + stored.pop("block_hash_hex") + else: + stored["block_hash_hex"] = payload_hash + server.block_candidate_retry_delays = {durable_hash: 1.0} - self.assertEqual(bundle, {"ok": True}) - self.assertEqual( - captured["payload"]["ctv_settlement"], - { - "direct_floor_sats": 10_485_760, - "config": { - "max_coinbase_settlement_outputs": 16, - "max_direct_coinbase_outputs": 12, - "max_fanout_recipients_per_transaction": 1000, - "reserved_coinbase_outputs": 0, - }, - "fanout_fee_rate_policy": { - "market_fee_rate_sats_per_1000_weight": 25, - "premium_bps": 12_000, - }, - }, - ) + self.assertEqual(server.replay_pending_block_candidates(), 0) + # Malformed-row cleanup is lower-priority maintenance, so it + # cannot form an N x database-timeout convoy ahead of valid + # recovered blocks on the node-offer lane. + self.assertTrue( + server._run_one_invalid_block_candidate_quarantine() + ) - def test_build_audit_bundle_preserves_exact_canonical_output_file(self) -> None: - server = coordinator() - server.signing_seed_hex = "42" * 32 - server.ledger_attestation_signing_seed_hex = "43" * 32 - canonical_bytes = b'{"ok":true,"nested":[1,2]}' - captured: dict[str, object] = {} + self.assertEqual(ledger.pending_block_candidates(), []) + self.assertNotIn(durable_hash, server.block_candidate_retry_delays) + self.assertIn( + "qbit_prism_block_candidate_poisoned_total 1", + server.metrics_payload(), + ) - with tempfile.TemporaryDirectory() as tmp, patch( - "lab.prism.prism_coordinator.subprocess.Popen", - fake_audit_bundle_popen( - captured, - output_text=canonical_bytes.decode("utf-8"), - ), - ): - output_path = Path(tmp) / "candidate.audit.json" - bundle = server.build_audit_bundle( - shares=[], - found_block={"block_height": 10, "coinbase_value_sats": 50_00000000}, - prior_balances=[], - coinbase_script_sig_suffix_hex="00", - canonical_output_path=output_path, - ) + def test_block_submitter_drops_candidate_when_pool_closed(self) -> None: + server, state, ledger = submit_coordinator() + server.accepted_block_count = 1 + server.max_blocks = 1 + submission = SimpleNamespace( + coinbase_tx_hex="c0ffee", + block_hash_hex="dd" * 32, + block_hex="00", + ) - self.assertEqual(bundle, {"ok": True, "nested": [1, 2]}) - self.assertEqual(output_path.read_bytes(), canonical_bytes) - self.assertIn("--canonical-output", captured["cmd"]) - self.assertEqual(captured["payload"]["shares"], []) + accepted = server.submit_block_candidate(block_candidate(server, state, submission)) - def test_build_audit_bundle_summary_only_requests_and_parses_job_summary(self) -> None: - server = coordinator() - server.signing_seed_hex = "42" * 32 - server.ledger_attestation_signing_seed_hex = "43" * 32 - summary = { - "found_block": { - "block_height": 10, - "coinbase_value_sats": 50_00000000, - }, - "signed_coinbase_manifest": { - "manifest": {"coinbase_tx_hex": "c0ffee"}, - "signature": {"signature_hex": "11" * 64}, - }, - } - captured: dict[str, object] = {} + self.assertFalse(accepted) + self.assertEqual(server.block_candidate_abandoned_counts[PRISM_REJECTION_POOL_CLOSED], 1) + self.assertEqual(server.rejection_counts_by_reason[PRISM_REJECTION_POOL_CLOSED], 0) + self.assertEqual(ledger.persisted, []) + + def test_block_worthy_share_is_credited_and_enqueued_before_block_submission(self) -> None: + # The share ack must never wait on the block path: a block-worthy + # share that met its target is credited immediately and the candidate + # is queued for the submitter thread. Nothing submits synchronously + # (the fixture RPC would raise on an unexpected submitblock call). + server, state, ledger = submit_coordinator() + submission = SimpleNamespace( + header_hex="aa" * 80, + block_hash_hex="cc" * 32, + share_pass=True, + block_pass=True, + ) with patch( - "lab.prism.prism_coordinator.subprocess.Popen", - fake_audit_bundle_popen( - captured, - output_text=json.dumps(summary, separators=(",", ":")), - ), - ), patch.object( - Path, - "open", - side_effect=AssertionError("summary-only build must not open an output path"), + "lab.prism.prism_coordinator.direct_stratum.assemble_submission", + return_value=submission, ): - bundle = server.build_audit_bundle( - shares=[], - found_block={"block_height": 10, "coinbase_value_sats": 50_00000000}, - prior_balances=[], - coinbase_script_sig_suffix_hex="00", - summary_only=True, + should_close = server.handle_submit( + state, + ["miner-a", "job-1", "00" * 8, "00000001", "00000002"], ) - self.assertEqual(bundle, summary) - self.assertEqual(set(bundle), {"found_block", "signed_coinbase_manifest"}) - self.assertIn("--job-summary-output", captured["cmd"]) - self.assertNotIn("--canonical-output", captured["cmd"]) - - def test_build_audit_bundle_removes_partial_output_after_builder_failure(self) -> None: - server = coordinator() - server.signing_seed_hex = "42" * 32 - server.ledger_attestation_signing_seed_hex = "43" * 32 - - captured: dict[str, object] = {} - with tempfile.TemporaryDirectory() as tmp: - output_path = Path(tmp) / "candidate.audit.json" - with patch( - "lab.prism.prism_coordinator.subprocess.Popen", - fake_audit_bundle_popen( - captured, - output_text='{"partial":', - returncode=9, - stderr_text="synthetic builder failure", - ), - ): - with self.assertRaisesRegex(RuntimeError, "synthetic builder failure"): - server.build_audit_bundle( - shares=[], - found_block={"block_height": 10, "coinbase_value_sats": 50_00000000}, - prior_balances=[], - coinbase_script_sig_suffix_hex="00", - canonical_output_path=output_path, - ) - self.assertFalse(output_path.exists()) - self.assertEqual(server.job_build_worker_counts["crashes"], 1) + self.assertFalse(should_close) + self.assertEqual(len(ledger.pending), 1) + self.assertIsNone(ledger.pending[0].credit_policy) + self.assertEqual(server.block_candidate_queue.qsize(), 1) + queued = server.block_candidate_queue.get_nowait() + self.assertIs(queued.submission, submission) + self.assertFalse(queued.credit_share_on_accept) - with patch( - "lab.prism.prism_coordinator.subprocess.Popen", - fake_audit_bundle_popen(captured), - ): - recovered = server.build_audit_bundle( - shares=[], - found_block={"block_height": 10, "coinbase_value_sats": 50_00000000}, - prior_balances=[], - coinbase_script_sig_suffix_hex="00", - ) + def test_block_candidate_submits_before_full_audit_persistence(self) -> None: + server, state, ledger = submit_coordinator() + server._ensure_job_cache_state() + server._ensure_tip_refresh_state() + stale_bundle_key = ("stale-payout-bundle",) + server._job_bundle_cache[stale_bundle_key] = object() # type: ignore[assignment] + active_fanout = _FanoutCancellation() + server._active_tip_refresh = ( # type: ignore[assignment] + SimpleNamespace(payout_state_generation=0), + active_fanout, + ) + with tempfile.TemporaryDirectory() as tempdir: + server.audit_dir = Path(tempdir) + server.evidence_path = Path(tempdir) / "evidence.json" + server.ledger_writer_public_key_hex = "aa" * 32 + block_hash = "cc" * 32 + rpc = SubmitRpc(tip="00" * 32, block_hash=block_hash, ledger=ledger) + server.rpc = rpc + witness_tx = synthetic_witness_transaction("44") + server.jobs["job-1"].job.transaction_hexes = (witness_tx,) + build_kwargs: list[dict[str, object]] = [] + # Deliberately use insertion order that differs from this fixture's + # stand-in canonical serialization. The alternate builder creates + # the requested output but writes pretty, noncanonical JSON, so + # existence alone must not make the path eligible for persistence. + alternate_bundle = { + "signed_coinbase_manifest": { + "manifest": { + "coinbase_tx_hex": "c0ffee", + "payout_count": 1, + } + }, + "payout_policy_manifest": {"accounts": []}, + "ledger_window_attestation": {"signature": {"public_key_hex": "aa" * 32}}, + "found_block": {"coinbase_value_sats": 50_00000000}, + } + canonical_bytes = json.dumps( + alternate_bundle, + sort_keys=True, + separators=(",", ":"), + ).encode() + canonical_sha256 = hashlib.sha256(canonical_bytes).hexdigest() - self.assertEqual(recovered, {"ok": True}) - self.assertEqual(server.job_build_worker_counts["restarts"], 1) + def fake_build_audit_bundle(**kwargs: object) -> dict[str, object]: + build_kwargs.append(kwargs) + output_path = kwargs["canonical_output_path"] + assert isinstance(output_path, Path) + adopter = kwargs["canonical_output_adopter"] + assert callable(adopter) + with output_path.open("x+", encoding="utf-8") as output: + output.write(json.dumps(alternate_bundle, indent=2)) + output.flush() + os.fsync(output.fileno()) + adopter(output_path, os.fstat(output.fileno())) + return alternate_bundle - def test_build_audit_bundle_recovers_after_cancelled_worker_timeout(self) -> None: - server = coordinator() - server.signing_seed_hex = "42" * 32 - server.ledger_attestation_signing_seed_hex = "43" * 32 - cancellation = _JobBuildCancellation(timeout_seconds=60) - process_calls = 0 - - class FakeStdin: - def write(self, value: str) -> int: - return len(value) - - def close(self) -> None: - return None - - class HungThenHealthyPopen: - def __init__(self, _cmd: list[str], **kwargs: object) -> None: - nonlocal process_calls - process_calls += 1 - self.healthy = process_calls == 2 - self.stdin = FakeStdin() - self.stdout = kwargs["stdout"] - self.stderr = kwargs["stderr"] - self.returncode: int | None = None - self.output_written = False - - def poll(self) -> int | None: - if self.returncode is not None: - return self.returncode - if self.healthy: - if not self.output_written: - self.stdout.write('{"ok":true}') # type: ignore[union-attr] - self.output_written = True - self.returncode = 0 - return 0 - cancellation.cancel("timeout") - return None - - def terminate(self) -> None: - self.returncode = -15 - - def kill(self) -> None: - self.returncode = -9 - - def wait(self, timeout: float | None = None) -> int: - del timeout - assert self.returncode is not None - return self.returncode - - build_kwargs = { - "shares": [], - "found_block": { - "block_height": 10, - "coinbase_value_sats": 50_00000000, - }, - "prior_balances": [], - "coinbase_script_sig_suffix_hex": "00", - } - with patch( - "lab.prism.prism_coordinator.subprocess.Popen", - HungThenHealthyPopen, - ): - with self.assertRaisesRegex(JobBuildCancelled, "timeout"): - server.build_audit_bundle( - **build_kwargs, - cancellation=cancellation, + def fake_verify_bundle(bundle_path: Path, *_args: object, **_kwargs: object) -> dict[str, object]: + candidate_bytes = bundle_path.read_bytes() + self.assertEqual(json.loads(candidate_bytes), alternate_bundle) + self.assertNotEqual(candidate_bytes, canonical_bytes) + self.assertNotEqual( + hashlib.sha256(candidate_bytes).hexdigest(), + canonical_sha256, ) - recovered = server.build_audit_bundle( - **build_kwargs, - cancellation=_JobBuildCancellation(timeout_seconds=60), - ) - - self.assertEqual(recovered, {"ok": True}) - self.assertEqual(process_calls, 2) - self.assertEqual(server.job_build_worker_counts["terminations"], 1) - self.assertEqual(server.job_build_worker_counts["restarts"], 1) + return { + **verified_audit_report(), + "audit_bundle_sha256_hex": canonical_sha256, + } - def test_build_audit_bundle_does_not_unlink_preexisting_output_path(self) -> None: - server = coordinator() - server.signing_seed_hex = "42" * 32 - server.ledger_attestation_signing_seed_hex = "43" * 32 + persist_accepted_block = ledger.persist_accepted_block - with tempfile.TemporaryDirectory() as tmp, patch( - "lab.prism.prism_coordinator.subprocess.Popen" - ) as popen: - output_path = Path(tmp) / "preexisting-candidate.audit.json" - output_path.write_bytes(b"do-not-clobber") - with self.assertRaises(FileExistsError): - server.build_audit_bundle( - shares=[], - found_block={"block_height": 10, "coinbase_value_sats": 50_00000000}, - prior_balances=[], - coinbase_script_sig_suffix_hex="00", - canonical_output_path=output_path, - ) + def persist_with_canonicalization(**kwargs: object) -> dict[str, object]: + self.assertIsNone(kwargs["canonical_bundle_path"]) + self.assertEqual(kwargs["final_bundle"], alternate_bundle) + report = kwargs["audit_report"] + assert isinstance(report, dict) + self.assertEqual(report["audit_bundle_sha256_hex"], canonical_sha256) + return persist_accepted_block(**kwargs) - self.assertEqual(output_path.read_bytes(), b"do-not-clobber") - popen.assert_not_called() + server.build_audit_bundle = fake_build_audit_bundle # type: ignore[method-assign] + server.verify_bundle = fake_verify_bundle # type: ignore[method-assign] + ledger.persist_accepted_block = persist_with_canonicalization # type: ignore[method-assign] + submission = SimpleNamespace( + coinbase_tx_hex="c0ffee", + block_hash_hex=block_hash, + block_hex="00", + ) + pending = SimpleNamespace(share_id="miner-a:" + block_hash) - def test_prism_ctv_settlement_config_uses_legacy_unit_aliases(self) -> None: - server = coordinator() + accepted = server.submit_block_candidate( + block_candidate(server, state, submission, pending_share=pending) + ) + self.assertTrue(accepted) + self.assertTrue( + server._ensure_shutdown_controller().writer_admission_closed() + ) - with patch.dict( - os.environ, - { - "PRISM_CTV_SETTLEMENT_ENABLED": "1", - "PRISM_DIRECT_COINBASE_PAYOUT_FLOOR_SATS": "10485760", - "PRISM_CTV_FANOUT_FEE_MARKET_RATE_SATS_PER_1000_WEIGHT": "25", - }, - clear=True, - ): - config = server.prism_ctv_settlement_config() + live_files = sorted(Path(tempdir).glob("prism-live-audit-bundle-[0-9]*.json")) + self.assertEqual(len(live_files), 1) + self.assertEqual(list(Path(tempdir).glob("prism-live-audit-bundle-candidate-*.json")), []) + self.assertEqual(list(Path(tempdir).glob(".prism-live-audit-bundle-candidate-*.tmp")), []) + envelope = json.loads(live_files[0].read_text(encoding="utf-8")) + self.assertEqual(envelope["schema"], "qbit.prism.live-audit-bundle-envelope.v1") + self.assertEqual(envelope["block_hash"], block_hash) + self.assertEqual(envelope["block_height"], 10) + self.assertEqual(envelope["audit_bundle_sha256"], canonical_sha256) + self.assertNotIn("signed_coinbase_manifest", envelope) + self.assertEqual( + Path(server.latest_evidence["audit_bundle_path"]), + live_files[0].resolve(), + ) - self.assertIsNotNone(config) - assert config is not None - self.assertEqual(config["direct_floor_sats"], 10_485_760) + self.assertTrue(rpc.submitted) self.assertEqual( - config["fanout_fee_rate_policy"], - {"market_fee_rate_sats_per_1000_weight": 25, "premium_bps": 12_000}, + build_kwargs[0]["witness_merkle_leaves_hex"], + direct_stratum.witness_merkle_leaves_hex((witness_tx,)), ) - - def test_prism_ctv_settlement_config_uses_node_fee_estimate_by_default(self) -> None: - server = coordinator() - rpc = FeeEstimateRpc({"feerate": "0.00001001"}) - server.rpc = rpc - - with patch.dict(os.environ, {"PRISM_CTV_SETTLEMENT_ENABLED": "1"}, clear=True): - config = server.prism_ctv_settlement_config() - - self.assertIsNotNone(config) - assert config is not None self.assertEqual( - config["fanout_fee_rate_policy"], - {"market_fee_rate_sats_per_1000_weight": 1001, "premium_bps": 12_000}, + build_kwargs[0]["coinbase_script_sig_suffix_hex"], + server.coinbase_tag_hex + state.extranonce1_hex + "00" * 8, ) - self.assertIn(("estimatesmartfee", [2]), rpc.calls) - - def test_prism_ctv_settlement_config_caches_node_fee_estimate_per_block_height(self) -> None: - server = coordinator() - rpc = FeeEstimateRpc({"feerate": "0.00001001"}) - server.rpc = rpc - - with patch.dict(os.environ, {"PRISM_CTV_SETTLEMENT_ENABLED": "1"}, clear=True): - first = server.prism_ctv_settlement_config(block_height=10) - rpc.estimate = {"feerate": "0.00002000"} - second = server.prism_ctv_settlement_config(block_height=10) - next_height = server.prism_ctv_settlement_config(block_height=11) + self.assertEqual(ledger.persisted[0]["block_hash"], block_hash) + self.assertEqual(ledger.persisted[0]["block_height"], 10) + self.assertTrue(ledger.persisted[0]["submit_seen_at_persist"]) + # This alternate test builder wrote noncanonical bytes to the requested + # path. The verified content is valid, but the path is never claimed as + # a byte-canonical artifact for ledger persistence. + self.assertIsNone(ledger.persisted[0]["canonical_bundle_path"]) + self.assertEqual(server._payout_state_generation, 1) + self.assertEqual(server._job_bundle_cache, {}) + self.assertTrue(active_fanout.is_set()) + self.assertTrue(server._tip_refresh_retry.is_set()) - assert first is not None - assert second is not None - assert next_height is not None - self.assertEqual( - first["fanout_fee_rate_policy"], - {"market_fee_rate_sats_per_1000_weight": 1001, "premium_bps": 12_000}, - ) - self.assertEqual(second["fanout_fee_rate_policy"], first["fanout_fee_rate_policy"]) - self.assertEqual( - next_height["fanout_fee_rate_policy"], - {"market_fee_rate_sats_per_1000_weight": 2000, "premium_bps": 12_000}, + def test_issued_preview_is_invalidated_when_final_coinbase_mismatches(self) -> None: + server, state, ledger = submit_coordinator() + server._ensure_job_cache_state() + block_hash = "cf" * 32 + server.rpc = SubmitRpc(tip="00" * 32, block_hash=block_hash, ledger=ledger) + server.build_audit_bundle = ( # type: ignore[method-assign] + lambda **_kwargs: verified_block_bundle("deadbeef") ) - self.assertEqual( - [call for call in rpc.calls if call[0] == "estimatesmartfee"], - [("estimatesmartfee", [2]), ("estimatesmartfee", [2])], + submission = SimpleNamespace( + coinbase_tx_hex="c0ffee", + block_hash_hex=block_hash, + block_hex="00", ) + candidate = block_candidate(server, state, submission) + candidate.context.prospective_prior_balances = () - def test_prism_ctv_settlement_config_separates_fee_cache_by_parent_hash(self) -> None: - server = coordinator() - rpc = FeeEstimateRpc({"feerate": "0.00001001"}) - server.rpc = rpc - - with patch.dict(os.environ, {"PRISM_CTV_SETTLEMENT_ENABLED": "1"}, clear=True): - first = server.prism_ctv_settlement_config(block_height=10, parent_hash="aa" * 32) - rpc.estimate = {"feerate": "0.00002000"} - same_parent = server.prism_ctv_settlement_config(block_height=10, parent_hash="aa" * 32) - reorg_parent = server.prism_ctv_settlement_config(block_height=10, parent_hash="bb" * 32) + with tempfile.TemporaryDirectory() as tempdir: + server.audit_dir = Path(tempdir) + self.assertFalse(server.submit_block_candidate(candidate)) - assert first is not None - assert same_parent is not None - assert reorg_parent is not None - self.assertEqual(same_parent["fanout_fee_rate_policy"], first["fanout_fee_rate_policy"]) - self.assertEqual( - reorg_parent["fanout_fee_rate_policy"], - {"market_fee_rate_sats_per_1000_weight": 2000, "premium_bps": 12_000}, - ) + self.assertEqual(server._payout_state_generation, 2) + self.assertEqual(server._accepted_block_payout_previews, {}) + self.assertEqual(ledger.persisted, []) self.assertEqual( - [call for call in rpc.calls if call[0] == "estimatesmartfee"], - [("estimatesmartfee", [2]), ("estimatesmartfee", [2])], + server._block_candidate_outcome.reason, + PRISM_REJECTION_CANDIDATE_AUDIT_MISMATCH, ) - def test_prism_ctv_settlement_config_fails_closed_when_fee_estimate_unavailable(self) -> None: - server = coordinator() - server.rpc = FeeEstimateRpc({"errors": ["insufficient data"]}) - - with patch.dict(os.environ, {"PRISM_CTV_SETTLEMENT_ENABLED": "1"}, clear=True): - with self.assertRaisesRegex(RuntimeError, "unable to compute PRISM CTV fanout fee rate"): - server.prism_ctv_settlement_config() + def test_accepted_block_persistence_allows_delivery_but_serializes_reconciliation( + self, + ) -> None: + server, state, ledger = submit_coordinator() + server._ensure_job_cache_state() + persist_started = threading.Event() + persist_finished = threading.Event() + release_persist = threading.Event() + confirm_started = threading.Event() + release_delivery = threading.Event() + reconcile_lock_attempted = threading.Event() + reconcile_mutated = threading.Event() + event_lock = threading.Lock() + events: list[str] = [] + original_persist = ledger.persist_accepted_block + original_confirm = ledger.confirm_accepted_block - def test_prism_ctv_settlement_config_retries_after_fee_estimate_failure(self) -> None: - server = coordinator() - rpc = FeeEstimateRpc({"errors": ["insufficient data"]}) - server.rpc = rpc + def note(name: str) -> None: + with event_lock: + events.append(name) - with patch.dict(os.environ, {"PRISM_CTV_SETTLEMENT_ENABLED": "1"}, clear=True): - with self.assertRaises(RuntimeError): - server.prism_ctv_settlement_config(block_height=10) - rpc.estimate = {"feerate": "0.00002000"} - recovered = server.prism_ctv_settlement_config(block_height=10) + def blocking_persist(**kwargs: object) -> dict[str, object]: + note("persist-start") + persist_started.set() + if not release_persist.wait(15): + raise AssertionError("timed out waiting to release accepted-block persistence") + result = original_persist(**kwargs) + note("persist-end") + persist_finished.set() + return result - assert recovered is not None - self.assertEqual( - recovered["fanout_fee_rate_policy"], - {"market_fee_rate_sats_per_1000_weight": 2000, "premium_bps": 12_000}, - ) - self.assertEqual( - [call for call in rpc.calls if call[0] == "estimatesmartfee"], - [("estimatesmartfee", [2]), ("estimatesmartfee", [2])], - ) + def observed_confirm(**kwargs: object) -> dict[str, object]: + note("confirm") + confirm_started.set() + return original_confirm(**kwargs) - def test_ctv_broadcaster_daemon_uses_coordinator_ledger_and_config(self) -> None: - server = coordinator() - server.ctv_broadcaster_wallet = None - server.ctv_broadcaster_fee_sats = 0 - server.ctv_broadcaster_limit = 7 - captured: dict[str, object] = {} + ledger.reorg_watch_blocks = lambda *, active_tip_height: [ # type: ignore[method-assign] + { + "block_height": active_tip_height - 2, + "block_hash": "aa" * 32, + "chain_state": "confirmed", + } + ] - class FakeDaemon: - def __init__(self, ledger: object, broadcaster: object, *, fee_sats: int) -> None: - captured["ledger"] = ledger - captured["broadcaster"] = broadcaster - captured["fee_sats"] = fee_sats + def mark_pool_block_inactive(**_kwargs: object) -> dict[str, object]: + note("reconcile-mutation") + reconcile_mutated.set() + return {"backend": "fake", "inactive_count": 1} - def run_once( - self, - *, - limit: int, - progress_callback: object, - chunk_size: int, - tip_refresh_pending: object, - chunk_callback: object, - ) -> object: - captured["limit"] = limit - captured["chunk_size"] = chunk_size - captured["tip_refresh_pending"] = tip_refresh_pending - captured["chunk_callback"] = chunk_callback - return SimpleNamespace( - scanned_count=1, - submitted_count=0, - updated_count=1, - failed_count=0, - yielded_to_tip_refresh=False, - ) + ledger.persist_accepted_block = blocking_persist # type: ignore[method-assign] + ledger.confirm_accepted_block = observed_confirm # type: ignore[method-assign] + ledger.mark_pool_block_inactive = mark_pool_block_inactive # type: ignore[method-assign] + accepted: list[bool] = [] + reconcile_results: list[dict[str, object]] = [] + errors: list[BaseException] = [] + delivery_admitted = threading.Event() + delivery_thread: threading.Thread | None = None + reconcile_thread: threading.Thread | None = None + mutation_lock = server._payout_balance_mutation_lock - with patch("lab.prism.prism_coordinator.CtvFanoutBroadcastDaemon", FakeDaemon): - result = server.run_ctv_fanout_broadcaster_once() + class ObservedBalanceLock: + def __enter__(self) -> ObservedBalanceLock: + if threading.current_thread() is reconcile_thread: + reconcile_lock_attempted.set() + mutation_lock.acquire() + return self - self.assertIs(captured["ledger"], server.ledger) - self.assertEqual(captured["fee_sats"], 0) - self.assertEqual(captured["limit"], 7) - self.assertEqual(captured["chunk_size"], 5) - self.assertTrue(callable(captured["tip_refresh_pending"])) - self.assertTrue(callable(captured["chunk_callback"])) - self.assertEqual(result.updated_count, 1) - self.assertIsNotNone(captured["broadcaster"]) + def __exit__(self, *_args: object) -> None: + mutation_lock.release() - def test_ctv_broadcaster_daemon_requires_wallet_for_cpfp_fee(self) -> None: - server = coordinator() - server.ctv_broadcaster_wallet = None - server.ctv_broadcaster_fee_sats = 1 + # The landing's audit-build release window drives the serializer + # through the plain lock interface as well. + def acquire(self, *args: object, **kwargs: object) -> bool: + return mutation_lock.acquire(*args, **kwargs) # type: ignore[arg-type] - with self.assertRaisesRegex(ValueError, "ctv_broadcaster_wallet is required"): - server.make_ctv_fanout_broadcast_daemon() + def release(self) -> None: + mutation_lock.release() - def test_clean_job_prunes_previous_active_prism_job(self) -> None: - server = coordinator() - state = client() - state.worker = WorkerIdentity( - username="miner-a", - payout_address="miner-a", - worker_name=None, - script_pubkey_hex="5220" + "11" * 32, - p2mr_program_hex="11" * 32, - ) - server.jobs = {} - counter = {"value": 0} + server._payout_balance_mutation_lock = ObservedBalanceLock() # type: ignore[assignment] - def build_context(client: ClientState, *, clean_jobs: bool) -> object: - counter["value"] += 1 - return SimpleNamespace( - job=SimpleNamespace(job_id=f"job-{counter['value']}", share_difficulty=Decimal("1")), - template={"previousblockhash": "00" * 32}, - collection_only=False, + with tempfile.TemporaryDirectory() as tempdir: + server.audit_dir = Path(tempdir) + server.evidence_path = Path(tempdir) / "evidence.json" + server.ledger_writer_public_key_hex = "aa" * 32 + block_hash = "cd" * 32 + server.rpc = SubmitRpc( + tip="00" * 32, + block_hash=block_hash, + ledger=ledger, ) + server.build_audit_bundle = ( # type: ignore[method-assign] + lambda **_kwargs: verified_block_bundle() + ) + server.verify_bundle = ( # type: ignore[method-assign] + lambda *_args, **_kwargs: verified_audit_report() + ) + submission = SimpleNamespace( + coinbase_tx_hex="c0ffee", + block_hash_hex=block_hash, + block_hex="00", + ) + candidate = block_candidate(server, state, submission) - server.build_job_for_client = build_context # type: ignore[method-assign] - server.send_difficulty = lambda *args, **kwargs: None - server.send_job = lambda *args, **kwargs: None - server.apply_job_difficulty = lambda *args, **kwargs: None - - server.maybe_send_job(state, clean_jobs=True) - first_job_id = next(iter(state.active_job_ids)) - server.maybe_send_job(state, clean_jobs=True) - second_job_id = next(iter(state.active_job_ids)) - - self.assertNotEqual(first_job_id, second_job_id) - self.assertNotIn(first_job_id, server.jobs) - self.assertIn(second_job_id, server.jobs) - - state.sock = SimpleNamespace(shutdown=lambda *_args: None, close=lambda: None) - server.disconnect_client(state) - self.assertNotIn(second_job_id, server.jobs) - self.assertEqual(state.active_job_ids, set()) - - def test_non_clean_job_retention_caps_previous_active_prism_jobs(self) -> None: - server = coordinator() - state = client() - state.worker = WorkerIdentity( - username="miner-a", - payout_address="miner-a", - worker_name=None, - script_pubkey_hex="5220" + "11" * 32, - p2mr_program_hex="11" * 32, - ) - server.jobs = {} - counter = {"value": 0} + def submit() -> None: + try: + accepted.append(server.submit_block_candidate(candidate)) + except BaseException as exc: # noqa: BLE001 - asserted below + errors.append(exc) - def build_context(client: ClientState, *, clean_jobs: bool) -> object: - counter["value"] += 1 - return SimpleNamespace( - job=SimpleNamespace(job_id=f"job-{counter['value']}", share_difficulty=Decimal("1")), - template={"previousblockhash": "00" * 32}, - collection_only=False, - ) + def deliver() -> None: + try: + with server._payout_state_delivery_gate.delivery(): + note("replacement-delivery") + delivery_admitted.set() + if not release_delivery.wait(15): + raise AssertionError( + "timed out waiting to release payout delivery" + ) + except BaseException as exc: # noqa: BLE001 - asserted below + errors.append(exc) - server.build_job_for_client = build_context # type: ignore[method-assign] - server.send_difficulty = lambda *args, **kwargs: None - server.send_job = lambda *args, **kwargs: None - server.apply_job_difficulty = lambda *args, **kwargs: None + def reconcile() -> None: + try: + reconcile_results.append( + server.reconcile_prism_pool_blocks_once(tip_hash=block_hash) + ) + except BaseException as exc: # noqa: BLE001 - asserted below + errors.append(exc) - total_jobs = MAX_ACTIVE_PRISM_JOBS_PER_CLIENT + 3 - for _ in range(total_jobs): - server.maybe_send_job(state, clean_jobs=False) + submit_thread = threading.Thread(target=submit) + submit_thread.start() + confirmation_waited_for_delivery = False + try: + self.assertTrue(persist_started.wait(5)) + server.reorg_reconciler_enabled = True + reconcile_thread = threading.Thread(target=reconcile) + reconcile_thread.start() + self.assertTrue(reconcile_lock_attempted.wait(5)) + self.assertFalse(reconcile_mutated.is_set()) + delivery_thread = threading.Thread(target=deliver) + delivery_thread.start() + admitted_while_persisting = delivery_admitted.wait(5) + if admitted_while_persisting: + self.assertFalse(reconcile_mutated.is_set()) + release_persist.set() + self.assertTrue(persist_finished.wait(5)) + confirmation_waited_for_delivery = not confirm_started.wait(0.1) + finally: + release_persist.set() + release_delivery.set() + submit_thread.join(10) + if delivery_thread is not None: + delivery_thread.join(10) + if reconcile_thread is not None: + reconcile_thread.join(10) - retained_ids = { - f"job-{index}" - for index in range(4, total_jobs + 1) - } - self.assertEqual(state.active_job_ids, retained_ids) - self.assertEqual(set(server.jobs), retained_ids) - self.assertNotIn("job-1", server.jobs) - self.assertEqual(state.active_job.job.job_id, f"job-{total_jobs}") + self.assertFalse(submit_thread.is_alive()) + self.assertIsNotNone(delivery_thread) + self.assertFalse(delivery_thread.is_alive()) + self.assertIsNotNone(reconcile_thread) + self.assertFalse(reconcile_thread.is_alive()) + if errors: + raise errors[0] + self.assertTrue(admitted_while_persisting) + # Confirmation is durable catch-up to the already-published + # prospective state and therefore does not wait on delivery. + self.assertFalse(confirmation_waited_for_delivery) + self.assertTrue(confirm_started.is_set()) + self.assertTrue(reconcile_mutated.is_set()) + self.assertEqual(accepted, [True]) + self.assertEqual(len(reconcile_results), 1) + self.assertEqual(reconcile_results[0]["inactive_blocks"], 1) + self.assertEqual(server._payout_state_generation, 2) + self.assertLess(events.index("persist-start"), events.index("replacement-delivery")) + self.assertLess(events.index("replacement-delivery"), events.index("persist-end")) + self.assertLess(events.index("persist-end"), events.index("confirm")) + self.assertLess(events.index("confirm"), events.index("reconcile-mutation")) - def test_tip_change_refreshes_clean_job_and_old_job_becomes_stale_without_reconnect(self) -> None: + def test_next_tip_preview_job_lands_after_parent_persistence(self) -> None: old_tip = "00" * 32 - new_tip = "11" * 32 - server = coordinator() - server.accepted_block_count = 0 - server.max_blocks = 1 - server.stop_after_block = True - server.stale_grace_seconds = 0 - server.jobs = {} - server.recent_share_keys = set() - server.share_weights_by_username = {} - ledger = RecordingLedger() - server.ledger = ledger - worker = worker_identity() - state = client() - state.username = worker.username - state.worker = worker - state.share_difficulty = Decimal("1") - sent: list[dict[str, object]] = [] - state.send = lambda payload: sent.append(payload) # type: ignore[method-assign] + parent_hash = "cd" * 32 + child_hash = "ce" * 32 + server, state, ledger = submit_coordinator(tip=old_tip) + server._ensure_job_cache_state() + ledger.durable_payout_state = True + server.reorg_reconciler_enabled = False + server.stop_after_block = False + server.max_blocks = 10 server.clients = {state} - - old_context = prism_context("old-job", old_tip, worker=worker) - state.active_job = old_context - state.active_job_ids = {"old-job"} - server.jobs["old-job"] = old_context - server.tip_template_snapshot = QbitTipTemplateSnapshot( - bestblockhash=old_tip, - previousblockhash=old_tip, - template_fingerprint=qbit_template_fingerprint(old_context.template), + server.refresh_jobs_after_accepted_block = ( # type: ignore[method-assign] + lambda **_kwargs: None ) - server.rpc = TipTemplateRpc(tip=new_tip, template=gbt_template(new_tip, height=11)) - - def build_fresh_job(client: ClientState, *, clean_jobs: bool) -> object: - self.assertIs(client, state) - return prism_context( - "fresh-job", - new_tip, - worker=worker, - difficulty=client.pending_share_difficulty or client.share_difficulty, - clean_jobs=clean_jobs, - ) + build_started = threading.Event() + release_build = threading.Event() + persist_started = threading.Event() + release_persist = threading.Event() + original_persist = ledger.persist_accepted_block + original_confirm = ledger.confirm_accepted_block + preview = [ + { + "recipient_id": "miner-a", + "order_key": "miner-a", + "p2mr_program_hex": "11" * 32, + "balance_sats": 25, + } + ] - server.build_job_for_client = build_fresh_job # type: ignore[method-assign] + def payout_bundle_payload() -> dict[str, object]: + return { + "found_block": {"coinbase_value_sats": 50_00000000}, + "ledger_window_attestation": { + "signature": {"public_key_hex": "aa" * 32} + }, + "payout_policy_manifest": { + "accounts": [ + { + "recipient_id": "miner-a", + "order_key": "miner-a", + "p2mr_program_hex": "11" * 32, + "gross_amount_sats": 25, + "prior_balance_sats": 0, + "candidate_balance_sats": 25, + "onchain_amount_sats": 0, + "carry_forward_balance_sats": 25, + "action": "accrued", + } + ] + }, + "signed_coinbase_manifest": { + "manifest": {"coinbase_tx_hex": "c0ffee", "payout_count": 1} + }, + } - refreshed = server.poll_qbit_tip_template_once() + def blocking_payout_bundle(**_kwargs: object) -> dict[str, object]: + if not build_started.is_set(): + build_started.set() + if not release_build.wait(15): + raise AssertionError("timed out waiting to release parent bundle build") + return payout_bundle_payload() - self.assertEqual(refreshed, 1) - self.assertEqual(server.tip_refresh_job_count, 1) - self.assertIn(state, server.clients) - self.assertNotIn("old-job", server.jobs) - self.assertEqual(state.active_job_ids, {"fresh-job"}) - self.assertIn("fresh-job", server.jobs) - self.assertEqual([payload["method"] for payload in sent], ["mining.set_difficulty", "mining.notify"]) - self.assertEqual(sent[1]["params"][0], "fresh-job") - self.assertTrue(sent[1]["params"][8]) + def blocking_persist(**kwargs: object) -> dict[str, object]: + if str(kwargs["block_hash"]).lower() == parent_hash: + persist_started.set() + if not release_persist.wait(15): + raise AssertionError("timed out waiting to release parent persistence") + return original_persist(**kwargs) - with self.assertRaises(StratumError) as raised: - server.handle_submit( - state, - ["miner-a", "old-job", "00" * 8, "00000001", "00000002"], + def expose_confirmed_preview(**kwargs: object) -> dict[str, object]: + persisted = next( + row + for row in reversed(ledger.persisted) + if str(row["block_hash"]).lower() + == str(kwargs["block_hash"]).lower() ) - self.assertEqual(raised.exception.code, 21) - self.assertEqual(raised.exception.reason, PRISM_REJECTION_UNKNOWN_JOB) - self.assertEqual(server.stale_share_count, 1) - self.assertEqual(server.rejection_counts_by_reason[PRISM_REJECTION_UNKNOWN_JOB], 1) - self.assertEqual(len(ledger.pending), 0) - - submission = SimpleNamespace( - header_hex="aa" * 80, - block_hash_hex="bb" * 32, - share_pass=True, - block_pass=False, - ) - with patch( - "lab.prism.prism_coordinator.direct_stratum.assemble_submission", - return_value=submission, - ): - should_close = server.handle_submit( - state, - ["miner-a", "fresh-job", "00" * 8, "00000001", "00000002"], + ledger.prior_balances = server._accepted_block_payout_preview_from_bundle( + persisted["final_bundle"] # type: ignore[arg-type] ) + return original_confirm(**kwargs) - self.assertFalse(should_close) - self.assertEqual(len(ledger.pending), 1) - self.assertEqual(ledger.pending[0].job_id, "fresh-job") - self.assertIn(state, server.clients) - - def test_tip_refresh_rpc_race_blocks_mismatched_tip_template_snapshot(self) -> None: - old_tip = "00" * 32 - new_tip = "11" * 32 - server = coordinator() - server.jobs = {} - worker = worker_identity() - state = client() - state.username = worker.username - state.worker = worker - sent: list[dict[str, object]] = [] - state.send = lambda payload: sent.append(payload) # type: ignore[method-assign] - server.clients = {state} - - old_context = prism_context("old-job", old_tip, worker=worker) - state.active_job = old_context - state.active_job_ids = {"old-job"} - server.jobs["old-job"] = old_context - server.tip_template_snapshot = QbitTipTemplateSnapshot( - bestblockhash=old_tip, - previousblockhash=old_tip, - template_fingerprint=qbit_template_fingerprint(old_context.template), - ) - server.rpc = TipTemplateRpc(tip=old_tip, template=gbt_template(new_tip, height=11)) - - with self.assertRaisesRegex( - TemplateRefreshBlocked, - "tip changed while fetching block template", - ): - server.poll_qbit_tip_template_once() - - self.assertIs(state.active_job, old_context) - self.assertEqual(server.jobs, {"old-job": old_context}) - self.assertEqual(state.active_job_ids, {"old-job"}) - self.assertEqual(sent, []) - # Tip observation is recorded as a detection before expensive template - # construction so obsolete builders can be cancelled immediately, but - # submit authority is published only alongside a coherent snapshot. - # The incoherent template never enters the artifact cache, client job - # maps, or the published tip state. - self.assertEqual(server.latest_detected_tip[0], old_tip) - self.assertIsNone(server.current_tip_first_seen) - self.assertIsNone(server._template_artifacts) - - def test_slow_tip_poll_cannot_regress_newer_blockwait_observation(self) -> None: - old_tip = "00" * 32 - new_tip = "11" * 32 - server = coordinator() - old_snapshot = QbitTipTemplateSnapshot( - bestblockhash=old_tip, - previousblockhash=old_tip, - template_fingerprint="22" * 32, - ) - server.rpc = TipRpc(old_tip) - self.assertTrue(server.observe_tip_first_seen(old_tip)) - - def overtake_poll() -> QbitTipTemplateSnapshot: - self.assertTrue(server.observe_tip_for_refresh(new_tip)) - return old_snapshot - - server.fetch_qbit_tip_template_snapshot = overtake_poll # type: ignore[method-assign] - - with self.assertRaisesRegex( - TemplateRefreshSuperseded, - "tip/template poll was superseded during template fetch", - ): - server.poll_qbit_tip_template_once() - - self.assertEqual(server.latest_detected_tip[0], new_tip) - self.assertEqual(server.current_tip_first_seen[0], old_tip) - self.assertIsNone(server.tip_template_snapshot) - self.assertIsNone( - getattr(server, "template_refresh_failure_started_monotonic", None) - ) - self.assertTrue(server._consume_tip_refresh_retry()) - self.assertFalse(server._consume_tip_refresh_retry()) + ledger.persist_accepted_block = blocking_persist # type: ignore[method-assign] + ledger.confirm_accepted_block = expose_confirmed_preview # type: ignore[method-assign] - def test_same_tip_template_refresh_sends_non_clean_job_and_keeps_old_job_submittable(self) -> None: - tip = "00" * 32 - server = coordinator() - server.accepted_block_count = 0 - server.max_blocks = 1 - server.stop_after_block = True - server.jobs = {} - server.recent_share_keys = set() - server.share_weights_by_username = {} - ledger = RecordingLedger() - server.ledger = ledger - worker = worker_identity() - state = client() - state.username = worker.username - state.worker = worker - state.share_difficulty = Decimal("1") - sent: list[dict[str, object]] = [] - state.send = lambda payload: sent.append(payload) # type: ignore[method-assign] - server.clients = {state} + class TwoBlockRpc: + def __init__(self) -> None: + self.tip = old_tip + self.height = 9 + self.hashes = {9: old_tip} + self.submitted: list[str] = [] - old_context = prism_context("old-job", tip, worker=worker) - state.active_job = old_context - state.active_job_ids = {"old-job"} - server.jobs["old-job"] = old_context - server.tip_template_snapshot = QbitTipTemplateSnapshot( - bestblockhash=tip, - previousblockhash=tip, - template_fingerprint=qbit_template_fingerprint(old_context.template), - ) - refreshed_template = gbt_template(tip, height=10, coinbasevalue=50_00000001) - server.rpc = TipTemplateRpc(tip=tip, template=refreshed_template) + def call(self, method: str, params: object = None) -> object: + if method == "getbestblockhash": + return self.tip + if method == "getblockcount": + return self.height + if method == "submitblock": + block_hex = str((params or [""])[0]) # type: ignore[index] + block_hash = parent_hash if block_hex == "00" else child_hash + self.height += 1 + self.tip = block_hash + self.hashes[self.height] = block_hash + self.submitted.append(block_hash) + ledger.submit_seen = True + return None + if method == "getblockhash": + return self.hashes[int((params or [0])[0])] # type: ignore[index] + raise RuntimeError(method) - def build_fresh_job(client: ClientState, *, clean_jobs: bool) -> object: - self.assertIs(client, state) - self.assertFalse(clean_jobs) - fresh_context = prism_context( - "fresh-job", - tip, - worker=worker, - difficulty=client.pending_share_difficulty or client.share_difficulty, - clean_jobs=clean_jobs, + rpc = TwoBlockRpc() + server.rpc = rpc + server.build_audit_bundle = blocking_payout_bundle # type: ignore[method-assign] + server.verify_bundle = ( # type: ignore[method-assign] + lambda *_args, **kwargs: verified_audit_report( + block_height=int(kwargs["expected_block_height"]) ) - fresh_context.template["coinbasevalue"] = refreshed_template["coinbasevalue"] - fresh_context.template_fingerprint = qbit_template_fingerprint(fresh_context.template) - return fresh_context - - server.build_job_for_client = build_fresh_job # type: ignore[method-assign] - - refreshed = server.poll_qbit_tip_template_once() - - self.assertEqual(refreshed, 1) - self.assertEqual(server.tip_refresh_job_count, 1) - self.assertIn(state, server.clients) - self.assertIn("old-job", server.jobs) - self.assertIn("fresh-job", server.jobs) - self.assertEqual(state.active_job_ids, {"old-job", "fresh-job"}) - self.assertEqual([payload["method"] for payload in sent], ["mining.set_difficulty", "mining.notify"]) - self.assertEqual(sent[1]["params"][0], "fresh-job") - self.assertFalse(sent[1]["params"][8]) - self.assertIn("qbit_prism_active_job_contexts 2", server.metrics_payload()) - - submission = SimpleNamespace( - header_hex="aa" * 80, - block_hash_hex="bb" * 32, - share_pass=True, - block_pass=False, ) - with patch( - "lab.prism.prism_coordinator.direct_stratum.assemble_submission", - return_value=submission, - ): - should_close = server.handle_submit( - state, - ["miner-a", "old-job", "00" * 8, "00000001", "00000002"], - ) - - self.assertFalse(should_close) - self.assertEqual(len(ledger.pending), 1) - self.assertEqual(ledger.pending[0].job_id, "old-job") - self.assertEqual(server.stale_share_count, 0) - self.assertEqual(server.rejection_counts_by_reason[PRISM_REJECTION_UNKNOWN_JOB], 0) - self.assertEqual(server.rejection_counts_by_reason[PRISM_REJECTION_STALE_JOB], 0) - - def test_tip_refresh_uses_pending_vardiff_difficulty_for_consistent_pair(self) -> None: - old_tip = "00" * 32 - new_tip = "22" * 32 - server = coordinator() - server.jobs = {} - worker = worker_identity() - state = client() - state.username = worker.username - state.worker = worker - state.share_difficulty = Decimal("1") - state.pending_share_difficulty = Decimal("8") sent: list[dict[str, object]] = [] state.send = lambda payload: sent.append(payload) # type: ignore[method-assign] - server.clients = {state} - old_context = prism_context("old-job", old_tip, worker=worker, difficulty=Decimal("1")) - state.active_job = old_context - state.active_job_ids = {"old-job"} - server.jobs["old-job"] = old_context - server.tip_template_snapshot = QbitTipTemplateSnapshot( - bestblockhash=old_tip, - previousblockhash=old_tip, - template_fingerprint=qbit_template_fingerprint(old_context.template), - ) - server.rpc = TipTemplateRpc(tip=new_tip, template=gbt_template(new_tip, height=11)) - def build_fresh_job(client: ClientState, *, clean_jobs: bool) -> object: - return prism_context( - "fresh-vardiff-job", - new_tip, - worker=worker, - difficulty=server.desired_client_share_difficulty(client), + def build_child_job(client: ClientState, *, clean_jobs: bool) -> object: + context = prism_context( + "child-job", + parent_hash, + worker=client.worker, clean_jobs=clean_jobs, ) + context.template["height"] = 11 + context.prior_balances = server._prior_balances_for_job_parent(parent_hash) + context.payout_state_generation = server._payout_state_generation + return context - server.build_job_for_client = build_fresh_job # type: ignore[method-assign] - - refreshed = server.poll_qbit_tip_template_once() - - self.assertEqual(refreshed, 1) - self.assertEqual(sent[0]["method"], "mining.set_difficulty") - self.assertEqual(sent[0]["params"], [8.0]) - self.assertEqual(sent[1]["method"], "mining.notify") - self.assertEqual(sent[1]["params"][0], "fresh-vardiff-job") - self.assertTrue(sent[1]["params"][8]) - self.assertEqual(state.share_difficulty, Decimal("8")) - self.assertIsNone(state.pending_share_difficulty) - self.assertEqual(server.jobs["fresh-vardiff-job"].job.share_difficulty, Decimal("8")) - - def test_tip_refresh_build_failure_keeps_client_connected_and_old_job_registered(self) -> None: - old_tip = "00" * 32 - new_tip = "33" * 32 - server = coordinator() - server.jobs = {} - worker = worker_identity() - state = client() - state.username = worker.username - state.worker = worker - sent: list[dict[str, object]] = [] - state.send = lambda payload: sent.append(payload) # type: ignore[method-assign] - server.clients = {state} - old_context = prism_context("old-job", old_tip, worker=worker) - state.active_job = old_context - state.active_job_ids = {"old-job"} - server.jobs["old-job"] = old_context - server.tip_template_snapshot = QbitTipTemplateSnapshot( - bestblockhash=old_tip, - previousblockhash=old_tip, - template_fingerprint=qbit_template_fingerprint(old_context.template), + server.build_job_for_client = build_child_job # type: ignore[method-assign] + parent_submission = SimpleNamespace( + coinbase_tx_hex="c0ffee", + block_hash_hex=parent_hash, + block_hex="00", ) - server.rpc = TipTemplateRpc(tip=new_tip, template=gbt_template(new_tip, height=11)) - - def failing_build(client: ClientState, *, clean_jobs: bool) -> object: - raise RuntimeError("transient getblocktemplate failure") - - server.build_job_for_client = failing_build # type: ignore[method-assign] + parent_candidate = block_candidate(server, state, parent_submission) + parent_candidate.context.prospective_prior_balances = ( + server._serialize_prior_balance_preview(preview) + ) + parent_results: list[bool] = [] + parent_errors: list[BaseException] = [] - with self.assertRaisesRegex(TemplateRefreshBlocked, "no refreshed work was issued"): - server.poll_qbit_tip_template_once() + def submit_parent() -> None: + try: + parent_results.append(server.submit_block_candidate(parent_candidate)) + except BaseException as exc: # noqa: BLE001 - asserted below + parent_errors.append(exc) - self.assertEqual(server.job_build_failure_count, 1) - self.assertEqual(server.tip_refresh_job_count, 0) - self.assertIn(state, server.clients) - self.assertEqual(state.active_job_ids, {"old-job"}) - self.assertIn("old-job", server.jobs) - self.assertEqual(sent, []) + with tempfile.TemporaryDirectory() as tempdir: + server.audit_dir = Path(tempdir) + server.evidence_path = Path(tempdir) / "evidence.json" + server.ledger_writer_public_key_hex = "aa" * 32 + parent_thread = threading.Thread(target=submit_parent) + parent_thread.start() + try: + self.assertTrue(build_started.wait(5)) + self.assertEqual(ledger.current_prior_balances(), []) + self.assertTrue(server.maybe_send_job(state, clean_jobs=True)) + child_context = state.active_job + self.assertIsNotNone(child_context) + assert child_context is not None + self.assertTrue(parent_thread.is_alive()) + self.assertEqual(child_context.prior_balances, preview) + self.assertEqual( + child_context.payout_state_generation, + server._payout_state_generation, + ) + preview_generation = server._payout_state_generation + self.assertEqual(sent[-1]["method"], "mining.notify") + self.assertTrue(sent[-1]["params"][8]) # type: ignore[index] - def test_tip_refresh_build_failure_is_not_masked_by_disconnected_client(self) -> None: - old_tip = "00" * 32 - new_tip = "33" * 32 - server = coordinator() - server.jobs = {} + child_submission = SimpleNamespace( + header_hex="bb" * 80, + coinbase_tx_hex="c0ffee", + block_hash_hex=child_hash, + block_hex="01", + share_pass=True, + block_pass=True, + ) + with patch( + "lab.prism.prism_coordinator.direct_stratum.assemble_submission", + return_value=child_submission, + ): + self.assertFalse( + server.handle_submit( + state, + ["miner-a", "child-job", "00" * 8, "00000001", "00000002"], + ) + ) + self.assertEqual(server.block_candidate_queue.qsize(), 1) + release_build.set() + self.assertTrue(persist_started.wait(5)) + self.assertTrue(parent_thread.is_alive()) + finally: + release_build.set() + release_persist.set() + parent_thread.join(10) - build_failed = client() - build_failed.worker = worker_identity("miner-build-failed") - build_failed.username = build_failed.worker.username - build_failed.active_job = prism_context( - "old-build-failed-job", old_tip, worker=build_failed.worker - ) - build_failed.active_job_ids = {"old-build-failed-job"} + self.assertFalse(parent_thread.is_alive()) + if parent_errors: + raise parent_errors[0] + self.assertEqual(parent_results, [True]) + self.assertEqual(ledger.current_prior_balances(), preview) + self.assertEqual(server._payout_state_generation, preview_generation) + self.assertTrue(server.submit_next_block_candidate()) - disconnected = client() - disconnected.connection_id = 2 - disconnected.worker = worker_identity("miner-disconnected") - disconnected.username = disconnected.worker.username - disconnected.active_job = prism_context( - "old-disconnected-job", old_tip, worker=disconnected.worker + self.assertEqual(rpc.submitted, [parent_hash, child_hash]) + self.assertEqual(server.accepted_block_count, 2) + self.assertEqual(len(ledger.persisted), 2) + self.assertEqual(len(ledger.confirmed), 2) + self.assertEqual( + server.block_candidate_abandoned_counts.get(PRISM_REJECTION_STALE_JOB, 0), + 0, ) - disconnected.active_job_ids = {"old-disconnected-job"} - - def disconnect_on_send(_payload: object) -> None: - raise OSError("socket closed") + self.assertEqual(server._accepted_block_payout_previews, {}) - disconnected.send = disconnect_on_send # type: ignore[method-assign] - server.clients = {build_failed, disconnected} - server.jobs = { - "old-build-failed-job": build_failed.active_job, - "old-disconnected-job": disconnected.active_job, - } - server.tip_template_snapshot = QbitTipTemplateSnapshot( - bestblockhash=old_tip, - previousblockhash=old_tip, - template_fingerprint=qbit_template_fingerprint(build_failed.active_job.template), - ) - server.rpc = TipTemplateRpc(tip=new_tip, template=gbt_template(new_tip, height=11)) + def test_direct_block_preparation_does_not_hold_delivery_gate(self) -> None: + server, state, ledger = submit_coordinator() + server._ensure_job_cache_state() + entered = threading.Event() + release = threading.Event() + accepted: list[bool] = [] + errors: list[BaseException] = [] + block_hash = "cf" * 32 + original_persist = ledger.persist_accepted_block - def mixed_build(state: ClientState, *, clean_jobs: bool) -> object: - if state is build_failed: - raise RuntimeError("template build unavailable") - return prism_context( - "disconnected-fresh-job", - new_tip, - worker=state.worker, - clean_jobs=clean_jobs, + def blocking_persist(**kwargs: object) -> dict[str, object]: + self.assertIsNone( + server._payout_state_delivery_gate._mutation_owner ) + entered.set() + if not release.wait(5): + raise AssertionError("test did not release direct-block preparation") + return original_persist(**kwargs) - disconnected_clients: list[ClientState] = [] - server.build_job_for_client = mixed_build # type: ignore[method-assign] - server.disconnect_client = disconnected_clients.append # type: ignore[method-assign] + ledger.persist_accepted_block = blocking_persist # type: ignore[method-assign] + with tempfile.TemporaryDirectory() as tempdir: + server.audit_dir = Path(tempdir) + server.evidence_path = Path(tempdir) / "evidence.json" + server.ledger_writer_public_key_hex = "aa" * 32 + server.rpc = SubmitRpc( + tip="00" * 32, + block_hash=block_hash, + ledger=ledger, + ) + server.build_audit_bundle = ( # type: ignore[method-assign] + lambda **_kwargs: verified_block_bundle() + ) + server.verify_bundle = ( # type: ignore[method-assign] + lambda *_args, **_kwargs: verified_audit_report() + ) + submission = SimpleNamespace( + coinbase_tx_hex="c0ffee", + block_hash_hex=block_hash, + block_hex="00", + ) - with self.assertRaisesRegex(TemplateRefreshBlocked, "no refreshed work was issued"): - server.poll_qbit_tip_template_once() + def submit() -> None: + try: + accepted.append( + server.submit_block_candidate( + block_candidate(server, state, submission) + ) + ) + except BaseException as exc: # pragma: no cover - asserted below + errors.append(exc) - self.assertEqual(server.job_build_failure_count, 1) - self.assertEqual(disconnected_clients, [disconnected]) - self.assertIn(build_failed, server.clients) + thread = threading.Thread(target=submit) + thread.start() + try: + self.assertTrue(entered.wait(5)) + with server._payout_state_delivery_gate.delivery_cancelable( + lambda: False, + generation=server._payout_state_generation, + priority=True, + ) as admission: + self.assertTrue(admission) + release.set() + thread.join(5) + self.assertFalse(thread.is_alive()) + self.assertTrue( + server._payout_state_prepare_lock.acquire(timeout=1) + ) + server._payout_state_prepare_lock.release() + admission.mark_delivered() + finally: + release.set() + thread.join(5) - def test_tip_reconciliation_quarantines_disconnected_block_before_refresh_job(self) -> None: - old_tip = "00" * 32 - new_tip = "44" * 32 - pool_block_hash = "aa" * 32 - server = coordinator() - server.reorg_reconciler_enabled = True - server.jobs = {} - worker = worker_identity() - state = client() - state.username = worker.username - state.worker = worker - state.share_difficulty = Decimal("1") - sent: list[dict[str, object]] = [] - state.send = lambda payload: sent.append(payload) # type: ignore[method-assign] - server.clients = {state} - old_context = prism_context("old-job", old_tip, worker=worker) - state.active_job = old_context - state.active_job_ids = {"old-job"} - server.jobs["old-job"] = old_context - server.tip_template_snapshot = QbitTipTemplateSnapshot( - bestblockhash=old_tip, - previousblockhash=old_tip, - template_fingerprint=qbit_template_fingerprint(old_context.template), - ) - ledger = ReorgLedger( - [ - { - "block_hash": pool_block_hash, - "block_height": 10, - "chain_state": "confirmed", - "maturity_state": "immature", - } - ] - ) - server.ledger = ledger - server.rpc = ReorgRpc( - tip=new_tip, - template=gbt_template(new_tip, height=11), - height=10, - block_hashes={10: "bb" * 32}, - ) + self.assertFalse(thread.is_alive()) + self.assertEqual(errors, []) + self.assertEqual(accepted, [True]) + self.assertEqual(server._payout_state_generation, 1) - def build_fresh_job(client: ClientState, *, clean_jobs: bool) -> object: - self.assertIn(("inactive", pool_block_hash, 10), ledger.events) - ledger.events.append(("build", client.connection_id)) - return prism_context("fresh-job", new_tip, worker=worker, clean_jobs=clean_jobs) + def test_failed_direct_block_audit_does_not_reserve_payout_source(self) -> None: + for failure_phase in ("build", "verify"): + with self.subTest(failure_phase=failure_phase): + server, state, ledger = submit_coordinator() + server._ensure_job_cache_state() + initial_source = server._payout_state_source + initial_published = server._published_payout_state + block_hash = "ce" * 32 + server.rpc = SubmitRpc( + tip="00" * 32, + block_hash=block_hash, + ledger=ledger, + ) + submission = SimpleNamespace( + coinbase_tx_hex="c0ffee", + block_hash_hex=block_hash, + block_hex="00", + ) - server.build_job_for_client = build_fresh_job # type: ignore[method-assign] + with tempfile.TemporaryDirectory() as tempdir: + server.audit_dir = Path(tempdir) + server.ledger_writer_public_key_hex = "aa" * 32 + if failure_phase == "build": + server.build_audit_bundle = ( # type: ignore[method-assign] + lambda **_kwargs: (_ for _ in ()).throw( + RuntimeError("audit reconstruction failed") + ) + ) + else: + server.build_audit_bundle = ( # type: ignore[method-assign] + lambda **_kwargs: verified_block_bundle() + ) + server.verify_bundle = ( # type: ignore[method-assign] + lambda *_args, **_kwargs: (_ for _ in ()).throw( + RuntimeError("audit verification failed") + ) + ) - refreshed = server.poll_qbit_tip_template_once() + with self.assertRaisesRegex( + RuntimeError, + "audit (reconstruction|verification) failed", + ): + server.submit_block_candidate( + block_candidate(server, state, submission) + ) - self.assertEqual(refreshed, 1) - self.assertLess( - ledger.events.index(("inactive", pool_block_hash, 10)), - ledger.events.index(("build", state.connection_id)), - ) - self.assertEqual(server.reorg_inactive_block_count, 1) - self.assertEqual(ledger.rows[0]["chain_state"], "inactive") - self.assertEqual(sent[1]["params"][0], "fresh-job") + self.assertEqual(server._payout_state_source, initial_source) + self.assertEqual(server._published_payout_state, initial_published) + self.assertEqual(server._payout_state_generation, 0) + self.assertEqual(ledger.persisted, []) - def test_reconciliation_quarantines_confirmed_block_above_shortened_tip(self) -> None: - pool_block_hash = "af" * 32 - server = coordinator() - server.reorg_reconciler_enabled = True - ledger = ReorgLedger( - [ - { - "block_hash": pool_block_hash, - "block_height": 12, - "chain_state": "confirmed", - "maturity_state": "immature", - } - ] + def test_uncertain_direct_block_ledger_commit_fences_delivery(self) -> None: + server, state, ledger = submit_coordinator() + server._ensure_job_cache_state() + block_hash = "cd" * 32 + server.rpc = SubmitRpc( + tip="00" * 32, + block_hash=block_hash, + ledger=ledger, ) - server.ledger = ledger - server.rpc = ReorgRpc( - tip="77" * 32, - template=gbt_template("77" * 32, height=11), - height=10, - block_hashes={}, + server.build_audit_bundle = ( # type: ignore[method-assign] + lambda **_kwargs: verified_block_bundle() ) - - summary = server.reconcile_prism_pool_blocks_once(tip_hash="77" * 32) - - self.assertEqual(summary["inactive_blocks"], 1) - self.assertEqual(server.reorg_inactive_block_count, 1) - self.assertEqual(ledger.rows[0]["chain_state"], "inactive") - - def test_tip_reconciliation_skips_jobs_when_qbit_chain_view_is_untrusted(self) -> None: - old_tip = "00" * 32 - new_tip = "55" * 32 - server = coordinator() - server.reorg_reconciler_enabled = True - server.jobs = {} - worker = worker_identity() - state = client() - state.username = worker.username - state.worker = worker - state.active_job = prism_context("old-job", old_tip, worker=worker) - state.active_job_ids = {"old-job"} - server.jobs["old-job"] = state.active_job - server.clients = {state} - server.tip_template_snapshot = QbitTipTemplateSnapshot( - bestblockhash=old_tip, - previousblockhash=old_tip, - template_fingerprint=qbit_template_fingerprint(state.active_job.template), + server.verify_bundle = ( # type: ignore[method-assign] + lambda *_args, **_kwargs: verified_audit_report() ) - ledger = ReorgLedger([]) - server.ledger = ledger - server.rpc = ReorgRpc( - tip=new_tip, - template=gbt_template(new_tip, height=11), - height=10, - block_hashes={10: new_tip}, - initialblockdownload=True, + ledger.confirm_accepted_block = ( # type: ignore[method-assign] + lambda **_kwargs: (_ for _ in ()).throw( + RuntimeError("ledger confirmation unavailable") + ) + ) + submission = SimpleNamespace( + coinbase_tx_hex="c0ffee", + block_hash_hex=block_hash, + block_hex="00", ) - def unexpected_build(client: ClientState, *, clean_jobs: bool) -> object: - raise AssertionError("job build should be skipped while qbitd is in IBD") - - server.build_job_for_client = unexpected_build # type: ignore[method-assign] - - with self.assertRaisesRegex(TemplateRefreshBlocked, "chain view remained untrusted"): - server.poll_qbit_tip_template_once() + with tempfile.TemporaryDirectory() as tempdir: + server.audit_dir = Path(tempdir) + server.ledger_writer_public_key_hex = "aa" * 32 + with self.assertRaisesRegex( + RuntimeError, + "ledger confirmation unavailable", + ): + server.submit_block_candidate( + block_candidate(server, state, submission) + ) - self.assertEqual(server.reorg_reconcile_skip_count, 1) - self.assertEqual(ledger.events, []) - self.assertEqual(state.active_job_ids, {"old-job"}) - self.assertEqual(server.tip_template_snapshot.bestblockhash, old_tip) + self.assertEqual(len(ledger.persisted), 1) + # The prospective preview was source/generation 1; the uncertain + # durable commit supersedes it with fenced source 2. + self.assertEqual(server._payout_state_generation, 1) + self.assertEqual(server._payout_state_source[0], 2) + self.assertEqual(server._published_payout_state.source_generation, 1) + self.assertTrue(server._payout_state_publication_blocked) + self.assertTrue(server._payout_state_delivery_gate._delivery_blocked) - def test_reconciliation_error_before_job_build_is_not_counted_as_build_failure(self) -> None: - tip = "59" * 32 - server = coordinator() - server.reorg_reconciler_enabled = True - state = client() - state.username = "miner-a" - state.worker = worker_identity() - state.share_difficulty = Decimal("1") - server.rpc = ReorgRpc( - tip=tip, - template=gbt_template(tip, height=11), - height=10, - block_hashes={10: tip}, + def test_uncertain_commit_supersedes_concurrently_published_source(self) -> None: + server, state, ledger = submit_coordinator() + server._ensure_job_cache_state() + block_hash = "cb" * 32 + newer_tip = "dc" * 32 + server.rpc = SubmitRpc( + tip="00" * 32, + block_hash=block_hash, + ledger=ledger, ) - - class FailingReorgLedger(FakeLedger): - def reorg_watch_blocks(self, *, active_tip_height: int) -> list[dict[str, object]]: - raise RuntimeError("ledger unavailable") - - server.ledger = FailingReorgLedger() - server.build_job_for_client = lambda _client, *, clean_jobs: (_ for _ in ()).throw( # type: ignore[method-assign] - AssertionError("job build should not run after reconcile failure") + server.build_audit_bundle = ( # type: ignore[method-assign] + lambda **_kwargs: verified_block_bundle() ) - - sent_job = server.maybe_send_job(state, clean_jobs=True) - - self.assertFalse(sent_job) - self.assertEqual(server.reorg_reconcile_error_count, 1) - self.assertEqual(server.job_build_failure_count, 0) - - def test_reconciliation_runs_again_for_same_tip_hash(self) -> None: - tip = "5a" * 32 - server = coordinator() - server.reorg_reconciler_enabled = True - ledger = ReorgLedger([]) - server.ledger = ledger - server.rpc = ReorgRpc( - tip=tip, - template=gbt_template(tip, height=11), - height=10, - block_hashes={10: tip}, + server.verify_bundle = ( # type: ignore[method-assign] + lambda *_args, **_kwargs: verified_audit_report() ) - self.assertTrue(server.ensure_reorg_reconciled_for_tip(tip)) - self.assertTrue(server.ensure_reorg_reconciled_for_tip(tip)) + def publish_newer_source_then_fail(**_kwargs: object) -> dict[str, object]: + server._reserve_payout_state_source( + "external_tip", + tip_hash=newer_tip, + ) + self.assertEqual( + server._publish_payout_state_candidate( + server._current_payout_state_candidate() + ), + 2, + ) + raise RuntimeError("ledger confirmation unavailable") - self.assertEqual(ledger.events, [("watch", 10), ("mature", 10), ("watch", 10), ("mature", 10)]) - self.assertEqual(server._payout_state_generation, 1) - self.assertEqual(server._payout_state_source[0], 1) + ledger.confirm_accepted_block = publish_newer_source_then_fail # type: ignore[method-assign] + submission = SimpleNamespace( + coinbase_tx_hex="c0ffee", + block_hash_hex=block_hash, + block_hex="00", + ) - @staticmethod - def _reconcile_coordinator_with_artifact_counter( - ledger: ReorgLedger, - rpc: ReorgRpc, - ) -> tuple[PrismCoordinator, list[tuple[int, int, bool]]]: - """Coordinator armed so candidate preparation would build an artifact.""" - server = coordinator() - server.reorg_reconciler_enabled = True - server.ledger = ledger - server.rpc = rpc - server._ensure_job_cache_state() - # Keep the speculative background preparer out of the way so the - # counter observes only builds requested by reconcile passes. - server._payout_artifact_executor_shutdown = True - server._pool_ready_latched = True - server._template_artifacts = SimpleNamespace(network_difficulty=1) - build_calls: list[tuple[int, int, bool]] = [] - - def fake_build( - expected_payout_state_generation: int, - artifact_payout_state_generation: int, - network_difficulty: int, - force_full_rescan: bool = False, - bypass_build_interval: bool = False, - during_publication: bool = False, - ) -> None: - del network_difficulty, bypass_build_interval, during_publication - build_calls.append( - ( - expected_payout_state_generation, - artifact_payout_state_generation, - force_full_rescan, + with tempfile.TemporaryDirectory() as tempdir: + server.audit_dir = Path(tempdir) + server.ledger_writer_public_key_hex = "aa" * 32 + with self.assertRaisesRegex( + RuntimeError, + "ledger confirmation unavailable", + ): + server.submit_block_candidate( + block_candidate(server, state, submission) ) - ) - return None - server._build_payout_ledger_artifact = fake_build # type: ignore[method-assign] - return server, build_calls - - def test_ledger_artifact_not_built_when_publication_not_required(self) -> None: - tip = "5b" * 32 - ledger = ReorgLedger([]) - server, build_calls = self._reconcile_coordinator_with_artifact_counter( - ledger, - ReorgRpc( - tip=tip, - template=gbt_template(tip, height=11), - height=10, - block_hashes={10: tip}, - ), + self.assertEqual(server._payout_state_generation, 2) + self.assertEqual(server._published_payout_state.source_generation, 2) + self.assertEqual(server._payout_state_source[0], 3) + self.assertEqual(server._payout_state_source[1], newer_tip) + self.assertEqual( + server._payout_state_source[2], + "direct_block_uncertain", ) + self.assertTrue(server._payout_state_publication_blocked) + self.assertTrue(server._payout_state_delivery_gate._delivery_blocked) - first = server.reconcile_prism_pool_blocks_once(tip_hash=tip) - self.assertEqual(first["published_generation"], 1) - self.assertEqual(len(build_calls), 1) - self.assertFalse(build_calls[0][2]) - - second = server.reconcile_prism_pool_blocks_once(tip_hash=tip) - - self.assertIsNone(second["published_generation"]) - self.assertEqual( - len(build_calls), - 1, - "a pass without publication need must not build the ledger artifact", + def test_post_confirm_publication_loss_completes_candidate_and_fences(self) -> None: + # Once persist + confirm are durable, losing the forced payout + # publication must not abort the candidate: the outbox row is marked + # submitted and the success tail runs, while delivery stays fenced + # until the scheduled refresh publishes the newest source. + server, state, ledger = submit_coordinator() + server._ensure_job_cache_state() + server.max_blocks = 2 + server.stop_after_block = False + block_hash = "e1" * 32 + server.rpc = SubmitRpc( + tip="00" * 32, + block_hash=block_hash, + ledger=ledger, ) - self.assertEqual( - ledger.events, - [("watch", 10), ("mature", 10), ("watch", 10), ("mature", 10)], + server.build_audit_bundle = ( # type: ignore[method-assign] + lambda **_kwargs: verified_block_bundle() ) - - def test_payout_change_forces_publication_and_builds_artifact(self) -> None: - tip = "5d" * 32 - pool_block_hash = "ae" * 32 - ledger = ReorgLedger([]) - server, build_calls = self._reconcile_coordinator_with_artifact_counter( - ledger, - ReorgRpc( - tip=tip, - template=gbt_template(tip, height=11), - height=10, - block_hashes={10: tip}, - ), + server.verify_bundle = ( # type: ignore[method-assign] + lambda *_args, **_kwargs: verified_audit_report() ) - - first = server.reconcile_prism_pool_blocks_once(tip_hash=tip) - self.assertEqual(first["published_generation"], 1) - self.assertEqual(len(build_calls), 1) - - # An orphaned confirmed block appears without any new payout source: - # payout_changed alone must still force a publication. - ledger.rows.append( - { - "block_hash": pool_block_hash, - "block_height": 12, - "chain_state": "confirmed", - "maturity_state": "immature", - } + submitted: list[dict[str, object]] = [] + ledger.mark_block_candidate_submitted = ( # type: ignore[attr-defined] + lambda **kwargs: submitted.append(kwargs) or True ) + real_confirm = ledger.confirm_accepted_block - second = server.reconcile_prism_pool_blocks_once(tip_hash=tip) - - self.assertEqual(second["inactive_blocks"], 1) - self.assertEqual(second["published_generation"], 2) - self.assertEqual(len(build_calls), 2) - self.assertTrue(build_calls[1][2]) - self.assertEqual(ledger.rows[0]["chain_state"], "inactive") - self.assertIn(("inactive", pool_block_hash, 10), ledger.events) - - def test_lost_mutation_response_publishes_fresh_balances_with_forced_rescan( - self, - ) -> None: - tip = "5e" * 32 - pool_block_hash = "af" * 32 - - class LostResponseLedger(ReorgLedger): - def __init__(self) -> None: - super().__init__([]) - self.prior_balance_reads = 0 - - def current_prior_balances(self) -> list[dict[str, object]]: - self.prior_balance_reads += 1 - if any(row["chain_state"] == "confirmed" for row in self.rows): - return [ - { - "recipient_id": "stale-carry", - "order_key": "01:stale-carry", - "p2mr_program_hex": "44" * 32, - "balance_sats": 546, - } - ] - return [] - - def mark_pool_block_inactive( - self, - *, - block_hash: str, - active_tip_height: int, - ) -> dict[str, object]: - result = super().mark_pool_block_inactive( - block_hash=block_hash, - active_tip_height=active_tip_height, - ) - if int(result["inactive_count"]): - raise ConnectionError("mutation response lost") - return result + def confirm_then_reserve_newer_source(**kwargs: object) -> dict[str, object]: + result = real_confirm(**kwargs) + server._reserve_payout_state_source( + "external_tip", + tip_hash="ee" * 32, + ) + return result - ledger = LostResponseLedger() - server = coordinator() - server.reorg_reconciler_enabled = True - server.ledger = ledger - server.rpc = ReorgRpc( - tip=tip, - template=gbt_template(tip, height=11), - height=10, - block_hashes={10: tip}, + ledger.confirm_accepted_block = confirm_then_reserve_newer_source # type: ignore[method-assign] + server._publish_current_payout_state_with_retry_budget = ( # type: ignore[method-assign] + lambda **_kwargs: None + ) + submission = SimpleNamespace( + coinbase_tx_hex="c0ffee", + block_hash_hex=block_hash, + block_hex="00", ) - forced_rescans: list[bool] = [] - original_prepare = server._prepared_payout_state_candidate - def record_prepare( - captured: tuple[int, int, str | None, str, float], - *, - force_full_window_rescan: bool = False, - bypass_build_interval: bool = False, - ) -> object: - forced_rescans.append(force_full_window_rescan) - return original_prepare( - captured, - force_full_window_rescan=force_full_window_rescan, - bypass_build_interval=bypass_build_interval, + with tempfile.TemporaryDirectory() as tempdir: + server.audit_dir = Path(tempdir) + server.evidence_path = Path(tempdir) / "evidence.json" + server.ledger_writer_public_key_hex = "aa" * 32 + self.assertTrue( + server._submit_next_block_candidate_writer( + block_candidate(server, state, submission) + ) ) - server._prepared_payout_state_candidate = record_prepare # type: ignore[method-assign] + self.assertIsNone(getattr(server, "_retry_block_candidate", None)) + self.assertEqual(len(ledger.confirmed), 1) + self.assertEqual(len(submitted), 1) + self.assertEqual(submitted[0]["block_hash"], block_hash) + self.assertEqual(server.accepted_block_count, 1) + self.assertIn(block_hash, server._accounted_accepted_block_hashes) + self.assertTrue(server._payout_state_publication_blocked) + self.assertTrue(server._payout_state_delivery_gate._delivery_blocked) + self.assertTrue(server.tip_refresh_is_pending()) - first = server.reconcile_prism_pool_blocks_once(tip_hash=tip) - self.assertEqual(first["published_generation"], 1) - with server._job_cache_lock: - initially_published = server._published_payout_state.artifact - assert initially_published is not None - self.assertEqual(len(initially_published.prior_balances()), 0) + # The scheduled refresh publishes the pending source and reopens + # delivery without any candidate replay. + del server._publish_current_payout_state_with_retry_budget + self.assertIsNotNone(server._publish_current_payout_state_with_retry_budget()) + self.assertFalse(server._payout_state_publication_blocked) + self.assertFalse(server._payout_state_delivery_gate._delivery_blocked) - ledger.rows.append( - { - "block_hash": pool_block_hash, - "block_height": 12, - "chain_state": "confirmed", - "maturity_state": "immature", - } - ) - # Rebuild the current generation's immutable state so the test starts - # from balances that predate the lost-response mutation. - with server._job_cache_lock: - server._published_payout_state = dataclass_replace( - server._published_payout_state, - artifact=None, + def test_idempotent_direct_block_replay_skips_publication(self) -> None: + server, state, ledger = submit_coordinator() + server._ensure_job_cache_state() + server.max_blocks = 2 + server.stop_after_block = False + block_hash = "d0" * 32 + with tempfile.TemporaryDirectory() as tempdir: + server.audit_dir = Path(tempdir) + server.evidence_path = Path(tempdir) / "evidence.json" + server.ledger_writer_public_key_hex = "aa" * 32 + rpc = SubmitRpc( + tip="00" * 32, + block_hash=block_hash, + ledger=ledger, + ) + server.rpc = rpc + server.build_audit_bundle = ( # type: ignore[method-assign] + lambda **_kwargs: verified_block_bundle() + ) + server.verify_bundle = ( # type: ignore[method-assign] + lambda *_args, **_kwargs: verified_audit_report() + ) + submission = SimpleNamespace( + coinbase_tx_hex="c0ffee", + block_hash_hex=block_hash, + block_hex="00", ) - stale_published = server._current_payout_state_artifact() - self.assertEqual(len(stale_published.prior_balances()), 1) - reads_before_error = ledger.prior_balance_reads - forced_rescans.clear() - - with self.assertRaisesRegex(ConnectionError, "mutation response lost"): - server.reconcile_prism_pool_blocks_once(tip_hash=tip) - - with server._job_cache_lock: - healed_state = server._published_payout_state - assert healed_state.artifact is not None - self.assertEqual(healed_state.generation, 2) - self.assertEqual(healed_state.artifact.prior_balances(), []) - self.assertEqual(ledger.prior_balance_reads, reads_before_error + 1) - self.assertEqual(forced_rescans, [True]) - self.assertEqual(ledger.rows[0]["chain_state"], "inactive") - - def test_read_phase_reconcile_failure_does_not_force_balance_rescan( - self, - ) -> None: - tip = "5f" * 32 - - class PrefetchFailureLedger(ReorgLedger): - fail_prefetch = False - - def __init__(self) -> None: - super().__init__([]) - self.prior_balance_reads = 0 - - def current_prior_balances(self) -> list[dict[str, object]]: - self.prior_balance_reads += 1 - return [] - def reorg_watch_blocks( - self, - *, - active_tip_height: int, - ) -> list[dict[str, object]]: - if self.fail_prefetch: - raise TimeoutError("reconcile prefetch join exceeded 20s") - return super().reorg_watch_blocks( - active_tip_height=active_tip_height + self.assertTrue( + server.submit_block_candidate( + block_candidate(server, state, submission) ) + ) + self.assertEqual(server._payout_state_generation, 1) + self.assertEqual( + server._published_payout_state.source_tip_hash, + block_hash, + ) + self.assertEqual(server._payout_state_source[0], 1) - ledger = PrefetchFailureLedger() - server = coordinator() - server.reorg_reconciler_enabled = True - server.ledger = ledger - server.rpc = ReorgRpc( - tip=tip, - template=gbt_template(tip, height=11), - height=10, - block_hashes={10: tip}, - ) - first = server.reconcile_prism_pool_blocks_once(tip_hash=tip) - self.assertEqual(first["published_generation"], 1) - reads_before_failure = ledger.prior_balance_reads - with server._job_cache_lock: - generation_before_failure = server._published_payout_state.generation + # Replay the durable candidate after its block landed, its + # confirmation committed, and the network built on top of it. + # Exact confirmed replay reports confirmed_count=1 at the block's + # own expected height even though the active chain tip is now a + # child. The published payout state already covers the + # candidate, so the replay must not reserve a source, bump the + # generation, wipe the job-bundle cache, or schedule refresh + # churn. + child_tip = "d7" * 32 - forced_rescans: list[bool] = [] - original_prepare = server._prepared_payout_state_candidate + class AncestorReplayRpc: + def call(self, method: str, params: object = None) -> object: + if method == "getbestblockhash": + return child_tip + if method == "getblockheader": + if params != [block_hash]: + raise AssertionError(params) + return {"height": 10, "confirmations": 2} + if method == "getblockhash": + if params != [10]: + raise AssertionError(params) + return block_hash + if method == "getblockcount": + return 11 + if method == "submitblock": + raise AssertionError( + "active ancestor must not be resubmitted" + ) + raise RuntimeError(method) - def record_prepare( - captured: tuple[int, int, str | None, str, float], - *, - force_full_window_rescan: bool = False, - bypass_build_interval: bool = False, - ) -> object: - forced_rescans.append(force_full_window_rescan) - return original_prepare( - captured, - force_full_window_rescan=force_full_window_rescan, - bypass_build_interval=bypass_build_interval, + server.rpc = AncestorReplayRpc() + ledger.pool_block_state = ( # type: ignore[attr-defined] + lambda **_kwargs: { + "chain_state": "confirmed", + "maturity_state": "immature", + } ) + def confirm_exact_ancestor(**kwargs: object) -> dict[str, object]: + self.assertEqual(kwargs["block_hash"], block_hash) + self.assertEqual(kwargs["active_tip_height"], 10) + return {"backend": "fake", "confirmed_count": 1} - server._prepared_payout_state_candidate = record_prepare # type: ignore[method-assign] - ledger.fail_prefetch = True - - with self.assertRaisesRegex( - TimeoutError, - "reconcile prefetch join exceeded 20s", - ): - server.reconcile_prism_pool_blocks_once(tip_hash=tip) + ledger.confirm_accepted_block = confirm_exact_ancestor # type: ignore[method-assign] + retry_calls = 0 - with server._job_cache_lock: - generation_after_failure = server._published_payout_state.generation - self.assertEqual(forced_rescans, []) - self.assertEqual(ledger.prior_balance_reads, reads_before_failure) - self.assertEqual(generation_after_failure, generation_before_failure) + def count_retry() -> None: + nonlocal retry_calls + retry_calls += 1 - def test_concurrent_same_tip_reconciles_share_one_pass(self) -> None: - tip = "5c" * 32 - entered = threading.Event() - release = threading.Event() + server._schedule_tip_refresh_retry = count_retry # type: ignore[method-assign] + cache_key = ("sentinel",) + server._job_bundle_cache[cache_key] = object() + discarded_before = server.payout_state_candidates_discarded + pending_marks_before = server._tip_refresh_pending_counter - class BlockingReorgLedger(ReorgLedger): - def reorg_watch_blocks( - self, - *, - active_tip_height: int, - ) -> list[dict[str, object]]: - entered.set() - if not release.wait(timeout=5.0): - raise AssertionError("reconcile release was never signaled") - return super().reorg_watch_blocks( - active_tip_height=active_tip_height + self.assertTrue( + server.submit_block_candidate( + block_candidate(server, state, submission) ) + ) - ledger = BlockingReorgLedger([]) - server = coordinator() - server.reorg_reconciler_enabled = True - server.ledger = ledger - server.rpc = ReorgRpc( - tip=tip, - template=gbt_template(tip, height=11), - height=10, - block_hashes={10: tip}, + self.assertEqual(server._payout_state_generation, 1) + self.assertEqual(server._published_payout_state.source_generation, 1) + self.assertEqual(server._published_payout_state.source_tip_hash, block_hash) + self.assertEqual(server._payout_state_source[0], 1) + self.assertIn(cache_key, server._job_bundle_cache) + self.assertEqual(retry_calls, 0) + self.assertEqual( + server._tip_refresh_pending_counter, + pending_marks_before, ) - server._ensure_job_cache_state() - - results: list[dict[str, object] | None] = [None, None] - - def reconcile(slot: int) -> None: - results[slot] = server.reconcile_prism_pool_blocks_once(tip_hash=tip) - - leader = threading.Thread(target=reconcile, args=(0,)) - leader.start() - self.assertTrue(entered.wait(timeout=5.0)) - with server._reconcile_flight_lock: - flight = server._reconcile_flights[tip] - - follower_waiting = threading.Event() - - class SignalingEvent: - def __init__(self, inner: threading.Event) -> None: - self._inner = inner - - def wait(self, timeout: float | None = None) -> bool: - follower_waiting.set() - return self._inner.wait(timeout) - - def set(self) -> None: - self._inner.set() - - flight.event = SignalingEvent(flight.event) # type: ignore[assignment] - follower = threading.Thread(target=reconcile, args=(1,)) - follower.start() - self.assertTrue(follower_waiting.wait(timeout=5.0)) - - release.set() - leader.join(timeout=5.0) - follower.join(timeout=5.0) - self.assertFalse(leader.is_alive()) - self.assertFalse(follower.is_alive()) - self.assertEqual( - ledger.events, - [("watch", 10), ("mature", 10)], - "the follower must reuse the leader's pass, not run its own", + server.payout_state_candidates_discarded, + discarded_before, ) - self.assertEqual(results[0], results[1]) - self.assertIsNotNone(results[0]) - self.assertIsNot(results[0], results[1]) - with server._reconcile_flight_lock: - self.assertEqual(server._reconcile_flights, {}) + self.assertFalse(server._payout_state_publication_blocked) + self.assertFalse(server._payout_state_delivery_gate._delivery_blocked) - def test_lock_owner_reconcile_bypasses_same_tip_flight(self) -> None: - tip = "5f" * 32 - ledger = ReorgLedger([]) - server = coordinator() - server.reorg_reconciler_enabled = True - server.ledger = ledger - server.rpc = ReorgRpc( - tip=tip, - template=gbt_template(tip, height=11), - height=10, - block_hashes={10: tip}, - ) + def test_at_tip_idempotent_confirm_replay_skips_publication(self) -> None: + """An at-tip replay reports the idempotent disposition, not a flip. + + qbit_confirm_pool_block returns 2 when its UPDATE matched nothing but + the row is already confirmed at that (hash, height): the call changed + nothing and allocated no publication ordinal. Before that split the + same replay reported 1 and was indistinguishable from a fresh flip, + so the window after finding a block -- where the candidate replays + while its own block is still the best tip -- republished payout state + on every pass. The replay must complete the candidate without + reserving a source, bumping the generation, wiping the job-bundle + cache or scheduling refresh churn, and must never be mistaken for a + failed confirmation, which fences delivery, republishes a withdrawal + and halts the pool (issue #61). + """ + server, state, ledger = submit_coordinator() server._ensure_job_cache_state() + server.max_blocks = 2 + server.stop_after_block = False + block_hash = "da" * 32 + with tempfile.TemporaryDirectory() as tempdir: + server.audit_dir = Path(tempdir) + server.evidence_path = Path(tempdir) / "evidence.json" + server.ledger_writer_public_key_hex = "aa" * 32 + server.rpc = SubmitRpc( + tip="00" * 32, + block_hash=block_hash, + ledger=ledger, + ) + server.build_audit_bundle = ( # type: ignore[method-assign] + lambda **_kwargs: verified_block_bundle() + ) + server.verify_bundle = ( # type: ignore[method-assign] + lambda *_args, **_kwargs: verified_audit_report() + ) + submission = SimpleNamespace( + coinbase_tx_hex="c0ffee", + block_hash_hex=block_hash, + block_hex="00", + ) + + self.assertTrue( + server.submit_block_candidate( + block_candidate(server, state, submission) + ) + ) + self.assertEqual(server._payout_state_generation, 1) + self.assertEqual( + server._published_payout_state.source_tip_hash, + block_hash, + ) + self.assertEqual(server._payout_state_source[0], 1) + flipped_sequence = int( + ledger._audit_publication_sequences[block_hash] + ) - leader_lock_attempted = threading.Event() - flight_waited = threading.Event() - flight_completed = threading.Event() - underlying_lock = threading.RLock() - leader: threading.Thread | None = None + # Replay the durable candidate while its own block is still the + # best tip -- no successor has arrived, so nothing about the + # active chain distinguishes this pass from the original landing. - class ObservedBalanceLock: - def __enter__(self) -> ObservedBalanceLock: - if threading.current_thread() is leader: - if underlying_lock.acquire(blocking=False): - underlying_lock.release() + class AtTipReplayRpc: + def call(self, method: str, params: object = None) -> object: + if method == "getbestblockhash": + return block_hash + if method == "getblockheader": + if params != [block_hash]: + raise AssertionError(params) + return {"height": 10, "confirmations": 1} + if method == "getblockhash": + if params != [10]: + raise AssertionError(params) + return block_hash + if method == "getblockcount": + return 10 + if method == "submitblock": raise AssertionError( - "reconcile leader unexpectedly acquired the owned lock" + "active tip must not be resubmitted" ) - leader_lock_attempted.set() - underlying_lock.acquire() - else: - underlying_lock.acquire() - return self - - def __exit__(self, *_args: object) -> None: - underlying_lock.release() - - class WaitForbiddenEvent: - def __init__(self, inner: threading.Event) -> None: - self._inner = inner + raise RuntimeError(method) - def wait(self, timeout: float | None = None) -> bool: - flight_waited.set() - raise AssertionError( - "lock-owning reconciliation joined the same-tip flight" - ) + server.rpc = AtTipReplayRpc() + ledger.pool_block_state = ( # type: ignore[attr-defined] + lambda **_kwargs: { + "chain_state": "confirmed", + "maturity_state": "immature", + } + ) - def set(self) -> None: - flight_completed.set() - self._inner.set() + def confirm_at_tip_replay(**kwargs: object) -> dict[str, object]: + self.assertEqual(kwargs["block_hash"], block_hash) + self.assertEqual(kwargs["active_tip_height"], 10) + # The idempotent arm returns the ordinal the original flip + # allocated and burns no sequence. + return { + "backend": "fake", + "confirmed_count": 2, + "audit_publication_sequence": flipped_sequence, + } - server._payout_balance_mutation_lock = ObservedBalanceLock() # type: ignore[assignment] - leader_results: list[dict[str, object]] = [] - leader_errors: list[BaseException] = [] + ledger.confirm_accepted_block = confirm_at_tip_replay # type: ignore[method-assign] + retry_calls = 0 - def lead_reconcile() -> None: - try: - leader_results.append( - server.reconcile_prism_pool_blocks_once(tip_hash=tip) - ) - except BaseException as exc: # noqa: BLE001 - asserted below - leader_errors.append(exc) + def count_retry() -> None: + nonlocal retry_calls + retry_calls += 1 - leader = threading.Thread(target=lead_reconcile) - try: - with server._payout_balance_mutation_lock: - leader.start() - self.assertTrue(leader_lock_attempted.wait(timeout=5.0)) - with server._reconcile_flight_lock: - flight = server._reconcile_flights[tip] - flight.event = WaitForbiddenEvent( # type: ignore[assignment] - flight.event - ) + server._schedule_tip_refresh_retry = count_retry # type: ignore[method-assign] + cache_key = ("sentinel",) + server._job_bundle_cache[cache_key] = object() + discarded_before = server.payout_state_candidates_discarded + pending_marks_before = server._tip_refresh_pending_counter - bypassed = server.ensure_reorg_reconciled_for_tip( - tip, - _coalesce_same_tip=False, + self.assertTrue( + server.submit_block_candidate( + block_candidate(server, state, submission) ) + ) + latest_evidence = server.latest_evidence - self.assertTrue(bypassed) - self.assertFalse(flight_waited.is_set()) - self.assertTrue(leader.is_alive()) - self.assertEqual( - ledger.events, - [("watch", 10), ("mature", 10)], - ) - finally: - leader.join(timeout=5.0) - - self.assertFalse(leader.is_alive()) - if leader_errors: - raise leader_errors[0] - self.assertEqual(len(leader_results), 1) - self.assertTrue(flight_completed.is_set()) - self.assertFalse(flight_waited.is_set()) + self.assertEqual(server._payout_state_generation, 1) + self.assertEqual(server._published_payout_state.source_generation, 1) + self.assertEqual(server._published_payout_state.source_tip_hash, block_hash) + self.assertEqual(server._payout_state_source[0], 1) + self.assertIn(cache_key, server._job_bundle_cache) + self.assertEqual(retry_calls, 0) self.assertEqual( - ledger.events, - [ - ("watch", 10), - ("mature", 10), - ("watch", 10), - ("mature", 10), - ], + server._tip_refresh_pending_counter, + pending_marks_before, ) - with server._reconcile_flight_lock: - self.assertEqual(server._reconcile_flights, {}) - - def test_lock_owner_reconcile_leads_visible_flight_when_absent(self) -> None: - tip = "60" * 32 - entered = threading.Event() - release = threading.Event() - - class BlockingReorgLedger(ReorgLedger): - def reorg_watch_blocks( - self, - *, - active_tip_height: int, - ) -> list[dict[str, object]]: - entered.set() - if not release.wait(timeout=5.0): - raise AssertionError("reconcile release was never signaled") - return super().reorg_watch_blocks( - active_tip_height=active_tip_height - ) - - ledger = BlockingReorgLedger([]) - server = coordinator() - server.reorg_reconciler_enabled = True - server.ledger = ledger - server.rpc = ReorgRpc( - tip=tip, - template=gbt_template(tip, height=11), - height=10, - block_hashes={10: tip}, + self.assertEqual( + server.payout_state_candidates_discarded, + discarded_before, ) - server._ensure_job_cache_state() - - lock_owner_result: list[bool] = [] - follower_result: list[dict[str, object]] = [] - errors: list[BaseException] = [] - follower_waiting = threading.Event() - - class SignalingEvent: - def __init__(self, inner: threading.Event) -> None: - self._inner = inner - - def wait(self, timeout: float | None = None) -> bool: - follower_waiting.set() - return self._inner.wait(timeout) - - def set(self) -> None: - self._inner.set() - - def reconcile_while_owning_lock() -> None: - try: - with server._payout_balance_mutation_lock: - lock_owner_result.append( - server.ensure_reorg_reconciled_for_tip( - tip, - _coalesce_same_tip=False, - ) - ) - except BaseException as exc: # noqa: BLE001 - asserted below - errors.append(exc) - - def follow_reconcile() -> None: - try: - follower_result.append( - server.reconcile_prism_pool_blocks_once(tip_hash=tip) - ) - except BaseException as exc: # noqa: BLE001 - asserted below - errors.append(exc) - - lock_owner = threading.Thread(target=reconcile_while_owning_lock) - follower = threading.Thread(target=follow_reconcile) - lock_owner.start() - try: - self.assertTrue(entered.wait(timeout=5.0)) - with server._reconcile_flight_lock: - flight = server._reconcile_flights[tip] - flight.event = SignalingEvent(flight.event) # type: ignore[assignment] - - follower.start() - self.assertTrue(follower_waiting.wait(timeout=5.0)) - self.assertTrue(lock_owner.is_alive()) - self.assertTrue(follower.is_alive()) - self.assertEqual(ledger.events, []) - finally: - release.set() - lock_owner.join(timeout=5.0) - if follower.ident is not None: - follower.join(timeout=5.0) - - self.assertFalse(lock_owner.is_alive()) - self.assertFalse(follower.is_alive()) - if errors: - raise errors[0] - self.assertEqual(lock_owner_result, [True]) - self.assertEqual(len(follower_result), 1) + self.assertFalse(server._payout_state_publication_blocked) + self.assertFalse(server._payout_state_delivery_gate._delivery_blocked) + # A covered replay is not a confirmation failure: no shutdown, and the + # candidate stays accepted rather than being abandoned. + self.assertFalse(server.stop_event.is_set()) + self.assertEqual(server.accepted_block_count, 1) + # The audit publication replays exactly against the ordinal the flip + # allocated. The disposition itself is observational -- reporting 2 + # where the published evidence recorded 1 must not strand the replay + # as a payload conflict. + self.assertIsNotNone(latest_evidence) + assert latest_evidence is not None self.assertEqual( - ledger.events, - [("watch", 10), ("mature", 10)], - "the ordinary caller must reuse the lock owner's visible pass", + latest_evidence["audit_publication_identity"]["sequence"], + flipped_sequence, ) - with server._reconcile_flight_lock: - self.assertEqual(server._reconcile_flights, {}) + self.assertEqual(latest_evidence["confirmation"]["confirmed_count"], 1) - def test_forced_and_reserved_reconciles_bypass_flight_reuse(self) -> None: - tip = "5e" * 32 - ledger = ReorgLedger([]) - server = coordinator() - server.reorg_reconciler_enabled = True - server.ledger = ledger - server.rpc = ReorgRpc( - tip=tip, - template=gbt_template(tip, height=11), - height=10, - block_hashes={10: tip}, - ) + def test_leaked_publication_fence_replay_republishes(self) -> None: + server, state, ledger = submit_coordinator() server._ensure_job_cache_state() + server.max_blocks = 2 + server.stop_after_block = False + block_hash = "d8" * 32 + with tempfile.TemporaryDirectory() as tempdir: + server.audit_dir = Path(tempdir) + server.evidence_path = Path(tempdir) / "evidence.json" + server.ledger_writer_public_key_hex = "aa" * 32 + server.rpc = SubmitRpc( + tip="00" * 32, + block_hash=block_hash, + ledger=ledger, + ) + server.build_audit_bundle = ( # type: ignore[method-assign] + lambda **_kwargs: verified_block_bundle() + ) + server.verify_bundle = ( # type: ignore[method-assign] + lambda *_args, **_kwargs: verified_audit_report() + ) + submission = SimpleNamespace( + coinbase_tx_hex="c0ffee", + block_hash_hex=block_hash, + block_hex="00", + ) - sentinel_summary: dict[str, object] = {"sentinel": True} - flight = _ReconcileFlight() - flight.summary = sentinel_summary - flight.event.set() - with server._reconcile_flight_lock: - server._reconcile_flights[tip] = flight - - joined = server.reconcile_prism_pool_blocks_once(tip_hash=tip) - self.assertEqual(joined, sentinel_summary) - self.assertIsNot(joined, sentinel_summary) - self.assertEqual(ledger.events, []) - - forced = server.reconcile_prism_pool_blocks_once( - tip_hash=tip, - _force_publish=True, - ) - self.assertEqual(forced["published_generation"], 1) - self.assertEqual(ledger.events, [("watch", 10), ("mature", 10)]) - - reserved = server.reconcile_prism_pool_blocks_once( - tip_hash=tip, - _source_reserved=True, - ) - self.assertIsNone(reserved["published_generation"]) - self.assertEqual( - ledger.events, - [("watch", 10), ("mature", 10), ("watch", 10), ("mature", 10)], - ) - - with server._reconcile_flight_lock: - server._reconcile_flights.pop(tip, None) - - def test_pass_spanning_detection_cycle_does_not_arm_memo(self) -> None: - # A pass whose reads straddle a flip away and back finishes with the - # latest detected hash matching its tip again, but its proof belongs - # to the closed epoch: arming must be refused so every memo consumer - # (refresh joins, initial-job and vardiff-idle builds) re-proves. - tip = "5c" * 32 - server = coordinator() - server.reorg_reconciler_enabled = True - ledger = ReorgLedger([]) - - def epoch_bumping_watch(*, active_tip_height: int) -> list[dict[str, object]]: - with server.lock: - server.tip_detection_epoch = ( - int(getattr(server, "tip_detection_epoch", 0)) + 2 + self.assertTrue( + server.submit_block_candidate( + block_candidate(server, state, submission) ) - return [] - - ledger.reorg_watch_blocks = epoch_bumping_watch # type: ignore[method-assign] - server.ledger = ledger - server.rpc = ReorgRpc( - tip=tip, - template=gbt_template(tip, height=11), - height=10, - block_hashes={10: tip}, - ) - - self.assertTrue(server.ensure_reorg_reconciled_for_tip(tip)) - - with server.lock: - self.assertNotIn(tip, server._reorg_reconcile_trusted_memo) - - def test_row_mutating_pass_evicts_other_tip_memo_entries(self) -> None: - tip_a = "a1" * 32 - tip_b = "b2" * 32 - pool_block_hash = "cd" * 32 - server = coordinator() - server.reorg_reconciler_enabled = True - ledger = ReorgLedger([]) - server.ledger = ledger - server.rpc = ReorgRpc( - tip=tip_b, - template=gbt_template(tip_b, height=11), - height=10, - block_hashes={10: tip_b}, - ) - server._ensure_job_cache_state() - server._note_reorg_reconcile_outcome(tip_a, trusted=True) - - # A pass for another tip that mutates no rows leaves tip A's cached - # proof valid: rows and tip-A chain state are both unchanged. - first = server.reconcile_prism_pool_blocks_once(tip_hash=tip_b) - self.assertEqual(first["published_generation"], 1) - with server.lock: - self.assertIn(tip_a, server._reorg_reconcile_trusted_memo) - self.assertIn(tip_b, server._reorg_reconcile_trusted_memo) - - # A pass that applies orphan/maturity row mutations ends every other - # tip's memo epoch, even though tip B was never observed by - # observe_tip_for_refresh. - ledger.rows.append( - { - "block_hash": pool_block_hash, - "block_height": 12, - "chain_state": "confirmed", - "maturity_state": "immature", - } - ) - second = server.reconcile_prism_pool_blocks_once(tip_hash=tip_b) - self.assertEqual(second["inactive_blocks"], 1) - with server.lock: - self.assertNotIn(tip_a, server._reorg_reconcile_trusted_memo) - self.assertIn(tip_b, server._reorg_reconcile_trusted_memo) - - def test_reconciliation_reactivates_inactive_block_that_returns_to_active_chain(self) -> None: - pool_block_hash = "cc" * 32 - server = coordinator() - server.reorg_reconciler_enabled = True - ledger = ReorgLedger( - [ - { - "block_hash": pool_block_hash, - "block_height": 12, - "chain_state": "inactive", - "maturity_state": "immature", - } - ] - ) - server.ledger = ledger - server.rpc = ReorgRpc( - tip=pool_block_hash, - template=gbt_template(pool_block_hash, height=13), - height=12, - block_hashes={12: pool_block_hash}, - ) + ) + self.assertEqual(server._payout_state_generation, 1) - summary = server.reconcile_prism_pool_blocks_once(tip_hash=pool_block_hash) + # Simulate the exception tail a replay must heal: a prior attempt + # force-blocked delivery while its source generation already + # matched the published source, then failed before republishing. + # The leaked fence blocks every job build until a publication + # lands, so an exact confirmed replay must heal the leaked fence + # before taking the covered-state skip. + server._block_payout_state_publication(force=True) + self.assertTrue(server._payout_state_publication_blocked) + self.assertEqual( + server._payout_state_source[0], + server._published_payout_state.source_generation, + ) + child_tip = "d9" * 32 - self.assertEqual(summary["reactivated_blocks"], 1) - self.assertEqual(server.reorg_reactivated_block_count, 1) - self.assertEqual(ledger.rows[0]["chain_state"], "confirmed") - self.assertIn(("mature", 12), ledger.events) + class AncestorReplayRpc: + def call(self, method: str, params: object = None) -> object: + if method == "getbestblockhash": + return child_tip + if method == "getblockheader": + if params != [block_hash]: + raise AssertionError(params) + return {"height": 10, "confirmations": 2} + if method == "getblockhash": + if params != [10]: + raise AssertionError(params) + return block_hash + if method == "getblockcount": + return 11 + if method == "submitblock": + raise AssertionError( + "active ancestor must not be resubmitted" + ) + raise RuntimeError(method) - def test_maybe_send_job_reconciles_before_direct_job_build(self) -> None: - tip = "66" * 32 - pool_block_hash = "dd" * 32 - server = coordinator() - server.reorg_reconciler_enabled = True - server.jobs = {} - worker = worker_identity() - state = client() - state.username = worker.username - state.worker = worker - state.share_difficulty = Decimal("1") - sent: list[dict[str, object]] = [] - state.send = lambda payload: sent.append(payload) # type: ignore[method-assign] - ledger = ReorgLedger( - [ - { - "block_hash": pool_block_hash, - "block_height": 20, + server.rpc = AncestorReplayRpc() + ledger.pool_block_state = ( # type: ignore[attr-defined] + lambda **_kwargs: { "chain_state": "confirmed", "maturity_state": "immature", } - ] - ) - server.ledger = ledger - server.rpc = ReorgRpc( - tip=tip, - template=gbt_template(tip, height=21), - height=20, - block_hashes={20: "ee" * 32}, - ) - - def build_direct_job(client: ClientState, *, clean_jobs: bool) -> object: - self.assertIn(("inactive", pool_block_hash, 20), ledger.events) - ledger.events.append(("build", client.connection_id)) - return prism_context("direct-job", tip, worker=worker, clean_jobs=clean_jobs) - - server.build_job_for_client = build_direct_job # type: ignore[method-assign] - - sent_job = server.maybe_send_job(state, clean_jobs=True) - - self.assertTrue(sent_job) - self.assertLess( - ledger.events.index(("inactive", pool_block_hash, 20)), - ledger.events.index(("build", state.connection_id)), - ) - self.assertEqual(sent[1]["params"][0], "direct-job") - - def test_make_ledger_requires_explicit_memory_opt_in(self) -> None: - server = PrismCoordinator.__new__(PrismCoordinator) - - with patch.dict(os.environ, {}, clear=True): - with self.assertRaises(SystemExit): - server.make_ledger() - - with patch.dict(os.environ, {"PRISM_ALLOW_MEMORY_LEDGER": "1"}, clear=True): - ledger = server.make_ledger() - - self.assertEqual(ledger.backend_name, "memory") + ) + def confirm_exact_ancestor(**kwargs: object) -> dict[str, object]: + self.assertEqual(kwargs["block_hash"], block_hash) + self.assertEqual(kwargs["active_tip_height"], 10) + return {"backend": "fake", "confirmed_count": 1} - def test_trusted_ledger_key_must_be_configured_or_explicitly_test_mode(self) -> None: - server = PrismCoordinator.__new__(PrismCoordinator) + ledger.confirm_accepted_block = confirm_exact_ancestor # type: ignore[method-assign] - with patch.dict(os.environ, {}, clear=True): - with self.assertRaises(SystemExit): - server.load_trusted_ledger_writer_public_key() + self.assertTrue( + server.submit_block_candidate( + block_candidate(server, state, submission) + ) + ) - with patch.dict(os.environ, {"PRISM_LEDGER_WRITER_PUBLIC_KEY_HEX": "aa" * 32}, clear=True): - self.assertEqual(server.load_trusted_ledger_writer_public_key(), "aa" * 32) + self.assertEqual(server._payout_state_generation, 2) + # Fence healing republishes identical covered state. It advances the + # delivery generation without inventing a logical invalidation source. + self.assertEqual(server._published_payout_state.source_generation, 1) + self.assertEqual(server._payout_state_source[0], 1) + self.assertFalse(server._payout_state_publication_blocked) + self.assertFalse(server._payout_state_delivery_gate._delivery_blocked) - with patch.dict(os.environ, {"PRISM_ALLOW_BUNDLE_EMBEDDED_LEDGER_KEY": "1"}, clear=True): - self.assertIsNone(server.load_trusted_ledger_writer_public_key()) + def test_direct_block_disabled_reconciler_bounds_publish_supersession(self) -> None: + server, state, ledger = submit_coordinator() + server._ensure_job_cache_state() + server.payout_reconcile_supersession_retries = 2 + block_hash = "d3" * 32 + real_publish = server._publish_payout_state_candidate + publish_attempts = 0 - def test_fixed_ledger_session_token_requires_explicit_opt_in(self) -> None: - server = PrismCoordinator.__new__(PrismCoordinator) - env = { - "PRISM_POSTGRES_PSQL_COMMAND": "psql postgresql://example.invalid/qbit", - "PRISM_LEDGER_WRITER_SESSION_TOKEN": "fixed-session", - } + def supersede_before_publish(candidate: object) -> int | None: + nonlocal publish_attempts + publish_attempts += 1 + server._reserve_payout_state_source( + "external_tip", + tip_hash=f"{publish_attempts + 20:064x}", + ) + return real_publish(candidate) # type: ignore[arg-type] - with patch.dict(os.environ, env, clear=True): - with self.assertRaises(SystemExit): - server.make_ledger() + server._publish_payout_state_candidate = supersede_before_publish # type: ignore[method-assign] + with tempfile.TemporaryDirectory() as tempdir: + server.audit_dir = Path(tempdir) + server.evidence_path = Path(tempdir) / "evidence.json" + server.ledger_writer_public_key_hex = "aa" * 32 + server.rpc = SubmitRpc( + tip="00" * 32, + block_hash=block_hash, + ledger=ledger, + ) + server.build_audit_bundle = ( # type: ignore[method-assign] + lambda **_kwargs: verified_block_bundle() + ) + server.verify_bundle = ( # type: ignore[method-assign] + lambda *_args, **_kwargs: verified_audit_report() + ) + submission = SimpleNamespace( + coinbase_tx_hex="c0ffee", + block_hash_hex=block_hash, + block_hex="00", + ) - with patch.dict(os.environ, {**env, "PRISM_ALLOW_FIXED_LEDGER_SESSION_TOKEN": "1"}, clear=True): - with patch("lab.prism.prism_coordinator.PsqlShareLedger") as fake_ledger: - fake_ledger.return_value = SimpleNamespace(backend_name="postgres-psql") - ledger = server.make_ledger() + accepted = server.submit_block_candidate( + block_candidate(server, state, submission) + ) - self.assertEqual(ledger.backend_name, "postgres-psql") - self.assertEqual(fake_ledger.call_args.kwargs["writer_session_token"], "fixed-session") + self.assertTrue(accepted) + self.assertEqual(publish_attempts, 3) + self.assertEqual(server._payout_state_generation, 0) + self.assertTrue(server._payout_state_publication_blocked) + self.assertTrue(server._payout_state_delivery_gate._delivery_blocked) + self.assertTrue(server.tip_refresh_is_pending()) - def test_generated_coordinator_session_advertises_heartbeat_capability(self) -> None: - server = PrismCoordinator.__new__(PrismCoordinator) - env = { - "PRISM_POSTGRES_PSQL_COMMAND": "psql postgresql://example.invalid/qbit", - } + def test_untrusted_direct_block_reconcile_publishes_newer_source_once(self) -> None: + server, state, ledger = submit_coordinator() + server._ensure_job_cache_state() + server.reorg_reconciler_enabled = True + server.ensure_reorg_reconciled_for_tip = ( # type: ignore[method-assign] + lambda _tip, *, _coalesce_same_tip: not _coalesce_same_tip + ) + server.qbit_chain_view_untrusted = lambda: True # type: ignore[method-assign] + block_hash = "d1" * 32 + newer_tip = "d2" * 32 + real_confirm = ledger.confirm_accepted_block - with patch.dict(os.environ, env, clear=True), patch( - "lab.prism.prism_coordinator.PsqlShareLedger" - ) as fake_ledger: - fake_ledger.return_value = SimpleNamespace(backend_name="postgres-psql") - server.make_ledger() + def superseding_noop_confirmation(**kwargs: object) -> dict[str, object]: + server._reserve_payout_state_source( + "external_tip", + tip_hash=newer_tip, + ) + return real_confirm(**kwargs) - self.assertTrue( - fake_ledger.call_args.kwargs["writer_session_token"].startswith( - WRITER_LEASE_HEARTBEAT_SESSION_PREFIX + ledger.confirm_accepted_block = superseding_noop_confirmation # type: ignore[method-assign] + with tempfile.TemporaryDirectory() as tempdir: + server.audit_dir = Path(tempdir) + server.evidence_path = Path(tempdir) / "evidence.json" + server.ledger_writer_public_key_hex = "aa" * 32 + server.rpc = SubmitRpc( + tip="00" * 32, + block_hash=block_hash, + ledger=ledger, + ) + server.build_audit_bundle = ( # type: ignore[method-assign] + lambda **_kwargs: verified_block_bundle() + ) + server.verify_bundle = ( # type: ignore[method-assign] + lambda *_args, **_kwargs: verified_audit_report() + ) + submission = SimpleNamespace( + coinbase_tx_hex="c0ffee", + block_hash_hex=block_hash, + block_hex="00", ) - ) - def test_same_tip_retention_requires_connection_derived_production_bound(self) -> None: - with self.assertRaisesRegex( - SystemExit, - "PRISM_STRATUM_SAME_TIP_JOB_RETENTION_PER_CONNECTION", - ): - validate_same_tip_job_retention_limits( - retention_seconds=30, - per_connection=0, - max_connections=0, - production=False, - ) - with self.assertRaisesRegex(SystemExit, "PRISM_STRATUM_MAX_CONNECTIONS"): - validate_same_tip_job_retention_limits( - retention_seconds=30, - per_connection=64, - max_connections=0, - production=True, - ) - - validate_same_tip_job_retention_limits( - retention_seconds=30, - per_connection=64, - max_connections=1_900, - production=True, - ) - validate_same_tip_job_retention_limits( - retention_seconds=30, - per_connection=64, - max_connections=0, - production=False, - ) - validate_same_tip_job_retention_limits( - retention_seconds=0, - per_connection=0, - max_connections=0, - production=True, - ) - - def test_production_gate_rejects_prism_test_bypasses_without_capacity_evidence(self) -> None: - base = { - "QBIT_PRODUCTION": "1", - "QBIT_RPC_USER": "qbitrpc", - "QBIT_RPC_PASSWORD": "not-default", - "PRISM_POSTGRES_PSQL_COMMAND": "psql postgresql://example.invalid/qbit", - "PRISM_POSTGRES_PASSWORD": "not-default", - "PRISM_MANIFEST_SIGNING_SEED_HEX": "42" * 32, - "PRISM_LEDGER_ATTESTATION_SIGNING_SEED_HEX": "43" * 32, - "PRISM_LEDGER_WRITER_PUBLIC_KEY_HEX": "44" * 32, - "PRISM_LEDGER_WRITER_ID": "managed-writer", - "PRISM_LEDGER_WRITER_EPOCH": "7", - "PRISM_AUDIT_DIR": "/var/lib/qbit/prism/audit", - "PRISM_EVIDENCE_PATH": "/var/lib/qbit/prism/evidence.json", - "PRISM_STRATUM_STALE_GRACE_SECONDS": "0", - "PRISM_STRATUM_SHARE_DIFF": "1024", - "PRISM_STRATUM_VARDIFF_MIN_DIFF": "1024", - "PRISM_STRATUM_VARDIFF_START_DIFF": "4096", - "PRISM_STRATUM_VARDIFF_MAX_DIFF": "65536", - "PRISM_STRATUM_MAX_CONNECTIONS": "1900", - } - for name in ( - "PRISM_ALLOW_MEMORY_LEDGER", - "PRISM_ALLOW_TEST_SIGNING_SEEDS", - "PRISM_ALLOW_BUNDLE_EMBEDDED_LEDGER_KEY", - "PRISM_ALLOW_FIXED_LEDGER_SESSION_TOKEN", - ): - with self.subTest(name=name), patch.dict(os.environ, {**base, name: "1"}, clear=True): - with self.assertRaisesRegex(SystemExit, name): - validate_prism_production_gate() + accepted = server.submit_block_candidate( + block_candidate(server, state, submission) + ) - with patch.dict(os.environ, base, clear=True): - validate_prism_production_gate() + self.assertTrue(accepted) + self.assertEqual(server._payout_state_generation, 2) + self.assertEqual(server._published_payout_state.source_tip_hash, newer_tip) + self.assertEqual(server.payout_state_candidates_discarded, 0) + self.assertEqual(server.reorg_reconcile_skip_count, 1) - with patch.dict( - os.environ, - {**base, "PRISM_STRATUM_MAX_CONNECTIONS": "0"}, - clear=True, - ): - with self.assertRaisesRegex(SystemExit, "PRISM_STRATUM_MAX_CONNECTIONS"): - validate_prism_production_gate() - - with patch.dict( - os.environ, - {**base, "PRISM_STRATUM_INITIAL_JOB_TIMEOUT_SECONDS": "0"}, - clear=True, - ): - with self.assertRaisesRegex( - SystemExit, - "PRISM_STRATUM_INITIAL_JOB_TIMEOUT_SECONDS", - ): - validate_prism_production_gate() - - with patch.dict( - os.environ, - {**base, "PRISM_STRATUM_MAX_PENDING_INITIAL_JOBS": "0"}, - clear=True, - ): - with self.assertRaisesRegex( - SystemExit, - "PRISM_STRATUM_MAX_PENDING_INITIAL_JOBS", - ): - validate_prism_production_gate() - - with patch.dict( - os.environ, - {**base, "QBIT_CHAIN": "mainnet", "PRISM_STRATUM_STALE_GRACE_SECONDS": "3"}, - clear=True, - ): - with self.assertRaisesRegex(SystemExit, "mainnet requires PRISM_STRATUM_STALE_GRACE_SECONDS=0"): - validate_prism_production_gate() - - # Off mainnet, production mode accepts a bounded grace window. - with patch.dict( - os.environ, - {**base, "PRISM_STRATUM_STALE_GRACE_SECONDS": "2"}, - clear=True, - ): - validate_prism_production_gate() - - with patch.dict(os.environ, {**base, "PRISM_POSTGRES_PASSWORD": "change-this"}, clear=True): - with self.assertRaisesRegex(SystemExit, "PRISM_POSTGRES_PASSWORD"): - validate_prism_production_gate() - - with patch.dict( - os.environ, - { - **base, - "PRISM_POSTGRES_PASSWORD": "not-default", - "PRISM_POSTGRES_PSQL_COMMAND": "", - "PRISM_DATABASE_URL": "postgresql://qbit:change-this@prism-postgres:5432/qbit", - }, - clear=True, - ): - with self.assertRaisesRegex(SystemExit, "PRISM_DATABASE_URL"): - validate_prism_production_gate() - - def test_production_gate_rejects_unsafe_difficulty_without_capacity_gate(self) -> None: - base = { - "QBIT_PRODUCTION": "1", - "QBIT_RPC_USER": "qbitrpc", - "QBIT_RPC_PASSWORD": "not-default", - "PRISM_POSTGRES_PSQL_COMMAND": "psql postgresql://example.invalid/qbit", - "PRISM_POSTGRES_PASSWORD": "not-default", - "PRISM_MANIFEST_SIGNING_SEED_HEX": "42" * 32, - "PRISM_LEDGER_ATTESTATION_SIGNING_SEED_HEX": "43" * 32, - "PRISM_LEDGER_WRITER_PUBLIC_KEY_HEX": "44" * 32, - "PRISM_LEDGER_WRITER_ID": "managed-writer", - "PRISM_LEDGER_WRITER_EPOCH": "7", - "PRISM_AUDIT_DIR": "/var/lib/qbit/prism/audit", - "PRISM_EVIDENCE_PATH": "/var/lib/qbit/prism/evidence.json", - "PRISM_STRATUM_STALE_GRACE_SECONDS": "0", - "PRISM_STRATUM_SHARE_DIFF": "1024", - "PRISM_STRATUM_VARDIFF_MIN_DIFF": "1024", - "PRISM_STRATUM_VARDIFF_START_DIFF": "4096", - "PRISM_STRATUM_VARDIFF_MAX_DIFF": "65536", - } - cases = ( - ({"PRISM_STRATUM_SHARE_DIFF": ""}, "requires PRISM_STRATUM_SHARE_DIFF"), - ({"PRISM_STRATUM_SHARE_DIFF": "not-a-decimal"}, "must be a decimal number"), - ({"PRISM_STRATUM_SHARE_DIFF": "NaN"}, "PRISM_STRATUM_SHARE_DIFF must be positive"), - ({"PRISM_STRATUM_SHARE_DIFF": "0"}, "PRISM_STRATUM_SHARE_DIFF must be positive"), - ({"PRISM_STRATUM_SHARE_DIFF": "1e-9"}, "lab-only 1e-9 difficulty"), - ({"PRISM_STRATUM_VARDIFF_MIN_DIFF": "8192"}, "minimum exceeds its start"), - ({"PRISM_STRATUM_VARDIFF_START_DIFF": "131072"}, "start exceeds its maximum"), - ) - for overrides, message in cases: - with self.subTest(overrides=overrides), patch.dict( - os.environ, - {**base, **overrides}, - clear=True, - ): - with self.assertRaisesRegex(SystemExit, message): - validate_prism_production_gate() - - def test_mainnet_implies_production_gate(self) -> None: - with patch.dict( - os.environ, - { - "QBIT_CHAIN": "mainnet", - "PRISM_STRATUM_STALE_GRACE_SECONDS": "0", - }, - clear=True, - ): - with self.assertRaisesRegex(SystemExit, "requires PRISM_STRATUM_SHARE_DIFF"): - validate_prism_production_gate() - - def test_compatibility_production_flag_implies_production_gate(self) -> None: - with patch.dict( - os.environ, - { - "QBIT_CHAIN": "testnet4", - "QBIT_TOOLS_PRODUCTION": "1", - "PRISM_STRATUM_STALE_GRACE_SECONDS": "0", - }, - clear=True, - ): - with self.assertRaisesRegex(SystemExit, "requires PRISM_STRATUM_SHARE_DIFF"): - validate_prism_production_gate() - - def test_mainnet_ctv_requires_static_fee_rate_before_runtime_startup(self) -> None: - env = { - "QBIT_CHAIN": "mainnet", - "QBIT_RPC_USER": "qbitrpc", - "QBIT_RPC_PASSWORD": "not-default", - "PRISM_POSTGRES_PSQL_COMMAND": "psql postgresql://example.invalid/qbit", - "PRISM_POSTGRES_PASSWORD": "not-default", - "PRISM_MANIFEST_SIGNING_SEED_HEX": "42" * 32, - "PRISM_LEDGER_ATTESTATION_SIGNING_SEED_HEX": "43" * 32, - "PRISM_LEDGER_WRITER_PUBLIC_KEY_HEX": "44" * 32, - "PRISM_LEDGER_WRITER_ID": "managed-writer", - "PRISM_LEDGER_WRITER_EPOCH": "7", - "PRISM_AUDIT_DIR": "/var/lib/qbit/prism/audit", - "PRISM_EVIDENCE_PATH": "/var/lib/qbit/prism/evidence.json", - "PRISM_STRATUM_STALE_GRACE_SECONDS": "0", - "PRISM_STRATUM_SHARE_DIFF": "1024", - "PRISM_STRATUM_VARDIFF_MIN_DIFF": "1024", - "PRISM_STRATUM_VARDIFF_START_DIFF": "4096", - "PRISM_STRATUM_VARDIFF_MAX_DIFF": "65536", - "PRISM_CTV_SETTLEMENT_ENABLED": "1", - } - with ( - patch.dict(os.environ, env, clear=True), - self.assertRaisesRegex( - SystemExit, - "PRISM_CTV_FANOUT_FEE_MARKET_RATE_BITS_PER_1000_WEIGHT", - ), - ): - validate_prism_production_gate() - - def test_live_chain_identity_accepts_main_alias_and_pinned_genesis(self) -> None: - genesis = "12" * 32 - server = PrismCoordinator.__new__(PrismCoordinator) - server.qbit_chain = "mainnet" - - class Rpc: - def call(self, method: str, params: object = None) -> object: - if method == "getblockchaininfo": - return { - "chain": "main", - "initialblockdownload": False, - "blocks": 100, - "headers": 100, - } - if method == "getnetworkinfo": - return {"connections": 2} - if method == "getblockhash" and params == [0]: - return genesis - raise RuntimeError(method) - - server.rpc = Rpc() - with patch.dict(os.environ, {"QBIT_EXPECTED_GENESIS_HASH": genesis}, clear=True): - server.validate_live_chain_identity() - - def test_live_chain_identity_rejects_incomplete_public_readiness(self) -> None: - server = PrismCoordinator.__new__(PrismCoordinator) - server.qbit_chain = "mainnet" - genesis = "12" * 32 - - cases = ( - ({"chain": "main", "blocks": 10, "headers": 10}, {"connections": 1}, "initial block"), - ( - {"chain": "main", "initialblockdownload": False, "blocks": 9, "headers": 10}, - {"connections": 1}, - "not caught up", - ), - ( - {"chain": "main", "initialblockdownload": False, "blocks": 10, "headers": 10}, - {"connections": 0}, - "requires at least 1", - ), + def _superseded_confirm_coordinator( + self, + confirmed_count: int, + ) -> tuple[PrismCoordinator, object, object, str, list[object]]: + server, state, ledger = submit_coordinator() + server._ensure_job_cache_state() + block_hash = "d3" * 32 + server.rpc = SubmitRpc( + tip="00" * 32, + block_hash=block_hash, + ledger=ledger, ) - for blockchain_info, network_info, message in cases: - with self.subTest(message=message): - server.rpc = SimpleNamespace( - call=lambda method, params=None: ( - blockchain_info - if method == "getblockchaininfo" - else network_info - if method == "getnetworkinfo" - else genesis - ) - ) - with ( - patch.dict(os.environ, {"QBIT_EXPECTED_GENESIS_HASH": genesis}, clear=True), - self.assertRaisesRegex(RuntimeError, message), - ): - server.validate_live_chain_identity() - - def test_live_template_preflight_enforces_freshness_and_relay_fee_floor(self) -> None: - server = PrismCoordinator.__new__(PrismCoordinator) - previous_hash = "34" * 32 - template = {"height": 1, "curtime": int(time.time()), "previousblockhash": previous_hash} - server.current_template_artifacts = lambda: SimpleNamespace( - template=template, - previousblockhash=previous_hash, + server.build_audit_bundle = ( # type: ignore[method-assign] + lambda **_kwargs: verified_block_bundle() ) - server.rpc = SimpleNamespace( - call=lambda method, params=None: { - "minrelaytxfee": "0.00001000", - "mempoolminfee": "0.00001000", + server.verify_bundle = ( # type: ignore[method-assign] + lambda *_args, **_kwargs: verified_audit_report() + ) + ledger.confirm_accepted_block = ( # type: ignore[method-assign] + lambda **_kwargs: { + "backend": "fake", + "confirmed_count": confirmed_count, } ) - server._ctv_fanout_market_fee_rate_cache = {} - - enabled = { - "PRISM_CTV_SETTLEMENT_ENABLED": "1", - "PRISM_CTV_FANOUT_FEE_MARKET_RATE_BITS_PER_1000_WEIGHT": "1000", - "PRISM_TEMPLATE_MAX_AGE_SECONDS": "120", - } - with patch.dict(os.environ, enabled, clear=True): - server.validate_live_template_and_fee_policy() - - server._ctv_fanout_market_fee_rate_cache = {} - with ( - patch.dict( - os.environ, - {**enabled, "PRISM_CTV_FANOUT_FEE_MARKET_RATE_BITS_PER_1000_WEIGHT": "1"}, - clear=True, - ), - self.assertRaisesRegex(RuntimeError, "below the connected node relay floor"), - ): - server.validate_live_template_and_fee_policy() - - template["curtime"] = int(time.time()) - 121 - with ( - patch.dict(os.environ, enabled, clear=True), - self.assertRaisesRegex(RuntimeError, "block template is stale"), - ): - server.validate_live_template_and_fee_policy() - - def test_template_refresh_failure_budget_starts_at_first_failure(self) -> None: - server = PrismCoordinator.__new__(PrismCoordinator) - server.template_refresh_failure_exit_seconds = 120 - server.last_successful_template_refresh_monotonic = 100.0 - - server._record_template_refresh_failure(500.0) - self.assertFalse(server.template_refresh_failure_expired(500.0)) - self.assertEqual(server.template_refresh_failure_started_monotonic, 500.0) - self.assertFalse(server.template_refresh_failure_expired(619.999)) - self.assertTrue(server.template_refresh_failure_expired(620.0)) - - def test_disabled_template_refresh_failure_budget_does_not_start_or_expire(self) -> None: - server = PrismCoordinator.__new__(PrismCoordinator) - server.template_refresh_failure_exit_seconds = 0 - - server._record_template_refresh_failure(500.0) - self.assertFalse(server.template_refresh_failure_expired(500.0)) - self.assertFalse(hasattr(server, "template_refresh_failure_started_monotonic")) - - def test_coordination_blocked_refresh_does_not_start_failure_budget(self) -> None: - for blocked_error in ( - TemplateRefreshSuperseded("qbit tip changed during sequential refresh"), - _PayoutStatePublicationBlocked( - "accepted block payout confirmation is still pending" - ), - ): - with self.subTest(blocked=type(blocked_error).__name__): - server = coordinator() - server.template_refresh_failure_exit_seconds = 10.0 - server.coordination_blocked_exit_seconds = 30.0 - server._record_heartbeat = lambda _name: None # type: ignore[method-assign] - server.rpc = TipRpc("11" * 32) - - def raise_blocked(error: Exception = blocked_error) -> QbitTipTemplateSnapshot: - raise error - - server.fetch_qbit_tip_template_snapshot = raise_blocked # type: ignore[method-assign] - with ( - patch( - "lab.prism.prism_coordinator.time.monotonic", - return_value=100.0, - ), - self.assertRaises(type(blocked_error)), - ): - server.poll_qbit_tip_template_once() - - self.assertIsNone( - getattr(server, "template_refresh_failure_started_monotonic", None) - ) - self.assertFalse(server.template_refresh_failure_expired(10_000.0)) - self.assertAlmostEqual( - server.coordination_blocked_streak_age_seconds(109.999), - 9.999, - ) - self.assertFalse( - server.coordination_blocked_streak_expired(129.999) - ) - - def test_coordination_blocked_default_budget_is_fifteen_minutes(self) -> None: - server = PrismCoordinator.__new__(PrismCoordinator) - server._record_coordination_blocked_refresh(100.0) - - self.assertFalse(server.coordination_blocked_streak_expired(999.999)) - self.assertTrue(server.coordination_blocked_streak_expired(1000.0)) - - def test_non_coordination_blocked_refresh_still_starts_failure_budget(self) -> None: - # Plain TemplateRefreshBlocked also wraps genuine failures (malformed - # template artifacts, job builds failing, untrusted chain views); only - # the TemplateRefreshSuperseded/payout-fence subclasses are exempt. - server = coordinator() - server.template_refresh_failure_exit_seconds = 10.0 - server.coordination_blocked_exit_seconds = 10.0 - server._record_coordination_blocked_refresh(90.0) - server._record_heartbeat = lambda _name: None # type: ignore[method-assign] - server.rpc = TipRpc("11" * 32) + shutdown_requests: list[object] = [] + real_shutdown = server.request_shutdown + + def observed_shutdown(signum: int | None = None) -> None: + shutdown_requests.append(signum) + real_shutdown(signum) + + server.request_shutdown = observed_shutdown # type: ignore[method-assign] + return server, state, ledger, block_hash, shutdown_requests + + def test_superseded_ledger_confirmation_abandons_without_shutdown(self) -> None: + # A confirm disposition of -1 means the row was terminally disposed + # (reorg quarantine, rejection, or reversal) before the confirmation + # landed: terminal for the candidate, benign for the pool. The + # coordinator must abandon the candidate and keep serving instead of + # converting the routine race into a full-pool outage. + server, state, ledger, block_hash, shutdown_requests = ( + self._superseded_confirm_coordinator(-1) + ) + # The definitive submitblock ack normally defers terminal + # abandonment; age that evidence out immediately to model the + # settled post-reorg view in which the chain no longer carries the + # candidate hash. + server.observed_tip_accept_window_seconds = 1e-9 + submission = SimpleNamespace( + coinbase_tx_hex="c0ffee", + block_hash_hex=block_hash, + block_hex="00", + ) - def raise_blocked() -> QbitTipTemplateSnapshot: - raise TemplateRefreshBlocked( - "unable to derive exact artifacts for observed qbit template" + with tempfile.TemporaryDirectory() as tempdir: + server.audit_dir = Path(tempdir) + server.evidence_path = Path(tempdir) / "evidence.json" + server.ledger_writer_public_key_hex = "aa" * 32 + accepted = server.submit_block_candidate( + block_candidate(server, state, submission) ) - server.fetch_qbit_tip_template_snapshot = raise_blocked # type: ignore[method-assign] - with ( - patch("lab.prism.prism_coordinator.time.monotonic", return_value=100.0), - self.assertRaises(TemplateRefreshBlocked), - ): - server.poll_qbit_tip_template_once() + self.assertFalse(accepted) + self.assertEqual(len(ledger.persisted), 1) + self.assertEqual( + server._block_candidate_outcome.reason, + PRISM_REJECTION_LEDGER_CONFIRMATION_SUPERSEDED, + ) + self.assertEqual(shutdown_requests, []) + self.assertFalse(server.stop_event.is_set()) + self.assertEqual(server.accepted_block_count, 0) + # The prospective preview was withdrawn with publication + # invalidation, exactly as on the fail-closed path. + transition = server._accepted_block_payout_previews.get(block_hash) + self.assertTrue(transition is None or transition.preview is None) - self.assertEqual(server.template_refresh_failure_started_monotonic, 100.0) - self.assertEqual(server.coordination_blocked_streak_age_seconds(110.0), 0.0) - self.assertTrue(server.template_refresh_failure_expired(110.0)) + def test_superseded_confirmation_with_fresh_acceptance_evidence_defers( + self, + ) -> None: + # While first-party acceptance evidence (the definitive submitblock + # ack) is still fresh, even a superseded disposition must not commit + # a terminal abandonment: the reorg reconciler may still reactivate + # the row. The candidate defers -- and the pool keeps serving. + server, state, _ledger, block_hash, shutdown_requests = ( + self._superseded_confirm_coordinator(-1) + ) + submission = SimpleNamespace( + coinbase_tx_hex="c0ffee", + block_hash_hex=block_hash, + block_hex="00", + ) - def test_sustained_blocked_refresh_storm_exits_via_coordination_budget(self) -> None: - server = coordinator() - server.blockpoll_seconds = 0 - server.template_refresh_failure_exit_seconds = 10.0 - server.coordination_blocked_exit_seconds = 10.0 - server._record_heartbeat = lambda _name: None # type: ignore[method-assign] - server.rpc = TipRpc("11" * 32) - clock = {"now": 100.0} - blocked_polls = 0 - - def blocked_fetch() -> QbitTipTemplateSnapshot: - nonlocal blocked_polls - blocked_polls += 1 - clock["now"] += 6.0 - if blocked_polls >= 4: - server.stop_event.set() - if blocked_polls % 2: - raise TemplateRefreshSuperseded( - "qbit tip changed during sequential refresh; immediate retry scheduled" - ) - raise _PayoutStatePublicationBlocked( - "payout state invalidation is pending publication" + with tempfile.TemporaryDirectory() as tempdir: + server.audit_dir = Path(tempdir) + server.evidence_path = Path(tempdir) / "evidence.json" + server.ledger_writer_public_key_hex = "aa" * 32 + accepted = server.submit_block_candidate( + block_candidate(server, state, submission) ) - server.fetch_qbit_tip_template_snapshot = blocked_fetch # type: ignore[method-assign] - with ( - patch( - "lab.prism.prism_coordinator.time.monotonic", - side_effect=lambda: clock["now"], - ), - patch("lab.prism.prism_coordinator.traceback.print_exc"), - ): - server.blockpoll_loop() - - self.assertEqual(blocked_polls, 4) - self.assertTrue(server.coordination_blocked_streak_expired(clock["now"])) - self.assertIsNone( - getattr(server, "template_refresh_failure_started_monotonic", None) + self.assertFalse(accepted) + self.assertEqual( + server._block_candidate_outcome.reason, + PRISM_REJECTION_BLOCK_ACCEPT_PENDING, ) - server.stop_event.clear() - server.watchdog_enabled = False - server.watchdog_interval_seconds = 0.001 - with ( - patch( - "lab.prism.prism_coordinator.time.monotonic", - return_value=clock["now"], - ), - patch( - "lab.prism.prism_coordinator.os._exit", - side_effect=SystemExit(1), - ) as exit_process, - patch("builtins.print"), - self.assertRaises(SystemExit), - ): - server.watchdog_loop() + self.assertEqual(shutdown_requests, []) + self.assertFalse(server.stop_event.is_set()) - exit_process.assert_called_once_with(1) + def test_unexplained_zero_ledger_confirmation_still_requests_shutdown( + self, + ) -> None: + # A 0 with no superseding row remains unexplained: the fail-closed + # shutdown escalation is preserved exactly as before the disposition + # split. + server, state, _ledger, block_hash, shutdown_requests = ( + self._superseded_confirm_coordinator(0) + ) + submission = SimpleNamespace( + coinbase_tx_hex="c0ffee", + block_hash_hex=block_hash, + block_hex="00", + ) - def test_armed_budget_is_not_fired_by_coordination_blocked_refresh(self) -> None: - # A transient budgeted failure armed the clock, qbitd recovered, and - # only coordination churn follows. Blocked attempts must not trip the - # armed budget in blockpoll_loop: the exit is reserved for the next - # budgeted failure, and the clock clears on the next completed refresh. - server = coordinator() - server.blockpoll_seconds = 0 - server.template_refresh_failure_exit_seconds = 10.0 - server.template_refresh_failure_started_monotonic = 100.0 - server._record_heartbeat = lambda _name: None # type: ignore[method-assign] - server.rpc = TipRpc("11" * 32) - clock = {"now": 100.0} - blocked_polls = 0 - - def blocked_fetch() -> QbitTipTemplateSnapshot: - nonlocal blocked_polls - blocked_polls += 1 - clock["now"] += 6.0 - if blocked_polls >= 4: - server.stop_event.set() - raise TemplateRefreshSuperseded( - "qbit tip changed during sequential refresh; immediate retry scheduled" + with tempfile.TemporaryDirectory() as tempdir: + server.audit_dir = Path(tempdir) + server.evidence_path = Path(tempdir) / "evidence.json" + server.ledger_writer_public_key_hex = "aa" * 32 + accepted = server.submit_block_candidate( + block_candidate(server, state, submission) ) - server.fetch_qbit_tip_template_snapshot = blocked_fetch # type: ignore[method-assign] - with ( - patch( - "lab.prism.prism_coordinator.time.monotonic", - side_effect=lambda: clock["now"], - ), - patch("lab.prism.prism_coordinator.traceback.print_exc"), - patch("lab.prism.prism_coordinator.os._exit", side_effect=SystemExit(1)) as exit_process, - ): - server.blockpoll_loop() - - self.assertEqual(blocked_polls, 4) - self.assertGreater( - clock["now"], - server.template_refresh_failure_started_monotonic - + server.template_refresh_failure_exit_seconds, + self.assertFalse(accepted) + self.assertEqual(len(shutdown_requests), 1) + self.assertTrue(server.stop_event.is_set()) + self.assertEqual( + server._block_candidate_outcome.reason, + PRISM_REJECTION_LEDGER_CONFIRMATION_FAILED, ) - exit_process.assert_not_called() - self.assertEqual(server.template_refresh_failure_started_monotonic, 100.0) - - def test_rpc_outage_arms_and_exhausts_failure_budget(self) -> None: - server = coordinator() - server.blockpoll_seconds = 0 - server.template_refresh_failure_exit_seconds = 10.0 - server._record_heartbeat = lambda _name: None # type: ignore[method-assign] - clock = {"now": 100.0} - - class OutageRpc: - def call(self, method: str, params: list[object] | None = None, **_kwargs: object) -> object: - clock["now"] += 6.0 - raise ConnectionError("qbitd unreachable") - - server.rpc = OutageRpc() - with ( - patch( - "lab.prism.prism_coordinator.time.monotonic", - side_effect=lambda: clock["now"], - ), - patch("lab.prism.prism_coordinator.traceback.print_exc"), - patch("lab.prism.prism_coordinator.os._exit", side_effect=SystemExit(1)) as exit_process, - self.assertRaises(SystemExit), - ): - server.blockpoll_loop() - - exit_process.assert_called_once_with(1) - self.assertIsNotNone(server.template_refresh_failure_started_monotonic) - - def test_persistent_blocked_template_derivation_arms_and_exhausts_failure_budget(self) -> None: - # Top-of-poll RPCs stay healthy, but every refresh fails with plain - # TemplateRefreshBlocked (e.g. malformed template artifacts, all job - # builds failing). A fresh process must still arm the budget from its - # first such failure and take the budgeted restart path. - server = coordinator() - server.blockpoll_seconds = 0 - server.template_refresh_failure_exit_seconds = 10.0 - server._record_heartbeat = lambda _name: None # type: ignore[method-assign] - server.rpc = TipRpc("11" * 32) - clock = {"now": 100.0} - - def blocked_fetch() -> QbitTipTemplateSnapshot: - clock["now"] += 6.0 - raise TemplateRefreshBlocked( - "unable to derive exact artifacts for observed qbit template" - ) - - server.fetch_qbit_tip_template_snapshot = blocked_fetch # type: ignore[method-assign] - with ( - patch( - "lab.prism.prism_coordinator.time.monotonic", - side_effect=lambda: clock["now"], - ), - patch("lab.prism.prism_coordinator.traceback.print_exc"), - patch("lab.prism.prism_coordinator.os._exit", side_effect=SystemExit(1)) as exit_process, - self.assertRaises(SystemExit), - ): - server.blockpoll_loop() - exit_process.assert_called_once_with(1) - self.assertIsNotNone(server.template_refresh_failure_started_monotonic) - - def test_healthy_noop_template_poll_resets_refresh_failure_clock(self) -> None: - server = coordinator() - server.blockpoll_seconds = 0 - server.last_successful_template_refresh_monotonic = 100.0 - server.template_refresh_failure_started_monotonic = 190.0 - server.template_refresh_failure_exit_seconds = 10.0 - server.coordination_blocked_exit_seconds = 10.0 - server._record_coordination_blocked_refresh(190.0) - server._record_heartbeat = lambda _name: None # type: ignore[method-assign] - snapshot = QbitTipTemplateSnapshot( - bestblockhash="11" * 32, - previousblockhash="11" * 32, - template_fingerprint="22" * 32, - ) - server.tip_template_snapshot = snapshot - server.rpc = TipRpc(snapshot.bestblockhash) - server.fetch_qbit_tip_template_snapshot = lambda: snapshot # type: ignore[method-assign] - - def trusted_chain_view(_tip: str) -> bool: - server.stop_event.set() - return True - - server.ensure_reorg_reconciled_for_tip = trusted_chain_view # type: ignore[method-assign] - with patch("lab.prism.prism_coordinator.time.monotonic", return_value=200.0): - server.blockpoll_loop() + def test_verified_canonical_bundle_path_requires_exact_bytes(self) -> None: + with tempfile.TemporaryDirectory() as tempdir: + candidate_path = Path(tempdir) / "candidate.json" + candidate_bytes = b'{"canonical":true}' + candidate_path.write_bytes(candidate_bytes) + canonical_sha256 = hashlib.sha256(candidate_bytes).hexdigest() - self.assertEqual(server.last_successful_template_refresh_monotonic, 200.0) - self.assertIsNone(server.template_refresh_failure_started_monotonic) - self.assertEqual(server.coordination_blocked_streak_age_seconds(300.0), 0.0) - self.assertFalse(server.coordination_blocked_streak_expired(300.0)) - self.assertFalse(server.template_refresh_failure_expired(300.0)) - self.assertIsNone(server.template_refresh_failure_started_monotonic) - server._record_template_refresh_failure(300.0) - self.assertEqual(server.template_refresh_failure_started_monotonic, 300.0) + self.assertEqual( + PrismCoordinator.verified_canonical_bundle_path( + candidate_path, + {"audit_bundle_sha256_hex": canonical_sha256.upper()}, + ), + candidate_path, + ) + self.assertIsNone( + PrismCoordinator.verified_canonical_bundle_path( + candidate_path, + {"audit_bundle_sha256_hex": "00" * 32}, + ) + ) - def test_shared_template_poll_records_success_for_blockwait_callers(self) -> None: - server = coordinator() - server.last_successful_template_refresh_monotonic = 100.0 - snapshot = QbitTipTemplateSnapshot( - bestblockhash="11" * 32, - previousblockhash="11" * 32, - template_fingerprint="22" * 32, + def test_active_ancestor_candidate_resumes_full_finalization_without_resubmit(self) -> None: + server, state, ledger = submit_coordinator() + server.max_blocks = 10 + server.stop_after_block = False + refreshes: list[str] = [] + server.refresh_jobs_after_accepted_block = ( # type: ignore[method-assign] + lambda **kwargs: refreshes.append(str(kwargs["block_hash"])) ) - server.rpc = TipRpc(snapshot.bestblockhash) - server.fetch_qbit_tip_template_snapshot = lambda: snapshot # type: ignore[method-assign] - server.ensure_reorg_reconciled_for_tip = lambda _tip: True # type: ignore[method-assign] + with tempfile.TemporaryDirectory() as tempdir: + server.audit_dir = Path(tempdir) + server.evidence_path = Path(tempdir) / "evidence.json" + server.ledger_writer_public_key_hex = "aa" * 32 + block_hash = "ac" * 32 - with patch("lab.prism.prism_coordinator.time.monotonic", return_value=200.0): - refreshed = server.poll_qbit_tip_template_once(heartbeat_name="qbit_blockwait") + class ActiveAncestorRpc: + def __init__(self) -> None: + self.revalidated_heights: list[int] = [] - self.assertEqual(refreshed, 0) - self.assertEqual(server.last_successful_template_refresh_monotonic, 200.0) + def call(self, method: str, params: object = None) -> object: + if method == "getbestblockhash": + return "ef" * 32 + if method == "getblockheader": + self.assert_candidate(params) + return {"height": 10, "confirmations": 2} + if method == "getblockcount": + return 11 + if method == "getblockhash": + if params != [10]: + raise AssertionError(params) + self.revalidated_heights.append(10) + return block_hash + if method == "submitblock": + raise AssertionError("active ancestor must not be resubmitted") + raise RuntimeError(method) - def test_untrusted_reconciliation_exhausts_template_refresh_failure_budget(self) -> None: - server = coordinator() - server.blockpoll_seconds = 0 - server.last_successful_template_refresh_monotonic = 100.0 - server.template_refresh_failure_started_monotonic = 100.0 - server.template_refresh_failure_exit_seconds = 10.0 - server._record_heartbeat = lambda _name: None # type: ignore[method-assign] - snapshot = QbitTipTemplateSnapshot( - bestblockhash="11" * 32, - previousblockhash="11" * 32, - template_fingerprint="22" * 32, - ) - server.rpc = TipRpc(snapshot.bestblockhash) - server.fetch_qbit_tip_template_snapshot = lambda: snapshot # type: ignore[method-assign] - server.ensure_reorg_reconciled_for_tip = lambda _tip: False # type: ignore[method-assign] - - with ( - patch("lab.prism.prism_coordinator.time.monotonic", return_value=110.0), - patch("lab.prism.prism_coordinator.traceback.print_exc"), - patch("lab.prism.prism_coordinator.os._exit", side_effect=SystemExit(1)) as exit_process, - self.assertRaises(SystemExit), - ): - server.blockpoll_loop() + @staticmethod + def assert_candidate(params: object) -> None: + if params != [block_hash]: + raise AssertionError(params) - exit_process.assert_called_once_with(1) - self.assertEqual(server.last_successful_template_refresh_monotonic, 100.0) + active_rpc = ActiveAncestorRpc() + server.rpc = active_rpc + server.ensure_reorg_reconciled_for_tip = ( # type: ignore[method-assign] + lambda _tip, *, _coalesce_same_tip: not _coalesce_same_tip + ) + server.build_audit_bundle = lambda **_kwargs: { # type: ignore[method-assign] + "found_block": {"coinbase_value_sats": 50_00000000}, + "ledger_window_attestation": {"signature": {"public_key_hex": "aa" * 32}}, + "payout_policy_manifest": {"accounts": []}, + "signed_coinbase_manifest": { + "manifest": {"coinbase_tx_hex": "c0ffee", "payout_count": 1} + }, + } + server.verify_bundle = ( # type: ignore[method-assign] + lambda *_args, **_kwargs: verified_audit_report() + ) + submission = SimpleNamespace( + coinbase_tx_hex="c0ffee", + block_hash_hex=block_hash, + block_hex="00", + ) + candidate = block_candidate(server, state, submission) - def test_all_refresh_job_builds_failing_exhausts_failure_budget(self) -> None: - old_tip = "00" * 32 - new_tip = "33" * 32 - server = coordinator() - server.blockpoll_seconds = 0 - server.last_successful_template_refresh_monotonic = 100.0 - server.template_refresh_failure_started_monotonic = 100.0 - server.template_refresh_failure_exit_seconds = 10.0 - server._record_heartbeat = lambda _name: None # type: ignore[method-assign] - state = client() - state.username = "miner-a" - state.worker = worker_identity() - state.active_job = prism_context("old-job", old_tip, worker=state.worker) - state.active_job_ids = {"old-job"} - server.clients = {state} - server.jobs = {"old-job": state.active_job} - server.tip_template_snapshot = QbitTipTemplateSnapshot( - bestblockhash=old_tip, - previousblockhash=old_tip, - template_fingerprint=qbit_template_fingerprint(state.active_job.template), - ) - snapshot = QbitTipTemplateSnapshot( - bestblockhash=new_tip, - previousblockhash=new_tip, - template_fingerprint="44" * 32, - ) - server.rpc = TipRpc(new_tip) - server.fetch_qbit_tip_template_snapshot = lambda: snapshot # type: ignore[method-assign] - server.build_job_for_client = lambda *_args, **_kwargs: (_ for _ in ()).throw( # type: ignore[method-assign] - RuntimeError("template build unavailable") - ) + self.assertTrue(server.submit_block_candidate(candidate)) + # Direct finalization records the refresh for its caller so slow + # job fanout happens after writer admission has been released. + self.assertEqual( + state.post_accept_refresh_block, + (10, block_hash), + ) + server.refresh_jobs_after_pending_accepted_block( + state, + heartbeat_name="block_submitter", + ) + # A failed outbox terminal update can replay A after later block B + # changes global balances. Validate A against its height-bounded + # active-chain view and suppress duplicate process-local accounting. + ledger.durable_payout_state = True + ledger.prior_balances = [ + { + "recipient_id": "later-miner", + "order_key": "later-miner", + "p2mr_program_hex": "22" * 32, + "balance_sats": 50, + } + ] + ledger.pool_block_state = lambda **_kwargs: { # type: ignore[attr-defined] + "block_hash": block_hash, + "block_height": 10, + "parent_hash": "00" * 32, + "chain_state": "confirmed", + "maturity_state": "immature", + } + historical_reads: list[dict[str, object]] = [] + ledger.prior_balances_after_pool_block = ( # type: ignore[attr-defined] + lambda **kwargs: historical_reads.append(dict(kwargs)) or [] + ) + self.assertTrue(server.submit_block_candidate(candidate)) + # The confirmed-ancestor replay keeps its height-bounded + # active-chain view (issue #209 leaves replay behavior intact). + self.assertEqual(historical_reads, [{"block_hash": block_hash}]) + latest_evidence = server.latest_evidence - with ( - patch("lab.prism.prism_coordinator.time.monotonic", return_value=110.0), - patch("lab.prism.prism_coordinator.traceback.print_exc"), - patch("lab.prism.prism_coordinator.os._exit", side_effect=SystemExit(1)) as exit_process, - self.assertRaises(SystemExit), - ): - server.blockpoll_loop() - - exit_process.assert_called_once_with(1) - self.assertEqual(server.job_build_failure_count, 1) - self.assertEqual(server.last_successful_template_refresh_monotonic, 100.0) - - def test_guarded_sequential_build_supersession_does_not_arm_failure_budget(self) -> None: - # Non-ready/collection mode: the sequential loop's guarded client - # build detects a superseded snapshot inside maybe_send_job. That is - # coordination churn, not template unhealthiness -- the real raise - # site must carry TemplateRefreshSuperseded so sustained pre-ready - # churn neither arms nor fires the budget. - old_tip = "00" * 32 - new_tip = "33" * 32 - server = coordinator() - server.blockpoll_seconds = 0 - server.template_refresh_failure_exit_seconds = 10.0 - server._record_heartbeat = lambda _name: None # type: ignore[method-assign] - state = client() - state.username = "miner-a" - state.worker = worker_identity() - state.active_job = prism_context("old-job", old_tip, worker=state.worker) - state.active_job_ids = {"old-job"} - server.clients = {state} - server.jobs = {"old-job": state.active_job} - server.tip_template_snapshot = QbitTipTemplateSnapshot( - bestblockhash=old_tip, - previousblockhash=old_tip, - template_fingerprint=qbit_template_fingerprint(state.active_job.template), - ) - snapshot = QbitTipTemplateSnapshot( - bestblockhash=new_tip, - previousblockhash=new_tip, - template_fingerprint="44" * 32, + self.assertEqual([row["block_hash"] for row in ledger.persisted], [block_hash] * 2) + # This compatibility builder ignores canonical_output_path, so its + # Python fallback remains verifier-only. + self.assertIsNone(ledger.persisted[0]["canonical_bundle_path"]) + self.assertEqual(ledger.confirmed[0]["active_tip_height"], 10) + self.assertEqual(ledger.confirmed[0]["block_hash"], block_hash) + self.assertEqual(active_rpc.revalidated_heights, [10, 10]) + self.assertFalse(ledger.confirmed[0]["submit_seen_at_confirm"]) + # The share credit happens on the client thread at submit time now; + # the block path itself appends nothing. + self.assertEqual(len(ledger.pending), 0) + self.assertFalse(server.stop_event.is_set()) + self.assertEqual(server.accepted_block_count, 1) + self.assertEqual(refreshes, [block_hash]) + self.assertIsNotNone(latest_evidence) + assert latest_evidence is not None + self.assertEqual(latest_evidence["persistence"]["block_count"], 1) + self.assertEqual(latest_evidence["confirmation"]["confirmed_count"], 1) + # Evidence carries an aggregate miner count, not a materialized list of + # every miner id (which scanned the whole ledger twice under the lock). + self.assertEqual(latest_evidence["accepted_share_count"], 0) + self.assertEqual(latest_evidence["distinct_miner_count"], 0) + self.assertNotIn("distinct_miners", latest_evidence) + + def test_fresh_confirmation_never_reads_historical_balances(self) -> None: + # Issue #209: a fresh confirmation lands as the ledger's newest + # confirmed pool block, so its post-confirm validation reads the + # O(recipients) current-balance summary. The O(history) height-bounded + # reader -- whose carry-table scan once held payout serialization past + # the parent-preview wait during back-to-back blocks -- is replay-only. + server, state, ledger = submit_coordinator() + server._ensure_job_cache_state() + server.max_blocks = 2 + server.stop_after_block = False + ledger.durable_payout_state = True + block_hash = "e3" * 32 + historical_reads: list[dict[str, object]] = [] + # A poisoned historical view: touching it on the fresh path would + # fail the post-confirm comparison and force shutdown. + ledger.prior_balances_after_pool_block = ( # type: ignore[attr-defined] + lambda **kwargs: historical_reads.append(dict(kwargs)) + or [ + { + "recipient_id": "history-miner", + "order_key": "history-miner", + "p2mr_program_hex": "33" * 32, + "balance_sats": 999, + } + ] ) - server.rpc = TipRpc(new_tip) - clock = {"now": 100.0} - fetches = 0 - - def fetch_snapshot() -> QbitTipTemplateSnapshot: - nonlocal fetches - fetches += 1 - clock["now"] += 6.0 - if fetches >= 4: - server.stop_event.set() - return snapshot + current_reads = 0 + original_current = ledger.current_prior_balances - server.fetch_qbit_tip_template_snapshot = fetch_snapshot # type: ignore[method-assign] - # The guarded pre-build currency check loses the race on every pass. - server._tip_refresh_snapshot_current_locked = ( # type: ignore[method-assign] - lambda *_args, **_kwargs: False - ) + def counting_current_prior_balances() -> list[dict[str, object]]: + nonlocal current_reads + current_reads += 1 + return original_current() - with ( - patch( - "lab.prism.prism_coordinator.time.monotonic", - side_effect=lambda: clock["now"], - ), - patch("lab.prism.prism_coordinator.traceback.print_exc"), - patch("lab.prism.prism_coordinator.os._exit", side_effect=SystemExit(1)) as exit_process, - ): - server.blockpoll_loop() + ledger.current_prior_balances = counting_current_prior_balances # type: ignore[method-assign] + with tempfile.TemporaryDirectory() as tempdir: + server.audit_dir = Path(tempdir) + server.evidence_path = Path(tempdir) / "evidence.json" + server.ledger_writer_public_key_hex = "aa" * 32 + server.rpc = SubmitRpc( + tip="00" * 32, + block_hash=block_hash, + ledger=ledger, + ) + server.build_audit_bundle = ( # type: ignore[method-assign] + lambda **_kwargs: verified_block_bundle() + ) + server.verify_bundle = ( # type: ignore[method-assign] + lambda *_args, **_kwargs: verified_audit_report() + ) + submission = SimpleNamespace( + coinbase_tx_hex="c0ffee", + block_hash_hex=block_hash, + block_hex="00", + ) - self.assertEqual(fetches, 4) - self.assertGreater(clock["now"] - 100.0, server.template_refresh_failure_exit_seconds) - exit_process.assert_not_called() - self.assertIsNone( - getattr(server, "template_refresh_failure_started_monotonic", None) - ) + self.assertTrue( + server.submit_block_candidate( + block_candidate(server, state, submission) + ) + ) + + self.assertEqual(historical_reads, []) + self.assertGreaterEqual(current_reads, 1) + self.assertEqual(len(ledger.confirmed), 1) + self.assertFalse(server.stop_event.is_set()) + self.assertEqual(server.accepted_block_count, 1) - def test_transient_template_refresh_failure_recovers_on_healthy_noop(self) -> None: + def test_audit_retention_prunes_only_live_and_candidate_files(self) -> None: server = coordinator() - server.blockpoll_seconds = 0 - server.last_successful_template_refresh_monotonic = 100.0 - server.template_refresh_failure_exit_seconds = 10.0 - server._record_heartbeat = lambda _name: None # type: ignore[method-assign] - poll_count = 0 - - def fail_then_noop() -> int: - nonlocal poll_count - poll_count += 1 - if poll_count == 1: - raise RuntimeError("transient RPC failure") - server.last_successful_template_refresh_monotonic = time.monotonic() - server.stop_event.set() - return 0 + with tempfile.TemporaryDirectory() as tempdir: + server.audit_dir = Path(tempdir) + server.audit_live_bundle_retention = 2 + server.audit_candidate_retention_seconds = 0 + for index in range(4): + path = Path(tempdir) / f"prism-live-audit-bundle-{index + 1}-{'aa' * 32}.json" + path.write_text("{}", encoding="utf-8") + os.utime(path, (100 + index, 100 + index)) + candidate = Path(tempdir) / f"prism-live-audit-bundle-candidate-{'bb' * 32}.json" + candidate.write_text("{}", encoding="utf-8") + temp_candidate = Path(tempdir) / f".prism-live-audit-bundle-candidate-{'bb' * 32}.json.tmp" + temp_candidate.write_text("{}", encoding="utf-8") + body = Path(tempdir) / f"prism-audit-bundle-body-{'cc' * 32}-{'dd' * 32}.json" + body.write_text("{}", encoding="utf-8") + segment = Path(tempdir) / f"prism-audit-share-segment-1-2-{'ee' * 32}.json" + segment.write_text("{}", encoding="utf-8") - server.poll_qbit_tip_template_once = fail_then_noop # type: ignore[method-assign] - with ( - patch("lab.prism.prism_coordinator.time.monotonic", side_effect=[105.0, 106.0]), - patch("lab.prism.prism_coordinator.traceback.print_exc"), - ): - server.blockpoll_loop() + # Live deletion is fail-closed unless durable evidence has supplied + # publication authority. This facade test supplies that authority + # explicitly while keeping its hand-written retention fixtures. + store = server._ensure_audit_artifact_store() + store._compatibility_evidence_override = True + store._evidence_state = "valid" - self.assertEqual(poll_count, 2) - self.assertEqual(server.last_successful_template_refresh_monotonic, 106.0) + server.prune_audit_artifacts() - def test_live_chain_identity_rejects_wrong_chain_or_genesis(self) -> None: - server = PrismCoordinator.__new__(PrismCoordinator) - server.qbit_chain = "mainnet" - server.rpc = SimpleNamespace( - call=lambda method, params=None: ( - {"chain": "regtest"} if method == "getblockchaininfo" else "34" * 32 + live_names = sorted(path.name for path in Path(tempdir).glob("prism-live-audit-bundle-[0-9]*.json")) + self.assertEqual( + live_names, + [ + f"prism-live-audit-bundle-3-{'aa' * 32}.json", + f"prism-live-audit-bundle-4-{'aa' * 32}.json", + ], ) - ) - with patch.dict(os.environ, {"QBIT_EXPECTED_GENESIS_HASH": "12" * 32}, clear=True): - with self.assertRaisesRegex(RuntimeError, "does not match RPC chain"): - server.validate_live_chain_identity() + self.assertFalse(candidate.exists()) + self.assertFalse(temp_candidate.exists()) + self.assertTrue(body.exists()) + self.assertTrue(segment.exists()) - server.rpc = SimpleNamespace( - call=lambda method, params=None: ( - {"chain": "main"} if method == "getblockchaininfo" else "34" * 32 - ) - ) - with patch.dict(os.environ, {"QBIT_EXPECTED_GENESIS_HASH": "12" * 32}, clear=True): - with self.assertRaisesRegex(RuntimeError, "does not match the connected"): - server.validate_live_chain_identity() + def test_audit_retention_zero_preserves_current_live_envelope(self) -> None: + server = coordinator() + with tempfile.TemporaryDirectory() as tempdir: + server.audit_dir = Path(tempdir) + server.audit_live_bundle_retention = 0 + old = Path(tempdir) / f"prism-live-audit-bundle-1-{'aa' * 32}.json" + old.write_text("{}", encoding="utf-8") + current = Path(tempdir) / f"prism-live-audit-bundle-2-{'bb' * 32}.json" + current.write_text("{}", encoding="utf-8") - def test_normal_accepted_share_does_not_close_client(self) -> None: - server, state, ledger = submit_coordinator() - submission = SimpleNamespace( - header_hex="aa" * 80, - block_hash_hex="bb" * 32, - share_pass=True, - block_pass=False, - ) + store = server._ensure_audit_artifact_store() + store._compatibility_evidence_override = True + store._evidence_state = "valid" + store._current_envelope = current.resolve() - with patch( - "lab.prism.prism_coordinator.direct_stratum.assemble_submission", - return_value=submission, - ): - should_close = server.handle_submit( - state, - ["miner-a", "job-1", "00" * 8, "00000001", "00000002"], - ) + server.prune_audit_artifacts(keep_live_path=current) - self.assertFalse(should_close) - self.assertEqual(len(ledger.pending), 1) - self.assertEqual(ledger.pending[0].share_id, "miner-a:" + "bb" * 32) + self.assertFalse(old.exists()) + self.assertTrue(current.exists()) - def test_prior_tip_share_inside_grace_is_credited_without_submitblock(self) -> None: + def test_accepted_direct_block_refreshes_clean_job_after_submit_response(self) -> None: old_tip = "00" * 32 - new_tip = "11" * 32 + block_hash = "ab" * 32 server, state, ledger = submit_coordinator(tip=old_tip) - rpc = ParentTipRpc(tip=new_tip, parent=old_tip) - server.rpc = rpc - server.current_tip_first_seen = (new_tip, time.monotonic()) - server.stale_grace_seconds = 3 - submission = SimpleNamespace( - header_hex="aa" * 80, - block_hash_hex="cc" * 32, - share_pass=True, - block_pass=True, + server.stop_after_block = False + server.max_blocks = 10 + server.clients = {state} + state.share_difficulty = Decimal("1") + sent: list[dict[str, object]] = [] + state.send = lambda payload: sent.append(payload) # type: ignore[method-assign] + old_context = server.jobs["job-1"] + server.tip_template_snapshot = QbitTipTemplateSnapshot( + bestblockhash=old_tip, + previousblockhash=old_tip, + template_fingerprint=qbit_template_fingerprint(old_context.template), ) - with ( - patch("builtins.print") as emitted, - patch( - "lab.prism.prism_coordinator.direct_stratum.assemble_submission", - return_value=submission, - ), - ): - should_close = server.handle_submit( - state, - ["miner-a", "job-1", "00" * 8, "00000001", "00000002"], + def build_fresh_job(client: ClientState, *, clean_jobs: bool) -> object: + self.assertIs(client, state) + self.assertNotIn( + "accepted_block_handling", + server._ensure_shutdown_controller().snapshot()["active_writers"], + ) + return prism_context( + "fresh-job", + block_hash, + worker=state.worker, + difficulty=server.desired_client_share_difficulty(client), + clean_jobs=clean_jobs, ) - self.assertFalse(should_close) - self.assertEqual(rpc.submitblock_calls, 0) - self.assertEqual(len(ledger.pending), 1) - self.assertEqual(ledger.pending[0].credit_policy, PRISM_CREDIT_POLICY_STALE_GRACE) - self.assertEqual(server.grace_credited_share_count, 1) - self.assertEqual(server.worker_share_counts["miner-a"]["grace"], 1) - self.assertEqual(server.block_solves_dropped_counts, {"stale_grace": 1}) - self.assertIn( - 'qbit_prism_block_solves_dropped_total{reason="stale_grace"} 1', - server.metrics_payload(), - ) - self.assertEqual(emitted.call_count, 1) - log_line = " ".join(str(value) for value in emitted.call_args.args) - self.assertIn(submission.block_hash_hex, log_line) - self.assertIn(old_tip, log_line) + server.build_job_for_client = build_fresh_job # type: ignore[method-assign] - def test_evicted_prior_tip_share_inside_grace_is_credited(self) -> None: - old_tip = "00" * 32 - new_tip = "11" * 32 - server, state, ledger = submit_coordinator(tip=old_tip) - rpc = ParentTipRpc(tip=new_tip, parent=old_tip) - server.rpc = rpc - server.current_tip_first_seen = (new_tip, time.monotonic()) - server.bury_evicted_job(state, "job-1") - server.jobs.pop("job-1") - state.active_job_ids.clear() - submission = SimpleNamespace( - header_hex="ad" * 80, - block_hash_hex="cd" * 32, - share_pass=True, - block_pass=False, - ) + with tempfile.TemporaryDirectory() as tempdir: + server.audit_dir = Path(tempdir) + server.evidence_path = Path(tempdir) / "evidence.json" + server.ledger_writer_public_key_hex = "aa" * 32 + server.rpc = SubmitAcceptingTemplateRpc(old_tip=old_tip, block_hash=block_hash, ledger=ledger) + server.build_audit_bundle = lambda **_kwargs: verified_block_bundle() # type: ignore[method-assign] + server.verify_bundle = lambda *_args, **_kwargs: verified_audit_report() # type: ignore[method-assign] + submission = SimpleNamespace( + header_hex="aa" * 80, + share_pass=True, + block_pass=True, + coinbase_tx_hex="c0ffee", + block_hash_hex=block_hash, + block_hex="00", + ) - with patch( - "lab.prism.prism_coordinator.direct_stratum.assemble_submission", - return_value=submission, - ): - should_close = server.handle_submit( + with patch( + "lab.prism.prism_coordinator.direct_stratum.assemble_submission", + return_value=submission, + ): + server.handle_request( + state, + { + "id": "submit-1", + "method": "mining.submit", + "params": ["miner-a", "job-1", "00" * 8, "00000001", "00000002"], + }, + ) + # The ack goes out before the block path runs; draining the + # submitter queue lands the block and pushes fresh work. + self.assertEqual(sent, [{"id": "submit-1", "result": True, "error": None}]) + self.assertTrue(server.submit_next_block_candidate()) + self.assertEqual(server.poll_qbit_tip_template_once(), 1) + + self.assertEqual(sent[0], {"id": "submit-1", "result": True, "error": None}) + self.assertEqual([payload.get("method") for payload in sent[1:]], ["mining.set_difficulty", "mining.notify"]) + self.assertEqual(sent[2]["params"][0], "fresh-job") + self.assertTrue(sent[2]["params"][8]) + self.assertEqual(server.tip_refresh_job_count, 1) + self.assertEqual(server.post_accept_refresh_failure_count, 0) + self.assertEqual(server.accepted_block_count, 1) + self.assertNotIn("job-1", server.jobs) + self.assertIn("fresh-job", server.jobs) + self.assertEqual(state.active_job_ids, {"fresh-job"}) + self.assertIn(state, server.clients) + server.stale_grace_seconds = 0 + + with self.assertRaises(StratumError) as raised: + server.handle_submit( state, ["miner-a", "job-1", "00" * 8, "00000001", "00000002"], ) + self.assertEqual(raised.exception.code, 21) + self.assertEqual(raised.exception.reason, PRISM_REJECTION_UNKNOWN_JOB) - self.assertFalse(should_close) - self.assertEqual(len(ledger.pending), 1) - self.assertEqual(ledger.pending[0].credit_policy, PRISM_CREDIT_POLICY_STALE_GRACE) - - def test_evicted_same_tip_share_is_credited_without_stale_grace_policy(self) -> None: - tip = "00" * 32 - server, state, ledger = submit_coordinator(tip=tip) - server.bury_evicted_job(state, "job-1") - server.jobs.pop("job-1") - state.active_job_ids.clear() - submission = SimpleNamespace( - header_hex="af" * 80, - block_hash_hex="cf" * 32, + fresh_submission = SimpleNamespace( + header_hex="bb" * 80, + block_hash_hex="bc" * 32, share_pass=True, block_pass=False, ) - with patch( "lab.prism.prism_coordinator.direct_stratum.assemble_submission", - return_value=submission, + return_value=fresh_submission, ): should_close = server.handle_submit( state, - ["miner-a", "job-1", "00" * 8, "00000001", "00000002"], + ["miner-a", "fresh-job", "00" * 8, "00000001", "00000002"], ) self.assertFalse(should_close) - self.assertEqual(len(ledger.pending), 1) - self.assertIsNone(ledger.pending[0].credit_policy) + self.assertEqual(ledger.pending[-1].job_id, "fresh-job") - def test_share_ack_latency_histogram_observes_accept_and_reject(self) -> None: - # The read-to-ack instrument for share-ingest saturation, driven - # through the real handler loop: both outcomes are observed after - # their response write, so accepted-vs-rejected separates commit - # pressure from transport/thread saturation. - tip = "00" * 32 - server, state, _ledger = submit_coordinator(tip=tip) + def test_post_accept_notification_does_not_run_failing_template_build(self) -> None: + old_tip = "00" * 32 + block_hash = "ad" * 32 + server, state, ledger = submit_coordinator(tip=old_tip) + server.stop_after_block = False + server.max_blocks = 10 + server.clients = {state} sent: list[dict[str, object]] = [] state.send = lambda payload: sent.append(payload) # type: ignore[method-assign] - submission = SimpleNamespace( - header_hex="af" * 80, - block_hash_hex="cf" * 32, - share_pass=True, - block_pass=False, - ) - coordinator_sock, peer = socket.socketpair() - state.sock = coordinator_sock - accepted_line = json.dumps( - { - "id": 7, - "method": "mining.submit", - "params": ["miner-a", "job-1", "00" * 8, "00000001", "00000002"], - } - ) - rejected_line = json.dumps( - { - "id": 8, - "method": "mining.submit", - "params": [ - "miner-a", - "missing-job", - "00" * 8, - "00000001", - "00000002", - ], - } - ) - with patch( - "lab.prism.prism_coordinator.direct_stratum.assemble_submission", - return_value=submission, - ): - handler = threading.Thread(target=server.handle_client, args=(state,)) - handler.start() - peer.sendall((accepted_line + "\n" + rejected_line + "\n").encode()) - peer.close() - handler.join(5) - - self.assertFalse(handler.is_alive()) - self.assertEqual(sent[0], {"id": 7, "result": True, "error": None}) - self.assertEqual(sent[1]["id"], 8) - self.assertIsNotNone(sent[1]["error"]) - metrics = "\n".join(server.share_ack_metrics_lines()) - self.assertIn( - 'qbit_prism_share_ack_seconds_count{result="accepted"} 1', - metrics, - ) - self.assertIn( - 'qbit_prism_share_ack_seconds_count{result="rejected"} 1', - metrics, - ) + def unexpected_build(client: ClientState, *, clean_jobs: bool) -> object: + raise AssertionError("job build should not run when template refresh fails") - def test_reconnected_same_username_submits_against_disconnected_job(self) -> None: - # A proxy flap leaves devices mining the dead connection's jobs; - # their shares arrive on the replacement connection and must credit - # against the retained context instead of rejecting unknown-job. - tip = "00" * 32 - server, state, ledger = submit_coordinator(tip=tip) - server.clients = {state} - state.close = lambda: None # type: ignore[method-assign] - server.disconnect_client(state) - retained = server.evicted_job_graveyard.get("job-1") - self.assertIsNotNone(retained) - assert retained is not None - self.assertIsNone(retained.client) - - reconnected = ClientState( - sock=object(), - address=("127.0.0.1", 2), - connection_id=2, - extranonce1_hex="00000002", - ) - reconnected.subscribed = True - reconnected.authorized = True - reconnected.username = "miner-a" - reconnected.worker = worker_identity("miner-a") - submission = SimpleNamespace( - header_hex="af" * 80, - block_hash_hex="cf" * 32, - share_pass=True, - block_pass=False, - ) + server.build_job_for_client = unexpected_build # type: ignore[method-assign] - with patch( - "lab.prism.prism_coordinator.direct_stratum.assemble_submission", - return_value=submission, - ): - should_close = server.handle_submit( - reconnected, - ["miner-a", "job-1", "00" * 8, "00000001", "00000002"], + with tempfile.TemporaryDirectory() as tempdir: + server.audit_dir = Path(tempdir) + server.evidence_path = Path(tempdir) / "evidence.json" + server.ledger_writer_public_key_hex = "aa" * 32 + server.rpc = SubmitAcceptingTemplateRpc( + old_tip=old_tip, + block_hash=block_hash, + fail_template_after_submit=True, + ledger=ledger, + ) + server.build_audit_bundle = lambda **_kwargs: verified_block_bundle() # type: ignore[method-assign] + server.verify_bundle = lambda *_args, **_kwargs: verified_audit_report() # type: ignore[method-assign] + submission = SimpleNamespace( + header_hex="aa" * 80, + share_pass=True, + block_pass=True, + coinbase_tx_hex="c0ffee", + block_hash_hex=block_hash, + block_hex="00", ) - self.assertFalse(should_close) - self.assertEqual(len(ledger.pending), 1) - self.assertIsNone(ledger.pending[0].credit_policy) - # Credit follows the retained context's original worker identity. - self.assertEqual(ledger.pending[0].share_id, "miner-a:" + "cf" * 32) - self.assertEqual( - server.evicted_job_submit_counts[ - "accepted_same_tip_cross_connection" - ], - 1, - ) - - def test_cross_connection_submit_requires_same_username(self) -> None: - tip = "00" * 32 - server, state, _ledger = submit_coordinator(tip=tip) - server.clients = {state} - state.close = lambda: None # type: ignore[method-assign] - server.disconnect_client(state) - self.assertIn("job-1", server.evicted_job_graveyard) - - intruder = ClientState( - sock=object(), - address=("127.0.0.1", 3), - connection_id=3, - extranonce1_hex="00000003", - ) - intruder.subscribed = True - intruder.authorized = True - intruder.username = "miner-b" - intruder.worker = worker_identity("miner-b") - submission = SimpleNamespace( - header_hex="af" * 80, - block_hash_hex="cf" * 32, - share_pass=True, - block_pass=False, - ) - - with patch( - "lab.prism.prism_coordinator.direct_stratum.assemble_submission", - return_value=submission, - ): - with self.assertRaises(StratumError) as raised: - server.handle_submit( - intruder, - ["miner-b", "job-1", "00" * 8, "00000001", "00000002"], + with patch( + "lab.prism.prism_coordinator.direct_stratum.assemble_submission", + return_value=submission, + ): + server.handle_request( + state, + { + "id": "submit-1", + "method": "mining.submit", + "params": ["miner-a", "job-1", "00" * 8, "00000001", "00000002"], + }, ) + self.assertTrue(server.submit_next_block_candidate()) - self.assertEqual(raised.exception.reason, PRISM_REJECTION_UNKNOWN_JOB) - - def test_cross_connection_submit_uses_retained_version_mask(self) -> None: - # In-flight work from the dead connection was version-rolled under - # the mask negotiated there; the replacement's own mask (0 before - # mining.configure) must not judge those version bits. - tip = "00" * 32 - server, state, _ledger = submit_coordinator(tip=tip) - server.jobs["job-1"].version_mask = 0x1FFFE000 - server.clients = {state} - state.close = lambda: None # type: ignore[method-assign] - server.disconnect_client(state) - - reconnected = ClientState( - sock=object(), - address=("127.0.0.1", 2), - connection_id=2, - extranonce1_hex="00000002", - ) - reconnected.subscribed = True - reconnected.authorized = True - reconnected.username = "miner-a" - reconnected.worker = worker_identity("miner-a") - self.assertEqual(reconnected.version_mask, 0) - submission = SimpleNamespace( - header_hex="af" * 80, - block_hash_hex="cf" * 32, - share_pass=True, - block_pass=False, - ) - - with patch( - "lab.prism.prism_coordinator.direct_stratum.assemble_submission", - return_value=submission, - ) as assemble: - self.assertFalse( - server.handle_submit( - reconnected, - [ - "miner-a", - "job-1", - "00" * 8, - "00000001", - "00000002", - "1fffe000", - ], - ) - ) + self.assertEqual(sent, [{"id": "submit-1", "result": True, "error": None}]) + self.assertEqual(server.accepted_block_count, 1) + self.assertEqual(len(ledger.persisted), 1) + self.assertEqual(len(ledger.confirmed), 1) + self.assertEqual(len(ledger.pending), 1) + self.assertEqual(server.tip_refresh_job_count, 0) + self.assertEqual(server.post_accept_refresh_failure_count, 0) + self.assertEqual(state.active_job_ids, {"job-1"}) + self.assertIn("job-1", server.jobs) + self.assertTrue(server.tip_refresh_is_pending()) + self.assertTrue(server._tip_refresh_retry.is_set()) + self.assertIn("qbit_prism_post_accept_refresh_failures_total 0", server.metrics_payload()) - self.assertEqual( - assemble.call_args.kwargs["version_mask"], - 0x1FFFE000, + def test_post_accept_refresh_preserves_pending_vardiff_difficulty_pair(self) -> None: + old_tip = "00" * 32 + block_hash = "ae" * 32 + server, state, ledger = submit_coordinator(tip=old_tip) + server.vardiff_config = vardiff.VardiffConfig( + enabled=True, + target_share_interval_seconds=Decimal("15"), + min_difficulty=Decimal("1"), + max_difficulty=Decimal("1024"), + retarget_interval_seconds=Decimal("90"), + max_step_factor=Decimal("4"), + startup_difficulty=Decimal("1"), + max_step_down_factor=Decimal("4"), + ewma_alpha=Decimal("1"), + retarget_tolerance=Decimal("0"), ) - - def test_cross_connection_block_candidate_keeps_original_extranonce(self) -> None: - # The mined coinbase embeds the extranonce1 the retained job was - # stamped with; a candidate recording the replacement connection's - # extranonce would fail the audit fence after submitblock. - tip = "00" * 32 - server, state, _ledger = submit_coordinator(tip=tip) server.stop_after_block = False server.max_blocks = 10 - server.jobs["job-1"].job.extranonce1_hex = state.extranonce1_hex server.clients = {state} - state.close = lambda: None # type: ignore[method-assign] - server.disconnect_client(state) - - reconnected = ClientState( - sock=object(), - address=("127.0.0.1", 2), - connection_id=2, - extranonce1_hex="00000002", - ) - reconnected.subscribed = True - reconnected.authorized = True - reconnected.username = "miner-a" - reconnected.worker = worker_identity("miner-a") - submission = SimpleNamespace( - header_hex="af" * 80, - block_hash_hex="cf" * 32, - share_pass=True, - block_pass=True, - coinbase_tx_hex="c0ffee", - block_hex="00", - ) + state.share_difficulty = Decimal("1") + state.pending_share_difficulty = Decimal("8") + state.vardiff_window_started_monotonic = time.monotonic() + sent: list[dict[str, object]] = [] + state.send = lambda payload: sent.append(payload) # type: ignore[method-assign] - with patch( - "lab.prism.prism_coordinator.direct_stratum.assemble_submission", - return_value=submission, - ): - self.assertFalse( - server.handle_submit( - reconnected, - ["miner-a", "job-1", "00" * 8, "00000001", "00000002"], - ) + def build_fresh_job(client: ClientState, *, clean_jobs: bool) -> object: + return prism_context( + "fresh-vardiff-job", + block_hash, + worker=state.worker, + difficulty=server.desired_client_share_difficulty(client), + clean_jobs=clean_jobs, ) - self.assertEqual(server.block_candidate_queue.qsize(), 1) - candidate = server.block_candidate_queue.get_nowait() - self.assertEqual(candidate.extranonce1_hex, "00000001") - self.assertIs(candidate.client, reconnected) - - def test_detached_entry_never_falls_back_to_stale_grace(self) -> None: - # A tip moving between the graveyard lookup and submit - # classification must not let a detached cross-connection entry - # borrow the reconnect's stale-grace window: cross-connection - # resumes are same-tip only. - tip_old = "00" * 32 - tip_new = "11" * 32 - server, state, _ledger = submit_coordinator(tip=tip_old) - server.clients = {state} - state.close = lambda: None # type: ignore[method-assign] - server.disconnect_client(state) - detached = server.evicted_job_graveyard.get("job-1") - self.assertIsNotNone(detached) - assert detached is not None - self.assertIsNone(detached.client) - - reconnected = ClientState( - sock=object(), - address=("127.0.0.1", 2), - connection_id=2, - extranonce1_hex="00000002", - ) - reconnected.subscribed = True - reconnected.authorized = True - reconnected.username = "miner-a" - reconnected.worker = worker_identity("miner-a") - eligibility_calls: list[str] = [] - - def recording_eligibility( - _client: ClientState, - _context: object, - current_tip: str, - ) -> bool: - eligibility_calls.append(current_tip) - return True - - server.context_eligible_for_stale_grace = ( # type: ignore[method-assign] - recording_eligibility - ) - - self.assertIsNone( - server.evicted_submit_context(reconnected, detached, tip_new) - ) - self.assertEqual(eligibility_calls, []) - - # Same-connection entries keep today's stale-grace classification. - live_state = client() - server.jobs["job-2"] = prism_context( - "job-2", - tip_old, - worker=live_state.worker, - ) - server.bury_evicted_job(live_state, "job-2") - live_entry = server.evicted_job_graveyard["job-2"] - result = server.evicted_submit_context(live_state, live_entry, tip_new) - self.assertIsNotNone(result) - assert result is not None - self.assertEqual(result[1], PRISM_CREDIT_POLICY_STALE_GRACE) - self.assertEqual(eligibility_calls, [tip_new]) - - def test_disconnected_retention_capacity_is_bounded(self) -> None: - tip = "00" * 32 - server, state, _ledger = submit_coordinator(tip=tip) - server.disconnected_job_retention = 1 - server.jobs["job-2"] = prism_context("job-2", tip, worker=state.worker) - state.active_job_ids = {"job-1", "job-2"} - server.clients = {state} - state.close = lambda: None # type: ignore[method-assign] - - server.disconnect_client(state) + server.build_job_for_client = build_fresh_job # type: ignore[method-assign] - self.assertEqual(len(server._disconnected_evicted_job_ids), 1) - self.assertEqual(len(server.evicted_job_graveyard), 1) - self.assertEqual( - server.evicted_job_capacity_eviction_counts["disconnected"], - 1, - ) - - def test_retained_share_dedup_uses_original_worker_after_reauthorization(self) -> None: - tip = "00" * 32 - server, state, ledger = submit_coordinator(tip=tip) - server.bury_evicted_job(state, "job-1") - server.jobs.pop("job-1") - state.active_job_ids.clear() - submission = SimpleNamespace( - header_hex="af" * 80, - block_hash_hex="cf" * 32, - share_pass=True, - block_pass=False, - ) - - with patch( - "lab.prism.prism_coordinator.direct_stratum.assemble_submission", - return_value=submission, - ): - self.assertFalse( - server.handle_submit( - state, - ["miner-a", "job-1", "00" * 8, "00000001", "00000002"], - ) + with tempfile.TemporaryDirectory() as tempdir: + server.audit_dir = Path(tempdir) + server.evidence_path = Path(tempdir) / "evidence.json" + server.ledger_writer_public_key_hex = "aa" * 32 + server.rpc = SubmitAcceptingTemplateRpc(old_tip=old_tip, block_hash=block_hash, ledger=ledger) + server.build_audit_bundle = lambda **_kwargs: verified_block_bundle() # type: ignore[method-assign] + server.verify_bundle = lambda *_args, **_kwargs: verified_audit_report() # type: ignore[method-assign] + submission = SimpleNamespace( + header_hex="aa" * 80, + share_pass=True, + block_pass=True, + coinbase_tx_hex="c0ffee", + block_hash_hex=block_hash, + block_hex="00", ) - state.username = "miner-b" - state.worker = worker_identity("miner-b") - with self.assertRaises(StratumError) as raised: - server.handle_submit( - state, - ["miner-b", "job-1", "00" * 8, "00000001", "00000002"], - ) - - self.assertEqual(raised.exception.reason, PRISM_REJECTION_DUPLICATE_SHARE) - self.assertEqual(len(ledger.pending), 1) - self.assertEqual(ledger.pending[0].share_id, "miner-a:" + "cf" * 32) - self.assertEqual(server.worker_share_counts["miner-a"]["accepted"], 1) - self.assertEqual(server.worker_share_counts["miner-b"]["accepted"], 0) - - def test_evicted_same_tip_share_survives_beyond_legacy_one_second_floor(self) -> None: - tip = "00" * 32 - server, state, ledger = submit_coordinator(tip=tip) - server.current_tip_first_seen = (tip, None) - server.same_tip_job_retention_seconds = 30 - server.bury_evicted_job(state, "job-1", now=100.0) - server.jobs.pop("job-1") - state.active_job_ids.clear() - submission = SimpleNamespace( - header_hex="a1" * 80, - block_hash_hex="c1" * 32, - share_pass=True, - block_pass=False, - ) - with patch( - "lab.prism.prism_coordinator.time.monotonic", - return_value=102.0, - ), patch( - "lab.prism.prism_coordinator.direct_stratum.assemble_submission", - return_value=submission, - ): - self.assertFalse( - server.handle_submit( + with patch( + "lab.prism.prism_coordinator.direct_stratum.assemble_submission", + return_value=submission, + ): + server.handle_request( state, - ["miner-a", "job-1", "00" * 8, "00000001", "00000002"], + { + "id": "submit-1", + "method": "mining.submit", + "params": ["miner-a", "job-1", "00" * 8, "00000001", "00000002"], + }, ) - ) - - self.assertEqual(len(ledger.pending), 1) - self.assertIsNone(ledger.pending[0].credit_policy) - self.assertIn("job-1", server.evicted_job_graveyard) - self.assertEqual(server.evicted_job_submit_counts["accepted_same_tip"], 1) - - def test_evicted_same_tip_submit_uses_original_job_difficulty(self) -> None: - tip = "00" * 32 - server, state, ledger = submit_coordinator(tip=tip) - server.current_tip_first_seen = (tip, None) - original_context = server.jobs["job-1"] - original_context.job.share_difficulty = Decimal("2") - state.share_difficulty = Decimal("32") - server.bury_evicted_job(state, "job-1") - server.jobs.pop("job-1") - state.active_job_ids.clear() - submission = SimpleNamespace( - header_hex="a2" * 80, - block_hash_hex="c2" * 32, - share_pass=True, - block_pass=False, - ) + self.assertTrue(server.submit_next_block_candidate()) + self.assertEqual(server.poll_qbit_tip_template_once(), 1) - def assemble(job: object, **_kwargs: object) -> object: - self.assertIs(job, original_context.job) - self.assertEqual(job.share_difficulty, Decimal("2")) - return submission + self.assertEqual(sent[0], {"id": "submit-1", "result": True, "error": None}) + self.assertEqual(sent[1]["method"], "mining.set_difficulty") + self.assertEqual(sent[1]["params"], [8.0]) + self.assertEqual(sent[2]["method"], "mining.notify") + self.assertEqual(sent[2]["params"][0], "fresh-vardiff-job") + self.assertTrue(sent[2]["params"][8]) + self.assertEqual(state.share_difficulty, Decimal("8")) + self.assertIsNone(state.pending_share_difficulty) + self.assertEqual(state.vardiff_window_submitted, 1) + self.assertEqual(state.vardiff_window_accepted, 1) - with patch( - "lab.prism.prism_coordinator.direct_stratum.assemble_submission", - side_effect=assemble, - ): - self.assertFalse( - server.handle_submit( - state, - ["miner-a", "job-1", "00" * 8, "00000001", "00000002"], - ) + def test_rejected_candidate_never_creates_prepared_payout_state(self) -> None: + server, state, ledger = submit_coordinator() + with tempfile.TemporaryDirectory() as tempdir: + server.audit_dir = Path(tempdir) + server.evidence_path = Path(tempdir) / "evidence.json" + server.ledger_writer_public_key_hex = "aa" * 32 + block_hash = "dd" * 32 + server.rpc = SubmitRpc(tip="00" * 32, block_hash=block_hash, submit_result="rejected") + server.build_audit_bundle = lambda **_kwargs: { # type: ignore[method-assign] + "found_block": {"coinbase_value_sats": 50_00000000}, + "ledger_window_attestation": {"signature": {"public_key_hex": "aa" * 32}}, + "payout_policy_manifest": {"accounts": []}, + "signed_coinbase_manifest": { + "manifest": { + "coinbase_tx_hex": "c0ffee", + "payout_count": 1, + } + }, + } + server.verify_bundle = ( # type: ignore[method-assign] + lambda *_args, **_kwargs: verified_audit_report() + ) + submission = SimpleNamespace( + coinbase_tx_hex="c0ffee", + block_hash_hex=block_hash, + block_hex="00", ) - self.assertEqual(len(ledger.pending), 1) - self.assertIsNone(ledger.pending[0].credit_policy) + accepted = server.submit_block_candidate(block_candidate(server, state, submission)) - def test_same_tip_retention_ttl_and_capacity_are_bounded(self) -> None: - tip = "00" * 32 - server, state, _ledger = submit_coordinator(tip=tip) - server.current_tip_first_seen = (tip, None) - server.same_tip_job_retention_seconds = 30 - server.same_tip_job_retention_per_connection = 2 - identity = state.worker - for index in range(3): - job_id = f"job-{index + 1}" - server.jobs[job_id] = prism_context(job_id, tip, worker=identity) - server.bury_evicted_job(state, job_id, now=100.0 + index) - - self.assertNotIn("job-1", server.evicted_job_graveyard) + self.assertFalse(accepted) + self.assertEqual(ledger.persisted, []) + self.assertEqual(ledger.rejected, []) + self.assertEqual(ledger.reversed, []) + self.assertEqual(len(ledger.pending), 0) self.assertEqual( - list(server.evicted_job_graveyard), - ["job-2", "job-3"], + server.block_candidate_abandoned_counts[PRISM_REJECTION_SUBMITBLOCK_REJECTED], 1 ) - self.assertEqual(server.evicted_job_capacity_eviction_counts["connection"], 1) - server.prune_evicted_job_graveyard(now=133.1) - self.assertEqual(server.evicted_job_graveyard, {}) - self.assertEqual(server.evicted_job_expiration_counts["same_tip"], 2) - - def test_evicted_job_hit_is_constant_work_in_large_graveyard(self) -> None: - tip = "00" * 32 - server, state, _ledger = submit_coordinator(tip=tip) - server.current_tip_first_seen = (tip, None) - server.same_tip_job_retention_seconds = 30 - server.same_tip_job_retention_per_connection = 4_096 - for index in range(4_096): - job_id = f"retained-{index}" - server.jobs[job_id] = prism_context(job_id, tip, worker=state.worker) - server.bury_evicted_job(state, job_id, now=100.0, prune=False) - - self.assertEqual(len(server.evicted_job_graveyard), 4_096) - self.assertEqual(len(server.evicted_same_tip_job_ids), 4_096) - classify_calls = 0 - original_classify = server._evicted_job_class_locked - - def counted_classify(entry: object) -> str: - nonlocal classify_calls - classify_calls += 1 - return original_classify(entry) - - server._evicted_job_class_locked = counted_classify # type: ignore[method-assign] - with patch("lab.prism.prism_coordinator.time.monotonic", return_value=101.0): - for _ in range(100): - self.assertIsNotNone( - server.evicted_job_entry(state, "retained-2048") - ) + def test_build_audit_bundle_passes_coinbase_output_policy_to_cli_payload(self) -> None: + server = coordinator() + server.rpc = AddressRpc(valid_address="tq1fee", script_byte="99") + server.signing_seed_hex = "42" * 32 + server.ledger_attestation_signing_seed_hex = "43" * 32 + captured: dict[str, object] = {} - self.assertEqual(classify_calls, 100) - - def test_pool_width_does_not_evict_other_connections_retained_jobs(self) -> None: - tip = "00" * 32 - server, _state, _ledger = submit_coordinator(tip=tip) - server.current_tip_first_seen = (tip, None) - server.same_tip_job_retention_seconds = 30 - server.same_tip_job_retention_per_connection = 1 - clients: list[ClientState] = [] - for index in range(4_097): - state = client() - state.connection_id = index + 1 - state.worker = worker_identity(f"miner-{index}") - clients.append(state) - job_id = f"wide-{index}" - server.jobs[job_id] = prism_context(job_id, tip, worker=state.worker) - server.bury_evicted_job(state, job_id, now=100.0, prune=False) - - self.assertEqual(len(server.evicted_job_graveyard), 4_097) - self.assertIn("wide-0", server.evicted_job_graveyard) - self.assertIn("wide-4096", server.evicted_job_graveyard) - - replacement_id = "wide-0-replacement" - server.jobs[replacement_id] = prism_context( - replacement_id, - tip, - worker=clients[0].worker, - ) - server.bury_evicted_job( - clients[0], - replacement_id, - now=101.0, - prune=False, - ) - - self.assertNotIn("wide-0", server.evicted_job_graveyard) - self.assertIn("wide-1", server.evicted_job_graveyard) - self.assertIn(replacement_id, server.evicted_job_graveyard) - self.assertEqual(len(server.evicted_job_graveyard), 4_097) - self.assertEqual(server.evicted_job_capacity_eviction_counts["connection"], 1) - - def test_tip_change_prunes_and_disconnect_detaches_retained_contexts(self) -> None: - old_tip = "00" * 32 - new_tip = "11" * 32 - server, state, _ledger = submit_coordinator(tip=old_tip) - server.current_tip_first_seen = (old_tip, None) - server.stale_grace_seconds = 0 - server.bury_evicted_job(state, "job-1") - self.assertIn("job-1", server.evicted_job_graveyard) + with patch.dict( + os.environ, + { + "PRISM_COINBASE_OUTPUT_POLICY": "pool-fee-first", + "PRISM_POOL_FEE_ENABLED": "1", + "PRISM_POOL_FEE_BPS": "125", + "PRISM_POOL_FEE_ADDRESS": "tq1fee", + }, + clear=True, + ), patch( + "lab.prism.prism_coordinator.subprocess.Popen", + fake_audit_bundle_popen(captured), + ): + server.build_audit_bundle( + shares=[], + found_block={"block_height": 10, "coinbase_value_sats": 50_00000000}, + prior_balances=[], + coinbase_script_sig_suffix_hex="00", + ) - server.observe_tip_first_seen(new_tip) - self.assertNotIn("job-1", server.evicted_job_graveyard) + payout_policy = captured["payload"]["payout_policy"] + self.assertEqual(payout_policy["coinbase_output_policy"], "pool-fee-first") + self.assertEqual(payout_policy["pool_fee_policy"]["fee_bps"], 125) - server.current_tip_first_seen = (new_tip, None) - server.jobs["job-2"] = prism_context("job-2", new_tip, worker=state.worker) - server.bury_evicted_job(state, "job-2") - server.clients = {state} - state.close = lambda: None # type: ignore[method-assign] - server.disconnect_client(state) - # Same-tip work survives the disconnect for a same-username - # reconnect; the dead connection's entry is detached and ages out on - # the normal same-tip TTL instead of vanishing with the socket. - retained = server.evicted_job_graveyard.get("job-2") - self.assertIsNotNone(retained) - assert retained is not None - self.assertIsNone(retained.client) - server.prune_evicted_job_graveyard( - now=time.monotonic() - + float(server.same_tip_job_retention_seconds) - + 1.0 - ) - self.assertEqual(server.evicted_job_graveyard, {}) - - def test_tip_flip_reanchors_retained_job_grace_to_client_delivery(self) -> None: - old_tip = "00" * 32 - new_tip = "11" * 32 - server, state, _ledger = submit_coordinator(tip=old_tip) - server.clients = {state} - server.current_tip_first_seen = (old_tip, None) - server.stale_grace_seconds = 3 - server.bury_evicted_job(state, "job-1", now=100.0) - - with patch("lab.prism.prism_coordinator.time.monotonic", return_value=120.0): - server.observe_tip_first_seen(new_tip) - - # Burial predates the flip by twenty seconds, but grace does not begin - # until this connection actually receives replacement work. - server.prune_evicted_job_graveyard(now=130.0) - self.assertIn("job-1", server.evicted_job_graveyard) - - state.tip_work_delivered = (new_tip, 130.0) - server.prune_evicted_job_graveyard(now=132.9) - self.assertIn("job-1", server.evicted_job_graveyard) - server.prune_evicted_job_graveyard(now=133.1) - self.assertNotIn("job-1", server.evicted_job_graveyard) - self.assertEqual(server.evicted_job_expiration_counts["stale_grace"], 1) - - def test_tip_flip_prunes_by_chain_parent_when_poller_skips_observed_tip(self) -> None: - observed_tip = "00" * 32 - intermediate_tip = "11" * 32 - current_tip = "22" * 32 - server, state, _ledger = submit_coordinator(tip=intermediate_tip) - server.clients = {state} - server.current_tip_first_seen = (observed_tip, None) - server.stale_grace_seconds = 3 - server.jobs["older-job"] = prism_context( - "older-job", - observed_tip, - worker=state.worker, + def test_ledger_artifact_not_built_when_publication_not_required(self) -> None: + tip = "5b" * 32 + ledger = ReorgLedger([]) + server, build_calls = self._reconcile_coordinator_with_artifact_counter( + ledger, + ReorgRpc( + tip=tip, + template=gbt_template(tip, height=11), + height=10, + block_hashes={10: tip}, + ), ) - server.bury_evicted_job(state, "older-job", now=100.0, prune=False) - server.bury_evicted_job(state, "job-1", now=110.0, prune=False) - server.rpc = ParentTipRpc(tip=current_tip, parent=intermediate_tip) - with patch("lab.prism.prism_coordinator.time.monotonic", return_value=120.0): - server.observe_tip_first_seen(current_tip) + first = server.reconcile_prism_pool_blocks_once(tip_hash=tip) + self.assertEqual(first["published_generation"], 1) + self.assertEqual(len(build_calls), 1) + self.assertFalse(build_calls[0][2]) + + second = server.reconcile_prism_pool_blocks_once(tip_hash=tip) - # The poller's previous observation is not authoritative. Tip - # observation proactively loads the actual parent, drops older work, - # and preserves the intermediate-tip context that submit can credit. - self.assertNotIn("older-job", server.evicted_job_graveyard) - self.assertIn("job-1", server.evicted_job_graveyard) + self.assertIsNone(second["published_generation"]) self.assertEqual( - server.current_tip_parent_hash(current_tip), - intermediate_tip, + len(build_calls), + 1, + "a pass without publication need must not build the ledger artifact", ) - entry = server.evicted_job_entry(state, "job-1") - self.assertIsNotNone(entry) - assert entry is not None self.assertEqual( - server.evicted_submit_context(state, entry, current_tip), - (entry.context, PRISM_CREDIT_POLICY_STALE_GRACE), - ) - - def test_slow_parent_lookup_cannot_overwrite_newer_tip_parent_cache(self) -> None: - old_tip = "00" * 32 - old_parent = "ff" * 32 - new_tip = "11" * 32 - new_parent = old_tip - server = coordinator() - server.current_tip_first_seen = (old_tip, None) - server.current_tip_observation_sequence = 1 - server.current_tip_parent = None - - def overtake_parent_lookup(tip_hash: str) -> str: - self.assertEqual(tip_hash, old_tip) - with server.lock: - server.current_tip_first_seen = (new_tip, 100.0) - server.current_tip_observation_sequence = 2 - server.current_tip_parent = (new_tip, new_parent) - return old_parent - - server._fetch_tip_parent_hash = overtake_parent_lookup # type: ignore[method-assign] - - self.assertEqual(server.current_tip_parent_hash(old_tip), old_parent) - self.assertEqual(server.current_tip_parent, (new_tip, new_parent)) - - def test_retained_same_tip_duplicate_remains_duplicate_share(self) -> None: - tip = "00" * 32 - server, state, ledger = submit_coordinator(tip=tip) - server.current_tip_first_seen = (tip, None) - server.bury_evicted_job(state, "job-1") - server.jobs.pop("job-1") - state.active_job_ids.clear() - submission = SimpleNamespace( - header_hex="a3" * 80, - block_hash_hex="c3" * 32, - share_pass=True, - block_pass=False, + ledger.events, + [("watch", 10), ("mature", 10), ("watch", 10), ("mature", 10)], ) - params = ["miner-a", "job-1", "00" * 8, "00000001", "00000002"] - - with patch( - "lab.prism.prism_coordinator.direct_stratum.assemble_submission", - return_value=submission, - ): - self.assertFalse(server.handle_submit(state, params)) - with self.assertRaises(StratumError) as raised: - server.handle_submit(state, params) - self.assertEqual(raised.exception.reason, PRISM_REJECTION_DUPLICATE_SHARE) - self.assertEqual(len(ledger.pending), 1) - self.assertIn("job-1", server.evicted_job_graveyard) - metrics = server.metrics_payload() - self.assertIn('qbit_prism_evicted_job_contexts{class="same_tip"} 1', metrics) - self.assertIn( - 'qbit_prism_evicted_job_submits_total{outcome="accepted_same_tip"} 1', - metrics, + def test_payout_change_forces_publication_and_rereads_balances(self) -> None: + tip = "5d" * 32 + pool_block_hash = "ae" * 32 + ledger = ReorgLedger([]) + server, build_calls = self._reconcile_coordinator_with_artifact_counter( + ledger, + ReorgRpc( + tip=tip, + template=gbt_template(tip, height=11), + height=10, + block_hashes={10: tip}, + ), ) - def test_pool_closed_submit_rejects_before_any_share_accounting(self) -> None: - # Post-close submits must not inflate submitted totals (the - # stale-percent denominator), per-worker submitted counters, or the - # vardiff window; only the pool-closed rejection itself is recorded. - server, state, ledger = submit_coordinator() - server.accepted_block_count = 1 - server.max_blocks = 1 - state.vardiff_config = SimpleNamespace(enabled=True) - submitted_before = server.submitted_share_count - - with self.assertRaises(StratumError) as raised: - server.handle_submit( - state, - ["miner-a", "job-1", "00" * 8, "00000001", "00000002"], - ) + first = server.reconcile_prism_pool_blocks_once(tip_hash=tip) + self.assertEqual(first["published_generation"], 1) + self.assertEqual(len(build_calls), 1) - self.assertEqual(raised.exception.reason, PRISM_REJECTION_POOL_CLOSED) - self.assertEqual(server.submitted_share_count, submitted_before) - # The rejection itself may admit the label, but no submission counted. - self.assertEqual(server.worker_share_counts["miner-a"]["submitted"], 0) - self.assertEqual(state.vardiff_window_submitted, 0) - self.assertEqual(len(ledger.pending), 0) - self.assertEqual( - server.worker_rejection_counts[("miner-a", PRISM_REJECTION_POOL_CLOSED)], 1 + # An orphaned confirmed block appears without any new payout source: + # payout_changed alone must still force a publication. + ledger.rows.append( + { + "block_hash": pool_block_hash, + "block_height": 12, + "chain_state": "confirmed", + "maturity_state": "immature", + } ) - def test_malformed_submit_does_not_diverge_worker_and_aggregate_submitted(self) -> None: - # A malformed-ntime submit must count identically in the per-worker and - # aggregate submitted counters (i.e. not at all) so the two never drift. - server, state, _ledger = submit_coordinator() - submitted_before = server.submitted_share_count - - with self.assertRaises(StratumError) as raised: - server.handle_submit( - state, - ["miner-a", "job-1", "00" * 8, "bad-ntime", "00000002"], - ) + second = server.reconcile_prism_pool_blocks_once(tip_hash=tip) - self.assertEqual(raised.exception.reason, PRISM_REJECTION_INVALID_NTIME_OR_NONCE) - self.assertEqual(server.submitted_share_count, submitted_before) - self.assertEqual(server.worker_share_counts["miner-a"]["submitted"], 0) + self.assertEqual(second["inactive_blocks"], 1) + self.assertEqual(second["published_generation"], 2) + self.assertEqual(len(build_calls), 2) self.assertEqual( - server.worker_rejection_counts[("miner-a", PRISM_REJECTION_INVALID_NTIME_OR_NONCE)], - 1, - ) - - def test_stale_grace_closed_when_refresh_path_has_not_observed_tip(self) -> None: - # Only blockpoll/blockwait may open the grace window. If the refresh path - # has not anchored the new tip (current_tip_first_seen is None) and only - # this submit's getbestblockhash sees it, the prior-tip share must reject - # as stale-job -- not get credited from a submit-anchored window. - old_tip = "00" * 32 - new_tip = "11" * 32 - server, state, ledger = submit_coordinator(tip=old_tip) - rpc = ParentTipRpc(tip=new_tip, parent=old_tip) - server.rpc = rpc - server.current_tip_first_seen = None - server.stale_grace_seconds = 3 - submission = SimpleNamespace( - header_hex="aa" * 80, - block_hash_hex="cc" * 32, - share_pass=True, - block_pass=False, - ) - - with patch( - "lab.prism.prism_coordinator.direct_stratum.assemble_submission", - return_value=submission, - ): - with self.assertRaises(StratumError) as raised: - server.handle_submit( - state, - ["miner-a", "job-1", "00" * 8, "00000001", "00000002"], - ) - - self.assertEqual(raised.exception.reason, PRISM_REJECTION_STALE_JOB) - self.assertEqual(len(ledger.pending), 0) - self.assertEqual(rpc.submitblock_calls, 0) - # The submit must not have anchored the window either. - self.assertIsNone(server.current_tip_first_seen) - - def test_stale_grace_rejected_after_window_expires(self) -> None: - # This connection received current-tip work well outside the grace - # window; a prior-tip share arriving now must reject rather than be - # credited late. - old_tip = "00" * 32 - new_tip = "11" * 32 - server, state, ledger = submit_coordinator(tip=old_tip) - rpc = ParentTipRpc(tip=new_tip, parent=old_tip) - server.rpc = rpc - server.stale_grace_seconds = 3 - server.current_tip_first_seen = (new_tip, time.monotonic() - 10) - state.tip_work_delivered = (new_tip, time.monotonic() - 10) - submission = SimpleNamespace( - header_hex="ab" * 80, - block_hash_hex="ce" * 32, - share_pass=True, - block_pass=False, - ) - - with patch( - "lab.prism.prism_coordinator.direct_stratum.assemble_submission", - return_value=submission, - ): - with self.assertRaises(StratumError) as raised: - server.handle_submit( - state, - ["miner-a", "job-1", "00" * 8, "00000001", "00000002"], - ) - - self.assertEqual(raised.exception.reason, PRISM_REJECTION_STALE_JOB) - self.assertEqual(len(ledger.pending), 0) - self.assertEqual(rpc.submitblock_calls, 0) - - def test_stale_grace_open_until_connection_receives_new_tip_work(self) -> None: - # The refresh pass may be slow or aborted (reorg reconcile failure, - # transient build errors). Until THIS connection is sent current-tip - # work, its prior-tip shares are still in flight and must stay - # creditable even after the global first-seen stamp ages past the - # grace window. - old_tip = "00" * 32 - new_tip = "11" * 32 - server, state, ledger = submit_coordinator(tip=old_tip) - rpc = ParentTipRpc(tip=new_tip, parent=old_tip) - server.rpc = rpc - server.stale_grace_seconds = 3 - server.current_tip_first_seen = (new_tip, time.monotonic() - 10) - state.tip_work_delivered = (old_tip, time.monotonic() - 60) - submission = SimpleNamespace( - header_hex="ac" * 80, - block_hash_hex="cd" * 32, - share_pass=True, - block_pass=False, - ) - - with patch( - "lab.prism.prism_coordinator.direct_stratum.assemble_submission", - return_value=submission, - ): - should_close = server.handle_submit( - state, - ["miner-a", "job-1", "00" * 8, "00000001", "00000002"], - ) - - self.assertFalse(should_close) - self.assertEqual(rpc.submitblock_calls, 0) - self.assertEqual(len(ledger.pending), 1) - self.assertEqual(ledger.pending[0].credit_policy, PRISM_CREDIT_POLICY_STALE_GRACE) - - def test_stale_grace_window_runs_from_per_connection_delivery_not_first_seen(self) -> None: - # A slow refresh pass can deliver current-tip work to a connection - # after the global first-seen stamp has already aged past the grace - # window. The window for that connection runs from ITS delivery. - old_tip = "00" * 32 - new_tip = "11" * 32 - server, state, ledger = submit_coordinator(tip=old_tip) - rpc = ParentTipRpc(tip=new_tip, parent=old_tip) - server.rpc = rpc - server.stale_grace_seconds = 3 - server.current_tip_first_seen = (new_tip, time.monotonic() - 10) - state.tip_work_delivered = (new_tip, time.monotonic() - 1) - submission = SimpleNamespace( - header_hex="ae" * 80, - block_hash_hex="cb" * 32, - share_pass=True, - block_pass=False, - ) - - with patch( - "lab.prism.prism_coordinator.direct_stratum.assemble_submission", - return_value=submission, - ): - should_close = server.handle_submit( - state, - ["miner-a", "job-1", "00" * 8, "00000001", "00000002"], - ) - - self.assertFalse(should_close) - self.assertEqual(len(ledger.pending), 1) - self.assertEqual(ledger.pending[0].credit_policy, PRISM_CREDIT_POLICY_STALE_GRACE) - - def test_startup_baseline_tip_does_not_open_stale_grace_window(self) -> None: - # The first tip observed after process start is a baseline, not a tip - # flip: it must not open the grace window. A later real flip must. - old_tip = "00" * 32 - new_tip = "11" * 32 - server, state, _ledger = submit_coordinator(tip=old_tip) - server.stale_grace_seconds = 3 - server.current_tip_first_seen = None - - server.observe_tip_first_seen(new_tip) - self.assertEqual(server.current_tip_first_seen, (new_tip, None)) - self.assertFalse(server.stale_grace_deadline_open(state, new_tip)) - - # A change away from the observed baseline is a real flip and opens - # the window for connections that have not yet received the new work. - flip_tip = "22" * 32 - server.observe_tip_first_seen(flip_tip) - self.assertIsNotNone(server.current_tip_first_seen[1]) - self.assertTrue(server.stale_grace_deadline_open(state, flip_tip)) - - def test_note_tip_work_delivered_keeps_first_delivery_per_tip(self) -> None: - # Same-tip template refreshes must not slide the grace anchor forward. - server, state, _ledger = submit_coordinator() - tip = "11" * 32 - - server.note_tip_work_delivered(state, tip) - first = state.tip_work_delivered - self.assertEqual(first[0], tip) - server.note_tip_work_delivered(state, tip) - self.assertEqual(state.tip_work_delivered, first) - - # A new tip re-anchors. - server.note_tip_work_delivered(state, "22" * 32) - self.assertEqual(state.tip_work_delivered[0], "22" * 32) - self.assertGreaterEqual(state.tip_work_delivered[1], first[1]) - - def test_evicted_graveyard_keeps_unexpired_entries_above_previous_cap_for_grace_credit(self) -> None: - old_tip = "00" * 32 - new_tip = "11" * 32 - server, state, ledger = submit_coordinator(tip=old_tip) - rpc = ParentTipRpc(tip=new_tip, parent=old_tip) - server.rpc = rpc - server.current_tip_first_seen = (new_tip, time.monotonic()) - context = server.jobs["job-1"] - evicted_at = time.monotonic() - server.evicted_job_graveyard = { - "job-1": (context, state.connection_id, evicted_at), - } - previous_hard_cap = 512 - for index in range(previous_hard_cap): - server.evicted_job_graveyard[f"filler-{index}"] = ( - context, - state.connection_id, - evicted_at + 0.001 + (index / 1_000_000), - ) - server.prune_evicted_job_graveyard(now=evicted_at + 0.5) - self.assertIn("job-1", server.evicted_job_graveyard) - server.jobs.pop("job-1") - state.active_job_ids.clear() - submission = SimpleNamespace( - header_hex="ae" * 80, - block_hash_hex="ce" * 32, - share_pass=True, - block_pass=False, - ) - - with patch( - "lab.prism.prism_coordinator.direct_stratum.assemble_submission", - return_value=submission, - ): - should_close = server.handle_submit( - state, - ["miner-a", "job-1", "00" * 8, "00000001", "00000002"], - ) - - self.assertFalse(should_close) - self.assertEqual(len(ledger.pending), 1) - self.assertEqual(ledger.pending[0].credit_policy, PRISM_CREDIT_POLICY_STALE_GRACE) - - def test_stale_grace_parent_rpc_failure_rejects_as_backend_unavailable(self) -> None: - old_tip = "00" * 32 - new_tip = "11" * 32 - server, state, _ledger = submit_coordinator(tip=old_tip) - server.rpc = TipRpc(new_tip) - server.current_tip_first_seen = (new_tip, time.monotonic()) - - with self.assertRaises(StratumError) as raised: - server.handle_submit( - state, - ["miner-a", "job-1", "00" * 8, "00000001", "00000002"], - ) - - self.assertEqual(raised.exception.reason, PRISM_REJECTION_BACKEND_RPC_UNAVAILABLE) - - def test_unknown_job_rejects_before_getbestblockhash_rpc(self) -> None: - class CountingTipRpc(TipRpc): - def __init__(self, tip: str) -> None: - super().__init__(tip) - self.getbest_calls = 0 - - def call(self, method: str, params: list[object] | None = None) -> object: - if method == "getbestblockhash": - self.getbest_calls += 1 - return super().call(method, params) - - server, state, _ledger = submit_coordinator() - rpc = CountingTipRpc("00" * 32) - server.rpc = rpc - - with self.assertRaises(StratumError) as raised: - server.handle_submit( - state, - ["miner-a", "garbage-job", "00" * 8, "00000001", "00000002"], - ) - - self.assertEqual(raised.exception.reason, PRISM_REJECTION_UNKNOWN_JOB) - self.assertEqual(rpc.getbest_calls, 0) - - def test_submit_passes_negotiated_version_bits_and_mask_to_stratum_assembly(self) -> None: - server, state, _ledger = submit_coordinator() - state.version_mask = 0x1FFFE000 - submission = SimpleNamespace( - header_hex="ac" * 80, - block_hash_hex="ba" * 32, - share_pass=True, - block_pass=False, - ) - - with patch( - "lab.prism.prism_coordinator.direct_stratum.assemble_submission", - return_value=submission, - ) as assemble_submission: - server.handle_submit( - state, - ["miner-a", "job-1", "00" * 8, "00000001", "00000002", "00002000"], - ) - - self.assertEqual(assemble_submission.call_args.kwargs["version_bits_hex"], "00002000") - self.assertEqual(assemble_submission.call_args.kwargs["version_mask"], 0x1FFFE000) - - def test_address_worker_submit_accrues_to_base_payout_address(self) -> None: - server, state, ledger = submit_coordinator() - username = f"{PAYOUT_ADDRESS}.rig-a" - worker = WorkerIdentity( - username=username, - payout_address=PAYOUT_ADDRESS, - worker_name="rig-a", - script_pubkey_hex="5220" + "44" * 32, - p2mr_program_hex="44" * 32, - ) - state.username = username - state.worker = worker - server.jobs["job-1"].worker = worker - server.share_weights_by_username = {PAYOUT_ADDRESS: 9} - submission = SimpleNamespace( - header_hex="aa" * 80, - block_hash_hex="bc" * 32, - share_pass=True, - block_pass=False, - ) - - with patch( - "lab.prism.prism_coordinator.direct_stratum.assemble_submission", - return_value=submission, - ): - should_close = server.handle_submit( - state, - [username, "job-1", "00" * 8, "00000001", "00000002"], - ) - - self.assertFalse(should_close) - self.assertEqual(len(ledger.pending), 1) - self.assertEqual(ledger.pending[0].share_id, username + ":" + "bc" * 32) - self.assertEqual(ledger.pending[0].miner_id, PAYOUT_ADDRESS) - self.assertEqual(ledger.pending[0].order_key, PAYOUT_ADDRESS) - self.assertEqual(ledger.pending[0].share_difficulty, 9) - - def test_address_worker_submit_still_requires_authorized_full_username(self) -> None: - server, state, ledger = submit_coordinator() - username = f"{PAYOUT_ADDRESS}.rig-a" - worker = WorkerIdentity( - username=username, - payout_address=PAYOUT_ADDRESS, - worker_name="rig-a", - script_pubkey_hex="5220" + "44" * 32, - p2mr_program_hex="44" * 32, - ) - state.username = username - state.worker = worker - server.jobs["job-1"].worker = worker - - with self.assertRaises(StratumError) as raised: - server.handle_submit( - state, - [PAYOUT_ADDRESS, "job-1", "00" * 8, "00000001", "00000002"], - ) - - self.assertEqual(raised.exception.code, 20) - self.assertEqual(len(ledger.pending), 0) - - def test_block_submit_rejects_job_when_prior_balances_changed_before_persist(self) -> None: - server, state, _recording = submit_coordinator() - ledger = SingleWriterShareLedger() - ledger.durable_payout_state = True # type: ignore[attr-defined] - server.ledger = ledger - server.jobs["job-1"].prior_balances = [ - { - "recipient_id": "miner-a", - "order_key": "miner-a", - "p2mr_program_hex": "11" * 32, - "balance_sats": 25, - } - ] - server.build_audit_bundle = lambda **_kwargs: (_ for _ in ()).throw( # type: ignore[method-assign] - AssertionError("audit bundle should not be rebuilt from stale prior balances") - ) - submission = SimpleNamespace( - coinbase_tx_hex="c0ffee", - block_hash_hex="ef" * 32, - block_hex="00", - ) - pending = self._pending_append("balance-stale").pending_share - candidate = block_candidate( - server, - state, - submission, - pending_share=pending, - ) - ledger.append_batch( - [(pending, server.block_candidate_intent(candidate))] - ) - server.enqueue_block_candidate(candidate) - - self.assertTrue(server.submit_next_block_candidate()) - # The share was already accepted at submit time, so a lost block is a - # block-abandonment, not a stale share rejection. - self.assertEqual(server.stale_share_count, 0) - self.assertEqual(server.block_candidate_abandoned_counts[PRISM_REJECTION_STALE_JOB], 1) - self.assertEqual( - server.stale_job_abandon_counts, - {"tip_moved": 0, "balance_stale": 1, "append_epoch_stale": 0}, - ) - outbox_row = ledger._block_candidate_outbox[submission.block_hash_hex] - self.assertEqual(outbox_row["state"], "abandoned") - self.assertEqual( - outbox_row["last_error"], - "prior balances changed since the job was issued", - ) - metrics = server.metrics_payload() - self.assertIn( - 'qbit_prism_stale_job_abandons_total{class="tip_moved"} 0', - metrics, - ) - self.assertIn( - 'qbit_prism_stale_job_abandons_total{class="balance_stale"} 1', - metrics, + build_calls[1][2:], + (False, True), + "a confirmed payout mutation keeps the exact share window and " + "only rereads current prior balances", ) + self.assertEqual(ledger.rows[0]["chain_state"], "inactive") + self.assertIn(("inactive", pool_block_hash, 10), ledger.events) def test_block_submit_lands_as_issued_after_late_append_epoch_invalidation(self) -> None: # A late-visible replay append advances the live epoch and schedules @@ -7910,107 +5599,16 @@ def gated_append_batch(entries: list[object]) -> list[object]: self.assertIsNone(outbox_row["last_error"]) self.assertEqual(server.accepted_block_count, 1) - def test_predating_append_commits_while_fast_lane_offer_is_in_flight( - self, - ) -> None: - # The other side of the fence boundary flipped with the node fast - # lane: submitblock no longer holds the landing fence, so a - # predating append's durable commit does NOT wait for the RPC to - # return. The row may become durable mid-offer with its epoch bump - # landing under the fence; the landing's post-offer accounting - # fences (not the offer itself) are what observe the invalidation. - # Here the node rejects the block outright, so the rejection -- not - # the epoch race -- terminally abandons the candidate. - server, state, _recording = submit_coordinator() - ledger = SingleWriterShareLedger() - server.ledger = ledger - context = server.jobs["job-1"] - context.found_block = { - "network_difficulty": 1, - "anchor_job_issued_at_ms": 12000, - } - # Keep an anchor the late row predates exposed for the whole test, - # independent of the landing's own declared-anchor lifetime. - anchor_token = server._expose_inflight_scan_anchor(12000) - self.addCleanup(server._retire_inflight_scan_anchor, anchor_token) - - late_entry = self._pending_append("mid-rpc-append") - durable_mid_rpc: list[bool] = [] - append_threads: list[threading.Thread] = [] - - class MidRpcAppendRpc(TipRpc): - def call( - rpc_self, - method: str, - params: list[object] | None = None, - ) -> object: - if method == "getblockcount": - return 9 - if method == "submitblock": - thread = threading.Thread( - target=server._append_share_entry, - args=(late_entry,), - daemon=True, - ) - thread.start() - append_threads.append(thread) - deadline = time.monotonic() + 0.25 - became_durable = False - while time.monotonic() < deadline: - if late_entry.pending_share.share_id in ledger._share_ids: - became_durable = True - break - time.sleep(0.005) - durable_mid_rpc.append(became_durable) - return "rejected-by-test" - return super().call(method, params) - - server.rpc = MidRpcAppendRpc("00" * 32) - submission = SimpleNamespace( - coinbase_tx_hex="c0ffee", - block_hash_hex="dd" * 32, - block_hex="00", - ) - pending = self._pending_append("landing-holds-fence").pending_share - candidate = block_candidate( - server, - state, - submission, - pending_share=pending, - ) - ledger.append_batch( - [(pending, server.block_candidate_intent(candidate))] - ) - server.enqueue_block_candidate(candidate) - - self.assertTrue(server.submit_next_block_candidate()) - - # The fast-lane RPC holds no fence, so the predating row became - # durable while the offer was still in flight. - self.assertEqual(durable_mid_rpc, [True]) - self.assertTrue(append_threads) - append_threads[0].join(timeout=10.0) - self.assertFalse(append_threads[0].is_alive()) - self.assertIn(late_entry.pending_share.share_id, ledger._share_ids) - with server._job_cache_lock: - self.assertEqual(server._payout_ledger_append_invalidation_epoch, 1) - self.assertEqual( - server.block_candidate_abandoned_counts[ - PRISM_REJECTION_SUBMITBLOCK_REJECTED - ], - 1, - ) - - def test_lease_deferral_unwinds_landed_bar_before_submitblock(self) -> None: - # The landed bar is armed before the lease fence to keep a transport - # failure's uncertain submitblock outcome conservative. A - # WriterLeaseRenewalDeferred refusal fires before the RPC, so the - # outcome is not uncertain: qbitd provably never saw the block, and - # leaving the bar armed in the (now surviving) process would bar - # reconciliation and payout-state publication for as long as the - # writer's own fenced write keeps the renewal deferred. The pending - # preview must survive, though: child work keeps waiting for the - # retry instead of snapshotting pre-accept balances. + def test_lease_deferral_unwinds_landed_bar_before_submitblock(self) -> None: + # The landed bar is armed before the lease fence to keep a transport + # failure's uncertain submitblock outcome conservative. A + # WriterLeaseRenewalDeferred refusal fires before the RPC, so the + # outcome is not uncertain: qbitd provably never saw the block, and + # leaving the bar armed in the (now surviving) process would bar + # reconciliation and payout-state publication for as long as the + # writer's own fenced write keeps the renewal deferred. The pending + # preview must survive, though: child work keeps waiting for the + # retry instead of snapshotting pre-accept balances. server, state, _recording = submit_coordinator() ledger = SingleWriterShareLedger() server.ledger = ledger @@ -8278,549 +5876,745 @@ def call( self.assertIsNone(outbox_row["last_error"]) self.assertEqual(server.accepted_block_count, 1) - def test_completed_append_predating_seedless_published_window_fences( - self, - ) -> None: - # A build that seeds no artifact (the documented - # PRISM_PAYOUT_ARTIFACT_REUSE=0 rollback mode) retires its walk - # exposure at the publication fence, yet jobs stamped from it keep - # serving that window until they retire. A replay-shaped append - # that started AND finished in that gap used to see no live - # anchor: no epoch bump, and its registry entry was popped on - # completion -- so a later landing drained nothing, its - # nonnegative context epoch skipped durable revalidation, and - # submitblock accepted a coinbase omitting the durable share. - # Publication now hands the declared anchor to the - # published-window watermark, so the append commits under the - # landing fence with its epoch bump -- and the landing, whose - # offer already landed, keeps the as-issued snapshot for - # accounting. - server, state, _recording = submit_coordinator() - ledger = SingleWriterShareLedger() - server.ledger = ledger - bundle_builds: list[dict[str, object]] = [] + def test_landing_builds_audit_bundle_outside_balance_serializer(self) -> None: + # The builder/verifier subprocess time must stay off the + # payout-balance mutation lock: job delivery and reconciliation + # queue behind that lock, and holding it across the bundle build was + # the dominant term of the finalization delivery stall. + server, state, ledger = submit_coordinator() + server.stop_after_block = False + server.max_blocks = 10 + lock_held_during_build: list[bool] = [] + build_calls: list[int] = [] - def recording_build_audit_bundle(**kwargs: object) -> dict[str, object]: - bundle_builds.append(dict(kwargs)) - return verified_block_bundle() + def probe_lock_from_other_thread() -> bool: + # An RLock re-acquires freely on the owning thread, so the probe + # must come from a thread that cannot be the owner. + acquired: list[bool] = [] - server.build_audit_bundle = recording_build_audit_bundle # type: ignore[method-assign] - server.verify_bundle = lambda *_args, **_kwargs: verified_audit_report() # type: ignore[method-assign] - tail_dir = tempfile.TemporaryDirectory() - self.addCleanup(tail_dir.cleanup) - server.audit_dir = Path(tail_dir.name) - server.evidence_path = Path(tail_dir.name) / "evidence.json" - server.ledger_writer_public_key_hex = "aa" * 32 - context = server.jobs["job-1"] - context.found_block = { - "network_difficulty": 1, - "anchor_job_issued_at_ms": 12000, - } - submission = SimpleNamespace( - coinbase_tx_hex="c0ffee", - block_hash_hex="dd" * 32, - block_hex="00", - ) - pending = self._pending_append("seedless-window").pending_share - candidate = block_candidate( - server, - state, - submission, - pending_share=pending, - ) - ledger.append_batch( - [(pending, server.block_candidate_intent(candidate))] - ) - server.enqueue_block_candidate(candidate) + def attempt() -> None: + got = server._payout_balance_mutation_lock.acquire( + blocking=False + ) + if got: + server._payout_balance_mutation_lock.release() + acquired.append(not got) - # The publication fence of a seedless build hands the bundle's - # declared anchor to the watermark when it retires the walk - # exposure. - server._ensure_job_cache_state() - with server._job_cache_lock: - server._publish_seedless_job_window_anchor_locked(12000) + prober = threading.Thread(target=attempt) + prober.start() + prober.join(5) + return bool(acquired and acquired[0]) - # The ordinary share hot path stays off the fence: a row stamped - # above the watermark commits unfenced and bumps nothing. - ordinary_entry = self._pending_append( - "ordinary-above-watermark", accepted_at_ms=13000 - ) - server._append_share_entry(ordinary_entry) - with server._job_cache_lock: - self.assertEqual(server._payout_ledger_append_invalidation_epoch, 0) - self.assertEqual( - server._payout_unfenced_append_inflight_stamps, {} - ) + def fake_build_audit_bundle(**_kwargs: object) -> dict[str, object]: + build_calls.append(1) + lock_held_during_build.append(probe_lock_from_other_thread()) + return verified_block_bundle() - # The replay-shaped append starts and finishes entirely before the - # landing begins: nothing is in flight for the landing to drain. - late_entry = self._pending_append("completed-before-landing") - server._append_share_entry(late_entry) - self.assertIn(late_entry.pending_share.share_id, ledger._share_ids) - with server._job_cache_lock: - self.assertEqual(server._payout_ledger_append_invalidation_epoch, 1) - self.assertEqual( - server._payout_unfenced_append_inflight_stamps, {} + with tempfile.TemporaryDirectory() as tempdir: + server.audit_dir = Path(tempdir) + server.evidence_path = Path(tempdir) / "evidence.json" + server.ledger_writer_public_key_hex = "aa" * 32 + block_hash = "cc" * 32 + server.rpc = SubmitRpc( + tip="00" * 32, + block_hash=block_hash, + ledger=ledger, + ) + server.build_audit_bundle = fake_build_audit_bundle # type: ignore[method-assign] + server.verify_bundle = ( # type: ignore[method-assign] + lambda *_args, **_kwargs: verified_audit_report() + ) + submission = SimpleNamespace( + coinbase_tx_hex="c0ffee", + block_hash_hex=block_hash, + block_hex="00", + ) + pending = SimpleNamespace(share_id="miner-a:" + block_hash) + accepted = server.submit_block_candidate( + block_candidate(server, state, submission, pending_share=pending) ) - submitblock_calls: list[object] = [] + self.assertTrue(accepted) + self.assertEqual(build_calls, [1]) + self.assertEqual(lock_held_during_build, [False]) + self.assertEqual(len(ledger.persisted), 1) + self.assertEqual(len(ledger.confirmed), 1) - class RecordingSubmitRpc(TipRpc): + def test_block_submitted_before_audit_bundle_build(self) -> None: + # The audit build runs with the balance serializer released, but + # never before submitblock: announcement of a freshly solved block + # must not wait on audit construction (a lost race is a lost round). + server, state, ledger = submit_coordinator() + server.stop_after_block = False + server.max_blocks = 10 + order: list[str] = [] + block_hash = "cc" * 32 + + class OrderRecordingRpc(SubmitRpc): def call( - rpc_self, + self, method: str, params: list[object] | None = None, ) -> object: - if method == "getblockcount": - return 9 if method == "submitblock": - submitblock_calls.append(params) - return None - if method == "getblockhash": - return "dd" * 32 + order.append("submitblock") return super().call(method, params) - server.rpc = RecordingSubmitRpc("00" * 32) + def ordered_build(**_kwargs: object) -> dict[str, object]: + order.append("build_audit_bundle") + return verified_block_bundle() - self.assertTrue(server.submit_next_block_candidate()) + with tempfile.TemporaryDirectory() as tempdir: + server.audit_dir = Path(tempdir) + server.evidence_path = Path(tempdir) / "evidence.json" + server.ledger_writer_public_key_hex = "aa" * 32 + server.rpc = OrderRecordingRpc( + tip="00" * 32, + block_hash=block_hash, + ledger=ledger, + ) + server.build_audit_bundle = ordered_build # type: ignore[method-assign] + server.verify_bundle = ( # type: ignore[method-assign] + lambda *_args, **_kwargs: verified_audit_report() + ) + submission = SimpleNamespace( + coinbase_tx_hex="c0ffee", + block_hash_hex=block_hash, + block_hex="00", + ) + pending = SimpleNamespace(share_id="miner-a:" + block_hash) + accepted = server.submit_block_candidate( + block_candidate(server, state, submission, pending_share=pending) + ) - # The bump already happened at commit time; the landing's epoch - # fence observes it and, with the node offer already accepted, - # accounts the as-issued coinbase (the durable predating share - # rides the next window). - self.assertEqual(submitblock_calls, [["00"]]) - # The landing observed the bump but the node already accepted - # the block: accounting keeps the as-issued snapshot instead of - # terminally abandoning a live block's payouts. - self.assertEqual(len(bundle_builds), 1) - self.assertNotIn( - PRISM_REJECTION_STALE_JOB, - server.block_candidate_abandoned_counts, - ) - self.assertEqual( - getattr(server, "stale_job_abandon_counts", {}).get( - "append_epoch_stale", 0 - ), - 0, - ) - outbox_row = ledger._block_candidate_outbox[submission.block_hash_hex] - self.assertEqual(outbox_row["state"], "submitted") - self.assertIsNone(outbox_row["last_error"]) - self.assertEqual(server.accepted_block_count, 1) + self.assertTrue(accepted) + self.assertEqual(order, ["submitblock", "build_audit_bundle"]) + self.assertEqual(len(ledger.persisted), 1) + self.assertEqual(len(ledger.confirmed), 1) - def test_block_submit_defers_descendant_until_active_ancestor_is_durable( - self, - ) -> None: - parent_hash = "ed" * 32 - ancestor_hash = "ee" * 32 + def test_successful_fast_lane_reconciles_observed_post_submit_tip(self) -> None: + parent_hash = "00" * 32 + block_hash = "cd" * 32 server, state, ledger = submit_coordinator(tip=parent_hash) - server.max_blocks = 10 server.stop_after_block = False - ledger.durable_payout_state = True - preview = [ - { - "recipient_id": "miner-a", - "order_key": "miner-a", - "p2mr_program_hex": "11" * 32, - "balance_sats": 25, - } - ] - context = server.jobs["job-1"] - context.template["height"] = 12 - context.prior_balances = preview - original_rpc_call = server.rpc.call - submit_calls: list[str] = [] - - def active_ancestor_call( - method: str, - params: list[object] | None = None, + server.max_blocks = 10 + server.rpc = SubmitAcceptingTemplateRpc( + old_tip=parent_hash, + block_hash=block_hash, + ledger=ledger, + ) + reconciled: list[tuple[str, bool]] = [] + landing_inputs: list[tuple[str, bool]] = [] + + def reconcile( + tip_hash: str, + *, + _coalesce_same_tip: bool = True, + ) -> bool: + reconciled.append((tip_hash, _coalesce_same_tip)) + return True + + original_land = server._land_and_confirm_block_candidate + + def observe_landing( + candidate: PrismBlockCandidate, + **kwargs: object, ) -> object: - if method == "getblockhash": - self.assertEqual(params, [10]) - return ancestor_hash - if method == "submitblock": - submit_calls.append(method) - return original_rpc_call(method, params) + landing_inputs.append( + (str(kwargs["current_tip"]), bool(kwargs["already_active"])) + ) + return original_land(candidate, **kwargs) # type: ignore[arg-type] - server.rpc.call = active_ancestor_call # type: ignore[method-assign] - server._begin_accepted_block_payout_preview( - ancestor_hash, - block_height=10, + server.ensure_reorg_reconciled_for_tip = reconcile # type: ignore[method-assign] + server._land_and_confirm_block_candidate = observe_landing # type: ignore[method-assign] + server.build_audit_bundle = ( # type: ignore[method-assign] + lambda **_kwargs: verified_block_bundle() ) - server._publish_accepted_block_payout_preview(ancestor_hash, preview) - server.build_audit_bundle = lambda **_kwargs: (_ for _ in ()).throw( # type: ignore[method-assign] - AssertionError("descendant must wait for ancestor durability") + server.verify_bundle = ( # type: ignore[method-assign] + lambda *_args, **_kwargs: verified_audit_report() ) submission = SimpleNamespace( coinbase_tx_hex="c0ffee", - block_hash_hex="ef" * 32, + block_hash_hex=block_hash, block_hex="00", ) - accepted = server.submit_block_candidate( - block_candidate(server, state, submission) + with tempfile.TemporaryDirectory() as tempdir: + server.audit_dir = Path(tempdir) + server.evidence_path = Path(tempdir) / "evidence.json" + server.ledger_writer_public_key_hex = "aa" * 32 + accepted = server.submit_block_candidate( + block_candidate(server, state, submission) + ) + + self.assertTrue(accepted) + self.assertEqual(landing_inputs, [(block_hash, False)]) + self.assertEqual(reconciled, [(block_hash, False)]) + + def test_submitter_offers_block_before_writer_admission_with_rpc_deadline(self) -> None: + server, state, _ledger = submit_coordinator() + server.block_submit_rpc_timeout_seconds = 0.75 + order: list[object] = [] + + class TimeoutRecordingRpc(TipRpc): + def call( + self, + method: str, + params: list[object] | None = None, + *, + timeout: float | None = None, + ) -> object: + if method == "submitblock": + order.append(("submitblock", timeout)) + return None + return super().call(method, params) + + class WriterAdmission: + def __enter__(self) -> None: + order.append("writer-admission") + + def __exit__(self, *_args: object) -> None: + return None + + candidate = block_candidate( + server, + state, + SimpleNamespace( + coinbase_tx_hex="00", + block_hash_hex="ce" * 32, + block_hex="00", + share_pass=True, + block_pass=True, + ), ) + server.rpc = TimeoutRecordingRpc("00" * 32) + server._writer_operation = lambda _component: WriterAdmission() # type: ignore[method-assign] - self.assertFalse(accepted) - # Node propagation is the fast lane: the durable descendant is offered - # to qbitd before payout/accounting notices the ancestor transition. - # Its accounting still defers until that ancestor is durable. - self.assertEqual(submit_calls, ["submitblock"]) - self.assertEqual(ledger.persisted, []) + def account( + _candidate: PrismBlockCandidate, + *, + node_submission: object, + ) -> bool: + order.append("accounting") + self.assertIsNone(getattr(node_submission, "result")) + return True + + server._submit_next_block_candidate_writer = account # type: ignore[method-assign] + server.enqueue_block_candidate(candidate) + + self.assertTrue(server.submit_next_block_candidate()) self.assertEqual( - server._block_candidate_outcome.reason, - PRISM_REJECTION_LEDGER_CONFIRMATION_FAILED, + order, + [("submitblock", 0.75), "writer-admission", "accounting"], ) - self.assertNotIn( - PRISM_REJECTION_STALE_JOB, - server.block_candidate_abandoned_counts, + + def test_accepted_candidate_with_moved_tip_finalizes_as_submitted(self) -> None: + old_tip = "00" * 32 + new_tip = "11" * 32 + block_hash = "cc" * 32 + server, state, _recording = submit_coordinator(tip=old_tip) + ledger = SingleWriterShareLedger() + server.ledger = ledger + pending = self._pending_append("accepted-tip-race").pending_share + candidate = block_candidate( + server, + state, + SimpleNamespace( + coinbase_tx_hex="00", + block_hash_hex=block_hash, + block_hex="00", + share_pass=True, + block_pass=True, + ), + pending_share=pending, ) - self.assertFalse(server.stop_event.is_set()) + ledger.append_batch( + [(pending, server.block_candidate_intent(candidate))] + ) + submitted: list[str] = [] + abandoned: list[str] = [] + original_submitted = ledger.mark_block_candidate_submitted + original_abandoned = ledger.mark_block_candidate_abandoned - def test_active_descendant_replay_stays_landed_until_ancestor_is_durable( - self, - ) -> None: - parent_hash = "ea" * 32 - ancestor_hash = "eb" * 32 - descendant_hash = "ec" * 32 - server, state, ledger = submit_coordinator(tip=parent_hash) - server.max_blocks = 10 - server.stop_after_block = False - ledger.durable_payout_state = True - preview = [ - { - "recipient_id": "miner-a", - "order_key": "miner-a", - "p2mr_program_hex": "11" * 32, - "balance_sats": 25, - } - ] - context = server.jobs["job-1"] - context.template["height"] = 12 - context.prior_balances = preview - original_rpc_call = server.rpc.call + def mark_submitted(*, block_hash: str) -> bool: + submitted.append(block_hash) + return original_submitted(block_hash=block_hash) - def active_descendant_call( - method: str, - params: list[object] | None = None, - ) -> object: - if method == "getbestblockhash": - return descendant_hash - if method == "getblockhash": - self.assertEqual(params, [10]) - return ancestor_hash - if method == "submitblock": - raise AssertionError("active descendant must not be resubmitted") - return original_rpc_call(method, params) + def mark_abandoned(*, block_hash: str, error: str) -> bool: + abandoned.append(block_hash) + return original_abandoned(block_hash=block_hash, error=error) - server.rpc.call = active_descendant_call # type: ignore[method-assign] + ledger.mark_block_candidate_submitted = mark_submitted # type: ignore[method-assign] + ledger.mark_block_candidate_abandoned = mark_abandoned # type: ignore[method-assign] server._begin_accepted_block_payout_preview( - ancestor_hash, + block_hash, block_height=10, ) - server._publish_accepted_block_payout_preview(ancestor_hash, preview) - server.build_audit_bundle = lambda **_kwargs: (_ for _ in ()).throw( # type: ignore[method-assign] - AssertionError("active descendant must wait for ancestor durability") + server._mark_accepted_block_payout_landed( + block_hash, + block_height=10, ) - submission = SimpleNamespace( - coinbase_tx_hex="c0ffee", - block_hash_hex=descendant_hash, - block_hex="00", + with server.lock: + # This is the process-local record written immediately before the + # "qbit accepted direct PRISM block" log. + server._accounted_accepted_block_hashes.add(block_hash) + server.rpc = TipRpc(new_tip) + server.build_audit_bundle = ( # type: ignore[method-assign] + lambda **_kwargs: (_ for _ in ()).throw( + AssertionError("an accounted candidate must not be rebuilt") + ) ) + server.enqueue_block_candidate(candidate) - accepted = server.submit_block_candidate( - block_candidate(server, state, submission) - ) + self.assertTrue(server.submit_next_block_candidate()) - self.assertFalse(accepted) - self.assertEqual(ledger.persisted, []) - self.assertEqual( - server._block_candidate_outcome.reason, - PRISM_REJECTION_LEDGER_CONFIRMATION_FAILED, + self.assertEqual(submitted, [block_hash]) + self.assertEqual(abandoned, []) + self.assertEqual(ledger.pending_block_candidates(), []) + self.assertNotIn( + PRISM_REJECTION_STALE_JOB, + server.block_candidate_abandoned_counts, ) - transition = server._accepted_block_payout_previews[descendant_hash] - self.assertTrue(transition.landed) - self.assertIsNone(transition.preview) - self.assertFalse(server.stop_event.is_set()) - - def test_block_submit_reconciliation_error_is_structured_rejection(self) -> None: - tip = "f0" * 32 - server, state, _ledger = submit_coordinator(tip=tip) - server.reorg_reconciler_enabled = True - server.rpc = ReorgRpc( - tip=tip, - template=gbt_template(tip, height=10), - height=9, - block_hashes={9: tip}, + self.assertNotIn(block_hash, server._accepted_block_payout_previews) + self.assertNotIn( + block_hash, + server._invalidated_accepted_block_payout_previews, ) - class FailingSubmitReorgLedger(RecordingLedger): - def reorg_watch_blocks(self, *, active_tip_height: int) -> list[dict[str, object]]: - raise RuntimeError("ledger unavailable") - - ledger = FailingSubmitReorgLedger() - server.ledger = ledger - server.build_audit_bundle = lambda **_kwargs: (_ for _ in ()).throw( # type: ignore[method-assign] - AssertionError("audit bundle should not be rebuilt after reconcile failure") + def test_terminal_abandon_immediately_clears_payout_preview(self) -> None: + server, _state, _ledger = submit_coordinator() + block_hash = "a4" * 32 + server._begin_accepted_block_payout_preview( + block_hash, + block_height=10, ) - submission = SimpleNamespace( - coinbase_tx_hex="c0ffee", - block_hash_hex="f1" * 32, - block_hex="00", + server._mark_accepted_block_payout_landed( + block_hash, + block_height=10, ) - with patch("builtins.print") as printed: - accepted = server.submit_block_candidate(block_candidate(server, state, submission)) + accepted_race_won = server._abandon_block_candidate( + PRISM_REJECTION_STALE_JOB, + "tip moved", + block_hash=block_hash, + worker="miner-a", + ) - self.assertFalse(accepted) - self.assertEqual( - server.block_candidate_abandoned_counts.get(PRISM_REJECTION_BACKEND_RPC_UNAVAILABLE, 0), - 0, + self.assertFalse(accepted_race_won) + self.assertNotIn(block_hash, server._accepted_block_payout_previews) + self.assertIn( + block_hash, + server._invalidated_accepted_block_payout_previews, ) - messages = [str(call.args[0]) for call in printed.call_args_list if call.args] - self.assertTrue(any("block candidate deferred" in message for message in messages)) - self.assertFalse(any("block candidate abandoned" in message for message in messages)) - self.assertEqual(server.rejection_counts_by_reason[PRISM_REJECTION_BACKEND_RPC_UNAVAILABLE], 0) - self.assertEqual(ledger.persisted, []) - self.assertEqual(ledger.pending, []) + # The tombstone protects pending durable replay, but unlike the leaked + # landed transition it does not block payout reconciliation. + with server._payout_balance_mutation(): + pass - def test_address_worker_suffixes_share_one_payout_account(self) -> None: - server, state_a, ledger = submit_coordinator() - usernames = [f"{PAYOUT_ADDRESS}.rig-a", f"{PAYOUT_ADDRESS}.rig-b"] - submissions = [ - SimpleNamespace( - header_hex="aa" * 80, - block_hash_hex="bd" * 32, - share_pass=True, - block_pass=False, - ), + def test_tip_moved_abandon_yields_to_acceptance_during_cleanup(self) -> None: + old_tip = "00" * 32 + new_tip = "11" * 32 + block_hash = "a5" * 32 + server, state, _recording = submit_coordinator(tip=old_tip) + configure_temporary_audit_root(self, server) + ledger = SingleWriterShareLedger() + server.ledger = ledger + pending = self._pending_append("accepted-during-clear").pending_share + candidate = block_candidate( + server, + state, SimpleNamespace( - header_hex="bb" * 80, - block_hash_hex="be" * 32, + coinbase_tx_hex="00", + block_hash_hex=block_hash, + block_hex="00", share_pass=True, - block_pass=False, + block_pass=True, ), - ] - states = [state_a, client()] - states[1].active_job_ids = {"job-1"} - for state, username in zip(states, usernames, strict=True): - worker = WorkerIdentity( - username=username, - payout_address=PAYOUT_ADDRESS, - worker_name=username.rsplit(".", 1)[1], - script_pubkey_hex="5220" + "55" * 32, - p2mr_program_hex="55" * 32, - ) - state.username = username - state.worker = worker - state.subscribed = True - state.authorized = True - server.jobs["job-1"].worker = worker - with patch( - "lab.prism.prism_coordinator.direct_stratum.assemble_submission", - return_value=submissions.pop(0), - ): - server.handle_submit( - state, - [username, "job-1", "00" * 8, "00000001", "00000002"], - ) + pending_share=pending, + ) + ledger.append_batch( + [(pending, server.block_candidate_intent(candidate))] + ) + submitted: list[str] = [] + abandoned: list[str] = [] + original_submitted = ledger.mark_block_candidate_submitted + original_abandoned = ledger.mark_block_candidate_abandoned + original_clear = server._clear_accepted_block_payout_preview - self.assertEqual([pending.miner_id for pending in ledger.pending], [PAYOUT_ADDRESS, PAYOUT_ADDRESS]) - self.assertEqual([pending.order_key for pending in ledger.pending], [PAYOUT_ADDRESS, PAYOUT_ADDRESS]) + def mark_submitted(*, block_hash: str) -> bool: + submitted.append(block_hash) + return original_submitted(block_hash=block_hash) - def test_stale_tip_rejects_without_appending_share(self) -> None: - old_tip = "00" * 32 - new_tip = "11" * 32 - server, state, ledger = submit_coordinator(tip=old_tip) - server.rpc = ParentTipRpc(tip=new_tip, parent="22" * 32) + def mark_abandoned(*, block_hash: str, error: str) -> bool: + abandoned.append(block_hash) + return original_abandoned(block_hash=block_hash, error=error) - with self.assertRaises(StratumError) as raised: - server.handle_submit( - state, - ["miner-a", "job-1", "00" * 8, "00000001", "00000002"], + def clear_while_accepting( + candidate_hash: str, + *, + invalidate_published: bool = False, + ) -> None: + original_clear( + candidate_hash, + invalidate_published=invalidate_published, ) + if invalidate_published: + with server.lock: + server._accounted_accepted_block_hashes.add( + candidate_hash.lower() + ) - self.assertEqual(raised.exception.code, 21) - self.assertEqual(raised.exception.reason, PRISM_REJECTION_STALE_JOB) - self.assertEqual(len(ledger.pending), 0) - self.assertEqual(server.stale_share_count, 1) - self.assertEqual(server.rejection_counts_by_reason[PRISM_REJECTION_STALE_JOB], 1) - - def test_share_ack_and_counters_wait_for_group_commit(self) -> None: - server, state, ledger = submit_coordinator() - server.share_writer_active = True - server.share_commit_timeout_seconds = 2.0 - server.share_commit_linger_seconds = 0.0 - commit_started = threading.Event() - release_commit = threading.Event() + ledger.mark_block_candidate_submitted = mark_submitted # type: ignore[method-assign] + ledger.mark_block_candidate_abandoned = mark_abandoned # type: ignore[method-assign] + server._clear_accepted_block_payout_preview = clear_while_accepting # type: ignore[method-assign] + server._begin_accepted_block_payout_preview( + block_hash, + block_height=10, + ) + server._mark_accepted_block_payout_landed( + block_hash, + block_height=10, + ) + server.rpc = RejectingSubmitTipRpc(new_tip) + server.enqueue_block_candidate(candidate) - class BlockingBatchLedger(type(ledger)): - def append_batch(self, entries: object) -> list[object]: - commit_started.set() - release_commit.wait(timeout=2) - return [self.append(pending) for pending, _candidate in entries] + self.assertTrue(server.submit_next_block_candidate()) - ledger = BlockingBatchLedger() - server.ledger = ledger - submission = SimpleNamespace( - header_hex="aa" * 80, - block_hash_hex="cc" * 32, - share_pass=True, - block_pass=False, + self.assertEqual(submitted, [block_hash]) + self.assertEqual(abandoned, []) + self.assertNotIn( + PRISM_REJECTION_STALE_JOB, + server.block_candidate_abandoned_counts, + ) + metrics = server.metrics_payload() + self.assertIn( + 'qbit_prism_stale_job_abandons_total{class="tip_moved"} 0', + metrics, + ) + self.assertIn( + 'qbit_prism_stale_job_abandons_total{class="balance_stale"} 0', + metrics, + ) + self.assertEqual(ledger.pending_block_candidates(), []) + outbox_row = ledger._block_candidate_outbox[block_hash] + self.assertEqual(outbox_row["state"], "submitted") + self.assertIsNone(outbox_row["last_error"]) + self.assertNotIn(block_hash, server._accepted_block_payout_previews) + self.assertNotIn( + block_hash, + server._invalidated_accepted_block_payout_previews, ) - writer = threading.Thread(target=server.share_append_loop, daemon=True) - writer.start() - outcome: list[object] = [] + def test_crash_after_submit_before_attempt_mark_replays_duplicate_once(self) -> None: + parent_hash = "00" * 32 + block_hash = "cf" * 32 + ledger = SingleWriterShareLedger() + first, state, _recording = submit_coordinator(tip=parent_hash) + first.ledger = ledger + first.stop_after_block = False + first.max_blocks = 10 + pending = self._pending_append("submit-before-mark-crash").pending_share + candidate = block_candidate( + first, + state, + SimpleNamespace( + coinbase_tx_hex="c0ffee", + block_hash_hex=block_hash, + block_hex="00", + share_pass=True, + block_pass=True, + ), + pending_share=pending, + ) + intent = first.block_candidate_intent(candidate) + ledger.append_batch([(pending, intent)]) - def submit() -> None: - with patch( - "lab.prism.prism_coordinator.direct_stratum.assemble_submission", - return_value=submission, - ): - outcome.append( - server.handle_submit( - state, - ["miner-a", "job-1", "00" * 8, "00000001", "00000002"], - ) - ) + class RestartAwareRpc(FakeRpc): + def __init__(self) -> None: + self.submit_results: list[object] = [] - submitter = threading.Thread(target=submit) - submitter.start() - self.assertTrue(commit_started.wait(timeout=1)) - self.assertTrue(submitter.is_alive()) - self.assertEqual(len(ledger.pending), 0) - self.assertEqual(server.worker_share_counts["miner-a"]["accepted"], 0) + def call( + self, + method: str, + params: list[object] | None = None, + *, + timeout: float | None = None, + ) -> object: + if method == "submitblock": + result = None if not self.submit_results else "duplicate" + self.submit_results.append(result) + return result + if method == "getbestblockhash": + return parent_hash + if method == "getblockhash": + return block_hash + if method == "getblockcount": + return 9 + return super().call(method, params) - release_commit.set() - submitter.join(timeout=2) - self.assertFalse(submitter.is_alive()) - self.assertEqual(outcome, [False]) - self.assertEqual(len(ledger.pending), 1) - self.assertEqual(ledger.pending[0].share_id, "miner-a:" + "cc" * 32) - self.assertEqual(server.worker_share_counts["miner-a"]["accepted"], 1) - server.request_shutdown() - writer.join(timeout=2) - - def test_job_snapshot_anchor_precedes_stamped_uncommitted_share(self) -> None: - # A share is stamped accepted_at_ms at validation time, before its - # group commit. Anchors chosen while that commit is pending must - # predate the stamp, or the issued window would omit a share that a - # later re-derivation at the same anchor includes. - server, state, ledger = submit_coordinator() - server.share_writer_active = True - server.share_commit_timeout_seconds = 2.0 - server.share_commit_linger_seconds = 0.0 - commit_started = threading.Event() - release_commit = threading.Event() + rpc = RestartAwareRpc() + first.rpc = rpc + original_mark = ledger.mark_block_candidate_attempted - class BlockingBatchLedger(type(ledger)): - def append_batch(self, entries: object) -> list[object]: - commit_started.set() - release_commit.wait(timeout=2) - return [self.append(pending) for pending, _candidate in entries] + def crash_before_mark(*, block_hash: str) -> bool: + raise SystemExit(f"simulated crash before marking {block_hash}") - ledger = BlockingBatchLedger() - server.ledger = ledger - submission = SimpleNamespace( - header_hex="aa" * 80, - block_hash_hex="cc" * 32, - share_pass=True, - block_pass=False, - ) + ledger.mark_block_candidate_attempted = crash_before_mark # type: ignore[method-assign] + first.enqueue_block_candidate(candidate) + with self.assertRaisesRegex(SystemExit, "simulated crash"): + first.submit_next_block_candidate() - writer = threading.Thread(target=server.share_append_loop, daemon=True) - writer.start() + self.assertEqual(rpc.submit_results, [None]) + self.assertEqual(ledger.pending_block_candidates(), [intent]) + self.assertEqual( + ledger._block_candidate_outbox[block_hash]["attempt_count"], + 0, + ) - def submit() -> None: - with patch( - "lab.prism.prism_coordinator.direct_stratum.assemble_submission", - return_value=submission, - ): - server.handle_submit( - state, - ["miner-a", "job-1", "00" * 8, "00000001", "00000002"], - ) + ledger.mark_block_candidate_attempted = original_mark # type: ignore[method-assign] + persisted_calls = 0 + confirmed_calls = 0 + original_persist = ledger.persist_accepted_block + original_confirm = ledger.confirm_accepted_block + + def persist_once(**kwargs: object) -> dict[str, object]: + nonlocal persisted_calls + persisted_calls += 1 + return original_persist(**kwargs) + + def confirm_once(**kwargs: object) -> dict[str, object]: + nonlocal confirmed_calls + confirmed_calls += 1 + return original_confirm(**kwargs) + + ledger.persist_accepted_block = persist_once # type: ignore[method-assign] + ledger.confirm_accepted_block = confirm_once # type: ignore[method-assign] - submitter = threading.Thread(target=submit) - submitter.start() - self.assertTrue(commit_started.wait(timeout=1)) + restarted, _restart_state, _recording = submit_coordinator(tip=parent_hash) + restarted.ledger = ledger + restarted.rpc = rpc + restarted.stop_after_block = False + restarted.max_blocks = 10 + with tempfile.TemporaryDirectory() as tempdir: + restarted.audit_dir = Path(tempdir) + restarted.evidence_path = Path(tempdir) / "evidence.json" + restarted.ledger_writer_public_key_hex = "aa" * 32 + restarted.build_audit_bundle = ( # type: ignore[method-assign] + lambda **_kwargs: verified_block_bundle() + ) + restarted.verify_bundle = ( # type: ignore[method-assign] + lambda *_args, **_kwargs: verified_audit_report() + ) + + self.assertEqual(restarted.replay_pending_block_candidates(), 1) + self.assertTrue(restarted.submit_next_block_candidate()) + self.assertEqual(restarted.replay_pending_block_candidates(), 0) - floor = dict(server._pending_share_commit_floor) - self.assertEqual(len(floor), 1) - (floor_entry,) = floor.values() - stamped_ms = int(floor_entry[0].accepted_at_ms) + self.assertEqual(rpc.submit_results, [None, "duplicate"]) + self.assertEqual(persisted_calls, 1) + self.assertEqual(confirmed_calls, 1) + self.assertEqual(restarted.accepted_block_count, 1) + self.assertEqual(ledger.pending_block_candidates(), []) self.assertEqual( - server._job_snapshot_anchor_ms(stamped_ms + 60_000), - stamped_ms - 1, + ledger._block_candidate_outbox[block_hash]["state"], + "submitted", ) - release_commit.set() - submitter.join(timeout=2) - self.assertFalse(submitter.is_alive()) - self.assertEqual(server._pending_share_commit_floor, {}) - # Drained floor: the anchor still sits strictly below the clamp - # instant so a same-millisecond stamp can never tie it. + def test_block_submitter_honors_stop_after_block_above_one_block_capacity( + self, + ) -> None: + server, state, ledger = submit_coordinator() + server.accepted_block_count = 1 + server.max_blocks = 2 + server.stop_after_block = True + submission = SimpleNamespace( + coinbase_tx_hex="c0ffee", + block_hash_hex="de" * 32, + block_hex="00", + ) + + accepted = server.submit_block_candidate( + block_candidate(server, state, submission) + ) + + self.assertFalse(accepted) self.assertEqual( - server._job_snapshot_anchor_ms(stamped_ms + 60_000), - stamped_ms + 60_000 - 1, + server.block_candidate_abandoned_counts[PRISM_REJECTION_POOL_CLOSED], + 1, ) - server.request_shutdown() - writer.join(timeout=2) + self.assertEqual(ledger.persisted, []) - def test_share_batch_failure_still_restores_snapshot_anchor(self) -> None: - server, _state, _ledger = submit_coordinator() - entry = self._pending_append("aa", accepted_at_ms=2) - server._ensure_pending_share_commit_state() - with server._pending_share_commit_lock: - server._pending_share_commit_floor[id(entry.pending_share)] = [ - entry.pending_share, - time.monotonic(), - False, - ] - self.assertEqual(server._job_snapshot_anchor_ms(10_000), 1) + @staticmethod + def _reconcile_coordinator_with_artifact_counter( + ledger: ReorgLedger, + rpc: ReorgRpc, + ) -> tuple[PrismCoordinator, list[tuple[int, int, bool, bool]]]: + """Coordinator armed so candidate preparation would build an artifact.""" + server = coordinator() + server.reorg_reconciler_enabled = True + server.ledger = ledger + server.rpc = rpc + server._ensure_job_cache_state() + # Keep the speculative background preparer out of the way so the + # counter observes only builds requested by reconcile passes. + server._payout_artifact_executor_shutdown = True + server._pool_ready_latched = True + server._template_artifacts = SimpleNamespace(network_difficulty=1) + build_calls: list[tuple[int, int, bool, bool]] = [] - class FailingLedger(FakeLedger): - def append(self, pending: object) -> object: - raise RuntimeError("ledger unavailable") + def fake_build( + expected_payout_state_generation: int, + artifact_payout_state_generation: int, + network_difficulty: int, + force_full_rescan: bool = False, + bypass_build_interval: bool = False, + during_publication: bool = False, + *, + force_prior_balances_read: bool = False, + ) -> None: + del network_difficulty, bypass_build_interval, during_publication + build_calls.append( + ( + expected_payout_state_generation, + artifact_payout_state_generation, + force_full_rescan, + force_prior_balances_read, + ) + ) + return None - server.ledger = FailingLedger() - self.assertFalse(server._append_share_batch([entry])) - self.assertTrue(entry.committed.is_set()) - self.assertEqual(server._pending_share_commit_floor, {}) - self.assertEqual(server._job_snapshot_anchor_ms(10_000), 9_999) + server._build_payout_ledger_artifact = fake_build # type: ignore[method-assign] + return server, build_calls - def test_failed_commit_releases_duplicate_key_for_exact_retry(self) -> None: - server, state, healthy = submit_coordinator() - server.share_writer_active = True - server.share_commit_linger_seconds = 0.0 - server.share_commit_timeout_seconds = 1.0 - class FailedLedger: - def append_batch(self, _entries: object) -> list[object]: - raise RuntimeError("postgres unavailable") +class PrismStampedJobFloorTests(unittest.TestCase): + """The listener floor must hold on the wire, not just in vardiff policy. - server.ledger = FailedLedger() - writer = threading.Thread(target=server.share_append_loop, daemon=True) - writer.start() + Stamped jobs are the single choke point for every mining.set_difficulty + the coordinator sends, and marketplace verification judges the first one. + The regression here is a young chain: qbit network difficulty below the + high-diff floor used to drag the advertised difficulty down with it. + """ + + def stamp_coordinator(self) -> PrismCoordinator: + server = coordinator() + server.job_counter = 0 + server.share_weights_by_username = {} + server.default_share_weight = 1 + return server + + def cached_bundle(self) -> CachedJobBundle: + # bits 207fffff: regtest-grade network difficulty (~4.7e-10), far + # below the 500k marketplace floor. + qbit_target = target_from_compact("207fffff") + base_job = direct_stratum.DirectQbitStratumJob( + job_id="prism-template-base", + previousblockhash_display="00" * 32, + prevhash="00" * 32, + coinb1="", + coinb2="", + full_coinbase_prefix="", + full_coinbase_suffix="", + merkle_branch=(), + transaction_hexes=(), + version="20000000", + nbits="207fffff", + ntime="6553f100", + qbit_target=qbit_target, + share_target=qbit_target, + share_difficulty=Decimal("1"), + extranonce1_hex="ffffffff", + extranonce2_size=8, + clean_jobs=True, + ) + return CachedJobBundle( + key=("test",), + template=gbt_template("00" * 32), + template_fingerprint="fp", + coinbase_manifest={}, + shares_json=[], + prior_balances=[], + found_block={"network_difficulty": 1}, + collection_only=False, + issued_at_ms=12345, + base_job=base_job, + built_monotonic=time.monotonic(), + ) + + def highdiff_client(self) -> ClientState: + state = client() + state.worker = worker_identity() + state.listener_vardiff_config = highdiff_vardiff_config() + state.minimum_advertised_difficulty = Decimal("500000") + state.share_difficulty = Decimal("500000") + return state + + def test_block_worthy_submission_below_share_target_submits_synchronously(self) -> None: + # With the floor above network difficulty a hash can solve a block + # while missing the advertised share target. It is a valid share only + # if the block lands, so it submits synchronously (not via the async + # queue) and the share credit lands with it. + server, state, ledger = submit_coordinator() submission = SimpleNamespace( header_hex="aa" * 80, - block_hash_hex="dd" * 32, - share_pass=True, - block_pass=False, + block_hash_hex="bb" * 32, + share_pass=False, + block_pass=True, ) + submitted: list[object] = [] + + def fake_submit(candidate: object) -> bool: + submitted.append(candidate) + server.append_accepted_share( + candidate.client, candidate.context, candidate.submission, candidate.pending_share + ) + return True + + server.submit_block_candidate = fake_submit # type: ignore[method-assign] with patch( "lab.prism.prism_coordinator.direct_stratum.assemble_submission", return_value=submission, ): - with self.assertRaisesRegex(StratumError, "commit failed"): - server.handle_submit( - state, - ["miner-a", "job-1", "00" * 8, "00000001", "00000002"], - ) + should_close = server.handle_submit( + state, + ["miner-a", "job-1", "00" * 8, "00000001", "00000002"], + ) + + self.assertFalse(should_close) + self.assertEqual(server.rejection_counts_by_reason[PRISM_REJECTION_LOW_DIFFICULTY], 0) + self.assertEqual(len(submitted), 1) + self.assertTrue(submitted[0].credit_share_on_accept) + # Nothing was queued to the async submitter; it landed inline. + self.assertEqual(server.block_candidate_queue.qsize(), 0) + self.assertEqual(len(ledger.pending), 1) + + def test_below_target_block_intent_is_durable_before_synchronous_submit(self) -> None: + server, state, _recording = submit_coordinator() + ledger = SingleWriterShareLedger() + server.ledger = ledger + submission = SimpleNamespace( + header_hex="aa" * 80, + coinbase_tx_hex="00", + block_hex="00", + block_hash_hex="bc" * 32, + share_pass=False, + block_pass=True, + ) + + def fake_submit(candidate: PrismBlockCandidate) -> bool: + pending = ledger.pending_block_candidates() + self.assertEqual(len(pending), 1) + self.assertTrue(pending[0]["credit_share_on_accept"]) + server.append_accepted_share( + candidate.client, + candidate.context, + candidate.submission, + candidate.pending_share, + candidate_intent=server.block_candidate_intent(candidate), + ) + return True - server.share_writer_active = False - server.ledger = healthy + server.submit_block_candidate = fake_submit # type: ignore[method-assign] with patch( "lab.prism.prism_coordinator.direct_stratum.assemble_submission", return_value=submission, @@ -8831,8481 +6625,5840 @@ def append_batch(self, _entries: object) -> list[object]: ["miner-a", "job-1", "00" * 8, "00000001", "00000002"], ) ) - self.assertEqual(len(healthy.pending), 1) - self.assertEqual(server.worker_share_counts["miner-a"]["accepted"], 1) - server.request_shutdown() - writer.join(timeout=2) - def test_share_writer_retries_failed_append_until_it_lands(self) -> None: - server, state, ledger = submit_coordinator() + self.assertEqual(len(ledger), 1) + self.assertEqual(ledger.pending_block_candidates(), []) - class FlakyLedger(type(ledger)): - def __init__(self) -> None: - super().__init__() - self.failures_remaining = 2 - - def append(self, pending: object) -> object: - if self.failures_remaining > 0: - self.failures_remaining -= 1 - raise RuntimeError("ledger briefly unavailable") - return super().append(pending) - - flaky = FlakyLedger() - server.ledger = flaky - entry = PendingShareAppend( - pending_share=SimpleNamespace( - share_id="miner-a:" + "ee" * 32, - job_issued_at_ms=0, - accepted_at_ms=0, - ), - username="miner-a", - job_id="job-1", - block_hash_hex="ee" * 32, - collection_only=False, - credit_policy=None, - ) - - with patch.object(server.stop_event, "wait", return_value=False) as waited: - server._append_share_entry(entry, retry_until_stopped=True) - - self.assertEqual(len(flaky.pending), 1) - self.assertEqual(server.share_append_failure_count, 2) - self.assertEqual(waited.call_count, 2) - - def _pending_append(self, tag: str, accepted_at_ms: int = 2) -> PendingShareAppend: - from lab.prism.share_ledger import PendingShare - - return PendingShareAppend( - pending_share=PendingShare( - share_id=f"miner-a:{tag}", - miner_id="miner-a", - order_key="miner-a", - p2mr_program_hex="11" * 32, - share_difficulty=1, - network_difficulty=1, - template_height=10, - job_id="job-1", - job_issued_at_ms=1, - accepted_at_ms=accepted_at_ms, - ntime=1_700_000_000, - ), - username="miner-a", - job_id="job-1", - block_hash_hex=tag * 32, - collection_only=False, - credit_policy=None, + def test_below_target_intent_failure_does_not_create_unsafe_retry_slot(self) -> None: + server, state, _recording = submit_coordinator() + ledger = SingleWriterShareLedger() + server.ledger = ledger + submission = SimpleNamespace( + header_hex="aa" * 80, + coinbase_tx_hex="00", + block_hex="00", + block_hash_hex="be" * 32, + share_pass=False, + block_pass=True, ) + submit_calls = 0 - def test_share_append_backlog_overflow_never_reports_success(self) -> None: - server, _state, _ledger = submit_coordinator() - server.share_append_queue = queue.Queue(maxsize=2) - with tempfile.TemporaryDirectory() as tempdir: - server.share_recovery_path = Path(tempdir) / "recovery.jsonl" - - server.enqueue_share_append(self._pending_append("aa")) - server.enqueue_share_append(self._pending_append("bb")) - with self.assertRaisesRegex(StratumError, "queue is full"): - server.enqueue_share_append(self._pending_append("cc")) - - self.assertEqual(server.shares_recovered_to_disk, 0) - remaining = [ - server.share_append_queue.get_nowait().pending_share.share_id - for _ in range(2) - ] - self.assertEqual(remaining, ["miner-a:aa", "miner-a:bb"]) - self.assertFalse(server.share_recovery_path.exists()) - - def test_writer_recovers_acked_share_on_shutdown_during_outage(self) -> None: - server, _state, ledger = submit_coordinator() - with tempfile.TemporaryDirectory() as tempdir: - server.share_recovery_path = Path(tempdir) / "recovery.jsonl" - - class DownLedger(type(ledger)): - def append(self, pending: object) -> object: - raise RuntimeError("postgres unavailable") - - server.ledger = DownLedger() - entry = self._pending_append("ee") - # First backoff wait returns True (stop requested): the share must - # be recovered, not silently dropped. - with patch.object(server.stop_event, "wait", return_value=True): - server._append_share_entry(entry, retry_until_stopped=True) - - self.assertEqual(server.shares_recovered_to_disk, 1) - recovered = json.loads(server.share_recovery_path.read_text().strip()) - self.assertEqual(recovered["share_id"], "miner-a:ee") + def fail_intent(_intent: dict[str, object]) -> bool: + raise RuntimeError("outbox unavailable") - def test_replay_recovered_shares_appends_and_clears_file(self) -> None: - server, _state, ledger = submit_coordinator() - with tempfile.TemporaryDirectory() as tempdir: - server.share_recovery_path = Path(tempdir) / "recovery.jsonl" - server._recover_share_to_disk(self._pending_append("f1"), "test") - server._recover_share_to_disk(self._pending_append("f2"), "test") + def unsafe_submit(_candidate: PrismBlockCandidate) -> bool: + nonlocal submit_calls + submit_calls += 1 + return True - replayed = server.replay_recovered_shares() + ledger.persist_block_candidate_intent = fail_intent # type: ignore[method-assign] + server.submit_block_candidate = unsafe_submit # type: ignore[method-assign] + with patch( + "lab.prism.prism_coordinator.direct_stratum.assemble_submission", + return_value=submission, + ): + with self.assertRaisesRegex(RuntimeError, "outbox unavailable"): + server.handle_submit( + state, + ["miner-a", "job-1", "00" * 8, "00000001", "00000002"], + ) - self.assertEqual(replayed, 2) - self.assertEqual(server.shares_replayed, 2) - self.assertEqual( - [p.share_id for p in ledger.pending], ["miner-a:f1", "miner-a:f2"] - ) - # File is cleared after a clean replay so shares are not re-added. - self.assertFalse(server.share_recovery_path.exists()) - self.assertEqual(server.replay_recovered_shares(), 0) - - def test_replay_recovered_shares_commit_under_the_landing_fence(self) -> None: - # Recovered rows reconstruct pre-crash timestamps, so they predate - # live anchors. Each replay append must hold the landing fence across - # the ledger commit and its epoch bump -- never bump after the row is - # already durable, where a landing could verify the pre-bump epoch - # and submit a coinbase omitting the replayed share. - server, _state, ledger = submit_coordinator() - anchor_token = server._expose_inflight_scan_anchor(12000) - self.addCleanup(server._retire_inflight_scan_anchor, anchor_token) - with tempfile.TemporaryDirectory() as tempdir: - server.share_recovery_path = Path(tempdir) / "recovery.jsonl" - server._recover_share_to_disk(self._pending_append("h1"), "test") + self.assertEqual(submit_calls, 0) + self.assertIsNone(getattr(server, "_retry_block_candidate", None)) + self.assertEqual(ledger.pending_block_candidates(), []) - fence_held_during_commit: list[bool] = [] - original_append = ledger.append + def test_block_worthy_below_target_rejects_low_difficulty_when_block_fails(self) -> None: + # If the block does not land, the below-share-target hash earns nothing + # and the miner is rejected as low-difficulty -- never acked accepted + # with no ledger row. + server, state, ledger = submit_coordinator() + submission = SimpleNamespace( + header_hex="aa" * 80, + block_hash_hex="bb" * 32, + share_pass=False, + block_pass=True, + ) + def reject_candidate(_candidate: PrismBlockCandidate) -> bool: + server._abandon_block_candidate( + PRISM_REJECTION_SUBMITBLOCK_REJECTED, + "rejected", + block_hash=_candidate.submission.block_hash_hex, + worker="miner-a", + ) + return False - def recording_append(pending: object) -> object: - fence_held_during_commit.append( - server._payout_append_landing_fence_lock.locked() + server.submit_block_candidate = reject_candidate # type: ignore[method-assign] + with patch( + "lab.prism.prism_coordinator.direct_stratum.assemble_submission", + return_value=submission, + ): + with self.assertRaises(StratumError) as raised: + server.handle_submit( + state, + ["miner-a", "job-1", "00" * 8, "00000001", "00000002"], ) - return original_append(pending) - ledger.append = recording_append # type: ignore[method-assign] - replayed = server.replay_recovered_shares() - - self.assertEqual(replayed, 1) - self.assertEqual(fence_held_during_commit, [True]) - with server._job_cache_lock: - self.assertEqual(server._payout_ledger_append_invalidation_epoch, 1) + self.assertEqual(raised.exception.reason, PRISM_REJECTION_LOW_DIFFICULTY) + self.assertEqual(len(ledger.pending), 0) + self.assertEqual(server.block_candidate_queue.qsize(), 0) + # The reject is counted (globally and for the worker), not just the + # block-abandonment reason -- this synchronous path used to skip it. + self.assertEqual(server.rejection_counts_by_reason[PRISM_REJECTION_LOW_DIFFICULTY], 1) + self.assertEqual(server.low_difficulty_share_count, 1) - def test_sync_append_commits_predating_row_under_the_landing_fence( - self, - ) -> None: - # The synchronous (no-writer) append path takes the same fence - # boundary as the group-commit writer for a row that predates a - # live anchor. - server, _state, ledger = submit_coordinator() - anchor_token = server._expose_inflight_scan_anchor(12000) - self.addCleanup(server._retire_inflight_scan_anchor, anchor_token) + def test_below_target_transient_outcome_closes_without_definitive_reject(self) -> None: + server, state, _recording = submit_coordinator() + ledger = SingleWriterShareLedger() + server.ledger = ledger + submission = SimpleNamespace( + header_hex="aa" * 80, + coinbase_tx_hex="00", + block_hex="00", + block_hash_hex="bd" * 32, + share_pass=False, + block_pass=True, + ) + server.submit_block_candidate = lambda _candidate: False # type: ignore[method-assign] - fence_held_during_commit: list[bool] = [] - original_append = ledger.append + submit_params = ["miner-a", "job-1", "00" * 8, "00000001", "00000002"] + # Each retry rebuilds its candidate intent with a fresh acknowledgment + # stamp. Force every call onto a new millisecond so the durable-outbox + # idempotency is exercised across acknowledgment-stamp drift instead of + # depending on both attempts landing within the same millisecond. + clock_ms = iter(range(1_700_000_000_000, 1_700_000_070_000, 7)) + with patch( + "lab.prism.prism_coordinator.direct_stratum.assemble_submission", + return_value=submission, + ), patch( + "lab.prism.prism_coordinator.now_ms", + side_effect=clock_ms.__next__, + ): + for _attempt in range(2): + with self.assertRaisesRegex(RuntimeError, "pending durable retry"): + server.handle_submit(state, submit_params) + self.assertEqual(server.recent_share_keys, set()) - def recording_append(pending: object) -> object: - fence_held_during_commit.append( - server._payout_append_landing_fence_lock.locked() - ) - return original_append(pending) + self.assertEqual(server.rejection_counts_by_reason[PRISM_REJECTION_LOW_DIFFICULTY], 0) + self.assertEqual(server.duplicate_share_count, 0) + self.assertEqual(len(ledger), 0) + self.assertEqual(len(ledger.pending_block_candidates()), 1) + self.assertIsNotNone(server._retry_block_candidate) + assert server._retry_block_candidate is not None + self.assertEqual( + server._retry_block_candidate.submission.block_hash_hex, + submission.block_hash_hex, + ) - ledger.append = recording_append # type: ignore[method-assign] - self.assertTrue(server._append_share_entry(self._pending_append("h2"))) - self.assertEqual(fence_held_during_commit, [True]) - with server._job_cache_lock: - self.assertEqual(server._payout_ledger_append_invalidation_epoch, 1) + def test_low_difficulty_submission_without_block_solve_is_rejected(self) -> None: + server, state, ledger = submit_coordinator() + submission = SimpleNamespace( + header_hex="aa" * 80, + block_hash_hex="bb" * 32, + share_pass=False, + block_pass=False, + ) + with patch( + "lab.prism.prism_coordinator.direct_stratum.assemble_submission", + return_value=submission, + ): + with self.assertRaises(StratumError) as raised: + server.handle_submit( + state, + ["miner-a", "job-1", "00" * 8, "00000001", "00000002"], + ) - def test_replay_recovered_shares_orders_by_accepted_at(self) -> None: - # A share can be recovered out of FIFO order (overflow of the newest, or - # a ledger flap during the shutdown drain). Replay must reorder by - # accepted_at_ms so share_seq reflects acceptance order, keeping the - # reward window correctly ordered. - server, _state, ledger = submit_coordinator() - with tempfile.TemporaryDirectory() as tempdir: - server.share_recovery_path = Path(tempdir) / "recovery.jsonl" - server._recover_share_to_disk(self._pending_append("late", accepted_at_ms=300), "test") - server._recover_share_to_disk(self._pending_append("early", accepted_at_ms=100), "test") - server._recover_share_to_disk(self._pending_append("mid", accepted_at_ms=200), "test") + self.assertEqual(raised.exception.code, 23) + self.assertEqual(raised.exception.reason, PRISM_REJECTION_LOW_DIFFICULTY) + self.assertEqual(server.rejection_counts_by_reason[PRISM_REJECTION_LOW_DIFFICULTY], 1) + self.assertEqual(len(ledger.pending), 0) - replayed = server.replay_recovered_shares() + def test_collection_only_below_target_block_solve_submits_solver_pays_all(self) -> None: + # A collection job's signed bootstrap manifest already commits the + # whole coinbase to the submitting worker, so a solved block on a + # collection job is submitted (synchronously here, since the share + # missed its target) instead of being withheld -- the first block on a + # fresh ledger must never be silently ledgered away. + server, state, ledger = submit_coordinator() + server.jobs["job-1"].collection_only = True + submission = SimpleNamespace( + header_hex="aa" * 80, + block_hash_hex="bb" * 32, + share_pass=False, + block_pass=True, + ) + submitted: list[object] = [] - self.assertEqual(replayed, 3) - self.assertEqual( - [p.share_id for p in ledger.pending], - ["miner-a:early", "miner-a:mid", "miner-a:late"], + def fake_submit(candidate: object) -> bool: + submitted.append(candidate) + server.append_accepted_share( + candidate.client, candidate.context, candidate.submission, candidate.pending_share ) + return True - def test_replay_skips_torn_line_and_keeps_file(self) -> None: - # A crash mid-append can leave the last line torn. That one line must - # not block the intact shares before it, and the file is kept so the - # torn line is preserved rather than silently discarded. - server, _state, ledger = submit_coordinator() - with tempfile.TemporaryDirectory() as tempdir: - server.share_recovery_path = Path(tempdir) / "recovery.jsonl" - server._recover_share_to_disk(self._pending_append("g1", accepted_at_ms=100), "test") - server._recover_share_to_disk(self._pending_append("g2", accepted_at_ms=200), "test") - with open(server.share_recovery_path, "a", encoding="utf-8") as handle: - handle.write('{"share_id": "miner-a:torn", "miner_i') # truncated, no newline - - replayed = server.replay_recovered_shares() - - self.assertEqual(replayed, 2) - self.assertEqual( - [p.share_id for p in ledger.pending], ["miner-a:g1", "miner-a:g2"] + server.submit_block_candidate = fake_submit # type: ignore[method-assign] + with patch( + "lab.prism.prism_coordinator.direct_stratum.assemble_submission", + return_value=submission, + ): + should_close = server.handle_submit( + state, + ["miner-a", "job-1", "00" * 8, "00000001", "00000002"], ) - # File kept because a line could not be parsed. - self.assertTrue(server.share_recovery_path.exists()) - # Re-running dedups the good shares (ledger is idempotent by id). - self.assertEqual(server.replay_recovered_shares(), 2) - def test_replay_is_idempotent_across_partial_replay(self) -> None: - # Finding: a partial replay (A commits, B fails transiently) kept the - # whole file; on retry, replay hit A's duplicate and stopped, stranding - # B forever. Replay must skip the already-committed A and reach B. - server, _state, _ledger = submit_coordinator() - with tempfile.TemporaryDirectory() as tempdir: - server.share_recovery_path = Path(tempdir) / "recovery.jsonl" - server._recover_share_to_disk(self._pending_append("A", accepted_at_ms=100), "test") - server._recover_share_to_disk(self._pending_append("B", accepted_at_ms=200), "test") + self.assertFalse(should_close) + self.assertEqual(server.rejection_counts_by_reason[PRISM_REJECTION_LOW_DIFFICULTY], 0) + self.assertEqual(len(submitted), 1) + self.assertTrue(submitted[0].credit_share_on_accept) + self.assertTrue(submitted[0].context.collection_only) + self.assertEqual(len(ledger.pending), 1) + self.assertEqual(server.collection_block_submission_count, 1) - class DedupLedger: - def __init__(self) -> None: - self.ids: list[str] = [] - self.fail_b_once = True - - def append(self, pending: object) -> object: - if pending.share_id == "miner-a:B" and self.fail_b_once: - self.fail_b_once = False - raise RuntimeError("postgres unavailable") - if pending.share_id in self.ids: - raise RuntimeError("duplicate share_id") - self.ids.append(pending.share_id) - return SimpleNamespace(share_seq=len(self.ids)) - - server.ledger = DedupLedger() - - # Pass 1: A commits, B raises a transient (non-duplicate) error, so - # the pass stops and keeps the file. - self.assertEqual(server.replay_recovered_shares(), 1) - self.assertTrue(server.share_recovery_path.exists()) - self.assertEqual(server.ledger.ids, ["miner-a:A"]) - - # Pass 2: A is now a duplicate (skipped, not fatal); B commits and - # the file is cleared. - self.assertEqual(server.replay_recovered_shares(), 1) - self.assertEqual(server.ledger.ids, ["miner-a:A", "miner-a:B"]) - self.assertFalse(server.share_recovery_path.exists()) - - def test_append_share_entry_reports_persisted_vs_recovered(self) -> None: - server, _state, ledger = submit_coordinator() - with tempfile.TemporaryDirectory() as tempdir: - server.share_recovery_path = Path(tempdir) / "recovery.jsonl" - # Healthy ledger: reports persisted. - self.assertTrue( - server._append_share_entry(self._pending_append("ok"), retry_until_stopped=True) + def test_collection_job_block_solve_is_credited_and_enqueued_not_withheld(self) -> None: + # A solved block that also met its share target on a collection job is + # credited immediately and queued for the submitter thread, exactly + # like a ready-window candidate. + server, state, ledger = submit_coordinator() + server.jobs["job-1"].collection_only = True + submission = SimpleNamespace( + header_hex="aa" * 80, + block_hash_hex="cc" * 32, + share_pass=True, + block_pass=True, + ) + with patch( + "lab.prism.prism_coordinator.direct_stratum.assemble_submission", + return_value=submission, + ): + should_close = server.handle_submit( + state, + ["miner-a", "job-1", "00" * 8, "00000001", "00000002"], ) - class DownLedger(type(ledger)): - def append(self, pending: object) -> object: - raise RuntimeError("postgres unavailable") - - server.ledger = DownLedger() - with patch.object(server.stop_event, "wait", return_value=True): - # Shutdown mid-outage: reports recovered, not persisted. - self.assertFalse( - server._append_share_entry(self._pending_append("down"), retry_until_stopped=True) - ) + self.assertFalse(should_close) + self.assertEqual(len(ledger.pending), 1) + self.assertEqual(server.block_candidate_queue.qsize(), 1) + queued = server.block_candidate_queue.get_nowait() + self.assertTrue(queued.context.collection_only) + self.assertFalse(queued.credit_share_on_accept) + self.assertEqual(server.collection_block_submission_count, 1) - def test_group_commit_failure_releases_all_waiters_without_recovery_file(self) -> None: - server, _state, ledger = submit_coordinator() - with tempfile.TemporaryDirectory() as tempdir: - server.share_recovery_path = Path(tempdir) / "recovery.jsonl" - server.share_append_queue = queue.Queue(maxsize=MAX_PENDING_SHARE_APPENDS) - - append_calls: list[str] = [] - - class DownLedger(type(ledger)): - def append(self, pending: object) -> object: - append_calls.append(pending.share_id) - raise RuntimeError("postgres unavailable") - - server.ledger = DownLedger() - entries = [] - for tag in ("s1", "s2", "s3"): - entry = self._pending_append(tag) - entries.append(entry) - server.enqueue_share_append(entry) - - server.request_shutdown() - server.share_append_loop() - - # The compatibility ledger fails on the first row; the whole batch - # is reported failed and no uncommitted share is called durable. - self.assertEqual(append_calls, ["miner-a:s1"]) - self.assertEqual(ledger.pending, []) - self.assertTrue(all(entry.committed.is_set() for entry in entries)) - self.assertTrue(all(entry.error is not None for entry in entries)) - self.assertFalse(server.share_recovery_path.exists()) - - def test_landing_builds_audit_bundle_outside_balance_serializer(self) -> None: - # The builder/verifier subprocess time must stay off the - # payout-balance mutation lock: job delivery and reconciliation - # queue behind that lock, and holding it across the bundle build was - # the dominant term of the finalization delivery stall. - server, state, ledger = submit_coordinator() - server.stop_after_block = False - server.max_blocks = 10 - lock_held_during_build: list[bool] = [] - build_calls: list[int] = [] - - def probe_lock_from_other_thread() -> bool: - # An RLock re-acquires freely on the owning thread, so the probe - # must come from a thread that cannot be the owner. - acquired: list[bool] = [] - - def attempt() -> None: - got = server._payout_balance_mutation_lock.acquire( - blocking=False - ) - if got: - server._payout_balance_mutation_lock.release() - acquired.append(not got) - - prober = threading.Thread(target=attempt) - prober.start() - prober.join(5) - return bool(acquired and acquired[0]) - - def fake_build_audit_bundle(**_kwargs: object) -> dict[str, object]: - build_calls.append(1) - lock_held_during_build.append(probe_lock_from_other_thread()) - return verified_block_bundle() + def test_block_candidate_intent_round_trips_compact_payout_state(self) -> None: + server, state, _ledger = submit_coordinator() + server.jobs["job-1"].collection_only = True + server.jobs["job-1"].prospective_prior_balances = ( + ("miner-a", "miner-a", "11" * 32, 25), + ) + pending = PendingShare( + share_id="miner-a:" + "dd" * 32, + miner_id="miner-a", + order_key="miner-a", + p2mr_program_hex="11" * 32, + share_difficulty=1, + network_difficulty=1, + template_height=9, + job_id="job-1", + job_issued_at_ms=1, + accepted_at_ms=2, + ntime=1, + ) + submission = SimpleNamespace( + coinbase_tx_hex="00", + block_hash_hex="dd" * 32, + block_hex="00", + share_pass=True, + block_pass=True, + ) + candidate = block_candidate(server, state, submission, pending_share=pending) - with tempfile.TemporaryDirectory() as tempdir: - server.audit_dir = Path(tempdir) - server.evidence_path = Path(tempdir) / "evidence.json" - server.ledger_writer_public_key_hex = "aa" * 32 - block_hash = "cc" * 32 - server.rpc = SubmitRpc( - tip="00" * 32, - block_hash=block_hash, - ledger=ledger, - ) - server.build_audit_bundle = fake_build_audit_bundle # type: ignore[method-assign] - server.verify_bundle = ( # type: ignore[method-assign] - lambda *_args, **_kwargs: verified_audit_report() - ) - submission = SimpleNamespace( - coinbase_tx_hex="c0ffee", - block_hash_hex=block_hash, - block_hex="00", - ) - pending = SimpleNamespace(share_id="miner-a:" + block_hash) - accepted = server.submit_block_candidate( - block_candidate(server, state, submission, pending_share=pending) - ) + intent = server.block_candidate_intent(candidate) + self.assertIs(intent["collection_only"], True) + replayed = server.block_candidate_from_intent(intent) + self.assertTrue(replayed.context.collection_only) + self.assertEqual( + replayed.context.prospective_prior_balances, + (("miner-a", "miner-a", "11" * 32, 25),), + ) - self.assertTrue(accepted) - self.assertEqual(build_calls, [1]) - self.assertEqual(lock_held_during_build, [False]) - self.assertEqual(len(ledger.persisted), 1) - self.assertEqual(len(ledger.confirmed), 1) + # Intents persisted before the flag existed replay as ready-window + # candidates, which is all the outbox could ever have contained then. + intent.pop("collection_only") + intent.pop("prospective_prior_balances") + replayed = server.block_candidate_from_intent(intent) + self.assertFalse(replayed.context.collection_only) + self.assertIsNone(replayed.context.prospective_prior_balances) - def test_block_submitted_before_audit_bundle_build(self) -> None: - # The audit build runs with the balance serializer released, but - # never before submitblock: announcement of a freshly solved block - # must not wait on audit construction (a lost race is a lost round). - server, state, ledger = submit_coordinator() - server.stop_after_block = False + def test_below_target_accepted_tail_serializes_moved_tip_replay(self) -> None: + old_tip = "00" * 32 + new_tip = "11" * 32 + block_hash = "b7" * 32 + server, state, _recording = submit_coordinator(tip=old_tip) server.max_blocks = 10 - order: list[str] = [] - block_hash = "cc" * 32 - - class OrderRecordingRpc(SubmitRpc): - def call( - self, - method: str, - params: list[object] | None = None, - ) -> object: - if method == "submitblock": - order.append("submitblock") - return super().call(method, params) - - def ordered_build(**_kwargs: object) -> dict[str, object]: - order.append("build_audit_bundle") - return verified_block_bundle() - - with tempfile.TemporaryDirectory() as tempdir: - server.audit_dir = Path(tempdir) - server.evidence_path = Path(tempdir) / "evidence.json" - server.ledger_writer_public_key_hex = "aa" * 32 - server.rpc = OrderRecordingRpc( - tip="00" * 32, - block_hash=block_hash, - ledger=ledger, - ) - server.build_audit_bundle = ordered_build # type: ignore[method-assign] - server.verify_bundle = ( # type: ignore[method-assign] - lambda *_args, **_kwargs: verified_audit_report() - ) - submission = SimpleNamespace( - coinbase_tx_hex="c0ffee", - block_hash_hex=block_hash, - block_hex="00", - ) - pending = SimpleNamespace(share_id="miner-a:" + block_hash) - accepted = server.submit_block_candidate( - block_candidate(server, state, submission, pending_share=pending) - ) - - self.assertTrue(accepted) - self.assertEqual(order, ["submitblock", "build_audit_bundle"]) - self.assertEqual(len(ledger.persisted), 1) - self.assertEqual(len(ledger.confirmed), 1) - - def test_successful_fast_lane_reconciles_observed_post_submit_tip(self) -> None: - parent_hash = "00" * 32 - block_hash = "cd" * 32 - server, state, ledger = submit_coordinator(tip=parent_hash) server.stop_after_block = False - server.max_blocks = 10 - server.rpc = SubmitAcceptingTemplateRpc( - old_tip=parent_hash, + ledger = SingleWriterShareLedger() + server.ledger = ledger + server.rpc = SubmitRpc( + tip=old_tip, block_hash=block_hash, ledger=ledger, ) - reconciled: list[tuple[str, bool]] = [] - landing_inputs: list[tuple[str, bool]] = [] - - def reconcile( - tip_hash: str, - *, - _coalesce_same_tip: bool = True, - ) -> bool: - reconciled.append((tip_hash, _coalesce_same_tip)) - return True - - original_land = server._land_and_confirm_block_candidate - - def observe_landing( - candidate: PrismBlockCandidate, - **kwargs: object, - ) -> object: - landing_inputs.append( - (str(kwargs["current_tip"]), bool(kwargs["already_active"])) - ) - return original_land(candidate, **kwargs) # type: ignore[arg-type] - - server.ensure_reorg_reconciled_for_tip = reconcile # type: ignore[method-assign] - server._land_and_confirm_block_candidate = observe_landing # type: ignore[method-assign] server.build_audit_bundle = ( # type: ignore[method-assign] lambda **_kwargs: verified_block_bundle() ) server.verify_bundle = ( # type: ignore[method-assign] lambda *_args, **_kwargs: verified_audit_report() ) + server.ledger_writer_public_key_hex = "aa" * 32 + server.refresh_jobs_after_pending_accepted_block = ( # type: ignore[method-assign] + lambda *_args, **_kwargs: 0 + ) submission = SimpleNamespace( + header_hex="b7" * 80, coinbase_tx_hex="c0ffee", - block_hash_hex=block_hash, block_hex="00", + block_hash_hex=block_hash, + share_pass=False, + block_pass=True, ) - with tempfile.TemporaryDirectory() as tempdir: - server.audit_dir = Path(tempdir) - server.evidence_path = Path(tempdir) / "evidence.json" - server.ledger_writer_public_key_hex = "aa" * 32 - accepted = server.submit_block_candidate( - block_candidate(server, state, submission) - ) - - self.assertTrue(accepted) - self.assertEqual(landing_inputs, [(block_hash, False)]) - self.assertEqual(reconciled, [(block_hash, False)]) + durable_confirmation = threading.Event() + accepted_tail_paused = threading.Event() + release_accepted_tail = threading.Event() + replay_guard_blocked = threading.Event() + confirmation_calls: list[str] = [] + submitted: list[str] = [] + abandoned: list[str] = [] + synchronous_results: list[bool] = [] + replay_results: list[bool] = [] + errors: list[BaseException] = [] + replay_thread: threading.Thread | None = None + original_confirm = ledger.confirm_accepted_block + original_stats = server.accepted_share_stats + original_submitted = ledger.mark_block_candidate_submitted + original_abandoned = ledger.mark_block_candidate_abandoned - def test_submitter_offers_block_before_writer_admission_with_rpc_deadline(self) -> None: - server, state, _ledger = submit_coordinator() - server.block_submit_rpc_timeout_seconds = 0.75 - order: list[object] = [] + class ObservedDispositionLock: + def __init__(self) -> None: + self.lock = threading.Lock() - class TimeoutRecordingRpc(TipRpc): - def call( + def acquire( self, - method: str, - params: list[object] | None = None, - *, - timeout: float | None = None, - ) -> object: - if method == "submitblock": - order.append(("submitblock", timeout)) - return None - return super().call(method, params) - - class WriterAdmission: - def __enter__(self) -> None: - order.append("writer-admission") - - def __exit__(self, *_args: object) -> None: - return None + blocking: bool = True, + timeout: float = -1, + ) -> bool: + if threading.current_thread() is replay_thread: + if self.lock.acquire(blocking=False): + return True + # A failed non-blocking acquisition proves the accepted + # attempt still owns this exact same-hash guard. + replay_guard_blocked.set() + if not blocking: + return self.lock.acquire(blocking=False) + if timeout < 0: + return self.lock.acquire() + return self.lock.acquire(timeout=timeout) - candidate = block_candidate( - server, - state, - SimpleNamespace( - coinbase_tx_hex="00", - block_hash_hex="ce" * 32, - block_hex="00", - share_pass=True, - block_pass=True, - ), - ) - server.rpc = TimeoutRecordingRpc("00" * 32) - server._writer_operation = lambda _component: WriterAdmission() # type: ignore[method-assign] + def release(self) -> None: + self.lock.release() - def account( - _candidate: PrismBlockCandidate, + server._ensure_block_candidate_disposition_state() + with server._block_candidate_disposition_registry_lock: + server._block_candidate_disposition_flights[block_hash] = ( # type: ignore[assignment] + SimpleNamespace(lock=ObservedDispositionLock(), users=0) + ) + + def confirm_accepted_block( *, - node_submission: object, - ) -> bool: - order.append("accounting") - self.assertIsNone(getattr(node_submission, "result")) - return True + block_hash: str, + active_tip_height: int, + ) -> dict[str, int | str]: + result = original_confirm( + block_hash=block_hash, + active_tip_height=active_tip_height, + ) + confirmation_calls.append(block_hash) + durable_confirmation.set() + return result - server._submit_next_block_candidate_writer = account # type: ignore[method-assign] - server.enqueue_block_candidate(candidate) + def pause_in_accepted_tail() -> tuple[int, int]: + accepted_tail_paused.set() + if not release_accepted_tail.wait(10): + raise AssertionError("timed out waiting to release accepted success tail") + return original_stats() - self.assertTrue(server.submit_next_block_candidate()) - self.assertEqual( - order, - [("submitblock", 0.75), "writer-admission", "accounting"], - ) + def mark_submitted(*, block_hash: str) -> bool: + submitted.append(block_hash) + return original_submitted(block_hash=block_hash) - def test_block_submit_histogram_measures_landed_to_rpc_interval(self) -> None: - # The race-critical span (candidate landed -> submitblock returned) - # must be observed exactly once per attempted submit, independent of - # the post-submit audit/persistence bookkeeping the outbox - # created_at/completed_at interval also includes. - old_tip = "00" * 32 - block_hash = "ab" * 32 - server, state, ledger = submit_coordinator(tip=old_tip) - server.stop_after_block = False - server.max_blocks = 10 - server.clients = {state} - sent: list[dict[str, object]] = [] - state.send = lambda payload: sent.append(payload) # type: ignore[method-assign] + def mark_abandoned(*, block_hash: str, error: str) -> bool: + abandoned.append(block_hash) + return original_abandoned(block_hash=block_hash, error=error) - def build_fresh_job(client: ClientState, *, clean_jobs: bool) -> object: - return prism_context( - "fresh-job", - block_hash, - worker=state.worker, - difficulty=Decimal("1"), - clean_jobs=clean_jobs, - ) + ledger.confirm_accepted_block = confirm_accepted_block # type: ignore[method-assign] + ledger.mark_block_candidate_submitted = mark_submitted # type: ignore[method-assign] + ledger.mark_block_candidate_abandoned = mark_abandoned # type: ignore[method-assign] + server.accepted_share_stats = pause_in_accepted_tail # type: ignore[method-assign] - server.build_job_for_client = build_fresh_job # type: ignore[method-assign] - submission = SimpleNamespace( - header_hex="aa" * 80, - share_pass=True, - block_pass=True, - coinbase_tx_hex="c0ffee", - block_hash_hex=block_hash, - block_hex="00", - ) + def submit_synchronously() -> None: + try: + synchronous_results.append( + server.handle_submit( + state, + [ + "miner-a", + "job-1", + "00" * 8, + "00000001", + "00000002", + ], + ) + ) + except BaseException as exc: # noqa: BLE001 - asserted below + errors.append(exc) - with tempfile.TemporaryDirectory() as tempdir: + def replay_pending_candidate() -> None: + try: + replay_results.append(server.submit_next_block_candidate()) + except BaseException as exc: # noqa: BLE001 - asserted below + errors.append(exc) + + with tempfile.TemporaryDirectory() as tempdir, patch( + "lab.prism.prism_coordinator.direct_stratum.assemble_submission", + return_value=submission, + ): server.audit_dir = Path(tempdir) server.evidence_path = Path(tempdir) / "evidence.json" - server.ledger_writer_public_key_hex = "aa" * 32 - server.rpc = SubmitAcceptingTemplateRpc( - old_tip=old_tip, - block_hash=block_hash, - ledger=ledger, - ) - server.build_audit_bundle = lambda **_kwargs: verified_block_bundle() # type: ignore[method-assign] - server.verify_bundle = lambda *_args, **_kwargs: verified_audit_report() # type: ignore[method-assign] - - with patch( - "lab.prism.prism_coordinator.direct_stratum.assemble_submission", - return_value=submission, - ): - server.handle_submit( - state, - ["miner-a", "job-1", "00" * 8, "00000001", "00000002"], + synchronous_thread = threading.Thread(target=submit_synchronously) + synchronous_thread.start() + try: + self.assertTrue( + accepted_tail_paused.wait(5), + msg=f"synchronous submit exited early: {errors!r}", ) - self.assertTrue(server.submit_next_block_candidate()) + self.assertTrue(durable_confirmation.is_set()) + self.assertEqual(len(ledger), 1) + self.assertEqual(len(ledger.pending_block_candidates()), 1) + with server.lock: + self.assertNotIn( + block_hash, + server._accounted_accepted_block_hashes, + ) + + self.assertEqual(server.replay_pending_block_candidates(), 1) + server.rpc = TipRpc(new_tip) + replay_thread = threading.Thread(target=replay_pending_candidate) + replay_thread.start() + self.assertTrue(replay_guard_blocked.wait(5)) + self.assertTrue(replay_thread.is_alive()) + self.assertEqual(submitted, []) + self.assertEqual(abandoned, []) + finally: + release_accepted_tail.set() + synchronous_thread.join(10) + if replay_thread is not None: + replay_thread.join(10) + self.assertFalse(synchronous_thread.is_alive()) + self.assertIsNotNone(replay_thread) + assert replay_thread is not None + self.assertFalse(replay_thread.is_alive()) + if errors: + raise errors[0] + self.assertEqual(synchronous_results, [False]) + self.assertEqual(replay_results, [True]) + self.assertEqual(confirmation_calls, [block_hash]) + self.assertGreaterEqual(len(submitted), 1) + self.assertEqual(abandoned, []) + self.assertEqual(len(ledger), 1) + self.assertEqual(ledger.pending_block_candidates(), []) self.assertEqual(server.accepted_block_count, 1) - metrics = "\n".join(server.block_submitter_metrics_lines()) - self.assertIn("qbit_prism_block_submit_seconds_count 1", metrics) - self.assertIn( - 'qbit_prism_block_submit_seconds_bucket{le="+Inf"} 1', - metrics, + self.assertIn(block_hash, server._accounted_accepted_block_hashes) + self.assertNotIn( + PRISM_REJECTION_STALE_JOB, + server.block_candidate_abandoned_counts, ) - # The candidate landed moments ago in this process, so the interval - # must fall inside the finite buckets, not only the +Inf overflow. - self.assertIn( - 'qbit_prism_block_submit_seconds_bucket{le="30"} 1', - metrics, + self.assertNotIn(block_hash, server._accepted_block_payout_previews) + self.assertNotIn( + block_hash, + server._invalidated_accepted_block_payout_previews, ) + self.assertEqual(server._block_candidate_disposition_flights, {}) - def test_orphaned_block_candidate_keeps_share_credit(self) -> None: - # Option-A semantics: a share that met its target stays credited even - # when its block candidate loses the tip race in the submitter. + def _drive_interleaved_distinct_hash_landings( + self, + *, + lease_error: BaseException | None = None, + lease_error_only_inside_publication_guard: bool = False, + ) -> SimpleNamespace: + """Interleave two distinct-hash landings across the production tails. + + Block A lands on the synchronous miner-connection tail and pauses in + its confirm->publish gap (the accepted-share-stats aggregate) with the + payout balance lock and the publication order guard released. Block B + lands through the production submitter->accounting-actor handoff + inside that gap and publishes first, so A reaches publication already + superseded on the publication ordinal. ``lease_error`` is raised from + the coordinator's fresh-lease fence when provided, modelling a writer + whose lease authority cannot be proven inside the gap. With + ``lease_error_only_inside_publication_guard`` the restore-authority + proof succeeds anywhere outside the store's publication order guard + and fails inside it, modelling a writer deposed while queued on the + publication locks. + """ old_tip = "00" * 32 - new_tip = "11" * 32 + hash_a = "a1" * 32 + hash_b = "b2" * 32 server, state, _recording = submit_coordinator(tip=old_tip) + server.max_blocks = 10 + server.stop_after_block = False ledger = SingleWriterShareLedger() server.ledger = ledger - submission = SimpleNamespace( - header_hex="aa" * 80, - block_hash_hex="cc" * 32, - block_hex="00", - share_pass=True, - block_pass=True, + server.build_audit_bundle = ( # type: ignore[method-assign] + lambda **_kwargs: verified_block_bundle() ) - - with patch( - "lab.prism.prism_coordinator.direct_stratum.assemble_submission", - return_value=submission, - ): - server.handle_submit( - state, - ["miner-a", "job-1", "00" * 8, "00000001", "00000002"], + server.verify_bundle = ( # type: ignore[method-assign] + lambda *_args, **kwargs: verified_audit_report( + block_height=int(kwargs["expected_block_height"]) ) + ) + server.ledger_writer_public_key_hex = "aa" * 32 + server.refresh_jobs_after_pending_accepted_block = ( # type: ignore[method-assign] + lambda *_args, **_kwargs: 0 + ) + publication_guard_holders: set[int] = set() + if lease_error is not None: - self.assertEqual(len(ledger.all_shares()), 1) - # The tip moves before the submitter drains the candidate. - server.rpc = RejectingSubmitTipRpc(new_tip) - - self.assertTrue(server.submit_next_block_candidate()) + def failing_lease_fence(component: str) -> None: + if lease_error_only_inside_publication_guard: + if component != "superseded_audit_envelope_restore": + return + if threading.get_ident() not in publication_guard_holders: + return + raise lease_error - self.assertEqual(server.accepted_block_count, 0) - # Counted as a block abandonment, not a share rejection. - self.assertEqual(server.block_candidate_abandoned_counts[PRISM_REJECTION_STALE_JOB], 1) - self.assertEqual( - server.stale_job_abandon_counts, - {"tip_moved": 1, "balance_stale": 0, "append_epoch_stale": 0}, - ) - self.assertEqual(server.stale_share_count, 0) - # The credited share survives the lost block race. - self.assertEqual(len(ledger.all_shares()), 1) - outbox_row = ledger._block_candidate_outbox[submission.block_hash_hex] - self.assertEqual(outbox_row["state"], "abandoned") - self.assertEqual( - outbox_row["last_error"], - f"tip moved before submit: {new_tip}", + server._require_fresh_ledger_lease_for_external_side_effect = ( # type: ignore[method-assign] + failing_lease_fence + ) + submission_a = SimpleNamespace( + header_hex="a1" * 80, + coinbase_tx_hex="c0ffee", + block_hex="00", + block_hash_hex=hash_a, + share_pass=False, + block_pass=True, ) - metrics = server.metrics_payload() - self.assertIn( - 'qbit_prism_stale_job_abandons_total{class="tip_moved"} 1', - metrics, - ) - self.assertIn( - 'qbit_prism_stale_job_abandons_total{class="balance_stale"} 0', - metrics, - ) - - def test_accepted_candidate_with_moved_tip_finalizes_as_submitted(self) -> None: - old_tip = "00" * 32 - new_tip = "11" * 32 - block_hash = "cc" * 32 - server, state, _recording = submit_coordinator(tip=old_tip) - ledger = SingleWriterShareLedger() - server.ledger = ledger - pending = self._pending_append("accepted-tip-race").pending_share - candidate = block_candidate( - server, - state, - SimpleNamespace( - coinbase_tx_hex="00", - block_hash_hex=block_hash, - block_hex="00", - share_pass=True, - block_pass=True, + context_b = SimpleNamespace( + job=SimpleNamespace( + job_id="job-2", + share_target=target_from_compact("207fffff"), + share_difficulty=Decimal("1"), + transaction_hexes=(), ), - pending_share=pending, + template={ + "previousblockhash": hash_a, + "height": 11, + "coinbasevalue": 50_00000000, + }, + found_block={"network_difficulty": 1}, + issued_at_ms=12345, + collection_only=False, + worker=state.worker, + shares_json=[], + prior_balances=[], ) - ledger.append_batch( - [(pending, server.block_candidate_intent(candidate))] + server.jobs["job-2"] = context_b + submission_b = SimpleNamespace( + header_hex="b2" * 80, + coinbase_tx_hex="c0ffee", + block_hex="01", + block_hash_hex=hash_b, + share_pass=True, + block_pass=True, ) - submitted: list[str] = [] - abandoned: list[str] = [] - original_submitted = ledger.mark_block_candidate_submitted - original_abandoned = ledger.mark_block_candidate_abandoned - - def mark_submitted(*, block_hash: str) -> bool: - submitted.append(block_hash) - return original_submitted(block_hash=block_hash) - - def mark_abandoned(*, block_hash: str, error: str) -> bool: - abandoned.append(block_hash) - return original_abandoned(block_hash=block_hash, error=error) - ledger.mark_block_candidate_submitted = mark_submitted # type: ignore[method-assign] - ledger.mark_block_candidate_abandoned = mark_abandoned # type: ignore[method-assign] - server._begin_accepted_block_payout_preview( - block_hash, - block_height=10, - ) - server._mark_accepted_block_payout_landed( - block_hash, - block_height=10, - ) - with server.lock: - # This is the process-local record written immediately before the - # "qbit accepted direct PRISM block" log. - server._accounted_accepted_block_hashes.add(block_hash) - server.rpc = TipRpc(new_tip) - server.build_audit_bundle = ( # type: ignore[method-assign] - lambda **_kwargs: (_ for _ in ()).throw( - AssertionError("an accounted candidate must not be rebuilt") - ) - ) - server.enqueue_block_candidate(candidate) + class TwoBlockRpc: + def __init__(self) -> None: + self.tip = old_tip + self.height = 9 + self.hashes = {9: old_tip} + self.submitted: list[str] = [] - self.assertTrue(server.submit_next_block_candidate()) + def call(self, method: str, params: object = None) -> object: + if method == "getbestblockhash": + return self.tip + if method == "getblockcount": + return self.height + if method == "submitblock": + block_hex = str((params or [""])[0]) # type: ignore[index] + block_hash = hash_a if block_hex == "00" else hash_b + self.height += 1 + self.tip = block_hash + self.hashes[self.height] = block_hash + self.submitted.append(block_hash) + return None + if method == "getblockhash": + return self.hashes[int((params or [0])[0])] # type: ignore[index] + raise RuntimeError(method) - self.assertEqual(submitted, [block_hash]) - self.assertEqual(abandoned, []) - self.assertEqual(ledger.pending_block_candidates(), []) - self.assertNotIn( - PRISM_REJECTION_STALE_JOB, - server.block_candidate_abandoned_counts, - ) - self.assertNotIn(block_hash, server._accepted_block_payout_previews) - self.assertNotIn( - block_hash, - server._invalidated_accepted_block_payout_previews, - ) + server.rpc = TwoBlockRpc() - def test_block_candidate_queue_overflow_coalesces_wakeup_without_drop(self) -> None: - server, state, _ledger = submit_coordinator() - server.block_candidate_queue = queue.Queue(maxsize=2) + sync_gap_open = threading.Event() + release_sync_gap = threading.Event() + confirmed_hashes: list[str] = [] + synchronous_results: list[bool] = [] + sync_errors: list[BaseException] = [] + actor_errors: list[BaseException] = [] + synchronous_thread: threading.Thread | None = None + original_confirm = ledger.confirm_accepted_block + original_stats = server.accepted_share_stats - def candidate(tag: str) -> PrismBlockCandidate: - return block_candidate( - server, - state, - SimpleNamespace(block_hash_hex=tag * 32, share_pass=True, block_pass=True), + def confirm_accepted_block( + *, + block_hash: str, + active_tip_height: int, + ) -> dict[str, int | str]: + result = original_confirm( + block_hash=block_hash, + active_tip_height=active_tip_height, ) + confirmed_hashes.append(block_hash) + return result - server.enqueue_block_candidate(candidate("aa")) - server.enqueue_block_candidate(candidate("bb")) - server.enqueue_block_candidate(candidate("cc")) + def pause_synchronous_gap() -> tuple[int, int]: + # Only A's synchronous tail pauses; B's accounting-actor landing + # runs the same aggregate inside the gap unimpeded. + if threading.current_thread() is synchronous_thread: + sync_gap_open.set() + if not release_sync_gap.wait(10): + raise AssertionError( + "timed out waiting to release the synchronous gap" + ) + return original_stats() - self.assertEqual(server.block_candidates_dropped, 0) - self.assertEqual(server.block_candidate_queue.qsize(), 2) - remaining = [ - server.block_candidate_queue.get_nowait().submission.block_hash_hex - for _ in range(2) - ] - # Existing wakeups remain ordered; the third candidate remains durable - # in the outbox and will be re-read after the queue drains. - self.assertEqual(remaining, ["aa" * 32, "bb" * 32]) - self.assertIn( - "qbit_prism_block_candidates_dropped_total 0", server.metrics_payload() - ) - self.assertIn( - "qbit_prism_block_candidate_wakeups_coalesced_total 1", - server.metrics_payload(), - ) + ledger.confirm_accepted_block = confirm_accepted_block # type: ignore[method-assign] + server.accepted_share_stats = pause_synchronous_gap # type: ignore[method-assign] - def test_durable_block_candidates_replay_after_queue_drains(self) -> None: - server, state, _recording = submit_coordinator() - ledger = SingleWriterShareLedger() - server.ledger = ledger - server.block_candidate_queue = queue.Queue(maxsize=2) + def submit_synchronously() -> None: + try: + synchronous_results.append( + server.handle_submit( + state, + [ + "miner-a", + "job-1", + "00" * 8, + "00000001", + "00000002", + ], + ) + ) + except BaseException as exc: # noqa: BLE001 - asserted below + sync_errors.append(exc) - for index, tag in enumerate(("aa", "bb", "cc"), start=1): - pending = PendingShare( - share_id=f"miner-a:{tag * 32}", - miner_id="miner-a", - order_key="miner-a", - p2mr_program_hex="11" * 32, - share_difficulty=1, - network_difficulty=1, - template_height=9, - job_id="job-1", - job_issued_at_ms=1, - accepted_at_ms=index, - ntime=1, + with tempfile.TemporaryDirectory() as tempdir, patch( + "lab.prism.prism_coordinator.direct_stratum.assemble_submission", + return_value=submission_a, + ): + server.audit_dir = Path(tempdir) + server.evidence_path = Path(tempdir) / "evidence.json" + audit_store = server._ensure_audit_artifact_store() + if lease_error_only_inside_publication_guard: + real_publication_order_guard = audit_store.publication_order_guard + + @contextlib.contextmanager + def tracking_publication_order_guard(): + with real_publication_order_guard(): + publication_guard_holders.add(threading.get_ident()) + try: + yield + finally: + publication_guard_holders.discard( + threading.get_ident() + ) + + audit_store.publication_order_guard = ( # type: ignore[method-assign] + tracking_publication_order_guard + ) + envelope_a = audit_store.live_envelope_path( + block_height=10, + block_hash=hash_a, ) - submission = SimpleNamespace( - coinbase_tx_hex="00", - block_hash_hex=tag * 32, - block_hex="00", - share_pass=True, - block_pass=True, + envelope_b = audit_store.live_envelope_path( + block_height=11, + block_hash=hash_b, ) - candidate = block_candidate( - server, state, submission, pending_share=pending + synchronous_thread = threading.Thread(target=submit_synchronously) + synchronous_thread.start() + actor_thread: threading.Thread | None = None + try: + self.assertTrue( + sync_gap_open.wait(5), + msg=f"synchronous submit exited early: {sync_errors!r}", + ) + # A is durably confirmed with ordinal 1 and both the payout + # balance lock and the publication order guard are released, + # but its live envelope is not yet written. + self.assertEqual(confirmed_hashes, [hash_a]) + self.assertEqual(ledger.audit_publication_sequence_floor(), 1) + self.assertFalse(envelope_a.exists()) + + # B rides the production handoff: the submitter claims the + # disposition, offers the block to the node, and transfers the + # lease to the accounting actor, which lands and publishes. + server.enqueue_block_candidate( + block_candidate(server, state, submission_b, job_id="job-2") + ) + handoff: list[object] = [] + server._enqueue_block_accounting_task = ( # type: ignore[method-assign] + lambda task: (handoff.append(task), True)[1] + ) + self.assertTrue( + server.submit_next_block_candidate(defer_accounting=True) + ) + self.assertEqual(len(handoff), 1) + + def run_accounting_actor() -> None: + try: + server._run_block_accounting_task(handoff[0]) + except BaseException as exc: # noqa: BLE001 - asserted below + actor_errors.append(exc) + + actor_thread = threading.Thread(target=run_accounting_actor) + actor_thread.start() + actor_thread.join(10) + self.assertFalse(actor_thread.is_alive()) + self.assertEqual(actor_errors, []) + self.assertEqual(confirmed_hashes, [hash_a, hash_b]) + self.assertEqual(ledger.audit_publication_sequence_floor(), 2) + self.assertTrue(envelope_b.exists()) + # The interleave is real: B published while A had not. + self.assertFalse(envelope_a.exists()) + finally: + release_sync_gap.set() + synchronous_thread.join(10) + if actor_thread is not None: + actor_thread.join(10) + + self.assertFalse(synchronous_thread.is_alive()) + if sync_errors: + raise sync_errors[0] + # Whatever happened to A's envelope, the publication itself must + # have completed on both tails. + self.assertEqual(synchronous_results, [False]) + self.assertEqual(server.accepted_block_count, 2) + + # The current evidence reference still names the highest + # published sequence; a superseded publication never moves it. + self.assertTrue(envelope_b.exists()) + latest = audit_store.latest_evidence() + assert latest is not None + self.assertEqual(latest["block_hash"], hash_b) + self.assertEqual( + latest["audit_publication_identity"]["sequence"], + 2, + ) + return SimpleNamespace( + hash_a=hash_a, + envelope_a_exists=envelope_a.exists(), + landed_a=( + json.loads(envelope_a.read_text(encoding="utf-8")) + if envelope_a.exists() + else None + ), ) - ledger.append_batch([(pending, server.block_candidate_intent(candidate))]) - self.assertEqual(server.replay_pending_block_candidates(), 3) - self.assertTrue(server.block_candidate_queue.empty()) - first = server._block_replay_candidate_queue.get_nowait() - second = server._block_replay_candidate_queue.get_nowait() - third = server._block_replay_candidate_queue.get_nowait() - self.assertEqual( - [ - first.submission.block_hash_hex, - second.submission.block_hash_hex, - third.submission.block_hash_hex, - ], - ["aa" * 32, "bb" * 32, "cc" * 32], + def test_interleaved_distinct_hash_landings_write_both_live_envelopes(self) -> None: + """Both live envelopes survive an interleaved distinct-hash landing. + + A's own height+hash live envelope is its block's only published + evidence pointer and must be written even though A publishes already + superseded; skipping it was the silent permanent loss of the original + defect. This pins the production opt-in wiring end to end: with + ``restore_superseded_envelope=False`` at the publish call site, this + test fails. + """ + result = self._drive_interleaved_distinct_hash_landings() + self.assertTrue(result.envelope_a_exists) + assert result.landed_a is not None + self.assertEqual(result.landed_a["block_hash"], result.hash_a) + self.assertEqual(result.landed_a["block_height"], 10) + + def test_withheld_lease_authority_skips_restore_without_failing_publication( + self, + ) -> None: + """A failed lease proof degrades the restore; it never fails the landing. + + The fresh-lease fence gates only the superseded-envelope restore + authority. When it raises, the publication must still complete + exactly as before the restore existed -- withholding degrades to the + historical behaviour of skipping the superseded envelope write, + because failing the publication on a lease hiccup would be a live + regression on the found-block path. + """ + from lab.prism.share_ledger import WriterLeaseRenewalDeferred + + result = self._drive_interleaved_distinct_hash_landings( + lease_error=WriterLeaseRenewalDeferred( + "writer lease renewal is deferred behind an in-flight write" + ), ) - ledger.mark_block_candidate_submitted(block_hash="aa" * 32) - ledger.mark_block_candidate_abandoned(block_hash="bb" * 32, error="stale") - self.assertEqual(third.pending_share.share_id, "miner-a:" + "cc" * 32) + self.assertFalse(result.envelope_a_exists) - def test_candidate_intent_avoids_duplicate_template_transaction_bodies(self) -> None: - server, state, _ledger = submit_coordinator() - witness_tx = synthetic_witness_transaction("55") - server.jobs["job-1"].template["transactions"] = [{"data": witness_tx}] - server.jobs["job-1"].job.transaction_hexes = (witness_tx,) - pending = self._pending_append("ca").pending_share - candidate = block_candidate( - server, - state, - SimpleNamespace( - coinbase_tx_hex="00", - block_hash_hex="ca" * 32, - block_hex="00" + witness_tx, - share_pass=True, - block_pass=True, + def test_restore_authority_is_proven_after_the_publication_locks( + self, + ) -> None: + """Deposal during the publication lock wait withholds the restore. + + The fresh-lease proof once ran before the payout balance lock and + the publication order guard were acquired -- a check-then-act across + an unbounded wait, so a writer deposed while queued on those locks + still restored. This fence models that deposal: the proof succeeds + anywhere outside the publication order guard and fails inside it, + so the restore is withheld only if authority is evaluated at the + moment of action, with the locks held. + """ + from lab.prism.share_ledger import WriterLeaseRenewalDeferred + + result = self._drive_interleaved_distinct_hash_landings( + lease_error=WriterLeaseRenewalDeferred( + "writer lease was deposed during the publication lock wait" ), - pending_share=pending, + lease_error_only_inside_publication_guard=True, ) + self.assertFalse(result.envelope_a_exists) - intent = server.block_candidate_intent(candidate) - self.assertEqual( - set(intent["template"]), - {"previousblockhash", "height", "coinbasevalue"}, - ) - self.assertNotIn("transaction_hexes", intent) - self.assertEqual( - intent["witness_merkle_leaves_hex"], - direct_stratum.witness_merkle_leaves_hex((witness_tx,)), - ) +class PrismCoordinatorAcceptedBlockGapTests(unittest.TestCase): + """Blockwait-first acceptance must survive every abandon-capable path. - def test_transient_candidate_failure_remains_pending_for_retry(self) -> None: - server, state, _recording = submit_coordinator() - ledger = SingleWriterShareLedger() - server.ledger = ledger - pending = PendingShare( - share_id="miner-a:" + "aa" * 32, - miner_id="miner-a", - order_key="miner-a", - p2mr_program_hex="11" * 32, - share_difficulty=1, - network_difficulty=1, - template_height=9, - job_id="job-1", - job_issued_at_ms=1, - accepted_at_ms=2, - ntime=1, + Regression coverage for the accepted-block blind spot left open by #89: + the accepted registry was only written in the direct-accept submit tail, + so a candidate whose submitblock ack was lost -- acceptance arriving via + a blockwait tip observation instead -- could be terminally abandoned as + stale-job by a retry that read one racing chain snapshot. The abandon + withdrew the landed payout preview, fenced payout publication, and left + template refreshes coordination-blocked until the watchdog restart. + """ + + def _accepted_tail_scaffolding(self, server: PrismCoordinator, tempdir: str) -> tuple[list[str], list[str]]: + server._ensure_job_cache_state() + server.reorg_reconciler_enabled = False + server.block_candidate_retry_initial_seconds = 0.0 + server.audit_dir = Path(tempdir) + server.evidence_path = Path(tempdir) / "evidence.json" + server.ledger_writer_public_key_hex = "aa" * 32 + server.build_audit_bundle = ( # type: ignore[method-assign] + lambda **_kwargs: verified_block_bundle() ) - candidate = block_candidate( - server, - state, - SimpleNamespace( - coinbase_tx_hex="00", - block_hash_hex="aa" * 32, - block_hex="00", - share_pass=True, - block_pass=True, - ), - pending_share=pending, + server.verify_bundle = ( # type: ignore[method-assign] + lambda *_args, **kwargs: verified_audit_report( + block_height=int(kwargs["expected_block_height"]) + ) ) - intent = server.block_candidate_intent(candidate) - ledger.append_batch([(pending, intent)]) - server.enqueue_block_candidate(candidate) - server.submit_block_candidate = ( # type: ignore[method-assign] - lambda _candidate: (_ for _ in ()).throw(RuntimeError("rpc unavailable")) + submitted: list[str] = [] + abandoned: list[str] = [] + server.ledger.mark_block_candidate_submitted = ( # type: ignore[attr-defined] + lambda **kwargs: submitted.append(str(kwargs["block_hash"])) or True ) - - self.assertTrue(server.submit_next_block_candidate()) - - self.assertEqual(ledger.pending_block_candidates(), [intent]) - self.assertEqual(server.block_candidate_abandoned_counts, {}) - self.assertIn( - "qbit_prism_block_candidate_retries_total 1", - server.metrics_payload(), + server.ledger.mark_block_candidate_abandoned = ( # type: ignore[attr-defined] + lambda **kwargs: abandoned.append(str(kwargs["block_hash"])) or True ) + return submitted, abandoned - def test_retryable_parent_stays_ahead_of_queued_child(self) -> None: - server, state, _ledger = submit_coordinator() - server.block_candidate_retry_initial_seconds = 0 - server.block_candidate_retry_max_seconds = 0 - parent = block_candidate( - server, - state, - SimpleNamespace( - coinbase_tx_hex="00", - block_hash_hex="a1" * 32, - block_hex="00", - share_pass=True, - block_pass=True, - ), - ) - child = block_candidate( - server, - state, - SimpleNamespace( - coinbase_tx_hex="00", - block_hash_hex="b1" * 32, - block_hex="01", - share_pass=True, - block_pass=True, - ), - ) - attempts: list[str] = [] + def _age_retained_acceptance_past_window( + self, server: PrismCoordinator + ) -> None: + """Backdate any stashed definitive ack past the acceptance window. - def submit(candidate: PrismBlockCandidate) -> bool: - block_hash = str(candidate.submission.block_hash_hex) - attempts.append(block_hash) - if block_hash == "a1" * 32 and attempts.count(block_hash) == 1: - server._abandon_block_candidate( - PRISM_REJECTION_BACKEND_RPC_UNAVAILABLE, - "temporary parent finalization failure", - block_hash=block_hash, - worker="miner-a", - ) - return False - return True + These interleavings pin the terminal machinery that runs once + first-party acceptance evidence has gone stale; a fresh stash + correctly vetoes the terminal decision and defers instead (see + test_definitive_ack_with_unprovable_chain_view_defers_not_abandons). + """ + real_stash = server._stash_retained_block_candidate_node_submission - server.submit_block_candidate = submit # type: ignore[method-assign] - self.assertTrue(server.enqueue_block_candidate(parent)) - self.assertTrue(server.enqueue_block_candidate(child)) + def stash_then_age(block_hash: str, node_submission: object) -> None: + real_stash(block_hash, node_submission) + with server.lock: + stamped = getattr( + server, + "_block_candidate_retained_submission_monotonic", + None, + ) + if stamped and block_hash.lower() in stamped: + stamped[block_hash.lower()] -= 400.0 - self.assertTrue(server.submit_next_block_candidate()) - self.assertIs(server._retry_block_candidate, parent) - self.assertEqual(server.block_candidate_queue.qsize(), 1) - self.assertTrue(server.submit_next_block_candidate()) - self.assertIsNone(getattr(server, "_retry_block_candidate", None)) - self.assertEqual(server.block_candidate_queue.qsize(), 1) - self.assertTrue(server.submit_next_block_candidate()) + server._stash_retained_block_candidate_node_submission = stash_then_age # type: ignore[method-assign] - self.assertEqual(attempts, ["a1" * 32, "a1" * 32, "b1" * 32]) - self.assertTrue(server.block_candidate_queue.empty()) + def _retained_candidate(self, server: PrismCoordinator) -> PrismBlockCandidate: + with server.lock: + candidate = server._retry_block_candidate + server._retry_block_candidate = None + self.assertIsNotNone(candidate) + return candidate - def test_candidate_retry_backoff_is_capped_and_cleared_on_success(self) -> None: - server, state, _recording = submit_coordinator() - ledger = SingleWriterShareLedger() - server.ledger = ledger - pending = self._pending_append("retry-success").pending_share - candidate = block_candidate( - server, - state, - SimpleNamespace( - coinbase_tx_hex="00", - block_hash_hex="a1" * 32, - block_hex="00", - share_pass=True, - block_pass=True, + def _chained_context( + self, + server: PrismCoordinator, + *, + job_id: str, + parent_hash: str, + height: int, + ) -> None: + base = server.jobs["job-1"] + server.jobs[job_id] = SimpleNamespace( + job=SimpleNamespace( + job_id=job_id, + share_target=base.job.share_target, + share_difficulty=base.job.share_difficulty, + transaction_hexes=(), ), - pending_share=pending, + template={ + "previousblockhash": parent_hash, + "height": height, + "coinbasevalue": 50_00000000, + }, + found_block={"network_difficulty": 1}, + issued_at_ms=12345, + collection_only=False, + worker=base.worker, + shares_json=[], + prior_balances=[], ) - ledger.append_batch([(pending, server.block_candidate_intent(candidate))]) - server.block_candidate_retry_initial_seconds = 0.1 - server.block_candidate_retry_max_seconds = 0.4 - attempts = 0 - def retry_then_succeed(_candidate: PrismBlockCandidate) -> bool: - nonlocal attempts - attempts += 1 - if attempts <= 4: - server._defer_block_candidate( - PRISM_REJECTION_BACKEND_RPC_UNAVAILABLE, - "temporary RPC outage", - worker="miner-a", - ) - return False - return True + def test_blockwait_observed_acceptance_survives_stale_chain_probe(self) -> None: + # The 2026-07-25 mainnet interleaving, end to end: lost submitblock + # ack, blockwait reports the pool's own hash as the new tip, and the + # retry disposition reads a racing snapshot in which the tip moved on + # and the header probe cannot prove the candidate active. The + # candidate must defer (never terminally abandon), keep its landed + # payout preview, and finalize as submitted once the view settles -- + # with no "qbit accepted direct" tail having run before the defer. + parent = "00" * 32 + block_hash = "b1" * 32 + racing_tip = "77" * 32 + server, state, ledger = submit_coordinator(tip=parent) + server.max_blocks = 2 + server.stop_after_block = False + with tempfile.TemporaryDirectory() as tempdir: + submitted, abandoned = self._accepted_tail_scaffolding(server, tempdir) + rpc = LostAckSubmitRpc( + start_tip=parent, + hash_by_hex={"00": block_hash}, + ) + server.rpc = rpc + candidate = block_candidate( + server, + state, + SimpleNamespace( + coinbase_tx_hex="c0ffee", + block_hash_hex=block_hash, + block_hex="00", + share_pass=True, + block_pass=True, + ), + ) - server.submit_block_candidate = retry_then_succeed # type: ignore[method-assign] - waits: list[float] = [] - with patch.object( - server.stop_event, - "wait", - side_effect=lambda delay: waits.append(delay) or False, - ): - for _attempt in range(5): - server.enqueue_block_candidate(candidate) - self.assertTrue(server.submit_next_block_candidate()) - - self.assertEqual(len(waits), 6) - for observed, expected in zip( - waits, - [0.1, 0.2, 0.25, 0.15, 0.25, 0.15], - strict=True, - ): - self.assertAlmostEqual(observed, expected) - self.assertNotIn(candidate.submission.block_hash_hex, server.block_candidate_retry_delays) - self.assertEqual(server.block_candidate_abandoned_counts, {}) - self.assertEqual(ledger.pending_block_candidates(), []) + # Attempt 1: submitblock lands the block but the ack is lost. + self.assertTrue(server._submit_next_block_candidate_writer(candidate)) + self.assertEqual(rpc.submitblock_calls, 1) + self.assertIn(block_hash, server._accepted_block_payout_previews) + self.assertTrue( + server._accepted_block_payout_previews[block_hash].landed + ) + candidate = self._retained_candidate(server) - def test_candidate_retry_state_is_cleared_on_terminal_abandonment(self) -> None: - server, state, _recording = submit_coordinator() - ledger = SingleWriterShareLedger() - server.ledger = ledger - pending = self._pending_append("retry-terminal").pending_share - candidate = block_candidate( - server, - state, - SimpleNamespace( - coinbase_tx_hex="00", - block_hash_hex="a2" * 32, - block_hex="00", - share_pass=True, - block_pass=True, - ), - pending_share=pending, - ) - ledger.append_batch([(pending, server.block_candidate_intent(candidate))]) - server.block_candidate_retry_delays = {candidate.submission.block_hash_hex: 2.0} + # Blockwait sees the pool's own hash as the new tip. This is the + # only acceptance signal: no direct-submit ack ever arrived and + # the accepted success tail has not run. + self.assertTrue(server.observe_tip_for_refresh(block_hash)) + self.assertIn(block_hash, server._tip_observed_accepted_block_hashes) + self.assertNotIn(block_hash, server._accounted_accepted_block_hashes) - def terminal(_candidate: PrismBlockCandidate) -> bool: - block_hash = _candidate.submission.block_hash_hex - block_height = int(_candidate.context.template["height"]) - server._begin_accepted_block_payout_preview( + # Attempt 2: the disposition reads a racing snapshot -- tip moved + # past the candidate and the header probe reports not-found. #89 + # abandoned here; the observation evidence must defer instead. + rpc.racing_tip = racing_tip + self.assertTrue(server._submit_next_block_candidate_writer(candidate)) + self.assertNotIn( + PRISM_REJECTION_STALE_JOB, + getattr(server, "block_candidate_abandoned_counts", {}), + ) + self.assertEqual(abandoned, []) + self.assertEqual(server.block_candidate_accept_pending_defer_count, 1) + # The landed preview is preserved -- not withdrawn, not + # tombstoned -- so payout publication stays unfenced and the + # reconcile/template path stays unblocked. + self.assertIn(block_hash, server._accepted_block_payout_previews) + self.assertTrue( + server._accepted_block_payout_previews[block_hash].landed + ) + self.assertNotIn( block_hash, - block_height=block_height, + server._invalidated_accepted_block_payout_previews, ) - server._mark_accepted_block_payout_landed( + self.assertFalse(server._payout_state_publication_blocked) + candidate = self._retained_candidate(server) + + # Attempt 3: the chain view settles with the pool's own block as + # the tip; the resume path runs the full accepted success tail + # and finalizes the durable outbox as submitted. + rpc.racing_tip = None + self.assertTrue(server._submit_next_block_candidate_writer(candidate)) + + self.assertEqual(submitted, [block_hash]) + self.assertEqual(abandoned, []) + self.assertEqual(rpc.submitblock_calls, 3) + self.assertIn(block_hash, server._accounted_accepted_block_hashes) + self.assertEqual(server.accepted_block_count, 1) + self.assertEqual(len(ledger.persisted), 1) + self.assertEqual(len(ledger.confirmed), 1) + self.assertNotIn(block_hash, server._accepted_block_payout_previews) + self.assertNotIn( block_hash, - block_height=block_height, + server._invalidated_accepted_block_payout_previews, ) - server._abandon_block_candidate( - PRISM_REJECTION_STALE_JOB, - "tip moved", - block_hash=block_hash, - worker="miner-a", + self.assertFalse(server._payout_state_publication_blocked) + self.assertIsNone( + server._accepted_block_payout_transition_for_parent( + block_hash, + parent_height=10, + ) + ) + self.assertNotIn( + block_hash, server._outstanding_block_candidate_hashes + ) + self.assertNotIn( + block_hash, server._tip_observed_accepted_block_hashes + ) + self.assertIn( + "qbit_prism_block_candidate_accept_pending_defers_total 1", + server.metrics_payload(), ) - return False - - server.submit_block_candidate = terminal # type: ignore[method-assign] - server.enqueue_block_candidate(candidate) - self.assertTrue(server.submit_next_block_candidate()) + def test_three_blocks_quick_succession_lost_acks_finalize_all(self) -> None: + # Three pool blocks land back to back (the 20:58 / 21:06:58 / + # 21:07:20 pattern), every submitblock ack is lost, and one retry + # even reads a transient foreign-tip snapshot. All three candidates + # must finalize as submitted with payout publication unfenced. + parent = "00" * 32 + hashes = ["b1" * 32, "b2" * 32, "b3" * 32] + block_hexes = ["0a", "0b", "0c"] + server, state, ledger = submit_coordinator(tip=parent) + server.max_blocks = 5 + server.stop_after_block = False + with tempfile.TemporaryDirectory() as tempdir: + submitted, abandoned = self._accepted_tail_scaffolding(server, tempdir) + rpc = LostAckSubmitRpc( + start_tip=parent, + hash_by_hex=dict(zip(block_hexes, hashes)), + ) + server.rpc = rpc + parents = [parent, hashes[0], hashes[1]] + for index, (block_hash, block_hex) in enumerate( + zip(hashes, block_hexes) + ): + job_id = f"job-chain-{index}" + self._chained_context( + server, + job_id=job_id, + parent_hash=parents[index], + height=10 + index, + ) + candidate = block_candidate( + server, + state, + SimpleNamespace( + coinbase_tx_hex="c0ffee", + block_hash_hex=block_hash, + block_hex=block_hex, + share_pass=True, + block_pass=True, + ), + job_id=job_id, + ) - self.assertNotIn(candidate.submission.block_hash_hex, server.block_candidate_retry_delays) - self.assertEqual(server.block_candidate_abandoned_counts[PRISM_REJECTION_STALE_JOB], 1) - self.assertEqual(ledger.pending_block_candidates(), []) - self.assertNotIn( - candidate.submission.block_hash_hex, - server._accepted_block_payout_previews, - ) - self.assertNotIn( - candidate.submission.block_hash_hex, - server._invalidated_accepted_block_payout_previews, - ) + # Lost-ack landing. + self.assertTrue( + server._submit_next_block_candidate_writer(candidate) + ) + candidate = self._retained_candidate(server) + # Blockwait-first acceptance. + self.assertTrue(server.observe_tip_for_refresh(block_hash)) - def test_terminal_abandon_immediately_clears_payout_preview(self) -> None: - server, _state, _ledger = submit_coordinator() - block_hash = "a4" * 32 - server._begin_accepted_block_payout_preview( - block_hash, - block_height=10, - ) - server._mark_accepted_block_payout_landed( - block_hash, - block_height=10, - ) + if index == 1: + # One retry reads a transient foreign-tip snapshot in + # which nothing can be proven active; it must defer. + rpc.racing_tip = "77" * 32 + self.assertTrue( + server._submit_next_block_candidate_writer(candidate) + ) + self.assertEqual(abandoned, []) + rpc.racing_tip = None + candidate = self._retained_candidate(server) - accepted_race_won = server._abandon_block_candidate( - PRISM_REJECTION_STALE_JOB, - "tip moved", - block_hash=block_hash, - worker="miner-a", - ) + # Settled view: the resume path finalizes as submitted. + self.assertTrue( + server._submit_next_block_candidate_writer(candidate) + ) + self.assertIn( + block_hash, server._accounted_accepted_block_hashes + ) - self.assertFalse(accepted_race_won) - self.assertNotIn(block_hash, server._accepted_block_payout_previews) - self.assertIn( - block_hash, - server._invalidated_accepted_block_payout_previews, - ) - # The tombstone protects pending durable replay, but unlike the leaked - # landed transition it does not block payout reconciliation. - with server._payout_balance_mutation(): - pass + self.assertEqual(submitted, hashes) + self.assertEqual(abandoned, []) + self.assertEqual(server.accepted_block_count, 3) + self.assertEqual( + getattr(server, "block_candidate_abandoned_counts", {}), + {}, + ) + self.assertEqual(len(ledger.persisted), 3) + self.assertEqual(len(ledger.confirmed), 3) + self.assertFalse(server._payout_state_publication_blocked) + self.assertEqual(server._accepted_block_payout_previews, {}) + self.assertEqual( + server._invalidated_accepted_block_payout_previews, {} + ) + self.assertEqual(server._outstanding_block_candidate_hashes, set()) + self.assertEqual(server._tip_observed_accepted_block_hashes, {}) - def test_tip_moved_abandon_yields_to_acceptance_during_cleanup(self) -> None: - old_tip = "00" * 32 - new_tip = "11" * 32 - block_hash = "a5" * 32 - server, state, _recording = submit_coordinator(tip=old_tip) - ledger = SingleWriterShareLedger() - server.ledger = ledger - pending = self._pending_append("accepted-during-clear").pending_share - candidate = block_candidate( - server, - state, - SimpleNamespace( - coinbase_tx_hex="00", - block_hash_hex=block_hash, - block_hex="00", - share_pass=True, - block_pass=True, - ), - pending_share=pending, - ) - ledger.append_batch( - [(pending, server.block_candidate_intent(candidate))] - ) - submitted: list[str] = [] - abandoned: list[str] = [] - original_submitted = ledger.mark_block_candidate_submitted - original_abandoned = ledger.mark_block_candidate_abandoned - original_clear = server._clear_accepted_block_payout_preview + def test_observed_window_expiry_keeps_genuinely_stale_abandon_terminal(self) -> None: + # A candidate that was once observed as the tip but has stayed off + # the active chain past the observation window is genuinely stale + # (permanently reorged out): the terminal abandon and its payout + # preview withdrawal must still happen. + parent = "00" * 32 + block_hash = "d4" * 32 + server, state, _ledger = submit_coordinator(tip=parent) + server.observed_tip_accept_window_seconds = 60.0 + with tempfile.TemporaryDirectory() as tempdir: + submitted, abandoned = self._accepted_tail_scaffolding(server, tempdir) + rpc = LostAckSubmitRpc( + start_tip=parent, + hash_by_hex={"00": block_hash}, + ) + server.rpc = rpc + candidate = block_candidate( + server, + state, + SimpleNamespace( + coinbase_tx_hex="c0ffee", + block_hash_hex=block_hash, + block_hex="00", + share_pass=True, + block_pass=True, + ), + ) + self.assertTrue(server._submit_next_block_candidate_writer(candidate)) + candidate = self._retained_candidate(server) + self.assertTrue(server.observe_tip_for_refresh(block_hash)) - def mark_submitted(*, block_hash: str) -> bool: - submitted.append(block_hash) - return original_submitted(block_hash=block_hash) + # The chain moved on without the candidate and the observation + # aged out: the block was reorged away for good. + rpc.racing_tip = "77" * 32 + rpc.active.pop(block_hash, None) + with server.lock: + server._tip_observed_accepted_block_hashes[block_hash] = ( + time.monotonic() - 61.0 + ) - def mark_abandoned(*, block_hash: str, error: str) -> bool: - abandoned.append(block_hash) - return original_abandoned(block_hash=block_hash, error=error) + self.assertTrue(server._submit_next_block_candidate_writer(candidate)) - def clear_while_accepting( - candidate_hash: str, - *, - invalidate_published: bool = False, - ) -> None: - original_clear( - candidate_hash, - invalidate_published=invalidate_published, + self.assertEqual(abandoned, [block_hash]) + self.assertEqual(submitted, []) + self.assertEqual( + server.block_candidate_abandoned_counts[PRISM_REJECTION_STALE_JOB], + 1, ) - if invalidate_published: - with server.lock: - server._accounted_accepted_block_hashes.add( - candidate_hash.lower() - ) + self.assertNotIn(block_hash, server._accepted_block_payout_previews) + self.assertNotIn(block_hash, server._accounted_accepted_block_hashes) - ledger.mark_block_candidate_submitted = mark_submitted # type: ignore[method-assign] - ledger.mark_block_candidate_abandoned = mark_abandoned # type: ignore[method-assign] - server._clear_accepted_block_payout_preview = clear_while_accepting # type: ignore[method-assign] - server._begin_accepted_block_payout_preview( - block_hash, - block_height=10, - ) - server._mark_accepted_block_payout_landed( - block_hash, - block_height=10, - ) - server.rpc = RejectingSubmitTipRpc(new_tip) - server.enqueue_block_candidate(candidate) + def test_block_candidate_acceptance_pending_probe_semantics(self) -> None: + block_hash = "e5" * 32 + server, _state, _ledger = submit_coordinator() + server._ensure_job_cache_state() + server.observed_tip_accept_window_seconds = 60.0 + server._register_outstanding_block_candidate(block_hash) - self.assertTrue(server.submit_next_block_candidate()) + def observed(age_seconds: float | None) -> None: + with server.lock: + server._tip_observed_accepted_block_hashes.pop(block_hash, None) + if age_seconds is not None: + server._tip_observed_accepted_block_hashes[block_hash] = ( + time.monotonic() - age_seconds + ) - self.assertEqual(submitted, [block_hash]) - self.assertEqual(abandoned, []) - self.assertNotIn( - PRISM_REJECTION_STALE_JOB, - server.block_candidate_abandoned_counts, + # Own hash is the fresh best tip: accepted, and the probe itself + # registers the observation for later checks. + observed(None) + server.rpc = AcceptanceProbeRpc(tip=block_hash) + self.assertTrue(server._block_candidate_acceptance_pending(block_hash)) + self.assertIn(block_hash, server._tip_observed_accepted_block_hashes) + + # Active header at the expected height: accepted. + observed(None) + server.rpc = AcceptanceProbeRpc( + tip="11" * 32, + header={"height": 10, "confirmations": 2}, ) - metrics = server.metrics_payload() - self.assertIn( - 'qbit_prism_stale_job_abandons_total{class="tip_moved"} 0', - metrics, + self.assertTrue( + server._block_candidate_acceptance_pending( + block_hash, expected_height=10 + ) ) - self.assertIn( - 'qbit_prism_stale_job_abandons_total{class="balance_stale"} 0', - metrics, + + # Provably active at the wrong height: abandonable even when the + # hash was recently observed. + observed(1.0) + server.rpc = AcceptanceProbeRpc( + tip="11" * 32, + header={"height": 9, "confirmations": 2}, ) - self.assertEqual(ledger.pending_block_candidates(), []) - outbox_row = ledger._block_candidate_outbox[block_hash] - self.assertEqual(outbox_row["state"], "submitted") - self.assertIsNone(outbox_row["last_error"]) - self.assertNotIn(block_hash, server._accepted_block_payout_previews) - self.assertNotIn( - block_hash, - server._invalidated_accepted_block_payout_previews, + self.assertFalse( + server._block_candidate_acceptance_pending( + block_hash, expected_height=10 + ) ) - def test_terminal_abandonment_keeps_tombstone_when_outbox_update_fails( - self, - ) -> None: - server, state, _recording = submit_coordinator() - ledger = SingleWriterShareLedger() - server.ledger = ledger - pending = self._pending_append("terminal-outbox-failure").pending_share - candidate = block_candidate( - server, - state, - SimpleNamespace( - coinbase_tx_hex="00", - block_hash_hex="a3" * 32, - block_hex="00", - share_pass=True, - block_pass=True, - ), - pending_share=pending, + # Unprovable probe + fresh observation: accepted (defer). + observed(1.0) + server.rpc = AcceptanceProbeRpc(tip="11" * 32, header=None) + self.assertTrue( + server._block_candidate_acceptance_pending( + block_hash, expected_height=10 + ) ) - block_hash = candidate.submission.block_hash_hex - block_height = int(candidate.context.template["height"]) - ledger.append_batch([(pending, server.block_candidate_intent(candidate))]) - def terminal(_candidate: PrismBlockCandidate) -> bool: - server._begin_accepted_block_payout_preview( - block_hash, - block_height=block_height, + # Unprovable probe + expired observation: genuinely stale. + observed(61.0) + self.assertFalse( + server._block_candidate_acceptance_pending( + block_hash, expected_height=10 ) - server._mark_accepted_block_payout_landed( - block_hash, - block_height=block_height, + ) + + # Unprovable probe + no observation: genuinely stale. + observed(None) + self.assertFalse( + server._block_candidate_acceptance_pending( + block_hash, expected_height=10 ) - server._abandon_block_candidate( - PRISM_REJECTION_STALE_JOB, - "tip moved", - block_hash=block_hash, - worker="miner-a", + ) + + # Probe failure + fresh observation: acceptance evidence wins. + observed(1.0) + server.rpc = AcceptanceProbeRpc(tip="11" * 32, fail=True) + self.assertTrue( + server._block_candidate_acceptance_pending( + block_hash, expected_height=10 ) - return False + ) - server.submit_block_candidate = terminal # type: ignore[method-assign] - server.enqueue_block_candidate(candidate) - with patch.object( - ledger, - "mark_block_candidate_abandoned", - side_effect=RuntimeError("postgres unavailable"), - ): - self.assertTrue(server.submit_next_block_candidate()) + # Probe failure + no observation: unchanged legacy behavior. + observed(None) + self.assertFalse( + server._block_candidate_acceptance_pending( + block_hash, expected_height=10 + ) + ) - self.assertNotIn(block_hash, server._accepted_block_payout_previews) - self.assertIn(block_hash, server._invalidated_accepted_block_payout_previews) - self.assertEqual( - [intent["block_hash_hex"] for intent in ledger.pending_block_candidates()], - [block_hash], + # A best-tip lookup failure must not suppress the active-header + # probe: the header alone proves acceptance, with no observation. + observed(None) + server.rpc = AcceptanceProbeRpc( + tip="11" * 32, + header={"height": 10, "confirmations": 2}, + fail_tip=True, + ) + self.assertTrue( + server._block_candidate_acceptance_pending( + block_hash, expected_height=10 + ) ) - def test_finalize_failure_replays_with_candidate_backoff(self) -> None: - server, state, _recording = submit_coordinator() - ledger = SingleWriterShareLedger() - server.ledger = ledger - pending = self._pending_append("f1").pending_share - candidate = block_candidate( - server, - state, - SimpleNamespace( - coinbase_tx_hex="00", - block_hash_hex="f1" * 32, - block_hex="00", - share_pass=True, - block_pass=True, - ), - pending_share=pending, + # ... and the header's wrong-height verdict also stands alone, + # overriding even a fresh observation. + observed(1.0) + server.rpc = AcceptanceProbeRpc( + tip="11" * 32, + header={"height": 9, "confirmations": 2}, + fail_tip=True, + ) + self.assertFalse( + server._block_candidate_acceptance_pending( + block_hash, expected_height=10 + ) ) - candidate_intent = server.block_candidate_intent(candidate) - ledger.append_batch([(pending, candidate_intent)]) - server.block_candidate_retry_initial_seconds = 0.1 - server.block_candidate_retry_max_seconds = 0.4 - # The block itself lands once; only the terminal outbox update fails. - # Replays must pace like any other candidate retry AND run finalize- - # only, never re-entering submission. - submit_calls = 0 - - def accepting_submit(_candidate: PrismBlockCandidate) -> bool: - nonlocal submit_calls - submit_calls += 1 - return True - server.submit_block_candidate = accepting_submit # type: ignore[method-assign] - original_finish = ledger.mark_block_candidate_submitted - original_mark_attempted = ledger.mark_block_candidate_attempted - finish_attempts = 0 - attempt_marks = 0 + def test_post_persist_stale_view_defers_before_rejecting_prepared_rows(self) -> None: + # Codex P1 / Bugbot: the post-persistence active-hash check could + # reject the prepared payout rows and only then reach the abandon, + # which now defers on acceptance evidence -- leaving rows + # rejected/reversed that a later confirmation can never promote. + # The acceptance check must run BEFORE reject_prepared_block. + parent = "00" * 32 + block_hash = "c7" * 32 + racing_winner = "77" * 32 + server, state, ledger = submit_coordinator(tip=parent) + server.max_blocks = 2 + server.stop_after_block = False + with tempfile.TemporaryDirectory() as tempdir: + submitted, abandoned = self._accepted_tail_scaffolding(server, tempdir) + rpc = LostAckSubmitRpc( + start_tip=parent, + hash_by_hex={"00": block_hash}, + ) + server.rpc = rpc + candidate = block_candidate( + server, + state, + SimpleNamespace( + coinbase_tx_hex="c0ffee", + block_hash_hex=block_hash, + block_hex="00", + share_pass=True, + block_pass=True, + ), + ) + self.assertTrue(server._submit_next_block_candidate_writer(candidate)) + candidate = self._retained_candidate(server) + self.assertTrue(server.observe_tip_for_refresh(block_hash)) - def mark_attempted(*, block_hash: str) -> bool: - nonlocal attempt_marks - attempt_marks += 1 - return original_mark_attempted(block_hash=block_hash) + # Retry resumes on the own-hash tip, but the post-persist + # getblockhash read races to a foreign winner for the height. + rpc.getblockhash_override = racing_winner + self.assertTrue(server._submit_next_block_candidate_writer(candidate)) - def flaky_finish(*, block_hash: str) -> bool: - nonlocal finish_attempts - finish_attempts += 1 - if finish_attempts <= 4: - raise RuntimeError("ledger unavailable") - return original_finish(block_hash=block_hash) + # Deferred without touching the prepared payout rows. + self.assertEqual(ledger.rejected, []) + self.assertEqual(abandoned, []) + self.assertNotIn( + "block-stale", + getattr(server, "block_candidate_abandoned_counts", {}), + ) + candidate = self._retained_candidate(server) - ledger.mark_block_candidate_attempted = mark_attempted # type: ignore[method-assign] - ledger.mark_block_candidate_submitted = flaky_finish # type: ignore[method-assign] - waits: list[float] = [] - with patch.object( - server.stop_event, - "wait", - side_effect=lambda delay: waits.append(delay) or False, - ): - for _attempt in range(4): - server.enqueue_block_candidate(candidate) - self.assertTrue(server.submit_next_block_candidate()) - self.assertEqual(ledger.pending_block_candidates(), [candidate_intent]) - server.enqueue_block_candidate(candidate) - self.assertTrue(server.submit_next_block_candidate()) + # The view settles with the pool's block active again: the + # replayed tail confirms the still-prepared rows. + rpc.getblockhash_override = None + self.assertTrue(server._submit_next_block_candidate_writer(candidate)) + self.assertEqual(submitted, [block_hash]) + self.assertEqual(ledger.rejected, []) + self.assertEqual(len(ledger.confirmed), 1) + self.assertIn(block_hash, server._accounted_accepted_block_hashes) - # The first accepted finalize failure returns unpaced so the caller - # can refresh the fleet immediately; the ladder starts at the first - # finalize-only replay. - self.assertEqual(len(waits), 4) - for observed, expected in zip( - waits, - [0.1, 0.2, 0.25, 0.15], - strict=True, - ): - self.assertAlmostEqual(observed, expected) - self.assertEqual(finish_attempts, 5) - self.assertEqual(attempt_marks, 1) - self.assertEqual(submit_calls, 1) - self.assertNotIn(candidate.submission.block_hash_hex, server.block_candidate_retry_delays) - self.assertEqual(server.block_candidate_abandoned_counts, {}) - self.assertEqual(ledger.pending_block_candidates(), []) - self.assertEqual(server._block_candidate_finalize_retries, {}) - self.assertIn( - "qbit_prism_block_candidate_retries_total 4", - server.metrics_payload(), - ) + def test_post_persist_stale_view_still_rejects_without_acceptance_evidence(self) -> None: + # The terminal half of the same site: submitblock succeeds but a + # sibling steals the height right after persistence and the pool's + # block was never observed as the tip. With no FRESH acceptance + # evidence (the retained ack has aged past the acceptance window), + # the terminal decision seals first and only then are the prepared + # rows rejected, exactly once. + parent = "00" * 32 + block_hash = "c8" * 32 + racing_winner = "77" * 32 + server, state, ledger = submit_coordinator(tip=parent) + server.max_blocks = 2 + server.stop_after_block = False + with tempfile.TemporaryDirectory() as tempdir: + submitted, abandoned = self._accepted_tail_scaffolding(server, tempdir) + rpc = LostAckSubmitRpc( + start_tip=parent, + hash_by_hex={"00": block_hash}, + ) + rpc.lose_acks = False + server.rpc = rpc + # The definitive ack retained at the offer would veto the + # terminal decision while fresh; these tests pin the sealed + # terminal machinery, so age it past the acceptance window. + self._age_retained_acceptance_past_window(server) + candidate = block_candidate( + server, + state, + SimpleNamespace( + coinbase_tx_hex="c0ffee", + block_hash_hex=block_hash, + block_hex="00", + share_pass=True, + block_pass=True, + ), + ) + real_persist = ledger.persist_accepted_block - def test_crash_after_submit_before_attempt_mark_replays_duplicate_once(self) -> None: - parent_hash = "00" * 32 - block_hash = "cf" * 32 - ledger = SingleWriterShareLedger() - first, state, _recording = submit_coordinator(tip=parent_hash) - first.ledger = ledger - first.stop_after_block = False - first.max_blocks = 10 - pending = self._pending_append("submit-before-mark-crash").pending_share - candidate = block_candidate( - first, - state, - SimpleNamespace( - coinbase_tx_hex="c0ffee", - block_hash_hex=block_hash, - block_hex="00", - share_pass=True, - block_pass=True, - ), - pending_share=pending, - ) - intent = first.block_candidate_intent(candidate) - ledger.append_batch([(pending, intent)]) + def persist_then_lose_race(**kwargs: object) -> dict[str, object]: + result = real_persist(**kwargs) + # A sibling wins the height after persistence: the tip and + # the height's active hash both belong to the foreign winner + # and the pool's block never became the chain tip. + rpc.tip = racing_winner + rpc.active.pop(block_hash, None) + rpc.getblockhash_override = racing_winner + return result - class RestartAwareRpc(FakeRpc): - def __init__(self) -> None: - self.submit_results: list[object] = [] + ledger.persist_accepted_block = persist_then_lose_race # type: ignore[method-assign] - def call( - self, - method: str, - params: list[object] | None = None, - *, - timeout: float | None = None, - ) -> object: - if method == "submitblock": - result = None if not self.submit_results else "duplicate" - self.submit_results.append(result) - return result - if method == "getbestblockhash": - return parent_hash - if method == "getblockhash": - return block_hash - if method == "getblockcount": - return 9 - return super().call(method, params) + self.assertTrue(server._submit_next_block_candidate_writer(candidate)) - rpc = RestartAwareRpc() - first.rpc = rpc - original_mark = ledger.mark_block_candidate_attempted + self.assertEqual(len(ledger.rejected), 1) + self.assertEqual(abandoned, [block_hash]) + self.assertEqual(submitted, []) + self.assertEqual( + server.block_candidate_abandoned_counts["block-stale"], + 1, + ) + self.assertNotIn(block_hash, server._accounted_accepted_block_hashes) - def crash_before_mark(*, block_hash: str) -> bool: - raise SystemExit(f"simulated crash before marking {block_hash}") + def test_acceptance_evidence_arriving_during_withdrawal_defers(self) -> None: + # Bugbot: the payout-preview withdrawal inside the abandon can block + # long enough for a blockwait observation to arrive; the terminal + # commitment must consult that late evidence, not only the + # completed-tail registry. + block_hash = "a9" * 32 + server, _state, _ledger = submit_coordinator() + server._ensure_job_cache_state() + server._register_outstanding_block_candidate(block_hash) + server._begin_accepted_block_payout_preview(block_hash, block_height=10) + server._mark_accepted_block_payout_landed(block_hash, block_height=10) + server.rpc = AcceptanceProbeRpc(tip="11" * 32, header=None) + real_clear = server._clear_accepted_block_payout_preview - ledger.mark_block_candidate_attempted = crash_before_mark # type: ignore[method-assign] - first.enqueue_block_candidate(candidate) - with self.assertRaisesRegex(SystemExit, "simulated crash"): - first.submit_next_block_candidate() - - self.assertEqual(rpc.submit_results, [None]) - self.assertEqual(ledger.pending_block_candidates(), [intent]) - self.assertEqual( - ledger._block_candidate_outbox[block_hash]["attempt_count"], - 0, - ) - - ledger.mark_block_candidate_attempted = original_mark # type: ignore[method-assign] - persisted_calls = 0 - confirmed_calls = 0 - original_persist = ledger.persist_accepted_block - original_confirm = ledger.confirm_accepted_block - - def persist_once(**kwargs: object) -> dict[str, object]: - nonlocal persisted_calls - persisted_calls += 1 - return original_persist(**kwargs) - - def confirm_once(**kwargs: object) -> dict[str, object]: - nonlocal confirmed_calls - confirmed_calls += 1 - return original_confirm(**kwargs) - - ledger.persist_accepted_block = persist_once # type: ignore[method-assign] - ledger.confirm_accepted_block = confirm_once # type: ignore[method-assign] - - restarted, _restart_state, _recording = submit_coordinator(tip=parent_hash) - restarted.ledger = ledger - restarted.rpc = rpc - restarted.stop_after_block = False - restarted.max_blocks = 10 - with tempfile.TemporaryDirectory() as tempdir: - restarted.audit_dir = Path(tempdir) - restarted.evidence_path = Path(tempdir) / "evidence.json" - restarted.ledger_writer_public_key_hex = "aa" * 32 - restarted.build_audit_bundle = ( # type: ignore[method-assign] - lambda **_kwargs: verified_block_bundle() - ) - restarted.verify_bundle = ( # type: ignore[method-assign] - lambda *_args, **_kwargs: verified_audit_report() + def observing_clear( + hash_arg: str, + *, + invalidate_published: bool = False, + ) -> None: + if invalidate_published: + # A blockwait observation lands while the withdrawal blocks. + with server.lock: + server._tip_observed_accepted_block_hashes[block_hash] = ( + time.monotonic() + ) + return real_clear( + hash_arg, + invalidate_published=invalidate_published, ) - self.assertEqual(restarted.replay_pending_block_candidates(), 1) - self.assertTrue(restarted.submit_next_block_candidate()) - self.assertEqual(restarted.replay_pending_block_candidates(), 0) + server._clear_accepted_block_payout_preview = observing_clear # type: ignore[method-assign] - self.assertEqual(rpc.submit_results, [None, "duplicate"]) - self.assertEqual(persisted_calls, 1) - self.assertEqual(confirmed_calls, 1) - self.assertEqual(restarted.accepted_block_count, 1) - self.assertEqual(ledger.pending_block_candidates(), []) + accepted_race_won = server._abandon_block_candidate( + PRISM_REJECTION_STALE_JOB, + "tip moved before submit: test", + block_hash=block_hash, + worker=None, + preserve_if_accepted=True, + expected_height=10, + ) + + self.assertFalse(accepted_race_won) + outcome = getattr(server, "_block_candidate_outcome", None) self.assertEqual( - ledger._block_candidate_outbox[block_hash]["state"], - "submitted", + getattr(outcome, "reason", None), + PRISM_REJECTION_BLOCK_ACCEPT_PENDING, + ) + self.assertNotIn( + PRISM_REJECTION_STALE_JOB, + getattr(server, "block_candidate_abandoned_counts", {}), + ) + self.assertEqual(server.block_candidate_accept_pending_defer_count, 1) + # The landed barrier is restored (and its fail-closed tombstone + # popped) so descendant builders wait on the preview instead of + # failing closed while the deferred retry heals it. + self.assertIn(block_hash, server._accepted_block_payout_previews) + self.assertTrue( + server._accepted_block_payout_previews[block_hash].landed + ) + self.assertNotIn( + block_hash, + server._invalidated_accepted_block_payout_previews, ) - def test_abandon_finalize_failure_counts_one_abandonment(self) -> None: + def test_replay_restores_acceptance_evidence_from_durable_block_state(self) -> None: + # Codex round 5: the observation registry dies with the process. A + # durable prepared/confirmed pool-block row proves a prior process's + # submitblock succeeded, so startup replay must restore acceptance + # evidence before a replay disposition can race a transient fork + # view and terminally abandon the accepted block. server, state, _recording = submit_coordinator() ledger = SingleWriterShareLedger() server.ledger = ledger - moved_tip = "11" * 32 - expected_error = f"tip moved before submit: {moved_tip}" - pending = self._pending_append("f2").pending_share - candidate = block_candidate( - server, - state, - SimpleNamespace( - coinbase_tx_hex="00", - block_hash_hex="f2" * 32, - block_hex="00", - share_pass=True, - block_pass=True, - ), - pending_share=pending, - ) - ledger.append_batch([(pending, server.block_candidate_intent(candidate))]) - server.block_candidate_retry_initial_seconds = 0.1 - server.block_candidate_retry_max_seconds = 0.4 - submit_calls = 0 - - def terminal_submit(_candidate: PrismBlockCandidate) -> bool: - nonlocal submit_calls - submit_calls += 1 - server._abandon_block_candidate( - PRISM_REJECTION_STALE_JOB, - expected_error, - block_hash=_candidate.submission.block_hash_hex, - worker="miner-a", - stale_job_class="tip_moved", + accepted_hash = "e7" * 32 + unproven_hash = "e8" * 32 + unreadable_hash = "e9" * 32 + for index, block_hash in enumerate( + (accepted_hash, unproven_hash, unreadable_hash), start=1 + ): + pending = PendingShare( + share_id=f"miner-a:{block_hash}", + miner_id="miner-a", + order_key="miner-a", + p2mr_program_hex="11" * 32, + share_difficulty=1, + network_difficulty=1, + template_height=9, + job_id="job-1", + job_issued_at_ms=1, + accepted_at_ms=index, + ntime=1, ) - return False - - server.submit_block_candidate = terminal_submit # type: ignore[method-assign] - original_finish = ledger.mark_block_candidate_abandoned - finish_attempts = 0 - - def flaky_finish(*, block_hash: str, error: str) -> bool: - nonlocal finish_attempts - finish_attempts += 1 - if finish_attempts <= 2: - raise RuntimeError("ledger unavailable") - return original_finish(block_hash=block_hash, error=error) + candidate = block_candidate( + server, + state, + SimpleNamespace( + coinbase_tx_hex="00", + block_hash_hex=block_hash, + block_hex="00", + share_pass=True, + block_pass=True, + ), + pending_share=pending, + ) + ledger.append_batch( + [(pending, server.block_candidate_intent(candidate))] + ) + def durable_block_state(*, block_hash: str) -> dict[str, object] | None: + if block_hash == accepted_hash: + return {"chain_state": "prepared", "maturity_state": "immature"} + if block_hash == unreadable_hash: + raise RuntimeError("postgres unavailable") + return None - ledger.mark_block_candidate_abandoned = flaky_finish # type: ignore[method-assign] - waits: list[float] = [] - with patch.object( - server.stop_event, - "wait", - side_effect=lambda delay: waits.append(delay) or False, - ): - for _attempt in range(3): - server.enqueue_block_candidate(candidate) - self.assertTrue(server.submit_next_block_candidate()) + ledger.pool_block_state = durable_block_state # type: ignore[attr-defined] - self.assertEqual(waits, [0.1, 0.2]) - self.assertEqual(submit_calls, 1) - self.assertEqual(finish_attempts, 3) - # Finalize-only replays must not recount the terminal abandonment. - self.assertEqual( - server.block_candidate_abandoned_counts, - {PRISM_REJECTION_STALE_JOB: 1}, - ) - self.assertEqual( - server.stale_job_abandon_counts, - {"tip_moved": 1, "balance_stale": 0, "append_epoch_stale": 0}, - ) - self.assertEqual(ledger.pending_block_candidates(), []) - outbox_row = ledger._block_candidate_outbox[ - candidate.submission.block_hash_hex + self.assertEqual(server.replay_pending_block_candidates(), 3) + replayed = [ + server._block_replay_candidate_queue.get_nowait() + for _ in range(3) ] - self.assertEqual(outbox_row["state"], "abandoned") - self.assertEqual(outbox_row["last_error"], expected_error) - self.assertEqual(server._block_candidate_finalize_retries, {}) + for candidate in replayed: + server._restore_replayed_candidate_acceptance_evidence(candidate) - def test_accepted_finalize_failure_still_triggers_post_accept_refresh(self) -> None: - server, state, _recording = submit_coordinator() - ledger = SingleWriterShareLedger() - server.ledger = ledger - pending = self._pending_append("f3").pending_share - candidate = block_candidate( - server, - state, - SimpleNamespace( - coinbase_tx_hex="00", - block_hash_hex="f3" * 32, - block_hex="00", - share_pass=True, - block_pass=True, - ), - pending_share=pending, + self.assertIn(accepted_hash, server._tip_observed_accepted_block_hashes) + self.assertIn(accepted_hash, server._outstanding_block_candidate_hashes) + # No durable acceptance proof: replays without synthetic evidence. + self.assertNotIn( + unproven_hash, server._tip_observed_accepted_block_hashes + ) + # An unreadable durable state fails safe: the candidate replays + # protected, bounded by the observation window, instead of exposed + # to a transient fork view racing its first disposition. + self.assertIn( + unreadable_hash, server._tip_observed_accepted_block_hashes ) - ledger.append_batch([(pending, server.block_candidate_intent(candidate))]) - server.block_candidate_retry_initial_seconds = 0.1 - server.block_candidate_retry_max_seconds = 0.4 - server.submit_block_candidate = lambda _candidate: True # type: ignore[method-assign] - - def failing_finish(*, block_hash: str) -> bool: - raise RuntimeError("ledger unavailable") - ledger.mark_block_candidate_submitted = failing_finish # type: ignore[method-assign] - refreshed_clients: list[object] = [] - server.refresh_jobs_after_pending_accepted_block = ( # type: ignore[method-assign] - lambda client, **_kwargs: refreshed_clients.append(client) or 0 - ) - released_shares: list[object] = [] - original_release = server._finish_pending_share_commit + def test_late_defer_republishes_withdrawn_preview_and_unfences(self) -> None: + # Bugbot round 5: when the withdrawal's own publication loses its + # race, delivery is fenced before the late-acceptance defer runs. + # The defer must republish the withdrawn preview so admission + # reopens immediately instead of staying coordination-blocked + # across the deferral cycles. + block_hash = "ba" * 32 + server, _state, _ledger = submit_coordinator() + server._ensure_job_cache_state() + server._register_outstanding_block_candidate(block_hash) + server._begin_accepted_block_payout_preview(block_hash, block_height=10) + server._mark_accepted_block_payout_landed(block_hash, block_height=10) + server._publish_accepted_block_payout_preview(block_hash, []) + self.assertFalse(server._payout_state_publication_blocked) + server.rpc = AcceptanceProbeRpc(tip="11" * 32, header=None) - def recording_release(pending_share: object) -> None: - released_shares.append(pending_share) - original_release(pending_share) + real_publish = server._publish_payout_state_candidate - server._finish_pending_share_commit = recording_release # type: ignore[method-assign] - waits: list[float] = [] - with patch.object( - server.stop_event, - "wait", - side_effect=lambda delay: waits.append(delay) or False, - ): - server.enqueue_block_candidate(candidate) - self.assertTrue(server.submit_next_block_candidate()) - # The block is active on-chain; the fleet refresh fires on the - # first finalize failure without waiting out a backoff, and the - # snapshot anchor floor is released despite the pending outbox - # mark. - self.assertEqual(refreshed_clients, [state]) - self.assertEqual(waits, []) - self.assertIn(pending, released_shares) - self.assertTrue(server.submit_next_block_candidate()) - self.assertEqual(refreshed_clients, [state]) - self.assertEqual(waits, [0.1]) + def withdrawal_publication_lost(candidate: object) -> object: + if getattr(candidate, "accepted_block_withdrawal", False): + return None + return real_publish(candidate) - def test_invalid_durable_candidate_is_quarantined_by_outbox_row_key(self) -> None: - for payload_hash in (None, "ff" * 32): - with self.subTest(payload_hash=payload_hash): - server, _state, _recording = submit_coordinator() - ledger = SingleWriterShareLedger() - server.ledger = ledger - durable_hash = "de" * 32 - invalid = { - "schema": "unsupported", - "block_hash_hex": durable_hash, - "block_hex": "00", - } - ledger.persist_block_candidate_intent(invalid) - stored = ledger._block_candidate_outbox[durable_hash]["candidate"] - if payload_hash is None: - stored.pop("block_hash_hex") - else: - stored["block_hash_hex"] = payload_hash - server.block_candidate_retry_delays = {durable_hash: 1.0} + server._publish_payout_state_candidate = withdrawal_publication_lost # type: ignore[method-assign] + real_clear = server._clear_accepted_block_payout_preview - self.assertEqual(server.replay_pending_block_candidates(), 0) - # Malformed-row cleanup is lower-priority maintenance, so it - # cannot form an N x database-timeout convoy ahead of valid - # recovered blocks on the node-offer lane. - self.assertTrue( - server._run_one_invalid_block_candidate_quarantine() - ) + def observing_clear( + hash_arg: str, + *, + invalidate_published: bool = False, + ) -> None: + if invalidate_published: + with server.lock: + server._tip_observed_accepted_block_hashes[block_hash] = ( + time.monotonic() + ) + return real_clear( + hash_arg, + invalidate_published=invalidate_published, + ) - self.assertEqual(ledger.pending_block_candidates(), []) - self.assertNotIn(durable_hash, server.block_candidate_retry_delays) - self.assertIn( - "qbit_prism_block_candidate_poisoned_total 1", - server.metrics_payload(), - ) + server._clear_accepted_block_payout_preview = observing_clear # type: ignore[method-assign] - def test_block_submitter_drops_candidate_when_pool_closed(self) -> None: - server, state, ledger = submit_coordinator() - server.accepted_block_count = 1 - server.max_blocks = 1 - submission = SimpleNamespace( - coinbase_tx_hex="c0ffee", - block_hash_hex="dd" * 32, - block_hex="00", + accepted_race_won = server._abandon_block_candidate( + PRISM_REJECTION_STALE_JOB, + "tip moved before submit: test", + block_hash=block_hash, + worker=None, + preserve_if_accepted=True, + expected_height=10, ) - accepted = server.submit_block_candidate(block_candidate(server, state, submission)) + self.assertFalse(accepted_race_won) + outcome = getattr(server, "_block_candidate_outcome", None) + self.assertEqual( + getattr(outcome, "reason", None), + PRISM_REJECTION_BLOCK_ACCEPT_PENDING, + ) + # The barrier is restored AND publication admission reopened: the + # withdrawn preview was republished despite the withdrawal's own + # publication loss having fenced delivery. + self.assertIn(block_hash, server._accepted_block_payout_previews) + self.assertTrue( + server._accepted_block_payout_previews[block_hash].landed + ) + self.assertIsNotNone( + server._accepted_block_payout_previews[block_hash].preview + ) + self.assertNotIn( + block_hash, + server._invalidated_accepted_block_payout_previews, + ) + self.assertFalse(server._payout_state_publication_blocked) - self.assertFalse(accepted) - self.assertEqual(server.block_candidate_abandoned_counts[PRISM_REJECTION_POOL_CLOSED], 1) - self.assertEqual(server.rejection_counts_by_reason[PRISM_REJECTION_POOL_CLOSED], 0) - self.assertEqual(ledger.persisted, []) + def test_late_gate_reprobes_after_withdrawal_heals_buried_block(self) -> None: + # Bugbot round 7: an unknown pre-withdrawal probe verdict must + # re-probe after the (blocking) withdrawal. A buried accepted block + # is not always re-observed as the tip -- blockwait reports only the + # newest of rapid connects -- so the recovered probe alone, with no + # observation evidence at all, must defer the terminal abandon. + block_hash = "bc" * 32 + server, _state, _ledger = submit_coordinator() + server._ensure_job_cache_state() + server._register_outstanding_block_candidate(block_hash) + server._begin_accepted_block_payout_preview(block_hash, block_height=10) + server._mark_accepted_block_payout_landed(block_hash, block_height=10) + rpc = AcceptanceProbeRpc(tip="11" * 32, header=None) + server.rpc = rpc + real_clear = server._clear_accepted_block_payout_preview - def test_block_submitter_honors_stop_after_block_above_one_block_capacity( - self, - ) -> None: - server, state, ledger = submit_coordinator() - server.accepted_block_count = 1 - server.max_blocks = 2 - server.stop_after_block = True - submission = SimpleNamespace( - coinbase_tx_hex="c0ffee", - block_hash_hex="de" * 32, - block_hex="00", - ) + def healing_clear( + hash_arg: str, + *, + invalidate_published: bool = False, + ) -> None: + if invalidate_published: + # The chain view heals while the withdrawal blocks: the + # candidate is provably active again, two blocks deep. + rpc.header = {"height": 10, "confirmations": 2} + return real_clear( + hash_arg, + invalidate_published=invalidate_published, + ) - accepted = server.submit_block_candidate( - block_candidate(server, state, submission) + server._clear_accepted_block_payout_preview = healing_clear # type: ignore[method-assign] + + accepted_race_won = server._abandon_block_candidate( + PRISM_REJECTION_STALE_JOB, + "tip moved before submit: test", + block_hash=block_hash, + worker=None, + preserve_if_accepted=True, + expected_height=10, ) - self.assertFalse(accepted) + self.assertFalse(accepted_race_won) + outcome = getattr(server, "_block_candidate_outcome", None) self.assertEqual( - server.block_candidate_abandoned_counts[PRISM_REJECTION_POOL_CLOSED], - 1, + getattr(outcome, "reason", None), + PRISM_REJECTION_BLOCK_ACCEPT_PENDING, + ) + self.assertNotIn( + PRISM_REJECTION_STALE_JOB, + getattr(server, "block_candidate_abandoned_counts", {}), + ) + self.assertIn(block_hash, server._accepted_block_payout_previews) + self.assertTrue( + server._accepted_block_payout_previews[block_hash].landed + ) + self.assertNotIn( + block_hash, + server._invalidated_accepted_block_payout_previews, ) - self.assertEqual(ledger.persisted, []) - def test_block_worthy_share_is_credited_and_enqueued_before_block_submission(self) -> None: - # The share ack must never wait on the block path: a block-worthy - # share that met its target is credited immediately and the candidate - # is queued for the submitter thread. Nothing submits synchronously - # (the fixture RPC would raise on an unexpected submitblock call). - server, state, ledger = submit_coordinator() - submission = SimpleNamespace( - header_hex="aa" * 80, - block_hash_hex="cc" * 32, - share_pass=True, - block_pass=True, + def test_wrong_height_probe_verdict_overrules_late_observation(self) -> None: + # Bugbot round 6: the probe must win both directions in the late + # check too. An observation arriving during the withdrawal cannot + # revive a candidate the pre-check probe proved active at the wrong + # height -- header heights are immutable, so that verdict stands and + # the abandon commits terminally instead of deferring to window + # expiry. + block_hash = "bb" * 32 + server, _state, _ledger = submit_coordinator() + server._ensure_job_cache_state() + server._register_outstanding_block_candidate(block_hash) + server._begin_accepted_block_payout_preview(block_hash, block_height=10) + server._mark_accepted_block_payout_landed(block_hash, block_height=10) + server.rpc = AcceptanceProbeRpc( + tip="11" * 32, + header={"height": 9, "confirmations": 2}, ) + real_clear = server._clear_accepted_block_payout_preview - with patch( - "lab.prism.prism_coordinator.direct_stratum.assemble_submission", - return_value=submission, - ): - should_close = server.handle_submit( - state, - ["miner-a", "job-1", "00" * 8, "00000001", "00000002"], + def observing_clear( + hash_arg: str, + *, + invalidate_published: bool = False, + ) -> None: + if invalidate_published: + with server.lock: + server._tip_observed_accepted_block_hashes[block_hash] = ( + time.monotonic() + ) + return real_clear( + hash_arg, + invalidate_published=invalidate_published, ) - self.assertFalse(should_close) - self.assertEqual(len(ledger.pending), 1) - self.assertIsNone(ledger.pending[0].credit_policy) - self.assertEqual(server.block_candidate_queue.qsize(), 1) - queued = server.block_candidate_queue.get_nowait() - self.assertIs(queued.submission, submission) - self.assertFalse(queued.credit_share_on_accept) + server._clear_accepted_block_payout_preview = observing_clear # type: ignore[method-assign] - def test_block_candidate_submits_before_full_audit_persistence(self) -> None: - server, state, ledger = submit_coordinator() - server._ensure_job_cache_state() - server._ensure_tip_refresh_state() - stale_bundle_key = ("stale-payout-bundle",) - server._job_bundle_cache[stale_bundle_key] = object() # type: ignore[assignment] - active_fanout = _FanoutCancellation() - server._active_tip_refresh = ( # type: ignore[assignment] - SimpleNamespace(payout_state_generation=0), - active_fanout, + accepted_race_won = server._abandon_block_candidate( + "block-stale", + "candidate active at unexpected height 9", + block_hash=block_hash, + worker=None, + expected_height=10, ) - with tempfile.TemporaryDirectory() as tempdir: - server.audit_dir = Path(tempdir) - server.evidence_path = Path(tempdir) / "evidence.json" - server.ledger_writer_public_key_hex = "aa" * 32 - block_hash = "cc" * 32 - rpc = SubmitRpc(tip="00" * 32, block_hash=block_hash, ledger=ledger) - server.rpc = rpc - witness_tx = synthetic_witness_transaction("44") - server.jobs["job-1"].job.transaction_hexes = (witness_tx,) - build_kwargs: list[dict[str, object]] = [] - # Deliberately use insertion order that differs from this fixture's - # stand-in canonical serialization. The alternate builder creates - # the requested output but writes pretty, noncanonical JSON, so - # existence alone must not make the path eligible for persistence. - alternate_bundle = { - "signed_coinbase_manifest": { - "manifest": { - "coinbase_tx_hex": "c0ffee", - "payout_count": 1, - } - }, - "payout_policy_manifest": {"accounts": []}, - "ledger_window_attestation": {"signature": {"public_key_hex": "aa" * 32}}, - "found_block": {"coinbase_value_sats": 50_00000000}, - } - canonical_bytes = json.dumps( - alternate_bundle, - sort_keys=True, - separators=(",", ":"), - ).encode() - canonical_sha256 = hashlib.sha256(canonical_bytes).hexdigest() - def fake_build_audit_bundle(**kwargs: object) -> dict[str, object]: - build_kwargs.append(kwargs) - output_path = kwargs["canonical_output_path"] - assert isinstance(output_path, Path) - output_path.write_text( - json.dumps(alternate_bundle, indent=2), - encoding="utf-8", - ) - return alternate_bundle + self.assertFalse(accepted_race_won) + outcome = getattr(server, "_block_candidate_outcome", None) + self.assertEqual(getattr(outcome, "reason", None), "block-stale") + self.assertNotIn( + "block-stale", + server.block_candidate_abandoned_counts, + ) + self.assertEqual( + getattr(server, "block_candidate_accept_pending_defer_count", 0), + 0, + ) - def fake_verify_bundle(bundle_path: Path, *_args: object, **_kwargs: object) -> dict[str, object]: - candidate_bytes = bundle_path.read_bytes() - self.assertEqual(json.loads(candidate_bytes), alternate_bundle) - self.assertNotEqual(candidate_bytes, canonical_bytes) - self.assertNotEqual( - hashlib.sha256(candidate_bytes).hexdigest(), - canonical_sha256, - ) - return { - "coinbase_txid": "11" * 32, - "coinbase_manifest_sha256_hex": "22" * 32, - "audit_bundle_sha256_hex": canonical_sha256, - "coinbase_tx_hex": "c0ffee", - } + def test_terminal_seal_excludes_observations_after_the_commit(self) -> None: + # Codex round 3: acceptance evidence arriving after the terminal + # commit but before the follow-up durable rejection must be excluded + # deterministically. The terminal commit seals the hash (drops it + # from outstanding) in the same critical section, so a later + # observation is a no-op instead of contradicting the sealed + # disposition mid-rejection. + block_hash = "b8" * 32 + server, _state, _ledger = submit_coordinator() + server._ensure_job_cache_state() + server._register_outstanding_block_candidate(block_hash) + server.rpc = AcceptanceProbeRpc(tip="11" * 32, header=None) - persist_accepted_block = ledger.persist_accepted_block + accepted_race_won = server._abandon_block_candidate( + PRISM_REJECTION_STALE_JOB, + "tip moved before submit: test", + block_hash=block_hash, + worker=None, + preserve_if_accepted=True, + expected_height=10, + ) - def persist_with_canonicalization(**kwargs: object) -> dict[str, object]: - self.assertIsNone(kwargs["canonical_bundle_path"]) - self.assertEqual(kwargs["final_bundle"], alternate_bundle) - report = kwargs["audit_report"] - assert isinstance(report, dict) - self.assertEqual(report["audit_bundle_sha256_hex"], canonical_sha256) - return persist_accepted_block(**kwargs) + self.assertFalse(accepted_race_won) + self.assertNotIn( + PRISM_REJECTION_STALE_JOB, + server.block_candidate_abandoned_counts, + ) + self.assertNotIn(block_hash, server._outstanding_block_candidate_hashes) - server.build_audit_bundle = fake_build_audit_bundle # type: ignore[method-assign] - server.verify_bundle = fake_verify_bundle # type: ignore[method-assign] - ledger.persist_accepted_block = persist_with_canonicalization # type: ignore[method-assign] - submission = SimpleNamespace( - coinbase_tx_hex="c0ffee", - block_hash_hex=block_hash, - block_hex="00", - ) - pending = SimpleNamespace(share_id="miner-a:" + block_hash) + # A blockwait observation lands right after the sealed commit: it + # must not register acceptance evidence for the sealed hash. + self.assertTrue(server.observe_tip_for_refresh(block_hash)) + self.assertNotIn(block_hash, server._tip_observed_accepted_block_hashes) + outcome = getattr(server, "_block_candidate_outcome", None) + self.assertIsNotNone(outcome) + server._record_committed_block_candidate_abandonment(block_hash, outcome) + self.assertEqual( + server.block_candidate_abandoned_counts[PRISM_REJECTION_STALE_JOB], + 1, + ) - accepted = server.submit_block_candidate( - block_candidate(server, state, submission, pending_share=pending) + def test_observation_during_prepared_row_rejection_cannot_split_state(self) -> None: + # The exact round-3 interleaving at the post-persist site: a + # blockwait observation fires inside the gap between the sealed + # terminal decision and reject_prepared_block (during the + # getblockcount call). The seal excludes it, so the rows are + # rejected exactly once with no contradictory evidence registered + # and no defer emitted after the seal. + parent = "00" * 32 + block_hash = "c9" * 32 + racing_winner = "77" * 32 + server, state, ledger = submit_coordinator(tip=parent) + server.max_blocks = 2 + server.stop_after_block = False + with tempfile.TemporaryDirectory() as tempdir: + submitted, abandoned = self._accepted_tail_scaffolding(server, tempdir) + rpc = LostAckSubmitRpc( + start_tip=parent, + hash_by_hex={"00": block_hash}, ) - self.assertTrue(accepted) - self.assertTrue( - server._ensure_shutdown_controller().writer_admission_closed() + rpc.lose_acks = False + server.rpc = rpc + # The definitive ack retained at the offer would veto the + # terminal decision while fresh; these tests pin the sealed + # terminal machinery, so age it past the acceptance window. + self._age_retained_acceptance_past_window(server) + candidate = block_candidate( + server, + state, + SimpleNamespace( + coinbase_tx_hex="c0ffee", + block_hash_hex=block_hash, + block_hex="00", + share_pass=True, + block_pass=True, + ), ) + race_state: dict[str, bool] = {"post_persist": False} + real_persist = ledger.persist_accepted_block - live_files = sorted(Path(tempdir).glob("prism-live-audit-bundle-[0-9]*.json")) - self.assertEqual(len(live_files), 1) - self.assertEqual(list(Path(tempdir).glob("prism-live-audit-bundle-candidate-*.json")), []) - self.assertEqual(list(Path(tempdir).glob(".prism-live-audit-bundle-candidate-*.tmp")), []) - envelope = json.loads(live_files[0].read_text(encoding="utf-8")) - self.assertEqual(envelope["schema"], "qbit.prism.live-audit-bundle-envelope.v1") - self.assertEqual(envelope["block_hash"], block_hash) - self.assertEqual(envelope["block_height"], 10) - self.assertEqual(envelope["audit_bundle_sha256"], canonical_sha256) - self.assertNotIn("signed_coinbase_manifest", envelope) - self.assertEqual(server.latest_evidence["audit_bundle_path"], str(live_files[0])) + def persist_then_lose_race(**kwargs: object) -> dict[str, object]: + result = real_persist(**kwargs) + rpc.tip = racing_winner + rpc.active.pop(block_hash, None) + rpc.getblockhash_override = racing_winner + race_state["post_persist"] = True + return result - self.assertTrue(rpc.submitted) - self.assertEqual( - build_kwargs[0]["witness_merkle_leaves_hex"], - direct_stratum.witness_merkle_leaves_hex((witness_tx,)), - ) - self.assertEqual( - build_kwargs[0]["coinbase_script_sig_suffix_hex"], - server.coinbase_tag_hex + state.extranonce1_hex + "00" * 8, - ) - self.assertEqual(ledger.persisted[0]["block_hash"], block_hash) - self.assertEqual(ledger.persisted[0]["block_height"], 10) - self.assertTrue(ledger.persisted[0]["submit_seen_at_persist"]) - # This alternate test builder wrote noncanonical bytes to the requested - # path. The verified content is valid, but the path is never claimed as - # a byte-canonical artifact for ledger persistence. - self.assertIsNone(ledger.persisted[0]["canonical_bundle_path"]) - self.assertEqual(server._payout_state_generation, 1) - self.assertEqual(server._job_bundle_cache, {}) - self.assertTrue(active_fanout.is_set()) - self.assertTrue(server._tip_refresh_retry.is_set()) + ledger.persist_accepted_block = persist_then_lose_race # type: ignore[method-assign] + real_call = rpc.call - def test_issued_preview_is_invalidated_when_final_coinbase_mismatches(self) -> None: - server, state, ledger = submit_coordinator() - server._ensure_job_cache_state() - block_hash = "cf" * 32 - server.rpc = SubmitRpc(tip="00" * 32, block_hash=block_hash, ledger=ledger) - server.build_audit_bundle = ( # type: ignore[method-assign] - lambda **_kwargs: verified_block_bundle("deadbeef") - ) - submission = SimpleNamespace( - coinbase_tx_hex="c0ffee", - block_hash_hex=block_hash, - block_hex="00", - ) - candidate = block_candidate(server, state, submission) - candidate.context.prospective_prior_balances = () + def observing_call( + method: str, + params: list[object] | None = None, + ) -> object: + if method == "getblockcount" and race_state["post_persist"]: + # Blockwait reports the candidate as tip inside the + # seal -> reject gap. + server.observe_tip_for_refresh(block_hash) + return real_call(method, params) - with tempfile.TemporaryDirectory() as tempdir: - server.audit_dir = Path(tempdir) - self.assertFalse(server.submit_block_candidate(candidate)) + rpc.call = observing_call # type: ignore[method-assign] - self.assertEqual(server._payout_state_generation, 2) - self.assertEqual(server._accepted_block_payout_previews, {}) - self.assertEqual(ledger.persisted, []) - self.assertEqual( - server._block_candidate_outcome.reason, - PRISM_REJECTION_CANDIDATE_AUDIT_MISMATCH, - ) + self.assertTrue(server._submit_next_block_candidate_writer(candidate)) - def test_accepted_block_persistence_allows_delivery_but_serializes_reconciliation( - self, - ) -> None: - server, state, ledger = submit_coordinator() - server._ensure_job_cache_state() - persist_started = threading.Event() - persist_finished = threading.Event() - release_persist = threading.Event() - confirm_started = threading.Event() - release_delivery = threading.Event() - reconcile_lock_attempted = threading.Event() - reconcile_mutated = threading.Event() - event_lock = threading.Lock() - events: list[str] = [] - original_persist = ledger.persist_accepted_block - original_confirm = ledger.confirm_accepted_block - - def note(name: str) -> None: - with event_lock: - events.append(name) - - def blocking_persist(**kwargs: object) -> dict[str, object]: - note("persist-start") - persist_started.set() - if not release_persist.wait(15): - raise AssertionError("timed out waiting to release accepted-block persistence") - result = original_persist(**kwargs) - note("persist-end") - persist_finished.set() - return result - - def observed_confirm(**kwargs: object) -> dict[str, object]: - note("confirm") - confirm_started.set() - return original_confirm(**kwargs) - - ledger.reorg_watch_blocks = lambda *, active_tip_height: [ # type: ignore[method-assign] - { - "block_height": active_tip_height - 2, - "block_hash": "aa" * 32, - "chain_state": "confirmed", - } - ] + self.assertEqual(len(ledger.rejected), 1) + self.assertEqual(abandoned, [block_hash]) + self.assertEqual(submitted, []) + self.assertEqual( + server.block_candidate_abandoned_counts["block-stale"], + 1, + ) + self.assertNotIn( + block_hash, server._tip_observed_accepted_block_hashes + ) + self.assertNotIn( + block_hash, server._outstanding_block_candidate_hashes + ) + self.assertEqual( + getattr(server, "block_candidate_accept_pending_defer_count", 0), + 0, + ) - def mark_pool_block_inactive(**_kwargs: object) -> dict[str, object]: - note("reconcile-mutation") - reconcile_mutated.set() - return {"backend": "fake", "inactive_count": 1} + def test_retained_candidate_reregisters_for_tip_observations(self) -> None: + # Codex round 8: the terminal seal stops observation matching, but + # when the terminal cleanup itself fails (reject_prepared_block + # raising) the candidate is retained for retry -- and evidence + # arriving during that backoff gap must register again, not vanish. + parent = "00" * 32 + block_hash = "cd" * 32 + racing_winner = "77" * 32 + server, state, ledger = submit_coordinator(tip=parent) + server.max_blocks = 2 + server.stop_after_block = False + with tempfile.TemporaryDirectory() as tempdir: + submitted, abandoned = self._accepted_tail_scaffolding(server, tempdir) + rpc = LostAckSubmitRpc( + start_tip=parent, + hash_by_hex={"00": block_hash}, + ) + rpc.lose_acks = False + server.rpc = rpc + # The definitive ack retained at the offer would veto the + # terminal decision while fresh; these tests pin the sealed + # terminal machinery, so age it past the acceptance window. + self._age_retained_acceptance_past_window(server) + candidate = block_candidate( + server, + state, + SimpleNamespace( + coinbase_tx_hex="c0ffee", + block_hash_hex=block_hash, + block_hex="00", + share_pass=True, + block_pass=True, + ), + ) + real_persist = ledger.persist_accepted_block - ledger.persist_accepted_block = blocking_persist # type: ignore[method-assign] - ledger.confirm_accepted_block = observed_confirm # type: ignore[method-assign] - ledger.mark_pool_block_inactive = mark_pool_block_inactive # type: ignore[method-assign] - accepted: list[bool] = [] - reconcile_results: list[dict[str, object]] = [] - errors: list[BaseException] = [] - delivery_admitted = threading.Event() - delivery_thread: threading.Thread | None = None - reconcile_thread: threading.Thread | None = None - mutation_lock = server._payout_balance_mutation_lock + def persist_then_lose_race(**kwargs: object) -> dict[str, object]: + result = real_persist(**kwargs) + rpc.tip = racing_winner + rpc.active.pop(block_hash, None) + rpc.getblockhash_override = racing_winner + return result - class ObservedBalanceLock: - def __enter__(self) -> ObservedBalanceLock: - if threading.current_thread() is reconcile_thread: - reconcile_lock_attempted.set() - mutation_lock.acquire() - return self + ledger.persist_accepted_block = persist_then_lose_race # type: ignore[method-assign] + reject_attempts: list[dict[str, object]] = [] - def __exit__(self, *_args: object) -> None: - mutation_lock.release() + def failing_reject(**kwargs: object) -> dict[str, object]: + reject_attempts.append(kwargs) + raise RuntimeError("psql briefly unavailable") - # The landing's audit-build release window drives the serializer - # through the plain lock interface as well. - def acquire(self, *args: object, **kwargs: object) -> bool: - return mutation_lock.acquire(*args, **kwargs) # type: ignore[arg-type] + ledger.reject_prepared_block = failing_reject # type: ignore[method-assign] + ledger.pool_block_state = ( # type: ignore[attr-defined] + lambda *, block_hash: { + "chain_state": "prepared", + "maturity_state": "immature", + } + ) - def release(self) -> None: - mutation_lock.release() + # The sealed terminal pass aborts inside the rejection (site + # attempt, then the writer's terminal-cleanup attempt): the + # candidate is retained and must match observations again. + self.assertTrue(server._submit_next_block_candidate_writer(candidate)) + self.assertEqual(len(reject_attempts), 2) + self.assertIsNotNone(getattr(server, "_retry_block_candidate", None)) + self.assertIn( + block_hash, server._outstanding_block_candidate_hashes + ) + self.assertNotIn( + "block-stale", + server.block_candidate_abandoned_counts, + ) - server._payout_balance_mutation_lock = ObservedBalanceLock() # type: ignore[assignment] + # Blockwait reports the pool's own hash during the backoff gap: + # the evidence registers instead of vanishing behind the seal. + self.assertTrue(server.observe_tip_for_refresh(block_hash)) + self.assertIn( + block_hash, server._tip_observed_accepted_block_hashes + ) + self.assertEqual(abandoned, []) - with tempfile.TemporaryDirectory() as tempdir: - server.audit_dir = Path(tempdir) - server.evidence_path = Path(tempdir) / "evidence.json" - server.ledger_writer_public_key_hex = "aa" * 32 - block_hash = "cd" * 32 - server.rpc = SubmitRpc( - tip="00" * 32, - block_hash=block_hash, - ledger=ledger, + # The retry can subsequently prove the candidate active and + # complete as submitted. The earlier reversible seals must never + # leave a contradictory abandonment count behind. + ledger.persist_accepted_block = real_persist # type: ignore[method-assign] + rpc.tip = block_hash + rpc.height = 10 + rpc.active[block_hash] = 10 + rpc.getblockhash_override = None + recovered_candidate = self._retained_candidate(server) + self.assertTrue( + server._submit_next_block_candidate_writer(recovered_candidate) ) - server.build_audit_bundle = ( # type: ignore[method-assign] - lambda **_kwargs: verified_block_bundle() + self.assertEqual(submitted, [block_hash]) + self.assertEqual(abandoned, []) + self.assertEqual(server.accepted_block_count, 1) + self.assertNotIn( + "block-stale", + server.block_candidate_abandoned_counts, ) - server.verify_bundle = ( # type: ignore[method-assign] - lambda *_args, **_kwargs: verified_audit_report() + + def test_definitive_ack_with_unprovable_chain_view_defers_not_abandons( + self, + ) -> None: + # Release-review finding: a definitive submitblock success followed + # by a chain view that cannot prove the block landed (foreign tip, + # header unknown, height taken by a sibling in that instant's view) + # must defer as accept-pending on the strength of the retained ack + # -- never terminally abandon accounting for a block the node + # accepted. The stash ages on the observation window, so a real + # orphan still terminalizes once the ack is stale (the sealed + # rejection tests above). + parent = "00" * 32 + block_hash = "ce" * 32 + racing_winner = "77" * 32 + server, state, ledger = submit_coordinator(tip=parent) + server.max_blocks = 2 + server.stop_after_block = False + with tempfile.TemporaryDirectory() as tempdir: + submitted, abandoned = self._accepted_tail_scaffolding(server, tempdir) + rpc = LostAckSubmitRpc( + start_tip=parent, + hash_by_hex={"00": block_hash}, ) - submission = SimpleNamespace( - coinbase_tx_hex="c0ffee", - block_hash_hex=block_hash, - block_hex="00", + rpc.lose_acks = False + server.rpc = rpc + candidate = block_candidate( + server, + state, + SimpleNamespace( + coinbase_tx_hex="c0ffee", + block_hash_hex=block_hash, + block_hex="00", + share_pass=True, + block_pass=True, + ), ) - candidate = block_candidate(server, state, submission) + real_persist = ledger.persist_accepted_block - def submit() -> None: - try: - accepted.append(server.submit_block_candidate(candidate)) - except BaseException as exc: # noqa: BLE001 - asserted below - errors.append(exc) + def persist_then_lose_race(**kwargs: object) -> dict[str, object]: + result = real_persist(**kwargs) + rpc.tip = racing_winner + rpc.active.pop(block_hash, None) + rpc.getblockhash_override = racing_winner + return result - def deliver() -> None: - try: - with server._payout_state_delivery_gate.delivery(): - note("replacement-delivery") - delivery_admitted.set() - if not release_delivery.wait(15): - raise AssertionError( - "timed out waiting to release payout delivery" - ) - except BaseException as exc: # noqa: BLE001 - asserted below - errors.append(exc) + ledger.persist_accepted_block = persist_then_lose_race # type: ignore[method-assign] - def reconcile() -> None: - try: - reconcile_results.append( - server.reconcile_prism_pool_blocks_once(tip_hash=block_hash) - ) - except BaseException as exc: # noqa: BLE001 - asserted below - errors.append(exc) - - submit_thread = threading.Thread(target=submit) - submit_thread.start() - confirmation_waited_for_delivery = False - try: - self.assertTrue(persist_started.wait(5)) - server.reorg_reconciler_enabled = True - reconcile_thread = threading.Thread(target=reconcile) - reconcile_thread.start() - self.assertTrue(reconcile_lock_attempted.wait(5)) - self.assertFalse(reconcile_mutated.is_set()) - delivery_thread = threading.Thread(target=deliver) - delivery_thread.start() - admitted_while_persisting = delivery_admitted.wait(5) - if admitted_while_persisting: - self.assertFalse(reconcile_mutated.is_set()) - release_persist.set() - self.assertTrue(persist_finished.wait(5)) - confirmation_waited_for_delivery = not confirm_started.wait(0.1) - finally: - release_persist.set() - release_delivery.set() - submit_thread.join(10) - if delivery_thread is not None: - delivery_thread.join(10) - if reconcile_thread is not None: - reconcile_thread.join(10) - - self.assertFalse(submit_thread.is_alive()) - self.assertIsNotNone(delivery_thread) - self.assertFalse(delivery_thread.is_alive()) - self.assertIsNotNone(reconcile_thread) - self.assertFalse(reconcile_thread.is_alive()) - if errors: - raise errors[0] - self.assertTrue(admitted_while_persisting) - # Confirmation is durable catch-up to the already-published - # prospective state and therefore does not wait on delivery. - self.assertFalse(confirmation_waited_for_delivery) - self.assertTrue(confirm_started.is_set()) - self.assertTrue(reconcile_mutated.is_set()) - self.assertEqual(accepted, [True]) - self.assertEqual(len(reconcile_results), 1) - self.assertEqual(reconcile_results[0]["inactive_blocks"], 1) - self.assertEqual(server._payout_state_generation, 2) - self.assertLess(events.index("persist-start"), events.index("replacement-delivery")) - self.assertLess(events.index("replacement-delivery"), events.index("persist-end")) - self.assertLess(events.index("persist-end"), events.index("confirm")) - self.assertLess(events.index("confirm"), events.index("reconcile-mutation")) - - def test_next_tip_preview_job_lands_after_parent_persistence(self) -> None: - old_tip = "00" * 32 - parent_hash = "cd" * 32 - child_hash = "ce" * 32 - server, state, ledger = submit_coordinator(tip=old_tip) - server._ensure_job_cache_state() - ledger.durable_payout_state = True - server.reorg_reconciler_enabled = False - server.stop_after_block = False - server.max_blocks = 10 - server.clients = {state} - server.refresh_jobs_after_accepted_block = ( # type: ignore[method-assign] - lambda **_kwargs: None - ) - build_started = threading.Event() - release_build = threading.Event() - persist_started = threading.Event() - release_persist = threading.Event() - original_persist = ledger.persist_accepted_block - original_confirm = ledger.confirm_accepted_block - preview = [ - { - "recipient_id": "miner-a", - "order_key": "miner-a", - "p2mr_program_hex": "11" * 32, - "balance_sats": 25, - } - ] - - def payout_bundle_payload() -> dict[str, object]: - return { - "found_block": {"coinbase_value_sats": 50_00000000}, - "ledger_window_attestation": { - "signature": {"public_key_hex": "aa" * 32} - }, - "payout_policy_manifest": { - "accounts": [ - { - "recipient_id": "miner-a", - "order_key": "miner-a", - "p2mr_program_hex": "11" * 32, - "gross_amount_sats": 25, - "prior_balance_sats": 0, - "candidate_balance_sats": 25, - "onchain_amount_sats": 0, - "carry_forward_balance_sats": 25, - "action": "accrued", - } - ] - }, - "signed_coinbase_manifest": { - "manifest": {"coinbase_tx_hex": "c0ffee", "payout_count": 1} - }, - } - - def blocking_payout_bundle(**_kwargs: object) -> dict[str, object]: - if not build_started.is_set(): - build_started.set() - if not release_build.wait(15): - raise AssertionError("timed out waiting to release parent bundle build") - return payout_bundle_payload() - - def blocking_persist(**kwargs: object) -> dict[str, object]: - if str(kwargs["block_hash"]).lower() == parent_hash: - persist_started.set() - if not release_persist.wait(15): - raise AssertionError("timed out waiting to release parent persistence") - return original_persist(**kwargs) - - def expose_confirmed_preview(**kwargs: object) -> dict[str, object]: - persisted = next( - row - for row in reversed(ledger.persisted) - if str(row["block_hash"]).lower() - == str(kwargs["block_hash"]).lower() - ) - ledger.prior_balances = server._accepted_block_payout_preview_from_bundle( - persisted["final_bundle"] # type: ignore[arg-type] - ) - return original_confirm(**kwargs) - - ledger.persist_accepted_block = blocking_persist # type: ignore[method-assign] - ledger.confirm_accepted_block = expose_confirmed_preview # type: ignore[method-assign] - - class TwoBlockRpc: - def __init__(self) -> None: - self.tip = old_tip - self.height = 9 - self.hashes = {9: old_tip} - self.submitted: list[str] = [] - - def call(self, method: str, params: object = None) -> object: - if method == "getbestblockhash": - return self.tip - if method == "getblockcount": - return self.height - if method == "submitblock": - block_hex = str((params or [""])[0]) # type: ignore[index] - block_hash = parent_hash if block_hex == "00" else child_hash - self.height += 1 - self.tip = block_hash - self.hashes[self.height] = block_hash - self.submitted.append(block_hash) - ledger.submit_seen = True - return None - if method == "getblockhash": - return self.hashes[int((params or [0])[0])] # type: ignore[index] - raise RuntimeError(method) - - rpc = TwoBlockRpc() - server.rpc = rpc - server.build_audit_bundle = blocking_payout_bundle # type: ignore[method-assign] - server.verify_bundle = ( # type: ignore[method-assign] - lambda *_args, **_kwargs: verified_audit_report() - ) - sent: list[dict[str, object]] = [] - state.send = lambda payload: sent.append(payload) # type: ignore[method-assign] - - def build_child_job(client: ClientState, *, clean_jobs: bool) -> object: - context = prism_context( - "child-job", - parent_hash, - worker=client.worker, - clean_jobs=clean_jobs, - ) - context.template["height"] = 11 - context.prior_balances = server._prior_balances_for_job_parent(parent_hash) - context.payout_state_generation = server._payout_state_generation - return context - - server.build_job_for_client = build_child_job # type: ignore[method-assign] - parent_submission = SimpleNamespace( - coinbase_tx_hex="c0ffee", - block_hash_hex=parent_hash, - block_hex="00", - ) - parent_candidate = block_candidate(server, state, parent_submission) - parent_candidate.context.prospective_prior_balances = ( - server._serialize_prior_balance_preview(preview) - ) - parent_results: list[bool] = [] - parent_errors: list[BaseException] = [] - - def submit_parent() -> None: - try: - parent_results.append(server.submit_block_candidate(parent_candidate)) - except BaseException as exc: # noqa: BLE001 - asserted below - parent_errors.append(exc) - - with tempfile.TemporaryDirectory() as tempdir: - server.audit_dir = Path(tempdir) - server.evidence_path = Path(tempdir) / "evidence.json" - server.ledger_writer_public_key_hex = "aa" * 32 - parent_thread = threading.Thread(target=submit_parent) - parent_thread.start() - try: - self.assertTrue(build_started.wait(5)) - self.assertEqual(ledger.current_prior_balances(), []) - self.assertTrue(server.maybe_send_job(state, clean_jobs=True)) - child_context = state.active_job - self.assertIsNotNone(child_context) - assert child_context is not None - self.assertTrue(parent_thread.is_alive()) - self.assertEqual(child_context.prior_balances, preview) - self.assertEqual( - child_context.payout_state_generation, - server._payout_state_generation, - ) - preview_generation = server._payout_state_generation - self.assertEqual(sent[-1]["method"], "mining.notify") - self.assertTrue(sent[-1]["params"][8]) # type: ignore[index] - - child_submission = SimpleNamespace( - header_hex="bb" * 80, - coinbase_tx_hex="c0ffee", - block_hash_hex=child_hash, - block_hex="01", - share_pass=True, - block_pass=True, - ) - with patch( - "lab.prism.prism_coordinator.direct_stratum.assemble_submission", - return_value=child_submission, - ): - self.assertFalse( - server.handle_submit( - state, - ["miner-a", "child-job", "00" * 8, "00000001", "00000002"], - ) - ) - self.assertEqual(server.block_candidate_queue.qsize(), 1) - release_build.set() - self.assertTrue(persist_started.wait(5)) - self.assertTrue(parent_thread.is_alive()) - finally: - release_build.set() - release_persist.set() - parent_thread.join(10) - - self.assertFalse(parent_thread.is_alive()) - if parent_errors: - raise parent_errors[0] - self.assertEqual(parent_results, [True]) - self.assertEqual(ledger.current_prior_balances(), preview) - self.assertEqual(server._payout_state_generation, preview_generation) - self.assertTrue(server.submit_next_block_candidate()) - - self.assertEqual(rpc.submitted, [parent_hash, child_hash]) - self.assertEqual(server.accepted_block_count, 2) - self.assertEqual(len(ledger.persisted), 2) - self.assertEqual(len(ledger.confirmed), 2) - self.assertEqual( - server.block_candidate_abandoned_counts.get(PRISM_REJECTION_STALE_JOB, 0), - 0, - ) - self.assertEqual(server._accepted_block_payout_previews, {}) - - def test_direct_block_preparation_does_not_hold_delivery_gate(self) -> None: - server, state, ledger = submit_coordinator() - server._ensure_job_cache_state() - entered = threading.Event() - release = threading.Event() - accepted: list[bool] = [] - errors: list[BaseException] = [] - block_hash = "cf" * 32 - original_persist = ledger.persist_accepted_block - - def blocking_persist(**kwargs: object) -> dict[str, object]: - self.assertIsNone( - server._payout_state_delivery_gate._mutation_owner - ) - entered.set() - if not release.wait(5): - raise AssertionError("test did not release direct-block preparation") - return original_persist(**kwargs) - - ledger.persist_accepted_block = blocking_persist # type: ignore[method-assign] - with tempfile.TemporaryDirectory() as tempdir: - server.audit_dir = Path(tempdir) - server.evidence_path = Path(tempdir) / "evidence.json" - server.ledger_writer_public_key_hex = "aa" * 32 - server.rpc = SubmitRpc( - tip="00" * 32, - block_hash=block_hash, - ledger=ledger, - ) - server.build_audit_bundle = ( # type: ignore[method-assign] - lambda **_kwargs: verified_block_bundle() - ) - server.verify_bundle = ( # type: ignore[method-assign] - lambda *_args, **_kwargs: verified_audit_report() - ) - submission = SimpleNamespace( - coinbase_tx_hex="c0ffee", - block_hash_hex=block_hash, - block_hex="00", - ) - - def submit() -> None: - try: - accepted.append( - server.submit_block_candidate( - block_candidate(server, state, submission) - ) - ) - except BaseException as exc: # pragma: no cover - asserted below - errors.append(exc) - - thread = threading.Thread(target=submit) - thread.start() - try: - self.assertTrue(entered.wait(5)) - with server._payout_state_delivery_gate.delivery_cancelable( - lambda: False, - generation=server._payout_state_generation, - priority=True, - ) as admission: - self.assertTrue(admission) - release.set() - thread.join(5) - self.assertFalse(thread.is_alive()) - self.assertTrue( - server._payout_state_prepare_lock.acquire(timeout=1) - ) - server._payout_state_prepare_lock.release() - admission.mark_delivered() - finally: - release.set() - thread.join(5) - - self.assertFalse(thread.is_alive()) - self.assertEqual(errors, []) - self.assertEqual(accepted, [True]) - self.assertEqual(server._payout_state_generation, 1) - - def test_failed_direct_block_audit_does_not_reserve_payout_source(self) -> None: - for failure_phase in ("build", "verify"): - with self.subTest(failure_phase=failure_phase): - server, state, ledger = submit_coordinator() - server._ensure_job_cache_state() - initial_source = server._payout_state_source - initial_published = server._published_payout_state - block_hash = "ce" * 32 - server.rpc = SubmitRpc( - tip="00" * 32, - block_hash=block_hash, - ledger=ledger, - ) - submission = SimpleNamespace( - coinbase_tx_hex="c0ffee", - block_hash_hex=block_hash, - block_hex="00", - ) - - with tempfile.TemporaryDirectory() as tempdir: - server.audit_dir = Path(tempdir) - server.ledger_writer_public_key_hex = "aa" * 32 - if failure_phase == "build": - server.build_audit_bundle = ( # type: ignore[method-assign] - lambda **_kwargs: (_ for _ in ()).throw( - RuntimeError("audit reconstruction failed") - ) - ) - else: - server.build_audit_bundle = ( # type: ignore[method-assign] - lambda **_kwargs: verified_block_bundle() - ) - server.verify_bundle = ( # type: ignore[method-assign] - lambda *_args, **_kwargs: (_ for _ in ()).throw( - RuntimeError("audit verification failed") - ) - ) - - with self.assertRaisesRegex( - RuntimeError, - "audit (reconstruction|verification) failed", - ): - server.submit_block_candidate( - block_candidate(server, state, submission) - ) - - self.assertEqual(server._payout_state_source, initial_source) - self.assertEqual(server._published_payout_state, initial_published) - self.assertEqual(server._payout_state_generation, 0) - self.assertEqual(ledger.persisted, []) - - def test_uncertain_direct_block_ledger_commit_fences_delivery(self) -> None: - server, state, ledger = submit_coordinator() - server._ensure_job_cache_state() - block_hash = "cd" * 32 - server.rpc = SubmitRpc( - tip="00" * 32, - block_hash=block_hash, - ledger=ledger, - ) - server.build_audit_bundle = ( # type: ignore[method-assign] - lambda **_kwargs: verified_block_bundle() - ) - server.verify_bundle = ( # type: ignore[method-assign] - lambda *_args, **_kwargs: verified_audit_report() - ) - ledger.confirm_accepted_block = ( # type: ignore[method-assign] - lambda **_kwargs: (_ for _ in ()).throw( - RuntimeError("ledger confirmation unavailable") - ) - ) - submission = SimpleNamespace( - coinbase_tx_hex="c0ffee", - block_hash_hex=block_hash, - block_hex="00", - ) - - with tempfile.TemporaryDirectory() as tempdir: - server.audit_dir = Path(tempdir) - server.ledger_writer_public_key_hex = "aa" * 32 - with self.assertRaisesRegex( - RuntimeError, - "ledger confirmation unavailable", - ): - server.submit_block_candidate( - block_candidate(server, state, submission) - ) - - self.assertEqual(len(ledger.persisted), 1) - # The prospective preview was source/generation 1; the uncertain - # durable commit supersedes it with fenced source 2. - self.assertEqual(server._payout_state_generation, 1) - self.assertEqual(server._payout_state_source[0], 2) - self.assertEqual(server._published_payout_state.source_generation, 1) - self.assertTrue(server._payout_state_publication_blocked) - self.assertTrue(server._payout_state_delivery_gate._delivery_blocked) - - def test_uncertain_commit_supersedes_concurrently_published_source(self) -> None: - server, state, ledger = submit_coordinator() - server._ensure_job_cache_state() - block_hash = "cb" * 32 - newer_tip = "dc" * 32 - server.rpc = SubmitRpc( - tip="00" * 32, - block_hash=block_hash, - ledger=ledger, - ) - server.build_audit_bundle = ( # type: ignore[method-assign] - lambda **_kwargs: verified_block_bundle() - ) - server.verify_bundle = ( # type: ignore[method-assign] - lambda *_args, **_kwargs: verified_audit_report() - ) - - def publish_newer_source_then_fail(**_kwargs: object) -> dict[str, object]: - server._reserve_payout_state_source( - "external_tip", - tip_hash=newer_tip, - ) - self.assertEqual( - server._publish_payout_state_candidate( - server._current_payout_state_candidate() - ), - 2, - ) - raise RuntimeError("ledger confirmation unavailable") - - ledger.confirm_accepted_block = publish_newer_source_then_fail # type: ignore[method-assign] - submission = SimpleNamespace( - coinbase_tx_hex="c0ffee", - block_hash_hex=block_hash, - block_hex="00", - ) - - with tempfile.TemporaryDirectory() as tempdir: - server.audit_dir = Path(tempdir) - server.ledger_writer_public_key_hex = "aa" * 32 - with self.assertRaisesRegex( - RuntimeError, - "ledger confirmation unavailable", - ): - server.submit_block_candidate( - block_candidate(server, state, submission) - ) - - self.assertEqual(server._payout_state_generation, 2) - self.assertEqual(server._published_payout_state.source_generation, 2) - self.assertEqual(server._payout_state_source[0], 3) - self.assertEqual(server._payout_state_source[1], newer_tip) - self.assertEqual( - server._payout_state_source[2], - "direct_block_uncertain", - ) - self.assertTrue(server._payout_state_publication_blocked) - self.assertTrue(server._payout_state_delivery_gate._delivery_blocked) - - def test_post_confirm_publication_loss_completes_candidate_and_fences(self) -> None: - # Once persist + confirm are durable, losing the forced payout - # publication must not abort the candidate: the outbox row is marked - # submitted and the success tail runs, while delivery stays fenced - # until the scheduled refresh publishes the newest source. - server, state, ledger = submit_coordinator() - server._ensure_job_cache_state() - server.max_blocks = 2 - server.stop_after_block = False - block_hash = "e1" * 32 - server.rpc = SubmitRpc( - tip="00" * 32, - block_hash=block_hash, - ledger=ledger, - ) - server.build_audit_bundle = ( # type: ignore[method-assign] - lambda **_kwargs: verified_block_bundle() - ) - server.verify_bundle = ( # type: ignore[method-assign] - lambda *_args, **_kwargs: verified_audit_report() - ) - submitted: list[dict[str, object]] = [] - ledger.mark_block_candidate_submitted = ( # type: ignore[attr-defined] - lambda **kwargs: submitted.append(kwargs) or True - ) - real_confirm = ledger.confirm_accepted_block - - def confirm_then_reserve_newer_source(**kwargs: object) -> dict[str, object]: - result = real_confirm(**kwargs) - server._reserve_payout_state_source( - "external_tip", - tip_hash="ee" * 32, - ) - return result - - ledger.confirm_accepted_block = confirm_then_reserve_newer_source # type: ignore[method-assign] - server._publish_current_payout_state_with_retry_budget = ( # type: ignore[method-assign] - lambda **_kwargs: None - ) - submission = SimpleNamespace( - coinbase_tx_hex="c0ffee", - block_hash_hex=block_hash, - block_hex="00", - ) - - with tempfile.TemporaryDirectory() as tempdir: - server.audit_dir = Path(tempdir) - server.evidence_path = Path(tempdir) / "evidence.json" - server.ledger_writer_public_key_hex = "aa" * 32 - self.assertTrue( - server._submit_next_block_candidate_writer( - block_candidate(server, state, submission) - ) - ) - - self.assertIsNone(getattr(server, "_retry_block_candidate", None)) - self.assertEqual(len(ledger.confirmed), 1) - self.assertEqual(len(submitted), 1) - self.assertEqual(submitted[0]["block_hash"], block_hash) - self.assertEqual(server.accepted_block_count, 1) - self.assertIn(block_hash, server._accounted_accepted_block_hashes) - self.assertTrue(server._payout_state_publication_blocked) - self.assertTrue(server._payout_state_delivery_gate._delivery_blocked) - self.assertTrue(server.tip_refresh_is_pending()) - - # The scheduled refresh publishes the pending source and reopens - # delivery without any candidate replay. - del server._publish_current_payout_state_with_retry_budget - self.assertIsNotNone(server._publish_current_payout_state_with_retry_budget()) - self.assertFalse(server._payout_state_publication_blocked) - self.assertFalse(server._payout_state_delivery_gate._delivery_blocked) - - def test_idempotent_direct_block_replay_skips_publication(self) -> None: - server, state, ledger = submit_coordinator() - server._ensure_job_cache_state() - server.max_blocks = 2 - server.stop_after_block = False - block_hash = "d0" * 32 - with tempfile.TemporaryDirectory() as tempdir: - server.audit_dir = Path(tempdir) - server.evidence_path = Path(tempdir) / "evidence.json" - server.ledger_writer_public_key_hex = "aa" * 32 - rpc = SubmitRpc( - tip="00" * 32, - block_hash=block_hash, - ledger=ledger, - ) - server.rpc = rpc - server.build_audit_bundle = ( # type: ignore[method-assign] - lambda **_kwargs: verified_block_bundle() - ) - server.verify_bundle = ( # type: ignore[method-assign] - lambda *_args, **_kwargs: verified_audit_report() - ) - submission = SimpleNamespace( - coinbase_tx_hex="c0ffee", - block_hash_hex=block_hash, - block_hex="00", - ) - - self.assertTrue( - server.submit_block_candidate( - block_candidate(server, state, submission) - ) - ) - self.assertEqual(server._payout_state_generation, 1) - self.assertEqual( - server._published_payout_state.source_tip_hash, - block_hash, - ) - self.assertEqual(server._payout_state_source[0], 1) - - # Replay the durable candidate after its block landed, its - # confirmation committed, and the network built on top of it. - # qbit_confirm_pool_block reports confirmed_count=0 for an - # already-confirmed block whose height no longer matches the - # active tip — the sustained replay state of the post-block - # livelock. The published payout state already covers the - # candidate, so the replay must not reserve a source, bump the - # generation, wipe the job-bundle cache, or schedule refresh - # churn. - child_tip = "d7" * 32 - - class AncestorReplayRpc: - def call(self, method: str, params: object = None) -> object: - if method == "getbestblockhash": - return child_tip - if method == "getblockheader": - if params != [block_hash]: - raise AssertionError(params) - return {"height": 10, "confirmations": 2} - if method == "getblockhash": - if params != [10]: - raise AssertionError(params) - return block_hash - if method == "getblockcount": - return 11 - if method == "submitblock": - raise AssertionError( - "active ancestor must not be resubmitted" - ) - raise RuntimeError(method) - - server.rpc = AncestorReplayRpc() - ledger.pool_block_state = ( # type: ignore[attr-defined] - lambda **_kwargs: { - "chain_state": "confirmed", - "maturity_state": "immature", - } - ) - ledger.confirm_accepted_block = ( # type: ignore[method-assign] - lambda **_kwargs: {"backend": "fake", "confirmed_count": 0} - ) - retry_calls = 0 - - def count_retry() -> None: - nonlocal retry_calls - retry_calls += 1 - - server._schedule_tip_refresh_retry = count_retry # type: ignore[method-assign] - cache_key = ("sentinel",) - server._job_bundle_cache[cache_key] = object() - discarded_before = server.payout_state_candidates_discarded - pending_marks_before = server._tip_refresh_pending_counter - - self.assertTrue( - server.submit_block_candidate( - block_candidate(server, state, submission) - ) - ) - - self.assertEqual(server._payout_state_generation, 1) - self.assertEqual(server._published_payout_state.source_generation, 1) - self.assertEqual(server._published_payout_state.source_tip_hash, block_hash) - self.assertEqual(server._payout_state_source[0], 1) - self.assertIn(cache_key, server._job_bundle_cache) - self.assertEqual(retry_calls, 0) - self.assertEqual( - server._tip_refresh_pending_counter, - pending_marks_before, - ) - self.assertEqual( - server.payout_state_candidates_discarded, - discarded_before, - ) - self.assertFalse(server._payout_state_publication_blocked) - self.assertFalse(server._payout_state_delivery_gate._delivery_blocked) - - def test_leaked_publication_fence_replay_republishes(self) -> None: - server, state, ledger = submit_coordinator() - server._ensure_job_cache_state() - server.max_blocks = 2 - server.stop_after_block = False - block_hash = "d8" * 32 - with tempfile.TemporaryDirectory() as tempdir: - server.audit_dir = Path(tempdir) - server.evidence_path = Path(tempdir) / "evidence.json" - server.ledger_writer_public_key_hex = "aa" * 32 - server.rpc = SubmitRpc( - tip="00" * 32, - block_hash=block_hash, - ledger=ledger, - ) - server.build_audit_bundle = ( # type: ignore[method-assign] - lambda **_kwargs: verified_block_bundle() - ) - server.verify_bundle = ( # type: ignore[method-assign] - lambda *_args, **_kwargs: verified_audit_report() - ) - submission = SimpleNamespace( - coinbase_tx_hex="c0ffee", - block_hash_hex=block_hash, - block_hex="00", - ) - - self.assertTrue( - server.submit_block_candidate( - block_candidate(server, state, submission) - ) - ) - self.assertEqual(server._payout_state_generation, 1) - - # Simulate the exception tail a replay must heal: a prior attempt - # force-blocked delivery while its source generation already - # matched the published source, then failed before republishing. - # The leaked fence blocks every job build until a publication - # lands, so a confirmed_count=0 replay must not take the covered - # skip here. - server._block_payout_state_publication(force=True) - self.assertTrue(server._payout_state_publication_blocked) - self.assertEqual( - server._payout_state_source[0], - server._published_payout_state.source_generation, - ) - child_tip = "d9" * 32 - - class AncestorReplayRpc: - def call(self, method: str, params: object = None) -> object: - if method == "getbestblockhash": - return child_tip - if method == "getblockheader": - if params != [block_hash]: - raise AssertionError(params) - return {"height": 10, "confirmations": 2} - if method == "getblockhash": - if params != [10]: - raise AssertionError(params) - return block_hash - if method == "getblockcount": - return 11 - if method == "submitblock": - raise AssertionError( - "active ancestor must not be resubmitted" - ) - raise RuntimeError(method) - - server.rpc = AncestorReplayRpc() - ledger.pool_block_state = ( # type: ignore[attr-defined] - lambda **_kwargs: { - "chain_state": "confirmed", - "maturity_state": "immature", - } - ) - ledger.confirm_accepted_block = ( # type: ignore[method-assign] - lambda **_kwargs: {"backend": "fake", "confirmed_count": 0} - ) - - self.assertTrue( - server.submit_block_candidate( - block_candidate(server, state, submission) - ) - ) - - self.assertEqual(server._payout_state_generation, 2) - # Fence healing republishes identical covered state. It advances the - # delivery generation without inventing a logical invalidation source. - self.assertEqual(server._published_payout_state.source_generation, 1) - self.assertEqual(server._payout_state_source[0], 1) - self.assertFalse(server._payout_state_publication_blocked) - self.assertFalse(server._payout_state_delivery_gate._delivery_blocked) - - def test_direct_block_disabled_reconciler_bounds_publish_supersession(self) -> None: - server, state, ledger = submit_coordinator() - server._ensure_job_cache_state() - server.payout_reconcile_supersession_retries = 2 - block_hash = "d3" * 32 - real_publish = server._publish_payout_state_candidate - publish_attempts = 0 - - def supersede_before_publish(candidate: object) -> int | None: - nonlocal publish_attempts - publish_attempts += 1 - server._reserve_payout_state_source( - "external_tip", - tip_hash=f"{publish_attempts + 20:064x}", - ) - return real_publish(candidate) # type: ignore[arg-type] - - server._publish_payout_state_candidate = supersede_before_publish # type: ignore[method-assign] - with tempfile.TemporaryDirectory() as tempdir: - server.audit_dir = Path(tempdir) - server.evidence_path = Path(tempdir) / "evidence.json" - server.ledger_writer_public_key_hex = "aa" * 32 - server.rpc = SubmitRpc( - tip="00" * 32, - block_hash=block_hash, - ledger=ledger, - ) - server.build_audit_bundle = ( # type: ignore[method-assign] - lambda **_kwargs: verified_block_bundle() - ) - server.verify_bundle = ( # type: ignore[method-assign] - lambda *_args, **_kwargs: verified_audit_report() - ) - submission = SimpleNamespace( - coinbase_tx_hex="c0ffee", - block_hash_hex=block_hash, - block_hex="00", - ) - - accepted = server.submit_block_candidate( - block_candidate(server, state, submission) - ) - - self.assertTrue(accepted) - self.assertEqual(publish_attempts, 3) - self.assertEqual(server._payout_state_generation, 0) - self.assertTrue(server._payout_state_publication_blocked) - self.assertTrue(server._payout_state_delivery_gate._delivery_blocked) - self.assertTrue(server.tip_refresh_is_pending()) - - def test_untrusted_direct_block_reconcile_publishes_newer_source_once(self) -> None: - server, state, ledger = submit_coordinator() - server._ensure_job_cache_state() - server.reorg_reconciler_enabled = True - server.ensure_reorg_reconciled_for_tip = ( # type: ignore[method-assign] - lambda _tip, *, _coalesce_same_tip: not _coalesce_same_tip - ) - server.qbit_chain_view_untrusted = lambda: True # type: ignore[method-assign] - block_hash = "d1" * 32 - newer_tip = "d2" * 32 - - def superseding_noop_confirmation(**_kwargs: object) -> dict[str, object]: - server._reserve_payout_state_source( - "external_tip", - tip_hash=newer_tip, - ) - return {"backend": "fake", "confirmed_count": 0} - - ledger.confirm_accepted_block = superseding_noop_confirmation # type: ignore[method-assign] - with tempfile.TemporaryDirectory() as tempdir: - server.audit_dir = Path(tempdir) - server.evidence_path = Path(tempdir) / "evidence.json" - server.ledger_writer_public_key_hex = "aa" * 32 - server.rpc = SubmitRpc( - tip="00" * 32, - block_hash=block_hash, - ledger=ledger, - ) - server.build_audit_bundle = ( # type: ignore[method-assign] - lambda **_kwargs: verified_block_bundle() - ) - server.verify_bundle = ( # type: ignore[method-assign] - lambda *_args, **_kwargs: verified_audit_report() - ) - submission = SimpleNamespace( - coinbase_tx_hex="c0ffee", - block_hash_hex=block_hash, - block_hex="00", - ) - - accepted = server.submit_block_candidate( - block_candidate(server, state, submission) - ) - - self.assertTrue(accepted) - self.assertEqual(server._payout_state_generation, 2) - self.assertEqual(server._published_payout_state.source_tip_hash, newer_tip) - self.assertEqual(server.payout_state_candidates_discarded, 0) - self.assertEqual(server.reorg_reconcile_skip_count, 1) - - def test_verified_canonical_bundle_path_requires_exact_bytes(self) -> None: - with tempfile.TemporaryDirectory() as tempdir: - candidate_path = Path(tempdir) / "candidate.json" - candidate_bytes = b'{"canonical":true}' - candidate_path.write_bytes(candidate_bytes) - canonical_sha256 = hashlib.sha256(candidate_bytes).hexdigest() - - self.assertEqual( - PrismCoordinator.verified_canonical_bundle_path( - candidate_path, - {"audit_bundle_sha256_hex": canonical_sha256.upper()}, - ), - candidate_path, - ) - self.assertIsNone( - PrismCoordinator.verified_canonical_bundle_path( - candidate_path, - {"audit_bundle_sha256_hex": "00" * 32}, - ) - ) - - def test_active_ancestor_candidate_resumes_full_finalization_without_resubmit(self) -> None: - server, state, ledger = submit_coordinator() - server.max_blocks = 10 - server.stop_after_block = False - refreshes: list[str] = [] - server.refresh_jobs_after_accepted_block = ( # type: ignore[method-assign] - lambda **kwargs: refreshes.append(str(kwargs["block_hash"])) - ) - with tempfile.TemporaryDirectory() as tempdir: - server.audit_dir = Path(tempdir) - server.evidence_path = Path(tempdir) / "evidence.json" - server.ledger_writer_public_key_hex = "aa" * 32 - block_hash = "ac" * 32 - - class ActiveAncestorRpc: - def __init__(self) -> None: - self.revalidated_heights: list[int] = [] - - def call(self, method: str, params: object = None) -> object: - if method == "getbestblockhash": - return "ef" * 32 - if method == "getblockheader": - self.assert_candidate(params) - return {"height": 10, "confirmations": 2} - if method == "getblockcount": - return 11 - if method == "getblockhash": - if params != [10]: - raise AssertionError(params) - self.revalidated_heights.append(10) - return block_hash - if method == "submitblock": - raise AssertionError("active ancestor must not be resubmitted") - raise RuntimeError(method) - - @staticmethod - def assert_candidate(params: object) -> None: - if params != [block_hash]: - raise AssertionError(params) - - active_rpc = ActiveAncestorRpc() - server.rpc = active_rpc - server.ensure_reorg_reconciled_for_tip = ( # type: ignore[method-assign] - lambda _tip, *, _coalesce_same_tip: not _coalesce_same_tip - ) - server.build_audit_bundle = lambda **_kwargs: { # type: ignore[method-assign] - "found_block": {"coinbase_value_sats": 50_00000000}, - "ledger_window_attestation": {"signature": {"public_key_hex": "aa" * 32}}, - "payout_policy_manifest": {"accounts": []}, - "signed_coinbase_manifest": { - "manifest": {"coinbase_tx_hex": "c0ffee", "payout_count": 1} - }, - } - server.verify_bundle = lambda *_args, **_kwargs: { # type: ignore[method-assign] - "coinbase_txid": "11" * 32, - "coinbase_manifest_sha256_hex": "22" * 32, - "audit_bundle_sha256_hex": "33" * 32, - "coinbase_tx_hex": "c0ffee", - } - submission = SimpleNamespace( - coinbase_tx_hex="c0ffee", - block_hash_hex=block_hash, - block_hex="00", - ) - candidate = block_candidate(server, state, submission) - - self.assertTrue(server.submit_block_candidate(candidate)) - # Direct finalization records the refresh for its caller so slow - # job fanout happens after writer admission has been released. - self.assertEqual( - state.post_accept_refresh_block, - (10, block_hash), - ) - server.refresh_jobs_after_pending_accepted_block( - state, - heartbeat_name="block_submitter", - ) - # A failed outbox terminal update can replay A after later block B - # changes global balances. Validate A against its height-bounded - # active-chain view and suppress duplicate process-local accounting. - ledger.durable_payout_state = True - ledger.prior_balances = [ - { - "recipient_id": "later-miner", - "order_key": "later-miner", - "p2mr_program_hex": "22" * 32, - "balance_sats": 50, - } - ] - ledger.pool_block_state = lambda **_kwargs: { # type: ignore[attr-defined] - "block_hash": block_hash, - "block_height": 10, - "parent_hash": "00" * 32, - "chain_state": "confirmed", - "maturity_state": "immature", - } - ledger.prior_balances_after_pool_block = ( # type: ignore[attr-defined] - lambda **_kwargs: [] - ) - self.assertTrue(server.submit_block_candidate(candidate)) - - self.assertEqual([row["block_hash"] for row in ledger.persisted], [block_hash] * 2) - # This compatibility builder ignores canonical_output_path, so its - # Python fallback remains verifier-only. - self.assertIsNone(ledger.persisted[0]["canonical_bundle_path"]) - self.assertEqual(ledger.confirmed[0]["active_tip_height"], 10) - self.assertEqual(ledger.confirmed[0]["block_hash"], block_hash) - self.assertEqual(active_rpc.revalidated_heights, [10, 10]) - self.assertFalse(ledger.confirmed[0]["submit_seen_at_confirm"]) - # The share credit happens on the client thread at submit time now; - # the block path itself appends nothing. - self.assertEqual(len(ledger.pending), 0) - self.assertFalse(server.stop_event.is_set()) - self.assertEqual(server.accepted_block_count, 1) - self.assertEqual(refreshes, [block_hash]) - self.assertEqual(server.latest_evidence["persistence"]["block_count"], 1) - self.assertEqual(server.latest_evidence["confirmation"]["confirmed_count"], 1) - # Evidence carries an aggregate miner count, not a materialized list of - # every miner id (which scanned the whole ledger twice under the lock). - self.assertEqual(server.latest_evidence["accepted_share_count"], 0) - self.assertEqual(server.latest_evidence["distinct_miner_count"], 0) - self.assertNotIn("distinct_miners", server.latest_evidence) - - def test_audit_retention_prunes_only_live_and_candidate_files(self) -> None: - server = coordinator() - with tempfile.TemporaryDirectory() as tempdir: - server.audit_dir = Path(tempdir) - server.audit_live_bundle_retention = 2 - server.audit_candidate_retention_seconds = 0 - for index in range(4): - path = Path(tempdir) / f"prism-live-audit-bundle-{index + 1}-{'aa' * 32}.json" - path.write_text("{}", encoding="utf-8") - os.utime(path, (100 + index, 100 + index)) - candidate = Path(tempdir) / f"prism-live-audit-bundle-candidate-{'bb' * 32}.json" - candidate.write_text("{}", encoding="utf-8") - temp_candidate = Path(tempdir) / f".prism-live-audit-bundle-candidate-{'bb' * 32}.json.tmp" - temp_candidate.write_text("{}", encoding="utf-8") - body = Path(tempdir) / f"prism-audit-bundle-body-{'cc' * 32}-{'dd' * 32}.json" - body.write_text("{}", encoding="utf-8") - segment = Path(tempdir) / f"prism-audit-share-segment-1-2-{'ee' * 32}.json" - segment.write_text("{}", encoding="utf-8") - - server.prune_audit_artifacts() - - live_names = sorted(path.name for path in Path(tempdir).glob("prism-live-audit-bundle-[0-9]*.json")) - self.assertEqual( - live_names, - [ - f"prism-live-audit-bundle-3-{'aa' * 32}.json", - f"prism-live-audit-bundle-4-{'aa' * 32}.json", - ], - ) - self.assertFalse(candidate.exists()) - self.assertFalse(temp_candidate.exists()) - self.assertTrue(body.exists()) - self.assertTrue(segment.exists()) - - def test_audit_retention_zero_preserves_current_live_envelope(self) -> None: - server = coordinator() - with tempfile.TemporaryDirectory() as tempdir: - server.audit_dir = Path(tempdir) - server.audit_live_bundle_retention = 0 - old = Path(tempdir) / f"prism-live-audit-bundle-1-{'aa' * 32}.json" - old.write_text("{}", encoding="utf-8") - current = Path(tempdir) / f"prism-live-audit-bundle-2-{'bb' * 32}.json" - current.write_text("{}", encoding="utf-8") - - server.prune_audit_artifacts(keep_live_path=current) - - self.assertFalse(old.exists()) - self.assertTrue(current.exists()) - - def test_accepted_direct_block_refreshes_clean_job_after_submit_response(self) -> None: - old_tip = "00" * 32 - block_hash = "ab" * 32 - server, state, ledger = submit_coordinator(tip=old_tip) - server.stop_after_block = False - server.max_blocks = 10 - server.clients = {state} - state.share_difficulty = Decimal("1") - sent: list[dict[str, object]] = [] - state.send = lambda payload: sent.append(payload) # type: ignore[method-assign] - old_context = server.jobs["job-1"] - server.tip_template_snapshot = QbitTipTemplateSnapshot( - bestblockhash=old_tip, - previousblockhash=old_tip, - template_fingerprint=qbit_template_fingerprint(old_context.template), - ) - - def build_fresh_job(client: ClientState, *, clean_jobs: bool) -> object: - self.assertIs(client, state) - self.assertNotIn( - "accepted_block_handling", - server._ensure_shutdown_controller().snapshot()["active_writers"], - ) - return prism_context( - "fresh-job", - block_hash, - worker=state.worker, - difficulty=server.desired_client_share_difficulty(client), - clean_jobs=clean_jobs, - ) - - server.build_job_for_client = build_fresh_job # type: ignore[method-assign] - - with tempfile.TemporaryDirectory() as tempdir: - server.audit_dir = Path(tempdir) - server.evidence_path = Path(tempdir) / "evidence.json" - server.ledger_writer_public_key_hex = "aa" * 32 - server.rpc = SubmitAcceptingTemplateRpc(old_tip=old_tip, block_hash=block_hash, ledger=ledger) - server.build_audit_bundle = lambda **_kwargs: verified_block_bundle() # type: ignore[method-assign] - server.verify_bundle = lambda *_args, **_kwargs: verified_audit_report() # type: ignore[method-assign] - submission = SimpleNamespace( - header_hex="aa" * 80, - share_pass=True, - block_pass=True, - coinbase_tx_hex="c0ffee", - block_hash_hex=block_hash, - block_hex="00", - ) - - with patch( - "lab.prism.prism_coordinator.direct_stratum.assemble_submission", - return_value=submission, - ): - server.handle_request( - state, - { - "id": "submit-1", - "method": "mining.submit", - "params": ["miner-a", "job-1", "00" * 8, "00000001", "00000002"], - }, - ) - # The ack goes out before the block path runs; draining the - # submitter queue lands the block and pushes fresh work. - self.assertEqual(sent, [{"id": "submit-1", "result": True, "error": None}]) - self.assertTrue(server.submit_next_block_candidate()) - self.assertEqual(server.poll_qbit_tip_template_once(), 1) - - self.assertEqual(sent[0], {"id": "submit-1", "result": True, "error": None}) - self.assertEqual([payload.get("method") for payload in sent[1:]], ["mining.set_difficulty", "mining.notify"]) - self.assertEqual(sent[2]["params"][0], "fresh-job") - self.assertTrue(sent[2]["params"][8]) - self.assertEqual(server.tip_refresh_job_count, 1) - self.assertEqual(server.post_accept_refresh_failure_count, 0) - self.assertEqual(server.accepted_block_count, 1) - self.assertNotIn("job-1", server.jobs) - self.assertIn("fresh-job", server.jobs) - self.assertEqual(state.active_job_ids, {"fresh-job"}) - self.assertIn(state, server.clients) - server.stale_grace_seconds = 0 - - with self.assertRaises(StratumError) as raised: - server.handle_submit( - state, - ["miner-a", "job-1", "00" * 8, "00000001", "00000002"], - ) - self.assertEqual(raised.exception.code, 21) - self.assertEqual(raised.exception.reason, PRISM_REJECTION_UNKNOWN_JOB) - - fresh_submission = SimpleNamespace( - header_hex="bb" * 80, - block_hash_hex="bc" * 32, - share_pass=True, - block_pass=False, - ) - with patch( - "lab.prism.prism_coordinator.direct_stratum.assemble_submission", - return_value=fresh_submission, - ): - should_close = server.handle_submit( - state, - ["miner-a", "fresh-job", "00" * 8, "00000001", "00000002"], - ) - - self.assertFalse(should_close) - self.assertEqual(ledger.pending[-1].job_id, "fresh-job") - - def test_post_accept_notification_does_not_run_failing_template_build(self) -> None: - old_tip = "00" * 32 - block_hash = "ad" * 32 - server, state, ledger = submit_coordinator(tip=old_tip) - server.stop_after_block = False - server.max_blocks = 10 - server.clients = {state} - sent: list[dict[str, object]] = [] - state.send = lambda payload: sent.append(payload) # type: ignore[method-assign] - - def unexpected_build(client: ClientState, *, clean_jobs: bool) -> object: - raise AssertionError("job build should not run when template refresh fails") - - server.build_job_for_client = unexpected_build # type: ignore[method-assign] - - with tempfile.TemporaryDirectory() as tempdir: - server.audit_dir = Path(tempdir) - server.evidence_path = Path(tempdir) / "evidence.json" - server.ledger_writer_public_key_hex = "aa" * 32 - server.rpc = SubmitAcceptingTemplateRpc( - old_tip=old_tip, - block_hash=block_hash, - fail_template_after_submit=True, - ledger=ledger, - ) - server.build_audit_bundle = lambda **_kwargs: verified_block_bundle() # type: ignore[method-assign] - server.verify_bundle = lambda *_args, **_kwargs: verified_audit_report() # type: ignore[method-assign] - submission = SimpleNamespace( - header_hex="aa" * 80, - share_pass=True, - block_pass=True, - coinbase_tx_hex="c0ffee", - block_hash_hex=block_hash, - block_hex="00", - ) - - with patch( - "lab.prism.prism_coordinator.direct_stratum.assemble_submission", - return_value=submission, - ): - server.handle_request( - state, - { - "id": "submit-1", - "method": "mining.submit", - "params": ["miner-a", "job-1", "00" * 8, "00000001", "00000002"], - }, - ) - self.assertTrue(server.submit_next_block_candidate()) - - self.assertEqual(sent, [{"id": "submit-1", "result": True, "error": None}]) - self.assertEqual(server.accepted_block_count, 1) - self.assertEqual(len(ledger.persisted), 1) - self.assertEqual(len(ledger.confirmed), 1) - self.assertEqual(len(ledger.pending), 1) - self.assertEqual(server.tip_refresh_job_count, 0) - self.assertEqual(server.post_accept_refresh_failure_count, 0) - self.assertEqual(state.active_job_ids, {"job-1"}) - self.assertIn("job-1", server.jobs) - self.assertTrue(server.tip_refresh_is_pending()) - self.assertTrue(server._tip_refresh_retry.is_set()) - self.assertIn("qbit_prism_post_accept_refresh_failures_total 0", server.metrics_payload()) - - def test_post_accept_refresh_preserves_pending_vardiff_difficulty_pair(self) -> None: - old_tip = "00" * 32 - block_hash = "ae" * 32 - server, state, ledger = submit_coordinator(tip=old_tip) - server.vardiff_config = vardiff.VardiffConfig( - enabled=True, - target_share_interval_seconds=Decimal("15"), - min_difficulty=Decimal("1"), - max_difficulty=Decimal("1024"), - retarget_interval_seconds=Decimal("90"), - max_step_factor=Decimal("4"), - startup_difficulty=Decimal("1"), - max_step_down_factor=Decimal("4"), - ewma_alpha=Decimal("1"), - retarget_tolerance=Decimal("0"), - ) - server.stop_after_block = False - server.max_blocks = 10 - server.clients = {state} - state.share_difficulty = Decimal("1") - state.pending_share_difficulty = Decimal("8") - state.vardiff_window_started_monotonic = time.monotonic() - sent: list[dict[str, object]] = [] - state.send = lambda payload: sent.append(payload) # type: ignore[method-assign] - - def build_fresh_job(client: ClientState, *, clean_jobs: bool) -> object: - return prism_context( - "fresh-vardiff-job", - block_hash, - worker=state.worker, - difficulty=server.desired_client_share_difficulty(client), - clean_jobs=clean_jobs, - ) - - server.build_job_for_client = build_fresh_job # type: ignore[method-assign] - - with tempfile.TemporaryDirectory() as tempdir: - server.audit_dir = Path(tempdir) - server.evidence_path = Path(tempdir) / "evidence.json" - server.ledger_writer_public_key_hex = "aa" * 32 - server.rpc = SubmitAcceptingTemplateRpc(old_tip=old_tip, block_hash=block_hash, ledger=ledger) - server.build_audit_bundle = lambda **_kwargs: verified_block_bundle() # type: ignore[method-assign] - server.verify_bundle = lambda *_args, **_kwargs: verified_audit_report() # type: ignore[method-assign] - submission = SimpleNamespace( - header_hex="aa" * 80, - share_pass=True, - block_pass=True, - coinbase_tx_hex="c0ffee", - block_hash_hex=block_hash, - block_hex="00", - ) - - with patch( - "lab.prism.prism_coordinator.direct_stratum.assemble_submission", - return_value=submission, - ): - server.handle_request( - state, - { - "id": "submit-1", - "method": "mining.submit", - "params": ["miner-a", "job-1", "00" * 8, "00000001", "00000002"], - }, - ) - self.assertTrue(server.submit_next_block_candidate()) - self.assertEqual(server.poll_qbit_tip_template_once(), 1) - - self.assertEqual(sent[0], {"id": "submit-1", "result": True, "error": None}) - self.assertEqual(sent[1]["method"], "mining.set_difficulty") - self.assertEqual(sent[1]["params"], [8.0]) - self.assertEqual(sent[2]["method"], "mining.notify") - self.assertEqual(sent[2]["params"][0], "fresh-vardiff-job") - self.assertTrue(sent[2]["params"][8]) - self.assertEqual(state.share_difficulty, Decimal("8")) - self.assertIsNone(state.pending_share_difficulty) - self.assertEqual(state.vardiff_window_submitted, 1) - self.assertEqual(state.vardiff_window_accepted, 1) - - def test_rejected_candidate_never_creates_prepared_payout_state(self) -> None: - server, state, ledger = submit_coordinator() - with tempfile.TemporaryDirectory() as tempdir: - server.audit_dir = Path(tempdir) - server.evidence_path = Path(tempdir) / "evidence.json" - server.ledger_writer_public_key_hex = "aa" * 32 - block_hash = "dd" * 32 - server.rpc = SubmitRpc(tip="00" * 32, block_hash=block_hash, submit_result="rejected") - server.build_audit_bundle = lambda **_kwargs: { # type: ignore[method-assign] - "found_block": {"coinbase_value_sats": 50_00000000}, - "ledger_window_attestation": {"signature": {"public_key_hex": "aa" * 32}}, - "payout_policy_manifest": {"accounts": []}, - "signed_coinbase_manifest": { - "manifest": { - "coinbase_tx_hex": "c0ffee", - "payout_count": 1, - } - }, - } - server.verify_bundle = lambda *_args, **_kwargs: { # type: ignore[method-assign] - "coinbase_txid": "11" * 32, - "coinbase_manifest_sha256_hex": "22" * 32, - "audit_bundle_sha256_hex": "33" * 32, - "coinbase_tx_hex": "c0ffee", - } - submission = SimpleNamespace( - coinbase_tx_hex="c0ffee", - block_hash_hex=block_hash, - block_hex="00", - ) - - accepted = server.submit_block_candidate(block_candidate(server, state, submission)) - - self.assertFalse(accepted) - self.assertEqual(ledger.persisted, []) - self.assertEqual(ledger.rejected, []) - self.assertEqual(ledger.reversed, []) - self.assertEqual(len(ledger.pending), 0) - self.assertEqual( - server.block_candidate_abandoned_counts[PRISM_REJECTION_SUBMITBLOCK_REJECTED], 1 - ) - - -class PrismCoordinatorReliabilityTests(unittest.TestCase): - def _bare_coordinator(self) -> PrismCoordinator: - server = PrismCoordinator.__new__(PrismCoordinator) - server.lock = threading.RLock() - server.stop_event = threading.Event() - server._heartbeats = {} - server._watchdog_pauses = {} - server._heartbeats_lock = threading.Lock() - server.watchdog_timeout_seconds = 120.0 - server.watchdog_interval_seconds = 15.0 - return server - - def test_block_candidate_progress_stamps_only_on_submitter_thread(self) -> None: - server = self._bare_coordinator() - clock = {"now": 1000.0} - with patch( - "lab.prism.prism_coordinator.time.monotonic", - side_effect=lambda: clock["now"], - ): - server._record_heartbeat("block_submitter") - clock["now"] += server.watchdog_timeout_seconds + 1.0 - self.assertEqual( - server._overdue_heartbeats(clock["now"]), ["block_submitter"] - ) - # Without a registered owner the helper never stamps. - server._record_block_candidate_progress() - self.assertEqual( - server._overdue_heartbeats(clock["now"]), ["block_submitter"] - ) - # The owner thread's stamps clear the overdue state. - server._block_submitter_thread_ident = threading.get_ident() - server._record_block_candidate_progress() - self.assertEqual(server._overdue_heartbeats(clock["now"]), []) - - def test_client_thread_disposition_cannot_mask_frozen_submitter(self) -> None: - server = self._bare_coordinator() - clock = {"now": 1000.0} - owner_ready = threading.Event() - release_owner = threading.Event() - - def frozen_owner() -> None: - server._block_submitter_thread_ident = threading.get_ident() - server._record_heartbeat("block_submitter") - owner_ready.set() - release_owner.wait(5) - - with patch( - "lab.prism.prism_coordinator.time.monotonic", - side_effect=lambda: clock["now"], - ): - owner = threading.Thread(target=frozen_owner) - owner.start() - try: - self.assertTrue(owner_ready.wait(5)) - clock["now"] += server.watchdog_timeout_seconds + 1.0 - self.assertEqual( - server._overdue_heartbeats(clock["now"]), ["block_submitter"] - ) - # A client connection thread reaching a disposition boundary - # (synchronous below-target solve) must not refresh the - # frozen dedicated thread's liveness budget. - foreign = threading.Thread( - target=server._record_block_candidate_progress - ) - foreign.start() - foreign.join(5) - self.assertEqual( - server._overdue_heartbeats(clock["now"]), ["block_submitter"] - ) - finally: - release_owner.set() - owner.join(5) - - def test_wedged_evidence_read_stays_watchdog_eligible(self) -> None: - server = self._bare_coordinator() - clock = {"now": 1000.0} - entered = threading.Event() - release = threading.Event() - - def wedged_stats() -> tuple[int, int]: - entered.set() - if not release.wait(5): - raise AssertionError("evidence read was not released") - return (7, 3) - - server.accepted_share_stats = wedged_stats # type: ignore[method-assign] - with patch( - "lab.prism.prism_coordinator.time.monotonic", - side_effect=lambda: clock["now"], - ): - server._record_heartbeat("block_submitter") - reader = threading.Thread(target=server.accepted_share_stats) - reader.start() - try: - self.assertTrue(entered.wait(5)) - clock["now"] += server.watchdog_timeout_seconds + 1.0 - # A blocked evidence read is deliberately NOT excused from - # the liveness budget: exit-and-replay is the recovery path - # for a genuinely wedged ledger read, and no disposition - # code may pause the submitter's heartbeat to hide it. - self.assertEqual( - server._overdue_heartbeats(clock["now"]), ["block_submitter"] - ) - self.assertNotIn("block_submitter", server._watchdog_pauses) - finally: - release.set() - reader.join(5) - - def test_metrics_include_accepted_stats_reconcile_status(self) -> None: - server = self._bare_coordinator() - server.ledger = SimpleNamespace( - accepted_stats_reconcile_status=lambda: { - "age_seconds": 12.5, - "failures": 3, - } - ) - lines = server._accepted_stats_reconcile_metric_lines() - self.assertIn( - "qbit_prism_accepted_stats_reconcile_failures_total 3", lines - ) - self.assertIn( - "qbit_prism_accepted_stats_reconcile_age_seconds 12.500000", lines - ) - # Ledgers without the accessor (file-backed, duck-typed) emit no - # reconcile lines rather than failing the scrape. - server.ledger = SimpleNamespace() - self.assertEqual(server._accepted_stats_reconcile_metric_lines(), []) - - def test_positive_float_env_rejects_non_finite_values(self) -> None: - for raw in ("nan", "inf", "-inf"): - with self.subTest(raw=raw), patch.dict( - os.environ, {"PRISM_WATCHDOG_TIMEOUT_SECONDS": raw}, clear=True - ): - with self.assertRaisesRegex(SystemExit, "PRISM_WATCHDOG_TIMEOUT_SECONDS must be finite"): - env_positive_float("PRISM_WATCHDOG_TIMEOUT_SECONDS", 120.0) - - def test_audit_verifier_subprocess_has_explicit_timeout(self) -> None: - server = self._bare_coordinator() - server.bundle_build_timeout_seconds = 0.25 - with patch( - "lab.prism.prism_coordinator.subprocess.run", - side_effect=subprocess.TimeoutExpired("audit-verify", 0.25), - ) as run: - with self.assertRaisesRegex(RuntimeError, "timed out after 0.25s"): - server.verify_bundle( - Path("candidate.json"), - "00", - "11" * 32, - expected_coinbase_value_sats=1, - ) - - self.assertEqual(run.call_args.kwargs["timeout"], 0.25) - - def test_overdue_heartbeats_flags_only_stale_subsystems(self) -> None: - server = self._bare_coordinator() - server._record_heartbeat("stratum_accept") - server._record_heartbeat("qbit_blockpoll") - now = time.monotonic() - - self.assertEqual(server._overdue_heartbeats(now), []) - - with server._heartbeats_lock: - server._heartbeats["qbit_blockpoll"] = now - 1_000.0 - - self.assertEqual(server._overdue_heartbeats(now), ["qbit_blockpoll"]) - - def test_overdue_submitter_heartbeat_names_the_stuck_phase(self) -> None: - server = self._bare_coordinator() - clock = {"now": 1_000.0} - server._block_submitter_thread_ident = threading.get_ident() - with patch( - "lab.prism.prism_coordinator.time.monotonic", - side_effect=lambda: clock["now"], - ): - server._record_block_submitter_phase("replay-outbox-query") - clock["now"] += server.watchdog_timeout_seconds + 1.0 - self.assertEqual( - server._overdue_heartbeats(clock["now"]), - ["block_submitter:replay-outbox-query"], - ) - - def test_sixty_second_attempt_mark_stall_does_not_delay_rpc_or_heartbeat(self) -> None: - server, state, recording = submit_coordinator() - entered_mark = threading.Event() - release_mark = threading.Event() - submitted = threading.Event() - - class StallingLedger(RecordingLedger): - def __init__(self) -> None: - super().__init__() - self.mark_calls = 0 - - def mark_block_candidate_attempted(self, *, block_hash: str) -> bool: - self.mark_calls += 1 - entered_mark.set() - release_mark.wait(60.0) - return True - - class DeadlineRpc(SubmitRpc): - def __init__(self, ledger: RecordingLedger) -> None: - super().__init__( - tip="00" * 32, - block_hash="d2" * 32, - ledger=ledger, - ) - self.timeouts: list[float | None] = [] - - def call( - self, - method: str, - params: list[object] | None = None, - *, - timeout: float | None = None, - ) -> object: - if method == "submitblock": - self.timeouts.append(timeout) - submitted.set() - if len(self.timeouts) > 1: - return "duplicate" - return super().call(method, params) - - ledger = StallingLedger() - server.ledger = ledger - rpc = DeadlineRpc(ledger) - server.rpc = rpc - server.block_submit_rpc_timeout_seconds = 0.4 - server.block_submit_db_timeout_seconds = 0.05 - server.block_candidate_retry_initial_seconds = 0.01 - server.block_candidate_retry_max_seconds = 0.01 - server.watchdog_timeout_seconds = 0.2 - server._heartbeats = {} - server._heartbeat_phases = {} - server._watchdog_pauses = {} - server._heartbeats_lock = threading.Lock() - candidate = block_candidate( - server, - state, - SimpleNamespace( - coinbase_tx_hex="00", - block_hash_hex="d2" * 32, - block_hex="00", - share_pass=True, - block_pass=True, - ), - ) - server.enqueue_block_candidate(candidate) - started = time.monotonic() - submitter = threading.Thread(target=server.block_submit_loop) - with patch("builtins.print"): - submitter.start() - try: - self.assertTrue(submitted.wait(2.0)) - self.assertLess(time.monotonic() - started, 2.0) - self.assertTrue(entered_mark.wait(1.0)) - with server._heartbeats_lock: - first_heartbeat = server._heartbeats["block_submitter"] - heartbeat_deadline = time.monotonic() + 1.0 - while time.monotonic() < heartbeat_deadline: - with server._heartbeats_lock: - latest_heartbeat = server._heartbeats["block_submitter"] - if latest_heartbeat > first_heartbeat: - break - time.sleep(0.01) - self.assertGreater(latest_heartbeat, first_heartbeat) - self.assertEqual( - server._overdue_heartbeats(time.monotonic()), - [], - ) - self.assertEqual(ledger.mark_calls, 1) - self.assertEqual(rpc.timeouts[0], 0.4) - finally: - server.stop_event.set() - submitter.join(2.0) - release_mark.set() - self.assertFalse(submitter.is_alive()) - - def test_accounting_retry_backoff_defers_instead_of_sleeping_under_admission(self) -> None: - server, state, _recording = submit_coordinator() - server.block_candidate_retry_initial_seconds = 30.0 - server.block_candidate_retry_max_seconds = 30.0 - - class FailingMarkLedger(RecordingLedger): - def mark_block_candidate_attempted(self, *, block_hash: str) -> bool: - raise RuntimeError("attempt marker unavailable") - - server.ledger = FailingMarkLedger() - candidate = block_candidate( - server, - state, - SimpleNamespace( - coinbase_tx_hex="00", - block_hash_hex="ab" * 32, - block_hex="ab", - share_pass=True, - block_pass=True, - ), - ) - block_hash = "ab" * 32 - server._block_accounting_thread_ident = threading.get_ident() - server._block_accounting_holds_disposition = True - started = time.monotonic() - with patch("builtins.print"): - handled = server._submit_next_block_candidate_writer( - candidate, - node_submission=SimpleNamespace( - attempted=False, - result=None, - error=None, - ), - disposition_held=True, - ) - elapsed = time.monotonic() - started - self.assertTrue(handled) - # The 30s backoff must be recorded as a deadline, never slept while - # the accounting thread holds admission and the disposition lease. - self.assertLess(elapsed, 10.0) - self.assertIs( - getattr(server, "_block_accounting_deferred_retry_candidate", None), - candidate, - ) - with server.lock: - deadline = server._block_candidate_retry_not_before.get(block_hash) - self.assertIsNotNone(deadline) - self.assertGreater(deadline, time.monotonic()) - - # After the lease releases, the deferred candidate parks in the retry - # slot and the dequeue honors the backoff deadline. - server._block_accounting_holds_disposition = False - with server.lock: - server._block_accounting_deferred_retry_candidate = None - server._merge_block_candidate_retry_locked( - "_retry_block_candidate", - candidate, - ) - with patch("builtins.print"): - self.assertFalse( - server.submit_next_block_candidate(defer_accounting=True) - ) - self.assertIs(getattr(server, "_retry_block_candidate", None), candidate) - - # Once the deadline passes the same candidate dequeues again. - with server.lock: - server._block_candidate_retry_not_before[block_hash] = ( - time.monotonic() - 1.0 - ) - with patch("builtins.print"): - self.assertTrue( - server.submit_next_block_candidate(defer_accounting=True) - ) - - def test_live_retry_reuses_definitive_node_acceptance_without_reoffer(self) -> None: - parent_hash = "00" * 32 - block_hash = "ce" * 32 - moved_tip = "11" * 32 - ledger = SingleWriterShareLedger() - server, state, _recording = submit_coordinator(tip=parent_hash) - server.ledger = ledger - server.stop_after_block = False - server.max_blocks = 10 - server.block_candidate_retry_initial_seconds = 0.0 - server.block_candidate_retry_max_seconds = 0.0 - from lab.prism.share_ledger import PendingShare - - pending = PendingShare( - share_id="miner-a:live-retry-duplicate", - miner_id="miner-a", - order_key="miner-a", - p2mr_program_hex="11" * 32, - share_difficulty=1, - network_difficulty=1, - template_height=10, - job_id="job-1", - job_issued_at_ms=1, - accepted_at_ms=2, - ntime=1_700_000_000, - ) - candidate = block_candidate( - server, - state, - SimpleNamespace( - coinbase_tx_hex="c0ffee", - block_hash_hex=block_hash, - block_hex="00", - share_pass=True, - block_pass=True, - ), - pending_share=pending, - ) - intent = server.block_candidate_intent(candidate) - ledger.append_batch([(pending, intent)]) - - expected_height = int(candidate.context.template["height"]) - header_state = {"confirmations": -1} - - class MovedTipRpc(FakeRpc): - def __init__(self) -> None: - self.submit_results: list[object] = [] - - def call( - self, - method: str, - params: list[object] | None = None, - *, - timeout: float | None = None, - ) -> object: - if method == "submitblock": - result = None if not self.submit_results else "duplicate" - self.submit_results.append(result) - return result - if method == "getbestblockhash": - # The chain advances as soon as the first offer lands, so - # the in-process retry observes a moved tip. - return parent_hash if not self.submit_results else moved_tip - if method == "getblockhash": - return block_hash - if method == "getblockcount": - return 9 - if method == "getblockheader": - # Unprovable during the tip race (found, not yet active), - # then settled and active for the final pass. - return { - "height": expected_height, - "confirmations": header_state["confirmations"], - } - return super().call(method, params) - - rpc = MovedTipRpc() - server.rpc = rpc - original_mark = ledger.mark_block_candidate_attempted - mark_state = {"failures": 0} - - def flaky_mark(*, block_hash: str) -> bool: - # Fail twice, starting at the first attempt marker after the - # fresh node offer, mirroring statement timeouts under sustained - # saturation. The retained acceptance must survive the failed - # retry too, not just the first failure. - if len(rpc.submit_results) == 1 and mark_state["failures"] < 2: - mark_state["failures"] += 1 - raise RuntimeError("attempt marker timed out") - return original_mark(block_hash=block_hash) - - ledger.mark_block_candidate_attempted = flaky_mark # type: ignore[method-assign] - server.enqueue_block_candidate(candidate) - with tempfile.TemporaryDirectory() as tempdir: - server.audit_dir = Path(tempdir) - server.evidence_path = Path(tempdir) / "evidence.json" - server.ledger_writer_public_key_hex = "aa" * 32 - server.build_audit_bundle = ( # type: ignore[method-assign] - lambda **_kwargs: verified_block_bundle() - ) - server.verify_bundle = ( # type: ignore[method-assign] - lambda *_args, **_kwargs: verified_audit_report() - ) - with patch("builtins.print"): - # First pass: fresh accept from the node, then the attempt - # marker fails and the candidate is retained for an - # in-process retry carrying the definitive node result. - self.assertTrue(server.submit_next_block_candidate()) - self.assertEqual(rpc.submit_results, [None]) - self.assertEqual( - ledger._block_candidate_outbox[block_hash]["state"], - "pending", - ) - # Retry 1 fails the marker again: the retained acceptance - # must survive a consumed-and-failed retry, not just the - # first failure. - self.assertTrue(server.submit_next_block_candidate()) - self.assertEqual(rpc.submit_results, [None]) - self.assertEqual( - ledger._block_candidate_outbox[block_hash]["state"], - "pending", - ) - # Retry 2: the stashed acceptance is still reused, so the - # node is never asked again (no "duplicate" classification, - # no chain probe reliance) and the landing tail finalizes - # exactly as if the first pass had continued. - self.assertTrue(server.submit_next_block_candidate()) - self.assertEqual(rpc.submit_results, [None]) - self.assertEqual( - ledger._block_candidate_outbox[block_hash]["state"], - "submitted", - ) - self.assertNotIn( - PRISM_REJECTION_STALE_JOB, - getattr(server, "block_candidate_abandoned_counts", {}), - ) - self.assertEqual(server.accepted_block_count, 1) - - def test_block_accounting_primary_queue_is_bounded_by_default(self) -> None: - server, _state, _recording = submit_coordinator() - server._ensure_block_accounting_state() - # A bounded primary is what makes the documented result-preserving - # spillover ordering reachable; the overflow queue stays unbounded. - self.assertEqual(server._block_accounting_queue.maxsize, 8) - self.assertEqual(server._block_accounting_overflow_queue.maxsize, 0) - - def test_accounting_saturation_does_not_convoy_node_offers(self) -> None: - server, state, _recording = submit_coordinator() - server.max_blocks = 10 - server.stop_after_block = False - server.block_candidate_retry_initial_seconds = 0.01 - server._block_accounting_queue = queue.PriorityQueue(maxsize=1) - entered_accounting = threading.Event() - release_accounting = threading.Event() - submitted: list[str] = [] - - class RecordingRpc(TipRpc): - def call( - self, - method: str, - params: list[object] | None = None, - *, - timeout: float | None = None, - ) -> object: - if method == "submitblock": - submitted.append(str((params or [""])[0])) - return None - return super().call(method, params) - - server.rpc = RecordingRpc("00" * 32) - - def blocked_accounting( - _candidate: PrismBlockCandidate, - **_kwargs: object, - ) -> bool: - entered_accounting.set() - release_accounting.wait(5) - return True - - server._call_block_candidate_writer = blocked_accounting # type: ignore[method-assign] - for tag in ("a1", "b2", "c3", "d4"): - candidate = block_candidate( - server, - state, - SimpleNamespace( - coinbase_tx_hex="00", - block_hash_hex=tag * 32, - block_hex=tag, - share_pass=True, - block_pass=True, - ), - ) - server.enqueue_block_candidate(candidate) - - submitter = threading.Thread(target=server.block_submit_loop) - with patch("builtins.print"): - submitter.start() - try: - self.assertTrue(entered_accounting.wait(1)) - deadline = time.monotonic() + 2 - while len(submitted) < 4 and time.monotonic() < deadline: - time.sleep(0.01) - self.assertEqual(submitted, ["a1", "b2", "c3", "d4"]) - finally: - server.stop_event.set() - release_accounting.set() - submitter.join(2) - accounting = getattr(server, "_block_accounting_thread", None) - if accounting is not None: - accounting.join(2) - self.assertFalse(submitter.is_alive()) - - def test_replay_batch_reaches_node_while_oldest_accounting_stalls(self) -> None: - server, state, _recording = submit_coordinator() - ledger = SingleWriterShareLedger() - server.ledger = ledger - server.max_blocks = 10 - server.stop_after_block = False - submitted: list[str] = [] - release_accounting = threading.Event() - entered_accounting = threading.Event() - - for index, tag in enumerate(("a5", "b6"), start=1): - candidate = block_candidate( - server, - state, - SimpleNamespace( - coinbase_tx_hex="00", - block_hash_hex=tag * 32, - block_hex=tag, - share_pass=True, - block_pass=True, - ), - ) - pending = PendingShare( - share_id=f"miner-a:{tag * 32}", - miner_id="miner-a", - order_key="miner-a", - p2mr_program_hex="11" * 32, - share_difficulty=1, - network_difficulty=1, - template_height=9, - job_id="job-1", - job_issued_at_ms=1, - accepted_at_ms=index, - ntime=1, - ) - candidate = dataclass_replace(candidate, pending_share=pending) - ledger.append_batch( - [(pending, server.block_candidate_intent(candidate))] - ) - - self.assertEqual(server.replay_pending_block_candidates(), 2) - - class RecordingRpc(TipRpc): - def call( - self, - method: str, - params: list[object] | None = None, - *, - timeout: float | None = None, - ) -> object: - if method == "submitblock": - submitted.append(str((params or [""])[0])) - return None - return super().call(method, params) - - server.rpc = RecordingRpc("00" * 32) - - def blocked_accounting( - _candidate: PrismBlockCandidate, - **_kwargs: object, - ) -> bool: - entered_accounting.set() - release_accounting.wait(5) - return True - - server._call_block_candidate_writer = blocked_accounting # type: ignore[method-assign] - submitter = threading.Thread(target=server.block_submit_loop) - with patch("builtins.print"): - submitter.start() - try: - self.assertTrue(entered_accounting.wait(1)) - deadline = time.monotonic() + 2 - while len(submitted) < 2 and time.monotonic() < deadline: - time.sleep(0.01) - self.assertEqual(submitted, ["a5", "b6"]) - finally: - server.stop_event.set() - release_accounting.set() - submitter.join(2) - accounting = getattr(server, "_block_accounting_thread", None) - if accounting is not None: - accounting.join(2) - - def test_startup_block_replay_timeout_starts_submitter_and_converges(self) -> None: - server, state, _recording = submit_coordinator() - server.max_blocks = 10 - server.stop_after_block = False - server.block_submit_db_timeout_seconds = 0.01 - # The startup enumeration is landing-class (issue #188 fix 4). - server.block_landing_db_timeout_seconds = 0.01 - server.block_landing_db_timeout_max_seconds = 0.01 - server.block_candidate_retry_initial_seconds = 0.01 - ledger = SingleWriterShareLedger() - server.ledger = ledger - block_hash = "e5" * 32 - pending = PendingShare( - share_id=f"miner-a:{block_hash}", - miner_id="miner-a", - order_key="miner-a", - p2mr_program_hex="11" * 32, - share_difficulty=1, - network_difficulty=1, - template_height=9, - job_id="job-1", - job_issued_at_ms=1, - accepted_at_ms=1, - ntime=1, - ) - candidate = block_candidate( - server, - state, - SimpleNamespace( - coinbase_tx_hex="00", - block_hash_hex=block_hash, - block_hex="e5", - share_pass=True, - block_pass=True, - ), - pending_share=pending, - ) - ledger.append_batch( - [(pending, server.block_candidate_intent(candidate))] - ) - - query_started = threading.Event() - release_query = threading.Event() - query_calls = 0 - startup_call: list[object] = [] - original_pending_rows = ledger.pending_block_candidate_rows - - def slow_pending_rows(*, limit: int = 32) -> list[dict[str, object]]: - nonlocal query_calls - query_calls += 1 - if query_calls == 1: - with server._block_submitter_ledger_calls_lock: - startup_call.append( - server._block_submitter_ledger_calls[ - ("replay-outbox-query", 32) - ] - ) - query_started.set() - if not release_query.wait(5): - raise AssertionError("timed out waiting to release outbox query") - return original_pending_rows(limit=limit) - - ledger.pending_block_candidate_rows = slow_pending_rows # type: ignore[method-assign] - original_record_wait = server._record_block_submitter_wait - loop_reuse_waiting = threading.Event() - - def observed_record_wait(phase: str) -> None: - original_record_wait(phase) - if ( - phase == "replay-outbox-query" - and getattr(server, "_block_submitter_thread_ident", None) - == threading.get_ident() - ): - loop_reuse_waiting.set() - - server._record_block_submitter_wait = observed_record_wait # type: ignore[method-assign] - - submitted: list[str] = [] - - class RecordingRpc(TipRpc): - def call( - self, - method: str, - params: list[object] | None = None, - *, - timeout: float | None = None, - ) -> object: - if method == "getblockcount": - return 9 - if method == "submitblock": - submitted.append(str((params or [""])[0])) - return None - return super().call(method, params) - - server.rpc = RecordingRpc("00" * 32) - accounted: list[str] = [] - - def account_candidate( - replayed: PrismBlockCandidate, - **_kwargs: object, - ) -> bool: - replayed_hash = replayed.submission.block_hash_hex - accounted.append(replayed_hash) - ledger.mark_block_candidate_submitted(block_hash=replayed_hash) - server.stop_event.set() - return True - - server._call_block_candidate_writer = account_candidate # type: ignore[method-assign] - - profile = SimpleNamespace(heartbeat_name="stratum_accept_default") - listener_closed = threading.Event() - listener = SimpleNamespace(close=listener_closed.set) - server.listener_profiles = [profile] - server.bind = "127.0.0.1" - server.port = 0 - server.min_ready_miners = 1 - server.audit_bind = None - server.audit_port = 0 - server.hot_path_log_enabled = False - server.blockwait_enabled = False - server.vardiff_idle_sweep_seconds = 0.0 - server.stratum_initial_job_timeout_seconds = 0.0 - server.watchdog_enabled = False - server.watchdog_interval_seconds = 0.1 - server.template_refresh_failure_exit_seconds = 120.0 - server.coordination_blocked_exit_seconds = 120.0 - server.open_stratum_listeners = ( # type: ignore[method-assign] - lambda _stack: [(listener, profile)] - ) - server.validate_live_chain_identity = lambda: None # type: ignore[method-assign] - server.validate_live_template_and_fee_policy = lambda: None # type: ignore[method-assign] - server.prism_payout_policy = lambda: {} # type: ignore[method-assign] - server.prewarm_startup_jobs = lambda: None # type: ignore[method-assign] - server.watchdog_loop = lambda: None # type: ignore[method-assign] - server.blockpoll_loop = lambda: None # type: ignore[method-assign] - server.replay_recovered_shares = lambda: 0 # type: ignore[method-assign] - server.share_append_loop = lambda: None # type: ignore[method-assign] - server.shutdown = lambda *, reason="graceful": True # type: ignore[method-assign] - - def drain_threads(threads: list[tuple[threading.Thread, float]]) -> None: - for thread, timeout in threads: - thread.join(timeout) - - server.drain_non_writer_components = drain_threads # type: ignore[method-assign] - - def accept_after_startup(_listener: object, _profile: object) -> None: - self.assertTrue(loop_reuse_waiting.wait(2)) - self.assertIsNotNone( - getattr(server, "_block_submitter_thread_ident", None) - ) - self.assertTrue(query_started.wait(1)) - # The loop is waiting on the exact startup call while that worker - # remains blocked; no replacement query was spawned. - with server._block_submitter_ledger_calls_lock: - self.assertIs( - server._block_submitter_ledger_calls.get( - ("replay-outbox-query", 32) - ), - startup_call[0], - ) - self.assertEqual(query_calls, 1) - release_query.set() - deadline = time.monotonic() + 2 - while not accounted and time.monotonic() < deadline: - time.sleep(0.01) - self.assertTrue(accounted) - - server.accept_loop = accept_after_startup # type: ignore[method-assign] - - try: - with patch("builtins.print") as printed: - server._serve_with_listener_stack(SimpleNamespace()) # type: ignore[arg-type] - finally: - server.stop_event.set() - release_query.set() - - ledger.pending_block_candidate_rows = original_pending_rows # type: ignore[method-assign] - startup_logs = [ - " ".join(str(value) for value in call.args) - for call in printed.call_args_list - ] - self.assertTrue( - any( - "prism coordinator: startup block candidate replay timed out " - "phase=replay-outbox-query timeout=0.01s" in message - for message in startup_logs - ) - ) - self.assertTrue(query_started.is_set()) - self.assertTrue(listener_closed.is_set()) - self.assertEqual(submitted, ["e5"]) - self.assertEqual(accounted, [block_hash]) - self.assertEqual(ledger.pending_block_candidates(), []) - - def test_startup_block_replay_catches_psql_server_timeout(self) -> None: - server, _state, _recording = submit_coordinator() - ledger = PsqlShareLedger.__new__(PsqlShareLedger) - ledger._command = ["psql"] - ledger._native = None - ledger._operation_timeout_local = threading.local() - - def timed_out_rows(*, limit: int = 32) -> list[dict[str, object]]: - ledger._run_sql(f"SELECT {limit};") - return [] - - ledger.pending_block_candidate_rows = timed_out_rows # type: ignore[method-assign] - server.ledger = ledger - completed = subprocess.CompletedProcess( - args=["psql"], - returncode=3, - stdout="", - stderr=( - "ERROR: 57014: canceling statement due to statement timeout" - ), - ) - with patch( - "lab.prism.share_ledger.subprocess.run", - return_value=completed, - ), patch("builtins.print") as printed: - self.assertTrue(server._run_startup_block_candidate_replay()) - - startup_logs = [ - " ".join(str(value) for value in call.args) - for call in printed.call_args_list - ] - self.assertTrue( - any( - "startup block candidate replay timed out " - "phase=replay-outbox-query" in message - for message in startup_logs - ) - ) - - def test_startup_block_replay_keeps_hard_database_errors_fatal(self) -> None: - server, _state, _recording = submit_coordinator() - - def failed_rows(*, limit: int = 32) -> list[dict[str, object]]: - raise RuntimeError(f"postgres failed limit={limit}") - - server.ledger = SimpleNamespace( - pending_block_candidate_rows=failed_rows, - ) - with self.assertRaisesRegex(RuntimeError, "postgres failed"): - server._run_startup_block_candidate_replay() - - def test_startup_block_replay_preserves_shutdown_stop(self) -> None: - server = self._bare_coordinator() - - def shutdown_replay() -> int: - raise ShutdownInProgress("shutdown won") - - server.replay_pending_block_candidates = shutdown_replay # type: ignore[method-assign] - - self.assertFalse(server._run_startup_block_candidate_replay()) - - def test_stuck_rpc_worker_pool_requests_nonzero_restart(self) -> None: - server, _state, _recording = submit_coordinator() - server.block_submit_stuck_call_exit_seconds = 0.04 - release = threading.Event() - entered: list[str] = [] - - class IgnoringRpc: - def call( - self, - method: str, - params: list[object] | None = None, - *, - timeout: float | None = None, - ) -> object: - entered.append(str((params or [""])[0])) - release.wait(5) - return None - - server.rpc = IgnoringRpc() - try: - for tag in ("11", "22"): - with self.assertRaises(TimeoutError): - server._run_submitblock_rpc_with_hard_deadline( - block_hash=tag * 32, - block_hex=tag, - timeout_seconds=0.01, - ) - time.sleep(0.05) - with self.assertRaises(TimeoutError): - server._run_submitblock_rpc_with_hard_deadline( - block_hash="33" * 32, - block_hex="33", - timeout_seconds=0.01, - ) - self.assertEqual(entered, ["11", "22"]) - self.assertTrue(server.stop_event.is_set()) - self.assertTrue(server._fatal_exit_requested) - finally: - release.set() - - def test_block_landing_db_timeout_escalates_and_clears(self) -> None: - server, _state, _recording = submit_coordinator() - server.block_landing_db_timeout_seconds = 30.0 - server.block_landing_db_timeout_max_seconds = 120.0 - block_hash = "ab" * 32 - self.assertEqual(server._block_landing_db_timeout(block_hash), 30.0) - server._note_block_landing_timeout(block_hash) - self.assertEqual(server._block_landing_db_timeout(block_hash), 60.0) - server._note_block_landing_timeout(block_hash) - self.assertEqual(server._block_landing_db_timeout(block_hash), 120.0) - server._note_block_landing_timeout(block_hash) - self.assertEqual(server._block_landing_db_timeout(block_hash), 120.0) - self.assertEqual(server._block_landing_db_timeout("cd" * 32), 30.0) - server._clear_block_candidate_retry_state(block_hash) - self.assertEqual(server._block_landing_db_timeout(block_hash), 30.0) - - def test_landing_scope_uses_landing_budget_from_first_attempt(self) -> None: - server, _state, _recording = submit_coordinator() - server.block_submit_db_timeout_seconds = 0.01 - server.block_landing_db_timeout_seconds = 45.0 - server.block_landing_db_timeout_max_seconds = 120.0 - scopes: list[float] = [] - - @contextlib.contextmanager - def statement_timeout(seconds: float): - scopes.append(seconds) - yield - - server.ledger = SimpleNamespace(statement_timeout=statement_timeout) - with server._block_landing_ledger_statement_timeout_scope("ab" * 32): - pass - self.assertEqual(scopes, [45.0]) - metrics = server.block_ledger_call_class_metrics() - self.assertEqual(metrics["landing"]["calls_total"], 1) - self.assertEqual(metrics["landing"]["timeouts_total"], 0) - self.assertEqual(metrics["landing"]["last_budget_seconds"], 45.0) - - def test_landing_scope_timeout_escalates_next_attempt_budget(self) -> None: - server, _state, _recording = submit_coordinator() - server.block_landing_db_timeout_seconds = 45.0 - server.block_landing_db_timeout_max_seconds = 120.0 - scopes: list[float] = [] - - @contextlib.contextmanager - def statement_timeout(seconds: float): - scopes.append(seconds) - yield - - server.ledger = SimpleNamespace(statement_timeout=statement_timeout) - block_hash = "ab" * 32 - with self.assertRaises(TimeoutError): - with server._block_landing_ledger_statement_timeout_scope(block_hash): - raise LedgerOperationTimeout("postgres operation exceeded 45s") - with server._block_landing_ledger_statement_timeout_scope(block_hash): - pass - self.assertEqual(scopes, [45.0, 90.0]) - metrics = server.block_ledger_call_class_metrics() - self.assertEqual(metrics["landing"]["calls_total"], 2) - self.assertEqual(metrics["landing"]["timeouts_total"], 1) - self.assertEqual(metrics["landing"]["last_budget_seconds"], 90.0) - - def test_landing_scope_ignores_non_ledger_timeouts(self) -> None: - """The scope guards a body that also runs node RPCs; an RPC timeout - is not a database cancellation and must not escalate the next - PostgreSQL budget or fire the landing-timeout alert.""" - server, _state, _recording = submit_coordinator() - server.block_landing_db_timeout_seconds = 45.0 - server.block_landing_db_timeout_max_seconds = 120.0 - scopes: list[float] = [] - - @contextlib.contextmanager - def statement_timeout(seconds: float): - scopes.append(seconds) - yield - - server.ledger = SimpleNamespace(statement_timeout=statement_timeout) - block_hash = "ab" * 32 - with self.assertRaises(TimeoutError): - with server._block_landing_ledger_statement_timeout_scope(block_hash): - raise TimeoutError("getblockhash exceeded 1s") - with server._block_landing_ledger_statement_timeout_scope(block_hash): - pass - self.assertEqual(scopes, [45.0, 45.0]) - metrics = server.block_ledger_call_class_metrics() - self.assertEqual(metrics["landing"]["calls_total"], 2) - self.assertEqual(metrics["landing"]["timeouts_total"], 0) - - def test_fast_class_ledger_call_metrics_record_timeouts(self) -> None: - server, _state, _recording = submit_coordinator() - server.block_submit_db_timeout_seconds = 0.01 - release = threading.Event() - - def blocking_operation() -> None: - release.wait(5) - - try: - with self.assertRaises(TimeoutError): - server._run_block_submitter_ledger_call( - ("metrics-key",), - "metrics-test", - blocking_operation, - ) - finally: - release.set() - metrics = server.block_ledger_call_class_metrics() - self.assertEqual(metrics["fast"]["timeouts_total"], 1) - self.assertEqual(metrics["fast"]["last_budget_seconds"], 0.01) - - def test_stuck_ledger_worker_pool_requests_nonzero_restart(self) -> None: - server, _state, _recording = submit_coordinator() - server.block_submit_db_timeout_seconds = 0.01 - server.block_submit_stuck_call_exit_seconds = 0.04 - release = threading.Event() - entered: list[str] = [] - - def blocking_operation(tag: str) -> str: - entered.append(tag) - release.wait(5) - return tag - - try: - for tag in ("one", "two"): - with self.assertRaises(TimeoutError): - server._run_block_submitter_ledger_call( - (tag,), - f"test-{tag}", - lambda tag=tag: blocking_operation(tag), - ) - time.sleep(0.05) - with self.assertRaises(TimeoutError): - server._run_block_submitter_ledger_call( - ("three",), - "test-three", - lambda: blocking_operation("three"), - ) - self.assertEqual(entered, ["one", "two"]) - self.assertTrue(server.stop_event.is_set()) - self.assertTrue(server._fatal_exit_requested) - finally: - release.set() - - def test_one_stuck_ledger_call_does_not_restart_with_spare_capacity(self) -> None: - server, _state, _recording = submit_coordinator() - server.block_submit_db_timeout_seconds = 0.01 - server.block_submit_stuck_call_exit_seconds = 0.04 - release = threading.Event() - entered = threading.Event() - - def blocking_operation() -> None: - entered.set() - release.wait(5) - - try: - with self.assertRaises(TimeoutError): - server._run_block_submitter_ledger_call( - ("same-key",), - "same-key", - blocking_operation, - ) - self.assertTrue(entered.wait(1)) - time.sleep(0.05) - with self.assertRaises(TimeoutError): - server._run_block_submitter_ledger_call( - ("same-key",), - "same-key", - blocking_operation, - ) - self.assertFalse(server.stop_event.is_set()) - self.assertFalse(getattr(server, "_fatal_exit_requested", False)) - finally: - release.set() - - def test_two_stuck_ledger_calls_restart_on_existing_key_retry(self) -> None: - server, _state, _recording = submit_coordinator() - server.block_submit_db_timeout_seconds = 0.01 - server.block_submit_stuck_call_exit_seconds = 0.04 - release = threading.Event() - entered: list[str] = [] - - def blocking_operation(tag: str) -> None: - entered.append(tag) - release.wait(5) - - try: - for tag in ("one", "two"): - with self.assertRaises(TimeoutError): - server._run_block_submitter_ledger_call( - (tag,), - tag, - lambda tag=tag: blocking_operation(tag), - ) - time.sleep(0.05) - # No third key is required: a retry reusing either poisoned call - # still observes that every bounded worker slot is exhausted. - with self.assertRaises(TimeoutError): - server._run_block_submitter_ledger_call( - ("one",), - "one", - lambda: blocking_operation("one"), - ) - self.assertEqual(entered, ["one", "two"]) - self.assertTrue(server.stop_event.is_set()) - self.assertTrue(server._fatal_exit_requested) - finally: - release.set() - - def test_watchdog_starts_before_synchronous_startup_work(self) -> None: - source = inspect.getsource(PrismCoordinator._serve_with_listener_stack) - watchdog_start = source.index("target=self.watchdog_loop") - - self.assertEqual(source.count("target=self.watchdog_loop"), 1) - self.assertLess(watchdog_start, source.index("self.prewarm_startup_jobs")) - self.assertLess(watchdog_start, source.index("self.replay_recovered_shares")) - - def test_watchdog_hard_exits_when_fatal_stop_wins_during_startup(self) -> None: - server = self._bare_coordinator() - server._fatal_exit_requested = True - server.stop_event.set() - - with patch("lab.prism.prism_coordinator.os._exit") as hard_exit: - server.watchdog_loop() - - hard_exit.assert_called_once_with(1) - - def test_parent_retry_displacement_preserves_durable_descendant(self) -> None: - server, state, _recording = submit_coordinator() - - def candidate_at(tag: str, height: int) -> PrismBlockCandidate: - candidate = block_candidate( - server, - state, - SimpleNamespace( - coinbase_tx_hex="00", - block_hash_hex=tag * 32, - block_hex=tag, - share_pass=True, - block_pass=True, - ), - ) - context = SimpleNamespace(**vars(candidate.context)) - context.template = {**candidate.context.template, "height": height} - return dataclass_replace( - candidate, - context=context, - durable_replay=True, - ) - - descendant = candidate_at("b8", 11) - parent = candidate_at("a8", 10) - descendant_hash = descendant.submission.block_hash_hex - server._ensure_block_candidate_disposition_state() - server._ensure_block_replay_state() - server._block_replay_inflight_hashes.add(descendant_hash) - with server.lock: - server._retry_block_candidate = descendant - server._merge_block_candidate_retry_locked( - "_retry_block_candidate", - parent, - ) - - self.assertIs(server._retry_block_candidate, parent) - self.assertIs( - server._block_disposition_waiting_retries[descendant_hash], - descendant, - ) - - # Once the parent reaches a terminal state, the descendant's preserved - # wakeup is selected even though durable replay still deduplicates it. - with server.lock: - server._retry_block_candidate = None - captured: list[object] = [] - server.max_blocks = 10 - server.stop_after_block = False - server._node_submission_for_candidate = ( # type: ignore[method-assign] - lambda _candidate: SimpleNamespace( - attempted=True, - result="duplicate", - error=None, - ) - ) - server._enqueue_block_accounting_task = ( # type: ignore[method-assign] - lambda task: (captured.append(task), True)[1] - ) - - self.assertTrue(server.submit_next_block_candidate(defer_accounting=True)) - self.assertEqual(len(captured), 1) - task = captured[0] - self.assertIs(task.candidate, descendant) - server._release_block_candidate_disposition(task.disposition_lease) - - def test_synchronous_waiter_joins_finalize_only_registry(self) -> None: - server, state, _recording = submit_coordinator() - candidate = block_candidate( - server, - state, - SimpleNamespace( - coinbase_tx_hex="00", - block_hash_hex="c8" * 32, - block_hex="c8", - share_pass=False, - block_pass=True, - ), - credit_share_on_accept=True, - ) - block_hash = candidate.submission.block_hash_hex - server._block_candidate_finalize_retries = {} - server._block_candidate_finalize_retries[block_hash] = (True, "") - server._node_submission_for_candidate = ( # type: ignore[method-assign] - lambda _candidate: (_ for _ in ()).throw( - AssertionError("finalize-only retry must not call qbitd") - ) - ) - finalized: list[dict[str, object]] = [] - - def finalize( - _candidate: PrismBlockCandidate, - **kwargs: object, - ) -> bool: - finalized.append(kwargs) - return True - - server._finalize_block_candidate = finalize # type: ignore[method-assign] - - self.assertTrue(server._submit_synchronous_block_candidate(candidate)) - self.assertEqual(len(finalized), 1) - self.assertTrue(finalized[0]["accepted"]) - self.assertEqual(finalized[0]["block_hash"], block_hash) - - def test_synchronous_waiter_preserves_finalize_only_abandonment(self) -> None: - server, state, _recording = submit_coordinator() - candidate = block_candidate( - server, - state, - SimpleNamespace( - coinbase_tx_hex="00", - block_hash_hex="c9" * 32, - block_hex="c9", - share_pass=False, - block_pass=True, - ), - credit_share_on_accept=True, - ) - block_hash = candidate.submission.block_hash_hex - server._block_candidate_finalize_retries = { - block_hash: (False, "terminal rejection") - } - server._node_submission_for_candidate = ( # type: ignore[method-assign] - lambda _candidate: (_ for _ in ()).throw( - AssertionError("finalize-only retry must not call qbitd") - ) - ) - finalized: list[dict[str, object]] = [] - server._finalize_block_candidate = ( # type: ignore[method-assign] - lambda _candidate, **kwargs: (finalized.append(kwargs), True)[1] - ) - - self.assertFalse(server._submit_synchronous_block_candidate(candidate)) - self.assertEqual(len(finalized), 1) - self.assertFalse(finalized[0]["accepted"]) - self.assertEqual(finalized[0]["error"], "terminal rejection") - - def test_synchronous_abandon_rejects_prepared_state_before_outbox(self) -> None: - server, state, ledger = submit_coordinator() - block_hash = "ca" * 32 - candidate = block_candidate( - server, - state, - SimpleNamespace( - coinbase_tx_hex="00", - block_hash_hex=block_hash, - block_hex="ca", - share_pass=False, - block_pass=True, - ), - credit_share_on_accept=True, - ) - events: list[str] = [] - prepared = {"value": True} - - def pool_block_state(*, block_hash: str) -> dict[str, str]: - self.assertEqual(block_hash, candidate.submission.block_hash_hex) - events.append("state") - return { - "chain_state": "prepared" if prepared["value"] else "rejected", - "maturity_state": "immature", - } - - def reject_prepared_block(**_kwargs: object) -> dict[str, object]: - events.append("reject") - prepared["value"] = False - return {"backend": "fake", "rejected_count": 1} - - def mark_abandoned(**_kwargs: object) -> bool: - events.append("abandon") - return True - - ledger.pool_block_state = pool_block_state # type: ignore[attr-defined] - ledger.reject_prepared_block = reject_prepared_block # type: ignore[method-assign] - ledger.mark_block_candidate_abandoned = mark_abandoned # type: ignore[attr-defined] - - class CleanupRpc(TipRpc): - def call( - self, - method: str, - params: list[object] | None = None, - ) -> object: - if method == "getblockcount": - return 9 - return super().call(method, params) - - server.rpc = CleanupRpc("00" * 32) - server._node_submission_for_candidate = ( # type: ignore[method-assign] - lambda _candidate: SimpleNamespace( - attempted=False, - result=None, - error=None, - ) - ) - server._mark_block_candidate_attempted = ( # type: ignore[method-assign] - lambda _block_hash: True - ) - - def stage_terminal_rejection( - _candidate: PrismBlockCandidate, - *, - node_submission: object, - ) -> bool: - self.assertIsNotNone(node_submission) - outcome = server._block_candidate_outcome - outcome.reason = PRISM_REJECTION_POOL_CLOSED - outcome.error = "pool is no longer accepting blocks" - outcome.stale_job_class = None - return False - - server._submit_block_candidate_serialized = ( # type: ignore[method-assign] - stage_terminal_rejection - ) - - self.assertFalse(server._submit_synchronous_block_candidate(candidate)) - self.assertEqual(events, ["state", "reject", "abandon"]) - self.assertEqual( - server.block_candidate_abandoned_counts, - {PRISM_REJECTION_POOL_CLOSED: 1}, - ) - self.assertFalse(server._block_candidate_terminal_outcome(block_hash)) - - def test_synchronous_cleanup_failure_keeps_candidate_pending(self) -> None: - server, state, ledger = submit_coordinator() - block_hash = "cb" * 32 - candidate = block_candidate( - server, - state, - SimpleNamespace( - coinbase_tx_hex="00", - block_hash_hex=block_hash, - block_hex="cb", - share_pass=False, - block_pass=True, - ), - credit_share_on_accept=True, - ) - events: list[str] = [] - ledger.pool_block_state = ( # type: ignore[attr-defined] - lambda *, block_hash: { - "chain_state": "prepared", - "maturity_state": "immature", - } - ) - - def fail_reject(**_kwargs: object) -> dict[str, object]: - events.append("reject") - raise RuntimeError("postgres unavailable") - - def unexpected_abandon(**_kwargs: object) -> bool: - events.append("abandon") - return True - - ledger.reject_prepared_block = fail_reject # type: ignore[method-assign] - ledger.mark_block_candidate_abandoned = unexpected_abandon # type: ignore[attr-defined] - - class CleanupRpc(TipRpc): - def call( - self, - method: str, - params: list[object] | None = None, - ) -> object: - if method == "getblockcount": - return 9 - return super().call(method, params) - - server.rpc = CleanupRpc("00" * 32) - server._node_submission_for_candidate = ( # type: ignore[method-assign] - lambda _candidate: SimpleNamespace( - attempted=False, - result=None, - error=None, - ) - ) - server._mark_block_candidate_attempted = ( # type: ignore[method-assign] - lambda _block_hash: True - ) - - def stage_terminal_rejection( - _candidate: PrismBlockCandidate, - *, - node_submission: object, - ) -> bool: - self.assertIsNotNone(node_submission) - outcome = server._block_candidate_outcome - outcome.reason = PRISM_REJECTION_POOL_CLOSED - outcome.error = "pool is no longer accepting blocks" - outcome.stale_job_class = None - return False - - server._submit_block_candidate_serialized = ( # type: ignore[method-assign] - stage_terminal_rejection - ) - - with self.assertRaisesRegex( - RuntimeError, - "could not reject prepared state for terminal candidate", - ): - server._submit_synchronous_block_candidate(candidate) - - self.assertEqual(events, ["reject"]) - self.assertIs(server._retry_block_candidate, candidate) - self.assertEqual(server.block_candidate_abandoned_counts, {}) - self.assertIsNone(server._block_candidate_terminal_outcome(block_hash)) - self.assertEqual( - server._block_candidate_outcome.reason, - PRISM_REJECTION_BACKEND_RPC_UNAVAILABLE, - ) - - def test_restart_resubmit_honors_terminal_abandoned_outbox(self) -> None: - server, state, _recording = submit_coordinator() - ledger = SingleWriterShareLedger() - server.ledger = ledger - block_hash = "e8" * 32 - submission = SimpleNamespace( - header_hex="aa" * 80, - coinbase_tx_hex="00", - block_hash_hex=block_hash, - block_hex="e8", - share_pass=False, - block_pass=True, - ) - first_pending = server.pending_share_from_submission( - context=server.jobs["job-1"], - submission=submission, - ntime_hex="00000001", - ) - first_candidate = block_candidate( - server, - state, - submission, - pending_share=first_pending, - credit_share_on_accept=True, - ) - self.assertTrue( - ledger.persist_block_candidate_intent( - server.block_candidate_intent(first_candidate) - ) - ) - self.assertTrue( - ledger.mark_block_candidate_abandoned( - block_hash=block_hash, - error="terminal before restart", - ) - ) - server._finish_pending_share_commit(first_pending) - submitblock_calls: list[str] = [] - - class NoResubmitRpc(TipRpc): - def call( - self, - method: str, - params: list[object] | None = None, - *, - timeout: float | None = None, - ) -> object: - if method == "submitblock": - submitblock_calls.append(str((params or [""])[0])) - return None - return super().call(method, params) - - server.rpc = NoResubmitRpc("00" * 32) - with patch( - "lab.prism.prism_coordinator.direct_stratum.assemble_submission", - return_value=submission, - ): - with self.assertRaises(StratumError) as raised: - server.handle_submit( - state, - ["miner-a", "job-1", "00" * 8, "00000001", "00000002"], - ) - - self.assertEqual(raised.exception.code, 23) - self.assertEqual(submitblock_calls, []) - self.assertEqual(ledger._block_candidate_outbox[block_hash]["state"], "abandoned") - self.assertEqual(len(ledger), 0) - - def test_restart_resubmit_coalesces_terminal_submitted_outbox(self) -> None: - first_server, first_state, _recording = submit_coordinator() - ledger = SingleWriterShareLedger() - first_server.ledger = ledger - block_hash = "f8" * 32 - submission = SimpleNamespace( - header_hex="ab" * 80, - coinbase_tx_hex="00", - block_hash_hex=block_hash, - block_hex="f8", - share_pass=True, - block_pass=True, - ) - first_pending = PendingShare( - share_id=f"miner-a:{block_hash}", - miner_id="miner-a", - order_key="miner-a", - p2mr_program_hex="11" * 32, - share_difficulty=7, - network_difficulty=1, - template_height=9, - job_id="job-1", - job_issued_at_ms=12345, - accepted_at_ms=1, - ntime=1, - ) - first_candidate = block_candidate( - first_server, - first_state, - submission, - pending_share=first_pending, - ) - first_record = ledger.append_batch( - [ - ( - first_pending, - first_server.block_candidate_intent(first_candidate), - ) - ] - )[0] - self.assertTrue(first_record.newly_inserted) - self.assertTrue( - ledger.mark_block_candidate_submitted(block_hash=block_hash) - ) - - # Model a clean restart: volatile duplicate/disposition state is gone, - # while the ledger and terminal candidate outbox remain authoritative. - server, state, _recording = submit_coordinator() - server.ledger = ledger - submitblock_calls: list[str] = [] - - class NoResubmitRpc(TipRpc): - def call( - self, - method: str, - params: list[object] | None = None, - *, - timeout: float | None = None, - ) -> object: - if method == "submitblock": - submitblock_calls.append(str((params or [""])[0])) - return None - return super().call(method, params) - - server.rpc = NoResubmitRpc("00" * 32) - with patch( - "lab.prism.prism_coordinator.direct_stratum.assemble_submission", - return_value=submission, - ): - self.assertFalse( - server.handle_submit( - state, - ["miner-a", "job-1", "00" * 8, "00000001", "00000002"], - ) - ) - - self.assertEqual(submitblock_calls, []) - self.assertEqual( - ledger._block_candidate_outbox[block_hash]["state"], "submitted" - ) - self.assertEqual(len(ledger), 1) - self.assertEqual(ledger._shares[0].accepted_at_ms, 1) - self.assertTrue(server.block_candidate_queue.empty()) - self.assertEqual( - server.worker_share_counts["miner-a"], - {"submitted": 1, "accepted": 0, "grace": 0}, - ) - - def test_exact_share_replay_does_not_repeat_process_credit(self) -> None: - server, state, _recording = submit_coordinator() - ledger = SingleWriterShareLedger() - server.ledger = ledger - block_hash = "d8" * 32 - submission = SimpleNamespace( - coinbase_tx_hex="00", - block_hash_hex=block_hash, - block_hex="d8", - share_pass=True, - block_pass=True, - ) - pending = PendingShare( - share_id=f"miner-a:{block_hash}", - miner_id="miner-a", - order_key="miner-a", - p2mr_program_hex="11" * 32, - share_difficulty=1, - network_difficulty=1, - template_height=9, - job_id="job-1", - job_issued_at_ms=12345, - accepted_at_ms=12346, - ntime=1, - ) - candidate = block_candidate( - server, - state, - submission, - pending_share=pending, - ) - intent = server.block_candidate_intent(candidate) - worker_credits: list[str] = [] - vardiff_credits: list[str] = [] - server.note_worker_accepted_share = ( # type: ignore[method-assign] - lambda worker, _policy: worker_credits.append(worker) - ) - server.note_vardiff_accepted_share = ( # type: ignore[method-assign] - lambda _client, job: vardiff_credits.append(job.job_id) - ) - - self.assertEqual( - server.append_accepted_share( - state, - candidate.context, - submission, - pending, - candidate_intent=intent, - ), - "pending", - ) - self.assertEqual( - server.append_accepted_share( - state, - candidate.context, - submission, - pending, - candidate_intent=intent, - ), - "pending", - ) + self.assertTrue(server._submit_next_block_candidate_writer(candidate)) - self.assertEqual(len(ledger), 1) - self.assertEqual(worker_credits, ["miner-a"]) - self.assertEqual(vardiff_credits, ["job-1"]) + # Deferred, not terminal: prepared rows untouched, candidate + # retained for retry, observation matching still open. + self.assertEqual(ledger.rejected, []) + self.assertEqual(abandoned, []) + self.assertEqual(submitted, []) + self.assertGreaterEqual( + getattr(server, "block_candidate_accept_pending_defer_count", 0), + 1, + ) + self.assertIsNotNone(getattr(server, "_retry_block_candidate", None)) + self.assertNotIn( + "block-stale", + server.block_candidate_abandoned_counts, + ) + self.assertIn( + block_hash, server._outstanding_block_candidate_hashes + ) - def test_same_hash_busy_lease_preserves_retry_behind_live_work(self) -> None: - server, state, _recording = submit_coordinator() - server.block_candidate_retry_initial_seconds = 0.0 - candidate = block_candidate( - server, - state, - SimpleNamespace( - coinbase_tx_hex="00", - block_hash_hex="a7" * 32, - block_hex="a7", - share_pass=True, - block_pass=True, - ), - ) - block_hash = candidate.submission.block_hash_hex - lease = server._claim_block_candidate_disposition( + def test_retained_acceptance_evidence_ages_on_observation_window( + self, + ) -> None: + # The stash is acceptance evidence only while fresh: past the + # observation window it stands down so a genuinely orphaned block + # (whose probes never prove anything either way) regains + # abandonability instead of deferring forever. + server, _state, _ledger = submit_coordinator() + block_hash = "cf" * 32 + server._stash_retained_block_candidate_node_submission( block_hash, - blocking=True, + SimpleNamespace(attempted=True, error=None, result=None), ) - assert lease is not None - server._retry_block_candidate = candidate - server._ensure_block_replay_state() - server._block_replay_inflight_hashes.add(block_hash) - try: - self.assertTrue( - server.submit_next_block_candidate(defer_accounting=True) + self.assertTrue(server._block_candidate_acceptance_retained(block_hash)) + with server.lock: + server._block_candidate_retained_submission_monotonic[ + block_hash + ] -= 400.0 + self.assertFalse(server._block_candidate_acceptance_retained(block_hash)) + # A non-positive window means evidence never expires, mirroring the + # tip-observation semantics. + server.observed_tip_accept_window_seconds = 0.0 + self.assertTrue(server._block_candidate_acceptance_retained(block_hash)) + + def test_pool_closed_gate_requires_probe_proven_acceptance(self) -> None: + # Bugbot: observation evidence alone must not open the pool-closed + # gate -- an off-chain candidate would fall through toward + # submitblock after the pool stopped accepting blocks. Unprovable + # views defer via the abandon path; expired evidence goes terminal. + parent = "00" * 32 + block_hash = "d9" * 32 + server, state, _ledger = submit_coordinator(tip=parent) + server.max_blocks = 1 + server.stop_after_block = False + server.accepted_block_count = 1 + server.observed_tip_accept_window_seconds = 60.0 + with tempfile.TemporaryDirectory() as tempdir: + submitted, abandoned = self._accepted_tail_scaffolding(server, tempdir) + rpc = LostAckSubmitRpc( + start_tip=parent, + hash_by_hex={"00": block_hash}, ) - self.assertIsNone(server._retry_block_candidate) - self.assertIs( - server._block_disposition_waiting_retries[block_hash], - candidate, + server.rpc = rpc + server._register_outstanding_block_candidate(block_hash) + with server.lock: + server._tip_observed_accepted_block_hashes[block_hash] = ( + time.monotonic() + ) + candidate = block_candidate( + server, + state, + SimpleNamespace( + coinbase_tx_hex="c0ffee", + block_hash_hex=block_hash, + block_hex="00", + share_pass=True, + block_pass=True, + ), ) - finally: - server._release_block_candidate_disposition(lease) - captured: list[object] = [] - server._node_submission_for_candidate = ( # type: ignore[method-assign] - lambda _candidate: SimpleNamespace( - attempted=True, - result="duplicate", - error=None, + # Fresh observation, unprovable chain view: defer, and above all + # never reach submitblock through the closed pool. + self.assertTrue(server._submit_next_block_candidate_writer(candidate)) + self.assertEqual(rpc.submitblock_calls, 0) + self.assertEqual(abandoned, []) + candidate = self._retained_candidate(server) + + # Evidence expires with the candidate still absent: terminal. + with server.lock: + server._tip_observed_accepted_block_hashes[block_hash] = ( + time.monotonic() - 61.0 + ) + self.assertTrue(server._submit_next_block_candidate_writer(candidate)) + self.assertEqual(rpc.submitblock_calls, 0) + self.assertEqual(abandoned, [block_hash]) + self.assertEqual(submitted, []) + self.assertEqual( + server.block_candidate_abandoned_counts[ + PRISM_REJECTION_POOL_CLOSED + ], + 1, ) - ) - server._enqueue_block_accounting_task = ( # type: ignore[method-assign] - lambda task: (captured.append(task), True)[1] - ) - self.assertTrue(server.submit_next_block_candidate(defer_accounting=True)) - self.assertEqual(len(captured), 1) - task = captured[0] - server._release_block_candidate_disposition(task.disposition_lease) - def test_permanently_closed_pool_hands_outbox_to_accounting(self) -> None: - server, state, _recording = submit_coordinator() + def test_pool_closed_gate_yields_to_on_chain_candidate(self) -> None: + # A candidate already on the active chain must complete its payout + # accounting even after the pool stops accepting new blocks; the + # pool-closed gate previously abandoned it terminally. + parent = "00" * 32 + block_hash = "f6" * 32 + server, state, ledger = submit_coordinator(tip=parent) server.max_blocks = 1 + server.stop_after_block = False server.accepted_block_count = 1 - candidate = block_candidate( - server, - state, - SimpleNamespace( - coinbase_tx_hex="00", - block_hash_hex="f2" * 32, - block_hex="f2", - share_pass=True, - block_pass=True, - ), - ) - captured: list[object] = [] - server._enqueue_block_accounting_task = ( # type: ignore[method-assign] - lambda task: (captured.append(task), True)[1] - ) - server.rpc = SimpleNamespace( - call=lambda *_args, **_kwargs: (_ for _ in ()).throw( - AssertionError("closed pool must not call submitblock") + with tempfile.TemporaryDirectory() as tempdir: + submitted, abandoned = self._accepted_tail_scaffolding(server, tempdir) + rpc = LostAckSubmitRpc( + start_tip=parent, + hash_by_hex={"00": block_hash}, + ) + # The block is already active with the pool's own hash as tip + # (for example after a watchdog restart replayed the outbox). + rpc.tip = block_hash + rpc.height = 10 + rpc.active[block_hash] = 10 + server.rpc = rpc + server.ensure_reorg_reconciled_for_tip = ( # type: ignore[method-assign] + lambda _tip, *, _coalesce_same_tip: not _coalesce_same_tip + ) + candidate = block_candidate( + server, + state, + SimpleNamespace( + coinbase_tx_hex="c0ffee", + block_hash_hex=block_hash, + block_hex="00", + share_pass=True, + block_pass=True, + ), ) - ) - server.enqueue_block_candidate(candidate) - self.assertTrue(server.submit_next_block_candidate(defer_accounting=True)) - self.assertEqual(len(captured), 1) - task = captured[0] - self.assertFalse(task.node_submission.attempted) - self.assertIsNone(getattr(server, "_retry_block_candidate", None)) - server._release_block_candidate_disposition(task.disposition_lease) + self.assertTrue(server._submit_next_block_candidate_writer(candidate)) - def test_accounted_block_replaces_its_capacity_reservation(self) -> None: - server, state, _recording = submit_coordinator() - server.max_blocks = 2 - server.stop_after_block = False - block_hash = "e2" * 32 - candidate = block_candidate( - server, - state, - SimpleNamespace( - coinbase_tx_hex="c0ffee", - block_hash_hex=block_hash, - block_hex="e2", - share_pass=True, - block_pass=True, - ), - ) - server._ensure_block_candidate_disposition_state() - server._block_fast_lane_reservations.add(block_hash) - server._land_and_confirm_block_candidate = ( # type: ignore[method-assign] - lambda *_args, **_kwargs: ( - verified_block_bundle(), - verified_audit_report(), - {"persisted": True}, - {"confirmed_count": 1}, + self.assertEqual(submitted, [block_hash]) + self.assertEqual(abandoned, []) + self.assertNotIn( + PRISM_REJECTION_POOL_CLOSED, + getattr(server, "block_candidate_abandoned_counts", {}), + ) + self.assertIn(block_hash, server._accounted_accepted_block_hashes) + self.assertEqual(server.accepted_block_count, 2) + + +class PrismCoordinatorReliabilityTests(unittest.TestCase): + def _bare_coordinator(self) -> PrismCoordinator: + server = PrismCoordinator.__new__(PrismCoordinator) + server.lock = threading.RLock() + server.stop_event = threading.Event() + server._heartbeats = {} + server._watchdog_pauses = {} + server._heartbeats_lock = threading.Lock() + server.watchdog_timeout_seconds = 120.0 + server.watchdog_interval_seconds = 15.0 + return server + + def test_block_candidate_progress_stamps_only_on_submitter_thread(self) -> None: + server = self._bare_coordinator() + clock = {"now": 1000.0} + with patch( + "lab.prism.prism_coordinator.time.monotonic", + side_effect=lambda: clock["now"], + ): + server._record_heartbeat("block_submitter") + clock["now"] += server.watchdog_timeout_seconds + 1.0 + self.assertEqual( + server._overdue_heartbeats(clock["now"]), ["block_submitter"] ) - ) - server.accepted_share_stats = lambda: (0, 0) # type: ignore[method-assign] - with tempfile.TemporaryDirectory() as tempdir: - server.audit_dir = Path(tempdir) - server.evidence_path = Path(tempdir) / "evidence.json" - self.assertTrue( - server._submit_block_candidate_serialized( - candidate, - node_submission=SimpleNamespace( - attempted=True, - result=None, - error=None, - ), - ) + # Without a registered owner the helper never stamps. + server._record_block_candidate_progress() + self.assertEqual( + server._overdue_heartbeats(clock["now"]), ["block_submitter"] ) + # The owner thread's stamps clear the overdue state. + server._block_submitter_thread_ident = threading.get_ident() + server._record_block_candidate_progress() + self.assertEqual(server._overdue_heartbeats(clock["now"]), []) - self.assertEqual(server.accepted_block_count, 1) - self.assertNotIn(block_hash, server._block_fast_lane_reservations) - self.assertTrue(server._reserve_block_fast_lane_slot("e3" * 32)) + def test_client_thread_disposition_cannot_mask_frozen_submitter(self) -> None: + server = self._bare_coordinator() + clock = {"now": 1000.0} + owner_ready = threading.Event() + release_owner = threading.Event() - def test_synchronous_candidate_waits_for_fast_lane_capacity(self) -> None: - server, state, _recording = submit_coordinator() - server.max_blocks = 1 - server.stop_after_block = True - reserved_hash = "e4" * 32 - candidate = block_candidate( - server, - state, - SimpleNamespace( - coinbase_tx_hex="c0ffee", - block_hash_hex="e5" * 32, - block_hex="e5", - share_pass=False, - block_pass=True, - ), - credit_share_on_accept=True, - ) - server._ensure_block_candidate_disposition_state() - self.assertTrue(server._reserve_block_fast_lane_slot(reserved_hash)) - server._node_submission_for_candidate = ( # type: ignore[method-assign] - lambda _candidate: (_ for _ in ()).throw( - AssertionError("capacity-blocked candidate must not reach qbitd") - ) - ) + def frozen_owner() -> None: + server._block_submitter_thread_ident = threading.get_ident() + server._record_heartbeat("block_submitter") + owner_ready.set() + release_owner.wait(5) - with self.assertRaisesRegex(RuntimeError, "fast-lane capacity"): - server._submit_synchronous_block_candidate(candidate) + with patch( + "lab.prism.prism_coordinator.time.monotonic", + side_effect=lambda: clock["now"], + ): + owner = threading.Thread(target=frozen_owner) + owner.start() + try: + self.assertTrue(owner_ready.wait(5)) + clock["now"] += server.watchdog_timeout_seconds + 1.0 + self.assertEqual( + server._overdue_heartbeats(clock["now"]), ["block_submitter"] + ) + # A client connection thread reaching a disposition boundary + # (synchronous below-target solve) must not refresh the + # frozen dedicated thread's liveness budget. + foreign = threading.Thread( + target=server._record_block_candidate_progress + ) + foreign.start() + foreign.join(5) + self.assertEqual( + server._overdue_heartbeats(clock["now"]), ["block_submitter"] + ) + finally: + release_owner.set() + owner.join(5) - self.assertIs(server._retry_block_candidate, candidate) - self.assertEqual(server._block_fast_lane_reservations, {reserved_hash}) + def test_audit_verifier_subprocess_has_explicit_timeout(self) -> None: + server = self._bare_coordinator() + server.bundle_build_timeout_seconds = 0.25 + with tempfile.TemporaryDirectory() as tempdir: + server.audit_dir = Path(tempdir) / "audit" + server.evidence_path = Path(tempdir) / "evidence.json" + store = server._ensure_audit_artifact_store() + try: + # The verifier budget is the coordinator-configured bundle + # build budget; the store must never silently fall back to a + # fixed default while a shorter budget is configured. + self.assertEqual(store.verifier_timeout_seconds, 0.25) + started = time.monotonic() + with patch( + "lab.prism.audit_artifacts.prism_tool_command", + lambda _name: ["python3", "-c", "import time; time.sleep(5)"], + ): + with self.assertRaisesRegex(RuntimeError, "timed out"): + server.verify_bundle( + Path(tempdir) / "candidate.json", + "00", + "11" * 32, + expected_coinbase_value_sats=1, + ) + self.assertLess(time.monotonic() - started, 5.0) + finally: + store.close() - def test_node_offer_exception_retains_reserved_candidate(self) -> None: - server, state, _recording = submit_coordinator() - server.max_blocks = 1 - server.stop_after_block = True - candidate = block_candidate( - server, - state, - SimpleNamespace( - coinbase_tx_hex="c0ffee", - block_hash_hex="e6" * 32, - block_hex="e6", - share_pass=True, - block_pass=True, - ), - ) - block_hash = candidate.submission.block_hash_hex - server._node_submission_for_candidate = ( # type: ignore[method-assign] - lambda _candidate: (_ for _ in ()).throw( - RuntimeError("node offer bookkeeping failed") - ) - ) - server.enqueue_block_candidate(candidate) + def test_sixty_second_attempt_mark_stall_does_not_delay_rpc_or_heartbeat(self) -> None: + server, state, recording = submit_coordinator() + entered_mark = threading.Event() + release_mark = threading.Event() + submitted = threading.Event() - with self.assertRaisesRegex(RuntimeError, "node offer bookkeeping failed"): - server.submit_next_block_candidate(defer_accounting=True) + class StallingLedger(RecordingLedger): + def __init__(self) -> None: + super().__init__() + self.mark_calls = 0 - self.assertIs(server._retry_block_candidate, candidate) - self.assertIn(block_hash, server._block_fast_lane_reservations) - self.assertFalse(server._reserve_block_fast_lane_slot("e7" * 32)) - lease = server._claim_block_candidate_disposition( - block_hash, - blocking=False, - ) - self.assertIsNotNone(lease) - assert lease is not None - server._release_block_candidate_disposition(lease) + def mark_block_candidate_attempted(self, *, block_hash: str) -> bool: + self.mark_calls += 1 + entered_mark.set() + release_mark.wait(60.0) + return True - def test_accounting_enqueue_exception_retains_reserved_candidate(self) -> None: - server, state, _recording = submit_coordinator() - server.max_blocks = 1 - server.stop_after_block = True + class DeadlineRpc(SubmitRpc): + def __init__(self, ledger: RecordingLedger) -> None: + super().__init__( + tip="00" * 32, + block_hash="d2" * 32, + ledger=ledger, + ) + self.timeouts: list[float | None] = [] + + def call( + self, + method: str, + params: list[object] | None = None, + *, + timeout: float | None = None, + ) -> object: + if method == "submitblock": + self.timeouts.append(timeout) + submitted.set() + if len(self.timeouts) > 1: + return "duplicate" + return super().call(method, params) + + ledger = StallingLedger() + server.ledger = ledger + rpc = DeadlineRpc(ledger) + server.rpc = rpc + server.block_submit_rpc_timeout_seconds = 0.4 + server.block_submit_db_timeout_seconds = 0.05 + server.block_candidate_retry_initial_seconds = 0.01 + server.block_candidate_retry_max_seconds = 0.01 + server.watchdog_timeout_seconds = 0.2 + server._heartbeats = {} + server._heartbeat_phases = {} + server._watchdog_pauses = {} + server._heartbeats_lock = threading.Lock() candidate = block_candidate( server, state, SimpleNamespace( - coinbase_tx_hex="c0ffee", - block_hash_hex="e8" * 32, - block_hex="e8", + coinbase_tx_hex="00", + block_hash_hex="d2" * 32, + block_hex="00", share_pass=True, block_pass=True, ), ) - block_hash = candidate.submission.block_hash_hex - server._node_submission_for_candidate = ( # type: ignore[method-assign] - lambda _candidate: SimpleNamespace( - attempted=True, - result=None, - error=None, - ) - ) - server._enqueue_block_accounting_task = ( # type: ignore[method-assign] - lambda _task: (_ for _ in ()).throw( - RuntimeError("accounting enqueue failed") - ) - ) server.enqueue_block_candidate(candidate) + started = time.monotonic() + submitter = threading.Thread(target=server.block_submit_loop) + with patch("builtins.print"): + submitter.start() + try: + self.assertTrue(submitted.wait(2.0)) + self.assertLess(time.monotonic() - started, 2.0) + self.assertTrue(entered_mark.wait(1.0)) + with server._heartbeats_lock: + first_heartbeat = server._heartbeats["block_submitter"] + heartbeat_deadline = time.monotonic() + 1.0 + while time.monotonic() < heartbeat_deadline: + with server._heartbeats_lock: + latest_heartbeat = server._heartbeats["block_submitter"] + if latest_heartbeat > first_heartbeat: + break + time.sleep(0.01) + self.assertGreater(latest_heartbeat, first_heartbeat) + self.assertEqual( + server._overdue_heartbeats(time.monotonic()), + [], + ) + self.assertEqual(ledger.mark_calls, 1) + self.assertEqual(rpc.timeouts[0], 0.4) + finally: + server.stop_event.set() + submitter.join(2.0) + release_mark.set() + self.assertFalse(submitter.is_alive()) - with self.assertRaisesRegex(RuntimeError, "accounting enqueue failed"): - server.submit_next_block_candidate(defer_accounting=True) - - self.assertIs(server._retry_block_candidate, candidate) - self.assertIn(block_hash, server._block_fast_lane_reservations) - self.assertFalse(server._reserve_block_fast_lane_slot("e9" * 32)) - lease = server._claim_block_candidate_disposition( - block_hash, - blocking=False, - ) - self.assertIsNotNone(lease) - assert lease is not None - server._release_block_candidate_disposition(lease) - - def test_block_submitter_retry_wait_heartbeats_in_bounded_slices(self) -> None: - server = self._bare_coordinator() - server.watchdog_timeout_seconds = 0.3 - clock = {"now": 0.0} - waits: list[float] = [] - overdue_samples: list[list[str]] = [] - - class AdvancingStopEvent: - def is_set(self) -> bool: - return False - - def wait(self, timeout: float) -> bool: - waits.append(timeout) - clock["now"] += timeout - overdue_samples.append(server._overdue_heartbeats(clock["now"])) - return False - - server.stop_event = AdvancingStopEvent() # type: ignore[assignment] - with patch( - "lab.prism.prism_coordinator.time.monotonic", - side_effect=lambda: clock["now"], - ): - self.assertFalse(server._wait_for_block_candidate_retry(1.0)) + def test_accounting_retry_backoff_defers_instead_of_sleeping_under_admission(self) -> None: + server, state, _recording = submit_coordinator() + server.block_candidate_retry_initial_seconds = 30.0 + server.block_candidate_retry_max_seconds = 30.0 - self.assertEqual(waits, [0.25, 0.25, 0.25, 0.25]) - self.assertEqual(overdue_samples, [[], [], [], []]) - self.assertEqual( - server._heartbeats["block_submitter"], - 1.0, - ) + class FailingMarkLedger(RecordingLedger): + def mark_block_candidate_attempted(self, *, block_hash: str) -> bool: + raise RuntimeError("attempt marker unavailable") - def test_blocked_candidate_phase_remains_watchdog_eligible(self) -> None: - server, state, _recording = submit_coordinator() - server._heartbeats = {} - server._watchdog_pauses = {} - server._heartbeats_lock = threading.Lock() - server.watchdog_timeout_seconds = 0.05 - entered_submission = threading.Event() - release_submission = threading.Event() + server.ledger = FailingMarkLedger() candidate = block_candidate( server, state, SimpleNamespace( coinbase_tx_hex="00", - block_hash_hex="fa" * 32, - block_hex="00", + block_hash_hex="ab" * 32, + block_hex="ab", share_pass=True, block_pass=True, ), ) - - def blocked_submission(_candidate: PrismBlockCandidate) -> bool: - entered_submission.set() - release_submission.wait(2) - return True - - server.submit_block_candidate = blocked_submission # type: ignore[method-assign] - server.enqueue_block_candidate(candidate) - server._record_heartbeat("block_submitter") - submitter = threading.Thread(target=server.submit_next_block_candidate) - submitter.start() - try: - self.assertTrue(entered_submission.wait(1)) - time.sleep(0.08) - self.assertEqual( - server._overdue_heartbeats(time.monotonic()), - ["block_submitter"], - ) - finally: - release_submission.set() - submitter.join(2) - self.assertFalse(submitter.is_alive()) - - def test_progressing_ctv_pass_longer_than_watchdog_timeout_stays_healthy(self) -> None: - server = self._bare_coordinator() - server.ctv_broadcaster_limit = 200 - server.ctv_broadcaster_interval_seconds = 30.0 - clock = {"now": 0.0} - overdue_samples: list[list[str]] = [] - seen_limits: list[int] = [] - - class StopAfterOnePass: - def is_set(self) -> bool: - return False - - def wait(self, timeout: float) -> bool: - return True - - class ProgressingDaemon: - def run_once(self, *, limit: int, progress_callback: object, **_kwargs: object) -> object: - seen_limits.append(limit) - assert callable(progress_callback) - for _ in range(3): - clock["now"] += 80.0 - overdue_samples.append(server._overdue_heartbeats(clock["now"])) - progress_callback() - return SimpleNamespace( - scanned_count=3, - submitted_count=0, - updated_count=3, - failed_count=0, - yielded_to_tip_refresh=False, - ) - - server.stop_event = StopAfterOnePass() # type: ignore[assignment] - server.ctv_fanout_broadcast_daemon = ProgressingDaemon() - - with patch("lab.prism.prism_coordinator.time.monotonic", side_effect=lambda: clock["now"]), patch( - "builtins.print" - ): - server.ctv_fanout_broadcaster_loop() - - self.assertGreater(clock["now"], server.watchdog_timeout_seconds) - self.assertEqual(seen_limits, [200]) - self.assertEqual(overdue_samples, [[], [], []]) - self.assertEqual(server.ctv_broadcaster_processed_rows_total, 3) - self.assertEqual(server.ctv_broadcaster_pass_count, 1) - - def test_ctv_pass_completion_heartbeat_precedes_interval_wait(self) -> None: - server = self._bare_coordinator() - server.ctv_broadcaster_limit = 200 - server.ctv_broadcaster_interval_seconds = 30.0 - clock = {"now": 0.0} - wait_observation: dict[str, object] = {} - - class StopAfterIntervalWait: - def is_set(self) -> bool: - return False - - def wait(self, timeout: float) -> bool: - wait_observation["timeout"] = timeout - wait_observation["heartbeat"] = server._heartbeats["ctv_fanout_broadcaster"] - wait_observation["overdue_after_wait"] = server._overdue_heartbeats( - clock["now"] + timeout - ) - return True - - class IncidentDurationDaemon: - def run_once(self, *, limit: int, progress_callback: object, **_kwargs: object) -> object: - clock["now"] += 102.0 - return SimpleNamespace( - scanned_count=200, - submitted_count=0, - updated_count=200, - failed_count=0, - yielded_to_tip_refresh=False, - ) - - server.stop_event = StopAfterIntervalWait() # type: ignore[assignment] - server.ctv_fanout_broadcast_daemon = IncidentDurationDaemon() - - with patch("lab.prism.prism_coordinator.time.monotonic", side_effect=lambda: clock["now"]), patch( - "builtins.print" - ): - server.ctv_fanout_broadcaster_loop() - - self.assertEqual(wait_observation["timeout"], 30.0) - self.assertEqual(wait_observation["heartbeat"], 102.0) - self.assertEqual(wait_observation["overdue_after_wait"], []) - - def test_ctv_pass_without_progress_remains_watchdog_eligible(self) -> None: - server = self._bare_coordinator() - server.ctv_broadcaster_limit = 200 - server.ctv_broadcaster_interval_seconds = 30.0 - clock = {"now": 0.0} - entered_row = threading.Event() - release_row = threading.Event() - - class BlockingDaemon: - def run_once(self, *, limit: int, progress_callback: object, **_kwargs: object) -> object: - entered_row.set() - release_row.wait() - return SimpleNamespace( - scanned_count=1, - submitted_count=0, - updated_count=1, - failed_count=0, - yielded_to_tip_refresh=False, - ) - - server.ctv_fanout_broadcast_daemon = BlockingDaemon() - broadcaster_thread = threading.Thread(target=server.ctv_fanout_broadcaster_loop) - with patch("lab.prism.prism_coordinator.time.monotonic", side_effect=lambda: clock["now"]), patch( - "builtins.print" - ): - broadcaster_thread.start() - self.assertTrue(entered_row.wait(timeout=1.0)) - clock["now"] = server.watchdog_timeout_seconds + 1.0 - self.assertEqual( - server._overdue_heartbeats(clock["now"]), - ["ctv_fanout_broadcaster"], + block_hash = "ab" * 32 + server._block_accounting_thread_ident = threading.get_ident() + server._block_accounting_holds_disposition = True + started = time.monotonic() + with patch("builtins.print"): + handled = server._submit_next_block_candidate_writer( + candidate, + node_submission=SimpleNamespace( + attempted=False, + result=None, + error=None, + ), + disposition_held=True, ) - server.stop_event.set() - release_row.set() - broadcaster_thread.join(timeout=1.0) - - self.assertFalse(broadcaster_thread.is_alive()) - - def test_watchdog_pause_suppresses_known_long_critical_section(self) -> None: - server = self._bare_coordinator() - server._record_heartbeat("stratum_accept") - server._record_heartbeat("qbit_blockpoll") - now = time.monotonic() - with server._heartbeats_lock: - server._heartbeats["stratum_accept"] = now - 1_000.0 - server._heartbeats["qbit_blockpoll"] = now - 1_000.0 - - self.assertEqual(server._overdue_heartbeats(now), ["qbit_blockpoll", "stratum_accept"]) - - with server._watchdog_paused("qbit_blockpoll", "stratum_accept"): - self.assertEqual(server._overdue_heartbeats(now + 1_000.0), []) - - self.assertEqual(server._overdue_heartbeats(time.monotonic()), []) - - def test_block_submit_pause_names_cover_registered_refresh_and_idle_threads(self) -> None: - server = self._bare_coordinator() - for name in ("stratum_accept", "qbit_blockpoll", "qbit_blockwait", "vardiff_idle_sweep"): - server._record_heartbeat(name) - now = time.monotonic() - with server._heartbeats_lock: - for name in server._heartbeats: - server._heartbeats[name] = now - 1_000.0 - - pause_names = server._registered_watchdog_heartbeat_names( - "qbit_blockpoll", - "qbit_blockwait", - "vardiff_idle_sweep", - "stratum_accept", + elapsed = time.monotonic() - started + self.assertTrue(handled) + # The 30s backoff must be recorded as a deadline, never slept while + # the accounting thread holds admission and the disposition lease. + self.assertLess(elapsed, 10.0) + self.assertIs( + getattr(server, "_block_accounting_deferred_retry_candidate", None), + candidate, ) + with server.lock: + deadline = server._block_candidate_retry_not_before.get(block_hash) + self.assertIsNotNone(deadline) + self.assertGreater(deadline, time.monotonic()) - with server._watchdog_paused(*pause_names): - self.assertEqual(server._overdue_heartbeats(now + 1_000.0), []) - - def test_pause_names_skip_removed_blockwait_without_resurrecting_heartbeat(self) -> None: - server = self._bare_coordinator() - server._record_heartbeat("qbit_blockpoll") - server._record_heartbeat("qbit_blockwait") - server._remove_watchdog_heartbeat("qbit_blockwait") - - pause_names = server._registered_watchdog_heartbeat_names("qbit_blockpoll", "qbit_blockwait") - - self.assertEqual(pause_names, ("qbit_blockpoll",)) - with server._watchdog_paused(*pause_names): - pass - self.assertNotIn("qbit_blockwait", server._heartbeats) - - def test_blockwait_parameter_mismatch_is_treated_as_unsupported(self) -> None: - self.assertTrue( - PrismCoordinator._blockwait_unsupported( - RuntimeError("RPC error -32602: invalid params: wrong number of parameters") + # After the lease releases, the deferred candidate parks in the retry + # slot and the dequeue honors the backoff deadline. + server._block_accounting_holds_disposition = False + with server.lock: + server._block_accounting_deferred_retry_candidate = None + server._merge_block_candidate_retry_locked( + "_retry_block_candidate", + candidate, ) - ) - - def test_blockwait_advances_known_tip_before_notification_failure(self) -> None: - server = self._bare_coordinator() - tip_a = "aa" * 32 - tip_b = "bb" * 32 - server.rpc = SimpleNamespace(call=lambda method: tip_a) - server.blockpoll_seconds = 1.0 - known_tips: list[str] = [] - detected_tips: list[str] = [] - - def blockwait_once(known_tip: str) -> str: - known_tips.append(known_tip) - if len(known_tips) == 2: - server.stop_event.set() - return tip_b - - server.blockwait_once = blockwait_once # type: ignore[method-assign] - - def observe_tip_for_refresh(tip_hash: str, **_kwargs: object) -> bool: - detected_tips.append(tip_hash) - if tip_hash == tip_b: - raise TemplateRefreshBlocked("notification failed") - return True - - def reject_premature_publication(*_args: object, **_kwargs: object) -> bool: - raise AssertionError("blockwait must not publish submit authority") - - server.observe_tip_for_refresh = observe_tip_for_refresh # type: ignore[method-assign] - server.observe_tip_first_seen = reject_premature_publication # type: ignore[method-assign] - - with patch("builtins.print"), patch( - "lab.prism.prism_coordinator.traceback.print_exc" - ), patch.object( - server.stop_event, - "wait", - side_effect=lambda _timeout: server.stop_event.is_set(), - ): - server.blockwait_loop() - - self.assertEqual(known_tips, [tip_a, tip_b]) - self.assertEqual(detected_tips, [tip_a, tip_b]) - - def test_blockwait_unsupported_removes_watchdog_heartbeat(self) -> None: - server = coordinator() - server.rpc = UnsupportedBlockwaitRpc("00" * 32) - server._record_heartbeat("qbit_blockwait") - - server.blockwait_loop() - - self.assertNotIn("qbit_blockwait", server._heartbeats) - self.assertNotIn("qbit_blockwait", server._watchdog_pauses) - - def test_release_ledger_lease_is_noop_without_lease_support(self) -> None: - server = self._bare_coordinator() - server.ledger = SimpleNamespace() - - # In-memory/regtest ledgers have no release_writer_lease; must not raise. - server.release_ledger_lease() - - def test_release_ledger_lease_invokes_ledger_release(self) -> None: - server = self._bare_coordinator() - calls: list[bool] = [] - server.ledger = SimpleNamespace( - release_writer_lease=lambda: (calls.append(True), True)[1] - ) - - server.release_ledger_lease() - - self.assertEqual(calls, [True]) - - def test_release_ledger_lease_swallows_release_errors(self) -> None: - server = self._bare_coordinator() - - def _boom() -> bool: - raise RuntimeError("db unreachable during shutdown") - - server.ledger = SimpleNamespace(release_writer_lease=_boom) - - # Shutdown must not raise even if the lease release fails. - server.release_ledger_lease() - - -def highdiff_vardiff_config(**overrides: object) -> vardiff.VardiffConfig: - values: dict[str, object] = dict( - enabled=True, - target_share_interval_seconds=Decimal("15"), - min_difficulty=Decimal("500000"), - max_difficulty=Decimal("4294967296"), - retarget_interval_seconds=Decimal("1"), - max_step_factor=Decimal("4"), - startup_difficulty=Decimal("500000"), - max_step_down_factor=Decimal("4"), - ewma_alpha=Decimal("1"), - retarget_tolerance=Decimal("0"), - ) - values.update(overrides) - return vardiff.VardiffConfig(**values) # type: ignore[arg-type] - - -def clear_stratum_diff_env() -> None: - for name in [ - name - for name in os.environ - if name.startswith("PRISM_STRATUM_HIGHDIFF") or name.startswith("PRISM_STRATUM_VARDIFF") - ]: - os.environ.pop(name, None) - - -class PrismListenerProfileTests(unittest.TestCase): - def test_highdiff_listener_disabled_without_port(self) -> None: - with patch.dict(os.environ, {}, clear=False): - clear_stratum_diff_env() - base = load_prism_vardiff_config(Decimal("0.000000001")) - self.assertIsNone(load_prism_highdiff_listener("0.0.0.0", base)) - - def test_highdiff_listener_defaults_to_nicehash_floor(self) -> None: - with patch.dict(os.environ, {}, clear=False): - clear_stratum_diff_env() - os.environ["PRISM_STRATUM_HIGHDIFF_PORT"] = "4334" - base = load_prism_vardiff_config(Decimal("0.000000001")) - profile = load_prism_highdiff_listener("10.0.0.1", base) - - assert profile is not None - self.assertEqual(profile.name, "highdiff") - self.assertEqual(profile.bind, "10.0.0.1") - self.assertEqual(profile.port, 4334) - self.assertEqual(profile.heartbeat_name, "stratum_accept_highdiff") - self.assertEqual(profile.share_difficulty, Decimal("500000")) - self.assertEqual(profile.minimum_advertised_difficulty, Decimal("500000")) - self.assertEqual(profile.vardiff_config.min_difficulty, Decimal("500000")) - self.assertEqual(profile.vardiff_config.startup_difficulty, Decimal("500000")) - self.assertEqual(profile.vardiff_config.max_difficulty, Decimal("4294967296")) - # Everything but the difficulty bounds is inherited from the base config. - self.assertEqual(profile.vardiff_config.enabled, base.enabled) - self.assertEqual( - profile.vardiff_config.target_share_interval_seconds, - base.target_share_interval_seconds, - ) - self.assertEqual( - profile.vardiff_config.retarget_interval_seconds, - base.retarget_interval_seconds, - ) - self.assertEqual(profile.vardiff_config.max_step_factor, base.max_step_factor) - - def test_highdiff_listener_env_overrides(self) -> None: - with patch.dict(os.environ, {}, clear=False): - clear_stratum_diff_env() - os.environ["PRISM_STRATUM_HIGHDIFF_PORT"] = "4335" - os.environ["PRISM_STRATUM_HIGHDIFF_BIND"] = "127.0.0.2" - os.environ["PRISM_STRATUM_HIGHDIFF_MIN_DIFF"] = "600000" - os.environ["PRISM_STRATUM_HIGHDIFF_START_DIFF"] = "1000000" - os.environ["PRISM_STRATUM_HIGHDIFF_MAX_DIFF"] = "8000000" - os.environ["PRISM_STRATUM_HIGHDIFF_SHARE_DIFF"] = "700000" - base = load_prism_vardiff_config(Decimal("0.000000001")) - profile = load_prism_highdiff_listener("0.0.0.0", base) - - assert profile is not None - self.assertEqual(profile.bind, "127.0.0.2") - self.assertEqual(profile.port, 4335) - self.assertEqual(profile.share_difficulty, Decimal("700000")) - self.assertEqual(profile.minimum_advertised_difficulty, Decimal("600000")) - self.assertEqual(profile.vardiff_config.min_difficulty, Decimal("600000")) - self.assertEqual(profile.vardiff_config.startup_difficulty, Decimal("1000000")) - self.assertEqual(profile.vardiff_config.max_difficulty, Decimal("8000000")) - - def test_highdiff_listener_rejects_inconsistent_bounds(self) -> None: - base = load_prism_vardiff_config(Decimal("0.000000001")) - with patch.dict(os.environ, {}, clear=False): - clear_stratum_diff_env() - os.environ["PRISM_STRATUM_HIGHDIFF_PORT"] = "4334" - os.environ["PRISM_STRATUM_HIGHDIFF_MIN_DIFF"] = "1000000" - with self.assertRaises(SystemExit): - load_prism_highdiff_listener("0.0.0.0", base) - with patch.dict(os.environ, {}, clear=False): - clear_stratum_diff_env() - os.environ["PRISM_STRATUM_HIGHDIFF_PORT"] = "4334" - os.environ["PRISM_STRATUM_HIGHDIFF_MAX_DIFF"] = "400000" - with self.assertRaises(SystemExit): - load_prism_highdiff_listener("0.0.0.0", base) - for bad_port in ("not-a-port", "0", "70000"): - with patch.dict(os.environ, {}, clear=False): - clear_stratum_diff_env() - os.environ["PRISM_STRATUM_HIGHDIFF_PORT"] = bad_port - with self.assertRaises(SystemExit): - load_prism_highdiff_listener("0.0.0.0", base) - - def test_client_startup_difficulty_uses_listener_profile(self) -> None: - server = coordinator() - profile = StratumListenerProfile( - name="highdiff", - bind="0.0.0.0", - port=4334, - share_difficulty=Decimal("700000"), - vardiff_config=highdiff_vardiff_config(), - heartbeat_name="stratum_accept_highdiff", - ) - self.assertEqual(server.client_startup_difficulty(profile), Decimal("500000")) - # Without a profile the default listener behavior is unchanged. - self.assertEqual(server.client_startup_difficulty(), Decimal("0.000000001")) - # With vardiff disabled the listener's fixed share difficulty applies. - fixed_profile = StratumListenerProfile( - name="highdiff", - bind="0.0.0.0", - port=4334, - share_difficulty=Decimal("700000"), - vardiff_config=highdiff_vardiff_config(enabled=False), - heartbeat_name="stratum_accept_highdiff", - ) - self.assertEqual(server.client_startup_difficulty(fixed_profile), Decimal("700000")) - - def test_stratum_accept_heartbeat_names(self) -> None: - server = coordinator() - # Coordinators built without listener profiles (tests, legacy) keep the - # historical single heartbeat name. - self.assertEqual(server.stratum_accept_heartbeat_names(), ("stratum_accept",)) - server.listener_profiles = [ - StratumListenerProfile( - name="default", - bind="0.0.0.0", - port=3340, - share_difficulty=Decimal("1"), - vardiff_config=server.vardiff_config, - heartbeat_name="stratum_accept", - ), - StratumListenerProfile( - name="highdiff", - bind="0.0.0.0", - port=4334, - share_difficulty=Decimal("500000"), - vardiff_config=highdiff_vardiff_config(), - heartbeat_name="stratum_accept_highdiff", - ), - ] - self.assertEqual( - server.stratum_accept_heartbeat_names(), - ("stratum_accept", "stratum_accept_highdiff"), - ) - - def test_parse_stratum_password_options(self) -> None: - self.assertEqual(parse_stratum_password_options(""), (None, None)) - self.assertEqual(parse_stratum_password_options("x"), (None, None)) - self.assertEqual(parse_stratum_password_options("d=8192"), (Decimal("8192"), None)) - self.assertEqual( - parse_stratum_password_options("md=4096,d=8192"), - (Decimal("8192"), Decimal("4096")), - ) - self.assertEqual( - parse_stratum_password_options("D=500000, MD=500000"), - (Decimal("500000"), Decimal("500000")), - ) - self.assertEqual(parse_stratum_password_options("d=abc"), (None, None)) - self.assertEqual(parse_stratum_password_options("d=-5,md=0"), (None, None)) - self.assertEqual(parse_stratum_password_options("foo=1,md=2048"), (None, Decimal("2048"))) - - def test_password_d_below_highdiff_floor_is_clamped(self) -> None: - server = coordinator() - state = client() - state.listener_vardiff_config = highdiff_vardiff_config() - state.share_difficulty = Decimal("500000") - state.requested_difficulty = Decimal("1000") - - target = server.apply_client_difficulty_requests(state) - - self.assertEqual(target, Decimal("500000")) - assert state.vardiff_config is not None - self.assertEqual(state.vardiff_config.min_difficulty, Decimal("500000")) - self.assertEqual(state.vardiff_config.startup_difficulty, Decimal("500000")) - - def test_password_md_raises_personal_floor_and_retarget_respects_it(self) -> None: - server = coordinator() - state = client() - state.requested_min_difficulty = Decimal("256") - state.share_difficulty = Decimal("1") - - target = server.apply_client_difficulty_requests(state) - - self.assertEqual(target, Decimal("256")) - assert state.vardiff_config is not None - self.assertEqual(state.vardiff_config.min_difficulty, Decimal("256")) - self.assertEqual(state.vardiff_config.max_difficulty, Decimal("1024")) - - # A zero-share retarget window wants to step down 4x; the personal - # floor must hold it at 256. - state.share_difficulty = Decimal("256") - server.retarget_client( - state, - current_difficulty=Decimal("256"), - accepted_shares=0, - submitted_shares=0, - accepted_difficulty=Decimal("0"), - elapsed_seconds=Decimal("2"), - ) - self.assertIsNone(state.pending_share_difficulty) - self.assertEqual(state.share_difficulty, Decimal("256")) - - def test_apply_requests_is_stable_across_reapplication(self) -> None: - server = coordinator() - state = client() - state.listener_vardiff_config = highdiff_vardiff_config() - state.requested_min_difficulty = Decimal("600000") - state.requested_difficulty = Decimal("700000") - - first = server.apply_client_difficulty_requests(state) - second = server.apply_client_difficulty_requests(state) - - self.assertEqual(first, Decimal("700000")) - self.assertEqual(second, Decimal("700000")) - assert state.vardiff_config is not None - # Recomputed from the pristine listener config: the floor is the md= - # value, not a compounded one. - self.assertEqual(state.vardiff_config.min_difficulty, Decimal("600000")) - - def test_suggest_difficulty_before_subscribe_applies_directly(self) -> None: - server = coordinator() - state = ClientState(sock=object(), address=("127.0.0.1", 1), connection_id=2, extranonce1_hex="00000002") - sent: list[object] = [] - state.send = lambda payload: sent.append(payload) # type: ignore[method-assign] - - server.handle_suggest_difficulty(state, 7, [512]) - - self.assertEqual(state.suggested_difficulty, Decimal("512")) - self.assertEqual(state.share_difficulty, Decimal("512")) - self.assertIsNone(state.pending_share_difficulty) - self.assertEqual(sent, [{"id": 7, "result": True, "error": None}]) - - def test_suggest_difficulty_post_authorize_advertises_with_job(self) -> None: - server = coordinator() - state = client() - state.share_difficulty = Decimal("1") - sent: list[object] = [] - state.send = lambda payload: sent.append(payload) # type: ignore[method-assign] - jobs: dict[str, object] = {"count": 0} - - def fake_send_job(client: object, clean_jobs: bool) -> bool: - jobs.update({"count": jobs["count"] + 1, "clean": clean_jobs}) - return True - - server.maybe_send_job = fake_send_job # type: ignore[method-assign] - - server.handle_suggest_difficulty(state, 8, [512]) - - self.assertEqual(state.pending_share_difficulty, Decimal("512")) - self.assertEqual(jobs["count"], 1) - self.assertTrue(jobs["clean"]) - self.assertEqual(sent, [{"id": 8, "result": True, "error": None}]) - - def test_suggest_difficulty_rolls_back_pending_on_build_failure(self) -> None: - server = coordinator() - state = client() - state.share_difficulty = Decimal("1") - state.difficulty_generation = 7 - state.send = lambda payload: None # type: ignore[method-assign] - server.maybe_send_job = lambda client, *, clean_jobs: False # type: ignore[method-assign] - - server.handle_suggest_difficulty(state, 9, [512]) - - self.assertIsNone(state.pending_share_difficulty) - self.assertEqual(state.share_difficulty, Decimal("1")) - self.assertEqual(state.difficulty_generation, 7) - - def test_suggest_difficulty_yields_to_password_d_option(self) -> None: - server = coordinator() - state = client() - state.requested_difficulty = Decimal("512") - state.share_difficulty = Decimal("512") - state.send = lambda payload: None # type: ignore[method-assign] - server.maybe_send_job = lambda client, *, clean_jobs: True # type: ignore[method-assign] - - server.handle_suggest_difficulty(state, 10, [128]) - - # d= wins: the suggestion is recorded but the resolved target stays at - # the explicit password difficulty, so nothing is re-advertised. - self.assertEqual(state.suggested_difficulty, Decimal("128")) - self.assertEqual(state.share_difficulty, Decimal("512")) - self.assertIsNone(state.pending_share_difficulty) - - def test_suggest_difficulty_ignores_junk_values(self) -> None: - server = coordinator() - state = client() - state.share_difficulty = Decimal("1") - sent: list[object] = [] - state.send = lambda payload: sent.append(payload) # type: ignore[method-assign] - - for junk in ([], ["nan"], ["-4"], ["0"], [None]): - server.handle_suggest_difficulty(state, 11, junk) # type: ignore[arg-type] - - self.assertIsNone(state.vardiff_config) - self.assertEqual(state.share_difficulty, Decimal("1")) - self.assertEqual(len(sent), 5) - - def test_highdiff_share_diff_tracks_start_and_validates_bounds(self) -> None: - base = load_prism_vardiff_config(Decimal("0.000000001")) - with patch.dict(os.environ, {}, clear=False): - clear_stratum_diff_env() - os.environ["PRISM_STRATUM_HIGHDIFF_PORT"] = "4334" - os.environ["PRISM_STRATUM_HIGHDIFF_MIN_DIFF"] = "1000000" - os.environ["PRISM_STRATUM_HIGHDIFF_START_DIFF"] = "1000000" - profile = load_prism_highdiff_listener("0.0.0.0", base) - assert profile is not None - # Unset fixed difficulty tracks the start difficulty instead of a - # constant that could fall below a raised floor. - self.assertEqual(profile.share_difficulty, Decimal("1000000")) - - # An explicit fixed difficulty outside the listener bounds must fail - # startup: advertising below the floor is exactly what the high-diff - # listener exists to prevent. - with patch.dict(os.environ, {}, clear=False): - clear_stratum_diff_env() - os.environ["PRISM_STRATUM_HIGHDIFF_PORT"] = "4334" - os.environ["PRISM_STRATUM_HIGHDIFF_SHARE_DIFF"] = "1000" - with self.assertRaises(SystemExit): - load_prism_highdiff_listener("0.0.0.0", base) - with patch.dict(os.environ, {}, clear=False): - clear_stratum_diff_env() - os.environ["PRISM_STRATUM_HIGHDIFF_PORT"] = "4334" - os.environ["PRISM_STRATUM_HIGHDIFF_SHARE_DIFF"] = "8589934592" - with self.assertRaises(SystemExit): - load_prism_highdiff_listener("0.0.0.0", base) - - def authorize_server_and_client(self) -> tuple[PrismCoordinator, ClientState, list[object]]: - server = coordinator() - server.rpc = AddressValidationRpc() - server.username_fallback_address = None - server.maybe_send_job = lambda client, *, clean_jobs: True # type: ignore[method-assign] - state = ClientState(sock=object(), address=("127.0.0.1", 1), connection_id=3, extranonce1_hex="00000003") - state.subscribed = True - sent: list[object] = [] - state.send = lambda payload: sent.append(payload) # type: ignore[method-assign] - return server, state, sent - - def test_authorize_password_applies_before_first_job(self) -> None: - server, state, sent = self.authorize_server_and_client() + with patch("builtins.print"): + self.assertFalse( + server.submit_next_block_candidate(defer_accounting=True) + ) + self.assertIs(getattr(server, "_retry_block_candidate", None), candidate) - server.handle_request( - state, - {"id": 5, "method": "mining.authorize", "params": [PAYOUT_ADDRESS, "d=0.5,md=0.25"]}, - ) + # Once the deadline passes the same candidate dequeues again. + with server.lock: + server._block_candidate_retry_not_before[block_hash] = ( + time.monotonic() - 1.0 + ) + with patch("builtins.print"): + self.assertTrue( + server.submit_next_block_candidate(defer_accounting=True) + ) - self.assertTrue(state.authorized) - self.assertEqual(state.requested_difficulty, Decimal("0.5")) - self.assertEqual(state.requested_min_difficulty, Decimal("0.25")) - # Applied directly (no job exists yet), so the first - # set_difficulty/notify pair advertises the requested value. - self.assertEqual(state.share_difficulty, Decimal("0.5")) - self.assertIsNone(state.pending_share_difficulty) - assert state.vardiff_config is not None - self.assertEqual(state.vardiff_config.min_difficulty, Decimal("0.25")) - self.assertEqual(sent, [{"id": 5, "result": True, "error": None}]) + def test_live_retry_reuses_definitive_node_acceptance_without_reoffer(self) -> None: + parent_hash = "00" * 32 + block_hash = "ce" * 32 + moved_tip = "11" * 32 + ledger = SingleWriterShareLedger() + server, state, _recording = submit_coordinator(tip=parent_hash) + server.ledger = ledger + server.stop_after_block = False + server.max_blocks = 10 + server.block_candidate_retry_initial_seconds = 0.0 + server.block_candidate_retry_max_seconds = 0.0 + from lab.prism.share_ledger import PendingShare - def test_reauthorize_with_plain_password_clears_stale_overrides(self) -> None: - server, state, _ = self.authorize_server_and_client() - server.handle_request( - state, - {"id": 5, "method": "mining.authorize", "params": [PAYOUT_ADDRESS, "d=0.5,md=0.25"]}, + pending = PendingShare( + share_id="miner-a:live-retry-duplicate", + miner_id="miner-a", + order_key="miner-a", + p2mr_program_hex="11" * 32, + share_difficulty=1, + network_difficulty=1, + template_height=10, + job_id="job-1", + job_issued_at_ms=1, + accepted_at_ms=2, + ntime=1_700_000_000, ) - assert state.vardiff_config is not None - - server.handle_request( + candidate = block_candidate( + server, state, - {"id": 6, "method": "mining.authorize", "params": [PAYOUT_ADDRESS, "x"]}, + SimpleNamespace( + coinbase_tx_hex="c0ffee", + block_hash_hex=block_hash, + block_hex="00", + share_pass=True, + block_pass=True, + ), + pending_share=pending, ) + intent = server.block_candidate_intent(candidate) + ledger.append_batch([(pending, intent)]) - # The new password carries no options: prior overrides are dropped and - # the client falls back to the pristine listener policy (its current - # difficulty is left alone; vardiff drifts it under listener bounds). - self.assertIsNone(state.requested_difficulty) - self.assertIsNone(state.requested_min_difficulty) - self.assertIsNone(state.vardiff_config) - self.assertEqual(state.share_difficulty, Decimal("0.5")) - - def test_reauthorize_with_new_difficulty_sends_single_job_pair(self) -> None: - server, state, _ = self.authorize_server_and_client() - send_job_calls: list[bool] = [] - - def counting_send_job(current: ClientState, *, clean_jobs: bool) -> bool: - send_job_calls.append(clean_jobs) - current.active_job = SimpleNamespace( - template={"previousblockhash": "aa" * 32}, - payout_state_generation=0, - ) - server.current_tip_first_seen = ("aa" * 32, None) - return True - - server.maybe_send_job = counting_send_job # type: ignore[method-assign] + expected_height = int(candidate.context.template["height"]) + header_state = {"confirmations": -1} - server.handle_request( - state, - {"id": 5, "method": "mining.authorize", "params": [PAYOUT_ADDRESS, "x"]}, - ) - self.assertEqual(len(send_job_calls), 1) + class MovedTipRpc(FakeRpc): + def __init__(self) -> None: + self.submit_results: list[object] = [] - # A re-authorize whose new d= advertises a fresh difficulty/job pair - # must not be followed by a second back-to-back pair. - server.handle_request( - state, - {"id": 6, "method": "mining.authorize", "params": [PAYOUT_ADDRESS, "d=0.5"]}, - ) - self.assertEqual(len(send_job_calls), 2) - self.assertEqual(state.pending_share_difficulty, Decimal("0.5")) + def call( + self, + method: str, + params: list[object] | None = None, + *, + timeout: float | None = None, + ) -> object: + if method == "submitblock": + result = None if not self.submit_results else "duplicate" + self.submit_results.append(result) + return result + if method == "getbestblockhash": + # The chain advances as soon as the first offer lands, so + # the in-process retry observes a moved tip. + return parent_hash if not self.submit_results else moved_tip + if method == "getblockhash": + return block_hash + if method == "getblockcount": + return 9 + if method == "getblockheader": + # Unprovable during the tip race (found, not yet active), + # then settled and active for the final pass. + return { + "height": expected_height, + "confirmations": header_state["confirmations"], + } + return super().call(method, params) - def test_authorize_rejects_and_disconnects_above_username_connection_limit(self) -> None: - server, first, _ = self.authorize_server_and_client() - server.stratum_max_connections_per_username = 1 - second = ClientState( - sock=object(), - address=("127.0.0.1", 2), - connection_id=4, - extranonce1_hex="00000004", - ) - second.send = lambda payload: None # type: ignore[method-assign] - server.clients.update({first, second}) + rpc = MovedTipRpc() + server.rpc = rpc + original_mark = ledger.mark_block_candidate_attempted + mark_state = {"failures": 0} - server.handle_request( - first, - {"id": 5, "method": "mining.authorize", "params": [PAYOUT_ADDRESS, "x"]}, - ) - with self.assertRaises(StratumError) as raised: - server.handle_request( - second, - {"id": 6, "method": "mining.authorize", "params": [PAYOUT_ADDRESS, "x"]}, - ) - - self.assertTrue(raised.exception.disconnect) - self.assertEqual(raised.exception.message, "too many connections for username") - self.assertEqual(server.connection_limit_rejection_counts["username"], 1) - self.assertFalse(second.authorized) - - def test_reauthorize_limit_error_preserves_live_session(self) -> None: - server, live, _ = self.authorize_server_and_client() - server.stratum_max_connections_per_username = 1 - occupant = ClientState( - sock=object(), - address=("127.0.0.1", 2), - connection_id=4, - extranonce1_hex="00000004", - ) - occupant.send = lambda payload: None # type: ignore[method-assign] - server.clients.update({live, occupant}) - - server.handle_request( - live, - { - "id": 5, - "method": "mining.authorize", - "params": [f"{PAYOUT_ADDRESS}.original", "x"], - }, - ) - server.handle_request( - occupant, - { - "id": 6, - "method": "mining.authorize", - "params": [f"{PAYOUT_ADDRESS}.full", "x"], - }, - ) - original_worker = live.worker + def flaky_mark(*, block_hash: str) -> bool: + # Fail twice, starting at the first attempt marker after the + # fresh node offer, mirroring statement timeouts under sustained + # saturation. The retained acceptance must survive the failed + # retry too, not just the first failure. + if len(rpc.submit_results) == 1 and mark_state["failures"] < 2: + mark_state["failures"] += 1 + raise RuntimeError("attempt marker timed out") + return original_mark(block_hash=block_hash) - with self.assertRaises(StratumError) as raised: - server.handle_request( - live, - { - "id": 7, - "method": "mining.authorize", - "params": [f"{PAYOUT_ADDRESS}.full", "x"], - }, + ledger.mark_block_candidate_attempted = flaky_mark # type: ignore[method-assign] + server.enqueue_block_candidate(candidate) + with tempfile.TemporaryDirectory() as tempdir: + server.audit_dir = Path(tempdir) + server.evidence_path = Path(tempdir) / "evidence.json" + server.ledger_writer_public_key_hex = "aa" * 32 + server.build_audit_bundle = ( # type: ignore[method-assign] + lambda **_kwargs: verified_block_bundle() ) - - self.assertFalse(raised.exception.disconnect) - self.assertTrue(live.authorized) - self.assertIs(live.worker, original_worker) - self.assertEqual(live.username, f"{PAYOUT_ADDRESS}.original") - - def test_username_connection_limit_is_disabled_by_default(self) -> None: - server = coordinator() - first = client() - second = client() - server.clients.update({first, second}) - worker = WorkerIdentity( - username=PAYOUT_ADDRESS, - payout_address=PAYOUT_ADDRESS, - worker_name=None, - script_pubkey_hex="5220" + "11" * 32, - p2mr_program_hex="11" * 32, + server.verify_bundle = ( # type: ignore[method-assign] + lambda *_args, **_kwargs: verified_audit_report() + ) + with patch("builtins.print"): + # First pass: fresh accept from the node, then the attempt + # marker fails and the candidate is retained for an + # in-process retry carrying the definitive node result. + self.assertTrue(server.submit_next_block_candidate()) + self.assertEqual(rpc.submit_results, [None]) + self.assertEqual( + ledger._block_candidate_outbox[block_hash]["state"], + "pending", + ) + # Retry 1 fails the marker again: the retained acceptance + # must survive a consumed-and-failed retry, not just the + # first failure. + self.assertTrue(server.submit_next_block_candidate()) + self.assertEqual(rpc.submit_results, [None]) + self.assertEqual( + ledger._block_candidate_outbox[block_hash]["state"], + "pending", + ) + # Retry 2: the stashed acceptance is still reused, so the + # node is never asked again (no "duplicate" classification, + # no chain probe reliance) and the landing tail finalizes + # exactly as if the first pass had continued. + self.assertTrue(server.submit_next_block_candidate()) + self.assertEqual(rpc.submit_results, [None]) + self.assertEqual( + ledger._block_candidate_outbox[block_hash]["state"], + "submitted", ) - - self.assertTrue(server.reserve_client_username(first, worker)) - self.assertTrue(server.reserve_client_username(second, worker)) - self.assertEqual(server.connection_limit_rejection_counts["username"], 0) - - def test_username_limit_does_not_count_idle_clients_for_empty_username(self) -> None: - server = coordinator() - server.stratum_max_connections_per_username = 1 - idle = client() - first = client() - second = client() - server.clients.update({idle, first, second}) - worker = WorkerIdentity( - username="", - payout_address=PAYOUT_ADDRESS, - worker_name=None, - script_pubkey_hex="5220" + "11" * 32, - p2mr_program_hex="11" * 32, + self.assertNotIn( + PRISM_REJECTION_STALE_JOB, + getattr(server, "block_candidate_abandoned_counts", {}), ) + self.assertEqual(server.accepted_block_count, 1) - self.assertTrue(server.reserve_client_username(first, worker)) - self.assertFalse(server.reserve_client_username(second, worker)) - self.assertEqual(server.connection_limit_rejection_counts["username"], 1) + def test_block_accounting_primary_queue_is_bounded_by_default(self) -> None: + server, _state, _recording = submit_coordinator() + server._ensure_block_accounting_state() + # A bounded primary is what makes the documented result-preserving + # spillover ordering reachable; the overflow queue stays unbounded. + self.assertEqual(server._block_accounting_queue.maxsize, 8) + self.assertEqual(server._block_accounting_overflow_queue.maxsize, 0) - def test_accept_loop_rejects_above_global_connection_limit(self) -> None: - server = coordinator() - server.stratum_max_connections = 1 - server.clients.add(client()) + def test_accounting_saturation_does_not_convoy_node_offers(self) -> None: + server, state, _recording = submit_coordinator() + server.max_blocks = 10 + server.stop_after_block = False + server.block_candidate_retry_initial_seconds = 0.01 + server._block_accounting_queue = queue.PriorityQueue(maxsize=1) + entered_accounting = threading.Event() + release_accounting = threading.Event() + submitted: list[str] = [] - class AcceptedSocket: - def __init__(self) -> None: - self.closed = False + class RecordingRpc(TipRpc): + def call( + self, + method: str, + params: list[object] | None = None, + *, + timeout: float | None = None, + ) -> object: + if method == "submitblock": + submitted.append(str((params or [""])[0])) + return None + return super().call(method, params) - def close(self) -> None: - self.closed = True + server.rpc = RecordingRpc("00" * 32) - accepted = AcceptedSocket() + def blocked_accounting( + _candidate: PrismBlockCandidate, + **_kwargs: object, + ) -> bool: + entered_accounting.set() + release_accounting.wait(5) + return True - class OneConnectionListener: - def __init__(self) -> None: - self.calls = 0 + server._call_block_candidate_writer = blocked_accounting # type: ignore[method-assign] + for tag in ("a1", "b2", "c3", "d4"): + candidate = block_candidate( + server, + state, + SimpleNamespace( + coinbase_tx_hex="00", + block_hash_hex=tag * 32, + block_hex=tag, + share_pass=True, + block_pass=True, + ), + ) + server.enqueue_block_candidate(candidate) - def accept(self) -> tuple[object, tuple[str, int]]: - self.calls += 1 - if self.calls == 1: - return accepted, ("127.0.0.1", 1000) + submitter = threading.Thread(target=server.block_submit_loop) + with patch("builtins.print"): + submitter.start() + try: + self.assertTrue(entered_accounting.wait(1)) + deadline = time.monotonic() + 2 + while len(submitted) < 4 and time.monotonic() < deadline: + time.sleep(0.01) + self.assertEqual(submitted, ["a1", "b2", "c3", "d4"]) + finally: server.stop_event.set() - raise socket.timeout() + release_accounting.set() + submitter.join(2) + accounting = getattr(server, "_block_accounting_thread", None) + if accounting is not None: + accounting.join(2) + self.assertFalse(submitter.is_alive()) - profile = StratumListenerProfile( - name="default", - bind="127.0.0.1", - port=3340, - share_difficulty=server.share_difficulty, - vardiff_config=server.vardiff_config, - heartbeat_name="stratum_accept", - ) + def test_replay_batch_reaches_node_while_oldest_accounting_stalls(self) -> None: + server, state, _recording = submit_coordinator() + ledger = SingleWriterShareLedger() + server.ledger = ledger + server.max_blocks = 10 + server.stop_after_block = False + submitted: list[str] = [] + release_accounting = threading.Event() + entered_accounting = threading.Event() + + for index, tag in enumerate(("a5", "b6"), start=1): + candidate = block_candidate( + server, + state, + SimpleNamespace( + coinbase_tx_hex="00", + block_hash_hex=tag * 32, + block_hex=tag, + share_pass=True, + block_pass=True, + ), + ) + pending = PendingShare( + share_id=f"miner-a:{tag * 32}", + miner_id="miner-a", + order_key="miner-a", + p2mr_program_hex="11" * 32, + share_difficulty=1, + network_difficulty=1, + template_height=9, + job_id="job-1", + job_issued_at_ms=1, + accepted_at_ms=index, + ntime=1, + ) + candidate = dataclass_replace(candidate, pending_share=pending) + ledger.append_batch( + [(pending, server.block_candidate_intent(candidate))] + ) - server.accept_loop(OneConnectionListener(), profile) # type: ignore[arg-type] + self.assertEqual(server.replay_pending_block_candidates(), 2) - self.assertTrue(accepted.closed) - self.assertEqual(server.connection_limit_rejection_counts["global"], 1) + class RecordingRpc(TipRpc): + def call( + self, + method: str, + params: list[object] | None = None, + *, + timeout: float | None = None, + ) -> object: + if method == "submitblock": + submitted.append(str((params or [""])[0])) + return None + return super().call(method, params) - def test_accept_loop_recovers_from_descriptor_exhaustion(self) -> None: - server = coordinator() - server.stratum_accept_resource_exhaustion_backoff_seconds = 0 + server.rpc = RecordingRpc("00" * 32) - class ExhaustedListener: - def __init__(self) -> None: - self.calls = 0 + def blocked_accounting( + _candidate: PrismBlockCandidate, + **_kwargs: object, + ) -> bool: + entered_accounting.set() + release_accounting.wait(5) + return True - def accept(self) -> tuple[object, tuple[str, int]]: - self.calls += 1 - if self.calls == 1: - raise OSError(errno.EMFILE, "too many open files") + server._call_block_candidate_writer = blocked_accounting # type: ignore[method-assign] + submitter = threading.Thread(target=server.block_submit_loop) + with patch("builtins.print"): + submitter.start() + try: + self.assertTrue(entered_accounting.wait(1)) + deadline = time.monotonic() + 2 + while len(submitted) < 2 and time.monotonic() < deadline: + time.sleep(0.01) + self.assertEqual(submitted, ["a5", "b6"]) + finally: server.stop_event.set() - raise socket.timeout() + release_accounting.set() + submitter.join(2) + accounting = getattr(server, "_block_accounting_thread", None) + if accounting is not None: + accounting.join(2) - listener = ExhaustedListener() - profile = StratumListenerProfile( - name="default", - bind="127.0.0.1", - port=3340, - share_difficulty=server.share_difficulty, - vardiff_config=server.vardiff_config, - heartbeat_name="stratum_accept", + def test_startup_block_replay_timeout_starts_submitter_and_converges(self) -> None: + server, state, _recording = submit_coordinator() + server.max_blocks = 10 + server.stop_after_block = False + server.block_submit_db_timeout_seconds = 0.01 + # The startup enumeration is landing-class (issue #188 fix 4). + server.block_landing_db_timeout_seconds = 0.01 + server.block_landing_db_timeout_max_seconds = 0.01 + server.block_candidate_retry_initial_seconds = 0.01 + ledger = SingleWriterShareLedger() + server.ledger = ledger + block_hash = "e5" * 32 + pending = PendingShare( + share_id=f"miner-a:{block_hash}", + miner_id="miner-a", + order_key="miner-a", + p2mr_program_hex="11" * 32, + share_difficulty=1, + network_difficulty=1, + template_height=9, + job_id="job-1", + job_issued_at_ms=1, + accepted_at_ms=1, + ntime=1, + ) + candidate = block_candidate( + server, + state, + SimpleNamespace( + coinbase_tx_hex="00", + block_hash_hex=block_hash, + block_hex="e5", + share_pass=True, + block_pass=True, + ), + pending_share=pending, + ) + ledger.append_batch( + [(pending, server.block_candidate_intent(candidate))] ) - server.accept_loop(listener, profile) # type: ignore[arg-type] - - self.assertEqual(listener.calls, 2) - self.assertEqual(server.accept_resource_exhaustion_count, 1) - - def test_resource_backoff_keeps_accept_watchdog_heartbeat_fresh(self) -> None: - server = coordinator() - server.stratum_accept_resource_exhaustion_backoff_seconds = 0.03 - server.watchdog_timeout_seconds = 0.01 - - server._wait_after_stratum_resource_failure("stratum_accept") + query_started = threading.Event() + release_query = threading.Event() + query_calls = 0 + startup_call: list[object] = [] + original_pending_rows = ledger.pending_block_candidate_rows - self.assertFalse(server._overdue_heartbeats(time.monotonic())) + def slow_pending_rows(*, limit: int = 32) -> list[dict[str, object]]: + nonlocal query_calls + query_calls += 1 + if query_calls == 1: + with server._block_submitter_ledger_calls_lock: + startup_call.append( + server._block_submitter_ledger_calls[ + ("replay-outbox-query", 32) + ] + ) + query_started.set() + if not release_query.wait(5): + raise AssertionError("timed out waiting to release outbox query") + return original_pending_rows(limit=limit) - def test_accept_loop_recovers_when_handler_thread_cannot_start(self) -> None: - server = coordinator() - server.connection_counter = 0 - server.stratum_send_timeout_seconds = 0 - server.stratum_accept_resource_exhaustion_backoff_seconds = 0 + ledger.pending_block_candidate_rows = slow_pending_rows # type: ignore[method-assign] + original_record_wait = server._record_block_submitter_wait + loop_reuse_waiting = threading.Event() - class AcceptedSocket: - def __init__(self) -> None: - self.closed = False + def observed_record_wait(phase: str) -> None: + original_record_wait(phase) + if ( + phase == "replay-outbox-query" + and getattr(server, "_block_submitter_thread_ident", None) + == threading.get_ident() + ): + loop_reuse_waiting.set() - def settimeout(self, timeout: object) -> None: - pass + server._record_block_submitter_wait = observed_record_wait # type: ignore[method-assign] - def shutdown(self, how: int) -> None: - pass + submitted: list[str] = [] - def close(self) -> None: - self.closed = True + class RecordingRpc(TipRpc): + def call( + self, + method: str, + params: list[object] | None = None, + *, + timeout: float | None = None, + ) -> object: + if method == "getblockcount": + return 9 + if method == "submitblock": + submitted.append(str((params or [""])[0])) + return None + return super().call(method, params) - accepted = AcceptedSocket() + server.rpc = RecordingRpc("00" * 32) + accounted: list[str] = [] - class OneConnectionListener: - def __init__(self) -> None: - self.calls = 0 + def account_candidate( + replayed: PrismBlockCandidate, + **_kwargs: object, + ) -> bool: + replayed_hash = replayed.submission.block_hash_hex + accounted.append(replayed_hash) + ledger.mark_block_candidate_submitted(block_hash=replayed_hash) + server.stop_event.set() + return True - def accept(self) -> tuple[object, tuple[str, int]]: - self.calls += 1 - if self.calls == 1: - return accepted, ("127.0.0.1", 1000) - server.stop_event.set() - raise socket.timeout() + server._call_block_candidate_writer = account_candidate # type: ignore[method-assign] - listener = OneConnectionListener() - profile = StratumListenerProfile( - name="default", - bind="127.0.0.1", - port=3340, - share_difficulty=server.share_difficulty, - vardiff_config=server.vardiff_config, - heartbeat_name="stratum_accept", + profile = SimpleNamespace(heartbeat_name="stratum_accept_default") + listener_closed = threading.Event() + listener = SimpleNamespace(close=listener_closed.set) + server.listener_profiles = [profile] + server.bind = "127.0.0.1" + server.port = 0 + server.min_ready_miners = 1 + server.audit_bind = None + server.audit_port = 0 + server.hot_path_log_enabled = False + server.blockwait_enabled = False + server.vardiff_idle_sweep_seconds = 0.0 + server.stratum_initial_job_timeout_seconds = 0.0 + server.watchdog_enabled = False + server.watchdog_interval_seconds = 0.1 + server.template_refresh_failure_exit_seconds = 120.0 + server.coordination_blocked_exit_seconds = 120.0 + server.open_stratum_listeners = ( # type: ignore[method-assign] + lambda _stack: [(listener, profile)] ) + server.validate_live_chain_identity = lambda: None # type: ignore[method-assign] + server.validate_live_template_and_fee_policy = lambda: None # type: ignore[method-assign] + server.prism_payout_policy = lambda: {} # type: ignore[method-assign] + server.prewarm_startup_jobs = lambda: None # type: ignore[method-assign] + server.watchdog_loop = lambda: None # type: ignore[method-assign] + server.blockpoll_loop = lambda: None # type: ignore[method-assign] + server.replay_recovered_shares = lambda: 0 # type: ignore[method-assign] + server.share_append_loop = lambda: None # type: ignore[method-assign] + server.shutdown = lambda *, reason="graceful": True # type: ignore[method-assign] - with patch.object(threading.Thread, "start", side_effect=RuntimeError("can't start new thread")): - server.accept_loop(listener, profile) # type: ignore[arg-type] - - self.assertEqual(listener.calls, 2) - self.assertTrue(accepted.closed) - self.assertFalse(server.clients) - self.assertEqual(server.connection_setup_failure_count, 1) - - def test_handle_client_cleans_up_when_makefile_hits_descriptor_limit(self) -> None: - server = coordinator() - - class MakefileFailureSocket: - def __init__(self) -> None: - self.closed = False - - def makefile(self, *args: object, **kwargs: object) -> object: - raise OSError(errno.EMFILE, "too many open files") - - def shutdown(self, how: int) -> None: - pass + def drain_threads(threads: list[tuple[threading.Thread, float]]) -> None: + for thread, timeout in threads: + thread.join(timeout) - def close(self) -> None: - self.closed = True + server.drain_non_writer_components = drain_threads # type: ignore[method-assign] - sock = MakefileFailureSocket() - state = ClientState( - sock=sock, # type: ignore[arg-type] - address=("127.0.0.1", 1000), - connection_id=1, - extranonce1_hex="00000001", - ) - server.clients.add(state) + def accept_after_startup(_listener: object, _profile: object) -> None: + self.assertTrue(loop_reuse_waiting.wait(2)) + self.assertIsNotNone( + getattr(server, "_block_submitter_thread_ident", None) + ) + self.assertTrue(query_started.wait(1)) + # The loop is waiting on the exact startup call while that worker + # remains blocked; no replacement query was spawned. + with server._block_submitter_ledger_calls_lock: + self.assertIs( + server._block_submitter_ledger_calls.get( + ("replay-outbox-query", 32) + ), + startup_call[0], + ) + self.assertEqual(query_calls, 1) + release_query.set() + deadline = time.monotonic() + 2 + while not accounted and time.monotonic() < deadline: + time.sleep(0.01) + self.assertTrue(accounted) - server.handle_client(state) + server.accept_loop = accept_after_startup # type: ignore[method-assign] - self.assertTrue(sock.closed) - self.assertNotIn(state, server.clients) - self.assertEqual(server.accept_resource_exhaustion_count, 1) + try: + with patch("builtins.print") as printed: + server._serve_with_listener_stack(SimpleNamespace()) # type: ignore[arg-type] + finally: + server.stop_event.set() + release_query.set() - def test_accept_loop_assigns_listener_profiles_and_unique_extranonce(self) -> None: - server = coordinator() - server.connection_counter = 0 - server.stratum_send_timeout_seconds = 0.0 - - def listening_socket() -> socket.socket: - listener = socket.socket(socket.AF_INET, socket.SOCK_STREAM) - listener.bind(("127.0.0.1", 0)) - listener.listen() - listener.settimeout(0.1) - return listener - - default_listener = listening_socket() - highdiff_listener = listening_socket() - default_profile = StratumListenerProfile( - name="default", - bind="127.0.0.1", - port=default_listener.getsockname()[1], - share_difficulty=server.share_difficulty, - vardiff_config=server.vardiff_config, - heartbeat_name="stratum_accept", - ) - highdiff_profile = StratumListenerProfile( - name="highdiff", - bind="127.0.0.1", - port=highdiff_listener.getsockname()[1], - share_difficulty=Decimal("500000"), - vardiff_config=highdiff_vardiff_config(), - heartbeat_name="stratum_accept_highdiff", - minimum_advertised_difficulty=Decimal("500000"), - ) - threads = [ - threading.Thread(target=server.accept_loop, args=(default_listener, default_profile), daemon=True), - threading.Thread(target=server.accept_loop, args=(highdiff_listener, highdiff_profile), daemon=True), + ledger.pending_block_candidate_rows = original_pending_rows # type: ignore[method-assign] + startup_logs = [ + " ".join(str(value) for value in call.args) + for call in printed.call_args_list ] - for thread in threads: - thread.start() - connections = [] - try: - connections.append(socket.create_connection(("127.0.0.1", default_profile.port), timeout=5)) - connections.append(socket.create_connection(("127.0.0.1", highdiff_profile.port), timeout=5)) - deadline = time.monotonic() + 5 - while time.monotonic() < deadline: - with server.lock: - if len(server.clients) == 2: - break - time.sleep(0.01) - with server.lock: - clients_by_listener = {c.listener_name: c for c in server.clients} - self.assertEqual(set(clients_by_listener), {"default", "highdiff"}) - self.assertEqual( - clients_by_listener["default"].share_difficulty, - Decimal("0.000000001"), - ) - self.assertEqual( - clients_by_listener["highdiff"].share_difficulty, - Decimal("500000"), + self.assertTrue( + any( + "prism coordinator: startup block candidate replay timed out " + "phase=replay-outbox-query timeout=0.01s" in message + for message in startup_logs ) - self.assertIs( - clients_by_listener["highdiff"].listener_vardiff_config, - highdiff_profile.vardiff_config, + ) + self.assertTrue(query_started.is_set()) + self.assertTrue(listener_closed.is_set()) + self.assertEqual(submitted, ["e5"]) + self.assertEqual(accounted, [block_hash]) + self.assertEqual(ledger.pending_block_candidates(), []) + + def _pending_outbox_backlog( + self, + server: PrismCoordinator, + state: ClientState, + ledger: SingleWriterShareLedger, + count: int, + ) -> list[str]: + """Persist ``count`` pending outbox rows and return their hashes.""" + hashes: list[str] = [] + for index in range(1, count + 1): + block_hash = f"{index:064x}" + hashes.append(block_hash) + pending = PendingShare( + share_id=f"miner-a:{block_hash}", + miner_id="miner-a", + order_key="miner-a", + p2mr_program_hex="11" * 32, + share_difficulty=1, + network_difficulty=1, + template_height=9, + job_id="job-1", + job_issued_at_ms=1, + accepted_at_ms=index, + ntime=1, ) - self.assertEqual( - clients_by_listener["highdiff"].minimum_advertised_difficulty, - Decimal("500000"), + candidate = block_candidate( + server, + state, + SimpleNamespace( + coinbase_tx_hex="00", + block_hash_hex=block_hash, + block_hex="00", + share_pass=True, + block_pass=True, + ), + pending_share=pending, ) - self.assertEqual( - clients_by_listener["default"].minimum_advertised_difficulty, - Decimal("0"), + ledger.append_batch( + [(pending, server.block_candidate_intent(candidate))] ) - extranonces = {c.extranonce1_hex for c in clients_by_listener.values()} - self.assertEqual(extranonces, {"00000001", "00000002"}) - self.assertIn("stratum_accept", server._heartbeats) - self.assertIn("stratum_accept_highdiff", server._heartbeats) - finally: - server.stop_event.set() - for connection in connections: - connection.close() - default_listener.close() - highdiff_listener.close() - for thread in threads: - thread.join(timeout=5) - - -class PrismStampedJobFloorTests(unittest.TestCase): - """The listener floor must hold on the wire, not just in vardiff policy. - - Stamped jobs are the single choke point for every mining.set_difficulty - the coordinator sends, and marketplace verification judges the first one. - The regression here is a young chain: qbit network difficulty below the - high-diff floor used to drag the advertised difficulty down with it. - """ + return hashes - def stamp_coordinator(self) -> PrismCoordinator: - server = coordinator() - server.job_counter = 0 - server.share_weights_by_username = {} - server.default_share_weight = 1 - return server + def test_configured_single_candidate_pages_preserve_complete_startup_replay(self) -> None: + server, state, _recording = submit_coordinator() + ledger = SingleWriterShareLedger() + server.ledger = ledger + server.config = SimpleNamespace(block=SimpleNamespace(replay_page_size=1)) + hashes = self._pending_outbox_backlog(server, state, ledger, 2) + original_rows = ledger.pending_block_candidate_rows + queries = [] + + def recording_rows(*, limit=32, after_cursor=None): + # Even after adopting the first parent, later accepted work must + # remain discoverable before the job-build gate can open. + self.assertTrue(server._block_replay_enumeration_owed()) + rows = original_rows(limit=limit, after_cursor=after_cursor) + queries.append((limit, after_cursor, len(rows))) + return rows + + ledger.pending_block_candidate_rows = recording_rows + server._note_block_replay_enumeration_owed() + with patch("builtins.print"): + self.assertEqual(server.replay_pending_block_candidates(), 2) - def cached_bundle(self) -> CachedJobBundle: - # bits 207fffff: regtest-grade network difficulty (~4.7e-10), far - # below the 500k marketplace floor. - qbit_target = target_from_compact("207fffff") - base_job = direct_stratum.DirectQbitStratumJob( - job_id="prism-template-base", - previousblockhash_display="00" * 32, - prevhash="00" * 32, - coinb1="", - coinb2="", - full_coinbase_prefix="", - full_coinbase_suffix="", - merkle_branch=(), - transaction_hexes=(), - version="20000000", - nbits="207fffff", - ntime="6553f100", - qbit_target=qbit_target, - share_target=qbit_target, - share_difficulty=Decimal("1"), - extranonce1_hex="ffffffff", - extranonce2_size=8, - clean_jobs=True, - ) - return CachedJobBundle( - key=("test",), - template=gbt_template("00" * 32), - template_fingerprint="fp", - coinbase_manifest={}, - shares_json=[], - prior_balances=[], - found_block={"network_difficulty": 1}, - collection_only=False, - issued_at_ms=12345, - base_job=base_job, - built_monotonic=time.monotonic(), + self.assertEqual([query[0] for query in queries], [1, 1, 1]) + self.assertEqual([query[2] for query in queries], [1, 1, 0]) + self.assertIsNone(queries[0][1]) + self.assertNotEqual(queries[1][1], queries[2][1]) + self.assertEqual( + [ + server._block_replay_candidate_queue.get_nowait().submission.block_hash_hex + for _ in hashes + ], + hashes, ) + self.assertEqual(set(server._accepted_block_payout_previews), set(hashes)) + self.assertFalse(server._block_replay_enumeration_owed()) + # Enumeration never marks accepted-but-unaccounted work terminal. + self.assertEqual(len(original_rows()), 2) - def highdiff_client(self) -> ClientState: - state = client() - state.worker = worker_identity() - state.listener_vardiff_config = highdiff_vardiff_config() - state.minimum_advertised_difficulty = Decimal("500000") - state.share_difficulty = Decimal("500000") - return state - - def test_stamped_job_enforces_floor_below_network_difficulty(self) -> None: - server = self.stamp_coordinator() - state = self.highdiff_client() - - context = server.stamp_job_for_client(state, self.cached_bundle(), clean_jobs=True) + def test_single_candidate_page_failure_keeps_gate_closed_and_retry_deduplicates(self) -> None: + server, state, _recording = submit_coordinator() + ledger = SingleWriterShareLedger() + server.ledger = ledger + server.config = SimpleNamespace(block=SimpleNamespace(replay_page_size=1)) + hashes = self._pending_outbox_backlog(server, state, ledger, 2) + original_rows = ledger.pending_block_candidate_rows - self.assertEqual( - context.job.share_target, - direct_stratum.difficulty_target(Decimal("500000")), - ) - # Decimal round-tripping can land within 1e-27 of the floor; the wire - # value is float(difficulty), which is what marketplaces judge. - self.assertGreaterEqual(float(context.job.share_difficulty), 500000.0) + def failing_rows(*, limit=32, after_cursor=None): + if after_cursor is not None: + raise TimeoutError("second candidate unavailable") + return original_rows(limit=limit, after_cursor=after_cursor) - def test_stamped_job_keeps_network_cap_without_listener_floor(self) -> None: - server = self.stamp_coordinator() - state = client() - state.worker = worker_identity() - # Even an absurd desired difficulty stays capped at the network - # target on the default listener: shares are never required to be - # harder than blocks there. - state.share_difficulty = Decimal("500000") + ledger.pending_block_candidate_rows = failing_rows + server._note_block_replay_enumeration_owed() + with patch("builtins.print"): + with self.assertRaisesRegex(TimeoutError, "second candidate unavailable"): + server.replay_pending_block_candidates() + self.assertTrue(server._block_replay_enumeration_owed()) + self.assertEqual(server._block_replay_candidate_queue.qsize(), 1) + self.assertEqual(len(original_rows()), 2) + ledger.pending_block_candidate_rows = original_rows + self.assertEqual(server.replay_pending_block_candidates(), 1) + + self.assertFalse(server._block_replay_enumeration_owed()) + self.assertEqual(server._block_replay_candidate_queue.qsize(), 2) + self.assertEqual(set(server._accepted_block_payout_previews), set(hashes)) + + def test_configured_single_candidate_pages_apply_to_ancestor_redrive(self) -> None: + server, state, _recording = submit_coordinator() + ledger = SingleWriterShareLedger() + server.ledger = ledger + server.config = SimpleNamespace(block=SimpleNamespace(replay_page_size=1)) + hashes = self._pending_outbox_backlog(server, state, ledger, 2) + service = server._ensure_block_candidate_service() + self.assertFalse(server._block_replay_enumeration_owed()) - context = server.stamp_job_for_client(state, self.cached_bundle(), clean_jobs=True) + with patch.object( + service, "_consume_ancestor_redrive_requests", return_value=(hashes[1],) + ), patch.object( + server, + "_run_block_submitter_ledger_call", + wraps=server._run_block_submitter_ledger_call, + ) as calls, patch("builtins.print"): + self.assertEqual(server.replay_pending_block_candidates(), 2) + + queries = [call for call in calls.call_args_list if call.args[1] == "replay-outbox-query"] + keys = [call.args[0] for call in queries] + self.assertEqual([key[1] for key in keys], [1, 1, 1]) + self.assertEqual([key[2] for key in keys], [1, 2, 3]) + self.assertEqual(len(set(keys)), 3) + self.assertTrue(all(call.kwargs["call_class"] == "landing" for call in queries)) + self.assertEqual(server._block_replay_candidate_queue.qsize(), 2) + self.assertEqual(set(server._accepted_block_payout_previews), set(hashes)) + + def test_startup_enumeration_pages_a_backlog_larger_than_one_window( + self, + ) -> None: + """A backlog past the enumeration cap must still enumerate completely. + + The doubling window this replaces fails closed once one query would + have to hold the entire backlog, so ~5,000 pending rows kept the + job-build gate shut until the outbox drained under the cap on its + own (testnet4, 2026-08-18: ~25 minutes of refused job builds). A + keyset walk adopts every row in bounded pages and ends only on a + page proven short. + """ + server, state, _recording = submit_coordinator() + ledger = SingleWriterShareLedger() + server.ledger = ledger + hashes = self._pending_outbox_backlog(server, state, ledger, 5) - self.assertEqual(context.job.share_target, target_from_compact("207fffff")) - self.assertLess(context.job.share_difficulty, Decimal("1")) + queries: list[tuple[int, object]] = [] + original_rows = ledger.pending_block_candidate_rows - def test_stamped_job_honors_md_raised_floor_on_highdiff_listener(self) -> None: - server = self.stamp_coordinator() - state = self.highdiff_client() - state.requested_min_difficulty = Decimal("2000000") - server.apply_client_difficulty_requests(state) + def recording_rows( + *, + limit: int = 32, + after_cursor: object | None = None, + ) -> list[dict[str, object]]: + queries.append((limit, after_cursor)) + return original_rows(limit=limit, after_cursor=after_cursor) - context = server.stamp_job_for_client(state, self.cached_bundle(), clean_jobs=True) + ledger.pending_block_candidate_rows = recording_rows # type: ignore[method-assign] + server._note_block_replay_enumeration_owed() + # Both constants shrink together so five pending rows reproduce the + # incident's shape: under the doubling window this backlog starts at + # the cap, every pass is "truncated", and the gate never opens. + with patch( + "lab.prism.block_candidates.MAX_BLOCK_REPLAY_ENUMERATION_ROWS", + 2, + ), patch( + "lab.prism.block_candidates.MAX_PENDING_BLOCK_CANDIDATES", + 2, + ), patch("builtins.print"): + queued = server.replay_pending_block_candidates() + + # Every durable row was adopted exactly once. + self.assertEqual(queued, len(hashes)) + self.assertEqual(server._block_replay_candidate_queue.qsize(), len(hashes)) + replayed = [ + server._block_replay_candidate_queue.get_nowait() + for _ in range(len(hashes)) + ] self.assertEqual( - context.job.share_target, - direct_stratum.difficulty_target(Decimal("2000000")), - ) - self.assertGreaterEqual(float(context.job.share_difficulty), 2000000.0) + sorted( + str(candidate.submission.block_hash_hex) for candidate in replayed + ), + sorted(hashes), + ) + # Three pages of two, two, one: each page is the same bounded query, + # and the short one is what proves the enumeration complete. + self.assertEqual([limit for limit, _cursor in queries], [2, 2, 2]) + self.assertIsNone(queries[0][1]) + self.assertIsNotNone(queries[1][1]) + self.assertNotEqual(queries[1][1], queries[2][1]) + # Every restored row armed its payout barrier before the gate opened. + self.assertEqual(len(server._accepted_block_payout_previews), len(hashes)) + self.assertFalse(server._block_replay_enumeration_owed()) + + def test_startup_enumeration_pages_are_distinct_ledger_calls(self) -> None: + """Distinct pages must not collapse onto one in-flight ledger call. + + A timed-out outbox call stays registered under its dedupe key so the + next paced retry can reuse it. If every page shared one key, page + N+1 would be answered by page N's call and a whole page of pending + candidates would vanish from an enumeration that then declared + itself complete. + """ + server, state, _recording = submit_coordinator() + ledger = SingleWriterShareLedger() + server.ledger = ledger + self._pending_outbox_backlog(server, state, ledger, 3) - def test_block_worthy_submission_below_share_target_submits_synchronously(self) -> None: - # With the floor above network difficulty a hash can solve a block - # while missing the advertised share target. It is a valid share only - # if the block lands, so it submits synchronously (not via the async - # queue) and the share credit lands with it. - server, state, ledger = submit_coordinator() - submission = SimpleNamespace( - header_hex="aa" * 80, - block_hash_hex="bb" * 32, - share_pass=False, - block_pass=True, - ) - submitted: list[object] = [] + keys: list[tuple[object, ...]] = [] + original_call = server._run_block_submitter_ledger_call - def fake_submit(candidate: object) -> bool: - submitted.append(candidate) - server.append_accepted_share( - candidate.client, candidate.context, candidate.submission, candidate.pending_share - ) - return True + def recording_call(key, phase, operation, **kwargs): # type: ignore[no-untyped-def] + if phase == "replay-outbox-query": + keys.append(key) + return original_call(key, phase, operation, **kwargs) + + server._run_block_submitter_ledger_call = recording_call # type: ignore[method-assign] + server._note_block_replay_enumeration_owed() - server.submit_block_candidate = fake_submit # type: ignore[method-assign] with patch( - "lab.prism.prism_coordinator.direct_stratum.assemble_submission", - return_value=submission, - ): - should_close = server.handle_submit( - state, - ["miner-a", "job-1", "00" * 8, "00000001", "00000002"], - ) + "lab.prism.block_candidates.MAX_BLOCK_REPLAY_ENUMERATION_ROWS", + 1, + ), patch("builtins.print"): + self.assertEqual(server.replay_pending_block_candidates(), 3) - self.assertFalse(should_close) - self.assertEqual(server.rejection_counts_by_reason[PRISM_REJECTION_LOW_DIFFICULTY], 0) - self.assertEqual(len(submitted), 1) - self.assertTrue(submitted[0].credit_share_on_accept) - # Nothing was queued to the async submitter; it landed inline. - self.assertEqual(server.block_candidate_queue.qsize(), 0) - self.assertEqual(len(ledger.pending), 1) + self.assertEqual(len(keys), 4) + self.assertEqual(len(set(keys)), 4) + # Every key is hashable and carries its page ordinal. + self.assertEqual([key[2] for key in keys], [1, 2, 3, 4]) - def test_below_target_block_intent_is_durable_before_synchronous_submit(self) -> None: + def test_steady_state_poll_never_paginates(self) -> None: + """Without an owed enumeration the poll stays one small query.""" server, state, _recording = submit_coordinator() ledger = SingleWriterShareLedger() server.ledger = ledger - submission = SimpleNamespace( - header_hex="aa" * 80, - coinbase_tx_hex="00", - block_hex="00", - block_hash_hex="bc" * 32, - share_pass=False, - block_pass=True, - ) + self._pending_outbox_backlog(server, state, ledger, 3) - def fake_submit(candidate: PrismBlockCandidate) -> bool: - pending = ledger.pending_block_candidates() - self.assertEqual(len(pending), 1) - self.assertTrue(pending[0]["credit_share_on_accept"]) - server.append_accepted_share( - candidate.client, - candidate.context, - candidate.submission, - candidate.pending_share, - candidate_intent=server.block_candidate_intent(candidate), - ) - return True + queries: list[tuple[int, object]] = [] + original_rows = ledger.pending_block_candidate_rows - server.submit_block_candidate = fake_submit # type: ignore[method-assign] - with patch( - "lab.prism.prism_coordinator.direct_stratum.assemble_submission", - return_value=submission, - ): - self.assertFalse( - server.handle_submit( - state, - ["miner-a", "job-1", "00" * 8, "00000001", "00000002"], - ) - ) + def recording_rows( + *, + limit: int = 32, + after_cursor: object | None = None, + ) -> list[dict[str, object]]: + queries.append((limit, after_cursor)) + return original_rows(limit=limit, after_cursor=after_cursor) - self.assertEqual(len(ledger), 1) - self.assertEqual(ledger.pending_block_candidates(), []) + ledger.pending_block_candidate_rows = recording_rows # type: ignore[method-assign] - def test_below_target_accepted_tail_serializes_moved_tip_replay(self) -> None: - old_tip = "00" * 32 - new_tip = "11" * 32 - block_hash = "b7" * 32 - server, state, _recording = submit_coordinator(tip=old_tip) - server.max_blocks = 10 - server.stop_after_block = False - ledger = SingleWriterShareLedger() - server.ledger = ledger - server.rpc = SubmitRpc( - tip=old_tip, - block_hash=block_hash, - ledger=ledger, - ) - server.build_audit_bundle = ( # type: ignore[method-assign] - lambda **_kwargs: verified_block_bundle() - ) - server.verify_bundle = ( # type: ignore[method-assign] - lambda *_args, **_kwargs: verified_audit_report() - ) - server.ledger_writer_public_key_hex = "aa" * 32 - server.refresh_jobs_after_pending_accepted_block = ( # type: ignore[method-assign] - lambda *_args, **_kwargs: 0 - ) - submission = SimpleNamespace( - header_hex="b7" * 80, - coinbase_tx_hex="c0ffee", - block_hex="00", - block_hash_hex=block_hash, - share_pass=False, - block_pass=True, - ) + with patch( + "lab.prism.block_candidates.MAX_BLOCK_REPLAY_ENUMERATION_ROWS", + 1, + ), patch("builtins.print"): + self.assertEqual(server.replay_pending_block_candidates(), 3) - durable_confirmation = threading.Event() - accepted_tail_paused = threading.Event() - release_accepted_tail = threading.Event() - replay_guard_blocked = threading.Event() - confirmation_calls: list[str] = [] - submitted: list[str] = [] - abandoned: list[str] = [] - synchronous_results: list[bool] = [] - replay_results: list[bool] = [] - errors: list[BaseException] = [] - replay_thread: threading.Thread | None = None - original_confirm = ledger.confirm_accepted_block - original_stats = server.accepted_share_stats - original_submitted = ledger.mark_block_candidate_submitted - original_abandoned = ledger.mark_block_candidate_abandoned + self.assertEqual(queries, [(MAX_PENDING_BLOCK_CANDIDATES, None)]) - class ObservedDispositionLock: - def __init__(self) -> None: - self.lock = threading.Lock() + def test_startup_enumeration_falls_back_when_cursors_are_unsupported( + self, + ) -> None: + """A ledger without cursor support keeps today's exact semantics. - def acquire( - self, - blocking: bool = True, - timeout: float = -1, - ) -> bool: - if threading.current_thread() is replay_thread: - if self.lock.acquire(blocking=False): - return True - # A failed non-blocking acquisition proves the accepted - # attempt still owns this exact same-hash guard. - replay_guard_blocked.set() - if not blocking: - return self.lock.acquire(blocking=False) - if timeout < 0: - return self.lock.acquire() - return self.lock.acquire(timeout=timeout) + The wrapper here takes ``**kwargs`` -- the shape an instrumenting or + forwarding double really has -- so signature introspection cannot + rule the cursor out and the keyword probe is what discovers the + legacy ledger underneath. The pass must then walk the old doubling + window and, past the cap, still fail closed with enumeration owed. + """ + server, _state, _recording = submit_coordinator() + requested: list[int] = [] - def release(self) -> None: - self.lock.release() + def legacy_rows(*, limit: int = 32) -> list[dict[str, object]]: + requested.append(limit) + return [durable_candidate_row(index) for index in range(limit)] - server._ensure_block_candidate_disposition_state() - with server._block_candidate_disposition_registry_lock: - server._block_candidate_disposition_flights[block_hash] = ( # type: ignore[assignment] - SimpleNamespace(lock=ObservedDispositionLock(), users=0) - ) + def forwarding_rows(**kwargs: object) -> list[dict[str, object]]: + return legacy_rows(**kwargs) # type: ignore[arg-type] - def confirm_accepted_block( - *, - block_hash: str, - active_tip_height: int, - ) -> dict[str, int | str]: - result = original_confirm( - block_hash=block_hash, - active_tip_height=active_tip_height, - ) - confirmation_calls.append(block_hash) - durable_confirmation.set() - return result + server.ledger = SimpleNamespace( + pending_block_candidate_rows=forwarding_rows, + ) + server._note_block_replay_enumeration_owed() - def pause_in_accepted_tail() -> tuple[int, int]: - accepted_tail_paused.set() - if not release_accepted_tail.wait(10): - raise AssertionError("timed out waiting to release accepted success tail") - return original_stats() + with patch("builtins.print"): + queued = server.replay_pending_block_candidates() - def mark_submitted(*, block_hash: str) -> bool: - submitted.append(block_hash) - return original_submitted(block_hash=block_hash) + # The probe raised TypeError and adopted nothing, so the windowed + # walk starts from the first batch exactly as it always did. + self.assertEqual(requested, [32, 64, 128, 256, 512, 1024]) + self.assertEqual(queued, MAX_BLOCK_REPLAY_ENUMERATION_ROWS) + # Fail closed: an unproven enumeration must not open the job-build + # gate. + self.assertTrue(server._block_replay_enumeration_owed()) - def mark_abandoned(*, block_hash: str, error: str) -> bool: - abandoned.append(block_hash) - return original_abandoned(block_hash=block_hash, error=error) + def test_startup_enumeration_fails_closed_on_a_page_without_a_cursor( + self, + ) -> None: + """A full page that cannot advance the walk must not claim success.""" + server, _state, _recording = submit_coordinator() + pages: list[object] = [] - ledger.confirm_accepted_block = confirm_accepted_block # type: ignore[method-assign] - ledger.mark_block_candidate_submitted = mark_submitted # type: ignore[method-assign] - ledger.mark_block_candidate_abandoned = mark_abandoned # type: ignore[method-assign] - server.accepted_share_stats = pause_in_accepted_tail # type: ignore[method-assign] + def uncursored_rows( + *, + limit: int = 32, + after_cursor: object | None = None, + ) -> list[dict[str, object]]: + pages.append(after_cursor) + return [durable_candidate_row(index) for index in range(limit)] - def submit_synchronously() -> None: - try: - synchronous_results.append( - server.handle_submit( - state, - [ - "miner-a", - "job-1", - "00" * 8, - "00000001", - "00000002", - ], - ) - ) - except BaseException as exc: # noqa: BLE001 - asserted below - errors.append(exc) + server.ledger = SimpleNamespace( + pending_block_candidate_rows=uncursored_rows, + ) + server._note_block_replay_enumeration_owed() - def replay_pending_candidate() -> None: - try: - replay_results.append(server.submit_next_block_candidate()) - except BaseException as exc: # noqa: BLE001 - asserted below - errors.append(exc) + with patch( + "lab.prism.block_candidates.MAX_BLOCK_REPLAY_ENUMERATION_ROWS", + 2, + ), patch("builtins.print"): + self.assertEqual(server.replay_pending_block_candidates(), 2) - with tempfile.TemporaryDirectory() as tempdir, patch( - "lab.prism.prism_coordinator.direct_stratum.assemble_submission", - return_value=submission, - ): - server.audit_dir = Path(tempdir) - server.evidence_path = Path(tempdir) / "evidence.json" - synchronous_thread = threading.Thread(target=submit_synchronously) - synchronous_thread.start() - try: - self.assertTrue( - accepted_tail_paused.wait(5), - msg=f"synchronous submit exited early: {errors!r}", - ) - self.assertTrue(durable_confirmation.is_set()) - self.assertEqual(len(ledger), 1) - self.assertEqual(len(ledger.pending_block_candidates()), 1) - with server.lock: - self.assertNotIn( - block_hash, - server._accounted_accepted_block_hashes, - ) + # One page, then a stop: the rows carry no cursor, so there is no + # way to prove what follows them. + self.assertEqual(pages, [None]) + self.assertTrue(server._block_replay_enumeration_owed()) - self.assertEqual(server.replay_pending_block_candidates(), 1) - server.rpc = TipRpc(new_tip) - replay_thread = threading.Thread(target=replay_pending_candidate) - replay_thread.start() - self.assertTrue(replay_guard_blocked.wait(5)) - self.assertTrue(replay_thread.is_alive()) - self.assertEqual(submitted, []) - self.assertEqual(abandoned, []) - finally: - release_accepted_tail.set() - synchronous_thread.join(10) - if replay_thread is not None: - replay_thread.join(10) + def test_startup_block_replay_catches_psql_server_timeout(self) -> None: + server, _state, _recording = submit_coordinator() + ledger = PsqlShareLedger.__new__(PsqlShareLedger) + ledger._command = ["psql"] + ledger._native = None + ledger._operation_timeout_local = threading.local() - self.assertFalse(synchronous_thread.is_alive()) - self.assertIsNotNone(replay_thread) - assert replay_thread is not None - self.assertFalse(replay_thread.is_alive()) - if errors: - raise errors[0] - self.assertEqual(synchronous_results, [False]) - self.assertEqual(replay_results, [True]) - self.assertEqual(confirmation_calls, [block_hash]) - self.assertGreaterEqual(len(submitted), 1) - self.assertEqual(abandoned, []) - self.assertEqual(len(ledger), 1) - self.assertEqual(ledger.pending_block_candidates(), []) - self.assertEqual(server.accepted_block_count, 1) - self.assertIn(block_hash, server._accounted_accepted_block_hashes) - self.assertNotIn( - PRISM_REJECTION_STALE_JOB, - server.block_candidate_abandoned_counts, + def timed_out_rows(*, limit: int = 32) -> list[dict[str, object]]: + ledger._run_sql(f"SELECT {limit};") + return [] + + ledger.pending_block_candidate_rows = timed_out_rows # type: ignore[method-assign] + server.ledger = ledger + completed = subprocess.CompletedProcess( + args=["psql"], + returncode=3, + stdout="", + stderr=( + "ERROR: 57014: canceling statement due to statement timeout" + ), ) - self.assertNotIn(block_hash, server._accepted_block_payout_previews) - self.assertNotIn( - block_hash, - server._invalidated_accepted_block_payout_previews, + with patch( + "lab.prism.share_ledger.subprocess.run", + return_value=completed, + ), patch("builtins.print") as printed: + self.assertTrue(server._run_startup_block_candidate_replay()) + + startup_logs = [ + " ".join(str(value) for value in call.args) + for call in printed.call_args_list + ] + self.assertTrue( + any( + "startup block candidate replay timed out " + "phase=replay-outbox-query" in message + for message in startup_logs + ) ) - self.assertEqual(server._block_candidate_disposition_flights, {}) - def test_below_target_intent_failure_does_not_create_unsafe_retry_slot(self) -> None: - server, state, _recording = submit_coordinator() - ledger = SingleWriterShareLedger() - server.ledger = ledger - submission = SimpleNamespace( - header_hex="aa" * 80, - coinbase_tx_hex="00", - block_hex="00", - block_hash_hex="be" * 32, - share_pass=False, - block_pass=True, + def test_startup_block_replay_keeps_hard_database_errors_fatal(self) -> None: + server, _state, _recording = submit_coordinator() + + def failed_rows(*, limit: int = 32) -> list[dict[str, object]]: + raise RuntimeError(f"postgres failed limit={limit}") + + server.ledger = SimpleNamespace( + pending_block_candidate_rows=failed_rows, ) - submit_calls = 0 + with self.assertRaisesRegex(RuntimeError, "postgres failed"): + server._run_startup_block_candidate_replay() - def fail_intent(_intent: dict[str, object]) -> bool: - raise RuntimeError("outbox unavailable") + def test_startup_block_replay_preserves_shutdown_stop(self) -> None: + server = self._bare_coordinator() - def unsafe_submit(_candidate: PrismBlockCandidate) -> bool: - nonlocal submit_calls - submit_calls += 1 - return True + def shutdown_replay() -> int: + raise ShutdownInProgress("shutdown won") - ledger.persist_block_candidate_intent = fail_intent # type: ignore[method-assign] - server.submit_block_candidate = unsafe_submit # type: ignore[method-assign] - with patch( - "lab.prism.prism_coordinator.direct_stratum.assemble_submission", - return_value=submission, - ): - with self.assertRaisesRegex(RuntimeError, "outbox unavailable"): - server.handle_submit( - state, - ["miner-a", "job-1", "00" * 8, "00000001", "00000002"], + server.replay_pending_block_candidates = shutdown_replay # type: ignore[method-assign] + + self.assertFalse(server._run_startup_block_candidate_replay()) + + def test_stuck_rpc_worker_pool_requests_nonzero_restart(self) -> None: + server, _state, _recording = submit_coordinator() + server.block_submit_stuck_call_exit_seconds = 0.04 + release = threading.Event() + entered: list[str] = [] + + class IgnoringRpc: + def call( + self, + method: str, + params: list[object] | None = None, + *, + timeout: float | None = None, + ) -> object: + entered.append(str((params or [""])[0])) + release.wait(5) + return None + + server.rpc = IgnoringRpc() + try: + for tag in ("11", "22"): + with self.assertRaises(TimeoutError): + server._run_submitblock_rpc_with_hard_deadline( + block_hash=tag * 32, + block_hex=tag, + timeout_seconds=0.01, + ) + time.sleep(0.05) + with self.assertRaises(TimeoutError): + server._run_submitblock_rpc_with_hard_deadline( + block_hash="33" * 32, + block_hex="33", + timeout_seconds=0.01, ) + self.assertEqual(entered, ["11", "22"]) + self.assertTrue(server.stop_event.is_set()) + self.assertTrue(server._fatal_exit_requested) + finally: + release.set() + + def test_block_landing_db_timeout_escalates_and_clears(self) -> None: + server, _state, _recording = submit_coordinator() + # The doubling sequence below reaches the reviewed 120s cap, which + # only a tolerance of at least 240s can grant: the landing budget is + # clamped to half the configured watchdog. Pin the production + # tolerance explicitly so this test keeps asserting the escalation + # ladder itself rather than the ceiling. + server.watchdog_timeout_seconds = 300.0 + server.block_landing_db_timeout_seconds = 30.0 + server.block_landing_db_timeout_max_seconds = 120.0 + block_hash = "ab" * 32 + self.assertEqual(server._block_landing_db_timeout(block_hash), 30.0) + server._note_block_landing_timeout(block_hash) + self.assertEqual(server._block_landing_db_timeout(block_hash), 60.0) + server._note_block_landing_timeout(block_hash) + self.assertEqual(server._block_landing_db_timeout(block_hash), 120.0) + server._note_block_landing_timeout(block_hash) + self.assertEqual(server._block_landing_db_timeout(block_hash), 120.0) + self.assertEqual(server._block_landing_db_timeout("cd" * 32), 30.0) + server._clear_block_candidate_retry_state(block_hash) + self.assertEqual(server._block_landing_db_timeout(block_hash), 30.0) - self.assertEqual(submit_calls, 0) - self.assertIsNone(getattr(server, "_retry_block_candidate", None)) - self.assertEqual(ledger.pending_block_candidates(), []) + def test_escalated_landing_budget_stays_inside_watchdog_tolerance(self) -> None: + """Escalation may never grant a budget the watchdog will not wait for. - def test_block_worthy_below_target_rejects_low_difficulty_when_block_fails(self) -> None: - # If the block does not land, the below-share-target hash earns nothing - # and the miner is rejected as low-difficulty -- never acked accepted - # with no ledger row. - server, state, ledger = submit_coordinator() - submission = SimpleNamespace( - header_hex="aa" * 80, - block_hash_hex="bb" * 32, - share_pass=False, - block_pass=True, - ) - def reject_candidate(_candidate: PrismBlockCandidate) -> bool: - server._abandon_block_candidate( - PRISM_REJECTION_SUBMITBLOCK_REJECTED, - "rejected", - block_hash=_candidate.submission.block_hash_hex, - worker="miner-a", - ) - return False + A landing step is spent on the watchdog-monitored block-work thread, + so a budget above the tolerance does not buy a longer attempt: the + watchdog hard-exits mid-landing, the in-memory escalation counts die + with the process, and the restart replays the same doomed attempt + from the base budget forever (issue #125). The ceiling is derived + from the configured tolerance, so raising the watchdog restores the + reviewed cap without touching this clamp. + """ + server, _state, _recording = submit_coordinator() + server.watchdog_timeout_seconds = 120.0 + server.block_landing_db_timeout_seconds = 30.0 + server.block_landing_db_timeout_max_seconds = 120.0 + block_hash = "ab" * 32 + for _ in range(8): + server._note_block_landing_timeout(block_hash) + escalated = server._block_landing_db_timeout(block_hash) + self.assertLessEqual(escalated, server.watchdog_timeout_seconds / 2.0) + self.assertEqual(escalated, 60.0) + # The production override is pinned too: a 300s tolerance leaves a + # 150s ceiling, so the reviewed 120s cap is still granted in full and + # deployments running that override see no behavior change. + server.watchdog_timeout_seconds = 300.0 + self.assertEqual(server._block_landing_db_timeout(block_hash), 120.0) - server.submit_block_candidate = reject_candidate # type: ignore[method-assign] - with patch( - "lab.prism.prism_coordinator.direct_stratum.assemble_submission", - return_value=submission, - ): - with self.assertRaises(StratumError) as raised: - server.handle_submit( - state, - ["miner-a", "job-1", "00" * 8, "00000001", "00000002"], - ) + def test_watchdog_ceiling_clamps_the_configured_base_not_only_the_cap(self) -> None: + """A tolerance below the base budget must lower the base too. - self.assertEqual(raised.exception.reason, PRISM_REJECTION_LOW_DIFFICULTY) - self.assertEqual(len(ledger.pending), 0) - self.assertEqual(server.block_candidate_queue.qsize(), 0) - # The reject is counted (globally and for the worker), not just the - # block-abandonment reason -- this synchronous path used to skip it. - self.assertEqual(server.rejection_counts_by_reason[PRISM_REJECTION_LOW_DIFFICULTY], 1) - self.assertEqual(server.low_difficulty_share_count, 1) + The first landing attempt already spends the base budget on the + watchdog-monitored thread, so a base above the ceiling trips the + watchdog before any escalation has happened at all -- the clamp + cannot be a cap-only concern. + """ + server, _state, _recording = submit_coordinator() + server.watchdog_timeout_seconds = 40.0 + server.block_landing_db_timeout_seconds = 30.0 + server.block_landing_db_timeout_max_seconds = 120.0 + block_hash = "ab" * 32 + service = server._ensure_block_candidate_service() + self.assertEqual(service._block_landing_watchdog_ceiling(), 20.0) + with contextlib.redirect_stdout(io.StringIO()): + self.assertEqual(server._block_landing_db_timeout(block_hash), 20.0) + for _ in range(8): + server._note_block_landing_timeout(block_hash) + self.assertEqual(server._block_landing_db_timeout(block_hash), 20.0) + + def test_disabled_watchdog_grants_the_configured_landing_budget(self) -> None: + """No hard-exit hazard means no reason to spend the operator's cap. + + The clamp exists solely because a landing budget above the watchdog + tolerance buys a hard exit instead of a longer attempt. A deployment + that turned the watchdog off has no such exit to avoid, so halving + its cap would be pure loss. The attribute is read defensively and + defaults to enabled, since clamping is the safe answer. + """ + server, _state, _recording = submit_coordinator() + server.watchdog_timeout_seconds = 120.0 + server.block_landing_db_timeout_seconds = 30.0 + server.block_landing_db_timeout_max_seconds = 120.0 + block_hash = "ab" * 32 + for _ in range(8): + server._note_block_landing_timeout(block_hash) - def test_below_target_transient_outcome_closes_without_definitive_reject(self) -> None: - server, state, _recording = submit_coordinator() - ledger = SingleWriterShareLedger() - server.ledger = ledger - submission = SimpleNamespace( - header_hex="aa" * 80, - coinbase_tx_hex="00", - block_hex="00", - block_hash_hex="bd" * 32, - share_pass=False, - block_pass=True, + server.watchdog_enabled = False + service = server._ensure_block_candidate_service() + self.assertEqual(service._block_landing_watchdog_ceiling(), float("inf")) + with contextlib.redirect_stdout(io.StringIO()) as quiet: + self.assertEqual(server._block_landing_db_timeout(block_hash), 120.0) + self.assertEqual(server._block_landing_db_timeout(None), 30.0) + # An infinite ceiling never compares below a configured value, so the + # clamp notice must stay silent as well. + self.assertEqual(quiet.getvalue(), "") + + server.watchdog_enabled = True + with contextlib.redirect_stdout(io.StringIO()): + self.assertEqual(server._block_landing_db_timeout(block_hash), 60.0) + + def test_lifecycle_config_still_carries_the_watchdog_enabled_switch(self) -> None: + """The clamp's defensive read must not outlive the attribute itself. + + ``getattr(..., True)`` would keep clamping silently if the setting + were ever renamed, so pin the name the coordinator assigns from. + """ + self.assertIn( + "watchdog_enabled", + {field.name for field in dataclasses.fields(LifecycleConfig)}, ) - server.submit_block_candidate = lambda _candidate: False # type: ignore[method-assign] - submit_params = ["miner-a", "job-1", "00" * 8, "00000001", "00000002"] - # Each retry rebuilds its candidate intent with a fresh acknowledgment - # stamp. Force every call onto a new millisecond so the durable-outbox - # idempotency is exercised across acknowledgment-stamp drift instead of - # depending on both attempts landing within the same millisecond. - clock_ms = iter(range(1_700_000_000_000, 1_700_000_070_000, 7)) - with patch( - "lab.prism.prism_coordinator.direct_stratum.assemble_submission", - return_value=submission, - ), patch( - "lab.prism.prism_coordinator.now_ms", - side_effect=clock_ms.__next__, - ): - for _attempt in range(2): - with self.assertRaisesRegex(RuntimeError, "pending durable retry"): - server.handle_submit(state, submit_params) - self.assertEqual(server.recent_share_keys, set()) + def test_landing_budget_clamp_is_announced_exactly_once(self) -> None: + """A silently reduced budget is a config mismatch nobody can see. - self.assertEqual(server.rejection_counts_by_reason[PRISM_REJECTION_LOW_DIFFICULTY], 0) - self.assertEqual(server.duplicate_share_count, 0) - self.assertEqual(len(ledger), 0) - self.assertEqual(len(ledger.pending_block_candidates()), 1) - self.assertIsNotNone(server._retry_block_candidate) - assert server._retry_block_candidate is not None - self.assertEqual( - server._retry_block_candidate.submission.block_hash_hex, - submission.block_hash_hex, - ) + Without the notice the operator's configured cap simply does not + happen -- at the 120s default tolerance a 120s cap becomes 60s -- and + "escalation exhausted at my cap" reads identically to "escalation + never reached my cap". It is emitted once because the budget is + recomputed on every landing attempt, and the flag flips under the + coordinator lock so concurrent first landings cannot both print. + """ + server, _state, _recording = submit_coordinator() + server.watchdog_timeout_seconds = 120.0 + server.block_landing_db_timeout_seconds = 30.0 + server.block_landing_db_timeout_max_seconds = 120.0 + block_hash = "ab" * 32 + captured = io.StringIO() + with contextlib.redirect_stdout(captured): + barrier = threading.Barrier(4, timeout=10) - def test_post_block_notification_stamps_caller_heartbeat(self) -> None: - server, _state, _ledger = submit_coordinator() - seen: list[str] = [] - server._record_heartbeat = seen.append # type: ignore[method-assign] - server.refresh_jobs_after_accepted_block( - block_height=10, block_hash="bb" * 32, heartbeat_name="block_submitter" - ) - self.assertEqual(seen, ["block_submitter"]) + def landing_budget() -> None: + barrier.wait() + for _ in range(5): + server._block_landing_db_timeout(block_hash) - # The client-thread pending refresh keeps the default poller heartbeat. - seen.clear() - server.refresh_jobs_after_accepted_block(block_height=11, block_hash="cc" * 32) - self.assertEqual(seen, ["qbit_blockpoll"]) + threads = [threading.Thread(target=landing_budget) for _ in range(4)] + for thread in threads: + thread.start() + for thread in threads: + thread.join(10) + server._block_landing_db_timeout(block_hash) + lines = [ + line + for line in captured.getvalue().splitlines() + if "landing db budget clamped by watchdog" in line + ] + self.assertEqual(len(lines), 1) + self.assertIn("configured_max=120s", lines[0]) + self.assertIn("granted_max=60s", lines[0]) + self.assertIn("ceiling=60s", lines[0]) + self.assertIn("watchdog_timeout=120s", lines[0]) + + def test_landing_scope_stamps_progress_during_ledger_admission(self) -> None: + """Waiting for the ledger's writer lock must not be heartbeat-silent. + + Admission happens before any statement is sent, so no server-side + deadline bounds it and nothing reports until the gate opens. The + landing scope runs on the block-work owner thread the watchdog + monitors, so it installs the ledger's progress hook and the wait + stamps its phase in watchdog-sized slices instead. + """ + server, _state, _recording = submit_coordinator() + server.watchdog_timeout_seconds = 120.0 + server.block_landing_db_timeout_seconds = 30.0 + server.block_submit_db_timeout_seconds = 1.0 + installed: list[float] = [] + stamped: list[str] = [] - def test_low_difficulty_submission_without_block_solve_is_rejected(self) -> None: - server, state, ledger = submit_coordinator() - submission = SimpleNamespace( - header_hex="aa" * 80, - block_hash_hex="bb" * 32, - share_pass=False, - block_pass=False, - ) - with patch( - "lab.prism.prism_coordinator.direct_stratum.assemble_submission", - return_value=submission, - ): - with self.assertRaises(StratumError) as raised: - server.handle_submit( - state, - ["miner-a", "job-1", "00" * 8, "00000001", "00000002"], - ) + @contextlib.contextmanager + def operation_progress(on_progress, *, slice_seconds: float): + installed.append(slice_seconds) + on_progress() + yield - self.assertEqual(raised.exception.code, 23) - self.assertEqual(raised.exception.reason, PRISM_REJECTION_LOW_DIFFICULTY) - self.assertEqual(server.rejection_counts_by_reason[PRISM_REJECTION_LOW_DIFFICULTY], 1) - self.assertEqual(len(ledger.pending), 0) + @contextlib.contextmanager + def statement_timeout(seconds: float): + yield - def test_collection_only_below_target_block_solve_submits_solver_pays_all(self) -> None: - # A collection job's signed bootstrap manifest already commits the - # whole coinbase to the submitting worker, so a solved block on a - # collection job is submitted (synchronously here, since the share - # missed its target) instead of being withheld -- the first block on a - # fresh ledger must never be silently ledgered away. - server, state, ledger = submit_coordinator() - server.jobs["job-1"].collection_only = True - submission = SimpleNamespace( - header_hex="aa" * 80, - block_hash_hex="bb" * 32, - share_pass=False, - block_pass=True, + server.ledger = SimpleNamespace( + statement_timeout=statement_timeout, + operation_progress=operation_progress, + ) + server._record_block_submitter_wait = lambda phase: stamped.append(phase) + with server._block_landing_ledger_statement_timeout_scope("ab" * 32): + pass + with server._block_submitter_ledger_statement_timeout_scope(): + pass + self.assertEqual(len(installed), 2) + for slice_seconds in installed: + self.assertGreater(slice_seconds, 0.0) + self.assertLessEqual(slice_seconds, server.watchdog_timeout_seconds) + self.assertEqual( + stamped, + [ + "wait-ledger-admission:landing", + "wait-ledger-admission:submit", + ], ) - submitted: list[object] = [] - - def fake_submit(candidate: object) -> bool: - submitted.append(candidate) - server.append_accepted_share( - candidate.client, candidate.context, candidate.submission, candidate.pending_share - ) - return True - server.submit_block_candidate = fake_submit # type: ignore[method-assign] - with patch( - "lab.prism.prism_coordinator.direct_stratum.assemble_submission", - return_value=submission, - ): - should_close = server.handle_submit( - state, - ["miner-a", "job-1", "00" * 8, "00000001", "00000002"], - ) + def test_landing_scope_accepts_ledgers_without_a_progress_hook(self) -> None: + """Duck-typed ledgers predating the hook keep working unchanged.""" + server, _state, _recording = submit_coordinator() + server.watchdog_timeout_seconds = 120.0 + server.block_landing_db_timeout_seconds = 30.0 + scopes: list[float] = [] - self.assertFalse(should_close) - self.assertEqual(server.rejection_counts_by_reason[PRISM_REJECTION_LOW_DIFFICULTY], 0) - self.assertEqual(len(submitted), 1) - self.assertTrue(submitted[0].credit_share_on_accept) - self.assertTrue(submitted[0].context.collection_only) - self.assertEqual(len(ledger.pending), 1) - self.assertEqual(server.collection_block_submission_count, 1) + @contextlib.contextmanager + def statement_timeout(seconds: float): + scopes.append(seconds) + yield - def test_collection_job_block_solve_is_credited_and_enqueued_not_withheld(self) -> None: - # A solved block that also met its share target on a collection job is - # credited immediately and queued for the submitter thread, exactly - # like a ready-window candidate. - server, state, ledger = submit_coordinator() - server.jobs["job-1"].collection_only = True + server.ledger = SimpleNamespace(statement_timeout=statement_timeout) + with server._block_landing_ledger_statement_timeout_scope("ab" * 32): + pass + self.assertEqual(scopes, [30.0]) + + def _landing_phase_recorder(self, server: PrismCoordinator) -> list[str]: + """Capture the block-work phases one landing pass stamps, in order.""" + phases: list[str] = [] + server._record_block_submitter_phase = phases.append # type: ignore[method-assign] + return phases + + def test_landing_brackets_the_reorg_walk_and_prior_balance_check(self) -> None: + """Neither stretch may sit inside the landing scope unnamed. + + Between the current-tip RPC and the audit build the landing thread + spends a full reorg reconciliation -- ledger statements and chain + RPCs, one pair per watched block -- and then a prior-balances read. + The budget granted to a landing step is derived from the watchdog on + the assumption that a heartbeat-silent stretch is about one + statement long, so an unstamped multi-statement stretch here is + exactly the hazard the derivation was supposed to remove (issue + #125). + """ + parent_hash = "00" * 32 + block_hash = "cd" * 32 + server, state, ledger = submit_coordinator(tip=parent_hash) + ledger.durable_payout_state = True + phases = self._landing_phase_recorder(server) + server.ensure_reorg_reconciled_for_tip = ( # type: ignore[method-assign] + lambda _tip, **_kwargs: True + ) + # A stale payout base abandons the candidate right after the check, + # so the landing stops with both bracketed stretches behind it. + server.prior_balances_match_current = ( # type: ignore[method-assign] + lambda _balances: False + ) submission = SimpleNamespace( - header_hex="aa" * 80, - block_hash_hex="cc" * 32, - share_pass=True, - block_pass=True, + coinbase_tx_hex="c0ffee", + block_hash_hex=block_hash, + block_hex="00", + ) + landed = server._land_and_confirm_block_candidate( + block_candidate(server, state, submission), + current_tip=block_hash, + already_active=False, + worker="miner-a", + node_submission=_BlockCandidateNodeSubmission(attempted=False), + ) + self.assertIsNone(landed) + self.assertEqual( + phases, + [ + "reorg-reconcile", + "reorg-reconcile:complete", + "prior-balances-check", + "prior-balances-check:complete", + ], ) - with patch( - "lab.prism.prism_coordinator.direct_stratum.assemble_submission", - return_value=submission, - ): - should_close = server.handle_submit( - state, - ["miner-a", "job-1", "00" * 8, "00000001", "00000002"], - ) - self.assertFalse(should_close) - self.assertEqual(len(ledger.pending), 1) - self.assertEqual(server.block_candidate_queue.qsize(), 1) - queued = server.block_candidate_queue.get_nowait() - self.assertTrue(queued.context.collection_only) - self.assertFalse(queued.credit_share_on_accept) - self.assertEqual(server.collection_block_submission_count, 1) + def test_landing_brackets_the_tip_height_rpc(self) -> None: + """A node round trip between two stamped stretches gets its own name.""" - def test_block_candidate_intent_round_trips_compact_payout_state(self) -> None: - server, state, _ledger = submit_coordinator() - server.jobs["job-1"].collection_only = True - server.jobs["job-1"].prospective_prior_balances = ( - ("miner-a", "miner-a", "11" * 32, 25), + class StaleHeightTipRpc(TipRpc): + def call( + self, + method: str, + params: list[object] | None = None, + ) -> object: + if method == "getblockcount": + return 42 + return super().call(method, params) + + parent_hash = "00" * 32 + block_hash = "cd" * 32 + server, state, ledger = submit_coordinator(tip=parent_hash) + ledger.durable_payout_state = True + server.rpc = StaleHeightTipRpc(parent_hash) + phases = self._landing_phase_recorder(server) + server.ensure_reorg_reconciled_for_tip = ( # type: ignore[method-assign] + lambda _tip, **_kwargs: True ) - pending = PendingShare( - share_id="miner-a:" + "dd" * 32, - miner_id="miner-a", - order_key="miner-a", - p2mr_program_hex="11" * 32, - share_difficulty=1, - network_difficulty=1, - template_height=9, - job_id="job-1", - job_issued_at_ms=1, - accepted_at_ms=2, - ntime=1, + server.prior_balances_match_current = ( # type: ignore[method-assign] + lambda _balances: True ) submission = SimpleNamespace( - coinbase_tx_hex="00", - block_hash_hex="dd" * 32, + coinbase_tx_hex="c0ffee", + block_hash_hex=block_hash, block_hex="00", - share_pass=True, - block_pass=True, ) - candidate = block_candidate(server, state, submission, pending_share=pending) - - intent = server.block_candidate_intent(candidate) - self.assertIs(intent["collection_only"], True) - replayed = server.block_candidate_from_intent(intent) - self.assertTrue(replayed.context.collection_only) + landed = server._land_and_confirm_block_candidate( + block_candidate(server, state, submission), + current_tip=block_hash, + already_active=False, + worker="miner-a", + node_submission=_BlockCandidateNodeSubmission(attempted=False), + ) + # The reported tip is far ahead of the template, so the candidate + # abandons as stale immediately after the bracketed RPC. + self.assertIsNone(landed) self.assertEqual( - replayed.context.prospective_prior_balances, - (("miner-a", "miner-a", "11" * 32, 25),), + phases[-2:], + ["tip-height-rpc", "tip-height-rpc:complete"], ) - # Intents persisted before the flag existed replay as ready-window - # candidates, which is all the outbox could ever have contained then. - intent.pop("collection_only") - intent.pop("prospective_prior_balances") - replayed = server.block_candidate_from_intent(intent) - self.assertFalse(replayed.context.collection_only) - self.assertIsNone(replayed.context.prospective_prior_balances) + def test_block_submitter_phase_never_stamps_from_a_foreign_thread(self) -> None: + """A stamp off the owner thread must not refresh a wedged owner. - def test_ready_pool_refreshes_clients_left_on_collection_jobs(self) -> None: - # Once the pool crosses min_ready_miners, the poller must replace - # collection jobs with windowed work even when the template snapshot - # is otherwise unchanged -- readiness itself is invisible to the - # template fingerprint. - server, state, _ledger = submit_coordinator() - tip = "00" * 32 - snapshot = SimpleNamespace( - bestblockhash=tip, - previousblockhash=tip, - template_fingerprint="fp", - ) - context = SimpleNamespace( - template={"previousblockhash": tip}, - template_fingerprint="fp", - collection_only=True, - ) - state.active_job = context + The landing brackets added around the reorg walk, the prior-balance + check and the tip-height RPC all run through this stamper, and the + same code paths also execute on client connection threads. If those + stamps counted, a frozen dedicated thread would look alive for as + long as clients kept solving. + """ + server, _state, _ledger = submit_coordinator() + server.watchdog_timeout_seconds = 120.0 + clock = {"now": 1000.0} + owner_ready = threading.Event() + release_owner = threading.Event() - server.min_ready_miners = 1 - server.accepted_share_stats = lambda: (0, 0) # type: ignore[method-assign] - self.assertFalse(server.pool_readiness_latched()) - self.assertFalse(server.client_needs_tip_template_refresh(state, snapshot)) + def frozen_owner() -> None: + server._block_submitter_thread_ident = threading.get_ident() + server._record_heartbeat("block_submitter") + owner_ready.set() + release_owner.wait(5) - server.accepted_share_stats = lambda: (1, 1) # type: ignore[method-assign] - self.assertTrue(server.pool_readiness_latched()) - self.assertTrue(server.client_needs_tip_template_refresh(state, snapshot)) + with patch( + "lab.prism.prism_coordinator.time.monotonic", + side_effect=lambda: clock["now"], + ): + owner = threading.Thread(target=frozen_owner) + owner.start() + try: + self.assertTrue(owner_ready.wait(5)) + clock["now"] += server.watchdog_timeout_seconds + 1.0 + self.assertEqual( + server._overdue_heartbeats(clock["now"]), + ["block_submitter"], + ) + foreign = threading.Thread( + target=lambda: server._record_block_submitter_phase( + "prior-balances-check" + ) + ) + foreign.start() + foreign.join(5) + self.assertEqual( + server._overdue_heartbeats(clock["now"]), + ["block_submitter"], + ) + self.assertNotEqual( + getattr( + server._ensure_block_candidate_service(), + "_block_submitter_phase", + None, + ), + "prior-balances-check", + ) + finally: + release_owner.set() + owner.join(5) - # Readiness is monotonic: once latched the ledger is never consulted - # again, and ready (non-collection) jobs still need no refresh. - server.accepted_share_stats = None # type: ignore[assignment] - self.assertTrue(server.pool_readiness_latched()) - context.collection_only = False - self.assertFalse(server.client_needs_tip_template_refresh(state, snapshot)) + def test_landing_scope_uses_landing_budget_from_first_attempt(self) -> None: + server, _state, _recording = submit_coordinator() + server.block_submit_db_timeout_seconds = 0.01 + server.block_landing_db_timeout_seconds = 45.0 + server.block_landing_db_timeout_max_seconds = 120.0 + scopes: list[float] = [] + @contextlib.contextmanager + def statement_timeout(seconds: float): + scopes.append(seconds) + yield -class LostAckSubmitRpc(FakeRpc): - """qbitd accepts submitted blocks while every submitblock ack is lost. + server.ledger = SimpleNamespace(statement_timeout=statement_timeout) + with server._block_landing_ledger_statement_timeout_scope("ab" * 32): + pass + self.assertEqual(scopes, [45.0]) + metrics = server.block_ledger_call_class_metrics() + self.assertEqual(metrics["landing"]["calls_total"], 1) + self.assertEqual(metrics["landing"]["timeouts_total"], 0) + self.assertEqual(metrics["landing"]["last_budget_seconds"], 45.0) - Models the mainnet 2026-07-25 interleaving: the block connects (the tip - flips to the candidate hash, so blockwait reports it) but the RPC - response never reaches the coordinator, and the retry disposition can - then read a transient racing snapshot in which the tip has moved on and - the header probe cannot prove the candidate active. - """ + def test_landing_scope_timeout_escalates_next_attempt_budget(self) -> None: + server, _state, _recording = submit_coordinator() + # The escalated 90s budget asserted below needs a tolerance of at + # least 180s; the landing budget is otherwise clamped to half the + # configured watchdog. + server.watchdog_timeout_seconds = 300.0 + server.block_landing_db_timeout_seconds = 45.0 + server.block_landing_db_timeout_max_seconds = 120.0 + scopes: list[float] = [] - def __init__(self, *, start_tip: str, hash_by_hex: dict[str, str]) -> None: - self.tip = start_tip - self.height = 9 - self.hash_by_hex = dict(hash_by_hex) - self.active: dict[str, int] = {} - self.racing_tip: str | None = None - self.getblockhash_override: str | None = None - self.lose_acks = True - self.submitblock_calls = 0 - - def call(self, method: str, params: list[object] | None = None) -> object: - if method == "getbestblockhash": - return self.racing_tip or self.tip - if method == "getblockcount": - return self.height - if method == "submitblock": - self.submitblock_calls += 1 - block_hash = self.hash_by_hex[str((params or [""])[0])] - if block_hash in self.active: - return "duplicate" - if self.racing_tip is not None: - return "bad-prevblk" - self.height += 1 - self.active[block_hash] = self.height - self.tip = block_hash - if self.lose_acks: - raise OSError("connection reset by peer before submitblock ack") - return None - if method == "getblockheader": - requested = str((params or [""])[0]).lower() - if self.racing_tip is None and requested in self.active: - block_height = self.active[requested] - return { - "height": block_height, - "confirmations": self.height - block_height + 1, - } - raise RuntimeError("qbit RPC getblockheader failed: -5 Block not found") - if method == "getblockhash": - if self.getblockhash_override is not None: - return self.getblockhash_override - requested_height = int((params or [0])[0]) - for block_hash, block_height in self.active.items(): - if block_height == requested_height: - return block_hash - raise RuntimeError(f"unknown height {requested_height}") - return super().call(method, params) + @contextlib.contextmanager + def statement_timeout(seconds: float): + scopes.append(seconds) + yield + server.ledger = SimpleNamespace(statement_timeout=statement_timeout) + block_hash = "ab" * 32 + with self.assertRaises(TimeoutError): + with server._block_landing_ledger_statement_timeout_scope(block_hash): + raise LedgerOperationTimeout("postgres operation exceeded 45s") + with server._block_landing_ledger_statement_timeout_scope(block_hash): + pass + self.assertEqual(scopes, [45.0, 90.0]) + metrics = server.block_ledger_call_class_metrics() + self.assertEqual(metrics["landing"]["calls_total"], 2) + self.assertEqual(metrics["landing"]["timeouts_total"], 1) + self.assertEqual(metrics["landing"]["last_budget_seconds"], 90.0) -class AcceptanceProbeRpc(FakeRpc): - def __init__( - self, - *, - tip: str, - header: dict[str, object] | None = None, - fail: bool = False, - fail_tip: bool = False, - ) -> None: - self.tip = tip - self.header = header - self.fail = fail - self.fail_tip = fail_tip - - def call(self, method: str, params: list[object] | None = None) -> object: - if self.fail: - raise RuntimeError("qbit RPC unavailable") - if method == "getbestblockhash": - if self.fail_tip: - raise RuntimeError("qbit RPC getbestblockhash unavailable") - return self.tip - if method == "getblockheader": - if self.header is None: - raise RuntimeError("qbit RPC getblockheader failed: -5 Block not found") - return self.header - return super().call(method, params) + def test_landing_scope_ignores_non_ledger_timeouts(self) -> None: + """The scope guards a body that also runs node RPCs; an RPC timeout + is not a database cancellation and must not escalate the next + PostgreSQL budget or fire the landing-timeout alert.""" + server, _state, _recording = submit_coordinator() + server.block_landing_db_timeout_seconds = 45.0 + server.block_landing_db_timeout_max_seconds = 120.0 + scopes: list[float] = [] + @contextlib.contextmanager + def statement_timeout(seconds: float): + scopes.append(seconds) + yield -class PrismCoordinatorAcceptedBlockGapTests(unittest.TestCase): - """Blockwait-first acceptance must survive every abandon-capable path. + server.ledger = SimpleNamespace(statement_timeout=statement_timeout) + block_hash = "ab" * 32 + with self.assertRaises(TimeoutError): + with server._block_landing_ledger_statement_timeout_scope(block_hash): + raise TimeoutError("getblockhash exceeded 1s") + with server._block_landing_ledger_statement_timeout_scope(block_hash): + pass + self.assertEqual(scopes, [45.0, 45.0]) + metrics = server.block_ledger_call_class_metrics() + self.assertEqual(metrics["landing"]["calls_total"], 2) + self.assertEqual(metrics["landing"]["timeouts_total"], 0) - Regression coverage for the accepted-block blind spot left open by #89: - the accepted registry was only written in the direct-accept submit tail, - so a candidate whose submitblock ack was lost -- acceptance arriving via - a blockwait tip observation instead -- could be terminally abandoned as - stale-job by a retry that read one racing chain snapshot. The abandon - withdrew the landed payout preview, fenced payout publication, and left - template refreshes coordination-blocked until the watchdog restart. - """ + def test_fast_class_ledger_call_metrics_record_timeouts(self) -> None: + server, _state, _recording = submit_coordinator() + server.block_submit_db_timeout_seconds = 0.01 + release = threading.Event() - def _accepted_tail_scaffolding(self, server: PrismCoordinator, tempdir: str) -> tuple[list[str], list[str]]: - server._ensure_job_cache_state() - server.reorg_reconciler_enabled = False - server.block_candidate_retry_initial_seconds = 0.0 - server.audit_dir = Path(tempdir) - server.evidence_path = Path(tempdir) / "evidence.json" - server.ledger_writer_public_key_hex = "aa" * 32 - server.build_audit_bundle = ( # type: ignore[method-assign] - lambda **_kwargs: verified_block_bundle() - ) - server.verify_bundle = ( # type: ignore[method-assign] - lambda *_args, **_kwargs: verified_audit_report() - ) - submitted: list[str] = [] - abandoned: list[str] = [] - server.ledger.mark_block_candidate_submitted = ( # type: ignore[attr-defined] - lambda **kwargs: submitted.append(str(kwargs["block_hash"])) or True - ) - server.ledger.mark_block_candidate_abandoned = ( # type: ignore[attr-defined] - lambda **kwargs: abandoned.append(str(kwargs["block_hash"])) or True - ) - return submitted, abandoned + def blocking_operation() -> None: + release.wait(5) + + try: + with self.assertRaises(TimeoutError): + server._run_block_submitter_ledger_call( + ("metrics-key",), + "metrics-test", + blocking_operation, + ) + finally: + release.set() + metrics = server.block_ledger_call_class_metrics() + self.assertEqual(metrics["fast"]["timeouts_total"], 1) + self.assertEqual(metrics["fast"]["last_budget_seconds"], 0.01) + + def test_stuck_ledger_worker_pool_requests_nonzero_restart(self) -> None: + server, _state, _recording = submit_coordinator() + server.block_submit_db_timeout_seconds = 0.01 + server.block_submit_stuck_call_exit_seconds = 0.04 + release = threading.Event() + entered: list[str] = [] + + def blocking_operation(tag: str) -> str: + entered.append(tag) + release.wait(5) + return tag + + try: + for tag in ("one", "two"): + with self.assertRaises(TimeoutError): + server._run_block_submitter_ledger_call( + (tag,), + f"test-{tag}", + lambda tag=tag: blocking_operation(tag), + ) + time.sleep(0.05) + with self.assertRaises(TimeoutError): + server._run_block_submitter_ledger_call( + ("three",), + "test-three", + lambda: blocking_operation("three"), + ) + self.assertEqual(entered, ["one", "two"]) + self.assertTrue(server.stop_event.is_set()) + self.assertTrue(server._fatal_exit_requested) + finally: + release.set() + + def test_one_stuck_ledger_call_does_not_restart_with_spare_capacity(self) -> None: + server, _state, _recording = submit_coordinator() + server.block_submit_db_timeout_seconds = 0.01 + server.block_submit_stuck_call_exit_seconds = 0.04 + release = threading.Event() + entered = threading.Event() + + def blocking_operation() -> None: + entered.set() + release.wait(5) + + try: + with self.assertRaises(TimeoutError): + server._run_block_submitter_ledger_call( + ("same-key",), + "same-key", + blocking_operation, + ) + self.assertTrue(entered.wait(1)) + time.sleep(0.05) + with self.assertRaises(TimeoutError): + server._run_block_submitter_ledger_call( + ("same-key",), + "same-key", + blocking_operation, + ) + self.assertFalse(server.stop_event.is_set()) + self.assertFalse(getattr(server, "_fatal_exit_requested", False)) + finally: + release.set() - def _age_retained_acceptance_past_window( - self, server: PrismCoordinator - ) -> None: - """Backdate any stashed definitive ack past the acceptance window. + def test_two_stuck_ledger_calls_restart_on_existing_key_retry(self) -> None: + server, _state, _recording = submit_coordinator() + server.block_submit_db_timeout_seconds = 0.01 + server.block_submit_stuck_call_exit_seconds = 0.04 + release = threading.Event() + entered: list[str] = [] - These interleavings pin the terminal machinery that runs once - first-party acceptance evidence has gone stale; a fresh stash - correctly vetoes the terminal decision and defers instead (see - test_definitive_ack_with_unprovable_chain_view_defers_not_abandons). - """ - real_stash = server._stash_retained_block_candidate_node_submission + def blocking_operation(tag: str) -> None: + entered.append(tag) + release.wait(5) - def stash_then_age(block_hash: str, node_submission: object) -> None: - real_stash(block_hash, node_submission) - with server.lock: - stamped = getattr( - server, - "_block_candidate_retained_submission_monotonic", - None, + try: + for tag in ("one", "two"): + with self.assertRaises(TimeoutError): + server._run_block_submitter_ledger_call( + (tag,), + tag, + lambda tag=tag: blocking_operation(tag), + ) + time.sleep(0.05) + # No third key is required: a retry reusing either poisoned call + # still observes that every bounded worker slot is exhausted. + with self.assertRaises(TimeoutError): + server._run_block_submitter_ledger_call( + ("one",), + "one", + lambda: blocking_operation("one"), ) - if stamped and block_hash.lower() in stamped: - stamped[block_hash.lower()] -= 400.0 - - server._stash_retained_block_candidate_node_submission = stash_then_age # type: ignore[method-assign] + self.assertEqual(entered, ["one", "two"]) + self.assertTrue(server.stop_event.is_set()) + self.assertTrue(server._fatal_exit_requested) + finally: + release.set() - def _retained_candidate(self, server: PrismCoordinator) -> PrismBlockCandidate: - with server.lock: - candidate = server._retry_block_candidate - server._retry_block_candidate = None - self.assertIsNotNone(candidate) - return candidate + def _ledger_worker_thread(self, phase: str) -> threading.Thread: + """The bounded worker still running one stalled ledger call.""" + name = f"prism-block-ledger-{phase}" + for thread in threading.enumerate(): + if thread.name == name: + return thread + raise AssertionError(f"no bounded ledger worker named {name}") + + def _assert_ledger_registry_drained(self, server: PrismCoordinator) -> None: + """Neither an entry nor a worker slot outlives a finished call.""" + with server._block_submitter_ledger_calls_lock: + self.assertEqual(dict(server._block_submitter_ledger_calls), {}) + acquired = [ + server._block_submitter_ledger_worker_slots.acquire(blocking=False) + for _ in range(MAX_BLOCK_SUBMITTER_STUCK_CALL_WORKERS) + ] + for _ in [slot for slot in acquired if slot]: + server._block_submitter_ledger_worker_slots.release() + self.assertEqual(acquired, [True] * MAX_BLOCK_SUBMITTER_STUCK_CALL_WORKERS) - def _chained_context( + def _drive_late_ledger_call( self, server: PrismCoordinator, - *, - job_id: str, - parent_hash: str, - height: int, - ) -> None: - base = server.jobs["job-1"] - server.jobs[job_id] = SimpleNamespace( - job=SimpleNamespace( - job_id=job_id, - share_target=base.job.share_target, - share_difficulty=base.job.share_difficulty, - transaction_hexes=(), - ), - template={ - "previousblockhash": parent_hash, - "height": height, - "coinbasevalue": 50_00000000, - }, - found_block={"network_difficulty": 1}, - issued_at_ms=12345, - collection_only=False, - worker=base.worker, - shares_json=[], - prior_balances=[], - ) - - def test_blockwait_observed_acceptance_survives_stale_chain_probe(self) -> None: - # The 2026-07-25 mainnet interleaving, end to end: lost submitblock - # ack, blockwait reports the pool's own hash as the new tip, and the - # retry disposition reads a racing snapshot in which the tip moved on - # and the header probe cannot prove the candidate active. The - # candidate must defer (never terminally abandon), keep its landed - # payout preview, and finalize as submitted once the view settles -- - # with no "qbit accepted direct" tail having run before the defer. - parent = "00" * 32 - block_hash = "b1" * 32 - racing_tip = "77" * 32 - server, state, ledger = submit_coordinator(tip=parent) - server.max_blocks = 2 - server.stop_after_block = False - with tempfile.TemporaryDirectory() as tempdir: - submitted, abandoned = self._accepted_tail_scaffolding(server, tempdir) - rpc = LostAckSubmitRpc( - start_tip=parent, - hash_by_hex={"00": block_hash}, - ) - server.rpc = rpc - candidate = block_candidate( - server, - state, - SimpleNamespace( - coinbase_tx_hex="c0ffee", - block_hash_hex=block_hash, - block_hex="00", - share_pass=True, - block_pass=True, - ), - ) - - # Attempt 1: submitblock lands the block but the ack is lost. - self.assertTrue(server._submit_next_block_candidate_writer(candidate)) - self.assertEqual(rpc.submitblock_calls, 1) - self.assertIn(block_hash, server._accepted_block_payout_previews) - self.assertTrue( - server._accepted_block_payout_previews[block_hash].landed - ) - candidate = self._retained_candidate(server) - - # Blockwait sees the pool's own hash as the new tip. This is the - # only acceptance signal: no direct-submit ack ever arrived and - # the accepted success tail has not run. - self.assertTrue(server.observe_tip_for_refresh(block_hash)) - self.assertIn(block_hash, server._tip_observed_accepted_block_hashes) - self.assertNotIn(block_hash, server._accounted_accepted_block_hashes) - - # Attempt 2: the disposition reads a racing snapshot -- tip moved - # past the candidate and the header probe reports not-found. #89 - # abandoned here; the observation evidence must defer instead. - rpc.racing_tip = racing_tip - self.assertTrue(server._submit_next_block_candidate_writer(candidate)) - self.assertNotIn( - PRISM_REJECTION_STALE_JOB, - getattr(server, "block_candidate_abandoned_counts", {}), - ) - self.assertEqual(abandoned, []) - self.assertEqual(server.block_candidate_accept_pending_defer_count, 1) - # The landed preview is preserved -- not withdrawn, not - # tombstoned -- so payout publication stays unfenced and the - # reconcile/template path stays unblocked. - self.assertIn(block_hash, server._accepted_block_payout_previews) - self.assertTrue( - server._accepted_block_payout_previews[block_hash].landed - ) - self.assertNotIn( - block_hash, - server._invalidated_accepted_block_payout_previews, - ) - self.assertFalse(server._payout_state_publication_blocked) - candidate = self._retained_candidate(server) + key: tuple[object, ...], + phase: str, + result: object = None, + error: BaseException | None = None, + ) -> _BlockSubmitterLedgerCall: + """Expire the coordinator-side wait, then let the worker land late. + + Returns the call the registry held, once its worker has fully + exited, so a caller can assert on the registry without racing the + removal the worker performs on its way out. + """ + release = threading.Event() + self.addCleanup(release.set) - # Attempt 3: the chain view settles with the pool's own block as - # the tip; the resume path runs the full accepted success tail - # and finalizes the durable outbox as submitted. - rpc.racing_tip = None - self.assertTrue(server._submit_next_block_candidate_writer(candidate)) + def stall() -> object: + if not release.wait(5): + raise AssertionError("the stalled ledger call was never released") + if error is not None: + raise error + return result - self.assertEqual(submitted, [block_hash]) - self.assertEqual(abandoned, []) - self.assertEqual(rpc.submitblock_calls, 3) - self.assertIn(block_hash, server._accounted_accepted_block_hashes) - self.assertEqual(server.accepted_block_count, 1) - self.assertEqual(len(ledger.persisted), 1) - self.assertEqual(len(ledger.confirmed), 1) - self.assertNotIn(block_hash, server._accepted_block_payout_previews) - self.assertNotIn( - block_hash, - server._invalidated_accepted_block_payout_previews, - ) - self.assertFalse(server._payout_state_publication_blocked) - self.assertIsNone( - server._accepted_block_payout_transition_for_parent( - block_hash, - parent_height=10, - ) - ) - self.assertNotIn( - block_hash, server._outstanding_block_candidate_hashes - ) - self.assertNotIn( - block_hash, server._tip_observed_accepted_block_hashes - ) - self.assertIn( - "qbit_prism_block_candidate_accept_pending_defers_total 1", - server.metrics_payload(), - ) + with self.assertRaises(TimeoutError): + server._run_block_submitter_ledger_call(key, phase, stall) + with server._block_submitter_ledger_calls_lock: + call = server._block_submitter_ledger_calls[key] + worker = self._ledger_worker_thread(phase) + release.set() + worker.join(5) + self.assertFalse(worker.is_alive()) + return call - def test_three_blocks_quick_succession_lost_acks_finalize_all(self) -> None: - # Three pool blocks land back to back (the 20:58 / 21:06:58 / - # 21:07:20 pattern), every submitblock ack is lost, and one retry - # even reads a transient foreign-tip snapshot. All three candidates - # must finalize as submitted with payout publication unfenced. - parent = "00" * 32 - hashes = ["b1" * 32, "b2" * 32, "b3" * 32] - block_hexes = ["0a", "0b", "0c"] - server, state, ledger = submit_coordinator(tip=parent) - server.max_blocks = 5 - server.stop_after_block = False - with tempfile.TemporaryDirectory() as tempdir: - submitted, abandoned = self._accepted_tail_scaffolding(server, tempdir) - rpc = LostAckSubmitRpc( - start_tip=parent, - hash_by_hex=dict(zip(block_hexes, hashes)), - ) - server.rpc = rpc - parents = [parent, hashes[0], hashes[1]] - for index, (block_hash, block_hex) in enumerate( - zip(hashes, block_hexes) - ): - job_id = f"job-chain-{index}" - self._chained_context( - server, - job_id=job_id, - parent_hash=parents[index], - height=10 + index, - ) - candidate = block_candidate( - server, - state, - SimpleNamespace( - coinbase_tx_hex="c0ffee", - block_hash_hex=block_hash, - block_hex=block_hex, - share_pass=True, - block_pass=True, - ), - job_id=job_id, - ) + def _overlapping_ledger_callers( + self, + server: PrismCoordinator, + key: tuple[object, ...], + phase: str, + result: object = None, + error: BaseException | None = None, + count: int = 3, + ) -> tuple[list[object], list[BaseException], list[str]]: + """Park ``count`` callers on one in-flight call, then let it land.""" + started = threading.Event() + release = threading.Event() + self.addCleanup(release.set) + runs: list[str] = [] - # Lost-ack landing. - self.assertTrue( - server._submit_next_block_candidate_writer(candidate) - ) - candidate = self._retained_candidate(server) - # Blockwait-first acceptance. - self.assertTrue(server.observe_tip_for_refresh(block_hash)) + def operation() -> object: + runs.append("run") + started.set() + if not release.wait(5): + raise AssertionError("the shared ledger call was never released") + if error is not None: + raise error + return result - if index == 1: - # One retry reads a transient foreign-tip snapshot in - # which nothing can be proven active; it must defer. - rpc.racing_tip = "77" * 32 - self.assertTrue( - server._submit_next_block_candidate_writer(candidate) - ) - self.assertEqual(abandoned, []) - rpc.racing_tip = None - candidate = self._retained_candidate(server) + waiting: set[int] = set() + waiting_lock = threading.Lock() + original_record_wait = server._record_block_submitter_wait - # Settled view: the resume path finalizes as submitted. - self.assertTrue( - server._submit_next_block_candidate_writer(candidate) - ) - self.assertIn( - block_hash, server._accounted_accepted_block_hashes + def observed_record_wait(observed: str) -> None: + original_record_wait(observed) + if observed == phase: + with waiting_lock: + waiting.add(threading.get_ident()) + + server._record_block_submitter_wait = observed_record_wait # type: ignore[method-assign] + + results: list[object] = [] + failures: list[BaseException] = [] + + def caller() -> None: + try: + results.append( + server._run_block_submitter_ledger_call(key, phase, operation) ) + except BaseException as exc: # pragma: no cover - failure detail + failures.append(exc) - self.assertEqual(submitted, hashes) - self.assertEqual(abandoned, []) - self.assertEqual(server.accepted_block_count, 3) - self.assertEqual( - getattr(server, "block_candidate_abandoned_counts", {}), - {}, + callers = [threading.Thread(target=caller) for _ in range(count)] + callers[0].start() + self.assertTrue(started.wait(5)) + for thread in callers[1:]: + thread.start() + # Every late arrival has to be parked on the one in-flight call + # before it lands, or nothing here is a statement about overlap. + deadline = time.monotonic() + 5 + while time.monotonic() < deadline: + with waiting_lock: + if len(waiting) == count: + break + time.sleep(0.005) + with waiting_lock: + self.assertEqual(len(waiting), count) + release.set() + for thread in callers: + thread.join(5) + self.assertFalse(thread.is_alive()) + return results, failures, runs + + def test_late_ledger_completions_return_the_registry_to_empty(self) -> None: + """Unique keys that time out and then land late must not pile up. + + The shipped collapse write is keyed by the exact page of block + hashes it abandons, so a write that finally lands after the caller's + deadline is the very thing that empties that page out of the outbox: + no later invocation of that key exists to observe the completion and + drop the entry. The registry has to come back to empty by itself, or + every timed-out page leaks a completed call object and its whole + hash tuple for the life of the process. + """ + server, _state, _recording = submit_coordinator() + server.block_submit_db_timeout_seconds = 0.01 + rounds = 6 + sizes: list[int] = [] + for index in range(rounds): + call = self._drive_late_ledger_call( + server, + ("late", index), + f"late-{index}", + result=index, ) - self.assertEqual(len(ledger.persisted), 3) - self.assertEqual(len(ledger.confirmed), 3) - self.assertFalse(server._payout_state_publication_blocked) - self.assertEqual(server._accepted_block_payout_previews, {}) + # The worker really did finish its operation before retiring it. + self.assertEqual(call.result, index) + with server._block_submitter_ledger_calls_lock: + sizes.append(len(server._block_submitter_ledger_calls)) + + self.assertEqual(sizes, [0] * rounds) + # Nothing was retired while it still owed a bounded worker slot. + self._assert_ledger_registry_drained(server) + + def test_a_late_ledger_success_is_retired_and_then_replayed(self) -> None: + """A late success answered nobody, so it leaves nothing behind.""" + server, _state, _recording = submit_coordinator() + server.block_submit_db_timeout_seconds = 0.01 + runs: list[str] = [] + + def fresh() -> str: + runs.append("fresh") + return "fresh-result" + + call = self._drive_late_ledger_call( + server, + ("late-success",), + "late-success", + result="late-result", + ) + self.assertEqual(call.result, "late-result") + self._assert_ledger_registry_drained(server) + # The key is free again, so the next caller replays the idempotent + # operation instead of being answered by a call it never made. + server.block_submit_db_timeout_seconds = 5.0 + self.assertEqual( + server._run_block_submitter_ledger_call( + ("late-success",), + "late-success", + fresh, + ), + "fresh-result", + ) + self.assertEqual(runs, ["fresh"]) + self._assert_ledger_registry_drained(server) + + def test_a_late_ledger_error_is_retired_and_then_replayed(self) -> None: + """A late failure is not owed to a caller that never waited on it.""" + server, _state, _recording = submit_coordinator() + server.block_submit_db_timeout_seconds = 0.01 + failure = RuntimeError("late ledger failure") + runs: list[str] = [] + + def fresh() -> str: + runs.append("fresh") + return "fresh-result" + + call = self._drive_late_ledger_call( + server, + ("late-error",), + "late-error", + error=failure, + ) + self.assertIs(call.error, failure) + self._assert_ledger_registry_drained(server) + server.block_submit_db_timeout_seconds = 5.0 + self.assertEqual( + server._run_block_submitter_ledger_call( + ("late-error",), + "late-error", + fresh, + ), + "fresh-result", + ) + self.assertEqual(runs, ["fresh"]) + self._assert_ledger_registry_drained(server) + + def test_retirement_never_drops_an_unfinished_or_replaced_call(self) -> None: + """The two guards single flight and slot accounting rest on.""" + server, _state, _recording = submit_coordinator() + service = server._ensure_block_candidate_service() + service._ensure_block_submitter_ledger_call_state() + running = _BlockSubmitterLedgerCall() + successor = _BlockSubmitterLedgerCall() + stale = _BlockSubmitterLedgerCall() + stale.done.set() + finished = _BlockSubmitterLedgerCall() + finished.done.set() + with server._block_submitter_ledger_calls_lock: + server._block_submitter_ledger_calls[("running",)] = running + server._block_submitter_ledger_calls[("replaced",)] = successor + server._block_submitter_ledger_calls[("finished",)] = finished + + # A worker that is still out is what same-key single flight and the + # bounded pool are counted from; it is never dropped. + service._retire_finished_block_submitter_ledger_call(("running",), running) + # A finished call cannot drop the fresh call that has taken its key. + service._retire_finished_block_submitter_ledger_call(("replaced",), stale) + # Its own finished entry does go. + service._retire_finished_block_submitter_ledger_call(("finished",), finished) + + with server._block_submitter_ledger_calls_lock: self.assertEqual( - server._invalidated_accepted_block_payout_previews, {} + dict(server._block_submitter_ledger_calls), + {("running",): running, ("replaced",): successor}, ) - self.assertEqual(server._outstanding_block_candidate_hashes, set()) - self.assertEqual(server._tip_observed_accepted_block_hashes, {}) - def test_observed_window_expiry_keeps_genuinely_stale_abandon_terminal(self) -> None: - # A candidate that was once observed as the tip but has stayed off - # the active chain past the observation window is genuinely stale - # (permanently reorged out): the terminal abandon and its payout - # preview withdrawal must still happen. - parent = "00" * 32 - block_hash = "d4" * 32 - server, state, _ledger = submit_coordinator(tip=parent) - server.observed_tip_accept_window_seconds = 60.0 - with tempfile.TemporaryDirectory() as tempdir: - submitted, abandoned = self._accepted_tail_scaffolding(server, tempdir) - rpc = LostAckSubmitRpc( - start_tip=parent, - hash_by_hex={"00": block_hash}, - ) - server.rpc = rpc + def test_overlapping_same_key_waiters_share_one_bounded_worker(self) -> None: + """Single flight is unchanged: one operation, one slot, one answer.""" + server, _state, _recording = submit_coordinator() + server.block_submit_db_timeout_seconds = 5.0 + results, failures, runs = self._overlapping_ledger_callers( + server, + ("shared",), + "shared", + result="shared-result", + ) + self.assertEqual(failures, []) + self.assertEqual(runs, ["run"]) + self.assertEqual(results, ["shared-result"] * 3) + self._assert_ledger_registry_drained(server) + + def test_overlapping_same_key_waiters_all_receive_the_failure(self) -> None: + """Retiring the entry cannot cost a parked waiter what it is owed.""" + server, _state, _recording = submit_coordinator() + server.block_submit_db_timeout_seconds = 5.0 + failure = RuntimeError("shared ledger failure") + results, failures, runs = self._overlapping_ledger_callers( + server, + ("shared-error",), + "shared-error", + error=failure, + ) + self.assertEqual(results, []) + self.assertEqual(runs, ["run"]) + self.assertEqual(failures, [failure] * 3) + self._assert_ledger_registry_drained(server) + + def test_parent_retry_displacement_preserves_durable_descendant(self) -> None: + server, state, _recording = submit_coordinator() + + def candidate_at(tag: str, height: int) -> PrismBlockCandidate: candidate = block_candidate( server, state, SimpleNamespace( - coinbase_tx_hex="c0ffee", - block_hash_hex=block_hash, - block_hex="00", + coinbase_tx_hex="00", + block_hash_hex=tag * 32, + block_hex=tag, share_pass=True, block_pass=True, ), ) - self.assertTrue(server._submit_next_block_candidate_writer(candidate)) - candidate = self._retained_candidate(server) - self.assertTrue(server.observe_tip_for_refresh(block_hash)) - - # The chain moved on without the candidate and the observation - # aged out: the block was reorged away for good. - rpc.racing_tip = "77" * 32 - rpc.active.pop(block_hash, None) - with server.lock: - server._tip_observed_accepted_block_hashes[block_hash] = ( - time.monotonic() - 61.0 - ) - - self.assertTrue(server._submit_next_block_candidate_writer(candidate)) - - self.assertEqual(abandoned, [block_hash]) - self.assertEqual(submitted, []) - self.assertEqual( - server.block_candidate_abandoned_counts[PRISM_REJECTION_STALE_JOB], - 1, + context = SimpleNamespace(**vars(candidate.context)) + context.template = {**candidate.context.template, "height": height} + return dataclass_replace( + candidate, + context=context, + durable_replay=True, ) - self.assertNotIn(block_hash, server._accepted_block_payout_previews) - self.assertNotIn(block_hash, server._accounted_accepted_block_hashes) - - def test_block_candidate_acceptance_pending_probe_semantics(self) -> None: - block_hash = "e5" * 32 - server, _state, _ledger = submit_coordinator() - server._ensure_job_cache_state() - server.observed_tip_accept_window_seconds = 60.0 - server._register_outstanding_block_candidate(block_hash) - - def observed(age_seconds: float | None) -> None: - with server.lock: - server._tip_observed_accepted_block_hashes.pop(block_hash, None) - if age_seconds is not None: - server._tip_observed_accepted_block_hashes[block_hash] = ( - time.monotonic() - age_seconds - ) - # Own hash is the fresh best tip: accepted, and the probe itself - # registers the observation for later checks. - observed(None) - server.rpc = AcceptanceProbeRpc(tip=block_hash) - self.assertTrue(server._block_candidate_acceptance_pending(block_hash)) - self.assertIn(block_hash, server._tip_observed_accepted_block_hashes) + descendant = candidate_at("b8", 11) + parent = candidate_at("a8", 10) + descendant_hash = descendant.submission.block_hash_hex + server._ensure_block_candidate_disposition_state() + server._ensure_block_replay_state() + server._block_replay_inflight_hashes.add(descendant_hash) + with server.lock: + server._retry_block_candidate = descendant + server._merge_block_candidate_retry_locked( + "_retry_block_candidate", + parent, + ) - # Active header at the expected height: accepted. - observed(None) - server.rpc = AcceptanceProbeRpc( - tip="11" * 32, - header={"height": 10, "confirmations": 2}, + self.assertIs(server._retry_block_candidate, parent) + self.assertIs( + server._block_disposition_waiting_retries[descendant_hash], + descendant, ) - self.assertTrue( - server._block_candidate_acceptance_pending( - block_hash, expected_height=10 + + # Once the parent reaches a terminal state, the descendant's preserved + # wakeup is selected even though durable replay still deduplicates it. + with server.lock: + server._retry_block_candidate = None + captured: list[object] = [] + server.max_blocks = 10 + server.stop_after_block = False + server._node_submission_for_candidate = ( # type: ignore[method-assign] + lambda _candidate: SimpleNamespace( + attempted=True, + result="duplicate", + error=None, ) ) + server._enqueue_block_accounting_task = ( # type: ignore[method-assign] + lambda task: (captured.append(task), True)[1] + ) - # Provably active at the wrong height: abandonable even when the - # hash was recently observed. - observed(1.0) - server.rpc = AcceptanceProbeRpc( - tip="11" * 32, - header={"height": 9, "confirmations": 2}, + self.assertTrue(server.submit_next_block_candidate(defer_accounting=True)) + self.assertEqual(len(captured), 1) + task = captured[0] + self.assertIs(task.candidate, descendant) + server._release_block_candidate_disposition(task.disposition_lease) + + def test_synchronous_waiter_joins_finalize_only_registry(self) -> None: + server, state, _recording = submit_coordinator() + candidate = block_candidate( + server, + state, + SimpleNamespace( + coinbase_tx_hex="00", + block_hash_hex="c8" * 32, + block_hex="c8", + share_pass=False, + block_pass=True, + ), + credit_share_on_accept=True, ) - self.assertFalse( - server._block_candidate_acceptance_pending( - block_hash, expected_height=10 + block_hash = candidate.submission.block_hash_hex + server._block_candidate_finalize_retries = {} + server._block_candidate_finalize_retries[block_hash] = (True, "") + server._node_submission_for_candidate = ( # type: ignore[method-assign] + lambda _candidate: (_ for _ in ()).throw( + AssertionError("finalize-only retry must not call qbitd") ) ) + finalized: list[dict[str, object]] = [] - # Unprovable probe + fresh observation: accepted (defer). - observed(1.0) - server.rpc = AcceptanceProbeRpc(tip="11" * 32, header=None) - self.assertTrue( - server._block_candidate_acceptance_pending( - block_hash, expected_height=10 - ) + def finalize( + _candidate: PrismBlockCandidate, + **kwargs: object, + ) -> bool: + finalized.append(kwargs) + return True + + server._finalize_block_candidate = finalize # type: ignore[method-assign] + + self.assertTrue(server._submit_synchronous_block_candidate(candidate)) + self.assertEqual(len(finalized), 1) + self.assertTrue(finalized[0]["accepted"]) + self.assertEqual(finalized[0]["block_hash"], block_hash) + + def test_synchronous_waiter_preserves_finalize_only_abandonment(self) -> None: + server, state, _recording = submit_coordinator() + candidate = block_candidate( + server, + state, + SimpleNamespace( + coinbase_tx_hex="00", + block_hash_hex="c9" * 32, + block_hex="c9", + share_pass=False, + block_pass=True, + ), + credit_share_on_accept=True, ) - - # Unprovable probe + expired observation: genuinely stale. - observed(61.0) - self.assertFalse( - server._block_candidate_acceptance_pending( - block_hash, expected_height=10 + block_hash = candidate.submission.block_hash_hex + server._block_candidate_finalize_retries = { + block_hash: (False, "terminal rejection") + } + server._node_submission_for_candidate = ( # type: ignore[method-assign] + lambda _candidate: (_ for _ in ()).throw( + AssertionError("finalize-only retry must not call qbitd") ) ) - - # Unprovable probe + no observation: genuinely stale. - observed(None) - self.assertFalse( - server._block_candidate_acceptance_pending( - block_hash, expected_height=10 - ) + finalized: list[dict[str, object]] = [] + server._finalize_block_candidate = ( # type: ignore[method-assign] + lambda _candidate, **kwargs: (finalized.append(kwargs), True)[1] ) - # Probe failure + fresh observation: acceptance evidence wins. - observed(1.0) - server.rpc = AcceptanceProbeRpc(tip="11" * 32, fail=True) - self.assertTrue( - server._block_candidate_acceptance_pending( - block_hash, expected_height=10 - ) - ) + self.assertFalse(server._submit_synchronous_block_candidate(candidate)) + self.assertEqual(len(finalized), 1) + self.assertFalse(finalized[0]["accepted"]) + self.assertEqual(finalized[0]["error"], "terminal rejection") - # Probe failure + no observation: unchanged legacy behavior. - observed(None) - self.assertFalse( - server._block_candidate_acceptance_pending( - block_hash, expected_height=10 - ) + def test_synchronous_abandon_rejects_prepared_state_before_outbox(self) -> None: + server, state, ledger = submit_coordinator() + block_hash = "ca" * 32 + candidate = block_candidate( + server, + state, + SimpleNamespace( + coinbase_tx_hex="00", + block_hash_hex=block_hash, + block_hex="ca", + share_pass=False, + block_pass=True, + ), + credit_share_on_accept=True, ) + events: list[str] = [] + prepared = {"value": True} - # A best-tip lookup failure must not suppress the active-header - # probe: the header alone proves acceptance, with no observation. - observed(None) - server.rpc = AcceptanceProbeRpc( - tip="11" * 32, - header={"height": 10, "confirmations": 2}, - fail_tip=True, - ) - self.assertTrue( - server._block_candidate_acceptance_pending( - block_hash, expected_height=10 - ) - ) + def pool_block_state(*, block_hash: str) -> dict[str, str]: + self.assertEqual(block_hash, candidate.submission.block_hash_hex) + events.append("state") + return { + "chain_state": "prepared" if prepared["value"] else "rejected", + "maturity_state": "immature", + } - # ... and the header's wrong-height verdict also stands alone, - # overriding even a fresh observation. - observed(1.0) - server.rpc = AcceptanceProbeRpc( - tip="11" * 32, - header={"height": 9, "confirmations": 2}, - fail_tip=True, - ) - self.assertFalse( - server._block_candidate_acceptance_pending( - block_hash, expected_height=10 - ) - ) + def reject_prepared_block(**_kwargs: object) -> dict[str, object]: + events.append("reject") + prepared["value"] = False + return {"backend": "fake", "rejected_count": 1} - def test_post_persist_stale_view_defers_before_rejecting_prepared_rows(self) -> None: - # Codex P1 / Bugbot: the post-persistence active-hash check could - # reject the prepared payout rows and only then reach the abandon, - # which now defers on acceptance evidence -- leaving rows - # rejected/reversed that a later confirmation can never promote. - # The acceptance check must run BEFORE reject_prepared_block. - parent = "00" * 32 - block_hash = "c7" * 32 - racing_winner = "77" * 32 - server, state, ledger = submit_coordinator(tip=parent) - server.max_blocks = 2 - server.stop_after_block = False - with tempfile.TemporaryDirectory() as tempdir: - submitted, abandoned = self._accepted_tail_scaffolding(server, tempdir) - rpc = LostAckSubmitRpc( - start_tip=parent, - hash_by_hex={"00": block_hash}, - ) - server.rpc = rpc - candidate = block_candidate( - server, - state, - SimpleNamespace( - coinbase_tx_hex="c0ffee", - block_hash_hex=block_hash, - block_hex="00", - share_pass=True, - block_pass=True, - ), - ) - self.assertTrue(server._submit_next_block_candidate_writer(candidate)) - candidate = self._retained_candidate(server) - self.assertTrue(server.observe_tip_for_refresh(block_hash)) + def mark_abandoned(**_kwargs: object) -> bool: + events.append("abandon") + return True - # Retry resumes on the own-hash tip, but the post-persist - # getblockhash read races to a foreign winner for the height. - rpc.getblockhash_override = racing_winner - self.assertTrue(server._submit_next_block_candidate_writer(candidate)) + ledger.pool_block_state = pool_block_state # type: ignore[attr-defined] + ledger.reject_prepared_block = reject_prepared_block # type: ignore[method-assign] + ledger.mark_block_candidate_abandoned = mark_abandoned # type: ignore[attr-defined] - # Deferred without touching the prepared payout rows. - self.assertEqual(ledger.rejected, []) - self.assertEqual(abandoned, []) - self.assertNotIn( - "block-stale", - getattr(server, "block_candidate_abandoned_counts", {}), + class CleanupRpc(TipRpc): + def call( + self, + method: str, + params: list[object] | None = None, + ) -> object: + if method == "getblockcount": + return 9 + return super().call(method, params) + + server.rpc = CleanupRpc("00" * 32) + server._node_submission_for_candidate = ( # type: ignore[method-assign] + lambda _candidate: SimpleNamespace( + attempted=False, + result=None, + error=None, ) - candidate = self._retained_candidate(server) + ) + server._mark_block_candidate_attempted = ( # type: ignore[method-assign] + lambda _block_hash: True + ) - # The view settles with the pool's block active again: the - # replayed tail confirms the still-prepared rows. - rpc.getblockhash_override = None - self.assertTrue(server._submit_next_block_candidate_writer(candidate)) - self.assertEqual(submitted, [block_hash]) - self.assertEqual(ledger.rejected, []) - self.assertEqual(len(ledger.confirmed), 1) - self.assertIn(block_hash, server._accounted_accepted_block_hashes) + def stage_terminal_rejection( + _candidate: PrismBlockCandidate, + *, + node_submission: object, + ) -> bool: + self.assertIsNotNone(node_submission) + outcome = server._block_candidate_outcome + outcome.reason = PRISM_REJECTION_POOL_CLOSED + outcome.error = "pool is no longer accepting blocks" + outcome.stale_job_class = None + return False - def test_post_persist_stale_view_still_rejects_without_acceptance_evidence(self) -> None: - # The terminal half of the same site: submitblock succeeds but a - # sibling steals the height right after persistence and the pool's - # block was never observed as the tip. With no FRESH acceptance - # evidence (the retained ack has aged past the acceptance window), - # the terminal decision seals first and only then are the prepared - # rows rejected, exactly once. - parent = "00" * 32 - block_hash = "c8" * 32 - racing_winner = "77" * 32 - server, state, ledger = submit_coordinator(tip=parent) - server.max_blocks = 2 - server.stop_after_block = False - with tempfile.TemporaryDirectory() as tempdir: - submitted, abandoned = self._accepted_tail_scaffolding(server, tempdir) - rpc = LostAckSubmitRpc( - start_tip=parent, - hash_by_hex={"00": block_hash}, - ) - rpc.lose_acks = False - server.rpc = rpc - # The definitive ack retained at the offer would veto the - # terminal decision while fresh; these tests pin the sealed - # terminal machinery, so age it past the acceptance window. - self._age_retained_acceptance_past_window(server) - candidate = block_candidate( - server, - state, - SimpleNamespace( - coinbase_tx_hex="c0ffee", - block_hash_hex=block_hash, - block_hex="00", - share_pass=True, - block_pass=True, - ), - ) - real_persist = ledger.persist_accepted_block + server._submit_block_candidate_serialized = ( # type: ignore[method-assign] + stage_terminal_rejection + ) - def persist_then_lose_race(**kwargs: object) -> dict[str, object]: - result = real_persist(**kwargs) - # A sibling wins the height after persistence: the tip and - # the height's active hash both belong to the foreign winner - # and the pool's block never became the chain tip. - rpc.tip = racing_winner - rpc.active.pop(block_hash, None) - rpc.getblockhash_override = racing_winner - return result + self.assertFalse(server._submit_synchronous_block_candidate(candidate)) + self.assertEqual(events, ["state", "reject", "abandon"]) + self.assertEqual( + server.block_candidate_abandoned_counts, + {PRISM_REJECTION_POOL_CLOSED: 1}, + ) + self.assertFalse(server._block_candidate_terminal_outcome(block_hash)) + + def test_synchronous_cleanup_failure_keeps_candidate_pending(self) -> None: + server, state, ledger = submit_coordinator() + block_hash = "cb" * 32 + candidate = block_candidate( + server, + state, + SimpleNamespace( + coinbase_tx_hex="00", + block_hash_hex=block_hash, + block_hex="cb", + share_pass=False, + block_pass=True, + ), + credit_share_on_accept=True, + ) + events: list[str] = [] + ledger.pool_block_state = ( # type: ignore[attr-defined] + lambda *, block_hash: { + "chain_state": "prepared", + "maturity_state": "immature", + } + ) - ledger.persist_accepted_block = persist_then_lose_race # type: ignore[method-assign] + def fail_reject(**_kwargs: object) -> dict[str, object]: + events.append("reject") + raise RuntimeError("postgres unavailable") - self.assertTrue(server._submit_next_block_candidate_writer(candidate)) + def unexpected_abandon(**_kwargs: object) -> bool: + events.append("abandon") + return True - self.assertEqual(len(ledger.rejected), 1) - self.assertEqual(abandoned, [block_hash]) - self.assertEqual(submitted, []) - self.assertEqual( - server.block_candidate_abandoned_counts["block-stale"], - 1, - ) - self.assertNotIn(block_hash, server._accounted_accepted_block_hashes) + ledger.reject_prepared_block = fail_reject # type: ignore[method-assign] + ledger.mark_block_candidate_abandoned = unexpected_abandon # type: ignore[attr-defined] - def test_acceptance_evidence_arriving_during_withdrawal_defers(self) -> None: - # Bugbot: the payout-preview withdrawal inside the abandon can block - # long enough for a blockwait observation to arrive; the terminal - # commitment must consult that late evidence, not only the - # completed-tail registry. - block_hash = "a9" * 32 - server, _state, _ledger = submit_coordinator() - server._ensure_job_cache_state() - server._register_outstanding_block_candidate(block_hash) - server._begin_accepted_block_payout_preview(block_hash, block_height=10) - server._mark_accepted_block_payout_landed(block_hash, block_height=10) - server.rpc = AcceptanceProbeRpc(tip="11" * 32, header=None) - real_clear = server._clear_accepted_block_payout_preview + class CleanupRpc(TipRpc): + def call( + self, + method: str, + params: list[object] | None = None, + ) -> object: + if method == "getblockcount": + return 9 + return super().call(method, params) - def observing_clear( - hash_arg: str, - *, - invalidate_published: bool = False, - ) -> None: - if invalidate_published: - # A blockwait observation lands while the withdrawal blocks. - with server.lock: - server._tip_observed_accepted_block_hashes[block_hash] = ( - time.monotonic() - ) - return real_clear( - hash_arg, - invalidate_published=invalidate_published, + server.rpc = CleanupRpc("00" * 32) + server._node_submission_for_candidate = ( # type: ignore[method-assign] + lambda _candidate: SimpleNamespace( + attempted=False, + result=None, + error=None, ) + ) + server._mark_block_candidate_attempted = ( # type: ignore[method-assign] + lambda _block_hash: True + ) - server._clear_accepted_block_payout_preview = observing_clear # type: ignore[method-assign] + def stage_terminal_rejection( + _candidate: PrismBlockCandidate, + *, + node_submission: object, + ) -> bool: + self.assertIsNotNone(node_submission) + outcome = server._block_candidate_outcome + outcome.reason = PRISM_REJECTION_POOL_CLOSED + outcome.error = "pool is no longer accepting blocks" + outcome.stale_job_class = None + return False - accepted_race_won = server._abandon_block_candidate( - PRISM_REJECTION_STALE_JOB, - "tip moved before submit: test", - block_hash=block_hash, - worker=None, - preserve_if_accepted=True, - expected_height=10, + server._submit_block_candidate_serialized = ( # type: ignore[method-assign] + stage_terminal_rejection ) - self.assertFalse(accepted_race_won) - outcome = getattr(server, "_block_candidate_outcome", None) + with self.assertRaisesRegex( + RuntimeError, + "could not reject prepared state for terminal candidate", + ): + server._submit_synchronous_block_candidate(candidate) + + self.assertEqual(events, ["reject"]) + self.assertIs(server._retry_block_candidate, candidate) + self.assertEqual(server.block_candidate_abandoned_counts, {}) + self.assertIsNone(server._block_candidate_terminal_outcome(block_hash)) self.assertEqual( - getattr(outcome, "reason", None), - PRISM_REJECTION_BLOCK_ACCEPT_PENDING, - ) - self.assertNotIn( - PRISM_REJECTION_STALE_JOB, - getattr(server, "block_candidate_abandoned_counts", {}), - ) - self.assertEqual(server.block_candidate_accept_pending_defer_count, 1) - # The landed barrier is restored (and its fail-closed tombstone - # popped) so descendant builders wait on the preview instead of - # failing closed while the deferred retry heals it. - self.assertIn(block_hash, server._accepted_block_payout_previews) - self.assertTrue( - server._accepted_block_payout_previews[block_hash].landed - ) - self.assertNotIn( - block_hash, - server._invalidated_accepted_block_payout_previews, + server._block_candidate_outcome.reason, + PRISM_REJECTION_BACKEND_RPC_UNAVAILABLE, ) - def test_replay_restores_acceptance_evidence_from_durable_block_state(self) -> None: - # Codex round 5: the observation registry dies with the process. A - # durable prepared/confirmed pool-block row proves a prior process's - # submitblock succeeded, so startup replay must restore acceptance - # evidence before a replay disposition can race a transient fork - # view and terminally abandon the accepted block. + def test_restart_resubmit_honors_terminal_abandoned_outbox(self) -> None: server, state, _recording = submit_coordinator() ledger = SingleWriterShareLedger() server.ledger = ledger - accepted_hash = "e7" * 32 - unproven_hash = "e8" * 32 - unreadable_hash = "e9" * 32 - for index, block_hash in enumerate( - (accepted_hash, unproven_hash, unreadable_hash), start=1 - ): - pending = PendingShare( - share_id=f"miner-a:{block_hash}", - miner_id="miner-a", - order_key="miner-a", - p2mr_program_hex="11" * 32, - share_difficulty=1, - network_difficulty=1, - template_height=9, - job_id="job-1", - job_issued_at_ms=1, - accepted_at_ms=index, - ntime=1, - ) - candidate = block_candidate( - server, - state, - SimpleNamespace( - coinbase_tx_hex="00", - block_hash_hex=block_hash, - block_hex="00", - share_pass=True, - block_pass=True, - ), - pending_share=pending, + block_hash = "e8" * 32 + submission = SimpleNamespace( + header_hex="aa" * 80, + coinbase_tx_hex="00", + block_hash_hex=block_hash, + block_hex="e8", + share_pass=False, + block_pass=True, + ) + first_pending = server.pending_share_from_submission( + context=server.jobs["job-1"], + submission=submission, + ntime_hex="00000001", + ) + first_candidate = block_candidate( + server, + state, + submission, + pending_share=first_pending, + credit_share_on_accept=True, + ) + self.assertTrue( + ledger.persist_block_candidate_intent( + server.block_candidate_intent(first_candidate) ) - ledger.append_batch( - [(pending, server.block_candidate_intent(candidate))] + ) + self.assertTrue( + ledger.mark_block_candidate_abandoned( + block_hash=block_hash, + error="terminal before restart", ) - def durable_block_state(*, block_hash: str) -> dict[str, object] | None: - if block_hash == accepted_hash: - return {"chain_state": "prepared", "maturity_state": "immature"} - if block_hash == unreadable_hash: - raise RuntimeError("postgres unavailable") - return None + ) + server._finish_pending_share_commit(first_pending) + submitblock_calls: list[str] = [] - ledger.pool_block_state = durable_block_state # type: ignore[attr-defined] + class NoResubmitRpc(TipRpc): + def call( + self, + method: str, + params: list[object] | None = None, + *, + timeout: float | None = None, + ) -> object: + if method == "submitblock": + submitblock_calls.append(str((params or [""])[0])) + return None + return super().call(method, params) - self.assertEqual(server.replay_pending_block_candidates(), 3) - replayed = [ - server._block_replay_candidate_queue.get_nowait() - for _ in range(3) - ] - for candidate in replayed: - server._restore_replayed_candidate_acceptance_evidence(candidate) + server.rpc = NoResubmitRpc("00" * 32) + with patch( + "lab.prism.prism_coordinator.direct_stratum.assemble_submission", + return_value=submission, + ): + with self.assertRaises(StratumError) as raised: + server.handle_submit( + state, + ["miner-a", "job-1", "00" * 8, "00000001", "00000002"], + ) - self.assertIn(accepted_hash, server._tip_observed_accepted_block_hashes) - self.assertIn(accepted_hash, server._outstanding_block_candidate_hashes) - # No durable acceptance proof: replays without synthetic evidence. - self.assertNotIn( - unproven_hash, server._tip_observed_accepted_block_hashes + self.assertEqual(raised.exception.code, 23) + self.assertEqual(submitblock_calls, []) + self.assertEqual(ledger._block_candidate_outbox[block_hash]["state"], "abandoned") + self.assertEqual(len(ledger), 0) + + def test_restart_resubmit_coalesces_terminal_submitted_outbox(self) -> None: + first_server, first_state, _recording = submit_coordinator() + ledger = SingleWriterShareLedger() + first_server.ledger = ledger + block_hash = "f8" * 32 + submission = SimpleNamespace( + header_hex="ab" * 80, + coinbase_tx_hex="00", + block_hash_hex=block_hash, + block_hex="f8", + share_pass=True, + block_pass=True, ) - # An unreadable durable state fails safe: the candidate replays - # protected, bounded by the observation window, instead of exposed - # to a transient fork view racing its first disposition. - self.assertIn( - unreadable_hash, server._tip_observed_accepted_block_hashes + first_pending = PendingShare( + share_id=f"miner-a:{block_hash}", + miner_id="miner-a", + order_key="miner-a", + p2mr_program_hex="11" * 32, + share_difficulty=7, + network_difficulty=1, + template_height=9, + job_id="job-1", + job_issued_at_ms=12345, + accepted_at_ms=1, + ntime=1, + ) + first_candidate = block_candidate( + first_server, + first_state, + submission, + pending_share=first_pending, + ) + first_record = ledger.append_batch( + [ + ( + first_pending, + first_server.block_candidate_intent(first_candidate), + ) + ] + )[0] + self.assertTrue(first_record.newly_inserted) + self.assertTrue( + ledger.mark_block_candidate_submitted(block_hash=block_hash) ) - def test_late_defer_republishes_withdrawn_preview_and_unfences(self) -> None: - # Bugbot round 5: when the withdrawal's own publication loses its - # race, delivery is fenced before the late-acceptance defer runs. - # The defer must republish the withdrawn preview so admission - # reopens immediately instead of staying coordination-blocked - # across the deferral cycles. - block_hash = "ba" * 32 - server, _state, _ledger = submit_coordinator() - server._ensure_job_cache_state() - server._register_outstanding_block_candidate(block_hash) - server._begin_accepted_block_payout_preview(block_hash, block_height=10) - server._mark_accepted_block_payout_landed(block_hash, block_height=10) - server._publish_accepted_block_payout_preview(block_hash, []) - self.assertFalse(server._payout_state_publication_blocked) - server.rpc = AcceptanceProbeRpc(tip="11" * 32, header=None) - - real_publish = server._publish_payout_state_candidate - - def withdrawal_publication_lost(candidate: object) -> object: - if getattr(candidate, "accepted_block_withdrawal", False): - return None - return real_publish(candidate) + # Model a clean restart: volatile duplicate/disposition state is gone, + # while the ledger and terminal candidate outbox remain authoritative. + server, state, _recording = submit_coordinator() + server.ledger = ledger + submitblock_calls: list[str] = [] - server._publish_payout_state_candidate = withdrawal_publication_lost # type: ignore[method-assign] - real_clear = server._clear_accepted_block_payout_preview + class NoResubmitRpc(TipRpc): + def call( + self, + method: str, + params: list[object] | None = None, + *, + timeout: float | None = None, + ) -> object: + if method == "submitblock": + submitblock_calls.append(str((params or [""])[0])) + return None + return super().call(method, params) - def observing_clear( - hash_arg: str, - *, - invalidate_published: bool = False, - ) -> None: - if invalidate_published: - with server.lock: - server._tip_observed_accepted_block_hashes[block_hash] = ( - time.monotonic() - ) - return real_clear( - hash_arg, - invalidate_published=invalidate_published, + server.rpc = NoResubmitRpc("00" * 32) + with patch( + "lab.prism.prism_coordinator.direct_stratum.assemble_submission", + return_value=submission, + ): + self.assertFalse( + server.handle_submit( + state, + ["miner-a", "job-1", "00" * 8, "00000001", "00000002"], + ) ) - server._clear_accepted_block_payout_preview = observing_clear # type: ignore[method-assign] + self.assertEqual(submitblock_calls, []) + self.assertEqual( + ledger._block_candidate_outbox[block_hash]["state"], "submitted" + ) + self.assertEqual(len(ledger), 1) + self.assertEqual(ledger._shares[0].accepted_at_ms, 1) + self.assertTrue(server.block_candidate_queue.empty()) + self.assertEqual( + server.worker_share_counts["miner-a"], + {"submitted": 1, "accepted": 0, "grace": 0}, + ) - accepted_race_won = server._abandon_block_candidate( - PRISM_REJECTION_STALE_JOB, - "tip moved before submit: test", - block_hash=block_hash, - worker=None, - preserve_if_accepted=True, - expected_height=10, + def test_exact_share_replay_does_not_repeat_process_credit(self) -> None: + server, state, _recording = submit_coordinator() + ledger = SingleWriterShareLedger() + server.ledger = ledger + block_hash = "d8" * 32 + submission = SimpleNamespace( + coinbase_tx_hex="00", + block_hash_hex=block_hash, + block_hex="d8", + share_pass=True, + block_pass=True, + ) + pending = PendingShare( + share_id=f"miner-a:{block_hash}", + miner_id="miner-a", + order_key="miner-a", + p2mr_program_hex="11" * 32, + share_difficulty=1, + network_difficulty=1, + template_height=9, + job_id="job-1", + job_issued_at_ms=12345, + accepted_at_ms=12346, + ntime=1, + ) + candidate = block_candidate( + server, + state, + submission, + pending_share=pending, + ) + intent = server.block_candidate_intent(candidate) + worker_credits: list[str] = [] + vardiff_credits: list[str] = [] + server.note_worker_accepted_share = ( # type: ignore[method-assign] + lambda worker, _policy: worker_credits.append(worker) + ) + server.note_vardiff_accepted_share = ( # type: ignore[method-assign] + lambda _client, job: vardiff_credits.append(job.job_id) ) - self.assertFalse(accepted_race_won) - outcome = getattr(server, "_block_candidate_outcome", None) self.assertEqual( - getattr(outcome, "reason", None), - PRISM_REJECTION_BLOCK_ACCEPT_PENDING, + server.append_accepted_share( + state, + candidate.context, + submission, + pending, + candidate_intent=intent, + ), + "pending", ) - # The barrier is restored AND publication admission reopened: the - # withdrawn preview was republished despite the withdrawal's own - # publication loss having fenced delivery. - self.assertIn(block_hash, server._accepted_block_payout_previews) - self.assertTrue( - server._accepted_block_payout_previews[block_hash].landed + self.assertEqual( + server.append_accepted_share( + state, + candidate.context, + submission, + pending, + candidate_intent=intent, + ), + "pending", ) - self.assertIsNotNone( - server._accepted_block_payout_previews[block_hash].preview + + self.assertEqual(len(ledger), 1) + self.assertEqual(worker_credits, ["miner-a"]) + self.assertEqual(vardiff_credits, ["job-1"]) + + def test_same_hash_busy_lease_preserves_retry_behind_live_work(self) -> None: + server, state, _recording = submit_coordinator() + server.block_candidate_retry_initial_seconds = 0.0 + candidate = block_candidate( + server, + state, + SimpleNamespace( + coinbase_tx_hex="00", + block_hash_hex="a7" * 32, + block_hex="a7", + share_pass=True, + block_pass=True, + ), ) - self.assertNotIn( + block_hash = candidate.submission.block_hash_hex + lease = server._claim_block_candidate_disposition( block_hash, - server._invalidated_accepted_block_payout_previews, + blocking=True, ) - self.assertFalse(server._payout_state_publication_blocked) - - def test_late_gate_reprobes_after_withdrawal_heals_buried_block(self) -> None: - # Bugbot round 7: an unknown pre-withdrawal probe verdict must - # re-probe after the (blocking) withdrawal. A buried accepted block - # is not always re-observed as the tip -- blockwait reports only the - # newest of rapid connects -- so the recovered probe alone, with no - # observation evidence at all, must defer the terminal abandon. - block_hash = "bc" * 32 - server, _state, _ledger = submit_coordinator() - server._ensure_job_cache_state() - server._register_outstanding_block_candidate(block_hash) - server._begin_accepted_block_payout_preview(block_hash, block_height=10) - server._mark_accepted_block_payout_landed(block_hash, block_height=10) - rpc = AcceptanceProbeRpc(tip="11" * 32, header=None) - server.rpc = rpc - real_clear = server._clear_accepted_block_payout_preview + assert lease is not None + server._retry_block_candidate = candidate + server._ensure_block_replay_state() + server._block_replay_inflight_hashes.add(block_hash) + try: + self.assertTrue( + server.submit_next_block_candidate(defer_accounting=True) + ) + self.assertIsNone(server._retry_block_candidate) + self.assertIs( + server._block_disposition_waiting_retries[block_hash], + candidate, + ) + finally: + server._release_block_candidate_disposition(lease) - def healing_clear( - hash_arg: str, - *, - invalidate_published: bool = False, - ) -> None: - if invalidate_published: - # The chain view heals while the withdrawal blocks: the - # candidate is provably active again, two blocks deep. - rpc.header = {"height": 10, "confirmations": 2} - return real_clear( - hash_arg, - invalidate_published=invalidate_published, + captured: list[object] = [] + server._node_submission_for_candidate = ( # type: ignore[method-assign] + lambda _candidate: SimpleNamespace( + attempted=True, + result="duplicate", + error=None, ) - - server._clear_accepted_block_payout_preview = healing_clear # type: ignore[method-assign] - - accepted_race_won = server._abandon_block_candidate( - PRISM_REJECTION_STALE_JOB, - "tip moved before submit: test", - block_hash=block_hash, - worker=None, - preserve_if_accepted=True, - expected_height=10, ) - - self.assertFalse(accepted_race_won) - outcome = getattr(server, "_block_candidate_outcome", None) - self.assertEqual( - getattr(outcome, "reason", None), - PRISM_REJECTION_BLOCK_ACCEPT_PENDING, + server._enqueue_block_accounting_task = ( # type: ignore[method-assign] + lambda task: (captured.append(task), True)[1] ) - self.assertNotIn( - PRISM_REJECTION_STALE_JOB, - getattr(server, "block_candidate_abandoned_counts", {}), + self.assertTrue(server.submit_next_block_candidate(defer_accounting=True)) + self.assertEqual(len(captured), 1) + task = captured[0] + server._release_block_candidate_disposition(task.disposition_lease) + + def test_permanently_closed_pool_hands_outbox_to_accounting(self) -> None: + server, state, _recording = submit_coordinator() + server.max_blocks = 1 + server.accepted_block_count = 1 + candidate = block_candidate( + server, + state, + SimpleNamespace( + coinbase_tx_hex="00", + block_hash_hex="f2" * 32, + block_hex="f2", + share_pass=True, + block_pass=True, + ), ) - self.assertIn(block_hash, server._accepted_block_payout_previews) - self.assertTrue( - server._accepted_block_payout_previews[block_hash].landed + captured: list[object] = [] + server._enqueue_block_accounting_task = ( # type: ignore[method-assign] + lambda task: (captured.append(task), True)[1] ) - self.assertNotIn( - block_hash, - server._invalidated_accepted_block_payout_previews, + server.rpc = SimpleNamespace( + call=lambda *_args, **_kwargs: (_ for _ in ()).throw( + AssertionError("closed pool must not call submitblock") + ) ) + server.enqueue_block_candidate(candidate) - def test_wrong_height_probe_verdict_overrules_late_observation(self) -> None: - # Bugbot round 6: the probe must win both directions in the late - # check too. An observation arriving during the withdrawal cannot - # revive a candidate the pre-check probe proved active at the wrong - # height -- header heights are immutable, so that verdict stands and - # the abandon commits terminally instead of deferring to window - # expiry. - block_hash = "bb" * 32 - server, _state, _ledger = submit_coordinator() - server._ensure_job_cache_state() - server._register_outstanding_block_candidate(block_hash) - server._begin_accepted_block_payout_preview(block_hash, block_height=10) - server._mark_accepted_block_payout_landed(block_hash, block_height=10) - server.rpc = AcceptanceProbeRpc( - tip="11" * 32, - header={"height": 9, "confirmations": 2}, - ) - real_clear = server._clear_accepted_block_payout_preview + self.assertTrue(server.submit_next_block_candidate(defer_accounting=True)) + self.assertEqual(len(captured), 1) + task = captured[0] + self.assertFalse(task.node_submission.attempted) + self.assertIsNone(getattr(server, "_retry_block_candidate", None)) + server._release_block_candidate_disposition(task.disposition_lease) - def observing_clear( - hash_arg: str, - *, - invalidate_published: bool = False, - ) -> None: - if invalidate_published: - with server.lock: - server._tip_observed_accepted_block_hashes[block_hash] = ( - time.monotonic() - ) - return real_clear( - hash_arg, - invalidate_published=invalidate_published, + def test_accounted_block_replaces_its_capacity_reservation(self) -> None: + server, state, _recording = submit_coordinator() + server.max_blocks = 2 + server.stop_after_block = False + block_hash = "e2" * 32 + candidate = block_candidate( + server, + state, + SimpleNamespace( + coinbase_tx_hex="c0ffee", + block_hash_hex=block_hash, + block_hex="e2", + share_pass=True, + block_pass=True, + ), + ) + server._ensure_block_candidate_disposition_state() + server._block_fast_lane_reservations.add(block_hash) + report = verified_audit_report() + verification_identity = AuditArtifactStore.build_verification_identity( + trust_source="embedded_test_only", + trusted_writer_public_key_hex="aa" * 32, + literal_sha256="cd" * 32, + literal_byte_len=1, + report=report, + ) + _recording._audit_publication_sequences[block_hash] = 1 + server._land_and_confirm_block_candidate = ( # type: ignore[method-assign] + lambda *_args, **_kwargs: ( + verified_block_bundle(), + report, + {"persisted": True}, + {"confirmed_count": 1}, + AuditPublicationIdentity(1, 10, block_hash), + verification_identity, + ) + ) + server.accepted_share_stats = lambda: (0, 0) # type: ignore[method-assign] + with tempfile.TemporaryDirectory() as tempdir: + server.audit_dir = Path(tempdir) + server.evidence_path = Path(tempdir) / "evidence.json" + self.assertTrue( + server._submit_block_candidate_serialized( + candidate, + node_submission=SimpleNamespace( + attempted=True, + result=None, + error=None, + ), + ) ) - server._clear_accepted_block_payout_preview = observing_clear # type: ignore[method-assign] + self.assertEqual(server.accepted_block_count, 1) + self.assertNotIn(block_hash, server._block_fast_lane_reservations) + self.assertTrue(server._reserve_block_fast_lane_slot("e3" * 32)) - accepted_race_won = server._abandon_block_candidate( - "block-stale", - "candidate active at unexpected height 9", - block_hash=block_hash, - worker=None, - expected_height=10, + def test_synchronous_candidate_waits_for_fast_lane_capacity(self) -> None: + server, state, _recording = submit_coordinator() + server.max_blocks = 1 + server.stop_after_block = True + reserved_hash = "e4" * 32 + candidate = block_candidate( + server, + state, + SimpleNamespace( + coinbase_tx_hex="c0ffee", + block_hash_hex="e5" * 32, + block_hex="e5", + share_pass=False, + block_pass=True, + ), + credit_share_on_accept=True, + ) + server._ensure_block_candidate_disposition_state() + self.assertTrue(server._reserve_block_fast_lane_slot(reserved_hash)) + server._node_submission_for_candidate = ( # type: ignore[method-assign] + lambda _candidate: (_ for _ in ()).throw( + AssertionError("capacity-blocked candidate must not reach qbitd") + ) ) - self.assertFalse(accepted_race_won) - outcome = getattr(server, "_block_candidate_outcome", None) - self.assertEqual(getattr(outcome, "reason", None), "block-stale") - self.assertNotIn( - "block-stale", - server.block_candidate_abandoned_counts, + with self.assertRaisesRegex(RuntimeError, "fast-lane capacity"): + server._submit_synchronous_block_candidate(candidate) + + self.assertIs(server._retry_block_candidate, candidate) + self.assertEqual(server._block_fast_lane_reservations, {reserved_hash}) + + def test_node_offer_exception_retains_reserved_candidate(self) -> None: + server, state, _recording = submit_coordinator() + server.max_blocks = 1 + server.stop_after_block = True + candidate = block_candidate( + server, + state, + SimpleNamespace( + coinbase_tx_hex="c0ffee", + block_hash_hex="e6" * 32, + block_hex="e6", + share_pass=True, + block_pass=True, + ), ) - self.assertEqual( - getattr(server, "block_candidate_accept_pending_defer_count", 0), - 0, + block_hash = candidate.submission.block_hash_hex + server._node_submission_for_candidate = ( # type: ignore[method-assign] + lambda _candidate: (_ for _ in ()).throw( + RuntimeError("node offer bookkeeping failed") + ) ) + server.enqueue_block_candidate(candidate) - def test_terminal_seal_excludes_observations_after_the_commit(self) -> None: - # Codex round 3: acceptance evidence arriving after the terminal - # commit but before the follow-up durable rejection must be excluded - # deterministically. The terminal commit seals the hash (drops it - # from outstanding) in the same critical section, so a later - # observation is a no-op instead of contradicting the sealed - # disposition mid-rejection. - block_hash = "b8" * 32 - server, _state, _ledger = submit_coordinator() - server._ensure_job_cache_state() - server._register_outstanding_block_candidate(block_hash) - server.rpc = AcceptanceProbeRpc(tip="11" * 32, header=None) + with self.assertRaisesRegex(RuntimeError, "node offer bookkeeping failed"): + server.submit_next_block_candidate(defer_accounting=True) - accepted_race_won = server._abandon_block_candidate( - PRISM_REJECTION_STALE_JOB, - "tip moved before submit: test", - block_hash=block_hash, - worker=None, - preserve_if_accepted=True, - expected_height=10, + self.assertIs(server._retry_block_candidate, candidate) + self.assertIn(block_hash, server._block_fast_lane_reservations) + self.assertFalse(server._reserve_block_fast_lane_slot("e7" * 32)) + lease = server._claim_block_candidate_disposition( + block_hash, + blocking=False, + ) + self.assertIsNotNone(lease) + assert lease is not None + server._release_block_candidate_disposition(lease) + + def test_accounting_enqueue_exception_retains_reserved_candidate(self) -> None: + server, state, _recording = submit_coordinator() + server.max_blocks = 1 + server.stop_after_block = True + candidate = block_candidate( + server, + state, + SimpleNamespace( + coinbase_tx_hex="c0ffee", + block_hash_hex="e8" * 32, + block_hex="e8", + share_pass=True, + block_pass=True, + ), + ) + block_hash = candidate.submission.block_hash_hex + server._node_submission_for_candidate = ( # type: ignore[method-assign] + lambda _candidate: SimpleNamespace( + attempted=True, + result=None, + error=None, + ) + ) + server._enqueue_block_accounting_task = ( # type: ignore[method-assign] + lambda _task: (_ for _ in ()).throw( + RuntimeError("accounting enqueue failed") + ) ) + server.enqueue_block_candidate(candidate) - self.assertFalse(accepted_race_won) - self.assertNotIn( - PRISM_REJECTION_STALE_JOB, - server.block_candidate_abandoned_counts, - ) - self.assertNotIn(block_hash, server._outstanding_block_candidate_hashes) + with self.assertRaisesRegex(RuntimeError, "accounting enqueue failed"): + server.submit_next_block_candidate(defer_accounting=True) - # A blockwait observation lands right after the sealed commit: it - # must not register acceptance evidence for the sealed hash. - self.assertTrue(server.observe_tip_for_refresh(block_hash)) - self.assertNotIn(block_hash, server._tip_observed_accepted_block_hashes) - outcome = getattr(server, "_block_candidate_outcome", None) - self.assertIsNotNone(outcome) - server._record_committed_block_candidate_abandonment(block_hash, outcome) - self.assertEqual( - server.block_candidate_abandoned_counts[PRISM_REJECTION_STALE_JOB], - 1, + self.assertIs(server._retry_block_candidate, candidate) + self.assertIn(block_hash, server._block_fast_lane_reservations) + self.assertFalse(server._reserve_block_fast_lane_slot("e9" * 32)) + lease = server._claim_block_candidate_disposition( + block_hash, + blocking=False, ) + self.assertIsNotNone(lease) + assert lease is not None + server._release_block_candidate_disposition(lease) - def test_observation_during_prepared_row_rejection_cannot_split_state(self) -> None: - # The exact round-3 interleaving at the post-persist site: a - # blockwait observation fires inside the gap between the sealed - # terminal decision and reject_prepared_block (during the - # getblockcount call). The seal excludes it, so the rows are - # rejected exactly once with no contradictory evidence registered - # and no defer emitted after the seal. - parent = "00" * 32 - block_hash = "c9" * 32 - racing_winner = "77" * 32 - server, state, ledger = submit_coordinator(tip=parent) - server.max_blocks = 2 - server.stop_after_block = False - with tempfile.TemporaryDirectory() as tempdir: - submitted, abandoned = self._accepted_tail_scaffolding(server, tempdir) - rpc = LostAckSubmitRpc( - start_tip=parent, - hash_by_hex={"00": block_hash}, - ) - rpc.lose_acks = False - server.rpc = rpc - # The definitive ack retained at the offer would veto the - # terminal decision while fresh; these tests pin the sealed - # terminal machinery, so age it past the acceptance window. - self._age_retained_acceptance_past_window(server) - candidate = block_candidate( - server, - state, - SimpleNamespace( - coinbase_tx_hex="c0ffee", - block_hash_hex=block_hash, - block_hex="00", - share_pass=True, - block_pass=True, - ), - ) - race_state: dict[str, bool] = {"post_persist": False} - real_persist = ledger.persist_accepted_block - - def persist_then_lose_race(**kwargs: object) -> dict[str, object]: - result = real_persist(**kwargs) - rpc.tip = racing_winner - rpc.active.pop(block_hash, None) - rpc.getblockhash_override = racing_winner - race_state["post_persist"] = True - return result - ledger.persist_accepted_block = persist_then_lose_race # type: ignore[method-assign] - real_call = rpc.call +class PendingShareFloorSeamTests(unittest.TestCase): + """S3 floor-holder seams used by the block-candidate credit paths. - def observing_call( - method: str, - params: list[object] | None = None, - ) -> object: - if method == "getblockcount" and race_state["post_persist"]: - # Blockwait reports the candidate as tip inside the - # seal -> reject gap. - server.observe_tip_for_refresh(block_hash) - return real_call(method, params) + At this layer the pending-commit floor keeps the current id-keyed single + holder per stamped share; the attempt/candidate release seams are the + staged split points a later layer widens into independent holders. These + cases pin the seam wiring so a lost release path cannot silently unhook + the snapshot-anchor floor from candidate handling. + """ - rpc.call = observing_call # type: ignore[method-assign] + def _stamped_pending_share(self, server, share_id: str): + share_writer = server._ensure_share_writer_service() + pending = PendingShare( + share_id=share_id, + miner_id="miner-a", + order_key="miner-a", + p2mr_program_hex="11" * 32, + share_difficulty=1, + network_difficulty=1, + template_height=9, + job_id="job-1", + job_issued_at_ms=1, + accepted_at_ms=100, + ntime=1_700_000_000, + ) + with share_writer._pending_share_commit_lock: + share_writer._pending_share_commit_floor[id(pending)] = [ + pending, + time.monotonic(), + False, + ] + return pending - self.assertTrue(server._submit_next_block_candidate_writer(candidate)) + def test_attempt_and_candidate_seams_release_the_owner_floor(self) -> None: + server = coordinator() + share_writer = server._ensure_share_writer_service() - self.assertEqual(len(ledger.rejected), 1) - self.assertEqual(abandoned, [block_hash]) - self.assertEqual(submitted, []) - self.assertEqual( - server.block_candidate_abandoned_counts["block-stale"], - 1, - ) - self.assertNotIn( - block_hash, server._tip_observed_accepted_block_hashes - ) - self.assertNotIn( - block_hash, server._outstanding_block_candidate_hashes - ) - self.assertEqual( - getattr(server, "block_candidate_accept_pending_defer_count", 0), - 0, - ) + attempt = self._stamped_pending_share(server, "miner-a:attempt") + self.assertEqual(server._job_snapshot_anchor_ms(1_000), 99) + server._finish_pending_share_attempt(attempt) + self.assertEqual(share_writer._pending_share_commit_floor, {}) + self.assertEqual(server._job_snapshot_anchor_ms(1_000), 999) - def test_retained_candidate_reregisters_for_tip_observations(self) -> None: - # Codex round 8: the terminal seal stops observation matching, but - # when the terminal cleanup itself fails (reject_prepared_block - # raising) the candidate is retained for retry -- and evidence - # arriving during that backoff gap must register again, not vanish. - parent = "00" * 32 - block_hash = "cd" * 32 - racing_winner = "77" * 32 - server, state, ledger = submit_coordinator(tip=parent) - server.max_blocks = 2 - server.stop_after_block = False - with tempfile.TemporaryDirectory() as tempdir: - submitted, abandoned = self._accepted_tail_scaffolding(server, tempdir) - rpc = LostAckSubmitRpc( - start_tip=parent, - hash_by_hex={"00": block_hash}, - ) - rpc.lose_acks = False - server.rpc = rpc - # The definitive ack retained at the offer would veto the - # terminal decision while fresh; these tests pin the sealed - # terminal machinery, so age it past the acceptance window. - self._age_retained_acceptance_past_window(server) - candidate = block_candidate( - server, - state, - SimpleNamespace( - coinbase_tx_hex="c0ffee", - block_hash_hex=block_hash, - block_hex="00", - share_pass=True, - block_pass=True, - ), - ) - real_persist = ledger.persist_accepted_block + candidate = self._stamped_pending_share(server, "miner-a:candidate") + server._finish_pending_share_candidate(candidate) + self.assertEqual(share_writer._pending_share_commit_floor, {}) - def persist_then_lose_race(**kwargs: object) -> dict[str, object]: - result = real_persist(**kwargs) - rpc.tip = racing_winner - rpc.active.pop(block_hash, None) - rpc.getblockhash_override = racing_winner - return result + def test_distinct_same_hash_stamps_hold_independent_floor_entries(self) -> None: + server = coordinator() + share_writer = server._ensure_share_writer_service() - ledger.persist_accepted_block = persist_then_lose_race # type: ignore[method-assign] - reject_attempts: list[dict[str, object]] = [] + first = self._stamped_pending_share(server, "miner-a:same-hash") + second = self._stamped_pending_share(server, "miner-a:same-hash") - def failing_reject(**kwargs: object) -> dict[str, object]: - reject_attempts.append(kwargs) - raise RuntimeError("psql briefly unavailable") + # Releasing one stamped object's holder cannot drop the other live + # holder for the same durable hash: the anchor floor stays clamped. + server._finish_pending_share_candidate(second) + self.assertEqual(len(share_writer._pending_share_commit_floor), 1) + self.assertIs( + share_writer._pending_share_commit_floor[id(first)][0], + first, + ) + self.assertEqual(server._job_snapshot_anchor_ms(1_000), 99) - ledger.reject_prepared_block = failing_reject # type: ignore[method-assign] - ledger.pool_block_state = ( # type: ignore[attr-defined] - lambda *, block_hash: { - "chain_state": "prepared", - "maturity_state": "immature", - } - ) + server._finish_pending_share_attempt(first) + self.assertEqual(share_writer._pending_share_commit_floor, {}) - # The sealed terminal pass aborts inside the rejection (site - # attempt, then the writer's terminal-cleanup attempt): the - # candidate is retained and must match observations again. - self.assertTrue(server._submit_next_block_candidate_writer(candidate)) - self.assertEqual(len(reject_attempts), 2) - self.assertIsNotNone(getattr(server, "_retry_block_candidate", None)) - self.assertIn( - block_hash, server._outstanding_block_candidate_hashes - ) - self.assertNotIn( - "block-stale", - server.block_candidate_abandoned_counts, - ) - # Blockwait reports the pool's own hash during the backoff gap: - # the evidence registers instead of vanishing behind the seal. - self.assertTrue(server.observe_tip_for_refresh(block_hash)) - self.assertIn( - block_hash, server._tip_observed_accepted_block_hashes - ) - self.assertEqual(abandoned, []) +class DuplicateDropSnapshotFloorTests(unittest.TestCase): + """Duplicate-dropped credit candidates release their adopted floor holders. - # The retry can subsequently prove the candidate active and - # complete as submitted. The earlier reversible seals must never - # leave a contradictory abandonment count behind. - ledger.persist_accepted_block = real_persist # type: ignore[method-assign] - rpc.tip = block_hash - rpc.height = 10 - rpc.active[block_hash] = 10 - rpc.getblockhash_override = None - recovered_candidate = self._retained_candidate(server) - self.assertTrue( - server._submit_next_block_candidate_writer(recovered_candidate) - ) - self.assertEqual(submitted, [block_hash]) - self.assertEqual(abandoned, []) - self.assertEqual(server.accepted_block_count, 1) - self.assertNotIn( - "block-stale", - server.block_candidate_abandoned_counts, - ) + A replayed credit-bearing candidate adopts its reconstructed PendingShare + onto the snapshot-anchor floor at decode, keyed by object identity, so a + same-hash duplicate dropped on any path carries a holder no other object's + disposition can release. Without a release at the drop, the job/payout + snapshot anchor stays clamped below the replayed stamp until restart + (#76 review finding). + """ - def test_definitive_ack_with_unprovable_chain_view_defers_not_abandons( + def _durable_credit_candidate( self, - ) -> None: - # Release-review finding: a definitive submitblock success followed - # by a chain view that cannot prove the block landed (foreign tip, - # header unknown, height taken by a sibling in that instant's view) - # must defer as accept-pending on the strength of the retained ack - # -- never terminally abandon accounting for a block the node - # accepted. The stash ages on the observation window, so a real - # orphan still terminalizes once the ack is stale (the sealed - # rejection tests above). - parent = "00" * 32 - block_hash = "ce" * 32 - racing_winner = "77" * 32 - server, state, ledger = submit_coordinator(tip=parent) - server.max_blocks = 2 - server.stop_after_block = False - with tempfile.TemporaryDirectory() as tempdir: - submitted, abandoned = self._accepted_tail_scaffolding(server, tempdir) - rpc = LostAckSubmitRpc( - start_tip=parent, - hash_by_hex={"00": block_hash}, - ) - rpc.lose_acks = False - server.rpc = rpc - candidate = block_candidate( + server: PrismCoordinator, + state: ClientState, + ledger: SingleWriterShareLedger, + tag: str, + stamp: int, + ) -> PrismBlockCandidate: + pending = PendingShare( + share_id=f"miner-a:{tag * 32}", + miner_id="miner-a", + order_key="miner-a", + p2mr_program_hex="11" * 32, + share_difficulty=1, + network_difficulty=1, + template_height=9, + job_id="job-1", + job_issued_at_ms=1, + accepted_at_ms=stamp, + ntime=1, + ) + value = dataclass_replace( + block_candidate( server, state, SimpleNamespace( - coinbase_tx_hex="c0ffee", - block_hash_hex=block_hash, + coinbase_tx_hex="00", + block_hash_hex=tag * 32, block_hex="00", - share_pass=True, + share_pass=False, block_pass=True, ), - ) - real_persist = ledger.persist_accepted_block + pending_share=pending, + ), + credit_share_on_accept=True, + ) + ledger.append_batch([(pending, server.block_candidate_intent(value))]) + return value - def persist_then_lose_race(**kwargs: object) -> dict[str, object]: - result = real_persist(**kwargs) - rpc.tip = racing_winner - rpc.active.pop(block_hash, None) - rpc.getblockhash_override = racing_winner - return result + def test_replay_dedupe_drop_releases_adopted_floor_holder(self) -> None: + server, state, _recording = submit_coordinator() + ledger = SingleWriterShareLedger() + server.ledger = ledger + self._durable_credit_candidate(server, state, ledger, "a7", 100) + share_writer = server._ensure_share_writer_service() - ledger.persist_accepted_block = persist_then_lose_race # type: ignore[method-assign] + with patch("builtins.print"): + self.assertEqual(server.replay_pending_block_candidates(), 1) + first = server._block_replay_candidate_queue.get_nowait() + self.assertEqual(len(share_writer._pending_share_commit_floor), 1) - self.assertTrue(server._submit_next_block_candidate_writer(candidate)) + # The submitter holds the dequeued candidate mid-disposition, so the + # steady-state poll re-reads the still-pending outbox row: the decode + # adopts a fresh holder before the in-flight dedupe drops the object. + with patch("builtins.print"): + self.assertEqual(server.replay_pending_block_candidates(), 0) + self.assertEqual(len(share_writer._pending_share_commit_floor), 1) - # Deferred, not terminal: prepared rows untouched, candidate - # retained for retry, observation matching still open. - self.assertEqual(ledger.rejected, []) - self.assertEqual(abandoned, []) - self.assertEqual(submitted, []) - self.assertGreaterEqual( - getattr(server, "block_candidate_accept_pending_defer_count", 0), - 1, - ) - self.assertIsNotNone(getattr(server, "_retry_block_candidate", None)) - self.assertNotIn( - "block-stale", - server.block_candidate_abandoned_counts, - ) - self.assertIn( - block_hash, server._outstanding_block_candidate_hashes - ) + # Once the surviving object's disposition releases its own holder, no + # leaked duplicate keeps clamping the anchor below the replayed stamp. + server._finish_pending_share_candidate(first.pending_share) + self.assertEqual(server._job_snapshot_anchor_ms(1_000), 999) - def test_retained_acceptance_evidence_ages_on_observation_window( - self, - ) -> None: - # The stash is acceptance evidence only while fresh: past the - # observation window it stands down so a genuinely orphaned block - # (whose probes never prove anything either way) regains - # abandonability instead of deferring forever. - server, _state, _ledger = submit_coordinator() - block_hash = "cf" * 32 - server._stash_retained_block_candidate_node_submission( - block_hash, - SimpleNamespace(attempted=True, error=None, result=None), + def test_terminal_short_circuit_releases_replayed_duplicate_holder(self) -> None: + server, state, _recording = submit_coordinator() + ledger = SingleWriterShareLedger() + server.ledger = ledger + candidate = self._durable_credit_candidate(server, state, ledger, "b7", 100) + with patch("builtins.print"): + self.assertEqual(server.replay_pending_block_candidates(), 1) + + # A same-hash disposition (e.g. the miner's synchronous resubmit of + # the below-target share) lands terminally while the replayed + # duplicate is still queued behind live solves. + server._record_block_candidate_terminal_outcome( + candidate.submission.block_hash_hex, + accepted=True, ) - self.assertTrue(server._block_candidate_acceptance_retained(block_hash)) + self.assertEqual(server._job_snapshot_anchor_ms(1_000), 99) + + self.assertTrue(server.submit_next_block_candidate()) + + self.assertEqual(server._job_snapshot_anchor_ms(1_000), 999) + + def test_terminal_outcome_releases_parked_same_hash_duplicate_holder(self) -> None: + server, state, _recording = submit_coordinator() + ledger = SingleWriterShareLedger() + server.ledger = ledger + candidate = self._durable_credit_candidate(server, state, ledger, "c7", 100) + with patch("builtins.print"): + self.assertEqual(server.replay_pending_block_candidates(), 1) + parked = server._block_replay_candidate_queue.get_nowait() + + # submit_next parks a candidate that lost the same-hash disposition + # claim; the winning flight then records the terminal outcome and + # discards the parked wakeup. + server._ensure_block_candidate_disposition_state() with server.lock: - server._block_candidate_retained_submission_monotonic[ - block_hash - ] -= 400.0 - self.assertFalse(server._block_candidate_acceptance_retained(block_hash)) - # A non-positive window means evidence never expires, mirroring the - # tip-observation semantics. - server.observed_tip_accept_window_seconds = 0.0 - self.assertTrue(server._block_candidate_acceptance_retained(block_hash)) + server._block_disposition_waiting_retries[ + candidate.submission.block_hash_hex + ] = parked + server._record_block_candidate_terminal_outcome( + candidate.submission.block_hash_hex, + accepted=True, + ) - def test_pool_closed_gate_requires_probe_proven_acceptance(self) -> None: - # Bugbot: observation evidence alone must not open the pool-closed - # gate -- an off-chain candidate would fall through toward - # submitblock after the pool stopped accepting blocks. Unprovable - # views defer via the abandon path; expired evidence goes terminal. - parent = "00" * 32 - block_hash = "d9" * 32 - server, state, _ledger = submit_coordinator(tip=parent) - server.max_blocks = 1 - server.stop_after_block = False - server.accepted_block_count = 1 - server.observed_tip_accept_window_seconds = 60.0 - with tempfile.TemporaryDirectory() as tempdir: - submitted, abandoned = self._accepted_tail_scaffolding(server, tempdir) - rpc = LostAckSubmitRpc( - start_tip=parent, - hash_by_hex={"00": block_hash}, - ) - server.rpc = rpc - server._register_outstanding_block_candidate(block_hash) - with server.lock: - server._tip_observed_accepted_block_hashes[block_hash] = ( - time.monotonic() - ) - candidate = block_candidate( + self.assertEqual(server._job_snapshot_anchor_ms(1_000), 999) + + def test_same_hash_retry_merge_releases_displaced_duplicate_holder(self) -> None: + server, state, _recording = submit_coordinator() + ledger = SingleWriterShareLedger() + server.ledger = ledger + self._durable_credit_candidate(server, state, ledger, "d7", 100) + with patch("builtins.print"): + self.assertEqual(server.replay_pending_block_candidates(), 1) + displaced = server._block_replay_candidate_queue.get_nowait() + server._retain_block_candidate_for_retry(displaced) + + # The miner resubmits the same below-target block: a distinct live + # stamped object takes the retry slot and the replayed object dies. + live = dataclass_replace( + block_candidate( server, state, SimpleNamespace( - coinbase_tx_hex="c0ffee", - block_hash_hex=block_hash, + coinbase_tx_hex="00", + block_hash_hex="d7" * 32, block_hex="00", - share_pass=True, + share_pass=False, block_pass=True, ), - ) + pending_share=PendingShare( + share_id="miner-a:" + "d7" * 32, + miner_id="miner-a", + order_key="miner-a", + p2mr_program_hex="11" * 32, + share_difficulty=1, + network_difficulty=1, + template_height=9, + job_id="job-1", + job_issued_at_ms=1, + accepted_at_ms=100, + ntime=1, + ), + ), + credit_share_on_accept=True, + ) + server._retain_block_candidate_for_retry(live) - # Fresh observation, unprovable chain view: defer, and above all - # never reach submitblock through the closed pool. - self.assertTrue(server._submit_next_block_candidate_writer(candidate)) - self.assertEqual(rpc.submitblock_calls, 0) - self.assertEqual(abandoned, []) - candidate = self._retained_candidate(server) + self.assertIs(server._retry_block_candidate, live) + self.assertEqual(server._job_snapshot_anchor_ms(1_000), 999) - # Evidence expires with the candidate still absent: terminal. - with server.lock: - server._tip_observed_accepted_block_hashes[block_hash] = ( - time.monotonic() - 61.0 - ) - self.assertTrue(server._submit_next_block_candidate_writer(candidate)) - self.assertEqual(rpc.submitblock_calls, 0) - self.assertEqual(abandoned, [block_hash]) - self.assertEqual(submitted, []) + +# -- issue #198: bounded collapse cleanup-retry backlog ------------------------ + +from contextlib import redirect_stderr, redirect_stdout # noqa: E402 +from io import StringIO # noqa: E402 +from typing import Any # noqa: E402 + +from lab.prism.block_candidates import ( # noqa: E402 + BLOCK_CANDIDATE_COLLAPSE_CLEANUP_STEPS, + MAX_BLOCK_CANDIDATE_TERMINAL_OUTCOMES, + PRISM_BLOCK_CANDIDATE_CLEANUP_BACKPRESSURE_CALLERS, + PRISM_BLOCK_CANDIDATE_COLLAPSE_OUTCOMES, + PRISM_BLOCK_CANDIDATE_COLLAPSE_REASON, +) +from lab.prism.coordinator_config import ( # noqa: E402 + DEFAULT_BLOCK_CANDIDATE_CLEANUP_RETRY_BACKLOG_MAX, + MAX_BLOCK_CANDIDATE_CLEANUP_RETRY_BACKLOG_MAX, + load_coordinator_config, +) +from tests.prism_candidate_storm import OBSERVED_TESTNET_CANDIDATE_STORM # noqa: E402 +from tests.test_prism_block_candidate_collapse import ( # noqa: E402 + CollapseFixture as _CollapseFixture, + _break_cleanup_step as _break_collapse_cleanup_step, + _hash as _collapse_hash, +) + +_BACKLOG_KNOB = "PRISM_BLOCK_CANDIDATE_CLEANUP_RETRY_BACKLOG_MAX" + + +def _minimal_config_environment(root: Path) -> dict[str, str]: + return { + "QBIT_RPC_HOST": "qbit.example", + "QBIT_RPC_USER": "rpc-user", + "QBIT_RPC_PASSWORD": "rpc-password", + "PRISM_ALLOW_MEMORY_LEDGER": "1", + "PRISM_ALLOW_TEST_SIGNING_SEEDS": "1", + "PRISM_ALLOW_BUNDLE_EMBEDDED_LEDGER_KEY": "1", + "PRISM_AUDIT_DIR": str(root), + "PRISM_EVIDENCE_PATH": str(root / "evidence.json"), + } + + +class BlockCandidateCleanupRetryBacklogBoundTests(unittest.TestCase): + """Issue #198: the cleanup-retry backlog is bounded at admission. + + A won row whose post-write cleanup failed keeps one in-memory retry + record, the only replay source it has left. Under a systemic cleanup + fault the collapse would otherwise mint one record per row it wins -- + unbounded at storm scale -- so once the registry holds the configured + number of records the collapse stops handing rows to its fenced bulk + terminalization. Everything already terminal keeps its record, its + exact pending-share holders, and its fence; everything declined stays + durable and pending for the per-row path; and once the backlog drains + admission resumes and the preserved rows collapse on the next walk. + + Every test here is deterministic: the retry pass is driven directly, + the walk is the shipped enumeration, and no thread or sleep is used. + """ + + BOUND = DEFAULT_BLOCK_CANDIDATE_CLEANUP_RETRY_BACKLOG_MAX + + @staticmethod + def _quiet() -> Any: + stack = contextlib.ExitStack() + stack.enter_context(redirect_stdout(StringIO())) + stack.enter_context(redirect_stderr(StringIO())) + return stack + + @staticmethod + def _snapshot(fixture: _CollapseFixture) -> dict[str, Any]: + return fixture.service.collapsed_candidate_cleanup_backlog_snapshot() + + def _drain(self, fixture: _CollapseFixture) -> int: + passes = 0 + with self._quiet(): + while fixture.service._run_one_collapsed_block_candidate_cleanup_retry(): + passes += 1 + return passes + + def _walk(self, fixture: _CollapseFixture) -> tuple[int, str]: + """The shipped enumeration walk, both streams captured.""" + errors = StringIO() + with redirect_stderr(errors): + queued, log = fixture.replay() + return queued, log + + # -- the knob ---------------------------------------------------------- + + def test_the_default_bound_is_derived_from_the_pinned_contracts(self) -> None: + fixture = _CollapseFixture() + self.assertEqual( + fixture.service._collapse_cleanup_retry_backlog_max(), + self.BOUND, + ) + # The restated limit is the fence registry's bound, and the default + # pins at most half of it. + self.assertEqual( + MAX_BLOCK_CANDIDATE_CLEANUP_RETRY_BACKLOG_MAX, + MAX_BLOCK_CANDIDATE_TERMINAL_OUTCOMES, + ) + self.assertLessEqual(self.BOUND, MAX_BLOCK_CANDIDATE_TERMINAL_OUTCOMES // 2) + # It absorbs the observed storm's siblings and one full replay page + # without engaging. + self.assertGreaterEqual(self.BOUND, OBSERVED_TESTNET_CANDIDATE_STORM - 1) + self.assertGreaterEqual(self.BOUND, MAX_BLOCK_REPLAY_ENUMERATION_ROWS) + self.assertIn("backlog_deferred", PRISM_BLOCK_CANDIDATE_COLLAPSE_OUTCOMES) + self.assertEqual( + PRISM_BLOCK_CANDIDATE_CLEANUP_BACKPRESSURE_CALLERS, + ("replay-page", "dequeue"), + ) + + def test_the_bound_reads_the_coordinator_then_the_loaded_config(self) -> None: + fixture = _CollapseFixture() + server = fixture.server + server.config = SimpleNamespace( + block=SimpleNamespace(candidate_cleanup_retry_backlog_max=77) + ) + self.assertEqual(fixture.service._collapse_cleanup_retry_backlog_max(), 77) + server.block_candidate_cleanup_retry_backlog_max = 5 + self.assertEqual(fixture.service._collapse_cleanup_retry_backlog_max(), 5) + # A misconfigured runtime value degrades to admitting one row at a + # time from an empty backlog, never to a collapse that admits nothing. + server.block_candidate_cleanup_retry_backlog_max = 0 + self.assertEqual(fixture.service._collapse_cleanup_retry_backlog_max(), 1) + self.assertEqual( + fixture.service._collapse_cleanup_admission_headroom(), + (1, 0, 1), + ) + + # Direct embedder/test overrides bypass startup validation, so they + # are clamped to the terminal-fence registry's hard capacity. + server.block_candidate_cleanup_retry_backlog_max = ( + MAX_BLOCK_CANDIDATE_CLEANUP_RETRY_BACKLOG_MAX + 1 + ) + self.assertEqual( + fixture.service._collapse_cleanup_retry_backlog_max(), + MAX_BLOCK_CANDIDATE_CLEANUP_RETRY_BACKLOG_MAX, + ) + self.assertEqual( + fixture.service._collapse_cleanup_admission_headroom(), + ( + MAX_BLOCK_CANDIDATE_CLEANUP_RETRY_BACKLOG_MAX, + 0, + MAX_BLOCK_CANDIDATE_CLEANUP_RETRY_BACKLOG_MAX, + ), + ) + + def test_startup_validation_refuses_an_unusable_bound(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + base = _minimal_config_environment(Path(tmp)) self.assertEqual( - server.block_candidate_abandoned_counts[ - PRISM_REJECTION_POOL_CLOSED - ], - 1, + load_coordinator_config(base).block.candidate_cleanup_retry_backlog_max, + self.BOUND, ) + for raw, message in ( + ("0", f"{_BACKLOG_KNOB} must be positive"), + ("-1", f"{_BACKLOG_KNOB} must be positive"), + ("abc", f"{_BACKLOG_KNOB} must be an integer"), + ( + str(MAX_BLOCK_CANDIDATE_CLEANUP_RETRY_BACKLOG_MAX + 1), + f"{_BACKLOG_KNOB} cannot exceed " + f"{MAX_BLOCK_CANDIDATE_CLEANUP_RETRY_BACKLOG_MAX}", + ), + ): + with self.subTest(raw=raw), self.assertRaisesRegex(SystemExit, message): + load_coordinator_config({**base, _BACKLOG_KNOB: raw}) + for raw in ("1", "2048", str(MAX_BLOCK_CANDIDATE_CLEANUP_RETRY_BACKLOG_MAX)): + with self.subTest(raw=raw): + config = load_coordinator_config({**base, _BACKLOG_KNOB: raw}) + self.assertEqual( + config.block.candidate_cleanup_retry_backlog_max, + int(raw), + ) - def test_pool_closed_gate_yields_to_on_chain_candidate(self) -> None: - # A candidate already on the active chain must complete its payout - # accounting even after the pool stops accepting new blocks; the - # pool-closed gate previously abandoned it terminally. - parent = "00" * 32 - block_hash = "f6" * 32 - server, state, ledger = submit_coordinator(tip=parent) - server.max_blocks = 1 - server.stop_after_block = False - server.accepted_block_count = 1 - with tempfile.TemporaryDirectory() as tempdir: - submitted, abandoned = self._accepted_tail_scaffolding(server, tempdir) - rpc = LostAckSubmitRpc( - start_tip=parent, - hash_by_hex={"00": block_hash}, - ) - # The block is already active with the pool's own hash as tip - # (for example after a watchdog restart replayed the outbox). - rpc.tip = block_hash - rpc.height = 10 - rpc.active[block_hash] = 10 - server.rpc = rpc - server.ensure_reorg_reconciled_for_tip = ( # type: ignore[method-assign] - lambda _tip, *, _coalesce_same_tip: not _coalesce_same_tip + # -- the gauges -------------------------------------------------------- + + def test_the_snapshot_reports_depth_age_holders_and_pins(self) -> None: + fixture = _CollapseFixture() + empty = self._snapshot(fixture) + self.assertEqual( + empty, + { + "depth": 0, + "backlog_max": self.BOUND, + "oldest_age_seconds": -1.0, + "pending_share_holders": 0, + "terminal_outcome_pins": 0, + "backpressure_active": False, + "backpressure_engagements": 0, + }, + ) + target = _collapse_hash(1) + candidate = fixture.seed([target], credit_share_on_accept=True)[0] + fixture.server._ensure_share_writer_service().adopt_pending_share( + candidate.pending_share + ) + fixture.service.candidate_queue.put_nowait(candidate) + broken = _break_collapse_cleanup_step( + fixture.server, + "_finish_pending_share_commit", + ) + with self._quiet(): + fixture.collapse() + snapshot = self._snapshot(fixture) + self.assertEqual(snapshot["depth"], 1) + self.assertEqual(snapshot["pending_share_holders"], 1) + self.assertEqual(snapshot["terminal_outcome_pins"], 1) + self.assertFalse(snapshot["backpressure_active"]) + self.assertGreaterEqual(snapshot["oldest_age_seconds"], 0.0) + # The retained holder is the exact object the queue held. + record = fixture.service._block_candidate_collapse_cleanup_retries[target] + self.assertEqual(len(record.shares), 1) + self.assertIs(record.shares[0], candidate.pending_share) + broken["on"] = False + self.assertEqual(self._drain(fixture), 1) + self.assertEqual(self._snapshot(fixture)["depth"], 0) + self.assertEqual(self._snapshot(fixture)["oldest_age_seconds"], -1.0) + + def test_pins_count_only_records_whose_fence_is_published(self) -> None: + """A record whose fence publication degraded still owes it.""" + fixture = _CollapseFixture() + target = _collapse_hash(1) + fixture.seed([target]) + # The degraded branch the publication itself contains: nothing is + # stamped, and the terminal-outcome step is what still owes it. + fixture.service._publish_collapsed_candidate_terminal_fence = ( + lambda abandoned: None + ) + broken = _break_collapse_cleanup_step( + fixture.server, + "_record_block_candidate_terminal_outcome", + ) + with self._quiet(): + fixture.collapse() + snapshot = self._snapshot(fixture) + self.assertEqual(snapshot["depth"], 1) + self.assertEqual(snapshot["terminal_outcome_pins"], 0) + self.assertIn("terminal-outcome", fixture.cleanup_backlog()[target]) + broken["on"] = False + self.assertEqual(self._drain(fixture), 1) + with fixture.server.lock: + self.assertIs( + fixture.service._block_candidate_terminal_outcomes[target], + False, ) - candidate = block_candidate( - server, - state, - SimpleNamespace( - coinbase_tx_hex="c0ffee", - block_hash_hex=block_hash, - block_hex="00", - share_pass=True, - block_pass=True, - ), + self.assertEqual(self._snapshot(fixture)["depth"], 0) + + def test_the_oldest_age_is_measured_from_the_first_deferral(self) -> None: + fixture = _CollapseFixture() + target = _collapse_hash(1) + fixture.seed([target]) + broken = _break_collapse_cleanup_step( + fixture.server, + "_clear_block_candidate_retry_state", + ) + with self._quiet(): + fixture.collapse() + registry = fixture.service._block_candidate_collapse_cleanup_retries + with fixture.server.lock: + registry[target].deferred_monotonic = time.monotonic() - 60.0 + # A failed retry takes the record out and re-registers it; the + # stamp must survive that round trip. + with self._quiet(): + self.assertTrue(fixture.retry_cleanup(force_due=True)[0]) + with fixture.server.lock: + self.assertEqual(registry[target].attempts, 1) + self.assertGreaterEqual(self._snapshot(fixture)["oldest_age_seconds"], 60.0) + broken["on"] = False + self.assertEqual(self._drain(fixture), 1) + + def test_an_inflight_retry_remains_counted_and_pins_its_fence(self) -> None: + """The accounting attempt cannot open an admission or fence gap.""" + fixture = _CollapseFixture() + service = fixture.service + server = fixture.server + server.block_candidate_cleanup_retry_backlog_max = 1 + target = _collapse_hash(1) + service._defer_collapsed_candidate_cleanup(target, ("retry-state",)) + service._ensure_block_candidate_disposition_state() + with server.lock: + service._stamp_block_candidate_terminal_outcome( + service._block_candidate_terminal_outcomes, + target, + False, ) - self.assertTrue(server._submit_next_block_candidate_writer(candidate)) + observed: dict[str, Any] = {} - self.assertEqual(submitted, [block_hash]) - self.assertEqual(abandoned, []) - self.assertNotIn( - PRISM_REJECTION_POOL_CLOSED, - getattr(server, "block_candidate_abandoned_counts", {}), + def observe_inflight(*_args: Any, **_kwargs: Any) -> tuple[dict, dict]: + observed["registry_empty"] = not ( + service._block_candidate_collapse_cleanup_retries ) - self.assertIn(block_hash, server._accounted_accepted_block_hashes) - self.assertEqual(server.accepted_block_count, 2) + observed["inflight"] = target in ( + service._block_candidate_collapse_cleanup_inflight + ) + observed["headroom"] = service._collapse_cleanup_admission_headroom() + observed["snapshot"] = self._snapshot(fixture) + # Make this fence the oldest entry in an overflowing outcome + # registry. The in-flight authority must still pin it while an + # unrelated unpinned outcome is evicted. + with server.lock: + outcomes = service._block_candidate_terminal_outcomes + for index in range(MAX_BLOCK_CANDIDATE_TERMINAL_OUTCOMES + 1): + other = _collapse_hash((1 << 40) + index) + service._stamp_block_candidate_terminal_outcome( + outcomes, + other, + False, + ) + service._bound_block_candidate_terminal_outcomes() + observed["fence_pinned"] = target in outcomes + return {}, {target: ()} + + service._run_collapsed_candidate_cleanup_steps = observe_inflight + with self._quiet(): + self.assertTrue( + service._run_one_collapsed_block_candidate_cleanup_retry() + ) + + self.assertTrue(observed["registry_empty"]) + self.assertTrue(observed["inflight"]) + self.assertEqual(observed["headroom"], (0, 1, 1)) + snapshot = observed["snapshot"] + self.assertEqual(snapshot["depth"], 1) + self.assertEqual(snapshot["terminal_outcome_pins"], 1) + self.assertTrue(snapshot["backpressure_active"]) + self.assertTrue(observed["fence_pinned"]) + self.assertEqual(self._snapshot(fixture)["depth"], 0) + + # -- backpressure at the shipped bound: the replay-page walk ------------ + + def test_bulk_terminalization_stops_admitting_rows_at_the_bound(self) -> None: + """Sustained failure at the shipped bound, then recovery, page path.""" + overflow = 128 + hashes = [_collapse_hash(index) for index in range(1, self.BOUND + overflow + 1)] + fixture = _CollapseFixture() + fixture.seed(hashes) + broken = _break_collapse_cleanup_step( + fixture.server, + "_clear_block_candidate_retry_state", + ) + # Four full pages are admitted and mint BOUND records; the fifth + # page finds no headroom and is preserved before any chain read. + queued, log = self._walk(fixture) + counts = fixture.counts() + snapshot = self._snapshot(fixture) + self.assertEqual(snapshot["depth"], self.BOUND) + self.assertTrue(snapshot["backpressure_active"]) + self.assertEqual(snapshot["backpressure_engagements"], 1) + self.assertEqual(snapshot["terminal_outcome_pins"], self.BOUND) + self.assertEqual(counts["abandoned"], self.BOUND) + self.assertEqual(counts["cleanup_failed"], self.BOUND) + self.assertEqual(counts["backlog_deferred"], overflow) + self.assertEqual(counts["considered"], self.BOUND) + # The preserved rows stayed durable and were adopted for the + # per-row path exactly as an unprobed height leaves them. + self.assertEqual(len(fixture.pending()), overflow) + self.assertEqual(queued, overflow) + self.assertEqual(len(fixture.ledger.abandon_calls), 4) + self.assertIn( + "cleanup backpressure engaged caller=replay-page " + f"rows_preserved={overflow} admitted=0 backlog={self.BOUND} " + f"backlog_max={self.BOUND}", + log, + ) + warning = next(line for line in log.splitlines() if "backpressure engaged" in line) + for block_hash in hashes: + self.assertNotIn(block_hash, warning) + # Sustained failure: retries keep failing, nothing is shed. + for attempt in range(1, 17): + with self._quiet(): + self.assertTrue(fixture.retry_cleanup(force_due=True)[0]) + self.assertEqual(self._snapshot(fixture)["depth"], self.BOUND) + self.assertEqual(fixture.counts()["cleanup_retry_failed"], 16) + self.assertEqual(self._snapshot(fixture)["terminal_outcome_pins"], self.BOUND) + # A second walk while the fault persists admits nothing more and + # re-adopts nothing; the warning is rate limited, the counter is not. + queued, log = self._walk(fixture) + self.assertEqual(queued, 0) + self.assertNotIn("backpressure engaged", log) + counts = fixture.counts() + self.assertEqual(counts["abandoned"], self.BOUND) + self.assertEqual(counts["backlog_deferred"], 2 * overflow) + self.assertEqual(self._snapshot(fixture)["backpressure_engagements"], 2) + self.assertEqual(len(fixture.ledger.abandon_calls), 4) + # Recovery: the backlog drains one hash per pass, admission resumes, + # and the preserved remainder collapses on the next walk. + broken["on"] = False + self.assertEqual(self._drain(fixture), self.BOUND) + snapshot = self._snapshot(fixture) + self.assertEqual(snapshot["depth"], 0) + self.assertFalse(snapshot["backpressure_active"]) + self.assertEqual(fixture.counts()["cleanup_recovered"], self.BOUND) + self.assertEqual(fixture.cleanup_backlog(), {}) + queued, log = self._walk(fixture) + self.assertEqual(queued, 0) + self.assertEqual(fixture.pending(), set()) + counts = fixture.counts() + self.assertEqual(counts["abandoned"], self.BOUND + overflow) + self.assertEqual(counts["backlog_deferred"], 2 * overflow) + self.assertEqual(len(fixture.ledger.abandon_calls), 5) + # Every terminal row is fenced and none was ever offered. + self.assertNotIn("submitblock", fixture.rpc.counts()) + with fixture.server.lock: + outcomes = fixture.service._block_candidate_terminal_outcomes + self.assertTrue(all(outcomes.get(block_hash) is False for block_hash in hashes)) + self.assertEqual( + fixture.server.block_candidate_abandoned_counts, + {PRISM_BLOCK_CANDIDATE_COLLAPSE_REASON: self.BOUND + overflow}, + ) + + def test_forced_walk_pauses_after_one_full_backpressured_page(self) -> None: + """A deep fault cannot move the durable storm into the replay queue.""" + page_rows = MAX_BLOCK_REPLAY_ENUMERATION_ROWS + total = 2 * page_rows + 7 + fixture = _CollapseFixture() + fixture.server.block_candidate_cleanup_retry_backlog_max = page_rows + fixture.seed([_collapse_hash(index) for index in range(1, total + 1)]) + _break_collapse_cleanup_step( + fixture.server, + "_clear_block_candidate_retry_state", + ) + queued, log = self._walk(fixture) + + self.assertEqual(self._snapshot(fixture)["depth"], page_rows) + self.assertEqual(queued, page_rows) + self.assertEqual( + fixture.service._block_replay_candidate_queue.qsize(), + page_rows, + ) + self.assertEqual(len(fixture.pending()), page_rows + 7) + self.assertIn("enumeration page=1", log) + self.assertIn("enumeration page=2", log) + self.assertNotIn("enumeration page=3", log) + self.assertIn("paused after cleanup backpressure", log) + self.assertTrue(fixture.server._block_replay_enumeration_owed()) + + # Even a direct forced re-entry waits for the adopted page to drain; + # it cannot append the seven rows still sitting on a later page. + engagements = self._snapshot(fixture)["backpressure_engagements"] + queued_again, log_again = self._walk(fixture) + self.assertEqual(queued_again, 0) + self.assertEqual(log_again, "") + self.assertEqual( + fixture.service._block_replay_candidate_queue.qsize(), + page_rows, + ) + self.assertEqual( + self._snapshot(fixture)["backpressure_engagements"], + engagements, + ) + self.assertTrue(fixture.server._block_replay_enumeration_owed()) + + # -- backpressure at the shipped bound: the dequeue-time skip ----------- + + def test_the_dequeue_time_skip_honours_the_same_bound(self) -> None: + fixture = _CollapseFixture() + service = fixture.service + server = fixture.server + # Fill the backlog to the bound through the shipped deferral. + with self._quiet(): + for index in range(self.BOUND): + service._defer_collapsed_candidate_cleanup( + _collapse_hash((1 << 20) + index), + ("retry-state",), + ) + self.assertEqual(self._snapshot(fixture)["depth"], self.BOUND) + fixture.rpc.results["submitblock"] = None + server.block_submit_rpc_timeout_seconds = 0.5 + + def offers() -> int: + return sum(1 for method, _ in fixture.rpc.calls if method == "submitblock") + + first = _collapse_hash(1) + candidate = fixture.seed([first])[0] + service.candidate_queue.put_nowait(candidate) + with self._quiet(): + self.assertTrue(server.submit_next_block_candidate(defer_accounting=True)) + counts = fixture.counts() + # Not considered, not skipped: preserved for the ordinary offer path, + # which offered it. + self.assertEqual(counts["backlog_deferred"], 1) + self.assertEqual(counts["dequeue_considered"], 0) + self.assertEqual(counts["dequeue_skipped"], 0) + self.assertEqual(offers(), 1) + self.assertIn(first, fixture.pending()) + self.assertEqual(self._snapshot(fixture)["depth"], self.BOUND) + self.assertEqual(self._snapshot(fixture)["backpressure_engagements"], 1) + # Once the backlog drains the skip is back. + self.assertEqual(self._drain(fixture), self.BOUND) + second = _collapse_hash(2) + candidate = fixture.seed([second])[0] + service.candidate_queue.put_nowait(candidate) + with self._quiet(): + self.assertTrue(server.submit_next_block_candidate(defer_accounting=True)) + counts = fixture.counts() + self.assertEqual(counts["dequeue_considered"], 1) + self.assertEqual(counts["dequeue_skipped"], 1) + self.assertEqual(counts["backlog_deferred"], 1) + self.assertEqual(offers(), 1) + self.assertNotIn(second, fixture.pending()) + + # -- what backpressure preserves, and what it never touches ------------- + + def test_holders_and_fences_survive_backpressure_and_release_once(self) -> None: + fixture = _CollapseFixture() + server = fixture.server + server.block_candidate_cleanup_retry_backlog_max = 2 + hashes = [_collapse_hash(index) for index in range(1, 5)] + candidates = fixture.seed(hashes, credit_share_on_accept=True) + writer = server._ensure_share_writer_service() + for candidate in candidates: + writer.adopt_pending_share(candidate.pending_share) + fixture.service.candidate_queue.put_nowait(candidate) + by_hash = dict(zip(hashes, candidates)) + releases: dict[int, int] = {} + original = server._finish_pending_share_commit + broken = {"on": True} + + def finish(pending_share: Any) -> None: + if broken["on"]: + raise RuntimeError("floor release boom") + releases[id(pending_share)] = releases.get(id(pending_share), 0) + 1 + original(pending_share) + + server._finish_pending_share_commit = finish + with self._quiet(): + fixture.collapse() + admitted = set(fixture.cleanup_backlog()) + preserved = set(hashes) - admitted + self.assertEqual(len(admitted), 2) + self.assertEqual(fixture.pending(), preserved) + self.assertEqual(fixture.counts()["backlog_deferred"], 2) + registry = fixture.service._block_candidate_collapse_cleanup_retries + for block_hash in admitted: + self.assertEqual(registry[block_hash].shares, (by_hash[block_hash].pending_share,)) + with server.lock: + outcomes = fixture.service._block_candidate_terminal_outcomes + for block_hash in admitted: + self.assertIs(outcomes[block_hash], False) + for block_hash in preserved: + self.assertNotIn(block_hash, outcomes) + # Sustained failure keeps the exact holder objects and the fences. + for _ in range(2): + with self._quiet(): + self.assertTrue(fixture.retry_cleanup(force_due=True)[0]) + for block_hash in admitted: + self.assertEqual(registry[block_hash].shares, (by_hash[block_hash].pending_share,)) + snapshot = self._snapshot(fixture) + self.assertEqual(snapshot["pending_share_holders"], 2) + self.assertEqual(snapshot["terminal_outcome_pins"], 2) + self.assertEqual({id(candidate.pending_share) for candidate in candidates}, set(fixture.floor())) + # Recovery releases exactly the admitted holders, once each; the + # preserved candidates keep theirs until their own collapse. + broken["on"] = False + self.assertEqual(self._drain(fixture), 2) + self.assertEqual( + set(fixture.floor()), + {id(by_hash[block_hash].pending_share) for block_hash in preserved}, + ) + with self._quiet(): + fixture.collapse() + self.assertEqual(fixture.pending(), set()) + self.assertEqual(fixture.floor(), {}) + self.assertEqual(set(releases.values()), {1}) + self.assertEqual(len(releases), 4) + self.assertEqual(fixture.cleanup_backlog(), {}) + self.assertNotIn("submitblock", fixture.rpc.counts()) + + def test_admission_resumes_as_the_backlog_drains(self) -> None: + fixture = _CollapseFixture() + fixture.server.block_candidate_cleanup_retry_backlog_max = 4 + hashes = [_collapse_hash(index) for index in range(1, 9)] + fixture.seed(hashes) + broken = _break_collapse_cleanup_step( + fixture.server, + "_clear_block_candidate_retry_state", + ) + with self._quiet(): + fixture.collapse() + self.assertEqual(self._snapshot(fixture)["depth"], 4) + self.assertEqual(len(fixture.pending()), 4) + self.assertEqual(fixture.counts()["backlog_deferred"], 4) + broken["on"] = False + for _ in range(2): + with self._quiet(): + self.assertTrue(fixture.retry_cleanup()[0]) + # Two records drained, so two rows of headroom: exactly two more + # rows are admitted and the other two preserved again. + with self._quiet(): + fixture.collapse() + counts = fixture.counts() + self.assertEqual(counts["abandoned"], 6) + self.assertEqual(counts["backlog_deferred"], 6) + self.assertEqual(len(fixture.pending()), 2) + self.assertEqual(self._snapshot(fixture)["depth"], 2) + self.assertEqual(self._drain(fixture), 2) + with self._quiet(): + fixture.collapse() + self.assertEqual(fixture.pending(), set()) + self.assertEqual(fixture.counts()["abandoned"], 8) + self.assertEqual(self._snapshot(fixture)["backpressure_engagements"], 2) + + def test_the_backpressure_warning_is_bounded_and_hash_free(self) -> None: + fixture = _CollapseFixture() + fixture.server.block_candidate_cleanup_retry_backlog_max = 2 + hashes = [_collapse_hash(index) for index in range(1, 7)] + fixture.seed(hashes) + broken = _break_collapse_cleanup_step( + fixture.server, + "_clear_block_candidate_retry_state", + ) + with redirect_stderr(StringIO()): + _retained, first_log = fixture.collapse() + _retained, second_log = fixture.collapse() + broken["on"] = False + warnings = [ + line + for log in (first_log, second_log) + for line in log.splitlines() + if "cleanup backpressure engaged" in line + ] + self.assertEqual(len(warnings), 1) + # Two admitted from an empty backlog, four preserved: the bound is + # what the pass would have exceeded, not what it had reached. + self.assertIn( + "caller=replay-page rows_preserved=4 admitted=2 backlog=0 backlog_max=2", + warnings[0], + ) + self.assertIn("oldest_seconds=", warnings[0]) + for block_hash in hashes: + self.assertNotIn(block_hash, warnings[0]) + # The counter and the engagement count moved for both occasions. + self.assertEqual(fixture.counts()["backlog_deferred"], 8) + self.assertEqual(self._snapshot(fixture)["backpressure_engagements"], 2) + self.assertEqual(fixture.counts()["selected"], 6) + self.assertEqual(fixture.counts()["considered"], 6) + with self.assertRaises(ValueError): + fixture.service._note_block_candidate_cleanup_backpressure( + caller="not-a-caller", + rows=1, + admitted=0, + depth=2, + maximum=2, + ) + + def test_the_fault_vocabulary_the_rig_injects_matches_the_cleanup_steps(self) -> None: + from tests.prism_candidate_storm import CLEANUP_FAULTS -if __name__ == "__main__": - unittest.main() + self.assertEqual( + set(CLEANUP_FAULTS) - {"floor-index"}, + set(BLOCK_CANDIDATE_COLLAPSE_CLEANUP_STEPS), + ) diff --git a/tests/test_prism_block_finalization.py b/tests/test_prism_block_finalization.py new file mode 100644 index 000000000..1ac94f367 --- /dev/null +++ b/tests/test_prism_block_finalization.py @@ -0,0 +1,271 @@ +#!/usr/bin/env python3 +"""Direct tests for the PRISM accepted-block finalization owner.""" + +from __future__ import annotations + +import ast +from pathlib import Path +from types import SimpleNamespace +import unittest +from unittest import mock + +import lab.prism.block_finalization as block_finalization_module +from lab.prism.block_finalization import ( + FINALIZATION_PHASES, + BlockFinalizationService, +) + + +class RecordingFinalizationService(BlockFinalizationService): + def __init__(self, *, admission: bool = True, landing: bool = True) -> None: + super().__init__(SimpleNamespace()) + self.events: list[str] = [] + self.admission_result: object = ( + SimpleNamespace(block_hash="ab" * 32) if admission else None + ) + self.landing_result = object() if landing else None + self.already_accounted = False + + def _record_block_candidate_progress( + self, + phase: str = "accounting-progress", + ) -> None: + return None + + def _admit_candidate( + self, + candidate: object, + *, + node_submission: object, + ) -> object | None: + self.events.append("admission") + return self.admission_result + + def _land_candidate(self, admission: object) -> object | None: + self.events.append("land_confirm") + return self.landing_result + + def _candidate_already_accounted(self, block_hash: str) -> bool: + self.events.append("accounted_check") + return self.already_accounted + + def _persist_ctv_and_credit( + self, + admission: object, + landed: object, + ) -> dict[str, object]: + self.events.append("ctv_credit") + return {"stored": True} + + def _build_finalization_evidence( + self, + admission: object, + landed: object, + ctv_persistence: dict[str, object], + ) -> object: + self.events.append("evidence") + return object() + + def _publish_finalization_evidence( + self, + landed: object, + prepared: object, + ) -> dict[str, object]: + self.events.append("audit_publish") + return {"published": True} + + def _account_finalized_candidate( + self, + admission: object, + landed: object, + published_evidence: dict[str, object], + ) -> bool: + self.events.append("accounting") + return True + + +class BlockFinalizationServiceTests(unittest.TestCase): + def test_success_runs_named_phases_once_in_durable_order(self) -> None: + service = RecordingFinalizationService() + monotonic_values = iter(range(13)) + + with mock.patch( + "lab.prism.block_finalization.time.monotonic", + side_effect=lambda: float(next(monotonic_values)), + ): + self.assertTrue( + service._submit_block_candidate_serialized( + object(), # type: ignore[arg-type] + node_submission=object(), # type: ignore[arg-type] + ) + ) + + self.assertEqual( + service.events, + [ + "admission", + "land_confirm", + "accounted_check", + "ctv_credit", + "evidence", + "audit_publish", + "accounting", + ], + ) + snapshot = service.metrics_snapshot() + for phase in FINALIZATION_PHASES: + self.assertEqual(snapshot["phases"][phase]["count"], 1) + self.assertEqual(snapshot["phases"][phase]["sum"], 1.0) + self.assertEqual(snapshot["phases"][phase]["max"], 1.0) + + def test_terminal_boundaries_do_not_run_later_phases(self) -> None: + rejected = RecordingFinalizationService(admission=False) + self.assertFalse( + rejected._submit_block_candidate_serialized( + object(), # type: ignore[arg-type] + node_submission=object(), # type: ignore[arg-type] + ) + ) + self.assertEqual(rejected.events, ["admission"]) + + # A tri-state admission verdict (already-recorded acceptance or an + # accepted race win) is terminal for the whole run: the ordered + # phases after admission never execute in either direction. + for terminal_result in (True, False): + short_circuit = RecordingFinalizationService() + short_circuit.admission_result = terminal_result + self.assertIs( + short_circuit._submit_block_candidate_serialized( + object(), # type: ignore[arg-type] + node_submission=object(), # type: ignore[arg-type] + ), + terminal_result, + ) + self.assertEqual(short_circuit.events, ["admission"]) + + not_landed = RecordingFinalizationService(landing=False) + self.assertFalse( + not_landed._submit_block_candidate_serialized( + object(), # type: ignore[arg-type] + node_submission=object(), # type: ignore[arg-type] + ) + ) + self.assertEqual(not_landed.events, ["admission", "land_confirm"]) + + replay = RecordingFinalizationService() + replay.already_accounted = True + self.assertTrue( + replay._submit_block_candidate_serialized( + object(), # type: ignore[arg-type] + node_submission=object(), # type: ignore[arg-type] + ) + ) + self.assertEqual( + replay.events, + ["admission", "land_confirm", "accounted_check"], + ) + + def test_candidate_interarrival_metric_is_bounded_aggregate_evidence(self) -> None: + service = BlockFinalizationService(SimpleNamespace()) + with mock.patch( + "lab.prism.block_finalization.time.monotonic", + side_effect=[3.0, 7.5, 13.0], + ): + service._note_candidate_started() + service._note_candidate_started() + service._note_candidate_started() + + intervals = service.metrics_snapshot()["candidate_intervals"] + self.assertEqual(intervals["count"], 2) + self.assertEqual(intervals["sum"], 10.0) + self.assertEqual(intervals["min"], 4.5) + lines = service.metrics_lines() + self.assertIn( + "qbit_prism_block_candidate_interarrival_seconds_count 2", + lines, + ) + self.assertIn( + "qbit_prism_block_candidate_interarrival_seconds_min 4.500000", + lines, + ) + + +class BlockFinalizationPortSurfaceTests(unittest.TestCase): + """Mechanically compare runtime accesses against the declared port.""" + + # The verify_bundle override is read through runtime.__dict__ on purpose + # (a per-instance escape hatch kept out of the typed port), so the AST + # sees a "__dict__" attribute access that no Protocol should declare. + PORT_ACCESS_ALLOWLIST = frozenset({"__dict__"}) + + @staticmethod + def _module_tree() -> ast.Module: + source_path = Path(block_finalization_module.__file__) + return ast.parse(source_path.read_text(encoding="utf-8")) + + @classmethod + def _class_node(cls, tree: ast.Module, name: str) -> ast.ClassDef: + for node in ast.walk(tree): + if isinstance(node, ast.ClassDef) and node.name == name: + return node + raise AssertionError(f"class {name} not found in module source") + + def test_every_runtime_access_is_declared_in_the_port(self) -> None: + tree = self._module_tree() + service = self._class_node(tree, "BlockFinalizationService") + port = self._class_node(tree, "BlockFinalizationPort") + + port_members: set[str] = set() + for statement in port.body: + if isinstance( + statement, (ast.FunctionDef, ast.AsyncFunctionDef) + ): + port_members.add(statement.name) + elif isinstance(statement, ast.AnnAssign) and isinstance( + statement.target, ast.Name + ): + port_members.add(statement.target.id) + self.assertTrue(port_members, "port declared no members") + + runtime_accesses: set[str] = set() + for node in ast.walk(service): + if ( + isinstance(node, ast.Attribute) + and isinstance(node.value, ast.Attribute) + and isinstance(node.value.value, ast.Name) + and node.value.value.id == "self" + and node.value.attr == "runtime" + ): + runtime_accesses.add(node.attr) + if ( + isinstance(node, ast.Call) + and isinstance(node.func, ast.Name) + and node.func.id == "getattr" + and len(node.args) >= 2 + and isinstance(node.args[0], ast.Attribute) + and isinstance(node.args[0].value, ast.Name) + and node.args[0].value.id == "self" + and node.args[0].attr == "runtime" + and isinstance(node.args[1], ast.Constant) + and isinstance(node.args[1].value, str) + ): + runtime_accesses.add(node.args[1].value) + self.assertTrue(runtime_accesses, "service accessed nothing via runtime") + + undeclared = ( + runtime_accesses - port_members - self.PORT_ACCESS_ALLOWLIST + ) + self.assertEqual( + undeclared, + set(), + "runtime accesses missing from BlockFinalizationPort: " + f"{sorted(undeclared)}", + ) + + def test_service_no_longer_forwards_attribute_reads(self) -> None: + self.assertNotIn("__getattr__", vars(BlockFinalizationService)) + self.assertNotIn("__setattr__", vars(BlockFinalizationService)) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_prism_bounded_canonical_finalization.py b/tests/test_prism_bounded_canonical_finalization.py new file mode 100644 index 000000000..06e3d328e --- /dev/null +++ b/tests/test_prism_bounded_canonical_finalization.py @@ -0,0 +1,1304 @@ +#!/usr/bin/env python3 +"""Bounded canonical candidate finalization and audit body publication (#255). + +End to end through the real Rust builder and verifier when the tools are +available: the compiler streams any share sequence into the builder and +hands back a bounded view of the canonical artifact; the audit store +publishes the canonical bundle, the compact body and its share segments +from that artifact through bounded chunks while preserving exact canonical +bytes, digests, signature and coinbase validation, inode identity binding +and idempotent retries; and accepted-block persistence consumes the view. +The fake-builder cases cover the compiler seam without the tools. +""" + +from __future__ import annotations + +from collections.abc import Sequence +import gc +import gzip +import hashlib +import json +import os +from pathlib import Path +import re +import subprocess +import tempfile +import unittest +from unittest import mock +import weakref +from typing import Any + +from lab.prism.audit_artifacts import ( + AuditArtifactConfig, + AuditArtifactStore, + gzip_canonical_bundle_bytes, +) +from lab.prism.audit_bundle_view import ( + SCAN_CHUNK_BYTES, + CanonicalAuditBundleView, + LazyRecordSequence, +) +from lab.prism.bundle_compiler import _iter_build_input_chunks +from lab.prism.prism_tools import prism_tool_command +from lab.prism.share_json_stream import iter_json_object_text_chunks +from lab.prism.share_ledger import PsqlShareLedger +from tests.prism_coordinator_test_support import coordinator as make_coordinator +from tests.prism_vardiff_test_support import fake_audit_bundle_popen + + +BLOCK_HASH = "a1" * 32 +PARENT_HASH = "b2" * 32 +ANCHOR_MS = 1_700_000_000_000 +COINBASE_VALUE_SATS = 50_00000000 +SIGNING_SEED = "42" * 32 +LEDGER_SEED = "43" * 32 +_TOOLS_STATE: dict[str, bool] = {} + + +def _repo_root() -> Path: + return Path(__file__).resolve().parents[1] + + +def real_tools_available() -> bool: + """True when the qbit-prism builder and verifier can run here.""" + if "available" in _TOOLS_STATE: + return _TOOLS_STATE["available"] + if not os.environ.get("PRISM_TOOL_BIN_DIR"): + local = _repo_root() / "target" / "release" + if (local / "qbit-prism-build-audit-bundle").exists(): + os.environ["PRISM_TOOL_BIN_DIR"] = str(local) + available = True + for name in ("qbit-prism-build-audit-bundle", "qbit-prism-audit-verify"): + try: + completed = subprocess.run( + prism_tool_command(name) + ["--help"], + capture_output=True, + timeout=600, + check=False, + ) + except (OSError, subprocess.TimeoutExpired): + available = False + break + if completed.returncode != 0: + available = False + break + _TOOLS_STATE["available"] = available + return available + + +def share_record(index: int, *, oversized_bytes: int = 0) -> dict[str, object]: + record: dict[str, object] = { + "share_seq": index + 1, + "share_id": "Z" * oversized_bytes if oversized_bytes else f"share-{index + 1}", + "miner_id": f"miner-{index % 3}", + "order_key": f"o{index % 3}", + "p2mr_program_hex": ("%02x" % (index % 3)) * 32, + "share_difficulty": 1, + "network_difficulty": 1000, + "template_height": 10, + "job_id": "job-1", + "job_issued_at_ms": ANCHOR_MS - 1000, + "accepted_at_ms": ANCHOR_MS - 500, + "ntime": 1_700_000_000, + "credit_policy": None if index % 2 else "standard", + } + return record + + +def found_block(height: int = 10) -> dict[str, object]: + return { + "block_height": height, + "coinbase_value_sats": COINBASE_VALUE_SATS, + "network_difficulty": 1000, + "anchor_job_issued_at_ms": ANCHOR_MS, + } + + +class GuardedShareSequence(Sequence): + """A replayable share window that refuses to be copied into a list. + + ``list(seq)`` consults ``__len__`` for its length hint; the streaming + encoder only iterates. Indexing is allowed for the segment slices. + """ + + def __init__(self, records: list[dict[str, object]]) -> None: + self._records = records + self.iterations = 0 + + def __iter__(self): # type: ignore[no-untyped-def] + self.iterations += 1 + yield from self._records + + def __len__(self) -> int: + raise AssertionError("share window must not be materialized") + + def __getitem__(self, index: int | slice) -> Any: + return self._records[index] + + +def coordinator_server() -> Any: + result = make_coordinator() + server = result[0] if isinstance(result, tuple) else result + server.signing_seed_hex = SIGNING_SEED + server.ledger_attestation_signing_seed_hex = LEDGER_SEED + return server + + +class FakeLeasePsqlShareLedger(PsqlShareLedger): + """Scripted psql ledger: every statement is recorded, results are queued.""" + + def __init__(self, lease_results: list[dict[str, object]], **kwargs: Any) -> None: + self.lease_results = list(lease_results) + self.lease_queries: list[str] = [] + kwargs.setdefault("native_client_mode", "0") + super().__init__( + psql_command="psql postgresql://example.invalid/qbit", + lease_retry_sleep=lambda _seconds: None, + **kwargs, + ) + + def _make_writer_lease_guard(self, _database_url: str | None) -> Any: + class FakeGuard: + held = True + + def try_acquire(self) -> bool: + return True + + def close(self) -> None: + self.held = False + + return FakeGuard() + + def _run_json(self, sql: str) -> Any: + self.lease_queries.append(sql) + if not self.lease_results: + raise AssertionError("unexpected extra ledger statement") + return self.lease_results.pop(0) + + +def acquired_lease() -> dict[str, object]: + return { + "acquired": True, + "writer_id": "writer-a", + "writer_epoch": 1, + "writer_session_token": "session-a", + } + + +def persist_result() -> dict[str, object]: + return { + "backend": "postgres-psql", + "share_count": 0, + "block_count": 1, + "bundle_count": 1, + "payout_entry_count": 3, + "carry_forward_count": 3, + "onchain_output_count": 0, + } + + +def payload_from_sql(sql: str) -> dict[str, Any]: + match = re.search(r"\$(qbit_prism_json(?:_x)*)\$(.*)\$\1\$::jsonb", sql, re.S) + if match is None: + raise AssertionError("persist statement carries no payload literal") + return json.loads(match.group(2)) + + +class _RealBuildMixin: + def build_candidate( + self, + store: AuditArtifactStore, + server: Any, + shares: Sequence[dict[str, object]], + *, + block_hash: str = BLOCK_HASH, + height: int = 10, + cancellation: Any = None, + ) -> tuple[Any, CanonicalAuditBundleView]: + candidate = store.issue_candidate(block_hash=block_hash) + adopted: list[Path] = [] + + def adopt(path: Path, value: os.stat_result) -> None: + store.adopt_compiler_candidate(candidate, path=path, value=value) + adopted.append(path) + + parent_fd = store.duplicate_root_directory_fd() + try: + view = server.build_audit_bundle( + shares=shares, + found_block=found_block(height), + prior_balances=[], + coinbase_script_sig_suffix_hex="00", + witness_merkle_leaves_hex=["ab" * 32], + canonical_output_path=candidate.path, + canonical_output_parent_fd=parent_fd, + canonical_output_adopter=adopt, + cancellation=cancellation, + ) + except BaseException: + store.discard_candidate(candidate) + raise + finally: + os.close(parent_fd) + assert adopted == [candidate.path] + return candidate, view + + def verify_candidate( + self, + store: AuditArtifactStore, + candidate: Any, + view: CanonicalAuditBundleView, + *, + height: int = 10, + ) -> dict[str, Any]: + verified = store.verify_candidate( + candidate, + coinbase_tx_hex=view["signed_coinbase_manifest"]["manifest"]["coinbase_tx_hex"], + expected_coinbase_value_sats=COINBASE_VALUE_SATS, + expected_block_height=height, + trusted_writer_public_key_hex=AuditArtifactStore.trusted_writer_key( + None, + view, + allow_embedded_test_key=True, + ), + trust_source="embedded_test_only", + ) + store.require_current_verified_candidate(verified, candidate) + assert verified.canonical_copy_eligible + return dict(verified.report) + + +@unittest.skipUnless(real_tools_available(), "qbit-prism builder and verifier are unavailable") +class RealBuilderBoundedFinalizationTests(_RealBuildMixin, unittest.TestCase): + def make_store(self, root: Path, *, share_segment_size: int) -> tuple[AuditArtifactStore, mock.Mock]: + canonicalizer = mock.Mock( + side_effect=AssertionError("bounded publication must not canonicalize a bundle"), + ) + store = AuditArtifactStore( + AuditArtifactConfig( + root=root, + evidence_path=root / "evidence.json", + share_segment_size=share_segment_size, + ), + canonicalizer=canonicalizer, + ) + self.addCleanup(store.close) + return store, canonicalizer + + def test_canonical_build_returns_bounded_view_from_a_streamed_sequence(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + store, _canonicalizer = self.make_store(root, share_segment_size=100) + server = coordinator_server() + records = [share_record(index) for index in range(1300)] + window = GuardedShareSequence(records) + candidate, view = self.build_candidate(store, server, window) + try: + self.assertIsInstance(view, CanonicalAuditBundleView) + self.assertGreaterEqual(window.iterations, 1) + raw = candidate.path.read_bytes() + self.assertEqual(view.sha256_hex, hashlib.sha256(raw).hexdigest()) + self.assertEqual(view.byte_length, len(raw)) + loaded = json.loads(raw) + self.assertEqual(view, loaded) + self.assertEqual(loaded, view) + self.assertIsInstance(view["shares"], LazyRecordSequence) + self.assertEqual(len(view["shares"]), 1300) + self.assertIsInstance(view["reward_manifest"]["shares"], LazyRecordSequence) + self.assertEqual(view["shares"][0]["share_seq"], 1) + self.assertEqual(view["shares"][-1]["share_seq"], 1300) + # The typed Rust order is preserved; the sorted v1 identity + # encoding is a different contract and is not used here. + self.assertEqual(list(view)[:3], ["schema", "shares", "found_block"]) + report = self.verify_candidate(store, candidate, view) + self.assertEqual(report["audit_bundle_sha256_hex"], view.sha256_hex) + self.assertEqual( + AuditArtifactStore.trusted_writer_key(None, view, allow_embedded_test_key=True), + AuditArtifactStore.trusted_writer_key(None, loaded, allow_embedded_test_key=True), + ) + self.assertEqual( + view["payout_policy_manifest"]["accounts"], + loaded["payout_policy_manifest"]["accounts"], + ) + finally: + view.close() + store.discard_candidate(candidate) + + def test_bounded_publication_preserves_bytes_digests_and_compact_roundtrip(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + store, canonicalizer = self.make_store(root, share_segment_size=250) + server = coordinator_server() + records = [share_record(index) for index in range(3000)] + candidate, view = self.build_candidate(store, server, GuardedShareSequence(records)) + try: + raw = candidate.path.read_bytes() + self.assertGreater(len(raw), 2 * SCAN_CHUNK_BYTES) + report = self.verify_candidate(store, candidate, view) + expected = report["audit_bundle_sha256_hex"] + body_uri = str(store.body_path(BLOCK_HASH, expected)) + payload = {"block_hash": BLOCK_HASH, "audit_bundle_sha256": expected} + real_loads = json.loads + real_dumps = json.dumps + + def bounded_loads(text: Any, *args: Any, **kwargs: Any) -> Any: + if len(text) > SCAN_CHUNK_BYTES: + raise AssertionError("publication decoded a window-sized document") + return real_loads(text, *args, **kwargs) + + def bounded_dumps(value: Any, *args: Any, **kwargs: Any) -> str: + encoded = real_dumps(value, *args, **kwargs) + if len(encoded) > SCAN_CHUNK_BYTES: + raise AssertionError("publication encoded a window-sized document") + return encoded + + with mock.patch("lab.prism.audit_artifacts.json.loads", side_effect=bounded_loads), mock.patch( + "lab.prism.audit_artifacts.json.dumps", + side_effect=bounded_dumps, + ): + published = store.prepare_external_audit_body( + payload, + view, + body_uri=body_uri, + canonical_bundle_path=candidate.path, + ) + self.assertEqual(published, body_uri) + canonicalizer.assert_not_called() + body_path = Path(body_uri) + body = json.loads(body_path.read_bytes()) + self.assertEqual(body["schema"], "qbit.prism.audit-bundle.v2") + self.assertEqual(body["share_count"], 3000) + parts = body["share_window_proof"]["share_parts"] + self.assertEqual(len(parts), 12) + self.assertTrue(all(part["kind"] == "segment_range" for part in parts)) + # Exact canonical bytes and their reproducible compression. + self.assertEqual(store.read_canonical_audit_bundle(BLOCK_HASH, expected), raw) + self.assertEqual(store.stored_canonical_bundle_sha256(BLOCK_HASH, expected), expected) + gz_path = store.canonical_bundle_path(BLOCK_HASH, expected) + self.assertEqual(gz_path.read_bytes(), gzip_canonical_bundle_bytes(raw)) + self.assertEqual(gzip.decompress(gz_path.read_bytes()), raw) + # The compact body reconstructs to the exact logical bundle and + # the reconstruction's Rust canonical digest is the advertised + # one (the reader canonicalizes; the publication never did). + canonicalizer.side_effect = None + canonicalizer.return_value = None + from lab.prism.bundle_compiler import canonical_bundle_bytes + + canonicalizer.side_effect = canonical_bundle_bytes + reconstructed = store.read_external_body(body_uri, expected_sha256=expected) + self.assertEqual(reconstructed, json.loads(raw)) + self.assertEqual(reconstructed, view) + canonicalizer.side_effect = AssertionError("retry must not canonicalize") + self.assertEqual( + store.audit_body_byte_len(body_uri, view, candidate.path), + body_path.stat().st_size, + ) + metrics = store.metrics_snapshot() + self.assertEqual(metrics["canonical_bundle"]["files"], 1) + self.assertEqual(metrics["body"]["files"], 1) + self.assertEqual(metrics["share_segment"]["files"], 12) + # An exact retry after the body landed is a durable no-op that + # neither rewrites nor reconstructs. + before = { + path: (path.stat().st_ino, path.stat().st_mtime_ns) + for path in root.iterdir() + if path.is_file() + } + with mock.patch.object( + store, + "resolve_audit_bundle_v2", + side_effect=AssertionError("retry must not reconstruct the window"), + ), mock.patch.object( + store, + "read_audit_share_segment", + side_effect=AssertionError("retry must not load a segment whole"), + ): + retried = store.prepare_external_audit_body( + payload, + view, + body_uri=body_uri, + canonical_bundle_path=candidate.path, + ) + self.assertEqual(retried, body_uri) + after = { + path: (path.stat().st_ino, path.stat().st_mtime_ns) + for path in root.iterdir() + if path.is_file() + } + self.assertEqual(before, after) + # The candidate artifact itself is a dot-prefixed .tmp name; + # no writer temporaries may remain beside it. + self.assertEqual( + [path for path in root.glob(".*.tmp") if "candidate" not in path.name], + [], + ) + finally: + view.close() + store.discard_candidate(candidate) + + def test_view_without_a_canonical_path_publishes_from_its_own_source(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + store, canonicalizer = self.make_store(root, share_segment_size=0) + server = coordinator_server() + candidate, view = self.build_candidate(store, server, [share_record(i) for i in range(7)]) + try: + raw = candidate.path.read_bytes() + expected = view.sha256_hex + body_uri = str(store.body_path(BLOCK_HASH, expected)) + published = store.prepare_external_audit_body( + {"block_hash": BLOCK_HASH, "audit_bundle_sha256": expected}, + view, + body_uri=body_uri, + ) + self.assertEqual(published, body_uri) + # No segmenting configured: the body is the literal canonical + # bundle, copied from the artifact source. + self.assertEqual(Path(body_uri).read_bytes(), raw) + self.assertEqual(store.read_canonical_audit_bundle(BLOCK_HASH, expected), raw) + canonicalizer.assert_not_called() + self.assertEqual(store.audit_body_byte_len(None, view), len(raw)) + finally: + view.close() + store.discard_candidate(candidate) + + def test_tampered_or_foreign_artifacts_are_rejected(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + store, _canonicalizer = self.make_store(root, share_segment_size=4) + server = coordinator_server() + candidate, view = self.build_candidate(store, server, [share_record(i) for i in range(9)]) + other, other_view = self.build_candidate( + store, + server, + [share_record(i) for i in range(10)], + block_hash=PARENT_HASH, + ) + try: + raw = candidate.path.read_bytes() + expected = view.sha256_hex + payload = {"block_hash": BLOCK_HASH, "audit_bundle_sha256": expected} + body_uri = str(store.body_path(BLOCK_HASH, expected)) + # A view over a different artifact than the canonical path. + with self.assertRaisesRegex(RuntimeError, "does not match logical bundle"): + store.prepare_external_audit_body( + payload, + other_view, + body_uri=body_uri, + canonical_bundle_path=candidate.path, + ) + # A dictionary that differs in one record. + tampered = json.loads(raw) + tampered["shares"][4]["share_difficulty"] = 2 + with self.assertRaisesRegex(RuntimeError, "does not match logical bundle"): + store.validate_canonical_source(candidate.path, expected, tampered) + store.validate_canonical_source(candidate.path, expected, json.loads(raw)) + store.validate_canonical_source(candidate.path, expected, view) + # The artifact rewritten under the view after the scan. + candidate.path.write_bytes(raw + b" ") + with self.assertRaisesRegex(RuntimeError, "sha256 mismatch"): + store.prepare_external_audit_body( + payload, + view, + body_uri=body_uri, + canonical_bundle_path=candidate.path, + ) + with self.assertRaises(Exception): + store.prepare_external_audit_body(payload, view, body_uri=body_uri) + self.assertEqual(list(root.glob("prism-audit-bundle-body-*")), []) + finally: + view.close() + other_view.close() + store.discard_candidate(candidate) + store.discard_candidate(other) + + def test_oversized_single_field_publishes_through_bounded_writers(self) -> None: + oversized = 2 * SCAN_CHUNK_BYTES + 3 + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + store, canonicalizer = self.make_store(root, share_segment_size=3) + server = coordinator_server() + records = [ + share_record(index, oversized_bytes=oversized if index == 4 else 0) + for index in range(11) + ] + candidate, view = self.build_candidate(store, server, records) + try: + raw = candidate.path.read_bytes() + self.assertGreater(len(raw), 2 * oversized) + report = self.verify_candidate(store, candidate, view) + expected = report["audit_bundle_sha256_hex"] + body_uri = str(store.body_path(BLOCK_HASH, expected)) + published = store.prepare_external_audit_body( + {"block_hash": BLOCK_HASH, "audit_bundle_sha256": expected}, + view, + body_uri=body_uri, + canonical_bundle_path=candidate.path, + ) + self.assertEqual(published, body_uri) + canonicalizer.assert_not_called() + self.assertEqual(store.read_canonical_audit_bundle(BLOCK_HASH, expected), raw) + from lab.prism.bundle_compiler import canonical_bundle_bytes + + canonicalizer.side_effect = canonical_bundle_bytes + self.assertEqual( + store.read_external_body(body_uri, expected_sha256=expected), + json.loads(raw), + ) + segments = sorted(root.glob("prism-audit-share-segment-slot-*.json")) + self.assertEqual(len(segments), 4) + self.assertTrue(any(path.stat().st_size > oversized for path in segments)) + finally: + view.close() + store.discard_candidate(candidate) + + def test_multiple_generations_publish_and_release_independently(self) -> None: + gc.disable() + try: + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + store, _canonicalizer = self.make_store(root, share_segment_size=20) + server = coordinator_server() + digests: list[str] = [] + refs: list[weakref.ref] = [] + for generation in range(3): + block_hash = ("%02x" % (0xC0 + generation)) * 32 + records = [share_record(index) for index in range(40 + generation)] + candidate, view = self.build_candidate( + store, + server, + GuardedShareSequence(records), + block_hash=block_hash, + height=10 + generation, + ) + try: + report = self.verify_candidate(store, candidate, view, height=10 + generation) + expected = report["audit_bundle_sha256_hex"] + body_uri = str(store.body_path(block_hash, expected)) + self.assertEqual( + store.prepare_external_audit_body( + {"block_hash": block_hash, "audit_bundle_sha256": expected}, + view, + body_uri=body_uri, + canonical_bundle_path=candidate.path, + ), + body_uri, + ) + digests.append(expected) + refs.append(weakref.ref(view)) + source_fd = view.source.fileno() + finally: + view.close() + store.discard_candidate(candidate) + with self.assertRaises(OSError): + os.fstat(source_fd) + del view + self.assertIsNone(refs[-1]()) + self.assertEqual(len(set(digests)), 3) + metrics = store.metrics_snapshot() + self.assertEqual(metrics["body"]["files"], 3) + self.assertEqual(metrics["canonical_bundle"]["files"], 3) + self.assertEqual(metrics["candidate"]["files"], 0) + finally: + gc.enable() + + def test_cancellation_during_output_scan_cleans_the_candidate(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + store, _canonicalizer = self.make_store(root, share_segment_size=0) + server = coordinator_server() + + class Cancelled(RuntimeError): + pass + + class Cancellation: + def __init__(self) -> None: + self.scan_checks = 0 + + def is_set(self) -> bool: + return False + + def raise_if_cancelled(self, phase: str) -> None: + if phase == "builder output scan": + self.scan_checks += 1 + if self.scan_checks == 2: + raise Cancelled(phase) + + cancellation = Cancellation() + records = [share_record(index) for index in range(1500)] + with self.assertRaises(Cancelled): + self.build_candidate(store, server, records, cancellation=cancellation) + self.assertEqual(cancellation.scan_checks, 2) + self.assertEqual(list(root.glob(".prism-live-audit-bundle-candidate-*")), []) + self.assertEqual(store.metrics_snapshot()["candidate"]["files"], 0) + + def test_accepted_block_persistence_consumes_the_view(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) / "audit" + root.mkdir() + canonicalizer = mock.Mock(side_effect=AssertionError("persistence must not canonicalize")) + ledger = FakeLeasePsqlShareLedger( + [ + acquired_lease(), + {"existing_block": False, "existing_body_uri": None}, + persist_result(), + ], + audit_body_dir=root, + audit_bundle_canonicalizer=canonicalizer, + audit_share_segment_size=50, + ) + store = ledger._audit_artifact_store + assert store is not None + server = coordinator_server() + candidate, view = self.build_candidate(store, server, GuardedShareSequence( + [share_record(index) for index in range(120)] + )) + try: + raw = candidate.path.read_bytes() + report = self.verify_candidate(store, candidate, view) + persistence = ledger.persist_accepted_block( + block_hash=BLOCK_HASH, + block_height=10, + parent_hash=PARENT_HASH, + final_bundle=view, + audit_report=report, + canonical_bundle_path=candidate.path, + ) + canonicalizer.assert_not_called() + body_uri = str(store.body_path(BLOCK_HASH, report["audit_bundle_sha256_hex"])) + self.assertEqual(persistence["body_uri"], body_uri) + self.assertEqual(persistence["audit_body_byte_len"], Path(body_uri).stat().st_size) + payload = payload_from_sql(ledger.lease_queries[-1]) + loaded = json.loads(raw) + self.assertIsNone(payload["audit_bundle"]) + self.assertEqual(payload["body_uri"], body_uri) + self.assertEqual(payload["accounts"], loaded["payout_policy_manifest"]["accounts"]) + self.assertEqual(payload["witness_merkle_leaves_hex"], loaded["witness_merkle_leaves_hex"]) + self.assertEqual( + payload["audit_commitment_leaves_hex"], + loaded["audit_commitment_leaves_hex"], + ) + self.assertEqual(payload["found_block_network_difficulty"], 1000) + self.assertEqual(payload["schema_version"], loaded["schema"]) + self.assertEqual(store.read_canonical_audit_bundle(BLOCK_HASH, view.sha256_hex), raw) + finally: + view.close() + store.discard_candidate(candidate) + + def test_inline_persistence_lane_splices_canonical_bytes(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + store, _canonicalizer = self.make_store(root, share_segment_size=0) + server = coordinator_server() + candidate, view = self.build_candidate(store, server, [share_record(i) for i in range(5)]) + try: + raw = candidate.path.read_bytes() + report = self.verify_candidate(store, candidate, view) + ledger = FakeLeasePsqlShareLedger([acquired_lease(), persist_result()]) + persistence = ledger.persist_accepted_block( + block_hash=BLOCK_HASH, + block_height=10, + parent_hash=PARENT_HASH, + final_bundle=view, + audit_report=report, + ) + self.assertEqual(persistence["body_uri"], "") + self.assertEqual(persistence["audit_body_byte_len"], len(raw)) + payload = payload_from_sql(ledger.lease_queries[-1]) + self.assertEqual(payload["audit_bundle"], json.loads(raw)) + self.assertIsNone(payload["body_uri"]) + self.assertIn( + json.dumps(json.loads(raw), separators=(",", ":"))[:64], + ledger.lease_queries[-1], + ) + finally: + view.close() + store.discard_candidate(candidate) + + +def ctv_manifest_set() -> dict[str, Any]: + """A two-chunk CTV fanout manifest set shaped for the recovery payload.""" + + def manifest(index: int, output_sum: int, covenant: int) -> dict[str, Any]: + return { + "schema": "qbit.prism.ctv-fanout-manifest.v1", + "precommitment": { + "chunk_index": index, + "chunk_count": 2, + "fanout_fee_sats": 0, + "anchor_vout": 1, + "ctv_hash_hex": ("e%x" % index) * 32, + "fanout_tx_template_hex": ("0a%02x" % index) * 40, + "fanout_output_sum_sats": output_sum, + "block_height": 10, + }, + "precommitment_sha256_hex": ("f%x" % index) * 32, + "commitment_witness_leaf_hex": ("a%x" % index) * 32, + "parent_coinbase_txid": "cc" * 32, + "parent_coinbase_tx_hex": "0102", + "parent_coinbase_vout": 1, + "covenant_output_value_sats": covenant, + "fanout_tx_hex": ("0b%02x" % index) * 60, + "fanout_txid": ("d%x" % index) * 32, + } + + return { + "schema": "qbit.prism.ctv-fanout-manifest-set.v1", + "block_height": 10, + "settlement_mode": "ctv_fanout", + "parent_coinbase_txid": "cc" * 32, + "fanout_count": 2, + "fanout_output_sum_sats": 30, + "covenant_output_value_sats": 40, + "manifests": [manifest(0, 10, 15), manifest(1, 20, 25)], + } + + +def synthetic_bundle(share_count: int, *, recipients: int = 3) -> dict[str, Any]: + """A canonical-shaped bundle with recipient-scaled members for the store tests.""" + shares = [share_record(index) for index in range(share_count)] + accounts = [ + { + "recipient_id": f"miner-{index}", + "order_key": f"o{index}", + "p2mr_program_hex": ("%02x" % (index % 251)) * 32, + "gross_amount_sats": 1000 + index, + "prior_balance_sats": 0, + "candidate_balance_sats": 1000 + index, + "onchain_amount_sats": 0, + "carry_forward_balance_sats": 1000 + index, + "action": "accrued", + } + for index in range(recipients) + ] + return { + "schema": "qbit.prism.audit-bundle.v1", + "shares": shares, + "found_block": found_block(), + "prior_balances": [ + {"recipient_id": f"miner-{index}", "order_key": f"o{index}", "p2mr_program_hex": ("%02x" % (index % 251)) * 32, "balance_sats": index} + for index in range(recipients) + ], + "payout_policy": {"p2mr_spend_input_bytes": 1}, + "witness_merkle_leaves_hex": ["ab" * 32], + "audit_commitment_leaves_hex": ["cd" * 32], + "ledger_window_attestation": {"signature": {"public_key_hex": "44" * 32}}, + "reward_manifest": { + "schema": "qbit.prism.reward-manifest.v1", + "included_share_count": share_count, + "shares": [{"share_seq": index + 1, "counted_difficulty": 1} for index in range(share_count)], + "entitlements": [{"recipient_id": f"miner-{index}", "weight": 1} for index in range(recipients)], + }, + "payout_policy_manifest": {"accounts": accounts, "onchain_entitlements": []}, + "ctv_fanout_manifest_set": ctv_manifest_set(), + "signed_coinbase_manifest": {"manifest": {"coinbase_tx_hex": "00", "payout_count": 0}}, + } + + +def bounded_json_guards(module_path: str, limit: int, *, guard_loads: bool = True) -> Any: + """Refuse json.dumps/json.loads calls above ``limit`` while active. + + The patch lands on the shared ``json`` module, so it covers every + caller in the process; tests whose fakes legitimately decode a whole + captured payload pass ``guard_loads=False``. + """ + real_dumps = json.dumps + real_loads = json.loads + + def guarded_dumps(value: Any, *args: Any, **kwargs: Any) -> str: + encoded = real_dumps(value, *args, **kwargs) + if len(encoded) > limit: + raise AssertionError(f"{module_path} encoded {len(encoded)} bytes in one call") + return encoded + + def guarded_loads(text: Any, *args: Any, **kwargs: Any) -> Any: + if len(text) > limit: + raise AssertionError(f"{module_path} decoded {len(text)} bytes in one call") + return real_loads(text, *args, **kwargs) + + class Guards: + def __enter__(self) -> "Guards": + self._patches = [ + mock.patch(f"{module_path}.json.dumps", side_effect=guarded_dumps), + ] + if guard_loads: + self._patches.append( + mock.patch(f"{module_path}.json.loads", side_effect=guarded_loads) + ) + for patch in self._patches: + patch.start() + return self + + def __exit__(self, *exc: object) -> None: + for patch in reversed(self._patches): + patch.stop() + + return Guards() + + +class RemainingBoundaryTests(unittest.TestCase): + """The follow-up closures: parts index, pressure, persistence and CTV.""" + + def make_store(self, root: Path, *, share_segment_size: int) -> AuditArtifactStore: + store = AuditArtifactStore( + AuditArtifactConfig( + root=root, + evidence_path=root / "evidence.json", + share_segment_size=share_segment_size, + ), + canonicalizer=lambda bundle: json.dumps(bundle, separators=(",", ":")).encode(), + ) + self.addCleanup(store.close) + return store + + def write_view(self, root: Path, bundle: dict[str, Any]) -> tuple[Path, bytes, CanonicalAuditBundleView]: + raw = json.dumps(bundle, separators=(",", ":")).encode() + path = root / f".prism-live-audit-bundle-candidate-{BLOCK_HASH}-{'0' * 32}.json.tmp" + path.write_bytes(raw) + view = CanonicalAuditBundleView.scan_path(path) + self.addCleanup(view.close) + return path, raw, view + + def test_small_segment_size_keeps_the_part_index_on_disk(self) -> None: + from lab.prism.audit_bundle_view import LazyRecordSequence as Lazy + + # segment_size=1 is the stress (one part per share); 250 the control. + for segment_size, expected_parts in ((1, 1500), (7, 215), (250, 6)): + with self.subTest(segment_size=segment_size), tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + store = self.make_store(root, share_segment_size=segment_size) + bundle = synthetic_bundle(1500) + path, raw, view = self.write_view(root, bundle) + expected = hashlib.sha256(raw).hexdigest() + body_uri = str(store.body_path(BLOCK_HASH, expected)) + with bounded_json_guards("lab.prism.audit_artifacts", 256 * 1024): + scratch: list[Any] = [] + parts = store.audit_share_range_parts(view["shares"], scratch=scratch) + assert parts is not None + self.assertIsInstance(parts, Lazy) + self.assertEqual(len(parts), expected_parts) + self.assertEqual(parts[0]["first_share_seq"], 1) + self.assertEqual(parts[-1]["last_share_seq"], 1500) + for scratch_view in scratch: + scratch_view.close() + published = store.prepare_external_audit_body( + {"block_hash": BLOCK_HASH, "audit_bundle_sha256": expected}, + view, + body_uri=body_uri, + canonical_bundle_path=path, + ) + self.assertEqual(published, body_uri) + body = json.loads(Path(body_uri).read_bytes()) + proof = body["share_window_proof"] + self.assertEqual(len(proof["share_parts"]), expected_parts) + # The incremental parts digest equals the reader's whole + # encoding, and the compact body reconstructs exactly. + self.assertEqual( + proof["share_parts_digest_hex"], + hashlib.sha256(store.storage_json_bytes({"share_parts": proof["share_parts"]})).hexdigest(), + ) + self.assertEqual(store.read_external_body(body_uri, expected_sha256=expected), bundle) + # A retry verifies through the lazily parsed part index. + with bounded_json_guards("lab.prism.audit_artifacts", 256 * 1024): + self.assertEqual( + store.prepare_external_audit_body( + {"block_hash": BLOCK_HASH, "audit_bundle_sha256": expected}, + view, + body_uri=body_uri, + canonical_bundle_path=path, + ), + body_uri, + ) + + def test_owned_open_failures_retire_descriptors_before_the_exception_returns(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + store = self.make_store(root, share_segment_size=4) + bundle = synthetic_bundle(6) + path, _raw, _view = self.write_view(root, bundle) + opened: list[int] = [] + real_open = os.open + + def recording_open(*args: Any, **kwargs: Any) -> int: + fd = real_open(*args, **kwargs) + if kwargs.get("dir_fd") == store._root_fd and args[0] == path.name: + opened.append(fd) + return fd + + gc_was_enabled = gc.isenabled() + gc.disable() + try: + for opener in ( + lambda: store._open_owned_artifact_source(path), + lambda: store._scan_owned_artifact(path, lazy_paths=(("shares",),)), + ): + opened.clear() + checks = {"count": 0} + real_validate = store._validate_owned_parent + + def failing_second_check(target: Path) -> None: + checks["count"] += 1 + if checks["count"] == 2: + raise RuntimeError("audit artifact root identity changed") + real_validate(target) + + retained: BaseException | None = None + with mock.patch("lab.prism.audit_artifacts.os.open", side_effect=recording_open), mock.patch.object( + store, + "_validate_owned_parent", + side_effect=failing_second_check, + ): + try: + opener() + except RuntimeError as exc: + retained = exc + # The exception and its traceback frames (which reference + # the view or source locals) are still retained here; the + # descriptors must already be closed regardless. + assert retained is not None + self.assertIn("identity changed", str(retained)) + self.assertIsNotNone(retained.__traceback__) + self.assertEqual(len(opened), 1) + for fd in opened: + with self.assertRaises(OSError): + os.fstat(fd) + del retained + finally: + if gc_was_enabled: + gc.enable() + + def test_scratch_pressure_is_reported_not_classified_as_mismatch(self) -> None: + from lab.prism.audit_bundle_view import ArtifactResourcePressure + + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + store = self.make_store(root, share_segment_size=100) + bundle = synthetic_bundle(500) + path, raw, view = self.write_view(root, bundle) + expected = hashlib.sha256(raw).hexdigest() + body_uri = str(store.body_path(BLOCK_HASH, expected)) + real_temporary_file = tempfile.TemporaryFile + calls = {"count": 0} + + def failing_after_first(*args: Any, **kwargs: Any) -> Any: + calls["count"] += 1 + if calls["count"] > 1: + raise OSError("no scratch space") + return real_temporary_file(*args, **kwargs) + + with mock.patch("lab.prism.audit_bundle_view.tempfile.TemporaryFile", side_effect=failing_after_first): + with self.assertRaises(ArtifactResourcePressure): + store.prepare_external_audit_body( + {"block_hash": BLOCK_HASH, "audit_bundle_sha256": expected}, + view, + body_uri=body_uri, + canonical_bundle_path=path, + ) + self.assertFalse(Path(body_uri).exists()) + # Once the pressure passes the same call publishes normally. + self.assertEqual( + store.prepare_external_audit_body( + {"block_hash": BLOCK_HASH, "audit_bundle_sha256": expected}, + view, + body_uri=body_uri, + canonical_bundle_path=path, + ), + body_uri, + ) + + def test_persistence_streams_lazy_accounts_and_the_inline_artifact(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + bundle = synthetic_bundle(1200, recipients=900) + path, raw, view = self.write_view(root, bundle) + expected = hashlib.sha256(raw).hexdigest() + report = { + "coinbase_txid": "11" * 32, + "coinbase_manifest_sha256_hex": "22" * 32, + "audit_bundle_sha256_hex": expected, + "coinbase_tx_hex": "00", + } + # Inline lane: no body store; the artifact bytes are spliced. + inline = FakeLeasePsqlShareLedger([acquired_lease(), persist_result()]) + with bounded_json_guards("lab.prism.share_ledger", 256 * 1024): + persistence = inline.persist_accepted_block( + block_hash=BLOCK_HASH, + block_height=10, + parent_hash=PARENT_HASH, + final_bundle=view, + audit_report=report, + ) + self.assertEqual(persistence["audit_body_byte_len"], len(raw)) + sql = inline.lease_queries[-1] + self.assertEqual(sql.count(raw.decode("utf-8")), 1) + payload = payload_from_sql(sql) + self.assertEqual(payload["audit_bundle"], bundle) + self.assertEqual(payload["accounts"], bundle["payout_policy_manifest"]["accounts"]) + self.assertEqual(payload["witness_merkle_leaves_hex"], bundle["witness_merkle_leaves_hex"]) + # External lane: the body pointer replaces the inline body and the + # recipient-scaled accounts stream from the view. + external = FakeLeasePsqlShareLedger( + [acquired_lease(), {"existing_block": False, "existing_body_uri": None}, persist_result()], + audit_body_dir=root, + audit_bundle_canonicalizer=mock.Mock(side_effect=AssertionError("must not canonicalize")), + audit_share_segment_size=100, + ) + with bounded_json_guards("lab.prism.share_ledger", 256 * 1024): + persistence = external.persist_accepted_block( + block_hash=BLOCK_HASH, + block_height=10, + parent_hash=PARENT_HASH, + final_bundle=view, + audit_report=report, + canonical_bundle_path=path, + ) + payload = payload_from_sql(external.lease_queries[-1]) + self.assertIsNone(payload["audit_bundle"]) + self.assertEqual(payload["body_uri"], persistence["body_uri"]) + self.assertEqual(payload["accounts"], bundle["payout_policy_manifest"]["accounts"]) + + def test_ctv_manifest_set_persists_and_digests_through_the_view(self) -> None: + from lab.prism.audit_bundle_view import LazyRecordSequence as Lazy, streamed_sha256_json_hex + from lab.prism.share_ledger import ctv_fanout_recovery_payload, sha256_json_hex + + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + bundle = synthetic_bundle(20) + _path, _raw, view = self.write_view(root, bundle) + manifest_set = view["ctv_fanout_manifest_set"] + self.assertIsInstance(manifest_set["manifests"], Lazy) + digest = streamed_sha256_json_hex(manifest_set) + self.assertEqual(digest, sha256_json_hex(bundle["ctv_fanout_manifest_set"])) + reference = ctv_fanout_recovery_payload( + block_hash=BLOCK_HASH, + manifest_set=bundle["ctv_fanout_manifest_set"], + manifest_set_sha256=digest, + ) + ledger = FakeLeasePsqlShareLedger( + [acquired_lease(), {"backend": "postgres-psql", "fanout_set_count": 1, "fanout_artifact_count": 2}], + ) + with bounded_json_guards("lab.prism.share_ledger", 256 * 1024): + ledger.persist_ctv_fanout_manifest_set( + block_hash=BLOCK_HASH, + manifest_set=manifest_set, + manifest_set_sha256=digest, + ) + payload = payload_from_sql(ledger.lease_queries[-1]) + for key in ("writer_id", "writer_epoch", "writer_session_token"): + payload.pop(key) + self.assertEqual(payload, reference) + # The in-memory ledger keeps plain state from the same payload. + from lab.prism.share_ledger import SingleWriterShareLedger + + memory = SingleWriterShareLedger() + memory.persist_ctv_fanout_manifest_set( + block_hash=BLOCK_HASH, + manifest_set=manifest_set, + manifest_set_sha256=digest, + ) + self.assertEqual(memory._ctv_fanout_sets[BLOCK_HASH], reference) + + def test_finalization_passes_lazy_members_to_its_consumers(self) -> None: + from types import SimpleNamespace + + from lab.prism.audit_bundle_view import streamed_sha256_json_hex + from lab.prism.block_finalization import BlockFinalizationService, FinalizationAdmission, LandedCandidate + from lab.prism.share_ledger import sha256_json_hex + + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + bundle = synthetic_bundle(20) + _path, _raw, view = self.write_view(root, bundle) + recorded: dict[str, Any] = {} + + def persist(**kwargs: Any) -> dict[str, Any]: + recorded.update(kwargs) + return {"backend": "memory"} + + runtime = SimpleNamespace(ledger=SimpleNamespace(persist_ctv_fanout_manifest_set=persist)) + service = BlockFinalizationService(runtime) # type: ignore[arg-type] + service._record_block_candidate_progress = lambda *args, **kwargs: None # type: ignore[method-assign] + admission = SimpleNamespace( + candidate=SimpleNamespace(credit_share_on_accept=False), + block_hash=BLOCK_HASH, + context=None, + submission=None, + ) + landed = LandedCandidate( + final_bundle=view, + report={}, + persistence={}, + confirmation={}, + audit_publication_identity=None, # type: ignore[arg-type] + audit_verification_identity={}, + ) + result = service._persist_ctv_and_credit(admission, landed) # type: ignore[arg-type] + self.assertEqual(result, {"backend": "memory"}) + self.assertIs(recorded["manifest_set"], view["ctv_fanout_manifest_set"]) + self.assertEqual(recorded["manifest_set_sha256"], sha256_json_hex(bundle["ctv_fanout_manifest_set"])) + self.assertEqual(recorded["manifest_set_sha256"], streamed_sha256_json_hex(view["ctv_fanout_manifest_set"])) + del FinalizationAdmission + + def test_payout_preview_walks_lazy_accounts(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + bundle = synthetic_bundle(20, recipients=40) + _path, _raw, view = self.write_view(root, bundle) + server = coordinator_server() + self.assertEqual( + server._accepted_block_payout_preview_from_bundle(view, prior_balances=[]), + server._accepted_block_payout_preview_from_bundle(bundle, prior_balances=[]), + ) + self.assertEqual( + len(server._accepted_block_payout_preview_from_bundle(view, prior_balances=[])), + 40, + ) + + +class CompilerInputStreamingTests(unittest.TestCase): + def test_build_input_chunks_match_json_dumps_for_lists_and_sequences(self) -> None: + records = [share_record(index) for index in range(700)] + payload = { + "found_block": found_block(), + "prior_balances": [], + "payout_policy": {"p2mr_spend_input_bytes": 1}, + "coinbase_script_sig_suffix_hex": "00", + "witness_merkle_leaves_hex": ["ab" * 32], + "shares": records, + "ctv_settlement": None, + } + expected = json.dumps(payload, separators=(",", ":")) + self.assertEqual( + "".join(iter_json_object_text_chunks(payload, array_keys=("shares",))), + expected, + ) + for shares in (records, tuple(records), GuardedShareSequence(records)): + with self.subTest(kind=type(shares).__name__): + chunks = list( + _iter_build_input_chunks( + {**payload, "shares": shares}, + array_keys=("shares", "compact_share_identities", "compact_shares"), + batch_records=64, + chunk_chars=4096, + ) + ) + self.assertEqual("".join(chunks), expected) + self.assertTrue(all(chunk.isascii() for chunk in chunks)) + self.assertGreater(len(chunks), 1) + + def test_compiler_streams_a_sequence_without_copying_it(self) -> None: + server = coordinator_server() + captured: dict[str, object] = {} + records = [share_record(index) for index in range(300)] + window = GuardedShareSequence(records) + canonical = json.dumps({"ok": True, "shares": records[:2]}, separators=(",", ":")) + with tempfile.TemporaryDirectory() as tmp, mock.patch( + "lab.prism.bundle_compiler.subprocess.Popen", + fake_audit_bundle_popen(captured, output_text=canonical), + ): + output_path = Path(tmp) / "candidate.audit.json" + bundle = server.build_audit_bundle( + shares=window, + found_block=found_block(), + prior_balances=[], + coinbase_script_sig_suffix_hex="00", + canonical_output_path=output_path, + ) + try: + self.assertIsInstance(bundle, CanonicalAuditBundleView) + self.assertEqual(captured["payload"]["shares"], records) + self.assertEqual(window.iterations, 1) + self.assertEqual(bundle, json.loads(canonical)) + self.assertIsInstance(bundle["shares"], LazyRecordSequence) + self.assertEqual(output_path.read_bytes(), canonical.encode()) + finally: + bundle.close() + + def test_compiler_splices_pre_encoded_windows_without_parsing(self) -> None: + from lab.prism.bundle_compiler import _iter_share_window_items + from lab.prism.share_json_stream import canonical_share_items_bytes + + records = [share_record(index) for index in range(1500)] + items = canonical_share_items_bytes(records) + + class MirrorLike(Sequence): + def __init__(self) -> None: + self.canonical_items = items + self.record_count = len(records) + + def __len__(self) -> int: + return self.record_count + + def __iter__(self): # type: ignore[no-untyped-def] + raise AssertionError("a pre-encoded window must not be parsed") + + def __getitem__(self, index: int | slice) -> Any: + raise AssertionError("a pre-encoded window must not be parsed") + + class PageLike: + def __init__(self, records_slice: list[dict[str, object]]) -> None: + self.canonical_json_items = canonical_share_items_bytes(records_slice) + + class PagedLike(Sequence): + def __init__(self) -> None: + self.pages = (PageLike(records[:700]), PageLike([]), PageLike(records[700:])) + + def __len__(self) -> int: + return len(records) + + def __iter__(self): # type: ignore[no-untyped-def] + raise AssertionError("a page-backed window must not be parsed") + + def __getitem__(self, index: int | slice) -> Any: + raise AssertionError("a page-backed window must not be parsed") + + for window in (MirrorLike(), PagedLike()): + with self.subTest(kind=type(window).__name__): + spliced = b"".join( + piece if isinstance(piece, bytes) else piece.encode() + for piece in _iter_share_window_items(window, batch_records=64, chunk_chars=4096) + ) + self.assertEqual(spliced, items) + self.assertEqual(json.loads(b"[" + spliced + b"]"), json.loads(b"[" + items + b"]")) + # Through the compiler: the builder receives those exact records and + # the recipient-scaled balances stream in batches. + server = coordinator_server() + captured: dict[str, object] = {} + balances = [ + {"recipient_id": f"m{index}", "order_key": f"o{index}", "p2mr_program_hex": "aa" * 32, "balance_sats": index} + for index in range(5000) + ] + with tempfile.TemporaryDirectory() as tmp, mock.patch( + "lab.prism.bundle_compiler.subprocess.Popen", + fake_audit_bundle_popen(captured, output_text='{"ok":true}'), + ), bounded_json_guards("lab.prism.bundle_compiler", 256 * 1024, guard_loads=False): + bundle = server.build_audit_bundle( + shares=MirrorLike(), + found_block=found_block(), + prior_balances=balances, + coinbase_script_sig_suffix_hex="00", + witness_merkle_leaves_hex=["ab" * 32] * 3000, + canonical_output_path=Path(tmp) / "candidate.audit.json", + ) + try: + self.assertEqual(captured["payload"]["shares"], json.loads(b"[" + items + b"]")) + self.assertEqual(captured["payload"]["prior_balances"], balances) + self.assertEqual(len(captured["payload"]["witness_merkle_leaves_hex"]), 3000) + finally: + bundle.close() + + def test_compiler_rejects_non_sequence_windows_before_writing(self) -> None: + server = coordinator_server() + with tempfile.TemporaryDirectory() as tmp: + with self.assertRaises(TypeError): + server.build_audit_bundle( + shares="not-a-window", # type: ignore[arg-type] + found_block=found_block(), + prior_balances=[], + coinbase_script_sig_suffix_hex="00", + canonical_output_path=Path(tmp) / "candidate.audit.json", + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_prism_bounded_executor.py b/tests/test_prism_bounded_executor.py new file mode 100644 index 000000000..6dc5c9ccf --- /dev/null +++ b/tests/test_prism_bounded_executor.py @@ -0,0 +1,181 @@ +#!/usr/bin/env python3 +"""Focused tests for PRISM's bounded priority executor.""" + +from __future__ import annotations + +import gc +import queue +import threading +import unittest +import weakref + +from lab.prism import prism_coordinator +from lab.prism.bounded_executor import ( + _BoundedPriorityExecutor, + _DeliveryQueueFull, +) + + +class _Payload: + pass + + +class _Task: + def __init__(self, result: _Payload, *, fail: bool) -> None: + self.result = result + self.fail = fail + + def __call__(self, argument: _Payload, *, keyword: _Payload) -> _Payload: + if self.fail: + raise RuntimeError("injected task failure") + return self.result + + +class BoundedExecutorRetentionTests(unittest.TestCase): + def check_idle_worker_releases_task(self, outcome: str) -> None: + # Queue the task before starting a worker, so cancellation and the + # return to an idle get() are deterministic and require no sleeps. + waiting = threading.Semaphore(0) + + class ObservedQueue(queue.PriorityQueue): + def get(self, *args, **kwargs): + waiting.release() + return super().get(*args, **kwargs) + + gc.collect() + was_enabled = gc.isenabled() + gc.disable() + executor = _BoundedPriorityExecutor(max_workers=0, max_queue_size=1) + executor._queue = ObservedQueue(maxsize=1) + try: + argument, keyword, result = _Payload(), _Payload(), _Payload() + task = _Task(result, fail=outcome == "failed") + future = executor.submit(task, argument, keyword=keyword) + references = { + "argument": weakref.ref(argument), + "keyword": weakref.ref(keyword), + "result": weakref.ref(result), + "task": weakref.ref(task), + "future": weakref.ref(future), + } + if outcome == "cancelled": + self.assertTrue(future.cancel()) + del argument, keyword, result, task, future + thread = threading.Thread(target=executor._worker, daemon=True) + executor._threads.append(thread) + thread.start() + self.assertTrue(waiting.acquire(timeout=5)) + self.assertTrue(waiting.acquire(timeout=5)) + self.assertEqual(executor.stats(), (0, 0)) + self.assertEqual(executor._queue.unfinished_tasks, 0) + self.assertTrue(thread.is_alive()) + self.assertEqual( + {name: ref() is not None for name, ref in references.items()}, + dict.fromkeys(references, False), + ) + finally: + executor.shutdown(wait=True, cancel_futures=True) + if was_enabled: + gc.enable() + gc.collect() + + def test_idle_worker_releases_successful_task(self) -> None: + self.check_idle_worker_releases_task("succeeded") + + def test_idle_worker_releases_failed_task(self) -> None: + self.check_idle_worker_releases_task("failed") + + def test_idle_worker_releases_cancelled_task(self) -> None: + self.check_idle_worker_releases_task("cancelled") + + +class BoundedPriorityExecutorTests(unittest.TestCase): + def test_compatibility_reexports_reference_executor_owner(self) -> None: + self.assertIs( + prism_coordinator._BoundedPriorityExecutor, + _BoundedPriorityExecutor, + ) + self.assertIs(prism_coordinator._DeliveryQueueFull, _DeliveryQueueFull) + + def test_queue_bound_excludes_active_worker_and_reports_active_count(self) -> None: + executor = _BoundedPriorityExecutor(max_workers=1, max_queue_size=1) + blocker_started = threading.Event() + release = threading.Event() + + def blocker() -> None: + blocker_started.set() + release.wait(5) + + try: + executor.submit(blocker) + self.assertTrue(blocker_started.wait(5)) + queued = executor.submit(lambda: None) + self.assertEqual(executor.stats(), (1, 1)) + with self.assertRaisesRegex(_DeliveryQueueFull, "queue is full"): + executor.submit(lambda: None) + release.set() + queued.result(5) + finally: + release.set() + executor.shutdown(wait=True, cancel_futures=True) + + def test_shutdown_cancels_queued_future_and_joins_named_workers(self) -> None: + executor = _BoundedPriorityExecutor(max_workers=2, max_queue_size=2) + blocker_started = [threading.Event(), threading.Event()] + release = threading.Event() + + def blocker(index: int) -> None: + blocker_started[index].set() + release.wait(5) + + executor.submit(blocker, 0) + executor.submit(blocker, 1) + self.assertTrue(all(started.wait(5) for started in blocker_started)) + queued = executor.submit(lambda: None) + + executor.shutdown(wait=False, cancel_futures=True) + self.assertTrue(queued.cancelled()) + self.assertEqual( + [thread.name for thread in executor._threads], + ["prism-job-delivery-1", "prism-job-delivery-2"], + ) + release.set() + executor.shutdown(wait=True) + + self.assertTrue(all(not thread.is_alive() for thread in executor._threads)) + + def test_cancel_discards_queued_entry_and_frees_bounded_admission(self) -> None: + executor = _BoundedPriorityExecutor(max_workers=1, max_queue_size=1) + blocker_started = threading.Event() + release = threading.Event() + + def blocker() -> None: + blocker_started.set() + release.wait(5) + + try: + executor.submit(blocker) + self.assertTrue(blocker_started.wait(5)) + queued = executor.submit(lambda: "victim") + with self.assertRaises(_DeliveryQueueFull): + executor.submit(lambda: "rejected") + + self.assertTrue(executor.cancel(queued)) + self.assertTrue(queued.cancelled()) + # The cancelled entry is removed from the priority queue itself, + # so bounded admission is immediately available again. + replacement = executor.submit(lambda: "replacement") + self.assertEqual(executor.stats()[0], 1) + + release.set() + self.assertEqual(replacement.result(5), "replacement") + # A worker that already dequeued an entry owns the task_done + # path; cancelling a running/finished future removes nothing. + self.assertFalse(executor.cancel(replacement)) + finally: + release.set() + executor.shutdown(wait=True, cancel_futures=True) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_prism_builder_pipe_transport.py b/tests/test_prism_builder_pipe_transport.py new file mode 100644 index 000000000..b2c535c2d --- /dev/null +++ b/tests/test_prism_builder_pipe_transport.py @@ -0,0 +1,1771 @@ +"""Builder pipe transport: readiness waits in place of fixed polling (#236). + +The --serve daemon transport and the one-shot writer drive non-blocking +pipes so cancellation, supersession and the absolute deadline are checked +between syscalls. They used to sleep a fixed 20 ms after every EAGAIN; a +readiness wait bounded by the deadline and the same check cadence now takes +its place. These tests pin the transport contract that must survive that +swap -- exact bytes and framing, exactly-once failure accounting, the +timeout/cancellation/supersession classifications, spool poisoning without +prefix replay, and selector cleanup on every path -- first with scripted +syscalls (deterministic EAGAIN/EINTR/spurious-readiness/short-write/EOF +sequences) and then over real pipes with a peer thread or subprocess. + +No timing threshold here is tight enough to be a performance claim; the +throughput measurement lives in ``tests/perf/window_builder_pipe_latency.py``. +""" + +from __future__ import annotations + +import hashlib +import json +import os +import selectors +import sys +import tempfile +import threading +import time +import unittest +from pathlib import Path +from types import SimpleNamespace +from typing import Any, Callable +from unittest.mock import patch + +from lab.prism import bundle_compiler as compiler_module +from lab.prism.bundle_compiler import ( + PRISM_BUILDER_PIPE_READ_CHUNK_BYTES, + PRISM_BUILDER_PIPE_WAIT_SLICE_SECONDS, + PRISM_TIP_REFRESH_ADMISSION_POLL_SECONDS, + BundleCompiler, + _PipeReadinessWaiter, + _ServeBuilderClient, + _ServeBuilderUnavailable, +) +from tests.prism_coordinator_test_support import ( + FAKE_SERVE_BUILDER_COMMAND, + PayoutLedgerArtifact, + coordinator, + spool_share, +) + + +FAR_DEADLINE_SECONDS = 30.0 + + +class _Superseded(RuntimeError): + """Stands in for JobBuildSuperseded.""" + + +class _Cancelled(RuntimeError): + """Stands in for the coordinator's cancellation error.""" + + +class _Control: + """Minimal BundleBuildControlPort.""" + + def __init__(self) -> None: + self.cancel_event = threading.Event() + self.process = None + + +class _Cancellation: + """Minimal CancellationPort backed by an event.""" + + def __init__(self) -> None: + self.event = threading.Event() + self.phases: list[str] = [] + + def is_set(self) -> bool: + return self.event.is_set() + + def raise_if_cancelled(self, phase: str) -> None: + self.phases.append(phase) + if self.event.is_set(): + raise _Cancelled(phase) + + +class _Runtime: + """Only the scheduler counters the transport helpers touch.""" + + def __init__(self) -> None: + self._job_build_scheduler_lock = threading.Lock() + self.job_build_worker_counts = { + "starts": 0, + "restarts": 0, + "crashes": 0, + "terminations": 0, + } + self._job_build_worker_restart_pending = False + + +class _Serialization: + """Only the spool-poisoning surface the splice helper touches.""" + + def __init__(self) -> None: + self.spool_failures = 0 + + def mark_spool_failed(self) -> None: + self.spool_failures += 1 + + +class _PipeEnd: + def __init__(self, file_descriptor: int) -> None: + self._file_descriptor = file_descriptor + + def fileno(self) -> int: + return self._file_descriptor + + +class _FakeClock: + """A monotonic clock the scripted tests advance by hand.""" + + def __init__(self, start: float = 1_000.0) -> None: + self.now = start + + def monotonic(self) -> float: + return self.now + + def advance(self, seconds: float) -> None: + self.now += seconds + + def sleep(self, seconds: float) -> None: + self.advance(seconds) + + +class _ScriptedWaiter: + """Drop-in for _PipeReadinessWaiter that never blocks. + + Records every wait with the deadline slack it was given (against the + scripted fake clock) and answers readiness from a class-level script + (True when the script is empty). Each wait may advance the fake clock, + so a deadline is exhausted by construction rather than by real time + passing: the transport loops can be walked through EAGAIN, spurious + readiness and timeouts deterministically. + """ + + instances: list[_ScriptedWaiter] = [] + readiness: list[bool] = [] + clock: _FakeClock = _FakeClock() + advance_per_wait: float = 0.0 + + def __init__(self, file_descriptor: int, events: int) -> None: + self.file_descriptor = file_descriptor + self.events = events + self.waits = 0 + self.slack: list[float] = [] + self.closed = 0 + type(self).instances.append(self) + + def __enter__(self) -> _ScriptedWaiter: + return self + + def __exit__(self, *exc_info: object) -> None: + self.close() + + def wait(self, deadline: float) -> bool: + self.waits += 1 + cls = type(self) + self.slack.append(deadline - cls.clock.monotonic()) + cls.clock.advance(cls.advance_per_wait) + script = cls.readiness + if script: + return script.pop(0) + return True + + def close(self) -> None: + self.closed += 1 + + +class _RecordingWaiter(_PipeReadinessWaiter): + """The real waiter, with its instances and waits observable.""" + + instances: list[_RecordingWaiter] = [] + + def __init__(self, file_descriptor: int, events: int) -> None: + super().__init__(file_descriptor, events) + self.closed = 0 + type(self).instances.append(self) + + def close(self) -> None: + self.closed += 1 + super().close() + + +class _GatedWaiter(_RecordingWaiter): + """The real waiter, releasing a held-back peer when the transport blocks. + + Tests that need the transport to block by construction hold their peer + back until the transport has actually entered a wait: at that moment + the pipe is empty (reads) or full (writes) whatever the scheduler did, + so a wait-count assertion is not a timing assumption. The release is an + in-process event for thread peers and a marker file for child processes. + """ + + instances: list[_GatedWaiter] = [] + released = threading.Event() + marker: str | None = None + + @classmethod + def arm(cls, marker: str | None = None) -> None: + cls.instances = [] + cls.released = threading.Event() + cls.marker = marker + + def wait(self, deadline: float) -> bool: + gate = type(self) + if not gate.released.is_set(): + if gate.marker is not None: + Path(gate.marker).touch() + gate.released.set() + return super().wait(deadline) + + +class _PollFailingSelector: + """A DefaultSelector stand-in that registers, then cannot poll. + + Models SelectSelector over a Windows pipe handle (registration succeeds, + ``select`` raises), with a hook to spend fake time inside the failed + poll. Lifecycle counts are class-level so a test can assert the + selector was built and closed exactly once. + """ + + built = 0 + closes = 0 + selects: list[float | None] = [] + error: BaseException = OSError(10038, "not a socket") + on_select: Callable[[], None] | None = None + + @classmethod + def reset( + cls, + *, + error: BaseException | None = None, + on_select: Callable[[], None] | None = None, + ) -> None: + cls.built = 0 + cls.closes = 0 + cls.selects = [] + cls.error = OSError(10038, "not a socket") if error is None else error + cls.on_select = on_select + + def __init__(self) -> None: + type(self).built += 1 + self.registered: list[tuple[int, int]] = [] + + def register(self, file_descriptor: int, events: int, data: Any = None) -> None: + self.registered.append((file_descriptor, events)) + + def select(self, timeout: float | None = None) -> list[Any]: + cls = type(self) + cls.selects.append(timeout) + if cls.on_select is not None: + cls.on_select() + raise cls.error + + def close(self) -> None: + type(self).closes += 1 + + +class _Do: + """A scripted syscall step: run a side effect, then answer or raise.""" + + def __init__(self, effect: Callable[[], None], answer: Any) -> None: + self.effect = effect + self.answer = answer + + +class _SyscallScript: + """Scripted os.read / os.write / os.splice answers for one descriptor. + + Answers are consumed in order: bytes (read) or an int (write/splice) + are returned, an exception instance is raised, and a _Do runs its + effect first. Every other descriptor reaches the real syscall. + """ + + def __init__(self, file_descriptor: int) -> None: + self.file_descriptor = file_descriptor + self.reads: list[Any] = [] + self.writes: list[Any] = [] + self.splices: list[Any] = [] + self.read_sizes: list[int] = [] + self.write_log: list[bytes] = [] + self.splice_log: list[tuple[int, int]] = [] + + def _answer(self, script: list[Any], name: str) -> Any: + if not script: + raise AssertionError(f"unscripted os.{name} call") + action = script.pop(0) + if isinstance(action, _Do): + action.effect() + action = action.answer + if isinstance(action, BaseException): + raise action + return action + + def install(self, case: unittest.TestCase) -> None: + real_read = os.read + real_write = os.write + real_splice = getattr(os, "splice", None) + + def read(file_descriptor: int, size: int) -> bytes: + if file_descriptor != self.file_descriptor: + return real_read(file_descriptor, size) + self.read_sizes.append(size) + chunk = self._answer(self.reads, "read") + if len(chunk) > size: + raise AssertionError("scripted read exceeds the requested size") + return chunk + + def write(file_descriptor: int, data: Any) -> int: + if file_descriptor != self.file_descriptor: + return real_write(file_descriptor, data) + written = self._answer(self.writes, "write") + self.write_log.append(bytes(data[:written])) + return written + + def splice(src: int, dst: int, count: int, **kwargs: Any) -> int: + if dst != self.file_descriptor: + assert real_splice is not None + return real_splice(src, dst, count, **kwargs) + self.splice_log.append((int(kwargs.get("offset_src", -1)), count)) + return self._answer(self.splices, "splice") + + for name, replacement in (("read", read), ("write", write)): + patcher = patch.object(os, name, replacement) + patcher.start() + case.addCleanup(patcher.stop) + splice_patcher = patch.object(os, "splice", splice, create=True) + splice_patcher.start() + case.addCleanup(splice_patcher.stop) + + +class _TransportCase(unittest.TestCase): + """Shared fixtures: a compiler over a counter-only runtime and pipes.""" + + def setUp(self) -> None: + self.runtime = _Runtime() + self.compiler = BundleCompiler( + self.runtime, # type: ignore[arg-type] + superseded_error=_Superseded, + cancellation_error_types=(_Cancelled,), + build_control_type=_Control, + ) + self._open_fds: list[int] = [] + + def tearDown(self) -> None: + for file_descriptor in self._open_fds: + try: + os.close(file_descriptor) + except OSError: + pass + + def _close_fd(self, file_descriptor: int) -> None: + try: + os.close(file_descriptor) + except OSError: + pass + if file_descriptor in self._open_fds: + self._open_fds.remove(file_descriptor) + + def _client(self) -> tuple[_ServeBuilderClient, int, int]: + """A client over two real pipes. + + Returns the client plus the peer ends: the descriptor a fake daemon + writes its stdout into, and the one it reads its stdin from. + """ + stdout_read, stdout_write = os.pipe() + stdin_read, stdin_write = os.pipe() + self._open_fds += [stdout_read, stdout_write, stdin_read, stdin_write] + os.set_blocking(stdout_read, False) + os.set_blocking(stdin_write, False) + client = _ServeBuilderClient( + process=SimpleNamespace( # type: ignore[arg-type] + stdin=_PipeEnd(stdin_write), + stdout=_PipeEnd(stdout_read), + poll=lambda: None, + ) + ) + return client, stdout_write, stdin_read + + def _scripted(self) -> None: + """Scripted waiter plus a hand-advanced clock for the module's deadlines.""" + self.clock = _FakeClock() + _ScriptedWaiter.instances = [] + _ScriptedWaiter.readiness = [] + _ScriptedWaiter.clock = self.clock + _ScriptedWaiter.advance_per_wait = 0.0 + for target, replacement in ( + ("_PipeReadinessWaiter", _ScriptedWaiter), + ( + "time", + SimpleNamespace( + monotonic=self.clock.monotonic, + sleep=self.clock.sleep, + ), + ), + ): + patcher = patch.object(compiler_module, target, replacement) + patcher.start() + self.addCleanup(patcher.stop) + + def _gated(self, marker: str | None = None) -> None: + _GatedWaiter.arm(marker) + patcher = patch.object( + compiler_module, + "_PipeReadinessWaiter", + _GatedWaiter, + ) + patcher.start() + self.addCleanup(patcher.stop) + + def _recording(self) -> None: + _RecordingWaiter.instances = [] + patcher = patch.object( + compiler_module, + "_PipeReadinessWaiter", + _RecordingWaiter, + ) + patcher.start() + self.addCleanup(patcher.stop) + + def assertNoFailureAccounted(self) -> None: + self.assertEqual( + self.runtime.job_build_worker_counts, + {"starts": 0, "restarts": 0, "crashes": 0, "terminations": 0}, + ) + self.assertFalse(self.runtime._job_build_worker_restart_pending) + + def assertOneCrashAccounted(self) -> None: + self.assertEqual(self.runtime.job_build_worker_counts["crashes"], 1) + self.assertEqual(self.runtime.job_build_worker_counts["terminations"], 0) + self.assertTrue(self.runtime._job_build_worker_restart_pending) + + +class ReadinessWaiterTests(_TransportCase): + """The helper itself: laziness, bounds, degradation and cleanup.""" + + def test_wait_slice_matches_the_retired_polling_cadence(self) -> None: + self.assertEqual( + PRISM_BUILDER_PIPE_WAIT_SLICE_SECONDS, + min(0.02, PRISM_TIP_REFRESH_ADMISSION_POLL_SECONDS), + ) + + def test_registration_is_lazy_and_close_is_idempotent(self) -> None: + read_end, write_end = os.pipe() + self._open_fds += [read_end, write_end] + with patch.object( + selectors, + "DefaultSelector", + side_effect=AssertionError("selector built without a wait"), + ): + waiter = _PipeReadinessWaiter(read_end, selectors.EVENT_READ) + self.assertEqual(waiter.waits, 0) + waiter.close() + waiter.close() + with waiter: + pass + + def test_ready_descriptor_returns_true_without_sleeping(self) -> None: + read_end, write_end = os.pipe() + self._open_fds += [read_end, write_end] + os.write(write_end, b"x") + with _PipeReadinessWaiter(read_end, selectors.EVENT_READ) as waiter: + started = time.monotonic() + ready = waiter.wait(time.monotonic() + FAR_DEADLINE_SECONDS) + elapsed = time.monotonic() - started + self.assertTrue(ready) + self.assertEqual(waiter.waits, 1) + self.assertLess(elapsed, 1.0) + + def test_idle_descriptor_wakes_on_the_cadence_not_the_deadline(self) -> None: + read_end, write_end = os.pipe() + self._open_fds += [read_end, write_end] + with _PipeReadinessWaiter(read_end, selectors.EVENT_READ) as waiter: + started = time.monotonic() + ready = waiter.wait(time.monotonic() + FAR_DEADLINE_SECONDS) + elapsed = time.monotonic() - started + self.assertFalse(ready) + # Bounded by the cancellation-check slice (20 ms), not by the far + # deadline; the upper bound is generous for a loaded runner. + self.assertGreaterEqual(elapsed, PRISM_BUILDER_PIPE_WAIT_SLICE_SECONDS / 2) + self.assertLess(elapsed, 1.0) + + def test_wait_is_bounded_by_the_remaining_deadline(self) -> None: + read_end, write_end = os.pipe() + self._open_fds += [read_end, write_end] + with _PipeReadinessWaiter(read_end, selectors.EVENT_READ) as waiter: + started = time.monotonic() + ready = waiter.wait(time.monotonic() + 0.005) + elapsed = time.monotonic() - started + self.assertFalse(ready) + self.assertLess(elapsed, 1.0) + + def test_exhausted_deadline_returns_false_without_a_selector(self) -> None: + read_end, write_end = os.pipe() + self._open_fds += [read_end, write_end] + with patch.object( + selectors, + "DefaultSelector", + side_effect=AssertionError("selector built for a dead deadline"), + ), patch.object(compiler_module.time, "sleep") as sleep: + with _PipeReadinessWaiter(read_end, selectors.EVENT_READ) as waiter: + self.assertFalse(waiter.wait(time.monotonic() - 1.0)) + self.assertEqual(waiter.waits, 1) + sleep.assert_not_called() + + def test_unwatchable_descriptor_degrades_to_one_bounded_sleep(self) -> None: + class _Refusing: + built = 0 + + def __init__(self) -> None: + type(self).built += 1 + + def register(self, *args: object) -> None: + raise PermissionError("regular files cannot be polled") + + def close(self) -> None: + pass + + read_end, write_end = os.pipe() + self._open_fds += [read_end, write_end] + with patch.object(selectors, "DefaultSelector", _Refusing), patch.object( + compiler_module.time, + "sleep", + ) as sleep: + with _PipeReadinessWaiter(read_end, selectors.EVENT_READ) as waiter: + self.assertFalse(waiter.wait(time.monotonic() + FAR_DEADLINE_SECONDS)) + self.assertFalse(waiter.wait(time.monotonic() + FAR_DEADLINE_SECONDS)) + self.assertEqual(_Refusing.built, 1) + self.assertEqual(sleep.call_count, 2) + for call in sleep.call_args_list: + self.assertLessEqual(call.args[0], PRISM_BUILDER_PIPE_WAIT_SLICE_SECONDS) + self.assertGreater(call.args[0], 0.0) + + def test_close_releases_the_registration(self) -> None: + closed: list[object] = [] + real_selector = selectors.DefaultSelector + + class _Observed(real_selector): # type: ignore[misc,valid-type] + def close(self) -> None: + closed.append(self) + super().close() + + read_end, write_end = os.pipe() + self._open_fds += [read_end, write_end] + with patch.object(selectors, "DefaultSelector", _Observed): + waiter = _PipeReadinessWaiter(read_end, selectors.EVENT_READ) + waiter.wait(time.monotonic() + 0.001) + self.assertEqual(closed, []) + waiter.close() + self.assertEqual(len(closed), 1) + self.assertIsNone(waiter._selector) + + def _fake_clock_module(self) -> tuple[_FakeClock, list[float]]: + """Drive the waiter's clock and record its sleeps.""" + clock = _FakeClock() + sleeps: list[float] = [] + + def sleep(seconds: float) -> None: + sleeps.append(seconds) + clock.advance(seconds) + + patcher = patch.object( + compiler_module, + "time", + SimpleNamespace(monotonic=clock.monotonic, sleep=sleep), + ) + patcher.start() + self.addCleanup(patcher.stop) + return clock, sleeps + + def test_poll_failure_detaches_once_and_falls_back_to_bounded_sleep(self) -> None: + read_end, write_end = os.pipe() + self._open_fds += [read_end, write_end] + clock, sleeps = self._fake_clock_module() + _PollFailingSelector.reset() + with patch.object(selectors, "DefaultSelector", _PollFailingSelector): + waiter = _PipeReadinessWaiter(read_end, selectors.EVENT_READ) + first = waiter.wait(clock.now + FAR_DEADLINE_SECONDS) + second = waiter.wait(clock.now + FAR_DEADLINE_SECONDS) + waiter.close() + waiter.close() + + self.assertEqual((first, second), (False, False)) + self.assertEqual(waiter.waits, 2) + # Built and polled once; released exactly once at the failed poll, + # never rebuilt for the next wait, and not closed again on exit. + self.assertEqual(_PollFailingSelector.built, 1) + self.assertEqual(len(_PollFailingSelector.selects), 1) + self.assertEqual(_PollFailingSelector.closes, 1) + self.assertIsNone(waiter._selector) + self.assertTrue(waiter._unwatchable) + # The failed poll spent no clock, so the first wait sleeps its whole + # slice; the second, unwatchable, sleeps one full slice too. + self.assertEqual( + [round(seconds, 6) for seconds in sleeps], + [PRISM_BUILDER_PIPE_WAIT_SLICE_SECONDS] * 2, + ) + + def test_poll_failure_sleeps_only_the_remainder_of_the_slice(self) -> None: + read_end, write_end = os.pipe() + self._open_fds += [read_end, write_end] + clock, sleeps = self._fake_clock_module() + spent = 0.005 + _PollFailingSelector.reset(on_select=lambda: clock.advance(spent)) + with patch.object(selectors, "DefaultSelector", _PollFailingSelector): + with _PipeReadinessWaiter(read_end, selectors.EVENT_READ) as waiter: + ready = waiter.wait(clock.now + FAR_DEADLINE_SECONDS) + after_first = clock.now + again = waiter.wait(clock.now + FAR_DEADLINE_SECONDS) + + self.assertEqual((ready, again), (False, False)) + # The failed poll consumed 5 ms of the 20 ms slice: the fallback + # sleeps the 15 ms left, so the wait as a whole still spans exactly + # one slice; the next (unwatchable) wait sleeps one full slice. + self.assertEqual( + [round(seconds, 6) for seconds in sleeps], + [ + round(PRISM_BUILDER_PIPE_WAIT_SLICE_SECONDS - spent, 6), + PRISM_BUILDER_PIPE_WAIT_SLICE_SECONDS, + ], + ) + self.assertEqual(round(after_first - 1_000.0, 6), PRISM_BUILDER_PIPE_WAIT_SLICE_SECONDS) + self.assertEqual(_PollFailingSelector.built, 1) + self.assertEqual(_PollFailingSelector.closes, 1) + + def test_poll_failure_after_spending_the_slice_sleeps_no_further(self) -> None: + read_end, write_end = os.pipe() + self._open_fds += [read_end, write_end] + clock, sleeps = self._fake_clock_module() + _PollFailingSelector.reset( + on_select=lambda: clock.advance(PRISM_BUILDER_PIPE_WAIT_SLICE_SECONDS * 3) + ) + with patch.object(selectors, "DefaultSelector", _PollFailingSelector): + with _PipeReadinessWaiter(read_end, selectors.EVENT_READ) as waiter: + ready = waiter.wait(clock.now + FAR_DEADLINE_SECONDS) + + self.assertFalse(ready) + self.assertEqual(sleeps, []) + self.assertEqual(_PollFailingSelector.closes, 1) + self.assertTrue(waiter._unwatchable) + + def test_poll_failure_never_overruns_the_absolute_deadline(self) -> None: + read_end, write_end = os.pipe() + self._open_fds += [read_end, write_end] + clock, sleeps = self._fake_clock_module() + _PollFailingSelector.reset(on_select=lambda: clock.advance(0.004)) + with patch.object(selectors, "DefaultSelector", _PollFailingSelector): + with _PipeReadinessWaiter(read_end, selectors.EVENT_READ) as waiter: + deadline = clock.now + 0.010 + ready = waiter.wait(deadline) + overshoot = clock.now - deadline + + self.assertFalse(ready) + # 10 ms of budget, 4 ms spent by the failed poll: the fallback sleeps + # the 6 ms to the deadline, not the 16 ms left of the slice. + self.assertEqual([round(seconds, 6) for seconds in sleeps], [0.006]) + self.assertLessEqual(round(overshoot, 9), 0.0) + + def test_poll_failure_with_a_spent_deadline_returns_without_sleeping(self) -> None: + read_end, write_end = os.pipe() + self._open_fds += [read_end, write_end] + clock, sleeps = self._fake_clock_module() + _PollFailingSelector.reset(on_select=lambda: clock.advance(0.050)) + with patch.object(selectors, "DefaultSelector", _PollFailingSelector): + with _PipeReadinessWaiter(read_end, selectors.EVENT_READ) as waiter: + ready = waiter.wait(clock.now + 0.010) + # Subsequent waits against a dead deadline stay free too. + later = waiter.wait(clock.now - 1.0) + + self.assertEqual((ready, later), (False, False)) + self.assertEqual(sleeps, []) + self.assertEqual(_PollFailingSelector.built, 1) + self.assertEqual(_PollFailingSelector.closes, 1) + + def test_interrupted_poll_is_a_plain_wakeup_and_keeps_the_selector(self) -> None: + read_end, write_end = os.pipe() + self._open_fds += [read_end, write_end] + _PollFailingSelector.reset(error=InterruptedError()) + with patch.object(selectors, "DefaultSelector", _PollFailingSelector), patch.object( + compiler_module.time, + "sleep", + ) as sleep: + waiter = _PipeReadinessWaiter(read_end, selectors.EVENT_READ) + first = waiter.wait(time.monotonic() + FAR_DEADLINE_SECONDS) + second = waiter.wait(time.monotonic() + FAR_DEADLINE_SECONDS) + self.assertEqual(_PollFailingSelector.closes, 0) + waiter.close() + + self.assertEqual((first, second), (False, False)) + # EINTR is not a fault: no sleep, the same selector polled again, + # released only by close(). + sleep.assert_not_called() + self.assertEqual(_PollFailingSelector.built, 1) + self.assertEqual(len(_PollFailingSelector.selects), 2) + self.assertEqual(_PollFailingSelector.closes, 1) + self.assertFalse(waiter._unwatchable) + + +class UnpollableDescriptorTransportTests(_TransportCase): + """A transport helper over a descriptor the selector cannot poll.""" + + def setUp(self) -> None: + super().setUp() + self._recording() + self.client, _stdout_write, _stdin_read = self._client() + self.script = _SyscallScript(self.client.process.stdout.fileno()) + self.script.install(self) + self.clock = _FakeClock() + self.sleeps: list[float] = [] + + def sleep(seconds: float) -> None: + self.sleeps.append(seconds) + self.clock.advance(seconds) + + for target, replacement in ( + ("time", SimpleNamespace(monotonic=self.clock.monotonic, sleep=sleep)), + ): + patcher = patch.object(compiler_module, target, replacement) + patcher.start() + self.addCleanup(patcher.stop) + selector_patch = patch.object(selectors, "DefaultSelector", _PollFailingSelector) + selector_patch.start() + self.addCleanup(selector_patch.stop) + + def test_read_line_completes_through_the_bounded_sleep_fallback(self) -> None: + _PollFailingSelector.reset() + self.script.reads = [BlockingIOError(), BlockingIOError(), b"ok\n"] + + line = self.compiler._serve_builder_read_line( + self.client, + self.clock.now + FAR_DEADLINE_SECONDS, + None, + None, + ) + + self.assertEqual(line, b"ok") + self.assertEqual(self.script.reads, []) + # First EAGAIN: the poll fails and the wait finishes on the sleep; + # second EAGAIN: unwatchable, straight to the sleep. One selector, + # released once at the failed poll, nothing left for close(). + self.assertEqual( + [round(seconds, 6) for seconds in self.sleeps], + [PRISM_BUILDER_PIPE_WAIT_SLICE_SECONDS] * 2, + ) + self.assertEqual(_PollFailingSelector.built, 1) + self.assertEqual(len(_PollFailingSelector.selects), 1) + self.assertEqual(_PollFailingSelector.closes, 1) + (waiter,) = _RecordingWaiter.instances + self.assertEqual(waiter.waits, 2) + self.assertEqual(waiter.closed, 1) + self.assertIsNone(waiter._selector) + self.assertNoFailureAccounted() + + def test_failed_poll_that_spends_the_deadline_times_out_without_sleeping(self) -> None: + _PollFailingSelector.reset(on_select=lambda: self.clock.advance(0.050)) + self.script.reads = [BlockingIOError(), b"never\n"] + + with self.assertRaisesRegex(_ServeBuilderUnavailable, "timed out"): + self.compiler._serve_builder_read_line( + self.client, + self.clock.now + 0.010, + None, + None, + ) + + self.assertEqual(self.sleeps, []) + self.assertEqual(self.script.reads, [b"never\n"]) + self.assertEqual(_PollFailingSelector.closes, 1) + (waiter,) = _RecordingWaiter.instances + self.assertEqual(waiter.waits, 1) + self.assertEqual(waiter.closed, 1) + self.assertNoFailureAccounted() + + +class ScriptedReadLineTests(_TransportCase): + """Envelope line reads walked through scripted syscalls.""" + + def setUp(self) -> None: + super().setUp() + self._scripted() + self.client, _stdout_write, _stdin_read = self._client() + self.script = _SyscallScript(self.client.process.stdout.fileno()) + self.script.install(self) + self.deadline = self.clock.now + FAR_DEADLINE_SECONDS + + def _read_line(self, cancellation=None, control=None) -> bytes: + return self.compiler._serve_builder_read_line( + self.client, + self.deadline, + cancellation, + control, + ) + + def test_eagain_waits_once_then_returns_line_and_keeps_tail(self) -> None: + self.script.reads = [BlockingIOError(), b'{"ok":true}\nTAIL'] + + line = self._read_line() + + self.assertEqual(line, b'{"ok":true}') + self.assertEqual(bytes(self.client.stdout_buffer), b"TAIL") + self.assertEqual(self.script.reads, []) + (waiter,) = _ScriptedWaiter.instances + self.assertEqual(waiter.events, selectors.EVENT_READ) + self.assertEqual(waiter.file_descriptor, self.script.file_descriptor) + self.assertEqual(waiter.waits, 1) + self.assertEqual(waiter.closed, 1) + self.assertEqual(waiter.slack, [FAR_DEADLINE_SECONDS]) + self.assertEqual(self.script.read_sizes, [PRISM_BUILDER_PIPE_READ_CHUNK_BYTES] * 2) + self.assertNoFailureAccounted() + + def test_line_already_buffered_takes_no_syscall_and_no_wait(self) -> None: + self.client.stdout_buffer += b"first\nsecond" + + self.assertEqual(self._read_line(), b"first") + + self.assertEqual(bytes(self.client.stdout_buffer), b"second") + self.assertEqual(self.script.read_sizes, []) + (waiter,) = _ScriptedWaiter.instances + self.assertEqual(waiter.waits, 0) + self.assertEqual(waiter.closed, 1) + + def test_line_assembled_across_chunks(self) -> None: + self.script.reads = [b"ab", BlockingIOError(), b"cd", b"ef\n"] + + self.assertEqual(self._read_line(), b"abcdef") + + self.assertEqual(bytes(self.client.stdout_buffer), b"") + self.assertEqual(_ScriptedWaiter.instances[0].waits, 1) + + def test_eintr_retries_immediately_without_waiting(self) -> None: + self.script.reads = [InterruptedError(), InterruptedError(), b"line\n"] + + self.assertEqual(self._read_line(), b"line") + + (waiter,) = _ScriptedWaiter.instances + self.assertEqual(waiter.waits, 0) + self.assertEqual(waiter.closed, 1) + + def test_spurious_readiness_retries_and_waits_again(self) -> None: + self.script.reads = [BlockingIOError(), BlockingIOError(), b"x\n"] + _ScriptedWaiter.readiness = [True, True] + + self.assertEqual(self._read_line(), b"x") + + self.assertEqual(_ScriptedWaiter.instances[0].waits, 2) + + def test_cadence_wakeup_without_readiness_retries(self) -> None: + self.script.reads = [BlockingIOError(), BlockingIOError(), b"x\n"] + _ScriptedWaiter.readiness = [False, False] + + self.assertEqual(self._read_line(), b"x") + + self.assertEqual(_ScriptedWaiter.instances[0].waits, 2) + + def test_eof_mid_frame_is_one_crash(self) -> None: + self.script.reads = [b"partial", b""] + + with self.assertRaisesRegex(_ServeBuilderUnavailable, "exited mid-request"): + self._read_line() + + self.assertOneCrashAccounted() + self.assertEqual(_ScriptedWaiter.instances[0].closed, 1) + + def test_eof_after_supersession_is_superseded_not_a_crash(self) -> None: + control = _Control() + self.script.reads = [ + b"partial", + _Do(control.cancel_event.set, b""), + ] + + with self.assertRaisesRegex(_Superseded, "terminated after supersession"): + self._read_line(control=control) + + self.assertNoFailureAccounted() + self.assertEqual(_ScriptedWaiter.instances[0].closed, 1) + + def test_deadline_while_blocked_times_out_without_accounting(self) -> None: + self.deadline = self.clock.now + 1.0 + self.script.reads = [BlockingIOError()] * 3 + _ScriptedWaiter.readiness = [False, False] + _ScriptedWaiter.advance_per_wait = 0.6 + + with self.assertRaisesRegex(_ServeBuilderUnavailable, "timed out"): + self._read_line() + + self.assertNoFailureAccounted() + (waiter,) = _ScriptedWaiter.instances + # Two waits, each handed the true remaining slack: the second + # exhausts the deadline and the loop's own check raises before any + # further syscall. + self.assertEqual(waiter.waits, 2) + self.assertEqual([round(slack, 6) for slack in waiter.slack], [1.0, 0.4]) + self.assertEqual(len(self.script.reads), 1) + self.assertEqual(waiter.closed, 1) + + def test_cancellation_while_blocked_raises_and_cleans_up(self) -> None: + cancellation = _Cancellation() + self.script.reads = [ + _Do(cancellation.event.set, BlockingIOError()), + b"never\n", + ] + + with self.assertRaises(_Cancelled): + self._read_line(cancellation=cancellation) + + self.assertEqual(self.script.reads, [b"never\n"]) + self.assertNoFailureAccounted() + self.assertEqual(_ScriptedWaiter.instances[0].closed, 1) + + def test_supersession_while_blocked_raises_and_cleans_up(self) -> None: + control = _Control() + self.script.reads = [ + _Do(control.cancel_event.set, BlockingIOError()), + b"never\n", + ] + + with self.assertRaisesRegex(_Superseded, "canceled after supersession"): + self._read_line(control=control) + + self.assertEqual(self.script.reads, [b"never\n"]) + self.assertNoFailureAccounted() + self.assertEqual(_ScriptedWaiter.instances[0].closed, 1) + + def test_read_error_is_a_daemon_anomaly(self) -> None: + self.script.reads = [OSError(5, "io error")] + + with self.assertRaisesRegex(_ServeBuilderUnavailable, "read failed"): + self._read_line() + + self.assertNoFailureAccounted() + self.assertEqual(_ScriptedWaiter.instances[0].closed, 1) + + +class ScriptedReadExactTests(_TransportCase): + """Raw-section reads walked through scripted syscalls.""" + + def setUp(self) -> None: + super().setUp() + self._scripted() + self.client, _stdout_write, _stdin_read = self._client() + self.script = _SyscallScript(self.client.process.stdout.fileno()) + self.script.install(self) + self.deadline = self.clock.now + FAR_DEADLINE_SECONDS + + def _read_exact(self, byte_count: int, cancellation=None, control=None) -> bytes: + return self.compiler._serve_builder_read_exact( + self.client, + byte_count, + self.deadline, + cancellation, + control, + ) + + def test_drains_buffer_then_reads_exactly_across_short_reads(self) -> None: + self.client.stdout_buffer += b"ABC" + self.script.reads = [b"DE", BlockingIOError(), b"FGH", b"UNREAD"] + + self.assertEqual(self._read_exact(8), b"ABCDEFGH") + + self.assertEqual(bytes(self.client.stdout_buffer), b"") + self.assertEqual(self.script.reads, [b"UNREAD"]) + # Never asks the pipe for more than the frame still owes. + self.assertEqual(self.script.read_sizes, [5, 3, 3]) + (waiter,) = _ScriptedWaiter.instances + self.assertEqual(waiter.waits, 1) + self.assertEqual(waiter.closed, 1) + self.assertNoFailureAccounted() + + def test_satisfied_from_buffer_leaves_the_rest_for_the_next_frame(self) -> None: + self.client.stdout_buffer += b"RAW\nnext-line\n" + + self.assertEqual(self._read_exact(3), b"RAW") + self.assertEqual(self._read_exact(1), b"\n") + + self.assertEqual(bytes(self.client.stdout_buffer), b"next-line\n") + self.assertEqual(self.script.read_sizes, []) + self.assertTrue(all(w.waits == 0 for w in _ScriptedWaiter.instances)) + self.assertTrue(all(w.closed == 1 for w in _ScriptedWaiter.instances)) + + def test_large_frame_reads_in_pipe_sized_chunks(self) -> None: + payload = bytes(range(256)) * 1024 # 256 KiB + chunk = PRISM_BUILDER_PIPE_READ_CHUNK_BYTES + self.script.reads = [ + payload[:chunk], + BlockingIOError(), + payload[chunk : 2 * chunk], + payload[2 * chunk : 2 * chunk + 10], + payload[2 * chunk + 10 : 3 * chunk], + payload[3 * chunk :], + ] + + self.assertEqual(self._read_exact(len(payload)), payload) + + # Every read asks for one pipe capacity while more than that is + # owed, whatever the pipe actually delivered last time. + self.assertEqual(self.script.read_sizes, [chunk] * 6) + self.assertEqual(_ScriptedWaiter.instances[0].waits, 1) + + def test_zero_length_frame_returns_empty_without_io(self) -> None: + self.assertEqual(self._read_exact(0), b"") + self.assertEqual(self.script.read_sizes, []) + + def test_eof_mid_frame_is_one_crash(self) -> None: + self.script.reads = [b"12", b""] + + with self.assertRaisesRegex(_ServeBuilderUnavailable, "exited mid-request"): + self._read_exact(4) + + self.assertOneCrashAccounted() + self.assertEqual(_ScriptedWaiter.instances[0].closed, 1) + + def test_eintr_and_spurious_readiness(self) -> None: + self.script.reads = [ + InterruptedError(), + BlockingIOError(), + BlockingIOError(), + b"ok", + ] + _ScriptedWaiter.readiness = [True, False] + + self.assertEqual(self._read_exact(2), b"ok") + + self.assertEqual(_ScriptedWaiter.instances[0].waits, 2) + + def test_deadline_times_out_without_accounting(self) -> None: + self.deadline = self.clock.now + 1.0 + self.script.reads = [BlockingIOError()] * 3 + _ScriptedWaiter.readiness = [False, False] + _ScriptedWaiter.advance_per_wait = 0.6 + + with self.assertRaisesRegex(_ServeBuilderUnavailable, "timed out"): + self._read_exact(1) + + self.assertNoFailureAccounted() + (waiter,) = _ScriptedWaiter.instances + self.assertEqual(waiter.waits, 2) + self.assertEqual([round(slack, 6) for slack in waiter.slack], [1.0, 0.4]) + self.assertEqual(len(self.script.reads), 1) + self.assertEqual(waiter.closed, 1) + + def test_cancellation_and_supersession_while_blocked(self) -> None: + cancellation = _Cancellation() + self.script.reads = [_Do(cancellation.event.set, BlockingIOError())] + with self.assertRaises(_Cancelled): + self._read_exact(1, cancellation=cancellation) + + control = _Control() + self.script.reads = [_Do(control.cancel_event.set, BlockingIOError())] + with self.assertRaisesRegex(_Superseded, "canceled after supersession"): + self._read_exact(1, control=control) + + self.assertNoFailureAccounted() + self.assertTrue(all(w.closed == 1 for w in _ScriptedWaiter.instances)) + + +class ScriptedWriteTests(_TransportCase): + """Request writes walked through scripted syscalls.""" + + def setUp(self) -> None: + super().setUp() + self._scripted() + self.client, _stdout_write, _stdin_read = self._client() + self.script = _SyscallScript(self.client.process.stdin.fileno()) + self.script.install(self) + self.deadline = self.clock.now + FAR_DEADLINE_SECONDS + + def _write(self, data: bytes, cancellation=None, control=None) -> int: + return self.compiler._serve_builder_write( + self.client, + data, + self.deadline, + cancellation, + control, + ) + + def test_short_writes_and_eagain_deliver_every_byte_once_in_order(self) -> None: + self.script.writes = [3, BlockingIOError(), InterruptedError(), 4, 3] + + self.assertEqual(self._write(b"0123456789"), 10) + + self.assertEqual(b"".join(self.script.write_log), b"0123456789") + self.assertEqual(self.script.write_log, [b"012", b"3456", b"789"]) + (waiter,) = _ScriptedWaiter.instances + self.assertEqual(waiter.events, selectors.EVENT_WRITE) + self.assertEqual(waiter.file_descriptor, self.script.file_descriptor) + self.assertEqual(waiter.waits, 1) + self.assertEqual(waiter.closed, 1) + self.assertNoFailureAccounted() + + def test_empty_write_takes_no_syscall(self) -> None: + self.assertEqual(self._write(b""), 0) + self.assertEqual(self.script.write_log, []) + + def test_broken_pipe_is_closed_input(self) -> None: + self.script.writes = [2, BrokenPipeError()] + + with self.assertRaisesRegex(_ServeBuilderUnavailable, "input pipe closed"): + self._write(b"abcd") + + self.assertEqual(self.script.write_log, [b"ab"]) + self.assertNoFailureAccounted() + self.assertEqual(_ScriptedWaiter.instances[0].closed, 1) + + def test_broken_pipe_after_supersession_is_superseded(self) -> None: + control = _Control() + self.script.writes = [_Do(control.cancel_event.set, BrokenPipeError())] + + with self.assertRaisesRegex(_Superseded, "terminated after supersession"): + self._write(b"abcd", control=control) + + self.assertNoFailureAccounted() + + def test_zero_progress_is_closed_input(self) -> None: + self.script.writes = [0] + + with self.assertRaisesRegex(_ServeBuilderUnavailable, "input pipe closed"): + self._write(b"abcd") + + self.assertNoFailureAccounted() + + def test_write_error_is_a_daemon_anomaly(self) -> None: + self.script.writes = [OSError(5, "io error")] + + with self.assertRaisesRegex(_ServeBuilderUnavailable, "write failed"): + self._write(b"abcd") + + self.assertNoFailureAccounted() + + def test_deadline_times_out_without_accounting(self) -> None: + self.deadline = self.clock.now + 1.0 + self.script.writes = [BlockingIOError()] * 3 + _ScriptedWaiter.readiness = [False, False] + _ScriptedWaiter.advance_per_wait = 0.6 + + with self.assertRaisesRegex(_ServeBuilderUnavailable, "timed out"): + self._write(b"abcd") + + self.assertNoFailureAccounted() + (waiter,) = _ScriptedWaiter.instances + self.assertEqual(waiter.waits, 2) + self.assertEqual([round(slack, 6) for slack in waiter.slack], [1.0, 0.4]) + self.assertEqual(len(self.script.writes), 1) + self.assertEqual(self.script.write_log, []) + self.assertEqual(waiter.closed, 1) + + def test_cancellation_and_supersession_while_blocked(self) -> None: + cancellation = _Cancellation() + self.script.writes = [_Do(cancellation.event.set, BlockingIOError())] + with self.assertRaises(_Cancelled): + self._write(b"abcd", cancellation=cancellation) + + control = _Control() + self.script.writes = [_Do(control.cancel_event.set, BlockingIOError())] + with self.assertRaisesRegex(_Superseded, "canceled after supersession"): + self._write(b"abcd", control=control) + + self.assertNoFailureAccounted() + self.assertTrue(all(w.closed == 1 for w in _ScriptedWaiter.instances)) + + +class ScriptedSpliceTests(_TransportCase): + """Spool splices walked through scripted syscalls.""" + + def setUp(self) -> None: + super().setUp() + self._scripted() + self.client, _stdout_write, _stdin_read = self._client() + self.script = _SyscallScript(self.client.process.stdin.fileno()) + self.script.install(self) + self.deadline = self.clock.now + FAR_DEADLINE_SECONDS + self.serialization = _Serialization() + self.spool = tempfile.TemporaryFile() + self.addCleanup(self.spool.close) + + def _splice(self, spool_size: int, cancellation=None, control=None) -> int: + return self.compiler._serve_builder_splice_spool( + self.client, + self.serialization, # type: ignore[arg-type] + self.spool, + spool_size, + self.deadline, + cancellation, + control, + ) + + def test_eagain_waits_for_the_pipe_and_resumes_from_the_exact_offset(self) -> None: + self.script.splices = [4, BlockingIOError(), InterruptedError(), 6] + + self.assertEqual(self._splice(10), 10) + + self.assertEqual( + self.script.splice_log, + [(0, 10), (4, 6), (4, 6), (4, 6)], + ) + (waiter,) = _ScriptedWaiter.instances + self.assertEqual(waiter.events, selectors.EVENT_WRITE) + self.assertEqual(waiter.file_descriptor, self.script.file_descriptor) + self.assertEqual(waiter.waits, 1) + self.assertEqual(waiter.closed, 1) + self.assertEqual(self.serialization.spool_failures, 0) + self.assertNoFailureAccounted() + + def test_failure_after_partial_progress_poisons_spool_and_never_replays(self) -> None: + self.script.splices = [4, OSError(22, "unsupported")] + + with self.assertRaisesRegex(_ServeBuilderUnavailable, "spool transfer failed"): + self._splice(10) + + self.assertEqual(self.script.splice_log, [(0, 10), (4, 6)]) + self.assertEqual(self.script.write_log, []) + self.assertEqual(self.serialization.spool_failures, 1) + self.assertNoFailureAccounted() + self.assertEqual(_ScriptedWaiter.instances[0].closed, 1) + + def test_broken_pipe_is_closed_input_and_keeps_the_spool(self) -> None: + self.script.splices = [BrokenPipeError()] + + with self.assertRaisesRegex(_ServeBuilderUnavailable, "input pipe closed"): + self._splice(10) + + self.assertEqual(self.serialization.spool_failures, 0) + self.assertNoFailureAccounted() + + def test_errors_after_supersession_are_superseded(self) -> None: + control = _Control() + self.script.splices = [_Do(control.cancel_event.set, BrokenPipeError())] + with self.assertRaisesRegex(_Superseded, "terminated after supersession"): + self._splice(10, control=control) + + control = _Control() + self.script.splices = [_Do(control.cancel_event.set, OSError(5, "io"))] + with self.assertRaisesRegex(_Superseded, "terminated after supersession"): + self._splice(10, control=control) + + self.assertEqual(self.serialization.spool_failures, 0) + self.assertNoFailureAccounted() + + def test_zero_progress_is_closed_input(self) -> None: + self.script.splices = [0] + + with self.assertRaisesRegex(_ServeBuilderUnavailable, "input pipe closed"): + self._splice(10) + + def test_deadline_times_out_without_accounting(self) -> None: + self.deadline = self.clock.now + 1.0 + self.script.splices = [BlockingIOError()] * 3 + _ScriptedWaiter.readiness = [False, False] + _ScriptedWaiter.advance_per_wait = 0.6 + + with self.assertRaisesRegex(_ServeBuilderUnavailable, "timed out"): + self._splice(10) + + self.assertNoFailureAccounted() + self.assertEqual(self.serialization.spool_failures, 0) + (waiter,) = _ScriptedWaiter.instances + self.assertEqual(waiter.waits, 2) + self.assertEqual([round(slack, 6) for slack in waiter.slack], [1.0, 0.4]) + self.assertEqual(len(self.script.splices), 1) + self.assertEqual(waiter.closed, 1) + + def test_cancellation_while_blocked(self) -> None: + cancellation = _Cancellation() + self.script.splices = [_Do(cancellation.event.set, BlockingIOError())] + + with self.assertRaises(_Cancelled): + self._splice(10, cancellation=cancellation) + + self.assertNoFailureAccounted() + + +def _slow_reader(file_descriptor: int, sink: bytearray, *, chunk: int, pause: float) -> None: + """Drain a blocking pipe end slowly so the writer side fills the pipe.""" + while True: + try: + data = os.read(file_descriptor, chunk) + except OSError: + break + if not data: + break + sink += data + time.sleep(pause) + + +def _bursty_writer(file_descriptor: int, payload: bytes, *, burst: int, pause: float) -> None: + """Write a payload in paced bursts, then close the pipe end.""" + view = memoryview(payload) + try: + while view: + written = os.write(file_descriptor, view[:burst]) + view = view[written:] + time.sleep(pause) + finally: + os.close(file_descriptor) + + +class RealPipeTransportTests(_TransportCase): + """The transport over real pipes with a peer thread.""" + + def setUp(self) -> None: + super().setUp() + self._recording() + self.client, self.stdout_write, self.stdin_read = self._client() + self.deadline = time.monotonic() + FAR_DEADLINE_SECONDS + self.threads: list[threading.Thread] = [] + + def tearDown(self) -> None: + for thread in self.threads: + thread.join(10.0) + super().tearDown() + + def _start(self, target: Callable[[], None]) -> None: + thread = threading.Thread(target=target, daemon=True) + thread.start() + self.threads.append(thread) + + def test_read_exact_streams_a_multi_megabyte_frame_exactly(self) -> None: + self._gated() + payload = os.urandom(4 << 20) + stdout_write = self.stdout_write + self._open_fds.remove(stdout_write) + + def writer() -> None: + # Nothing is written until the reader has blocked on the empty + # pipe, so the first read meets EAGAIN by construction. + _GatedWaiter.released.wait(10.0) + _bursty_writer(stdout_write, payload, burst=256 << 10, pause=0.0) + + self._start(writer) + + received = self.compiler._serve_builder_read_exact( + self.client, + len(payload), + self.deadline, + None, + None, + ) + + self.assertEqual(len(received), len(payload)) + self.assertEqual( + hashlib.sha256(received).hexdigest(), + hashlib.sha256(payload).hexdigest(), + ) + self.assertEqual(bytes(self.client.stdout_buffer), b"") + (waiter,) = _GatedWaiter.instances + self.assertTrue(_GatedWaiter.released.is_set()) + self.assertGreaterEqual(waiter.waits, 1) + self.assertEqual(waiter.closed, 1) + self.assertNoFailureAccounted() + + def test_line_then_raw_section_then_line_keeps_framing(self) -> None: + raw = os.urandom(200_000) + envelope = json.dumps({"ok": True, "window_items_len": len(raw)}).encode() + stream = envelope + b"\n" + raw + b"\n" + b'{"ok":true,"next":1}\n' + stdout_write = self.stdout_write + self._open_fds.remove(stdout_write) + self._start(lambda: _bursty_writer(stdout_write, stream, burst=7_000, pause=0.0)) + + line = self.compiler._serve_builder_read_line( + self.client, self.deadline, None, None + ) + self.assertEqual(json.loads(line)["window_items_len"], len(raw)) + section = self.compiler._serve_builder_read_exact( + self.client, len(raw), self.deadline, None, None + ) + self.assertEqual(section, raw) + self.assertEqual( + self.compiler._serve_builder_read_exact( + self.client, 1, self.deadline, None, None + ), + b"\n", + ) + self.assertEqual( + json.loads( + self.compiler._serve_builder_read_line( + self.client, self.deadline, None, None + ) + ), + {"ok": True, "next": 1}, + ) + self.assertEqual(bytes(self.client.stdout_buffer), b"") + self.assertTrue(all(w.closed == 1 for w in _RecordingWaiter.instances)) + + def test_write_streams_to_a_slow_reader_exactly(self) -> None: + self._gated() + payload = os.urandom(2 << 20) + received = bytearray() + stdin_read = self.stdin_read + + def reader() -> None: + # Nothing is drained until the writer has blocked on the full + # pipe, so the transfer meets EAGAIN by construction. + _GatedWaiter.released.wait(10.0) + _slow_reader(stdin_read, received, chunk=32 << 10, pause=0.0) + + self._start(reader) + + written = self.compiler._serve_builder_write( + self.client, payload, self.deadline, None, None + ) + self._close_fd(self.client.process.stdin.fileno()) + self.threads[-1].join(10.0) + + self.assertEqual(written, len(payload)) + self.assertEqual(bytes(received), payload) + (waiter,) = _GatedWaiter.instances + self.assertTrue(_GatedWaiter.released.is_set()) + self.assertGreaterEqual(waiter.waits, 1) + self.assertEqual(waiter.closed, 1) + self.assertNoFailureAccounted() + + @unittest.skipUnless(hasattr(os, "splice"), "os.splice is Linux-only") + def test_splice_streams_the_spool_to_a_slow_reader_exactly(self) -> None: + payload = os.urandom(2 << 20) + spool = tempfile.TemporaryFile() + self.addCleanup(spool.close) + spool.write(payload) + spool.flush() + serialization = _Serialization() + received = bytearray() + stdin_read = self.stdin_read + self._gated() + + def reader() -> None: + _GatedWaiter.released.wait(10.0) + _slow_reader(stdin_read, received, chunk=32 << 10, pause=0.0) + + self._start(reader) + + moved = self.compiler._serve_builder_splice_spool( + self.client, + serialization, # type: ignore[arg-type] + spool, + len(payload), + self.deadline, + None, + None, + ) + self._close_fd(self.client.process.stdin.fileno()) + self.threads[-1].join(10.0) + + self.assertEqual(moved, len(payload)) + self.assertEqual(bytes(received), payload) + self.assertEqual(serialization.spool_failures, 0) + (waiter,) = _GatedWaiter.instances + self.assertTrue(_GatedWaiter.released.is_set()) + self.assertGreaterEqual(waiter.waits, 1) + self.assertEqual(waiter.closed, 1) + self.assertNoFailureAccounted() + + def test_idle_daemon_times_out_at_the_deadline_without_accounting(self) -> None: + deadline = time.monotonic() + 0.1 + started = time.monotonic() + + with self.assertRaisesRegex(_ServeBuilderUnavailable, "timed out"): + self.compiler._serve_builder_read_line(self.client, deadline, None, None) + + elapsed = time.monotonic() - started + self.assertGreaterEqual(elapsed, 0.05) + self.assertLess(elapsed, 5.0) + # The wait path itself is pinned by the scripted timeout tests; a + # real deadline cannot guarantee the loop reached a wait first. + (waiter,) = _RecordingWaiter.instances + self.assertEqual(waiter.closed, 1) + self.assertNoFailureAccounted() + + def test_supersession_interrupts_a_blocked_read_on_the_cadence(self) -> None: + control = _Control() + threading.Timer(0.05, control.cancel_event.set).start() + started = time.monotonic() + + with self.assertRaisesRegex(_Superseded, "canceled after supersession"): + self.compiler._serve_builder_read_line( + self.client, self.deadline, None, control + ) + + self.assertLess(time.monotonic() - started, 5.0) + self.assertNoFailureAccounted() + self.assertEqual(_RecordingWaiter.instances[0].closed, 1) + + def test_cancellation_interrupts_a_blocked_write_on_the_cadence(self) -> None: + cancellation = _Cancellation() + stdin_write = self.client.process.stdin.fileno() + # Fill the pipe so the transport must wait for the peer to drain. + filler = b"\0" * (64 << 10) + while True: + try: + os.write(stdin_write, filler) + except BlockingIOError: + break + threading.Timer(0.05, cancellation.event.set).start() + started = time.monotonic() + + with self.assertRaises(_Cancelled): + self.compiler._serve_builder_write( + self.client, b"more", self.deadline, cancellation, None + ) + + self.assertLess(time.monotonic() - started, 5.0) + self.assertNoFailureAccounted() + self.assertEqual(_RecordingWaiter.instances[0].closed, 1) + + def test_daemon_exit_mid_frame_is_one_crash(self) -> None: + os.write(self.stdout_write, b"only-part") + self._close_fd(self.stdout_write) + + with self.assertRaisesRegex(_ServeBuilderUnavailable, "exited mid-request"): + self.compiler._serve_builder_read_exact( + self.client, 200, self.deadline, None, None + ) + + self.assertOneCrashAccounted() + self.assertEqual(_RecordingWaiter.instances[0].closed, 1) + + def test_daemon_reader_gone_is_closed_input(self) -> None: + self._close_fd(self.stdin_read) + + with self.assertRaisesRegex(_ServeBuilderUnavailable, "input pipe closed"): + self.compiler._serve_builder_write( + self.client, b"request\n", self.deadline, None, None + ) + + self.assertNoFailureAccounted() + self.assertEqual(_RecordingWaiter.instances[0].closed, 1) + + def test_daemon_reader_gone_while_blocked_wakes_the_writer(self) -> None: + stdin_write = self.client.process.stdin.fileno() + filler = b"\0" * (64 << 10) + while True: + try: + os.write(stdin_write, filler) + except BlockingIOError: + break + stdin_read = self.stdin_read + threading.Timer(0.05, lambda: self._close_fd(stdin_read)).start() + started = time.monotonic() + + with self.assertRaisesRegex(_ServeBuilderUnavailable, "input pipe closed"): + self.compiler._serve_builder_write( + self.client, b"more", self.deadline, None, None + ) + + self.assertLess(time.monotonic() - started, 5.0) + self.assertNoFailureAccounted() + + +# Echoes its stdin payload like the echo builder, but reads nothing until +# the marker file named by PRISM_TEST_READER_GATE exists: the coordinator's +# writer therefore fills the pipe and blocks by construction, and the gated +# waiter creates the marker on that first wait. +GATED_READER_BUILDER_COMMAND = [ + sys.executable, + "-c", + ( + "import json, os, sys, time\n" + "marker = os.environ.get('PRISM_TEST_READER_GATE')\n" + "while marker and not os.path.exists(marker):\n" + " time.sleep(0.001)\n" + "data = bytearray()\n" + "while True:\n" + " chunk = sys.stdin.buffer.read(65536)\n" + " if not chunk:\n" + " break\n" + " data += chunk\n" + "json.dump({'received': json.loads(bytes(data)), 'transport': 'one-shot'}," + " sys.stdout)\n" + ), +] + + +class OneShotWriterTests(unittest.TestCase): + """The one-shot subprocess writer waits for readiness like the daemon.""" + + def _coordinator(self): + server, _rpc = coordinator() + server.signing_seed_hex = "42" * 32 + server.ledger_attestation_signing_seed_hex = "43" * 32 + self.addCleanup(server.shutdown_serve_builder) + self.addCleanup(server.retire_share_window_spool) + return server + + def _serialization(self, server, shares): + return server._share_window_serialization_for_artifact( + PayoutLedgerArtifact( + generation=1, + payout_state_generation=0, + network_difficulty=1, + accepted_share_count=len(shares), + shares_json=tuple(shares), + prior_balances=(), + prepared_monotonic=time.monotonic(), + snapshot_anchor_ms=None, + ), + shares, + ) + + def _gate(self) -> str: + directory = tempfile.mkdtemp(prefix="prism-reader-gate-") + self.addCleanup(lambda: __import__("shutil").rmtree(directory, ignore_errors=True)) + return os.path.join(directory, "go") + + def _build(self, server, shares, *, serialization=None, command=None, gate=None): + environment = {"PRISM_BUILDER_SERVE": "0"} + if gate is not None: + environment["PRISM_TEST_READER_GATE"] = gate + _GatedWaiter.arm(gate) + waiter_class: type = _GatedWaiter + else: + _RecordingWaiter.instances = [] + waiter_class = _RecordingWaiter + with patch.dict(os.environ, environment), patch( + "lab.prism.prism_coordinator.prism_tool_command", + return_value=list(command or GATED_READER_BUILDER_COMMAND), + ), patch.object(compiler_module, "_PipeReadinessWaiter", waiter_class): + return server.build_audit_bundle( + shares=shares, + found_block={ + "block_height": 10, + "coinbase_value_sats": 50_00000000, + "network_difficulty": 1, + "anchor_job_issued_at_ms": 1_700_000_000_000, + }, + prior_balances=[], + coinbase_script_sig_suffix_hex="00", + summary_only=True, + payout_policy={"policy": "day-one"}, + share_serialization=serialization, + ) + + def test_buffered_writer_streams_to_a_gated_child_exactly(self) -> None: + server = self._coordinator() + server.bundle_build_timeout_seconds = 30.0 + # Well over one pipe capacity, so the fragment writer must block + # before the child is released to read anything. + shares = [spool_share(seq) for seq in range(1, 6_001)] + gate = self._gate() + + result = self._build(server, shares, gate=gate) + + self.assertEqual(result["transport"], "one-shot") + self.assertEqual(len(result["received"]["compact_shares"]), 6_000) + self.assertEqual(result["received"]["found_block"]["block_height"], 10) + self.assertTrue(os.path.exists(gate)) + (waiter,) = _GatedWaiter.instances + self.assertEqual(waiter._events, selectors.EVENT_WRITE) + self.assertGreaterEqual(waiter.waits, 1) + self.assertEqual(waiter.closed, 1) + with server._job_build_scheduler_lock: + counts = dict(server.job_build_worker_counts) + self.assertEqual(counts["crashes"], 0) + self.assertEqual(counts["terminations"], 0) + + @unittest.skipUnless(hasattr(os, "splice"), "os.splice is Linux-only") + def test_spool_splice_streams_to_a_gated_child_exactly(self) -> None: + server = self._coordinator() + server.bundle_build_timeout_seconds = 30.0 + shares = [spool_share(seq) for seq in range(1, 6_001)] + serialization = self._serialization(server, shares) + gate = self._gate() + + result = self._build(server, shares, serialization=serialization, gate=gate) + + self.assertEqual(result["transport"], "one-shot") + self.assertEqual(len(result["received"]["compact_shares"]), 6_000) + self.assertFalse(serialization._spool_failed) + self.assertTrue(os.path.exists(gate)) + (waiter,) = _GatedWaiter.instances + self.assertGreaterEqual(waiter.waits, 1) + self.assertEqual(waiter.closed, 1) + + def test_child_that_stops_reading_fails_the_build_without_hanging(self) -> None: + server = self._coordinator() + server.bundle_build_timeout_seconds = 0.5 + shares = [spool_share(seq) for seq in range(1, 6_001)] + stalled = [ + sys.executable, + "-c", + "import sys, time\nsys.stdin.buffer.read(1024)\ntime.sleep(30)\n", + ] + started = time.monotonic() + + with self.assertRaisesRegex(RuntimeError, "timed out"): + self._build(server, shares, command=stalled) + + self.assertLess(time.monotonic() - started, 10.0) + (waiter,) = _RecordingWaiter.instances + self.assertGreaterEqual(waiter.waits, 1) + self.assertEqual(waiter.closed, 1) + self.assertEqual(server.tip_refresh_worker_failures, 1) + + +class DaemonRoundTripTests(unittest.TestCase): + """A large window through the real daemon protocol over real pipes.""" + + def test_large_window_upload_and_hit_over_subprocess_pipes(self) -> None: + server, _rpc = coordinator() + server.signing_seed_hex = "42" * 32 + server.ledger_attestation_signing_seed_hex = "43" * 32 + self.addCleanup(server.shutdown_serve_builder) + self.addCleanup(server.retire_share_window_spool) + shares = [spool_share(seq) for seq in range(1, 20_001)] + serialization = server._share_window_serialization_for_artifact( + PayoutLedgerArtifact( + generation=1, + payout_state_generation=0, + network_difficulty=1, + accepted_share_count=len(shares), + shares_json=tuple(shares), + prior_balances=(), + prepared_monotonic=time.monotonic(), + snapshot_anchor_ms=None, + ), + shares, + ) + _RecordingWaiter.instances = [] + + def build(height: int): + with patch.dict(os.environ, {"FAKE_SERVE_BUILDER_MODE": "ok"}), patch( + "lab.prism.prism_coordinator.prism_tool_command", + return_value=list(FAKE_SERVE_BUILDER_COMMAND), + ), patch.object( + compiler_module, + "_PipeReadinessWaiter", + _RecordingWaiter, + ): + return server.build_audit_bundle( + shares=shares, + found_block={ + "block_height": height, + "coinbase_value_sats": 50_00000000, + "network_difficulty": 1, + "anchor_job_issued_at_ms": 1_700_000_000_000, + }, + prior_balances=[], + coinbase_script_sig_suffix_hex="00", + summary_only=True, + payout_policy={"policy": "day-one"}, + share_serialization=serialization, + ) + + first = build(10) + second = build(11) + + self.assertEqual(first["transport"], "serve") + self.assertTrue(first["request_had_window"]) + self.assertEqual(len(first["window"]["compact_shares"]), 20_000) + self.assertEqual(second["transport"], "serve") + self.assertFalse(second["request_had_window"]) + self.assertEqual(len(second["window"]["compact_shares"]), 20_000) + with server._serve_builder_metrics_lock: + counts = dict(server.serve_builder_counts) + self.assertEqual(counts["requests"], 2) + self.assertEqual(counts["fallbacks"], 0) + # Every transport operation registered and released its waiter; how + # often the fake daemon's parse outran the coordinator is scheduling + # and is not asserted. + self.assertTrue(_RecordingWaiter.instances) + self.assertTrue(all(w.closed == 1 for w in _RecordingWaiter.instances)) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_prism_candidate_codec.py b/tests/test_prism_candidate_codec.py new file mode 100644 index 000000000..246a2b8c3 --- /dev/null +++ b/tests/test_prism_candidate_codec.py @@ -0,0 +1,457 @@ +#!/usr/bin/env python3 +"""Bounded candidate codec regressions (issue #255). + +Two contracts are pinned here. First, byte identity with the historical +v1 identity oracle: for every valid fixture the chunked body concatenates +to exactly ``json.dumps(block_candidate_identity(intent), sort_keys=True, +separators=(",", ":"))`` and ``candidate_sha256`` is the oracle's digest. +Second, boundedness by mechanism, not by timing: every ``json.dumps`` call +the encoder makes is instrumented and asserted to cover at most the batch +record cap, and every chunk it emits is asserted to be the fixed chunk +size except the last. Nothing here measures elapsed time. +""" + +from __future__ import annotations + +import hashlib +import json +import os +import sys +import tempfile +import unittest +from pathlib import Path +from typing import Any +from unittest import mock + +sys.path.insert(0, str(Path(__file__).resolve().parents[1])) + +from lab.prism import candidate_codec as codec # noqa: E402 +from lab.prism.share_ledger import ( # noqa: E402 + AcceptedShareRecord, + DaemonShareJsonSequence, + IncrementalShareJsonSequence, + _IncrementalShareWindowPage, + block_candidate_identity, + block_candidate_identity_sha256, + sha256_json_hex, +) + +HASH_A = "aa" * 32 +PARENT = "bb" * 32 +RUN_SLOW = os.environ.get("PRISM_CANDIDATE_CODEC_SLOW") == "1" + + +def oracle_bytes(intent: dict[str, Any]) -> bytes: + """The historical identity JSON, computed the historical way.""" + identity = block_candidate_identity({**intent, "block_hash_hex": intent["block_hash_hex"].lower()}) + return json.dumps(identity, sort_keys=True, separators=(",", ":")).encode("utf-8") + + +def share(index: int) -> dict[str, Any]: + return { + "share_seq": index, + "share_id": f"miner-{index % 7}:é\U0001F600{'x' * (index % 3)}", + "miner_id": f"m{index % 7}", + "order_key": "k", + "p2mr_program_hex": "ab" * 32, + "share_difficulty": 10**30 + index, + "network_difficulty": 1.5e16 if index % 5 else 3, + "template_height": 5, + "job_id": 'j"q\\', + "job_issued_at_ms": 1_700_000_000_000 + index, + "accepted_at_ms": 1_700_000_000_001 + index, + "ntime": 1_700_000_000, + "credit_policy": None if index % 2 else "stale-grace", + } + + +def intent_for(share_count: int, **overrides: Any) -> dict[str, Any]: + fields: dict[str, Any] = { + "schema": codec.CANDIDATE_INTENT_SCHEMA, + "block_hash_hex": HASH_A.upper(), + "block_hex": "00" * 100, + "coinbase_tx_hex": "01", + "parent_hash": PARENT, + "expected_height": 10, + "template": {"previousblockhash": PARENT, "height": 10, "coinbasevalue": 5}, + "shares_json": [share(index) for index in range(share_count)], + "prior_balances": [{"recipient_id": "a", "balance_sats": 1}], + "found_block": {"network_difficulty": 1.0, "witness": [1, 2, {"z": None}]}, + "prospective_prior_balances": None, + "witness_merkle_leaves_hex": ["cc" * 32] * 3, + "extranonce1_hex": "00", + "extranonce2_hex": "01", + "username": "u", + "pending_share": {"share_id": "s", "accepted_at_ms": 123}, + "credit_share_on_accept": False, + "collection_only": False, + } + fields.update(overrides) + return fields + + +class _Instrumented: + """Instrument ``json.dumps`` inside the codec and collect chunk facts.""" + + def __init__(self) -> None: + self.dumps_inputs: list[int] = [] + self.dumps_outputs: list[int] = [] + self.chunks: list[codec.BodyChunk] = [] + self._real = json.dumps + + def dumps(self, value: Any, *args: Any, **kwargs: Any) -> str: + text = self._real(value, *args, **kwargs) + self.dumps_inputs.append(len(value) if isinstance(value, (list, tuple)) else 1) + self.dumps_outputs.append(len(text)) + return text + + def collect(self, chunk: codec.BodyChunk) -> None: + self.chunks.append(chunk) + + +class CodecByteIdentityTests(unittest.TestCase): + def assert_body_matches_oracle(self, fields: dict[str, Any], *, chunk_bytes: int = 4096) -> codec.PreparedCandidateIntent: + prepared = codec.prepare_candidate_intent(fields, chunk_bytes=chunk_bytes) + got = bytearray() + chunks: list[codec.BodyChunk] = [] + + def collect(chunk: codec.BodyChunk) -> None: + chunks.append(chunk) + got.extend(chunk.data) + + prepared.body.write_chunks(collect) + expected = oracle_bytes(fields) + self.assertEqual(bytes(got), expected) + self.assertEqual(prepared.candidate_sha256, hashlib.sha256(expected).hexdigest()) + self.assertEqual(prepared.candidate_sha256, sha256_json_hex(block_candidate_identity({**fields, "block_hash_hex": HASH_A}))) + self.assertEqual(block_candidate_identity_sha256(prepared), prepared.candidate_sha256) + # Fixed chunk boundaries: every chunk but the last is exactly the size. + self.assertEqual([chunk.ordinal for chunk in chunks], list(range(len(chunks)))) + self.assertTrue(all(len(chunk.data) == chunk_bytes for chunk in chunks[:-1])) + self.assertEqual(sum(len(chunk.data) for chunk in chunks), prepared.manifest.byte_count) + self.assertEqual(len(chunks), prepared.manifest.chunk_count) + for chunk in chunks: + self.assertEqual(hashlib.sha256(chunk.data).hexdigest(), chunk.sha256) + return prepared + + def test_share_counts_around_batch_boundaries(self) -> None: + for count in (0, 1, 255, 256, 257, 511, 512, 513, 1300): + with self.subTest(count=count): + prepared = self.assert_body_matches_oracle(intent_for(count)) + self.assertEqual(prepared.manifest.share_count, count) + self.assertIs(prepared["shares_json"], prepared.shares) + self.assertEqual(prepared["pending_share"]["accepted_at_ms"], 123) + self.assertTrue(prepared.accepted_at_present) + + @unittest.skipUnless(RUN_SLOW, "set PRISM_CANDIDATE_CODEC_SLOW=1 for the 375,000-share oracle") + def test_stress_window_matches_oracle(self) -> None: + prepared = self.assert_body_matches_oracle(intent_for(375_000), chunk_bytes=codec.CANDIDATE_BODY_CHUNK_BYTES) + self.assertEqual(prepared.manifest.share_count, 375_000) + + def test_giant_string_field_spans_chunk_and_slice_boundaries(self) -> None: + big = ("é\"\\" + "x" * 1000 + "\U0001F600") * 300 + prepared = self.assert_body_matches_oracle(intent_for(3, block_hex=big), chunk_bytes=1000) + self.assertGreater(prepared.manifest.span_count, 1) + + def test_giant_field_inside_one_record(self) -> None: + fields = intent_for(600) + fields["shares_json"][300]["share_id"] = "y" * (5 * 1024 * 1024) + self.assert_body_matches_oracle(fields, chunk_bytes=codec.CANDIDATE_BODY_CHUNK_BYTES) + + def test_first_oversized_record_is_bounded_before_encoding(self) -> None: + fields = intent_for(260) + fields["shares_json"][0]["share_id"] = "😀" * 100_000 + fields["shares_json"][257]["share_id"] = "x" * (5 * 1024 * 1024) + expected = oracle_bytes(fields) + observer = _Instrumented() + with mock.patch.object(codec.json, "dumps", observer.dumps): + prepared = codec.prepare_candidate_intent(fields) + prepared.body.write_chunks(observer.collect) + self.assertEqual(b"".join(chunk.data for chunk in observer.chunks), expected) + self.assertLessEqual(max(observer.dumps_outputs), codec.CODEC_BATCH_TARGET_BYTES) + + def test_large_integers_floats_nulls_and_field_order(self) -> None: + fields = intent_for( + 2, + found_block={"network_difficulty": 2**200, "float": 1e300, "tiny": 5e-324, "neg": -0.0, "none": None}, + extension_zzz={"b": [1, {"a": None}], "a": ""}, + extension_aaa=[[], {}, "", 0, False], + ) + prepared = self.assert_body_matches_oracle(fields) + self.assertEqual(list(prepared), list(fields)) + + def test_numeric_keys_and_oversized_nested_key_keep_bounded_encoding(self) -> None: + fields = intent_for(1, extension={1: "x" * 300_000, 2.5: ["y" * 300_000]}, + found_block={"z" * 300_000: True}) + expected = oracle_bytes(fields) + observer = _Instrumented() + with mock.patch.object(codec.json, "dumps", observer.dumps): + prepared = codec.prepare_candidate_intent(fields) + prepared.body.write_chunks(observer.collect) + self.assertEqual(b"".join(chunk.data for chunk in observer.chunks), expected) + self.assertLessEqual(max(observer.dumps_outputs), codec.CODEC_BATCH_TARGET_BYTES) + + def test_absent_and_null_stamp_are_distinct(self) -> None: + absent = intent_for(2) + del absent["pending_share"]["accepted_at_ms"] + prepared_absent = self.assert_body_matches_oracle(absent) + self.assertFalse(prepared_absent.accepted_at_present) + null = intent_for(2, pending_share={"share_id": "s", "accepted_at_ms": None}) + prepared_null = self.assert_body_matches_oracle(null) + self.assertTrue(prepared_null.accepted_at_present) + self.assertNotEqual(prepared_absent.candidate_sha256, prepared_null.candidate_sha256) + + def test_stamp_drift_keeps_identity(self) -> None: + first = codec.prepare_candidate_intent(intent_for(3)) + second = codec.prepare_candidate_intent(intent_for(3, pending_share={"share_id": "s", "accepted_at_ms": 999})) + self.assertEqual(first.candidate_sha256, second.candidate_sha256) + self.assertEqual(second.accepted_at_ms, 999) + + def test_candidate_only_intent_without_shares(self) -> None: + fields = {"schema": codec.CANDIDATE_INTENT_SCHEMA, "block_hash_hex": HASH_A, "block_hex": "00"} + prepared = self.assert_body_matches_oracle(fields) + self.assertFalse(prepared.has_shares) + self.assertNotIn("shares_json", prepared) + + def test_empty_and_large_balances_and_witness_arrays(self) -> None: + empty = intent_for(1, prior_balances=[], witness_merkle_leaves_hex=[], prospective_prior_balances=[]) + self.assert_body_matches_oracle(empty) + large = intent_for( + 1, + prior_balances=[{"recipient_id": f"r{index}", "balance_sats": index} for index in range(5000)], + prospective_prior_balances=[["a", "b", "c", index] for index in range(5000)], + witness_merkle_leaves_hex=["cc" * 32] * 20_000, + ) + prepared = self.assert_body_matches_oracle(large) + self.assertGreaterEqual(prepared.manifest.span_count, 4) + + +class CodecRejectionTests(unittest.TestCase): + def test_nan_and_infinity_are_rejected_before_credit(self) -> None: + for value in (float("nan"), float("inf"), -float("inf")): + with self.subTest(value=value), self.assertRaises(codec.CandidateCodecError): + codec.prepare_candidate_intent(intent_for(1, found_block={"network_difficulty": value})) + with self.subTest(value=value, where="share"), self.assertRaises(codec.CandidateCodecError): + codec.prepare_candidate_intent(intent_for(0, shares_json=[{"a": value}])) + + def test_unsupported_types_raise_type_error_like_json_dumps(self) -> None: + for value in (b"bytes", {1, 2}, object()): + with self.subTest(value=type(value).__name__), self.assertRaises(TypeError): + codec.prepare_candidate_intent(intent_for(1, found_block={"blob": value})) + + def test_jsonb_incompatible_text_is_rejected_everywhere(self) -> None: + nul = chr(0) + high, low = "\ud83d", "\ude00" + cases = { + "nul in field": intent_for(1, username="a" + nul + "b"), + "nul in key": intent_for(1, **{"ext" + nul: 1}), + "nul in share": intent_for(0, shares_json=[{"a": "x" + nul}]), + "nul in giant": intent_for(1, block_hex="y" * 70000 + nul), + "nul in nested share list": intent_for(0, shares_json=[{"a": [nul]}]), + "lone high": intent_for(1, username=high), + "lone low in share": intent_for(0, shares_json=[{"a": low}]), + "lone high in giant": intent_for(1, block_hex="y" * 70000 + high), + "nul in non-string-key dict": intent_for(1, found_block={1: "x" + nul}), + "lone in non-string-key dict": intent_for(1, found_block={1: high}), + "high then high": intent_for(0, shares_json=[{"a": high + high + low}]), + } + for label, fields in cases.items(): + with self.subTest(label=label), self.assertRaises(codec.CandidateCodecError): + codec.prepare_candidate_intent(fields) + + def test_jsonb_compatible_escapes_are_accepted(self) -> None: + high, low = "\ud83d", "\ude00" + for label, fields in { + "valid pair": intent_for(1, username="\U0001F600", shares_json=[{"a": "\U0001F600"}]), + "adjacent lone units forming a pair": intent_for(1, username=high + low), + "escaped backslash-u text": intent_for(1, username="\\u0000", shares_json=[{"a": "\\u0000", "b": "\\\\ud83d"}]), + "pair at slice edge": intent_for(1, block_hex="y" * 65535 + "\U0001F600" + "z"), + "literal surrogate pair at slice edge": intent_for(1, block_hex="y" * (codec.CODEC_STRING_SLICE_CHARS - 1) + high + low + "z"), + }.items(): + with self.subTest(label=label): + prepared = codec.prepare_candidate_intent(fields) + got = bytearray() + prepared.body.write_chunks(lambda chunk: got.extend(chunk.data)) + self.assertEqual(bytes(got), oracle_bytes(fields)) + + def test_missing_hash_and_bad_share_container(self) -> None: + with self.assertRaises(codec.CandidateCodecError): + codec.prepare_candidate_intent({"schema": "x", "shares_json": []}) + with self.assertRaises(codec.CandidateCodecError): + codec.prepare_candidate_intent(intent_for(0, shares_json="not a sequence")) + + def test_changed_source_cannot_publish(self) -> None: + fields = intent_for(5) + prepared = codec.prepare_candidate_intent(fields) + fields["shares_json"].append(share(99)) + with self.assertRaises(codec.CandidateBodyIntegrityError): + prepared.body.write_chunks(lambda chunk: None) + + def test_mutated_prepared_intent_is_re_prepared(self) -> None: + prepared = codec.prepare_candidate_intent(intent_for(2)) + original = prepared.candidate_sha256 + prepared.pop("collection_only") + self.assertTrue(prepared.dirty) + refreshed = codec.prepare_candidate_intent(prepared) + self.assertNotEqual(refreshed.candidate_sha256, original) + self.assertEqual(refreshed.candidate_sha256, hashlib.sha256(oracle_bytes(dict(prepared))).hexdigest()) + + +class CodecBoundednessTests(unittest.TestCase): + def test_every_dumps_call_and_chunk_is_bounded(self) -> None: + fields = intent_for(3000, prior_balances=[{"recipient_id": f"r{i}", "balance_sats": i} for i in range(3000)]) + probe = _Instrumented() + with mock.patch.object(codec.json, "dumps", probe.dumps): + prepared = codec.prepare_candidate_intent(fields, chunk_bytes=codec.CANDIDATE_BODY_CHUNK_BYTES) + prepared.body.write_chunks(probe.collect) + self.assertTrue(probe.dumps_inputs) + self.assertLessEqual(max(probe.dumps_inputs), codec.CODEC_BATCH_RECORDS) + self.assertLessEqual(max(probe.dumps_outputs), codec.CODEC_BATCH_OVERSIZED_BYTES) + self.assertTrue(all(len(chunk.data) <= codec.CANDIDATE_BODY_CHUNK_BYTES for chunk in probe.chunks)) + + def test_daemon_sequence_is_copied_verbatim_without_parsing(self) -> None: + records = [share(index) for index in range(2000)] + records[700]["share_id"] = 'trap},{"accepted_at_ms":1' + items = b",".join(json.dumps(record, sort_keys=True, separators=(",", ":"), default=str).encode() for record in records) + daemon = DaemonShareJsonSequence(items, len(records)) + fields = intent_for(0, shares_json=daemon) + prepared = codec.prepare_candidate_intent(fields, chunk_bytes=4096) + got = bytearray() + prepared.body.write_chunks(lambda chunk: got.extend(chunk.data)) + self.assertIsNone(daemon._parsed, "the daemon sequence must not be parsed to stage it") + self.assertEqual(bytes(got), oracle_bytes(intent_for(0, shares_json=records))) + self.assertEqual(prepared.manifest.share_count, 2000) + + def test_page_backed_window_pages_are_copied_verbatim(self) -> None: + records = tuple( + AcceptedShareRecord(share_seq=i, share_id=f"s{i}", miner_id="m", order_key="k", p2mr_program_hex="ab" * 32, share_difficulty=1, network_difficulty=2, template_height=3, job_id="j", job_issued_at_ms=4, accepted_at_ms=5, ntime=6) + for i in range(1100) + ) + pages = tuple(_IncrementalShareWindowPage.from_records(records[i : i + 512]) for i in range(0, 1100, 512)) + window = IncrementalShareJsonSequence(pages=pages, record_count=1100) + prepared = codec.prepare_candidate_intent(intent_for(0, shares_json=window), chunk_bytes=4096) + got = bytearray() + prepared.body.write_chunks(lambda chunk: got.extend(chunk.data)) + self.assertEqual(bytes(got), oracle_bytes(intent_for(0, shares_json=list(window)))) + self.assertGreaterEqual(prepared.manifest.page_count, 5) + + def test_replay_header_is_bounded_and_defers_oversized_facts(self) -> None: + bounded = codec.replay_header_from_fields(intent_for(1)) + self.assertFalse(bounded["oversized"]) + self.assertEqual(bounded["pending_share"]["accepted_at_ms"], 123) + oversized = codec.replay_header_from_fields(intent_for(1, username="u" * 1000, pending_share={"share_id": "s" * 1000, "accepted_at_ms": 123})) + self.assertTrue(oversized["oversized"]) + self.assertIsNone(oversized["username"]) + self.assertIsNone(oversized["pending_share"]["share_id"]) + self.assertEqual(oversized["pending_share"]["accepted_at_ms"], 123) + self.assertLess(len(json.dumps(oversized)), 4096) + + def test_replay_header_rejects_giant_text_before_encoding_it(self) -> None: + observer = _Instrumented() + fields = intent_for(1, username="u" * (5 * 1024 * 1024)) + with mock.patch.object(codec.json, "dumps", observer.dumps): + header = codec.replay_header_from_fields(fields) + self.assertIsNone(header["username"]) + self.assertLessEqual(max(observer.dumps_outputs), 4096) + + +class SpoolRoundTripTests(unittest.TestCase): + def _spool(self, prepared: codec.PreparedCandidateIntent) -> codec.SpoolCandidateBody: + directory = tempfile.mkdtemp() + self.addCleanup(lambda: [os.unlink(os.path.join(directory, name)) for name in os.listdir(directory)] and os.rmdir(directory) if os.path.isdir(directory) else None) + return codec.write_spool_from_body(os.path.join(directory, "b.body"), os.path.join(directory, "b.idx"), prepared.body) + + def test_pages_decode_and_random_access_use_the_disk_index(self) -> None: + for count in (0, 1, 256, 1300): + with self.subTest(count=count): + fields = intent_for(count, prior_balances=[{"recipient_id": f"r{i}", "balance_sats": i} for i in range(4000 if count == 1300 else 1)]) + prepared = codec.prepare_candidate_intent(fields, chunk_bytes=4096) + spool = self._spool(prepared) + hydrated = codec.prepared_intent_from_spool(spool, accepted_at_present=True, accepted_at_ms=123) + shares = hydrated["shares_json"] + self.assertEqual(len(shares), count) + self.assertEqual(list(shares), fields["shares_json"]) + if count: + self.assertEqual(shares[0], fields["shares_json"][0]) + self.assertEqual(shares[-1], fields["shares_json"][-1]) + self.assertEqual(shares[count // 2], fields["shares_json"][count // 2]) + self.assertEqual(shares.canonical_json_sha256(), hashlib.sha256(json.dumps(fields["shares_json"], sort_keys=True, separators=(",", ":"), default=str).encode()).hexdigest()) + self.assertEqual(list(hydrated["prior_balances"]), fields["prior_balances"]) + if count == 1300: + self.assertIsInstance(hydrated["prior_balances"], codec.SpoolJsonArraySequence) + self.assertEqual(hydrated["pending_share"]["accepted_at_ms"], 123) + regot = bytearray() + hydrated.body.write_chunks(lambda chunk: regot.extend(chunk.data)) + self.assertEqual(bytes(regot), oracle_bytes(fields)) + self.assertEqual(codec.prepare_candidate_intent(dict(hydrated), chunk_bytes=4096).candidate_sha256, prepared.candidate_sha256) + paths = (spool.path, spool.index.path) + spool.close() + self.assertFalse(any(os.path.exists(path) for path in paths)) + + def test_daemon_pages_ignore_quoted_and_nested_boundary_traps(self) -> None: + records = [share(index) for index in range(2000)] + records[700]["share_id"] = 'trap},{"accepted_at_ms":1' + records[511]["nested"] = [{"a": 1}, {"b": "\\\"},{"}] + items = b",".join(json.dumps(record, sort_keys=True, separators=(",", ":"), default=str).encode() for record in records) + prepared = codec.prepare_candidate_intent(intent_for(0, shares_json=DaemonShareJsonSequence(items, 2000)), chunk_bytes=4096) + spool = self._spool(prepared) + span = spool.index.span("shares_json") + self.assertTrue(span.pages_exact) + sequence = codec.SpoolJsonArraySequence(spool, span) + self.assertEqual(sequence[1999], records[1999]) + self.assertEqual(list(sequence), records) + self.assertTrue(spool.index.span("shares_json").pages_exact) + self.assertEqual(sequence[700], records[700]) + spool.close() + + def test_giant_string_and_oversized_record_decode_by_slices(self) -> None: + big = ("é\"\\" + "x" * 1000 + "\U0001F600") * 300 + fields = intent_for(600, block_hex=big) + fields["shares_json"][300]["share_id"] = "y" * (5 * 1024 * 1024) + prepared = codec.prepare_candidate_intent(fields, chunk_bytes=codec.CANDIDATE_BODY_CHUNK_BYTES) + spool = self._spool(prepared) + hydrated = codec.prepared_intent_from_spool(spool, accepted_at_present=True, accepted_at_ms=123) + self.assertEqual(hydrated["block_hex"], big) + self.assertEqual(hydrated["shares_json"][300], fields["shares_json"][300]) + spool.close() + + def test_truncated_and_corrupt_spools_fail_closed(self) -> None: + prepared = codec.prepare_candidate_intent(intent_for(300), chunk_bytes=4096) + spool = self._spool(prepared) + with open(spool.path, "r+b") as handle: + handle.seek(10) + handle.write(b"Z") + with self.assertRaises(codec.CandidateBodyIntegrityError): + list(spool.iter_chunks()) + with open(spool.path, "r+b") as handle: + handle.truncate(100) + with self.assertRaises(codec.CandidateBodyIntegrityError): + list(spool.iter_chunks()) + spool.close() + + def test_manifest_and_span_validation(self) -> None: + manifest = codec.prepare_candidate_intent(intent_for(3)).manifest + self.assertEqual(codec.CandidateBodyManifest.from_json(manifest.to_json()), manifest) + for broken in ( + {**manifest.to_json(), "chunk_count": manifest.chunk_count + 1}, + {**manifest.to_json(), "storage_version": 3}, + {**manifest.to_json(), "shares_end": manifest.byte_count + 1}, + {**manifest.to_json(), "candidate_sha256": "zz"}, + ): + with self.subTest(broken=broken), self.assertRaises(codec.CandidateBodyIntegrityError): + codec.CandidateBodyManifest.from_json(broken) + with self.assertRaises(codec.CandidateBodyIntegrityError): + codec.FieldSpan.from_json({"field": "x", "kind": "blob", "start": 0, "end": 1}) + + def test_jsonb_equivalence_matches_postgres_rules(self) -> None: + from decimal import Decimal + + self.assertTrue(codec.jsonb_equivalent({"a": Decimal("1.0"), "b": [1, 2]}, {"b": [Decimal(1), 2], "a": 1})) + self.assertTrue(codec.jsonb_equivalent(Decimal("1E+16"), 10**16)) + self.assertFalse(codec.jsonb_equivalent(True, 1)) + self.assertFalse(codec.jsonb_equivalent({"a": 1}, {"a": 1, "b": None})) + self.assertFalse(codec.jsonb_equivalent([1, 2], [2, 1])) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_prism_candidate_helper.py b/tests/test_prism_candidate_helper.py new file mode 100644 index 000000000..3c8f58212 --- /dev/null +++ b/tests/test_prism_candidate_helper.py @@ -0,0 +1,75 @@ +"""Exercise helper backpressure and shutdown with real child processes.""" + +from __future__ import annotations + +import subprocess +import sys +import threading +import time +import unittest +from unittest import mock + +from lab.prism import candidate_store as store +from lab.prism.candidate_codec import prepare_candidate_intent + + +class CandidateHelperLifecycleTests(unittest.TestCase): + def helper(self, **kwargs): + return store.LegacyCandidateHelper( + store.LegacyTransport(database_url="unused", psql_command=()), **kwargs + ) + + def test_stalled_input_obeys_deadline_and_reaps_child(self): + body = prepare_candidate_intent({ + "block_hash_hex": "aa" * 32, "shares_json": [], "pad": "x" * (2 * 1024 * 1024), + }).body + real_popen = subprocess.Popen + children = [] + + def sleeping_child(_args, **kwargs): + process = real_popen([sys.executable, "-c", "import time; time.sleep(60)"], **kwargs) + children.append(process) + return process + + started = time.monotonic() + with mock.patch.object(store.subprocess, "Popen", sleeping_child): + with self.assertRaisesRegex(store.CandidateStorageError, "deadline"): + self.helper(timeout_seconds=0.2).compare("aa" * 32, body) + self.assertLess(time.monotonic() - started, 3.0) + self.assertEqual(len(children), 1) + self.assertIsNotNone(children[0].poll()) + self.assertTrue(all(pipe.closed for pipe in (children[0].stdin, children[0].stdout, children[0].stderr))) + self.assertFalse(any(t.name.startswith("prism-legacy-helper-") for t in threading.enumerate())) + + def test_cancellation_wrapper_waits_for_same_admission_slot(self): + cancelled = threading.Event() + helper = self.helper(timeout_seconds=1.0) + wrapper = helper.with_cancellation(cancelled.is_set) + helper._lock.acquire() + cancelled.set() + try: + with mock.patch.object(store.subprocess, "Popen") as spawn: + with self.assertRaisesRegex(store.CandidateStorageError, "cancelled"): + wrapper.convert("aa" * 32, "unused.body", "unused.idx") + spawn.assert_not_called() + finally: + helper._lock.release() + + def test_child_diagnostic_backpressure_does_not_deadlock(self): + real_popen = subprocess.Popen + + def noisy_child(_args, **kwargs): + return real_popen([ + sys.executable, "-c", + "import sys; sys.stdin.readline(); " + "sys.stderr.write('x' * 1000000); sys.stderr.flush(); " + "sys.stdout.write('{\"equal\":true}\\n'); sys.stdout.flush(); sys.stdin.read()", + ], **kwargs) + + body = prepare_candidate_intent({"block_hash_hex": "aa" * 32, "shares_json": []}).body + with mock.patch.object(store.subprocess, "Popen", noisy_child): + self.assertTrue(self.helper(timeout_seconds=3.0).compare("aa" * 32, body)) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_prism_candidate_replay_bounds.py b/tests/test_prism_candidate_replay_bounds.py new file mode 100644 index 000000000..221ba629f --- /dev/null +++ b/tests/test_prism_candidate_replay_bounds.py @@ -0,0 +1,119 @@ +"""Descriptor backpressure bounds obligations and preserves complete replay.""" + +from types import SimpleNamespace +import unittest +from unittest import mock + +from lab.prism.candidate_codec import replay_header_from_fields +from lab.prism.candidate_store import CandidateHeaderPage +from tests.prism_postgres_candidate_gate import intent +from tests.prism_vardiff_test_support import submit_coordinator + + +class ReplayDescriptorBoundsTests(unittest.TestCase): + def test_overflow_pauses_before_adoption_and_replays_every_row_after_drain(self): + for page_size in (2, 1024): + with self.subTest(page_size=page_size): + server, _state, _recording = submit_coordinator() + service = server._ensure_block_candidate_service() + rows = [] + for index in range(1, 12): + fields = intent(f"{index:064x}", 0, credit=True) + rows.append({"block_hash": fields["block_hash_hex"], "storage_version": 1, + "candidate_sha256": "ab" * 32, "cursor": index, + "header": replay_header_from_fields(fields)}) + pending = list(rows) + queries = [] + + def headers(*, limit, after_cursor, max_bytes): + remaining = [row for row in pending if row["cursor"] > (after_cursor or 0)] + page = remaining[:limit] + queries.append((after_cursor, len(page))) + return CandidateHeaderPage(tuple(page), page[-1]["cursor"] if page else None, + len(remaining) <= limit, len(page), False) + + server.ledger = SimpleNamespace(candidate_hydration_deferred=True, + pending_block_candidate_headers=headers) + server.config = SimpleNamespace(block=SimpleNamespace(replay_page_size=page_size)) + server._run_block_submitter_ledger_call = lambda _key, _phase, call, **_kwargs: call() + service._collapse_superseded_block_candidates = lambda rows, **_kwargs: rows + previews = set() + server._begin_accepted_block_payout_preview = lambda key, **_kwargs: previews.add(key) + server._clear_accepted_block_payout_preview = previews.discard + server._finish_pending_share_candidate = mock.Mock() + server._note_block_replay_enumeration_owed() + seen = [] + writer = service.ports.share_writer() + with mock.patch.object(writer, "adopt_pending_share"), \ + mock.patch("lab.prism.block_candidates.MAX_BLOCK_REPLAY_DESCRIPTORS_IN_MEMORY", 3), \ + mock.patch("builtins.print"): + while pending: + queued = server.replay_pending_block_candidates() + self.assertEqual(queued, min(3, len(pending))) + self.assertLessEqual(len(service._block_replay_floor_holders), 3) + self.assertEqual(len(previews), queued) + if len(pending) > 3: + self.assertTrue(server._block_replay_enumeration_owed()) + before = len(queries) + self.assertEqual(server.replay_pending_block_candidates(), 0) + self.assertEqual(len(queries), before) + else: + self.assertFalse(server._block_replay_enumeration_owed()) + while not service._block_replay_candidate_queue.empty(): + descriptor = service._block_replay_candidate_queue.get_nowait() + seen.append(descriptor.block_hash) + pending[:] = [row for row in pending if row["block_hash"] != descriptor.block_hash] + service._drop_replay_descriptor(descriptor) + self.assertEqual(service._block_replay_floor_holders, {}) + self.assertEqual(previews, set()) + self.assertEqual(seen, [row["block_hash"] for row in rows]) + self.assertEqual(server._finish_pending_share_candidate.call_count, len(rows)) + + def test_oversized_header_is_registered_and_hydrates_the_body_pending_share(self): + # share_id is ":": at 200 bytes only share_id + # exceeds the header bound, at 300 bytes username does too. + for username_bytes in (200, 300): + with self.subTest(username_bytes=username_bytes): + server, _state, _recording = submit_coordinator() + service = server._ensure_block_candidate_service() + fields = intent("ab" * 32, 0, credit=True) + fields["username"] = "q" * username_bytes + fields["pending_share"]["share_id"] = f"{fields['username']}:{fields['block_hash_hex']}" + header = replay_header_from_fields(fields) + self.assertTrue(header["oversized"]) + self.assertIsNone(header["pending_share"]["share_id"]) + row = {"block_hash": fields["block_hash_hex"], "storage_version": 1, + "candidate_sha256": "ab" * 32, "cursor": 1, "header": header} + + def headers(*, limit, after_cursor, max_bytes): + page = (row,) if after_cursor is None else () + return CandidateHeaderPage(page, 1, True, len(page), False) + + server.ledger = SimpleNamespace( + candidate_hydration_deferred=True, + pending_block_candidate_headers=headers, + hydrate_block_candidate_intent=lambda _row, **_kwargs: fields, + ) + server.config = SimpleNamespace(block=SimpleNamespace(replay_page_size=8)) + server._run_block_submitter_ledger_call = lambda _key, _phase, call, **_kwargs: call() + server._block_landing_db_timeout = lambda: 5.0 + service._collapse_superseded_block_candidates = lambda rows, **_kwargs: rows + server._begin_accepted_block_payout_preview = mock.Mock() + server._queue_invalid_block_candidate_for_quarantine = mock.Mock() + server._finish_pending_share_candidate = mock.Mock() + server._note_block_replay_enumeration_owed() + writer = service.ports.share_writer() + with mock.patch.object(writer, "adopt_pending_share") as adopt, \ + mock.patch("builtins.print"): + self.assertEqual(server.replay_pending_block_candidates(), 1) + descriptor = service._block_replay_candidate_queue.get_nowait() + self.assertTrue(descriptor.header_oversized) + stand_in = service._block_replay_floor_holders[descriptor.block_hash] + candidate = service._hydrate_replay_descriptor(descriptor) + server._queue_invalid_block_candidate_for_quarantine.assert_not_called() + self.assertEqual(candidate.pending_share.share_id, fields["pending_share"]["share_id"]) + # The body's share takes the floor before the stand-in leaves it. + self.assertEqual([call.args[0] for call in adopt.call_args_list], + [stand_in, candidate.pending_share]) + server._finish_pending_share_candidate.assert_called_once_with(stand_in) + self.assertEqual(service._block_replay_floor_holders, {}) diff --git a/tests/test_prism_candidate_schema_refusal.py b/tests/test_prism_candidate_schema_refusal.py new file mode 100644 index 000000000..d9e179260 --- /dev/null +++ b/tests/test_prism_candidate_schema_refusal.py @@ -0,0 +1,82 @@ +"""Every storage version refuses a database without the 002 migration (#255).""" + +from types import SimpleNamespace +import unittest +from unittest.mock import Mock + +from lab.prism import recover_pending_blocks as recovery +from lab.prism.candidate_store import IncompatibleCandidateSchema +from lab.prism.share_ledger import PsqlShareLedger + +# (has_body_table, declared capability): nothing migrated, and a partial +# migration whose capability row (declared last) never landed. +UNMIGRATED = ((False, None), (True, None)) + + +def ledger(storage_version, has_body_table, declared): + value = PsqlShareLedger.__new__(PsqlShareLedger) + value._candidate_storage_version = storage_version + + def run_json(sql): + if "to_regclass" in sql: + return {"has_body_table": has_body_table, "has_capabilities": declared is not None} + return {"declared": declared} + + value._run_json = run_json + return value + + +def reader(has_body_table, declared): + def execute(sql, _params=None): + if "to_regclass" in sql: + row = {"has_body_table": has_body_table, "has_capabilities": declared is not None} + else: + row = {"capability_value": declared} + return SimpleNamespace(fetchone=lambda: row) + + value = recovery.RecoveryReader.__new__(recovery.RecoveryReader) + value.connection = SimpleNamespace(execute=execute) + return value + + +class LedgerSchemaRefusalTests(unittest.TestCase): + def test_unmigrated_schema_is_refused_for_every_storage_version(self): + for storage_version in (1, 2): + for has_body_table, declared in UNMIGRATED: + with self.subTest(storage_version=storage_version, has_body_table=has_body_table), \ + self.assertRaisesRegex(IncompatibleCandidateSchema, "002_candidate_bodies.sql"): + ledger(storage_version, has_body_table, declared).verify_candidate_schema() + + def test_migrated_schema_is_accepted_and_a_newer_one_refused(self): + for storage_version in (1, 2): + with self.subTest(storage_version=storage_version): + self.assertEqual( + ledger(storage_version, True, 2).verify_candidate_schema(), + {"declared": 2, "has_body_table": True}, + ) + with self.assertRaisesRegex(IncompatibleCandidateSchema, "rollback floor"): + ledger(storage_version, True, 3).verify_candidate_schema() + + +class RecoverySchemaRefusalTests(unittest.TestCase): + def test_reader_applies_the_ledger_rule(self): + for has_body_table, declared in UNMIGRATED: + with self.subTest(has_body_table=has_body_table), \ + self.assertRaisesRegex(recovery.RecoveryError, "002_candidate_bodies.sql"): + reader(has_body_table, declared).require_candidate_schema() + with self.assertRaisesRegex(recovery.RecoveryError, "rollback floor"): + reader(True, 3).require_candidate_schema() + reader(True, 2).require_candidate_schema() + + def test_plan_refuses_an_unmigrated_database_before_reading_candidates(self): + legacy = reader(False, None) + legacy.metadata = Mock() + rpc = SimpleNamespace(call=Mock()) + with self.assertRaisesRegex(recovery.RecoveryError, "002_candidate_bodies.sql"): + recovery.plan_recovery(legacy, rpc, ["ab" * 32]) + legacy.metadata.assert_not_called() + rpc.call.assert_not_called() + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_prism_candidate_spool_integration.py b/tests/test_prism_candidate_spool_integration.py new file mode 100644 index 000000000..30cbed82a --- /dev/null +++ b/tests/test_prism_candidate_spool_integration.py @@ -0,0 +1,80 @@ +"""Lazy candidate fields remain usable by the replay and membership callers.""" + +import gc +import json +from pathlib import Path +import tempfile +import hashlib +import unittest +from unittest import mock +import weakref + +from lab.prism.block_candidates import block_candidate_from_intent +from lab.prism.candidate_codec import prepare_candidate_intent, prepared_intent_from_spool, write_spool_from_body +from lab.prism.candidate_window import copy_text_chunks, recorded_share_ids +from tests.prism_postgres_candidate_gate import intent + + +class CandidateSpoolIntegrationTests(unittest.TestCase): + def test_metadata_aggregate_and_giant_key_keep_native_decodes_bounded(self): + fields = intent("cd" * 32, 1, credit=False) + fields.update({f"extra_{index}": "x" * 24000 for index in range(60)}) + fields["found_block"]["k" * (2 * 1024 * 1024)] = True + prepared = prepare_candidate_intent(fields) + loads = json.loads + sizes = [] + + def bounded_loads(data, *args, **kwargs): + sizes.append(len(data)) + return loads(data, *args, **kwargs) + + with tempfile.TemporaryDirectory() as directory: + body = write_spool_from_body(str(Path(directory) / "body"), str(Path(directory) / "index"), prepared.body) + try: + with mock.patch("json.loads", side_effect=bounded_loads): + hydrated = prepared_intent_from_spool(body, accepted_at_present=True, accepted_at_ms=123) + self.assertEqual(hydrated["found_block"]["network_difficulty"], 1000) + self.assertEqual(hydrated["extra_59"], fields["extra_59"]) + self.assertLessEqual(max(sizes), 1024 * 1024) + finally: + body.close() + + def test_large_block_and_share_identifier_survive_replay_without_coercion(self): + fields = intent("ab" * 32, 3, credit=False) + fields["block_hex"] = "ab" * (1024 * 1024) + fields["shares_json"][0]["share_id"] = "id" * (1024 * 1024) + prepared = prepare_candidate_intent(fields) + enabled = gc.isenabled() + with tempfile.TemporaryDirectory() as directory: + gc.disable() + try: + body = write_spool_from_body(str(Path(directory) / "body"), str(Path(directory) / "index"), prepared.body) + observed = weakref.ref(body) + hydrated = prepared_intent_from_spool( + body, accepted_at_present=prepared.accepted_at_present, + accepted_at_ms=prepared.accepted_at_ms, + ) + candidate = block_candidate_from_intent(hydrated) + self.assertTrue(callable(getattr(candidate.submission.block_hex, "iter_byte_chunks", None))) + self.assertEqual(candidate.submission.block_hex.byte_length, len(fields["block_hex"]) + 2) + # A changed hydrated intent must re-encode its lazy fields, + # rather than accidentally reusing the original digest/body. + hydrated["username"] = "changed" + updated = prepare_candidate_intent(hydrated) + from tests.test_prism_candidate_codec import oracle_bytes + fields["username"] = "changed" + self.assertEqual(updated.candidate_sha256, hashlib.sha256(oracle_bytes(fields)).hexdigest()) + del updated + copied = b"".join(copy_text_chunks(recorded_share_ids(candidate.context.shares_json))) + expected = b"".join(copy_text_chunks(recorded_share_ids(fields["shares_json"]))) + self.assertEqual(copied, expected) + del candidate, hydrated, body + self.assertIsNone(observed(), "replay's caller graph retained the body without cyclic GC") + self.assertEqual(list(Path(directory).iterdir()), []) + finally: + if enabled: + gc.enable() + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_prism_candidate_spool_view.py b/tests/test_prism_candidate_spool_view.py new file mode 100644 index 000000000..72bcc1032 --- /dev/null +++ b/tests/test_prism_candidate_spool_view.py @@ -0,0 +1,845 @@ +#!/usr/bin/env python3 +"""Bounded candidate spool hydration regressions (issue #255). + +Every C call the reader makes is instrumented and asserted against the +view module's hard threshold, for a body carrying an oversized share +record, an oversized nested array and oversized top-level strings. The +remaining tests pin the adapter contract root builds on: streamed strings +that never split an escape or a surrogate pair, mapping/sequence views +that compare with plain values, verbatim byte access that preserves every +digest, random access through the disk index, legacy hinted pages, failure +classification (corrupt versus closed versus resource pressure), ownership +under retained exceptions with the cyclic collector disabled, and the +supervised isolated-record helper. +""" + +from __future__ import annotations + +import contextlib +import gc +import hashlib +import json +import os +import shutil +import subprocess +import sys +import tempfile +import threading +import time +import unittest +from dataclasses import replace +from pathlib import Path +from typing import Any +from unittest import mock + +sys.path.insert(0, str(Path(__file__).resolve().parents[1])) + +from lab.prism import audit_bundle_view as audit # noqa: E402 +from lab.prism import candidate_codec as codec # noqa: E402 +from lab.prism import candidate_spool_view as views # noqa: E402 +from lab.prism.share_ledger import block_candidate_identity # noqa: E402 + +HASH_A = "aa" * 32 +PARENT = "bb" * 32 +COMPACT = (",", ":") + + +def canonical(value: Any) -> bytes: + return json.dumps(value, sort_keys=True, separators=COMPACT).encode("utf-8") + + +def oracle_bytes(intent: dict[str, Any]) -> bytes: + identity = block_candidate_identity({**intent, "block_hash_hex": intent["block_hash_hex"].lower()}) + return canonical(identity) + + +def share(index: int) -> dict[str, Any]: + return { + "share_seq": index, + "share_id": f"miner-{index % 7}:é\U0001F600{'x' * (index % 3)}", + "miner_id": f"m{index % 7}", + "order_key": "k", + "p2mr_program_hex": "ab" * 32, + "share_difficulty": 10**30 + index, + "network_difficulty": 1.5e16 if index % 5 else 3, + "template_height": 5, + "job_id": 'j"q\\', + "job_issued_at_ms": 1_700_000_000_000 + index, + "accepted_at_ms": 1_700_000_000_001 + index, + "ntime": 1_700_000_000, + "credit_policy": None if index % 2 else "stale-grace", + } + + +def intent_for(share_count: int, **overrides: Any) -> dict[str, Any]: + fields: dict[str, Any] = { + "schema": codec.CANDIDATE_INTENT_SCHEMA, + "block_hash_hex": HASH_A.upper(), + "block_hex": "00" * 100, + "coinbase_tx_hex": "01", + "parent_hash": PARENT, + "expected_height": 10, + "template": {"previousblockhash": PARENT, "height": 10, "coinbasevalue": 5}, + "shares_json": [share(index) for index in range(share_count)], + "prior_balances": [{"recipient_id": "a", "balance_sats": 1}], + "found_block": {"network_difficulty": 1.0, "witness": [1, 2, {"z": None}]}, + "prospective_prior_balances": None, + "witness_merkle_leaves_hex": ["cc" * 32] * 3, + "extranonce1_hex": "00", + "extranonce2_hex": "01", + "username": "u", + "pending_share": {"share_id": "s", "accepted_at_ms": 123}, + "credit_share_on_accept": False, + "collection_only": False, + } + fields.update(overrides) + return fields + + +class _LoadsProbe: + """Record the input and output size of every ``json.loads`` call.""" + + def __init__(self) -> None: + self.inputs: list[int] = [] + self.outputs: list[int] = [] + self._real = json.loads + + def loads(self, data: Any, *args: Any, **kwargs: Any) -> Any: + value = self._real(data, *args, **kwargs) + self.inputs.append(len(data)) + if isinstance(value, (str, list, tuple, dict)): + self.outputs.append(len(value)) + else: + self.outputs.append(1) + return value + + +class SpoolFixture(unittest.TestCase): + def bounded(self, threshold: int) -> Any: + """Lower the view threshold for a test, keeping the invariants. + + The encoder spans a top-level field only above its fast-path + ceiling and the reader slices strings below the threshold, so both + knobs follow the threshold down; in production the ordering is the + same (64 KiB fast path, 64 KiB slices, 1 MiB threshold). + """ + stack = contextlib.ExitStack() + stack.enter_context(mock.patch.object(views, "SPOOL_VIEW_DECODE_BYTES", threshold)) + stack.enter_context(mock.patch.object(codec, "CODEC_FAST_PATH_BYTES", min(codec.CODEC_FAST_PATH_BYTES, threshold))) + stack.enter_context(mock.patch.object(views, "SPOOL_VIEW_STRING_SLICE_BYTES", min(views.SPOOL_VIEW_STRING_SLICE_BYTES, threshold))) + return stack + + def spool(self, fields: dict[str, Any], *, chunk_bytes: int = 4096) -> tuple[codec.PreparedCandidateIntent, codec.SpoolCandidateBody]: + prepared = codec.prepare_candidate_intent(fields, chunk_bytes=chunk_bytes) + directory = tempfile.mkdtemp() + self.addCleanup(shutil.rmtree, directory, True) + body = codec.write_spool_from_body( + os.path.join(directory, "b.body"), os.path.join(directory, "b.idx"), prepared.body + ) + self.addCleanup(body.close) + return prepared, body + + def hydrate(self, fields: dict[str, Any], **kwargs: Any) -> tuple[codec.PreparedCandidateIntent, codec.SpoolCandidateBody]: + _prepared, body = self.spool(fields, **kwargs) + hydrated = codec.prepared_intent_from_spool(body, accepted_at_present=True, accepted_at_ms=123) + return hydrated, body + + +class BoundednessTests(SpoolFixture): + def test_every_reader_c_call_is_bounded_by_the_threshold(self) -> None: + threshold = 32 * 1024 + big = ("é\"\\" + "y" * 500 + "\U0001F600") * 400 # ~200 KiB encoded + fields = intent_for(700, username=big, block_hex="ab" * 40_000, coinbase_tx_hex="cd" * 40_000) + fields["shares_json"][300]["share_id"] = "s" * (6 * threshold) + fields["shares_json"][300]["nested"] = [{"k": i, "v": "w" * 100} for i in range(2000)] + fields["found_block"]["witness"] = [[i, i * 1.5, f"leaf-{i}"] for i in range(20_000)] + fields["pending_share"] = {"share_id": "p" * (3 * threshold), "accepted_at_ms": 123, "miner_id": "m"} + probe = _LoadsProbe() + reads: list[int] = [] + real_read = codec.SpoolCandidateBody.read_span + + def read_span(self_: Any, start: int, end: int) -> bytes: + reads.append(end - start) + return real_read(self_, start, end) + + with self.bounded(threshold), \ + mock.patch.object(codec.json, "loads", probe.loads), \ + mock.patch.object(views.json, "loads", probe.loads), \ + mock.patch.object(codec.SpoolCandidateBody, "read_span", read_span): + hydrated, _body = self.hydrate(fields) + # Touch everything: every share, every member of the oversized + # record, every top-level fact, every streamed string. + self.assertEqual(len(list(hydrated["shares_json"])), 700) + record = hydrated["shares_json"][300] + self.assertIsInstance(record, views.SpoolObjectView) + self.assertEqual(dict(record).keys(), fields["shares_json"][300].keys()) + self.assertEqual(record["share_id"], fields["shares_json"][300]["share_id"]) + self.assertEqual(list(record["nested"]), fields["shares_json"][300]["nested"]) + self.assertIsInstance(hydrated["username"], views.SpoolStringView) + self.assertEqual("".join(hydrated["username"].iter_text_chunks()), big) + self.assertIsInstance(hydrated["block_hex"], views.SpoolStringView) + self.assertEqual(hydrated["block_hex"], "ab" * 40_000) + self.assertEqual(hydrated["coinbase_tx_hex"], "cd" * 40_000) + self.assertIsInstance(hydrated["coinbase_tx_hex"], str) + witness = hydrated["found_block"]["witness"] + self.assertIsInstance(witness, views.SpoolArrayView) + self.assertEqual(len(witness), 20_000) + self.assertEqual(list(witness), fields["found_block"]["witness"]) + self.assertEqual(witness[19_999], fields["found_block"]["witness"][19_999]) + self.assertEqual(hydrated["pending_share"]["accepted_at_ms"], 123) + self.assertEqual(hydrated["pending_share"]["share_id"], "p" * (3 * threshold)) + header = hydrated.replay_header() + self.assertTrue(header["oversized"]) + self.assertIsNone(header["username"]) + self.assertEqual(header["pending_share"]["accepted_at_ms"], 123) + self.assertTrue(probe.inputs) + # The skeleton is the one call bounded by structure rather than the + # threshold: every non-spanned top-level field is under the encoder's + # fast-path ceiling, and this intent has twenty of them. + skeleton_bound = 20 * threshold + self.assertLessEqual(max(probe.inputs), max(threshold + 2, skeleton_bound)) + self.assertLessEqual(sorted(probe.inputs)[-2], threshold + 2) + self.assertLessEqual(max(reads), max(threshold, views.SPOOL_VIEW_READ_BYTES)) + string_outputs = [size for size, inp in zip(probe.outputs, probe.inputs) if inp <= views.SPOOL_VIEW_STRING_SLICE_BYTES + 16] + self.assertTrue(string_outputs) + self.assertLessEqual(max(probe.outputs), max(threshold, codec.CODEC_BATCH_RECORDS * 4)) + + def test_share_id_above_the_page_decode_ceiling_is_a_streamed_string(self) -> None: + # The exact defect: one record over CODEC_PAGE_DECODE_BYTES made the + # page "not valid JSON". It is now an object view whose share_id + # streams; every other record on the page stays a plain dict. + giant = "g" * (codec.CODEC_PAGE_DECODE_BYTES + 1024 * 1024) + fields = intent_for(600) + fields["shares_json"][300]["share_id"] = giant + probe = _LoadsProbe() + with mock.patch.object(codec.json, "loads", probe.loads), mock.patch.object(views.json, "loads", probe.loads): + hydrated, body = self.hydrate(fields, chunk_bytes=codec.CANDIDATE_BODY_CHUNK_BYTES) + shares = hydrated["shares_json"] + record = shares[300] + self.assertIsInstance(record, views.SpoolObjectView) + self.assertIsInstance(record["share_id"], views.SpoolStringView) + self.assertEqual(record["share_id"].byte_length, len(giant) + 2) + self.assertEqual(record["share_id"].text_length(), len(giant)) + self.assertTrue(record["share_id"] == giant) + self.assertEqual(record["share_seq"], 300) + self.assertEqual(record, fields["shares_json"][300]) + self.assertIsInstance(shares[299], dict) + self.assertIsInstance(shares[301], dict) + self.assertEqual(shares[299], fields["shares_json"][299]) + self.assertEqual(shares[599], fields["shares_json"][599]) + self.assertLessEqual(max(probe.inputs), views.SPOOL_VIEW_DECODE_BYTES + 2) + digest = hashlib.sha256() + body.write_chunks(lambda chunk: digest.update(chunk.data)) + self.assertEqual(digest.hexdigest(), hashlib.sha256(oracle_bytes(fields)).hexdigest()) + + +class RoundTripTests(SpoolFixture): + def test_small_and_unicode_strings_round_trip_at_every_slice_cut(self) -> None: + unit = "\U0001F600" * 5 + "\\" * 7 + 'q"' + "é" + "\U0001F600" + "\\\\u0041" + "\n\t" + text = unit * 40 + fields = intent_for(1, username=text, block_hex="0" * 10) + with self.bounded(256): + hydrated, _body = self.hydrate(fields) + view = hydrated["username"] + self.assertIsInstance(view, views.SpoolStringView) + encoded = json.dumps(text) + for slice_bytes in range(32, 97): + with self.subTest(slice_bytes=slice_bytes): + chunks = list(view.iter_text_chunks(slice_bytes=slice_bytes)) + self.assertEqual("".join(chunks), text) + self.assertTrue(all(chunks)) + self.assertEqual(view, text) + self.assertNotEqual(view, text + "x") + self.assertTrue(text == view) + self.assertEqual("".join(view.iter_encoded_chunks()), encoded) + self.assertEqual(b"".join(view.iter_byte_chunks(chunk_bytes=7)), encoded.encode("ascii")) + self.assertEqual(view.canonical_json_sha256(), hashlib.sha256(encoded.encode()).hexdigest()) + self.assertEqual(views.materialize_spool_views(view), text) + self.assertEqual("".join(views.iter_string_text_chunks(view)), text) + self.assertEqual(list(views.iter_string_text_chunks("plain")), ["plain"]) + with self.assertRaises(TypeError): + str(view) + self.assertIn("SpoolStringView", repr(view)) + self.assertTrue(view) + # Small strings are plain values with the historical shape. + self.assertEqual(hydrated["block_hex"], "0" * 10) + self.assertIsInstance(hydrated["block_hex"], str) + self.assertEqual(hydrated["shares_json"][0]["share_id"], share(0)["share_id"]) + + def test_raw_utf8_and_split_numbers_survive_tiny_reads(self) -> None: + # A body written with raw UTF-8 (a legacy encoder that did not + # escape) and read five bytes at a time: no code point, escape or + # number token may be split by a read boundary. + witness = [ + 1, -2, 3.5e-7, 12345678901234567890, -0.0, 1e300, True, None, + 'trap},{"k":1', "\\\\\"", "☃ snow \U0001F600", {"n": [1.25, "x", {"deep": [[], {}]}]}, [], {}, + ] * 40 + fields = intent_for(1, found_block={"network_difficulty": 1.0, "witness": witness}) + _prepared, body = self.spool(fields) + raw_path = body.path + ".raw" + text = json.loads(body.read_span(0, body.manifest.byte_count).decode("ascii")) + raw = json.dumps(text, sort_keys=True, separators=COMPACT, ensure_ascii=False).encode("utf-8") + with open(raw_path, "wb") as handle: + handle.write(raw) + self.addCleanup(lambda: os.path.exists(raw_path) and os.unlink(raw_path)) + start = raw.index(b'"found_block":{') + len(b'"found_block":') + end = raw.index(b',"parent_hash"') + raw_body = mock.Mock(spec=codec.SpoolCandidateBody) + raw_body.alive = True + raw_body.read_span = lambda s, e: raw[s:e] + with mock.patch.object(views, "SPOOL_VIEW_READ_BYTES", 5), \ + mock.patch.object(views, "SPOOL_VIEW_STRING_SLICE_BYTES", 33), \ + self.bounded(40): + found = views.decode_body_span(raw_body, start, end) + self.assertIsInstance(found, views.SpoolObjectView) + items = found["witness"] + self.assertIsInstance(items, views.SpoolArrayView) + self.assertEqual(len(items), len(witness)) + self.assertEqual(list(items), witness) + self.assertEqual(items[10], witness[10]) + self.assertEqual(items[-1], witness[-1]) + self.assertEqual(items[3:6], tuple(witness[3:6])) + self.assertEqual(items, witness) + self.assertEqual(found, fields["found_block"]) + self.assertEqual("".join(items.iter_encoded_chunks()), json.dumps(witness, sort_keys=True, separators=COMPACT, ensure_ascii=False)) + + def test_random_access_and_slicing_through_the_disk_index(self) -> None: + records = [share(index) for index in range(1300)] + records[700]["share_id"] = 'trap},{"accepted_at_ms":1' + records[511]["nested"] = [{"a": 1}, {"b": "\\\"},{"}] + records[900]["share_id"] = "big" * 30_000 + fields = intent_for(0, shares_json=records) + with self.bounded(16 * 1024): + hydrated, _body = self.hydrate(fields) + shares = hydrated["shares_json"] + self.assertIsInstance(shares, codec.SpoolJsonArraySequence) + self.assertEqual(len(shares), 1300) + for index in (0, 1, 255, 256, 511, 700, 899, 900, 901, 1299, -1, -300): + with self.subTest(index=index): + self.assertEqual(shares[index], records[index]) + self.assertEqual(shares[898:903], tuple(records[898:903])) + self.assertEqual(shares[::400], tuple(records[::400])) + with self.assertRaises(IndexError): + shares[1300] + with self.assertRaises(TypeError): + shares["x"] + self.assertIsInstance(shares[900], views.SpoolObjectView) + self.assertIsInstance(shares[900]["share_id"], views.SpoolStringView) + self.assertEqual(list(shares), records) + self.assertTrue(shares == records) + self.assertFalse(shares == records[:-1]) + expected_array = canonical(records) + self.assertEqual(shares.canonical_json_sha256(), hashlib.sha256(expected_array).hexdigest()) + self.assertEqual(b"".join(shares.iter_byte_chunks()), expected_array) + staged = bytearray(b"[") + for data, _count in shares.canonical_share_pages(): + if staged[-1:] != b"[" and staged[-1:] != b",": + staged += b"," + staged += data + staged += b"]" + self.assertEqual(json.loads(bytes(staged)), records) + + +class RestagingTests(SpoolFixture): + def test_hydrated_body_and_every_field_keep_their_bytes(self) -> None: + fields = intent_for(600, username="u" * 200_000, block_hex="ef" * 100_000) + fields["shares_json"][300]["share_id"] = "y" * 300_000 + fields["found_block"]["witness"] = [{"leaf": i} for i in range(30_000)] + fields["pending_share"] = {"share_id": "p" * 200_000, "accepted_at_ms": 123} + expected = oracle_bytes(fields) + with self.bounded(64 * 1024): + hydrated, body = self.hydrate(fields) + got = bytearray() + hydrated.body.write_chunks(lambda chunk: got.extend(chunk.data)) + self.assertEqual(bytes(got), expected) + self.assertIs(codec.prepare_candidate_intent(hydrated), hydrated) + for name in ("username", "block_hex"): + view = hydrated[name] + self.assertIsInstance(view, views.SpoolStringView) + self.assertEqual(b"".join(view.iter_byte_chunks()), canonical(fields[name])) + found = hydrated["found_block"] + self.assertIsInstance(found, views.SpoolObjectView) + self.assertEqual(b"".join(found.iter_byte_chunks()), canonical(fields["found_block"])) + self.assertEqual(found.canonical_json_sha256(), hashlib.sha256(canonical(fields["found_block"])).hexdigest()) + self.assertEqual(b"".join(found["witness"].iter_byte_chunks()), canonical(fields["found_block"]["witness"])) + record = hydrated["shares_json"][300] + self.assertEqual(b"".join(record.iter_byte_chunks()), canonical(fields["shares_json"][300])) + self.assertEqual(b"".join(record["share_id"].iter_byte_chunks()), canonical(fields["shares_json"][300]["share_id"])) + # Re-staging the hydrated body to a second spool reproduces it + # byte for byte, oversized page and all, and hydrates equal. + directory = tempfile.mkdtemp() + self.addCleanup(shutil.rmtree, directory, True) + second = codec.write_spool_from_body(os.path.join(directory, "c.body"), os.path.join(directory, "c.idx"), hydrated.body) + self.addCleanup(second.close) + self.assertEqual(second.manifest, body.manifest) + again = codec.prepared_intent_from_spool(second, accepted_at_present=True, accepted_at_ms=123) + self.assertEqual(again["shares_json"][300], fields["shares_json"][300]) + self.assertEqual(again["found_block"], fields["found_block"]) + # The shares sequence re-encodes through the verbatim page + # protocol, streaming the oversized page as raw pieces. + restaged = codec.prepare_candidate_intent(intent_for(0, shares_json=hydrated["shares_json"]), chunk_bytes=4096) + regot = bytearray() + restaged.body.write_chunks(lambda chunk: regot.extend(chunk.data)) + self.assertEqual(bytes(regot), oracle_bytes(intent_for(0, shares_json=fields["shares_json"]))) + + def test_streaming_encoder_and_materializer_accept_spool_views(self) -> None: + fields = intent_for(2, username="w" * 5000, found_block={"network_difficulty": 1.0, "witness": list(range(3000))}) + with self.bounded(1024): + hydrated, _body = self.hydrate(fields) + username = hydrated["username"] + found = hydrated["found_block"] + self.assertIsInstance(username, views.SpoolStringView) + self.assertIsInstance(found, views.SpoolObjectView) + payload = {"u": username, "f": found, "w": found["witness"], "n": 1} + plain = {"u": fields["username"], "f": fields["found_block"], "w": fields["found_block"]["witness"], "n": 1} + self.assertEqual("".join(audit.iter_json_chunks(payload, sort_keys=True)), json.dumps(plain, sort_keys=True, separators=COMPACT)) + self.assertEqual("".join(audit.iter_json_chunks(payload)), json.dumps(plain, separators=COMPACT)) + self.assertEqual(audit.materialize_json(payload), plain) + self.assertEqual(views.materialize_spool_views(payload), plain) + self.assertEqual(audit.streamed_sha256_json_hex(payload), hashlib.sha256(canonical(plain)).hexdigest()) + self.assertTrue(views.is_spool_view(username)) + self.assertFalse(views.is_spool_view("x")) + + +class MappingViewTests(SpoolFixture): + def test_object_view_mapping_protocol_and_pending_stamp(self) -> None: + members: dict[str, Any] = {f"k{i:04d}": {"v": i, "s": "x" * 50} for i in range(600)} + members["share_id"] = "S" * 5000 + members["accepted_at_ms"] = 123 + # The body stores keys sorted and the header projection keeps at + # most 32 members (the historical shape), so the pending share stays + # small in members while its share_id alone crosses the threshold. + pending = {"share_id": "S" * 5000, "accepted_at_ms": 123, "miner_id": "m"} + fields = intent_for(1, pending_share=pending, found_block=dict(members)) + with self.bounded(2048), \ + mock.patch.object(views, "SPOOL_VIEW_INDEX_MEMORY_ENTRIES", 8): + hydrated, _body = self.hydrate(fields) + pending = hydrated["pending_share"] + self.assertIsInstance(pending, dict) + self.assertEqual(pending["accepted_at_ms"], 123) + self.assertIsInstance(pending["share_id"], views.SpoolStringView) + self.assertEqual(pending["share_id"], "S" * 5000) + self.assertEqual(pending["miner_id"], "m") + self.assertEqual(set(pending), {"share_id", "accepted_at_ms", "miner_id"}) + header = hydrated.replay_header() + self.assertTrue(header["oversized"]) + self.assertEqual(header["pending_share"]["accepted_at_ms"], 123) + self.assertIsNone(header["pending_share"]["share_id"]) + self.assertTrue(header["accepted_at_present"]) + found = hydrated["found_block"] + self.assertIsInstance(found, views.SpoolObjectView) + # The body stores keys sorted, as the historical json.loads did. + ordered = sorted(members) + self.assertEqual(len(found), len(members)) + self.assertEqual(list(found), ordered) + self.assertEqual(list(found.keys()), ordered) + self.assertEqual([key for key, _ in found.items()], ordered) + self.assertEqual(list(found.values())[1], members[ordered[1]]) + self.assertEqual(found.get("absent", "d"), "d") + self.assertIn("k0599", found) + self.assertNotIn("nope", found) + self.assertNotIn(5, found) + with self.assertRaises(KeyError): + found["nope"] + self.assertEqual(found, members) + self.assertTrue(members == found) + self.assertNotEqual(found, {**members, "extra": 1}) + self.assertNotEqual(found, {**members, "k0000": 0}) + self.assertEqual(dict(found), members) + self.assertEqual(views.materialize_spool_views(found), members) + self.assertTrue(found._ensure_indexed().spilled) + self.assertIn("members=602", repr(found)) + found.close() + with self.assertRaises(codec.CandidateBodyIntegrityError): + found["k0001"] + # Without the key cache (too many members) lookups scan keys. + with mock.patch.object(views, "SPOOL_VIEW_KEY_CACHE_ENTRIES", 4): + again = views.SpoolObjectView(found.body, *found.byte_range) + self.assertEqual(again["k0450"], members["k0450"]) + self.assertIsNone(again._keys) + self.assertEqual(again["share_id"], members["share_id"]) + with self.assertRaises(KeyError): + again["missing"] + + def test_decoded_metadata_boundary_and_lazy_block(self) -> None: + fields = intent_for(1, coinbase_tx_hex="ab" * 3000, block_hex="cd" * 3000, extranonce1_hex="ef" * 3000, username="u" * 6000) + with self.bounded(1024): + hydrated, _body = self.hydrate(fields) + for name in ("coinbase_tx_hex", "extranonce1_hex"): + self.assertIsInstance(hydrated[name], str) + self.assertEqual(hydrated[name], fields[name]) + self.assertIsInstance(hydrated["block_hex"], views.SpoolStringView) + self.assertEqual(hydrated["block_hex"].byte_length, len(fields["block_hex"]) + 2) + self.assertEqual(b"".join(hydrated["block_hex"].iter_byte_chunks()), canonical(fields["block_hex"])) + self.assertIsInstance(hydrated["username"], views.SpoolStringView) + with self.assertRaises(TypeError) as caught: + str(hydrated["username"]) + self.assertNotIsInstance(caught.exception, codec.CandidateBodyIntegrityError) + self.assertEqual(codec.decode_json_string_span(hydrated.body, hydrated.body.index.span("username")), fields["username"]) + self.assertIn("block_hex", hydrated) + self.assertEqual(hydrated.block_hash, HASH_A) + + +class FailureTests(SpoolFixture): + def test_corrupt_truncated_missing_and_closed_bodies_are_classified(self) -> None: + fields = intent_for(300, username="u" * 20_000) + fields["found_block"]["witness"] = list(range(5000)) + with self.bounded(4096): + hydrated, body = self.hydrate(fields) + username = hydrated["username"] + witness = hydrated["found_block"]["witness"] + self.assertEqual(username, fields["username"]) + self.assertEqual(len(witness), 5000) + span = body.index.span("username") + # An invalid escape inside the streamed string: corruption. + with open(body.path, "r+b") as handle: + handle.seek(span.start + 10) + handle.write(b"\\x") + with self.assertRaises(codec.CandidateBodyIntegrityError): + list(username.iter_text_chunks()) + with self.assertRaises(codec.CandidateBodyIntegrityError): + list(body.iter_chunks()) + # Unbalanced delimiters in a lazy array: corruption. + witness_span = body.index.span("found_block") + with open(body.path, "r+b") as handle: + handle.seek(witness_span.start + 20) + handle.write(b"]") + fresh = views.SpoolObjectView(body, witness_span.start, witness_span.end) + with self.assertRaises(codec.CandidateBodyIntegrityError): + fresh["witness"] + # A page index that lies about its span: corruption, not a crash. + with self.assertRaises(codec.CandidateBodyIntegrityError): + codec.SpoolJsonArraySequence(body, replace(body.index.span("shares_json"), start=10**9))[0] + # Truncation inside the share array: the read is shorter than + # the manifest (username follows the shares in key order). + with open(body.path, "r+b") as handle: + handle.truncate(body.index.span("shares_json").start + 100) + with self.assertRaises(codec.CandidateBodyIntegrityError): + hydrated["shares_json"][299] + with self.assertRaises(codec.CandidateBodyIntegrityError): + b"".join(username.iter_byte_chunks()) + # A vanished index file: corruption-class, never a bare OSError. + os.unlink(body.index.path) + with self.assertRaises(codec.CandidateBodyIntegrityError): + hydrated["shares_json"][0] + # A released body: every view fails closed. + hydrated.release() + self.assertFalse(body.alive) + with self.assertRaises(codec.CandidateBodyIntegrityError): + username.iter_text_chunks().__next__() + with self.assertRaises(codec.CandidateBodyIntegrityError): + views.SpoolArrayView(body, witness_span.start, witness_span.end)[0] + with self.assertRaises(codec.CandidateBodyIntegrityError): + hydrated["shares_json"][1] + + def test_mismatched_delimiters_and_empty_values_are_corruption(self) -> None: + raw = b'{"a":[1,2}' + fake = mock.Mock(spec=codec.SpoolCandidateBody) + fake.alive = True + fake.read_span = lambda s, e: raw[s:e] + with self.bounded(2): + with self.assertRaises(codec.CandidateBodyIntegrityError): + views.decode_body_span(fake, 5, 10) + with self.assertRaises(codec.CandidateBodyIntegrityError): + views.decode_body_span(fake, 5, 5) + with self.assertRaises(codec.CandidateBodyIntegrityError): + list(views.iter_member_spans(fake, 1, 9)) + with self.assertRaises(codec.CandidateBodyIntegrityError): + list(views.iter_json_string_text_chunks(fake, 0, 4)) + + def test_legacy_hinted_pages_stay_readable_ignoring_bad_hints(self) -> None: + records = [share(index) for index in range(900)] + records[400]["share_id"] = 'trap},{"share_seq":9' * 20 + records[650]["nested"] = [{"b": "\\\"},{"}] * 3 + fields = intent_for(0, shares_json=records) + _prepared, body = self.spool(fields) + span = body.index.span("shares_json") + data = body.read_span(span.start, span.end) + # A legacy index whose hints came from a substring scan: one lands + # inside a quoted trap, one inside a nested object. + directory = tempfile.mkdtemp() + self.addCleanup(shutil.rmtree, directory, True) + legacy_path = os.path.join(directory, "legacy.body") + shutil.copyfile(body.path, legacy_path) + index = codec.SpoolFieldIndex.create(os.path.join(directory, "legacy.idx")) + hints = [span.start + 1] + hints.append(span.start + data.index(b"trap},{") + 5) + hints.append(span.start + data.index(b'\\"},{"') + 4) + hinted = replace(span, page_count=len(hints), pages_exact=False) + index.begin_field(hinted) + index.append_pages("shares_json", [(offset, 0) for offset in hints]) + legacy = codec.SpoolCandidateBody(legacy_path, body.manifest, index) + self.addCleanup(legacy.close) + sequence = codec.SpoolJsonArraySequence(legacy, hinted) + with self.bounded(8192): + self.assertEqual(sequence[400], records[400]) + self.assertEqual(sequence[899], records[899]) + self.assertEqual(sequence[0], records[0]) + self.assertEqual(list(sequence), records) + self.assertFalse(legacy.index.span("shares_json").pages_exact) + self.assertIsNotNone(sequence._walk_index) + staged = codec.prepare_candidate_intent(intent_for(0, shares_json=sequence), chunk_bytes=4096) + self.assertEqual(staged.candidate_sha256, hashlib.sha256(oracle_bytes(fields)).hexdigest()) + sequence.close() + + +class OwnershipTests(SpoolFixture): + def test_scratch_indices_close_on_error_and_by_refcount_with_gc_disabled(self) -> None: + fields = intent_for(1, found_block={"network_difficulty": 1.0, "witness": list(range(4000))}) + created: list[Any] = [] + real_temporary_file = tempfile.TemporaryFile + + def recording_temporary_file(*args: Any, **kwargs: Any) -> Any: + handle = real_temporary_file(*args, **kwargs) + created.append(handle) + return handle + + gc_was_enabled = gc.isenabled() + gc.disable() + self.addCleanup(lambda: gc.enable() if gc_was_enabled else None) + with self.bounded(512), \ + mock.patch.object(views, "SPOOL_VIEW_INDEX_MEMORY_ENTRIES", 4), \ + mock.patch.object(views.tempfile, "TemporaryFile", recording_temporary_file): + hydrated, body = self.hydrate(fields) + found = hydrated["found_block"] + # Hydration may spill its aggregate metadata index first. + self.assertTrue(all(handle.closed for handle in created)) + created.clear() + # 1. An error while the index is being built closes its scratch + # file before the exception leaves, even though the retained + # exception keeps the frame (and the index) alive. + real_spans = views.iter_item_spans + + def failing_spans(*args: Any, **kwargs: Any) -> Any: + for count, item in enumerate(real_spans(*args, **kwargs)): + if count == 50: + raise OSError("disk went away") + yield item + + retained: list[BaseException] = [] + with mock.patch.object(views, "iter_item_spans", failing_spans): + try: + len(found["witness"]) + except OSError as exc: + retained.append(exc) + self.assertEqual(len(retained), 1) + self.assertEqual(len(created), 1) + self.assertTrue(created[0].closed) + # 2. A successful index spills to a scratch file that closes when + # the last view referencing it is dropped -- by reference + # count alone, since no view takes part in a cycle. + witness = found["witness"] + self.assertEqual(len(witness), 4000) + self.assertEqual(len(created), 2) + self.assertFalse(created[1].closed) + fd = created[1].fileno() + os.fstat(fd) + del witness + self.assertTrue(created[1].closed) + with self.assertRaises(OSError): + os.fstat(fd) + # 3. Resource pressure while spilling is retryable, and leaves + # no descriptor behind. + with mock.patch.object(views.tempfile, "TemporaryFile", side_effect=OSError("no temp")): + with self.assertRaises(audit.ArtifactResourcePressure) as pressure: + len(views.SpoolArrayView(body, *found["witness"].byte_range)) + self.assertNotIsInstance(pressure.exception, codec.CandidateBodyIntegrityError) + self.assertEqual(len(created), 2) + # 4. Explicit release: the spool pair is unlinked while a view + # (kept alive by a retained traceback) still exists, and the + # view fails closed instead of reading a stale file. + witness = found["witness"] + paths = (body.path, body.index.path) + try: + raise RuntimeError("retained while views are alive") + except RuntimeError as exc: + retained.append(exc) + hydrated.release() + self.assertFalse(any(os.path.exists(path) for path in paths)) + with self.assertRaises(codec.CandidateBodyIntegrityError): + witness[0] + views.close_spool_views(hydrated.facts) + self.assertTrue(all(handle.closed for handle in created)) + del retained + + def test_hydration_failure_leaves_no_open_scratch(self) -> None: + # A corrupt member value inside a lazy pending_share: hydration's + # shallow copy fails after the (spilled) member index was built, the + # exception is retained, and the scratch file is closed anyway. + pending = {f"k{i:03d}": {"s": "x" * 20, "v": i} for i in range(40)} + pending["accepted_at_ms"] = 123 + fields = intent_for(2, pending_share=pending) + with self.bounded(512): + _prepared, body = self.spool(fields) + span = body.index.span("pending_share") + data = body.read_span(span.start, span.end) + with open(body.path, "r+b") as handle: + handle.seek(span.start + data.index(b'"v":7}') + 4) + handle.write(b"x") + created: list[Any] = [] + real_temporary_file = tempfile.TemporaryFile + + def recording(*args: Any, **kwargs: Any) -> Any: + handle = real_temporary_file(*args, **kwargs) + created.append(handle) + return handle + + with self.bounded(512), \ + mock.patch.object(views, "SPOOL_VIEW_INDEX_MEMORY_ENTRIES", 2), \ + mock.patch.object(views.tempfile, "TemporaryFile", recording): + retained: list[BaseException] = [] + try: + codec.prepared_intent_from_spool(body, accepted_at_present=True, accepted_at_ms=123) + except codec.CandidateBodyIntegrityError as exc: + retained.append(exc) + self.assertEqual(len(retained), 1) + self.assertTrue(created) + self.assertTrue(all(handle.closed for handle in created)) + del retained + + +class HelperSupervisionTests(unittest.TestCase): + def source(self, raw: bytes) -> audit.ArtifactSource: + directory = tempfile.mkdtemp() + self.addCleanup(shutil.rmtree, directory, True) + path = os.path.join(directory, "record.json") + with open(path, "wb") as handle: + handle.write(raw) + source = audit.ArtifactSource(os.open(path, os.O_RDONLY), path=Path(path)) + self.addCleanup(source.close) + return source + + def helper_threads(self) -> list[str]: + return [thread.name for thread in threading.enumerate() if thread.name.startswith("prism-audit-record-helper")] + + def test_hanging_helper_obeys_the_deadline_and_is_reaped(self) -> None: + raw = b'{"a":"' + b"x" * 5000 + b'"}' + source = self.source(raw) + real_popen = subprocess.Popen + children: list[subprocess.Popen[bytes]] = [] + + def sleeping_child(_args: Any, **kwargs: Any) -> Any: + process = real_popen([sys.executable, "-c", "import time; time.sleep(60)"], **kwargs) + children.append(process) + return process + + started = time.monotonic() + with mock.patch.object(audit.subprocess, "Popen", sleeping_child): + with self.assertRaises(audit.ArtifactResourcePressure): + audit.normalize_record_isolated(source, 0, len(raw), timeout_seconds=0.3) + self.assertLess(time.monotonic() - started, 5.0) + self.assertEqual(len(children), 1) + self.assertIsNotNone(children[0].poll()) + self.assertTrue(all(pipe.closed for pipe in (children[0].stdin, children[0].stdout, children[0].stderr))) + self.assertEqual(self.helper_threads(), []) + + def test_cancellation_abandons_the_helper(self) -> None: + raw = b'{"a":"' + b"x" * 5000 + b'"}' + source = self.source(raw) + real_popen = subprocess.Popen + children: list[subprocess.Popen[bytes]] = [] + calls = {"count": 0} + + class Cancelled(Exception): + pass + + def cancellation() -> None: + calls["count"] += 1 + if calls["count"] > 2: + raise Cancelled() + + def sleeping_child(_args: Any, **kwargs: Any) -> Any: + process = real_popen([sys.executable, "-c", "import time; time.sleep(60)"], **kwargs) + children.append(process) + return process + + with mock.patch.object(audit.subprocess, "Popen", sleeping_child): + with self.assertRaises(Cancelled): + audit.normalize_record_isolated(source, 0, len(raw), timeout_seconds=30.0, cancellation=cancellation) + self.assertIsNotNone(children[0].poll()) + self.assertEqual(self.helper_threads(), []) + + def test_admission_slots_are_shared_and_deadline_bounded(self) -> None: + raw = b'{"a":1}' + source = self.source(raw) + admission = audit.HelperAdmission(1) + admission.acquire(lambda: None) + try: + with mock.patch.object(audit.subprocess, "Popen") as spawn: + with self.assertRaises(audit.ArtifactResourcePressure): + audit.normalize_record_isolated(source, 0, len(raw), timeout_seconds=0.2, admission=admission) + spawn.assert_not_called() + finally: + admission.release() + # The slot is released after use, so a second run proceeds. + record = audit.normalize_record_isolated(source, 0, len(raw), admission=admission) + try: + self.assertEqual(record, {"a": 1}) + finally: + record.close() + admission.acquire(lambda: None) + admission.release() + + def test_helper_exit_statuses_and_noisy_diagnostics_are_classified(self) -> None: + raw = b'{"a":"' + b"x" * 100 + b'"}' + source = self.source(raw) + real_popen = subprocess.Popen + + def child(script: str) -> Any: + def spawn(_args: Any, **kwargs: Any) -> Any: + return real_popen([sys.executable, "-c", script], **kwargs) + return spawn + + noisy = "import sys; sys.stdin.read(); sys.stderr.write('e' * 1000000); sys.stderr.flush(); sys.exit(2)" + with mock.patch.object(audit.subprocess, "Popen", child(noisy)): + with self.assertRaises(json.JSONDecodeError) as malformed: + audit.normalize_record_isolated(source, 0, len(raw), timeout_seconds=30.0) + self.assertIsInstance(malformed.exception, audit.CanonicalArtifactSyntaxError) + self.assertLessEqual(len(str(malformed.exception)), audit.RAW_RECORD_HELPER_DIAGNOSTIC_BYTES + 100) + with mock.patch.object(audit.subprocess, "Popen", child("import sys; sys.stdin.read(); sys.exit(3)")): + with self.assertRaises(audit.ArtifactResourcePressure): + audit.normalize_record_isolated(source, 0, len(raw), timeout_seconds=30.0) + with mock.patch.object(audit.subprocess, "Popen", child("import sys; sys.stdin.read(); sys.stdout.write('not json\\n'); sys.stdout.flush()")): + with self.assertRaises(audit.ArtifactResourcePressure): + audit.normalize_record_isolated(source, 0, len(raw), timeout_seconds=30.0) + with mock.patch.object(audit.tempfile, "TemporaryFile", side_effect=OSError("disk full")): + with self.assertRaises(audit.ArtifactResourcePressure): + audit.normalize_record_isolated(source, 0, len(raw), timeout_seconds=30.0) + self.assertEqual(self.helper_threads(), []) + record = audit.normalize_record_isolated(source, 0, len(raw), member_limit=16) + try: + self.assertIn("a", record.omitted_members) + self.assertEqual(record, {"a": "x" * 100}) + finally: + record.close() + + def test_cancellation_reaches_lazy_audit_reads(self) -> None: + bundle = {"shares": [{"share_seq": i, "share_id": "s" * 3000} for i in range(40)]} + source_path = Path(tempfile.mkdtemp()) + self.addCleanup(shutil.rmtree, source_path, True) + path = source_path / "bundle.json" + path.write_bytes(json.dumps(bundle, separators=COMPACT).encode()) + state = {"cancel": False} + + class Cancelled(Exception): + pass + + def cancellation() -> None: + if state["cancel"]: + raise Cancelled() + + view = audit.CanonicalAuditBundleView.scan_path(path, cancellation=cancellation, chunk_bytes=1024) + try: + self.assertEqual(view["shares"][39]["share_seq"], 39) + state["cancel"] = True + with self.assertRaises(Cancelled): + view["shares"][39] + with mock.patch.object(audit, "RECORD_DECODE_SOFT_LIMIT_BYTES", 512): + with self.assertRaises(Cancelled): + view["shares"][3] + finally: + view.close() + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_prism_candidate_storage_races.py b/tests/test_prism_candidate_storage_races.py new file mode 100644 index 000000000..85bb026aa --- /dev/null +++ b/tests/test_prism_candidate_storage_races.py @@ -0,0 +1,252 @@ +"""Two-session PostgreSQL regressions for immutable candidate publication.""" + +from __future__ import annotations + +import os +import threading +import time +import unittest +from unittest import mock +import uuid + +from lab.prism.candidate_codec import prepare_candidate_intent +from lab.prism.candidate_store import retire_orphan_bodies_sql +from lab.prism.share_ledger import PsqlShareLedger +from tests.prism_postgres_candidate_gate import intent, pending_share_for + + +@unittest.skipUnless(os.environ.get("PRISM_TEST_DATABASE_URL"), "requires disposable PostgreSQL") +class CandidateStorageRaceTests(unittest.TestCase): + def setUp(self): + import psycopg + from psycopg.conninfo import make_conninfo + + self.admin = psycopg.connect(os.environ["PRISM_TEST_DATABASE_URL"], autocommit=True) + self.addCleanup(self.admin.close) + schema = "candidate_race_" + uuid.uuid4().hex + self.admin.execute(f'CREATE SCHEMA "{schema}"') + self.addCleanup(lambda: self.admin.execute(f'DROP SCHEMA "{schema}" CASCADE')) + url = make_conninfo(os.environ["PRISM_TEST_DATABASE_URL"], options=f"-csearch_path={schema}") + self.ledger = PsqlShareLedger( + database_url=url, psql_command="psql", native_client_mode="native", + writer_id="candidate-race", initialize_schema=True, + ) + self.addCleanup(self.ledger.close) + self.addCleanup(self.ledger.release_writer_lease) + self.first = psycopg.connect(url) + self.second = psycopg.connect(url) + self.addCleanup(self.first.close) + self.addCleanup(self.second.close) + self.prepared = prepare_candidate_intent(intent(uuid.uuid4().hex * 2, 4, credit=False)) + + def stage_without_sealing(self): + execute = self.ledger._run_json + captured = [] + + def capture(sql): + if "observed_chunk_count" in sql: + captured.append(sql) + return {"sealed": 1} + return execute(sql) + + with mock.patch.object(self.ledger, "_run_json", side_effect=capture): + body_id = self.ledger.stage_candidate_body(self.prepared) + self.assertEqual(len(captured), 1) + return body_id, captured[0] + + def publication(self): + body_id = self.ledger.stage_candidate_body(self.prepared) + with mock.patch.object(self.ledger, "stage_candidate_body", return_value=body_id), \ + mock.patch.object(self.ledger, "_run_fenced_json", return_value={"inserted": 1}) as execute: + self.ledger.persist_block_candidate_intent(self.prepared) + return body_id, execute.call_args.args[0] + + def retire_sql(self): + return retire_orphan_bodies_sql( + {**self.ledger._writer_identity_payload(), "stale_staging_seconds": 0}, + jsonb=self.ledger._jsonb_literal, + ) + + def race(self, first_sql, second_sql): + """Commit the first statement only after the second demonstrably waits.""" + initial = self.first.execute(first_sql).fetchone() + result = [] + + def run(): + try: + cursor = self.second.execute(second_sql) + result.append(cursor.fetchone() if cursor.description else None) + self.second.commit() + except Exception as exc: + result.append(exc) + self.second.rollback() + + thread = threading.Thread(target=run) + thread.start() + deadline = time.monotonic() + 10 + blocked = False + try: + while thread.is_alive() and time.monotonic() < deadline: + row = self.admin.execute( + "SELECT wait_event_type FROM pg_stat_activity WHERE pid = %s", + (self.second.info.backend_pid,), + ).fetchone() + if row and row[0] == "Lock": + blocked = True + break + time.sleep(0.01) + finally: + self.first.commit() + thread.join(10) + if thread.is_alive(): + self.second.cancel() + thread.join(5) + self.assertTrue(blocked, "second statement never reached the row-lock barrier") + self.assertFalse(thread.is_alive()) + self.assertEqual(len(result), 1) + return initial, result[0] + + def extra_part_sql(self, body_id, kind): + if kind == "chunk": + return ( + "INSERT INTO qbit_block_candidate_body_chunk VALUES " + f"('{body_id}', 999999, '\\x00', '{'0' * 64}') RETURNING ordinal" + ) + return ( + "INSERT INTO qbit_block_candidate_body_page VALUES " + f"('{body_id}', 'shares_json', 999999, 0, 0) RETURNING page_ordinal" + ) + + def test_seal_first_rejects_late_chunk(self): + body_id, seal = self.stage_without_sealing() + first, second = self.race(seal, self.extra_part_sql(body_id, "chunk")) + self.assertEqual(first[0]["sealed"], 1) + self.assertIsInstance(second, Exception) + self.assertIn("accepts no parts", str(second)) + + def test_seal_first_rejects_late_page(self): + body_id, seal = self.stage_without_sealing() + _, second = self.race(seal, self.extra_part_sql(body_id, "page")) + self.assertIsInstance(second, Exception) + self.assertIn("accepts no parts", str(second)) + + def assert_part_first_prevents_seal(self, kind): + body_id, seal = self.stage_without_sealing() + _, second = self.race(self.extra_part_sql(body_id, kind), seal) + if isinstance(second, Exception): + self.assertTrue("incomplete" in str(second) or "could not serialize" in str(second), str(second)) + else: + self.assertEqual(second[0]["sealed"], 0, "seal missed the part committed during its lock wait") + state = self.first.execute("SELECT state FROM qbit_block_candidate_body WHERE body_id = %s", (body_id,)).fetchone()[0] + self.assertEqual(state, "staging") + + def test_chunk_first_is_included_in_seal_validation(self): + self.assert_part_first_prevents_seal("chunk") + + def test_page_first_is_included_in_seal_validation(self): + self.assert_part_first_prevents_seal("page") + + def test_repeatable_read_sealer_cannot_miss_concurrent_upload(self): + self.second.execute("SET TRANSACTION ISOLATION LEVEL REPEATABLE READ") + self.assert_part_first_prevents_seal("page") + + def test_publication_first_prevents_orphan_retirement(self): + body_id, publish = self.publication() + first, second = self.race(publish, self.retire_sql()) + self.assertEqual(first[0]["inserted"], 1) + if isinstance(second, Exception): + self.assertIn("referenced by a pending outbox row", str(second)) + else: + self.assertEqual(second[0]["retired"], []) + self.assertEqual(self.first.execute("SELECT state FROM qbit_block_candidate_body WHERE body_id = %s", (body_id,)).fetchone()[0], "sealed") + + def test_retirement_first_prevents_publication(self): + body_id, publish = self.publication() + first, second = self.race(self.retire_sql(), publish) + self.assertEqual(first[0]["retired"], [body_id]) + self.assertEqual(second[0]["error"], "block candidate body is not sealed") + self.assertEqual(self.first.execute("SELECT count(*) FROM qbit_block_candidate_outbox WHERE body_id = %s", (body_id,)).fetchone()[0], 0) + + def test_repeatable_read_retirement_cannot_miss_publication(self): + body_id, publish = self.publication() + self.second.execute("SET TRANSACTION ISOLATION LEVEL REPEATABLE READ") + _, second = self.race(publish, self.retire_sql()) + self.assertIsInstance(second, Exception) + self.assertIn("could not serialize", str(second)) + self.assertEqual(self.first.execute("SELECT state FROM qbit_block_candidate_body WHERE body_id = %s", (body_id,)).fetchone()[0], "sealed") + + def test_database_publication_guard_invalidates_older_retirement_snapshot(self): + body_id = self.ledger.stage_candidate_body(self.prepared) + publish = ( + "INSERT INTO qbit_block_candidate_outbox " + "(block_hash, candidate_sha256, storage_version, body_id) VALUES " + f"('{self.prepared.block_hash}', '{self.prepared.candidate_sha256}', 2, '{body_id}') " + "RETURNING body_id" + ) + self.second.execute("SET TRANSACTION ISOLATION LEVEL REPEATABLE READ") + _, second = self.race(publish, self.retire_sql()) + self.assertIsInstance(second, Exception) + self.assertIn("could not serialize", str(second)) + self.assertEqual(self.first.execute("SELECT state FROM qbit_block_candidate_body WHERE body_id = %s", (body_id,)).fetchone()[0], "sealed") + + def test_content_comparator_rejects_forced_digest_collision(self): + body_id = self.ledger.stage_candidate_body(self.prepared) + fields = dict(self.prepared) + fields["username"] = "distinct" + different = prepare_candidate_intent(fields) + # The append caller has already accepted a matching digest. Exercise + # its independent content check with deliberately different bytes. + self.assertFalse(self.ledger.compare_candidate_body(body_id, different)) + + def test_content_comparator_preserves_jsonb_numeric_equality(self): + body_id = self.ledger.stage_candidate_body(self.prepared) + fields = dict(self.prepared) + fields["found_block"] = {"network_difficulty": 1000.0} + equivalent = prepare_candidate_intent(fields) + self.assertNotEqual(equivalent.candidate_sha256, self.prepared.candidate_sha256) + self.assertTrue(self.ledger.compare_candidate_body(body_id, equivalent)) + + def test_takeover_after_comparison_cannot_credit_share(self): + self.ledger.persist_block_candidate_intent(self.prepared) + compare = self.ledger.compare_candidate_body + + def compare_then_takeover(*args): + result = compare(*args) + self.first.execute( + "UPDATE qbit_ledger_writer_lease SET writer_session_token = %s WHERE singleton", + (uuid.uuid4().hex,), + ) + self.first.commit() + return result + + with mock.patch.object(self.ledger, "compare_candidate_body", side_effect=compare_then_takeover): + with self.assertRaisesRegex(RuntimeError, "writer lease is not active"): + self.ledger.append_batch([(pending_share_for(self.prepared), self.prepared)]) + self.assertEqual(self.first.execute("SELECT count(*) FROM qbit_share_ledger").fetchone()[0], 0) + self.assertEqual(self.first.execute("SELECT state, share_id FROM qbit_block_candidate_outbox").fetchone(), ("pending", None)) + + def test_body_retired_before_publication_is_restaged_without_duplicate_credit(self): + publish = self.ledger._publish_candidate_batch + retired = [] + + def retire_once(payloads, count): + if not retired: + body_id = payloads[0]["candidate"]["body_id"] + self.first.execute( + "UPDATE qbit_block_candidate_body SET state = 'retired', retired_at = clock_timestamp() WHERE body_id = %s", + (body_id,), + ) + self.first.commit() + retired.append(body_id) + return publish(payloads, count) + + with mock.patch.object(self.ledger, "_publish_candidate_batch", side_effect=retire_once): + self.ledger.append_batch([(pending_share_for(self.prepared), self.prepared)]) + self.assertEqual(self.first.execute("SELECT count(*) FROM qbit_share_ledger").fetchone()[0], 1) + row = self.first.execute("SELECT state, body_id FROM qbit_block_candidate_outbox").fetchone() + self.assertEqual(row[0], "pending") + self.assertNotEqual(row[1], retired[0]) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_prism_candidate_storm.py b/tests/test_prism_candidate_storm.py new file mode 100644 index 000000000..c5028fa08 --- /dev/null +++ b/tests/test_prism_candidate_storm.py @@ -0,0 +1,1150 @@ +#!/usr/bin/env python3 +"""Regression coverage for the reusable PRISM candidate-storm rig.""" + +from __future__ import annotations + +import unittest + +from lab.prism.block_candidates import ( + BLOCK_CANDIDATE_COLLAPSE_CLEANUP_STEPS, + MAX_BLOCK_REPLAY_ENUMERATION_ROWS, + MAX_PENDING_BLOCK_CANDIDATES, +) +from lab.prism.coordinator_config import ( + DEFAULT_BLOCK_CANDIDATE_CLEANUP_RETRY_BACKLOG_MAX, +) +from tests.prism_candidate_storm import ( + CLEANUP_FAULT_NONE, + CLEANUP_FAULTS, + OBSERVED_TESTNET_CANDIDATE_STORM, + PERTURBATIONS, + STORM_PARENT_HASH, + CandidateStormRig, + CleanupBacklogReport, + CleanupFaultInjector, + cleanup_backlog_marginal_bytes_per_record, +) + +# A storm large enough to overflow the live queue by 2x and to exercise the +# replay enumeration's legacy widening window more than once, small enough +# to drain through the per-row path in well under a second. +DECIDED_STORM_CANDIDATES = 100 + + +class CandidateStormInstrumentTests(unittest.TestCase): + def test_observed_storm_exposes_live_and_restart_marker_cardinality( + self, + ) -> None: + """Pin the ownership semantics that falsified the first #183 design. + + Outstanding covers all live admissions, not just the bounded queue, + and replay-inflight covers every adopted restart row. Neither marker + is evidence that the candidate has already reached qbitd. + """ + + self.assertGreater( + OBSERVED_TESTNET_CANDIDATE_STORM, + MAX_BLOCK_REPLAY_ENUMERATION_ROWS, + ) + self.assertGreater( + OBSERVED_TESTNET_CANDIDATE_STORM, + MAX_PENDING_BLOCK_CANDIDATES, + ) + rig = CandidateStormRig() + + live = rig.seed_live() + self.assertEqual(live.durable_pending, OBSERVED_TESTNET_CANDIDATE_STORM) + self.assertEqual(live.live_queue_capacity, MAX_PENDING_BLOCK_CANDIDATES) + self.assertEqual(live.live_queued, MAX_PENDING_BLOCK_CANDIDATES) + self.assertEqual( + live.live_coalesced, + OBSERVED_TESTNET_CANDIDATE_STORM - MAX_PENDING_BLOCK_CANDIDATES, + ) + self.assertEqual( + live.outstanding_marked, + OBSERVED_TESTNET_CANDIDATE_STORM, + ) + self.assertTrue(live.outstanding_covers_all_durable) + self.assertEqual(live.replay_queued, 0) + self.assertEqual(live.replay_inflight_marked, 0) + + restarted = rig.restart_and_enumerate() + self.assertEqual( + restarted.durable_pending, + OBSERVED_TESTNET_CANDIDATE_STORM, + ) + self.assertEqual(restarted.live_queued, 0) + self.assertEqual(restarted.live_coalesced, 0) + self.assertEqual(restarted.outstanding_marked, 0) + self.assertEqual( + restarted.replay_queued, + OBSERVED_TESTNET_CANDIDATE_STORM, + ) + self.assertEqual( + restarted.replay_inflight_marked, + OBSERVED_TESTNET_CANDIDATE_STORM, + ) + self.assertTrue(restarted.replay_inflight_covers_all_durable) + self.assertEqual( + restarted.accepted_parent_previews, + OBSERVED_TESTNET_CANDIDATE_STORM, + ) + self.assertFalse(restarted.replay_enumeration_owed) + + +class DecidedCandidateStormTests(unittest.TestCase): + """Pin what the rig reports once the storm height is decided. + + These are component facts about the shipped tree, not a selector: they + say which in-memory and durable facts exist for never-offered siblings + versus candidates with a real or ambiguous node offer, and what the + per-row disposition path costs to drain the sibling set. + """ + + def test_decided_height_leaves_siblings_without_offer_evidence(self) -> None: + rig = CandidateStormRig(candidates=DECIDED_STORM_CANDIDATES) + rig.seed_live() + winner = rig.decide_height() + self.assertEqual(winner, rig.block_hashes[-1]) + self.assertEqual(rig.decided_rpc.call("getbestblockhash"), winner) + self.assertEqual( + rig.decided_rpc.call("getblockhash", [rig.storm_height]), + winner, + ) + + live = rig.snapshot(rig.live_server) + self.assertEqual(live.outstanding_marked, DECIDED_STORM_CANDIDATES) + self.assertEqual(live.selector_evidence_marked, 0) + ownership = rig.ownership(rig.live_server) + self.assertEqual(len(ownership), DECIDED_STORM_CANDIDATES) + decided_header = rig.decided_rpc.call("getblockheader", [winner]) + for record in ownership: + self.assertEqual(record.outbox_state, "pending") + self.assertEqual(record.attempt_count, 0) + self.assertEqual(record.parent_hash, STORM_PARENT_HASH) + self.assertEqual(record.expected_height, rig.storm_height) + # Equal work: the decided block and every sibling were built at + # the same difficulty, so height occupancy is decisive here. + self.assertEqual(record.network_difficulty, decided_header["difficulty"]) + self.assertTrue(record.outstanding) + self.assertFalse(record.selector_evidence) + self.assertEqual( + sum(1 for record in ownership if record.live_queued), + MAX_PENDING_BLOCK_CANDIDATES, + ) + # The winner is the last seeded candidate: a coalesced wakeup in the + # live view, so the queue holds only siblings. + self.assertFalse(ownership[-1].live_queued) + + restarted = rig.restart_and_enumerate() + self.assertEqual(restarted.replay_inflight_marked, DECIDED_STORM_CANDIDATES) + self.assertEqual(restarted.accepted_parent_previews, DECIDED_STORM_CANDIDATES) + self.assertEqual(restarted.selector_evidence_marked, 0) + self.assertIs(rig.restarted_server.rpc, rig.decided_rpc) + self.assertTrue( + all(record.replay_queued for record in rig.ownership(rig.restarted_server)) + ) + + def test_selector_evidence_gauge_counts_each_evidence_kind_once(self) -> None: + rig = CandidateStormRig(candidates=DECIDED_STORM_CANDIDATES) + rig.seed_live() + rig.decide_height() + server = rig.live_server + hashes = rig.block_hashes + undo = [ + rig.perturb(server, kind, hashes[index]) + for index, kind in enumerate(PERTURBATIONS) + ] + try: + by_hash = {record.block_hash: record for record in rig.ownership(server)} + self.assertTrue(by_hash[hashes[0]].disposition_held) + self.assertTrue(by_hash[hashes[1]].node_acceptance_retained) + self.assertTrue(by_hash[hashes[2]].tip_observed) + self.assertTrue(by_hash[hashes[3]].retry_held) + self.assertEqual(by_hash[hashes[4]].pool_block_chain_state, "prepared") + self.assertIs(by_hash[hashes[5]].terminal_outcome, False) + for block_hash in hashes[:len(PERTURBATIONS)]: + self.assertTrue(by_hash[block_hash].selector_evidence) + # Process ownership is unchanged by evidence: still outstanding. + self.assertTrue(by_hash[block_hash].outstanding) + self.assertEqual( + sum(1 for record in by_hash.values() if record.selector_evidence), + len(PERTURBATIONS), + ) + # The union is conservative, but the rig still says which rows + # actually attest a node offer. Three do not: the retry holder, + # the disposition lease and the terminal registry are each + # reachable without any submitblock. + widenings = { + "retry_slot": "retry_held_without_offer", + "lease_held": "lease_held_without_offer", + "terminal": "terminal_without_offer", + } + for kind, attested in widenings.items(): + with self.subTest(perturbation=kind): + record = by_hash[hashes[PERTURBATIONS.index(kind)]] + self.assertFalse(record.node_offer_evidence) + for name in widenings.values(): + self.assertEqual(getattr(record, name), name == attested) + for index, kind in enumerate(PERTURBATIONS): + if kind in widenings: + continue + with self.subTest(perturbation=kind): + self.assertTrue(by_hash[hashes[index]].node_offer_evidence) + for name in widenings.values(): + self.assertFalse(getattr(by_hash[hashes[index]], name)) + snapshot = rig.snapshot(server) + self.assertEqual( + snapshot.selector_evidence_marked, + len(PERTURBATIONS), + ) + self.assertEqual( + snapshot.node_offer_evidence_marked, + len(PERTURBATIONS) - len(widenings), + ) + self.assertEqual(snapshot.unoffered_retry_marked, 1) + self.assertEqual(snapshot.unoffered_lease_marked, 1) + self.assertEqual(snapshot.unoffered_terminal_marked, 1) + # The published split accounts for the whole union: every row the + # conservative contract covers is either an attested offer or one + # of the three named widenings, and here the widenings are + # disjoint. + self.assertEqual( + snapshot.selector_evidence_marked, + snapshot.node_offer_evidence_marked + + snapshot.unoffered_retry_marked + + snapshot.unoffered_lease_marked + + snapshot.unoffered_terminal_marked, + ) + finally: + for release in undo: + if release is not None: + release() + + def test_retry_retention_alone_is_not_node_offer_evidence(self) -> None: + """Parking a wakeup in the retry holder must not read as an offer. + + ``_retain_block_candidate_for_retry`` runs on shipped paths that + never reached qbitd -- most plainly when + ``_reserve_block_fast_lane_slot`` declines and ``submit_next`` parks + the candidate *before* any submitblock -- so a rig that folded the + holder into its offer evidence would credit a node call that never + happened. The #183 must-not-abandon union still has to cover the + row, because the holder cannot distinguish that path from a genuine + offer retrying its tail. + """ + + rig = CandidateStormRig(candidates=DECIDED_STORM_CANDIDATES) + rig.seed_live() + rig.decide_height() + server = rig.live_server + parked = rig.block_hashes[0] + + before = {record.block_hash: record for record in rig.ownership(server)} + self.assertTrue(before[parked].live_queued) + self.assertFalse(before[parked].retry_held) + self.assertFalse(before[parked].selector_evidence) + calls_before = dict(rig.decided_rpc.calls) + + # Move a queued, never-offered wakeup into the retry slot, exactly as + # a declined fast-lane reservation does. + rig.perturb(server, "retry_slot", parked) + record = {r.block_hash: r for r in rig.ownership(server)}[parked] + self.assertTrue(record.retry_held) + + # Nothing was offered: no RPC happened, and no fact downstream of an + # offer exists for the row. + self.assertEqual(rig.decided_rpc.calls, calls_before) + self.assertFalse(record.node_offer_evidence) + self.assertTrue(record.retry_held_without_offer) + self.assertFalse(record.disposition_held) + self.assertFalse(record.node_acceptance_retained) + self.assertFalse(record.tip_observed) + self.assertFalse(record.accounted_accepted) + self.assertIsNone(record.pool_block_chain_state) + # Nor is the row reported terminal: retry retention is not a + # disposition, so the durable row is untouched and unattempted. + self.assertIsNone(record.terminal_outcome) + self.assertEqual(record.outbox_state, "pending") + self.assertEqual(record.attempt_count, 0) + # The conservative contract still covers it. + self.assertTrue(record.selector_evidence) + + snapshot = rig.snapshot(server) + self.assertEqual(snapshot.selector_evidence_marked, 1) + self.assertEqual(snapshot.node_offer_evidence_marked, 0) + self.assertEqual(snapshot.unoffered_retry_marked, 1) + + def test_lease_retention_alone_is_not_node_offer_evidence(self) -> None: + """Holding the disposition lease must not read as an offer. + + ``submit_next`` claims the lease on the dequeued hash *before* it + consults the terminal outcome, the accounted/capacity close and + ``_reserve_block_fast_lane_slot``; each of those paths releases it + again without ever reaching submitblock. The flight registry this is + read from is weaker still -- ``_claim_block_candidate_disposition`` + registers a user before taking the flight's lock, so a blocked waiter + is present too. A rig that folded the lease into its offer evidence + would credit a node call that never happened, and #183's safety + comparison is published against exactly that split. The + must-not-abandon union still has to cover the row, because a held + lease may equally span an offer already in flight. + """ + + rig = CandidateStormRig(candidates=DECIDED_STORM_CANDIDATES) + rig.seed_live() + rig.decide_height() + server = rig.live_server + leased = rig.block_hashes[0] + + before = {record.block_hash: record for record in rig.ownership(server)} + self.assertFalse(before[leased].disposition_held) + self.assertFalse(before[leased].selector_evidence) + calls_before = dict(rig.decided_rpc.calls) + + # Claim the lease through the shipped entry point, exactly as + # submit_next does before it has decided anything about the row. + release = rig.perturb(server, "lease_held", leased) + try: + record = {r.block_hash: r for r in rig.ownership(server)}[leased] + self.assertTrue(record.disposition_held) + + # Nothing was offered: no RPC happened, and no fact downstream of + # an offer exists for the row. + self.assertEqual(rig.decided_rpc.calls, calls_before) + self.assertFalse(record.node_offer_evidence) + self.assertTrue(record.lease_held_without_offer) + self.assertFalse(record.retry_held) + self.assertFalse(record.retry_held_without_offer) + self.assertFalse(record.node_acceptance_retained) + self.assertFalse(record.tip_observed) + self.assertFalse(record.accounted_accepted) + self.assertIsNone(record.pool_block_chain_state) + # Nor is the row disposed: the lease is a claim on the decision, + # not the decision, so the durable row is untouched and + # unattempted. + self.assertIsNone(record.terminal_outcome) + self.assertEqual(record.outbox_state, "pending") + self.assertEqual(record.attempt_count, 0) + # The conservative contract still covers it. + self.assertTrue(record.selector_evidence) + + snapshot = rig.snapshot(server) + self.assertEqual(snapshot.selector_evidence_marked, 1) + self.assertEqual(snapshot.node_offer_evidence_marked, 0) + self.assertEqual(snapshot.unoffered_lease_marked, 1) + self.assertEqual(snapshot.unoffered_retry_marked, 0) + finally: + release() + + # Releasing drops the row out of the union again: the widening reads + # the live flight registry rather than a sticky marker, so it cannot + # accumulate the storm the way outstanding/replay-inflight do. + after = {r.block_hash: r for r in rig.ownership(server)}[leased] + self.assertFalse(after.disposition_held) + self.assertFalse(after.selector_evidence) + self.assertEqual(rig.snapshot(server).selector_evidence_marked, 0) + self.assertEqual(rig.decided_rpc.calls, calls_before) + + def test_capacity_closed_terminalization_is_not_node_offer_evidence( + self, + ) -> None: + """A terminal outcome reached with zero submitblock is not an offer. + + Whenever pool capacity is already closed, ``submit_next`` builds a + ``_BlockCandidateNodeSubmission(attempted=False)`` and hands it + straight to the accounting queue, which marks the durable attempt and + terminalizes the outbox row without calling qbitd once. A rig that + folded the terminal registry into its offer evidence would publish + that row as an attested node offer, and #183's safety comparison is + published against exactly that split. The conservative + must-not-abandon union still has to cover the row, because a terminal + outcome may equally be the seal on an offer that returned. + """ + + rig = CandidateStormRig(candidates=DECIDED_STORM_CANDIDATES) + rig.seed_live() + winner = rig.decide_height() + server = rig.live_server + closed = rig.block_hashes[0] + + before = {r.block_hash: r for r in rig.ownership(server)}[closed] + self.assertIsNone(before.terminal_outcome) + self.assertFalse(before.selector_evidence) + self.assertNotIn("submitblock", rig.decided_rpc.calls) + + # Close pool capacity through the shipped coordinator tunable the + # production check reads, then drive one row of the shipped path. + # ``max_rounds`` cannot bound this call: it budgets node offers and + # this path makes none, so the single row is stepped by withholding + # every other one instead. + server.max_blocks = 0 + drain = rig.drain_per_row( + server, + stop_before_hash=winner, + preserve_hashes=[ + block_hash for block_hash in rig.block_hashes if block_hash != closed + ], + ) + + # The rig's own RPC call log is the proof: the row was terminalized + # and not one submitblock was made, on this call or ever -- so the + # call performed no rounds at all, and the report's offer gauge + # agrees with its RPC delta instead of contradicting it. + self.assertEqual(drain.rounds, 0) + self.assertEqual(drain.rpc_calls.get("submitblock", 0), 0) + self.assertNotIn("submitblock", rig.decided_rpc.calls) + self.assertEqual(drain.abandoned_hashes, frozenset({closed})) + self.assertEqual(drain.submitted_hashes, frozenset()) + self.assertEqual( + drain.withheld_hashes, + frozenset(rig.block_hashes) - {closed}, + ) + # The durable row still records an attempt, so attempt_count is no + # substitute for the missing RPC: submit_writer marks admission to a + # processing phase before the offer branch it never took. + self.assertEqual(drain.ledger_attempt_marks, 1) + + record = {r.block_hash: r for r in rig.ownership(server)}[closed] + self.assertEqual(record.outbox_state, "abandoned") + self.assertIs(record.terminal_outcome, False) + self.assertEqual(record.attempt_count, 1) + self.assertFalse(record.node_offer_evidence) + self.assertTrue(record.terminal_without_offer) + # Nothing else attests an offer for the row either. + self.assertFalse(record.node_acceptance_retained) + self.assertFalse(record.tip_observed) + self.assertFalse(record.accounted_accepted) + self.assertIsNone(record.pool_block_chain_state) + self.assertFalse(record.retry_held) + self.assertFalse(record.retry_held_without_offer) + self.assertFalse(record.disposition_held) + self.assertFalse(record.lease_held_without_offer) + # The conservative contract still covers it. + self.assertTrue(record.selector_evidence) + + snapshot = rig.snapshot(server) + self.assertEqual(snapshot.node_offer_evidence_marked, 0) + self.assertEqual(snapshot.unoffered_terminal_marked, 1) + self.assertEqual(snapshot.unoffered_retry_marked, 0) + self.assertEqual(snapshot.unoffered_lease_marked, 0) + self.assertEqual(snapshot.selector_evidence_marked, 1) + + # The oracle's exact-set properties are unchanged by the narrowed + # gauge: reopening capacity drains every remaining sibling through + # the ordinary offer path, and the two calls partition the sibling + # set exactly, each reporting only its own transitions. + server.max_blocks = DECIDED_STORM_CANDIDATES + 1 + rest = rig.drain_per_row(server, stop_before_hash=winner) + self.assertEqual( + rest.rpc_calls["submitblock"], + DECIDED_STORM_CANDIDATES - 2, + ) + self.assertTrue(rest.abandoned_hashes.isdisjoint(drain.abandoned_hashes)) + self.assertEqual( + drain.abandoned_hashes | rest.abandoned_hashes, + frozenset(rig.block_hashes) - {winner}, + ) + self.assertEqual(rest.pending_hashes, frozenset({winner})) + self.assertEqual(rest.submitted_hashes, frozenset()) + self.assertEqual(rest.withheld_hashes, frozenset()) + # Every abandoned row is now conservatively covered and none of them + # is published as an attested offer. + after = rig.ownership(server) + self.assertEqual( + sum(1 for r in after if r.terminal_without_offer), + DECIDED_STORM_CANDIDATES - 1, + ) + self.assertEqual( + rig.snapshot(server).node_offer_evidence_marked, + 0, + ) + + def test_replay_adoption_alone_drops_a_tip_observation(self) -> None: + """Why ``perturb('tip_observed')`` also registers the hash outstanding. + + Restart adoption never marks a hash outstanding -- only the node + offer does -- so a blockwait observation of a replayed candidate is + dropped until it is dequeued and offered. A selector that reads the + observation set alone therefore cannot see a replayed candidate's + acceptance, which is why the #181 dequeue-time skip has to read the + chain for replayed rows. + """ + + rig = CandidateStormRig(candidates=DECIDED_STORM_CANDIDATES) + rig.seed_live() + rig.restart_and_enumerate() + rig.decide_height() + server = rig.restarted_server + replayed = rig.block_hashes[0] + self.assertEqual(rig.snapshot(server).outstanding_marked, 0) + + server._note_tip_observation_for_candidates(replayed) + by_hash = {record.block_hash: record for record in rig.ownership(server)} + self.assertFalse(by_hash[replayed].tip_observed) + self.assertFalse(by_hash[replayed].selector_evidence) + self.assertEqual(rig.snapshot(server).selector_evidence_marked, 0) + + rig.perturb(server, "tip_observed", replayed) + by_hash = {record.block_hash: record for record in rig.ownership(server)} + self.assertTrue(by_hash[replayed].tip_observed) + self.assertEqual(rig.snapshot(server).selector_evidence_marked, 1) + + def test_per_row_drain_cost_is_linear_in_siblings_live_and_after_restart( + self, + ) -> None: + siblings = DECIDED_STORM_CANDIDATES - 1 + expected_rpc = { + # One fast-lane offer per sibling, then the admission tip read plus + # the abandon seal's pre- and post-withdrawal chain probes: three + # getbestblockhash/getblockheader pairs per sibling. + "submitblock": siblings, + "getbestblockhash": 3 * siblings, + "getblockheader": 3 * siblings, + } + + live_rig = CandidateStormRig(candidates=DECIDED_STORM_CANDIDATES) + live_rig.seed_live() + winner = live_rig.decide_height() + drain = live_rig.drain_per_row(live_rig.live_server, stop_before_hash=winner) + self.assertEqual(drain.rounds, siblings) + # The exact set, not just its size: this is what a selector is + # compared against. + self.assertEqual( + drain.abandoned_hashes, + frozenset(live_rig.block_hashes) - {winner}, + ) + self.assertEqual(drain.pending_hashes, frozenset({winner})) + self.assertEqual(drain.submitted_hashes, frozenset()) + self.assertEqual(drain.withheld_hashes, frozenset()) + # Every offer was handed to the real accounting queue and run there. + self.assertEqual(drain.accounting_tasks, siblings) + self.assertEqual(drain.ledger_attempt_marks, siblings) + self.assertEqual(drain.rpc_calls, expected_rpc) + # 32 queued wakeups drain first; the coalesced remainder is recovered + # through the outbox query in 32-row windows: 32 + 32 + 4 rows. + self.assertEqual(drain.replay_enumerations, 3) + after = {record.block_hash: record for record in live_rig.ownership(live_rig.live_server)} + self.assertEqual(after[winner].outbox_state, "pending") + self.assertFalse(after[winner].selector_evidence) + self.assertTrue( + all( + record.outbox_state == "abandoned" and record.terminal_outcome is False + for block_hash, record in after.items() + if block_hash != winner + ) + ) + # Every sibling's offer-time barrier was withdrawn by its abandonment; + # the one left is the winner's, armed when the live process's own + # recovery enumeration adopted it from the outbox (adoption arms a + # barrier per row exactly as restart adoption does). + self.assertEqual(live_rig.snapshot(live_rig.live_server).accepted_parent_previews, 1) + + restart_rig = CandidateStormRig(candidates=DECIDED_STORM_CANDIDATES) + restart_rig.seed_live() + restart_rig.restart_and_enumerate() + winner = restart_rig.decide_height() + drain = restart_rig.drain_per_row( + restart_rig.restarted_server, + stop_before_hash=winner, + ) + self.assertEqual(drain.rounds, siblings) + self.assertEqual( + drain.abandoned_hashes, + frozenset(restart_rig.block_hashes) - {winner}, + ) + self.assertEqual(drain.pending_hashes, frozenset({winner})) + self.assertEqual(drain.accounting_tasks, siblings) + self.assertEqual(drain.ledger_attempt_marks, siblings) + self.assertEqual(drain.rpc_calls, expected_rpc) + # Restart adoption already queued every row; no enumeration is needed. + self.assertEqual(drain.replay_enumerations, 0) + # Only the undrained winner's adoption-time barrier remains. + self.assertEqual( + restart_rig.snapshot(restart_rig.restarted_server).accepted_parent_previews, + 1, + ) + + def test_per_row_drain_runs_the_production_submitter_split(self) -> None: + """The drain must defer accounting, as ``BlockCandidateService.run`` does. + + Two shipped behaviours exist only on that split, and both decide + whether a row is abandoned: ``submit_next`` claims the disposition + lease *blocking* when accounting is inline, so one candidate whose + lease is held elsewhere would stop the drain forever; and + ``_restore_replayed_candidate_acceptance_evidence`` runs only inside + the accounting task, so an inline drain abandons a replayed candidate + with a prepared pool-block row that production preserves. + """ + + rig = CandidateStormRig(candidates=DECIDED_STORM_CANDIDATES) + rig.seed_live() + winner = rig.decide_height() + server = rig.live_server + held = rig.block_hashes[0] + release = rig.perturb(server, "lease_held", held) + try: + drain = rig.drain_per_row(server, stop_before_hash=winner) + finally: + release() + # Bounded: the held lease parked one wakeup instead of blocking, and + # every other sibling still drained. + self.assertEqual(drain.lease_blocked_hashes, frozenset({held})) + self.assertNotIn(held, drain.abandoned_hashes) + self.assertIn(held, drain.pending_hashes) + self.assertEqual( + drain.abandoned_hashes, + frozenset(rig.block_hashes) - {winner, held}, + ) + # The lease-held row was never offered, so it cost no submitblock. + self.assertEqual( + drain.rpc_calls["submitblock"], + DECIDED_STORM_CANDIDATES - 2, + ) + + def test_max_rounds_budgets_offers_not_wakeups_when_a_lease_is_held( + self, + ) -> None: + """A parked no-offer row must not spend the bounded drain's budget. + + ``submit_next_block_candidate(defer_accounting=True)`` returns True + for the head whose disposition lease it could not claim: it only + parks the wakeup in ``_block_disposition_waiting_retries`` and + returns, having offered qbitd nothing. A ``max_rounds`` budget spent + on that boolean would end a bounded drain with zero submitblock + calls, zero accounting tasks and zero terminalizations, while the + report still claimed a round -- so the gauge would contradict this + report's own RPC, accounting and terminal-set fields, and a caller + stepping the oracle one offered row at a time would silently get no + row at all. + """ + + rig = CandidateStormRig(candidates=DECIDED_STORM_CANDIDATES) + rig.seed_live() + winner = rig.decide_height() + server = rig.live_server + service = server._ensure_block_candidate_service() + # The head of the live lane, so the parked row is the first one the + # bounded drain reaches. + held = str( + list(service.candidate_queue.queue)[0].submission.block_hash_hex + ).lower() + + release = rig.perturb(server, "lease_held", held) + try: + drain = rig.drain_per_row( + server, + stop_before_hash=winner, + max_rounds=1, + ) + # Read while the lease still stands: that is the state the drain + # saw and the reason it parked the row. + record = {r.block_hash: r for r in rig.ownership(server)}[held] + finally: + release() + + # The parked row cost no offer, so it cost no round; the drain moved + # past it and spent its single round on a row it really offered. + self.assertEqual(drain.rounds, 1) + self.assertEqual(drain.rpc_calls.get("submitblock", 0), 1) + self.assertEqual(drain.accounting_tasks, 1) + self.assertEqual(drain.ledger_attempt_marks, 1) + # Every cost field agrees with the terminal sets: exactly one row was + # terminalized, and it is not the parked one. + self.assertEqual(len(drain.abandoned_hashes), 1) + self.assertNotIn(held, drain.abandoned_hashes) + self.assertEqual(drain.submitted_hashes, frozenset()) + self.assertEqual(drain.lease_blocked_hashes, frozenset({held})) + self.assertEqual(drain.withheld_hashes, frozenset({held})) + self.assertIn(held, drain.pending_hashes) + self.assertEqual( + drain.pending_hashes, + frozenset(rig.block_hashes) - drain.abandoned_hashes, + ) + + # The parked row itself is untouched: no offer, no durable attempt, + # no terminal outcome -- only the lease this test holds. + self.assertEqual(record.outbox_state, "pending") + self.assertEqual(record.attempt_count, 0) + self.assertIsNone(record.terminal_outcome) + self.assertFalse(record.node_offer_evidence) + self.assertTrue(record.disposition_held) + self.assertTrue(record.lease_held_without_offer) + self.assertTrue(record.selector_evidence) + + def test_per_row_drain_preserves_every_offer_evidence_perturbation( + self, + ) -> None: + """What the shipped path does with each evidence shape, in both views. + + Only two shapes reach a terminal abandonment: a wakeup sitting in the + retry slot (dequeued and re-offered like any other), and, in the live + view only, a prepared pool-block row (rejected first, then + abandoned). The same prepared row after restart is preserved, because + the accounting task restores its acceptance evidence. ``lease_held`` + carries no node-offer evidence either, but it is spared for an + unrelated reason: the drain cannot claim its lease, so the shipped + path parks that wakeup instead of deciding it. + """ + + for view in ("live", "restart"): + with self.subTest(view=view): + rig = CandidateStormRig(candidates=DECIDED_STORM_CANDIDATES) + rig.seed_live() + if view == "restart": + rig.restart_and_enumerate() + winner = rig.decide_height() + server = rig.restarted_server if view == "restart" else rig.live_server + hashes = dict(zip(PERTURBATIONS, rig.block_hashes)) + undo = [ + rig.perturb(server, kind, block_hash) + for kind, block_hash in hashes.items() + ] + self.assertEqual( + rig.snapshot(server).selector_evidence_marked, + len(PERTURBATIONS), + ) + try: + drain = rig.drain_per_row( + server, + stop_before_hash=winner, + # A retained definitive success re-runs the landing + # tail rather than the abandon path, and that tail is + # outside this instrument. + preserve_hashes=[hashes["retained_success"]], + ) + finally: + for release in undo: + if release is not None: + release() + + abandoned = {"retry_slot"} + if view == "live": + abandoned.add("prepared_pool_block") + for kind, block_hash in hashes.items(): + with self.subTest(perturbation=kind): + if kind in abandoned: + self.assertIn(block_hash, drain.abandoned_hashes) + else: + self.assertNotIn(block_hash, drain.abandoned_hashes) + self.assertIn(block_hash, drain.pending_hashes) + self.assertEqual( + drain.lease_blocked_hashes, + frozenset({hashes["lease_held"]}), + ) + expected_deferred = {hashes["tip_observed"]} + if view == "restart": + expected_deferred.add(hashes["prepared_pool_block"]) + self.assertEqual(drain.deferred_hashes, frozenset(expected_deferred)) + self.assertEqual( + drain.abandoned_rows, + DECIDED_STORM_CANDIDATES - 1 - len(hashes) + len(abandoned), + ) + self.assertEqual(drain.submitted_hashes, frozenset()) + self.assertIn(winner, drain.pending_hashes) + + def test_preserved_wakeups_survive_the_drain_that_withheld_them(self) -> None: + """``preserve_hashes`` is invocation-scoped, so it must not eat wakeups. + + A withheld row is lifted out of the dequeue order without being + offered or disposed, which leaves its ownership markers standing. + After restart adoption that includes ``_block_replay_inflight_hashes``, + and ``_enqueue_replayed_block_candidate`` drops any re-adoption of an + in-flight hash as a duplicate -- so a drain that discarded the lifted + wakeup would leave the row pending with nothing left to dispose of it + and no enumeration able to rebuild it. Partial and repeated drains + re-apply the withholding every call, so the wakeup has to survive each + one, and a later unprotected drain has to be able to process the row. + """ + + for view in ("live", "restart"): + with self.subTest(view=view): + rig = CandidateStormRig(candidates=DECIDED_STORM_CANDIDATES) + rig.seed_live() + if view == "restart": + rig.restart_and_enumerate() + winner = rig.decide_height() + server = rig.restarted_server if view == "restart" else rig.live_server + service = server._ensure_block_candidate_service() + protected = rig.block_hashes[0] + + def queued_wakeups() -> set[str]: + return { + str(item.submission.block_hash_hex).lower() + for lane in ( + service.candidate_queue, + service._block_replay_candidate_queue, + ) + for item in list(lane.queue) + } + + self.assertIn(protected, queued_wakeups()) + + # Repeated partial drains: each one withholds the row again. + steps = [ + rig.drain_per_row( + server, + stop_before_hash=winner, + preserve_hashes=[protected], + max_rounds=2, + ) + for _ in range(2) + ] + for index, step in enumerate(steps): + with self.subTest(step=index): + self.assertEqual(step.rounds, 2) + self.assertEqual(step.withheld_hashes, frozenset({protected})) + self.assertNotIn(protected, step.abandoned_hashes) + self.assertIn(protected, step.pending_hashes) + self.assertEqual(len(step.abandoned_hashes), 2) + self.assertTrue( + steps[0].abandoned_hashes.isdisjoint(steps[1].abandoned_hashes) + ) + self.assertIn(protected, queued_wakeups()) + + # ...then an unbounded preserving drain over the remainder. + first = rig.drain_per_row( + server, + stop_before_hash=winner, + preserve_hashes=[protected], + ) + self.assertEqual(first.withheld_hashes, frozenset({protected})) + self.assertNotIn(protected, first.abandoned_hashes) + self.assertEqual( + first.pending_hashes, + frozenset({protected, winner}), + ) + + # The protected row still owns exactly one in-memory wakeup, + # in a lane a later drain dequeues from... + self.assertIn(protected, queued_wakeups()) + self.assertEqual( + sum( + 1 + for lane in ( + service.candidate_queue, + service._block_replay_candidate_queue, + ) + for item in list(lane.queue) + if str(item.submission.block_hash_hex).lower() == protected + ), + 1, + ) + record = {r.block_hash: r for r in rig.ownership(server)}[protected] + self.assertTrue(record.live_queued or record.replay_queued) + # ...and its durable row and evidence are untouched, exactly + # as the caller arranged them. + self.assertEqual(record.outbox_state, "pending") + self.assertEqual(record.attempt_count, 0) + self.assertIsNone(record.terminal_outcome) + self.assertFalse(record.selector_evidence) + self.assertFalse(record.node_offer_evidence) + + # A later unprotected drain reaches it and disposes of it for + # real -- one offer, one accounting task, one durable attempt. + second = rig.drain_per_row(server, stop_before_hash=winner) + self.assertEqual(second.rounds, 1) + self.assertEqual(second.accounting_tasks, 1) + self.assertEqual(second.ledger_attempt_marks, 1) + self.assertEqual(second.rpc_calls["submitblock"], 1) + self.assertEqual(second.abandoned_hashes, frozenset({protected})) + self.assertEqual(second.withheld_hashes, frozenset()) + self.assertEqual(second.pending_hashes, frozenset({winner})) + # Every call partitions the sibling set; nothing was stranded. + self.assertEqual( + steps[0].abandoned_hashes + | steps[1].abandoned_hashes + | first.abandoned_hashes + | second.abandoned_hashes, + frozenset(rig.block_hashes) - {winner}, + ) + + def test_bounded_drains_report_only_their_own_transitions(self) -> None: + """``max_rounds`` reports this call's work, not the rig's history. + + A bounded drain is how a caller steps the oracle one row at a time. + If the report named every terminal row standing in the outbox instead + of the ones this call moved, the second step would claim the first + step's abandonment, and no per-step comparison against a selector + would mean anything. + """ + + steps = 3 + rig = CandidateStormRig(candidates=DECIDED_STORM_CANDIDATES) + rig.seed_live() + winner = rig.decide_height() + server = rig.live_server + + seen: frozenset[str] = frozenset() + for step in range(steps): + with self.subTest(step=step): + report = rig.drain_per_row( + server, + stop_before_hash=winner, + max_rounds=1, + ) + # One row offered, one accounted, one durable attempt marked. + self.assertEqual(report.rounds, 1) + self.assertEqual(report.accounting_tasks, 1) + self.assertEqual(report.ledger_attempt_marks, 1) + self.assertEqual(report.rpc_calls["submitblock"], 1) + # The first 32 wakeups are already queued, so no step needs + # the recovery enumeration. + self.assertEqual(report.replay_enumerations, 0) + # ...and exactly one transition, never an earlier step's. + self.assertEqual(len(report.abandoned_hashes), 1) + self.assertEqual(report.submitted_hashes, frozenset()) + self.assertEqual(report.withheld_hashes, frozenset()) + self.assertTrue(report.abandoned_hashes.isdisjoint(seen)) + seen |= report.abandoned_hashes + # pending_hashes is the complementary residue of the same + # outbox, so it shrinks by exactly the rows drained so far. + self.assertEqual( + report.pending_hashes, + frozenset(rig.block_hashes) - seen, + ) + self.assertEqual(len(seen), steps) + + # The bounded steps and the unbounded remainder partition the sibling + # set exactly: no row is reported twice and none is dropped. + final = rig.drain_per_row(server, stop_before_hash=winner) + self.assertEqual(final.rounds, DECIDED_STORM_CANDIDATES - 1 - steps) + self.assertTrue(final.abandoned_hashes.isdisjoint(seen)) + self.assertEqual( + seen | final.abandoned_hashes, + frozenset(rig.block_hashes) - {winner}, + ) + self.assertEqual(final.submitted_hashes, frozenset()) + self.assertEqual(final.withheld_hashes, frozenset()) + self.assertEqual(final.pending_hashes, frozenset({winner})) + + def test_per_row_drain_at_the_observed_cardinality(self) -> None: + """The same facts at the incident's 3,120 rows, live and after restart. + + 3,120 is above both the 32-entry live queue and the 1,024-row replay + page, so the live view exercises the recovery enumeration 97 times + and the restart view exercises multi-page adoption. + """ + + siblings = OBSERVED_TESTNET_CANDIDATE_STORM - 1 + expected_rpc = { + "submitblock": siblings, + "getbestblockhash": 3 * siblings, + "getblockheader": 3 * siblings, + } + for view, enumerations in (("live", 97), ("restart", 0)): + with self.subTest(view=view): + rig = CandidateStormRig() + rig.seed_live() + if view == "restart": + rig.restart_and_enumerate() + winner = rig.decide_height() + server = rig.restarted_server if view == "restart" else rig.live_server + drain = rig.drain_per_row(server, stop_before_hash=winner) + self.assertEqual( + drain.abandoned_hashes, + frozenset(rig.block_hashes) - {winner}, + ) + self.assertEqual(drain.pending_hashes, frozenset({winner})) + self.assertEqual(drain.submitted_hashes, frozenset()) + self.assertEqual(drain.withheld_hashes, frozenset()) + self.assertEqual(drain.rounds, siblings) + self.assertEqual(drain.accounting_tasks, siblings) + self.assertEqual(drain.ledger_attempt_marks, siblings) + self.assertEqual(drain.rpc_calls, expected_rpc) + self.assertEqual(sum(drain.rpc_calls.values()), 21_833) + self.assertEqual(drain.replay_enumerations, enumerations) + # Only the undrained winner's barrier is left standing. + self.assertEqual( + rig.snapshot(server).accepted_parent_previews, + 1, + ) + + def test_parent_transition_lookup_probes_once_per_distinct_height(self) -> None: + rig = CandidateStormRig(candidates=DECIDED_STORM_CANDIDATES) + rig.seed_live() + rig.restart_and_enumerate() + rig.decide_height() + server = rig.restarted_server + self.assertEqual( + rig.snapshot(server).accepted_parent_previews, + DECIDED_STORM_CANDIDATES, + ) + before = dict(rig.decided_rpc.calls) + # A parent with no exact transition: the lookup falls through to the + # ancestor scan, which #193 deduplicates by candidate height. + selected = server._accepted_block_payout_transition_for_parent( + "22" * 32, + parent_height=rig.storm_height, + ) + self.assertEqual( + rig.decided_rpc.calls.get("getblockhash", 0) - before.get("getblockhash", 0), + 1, + ) + self.assertEqual(selected, (rig.block_hashes[-1], False)) + + +class CleanupBacklogStormTests(unittest.TestCase): + """Issue #198: wall-clock corroboration at the observed cardinality. + + The deterministic contract lives in ``tests/test_prism_block_candidates``; + these runs drive the shipped collapse walk over 3,120 durable rows under + each injected cleanup dependency fault, hold the fault for a bounded run + of failing retries, heal it, and require the rig's recovery proof to + hold: the backlog drains through the shipped retry pass, every collapsed + hash is cleaned up exactly once, no terminal row is replay-adopted or + re-offered, and the decided winner keeps its pending-share floor + authority. Wall-clock bounds are deliberately loose; they corroborate + the measured throughput rather than pin it. + """ + + SIBLINGS = OBSERVED_TESTNET_CANDIDATE_STORM - 1 + + def _assert_recovered(self, report: CleanupBacklogReport) -> None: + self.assertEqual(report.depth_after_recovery, 0, report) + self.assertEqual(report.recovered_records, report.deferred_records, report) + self.assertEqual(report.recovery_passes, report.deferred_records, report) + self.assertEqual(set(report.excess_cleanup_calls.values()), {0}, report) + self.assertEqual(set(report.missing_cleanup_calls.values()), {0}, report) + self.assertEqual(report.post_recovery_cleanup_calls, 0, report) + self.assertEqual(report.floor_double_releases, 0, report) + self.assertEqual(report.terminal_rows_adopted, 0, report) + self.assertEqual(report.node_offers, 0, report) + self.assertEqual(report.drain_rounds, 0, report) + self.assertEqual(report.pending_rows_after_drain, 1, report) + self.assertTrue(report.recovered_cleanly, report) + + def test_the_fault_vocabulary_covers_every_cleanup_dependency(self) -> None: + self.assertEqual( + set(CLEANUP_FAULTS), + set(BLOCK_CANDIDATE_COLLAPSE_CLEANUP_STEPS) | {"floor-index"}, + ) + rig = CandidateStormRig(candidates=DECIDED_STORM_CANDIDATES) + rig.seed_live() + with self.assertRaises(ValueError): + CleanupFaultInjector(rig.live_server, "not-a-cleanup-dependency") + with self.assertRaises(ValueError): + rig.measure_cleanup_backlog("not-a-cleanup-dependency") + + def test_every_cleanup_dependency_fault_at_the_observed_storm_recovers( + self, + ) -> None: + for fault in CLEANUP_FAULTS: + with self.subTest(fault=fault): + rig = CandidateStormRig(credit_shares=True) + report = rig.measure_cleanup_backlog( + fault, + view="restart", + sustained_passes=16, + ) + # The shipped bound absorbs the whole storm: every sibling + # is collapsed and every one of them owes cleanup. + self.assertEqual(report.backlog_max, DEFAULT_BLOCK_CANDIDATE_CLEANUP_RETRY_BACKLOG_MAX) + self.assertEqual(report.collapsed_rows, self.SIBLINGS) + self.assertEqual(report.deferred_records, self.SIBLINGS) + self.assertEqual(report.preserved_rows, 0) + self.assertEqual(report.backpressure_engagements, 0) + self.assertFalse(report.backpressure_active) + self.assertEqual(report.terminal_outcome_pins, self.SIBLINGS) + # The restart view holds every row in the replay queue, so + # every record retains its holder -- except under the + # index abort, which never got to read them. + expected_holders = 0 if fault == "floor-index" else self.SIBLINGS + self.assertEqual(report.pending_share_holders, expected_holders) + self.assertGreater(report.registry_bytes_per_record, 0.0) + self.assertGreaterEqual(report.oldest_age_seconds, 0.0) + # Sustained failure sheds nothing. + self.assertEqual(report.sustained_passes, 16) + self.assertEqual(report.sustained_failed_passes, 16) + self.assertEqual(report.depth_after_sustained, self.SIBLINGS) + self.assertEqual(report.holders_after_sustained, expected_holders) + self.assertEqual(report.pins_after_sustained, self.SIBLINGS) + # Recovery, and the floor authority it leaves behind. + self._assert_recovered(report) + self.assertEqual(report.collapsed_rows_after_recovery, 0) + self.assertEqual(report.floor_holders_before, OBSERVED_TESTNET_CANDIDATE_STORM) + self.assertEqual(report.floor_holders_after, 1) + self.assertTrue(report.winner_floor_holder_retained) + # Wall-clock corroboration of the measured drain rate. + self.assertLess(report.recovery_seconds, 60.0) + self.assertGreater(report.retry_records_per_second, 50.0) + + def test_the_bound_engages_at_the_observed_storm_and_admission_resumes( + self, + ) -> None: + bound = MAX_BLOCK_REPLAY_ENUMERATION_ROWS + rig = CandidateStormRig(credit_shares=True, backlog_max=bound) + report = rig.measure_cleanup_backlog( + "retry-state", + view="restart", + sustained_passes=16, + ) + self.assertEqual(report.backlog_max, bound) + # The first page fills the backlog; exactly one following page is + # preserved and adopted before pagination pauses for it to drain. + self.assertEqual(report.collapsed_rows, bound) + self.assertEqual(report.deferred_records, bound) + self.assertEqual( + report.preserved_rows, + MAX_BLOCK_REPLAY_ENUMERATION_ROWS, + ) + self.assertEqual(report.backpressure_engagements, 1) + self.assertTrue(report.backpressure_active) + self.assertEqual(report.pending_share_holders, bound) + self.assertEqual(report.terminal_outcome_pins, bound) + self.assertEqual(report.depth_after_sustained, bound) + self.assertEqual(report.holders_after_sustained, bound) + # Once drained, the next walk collapses everything the bound + # preserved, and the rig's recovery proof still holds over the + # union. + self._assert_recovered(report) + self.assertEqual(report.collapsed_rows_after_recovery, self.SIBLINGS - bound) + self.assertEqual(report.floor_holders_after, 1) + self.assertTrue(report.winner_floor_holder_retained) + + def test_the_live_view_retains_only_the_queued_holders(self) -> None: + """Only a queued wakeup owns a holder the cleanup can retain.""" + rig = CandidateStormRig() + report = rig.measure_cleanup_backlog( + "pending-share-floor", + view="live", + sustained_passes=8, + ) + self.assertEqual(report.collapsed_rows, self.SIBLINGS) + # The coalesced siblings had no in-memory object, so their cleanup + # had no floor step to fail: only the bounded queue's wakeups owe. + self.assertEqual(report.deferred_records, MAX_PENDING_BLOCK_CANDIDATES) + self.assertEqual(report.pending_share_holders, MAX_PENDING_BLOCK_CANDIDATES) + self.assertEqual(report.terminal_outcome_pins, MAX_PENDING_BLOCK_CANDIDATES) + self._assert_recovered(report) + + def test_a_clean_walk_leaves_no_backlog_and_the_marginal_cost_is_measurable( + self, + ) -> None: + clean = CandidateStormRig(candidates=DECIDED_STORM_CANDIDATES).measure_cleanup_backlog( + CLEANUP_FAULT_NONE, + view="restart", + ) + self.assertEqual(clean.deferred_records, 0) + self.assertEqual(clean.collapsed_rows, DECIDED_STORM_CANDIDATES - 1) + self.assertEqual(clean.registry_bytes_per_record, 0.0) + self.assertEqual(clean.pending_share_holders, 0) + self.assertTrue(clean.recovered_cleanly) + faulted = CandidateStormRig(candidates=DECIDED_STORM_CANDIDATES).measure_cleanup_backlog( + "retry-state", + view="restart", + ) + self.assertEqual(faulted.deferred_records, DECIDED_STORM_CANDIDATES - 1) + # The process-wide live-byte delta also includes allocator and cache + # churn from the clean walk, so it is diagnostic rather than a + # monotonic assertion. The owner-filtered delta and the independent + # deep registry walk isolate the retained retry records. + self.assertGreater( + cleanup_backlog_marginal_bytes_per_record(faulted, clean, owner_only=True), + 0.0, + ) + self.assertGreater(faulted.registry_bytes_per_record, 0.0) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_prism_candidate_window.py b/tests/test_prism_candidate_window.py new file mode 100644 index 000000000..74aa9b4c3 --- /dev/null +++ b/tests/test_prism_candidate_window.py @@ -0,0 +1,411 @@ +from __future__ import annotations + +import gc +import itertools +import os +from pathlib import Path +import shlex +import shutil +import threading +import time +from types import SimpleNamespace +import unittest +from unittest import mock +import weakref + +from lab.prism.candidate_window import ( + COPY_TEXT_MAX_BYTES, + COPY_TEXT_MAX_RECORDS, + copy_text_chunks, + disk_window_covers, + recorded_share_ids, +) + + +class CandidateWindowTests(unittest.TestCase): + def test_membership_checks_omissions_not_equality_or_order(self) -> None: + self.assertTrue(disk_window_covers(iter(["b", "a", "a", "extra"]), [ + {"share_id": "a"}, {"share_id": "b"}, + ])) + self.assertFalse(disk_window_covers(iter(["a", "extra"]), [ + {"share_id": "a"}, {"share_id": "late"}, + ])) + self.assertTrue(disk_window_covers(iter([]), [])) + self.assertFalse(disk_window_covers(iter([]), [{"share_id": "a"}])) + + def test_legacy_string_and_non_dictionary_semantics(self) -> None: + recorded = [None, {"share_id": 42}, {}, {"share_id": "é"}] + self.assertTrue(disk_window_covers(recorded_share_ids(recorded), [ + "ignored", {"share_id": "42"}, {"share_id": None}, {"share_id": "é"}, + ])) + self.assertFalse(disk_window_covers(recorded_share_ids(recorded), [ + {"share_id": "e\u0301"}, + ])) + + def test_copy_escapes_commands_nulls_and_delimiters(self) -> None: + chunks = list(copy_text_chunks(["\\.\nDROP TABLE x;", "\\N", "a\tb\rc", ""])) + self.assertEqual( + b"".join(chunks), b"\\\\.\\nDROP TABLE x;\n\\\\N\na\\tb\\rc\n\n", + ) + + def test_oversized_field_obeys_byte_bound_and_checks_deadline(self) -> None: + value = "🌐\n\\\té" * 30000 + checks = 0 + + def check() -> None: + nonlocal checks + checks += 1 + + count = 0 + for chunk in copy_text_chunks([value], check=check): + self.assertLessEqual(len(chunk), COPY_TEXT_MAX_BYTES) + count += 1 + self.assertEqual(checks, count) + self.assertGreater(checks, 30) + + def test_lazy_identifier_is_streamed_and_compared_exactly_under_collision(self) -> None: + class Text: + def __init__(self, suffix): + self.suffix = suffix + + def iter_text_chunks(self): + yield "🌐\n\\\té" * 30000 + yield self.suffix + + def __str__(self): + raise AssertionError("lazy identifier was materialized") + + value = Text("a") + chunks = list(copy_text_chunks(recorded_share_ids([{"share_id": value}]))) + self.assertTrue(all(len(chunk) <= COPY_TEXT_MAX_BYTES for chunk in chunks)) + collision = SimpleNamespace(update=lambda data: None, digest=lambda: b"collision") + with mock.patch("lab.prism.candidate_window.hashlib.sha256", return_value=collision): + self.assertTrue(disk_window_covers([value], [{"share_id": Text("a")}])) + self.assertFalse(disk_window_covers([value], [{"share_id": Text("b")}])) + + def test_cancel_during_long_field_stops_consumption(self) -> None: + calls = 0 + + def cancel() -> None: + nonlocal calls + calls += 1 + if calls == 3: + raise InterruptedError("cancelled") + + with self.assertRaises(InterruptedError): + for _chunk in copy_text_chunks(["x" * 50000], check=cancel): + pass + self.assertEqual(calls, 3) + + def test_copy_rejects_nul_after_a_bounded_prefix(self) -> None: + emitted = 0 + with self.assertRaisesRegex(ValueError, "share ID contains a NUL"): + for chunk in copy_text_chunks(["x" * (3 * COPY_TEXT_MAX_BYTES) + "\x00"]): + self.assertLessEqual(len(chunk), COPY_TEXT_MAX_BYTES) + emitted += 1 + self.assertGreater(emitted, 0) + + def test_tiny_records_still_have_a_work_bound(self) -> None: + consumed = 0 + at_check = 0 + + def values(): + nonlocal consumed + for _ in range(1300): + consumed += 1 + yield "" + + def check(): + nonlocal at_check + self.assertLessEqual(consumed - at_check, COPY_TEXT_MAX_RECORDS) + at_check = consumed + + for chunk in copy_text_chunks(values(), check=check): + self.assertLessEqual(len(chunk), COPY_TEXT_MAX_BYTES) + self.assertEqual(at_check, 1300) + + def test_record_graphs_release_without_cyclic_gc(self) -> None: + class Row(dict): + pass + + references = [] + + def rows(): + for index in range(2000): + row = Row(share_id=f"share-{index}", payload=bytearray(10000)) + references.append(weakref.ref(row)) + yield row + + enabled = gc.isenabled() + gc.disable() + try: + self.assertTrue(disk_window_covers(recorded_share_ids(rows()), [ + {"share_id": "share-1999"}, + ])) + self.assertTrue(all(reference() is None for reference in references)) + finally: + if enabled: + gc.enable() + + +class PayoutMembershipPortTests(unittest.TestCase): + def setUp(self) -> None: + from lab.prism.payout_state import PayoutStateService + + self.covers = mock.Mock(return_value=True) + self.aggregate = mock.Mock(side_effect=AssertionError("aggregate window read")) + self.runtime = SimpleNamespace( + ledger=SimpleNamespace( + candidate_window_covers=self.covers, audit_share_window=self.aggregate, + ), + _job_cache_lock=threading.RLock(), + _record_payout_artifact_event=mock.Mock(), + _incremental_payout_artifact_window=object(), + ) + self.service = PayoutStateService( + self.runtime, shutdown_error=RuntimeError, now_ms=lambda: 0, + ) + # This object cannot be materialized or iterated by the payout caller. + self.shares = object() + self.context = SimpleNamespace( + found_block={"anchor_job_issued_at_ms": "2000", "network_difficulty": "1000000"}, + shares_json=self.shares, + ) + + def test_bounded_port_receives_original_view_and_returns_omission_verdict(self) -> None: + for verdict in (True, False): + with self.subTest(verdict=verdict): + self.covers.return_value = verdict + self.assertIs(self.service._replayed_payout_window_reproducible(self.context), verdict) + self.covers.assert_called_with( + self.shares, anchor_job_issued_at_ms=2000, network_difficulty=1000000, + ) + self.aggregate.assert_not_called() + + def test_missing_anchor_fails_closed_before_any_read(self) -> None: + self.context.found_block.pop("anchor_job_issued_at_ms") + self.assertFalse(self.service._replayed_payout_window_reproducible(self.context)) + self.covers.assert_not_called() + self.aggregate.assert_not_called() + + def test_bounded_port_divergence_invalidates_mirror_and_fails_closed(self) -> None: + from lab.prism.share_ledger import DaemonWindowMirrorDivergence + + self.covers.side_effect = DaemonWindowMirrorDivergence("refuted mirror") + self.assertFalse(self.service._replayed_payout_window_reproducible(self.context)) + self.runtime._record_payout_artifact_event.assert_called_once_with("window_mirror_divergence") + self.assertIsNone(self.runtime._incremental_payout_artifact_window) + self.aggregate.assert_not_called() + + def test_bounded_port_timeout_propagates_for_retry_without_aggregate_fallback(self) -> None: + from lab.prism.share_ledger import LedgerOperationTimeout + + error = LedgerOperationTimeout("expired") + self.covers.side_effect = error + with self.assertRaises(LedgerOperationTimeout) as raised: + self.service._replayed_payout_window_reproducible(self.context) + self.assertIs(raised.exception, error) + self.aggregate.assert_not_called() + + +@unittest.skipUnless(os.environ.get("PRISM_TEST_DATABASE_URL"), "requires disposable PostgreSQL") +class CandidateWindowPostgresTests(unittest.TestCase): + """Run with an initialized disposable database, never a production DSN.""" + + def setUp(self) -> None: + import psycopg + from lab.prism.share_ledger import PsqlShareLedger + + self.connection = psycopg.connect(os.environ["PRISM_TEST_DATABASE_URL"], autocommit=True) + self.addCleanup(self.connection.close) + self.connection.execute("TRUNCATE qbit_share_ledger CASCADE") + self.ledger = PsqlShareLedger( + database_url=os.environ["PRISM_TEST_DATABASE_URL"], psql_command="psql", + native_client_mode="native", writer_id="candidate-window-tests", + ) + self.addCleanup(self.ledger.close) + self.addCleanup(self.ledger.release_writer_lease) + + def insert(self, share_id: str, *, accepted_at_ms: int = 1000) -> None: + self.connection.execute(""" + INSERT INTO qbit_share_ledger ( + share_id, miner_id, payout_order_key, p2mr_program, + share_difficulty, network_difficulty, template_height, job_id, + job_issued_at, ntime, accepted_at, credit_policy, accepted, + writer_id, writer_epoch + ) VALUES (%s, 'miner', 'miner', decode(repeat('aa', 32), 'hex'), + 1, 1000000, 1, 'job', to_timestamp(0), 0, + to_timestamp(%s::double precision / 1000.0), NULL, true, 'test', 1) + """, (share_id, accepted_at_ms)) + + def covers(self, rows) -> bool: + return self.ledger.candidate_window_covers( + rows, anchor_job_issued_at_ms=2000, network_difficulty=1000000, + ) + + def test_native_exact_subset_and_late_visible_append(self) -> None: + self.insert("a") + self.assertTrue(self.covers([{"share_id": "a"}, {"share_id": "extra"}])) + self.insert("late") + self.assertFalse(self.covers([{"share_id": "a"}])) + self.assertTrue(self.covers([{"share_id": "late"}, {"share_id": "a"}])) + + def test_native_copy_special_values_and_no_temporary_table_leak(self) -> None: + for share_id in ("\\.\nSELECT 1;", "\\N", "🌐\té\r\n", "extra"): + self.insert(share_id) + self.assertTrue(self.covers([ + {"share_id": value} + for value in ("extra", "\\.\nSELECT 1;", "🌐\té\r\n", "\\N") + ])) + with self.ledger._native.connection() as connection: + self.assertIsNone(connection.execute( + "SELECT to_regclass('pg_temp.qbit_candidate_recorded_ids')" + ).fetchone()[0]) + + def test_native_failure_rolls_back_private_copy_and_can_retry(self) -> None: + self.insert("a") + + def failing_rows(): + yield {"share_id": "a"} + raise InterruptedError("stop") + + with self.assertRaises(InterruptedError): + self.covers(failing_rows()) + self.assertTrue(self.covers([{"share_id": "a"}])) + + def test_native_expired_deadline_does_not_publish_partial_answer(self) -> None: + from lab.prism.share_ledger import LedgerOperationTimeout + + self.insert("a") + with self.ledger.operation_timeout(1): + self.ledger._operation_timeout_local.deadline = self.ledger._monotonic() - 1 + with self.assertRaises(LedgerOperationTimeout): + self.covers([{"share_id": "a"}]) + self.assertTrue(self.covers([{"share_id": "a"}])) + + def test_native_nul_failure_rolls_back_and_can_retry(self) -> None: + self.insert("a") + with self.assertRaisesRegex(ValueError, "share ID contains a NUL"): + self.covers([ + {"share_id": "a"}, + {"share_id": "x" * (3 * COPY_TEXT_MAX_BYTES) + "\x00"}, + ]) + self.assertTrue(self.covers([{"share_id": "a"}])) + + def assert_streaming_timeout_and_retry(self, share_id: str) -> None: + from lab.prism.share_ledger import LedgerOperationTimeout + + with self.ledger.operation_timeout(0.1): + with self.assertRaises(LedgerOperationTimeout): + self.covers(itertools.repeat({"share_id": share_id})) + self.assertEqual(self.connection.execute(""" + SELECT count(*) FROM pg_class c JOIN pg_namespace n ON c.relnamespace = n.oid + WHERE n.nspname LIKE 'pg_temp_%' AND c.relname = 'qbit_candidate_recorded_ids' + """).fetchone()[0], 0) + self.assertEqual(self.connection.execute(""" + SELECT count(*) FROM pg_stat_activity + WHERE datname = current_database() AND state = 'idle in transaction' + """).fetchone()[0], 0) + self.assertTrue(self.covers([{"share_id": share_id}])) + + def test_native_deadline_expires_while_streaming(self) -> None: + self.insert("a") + self.assert_streaming_timeout_and_retry("a") + + def test_native_slow_consumer_applies_backpressure_and_rolls_back(self) -> None: + from lab.prism import candidate_window + from lab.prism.share_ledger import LedgerOperationTimeout + + self.insert("a") + create = candidate_window._CREATE + """ +CREATE FUNCTION pg_temp.slow_candidate_copy() RETURNS trigger LANGUAGE plpgsql AS $$ +BEGIN PERFORM pg_sleep(0.02); RETURN NEW; END $$; +CREATE TRIGGER slow_candidate_copy BEFORE INSERT ON pg_temp.qbit_candidate_recorded_ids +FOR EACH ROW EXECUTE FUNCTION pg_temp.slow_candidate_copy(); +""" + factory = candidate_window._flushing_copy_writer + longest_write = 0.0 + maximum_rss = 0 + + def rss() -> int: + status = Path("/proc/self/status") + if not status.exists(): + return 0 + for line in status.read_text().splitlines(): + if line.startswith("VmRSS:"): + return int(line.split()[1]) * 1024 + return 0 + + initial_rss = rss() + + def instrument(cursor): + writer = factory(cursor) + write = writer.write + + def measured(data): + nonlocal longest_write, maximum_rss + started = time.monotonic() + try: + return write(data) + finally: + longest_write = max(longest_write, time.monotonic() - started) + maximum_rss = max(maximum_rss, rss()) + + writer.write = measured + return writer + + # Reuse one record so the only potential window-sized allocation is + # inside the transport. The stock Linux writer buffers this 64 MiB + # stream without blocking when the trigger stalls the backend. + rows = itertools.repeat({"share_id": "x" * 4095}, 16384) + with mock.patch.object(candidate_window, "_CREATE", create), mock.patch.object( + candidate_window, "_flushing_copy_writer", instrument, + ), self.ledger.operation_timeout(1.5): + with self.assertRaises(LedgerOperationTimeout): + self.covers(rows) + self.assertGreater(longest_write, 0.1) + if initial_rss: + self.assertLess(maximum_rss - initial_rss, 32 * 1024 * 1024) + self.assertEqual(self.connection.execute(""" + SELECT count(*) FROM pg_class c JOIN pg_namespace n ON c.relnamespace = n.oid + WHERE n.nspname LIKE 'pg_temp_%' AND c.relname = 'qbit_candidate_recorded_ids' + """).fetchone()[0], 0) + self.assertEqual(self.connection.execute(""" + SELECT count(*) FROM pg_stat_activity + WHERE datname = current_database() AND state = 'idle in transaction' + """).fetchone()[0], 0) + self.assertTrue(self.covers([{"share_id": "a"}])) + + @unittest.skipUnless( + shutil.which("psql") or os.environ.get("PRISM_TEST_PSQL_COMMAND_WITH_ENV"), + "requires PostgreSQL client", + ) + def test_psql_stream_roundtrip_and_failure(self) -> None: + # Use the existing instance's guards/gates and an independent psql + # connection. The writer lease is not read or updated by this call. + native = self.ledger._native + old_command = self.ledger._command + self.ledger._native = None + self.ledger._command = shlex.split(os.environ.get( + "PRISM_TEST_PSQL_COMMAND_WITH_ENV", + "psql " + shlex.quote(os.environ["PRISM_TEST_DATABASE_URL"]), + )) + try: + value = "\\.\nSELECT 1;🌐\t\\N" + self.insert(value) + self.assertTrue(self.covers([{"share_id": value}, {"share_id": "extra"}])) + self.assertFalse(self.covers([{"share_id": "extra"}])) + with self.assertRaisesRegex(ValueError, "share ID contains a NUL"): + self.covers([{"share_id": value + "\x00"}]) + self.assertTrue(self.covers([{"share_id": value}])) + self.assert_streaming_timeout_and_retry(value) + self.assertEqual(self.connection.execute( + "SELECT count(*) FROM qbit_share_ledger" + ).fetchone()[0], 1) + finally: + self.ledger._native = native + self.ledger._command = old_command + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_prism_completion_callbacks.py b/tests/test_prism_completion_callbacks.py new file mode 100644 index 000000000..b5fdf9948 --- /dev/null +++ b/tests/test_prism_completion_callbacks.py @@ -0,0 +1,238 @@ +#!/usr/bin/env python3 +"""A completed future must not keep obsolete callback owners alive.""" + +from __future__ import annotations + +from concurrent.futures import Future +from contextlib import redirect_stderr, redirect_stdout +import gc +import io +import threading +from types import SimpleNamespace +import unittest +import weakref + +from lab.prism.future_callbacks import add_releasing_done_callback +from lab.prism.job_delivery import JobDeliveryService, PendingInitialJob +from lab.prism.reorg_reconciler import ReorgReconcilerService + + +class _Callback: + def __init__(self, calls: list[str], name: str) -> None: + self.calls = calls + self.name = name + + def __call__(self, future: Future) -> None: + assert future.done() + self.calls.append(self.name) + + +class CompletionCallbackTests(unittest.TestCase): + def test_pending_callbacks_stay_alive_and_complete_in_registration_order(self) -> None: + future = Future() + calls: list[str] = [] + references = [] + for name in ("first", "second"): + callback = _Callback(calls, name) + references.append(weakref.ref(callback)) + add_releasing_done_callback(future, callback) + del callback + self.assertTrue(all(ref() is not None for ref in references)) + future.set_result(17) + self.assertEqual(calls, ["first", "second"]) + self.assertTrue(all(ref() is None for ref in references)) + # The caller can keep the result future without keeping its callbacks. + self.assertEqual(future.result(), 17) + + def test_finished_future_invokes_and_releases_callback_inline(self) -> None: + future = Future() + future.set_result(17) + calls: list[str] = [] + callback = _Callback(calls, "inline") + reference = weakref.ref(callback) + add_releasing_done_callback(future, callback) + del callback + self.assertEqual(calls, ["inline"]) + self.assertIsNone(reference()) + + +class InitialDeliveryCallbackRetentionTests(unittest.TestCase): + def test_failure_handler_preserves_error_without_capturing_request(self) -> None: + gc.collect() + was_enabled = gc.isenabled() + gc.disable() + try: + references = self.finish_failed_request() + self.assertTrue(all(ref() is None for ref in references)) + finally: + if was_enabled: + gc.enable() + gc.collect() + + def finish_failed_request(self): + class Client: + connection_id = 17 + + client = Client() + request = PendingInitialJob( + client=client, + authorization_generation=1, + worker=None, + requested_monotonic=0, + deadline_monotonic=None, + ) + future = Future() + error = RuntimeError("initial build failed") + future.set_exception(error) + request.future = future + pending = {client: request} + disconnected = [] + runtime = SimpleNamespace( + lock=threading.Lock(), + job_build_failure_count=0, + disconnect_client=lambda peer: disconnected.append(peer.connection_id), + ) + owner = SimpleNamespace( + _runtime=runtime, + _initial_job_admission_lock=threading.Lock(), + _pending_initial_jobs=pending, + initial_job_failed_count=0, + ) + output, errors = io.StringIO(), io.StringIO() + with redirect_stdout(output), redirect_stderr(errors): + JobDeliveryService._initial_job_future_finished(owner, request, future) + self.assertEqual(runtime.job_build_failure_count, 1) + self.assertEqual(owner.initial_job_failed_count, 1) + self.assertEqual(disconnected, [17]) + self.assertFalse(pending) + self.assertIn("connection=17", output.getvalue()) + self.assertIn("RuntimeError: initial build failed", errors.getvalue()) + self.assertIs(future.exception(), error) + self.assertIsNone(error.__traceback__) + return [weakref.ref(obj) for obj in (client, request, future)] + + def check_releases_request(self, *, cancel: bool, already_done: bool) -> None: + gc.collect() + was_enabled = gc.isenabled() + gc.disable() + try: + references = self.complete_request(cancel=cancel, already_done=already_done) + self.assertEqual( + {name: ref() is not None for name, ref in references.items()}, + dict.fromkeys(references, False), + ) + finally: + if was_enabled: + gc.enable() + gc.collect() + + def complete_request(self, *, cancel: bool, already_done: bool): + class Client: + pass + + client = Client() + request = PendingInitialJob( + client=client, + authorization_generation=1, + worker=None, + requested_monotonic=0, + deadline_monotonic=None, + ) + future = Future() + pending = {client: request} + calls = [] + + def finish(completed_request, completed_future): + self.assertIs(completed_request, pending.pop(completed_request.client)) + self.assertIs(completed_request.future, completed_future) + calls.append(True) + + runtime = SimpleNamespace( + _submit_delivery_task=lambda *args, **kwargs: future, + initial_job_executor=lambda: None, + _run_initial_job=lambda *args: None, + _initial_job_future_finished=finish, + ) + owner = SimpleNamespace( + _runtime=runtime, + _initial_job_admission_lock=threading.Lock(), + _pending_initial_jobs=pending, + ) + references = { + "client": weakref.ref(client), + "request": weakref.ref(request), + "future": weakref.ref(future), + } + if already_done: + future.set_result(True) + self.assertTrue(JobDeliveryService._submit_initial_job_request(owner, request)) + if cancel: + self.assertTrue(future.cancel()) + elif not already_done: + future.set_result(True) + self.assertEqual(calls, [True]) + self.assertFalse(pending) + return references + + def test_completed_request_releases_disconnected_client(self) -> None: + self.check_releases_request(cancel=False, already_done=False) + + def test_cancelled_request_releases_disconnected_client(self) -> None: + self.check_releases_request(cancel=True, already_done=False) + + def test_already_completed_request_releases_disconnected_client(self) -> None: + self.check_releases_request(cancel=False, already_done=True) + + +class StalePrefetchCallbackRetentionTests(unittest.TestCase): + def check_releases_prefetch(self, *, already_done: bool, outcome: str) -> None: + gc.collect() + was_enabled = gc.isenabled() + gc.disable() + try: + reference = self.discard_prefetch(already_done=already_done, outcome=outcome) + self.assertIsNone(reference()) + finally: + if was_enabled: + gc.enable() + gc.collect() + + def discard_prefetch(self, *, already_done: bool, outcome: str): + future = Future() + error = KeyboardInterrupt("prefetch failed") + + def complete(): + if outcome == "failed": + future.set_exception(error) + elif outcome == "cancelled": + future.cancel() + else: + future.set_result(True) + + if already_done: + complete() + ReorgReconcilerService.discard_stale_prefetch(future) + if not already_done: + complete() + if outcome == "failed": + self.assertIs(future.exception(), error) + # Observing an obsolete error must not add a new traceback whose + # callback frame points back to the discarded future. + self.assertIsNone(error.__traceback__) + return weakref.ref(future) + + def test_failed_pending_prefetch_does_not_capture_consumer_frame(self) -> None: + self.check_releases_prefetch(already_done=False, outcome="failed") + + def test_failed_finished_prefetch_does_not_capture_consumer_frame(self) -> None: + self.check_releases_prefetch(already_done=True, outcome="failed") + + def test_cancelled_prefetch_is_discarded(self) -> None: + self.check_releases_prefetch(already_done=False, outcome="cancelled") + + def test_successful_prefetch_is_discarded(self) -> None: + self.check_releases_prefetch(already_done=False, outcome="succeeded") + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_prism_compose_profile.py b/tests/test_prism_compose_profile.py index d3dc475db..494301c46 100644 --- a/tests/test_prism_compose_profile.py +++ b/tests/test_prism_compose_profile.py @@ -8,7 +8,9 @@ import subprocess import unittest from pathlib import Path -from typing import Any +from unittest.mock import patch + +from lab.prism import public_read_service ROOT = Path(__file__).resolve().parents[1] @@ -39,6 +41,7 @@ def setUpClass(cls) -> None: "QBIT_NODE_EXTRA_ARG": "-listen=1", "PRISM_STRATUM_PORT": "43340", "PRISM_STRATUM_PORT_HOST": "127.0.0.1:43340", + "PRISM_STRATUM_HIGHDIFF_PORT": "44334", "PRISM_PUBLIC_STRATUM_URL": "stratum+tcp://public-pool.example:3335", "PRISM_PUBLIC_STRATUM_HIGHDIFF_URL": "stratum+tcp://public-pool.example:4334", "PRISM_PUBLIC_POOL_FEE_BPS": "200", @@ -61,9 +64,11 @@ def setUpClass(cls) -> None: "PRISM_CTV_BROADCAST_RETRY_BACKOFF_SECONDS": "17", "PRISM_BLOCKWAIT_ENABLED": "0", "PRISM_BLOCKWAIT_TIMEOUT_SECONDS": "13", + "PRISM_BLOCK_REPLAY_PAGE_SIZE": "1", "PRISM_COORDINATION_BLOCKED_EXIT_SECONDS": "37", "PRISM_HEALTH_PENDING_REFRESH_MAX_AGE_SECONDS": "23", "PRISM_HEALTH_TIP_POLL_MAX_AGE_SECONDS": "29", + "PRISM_METRICS_REFRESH_SECONDS": "17", "PRISM_STRATUM_STALE_GRACE_SECONDS": "4", "PRISM_STRATUM_SAME_TIP_JOB_RETENTION_SECONDS": "31", "PRISM_STRATUM_SAME_TIP_JOB_RETENTION_PER_CONNECTION": "65", @@ -149,16 +154,24 @@ def test_prism_coordinator_gets_required_environment(self) -> None: self.assertEqual(env["PRISM_DATABASE_URL"], "postgresql://qbit:change-this@prism-postgres:5432/qbit") self.assertEqual(env["PRISM_POSTGRES_INIT_SCHEMA"], "1") self.assertEqual(env["PRISM_POSTGRES_READ_CONCURRENCY"], "4") + self.assertEqual(env["PRISM_POSTGRES_IDLE_IN_TRANSACTION_TIMEOUT_SECONDS"], "15") + self.assertEqual(env["PRISM_POSTGRES_TCP_KEEPALIVES_IDLE_SECONDS"], "30") + self.assertEqual(env["PRISM_POSTGRES_TCP_KEEPALIVES_INTERVAL_SECONDS"], "10") + self.assertEqual(env["PRISM_POSTGRES_TCP_KEEPALIVES_COUNT"], "3") self.assertEqual(env["PRISM_LEDGER_LEASE_TTL_SECONDS"], "60") + self.assertEqual(env["PRISM_LEDGER_LEASE_ACQUIRE_LOCK_TIMEOUT_SECONDS"], "5") + self.assertEqual(env["PRISM_LEDGER_LEASE_ACQUIRE_ATTEMPTS"], "5") self.assertEqual(env["PRISM_WRITER_QUIESCENCE_TIMEOUT_SECONDS"], "9") self.assertEqual(env["PRISM_WATCHDOG_ENABLED"], "1") self.assertEqual(env["PRISM_WATCHDOG_TIMEOUT_SECONDS"], "120") self.assertEqual(env["PRISM_WATCHDOG_INTERVAL_SECONDS"], "15") self.assertEqual(env["PRISM_BLOCKWAIT_ENABLED"], "0") self.assertEqual(env["PRISM_BLOCKWAIT_TIMEOUT_SECONDS"], "13") + self.assertEqual(env["PRISM_BLOCK_REPLAY_PAGE_SIZE"], "1") self.assertEqual(env["PRISM_COORDINATION_BLOCKED_EXIT_SECONDS"], "37") self.assertEqual(env["PRISM_HEALTH_PENDING_REFRESH_MAX_AGE_SECONDS"], "23") self.assertEqual(env["PRISM_HEALTH_TIP_POLL_MAX_AGE_SECONDS"], "29") + self.assertEqual(env["PRISM_METRICS_REFRESH_SECONDS"], "17") self.assertEqual(env["PRISM_STRATUM_STALE_GRACE_SECONDS"], "4") self.assertEqual(env["PRISM_STRATUM_SAME_TIP_JOB_RETENTION_SECONDS"], "31") self.assertEqual( @@ -182,22 +195,7 @@ def test_prism_coordinator_gets_required_environment(self) -> None: self.assertEqual(env["PRISM_PAYOUT_ADDRESS_CACHE_TTL_SECONDS"], "1800") self.assertEqual(env["PRISM_STRATUM_BIND"], "0.0.0.0") self.assertEqual(env["PRISM_STRATUM_PORT"], "43340") - self.assertEqual(env["PRISM_PUBLIC_STRATUM_URL"], "stratum+tcp://public-pool.example:3335") - self.assertEqual(env["PRISM_PUBLIC_STRATUM_HIGHDIFF_URL"], "stratum+tcp://public-pool.example:4334") - self.assertEqual(env["PRISM_PUBLIC_POOL_FEE_BPS"], "200") - self.assertEqual(env["PRISM_PUBLIC_CACHE_ENABLED"], "1") - self.assertEqual(env["PRISM_PUBLIC_CACHE_TTL_SECONDS"], "5") - self.assertEqual(env["PRISM_PUBLIC_CACHE_STALE_WHILE_REVALIDATE_SECONDS"], "30") - self.assertEqual(env["PRISM_PUBLIC_AGGREGATE_CACHE_TTL_SECONDS"], "30") - self.assertEqual(env["PRISM_PUBLIC_AGGREGATE_CACHE_STALE_WHILE_REVALIDATE_SECONDS"], "30") self.assertEqual(env["PRISM_PUBLIC_REWARD_WINDOW_CACHE_SECONDS"], "30") - self.assertEqual(env["PRISM_PUBLIC_CONFIG_CACHE_TTL_SECONDS"], "300") - self.assertEqual(env["PRISM_PUBLIC_CONFIG_CACHE_STALE_WHILE_REVALIDATE_SECONDS"], "3600") - self.assertEqual(env["PRISM_PUBLIC_ARTIFACT_CACHE_TTL_SECONDS"], "86400") - self.assertEqual(env["PRISM_PUBLIC_ARTIFACT_CACHE_STALE_WHILE_REVALIDATE_SECONDS"], "86400") - self.assertEqual(env["PRISM_PUBLIC_CACHE_MAX_ENTRIES"], "512") - self.assertEqual(env["PRISM_PUBLIC_CACHE_MAX_RESPONSE_BYTES"], "1048576") - self.assertEqual(env["PRISM_PUBLIC_CACHE_DEBUG_HEADERS"], "0") self.assertEqual(env["PRISM_AUDIT_BIND"], "127.0.0.1") self.assertEqual(env["PRISM_AUDIT_PORT"], "3341") self.assertEqual(env["PRISM_AUDIT_SHARE_SEGMENT_SIZE"], "10000") @@ -277,6 +275,278 @@ def test_prism_coordinator_descriptor_limit_is_configurable(self) -> None: self.assertEqual(nofile, {"soft": 60000, "hard": 65000}) + # -- the extracted public read tier (issue #145) ------------------------ + + def test_public_api_runs_the_public_read_service_from_the_prism_image(self) -> None: + service = self.config["services"]["prism-public-api"] + + self.assertEqual(service["command"], ["python3", "-m", "lab.prism.public_read_service"]) + self.assertEqual( + service["build"]["dockerfile"], + "lab/prism/Dockerfile", + ) + self.assertEqual( + service["image"], + self.config["services"]["prism-coordinator"]["image"], + ) + self.assertEqual(service.get("restart"), "on-failure") + + def test_public_api_survives_coordinator_restarts(self) -> None: + # The public tier must not depend on the coordinator: independence from + # coordinator restarts is a large part of why the surface was extracted. + # It waits on the standby it reads, which is also not the coordinator's + # primary -- so a primary restart does not take the dashboard with it. + depends_on = self.config["services"]["prism-public-api"].get("depends_on", {}) + + self.assertIn("prism-postgres-replica", depends_on) + self.assertEqual( + depends_on["prism-postgres-replica"]["condition"], + "service_healthy", + ) + self.assertNotIn("prism-coordinator", depends_on) + + def test_public_api_mounts_the_audit_volume_read_only(self) -> None: + volumes = self.config["services"]["prism-public-api"].get("volumes", []) + audit = [ + volume + for volume in volumes + if isinstance(volume, dict) and volume.get("source") == "prism-audit-data" + ] + + self.assertEqual(1, len(audit), f"expected one audit mount in {volumes!r}") + self.assertTrue( + audit[0].get("read_only"), + f"the audit artifact volume must be mounted read-only: {audit[0]!r}", + ) + + def test_public_api_receives_the_moved_public_knobs(self) -> None: + env = self._service_environment("prism-public-api") + + self.assertEqual(env["PRISM_PUBLIC_STRATUM_URL"], "stratum+tcp://public-pool.example:3335") + self.assertEqual(env["PRISM_PUBLIC_STRATUM_HIGHDIFF_URL"], "stratum+tcp://public-pool.example:4334") + self.assertEqual(env["PRISM_PUBLIC_POOL_FEE_BPS"], "200") + self.assertEqual(env["PRISM_PUBLIC_CACHE_ENABLED"], "1") + self.assertEqual(env["PRISM_PUBLIC_CACHE_TTL_SECONDS"], "5") + self.assertEqual(env["PRISM_PUBLIC_AGGREGATE_CACHE_TTL_SECONDS"], "30") + self.assertEqual(env["PRISM_PUBLIC_CONFIG_CACHE_TTL_SECONDS"], "300") + self.assertEqual(env["PRISM_PUBLIC_ARTIFACT_CACHE_TTL_SECONDS"], "86400") + self.assertEqual(env["PRISM_PUBLIC_CACHE_MAX_ENTRIES"], "512") + self.assertEqual(env["PRISM_PUBLIC_CACHE_DEBUG_HEADERS"], "0") + self.assertEqual( + self._public_database_url(), + "postgresql://qbit:change-this@prism-postgres-replica:5432/qbit", + ) + self.assertEqual(env["PRISM_AUDIT_DIR"], "/var/lib/qbit-prism/audit") + self.assertEqual(env["QBIT_RPC_HOST"], "qbitd") + + def test_public_api_receives_the_advertised_stratum_ports(self) -> None: + # The service runs no Stratum listener, but mining-configuration still + # renders from these: PRISM_STRATUM_PORT is the primary endpoint's + # fallback port and the High-diff endpoint appears only when + # PRISM_STRATUM_HIGHDIFF_PORT is set. Without the pass-through the + # rendered body silently drops the high-diff listener the coordinator + # is running. + env = self._service_environment("prism-public-api") + + self.assertEqual(env["PRISM_STRATUM_PORT"], "43340") + self.assertEqual(env["PRISM_STRATUM_HIGHDIFF_PORT"], "44334") + + def test_the_coordinator_no_longer_carries_the_moved_public_knobs(self) -> None: + # The coordinator serves no route that reads these any more; leaving + # them behind would imply it still did. + env = self._service_environment("prism-coordinator") + + for name in ( + "PRISM_PUBLIC_STRATUM_URL", + "PRISM_PUBLIC_POOL_NAME", + "PRISM_PUBLIC_CACHE_ENABLED", + "PRISM_PUBLIC_CACHE_TTL_SECONDS", + "PRISM_PUBLIC_ARTIFACT_CACHE_TTL_SECONDS", + "PRISM_PUBLIC_EXPLORER_TX_URL_PREFIX", + ): + with self.subTest(name=name): + self.assertNotIn(name, env) + + def test_the_reward_window_cache_knob_stays_on_both(self) -> None: + # Read by the ledger itself, and both processes construct one. + for service in ("prism-coordinator", "prism-public-api"): + with self.subTest(service=service): + env = self._service_environment(service) + self.assertEqual(env["PRISM_PUBLIC_REWARD_WINDOW_CACHE_SECONDS"], "30") + + def test_public_api_never_receives_writer_or_signing_configuration(self) -> None: + # A read tier that could acquire a writer lease or sign a manifest + # would not be a read tier. + env = self._service_environment("prism-public-api") + + self.assertEqual(env.get("PRISM_ALLOW_MEMORY_LEDGER"), "0") + for name in env: + self.assertFalse( + name.startswith("PRISM_LEDGER_WRITER_"), + f"{name} must not reach the public read tier", + ) + for name in ( + "PRISM_MANIFEST_SIGNING_SEED_HEX", + "PRISM_LEDGER_ATTESTATION_SIGNING_SEED_HEX", + "PRISM_POSTGRES_INIT_SCHEMA", + ): + with self.subTest(name=name): + self.assertNotIn(name, env) + + def test_public_api_publishes_its_port_and_healthcheck(self) -> None: + service = self.config["services"]["prism-public-api"] + published = { + str(port.get("published")) + for port in service.get("ports", []) + if isinstance(port, dict) + } + self.assertIn("3342", published) + + command = " ".join(service["healthcheck"]["test"]) + self.assertIn("/healthz", command) + self.assertIn("urllib.request.urlopen", command) + + def test_primary_pins_the_hba_file_carrying_the_replication_rules(self) -> None: + postgres = self.config["services"]["prism-postgres"] + + self.assertEqual( + postgres["command"], + ["postgres", "-c", "hba_file=/etc/postgresql/pg_hba.conf"], + ) + mounts = [ + volume + for volume in postgres.get("volumes", []) + if isinstance(volume, dict) + and volume.get("target") == "/etc/postgresql/pg_hba.conf" + ] + self.assertEqual( + len(mounts), 1, f"missing pg_hba mount in {postgres.get('volumes')!r}" + ) + self.assertTrue(mounts[0].get("read_only", False)) + self.assertEqual( + mounts[0].get("source"), + str(ROOT / "config/prism-postgres/pg_hba.conf"), + ) + + def test_primary_hba_file_authorizes_replication_and_keeps_local_access( + self, + ) -> None: + rules = [ + line.split() + for line in (ROOT / "config/prism-postgres/pg_hba.conf") + .read_text(encoding="utf-8") + .splitlines() + if line.strip() and not line.lstrip().startswith("#") + ] + + # The image's initdb bootstrap, pg_isready and `docker exec psql` all + # come in over the unix socket, and this file replaces the generated + # one, so losing the local rule would break the existing healthcheck. + self.assertEqual(rules[0][:4], ["local", "all", "all", "trust"]) + self.assertIn(["host", "all", "all", "127.0.0.1/32", "scram-sha-256"], rules) + self.assertIn(["host", "all", "all", "::1/128", "scram-sha-256"], rules) + self.assertIn(["host", "all", "all", "0.0.0.0/0", "scram-sha-256"], rules) + self.assertIn(["host", "all", "all", "::/0", "scram-sha-256"], rules) + # Compose cannot interpolate this file, so the replication rules name + # the default superuser literally. + self.assertIn( + ["host", "replication", "qbit", "0.0.0.0/0", "scram-sha-256"], rules + ) + self.assertIn(["host", "replication", "qbit", "::/0", "scram-sha-256"], rules) + + def test_replica_bootstraps_from_the_primary(self) -> None: + replica = self.config["services"]["prism-postgres-replica"] + env = self._service_environment("prism-postgres-replica") + + self.assertEqual(replica["image"], "postgres:16-alpine") + self.assertEqual( + replica["depends_on"]["prism-postgres"]["condition"], "service_healthy" + ) + self.assertEqual( + replica["entrypoint"], ["/usr/local/bin/prism-replica-entrypoint.sh"] + ) + self.assertEqual( + env["PRISM_POSTGRES_REPLICATION_SLOT"], "prism_public_replica" + ) + # The data directory is cloned by pg_basebackup, so these credentials + # exist for the replication connection and the healthcheck's psql. + self.assertEqual(env["POSTGRES_USER"], "qbit") + self.assertEqual(env["POSTGRES_DB"], "qbit") + self.assertEqual(env["PGPASSWORD"], env["POSTGRES_PASSWORD"]) + + targets = { + volume.get("target"): volume + for volume in replica.get("volumes", []) + if isinstance(volume, dict) + } + self.assertEqual( + targets["/var/lib/postgresql/data"].get("source"), + "prism-postgres-replica-data", + ) + self.assertEqual( + targets["/usr/local/bin/prism-replica-entrypoint.sh"].get("source"), + str(ROOT / "config/prism-postgres/replica-entrypoint.sh"), + ) + + def test_replica_bootstrap_script_retries_and_creates_the_slot(self) -> None: + script = (ROOT / "config/prism-postgres/replica-entrypoint.sh").read_text( + encoding="utf-8" + ) + + self.assertIn("pg_basebackup", script) + self.assertIn("--wal-method=stream", script) + self.assertIn("--write-recovery-conf", script) + self.assertIn('--slot="$REPLICATION_SLOT"', script) + self.assertIn("--create-slot", script) + # A cold `docker compose up` must survive the primary not being + # reachable yet, on a bounded budget rather than forever. + self.assertIn("PRISM_POSTGRES_REPLICA_BASEBACKUP_ATTEMPTS:-60", script) + self.assertIn("PRISM_POSTGRES_REPLICA_BASEBACKUP_RETRY_SECONDS:-5", script) + # Hand off to the stock entrypoint, which skips init on a populated + # data directory and drops privileges to the postgres account. + self.assertIn("exec docker-entrypoint.sh postgres", script) + # A bind mount aimed by mistake at a real cluster must stop the script + # rather than be cleared by the incomplete-bootstrap path below it. + self.assertIn("pg_controldata", script) + + def test_replica_healthcheck_requires_a_live_standby(self) -> None: + command = " ".join( + self.config["services"]["prism-postgres-replica"]["healthcheck"]["test"] + ) + + # An out-of-band promotion must report unhealthy rather than quietly + # become a second writable cluster behind the public read tier. + self.assertIn("pg_is_in_recovery()", command) + self.assertIn("pg_isready", command) + + def test_public_read_service_reads_the_replica_not_the_primary(self) -> None: + env = self._service_environment("prism-public-api") + + self.assertEqual(env["PRISM_DATABASE_URL"], "") + self.assertEqual(env["PRISM_PUBLIC_DATABASE_HOST"], "prism-postgres-replica") + database_url = self._public_database_url() + self.assertIn("prism-postgres-replica", database_url) + self.assertNotIn("@prism-postgres:", database_url) + self.assertEqual(env["PRISM_PUBLIC_REPLICA_MODE"], "require") + depends_on = self.config["services"]["prism-public-api"]["depends_on"] + self.assertEqual({"prism-postgres-replica"}, set(depends_on)) + self.assertEqual( + "service_healthy", depends_on["prism-postgres-replica"]["condition"] + ) + + # The coordinator keeps the primary; the split is the point. + coordinator = self._service_environment("prism-coordinator") + self.assertIn("@prism-postgres:", coordinator["PRISM_DATABASE_URL"]) + + def _public_database_url(self) -> str: + # Inspect the DSN consumed by both ledger backends after runtime encoding. + env = self._service_environment("prism-public-api") + with patch.dict(os.environ, env, clear=True), patch.object( + public_read_service, "PsqlShareLedger" + ) as ledger, patch.object(public_read_service, "build_audit_artifact_store", return_value=None): + public_read_service.build_ledger_from_env(env) + return ledger.call_args.kwargs["database_url"] + def _service_environment(self, name: str) -> dict[str, str]: raw_env = self.config["services"][name]["environment"] if isinstance(raw_env, dict): diff --git a/tests/test_prism_concurrency_harness.py b/tests/test_prism_concurrency_harness.py new file mode 100644 index 000000000..3fab3c18e --- /dev/null +++ b/tests/test_prism_concurrency_harness.py @@ -0,0 +1,992 @@ +#!/usr/bin/env python3 +"""Self-tests for the deterministic concurrency harness. + +A harness that lies is worse than no harness: every scenario built on it +inherits the lie, and a green suite then argues *against* fixing a real +defect. These tests pin the three properties the lease scenarios depend on. + +1. **Classification is anchored to production SQL.** ``FakePostgres`` answers + statements by recognising them, so a statement it recognises wrongly is a + silent divergence. The classifier is driven from real ``PsqlShareLedger`` + calls, and anything unrecognised raises rather than returning a plausible + answer. + +2. **The ports match production.** The fakes stand in for + ``_NativePostgresClient`` and ``_NativePostgresLeaseGuard``; if those + signatures move, the fakes must break loudly rather than silently stop + modelling something. + +3. **Order is an input, not an accident.** The scheduler must produce an + identical trace every run, must refuse to step an actor that cannot + progress, and must fail fast when an actor blocks on something the + harness does not model instead of hanging the suite. +""" + +from __future__ import annotations + +import inspect +import threading +import unittest +from typing import Any + +from lab.prism import share_ledger as share_ledger_module +from lab.prism.share_ledger import ( + DEFAULT_LEASE_ACQUIRE_LOCK_TIMEOUT_SECONDS, + PendingShare, +) +from tests.prism_concurrency_harness import ( + HarnessError, + acquire_result_arms, + acquire_terminal_arm_keys, + LandingOp, + LeaseHarness, + LeaseOp, + NotRunnable, + SchedulerStall, + UnsupportedStatement, + VirtualClock, + advisory_lock_pair, + assert_deterministic, + classify, +) + + +#: The explicit not-acquired/retry terminal arm, captured verbatim from the +#: signed ledger change that introduces it (PR #168, commit 869bb7e). That +#: reference is documentation of where these bytes came from and nothing more: +#: this suite never reads that commit, and must not. A test that shells out to +#: git is a test that fails for reasons which have nothing to do with the tree +#: under test -- a shallow clone, a pruned object, an offline runner -- and one +#: that can fetch on demand makes the suite's result depend on the network. +#: The fixture is checked in so the model is pinned against a fixed input. +#: +#: Re-capture by hand if the ledger change re-words the arm; the constants it +#: interpolates are WRITER_LEASE_ACQUIRE_RETRY_KEY and +#: WRITER_LEASE_ACQUIRE_RETRY_SUBJECT. +LEDGER_RETRY_TERMINAL_ARM = """, + json_build_object( + 'acquired', false, + 'lease_snapshot_retry', true, + 'lease', 'qbit ledger writer lease', + 'retry_reason', + 'the qbit ledger writer lease row was committed by a concurrent first acquisition after this statement snapshot' + ) +);""" + +#: What the model must report for that arm. Written out independently of the +#: fixture above rather than derived from parsing it: deriving both sides from +#: one parse would make the assertion agree with itself no matter what the +#: parse did. +LEDGER_RETRY_TERMINAL_OBJECT = { + "acquired": False, + "lease_snapshot_retry": True, + "lease": "qbit ledger writer lease", + "retry_reason": ( + "the qbit ledger writer lease row was committed by a concurrent " + "first acquisition after this statement snapshot" + ), +} + + +def _without_ledger_retry_terminal_arm(sql: str) -> str: + """Return the historical two-arm acquire shape from current shipped SQL.""" + stripped = sql.rstrip() + if not stripped.endswith(LEDGER_RETRY_TERMINAL_ARM): + raise AssertionError("shipped acquire SQL does not carry the pinned terminal arm") + return stripped[: -len(LEDGER_RETRY_TERMINAL_ARM)] + "\n);" + + +def _source_of(member: Any) -> str: + """Return a member's source, or an empty string when it has none.""" + try: + return inspect.getsource(member) + except (OSError, TypeError): + return "" + + +class StatementClassificationTests(unittest.TestCase): + """The classifier is pinned to the SQL PsqlShareLedger actually emits.""" + + def test_every_lease_statement_is_recognised_from_real_calls(self) -> None: + """Drive each lease operation through production code and classify it. + + This is the pin that matters: the fragments in ``_SIGNATURES`` are + lifted from ``share_ledger.py``, and this test proves they still + match what that module generates rather than what it generated when + the harness was written. + """ + with LeaseHarness() as harness: + alpha = harness.coordinator("alpha") + alpha.start() + harness.run_until(alpha, "done:startup") + + alpha.call(alpha.ledger.renew_writer_lease, label="renew") + alpha.call( + alpha.ledger.verify_writer_lease_guard_session, + label="verify", + ) + alpha.call(alpha.ledger.release_writer_lease, label="release") + + # A second coordinator, to pin the acquire path a successor takes. + # This one finds the lease released and takes it through the + # ordinary expiry CAS rather than adopting; the adoption CAS is + # pinned separately by test_adoption_statement_classifies, which + # is the only place that statement is emitted here. + beta = harness.coordinator( + "beta", + session_token="heartbeat-v1:beta-successor", + ) + beta.start() + harness.run_until(beta, "done:startup") + + # Every statement above reached FakePostgres and was classified + # as the operation it performs; an unrecognised one would have + # raised UnsupportedStatement inside the ledger call. + self.assertEqual( + harness.statement_kinds(), + ["acquire", "renew", "verify", "release", "acquire"], + ) + + def test_adoption_statement_classifies(self) -> None: + """The adoption CAS is the one statement startup alone never emits.""" + ledger_sql = share_ledger_module.PsqlShareLedger.__dict__[ + "_try_adopt_writer_lease" + ] + self.assertIn("observed_writer_session_token", inspect.getsource(ledger_sql)) + + with LeaseHarness() as harness: + alpha = harness.coordinator("alpha") + alpha.start() + harness.run_until(alpha, "done:startup") + # Build the statement through the real method, capturing it at the + # port rather than reconstructing it here. + observed = { + "writer_session_token": "heartbeat-v1:predecessor", + "lease_updated_at": "2026-06-26 19:49:22.233718+00", + } + alpha.call( + lambda: alpha.ledger._try_adopt_writer_lease(observed), + label="adopt", + ) + self.assertEqual(harness.server.statements[-1].kind, LeaseOp.ADOPT) + self.assertEqual( + harness.server.statements[-1].payload[ + "observed_writer_session_token" + ], + "heartbeat-v1:predecessor", + ) + + def test_unknown_statement_fails_loudly(self) -> None: + with self.assertRaises(UnsupportedStatement) as caught: + classify("SELECT 1") + self.assertIn("does not model this statement", str(caught.exception)) + + def test_deferred_state_machines_name_their_follow_up(self) -> None: + for sql, expected in ( + ("INSERT INTO qbit_share_ledger (share_id)", "share submission"), + ("INSERT INTO qbit_ctv_fanouts (block_hash)", "CTV fanout"), + ): + with self.subTest(sql=sql): + with self.assertRaises(UnsupportedStatement) as caught: + classify(sql) + self.assertIn(expected, str(caught.exception)) + self.assertIn("#128", str(caught.exception)) + + def test_a_landing_statement_shape_that_moved_fails_loudly(self) -> None: + """A landing table alone is not a classification. + + The deferred marker for the outbox is gone now that landing is + modelled, so an outbox statement whose shape the classifier does not + recognise has to reach the unmodelled-statement error rather than + being answered by whichever landing evaluator matched loosest. + """ + with self.assertRaises(UnsupportedStatement) as caught: + classify("UPDATE qbit_block_candidate_outbox SET quarantined = true") + self.assertIn("does not model this statement", str(caught.exception)) + self.assertIn("_LANDING_SIGNATURES", str(caught.exception)) + + def test_shipped_share_append_is_not_read_as_the_outbox_state_machine( + self, + ) -> None: + """A share append must not be misnamed as the outbox state machine. + + ``_append_batch_with_replay_outcomes`` opens the very same + ``INSERT INTO qbit_block_candidate_outbox (`` and raises the very same + ``'block candidate payload mismatch'`` as + ``persist_block_candidate_intent``. Those two fragments alone therefore + match a share submission, and the classifier answered one with the + outbox evaluator: the statement was reported against the wrong state + machine, so the failure argued for an outbox bug that did not exist and + hid the deferral that did. This drives the shipped ``append_batch`` and + requires the captured statement to reach ``_DEFERRED_SIGNATURES``. + """ + share = PendingShare( + share_id="miner-a:1", + miner_id="miner-a", + order_key="miner-a", + p2mr_program_hex="22" * 32, + share_difficulty=1, + network_difficulty=1, + template_height=9, + job_id="job-1", + job_issued_at_ms=1, + accepted_at_ms=2, + ntime=1_700_000_000, + ) + candidate = {"block_hash_hex": "ab" * 32} + + captured: list[str] = [] + with LeaseHarness() as harness: + alpha = harness.coordinator("alpha") + alpha.start() + harness.run_until(alpha, "done:startup") + + # Capture at the port. The classifier rejects this statement, so it + # never reaches server.statements; the raw SQL has to be taken on + # the way in rather than read back out of the recognised trace. + executed = harness.server.execute + + def capturing_execute(backend: Any, sql: str, **kwargs: Any) -> Any: + captured.append(sql) + return executed(backend, sql, **kwargs) + + harness.server.execute = capturing_execute # type: ignore[method-assign] + + # Pin the historical ambiguous SQL discriminator. The v2 append + # has a distinct version-CAS shape and its own storage gate. + alpha.ledger._candidate_storage_version = 1 + with self.assertRaises(UnsupportedStatement) as caught: + alpha.call( + lambda: alpha.ledger.append_batch([(share, candidate)]), + label="append", + ) + + # The statement really is ambiguous under the two original fragments; + # if production stops emitting both, this test is no longer guarding + # anything and should fail rather than quietly pass. + appends = [sql for sql in captured if "INSERT INTO qbit_share_ledger" in sql] + self.assertEqual(len(appends), 1) + sql = appends[0] + self.assertIn("INSERT INTO qbit_block_candidate_outbox (", sql) + self.assertIn("'block candidate payload mismatch'", sql) + self.assertIn("INSERT INTO qbit_share_ledger", sql) + + # The discriminator: a share append claims a colliding outbox row, + # it does not leave it alone. + self.assertIn("ON CONFLICT (block_hash) DO UPDATE", sql) + self.assertNotIn("ON CONFLICT (block_hash) DO NOTHING", sql) + + # Both the in-flight rejection and a direct classification of the + # captured statement have to name the same state machine. + with self.assertRaises(UnsupportedStatement) as direct: + classify(sql) + for message in (str(caught.exception), str(direct.exception)): + self.assertIn("share submission and dedupe", message) + self.assertIn("#128", message) + self.assertNotIn(LandingOp.OUTBOX_RECORD.value, message) + + def test_block_candidate_intent_still_classifies_as_the_outbox_record( + self, + ) -> None: + """The discriminator must not cost the outbox its own classification. + + Narrowing ``OUTBOX_RECORD`` is only correct if the statement it exists + for still matches, so this pins the other direction of the same change. + """ + with LeaseHarness() as harness: + alpha = harness.coordinator("alpha") + alpha.start() + harness.run_until(alpha, "done:startup") + alpha.call( + lambda: alpha.ledger.persist_block_candidate_intent( + {"block_hash_hex": "ab" * 32} + ), + label="intent", + ) + statement = harness.server.statements[-1] + self.assertEqual(statement.kind, LandingOp.OUTBOX_RECORD) + self.assertEqual(statement.payload["block_hash"], "ab" * 32) + self.assertIn("ON CONFLICT (block_hash) DO NOTHING", statement.sql) + + def test_the_outbox_discriminator_is_unique_to_the_intent_writer( + self, + ) -> None: + """Pin the fragment's uniqueness to production, not to this test. + + ``OUTBOX_RECORD`` is now identified by a conflict action. That is only + a valid identity for the two versioned intent writers; if another + writer starts inserting the candidate with ``DO NOTHING`` the + classifier is ambiguous again and this fails at that moment rather + than at the next confusing scenario failure. + """ + discriminator = "ON CONFLICT (block_hash) DO NOTHING" + emitters = sorted( + name + for name, member in vars(share_ledger_module.PsqlShareLedger).items() + if discriminator in _source_of(member) + ) + self.assertEqual(emitters, ["_persist_block_candidate_intent_v1", "persist_block_candidate_intent"]) + + def test_classifier_extracts_the_configured_ttl(self) -> None: + with LeaseHarness(lease_ttl_seconds=17.5) as harness: + alpha = harness.coordinator("alpha") + alpha.start() + harness.run_until(alpha, "done:startup") + self.assertEqual( + harness.server.statements[0].lease_ttl_seconds, + 17.5, + ) + row = harness.lease_row() + assert row is not None + self.assertEqual( + (row.lease_expires_at - row.updated_at).total_seconds(), + 17.5, + ) + + def test_advisory_lock_pair_matches_production_splitting(self) -> None: + key = share_ledger_module._writer_lease_advisory_lock_key( + "prism-coordinator", + 1, + ) + classid, objid = advisory_lock_pair(key) + self.assertEqual(classid, (key >> 32) & 0xFFFFFFFF) + self.assertEqual(objid, key & 0xFFFFFFFF) + + +class PortContractTests(unittest.TestCase): + """The fakes must keep matching the production classes they replace. + + The repository runs no mypy — CI is `compileall` plus `unittest discover` + — so structural typing is never checked statically. Without these the two + protocols would be documentation: nothing would notice a fake that + stopped implementing one, or a production class that outgrew it. + """ + + def _guard_stub(self) -> Any: + guard = share_ledger_module._NativePostgresLeaseGuard.__new__( + share_ledger_module._NativePostgresLeaseGuard + ) + guard._connection = None + guard._advisory_lock_key = 0 + guard._query_lock = threading.Lock() + guard._closed = False + guard._held = True + return guard + + def test_production_classes_satisfy_their_own_ports(self) -> None: + client = share_ledger_module._NativePostgresClient.__new__( + share_ledger_module._NativePostgresClient + ) + self.assertIsInstance(client, share_ledger_module.LedgerSqlPort) + self.assertIsInstance( + self._guard_stub(), + share_ledger_module.LeaseGuardPort, + ) + + def test_fakes_satisfy_the_ports_they_stand_in_for(self) -> None: + from tests.prism_concurrency_harness import FakeLeaseGuard, FakeSqlBackend + + with LeaseHarness() as harness: + alpha = harness.coordinator("alpha") + alpha.start() + harness.run_until(alpha, "done:startup") + + self.assertIsInstance(alpha.sql_backend, FakeSqlBackend) + self.assertIsInstance(alpha.guard, FakeLeaseGuard) + self.assertIsInstance( + alpha.sql_backend, + share_ledger_module.LedgerSqlPort, + ) + self.assertIsInstance( + alpha.guard, + share_ledger_module.LeaseGuardPort, + ) + + def test_every_port_method_matches_production_signature(self) -> None: + """Full signatures, not just parameter names. + + Comparing `list(signature.parameters)` alone would stay green if a + keyword-only argument became positional or lost its default, which is + exactly the kind of drift that makes a fake silently stop modelling + the thing it replaces. + """ + from tests.prism_concurrency_harness import FakeLeaseGuard, FakeSqlBackend + + for production, fake, methods in ( + ( + share_ledger_module._NativePostgresClient, + FakeSqlBackend, + ("run_json", "run_script", "close"), + ), + ( + share_ledger_module._NativePostgresLeaseGuard, + FakeLeaseGuard, + ("run_json", "try_acquire", "close"), + ), + ): + for method in methods: + with self.subTest(production=production.__name__, method=method): + self.assertEqual( + inspect.signature(getattr(production, method)), + inspect.signature(getattr(fake, method)), + ) + + def test_sql_backend_matches_native_client_signature(self) -> None: + from tests.prism_concurrency_harness import FakeSqlBackend + + production = inspect.signature( + share_ledger_module._NativePostgresClient.run_json + ) + fake = inspect.signature(FakeSqlBackend.run_json) + self.assertEqual(list(production.parameters), list(fake.parameters)) + + def test_lease_guard_matches_native_guard_signature(self) -> None: + from tests.prism_concurrency_harness import FakeLeaseGuard + + production = inspect.signature( + share_ledger_module._NativePostgresLeaseGuard.run_json + ) + fake = inspect.signature(FakeLeaseGuard.run_json) + self.assertEqual(list(production.parameters), list(fake.parameters)) + + def test_ledger_accepts_the_fakes_through_its_constructor(self) -> None: + """No bound method is reassigned to install the harness.""" + parameters = inspect.signature( + share_ledger_module.PsqlShareLedger.__init__ + ).parameters + for name in ("sql_backend_factory", "lease_guard_factory", "monotonic"): + self.assertIn(name, parameters) + + +class VirtualClockTests(unittest.TestCase): + def test_time_only_moves_when_advanced(self) -> None: + clock = VirtualClock() + first = clock.monotonic() + for _ in range(1000): + clock.now() + self.assertEqual(clock.monotonic(), first) + clock.advance(2.5) + self.assertEqual(clock.monotonic(), first + 2.5) + + def test_time_cannot_move_backwards(self) -> None: + clock = VirtualClock() + with self.assertRaises(HarnessError): + clock.advance(-1) + + def test_timestamp_text_is_postgres_shaped(self) -> None: + clock = VirtualClock() + self.assertEqual( + VirtualClock.timestamp_text(clock.now()), + "2026-06-26 19:49:22.233718+00", + ) + + +class SchedulerTests(unittest.TestCase): + def test_stepping_a_blocked_actor_is_refused(self) -> None: + with LeaseHarness() as harness: + alpha = harness.coordinator("alpha") + alpha.start() + harness.run_until(alpha, "done:startup") + + def sleep_forever() -> None: + harness.sleep(30.0) + + alpha.submit(sleep_forever, label="sleep") + harness.run_until(alpha, "sleep:30") + + with self.assertRaises(NotRunnable): + harness.step(alpha) + + # Time is the only thing that can release it, and the harness can + # say exactly when. + self.assertEqual( + harness.scheduler.next_deadline(), + harness.clock.monotonic() + 30.0, + ) + harness.advance_to_next_deadline() + harness.run_until(alpha, "done:sleep") + + def test_blocking_outside_the_model_is_reported_not_hung(self) -> None: + """An actor waiting on something unmodelled must fail, not hang.""" + with LeaseHarness(baton_timeout_seconds=0.4) as harness: + alpha = harness.coordinator("alpha") + unmodelled = threading.Lock() + unmodelled.acquire() + alpha.submit(unmodelled.acquire, label="wedge") + + with self.assertRaises(SchedulerStall) as caught: + harness.run_until(alpha, "done:wedge") + self.assertIn("does not model", str(caught.exception)) + unmodelled.release() + + def test_repeated_runs_produce_an_identical_schedule(self) -> None: + def scenario(harness: LeaseHarness) -> dict[str, object]: + # Distinct epochs: each computes a different advisory-guard key, + # so both reach the lease upsert and genuinely race for the row. + alpha = harness.coordinator("alpha", writer_epoch=1) + beta = harness.coordinator("beta", writer_epoch=2) + alpha.start() + beta.start() + # Both statements are in flight before either executes. + harness.run_until(alpha, "alpha.begin:acquire") + harness.run_until(beta, "beta.begin:acquire") + harness.drain([alpha, beta]) + beta_error = beta.actor.calls[0].error + return { + "holder": harness.lease_holder_session(), + "kinds": harness.statement_kinds(), + "alpha_failed": alpha.actor.calls[0].error is not None, + "beta_error": None if beta_error is None else str(beta_error), + "acquire_arms": acquire_result_arms(harness.server.statements[0].sql), + } + + run = assert_deterministic(self, scenario, repeats=25) + # The schedule decides the winner, and it decides it the same way + # every run. That reproducibility is the property under test; the + # identity of the winner is just what this schedule happens to pick. + self.assertEqual(run.outcome["holder"], "heartbeat-v1:alpha-1") + # The race loser receives the explicit retry sentinel and performs one + # fresh-snapshot acquire, so the current three-arm statement executes + # three acquire operations in this schedule. + self.assertEqual(run.outcome["kinds"], ["acquire", "acquire", "acquire"]) + self.assertFalse(run.outcome["alpha_failed"]) + # What the loser gets is a property of the shipped statement, so it is + # asserted against the statement rather than pinned to a constant. On + # the two-arm shape its DO UPDATE matches nothing and the COALESCE + # fallback reads a snapshot taken before the winner's row existed, so + # the whole statement evaluates to SQL NULL. Issue #140's terminal arm + # retires that outcome. See + # test_first_acquire_race_loser_matches_the_shipped_acquire_shape. + if run.outcome["acquire_arms"] == 2: + self.assertEqual( + run.outcome["beta_error"], + "postgres query returned no JSON", + ) + else: + self.assertNotIn( + "returned no JSON", + str(run.outcome["beta_error"]), + ) + + def test_first_acquire_race_loser_matches_the_shipped_acquire_shape(self) -> None: + """Two coordinators racing the first-ever acquire. + + What the loser gets is decided by the shipped statement, so this test + reads the statement and asserts what that shape can produce, instead + of pinning whichever outcome was current when it was written. + + **Two-arm shape (this baseline).** The losing + ``INSERT ... ON CONFLICT DO UPDATE`` affects zero rows, and the + ``COALESCE`` arm that would report the holder reads this statement's + own snapshot -- taken before the winner's row was committed -- so it + finds nothing and the statement returns SQL NULL. + ``parse_single_json_value`` turns that into ``postgres query returned + no JSON``, which ``_ensure_writer_lease`` does not catch, so + ``PsqlShareLedger.__init__`` dies with an error naming nothing about + leases or about the coordinator that holds it. That is a real + production outcome that reads like a driver fault, which is why it was + pinned rather than left implicit. + + **Terminal-arm shape (issue #140).** The arm exists precisely so that + race reports a lease outcome, so the generic no-JSON diagnostic must + no longer be reachable. This asserts its retirement rather than a + message this branch has never seen: the operator-facing text is the + ledger PR's to choose, but "not the driver fault" is the whole point + of the change and is safe to require here. + """ + with LeaseHarness() as harness: + alpha = harness.coordinator("alpha", writer_epoch=1) + beta = harness.coordinator("beta", writer_epoch=2) + alpha.start() + beta.start() + harness.run_until(alpha, "alpha.begin:acquire") + harness.run_until(beta, "beta.begin:acquire") + harness.drain([alpha, beta]) + + alpha.actor.calls[0].value() + arms = acquire_result_arms(harness.server.statements[0].sql) + if arms == 2: + with self.assertRaisesRegex( + RuntimeError, + "postgres query returned no JSON", + ): + beta.actor.calls[0].value() + else: + error = beta.actor.calls[0].error + self.assertNotIn("returned no JSON", str(error)) + + # Contrast, and unchanged by either shape: once the winner's row + # is visible in the loser's snapshot, the same race produces the + # ordinary refusal. + gamma = harness.coordinator("gamma", writer_epoch=3) + gamma.start() + harness.run_until(gamma, "done:startup") + with self.assertRaisesRegex( + RuntimeError, + "qbit ledger writer lease is held by", + ): + gamma.actor.calls[0].value() + + def test_acquire_arm_count_is_read_from_the_shipped_statement(self) -> None: + """The compatibility switch is computed, never declared. + + If this read a constant or a version flag, the whole arrangement would + be a guess about production that drifts silently. It reads the + statement ``PsqlShareLedger`` actually emitted, and pins what that + statement is at this baseline so a shape change cannot pass unnoticed. + """ + with LeaseHarness() as harness: + alpha = harness.coordinator("alpha") + alpha.start() + harness.run_until(alpha, "done:startup") + statement = harness.server.statements[0] + self.assertEqual(statement.kind, LeaseOp.ACQUIRE) + sql = statement.sql + + # The shipped shape has the acquired, holder, and explicit retry arms. + # Asserting the terminal keys as well as the count stops a renamed arm + # from being read as unchanged. + self.assertEqual(acquire_result_arms(sql), 3) + self.assertEqual( + acquire_terminal_arm_keys(sql), + ( + "acquired", + "lease_snapshot_retry", + "lease", + "retry_reason", + ), + ) + + # A shape neither this model nor #140 describes stays loud. + with self.assertRaises(UnsupportedStatement) as caught: + acquire_result_arms("SELECT 1;") + self.assertIn("SELECT COALESCE", str(caught.exception)) + + def test_the_explicit_not_acquired_arm_is_modelled_when_present(self) -> None: + """Exercise the integrated #140 arm and the historical two-arm shape. + + The current statement must carry the pinned terminal arm and the model + must answer it. Removing only that captured arm recreates the old + two-arm statement and proves the compatibility switch still switches. + + The arm is the captured fixture and the expected object is written out + separately, so this holds the model against a fixed input without + reading anything outside this tree. + """ + with LeaseHarness() as harness: + alpha = harness.coordinator("alpha") + alpha.start() + harness.run_until(alpha, "done:startup") + server = harness.server + shipped = server.statements[0].sql + + terminal = shipped.rstrip() + baseline = _without_ledger_retry_terminal_arm(shipped) + self.assertEqual(acquire_result_arms(terminal), 3) + self.assertEqual( + acquire_terminal_arm_keys(terminal), + ("acquired", "lease_snapshot_retry", "lease", "retry_reason"), + ) + + statement = classify(terminal) + answered = server._unobserved_not_acquired(statement) + + # The whole point: this shape cannot evaluate to SQL NULL, so the + # ports can no longer raise "postgres query returned no JSON". The + # values are the statement's own literals, including the sentence -- + # a boolean here would mean the model was answering the key's name. + self.assertEqual(answered, LEDGER_RETRY_TERMINAL_OBJECT) + # Spelled out separately: answering the key's *name* is the exact + # defect the first policy had, and it would put a boolean here. Dict + # equality alone reads as a typo check and would not say that. + assert answered is not None + self.assertIsInstance(answered["retry_reason"], str) + self.assertIs(answered["lease_snapshot_retry"], True) + self.assertIs(answered["acquired"], False) + + # And the historical shape still must, or the switch is not switching. + self.assertIsNone(server._unobserved_not_acquired(classify(baseline))) + + def test_a_terminal_literal_survives_commas_and_quotes(self) -> None: + """Punctuation inside a reason string must not become an argument break. + + The sentence this arm ships today happens to contain no comma, so + nothing else in this suite exercises the splitter's quoting. That is + a thin place to leave under a model whose whole job is to report the + wording production chose: the first comma added to that sentence + would otherwise silently re-pair every key and value after it, and + the failure would look like a renamed key rather than a parser bug. + """ + reason = ( + "the lease row was committed by a concurrent acquisition, " + "after this statement''s snapshot" + ) + arm = ( + ",\n json_build_object(\n" + " 'acquired', false,\n" + " 'lease_snapshot_retry', true,\n" + " 'lease', 'qbit ledger writer lease',\n" + " 'retry_reason',\n" + f" '{reason}'\n" + " )\n);" + ) + with LeaseHarness() as harness: + alpha = harness.coordinator("alpha") + alpha.start() + harness.run_until(alpha, "done:startup") + server = harness.server + baseline = _without_ledger_retry_terminal_arm(server.statements[0].sql) + terminal = baseline.rstrip()[:-2] + arm + + self.assertEqual(acquire_result_arms(terminal), 3) + self.assertEqual( + acquire_terminal_arm_keys(terminal), + ("acquired", "lease_snapshot_retry", "lease", "retry_reason"), + ) + answered = server._unobserved_not_acquired(classify(terminal)) + + assert answered is not None + self.assertEqual( + answered["retry_reason"], + "the lease row was committed by a concurrent acquisition, " + "after this statement's snapshot", + ) + self.assertEqual(answered["lease"], "qbit ledger writer lease") + + def test_an_unteachable_terminal_arm_key_stays_loud(self) -> None: + """A key this model cannot answer must fail, not be invented. + + The value policy for the terminal arm is deliberately narrow: + ``acquired`` is false, the holder and lease columns are null because + nothing is visible, and a retry flag is true. Anything else is a shape + nobody has taught this model, and guessing at it would be exactly the + harness lie the suite's docstring warns about. + """ + with LeaseHarness() as harness: + alpha = harness.coordinator("alpha") + alpha.start() + harness.run_until(alpha, "done:startup") + server = harness.server + shipped = server.statements[0].sql + terminal = shipped.rstrip()[:-2] + ( + ",\n (\n SELECT json_build_object(\n" + " 'acquired', false,\n" + " 'backoff_hint_seconds', 3\n" + " )\n )\n);" + ) + statement = classify(terminal) + with self.assertRaises(UnsupportedStatement) as caught: + server._unobserved_not_acquired(statement) + self.assertIn("backoff_hint_seconds", str(caught.exception)) + + def test_trace_names_the_actor_and_the_stop(self) -> None: + with LeaseHarness() as harness: + alpha = harness.coordinator("alpha") + alpha.start() + harness.run_until(alpha, "done:startup") + self.assertEqual( + harness.trace, + [ + "alpha@alpha.begin:acquire", + "alpha@alpha.done:acquire", + # The acquire runs under #123's acquisition deadline, so + # it commits explicitly and offers a precommit stop. + "alpha@alpha.precommit", + "alpha@done:startup", + ], + ) + + +class PostgresModelTests(unittest.TestCase): + """The PostgreSQL behaviours the lease lifecycle actually relies on.""" + + def test_advisory_lock_is_exclusive_and_session_scoped(self) -> None: + with LeaseHarness() as harness: + alpha = harness.coordinator("alpha", writer_epoch=1) + alpha.start() + harness.run_until(alpha, "done:startup") + assert alpha.guard is not None + self.assertTrue(alpha.guard.held) + + # A same-identity twin computes the same key and cannot take it. + key = alpha.guard.advisory_lock_key + other = harness.server.connect(application_name="twin") + self.assertFalse(harness.server.try_advisory_lock(other, key)) + + # Closing the holder's session releases it server-side. + alpha.guard.close() + self.assertTrue(harness.server.try_advisory_lock(other, key)) + + def test_tuple_lock_is_held_for_the_life_of_a_transaction(self) -> None: + with LeaseHarness() as harness: + alpha = harness.coordinator("alpha") + alpha.start() + harness.run_until(alpha, "done:startup") + + def deadline_scoped_write() -> object: + with alpha.ledger.operation_timeout(30.0): + return alpha.ledger.renew_writer_lease() + + alpha.submit(deadline_scoped_write, label="write") + harness.run_until(alpha, "alpha.done:renew") + # Statement executed, COMMIT not yet sent: the lock is held and + # the committed row still carries the pre-write value. + self.assertIsNotNone(harness.server.lease_lock_holder) + row = harness.lease_row() + assert row is not None + self.assertIsNotNone(row.xmax) + + harness.run_until(alpha, "done:write") + self.assertIsNone(harness.server.lease_lock_holder) + row = harness.lease_row() + assert row is not None + self.assertIsNone(row.xmax) + + def test_vanishing_during_an_autocommit_statement_leaves_no_orphan(self) -> None: + """The model must not offer #123's shape on a path that cannot produce it. + + A client that vanishes mid-statement leaves an orphan only when the + COMMIT is a separate message it never sends. Autocommit has no such + message: PostgreSQL finishes the statement and commits it. A harness + that let a scenario build the orphan shape on such a path would argue + for a fix to a defect that path cannot reach. + + This used to drive the startup lease acquire, which was autocommit + because no caller armed a deadline on it. #123's fix arms one, so the + acquire is no longer an example of this property — see + ``test_vanishing_mid_acquire_orphans_now_that_it_is_deadline_scoped`` + for what that path does instead. An unscoped renewal is still a plain + autocommit statement and demonstrates the property unchanged; the + property under test never was about the acquire in particular. + """ + with LeaseHarness() as harness: + alpha = harness.coordinator("alpha") + alpha.start() + harness.run_until(alpha, "done:startup") + + # No operation_timeout: run_json gets timeout_seconds=None, so + # the model runs it the way _NativePostgresClient does, without + # an enclosing conn.transaction(). + alpha.submit(lambda: alpha.ledger.renew_writer_lease(), label="renew") + harness.run_until(alpha, "alpha.done:renew") + self.assertIsNotNone( + harness.server.lease_lock_holder, + "the statement is mid-flight and holds the tuple lock", + ) + + alpha.vanish() + + self.assertIsNone( + harness.server.lease_lock_holder, + "the server finished and committed the autocommit statement", + ) + row = harness.lease_row() + assert row is not None + self.assertEqual(row.writer_session_token, "heartbeat-v1:alpha-1") + self.assertIsNone( + row.xmax, + "no transaction is left holding the row", + ) + + def test_vanishing_mid_acquire_orphans_now_that_it_is_deadline_scoped( + self, + ) -> None: + """#123's fix widens the orphan shape onto the startup acquire itself. + + ``_run_lease_acquisition_json`` wraps the acquire in + ``statement_timeout(...)`` so that a successor cannot queue forever + on the lease tuple. That deadline is also what makes + ``_NativePostgresClient`` run the statement inside an explicit + transaction with a separate COMMIT — so a coordinator that vanishes + mid-acquire now leaves exactly the orphan #123 is about, on a path + that previously could not produce one. + + This is an accepted trade rather than a regression, and it is pinned + here so it stays deliberate. The orphan #123 describes was unbounded; + this one is bounded by ``idle_in_transaction_session_timeout`` + (``PRISM_POSTGRES_IDLE_IN_TRANSACTION_TIMEOUT_SECONDS``, 15s by + default), which the same fix sets on every coordinator session. That + is why the session guard is not optional, and + docs/prism-ledger-ops.md records the trade. + + The reap itself is not asserted here: this harness models no + idle-in-transaction reaper, and faking one would pin the model's + opinion rather than PostgreSQL's. That the option is actually set on + every connection path is pinned directly against the shipped client + by ``PostgresSessionGuardTests`` in tests/test_prism_share_ledger.py. + What this test pins is what the model does support: the orphan is + real, it holds the lease lock, and a successor that meets it is + bounded rather than stuck. + """ + with LeaseHarness() as harness: + alpha = harness.coordinator("alpha", writer_epoch=1) + alpha.start() + # A checkpoint that did not exist on this path before the fix. + harness.run_until(alpha, "alpha.precommit") + alpha.vanish() + + orphan = harness.server.lease_lock_holder + self.assertIsNotNone(orphan) + assert orphan is not None + self.assertTrue(orphan.orphaned) + self.assertTrue(orphan.explicit, "the deadline is what opened it") + self.assertIsNone( + harness.lease_row(), + "the acquire never committed, so no lease row exists at all", + ) + + # The successor still gets out, which is the half that matters: + # this orphan costs a bounded startup delay, not an outage. + beta = harness.coordinator("beta", writer_epoch=2) + beta.start() + harness.run_until_blocked(beta, "beta.lockwait:acquire") + self.assertEqual( + beta.actor.wake_at, + harness.clock.monotonic() + + DEFAULT_LEASE_ACQUIRE_LOCK_TIMEOUT_SECONDS, + ) + + def test_vanishing_inside_a_deadline_scoped_statement_does_orphan(self) -> None: + """The contrast: a separate COMMIT the client never sends.""" + with LeaseHarness() as harness: + alpha = harness.coordinator("alpha") + alpha.start() + harness.run_until(alpha, "done:startup") + + def deadline_scoped_write() -> object: + with alpha.ledger.operation_timeout(30.0): + return alpha.ledger.renew_writer_lease() + + alpha.submit(deadline_scoped_write, label="write") + harness.run_until(alpha, "alpha.precommit") + alpha.vanish() + + orphan = harness.server.lease_lock_holder + self.assertIsNotNone(orphan) + assert orphan is not None + self.assertTrue(orphan.orphaned) + + def test_autocommit_statements_never_leave_a_transaction_open(self) -> None: + """No caller deadline, so no explicit transaction to be caught inside. + + Also drove the startup acquire until #123's fix armed a deadline on + it. Asserting the absence of a precommit stop only means something on + a statement that genuinely has no caller deadline, so this drives an + unscoped renewal; on the acquire the assertion would now be checking + the wrong path and passing for the wrong reason. + """ + with LeaseHarness() as harness: + alpha = harness.coordinator("alpha") + alpha.start() + harness.run_until(alpha, "done:startup") + + mark = len(harness.trace) + alpha.submit(lambda: alpha.ledger.renew_writer_lease(), label="renew") + harness.run_until(alpha, "done:renew") + + self.assertNotIn("alpha@alpha.precommit", harness.trace[mark:]) + self.assertIsNone(harness.server.lease_lock_holder) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_prism_coordinator_config.py b/tests/test_prism_coordinator_config.py new file mode 100644 index 000000000..d31c83bd4 --- /dev/null +++ b/tests/test_prism_coordinator_config.py @@ -0,0 +1,2616 @@ +#!/usr/bin/env python3 +"""Focused PRISM coordinator config tests.""" +# ruff: noqa: F403, F405 + +from __future__ import annotations + +import sys +import tempfile +import unittest +from pathlib import Path + +from lab.prism.coordinator_config import ( + DEFAULT_PRISM_VARDIFF_INITIAL_MAX_STEP_UP, + MAX_PRISM_PYTHON_SWITCH_INTERVAL_SECONDS, + MIN_PRISM_PYTHON_SWITCH_INTERVAL_SECONDS, + apply_python_switch_interval, + env_decimal, + load_coordinator_config, + resolve_python_switch_interval_seconds, +) +from tests.prism_vardiff_test_support import * + + +class PrismPythonSwitchIntervalTests(unittest.TestCase): + """#143: the interpreter switch interval is a tunable, validated at startup.""" + + def setUp(self) -> None: + # setswitchinterval is process-global, so a test that applies one must + # put it back or every later test in this process inherits it. + original = sys.getswitchinterval() + self.addCleanup(sys.setswitchinterval, original) + + def test_unset_leaves_the_interpreter_default_untouched(self) -> None: + before = sys.getswitchinterval() + + self.assertIsNone(resolve_python_switch_interval_seconds(environ={})) + self.assertIsNone(apply_python_switch_interval(environ={})) + + # Absent means "do not call setswitchinterval", not "call it with 5ms". + self.assertEqual(sys.getswitchinterval(), before) + + def test_empty_value_behaves_as_unset(self) -> None: + # compose.yaml passes ${PRISM_PYTHON_SWITCH_INTERVAL_SECONDS:-}, which + # reaches the container as an empty string rather than as an absent + # variable. If that ever stopped reading as unset, every default + # deployment would refuse to start. + before = sys.getswitchinterval() + + self.assertIsNone( + apply_python_switch_interval( + environ={"PRISM_PYTHON_SWITCH_INTERVAL_SECONDS": ""} + ) + ) + + self.assertEqual(sys.getswitchinterval(), before) + + def test_configured_value_is_applied_and_returned(self) -> None: + applied = apply_python_switch_interval( + environ={"PRISM_PYTHON_SWITCH_INTERVAL_SECONDS": "0.001"} + ) + + self.assertEqual(applied, 0.001) + self.assertAlmostEqual(sys.getswitchinterval(), 0.001) + + def test_non_numeric_value_refuses_startup(self) -> None: + with self.assertRaisesRegex( + SystemExit, "PRISM_PYTHON_SWITCH_INTERVAL_SECONDS must be a number" + ): + resolve_python_switch_interval_seconds( + environ={"PRISM_PYTHON_SWITCH_INTERVAL_SECONDS": "fast"} + ) + + def test_non_positive_and_non_finite_values_refuse_startup(self) -> None: + for raw, message in ( + ("0", "must be positive"), + ("-0.005", "must be positive"), + ("nan", "must be finite"), + ("inf", "must be finite"), + ): + with self.subTest(raw=raw), self.assertRaisesRegex(SystemExit, message): + resolve_python_switch_interval_seconds( + environ={"PRISM_PYTHON_SWITCH_INTERVAL_SECONDS": raw} + ) + + def test_sub_microsecond_value_refuses_startup(self) -> None: + # CPython stores the interval in whole microseconds, so anything below + # one truncates to zero -- a switch check at every opportunity, which + # is the opposite of what lowering the value is reaching for. + for raw in ("0.0000009", "1e-07", "1e-09"): + with self.subTest(raw=raw), self.assertRaisesRegex( + SystemExit, "cannot be below .* truncate it to zero" + ): + resolve_python_switch_interval_seconds( + environ={"PRISM_PYTHON_SWITCH_INTERVAL_SECONDS": raw} + ) + + def test_the_floor_is_the_interpreters_actual_resolution(self) -> None: + # Pins the bound to the interpreter rather than to a number we chose: + # the floor must apply intact, and anything below it must truncate to + # zero. If a future CPython changes its resolution, this fails rather + # than letting the bound silently drift out of agreement with it. + floor = MIN_PRISM_PYTHON_SWITCH_INTERVAL_SECONDS + + applied = apply_python_switch_interval( + environ={"PRISM_PYTHON_SWITCH_INTERVAL_SECONDS": repr(floor)} + ) + self.assertEqual(applied, floor) + self.assertEqual(sys.getswitchinterval(), floor) + + sys.setswitchinterval(floor / 2) + self.assertEqual( + sys.getswitchinterval(), + 0.0, + "sub-floor intervals no longer truncate to zero; revisit the bound", + ) + + def test_value_above_the_ceiling_refuses_startup(self) -> None: + # A misplaced decimal point ("5" meaning 5ms) would otherwise let a + # thread hold the GIL for whole seconds, which reads as unexplained + # ack-latency rather than as a configuration error. + with self.assertRaisesRegex( + SystemExit, "PRISM_PYTHON_SWITCH_INTERVAL_SECONDS cannot exceed" + ): + resolve_python_switch_interval_seconds( + environ={"PRISM_PYTHON_SWITCH_INTERVAL_SECONDS": "5"} + ) + + def test_the_ceiling_itself_is_accepted(self) -> None: + ceiling = MAX_PRISM_PYTHON_SWITCH_INTERVAL_SECONDS + + self.assertEqual( + resolve_python_switch_interval_seconds( + environ={"PRISM_PYTHON_SWITCH_INTERVAL_SECONDS": str(ceiling)} + ), + ceiling, + ) + + def test_rejected_values_never_reach_the_interpreter(self) -> None: + before = sys.getswitchinterval() + + with self.assertRaises(SystemExit): + apply_python_switch_interval( + environ={"PRISM_PYTHON_SWITCH_INTERVAL_SECONDS": "5"} + ) + + self.assertEqual(sys.getswitchinterval(), before) + + +class PrismCoordinatorVardiffTests(unittest.TestCase): + def test_decimal_env_rejects_invalid_and_non_finite_values(self) -> None: + with self.assertRaisesRegex(SystemExit, "TEST_DECIMAL must be a decimal number"): + env_decimal("TEST_DECIMAL", "1", environ={"TEST_DECIMAL": "not-decimal"}) + for raw in ("NaN", "sNaN", "Infinity", "-Infinity"): + with self.subTest(raw=raw), self.assertRaisesRegex( + SystemExit, + "TEST_DECIMAL must be finite", + ): + env_decimal("TEST_DECIMAL", "1", environ={"TEST_DECIMAL": raw}) + + def test_send_error_includes_canonical_reason_id_data(self) -> None: + server = coordinator() + sent: list[dict[str, object]] = [] + client = SimpleNamespace(send=lambda payload: sent.append(payload)) + + server.send_error(client, "submit-1", 21, "stale job", reason=PRISM_REJECTION_STALE_JOB) # type: ignore[arg-type] + + self.assertEqual( + sent, + [ + { + "id": "submit-1", + "result": None, + "error": [21, "stale job", {"reason_id": PRISM_REJECTION_STALE_JOB}], + } + ], + ) + + def test_scaled_target_difficulty_uses_pow_limit_units(self) -> None: + pow_limit = target_from_compact("207fffff") + + self.assertEqual(scaled_target_difficulty(pow_limit), 1_000_000) + self.assertEqual(scaled_target_difficulty(pow_limit // 4), 4_000_000) + + def test_qbit_gbt_rules_include_signet_rule_only_for_signet(self) -> None: + self.assertEqual(qbit_gbt_rules("regtest"), ["segwit"]) + self.assertEqual(qbit_gbt_rules("testnet4"), ["segwit"]) + self.assertEqual(qbit_gbt_rules("signet"), ["segwit", "signet"]) + + def test_qbit_template_fingerprint_ignores_clock_only_fields(self) -> None: + base = gbt_template("00" * 32, curtime=1) + base["longpollid"] = "10:0" + base["mintime"] = 1 + clock_only = gbt_template("00" * 32, curtime=2) + clock_only["longpollid"] = "10:1" + clock_only["mintime"] = 2 + changed_value = dict(clock_only) + changed_value["coinbasevalue"] = 49_99999999 + + self.assertEqual(qbit_template_fingerprint(base), qbit_template_fingerprint(clock_only)) + self.assertNotEqual(qbit_template_fingerprint(base), qbit_template_fingerprint(changed_value)) + + def test_resolve_version_mask_uses_gbt_versionrollingmask(self) -> None: + server = PrismCoordinator.__new__(PrismCoordinator) + rpc = TemplateRpc({"versionrollingmask": "1fffe000"}) + server.rpc = rpc + server.qbit_chain = "signet" + + selection = server.resolve_version_rolling_mask(0x000000FF) + + self.assertEqual(selection.selected_mask, 0x1FFFE000) + self.assertEqual(selection.source, "qbit_getblocktemplate") + self.assertEqual(rpc.calls, [("getblocktemplate", [{"rules": ["segwit", "signet"]}])]) + + def test_resolve_version_mask_falls_back_only_when_gbt_missing_or_unavailable(self) -> None: + missing = PrismCoordinator.__new__(PrismCoordinator) + missing.rpc = TemplateRpc({}) + missing.qbit_chain = "regtest" + + missing_selection = missing.resolve_version_rolling_mask(direct_stratum.QBIT_VERSION_ROLLING_MASK) + + self.assertEqual(missing_selection.selected_mask, direct_stratum.QBIT_VERSION_ROLLING_MASK) + self.assertEqual(missing_selection.source, "fallback") + self.assertEqual(missing_selection.detail, "missing_versionrollingmask") + + unavailable = PrismCoordinator.__new__(PrismCoordinator) + unavailable.rpc = FakeRpc() + unavailable.qbit_chain = "regtest" + + unavailable_selection = unavailable.resolve_version_rolling_mask(direct_stratum.QBIT_VERSION_ROLLING_MASK) + + self.assertEqual(unavailable_selection.selected_mask, direct_stratum.QBIT_VERSION_ROLLING_MASK) + self.assertEqual(unavailable_selection.source, "fallback") + self.assertTrue(unavailable_selection.detail.startswith("probe_error:")) + + def test_resolve_version_mask_disables_only_on_gbt_zero_mask(self) -> None: + server = PrismCoordinator.__new__(PrismCoordinator) + server.rpc = TemplateRpc({"versionrollingmask": "00000000"}) + server.qbit_chain = "regtest" + + selection = server.resolve_version_rolling_mask(direct_stratum.QBIT_VERSION_ROLLING_MASK) + + self.assertEqual(selection.selected_mask, 0) + self.assertEqual(selection.source, "qbit_getblocktemplate") + self.assertEqual(selection.detail, "disabled_by_zero_mask") + + def test_resolve_version_mask_rejects_invalid_gbt_mask(self) -> None: + server = PrismCoordinator.__new__(PrismCoordinator) + server.rpc = TemplateRpc({"versionrollingmask": "not-hex"}) + server.qbit_chain = "regtest" + + with self.assertRaisesRegex(SystemExit, "invalid getblocktemplate.versionrollingmask"): + server.resolve_version_rolling_mask(direct_stratum.QBIT_VERSION_ROLLING_MASK) + + def test_configure_negotiates_requested_mask_with_gbt_server_mask(self) -> None: + server = coordinator() + server.version_mask = 0x1FFFE000 + state = client() + captured: dict[str, object] = {} + server.send_result = lambda _client, request_id, result: captured.update( # type: ignore[method-assign] + {"request_id": request_id, "result": result} + ) + + server.handle_configure( + state, + "configure-1", + [ + ["version-rolling"], + {"version-rolling.mask": "0000f000"}, + ], + ) + + self.assertEqual(captured["request_id"], "configure-1") + self.assertEqual( + captured["result"], + { + "version-rolling": True, + "version-rolling.mask": "0000e000", + }, + ) + self.assertEqual(state.version_mask, 0x0000E000) + + def test_configure_disables_version_rolling_when_gbt_mask_is_zero(self) -> None: + server = coordinator() + server.version_mask = 0 + state = client() + captured: dict[str, object] = {} + server.send_result = lambda _client, request_id, result: captured.update( # type: ignore[method-assign] + {"request_id": request_id, "result": result} + ) + + server.handle_configure( + state, + "configure-1", + [ + ["version-rolling"], + {"version-rolling.mask": "ffffffff"}, + ], + ) + + self.assertEqual( + captured["result"], + { + "version-rolling": False, + "version-rolling.mask": "00000000", + }, + ) + self.assertEqual(state.version_mask, 0) + + def test_accepted_share_difficulty_uses_actual_target_unless_overridden(self) -> None: + server = coordinator() + worker = WorkerIdentity( + username="miner-a", + payout_address="miner-a", + worker_name=None, + script_pubkey_hex="5220" + "11" * 32, + p2mr_program_hex="11" * 32, + ) + context = SimpleNamespace( + worker=worker, + job=SimpleNamespace(share_target=target_from_compact("207fffff") // 2), + ) + server.share_weights_by_username = {} + + self.assertEqual(server.accepted_share_difficulty(context), 2_000_000) + + server.share_weights_by_username = {"miner-a": 7} + self.assertEqual(server.accepted_share_difficulty(context), 7) + + def test_resolve_worker_accepts_bare_payout_address(self) -> None: + server = PrismCoordinator.__new__(PrismCoordinator) + rpc = AddressValidationRpc(script_byte="22") + server.rpc = rpc + + worker = server.resolve_worker(PAYOUT_ADDRESS) + + self.assertEqual(rpc.validated, [PAYOUT_ADDRESS]) + self.assertEqual(worker.username, PAYOUT_ADDRESS) + self.assertEqual(worker.payout_address, PAYOUT_ADDRESS) + self.assertIsNone(worker.worker_name) + self.assertEqual(worker.p2mr_program_hex, "22" * 32) + + def test_resolve_worker_accepts_address_worker_username(self) -> None: + server = PrismCoordinator.__new__(PrismCoordinator) + rpc = AddressValidationRpc(script_byte="33") + server.rpc = rpc + + worker = server.resolve_worker(f"{PAYOUT_ADDRESS}.rig-a") + + self.assertEqual(rpc.validated, [PAYOUT_ADDRESS]) + self.assertEqual(worker.username, f"{PAYOUT_ADDRESS}.rig-a") + self.assertEqual(worker.payout_address, PAYOUT_ADDRESS) + self.assertEqual(worker.worker_name, "rig-a") + self.assertEqual(worker.p2mr_program_hex, "33" * 32) + + def test_resolve_worker_caches_successful_address_validation(self) -> None: + server = PrismCoordinator.__new__(PrismCoordinator) + rpc = AddressValidationRpc(script_byte="33") + server.rpc = rpc + + first = server.resolve_worker(f"{PAYOUT_ADDRESS}.rig-a") + second = server.resolve_worker(f"{PAYOUT_ADDRESS}.rig-b") + + self.assertEqual(rpc.validated, [PAYOUT_ADDRESS]) + self.assertEqual(first.p2mr_program_hex, second.p2mr_program_hex) + + def test_payout_address_cache_evicts_least_recently_used_entry(self) -> None: + server = PrismCoordinator.__new__(PrismCoordinator) + server.payout_address_cache_max_entries = 2 + server.payout_address_cache_ttl_seconds = 60 + + class AnyAddressRpc: + def __init__(self) -> None: + self.validated: list[str] = [] + + def call(self, method: str, params: list[object] | None = None) -> object: + address = str((params or [""])[0]) + self.validated.append(address) + return {"isvalid": True, "scriptPubKey": "5220" + "33" * 32} + + rpc = AnyAddressRpc() + server.rpc = rpc + + server.validate_p2mr_address("address-a", label="test") + server.validate_p2mr_address("address-b", label="test") + server.validate_p2mr_address("address-a", label="test") + server.validate_p2mr_address("address-c", label="test") + + self.assertEqual(rpc.validated, ["address-a", "address-b", "address-c"]) + self.assertEqual(list(server._p2mr_address_cache), ["address-a", "address-c"]) + + server.validate_p2mr_address("address-b", label="test") + self.assertEqual(rpc.validated[-1], "address-b") + self.assertEqual(len(server._p2mr_address_cache), 2) + + def test_payout_address_cache_revalidates_expired_entry(self) -> None: + server = PrismCoordinator.__new__(PrismCoordinator) + server.payout_address_cache_max_entries = 2 + server.payout_address_cache_ttl_seconds = 5 + rpc = AddressValidationRpc(script_byte="33") + server.rpc = rpc + now = 100.0 + + with patch( + "lab.prism.prism_coordinator.time.monotonic", + side_effect=lambda: now, + ): + server.validate_p2mr_address(PAYOUT_ADDRESS, label="test") + now = 104.0 + server.validate_p2mr_address(PAYOUT_ADDRESS, label="test") + now = 106.0 + server.validate_p2mr_address(PAYOUT_ADDRESS, label="test") + + self.assertEqual(rpc.validated, [PAYOUT_ADDRESS, PAYOUT_ADDRESS]) + + def test_concurrent_worker_resolution_singleflights_address_validation(self) -> None: + server = PrismCoordinator.__new__(PrismCoordinator) + entered = threading.Event() + release = threading.Event() + + class BlockingAddressRpc(AddressValidationRpc): + def call(self, method: str, params: list[object] | None = None) -> object: + if method == "validateaddress": + entered.set() + if not release.wait(timeout=5): + raise TimeoutError("test did not release validateaddress") + return super().call(method, params) + + rpc = BlockingAddressRpc(script_byte="33") + server.rpc = rpc + server._ensure_p2mr_address_cache_state() + workers: list[WorkerIdentity] = [] + errors: list[BaseException] = [] + + def resolve(index: int) -> None: + try: + workers.append(server.resolve_worker(f"{PAYOUT_ADDRESS}.rig-{index}")) + except BaseException as exc: # pragma: no cover - surfaced below + errors.append(exc) + + threads = [threading.Thread(target=resolve, args=(index,)) for index in range(8)] + for thread in threads: + thread.start() + self.assertTrue(entered.wait(timeout=5)) + release.set() + for thread in threads: + thread.join(timeout=5) + + self.assertFalse(errors) + self.assertEqual(len(workers), 8) + self.assertEqual(rpc.validated, [PAYOUT_ADDRESS]) + + def test_concurrent_failed_worker_resolution_shares_singleflight_error(self) -> None: + server = PrismCoordinator.__new__(PrismCoordinator) + entered = threading.Event() + release = threading.Event() + + class FailingAddressRpc: + def __init__(self) -> None: + self.calls = 0 + + def call(self, method: str, params: list[object] | None = None) -> object: + self.calls += 1 + entered.set() + if not release.wait(timeout=5): + raise TimeoutError("test did not release validateaddress") + raise RuntimeError("qbitd unavailable") + + rpc = FailingAddressRpc() + server.rpc = rpc + server._ensure_p2mr_address_cache_state() + errors: list[BaseException] = [] + + def resolve() -> None: + try: + server.resolve_worker(PAYOUT_ADDRESS) + except BaseException as exc: + errors.append(exc) + + threads = [threading.Thread(target=resolve) for _ in range(8)] + for thread in threads: + thread.start() + self.assertTrue(entered.wait(timeout=5)) + deadline = time.monotonic() + 5 + while time.monotonic() < deadline: + with server._p2mr_address_cache_lock: + pending = server._p2mr_address_validation_inflight[PAYOUT_ADDRESS] + if pending.waiters == 7: + break + time.sleep(0.001) + self.assertEqual(pending.waiters, 7) + release.set() + for thread in threads: + thread.join(timeout=5) + + self.assertEqual(rpc.calls, 1) + self.assertEqual(len(errors), 8) + self.assertTrue(all("qbitd unavailable" in str(exc) for exc in errors)) + + def test_resolve_worker_rejects_invalid_base_address_with_worker_suffix(self) -> None: + server = PrismCoordinator.__new__(PrismCoordinator) + rpc = AddressValidationRpc() + server.rpc = rpc + server.username_fallback_address = None + + with self.assertRaises(StratumError) as raised: + server.resolve_worker("not-a-qbit-address.rig-a") + + self.assertEqual(raised.exception.code, 20) + self.assertEqual(rpc.validated, ["not-a-qbit-address"]) + + def test_resolve_worker_uses_configured_fallback_for_invalid_username(self) -> None: + fallback_address = "tq1fallback" + server = PrismCoordinator.__new__(PrismCoordinator) + rpc = AddressValidationRpc(valid_address=fallback_address, script_byte="44") + server.rpc = rpc + server.username_fallback_address = fallback_address + + worker = server.resolve_worker("not-a-qbit-address.rig-a") + + self.assertEqual(rpc.validated, ["not-a-qbit-address", fallback_address]) + self.assertEqual(worker.username, "not-a-qbit-address.rig-a") + self.assertEqual(worker.payout_address, fallback_address) + self.assertEqual(worker.worker_name, "rig-a") + self.assertEqual(worker.p2mr_program_hex, "44" * 32) + + def test_resolve_worker_uses_testnet_default_fallback_for_invalid_username(self) -> None: + server = PrismCoordinator.__new__(PrismCoordinator) + rpc = AddressValidationRpc(valid_address=DEFAULT_TESTNET_USERNAME_FALLBACK_ADDRESS, script_byte="55") + server.rpc = rpc + + with patch.dict(os.environ, {"QBIT_CHAIN": "testnet4"}, clear=True): + worker = server.resolve_worker("not-a-qbit-address") + + self.assertEqual(rpc.validated, ["not-a-qbit-address", DEFAULT_TESTNET_USERNAME_FALLBACK_ADDRESS]) + self.assertEqual(worker.username, "not-a-qbit-address") + self.assertEqual(worker.payout_address, DEFAULT_TESTNET_USERNAME_FALLBACK_ADDRESS) + self.assertIsNone(worker.worker_name) + self.assertEqual(worker.p2mr_program_hex, "55" * 32) + + def test_default_username_fallback_is_testnet_only_unless_configured(self) -> None: + with patch.dict(os.environ, {"QBIT_CHAIN": "testnet4"}, clear=True): + self.assertEqual(default_prism_username_fallback_address(), DEFAULT_TESTNET_USERNAME_FALLBACK_ADDRESS) + with patch.dict(os.environ, {"QBIT_CHAIN": "regtest"}, clear=True): + self.assertIsNone(default_prism_username_fallback_address()) + with patch.dict( + os.environ, + {"QBIT_CHAIN": "regtest", "PRISM_USERNAME_FALLBACK_ADDRESS": "qbrt1fallback"}, + clear=True, + ): + self.assertEqual(default_prism_username_fallback_address(), "qbrt1fallback") + + def test_prism_payout_policy_defaults_to_no_pool_fee(self) -> None: + server = coordinator() + + with patch.dict(os.environ, {}, clear=True): + policy = server.prism_payout_policy() + + self.assertEqual( + policy, + { + "p2mr_spend_input_bytes": 3_680, + "target_feerate_sats_per_byte": 1, + "safety_multiplier": 4, + }, + ) + + def test_prism_payout_policy_allows_fixed_min_output_bits_override(self) -> None: + server = coordinator() + + with patch.dict(os.environ, {"PRISM_PAYOUT_MIN_OUTPUT_BITS": "10000"}, clear=True): + policy = server.prism_payout_policy() + + self.assertEqual( + policy, + { + "p2mr_spend_input_bytes": 3_680, + "target_feerate_sats_per_byte": 1, + "safety_multiplier": 4, + "min_output_sats": 10_000, + }, + ) + + def test_prism_payout_policy_falls_back_to_legacy_min_output_sats_override(self) -> None: + server = coordinator() + + with patch.dict(os.environ, {"PRISM_PAYOUT_MIN_OUTPUT_SATS": "10000"}, clear=True): + policy = server.prism_payout_policy() + + self.assertEqual(policy["min_output_sats"], 10_000) + + def test_prism_payout_policy_min_output_bits_overrides_legacy_sats_override(self) -> None: + server = coordinator() + + with patch.dict( + os.environ, + { + "PRISM_PAYOUT_MIN_OUTPUT_BITS": "11000", + "PRISM_PAYOUT_MIN_OUTPUT_SATS": "10000", + }, + clear=True, + ): + policy = server.prism_payout_policy() + + self.assertEqual(policy["min_output_sats"], 11_000) + + def test_prism_coinbase_tag_defaults_to_prism(self) -> None: + with patch.dict(os.environ, {}, clear=True): + self.assertEqual(default_prism_coinbase_tag_hex(), "/PRISM/".encode("ascii").hex()) + + def test_prism_coinbase_tag_is_configurable_and_can_be_disabled(self) -> None: + with patch.dict(os.environ, {"PRISM_COINBASE_TAG": "/CUSTOM/"}, clear=True): + self.assertEqual(default_prism_coinbase_tag_hex(), "/CUSTOM/".encode("ascii").hex()) + with patch.dict(os.environ, {"PRISM_COINBASE_TAG": ""}, clear=True): + self.assertEqual(default_prism_coinbase_tag_hex(), "") + + def test_prism_coinbase_tag_rejects_non_printable_or_long_values(self) -> None: + for tag, message in ( + ("PRISM\n", "printable ASCII"), + ("P" * 41, "at most 40 bytes"), + ("PRISM-π", "ASCII"), + ): + with self.subTest(tag=tag), patch.dict( + os.environ, {"PRISM_COINBASE_TAG": tag}, clear=True + ): + with self.assertRaisesRegex(SystemExit, message): + default_prism_coinbase_tag_hex() + + def test_coinbase_script_sig_suffix_places_pool_tag_before_extranonce(self) -> None: + server = coordinator() + server.coinbase_tag_hex = "/PRISM/".encode("ascii").hex() + + suffix = server.coinbase_script_sig_suffix_hex("aabbccdd", "00" * 8) + + self.assertEqual(suffix, "/PRISM/".encode("ascii").hex() + "aabbccdd" + "00" * 8) + self.assertTrue(suffix.endswith("aabbccdd" + "00" * 8)) + + def test_prism_payout_policy_allows_formula_overrides(self) -> None: + server = coordinator() + + with patch.dict( + os.environ, + { + "PRISM_PAYOUT_P2MR_SPEND_INPUT_BYTES": "2500", + "PRISM_PAYOUT_TARGET_FEERATE_BITS_PER_BYTE": "2", + "PRISM_PAYOUT_SAFETY_MULTIPLIER": "3", + }, + clear=True, + ): + policy = server.prism_payout_policy() + + self.assertEqual( + policy, + { + "p2mr_spend_input_bytes": 2_500, + "target_feerate_sats_per_byte": 2, + "safety_multiplier": 3, + }, + ) + + def test_prism_payout_policy_formula_uses_legacy_feerate_alias(self) -> None: + server = coordinator() + + with patch.dict(os.environ, {"PRISM_PAYOUT_TARGET_FEERATE_SATS_PER_BYTE": "2"}, clear=True): + policy = server.prism_payout_policy() + + self.assertEqual(policy["target_feerate_sats_per_byte"], 2) + + def test_prism_payout_policy_formula_bits_feerate_overrides_legacy_alias(self) -> None: + server = coordinator() + + with patch.dict( + os.environ, + { + "PRISM_PAYOUT_TARGET_FEERATE_BITS_PER_BYTE": "3", + "PRISM_PAYOUT_TARGET_FEERATE_SATS_PER_BYTE": "2", + }, + clear=True, + ): + policy = server.prism_payout_policy() + + self.assertEqual(policy["target_feerate_sats_per_byte"], 3) + + def test_prism_payout_policy_rejects_invalid_floor_settings(self) -> None: + cases = [ + ({"PRISM_PAYOUT_MIN_OUTPUT_BITS": "0"}, "PRISM_PAYOUT_MIN_OUTPUT_BITS must be positive"), + ( + {"PRISM_PAYOUT_MIN_OUTPUT_BITS": "not-int"}, + "PRISM_PAYOUT_MIN_OUTPUT_BITS must be an integer", + ), + ({"PRISM_PAYOUT_MIN_OUTPUT_SATS": "0"}, "PRISM_PAYOUT_MIN_OUTPUT_SATS must be positive"), + ( + {"PRISM_PAYOUT_MIN_OUTPUT_SATS": "not-int"}, + "PRISM_PAYOUT_MIN_OUTPUT_SATS must be an integer", + ), + ( + {"PRISM_PAYOUT_SAFETY_MULTIPLIER": "0"}, + "PRISM_PAYOUT_SAFETY_MULTIPLIER must be positive", + ), + ( + {"PRISM_PAYOUT_TARGET_FEERATE_BITS_PER_BYTE": "0"}, + "PRISM_PAYOUT_TARGET_FEERATE_BITS_PER_BYTE must be positive", + ), + ] + for env_vars, expected in cases: + with self.subTest(env_vars=env_vars), patch.dict(os.environ, env_vars, clear=True): + server = coordinator() + with self.assertRaisesRegex(SystemExit, expected): + server.prism_payout_policy() + + def test_prism_pool_fee_address_config_validates_p2mr_address(self) -> None: + server = coordinator() + server.rpc = AddressRpc(valid_address="tq1fee", script_byte="88") + + with patch.dict( + os.environ, + { + "PRISM_POOL_FEE_ENABLED": "1", + "PRISM_POOL_FEE_BPS": "125", + "PRISM_POOL_FEE_ADDRESS": "tq1fee", + }, + clear=True, + ): + policy = server.prism_payout_policy() + + self.assertEqual(server.rpc.validated, ["tq1fee"]) + self.assertEqual( + policy["pool_fee_policy"], + { + "fee_bps": 125, + "recipient_id": "tq1fee", + "order_key": "tq1fee", + "p2mr_program_hex": "88" * 32, + }, + ) + + def test_prism_pool_fee_enabled_allows_zero_bps_policy(self) -> None: + server = coordinator() + server.rpc = AddressRpc(valid_address="tq1fee", script_byte="66") + + with patch.dict( + os.environ, + { + "PRISM_POOL_FEE_ENABLED": "1", + "PRISM_POOL_FEE_BPS": "0", + "PRISM_POOL_FEE_ADDRESS": "tq1fee", + }, + clear=True, + ): + policy = server.prism_payout_policy() + + self.assertEqual(policy["pool_fee_policy"]["fee_bps"], 0) + self.assertEqual(policy["pool_fee_policy"]["p2mr_program_hex"], "66" * 32) + + def test_prism_pool_fee_program_config_requires_recipient_identity(self) -> None: + server = coordinator() + + with patch.dict( + os.environ, + { + "PRISM_POOL_FEE_ENABLED": "1", + "PRISM_POOL_FEE_BPS": "125", + "PRISM_POOL_FEE_P2MR_PROGRAM_HEX": "55" * 32, + }, + clear=True, + ): + with self.assertRaisesRegex(SystemExit, "PRISM_POOL_FEE_RECIPIENT_ID"): + server.prism_payout_policy() + + def test_prism_pool_fee_program_config_uses_explicit_order_key(self) -> None: + server = coordinator() + + with patch.dict( + os.environ, + { + "PRISM_POOL_FEE_ENABLED": "1", + "PRISM_POOL_FEE_BPS": "125", + "PRISM_POOL_FEE_P2MR_PROGRAM_HEX": "55" * 32, + "PRISM_POOL_FEE_RECIPIENT_ID": "pool-fee", + "PRISM_POOL_FEE_ORDER_KEY": "000-pool-fee", + }, + clear=True, + ): + policy = server.prism_payout_policy() + + self.assertEqual( + policy["pool_fee_policy"], + { + "fee_bps": 125, + "recipient_id": "pool-fee", + "order_key": "000-pool-fee", + "p2mr_program_hex": "55" * 32, + }, + ) + + def test_prism_pool_fee_config_rejects_ambiguous_or_invalid_settings(self) -> None: + cases = [ + ( + {"PRISM_POOL_FEE_ENABLED": "1", "PRISM_POOL_FEE_ADDRESS": "tq1fee"}, + "PRISM_POOL_FEE_BPS", + ), + ( + { + "PRISM_POOL_FEE_ENABLED": "1", + "PRISM_POOL_FEE_BPS": "10001", + "PRISM_POOL_FEE_ADDRESS": "tq1fee", + }, + "between 0 and 10000", + ), + ( + { + "PRISM_POOL_FEE_ENABLED": "1", + "PRISM_POOL_FEE_BPS": "125", + "PRISM_POOL_FEE_ADDRESS": "tq1fee", + "PRISM_POOL_FEE_P2MR_PROGRAM_HEX": "55" * 32, + }, + "exactly one", + ), + ] + for env_vars, expected in cases: + with self.subTest(env_vars=env_vars), patch.dict(os.environ, env_vars, clear=True): + server = coordinator() + server.rpc = AddressRpc(valid_address="tq1fee") + with self.assertRaisesRegex(SystemExit, expected): + server.prism_payout_policy() + + def test_prism_pool_fee_config_rejects_disabled_fee_settings(self) -> None: + cases = [ + {"PRISM_POOL_FEE_BPS": "125"}, + {"PRISM_POOL_FEE_ADDRESS": "tq1fee"}, + {"PRISM_POOL_FEE_P2MR_PROGRAM_HEX": "55" * 32}, + {"PRISM_POOL_FEE_RECIPIENT_ID": "pool-fee"}, + { + "PRISM_POOL_FEE_ENABLED": "0", + "PRISM_POOL_FEE_BPS": "125", + "PRISM_POOL_FEE_ADDRESS": "tq1fee", + }, + ] + for env_vars in cases: + with self.subTest(env_vars=env_vars), patch.dict(os.environ, env_vars, clear=True): + server = coordinator() + server.rpc = AddressRpc(valid_address="tq1fee") + with self.assertRaisesRegex(SystemExit, "PRISM_POOL_FEE_ENABLED=1"): + server.prism_payout_policy() + + def test_prism_pool_fee_config_rejects_non_p2mr_fee_address(self) -> None: + server = coordinator() + server.rpc = AddressRpc(valid_address="tq1fee", p2mr=False) + + with patch.dict( + os.environ, + { + "PRISM_POOL_FEE_ENABLED": "1", + "PRISM_POOL_FEE_BPS": "125", + "PRISM_POOL_FEE_ADDRESS": "tq1fee", + }, + clear=True, + ): + with self.assertRaisesRegex(SystemExit, "P2MR"): + server.prism_payout_policy() + + def test_prism_ctv_settlement_config_uses_legacy_unit_aliases(self) -> None: + server = coordinator() + + with patch.dict( + os.environ, + { + "PRISM_CTV_SETTLEMENT_ENABLED": "1", + "PRISM_DIRECT_COINBASE_PAYOUT_FLOOR_SATS": "10485760", + "PRISM_CTV_FANOUT_FEE_MARKET_RATE_SATS_PER_1000_WEIGHT": "25", + }, + clear=True, + ): + config = server.prism_ctv_settlement_config() + + self.assertIsNotNone(config) + assert config is not None + self.assertEqual(config["direct_floor_sats"], 10_485_760) + self.assertEqual( + config["fanout_fee_rate_policy"], + {"market_fee_rate_sats_per_1000_weight": 25, "premium_bps": 12_000}, + ) + + def test_prism_ctv_settlement_config_uses_node_fee_estimate_by_default(self) -> None: + server = coordinator() + rpc = FeeEstimateRpc({"feerate": "0.00001001"}) + server.rpc = rpc + + with patch.dict(os.environ, {"PRISM_CTV_SETTLEMENT_ENABLED": "1"}, clear=True): + config = server.prism_ctv_settlement_config() + + self.assertIsNotNone(config) + assert config is not None + self.assertEqual( + config["fanout_fee_rate_policy"], + {"market_fee_rate_sats_per_1000_weight": 1001, "premium_bps": 12_000}, + ) + self.assertIn(("estimatesmartfee", [2]), rpc.calls) + + def test_prism_ctv_settlement_config_caches_node_fee_estimate_per_block_height(self) -> None: + server = coordinator() + rpc = FeeEstimateRpc({"feerate": "0.00001001"}) + server.rpc = rpc + + with patch.dict(os.environ, {"PRISM_CTV_SETTLEMENT_ENABLED": "1"}, clear=True): + first = server.prism_ctv_settlement_config(block_height=10) + rpc.estimate = {"feerate": "0.00002000"} + second = server.prism_ctv_settlement_config(block_height=10) + next_height = server.prism_ctv_settlement_config(block_height=11) + + assert first is not None + assert second is not None + assert next_height is not None + self.assertEqual( + first["fanout_fee_rate_policy"], + {"market_fee_rate_sats_per_1000_weight": 1001, "premium_bps": 12_000}, + ) + self.assertEqual(second["fanout_fee_rate_policy"], first["fanout_fee_rate_policy"]) + self.assertEqual( + next_height["fanout_fee_rate_policy"], + {"market_fee_rate_sats_per_1000_weight": 2000, "premium_bps": 12_000}, + ) + self.assertEqual( + [call for call in rpc.calls if call[0] == "estimatesmartfee"], + [("estimatesmartfee", [2]), ("estimatesmartfee", [2])], + ) + + def test_prism_ctv_settlement_config_separates_fee_cache_by_parent_hash(self) -> None: + server = coordinator() + rpc = FeeEstimateRpc({"feerate": "0.00001001"}) + server.rpc = rpc + + with patch.dict(os.environ, {"PRISM_CTV_SETTLEMENT_ENABLED": "1"}, clear=True): + first = server.prism_ctv_settlement_config(block_height=10, parent_hash="aa" * 32) + rpc.estimate = {"feerate": "0.00002000"} + same_parent = server.prism_ctv_settlement_config(block_height=10, parent_hash="aa" * 32) + reorg_parent = server.prism_ctv_settlement_config(block_height=10, parent_hash="bb" * 32) + + assert first is not None + assert same_parent is not None + assert reorg_parent is not None + self.assertEqual(same_parent["fanout_fee_rate_policy"], first["fanout_fee_rate_policy"]) + self.assertEqual( + reorg_parent["fanout_fee_rate_policy"], + {"market_fee_rate_sats_per_1000_weight": 2000, "premium_bps": 12_000}, + ) + self.assertEqual( + [call for call in rpc.calls if call[0] == "estimatesmartfee"], + [("estimatesmartfee", [2]), ("estimatesmartfee", [2])], + ) + + def test_prism_ctv_settlement_config_fails_closed_when_fee_estimate_unavailable(self) -> None: + server = coordinator() + server.rpc = FeeEstimateRpc({"errors": ["insufficient data"]}) + + with patch.dict(os.environ, {"PRISM_CTV_SETTLEMENT_ENABLED": "1"}, clear=True): + with self.assertRaisesRegex(RuntimeError, "unable to compute PRISM CTV fanout fee rate"): + server.prism_ctv_settlement_config() + + def test_prism_ctv_settlement_config_retries_after_fee_estimate_failure(self) -> None: + server = coordinator() + rpc = FeeEstimateRpc({"errors": ["insufficient data"]}) + server.rpc = rpc + + with patch.dict(os.environ, {"PRISM_CTV_SETTLEMENT_ENABLED": "1"}, clear=True): + with self.assertRaises(RuntimeError): + server.prism_ctv_settlement_config(block_height=10) + rpc.estimate = {"feerate": "0.00002000"} + recovered = server.prism_ctv_settlement_config(block_height=10) + + assert recovered is not None + self.assertEqual( + recovered["fanout_fee_rate_policy"], + {"market_fee_rate_sats_per_1000_weight": 2000, "premium_bps": 12_000}, + ) + self.assertEqual( + [call for call in rpc.calls if call[0] == "estimatesmartfee"], + [("estimatesmartfee", [2]), ("estimatesmartfee", [2])], + ) + + def test_ctv_broadcaster_daemon_uses_coordinator_ledger_and_config(self) -> None: + server = coordinator() + server.ctv_broadcaster_wallet = None + server.ctv_broadcaster_fee_sats = 0 + server.ctv_broadcaster_limit = 7 + captured: dict[str, object] = {} + + class FakeDaemon: + def __init__(self, ledger: object, broadcaster: object, *, fee_sats: int) -> None: + captured["ledger"] = ledger + captured["broadcaster"] = broadcaster + captured["fee_sats"] = fee_sats + + def run_once( + self, + *, + limit: int, + progress_callback: object, + chunk_size: int, + tip_refresh_pending: object, + chunk_callback: object, + ) -> object: + captured["limit"] = limit + captured["chunk_size"] = chunk_size + captured["tip_refresh_pending"] = tip_refresh_pending + captured["chunk_callback"] = chunk_callback + return SimpleNamespace( + scanned_count=1, + submitted_count=0, + updated_count=1, + failed_count=0, + yielded_to_tip_refresh=False, + ) + + with patch("lab.prism.prism_coordinator.CtvFanoutBroadcastDaemon", FakeDaemon): + result = server.run_ctv_fanout_broadcaster_once() + + self.assertIs(captured["ledger"], server.ledger) + self.assertEqual(captured["fee_sats"], 0) + self.assertEqual(captured["limit"], 7) + self.assertEqual(captured["chunk_size"], 5) + self.assertTrue(callable(captured["tip_refresh_pending"])) + self.assertTrue(callable(captured["chunk_callback"])) + self.assertEqual(result.updated_count, 1) + self.assertIsNotNone(captured["broadcaster"]) + + def test_ctv_broadcaster_daemon_requires_wallet_for_cpfp_fee(self) -> None: + server = coordinator() + server.ctv_broadcaster_wallet = None + server.ctv_broadcaster_fee_sats = 1 + + with self.assertRaisesRegex(ValueError, "ctv_broadcaster_wallet is required"): + server.make_ctv_fanout_broadcast_daemon() + + def test_make_ledger_requires_explicit_memory_opt_in(self) -> None: + server = PrismCoordinator.__new__(PrismCoordinator) + + with patch.dict(os.environ, {}, clear=True): + with self.assertRaises(SystemExit): + server.make_ledger() + + with patch.dict(os.environ, {"PRISM_ALLOW_MEMORY_LEDGER": "1"}, clear=True): + ledger = server.make_ledger() + + self.assertEqual(ledger.backend_name, "memory") + + def test_trusted_ledger_key_must_be_configured_or_explicitly_test_mode(self) -> None: + server = PrismCoordinator.__new__(PrismCoordinator) + + with patch.dict(os.environ, {}, clear=True): + with self.assertRaises(SystemExit): + server.load_trusted_ledger_writer_public_key() + + with patch.dict(os.environ, {"PRISM_LEDGER_WRITER_PUBLIC_KEY_HEX": "aa" * 32}, clear=True): + self.assertEqual(server.load_trusted_ledger_writer_public_key(), "aa" * 32) + + with patch.dict(os.environ, {"PRISM_ALLOW_BUNDLE_EMBEDDED_LEDGER_KEY": "1"}, clear=True): + self.assertIsNone(server.load_trusted_ledger_writer_public_key()) + + def test_fixed_ledger_session_token_requires_explicit_opt_in(self) -> None: + server = PrismCoordinator.__new__(PrismCoordinator) + env = { + "PRISM_POSTGRES_PSQL_COMMAND": "psql postgresql://example.invalid/qbit", + "PRISM_LEDGER_WRITER_SESSION_TOKEN": "fixed-session", + "PRISM_PUBLIC_REWARD_WINDOW_CACHE_SECONDS": "47", + } + + with patch.dict(os.environ, env, clear=True): + with self.assertRaises(SystemExit): + server.make_ledger() + + with patch.dict(os.environ, {**env, "PRISM_ALLOW_FIXED_LEDGER_SESSION_TOKEN": "1"}, clear=True): + with patch("lab.prism.prism_coordinator.PsqlShareLedger") as fake_ledger: + fake_ledger.return_value = SimpleNamespace(backend_name="postgres-psql") + ledger = server.make_ledger() + + self.assertEqual(ledger.backend_name, "postgres-psql") + self.assertEqual(fake_ledger.call_args.kwargs["writer_session_token"], "fixed-session") + self.assertEqual( + fake_ledger.call_args.kwargs["reward_window_cache_seconds"], + 47.0, + ) + + def test_same_tip_retention_requires_connection_derived_production_bound(self) -> None: + with self.assertRaisesRegex( + SystemExit, + "PRISM_STRATUM_SAME_TIP_JOB_RETENTION_PER_CONNECTION", + ): + validate_same_tip_job_retention_limits( + retention_seconds=30, + per_connection=0, + max_connections=0, + production=False, + ) + with self.assertRaisesRegex(SystemExit, "PRISM_STRATUM_MAX_CONNECTIONS"): + validate_same_tip_job_retention_limits( + retention_seconds=30, + per_connection=64, + max_connections=0, + production=True, + ) + + validate_same_tip_job_retention_limits( + retention_seconds=30, + per_connection=64, + max_connections=1_900, + production=True, + ) + validate_same_tip_job_retention_limits( + retention_seconds=30, + per_connection=64, + max_connections=0, + production=False, + ) + validate_same_tip_job_retention_limits( + retention_seconds=0, + per_connection=0, + max_connections=0, + production=True, + ) + + def test_production_gate_rejects_prism_test_bypasses_without_capacity_evidence(self) -> None: + base = { + "QBIT_PRODUCTION": "1", + "QBIT_RPC_USER": "qbitrpc", + "QBIT_RPC_PASSWORD": "not-default", + "PRISM_POSTGRES_PSQL_COMMAND": "psql postgresql://example.invalid/qbit", + "PRISM_POSTGRES_PASSWORD": "not-default", + "PRISM_MANIFEST_SIGNING_SEED_HEX": "42" * 32, + "PRISM_LEDGER_ATTESTATION_SIGNING_SEED_HEX": "43" * 32, + "PRISM_LEDGER_WRITER_PUBLIC_KEY_HEX": "44" * 32, + "PRISM_LEDGER_WRITER_ID": "managed-writer", + "PRISM_LEDGER_WRITER_EPOCH": "7", + "PRISM_AUDIT_DIR": "/var/lib/qbit/prism/audit", + "PRISM_EVIDENCE_PATH": "/var/lib/qbit/prism/evidence.json", + "PRISM_STRATUM_STALE_GRACE_SECONDS": "3", + "PRISM_STRATUM_SHARE_DIFF": "1024", + "PRISM_STRATUM_VARDIFF_MIN_DIFF": "1024", + "PRISM_STRATUM_VARDIFF_START_DIFF": "4096", + "PRISM_STRATUM_VARDIFF_MAX_DIFF": "65536", + "PRISM_STRATUM_MAX_CONNECTIONS": "1900", + } + for name in ( + "PRISM_ALLOW_MEMORY_LEDGER", + "PRISM_ALLOW_TEST_SIGNING_SEEDS", + "PRISM_ALLOW_BUNDLE_EMBEDDED_LEDGER_KEY", + "PRISM_ALLOW_FIXED_LEDGER_SESSION_TOKEN", + ): + with self.subTest(name=name), patch.dict(os.environ, {**base, name: "1"}, clear=True): + with self.assertRaisesRegex(SystemExit, name): + validate_prism_production_gate() + + with patch.dict(os.environ, base, clear=True): + validate_prism_production_gate() + + with patch.dict( + os.environ, + {**base, "PRISM_STRATUM_MAX_CONNECTIONS": "0"}, + clear=True, + ): + with self.assertRaisesRegex(SystemExit, "PRISM_STRATUM_MAX_CONNECTIONS"): + validate_prism_production_gate() + + with patch.dict( + os.environ, + {**base, "PRISM_STRATUM_INITIAL_JOB_TIMEOUT_SECONDS": "0"}, + clear=True, + ): + with self.assertRaisesRegex( + SystemExit, + "PRISM_STRATUM_INITIAL_JOB_TIMEOUT_SECONDS", + ): + validate_prism_production_gate() + + with patch.dict( + os.environ, + {**base, "PRISM_STRATUM_MAX_PENDING_INITIAL_JOBS": "0"}, + clear=True, + ): + with self.assertRaisesRegex( + SystemExit, + "PRISM_STRATUM_MAX_PENDING_INITIAL_JOBS", + ): + validate_prism_production_gate() + + # Mainnet follows the same bounded-grace rule as every other chain: + # zero grace rejects every in-flight prior-tip share at each block, + # which miners read as pool failure. + with patch.dict( + os.environ, + {**base, "QBIT_CHAIN": "mainnet", "PRISM_STRATUM_STALE_GRACE_SECONDS": "3"}, + clear=True, + ): + validate_prism_production_gate() + + # Off mainnet, production mode accepts a bounded grace window too. + with patch.dict( + os.environ, + {**base, "PRISM_STRATUM_STALE_GRACE_SECONDS": "2"}, + clear=True, + ): + validate_prism_production_gate() + + with patch.dict(os.environ, {**base, "PRISM_POSTGRES_PASSWORD": "change-this"}, clear=True): + with self.assertRaisesRegex(SystemExit, "PRISM_POSTGRES_PASSWORD"): + validate_prism_production_gate() + + with patch.dict( + os.environ, + { + **base, + "PRISM_POSTGRES_PASSWORD": "not-default", + "PRISM_POSTGRES_PSQL_COMMAND": "", + "PRISM_DATABASE_URL": "postgresql://qbit:change-this@prism-postgres:5432/qbit", + }, + clear=True, + ): + with self.assertRaisesRegex(SystemExit, "PRISM_DATABASE_URL"): + validate_prism_production_gate() + + def test_production_gate_rejects_unsafe_difficulty_without_capacity_gate(self) -> None: + base = { + "QBIT_PRODUCTION": "1", + "QBIT_RPC_USER": "qbitrpc", + "QBIT_RPC_PASSWORD": "not-default", + "PRISM_POSTGRES_PSQL_COMMAND": "psql postgresql://example.invalid/qbit", + "PRISM_POSTGRES_PASSWORD": "not-default", + "PRISM_MANIFEST_SIGNING_SEED_HEX": "42" * 32, + "PRISM_LEDGER_ATTESTATION_SIGNING_SEED_HEX": "43" * 32, + "PRISM_LEDGER_WRITER_PUBLIC_KEY_HEX": "44" * 32, + "PRISM_LEDGER_WRITER_ID": "managed-writer", + "PRISM_LEDGER_WRITER_EPOCH": "7", + "PRISM_AUDIT_DIR": "/var/lib/qbit/prism/audit", + "PRISM_EVIDENCE_PATH": "/var/lib/qbit/prism/evidence.json", + "PRISM_STRATUM_STALE_GRACE_SECONDS": "3", + "PRISM_STRATUM_SHARE_DIFF": "1024", + "PRISM_STRATUM_VARDIFF_MIN_DIFF": "1024", + "PRISM_STRATUM_VARDIFF_START_DIFF": "4096", + "PRISM_STRATUM_VARDIFF_MAX_DIFF": "65536", + } + cases = ( + ({"PRISM_STRATUM_SHARE_DIFF": ""}, "requires PRISM_STRATUM_SHARE_DIFF"), + ({"PRISM_STRATUM_SHARE_DIFF": "not-a-decimal"}, "must be a decimal number"), + ({"PRISM_STRATUM_SHARE_DIFF": "NaN"}, "PRISM_STRATUM_SHARE_DIFF must be positive"), + ({"PRISM_STRATUM_SHARE_DIFF": "0"}, "PRISM_STRATUM_SHARE_DIFF must be positive"), + ({"PRISM_STRATUM_SHARE_DIFF": "1e-9"}, "lab-only 1e-9 difficulty"), + ({"PRISM_STRATUM_VARDIFF_MIN_DIFF": "8192"}, "minimum exceeds its start"), + ({"PRISM_STRATUM_VARDIFF_START_DIFF": "131072"}, "start exceeds its maximum"), + ) + for overrides, message in cases: + with self.subTest(overrides=overrides), patch.dict( + os.environ, + {**base, **overrides}, + clear=True, + ): + with self.assertRaisesRegex(SystemExit, message): + validate_prism_production_gate() + + def test_mainnet_implies_production_gate(self) -> None: + with patch.dict( + os.environ, + { + "QBIT_CHAIN": "mainnet", + "PRISM_STRATUM_STALE_GRACE_SECONDS": "3", + }, + clear=True, + ): + with self.assertRaisesRegex(SystemExit, "requires PRISM_STRATUM_SHARE_DIFF"): + validate_prism_production_gate() + + def test_compatibility_production_flag_implies_production_gate(self) -> None: + with patch.dict( + os.environ, + { + "QBIT_CHAIN": "testnet4", + "QBIT_TOOLS_PRODUCTION": "1", + "PRISM_STRATUM_STALE_GRACE_SECONDS": "3", + }, + clear=True, + ): + with self.assertRaisesRegex(SystemExit, "requires PRISM_STRATUM_SHARE_DIFF"): + validate_prism_production_gate() + + def test_mainnet_ctv_requires_static_fee_rate_before_runtime_startup(self) -> None: + env = { + "QBIT_CHAIN": "mainnet", + "QBIT_RPC_USER": "qbitrpc", + "QBIT_RPC_PASSWORD": "not-default", + "PRISM_POSTGRES_PSQL_COMMAND": "psql postgresql://example.invalid/qbit", + "PRISM_POSTGRES_PASSWORD": "not-default", + "PRISM_MANIFEST_SIGNING_SEED_HEX": "42" * 32, + "PRISM_LEDGER_ATTESTATION_SIGNING_SEED_HEX": "43" * 32, + "PRISM_LEDGER_WRITER_PUBLIC_KEY_HEX": "44" * 32, + "PRISM_LEDGER_WRITER_ID": "managed-writer", + "PRISM_LEDGER_WRITER_EPOCH": "7", + "PRISM_AUDIT_DIR": "/var/lib/qbit/prism/audit", + "PRISM_EVIDENCE_PATH": "/var/lib/qbit/prism/evidence.json", + "PRISM_STRATUM_STALE_GRACE_SECONDS": "3", + "PRISM_STRATUM_SHARE_DIFF": "1024", + "PRISM_STRATUM_VARDIFF_MIN_DIFF": "1024", + "PRISM_STRATUM_VARDIFF_START_DIFF": "4096", + "PRISM_STRATUM_VARDIFF_MAX_DIFF": "65536", + "PRISM_CTV_SETTLEMENT_ENABLED": "1", + } + with ( + patch.dict(os.environ, env, clear=True), + self.assertRaisesRegex( + SystemExit, + "PRISM_CTV_FANOUT_FEE_MARKET_RATE_BITS_PER_1000_WEIGHT", + ), + ): + validate_prism_production_gate() + + def test_live_chain_identity_accepts_main_alias_and_pinned_genesis(self) -> None: + genesis = "12" * 32 + server = PrismCoordinator.__new__(PrismCoordinator) + server.qbit_chain = "mainnet" + + class Rpc: + def call(self, method: str, params: object = None) -> object: + if method == "getblockchaininfo": + return { + "chain": "main", + "initialblockdownload": False, + "blocks": 100, + "headers": 100, + } + if method == "getnetworkinfo": + return {"connections": 2} + if method == "getblockhash" and params == [0]: + return genesis + raise RuntimeError(method) + + server.rpc = Rpc() + with patch.dict(os.environ, {"QBIT_EXPECTED_GENESIS_HASH": genesis}, clear=True): + server.validate_live_chain_identity() + + def test_live_chain_identity_rejects_incomplete_public_readiness(self) -> None: + server = PrismCoordinator.__new__(PrismCoordinator) + server.qbit_chain = "mainnet" + genesis = "12" * 32 + + cases = ( + ({"chain": "main", "blocks": 10, "headers": 10}, {"connections": 1}, "initial block"), + ( + {"chain": "main", "initialblockdownload": False, "blocks": 9, "headers": 10}, + {"connections": 1}, + "not caught up", + ), + ( + {"chain": "main", "initialblockdownload": False, "blocks": 10, "headers": 10}, + {"connections": 0}, + "requires at least 1", + ), + ) + for blockchain_info, network_info, message in cases: + with self.subTest(message=message): + server.rpc = SimpleNamespace( + call=lambda method, params=None: ( + blockchain_info + if method == "getblockchaininfo" + else network_info + if method == "getnetworkinfo" + else genesis + ) + ) + with ( + patch.dict(os.environ, {"QBIT_EXPECTED_GENESIS_HASH": genesis}, clear=True), + self.assertRaisesRegex(RuntimeError, message), + ): + server.validate_live_chain_identity() + + def test_live_template_preflight_enforces_freshness_and_relay_fee_floor(self) -> None: + server = PrismCoordinator.__new__(PrismCoordinator) + previous_hash = "34" * 32 + template = {"height": 1, "curtime": int(time.time()), "previousblockhash": previous_hash} + server.current_template_artifacts = lambda: SimpleNamespace( + template=template, + previousblockhash=previous_hash, + ) + server.rpc = SimpleNamespace( + call=lambda method, params=None: { + "minrelaytxfee": "0.00001000", + "mempoolminfee": "0.00001000", + } + ) + server._ctv_fanout_market_fee_rate_cache = {} + + enabled = { + "PRISM_CTV_SETTLEMENT_ENABLED": "1", + "PRISM_CTV_FANOUT_FEE_MARKET_RATE_BITS_PER_1000_WEIGHT": "1000", + "PRISM_TEMPLATE_MAX_AGE_SECONDS": "120", + } + with patch.dict(os.environ, enabled, clear=True): + server.validate_live_template_and_fee_policy() + + server._ctv_fanout_market_fee_rate_cache = {} + with ( + patch.dict( + os.environ, + {**enabled, "PRISM_CTV_FANOUT_FEE_MARKET_RATE_BITS_PER_1000_WEIGHT": "1"}, + clear=True, + ), + self.assertRaisesRegex(RuntimeError, "below the connected node relay floor"), + ): + server.validate_live_template_and_fee_policy() + + template["curtime"] = int(time.time()) - 121 + with ( + patch.dict(os.environ, enabled, clear=True), + self.assertRaisesRegex(RuntimeError, "block template is stale"), + ): + server.validate_live_template_and_fee_policy() + + def test_live_chain_identity_rejects_wrong_chain_or_genesis(self) -> None: + server = PrismCoordinator.__new__(PrismCoordinator) + server.qbit_chain = "mainnet" + server.rpc = SimpleNamespace( + call=lambda method, params=None: ( + {"chain": "regtest"} if method == "getblockchaininfo" else "34" * 32 + ) + ) + with patch.dict(os.environ, {"QBIT_EXPECTED_GENESIS_HASH": "12" * 32}, clear=True): + with self.assertRaisesRegex(RuntimeError, "does not match RPC chain"): + server.validate_live_chain_identity() + + server.rpc = SimpleNamespace( + call=lambda method, params=None: ( + {"chain": "main"} if method == "getblockchaininfo" else "34" * 32 + ) + ) + with patch.dict(os.environ, {"QBIT_EXPECTED_GENESIS_HASH": "12" * 32}, clear=True): + with self.assertRaisesRegex(RuntimeError, "does not match the connected"): + server.validate_live_chain_identity() + + def _pending_append(self, tag: str, accepted_at_ms: int = 2) -> PendingShareAppend: + from lab.prism.share_ledger import PendingShare + + return PendingShareAppend( + pending_share=PendingShare( + share_id=f"miner-a:{tag}", + miner_id="miner-a", + order_key="miner-a", + p2mr_program_hex="11" * 32, + share_difficulty=1, + network_difficulty=1, + template_height=10, + job_id="job-1", + job_issued_at_ms=1, + accepted_at_ms=accepted_at_ms, + ntime=1_700_000_000, + ), + username="miner-a", + job_id="job-1", + block_hash_hex=tag * 32, + collection_only=False, + credit_policy=None, + ) + + def test_prism_coinbase_output_policy_defaults_to_canonical_and_is_omitted(self) -> None: + server = coordinator() + server.rpc = AddressRpc(valid_address="tq1fee", script_byte="88") + + with patch.dict( + os.environ, + { + "PRISM_POOL_FEE_ENABLED": "1", + "PRISM_POOL_FEE_BPS": "125", + "PRISM_POOL_FEE_ADDRESS": "tq1fee", + }, + clear=True, + ): + policy = server.prism_payout_policy() + + self.assertNotIn("coinbase_output_policy", policy) + + def test_prism_coinbase_output_policy_explicit_canonical_is_omitted(self) -> None: + server = coordinator() + server.rpc = AddressRpc(valid_address="tq1fee", script_byte="88") + + with patch.dict( + os.environ, + { + "PRISM_COINBASE_OUTPUT_POLICY": "canonical", + "PRISM_POOL_FEE_ENABLED": "1", + "PRISM_POOL_FEE_BPS": "125", + "PRISM_POOL_FEE_ADDRESS": "tq1fee", + }, + clear=True, + ): + policy = server.prism_payout_policy() + + self.assertNotIn("coinbase_output_policy", policy) + + def test_prism_coinbase_output_policy_pool_fee_first_is_included(self) -> None: + server = coordinator() + server.rpc = AddressRpc(valid_address="tq1fee", script_byte="88") + + with patch.dict( + os.environ, + { + "PRISM_COINBASE_OUTPUT_POLICY": "pool-fee-first", + "PRISM_POOL_FEE_ENABLED": "1", + "PRISM_POOL_FEE_BPS": "125", + "PRISM_POOL_FEE_ADDRESS": "tq1fee", + }, + clear=True, + ): + policy = server.prism_payout_policy() + + self.assertEqual(policy["coinbase_output_policy"], "pool-fee-first") + self.assertEqual(policy["pool_fee_policy"]["fee_bps"], 125) + + def test_prism_coinbase_output_policy_rejects_unknown_values(self) -> None: + for invalid in ("fee-first", "POOL-FEE-FIRST", "canonical "): + with self.subTest(invalid=invalid), patch.dict( + os.environ, + { + "PRISM_COINBASE_OUTPUT_POLICY": invalid, + "PRISM_POOL_FEE_ENABLED": "1", + "PRISM_POOL_FEE_BPS": "125", + "PRISM_POOL_FEE_ADDRESS": "tq1fee", + }, + clear=True, + ): + server = coordinator() + server.rpc = AddressRpc(valid_address="tq1fee") + with self.assertRaisesRegex( + SystemExit, + "PRISM_COINBASE_OUTPUT_POLICY must be one of: canonical, pool-fee-first", + ): + server.prism_payout_policy() + + def test_prism_coinbase_output_policy_pool_fee_first_requires_pool_fee(self) -> None: + cases = [ + {"PRISM_COINBASE_OUTPUT_POLICY": "pool-fee-first"}, + { + "PRISM_COINBASE_OUTPUT_POLICY": "pool-fee-first", + "PRISM_POOL_FEE_ENABLED": "0", + }, + ] + for env_vars in cases: + with self.subTest(env_vars=env_vars), patch.dict(os.environ, env_vars, clear=True): + server = coordinator() + with self.assertRaisesRegex(SystemExit, "requires PRISM_POOL_FEE_ENABLED=1"): + server.prism_payout_policy() + + def test_generated_coordinator_session_advertises_heartbeat_capability(self) -> None: + server = PrismCoordinator.__new__(PrismCoordinator) + env = { + "PRISM_POSTGRES_PSQL_COMMAND": "psql postgresql://example.invalid/qbit", + } + + with patch.dict(os.environ, env, clear=True), patch( + "lab.prism.prism_coordinator.PsqlShareLedger" + ) as fake_ledger: + fake_ledger.return_value = SimpleNamespace(backend_name="postgres-psql") + server.make_ledger() + + self.assertTrue( + fake_ledger.call_args.kwargs["writer_session_token"].startswith( + WRITER_LEASE_HEARTBEAT_SESSION_PREFIX + ) + ) + + +class PrismVardiffInitialConvergenceKillSwitchTests(unittest.TestCase): + """PRISM_STRATUM_VARDIFF_INITIAL_CONVERGENCE=0 must fully disengage. + + The switch is documented as restoring the pre-#132 behavior exactly, so + turning it off has to stop the initial-convergence settings from having + any say in whether the coordinator starts. VardiffConfig enforces its + invariants on every instance, so the loader is what has to keep an inert + setting from reaching them as a startup failure. + """ + + START = Decimal("0.000000001") + + # Each entry is one initial-convergence setting that is invalid only + # relative to the relaxation, plus the message the enabled loader rejects + # it with. Raising MAX_STEP_UP past the initial bound is the reproducer + # that motivated this: it is an ordinary tuning change that becomes + # unrelated to anything once the relaxation is off. + DISABLED_SHOULD_TOLERATE = ( + ( + {"PRISM_STRATUM_VARDIFF_MAX_STEP_UP": "128"}, + "PRISM_STRATUM_VARDIFF_INITIAL_MAX_STEP_UP is below", + ), + ( + {"PRISM_STRATUM_VARDIFF_INITIAL_MAX_STEP_UP": "2"}, + "PRISM_STRATUM_VARDIFF_INITIAL_MAX_STEP_UP is below", + ), + ( + {"PRISM_STRATUM_VARDIFF_INITIAL_MIN_SHARES": "0"}, + "PRISM_STRATUM_VARDIFF_INITIAL_MIN_SHARES must be at least 1", + ), + ( + {"PRISM_STRATUM_VARDIFF_INITIAL_MIN_STEP_UP": "0.5"}, + "PRISM_STRATUM_VARDIFF_INITIAL_MIN_STEP_UP must be at least 1", + ), + ( + {"PRISM_STRATUM_VARDIFF_INITIAL_MIN_SECONDS": "0"}, + "PRISM_STRATUM_VARDIFF_INITIAL_MIN_SECONDS must be positive", + ), + ) + + def test_disabled_initial_settings_never_fail_startup(self) -> None: + for overrides, _message in self.DISABLED_SHOULD_TOLERATE: + with self.subTest(overrides=overrides): + config = load_prism_vardiff_config( + self.START, + environ={ + "PRISM_STRATUM_VARDIFF_INITIAL_CONVERGENCE": "0", + **overrides, + }, + ) + + self.assertFalse(config.initial_convergence_enabled) + # Whatever the loader had to do to the inert values, the + # config it returns must satisfy the invariants rather than + # having escaped them. + self.assertGreaterEqual( + config.initial_max_step_factor, config.max_step_factor + ) + self.assertGreaterEqual(config.initial_min_step_factor, 1) + self.assertGreaterEqual(config.initial_min_accepted_shares, 1) + self.assertGreater(config.initial_min_elapsed_seconds, 0) + + def test_disabled_initial_settings_are_not_parsed(self) -> None: + config = load_prism_vardiff_config( + self.START, + environ={ + "PRISM_STRATUM_VARDIFF_INITIAL_CONVERGENCE": "0", + "PRISM_STRATUM_VARDIFF_INITIAL_MAX_STEP_UP": "not-a-decimal", + "PRISM_STRATUM_VARDIFF_INITIAL_MIN_SHARES": "not-an-integer", + "PRISM_STRATUM_VARDIFF_INITIAL_MIN_STEP_UP": "NaN", + "PRISM_STRATUM_VARDIFF_INITIAL_MIN_SECONDS": "-1", + }, + ) + + self.assertFalse(config.initial_convergence_enabled) + self.assertEqual( + config.initial_max_step_factor, + DEFAULT_PRISM_VARDIFF_INITIAL_MAX_STEP_UP, + ) + + def test_enabled_validation_is_unchanged(self) -> None: + for overrides, message in self.DISABLED_SHOULD_TOLERATE: + with self.subTest(overrides=overrides): + with self.assertRaises(SystemExit) as raised: + load_prism_vardiff_config(self.START, environ=dict(overrides)) + + self.assertIn(message, str(raised.exception)) + + def test_enabled_still_accepts_a_valid_custom_relaxation(self) -> None: + config = load_prism_vardiff_config( + self.START, + environ={ + "PRISM_STRATUM_VARDIFF_MAX_STEP_UP": "8", + "PRISM_STRATUM_VARDIFF_INITIAL_MAX_STEP_UP": "128", + "PRISM_STRATUM_VARDIFF_INITIAL_MIN_SHARES": "4", + "PRISM_STRATUM_VARDIFF_INITIAL_MIN_STEP_UP": "2", + }, + ) + + self.assertTrue(config.initial_convergence_enabled) + # Operator values reach the config untouched while the switch is on. + self.assertEqual(config.max_step_factor, Decimal("8")) + self.assertEqual(config.initial_max_step_factor, Decimal("128")) + self.assertEqual(config.initial_min_accepted_shares, 4) + self.assertEqual(config.initial_min_step_factor, Decimal("2")) + + def test_validation_unrelated_to_the_relaxation_still_applies_when_off( + self, + ) -> None: + # The switch relaxes the initial-convergence settings only. The + # observation-only arrival threshold and the ordinary step bound are + # read regardless of it, so they keep failing closed. + for overrides, message in ( + ( + {"PRISM_STRATUM_VARDIFF_HIGH_DIFF_ARRIVAL_DIFF": "0"}, + "PRISM_STRATUM_VARDIFF_HIGH_DIFF_ARRIVAL_DIFF must be positive", + ), + ( + {"PRISM_STRATUM_VARDIFF_MAX_STEP_UP": "not-a-number"}, + "PRISM_STRATUM_VARDIFF_MAX_STEP_UP must be a decimal number", + ), + ): + with self.subTest(overrides=overrides): + with self.assertRaises(SystemExit) as raised: + load_prism_vardiff_config( + self.START, + environ={ + "PRISM_STRATUM_VARDIFF_INITIAL_CONVERGENCE": "0", + **overrides, + }, + ) + + self.assertIn(message, str(raised.exception)) + + def test_tolerated_settings_stay_inert(self) -> None: + # The whole justification for accepting these configurations is that + # the relaxation is off, so pin that: neither the early trigger nor + # the wider step bound may act on a config the loader had to adjust. + config = load_prism_vardiff_config( + self.START, + environ={ + "PRISM_STRATUM_VARDIFF_INITIAL_CONVERGENCE": "0", + "PRISM_STRATUM_VARDIFF_MAX_STEP_UP": "128", + "PRISM_STRATUM_VARDIFF_MAX_DIFF": "1000000000", + }, + ) + current = Decimal("100") + + self.assertFalse( + vardiff.initial_convergence_ready( + current_difficulty=current, + accepted_shares=1000, + accepted_difficulty=Decimal("1000000"), + elapsed_seconds=Decimal("60"), + config=config, + ) + ) + # Even asked for the relaxation directly, the step stays on the + # ordinary bound: 128x, not the coerced initial factor. + self.assertEqual( + vardiff.calculate_next_difficulty( + current_difficulty=current, + accepted_shares=1000, + elapsed_seconds=Decimal("60"), + config=config, + accepted_difficulty=Decimal("100000000"), + initial_convergence=True, + ), + current * config.max_step_factor, + ) + + +class PrismCoordinatorReliabilityTests(unittest.TestCase): + def _bare_coordinator(self) -> PrismCoordinator: + server = PrismCoordinator.__new__(PrismCoordinator) + server.lock = threading.RLock() + server.stop_event = threading.Event() + server._heartbeats = {} + server._watchdog_pauses = {} + server._heartbeats_lock = threading.Lock() + server.watchdog_timeout_seconds = 120.0 + server.watchdog_interval_seconds = 15.0 + return server + + def test_positive_float_env_rejects_non_finite_values(self) -> None: + for raw in ("nan", "inf", "-inf"): + with self.subTest(raw=raw), patch.dict( + os.environ, {"PRISM_WATCHDOG_TIMEOUT_SECONDS": raw}, clear=True + ): + with self.assertRaisesRegex(SystemExit, "PRISM_WATCHDOG_TIMEOUT_SECONDS must be finite"): + env_positive_float("PRISM_WATCHDOG_TIMEOUT_SECONDS", 120.0) + + +class PrismListenerProfileTests(unittest.TestCase): + def test_highdiff_listener_disabled_without_port(self) -> None: + with patch.dict(os.environ, {}, clear=False): + clear_stratum_diff_env() + base = load_prism_vardiff_config(Decimal("0.000000001")) + self.assertIsNone(load_prism_highdiff_listener("0.0.0.0", base)) + + def test_highdiff_listener_defaults_to_nicehash_floor(self) -> None: + with patch.dict(os.environ, {}, clear=False): + clear_stratum_diff_env() + os.environ["PRISM_STRATUM_HIGHDIFF_PORT"] = "4334" + base = load_prism_vardiff_config(Decimal("0.000000001")) + profile = load_prism_highdiff_listener("10.0.0.1", base) + + assert profile is not None + self.assertEqual(profile.name, "highdiff") + self.assertEqual(profile.bind, "10.0.0.1") + self.assertEqual(profile.port, 4334) + self.assertEqual(profile.heartbeat_name, "stratum_accept_highdiff") + self.assertEqual(profile.share_difficulty, Decimal("500000")) + self.assertEqual(profile.minimum_advertised_difficulty, Decimal("500000")) + self.assertEqual(profile.vardiff_config.min_difficulty, Decimal("500000")) + self.assertEqual(profile.vardiff_config.startup_difficulty, Decimal("500000")) + self.assertEqual(profile.vardiff_config.max_difficulty, Decimal("4294967296")) + # Everything but the difficulty bounds is inherited from the base config. + self.assertEqual(profile.vardiff_config.enabled, base.enabled) + self.assertEqual( + profile.vardiff_config.target_share_interval_seconds, + base.target_share_interval_seconds, + ) + self.assertEqual( + profile.vardiff_config.retarget_interval_seconds, + base.retarget_interval_seconds, + ) + self.assertEqual(profile.vardiff_config.max_step_factor, base.max_step_factor) + + def test_highdiff_listener_env_overrides(self) -> None: + with patch.dict(os.environ, {}, clear=False): + clear_stratum_diff_env() + os.environ["PRISM_STRATUM_HIGHDIFF_PORT"] = "4335" + os.environ["PRISM_STRATUM_HIGHDIFF_BIND"] = "127.0.0.2" + os.environ["PRISM_STRATUM_HIGHDIFF_MIN_DIFF"] = "600000" + os.environ["PRISM_STRATUM_HIGHDIFF_START_DIFF"] = "1000000" + os.environ["PRISM_STRATUM_HIGHDIFF_MAX_DIFF"] = "8000000" + os.environ["PRISM_STRATUM_HIGHDIFF_SHARE_DIFF"] = "700000" + base = load_prism_vardiff_config(Decimal("0.000000001")) + profile = load_prism_highdiff_listener("0.0.0.0", base) + + assert profile is not None + self.assertEqual(profile.bind, "127.0.0.2") + self.assertEqual(profile.port, 4335) + self.assertEqual(profile.share_difficulty, Decimal("700000")) + self.assertEqual(profile.minimum_advertised_difficulty, Decimal("600000")) + self.assertEqual(profile.vardiff_config.min_difficulty, Decimal("600000")) + self.assertEqual(profile.vardiff_config.startup_difficulty, Decimal("1000000")) + self.assertEqual(profile.vardiff_config.max_difficulty, Decimal("8000000")) + + def test_highdiff_listener_rejects_inconsistent_bounds(self) -> None: + base = load_prism_vardiff_config(Decimal("0.000000001")) + with patch.dict(os.environ, {}, clear=False): + clear_stratum_diff_env() + os.environ["PRISM_STRATUM_HIGHDIFF_PORT"] = "4334" + os.environ["PRISM_STRATUM_HIGHDIFF_MIN_DIFF"] = "1000000" + with self.assertRaises(SystemExit): + load_prism_highdiff_listener("0.0.0.0", base) + with patch.dict(os.environ, {}, clear=False): + clear_stratum_diff_env() + os.environ["PRISM_STRATUM_HIGHDIFF_PORT"] = "4334" + os.environ["PRISM_STRATUM_HIGHDIFF_MAX_DIFF"] = "400000" + with self.assertRaises(SystemExit): + load_prism_highdiff_listener("0.0.0.0", base) + for bad_port in ("not-a-port", "0", "70000"): + with patch.dict(os.environ, {}, clear=False): + clear_stratum_diff_env() + os.environ["PRISM_STRATUM_HIGHDIFF_PORT"] = bad_port + with self.assertRaises(SystemExit): + load_prism_highdiff_listener("0.0.0.0", base) + + def test_client_startup_difficulty_uses_listener_profile(self) -> None: + server = coordinator() + profile = StratumListenerProfile( + name="highdiff", + bind="0.0.0.0", + port=4334, + share_difficulty=Decimal("700000"), + vardiff_config=highdiff_vardiff_config(), + heartbeat_name="stratum_accept_highdiff", + ) + self.assertEqual(server.client_startup_difficulty(profile), Decimal("500000")) + # Without a profile the default listener behavior is unchanged. + self.assertEqual(server.client_startup_difficulty(), Decimal("0.000000001")) + # With vardiff disabled the listener's fixed share difficulty applies. + fixed_profile = StratumListenerProfile( + name="highdiff", + bind="0.0.0.0", + port=4334, + share_difficulty=Decimal("700000"), + vardiff_config=highdiff_vardiff_config(enabled=False), + heartbeat_name="stratum_accept_highdiff", + ) + self.assertEqual(server.client_startup_difficulty(fixed_profile), Decimal("700000")) + + def test_stratum_accept_heartbeat_names(self) -> None: + server = coordinator() + # Coordinators built without listener profiles (tests, legacy) keep the + # historical single heartbeat name. + self.assertEqual(server.stratum_accept_heartbeat_names(), ("stratum_accept",)) + server.listener_profiles = [ + StratumListenerProfile( + name="default", + bind="0.0.0.0", + port=3340, + share_difficulty=Decimal("1"), + vardiff_config=server.vardiff_config, + heartbeat_name="stratum_accept", + ), + StratumListenerProfile( + name="highdiff", + bind="0.0.0.0", + port=4334, + share_difficulty=Decimal("500000"), + vardiff_config=highdiff_vardiff_config(), + heartbeat_name="stratum_accept_highdiff", + ), + ] + self.assertEqual( + server.stratum_accept_heartbeat_names(), + ("stratum_accept", "stratum_accept_highdiff"), + ) + + def test_parse_stratum_password_options(self) -> None: + self.assertEqual(parse_stratum_password_options(""), (None, None)) + self.assertEqual(parse_stratum_password_options("x"), (None, None)) + self.assertEqual(parse_stratum_password_options("d=8192"), (Decimal("8192"), None)) + self.assertEqual( + parse_stratum_password_options("md=4096,d=8192"), + (Decimal("8192"), Decimal("4096")), + ) + self.assertEqual( + parse_stratum_password_options("D=500000, MD=500000"), + (Decimal("500000"), Decimal("500000")), + ) + self.assertEqual(parse_stratum_password_options("d=abc"), (None, None)) + self.assertEqual(parse_stratum_password_options("d=-5,md=0"), (None, None)) + self.assertEqual(parse_stratum_password_options("foo=1,md=2048"), (None, Decimal("2048"))) + + def test_password_d_below_highdiff_floor_is_clamped(self) -> None: + server = coordinator() + state = client() + state.listener_vardiff_config = highdiff_vardiff_config() + state.share_difficulty = Decimal("500000") + state.requested_difficulty = Decimal("1000") + + target = server.apply_client_difficulty_requests(state) + + self.assertEqual(target, Decimal("500000")) + assert state.vardiff_config is not None + self.assertEqual(state.vardiff_config.min_difficulty, Decimal("500000")) + self.assertEqual(state.vardiff_config.startup_difficulty, Decimal("500000")) + + def test_password_md_raises_personal_floor_and_retarget_respects_it(self) -> None: + server = coordinator() + state = client() + state.requested_min_difficulty = Decimal("256") + state.share_difficulty = Decimal("1") + + target = server.apply_client_difficulty_requests(state) + + self.assertEqual(target, Decimal("256")) + assert state.vardiff_config is not None + self.assertEqual(state.vardiff_config.min_difficulty, Decimal("256")) + self.assertEqual(state.vardiff_config.max_difficulty, Decimal("1024")) + + # A zero-share retarget window wants to step down 4x; the personal + # floor must hold it at 256. + state.share_difficulty = Decimal("256") + server.retarget_client( + state, + current_difficulty=Decimal("256"), + accepted_shares=0, + submitted_shares=0, + accepted_difficulty=Decimal("0"), + elapsed_seconds=Decimal("2"), + ) + self.assertIsNone(state.pending_share_difficulty) + self.assertEqual(state.share_difficulty, Decimal("256")) + + def test_apply_requests_is_stable_across_reapplication(self) -> None: + server = coordinator() + state = client() + state.listener_vardiff_config = highdiff_vardiff_config() + state.requested_min_difficulty = Decimal("600000") + state.requested_difficulty = Decimal("700000") + + first = server.apply_client_difficulty_requests(state) + second = server.apply_client_difficulty_requests(state) + + self.assertEqual(first, Decimal("700000")) + self.assertEqual(second, Decimal("700000")) + assert state.vardiff_config is not None + # Recomputed from the pristine listener config: the floor is the md= + # value, not a compounded one. + self.assertEqual(state.vardiff_config.min_difficulty, Decimal("600000")) + + def test_suggest_difficulty_before_subscribe_applies_directly(self) -> None: + server = coordinator() + state = ClientState(sock=object(), address=("127.0.0.1", 1), connection_id=2, extranonce1_hex="00000002") + sent: list[object] = [] + state.send = lambda payload: sent.append(payload) # type: ignore[method-assign] + + server.handle_suggest_difficulty(state, 7, [512]) + + self.assertEqual(state.suggested_difficulty, Decimal("512")) + self.assertEqual(state.share_difficulty, Decimal("512")) + self.assertIsNone(state.pending_share_difficulty) + self.assertEqual(sent, [{"id": 7, "result": True, "error": None}]) + + def test_suggest_difficulty_post_authorize_advertises_with_job(self) -> None: + server = coordinator() + state = client() + state.share_difficulty = Decimal("1") + sent: list[object] = [] + state.send = lambda payload: sent.append(payload) # type: ignore[method-assign] + jobs: dict[str, object] = {"count": 0} + + def fake_send_job(client: object, clean_jobs: bool) -> bool: + jobs.update({"count": jobs["count"] + 1, "clean": clean_jobs}) + return True + + server.maybe_send_job = fake_send_job # type: ignore[method-assign] + + server.handle_suggest_difficulty(state, 8, [512]) + + self.assertEqual(state.pending_share_difficulty, Decimal("512")) + self.assertEqual(jobs["count"], 1) + self.assertTrue(jobs["clean"]) + self.assertEqual(sent, [{"id": 8, "result": True, "error": None}]) + + def test_suggest_difficulty_rolls_back_pending_on_build_failure(self) -> None: + server = coordinator() + state = client() + state.share_difficulty = Decimal("1") + state.difficulty_generation = 7 + state.send = lambda payload: None # type: ignore[method-assign] + server.maybe_send_job = lambda client, *, clean_jobs: False # type: ignore[method-assign] + + server.handle_suggest_difficulty(state, 9, [512]) + + self.assertIsNone(state.pending_share_difficulty) + self.assertEqual(state.share_difficulty, Decimal("1")) + self.assertEqual(state.difficulty_generation, 7) + + def test_suggest_difficulty_yields_to_password_d_option(self) -> None: + server = coordinator() + state = client() + state.requested_difficulty = Decimal("512") + state.share_difficulty = Decimal("512") + state.send = lambda payload: None # type: ignore[method-assign] + server.maybe_send_job = lambda client, *, clean_jobs: True # type: ignore[method-assign] + + server.handle_suggest_difficulty(state, 10, [128]) + + # d= wins: the suggestion is recorded but the resolved target stays at + # the explicit password difficulty, so nothing is re-advertised. + self.assertEqual(state.suggested_difficulty, Decimal("128")) + self.assertEqual(state.share_difficulty, Decimal("512")) + self.assertIsNone(state.pending_share_difficulty) + + def test_suggest_difficulty_ignores_junk_values(self) -> None: + server = coordinator() + state = client() + state.share_difficulty = Decimal("1") + sent: list[object] = [] + state.send = lambda payload: sent.append(payload) # type: ignore[method-assign] + + for junk in ([], ["nan"], ["-4"], ["0"], [None]): + server.handle_suggest_difficulty(state, 11, junk) # type: ignore[arg-type] + + self.assertIsNone(state.vardiff_config) + self.assertEqual(state.share_difficulty, Decimal("1")) + self.assertEqual(len(sent), 5) + + def test_highdiff_share_diff_tracks_start_and_validates_bounds(self) -> None: + base = load_prism_vardiff_config(Decimal("0.000000001")) + with patch.dict(os.environ, {}, clear=False): + clear_stratum_diff_env() + os.environ["PRISM_STRATUM_HIGHDIFF_PORT"] = "4334" + os.environ["PRISM_STRATUM_HIGHDIFF_MIN_DIFF"] = "1000000" + os.environ["PRISM_STRATUM_HIGHDIFF_START_DIFF"] = "1000000" + profile = load_prism_highdiff_listener("0.0.0.0", base) + assert profile is not None + # Unset fixed difficulty tracks the start difficulty instead of a + # constant that could fall below a raised floor. + self.assertEqual(profile.share_difficulty, Decimal("1000000")) + + # An explicit fixed difficulty outside the listener bounds must fail + # startup: advertising below the floor is exactly what the high-diff + # listener exists to prevent. + with patch.dict(os.environ, {}, clear=False): + clear_stratum_diff_env() + os.environ["PRISM_STRATUM_HIGHDIFF_PORT"] = "4334" + os.environ["PRISM_STRATUM_HIGHDIFF_SHARE_DIFF"] = "1000" + with self.assertRaises(SystemExit): + load_prism_highdiff_listener("0.0.0.0", base) + with patch.dict(os.environ, {}, clear=False): + clear_stratum_diff_env() + os.environ["PRISM_STRATUM_HIGHDIFF_PORT"] = "4334" + os.environ["PRISM_STRATUM_HIGHDIFF_SHARE_DIFF"] = "8589934592" + with self.assertRaises(SystemExit): + load_prism_highdiff_listener("0.0.0.0", base) + + def authorize_server_and_client(self) -> tuple[PrismCoordinator, ClientState, list[object]]: + server = coordinator() + server.rpc = AddressValidationRpc() + server.username_fallback_address = None + server.maybe_send_job = lambda client, *, clean_jobs: True # type: ignore[method-assign] + state = ClientState(sock=object(), address=("127.0.0.1", 1), connection_id=3, extranonce1_hex="00000003") + state.subscribed = True + sent: list[object] = [] + state.send = lambda payload: sent.append(payload) # type: ignore[method-assign] + return server, state, sent + + def test_authorize_password_applies_before_first_job(self) -> None: + server, state, sent = self.authorize_server_and_client() + + server.handle_request( + state, + {"id": 5, "method": "mining.authorize", "params": [PAYOUT_ADDRESS, "d=0.5,md=0.25"]}, + ) + + self.assertTrue(state.authorized) + self.assertEqual(state.requested_difficulty, Decimal("0.5")) + self.assertEqual(state.requested_min_difficulty, Decimal("0.25")) + # Applied directly (no job exists yet), so the first + # set_difficulty/notify pair advertises the requested value. + self.assertEqual(state.share_difficulty, Decimal("0.5")) + self.assertIsNone(state.pending_share_difficulty) + assert state.vardiff_config is not None + self.assertEqual(state.vardiff_config.min_difficulty, Decimal("0.25")) + self.assertEqual(sent, [{"id": 5, "result": True, "error": None}]) + + def test_reauthorize_with_plain_password_clears_stale_overrides(self) -> None: + server, state, _ = self.authorize_server_and_client() + server.handle_request( + state, + {"id": 5, "method": "mining.authorize", "params": [PAYOUT_ADDRESS, "d=0.5,md=0.25"]}, + ) + assert state.vardiff_config is not None + + server.handle_request( + state, + {"id": 6, "method": "mining.authorize", "params": [PAYOUT_ADDRESS, "x"]}, + ) + + # The new password carries no options: prior overrides are dropped and + # the client falls back to the pristine listener policy (its current + # difficulty is left alone; vardiff drifts it under listener bounds). + self.assertIsNone(state.requested_difficulty) + self.assertIsNone(state.requested_min_difficulty) + self.assertIsNone(state.vardiff_config) + self.assertEqual(state.share_difficulty, Decimal("0.5")) + + def test_reauthorize_with_new_difficulty_sends_single_job_pair(self) -> None: + server, state, _ = self.authorize_server_and_client() + send_job_calls: list[bool] = [] + + def counting_send_job(current: ClientState, *, clean_jobs: bool) -> bool: + send_job_calls.append(clean_jobs) + current.active_job = SimpleNamespace( + template={"previousblockhash": "aa" * 32}, + payout_state_generation=0, + ) + server.current_tip_first_seen = ("aa" * 32, None) + return True + + server.maybe_send_job = counting_send_job # type: ignore[method-assign] + + server.handle_request( + state, + {"id": 5, "method": "mining.authorize", "params": [PAYOUT_ADDRESS, "x"]}, + ) + self.assertEqual(len(send_job_calls), 1) + + # A re-authorize whose new d= advertises a fresh difficulty/job pair + # must not be followed by a second back-to-back pair. + server.handle_request( + state, + {"id": 6, "method": "mining.authorize", "params": [PAYOUT_ADDRESS, "d=0.5"]}, + ) + self.assertEqual(len(send_job_calls), 2) + self.assertEqual(state.pending_share_difficulty, Decimal("0.5")) + + def test_authorize_rejects_and_disconnects_above_username_connection_limit(self) -> None: + server, first, _ = self.authorize_server_and_client() + server.stratum_max_connections_per_username = 1 + second = ClientState( + sock=object(), + address=("127.0.0.1", 2), + connection_id=4, + extranonce1_hex="00000004", + ) + second.send = lambda payload: None # type: ignore[method-assign] + server.clients.update({first, second}) + + server.handle_request( + first, + {"id": 5, "method": "mining.authorize", "params": [PAYOUT_ADDRESS, "x"]}, + ) + with self.assertRaises(StratumError) as raised: + server.handle_request( + second, + {"id": 6, "method": "mining.authorize", "params": [PAYOUT_ADDRESS, "x"]}, + ) + + self.assertTrue(raised.exception.disconnect) + self.assertEqual(raised.exception.message, "too many connections for username") + self.assertEqual(server.connection_limit_rejection_counts["username"], 1) + self.assertFalse(second.authorized) + + def test_reauthorize_limit_error_preserves_live_session(self) -> None: + server, live, _ = self.authorize_server_and_client() + server.stratum_max_connections_per_username = 1 + occupant = ClientState( + sock=object(), + address=("127.0.0.1", 2), + connection_id=4, + extranonce1_hex="00000004", + ) + occupant.send = lambda payload: None # type: ignore[method-assign] + server.clients.update({live, occupant}) + + server.handle_request( + live, + { + "id": 5, + "method": "mining.authorize", + "params": [f"{PAYOUT_ADDRESS}.original", "x"], + }, + ) + server.handle_request( + occupant, + { + "id": 6, + "method": "mining.authorize", + "params": [f"{PAYOUT_ADDRESS}.full", "x"], + }, + ) + original_worker = live.worker + + with self.assertRaises(StratumError) as raised: + server.handle_request( + live, + { + "id": 7, + "method": "mining.authorize", + "params": [f"{PAYOUT_ADDRESS}.full", "x"], + }, + ) + + self.assertFalse(raised.exception.disconnect) + self.assertTrue(live.authorized) + self.assertIs(live.worker, original_worker) + self.assertEqual(live.username, f"{PAYOUT_ADDRESS}.original") + + def test_username_connection_limit_is_disabled_by_default(self) -> None: + server = coordinator() + first = client() + second = client() + server.clients.update({first, second}) + worker = WorkerIdentity( + username=PAYOUT_ADDRESS, + payout_address=PAYOUT_ADDRESS, + worker_name=None, + script_pubkey_hex="5220" + "11" * 32, + p2mr_program_hex="11" * 32, + ) + + self.assertTrue(server.reserve_client_username(first, worker)) + self.assertTrue(server.reserve_client_username(second, worker)) + self.assertEqual(server.connection_limit_rejection_counts["username"], 0) + + def test_username_limit_does_not_count_idle_clients_for_empty_username(self) -> None: + server = coordinator() + server.stratum_max_connections_per_username = 1 + idle = client() + first = client() + second = client() + server.clients.update({idle, first, second}) + worker = WorkerIdentity( + username="", + payout_address=PAYOUT_ADDRESS, + worker_name=None, + script_pubkey_hex="5220" + "11" * 32, + p2mr_program_hex="11" * 32, + ) + + self.assertTrue(server.reserve_client_username(first, worker)) + self.assertFalse(server.reserve_client_username(second, worker)) + self.assertEqual(server.connection_limit_rejection_counts["username"], 1) + + def test_accept_loop_rejects_above_global_connection_limit(self) -> None: + server = coordinator() + server.stratum_max_connections = 1 + server.clients.add(client()) + + class AcceptedSocket: + def __init__(self) -> None: + self.closed = False + + def close(self) -> None: + self.closed = True + + accepted = AcceptedSocket() + + class OneConnectionListener: + def __init__(self) -> None: + self.calls = 0 + + def accept(self) -> tuple[object, tuple[str, int]]: + self.calls += 1 + if self.calls == 1: + return accepted, ("127.0.0.1", 1000) + server.stop_event.set() + raise socket.timeout() + + profile = StratumListenerProfile( + name="default", + bind="127.0.0.1", + port=3340, + share_difficulty=server.share_difficulty, + vardiff_config=server.vardiff_config, + heartbeat_name="stratum_accept", + ) + + server.accept_loop(OneConnectionListener(), profile) # type: ignore[arg-type] + + self.assertTrue(accepted.closed) + self.assertEqual(server.connection_limit_rejection_counts["global"], 1) + + def test_accept_loop_recovers_from_descriptor_exhaustion(self) -> None: + server = coordinator() + server.stratum_accept_resource_exhaustion_backoff_seconds = 0 + + class ExhaustedListener: + def __init__(self) -> None: + self.calls = 0 + + def accept(self) -> tuple[object, tuple[str, int]]: + self.calls += 1 + if self.calls == 1: + raise OSError(errno.EMFILE, "too many open files") + server.stop_event.set() + raise socket.timeout() + + listener = ExhaustedListener() + profile = StratumListenerProfile( + name="default", + bind="127.0.0.1", + port=3340, + share_difficulty=server.share_difficulty, + vardiff_config=server.vardiff_config, + heartbeat_name="stratum_accept", + ) + + server.accept_loop(listener, profile) # type: ignore[arg-type] + + self.assertEqual(listener.calls, 2) + self.assertEqual(server.accept_resource_exhaustion_count, 1) + + def test_resource_backoff_keeps_accept_watchdog_heartbeat_fresh(self) -> None: + server = coordinator() + server.stratum_accept_resource_exhaustion_backoff_seconds = 0.03 + server.watchdog_timeout_seconds = 0.01 + + server._wait_after_stratum_resource_failure("stratum_accept") + + self.assertFalse(server._overdue_heartbeats(time.monotonic())) + + def test_accept_loop_recovers_when_handler_thread_cannot_start(self) -> None: + server = coordinator() + server.connection_counter = 0 + server.stratum_send_timeout_seconds = 0 + server.stratum_accept_resource_exhaustion_backoff_seconds = 0 + + class AcceptedSocket: + def __init__(self) -> None: + self.closed = False + + def settimeout(self, timeout: object) -> None: + pass + + def shutdown(self, how: int) -> None: + pass + + def close(self) -> None: + self.closed = True + + accepted = AcceptedSocket() + + class OneConnectionListener: + def __init__(self) -> None: + self.calls = 0 + + def accept(self) -> tuple[object, tuple[str, int]]: + self.calls += 1 + if self.calls == 1: + return accepted, ("127.0.0.1", 1000) + server.stop_event.set() + raise socket.timeout() + + listener = OneConnectionListener() + profile = StratumListenerProfile( + name="default", + bind="127.0.0.1", + port=3340, + share_difficulty=server.share_difficulty, + vardiff_config=server.vardiff_config, + heartbeat_name="stratum_accept", + ) + + with patch.object(threading.Thread, "start", side_effect=RuntimeError("can't start new thread")): + server.accept_loop(listener, profile) # type: ignore[arg-type] + + self.assertEqual(listener.calls, 2) + self.assertTrue(accepted.closed) + self.assertFalse(server.clients) + self.assertEqual(server.connection_setup_failure_count, 1) + + def test_handle_client_cleans_up_when_makefile_hits_descriptor_limit(self) -> None: + server = coordinator() + + class MakefileFailureSocket: + def __init__(self) -> None: + self.closed = False + + def makefile(self, *args: object, **kwargs: object) -> object: + raise OSError(errno.EMFILE, "too many open files") + + def shutdown(self, how: int) -> None: + pass + + def close(self) -> None: + self.closed = True + + sock = MakefileFailureSocket() + state = ClientState( + sock=sock, # type: ignore[arg-type] + address=("127.0.0.1", 1000), + connection_id=1, + extranonce1_hex="00000001", + ) + server.clients.add(state) + + server.handle_client(state) + + self.assertTrue(sock.closed) + self.assertNotIn(state, server.clients) + self.assertEqual(server.accept_resource_exhaustion_count, 1) + + def test_accept_loop_assigns_listener_profiles_and_unique_extranonce(self) -> None: + server = coordinator() + server.connection_counter = 0 + server.stratum_send_timeout_seconds = 0.0 + + def listening_socket() -> socket.socket: + listener = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + listener.bind(("127.0.0.1", 0)) + listener.listen() + listener.settimeout(0.1) + return listener + + default_listener = listening_socket() + highdiff_listener = listening_socket() + default_profile = StratumListenerProfile( + name="default", + bind="127.0.0.1", + port=default_listener.getsockname()[1], + share_difficulty=server.share_difficulty, + vardiff_config=server.vardiff_config, + heartbeat_name="stratum_accept", + ) + highdiff_profile = StratumListenerProfile( + name="highdiff", + bind="127.0.0.1", + port=highdiff_listener.getsockname()[1], + share_difficulty=Decimal("500000"), + vardiff_config=highdiff_vardiff_config(), + heartbeat_name="stratum_accept_highdiff", + minimum_advertised_difficulty=Decimal("500000"), + ) + threads = [ + threading.Thread(target=server.accept_loop, args=(default_listener, default_profile), daemon=True), + threading.Thread(target=server.accept_loop, args=(highdiff_listener, highdiff_profile), daemon=True), + ] + for thread in threads: + thread.start() + connections = [] + try: + connections.append(socket.create_connection(("127.0.0.1", default_profile.port), timeout=5)) + connections.append(socket.create_connection(("127.0.0.1", highdiff_profile.port), timeout=5)) + deadline = time.monotonic() + 5 + while time.monotonic() < deadline: + with server.lock: + if len(server.clients) == 2: + break + time.sleep(0.01) + with server.lock: + clients_by_listener = {c.listener_name: c for c in server.clients} + self.assertEqual(set(clients_by_listener), {"default", "highdiff"}) + self.assertEqual( + clients_by_listener["default"].share_difficulty, + Decimal("0.000000001"), + ) + self.assertEqual( + clients_by_listener["highdiff"].share_difficulty, + Decimal("500000"), + ) + self.assertIs( + clients_by_listener["highdiff"].listener_vardiff_config, + highdiff_profile.vardiff_config, + ) + self.assertEqual( + clients_by_listener["highdiff"].minimum_advertised_difficulty, + Decimal("500000"), + ) + self.assertEqual( + clients_by_listener["default"].minimum_advertised_difficulty, + Decimal("0"), + ) + extranonces = {c.extranonce1_hex for c in clients_by_listener.values()} + self.assertEqual(extranonces, {"00000001", "00000002"}) + self.assertIn("stratum_accept", server._heartbeats) + self.assertIn("stratum_accept_highdiff", server._heartbeats) + finally: + server.stop_event.set() + for connection in connections: + connection.close() + default_listener.close() + highdiff_listener.close() + for thread in threads: + thread.join(timeout=5) + + +class LedgerSessionGuardConfigTests(unittest.TestCase): + # (env var, LedgerConfig field, default) for the postgres session guards + # and the bounded startup lease acquisition, all six added for issue #123. + SESSION_GUARD_SETTINGS = ( + ( + "PRISM_POSTGRES_IDLE_IN_TRANSACTION_TIMEOUT_SECONDS", + "postgres_idle_in_transaction_timeout_seconds", + 15.0, + ), + ( + "PRISM_POSTGRES_TCP_KEEPALIVES_IDLE_SECONDS", + "postgres_tcp_keepalives_idle_seconds", + 30, + ), + ( + "PRISM_POSTGRES_TCP_KEEPALIVES_INTERVAL_SECONDS", + "postgres_tcp_keepalives_interval_seconds", + 10, + ), + ( + "PRISM_POSTGRES_TCP_KEEPALIVES_COUNT", + "postgres_tcp_keepalives_count", + 3, + ), + ( + "PRISM_LEDGER_LEASE_ACQUIRE_LOCK_TIMEOUT_SECONDS", + "lease_acquire_lock_timeout_seconds", + 5.0, + ), + ( + "PRISM_LEDGER_LEASE_ACQUIRE_ATTEMPTS", + "lease_acquire_attempts", + 5, + ), + ) + + @staticmethod + def minimal_environment(root: Path) -> dict[str, str]: + return { + "QBIT_RPC_HOST": "qbit.example", + "QBIT_RPC_USER": "rpc-user", + "QBIT_RPC_PASSWORD": "rpc-password", + "PRISM_ALLOW_MEMORY_LEDGER": "1", + "PRISM_ALLOW_TEST_SIGNING_SEEDS": "1", + "PRISM_ALLOW_BUNDLE_EMBEDDED_LEDGER_KEY": "1", + "PRISM_AUDIT_DIR": str(root), + "PRISM_EVIDENCE_PATH": str(root / "evidence.json"), + } + + def test_session_guard_defaults_land_on_ledger_config(self) -> None: + with tempfile.TemporaryDirectory() as temp_dir: + config = load_coordinator_config( + self.minimal_environment(Path(temp_dir)) + ) + + for _env_name, field_name, default in self.SESSION_GUARD_SETTINGS: + with self.subTest(field=field_name): + self.assertEqual(getattr(config.ledger, field_name), default) + + def test_replay_page_size_defaults_and_overrides(self) -> None: + with tempfile.TemporaryDirectory() as temp_dir: + source = self.minimal_environment(Path(temp_dir)) + self.assertEqual(load_coordinator_config(source).block.replay_page_size, 1024) + for size in (1, 2, 32, 1024): + with self.subTest(size=size): + config = load_coordinator_config( + {**source, "PRISM_BLOCK_REPLAY_PAGE_SIZE": str(size)} + ) + self.assertEqual(config.block.replay_page_size, size) + + def test_invalid_replay_page_size_refuses_startup(self) -> None: + with tempfile.TemporaryDirectory() as temp_dir: + source = self.minimal_environment(Path(temp_dir)) + for raw in ("", "0", "-1", "1025", "1.5", "invalid"): + with self.subTest(raw=raw), self.assertRaisesRegex( + SystemExit, "PRISM_BLOCK_REPLAY_PAGE_SIZE" + ): + load_coordinator_config({**source, "PRISM_BLOCK_REPLAY_PAGE_SIZE": raw}) + + def test_session_guard_overrides_land_on_ledger_config(self) -> None: + overrides = { + "PRISM_POSTGRES_IDLE_IN_TRANSACTION_TIMEOUT_SECONDS": "7.5", + "PRISM_POSTGRES_TCP_KEEPALIVES_IDLE_SECONDS": "17", + "PRISM_POSTGRES_TCP_KEEPALIVES_INTERVAL_SECONDS": "6", + "PRISM_POSTGRES_TCP_KEEPALIVES_COUNT": "2", + "PRISM_LEDGER_LEASE_ACQUIRE_LOCK_TIMEOUT_SECONDS": "3.5", + "PRISM_LEDGER_LEASE_ACQUIRE_ATTEMPTS": "9", + } + with tempfile.TemporaryDirectory() as temp_dir: + config = load_coordinator_config( + {**self.minimal_environment(Path(temp_dir)), **overrides} + ) + + ledger = config.ledger + self.assertEqual(ledger.postgres_idle_in_transaction_timeout_seconds, 7.5) + self.assertEqual(ledger.postgres_tcp_keepalives_idle_seconds, 17) + self.assertEqual(ledger.postgres_tcp_keepalives_interval_seconds, 6) + self.assertEqual(ledger.postgres_tcp_keepalives_count, 2) + self.assertEqual(ledger.lease_acquire_lock_timeout_seconds, 3.5) + self.assertEqual(ledger.lease_acquire_attempts, 9) + + def test_session_guard_settings_fail_closed_on_invalid_values(self) -> None: + # Empty, non-numeric, zero, and negative values must all abort the + # load: a disarmed or nonsensical guard silently recreates the + # unbounded startup blocking these settings exist to prevent. + for env_name, _field_name, _default in self.SESSION_GUARD_SETTINGS: + for raw in ("", "not-a-number", "0", "-1"): + with self.subTest(name=env_name, raw=raw): + with tempfile.TemporaryDirectory() as temp_dir: + source = { + **self.minimal_environment(Path(temp_dir)), + env_name: raw, + } + with self.assertRaises(SystemExit): + load_coordinator_config(source) diff --git a/tests/test_prism_coordinator_config_loading.py b/tests/test_prism_coordinator_config_loading.py new file mode 100644 index 000000000..f6acb97b8 --- /dev/null +++ b/tests/test_prism_coordinator_config_loading.py @@ -0,0 +1,352 @@ +#!/usr/bin/env python3 +"""Direct tests for the extracted coordinator configuration and RPC seams.""" + +from __future__ import annotations + +import contextlib +import dataclasses +import io +import subprocess +import sys +import tempfile +import time +import unittest +from dataclasses import replace +from pathlib import Path +from types import SimpleNamespace +from unittest.mock import patch + +from lab.prism.coordinator_config import ( + CoordinatorConfig, + LifecycleConfig, + load_coordinator_config, +) +from lab.prism.prism_coordinator import JsonRpc as CompatibilityJsonRpc +from lab.prism.prism_coordinator import PrismCoordinator +from lab.prism.rpc import JsonRpc +from lab.prism.run_ctv_broadcaster_daemon import JsonRpc as DaemonJsonRpc + + +def minimal_environment(root: Path) -> dict[str, str]: + return { + "QBIT_RPC_HOST": "qbit.example", + "QBIT_RPC_USER": "rpc-user", + "QBIT_RPC_PASSWORD": "rpc-password", + "PRISM_ALLOW_MEMORY_LEDGER": "1", + "PRISM_ALLOW_TEST_SIGNING_SEEDS": "1", + "PRISM_ALLOW_BUNDLE_EMBEDDED_LEDGER_KEY": "1", + "PRISM_AUDIT_DIR": str(root), + "PRISM_EVIDENCE_PATH": str(root / "evidence.json"), + } + + +class CoordinatorConfigLoadingTests(unittest.TestCase): + def construct(self, source: dict[str, str]) -> PrismCoordinator: + config = load_coordinator_config(source) + with patch.object(JsonRpc, "call", side_effect=RuntimeError("offline")): + return PrismCoordinator(config) + + def test_loader_accepts_mapping_without_reading_process_environment(self) -> None: + with tempfile.TemporaryDirectory() as temp_dir: + source = minimal_environment(Path(temp_dir)) + with patch.dict("os.environ", {}, clear=True): + config = load_coordinator_config(source) + + self.assertIsInstance(config, CoordinatorConfig) + self.assertEqual(config.rpc.host, "qbit.example") + self.assertEqual(config.rpc.port, 18452) + self.assertEqual(config.stratum.bind, "127.0.0.1") + self.assertEqual(config.lifecycle.pending_refresh_health_deadline_seconds, 15.0) + self.assertEqual(config.lifecycle.coherent_tip_poll_health_deadline_seconds, 15.0) + + def test_loader_captures_pr54_health_deadlines_in_frozen_lifecycle_config(self) -> None: + with tempfile.TemporaryDirectory() as temp_dir: + source = { + **minimal_environment(Path(temp_dir)), + "PRISM_HEALTH_REFRESH_SECONDS": "7", + "PRISM_METRICS_REFRESH_SECONDS": "11", + "PRISM_HEALTH_PENDING_REFRESH_MAX_AGE_SECONDS": "23", + "PRISM_HEALTH_TIP_POLL_MAX_AGE_SECONDS": "29", + "PRISM_MINING_HEALTH_STARTUP_GRACE_SECONDS": "31", + } + config = load_coordinator_config(source) + + self.assertEqual( + config.lifecycle, + replace( + config.lifecycle, + health_refresh_seconds=7.0, + metrics_refresh_seconds=11.0, + pending_refresh_health_deadline_seconds=23.0, + coherent_tip_poll_health_deadline_seconds=29.0, + mining_health_startup_grace_seconds=31.0, + ), + ) + self.assertIsInstance(config.lifecycle, LifecycleConfig) + with self.assertRaises(dataclasses.FrozenInstanceError): + config.lifecycle.health_refresh_seconds = 1.0 # type: ignore[misc] + + def test_loader_owns_public_reward_window_cache_interval(self) -> None: + with tempfile.TemporaryDirectory() as temp_dir: + source = { + **minimal_environment(Path(temp_dir)), + "PRISM_PUBLIC_REWARD_WINDOW_CACHE_SECONDS": "47", + } + config = load_coordinator_config(source) + + self.assertEqual(config.ledger.reward_window_cache_seconds, 47.0) + + def test_coordinator_uses_supplied_snapshot_without_reloading_environment(self) -> None: + with tempfile.TemporaryDirectory() as temp_dir: + config = load_coordinator_config(minimal_environment(Path(temp_dir))) + with ( + patch( + "lab.prism.prism_coordinator.load_coordinator_config", + side_effect=AssertionError("unexpected environment reload"), + ), + patch.object(JsonRpc, "call", side_effect=RuntimeError("offline")), + ): + coordinator = PrismCoordinator(config) + + self.assertIs(coordinator.config, config) + self.assertEqual(coordinator.bind, config.stratum.bind) + self.assertEqual( + coordinator.health_pending_refresh_max_age_seconds, + config.lifecycle.pending_refresh_health_deadline_seconds, + ) + self.assertEqual( + coordinator.metrics_refresh_seconds, + config.lifecycle.metrics_refresh_seconds, + ) + + def test_metrics_refresh_interval_must_be_positive(self) -> None: + with tempfile.TemporaryDirectory() as temp_dir: + source = { + **minimal_environment(Path(temp_dir)), + "PRISM_METRICS_REFRESH_SECONDS": "0", + } + with self.assertRaisesRegex( + SystemExit, + "PRISM_METRICS_REFRESH_SECONDS must be positive", + ): + load_coordinator_config(source) + + def test_watchdog_timeout_floor_rejects_sub_second_tolerances(self) -> None: + with tempfile.TemporaryDirectory() as temp_dir: + base = minimal_environment(Path(temp_dir)) + for raw in ("0.0005", "0.5", "0.999"): + with self.subTest(raw=raw): + with self.assertRaisesRegex( + SystemExit, + "PRISM_WATCHDOG_TIMEOUT_SECONDS must be at least 1", + ): + load_coordinator_config( + {**base, "PRISM_WATCHDOG_TIMEOUT_SECONDS": raw} + ) + for raw, expected in (("1.0", 1.0), ("120", 120.0)): + with self.subTest(raw=raw): + config = load_coordinator_config( + {**base, "PRISM_WATCHDOG_TIMEOUT_SECONDS": raw} + ) + self.assertEqual( + config.lifecycle.watchdog_timeout_seconds, expected + ) + + def test_idle_in_transaction_reap_outside_lease_budget_warns(self) -> None: + with tempfile.TemporaryDirectory() as temp_dir: + base = minimal_environment(Path(temp_dir)) + inverted = { + **base, + "PRISM_LEDGER_LEASE_ACQUIRE_LOCK_TIMEOUT_SECONDS": "2", + "PRISM_LEDGER_LEASE_ACQUIRE_ATTEMPTS": "3", + "PRISM_POSTGRES_IDLE_IN_TRANSACTION_TIMEOUT_SECONDS": "10", + } + stderr = io.StringIO() + with contextlib.redirect_stderr(stderr): + load_coordinator_config(inverted) + warning = stderr.getvalue() + self.assertIn("prism coordinator: ", warning) + self.assertIn( + "PRISM_POSTGRES_IDLE_IN_TRANSACTION_TIMEOUT_SECONDS (10s)", + warning, + ) + self.assertIn( + "PRISM_LEDGER_LEASE_ACQUIRE_LOCK_TIMEOUT_SECONDS (2s)", + warning, + ) + self.assertIn("PRISM_LEDGER_LEASE_ACQUIRE_ATTEMPTS (3)", warning) + self.assertIn("lower the idle-in-transaction timeout", warning) + + stderr = io.StringIO() + with contextlib.redirect_stderr(stderr): + load_coordinator_config(base) + self.assertEqual(stderr.getvalue(), "") + + def test_zero_argument_coordinator_still_loads_environment(self) -> None: + with tempfile.TemporaryDirectory() as temp_dir: + source = minimal_environment(Path(temp_dir)) + with ( + patch.dict("os.environ", source, clear=True), + patch.object(JsonRpc, "call", side_effect=RuntimeError("offline")), + ): + coordinator = PrismCoordinator() + + self.assertEqual(coordinator.rpc.host, "qbit.example") + self.assertEqual(coordinator.config.rpc.user, "rpc-user") + + def test_dormant_live_chain_validation_is_deferred_until_use(self) -> None: + with tempfile.TemporaryDirectory() as temp_dir: + source = { + **minimal_environment(Path(temp_dir)), + "PRISM_MIN_PEERS": "0", + "PRISM_TEMPLATE_MAX_AGE_SECONDS": "not-an-int", + } + coordinator = self.construct(source) + + class PublicChainRpc: + def call(self, method: str, params: list[object] | None = None) -> object: + if method == "getblockchaininfo": + return { + "chain": "testnet4", + "initialblockdownload": False, + "blocks": 10, + "headers": 10, + } + if method == "getnetworkinfo": + return {"connections": 1} + raise RuntimeError(method) + + coordinator.qbit_chain = "testnet4" + coordinator.rpc = PublicChainRpc() # type: ignore[assignment] + # The extracted immutable snapshot defers live-boundary validation; + # the consuming methods read the live process environment at call + # time in the current tree, so the invalid values are staged there. + deferred = { + "PRISM_MIN_PEERS": "0", + "PRISM_TEMPLATE_MAX_AGE_SECONDS": "not-an-int", + } + with patch.dict("os.environ", deferred): + with self.assertRaisesRegex(SystemExit, "PRISM_MIN_PEERS must be positive"): + coordinator.validate_live_chain_identity() + + coordinator.current_template_artifacts = lambda: SimpleNamespace( # type: ignore[method-assign] + template={"curtime": int(time.time())}, + previousblockhash="11" * 32, + ) + with self.assertRaisesRegex( + SystemExit, "PRISM_TEMPLATE_MAX_AGE_SECONDS must be an integer" + ): + coordinator.validate_live_template_and_fee_policy() + + def test_disabled_ctv_settlement_ignores_invalid_settlement_only_values(self) -> None: + with tempfile.TemporaryDirectory() as temp_dir: + source = { + **minimal_environment(Path(temp_dir)), + "PRISM_CTV_SETTLEMENT_ENABLED": "0", + "PRISM_DIRECT_COINBASE_PAYOUT_FLOOR_BITS": "not-an-int", + "PRISM_RESERVED_COINBASE_OUTPUTS": "also-not-an-int", + } + coordinator = self.construct(source) + + with patch.dict( + "os.environ", + { + "PRISM_CTV_SETTLEMENT_ENABLED": "0", + "PRISM_DIRECT_COINBASE_PAYOUT_FLOOR_BITS": "not-an-int", + "PRISM_RESERVED_COINBASE_OUTPUTS": "also-not-an-int", + }, + ): + self.assertIsNone(coordinator.prism_ctv_settlement_config()) + + def test_enabled_ctv_settlement_validates_at_consuming_boundary(self) -> None: + with tempfile.TemporaryDirectory() as temp_dir: + source = { + **minimal_environment(Path(temp_dir)), + "PRISM_CTV_SETTLEMENT_ENABLED": "1", + "PRISM_DIRECT_COINBASE_PAYOUT_FLOOR_BITS": "not-an-int", + } + coordinator = self.construct(source) + + with patch.dict( + "os.environ", + { + "PRISM_CTV_SETTLEMENT_ENABLED": "1", + "PRISM_DIRECT_COINBASE_PAYOUT_FLOOR_BITS": "not-an-int", + }, + ): + with self.assertRaisesRegex( + SystemExit, "PRISM_DIRECT_COINBASE_PAYOUT_FLOOR_BITS must be an integer" + ): + coordinator.prism_ctv_settlement_config() + + def test_payout_validation_is_deferred_until_policy_is_consumed(self) -> None: + with tempfile.TemporaryDirectory() as temp_dir: + source = { + **minimal_environment(Path(temp_dir)), + "PRISM_PAYOUT_P2MR_SPEND_INPUT_BYTES": "not-an-int", + "PRISM_POOL_FEE_ENABLED": "", + } + coordinator = self.construct(source) + + with patch.dict( + "os.environ", + { + "PRISM_PAYOUT_P2MR_SPEND_INPUT_BYTES": "not-an-int", + "PRISM_POOL_FEE_ENABLED": "", + }, + ): + with self.assertRaisesRegex( + SystemExit, "PRISM_PAYOUT_P2MR_SPEND_INPUT_BYTES must be an integer" + ): + coordinator.prism_payout_policy() + + def test_explicit_disabled_broadcaster_defers_empty_settlement_flag(self) -> None: + with tempfile.TemporaryDirectory() as temp_dir: + source = { + **minimal_environment(Path(temp_dir)), + "PRISM_CTV_BROADCASTER_ENABLED": "0", + "PRISM_CTV_SETTLEMENT_ENABLED": "", + } + coordinator = self.construct(source) + + self.assertFalse(coordinator.ctv_broadcaster_enabled) + with patch.dict("os.environ", {"PRISM_CTV_SETTLEMENT_ENABLED": ""}): + with self.assertRaisesRegex( + SystemExit, "PRISM_CTV_SETTLEMENT_ENABLED is required" + ): + coordinator.prism_ctv_settlement_config() + + def test_broadcaster_default_still_consumes_settlement_flag_during_construction(self) -> None: + with tempfile.TemporaryDirectory() as temp_dir: + source = { + **minimal_environment(Path(temp_dir)), + "PRISM_CTV_SETTLEMENT_ENABLED": "", + } + with self.assertRaisesRegex( + SystemExit, "PRISM_CTV_SETTLEMENT_ENABLED is required" + ): + load_coordinator_config(source) + + def test_rpc_compatibility_reexport_points_to_leaf_class(self) -> None: + self.assertIs(CompatibilityJsonRpc, JsonRpc) + self.assertIs(DaemonJsonRpc, JsonRpc) + + def test_importing_rpc_leaf_does_not_import_coordinator(self) -> None: + result = subprocess.run( + [ + sys.executable, + "-c", + ( + "import sys; import lab.prism.rpc; " + "assert 'lab.prism.prism_coordinator' not in sys.modules" + ), + ], + check=False, + capture_output=True, + text=True, + ) + self.assertEqual(result.returncode, 0, result.stderr) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_prism_coordinator_job_cache.py b/tests/test_prism_coordinator_job_cache.py index 62ea033b0..56304ddf0 100644 --- a/tests/test_prism_coordinator_job_cache.py +++ b/tests/test_prism_coordinator_job_cache.py @@ -1,10489 +1,383 @@ #!/usr/bin/env python3 -"""Per-template job-build cache, cached health snapshot, and latency metrics.""" +"""Direct ownership tests for the extracted PRISM template and job services. -from __future__ import annotations - -import contextlib -import io -import os -import queue -import socket -import subprocess -import sys -import threading -import time -import unittest -from pathlib import Path -from concurrent.futures import Future -from contextlib import contextmanager -from dataclasses import dataclass, replace as dataclass_replace -from decimal import Decimal -from types import SimpleNamespace -from unittest.mock import patch - -from lab.auxpow import vardiff -from lab.prism import direct_stratum -from lab.prism import prism_coordinator as prism_coordinator_module -from lab.prism.prism_coordinator import ( - ClientState, - JobBuildAdmissionDeadlineExceeded, - JobBuildSuperseded, - MAX_PRISM_JOB_BUNDLE_CACHE_ENTRIES, - PRISM_JOB_EXTRANONCE1_PLACEHOLDER_HEX, - PRISM_REJECTION_REASON_IDS, - PendingShareAppend, - PayoutLedgerArtifact, - PrismCoordinator, - ShutdownInProgress, - TemplateRefreshBlocked, - WorkerIdentity, - _PayoutStatePublicationBlocked, - _compact_share_payload, - canonical_json_sha256, - canonical_json_text, - default_prism_coinbase_tag_hex, - now_ms, - qbit_template_fingerprint, -) -from lab.prism.share_ledger import ( - LedgerOperationTimeout, - PendingShare, - SingleWriterShareLedger, -) - -PAYOUT_ADDRESS = "tq1z70ukpvs96kye6jmgvl3nttevtkrq8uu89snkpm6m8gwqukw8u5dsz32kwa" -EXTRANONCE2_SIZE = 8 - - -@dataclass(frozen=True) -class FakeShare: - miner_id: str - share_seq: int - - def to_prism_json(self) -> dict[str, object]: - return {"share_seq": self.share_seq, "miner_id": self.miner_id} - - -class FakeLedger: - backend_name = "fake" - - def __init__(self, miners: list[str] | None = None) -> None: - self.miners = miners if miners is not None else ["miner-a", "miner-b", "miner-c"] - self.snapshot_calls = 0 - self.stats_calls = 0 - - def accepted_share_stats(self) -> dict[str, int]: - self.stats_calls += 1 - return { - "accepted_share_count": len(self.miners), - "distinct_miner_count": len(set(self.miners)), - } - - def all_shares(self) -> list[FakeShare]: - raise AssertionError("all_shares must not be called when accepted_share_stats exists") - - def snapshot_at_job_issue(self, anchor_job_issued_at_ms: int, *, window_weight: int | None = None) -> list[FakeShare]: - self.snapshot_calls += 1 - return [FakeShare(miner_id=miner, share_seq=seq + 1) for seq, miner in enumerate(self.miners)] - - def current_prior_balances(self) -> list[dict[str, object]]: - return [] - - def metrics(self) -> dict[str, int]: - return {"blocks": 0, "owed_accounts": 0} - - -class ReadyLedgerWithEmptyFirstSnapshot(FakeLedger): - def __init__(self) -> None: - super().__init__(miners=["miner-a", "miner-b", "miner-c"]) - - def snapshot_at_job_issue(self, anchor_job_issued_at_ms: int, *, window_weight: int | None = None) -> list[FakeShare]: - self.snapshot_calls += 1 - if self.snapshot_calls == 1: - return [] - return [FakeShare(miner_id=miner, share_seq=seq + 1) for seq, miner in enumerate(self.miners)] - - -class FakeRpc: - def __init__(self, template: dict[str, object], tip: str) -> None: - self.template = template - self.tip = tip - self.blockchain_info: dict[str, object] = { - "initialblockdownload": False, - "blocks": 100, - "headers": 100, - } - self.calls: list[str] = [] - - def call(self, method: str, params: list[object] | None = None) -> object: - self.calls.append(method) - if method == "getblocktemplate": - return dict(self.template) - if method == "getbestblockhash": - return self.tip - if method == "getblockchaininfo": - return dict(self.blockchain_info) - if method == "getblockcount": - return int(self.blockchain_info["blocks"]) - raise AssertionError(f"unexpected RPC {method}") - - def count(self, method: str) -> int: - return sum(1 for name in self.calls if name == method) - - -def synthetic_manifest_coinbase_hex(suffix_hex: str) -> str: - """A structurally valid non-witness coinbase whose scriptSig ends with the - extranonce placeholder suffix, as the audit bundle builder produces.""" - height_push = "03aabbcc" - script_sig = height_push + suffix_hex - script_sig_bytes = bytes.fromhex(script_sig) - output = (50_00000000).to_bytes(8, "little").hex() + "0151" - return ( - "01000000" - + "01" - + "00" * 32 - + "ffffffff" - + direct_stratum.compact_size(len(script_sig_bytes)).hex() - + script_sig - + "ffffffff" - + "01" - + output - + "00000000" - ) - - -def base_template(height: int = 10, prevhash: str = "11" * 32) -> dict[str, object]: - # Realistic (non-regtest) bits: the network target must be harder than the - # vardiff range for per-client share targets to differ, as on testnet4. - return { - "height": height, - "previousblockhash": prevhash, - "bits": "1b00ffff", - "version": 0x20000000, - "curtime": 1_700_000_000, - "coinbasevalue": 50_00000000, - "transactions": [], - } - - -def durable_candidate_row(index: int) -> dict[str, object]: - """A minimal valid outbox row whose intent survives block_candidate_from_intent.""" - block_hash = f"{index + 1:064x}" - return { - "block_hash": block_hash, - "candidate": { - "schema": "qbit.prism.block-candidate-intent.v1", - "block_hash_hex": block_hash, - "block_hex": "00", - "coinbase_tx_hex": "00", - "parent_hash": "11" * 32, - "expected_height": 10, - "template": { - "previousblockhash": "11" * 32, - "height": 10, - "coinbasevalue": 50_00000000, - }, - "shares_json": [], - "prior_balances": [], - "found_block": {}, - "prospective_prior_balances": None, - "witness_merkle_leaves_hex": [], - "extranonce1_hex": "00000001", - "extranonce2_hex": "00", - "username": "miner-a", - "pending_share": { - "share_id": f"miner-a:{block_hash}", - "miner_id": "miner-a", - "order_key": "miner-a", - "p2mr_program_hex": "22" * 32, - "share_difficulty": 1, - "network_difficulty": 1, - "template_height": 10, - "job_id": "job-1", - "job_issued_at_ms": 1, - "accepted_at_ms": 1, - "ntime": 1, - }, - "credit_share_on_accept": False, - "collection_only": False, - }, - } - - -def worker(payout: str = PAYOUT_ADDRESS, username: str | None = None) -> WorkerIdentity: - return WorkerIdentity( - username=username or payout, - payout_address=payout, - worker_name=None, - script_pubkey_hex="5220" + "22" * 32, - p2mr_program_hex="22" * 32, - ) - - -def client(connection_id: int, identity: WorkerIdentity | None = None) -> ClientState: - state = ClientState.__new__(ClientState) - state.sock = None - state.address = ("127.0.0.1", 40_000 + connection_id) - state.connection_id = connection_id - state.extranonce1_hex = f"{connection_id:08x}" - state.subscribed = True - state.authorized = True - identity = identity or worker() - state.username = identity.username - state.worker = identity - state.version_mask = 0 - state.active_job = None - state.share_difficulty = Decimal("1") - state.pending_share_difficulty = None - state.active_job_ids = set() - state.post_accept_refresh_block = None - state.tip_work_delivered = None - state.closing = False - state.job_update_lock = threading.RLock() - state.send_lock = threading.Lock() - return state - - -class ObservedRLock: - """RLock test double that exposes a contending acquire without sleeps.""" - - def __init__(self) -> None: - self._lock = threading.RLock() - self.observe_acquires = False - self.acquire_attempted = threading.Event() - - def acquire(self, blocking: bool = True, timeout: float = -1) -> bool: - if self.observe_acquires: - self.acquire_attempted.set() - return self._lock.acquire(blocking, timeout) - - def release(self) -> None: - self._lock.release() - - def __enter__(self) -> ObservedRLock: - self.acquire() - return self - - def __exit__(self, *_args: object) -> None: - self.release() - - -def coordinator(*, ledger: object | None = None, template: dict[str, object] | None = None) -> tuple[PrismCoordinator, FakeRpc]: - server = PrismCoordinator.__new__(PrismCoordinator) - template = template or base_template() - rpc = FakeRpc(template, tip=str(template["previousblockhash"])) - server.rpc = rpc - server.qbit_chain = "regtest" - server.lock = threading.RLock() - server.stop_event = threading.Event() - server.clients = set() - server.jobs = {} - server.job_counter = 0 - server.connection_counter = 0 - server.accepted_block_count = 0 - server.max_blocks = 1_000 - server.started_monotonic = time.monotonic() - server.submitted_share_count = 0 - server.stale_share_count = 0 - server.duplicate_share_count = 0 - server.low_difficulty_share_count = 0 - server.rejection_counts_by_reason = {reason: 0 for reason in PRISM_REJECTION_REASON_IDS} - server.job_build_failure_count = 0 - server.tip_refresh_job_count = 0 - server.post_accept_refresh_failure_count = 0 - server.reorg_reconciler_enabled = False - server.reorg_inactive_block_count = 0 - server.reorg_reactivated_block_count = 0 - server.reorg_reconcile_skip_count = 0 - server.reorg_reconcile_error_count = 0 - server.matured_payout_count = 0 - server.last_reorg_reconciled_tip_hash = None - server.last_reorg_reconciled_trusted = False - server.last_reorg_reconciled_monotonic = None - server.latest_evidence = None - server.latest_coinbase_size_bytes = None - server.tip_template_snapshot = None - server.extranonce2_size = EXTRANONCE2_SIZE - server.coinbase_tag_hex = default_prism_coinbase_tag_hex() - server.share_difficulty = Decimal("1") - server.vardiff_config = vardiff.VardiffConfig( - enabled=True, - target_share_interval_seconds=Decimal("15"), - min_difficulty=Decimal("0.000000001"), - max_difficulty=Decimal("1024"), - retarget_interval_seconds=Decimal("90"), - max_step_factor=Decimal("4"), - startup_difficulty=Decimal("1"), - max_step_down_factor=Decimal("4"), - ewma_alpha=Decimal("0.4"), - retarget_tolerance=Decimal("0.25"), - ) - server.default_share_weight = 1 - server.share_weights_by_username = {} - server.min_ready_miners = 3 - server.ledger = ledger if ledger is not None else FakeLedger() - server.blockpoll_seconds = 2.0 - # Failed-refresh spacing is opt-in per test: its holdoff waits on real - # time, which deadlocks tests that freeze time.monotonic around failing - # polls. Pacing behavior is covered by test_prism_refresh_retry_pacing. - server.tip_refresh_failure_holdoff_seconds = 0.0 - server.job_bundle_cache_seconds = 10.0 - server.template_cache_seconds = 2.0 - server.reorg_reconcile_cache_seconds = 5.0 - server.health_refresh_seconds = 5.0 - server.stratum_send_timeout_seconds = 20.0 - server._ensure_job_cache_state() - return server, rpc - - -def install_fake_bundle_builder(server: PrismCoordinator) -> dict[str, object]: - """Replace the audit bundle subprocess with a counting fake whose manifest - coinbase embeds exactly the suffix the coordinator asked for.""" - recorded: dict[str, object] = {"calls": 0, "suffixes": []} - - def fake_build_audit_bundle(**kwargs: object) -> dict[str, object]: - recorded["calls"] = int(recorded["calls"]) + 1 - suffix_hex = str(kwargs["coinbase_script_sig_suffix_hex"]) - recorded["suffixes"].append(suffix_hex) - recorded["last_kwargs"] = kwargs - return { - "found_block": dict(kwargs["found_block"]), - "payout_policy_manifest": {"accounts": []}, - "signed_coinbase_manifest": { - "manifest": { - "coinbase_tx_hex": synthetic_manifest_coinbase_hex(suffix_hex), - } - }, - } - - server.build_audit_bundle = fake_build_audit_bundle # type: ignore[method-assign] - return recorded - - -def stamped_pending_share(accepted_at_ms: int) -> PendingShare: - return PendingShare( - share_id=f"miner-a:{accepted_at_ms}", - miner_id="miner-a", - order_key="miner-a", - p2mr_program_hex="22" * 32, - share_difficulty=1, - network_difficulty=1, - template_height=9, - job_id="job-1", - job_issued_at_ms=accepted_at_ms - 1, - accepted_at_ms=accepted_at_ms, - ntime=1_700_000_000, - ) - - -class AnchorRecordingLedger(FakeLedger): - def __init__(self) -> None: - super().__init__() - self.anchors: list[int] = [] - - def snapshot_at_job_issue( - self, anchor_job_issued_at_ms: int, *, window_weight: int | None = None - ) -> list[FakeShare]: - self.anchors.append(int(anchor_job_issued_at_ms)) - return super().snapshot_at_job_issue( - anchor_job_issued_at_ms, window_weight=window_weight - ) - - -class IncrementalRecordingLedger(SingleWriterShareLedger): - def __init__(self) -> None: - super().__init__() - self.full_snapshot_calls = 0 - self.delta_snapshot_calls = 0 - - def snapshot_at_job_issue( - self, - anchor_job_issued_at_ms: int, - *, - window_weight: int | None = None, - ) -> list[object]: - self.full_snapshot_calls += 1 - return super().snapshot_at_job_issue( - anchor_job_issued_at_ms, - window_weight=window_weight, - ) - - def snapshot_between_job_issues( - self, - previous_anchor_job_issued_at_ms: int, - anchor_job_issued_at_ms: int, - ) -> list[object]: - self.delta_snapshot_calls += 1 - return super().snapshot_between_job_issues( - previous_anchor_job_issued_at_ms, - anchor_job_issued_at_ms, - ) - - -def append_incremental_share( - ledger: SingleWriterShareLedger, - *, - share_seq: int, - accepted_at_ms: int, -) -> None: - ledger.append( - PendingShare( - share_id=f"incremental-{share_seq}", - miner_id=f"miner-{share_seq % 3}", - order_key=f"miner-{share_seq % 3}", - p2mr_program_hex=f"{share_seq % 256:02x}" * 32, - share_difficulty=1, - network_difficulty=1, - template_height=9, - job_id=f"job-{share_seq}", - job_issued_at_ms=accepted_at_ms - 1, - accepted_at_ms=accepted_at_ms, - ntime=1_700_000_000 + share_seq, - ) - ) - - -class SnapshotAnchorFloorTests(unittest.TestCase): - def _hold_floor(self, server: PrismCoordinator, share: PendingShare) -> None: - server._ensure_pending_share_commit_state() - with server._pending_share_commit_lock: - server._pending_share_commit_floor[id(share)] = [ - share, - time.monotonic(), - False, - ] - - def test_job_bundle_anchor_clamps_below_pending_share_commit(self) -> None: - # The issued snapshot must be reproducible from the durable ledger: - # while a stamped share's commit is pending, the job anchor (which the - # bundle declares as anchor_job_issued_at_ms) has to predate it, or - # qbit_audit_share_window at the declared anchor would include a share - # the published window omitted. - ledger = AnchorRecordingLedger() - server, _rpc = coordinator(ledger=ledger) - install_fake_bundle_builder(server) - stamped_ms = now_ms() - 5 - share = stamped_pending_share(stamped_ms) - self._hold_floor(server, share) - - bundle = server.build_shared_job_bundle( - server.current_template_artifacts(), - worker(), - ) - self.assertEqual(ledger.anchors[-1], stamped_ms - 1) - self.assertEqual( - bundle.found_block["anchor_job_issued_at_ms"], stamped_ms - 1 - ) - self.assertEqual(bundle.issued_at_ms, stamped_ms - 1) - - server._finish_pending_share_commit(share) - # The issued time is frozen per template generation; drop the frozen - # entry so the rebuild stamps a fresh anchor now that no commit is - # pending. - with server._job_cache_lock: - server._job_build_issued_at_ms.clear() - rebuilt = server.build_shared_job_bundle( - server.current_template_artifacts(), - worker(), - ) - self.assertGreaterEqual(ledger.anchors[-1], stamped_ms) - self.assertGreaterEqual( - int(rebuilt.found_block["anchor_job_issued_at_ms"]), stamped_ms - ) - - def test_payout_artifact_declares_its_own_snapshot_anchor(self) -> None: - # An artifact snapshot is taken at its own earlier anchor. A bundle - # reusing the artifact must declare that anchor rather than the - # fresher job-issue time: a share stamped between the two anchors is - # excluded from the artifact by construction, yet a re-derivation at - # the job-issue anchor would include it. - ledger = AnchorRecordingLedger() - server, _rpc = coordinator(ledger=ledger) - install_fake_bundle_builder(server) - artifacts = server.current_template_artifacts() - clamp_now_ms = now_ms() - 6 - # Anchor selection sits strictly below the clamp instant so a share - # stamped in the same millisecond can never tie the anchor. - artifact_anchor_ms = clamp_now_ms - 1 - with patch( - "lab.prism.prism_coordinator.now_ms", - return_value=clamp_now_ms, - ): - artifact = server._build_payout_ledger_artifact( - 0, 0, artifacts.network_difficulty - ) - assert artifact is not None - self.assertEqual(artifact.snapshot_anchor_ms, artifact_anchor_ms) - self.assertEqual(ledger.anchors[-1], artifact_anchor_ms) - - # Construction re-validates that the passed artifact is the installed - # current one. - with server._job_cache_lock: - server._payout_ledger_artifact = artifact - bundle = server.build_shared_job_bundle( - artifacts, - worker(), - payout_artifact=artifact, - ) - self.assertEqual( - bundle.found_block["anchor_job_issued_at_ms"], - artifact.snapshot_anchor_ms, - ) - self.assertGreater(bundle.issued_at_ms, int(artifact.snapshot_anchor_ms)) - - def test_artifact_paths_proceed_at_the_clamped_anchor_floor( - self, - ) -> None: - # The pending-commit clamp is anchor selection, not a fence: every - # share stamped at or below the clamped anchor is already durable, - # so the window read at that anchor is exact and reproducible, and - # both artifact producers publish it. Shares stamped above the - # anchor deterministically belong to the next window. - ledger = AnchorRecordingLedger() - server, _rpc = coordinator(ledger=ledger) - install_fake_bundle_builder(server) - artifacts = server.current_template_artifacts() - stamped_ms = now_ms() - 5 - share = stamped_pending_share(stamped_ms) - self._hold_floor(server, share) - try: - artifact = server._build_payout_ledger_artifact( - 0, 0, artifacts.network_difficulty - ) - self.assertIsNotNone(artifact) - assert artifact is not None - self.assertEqual(artifact.snapshot_anchor_ms, stamped_ms - 1) - self.assertEqual(ledger.anchors[-1], stamped_ms - 1) - - # The synchronous build proceeds at the clamped anchor and seeds - # its window for reuse under that same anchor; cache publication - # installs the seed. - bundle = server.build_shared_job_bundle(artifacts, worker()) - self.assertEqual(bundle.issued_at_ms, stamped_ms - 1) - seeded = bundle.prepared_ledger_artifact - self.assertIsNotNone(seeded) - assert seeded is not None - self.assertEqual(seeded.snapshot_anchor_ms, stamped_ms - 1) - server._install_payout_ledger_artifact(seeded) - with server._job_cache_lock: - published = server._payout_ledger_artifact - self.assertIsNotNone(published) - assert published is not None - self.assertEqual(published.snapshot_anchor_ms, stamped_ms - 1) - finally: - server._finish_pending_share_commit(share) - - def test_background_build_refuses_a_pathologically_old_floor(self) -> None: - # A floor held further below now than the audit anchor ceiling (a - # wedged writer or leaked release) would arm an artifact that is - # born expired; refuse before paying the window walk so the - # re-arm backoff paces retries. - ledger = AnchorRecordingLedger() - server, _rpc = coordinator(ledger=ledger) - install_fake_bundle_builder(server) - artifacts = server.current_template_artifacts() - share = stamped_pending_share(now_ms() - 400_000) - self._hold_floor(server, share) - try: - self.assertIsNone( - server._build_payout_ledger_artifact( - 0, 0, artifacts.network_difficulty - ) - ) - self.assertEqual(ledger.snapshot_calls, 0) - finally: - server._finish_pending_share_commit(share) - - def test_anchor_selection_excludes_same_millisecond_stamps(self) -> None: - # Millisecond granularity can hand a share stamped right after - # anchor selection the same accepted_at_ms as the clamp instant, - # and such a share is never protected by the pending floor. The - # window predicate is anchor-inclusive, so the anchor must sit - # strictly below the clamp-time millisecond or that share would - # join audit replays of a window that never contained it. - server, _rpc = coordinator(ledger=AnchorRecordingLedger()) - clamp_now = now_ms() - self.assertEqual( - server._job_snapshot_anchor_ms(clamp_now), - clamp_now - 1, - ) - - share = stamped_pending_share(clamp_now - 7) - self._hold_floor(server, share) - try: - self.assertEqual( - server._job_snapshot_anchor_ms(clamp_now), - clamp_now - 8, - ) - finally: - server._finish_pending_share_commit(share) - - -class IncrementalPayoutArtifactTests(unittest.TestCase): - def configured_server( - self, - ) -> tuple[PrismCoordinator, IncrementalRecordingLedger, object]: - ledger = IncrementalRecordingLedger() - append_incremental_share(ledger, share_seq=1, accepted_at_ms=999_900) - append_incremental_share(ledger, share_seq=2, accepted_at_ms=999_910) - append_incremental_share(ledger, share_seq=3, accepted_at_ms=999_920) - server, rpc = coordinator(ledger=ledger) - artifacts = server.store_template_artifacts(dict(rpc.template)) - assert artifacts is not None - server._pool_ready_latched = True - server.payout_artifact_min_build_interval_seconds = 60.0 - server.payout_artifact_full_rescan_seconds = 3_600.0 - return server, ledger, artifacts - - @staticmethod - def replay_shaped_append_entry( - *, - accepted_at_ms: int, - block_hash_hex: str, - ) -> PendingShareAppend: - persisted = stamped_pending_share(accepted_at_ms) - pending = PendingShare(**dict(vars(persisted))) - return PendingShareAppend( - pending_share=pending, - username="miner-a", - job_id=pending.job_id, - block_hash_hex=block_hash_hex, - collection_only=False, - credit_policy=None, - candidate_intent={ - "block_hash_hex": block_hash_hex, - "pending_share": dict(vars(pending)), - "credit_share_on_accept": True, - }, - ) - - def test_late_visible_replay_append_forces_next_build_to_full_oracle( - self, - ) -> None: - server, ledger, artifacts = self.configured_server() - clock_ms = [1_000_000] - with patch( - "lab.prism.prism_coordinator.now_ms", - side_effect=lambda: clock_ms[0], - ): - initial = server._build_payout_ledger_artifact( - 0, 0, artifacts.network_difficulty - ) - assert initial is not None - self.assertEqual(initial.window_build_mode, "full_rescan") - - server.payout_artifact_min_build_interval_seconds = 0.0 - clock_ms[0] = 1_000_020 - advanced = server._build_payout_ledger_artifact( - 0, 0, artifacts.network_difficulty - ) - assert advanced is not None - self.assertEqual(advanced.window_build_mode, "incremental") - self.assertTrue(server._install_payout_ledger_artifact(advanced)) - anchor_ms = int(advanced.snapshot_anchor_ms) - - entry = self.replay_shaped_append_entry( - accepted_at_ms=anchor_ms - 10, - block_hash_hex="44" * 32, - ) - self.assertTrue(server._append_share_batch([entry])) - self.assertIsNone(server._incremental_payout_artifact_window) - with server._job_cache_lock: - self.assertIsNone(server._payout_ledger_artifact) - self.assertFalse(server._install_payout_ledger_artifact(advanced)) - self.assertIsNone( - server._cached_found_block_payout_artifact( - base_generation=0, - artifact_payout_state_generation=1, - network_difficulty=artifacts.network_difficulty, - fallback_reason="prepare_lock_busy", - ) - ) - - clock_ms[0] = 1_000_040 - rebuilt = server._build_payout_ledger_artifact( - 0, 0, artifacts.network_difficulty - ) - - assert rebuilt is not None - self.assertEqual(rebuilt.window_build_mode, "full_rescan") - self.assertEqual( - rebuilt.window_full_rescan_reason, - "late_visible_append", - ) - self.assertEqual(ledger.full_snapshot_calls, 2) - self.assertIn( - entry.pending_share.share_id, - {str(share["share_id"]) for share in rebuilt.shares_json}, - ) - - def test_replay_shaped_batch_commits_under_the_landing_fence(self) -> None: - # The durable append and its epoch bump must share the landing fence - # boundary: a bump that only starts after append_batch() returned - # leaves a gap where a landing verifies the pre-bump epoch and enters - # submitblock with the row already durable but unpaid. A batch holding - # a replay-shaped row therefore holds the fence across the ledger - # commit itself; an ordinary batch stays off the lock entirely. - server, ledger, artifacts = self.configured_server() - clock_ms = [1_000_000] - with patch( - "lab.prism.prism_coordinator.now_ms", - side_effect=lambda: clock_ms[0], - ): - initial = server._build_payout_ledger_artifact( - 0, 0, artifacts.network_difficulty - ) - assert initial is not None - server.payout_artifact_min_build_interval_seconds = 0.0 - clock_ms[0] = 1_000_020 - advanced = server._build_payout_ledger_artifact( - 0, 0, artifacts.network_difficulty - ) - assert advanced is not None - self.assertTrue(server._install_payout_ledger_artifact(advanced)) - anchor_ms = int(advanced.snapshot_anchor_ms) - - fence_held_during_commit: list[bool] = [] - original_append_batch = ledger.append_batch - - def recording_append_batch(entries: list[object]) -> list[object]: - fence_held_during_commit.append( - server._payout_append_landing_fence_lock.locked() - ) - return original_append_batch(entries) - - ledger.append_batch = recording_append_batch # type: ignore[method-assign] - - fresh = self.replay_shaped_append_entry( - accepted_at_ms=anchor_ms + 50, - block_hash_hex="88" * 32, - ) - self.assertTrue(server._append_share_batch([fresh])) - self.assertEqual(fence_held_during_commit, [False]) - with server._job_cache_lock: - self.assertEqual( - server._payout_ledger_append_invalidation_epoch, 0 - ) - - predating = self.replay_shaped_append_entry( - accepted_at_ms=anchor_ms - 10, - block_hash_hex="99" * 32, - ) - self.assertTrue(server._append_share_batch([predating])) - self.assertEqual(fence_held_during_commit, [False, True]) - with server._job_cache_lock: - self.assertEqual( - server._payout_ledger_append_invalidation_epoch, 1 - ) - self.assertIsNone(server._payout_ledger_artifact) - self.assertIsNone(server._incremental_payout_artifact_window) - - def test_replay_append_during_cold_scan_invalidates_inflight_walk( - self, - ) -> None: - # A cold-start walk exposes no cached window or armed artifact while - # its ledger read runs. A replayed row committing mid-walk with - # timestamps below the walk's anchor must still record an - # invalidation, or the walk installs a window that can never - # rediscover the row through delta reads. - server, ledger, artifacts = self.configured_server() - clock_ms = [1_000_000] - entry = self.replay_shaped_append_entry( - accepted_at_ms=999_950, - block_hash_hex="55" * 32, - ) - original_snapshot = ledger.snapshot_at_job_issue - raced = threading.Event() - append_threads: list[threading.Thread] = [] - - def racing_snapshot( - anchor_job_issued_at_ms: int, - *, - window_weight: int | None = None, - ) -> list[object]: - # Capture the read result first: the replayed row committing - # afterwards models a database snapshot taken before the commit. - records = original_snapshot( - anchor_job_issued_at_ms, - window_weight=window_weight, - ) - if not raced.is_set(): - raced.set() - thread = threading.Thread( - target=server._append_share_batch, - args=([entry],), - ) - thread.start() - append_threads.append(thread) - deadline = time.monotonic() + 2.0 - while time.monotonic() < deadline: - with server._job_cache_lock: - if server._payout_ledger_append_invalidation_epoch: - break - time.sleep(0.001) - return records - - ledger.snapshot_at_job_issue = racing_snapshot # type: ignore[method-assign] - with patch( - "lab.prism.prism_coordinator.now_ms", - side_effect=lambda: clock_ms[0], - ): - initial = server._build_payout_ledger_artifact( - 0, 0, artifacts.network_difficulty - ) - assert initial is not None - self.assertEqual(initial.window_build_mode, "full_rescan") - self.assertNotIn( - entry.pending_share.share_id, - {str(share["share_id"]) for share in initial.shares_json}, - ) - self.assertTrue(append_threads) - append_threads[0].join(timeout=2.0) - self.assertFalse(append_threads[0].is_alive()) - with server._job_cache_lock: - self.assertEqual( - server._payout_ledger_append_invalidation_epoch, 1 - ) - self.assertFalse(server._install_payout_ledger_artifact(initial)) - with server._job_cache_lock: - self.assertIsNone(server._payout_ledger_artifact) - - server.payout_artifact_min_build_interval_seconds = 0.0 - clock_ms[0] = 1_000_040 - rebuilt = server._build_payout_ledger_artifact( - 0, 0, artifacts.network_difficulty - ) - - assert rebuilt is not None - self.assertEqual(rebuilt.window_build_mode, "full_rescan") - self.assertEqual( - rebuilt.window_full_rescan_reason, - "late_visible_append", - ) - self.assertIn( - entry.pending_share.share_id, - {str(share["share_id"]) for share in rebuilt.shares_json}, - ) - - def test_inflight_scan_anchor_records_invalidation_without_windows( - self, - ) -> None: - # Directly exercise the published in-flight anchor: with no cached - # window and no armed artifact, only a predating append may advance - # the epoch, and fresh shares above the anchor must leave the walk - # undisturbed. - server, _ledger, _artifacts = self.configured_server() - server._ensure_job_cache_state() - predating = stamped_pending_share(999_950) - fresh = stamped_pending_share(1_000_050) - - server._invalidate_incremental_payout_window_for_append(predating) - with server._job_cache_lock: - self.assertEqual(server._payout_ledger_append_invalidation_epoch, 0) - - with patch( - "lab.prism.prism_coordinator.now_ms", - side_effect=lambda: 1_000_000, - ): - token = server._expose_inflight_scan_anchor(999_999) - try: - server._invalidate_incremental_payout_window_for_append(fresh) - with server._job_cache_lock: - self.assertEqual( - server._payout_ledger_append_invalidation_epoch, 0 - ) - server._invalidate_incremental_payout_window_for_append(predating) - finally: - server._retire_inflight_scan_anchor(token) - with server._job_cache_lock: - self.assertEqual(server._payout_ledger_append_invalidation_epoch, 1) - self.assertEqual(server._payout_window_inflight_scan_anchors, {}) - self.assertEqual( - server._incremental_payout_artifact_window_invalidation_reason, - "late_visible_append", - ) - - def test_replay_append_during_bundle_construction_invalidates_build( - self, - ) -> None: - # A cold synchronous ready build exposes no cached window or armed - # artifact once its ledger read returns, while manifest construction - # and cache publication still lie ahead. A replayed row committing in - # that gap with timestamps below the read's anchor must still advance - # the epoch -- the exposure outlives the read -- so the publication - # fence supersedes the bundle instead of seeding a window that delta - # reads can never rediscover the row from. - server, _ledger, _artifacts = self.configured_server() - install_fake_bundle_builder(server) - clock_ms = [1_000_000] - entry = self.replay_shaped_append_entry( - accepted_at_ms=999_950, - block_hash_hex="66" * 32, - ) - raced = threading.Event() - original_build = server.build_audit_bundle - - def racing_build(**kwargs: object) -> dict[str, object]: - # The ledger read has already returned; the replayed row commits - # while the coinbase manifest is being constructed. - if not raced.is_set(): - raced.set() - server._append_share_batch([entry]) - return original_build(**kwargs) - - server.build_audit_bundle = racing_build # type: ignore[method-assign] - with patch( - "lab.prism.prism_coordinator.now_ms", - side_effect=lambda: clock_ms[0], - ): - with self.assertRaises(JobBuildSuperseded): - server.build_shared_job_bundle( - server.current_template_artifacts(), - worker(), - ) - with server._job_cache_lock: - self.assertEqual( - server._payout_ledger_append_invalidation_epoch, 1 - ) - self.assertIsNone(server._payout_ledger_artifact) - - # A fresh build keyed to the advanced epoch re-reads the ledger - # and rediscovers the replayed row. - with server._job_cache_lock: - server._job_build_issued_at_ms.clear() - clock_ms[0] = 1_000_040 - rebuilt = server.build_shared_job_bundle( - server.current_template_artifacts(), - worker(), - ) - self.assertIn( - entry.pending_share.share_id, - {str(share["share_id"]) for share in rebuilt.shares_json}, - ) - - def test_seeded_artifact_exposure_survives_until_install_fence( - self, - ) -> None: - # The synchronous build's exposed scan anchor rides its seeded - # artifact: a replayed row committing after the bundle returned but - # before cache publication installs the seed must advance the epoch, - # and the doomed seed's install fence must discard it (retiring the - # exposure) instead of arming the pre-append window. - server, _ledger, _artifacts = self.configured_server() - install_fake_bundle_builder(server) - with patch( - "lab.prism.prism_coordinator.now_ms", - side_effect=lambda: 1_000_000, - ): - bundle = server.build_shared_job_bundle( - server.current_template_artifacts(), - worker(), - ) - seed = bundle.prepared_ledger_artifact - assert seed is not None - self.assertIsNotNone(seed.inflight_scan_anchor_token) - - entry = self.replay_shaped_append_entry( - accepted_at_ms=999_950, - block_hash_hex="77" * 32, - ) - self.assertTrue(server._append_share_batch([entry])) - with server._job_cache_lock: - self.assertEqual( - server._payout_ledger_append_invalidation_epoch, 1 - ) - - self.assertFalse(server._install_payout_ledger_artifact(seed)) - with server._job_cache_lock: - self.assertIsNone(server._payout_ledger_artifact) - self.assertEqual(server._payout_window_inflight_scan_anchors, {}) - - def test_late_append_disarms_before_prepare_lock_is_available(self) -> None: - server, _ledger, artifacts = self.configured_server() - clock_ms = [1_000_000] - with patch( - "lab.prism.prism_coordinator.now_ms", - side_effect=lambda: clock_ms[0], - ): - artifact = server._build_payout_ledger_artifact( - 0, 0, artifacts.network_difficulty - ) - assert artifact is not None - self.assertTrue(server._install_payout_ledger_artifact(artifact)) - anchor_ms = int(artifact.snapshot_anchor_ms) - pending = stamped_pending_share(anchor_ms - 10) - - server._payout_state_prepare_lock.acquire() - # Exercise the artifact anchor independently: an already-cold - # incremental cache must not hide an unsafe armed artifact. - server._incremental_payout_artifact_window = None - invalidation_finished = threading.Event() - - def invalidate() -> None: - server._invalidate_incremental_payout_window_for_append( - pending - ) - invalidation_finished.set() - - thread = threading.Thread(target=invalidate) - thread.start() - try: - deadline = time.monotonic() + 1.0 - while time.monotonic() < deadline: - with server._job_cache_lock: - if server._payout_ledger_artifact is None: - break - time.sleep(0.001) - with server._job_cache_lock: - self.assertIsNone(server._payout_ledger_artifact) - self.assertGreater( - server._payout_ledger_append_invalidation_epoch, - artifact.append_invalidation_epoch, - ) - self.assertFalse(invalidation_finished.is_set()) - self.assertFalse( - server._install_payout_ledger_artifact(artifact) - ) - finally: - server._payout_state_prepare_lock.release() - thread.join(timeout=1.0) - self.assertFalse(thread.is_alive()) - self.assertTrue(invalidation_finished.is_set()) - - def test_atomic_publication_rejects_pre_append_artifact(self) -> None: - server, _ledger, artifacts = self.configured_server() - clock_ms = [1_000_000] - with patch( - "lab.prism.prism_coordinator.now_ms", - side_effect=lambda: clock_ms[0], - ): - candidate = server._current_payout_state_candidate() - stale = candidate.ledger_artifact - assert stale is not None - anchor_ms = int(stale.snapshot_anchor_ms) - server._invalidate_incremental_payout_window_for_append( - stamped_pending_share(anchor_ms - 10) - ) - with server._payout_artifact_executor_lock: - server._payout_artifact_executor_shutdown = True - published = server._publish_payout_state_candidate(candidate) - - self.assertIsNotNone(published) - with server._job_cache_lock: - self.assertIsNone(server._payout_ledger_artifact) - self.assertGreater( - server._payout_ledger_append_invalidation_epoch, - stale.append_invalidation_epoch, - ) - - def test_late_append_supersedes_in_flight_sync_seed_bundle(self) -> None: - server, _ledger, artifacts = self.configured_server() - clock_ms = [1_000_000] - with patch( - "lab.prism.prism_coordinator.now_ms", - side_effect=lambda: clock_ms[0], - ): - artifact = server._build_payout_ledger_artifact( - 0, 0, artifacts.network_difficulty - ) - assert artifact is not None - self.assertTrue(server._install_payout_ledger_artifact(artifact)) - pending = stamped_pending_share( - int(artifact.snapshot_anchor_ms) - 10 - ) - - def invalidating_builder(**kwargs: object) -> dict[str, object]: - server._invalidate_incremental_payout_window_for_append( - pending - ) - suffix_hex = str(kwargs["coinbase_script_sig_suffix_hex"]) - return { - "found_block": dict(kwargs["found_block"]), - "payout_policy_manifest": {"accounts": []}, - "signed_coinbase_manifest": { - "manifest": { - "coinbase_tx_hex": synthetic_manifest_coinbase_hex( - suffix_hex - ), - } - }, - } - - server.build_audit_bundle = invalidating_builder # type: ignore[method-assign] - with self.assertRaises(JobBuildSuperseded): - server.build_shared_job_bundle( - artifacts, - worker(), - payout_artifact=None, - ) - - with server._job_cache_lock: - self.assertIsNone(server._payout_ledger_artifact) - - def test_normal_append_preserves_incremental_window(self) -> None: - server, ledger, artifacts = self.configured_server() - clock_ms = [1_000_000] - with patch( - "lab.prism.prism_coordinator.now_ms", - side_effect=lambda: clock_ms[0], - ): - initial = server._build_payout_ledger_artifact( - 0, 0, artifacts.network_difficulty - ) - assert initial is not None - cached = server._incremental_payout_artifact_window - self.assertIsNotNone(cached) - self.assertTrue(server._install_payout_ledger_artifact(initial)) - with server._job_cache_lock: - armed = server._payout_ledger_artifact - append_epoch = ( - server._payout_ledger_append_invalidation_epoch - ) - self.assertIsNotNone(armed) - - entry = self.replay_shaped_append_entry( - accepted_at_ms=1_000_010, - block_hash_hex="55" * 32, - ) - self.assertTrue(server._append_share_entry(entry)) - self.assertIs(server._incremental_payout_artifact_window, cached) - with server._job_cache_lock: - self.assertIs(server._payout_ledger_artifact, armed) - self.assertEqual( - server._payout_ledger_append_invalidation_epoch, - append_epoch, - ) - self.assertIs( - server._usable_payout_ledger_artifact( - 0, - artifacts.network_difficulty, - ), - armed, - ) - - server.payout_artifact_min_build_interval_seconds = 0.0 - clock_ms[0] = 1_000_020 - advanced = server._build_payout_ledger_artifact( - 0, 0, artifacts.network_difficulty - ) - - assert advanced is not None - self.assertEqual(advanced.window_build_mode, "incremental") - self.assertIsNone(advanced.window_full_rescan_reason) - self.assertEqual(advanced.window_touched_pages, 1) - self.assertEqual(ledger.full_snapshot_calls, 1) - self.assertEqual(ledger.delta_snapshot_calls, 1) - self.assertIn( - entry.pending_share.share_id, - {str(share["share_id"]) for share in advanced.shares_json}, - ) - - def test_debounce_then_delta_and_forced_full_rescan(self) -> None: - server, ledger, artifacts = self.configured_server() - clock_ms = [1_000_000] - with patch( - "lab.prism.prism_coordinator.now_ms", - side_effect=lambda: clock_ms[0], - ): - initial = server._build_payout_ledger_artifact( - 0, 0, artifacts.network_difficulty - ) - assert initial is not None - self.assertEqual(initial.window_build_mode, "full_rescan") - self.assertEqual(ledger.full_snapshot_calls, 1) - - append_incremental_share( - ledger, - share_seq=4, - accepted_at_ms=1_000_010, - ) - clock_ms[0] = 1_000_020 - debounced = server._build_payout_ledger_artifact( - 0, 0, artifacts.network_difficulty - ) - assert debounced is not None - self.assertEqual(debounced.window_build_mode, "debounced") - self.assertEqual(ledger.full_snapshot_calls, 1) - self.assertEqual(ledger.delta_snapshot_calls, 0) - self.assertEqual(len(debounced.shares_json), 3) - - server.payout_artifact_min_build_interval_seconds = 0.0 - advanced = server._build_payout_ledger_artifact( - 0, 0, artifacts.network_difficulty - ) - assert advanced is not None - self.assertEqual(advanced.window_build_mode, "incremental") - self.assertEqual(advanced.window_delta_rows, 1) - self.assertEqual(ledger.delta_snapshot_calls, 1) - self.assertEqual(len(advanced.shares_json), 4) - - forced = server._build_payout_ledger_artifact( - 0, - 0, - artifacts.network_difficulty, - True, - ) - assert forced is not None - self.assertEqual(forced.window_build_mode, "full_rescan") - self.assertEqual( - forced.window_full_rescan_reason, - "reconcile_invalidation", - ) - self.assertEqual(ledger.full_snapshot_calls, 2) - - def test_periodic_self_check_compares_delta_to_full_oracle(self) -> None: - server, ledger, artifacts = self.configured_server() - clock_ms = [1_000_000] - with patch( - "lab.prism.prism_coordinator.now_ms", - side_effect=lambda: clock_ms[0], - ): - self.assertIsNotNone( - server._build_payout_ledger_artifact( - 0, 0, artifacts.network_difficulty - ) - ) - append_incremental_share( - ledger, - share_seq=4, - accepted_at_ms=1_000_010, - ) - clock_ms[0] = 1_000_020 - server.payout_artifact_min_build_interval_seconds = 0.0 - server.payout_artifact_full_rescan_seconds = 0.0 - - checked = server._build_payout_ledger_artifact( - 0, 0, artifacts.network_difficulty - ) - - assert checked is not None - self.assertEqual(checked.window_build_mode, "self_check_match") - self.assertEqual(ledger.delta_snapshot_calls, 1) - self.assertEqual(ledger.full_snapshot_calls, 2) - - def test_routine_build_runs_overdue_self_check_despite_preview_refresh( - self, - ) -> None: - # Accepted-block previews bypass the build interval and refresh - # refreshed_monotonic on every block, yet never run the periodic - # oracle themselves. Under a sustained sub-interval block cadence - # every routine build used to return from the debounce branch - # before reaching the self-check condition, so the configured - # full-rescan window and balance oracle could be postponed - # indefinitely. An overdue self-check now disarms the debounce - # for routine builds while the urgent preview path stays - # debounce-free and oracle-free. - server, ledger, artifacts = self.configured_server() - clock_ms = [1_000_000] - with patch( - "lab.prism.prism_coordinator.now_ms", - side_effect=lambda: clock_ms[0], - ): - self.assertIsNotNone( - server._build_payout_ledger_artifact( - 0, 0, artifacts.network_difficulty - ) - ) - append_incremental_share( - ledger, - share_seq=4, - accepted_at_ms=1_000_010, - ) - clock_ms[0] = 1_000_020 - preview = server._build_payout_ledger_artifact( - 0, 0, artifacts.network_difficulty, False, True - ) - assert preview is not None - self.assertEqual(preview.window_build_mode, "incremental") - # The next preview arrives with the self-check long overdue; - # the urgent path must still not pay the oracle walk. - server.payout_artifact_full_rescan_seconds = 0.0 - append_incremental_share( - ledger, - share_seq=5, - accepted_at_ms=1_000_030, - ) - clock_ms[0] = 1_000_040 - urgent = server._build_payout_ledger_artifact( - 0, 0, artifacts.network_difficulty, False, True - ) - assert urgent is not None - self.assertEqual(urgent.window_build_mode, "incremental") - self.assertEqual(ledger.full_snapshot_calls, 1) - # A routine build inside the min build interval used to - # debounce here -- forever, while blocks kept refreshing the - # window above -- and the self-check never ran. - checked = server._build_payout_ledger_artifact( - 0, 0, artifacts.network_difficulty - ) - - assert checked is not None - self.assertEqual(checked.window_build_mode, "self_check_match") - self.assertEqual(ledger.full_snapshot_calls, 2) - - def test_periodic_self_check_replaces_divergent_delta_with_full_bytes( - self, - ) -> None: - class OmittingDeltaLedger(IncrementalRecordingLedger): - def snapshot_between_job_issues( - self, - previous_anchor_job_issued_at_ms: int, - anchor_job_issued_at_ms: int, - ) -> list[object]: - del previous_anchor_job_issued_at_ms, anchor_job_issued_at_ms - self.delta_snapshot_calls += 1 - return [] - - ledger = OmittingDeltaLedger() - for share_seq in range(1, 4): - append_incremental_share( - ledger, - share_seq=share_seq, - accepted_at_ms=999_900 + share_seq, - ) - server, rpc = coordinator(ledger=ledger) - artifacts = server.store_template_artifacts(dict(rpc.template)) - assert artifacts is not None - server._pool_ready_latched = True - clock_ms = [1_000_000] - with patch( - "lab.prism.prism_coordinator.now_ms", - side_effect=lambda: clock_ms[0], - ): - self.assertIsNotNone( - server._build_payout_ledger_artifact( - 0, 0, artifacts.network_difficulty - ) - ) - append_incremental_share( - ledger, - share_seq=4, - accepted_at_ms=1_000_010, - ) - clock_ms[0] = 1_000_020 - server.payout_artifact_min_build_interval_seconds = 0.0 - server.payout_artifact_full_rescan_seconds = 0.0 - repaired = server._build_payout_ledger_artifact( - 0, 0, artifacts.network_difficulty - ) - forced = server._build_payout_ledger_artifact( - 0, - 0, - artifacts.network_difficulty, - True, - ) - - assert repaired is not None and forced is not None - self.assertEqual(repaired.window_build_mode, "self_check_mismatch") - self.assertEqual(list(repaired.shares_json), list(forced.shares_json)) - self.assertEqual( - repaired.share_snapshot_sha256, - forced.share_snapshot_sha256, - ) - - def test_periodic_self_check_invalidates_drifted_published_balances( - self, - ) -> None: - class DriftingCarryLedger(IncrementalRecordingLedger): - def __init__(self) -> None: - super().__init__() - self.balance_sats = 546 - self.prior_balance_reads = 0 - - def current_prior_balances(self) -> list[dict[str, object]]: - self.prior_balance_reads += 1 - return [ - { - "recipient_id": "carry", - "order_key": "01:carry", - "p2mr_program_hex": "66" * 32, - "balance_sats": self.balance_sats, - } - ] - - ledger = DriftingCarryLedger() - for share_seq in range(1, 4): - append_incremental_share( - ledger, - share_seq=share_seq, - accepted_at_ms=999_900 + share_seq, - ) - server, rpc = coordinator(ledger=ledger) - artifacts = server.store_template_artifacts(dict(rpc.template)) - assert artifacts is not None - server._pool_ready_latched = True - server.payout_artifact_min_build_interval_seconds = 0.0 - server.payout_artifact_full_rescan_seconds = 3_600.0 - clock_ms = [1_000_000] - monotonic_seconds = [100.0] - build_logs: list[dict[str, object]] = [] - server._payout_artifact_log = ( # type: ignore[method-assign] - lambda event, **fields: build_logs.append( - {"event": event, **fields} - ) - ) - - with ( - patch( - "lab.prism.prism_coordinator.now_ms", - side_effect=lambda: clock_ms[0], - ), - patch( - "lab.prism.prism_coordinator.time.monotonic", - side_effect=lambda: monotonic_seconds[0], - ), - ): - published = server._current_payout_state_artifact() - initial = server._build_payout_ledger_artifact( - 0, - 0, - artifacts.network_difficulty, - ) - assert initial is not None - self.assertTrue(server._install_payout_ledger_artifact(initial)) - - ledger.balance_sats = 777 - append_incremental_share( - ledger, - share_seq=4, - accepted_at_ms=1_000_010, - ) - clock_ms[0] = 1_000_020 - monotonic_seconds[0] = 3_701.0 - checked = server._build_payout_ledger_artifact( - 0, - 0, - artifacts.network_difficulty, - ) - - with server._job_cache_lock: - self.assertIsNone(server._payout_ledger_artifact) - - # The mismatch build publishes the repaired balances for the same - # generation, so the next build resumes byte reuse from the - # repaired snapshot instead of paying another carry read against - # a still-stale publication. - clock_ms[0] = 1_000_030 - monotonic_seconds[0] = 3_702.0 - following = server._build_payout_ledger_artifact( - 0, - 0, - artifacts.network_difficulty, - ) - - assert checked is not None and following is not None - self.assertEqual(checked.window_build_mode, "self_check_match") - self.assertEqual(checked.prior_balances[0]["balance_sats"], 777) - self.assertEqual(following.prior_balances[0]["balance_sats"], 777) - self.assertEqual(ledger.prior_balance_reads, 2) - self.assertEqual(ledger.full_snapshot_calls, 2) - repaired = server._published_payout_state.artifact - assert repaired is not None - self.assertNotEqual( - repaired.prior_balances_sha256, - published.prior_balances_sha256, - ) - self.assertEqual( - repaired.prior_balances_sha256, - checked.prior_balances_sha256, - ) - self.assertIsNone( - server._payout_prior_balances_reuse_invalidated_sha256 - ) - self.assertEqual( - server.payout_artifact_event_counts["balance_check_mismatch"], - 1, - ) - self.assertEqual( - server.payout_artifact_event_counts["balance_repair_published"], - 1, - ) - self.assertEqual( - server.payout_artifact_event_counts["self_check_match"], - 1, - ) - built_sources = [ - entry["prior_balances_source"] - for entry in build_logs - if entry["event"] == "payout_artifact_built" - ] - self.assertEqual(built_sources, ["published", "ledger", "published"]) - mismatch_logs = [ - entry - for entry in build_logs - if entry["event"] == "payout_artifact_balance_check_mismatch" - ] - self.assertEqual(len(mismatch_logs), 1) - repair_logs = [ - entry - for entry in build_logs - if entry["event"] == "payout_artifact_balance_repair_published" - ] - self.assertEqual(len(repair_logs), 1) - - def test_balance_backstop_failure_retains_oracle_repaired_window( - self, - ) -> None: - # The share-window oracle and the carry-balance backstop are - # independent reads. A backstop failure after a successful oracle - # must not resurrect the delta window the oracle just refuted. - class FailingCarryOmittingDeltaLedger(IncrementalRecordingLedger): - def __init__(self) -> None: - super().__init__() - self.fail_balance_read = False - self.balance_read_attempts = 0 - - def snapshot_between_job_issues( - self, - previous_anchor_job_issued_at_ms: int, - anchor_job_issued_at_ms: int, - ) -> list[object]: - del previous_anchor_job_issued_at_ms, anchor_job_issued_at_ms - self.delta_snapshot_calls += 1 - return [] - - def current_prior_balances(self) -> list[dict[str, object]]: - self.balance_read_attempts += 1 - if self.fail_balance_read: - raise RuntimeError("carry aggregate unavailable") - return [ - { - "recipient_id": "carry", - "order_key": "01:carry", - "p2mr_program_hex": "66" * 32, - "balance_sats": 546, - } - ] - - ledger = FailingCarryOmittingDeltaLedger() - for share_seq in range(1, 4): - append_incremental_share( - ledger, - share_seq=share_seq, - accepted_at_ms=999_900 + share_seq, - ) - server, rpc = coordinator(ledger=ledger) - artifacts = server.store_template_artifacts(dict(rpc.template)) - assert artifacts is not None - server._pool_ready_latched = True - server.payout_artifact_min_build_interval_seconds = 0.0 - server.payout_artifact_full_rescan_seconds = 3_600.0 - clock_ms = [1_000_000] - monotonic_seconds = [100.0] - with ( - patch( - "lab.prism.prism_coordinator.now_ms", - side_effect=lambda: clock_ms[0], - ), - patch( - "lab.prism.prism_coordinator.time.monotonic", - side_effect=lambda: monotonic_seconds[0], - ), - ): - server._current_payout_state_artifact() - initial = server._build_payout_ledger_artifact( - 0, - 0, - artifacts.network_difficulty, - ) - assert initial is not None - self.assertTrue(server._install_payout_ledger_artifact(initial)) - - ledger.fail_balance_read = True - append_incremental_share( - ledger, - share_seq=4, - accepted_at_ms=1_000_010, - ) - clock_ms[0] = 1_000_020 - monotonic_seconds[0] = 3_701.0 - checked = server._build_payout_ledger_artifact( - 0, - 0, - artifacts.network_difficulty, - ) - - append_incremental_share( - ledger, - share_seq=5, - accepted_at_ms=1_000_021, - ) - clock_ms[0] = 1_000_030 - monotonic_seconds[0] = 3_702.0 - following = server._build_payout_ledger_artifact( - 0, - 0, - artifacts.network_difficulty, - ) - - assert checked is not None and following is not None - self.assertEqual(checked.window_build_mode, "self_check_mismatch") - self.assertEqual( - checked.window_full_rescan_reason, - "periodic_self_check_balance_check_failed", - ) - # The omitting delta produced three retained shares; the oracle's - # four-share window must be the one served and cached. - self.assertEqual(len(checked.shares_json), 4) - # One read published the priming state; the only other attempt is - # the backstop that failed. - self.assertEqual(ledger.balance_read_attempts, 2) - # The unverified balances keep reusing the published snapshot. - self.assertEqual( - checked.prior_balances_sha256, - initial.prior_balances_sha256, - ) - self.assertIsNone( - server._payout_prior_balances_reuse_invalidated_sha256 - ) - self.assertEqual( - server.payout_artifact_event_counts["self_check_mismatch"], - 1, - ) - self.assertEqual( - server.payout_artifact_event_counts["self_check_failed"], - 0, - ) - self.assertEqual( - server.payout_artifact_event_counts["balance_check_mismatch"], - 0, - ) - # The failed backstop still stamps the attempt: the next build stays - # on the delta path instead of retrying the oracle immediately. - self.assertEqual(following.window_build_mode, "incremental") - self.assertEqual(ledger.full_snapshot_calls, 2) - - def test_periodic_self_check_repair_arms_and_serves(self) -> None: - # The repaired publication is what lets the mismatch build's fresh - # balances actually reach miners: its artifact must arm against the - # repaired digest and the reuse probe must serve it, instead of the - # armed-artifact fence clearing the repair against a stale published - # snapshot until an unrelated generation change. - class DriftingCarryLedger(IncrementalRecordingLedger): - def __init__(self) -> None: - super().__init__() - self.balance_sats = 546 - - def current_prior_balances(self) -> list[dict[str, object]]: - return [ - { - "recipient_id": "carry", - "order_key": "01:carry", - "p2mr_program_hex": "66" * 32, - "balance_sats": self.balance_sats, - } - ] - - ledger = DriftingCarryLedger() - for share_seq in range(1, 4): - append_incremental_share( - ledger, - share_seq=share_seq, - accepted_at_ms=999_900 + share_seq, - ) - server, rpc = coordinator(ledger=ledger) - artifacts = server.store_template_artifacts(dict(rpc.template)) - assert artifacts is not None - server._pool_ready_latched = True - server.payout_artifact_min_build_interval_seconds = 0.0 - server.payout_artifact_full_rescan_seconds = 3_600.0 - clock_ms = [1_000_000] - monotonic_seconds = [100.0] - - with ( - patch( - "lab.prism.prism_coordinator.now_ms", - side_effect=lambda: clock_ms[0], - ), - patch( - "lab.prism.prism_coordinator.time.monotonic", - side_effect=lambda: monotonic_seconds[0], - ), - ): - server._current_payout_state_artifact() - initial = server._build_payout_ledger_artifact( - 0, 0, artifacts.network_difficulty - ) - assert initial is not None - self.assertTrue(server._install_payout_ledger_artifact(initial)) - - ledger.balance_sats = 777 - clock_ms[0] = 1_000_020 - monotonic_seconds[0] = 3_701.0 - checked = server._build_payout_ledger_artifact( - 0, 0, artifacts.network_difficulty - ) - assert checked is not None - self.assertEqual(checked.prior_balances[0]["balance_sats"], 777) - - self.assertTrue(server._install_payout_ledger_artifact(checked)) - served = server._usable_payout_ledger_artifact( - 0, - artifacts.network_difficulty, - ) - - assert served is not None - self.assertEqual(served.prior_balances[0]["balance_sats"], 777) - self.assertEqual( - served.prior_balances_sha256, - checked.prior_balances_sha256, - ) - with server._job_cache_lock: - self.assertIsNotNone(server._payout_ledger_artifact) - - def test_self_check_repair_supersedes_active_jobs(self) -> None: - # The in-place repair changes neither the payout generation nor the - # append epoch, so an already-stamped job passes every admission - # fence while carrying the refuted balances. The repair must - # schedule a refresh wave, and the wave's reselection must identify - # such jobs by their payout digest, or they stay mineable (and a - # block solve commits the stale allocation) until an unrelated - # generation change. - server, _ledger, _artifacts = self.configured_server() - install_fake_bundle_builder(server) - server._ensure_tip_refresh_state() - state = client(1) - with patch( - "lab.prism.prism_coordinator.now_ms", - side_effect=lambda: 1_000_000, - ): - bundle = server.build_shared_job_bundle( - server.current_template_artifacts(), - worker(), - ) - context = server.stamp_job_for_client( - state, - bundle, - clean_jobs=True, - ) - state.active_job = context - snapshot = SimpleNamespace( - bestblockhash=str(bundle.template["previousblockhash"]), - previousblockhash=str(bundle.template["previousblockhash"]), - template_fingerprint=bundle.template_fingerprint, - ) - with server._job_cache_lock: - published = server._published_payout_state.artifact - assert published is not None - self.assertEqual( - context.payout_artifact_sha256, - published.prior_balances_sha256, - ) - self.assertFalse( - server.client_needs_tip_template_refresh(state, snapshot) - ) - - self.assertTrue( - server._publish_self_check_repaired_balances( - 0, - stale_prior_balances_sha256=published.prior_balances_sha256, - balances=[ - { - "recipient_id": "carry", - "order_key": "01:carry", - "p2mr_program_hex": "66" * 32, - "balance_sats": 999, - } - ], - ) - ) - - self.assertTrue( - server.client_needs_tip_template_refresh(state, snapshot) - ) - self.assertTrue(server._tip_refresh_pending_event.is_set()) - self.assertTrue(server._tip_refresh_retry.is_set()) - - def test_self_check_repair_retires_refuted_jobs_clean(self) -> None: - # Reselection alone is not enough: the wave stamps the replacement - # with clean_jobs from client_tip_changed_for_snapshot, and a - # clean_jobs=False delivery leaves the refuted job ID admissible in - # active_job_ids. The digest mismatch must force clean-job - # retirement, or a late block submission on the old job still - # commits the refuted allocation after corrected work is delivered. - server, _ledger, _artifacts = self.configured_server() - install_fake_bundle_builder(server) - server._ensure_tip_refresh_state() - state = client(1) - with patch( - "lab.prism.prism_coordinator.now_ms", - side_effect=lambda: 1_000_000, - ): - bundle = server.build_shared_job_bundle( - server.current_template_artifacts(), - worker(), - ) - context = server.stamp_job_for_client( - state, - bundle, - clean_jobs=True, - ) - state.active_job = context - snapshot = SimpleNamespace( - bestblockhash=str(bundle.template["previousblockhash"]), - previousblockhash=str(bundle.template["previousblockhash"]), - template_fingerprint=bundle.template_fingerprint, - ) - with server._job_cache_lock: - published = server._published_payout_state.artifact - assert published is not None - self.assertFalse( - server.client_tip_changed_for_snapshot(state, snapshot) - ) - - self.assertTrue( - server._publish_self_check_repaired_balances( - 0, - stale_prior_balances_sha256=published.prior_balances_sha256, - balances=[ - { - "recipient_id": "carry", - "order_key": "01:carry", - "p2mr_program_hex": "66" * 32, - "balance_sats": 999, - } - ], - ) - ) - - self.assertTrue( - server.client_tip_changed_for_snapshot(state, snapshot) - ) - - def test_late_append_invalidation_supersedes_active_jobs(self) -> None: - # A late-visible append advances only the append epoch: the payout - # generation and published digest both survive, so an active job - # mining the pre-append window passes every other fence. The - # invalidation must schedule a refresh wave, the reselection must - # identify the job by its stamped epoch, and the replacement must - # retire it clean -- otherwise miners keep working (and can solve a - # block from) a window that omitted the late share until some - # unrelated tip event. - server, _ledger, _artifacts = self.configured_server() - install_fake_bundle_builder(server) - server._ensure_tip_refresh_state() - state = client(1) - with patch( - "lab.prism.prism_coordinator.now_ms", - side_effect=lambda: 1_000_000, - ): - bundle = server.build_shared_job_bundle( - server.current_template_artifacts(), - worker(), - ) - context = server.stamp_job_for_client( - state, - bundle, - clean_jobs=True, - ) - state.active_job = context - snapshot = SimpleNamespace( - bestblockhash=str(bundle.template["previousblockhash"]), - previousblockhash=str(bundle.template["previousblockhash"]), - template_fingerprint=bundle.template_fingerprint, - ) - self.assertFalse( - server.client_needs_tip_template_refresh(state, snapshot) - ) - self.assertFalse( - server.client_tip_changed_for_snapshot(state, snapshot) - ) - - with patch( - "lab.prism.prism_coordinator.now_ms", - side_effect=lambda: 1_000_000, - ): - token = server._expose_inflight_scan_anchor(999_999) - try: - server._invalidate_incremental_payout_window_for_append( - stamped_pending_share(999_950) - ) - finally: - server._retire_inflight_scan_anchor(token) - with server._job_cache_lock: - self.assertEqual( - server._payout_ledger_append_invalidation_epoch, 1 - ) - - self.assertTrue( - server.client_needs_tip_template_refresh(state, snapshot) - ) - self.assertTrue( - server.client_tip_changed_for_snapshot(state, snapshot) - ) - self.assertTrue(server._tip_refresh_pending_event.is_set()) - self.assertTrue(server._tip_refresh_retry.is_set()) - - def test_direct_delivery_rechecks_published_digest_at_commit(self) -> None: - # A periodic self-check repair swaps the published balance snapshot - # in place: neither the payout generation nor the append epoch moves, - # so only the digest identifies a direct-delivery context keyed to - # the refuted balances. When the repair lands after the build but - # before the commit boundary, the delivery must re-check the digest - # or the miner's active job carries the refuted allocation. - server, _ledger, _artifacts = self.configured_server() - install_fake_bundle_builder(server) - server._ensure_tip_refresh_state() - state = client(1) - sent: list[dict[str, object]] = [] - state.send = sent.append # type: ignore[method-assign] - original_build_job_for_client = server.build_job_for_client - repaired = False - - def repair_after_build(*args: object, **kwargs: object) -> object: - nonlocal repaired - context = original_build_job_for_client(*args, **kwargs) # type: ignore[arg-type] - if not repaired: - repaired = True - with server._job_cache_lock: - published = server._published_payout_state.artifact - assert published is not None - self.assertEqual( - context.payout_artifact_sha256, - published.prior_balances_sha256, - ) - self.assertTrue( - server._publish_self_check_repaired_balances( - 0, - stale_prior_balances_sha256=( - published.prior_balances_sha256 - ), - balances=[ - { - "recipient_id": "carry", - "order_key": "01:carry", - "p2mr_program_hex": "66" * 32, - "balance_sats": 999, - } - ], - ) - ) - return context - - server.build_job_for_client = repair_after_build # type: ignore[method-assign] - - with patch( - "lab.prism.prism_coordinator.now_ms", - side_effect=lambda: 1_000_000, - ): - self.assertFalse(server.maybe_send_job(state, clean_jobs=True)) - self.assertEqual(sent, []) - self.assertIsNone(state.active_job) - self.assertEqual(state.active_job_ids, set()) - self.assertEqual(server.jobs, {}) - - def test_incremental_and_forced_full_artifacts_match_with_carry_boundaries( - self, - ) -> None: - class CarryLedger(IncrementalRecordingLedger): - def current_prior_balances(self) -> list[dict[str, object]]: - return [ - { - "recipient_id": "below", - "order_key": "01:below", - "p2mr_program_hex": "11" * 32, - "balance_sats": 545, - }, - { - "recipient_id": "at", - "order_key": "02:at", - "p2mr_program_hex": "22" * 32, - "balance_sats": 546, - }, - { - "recipient_id": "above", - "order_key": "03:above", - "p2mr_program_hex": "33" * 32, - "balance_sats": 547, - }, - ] - - ledger = CarryLedger() - for share_seq in range(1, 4): - append_incremental_share( - ledger, - share_seq=share_seq, - accepted_at_ms=999_900 + share_seq, - ) - server, rpc = coordinator(ledger=ledger) - artifacts = server.store_template_artifacts(dict(rpc.template)) - assert artifacts is not None - server._pool_ready_latched = True - server.payout_artifact_min_build_interval_seconds = 0.0 - clock_ms = [1_000_000] - with patch( - "lab.prism.prism_coordinator.now_ms", - side_effect=lambda: clock_ms[0], - ): - self.assertIsNotNone( - server._build_payout_ledger_artifact( - 0, 0, artifacts.network_difficulty - ) - ) - append_incremental_share( - ledger, - share_seq=4, - accepted_at_ms=1_000_010, - ) - clock_ms[0] = 1_000_020 - incremental = server._build_payout_ledger_artifact( - 0, 0, artifacts.network_difficulty - ) - forced = server._build_payout_ledger_artifact( - 0, - 0, - artifacts.network_difficulty, - True, - ) - - assert incremental is not None and forced is not None - incremental_bytes = canonical_json_text( - { - "shares": list(incremental.shares_json), - "prior_balances": list(incremental.prior_balances), - } - ).encode() - forced_bytes = canonical_json_text( - { - "shares": list(forced.shares_json), - "prior_balances": list(forced.prior_balances), - } - ).encode() - self.assertEqual(incremental_bytes, forced_bytes) - self.assertEqual( - incremental.share_snapshot_sha256, - forced.share_snapshot_sha256, - ) - self.assertEqual( - incremental.prior_balances_sha256, - forced.prior_balances_sha256, - ) - - def test_normal_refresh_reuses_generation_owned_prior_balances(self) -> None: - class CountingCarryLedger(IncrementalRecordingLedger): - prior_balance_reads = 0 - - def current_prior_balances(self) -> list[dict[str, object]]: - self.prior_balance_reads += 1 - return [ - { - "recipient_id": "carry", - "order_key": "01:carry", - "p2mr_program_hex": "66" * 32, - "balance_sats": 545, - } - ] - - ledger = CountingCarryLedger() - for share_seq in range(1, 4): - append_incremental_share( - ledger, - share_seq=share_seq, - accepted_at_ms=999_900 + share_seq, - ) - server, rpc = coordinator(ledger=ledger) - artifacts = server.store_template_artifacts(dict(rpc.template)) - assert artifacts is not None - server._pool_ready_latched = True - server.payout_artifact_min_build_interval_seconds = 0.0 - clock_ms = [1_000_000] - with patch( - "lab.prism.prism_coordinator.now_ms", - side_effect=lambda: clock_ms[0], - ): - published = server._current_payout_state_artifact() - initial = server._build_payout_ledger_artifact( - 0, 0, artifacts.network_difficulty - ) - append_incremental_share( - ledger, - share_seq=4, - accepted_at_ms=1_000_010, - ) - clock_ms[0] = 1_000_020 - incremental = server._build_payout_ledger_artifact( - 0, 0, artifacts.network_difficulty - ) - forced = server._build_payout_ledger_artifact( - 0, - 0, - artifacts.network_difficulty, - True, - ) - - assert initial is not None and incremental is not None and forced is not None - self.assertEqual(ledger.prior_balance_reads, 2) - self.assertEqual( - initial.prior_balances_sha256, - published.prior_balances_sha256, - ) - self.assertEqual( - incremental.prior_balances_sha256, - published.prior_balances_sha256, - ) - - def test_fifteen_second_generation_bumps_refresh_once_per_minute(self) -> None: - server, ledger, artifacts = self.configured_server() - clock_ms = [1_000_000] - monotonic_seconds = [100.0] - modes: list[str | None] = [] - with ( - patch( - "lab.prism.prism_coordinator.now_ms", - side_effect=lambda: clock_ms[0], - ), - patch( - "lab.prism.prism_coordinator.time.monotonic", - side_effect=lambda: monotonic_seconds[0], - ), - ): - initial = server._build_payout_ledger_artifact( - 0, 0, artifacts.network_difficulty - ) - assert initial is not None - modes.append(initial.window_build_mode) - for bump in range(1, 5): - monotonic_seconds[0] = 100.0 + (15.0 * bump) - clock_ms[0] = 1_000_000 + (15_000 * bump) - append_incremental_share( - ledger, - share_seq=3 + bump, - accepted_at_ms=clock_ms[0] - 2, - ) - built = server._build_payout_ledger_artifact( - 0, 0, artifacts.network_difficulty - ) - assert built is not None - modes.append(built.window_build_mode) - - self.assertEqual( - modes, - ["full_rescan", "debounced", "debounced", "debounced", "incremental"], - ) - self.assertEqual(ledger.full_snapshot_calls, 1) - self.assertEqual(ledger.delta_snapshot_calls, 1) - - def test_found_block_candidate_bypasses_normal_interval(self) -> None: - server, ledger, artifacts = self.configured_server() - clock_ms = [1_000_000] - with patch( - "lab.prism.prism_coordinator.now_ms", - side_effect=lambda: clock_ms[0], - ): - self.assertIsNotNone( - server._build_payout_ledger_artifact( - 0, 0, artifacts.network_difficulty - ) - ) - append_incremental_share( - ledger, - share_seq=4, - accepted_at_ms=1_000_010, - ) - # Even an overdue routine oracle must be deferred from the - # immediate found-block publication path. - server.payout_artifact_full_rescan_seconds = 0.0 - clock_ms[0] = 1_000_020 - candidate = server._prepared_payout_state_candidate( - ( - 0, - 1, - str(server.rpc.tip), - "accepted_block_preview", - time.monotonic(), - ) - ) - candidate = server._accepted_block_preview_candidate( - candidate, - block_hash="44" * 32, - preview=(("winner", "01:winner", "55" * 32, 546),), - ) - - artifact = candidate.ledger_artifact - assert artifact is not None - self.assertEqual(artifact.window_build_mode, "incremental") - self.assertEqual(artifact.window_delta_rows, 1) - self.assertEqual(ledger.delta_snapshot_calls, 1) - self.assertEqual(ledger.full_snapshot_calls, 1) - self.assertEqual(len(artifact.shares_json), 4) - - def test_failed_periodic_oracle_is_spaced_from_next_delta_build(self) -> None: - class FailingOracleLedger(IncrementalRecordingLedger): - fail_full_snapshot = False - - def snapshot_at_job_issue( - self, - anchor_job_issued_at_ms: int, - *, - window_weight: int | None = None, - ) -> list[object]: - if self.fail_full_snapshot: - self.full_snapshot_calls += 1 - raise RuntimeError("oracle unavailable") - return super().snapshot_at_job_issue( - anchor_job_issued_at_ms, - window_weight=window_weight, - ) - - ledger = FailingOracleLedger() - for share_seq in range(1, 4): - append_incremental_share( - ledger, - share_seq=share_seq, - accepted_at_ms=999_900 + share_seq, - ) - server, rpc = coordinator(ledger=ledger) - artifacts = server.store_template_artifacts(dict(rpc.template)) - assert artifacts is not None - server._pool_ready_latched = True - server.payout_artifact_min_build_interval_seconds = 0.0 - server.payout_artifact_full_rescan_seconds = 10.0 - clock_ms = [1_000_000] - monotonic_seconds = [100.0] - with ( - patch( - "lab.prism.prism_coordinator.now_ms", - side_effect=lambda: clock_ms[0], - ), - patch( - "lab.prism.prism_coordinator.time.monotonic", - side_effect=lambda: monotonic_seconds[0], - ), - ): - self.assertIsNotNone( - server._build_payout_ledger_artifact( - 0, 0, artifacts.network_difficulty - ) - ) - ledger.fail_full_snapshot = True - append_incremental_share( - ledger, - share_seq=4, - accepted_at_ms=1_000_010, - ) - clock_ms[0] = 1_000_020 - monotonic_seconds[0] = 110.0 - failed_check = server._build_payout_ledger_artifact( - 0, 0, artifacts.network_difficulty - ) - assert failed_check is not None - self.assertEqual( - failed_check.window_build_mode, - "incremental_self_check_failed", - ) - - append_incremental_share( - ledger, - share_seq=5, - accepted_at_ms=1_000_021, - ) - clock_ms[0] = 1_000_030 - monotonic_seconds[0] = 111.0 - next_delta = server._build_payout_ledger_artifact( - 0, 0, artifacts.network_difficulty - ) - - assert next_delta is not None - self.assertEqual(next_delta.window_build_mode, "incremental") - self.assertEqual(ledger.full_snapshot_calls, 2) - - def test_found_block_reuses_armed_window_when_prepare_lock_is_busy( - self, - ) -> None: - class BusyPrepareLock: - def acquire(self, blocking: bool = True) -> bool: - self.assert_nonblocking(blocking) - return False - - @staticmethod - def assert_nonblocking(blocking: bool) -> None: - if blocking: - raise AssertionError("found-block path attempted to wait") - - def release(self) -> None: - raise AssertionError("unacquired lock was released") - - server, ledger, artifacts = self.configured_server() - clock_ms = [1_000_000] - with patch( - "lab.prism.prism_coordinator.now_ms", - side_effect=lambda: clock_ms[0], - ): - initial = server._build_payout_ledger_artifact( - 0, 0, artifacts.network_difficulty - ) - assert initial is not None - self.assertTrue(server._install_payout_ledger_artifact(initial)) - server._payout_state_prepare_lock = BusyPrepareLock() # type: ignore[assignment] - candidate = server._prepared_payout_state_candidate( - ( - 0, - 1, - str(server.rpc.tip), - "accepted_block_preview", - time.monotonic(), - ) - ) - candidate = server._accepted_block_preview_candidate( - candidate, - block_hash="44" * 32, - preview=(("winner", "01:winner", "55" * 32, 546),), - ) - - artifact = candidate.ledger_artifact - assert artifact is not None - self.assertEqual(artifact.window_build_mode, "found_block_cached") - self.assertEqual(artifact.payout_state_generation, 1) - self.assertEqual( - list(artifact.prior_balances), - [ - { - "recipient_id": "winner", - "order_key": "01:winner", - "p2mr_program_hex": "55" * 32, - "balance_sats": 546, - } - ], - ) - self.assertEqual(ledger.full_snapshot_calls, 1) - self.assertEqual(ledger.delta_snapshot_calls, 0) - self.assertEqual( - server.payout_artifact_event_counts["found_block_cached"], - 1, - ) - - def test_found_block_build_failure_is_visible_and_reuses_exact_window( - self, - ) -> None: - class FailingDeltaLedger(IncrementalRecordingLedger): - fail_delta = False - - def snapshot_between_job_issues( - self, - previous_anchor_job_issued_at_ms: int, - anchor_job_issued_at_ms: int, - ) -> list[object]: - if self.fail_delta: - self.delta_snapshot_calls += 1 - raise RuntimeError("delta unavailable") - return super().snapshot_between_job_issues( - previous_anchor_job_issued_at_ms, - anchor_job_issued_at_ms, - ) - - ledger = FailingDeltaLedger() - for share_seq in range(1, 4): - append_incremental_share( - ledger, - share_seq=share_seq, - accepted_at_ms=999_900 + share_seq, - ) - server, rpc = coordinator(ledger=ledger) - artifacts = server.store_template_artifacts(dict(rpc.template)) - assert artifacts is not None - server._pool_ready_latched = True - clock_ms = [1_000_000] - with patch( - "lab.prism.prism_coordinator.now_ms", - side_effect=lambda: clock_ms[0], - ): - initial = server._build_payout_ledger_artifact( - 0, 0, artifacts.network_difficulty - ) - assert initial is not None - self.assertTrue(server._install_payout_ledger_artifact(initial)) - ledger.fail_delta = True - clock_ms[0] = 1_000_020 - candidate = server._prepared_payout_state_candidate( - ( - 0, - 1, - str(server.rpc.tip), - "accepted_block_preview", - time.monotonic(), - ) - ) - - artifact = candidate.ledger_artifact - assert artifact is not None - self.assertEqual(artifact.window_build_mode, "found_block_cached") - self.assertEqual( - server.payout_artifact_event_counts["build_aborted"], - 1, - ) - self.assertEqual( - server.payout_artifact_event_counts["found_block_cached"], - 1, - ) - - def test_old_anchor_does_not_rearm_before_window_cadence(self) -> None: - class RecordingExecutor: - def __init__(self) -> None: - self.submissions = 0 - - def submit(self, _fn: object) -> Future[None]: - self.submissions += 1 - return Future() - - server, _ledger, artifacts = self.configured_server() - clock_ms = [1_000_000] - monotonic_seconds = [100.0] - with ( - patch( - "lab.prism.prism_coordinator.now_ms", - side_effect=lambda: clock_ms[0], - ), - patch( - "lab.prism.prism_coordinator.time.monotonic", - side_effect=lambda: monotonic_seconds[0], - ), - ): - initial = server._build_payout_ledger_artifact( - 0, 0, artifacts.network_difficulty - ) - assert initial is not None - self.assertTrue(server._install_payout_ledger_artifact(initial)) - executor = RecordingExecutor() - with server._payout_artifact_executor_lock: - server._payout_artifact_executor = executor # type: ignore[assignment] - - # A pending-share floor can leave the declared anchor old even - # though its exact pages were just re-proved. Past-floor probes - # must not enqueue five-second debounced balance-only builds. - clock_ms[0] = 1_061_000 - monotonic_seconds[0] = 105.0 - self.assertIsNotNone( - server._usable_payout_ledger_artifact( - 0, - artifacts.network_difficulty, - ) - ) - with server._payout_artifact_executor_lock: - self.assertIsNone(server._payout_artifact_requested) - - monotonic_seconds[0] = 160.0 - self.assertIsNotNone( - server._usable_payout_ledger_artifact( - 0, - artifacts.network_difficulty, - ) - ) - - with server._payout_artifact_executor_lock: - self.assertEqual( - server._payout_artifact_requested, - (0, int(artifacts.network_difficulty)), - ) - self.assertEqual(executor.submissions, 1) - - -def mark_progress_healthy(server: PrismCoordinator) -> None: - snapshot = server.fetch_qbit_tip_template_snapshot() - server._record_progress_tip_poll(snapshot) - server._record_progress_publication( - snapshot, - int(getattr(server, "_payout_state_generation", 0)), - ) - - -class JobBundleCacheTests(unittest.TestCase): - def test_tip_template_snapshot_stays_coherent_across_tip_transition(self) -> None: - old_tip = "11" * 32 - new_tip = "22" * 32 - server, rpc = coordinator(template=base_template(prevhash=old_tip)) - new_template = base_template(height=11, prevhash=new_tip) - original_call = rpc.call - - def transition_during_template_fetch( - method: str, - params: list[object] | None = None, - ) -> object: - if method == "getblocktemplate": - rpc.tip = new_tip - rpc.template = new_template - return original_call(method, params) - - rpc.call = transition_during_template_fetch # type: ignore[method-assign] - - snapshot = server.fetch_qbit_tip_template_snapshot() - - self.assertEqual(snapshot.bestblockhash, new_tip) - self.assertEqual(snapshot.previousblockhash, new_tip) - self.assertEqual( - snapshot.template_fingerprint, - qbit_template_fingerprint(new_template), - ) - self.assertEqual(rpc.calls[:2], ["getblocktemplate", "getbestblockhash"]) - - def test_one_heavy_build_shared_across_clients_with_per_client_stamping(self) -> None: - server, rpc = coordinator() - recorded = install_fake_bundle_builder(server) - clients = [client(1), client(2), client(3)] - - contexts = [server.build_job_for_client(c, clean_jobs=True) for c in clients] - - self.assertEqual(recorded["calls"], 1) - self.assertEqual(rpc.count("getblocktemplate"), 1) - self.assertEqual(server.ledger.snapshot_calls, 1) - # The heavy build uses the placeholder extranonce1, never a client's. - self.assertEqual( - recorded["suffixes"], - [ - server.coinbase_tag_hex - + PRISM_JOB_EXTRANONCE1_PLACEHOLDER_HEX - + "00" * EXTRANONCE2_SIZE - ], - ) - job_ids = {context.job.job_id for context in contexts} - self.assertEqual(len(job_ids), 3) - self.assertEqual( - [context.job.extranonce1_hex for context in contexts], - [c.extranonce1_hex for c in clients], - ) - # coinb1/coinb2 exclude the extranonce window entirely, so the shared - # split is byte-identical for every client. - self.assertEqual(len({context.job.coinb1 for context in contexts}), 1) - self.assertEqual(len({context.job.coinb2 for context in contexts}), 1) - self.assertTrue(all(not hasattr(context, "bundle") for context in contexts)) - self.assertTrue( - all(context.prospective_prior_balances == () for context in contexts) - ) - cached = next(iter(server._job_bundle_cache.values())) - self.assertFalse(hasattr(cached, "bundle")) - self.assertEqual(cached.prospective_prior_balances, ()) - self.assertEqual( - cached.coinbase_manifest["coinbase_tx_hex"], - synthetic_manifest_coinbase_hex(recorded["suffixes"][0]), - ) - self.assertIs(contexts[0].shares_json, contexts[1].shares_json) - - def test_latest_wins_scheduler_preserves_synchronous_builder_output(self) -> None: - server, rpc = coordinator() - recorded = install_fake_bundle_builder(server) - artifacts = server.store_template_artifacts(dict(rpc.template)) - assert artifacts is not None - identity = worker() - cache_key = server._job_bundle_key( - artifacts, - mode="ready", - payout_state_generation=0, - worker=identity, - ) - - with patch("lab.prism.prism_coordinator.now_ms", return_value=1_700_000_001_000): - direct_request = server._new_job_build_request( - artifacts, - identity, - mode="ready", - payout_state_generation=0, - cache_key=cache_key, - ) - direct = server.build_shared_job_bundle( - artifacts, - identity, - mode="ready", - payout_state_generation=0, - key=cache_key, - build_request=direct_request, - ) - scheduled_request = server._new_job_build_request( - artifacts, - identity, - mode="ready", - payout_state_generation=0, - cache_key=cache_key, - ) - scheduled = server._request_job_build(scheduled_request).result(5) - - server.shutdown_job_build_executor() - self.assertEqual(recorded["calls"], 2) - self.assertEqual(scheduled.key, direct.key) - self.assertEqual(scheduled.template, direct.template) - self.assertEqual(scheduled.template_fingerprint, direct.template_fingerprint) - self.assertEqual(scheduled.coinbase_manifest, direct.coinbase_manifest) - self.assertEqual(scheduled.shares_json, direct.shares_json) - self.assertEqual(scheduled.prior_balances, direct.prior_balances) - self.assertEqual(scheduled.found_block, direct.found_block) - self.assertEqual(scheduled.collection_only, direct.collection_only) - self.assertEqual(scheduled.issued_at_ms, direct.issued_at_ms) - self.assertEqual(scheduled.base_job, direct.base_job) - self.assertEqual(scheduled.build_key, direct.build_key) - - def test_stamped_job_reassembles_coinbase_with_client_extranonce(self) -> None: - server, _ = coordinator() - install_fake_bundle_builder(server) - state = client(0x2A) - - context = server.build_job_for_client(state, clean_jobs=True) - extranonce2_hex = "11" * EXTRANONCE2_SIZE - submission = direct_stratum.assemble_submission( - context.job, - extranonce2_hex=extranonce2_hex, - ntime_hex="65000000", - nonce_hex="00000001", - ) - - expected_suffix = server.coinbase_tag_hex + state.extranonce1_hex + extranonce2_hex - coinbase = bytes.fromhex(submission.coinbase_tx_hex) - script_start, script_len = direct_stratum.coinbase_scriptsig_span( - coinbase, field_name="stamped coinbase" - ) - script_sig_hex = coinbase[script_start : script_start + script_len].hex() - self.assertTrue(script_sig_hex.endswith(expected_suffix)) - self.assertNotIn(PRISM_JOB_EXTRANONCE1_PLACEHOLDER_HEX, script_sig_hex[len("03aabbcc") :]) - - def test_template_fingerprint_change_invalidates_bundle_cache(self) -> None: - server, rpc = coordinator() - recorded = install_fake_bundle_builder(server) - server.build_job_for_client(client(1), clean_jobs=True) - - new_template = base_template(height=11, prevhash="22" * 32) - rpc.template = new_template - rpc.tip = str(new_template["previousblockhash"]) - server.store_template_artifacts(dict(new_template)) - - context = server.build_job_for_client(client(2), clean_jobs=True) - - self.assertEqual(recorded["calls"], 2) - self.assertEqual(context.template_fingerprint, qbit_template_fingerprint(new_template)) - # Bundles for the old fingerprint are evicted. - self.assertEqual( - {entry.template_fingerprint for entry in server._job_bundle_cache.values()}, - {qbit_template_fingerprint(new_template)}, - ) - - def test_bundle_cache_ttl_expiry_rebuilds(self) -> None: - server, _ = coordinator() - recorded = install_fake_bundle_builder(server) - server.job_bundle_cache_seconds = 0.05 - - server.build_job_for_client(client(1), clean_jobs=True) - time.sleep(0.06) - server.build_job_for_client(client(2), clean_jobs=True) - - self.assertEqual(recorded["calls"], 2) - - def test_bundle_cache_lookup_prunes_every_expired_snapshot(self) -> None: - server, _ = coordinator() - install_fake_bundle_builder(server) - artifacts = server.store_template_artifacts(base_template()) - assert artifacts is not None - current = server.shared_job_bundle(artifacts, worker()) - expired_key = ("expired-template", "ready") - expired = dataclass_replace( - current, - key=expired_key, - template_fingerprint="expired-template", - built_monotonic=time.monotonic() - 60, - ) - server._job_bundle_cache[expired_key] = expired - - looked_up = server._lookup_job_bundle(current.key) - - self.assertIs(looked_up, current) - self.assertNotIn(expired_key, server._job_bundle_cache) - self.assertEqual(list(server._job_bundle_cache.values()), [current]) - - def test_zero_ttl_disables_bundle_cache(self) -> None: - server, _ = coordinator() - recorded = install_fake_bundle_builder(server) - server.job_bundle_cache_seconds = 0.0 - - server.build_job_for_client(client(1), clean_jobs=True) - server.build_job_for_client(client(2), clean_jobs=True) - - self.assertEqual(recorded["calls"], 2) - - def test_payout_state_change_during_build_retries_before_cache_or_return(self) -> None: - server, rpc = coordinator() - recorded = install_fake_bundle_builder(server) - artifacts = server.store_template_artifacts(dict(rpc.template)) - self.assertIsNotNone(artifacts) - assert artifacts is not None - identity = worker() - original_build = server.build_shared_job_bundle - built_generations: list[int] = [] - - def mutate_after_first_build( - build_artifacts: object, - build_worker: WorkerIdentity | None, - **kwargs: object, - ) -> object: - bundle = original_build( # type: ignore[arg-type] - build_artifacts, - build_worker, - **kwargs, - ) - built_generations.append(bundle.payout_state_generation) - if len(built_generations) == 1: - server._advance_payout_state_generation() - return bundle - - server.build_shared_job_bundle = mutate_after_first_build # type: ignore[method-assign] - - bundle = server.shared_job_bundle(artifacts, identity) - cached = server.shared_job_bundle(artifacts, identity) - - self.assertEqual(built_generations, [0, 1]) - self.assertEqual(recorded["calls"], 2) - self.assertEqual(bundle.payout_state_generation, 1) - self.assertIs(cached, bundle) - - def test_child_bundle_waits_for_pending_parent_preview_without_confirmed_read( - self, - ) -> None: - class PreviewLedger(FakeLedger): - def __init__(self) -> None: - super().__init__() - self.current_balance_reads = 0 - - def current_prior_balances(self) -> list[dict[str, object]]: - self.current_balance_reads += 1 - return [] - - class ObservedCondition(threading.Condition): - def __init__(self) -> None: - super().__init__() - self.wait_entered = threading.Event() - - def wait(self, timeout: float | None = None) -> bool: - self.wait_entered.set() - return super().wait(timeout) - - ledger = PreviewLedger() - server, rpc = coordinator(ledger=ledger) - recorded = install_fake_bundle_builder(server) - artifacts = server.store_template_artifacts(dict(rpc.template)) - self.assertIsNotNone(artifacts) - assert artifacts is not None - parent_hash = str(rpc.template["previousblockhash"]) - preview_condition = ObservedCondition() - server._accepted_block_payout_preview_condition = preview_condition - server.accepted_block_payout_preview_wait_seconds = 10 - preview = [ - { - "recipient_id": "miner-a", - "order_key": "miner-a", - "p2mr_program_hex": "11" * 32, - "balance_sats": 25, - } - ] - bundles: list[object] = [] - errors: list[BaseException] = [] - - def build_child_bundle() -> None: - try: - bundles.append(server.shared_job_bundle(artifacts, worker())) - except BaseException as exc: # pragma: no cover - asserted below - errors.append(exc) - - server._begin_accepted_block_payout_preview(parent_hash) - thread = threading.Thread(target=build_child_bundle, daemon=True) - thread.start() - try: - preview_wait_reached = preview_condition.wait_entered.wait(2) - waiting_for_preview = thread.is_alive() - confirmed_reads_while_pending = ledger.current_balance_reads - builds_while_pending = int(recorded["calls"]) - finally: - server._publish_accepted_block_payout_preview(parent_hash, preview) - thread.join(5) - server._clear_accepted_block_payout_preview(parent_hash) - - self.assertTrue(preview_wait_reached) - self.assertTrue(waiting_for_preview) - self.assertEqual(confirmed_reads_while_pending, 0) - self.assertEqual(builds_while_pending, 0) - self.assertFalse(thread.is_alive()) - self.assertEqual(errors, []) - self.assertEqual(len(bundles), 1) - bundle = bundles[0] - self.assertEqual(bundle.prior_balances, preview) # type: ignore[union-attr] - self.assertEqual(recorded["last_kwargs"]["prior_balances"], preview) # type: ignore[index] - # Preview publication prepares an artifact from one confirmed snapshot; - # the zero count captured above proves the waiting child did not read it. - self.assertEqual(ledger.current_balance_reads, 1) - self.assertEqual( # type: ignore[union-attr] - bundle.payout_state_generation, - server._payout_state_generation, - ) - - self.assertEqual(server._prior_balances_for_job_parent(parent_hash), []) - self.assertEqual(ledger.current_balance_reads, 2) - - def test_parent_preview_publication_is_idempotent_and_withdrawal_invalidates( - self, - ) -> None: - server, _rpc = coordinator() - parent_hash = "ab" * 32 - preview = [ - { - "recipient_id": "miner-a", - "order_key": "miner-a", - "p2mr_program_hex": "11" * 32, - "balance_sats": 25, - } - ] - - server._begin_accepted_block_payout_preview(parent_hash) - self.assertEqual( - server._publish_accepted_block_payout_preview(parent_hash, preview), - preview, - ) - self.assertEqual(server._payout_state_generation, 1) - - self.assertEqual( - server._publish_accepted_block_payout_preview(parent_hash, preview), - preview, - ) - self.assertEqual(server._payout_state_generation, 1) - with self.assertRaisesRegex(RuntimeError, "changed during retry"): - server._publish_accepted_block_payout_preview( - parent_hash, - [{**preview[0], "balance_sats": 26}], - ) - - server._clear_accepted_block_payout_preview( - parent_hash, - invalidate_published=True, - ) - self.assertEqual(server._payout_state_generation, 2) - self.assertEqual(server._accepted_block_payout_previews, {}) - self.assertEqual( - server._invalidated_accepted_block_payout_previews, - {parent_hash: None}, - ) - with self.assertRaisesRegex(TemplateRefreshBlocked, "was withdrawn"): - server._prior_balances_for_job_parent(parent_hash) - - server._begin_accepted_block_payout_preview(parent_hash) - self.assertEqual(server._invalidated_accepted_block_payout_previews, {}) - server._clear_accepted_block_payout_preview(parent_hash) - - def test_unpublished_parent_preview_retries_and_reopens_delivery(self) -> None: - server, _rpc = coordinator() - server.payout_reconcile_supersession_retries = 2 - parent_hash = "ac" * 32 - preview = [ - { - "recipient_id": "miner-a", - "order_key": "miner-a", - "p2mr_program_hex": "11" * 32, - "balance_sats": 25, - } - ] - - server._begin_accepted_block_payout_preview(parent_hash) - with patch.object( - server, - "_publish_payout_state_candidate", - return_value=None, - ) as publish_candidate: - self.assertEqual( - server._publish_accepted_block_payout_preview(parent_hash, preview), - preview, - ) - - transition = server._accepted_block_payout_previews[parent_hash] - self.assertEqual(publish_candidate.call_count, 3) - self.assertIsNotNone(transition.preview) - self.assertIsNone(transition.published_generation) - self.assertEqual(server._payout_state_generation, 0) - self.assertTrue(server._payout_state_publication_fenced()) - self.assertTrue(server._payout_state_delivery_gate._delivery_blocked) - - self.assertEqual( - server._publish_accepted_block_payout_preview(parent_hash, preview), - preview, - ) - - transition = server._accepted_block_payout_previews[parent_hash] - self.assertEqual(transition.published_generation, 1) - self.assertEqual(server._payout_state_generation, 1) - self.assertFalse(server._payout_state_publication_fenced()) - self.assertFalse(server._payout_state_delivery_gate._delivery_blocked) - - def test_preview_publication_degrades_when_artifact_preparation_times_out( - self, - ) -> None: - server, _rpc = coordinator() - parent_hash = "ab" * 32 - preview = [ - { - "recipient_id": "miner-a", - "order_key": "miner-a", - "p2mr_program_hex": "11" * 32, - "balance_sats": 25, - } - ] - - def timing_out_preparation(*_args: object, **_kwargs: object) -> object: - raise TimeoutError("postgres operation exceeded 30s") - - server._prepared_payout_state_candidate = ( # type: ignore[method-assign] - timing_out_preparation - ) - server._begin_accepted_block_payout_preview(parent_hash) - with patch("builtins.print"): - self.assertEqual( - server._publish_accepted_block_payout_preview(parent_hash, preview), - preview, - ) - - # Children of the pending parent receive the immutable compact - # preview even though no payout-state generation was published. - transition = server._accepted_block_payout_previews[parent_hash] - self.assertIsNone(transition.published_generation) - self.assertTrue(transition.landed) - self.assertEqual( - server._prior_balances_for_job_parent(parent_hash), - preview, - ) - # Payout-state publication itself stays gated until the landing - # reaches a terminal durable state. - with self.assertRaisesRegex( - _PayoutStatePublicationBlocked, "still pending" - ): - with server._payout_balance_mutation(): - pass - server._clear_accepted_block_payout_preview(parent_hash) - - def test_unresolved_accepted_parent_depth_cap_fails_closed(self) -> None: - server, _rpc = coordinator() - server.accepted_parent_unresolved_depth_max = 2 - server.accepted_block_payout_preview_wait_seconds = 0.01 - hashes = ["a1" * 32, "a2" * 32, "a3" * 32] - for height, block_hash in enumerate(hashes, start=10): - server._begin_accepted_block_payout_preview( - block_hash, block_height=height - ) - server._mark_accepted_block_payout_landed( - block_hash, block_height=height - ) - self.assertEqual(len(server.accepted_parent_unresolved_ages_seconds()), 3) - with self.assertRaisesRegex(TemplateRefreshBlocked, "exceeds cap"): - server._await_pending_parent_payout_preview(hashes[0]) - # At the cap issuance still blocks: a child issued now could land - # and create a (cap + 1)th unresolved transition, so the configured - # maximum would no longer bound the prospective balance chain. - server._clear_accepted_block_payout_preview(hashes[2]) - with self.assertRaisesRegex(TemplateRefreshBlocked, "exceeds cap"): - server._await_pending_parent_payout_preview(hashes[0]) - # Below the cap the ordinary bounded wait applies instead. - server._clear_accepted_block_payout_preview(hashes[1]) - with self.assertRaisesRegex(TemplateRefreshBlocked, "not ready yet"): - server._await_pending_parent_payout_preview(hashes[0]) - server._clear_accepted_block_payout_preview(hashes[0]) - self.assertEqual(server.accepted_parent_unresolved_ages_seconds(), []) - - def test_landing_observability_metrics_exposition(self) -> None: - import contextlib as _contextlib - import time as _time - - server, _rpc = coordinator() - server.block_landing_db_timeout_seconds = 45.0 - server.block_landing_db_timeout_max_seconds = 90.0 - - @_contextlib.contextmanager - def statement_timeout(seconds: float): - yield - - original_ledger = server.ledger - server.ledger = SimpleNamespace( - statement_timeout=statement_timeout, - prior_balances_read_stats=lambda: { - "reads_total": 3, - "last_seconds": 0.004, - "max_seconds": 0.021, - }, - ) - block_hash = "ab" * 32 - with self.assertRaises(TimeoutError): - with server._block_landing_ledger_statement_timeout_scope(block_hash): - raise LedgerOperationTimeout("postgres operation exceeded 45s") - server.ledger = original_ledger - - server._begin_accepted_block_payout_preview(block_hash, block_height=10) - server._mark_accepted_block_payout_landed(block_hash, block_height=10) - server.accepted_block_payout_preview_wait_seconds = 0.0 - with self.assertRaisesRegex(TemplateRefreshBlocked, "not ready yet"): - server._await_pending_parent_payout_preview(block_hash) - server._startup_phase_origin_monotonic = _time.monotonic() - server._record_startup_phase_once("audit_listener_bound") - - prior_stats = { - "reads_total": 3, - "last_seconds": 0.004, - "max_seconds": 0.021, - } - server.ledger.prior_balances_read_stats = ( # type: ignore[attr-defined] - lambda: prior_stats - ) - lines = "\n".join(server.landing_observability_metrics_lines()) - self.assertIn( - 'qbit_prism_block_ledger_call_timeouts_total{call_class="landing"} 1', - lines, - ) - self.assertIn( - 'qbit_prism_block_ledger_call_budget_seconds{call_class="landing"} 45.000000', - lines, - ) - self.assertIn( - "qbit_prism_accepted_parent_unresolved_transitions 1", - lines, - ) - self.assertIn( - "qbit_prism_accepted_parent_preview_wait_timeouts_total 1", - lines, - ) - self.assertNotIn( - "qbit_prism_accepted_parent_unresolved_oldest_seconds -1", - lines, - ) - self.assertIn("qbit_prism_prior_balances_reads_total 3", lines) - self.assertIn( - "qbit_prism_prior_balances_read_max_seconds 0.021000", - lines, - ) - self.assertIn( - 'qbit_prism_startup_phase_seconds{phase="audit_listener_bound"}', - lines, - ) - server._clear_accepted_block_payout_preview(block_hash) - - def test_startup_replay_gate_blocks_job_builds_until_enumeration(self) -> None: - server, _rpc = coordinator() - enumerated: list[int] = [] - - class EnumLedger(FakeLedger): - def pending_block_candidate_rows( - self, *, limit: int = 32 - ) -> list[dict[str, object]]: - enumerated.append(limit) - return [] - - server.ledger = EnumLedger() - server._note_block_replay_enumeration_owed() - with self.assertRaisesRegex(TemplateRefreshBlocked, "has not enumerated"): - server._await_pending_parent_payout_preview("ab" * 32) - self.assertEqual(server.replay_pending_block_candidates(), 0) - self.assertEqual(enumerated, [32]) - self.assertFalse(server._block_replay_enumeration_owed()) - self.assertIsNone(server._await_pending_parent_payout_preview("ab" * 32)) - - def test_owed_enumeration_bypasses_live_candidate_short_circuits(self) -> None: - server, _rpc = coordinator() - enumerated: list[int] = [] - - class EnumLedger(FakeLedger): - def pending_block_candidate_rows( - self, *, limit: int = 32 - ) -> list[dict[str, object]]: - enumerated.append(limit) - return [] - - server.ledger = EnumLedger() - server.block_landing_db_timeout_seconds = 7.5 - server._retry_block_candidate = object() - # Steady state: a retained retry short-circuits the outbox poll. - self.assertEqual(server.replay_pending_block_candidates(), 0) - self.assertEqual(enumerated, []) - # Owed enumeration is correctness-critical and bypasses the - # short-circuits, running with the landing-class budget and - # recording under the landing metrics class. - server._note_block_replay_enumeration_owed() - self.assertEqual(server.replay_pending_block_candidates(), 0) - self.assertEqual(enumerated, [32]) - self.assertFalse(server._block_replay_enumeration_owed()) - metrics = server.block_ledger_call_class_metrics() - self.assertEqual(metrics["landing"]["last_budget_seconds"], 7.5) - self.assertNotIn("fast", metrics) - - def test_worker_reported_deadline_counts_as_landing_timeout(self) -> None: - """A server-side cancellation completes the ledger worker with a - timeout error before the coordinator-side wait expires; the - completion path must still count it for the landing-timeout alert.""" - server, _rpc = coordinator() - - class CancelledLedger(FakeLedger): - def pending_block_candidate_rows( - self, *, limit: int = 32 - ) -> list[dict[str, object]]: - raise LedgerOperationTimeout("postgres statement deadline expired") - - server.ledger = CancelledLedger() - server._note_block_replay_enumeration_owed() - with patch("builtins.print"): - self.assertTrue(server._run_startup_block_candidate_replay()) - self.assertTrue(server._block_replay_enumeration_owed()) - metrics = server.block_ledger_call_class_metrics() - self.assertEqual(metrics["landing"]["calls_total"], 1) - self.assertEqual(metrics["landing"]["timeouts_total"], 1) - self.assertNotIn("fast", metrics) - - def test_startup_enumeration_escalates_past_truncated_batches(self) -> None: - """A full first batch must not end enumeration: rows beyond the batch - window (potentially the active parent) still need their payout - barriers armed before job builds unblock.""" - server, _rpc = coordinator() - requested: list[int] = [] - total_pending = 33 - - class TruncatingLedger(FakeLedger): - def pending_block_candidate_rows( - self, *, limit: int = 32 - ) -> list[dict[str, object]]: - requested.append(limit) - return [ - durable_candidate_row(index) - for index in range(min(limit, total_pending)) - ] - - server.ledger = TruncatingLedger() - server._note_block_replay_enumeration_owed() - self.assertEqual(server.replay_pending_block_candidates(), total_pending) - self.assertEqual(requested, [32, 64]) - self.assertFalse(server._block_replay_enumeration_owed()) - self.assertEqual( - server._block_replay_candidate_queue.qsize(), total_pending - ) - # Every restored row armed its payout barrier before the gate opened. - self.assertEqual( - len(server._accepted_block_payout_previews), total_pending - ) - - def test_startup_enumeration_stays_owed_at_truncation_cap(self) -> None: - """An outbox larger than the enumeration cap keeps job builds blocked - instead of silently declaring enumeration complete.""" - server, _rpc = coordinator() - requested: list[int] = [] - - class EndlessLedger(FakeLedger): - def pending_block_candidate_rows( - self, *, limit: int = 32 - ) -> list[dict[str, object]]: - requested.append(limit) - return [durable_candidate_row(index) for index in range(limit)] - - server.ledger = EndlessLedger() - server._note_block_replay_enumeration_owed() - with patch("builtins.print"): - queued = server.replay_pending_block_candidates() - self.assertEqual(queued, 1024) - self.assertEqual(requested, [32, 64, 128, 256, 512, 1024]) - self.assertTrue(server._block_replay_enumeration_owed()) - with self.assertRaisesRegex(TemplateRefreshBlocked, "has not enumerated"): - server._await_pending_parent_payout_preview("ab" * 32) - - def test_steady_state_poll_restores_a_single_batch(self) -> None: - """Without an owed enumeration, a full batch stays a single query; - the next poll picks up later rows after the queue drains.""" - server, _rpc = coordinator() - requested: list[int] = [] - - class FullBatchLedger(FakeLedger): - def pending_block_candidate_rows( - self, *, limit: int = 32 - ) -> list[dict[str, object]]: - requested.append(limit) - return [durable_candidate_row(index) for index in range(limit)] - - server.ledger = FullBatchLedger() - self.assertEqual(server.replay_pending_block_candidates(), 32) - self.assertEqual(requested, [32]) - self.assertFalse(server._block_replay_enumeration_owed()) - - def test_startup_replay_timeout_keeps_job_builds_blocked(self) -> None: - server, _rpc = coordinator() - release = threading.Event() - - class StallLedger(FakeLedger): - def pending_block_candidate_rows( - self, *, limit: int = 32 - ) -> list[dict[str, object]]: - release.wait(5) - return [] - - server.ledger = StallLedger() - server.block_landing_db_timeout_seconds = 0.01 - server.block_landing_db_timeout_max_seconds = 0.01 - try: - with patch("builtins.print"): - self.assertTrue(server._run_startup_block_candidate_replay()) - self.assertTrue(server._block_replay_enumeration_owed()) - with self.assertRaisesRegex( - TemplateRefreshBlocked, "has not enumerated" - ): - server._await_pending_parent_payout_preview("ab" * 32) - # The startup enumeration ran with the landing budget, so its - # timeout fires the documented landing-timeout alert instead of - # hiding inside the fast-call series. - metrics = server.block_ledger_call_class_metrics() - self.assertEqual(metrics["landing"]["timeouts_total"], 1) - self.assertNotIn("fast", metrics) - finally: - release.set() - - def test_startup_phase_seconds_records_first_occurrence_only(self) -> None: - import time as _time - - server, _rpc = coordinator() - self.assertEqual(server.startup_phase_seconds(), {}) - server._record_startup_phase_once("audit_listener_bound") - self.assertEqual(server.startup_phase_seconds(), {}) - server._startup_phase_origin_monotonic = _time.monotonic() - 1.0 - server._record_startup_phase_once("audit_listener_bound") - first = server.startup_phase_seconds()["audit_listener_bound"] - self.assertGreaterEqual(first, 1.0) - server._record_startup_phase_once("audit_listener_bound") - self.assertEqual( - server.startup_phase_seconds()["audit_listener_bound"], first - ) - - def test_job_issuance_continues_during_padded_landing(self) -> None: - """Acceptance shape for issue #188: a landing padded past its normal - duration must not stop child job issuance, and payout-state - publication stays gated until the landing reaches a terminal state. - The pad is logical (the barrier simply stays unresolved) so the test - is instant while pinning the same ordering a 10s landing produces.""" - server, _rpc = coordinator() - parent_hash = "ab" * 32 - preview = [ - { - "recipient_id": "miner-a", - "order_key": "miner-a", - "p2mr_program_hex": "11" * 32, - "balance_sats": 25, - } - ] - server._begin_accepted_block_payout_preview(parent_hash, block_height=10) - server._mark_accepted_block_payout_landed(parent_hash, block_height=10) - server._publish_accepted_block_payout_preview(parent_hash, preview) - # The landing bookkeeping is still unresolved; every child build keeps - # receiving the immutable preview instead of blocking or reading - # confirmed balances. - for _ in range(5): - self.assertEqual( - server._prior_balances_for_job_parent(parent_hash), - preview, - ) - ages = server.accepted_parent_unresolved_ages_seconds() - self.assertEqual(len(ages), 1) - with self.assertRaisesRegex( - _PayoutStatePublicationBlocked, "still pending" - ): - with server._payout_balance_mutation(): - pass - # Landing completes: the transition clears and confirmed reads resume. - server._clear_accepted_block_payout_preview(parent_hash) - self.assertEqual( - server._prior_balances_for_job_parent( - parent_hash, fallback_balances=[] - ), - [], - ) - self.assertEqual(server.accepted_parent_unresolved_ages_seconds(), []) - - def test_unmark_landed_clears_unresolved_age(self) -> None: - server, _rpc = coordinator() - block_hash = "cd" * 32 - server._begin_accepted_block_payout_preview(block_hash, block_height=10) - self.assertEqual(server.accepted_parent_unresolved_ages_seconds(), []) - server._mark_accepted_block_payout_landed(block_hash, block_height=10) - ages = server.accepted_parent_unresolved_ages_seconds() - self.assertEqual(len(ages), 1) - self.assertGreaterEqual(ages[0], 0.0) - server._unmark_accepted_block_payout_landed(block_hash) - self.assertEqual(server.accepted_parent_unresolved_ages_seconds(), []) - server._clear_accepted_block_payout_preview(block_hash) - - def test_withdrawn_landed_transition_blocks_active_descendant_fallback( - self, - ) -> None: - class CountingLedger(FakeLedger): - def __init__(self) -> None: - super().__init__() - self.current_balance_reads = 0 - - def current_prior_balances(self) -> list[dict[str, object]]: - self.current_balance_reads += 1 - return [] - - ledger = CountingLedger() - server, rpc = coordinator(ledger=ledger) - accepted_hash = "bc" * 32 - descendant_hash = "bd" * 32 - original_rpc_call = rpc.call - - def active_chain_call( - method: str, - params: list[object] | None = None, - ) -> object: - if method == "getblockhash": - self.assertEqual(params, [10]) - return accepted_hash - return original_rpc_call(method, params) - - rpc.call = active_chain_call # type: ignore[method-assign] - server._begin_accepted_block_payout_preview( - accepted_hash, - block_height=10, - ) - server._mark_accepted_block_payout_landed( - accepted_hash, - block_height=10, - ) - server._clear_accepted_block_payout_preview( - accepted_hash, - invalidate_published=True, - ) - - with self.assertRaisesRegex(TemplateRefreshBlocked, "was withdrawn"): - server._prior_balances_for_job_parent( - descendant_hash, - parent_height=11, - ) - self.assertEqual(ledger.current_balance_reads, 0) - self.assertTrue(server._tip_refresh_retry.is_set()) - - def test_preview_publication_survives_window_build_timeout(self) -> None: - """Issue #188 acceptance: when the payout window build dies with the - slow ledger during found-block publication, the preview must still - cross the atomic publication boundary. Admission then reopens and - child jobs build against the verified compact preview instead of - stalling behind the global publication fence until the ledger - recovers.""" - accepted_hash = "c4" * 32 - server, rpc = coordinator( - template=base_template(height=12, prevhash=accepted_hash) - ) - recorded = install_fake_bundle_builder(server) - artifacts = server.store_template_artifacts(dict(rpc.template)) - assert artifacts is not None - self.assertTrue(server.pool_readiness_latched()) - preview = [ - { - "recipient_id": "miner-a", - "order_key": "miner-a", - "p2mr_program_hex": "11" * 32, - "balance_sats": 25, - } - ] - - def timing_out_window_build( - *_args: object, **_kwargs: object - ) -> PayoutLedgerArtifact: - raise LedgerOperationTimeout("postgres statement deadline expired") - - server._build_payout_ledger_artifact = ( # type: ignore[method-assign] - timing_out_window_build - ) - server._begin_accepted_block_payout_preview(accepted_hash, block_height=11) - server._mark_accepted_block_payout_landed(accepted_hash, block_height=11) - generation_before = server._payout_state_generation - with patch("builtins.print"): - published = server._publish_accepted_block_payout_preview( - accepted_hash, preview - ) - self.assertEqual(published, preview) - # The publication crossed the atomic boundary: a generation was - # installed and delivery admission reopened despite the dead build. - self.assertFalse(server._payout_state_publication_fenced()) - self.assertEqual(server._payout_state_generation, generation_before + 1) - transition = server._accepted_block_payout_previews[accepted_hash] - self.assertEqual(transition.published_generation, generation_before + 1) - # Child job construction is admitted and consumes the preview. - bundle = server.shared_job_bundle(artifacts, mode="ready") - self.assertIsNotNone(bundle) - self.assertEqual(recorded["calls"], 1) - last_kwargs = recorded["last_kwargs"] - assert isinstance(last_kwargs, dict) - self.assertEqual(list(last_kwargs["prior_balances"]), preview) - server._clear_accepted_block_payout_preview(accepted_hash) - - def test_inactive_landed_ancestor_rejects_preview_patched_artifact( - self, - ) -> None: - accepted_hash = "c0" * 32 - alternate_tip = "c1" * 32 - server, rpc = coordinator( - template=base_template(height=12, prevhash=alternate_tip) - ) - recorded = install_fake_bundle_builder(server) - artifacts = server.store_template_artifacts(dict(rpc.template)) - assert artifacts is not None - self.assertTrue(server.pool_readiness_latched()) - preview = [ - { - "recipient_id": "miner-a", - "order_key": "miner-a", - "p2mr_program_hex": "11" * 32, - "balance_sats": 25, - } - ] - server._begin_accepted_block_payout_preview( - accepted_hash, - block_height=10, - ) - server._mark_accepted_block_payout_landed( - accepted_hash, - block_height=10, - ) - server._publish_accepted_block_payout_preview(accepted_hash, preview) - with server._job_cache_lock: - artifact = server._payout_ledger_artifact - self.assertIsNotNone(artifact) - assert artifact is not None - self.assertEqual(list(artifact.prior_balances), preview) - - original_rpc_call = rpc.call - - def alternate_chain_call( - method: str, - params: list[object] | None = None, - ) -> object: - if method == "getblockhash": - self.assertEqual(params, [10]) - return "c2" * 32 - return original_rpc_call(method, params) - - rpc.call = alternate_chain_call # type: ignore[method-assign] - - with self.assertRaisesRegex( - TemplateRefreshBlocked, - "no longer active", - ): - server.shared_job_bundle(artifacts, mode="ready") - - self.assertEqual(recorded["calls"], 0) - self.assertTrue(server._tip_refresh_retry.is_set()) - - def test_waiting_child_does_not_fall_back_after_transition_withdrawal( - self, - ) -> None: - class CountingLedger(FakeLedger): - def __init__(self) -> None: - super().__init__() - self.current_balance_reads = 0 - - def current_prior_balances(self) -> list[dict[str, object]]: - self.current_balance_reads += 1 - return [] - - class ObservedCondition(threading.Condition): - def __init__(self) -> None: - super().__init__() - self.wait_entered = threading.Event() - - def wait(self, timeout: float | None = None) -> bool: - self.wait_entered.set() - return super().wait(timeout) - - ledger = CountingLedger() - server, _rpc = coordinator(ledger=ledger) - parent_hash = "be" * 32 - preview_condition = ObservedCondition() - server._accepted_block_payout_preview_condition = preview_condition - server.accepted_block_payout_preview_wait_seconds = 10 - server._begin_accepted_block_payout_preview( - parent_hash, - block_height=10, - ) - server._mark_accepted_block_payout_landed( - parent_hash, - block_height=10, - ) - errors: list[BaseException] = [] - - def read_parent_balances() -> None: - try: - server._prior_balances_for_job_parent( - parent_hash, - parent_height=10, - ) - except BaseException as exc: # pragma: no cover - asserted below - errors.append(exc) - - thread = threading.Thread(target=read_parent_balances, daemon=True) - thread.start() - try: - self.assertTrue(preview_condition.wait_entered.wait(2)) - self.assertTrue(thread.is_alive()) - finally: - server._clear_accepted_block_payout_preview( - parent_hash, - invalidate_published=True, - ) - thread.join(5) - - self.assertFalse(thread.is_alive()) - self.assertEqual(len(errors), 1) - self.assertIsInstance(errors[0], TemplateRefreshBlocked) - self.assertIn("was withdrawn", str(errors[0])) - self.assertEqual(ledger.current_balance_reads, 0) - - def test_pending_parent_preview_wait_is_bounded_and_retryable(self) -> None: - server, _rpc = coordinator() - parent_hash = "ac" * 32 - server.accepted_block_payout_preview_wait_seconds = 0.01 - server._begin_accepted_block_payout_preview(parent_hash) - - with self.assertRaisesRegex(TemplateRefreshBlocked, "not ready"): - server._prior_balances_for_job_parent(parent_hash) - - self.assertTrue(server._tip_refresh_retry.is_set()) - server._clear_accepted_block_payout_preview(parent_hash) - - def test_replayed_active_ancestor_blocks_descendant_until_preview(self) -> None: - class PreviewLedger(FakeLedger): - def __init__(self) -> None: - super().__init__() - self.current_balance_reads = 0 - - def current_prior_balances(self) -> list[dict[str, object]]: - self.current_balance_reads += 1 - return [] - - class ObservedCondition(threading.Condition): - def __init__(self) -> None: - super().__init__() - self.wait_entered = threading.Event() - - def wait(self, timeout: float | None = None) -> bool: - self.wait_entered.set() - return super().wait(timeout) - - ledger = PreviewLedger() - server, rpc = coordinator(ledger=ledger) - accepted_hash = "ad" * 32 - descendant_hash = "ae" * 32 - preview = [ - { - "recipient_id": "miner-a", - "order_key": "miner-a", - "p2mr_program_hex": "11" * 32, - "balance_sats": 25, - } - ] - original_rpc_call = rpc.call - - def active_chain_call( - method: str, - params: list[object] | None = None, - ) -> object: - if method == "getblockhash": - self.assertEqual(params, [10]) - return accepted_hash - return original_rpc_call(method, params) - - rpc.call = active_chain_call # type: ignore[method-assign] - preview_condition = ObservedCondition() - server._accepted_block_payout_preview_condition = preview_condition - server.accepted_block_payout_preview_wait_seconds = 10 - server._begin_accepted_block_payout_preview(accepted_hash, block_height=10) - balances: list[list[dict[str, object]]] = [] - errors: list[BaseException] = [] - - def read_descendant_balances() -> None: - try: - balances.append( - server._prior_balances_for_job_parent( - descendant_hash, - parent_height=11, - ) - ) - except BaseException as exc: # pragma: no cover - asserted below - errors.append(exc) - - thread = threading.Thread(target=read_descendant_balances, daemon=True) - thread.start() - try: - self.assertTrue(preview_condition.wait_entered.wait(2)) - self.assertTrue(thread.is_alive()) - self.assertEqual(ledger.current_balance_reads, 0) - finally: - server._publish_accepted_block_payout_preview(accepted_hash, preview) - thread.join(5) - server._clear_accepted_block_payout_preview(accepted_hash) - - self.assertFalse(thread.is_alive()) - self.assertEqual(errors, []) - self.assertEqual(balances, [preview]) - self.assertEqual(ledger.current_balance_reads, 0) - - def test_landed_transition_bars_reconciliation_before_preview(self) -> None: - server, _rpc = coordinator() - block_hash = "af" * 32 - server._begin_accepted_block_payout_preview(block_hash, block_height=10) - server._mark_accepted_block_payout_landed(block_hash, block_height=10) - - with self.assertRaisesRegex( - _PayoutStatePublicationBlocked, - "confirmation is still pending", - ): - with server._payout_balance_mutation(): - self.fail("landed transition must bar payout reconciliation") - with self.assertRaisesRegex( - _PayoutStatePublicationBlocked, - "confirmation is still pending", - ): - server.reconcile_prism_pool_blocks_once() - - server._clear_accepted_block_payout_preview(block_hash) - with server._payout_balance_mutation(): - pass - - def test_landed_transition_poll_uses_coordination_blocked_budget(self) -> None: - server, _rpc = coordinator() - server.reorg_reconciler_enabled = True - block_hash = "af" * 32 - server._begin_accepted_block_payout_preview(block_hash, block_height=10) - server._mark_accepted_block_payout_landed(block_hash, block_height=10) - - with ( - patch( - "lab.prism.prism_coordinator.time.monotonic", - return_value=100.0, - ), - self.assertRaisesRegex( - _PayoutStatePublicationBlocked, - "confirmation is still pending", - ), - ): - server.poll_qbit_tip_template_once() - - self.assertIsNone( - getattr(server, "template_refresh_failure_started_monotonic", None) - ) - self.assertEqual( - server.coordination_blocked_streak_age_seconds(105.0), - 5.0, - ) - - def test_readiness_latch_during_preparation_admission_reselects_ready_mode(self) -> None: - ledger = FakeLedger(miners=["solo"]) - server, rpc = coordinator(ledger=ledger) - recorded = install_fake_bundle_builder(server) - artifacts = server.store_template_artifacts(dict(rpc.template)) - self.assertIsNotNone(artifacts) - assert artifacts is not None - first_lookup_entered = threading.Event() - release_first_lookup = threading.Event() - original_lookup = server._lookup_job_bundle - lookup_calls = 0 - - def block_first_lookup(key: tuple[object, ...]) -> object: - nonlocal lookup_calls - lookup_calls += 1 - if lookup_calls == 1: - first_lookup_entered.set() - self.assertTrue(release_first_lookup.wait(2.0)) - return original_lookup(key) - - server._lookup_job_bundle = block_first_lookup # type: ignore[method-assign] - bundles: list[object] = [] - errors: list[BaseException] = [] - - def build_bundle() -> None: - try: - bundles.append(server.shared_job_bundle(artifacts, worker())) - except BaseException as exc: # pragma: no cover - assertion reports it - errors.append(exc) - - build_thread = threading.Thread(target=build_bundle) - build_thread.start() - self.assertTrue(first_lookup_entered.wait(2.0)) - ledger.miners = ["miner-a", "miner-b", "miner-c"] - release_first_lookup.set() - - build_thread.join(2.0) - - self.assertFalse(build_thread.is_alive()) - self.assertEqual(errors, []) - self.assertEqual(len(bundles), 1) - bundle = bundles[0] - self.assertFalse(bundle.collection_only) # type: ignore[union-attr] - self.assertEqual(recorded["calls"], 1) - self.assertEqual(ledger.snapshot_calls, 1) - - def test_payout_generation_advance_does_not_cancel_new_generation_fanout(self) -> None: - server, _rpc = coordinator() - server._ensure_tip_refresh_state() - cancellations: list[str] = [] - - class CurrentToken: - payout_state_generation = 1 - - class Cancellation: - def cancel(self) -> None: - cancellations.append("cancelled") - - class InjectCurrentFanoutLock: - def __enter__(self) -> object: - server._active_tip_refresh = (CurrentToken(), Cancellation()) - return self - - def __exit__(self, *_args: object) -> None: - return None - - # Model a new-generation refresh registering after the cache-state - # increment but before the invalidator reaches the coordinator lock. - server.lock = InjectCurrentFanoutLock() # type: ignore[assignment] - - self.assertEqual(server._advance_payout_state_generation(), 1) - self.assertEqual(cancellations, []) - self.assertFalse(server.tip_refresh_is_pending()) - self.assertFalse(server._tip_refresh_retry.is_set()) - - def test_payout_generation_retry_marks_tip_refresh_pending(self) -> None: - server, _rpc = coordinator() - server._ensure_tip_refresh_state() - - self.assertFalse(server.tip_refresh_is_pending()) - self.assertEqual(server._advance_payout_state_generation(), 1) - - self.assertTrue(server.tip_refresh_is_pending()) - self.assertTrue(server._tip_refresh_retry.is_set()) - - self.assertEqual(server.poll_qbit_tip_template_once(), 0) - self.assertFalse(server.tip_refresh_is_pending()) - - def test_payout_only_advance_bounds_publish_supersession(self) -> None: - server, _rpc = coordinator() - server.payout_reconcile_supersession_retries = 2 - real_publish = server._publish_payout_state_candidate - publish_attempts = 0 - - def supersede_before_publish(candidate: object) -> int | None: - nonlocal publish_attempts - publish_attempts += 1 - server._reserve_payout_state_source( - "external_tip", - tip_hash=f"{publish_attempts + 30:064x}", - ) - return real_publish(candidate) # type: ignore[arg-type] - - server._publish_payout_state_candidate = supersede_before_publish # type: ignore[method-assign] - - with self.assertRaisesRegex( - TemplateRefreshBlocked, - "payout-only invalidation was superseded", - ): - server._advance_payout_state_generation() - - self.assertEqual(publish_attempts, 3) - self.assertEqual(server._payout_state_generation, 0) - self.assertTrue(server._payout_state_publication_blocked) - self.assertTrue(server._payout_state_delivery_gate._delivery_blocked) - self.assertTrue(server.tip_refresh_is_pending()) - - def test_payout_publication_fence_is_not_a_job_build_failure(self) -> None: - server, _rpc = coordinator() - install_fake_bundle_builder(server) - state = client(1) - state.send = lambda _payload: None # type: ignore[method-assign] - server.clients = {state} - server._pool_ready_latched = True - server._reserve_payout_state_source("payout_only") - server._block_payout_state_publication() - - self.assertFalse(server.maybe_send_job(state, clean_jobs=True)) - with self.assertRaisesRegex(TemplateRefreshBlocked, "pending publication"): - server.poll_qbit_tip_template_once() - - self.assertEqual(server.job_build_failure_count, 0) - self.assertEqual(server.tip_refresh_client_counts["failed"], 0) - self.assertEqual(server.tip_refresh_client_counts["skipped"], 1) - - def test_successful_poll_clears_payout_pending_created_during_reconcile(self) -> None: - server, _rpc = coordinator() - server._ensure_tip_refresh_state() - reconcile_entered = threading.Event() - allow_reconcile = threading.Event() - results: list[int] = [] - errors: list[BaseException] = [] - - def reconcile(_tip_hash: str) -> bool: - reconcile_entered.set() - if not allow_reconcile.wait(5): - raise AssertionError("test did not release reconciliation") - return True - - def poll() -> None: - try: - results.append(server.poll_qbit_tip_template_once()) - except BaseException as exc: # pragma: no cover - asserted below - errors.append(exc) - - server.ensure_reorg_reconciled_for_tip = reconcile # type: ignore[method-assign] - server._mark_tip_refresh_pending("seed") - poll_thread = threading.Thread(target=poll) - poll_thread.start() - try: - self.assertTrue(reconcile_entered.wait(5)) - self.assertEqual(server._advance_payout_state_generation(), 1) - finally: - allow_reconcile.set() - poll_thread.join(5) - - self.assertFalse(poll_thread.is_alive()) - self.assertEqual(errors, []) - self.assertEqual(results, [0]) - self.assertEqual(server._payout_state_generation, 1) - self.assertIsNotNone(server.last_successful_template_refresh_monotonic) - self.assertFalse(server.tip_refresh_is_pending()) - - def test_failed_poll_preserves_pending_signal_until_successful_retry(self) -> None: - server, _rpc = coordinator() - server._ensure_tip_refresh_state() - pending_token = server._mark_tip_refresh_pending("seed") - - def fail_reconciliation(_tip_hash: str) -> bool: - server._schedule_tip_refresh_retry() - raise RuntimeError("ledger unavailable") - - server.ensure_reorg_reconciled_for_tip = fail_reconciliation # type: ignore[method-assign] - - with self.assertRaisesRegex( - TemplateRefreshBlocked, - "qbit reorg reconciliation failed", - ): - server.poll_qbit_tip_template_once() - - self.assertTrue(server._tip_refresh_retry.is_set()) - self.assertTrue(server.tip_refresh_is_pending()) - self.assertEqual(server._tip_refresh_pending_token, pending_token) - - # Model blockpoll claiming the immediate wake and completing the retry. - server._tip_refresh_retry.clear() - server.ensure_reorg_reconciled_for_tip = lambda _tip: True # type: ignore[method-assign] - - self.assertEqual(server.poll_qbit_tip_template_once(), 0) - self.assertFalse(server.tip_refresh_is_pending()) - - def test_shutdown_during_reconciliation_stops_poll_without_refresh_failure(self) -> None: - server, _rpc = coordinator() - - def rejected_reconciliation(_tip_hash: str) -> bool: - server.stop_event.set() - raise ShutdownInProgress("PRISM coordinator is shutting down") - - server.ensure_reorg_reconciled_for_tip = ( # type: ignore[method-assign] - rejected_reconciliation - ) - - self.assertEqual(server.poll_qbit_tip_template_once(), 0) - self.assertIsNone( - getattr(server, "last_successful_template_refresh_monotonic", None) - ) - - def test_completed_refresh_cannot_clear_newer_payout_pending(self) -> None: - server, _rpc = coordinator() - snapshot = server.fetch_qbit_tip_template_snapshot() - sequence = server._reserve_tip_observation_sequence() - self.assertTrue( - server.observe_tip_first_seen( - snapshot.bestblockhash, - observation_sequence=sequence, - publish_refresh_observation=True, - ) - ) - with server.lock: - server.tip_template_snapshot = snapshot - completed_generation = server._payout_state_generation - self.assertEqual(server._advance_payout_state_generation(), 1) - newer_token = server._tip_refresh_pending_token - - self.assertFalse( - server._clear_tip_refresh_pending_for_completed_refresh( - snapshot, - sequence, - completed_generation, - ) - ) - self.assertEqual(server._tip_refresh_pending_token, newer_token) - self.assertTrue(server.tip_refresh_is_pending()) - - def test_escaped_stale_bundle_is_rejected_before_direct_delivery(self) -> None: - server, _rpc = coordinator() - install_fake_bundle_builder(server) - server._ensure_tip_refresh_state() - state = client(1) - sent: list[dict[str, object]] = [] - state.send = sent.append # type: ignore[method-assign] - original_shared_job_bundle = server.shared_job_bundle - advanced = False - - def advance_after_bundle(*args: object, **kwargs: object) -> object: - nonlocal advanced - bundle = original_shared_job_bundle(*args, **kwargs) # type: ignore[arg-type] - if not advanced: - advanced = True - server._advance_payout_state_generation() - return bundle - - server.shared_job_bundle = advance_after_bundle # type: ignore[method-assign] - - self.assertFalse(server.maybe_send_job(state, clean_jobs=True)) - self.assertEqual(sent, []) - self.assertIsNone(state.active_job) - self.assertEqual(server._payout_state_generation, 1) - self.assertTrue(server._tip_refresh_retry.is_set()) - - self.assertTrue(server.maybe_send_job(state, clean_jobs=True)) - self.assertIsNotNone(state.active_job) - self.assertEqual(state.active_job.payout_state_generation, 1) - self.assertEqual( - [payload["method"] for payload in sent], - ["mining.set_difficulty", "mining.notify"], - ) - - def test_priority_decision_uses_one_publication_snapshot(self) -> None: - server, _rpc = coordinator() - state = client(1) - context = SimpleNamespace( - payout_state_generation=0, - template={"previousblockhash": "11" * 32}, - ) - server.ensure_reorg_reconciled_for_current_tip = ( # type: ignore[method-assign] - lambda **_kwargs: True - ) - server.build_job_for_client = ( # type: ignore[method-assign] - lambda *_args, **_kwargs: context - ) - original_lock = server._job_cache_lock - - class PublishAfterPrioritySnapshot: - advanced = False - - def __enter__(self) -> object: - original_lock.acquire() - return self - - def __exit__( - self, - _exc_type: object, - _exc: object, - _traceback: object, - ) -> None: - original_lock.release() - if not self.advanced: - self.advanced = True - server._payout_state_generation = 1 - - priorities: list[bool] = [] - - class RecordingGate: - @contextmanager - def delivery_cancelable( - self, - _cancelled: object, - *, - priority: bool, - **_kwargs: object, - ) -> object: - priorities.append(priority) - yield False - - server._job_cache_lock = PublishAfterPrioritySnapshot() # type: ignore[assignment] - server._payout_state_delivery_gate = RecordingGate() # type: ignore[assignment] - - self.assertFalse(server.maybe_send_job(state, clean_jobs=True)) - self.assertEqual(priorities, [True]) - self.assertEqual(server._payout_state_generation, 1) - - def test_zero_template_ttl_fetches_template_per_build(self) -> None: - server, rpc = coordinator() - install_fake_bundle_builder(server) - server.template_cache_seconds = 0.0 - - server.build_job_for_client(client(1), clean_jobs=True) - server.build_job_for_client(client(2), clean_jobs=True) - - self.assertEqual(rpc.count("getblocktemplate"), 2) - - def test_late_stale_template_fetch_cannot_replace_newer_artifacts(self) -> None: - server, rpc = coordinator() - server.template_cache_seconds = 0.0 - stale_template = dict(rpc.template) - current_template = base_template(height=11, prevhash="22" * 32) - fetch_started = threading.Event() - release_fetch = threading.Event() - results: list[object] = [] - errors: list[BaseException] = [] - original_call = rpc.call - thread: threading.Thread - - def blocking_call( - method: str, - params: list[object] | None = None, - ) -> object: - if method == "getblocktemplate" and threading.current_thread() is thread: - fetch_started.set() - if not release_fetch.wait(5): - raise AssertionError("stale template fetch was not released") - return dict(stale_template) - return original_call(method, params) - - def fetch_stale_artifacts() -> None: - try: - results.append(server.current_template_artifacts()) - except BaseException as exc: # noqa: BLE001 - surface to the test - errors.append(exc) - - rpc.call = blocking_call # type: ignore[method-assign] - thread = threading.Thread(target=fetch_stale_artifacts) - thread.start() - try: - self.assertTrue(fetch_started.wait(5)) - current_artifacts = server.store_template_artifacts(current_template) - self.assertIsNotNone(current_artifacts) - assert current_artifacts is not None - self.assertGreater(current_artifacts.generation, 1) - finally: - release_fetch.set() - thread.join(5) - - self.assertFalse(thread.is_alive()) - self.assertEqual(errors, []) - self.assertEqual(results, [current_artifacts]) - self.assertIs(server._template_artifacts, current_artifacts) - self.assertEqual( - current_artifacts.fingerprint, - qbit_template_fingerprint(current_template), - ) - - def test_collection_mode_bundles_are_keyed_per_worker(self) -> None: - server, _ = coordinator(ledger=FakeLedger(miners=["solo"])) - recorded = install_fake_bundle_builder(server) - server.min_ready_miners = 3 - - worker_a = worker(payout="tq1worker-a") - worker_b = worker(payout="tq1worker-b") - context_a1 = server.build_job_for_client(client(1, worker_a), clean_jobs=True) - context_a2 = server.build_job_for_client(client(2, worker_a), clean_jobs=True) - context_b = server.build_job_for_client(client(3, worker_b), clean_jobs=True) - - self.assertTrue(context_a1.collection_only) - self.assertTrue(context_b.collection_only) - self.assertEqual(recorded["calls"], 2) - self.assertTrue( - all(not hasattr(context, "bundle") for context in (context_a1, context_a2, context_b)) - ) - self.assertIs(context_a1.shares_json, context_a2.shares_json) - self.assertIsNot(context_a1.shares_json, context_b.shares_json) - - def test_collection_bundle_cache_rebuilds_when_pool_becomes_ready(self) -> None: - ledger = FakeLedger(miners=["solo"]) - server, _ = coordinator(ledger=ledger) - recorded = install_fake_bundle_builder(server) - state = client(1) - - collection_context = server.build_job_for_client(state, clean_jobs=True) - ledger.miners = ["miner-a", "miner-b", "miner-c"] - ready_context = server.build_job_for_client(state, clean_jobs=True) - - self.assertTrue(collection_context.collection_only) - self.assertFalse(ready_context.collection_only) - self.assertEqual(recorded["calls"], 2) - self.assertEqual(ledger.snapshot_calls, 1) - - def test_ready_empty_snapshot_does_not_fall_back_to_worker_collection(self) -> None: - ledger = ReadyLedgerWithEmptyFirstSnapshot() - server, _ = coordinator(ledger=ledger) - recorded = install_fake_bundle_builder(server) - state = client(1) - - with self.assertRaisesRegex( - RuntimeError, - "ready-pool ledger snapshot contained no payout shares", - ): - server.build_job_for_client(state, clean_jobs=True) - ready_context = server.build_job_for_client(state, clean_jobs=True) - - self.assertFalse(ready_context.collection_only) - self.assertEqual(recorded["calls"], 1) - self.assertEqual(ledger.snapshot_calls, 2) - - def test_vardiff_difficulty_is_stamped_per_client(self) -> None: - server, _ = coordinator() - install_fake_bundle_builder(server) - easy = client(1) - hard = client(2) - hard.pending_share_difficulty = Decimal("512") - - easy_context = server.build_job_for_client(easy, clean_jobs=True) - hard_context = server.build_job_for_client(hard, clean_jobs=True) - - self.assertEqual(easy_context.job.coinb1, hard_context.job.coinb1) - self.assertGreater(easy_context.job.share_target, hard_context.job.share_target) - self.assertEqual(hard_context.job.share_difficulty, Decimal("512")) - - def test_template_artifacts_reuse_derivations_when_fingerprint_unchanged(self) -> None: - server, _ = coordinator() - template = base_template() - first = server.store_template_artifacts(dict(template)) - refreshed_template = dict(template) - refreshed_template["curtime"] = int(template["curtime"]) + 30 - second = server.store_template_artifacts(refreshed_template) - - self.assertIsNotNone(first) - self.assertIsNotNone(second) - assert first is not None and second is not None - self.assertEqual(first.fingerprint, second.fingerprint) - self.assertIs(first.transaction_hexes, second.transaction_hexes) - self.assertIs(first.witness_merkle_leaves_hex, second.witness_merkle_leaves_hex) - - def test_poll_seeds_template_cache_for_client_builds(self) -> None: - server, rpc = coordinator() - install_fake_bundle_builder(server) - - refreshed = server.poll_qbit_tip_template_once() - self.assertEqual(refreshed, 0) - self.assertEqual(rpc.count("getblocktemplate"), 1) - - server.build_job_for_client(client(1), clean_jobs=True) - self.assertEqual(rpc.count("getblocktemplate"), 1) - - def test_reorg_reconciliation_cached_per_tip(self) -> None: - server, rpc = coordinator() - server.reorg_reconciler_enabled = True - reconcile_calls: list[str | None] = [] - - def fake_reconcile(*, tip_hash: str | None = None) -> dict[str, object]: - reconcile_calls.append(tip_hash) - server._note_reorg_reconcile_outcome(tip_hash, trusted=True) - return {"untrusted": False} - - server.reconcile_prism_pool_blocks_once = fake_reconcile # type: ignore[method-assign] - - self.assertTrue(server.ensure_reorg_reconciled_for_current_tip()) - self.assertTrue(server.ensure_reorg_reconciled_for_current_tip()) - self.assertEqual(len(reconcile_calls), 1) - - rpc.tip = "33" * 32 - self.assertTrue(server.ensure_reorg_reconciled_for_current_tip()) - self.assertEqual(len(reconcile_calls), 2) - self.assertEqual(reconcile_calls[-1], "33" * 32) - - def test_untrusted_outcome_for_other_tip_keeps_memo_armed(self) -> None: - server, rpc = coordinator() - server.reorg_reconciler_enabled = True - server._note_reorg_reconcile_outcome(rpc.tip, trusted=True) - reconcile_calls: list[str | None] = [] - - def fake_reconcile(*, tip_hash: str | None = None) -> dict[str, object]: - reconcile_calls.append(tip_hash) - server._note_reorg_reconcile_outcome(tip_hash, trusted=True) - return {"untrusted": False} - - server.reconcile_prism_pool_blocks_once = fake_reconcile # type: ignore[method-assign] - - # A superseded/untrusted pass for a different tip must not unarm the - # cached trusted outcome for the current tip. - server._note_reorg_reconcile_outcome("44" * 32, trusted=False) - self.assertTrue(server.ensure_reorg_reconciled_for_current_tip()) - self.assertEqual(reconcile_calls, []) - - # An untrusted outcome for the current tip itself unarms only it. - server._note_reorg_reconcile_outcome(rpc.tip, trusted=False) - self.assertTrue(server.ensure_reorg_reconciled_for_current_tip()) - self.assertEqual(reconcile_calls, [rpc.tip]) - - def test_reconcile_error_outcome_clears_every_memo_entry(self) -> None: - server, rpc = coordinator() - server.reorg_reconciler_enabled = True - server._note_reorg_reconcile_outcome(rpc.tip, trusted=True) - server._note_reorg_reconcile_outcome("55" * 32, trusted=True) - reconcile_calls: list[str | None] = [] - - def fake_reconcile(*, tip_hash: str | None = None) -> dict[str, object]: - reconcile_calls.append(tip_hash) - server._note_reorg_reconcile_outcome(tip_hash, trusted=True) - return {"untrusted": False} - - server.reconcile_prism_pool_blocks_once = fake_reconcile # type: ignore[method-assign] - - server._note_reorg_reconcile_outcome( - "55" * 32, - trusted=False, - clear_memo=True, - ) - - self.assertTrue(server.ensure_reorg_reconciled_for_current_tip()) - self.assertEqual(reconcile_calls, [rpc.tip]) - - def test_tip_flip_back_within_ttl_requires_fresh_reconcile(self) -> None: - server, rpc = coordinator() - server.reorg_reconciler_enabled = True - tip_a = rpc.tip - self.addCleanup(server.shutdown_tip_refresh_executor) - server.observe_tip_for_refresh(tip_a) - server._note_reorg_reconcile_outcome(tip_a, trusted=True) - reconcile_calls: list[str | None] = [] - - def fake_reconcile(*, tip_hash: str | None = None) -> dict[str, object]: - reconcile_calls.append(tip_hash) - return {"untrusted": False} - - server.reconcile_prism_pool_blocks_once = fake_reconcile # type: ignore[method-assign] - - self.assertTrue(server.ensure_reorg_reconciled_for_current_tip()) - self.assertEqual(reconcile_calls, []) - - # Detecting an intervening tip ends tip A's memo epoch: when the - # chain flips back to A within the cache TTL, pool-block chain state - # must be re-proven by a fresh pass, not assumed from the pre-flip - # outcome. - server.observe_tip_for_refresh("bb" * 32) - - self.assertTrue(server.ensure_reorg_reconciled_for_current_tip()) - self.assertEqual(reconcile_calls, [tip_a]) - - def test_stale_pass_cannot_rearm_memo_after_newer_tip_detected(self) -> None: - server, rpc = coordinator() - server.reorg_reconciler_enabled = True - tip_a = rpc.tip - self.addCleanup(server.shutdown_tip_refresh_executor) - server.observe_tip_for_refresh(tip_a) - server.observe_tip_for_refresh("bb" * 32) - - # A pass for tip A finishing after tip B was detected must not arm - # the memo for A: its epoch already ended at the newer observation. - server._note_reorg_reconcile_outcome(tip_a, trusted=True) - reconcile_calls: list[str | None] = [] - - def fake_reconcile(*, tip_hash: str | None = None) -> dict[str, object]: - reconcile_calls.append(tip_hash) - return {"untrusted": False} - - server.reconcile_prism_pool_blocks_once = fake_reconcile # type: ignore[method-assign] - - self.assertTrue(server.ensure_reorg_reconciled_for_current_tip()) - self.assertEqual(reconcile_calls, [tip_a]) - - def test_reorg_cache_rechecks_chain_view_before_reuse(self) -> None: - server, rpc = coordinator() - server.reorg_reconciler_enabled = True - rpc.blockchain_info["headers"] = 101 - server._note_reorg_reconcile_outcome(rpc.tip, trusted=True) - reconcile_calls: list[str | None] = [] - - def fake_reconcile(*, tip_hash: str | None = None) -> dict[str, object]: - reconcile_calls.append(tip_hash) - server._note_reorg_reconcile_outcome(tip_hash, trusted=False) - return {"untrusted": True} - - server.reconcile_prism_pool_blocks_once = fake_reconcile # type: ignore[method-assign] - - self.assertFalse(server.ensure_reorg_reconciled_for_current_tip()) - self.assertEqual(rpc.count("getblockchaininfo"), 1) - self.assertEqual(reconcile_calls, [rpc.tip]) - self.assertFalse(server.last_reorg_reconciled_trusted) - - def test_single_flight_builds_once_under_concurrency(self) -> None: - server, _ = coordinator() - recorded = install_fake_bundle_builder(server) - original_builder = server.build_audit_bundle - build_started = threading.Event() - - def slow_builder(**kwargs: object) -> dict[str, object]: - build_started.set() - time.sleep(0.05) - return original_builder(**kwargs) - - server.build_audit_bundle = slow_builder # type: ignore[method-assign] - errors: list[BaseException] = [] - - def build(connection_id: int) -> None: - try: - server.build_job_for_client(client(connection_id), clean_jobs=True) - except BaseException as exc: # noqa: BLE001 - surface to the test - errors.append(exc) - - threads = [threading.Thread(target=build, args=(index + 1,)) for index in range(6)] - for thread in threads: - thread.start() - for thread in threads: - thread.join() - - self.assertEqual(errors, []) - self.assertEqual(recorded["calls"], 1) - - def test_observed_tip_change_rejects_stale_cached_bundle_delivery(self) -> None: - old_tip = "11" * 32 - new_tip = "22" * 32 - server, rpc = coordinator(template=base_template(prevhash=old_tip)) - install_fake_bundle_builder(server) - state = client(1) - sent: list[dict[str, object]] = [] - state.send = sent.append # type: ignore[method-assign] - server.clients = {state} - server.observe_tip_first_seen(old_tip, observation_sequence=1) - - self.assertTrue(server.maybe_send_job(state, clean_jobs=True)) - self.assertEqual(len(sent), 2) - sent.clear() - - rpc.tip = new_tip - server.observe_tip_first_seen(new_tip, observation_sequence=2) - - self.assertFalse(server.maybe_send_job(state, clean_jobs=True)) - self.assertEqual(sent, []) - - def test_ready_ledger_snapshot_holds_payout_mutation_lock(self) -> None: - server, rpc = coordinator() - install_fake_bundle_builder(server) - artifacts = server.store_template_artifacts(dict(rpc.template)) - assert artifacts is not None - entered_snapshot = threading.Event() - release_snapshot = threading.Event() - original_snapshot = server.ledger.snapshot_at_job_issue - - def blocked_snapshot(*args: object, **kwargs: object) -> object: - entered_snapshot.set() - self.assertTrue(release_snapshot.wait(2)) - return original_snapshot(*args, **kwargs) - - server.ledger.snapshot_at_job_issue = blocked_snapshot # type: ignore[method-assign] - errors: list[BaseException] = [] - build_thread = threading.Thread( - target=lambda: self._capture_error( - errors, - lambda: server.shared_job_bundle(artifacts, mode="ready"), - ) - ) - build_thread.start() - try: - self.assertTrue(entered_snapshot.wait(2)) - mutation_acquired = server._payout_state_prepare_lock.acquire( - blocking=False - ) - if mutation_acquired: - server._payout_state_prepare_lock.release() - self.assertFalse(mutation_acquired) - finally: - release_snapshot.set() - build_thread.join(2) - - self.assertFalse(build_thread.is_alive()) - self.assertEqual(errors, []) - - def test_ready_build_identity_separates_clock_only_generations(self) -> None: - server, rpc = coordinator() - first = server.store_template_artifacts(dict(rpc.template)) - second_template = dict(rpc.template) - second_template["curtime"] = int(second_template["curtime"]) + 1 - second = server.store_template_artifacts(second_template) - assert first is not None and second is not None - self.assertEqual(first.fingerprint, second.fingerprint) - self.assertNotEqual(first.generation, second.generation) - payout_generation = server._payout_state_generation - - with patch( - "lab.prism.prism_coordinator.now_ms", - side_effect=[1_700_000_000_000, 1_700_000_001_000], - ): - first_request = server._new_job_build_request( - first, - None, - mode="ready", - payout_state_generation=payout_generation, - cache_key=server._job_bundle_key( - first, - mode="ready", - payout_state_generation=payout_generation, - worker=None, - ), - ) - second_request = server._new_job_build_request( - second, - None, - mode="ready", - payout_state_generation=payout_generation, - cache_key=server._job_bundle_key( - second, - mode="ready", - payout_state_generation=payout_generation, - worker=None, - ), - ) - - self.assertNotEqual( - first_request.equivalence_key, - second_request.equivalence_key, - ) - self.assertNotEqual( - first_request.key.issued_at_ms, - second_request.key.issued_at_ms, - ) - - def test_precomputed_payout_artifact_matches_inline_output_exactly(self) -> None: - server, rpc = coordinator() - install_fake_bundle_builder(server) - artifacts = server.store_template_artifacts(dict(rpc.template)) - assert artifacts is not None - - with patch("lab.prism.prism_coordinator.now_ms", return_value=1_700_000_123_000): - inline = server.shared_job_bundle(artifacts, mode="ready") - with server._job_cache_lock: - server._job_bundle_cache.clear() - server._prepare_payout_ledger_artifact( - server._payout_state_generation, - artifacts.network_difficulty, - ) - prepared = server.shared_job_bundle(artifacts, mode="ready") - - self.assertEqual(prepared.base_job, inline.base_job) - self.assertEqual(prepared.coinbase_manifest, inline.coinbase_manifest) - self.assertEqual(prepared.shares_json, inline.shares_json) - self.assertEqual(prepared.prior_balances, inline.prior_balances) - self.assertEqual(prepared.found_block, inline.found_block) - self.assertGreater(prepared.payout_artifact_generation, 0) - - def test_accepted_preview_patches_artifact_across_normal_clear(self) -> None: - class CountingBalanceLedger(FakeLedger): - def __init__(self) -> None: - super().__init__() - self.prior_balance_reads = 0 - self.database_balances = [ - { - "recipient_id": "stale-miner", - "order_key": "stale-miner", - "p2mr_program_hex": "22" * 32, - "balance_sats": 1, - } - ] - - def current_prior_balances(self) -> list[dict[str, object]]: - self.prior_balance_reads += 1 - return [dict(balance) for balance in self.database_balances] - - ledger = CountingBalanceLedger() - server, rpc = coordinator(ledger=ledger) - recorded = install_fake_bundle_builder(server) - artifacts = server.store_template_artifacts(dict(rpc.template)) - assert artifacts is not None - server._pool_ready_latched = True - parent_hash = str(rpc.template["previousblockhash"]) - parent_height = int(rpc.template["height"]) - 1 - preview = [ - { - "recipient_id": "miner-a", - "order_key": "miner-a", - "p2mr_program_hex": "11" * 32, - "balance_sats": 25, - } - ] - - server._begin_accepted_block_payout_preview( - parent_hash, - block_height=parent_height, - ) - server._publish_accepted_block_payout_preview(parent_hash, preview) - - artifact = server._payout_ledger_artifact - self.assertIsNotNone(artifact) - assert artifact is not None - self.assertEqual(artifact.prior_balances, tuple(preview)) - self.assertEqual( - artifact.payout_state_generation, - server._payout_state_generation, - ) - self.assertGreater(artifact.generation, 0) - - ledger.prior_balance_reads = 0 - preview_bundle = server.shared_job_bundle(artifacts, mode="ready") - - self.assertEqual(preview_bundle.prior_balances, preview) - self.assertEqual(recorded["last_kwargs"]["prior_balances"], preview) # type: ignore[index] - self.assertEqual( - preview_bundle.payout_artifact_generation, - artifact.generation, - ) - self.assertEqual(ledger.prior_balance_reads, 0) - - payout_generation = server._payout_state_generation - ledger.database_balances = [dict(balance) for balance in preview] - server._clear_accepted_block_payout_preview(parent_hash) - self.assertEqual(server._payout_state_generation, payout_generation) - self.assertIs(server._payout_ledger_artifact, artifact) - self.assertNotIn(parent_hash, server._accepted_block_payout_previews) - self.assertNotIn( - parent_hash, - server._invalidated_accepted_block_payout_previews, - ) - with server._job_cache_lock: - server._job_bundle_cache.clear() - - post_clear_bundle = server.shared_job_bundle(artifacts, mode="ready") - - self.assertEqual(post_clear_bundle.prior_balances, preview) - self.assertEqual(recorded["last_kwargs"]["prior_balances"], preview) # type: ignore[index] - self.assertEqual( - post_clear_bundle.payout_artifact_generation, - artifact.generation, - ) - self.assertEqual(ledger.prior_balance_reads, 0) - - def test_valid_precomputed_artifact_skips_tip_path_ledger_snapshot(self) -> None: - server, rpc = coordinator() - install_fake_bundle_builder(server) - snapshot = server.fetch_qbit_tip_template_snapshot() - assert snapshot.template_artifacts is not None - server._prepare_payout_ledger_artifact( - server._payout_state_generation, - snapshot.template_artifacts.network_difficulty, - ) - server.ledger.snapshot_calls = 0 - - bundle = server.prepare_tip_refresh_bundle(snapshot) - - self.assertFalse(bundle.collection_only) - self.assertGreater(bundle.payout_artifact_generation, 0) - self.assertEqual(server.ledger.snapshot_calls, 0) - - def test_mismatched_precomputed_artifact_falls_back_to_inline_snapshot(self) -> None: - server, rpc = coordinator() - install_fake_bundle_builder(server) - artifacts = server.store_template_artifacts(dict(rpc.template)) - assert artifacts is not None - server._current_payout_state_artifact() - server._prepare_payout_ledger_artifact( - server._payout_state_generation, - artifacts.network_difficulty, - ) - with server._job_cache_lock: - published = server._published_payout_state - assert published.artifact is not None - changed_balances = [{"miner_id": "miner-z", "balance_sats": 1}] - server._published_payout_state = dataclass_replace( - published, - artifact=dataclass_replace( - published.artifact, - prior_balances_json=canonical_json_text(changed_balances), - prior_balances_sha256=canonical_json_sha256(changed_balances), - ), - ) - - self.assertIsNone( - server._usable_payout_ledger_artifact( - server._payout_state_generation, - artifacts.network_difficulty, - ) - ) - self.assertIsNone(server._payout_ledger_artifact) - server.ledger.snapshot_calls = 0 - - bundle = server.shared_job_bundle(artifacts, mode="ready") - - self.assertEqual(bundle.payout_artifact_generation, 0) - self.assertEqual(server.ledger.snapshot_calls, 1) - - def test_sync_ledger_snapshot_publishes_reusable_artifact(self) -> None: - server, rpc = coordinator() - install_fake_bundle_builder(server) - first = server.store_template_artifacts(dict(rpc.template)) - assert first is not None - # Pin distinct per-generation anchors near the live clock so the - # armed window stays anchored inside the audit ceiling. - first_anchor_ms = now_ms() - 20 - with server._job_cache_lock: - server._job_build_issued_at_ms[first.generation] = first_anchor_ms - - inline = server.shared_job_bundle(first, mode="ready") - - self.assertEqual(inline.payout_artifact_generation, 0) - self.assertEqual(server.ledger.snapshot_calls, 1) - with server._job_cache_lock: - artifact = server._payout_ledger_artifact - self.assertIsNotNone(artifact) - assert artifact is not None - self.assertGreater(artifact.generation, 0) - self.assertEqual( - artifact.payout_state_generation, - server._payout_state_generation, - ) - self.assertEqual( - artifact.accepted_share_count, - len(server.ledger.miners), - ) - self.assertEqual(list(artifact.shares_json), inline.shares_json) - self.assertEqual(artifact.snapshot_anchor_ms, inline.issued_at_ms) - - second = server.store_template_artifacts( - base_template(height=11, prevhash="22" * 32) - ) - assert second is not None - with server._job_cache_lock: - server._job_build_issued_at_ms[second.generation] = ( - first_anchor_ms + 10 - ) - - reused = server.shared_job_bundle(second, mode="ready") - - self.assertEqual(reused.payout_artifact_generation, artifact.generation) - self.assertEqual(server.ledger.snapshot_calls, 1) - self.assertEqual(reused.shares_json, inline.shares_json) - self.assertNotEqual(reused.issued_at_ms, inline.issued_at_ms) - self.assertEqual( - reused.found_block["anchor_job_issued_at_ms"], - artifact.snapshot_anchor_ms, - ) - - def test_sync_artifact_publication_survives_share_commit_mid_read( - self, - ) -> None: - class MidReadCommitLedger(FakeLedger): - def snapshot_at_job_issue( - self, - anchor_job_issued_at_ms: int, - *, - window_weight: int | None = None, - ) -> list[FakeShare]: - result = super().snapshot_at_job_issue( - anchor_job_issued_at_ms, - window_weight=window_weight, - ) - # A share becomes durable while the window read is in - # flight. The read is scoped by the frozen anchor -- the - # commit lands above it and belongs to the next window -- so - # anchor-scoped publication proceeds. - self.miners = [*self.miners, f"mid-read-{self.snapshot_calls}"] - return result - - server, rpc = coordinator(ledger=MidReadCommitLedger()) - install_fake_bundle_builder(server) - artifacts = server.store_template_artifacts(dict(rpc.template)) - assert artifacts is not None - - bundle = server.shared_job_bundle(artifacts, mode="ready") - - self.assertFalse(bundle.collection_only) - self.assertEqual(bundle.payout_artifact_generation, 0) - with server._job_cache_lock: - artifact = server._payout_ledger_artifact - self.assertIsNotNone(artifact) - assert artifact is not None - self.assertEqual(artifact.snapshot_anchor_ms, bundle.issued_at_ms) - self.assertEqual(list(artifact.shares_json), bundle.shares_json) - - # The armed window keeps serving under continuous commits: the live - # durable count has moved past the artifact's, and reuse must not - # care. - second = server.store_template_artifacts( - base_template(height=11, prevhash="22" * 32) - ) - assert second is not None - reused = server.shared_job_bundle(second, mode="ready") - self.assertEqual(reused.payout_artifact_generation, artifact.generation) - self.assertEqual(server.ledger.snapshot_calls, 1) - self.assertEqual( - reused.found_block["anchor_job_issued_at_ms"], - artifact.snapshot_anchor_ms, - ) - - def test_sync_publication_allowed_when_anchor_predates_durable_share( - self, - ) -> None: - server, rpc = coordinator() - install_fake_bundle_builder(server) - artifacts = server.store_template_artifacts(dict(rpc.template)) - assert artifacts is not None - - first = server.shared_job_bundle(artifacts, mode="ready") - self.assertFalse(first.collection_only) - with server._job_cache_lock: - self.assertIsNotNone(server._payout_ledger_artifact) - # A share becomes durable after this generation's anchor was - # frozen. Force the synchronous path to run again for the same - # frozen anchor. - server._payout_ledger_artifact = None - server._job_bundle_cache.clear() - server.ledger.miners = [*server.ledger.miners, "post-anchor-share"] - - second = server.shared_job_bundle(artifacts, mode="ready") - - self.assertEqual(second.payout_artifact_generation, 0) - # The window read at the frozen anchor deterministically excludes - # the later durable share -- it belongs to the next window -- so the - # anchor-scoped publication proceeds even though the live count has - # moved past the frozen anchor. - with server._job_cache_lock: - republished = server._payout_ledger_artifact - self.assertIsNotNone(republished) - assert republished is not None - self.assertEqual(republished.snapshot_anchor_ms, second.issued_at_ms) - self.assertEqual(second.issued_at_ms, first.issued_at_ms) - - def test_rearmed_artifact_is_not_shadowed_by_stale_cached_bundle( - self, - ) -> None: - server, rpc = coordinator() - recorded = install_fake_bundle_builder(server) - artifacts = server.store_template_artifacts(dict(rpc.template)) - assert artifacts is not None - - stale = server.shared_job_bundle(artifacts, mode="ready") - self.assertEqual(recorded["calls"], 1) - self.assertEqual(len(stale.shares_json), 3) - - # A share lands and the debounced re-arm rebuilds a fresher window. - server.ledger.miners = [*server.ledger.miners, "late-share"] - server._prepare_payout_ledger_artifact( - server._payout_state_generation, - artifacts.network_difficulty, - ) - with server._job_cache_lock: - fresh_artifact = server._payout_ledger_artifact - assert fresh_artifact is not None - self.assertEqual(fresh_artifact.accepted_share_count, 4) - - rebuilt = server.shared_job_bundle(artifacts, mode="ready") - - # The pre-re-arm bundle is still inside its cache TTL under the - # no-artifact key; its older window must not shadow the fresher - # artifact. - self.assertEqual( - rebuilt.payout_artifact_generation, - fresh_artifact.generation, - ) - self.assertEqual(len(rebuilt.shares_json), 4) - self.assertEqual(recorded["calls"], 2) - - def test_racing_commit_does_not_disable_sync_artifact_seeding(self) -> None: - class RacingStatsLedger(FakeLedger): - def __init__(self) -> None: - super().__init__( - miners=["miner-a", "miner-b", "miner-c", "miner-d"] - ) - self._pending_counts = iter([3, 4]) - - def accepted_share_stats(self) -> dict[str, int]: - self.stats_calls += 1 - try: - count = next(self._pending_counts) - except StopIteration: - count = 4 - return { - "accepted_share_count": count, - "distinct_miner_count": 4, - } - - server, rpc = coordinator(ledger=RacingStatsLedger()) - install_fake_bundle_builder(server) - artifacts = server.store_template_artifacts(dict(rpc.template)) - assert artifacts is not None - - # A commit races the informational count read. The window is scoped - # by the frozen anchor regardless of which side of the read the - # commit landed on, so seeding proceeds; the count is diagnostics, - # not a fence. - bundle = server.shared_job_bundle(artifacts, mode="ready") - - self.assertFalse(bundle.collection_only) - with server._job_cache_lock: - artifact = server._payout_ledger_artifact - self.assertIsNotNone(artifact) - assert artifact is not None - self.assertEqual(artifact.snapshot_anchor_ms, bundle.issued_at_ms) - self.assertEqual(list(artifact.shares_json), bundle.shares_json) - - def test_same_window_background_rebuild_keeps_artifact_generation( - self, - ) -> None: - server, rpc = coordinator() - install_fake_bundle_builder(server) - artifacts = server.store_template_artifacts(dict(rpc.template)) - assert artifacts is not None - - server.shared_job_bundle(artifacts, mode="ready") - with server._job_cache_lock: - installed = server._payout_ledger_artifact - assert installed is not None - - # No share committed since: the speculative rebuild reads the same - # window under a fresh anchor and must not spin the generation, - # which would re-key bundle lookups for nothing -- but it is still a - # successful preparation, so an accumulated re-arm backoff releases - # and the fresher anchor advances the staleness clock in place. - with server._payout_artifact_executor_lock: - server._payout_artifact_rearm_backoff = 4 - server._prepare_payout_ledger_artifact( - server._payout_state_generation, - artifacts.network_difficulty, - ) - with server._job_cache_lock: - refreshed = server._payout_ledger_artifact - assert refreshed is not None - self.assertEqual(refreshed.generation, installed.generation) - self.assertEqual( - refreshed.share_snapshot_sha256, - installed.share_snapshot_sha256, - ) - assert installed.snapshot_anchor_ms is not None - assert refreshed.snapshot_anchor_ms is not None - self.assertGreaterEqual( - int(refreshed.snapshot_anchor_ms), - int(installed.snapshot_anchor_ms), - ) - with server._payout_artifact_executor_lock: - self.assertEqual(server._payout_artifact_rearm_backoff, 1) - - # A new durable share makes the rebuild a genuine replacement. - server.ledger.miners = [*server.ledger.miners, "late-share"] - server._prepare_payout_ledger_artifact( - server._payout_state_generation, - artifacts.network_difficulty, - ) - with server._job_cache_lock: - replaced = server._payout_ledger_artifact - assert replaced is not None - self.assertGreater(replaced.generation, installed.generation) - self.assertEqual(replaced.accepted_share_count, 4) - - def test_delayed_older_snapshot_does_not_regress_artifact(self) -> None: - server, rpc = coordinator() - install_fake_bundle_builder(server) - artifacts = server.store_template_artifacts(dict(rpc.template)) - assert artifacts is not None - server.shared_job_bundle(artifacts, mode="ready") - with server._job_cache_lock: - current = server._payout_ledger_artifact - assert current is not None - assert current.snapshot_anchor_ms is not None - - # A snapshot taken at an earlier anchor finishes its window - # conversion late, so its preparation timestamp is newer than the - # installed artifact's; the anchor still proves it is the older - # window. - older_window = [ - {"share_seq": seq, "miner_id": "miner-a"} for seq in (1, 2) - ] - delayed = PayoutLedgerArtifact( - generation=0, - payout_state_generation=server._payout_state_generation, - network_difficulty=int(current.network_difficulty), - accepted_share_count=2, - shares_json=tuple(older_window), - prior_balances=(), - prepared_monotonic=time.monotonic(), - snapshot_anchor_ms=int(current.snapshot_anchor_ms) - 5, - share_snapshot_sha256=canonical_json_sha256(older_window), - ) - server._install_payout_ledger_artifact(delayed) - with server._job_cache_lock: - self.assertIs(server._payout_ledger_artifact, current) - - # A window snapshotted at a fresher anchor replaces regardless of - # its preparation timestamp ordering. - newer_window = [ - {"share_seq": seq, "miner_id": "miner-a"} for seq in (1, 2, 3, 4) - ] - fresher = dataclass_replace( - delayed, - accepted_share_count=4, - shares_json=tuple(newer_window), - prepared_monotonic=current.prepared_monotonic - 1.0, - snapshot_anchor_ms=int(current.snapshot_anchor_ms) + 5, - share_snapshot_sha256=canonical_json_sha256(newer_window), - ) - server._install_payout_ledger_artifact(fresher) - with server._job_cache_lock: - replaced = server._payout_ledger_artifact - assert replaced is not None - self.assertGreater(replaced.generation, current.generation) - self.assertEqual(replaced.accepted_share_count, 4) - - def test_difficulty_change_replaces_same_window_artifact(self) -> None: - server, rpc = coordinator() - install_fake_bundle_builder(server) - artifacts = server.store_template_artifacts(dict(rpc.template)) - assert artifacts is not None - server.shared_job_bundle(artifacts, mode="ready") - with server._job_cache_lock: - installed = server._payout_ledger_artifact - assert installed is not None - - # A retarget moves the live template to a new difficulty while the - # share window stays identical. The rebuild at the live difficulty - # must replace the artifact: keeping the old-difficulty artifact - # would fail the reuse fence on every build while the same-window - # skip kept discarding its replacement. - retarget_difficulty = int(installed.network_difficulty) + 1 - with server._job_cache_lock: - live = server._template_artifacts - assert live is not None - server._template_artifacts = dataclass_replace( - live, - network_difficulty=retarget_difficulty, - ) - server._prepare_payout_ledger_artifact( - server._payout_state_generation, - retarget_difficulty, - ) - with server._job_cache_lock: - replaced = server._payout_ledger_artifact - assert replaced is not None - self.assertGreater(replaced.generation, installed.generation) - self.assertEqual(replaced.network_difficulty, retarget_difficulty) - - # A pre-retarget build delayed after its snapshot carries the same - # window at the old difficulty; whether its anchor trails or ties, - # the live-difficulty artifact must stay. - delayed = dataclass_replace( - installed, - generation=0, - prepared_monotonic=time.monotonic(), - ) - server._install_payout_ledger_artifact(delayed) - with server._job_cache_lock: - self.assertIs(server._payout_ledger_artifact, replaced) - - # Even a delayed pre-retarget build that clamped a fresher anchor - # must not displace it: a wrong-difficulty install would fail every - # reuse probe on the difficulty check, which never re-arms. - assert replaced.snapshot_anchor_ms is not None - leading = dataclass_replace( - installed, - generation=0, - prepared_monotonic=time.monotonic(), - snapshot_anchor_ms=int(replaced.snapshot_anchor_ms) + 5, - ) - server._install_payout_ledger_artifact(leading) - with server._job_cache_lock: - self.assertIs(server._payout_ledger_artifact, replaced) - - def test_fence_failure_rearms_artifact_preparation_with_debounce(self) -> None: - server, rpc = coordinator() - install_fake_bundle_builder(server) - artifacts = server.store_template_artifacts(dict(rpc.template)) - assert artifacts is not None - server.payout_artifact_rearm_min_seconds = 5.0 - server._prepare_payout_ledger_artifact( - server._payout_state_generation, - artifacts.network_difficulty, - ) - - class RecordingExecutor: - def __init__(self) -> None: - self.submissions = 0 - - def submit(self, _fn: object) -> Future[None]: - # Recorded but never run: the request slot stays observable. - self.submissions += 1 - return Future() - - executor = RecordingExecutor() - with server._payout_artifact_executor_lock: - server._payout_artifact_executor = executor # type: ignore[assignment] - - usable = server._usable_payout_ledger_artifact( - server._payout_state_generation, - artifacts.network_difficulty, - ) - self.assertIsNotNone(usable) - with server._payout_artifact_executor_lock: - self.assertIsNone(server._payout_artifact_requested) - - # Age the armed window past the re-anchor floor: the probe keeps - # serving the artifact while the debounced background re-anchor is - # scheduled. - server.payout_artifact_reanchor_seconds = 10.0 - with server._job_cache_lock: - armed = server._payout_ledger_artifact - assert armed is not None - assert armed.snapshot_anchor_ms is not None - server._payout_ledger_artifact = dataclass_replace( - armed, - snapshot_anchor_ms=int(armed.snapshot_anchor_ms) - 11_000, - ) - - self.assertIsNotNone( - server._usable_payout_ledger_artifact( - server._payout_state_generation, - artifacts.network_difficulty, - ) - ) - with server._payout_artifact_executor_lock: - self.assertEqual( - server._payout_artifact_requested, - ( - server._payout_state_generation, - int(artifacts.network_difficulty), - ), - ) - self.assertEqual(executor.submissions, 1) - # Every past-floor probe counts, even when the rearm itself is - # debounced -- the canary needs "serving past the floor" to be - # visible independently of rearm_scheduled. - self.assertEqual( - server.payout_artifact_event_counts["probe_past_floor"], - 1, - ) - # The one-slot worker dequeues the request; a served probe inside - # the interval must not slip a duplicate into the emptied slot. - server._payout_artifact_requested = None - - self.assertIsNotNone( - server._usable_payout_ledger_artifact( - server._payout_state_generation, - artifacts.network_difficulty, - ) - ) - with server._payout_artifact_executor_lock: - self.assertIsNone(server._payout_artifact_requested) - - # Once the interval has elapsed the next served probe re-arms. - with server._payout_artifact_executor_lock: - server._payout_artifact_last_schedule_monotonic = ( - time.monotonic() - 6.0 - ) - self.assertIsNotNone( - server._usable_payout_ledger_artifact( - server._payout_state_generation, - artifacts.network_difficulty, - ) - ) - with server._payout_artifact_executor_lock: - self.assertEqual( - server._payout_artifact_requested, - ( - server._payout_state_generation, - int(artifacts.network_difficulty), - ), - ) - - def test_aborted_speculative_rebuilds_back_off_and_reset(self) -> None: - ledger = AnchorRecordingLedger() - server, rpc = coordinator(ledger=ledger) - install_fake_bundle_builder(server) - artifacts = server.store_template_artifacts(dict(rpc.template)) - assert artifacts is not None - server.payout_artifact_rearm_min_seconds = 5.0 - - # A pending-commit floor older than the audit anchor ceiling aborts - # the rebuild before the window walk (the artifact would arm born - # expired); each abort doubles the re-arm interval instead of - # retrying the reward-window walk at the floor forever. - share = stamped_pending_share(now_ms() - 400_000) - server._ensure_pending_share_commit_state() - with server._pending_share_commit_lock: - server._pending_share_commit_floor[id(share)] = [ - share, - time.monotonic(), - False, - ] - server._prepare_payout_ledger_artifact(0, artifacts.network_difficulty) - with server._payout_artifact_executor_lock: - self.assertEqual(server._payout_artifact_rearm_backoff, 2) - server._prepare_payout_ledger_artifact(0, artifacts.network_difficulty) - with server._payout_artifact_executor_lock: - self.assertEqual(server._payout_artifact_rearm_backoff, 4) - self.assertEqual(ledger.snapshot_calls, 0) - - # Elapsed time beyond the floor but inside the scaled interval must - # not re-arm. - with server._payout_artifact_executor_lock: - server._payout_artifact_last_schedule_monotonic = ( - time.monotonic() - 10.0 - ) - server._rearm_payout_ledger_artifact_after_fence_failure( - 0, - artifacts.network_difficulty, - ) - with server._payout_artifact_executor_lock: - self.assertIsNone(server._payout_artifact_requested) - - # A rebuild that finally arms resets the backoff to the floor. - server._finish_pending_share_commit(share) - server._prepare_payout_ledger_artifact(0, artifacts.network_difficulty) - with server._job_cache_lock: - self.assertIsNotNone(server._payout_ledger_artifact) - with server._payout_artifact_executor_lock: - self.assertEqual(server._payout_artifact_rearm_backoff, 1) - - def test_payout_publication_resets_rearm_backoff(self) -> None: - server, rpc = coordinator() - install_fake_bundle_builder(server) - artifacts = server.store_template_artifacts(dict(rpc.template)) - assert artifacts is not None - server._pool_ready_latched = True - with server._payout_artifact_executor_lock: - server._payout_artifact_rearm_backoff = 8 - - # The accepted-block preview publishes a new payout generation whose - # candidate artifact installs through the atomic publication pointer - # swap; the accumulated backoff must release with it. - parent_hash = str(rpc.template["previousblockhash"]) - server._begin_accepted_block_payout_preview( - parent_hash, - block_height=int(rpc.template["height"]) - 1, - ) - server._publish_accepted_block_payout_preview( - parent_hash, - [ - { - "recipient_id": "miner-a", - "order_key": "miner-a", - "p2mr_program_hex": "11" * 32, - "balance_sats": 25, - } - ], - ) - - self.assertGreater(server._payout_state_generation, 0) - with server._job_cache_lock: - self.assertIsNotNone(server._payout_ledger_artifact) - with server._payout_artifact_executor_lock: - self.assertEqual(server._payout_artifact_rearm_backoff, 1) - - def test_landed_preview_suppresses_fence_failure_rearm(self) -> None: - server, rpc = coordinator() - install_fake_bundle_builder(server) - artifacts = server.store_template_artifacts(dict(rpc.template)) - assert artifacts is not None - server._prepare_payout_ledger_artifact( - server._payout_state_generation, - artifacts.network_difficulty, - ) - scheduled: list[tuple[int, int]] = [] - server._schedule_payout_ledger_artifact_preparation = ( # type: ignore[method-assign] - lambda generation, difficulty, **_kwargs: scheduled.append( - (generation, difficulty) - ) - ) - accepted_hash = "d0" * 32 - server._begin_accepted_block_payout_preview( - accepted_hash, - block_height=int(rpc.template["height"]) - 1, - ) - server._mark_accepted_block_payout_landed( - accepted_hash, - block_height=int(rpc.template["height"]) - 1, - ) - - # Age the armed window past the re-anchor floor: the probe keeps - # serving while attempting the background re-anchor. - server.payout_artifact_reanchor_seconds = 10.0 - with server._job_cache_lock: - armed = server._payout_ledger_artifact - assert armed is not None - assert armed.snapshot_anchor_ms is not None - server._payout_ledger_artifact = dataclass_replace( - armed, - snapshot_anchor_ms=int(armed.snapshot_anchor_ms) - 11_000, - ) - - self.assertIsNotNone( - server._usable_payout_ledger_artifact( - server._payout_state_generation, - artifacts.network_difficulty, - ) - ) - # A speculative rebuild would read database balances the landed - # prospective state supersedes; preparation resumes only through the - # durable-confirmation call site. - self.assertEqual(scheduled, []) - - def test_artifact_reuse_serves_past_install_age_to_anchor_ceiling( - self, - ) -> None: - server, rpc = coordinator() - install_fake_bundle_builder(server) - artifacts = server.store_template_artifacts(dict(rpc.template)) - assert artifacts is not None - server._prepare_payout_ledger_artifact( - server._payout_state_generation, - artifacts.network_difficulty, - ) - with server._job_cache_lock: - armed = server._payout_ledger_artifact - assert armed is not None - assert armed.snapshot_anchor_ms is not None - # The balances digest is memoized at construction so the serving - # probe and the reused-build fences never re-canonicalize the - # tuple on the request path. - self.assertEqual( - armed.prior_balances_sha256, - canonical_json_sha256(armed.prior_balances), - ) - - # A window whose anchor is past the re-anchor floor still serves -- - # post-anchor shares belong to the next window by construction, and - # validity is carried by the generation/balances fences, never by - # wall-clock age below the ceiling. (The re-anchor and - # anchor-ceiling env knobs wire these same attributes in __init__.) - server.payout_artifact_reanchor_seconds = 10.0 - server.ledger.miners = [*server.ledger.miners, "late-1", "late-2"] - with server._job_cache_lock: - server._payout_ledger_artifact = dataclass_replace( - armed, - snapshot_anchor_ms=int(armed.snapshot_anchor_ms) - 15_000, - ) - self.assertIsNotNone( - server._usable_payout_ledger_artifact( - server._payout_state_generation, - artifacts.network_difficulty, - rearm_on_fence_failure=False, - ) - ) - - # 2026-07-30 brownout regression: install age must never retire the - # window. A 10s install budget at the production build cadence (a - # shared build every 10-14s, a 4-8s walk) expired between - # consecutive builds, sending every build back through the - # synchronous reward-window walk. Only payout events and the audit - # ceiling end reuse. - with server._job_cache_lock: - server._payout_ledger_artifact = dataclass_replace( - armed, - prepared_monotonic=float(armed.prepared_monotonic) - 3600.0, - ) - self.assertIsNotNone( - server._usable_payout_ledger_artifact( - server._payout_state_generation, - artifacts.network_difficulty, - rearm_on_fence_failure=False, - ) - ) - - # The wall-clock anchor ceiling stays as the audit-facing backstop. - server.payout_artifact_max_anchor_age_seconds = 5.0 - with server._job_cache_lock: - server._payout_ledger_artifact = dataclass_replace( - armed, - snapshot_anchor_ms=int(armed.snapshot_anchor_ms) - 6_000, - ) - self.assertIsNone( - server._usable_payout_ledger_artifact( - server._payout_state_generation, - artifacts.network_difficulty, - rearm_on_fence_failure=False, - ) - ) - - def test_reuse_kill_switch_disables_probe_and_background_walks( - self, - ) -> None: - server, rpc = coordinator() - install_fake_bundle_builder(server) - artifacts = server.store_template_artifacts(dict(rpc.template)) - assert artifacts is not None - server._prepare_payout_ledger_artifact( - server._payout_state_generation, - artifacts.network_difficulty, - ) - with server._job_cache_lock: - assert server._payout_ledger_artifact is not None - - # Off: the probe refuses a perfectly valid armed artifact and the - # scheduler runs no background reward-window walks -- the disabled - # state restores pre-reuse delivery economics (the synchronous - # path keeps the re-landed anchor-selection semantics), so a - # delivery regression is an env flip instead of a rollback. - server.payout_artifact_reuse_enabled = False - self.assertIsNone( - server._usable_payout_ledger_artifact( - server._payout_state_generation, - artifacts.network_difficulty, - ) - ) - server._schedule_payout_ledger_artifact_preparation( - server._payout_state_generation, - artifacts.network_difficulty, - ) - with server._payout_artifact_executor_lock: - self.assertIsNone(server._payout_artifact_requested) - self.assertIsNone(server._payout_artifact_future) - - # Back on: the armed artifact serves again without a rebuild. - server.payout_artifact_reuse_enabled = True - self.assertIsNotNone( - server._usable_payout_ledger_artifact( - server._payout_state_generation, - artifacts.network_difficulty, - ) - ) - - # Off also gates arming itself: a prepared window is discarded at - # install, so nothing churns install events or re-keys the idle - # bundle fast path to an unusable generation while disabled. - server.payout_artifact_reuse_enabled = False - with server._job_cache_lock: - server._payout_ledger_artifact = None - server._prepare_payout_ledger_artifact( - server._payout_state_generation, - artifacts.network_difficulty, - ) - with server._job_cache_lock: - self.assertIsNone(server._payout_ledger_artifact) - - def test_anchor_ceiling_must_clear_reanchor_floor(self) -> None: - # One env var used to be able to re-create the 2026-07-30 incident: - # a ceiling the background re-anchor cannot beat (floor + debounce + - # the window walk) rejects every probe while the rebuild runs - # debounced in parallel -- walk-per-build. __init__ refuses the - # configuration at startup through this validation. - with self.assertRaises(SystemExit): - prism_coordinator_module.validate_payout_artifact_age_bounds( - 60.0, - 60.0, - ) - with self.assertRaises(SystemExit): - prism_coordinator_module.validate_payout_artifact_age_bounds( - 120.0, - 60.0, - ) - # The same margin applies to a custom build cadence; validating only - # the re-anchor floor allowed 299s builds under a 300s ceiling. - with self.assertRaises(SystemExit): - prism_coordinator_module.validate_payout_artifact_age_bounds( - 300.0, - 299.0, - ) - # The default split (300s ceiling, 60s floor) passes. - prism_coordinator_module.validate_payout_artifact_age_bounds( - 300.0, - 60.0, - ) - - def test_in_flight_build_keeps_aged_artifact_it_already_selected( - self, - ) -> None: - server, rpc = coordinator() - install_fake_bundle_builder(server) - artifacts = server.store_template_artifacts(dict(rpc.template)) - assert artifacts is not None - server._prepare_payout_ledger_artifact( - server._payout_state_generation, - artifacts.network_difficulty, - ) - server.payout_artifact_max_anchor_age_seconds = 5.0 - with server._job_cache_lock: - armed = server._payout_ledger_artifact - assert armed is not None - assert armed.snapshot_anchor_ms is not None - aged = dataclass_replace( - armed, - snapshot_anchor_ms=int(armed.snapshot_anchor_ms) - 6_000, - ) - server._payout_ledger_artifact = aged - - # New reuse decisions reject a window past the audit ceiling... - self.assertIsNone( - server._usable_payout_ledger_artifact( - server._payout_state_generation, - artifacts.network_difficulty, - rearm_on_fence_failure=False, - ) - ) - # ...but a build that selected it while fresh completes with it: the - # in-build re-validation checks supersession and the balances fence - # only, so queue delay past the bound cannot scrap the reuse into - # the full snapshot it was armed to avoid. - server.ledger.snapshot_calls = 0 - bundle = server.build_shared_job_bundle( - artifacts, - worker(), - payout_artifact=aged, - ) - self.assertEqual(server.ledger.snapshot_calls, 0) - self.assertEqual( - bundle.found_block["anchor_job_issued_at_ms"], - aged.snapshot_anchor_ms, - ) - - def test_in_flight_build_survives_same_window_anchor_refresh(self) -> None: - server, rpc = coordinator() - install_fake_bundle_builder(server) - artifacts = server.store_template_artifacts(dict(rpc.template)) - assert artifacts is not None - server._prepare_payout_ledger_artifact( - server._payout_state_generation, - artifacts.network_difficulty, - ) - with server._job_cache_lock: - armed = server._payout_ledger_artifact - assert armed is not None - assert armed.snapshot_anchor_ms is not None - # An in-flight build selected the armed artifact while it - # carried an older anchor. - selected = dataclass_replace( - armed, - snapshot_anchor_ms=int(armed.snapshot_anchor_ms) - 5, - ) - server._payout_ledger_artifact = selected - - # A same-window rebuild refreshes the anchor in place: the stored - # instance swaps while the generation -- the re-key authority -- - # and the window bytes stay identical. - server._prepare_payout_ledger_artifact( - server._payout_state_generation, - artifacts.network_difficulty, - ) - with server._job_cache_lock: - refreshed = server._payout_ledger_artifact - assert refreshed is not None - self.assertIsNot(refreshed, selected) - self.assertEqual(refreshed.generation, selected.generation) - self.assertEqual( - refreshed.share_snapshot_sha256, - selected.share_snapshot_sha256, - ) - - # The build holding the pre-refresh instance completes its reuse - # instead of being scrapped into a full snapshot by the swap. - server.ledger.snapshot_calls = 0 - bundle = server.build_shared_job_bundle( - artifacts, - worker(), - payout_artifact=selected, - ) - self.assertEqual(server.ledger.snapshot_calls, 0) - self.assertEqual( - bundle.found_block["anchor_job_issued_at_ms"], - selected.snapshot_anchor_ms, - ) - - def test_generation_current_cached_bundle_serves_past_reanchor_floor( - self, - ) -> None: - server, rpc = coordinator() - recorded = install_fake_bundle_builder(server) - first = server.store_template_artifacts(dict(rpc.template)) - assert first is not None - with server._payout_artifact_executor_lock: - server._payout_artifact_executor_shutdown = True - # Re-anchor floor below the bundle-cache TTL so the two gates are - # distinguishable inside the TTL window. - server.payout_artifact_reanchor_seconds = 5.0 - - server.shared_job_bundle(first, mode="ready") - self.assertEqual(recorded["calls"], 1) - with server._job_cache_lock: - armed = server._payout_ledger_artifact - assert armed is not None - - # A bundle keyed to the currently armed artifact generation carries - # exactly the armed window: it keeps serving past the re-anchor - # floor (its window is as fresh as reuse itself), bounded upstream - # by the cache TTL. - second = server.store_template_artifacts( - base_template(height=11, prevhash="22" * 32) - ) - assert second is not None - reused = server.shared_job_bundle(second, mode="ready") - self.assertEqual(reused.payout_artifact_generation, armed.generation) - self.assertEqual(recorded["calls"], 2) - with server._job_cache_lock: - for key, entry in list(server._job_bundle_cache.items()): - if entry.payout_artifact_generation == armed.generation: - server._job_bundle_cache[key] = dataclass_replace( - entry, - built_monotonic=float(entry.built_monotonic) - 7.0, - ) - server.shared_job_bundle(second, mode="ready") - self.assertEqual(recorded["calls"], 2) - - def test_cached_no_artifact_bundle_gates_on_build_age_not_anchor( - self, - ) -> None: - # 2026-07-29 regression companion: the gate must follow build age - # (freshness of the work) inside the audit ceiling, never the - # wall-clock anchor, which is frozen per template generation. - server, rpc = coordinator() - recorded = install_fake_bundle_builder(server) - artifacts = server.store_template_artifacts(dict(rpc.template)) - assert artifacts is not None - with server._payout_artifact_executor_lock: - server._payout_artifact_executor_shutdown = True - server.payout_artifact_reanchor_seconds = 5.0 - - seeded = server.shared_job_bundle(artifacts, mode="ready") - self.assertEqual(recorded["calls"], 1) - - # Fresh build age serves from cache once the armed artifact is - # gone. - with server._job_cache_lock: - server._payout_ledger_artifact = None - server.shared_job_bundle(artifacts, mode="ready") - self.assertEqual(recorded["calls"], 1) - - # Build age past the re-anchor floor rebuilds even though the - # bundle-cache TTL has not lapsed. - with server._job_cache_lock: - server._payout_ledger_artifact = None - for key, entry in list(server._job_bundle_cache.items()): - server._job_bundle_cache[key] = dataclass_replace( - entry, - built_monotonic=float(entry.built_monotonic) - 7.0, - ) - server.shared_job_bundle(artifacts, mode="ready") - self.assertEqual(recorded["calls"], 2) - - # The audit ceiling on the declared anchor stays as the backstop - # even for a freshly built entry. - aged_anchor_ms = ( - int(seeded.found_block["anchor_job_issued_at_ms"]) - 400_000 - ) - with server._job_cache_lock: - server._payout_ledger_artifact = None - for key, entry in list(server._job_bundle_cache.items()): - aged_found_block = dict(entry.found_block) - aged_found_block["anchor_job_issued_at_ms"] = aged_anchor_ms - server._job_bundle_cache[key] = dataclass_replace( - entry, - found_block=aged_found_block, - ) - server.shared_job_bundle(artifacts, mode="ready") - self.assertEqual(recorded["calls"], 3) - - def test_slow_window_walk_still_arms_reusable_artifact(self) -> None: - # 2026-07-29 regression: the reward-window walk takes multiple - # seconds at production volume, and freshness measured from the - # wall-clock anchor rejected every slow build on arrival. The walk - # here advances the wall clock past the freshness budget; the - # artifact must still arm, serve reuse, and reset the re-arm - # backoff. - clock = [now_ms()] - - class SlowWalkLedger(FakeLedger): - def snapshot_at_job_issue( - self, - anchor_job_issued_at_ms: int, - *, - window_weight: int | None = None, - ) -> list[FakeShare]: - result = super().snapshot_at_job_issue( - anchor_job_issued_at_ms, - window_weight=window_weight, - ) - clock[0] += 15_000 - return result - - server, rpc = coordinator(ledger=SlowWalkLedger()) - install_fake_bundle_builder(server) - artifacts = server.store_template_artifacts(dict(rpc.template)) - assert artifacts is not None - with server._payout_artifact_executor_lock: - server._payout_artifact_rearm_backoff = 4 - with patch( - "lab.prism.prism_coordinator.now_ms", - side_effect=lambda: clock[0], - ): - server._prepare_payout_ledger_artifact( - server._payout_state_generation, - artifacts.network_difficulty, - ) - with server._job_cache_lock: - armed = server._payout_ledger_artifact - self.assertIsNotNone(armed) - assert armed is not None - assert armed.snapshot_anchor_ms is not None - self.assertGreater( - clock[0] - int(armed.snapshot_anchor_ms), - 10_000, - ) - self.assertIs( - server._usable_payout_ledger_artifact( - server._payout_state_generation, - artifacts.network_difficulty, - ), - armed, - ) - with server._payout_artifact_executor_lock: - self.assertEqual(server._payout_artifact_rearm_backoff, 1) - metrics = server.metrics_payload() - self.assertIn( - 'qbit_prism_payout_artifact_events_total{event="installed"} 1', - metrics, - ) - self.assertIn( - 'qbit_prism_payout_artifact_events_total{event="built"} 1', - metrics, - ) - self.assertIn( - 'qbit_prism_payout_artifact_events_total{event="incremental"}', - metrics, - ) - self.assertIn( - 'qbit_prism_payout_artifact_events_total{event="self_check_failed"}', - metrics, - ) - self.assertIn( - 'qbit_prism_payout_artifact_events_total{event="found_block_cached"}', - metrics, - ) - - def test_born_expired_install_paces_backoff_instead_of_resetting( - self, - ) -> None: - # A walk that outlives the audit ceiling arms nothing. Crediting it - # as a success reset the re-arm backoff to the floor and re-walked - # the reward window continuously -- the 2026-07-29 livelock. It - # must pace exactly like a failed preparation. - clock = [now_ms()] - - class GlacialWalkLedger(FakeLedger): - def snapshot_at_job_issue( - self, - anchor_job_issued_at_ms: int, - *, - window_weight: int | None = None, - ) -> list[FakeShare]: - result = super().snapshot_at_job_issue( - anchor_job_issued_at_ms, - window_weight=window_weight, - ) - clock[0] += 400_000 - return result - - server, rpc = coordinator(ledger=GlacialWalkLedger()) - install_fake_bundle_builder(server) - artifacts = server.store_template_artifacts(dict(rpc.template)) - assert artifacts is not None - with patch( - "lab.prism.prism_coordinator.now_ms", - side_effect=lambda: clock[0], - ): - server._prepare_payout_ledger_artifact( - server._payout_state_generation, - artifacts.network_difficulty, - ) - with server._job_cache_lock: - self.assertIsNone(server._payout_ledger_artifact) - with server._payout_artifact_executor_lock: - self.assertEqual(server._payout_artifact_rearm_backoff, 2) - server._prepare_payout_ledger_artifact( - server._payout_state_generation, - artifacts.network_difficulty, - ) - with server._payout_artifact_executor_lock: - self.assertEqual(server._payout_artifact_rearm_backoff, 4) - with server._payout_artifact_executor_lock: - self.assertEqual( - server.payout_artifact_event_counts["born_expired"], - 2, - ) - - def test_equal_anchor_reprove_credits_freshness(self) -> None: - # A pending-commit floor pins the snapshot anchor, so a fence-failure - # rebuild re-proves the same window at the SAME anchor. That re-prove - # must advance the freshness clock: no fresher window is - # constructible while the floor holds, and rejecting the credit - # would re-walk the reward window (with backoff reset each round) - # until the audit ceiling -- the livelock shape this re-land - # removes. - server, rpc = coordinator() - install_fake_bundle_builder(server) - artifacts = server.store_template_artifacts(dict(rpc.template)) - assert artifacts is not None - share = stamped_pending_share(now_ms() - 5) - server._ensure_pending_share_commit_state() - with server._pending_share_commit_lock: - server._pending_share_commit_floor[id(share)] = [ - share, - time.monotonic(), - False, - ] - try: - server._prepare_payout_ledger_artifact( - server._payout_state_generation, - artifacts.network_difficulty, - ) - with server._job_cache_lock: - armed = server._payout_ledger_artifact - assert armed is not None - server._payout_ledger_artifact = dataclass_replace( - armed, - prepared_monotonic=float(armed.prepared_monotonic) - 11.0, - ) - # Install age no longer gates the probe (2026-07-30 brownout - # regression); the armed window keeps serving while pinned. - self.assertIsNotNone( - server._usable_payout_ledger_artifact( - server._payout_state_generation, - artifacts.network_difficulty, - rearm_on_fence_failure=False, - ) - ) - - server._prepare_payout_ledger_artifact( - server._payout_state_generation, - artifacts.network_difficulty, - ) - with server._job_cache_lock: - reproved = server._payout_ledger_artifact - assert reproved is not None - self.assertEqual(reproved.generation, armed.generation) - self.assertEqual( - reproved.snapshot_anchor_ms, - armed.snapshot_anchor_ms, - ) - self.assertIsNotNone( - server._usable_payout_ledger_artifact( - server._payout_state_generation, - artifacts.network_difficulty, - ) - ) - finally: - server._finish_pending_share_commit(share) - - def test_publication_restamps_candidate_artifact_freshness(self) -> None: - # A payout-state candidate builds its artifact before the atomic - # publication, and the delivery-gate drain between the two can - # outlive the reuse budget. The install must restamp freshness so a - # freshly published generation never arms an already-stale artifact. - server, rpc = coordinator() - install_fake_bundle_builder(server) - artifacts = server.store_template_artifacts(dict(rpc.template)) - assert artifacts is not None - server._pool_ready_latched = True - real_build = server._build_payout_ledger_artifact - - def slow_publication_build(*args: object) -> object: - built = real_build(*args) - if built is None: - return None - return dataclass_replace( - built, - prepared_monotonic=time.monotonic() - 11.0, - ) - - server._build_payout_ledger_artifact = slow_publication_build # type: ignore[method-assign] - parent_hash = str(rpc.template["previousblockhash"]) - server._begin_accepted_block_payout_preview( - parent_hash, - block_height=int(rpc.template["height"]) - 1, - ) - server._publish_accepted_block_payout_preview( - parent_hash, - [ - { - "recipient_id": "miner-a", - "order_key": "miner-a", - "p2mr_program_hex": "11" * 32, - "balance_sats": 25, - } - ], - ) - - self.assertGreater(server._payout_state_generation, 0) - usable = server._usable_payout_ledger_artifact( - server._payout_state_generation, - artifacts.network_difficulty, - ) - self.assertIsNotNone(usable) - - def test_usable_probe_survives_equal_window_freshness_restamp( - self, - ) -> None: - # An equal-window re-prove restamps the armed artifact by replacing - # the object while a reuse probe is hashing balances outside the - # cache lock. The probe must treat the restamped copy as the same - # armed window instead of failing closed into a synchronous - # reward-window walk: under a pinned pending-commit floor the - # restamp IS the intentional recovery path, so the race is routine. - server, rpc = coordinator() - install_fake_bundle_builder(server) - artifacts = server.store_template_artifacts(dict(rpc.template)) - assert artifacts is not None - server._prepare_payout_ledger_artifact( - server._payout_state_generation, - artifacts.network_difficulty, - ) - with server._job_cache_lock: - armed = server._payout_ledger_artifact - assert armed is not None - # Strip the memoized balances digest: the race window this test - # drives opens while the probe hashes balances outside the cache - # lock, which only happens on the fallback path for artifacts - # without a memo (legacy or test-constructed). The restamp - # admission below must keep protecting that path. - armed = dataclass_replace(armed, prior_balances_sha256=None) - server._payout_ledger_artifact = armed - - real_sha256 = prism_coordinator_module.canonical_json_sha256 - restamped_once = [False] - - def restamp_during_hash(value: object) -> str: - if not restamped_once[0]: - restamped_once[0] = True - self.assertTrue( - server._install_payout_ledger_artifact( - dataclass_replace( - armed, - generation=0, - prepared_monotonic=time.monotonic(), - ) - ) - ) - return real_sha256(value) - - with patch.object( - prism_coordinator_module, - "canonical_json_sha256", - restamp_during_hash, - ): - usable = server._usable_payout_ledger_artifact( - server._payout_state_generation, - artifacts.network_difficulty, - ) - self.assertTrue(restamped_once[0]) - with server._job_cache_lock: - restamped = server._payout_ledger_artifact - assert restamped is not None - self.assertIsNot(restamped, armed) - self.assertIsNotNone(usable) - self.assertIs(usable, restamped) - self.assertEqual(restamped.generation, armed.generation) - - def test_publication_discards_born_expired_candidate_artifact( - self, - ) -> None: - # Candidate construction plus the delivery-gate drain can push the - # candidate artifact's declared anchor past the audit ceiling. The - # atomic publication must apply the same born-expired admission rule - # as _install_payout_ledger_artifact: arming the artifact would fail - # every reuse probe on anchor age -- with the re-arm suppressed - # while the accepted preview awaits durability -- instead of letting - # the post-publication probe schedule recovery. - server, rpc = coordinator() - install_fake_bundle_builder(server) - artifacts = server.store_template_artifacts(dict(rpc.template)) - assert artifacts is not None - server._pool_ready_latched = True - real_build = server._build_payout_ledger_artifact - ceiling_ms = server._payout_artifact_max_anchor_age_ms() - - def expired_anchor_build(*args: object) -> object: - built = real_build(*args) - if built is None: - return None - assert built.snapshot_anchor_ms is not None - return dataclass_replace( - built, - snapshot_anchor_ms=int(built.snapshot_anchor_ms) - - int(ceiling_ms) - - 1_000, - ) - - server._build_payout_ledger_artifact = expired_anchor_build # type: ignore[method-assign] - parent_hash = str(rpc.template["previousblockhash"]) - server._begin_accepted_block_payout_preview( - parent_hash, - block_height=int(rpc.template["height"]) - 1, - ) - generation_before = server._payout_state_generation - server._publish_accepted_block_payout_preview( - parent_hash, - [ - { - "recipient_id": "miner-a", - "order_key": "miner-a", - "p2mr_program_hex": "11" * 32, - "balance_sats": 25, - } - ], - ) - self.assertGreater(server._payout_state_generation, generation_before) - with server._job_cache_lock: - self.assertIsNone(server._payout_ledger_artifact) - with server._payout_artifact_executor_lock: - born_expired = int( - server.payout_artifact_event_counts.get("born_expired", 0) - ) - self.assertGreaterEqual(born_expired, 1) - self.assertIsNone( - server._usable_payout_ledger_artifact( - server._payout_state_generation, - artifacts.network_difficulty, - ) - ) - - def test_publication_records_installed_event(self) -> None: - # The atomic publication arms the candidate artifact through its own - # pointer swap, not _install_payout_ledger_artifact. It must still - # count in the installed lifecycle event family, or every - # publication-path install is invisible to the observability the - # event counter exists for. - server, rpc = coordinator() - install_fake_bundle_builder(server) - artifacts = server.store_template_artifacts(dict(rpc.template)) - assert artifacts is not None - server._pool_ready_latched = True - with server._payout_artifact_executor_lock: - installed_before = int( - server.payout_artifact_event_counts.get("installed", 0) - ) - parent_hash = str(rpc.template["previousblockhash"]) - server._begin_accepted_block_payout_preview( - parent_hash, - block_height=int(rpc.template["height"]) - 1, - ) - server._publish_accepted_block_payout_preview( - parent_hash, - [ - { - "recipient_id": "miner-a", - "order_key": "miner-a", - "p2mr_program_hex": "11" * 32, - "balance_sats": 25, - } - ], - ) - with server._job_cache_lock: - self.assertIsNotNone(server._payout_ledger_artifact) - with server._payout_artifact_executor_lock: - installed_after = int( - server.payout_artifact_event_counts.get("installed", 0) - ) - self.assertEqual(installed_after, installed_before + 1) - - def test_generation_bump_does_not_scrap_in_flight_build(self) -> None: - # 2026-07-29 regression: every differing window bumps the armed - # artifact generation, and the in-build re-validation scrapped any - # build whose selected artifact lost the slot -- a rebuild storm at - # production share rates. A build must finish on the copy it - # selected: its window stays audit-reproducible at its declared - # anchor while the payout generation and published balances hold. - server, rpc = coordinator() - install_fake_bundle_builder(server) - artifacts = server.store_template_artifacts(dict(rpc.template)) - assert artifacts is not None - server._prepare_payout_ledger_artifact( - server._payout_state_generation, - artifacts.network_difficulty, - ) - with server._job_cache_lock: - selected = server._payout_ledger_artifact - assert selected is not None - assert selected.snapshot_anchor_ms is not None - - fresher_window = [ - {"share_seq": seq, "miner_id": "miner-a"} for seq in (1, 2, 3, 4) - ] - fresher = dataclass_replace( - selected, - generation=0, - accepted_share_count=4, - shares_json=tuple(fresher_window), - snapshot_anchor_ms=int(selected.snapshot_anchor_ms) + 5, - share_snapshot_sha256=canonical_json_sha256(fresher_window), - ) - self.assertTrue(server._install_payout_ledger_artifact(fresher)) - with server._job_cache_lock: - bumped = server._payout_ledger_artifact - assert bumped is not None - self.assertGreater(bumped.generation, selected.generation) - - server.ledger.snapshot_calls = 0 - bundle = server.build_shared_job_bundle( - artifacts, - worker(), - payout_artifact=selected, - ) - self.assertEqual(server.ledger.snapshot_calls, 0) - self.assertEqual( - bundle.found_block["anchor_job_issued_at_ms"], - selected.snapshot_anchor_ms, - ) - self.assertEqual(bundle.shares_json, list(selected.shares_json)) - - def test_rebuilt_bundle_serves_under_aged_template_generation( - self, - ) -> None: - # 2026-07-29 regression: issued_at_ms is frozen per template - # generation and predates the walk, so a wall-clock anchor gate on - # cached bundles declared every rebuilt bundle dead on arrival once - # its template generation outlived the bound. Below the audit - # ceiling, cache service must follow build age, not anchor age. - server, rpc = coordinator() - recorded = install_fake_bundle_builder(server) - artifacts = server.store_template_artifacts(dict(rpc.template)) - assert artifacts is not None - with server._payout_artifact_executor_lock: - server._payout_artifact_executor_shutdown = True - with server._job_cache_lock: - server._job_build_issued_at_ms[artifacts.generation] = ( - now_ms() - 20_000 - ) - - built = server.shared_job_bundle(artifacts, mode="ready") - self.assertEqual(recorded["calls"], 1) - self.assertEqual( - built.found_block["anchor_job_issued_at_ms"], - built.issued_at_ms, - ) - - served = server.shared_job_bundle(artifacts, mode="ready") - self.assertEqual(recorded["calls"], 1) - self.assertEqual(served.issued_at_ms, built.issued_at_ms) - - def test_admission_deadline_fails_fast_and_visibly(self) -> None: - server, _rpc = coordinator() - server.routine_admission_deadline_seconds = 0.05 - server._publication_priority_scheduled_locked = ( # type: ignore[method-assign] - lambda: True - ) - - with self.assertRaises(JobBuildAdmissionDeadlineExceeded) as raised: - server._begin_routine_job_build_preparation( - request_source="initial", - cancelled=None, - ) - - # The deadline is coordination churn, not a failure budget event: - # callers retry with their normal pacing. - self.assertIsInstance(raised.exception, TemplateRefreshBlocked) - with server._job_build_scheduler_lock: - self.assertEqual( - server.initial_job_prepared_work_counts["admission_deadline"], - 1, - ) - - def test_reused_anchor_bundle_preview_matches_fresh_build_through_guard( - self, - ) -> None: - # The #50 payout-preview guard compares the preview computed from - # the issued job against one recomputed at landing. A reused-anchor - # bundle must therefore reproduce, byte for byte, what a fresh - # ledger read at the same anchor produces -- the audit - # reproducibility contract the artifact's snapshot_anchor_ms - # declaration documents. - programs = { - "miner-a": "aa" * 32, - "miner-b": "bb" * 32, - "miner-c": "cc" * 32, - "miner-d": "dd" * 32, - } - - class AnchorScopedLedger(FakeLedger): - """Window reads respect the anchor like the real ledger.""" - - def __init__(self) -> None: - super().__init__(miners=[]) - self.stamped: list[tuple[int, FakeShare]] = [] - - def add_share(self, miner_id: str, accepted_at_ms: int) -> None: - self.stamped.append( - ( - int(accepted_at_ms), - FakeShare( - miner_id=miner_id, - share_seq=len(self.stamped) + 1, - ), - ) - ) - - def accepted_share_stats(self) -> dict[str, int]: - self.stats_calls += 1 - return { - "accepted_share_count": len(self.stamped), - "distinct_miner_count": len( - {share.miner_id for _, share in self.stamped} - ), - } - - def snapshot_at_job_issue( - self, - anchor_job_issued_at_ms: int, - *, - window_weight: int | None = None, - ) -> list[FakeShare]: - self.snapshot_calls += 1 - return [ - share - for stamp, share in self.stamped - if stamp <= int(anchor_job_issued_at_ms) - ] - - anchor_ms = now_ms() - 50 - ledger = AnchorScopedLedger() - ledger.add_share("miner-a", anchor_ms - 30) - ledger.add_share("miner-b", anchor_ms - 20) - ledger.add_share("miner-c", anchor_ms - 10) - server, rpc = coordinator(ledger=ledger) - - def preview_bundle_builder(**kwargs: object) -> dict[str, object]: - suffix_hex = str(kwargs["coinbase_script_sig_suffix_hex"]) - weights: dict[str, int] = {} - for share in kwargs["shares"]: # type: ignore[union-attr] - miner = str(share["miner_id"]) # type: ignore[index] - weights[miner] = weights.get(miner, 0) + 1 - return { - "found_block": dict(kwargs["found_block"]), # type: ignore[call-overload] - "payout_policy_manifest": { - "accounts": [ - { - "account_type": "miner", - "recipient_id": miner, - "order_key": miner, - "p2mr_program_hex": programs[miner], - "carry_forward_balance_sats": 1_000 * weight, - } - for miner, weight in sorted(weights.items()) - ] - }, - "signed_coinbase_manifest": { - "manifest": { - "coinbase_tx_hex": synthetic_manifest_coinbase_hex( - suffix_hex - ), - } - }, - } - - server.build_audit_bundle = preview_bundle_builder # type: ignore[method-assign] - - first = server.store_template_artifacts(dict(rpc.template)) - assert first is not None - with server._job_cache_lock: - server._job_build_issued_at_ms[first.generation] = anchor_ms - - seeded_bundle = server.shared_job_bundle(first, mode="ready") - self.assertEqual(seeded_bundle.payout_artifact_generation, 0) - with server._job_cache_lock: - artifact = server._payout_ledger_artifact - assert artifact is not None - self.assertEqual(artifact.snapshot_anchor_ms, anchor_ms) - - # Shares keep landing after the anchor; the durable count moves past - # the artifact's, which must not matter for reuse. - ledger.add_share("miner-a", anchor_ms + 20) - ledger.add_share("miner-d", anchor_ms + 25) - - second = server.store_template_artifacts( - base_template(height=11, prevhash="22" * 32) - ) - assert second is not None - reused = server.shared_job_bundle(second, mode="ready") - self.assertEqual(reused.payout_artifact_generation, artifact.generation) - self.assertEqual( - reused.found_block["anchor_job_issued_at_ms"], - anchor_ms, - ) - - # Control: a fresh synchronous build pinned to the same anchor pays - # its own window read and must reproduce the reused output exactly, - # excluding the post-anchor shares deterministically. - with server._job_cache_lock: - server._payout_ledger_artifact = None - server._job_bundle_cache.clear() - third = server.store_template_artifacts( - base_template(height=12, prevhash="33" * 32) - ) - assert third is not None - with server._job_cache_lock: - server._job_build_issued_at_ms[third.generation] = anchor_ms - snapshot_calls_before = ledger.snapshot_calls - fresh = server.shared_job_bundle(third, mode="ready") - self.assertEqual(ledger.snapshot_calls, snapshot_calls_before + 1) - self.assertEqual(fresh.payout_artifact_generation, 0) - - self.assertEqual(reused.shares_json, fresh.shares_json) - self.assertEqual( - fresh.found_block["anchor_job_issued_at_ms"], - anchor_ms, - ) - assert reused.prospective_prior_balances is not None - assert fresh.prospective_prior_balances is not None - self.assertEqual( - canonical_json_text(list(reused.prospective_prior_balances)), - canonical_json_text(list(fresh.prospective_prior_balances)), - ) - - # A window at a fresher anchor covers the post-anchor shares and - # produces a different preview; built now to prove the guard - # equality below is load-bearing. - with server._job_cache_lock: - server._payout_ledger_artifact = None - server._job_bundle_cache.clear() - fourth = server.store_template_artifacts( - base_template(height=13, prevhash="44" * 32) - ) - assert fourth is not None - with server._job_cache_lock: - server._job_build_issued_at_ms[fourth.generation] = anchor_ms + 30 - divergent = server.shared_job_bundle(fourth, mode="ready") - assert divergent.prospective_prior_balances is not None - self.assertNotEqual( - canonical_json_text(list(divergent.prospective_prior_balances)), - canonical_json_text(list(reused.prospective_prior_balances)), - ) - - # The #50 guard sequence at landing: the issued preview (from the - # reused-anchor job) publishes first; the verified preview - # recomputed at the same anchor must then publish idempotently. - block_hash = "d1" * 32 - server._begin_accepted_block_payout_preview( - block_hash, - block_height=int(rpc.template["height"]), - ) - issued = server._materialize_prior_balance_preview( - reused.prospective_prior_balances - ) - server._publish_accepted_block_payout_preview(block_hash, issued) - generation_after_issued = server._payout_state_generation - verified = server._materialize_prior_balance_preview( - fresh.prospective_prior_balances - ) - server._publish_accepted_block_payout_preview(block_hash, verified) - self.assertEqual( - server._payout_state_generation, - generation_after_issued, - ) - with server._accepted_block_payout_preview_condition: - transition = server._accepted_block_payout_previews[block_hash] - self.assertEqual( - transition.preview, - server._serialize_prior_balance_preview(issued), - ) - - # A preview from any other anchor's window trips the guard exactly - # as submit_block_candidate's landing rebuild would. - with self.assertRaisesRegex(RuntimeError, "changed during retry"): - server._publish_accepted_block_payout_preview( - block_hash, - server._materialize_prior_balance_preview( - divergent.prospective_prior_balances - ), - ) - - def test_new_tip_cancels_blocked_old_bundle_without_publication(self) -> None: - old_tip = "11" * 32 - new_tip = "22" * 32 - server, rpc = coordinator(template=base_template(prevhash=old_tip)) - recorded = install_fake_bundle_builder(server) - original_builder = server.build_audit_bundle - build_started = threading.Event() - - def cancelable_builder(**kwargs: object) -> dict[str, object]: - control = server._job_build_phase_local.bundle_build_control - build_started.set() - self.assertTrue(control.cancel_event.wait(2)) - return original_builder(**kwargs) - - server.build_audit_bundle = cancelable_builder # type: ignore[method-assign] - artifacts = server.store_template_artifacts(dict(rpc.template)) - assert artifacts is not None - server.observe_tip_first_seen(old_tip, observation_sequence=1) - errors: list[BaseException] = [] - thread = threading.Thread( - target=lambda: self._capture_error( - errors, - lambda: server.shared_job_bundle(artifacts, mode="ready"), - ) - ) - thread.start() - self.assertTrue(build_started.wait(2)) - - server.observe_tip_first_seen(new_tip, observation_sequence=2) - thread.join(2) - - self.assertFalse(thread.is_alive()) - self.assertEqual(len(errors), 1) - self.assertIsInstance(errors[0], TemplateRefreshBlocked) - self.assertEqual(recorded["calls"], 1) - self.assertEqual(server._active_job_bundle_builds, {}) - self.assertEqual(server.tip_refresh_build_inflight, 0) - self.assertFalse(any( - entry.template_fingerprint == artifacts.fingerprint - for entry in server._job_bundle_cache.values() - )) - self.assertEqual(server.tip_refresh_superseded_results, 1) - - def test_builder_crash_and_timeout_fail_closed_then_recover(self) -> None: - server, _rpc = coordinator() - server.prism_ctv_settlement_config = lambda **_kwargs: None # type: ignore[method-assign] - server.signing_seed_hex = "42" * 32 - server.ledger_attestation_signing_seed_hex = "43" * 32 - build_kwargs = { - "shares": [], - "found_block": { - "block_height": 10, - "coinbase_value_sats": 50_00000000, - "network_difficulty": 1, - "anchor_job_issued_at_ms": 1_700_000_000_000, - }, - "prior_balances": [], - "coinbase_script_sig_suffix_hex": "00", - } - - with patch( - "lab.prism.prism_coordinator.prism_tool_command", - return_value=[sys.executable, "-c", "raise SystemExit(7)"], - ): - with self.assertRaisesRegex(RuntimeError, "failed"): - server.build_audit_bundle(**build_kwargs) - - server.bundle_build_timeout_seconds = 0.01 - with patch( - "lab.prism.prism_coordinator.prism_tool_command", - return_value=[ - sys.executable, - "-c", - "import time; time.sleep(5)", - ], - ): - with self.assertRaisesRegex(RuntimeError, "timed out"): - server.build_audit_bundle(**build_kwargs) - - server.bundle_build_timeout_seconds = 1.0 - recovery_script = ( - "import json,sys; json.load(sys.stdin); " - "json.dump({'recovered': True}, sys.stdout)" - ) - with patch( - "lab.prism.prism_coordinator.prism_tool_command", - return_value=[sys.executable, "-c", recovery_script], - ): - recovered = server.build_audit_bundle(**build_kwargs) - self.assertEqual(recovered, {"recovered": True}) - - def test_audit_builder_child_does_not_inherit_open_socket(self) -> None: - server, _rpc = coordinator() - server.prism_ctv_settlement_config = lambda **_kwargs: None # type: ignore[method-assign] - server.signing_seed_hex = "42" * 32 - server.ledger_attestation_signing_seed_hex = "43" * 32 - probe_script = ( - "import json,os,sys; json.load(sys.stdin); fd=int(sys.argv[1]); " - "inherited=True; " - "\ntry: os.fstat(fd)" - "\nexcept OSError: inherited=False" - "\njson.dump({'inherited_socket': inherited}, sys.stdout)" - ) - with socket.socket() as parent_socket: - parent_socket.set_inheritable(True) - with patch( - "lab.prism.prism_coordinator.prism_tool_command", - return_value=[ - sys.executable, - "-c", - probe_script, - str(parent_socket.fileno()), - ], - ): - result = server.build_audit_bundle( - shares=[], - found_block={ - "block_height": 10, - "coinbase_value_sats": 50_00000000, - "network_difficulty": 1, - "anchor_job_issued_at_ms": 1_700_000_000_000, - }, - prior_balances=[], - coinbase_script_sig_suffix_hex="00", - ) - self.assertEqual(result, {"inherited_socket": False}) - - def test_repeated_superseded_builds_leave_state_bounded(self) -> None: - server, rpc = coordinator() - install_fake_bundle_builder(server) - original_builder = server.build_audit_bundle - starts: queue.Queue[None] = queue.Queue() - - def cancelable_builder(**kwargs: object) -> dict[str, object]: - control = server._job_build_phase_local.bundle_build_control - starts.put(None) - self.assertTrue(control.cancel_event.wait(2)) - return original_builder(**kwargs) - - server.build_audit_bundle = cancelable_builder # type: ignore[method-assign] - current_tip = str(rpc.tip) - server.observe_tip_first_seen(current_tip, observation_sequence=1) - errors: list[BaseException] = [] - for index in range(8): - rpc.template = base_template(height=10 + index, prevhash=current_tip) - artifacts = server.store_template_artifacts(dict(rpc.template)) - assert artifacts is not None - thread = threading.Thread( - target=lambda current=artifacts: self._capture_error( - errors, - lambda: server.shared_job_bundle(current, mode="ready"), - ) - ) - thread.start() - starts.get(timeout=2) - current_tip = f"{index + 2:064x}" - rpc.tip = current_tip - server.observe_tip_first_seen( - current_tip, - observation_sequence=index + 2, - ) - thread.join(2) - self.assertFalse(thread.is_alive()) - - self.assertEqual(len(errors), 8) - self.assertTrue(all(isinstance(exc, TemplateRefreshBlocked) for exc in errors)) - self.assertEqual(server._active_job_bundle_builds, {}) - self.assertEqual(server.tip_refresh_build_inflight, 0) - self.assertEqual(server.tip_refresh_build_queue_depth, 0) - self.assertLessEqual( - len(server._job_bundle_cache), - MAX_PRISM_JOB_BUNDLE_CACHE_ENTRIES, - ) - self.assertEqual(server.tip_refresh_superseded_results, 8) - - @staticmethod - def _capture_error( - errors: list[BaseException], - operation: object, - ) -> None: - try: - operation() # type: ignore[operator] - except BaseException as exc: # noqa: BLE001 - test thread handoff - errors.append(exc) - - def test_ready_tip_refresh_builds_once_and_stamps_every_client(self) -> None: - server, _ = coordinator() - recorded = install_fake_bundle_builder(server) - clients = [client(1), client(2), client(3)] - clients[1].pending_share_difficulty = Decimal("8") - sent: dict[int, list[dict[str, object]]] = {state.connection_id: [] for state in clients} - for state in clients: - state.send = ( # type: ignore[method-assign] - lambda payload, connection_id=state.connection_id: sent[connection_id].append(payload) - ) - server.clients = set(clients) - - try: - refreshed = server.poll_qbit_tip_template_once() - finally: - server.shutdown_tip_refresh_executor() - - self.assertEqual(refreshed, 3) - self.assertEqual(recorded["calls"], 1) - contexts = [state.active_job for state in clients] - self.assertEqual(len({context.job.job_id for context in contexts}), 3) - self.assertEqual( - [context.job.extranonce1_hex for context in contexts], - [state.extranonce1_hex for state in clients], - ) - self.assertEqual(contexts[1].job.share_difficulty, Decimal("8")) - self.assertEqual( - [payload["method"] for payload in sent[2]], - ["mining.set_difficulty", "mining.notify"], - ) - metrics = server.metrics_payload() - self.assertIn('qbit_prism_tip_refresh_clients_total{result="sent"} 3', metrics) - self.assertIn("qbit_prism_tip_refresh_first_delivery_seconds_count 1", metrics) - self.assertIn("qbit_prism_tip_refresh_last_delivery_seconds_count 1", metrics) - - def test_ready_tip_refresh_shares_one_bundle_across_250_clients(self) -> None: - server, rpc = coordinator() - recorded = install_fake_bundle_builder(server) - server.reorg_reconciler_enabled = True - reconciled: list[str] = [] - trust_checks = 0 - - def reconcile_once(tip_hash: str) -> bool: - reconciled.append(tip_hash) - return True - - def chain_view_untrusted() -> bool: - nonlocal trust_checks - trust_checks += 1 - return False - - server.ensure_reorg_reconciled_for_tip = reconcile_once # type: ignore[method-assign] - server.qbit_chain_view_untrusted = chain_view_untrusted # type: ignore[method-assign] - server.ensure_reorg_reconciled_for_current_tip = ( # type: ignore[method-assign] - lambda **_kwargs: self.fail("fanout repeated current-tip validation") - ) - clients = [client(index + 1) for index in range(250)] - sent: dict[int, list[dict[str, object]]] = { - state.connection_id: [] for state in clients - } - for state in clients: - state.send = ( # type: ignore[method-assign] - lambda payload, connection_id=state.connection_id: sent[ - connection_id - ].append(payload) - ) - server.clients = set(clients) - - try: - refreshed = server.poll_qbit_tip_template_once() - finally: - server.shutdown_tip_refresh_executor() - - self.assertEqual(refreshed, 250) - self.assertEqual(recorded["calls"], 1) - cached = next(iter(server._job_bundle_cache.values())) - self.assertEqual( - len({id(state.active_job.shares_json) for state in clients}), - 1, - ) - self.assertTrue( - all(state.active_job.shares_json is cached.shares_json for state in clients) - ) - self.assertEqual(reconciled, [rpc.tip]) - self.assertEqual(trust_checks, 2) - # The early priority probe, snapshot coherence, pre-fanout validation, - # and post-fanout detection are each constant-cost regardless of - # client count. - self.assertEqual(rpc.count("getbestblockhash"), 4) - self.assertTrue(all(len(payloads) == 2 for payloads in sent.values())) - fingerprints = {state.active_job.template_fingerprint for state in clients} - self.assertEqual(len(fingerprints), 1) - - def test_same_fingerprint_bundle_rebinds_exact_template_observation(self) -> None: - server, _ = coordinator() - install_fake_bundle_builder(server) - identity = worker() - first = server.store_template_artifacts(base_template()) - assert first is not None - original = server.shared_job_bundle(first, identity) - updated_template = dict(first.template) - updated_template["curtime"] = int(updated_template["curtime"]) + 30 - second = server.store_template_artifacts(updated_template) - assert second is not None - - rebound = server.shared_job_bundle(second, identity) - - self.assertEqual(first.fingerprint, second.fingerprint) - self.assertIs(rebound.template, second.template) - self.assertIsNot(rebound.template, original.template) - self.assertEqual(rebound.template_generation, second.generation) - self.assertEqual(rebound.base_job.ntime, f'{updated_template["curtime"]:08x}') - - def test_clock_only_refresh_does_not_discard_inflight_ready_build(self) -> None: - server, rpc = coordinator() - recorded = install_fake_bundle_builder(server) - first = server.store_template_artifacts(dict(rpc.template)) - assert first is not None - build_entered = threading.Event() - release_build = threading.Event() - original_build = server.build_shared_job_bundle - build_calls = 0 - build_calls_lock = threading.Lock() - - def blocking_build(*args: object, **kwargs: object) -> object: - nonlocal build_calls - with build_calls_lock: - build_calls += 1 - build_entered.set() - self.assertTrue(release_build.wait(5)) - return original_build(*args, **kwargs) # type: ignore[arg-type] - - server.build_shared_job_bundle = blocking_build # type: ignore[method-assign] - results: list[list[object]] = [[], []] - errors: list[list[BaseException]] = [[], []] - - def build(index: int, build_artifacts: object) -> None: - try: - results[index].append( - server.shared_job_bundle( # type: ignore[arg-type] - build_artifacts, - mode="ready", - ) - ) - except BaseException as exc: # noqa: BLE001 - surfaced below - errors[index].append(exc) - - first_thread = threading.Thread(target=build, args=(0, first)) - second_thread: threading.Thread | None = None - first_thread.start() - try: - self.assertTrue(build_entered.wait(5)) - updated_template = dict(first.template) - updated_template["curtime"] = int(updated_template["curtime"]) + 30 - second = server.store_template_artifacts(updated_template) - assert second is not None - second_thread = threading.Thread(target=build, args=(1, second)) - second_thread.start() - time.sleep(0.05) - with build_calls_lock: - self.assertEqual(build_calls, 1) - finally: - release_build.set() - first_thread.join(5) - if second_thread is not None: - second_thread.join(5) - - self.assertFalse(first_thread.is_alive()) - assert second_thread is not None - self.assertFalse(second_thread.is_alive()) - self.assertEqual(errors, [[], []]) - self.assertEqual([len(items) for items in results], [1, 1]) - built = results[0][0] - self.assertIs(built.template, first.template) # type: ignore[union-attr] - rebound = results[1][0] - self.assertIs(rebound.template, second.template) - self.assertEqual(rebound.template_generation, second.generation) - self.assertEqual(recorded["calls"], 1) - - def test_same_fingerprint_collection_bundle_rebuilds_exact_observation(self) -> None: - server, _ = coordinator(ledger=FakeLedger(miners=["miner-a"])) - recorded = install_fake_bundle_builder(server) - identity = worker() - first = server.store_template_artifacts(base_template()) - assert first is not None - original = server.shared_job_bundle(first, identity) - updated_template = dict(first.template) - updated_template["curtime"] = int(updated_template["curtime"]) + 30 - second = server.store_template_artifacts(updated_template) - assert second is not None - - rebuilt = server.shared_job_bundle(second, identity) - - self.assertTrue(original.collection_only) - self.assertTrue(rebuilt.collection_only) - self.assertEqual(first.fingerprint, second.fingerprint) - self.assertEqual(recorded["calls"], 2) - self.assertIsNot(rebuilt.coinbase_manifest, original.coinbase_manifest) - self.assertIs(rebuilt.template, second.template) - self.assertEqual(rebuilt.template_generation, second.generation) - - def test_job_bundle_cache_is_bounded(self) -> None: - server, _ = coordinator() - install_fake_bundle_builder(server) - artifacts = server.store_template_artifacts(base_template()) - assert artifacts is not None - bundle = server.shared_job_bundle(artifacts, worker()) - - for index in range(MAX_PRISM_JOB_BUNDLE_CACHE_ENTRIES + 5): - candidate = dataclass_replace( - bundle, - key=(artifacts.fingerprint, "test", index), - ) - server._cache_job_bundle_if_current(candidate, artifacts) - - self.assertEqual( - len(server._job_bundle_cache), - MAX_PRISM_JOB_BUNDLE_CACHE_ENTRIES, - ) - self.assertNotIn( - (artifacts.fingerprint, "test", 0), - server._job_bundle_cache, - ) - - def test_job_bundle_cache_preserves_coordinator_lock_order(self) -> None: - server, _ = coordinator() - install_fake_bundle_builder(server) - artifacts = server.store_template_artifacts(base_template()) - assert artifacts is not None - bundle = server.shared_job_bundle(artifacts, mode="ready") - observed_cache_lock = ObservedRLock() - server._job_cache_lock = observed_cache_lock # type: ignore[assignment] - errors: list[BaseException] = [] - - def cache_bundle() -> None: - try: - server._cache_job_bundle_if_current(bundle, artifacts) - except BaseException as exc: # noqa: BLE001 - surfaced below - errors.append(exc) - - observed_cache_lock.acquire() - observed_cache_lock.observe_acquires = True - cache_thread = threading.Thread(target=cache_bundle) - coordinator_lock_acquired = False - try: - cache_thread.start() - self.assertTrue(observed_cache_lock.acquire_attempted.wait(5)) - coordinator_lock_acquired = server.lock.acquire(timeout=0.25) - if coordinator_lock_acquired: - server.lock.release() - finally: - observed_cache_lock.release() - cache_thread.join(5) - - self.assertTrue(coordinator_lock_acquired) - self.assertFalse(cache_thread.is_alive()) - self.assertEqual(errors, []) - - def test_active_gap_replaces_older_pending_job_build(self) -> None: - server, rpc = coordinator() - first = server.store_template_artifacts(dict(rpc.template)) - second_template = dict(rpc.template) - second_template["coinbasevalue"] = int(second_template["coinbasevalue"]) + 1 - second = server.store_template_artifacts(second_template) - assert first is not None and second is not None - payout_generation = server._payout_state_generation - - def request_for(artifacts: object) -> object: - return server._new_job_build_request( - artifacts, # type: ignore[arg-type] - None, - mode="ready", - payout_state_generation=payout_generation, - cache_key=server._job_bundle_key( - artifacts, # type: ignore[arg-type] - mode="ready", - payout_state_generation=payout_generation, - worker=None, - ), - ) - - pending = request_for(first) - newest = request_for(second) - server._job_build_active = None - server._job_build_retiring = SimpleNamespace(request=pending) - server._job_build_pending = pending - server._start_job_build_locked = ( # type: ignore[method-assign] - lambda request: SimpleNamespace(request=request, future=None) - ) - server._arm_job_build_locked = lambda _flight: None # type: ignore[method-assign] - - promise = server._request_job_build(newest) # type: ignore[arg-type] - - self.assertIs(promise, newest.promise) # type: ignore[union-attr] - self.assertIsNone(server._job_build_pending) - assert server._job_build_active is not None - self.assertIs(server._job_build_active.request, newest) - self.assertTrue(pending.promise.done()) # type: ignore[union-attr] - self.assertIsInstance( - pending.promise.exception(), # type: ignore[union-attr] - JobBuildSuperseded, - ) - - def test_publication_critical_build_cannot_be_displaced_by_initial_work( - self, - ) -> None: - server, rpc = coordinator() - old_artifacts = server.store_template_artifacts(dict(rpc.template)) - new_artifacts = server.store_template_artifacts( - base_template(height=11, prevhash="22" * 32) - ) - assert old_artifacts is not None and new_artifacts is not None - payout_generation = server._payout_state_generation - - def request_for( - artifacts: object, - *, - publication_critical: bool, - request_source: str, - ) -> object: - return server._new_job_build_request( - artifacts, # type: ignore[arg-type] - None, - mode="ready", - payout_state_generation=payout_generation, - cache_key=server._job_bundle_key( - artifacts, # type: ignore[arg-type] - mode="ready", - payout_state_generation=payout_generation, - worker=None, - ), - publication_critical=publication_critical, - request_source=request_source, - ) - - latest = request_for( - new_artifacts, - publication_critical=True, - request_source="tip_refresh", - ) - reconnect = request_for( - old_artifacts, - publication_critical=False, - request_source="initial", - ) - same_tip_reconnect = request_for( - new_artifacts, - publication_critical=False, - request_source="initial", - ) - latest_flight = SimpleNamespace(request=latest) - server._job_build_active = latest_flight - server._job_build_retiring = None - server._job_build_pending = None - - deferred = server._request_job_build(reconnect) # type: ignore[arg-type] - coalesced = server._request_job_build( # type: ignore[arg-type] - same_tip_reconnect - ) - - self.assertIs(server._job_build_active, latest_flight) - self.assertIs(coalesced, latest.promise) # type: ignore[union-attr] - self.assertFalse(latest.cancellation.is_set()) # type: ignore[union-attr] - self.assertFalse(deferred.done()) - self.assertEqual( - server.job_build_priority_counts["routine_deferred"], - 1, - ) - self.assertEqual( - server.initial_job_prepared_work_counts["deferred"], - 1, - ) - self.assertEqual( - server.initial_job_prepared_work_counts["singleflight"], - 1, - ) - - latest.promise.set_result(object()) # type: ignore[union-attr] - with self.assertRaises(JobBuildSuperseded): - deferred.result() - - metrics = "\n".join(server.job_build_metrics_lines()) - self.assertIn( - 'qbit_prism_job_build_priority_events_total{result="routine_deferred"} 1', - metrics, - ) - self.assertIn( - 'qbit_prism_initial_job_prepared_work_total{result="deferred"} 1', - metrics, - ) - self.assertIn( - 'qbit_prism_initial_job_prepared_work_total{result="singleflight"} 1', - metrics, - ) - - def test_publication_critical_build_preempts_routine_builder_capacity( - self, - ) -> None: - server, rpc = coordinator() - old_artifacts = server.store_template_artifacts(dict(rpc.template)) - new_artifacts = server.store_template_artifacts( - base_template(height=11, prevhash="22" * 32) - ) - assert old_artifacts is not None and new_artifacts is not None - payout_generation = server._payout_state_generation - - def request_for( - artifacts: object, - *, - publication_critical: bool, - request_source: str, - ) -> object: - return server._new_job_build_request( - artifacts, # type: ignore[arg-type] - None, - mode="ready", - payout_state_generation=payout_generation, - cache_key=server._job_bundle_key( - artifacts, # type: ignore[arg-type] - mode="ready", - payout_state_generation=payout_generation, - worker=None, - ), - publication_critical=publication_critical, - request_source=request_source, - ) - - routine = request_for( - old_artifacts, - publication_critical=False, - request_source="initial", - ) - latest = request_for( - new_artifacts, - publication_critical=True, - request_source="tip_refresh", - ) - routine_flight = SimpleNamespace(request=routine) - server._job_build_active = routine_flight - server._job_build_retiring = None - server._job_build_pending = None - server._start_job_build_locked = ( # type: ignore[method-assign] - lambda request: SimpleNamespace(request=request, future=None) - ) - server._arm_job_build_locked = lambda _flight: None # type: ignore[method-assign] - - promise = server._request_job_build(latest) # type: ignore[arg-type] - - self.assertIs(promise, latest.promise) # type: ignore[union-attr] - self.assertTrue(routine.cancellation.is_set()) # type: ignore[union-attr] - self.assertIs(server._job_build_retiring, routine_flight) - assert server._job_build_active is not None - self.assertIs(server._job_build_active.request, latest) - self.assertEqual( - server.job_build_priority_counts["routine_preempted"], - 1, - ) - - def test_publication_critical_build_restarts_unhealthy_exact_flight( - self, - ) -> None: - for unhealthy in ("almost_expired", "stalled"): - with self.subTest(unhealthy=unhealthy): - server, rpc = coordinator() - artifacts = server.store_template_artifacts(dict(rpc.template)) - assert artifacts is not None - payout_generation = server._payout_state_generation - - def request_for( - *, - publication_critical: bool, - priority_requested_monotonic: float | None = None, - ) -> object: - return server._new_job_build_request( - artifacts, - None, - mode="ready", - payout_state_generation=payout_generation, - cache_key=server._job_bundle_key( - artifacts, - mode="ready", - payout_state_generation=payout_generation, - worker=None, - ), - publication_critical=publication_critical, - request_source=( - "tip_refresh" - if publication_critical - else "initial" - ), - priority_requested_monotonic=( - priority_requested_monotonic - ), - ) - - routine = request_for(publication_critical=False) - now = time.monotonic() - if unhealthy == "almost_expired": - routine.cancellation.started_monotonic = now - 59.99 - routine.cancellation.deadline_monotonic = now + 0.01 - routine.cancellation.last_checkpoint_monotonic = now - else: - routine.cancellation.started_monotonic = now - 1.0 - routine.cancellation.deadline_monotonic = now + 59.0 - routine.cancellation.last_checkpoint_monotonic = ( - now - server.job_build_cancel_grace_seconds - 0.01 - ) - priority_requested = now - 59.0 - latest = request_for( - publication_critical=True, - priority_requested_monotonic=priority_requested, - ) - routine_flight = SimpleNamespace(request=routine) - server._job_build_active = routine_flight - server._job_build_retiring = None - server._job_build_pending = None - server._start_job_build_locked = ( # type: ignore[method-assign] - lambda request: SimpleNamespace(request=request, future=None) - ) - server._arm_job_build_locked = lambda _flight: None # type: ignore[method-assign] - - promise = server._request_job_build(latest) # type: ignore[arg-type] - - self.assertIs(promise, latest.promise) - self.assertIsNot(promise, routine.promise) - self.assertTrue(routine.cancellation.is_set()) - self.assertIs(server._job_build_retiring, routine_flight) - assert server._job_build_active is not None - self.assertIs(server._job_build_active.request, latest) - self.assertEqual( - latest.requested_monotonic, - priority_requested, - ) - self.assertGreater( - latest.cancellation.deadline_monotonic - - time.monotonic(), - server.job_build_timeout_seconds - 1.0, - ) - self.assertEqual( - server.job_build_priority_counts["routine_preempted"], - 1, - ) - - def test_publication_priority_precedes_immutable_request_preparation( - self, - ) -> None: - server, rpc = coordinator() - install_fake_bundle_builder(server) - artifacts = server.store_template_artifacts(dict(rpc.template)) - assert artifacts is not None - critical_preparation_entered = threading.Event() - release_critical_preparation = threading.Event() - initial_preparation_entered = threading.Event() - original_new_request = server._new_job_build_request - - def observed_new_request(*args: object, **kwargs: object) -> object: - request_source = str(kwargs.get("request_source", "routine")) - if request_source == "tip_refresh": - critical_preparation_entered.set() - if not release_critical_preparation.wait(5): - raise AssertionError("test did not release priority preparation") - elif request_source == "initial": - initial_preparation_entered.set() - return original_new_request(*args, **kwargs) # type: ignore[arg-type] - - server._new_job_build_request = observed_new_request # type: ignore[method-assign] - results: dict[str, list[object]] = {"critical": [], "initial": []} - errors: list[BaseException] = [] - - def build(label: str, *, publication_critical: bool) -> None: - try: - results[label].append( - server.shared_job_bundle( - artifacts, - mode="ready", - publication_critical=publication_critical, - request_source=( - "tip_refresh" if publication_critical else "initial" - ), - ) - ) - except BaseException as exc: # noqa: BLE001 - surfaced below - errors.append(exc) - - critical_thread = threading.Thread( - target=build, - args=("critical",), - kwargs={"publication_critical": True}, - ) - initial_thread = threading.Thread( - target=build, - args=("initial",), - kwargs={"publication_critical": False}, - ) - critical_thread.start() - try: - self.assertTrue(critical_preparation_entered.wait(5)) - initial_thread.start() - self.assertFalse(initial_preparation_entered.wait(0.1)) - metrics = "\n".join(server.job_build_metrics_lines()) - self.assertIn("qbit_prism_job_build_priority_active 1", metrics) - finally: - release_critical_preparation.set() - critical_thread.join(5) - initial_thread.join(5) - - self.assertFalse(critical_thread.is_alive()) - self.assertFalse(initial_thread.is_alive()) - self.assertEqual(errors, []) - self.assertEqual([len(results[label]) for label in results], [1, 1]) - self.assertIs(results["critical"][0], results["initial"][0]) - self.assertFalse(initial_preparation_entered.is_set()) - self.assertEqual( - server.initial_job_prepared_work_counts["deferred"], - 1, - ) - self.assertEqual( - server.initial_job_prepared_work_counts["cache_hit"], - 1, - ) - self.assertEqual( - server.job_build_priority_admission_seconds["count"], - 1, - ) - self.assertGreaterEqual( - server.job_build_priority_admission_seconds["sum"], - 0.1, - ) - - def test_priority_reservation_cancels_admitted_routine_preparation( - self, - ) -> None: - server, rpc = coordinator() - install_fake_bundle_builder(server) - artifacts = server.store_template_artifacts(dict(rpc.template)) - assert artifacts is not None - routine_in_payout_lookup = threading.Event() - release_routine_lookup = threading.Event() - routine_request_constructed = threading.Event() - original_usable_artifact = server._usable_payout_ledger_artifact - original_new_request = server._new_job_build_request - routine_thread: threading.Thread - - def blocked_usable_artifact(*args: object, **kwargs: object) -> object: - if threading.current_thread() is routine_thread: - # The pre-admission cache probe also consults the payout - # artifact; only the lookup made under an admitted routine - # preparation exercises the cancellation contract here. - with server._job_build_scheduler_lock: - admitted = bool(server._job_build_routine_preparations) - if admitted: - routine_in_payout_lookup.set() - if not release_routine_lookup.wait(5): - raise AssertionError("test did not release payout lookup") - return original_usable_artifact(*args, **kwargs) # type: ignore[arg-type] - - def observed_new_request(*args: object, **kwargs: object) -> object: - if kwargs.get("request_source") == "initial": - routine_request_constructed.set() - return original_new_request(*args, **kwargs) # type: ignore[arg-type] - - server._usable_payout_ledger_artifact = blocked_usable_artifact # type: ignore[method-assign] - server._new_job_build_request = observed_new_request # type: ignore[method-assign] - errors: list[BaseException] = [] - - def build_routine() -> None: - try: - server.shared_job_bundle( - artifacts, - mode="ready", - retry_superseded=False, - request_source="initial", - ) - except BaseException as exc: # noqa: BLE001 - surfaced below - errors.append(exc) - - routine_thread = threading.Thread(target=build_routine) - routine_thread.start() - priority_token: int | None = None - try: - self.assertTrue(routine_in_payout_lookup.wait(5)) - with server._job_build_scheduler_lock: - routine_cancellations = tuple( - cancellation_ref() - for cancellation_ref in ( - server._job_build_routine_preparations.values() - ) - ) - self.assertEqual(len(routine_cancellations), 1) - self.assertIsNotNone(routine_cancellations[0]) - priority_token, _requested = ( - server._begin_job_build_priority_preparation() - ) - assert routine_cancellations[0] is not None - self.assertTrue(routine_cancellations[0].is_set()) - finally: - release_routine_lookup.set() - routine_thread.join(5) - if priority_token is not None: - server._finish_job_build_priority_preparation(priority_token) - - self.assertFalse(routine_thread.is_alive()) - self.assertEqual(len(errors), 1) - self.assertIsInstance(errors[0], JobBuildSuperseded) - self.assertFalse(routine_request_constructed.is_set()) - with server._job_build_scheduler_lock: - self.assertEqual(len(server._job_build_routine_preparations), 0) - - def test_publication_critical_collection_promotes_past_ready_work(self) -> None: - server, rpc = coordinator(ledger=FakeLedger(miners=["miner-a"])) - old_artifacts = server.store_template_artifacts(dict(rpc.template)) - new_artifacts = server.store_template_artifacts( - base_template(height=11, prevhash="22" * 32) - ) - assert old_artifacts is not None and new_artifacts is not None - payout_generation = server._payout_state_generation - ready = server._new_job_build_request( - old_artifacts, - None, - mode="ready", - payout_state_generation=payout_generation, - cache_key=server._job_bundle_key( - old_artifacts, - mode="ready", - payout_state_generation=payout_generation, - worker=None, - ), - request_source="initial", - ) - latest_identity = worker("tq1latest", "tq1latest.rig") - latest = server._new_job_build_request( - new_artifacts, - latest_identity, - mode="collection", - payout_state_generation=payout_generation, - cache_key=server._job_bundle_key( - new_artifacts, - mode="collection", - payout_state_generation=payout_generation, - worker=latest_identity, - ), - publication_critical=True, - request_source="tip_refresh", - ) - ready_flight = SimpleNamespace(request=ready) - server._job_build_active = ready_flight - server._job_build_retiring = None - server._job_build_pending = latest - armed: list[object] = [] - server._start_job_build_locked = ( # type: ignore[method-assign] - lambda request: SimpleNamespace(request=request, future=None) - ) - server._arm_job_build_locked = armed.append # type: ignore[method-assign] - - server._promote_pending_job_build_locked() - - self.assertTrue(ready.cancellation.is_set()) - self.assertIs(server._job_build_retiring, ready_flight) - self.assertIsNone(server._job_build_pending) - assert server._job_build_active is not None - self.assertIs(server._job_build_active.request, latest) - self.assertEqual(armed, [server._job_build_active]) - self.assertEqual( - server.job_build_priority_counts["routine_preempted"], - 1, - ) - - def test_routine_pending_never_promotes_over_publication_critical_flight( - self, - ) -> None: - for placement in ("active", "retiring"): - with self.subTest(placement=placement): - server, rpc = coordinator() - critical_artifacts = server.store_template_artifacts( - dict(rpc.template) - ) - routine_artifacts = server.store_template_artifacts( - base_template(height=11, prevhash="22" * 32) - ) - assert critical_artifacts is not None - assert routine_artifacts is not None - payout_generation = server._payout_state_generation - - def request_for( - artifacts: object, - *, - publication_critical: bool, - ) -> object: - return server._new_job_build_request( - artifacts, # type: ignore[arg-type] - None, - mode="ready", - payout_state_generation=payout_generation, - cache_key=server._job_bundle_key( - artifacts, # type: ignore[arg-type] - mode="ready", - payout_state_generation=payout_generation, - worker=None, - ), - publication_critical=publication_critical, - request_source=( - "tip_refresh" if publication_critical else "initial" - ), - ) - - critical = request_for( - critical_artifacts, - publication_critical=True, - ) - routine = request_for( - routine_artifacts, - publication_critical=False, - ) - critical_flight = SimpleNamespace(request=critical) - server._job_build_active = ( - critical_flight if placement == "active" else None - ) - server._job_build_retiring = ( - critical_flight if placement == "retiring" else None - ) - server._job_build_pending = routine - server._start_job_build_locked = ( # type: ignore[method-assign] - lambda _request: self.fail( - "routine pending work displaced publication-critical work" - ) - ) - - server._promote_pending_job_build_locked() - - self.assertIs(server._job_build_pending, routine) - self.assertFalse(critical.cancellation.is_set()) - self.assertIs( - ( - server._job_build_active - if placement == "active" - else server._job_build_retiring - ), - critical_flight, - ) - - def test_cancelled_ready_does_not_block_collection_promotion(self) -> None: - for placement in ("active", "retiring"): - with self.subTest(placement=placement): - server, rpc = coordinator(ledger=FakeLedger(miners=["miner-a"])) - artifacts = server.store_template_artifacts(dict(rpc.template)) - assert artifacts is not None - payout_generation = server._payout_state_generation - - def request_for( - mode: str, - identity: WorkerIdentity | None, - ) -> object: - return server._new_job_build_request( - artifacts, - identity, - mode=mode, - payout_state_generation=payout_generation, - cache_key=server._job_bundle_key( - artifacts, - mode=mode, - payout_state_generation=payout_generation, - worker=identity, - ), - ) - - ready = request_for("ready", None) - collection = request_for( - "collection", - worker("tq1collection", "tq1collection.rig"), - ) - self.assertTrue( # type: ignore[union-attr] - ready.cancellation.cancel("superseded") - ) - ready_flight = SimpleNamespace(request=ready) - if placement == "active": - server._job_build_active = ready_flight - server._job_build_retiring = None - else: - server._job_build_active = None - server._job_build_retiring = ready_flight - server._job_build_pending = collection - armed: list[object] = [] - server._start_job_build_locked = ( # type: ignore[method-assign] - lambda request: SimpleNamespace(request=request, future=None) - ) - server._arm_job_build_locked = armed.append # type: ignore[method-assign] - - server._promote_pending_job_build_locked() - - self.assertIsNone(server._job_build_pending) - assert server._job_build_active is not None - self.assertIs(server._job_build_active.request, collection) - self.assertIs(server._job_build_retiring, ready_flight) - self.assertEqual(armed, [server._job_build_active]) - - def test_immediate_collection_completion_does_not_reoccupy_slot(self) -> None: - server, rpc = coordinator(ledger=FakeLedger(miners=["miner-a"])) - artifacts = server.store_template_artifacts(dict(rpc.template)) - assert artifacts is not None - payout_generation = server._payout_state_generation - identities = [ - worker(f"tq1worker-{index}", f"tq1worker-{index}.rig") - for index in range(2) - ] - - def request_for(identity: WorkerIdentity) -> object: - return server._new_job_build_request( - artifacts, - identity, - mode="collection", - payout_state_generation=payout_generation, - cache_key=server._job_bundle_key( - artifacts, - mode="collection", - payout_state_generation=payout_generation, - worker=identity, - ), - ) - - pending = request_for(identities[0]) - incoming = request_for(identities[1]) - results: dict[int, object] = {} - - def completed_flight(request: object) -> object: - result = SimpleNamespace(request=request) - results[id(request)] = result - future: Future[object] = Future() - future.set_result(result) - return SimpleNamespace(request=request, future=future) - - server._job_build_pending = pending - server._start_job_build_locked = completed_flight # type: ignore[method-assign] - - promise = server._request_job_build(incoming) # type: ignore[arg-type] - - self.assertIs(promise.result(), results[id(incoming)]) - self.assertIs( # type: ignore[union-attr] - pending.promise.result(), - results[id(pending)], - ) - self.assertIsNone(server._job_build_active) - self.assertIsNone(server._job_build_retiring) - self.assertIsNone(server._job_build_pending) - - def test_independent_collection_workers_do_not_supersede_each_other(self) -> None: - server, rpc = coordinator(ledger=FakeLedger(miners=["miner-a"])) - recorded = install_fake_bundle_builder(server) - artifacts = server.store_template_artifacts(dict(rpc.template)) - assert artifacts is not None - identities = [ - worker(f"tq1worker-{index}", f"tq1worker-{index}.rig") - for index in range(4) - ] - entered = [threading.Event() for _identity in identities] - releases = [threading.Event() for _identity in identities] - original_build = server.build_shared_job_bundle - active_builds = 0 - max_active_builds = 0 - active_lock = threading.Lock() - - def blocking_build( - build_artifacts: object, - identity: WorkerIdentity, - **kwargs: object, - ) -> object: - nonlocal active_builds, max_active_builds - index = identities.index(identity) - with active_lock: - active_builds += 1 - max_active_builds = max(max_active_builds, active_builds) - entered[index].set() - try: - self.assertTrue(releases[index].wait(5)) - return original_build( - build_artifacts, # type: ignore[arg-type] - identity, - **kwargs, - ) - finally: - with active_lock: - active_builds -= 1 - - server.build_shared_job_bundle = blocking_build # type: ignore[method-assign] - results: list[list[object]] = [[] for _identity in identities] - errors: list[list[BaseException]] = [[] for _identity in identities] - - def build(index: int) -> None: - try: - results[index].append( - server.shared_job_bundle( - artifacts, - identities[index], - mode="collection", - ) - ) - except BaseException as exc: # noqa: BLE001 - surfaced below - errors[index].append(exc) - - threads = [ - threading.Thread(target=build, args=(index,)) - for index in range(len(identities)) - ] - threads[0].start() - try: - self.assertTrue(entered[0].wait(5)) - threads[1].start() - self.assertTrue(entered[1].wait(5)) - threads[2].start() - pending_deadline = time.monotonic() + 5 - while time.monotonic() < pending_deadline: - with server._job_build_scheduler_lock: - pending = server._job_build_pending - if pending is not None and pending.worker == identities[2]: - break - time.sleep(0.01) - else: - self.fail("third collection build was not queued") - threads[3].start() - self.assertEqual(server.job_build_scheduler_counts["starts"], 2) - releases[0].set() - self.assertTrue(entered[2].wait(5)) - releases[1].set() - self.assertTrue(entered[3].wait(5)) - self.assertEqual(server.job_build_scheduler_counts["starts"], 4) - finally: - for release in releases: - release.set() - for thread in threads: - thread.join(5) - - self.assertTrue(all(not thread.is_alive() for thread in threads)) - self.assertEqual(errors, [[], [], [], []]) - self.assertEqual([len(items) for items in results], [1, 1, 1, 1]) - self.assertEqual(recorded["calls"], 4) - self.assertLessEqual(max_active_builds, 2) - self.assertEqual(server.job_build_scheduler_counts["supersessions"], 0) - - def test_collection_independence_requires_one_immutable_cohort(self) -> None: - server, rpc = coordinator(ledger=FakeLedger(miners=["miner-a"])) - first = server.store_template_artifacts(dict(rpc.template)) - assert first is not None - second_template = dict(rpc.template) - second_template["curtime"] = int(second_template["curtime"]) + 1 - second = server.store_template_artifacts(second_template) - assert second is not None - payout_generation = server._payout_state_generation - first_worker = worker("tq1worker-1", "tq1worker-1.rig") - second_worker = worker("tq1worker-2", "tq1worker-2.rig") - - def request_for( - build_artifacts: object, - identity: WorkerIdentity, - ) -> object: - return server._new_job_build_request( - build_artifacts, # type: ignore[arg-type] - identity, - mode="collection", - payout_state_generation=payout_generation, - cache_key=server._job_bundle_key( - build_artifacts, # type: ignore[arg-type] - mode="collection", - payout_state_generation=payout_generation, - worker=identity, - ), - ) - - first_request = request_for(first, first_worker) - peer_request = request_for(first, second_worker) - newer_request = request_for(second, first_worker) - - self.assertTrue( - server._collection_job_builds_are_independent( - first_request, # type: ignore[arg-type] - peer_request, # type: ignore[arg-type] - ) - ) - self.assertFalse( - server._collection_job_builds_are_independent( - first_request, # type: ignore[arg-type] - newer_request, # type: ignore[arg-type] - ) - ) - - def test_ready_build_cancels_both_live_collection_flights(self) -> None: - server, rpc = coordinator(ledger=FakeLedger(miners=["miner-a"])) - server._ensure_tip_refresh_state() - install_fake_bundle_builder(server) - artifacts = server.store_template_artifacts(dict(rpc.template)) - assert artifacts is not None - identities = [ - worker(f"tq1worker-{index}", f"tq1worker-{index}.rig") - for index in range(2) - ] - entered = [threading.Event(), threading.Event()] - cancellation_observed = [threading.Event(), threading.Event()] - release_cancelled = threading.Event() - original_build = server.build_shared_job_bundle - - def blocking_build( - build_artifacts: object, - identity: WorkerIdentity | None, - **kwargs: object, - ) -> object: - request = kwargs["build_request"] - if identity in identities: - index = identities.index(identity) - entered[index].set() - while not request.cancellation.is_set(): # type: ignore[union-attr] - if release_cancelled.wait(0.01): - raise AssertionError("collection build was not cancelled") - cancellation_observed[index].set() - release_cancelled.wait(5) - request.cancellation.raise_if_cancelled( # type: ignore[union-attr] - "test collection hold" - ) - return original_build( - build_artifacts, # type: ignore[arg-type] - identity, - **kwargs, - ) - - server.build_shared_job_bundle = blocking_build # type: ignore[method-assign] - payout_generation = server._payout_state_generation - - def request_for( - mode: str, - identity: WorkerIdentity | None, - ) -> object: - return server._new_job_build_request( - artifacts, - identity, - mode=mode, - payout_state_generation=payout_generation, - cache_key=server._job_bundle_key( - artifacts, - mode=mode, - payout_state_generation=payout_generation, - worker=identity, - ), - ) - - collection_requests = [ - request_for("collection", identity) for identity in identities - ] - collection_promises = [] - ready_promise = None - try: - collection_promises.append( - server._request_job_build(collection_requests[0]) # type: ignore[arg-type] - ) - self.assertTrue(entered[0].wait(5)) - collection_promises.append( - server._request_job_build(collection_requests[1]) # type: ignore[arg-type] - ) - self.assertTrue(entered[1].wait(5)) - - ready_request = request_for("ready", None) - ready_promise = server._request_job_build( # type: ignore[arg-type] - ready_request - ) - self.assertTrue(cancellation_observed[0].wait(5)) - self.assertTrue(cancellation_observed[1].wait(5)) - self.assertFalse(collection_promises[0].done()) - self.assertFalse(collection_promises[1].done()) - self.assertFalse(ready_promise.done()) - self.assertEqual(server.job_build_scheduler_counts["starts"], 2) - finally: - release_cancelled.set() - - assert ready_promise is not None - ready_bundle = ready_promise.result(timeout=5) - self.assertFalse(ready_bundle.collection_only) - for request, promise in zip(collection_requests, collection_promises): - self.assertTrue(request.cancellation.is_set()) # type: ignore[union-attr] - self.assertIsInstance( - promise.exception(timeout=5), - JobBuildSuperseded, - ) - self.assertEqual(server.job_build_scheduler_counts["starts"], 3) - - def test_ready_build_cancels_retiring_only_collection_flight(self) -> None: - server, rpc = coordinator(ledger=FakeLedger(miners=["miner-a"])) - server._ensure_tip_refresh_state() - install_fake_bundle_builder(server) - artifacts = server.store_template_artifacts(dict(rpc.template)) - assert artifacts is not None - identities = [ - worker(f"tq1worker-{index}", f"tq1worker-{index}.rig") - for index in range(2) - ] - entered = [threading.Event(), threading.Event()] - release_active = threading.Event() - retiring_cancelled = threading.Event() - release_retiring = threading.Event() - original_build = server.build_shared_job_bundle - - def blocking_build( - build_artifacts: object, - identity: WorkerIdentity | None, - **kwargs: object, - ) -> object: - request = kwargs["build_request"] - if identity == identities[0]: - entered[0].set() - while not request.cancellation.is_set(): # type: ignore[union-attr] - if release_retiring.wait(0.01): - raise AssertionError("retiring build was not cancelled") - retiring_cancelled.set() - release_retiring.wait(5) - request.cancellation.raise_if_cancelled( # type: ignore[union-attr] - "test retiring-only hold" - ) - elif identity == identities[1]: - entered[1].set() - self.assertTrue(release_active.wait(5)) - return original_build( - build_artifacts, # type: ignore[arg-type] - identity, - **kwargs, - ) - - server.build_shared_job_bundle = blocking_build # type: ignore[method-assign] - payout_generation = server._payout_state_generation - - def request_for( - mode: str, - identity: WorkerIdentity | None, - ) -> object: - return server._new_job_build_request( - artifacts, - identity, - mode=mode, - payout_state_generation=payout_generation, - cache_key=server._job_bundle_key( - artifacts, - mode=mode, - payout_state_generation=payout_generation, - worker=identity, - ), - ) - - first_request = request_for("collection", identities[0]) - second_request = request_for("collection", identities[1]) - first_promise = server._request_job_build( # type: ignore[arg-type] - first_request - ) - self.assertTrue(entered[0].wait(5)) - second_promise = server._request_job_build( # type: ignore[arg-type] - second_request - ) - self.assertTrue(entered[1].wait(5)) - try: - release_active.set() - second_bundle = second_promise.result(timeout=5) - self.assertTrue(second_bundle.collection_only) - with server._job_build_scheduler_lock: - self.assertIsNone(server._job_build_active) - assert server._job_build_retiring is not None - self.assertIs( - server._job_build_retiring.request, - first_request, - ) - - ready_request = request_for("ready", None) - ready_promise = server._request_job_build( # type: ignore[arg-type] - ready_request - ) - self.assertTrue(retiring_cancelled.wait(5)) - ready_bundle = ready_promise.result(timeout=5) - self.assertFalse(ready_bundle.collection_only) - self.assertTrue(first_request.cancellation.is_set()) # type: ignore[union-attr] - finally: - release_active.set() - release_retiring.set() - - self.assertIsInstance( - first_promise.exception(timeout=5), - JobBuildSuperseded, - ) - - def test_collection_retries_do_not_supersede_ready_build(self) -> None: - server, rpc = coordinator(ledger=FakeLedger(miners=["miner-a"])) - server._ensure_tip_refresh_state() - install_fake_bundle_builder(server) - artifacts = server.store_template_artifacts(dict(rpc.template)) - assert artifacts is not None - identities = [ - worker(f"tq1worker-{index}", f"tq1worker-{index}.rig") - for index in range(2) - ] - collection_entered = [threading.Event(), threading.Event()] - collection_cancelled = [threading.Event(), threading.Event()] - release_cancelled = threading.Event() - ready_entered = threading.Event() - release_ready = threading.Event() - stop_collections = threading.Event() - ready_requests: list[object] = [] - collection_requests: list[object | None] = [None, None] - original_build = server.build_shared_job_bundle - - def blocking_build( - build_artifacts: object, - identity: WorkerIdentity | None, - **kwargs: object, - ) -> object: - request = kwargs["build_request"] - if request.mode == "ready": # type: ignore[union-attr] - ready_requests.append(request) - ready_entered.set() - self.assertTrue(release_ready.wait(5)) - elif identity in identities: - index = identities.index(identity) - collection_requests[index] = request - collection_entered[index].set() - while not request.cancellation.is_set(): # type: ignore[union-attr] - if release_cancelled.wait(0.01): - raise AssertionError("collection build was not cancelled") - collection_cancelled[index].set() - release_cancelled.wait(5) - request.cancellation.raise_if_cancelled( # type: ignore[union-attr] - "test collection retry hold" - ) - return original_build( - build_artifacts, # type: ignore[arg-type] - identity, - **kwargs, - ) - - server.build_shared_job_bundle = blocking_build # type: ignore[method-assign] - collection_results: list[list[object]] = [[], []] - collection_errors: list[list[BaseException]] = [[], []] - ready_results: list[object] = [] - ready_errors: list[BaseException] = [] - - def build_collection(index: int) -> None: - try: - collection_results[index].append( - server.shared_job_bundle( - artifacts, - identities[index], - mode="collection", - cancelled=stop_collections.is_set, - ) - ) - except BaseException as exc: # noqa: BLE001 - surfaced below - collection_errors[index].append(exc) - - def build_ready() -> None: - try: - ready_results.append( - server.shared_job_bundle( - artifacts, - mode="ready", - retry_superseded=False, - ) - ) - except BaseException as exc: # noqa: BLE001 - surfaced below - ready_errors.append(exc) - - collection_threads = [ - threading.Thread(target=build_collection, args=(index,)) - for index in range(2) - ] - ready_thread = threading.Thread(target=build_ready) - try: - collection_threads[0].start() - self.assertTrue(collection_entered[0].wait(5)) - collection_threads[1].start() - self.assertTrue(collection_entered[1].wait(5)) - ready_thread.start() - self.assertTrue(collection_cancelled[0].wait(5)) - self.assertTrue(collection_cancelled[1].wait(5)) - self.assertEqual(server.job_build_scheduler_counts["starts"], 2) - - release_cancelled.set() - self.assertTrue(ready_entered.wait(5)) - retry_deadline = time.monotonic() + 5 - while ( - server.job_build_scheduler_counts["requests"] < 5 - and time.monotonic() < retry_deadline - ): - time.sleep(0.01) - self.assertGreaterEqual( - server.job_build_scheduler_counts["requests"], - 5, - ) - self.assertEqual(server.job_build_scheduler_counts["starts"], 3) - self.assertEqual(len(ready_requests), 1) - self.assertFalse( - ready_requests[0].cancellation.is_set() # type: ignore[union-attr] - ) - - stop_collections.set() - for thread in collection_threads: - thread.join(2) - release_ready.set() - ready_thread.join(5) - finally: - stop_collections.set() - release_cancelled.set() - release_ready.set() - for thread in collection_threads: - if thread.ident is not None: - thread.join(5) - if ready_thread.ident is not None: - ready_thread.join(5) - - self.assertTrue(all(not thread.is_alive() for thread in collection_threads)) - self.assertFalse(ready_thread.is_alive()) - self.assertEqual(collection_results, [[], []]) - self.assertEqual([len(errors) for errors in collection_errors], [1, 1]) - self.assertEqual(ready_errors, []) - self.assertEqual(len(ready_results), 1) - self.assertFalse(ready_results[0].collection_only) # type: ignore[union-attr] - for request in collection_requests: - assert request is not None - self.assertTrue(request.cancellation.is_set()) # type: ignore[union-attr] - - def test_shutdown_cancels_builder_with_full_helper_input_pipe(self) -> None: - server, rpc = coordinator() - server.signing_seed_hex = "42" * 32 - server.ledger_attestation_signing_seed_hex = "43" * 32 - artifacts = server.store_template_artifacts(dict(rpc.template)) - assert artifacts is not None - payout_generation = server._payout_state_generation - request = server._new_job_build_request( - artifacts, - None, - mode="ready", - payout_state_generation=payout_generation, - cache_key=server._job_bundle_key( - artifacts, - mode="ready", - payout_state_generation=payout_generation, - worker=None, - ), - ) - helper_started = threading.Event() - helper_processes: list[subprocess.Popen[str]] = [] - real_popen = subprocess.Popen - - def capture_popen(*args: object, **kwargs: object) -> subprocess.Popen[str]: - process = real_popen(*args, **kwargs) # type: ignore[arg-type] - helper_processes.append(process) - helper_started.set() - return process - - def fill_helper_pipe(*_args: object, **kwargs: object) -> object: - build_request = kwargs["build_request"] - return server.build_audit_bundle( - shares=[], - found_block={ - "block_height": 10, - "coinbase_value_sats": 50_00000000, - "network_difficulty": 1, - "anchor_job_issued_at_ms": 1_700_000_000_000, - }, - prior_balances=[ - { - "miner_id": "pipe-filler", - "balance_sats": 1, - "padding": "x" * (4 * 1024 * 1024), - } - ], - coinbase_script_sig_suffix_hex="00", - cancellation=build_request.cancellation, # type: ignore[union-attr] - ) - - server.build_shared_job_bundle = fill_helper_pipe # type: ignore[method-assign] - shutdown_finished = threading.Event() - shutdown_errors: list[BaseException] = [] - - def shutdown() -> None: - try: - server.shutdown_job_build_executor() - except BaseException as exc: # noqa: BLE001 - surfaced below - shutdown_errors.append(exc) - finally: - shutdown_finished.set() - - with patch( - "lab.prism.prism_coordinator.prism_tool_command", - return_value=[sys.executable, "-c", "import time; time.sleep(30)"], - ), patch( - "lab.prism.prism_coordinator.subprocess.Popen", - side_effect=capture_popen, - ): - promise = server._request_job_build(request) - self.assertTrue( - helper_started.wait(5), - repr(promise.exception(timeout=1)) if promise.done() else None, - ) - time.sleep(0.1) - self.assertFalse(promise.done()) - shutdown_thread = threading.Thread(target=shutdown) - shutdown_thread.start() - shutdown_returned = shutdown_finished.wait(2) - if not shutdown_returned: - for process in helper_processes: - if process.poll() is None: - process.kill() - shutdown_thread.join(5) - - self.assertTrue(shutdown_returned) - self.assertFalse(shutdown_thread.is_alive()) - self.assertEqual(shutdown_errors, []) - self.assertIsInstance(promise.exception(timeout=1), JobBuildSuperseded) - - def test_control_cancel_during_serialization_is_supersession(self) -> None: - server, rpc = coordinator() - server.signing_seed_hex = "42" * 32 - server.ledger_attestation_signing_seed_hex = "43" * 32 - artifacts = server.store_template_artifacts(dict(rpc.template)) - assert artifacts is not None - helper_started = threading.Event() - helper_processes: list[subprocess.Popen[str]] = [] - real_popen = subprocess.Popen - - def capture_popen(*args: object, **kwargs: object) -> subprocess.Popen[str]: - process = real_popen(*args, **kwargs) # type: ignore[arg-type] - helper_processes.append(process) - helper_started.set() - return process - - def fill_helper_pipe(*_args: object, **kwargs: object) -> object: - build_request = kwargs["build_request"] - return server.build_audit_bundle( - shares=[], - found_block={ - "block_height": 10, - "coinbase_value_sats": 50_00000000, - "network_difficulty": 1, - "anchor_job_issued_at_ms": 1_700_000_000_000, - }, - prior_balances=[ - { - "miner_id": "pipe-filler", - "balance_sats": 1, - "padding": "x" * (4 * 1024 * 1024), - } - ], - coinbase_script_sig_suffix_hex="00", - cancellation=build_request.cancellation, # type: ignore[union-attr] - ) - - server.build_shared_job_bundle = fill_helper_pipe # type: ignore[method-assign] - errors: list[BaseException] = [] - - def build() -> None: - try: - server.shared_job_bundle( - artifacts, - mode="ready", - retry_superseded=False, - ) - except BaseException as exc: # noqa: BLE001 - surfaced below - errors.append(exc) - - with patch( - "lab.prism.prism_coordinator.prism_tool_command", - return_value=[sys.executable, "-c", "import time; time.sleep(30)"], - ), patch( - "lab.prism.prism_coordinator.subprocess.Popen", - side_effect=capture_popen, - ): - build_thread = threading.Thread(target=build) - build_thread.start() - self.assertTrue(helper_started.wait(5)) - with server._job_cache_lock: - controls = list(server._active_job_bundle_builds.values()) - self.assertEqual(len(controls), 1) - controls[0].cancel_event.set() - build_thread.join(2) - if build_thread.is_alive(): - for process in helper_processes: - if process.poll() is None: - process.kill() - build_thread.join(5) - - self.assertFalse(build_thread.is_alive()) - self.assertEqual(len(errors), 1) - self.assertIsInstance(errors[0], JobBuildSuperseded) - self.assertEqual(server.shared_bundle_build_counts["superseded"], 1) - self.assertEqual(server.shared_bundle_build_counts["failed"], 0) - self.assertEqual(server.tip_refresh_superseded_results, 1) - - def test_full_helper_input_pipe_obeys_builder_timeout(self) -> None: - server, _rpc = coordinator() - server.signing_seed_hex = "42" * 32 - server.ledger_attestation_signing_seed_hex = "43" * 32 - server.bundle_build_timeout_seconds = 0.05 - helper_started = threading.Event() - helper_processes: list[subprocess.Popen[str]] = [] - real_popen = subprocess.Popen - - def capture_popen(*args: object, **kwargs: object) -> subprocess.Popen[str]: - process = real_popen(*args, **kwargs) # type: ignore[arg-type] - helper_processes.append(process) - helper_started.set() - return process - - errors: list[BaseException] = [] - - def build() -> None: - try: - server.build_audit_bundle( - shares=[], - found_block={ - "block_height": 10, - "coinbase_value_sats": 50_00000000, - "network_difficulty": 1, - "anchor_job_issued_at_ms": 1_700_000_000_000, - }, - prior_balances=[ - { - "miner_id": "pipe-filler", - "balance_sats": 1, - "padding": "x" * (4 * 1024 * 1024), - } - ], - coinbase_script_sig_suffix_hex="00", - ) - except BaseException as exc: # noqa: BLE001 - surfaced below - errors.append(exc) - - with patch( - "lab.prism.prism_coordinator.prism_tool_command", - return_value=[sys.executable, "-c", "import time; time.sleep(30)"], - ), patch( - "lab.prism.prism_coordinator.subprocess.Popen", - side_effect=capture_popen, - ): - build_thread = threading.Thread(target=build) - build_thread.start() - self.assertTrue(helper_started.wait(5)) - build_thread.join(2) - if build_thread.is_alive(): - for process in helper_processes: - if process.poll() is None: - process.kill() - build_thread.join(5) - - self.assertFalse(build_thread.is_alive()) - self.assertEqual(len(errors), 1) - self.assertIsInstance(errors[0], RuntimeError) - self.assertIn("timed out", str(errors[0])) - - def test_supersession_retry_wakes_blockpoll_without_full_interval(self) -> None: - server, _ = coordinator() - server.blockpoll_seconds = 60.0 - server._ensure_tip_refresh_state() - poll_called = threading.Event() - - def poll_once() -> int: - poll_called.set() - server.stop_event.set() - return 0 - - server.poll_qbit_tip_template_once = poll_once # type: ignore[method-assign] - thread = threading.Thread(target=server.blockpoll_loop) - thread.start() - try: - server._schedule_tip_refresh_retry() - self.assertTrue(poll_called.wait(1)) - finally: - server.stop_event.set() - server._schedule_tip_refresh_retry() - thread.join(1) - - self.assertFalse(thread.is_alive()) - - def test_ready_tip_refresh_respects_executor_bound(self) -> None: - server, _ = coordinator() - install_fake_bundle_builder(server) - server.tip_refresh_max_workers = 2 - clients = [client(index + 1) for index in range(6)] - server.clients = set(clients) - release = threading.Event() - two_started = threading.Event() - counter_lock = threading.Lock() - active = 0 - maximum = 0 - - def send(payload: dict[str, object]) -> None: - nonlocal active, maximum - if payload["method"] != "mining.notify": - return - with counter_lock: - active += 1 - maximum = max(maximum, active) - if active == 2: - two_started.set() - try: - self.assertTrue(release.wait(5)) - finally: - with counter_lock: - active -= 1 - - for state in clients: - state.send = send # type: ignore[method-assign] - result: list[int] = [] - thread = threading.Thread(target=lambda: result.append(server.poll_qbit_tip_template_once())) - thread.start() - try: - self.assertTrue(two_started.wait(5)) - self.assertLessEqual(maximum, 2) - finally: - release.set() - thread.join(5) - server.shutdown_tip_refresh_executor() - - self.assertFalse(thread.is_alive()) - self.assertEqual(result, [6]) - self.assertEqual(maximum, 2) - - def test_blocked_socket_does_not_delay_another_client(self) -> None: - server, _ = coordinator() - install_fake_bundle_builder(server) - server.tip_refresh_max_workers = 2 - blocked = client(1) - healthy = client(2) - server.clients = {blocked, healthy} - blocked_started = threading.Event() - healthy_delivered = threading.Event() - release = threading.Event() - - def blocked_send(payload: dict[str, object]) -> None: - if payload["method"] == "mining.notify": - blocked_started.set() - self.assertTrue(release.wait(5)) - - def healthy_send(payload: dict[str, object]) -> None: - if payload["method"] == "mining.notify": - healthy_delivered.set() - - blocked.send = blocked_send # type: ignore[method-assign] - healthy.send = healthy_send # type: ignore[method-assign] - result: list[int] = [] - thread = threading.Thread(target=lambda: result.append(server.poll_qbit_tip_template_once())) - thread.start() - try: - self.assertTrue(blocked_started.wait(5)) - self.assertTrue(healthy_delivered.wait(5)) - finally: - release.set() - thread.join(5) - server.shutdown_tip_refresh_executor() - - self.assertEqual(result, [2]) - - def test_shutdown_drains_inflight_tip_refresh_worker(self) -> None: - server, _ = coordinator() - install_fake_bundle_builder(server) - server.tip_refresh_max_workers = 1 - state = client(1) - server.clients = {state} - worker_started = threading.Event() - worker_send_finished = threading.Event() - release_worker = threading.Event() - shutdown_complete = threading.Event() - poll_errors: list[BaseException] = [] - - def blocked_send(payload: dict[str, object]) -> None: - if payload["method"] != "mining.notify": - return - worker_started.set() - try: - self.assertTrue(release_worker.wait(5)) - finally: - worker_send_finished.set() - - def poll() -> None: - try: - server.poll_qbit_tip_template_once() - except BaseException as exc: # noqa: BLE001 - surface to the test - poll_errors.append(exc) - - def shutdown() -> None: - server.shutdown_tip_refresh_executor() - shutdown_complete.set() - - state.send = blocked_send # type: ignore[method-assign] - poll_thread = threading.Thread(target=poll) - shutdown_thread = threading.Thread(target=shutdown) - poll_thread.start() - try: - self.assertTrue(worker_started.wait(5)) - server.stop_event.set() - shutdown_thread.start() - self.assertFalse(shutdown_complete.wait(0.05)) - self.assertFalse(worker_send_finished.is_set()) - finally: - release_worker.set() - shutdown_thread.join(5) - poll_thread.join(5) - - self.assertFalse(shutdown_thread.is_alive()) - self.assertFalse(poll_thread.is_alive()) - self.assertTrue(shutdown_complete.is_set()) - self.assertTrue(worker_send_finished.is_set()) - self.assertEqual(poll_errors, []) - self.assertEqual(server.tip_refresh_inflight, 0) - with self.assertRaisesRegex(RuntimeError, "executor is shut down"): - server.tip_refresh_executor() - - def test_queued_fanout_stops_when_chain_view_becomes_untrusted(self) -> None: - server, _ = coordinator() - install_fake_bundle_builder(server) - server.reorg_reconciler_enabled = True - server.ensure_reorg_reconciled_for_tip = lambda _tip: True # type: ignore[method-assign] - trust_checks = 0 - - def chain_view_untrusted() -> bool: - nonlocal trust_checks - trust_checks += 1 - return True - - server.qbit_chain_view_untrusted = chain_view_untrusted # type: ignore[method-assign] - first = client(1) - second = client(2) - server.clients = [first, second] # type: ignore[assignment] - sent: list[dict[str, object]] = [] - first.send = sent.append # type: ignore[method-assign] - second.send = sent.append # type: ignore[method-assign] - try: - with self.assertRaisesRegex(TemplateRefreshBlocked, "became untrusted"): - server.poll_qbit_tip_template_once() - finally: - server.shutdown_tip_refresh_executor() - - self.assertEqual(trust_checks, 1) - self.assertEqual(sent, []) - self.assertIsNone(first.active_job) - self.assertIsNone(second.active_job) - - def test_queued_fanout_stops_when_live_tip_changes(self) -> None: - server, rpc = coordinator() - install_fake_bundle_builder(server) - server.tip_refresh_max_workers = 1 - first = client(1) - second = client(2) - server.clients = [first, second] # type: ignore[assignment] - first_blocked = threading.Event() - release_first = threading.Event() - second_sent: list[dict[str, object]] = [] - - def first_send(payload: dict[str, object]) -> None: - if payload["method"] == "mining.notify": - first_blocked.set() - self.assertTrue(release_first.wait(5)) - - first.send = first_send # type: ignore[method-assign] - second.send = second_sent.append # type: ignore[method-assign] - refreshed: list[int] = [] - errors: list[BaseException] = [] - - def poll() -> None: - try: - refreshed.append(server.poll_qbit_tip_template_once()) - except BaseException as exc: # noqa: BLE001 - surface to the test - errors.append(exc) - - thread = threading.Thread(target=poll) - thread.start() - try: - self.assertTrue(first_blocked.wait(5)) - rpc.tip = "33" * 32 - finally: - release_first.set() - thread.join(5) - server.shutdown_tip_refresh_executor() - - self.assertFalse(thread.is_alive()) - self.assertEqual(refreshed, []) - self.assertEqual(len(errors), 1) - self.assertIsInstance(errors[0], TemplateRefreshBlocked) - self.assertIn("immediate retry scheduled", str(errors[0])) - self.assertTrue(server._tip_refresh_retry.is_set()) - self.assertIsNotNone(first.active_job) - self.assertIsNotNone(second.active_job) - self.assertEqual( - [payload["method"] for payload in second_sent], - ["mining.set_difficulty", "mining.notify"], - ) - - def test_multiworker_cancel_releases_client_lock_while_draining_peer(self) -> None: - server, _ = coordinator() - install_fake_bundle_builder(server) - server.tip_refresh_max_workers = 1 - admitted = client(1) - queued = client(2) - server.clients = [admitted, queued] # type: ignore[assignment] - admitted_send_started = threading.Event() - release_admitted_send = threading.Event() - queued_sent: list[dict[str, object]] = [] - - def admitted_send(payload: dict[str, object]) -> None: - if payload["method"] == "mining.notify": - admitted_send_started.set() - self.assertTrue(release_admitted_send.wait(5)) - - admitted.send = admitted_send # type: ignore[method-assign] - queued.send = queued_sent.append # type: ignore[method-assign] - refreshed: list[int] = [] - errors: list[BaseException] = [] - - def poll() -> None: - try: - refreshed.append(server.poll_qbit_tip_template_once()) - except BaseException as exc: # noqa: BLE001 - surface to the test - errors.append(exc) - - thread = threading.Thread(target=poll) - thread.start() - try: - self.assertTrue(admitted_send_started.wait(5)) - server.observe_tip_for_refresh("33" * 32) - lock_acquired = queued.job_update_lock.acquire(timeout=0.1) - self.assertTrue(lock_acquired) - if lock_acquired: - queued.job_update_lock.release() - # The coordinator still waits for the admitted peer delivery, but - # queued workers observe cancellation without taking client state. - self.assertTrue(thread.is_alive()) - self.assertIsNone(queued.active_job) - finally: - release_admitted_send.set() - thread.join(5) - server.shutdown_tip_refresh_executor() - - self.assertFalse(thread.is_alive()) - self.assertEqual(refreshed, []) - self.assertEqual(len(errors), 1) - self.assertIsInstance(errors[0], TemplateRefreshBlocked) - self.assertIsNotNone(admitted.active_job) - self.assertIsNone(queued.active_job) - self.assertEqual(queued_sent, []) - - def test_same_tip_cache_refresh_during_fanout_does_not_abort(self) -> None: - server, rpc = coordinator() - install_fake_bundle_builder(server) - server.tip_refresh_max_workers = 1 - first = client(1) - second = client(2) - # Preserve task order so the cache replacement happens after one - # delivery and before the next worker task starts. - server.clients = [first, second] # type: ignore[assignment] - first_blocked = threading.Event() - release_first = threading.Event() - second_sent: list[dict[str, object]] = [] - - def first_send(payload: dict[str, object]) -> None: - if payload["method"] == "mining.notify": - first_blocked.set() - self.assertTrue(release_first.wait(5)) - - first.send = first_send # type: ignore[method-assign] - second.send = second_sent.append # type: ignore[method-assign] - refreshed: list[int] = [] - errors: list[BaseException] = [] - - def poll() -> None: - try: - refreshed.append(server.poll_qbit_tip_template_once()) - except BaseException as exc: # noqa: BLE001 - surface to the test - errors.append(exc) - - thread = threading.Thread(target=poll) - thread.start() - try: - self.assertTrue(first_blocked.wait(5)) - replacement = dict(rpc.template) - replacement["coinbasevalue"] = int(replacement["coinbasevalue"]) + 1 - replacement_artifacts = server.store_template_artifacts(replacement) - self.assertIsNotNone(replacement_artifacts) - self.assertNotEqual( - replacement_artifacts.fingerprint, - first.active_job.template_fingerprint, - ) - finally: - release_first.set() - thread.join(5) - server.shutdown_tip_refresh_executor() - - self.assertFalse(thread.is_alive()) - self.assertEqual(errors, []) - self.assertEqual(refreshed, [2]) - self.assertIsNotNone(server.last_successful_template_refresh_monotonic) - self.assertEqual( - [payload["method"] for payload in second_sent], - ["mining.set_difficulty", "mining.notify"], - ) - - def test_queued_fanout_does_not_overwrite_intervening_job(self) -> None: - server, rpc = coordinator() - install_fake_bundle_builder(server) - server.tip_refresh_max_workers = 1 - first = client(1) - second = client(2) - server.clients = [first, second] # type: ignore[assignment] - first_blocked = threading.Event() - release_first = threading.Event() - second_sent: list[dict[str, object]] = [] - - def first_send(payload: dict[str, object]) -> None: - if payload["method"] == "mining.notify": - first_blocked.set() - self.assertTrue(release_first.wait(5)) - - first.send = first_send # type: ignore[method-assign] - second.send = second_sent.append # type: ignore[method-assign] - refreshed: list[int] = [] - errors: list[BaseException] = [] - - def poll() -> None: - try: - refreshed.append(server.poll_qbit_tip_template_once()) - except BaseException as exc: # noqa: BLE001 - surface to the test - errors.append(exc) - - thread = threading.Thread(target=poll) - thread.start() - try: - self.assertTrue(first_blocked.wait(5)) - replacement = dict(rpc.template) - replacement["coinbasevalue"] = int(replacement["coinbasevalue"]) + 1 - replacement_artifacts = server.store_template_artifacts(replacement) - self.assertIsNotNone(replacement_artifacts) - self.assertTrue(server.maybe_send_job(second, clean_jobs=False)) - intervening_job = second.active_job - self.assertEqual( - intervening_job.template_fingerprint, - replacement_artifacts.fingerprint, - ) - self.assertGreater( - intervening_job.template_generation, - first.active_job.template_generation, - ) - finally: - release_first.set() - thread.join(5) - server.shutdown_tip_refresh_executor() - - self.assertFalse(thread.is_alive()) - self.assertEqual(errors, []) - self.assertEqual(refreshed, [1]) - self.assertIs(second.active_job, intervening_job) - self.assertEqual( - [payload["method"] for payload in second_sent], - ["mining.set_difficulty", "mining.notify"], - ) - - def test_queued_fanout_replaces_stale_intervening_job(self) -> None: - server, rpc = coordinator() - install_fake_bundle_builder(server) - server.tip_refresh_max_workers = 1 - # The poll must observe the same-tip template rotation immediately for - # the queued-fanout replacement race below; disable the same-tip - # template reuse window so the rotation is fetched, not deferred. - server.template_cache_seconds = 0.0 - first = client(1) - second = client(2) - server.clients = [first, second] # type: ignore[assignment] - old_artifacts = server.store_template_artifacts(dict(rpc.template)) - self.assertIsNotNone(old_artifacts) - assert old_artifacts is not None - assert second.worker is not None - old_bundle = server.shared_job_bundle(old_artifacts, second.worker) - refreshed_template = dict(rpc.template) - refreshed_template["coinbasevalue"] = int( - refreshed_template["coinbasevalue"] - ) + 1 - rpc.template = refreshed_template - first_blocked = threading.Event() - release_first = threading.Event() - second_sent: list[dict[str, object]] = [] - - def first_send(payload: dict[str, object]) -> None: - if payload["method"] == "mining.notify": - first_blocked.set() - self.assertTrue(release_first.wait(5)) - - first.send = first_send # type: ignore[method-assign] - second.send = second_sent.append # type: ignore[method-assign] - refreshed: list[int] = [] - errors: list[BaseException] = [] - - def poll() -> None: - try: - refreshed.append(server.poll_qbit_tip_template_once()) - except BaseException as exc: # noqa: BLE001 - surface to the test - errors.append(exc) - - thread = threading.Thread(target=poll) - thread.start() - try: - self.assertTrue(first_blocked.wait(5)) - with second.job_update_lock, server.lock: - stale_intervening_job = server.stamp_job_for_client( - second, - old_bundle, - clean_jobs=False, - ) - second.active_job = stale_intervening_job - second.active_job_ids.add(stale_intervening_job.job.job_id) - server.jobs[stale_intervening_job.job.job_id] = stale_intervening_job - finally: - release_first.set() - thread.join(5) - server.shutdown_tip_refresh_executor() - - self.assertFalse(thread.is_alive()) - self.assertEqual(errors, []) - self.assertEqual(refreshed, [2]) - self.assertIsNot(second.active_job, stale_intervening_job) - self.assertEqual( - second.active_job.template_fingerprint, - qbit_template_fingerprint(refreshed_template), - ) - self.assertGreater( - second.active_job.template_generation, - stale_intervening_job.template_generation, - ) - self.assertEqual( - [payload["method"] for payload in second_sent], - ["mining.set_difficulty", "mining.notify"], - ) - - def test_newer_template_does_not_supersede_current_payout_refresh(self) -> None: - server, rpc = coordinator() - install_fake_bundle_builder(server) - state = client(1) - artifacts = server.store_template_artifacts(dict(rpc.template)) - self.assertIsNotNone(artifacts) - assert artifacts is not None and state.worker is not None - stale_bundle = server.shared_job_bundle(artifacts, state.worker) - snapshot = server.fetch_qbit_tip_template_snapshot() - stale_intervening_job = dataclass_replace( - server.stamp_job_for_client( - state, - stale_bundle, - clean_jobs=False, - ), - template_generation=snapshot.template_generation + 1, - ) - server._advance_payout_state_generation() - - self.assertFalse( - server.intervening_job_supersedes_snapshot( - stale_intervening_job, - None, - snapshot, - ) - ) - current_intervening_job = dataclass_replace( - stale_intervening_job, - payout_state_generation=server._payout_state_generation, - ) - self.assertTrue( - server.intervening_job_supersedes_snapshot( - current_intervening_job, - None, - snapshot, - ) - ) - - def test_higher_generation_old_tip_does_not_supersede_new_tip_snapshot(self) -> None: - server, rpc = coordinator() - install_fake_bundle_builder(server) - state = client(1) - old_artifacts = server.store_template_artifacts(dict(rpc.template)) - self.assertIsNotNone(old_artifacts) - assert old_artifacts is not None and state.worker is not None - old_bundle = server.shared_job_bundle(old_artifacts, state.worker) - - new_tip = "22" * 32 - rpc.tip = new_tip - rpc.template = base_template(height=11, prevhash=new_tip) - snapshot = server.fetch_qbit_tip_template_snapshot() - old_tip_job = dataclass_replace( - server.stamp_job_for_client( - state, - old_bundle, - clean_jobs=False, - ), - template_generation=snapshot.template_generation + 1, - ) - - self.assertNotEqual( - old_tip_job.template_fingerprint, - snapshot.template_fingerprint, - ) - self.assertFalse( - server.intervening_job_supersedes_snapshot( - old_tip_job, - None, - snapshot, - ) - ) - - def test_broken_socket_disconnects_only_that_client(self) -> None: - server, _ = coordinator() - install_fake_bundle_builder(server) - broken = client(1) - healthy = client(2) - server.clients = {broken, healthy} - healthy_sent: list[dict[str, object]] = [] - disconnected: list[ClientState] = [] - broken.send = lambda _payload: (_ for _ in ()).throw(OSError("closed")) # type: ignore[method-assign] - healthy.send = healthy_sent.append # type: ignore[method-assign] - server.disconnect_client = disconnected.append # type: ignore[method-assign] - - try: - refreshed = server.poll_qbit_tip_template_once() - finally: - server.shutdown_tip_refresh_executor() - - self.assertEqual(refreshed, 1) - self.assertEqual(disconnected, [broken]) - self.assertEqual( - [payload["method"] for payload in healthy_sent], - ["mining.set_difficulty", "mining.notify"], - ) - - def test_client_removed_before_pending_task_runs_is_skipped(self) -> None: - server, _ = coordinator() - install_fake_bundle_builder(server) - server.tip_refresh_max_workers = 1 - first = client(1) - removed = client(2) - clients = [first, removed] - server.clients = set(clients) - snapshot = server.fetch_qbit_tip_template_snapshot() - server.observe_tip_first_seen(snapshot.bestblockhash) - server.pool_readiness_latched() - server.tip_template_snapshot = snapshot - bundle = server.prepare_tip_refresh_bundle(snapshot) - blocked = threading.Event() - release = threading.Event() - - def first_send(payload: dict[str, object]) -> None: - if payload["method"] == "mining.notify": - blocked.set() - self.assertTrue(release.wait(5)) - - first.send = first_send # type: ignore[method-assign] - removed.send = lambda _payload: self.fail("removed client received a job") # type: ignore[method-assign] - result: list[tuple[int, float | None, float | None, int]] = [] - thread = threading.Thread( - target=lambda: result.append( - server._fanout_prepared_tip_refresh( - clients, - bundle, - snapshot, - heartbeat_name="qbit_blockpoll", - ) - ) - ) - thread.start() - try: - self.assertTrue(blocked.wait(5)) - with server.lock: - server.clients.remove(removed) - finally: - release.set() - thread.join(5) - server.shutdown_tip_refresh_executor() - - self.assertEqual(result[0][0], 1) - self.assertIsNone(removed.active_job) - self.assertEqual(removed.active_job_ids, set()) - - def test_template_fingerprint_race_uses_snapshot_owned_artifacts(self) -> None: - server, _ = coordinator() - install_fake_bundle_builder(server) - states = [client(1), client(2)] - sent: list[dict[str, object]] = [] - for state in states: - state.send = sent.append # type: ignore[method-assign] - server.clients = set(states) - original_shared_job_bundle = server.shared_job_bundle - race_calls = 0 - - def race_artifacts( - artifacts: object, - identity: WorkerIdentity | None = None, - **kwargs: object, - ) -> object: - nonlocal race_calls - race_calls += 1 - bundle = original_shared_job_bundle(artifacts, identity, **kwargs) - with server._job_cache_lock: - server._template_artifacts = dataclass_replace( - server._template_artifacts, - fingerprint="ff" * 32, - ) - return bundle - - server.shared_job_bundle = race_artifacts # type: ignore[method-assign] - - refreshed = server.poll_qbit_tip_template_once() - - self.assertEqual(race_calls, 1) - self.assertEqual(refreshed, 2) - self.assertEqual(len(sent), 4) - self.assertIsNotNone(server.tip_template_snapshot) - snapshot = server.tip_template_snapshot - assert snapshot is not None and snapshot.template_artifacts is not None - for state in states: - self.assertIs(state.active_job.template, snapshot.template_artifacts.template) - self.assertEqual( - state.active_job.template_fingerprint, - snapshot.template_fingerprint, - ) - - -class ClientCleanupTests(unittest.TestCase): - def test_disconnect_retires_and_closes_before_job_lock_cleanup(self) -> None: - server, _ = coordinator() - state = client(1) - server.clients = {state} - socket_closed = threading.Event() - state.close = socket_closed.set # type: ignore[method-assign] - state.job_update_lock.acquire() - disconnect = threading.Thread(target=server.disconnect_client, args=(state,)) - try: - disconnect.start() - self.assertTrue(socket_closed.wait(5)) - with server.lock: - self.assertNotIn(state, server.clients) - self.assertTrue(state.closing) - self.assertTrue(disconnect.is_alive()) - finally: - state.job_update_lock.release() - disconnect.join(5) - - self.assertFalse(disconnect.is_alive()) - - def test_disconnect_during_prepared_refresh_skips_without_job_state(self) -> None: - server, _ = coordinator() - install_fake_bundle_builder(server) - state = client(1) - observed_lock = ObservedRLock() - state.job_update_lock = observed_lock # type: ignore[assignment] - server.clients = {state} - snapshot = server.fetch_qbit_tip_template_snapshot() - server.observe_tip_first_seen(snapshot.bestblockhash) - server.pool_readiness_latched() - server.tip_template_snapshot = snapshot - bundle = server.prepare_tip_refresh_bundle(snapshot) - state.send = lambda _payload: self.fail( # type: ignore[method-assign] - "disconnected client received prepared work" - ) - socket_closed = threading.Event() - state.close = socket_closed.set # type: ignore[method-assign] - results: list[tuple[int, float | None, float | None, int]] = [] - errors: list[BaseException] = [] - - def refresh() -> None: - try: - results.append( - server._fanout_prepared_tip_refresh( - [state], - bundle, - snapshot, - heartbeat_name="qbit_blockpoll", - ) - ) - except BaseException as exc: # noqa: BLE001 - surface thread failures - errors.append(exc) - - observed_lock.acquire() - observed_lock.observe_acquires = True - refresh_thread = threading.Thread(target=refresh) - disconnect_thread = threading.Thread( - target=server.disconnect_client, - args=(state,), - ) - try: - refresh_thread.start() - self.assertTrue(observed_lock.acquire_attempted.wait(5)) - disconnect_thread.start() - self.assertTrue(socket_closed.wait(5)) - refresh_thread.join(5) - self.assertFalse(refresh_thread.is_alive()) - self.assertTrue(disconnect_thread.is_alive()) - finally: - observed_lock.release() - refresh_thread.join(5) - disconnect_thread.join(5) - server.shutdown_tip_refresh_executor() - - self.assertFalse(disconnect_thread.is_alive()) - self.assertEqual(errors, []) - self.assertEqual(results[0][0], 0) - self.assertIsNone(state.active_job) - self.assertEqual(state.active_job_ids, set()) - self.assertEqual(server.jobs, {}) +These cases target the J1 owner boundary itself: symbol identity, descriptor +routing, template-artifact generation ordering, and bundle cache-admission +fencing through the real ``JobBundleService`` / +``TemplateArtifactRepository`` APIs. The publication-priority and scheduler +behavior itself is asserted by the upstream suites in +``tests/test_prism_job_builder.py`` and ``tests/test_prism_job_build_promises.py``. +""" - def test_mass_disconnect_releases_active_connection_accounting(self) -> None: - server, _ = coordinator() - states = [client(index) for index in range(1, 129)] - for state in states: - state.close = lambda: None # type: ignore[method-assign] - server.clients = set(states) - - for state in states: - server.disconnect_client(state) - - with server.lock: - self.assertEqual(len(server.clients), 0) - self.assertTrue(all(state.closing for state in states)) - - def test_concurrent_disconnect_is_idempotent_and_deadlock_free(self) -> None: - server, _ = coordinator() - state = client(1) - server.clients = {state} - close_count = 0 - close_count_lock = threading.Lock() - caller_count = 16 - start = threading.Barrier(caller_count + 1) - errors: list[BaseException] = [] - - def close() -> None: - nonlocal close_count - with close_count_lock: - close_count += 1 - - def disconnect() -> None: - try: - start.wait() - server.disconnect_client(state) - except BaseException as exc: # noqa: BLE001 - surface thread failures - errors.append(exc) - - state.close = close # type: ignore[method-assign] - callers = [threading.Thread(target=disconnect) for _ in range(caller_count)] - for caller in callers: - caller.start() - start.wait() - for caller in callers: - caller.join(5) - - self.assertTrue(all(not caller.is_alive() for caller in callers)) - self.assertEqual(errors, []) - self.assertEqual(close_count, 1) - self.assertNotIn(state, server.clients) - - def test_disconnect_removes_active_and_evicted_job_contexts(self) -> None: - server, _ = coordinator() - install_fake_bundle_builder(server) - state = client(1) - active = server.build_job_for_client(state, clean_jobs=True) - evicted = server.build_job_for_client(state, clean_jobs=False) - active_id = active.job.job_id - evicted_id = evicted.job.job_id - state.active_job = active - state.active_job_ids = {active_id} - server.jobs = {active_id: active, evicted_id: evicted} - server.bury_evicted_job(state, evicted_id) - server.jobs.pop(evicted_id) - server.clients = {state} - state.close = lambda: None # type: ignore[method-assign] - - server.disconnect_client(state) - - self.assertIsNone(state.active_job) - self.assertEqual(state.active_job_ids, set()) - self.assertNotIn(active_id, server.jobs) - # Same-tip contexts survive the disconnect (detached from the dead - # connection object) so a same-username reconnect can still submit - # in-flight work; they age out on the same-tip retention TTL. - for job_id in (active_id, evicted_id): - entry = server.evicted_job_graveyard.get(job_id) - self.assertIsNotNone(entry) - assert entry is not None - self.assertIsNone(entry.client) - self.assertIn(job_id, server._disconnected_evicted_job_ids) - - def test_disconnect_purges_evicted_contexts_when_retention_disabled( - self, - ) -> None: - server, _ = coordinator() - install_fake_bundle_builder(server) - server.disconnected_job_retention = 0 - state = client(1) - active = server.build_job_for_client(state, clean_jobs=True) - evicted = server.build_job_for_client(state, clean_jobs=False) - active_id = active.job.job_id - evicted_id = evicted.job.job_id - state.active_job = active - state.active_job_ids = {active_id} - server.jobs = {active_id: active, evicted_id: evicted} - server.bury_evicted_job(state, evicted_id) - server.jobs.pop(evicted_id) - server.clients = {state} - state.close = lambda: None # type: ignore[method-assign] - - server.disconnect_client(state) - - self.assertIsNone(state.active_job) - self.assertEqual(state.active_job_ids, set()) - self.assertNotIn(active_id, server.jobs) - self.assertNotIn(active_id, server.evicted_job_graveyard) - self.assertNotIn(evicted_id, server.evicted_job_graveyard) - self.assertNotIn(state.connection_id, server.evicted_jobs_by_connection) - - def test_reconnect_storm_leaves_no_handler_threads_or_ghost_clients(self) -> None: - server, _ = coordinator() - connection_count = 32 - start = threading.Barrier(connection_count + 1) - peers: list[socket.socket] = [] - handlers: list[threading.Thread] = [] - - def handle(state: ClientState) -> None: - start.wait() - server.handle_client(state) - - for connection_id in range(1, connection_count + 1): - coordinator_socket, peer_socket = socket.socketpair() - state = client(connection_id) - state.sock = coordinator_socket - server.clients.add(state) - peers.append(peer_socket) - handler = threading.Thread( - target=handle, - args=(state,), - name=f"prism-test-handler-{connection_id}", - ) - handlers.append(handler) - handler.start() - - start.wait() - for peer in peers: - peer.close() - for handler in handlers: - handler.join(5) - - self.assertTrue(all(not handler.is_alive() for handler in handlers)) - with server.lock: - self.assertEqual(server.clients, set()) - - -class HealthSnapshotTests(unittest.TestCase): - def test_health_payload_uses_aggregate_stats_not_all_shares(self) -> None: - ledger = FakeLedger() - server, _ = coordinator(ledger=ledger) - mark_progress_healthy(server) - payload = server.health_payload() - self.assertTrue(payload["ok"]) - self.assertEqual(payload["accepted_share_count"], 3) - self.assertEqual(payload["ready_miner_count"], 3) - self.assertGreaterEqual(ledger.stats_calls, 1) - - def test_cached_health_payload_computes_inline_without_refresher(self) -> None: - server, _ = coordinator() - mark_progress_healthy(server) - status, payload = server.cached_health_payload() - self.assertEqual(status, 200) - self.assertTrue(payload["ok"]) - - def test_audit_server_marks_refresher_before_listener_dispatch(self) -> None: - """A probe retrying connection-refused queues in the accept backlog at - listen() time and can be served the instant the listener thread - starts. The refresher flag must already be set by then, or - cached_health_payload's legacy inline path would run the cold - accepted-share aggregate synchronously on the handler thread instead - of returning the documented starting state.""" - server, _ = coordinator() - server.audit_bind = "127.0.0.1" - server.audit_port = 0 - # The spawned refresher loop exits immediately; the running flag is - # what gates the handler path and must survive. - server.stop_event.set() - flag_at_bind: list[bool] = [] - - class RecordingServer: - def __init__(self, address: object, handler_cls: object) -> None: - flag_at_bind.append(bool(server._health_refresh_loop_running)) - - def serve_forever(self) -> None: - return None - - with patch.object( - prism_coordinator_module, "ThreadingHTTPServer", RecordingServer - ), patch("builtins.print"): - server.start_audit_server() - self.assertEqual(flag_at_bind, [True]) - self.assertTrue(server._health_refresh_loop_running) - - def test_cached_health_payload_serves_snapshot_and_flags_staleness(self) -> None: - server, _ = coordinator() - mark_progress_healthy(server) - server.refresh_health_snapshot() - server._health_refresh_loop_running = True - - status, payload = server.cached_health_payload() - self.assertEqual(status, 200) - self.assertIn("snapshot_age_seconds", payload) - - # Even if the ledger becomes unusable, the snapshot keeps serving. - server.ledger = None # type: ignore[assignment] - status, payload = server.cached_health_payload() - self.assertEqual(status, 200) - - server._health_snapshot_monotonic = time.monotonic() - 1_000 - status, payload = server.cached_health_payload() - self.assertEqual(status, 503) - self.assertFalse(payload["ok"]) - - def test_accepted_share_stats_falls_back_to_all_shares(self) -> None: - server, _ = coordinator(ledger=SingleWriterShareLedger()) - self.assertEqual(server.accepted_share_stats(), (0, 0)) - - def test_single_writer_ledger_stats(self) -> None: - ledger = SingleWriterShareLedger() - self.assertEqual( - ledger.accepted_share_stats(), - {"accepted_share_count": 0, "distinct_miner_count": 0}, - ) - - -class JobBuildMetricsTests(unittest.TestCase): - def test_metrics_include_job_build_histogram_and_cache_counters(self) -> None: - server, _ = coordinator() - install_fake_bundle_builder(server) - server.build_job_for_client(client(1), clean_jobs=True) - server.build_job_for_client(client(2), clean_jobs=True) - server.observe_job_build_elapsed(0.3, {"bundle": 0.2, "stamp": 0.01}) - server._observe_tip_refresh_build_phase("payout_state_derivation", 0.25) - server._record_tip_refresh_ipc_bytes("input", 123) - - metrics = server.metrics_payload() - - self.assertIn('qbit_prism_job_build_seconds_bucket{le="0.5"} 1', metrics) - self.assertIn('qbit_prism_job_build_seconds_bucket{le="+Inf"} 1', metrics) - self.assertIn("qbit_prism_job_build_seconds_count 1", metrics) - self.assertIn('qbit_prism_job_cache_hits_total{cache="bundle"} 1', metrics) - self.assertIn('qbit_prism_job_cache_misses_total{cache="bundle"} 1', metrics) - self.assertIn('qbit_prism_job_build_phase_seconds_total{phase="bundle"} 0.2', metrics) - self.assertIn( - 'qbit_prism_tip_refresh_bundle_phase_seconds_count{phase="payout_state_derivation"} 1', - metrics, - ) - self.assertIn( - 'qbit_prism_tip_refresh_builder_ipc_bytes_total{direction="input"} 123', - metrics, - ) - self.assertIn("qbit_prism_tip_refresh_bundle_queue_depth 0", metrics) - self.assertIn("qbit_prism_tip_refresh_bundle_inflight 0", metrics) - self.assertIn("qbit_prism_connected_clients 0", metrics) - - def test_poll_observes_reorg_reconcile_phase(self) -> None: - # The reconcile stage runs serially before every refresh build; one - # poll must record exactly one reorg_reconcile phase observation. - server, _ = coordinator() - install_fake_bundle_builder(server) - state = client(1) - state.send = lambda _payload: None # type: ignore[method-assign] - server.clients = {state} - - try: - refreshed = server.poll_qbit_tip_template_once() - finally: - server.shutdown_tip_refresh_executor() - - self.assertEqual(refreshed, 1) - metrics = server.metrics_payload() - self.assertIn( - 'qbit_prism_tip_refresh_bundle_phase_seconds_count{phase="reorg_reconcile"} 1', - metrics, - ) - - def test_metrics_split_payout_preparation_publication_and_delivery(self) -> None: - server, _ = coordinator() - install_fake_bundle_builder(server) - state = client(1) - state.send = lambda _payload: None # type: ignore[method-assign] - - self.assertEqual(server._advance_payout_state_generation(), 1) - self.assertTrue(server.maybe_send_job(state, clean_jobs=True)) - - metrics = server.metrics_payload() - - self.assertIn("qbit_prism_payout_preparation_seconds_count 1", metrics) - self.assertIn("qbit_prism_payout_publish_seconds_count 1", metrics) - self.assertIn( - "qbit_prism_payout_invalidation_first_delivery_seconds_count 1", - metrics, - ) - self.assertIn( - 'qbit_prism_payout_gate_wait_seconds_count{generation="current"} 1', - metrics, - ) - self.assertIn("qbit_prism_payout_candidates_discarded_total 0", metrics) - - -def spool_share(seq: int) -> dict[str, object]: - return { - "share_seq": seq, - "share_id": f"share-{seq}", - "miner_id": "miner-a", - "order_key": "miner-a", - "p2mr_program_hex": "22" * 32, - "share_difficulty": 1, - "job_issued_at_ms": 1_700_000_000_000 + seq, - "accepted_at_ms": 1_700_000_000_100 + seq, - "credit_policy": None, - } +from __future__ import annotations +from collections import OrderedDict +from dataclasses import replace as dataclass_replace +import time +from types import SimpleNamespace +import unittest -ECHO_BUILDER_COMMAND = [ - sys.executable, - "-c", - "import json,sys; json.dump({'received': json.load(sys.stdin)}, sys.stdout)", -] +from lab.prism import job_bundle, prism_coordinator, template_artifacts +from lab.prism.job_bundle import CachedJobBundle, JobBuildKey +from lab.prism.prism_coordinator import PayoutStateArtifact +from lab.prism.template_artifacts import CachedTemplateArtifacts +from tests.prism_coordinator_test_support import base_template, coordinator -class ShareWindowSpoolTests(unittest.TestCase): - """Spool-file handoff of the serialized share window to the builder.""" +def cached_bundle( + artifacts: CachedTemplateArtifacts, + *, + payout_generation: int, + payout_artifact_sha256: str, + key: tuple[object, ...] = ("candidate",), + collection_only: bool = True, + payout_append_invalidation_epoch: int = 0, +) -> CachedJobBundle: + build_key = JobBuildKey( + best_tip_hash=artifacts.previousblockhash, + previous_block_hash=artifacts.previousblockhash, + template_fingerprint=artifacts.fingerprint, + template_generation=artifacts.generation, + payout_state_generation=payout_generation, + payout_artifact_sha256=payout_artifact_sha256, + mode="collection" if collection_only else "ready", + collection_identity=("miner", "22" * 32) if collection_only else None, + block_height=int(artifacts.template["height"]), + coinbase_value_sats=int(artifacts.template["coinbasevalue"]), + network_difficulty=artifacts.network_difficulty, + issued_at_ms=1, + payout_policy_sha256="payout", + ctv_settlement_sha256=None, + witness_merkle_sha256="witness", + transaction_set_sha256="transactions", + coinbase_suffix_hex="00", + signing_key_sha256="signing", + ledger_signing_key_sha256="ledger", + numeric_context_sha256="numeric", + payout_append_invalidation_epoch=payout_append_invalidation_epoch, + ) + return CachedJobBundle( + key=key, + template=artifacts.template, + template_fingerprint=artifacts.fingerprint, + coinbase_manifest={"coinbase_tx_hex": "00"}, + shares_json=[], + prior_balances=[], + found_block={"network_difficulty": artifacts.network_difficulty}, + collection_only=collection_only, + issued_at_ms=1, + base_job=SimpleNamespace(), # type: ignore[arg-type] + built_monotonic=time.monotonic(), + template_generation=artifacts.generation, + payout_state_generation=payout_generation, + collection_identity=build_key.collection_identity, + build_key=build_key, + ) - def _coordinator(self) -> PrismCoordinator: - server, _rpc = coordinator() - server.signing_seed_hex = "42" * 32 - server.ledger_attestation_signing_seed_hex = "43" * 32 - self.addCleanup(server.retire_share_window_spool) - return server - def _ledger_artifact( - self, - shares: list[dict[str, object]], - *, - generation: int, - ) -> PayoutLedgerArtifact: - return PayoutLedgerArtifact( +def seed_published_payout_artifact(server: object, sha256: str) -> int: + """Publish a payout artifact through the P1 owner and return the generation.""" + server._ensure_payout_state_service() # type: ignore[attr-defined] + generation = int(server._payout_state_generation) # type: ignore[attr-defined] + server._published_payout_state = dataclass_replace( # type: ignore[attr-defined] + server._published_payout_state, # type: ignore[attr-defined] + artifact=PayoutStateArtifact( generation=generation, - payout_state_generation=0, - network_difficulty=1, - accepted_share_count=len(shares), - shares_json=tuple(shares), - prior_balances=(), + source_generation=0, + prior_balances_json="[]", + prior_balances_sha256=sha256, prepared_monotonic=time.monotonic(), - snapshot_anchor_ms=None, - ) - - def _build_with_echo_builder( - self, - server: PrismCoordinator, - shares: list[dict[str, object]], - serialization: object, - *, - height: int, - ) -> dict[str, object]: - # These tests cover the one-shot spool handoff; pin the persistent - # builder off so the echo helper is never spawned as a daemon. - with patch.dict(os.environ, {"PRISM_BUILDER_SERVE": "0"}), patch( - "lab.prism.prism_coordinator.prism_tool_command", - return_value=list(ECHO_BUILDER_COMMAND), - ): - return server.build_audit_bundle( - shares=shares, - found_block={ - "block_height": height, - "coinbase_value_sats": 50_00000000, - "network_difficulty": 1, - "anchor_job_issued_at_ms": 1_700_000_000_000, - }, - prior_balances=[], - coinbase_script_sig_suffix_hex="00", - summary_only=True, - payout_policy={"policy": "day-one"}, - share_serialization=serialization, # type: ignore[arg-type] - ) - - def _expected_payload( - self, - shares: list[dict[str, object]], - *, - height: int, - ) -> dict[str, object]: - identities, compact_shares = _compact_share_payload(shares) - return { - "found_block": { - "block_height": height, - "coinbase_value_sats": 50_00000000, - "network_difficulty": 1, - "anchor_job_issued_at_ms": 1_700_000_000_000, - }, - "prior_balances": [], - "payout_policy": {"policy": "day-one"}, - "coinbase_script_sig_suffix_hex": "00", - "witness_merkle_leaves_hex": [], - "compact_share_identities": [ - list(identity) for identity in identities - ], - "compact_shares": [list(share) for share in compact_shares], - } - - def test_spool_feeds_builder_and_is_written_once_per_generation( - self, - ) -> None: - server = self._coordinator() - shares = [spool_share(seq) for seq in range(1, 4)] - serialization = server._share_window_serialization_for_artifact( - self._ledger_artifact(shares, generation=1), - shares, - ) - spool_creations: list[int] = [] - real_spool_file = prism_coordinator_module._share_window_spool_file - - def counting_spool_file() -> object: - spool_creations.append(1) - return real_spool_file() - - with patch( - "lab.prism.prism_coordinator._share_window_spool_file", - side_effect=counting_spool_file, - ): - first = self._build_with_echo_builder( - server, - shares, - serialization, - height=10, - ) - second = self._build_with_echo_builder( - server, - shares, - serialization, - height=11, - ) - - self.assertEqual( - first["received"], - self._expected_payload(shares, height=10), - ) - self.assertEqual( - second["received"], - self._expected_payload(shares, height=11), - ) - self.assertEqual(len(spool_creations), 1) - - def test_spool_creation_failure_falls_back_to_pipe_writes(self) -> None: - server = self._coordinator() - shares = [spool_share(seq) for seq in range(1, 4)] - serialization = server._share_window_serialization_for_artifact( - self._ledger_artifact(shares, generation=1), - shares, - ) + ), + ) + return generation - with patch( - "lab.prism.prism_coordinator._share_window_spool_file", - side_effect=OSError("temp filesystem unavailable"), - ) as spool_factory: - first = self._build_with_echo_builder( - server, - shares, - serialization, - height=10, - ) - second = self._build_with_echo_builder( - server, - shares, - serialization, - height=11, - ) - self.assertEqual( - first["received"], - self._expected_payload(shares, height=10), +class MovedSymbolIdentityTests(unittest.TestCase): + def test_moved_symbol_reexports_preserve_identity(self) -> None: + self.assertIs(prism_coordinator.CachedJobBundle, job_bundle.CachedJobBundle) + # PR 80 removed the JobBuildKey compatibility re-export. + self.assertFalse(hasattr(prism_coordinator, "JobBuildKey")) + self.assertIs( + prism_coordinator.JobBuildSuperseded, job_bundle.JobBuildSuperseded ) - self.assertEqual( - second["received"], - self._expected_payload(shares, height=11), + self.assertIs(prism_coordinator._JobBuildRequest, job_bundle.JobBuildRequest) + self.assertIs(prism_coordinator._JobBuildFlight, job_bundle.JobBuildFlight) + self.assertIs( + prism_coordinator._JobBuildCancellation, job_bundle.JobBuildCancellation ) - # The failure is sticky: a broken temp filesystem is not retried on - # every subsequent build of the same generation. - self.assertEqual(spool_factory.call_count, 1) - - def test_splice_failure_poisons_spool_and_falls_back_to_pipe(self) -> None: - server = self._coordinator() - shares = [spool_share(seq) for seq in range(1, 4)] - serialization = server._share_window_serialization_for_artifact( - self._ledger_artifact(shares, generation=1), - shares, + self.assertIs( + prism_coordinator._JobBundleBuildControl, + job_bundle.JobBundleBuildControl, ) - - with patch( - "lab.prism.prism_coordinator.os.splice", - side_effect=OSError("splice unsupported"), - ): - first = self._build_with_echo_builder( - server, - shares, - serialization, - height=10, - ) - second = self._build_with_echo_builder( - server, - shares, - serialization, - height=11, + self.assertIs( + prism_coordinator.CachedTemplateArtifacts, + template_artifacts.CachedTemplateArtifacts, ) - - self.assertEqual( - first["received"], - self._expected_payload(shares, height=10), + self.assertIs( + prism_coordinator.QbitTipTemplateSnapshot, + template_artifacts.QbitTipTemplateSnapshot, ) + # qbit_template_fingerprint intentionally remains a local coordinator + # compatibility definition at this layer; the owner carries its own + # copy with identical semantics (no reverse import). + template = base_template() self.assertEqual( - second["received"], - self._expected_payload(shares, height=11), - ) - self.assertTrue(serialization._spool_failed) - self.assertIsNone(serialization._spool_file) - - def test_mid_stream_splice_failure_poisons_spool_for_later_builds( - self, - ) -> None: - server = self._coordinator() - shares = [spool_share(seq) for seq in range(1, 4)] - serialization = server._share_window_serialization_for_artifact( - self._ledger_artifact(shares, generation=1), - shares, + prism_coordinator.qbit_template_fingerprint(template), + template_artifacts.qbit_template_fingerprint(template), ) - # The first chunk moves and the second fails: this build is - # unrepairable (partial tail already in the pipe), but the spool must - # be poisoned so later builds stream the cached fragments instead of - # repeating the failure. - with patch( - "lab.prism.prism_coordinator.os.splice", - side_effect=[16, OSError("splice io error")], - ): - with self.assertRaises(OSError): - self._build_with_echo_builder( - server, - shares, - serialization, - height=10, - ) - self.assertTrue(serialization._spool_failed) + def test_service_owns_job_cache_state_and_descriptors_route(self) -> None: + server, _rpc = coordinator() + service = server._ensure_job_bundle_service() - recovered = self._build_with_echo_builder( - server, - shares, - serialization, - height=11, + self.assertIs(server._job_cache_lock, service._job_cache_lock) + self.assertIs(server._job_bundle_cache, service._job_bundle_cache) + self.assertIs( + server._job_build_scheduler_lock, service._job_build_scheduler_lock ) - self.assertEqual( - recovered["received"], - self._expected_payload(shares, height=11), + self.assertIs(server.job_cache_hit_counts, service.job_cache_hit_counts) + # The repository deliberately shares the historical combined lock. + self.assertIs( + service.template_repository._job_cache_lock, + service._job_cache_lock, ) - def test_content_rotation_retires_spool_after_last_lease(self) -> None: - server = self._coordinator() - shares = [spool_share(seq) for seq in range(1, 4)] - first = server._share_window_serialization_for_artifact( - self._ledger_artifact(shares, generation=1), - shares, - ) - lease = first.acquire_spooled_tail(shares) - self.assertIsNotNone(lease) - assert lease is not None - spool_file, spool_size = lease - self.assertGreater(spool_size, 0) + replacement: OrderedDict[tuple[object, ...], CachedJobBundle] = OrderedDict() + server._job_bundle_cache = replacement + self.assertIs(service._job_bundle_cache, replacement) - changed_shares = [*shares, spool_share(4)] - second = server._share_window_serialization_for_artifact( - self._ledger_artifact(changed_shares, generation=2), - changed_shares, - ) - self.assertIsNot(second, first) - # Retired: no new leases, but the in-flight transfer keeps the - # descriptor alive until it releases. - self.assertIsNone(first.acquire_spooled_tail(shares)) - self.assertFalse(spool_file.closed) - first.release_spooled_tail() - self.assertTrue(spool_file.closed) - def test_generation_retag_reuses_content_keyed_serialization(self) -> None: - server = self._coordinator() - shares = [spool_share(seq) for seq in range(1, 4)] +class TemplateArtifactRepositoryTests(unittest.TestCase): + def test_repository_derive_reuses_expensive_fields_and_orders_stores(self) -> None: + server, _rpc = coordinator() + repository = server._ensure_job_bundle_service().template_repository - first = server._share_window_serialization_for_artifact( - self._ledger_artifact(shares, generation=1), - shares, - ) - second = server._share_window_serialization_for_artifact( - self._ledger_artifact(shares, generation=2), - shares, + first = repository.derive( + base_template(), generation=repository.reserve_generation() ) + self.assertTrue(repository.store_artifacts(first)) - self.assertIs(second, first) - - def test_shutdown_retires_current_spool(self) -> None: - server = self._coordinator() - shares = [spool_share(seq) for seq in range(1, 4)] - serialization = server._share_window_serialization_for_artifact( - self._ledger_artifact(shares, generation=1), - shares, + second = repository.derive( + first.template, generation=repository.reserve_generation() ) - lease = serialization.acquire_spooled_tail(shares) - assert lease is not None - serialization.release_spooled_tail() - self.assertFalse(lease[0].closed) - - server.retire_share_window_spool() - - self.assertTrue(lease[0].closed) - self.assertIsNone(serialization.acquire_spooled_tail(shares)) - - -FAKE_SERVE_BUILDER_COMMAND = [ - sys.executable, - str(Path(__file__).resolve().parent / "fixtures" / "fake_serve_builder.py"), -] - -class ServeBuilderTests(unittest.TestCase): - """Persistent --serve builder client, its fallback, and supersession.""" - - def _coordinator(self) -> PrismCoordinator: - server, _rpc = coordinator() - server.signing_seed_hex = "42" * 32 - server.ledger_attestation_signing_seed_hex = "43" * 32 - self.addCleanup(server.shutdown_serve_builder) - self.addCleanup(server.retire_share_window_spool) - return server - - def _serialization( - self, - server: PrismCoordinator, - shares: list[dict[str, object]], - *, - generation: int = 1, - ) -> object: - return server._share_window_serialization_for_artifact( - PayoutLedgerArtifact( - generation=generation, - payout_state_generation=0, - network_difficulty=1, - accepted_share_count=len(shares), - shares_json=tuple(shares), - prior_balances=(), - prepared_monotonic=time.monotonic(), - snapshot_anchor_ms=None, - ), - shares, + self.assertEqual(second.template, first.template) + # A same-fingerprint derivation reuses the expensive derived fields + # of the stored observation instead of recomputing them. + self.assertIs(second.transaction_hexes, first.transaction_hexes) + self.assertIs( + second.witness_merkle_leaves_hex, first.witness_merkle_leaves_hex ) + self.assertTrue(repository.store_artifacts(second)) + self.assertFalse(repository.store_artifacts(first)) + self.assertIs(repository.current_artifacts(), second) - def _build( - self, - server: PrismCoordinator, - shares: list[dict[str, object]], - serialization: object, - *, - height: int, - mode: str = "ok", - cancellation: object = None, - ) -> dict[str, object]: - with patch.dict( - os.environ, - {"FAKE_SERVE_BUILDER_MODE": mode}, - ), patch( - "lab.prism.prism_coordinator.prism_tool_command", - return_value=list(FAKE_SERVE_BUILDER_COMMAND), - ): - return server.build_audit_bundle( - shares=shares, - found_block={ - "block_height": height, - "coinbase_value_sats": 50_00000000, - "network_difficulty": 1, - "anchor_job_issued_at_ms": 1_700_000_000_000, - }, - prior_balances=[], - coinbase_script_sig_suffix_hex="00", - summary_only=True, - payout_policy={"policy": "day-one"}, - share_serialization=serialization, # type: ignore[arg-type] - cancellation=cancellation, # type: ignore[arg-type] - ) - - def test_cache_miss_uploads_window_then_hits_on_next_request(self) -> None: - server = self._coordinator() - shares = [spool_share(seq) for seq in range(1, 4)] - serialization = self._serialization(server, shares) - identities, compact_shares = _compact_share_payload(shares) - expected_window = { - "compact_share_identities": [ - list(identity) for identity in identities - ], - "compact_shares": [list(share) for share in compact_shares], - } - try: - first = self._build(server, shares, serialization, height=10) - second = self._build(server, shares, serialization, height=11) - - self.assertEqual(first["transport"], "serve") - self.assertTrue(first["request_had_window"]) - self.assertEqual(first["window"], expected_window) - self.assertEqual(first["found_block"]["block_height"], 10) - - self.assertEqual(second["transport"], "serve") - self.assertFalse(second["request_had_window"]) - self.assertEqual(second["window"], expected_window) - self.assertEqual(second["found_block"]["block_height"], 11) - - with server._serve_builder_metrics_lock: - counts = dict(server.serve_builder_counts) - window_counts = dict(server.serve_builder_window_cache_counts) - self.assertEqual(counts["requests"], 2) - self.assertEqual(counts["spawns"], 1) - self.assertEqual(counts["fallbacks"], 0) - self.assertEqual(counts["window_uploads"], 1) - self.assertEqual(window_counts, {"hits": 1, "misses": 1}) - finally: - server.shutdown_serve_builder() - - def test_needs_window_bounce_reuploads_and_succeeds(self) -> None: - server = self._coordinator() - shares = [spool_share(seq) for seq in range(1, 4)] - serialization = self._serialization(server, shares) - try: - first = self._build(server, shares, serialization, height=10) - self.assertTrue(first["request_had_window"]) - client = server._serve_builder - assert client is not None - second_shares = [spool_share(seq) for seq in range(1, 5)] - second_serialization = self._serialization( - server, - second_shares, - generation=2, - ) - # Pretend this window was already uploaded: the daemon's - # needs_window bounce must repair the divergence transparently. - client.note_uploaded_window( - second_serialization.share_snapshot_sha256 # type: ignore[attr-defined] - ) - - server._ensure_tip_refresh_state() - server._job_build_phase_local.tip_refresh_metrics = True - try: - second = self._build( - server, - second_shares, - second_serialization, - height=11, - ) - finally: - server._job_build_phase_local.tip_refresh_metrics = False - - self.assertEqual(second["transport"], "serve") - self.assertTrue(second["request_had_window"]) - self.assertEqual(second["found_block"]["block_height"], 11) - with server._serve_builder_metrics_lock: - counts = dict(server.serve_builder_counts) - self.assertEqual(counts["requests"], 2) - self.assertEqual(counts["fallbacks"], 0) - # One build observes serialization_copy exactly three times -- - # fragment precompose, the accumulated transport writes, and the - # daemon's reported timings -- matching the one-shot transport - # even when a needs_window bounce re-sent the request. - histogram = server.tip_refresh_build_phase_histograms[ - "serialization_copy" - ] - self.assertEqual(histogram["count"], 3) - finally: - server.shutdown_serve_builder() - - def test_hit_promotion_keeps_mirror_and_daemon_lru_aligned(self) -> None: - server = self._coordinator() - shares_a = [spool_share(seq) for seq in range(1, 4)] - shares_b = [spool_share(seq) for seq in range(1, 5)] - shares_c = [spool_share(seq) for seq in range(1, 6)] - ser_a = self._serialization(server, shares_a, generation=1) - ser_b = self._serialization(server, shares_b, generation=2) - ser_c = self._serialization(server, shares_c, generation=3) - try: - self.assertTrue( - self._build(server, shares_a, ser_a, height=10)[ - "request_had_window" - ] - ) - self.assertTrue( - self._build(server, shares_b, ser_b, height=11)[ - "request_had_window" - ] - ) - # The hit promotes window A on both sides; uploading a third - # window must therefore evict B everywhere. - self.assertFalse( - self._build(server, shares_a, ser_a, height=12)[ - "request_had_window" - ] - ) - self.assertTrue( - self._build(server, shares_c, ser_c, height=13)[ - "request_had_window" - ] - ) - - aligned = self._build(server, shares_a, ser_a, height=14) - - self.assertEqual(aligned["transport"], "serve") - self.assertFalse(aligned["request_had_window"]) - with server._serve_builder_metrics_lock: - counts = dict(server.serve_builder_counts) - self.assertEqual(counts["requests"], 5) - self.assertEqual(counts["window_uploads"], 3) - self.assertEqual(counts["fallbacks"], 0) - finally: - server.shutdown_serve_builder() - - def test_daemon_crash_falls_back_to_one_shot(self) -> None: - server = self._coordinator() - shares = [spool_share(seq) for seq in range(1, 4)] - serialization = self._serialization(server, shares) - try: - result = self._build( - server, - shares, - serialization, - height=10, - mode="crash-before-response", - ) - - self.assertEqual(result["transport"], "one-shot") - self.assertEqual( - result["received"]["found_block"]["block_height"], - 10, - ) - self.assertIn("compact_shares", result["received"]) - with server._serve_builder_metrics_lock: - counts = dict(server.serve_builder_counts) - self.assertIsNone(server._serve_builder) - self.assertEqual(counts["fallbacks"], 1) - self.assertEqual(counts["requests"], 0) - finally: - server.shutdown_serve_builder() - - def test_daemon_splice_failure_poisons_spool_and_one_shot_recovers( + def test_store_artifacts_prunes_stale_fingerprints_and_cancels_builds( self, ) -> None: - server = self._coordinator() - shares = [spool_share(seq) for seq in range(1, 4)] - serialization = self._serialization(server, shares) - try: - # The daemon upload dies mid-splice; the spool is poisoned before - # falling back, so the fresh one-shot subprocess streams the - # cached fragments instead of retrying the same failing transfer. - with patch( - "lab.prism.prism_coordinator.os.splice", - side_effect=[16, OSError("splice io error")], - ): - result = self._build(server, shares, serialization, height=10) - - self.assertEqual(result["transport"], "one-shot") - self.assertIn("compact_shares", result["received"]) - self.assertTrue(serialization._spool_failed) # type: ignore[attr-defined] - with server._serve_builder_metrics_lock: - counts = dict(server.serve_builder_counts) - self.assertEqual(counts["fallbacks"], 1) - finally: - server.shutdown_serve_builder() - - def test_protocol_mismatch_falls_back_to_one_shot(self) -> None: - server = self._coordinator() - shares = [spool_share(seq) for seq in range(1, 4)] - serialization = self._serialization(server, shares) - try: - result = self._build( - server, - shares, - serialization, - height=10, - mode="protocol-mismatch", - ) - - self.assertEqual(result["transport"], "one-shot") - with server._serve_builder_metrics_lock: - counts = dict(server.serve_builder_counts) - self.assertIsNone(server._serve_builder) - self.assertEqual(counts["fallbacks"], 1) - # The mismatched daemon was a real worker launch and must appear - # in the lifecycle totals alongside the one-shot fallback. - with server._job_build_scheduler_lock: - worker_counts = dict(server.job_build_worker_counts) - self.assertEqual(worker_counts["starts"], 2) - finally: - server.shutdown_serve_builder() - - def test_disabled_serve_builder_uses_one_shot_directly(self) -> None: - server = self._coordinator() - shares = [spool_share(seq) for seq in range(1, 4)] - serialization = self._serialization(server, shares) - with patch.dict(os.environ, {"PRISM_BUILDER_SERVE": "0"}): - result = self._build(server, shares, serialization, height=10) - - self.assertEqual(result["transport"], "one-shot") - with server._serve_builder_metrics_lock: - counts = dict(server.serve_builder_counts) - self.assertEqual(counts["spawns"], 0) - - def test_daemon_timeout_fallback_shares_the_build_deadline(self) -> None: - server = self._coordinator() - # An already-exhausted budget: the daemon attempt must not grant the - # one-shot fallback a fresh full deadline, and the timeout is counted - # as one worker failure, not two. - server.bundle_build_timeout_seconds = 0.0 - shares = [spool_share(seq) for seq in range(1, 4)] - serialization = self._serialization(server, shares) - - with self.assertRaisesRegex( - RuntimeError, - "qbit-prism-build-audit-bundle timed out", - ): - self._build( - server, - shares, - serialization, - height=10, - mode="hang-after-request", + server, rpc = coordinator() + service = server._ensure_job_bundle_service() + repository = service.template_repository + old = server.store_template_artifacts(dict(rpc.template)) + assert old is not None + cancellations: list[tuple[str, bool]] = [] + server._cancel_obsolete_job_builds = ( # type: ignore[method-assign] + lambda reason, *, keep_published_snapshot=False: cancellations.append( + (reason, keep_published_snapshot) ) - - with server._serve_builder_metrics_lock: - counts = dict(server.serve_builder_counts) - self.assertEqual(counts["fallbacks"], 1) - self.assertEqual(counts["spawns"], 0) - self.assertEqual(server.tip_refresh_worker_failures, 1) - # The live daemon killed at the handshake deadline is a recorded - # termination, and the one-shot replacement consumes the pending - # restart: two launches, one termination, one restart. - with server._job_build_scheduler_lock: - worker_counts = dict(server.job_build_worker_counts) - self.assertFalse(server._job_build_worker_restart_pending) - self.assertEqual(worker_counts["starts"], 2) - self.assertEqual(worker_counts["terminations"], 1) - self.assertEqual(worker_counts["restarts"], 1) - - def test_idle_daemon_death_counts_crash_and_restart(self) -> None: - server = self._coordinator() - shares = [spool_share(seq) for seq in range(1, 4)] - serialization = self._serialization(server, shares) - try: - first = self._build(server, shares, serialization, height=10) - self.assertEqual(first["transport"], "serve") - client = server._serve_builder - assert client is not None - client.process.kill() - client.process.wait(timeout=5) - - second = self._build(server, shares, serialization, height=11) - - self.assertEqual(second["transport"], "serve") - with server._job_build_scheduler_lock: - counts = dict(server.job_build_worker_counts) - self.assertFalse(server._job_build_worker_restart_pending) - self.assertEqual(counts["crashes"], 1) - self.assertEqual(counts["restarts"], 1) - self.assertEqual(server.tip_refresh_worker_restarts, 1) - with server._serve_builder_metrics_lock: - serve_counts = dict(server.serve_builder_counts) - self.assertEqual(serve_counts["spawns"], 2) - self.assertEqual(serve_counts["fallbacks"], 0) - finally: - server.shutdown_serve_builder() - - def test_daemon_respawn_counts_as_worker_restart(self) -> None: - server = self._coordinator() - shares = [spool_share(seq) for seq in range(1, 4)] - serialization = self._serialization(server, shares) - try: - # A prior worker ended abnormally (crash or supersession kill) - # with no immediate fallback consuming the flag; the replacement - # daemon spawn must claim the restart instead of leaking it to - # an unrelated later one-shot build. - server._ensure_tip_refresh_state() - with server._job_build_scheduler_lock: - server._job_build_worker_restart_pending = True - - result = self._build(server, shares, serialization, height=10) - - self.assertEqual(result["transport"], "serve") - with server._job_build_scheduler_lock: - counts = dict(server.job_build_worker_counts) - self.assertFalse(server._job_build_worker_restart_pending) - self.assertEqual(counts["restarts"], 1) - self.assertGreaterEqual(counts["starts"], 1) - self.assertEqual(server.tip_refresh_worker_restarts, 1) - finally: - server.shutdown_serve_builder() - - def test_daemon_detaches_from_build_control_after_request(self) -> None: - server = self._coordinator() - shares = [spool_share(seq) for seq in range(1, 4)] - serialization = self._serialization(server, shares) - control = prism_coordinator_module._JobBundleBuildControl( - key=("serve-detach",), - previousblockhash="11" * 32, - payout_state_generation=0, - payout_artifact_generation=1, - ) - with server._job_cache_lock: - server._active_job_bundle_builds[control.key] = control - server._job_build_phase_local.bundle_build_control = control - try: - result = self._build(server, shares, serialization, height=10) - - self.assertEqual(result["transport"], "serve") - # The completed request detached the daemon: a late supersession - # of this build has no process reference left to terminate. - self.assertIsNone(control.process) - control.cancel_event.set() - client = server._serve_builder - assert client is not None - self.assertIsNone(client.process.poll()) - finally: - server._job_build_phase_local.bundle_build_control = None - control.cancel_event.set() - with server._job_cache_lock: - server._active_job_bundle_builds.pop(control.key, None) - server.shutdown_serve_builder() - - def test_supersession_cancels_in_flight_daemon_request(self) -> None: - server = self._coordinator() - shares = [spool_share(seq) for seq in range(1, 4)] - serialization = self._serialization(server, shares) - control = prism_coordinator_module._JobBundleBuildControl( - key=("serve-test",), - previousblockhash="11" * 32, - payout_state_generation=0, - payout_artifact_generation=1, ) - with server._job_cache_lock: - server._active_job_bundle_builds[control.key] = control - errors: list[BaseException] = [] - results: list[object] = [] - - def build() -> None: - # The bundle-build control travels on the builder thread's local - # state, exactly as _execute_job_build installs it. - server._job_build_phase_local.bundle_build_control = control - try: - results.append( - self._build( - server, - shares, - serialization, - height=10, - mode="hang-after-request", - ) + with service._job_cache_lock: + service._job_bundle_cache = OrderedDict( + ( + (("old", 0), SimpleNamespace(template_fingerprint=old.fingerprint)), + (("other", 0), SimpleNamespace(template_fingerprint="ff" * 32)), ) - except BaseException as exc: # noqa: BLE001 - surfaced below - errors.append(exc) - finally: - server._job_build_phase_local.bundle_build_control = None - - thread = threading.Thread(target=build) - thread.start() - try: - deadline = time.monotonic() + 5.0 - while time.monotonic() < deadline: - if control.process is not None: - break - time.sleep(0.005) - self.assertIsNotNone(control.process) - - control.cancel_event.set() - thread.join(5.0) - finally: - control.cancel_event.set() - thread.join(1.0) - server.shutdown_serve_builder() - - self.assertFalse(thread.is_alive()) - self.assertEqual(results, []) - self.assertEqual(len(errors), 1) - self.assertIsInstance(errors[0], JobBuildSuperseded) - self.assertIsNone(server._serve_builder) - - -class ReorgReconcileRefreshPathTests(unittest.TestCase): - """The tip-refresh reconcile stage: memo reuse and fetch overlap.""" - - def test_poll_memo_hit_skips_full_reconcile_pass(self) -> None: - server, rpc = coordinator() - install_fake_bundle_builder(server) - server.reorg_reconciler_enabled = True - state = client(1) - state.send = lambda _payload: None # type: ignore[method-assign] - server.clients = {state} - with server.lock: - server._reorg_reconcile_trusted_memo[rpc.tip] = time.monotonic() - - def unexpected_pass(**_kwargs: object) -> dict[str, object]: - raise AssertionError( - "memo-armed poll must not queue a serialized reconcile pass" ) - server.reconcile_prism_pool_blocks_once = ( # type: ignore[method-assign] - unexpected_pass + replacement = repository.derive( + base_template(height=11, prevhash="22" * 32), + generation=repository.reserve_generation(), ) + self.assertNotEqual(replacement.fingerprint, old.fingerprint) + self.assertTrue(repository.store_artifacts(replacement)) - try: - refreshed = server.poll_qbit_tip_template_once() - finally: - server.shutdown_tip_refresh_executor() - - self.assertEqual(refreshed, 1) - # A memo hit never needs the prefetch worker. - self.assertIsNone(server._reconcile_prefetch_executor) - metrics = server.metrics_payload() - self.assertIn( - 'qbit_prism_reorg_reconcile_lookups_total{path="tip_refresh",source="memo_hit"} 1', - metrics, - ) - self.assertIn( - 'qbit_prism_tip_refresh_bundle_phase_seconds_count{phase="reorg_reconcile"} 1', - metrics, + self.assertEqual( + cancellations, + [("template fingerprint superseded", True)], ) + with service._job_cache_lock: + self.assertEqual(dict(service._job_bundle_cache), {}) + self.assertIs(repository.current_artifacts(), replacement) - def test_poll_overlaps_reconcile_with_template_fetch(self) -> None: - # Deadlock-free only when the reconcile pass and the template fetch - # genuinely run concurrently: the fetch refuses to return before the - # reconcile starts, and the reconcile refuses to return before the - # fetch starts. - fetch_started = threading.Event() - reconcile_running = threading.Event() - - class OverlapProbeRpc(FakeRpc): - def call( - self, - method: str, - params: list[object] | None = None, - ) -> object: - if method == "getblocktemplate": - fetch_started.set() - assert reconcile_running.wait(5.0), ( - "reconcile prefetch did not start while the template " - "fetch was in flight" - ) - return super().call(method, params) + def test_repository_adoption_seeds_generation_like_first_touch(self) -> None: + server, _rpc = coordinator() + repository = server._ensure_job_bundle_service().template_repository + seeded = repository.derive(base_template(), generation=5) - template = base_template() - rpc = OverlapProbeRpc(template, tip=str(template["previousblockhash"])) - server, _ = coordinator(template=template) - server.rpc = rpc - install_fake_bundle_builder(server) - server.reorg_reconciler_enabled = True - state = client(1) - state.send = lambda _payload: None # type: ignore[method-assign] - server.clients = {state} - reconciled_tips: list[str] = [] + repository.adopt_template_artifacts(seeded) - def fake_ensure(tip_hash: str) -> bool: - reconciled_tips.append(tip_hash) - reconcile_running.set() - assert fetch_started.wait(5.0), ( - "template fetch did not start while the reconcile pass was " - "in flight" - ) - # A real trusted pass arms the per-tip memo; the join validates - # the overlapped result against that armed entry. - with server.lock: - server._reorg_reconcile_trusted_memo[tip_hash] = time.monotonic() - return True + self.assertIs(repository.current_artifacts(), seeded) + self.assertEqual(repository.reserve_generation(), 6) - server.ensure_reorg_reconciled_for_tip = ( # type: ignore[method-assign] - fake_ensure + repository.adopt_template_artifact_generation(10) + # An explicitly seeded counter is authoritative: later artifact + # adoption may replace the artifacts but not rewind the counter. + repository.adopt_template_artifacts( + repository.derive(base_template(), generation=3) ) + self.assertEqual(repository.reserve_generation(), 11) - try: - refreshed = server.poll_qbit_tip_template_once() - finally: - server.shutdown_tip_refresh_executor() - self.assertEqual(refreshed, 1) - self.assertEqual(reconciled_tips, [rpc.tip]) - metrics = server.metrics_payload() - self.assertIn( - 'qbit_prism_reorg_reconcile_lookups_total{path="tip_refresh",source="overlap"} 1', - metrics, +class JobBundleCacheAdmissionTests(unittest.TestCase): + def admission_fixture(self): # type: ignore[no-untyped-def] + server, rpc = coordinator() + service = server._ensure_job_bundle_service() + artifacts = server.store_template_artifacts(dict(rpc.template)) + assert artifacts is not None + generation = seed_published_payout_artifact(server, "payout-sha") + server._artifacts_buildable_locked = lambda _artifacts: True # type: ignore[method-assign] + server._published_snapshot_artifacts_locked = ( # type: ignore[method-assign] + lambda _artifacts: False ) + return server, service, artifacts, generation - def test_poll_reconciles_snapshot_tip_serially_when_tip_moves_before_fetch( - self, - ) -> None: - # The cheap probe observed a tip that was replaced before the - # template fetch. The prefetched pass proved the superseded hash, so - # the poll must reconcile the snapshot's newer tip on its own thread. - template = base_template() - new_tip = str(template["previousblockhash"]) - old_tip = "ab" * 32 - - class MovingTipRpc(FakeRpc): - def __init__(self) -> None: - super().__init__(template, tip=new_tip) - self.best_tip_calls = 0 + def test_cache_admission_requires_current_payout_state(self) -> None: + server, service, artifacts, generation = self.admission_fixture() - def call( - self, - method: str, - params: list[object] | None = None, - ) -> object: - if method == "getbestblockhash": - self.best_tip_calls += 1 - if self.best_tip_calls == 1: - self.calls.append(method) - return old_tip - return super().call(method, params) - - rpc = MovingTipRpc() - server, _ = coordinator(template=template) - server.rpc = rpc - install_fake_bundle_builder(server) - server.reorg_reconciler_enabled = True - state = client(1) - state.send = lambda _payload: None # type: ignore[method-assign] - server.clients = {state} - reconcile_calls: list[str] = [] - prefetch_drained = threading.Event() - - def fake_ensure(tip_hash: str) -> bool: - reconcile_calls.append(tip_hash) - if tip_hash == old_tip: - prefetch_drained.set() - return True - - server.ensure_reorg_reconciled_for_tip = ( # type: ignore[method-assign] - fake_ensure + exact = cached_bundle( + artifacts, + payout_generation=generation, + payout_artifact_sha256="payout-sha", + key=("exact",), ) + self.assertTrue(service._cache_job_bundle_if_current(exact, artifacts)) + with service._job_cache_lock: + self.assertIn(("exact",), service._job_bundle_cache) - try: - refreshed = server.poll_qbit_tip_template_once() - finally: - server.shutdown_tip_refresh_executor() - - self.assertEqual(refreshed, 1) - self.assertTrue(prefetch_drained.wait(5.0)) - self.assertIn(new_tip, reconcile_calls) - self.assertEqual( - sorted(reconcile_calls), - sorted([old_tip, new_tip]), - ) - metrics = server.metrics_payload() - self.assertIn( - 'qbit_prism_reorg_reconcile_lookups_total{path="tip_refresh",source="serial"} 1', - metrics, + stale_generation = cached_bundle( + artifacts, + payout_generation=generation + 1, + payout_artifact_sha256="payout-sha", + key=("stale-generation",), ) - - def test_poll_reproves_when_memo_evicted_during_template_fetch(self) -> None: - # A tip that flips away and back during the template fetch evicts - # the memo entry so post-flip pool-block state is re-proven. The - # join must judge the memo at join time, never the pre-fetch probe. - template = base_template() - server, _ = coordinator(template=template) - server.reorg_reconciler_enabled = True - tip = str(template["previousblockhash"]) - - class EvictingRpc(FakeRpc): - def call( - self, - method: str, - params: list[object] | None = None, - ) -> object: - if method == "getblocktemplate": - # Models a detection during the fetch unarming the memo. - with server.lock: - server._reorg_reconcile_trusted_memo.clear() - return super().call(method, params) - - rpc = EvictingRpc(template, tip=tip) - server.rpc = rpc - install_fake_bundle_builder(server) - state = client(1) - state.send = lambda _payload: None # type: ignore[method-assign] - server.clients = {state} - with server.lock: - server._reorg_reconcile_trusted_memo[tip] = time.monotonic() - reconcile_calls: list[str] = [] - - def fake_ensure(tip_hash: str) -> bool: - reconcile_calls.append(tip_hash) - return True - - server.ensure_reorg_reconciled_for_tip = ( # type: ignore[method-assign] - fake_ensure + self.assertFalse( + service._cache_job_bundle_if_current(stale_generation, artifacts) ) - try: - refreshed = server.poll_qbit_tip_template_once() - finally: - server.shutdown_tip_refresh_executor() - - self.assertEqual(refreshed, 1) - self.assertEqual(reconcile_calls, [tip]) - metrics = server.metrics_payload() - self.assertIn( - 'qbit_prism_reorg_reconcile_lookups_total{path="tip_refresh",source="serial"} 1', - metrics, + stale_artifact = cached_bundle( + artifacts, + payout_generation=generation, + payout_artifact_sha256="different-sha", + key=("stale-artifact",), ) - self.assertIn( - 'qbit_prism_reorg_reconcile_lookups_total{path="tip_refresh",source="memo_hit"} 0', - metrics, + self.assertFalse( + service._cache_job_bundle_if_current(stale_artifact, artifacts) ) - def test_poll_reproves_when_memo_rearmed_across_detection_epoch(self) -> None: - # A pass whose execution straddles a flip away and back can re-arm - # the memo after the flip-back (the latest detected hash matches - # again), but its proof belongs to the closed epoch. The join must - # refuse memo entries when the detection epoch moved mid-fetch. - template = base_template() - server, _ = coordinator(template=template) - server.reorg_reconciler_enabled = True - tip = str(template["previousblockhash"]) - - class FlipBackRpc(FakeRpc): - def call( - self, - method: str, - params: list[object] | None = None, - ) -> object: - if method == "getblocktemplate": - # Models a flip away and back detected during the fetch - # (two epoch bumps) with a straddling pass re-arming the - # memo for the same hash afterwards. - with server.lock: - server.tip_detection_epoch = ( - int(getattr(server, "tip_detection_epoch", 0)) + 2 - ) - server._reorg_reconcile_trusted_memo[tip] = ( - time.monotonic() - ) - return super().call(method, params) - - rpc = FlipBackRpc(template, tip=tip) - server.rpc = rpc - install_fake_bundle_builder(server) - state = client(1) - state.send = lambda _payload: None # type: ignore[method-assign] - server.clients = {state} - with server.lock: - server._reorg_reconcile_trusted_memo[tip] = time.monotonic() - reconcile_calls: list[str] = [] - - def fake_ensure(tip_hash: str) -> bool: - reconcile_calls.append(tip_hash) - return True - - server.ensure_reorg_reconciled_for_tip = ( # type: ignore[method-assign] - fake_ensure + stale_epoch = cached_bundle( + artifacts, + payout_generation=generation, + payout_artifact_sha256="payout-sha", + key=("stale-epoch",), + collection_only=False, + payout_append_invalidation_epoch=7, ) - - try: - refreshed = server.poll_qbit_tip_template_once() - finally: - server.shutdown_tip_refresh_executor() - - self.assertEqual(refreshed, 1) - self.assertEqual(reconcile_calls, [tip]) - metrics = server.metrics_payload() - self.assertIn( - 'qbit_prism_reorg_reconcile_lookups_total{path="tip_refresh",source="serial"} 1', - metrics, + self.assertFalse( + service._cache_job_bundle_if_current(stale_epoch, artifacts) ) - def test_overlap_join_blocks_when_trust_flips_after_prefetch(self) -> None: - # The prefetched pass proves trust when it runs, up to a fetch - # window earlier. Headers running ahead without a best-hash change - # (an arriving reorg, no detection) must still block the refresh at - # the join, matching the memo branch's live trust check. - template = base_template() - server, _ = coordinator(template=template) - server.reorg_reconciler_enabled = True - tip = str(template["previousblockhash"]) - - class TrustFlipRpc(FakeRpc): - def call( - self, - method: str, - params: list[object] | None = None, - ) -> object: - if method == "getblocktemplate": - # Headers run ahead after the fetch: untrusted view with - # no tip detection (epoch unchanged). - self.blockchain_info["headers"] = 101 - return super().call(method, params) + with service._job_cache_lock: + self.assertEqual(list(service._job_bundle_cache), [("exact",)]) - rpc = TrustFlipRpc(template, tip=tip) - server.rpc = rpc - install_fake_bundle_builder(server) - state = client(1) - state.send = lambda _payload: None # type: ignore[method-assign] - server.clients = {state} - server.ensure_reorg_reconciled_for_tip = ( # type: ignore[method-assign] - lambda _tip: True + def test_cache_admission_requires_current_template_or_pinned_snapshot( + self, + ) -> None: + server, service, old, generation = self.admission_fixture() + repository = service.template_repository + stale_bundle = cached_bundle( + old, + payout_generation=generation, + payout_artifact_sha256="payout-sha", + key=("old-after-publication",), ) - - try: - with self.assertRaises(TemplateRefreshBlocked) as raised: - server.poll_qbit_tip_template_once() - finally: - server.shutdown_tip_refresh_executor() - - self.assertIn("untrusted", str(raised.exception)) - self.assertIsNone(state.active_job) - - def test_tip_refresh_join_bounds_a_stalled_reconcile_prefetch(self) -> None: - # The overlap join must never park the poll loop on a crawling - # prefetched pass: it times out into the normal blocked-retry path - # within the CONFIGURED budget (not merely eventually), retries - # re-join the identical still-running future without starting - # another pass, and the pass completing unblocks the next poll. - server, rpc = coordinator() - server.reorg_reconciler_enabled = True - server.reconcile_prefetch_join_timeout_seconds = 0.05 - started = threading.Event() - release = threading.Event() - pass_calls = [0] - - def slow_ensure(tip_hash: str) -> bool: - pass_calls[0] += 1 - started.set() - assert release.wait(10.0) - return True - - server.ensure_reorg_reconciled_for_tip = ( # type: ignore[method-assign] - slow_ensure + server._cancel_obsolete_job_builds = ( # type: ignore[method-assign] + lambda _reason, **_kwargs: None ) - try: - join_started = time.monotonic() - with self.assertRaises(TemplateRefreshBlocked): - server.poll_qbit_tip_template_once() - # Generous scheduler tolerance, but far below any hardcoded - # multi-second budget: the configured 0.05s must be what fired. - self.assertLess(time.monotonic() - join_started, 1.0) - self.assertTrue(started.is_set()) - with server._reconcile_prefetch_executor_lock: - pending = server._reconcile_prefetch_pending - self.assertIsNotNone(pending) - assert pending is not None - first_future = pending[1] - self.assertFalse(first_future.done()) - - # A retry re-joins the same running future: same slot identity, - # no additional pass started. - with self.assertRaises(TemplateRefreshBlocked): - server.poll_qbit_tip_template_once() - with server._reconcile_prefetch_executor_lock: - pending_again = server._reconcile_prefetch_pending - assert pending_again is not None - self.assertIs(pending_again[1], first_future) - self.assertEqual(pass_calls[0], 1) - - release.set() - self.assertTrue(first_future.result(5.0)) - server.poll_qbit_tip_template_once() - finally: - release.set() - server.shutdown_reconcile_prefetch_executor() - - def test_serial_reconcile_branch_uses_the_bounded_join(self) -> None: - # With the memo disabled no overlap prefetch exists, so the pass - # lands in the serial re-prove branch -- which must route through - # the same prefetch slot and bounded join instead of parking the - # poll loop synchronously inside a crawling pass. - server, rpc = coordinator() - server.reorg_reconciler_enabled = True - server.reorg_reconcile_cache_seconds = 0.0 - server.reconcile_prefetch_join_timeout_seconds = 0.05 - started = threading.Event() - release = threading.Event() - - def slow_ensure(tip_hash: str) -> bool: - started.set() - assert release.wait(10.0) - return True - - server.ensure_reorg_reconciled_for_tip = ( # type: ignore[method-assign] - slow_ensure + replacement = repository.derive( + base_template(height=11, prevhash="22" * 32), + generation=repository.reserve_generation(), ) - try: - join_started = time.monotonic() - with self.assertRaises(TemplateRefreshBlocked): - server.poll_qbit_tip_template_once() - self.assertLess(time.monotonic() - join_started, 1.0) - self.assertTrue(started.is_set()) - with server._reconcile_prefetch_executor_lock: - pending = server._reconcile_prefetch_pending - self.assertIsNotNone(pending) - assert pending is not None - self.assertFalse(pending[1].done()) - - release.set() - self.assertTrue(pending[1].result(5.0)) - server.poll_qbit_tip_template_once() - finally: - release.set() - server.shutdown_reconcile_prefetch_executor() - - def test_pass_raised_timeout_is_not_logged_as_join_expiry(self) -> None: - # socket.timeout IS builtins.TimeoutError: a pass that raises it - # completes the future exceptionally and must surface as an ordinary - # blocked pass -- not as a join expiry, which would misdirect - # operators toward the join while nothing is running. - server, rpc = coordinator() - server.reorg_reconciler_enabled = True - server.reorg_reconcile_cache_seconds = 0.0 - server.reconcile_prefetch_join_timeout_seconds = 5.0 - - def timing_out_ensure(tip_hash: str) -> bool: - raise TimeoutError("pass-owned timeout") + self.assertTrue(repository.store_artifacts(replacement)) - server.ensure_reorg_reconciled_for_tip = ( # type: ignore[method-assign] - timing_out_ensure + # An old completion cannot cache after a newer fingerprint published. + self.assertFalse( + service._cache_job_bundle_if_current(stale_bundle, old) ) - try: - captured = io.StringIO() - join_started = time.monotonic() - with contextlib.redirect_stdout(captured): - with self.assertRaises(TemplateRefreshBlocked): - server.poll_qbit_tip_template_once() - self.assertLess(time.monotonic() - join_started, 2.0) - self.assertNotIn("join exceeded", captured.getvalue()) - finally: - server.shutdown_reconcile_prefetch_executor() - - def test_reconcile_prefetch_slot_is_reused_across_failed_attempts(self) -> None: - # A refresh attempt that dies before its join (template-RPC outage) - # must not queue another serialized pass per retry: the slot holds - # at most one outstanding prefetch, reused for the same tip and - # replaced (with the queued task cancelled) on a tip change. - server, _ = coordinator() - server.reorg_reconciler_enabled = True - started = threading.Event() - release = threading.Event() + with service._job_cache_lock: + self.assertNotIn(("old-after-publication",), service._job_bundle_cache) - def slow_ensure(tip_hash: str) -> bool: - started.set() - assert release.wait(5.0) - return True - - server.ensure_reorg_reconciled_for_tip = ( # type: ignore[method-assign] - slow_ensure + # The sole exception is a pinned rebuild of exactly the published + # snapshot, which direct issuance would otherwise rebuild all window. + server._published_snapshot_artifacts_locked = ( # type: ignore[method-assign] + lambda candidate: candidate.fingerprint == old.fingerprint ) - try: - first = server._submit_reconcile_prefetch("aa" * 32) - assert first is not None - self.assertTrue(started.wait(5.0)) - second = server._submit_reconcile_prefetch("aa" * 32) - self.assertIs(second, first) - third = server._submit_reconcile_prefetch("bb" * 32) - assert third is not None - self.assertIsNot(third, first) - fourth = server._submit_reconcile_prefetch("cc" * 32) - assert fourth is not None - self.assertIsNot(fourth, third) - self.assertTrue(third.cancelled()) - release.set() - self.assertTrue(first.result(5.0)) - self.assertTrue(fourth.result(5.0)) - finally: - release.set() - server.shutdown_reconcile_prefetch_executor() - - def test_job_build_memo_lookups_are_counted(self) -> None: - server, rpc = coordinator() - server.reorg_reconciler_enabled = True - with server.lock: - server._reorg_reconcile_trusted_memo[rpc.tip] = time.monotonic() - - self.assertTrue(server.ensure_reorg_reconciled_for_current_tip()) - - with server.lock: - server._reorg_reconcile_trusted_memo[rpc.tip] = ( - time.monotonic() - 100.0 - ) - serial_tips: list[str] = [] + self.assertTrue(service._cache_job_bundle_if_current(stale_bundle, old)) + with service._job_cache_lock: + self.assertIn(("old-after-publication",), service._job_bundle_cache) - def fake_ensure(tip_hash: str) -> bool: - serial_tips.append(tip_hash) - return True - - server.ensure_reorg_reconciled_for_tip = ( # type: ignore[method-assign] - fake_ensure + def test_collection_admission_requires_exact_generation_unlike_ready( + self, + ) -> None: + server, service, artifacts, generation = self.admission_fixture() + repository = service.template_repository + # A same-fingerprint, newer-generation observation of the same + # template supersedes the stored one without a fingerprint change. + newer = repository.derive( + artifacts.template, generation=repository.reserve_generation() ) + self.assertTrue(repository.store_artifacts(newer)) - self.assertTrue(server.ensure_reorg_reconciled_for_current_tip()) - - self.assertEqual(serial_tips, [rpc.tip]) - metrics = server.metrics_payload() - self.assertIn( - 'qbit_prism_reorg_reconcile_lookups_total{path="job_build",source="memo_hit"} 1', - metrics, + old_collection = cached_bundle( + artifacts, + payout_generation=generation, + payout_artifact_sha256="payout-sha", + key=("old-collection",), + collection_only=True, ) - self.assertIn( - 'qbit_prism_reorg_reconcile_lookups_total{path="job_build",source="serial"} 1', - metrics, + old_ready = cached_bundle( + artifacts, + payout_generation=generation, + payout_artifact_sha256="payout-sha", + key=("old-ready",), + collection_only=False, ) - def test_poll_falls_back_to_serial_reconcile_after_prefetch_shutdown( - self, - ) -> None: - server, rpc = coordinator() - install_fake_bundle_builder(server) - server.reorg_reconciler_enabled = True - state = client(1) - state.send = lambda _payload: None # type: ignore[method-assign] - server.clients = {state} - server.shutdown_reconcile_prefetch_executor() - reconcile_calls: list[str] = [] - - def fake_ensure(tip_hash: str) -> bool: - reconcile_calls.append(tip_hash) - return True - - server.ensure_reorg_reconciled_for_tip = ( # type: ignore[method-assign] - fake_ensure + self.assertFalse( + service._cache_job_bundle_if_current(old_collection, artifacts) ) + self.assertTrue(service._cache_job_bundle_if_current(old_ready, artifacts)) + with service._job_cache_lock: + self.assertNotIn(("old-collection",), service._job_bundle_cache) + self.assertIn(("old-ready",), service._job_bundle_cache) - try: - refreshed = server.poll_qbit_tip_template_once() - finally: - server.shutdown_tip_refresh_executor() - - self.assertEqual(refreshed, 1) - self.assertEqual(reconcile_calls, [rpc.tip]) - self.assertIsNone(server._reconcile_prefetch_executor) - metrics = server.metrics_payload() - self.assertIn( - 'qbit_prism_reorg_reconcile_lookups_total{path="tip_refresh",source="serial"} 1', - metrics, - ) +class JobBuildExecutorOwnershipTests(unittest.TestCase): + def test_executor_is_single_flight_named_and_shutdown_is_terminal(self) -> None: + server, _rpc = coordinator() + service = server._ensure_job_bundle_service() -class JobContextStampTests(unittest.TestCase): - def test_job_context_stamps_client_version_mask(self) -> None: - # Cross-connection resumes validate version bits against the mask - # negotiated on the delivering connection, so the context must - # record it at stamp time. - server, _ = coordinator() - install_fake_bundle_builder(server) - state = client(1) - state.version_mask = 0x1FFFE000 + executor = service._job_build_executor_locked() + self.assertIs(service._job_build_executor_locked(), executor) + self.assertEqual(executor._thread_name_prefix, "prism-job-build") - context = server.build_job_for_client(state, clean_jobs=True) + service.shutdown_job_build_executor() - self.assertEqual(context.version_mask, 0x1FFFE000) + self.assertTrue(service._job_build_executor_shutdown) + with self.assertRaisesRegex(RuntimeError, "shut down"): + service._job_build_executor_locked() + # The coordinator facade delegates to the same owner state. + with self.assertRaisesRegex(RuntimeError, "shut down"): + server._job_build_executor_locked() if __name__ == "__main__": diff --git a/tests/test_prism_coordinator_metrics.py b/tests/test_prism_coordinator_metrics.py new file mode 100644 index 000000000..7cf402e0a --- /dev/null +++ b/tests/test_prism_coordinator_metrics.py @@ -0,0 +1,1198 @@ +#!/usr/bin/env python3 +"""Focused PRISM coordinator metrics tests.""" +# ruff: noqa: F821 + +from __future__ import annotations + +from pathlib import Path +import tempfile +import unittest +from unittest import mock +from lab.prism import audit_http as audit_http_module +from tests import prism_coordinator_test_support as _job_support +from tests import prism_vardiff_test_support as _vardiff_support + + +def install_isolated_coordinator( + test_case: unittest.TestCase, + factory: object, +) -> None: + assert callable(factory) + + def isolated(*args: object, **kwargs: object) -> object: + result = factory(*args, **kwargs) + server = result[0] if isinstance(result, tuple) else result + temporary = tempfile.TemporaryDirectory() + server.audit_dir = Path(temporary.name) / "audit" + server.evidence_path = Path(temporary.name) / "state" / "evidence.json" + + def cleanup() -> None: + store = server.__dict__.get("_audit_artifact_store") + if store is not None: + store.close() + temporary.cleanup() + + test_case.addCleanup(cleanup) + return result + + globals()["coordinator"] = isolated + + +class _VardiffSupportTestCase(unittest.TestCase): + def setUp(self) -> None: + globals().update( + {name: getattr(_vardiff_support, name) for name in _vardiff_support.__all__} + ) + install_isolated_coordinator(self, _vardiff_support.coordinator) + + +class _JobSupportTestCase(unittest.TestCase): + def setUp(self) -> None: + globals().update( + {name: getattr(_job_support, name) for name in _job_support.__all__} + ) + install_isolated_coordinator(self, _job_support.coordinator) + + +class PrismCoordinatorVardiffTests(_VardiffSupportTestCase): + def test_metrics_include_issue_scope_operational_gauges(self) -> None: + server = coordinator() + server.submitted_share_count = 10 + server.stale_share_count = 2 + server.duplicate_share_count = 1 + server.low_difficulty_share_count = 3 + server.grace_credited_share_count = 6 + server.idle_retarget_count = 7 + server.rejection_counts_by_reason[PRISM_REJECTION_STALE_JOB] = 2 + server.rejection_counts_by_reason["duplicate-share"] = 1 + server.rejection_counts_by_reason["low-difficulty"] = 3 + server.tip_refresh_job_count = 4 + server.post_accept_refresh_failure_count = 5 + server.connection_limit_rejection_counts = {"global": 2, "username": 3} + server.accept_resource_exhaustion_count = 4 + server.connection_setup_failure_count = 5 + server._ensure_vardiff_idle_state() + with server._vardiff_idle_lock: + server.vardiff_idle_clients_inspected = 8 + server.vardiff_idle_skip_counts["busy"] = 2 + server.vardiff_idle_queue_depth = 1 + server.vardiff_idle_inflight = 2 + server.vardiff_idle_task_failures = 1 + server._observe_vardiff_idle_seconds("sweep", 0.005) + server._observe_vardiff_idle_seconds("task", 0.01) + + metrics = server.metrics_payload() + + self.assertIn("qbit_prism_submitted_shares_total 10", metrics) + self.assertIn("qbit_prism_stale_shares_total 2", metrics) + self.assertIn("qbit_prism_duplicate_shares_total 1", metrics) + self.assertIn("qbit_prism_low_difficulty_shares_total 3", metrics) + self.assertIn("qbit_prism_grace_credited_shares_total 6", metrics) + self.assertIn("qbit_prism_stratum_active_connections 0", metrics) + self.assertIn("qbit_prism_stratum_connection_limit 384", metrics) + self.assertIn("qbit_prism_stratum_peak_active_connections 0", metrics) + self.assertIn("qbit_prism_stratum_pending_initial_jobs 0", metrics) + self.assertIn("qbit_prism_stratum_pending_initial_job_limit 128", metrics) + self.assertIn( + "qbit_prism_stratum_oldest_genuinely_pending_initial_job_seconds 0.0", + metrics, + ) + self.assertIn( + "qbit_prism_stratum_current_tip_coverage_gap_seconds 0.0", + metrics, + ) + self.assertIn("qbit_prism_stratum_current_tip_job_coverage 1.0", metrics) + self.assertIn("qbit_prism_stratum_handler_threads 0", metrics) + self.assertIn("qbit_prism_job_delivery_queue_depth 0", metrics) + self.assertIn("qbit_prism_job_delivery_active_workers 0", metrics) + self.assertIn( + 'qbit_prism_stratum_connection_limit_rejections_total{scope="global"} 2', + metrics, + ) + self.assertIn( + 'qbit_prism_stratum_connection_limit_rejections_total{scope="username"} 3', + metrics, + ) + self.assertIn("qbit_prism_stratum_accept_resource_exhaustions_total 4", metrics) + self.assertIn("qbit_prism_stratum_connection_setup_failures_total 5", metrics) + self.assertIn('qbit_prism_rejections_total{reason_id="stale-job"} 2', metrics) + self.assertIn('qbit_prism_rejections_total{reason_id="duplicate-share"} 1', metrics) + self.assertIn('qbit_prism_rejections_total{reason_id="low-difficulty"} 3', metrics) + self.assertIn("qbit_prism_tip_refresh_jobs_total 4", metrics) + self.assertIn("qbit_prism_post_accept_refresh_failures_total 5", metrics) + self.assertIn("qbit_prism_vardiff_idle_retargets_total 7", metrics) + self.assertIn("qbit_prism_vardiff_idle_clients_inspected_total 8", metrics) + self.assertIn('qbit_prism_vardiff_idle_skips_total{reason="busy"} 2', metrics) + self.assertIn("qbit_prism_vardiff_idle_queue_depth 1", metrics) + self.assertIn("qbit_prism_vardiff_idle_inflight 2", metrics) + self.assertIn("qbit_prism_vardiff_idle_task_failures_total 1", metrics) + self.assertIn("qbit_prism_vardiff_idle_sweep_seconds_count 1", metrics) + self.assertIn("qbit_prism_vardiff_idle_retarget_task_seconds_count 1", metrics) + self.assertIn("qbit_prism_stale_share_percent 20", metrics) + self.assertIn("qbit_prism_coinbase_weight_headroom_bytes 1999750", metrics) + self.assertIn("qbit_prism_vardiff_enabled 1", metrics) + self.assertIn("qbit_prism_qbitd_initial_block_download 0", metrics) + self.assertIn("qbit_prism_qbitd_peers 4", metrics) + + def test_hot_lock_and_pending_candidate_metrics_are_bounded(self) -> None: + server = coordinator() + server.lock = _ObservedRLock() + lock_wait_started = threading.Event() + + def contend() -> None: + lock_wait_started.set() + with server.lock: + pass + + with server.lock: + waiter = threading.Thread(target=contend) + waiter.start() + self.assertTrue(lock_wait_started.wait(1)) + time.sleep(0.02) + waiter.join(1) + self.assertFalse(waiter.is_alive()) + + ledger = SingleWriterShareLedger() + block_hash = "bc" * 32 + ledger.persist_block_candidate_intent( + { + "schema": "qbit.prism.block-candidate-intent.v1", + "block_hash_hex": block_hash, + "block_hex": "00", + } + ) + server.ledger = ledger + before_attempt = "\n".join( + server.coordinator_lock_metrics_lines() + + server.block_submitter_metrics_lines() + ) + self.assertIn("qbit_prism_coordinator_lock_contentions_total 1", before_attempt) + self.assertIn("qbit_prism_block_candidates_pending 1", before_attempt) + self.assertRegex( + before_attempt, + r"qbit_prism_block_candidate_oldest_unattempted_seconds 0\.\d+", + ) + + self.assertTrue(ledger.mark_block_candidate_attempted(block_hash=block_hash)) + after_attempt = "\n".join(server.block_submitter_metrics_lines()) + self.assertIn( + "qbit_prism_block_candidate_oldest_unattempted_seconds 0.000000", + after_attempt, + ) + + def test_coordinator_lock_counts_every_granted_acquisition(self) -> None: + server = coordinator() + server.lock = _ObservedRLock() + + with server.lock: + # Re-entrant re-acquisition is an acquire() call and counts as one: + # the contended counter is per call, so the denominator must be too. + with server.lock: + pass + self.assertTrue(server.lock.acquire(blocking=False)) + server.lock.release() + + acquisitions, contentions, _sum, _max = ( + server.coordinator_lock_contention_snapshot() + ) + self.assertEqual((acquisitions, contentions), (3, 0)) + + lock_wait_started = threading.Event() + + def contend() -> None: + lock_wait_started.set() + with server.lock: + pass + + with server.lock: + waiter = threading.Thread(target=contend) + waiter.start() + self.assertTrue(lock_wait_started.wait(1)) + time.sleep(0.02) + waiter.join(1) + self.assertFalse(waiter.is_alive()) + + acquisitions, contentions, _sum, _max = ( + server.coordinator_lock_contention_snapshot() + ) + # The waiter's grant is one acquisition and one contention; the holder's + # own acquisition was uncontended. + self.assertEqual((acquisitions, contentions), (5, 1)) + + metrics = "\n".join(server.coordinator_lock_metrics_lines()) + self.assertIn("qbit_prism_coordinator_lock_acquisitions_total 5", metrics) + self.assertIn("qbit_prism_coordinator_lock_contentions_total 1", metrics) + + def test_coordinator_lock_timeout_is_contention_without_acquisition(self) -> None: + server = coordinator() + server.lock = _ObservedRLock() + timed_out = threading.Event() + + def give_up() -> None: + if not server.lock.acquire(timeout=0.05): + timed_out.set() + + with server.lock: + waiter = threading.Thread(target=give_up) + waiter.start() + waiter.join(1) + self.assertFalse(waiter.is_alive()) + self.assertTrue(timed_out.is_set()) + + acquisitions, contentions, _sum, _max = ( + server.coordinator_lock_contention_snapshot() + ) + # Only the holder acquired. The abandoned wait is real contention, which + # is why the derived contended fraction is an upper bound. + self.assertEqual((acquisitions, contentions), (1, 1)) + + def test_accepted_share_rate_is_left_to_the_counter(self) -> None: + server = coordinator() + + metrics = server.metrics_payload() + + # The gauge divided the ledger's lifetime accepted count by this + # process's uptime. Removed rather than redefined: a series that + # disappears is loud, a series that silently changes meaning is not. + self.assertNotIn("qbit_prism_shares_per_second", metrics) + self.assertRegex(metrics, r"\nqbit_prism_accepted_shares_total \d+\n") + self.assertIn("qbit_prism_vardiff_lane_accepted_shares_total", metrics) + + def test_metrics_include_bounded_worker_share_and_rejection_counters(self) -> None: + server = coordinator() + server.worker_metrics_limit = 1 + + server.note_worker_submitted_share("miner-a") + server.note_worker_accepted_share("miner-a", PRISM_CREDIT_POLICY_STALE_GRACE) + server.note_worker_submitted_share("miner-b") + server.record_rejection(PRISM_REJECTION_LOW_DIFFICULTY, worker="miner-b") + + metrics = server.metrics_payload() + + self.assertIn('qbit_prism_worker_submitted_shares_total{worker="miner-a"} 1', metrics) + self.assertIn('qbit_prism_worker_accepted_shares_total{worker="miner-a"} 1', metrics) + self.assertIn('qbit_prism_worker_grace_credited_shares_total{worker="miner-a"} 1', metrics) + self.assertIn('qbit_prism_worker_submitted_shares_total{worker="_other"} 1', metrics) + self.assertIn( + 'qbit_prism_worker_rejections_total{worker="_other",reason_id="low-difficulty"} 1', + metrics, + ) + + def test_metrics_include_ctv_broadcaster_progress_and_pass_duration(self) -> None: + server = coordinator() + server._record_ctv_fanout_broadcaster_progress() + server._record_ctv_fanout_broadcaster_progress() + server.observe_ctv_fanout_broadcaster_pass(102.0) + server.observe_ctv_fanout_broadcaster_chunk( + SimpleNamespace(processed_count=1, elapsed_seconds=0.25) + ) + server._record_ctv_fanout_broadcaster_yield() + + metrics = server.metrics_payload() + + self.assertIn("qbit_prism_ctv_fanout_broadcaster_processed_rows_total 2", metrics) + self.assertIn( + 'qbit_prism_ctv_fanout_broadcaster_pass_seconds_bucket{le="60"} 0', + metrics, + ) + self.assertIn( + 'qbit_prism_ctv_fanout_broadcaster_pass_seconds_bucket{le="120"} 1', + metrics, + ) + self.assertIn("qbit_prism_ctv_fanout_broadcaster_pass_seconds_sum 102.000000", metrics) + self.assertIn("qbit_prism_ctv_fanout_broadcaster_pass_seconds_count 1", metrics) + self.assertIn( + "qbit_prism_ctv_fanout_broadcaster_tip_refresh_yields_total 1", + metrics, + ) + self.assertIn( + 'qbit_prism_ctv_fanout_broadcaster_chunk_seconds_bucket{le="0.25"} 1', + metrics, + ) + self.assertIn( + 'qbit_prism_ctv_fanout_broadcaster_chunk_rows_bucket{le="1"} 1', + metrics, + ) + + def test_zero_worker_metric_limit_uses_overflow_bucket(self) -> None: + server = coordinator() + server.worker_metrics_limit = 0 + + server.note_worker_submitted_share("miner-a") + + self.assertEqual(set(server.worker_share_counts), {"_other"}) + self.assertEqual(server.worker_share_counts["_other"]["submitted"], 1) + + def test_unauthorized_submit_does_not_admit_payload_worker_metric_label(self) -> None: + server, state, _ledger = submit_coordinator() + server.worker_metrics_limit = 1 + + with self.assertRaises(StratumError) as raised: + server.handle_submit( + state, + ["spoofed-miner", "job-1", "00" * 8, "00000001", "00000002"], + ) + + self.assertEqual(raised.exception.reason, PRISM_REJECTION_UNAUTHORIZED_WORKER) + self.assertNotIn("spoofed-miner", server.worker_share_counts) + self.assertEqual(server.worker_share_counts["miner-a"]["submitted"], 0) + self.assertEqual( + server.worker_rejection_counts[("miner-a", PRISM_REJECTION_UNAUTHORIZED_WORKER)], + 1, + ) + + server.note_worker_submitted_share("miner-a") + + self.assertNotIn(PRISM_WORKER_METRICS_OVERFLOW_LABEL, server.worker_share_counts) + self.assertEqual(server.worker_share_counts["miner-a"]["submitted"], 1) + + def test_metrics_include_audit_artifact_storage_gauges(self) -> None: + server = coordinator() + with tempfile.TemporaryDirectory() as tempdir: + server.audit_dir = Path(tempdir) + files = { + f"prism-audit-bundle-body-{'aa' * 32}-{'bb' * 32}.json": b"abc", + f"prism-audit-share-segment-1-1-{'cc' * 32}.json": b"defg", + f"prism-live-audit-bundle-1-{'dd' * 32}.json": b"hi", + f"prism-live-audit-bundle-candidate-{'ee' * 32}.json": b"j", + f".prism-live-audit-bundle-candidate-{'ff' * 32}.json.tmp": b"klmno", + "operator-note.txt": b"pqrstu", + } + for name, body in files.items(): + (Path(tempdir) / name).write_bytes(body) + + metrics = server.metrics_payload() + + self.assertIn('qbit_prism_audit_artifact_bytes{kind="body"} 3', metrics) + self.assertIn('qbit_prism_audit_artifact_files{kind="body"} 1', metrics) + self.assertIn('qbit_prism_audit_artifact_bytes{kind="share_segment"} 4', metrics) + self.assertIn('qbit_prism_audit_artifact_files{kind="share_segment"} 1', metrics) + self.assertIn('qbit_prism_audit_artifact_bytes{kind="live_bundle"} 2', metrics) + self.assertIn('qbit_prism_audit_artifact_files{kind="live_bundle"} 1', metrics) + self.assertIn('qbit_prism_audit_artifact_bytes{kind="candidate"} 6', metrics) + self.assertIn('qbit_prism_audit_artifact_files{kind="candidate"} 2', metrics) + self.assertIn('qbit_prism_audit_artifact_bytes{kind="other"} 6', metrics) + self.assertIn('qbit_prism_audit_artifact_files{kind="other"} 1', metrics) + self.assertIn("qbit_prism_audit_artifact_scan_error 0", metrics) + + def test_metrics_payload_reads_one_a1_snapshot(self) -> None: + server = coordinator() + snapshot = { + "body": {"files": 1, "bytes": 2}, + "share_segment": {"files": 3, "bytes": 4}, + "live_bundle": {"files": 5, "bytes": 6}, + "candidate": {"files": 7, "bytes": 8}, + "other": {"files": 9, "bytes": 10}, + "scan_error": 0, + } + reader = mock.Mock(return_value=snapshot) + server.audit_artifact_metrics = reader # type: ignore[method-assign] + + metrics = server.metrics_payload() + + reader.assert_called_once_with() + self.assertIn('qbit_prism_audit_artifact_files{kind="body"} 1', metrics) + self.assertIn('qbit_prism_audit_artifact_bytes{kind="other"} 10', metrics) + self.assertIn("qbit_prism_audit_artifact_scan_error 0", metrics) + + def _pending_append(self, tag: str, accepted_at_ms: int = 2) -> PendingShareAppend: + from lab.prism.share_ledger import PendingShare + + return PendingShareAppend( + pending_share=PendingShare( + share_id=f"miner-a:{tag}", + miner_id="miner-a", + order_key="miner-a", + p2mr_program_hex="11" * 32, + share_difficulty=1, + network_difficulty=1, + template_height=10, + job_id="job-1", + job_issued_at_ms=1, + accepted_at_ms=accepted_at_ms, + ntime=1_700_000_000, + ), + username="miner-a", + job_id="job-1", + block_hash_hex=tag * 32, + collection_only=False, + credit_policy=None, + ) + + def test_share_ack_latency_histogram_observes_accept_and_reject(self) -> None: + # The read-to-ack instrument for share-ingest saturation, driven + # through the real handler loop: both outcomes are observed after + # their response write, so accepted-vs-rejected separates commit + # pressure from transport/thread saturation. + tip = "00" * 32 + server, state, _ledger = submit_coordinator(tip=tip) + sent: list[dict[str, object]] = [] + state.send = lambda payload: sent.append(payload) # type: ignore[method-assign] + submission = SimpleNamespace( + header_hex="af" * 80, + block_hash_hex="cf" * 32, + share_pass=True, + block_pass=False, + ) + coordinator_sock, peer = socket.socketpair() + state.sock = coordinator_sock + accepted_line = json.dumps( + { + "id": 7, + "method": "mining.submit", + "params": ["miner-a", "job-1", "00" * 8, "00000001", "00000002"], + } + ) + rejected_line = json.dumps( + { + "id": 8, + "method": "mining.submit", + "params": [ + "miner-a", + "missing-job", + "00" * 8, + "00000001", + "00000002", + ], + } + ) + + with patch( + "lab.prism.prism_coordinator.direct_stratum.assemble_submission", + return_value=submission, + ): + handler = threading.Thread(target=server.handle_client, args=(state,)) + handler.start() + peer.sendall((accepted_line + "\n" + rejected_line + "\n").encode()) + peer.close() + handler.join(5) + + self.assertFalse(handler.is_alive()) + self.assertEqual(sent[0], {"id": 7, "result": True, "error": None}) + self.assertEqual(sent[1]["id"], 8) + self.assertIsNotNone(sent[1]["error"]) + metrics = "\n".join(server.share_ack_metrics_lines()) + self.assertIn( + 'qbit_prism_share_ack_seconds_count{result="accepted"} 1', + metrics, + ) + self.assertIn( + 'qbit_prism_share_ack_seconds_count{result="rejected"} 1', + metrics, + ) + + def test_block_submit_histogram_measures_landed_to_rpc_interval(self) -> None: + # The race-critical span (candidate landed -> submitblock returned) + # must be observed exactly once per attempted submit, independent of + # the post-submit audit/persistence bookkeeping the outbox + # created_at/completed_at interval also includes. + old_tip = "00" * 32 + block_hash = "ab" * 32 + server, state, ledger = submit_coordinator(tip=old_tip) + server.stop_after_block = False + server.max_blocks = 10 + server.clients = {state} + sent: list[dict[str, object]] = [] + state.send = lambda payload: sent.append(payload) # type: ignore[method-assign] + + def build_fresh_job(client: ClientState, *, clean_jobs: bool) -> object: + return prism_context( + "fresh-job", + block_hash, + worker=state.worker, + difficulty=Decimal("1"), + clean_jobs=clean_jobs, + ) + + server.build_job_for_client = build_fresh_job # type: ignore[method-assign] + submission = SimpleNamespace( + header_hex="aa" * 80, + share_pass=True, + block_pass=True, + coinbase_tx_hex="c0ffee", + block_hash_hex=block_hash, + block_hex="00", + ) + + with tempfile.TemporaryDirectory() as tempdir: + server.audit_dir = Path(tempdir) + server.evidence_path = Path(tempdir) / "evidence.json" + server.ledger_writer_public_key_hex = "aa" * 32 + server.rpc = SubmitAcceptingTemplateRpc( + old_tip=old_tip, + block_hash=block_hash, + ledger=ledger, + ) + server.build_audit_bundle = lambda **_kwargs: verified_block_bundle() # type: ignore[method-assign] + server.verify_bundle = lambda *_args, **_kwargs: verified_audit_report() # type: ignore[method-assign] + + with patch( + "lab.prism.prism_coordinator.direct_stratum.assemble_submission", + return_value=submission, + ): + server.handle_submit( + state, + ["miner-a", "job-1", "00" * 8, "00000001", "00000002"], + ) + self.assertTrue(server.submit_next_block_candidate()) + + self.assertEqual(server.accepted_block_count, 1) + metrics = "\n".join(server.block_submitter_metrics_lines()) + self.assertIn("qbit_prism_block_submit_seconds_count 1", metrics) + self.assertIn( + 'qbit_prism_block_submit_seconds_bucket{le="+Inf"} 1', + metrics, + ) + # The candidate landed moments ago in this process, so the interval + # must fall inside the finite buckets, not only the +Inf overflow. + self.assertIn( + 'qbit_prism_block_submit_seconds_bucket{le="30"} 1', + metrics, + ) + + +class PrismCoordinatorReliabilityTests(_VardiffSupportTestCase): + def _bare_coordinator(self) -> PrismCoordinator: + server = PrismCoordinator.__new__(PrismCoordinator) + server.lock = threading.RLock() + server.stop_event = threading.Event() + server._heartbeats = {} + server._watchdog_pauses = {} + server._heartbeats_lock = threading.Lock() + server.watchdog_timeout_seconds = 120.0 + server.watchdog_interval_seconds = 15.0 + return server + + def test_overdue_heartbeats_flags_only_stale_subsystems(self) -> None: + server = self._bare_coordinator() + server._record_heartbeat("stratum_accept") + server._record_heartbeat("qbit_blockpoll") + now = time.monotonic() + + self.assertEqual(server._overdue_heartbeats(now), []) + + with server._heartbeats_lock: + server._heartbeats["qbit_blockpoll"] = now - 1_000.0 + + self.assertEqual(server._overdue_heartbeats(now), ["qbit_blockpoll"]) + + def test_block_submitter_retry_wait_heartbeats_in_bounded_slices(self) -> None: + server = self._bare_coordinator() + server.watchdog_timeout_seconds = 0.3 + clock = {"now": 0.0} + waits: list[float] = [] + overdue_samples: list[list[str]] = [] + + class AdvancingStopEvent: + def is_set(self) -> bool: + return False + + def wait(self, timeout: float) -> bool: + waits.append(timeout) + clock["now"] += timeout + overdue_samples.append(server._overdue_heartbeats(clock["now"])) + return False + + server.stop_event = AdvancingStopEvent() # type: ignore[assignment] + with patch( + "lab.prism.block_candidates.time.monotonic", + side_effect=lambda: clock["now"], + ): + self.assertFalse(server._wait_for_block_candidate_retry(1.0)) + + self.assertEqual(waits, [0.25, 0.25, 0.25, 0.25]) + self.assertEqual(overdue_samples, [[], [], [], []]) + self.assertEqual( + server._heartbeats["block_submitter"], + 1.0, + ) + + def test_blocked_candidate_phase_remains_watchdog_eligible(self) -> None: + server, state, _recording = submit_coordinator() + server._heartbeats = {} + server._watchdog_pauses = {} + server._heartbeats_lock = threading.Lock() + server.watchdog_timeout_seconds = 0.05 + entered_submission = threading.Event() + release_submission = threading.Event() + candidate = block_candidate( + server, + state, + SimpleNamespace( + coinbase_tx_hex="00", + block_hash_hex="fa" * 32, + block_hex="00", + share_pass=True, + block_pass=True, + ), + ) + + def blocked_submission(_candidate: PrismBlockCandidate) -> bool: + entered_submission.set() + release_submission.wait(2) + return True + + server.submit_block_candidate = blocked_submission # type: ignore[method-assign] + server.enqueue_block_candidate(candidate) + server._record_heartbeat("block_submitter") + submitter = threading.Thread(target=server.submit_next_block_candidate) + submitter.start() + try: + self.assertTrue(entered_submission.wait(1)) + time.sleep(0.08) + self.assertEqual( + server._overdue_heartbeats(time.monotonic()), + ["block_submitter"], + ) + finally: + release_submission.set() + submitter.join(2) + self.assertFalse(submitter.is_alive()) + + def test_progressing_ctv_pass_longer_than_watchdog_timeout_stays_healthy(self) -> None: + server = self._bare_coordinator() + server.ctv_broadcaster_limit = 200 + server.ctv_broadcaster_interval_seconds = 30.0 + clock = {"now": 0.0} + overdue_samples: list[list[str]] = [] + seen_limits: list[int] = [] + + class StopAfterOnePass: + def is_set(self) -> bool: + return False + + def wait(self, timeout: float) -> bool: + return True + + class ProgressingDaemon: + def run_once(self, *, limit: int, progress_callback: object, **_kwargs: object) -> object: + seen_limits.append(limit) + assert callable(progress_callback) + for _ in range(3): + clock["now"] += 80.0 + overdue_samples.append(server._overdue_heartbeats(clock["now"])) + progress_callback() + return SimpleNamespace( + scanned_count=3, + submitted_count=0, + updated_count=3, + failed_count=0, + yielded_to_tip_refresh=False, + ) + + server.stop_event = StopAfterOnePass() # type: ignore[assignment] + server.ctv_fanout_broadcast_daemon = ProgressingDaemon() + + with patch("lab.prism.prism_coordinator.time.monotonic", side_effect=lambda: clock["now"]), patch( + "builtins.print" + ): + server.ctv_fanout_broadcaster_loop() + + self.assertGreater(clock["now"], server.watchdog_timeout_seconds) + self.assertEqual(seen_limits, [200]) + self.assertEqual(overdue_samples, [[], [], []]) + self.assertEqual(server.ctv_broadcaster_processed_rows_total, 3) + self.assertEqual(server.ctv_broadcaster_pass_count, 1) + + def test_ctv_pass_completion_heartbeat_precedes_interval_wait(self) -> None: + server = self._bare_coordinator() + server.ctv_broadcaster_limit = 200 + server.ctv_broadcaster_interval_seconds = 30.0 + clock = {"now": 0.0} + wait_observation: dict[str, object] = {} + + class StopAfterIntervalWait: + def is_set(self) -> bool: + return False + + def wait(self, timeout: float) -> bool: + wait_observation["timeout"] = timeout + wait_observation["heartbeat"] = server._heartbeats["ctv_fanout_broadcaster"] + wait_observation["overdue_after_wait"] = server._overdue_heartbeats( + clock["now"] + timeout + ) + return True + + class IncidentDurationDaemon: + def run_once(self, *, limit: int, progress_callback: object, **_kwargs: object) -> object: + clock["now"] += 102.0 + return SimpleNamespace( + scanned_count=200, + submitted_count=0, + updated_count=200, + failed_count=0, + yielded_to_tip_refresh=False, + ) + + server.stop_event = StopAfterIntervalWait() # type: ignore[assignment] + server.ctv_fanout_broadcast_daemon = IncidentDurationDaemon() + + with patch("lab.prism.prism_coordinator.time.monotonic", side_effect=lambda: clock["now"]), patch( + "builtins.print" + ): + server.ctv_fanout_broadcaster_loop() + + self.assertEqual(wait_observation["timeout"], 30.0) + self.assertEqual(wait_observation["heartbeat"], 102.0) + self.assertEqual(wait_observation["overdue_after_wait"], []) + + def test_ctv_pass_without_progress_remains_watchdog_eligible(self) -> None: + server = self._bare_coordinator() + server.ctv_broadcaster_limit = 200 + server.ctv_broadcaster_interval_seconds = 30.0 + clock = {"now": 0.0} + entered_row = threading.Event() + release_row = threading.Event() + + class BlockingDaemon: + def run_once(self, *, limit: int, progress_callback: object, **_kwargs: object) -> object: + entered_row.set() + release_row.wait() + return SimpleNamespace( + scanned_count=1, + submitted_count=0, + updated_count=1, + failed_count=0, + yielded_to_tip_refresh=False, + ) + + server.ctv_fanout_broadcast_daemon = BlockingDaemon() + broadcaster_thread = threading.Thread(target=server.ctv_fanout_broadcaster_loop) + with patch("lab.prism.prism_coordinator.time.monotonic", side_effect=lambda: clock["now"]), patch( + "builtins.print" + ): + broadcaster_thread.start() + self.assertTrue(entered_row.wait(timeout=1.0)) + clock["now"] = server.watchdog_timeout_seconds + 1.0 + self.assertEqual( + server._overdue_heartbeats(clock["now"]), + ["ctv_fanout_broadcaster"], + ) + server.stop_event.set() + release_row.set() + broadcaster_thread.join(timeout=1.0) + + self.assertFalse(broadcaster_thread.is_alive()) + + def test_watchdog_pause_suppresses_known_long_critical_section(self) -> None: + server = self._bare_coordinator() + server._record_heartbeat("stratum_accept") + server._record_heartbeat("qbit_blockpoll") + now = time.monotonic() + with server._heartbeats_lock: + server._heartbeats["stratum_accept"] = now - 1_000.0 + server._heartbeats["qbit_blockpoll"] = now - 1_000.0 + + self.assertEqual(server._overdue_heartbeats(now), ["qbit_blockpoll", "stratum_accept"]) + + with server._watchdog_paused("qbit_blockpoll", "stratum_accept"): + self.assertEqual(server._overdue_heartbeats(now + 1_000.0), []) + + self.assertEqual(server._overdue_heartbeats(time.monotonic()), []) + + def test_block_submit_pause_names_cover_registered_refresh_and_idle_threads(self) -> None: + server = self._bare_coordinator() + for name in ("stratum_accept", "qbit_blockpoll", "qbit_blockwait", "vardiff_idle_sweep"): + server._record_heartbeat(name) + now = time.monotonic() + with server._heartbeats_lock: + for name in server._heartbeats: + server._heartbeats[name] = now - 1_000.0 + + pause_names = server._registered_watchdog_heartbeat_names( + "qbit_blockpoll", + "qbit_blockwait", + "vardiff_idle_sweep", + "stratum_accept", + ) + + with server._watchdog_paused(*pause_names): + self.assertEqual(server._overdue_heartbeats(now + 1_000.0), []) + + def test_pause_names_skip_removed_blockwait_without_resurrecting_heartbeat(self) -> None: + server = self._bare_coordinator() + server._record_heartbeat("qbit_blockpoll") + server._record_heartbeat("qbit_blockwait") + server._remove_watchdog_heartbeat("qbit_blockwait") + + pause_names = server._registered_watchdog_heartbeat_names("qbit_blockpoll", "qbit_blockwait") + + self.assertEqual(pause_names, ("qbit_blockpoll",)) + with server._watchdog_paused(*pause_names): + pass + self.assertNotIn("qbit_blockwait", server._heartbeats) + + def test_wedged_evidence_read_stays_watchdog_eligible(self) -> None: + server = self._bare_coordinator() + clock = {"now": 1000.0} + entered = threading.Event() + release = threading.Event() + + def wedged_stats() -> tuple[int, int]: + entered.set() + if not release.wait(5): + raise AssertionError("evidence read was not released") + return (7, 3) + + server.accepted_share_stats = wedged_stats # type: ignore[method-assign] + with patch( + "lab.prism.prism_coordinator.time.monotonic", + side_effect=lambda: clock["now"], + ): + server._record_heartbeat("block_submitter") + reader = threading.Thread(target=server.accepted_share_stats) + reader.start() + try: + self.assertTrue(entered.wait(5)) + clock["now"] += server.watchdog_timeout_seconds + 1.0 + # A blocked evidence read is deliberately NOT excused from + # the liveness budget: exit-and-replay is the recovery path + # for a genuinely wedged ledger read, and no disposition + # code may pause the submitter's heartbeat to hide it. + self.assertEqual( + server._overdue_heartbeats(clock["now"]), ["block_submitter"] + ) + self.assertNotIn("block_submitter", server._watchdog_pauses) + finally: + release.set() + reader.join(5) + + def test_metrics_include_accepted_stats_reconcile_status(self) -> None: + server = self._bare_coordinator() + server.ledger = SimpleNamespace( + accepted_stats_reconcile_status=lambda: { + "age_seconds": 12.5, + "failures": 3, + } + ) + lines = server._accepted_stats_reconcile_metric_lines() + self.assertIn( + "qbit_prism_accepted_stats_reconcile_failures_total 3", lines + ) + self.assertIn( + "qbit_prism_accepted_stats_reconcile_age_seconds 12.500000", lines + ) + # Ledgers without the accessor (file-backed, duck-typed) emit no + # reconcile lines rather than failing the scrape. + server.ledger = SimpleNamespace() + self.assertEqual(server._accepted_stats_reconcile_metric_lines(), []) + + def test_overdue_submitter_heartbeat_names_the_stuck_phase(self) -> None: + server = self._bare_coordinator() + clock = {"now": 1_000.0} + server._block_submitter_thread_ident = threading.get_ident() + with patch( + "lab.prism.prism_coordinator.time.monotonic", + side_effect=lambda: clock["now"], + ): + server._record_block_submitter_phase("replay-outbox-query") + clock["now"] += server.watchdog_timeout_seconds + 1.0 + self.assertEqual( + server._overdue_heartbeats(clock["now"]), + ["block_submitter:replay-outbox-query"], + ) + + def test_watchdog_starts_before_synchronous_startup_work(self) -> None: + source = inspect.getsource(PrismCoordinator._serve_with_listener_stack) + watchdog_start = source.index("target=self.watchdog_loop") + + self.assertEqual(source.count("target=self.watchdog_loop"), 1) + self.assertLess(watchdog_start, source.index("self.prewarm_startup_jobs")) + self.assertLess(watchdog_start, source.index("self.replay_recovered_shares")) + + def test_watchdog_hard_exits_when_fatal_stop_wins_during_startup(self) -> None: + server = self._bare_coordinator() + server._fatal_exit_requested = True + server.stop_event.set() + + with patch("lab.prism.prism_coordinator.os._exit") as hard_exit: + server.watchdog_loop() + + hard_exit.assert_called_once_with(1) + + +class PrismStampedJobFloorTests(_VardiffSupportTestCase): + """The listener floor must hold on the wire, not just in vardiff policy. + + Stamped jobs are the single choke point for every mining.set_difficulty + the coordinator sends, and marketplace verification judges the first one. + The regression here is a young chain: qbit network difficulty below the + high-diff floor used to drag the advertised difficulty down with it. + """ + + def stamp_coordinator(self) -> PrismCoordinator: + server = coordinator() + server.job_counter = 0 + server.share_weights_by_username = {} + server.default_share_weight = 1 + return server + + def cached_bundle(self) -> CachedJobBundle: + # bits 207fffff: regtest-grade network difficulty (~4.7e-10), far + # below the 500k marketplace floor. + qbit_target = target_from_compact("207fffff") + base_job = direct_stratum.DirectQbitStratumJob( + job_id="prism-template-base", + previousblockhash_display="00" * 32, + prevhash="00" * 32, + coinb1="", + coinb2="", + full_coinbase_prefix="", + full_coinbase_suffix="", + merkle_branch=(), + transaction_hexes=(), + version="20000000", + nbits="207fffff", + ntime="6553f100", + qbit_target=qbit_target, + share_target=qbit_target, + share_difficulty=Decimal("1"), + extranonce1_hex="ffffffff", + extranonce2_size=8, + clean_jobs=True, + ) + return CachedJobBundle( + key=("test",), + template=gbt_template("00" * 32), + template_fingerprint="fp", + coinbase_manifest={}, + shares_json=[], + prior_balances=[], + found_block={"network_difficulty": 1}, + collection_only=False, + issued_at_ms=12345, + base_job=base_job, + built_monotonic=time.monotonic(), + ) + + def highdiff_client(self) -> ClientState: + state = client() + state.worker = worker_identity() + state.listener_vardiff_config = highdiff_vardiff_config() + state.minimum_advertised_difficulty = Decimal("500000") + state.share_difficulty = Decimal("500000") + return state + + def test_post_block_notification_stamps_caller_heartbeat(self) -> None: + server, _state, _ledger = submit_coordinator() + seen: list[str] = [] + server._record_heartbeat = seen.append # type: ignore[method-assign] + server.refresh_jobs_after_accepted_block( + block_height=10, block_hash="bb" * 32, heartbeat_name="block_submitter" + ) + self.assertEqual(seen, ["block_submitter"]) + + # The client-thread pending refresh keeps the default poller heartbeat. + seen.clear() + server.refresh_jobs_after_accepted_block(block_height=11, block_hash="cc" * 32) + self.assertEqual(seen, ["qbit_blockpoll"]) + + +class HealthSnapshotTests(_JobSupportTestCase): + def test_health_payload_uses_aggregate_stats_not_all_shares(self) -> None: + ledger = FakeLedger() + server, _ = coordinator(ledger=ledger) + mark_progress_healthy(server) + payload = server.health_payload() + self.assertTrue(payload["ok"]) + self.assertEqual(payload["accepted_share_count"], 3) + self.assertEqual(payload["ready_miner_count"], 3) + self.assertGreaterEqual(ledger.stats_calls, 1) + + def test_cached_health_payload_computes_inline_without_refresher(self) -> None: + server, _ = coordinator() + mark_progress_healthy(server) + status, payload = server.cached_health_payload() + self.assertEqual(status, 200) + self.assertTrue(payload["ok"]) + + def test_cached_health_payload_serves_snapshot_and_flags_staleness(self) -> None: + server, _ = coordinator() + mark_progress_healthy(server) + server.refresh_health_snapshot() + server._ensure_observability_service().set_loop_running_for_test(True) + + status, payload = server.cached_health_payload() + self.assertEqual(status, 200) + self.assertIn("snapshot_age_seconds", payload) + + # Even if the ledger becomes unusable, the snapshot keeps serving. + server.ledger = None # type: ignore[assignment] + status, payload = server.cached_health_payload() + self.assertEqual(status, 200) + + server._ensure_observability_service().set_health_snapshot_monotonic_for_test( + time.monotonic() - 1_000 + ) + status, payload = server.cached_health_payload() + self.assertEqual(status, 503) + self.assertFalse(payload["ok"]) + + def test_accepted_share_stats_falls_back_to_all_shares(self) -> None: + server, _ = coordinator(ledger=SingleWriterShareLedger()) + self.assertEqual(server.accepted_share_stats(), (0, 0)) + + def test_single_writer_ledger_stats(self) -> None: + ledger = SingleWriterShareLedger() + self.assertEqual( + ledger.accepted_share_stats(), + {"accepted_share_count": 0, "distinct_miner_count": 0}, + ) + + def test_audit_server_marks_refresher_before_listener_dispatch(self) -> None: + """A probe retrying connection-refused queues in the accept backlog at + listen() time and can be served the instant the listener thread + starts. The refresher flag must already be set by then, or + cached_health_payload's legacy inline path would run the cold + accepted-share aggregate synchronously on the handler thread instead + of returning the documented starting state.""" + server, _ = coordinator() + server.audit_bind = "127.0.0.1" + server.audit_port = 0 + # The spawned refresher loops exit immediately; the running flag is + # what gates the handler path and must survive. + server.stop_event.set() + flag_at_bind: list[bool] = [] + + class RecordingServer: + def __init__(self, address: object, handler_cls: object) -> None: + flag_at_bind.append( + server._ensure_observability_service() + .state() + .health_refresh_loop_running + ) + self.server_address = address + self.ready = threading.Event() + self.released = threading.Event() + + def serve_unless_startup_cancelled(self, *, poll_interval: float) -> None: + del poll_interval + self.ready.set() + self.released.wait(2.0) + + def cancel_startup(self) -> bool: + return True + + def shutdown(self) -> None: + self.released.set() + + def server_close(self) -> None: + return None + + with patch.object( + audit_http_module, "_BoundedThreadingHttpServer", RecordingServer + ), patch("builtins.print"): + server.start_audit_server() + try: + self.assertEqual(flag_at_bind, [True]) + self.assertTrue( + server._ensure_observability_service() + .state() + .health_refresh_loop_running + ) + finally: + self.assertTrue(server._ensure_audit_http_facade().stop()) + + +class MetricsSnapshotTests(_JobSupportTestCase): + def test_cached_metrics_do_not_reenter_complete_renderer(self) -> None: + server, _ = coordinator() + complete = server.refresh_metrics_snapshot() + server._render_metrics_payload = mock.Mock( # type: ignore[method-assign] + side_effect=AssertionError("cached scrape reached live renderer") + ) + + status, cached = server.cached_metrics_payload() + + self.assertEqual(status, 200) + self.assertTrue(cached.startswith(complete)) + server._render_metrics_payload.assert_not_called() + self.assertIn("qbit_prism_metrics_snapshot_generation 1", cached) + + def test_renderer_emits_semantic_current_work_ratio_gauge(self) -> None: + # Upstream #107: the complete document carries the semantic-currency + # gauge derived from fingerprint, payout generation, and client + # session currency. + server, _ = coordinator() + + metrics = server._render_metrics_payload() + + self.assertIn( + "qbit_prism_stratum_semantic_current_work_ratio 1.0", + metrics, + ) + + +class JobBuildMetricsTests(_JobSupportTestCase): + def test_metrics_include_job_build_histogram_and_cache_counters(self) -> None: + server, _ = coordinator() + install_fake_bundle_builder(server) + server.build_job_for_client(client(1), clean_jobs=True) + server.build_job_for_client(client(2), clean_jobs=True) + server.observe_job_build_elapsed(0.3, {"bundle": 0.2, "stamp": 0.01}) + server._observe_tip_refresh_build_phase("payout_state_derivation", 0.25) + server._record_tip_refresh_ipc_bytes("input", 123) + + metrics = server.metrics_payload() + + self.assertIn('qbit_prism_job_build_seconds_bucket{le="0.5"} 1', metrics) + self.assertIn('qbit_prism_job_build_seconds_bucket{le="+Inf"} 1', metrics) + self.assertIn("qbit_prism_job_build_seconds_count 1", metrics) + self.assertIn('qbit_prism_job_cache_hits_total{cache="bundle"} 1', metrics) + self.assertIn('qbit_prism_job_cache_misses_total{cache="bundle"} 1', metrics) + self.assertIn('qbit_prism_job_build_phase_seconds_total{phase="bundle"} 0.2', metrics) + self.assertIn( + 'qbit_prism_tip_refresh_bundle_phase_seconds_count{phase="payout_state_derivation"} 1', + metrics, + ) + self.assertIn( + 'qbit_prism_tip_refresh_builder_ipc_bytes_total{direction="input"} 123', + metrics, + ) + self.assertIn("qbit_prism_tip_refresh_bundle_queue_depth 0", metrics) + self.assertIn("qbit_prism_tip_refresh_bundle_inflight 0", metrics) + self.assertIn("qbit_prism_connected_clients 0", metrics) + + def test_metrics_split_payout_preparation_publication_and_delivery(self) -> None: + server, _ = coordinator() + install_fake_bundle_builder(server) + state = client(1) + state.send = lambda _payload: None # type: ignore[method-assign] + + self.assertEqual(server._advance_payout_state_generation(), 1) + self.assertTrue(server.maybe_send_job(state, clean_jobs=True)) + + metrics = server.metrics_payload() + + self.assertIn("qbit_prism_payout_preparation_seconds_count 1", metrics) + self.assertIn("qbit_prism_payout_publish_seconds_count 1", metrics) + self.assertIn( + "qbit_prism_payout_invalidation_first_delivery_seconds_count 1", + metrics, + ) + self.assertIn( + 'qbit_prism_payout_gate_wait_seconds_count{generation="current"} 1', + metrics, + ) + self.assertIn("qbit_prism_payout_candidates_discarded_total 0", metrics) + + def test_poll_observes_reorg_reconcile_phase(self) -> None: + # The reconcile stage runs serially before every refresh build; one + # poll must record exactly one reorg_reconcile phase observation. + server, _ = coordinator() + install_fake_bundle_builder(server) + state = client(1) + state.send = lambda _payload: None # type: ignore[method-assign] + server.clients = {state} + + try: + refreshed = server.poll_qbit_tip_template_once() + finally: + server.shutdown_tip_refresh_executor() + + self.assertEqual(refreshed, 1) + metrics = server.metrics_payload() + self.assertIn( + 'qbit_prism_tip_refresh_bundle_phase_seconds_count{phase="reorg_reconcile"} 1', + metrics, + ) diff --git a/tests/test_prism_coordinator_shutdown.py b/tests/test_prism_coordinator_shutdown.py index 8b0d0eb4a..1a3aa5db0 100644 --- a/tests/test_prism_coordinator_shutdown.py +++ b/tests/test_prism_coordinator_shutdown.py @@ -12,12 +12,14 @@ from unittest.mock import patch from lab.prism import prism_coordinator -from lab.prism.prism_coordinator import ( +from lab.prism.coordinator_shutdown import ( CoordinatorShutdownController, + ShutdownInProgress, +) +from lab.prism.prism_coordinator import ( PendingShareAppend, PRISM_REJECTION_POOL_CLOSED, PrismCoordinator, - ShutdownInProgress, StratumError, WriterLeaseRenewalDeferred, ) @@ -176,7 +178,112 @@ def coordinator( return server +class CoordinatorShutdownControllerTests(unittest.TestCase): + def test_compatibility_reexports_reference_shutdown_owner(self) -> None: + self.assertIs( + prism_coordinator.CoordinatorShutdownController, + CoordinatorShutdownController, + ) + self.assertIs(prism_coordinator.ShutdownInProgress, ShutdownInProgress) + + def test_nested_writer_inherits_admission_after_shutdown_request(self) -> None: + controller = CoordinatorShutdownController(0.5) + outer = controller.enter_writer("outer") + controller.request_shutdown(signal.SIGTERM) + inner = controller.enter_writer("inner") + + controller.exit_writer(inner) + controller.exit_writer(outer) + + self.assertEqual(controller.snapshot()["active_writers"], {}) + with self.assertRaisesRegex(ShutdownInProgress, "coordinator is shutting down"): + controller.enter_writer("late") + + def test_transferable_writer_token_finishes_idempotently_on_another_thread( + self, + ) -> None: + controller = CoordinatorShutdownController(0.5) + token = controller.reserve_writer("share_persistence") + + finisher = threading.Thread(target=lambda: (token.finish(), token.finish())) + finisher.start() + finisher.join(1) + + self.assertFalse(finisher.is_alive()) + self.assertTrue(token.finished) + self.assertEqual(controller.snapshot()["active_writers"], {}) + + class PrismCoordinatorShutdownTests(unittest.TestCase): + def test_refresh_timeout_still_drains_build_executors(self) -> None: + server = coordinator() + calls: list[str] = [] + server.shutdown_initial_job_executor = ( # type: ignore[method-assign] + lambda: calls.append("initial") + ) + server.shutdown_job_build_executor = ( # type: ignore[method-assign] + lambda: calls.append("job_build") + ) + server.shutdown_payout_artifact_executor = ( # type: ignore[method-assign] + lambda: calls.append("payout_artifact") + ) + server.shutdown_reconcile_prefetch_executor = ( # type: ignore[method-assign] + lambda: calls.append("reconcile_prefetch") + ) + server.retire_share_window_spool = ( # type: ignore[method-assign] + lambda: calls.append("spool") + ) + server.shutdown_serve_builder = ( # type: ignore[method-assign] + lambda: calls.append("serve_builder") + ) + + server.shutdown_tip_refresh_executor() + + self.assertEqual( + calls, + [ + "initial", + "job_build", + "payout_artifact", + "reconcile_prefetch", + "spool", + "serve_builder", + ], + ) + + def test_startup_replay_shutdown_stops_cleanly_and_releases_lease_once( + self, + ) -> None: + ledger = RecordingLeaseLedger() + server = coordinator(ledger) + + def rejected_replay() -> int: + server.request_shutdown(signal.SIGTERM) + raise ShutdownInProgress("PRISM coordinator is shutting down") + + started = time.monotonic() + with patch("builtins.print"): + self.assertFalse( + server._run_startup_writer_replay( + rejected_replay, + drain_threads=[], + ) + ) + elapsed = time.monotonic() - started + + # No writer is active, so the replay exit never waits out the + # quiescence budget before releasing the lease exactly once. + self.assertLess(elapsed, 0.45) + self.assertEqual(ledger.release_calls, 1) + snapshot = server._ensure_shutdown_controller().snapshot() + self.assertEqual(snapshot["active_writers"], {}) + self.assertFalse(snapshot["lease_release_withheld"]) + self.assertEqual(snapshot["release_withheld_total"], 0) + self.assertEqual(snapshot["lease_release_outcomes"]["success"], 1) + with patch("builtins.print"): + server.shutdown(reason="main_finally") + self.assertEqual(ledger.release_calls, 1) + def test_lease_heartbeat_start_without_ledger_is_noop(self) -> None: server = PrismCoordinator.__new__(PrismCoordinator) @@ -732,6 +839,11 @@ def renew_writer_lease_heartbeat(self) -> dict[str, int | str]: return {"backend": "recording", "renewed_count": 1} server = coordinator(BlockingHeartbeatLedger()) + # A scaled-down but internally coherent policy: the interval stays + # below the failure budget and the budget plus the exit envelope + # stays inside the adoption silence, so the heartbeat starts and the + # stall — not a refused configuration — is what fires the exit. + server.ledger_lease_heartbeat_seconds = 0.005 server.ledger_lease_heartbeat_failure_seconds = 0.03 server.ledger_lease_heartbeat_monitor_seconds = 0.005 server.ledger_lease_heartbeat_exit_timeout_seconds = 0.01 @@ -887,7 +999,7 @@ def test_server_proven_cap_preserves_adoption_envelope_after_death( death + interval + budget — past a replacement's CAS eligibility when interval + budget reaches the adoption silence. The server-proven cap measures from completed round trips, which - cannot postdate the death, and must fire first (cap 0.265s here + cannot postdate the death, and must fire first (cap 0.26s here versus interval + budget = 0.30s). """ first_done = threading.Event() @@ -921,6 +1033,11 @@ def verify_writer_lease_guard_session( server.ledger_lease_heartbeat_failure_seconds = 0.25 server.ledger_lease_heartbeat_monitor_seconds = 0.005 server.ledger_lease_heartbeat_exit_timeout_seconds = 0.005 + # The scheduler slack is a term of the safety inequality, so a + # scaled-down policy has to scale it too: left at the production + # value it would not fit inside this 0.28s silence and the + # heartbeat would (correctly) refuse to start. + server.ledger_lease_heartbeat_scheduler_slack_seconds = 0.005 thread: threading.Thread | None = None try: @@ -962,7 +1079,17 @@ def verify_writer_lease_guard_session( if thread is not None: thread.join(0.5) - def test_heartbeat_warning_honors_private_silence_attribute(self) -> None: + def test_heartbeat_rejection_honors_private_silence_attribute(self) -> None: + """An operator silence override reaches the safety inequality. + + PsqlShareLedger keeps the adoption silence on a private attribute + with no public alias. The policy must read it: a 0.5s silence + cannot contain the default 1.25s failure budget plus the exit + envelope, so this is a real double-writer hazard, not a tuning + preference, and the heartbeat must refuse to start rather than + run with a broken exit-before-adoption argument. + """ + class ShortSilenceLedger(GuardVerifyLeaseLedger): _lease_adoption_silence_seconds = 0.5 @@ -970,40 +1097,106 @@ class ShortSilenceLedger(GuardVerifyLeaseLedger): server = coordinator(ledger) server.ledger_lease_heartbeat_seconds = 0.01 - with patch("builtins.print") as printed: - thread = server._start_ledger_lease_heartbeat() - self.assertIsNotNone(thread) - self.assertTrue(server._stop_ledger_lease_heartbeat()) - assert thread is not None - thread.join(0.2) - warnings = [ - call.args[0] - for call in printed.call_args_list - if call.args - and "heartbeat timing is misconfigured" in str(call.args[0]) - ] - # The default 0.75s budget leaves no envelope headroom under a - # 0.5s operator silence override stored on the ledger's private - # attribute; the warning must see it rather than the 1.0s default. - self.assertEqual(len(warnings), 1) + with patch.object( + server, + "_ledger_lease_heartbeat_hard_exit", + ) as hard_exit: + self.assertIsNone(server._start_ledger_lease_heartbeat()) - def test_heartbeat_timing_misconfiguration_warns_at_start(self) -> None: + hard_exit.assert_called_once() + message = str(hard_exit.call_args.args[0]) + self.assertIn("refusing to start the ledger lease heartbeat", message) + self.assertIn("adoption silence 0.5s", message) + # No thread may exist: refusing means refusing to run, not + # starting and then complaining. + self.assertIsNone(server._ledger_lease_heartbeat_thread) + self.assertEqual(ledger.verify_calls, 0) + + def test_heartbeat_unsafe_timing_is_refused_at_start(self) -> None: + """A failure budget that overruns the adoption silence is refused. + + The budget the monitor enforces has to fit inside the silence + window alongside the exit envelope, or a coordinator that lost its + guarded session can still be running when a replacement becomes + adoption-eligible. Before issue #212 this printed a warning and + started anyway. + """ ledger = GuardVerifyLeaseLedger() server = coordinator(ledger) server.ledger_lease_heartbeat_seconds = 0.01 - server.ledger_lease_heartbeat_failure_seconds = 1.5 + server.ledger_lease_heartbeat_failure_seconds = 2.5 - with patch("builtins.print") as printed: + with patch.object( + server, + "_ledger_lease_heartbeat_hard_exit", + ) as hard_exit: + self.assertIsNone(server._start_ledger_lease_heartbeat()) + + hard_exit.assert_called_once() + message = str(hard_exit.call_args.args[0]) + self.assertIn("refusing to start the ledger lease heartbeat", message) + self.assertIn("failure budget 2.5s", message) + self.assertIsNone(server._ledger_lease_heartbeat_thread) + + def test_heartbeat_interval_at_or_above_budget_is_refused(self) -> None: + """One idle wait must not be able to exhaust the liveness budget.""" + ledger = GuardVerifyLeaseLedger() + server = coordinator(ledger) + server.ledger_lease_heartbeat_seconds = 1.25 + server.ledger_lease_heartbeat_failure_seconds = 1.25 + + with patch.object( + server, + "_ledger_lease_heartbeat_hard_exit", + ) as hard_exit: + self.assertIsNone(server._start_ledger_lease_heartbeat()) + + hard_exit.assert_called_once() + self.assertIn( + "must exceed the heartbeat interval", + str(hard_exit.call_args.args[0]), + ) + + def test_heartbeat_warns_without_tail_latency_headroom(self) -> None: + """Safe but unstable timing warns instead of refusing. + + A silence window that still satisfies the exit-before-adoption + inequality but leaves the server-proven cap below the largest gap a + healthy coordinator can produce permits no double writer — it just + guarantees issue #212's false exits. Labs and tests deliberately + run tiny policies, so this stays a warning. + """ + + class TightSilenceLedger(GuardVerifyLeaseLedger): + _lease_adoption_silence_seconds = 0.5 + + ledger = TightSilenceLedger() + server = coordinator(ledger) + server.ledger_lease_heartbeat_seconds = 0.01 + server.ledger_lease_heartbeat_failure_seconds = 0.2 + server.ledger_lease_heartbeat_monitor_seconds = 0.005 + server.ledger_lease_heartbeat_exit_timeout_seconds = 0.01 + # Scaled with the rest of the policy so this stays the safe-but-tight + # case: the guard's own statement timeout is not scaled, so the + # staleness cap still lands below the healthy-gap bound and the + # advisory fires. + server.ledger_lease_heartbeat_scheduler_slack_seconds = 0.005 + + with patch("builtins.print") as printed, patch.object( + server, + "_ledger_lease_heartbeat_hard_exit", + ) as hard_exit: thread = server._start_ledger_lease_heartbeat() self.assertIsNotNone(thread) self.assertTrue(server._stop_ledger_lease_heartbeat()) assert thread is not None thread.join(0.2) + + hard_exit.assert_not_called() warnings = [ call.args[0] for call in printed.call_args_list - if call.args - and "heartbeat timing is misconfigured" in str(call.args[0]) + if call.args and "no tail-latency headroom" in str(call.args[0]) ] self.assertEqual(len(warnings), 1) @@ -1020,6 +1213,10 @@ def test_heartbeat_default_timing_does_not_warn(self) -> None: thread.join(0.2) for call in printed.call_args_list: if call.args: + self.assertNotIn( + "no tail-latency headroom", + str(call.args[0]), + ) self.assertNotIn( "heartbeat timing is misconfigured", str(call.args[0]), @@ -1445,6 +1642,50 @@ def blocked_executor_drain() -> None: drain_thread.join(1) self.assertFalse(drain_thread.is_alive()) + def test_audit_listener_stops_during_non_writer_drain_after_lease_release( + self, + ) -> None: + ledger = RecordingLeaseLedger() + server = coordinator(ledger) + stop_saw_release: list[bool] = [] + server._audit_http_facade = SimpleNamespace( + stop=lambda: stop_saw_release.append(ledger.released.is_set()) or True + ) + server._background_services = SimpleNamespace( + threads_to_drain=lambda: (), + ) + server.shutdown_vardiff_idle_executor = lambda: None # type: ignore[method-assign] + server.shutdown_tip_refresh_executor = lambda: None # type: ignore[method-assign] + + with patch("builtins.print"): + self.assertTrue(server.shutdown()) + server.drain_non_writer_components() + + self.assertEqual(stop_saw_release, [True]) + + def test_audit_listener_stop_timeout_is_logged_during_drain(self) -> None: + # The facade's bounded stop() reports False on a stop_timeout; the + # drain must surface that as an audit_http_stop shutdown log line + # instead of blocking or swallowing it. + ledger = RecordingLeaseLedger() + server = coordinator(ledger) + shutdown_events: list[tuple[str, dict[str, object]]] = [] + server._shutdown_log = ( # type: ignore[method-assign] + lambda event, **fields: shutdown_events.append((event, fields)) + ) + server._audit_http_facade = SimpleNamespace(stop=lambda: False) + server._background_services = SimpleNamespace( + threads_to_drain=lambda: (), + ) + server.shutdown_vardiff_idle_executor = lambda: None # type: ignore[method-assign] + server.shutdown_tip_refresh_executor = lambda: None # type: ignore[method-assign] + + with patch("builtins.print"): + self.assertTrue(server.shutdown()) + server.drain_non_writer_components() + + self.assertIn(("audit_http_stop", {"outcome": "timeout"}), shutdown_events) + def test_pending_share_batch_flushes_before_release(self) -> None: append_started = threading.Event() allow_flush = threading.Event() @@ -1553,6 +1794,9 @@ def producer() -> None: writer_thread.join(1) shutdown_thread.join(1) + self.assertFalse(producer_thread.is_alive(), "producer thread leaked") + self.assertFalse(writer_thread.is_alive(), "share writer thread leaked") + self.assertFalse(shutdown_thread.is_alive(), "shutdown thread leaked") self.assertTrue(entry.committed.is_set()) self.assertEqual(ledger.release_calls, 1) diff --git a/tests/test_prism_ctv_runtime.py b/tests/test_prism_ctv_runtime.py new file mode 100644 index 000000000..f298742d1 --- /dev/null +++ b/tests/test_prism_ctv_runtime.py @@ -0,0 +1,547 @@ +"""Direct tests for the coordinator-facing CTV runtime service.""" + +from __future__ import annotations + +from contextlib import contextmanager +import threading +import unittest +from unittest.mock import patch + +from lab.prism.coordinator_shutdown import ShutdownInProgress +from lab.prism.ctv_broadcaster_daemon import ( + CtvFanoutChunkResult, + CtvFanoutDaemonResult, +) +from lab.prism.ctv_runtime import ( + CTV_BROADCAST_STATE_COMPONENT, + CtvRuntimeConfig, + CtvRuntimeService, +) +from lab.prism.prism_coordinator import PrismCoordinator + + +class StopAfterOnePass: + def is_set(self) -> bool: + return False + + def wait(self, _timeout: float) -> bool: + return True + + +class PrismCtvRuntimeTests(unittest.TestCase): + @staticmethod + def config(**overrides: object) -> CtvRuntimeConfig: + values: dict[str, object] = { + "enabled": True, + "wallet": None, + "fee_sats": 0, + "limit": 7, + "chunk_size": 2, + "interval_seconds": 30.0, + } + values.update(overrides) + return CtvRuntimeConfig(**values) # type: ignore[arg-type] + + def test_run_once_constructs_daemon_and_holds_exact_writer_admission(self) -> None: + ledger = object() + admission: list[str] = [] + captured: dict[str, object] = {} + heartbeats: list[bool] = [] + + def rpc_call(*_args: object, **_kwargs: object) -> None: + return None + + class FakeBroadcaster: + def __init__(self, call: object, *, funding_wallet: str | None) -> None: + captured["rpc_call"] = call + captured["wallet"] = funding_wallet + + class FakeDaemon: + def __init__( + self, + daemon_ledger: object, + broadcaster: object, + *, + fee_sats: int, + ) -> None: + captured["ledger"] = daemon_ledger + captured["broadcaster"] = broadcaster + captured["fee_sats"] = fee_sats + + def run_once(self, **kwargs: object) -> CtvFanoutDaemonResult: + captured.update(kwargs) + progress_callback = kwargs["progress_callback"] + chunk_callback = kwargs["chunk_callback"] + tip_refresh_pending = kwargs["tip_refresh_pending"] + assert callable(progress_callback) + assert callable(chunk_callback) + assert callable(tip_refresh_pending) + progress_callback() + chunk_callback( + CtvFanoutChunkResult(processed_count=1, elapsed_seconds=0.25) + ) + captured["tip_pending"] = tip_refresh_pending() + return CtvFanoutDaemonResult(1, 0, 1, 0) + + @contextmanager + def writer_admission(component: str): + admission.append(f"enter:{component}") + try: + yield + finally: + admission.append(f"exit:{component}") + + service = CtvRuntimeService( + rpc_call=rpc_call, + ledger_source=lambda: ledger, + writer_admission=writer_admission, + tip_refresh_pending=lambda: False, + heartbeat=lambda: heartbeats.append(True), + stop_signal=threading.Event, + config=self.config(wallet="fee-wallet", fee_sats=900), + daemon_type=FakeDaemon, # type: ignore[arg-type] + broadcaster_type=FakeBroadcaster, # type: ignore[arg-type] + ) + + result = service.run_once(progress_callback=service.record_progress) + + self.assertEqual(result.updated_count, 1) + self.assertEqual( + admission, + [ + f"enter:{CTV_BROADCAST_STATE_COMPONENT}", + f"exit:{CTV_BROADCAST_STATE_COMPONENT}", + ], + ) + self.assertIs(captured["ledger"], ledger) + self.assertIs(captured["rpc_call"], rpc_call) + self.assertEqual(captured["wallet"], "fee-wallet") + self.assertEqual(captured["fee_sats"], 900) + self.assertEqual(captured["limit"], 7) + self.assertEqual(captured["chunk_size"], 2) + self.assertFalse(captured["tip_pending"]) + self.assertEqual(len(heartbeats), 2) + metrics = "\n".join(service.metrics_lines()) + self.assertIn( + "qbit_prism_ctv_fanout_broadcaster_processed_rows_total 1", + metrics, + ) + self.assertIn( + "qbit_prism_ctv_fanout_broadcaster_chunk_seconds_sum 0.250000", + metrics, + ) + + def test_closed_writer_admission_does_not_construct_daemon(self) -> None: + constructed: list[bool] = [] + + class RejectingAdmission: + def __enter__(self) -> None: + raise ShutdownInProgress("closed") + + def __exit__(self, *_args: object) -> None: + return None + + class UnexpectedDaemon: + def __init__(self, *_args: object, **_kwargs: object) -> None: + constructed.append(True) + + service = CtvRuntimeService( + rpc_call=lambda *_args, **_kwargs: None, + ledger_source=lambda: object(), + writer_admission=lambda _component: RejectingAdmission(), + tip_refresh_pending=lambda: False, + heartbeat=lambda: None, + stop_signal=threading.Event, + config=self.config(), + daemon_type=UnexpectedDaemon, # type: ignore[arg-type] + ) + + with self.assertRaises(ShutdownInProgress): + service.run_once() + + self.assertEqual(constructed, []) + + def test_loop_records_completion_before_wait_and_preserves_summary(self) -> None: + clock_values = iter((10.0, 112.0)) + heartbeats: list[float] = [] + wait_observation: dict[str, object] = {} + + class ObservedStop(StopAfterOnePass): + def wait(self, timeout: float) -> bool: + wait_observation["timeout"] = timeout + wait_observation["pass_count"] = service.pass_count + return True + + @contextmanager + def writer_admission(_component: str): + yield + + class OnePassDaemon: + def run_once(self, **_kwargs: object) -> CtvFanoutDaemonResult: + return CtvFanoutDaemonResult(0, 0, 0, 0, True) + + observed_stop = ObservedStop() + service = CtvRuntimeService( + rpc_call=lambda *_args, **_kwargs: None, + ledger_source=lambda: object(), + writer_admission=writer_admission, + tip_refresh_pending=lambda: False, + heartbeat=lambda: heartbeats.append(1.0), + stop_signal=lambda: observed_stop, + config=self.config(), + monotonic=lambda: next(clock_values), + ) + service.daemon = OnePassDaemon() # type: ignore[assignment] + + service.loop() + + self.assertEqual(wait_observation, {"timeout": 30.0, "pass_count": 1}) + self.assertEqual(len(heartbeats), 2) + metrics = "\n".join(service.metrics_lines()) + self.assertIn( + "qbit_prism_ctv_fanout_broadcaster_pass_seconds_sum 102.000000", + metrics, + ) + self.assertIn( + "qbit_prism_ctv_fanout_broadcaster_tip_refresh_yields_total 1", + metrics, + ) + self.assertEqual( + service.startup_summary(), + "prism coordinator: CTV fanout broadcaster enabled " + "mode=direct fee_bits=0 wallet=none interval=30s limit=7 chunk_size=2", + ) + + def test_cpfp_wallet_validation_remains_at_daemon_construction(self) -> None: + @contextmanager + def writer_admission(_component: str): + yield + + service = CtvRuntimeService( + rpc_call=lambda *_args, **_kwargs: None, + ledger_source=lambda: object(), + writer_admission=writer_admission, + tip_refresh_pending=lambda: False, + heartbeat=lambda: None, + stop_signal=threading.Event, + config=self.config(wallet=None, fee_sats=1), + ) + + with self.assertRaisesRegex( + ValueError, + "ctv_broadcaster_wallet is required", + ): + service.make_daemon() + + def test_lazy_handoff_consumes_overrides_and_updates_only_live_config(self) -> None: + server = PrismCoordinator.__new__(PrismCoordinator) + server.ctv_broadcaster_limit = 7 + + self.assertEqual( + server.__dict__["_ctv_runtime_compat_config"], + {"limit": 7}, + ) + + runtime = server._ensure_ctv_runtime() + + self.assertEqual(runtime.config.limit, 7) + self.assertNotIn("_ctv_runtime_compat_config", server.__dict__) + + server.ctv_broadcaster_limit = 11 + + self.assertIs(server._ensure_ctv_runtime(), runtime) + self.assertEqual(runtime.config.limit, 11) + self.assertNotIn("_ctv_runtime_compat_config", server.__dict__) + self.assertEqual(server._legacy_ctv_runtime_config().limit, 100) + + def test_wallet_or_fee_config_change_rebuilds_daemon_for_next_pass(self) -> None: + constructions: list[tuple[str | None, int]] = [] + runs: list[tuple[int, int]] = [] + + class FakeBroadcaster: + def __init__( + self, + _rpc_call: object, + *, + funding_wallet: str | None, + ) -> None: + self.wallet = funding_wallet + + class FakeDaemon: + def __init__( + self, + _ledger: object, + broadcaster: FakeBroadcaster, + *, + fee_sats: int, + ) -> None: + constructions.append((broadcaster.wallet, fee_sats)) + + def run_once(self, **kwargs: object) -> CtvFanoutDaemonResult: + runs.append((int(kwargs["limit"]), int(kwargs["chunk_size"]))) + return CtvFanoutDaemonResult(0, 0, 0, 0) + + @contextmanager + def writer_admission(_component: str): + yield + + service = CtvRuntimeService( + rpc_call=lambda *_args, **_kwargs: None, + ledger_source=lambda: object(), + writer_admission=writer_admission, + tip_refresh_pending=lambda: False, + heartbeat=lambda: None, + stop_signal=threading.Event, + config=self.config(), + daemon_type=FakeDaemon, # type: ignore[arg-type] + broadcaster_type=FakeBroadcaster, # type: ignore[arg-type] + ) + + service.run_once() + first = service.daemon + service.replace_config(limit=11, chunk_size=4) + service.run_once() + self.assertIs(service.daemon, first) + + service.replace_config(wallet="fee-wallet", fee_sats=900) + service.run_once() + + self.assertIsNot(service.daemon, first) + self.assertEqual(constructions, [(None, 0), ("fee-wallet", 900)]) + self.assertEqual(runs, [(7, 2), (11, 4), (11, 4)]) + + def test_concurrent_live_config_updates_retain_both_frozen_fields(self) -> None: + server = PrismCoordinator.__new__(PrismCoordinator) + runtime = server._ensure_ctv_runtime() + start = threading.Barrier(3) + errors: list[BaseException] = [] + + def update_limit() -> None: + try: + start.wait(timeout=2.0) + server.ctv_broadcaster_limit = 17 + except BaseException as exc: # pragma: no cover - asserted below + errors.append(exc) + + def update_chunk_size() -> None: + try: + start.wait(timeout=2.0) + server.ctv_broadcaster_chunk_size = 3 + except BaseException as exc: # pragma: no cover - asserted below + errors.append(exc) + + threads = [ + threading.Thread(target=update_limit), + threading.Thread(target=update_chunk_size), + ] + for thread in threads: + thread.start() + start.wait(timeout=2.0) + for thread in threads: + thread.join(timeout=2.0) + + self.assertTrue(all(not thread.is_alive() for thread in threads)) + self.assertEqual(errors, []) + snapshot = runtime.config + self.assertEqual(snapshot.limit, 17) + self.assertEqual(snapshot.chunk_size, 3) + + def test_lazy_handoff_is_singleton_under_deterministic_contention(self) -> None: + server = PrismCoordinator.__new__(PrismCoordinator) + server._record_heartbeat = lambda _name: None # type: ignore[method-assign] + arrivals = threading.Barrier(2) + underlying_lock = threading.Lock() + candidates: list[CtvRuntimeService] = [] + results: list[CtvRuntimeService] = [] + errors: list[BaseException] = [] + + class CoordinatedInitLock: + def __enter__(self) -> None: + arrivals.wait(timeout=2.0) + underlying_lock.acquire() + + def __exit__(self, *_args: object) -> None: + underlying_lock.release() + + server._ctv_runtime_init_lock = CoordinatedInitLock() # type: ignore[assignment] + original_make = server._make_ctv_runtime_service + + def counted_make( + config: CtvRuntimeConfig | None = None, + ) -> CtvRuntimeService: + candidate = original_make(config) + candidates.append(candidate) + return candidate + + server._make_ctv_runtime_service = counted_make # type: ignore[method-assign] + + def ensure_runtime() -> None: + try: + results.append(server._ensure_ctv_runtime()) + except BaseException as exc: # pragma: no cover - asserted below + errors.append(exc) + + threads = [threading.Thread(target=ensure_runtime) for _ in range(2)] + for thread in threads: + thread.start() + for thread in threads: + thread.join(timeout=2.0) + + self.assertTrue(all(not thread.is_alive() for thread in threads)) + self.assertEqual(errors, []) + self.assertEqual(len(candidates), 1) + self.assertEqual(len(results), 2) + self.assertIs(results[0], candidates[0]) + self.assertIs(results[1], candidates[0]) + marker = object() + results[0].daemon = marker # type: ignore[assignment] + self.assertIs(results[1].daemon, marker) + results[0].record_progress() + results[1].record_progress() + self.assertEqual(results[0].processed_rows_total, 2) + + def test_daemon_getter_waits_for_paused_lazy_handoff_publish(self) -> None: + server = PrismCoordinator.__new__(PrismCoordinator) + daemon_marker = object() + server.ctv_fanout_broadcast_daemon = daemon_marker # type: ignore[assignment] + underlying_lock = threading.Lock() + getter_attempted = threading.Event() + factory_popped_compat = threading.Event() + release_factory = threading.Event() + getter_returned = threading.Event() + runtime_results: list[CtvRuntimeService] = [] + daemon_results: list[object | None] = [] + errors: list[BaseException] = [] + + class ObservedInitLock: + def __enter__(self) -> None: + if threading.current_thread().name == "ctv-daemon-getter": + getter_attempted.set() + underlying_lock.acquire() + + def __exit__(self, *_args: object) -> None: + underlying_lock.release() + + server._ctv_runtime_init_lock = ObservedInitLock() # type: ignore[assignment] + original_make = server._make_ctv_runtime_service + + def paused_make( + config: CtvRuntimeConfig | None = None, + ) -> CtvRuntimeService: + runtime = original_make(config) + if "_ctv_runtime_compat_daemon" in server.__dict__: + raise AssertionError("compat daemon was not consumed") + factory_popped_compat.set() + if not release_factory.wait(timeout=2.0): + raise AssertionError("test did not release runtime publication") + return runtime + + server._make_ctv_runtime_service = paused_make # type: ignore[method-assign] + + def initialize() -> None: + try: + runtime_results.append(server._ensure_ctv_runtime()) + except BaseException as exc: # pragma: no cover - asserted below + errors.append(exc) + + def read_daemon() -> None: + try: + daemon_results.append(server.ctv_fanout_broadcast_daemon) + except BaseException as exc: # pragma: no cover - asserted below + errors.append(exc) + finally: + getter_returned.set() + + initializer = threading.Thread(target=initialize, name="ctv-runtime-initializer") + getter = threading.Thread(target=read_daemon, name="ctv-daemon-getter") + initializer.start() + self.assertTrue(factory_popped_compat.wait(timeout=2.0)) + getter.start() + self.assertTrue(getter_attempted.wait(timeout=2.0)) + self.assertFalse(getter_returned.is_set()) + + release_factory.set() + initializer.join(timeout=2.0) + getter.join(timeout=2.0) + + self.assertFalse(initializer.is_alive()) + self.assertFalse(getter.is_alive()) + self.assertEqual(errors, []) + self.assertEqual(len(runtime_results), 1) + self.assertEqual(daemon_results, [daemon_marker]) + self.assertIs(runtime_results[0].daemon, daemon_marker) + self.assertNotIn("_ctv_runtime_compat_daemon", server.__dict__) + + def test_coordinator_facade_routes_every_retained_loop_patch_seam(self) -> None: + server = PrismCoordinator.__new__(PrismCoordinator) + server.stop_event = StopAfterOnePass() # type: ignore[assignment] + server.ctv_broadcaster_limit = 7 + server.ctv_broadcaster_chunk_size = 2 + server.ctv_broadcaster_interval_seconds = 30.0 + server.tip_refresh_is_pending = lambda: False # type: ignore[method-assign] + server._record_heartbeat = lambda _name: None # type: ignore[method-assign] + events: list[object] = [] + + @contextmanager + def writer_admission(component: str): + events.append(("admit", component)) + yield + + server._writer_operation = writer_admission # type: ignore[method-assign] + + class FacadeDaemon: + def run_once(self, **kwargs: object) -> CtvFanoutDaemonResult: + progress_callback = kwargs["progress_callback"] + chunk_callback = kwargs["chunk_callback"] + assert callable(progress_callback) + assert callable(chunk_callback) + progress_callback() + chunk_callback( + CtvFanoutChunkResult(processed_count=2, elapsed_seconds=0.25) + ) + return CtvFanoutDaemonResult(2, 0, 2, 0, True) + + server.ctv_fanout_broadcast_daemon = FacadeDaemon() # type: ignore[assignment] + server._record_ctv_fanout_broadcaster_progress = ( # type: ignore[method-assign] + lambda: events.append("progress") + ) + server.observe_ctv_fanout_broadcaster_chunk = ( # type: ignore[method-assign] + lambda result: events.append(("chunk", result.processed_count)) + ) + server.observe_ctv_fanout_broadcaster_pass = ( # type: ignore[method-assign] + lambda elapsed: events.append(("pass", elapsed)) + ) + server._record_ctv_fanout_broadcaster_yield = ( # type: ignore[method-assign] + lambda: events.append("yield") + ) + original_run_once = server.run_ctv_fanout_broadcaster_once + + def patched_run_once(**kwargs: object) -> CtvFanoutDaemonResult: + events.append("run_once") + return original_run_once(**kwargs) # type: ignore[arg-type] + + server.run_ctv_fanout_broadcaster_once = patched_run_once # type: ignore[method-assign] + + with patch("builtins.print"): + server.ctv_fanout_broadcaster_loop() + + self.assertEqual( + [event if isinstance(event, str) else event[0] for event in events], + ["run_once", "admit", "progress", "chunk", "pass", "yield"], + ) + self.assertEqual(events[1], ("admit", CTV_BROADCAST_STATE_COMPONENT)) + # The registry spec is declared with a call-time-resolved target + # (post-construction loop monkeypatches keep working), so the spec + # contract is asserted on its lifecycle fields rather than a bound + # method identity. + registry = server._make_background_service_registry() + specification = registry.snapshot("ctv_fanout_broadcaster").specification + self.assertEqual(specification.name, "ctv_fanout_broadcaster") + self.assertEqual(specification.thread_name, "prism-ctv-fanout-broadcaster") + self.assertEqual(specification.join_timeout, 1.0) + self.assertTrue(specification.daemon) + self.assertTrue(specification.watchdog_monitored) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_prism_dequeue_stale_skip.py b/tests/test_prism_dequeue_stale_skip.py new file mode 100644 index 000000000..aec43903c --- /dev/null +++ b/tests/test_prism_dequeue_stale_skip.py @@ -0,0 +1,928 @@ +#!/usr/bin/env python3 +"""Regression coverage for the dequeue-time stale sibling skip (#181 item 2). + +The 2026-08-21 testnet4 validation showed that #196 removed the +restart/replay candidate population but left the live/fast-path one: the +coordinator log still carried bursts of individual +``block candidate abandoned reason=stale-job: tip moved before submit``. +Each of those rows costs one ``submitblock``, three ``getbestblockhash``, +three ``getblockheader`` and two ledger writes -- plus an accounting task, a +fast-lane reservation and an accepted-block payout-preview barrier armed and +withdrawn -- to learn a fact the chain could have answered before the offer. + +These tests pin what the skip does about that, against two references it may +never disagree with: #194's per-row oracle (``drain_per_row``, the shipped +disposition path) and #196's predicate S (whose clauses, evidence set, +fencing and cleanup the skip reuses rather than restates). + +The invariant every efficacy test asserts is one-directional: +``skip_set - oracle_abandoned`` must be empty. Conservative oracle-only +deltas are acceptable -- the skip is an optimisation over a path that still +runs -- while a single selector-only hash is a block the per-row path would +have kept. +""" + +from __future__ import annotations + +import queue +import sys +import unittest +from contextlib import redirect_stdout +from io import StringIO +from pathlib import Path +from typing import Any + +if not __package__: + sys.path.insert(0, str(Path(__file__).resolve().parents[1])) + +from lab.prism.block_candidates import ( # noqa: E402 + MAX_PENDING_BLOCK_CANDIDATES, + PRISM_BLOCK_CANDIDATE_COLLAPSE_OUTCOMES, + _BlockCandidateNodeSubmission, + _dequeued_candidate_collapse_row, + block_candidate_intent, +) +from tests.prism_candidate_storm import ( # noqa: E402 + PERTURBATIONS, + CandidateStormRig, +) +from tests.test_prism_block_candidate_collapse import ( # noqa: E402 + DECIDED_HEIGHT, + DECIDED_WINNER, + MIN_WORK_BITS, + STORM_PARENT, + CollapseChainRpc, + CollapseFixture, + CollapseLedger, + _hash, + _scaled, +) + + +class _NoBatchLedger(CollapseLedger): + """A compatibility ledger with no fenced batch abandonment at all.""" + + def __getattr__(self, name: str) -> Any: + if name == "mark_block_candidates_abandoned": + raise AttributeError(name) + return super().__getattr__(name) + + mark_block_candidates_abandoned = None # type: ignore[assignment] + + +class _NoPoolBlockLedger(CollapseLedger): + """A compatibility ledger that cannot answer clause 2 at all.""" + + def __getattr__(self, name: str) -> Any: + if name == "pool_block_state": + raise AttributeError(name) + return super().__getattr__(name) + + pool_block_state = None # type: ignore[assignment] + + +def _offerable(fixture: CollapseFixture) -> CollapseFixture: + """Let the fixture's chain answer ``submitblock`` instead of raising. + + ``FakeRpc`` refuses every unknown method, which would turn a preserved + candidate's node offer into an error submission. These tests care + whether the offer *happened*, so the chain answers the way qbitd does + for a valid block that did not become the tip. + """ + fixture.rpc.results["submitblock"] = "inconclusive" + return fixture + + +def _drive(fixture: CollapseFixture, candidate: Any) -> str: + """Admit one candidate and run exactly one shipped ``submit_next`` over it. + + ``defer_accounting=True`` is the split ``BlockCandidateService.run`` + uses. No accounting lane is drained here, so a candidate the skip + preserved is left with its offer made and its durable row still pending + -- which is precisely the observable difference from one it skipped. + """ + buffer = StringIO() + with redirect_stdout(buffer): + fixture.server.enqueue_block_candidate(candidate) + fixture.server.submit_next_block_candidate(defer_accounting=True) + return buffer.getvalue() + + +def _record_skips(server: Any) -> set[str]: + """Capture the hashes the *skip itself* terminalized, and only those. + + ``PerRowDrainReport.abandoned_hashes`` names every durable transition a + drain made, which under ``dequeue_skip=True`` is the union of the skip's + own set and whatever the per-row path went on to abandon for candidates + the skip preserved. The contract's invariant is about the skip's set, so + it has to be read where the decision is made. + """ + service = server._ensure_block_candidate_service() + shipped = service._skip_superseded_block_candidate_at_dequeue + seen: set[str] = set() + + def recording(candidate: Any, **kwargs: Any) -> bool: + skipped = shipped(candidate, **kwargs) + if skipped: + seen.add(str(candidate.submission.block_hash_hex).lower()) + return skipped + + service._skip_superseded_block_candidate_at_dequeue = recording + return seen + + +def _accounting_depth(fixture: CollapseFixture) -> int: + service = fixture.service + service._ensure_block_accounting_state() + return ( + service._block_accounting_queue.qsize() + + service._block_accounting_overflow_queue.qsize() + + service._block_accounting_accepted_queue.qsize() + ) + + +def _skipped(fixture: CollapseFixture, candidate: Any) -> bool: + """Drive one candidate and report whether the skip terminalized it.""" + before = fixture.pending() + _drive(fixture, candidate) + block_hash = str(candidate.submission.block_hash_hex).lower() + gone = block_hash in before and block_hash not in fixture.pending() + skipped = fixture.counts()["dequeue_skipped"] > 0 + assert gone == skipped, "durable state and the skip counter disagree" + return skipped + + +class DequeueSkipRowShapeTests(unittest.TestCase): + """The skip judges an in-memory candidate; it must present the durable row.""" + + def test_the_dequeue_row_matches_the_durable_intent_field_by_field( + self, + ) -> None: + """Predicate S must read the same facts whichever side built the row. + + ``_dequeued_candidate_collapse_row`` restates a subset of + ``block_candidate_intent`` rather than calling it, because the full + intent re-serializes the block and recomputes every witness merkle + leaf. This is what keeps the restatement honest. + """ + fixture = CollapseFixture() + candidate = fixture.seed([_hash(1)], network_difficulty=7)[0] + intent = block_candidate_intent(candidate) + row = _dequeued_candidate_collapse_row(candidate, pool_block_exists=False) + self.assertEqual(row["block_hash"], intent["block_hash_hex"]) + built = row["candidate"] + for field in ("block_hash_hex", "parent_hash", "expected_height"): + self.assertEqual(built[field], intent[field], field) + for field in ("previousblockhash", "height"): + self.assertEqual( + built["template"][field], + intent["template"][field], + field, + ) + self.assertEqual(built["found_block"], intent["found_block"]) + self.assertEqual( + built["pending_share"]["job_id"], + intent["pending_share"]["job_id"], + ) + self.assertIs(row["pool_block_exists"], False) + self.assertIs( + _dequeued_candidate_collapse_row(candidate, pool_block_exists=True)[ + "pool_block_exists" + ], + True, + ) + + def test_the_outcome_label_set_stays_closed(self) -> None: + fixture = CollapseFixture() + self.assertEqual( + tuple(fixture.counts()), + PRISM_BLOCK_CANDIDATE_COLLAPSE_OUTCOMES, + ) + for outcome in ("dequeue_considered", "dequeue_skipped", "dequeue_preserved"): + self.assertIn(outcome, PRISM_BLOCK_CANDIDATE_COLLAPSE_OUTCOMES) + + +class DequeueSkipEfficacyTests(unittest.TestCase): + """The skip against #194's per-row oracle, live and after restart.""" + + def _rig(self, candidates: int, view: str) -> tuple[CandidateStormRig, Any, str]: + rig = CandidateStormRig(candidates=candidates) + rig.seed_live() + if view == "restart": + rig.restart_and_enumerate() + server = rig.restarted_server if view == "restart" else rig.live_server + winner = rig.decide_height() + return rig, server, winner + + def _oracle(self, candidates: int, view: str) -> tuple[set[str], Any]: + rig, server, winner = self._rig(candidates, view) + # drain_per_row suppresses both shipped set-oriented dispositions by + # default, so this stays the per-row path with no patching here. + report = rig.drain_per_row(server, stop_before_hash=winner) + return set(report.abandoned_hashes), report + + def _skip(self, candidates: int, view: str) -> tuple[set[str], Any, Any, str]: + rig, server, winner = self._rig(candidates, view) + skip_set = _record_skips(server) + report = rig.drain_per_row( + server, + stop_before_hash=winner, + dequeue_skip=True, + ) + return skip_set, report, server, winner + + def _compare(self, candidates: int, view: str) -> tuple[set[str], Any, Any]: + oracle, oracle_report = self._oracle(candidates, view) + skipped, skip_report, _server, winner = self._skip(candidates, view) + self.assertEqual( + skipped - oracle, + set(), + f"the skip abandoned {len(skipped - oracle)} hash(es) the shipped " + f"per-row path refused ({candidates} candidates, {view} view)", + ) + # And the drain as a whole -- skip plus whatever per-row disposed of + # afterwards -- terminalized nothing the oracle kept either. + self.assertEqual(set(skip_report.abandoned_hashes) - oracle, set()) + self.assertNotIn(winner, skipped) + return skipped, skip_report, oracle_report + + def test_every_stale_sibling_is_skipped_and_the_winner_is_not(self) -> None: + for view in ("live", "restart"): + with self.subTest(view=view): + skipped, _report, _oracle = self._compare(100, view) + self.assertEqual(len(skipped), 99) + + def test_agreement_holds_at_the_observed_storm_cardinality(self) -> None: + for view in ("live", "restart"): + with self.subTest(view=view): + skipped, _report, _oracle = self._compare(3_120, view) + self.assertEqual(len(skipped), 3_119) + + def test_the_skip_costs_no_submitblock_and_fewer_chain_reads(self) -> None: + """The cost claim, measured against the oracle on the same population.""" + for view in ("live", "restart"): + with self.subTest(view=view): + oracle, oracle_report = self._oracle(100, view) + skipped, skip_report, _server, _winner = self._skip(100, view) + self.assertEqual(len(oracle), 99) + self.assertEqual(len(skipped), 99) + self.assertEqual(set(skip_report.abandoned_hashes), skipped) + # The dominant per-sibling cost is the node offer itself. + self.assertEqual(oracle_report.rpc_calls["submitblock"], 99) + self.assertNotIn("submitblock", skip_report.rpc_calls) + oracle_rpcs = sum(oracle_report.rpc_calls.values()) + skip_rpcs = sum(skip_report.rpc_calls.values()) + self.assertEqual(oracle_rpcs, 693) + self.assertLess(skip_rpcs, oracle_rpcs) + # Four reads per skipped sibling: the clause-3 tip, the + # selector's tip height, and the pre-write revalidation's + # own fresh tip and occupant. The decided height and its + # header are read once for the whole burst, because the + # burst shares one probe budget for as long as the best tip + # holds. + self.assertEqual(skip_rpcs, 4 * 99 + 2) + self.assertEqual(skip_report.rpc_calls["getblockheader"], 1) + # Two durable writes per sibling become one: the attempt + # mark the offer stamps is never made at all. + self.assertEqual(oracle_report.ledger_attempt_marks, 99) + self.assertEqual(skip_report.ledger_attempt_marks, 0) + # And no stale sibling reaches per-row accounting. + self.assertEqual(oracle_report.accounting_tasks, 99) + self.assertEqual(skip_report.accounting_tasks, 0) + + def test_a_replay_adopted_row_is_classified_from_the_chain(self) -> None: + """The restart-specific fact: replay adoption marks nothing outstanding. + + A skip that consulted only the observation set would get both halves + of this wrong -- it would have no evidence that the winner is the + winner, and none that a sibling is superseded. + """ + rig = CandidateStormRig(candidates=8) + rig.seed_live() + rig.restart_and_enumerate() + server = rig.restarted_server + winner = rig.decide_height() + service = server._ensure_block_candidate_service() + with server.lock: + self.assertEqual( + set(service._outstanding_block_candidate_hashes), + set(), + ) + self.assertEqual( + set(service._tip_observed_accepted_block_hashes), + set(), + ) + self.assertEqual( + len(service._block_replay_inflight_hashes), + 8, + ) + report = rig.drain_per_row( + server, + stop_before_hash=winner, + dequeue_skip=True, + ) + self.assertEqual(len(report.abandoned_hashes), 7) + self.assertNotIn(winner, report.abandoned_hashes) + # Now drive the winner's own dequeue. Nothing in memory says it was + # accepted; only the chain does, through clause 4's "the occupant is + # this candidate". + offers_before = int(rig.decided_rpc.calls.get("submitblock", 0)) + with redirect_stdout(StringIO()): + server.submit_next_block_candidate(defer_accounting=True) + self.assertEqual( + int(rig.decided_rpc.calls.get("submitblock", 0)) - offers_before, + 1, + ) + self.assertIn( + winner, + { + str(row["block_hash"]).lower() + for row in rig.ledger.pending_block_candidate_rows(limit=64) + }, + ) + counts = service.block_candidate_collapse_snapshot() + self.assertEqual(counts["dequeue_skipped"], 7) + self.assertEqual(counts["dequeue_preserved"], 1) + + +class DequeueSkipEvidenceTests(unittest.TestCase): + """Every member of #196's evidence set still rejects at dequeue.""" + + def test_the_baseline_sibling_is_skipped(self) -> None: + fixture = _offerable(CollapseFixture()) + candidate = fixture.seed([_hash(1)])[0] + self.assertTrue(_skipped(fixture, candidate)) + self.assertEqual(_accounting_depth(fixture), 0) + self.assertNotIn("submitblock", fixture.rpc.counts()) + + def test_a_candidate_building_on_the_best_tip_is_never_skipped(self) -> None: + """Clause 3, read from the chain: this is the block waiting to land.""" + fixture = _offerable( + CollapseFixture( + rpc=CollapseChainRpc( + tip=STORM_PARENT, + tip_height=DECIDED_HEIGHT - 1, + active={DECIDED_HEIGHT - 1: STORM_PARENT}, + ) + ) + ) + candidate = fixture.seed([_hash(1)])[0] + self.assertFalse(_skipped(fixture, candidate)) + self.assertEqual(fixture.counts()["dequeue_preserved"], 1) + # The acceptance path pays exactly one extra chain read for the + # clause-3 answer, and never a durable one. + self.assertEqual(fixture.rpc.counts()["getbestblockhash"], 1) + self.assertNotIn("getblockcount", fixture.rpc.counts()) + self.assertEqual(fixture.rpc.counts()["submitblock"], 1) + + def test_ownership_markers_alone_do_not_preserve_a_stale_sibling(self) -> None: + """Outstanding/replay-inflight mean owned, not offered (#183's note).""" + fixture = _offerable(CollapseFixture()) + candidate = fixture.seed([_hash(1)])[0] + fixture.server._register_outstanding_block_candidate(_hash(1)) + with fixture.server.lock: + fixture.service._block_replay_inflight_hashes.add(_hash(1)) + self.assertTrue(_skipped(fixture, candidate)) + + def test_each_evidence_source_preserves_its_hash(self) -> None: + target = _hash(1) + + def deferred_accounting_retry(fixture: CollapseFixture) -> None: + duplicate = fixture.seed([target])[0] + with fixture.server.lock: + fixture.service._block_accounting_deferred_retry_candidate = ( + duplicate + ) + + def waiting_retry(fixture: CollapseFixture) -> None: + duplicate = fixture.seed([target])[0] + with fixture.server.lock: + fixture.service._block_disposition_waiting_retries[target] = ( + duplicate + ) + + def retained_submission(fixture: CollapseFixture) -> None: + fixture.service._stash_retained_block_candidate_node_submission( + target, + _BlockCandidateNodeSubmission(attempted=True, result=None), + ) + + def tip_observed(fixture: CollapseFixture) -> None: + fixture.server._register_outstanding_block_candidate(target) + with redirect_stdout(StringIO()): + fixture.server._note_tip_observation_for_candidates(target) + + def accounted_accepted(fixture: CollapseFixture) -> None: + fixture.server._ensure_job_cache_state() + with fixture.server.lock: + fixture.server._accounted_accepted_block_hashes.add(target) + + def prepared_pool_block(fixture: CollapseFixture) -> None: + fixture.ledger.inner.persist_accepted_block( + block_hash=target, + block_height=DECIDED_HEIGHT, + parent_hash=STORM_PARENT, + final_bundle={}, + audit_report={}, + ) + + for name, install in ( + ("deferred_accounting_retry", deferred_accounting_retry), + ("waiting_retry", waiting_retry), + ("retained_submission", retained_submission), + ("tip_observed", tip_observed), + ("accounted_accepted", accounted_accepted), + ("prepared_pool_block", prepared_pool_block), + ): + with self.subTest(evidence=name): + fixture = _offerable(CollapseFixture()) + candidate = fixture.seed([target])[0] + install(fixture) + self.assertFalse(_skipped(fixture, candidate)) + self.assertIn(target, fixture.pending()) + self.assertEqual(fixture.counts()["dequeue_skipped"], 0) + if name != "prepared_pool_block": + # In-memory evidence is probed before any round trip, so + # an excluded candidate leaves the skip having spent + # neither a chain read nor the durable pool-block probe. + # ``prepared_pool_block`` is the one clause that is a + # durable fact, so it is answered after the tip read. + self.assertNotIn( + "getbestblockhash", + fixture.rpc.counts(), + ) + else: + self.assertEqual( + fixture.rpc.counts()["getbestblockhash"], + 1, + ) + + def test_a_recorded_terminal_outcome_never_reaches_the_skip(self) -> None: + """The duplicate-drop fence runs first, and abandons nothing.""" + fixture = _offerable(CollapseFixture()) + candidate = fixture.seed([_hash(1)])[0] + fixture.service._record_block_candidate_terminal_outcome( + _hash(1), + accepted=False, + ) + _drive(fixture, candidate) + self.assertEqual(fixture.counts()["dequeue_considered"], 0) + self.assertIn(_hash(1), fixture.pending()) + + def test_a_finalize_retry_never_reaches_the_skip(self) -> None: + """A finalize-only replay is a different branch of ``submit_next``.""" + fixture = _offerable(CollapseFixture()) + candidate = fixture.seed([_hash(1)])[0] + with fixture.server.lock: + fixture.service.finalize_retries[_hash(1)] = (False, "boom") + _drive(fixture, candidate) + self.assertEqual(fixture.counts()["dequeue_considered"], 0) + self.assertEqual(fixture.counts()["dequeue_skipped"], 0) + + def test_a_lease_held_elsewhere_parks_the_wakeup_before_the_skip(self) -> None: + """``submit_next`` claims the lease first; an unclaimable one defers.""" + fixture = _offerable(CollapseFixture()) + candidate = fixture.seed([_hash(1)])[0] + lease = fixture.server._claim_block_candidate_disposition( + _hash(1), + blocking=False, + ) + self.assertIsNotNone(lease) + self.addCleanup( + fixture.server._release_block_candidate_disposition, + lease, + ) + _drive(fixture, candidate) + self.assertEqual(fixture.counts()["dequeue_considered"], 0) + self.assertIn(_hash(1), fixture.pending()) + self.assertIn( + _hash(1), + fixture.service._block_disposition_waiting_retries, + ) + + def test_the_passs_own_lease_is_the_only_flight_exempted(self) -> None: + """The self-exemption is what makes the skip reachable at all. + + ``submit_next`` holds this hash's disposition lease by the time the + skip runs, so #196's clause 1 would reject the row forever without + it -- and any exemption wider than this one hash would let a + genuinely mid-offer sibling through. + """ + fixture = _offerable(CollapseFixture()) + target, other = _hash(1), _hash(2) + first = fixture.seed([target])[0] + second = fixture.seed([other])[0] + held = fixture.server._claim_block_candidate_disposition( + other, + blocking=False, + ) + self.assertIsNotNone(held) + self.addCleanup( + fixture.server._release_block_candidate_disposition, + held, + ) + # The pass's own flight does not count; the sibling's does. + self.assertTrue(_skipped(fixture, first)) + self.assertNotIn(target, fixture.pending()) + _drive(fixture, second) + self.assertIn(other, fixture.pending()) + + +class DequeueSkipClauseTests(unittest.TestCase): + """The chain clauses, at the dequeue boundary.""" + + def test_a_lighter_occupant_never_destroys_a_heavier_sibling(self) -> None: + """Clause 4b, and the per-row path re-decides the height instead. + + Under testnet4's 20-minute minimum-difficulty rule a light foreign + block can occupy a height our full-difficulty sibling outweighs. + Skipping it pre-offer would destroy a block the per-row path wins. + """ + fixture = _offerable( + CollapseFixture( + rpc=CollapseChainRpc( + tip=DECIDED_WINNER, + tip_height=DECIDED_HEIGHT, + active={DECIDED_HEIGHT: DECIDED_WINNER}, + bits={DECIDED_WINNER: MIN_WORK_BITS}, + ) + ) + ) + occupant = _scaled(MIN_WORK_BITS) + heavier = fixture.seed([_hash(1)], network_difficulty=occupant * 8)[0] + self.assertFalse(_skipped(fixture, heavier)) + self.assertIn(_hash(1), fixture.pending()) + # Still offered, and still handed to per-row accounting to decide. + self.assertEqual(fixture.rpc.counts()["submitblock"], 1) + self.assertEqual(_accounting_depth(fixture), 1) + + def test_an_equal_work_occupant_still_supersedes(self) -> None: + fixture = _offerable( + CollapseFixture( + rpc=CollapseChainRpc( + tip=DECIDED_WINNER, + tip_height=DECIDED_HEIGHT, + active={DECIDED_HEIGHT: DECIDED_WINNER}, + bits={DECIDED_WINNER: MIN_WORK_BITS}, + ) + ) + ) + occupant = _scaled(MIN_WORK_BITS) + sibling = fixture.seed([_hash(2)], network_difficulty=occupant)[0] + self.assertTrue(_skipped(fixture, sibling)) + + def test_an_undecided_height_is_never_skipped(self) -> None: + """Clause 4: a height above the tip is still ours to win.""" + fixture = _offerable( + CollapseFixture( + rpc=CollapseChainRpc( + tip=DECIDED_WINNER, + tip_height=DECIDED_HEIGHT - 1, + active={DECIDED_HEIGHT - 1: DECIDED_WINNER}, + ) + ) + ) + candidate = fixture.seed([_hash(1)])[0] + self.assertFalse(_skipped(fixture, candidate)) + self.assertEqual(fixture.rpc.counts()["submitblock"], 1) + + def test_a_candidate_that_is_itself_the_occupant_is_never_skipped(self) -> None: + fixture = _offerable( + CollapseFixture( + rpc=CollapseChainRpc( + tip=_hash(1), + tip_height=DECIDED_HEIGHT, + active={DECIDED_HEIGHT: _hash(1)}, + ) + ) + ) + candidate = fixture.seed([_hash(1)])[0] + self.assertFalse(_skipped(fixture, candidate)) + + +class DequeueSkipFailClosedTests(unittest.TestCase): + """Every unreadable fact falls through to the offer path.""" + + def _preserved_on(self, method: str) -> None: + fixture = _offerable(CollapseFixture()) + candidate = fixture.seed([_hash(1)])[0] + fixture.rpc.failures.add(method) + self.assertFalse(_skipped(fixture, candidate)) + self.assertIn(_hash(1), fixture.pending()) + self.assertEqual(fixture.counts()["dequeue_preserved"], 1) + self.assertGreaterEqual(fixture.counts()["fail_closed"], 1) + self.assertEqual(fixture.rpc.counts()["submitblock"], 1) + + def test_a_failed_best_tip_read_preserves_the_candidate(self) -> None: + self._preserved_on("getbestblockhash") + + def test_a_failed_active_block_read_preserves_the_candidate(self) -> None: + self._preserved_on("getblockhash") + + def test_a_failed_header_read_preserves_the_candidate(self) -> None: + self._preserved_on("getblockheader") + + def test_a_failed_tip_height_read_preserves_the_candidate(self) -> None: + self._preserved_on("getblockcount") + + def test_a_failed_pool_block_probe_preserves_the_candidate(self) -> None: + fixture = _offerable(CollapseFixture()) + candidate = fixture.seed([_hash(1)])[0] + + def explode(**_kwargs: object) -> object: + raise RuntimeError("pool block state unavailable") + + fixture.ledger.inner.pool_block_state = explode + self.assertFalse(_skipped(fixture, candidate)) + self.assertEqual(fixture.counts()["dequeue_preserved"], 1) + self.assertGreaterEqual(fixture.counts()["fail_closed"], 1) + self.assertEqual(fixture.rpc.counts()["submitblock"], 1) + + def test_a_ledger_without_a_pool_block_probe_never_skips(self) -> None: + """Clause 2 has no durable answer, so nothing may be concluded.""" + fixture = _offerable(CollapseFixture(ledger=_NoPoolBlockLedger())) + candidate = fixture.seed([_hash(1)])[0] + self.assertFalse(_skipped(fixture, candidate)) + self.assertEqual(fixture.counts()["dequeue_considered"], 0) + self.assertEqual(fixture.rpc.counts()["submitblock"], 1) + + def test_a_ledger_without_a_fenced_batch_write_never_skips(self) -> None: + fixture = _offerable(CollapseFixture(ledger=_NoBatchLedger())) + candidate = fixture.seed([_hash(1)])[0] + self.assertFalse(_skipped(fixture, candidate)) + self.assertEqual(fixture.counts()["dequeue_considered"], 0) + # Not even a chain read is spent on a structural absence. + self.assertNotIn("getbestblockhash", fixture.rpc.counts()) + self.assertEqual(fixture.rpc.counts()["submitblock"], 1) + + def test_a_lost_fenced_write_preserves_the_candidate(self) -> None: + """The write is the authority: a row it did not return is not ours.""" + fixture = _offerable(CollapseFixture()) + candidate = fixture.seed([_hash(1)])[0] + fixture.ledger.abandon_hook = lambda _hashes, _error: () + self.assertFalse(_skipped(fixture, candidate)) + self.assertIn(_hash(1), fixture.pending()) + self.assertEqual(fixture.counts()["write_lost"], 1) + self.assertEqual(fixture.counts()["dequeue_preserved"], 1) + self.assertEqual(fixture.rpc.counts()["submitblock"], 1) + + def test_a_failed_fenced_write_preserves_the_candidate(self) -> None: + def explode(_hashes: object, _error: object) -> object: + raise RuntimeError("fenced write unavailable") + + fixture = _offerable(CollapseFixture()) + candidate = fixture.seed([_hash(1)])[0] + fixture.ledger.abandon_hook = explode + self.assertFalse(_skipped(fixture, candidate)) + self.assertIn(_hash(1), fixture.pending()) + self.assertEqual(fixture.rpc.counts()["submitblock"], 1) + + def test_a_tip_that_moved_under_the_lease_drops_the_write(self) -> None: + """The pre-write revalidation runs at dequeue exactly as it does on a page.""" + fixture = _offerable(CollapseFixture()) + candidate = fixture.seed([_hash(1)])[0] + seen: list[str] = [] + + def move_tip(rpc: Any, method: str, _params: object) -> None: + if method != "getbestblockhash": + return + seen.append(method) + if len(seen) == 2: + # Between selection and the fenced write, the candidate's + # own parent became the tip again. + rpc.tip = STORM_PARENT + + fixture.rpc.before_call = move_tip + self.assertFalse(_skipped(fixture, candidate)) + self.assertIn(_hash(1), fixture.pending()) + self.assertEqual(fixture.counts()["revalidation_dropped"], 1) + self.assertEqual(fixture.rpc.counts()["submitblock"], 1) + + +class DequeueSkipGuardTests(unittest.TestCase): + """An escape from the skip must not strand the lease or the object.""" + + def test_a_raising_skip_retains_the_candidate_and_frees_the_lease( + self, + ) -> None: + fixture = _offerable(CollapseFixture()) + candidate = fixture.seed([_hash(1)], credit_share_on_accept=True)[0] + + def explode(_candidate: Any, **_kwargs: Any) -> bool: + raise RuntimeError("skip exploded") + + fixture.service._skip_superseded_block_candidate_at_dequeue = explode + with self.assertRaises(RuntimeError): + with redirect_stdout(StringIO()): + fixture.server.enqueue_block_candidate(candidate) + fixture.server.submit_next_block_candidate(defer_accounting=True) + # The lease is free again, so the next pass for this hash is not + # parked forever... + lease = fixture.server._claim_block_candidate_disposition( + _hash(1), + blocking=False, + ) + self.assertIsNotNone(lease) + fixture.server._release_block_candidate_disposition(lease) + # ...and the object that owns the floor holder is still reachable. + with fixture.server.lock: + self.assertIs(fixture.service.retry_candidate, candidate) + self.assertIn(_hash(1), fixture.pending()) + + +class DequeueSkipCleanupTests(unittest.TestCase): + """A skipped candidate is terminalized with #196's full cleanup.""" + + def test_the_dequeued_objects_floor_holder_is_released(self) -> None: + """Asserted directly: the holder is keyed by the object's identity. + + The apply's own pending-share step indexes holders by scanning the + live and replay queues, and a dequeued object is in neither, so the + skip has to release this one itself. + """ + fixture = CollapseFixture() + candidate = fixture.seed([_hash(1)], credit_share_on_accept=True)[0] + writer = fixture.server._ensure_share_writer_service() + writer.adopt_pending_share(candidate.pending_share) + self.assertIn(id(candidate.pending_share), fixture.floor()) + self.assertTrue(_skipped(fixture, candidate)) + self.assertNotIn(id(candidate.pending_share), fixture.floor()) + + def test_every_skipped_candidate_in_a_burst_releases_its_floor(self) -> None: + fixture = CollapseFixture() + hashes = [_hash(index) for index in range(1, 6)] + candidates = fixture.seed(hashes, credit_share_on_accept=True) + writer = fixture.server._ensure_share_writer_service() + for candidate in candidates: + writer.adopt_pending_share(candidate.pending_share) + self.assertEqual(len(fixture.floor()), len(candidates)) + for candidate in candidates: + _drive(fixture, candidate) + self.assertEqual(fixture.counts()["dequeue_skipped"], len(candidates)) + self.assertEqual(fixture.floor(), {}) + self.assertEqual(fixture.pending(), set()) + + def test_the_preview_barrier_is_withdrawn_and_left_untombstoned(self) -> None: + fixture = CollapseFixture() + candidate = fixture.seed([_hash(1)])[0] + server = fixture.server + server._register_outstanding_block_candidate(_hash(1)) + server._begin_accepted_block_payout_preview( + _hash(1), + block_height=DECIDED_HEIGHT, + ) + server._mark_accepted_block_payout_landed( + _hash(1), + block_height=DECIDED_HEIGHT, + ) + with server._accepted_block_payout_preview_condition: + self.assertIn(_hash(1), server._accepted_block_payout_previews) + self.assertTrue(_skipped(fixture, candidate)) + with server._accepted_block_payout_preview_condition: + # Withdrawn *and* untombstoned: the durable row is terminal, so + # nothing is left to fence, and a retained tombstone is exactly + # the preview wait this work exists to end. + self.assertNotIn(_hash(1), server._accepted_block_payout_previews) + self.assertEqual( + server._invalidated_accepted_block_payout_previews, + {}, + ) + + def test_the_skip_publishes_the_terminal_fence_and_the_counters(self) -> None: + fixture = CollapseFixture() + candidate = fixture.seed([_hash(1)])[0] + self.assertTrue(_skipped(fixture, candidate)) + service = fixture.service + self.assertIs(service._block_candidate_terminal_outcome(_hash(1)), False) + with fixture.server.lock: + self.assertNotIn( + _hash(1), + service._outstanding_block_candidate_hashes, + ) + self.assertNotIn(_hash(1), service._block_fast_lane_reservations) + self.assertEqual( + fixture.server.block_candidate_abandoned_counts, + {"stale-job": 1}, + ) + self.assertEqual(fixture.cleanup_backlog(), {}) + + def test_a_skip_reserves_no_max_block_capacity(self) -> None: + """A stale sibling must not consume capacity on its way to abandonment.""" + fixture = CollapseFixture() + fixture.server.max_blocks = 1 + fixture.server.stop_after_block = False + candidates = fixture.seed( + [_hash(1), _hash(2), _hash(3)], + ) + for candidate in candidates: + _drive(fixture, candidate) + self.assertEqual(fixture.counts()["dequeue_skipped"], 3) + self.assertEqual(fixture.pending(), set()) + with fixture.server.lock: + self.assertEqual(fixture.service._block_fast_lane_reservations, set()) + + +class DequeueSkipPerturbationTests(unittest.TestCase): + """No evidence shape ever produces a skip-only hash, in either view.""" + + def test_no_perturbation_ever_produces_a_skip_only_hash(self) -> None: + candidates = 20 + target = _hash(7) + for view in ("live", "restart"): + for kind in PERTURBATIONS: + with self.subTest(view=view, perturbation=kind): + oracle = self._drain(candidates, view, kind, target, False) + skipped = self._drain(candidates, view, kind, target, True) + self.assertEqual( + skipped - oracle, + set(), + f"{kind} produced a skip-only hash in the {view} view", + ) + + def test_five_of_six_shapes_exclude_the_perturbed_hash_outright(self) -> None: + """And the sixth is the retry slot, which the dequeue itself consumes. + + ``perturb('retry_slot')`` moves a never-offered wakeup into the + global retry holder -- what the shipped path does when the fast-lane + reservation declines. ``submit_next`` clears that holder as it + dequeues, so by the time the skip judges the object, the holder no + longer names it and cannot be evidence about it. The per-row oracle + abandons that row too (it re-offers and finds the tip moved), so the + subset invariant above still holds and no block the per-row path + would have won is destroyed; it is simply a place where the skip is + no more conservative than the path it replaces. + """ + candidates = 20 + target = _hash(7) + for view in ("live", "restart"): + for kind in PERTURBATIONS: + with self.subTest(view=view, perturbation=kind): + skipped = self._drain(candidates, view, kind, target, True) + if kind == "retry_slot": + self.assertIn(target, skipped) + else: + self.assertNotIn(target, skipped) + + def _drain( + self, + candidates: int, + view: str, + kind: str, + target: str, + dequeue_skip: bool, + ) -> set[str]: + rig = CandidateStormRig(candidates=candidates) + rig.seed_live() + if view == "restart": + rig.restart_and_enumerate() + server = rig.restarted_server if view == "restart" else rig.live_server + winner = rig.decide_height() + with redirect_stdout(StringIO()): + undo = rig.perturb(server, kind, target) + skip_set = _record_skips(server) if dequeue_skip else set() + try: + report = rig.drain_per_row( + server, + stop_before_hash=winner, + dequeue_skip=dequeue_skip, + ) + finally: + if undo is not None: + undo() + return skip_set if dequeue_skip else set(report.abandoned_hashes) + + +class DequeueAdmissionUnchangedTests(unittest.TestCase): + """D5: the live queue stays FIFO and stays bounded at 32.""" + + def test_the_live_queue_is_still_a_bounded_fifo_of_thirty_two(self) -> None: + self.assertEqual(MAX_PENDING_BLOCK_CANDIDATES, 32) + fixture = CollapseFixture() + service = fixture.service + service.candidate_queue = None + candidates = fixture.seed([_hash(index) for index in range(1, 40)]) + with redirect_stdout(StringIO()): + admitted = [ + fixture.server.enqueue_block_candidate(candidate) + for candidate in candidates + ] + queue_obj = service.candidate_queue + self.assertIsInstance(queue_obj, queue.Queue) + self.assertNotIsInstance(queue_obj, queue.PriorityQueue) + self.assertEqual(queue_obj.maxsize, 32) + self.assertEqual(queue_obj.qsize(), 32) + # The first 32 wakeups were admitted; the rest coalesced, and their + # durable rows are what keeps them replayable. + self.assertEqual(admitted, [True] * 32 + [False] * 7) + self.assertEqual(int(service.wakeups_coalesced), 7) + self.assertEqual( + [ + str(item.submission.block_hash_hex).lower() + for item in list(queue_obj.queue) + ], + [_hash(index) for index in range(1, 33)], + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_prism_direct_stratum.py b/tests/test_prism_direct_stratum.py index 0e663142a..171b742df 100644 --- a/tests/test_prism_direct_stratum.py +++ b/tests/test_prism_direct_stratum.py @@ -2,8 +2,10 @@ from __future__ import annotations +import dataclasses import unittest from decimal import Decimal +from unittest import mock from lab.auxpow import stratum_codec from lab.prism import direct_stratum @@ -232,6 +234,7 @@ def test_builder_coinbase_template_splits_without_witness_and_reassembles_submit direct_stratum.strip_witness_transaction(submission.coinbase_tx_hex).hex(), submission.coinbase_txid_preimage_hex, ) + self.assertTrue(submission.block_pass) self.assertEqual(submission.block_hex, submission.header_hex + "01" + submission.coinbase_tx_hex) self.assertEqual(len(bytes.fromhex(submission.header_hex)), 80) @@ -270,6 +273,7 @@ def test_merkle_branch_uses_transaction_txids_excluding_witness(self) -> None: ) self.assertEqual(submission.header_hex, expected_header.hex()) + self.assertTrue(submission.block_pass) self.assertTrue(submission.block_hex.endswith(submission.coinbase_tx_hex + extra_tx)) def test_witness_commitment_uses_template_wtxids_for_serialized_block(self) -> None: @@ -299,12 +303,16 @@ def test_witness_commitment_uses_template_wtxids_for_serialized_block(self) -> N [direct_stratum.transaction_wtxid_internal(witness_tx).hex()], ) + # Nonce 1 rather than 0: bits 207fffff put the network target at + # roughly half the hash space, and only a hash that solved the block + # serializes one to inspect. submission = direct_stratum.assemble_submission( job, extranonce2_hex="0102030405060708", ntime_hex=job.ntime, - nonce_hex="00000000", + nonce_hex="00000001", ) + self.assertTrue(submission.block_pass) block_txs = block_transaction_hexes(submission.block_hex) self.assertEqual(block_txs, [submission.coinbase_tx_hex, witness_tx]) self.assertIn(witness_commitment_script(block_txs[1:]), block_txs[0]) @@ -496,6 +504,183 @@ def test_select_version_rolling_mask_uses_gbt_field_or_fallback(self) -> None: ) +class DeferredBlockSerializationTests(unittest.TestCase): + """The block is serialized only for a hash that actually solved it. + + ``block_hex`` costs O(block bytes) of GIL-held decode, concatenate and + re-encode, and only the block-landing path ever reads it, so an ordinary + accepted share must not pay for it (#143). These tests pin both halves of + that bargain: the share path never builds the bytes, and the block path + always has them before a candidate can exist. + """ + + # qbit_target == 1, so no hash solves the block. + UNSOLVABLE_BITS = "03000001" + # Difficulty this low clamps the share target to 2**256-1, so every hash + # passes the share target while missing the (unsolvable) network target. + ALWAYS_SHARE_DIFFICULTY = Decimal("1e-40") + + def job( + self, + *, + bits: str | None = None, + transactions: list[str] | None = None, + desired_share_difficulty: Decimal = Decimal("1"), + ) -> direct_stratum.DirectQbitStratumJob: + coinbase_tx_hex = synthetic_coinbase_template() + built_template = template(transactions) + if bits is not None: + built_template["bits"] = bits + return direct_stratum.make_job_from_builder_manifest( + job_id="job-serialization", + template=built_template, + manifest=manifest_for(coinbase_tx_hex), + extranonce1_hex=EXTRANONCE1, + extranonce2_size=EXTRANONCE2_SIZE, + desired_share_difficulty=desired_share_difficulty, + ) + + def assemble( + self, + job: direct_stratum.DirectQbitStratumJob, + ) -> direct_stratum.DirectQbitSubmission: + return direct_stratum.assemble_submission( + job, + extranonce2_hex="cafebabe00000001", + ntime_hex=job.ntime, + nonce_hex="0000002a", + ) + + def test_ordinary_share_never_serializes_the_block(self) -> None: + job = self.job( + bits=self.UNSOLVABLE_BITS, + transactions=[synthetic_transaction("22")], + desired_share_difficulty=self.ALWAYS_SHARE_DIFFICULTY, + ) + + with mock.patch.object( + direct_stratum, + "serialize_block", + side_effect=AssertionError("serialize_block ran on the share path"), + ) as serialize: + submission = self.assemble(job) + + serialize.assert_not_called() + self.assertTrue(submission.share_pass) + self.assertFalse(submission.block_pass) + self.assertEqual(submission.block_hex, "") + + def test_block_worthy_share_materializes_the_block_in_assembly(self) -> None: + extra_tx = synthetic_transaction("22") + job = self.job(transactions=[extra_tx]) + + submission = self.assemble(job) + + self.assertTrue(submission.block_pass) + # Byte-identical to what the unconditional call produced before the + # share path stopped paying for it. + self.assertEqual( + submission.block_hex, + direct_stratum.serialize_block( + bytes.fromhex(submission.header_hex), + submission.coinbase_tx_hex, + job.transaction_hexes, + ).hex(), + ) + self.assertEqual( + block_transaction_hexes(submission.block_hex), + [submission.coinbase_tx_hex, extra_tx], + ) + + def test_block_hex_attribute_exists_on_every_submission(self) -> None: + """The landing path's duck-typed guard must stay meaningful. + + ``block_candidates`` probes ``hasattr(submission, "block_hex")`` to + tell a real submission from an embedder that retains no block bytes. + Skipping the work must not skip the attribute, or every ordinary share + would start looking like such an embedder. + """ + share_only = self.job( + bits=self.UNSOLVABLE_BITS, + desired_share_difficulty=self.ALWAYS_SHARE_DIFFICULTY, + ) + + submission = self.assemble(share_only) + + self.assertTrue(hasattr(submission, "block_hex")) + self.assertIsInstance(submission.block_hex, str) + + def test_coinbase_validation_still_runs_on_a_share_that_missed_the_block( + self, + ) -> None: + """Deferring the block bytes must not defer coinbase validation. + + ``serialize_block`` used to decode the full coinbase on every share as + a side effect. The explicit txid-preimage check is what actually + guards that value, and it has to keep firing at assembly time -- not + at block landing, which is the worst possible moment to discover it. + """ + job = self.job( + bits=self.UNSOLVABLE_BITS, + desired_share_difficulty=self.ALWAYS_SHARE_DIFFICULTY, + ) + # Move the full coinbase's locktime without moving the Stratum txid + # preimage's: the witness strip no longer reproduces the preimage. + tampered = dataclasses.replace( + job, + full_coinbase_suffix=job.full_coinbase_suffix[:-8] + "01000000", + ) + + with self.assertRaisesRegex(ValueError, "does not match Stratum txid preimage"): + self.assemble(tampered) + + def test_template_transactions_are_validated_once_at_job_build(self) -> None: + """Per-template validation amortizes across every share of that job. + + ``serialize_block`` re-validated the hex of every transaction on every + share. The values are immutable and were already validated here, at + build, so dropping the per-share pass moves no detection anywhere. + """ + malformed = template(["zz"]) + + with self.assertRaisesRegex( + ValueError, + r"template transaction\[0\] data must be hexadecimal", + ): + direct_stratum.make_job_from_builder_manifest( + job_id="job-malformed", + template=malformed, + manifest=manifest_for(synthetic_coinbase_template()), + extranonce1_hex=EXTRANONCE1, + extranonce2_size=EXTRANONCE2_SIZE, + desired_share_difficulty=Decimal("1"), + ) + + # A well-formed template is validated and normalized at build, so the + # tuple every later share reads is already known good. + extra_tx = synthetic_transaction("22") + job = self.job(transactions=[extra_tx.upper()]) + self.assertEqual(job.transaction_hexes, (extra_tx,)) + + def test_explicit_transaction_hexes_are_validated_at_job_build(self) -> None: + """The bundle path passes transactions in directly; it validates too.""" + coinbase_tx_hex = synthetic_coinbase_template() + + with self.assertRaisesRegex( + ValueError, + r"transaction_hexes\[0\] must be hexadecimal", + ): + direct_stratum.make_job_from_builder_manifest( + job_id="job-malformed-explicit", + template=template(), + manifest=manifest_for(coinbase_tx_hex), + extranonce1_hex=EXTRANONCE1, + extranonce2_size=EXTRANONCE2_SIZE, + desired_share_difficulty=Decimal("1"), + transaction_hexes=("zz",), + ) + + class EffectiveShareTargetTests(unittest.TestCase): def test_network_cap_governs_without_minimum(self) -> None: # bits 207fffff: network difficulty ~4.7e-10, so a 500k desired diff --git a/tests/test_prism_endpoint_registry.py b/tests/test_prism_endpoint_registry.py new file mode 100644 index 000000000..c0e640171 --- /dev/null +++ b/tests/test_prism_endpoint_registry.py @@ -0,0 +1,672 @@ +#!/usr/bin/env python3 +"""Pin the endpoint classification registry against real HTTP dispatch. + +The registry in ``lab/prism/endpoint_registry.py`` is the reviewable form of +the issue #145 public/operator split. It is only worth reviewing if it cannot +drift from the code, so these tests assert both directions: every classified +path is actually routed, and every routed path is classified. +""" + +from __future__ import annotations + +import ast +import json +import unittest +import urllib.error +import urllib.request +from pathlib import Path + +from lab.prism import endpoint_registry, public_api +from lab.prism.audit_http import AuditHttpConfig, AuditHttpFacade +from lab.prism.endpoint_registry import ( + Audience, + Disposition, + ENDPOINTS, + LedgerAccess, +) +from lab.prism.observability import METRICS_STATE_FRESH, MetricsSnapshotResponse + + +ROOT = Path(__file__).resolve().parents[1] +AUDIT_HTTP_PATH = ROOT / "lab" / "prism" / "audit_http.py" +PUBLIC_API_PATH = ROOT / "lab" / "prism" / "public_api.py" + +HEX64 = "ab" * 32 +PLACEHOLDERS = { + "{block_hash}": HEX64, + "{fanout_txid}": HEX64, + "{commitment_leaf_hex}": HEX64, + "{sha256}": HEX64, + "{recipient_id}": "miner-1", +} + +UNKNOWN_ENDPOINT_BODY = {"error": "unknown endpoint"} +UNKNOWN_PUBLIC_MESSAGE = "unknown public dashboard endpoint" + + +def concrete_path(path: str) -> str: + for placeholder, value in PLACEHOLDERS.items(): + path = path.replace(placeholder, value) + if "{" in path: + raise AssertionError(f"unsubstituted placeholder in {path!r}") + return path + + +class _Sentinel(Exception): + """Raised by the fake coordinator once dispatch reaches a real handler.""" + + +class _ExplodingLedger: + backend_name = "fixture" + + def __getattr__(self, name: str): + def _call(*args: object, **kwargs: object): + raise _Sentinel(name) + + return _call + + +class _ExplodingRpc: + def call(self, *args: object, **kwargs: object): + raise _Sentinel("rpc") + + +class _FakeCoordinator: + def __init__(self) -> None: + self.ledger = _ExplodingLedger() + self.rpc = _ExplodingRpc() + + +class _RoutingPort: + """Answers every AuditHttpPort method so only routing is under test.""" + + def cached_health_payload(self) -> tuple[int, dict[str, object]]: + return 200, {"ok": True} + + def cached_metrics_payload(self) -> MetricsSnapshotResponse: + return MetricsSnapshotResponse( + status=200, + body="fixture 1\n", + state=METRICS_STATE_FRESH, + age_seconds=0, + ) + + def latest_evidence_payload(self) -> dict[str, object] | None: + return {"schema": "fixture"} + + def owed_balances_payload(self) -> dict[str, object]: + return {"balances": []} + + def carry_forward_integrity_payload(self) -> dict[str, object]: + return {"ok": True} + + def miner_status_payload(self, recipient_id: str) -> dict[str, object]: + return {"recipient_id": recipient_id} + + def ledger_backend(self) -> str: + return "fixture" + + def audit_share_window(self, **kwargs: object) -> list[dict[str, object]]: + return [{"fixture": True}] + + def audit_block_payouts(self, **kwargs: object) -> list[dict[str, object]]: + return [{"fixture": True}] + + def audit_ctv_fanouts(self, **kwargs: object) -> list[dict[str, object]]: + return [{"fixture": True}] + + def audit_ctv_fanout_manifest_set(self, **kwargs: object) -> dict[str, object]: + return {"fixture": True} + + def ctv_fanout_status(self, **kwargs: object) -> dict[str, object]: + return {"fixture": True} + + def pending_ctv_fanout_statuses(self, **kwargs: object) -> list[dict[str, object]]: + return [{"fixture": True}] + + def audit_bundle(self, **kwargs: object) -> dict[str, object]: + return {"fixture": True} + + def audit_bundle_by_commitment(self, **kwargs: object) -> dict[str, object]: + return {"fixture": True} + + +def string_literals(source_path: Path, function_name: str) -> set[str]: + """Absolute path literals mentioned inside one function.""" + + tree = ast.parse(source_path.read_text(encoding="utf-8")) + for node in ast.walk(tree): + if isinstance(node, ast.FunctionDef) and node.name == function_name: + return { + child.value + for child in ast.walk(node) + if isinstance(child, ast.Constant) + and isinstance(child.value, str) + and child.value.startswith("/") + } + raise AssertionError(f"{function_name} not found in {source_path.name}") + + +class EndpointRegistryShapeTests(unittest.TestCase): + def test_paths_are_unique_across_the_registry(self) -> None: + seen: dict[str, str] = {} + for endpoint in ENDPOINTS: + for path in endpoint.paths: + self.assertNotIn( + path, + seen, + f"{path} classified twice ({seen.get(path)} and {endpoint.primary_path})", + ) + seen[path] = endpoint.primary_path + + def test_only_the_public_read_surface_is_extracted(self) -> None: + for endpoint in ENDPOINTS: + expected = ( + Disposition.EXTRACT + if endpoint.audience is Audience.PUBLIC_READ + else Disposition.RETAIN + ) + self.assertIs( + endpoint.disposition, + expected, + f"{endpoint.primary_path}: {endpoint.audience} must be {expected}", + ) + + def test_extracted_surface_is_exactly_public_v1(self) -> None: + # PRISM.md: "operators can expose only /public/v1 through a reverse + # proxy or dashboard frontend". The extraction must not widen that. + for path in endpoint_registry.extracted_paths(): + self.assertTrue( + path.startswith("/public/v1/"), + f"{path} is not part of the documented public API surface", + ) + + def test_audit_routes_are_never_extracted(self) -> None: + # docs/public-dashboard-api/README.md and + # tests/test_public_dashboard_api_contract.py both forbid exposing + # /audit/* as public API. + for path in endpoint_registry.extracted_paths(): + self.assertFalse(path.startswith("/audit/")) + for prefix in ("/audit/", "/healthz", "/metrics", "/owed"): + self.assertTrue( + any(path.startswith(prefix) for path in endpoint_registry.retained_paths()), + f"expected retained paths under {prefix}", + ) + + def test_extracted_and_retained_partition_the_registry(self) -> None: + extracted = set(endpoint_registry.extracted_paths()) + retained = set(endpoint_registry.retained_paths()) + self.assertEqual(set(), extracted & retained) + every_path = {path for endpoint in ENDPOINTS for path in endpoint.paths} + self.assertEqual(every_path, extracted | retained) + + def test_writer_lock_extractions_record_a_blocker(self) -> None: + # The Endpoint constructor enforces this; assert the set is non-empty so + # the rule cannot be satisfied vacuously by dropping the classification. + blockers = dict(endpoint_registry.extraction_blockers()) + for endpoint in ENDPOINTS: + if ( + LedgerAccess.WRITER_LOCK in endpoint.access + and endpoint.disposition is Disposition.EXTRACT + ): + self.assertIn(endpoint.primary_path, blockers) + self.assertTrue(blockers) + + def test_health_and_metrics_are_retained_operator_surfaces(self) -> None: + retained = endpoint_registry.retained_paths() + self.assertIn("/healthz", retained) + self.assertIn("/metrics", retained) + + def test_drift_count_and_owed_dump_stay_on_the_coordinator(self) -> None: + retained = endpoint_registry.retained_paths() + for path in ( + "/audit/carry-forward-integrity", + "/audit/ledger-integrity", + "/owed", + "/owed-balances", + "/miners/{recipient_id}/status", + "/payouts/{recipient_id}/status", + ): + self.assertIn(path, retained) + + def test_payout_affecting_surface_is_named(self) -> None: + payout = { + path + for endpoint in endpoint_registry.endpoints_by_audience( + Audience.PAYOUT_AFFECTING + ) + for path in endpoint.paths + } + self.assertEqual( + { + "/audit/carry-forward-integrity", + "/audit/ledger-integrity", + "/owed", + "/owed-balances", + "/miners/{recipient_id}/status", + "/payouts/{recipient_id}/status", + }, + payout, + ) + + def test_no_extracted_route_takes_the_writer_lock_without_a_blocker(self) -> None: + blockers = dict(endpoint_registry.extraction_blockers()) + extracted = set(endpoint_registry.extracted_paths()) + for path in endpoint_registry.writer_lock_paths(): + if path in extracted: + self.assertIn(path, blockers) + + +class StalenessBudgetTests(unittest.TestCase): + """The staleness contract must be derived and stated, never hand-picked.""" + + def extracted_endpoints(self) -> list[endpoint_registry.Endpoint]: + return [ + endpoint + for endpoint in ENDPOINTS + if endpoint.disposition is Disposition.EXTRACT + ] + + def test_every_extracted_route_states_a_budget_or_declares_immutability(self) -> None: + for endpoint in self.extracted_endpoints(): + with self.subTest(path=endpoint.primary_path): + self.assertTrue( + endpoint.max_staleness_seconds is not None + or endpoint.immutable_content, + f"{endpoint.primary_path} states no staleness tolerance", + ) + + def test_retained_routes_carry_no_public_staleness_budget(self) -> None: + # The contract covers the extracted public surface. A budget on a + # retained route would imply the coordinator enforces one, which it + # does not. + for endpoint in ENDPOINTS: + if endpoint.disposition is Disposition.EXTRACT: + continue + with self.subTest(path=endpoint.primary_path): + self.assertIsNone(endpoint.max_staleness_seconds) + self.assertFalse(endpoint.immutable_content) + + def test_budgets_match_the_documented_derivation(self) -> None: + # Pin each budget to the formula rather than to a literal, so changing + # a cache default without revisiting the budget fails here. + aggregate = endpoint_registry.PUBLIC_AGGREGATE_CACHE_TTL_DEFAULT_SECONDS + plain = endpoint_registry.PUBLIC_CACHE_TTL_DEFAULT_SECONDS + config = endpoint_registry.PUBLIC_CONFIG_CACHE_TTL_DEFAULT_SECONDS + reward_window = endpoint_registry.POOL_REWARD_WINDOW_CACHE_DEFAULT_SECONDS + derive = endpoint_registry.staleness_budget_seconds + expected = { + "/public/v1/pool-summary": derive(cache_ttl_seconds=aggregate), + "/public/v1/blocks": derive(cache_ttl_seconds=plain), + "/public/v1/leaderboard": derive(cache_ttl_seconds=plain), + "/public/v1/hashrate-series": derive(cache_ttl_seconds=aggregate), + # A cheap grouped scan of the small pool-blocks table, so it is a + # plain row read rather than an aggregate despite serving a chart. + "/public/v1/block-markers": derive(cache_ttl_seconds=plain), + "/public/v1/mining-configuration": derive(cache_ttl_seconds=config), + # The one route with a second cache in series beneath it. + "/public/v1/miners/{recipient_id}": derive( + cache_ttl_seconds=plain, + underlying_cache_seconds=reward_window, + ), + "/public/v1/miners/{recipient_id}/earnings": derive(cache_ttl_seconds=plain), + "/public/v1/miners/{recipient_id}/payouts": derive(cache_ttl_seconds=plain), + "/public/v1/miners/{recipient_id}/workers": derive(cache_ttl_seconds=aggregate), + "/public/v1/blocks/{block_hash}/settlement-artifacts": derive( + cache_ttl_seconds=plain + ), + "/public/v1/fanouts/pending": derive(cache_ttl_seconds=plain), + "/public/v1/fanouts/{fanout_txid}": derive(cache_ttl_seconds=plain), + } + actual = { + endpoint.primary_path: endpoint.max_staleness_seconds + for endpoint in self.extracted_endpoints() + if endpoint.max_staleness_seconds is not None + } + self.assertEqual(expected, actual) + + def test_the_aggregate_ttl_really_is_what_the_cache_policy_applies(self) -> None: + # The derivation above assumes public_cache_policy classifies these + # three as aggregates and everything else as a plain row read. Assert + # that against the policy itself so the two cannot drift apart. + for path, expected_ttl in ( + ("/public/v1/pool-summary", endpoint_registry.PUBLIC_AGGREGATE_CACHE_TTL_DEFAULT_SECONDS), + ("/public/v1/hashrate-series", endpoint_registry.PUBLIC_AGGREGATE_CACHE_TTL_DEFAULT_SECONDS), + ("/public/v1/miners/miner-1/workers", endpoint_registry.PUBLIC_AGGREGATE_CACHE_TTL_DEFAULT_SECONDS), + ("/public/v1/blocks", endpoint_registry.PUBLIC_CACHE_TTL_DEFAULT_SECONDS), + ("/public/v1/block-markers", endpoint_registry.PUBLIC_CACHE_TTL_DEFAULT_SECONDS), + ("/public/v1/miners/miner-1/earnings", endpoint_registry.PUBLIC_CACHE_TTL_DEFAULT_SECONDS), + ("/public/v1/mining-configuration", endpoint_registry.PUBLIC_CONFIG_CACHE_TTL_DEFAULT_SECONDS), + ): + with self.subTest(path=path): + self.assertEqual( + expected_ttl, + public_api.public_cache_policy(path).ttl_seconds, + ) + + def test_no_budget_falls_below_the_documented_floor(self) -> None: + for endpoint in self.extracted_endpoints(): + if endpoint.max_staleness_seconds is None: + continue + with self.subTest(path=endpoint.primary_path): + self.assertGreaterEqual( + endpoint.max_staleness_seconds, + endpoint_registry.MINIMUM_PUBLIC_STALE_SECONDS, + ) + + def test_only_the_artifact_route_is_exempt_from_staleness(self) -> None: + # Content-addressed and immutable: correct at any age. Stated in the + # registry rather than left implicit, so the exemption is reviewable. + exempt = [ + endpoint.primary_path + for endpoint in ENDPOINTS + if endpoint.immutable_content + ] + self.assertEqual(["/public/v1/artifacts/{sha256}"], exempt) + + def test_immutable_content_and_a_budget_are_mutually_exclusive(self) -> None: + with self.assertRaises(ValueError): + endpoint_registry.Endpoint( + paths=("/public/v1/example",), + audience=Audience.PUBLIC_READ, + disposition=Disposition.EXTRACT, + access=(LedgerAccess.READ_SLOT,), + rationale="fixture", + immutable_content=True, + max_staleness_seconds=30, + ) + + def test_an_extracted_route_cannot_omit_its_staleness_tolerance(self) -> None: + with self.assertRaises(ValueError): + endpoint_registry.Endpoint( + paths=("/public/v1/example",), + audience=Audience.PUBLIC_READ, + disposition=Disposition.EXTRACT, + access=(LedgerAccess.READ_SLOT,), + rationale="fixture", + ) + + def test_derivation_rejects_negative_cache_seconds(self) -> None: + with self.assertRaises(ValueError): + endpoint_registry.staleness_budget_seconds(cache_ttl_seconds=-1) + with self.assertRaises(ValueError): + endpoint_registry.staleness_budget_seconds( + cache_ttl_seconds=5, + underlying_cache_seconds=-1, + ) + + +class EndpointForRequestPathTests(unittest.TestCase): + """Concrete request paths must resolve to the endpoint that serves them.""" + + def test_literal_paths_win_over_templates(self) -> None: + # /public/v1/fanouts/pending is a literal list route that would also + # match /public/v1/fanouts/{fanout_txid}; dispatch() gives the literal + # precedence, so the registry lookup must too. + endpoint = endpoint_registry.endpoint_for_request_path( + "/public/v1/fanouts/pending" + ) + assert endpoint is not None + self.assertEqual("/public/v1/fanouts/pending", endpoint.primary_path) + + def test_templates_match_concrete_segments(self) -> None: + for path, expected in ( + (f"/public/v1/fanouts/{HEX64}", "/public/v1/fanouts/{fanout_txid}"), + ("/public/v1/miners/miner-1", "/public/v1/miners/{recipient_id}"), + ( + "/public/v1/miners/miner-1/workers", + "/public/v1/miners/{recipient_id}/workers", + ), + ( + f"/public/v1/blocks/{HEX64}/settlement-artifacts", + "/public/v1/blocks/{block_hash}/settlement-artifacts", + ), + ): + with self.subTest(path=path): + endpoint = endpoint_registry.endpoint_for_request_path(path) + assert endpoint is not None + self.assertEqual(expected, endpoint.primary_path) + + def test_unknown_paths_resolve_to_nothing(self) -> None: + for path in ("/public/v1/not-a-route", "/public/v1", "/nope"): + with self.subTest(path=path): + self.assertIsNone(endpoint_registry.endpoint_for_request_path(path)) + + def test_every_extracted_path_resolves(self) -> None: + for path in endpoint_registry.extracted_paths(): + with self.subTest(path=path): + self.assertIsNotNone( + endpoint_registry.endpoint_for_request_path(concrete_path(path)) + ) + + +class EndpointRegistryRoutingTests(unittest.TestCase): + """Every classified path must actually be routed today.""" + + @classmethod + def setUpClass(cls) -> None: + cls.facade = AuditHttpFacade( + _RoutingPort(), # type: ignore[arg-type] + AuditHttpConfig("127.0.0.1", 0, join_timeout_seconds=1.0), + ) + state = cls.facade.start() + assert state.bound_address is not None + cls.host, cls.port = state.bound_address + + @classmethod + def tearDownClass(cls) -> None: + cls.facade.stop() + + @staticmethod + def decode(body: bytes) -> object: + # /metrics answers Prometheus text rather than JSON; the routing + # assertion only needs "not the unknown-endpoint body". + try: + return json.loads(body or b"null") + except json.JSONDecodeError: + return body.decode(errors="replace") + + def get(self, path: str) -> tuple[int, object]: + url = f"http://{self.host}:{self.port}{path}" + try: + with urllib.request.urlopen(url, timeout=5) as response: + return response.status, self.decode(response.read()) + except urllib.error.HTTPError as error: + with error: + return error.code, self.decode(error.read()) + + def test_every_non_public_registry_path_is_routed(self) -> None: + for endpoint in ENDPOINTS: + for path in endpoint.paths: + if path.startswith("/public/v1"): + continue + with self.subTest(path=path): + status, body = self.get(concrete_path(path)) + self.assertNotEqual( + body, + UNKNOWN_ENDPOINT_BODY, + f"{path} is classified but not routed", + ) + self.assertNotEqual(status, 404, f"{path} returned 404") + + def test_metrics_is_served_as_prometheus_text(self) -> None: + url = f"http://{self.host}:{self.port}/metrics" + with urllib.request.urlopen(url, timeout=5) as response: + self.assertEqual(response.status, 200) + self.assertTrue( + response.headers["Content-Type"].startswith("text/plain") + ) + + def test_unclassified_path_still_404s(self) -> None: + status, body = self.get("/definitely-not-a-route") + self.assertEqual(status, 404) + self.assertEqual(body, UNKNOWN_ENDPOINT_BODY) + + +class PublicDispatchRoutingTests(unittest.TestCase): + """Every /public/v1 path in the registry must be routed by dispatch().""" + + def dispatch(self, path: str) -> None: + public_api.dispatch(_FakeCoordinator(), path, {}) + + def test_every_public_registry_path_is_routed(self) -> None: + for path in endpoint_registry.extracted_paths(): + if not path.startswith("/public/v1"): + continue + with self.subTest(path=path): + try: + self.dispatch(concrete_path(path)) + except _Sentinel: + # Reached a real handler, which is what "routed" means. + continue + except public_api.PublicApiError as error: + self.assertNotEqual( + error.message, + UNKNOWN_PUBLIC_MESSAGE, + f"{path} is classified but dispatch() does not route it", + ) + + def test_unknown_public_path_is_rejected(self) -> None: + with self.assertRaises(public_api.PublicApiError) as caught: + self.dispatch("/public/v1/not-a-route") + self.assertEqual(caught.exception.message, UNKNOWN_PUBLIC_MESSAGE) + + +class RegistryCoverageTests(unittest.TestCase): + """The HTTP layer's route literals must match the registry exactly. + + The guard is a frozen snapshot of the string literals in each dispatch + function, compared for exact set equality. A looser substring match was + rejected on review: a new route that merely *extends* an existing literal + (for example a new ``/public/v1/blocks/...`` sub-route) would pass + unclassified, which is precisely the drift this guard exists to catch. + """ + + # Every "/"-prefixed string constant in audit_http.do_GET, including the + # prefix fragments the dispatcher matches against ("/audit/blocks/", + # "/status", ...). Adding a route without classifying it in + # lab/prism/endpoint_registry.py fails the snapshot equality first. + EXPECTED_DO_GET_LITERALS = frozenset( + { + "/", + "/audit/block/", + "/audit/blocks/", + "/audit/carry-forward-integrity", + "/audit/commitments/", + "/audit/fanouts/", + "/audit/fanouts/pending", + "/audit/latest", + "/audit/ledger-integrity", + "/audit/share-window", + "/bundle", + "/ctv-fanout-manifest-set", + "/ctv-fanouts", + "/healthz", + "/metrics", + "/miners/", + "/owed", + "/owed-balances", + "/payouts", + "/payouts/", + "/status", + } + ) + + # Every "/"-prefixed string constant in public_api.dispatch. + EXPECTED_DISPATCH_LITERALS = frozenset( + { + "/", + "/public/v1/artifacts/", + "/public/v1/block-markers", + "/public/v1/blocks", + "/public/v1/blocks/", + "/public/v1/fanouts/", + "/public/v1/fanouts/pending", + "/public/v1/hashrate-series", + "/public/v1/leaderboard", + "/public/v1/miners/", + "/public/v1/mining-configuration", + "/public/v1/pool-summary", + "/settlement-artifacts", + } + ) + + def registry_paths(self) -> set[str]: + return {path for endpoint in ENDPOINTS for path in endpoint.paths} + + def assert_snapshot(self, literals: set[str], expected: frozenset[str]) -> None: + added = sorted(literals - expected) + removed = sorted(expected - literals) + self.assertEqual( + (added, removed), + ([], []), + "route literals in the HTTP layer changed without a matching " + "classification update in lab/prism/endpoint_registry.py " + f"(new: {added}, gone: {removed})", + ) + + def test_audit_http_route_literals_match_snapshot(self) -> None: + literals = string_literals(AUDIT_HTTP_PATH, "do_GET") + self.assertIn("/healthz", literals) + self.assert_snapshot(literals, self.EXPECTED_DO_GET_LITERALS) + + def test_public_dispatch_route_literals_match_snapshot(self) -> None: + literals = string_literals(PUBLIC_API_PATH, "dispatch") + self.assertIn("/public/v1/pool-summary", literals) + self.assert_snapshot(literals, self.EXPECTED_DISPATCH_LITERALS) + + def test_every_registry_path_is_reachable_through_a_route_literal(self) -> None: + # The snapshots pin the literals; this pins the other direction: every + # classified path must actually be matched by a literal from the + # dispatcher that owns it, so the registry cannot list a path nothing + # serves. + public = set(endpoint_registry.extracted_paths()) + for path in self.registry_paths(): + surface = ( + self.EXPECTED_DISPATCH_LITERALS + if path in public + else self.EXPECTED_DO_GET_LITERALS + ) + with self.subTest(path=path): + self.assertTrue( + any( + path == literal or path.startswith(literal) + for literal in surface + if literal != "/" + ), + f"{path} is classified but no route literal in its " + "dispatcher can match it", + ) + + def test_path_counts_are_pinned(self) -> None: + # The extraction set and the total surface are contract numbers: + # PR 2 moved exactly the 13 original /public/v1 routes, + # /public/v1/block-markers joined the extracted surface later, and the + # registry classifies all 32 request paths the HTTP surface serves. + self.assertEqual(14, len(endpoint_registry.extracted_paths())) + self.assertEqual(32, len(self.registry_paths())) + + def test_http_handler_is_get_only(self) -> None: + # The registry -- and the snapshot guard above -- only inspect do_GET + # and dispatch. If a do_POST ever appears, its routes would be + # invisible to the coverage tests, so pin the assumption the registry + # rests on: the HTTP surface answers GET only. + source = AUDIT_HTTP_PATH.read_text(encoding="utf-8") + handler_methods = sorted( + node.name + for node in ast.walk(ast.parse(source)) + if isinstance(node, ast.FunctionDef) + and node.name.startswith("do_") + and node.name.isupper() is False + and node.name != "do_GET" + ) + self.assertEqual( + [], + handler_methods, + "the endpoint registry only covers GET; new HTTP verb methods " + "need registry coverage first", + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_prism_first_job_latency.py b/tests/test_prism_first_job_latency.py index 9ac76581d..4d2cdd1b6 100644 --- a/tests/test_prism_first_job_latency.py +++ b/tests/test_prism_first_job_latency.py @@ -11,7 +11,6 @@ import dataclasses import json -import re import threading import time import unittest @@ -32,17 +31,19 @@ PayoutLedgerArtifact, _compact_share_payload, _JobBuildCancellation, - _PayoutDeliveryAdmission, canonical_json_sha256, ) +from lab.prism.payout_state import ( + PayoutDeliveryAdmission as _PayoutDeliveryAdmission, +) from lab.prism.prism_coordinator import ClientState, PrismCoordinator -from tests.test_prism_coordinator_job_cache import ( +from tests.prism_coordinator_test_support import ( base_template, client, coordinator, install_fake_bundle_builder, ) -from tests.test_prism_coordinator_vardiff import fake_audit_bundle_popen +from tests.prism_vardiff_test_support import fake_audit_bundle_popen from tests.test_prism_initial_job_delivery import wait_until @@ -331,18 +332,21 @@ def test_generation_retags_reuse_and_content_change_invalidates(self) -> None: first, ) - identities_json, compact_json = first.compact_fragments(shares) + tail_chunks = first.compact_tail_chunks(shares) expected_identities, expected_compact = _compact_share_payload(shares) + # The tail completes a JSON object whose only other member is the + # (empty) head, so it can be parsed back by prefixing "{". + tail = json.loads("{" + "".join(tail_chunks)[1:]) self.assertEqual( - json.loads(identities_json), + tail["compact_share_identities"], [list(identity) for identity in expected_identities], ) self.assertEqual( - json.loads(compact_json), + tail["compact_shares"], [list(entry) for entry in expected_compact], ) - # Fragments encode once per canonical window and are reused verbatim. - self.assertIs(first.compact_fragments(shares)[0], identities_json) + # The tail encodes once per canonical window and is reused verbatim. + self.assertIs(first.compact_tail_chunks(shares), tail_chunks) artifact_bump = dataclasses.replace(artifact, generation=2) rebuilt = server._share_window_serialization_for_artifact( diff --git a/tests/test_prism_heap_census.py b/tests/test_prism_heap_census.py new file mode 100644 index 000000000..08ae40afa --- /dev/null +++ b/tests/test_prism_heap_census.py @@ -0,0 +1,1155 @@ +#!/usr/bin/env python3 +"""Issue #226 part 2: the bounded heap census, malloc_trim, allocator pins. + +Every test here pins a safety property of an instrument that walks a +multi-gigabyte heap or takes glibc's arena locks: that it is off unless an +operator switches it on, that it is bounded when on, that no request path +can reach it, that the trim is paced and reports its effect through the +always-on telemetry, that the image's allocator environment cannot change +silently, and that the soak's resident-set bound is an automated verdict. +""" + +from __future__ import annotations + +import ast +import gc +import json +import os +import re +import signal +import subprocess +import sys +import tempfile +import threading +import time +import tracemalloc +import unittest +from pathlib import Path +from types import SimpleNamespace +from unittest import mock + +from lab.prism import process_telemetry +from lab.prism.coordinator_config import ( + DEFAULT_PRISM_HEAP_CENSUS_CONFIG, + DEFAULT_PRISM_HEAP_CENSUS_MAX_BYTES, + DEFAULT_PRISM_HEAP_CENSUS_MAX_SECONDS, + DEFAULT_PRISM_HEAP_CENSUS_MIN_INTERVAL_SECONDS, + DEFAULT_PRISM_HEAP_CENSUS_TOP_N, + MAX_PRISM_HEAP_CENSUS_MAX_SECONDS, + MAX_PRISM_HEAP_CENSUS_TOP_N, + MIN_PRISM_MALLOC_TRIM_INTERVAL_SECONDS, + env_heap_census_config, + load_coordinator_config, +) +from lab.prism.metrics import MetricsRenderer +from lab.prism.process_telemetry import ( + HEAP_CENSUS_FORMAT, + HEAP_CENSUS_RETAINED_FILES, + HeapCensus, + HeapCensusConfig, + HeapCensusService, + MallocTrimmer, + ProcessHeapSample, + ProcessHeapTelemetry, + evaluate_rss_bound, + heap_census_signal, + install_heap_census, + malloc_trim_signal, + parse_malloc_info_xml, + read_anonymous_map_shape, + read_malloc_arena_count, + read_malloc_info_summary, + read_resident_memory_bytes, +) +from tests import prism_vardiff_test_support as support + +ROOT = Path(__file__).resolve().parents[1] +PRISM_DIR = ROOT / "lab" / "prism" +DOCKERFILE = PRISM_DIR / "Dockerfile" +COMPOSE_FILE = ROOT / "compose.yaml" +ENV_EXAMPLE = ROOT / ".env.example" + +# SIGRTMIN is Linux-only. The fake signal module carries a synthetic number so +# every fake-driven test runs on macOS too; only the test that delivers a real +# signal to this process gates its real-time-signal half on the platform. +_HAS_SIGRTMIN = hasattr(signal, "SIGRTMIN") +_SIGRTMIN = getattr(signal, "SIGRTMIN", 32) +_HAS_PROCFS = Path("/proc/self/statm").exists() + +# The image's allocator environment, as decided in lab/prism/Dockerfile and +# argued in docs/mainnet-deployment.md. A change here must be a decision. +PINNED_MALLOC_ARENA_MAX = "2" + +# Every name through which the walk or the trim can be reached. No module +# that serves a request may mention any of them. +CENSUS_ENTRY_POINTS = ( + "HeapCensus", + "HeapCensusService", + "install_heap_census", + "MallocTrimmer", + "resolve_malloc_trim", + "gc.get_objects", + "tracemalloc.take_snapshot", + "malloc_trim", +) + + +# A malloc_info document in glibc's shape: per heap, one row per +# non-empty size class (fastbin classes first), an row, the fast +# and rest totals (rest = bins + top, never fastbins), then the global totals. +_MALLOC_INFO_FIXTURE = """ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +""" + + +def _coordinator_service_environment() -> dict[str, str]: + """The prism-coordinator service's environment mapping, from the raw text. + + Parsed without a YAML dependency: the service block runs from its key to + the next two-space-indented key, and its environment entries are the + six-space-indented ``NAME: value`` lines inside it. + """ + lines = COMPOSE_FILE.read_text(encoding="utf-8").splitlines() + start = lines.index(" prism-coordinator:") + block: list[str] = [] + for line in lines[start + 1:]: + if re.match(r"^ [A-Za-z0-9_-]+:\s*$", line): + break + block.append(line) + environment: dict[str, str] = {} + inside = False + for line in block: + if re.match(r"^ environment:\s*$", line): + inside = True + continue + if inside and re.match(r"^ [A-Za-z0-9_-]+:", line): + inside = False + if inside: + match = re.match(r"^ ([A-Z0-9_]+): (.*)$", line) + if match: + environment[match.group(1)] = match.group(2).strip() + return environment + + +def _env_example_values() -> dict[str, str]: + values: dict[str, str] = {} + for line in ENV_EXAMPLE.read_text(encoding="utf-8").splitlines(): + if line and not line.startswith("#") and "=" in line: + key, value = line.split("=", 1) + values[key] = value + return values + + +def _config(**overrides: object) -> HeapCensusConfig: + values: dict[str, object] = { + "enabled": True, + "output_dir": "unused", + "top_n": 5, + "max_bytes": 65_536, + "max_seconds": 10.0, + "min_interval_seconds": 60.0, + "tracemalloc_enabled": False, + "malloc_trim_signal_enabled": False, + "malloc_trim_interval_seconds": 0.0, + } + values.update(overrides) + return HeapCensusConfig(**values) # type: ignore[arg-type] + + +def _sample(**overrides: int) -> ProcessHeapSample: + values: dict[str, object] = { + "allocated_blocks": 1_000, + "gc_trigger_count": (1, 2, 3), + "gc_collections": (4, 5, 6), + "gc_collected": (7, 8, 9), + "gc_uncollectable": (0, 0, 0), + "threads": 64, + "malloc_info_available": True, + "malloc_arena_bytes": 10_000, + "malloc_in_use_bytes": 6_000, + "malloc_free_bytes": 4_000, + "malloc_mmapped_bytes": 500, + } + values.update(overrides) + return ProcessHeapSample(**values) # type: ignore[arg-type] + + +class _FakeClock: + def __init__(self, now: float = 1_000.0) -> None: + self.now = now + + def __call__(self) -> float: + return self.now + + def advance(self, seconds: float) -> None: + self.now += seconds + + +class _FakeSignalModule: + """Records registrations instead of installing real handlers.""" + + SIGUSR1 = signal.SIGUSR1 + SIGRTMIN = _SIGRTMIN + SIG_DFL = signal.SIG_DFL + + def __init__(self) -> None: + self.registered: dict[int, object] = {} + + def signal(self, signum: int, handler: object) -> None: + self.registered[signum] = handler + + def getsignal(self, signum: int) -> object: + return self.registered.get(signum, self.SIG_DFL) + + +class HeapCensusDisabledByDefaultTests(unittest.TestCase): + def test_heap_census_is_disabled_by_default(self) -> None: + """Without the switch nothing is registered and the entry point is inert.""" + config = env_heap_census_config(environ={}) + self.assertFalse(config.enabled) + self.assertFalse(config.tracemalloc_enabled) + self.assertFalse(config.malloc_trim_signal_enabled) + self.assertEqual(config.malloc_trim_interval_seconds, 0.0) + self.assertFalse(config.anything_armed) + # The shipped lifecycle default is what the loader produces from an + # empty environment, switches and bounds alike. + self.assertEqual(config, DEFAULT_PRISM_HEAP_CENSUS_CONFIG) + fake_signal = _FakeSignalModule() + with ( + mock.patch.object( + gc, + "get_objects", + side_effect=AssertionError("gc.get_objects ran with the census off"), + ) as get_objects, + mock.patch.object( + tracemalloc, + "start", + side_effect=AssertionError("tracemalloc started with the census off"), + ) as tracemalloc_start, + ): + self.assertIsNone(install_heap_census(config, signal_module=fake_signal)) + self.assertEqual(fake_signal.registered, {}) + self.assertFalse(tracemalloc.is_tracing()) + with tempfile.TemporaryDirectory() as directory: + census = HeapCensus(_config(enabled=False, output_dir=directory)) + self.assertIsNone(census.take("direct call")) + self.assertIsNone(census.last_outcome) + self.assertEqual(os.listdir(directory), []) + service = HeapCensusService(_config(enabled=False, output_dir=directory)) + try: + self.assertFalse(service.request_census("signal")) + self.assertIsNone(service.dispatch("census", "direct")) + self.assertEqual(os.listdir(directory), []) + finally: + service.close() + get_objects.assert_not_called() + tracemalloc_start.assert_not_called() + # No worker thread exists for a census that never armed. + self.assertFalse( + any( + thread.name == HeapCensusService.THREAD_NAME + for thread in threading.enumerate() + ) + ) + + def test_census_switches_are_strict_and_bounded(self) -> None: + """A typo or an unbounded value refuses startup rather than arming.""" + for bad in ( + {"PRISM_HEAP_CENSUS": "maybe"}, + {"PRISM_HEAP_CENSUS_TRACEMALLOC": "1"}, + {"PRISM_HEAP_CENSUS": "1", "PRISM_HEAP_CENSUS_TRACEMALLOC": "sometimes"}, + {"PRISM_HEAP_CENSUS_TOP_N": "0"}, + {"PRISM_HEAP_CENSUS_TOP_N": str(MAX_PRISM_HEAP_CENSUS_TOP_N + 1)}, + {"PRISM_HEAP_CENSUS_MAX_BYTES": "10"}, + {"PRISM_HEAP_CENSUS_MAX_BYTES": str(64 * 1024 * 1024)}, + {"PRISM_HEAP_CENSUS_MAX_SECONDS": "0"}, + {"PRISM_HEAP_CENSUS_MAX_SECONDS": str(MAX_PRISM_HEAP_CENSUS_MAX_SECONDS + 1)}, + {"PRISM_HEAP_CENSUS_MIN_INTERVAL_SECONDS": "-1"}, + {"PRISM_MALLOC_TRIM_SIGNAL": "yes please"}, + { + "PRISM_MALLOC_TRIM_INTERVAL_SECONDS": str( + MIN_PRISM_MALLOC_TRIM_INTERVAL_SECONDS / 2 + ) + }, + ): + with self.subTest(bad=bad), self.assertRaises(SystemExit): + env_heap_census_config(environ=bad) + armed = env_heap_census_config( + environ={ + "PRISM_HEAP_CENSUS": "on", + "PRISM_HEAP_CENSUS_TRACEMALLOC": "true", + "PRISM_MALLOC_TRIM_SIGNAL": "1", + "PRISM_MALLOC_TRIM_INTERVAL_SECONDS": "600", + "PRISM_AUDIT_DIR": "/var/lib/qbit-prism/audit", + } + ) + self.assertTrue(armed.enabled) + self.assertTrue(armed.tracemalloc_enabled) + self.assertTrue(armed.malloc_trim_signal_enabled) + self.assertEqual(armed.malloc_trim_interval_seconds, 600.0) + self.assertEqual(armed.output_dir, "/var/lib/qbit-prism/audit/heap-census") + self.assertEqual(armed.top_n, DEFAULT_PRISM_HEAP_CENSUS_TOP_N) + self.assertEqual(armed.max_bytes, DEFAULT_PRISM_HEAP_CENSUS_MAX_BYTES) + self.assertEqual(armed.max_seconds, DEFAULT_PRISM_HEAP_CENSUS_MAX_SECONDS) + self.assertEqual( + armed.min_interval_seconds, DEFAULT_PRISM_HEAP_CENSUS_MIN_INTERVAL_SECONDS + ) + # The loader carries the validated snapshot, so a bad bound refuses + # coordinator startup, not just a standalone read. + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + base = { + "QBIT_RPC_HOST": "qbit.example", + "QBIT_RPC_USER": "rpc-user", + "QBIT_RPC_PASSWORD": "rpc-password", + "PRISM_ALLOW_MEMORY_LEDGER": "1", + "PRISM_ALLOW_TEST_SIGNING_SEEDS": "1", + "PRISM_ALLOW_BUNDLE_EMBEDDED_LEDGER_KEY": "1", + "PRISM_AUDIT_DIR": str(root), + "PRISM_EVIDENCE_PATH": str(root / "evidence.json"), + } + shipped = load_coordinator_config(base) + self.assertFalse(shipped.lifecycle.heap_census.enabled) + self.assertFalse(shipped.lifecycle.heap_census.anything_armed) + self.assertEqual( + shipped.lifecycle.heap_census.output_dir, str(root / "heap-census") + ) + with self.assertRaisesRegex(SystemExit, "PRISM_HEAP_CENSUS_TOP_N must be at most"): + load_coordinator_config( + {**base, "PRISM_HEAP_CENSUS": "1", "PRISM_HEAP_CENSUS_TOP_N": "9999"} + ) + with self.assertRaisesRegex(SystemExit, "PRISM_HEAP_CENSUS must be a boolean"): + load_coordinator_config({**base, "PRISM_HEAP_CENSUS": "maybe"}) + loaded = load_coordinator_config({**base, "PRISM_HEAP_CENSUS": "1"}) + self.assertTrue(loaded.lifecycle.heap_census.enabled) + self.assertEqual( + loaded.lifecycle.heap_census.output_dir, + str(loaded.audit.directory / "heap-census"), + ) + + +class HeapCensusBoundedReportTests(unittest.TestCase): + def test_heap_census_writes_a_bounded_report_to_a_file(self) -> None: + """On a heap with more types than N, the file holds exactly N and fits.""" + + class Retained: + pass + + # More distinct tracked types than the census may list, each with + # enough instances to outrank the interpreter's own bookkeeping. + kinds = [type(f"CensusProbe{index}", (Retained,), {}) for index in range(12)] + keep = [kind() for kind in kinds for _ in range(50)] + with tempfile.TemporaryDirectory() as directory: + config = _config(output_dir=os.path.join(directory, "nested"), top_n=5, max_bytes=16_384) + census = HeapCensus(config) + outcome = census.take("test") + self.assertIsNotNone(outcome) + assert outcome is not None + path = Path(outcome.path) + self.assertTrue(path.exists()) + self.assertEqual(path.parent, Path(config.output_dir)) + self.assertLessEqual(path.stat().st_size, config.max_bytes) + self.assertEqual(outcome.bytes_written, path.stat().st_size) + report = json.loads(path.read_text(encoding="utf-8")) + self.assertEqual(report["format"], HEAP_CENSUS_FORMAT) + self.assertEqual(report["reason"], "test") + self.assertEqual(report["bounds"], {"top_n": 5, "max_bytes": 16_384, "max_seconds": 10.0}) + self.assertEqual(len(report["types_by_count"]), 5) + self.assertEqual(len(report["types_by_bytes"]), 5) + self.assertEqual(report["top_n_written"], 5) + self.assertFalse(report["walk"]["truncated"]) + self.assertGreater(report["walk"]["distinct_types"], 5) + self.assertEqual(report["walk"]["objects_walked"], report["walk"]["tracked_objects"]) + self.assertGreaterEqual(report["walk"]["tracked_objects"], len(keep)) + self.assertEqual(report["tracemalloc"], {"tracing": False}) + self.assertEqual(report["pid"], os.getpid()) + self.assertIn("resident_memory_bytes", report["process"]) + self.assertIn("malloc_arena_count", report["process"]) + self.assertIn("anonymous_map", report["process"]) + # Each entry is a name, a count and a shallow byte total; the + # list is sorted by its key and nothing in it is a repr of an + # object (no addresses, no payload). + counts = [entry["count"] for entry in report["types_by_count"]] + self.assertEqual(counts, sorted(counts, reverse=True)) + for entry in report["types_by_count"] + report["types_by_bytes"]: + self.assertEqual(set(entry), {"type", "count", "shallow_bytes"}) + self.assertNotIn("0x", entry["type"]) + # The byte cap is met by shrinking N, never by cutting JSON. + tight = HeapCensus(_config(output_dir=directory, top_n=200, max_bytes=4_096)) + tight_outcome = tight.take("tight") + assert tight_outcome is not None + tight_report = json.loads(Path(tight_outcome.path).read_text(encoding="utf-8")) + self.assertLessEqual(Path(tight_outcome.path).stat().st_size, 4_096) + self.assertLess(tight_report["top_n_written"], 200) + self.assertTrue(tight_report["fit_to_max_bytes"]) + self.assertEqual(len(tight_report["types_by_count"]), tight_report["top_n_written"]) + # The walk cap: with no time at all the pass stops after its + # first stride and says so, and the report is still written. + clock = _FakeClock() + capped = HeapCensus( + _config(output_dir=directory, max_seconds=0.001), + clock=_advancing(clock, 1.0), + ) + capped_outcome = capped.take("capped") + assert capped_outcome is not None + self.assertTrue(capped_outcome.walk_truncated) + self.assertLess(capped_outcome.objects_walked, capped_outcome.tracked_objects) + capped_report = json.loads(Path(capped_outcome.path).read_text(encoding="utf-8")) + self.assertTrue(capped_report["walk"]["truncated"]) + # Files are retained up to a fixed count; the oldest go first, + # including the first report this census wrote. + for index in range(HEAP_CENSUS_RETAINED_FILES + 3): + self.assertIsNotNone(census.take(f"fill-{index}")) + names = sorted( + name + for name in os.listdir(config.output_dir) + if name.startswith("heap-census-") + ) + self.assertEqual(len(names), HEAP_CENSUS_RETAINED_FILES) + self.assertFalse(path.exists()) + self.assertFalse(any(name.endswith(".tmp") for name in os.listdir(config.output_dir))) + del keep + + def test_types_sharing_a_printed_name_merge_rather_than_overwrite(self) -> None: + """Two type objects with one printed name add up; neither hides the other.""" + + def make_probe_class() -> type: + class CensusMergeProbe: + pass + + # The same short printed name for both, as two modules defining + # one class name would have; the nested qualname would only + # truncate into a third collision. + CensusMergeProbe.__qualname__ = "CensusMergeProbe" + return CensusMergeProbe + + first = make_probe_class() + second = make_probe_class() + self.assertIsNot(first, second) + self.assertEqual( + (first.__module__, first.__qualname__), (second.__module__, second.__qualname__) + ) + # Two more whose long names truncate to the same printed name. + long_a = type("L" * 150, (object,), {}) + long_b = type("L" * 149 + "Z", (object,), {}) + keep = [first() for _ in range(30)] + [second() for _ in range(20)] + keep += [long_a() for _ in range(7)] + [long_b() for _ in range(5)] + with tempfile.TemporaryDirectory() as directory: + outcome = HeapCensus(_config(output_dir=directory, top_n=200)).take("merge") + assert outcome is not None + report = json.loads(Path(outcome.path).read_text(encoding="utf-8")) + del keep + by_name = {entry["type"]: entry for entry in report["types_by_count"]} + merged = [name for name in by_name if name.endswith(".CensusMergeProbe")] + self.assertEqual(len(merged), 1, merged) + self.assertEqual(by_name[merged[0]]["count"], 50) + self.assertGreater(by_name[merged[0]]["shallow_bytes"], 0) + truncated = [name for name in by_name if name.endswith("...") and "LLLL" in name] + self.assertEqual(len(truncated), 1, truncated) + self.assertEqual(by_name[truncated[0]]["count"], 12) + # The walk still reports how many type objects it saw, and how many + # printed names they merged into. + self.assertGreater(report["walk"]["distinct_types"], report["walk"]["distinct_type_names"]) + + def test_retention_keeps_the_newest_reports_regardless_of_mtime(self) -> None: + """A tight loop past the limit evicts the oldest even when every mtime ties.""" + with tempfile.TemporaryDirectory() as directory: + census = HeapCensus(_config(output_dir=directory, top_n=3)) + total = HEAP_CENSUS_RETAINED_FILES + 8 + for index in range(total - 1): + self.assertIsNotNone(census.take(f"fill-{index}")) + # Every existing report now carries the same timestamp, so the + # eviction that the final census triggers cannot lean on mtime. + same_stamp = 1_700_000_000 + for name in os.listdir(directory): + os.utime(os.path.join(directory, name), (same_stamp, same_stamp)) + self.assertIsNotNone(census.take(f"fill-{total - 1}")) + names = sorted( + name for name in os.listdir(directory) if name.startswith("heap-census-") + ) + self.assertEqual(len(names), HEAP_CENSUS_RETAINED_FILES) + reasons = [ + json.loads(Path(directory, name).read_text(encoding="utf-8"))["reason"] + for name in names + ] + expected = [f"fill-{index}" for index in range(total - HEAP_CENSUS_RETAINED_FILES, total)] + self.assertEqual(reasons, expected) + # Name order is write order: the sequence is zero-padded, so the + # tenth report does not sort before the ninth. + sequences = [int(name.rsplit("-", 1)[1].split(".")[0]) for name in names] + self.assertEqual(sequences, sorted(sequences)) + self.assertEqual(sequences, list(range(total - HEAP_CENSUS_RETAINED_FILES + 1, total + 1))) + + def test_heap_census_reports_tracemalloc_sites_only_while_tracing(self) -> None: + """The site lists appear only when tracing, and are capped at N too.""" + self.assertFalse(tracemalloc.is_tracing()) + tracemalloc.start(1) + try: + junk = [bytearray(2_048) for _ in range(200)] + with tempfile.TemporaryDirectory() as directory: + outcome = HeapCensus(_config(output_dir=directory, top_n=3)).take("traced") + assert outcome is not None + self.assertTrue(outcome.tracemalloc_traced) + report = json.loads(Path(outcome.path).read_text(encoding="utf-8")) + traced = report["tracemalloc"] + self.assertTrue(traced["tracing"]) + self.assertEqual(traced["frames"], 1) + self.assertLessEqual(len(traced["by_line"]), 3) + self.assertLessEqual(len(traced["by_file"]), 3) + self.assertGreater(traced["traced_bytes"], 0) + for entry in traced["by_line"] + traced["by_file"]: + self.assertEqual(set(entry), {"site", "bytes", "count"}) + del junk + finally: + tracemalloc.stop() + + def test_malloc_info_split_excludes_fastbins_from_bins(self) -> None: + """Fastbin rows are not part of ; top must not absorb them.""" + summary = parse_malloc_info_xml(_MALLOC_INFO_FIXTURE) + self.assertTrue(summary.available) + self.assertEqual(summary.arenas, 2) + self.assertEqual(summary.fast_bytes, 492) + # Heap 0: rows 198+294 (fast) + 145 + 40002 + unsorted 5000; rest is + # bins plus top = 145147, so bins are 45147 and top is 100000. + # Heap 1: nothing in any bin, rest 70000 is all top. + self.assertEqual(summary.bin_bytes, 45_147) + self.assertEqual(summary.top_bytes, 170_000) + self.assertEqual(summary.system_bytes, 1_200_000) + # The identity the split must keep: fast + bins + top is exactly the + # sum of glibc's own fast and rest totals. + self.assertEqual(summary.fast_bytes + summary.bin_bytes + summary.top_bytes, 492 + 215_147) + # A document without heaps folds to zero arenas, not to a crash. + empty = parse_malloc_info_xml('\n\n') + self.assertEqual((empty.arenas, empty.top_bytes, empty.bin_bytes), (0, 0, 0)) + # And the live reading on this host, where glibc offers it, is + # internally consistent: no negative part, parts under system bytes. + live = read_malloc_info_summary() + if live.available: + self.assertGreaterEqual(live.top_bytes, 0) + self.assertGreaterEqual(live.bin_bytes, 0) + self.assertGreaterEqual(live.fast_bytes, 0) + self.assertLessEqual(live.top_bytes + live.bin_bytes + live.fast_bytes, live.system_bytes) + + def test_process_readings_are_available_on_this_host(self) -> None: + """The readings the census and the storm instrument share are real here.""" + resident = read_resident_memory_bytes() + shape = read_anonymous_map_shape() + arenas = read_malloc_arena_count() + if not Path("/proc/self/maps").exists(): + self.skipTest("no procfs on this platform") + self.assertGreater(resident, 0) + self.assertTrue(shape.available) + self.assertGreaterEqual(shape.regions, 1) + self.assertEqual( + shape.regions, sum(shape.band_regions.values()) + ) + self.assertEqual(shape.total_bytes, sum(shape.band_bytes.values())) + if ProcessHeapTelemetry().sample().malloc_info_available: + self.assertGreaterEqual(arenas, 1) + # A pid that does not exist reads as unavailable, not as zero. + self.assertEqual(read_resident_memory_bytes(2**22 + 12345), -1) + self.assertFalse(read_anonymous_map_shape(2**22 + 12345).available) + + +def _advancing(clock: _FakeClock, step: float): + """A clock that moves ``step`` forward on every read.""" + + def read() -> float: + clock.advance(step) + return clock.now + + return read + + +class HeapCensusRequestPathTests(unittest.TestCase): + def test_heap_census_is_not_reachable_from_any_request_path(self) -> None: + """Structurally: only main() in the coordinator can arm it, and no server module names it.""" + # 1. The coordinator references the installer in exactly one place, + # inside main(), after the shutdown handlers. + source = (PRISM_DIR / "prism_coordinator.py").read_text(encoding="utf-8") + tree = ast.parse(source) + main_node = next( + node for node in tree.body + if isinstance(node, ast.FunctionDef) and node.name == "main" + ) + main_span = range(main_node.lineno, main_node.end_lineno + 1) + references = [ + node.lineno + for node in ast.walk(tree) + if isinstance(node, ast.Name) and node.id == "install_heap_census" + ] + self.assertEqual(len(references), 1, references) + self.assertIn(references[0], main_span) + for name in ("HeapCensus", "HeapCensusService", "MallocTrimmer", "gc.get_objects"): + self.assertNotIn(name, source) + # 2. No module that serves a request names any entry point. The + # census module itself, the config that validates its switches, + # and the coordinator's main() are the only exceptions. + exempt = {"process_telemetry.py", "coordinator_config.py", "prism_coordinator.py"} + offenders: dict[str, list[str]] = {} + for module in sorted(PRISM_DIR.glob("*.py")): + if module.name in exempt: + continue + text = module.read_text(encoding="utf-8") + hits = [name for name in CENSUS_ENTRY_POINTS if name in text] + if hits: + offenders[module.name] = hits + self.assertEqual(offenders, {}) + for served in ("audit_http.py", "public_read_service.py", "metrics.py"): + self.assertTrue((PRISM_DIR / served).exists(), served) + # 3. A full metrics render, the one path every scrape takes, never + # walks the heap or trims, even with the census armed in-process. + server = support.coordinator() + with tempfile.TemporaryDirectory() as directory: + service = HeapCensusService(_config(output_dir=directory)) + try: + with ( + mock.patch.object( + gc, + "get_objects", + side_effect=AssertionError("gc.get_objects walked the heap on a scrape"), + ) as get_objects, + mock.patch.object( + tracemalloc, + "take_snapshot", + side_effect=AssertionError("tracemalloc snapshot on a scrape"), + ) as take_snapshot, + mock.patch.object( + HeapCensus, + "take", + side_effect=AssertionError("census taken on a scrape"), + ) as take, + mock.patch.object( + MallocTrimmer, + "trim_once", + side_effect=AssertionError("malloc_trim on a scrape"), + ) as trim, + ): + document = MetricsRenderer(server).render() + self.assertIn("qbit_prism_process_allocated_blocks ", document) + # The signal handlers, run as the interpreter would run + # them, only wake the worker: no walk in handler context. + fake_signal = _FakeSignalModule() + armed = install_heap_census( + _config(output_dir=directory, malloc_trim_signal_enabled=True), + signal_module=fake_signal, + start_thread=False, + log=lambda _line: None, + ) + assert armed is not None + try: + census_handler = fake_signal.registered[heap_census_signal(fake_signal)] + trim_handler = fake_signal.registered[malloc_trim_signal(fake_signal)] + census_handler(signal.SIGUSR1, None) + trim_handler(_SIGRTMIN + 1, None) + finally: + armed.close() + get_objects.assert_not_called() + take_snapshot.assert_not_called() + take.assert_not_called() + trim.assert_not_called() + finally: + service.close() + self.assertEqual( + [name for name in os.listdir(directory) if name.startswith("heap-census-")], + [], + ) + + def test_signal_handlers_queue_work_for_the_worker_thread(self) -> None: + """A real SIGUSR1 to this process produces a census on the worker, not inline. + + The trim half needs a real-time signal, which macOS does not have; + there only the SIGUSR1 half runs. + """ + with tempfile.TemporaryDirectory() as directory: + previous_census = signal.getsignal(signal.SIGUSR1) + previous_trim = signal.getsignal(_SIGRTMIN + 1) if _HAS_SIGRTMIN else None + trimmer = MallocTrimmer( + SimpleNamespace(sample=lambda: _sample()), # type: ignore[arg-type] + resolver=lambda: (lambda _pad: 1), + rss_reader=lambda: 5, + log=lambda _line: None, + ) + service = HeapCensusService( + _config(output_dir=directory, malloc_trim_signal_enabled=True, min_interval_seconds=0.0), + trimmer=trimmer, + log=lambda _line: None, + ) + try: + fake_signal = _FakeSignalModule() + with mock.patch.object( + process_telemetry, "HeapCensusService", return_value=service + ): + armed = install_heap_census( + service.config, signal_module=fake_signal, log=lambda _line: None + ) + self.assertIs(armed, service) + # Install the recorded handlers for real, deliver the + # signals to ourselves, and wait for the worker. + signal.signal(signal.SIGUSR1, fake_signal.registered[signal.SIGUSR1]) + if _HAS_SIGRTMIN: + signal.signal(_SIGRTMIN + 1, fake_signal.registered[_SIGRTMIN + 1]) + try: + os.kill(os.getpid(), signal.SIGUSR1) + if _HAS_SIGRTMIN: + os.kill(os.getpid(), _SIGRTMIN + 1) + # Bounded wait: the worker has ten seconds to serve both. + deadline = time.monotonic() + 10.0 + while time.monotonic() < deadline and ( + service.census.last_outcome is None + or (_HAS_SIGRTMIN and trimmer.calls == 0) + ): + time.sleep(0.02) + finally: + signal.signal(signal.SIGUSR1, previous_census) + if _HAS_SIGRTMIN: + signal.signal(_SIGRTMIN + 1, previous_trim) + outcome = service.census.last_outcome + self.assertIsNotNone(outcome) + assert outcome is not None + self.assertTrue(Path(outcome.path).exists()) + self.assertIn("signal:", json.loads(Path(outcome.path).read_text())["reason"]) + if _HAS_SIGRTMIN: + self.assertEqual(trimmer.calls, 1) + assert trimmer.last_result is not None + self.assertTrue(trimmer.last_result.reason.startswith("signal:")) + else: + self.assertEqual(trimmer.calls, 0) + finally: + service.close() + + def test_close_disarms_signals_and_silences_wake(self) -> None: + """After close() the previous handlers are back and a late signal writes nothing.""" + fake_signal = _FakeSignalModule() + earlier_census = object() + earlier_trim = object() + fake_signal.registered[signal.SIGUSR1] = earlier_census + fake_signal.registered[_SIGRTMIN + 1] = earlier_trim + with tempfile.TemporaryDirectory() as directory: + service = install_heap_census( + _config(output_dir=directory, malloc_trim_signal_enabled=True), + signal_module=fake_signal, + start_thread=False, + log=lambda _line: None, + ) + assert service is not None + census_handler = fake_signal.registered[signal.SIGUSR1] + trim_handler = fake_signal.registered[_SIGRTMIN + 1] + self.assertIsNot(census_handler, earlier_census) + self.assertIsNot(trim_handler, earlier_trim) + self.assertTrue(service.request_census("before close")) + service.close() + # The handlers this service armed are gone, and what they + # replaced is back. + self.assertIs(fake_signal.registered[signal.SIGUSR1], earlier_census) + self.assertIs(fake_signal.registered[_SIGRTMIN + 1], earlier_trim) + # A handler someone kept a reference to, or a request made after + # close, is a no-op: no write into a recycled descriptor. + with mock.patch.object(os, "write", side_effect=AssertionError("wrote after close")) as write: + self.assertFalse(service.request_census("late")) + self.assertFalse(service.request_trim("late")) + self.assertFalse(service._wake(b"c")) + census_handler(signal.SIGUSR1, None) + trim_handler(_SIGRTMIN + 1, None) + write.assert_not_called() + service.close() # idempotent + self.assertEqual( + [name for name in os.listdir(directory) if name.startswith("heap-census-")], + [], + ) + # With nothing registered beforehand the restore lands on SIG_DFL, + # not on a stale handler. + bare = _FakeSignalModule() + service = install_heap_census( + _config(output_dir="unused-never-written", malloc_trim_signal_enabled=True), + signal_module=bare, + start_thread=False, + log=lambda _line: None, + ) + assert service is not None + service.close() + self.assertIs(bare.registered[signal.SIGUSR1], signal.SIG_DFL) + self.assertIs(bare.registered[_SIGRTMIN + 1], signal.SIG_DFL) + + +class MallocTrimTests(unittest.TestCase): + def test_malloc_trim_action_is_bounded_and_reports_its_effect(self) -> None: + """At most one trim per interval, deltas reported through the WP1a sample.""" + calls: list[int] = [] + samples = iter( + [ + _sample(malloc_arena_bytes=10_000, malloc_in_use_bytes=6_000, malloc_free_bytes=4_000), + _sample(malloc_arena_bytes=7_000, malloc_in_use_bytes=6_000, malloc_free_bytes=1_000), + _sample(malloc_arena_bytes=7_000, malloc_in_use_bytes=6_000, malloc_free_bytes=1_000), + _sample(malloc_arena_bytes=7_000, malloc_in_use_bytes=6_000, malloc_free_bytes=1_000), + ] + ) + rss = iter([9_000, 6_500, 6_500, 6_500]) + clock = _FakeClock() + lines: list[str] = [] + + def fake_trim(pad: int) -> int: + calls.append(pad) + return 1 + + trimmer = MallocTrimmer( + SimpleNamespace(sample=lambda: next(samples)), # type: ignore[arg-type] + resolver=lambda: fake_trim, + clock=clock, + rss_reader=lambda: next(rss), + log=lines.append, + ) + service = HeapCensusService( + _config(enabled=False, min_interval_seconds=60.0, malloc_trim_interval_seconds=0.0), + trimmer=trimmer, + clock=clock, + log=lines.append, + ) + try: + first = service.dispatch("trim", "signal:35") + assert first is not None + self.assertEqual(calls, [0]) + self.assertTrue(first.released) + self.assertEqual(first.arena_delta, -3_000) + self.assertEqual(first.free_delta, -3_000) + self.assertEqual(first.in_use_delta, 0) + self.assertEqual(first.resident_delta, -2_500) + self.assertEqual(first.as_dict()["reason"], "signal:35") + self.assertTrue(any(line.startswith("prism malloc_trim ") for line in lines)) + # Two more requests inside the interval are suppressed, and say so. + clock.advance(10.0) + self.assertIsNone(service.dispatch("trim", "signal:35")) + clock.advance(10.0) + self.assertIsNone(service.dispatch("trim", "signal:35")) + self.assertEqual(calls, [0]) + self.assertEqual(service.suppressed["trim"], 2) + self.assertTrue(any("malloc_trim suppressed" in line for line in lines)) + # Past the interval the next request runs. + clock.advance(60.0) + self.assertIsNotNone(service.dispatch("trim", "periodic")) + self.assertEqual(calls, [0, 0]) + self.assertEqual(trimmer.calls, 2) + finally: + service.close() + # The periodic action is paced by its own interval through the + # same gate, and is not due before it. + clock = _FakeClock() + periodic = HeapCensusService( + _config(enabled=False, min_interval_seconds=0.0, malloc_trim_interval_seconds=120.0), + trimmer=MallocTrimmer( + SimpleNamespace(sample=lambda: _sample()), # type: ignore[arg-type] + resolver=lambda: fake_trim, + clock=clock, + rss_reader=lambda: 1, + log=lambda _line: None, + ), + clock=clock, + log=lambda _line: None, + ) + try: + calls.clear() + self.assertFalse(periodic.periodic_trim_due()) + self.assertTrue(periodic.run_once(0.0)) + self.assertEqual(calls, []) + clock.advance(120.0) + self.assertTrue(periodic.periodic_trim_due()) + self.assertTrue(periodic.run_once(0.0)) + self.assertEqual(calls, [0]) + self.assertFalse(periodic.periodic_trim_due()) + finally: + periodic.close() + # An absent symbol is remembered: one log line, no call, no retry. + absent_lines: list[str] = [] + absent = MallocTrimmer( + SimpleNamespace(sample=lambda: _sample()), # type: ignore[arg-type] + resolver=mock.Mock(side_effect=AttributeError("malloc_trim: symbol not found")), + log=absent_lines.append, + ) + self.assertIsNone(absent.trim_once("a")) + self.assertIsNone(absent.trim_once("b")) + self.assertEqual(absent.calls, 0) + self.assertEqual(len(absent_lines), 1) + + def test_real_malloc_trim_releases_free_bytes_and_keeps_in_use(self) -> None: + """On glibc the real call frees retained free chunks and touches nothing live.""" + telemetry = ProcessHeapTelemetry() + if not telemetry.sample().malloc_info_available: + self.skipTest("mallinfo2 unavailable on this platform") + # Fragment the main arena a little: allocate and free medium chunks + # that stay under the mmap threshold, so glibc keeps them. + junk = [bytes([index % 251]) * (48 * 1024) for index in range(512)] + held = junk[::2] + del junk + result = MallocTrimmer(telemetry, log=lambda _line: None).trim_once("test") + self.assertIsNotNone(result) + assert result is not None + self.assertIsInstance(result.released, bool) + # Reading RSS and sampling mallinfo2 allocate Python/ctypes objects + # between the two samples. Their exact net allocation varies across + # Python builds, even though malloc_trim preserves every live chunk. + # Check the live allocation and its contents instead of requiring the + # sampling machinery itself to have a zero-byte delta. + live_bytes = sum(map(len, held)) + self.assertGreaterEqual(result.before.malloc_in_use_bytes, live_bytes) + self.assertGreaterEqual(result.after.malloc_in_use_bytes, live_bytes) + for index, chunk in enumerate(held): + self.assertEqual(chunk, bytes([(index * 2) % 251]) * (48 * 1024)) + self.assertLessEqual(result.free_delta, 0) + self.assertLessEqual(result.arena_delta, 0) + self.assertGreaterEqual(result.seconds, 0.0) + del held + + +class AllocatorEnvironmentPinTests(unittest.TestCase): + def test_allocator_environment_defaults_are_pinned(self) -> None: + """The image sets exactly MALLOC_ARENA_MAX, at the decided value, overridably.""" + dockerfile = DOCKERFILE.read_text(encoding="utf-8") + args = dict(re.findall(r"^ARG\s+([A-Z0-9_]+)=(\S*)\s*$", dockerfile, re.MULTILINE)) + envs = dict(re.findall(r"^ENV\s+([A-Z0-9_]+)=(\S*)\s*$", dockerfile, re.MULTILINE)) + self.assertEqual(args.get("MALLOC_ARENA_MAX"), PINNED_MALLOC_ARENA_MAX) + self.assertEqual(envs.get("MALLOC_ARENA_MAX"), "${MALLOC_ARENA_MAX}") + allocator_envs = { + name for name in envs + if name.startswith("MALLOC_") or name in ("GLIBC_TUNABLES", "PYTHONMALLOC") + } + self.assertEqual(allocator_envs, {"MALLOC_ARENA_MAX"}) + self.assertNotIn("PYTHONMALLOC", dockerfile.replace("# PYTHONMALLOC is deliberately unset", "")) + # Never an empty allocator value: glibc reads "" as 0. + for name, value in {**args, **envs}.items(): + if name.startswith("MALLOC_"): + self.assertNotEqual(value, "", name) + # The ARG precedes the ENV that expands it, and both precede CMD. + self.assertLess(dockerfile.index("ARG MALLOC_ARENA_MAX="), dockerfile.index("ENV MALLOC_ARENA_MAX=")) + self.assertLess(dockerfile.index("ENV MALLOC_ARENA_MAX="), dockerfile.index("CMD [")) + # The deployment notes record the same decision. + deployment = (ROOT / "docs" / "mainnet-deployment.md").read_text(encoding="utf-8") + self.assertIn(f"MALLOC_ARENA_MAX={PINNED_MALLOC_ARENA_MAX}", deployment) + # A dotenv value reaches the container only through the compose + # passthrough, whose default must equal the image default so the + # two cannot drift, and must never be empty (glibc reads "" as 0). + environment = _coordinator_service_environment() + self.assertEqual( + environment.get("MALLOC_ARENA_MAX"), + f"${{MALLOC_ARENA_MAX:-{PINNED_MALLOC_ARENA_MAX}}}", + ) + self.assertEqual(_env_example_values().get("MALLOC_ARENA_MAX"), PINNED_MALLOC_ARENA_MAX) + # glibc honours the variable: a child started with it capped cannot + # grow past the cap however many threads allocate. This is the + # mechanism the soak relies on, proven on this host. + probe = ( + "import ctypes, threading, sys\n" + "sys.path.insert(0, %r)\n" + "from lab.prism.process_telemetry import read_malloc_arena_count\n" + "barrier = threading.Barrier(16)\n" + "def work():\n" + " keep = [bytes(8192) for _ in range(32)]\n" + " barrier.wait()\n" + " del keep\n" + "threads = [threading.Thread(target=work) for _ in range(16)]\n" + "[t.start() for t in threads]; [t.join() for t in threads]\n" + "print(read_malloc_arena_count())\n" + ) % str(ROOT) + if read_malloc_arena_count() < 0: + self.skipTest("malloc_info unavailable on this platform") + capped = subprocess.run( + [sys.executable, "-c", probe], + env={**os.environ, "MALLOC_ARENA_MAX": PINNED_MALLOC_ARENA_MAX}, + capture_output=True, + text=True, + check=True, + ) + uncapped_env = {k: v for k, v in os.environ.items() if k != "MALLOC_ARENA_MAX"} + uncapped = subprocess.run( + [sys.executable, "-c", probe], + env=uncapped_env, + capture_output=True, + text=True, + check=True, + ) + self.assertLessEqual(int(capped.stdout.strip()), int(PINNED_MALLOC_ARENA_MAX)) + self.assertGreater(int(uncapped.stdout.strip()), int(PINNED_MALLOC_ARENA_MAX)) + + + def test_census_switches_pass_through_compose_with_the_shipped_defaults(self) -> None: + """Every switch reaches the coordinator container, defaulting to what the config ships.""" + environment = _coordinator_service_environment() + example = _env_example_values() + expected_defaults = { + "PRISM_HEAP_CENSUS": "0", + "PRISM_HEAP_CENSUS_TOP_N": str(DEFAULT_PRISM_HEAP_CENSUS_TOP_N), + "PRISM_HEAP_CENSUS_MAX_BYTES": str(DEFAULT_PRISM_HEAP_CENSUS_MAX_BYTES), + "PRISM_HEAP_CENSUS_MAX_SECONDS": f"{DEFAULT_PRISM_HEAP_CENSUS_MAX_SECONDS:g}", + "PRISM_HEAP_CENSUS_MIN_INTERVAL_SECONDS": f"{DEFAULT_PRISM_HEAP_CENSUS_MIN_INTERVAL_SECONDS:g}", + "PRISM_HEAP_CENSUS_TRACEMALLOC": "0", + "PRISM_MALLOC_TRIM_SIGNAL": "0", + "PRISM_MALLOC_TRIM_INTERVAL_SECONDS": "0", + } + for name, default in expected_defaults.items(): + with self.subTest(name=name): + self.assertEqual(environment.get(name), f"${{{name}:-{default}}}") + self.assertEqual(example.get(name), default) + # The directory is optional: an empty passthrough must reach the + # coordinator as unset, and the reader must then use the default. + self.assertEqual(environment.get("PRISM_HEAP_CENSUS_DIR"), "${PRISM_HEAP_CENSUS_DIR:-}") + config = env_heap_census_config( + environ={"PRISM_HEAP_CENSUS_DIR": "", "PRISM_AUDIT_DIR": "/var/lib/qbit-prism/audit"} + ) + self.assertEqual(config.output_dir, "/var/lib/qbit-prism/audit/heap-census") + # The defaults the passthroughs carry are the ones the loader + # produces from an empty environment. + shipped = env_heap_census_config(environ={}) + passthrough_environ = { + name: default for name, default in expected_defaults.items() + } + passthrough_environ["PRISM_HEAP_CENSUS_DIR"] = "" + self.assertEqual(env_heap_census_config(environ=passthrough_environ), shipped) + + +class RssBoundCheckTests(unittest.TestCase): + def test_rss_bound_check_fails_when_the_stated_multiple_is_exceeded(self) -> None: + """Synthetic series: the #226 slope fails, a flat post-warm-up series passes.""" + mib = 1024 * 1024 + hour = 3600.0 + # The mainnet shape: 140 MiB at start, +390 MB every hour, for 24 h. + leaking = [(h * hour, int(140 * mib + h * 390_000_000)) for h in range(25)] + verdict = evaluate_rss_bound(leaking, warmup_seconds=hour, multiple=2.0) + self.assertFalse(verdict.passed) + # The baseline is the warm-up peak (the one-hour sample), so the + # bound is twice that and the third-hour sample is the first over it. + self.assertEqual(verdict.baseline_bytes, int(140 * mib + 390_000_000)) + self.assertEqual(verdict.first_breach_at_seconds, 3 * hour) + self.assertGreater(verdict.peak_ratio, 2.0) + self.assertGreater(verdict.slope_bytes_per_hour or 0, 380_000_000) + self.assertTrue(any("exceeded 2x" in reason for reason in verdict.reasons)) + # Bounded: warm-up to 400 MiB (the window materializes), then a + # storm to 613 MiB that drains, then flat. Under 2x of 400 MiB. + bounded = [(0.0, 140 * mib), (0.5 * hour, 400 * mib), (hour, 390 * mib)] + bounded += [(h * hour, 613 * mib if h == 3 else 400 * mib + (h % 3) * mib) for h in range(2, 25)] + verdict = evaluate_rss_bound(bounded, warmup_seconds=hour, multiple=2.0) + self.assertTrue(verdict.passed, verdict.reasons) + self.assertEqual(verdict.baseline_bytes, 400 * mib) + self.assertEqual(verdict.bound_bytes, 800 * mib) + self.assertEqual(verdict.peak_bytes, 613 * mib) + self.assertIsNone(verdict.first_breach_at_seconds) + # The same series fails a tighter multiple, at the storm. + verdict = evaluate_rss_bound(bounded, warmup_seconds=hour, multiple=1.5) + self.assertFalse(verdict.passed) + self.assertEqual(verdict.first_breach_at_seconds, 3 * hour) + # Absolute epoch timestamps are re-based; -1 samples are ignored. + epoch = 1_788_000_000.0 + shifted = [(epoch + t, r) for t, r in bounded] + [(epoch + 5 * hour, -1)] + self.assertTrue(evaluate_rss_bound(shifted, warmup_seconds=hour, multiple=2.0).passed) + # Too short a series, or one with nothing after warm-up, is not a pass. + short = evaluate_rss_bound(bounded[:3], warmup_seconds=hour, multiple=2.0) + self.assertFalse(short.passed) + self.assertIn("no sample after the warm-up window", short.reasons) + brief = evaluate_rss_bound(bounded, warmup_seconds=hour, multiple=2.0, min_span_seconds=48 * hour) + self.assertFalse(brief.passed) + self.assertTrue(any("spans" in reason for reason in brief.reasons)) + self.assertFalse(evaluate_rss_bound([], warmup_seconds=hour, multiple=2.0).passed) + # The optional slope guard catches a slow leak that stays under 2x + # (15 MiB/h from a 420 MiB warm-up peak ends the day at 760 MiB). + slow = [(h * hour, int(400 * mib + h * 15 * mib)) for h in range(25)] + self.assertTrue(evaluate_rss_bound(slow, warmup_seconds=hour, multiple=2.0).passed) + guarded = evaluate_rss_bound( + slow, warmup_seconds=hour, multiple=2.0, max_slope_bytes_per_hour=10 * mib + ) + self.assertFalse(guarded.passed) + self.assertTrue(any("slope" in reason for reason in guarded.reasons)) + with self.assertRaises(ValueError): + evaluate_rss_bound(bounded, warmup_seconds=hour, multiple=1.0) + + def test_rss_bound_cli_exit_status_is_the_verdict(self) -> None: + """The operator's command exits 0 on pass, 1 on fail, 2 on bad input.""" + mib = 1024 * 1024 + with tempfile.TemporaryDirectory() as directory: + passing = Path(directory) / "pass.csv" + passing.write_text( + "# epoch_seconds,rss_bytes\n" + + "".join(f"{h * 3600},{400 * mib}\n" for h in range(25)), + encoding="utf-8", + ) + failing = Path(directory) / "fail.csv" + failing.write_text( + "".join(f"{h * 3600},{140 * mib + h * 390_000_000}\n" for h in range(25)), + encoding="utf-8", + ) + broken = Path(directory) / "broken.csv" + broken.write_text("not,a,number\n", encoding="utf-8") + base = [sys.executable, "-m", "lab.prism.process_telemetry", "rss-bound"] + ok = subprocess.run( + [*base, "--samples", str(passing)], + cwd=ROOT, + capture_output=True, + text=True, + ) + self.assertEqual(ok.returncode, 0, ok.stderr) + self.assertTrue(json.loads(ok.stdout)["passed"]) + bad = subprocess.run( + [*base, "--samples", str(failing)], + cwd=ROOT, + capture_output=True, + text=True, + ) + self.assertEqual(bad.returncode, 1, bad.stderr) + self.assertFalse(json.loads(bad.stdout)["passed"]) + self.assertEqual(json.loads(bad.stdout)["first_breach_at_seconds"], 10800.0) + usage = subprocess.run( + [*base, "--samples", str(broken)], + cwd=ROOT, + capture_output=True, + text=True, + ) + self.assertEqual(usage.returncode, 2) + sample = subprocess.run( + [sys.executable, "-m", "lab.prism.process_telemetry", "rss-sample", "--pid", str(os.getpid())], + cwd=ROOT, + capture_output=True, + text=True, + ) + epoch, rss = sample.stdout.strip().split(",") + self.assertGreater(int(epoch), 0) + if _HAS_PROCFS: + self.assertEqual(sample.returncode, 0, sample.stderr) + self.assertGreater(int(rss), 0) + else: + # Documented contract off Linux: the -1 sentinel and exit 1, + # so a capture loop cannot mistake "unavailable" for a value. + self.assertEqual(sample.returncode, 1, sample.stderr) + self.assertEqual(int(rss), -1) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_prism_hot_path.py b/tests/test_prism_hot_path.py index 92ae853cc..3e4b369e6 100644 --- a/tests/test_prism_hot_path.py +++ b/tests/test_prism_hot_path.py @@ -33,8 +33,8 @@ StratumError, TemplateRefreshBlocked, WorkerIdentity, - default_prism_coinbase_tag_hex, ) +from lab.prism.coordinator_config import default_prism_coinbase_tag_hex from lab.prism.share_ledger import AcceptedShareRecord, PendingShare PAYOUT_ADDRESS = "tq1z70ukpvs96kye6jmgvl3nttevtkrq8uu89snkpm6m8gwqukw8u5dsz32kwa" @@ -377,6 +377,9 @@ def fake_build_audit_bundle(**kwargs: object) -> dict[str, object]: } server.build_audit_bundle = fake_build_audit_bundle # type: ignore[method-assign] + server._ensure_bundle_compiler().build_audit_bundle = ( # type: ignore[method-assign] + fake_build_audit_bundle + ) return recorded @@ -647,6 +650,7 @@ def test_per_share_and_per_job_prints_enabled_by_flag(self) -> None: class HotPathLockIsolationTests(unittest.TestCase): def test_first_touch_installs_one_hot_path_lock_set(self) -> None: server = PrismCoordinator.__new__(PrismCoordinator) + server.extranonce2_size = EXTRANONCE2_SIZE state = ClientState.__new__(ClientState) start = threading.Barrier(32) @@ -656,7 +660,7 @@ def initialize(_index: int) -> tuple[object, object, object, bool]: reserved = server._reserve_recent_share_key(("same", "header")) return ( vardiff_lock, - server._recent_share_lock, + server._ensure_share_submission_service().recent_shares, server._share_accounting_lock, reserved, ) diff --git a/tests/test_prism_immutable_refresh_artifacts.py b/tests/test_prism_immutable_refresh_artifacts.py index 8ad9c82ab..f6af607b3 100644 --- a/tests/test_prism_immutable_refresh_artifacts.py +++ b/tests/test_prism_immutable_refresh_artifacts.py @@ -15,7 +15,7 @@ WorkerIdentity, qbit_template_fingerprint, ) -from tests.test_prism_coordinator_job_cache import ( +from tests.prism_coordinator_test_support import ( base_template, client, coordinator, diff --git a/tests/test_prism_incremental_payout_window.py b/tests/test_prism_incremental_payout_window.py index 62692a994..e468c9199 100644 --- a/tests/test_prism_incremental_payout_window.py +++ b/tests/test_prism_incremental_payout_window.py @@ -11,10 +11,20 @@ from typing import Iterable, Sequence from unittest.mock import patch +from lab.prism.share_json_stream import ( + ShareArrayJsonSequence, + canonical_share_array_sha256, + canonical_share_items_bytes, + iter_json_array_text_chunks, + iter_json_object_text_chunks, + release_share_list_incrementally, + share_array_json_view, +) from lab.prism.share_ledger import ( AcceptedShareRecord, IncrementalShareWindow, IncrementalWindowFallback, + _IncrementalShareWindowPage, ) @@ -589,5 +599,224 @@ def counting_to_prism_json( ) +def awkward_share_json(seq: int) -> dict[str, object]: + """A prism-JSON share that exercises every encoder edge at once. + + Non-ASCII and non-BMP text (surrogate-pair escapes), JSON-significant + punctuation inside strings, integers past 64 and 128 bits, and the + optional ``credit_policy`` member both present and absent. + """ + payload: dict[str, object] = { + "share_seq": seq, + "share_id": f"miner-\u00e9.rig{seq}:\U0001F600,}}{{\"\\", + "miner_id": f"miner-{seq % 5}", + "order_key": f"{seq % 5:02d}:miner-{seq % 5}", + "p2mr_program_hex": f"{seq % 256:02x}" * 32, + "share_difficulty": (1 << 130) + seq, + "network_difficulty": (1 << 70) + 7, + "template_height": 800_000 + seq, + "job_id": f"job-\u2603-{seq}", + "job_issued_at_ms": 1_700_000_000_000 + seq, + "accepted_at_ms": 1_700_000_000_100 + seq, + "ntime": 1_700_000_000 + seq, + } + if seq % 3 == 0: + payload["credit_policy"] = "template-\u00fc" + return payload + + +class BoundedShareJsonEncodingTests(unittest.TestCase): + """The batched encoders reproduce ``json.dumps`` byte for byte (#236). + + Every helper in ``share_json_stream`` exists so that no single C call + covers a whole 200k-share window; the contract that makes that safe is + that the concatenated output is identical to the historical one-call + encoding, whatever batch and chunk sizes are in force. + """ + + BATCH_SIZES = (1, 2, 3, 7, 512) + CHUNK_SIZES = (1, 16, 64 * 1024) + + def _shares(self, count: int) -> list[dict[str, object]]: + return [awkward_share_json(seq) for seq in range(1, count + 1)] + + def test_array_chunks_match_json_dumps_for_every_batch_and_chunk_size( + self, + ) -> None: + for count in (0, 1, 2, 13, 1_100): + shares = self._shares(count) + canonical = json.dumps( + shares, sort_keys=True, separators=(",", ":"), default=str + )[1:-1] + compact = json.dumps(shares, separators=(",", ":"))[1:-1] + for batch in self.BATCH_SIZES: + for chunk in self.CHUNK_SIZES: + with self.subTest(count=count, batch=batch, chunk=chunk): + canonical_chunks = list( + iter_json_array_text_chunks( + shares, + sort_keys=True, + default=str, + batch_records=batch, + chunk_chars=chunk, + ) + ) + self.assertEqual("".join(canonical_chunks), canonical) + self.assertEqual( + "".join( + iter_json_array_text_chunks( + shares, + batch_records=batch, + chunk_chars=chunk, + ) + ), + compact, + ) + # No chunk splits a record: every chunk is a run of + # whole records, so it parses once re-framed. + for piece in canonical_chunks: + json.loads("[" + piece.strip(",") + "]") + self.assertTrue( + all(piece.isascii() for piece in canonical_chunks) + ) + + def test_object_chunks_match_json_dumps(self) -> None: + shares = self._shares(700) + fields = { + "request": "prepare_window", + "mode": "full", + "anchor_job_issued_at_ms": 1_700_000_000_000, + "records": shares, + "window_weight": 1 << 100, + "empty": [], + "pair": (1, "two"), + "nested": {"z": 1, "a": [1.5, None, True]}, + } + for batch in (1, 512): + for chunk in self.CHUNK_SIZES: + with self.subTest(batch=batch, chunk=chunk): + self.assertEqual( + "".join( + iter_json_object_text_chunks( + fields, + array_keys=("records", "empty", "pair"), + batch_records=batch, + chunk_chars=chunk, + ) + ), + json.dumps(fields, separators=(",", ":")), + ) + self.assertEqual( + "".join( + iter_json_object_text_chunks( + fields, + array_keys=("records",), + sort_keys=True, + default=str, + batch_records=batch, + chunk_chars=chunk, + ) + ), + json.dumps( + fields, + sort_keys=True, + separators=(",", ":"), + default=str, + ), + ) + # A non-string key defers to json.dumps' own coercion rules. + mixed = {"a": 1, 2: [1, 2]} + self.assertEqual( + "".join(iter_json_object_text_chunks(mixed, array_keys=(2,))), + json.dumps(mixed, separators=(",", ":")), + ) + + def test_page_canonical_items_match_per_record_encoding(self) -> None: + records = tuple( + accepted_share( + seq, + share_difficulty=(1 << 129) + seq, + job_issued_at_ms=1_000 + seq, + accepted_at_ms=1_000 + seq, + credit_policy="unicode-\u00e9-\U0001F600" if seq % 2 else None, + ) + for seq in range(1, 600) + ) + page = _IncrementalShareWindowPage.from_records(records) + expected_items = b",".join( + canonical_json_text(record.to_prism_json()).encode() + for record in records + ) + self.assertEqual(page.canonical_json_items, expected_items) + self.assertEqual( + canonical_share_items_bytes(page.prism_json_records), + expected_items, + ) + window = IncrementalShareWindow.from_full_snapshot( + records, + anchor_job_issued_at_ms=10_000, + window_weight=sum(int(r.share_difficulty) for r in records), + page_size=128, + ) + self.assertEqual( + window.json_records().canonical_json_sha256(), + canonical_json_sha256( + tuple(record.to_prism_json() for record in records) + ), + ) + self.assertEqual( + _IncrementalShareWindowPage.from_records(()).canonical_json_items, + b"", + ) + + def test_incremental_release_empties_lists_in_bounded_slices(self) -> None: + values = list(range(10_001)) + release_share_list_incrementally(values, batch_records=4_000) + self.assertEqual(values, []) + # Non-lists (tuples, the paged view) are left to ordinary GC. + frozen = (1, 2, 3) + release_share_list_incrementally(frozen) + self.assertEqual(frozen, (1, 2, 3)) + # A batch size below one is clamped rather than looping forever. + values = [1, 2] + release_share_list_incrementally(values, batch_records=0) + self.assertEqual(values, []) + + def test_share_array_view_digests_like_the_whole_array(self) -> None: + shares = self._shares(1_500) + expected = canonical_json_sha256(shares) + self.assertEqual(canonical_share_array_sha256(shares), expected) + self.assertEqual(canonical_share_array_sha256(tuple(shares)), expected) + self.assertEqual(canonical_share_array_sha256([]), canonical_json_sha256([])) + view = share_array_json_view(shares) + self.assertIsInstance(view, ShareArrayJsonSequence) + self.assertEqual(view.canonical_json_sha256(), expected) + self.assertEqual(len(view), len(shares)) + self.assertEqual(list(view), shares) + self.assertIs(view[3], shares[3]) + self.assertEqual(view[-2:], shares[-2:]) + # A sequence that already carries its digest is handed back as is, + # and the streaming digest defers to it. + window = IncrementalShareWindow.from_full_snapshot( + [ + accepted_share( + seq, + share_difficulty=1, + job_issued_at_ms=seq, + accepted_at_ms=seq, + ) + for seq in range(1, 4) + ], + anchor_job_issued_at_ms=10, + window_weight=3, + ) + paged = window.json_records() + self.assertIs(share_array_json_view(paged), paged) + self.assertEqual( + canonical_share_array_sha256(paged), + paged.canonical_json_sha256(), + ) + + if __name__ == "__main__": unittest.main() diff --git a/tests/test_prism_initial_job_delivery.py b/tests/test_prism_initial_job_delivery.py index 1493d57ae..c4f120b6d 100644 --- a/tests/test_prism_initial_job_delivery.py +++ b/tests/test_prism_initial_job_delivery.py @@ -6,13 +6,15 @@ import threading import time import unittest +from unittest.mock import patch +from lab.prism.payout_state import AcceptedParentPayoutPreviewPending from lab.prism.prism_coordinator import ( PendingInitialJob, PRISM_JOB_EXTRANONCE1_PLACEHOLDER_HEX, TemplateRefreshBlocked, ) -from tests.test_prism_coordinator_job_cache import ( +from tests.prism_coordinator_test_support import ( EXTRANONCE2_SIZE, ObservedRLock, base_template, @@ -102,7 +104,7 @@ def test_initial_delivery_vardiff_wait_does_not_hold_coordinator_lock(self) -> N connection_id=state.connection_id, difficulty_generation=0, ) - server.pending_initial_jobs[state] = request + server.pending_initial_jobs = {state: request} results: list[bool | None] = [] with vardiff_lock: @@ -155,7 +157,7 @@ def test_initial_delivery_rechecks_append_epoch_after_vardiff_wait( connection_id=state.connection_id, difficulty_generation=0, ) - server.pending_initial_jobs[state] = request + server.pending_initial_jobs = {state: request} results: list[bool | None] = [] with vardiff_lock: @@ -211,7 +213,7 @@ def test_initial_delivery_rechecks_published_digest_after_vardiff_wait( connection_id=state.connection_id, difficulty_generation=0, ) - server.pending_initial_jobs[state] = request + server.pending_initial_jobs = {state: request} results: list[bool | None] = [] with vardiff_lock: @@ -298,6 +300,62 @@ def transient_artifacts() -> object: 0, ) + def test_initial_delivery_retries_preview_pending_without_counting_it( + self, + ) -> None: + # Boundary 1 (#182): the initial-job loop coalesces accepted-parent + # preview backpressure onto its ordinary bounded retry. The client + # stays connected, the job still lands, and the benign wait never + # reaches the generic job_build_failure_count accounting reserved for + # real build failures. + server, _rpc = coordinator() + install_fake_bundle_builder(server) + server.prewarm_current_tip_ready_bundle() + state = client(1) + state.authorization_generation = 1 + state.authorized_monotonic = time.monotonic() + state.send = lambda _payload: None # type: ignore[method-assign] + server.clients = {state} + + bundle_attempts = 0 + original_shared_job_bundle = server.shared_job_bundle + + def blocked_then_ready(*args: object, **kwargs: object) -> object: + nonlocal bundle_attempts + bundle_attempts += 1 + if bundle_attempts == 1: + raise AcceptedParentPayoutPreviewPending( + "accepted parent payout preview is not ready yet", + parent_hash="cc" * 32, + waited_seconds=0.25, + timeout_count=1, + ) + return original_shared_job_bundle(*args, **kwargs) # type: ignore[operator] + + server.shared_job_bundle = blocked_then_ready # type: ignore[method-assign] + + with patch("builtins.print") as logged: + try: + server.request_initial_job_delivery(state) + wait_until(lambda: state.active_job is not None) + finally: + server.shutdown_tip_refresh_executor() + log_lines = [ + str(call.args[0]) for call in logged.call_args_list if call.args + ] + + self.assertEqual(bundle_attempts, 2) + # Retried, not counted: the benign wait stays out of the generic + # build-failure budget. + self.assertEqual(server.job_build_failure_count, 0) + self.assertIn(state, server.clients) + self.assertIsNotNone(state.active_job) + # The retry carries an explicit reason instead of a failure traceback. + self.assertTrue( + any("accepted_parent_preview_pending" in line for line in log_lines), + log_lines, + ) + def test_initial_delivery_backs_off_after_superseded_work(self) -> None: server, rpc = coordinator() install_fake_bundle_builder(server) @@ -326,7 +384,7 @@ def test_initial_delivery_backs_off_after_superseded_work(self) -> None: connection_id=state.connection_id, difficulty_generation=0, ) - server.pending_initial_jobs[state] = request + server.pending_initial_jobs = {state: request} artifacts = server.current_template_artifacts() waits: list[float] = [] request.cancelled.wait = ( # type: ignore[method-assign] @@ -377,7 +435,7 @@ def test_initial_delivery_keeps_authoritative_published_work_on_tip_churn(self) connection_id=state.connection_id, difficulty_generation=0, ) - server.pending_initial_jobs[state] = request + server.pending_initial_jobs = {state: request} artifacts = server.current_template_artifacts() waits: list[float] = [] request.cancelled.wait = ( # type: ignore[method-assign] @@ -615,7 +673,7 @@ def blocked_build(*args: object, **kwargs: object) -> object: self.assertNotEqual(state.active_job.payout_state_generation, 0) def test_collection_mode_initial_bundles_remain_identity_specific(self) -> None: - from tests.test_prism_coordinator_job_cache import FakeLedger + from tests.prism_coordinator_test_support import FakeLedger server, _rpc = coordinator(ledger=FakeLedger(miners=["solo"])) install_fake_bundle_builder(server) @@ -726,7 +784,7 @@ def test_health_turns_non_green_after_stalled_delivery_deadline(self) -> None: self.assertEqual(payload["clients_with_current_tip_job"], 0) server.clients.clear() - server._health_snapshot = None + server._ensure_observability_service().clear_health_snapshot_for_test() status, payload = server.cached_health_payload() self.assertEqual(status, 200) self.assertTrue(payload["ok"]) @@ -769,7 +827,9 @@ def test_health_flags_sustained_eighty_seven_percent_coverage_loss(self) -> None server.started_monotonic = old server.mining_health_startup_grace_seconds = 5 server.stratum_initial_job_timeout_seconds = 5 - server._mining_delivery_failure_started_monotonic = time.monotonic() - 5 + server._ensure_observability_service().set_delivery_failure_started_monotonic_for_test( + time.monotonic() - 5 + ) status, payload = server.cached_health_payload() diff --git a/tests/test_prism_issue_224_cadence.py b/tests/test_prism_issue_224_cadence.py new file mode 100644 index 000000000..e6b30182c --- /dev/null +++ b/tests/test_prism_issue_224_cadence.py @@ -0,0 +1,596 @@ +"""Issue #224 Wave 2: a dense reconcile cadence over one exact share window. + +Wave 1 taught a confirmed payout mutation to ask candidate preparation for +a live prior-balances reread instead of the O(window) oracle rescan, on the +invariant that a reconcile pass never writes ``qbit_share_ledger`` +(recorded in ``lab.prism.accepted_preview_telemetry``). Its tests pin one +mutation at a time. The alert hour was a dense accepted-tip cadence, so +this module drives several consecutive trusted passes over one fixed +accepted-share window through the real coordinator adapters while payout +state moves through the reconciliation-owned mutations the ledger actually +performs -- a maturation, an inactive marking, a reactivation -- interleaved +with passes that change nothing and with the post-confirm and landing entry +points. + +Per pass it records the intent handed to candidate preparation, the ledger +artifact that came back, the balances that were published and the ledger +reads that were paid for, and pins them against each other and against the +fixed-cardinality telemetry: a mutating pass rereads balances over the +armed window and never rescans it, a non-mutating pass invents neither, and +a lost mutator response mid-cadence still fails closed to the full rescan. +Everything runs on a frozen anchor clock; nothing here sleeps or compares +elapsed time. +""" + +from __future__ import annotations + +import re +import unittest +from collections import Counter +from typing import Callable +from unittest.mock import patch + +from lab.prism.accepted_preview_telemetry import ( + PRISM_ACCEPTED_LANDING_PHASES, + PRISM_LEDGER_READ_OPERATIONS, + PRISM_PAYOUT_WINDOW_FULL_RESCAN_PATHS, + PRISM_PAYOUT_WINDOW_FULL_RESCAN_REASONS, + PRISM_REORG_RECONCILE_CALLERS, + PRISM_REORG_RECONCILE_STEPS, + ensure_accepted_preview_telemetry, + fold_ledger_read_stats, +) +from lab.prism.metrics import MetricsRenderer +from lab.prism.share_ledger import PsqlShareLedger +from tests.test_prism_issue_224_core import ( + BALANCE_REREAD_INTENT, + FULL_RESCAN_INTENT, + PLAIN_INTENT, + CarryLedger, + QuietCoordinatorTestCase, + configured_coordinator, + full_rescan_total, + pass_counts, + step_count, +) + + +# One confirmed pool block below the fake node's tip (height 100). The +# cadence flips the active chain at its height between its own hash and a +# competitor to drive the inactive and reactivated transitions. +POOL_BLOCK_HASH = "ab" * 32 +POOL_BLOCK_HEIGHT = 90 +COMPETING_BLOCK_HASH = "cd" * 32 +FIXED_NOW_MS = 1_000_000 + +# The carry aggregate after each mutation. The values are arbitrary but +# pairwise distinct, so a publication that reused an earlier generation's +# balances cannot equal the live read by coincidence. ``CarryLedger`` owns +# the initial 546 and the post-maturation 777. +INITIAL_BALANCE = 546 +MATURED_BALANCE = 777 +INACTIVE_BALANCE = 101 +REACTIVATED_BALANCE = 909 +LOST_RESPONSE_BALANCE = 4321 + + +class CadenceLedger(CarryLedger): + """A carry ledger whose pool-block state feeds the watch loop. + + ``SingleWriterShareLedger`` already models pool-block chain state (the + prepared/confirmed/inactive transitions with their audit ordinals); the + memory backend just never exposes those rows to ``reorg_watch_blocks``. + This fixture exposes them, and moves the carry aggregate on every + mutation that reports a row changed, the way the PostgreSQL functions + move ``qbit_payout_carry_forward`` -- and, exactly as in PostgreSQL, + never touches an accepted share. + + The prior-balances read is attributed with the recorder borrowed from + ``PsqlShareLedger``, so the operation name, the cell shape and the + rendering are the shipped ones; the fixture supplies only the trigger. + """ + + _ensure_ledger_read_timings = PsqlShareLedger._ensure_ledger_read_timings + _note_ledger_read_timing = PsqlShareLedger._note_ledger_read_timing + ledger_read_gate_stats = PsqlShareLedger.ledger_read_gate_stats + + def __init__(self) -> None: + super().__init__() + self.lose_next_maturation_response = False + + def current_prior_balances(self) -> list[dict[str, object]]: + balances = super().current_prior_balances() + self._note_ledger_read_timing( + "current_prior_balances", + gate_wait_seconds=0.0, + execute_seconds=0.0, + timed_out=False, + ) + return balances + + def reorg_watch_blocks(self, *, active_tip_height: int) -> list[dict[str, object]]: + with self._lock: + return [ + { + "block_hash": block_hash, + "block_height": block[0], + "chain_state": block[1], + } + for block_hash, block in self._memory_pool_blocks.items() + if block[1] in {"confirmed", "inactive"} + ] + + def mark_pool_block_inactive(self, **kwargs: object) -> dict[str, int | str]: + result = super().mark_pool_block_inactive(**kwargs) # type: ignore[arg-type] + if result["inactive_count"]: + self.balance_sats = INACTIVE_BALANCE + return result + + def reactivate_pool_block(self, **kwargs: object) -> dict[str, int | str]: + result = super().reactivate_pool_block(**kwargs) # type: ignore[arg-type] + if result["reactivated_count"]: + self.balance_sats = REACTIVATED_BALANCE + return result + + def mark_mature_pool_payouts(self, *, active_tip_height: int) -> dict[str, int | str]: + if self.lose_next_maturation_response: + # The statement committed server-side; only its response was + # lost, so the aggregate moved and the caller cannot know it. + self.lose_next_maturation_response = False + self.matured_pending = 0 + self.balance_sats = LOST_RESPONSE_BALANCE + raise ConnectionError("mutation response lost") + return super().mark_mature_pool_payouts(active_tip_height=active_tip_height) + + +class CandidateRecorder: + """Every candidate preparation: the intent asked for, the artifact built. + + The reconciler reaches ``_prepared_payout_state_candidate`` through a + port resolved at call time, so a per-instance wrapper sees exactly the + kwargs it forwarded and the real candidate the coordinator prepared. + """ + + def __init__(self, server) -> None: + self.records: list[tuple[dict[str, bool], object]] = [] + original = server._prepared_payout_state_candidate + + def prepared(captured, **kwargs): + candidate = original(captured, **kwargs) + self.records.append((dict(kwargs), candidate.ledger_artifact)) + return candidate + + server._prepared_payout_state_candidate = prepared + + +def cadence_coordinator(): + """A reconciling coordinator over one confirmed pool block. + + The debounce decision compares ``time.monotonic()`` against the minimum + build interval; it is held far above any test's wall-clock so the + cadence rides the armed window by construction rather than by racing an + elapsed-time threshold. + """ + ledger = CadenceLedger() + server, rpc, _ledger, _artifacts = configured_coordinator(ledger) + server.reorg_reconciler_enabled = True + server.reorg_reconcile_cache_seconds = 0.0 + server.payout_artifact_min_build_interval_seconds = 3_600.0 + active_chain = {POOL_BLOCK_HEIGHT: POOL_BLOCK_HASH} + original_call = rpc.call + + def call(method: str, params: list[object] | None = None) -> object: + if method == "getblockhash": + rpc.calls.append(method) + return active_chain[int(params[0])] # type: ignore[index] + return original_call(method, params) + + rpc.call = call # type: ignore[method-assign] + ledger.persist_accepted_block( + block_hash=POOL_BLOCK_HASH, + block_height=POOL_BLOCK_HEIGHT, + parent_hash="11" * 32, + final_bundle={}, + audit_report={}, + ) + ledger.confirm_accepted_block( + block_hash=POOL_BLOCK_HASH, + active_tip_height=POOL_BLOCK_HEIGHT, + ) + return server, rpc, ledger, active_chain + + +def label_values(lines: list[str]) -> dict[str, set[str]]: + values: dict[str, set[str]] = {} + for line in lines: + for name, value in re.findall(r'(\w+)="([^"]*)"', line): + values.setdefault(name, set()).add(value) + return values + + +def sample(lines: list[str], series: str) -> int: + """The integer value of one exact series line, which must exist once.""" + matches = [ + line for line in lines if line.startswith(series + " ") + ] + if len(matches) != 1: + raise AssertionError(f"{series}: {len(matches)} lines") + return int(float(matches[0].split(" ", 1)[1])) + + +class DenseCadenceTests(QuietCoordinatorTestCase): + """Consecutive trusted passes over one fixed window, through the real adapters.""" + + def setUp(self) -> None: + super().setUp() + self.server, self.rpc, self.ledger, self.active_chain = cadence_coordinator() + self.telemetry = ensure_accepted_preview_telemetry(self.server) + self.recorder = CandidateRecorder(self.server) + self.tip = str(self.rpc.tip) + # What the telemetry must report once the cadence is over, kept as + # the cadence runs so the story is told by the passes themselves. + self.expected_passes: Counter[str] = Counter() + self.expected_steps: Counter[tuple[str, str]] = Counter() + self.window_identity: tuple[object, ...] | None = None + self.armed_window: object = None + clock = patch("lab.prism.prism_coordinator.now_ms", return_value=FIXED_NOW_MS) + clock.start() + self.addCleanup(clock.stop) + + # -- helpers ----------------------------------------------------------- + + def published(self): + with self.server._job_cache_lock: + published = self.server._published_payout_state.artifact + return published + + def published_generation(self) -> int: + published = self.published() + return 0 if published is None else int(published.generation) + + def run_pass( + self, + label: str, + *, + intent: dict[str, bool] | None, + reads: int, + mutations: int, + window_mode: str | None = None, + caller: str = "other", + run: Callable[[], object] | None = None, + ) -> object: + """Run one pass and pin its per-pass contract. + + ``intent`` is the preparation kwargs the pass must forward, or + ``None`` when it must prepare nothing; ``reads`` is how many live + prior-balances reads it may pay for; ``mutations`` how many ledger + mutators it runs (the maturation sweep always runs once). + """ + reads_before = self.ledger.prior_balance_reads + records_before = len(self.recorder.records) + generation_before = self.published_generation() + + result = ( + run() + if run is not None + else self.server.reconcile_prism_pool_blocks_once(tip_hash=self.tip) + ) + + prepared = intent is not None + self.expected_passes[caller] += 1 + self.expected_steps[(caller, "admission_wait")] += 1 + # getblockcount plus one getblockhash for the watched row. + self.expected_steps[(caller, "chain_probe")] += 2 + # The watch rows plus the stranded-prepared sweep. + self.expected_steps[(caller, "watch_query")] += 2 + self.expected_steps[(caller, "mutations")] += mutations + self.expected_steps[(caller, "candidate_prepare")] += int(prepared) + self.expected_steps[(caller, "publish")] += int(prepared) + + records = self.recorder.records[records_before:] + self.assertEqual( + self.ledger.prior_balance_reads, reads_before + reads, label + ) + if not prepared: + self.assertEqual(records, [], label) + self.assertEqual(self.published_generation(), generation_before, label) + return result + + self.assertEqual(len(records), 1, label) + kwargs, artifact = records[0] + self.assertEqual(kwargs, intent, label) + self.assertIsNotNone(artifact, label) + self.assertEqual(artifact.window_build_mode, window_mode, label) + # The publication carries exactly the live carry aggregate. + published = self.published() + self.assertIsNotNone(published, label) + self.assertEqual(published.generation, generation_before + 1, label) + self.assertEqual( + [row["balance_sats"] for row in published.prior_balances()], + [self.ledger.balance_sats], + label, + ) + identity = ( + artifact.snapshot_anchor_ms, + artifact.share_snapshot_sha256, + tuple(artifact.shares_json), + ) + if self.window_identity is None: + self.window_identity = identity + self.armed_window = self.server._incremental_payout_artifact_window + self.assertIsNotNone(self.armed_window, label) + return result + # The same window: same anchor, same digest, same shares, and the + # very object the cold start armed -- never replaced, never + # rescanned. + self.assertEqual(identity, self.window_identity, label) + self.assertIs( + self.server._incremental_payout_artifact_window, + self.armed_window, + label, + ) + self.assertIsNone(artifact.window_full_rescan_reason, label) + return result + + def assert_no_window_rescan_since_cold_start(self) -> None: + self.assertEqual(self.ledger.full_snapshot_calls, 1) + self.assertEqual(self.ledger.delta_snapshot_calls, 0) + self.assertEqual(self.server.payout_artifact_event_counts["full_rescan"], 1) + snapshot = self.telemetry.snapshot() + self.assertEqual(full_rescan_total(snapshot), 1) + self.assertEqual(snapshot["full_rescans"][("cold_start", "in_process")]["count"], 1) + + # -- the cadence -------------------------------------------------------- + + def test_confirmed_mutations_reread_balances_over_one_exact_window(self) -> None: + ledger, chain = self.ledger, self.active_chain + self.assertEqual(ledger.balance_sats, INITIAL_BALANCE) + + # 1. Cold start: the tip is reserved, so the pass publishes; the + # window comes from the oracle once and the balances from the + # ledger once, with the plain pre-#224 intent. + first = self.run_pass( + "cold start", + intent=PLAIN_INTENT, + reads=1, + mutations=1, + window_mode="full_rescan", + ) + self.assertEqual(first["published_generation"], 1) + self.assertEqual(first["watched_blocks"], 1) + + # 2. A maturation moves the aggregate: reread, same window. + ledger.matured_pending = 1 + second = self.run_pass( + "maturation", + intent=BALANCE_REREAD_INTENT, + reads=1, + mutations=1, + window_mode="debounced", + ) + self.assertEqual(second["matured_payouts"], 1) + self.assertEqual(ledger.balance_sats, MATURED_BALANCE) + + # 3. Nothing moved and nothing requires publication: no + # preparation, no read, no generation. + third = self.run_pass("quiet", intent=None, reads=0, mutations=1) + self.assertIsNone(third["published_generation"]) + + # 4. The block leaves the active chain: inactive, reread. + chain[POOL_BLOCK_HEIGHT] = COMPETING_BLOCK_HASH + fourth = self.run_pass( + "inactive", + intent=BALANCE_REREAD_INTENT, + reads=1, + mutations=2, + window_mode="debounced", + ) + self.assertEqual(fourth["inactive_blocks"], 1) + self.assertEqual(ledger.balance_sats, INACTIVE_BALANCE) + + # 5. Still off the active chain: the inactive row is left alone. + fifth = self.run_pass("quiet while inactive", intent=None, reads=0, mutations=1) + self.assertEqual(fifth["inactive_blocks"], 0) + self.assertEqual(fifth["reactivated_blocks"], 0) + + # 6. The chain flips back: reactivated, reread. + chain[POOL_BLOCK_HEIGHT] = POOL_BLOCK_HASH + sixth = self.run_pass( + "reactivated", + intent=BALANCE_REREAD_INTENT, + reads=1, + mutations=2, + window_mode="debounced", + ) + self.assertEqual(sixth["reactivated_blocks"], 1) + self.assertEqual(ledger.balance_sats, REACTIVATED_BALANCE) + + # 7. A post-confirm forced publication with nothing to mutate keeps + # the plain intent and reuses the published balances verbatim. + seventh = self.run_pass( + "post-confirm republish", + caller="post_confirm", + run=lambda: self.server.reconcile_prism_pool_blocks_once( + tip_hash=self.tip, + _force_publish=True, + _source_reserved=True, + ), + intent=PLAIN_INTENT, + reads=0, + mutations=1, + window_mode="debounced", + ) + self.assertIsNotNone(seventh["published_generation"]) + + # 8. The landing's in-lock pass with a second maturation. + ledger.matured_pending = 1 + self.run_pass( + "landing maturation", + caller="landing", + run=lambda: self.assertTrue( + self.server.ensure_reorg_reconciled_for_tip( + self.tip, _coalesce_same_tip=False + ) + ), + intent=BALANCE_REREAD_INTENT, + reads=1, + mutations=1, + window_mode="debounced", + ) + self.assertEqual(ledger.balance_sats, MATURED_BALANCE) + + # The window was read from the oracle exactly once, for the cold + # start, and never rescanned; every later read was a balance read + # owned by a payout-changing pass. + self.assert_no_window_rescan_since_cold_start() + self.assertEqual(ledger.prior_balance_reads, 5) + self.assertEqual(self.published_generation(), 6) + self.assert_telemetry_tells_the_same_story() + + def assert_telemetry_tells_the_same_story(self) -> None: + snapshot = self.telemetry.snapshot() + self.assertEqual( + pass_counts(snapshot), + {caller: self.expected_passes[caller] for caller in PRISM_REORG_RECONCILE_CALLERS}, + ) + for caller in PRISM_REORG_RECONCILE_CALLERS: + for step in PRISM_REORG_RECONCILE_STEPS: + self.assertEqual( + step_count(snapshot, caller, step), + self.expected_steps[(caller, step)], + (caller, step), + ) + # Closed products, and nothing the cadence did grew them. + self.assertEqual( + set(snapshot["reconcile_steps"]), + { + (caller, step) + for caller in PRISM_REORG_RECONCILE_CALLERS + for step in PRISM_REORG_RECONCILE_STEPS + }, + ) + self.assertEqual( + set(snapshot["full_rescans"]), + { + (reason, path) + for reason in PRISM_PAYOUT_WINDOW_FULL_RESCAN_REASONS + for path in PRISM_PAYOUT_WINDOW_FULL_RESCAN_PATHS + }, + ) + # The reads the passes paid for are the reads the ledger attributed, + # under the one contract operation, and nothing else. + folded = fold_ledger_read_stats(self.ledger.ledger_read_gate_stats()) + self.assertEqual(list(folded), ["current_prior_balances"]) + self.assertIn("current_prior_balances", PRISM_LEDGER_READ_OPERATIONS) + self.assertEqual( + folded["current_prior_balances"]["calls_total"], + self.ledger.prior_balance_reads, + ) + # And the operator sees the same story on /metrics with only + # closed-vocabulary labels. + renderer = MetricsRenderer(self.server) + lines = ( + renderer.accepted_preview_attribution_metrics_lines() + + renderer._ledger_read_gate_metric_lines() + ) + labels = label_values(lines) + self.assertEqual(labels["phase"], set(PRISM_ACCEPTED_LANDING_PHASES)) + self.assertEqual(labels["caller"], set(PRISM_REORG_RECONCILE_CALLERS)) + self.assertEqual(labels["step"], set(PRISM_REORG_RECONCILE_STEPS)) + self.assertEqual(labels["reason"], set(PRISM_PAYOUT_WINDOW_FULL_RESCAN_REASONS)) + self.assertEqual(labels["path"], set(PRISM_PAYOUT_WINDOW_FULL_RESCAN_PATHS)) + self.assertEqual(labels["operation"], {"current_prior_balances"}) + self.assertEqual( + sample(lines, 'qbit_prism_ledger_read_calls_total{operation="current_prior_balances"}'), + self.ledger.prior_balance_reads, + ) + for caller in PRISM_REORG_RECONCILE_CALLERS: + self.assertEqual( + sample(lines, f'qbit_prism_reorg_reconcile_pass_seconds_count{{caller="{caller}"}}'), + self.expected_passes[caller], + ) + self.assertEqual( + sample( + lines, + 'qbit_prism_payout_window_full_rescan_seconds_count' + '{reason="reconcile_invalidation",path="in_process"}', + ), + 0, + ) + self.assertEqual( + sample( + lines, + 'qbit_prism_payout_window_full_rescan_seconds_count' + '{reason="cold_start",path="in_process"}', + ), + 1, + ) + + def test_a_lost_mutator_response_mid_cadence_keeps_the_full_rescan(self) -> None: + """The fail-closed path is unchanged under the same cadence. + + The maturation may have committed before its response was lost, so + the row counts are unknowable. The pass republishes against the + forced full rescan, which necessarily rereads the current balances, + and records the rescan under its own reason. + """ + ledger = self.ledger + self.run_pass( + "cold start", + intent=PLAIN_INTENT, + reads=1, + mutations=1, + window_mode="full_rescan", + ) + ledger.matured_pending = 1 + self.run_pass( + "maturation", + intent=BALANCE_REREAD_INTENT, + reads=1, + mutations=1, + window_mode="debounced", + ) + self.assert_no_window_rescan_since_cold_start() + generation_before = self.published_generation() + armed_before = self.server._incremental_payout_artifact_window + + ledger.lose_next_maturation_response = True + with self.assertRaisesRegex(ConnectionError, "mutation response lost"): + self.server.reconcile_prism_pool_blocks_once(tip_hash=self.tip) + + kwargs, artifact = self.recorder.records[-1] + self.assertEqual(kwargs, FULL_RESCAN_INTENT) + self.assertEqual(artifact.window_build_mode, "full_rescan") + self.assertEqual(artifact.window_full_rescan_reason, "reconcile_invalidation") + # The oracle ran again and re-armed a fresh window whose content is + # the same (no share landed) but which is no longer the cold-start + # object. + self.assertEqual(ledger.full_snapshot_calls, 2) + assert self.window_identity is not None + self.assertEqual(artifact.share_snapshot_sha256, self.window_identity[1]) + self.assertIsNot(self.server._incremental_payout_artifact_window, armed_before) + # The republication carries the balances the lost mutation left. + self.assertEqual(ledger.prior_balance_reads, 3) + published = self.published() + assert published is not None + self.assertEqual(published.generation, generation_before + 1) + self.assertEqual( + [row["balance_sats"] for row in published.prior_balances()], + [LOST_RESPONSE_BALANCE], + ) + service = self.server._ensure_reorg_reconciler_service() + self.assertEqual(service.snapshot().reconcile_error_count, 1) + snapshot = self.telemetry.snapshot() + self.assertEqual(pass_counts(snapshot)["other"], 3) + self.assertEqual(step_count(snapshot, "other", "mutations"), 3) + self.assertEqual(step_count(snapshot, "other", "candidate_prepare"), 3) + self.assertEqual(step_count(snapshot, "other", "publish"), 3) + rescans = snapshot["full_rescans"] + self.assertEqual(rescans[("reconcile_invalidation", "in_process")]["count"], 1) + self.assertEqual(rescans[("cold_start", "in_process")]["count"], 1) + self.assertEqual(full_rescan_total(snapshot), 2) + self.assertEqual(self.server.payout_artifact_event_counts["full_rescan"], 2) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_prism_issue_224_core.py b/tests/test_prism_issue_224_core.py new file mode 100644 index 000000000..0fc3064e5 --- /dev/null +++ b/tests/test_prism_issue_224_core.py @@ -0,0 +1,1040 @@ +"""Issue #224 Wave 1 core: reconciliation rereads balances, never the window. + +A reconcile pass mutates pool-block, payout-entry and carry state but never +``qbit_share_ledger`` (the invariant recorded in +``lab.prism.accepted_preview_telemetry``). At a fixed anchor the accepted +share window therefore stays exact after a maturation, an inactive marking +or a reactivation, and only the prior balances need a live reread. Before +this wave every confirmed mutation answered with ``force_full_window_rescan`` +and paid an O(window) oracle read under the writer lock on the accepted +preview's critical path. + +These tests pin the split between the two intents, the fail-closed paths +that keep the full rescan, and the fixed-cardinality attribution the +telemetry contract defines for reconcile passes, steps, callers and actual +full rescans. +""" + +from __future__ import annotations + +import threading +import time +import unittest +from contextlib import contextmanager, nullcontext +from types import SimpleNamespace +from unittest.mock import patch + +from lab.prism.accepted_preview_telemetry import ( + AcceptedPreviewTelemetry, + PRISM_PAYOUT_WINDOW_FULL_RESCAN_PATHS, + PRISM_PAYOUT_WINDOW_FULL_RESCAN_REASONS, + PRISM_REORG_RECONCILE_CALLERS, + PRISM_REORG_RECONCILE_STEPS, + ensure_accepted_preview_telemetry, +) +from lab.prism.payout_state import ( + PayoutStatePublicationBlocked, + _PayoutWindowMaterialization, +) +from lab.prism.reorg_reconciler import ReorgPorts, ReorgReconcilerService +from lab.prism.share_ledger import IncrementalWindowAdvanceStats +from tests.prism_coordinator_test_support import ( + IncrementalRecordingLedger, + append_incremental_share, + coordinator, +) + + +BALANCE_REREAD_INTENT = { + "force_full_window_rescan": False, + "force_prior_balances_read": True, +} +PLAIN_INTENT = {"force_full_window_rescan": False} +FULL_RESCAN_INTENT = {"force_full_window_rescan": True} + + +def full_rescan_total(snapshot: dict[str, object]) -> int: + rescans = snapshot["full_rescans"] + assert isinstance(rescans, dict) + return sum(int(stats["count"]) for stats in rescans.values()) + + +def pass_counts(snapshot: dict[str, object]) -> dict[str, int]: + passes = snapshot["reconcile_passes"] + assert isinstance(passes, dict) + return {caller: int(stats["count"]) for caller, stats in passes.items()} + + +def step_count(snapshot: dict[str, object], caller: str, step: str) -> int: + steps = snapshot["reconcile_steps"] + assert isinstance(steps, dict) + return int(steps[(caller, step)]["count"]) + + +# --- reconciler-level fixtures ------------------------------------------- + + +class MutatingLedger: + """One confirmed row that leaves the active chain plus a maturation.""" + + def __init__( + self, + *, + rows: list[dict[str, object]] | None = None, + matured: int = 1, + lose_mutation_response: bool = False, + ) -> None: + self.rows = ( + [{"block_height": 8, "block_hash": "aa", "chain_state": "confirmed"}] + if rows is None + else rows + ) + self.matured = matured + self.lose_mutation_response = lose_mutation_response + self.inactivated: list[str] = [] + + def reorg_watch_blocks(self, *, active_tip_height: int) -> list[dict[str, object]]: + return [dict(row) for row in self.rows] + + def mark_pool_block_inactive(self, **kwargs: object) -> dict[str, int]: + self.inactivated.append(str(kwargs["block_hash"])) + if self.lose_mutation_response: + raise ConnectionError("mutation response lost") + return {"inactive_count": 1} + + def reactivate_pool_block(self, **kwargs: object) -> dict[str, int]: + return {"reactivated_count": 0} + + def mark_mature_pool_payouts(self, **_kwargs: object) -> dict[str, int]: + return {"matured_count": self.matured} + + +def make_ports( + *, + ledger: object, + telemetry: AcceptedPreviewTelemetry | None = None, + chain_untrusted: bool = False, + publication_required: bool = False, + publish: object = 7, + serialized_refusals: list[BaseException] | None = None, +) -> tuple[ReorgPorts, list[tuple[str, object]], dict[str, ReorgReconcilerService]]: + """Ports whose coordinator seams route back into the service under test. + + ``reconcile_with_admission`` and ``reconcile_serialized`` stand in for + the coordinator's public entry and serialized adapter, so the caller + stamp and the admission stamp cross the same seams they cross in the + coordinator; ``serialized_refusals`` raises in the adapter before the + core pass, the way a landed preview refuses a pass. + """ + events: list[tuple[str, object]] = [] + holder: dict[str, ReorgReconcilerService] = {} + state_lock = threading.RLock() + active_chain = {8: "not-aa"} + refusals = list(serialized_refusals or []) + + def rpc_call(method: str, params: object = None) -> object: + if method == "getbestblockhash": + return "tip" + if method == "getblockcount": + return 10 + if method == "getblockhash": + height = int(params[0]) # type: ignore[index] + events.append(("getblockhash", height)) + return active_chain[height] + raise AssertionError(method) + + @contextmanager + def publication_guard(): + yield + + def prepared_candidate(captured, **kwargs): + events.append(("prepare", kwargs)) + return SimpleNamespace(captured=captured) + + def reconcile_serialized(**kwargs: object) -> dict[str, object]: + events.append(("serialized", dict(kwargs))) + if refusals: + raise refusals.pop(0) + return holder["service"].reconcile( + tip_hash=kwargs.get("tip_hash"), # type: ignore[arg-type] + force_publish=bool(kwargs.get("_force_publish", False)), + source_reserved=bool(kwargs.get("_source_reserved", False)), + ) + + ports = ReorgPorts( + rpc_call=rpc_call, + ledger=lambda: ledger, + ensure_job_cache_state=lambda: None, + state_lock=lambda: state_lock, + source_tip=lambda: "tip", + reserve_external_tip=lambda tip: events.append(("reserve", tip)), + max_supersession_retries=lambda: 1, + prepare_lock=nullcontext, + capture_source=lambda: (0, 0, "tip", "test", 0.0), + prepared_candidate=prepared_candidate, + captured_publication_required=lambda _captured: publication_required, + block_publication=lambda **kwargs: events.append(("block", kwargs)), + publication_guard=publication_guard, + publish_candidate=lambda _candidate: publish, # type: ignore[return-value] + observe_preparation=lambda elapsed: None, + chain_view_untrusted=lambda: chain_untrusted, + reorg_proof_snapshot=lambda: ("tip", 0), + flight_wait_seconds=lambda: 30.0, + prefetch_join_timeout_seconds=lambda: 20.0, + reconcile_with_admission=lambda **kwargs: ( + holder["service"].reconcile_with_flights(**kwargs) + ), + reconcile_serialized=reconcile_serialized, + ensure_tip=lambda tip: holder["service"].ensure_tip(tip), + accepted_preview_telemetry=( + (lambda: telemetry) if telemetry is not None else None + ), + ) + return ports, events, holder + + +def make_service(**kwargs: object) -> tuple[ + ReorgReconcilerService, + list[tuple[str, object]], + AcceptedPreviewTelemetry, +]: + telemetry = kwargs.pop("telemetry", None) or AcceptedPreviewTelemetry() + ports, events, holder = make_ports(telemetry=telemetry, **kwargs) # type: ignore[arg-type] + service = ReorgReconcilerService(ports, cache_seconds=0.0) + holder["service"] = service + return service, events, telemetry + + +def prepare_intents(events: list[tuple[str, object]]) -> list[object]: + return [payload for name, payload in events if name == "prepare"] + + +class ReconcileIntentTests(unittest.TestCase): + """The confirmed-mutation path asks for balances, not the window.""" + + def test_confirmed_mutations_request_a_balance_reread_over_the_window( + self, + ) -> None: + ledger = MutatingLedger() + service, events, _telemetry = make_service(ledger=ledger) + + summary = service.reconcile(tip_hash="tip") + + self.assertEqual(ledger.inactivated, ["aa"]) + self.assertEqual(summary["inactive_blocks"], 1) + self.assertEqual(summary["matured_payouts"], 1) + self.assertEqual(summary["published_generation"], 7) + # Exactly one preparation, and it carries the narrow #224 intent: + # the exact window is reused, only the carry aggregate is reread. + self.assertEqual(prepare_intents(events), [BALANCE_REREAD_INTENT]) + # Publication still fences delivery until the reread balances land. + self.assertIn(("block", {"force": True}), events) + + def test_maturation_alone_requests_the_reread(self) -> None: + ledger = MutatingLedger(rows=[], matured=3) + service, events, _telemetry = make_service(ledger=ledger) + + summary = service.reconcile(tip_hash="tip") + + self.assertEqual(summary["matured_payouts"], 3) + self.assertEqual(prepare_intents(events), [BALANCE_REREAD_INTENT]) + + def test_a_pass_without_mutations_keeps_the_plain_intent(self) -> None: + ledger = MutatingLedger(rows=[], matured=0) + service, events, _telemetry = make_service( + ledger=ledger, + publication_required=True, + ) + + service.reconcile(tip_hash="tip") + + # No reread is forced (the published balances are still exact) and + # the pre-#224 keyword shape reaches preparation seams unchanged. + self.assertEqual(prepare_intents(events), [PLAIN_INTENT]) + + def test_no_mutation_and_no_required_publication_prepares_nothing( + self, + ) -> None: + ledger = MutatingLedger(rows=[], matured=0) + service, events, _telemetry = make_service(ledger=ledger) + + summary = service.reconcile(tip_hash="tip") + + self.assertIsNone(summary["published_generation"]) + self.assertEqual(prepare_intents(events), []) + + def test_lost_mutator_response_keeps_the_forced_full_rescan(self) -> None: + """Error-after-possible-commit stays fail-closed and unchanged. + + The mutator may have committed before its response was lost, so the + row counts are unknowable. The pass republishes against the forced + full rescan exactly as before: that path necessarily rereads the + current prior balances, so stale balances can never be reused, and + the narrow reread intent is not what it asks for. + """ + ledger = MutatingLedger(lose_mutation_response=True) + service, events, telemetry = make_service(ledger=ledger) + + with self.assertRaisesRegex(ConnectionError, "mutation response lost"): + service.reconcile(tip_hash="tip") + + self.assertEqual(prepare_intents(events), [FULL_RESCAN_INTENT]) + self.assertIn(("block", {"force": True}), events) + self.assertEqual(service.snapshot().reconcile_error_count, 1) + # The error exit still records its pass and the steps it reached, + # including the error candidate's preparation and publication. + snapshot = telemetry.snapshot() + self.assertEqual(pass_counts(snapshot)["other"], 1) + self.assertEqual(step_count(snapshot, "other", "mutations"), 1) + self.assertEqual(step_count(snapshot, "other", "candidate_prepare"), 1) + self.assertEqual(step_count(snapshot, "other", "publish"), 1) + + def test_a_read_phase_failure_forces_nothing(self) -> None: + class ReadFailingLedger(MutatingLedger): + def reorg_watch_blocks(self, *, active_tip_height: int): + raise TimeoutError("reconcile prefetch join exceeded 20s") + + service, events, telemetry = make_service(ledger=ReadFailingLedger()) + + with self.assertRaises(TimeoutError): + service.reconcile(tip_hash="tip") + + self.assertEqual(prepare_intents(events), []) + snapshot = telemetry.snapshot() + self.assertEqual(pass_counts(snapshot)["other"], 1) + self.assertEqual(step_count(snapshot, "other", "watch_query"), 1) + self.assertEqual(step_count(snapshot, "other", "candidate_prepare"), 0) + + def test_untrusted_forced_publication_prepares_without_any_intent( + self, + ) -> None: + service, events, _telemetry = make_service( + ledger=MutatingLedger(), + chain_untrusted=True, + ) + + summary = service.reconcile(tip_hash="tip", force_publish=True) + + self.assertTrue(summary["untrusted"]) + self.assertEqual(prepare_intents(events), [{}]) + + +class ReconcileAttributionTests(unittest.TestCase): + """Pass/step/caller observations through the real entry points.""" + + def test_direct_pass_records_pass_and_every_step_it_ran(self) -> None: + service, _events, telemetry = make_service(ledger=MutatingLedger()) + + service.reconcile(tip_hash="tip") + + snapshot = telemetry.snapshot() + counts = pass_counts(snapshot) + self.assertEqual(counts["other"], 1) + for caller in PRISM_REORG_RECONCILE_CALLERS: + if caller != "other": + self.assertEqual(counts[caller], 0, caller) + # getblockcount plus one getblockhash; the watch query; the inactive + # marking plus the maturation; the preparation; the publication. + self.assertEqual(step_count(snapshot, "other", "chain_probe"), 2) + self.assertEqual(step_count(snapshot, "other", "watch_query"), 1) + self.assertEqual(step_count(snapshot, "other", "mutations"), 2) + self.assertEqual(step_count(snapshot, "other", "candidate_prepare"), 1) + self.assertEqual(step_count(snapshot, "other", "publish"), 1) + # A direct core call never crossed the serialized adapter. + self.assertEqual(step_count(snapshot, "other", "admission_wait"), 0) + # Nothing here is a full rescan. + self.assertEqual(full_rescan_total(snapshot), 0) + + def test_untrusted_pass_records_its_pass_and_no_steps(self) -> None: + service, _events, telemetry = make_service( + ledger=MutatingLedger(), + chain_untrusted=True, + ) + + service.reconcile(tip_hash="tip") + + snapshot = telemetry.snapshot() + self.assertEqual(pass_counts(snapshot)["other"], 1) + for step in PRISM_REORG_RECONCILE_STEPS: + self.assertEqual(step_count(snapshot, "other", step), 0, step) + + def test_forced_publication_without_context_is_post_confirm(self) -> None: + service, events, telemetry = make_service(ledger=MutatingLedger()) + + service.reconcile_with_flights( + tip_hash="tip", + _force_publish=True, + _source_reserved=True, + ) + + snapshot = telemetry.snapshot() + self.assertEqual(pass_counts(snapshot)["post_confirm"], 1) + self.assertEqual(pass_counts(snapshot)["other"], 0) + # Crossing the adapter records the admission wait for that caller. + self.assertEqual( + step_count(snapshot, "post_confirm", "admission_wait"), 1 + ) + # The adapter seam still receives exactly its historical kwargs. + self.assertIn( + ( + "serialized", + { + "tip_hash": "tip", + "_force_publish": True, + "_source_reserved": True, + }, + ), + events, + ) + + def test_direct_forced_core_call_is_post_confirm_too(self) -> None: + service, _events, telemetry = make_service(ledger=MutatingLedger()) + service.reconcile(tip_hash="tip", force_publish=True) + self.assertEqual(pass_counts(telemetry.snapshot())["post_confirm"], 1) + + def test_flight_bypass_attributes_the_landing(self) -> None: + service, events, telemetry = make_service(ledger=MutatingLedger()) + + self.assertTrue(service.ensure_tip("tip", _coalesce_same_tip=False)) + + snapshot = telemetry.snapshot() + self.assertEqual(pass_counts(snapshot)["landing"], 1) + self.assertEqual(step_count(snapshot, "landing", "admission_wait"), 1) + self.assertEqual(step_count(snapshot, "landing", "candidate_prepare"), 1) + self.assertIn(("serialized", {"tip_hash": "tip"}), events) + + def test_coalescing_ensure_tip_without_context_is_other(self) -> None: + service, _events, telemetry = make_service(ledger=MutatingLedger()) + self.assertTrue(service.ensure_tip("tip")) + self.assertEqual(pass_counts(telemetry.snapshot())["other"], 1) + + def test_job_build_entry_attributes_job_build(self) -> None: + service, _events, telemetry = make_service(ledger=MutatingLedger()) + + self.assertTrue(service.ensure_current()) + + snapshot = telemetry.snapshot() + self.assertEqual(pass_counts(snapshot)["job_build"], 1) + self.assertEqual(step_count(snapshot, "job_build", "admission_wait"), 1) + # The existing lookup family and the new pass family join on the + # same spelling. + self.assertEqual( + service.reorg_reconcile_lookup_snapshot()[("job_build", "serial")], + 1, + ) + + def test_prefetch_lane_attributes_tip_refresh(self) -> None: + service, _events, telemetry = make_service(ledger=MutatingLedger()) + + self.assertTrue(service.prefetch_pass("tip")) + self.assertTrue(service.prefetch_pass("tip", prove=True)) + # The bounded re-prove routes through the same lane. + service.shutdown_prefetch_executor() + self.assertTrue(service.snapshot_tip_bounded("tip")) + + self.assertEqual(pass_counts(telemetry.snapshot())["tip_refresh"], 3) + + def test_explicit_caller_wins_and_unknown_callers_are_refused(self) -> None: + service, _events, telemetry = make_service(ledger=MutatingLedger()) + + service.ensure_tip("tip", _caller="tip_refresh") + service.reconcile_with_flights(tip_hash="tip", _caller="job_build") + service.reconcile(tip_hash="tip", caller="landing") + + counts = pass_counts(telemetry.snapshot()) + self.assertEqual(counts["tip_refresh"], 1) + self.assertEqual(counts["job_build"], 1) + self.assertEqual(counts["landing"], 1) + with self.assertRaisesRegex(ValueError, "unknown reorg reconcile caller"): + service.reconcile(tip_hash="tip", caller="benchmark-run-42") + with self.assertRaisesRegex(ValueError, "unknown reorg reconcile caller"): + service.ensure_tip("tip", _caller="benchmark-run-42") + + def test_outer_entry_point_keeps_its_label_through_nested_seams(self) -> None: + """A prefetched pass reaching ensure_tip via the coordinator stays tip_refresh.""" + service, _events, telemetry = make_service(ledger=MutatingLedger()) + + with service._attributed("tip_refresh"): + service.reconcile_with_flights(tip_hash="tip", _force_publish=True) + + counts = pass_counts(telemetry.snapshot()) + self.assertEqual(counts["tip_refresh"], 1) + self.assertEqual(counts["post_confirm"], 0) + # The stamp is restored once the entry point returns. + self.assertIsNone(service._attributed_caller()) + + def test_adapter_refusal_clears_the_admission_stamp(self) -> None: + service, _events, telemetry = make_service( + ledger=MutatingLedger(), + serialized_refusals=[ + PayoutStatePublicationBlocked("accepted block payout pending") + ], + ) + + with self.assertRaises(PayoutStatePublicationBlocked): + service.reconcile_with_flights(tip_hash="tip") + # A later direct call on this thread must not inherit the stamp. + service.reconcile(tip_hash="tip") + + snapshot = telemetry.snapshot() + self.assertEqual(pass_counts(snapshot)["other"], 1) + self.assertEqual(step_count(snapshot, "other", "admission_wait"), 0) + + def test_disabled_service_records_nothing(self) -> None: + ports, _events, holder = make_ports(ledger=MutatingLedger()) + service = ReorgReconcilerService(ports, enabled=False) + holder["service"] = service + service.reconcile(tip_hash="tip") + self.assertTrue(service.ensure_tip("tip", _coalesce_same_tip=False)) + self.assertEqual( + sum(pass_counts(service.accepted_preview_telemetry.snapshot()).values()), + 0, + ) + + def test_ports_without_an_owner_get_one_private_stable_owner(self) -> None: + ports, _events, holder = make_ports(ledger=MutatingLedger()) + service = ReorgReconcilerService(ports) + holder["service"] = service + owner = service.accepted_preview_telemetry + self.assertIsInstance(owner, AcceptedPreviewTelemetry) + self.assertIs(service.accepted_preview_telemetry, owner) + + service.reconcile(tip_hash="tip") + self.assertEqual(pass_counts(owner.snapshot())["other"], 1) + + def test_port_supplied_owner_is_used_as_is(self) -> None: + shared = AcceptedPreviewTelemetry() + service, _events, telemetry = make_service( + ledger=MutatingLedger(), + telemetry=shared, + ) + self.assertIs(telemetry, shared) + self.assertIs(service.accepted_preview_telemetry, shared) + + +# --- payout-state level --------------------------------------------------- + + +class CarryLedger(IncrementalRecordingLedger): + """Incremental ledger whose carry aggregate moves under the tests' control.""" + + def __init__(self) -> None: + super().__init__() + self.balance_sats = 546 + self.prior_balance_reads = 0 + self.matured_pending = 0 + self.fail_full_snapshot = False + + def current_prior_balances(self) -> list[dict[str, object]]: + self.prior_balance_reads += 1 + return [ + { + "recipient_id": "carry", + "order_key": "01:carry", + "p2mr_program_hex": "66" * 32, + "balance_sats": self.balance_sats, + } + ] + + def snapshot_at_job_issue(self, anchor_job_issued_at_ms, *, window_weight=None): + if self.fail_full_snapshot: + self.full_snapshot_calls += 1 + raise TimeoutError("postgres statement deadline expired") + return super().snapshot_at_job_issue( + anchor_job_issued_at_ms, + window_weight=window_weight, + ) + + def mark_mature_pool_payouts(self, *, active_tip_height: int) -> dict[str, int | str]: + matured, self.matured_pending = self.matured_pending, 0 + if matured: + # Maturation moves carry balances; the share ledger is untouched. + self.balance_sats = 777 + return {"backend": "memory", "matured_count": matured} + + +def configured_coordinator(ledger: CarryLedger | None = None): + """A ready coordinator over a three-share window on a carry ledger. + + ``ledger`` lets the Wave 2 cadence module supply its subclass without + re-stating this setup. + """ + ledger = CarryLedger() if ledger is None else ledger + append_incremental_share(ledger, share_seq=1, accepted_at_ms=999_900) + append_incremental_share(ledger, share_seq=2, accepted_at_ms=999_910) + append_incremental_share(ledger, share_seq=3, accepted_at_ms=999_920) + server, rpc = coordinator(ledger=ledger) + artifacts = server.store_template_artifacts(dict(rpc.template)) + assert artifacts is not None + server._pool_ready_latched = True + server.payout_artifact_min_build_interval_seconds = 60.0 + server.payout_artifact_full_rescan_seconds = 3_600.0 + return server, rpc, ledger, artifacts + + +class QuietCoordinatorTestCase(unittest.TestCase): + """Keep the artifact lifecycle log lines out of the test output.""" + + def setUp(self) -> None: + super().setUp() + printer = patch("builtins.print") + printer.start() + self.addCleanup(printer.stop) + + +class PayoutWindowIntentTests(QuietCoordinatorTestCase): + def test_balance_reread_reuses_the_exact_window_and_reads_live_carry( + self, + ) -> None: + server, _rpc, ledger, artifacts = configured_coordinator() + telemetry = ensure_accepted_preview_telemetry(server) + with patch("lab.prism.prism_coordinator.now_ms", return_value=1_000_000): + published = server._current_payout_state_artifact() + initial = server._build_payout_ledger_artifact( + 0, 0, artifacts.network_difficulty + ) + assert initial is not None + self.assertEqual(initial.window_build_mode, "full_rescan") + self.assertEqual(initial.window_full_rescan_reason, "cold_start") + self.assertEqual(ledger.full_snapshot_calls, 1) + self.assertTrue(server._install_payout_ledger_artifact(initial)) + + ledger.balance_sats = 777 + reads_before = ledger.prior_balance_reads + reused = server._build_payout_ledger_artifact( + 0, 0, artifacts.network_difficulty + ) + assert reused is not None + # Without the intent the published bytes are reused verbatim. + self.assertEqual(reused.prior_balances[0]["balance_sats"], 546) + self.assertEqual( + reused.prior_balances_sha256, published.prior_balances_sha256 + ) + self.assertEqual(ledger.prior_balance_reads, reads_before) + + reread = server._build_payout_ledger_artifact( + 0, + 0, + artifacts.network_difficulty, + force_prior_balances_read=True, + ) + + assert reread is not None + # The window is the armed one: no oracle read, same digest, same + # anchor, and no rescan reason. + self.assertEqual(reread.window_build_mode, "debounced") + self.assertIsNone(reread.window_full_rescan_reason) + self.assertEqual(reread.share_snapshot_sha256, initial.share_snapshot_sha256) + self.assertEqual(reread.snapshot_anchor_ms, initial.snapshot_anchor_ms) + self.assertEqual(ledger.full_snapshot_calls, 1) + self.assertEqual(ledger.delta_snapshot_calls, 0) + # The balances are live. + self.assertEqual(reread.prior_balances[0]["balance_sats"], 777) + self.assertEqual(ledger.prior_balance_reads, reads_before + 1) + self.assertNotEqual( + reread.prior_balances_sha256, published.prior_balances_sha256 + ) + # A balance reread is never an observation of the rescan family. + snapshot = telemetry.snapshot() + self.assertEqual(full_rescan_total(snapshot), 1) + self.assertEqual(snapshot["full_rescans"][("cold_start", "in_process")]["count"], 1) + + def test_balance_reread_rides_the_delta_path_when_shares_landed(self) -> None: + server, _rpc, ledger, artifacts = configured_coordinator() + server.payout_artifact_min_build_interval_seconds = 0.0 + clock_ms = [1_000_000] + with patch( + "lab.prism.prism_coordinator.now_ms", + side_effect=lambda: clock_ms[0], + ): + server._current_payout_state_artifact() + initial = server._build_payout_ledger_artifact( + 0, 0, artifacts.network_difficulty + ) + assert initial is not None + self.assertTrue(server._install_payout_ledger_artifact(initial)) + append_incremental_share(ledger, share_seq=4, accepted_at_ms=1_000_010) + clock_ms[0] = 1_000_020 + ledger.balance_sats = 777 + reread = server._build_payout_ledger_artifact( + 0, + 0, + artifacts.network_difficulty, + force_prior_balances_read=True, + ) + + assert reread is not None + self.assertEqual(reread.window_build_mode, "incremental") + self.assertEqual(reread.window_delta_rows, 1) + self.assertEqual(len(reread.shares_json), 4) + self.assertEqual(ledger.full_snapshot_calls, 1) + self.assertEqual(ledger.delta_snapshot_calls, 1) + self.assertEqual(reread.prior_balances[0]["balance_sats"], 777) + self.assertEqual( + full_rescan_total(ensure_accepted_preview_telemetry(server).snapshot()), + 1, + ) + + def test_force_full_window_rescan_keeps_its_whole_meaning(self) -> None: + server, _rpc, ledger, artifacts = configured_coordinator() + telemetry = ensure_accepted_preview_telemetry(server) + with patch("lab.prism.prism_coordinator.now_ms", return_value=1_000_000): + server._current_payout_state_artifact() + initial = server._build_payout_ledger_artifact( + 0, 0, artifacts.network_difficulty + ) + assert initial is not None + self.assertTrue(server._install_payout_ledger_artifact(initial)) + ledger.balance_sats = 777 + reads_before = ledger.prior_balance_reads + forced = server._build_payout_ledger_artifact( + 0, + 0, + artifacts.network_difficulty, + True, + ) + + assert forced is not None + self.assertEqual(forced.window_build_mode, "full_rescan") + self.assertEqual(forced.window_full_rescan_reason, "reconcile_invalidation") + self.assertEqual(ledger.full_snapshot_calls, 2) + # A full rescan necessarily obtains the current prior balances. + self.assertEqual(forced.prior_balances[0]["balance_sats"], 777) + self.assertEqual(ledger.prior_balance_reads, reads_before + 1) + rescans = telemetry.snapshot()["full_rescans"] + self.assertEqual(rescans[("reconcile_invalidation", "in_process")]["count"], 1) + self.assertEqual(rescans[("cold_start", "in_process")]["count"], 1) + + def test_prepared_candidate_carries_both_intents_distinctly(self) -> None: + server, rpc, ledger, artifacts = configured_coordinator() + telemetry = ensure_accepted_preview_telemetry(server) + captured = (0, 0, str(rpc.tip), "external_tip", time.monotonic()) + with patch("lab.prism.prism_coordinator.now_ms", return_value=1_000_000): + server._current_payout_state_artifact() + initial = server._build_payout_ledger_artifact( + 0, 0, artifacts.network_difficulty + ) + assert initial is not None + self.assertTrue(server._install_payout_ledger_artifact(initial)) + ledger.balance_sats = 777 + + # Compatibility: the pre-#224 signature still works. + plain = server._prepared_payout_state_candidate(captured) + reread = server._prepared_payout_state_candidate( + captured, + force_prior_balances_read=True, + ) + forced = server._prepared_payout_state_candidate( + captured, + force_full_window_rescan=True, + ) + + assert plain.ledger_artifact is not None + assert reread.ledger_artifact is not None + assert forced.ledger_artifact is not None + self.assertEqual(plain.ledger_artifact.window_build_mode, "debounced") + self.assertEqual(plain.ledger_artifact.prior_balances[0]["balance_sats"], 546) + self.assertEqual(reread.ledger_artifact.window_build_mode, "debounced") + self.assertEqual(reread.ledger_artifact.prior_balances[0]["balance_sats"], 777) + self.assertEqual(forced.ledger_artifact.window_build_mode, "full_rescan") + self.assertEqual( + forced.ledger_artifact.window_full_rescan_reason, + "reconcile_invalidation", + ) + self.assertEqual(forced.ledger_artifact.prior_balances[0]["balance_sats"], 777) + self.assertEqual(ledger.full_snapshot_calls, 2) + self.assertEqual( + telemetry.snapshot()["full_rescans"][ + ("reconcile_invalidation", "in_process") + ]["count"], + 1, + ) + + def test_narrow_build_seams_are_not_handed_the_new_keyword(self) -> None: + """Preparation forwards the reread intent only when it applies.""" + server, rpc, _ledger, _artifacts = configured_coordinator() + seen: list[tuple[object, ...]] = [] + + def narrow_build( + expected_payout_state_generation: int, + artifact_payout_state_generation: int, + network_difficulty: int, + force_full_rescan: bool = False, + bypass_build_interval: bool = False, + during_publication: bool = False, + ) -> None: + seen.append( + ( + expected_payout_state_generation, + artifact_payout_state_generation, + force_full_rescan, + ) + ) + return None + + server._build_payout_ledger_artifact = narrow_build # type: ignore[method-assign] + captured = (0, 0, str(rpc.tip), "external_tip", time.monotonic()) + server._prepared_payout_state_candidate(captured) + server._prepared_payout_state_candidate(captured, force_full_window_rescan=True) + self.assertEqual(seen, [(0, 1, False), (0, 1, True)]) + with self.assertRaises(TypeError): + server._prepared_payout_state_candidate( + captured, + force_prior_balances_read=True, + ) + + +class FullRescanTelemetryTests(QuietCoordinatorTestCase): + def test_every_recorded_reason_and_path_is_in_the_contract(self) -> None: + server, _rpc, _ledger, _artifacts = configured_coordinator() + snapshot = ensure_accepted_preview_telemetry(server).snapshot() + for reason, path in snapshot["full_rescans"]: + self.assertIn(reason, PRISM_PAYOUT_WINDOW_FULL_RESCAN_REASONS) + self.assertIn(path, PRISM_PAYOUT_WINDOW_FULL_RESCAN_PATHS) + + def test_cache_lifecycle_rescans_record_their_fixed_reason(self) -> None: + server, _rpc, ledger, artifacts = configured_coordinator() + telemetry = ensure_accepted_preview_telemetry(server) + with patch("lab.prism.prism_coordinator.now_ms", return_value=1_000_000): + initial = server._build_payout_ledger_artifact( + 0, 0, artifacts.network_difficulty + ) + assert initial is not None + # A late-visible append bumps the epoch without waiting for the + # preparation lock; the next build must fail closed to the oracle. + with server._job_cache_lock: + server._payout_ledger_append_invalidation_epoch += 1 + appended = server._build_payout_ledger_artifact( + 0, 0, artifacts.network_difficulty + ) + assert appended is not None + self.assertEqual(appended.window_full_rescan_reason, "late_visible_append") + # An explicit append-side invalidation leaves its one-shot reason. + with server._payout_state_prepare_lock: + server._incremental_payout_artifact_window = None + server._incremental_payout_artifact_window_invalidation_reason = ( + "cache_invalidated" + ) + invalidated = server._build_payout_ledger_artifact( + 0, 0, artifacts.network_difficulty + ) + assert invalidated is not None + self.assertEqual(invalidated.window_full_rescan_reason, "cache_invalidated") + + self.assertEqual(ledger.full_snapshot_calls, 3) + rescans = telemetry.snapshot()["full_rescans"] + self.assertEqual(rescans[("cold_start", "in_process")]["count"], 1) + self.assertEqual(rescans[("late_visible_append", "in_process")]["count"], 1) + self.assertEqual(rescans[("cache_invalidated", "in_process")]["count"], 1) + self.assertEqual(full_rescan_total(telemetry.snapshot()), 3) + for stats in rescans.values(): + self.assertGreaterEqual(stats["sum"], 0.0) + self.assertGreaterEqual(stats["max"], 0.0) + + def test_daemon_fold_records_the_daemon_path(self) -> None: + server, _rpc, _ledger, artifacts = configured_coordinator() + telemetry = ensure_accepted_preview_telemetry(server) + service = server._ensure_payout_state_service() + + def daemon_fold(*, records, snapshot_anchor_ms, reason, **_kwargs): + shares_json = tuple(record.to_prism_json() for record in records) + return _PayoutWindowMaterialization( + shares_json=shares_json, + share_snapshot_sha256=service._canonical_json_sha256(shares_json), + snapshot_anchor_ms=int(snapshot_anchor_ms), + mode="full_rescan", + record_count=len(shares_json), + stats=IncrementalWindowAdvanceStats(0, 0, 0), + full_rescan_reason=reason, + ) + + with ( + patch.object(service, "_window_pipeline_rust_enabled", return_value=True), + patch.object(service, "_daemon_full_window_materialization", daemon_fold), + patch("lab.prism.prism_coordinator.now_ms", return_value=1_000_000), + ): + built = server._build_payout_ledger_artifact( + 0, 0, artifacts.network_difficulty + ) + + assert built is not None + self.assertEqual(built.window_build_mode, "full_rescan") + rescans = telemetry.snapshot()["full_rescans"] + self.assertEqual(rescans[("cold_start", "daemon")]["count"], 1) + self.assertEqual(rescans[("cold_start", "in_process")]["count"], 0) + + def test_periodic_self_check_records_the_periodic_reason_in_process( + self, + ) -> None: + server, _rpc, ledger, artifacts = configured_coordinator() + telemetry = ensure_accepted_preview_telemetry(server) + server.payout_artifact_min_build_interval_seconds = 0.0 + server.payout_artifact_full_rescan_seconds = 0.0 + with patch("lab.prism.prism_coordinator.now_ms", return_value=1_000_000): + initial = server._build_payout_ledger_artifact( + 0, 0, artifacts.network_difficulty + ) + assert initial is not None + checked = server._build_payout_ledger_artifact( + 0, 0, artifacts.network_difficulty + ) + + assert checked is not None + self.assertEqual(checked.window_build_mode, "self_check_match") + self.assertEqual(ledger.full_snapshot_calls, 2) + rescans = telemetry.snapshot()["full_rescans"] + self.assertEqual(rescans[("cold_start", "in_process")]["count"], 1) + self.assertEqual(rescans[("periodic_self_check", "in_process")]["count"], 1) + + def test_failed_self_check_records_its_failure_reason(self) -> None: + server, _rpc, ledger, artifacts = configured_coordinator() + telemetry = ensure_accepted_preview_telemetry(server) + server.payout_artifact_min_build_interval_seconds = 0.0 + server.payout_artifact_full_rescan_seconds = 0.0 + with patch("lab.prism.prism_coordinator.now_ms", return_value=1_000_000): + initial = server._build_payout_ledger_artifact( + 0, 0, artifacts.network_difficulty + ) + assert initial is not None + ledger.fail_full_snapshot = True + checked = server._build_payout_ledger_artifact( + 0, 0, artifacts.network_difficulty + ) + + assert checked is not None + self.assertEqual(checked.window_build_mode, "incremental_self_check_failed") + rescans = telemetry.snapshot()["full_rescans"] + self.assertEqual( + rescans[("periodic_self_check_failed", "in_process")]["count"], 1 + ) + + def test_a_rescan_that_dies_with_the_ledger_still_owns_its_time(self) -> None: + server, _rpc, ledger, artifacts = configured_coordinator() + telemetry = ensure_accepted_preview_telemetry(server) + ledger.fail_full_snapshot = True + with patch("lab.prism.prism_coordinator.now_ms", return_value=1_000_000): + self.assertIsNone( + server._build_payout_ledger_artifact( + 0, 0, artifacts.network_difficulty + ) + ) + rescans = telemetry.snapshot()["full_rescans"] + self.assertEqual(rescans[("cold_start", "in_process")]["count"], 1) + + +# --- coordinator integration ------------------------------------------------ + + +class CoordinatorReconcileIntegrationTests(QuietCoordinatorTestCase): + """The real adapters: writer admission, mutation lock, service core.""" + + def reconciling_coordinator(self): + server, rpc, ledger, artifacts = configured_coordinator() + server.reorg_reconciler_enabled = True + # Every entry point must run a real pass, not a memo hit. + server.reorg_reconcile_cache_seconds = 0.0 + return server, rpc, ledger, artifacts + + def test_maturation_reuses_the_window_and_publishes_reread_balances( + self, + ) -> None: + server, rpc, ledger, _artifacts = self.reconciling_coordinator() + telemetry = ensure_accepted_preview_telemetry(server) + tip = str(rpc.tip) + with patch("lab.prism.prism_coordinator.now_ms", return_value=1_000_000): + first = server.reconcile_prism_pool_blocks_once(tip_hash=tip) + self.assertEqual(first["published_generation"], 1) + self.assertEqual(ledger.full_snapshot_calls, 1) + with server._job_cache_lock: + first_published = server._published_payout_state.artifact + assert first_published is not None + self.assertEqual(first_published.prior_balances()[0]["balance_sats"], 546) + + ledger.matured_pending = 1 + reads_before = ledger.prior_balance_reads + second = server.reconcile_prism_pool_blocks_once(tip_hash=tip) + + self.assertEqual(second["matured_payouts"], 1) + self.assertEqual(second["published_generation"], 2) + # The exact window was reused: no second oracle read anywhere. + self.assertEqual(ledger.full_snapshot_calls, 1) + self.assertEqual(server.payout_artifact_event_counts["full_rescan"], 1) + # The balances the maturation moved were reread and published. + self.assertEqual(ledger.prior_balance_reads, reads_before + 1) + with server._job_cache_lock: + published = server._published_payout_state.artifact + assert published is not None + self.assertEqual(published.generation, 2) + self.assertEqual(published.prior_balances()[0]["balance_sats"], 777) + self.assertNotEqual( + published.prior_balances_sha256, first_published.prior_balances_sha256 + ) + # Attribution: two ``other`` passes that crossed the real adapter, + # each preparing and publishing; one cold-start rescan, and no + # reconcile_invalidation rescan at all. + snapshot = telemetry.snapshot() + self.assertEqual(pass_counts(snapshot)["other"], 2) + self.assertEqual(step_count(snapshot, "other", "admission_wait"), 2) + # Per pass: the watch query plus the stranded-prepared sweep read. + self.assertEqual(step_count(snapshot, "other", "watch_query"), 4) + self.assertEqual(step_count(snapshot, "other", "chain_probe"), 2) + self.assertEqual(step_count(snapshot, "other", "mutations"), 2) + self.assertEqual(step_count(snapshot, "other", "candidate_prepare"), 2) + self.assertEqual(step_count(snapshot, "other", "publish"), 2) + rescans = snapshot["full_rescans"] + self.assertEqual(rescans[("cold_start", "in_process")]["count"], 1) + self.assertEqual(rescans[("reconcile_invalidation", "in_process")]["count"], 0) + self.assertEqual(rescans[("reconcile_invalidation", "daemon")]["count"], 0) + self.assertEqual(full_rescan_total(snapshot), 1) + + def test_entry_points_attribute_through_the_real_adapters(self) -> None: + server, rpc, _ledger, _artifacts = self.reconciling_coordinator() + telemetry = ensure_accepted_preview_telemetry(server) + tip = str(rpc.tip) + with patch("lab.prism.prism_coordinator.now_ms", return_value=1_000_000): + self.assertTrue( + server.ensure_reorg_reconciled_for_tip(tip, _coalesce_same_tip=False) + ) + confirmed = server.reconcile_prism_pool_blocks_once( + tip_hash=tip, + _force_publish=True, + _source_reserved=True, + ) + self.assertIsNotNone(confirmed["published_generation"]) + self.assertTrue(server.ensure_reorg_reconciled_for_current_tip()) + prefetch = server._submit_reconcile_prefetch(tip) + assert prefetch is not None + try: + self.assertTrue(prefetch.result(timeout=10.0)) + finally: + server.shutdown_reconcile_prefetch_executor() + self.assertTrue(server.ensure_reorg_reconciled_for_tip(tip)) + + counts = pass_counts(telemetry.snapshot()) + self.assertEqual(counts["landing"], 1) + self.assertEqual(counts["post_confirm"], 1) + self.assertEqual(counts["job_build"], 1) + self.assertEqual(counts["tip_refresh"], 1) + self.assertEqual(counts["other"], 1) + snapshot = telemetry.snapshot() + for caller in PRISM_REORG_RECONCILE_CALLERS: + self.assertEqual(step_count(snapshot, caller, "admission_wait"), 1, caller) + # The service records into the same owner the renderer reads. + self.assertIs( + server._ensure_reorg_reconciler_service().accepted_preview_telemetry, + telemetry, + ) + + def test_landed_preview_refusal_records_no_pass(self) -> None: + server, rpc, _ledger, _artifacts = self.reconciling_coordinator() + telemetry = ensure_accepted_preview_telemetry(server) + tip = str(rpc.tip) + server._begin_accepted_block_payout_preview("aa" * 32, block_height=11) + server._mark_accepted_block_payout_landed("aa" * 32, block_height=11) + with self.assertRaises(PayoutStatePublicationBlocked): + server.reconcile_prism_pool_blocks_once(tip_hash=tip) + self.assertEqual(sum(pass_counts(telemetry.snapshot()).values()), 0) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_prism_issue_224_landing.py b/tests/test_prism_issue_224_landing.py new file mode 100644 index 000000000..a26da2f86 --- /dev/null +++ b/tests/test_prism_issue_224_landing.py @@ -0,0 +1,1094 @@ +#!/usr/bin/env python3 +"""Accepted-landing attribution for issue #224. + +``PrismAcceptedPreviewPublicationLatencyHigh`` proves a tail exists between +definitive node acceptance and the visible payout preview, but until this lane +nothing said *which stretch of the landing* owned a sample. These tests pin +the properties that make the answer trustworthy rather than merely present: + +* the phase label vocabulary is closed -- the seven names of + ``PRISM_ACCEPTED_LANDING_PHASES`` and nothing a call site can compute; +* the interval keeps its shipped definition -- it starts at definitive node + acceptance, not at the pre-RPC landed fence that is armed before qbitd has + seen the block, and it closes once at the first publication; +* one publication yields exactly one retained diagnostic and one structured + line, and a retry or republication yields none; +* a block hash and height reach the diagnostic and the log line only, never a + metric label; +* the in-flight per-hash timing state cannot leak -- terminal abandonment + drops it and the map is bounded exactly as the acceptance stamps are; +* the landing itself is unchanged apart from the telemetry: the balance + serializer is still taken once and released once around the same body, and + a landing still lands, confirms and publishes. +""" + +from __future__ import annotations + +import ast +import json +import threading +import unittest +from contextlib import contextmanager +from io import StringIO +from pathlib import Path +from types import SimpleNamespace +from typing import Any, Iterator +from unittest import mock + +import lab.prism.block_candidates as block_candidates_module +import lab.prism.block_finalization as block_finalization_module +from lab.prism.accepted_preview_telemetry import ( + AcceptedPreviewPublicationDiagnostic, + LANDING_PHASE_BALANCE_LOCK_WAIT, + LANDING_PHASE_CHAIN_PROBE, + LANDING_PHASE_LANE_WAIT, + LANDING_PHASE_PREVIEW_PREPARE, + LANDING_PHASE_PREVIEW_PUBLISH, + LANDING_PHASE_PRIOR_BALANCES_CHECK, + LANDING_PHASE_RECONCILE, + PRISM_ACCEPTED_LANDING_PHASES, + RECONCILE_CALLER_LANDING, + ensure_accepted_preview_telemetry, +) +from lab.prism.block_candidates import ( + MAX_ACCEPTED_BLOCK_PREVIEW_PUBLICATION_STAMPS, + PRISM_ACCEPTED_LANDING_LOG_EVENT, +) +from tests.prism_vardiff_test_support import ( + SubmitRpc, + block_candidate, + submit_coordinator, + tempfile, + verified_audit_report, + verified_block_bundle, +) + + +OWNED_MODULES = (block_candidates_module, block_finalization_module) +# Every call whose argument list carries a landing phase name. A phase that is +# a computed string anywhere in this list is a new metric series. +PHASE_ARGUMENT_CALLS = frozenset( + { + "observe_landing_phase", + "landing_phase", + "_landing_phase", + "_accepted_landing_phase", + "_commit_accepted_landing_phase", + } +) + + +def _module_tree(module: Any) -> ast.Module: + return ast.parse(Path(module.__file__).read_text(encoding="utf-8")) + + +def _builds_a_string(node: ast.AST) -> bool: + """Whether this expression produces a string the source did not fix.""" + if isinstance(node, ast.Constant): + return isinstance(node.value, str) + if isinstance(node, ast.JoinedStr): + return True + if isinstance(node, ast.BinOp): + return any( + _builds_a_string(operand) for operand in (node.left, node.right) + ) + if isinstance(node, ast.Call) and isinstance(node.func, ast.Attribute): + return node.func.attr in {"format", "join", "lower", "upper"} + return False + + +def _call_name(node: ast.Call) -> str | None: + if isinstance(node.func, ast.Attribute): + return node.func.attr + if isinstance(node.func, ast.Name): + return node.func.id + return None + + +class LandingPhaseVocabularyTests(unittest.TestCase): + """The seven fixed labels, and no way for a call site to invent an eighth.""" + + def test_owned_modules_reference_every_contract_phase_constant(self) -> None: + referenced: set[str] = set() + for module in OWNED_MODULES: + for node in ast.walk(_module_tree(module)): + if isinstance(node, ast.Name) and node.id.startswith( + "LANDING_PHASE_" + ): + referenced.add(node.id) + self.assertEqual( + referenced, + { + "LANDING_PHASE_LANE_WAIT", + "LANDING_PHASE_BALANCE_LOCK_WAIT", + "LANDING_PHASE_RECONCILE", + "LANDING_PHASE_PRIOR_BALANCES_CHECK", + "LANDING_PHASE_CHAIN_PROBE", + "LANDING_PHASE_PREVIEW_PREPARE", + "LANDING_PHASE_PREVIEW_PUBLISH", + }, + ) + self.assertEqual( + tuple(sorted(PRISM_ACCEPTED_LANDING_PHASES)), + tuple( + sorted( + ( + LANDING_PHASE_LANE_WAIT, + LANDING_PHASE_BALANCE_LOCK_WAIT, + LANDING_PHASE_RECONCILE, + LANDING_PHASE_PRIOR_BALANCES_CHECK, + LANDING_PHASE_CHAIN_PROBE, + LANDING_PHASE_PREVIEW_PREPARE, + LANDING_PHASE_PREVIEW_PUBLISH, + ) + ) + ), + ) + + def test_no_phase_argument_is_a_computed_string(self) -> None: + # A literal, an f-string, or a concatenation reaching one of these + # calls is how a dynamic label gets born. Names are the only shape + # allowed, and the recorder validates the name against the contract. + offenders: list[str] = [] + for module in OWNED_MODULES: + for node in ast.walk(_module_tree(module)): + if not isinstance(node, ast.Call): + continue + if _call_name(node) not in PHASE_ARGUMENT_CALLS: + continue + arguments = list(node.args) + [ + keyword.value for keyword in node.keywords + ] + for argument in arguments: + if _builds_a_string(argument): + offenders.append( + f"{Path(module.__file__).name}:{node.lineno}" + ) + self.assertEqual(offenders, []) + + def test_unknown_phase_names_are_refused_before_the_body_runs(self) -> None: + server, _state, _ledger = submit_coordinator() + service = server._ensure_block_candidate_service() + entered = False + + with self.assertRaises(ValueError): + with service._accepted_landing_phase("ab" * 32, "not_a_phase"): + entered = True # pragma: no cover - the guard runs first + self.assertFalse(entered) + + telemetry = ensure_accepted_preview_telemetry(server) + with self.assertRaises(ValueError): + telemetry.observe_landing_phase("not_a_phase", 1.0) + + def test_phase_metric_cells_carry_only_the_seven_fixed_labels(self) -> None: + server, _state, _ledger = submit_coordinator() + service = server._ensure_block_candidate_service() + block_hash = "ab" * 32 + service._note_accepted_block_preview_acceptance(block_hash) + for phase in PRISM_ACCEPTED_LANDING_PHASES: + with service._accepted_landing_phase(block_hash, phase): + pass + + snapshot = ensure_accepted_preview_telemetry(server).snapshot() + self.assertEqual( + set(snapshot["landing_phases"]), + set(PRISM_ACCEPTED_LANDING_PHASES), + ) + for phase in PRISM_ACCEPTED_LANDING_PHASES: + self.assertEqual(snapshot["landing_phases"][phase]["count"], 1) + # Nothing anywhere in the metric cells names a block. + self.assertNotIn(block_hash, repr(snapshot)) + + +class LandingLaneWaitTests(unittest.TestCase): + """``lane_wait`` measures from acceptance, not from the landed fence.""" + + def _service(self) -> tuple[Any, Any]: + server, _state, _ledger = submit_coordinator() + return server, server._ensure_block_candidate_service() + + def test_lane_wait_starts_at_definitive_acceptance(self) -> None: + server, service = self._service() + block_hash = "ab" * 32 + with mock.patch( + "lab.prism.block_candidates.time.monotonic", + side_effect=[100.0, 104.5], + ): + service._note_accepted_block_preview_acceptance(block_hash) + service._close_accepted_landing_lane_wait(block_hash, block_height=10) + + snapshot = ensure_accepted_preview_telemetry(server).snapshot() + lane_wait = snapshot["landing_phases"][LANDING_PHASE_LANE_WAIT] + self.assertEqual(lane_wait["count"], 1) + self.assertAlmostEqual(float(lane_wait["sum"]), 4.5) + self.assertEqual( + service.accepted_landing_attribution_snapshot()[block_hash][ + LANDING_PHASE_LANE_WAIT + ], + 4.5, + ) + + def test_pre_rpc_landed_fencing_does_not_start_the_lane_wait(self) -> None: + # _mark_accepted_block_payout_landed arms the landed barrier *before* + # the submitblock RPC, so a lane wait measured from it would include + # the offer itself and would restart on every retry. Only the + # definitive-acceptance stamp opens an interval. + server, service = self._service() + block_hash = "cd" * 32 + server._begin_accepted_block_payout_preview(block_hash, block_height=10) + server._mark_accepted_block_payout_landed(block_hash, block_height=10) + + service._close_accepted_landing_lane_wait(block_hash, block_height=10) + + snapshot = ensure_accepted_preview_telemetry(server).snapshot() + self.assertEqual( + snapshot["landing_phases"][LANDING_PHASE_LANE_WAIT]["count"], + 0, + ) + self.assertEqual(service.accepted_landing_attribution_snapshot(), {}) + + def test_lane_wait_is_recorded_once_per_open_interval(self) -> None: + # A retried landing re-enters the lane. Measuring again from the same + # acceptance stamp would charge the retry's queue time to a stretch + # the first attempt already owns, and would keep growing it. + server, service = self._service() + block_hash = "ef" * 32 + with mock.patch( + "lab.prism.block_candidates.time.monotonic", + side_effect=[10.0, 11.0, 30.0, 90.0], + ): + service._note_accepted_block_preview_acceptance(block_hash) + service._close_accepted_landing_lane_wait(block_hash) + service._close_accepted_landing_lane_wait(block_hash) + service._close_accepted_landing_lane_wait(block_hash) + + lane_wait = ensure_accepted_preview_telemetry(server).snapshot()[ + "landing_phases" + ][LANDING_PHASE_LANE_WAIT] + self.assertEqual(lane_wait["count"], 1) + self.assertAlmostEqual(float(lane_wait["sum"]), 1.0) + + def test_a_candidate_shape_without_a_hash_starts_nothing(self) -> None: + _server, service = self._service() + service._note_accepted_landing_lane_start(SimpleNamespace()) + self.assertEqual(service.accepted_landing_attribution_snapshot(), {}) + + def _candidate(self, block_hash: str, height: int = 10) -> Any: + return SimpleNamespace( + submission=SimpleNamespace(block_hash_hex=block_hash), + context=SimpleNamespace(template={"height": height}), + ) + + def test_a_lane_closed_before_the_stamp_is_closed_again_after_it( + self, + ) -> None: + # A fresh candidate carries no acceptance stamp until the node offer + # the landing itself makes returns definitively, so the close that + # runs before that offer finds no interval to measure. The second + # close is what turns a first-pass landing into a recorded sample + # instead of a silent gap: a zero sample and a missing sample read + # identically in the diagnostic, and only the family count separates + # them. + server, service = self._service() + block_hash = "5c" * 32 + candidate = self._candidate(block_hash) + + service._note_accepted_landing_lane_start(candidate) + service._note_accepted_block_preview_acceptance(block_hash) + service._note_accepted_landing_lane_start(candidate) + + lane_wait = ensure_accepted_preview_telemetry(server).snapshot()[ + "landing_phases" + ][LANDING_PHASE_LANE_WAIT] + self.assertEqual(lane_wait["count"], 1) + self.assertEqual( + service.accepted_landing_attribution_snapshot()[block_hash][ + LANDING_PHASE_LANE_WAIT + ], + float(lane_wait["sum"]), + ) + + def test_a_retained_acceptance_keeps_the_earlier_tighter_sample( + self, + ) -> None: + # The reverse case: the stamp already existed, so the close before + # the retained offer is looked up owns the sample and the one after + # it must not extend the stretch with the landing's own work. + server, service = self._service() + block_hash = "6d" * 32 + candidate = self._candidate(block_hash) + with mock.patch( + "lab.prism.block_candidates.time.monotonic", + side_effect=[10.0, 12.0, 45.0], + ): + service._note_accepted_block_preview_acceptance(block_hash) + service._note_accepted_landing_lane_start(candidate) + service._note_accepted_landing_lane_start(candidate) + + lane_wait = ensure_accepted_preview_telemetry(server).snapshot()[ + "landing_phases" + ][LANDING_PHASE_LANE_WAIT] + self.assertEqual(lane_wait["count"], 1) + self.assertAlmostEqual(float(lane_wait["sum"]), 2.0) + + def test_the_lane_closes_before_the_attempt_marker_is_written(self) -> None: + # ``_mark_block_candidate_attempted`` is the landing's own PostgreSQL + # write, not queue time. Closing the lane after it charges the write + # to ``lane_wait`` -- and a marker that times out keeps charging the + # retry backoff there too, so the one stretch that exonerates the + # landing grows precisely when the writer is degraded. The ordering is + # the whole guarantee, so it is pinned where it is written. + tree = _module_tree(block_candidates_module) + ordered = 0 + for function in ast.walk(tree): + if not isinstance(function, (ast.FunctionDef, ast.AsyncFunctionDef)): + continue + lane_starts = [ + node.lineno + for node in ast.walk(function) + if isinstance(node, ast.Call) + and _call_name(node) == "_note_accepted_landing_lane_start" + ] + markers = [ + node.lineno + for node in ast.walk(function) + if isinstance(node, ast.Call) + and _call_name(node) == "_mark_block_candidate_attempted" + ] + if not lane_starts or not markers: + continue + offers = [ + node.lineno + for node in ast.walk(function) + if isinstance(node, ast.Call) + and _call_name(node) + == "_node_submission_for_candidate_or_retained" + ] + ordered += 1 + self.assertLess( + max(lane_starts), + min(markers), + f"{function.name} closes the lane after its attempt marker", + ) + # A fresh candidate is stamped by the offer itself, so a path that + # only closed the lane before making it would record no sample at + # all for every first-pass landing. + self.assertTrue( + offers and max(lane_starts) > max(offers), + f"{function.name} never closes the lane after its node offer", + ) + # Both queued landing paths write the marker; a rename that loses one + # would otherwise make this test vacuously true. + self.assertEqual(ordered, 2) + + +class LandingPublicationDiagnosticTests(unittest.TestCase): + """One closed interval, one retained diagnostic, one structured line.""" + + def _service(self) -> tuple[Any, Any]: + server, _state, _ledger = submit_coordinator() + return server, server._ensure_block_candidate_service() + + def _publish(self, server: Any, block_hash: str, result: str) -> list[dict]: + stream = StringIO() + with mock.patch("sys.stdout", stream): + server._observe_accepted_block_preview_publication( + block_hash, + result=result, + ) + lines = [] + for line in stream.getvalue().splitlines(): + prefix, _, payload = line.partition("prism coordinator: ") + if prefix or not payload.startswith("{"): + continue + record = json.loads(payload) + if record.get("event") == PRISM_ACCEPTED_LANDING_LOG_EVENT: + lines.append(record) + return lines + + def test_first_published_publication_closes_one_interval(self) -> None: + server, service = self._service() + block_hash = "ab" * 32 + service._note_accepted_block_preview_acceptance(block_hash) + with service._accepted_landing_phase( + block_hash, + LANDING_PHASE_RECONCILE, + block_height=10, + ): + pass + + logged = self._publish(server, block_hash, "published") + + telemetry = ensure_accepted_preview_telemetry(server) + diagnostics = telemetry.diagnostics_snapshot() + self.assertEqual(len(diagnostics), 1) + self.assertEqual(telemetry.snapshot()["diagnostics_recorded"], 1) + diagnostic = diagnostics[0] + self.assertEqual(diagnostic.block_hash, block_hash) + self.assertEqual(diagnostic.block_height, 10) + self.assertEqual(diagnostic.result, "published") + self.assertEqual(diagnostic.reconcile_caller, RECONCILE_CALLER_LANDING) + self.assertGreaterEqual(diagnostic.acceptance_to_publication_seconds, 0.0) + self.assertIsNotNone(diagnostic.recorded_monotonic) + self.assertEqual( + set(diagnostic.phase_seconds), + set(PRISM_ACCEPTED_LANDING_PHASES), + ) + self.assertEqual(len(logged), 1) + self.assertEqual(logged[0]["block_hash"], block_hash) + self.assertEqual(logged[0]["block_height"], 10) + self.assertEqual(logged[0]["result"], "published") + # The shipped histogram still closed exactly one interval. + histograms = service.accepted_block_preview_publication_snapshot() + self.assertEqual(histograms["published"]["count"], 1) + self.assertEqual(histograms["degraded"]["count"], 0) + + def test_first_degraded_publication_closes_one_interval(self) -> None: + # The fenced local-retention branch also makes the preview visible to + # waiting children, so it closes the interval and earns a diagnostic + # of its own -- labelled apart, never merged into the healthy one. + server, service = self._service() + block_hash = "cd" * 32 + service._note_accepted_block_preview_acceptance(block_hash) + + logged = self._publish(server, block_hash, "degraded") + + telemetry = ensure_accepted_preview_telemetry(server) + diagnostics = telemetry.diagnostics_snapshot() + self.assertEqual(len(diagnostics), 1) + self.assertEqual(diagnostics[0].result, "degraded") + self.assertEqual(len(logged), 1) + self.assertEqual(logged[0]["result"], "degraded") + histograms = service.accepted_block_preview_publication_snapshot() + self.assertEqual(histograms["degraded"]["count"], 1) + self.assertEqual(histograms["published"]["count"], 0) + + def test_retry_and_republication_do_not_double_count(self) -> None: + server, service = self._service() + block_hash = "ef" * 32 + service._note_accepted_block_preview_acceptance(block_hash) + first = self._publish(server, block_hash, "published") + + # A matching republication, a degraded relabel, and a second + # definitive offer of the same hash: none of them is a new interval. + again = self._publish(server, block_hash, "published") + relabelled = self._publish(server, block_hash, "degraded") + service._note_accepted_block_preview_acceptance(block_hash) + reoffered = self._publish(server, block_hash, "published") + + telemetry = ensure_accepted_preview_telemetry(server) + self.assertEqual(len(first), 1) + self.assertEqual(again, []) + self.assertEqual(relabelled, []) + self.assertEqual(reoffered, []) + self.assertEqual(len(telemetry.diagnostics_snapshot()), 1) + self.assertEqual(telemetry.snapshot()["diagnostics_recorded"], 1) + histograms = service.accepted_block_preview_publication_snapshot() + self.assertEqual(histograms["published"]["count"], 1) + self.assertEqual(histograms["degraded"]["count"], 0) + + def test_phases_after_the_interval_closed_reach_no_diagnostic(self) -> None: + # A retry's stretches still belong in the process-wide family -- they + # are landing work that ran -- but they can never be claimed by a + # landing whose interval is already measured and recorded. + server, service = self._service() + block_hash = "12" * 32 + service._note_accepted_block_preview_acceptance(block_hash) + self._publish(server, block_hash, "published") + + with service._accepted_landing_phase( + block_hash, + LANDING_PHASE_RECONCILE, + block_height=10, + ): + pass + + telemetry = ensure_accepted_preview_telemetry(server) + self.assertEqual(len(telemetry.diagnostics_snapshot()), 1) + self.assertEqual( + telemetry.diagnostics_snapshot()[0].phase_seconds[ + LANDING_PHASE_RECONCILE + ], + 0.0, + ) + self.assertEqual( + telemetry.snapshot()["landing_phases"][LANDING_PHASE_RECONCILE][ + "count" + ], + 1, + ) + self.assertEqual(service.accepted_landing_attribution_snapshot(), {}) + + def test_publication_without_an_acceptance_stamp_records_nothing(self) -> None: + server, _service = self._service() + logged = self._publish(server, "ab" * 32, "published") + telemetry = ensure_accepted_preview_telemetry(server) + self.assertEqual(logged, []) + self.assertEqual(telemetry.diagnostics_snapshot(), ()) + + def test_publish_span_is_closed_by_the_visibility_instant(self) -> None: + # The publication closes the interval from inside the publish call, so + # a plain timer would record ``preview_publish`` after the diagnostic + # was already built from an empty cell. + server, service = self._service() + block_hash = "34" * 32 + service._note_accepted_block_preview_acceptance(block_hash) + with service._accepted_landing_publish_span(block_hash, block_height=11): + server._observe_accepted_block_preview_publication( + block_hash, + result="published", + ) + + telemetry = ensure_accepted_preview_telemetry(server) + diagnostics = telemetry.diagnostics_snapshot() + self.assertEqual(len(diagnostics), 1) + self.assertEqual(diagnostics[0].block_height, 11) + self.assertGreater( + diagnostics[0].phase_seconds[LANDING_PHASE_PREVIEW_PUBLISH], + 0.0, + ) + # Closed exactly once: the observer took the span, and the context + # manager's exit found nothing left to record. + self.assertEqual( + telemetry.snapshot()["landing_phases"][ + LANDING_PHASE_PREVIEW_PUBLISH + ]["count"], + 1, + ) + + def test_a_publish_that_publishes_nothing_still_records_its_phase(self) -> None: + server, service = self._service() + block_hash = "56" * 32 + service._note_accepted_block_preview_acceptance(block_hash) + with service._accepted_landing_publish_span(block_hash): + pass + + telemetry = ensure_accepted_preview_telemetry(server) + self.assertEqual( + telemetry.snapshot()["landing_phases"][ + LANDING_PHASE_PREVIEW_PUBLISH + ]["count"], + 1, + ) + self.assertEqual(telemetry.diagnostics_snapshot(), ()) + + def test_hash_and_height_reach_the_diagnostic_and_log_only(self) -> None: + server, service = self._service() + block_hash = "78" * 32 + service._note_accepted_block_preview_acceptance(block_hash) + with service._accepted_landing_phase( + block_hash, + LANDING_PHASE_CHAIN_PROBE, + block_height=4242, + ): + pass + logged = self._publish(server, block_hash, "published") + + telemetry = ensure_accepted_preview_telemetry(server) + self.assertEqual(len(logged), 1) + self.assertEqual(logged[0]["block_hash"], block_hash) + self.assertEqual(logged[0]["block_height"], 4242) + for phase in PRISM_ACCEPTED_LANDING_PHASES: + self.assertIn(f"phase_{phase}_seconds", logged[0]) + + rendered = repr(telemetry.snapshot()) + self.assertNotIn(block_hash, rendered) + self.assertNotIn("4242", rendered) + self.assertNotIn( + block_hash, + repr(service.accepted_block_preview_publication_snapshot()), + ) + + def test_the_shipped_metrics_render_carries_no_hash_or_height(self) -> None: + # End to end through the renderer the alert is read from: the phase + # family is there, its only label is the fixed phase name, and the + # block this landing measured is nowhere in the exposition. + server, service = self._service() + block_hash = "bc" * 32 + service._note_accepted_block_preview_acceptance(block_hash) + with service._accepted_landing_phase( + block_hash, + LANDING_PHASE_RECONCILE, + block_height=4242, + ): + pass + self._publish(server, block_hash, "published") + + rendered = server._ensure_metrics_renderer().render() + + self.assertIn( + 'qbit_prism_accepted_block_landing_phase_seconds_count{phase="reconcile"} 1', + rendered, + ) + for phase in PRISM_ACCEPTED_LANDING_PHASES: + self.assertIn( + "qbit_prism_accepted_block_landing_phase_seconds_sum" + f'{{phase="{phase}"}}', + rendered, + ) + self.assertNotIn(block_hash, rendered) + self.assertNotIn("4242", rendered) + self.assertNotIn(PRISM_ACCEPTED_LANDING_LOG_EVENT, rendered) + + def test_rescan_and_ledger_totals_stay_at_the_neutral_values(self) -> None: + # This lane can obtain no per-landing causal evidence for either + # without reading a process-wide delta, and such a read would claim + # whatever else the process was doing as this block's work. The + # separately attributed families answer those questions instead. + server, service = self._service() + block_hash = "9a" * 32 + service._note_accepted_block_preview_acceptance(block_hash) + logged = self._publish(server, block_hash, "published") + + diagnostic = ensure_accepted_preview_telemetry(server).diagnostics_snapshot()[0] + neutral = AcceptedPreviewPublicationDiagnostic( + block_hash=block_hash, + block_height=None, + result="published", + acceptance_to_publication_seconds=0.0, + ) + self.assertEqual(diagnostic.full_rescan_reason, neutral.full_rescan_reason) + self.assertEqual(diagnostic.full_rescan_path, neutral.full_rescan_path) + self.assertEqual(diagnostic.full_rescan_seconds, neutral.full_rescan_seconds) + self.assertEqual( + diagnostic.ledger_gate_wait_seconds, + neutral.ledger_gate_wait_seconds, + ) + self.assertEqual( + diagnostic.ledger_execute_seconds, + neutral.ledger_execute_seconds, + ) + self.assertIsNone(logged[0]["full_rescan_reason"]) + self.assertEqual(logged[0]["ledger_execute_seconds"], 0.0) + + +class LandingAttributionCleanupTests(unittest.TestCase): + """In-flight per-hash timing state is bounded and cannot leak.""" + + def _service(self) -> tuple[Any, Any]: + server, _state, _ledger = submit_coordinator() + return server, server._ensure_block_candidate_service() + + def test_terminal_abandonment_discards_the_in_flight_state(self) -> None: + server, service = self._service() + block_hash = "ab" * 32 + service._note_accepted_block_preview_acceptance(block_hash) + with service._accepted_landing_phase( + block_hash, + LANDING_PHASE_RECONCILE, + ): + pass + self.assertIn(block_hash, service.accepted_landing_attribution_snapshot()) + + service._record_committed_block_candidate_abandonment( + block_hash, + SimpleNamespace(reason="submitblock-rejected", stale_job_class=None), + ) + + self.assertEqual(service.accepted_landing_attribution_snapshot(), {}) + # The metric family keeps the stretch that actually ran. + self.assertEqual( + ensure_accepted_preview_telemetry(server).snapshot()["landing_phases"][ + LANDING_PHASE_RECONCILE + ]["count"], + 1, + ) + + def test_a_retryable_abandonment_keeps_the_open_interval(self) -> None: + # A retryable disposition is not terminal: the same interval is still + # running and its next attempt will close it. + _server, service = self._service() + block_hash = "cd" * 32 + service._note_accepted_block_preview_acceptance(block_hash) + with service._accepted_landing_phase( + block_hash, + LANDING_PHASE_RECONCILE, + ): + pass + retryable = next(iter(service.retryable_reasons)) + + service._record_committed_block_candidate_abandonment( + block_hash, + SimpleNamespace(reason=retryable, stale_job_class=None), + ) + + self.assertIn(block_hash, service.accepted_landing_attribution_snapshot()) + + def test_in_flight_state_is_bounded_like_the_acceptance_stamps(self) -> None: + _server, service = self._service() + for index in range(MAX_ACCEPTED_BLOCK_PREVIEW_PUBLICATION_STAMPS + 64): + block_hash = f"{index:064x}" + service._note_accepted_block_preview_acceptance(block_hash) + with service._accepted_landing_phase( + block_hash, + LANDING_PHASE_RECONCILE, + ): + pass + + retained = service.accepted_landing_attribution_snapshot() + self.assertLessEqual( + len(retained), + MAX_ACCEPTED_BLOCK_PREVIEW_PUBLICATION_STAMPS, + ) + self.assertLessEqual( + len(service._accepted_block_preview_acceptance_monotonic), + MAX_ACCEPTED_BLOCK_PREVIEW_PUBLICATION_STAMPS, + ) + # Eviction is oldest-first, so the newest landing is the survivor. + newest = f"{MAX_ACCEPTED_BLOCK_PREVIEW_PUBLICATION_STAMPS + 63:064x}" + self.assertIn(newest, retained) + + def test_an_unstamped_publication_leaves_nothing_in_flight(self) -> None: + # A startup replay or a chain-proven candidate publishes without an + # in-process acceptance stamp, so the observer returns before it can + # pop anything. A record opened by the publish span would then report + # a finished publication as in flight for as long as the bound kept + # it. + server, service = self._service() + block_hash = "7b" * 32 + with service._accepted_landing_publish_span(block_hash, block_height=12): + server._observe_accepted_block_preview_publication( + block_hash, + result="published", + ) + + self.assertEqual(service.accepted_landing_attribution_snapshot(), {}) + telemetry = ensure_accepted_preview_telemetry(server) + self.assertEqual(telemetry.diagnostics_snapshot(), ()) + # The stretch still ran, so the phase family still owns its sample. + self.assertEqual( + telemetry.snapshot()["landing_phases"][ + LANDING_PHASE_PREVIEW_PUBLISH + ]["count"], + 1, + ) + + def test_unstamped_publications_never_evict_a_live_landing(self) -> None: + # The failure this guards is the replay storm: enough unattributable + # publications to fill the shared cap would push live landings out of + # it before their diagnostics were ever emitted. + server, service = self._service() + live = "01" * 32 + service._note_accepted_block_preview_acceptance(live) + with service._accepted_landing_phase(live, LANDING_PHASE_RECONCILE): + pass + for index in range(MAX_ACCEPTED_BLOCK_PREVIEW_PUBLICATION_STAMPS + 8): + with service._accepted_landing_publish_span(f"{index:064x}"): + pass + + self.assertEqual( + list(service.accepted_landing_attribution_snapshot()), + [live], + ) + server._observe_accepted_block_preview_publication( + live, + result="published", + ) + diagnostics = ensure_accepted_preview_telemetry(server).diagnostics_snapshot() + self.assertEqual([entry.block_hash for entry in diagnostics], [live]) + + def test_publication_pops_the_in_flight_state(self) -> None: + server, service = self._service() + block_hash = "ef" * 32 + service._note_accepted_block_preview_acceptance(block_hash) + with service._accepted_landing_phase( + block_hash, + LANDING_PHASE_RECONCILE, + ): + pass + server._observe_accepted_block_preview_publication( + block_hash, + result="published", + ) + self.assertEqual(service.accepted_landing_attribution_snapshot(), {}) + + def test_a_raising_stretch_still_records_and_is_not_masked(self) -> None: + server, service = self._service() + block_hash = "12" * 32 + service._note_accepted_block_preview_acceptance(block_hash) + + with self.assertRaisesRegex(RuntimeError, "reconcile exploded"): + with service._accepted_landing_phase( + block_hash, + LANDING_PHASE_RECONCILE, + ): + raise RuntimeError("reconcile exploded") + + telemetry = ensure_accepted_preview_telemetry(server) + self.assertEqual( + telemetry.snapshot()["landing_phases"][LANDING_PHASE_RECONCILE][ + "count" + ], + 1, + ) + self.assertIn( + LANDING_PHASE_RECONCILE, + service.accepted_landing_attribution_snapshot()[block_hash], + ) + + def test_a_raising_publish_span_still_records_and_is_not_masked(self) -> None: + server, service = self._service() + block_hash = "34" * 32 + service._note_accepted_block_preview_acceptance(block_hash) + + with self.assertRaisesRegex(RuntimeError, "publication exploded"): + with service._accepted_landing_publish_span(block_hash): + raise RuntimeError("publication exploded") + + self.assertEqual( + ensure_accepted_preview_telemetry(server).snapshot()["landing_phases"][ + LANDING_PHASE_PREVIEW_PUBLISH + ]["count"], + 1, + ) + + def test_a_refused_diagnostic_never_fails_a_publication(self) -> None: + # Attribution is observability: the preview is already visible to + # waiting children by the time the record is built. + server, service = self._service() + block_hash = "56" * 32 + service._note_accepted_block_preview_acceptance(block_hash) + with mock.patch.object( + block_candidates_module, + "AcceptedPreviewPublicationDiagnostic", + side_effect=ValueError("refused"), + ): + server._observe_accepted_block_preview_publication( + block_hash, + result="published", + ) + + telemetry = ensure_accepted_preview_telemetry(server) + self.assertEqual(telemetry.diagnostics_snapshot(), ()) + # The shipped interval still closed exactly once. + self.assertEqual( + service.accepted_block_preview_publication_snapshot()["published"][ + "count" + ], + 1, + ) + + +class LandingBehaviourCompatibilityTests(unittest.TestCase): + """The landing itself is unchanged apart from the telemetry it emits.""" + + def test_balance_serializer_is_taken_once_around_the_same_body(self) -> None: + # The acquisition is timed through an ExitStack rather than a plain + # ``with``. The lock must still be entered once before the body and + # released once after it, on the early-return path too. + server, state, _ledger = submit_coordinator() + service = server._ensure_block_finalization_service() + events: list[str] = [] + real_lock = server._block_submitter_lock + + @contextmanager + def recording(lock: Any, name: str) -> Iterator[None]: + events.append(f"enter:{name}") + try: + with real_lock(lock, name): + yield + finally: + events.append(f"exit:{name}") + + server._block_submitter_lock = recording # type: ignore[method-assign] + server._defer_for_pending_parent_payout_transition = ( # type: ignore[method-assign] + lambda **_kwargs: (events.append("body") or True) + ) + submission = SimpleNamespace( + coinbase_tx_hex="c0ffee", + block_hash_hex="94" * 32, + block_hex="00", + ) + candidate = block_candidate(server, state, submission) + + landed = service._land_and_confirm_block_candidate( + candidate, + current_tip="00" * 32, + already_active=False, + worker="miner-a", + node_submission=block_candidates_module._BlockCandidateNodeSubmission( + attempted=False + ), + ) + + self.assertIsNone(landed) + self.assertEqual( + events, + [ + "enter:payout-balance-mutation", + "body", + "exit:payout-balance-mutation", + ], + ) + # The wait was attributed even though the landing deferred. + self.assertEqual( + ensure_accepted_preview_telemetry(server).snapshot()["landing_phases"][ + LANDING_PHASE_BALANCE_LOCK_WAIT + ]["count"], + 1, + ) + # And the serializer is genuinely free again. + self.assertTrue(server._payout_balance_mutation_lock.acquire(blocking=False)) + server._payout_balance_mutation_lock.release() + + def test_prior_balances_reread_keeps_its_stamps_and_verdict(self) -> None: + server, _state, _ledger = submit_coordinator() + service = server._ensure_block_finalization_service() + stamps: list[str] = [] + server._record_block_submitter_phase = ( # type: ignore[method-assign] + lambda phase: stamps.append(phase) + ) + server.prior_balances_match_current = ( # type: ignore[method-assign] + lambda _balances: False + ) + + without_context = service._stamped_prior_balances_match_current([]) + with_context = service._stamped_prior_balances_match_current( + [], + block_hash="ab" * 32, + block_height=10, + ) + + self.assertFalse(without_context) + self.assertFalse(with_context) + self.assertEqual( + stamps, + [ + "prior-balances-check", + "prior-balances-check:complete", + "prior-balances-check", + "prior-balances-check:complete", + ], + ) + # Only the landing-scoped call is a landing phase; the same helper + # called without a hash still times nothing. + self.assertEqual( + ensure_accepted_preview_telemetry(server).snapshot()["landing_phases"][ + LANDING_PHASE_PRIOR_BALANCES_CHECK + ]["count"], + 1, + ) + + def test_a_landing_still_lands_confirms_and_publishes(self) -> None: + server, state, ledger = submit_coordinator() + block_hash = "94" * 32 + with tempfile.TemporaryDirectory() as tmp: + server.audit_dir = Path(tmp) + server.evidence_path = Path(tmp) / "evidence.json" + server.ledger_writer_public_key_hex = "aa" * 32 + server._ensure_audit_artifact_store() + server.rpc = SubmitRpc( + tip="00" * 32, + block_hash=block_hash, + ledger=ledger, + ) + server.build_audit_bundle = ( # type: ignore[method-assign] + lambda **_kwargs: verified_block_bundle() + ) + server.verify_bundle = ( # type: ignore[method-assign] + lambda *_args, **_kwargs: verified_audit_report() + ) + submission = SimpleNamespace( + coinbase_tx_hex="c0ffee", + block_hash_hex=block_hash, + block_hex="00", + ) + candidate = block_candidate(server, state, submission) + + self.assertTrue(server.submit_block_candidate(candidate)) + + self.assertEqual(server.accepted_block_count, 1) + service = server._ensure_block_candidate_service() + histograms = service.accepted_block_preview_publication_snapshot() + published = int(histograms["published"]["count"]) + degraded = int(histograms["degraded"]["count"]) + self.assertEqual(published + degraded, 1) + + telemetry = ensure_accepted_preview_telemetry(server) + diagnostics = telemetry.diagnostics_snapshot() + self.assertEqual(len(diagnostics), 1) + diagnostic = diagnostics[0] + self.assertEqual(diagnostic.block_hash, block_hash) + self.assertEqual(diagnostic.block_height, 10) + self.assertEqual(diagnostic.reconcile_caller, RECONCILE_CALLER_LANDING) + # The measured stretches are stretches *of* the interval they close, + # so they can never sum past it. + self.assertLessEqual( + sum(diagnostic.phase_seconds.values()), + diagnostic.acceptance_to_publication_seconds + 1e-6, + ) + # Nothing is left in flight once the landing completed. + self.assertEqual(service.accepted_landing_attribution_snapshot(), {}) + + def test_the_service_records_nothing_without_the_b1_seam(self) -> None: + # The focused fakes that drive finalization directly carry no + # block-candidate service; the timers must degrade to no-ops rather + # than reaching through a runtime that has no seam. + service = block_finalization_module.BlockFinalizationService( + SimpleNamespace() + ) + self.assertIsNone(service._accepted_landing_owner()) + with service._landing_phase( + LANDING_PHASE_RECONCILE, + block_hash="ab" * 32, + ): + pass + with service._landing_publish_span(block_hash="ab" * 32): + pass + self.assertNotIn( + "_accepted_preview_telemetry", + vars(service.runtime), + ) + + +class LandingAttributionThreadingTests(unittest.TestCase): + """The recorders never nest the leaf lock under anything of their own.""" + + def test_concurrent_landings_keep_their_own_attribution(self) -> None: + server, _state, _ledger = submit_coordinator() + service = server._ensure_block_candidate_service() + hashes = [f"{index:064x}" for index in range(8)] + for block_hash in hashes: + service._note_accepted_block_preview_acceptance(block_hash) + + barrier = threading.Barrier(len(hashes)) + + def land(block_hash: str) -> None: + barrier.wait() + with service._accepted_landing_phase( + block_hash, + LANDING_PHASE_RECONCILE, + block_height=10, + ): + pass + server._observe_accepted_block_preview_publication( + block_hash, + result="published", + ) + + threads = [ + threading.Thread(target=land, args=(block_hash,)) + for block_hash in hashes + ] + for thread in threads: + thread.start() + for thread in threads: + thread.join() + + telemetry = ensure_accepted_preview_telemetry(server) + diagnostics = telemetry.diagnostics_snapshot() + self.assertEqual( + sorted(diagnostic.block_hash for diagnostic in diagnostics), + sorted(hashes), + ) + self.assertEqual( + telemetry.snapshot()["landing_phases"][LANDING_PHASE_RECONCILE][ + "count" + ], + len(hashes), + ) + self.assertEqual(service.accepted_landing_attribution_snapshot(), {}) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_prism_issue_224_ledger.py b/tests/test_prism_issue_224_ledger.py new file mode 100644 index 000000000..58ecc2d50 --- /dev/null +++ b/tests/test_prism_issue_224_ledger.py @@ -0,0 +1,759 @@ +#!/usr/bin/env python3 +"""Issue #224 Wave 1: attribution for the ledger reads a preview waits on. + +``PrismAcceptedPreviewPublicationLatencyHigh`` fired while the coordinator +stayed healthy, and nothing on ``/metrics`` said which stretch of the landing +owned the sample. Four ledger reads sit in that stretch -- the payout-window +snapshot and its incremental delta at job issue, and the two prior-balances +reads -- and each of them can be slow for either of two unrelated reasons: +coordinator-local admission (a convoy behind an accounting write) or +PostgreSQL execution. One duration covering both cannot tell an operator +which, which is exactly how #211 was misdiagnosed on the enumeration path +before it was split. + +This wave routes those four through the same attributed read helper the +enumeration already uses, under the closed operation vocabulary +``lab.prism.accepted_preview_telemetry`` froze in Wave 0. The scenarios here +pin the two things that could go wrong while doing it: + +* the *measurement* is right -- the fixed operation name, gate wait apart + from execution, an admission expiry counted as a gate timeout with no + execution sample, a statement expiry counted as an execution timeout; +* the *behavior* is unchanged -- each read keeps the admission primitive it + already had (the writer lock stays the writer lock, the read slot stays the + read slot), sends byte-identical SQL, returns identical records, gives the + primitive back on every failure path, and preserves the original exception. + +The gates below are real ``Lock``/``BoundedSemaphore`` objects behind a +recording wrapper, so "the read released what it took" is a fact about an +actual primitive rather than about a counter. The clock is scripted, so the +recorded seconds are exact. +""" + +from __future__ import annotations + +import copy +import re +import sys +import threading +import unittest +from pathlib import Path +from typing import Any + +sys.path.insert(0, str(Path(__file__).resolve().parents[1])) + +from lab.prism.accepted_preview_telemetry import ( # noqa: E402 + PRISM_LEDGER_READ_OPERATIONS, + fold_ledger_read_stats, +) +from lab.prism.share_ledger import ( # noqa: E402 + AcceptedShareRecord, + LedgerOperationTimeout, + PsqlShareLedger, + ReadOnlyLedgerError, + _RefusingWriterGate, +) + +REPO_ROOT = Path(__file__).resolve().parents[1] + +BLOCK_HASH = "aa" * 32 + +# The operation each method must record under, and the admission primitive it +# must keep. Both halves are the contract: an attribution that renamed the +# operation would break the join with the Wave 0 vocabulary, and one that +# moved a read between primitives would change which writes it serializes +# against -- a concurrency change wearing a metrics change's clothes. +READ_SLOT = "read slot" +WRITER_LOCK = "writer lock" + +# The longest a scenario will wait for admission before calling the gate +# stranded. Every legitimate acquisition below is uncontended and returns +# immediately, so this is never reached by a passing run -- it exists so a +# helper that stopped releasing its gate fails a scenario instead of parking +# the suite on a primitive nobody will ever give back. +STRANDED_GATE_SECONDS = 2.0 + + +class _Clock: + """A monotonic clock these scenarios advance explicitly. + + Every recorded duration below comes from this clock, so a gate wait of + 0.25s is 0.25s in the assertion whether or not the primitive was really + contended and whatever the host was doing at the time. + """ + + def __init__(self, start: float = 1_000.0) -> None: + self.now = float(start) + + def __call__(self) -> float: + return self.now + + def advance(self, seconds: float) -> None: + self.now += float(seconds) + + +class _RecordingGate: + """A real admission primitive that also reports who took it, and when. + + Deliberately a wrapper rather than a stub. The failure scenarios assert + that a read which raises still hands its gate back, and a counter-only + fake cannot tell a released primitive from a leaked one -- the next + acquisition would succeed either way. Wrapping a real ``Lock`` / + ``BoundedSemaphore`` makes a leak show up where it would in production, + as the next caller failing to be admitted (bounded here by + ``STRANDED_GATE_SECONDS`` so the failure is a report, not a hang). + """ + + def __init__( + self, + name: str, + primitive: Any, + journal: list[str], + clock: _Clock, + *, + wait_seconds: float = 0.0, + ) -> None: + self.name = name + self.primitive = primitive + self.journal = journal + self.clock = clock + self.wait_seconds = wait_seconds + self.admissions = 0 + self.releases = 0 + + def acquire(self, timeout: float | None = None) -> bool: + self.journal.append(f"{self.name}:acquire") + acquired = self.primitive.acquire( + timeout=STRANDED_GATE_SECONDS if timeout is None else timeout + ) + # Admission wall-clock is scripted, and is charged whether admission + # succeeded or expired: a caller that waited out its budget waited. + self.clock.advance(self.wait_seconds) + if acquired: + self.admissions += 1 + self.journal.append(f"{self.name}:{'admitted' if acquired else 'expired'}") + return acquired + + def release(self) -> None: + self.releases += 1 + self.journal.append(f"{self.name}:release") + self.primitive.release() + + @property + def outstanding(self) -> int: + return self.admissions - self.releases + + def is_free(self) -> bool: + """Whether the wrapped primitive can still be taken.""" + if not self.primitive.acquire(timeout=0): + return False + self.primitive.release() + return True + + +class _AttributedReadLedger(PsqlShareLedger): + """A ledger carrying only the read plumbing these scenarios touch. + + Built through ``__new__`` in the style the rest of the ledger suite uses + for single-statement scenarios: the class under test is the shipped + ``PsqlShareLedger``, and only the seams a read actually reaches are + supplied. + """ + + def __init__( + self, + results: list[Any], + *, + gate_wait_seconds: float = 0.0, + execute_seconds: float = 0.0, + read_concurrency: int = 1, + ) -> None: + self.clock = _Clock() + self.journal: list[str] = [] + self.queries: list[str] = [] + self.results = list(results) + self.execute_seconds = execute_seconds + self._monotonic = self.clock + self._native = None + self.writer_gate = _RecordingGate( + WRITER_LOCK, + threading.Lock(), + self.journal, + self.clock, + wait_seconds=gate_wait_seconds, + ) + self.read_gate = _RecordingGate( + READ_SLOT, + threading.BoundedSemaphore(read_concurrency), + self.journal, + self.clock, + wait_seconds=gate_wait_seconds, + ) + self._lock = self.writer_gate + self._read_semaphore = self.read_gate + self._prior_balances_read_stats_lock = threading.Lock() + self._prior_balances_reads_total = 0 + self._prior_balances_read_last_seconds = 0.0 + self._prior_balances_read_max_seconds = 0.0 + self._ledger_read_timings: dict[str, dict[str, float | int]] = {} + self._ledger_read_timings_lock = threading.Lock() + + def arm_deadline(self, remaining_seconds: float) -> None: + """Give the calling thread a bounded operation budget.""" + deadline_local = threading.local() + deadline_local.deadline = self.clock.now + remaining_seconds + self._operation_timeout_local = deadline_local + + def _run_json(self, sql: str) -> Any: + self.queries.append(sql) + self.journal.append("statement") + self.clock.advance(self.execute_seconds) + if not self.results: + raise AssertionError(f"unexpected extra query: {sql[:120]}") + result = self.results.pop(0) + if isinstance(result, BaseException): + raise result + return result + + +class _UnattributedReadLedger(_AttributedReadLedger): + """The same ledger with the pre-#224 admission body restored. + + ``_run_attributed_read_json`` is replaced by exactly what each call site + used to do: take the gate through ``_operation_gate`` and run the same + retry-safe statement, recording nothing. It is the differential control + for "the SQL and the results did not change" -- both ledgers run the + shipped method bodies, and only the read helper differs. + """ + + def _run_attributed_read_json( + self, + sql: str, + *, + operation: str, + gate: Any, + gate_name: str, + ) -> Any: + with self._operation_gate(gate, gate_name): + return self._run_retry_safe_read_json(sql) + + def _run_attributed_read_json_rows( + self, + sql: str, + *, + operation: str, + gate: Any, + gate_name: str, + row_converter: Any = None, + ) -> list[Any]: + # The row-result reads (#236: the payout-window snapshot and delta) + # get the same restoration: the caller's gate through + # ``_operation_gate`` and the same retry-safe row statement, + # recording nothing. + with self._operation_gate(gate, gate_name): + return self._run_retry_safe_read_json_rows( + sql, + row_converter=row_converter, + ) + + +def share_row(share_seq: int, *, miner: str = "miner-a") -> dict[str, Any]: + """One accepted-share payload in the shape the snapshot SQL returns.""" + return { + "share_seq": share_seq, + "share_id": f"share-{share_seq}", + "miner_id": miner, + "order_key": miner, + "p2mr_program_hex": "11" * 32, + "share_difficulty": "4096", + "network_difficulty": "1048576", + "template_height": 800_000 + share_seq, + "job_id": f"job-{share_seq}", + "job_issued_at_ms": 1_000 + share_seq, + "accepted_at_ms": 1_001 + share_seq, + "ntime": 1_700_000_000 + share_seq, + "credit_policy": None, + } + + +def balance_row(miner: str = "miner-a") -> dict[str, Any]: + """One carry-forward balance in the shape both balance reads return.""" + return { + "recipient_id": miner, + "order_key": miner, + "p2mr_program_hex": "22" * 32, + "balance_sats": "2500", + } + + +# Each attributed read, as (operation, gate name, canned result, call). The +# scenarios iterate this so a fifth read added to the contract without a +# scenario is a missing row here rather than a silently untested path. +def read_cases() -> tuple[tuple[str, str, Any, Any], ...]: + return ( + ( + "payout_window_snapshot", + READ_SLOT, + [share_row(1), share_row(2)], + lambda ledger: ledger.snapshot_at_job_issue(1_005, window_weight=64), + ), + ( + "payout_window_delta", + READ_SLOT, + [share_row(3)], + lambda ledger: ledger.snapshot_between_job_issues(1_005, 1_006), + ), + ( + "prior_balances_after_pool_block", + READ_SLOT, + [balance_row()], + lambda ledger: ledger.prior_balances_after_pool_block( + block_hash=BLOCK_HASH + ), + ), + ( + "current_prior_balances", + WRITER_LOCK, + [balance_row()], + lambda ledger: ledger.current_prior_balances(), + ), + ) + + +class LedgerReadAttributionTests(unittest.TestCase): + def test_each_read_records_under_its_contract_operation(self) -> None: + """The four names are the Wave 0 ones, spelled exactly.""" + for operation, _gate_name, result, call in read_cases(): + with self.subTest(operation=operation): + ledger = _AttributedReadLedger([result]) + + call(ledger) + + self.assertIn(operation, PRISM_LEDGER_READ_OPERATIONS) + stats = ledger.ledger_read_gate_stats() + # Exactly one operation, exactly once: a read that recorded + # twice would double-count the landing it is meant to explain. + self.assertEqual(list(stats), [operation]) + self.assertEqual(stats[operation]["calls_total"], 1) + + def test_the_shipped_enumeration_attribution_is_untouched(self) -> None: + """``pending_block_candidate_rows`` keeps its name and its read slot.""" + ledger = _AttributedReadLedger( + [[{"block_hash": BLOCK_HASH, "candidate": {}, "pool_block_exists": False}]] + ) + + rows = ledger.pending_block_candidate_rows(limit=8) + + self.assertEqual([row["block_hash"] for row in rows], [BLOCK_HASH]) + self.assertEqual(rows[0]["pool_block_exists"], False) + self.assertEqual( + list(ledger.ledger_read_gate_stats()), + ["pending_block_candidate_rows"], + ) + self.assertEqual(ledger.read_gate.admissions, 1) + self.assertEqual(ledger.writer_gate.admissions, 0) + + def test_each_read_keeps_the_admission_primitive_it_had(self) -> None: + """The writer-lock read stays a writer-lock read, and vice versa. + + This is the invariant most at risk while adding attribution: the + attributed helper used to be hard-wired to the read semaphore, so + routing the prior-balances reread through it is exactly where that + read could have been silently moved off the writer lock it needs. + """ + for operation, gate_name, result, call in read_cases(): + with self.subTest(operation=operation): + ledger = _AttributedReadLedger([result]) + + call(ledger) + + writer_lock = gate_name == WRITER_LOCK + taken = ledger.writer_gate if writer_lock else ledger.read_gate + untaken = ledger.read_gate if writer_lock else ledger.writer_gate + self.assertEqual(taken.admissions, 1) + self.assertEqual(taken.releases, 1) + # Not merely "the right gate was taken": the other one was + # never touched, so no read acquired both or swapped classes. + self.assertEqual(untaken.admissions, 0) + self.assertEqual(untaken.releases, 0) + # And admission precedes the statement it admits. + self.assertEqual( + ledger.journal, + [ + f"{gate_name}:acquire", + f"{gate_name}:admitted", + "statement", + f"{gate_name}:release", + ], + ) + + def test_read_slot_reads_never_reach_the_writer_lock(self) -> None: + """A refusing writer gate is invisible to the three read-slot reads. + + The read-only ledger substitutes ``_RefusingWriterGate`` for the + writer lock, so any path that quietly took it fails loudly. The three + read-slot reads must be unaffected; the prior-balances reread must + still be refused, which is the same statement made from the other + side: it really is on the writer lock. + """ + for operation, gate_name, result, call in read_cases(): + with self.subTest(operation=operation): + ledger = _AttributedReadLedger([result]) + ledger._lock = _RefusingWriterGate() + + if gate_name == WRITER_LOCK: + with self.assertRaises(ReadOnlyLedgerError): + call(ledger) + # Refused before any statement was sent. + self.assertEqual(ledger.queries, []) + self.assertEqual(ledger.read_gate.admissions, 0) + else: + self.assertEqual(len(call(ledger)), len(result)) + self.assertEqual(len(ledger.queries), 1) + self.assertEqual(ledger.read_gate.admissions, 1) + + def test_gate_wait_and_execution_are_separate_samples(self) -> None: + """The whole point: which half of a slow read was slow.""" + for operation, _gate_name, result, call in read_cases(): + with self.subTest(operation=operation): + ledger = _AttributedReadLedger( + [result], + gate_wait_seconds=0.25, + execute_seconds=1.5, + ) + + call(ledger) + + stats = ledger.ledger_read_gate_stats()[operation] + self.assertAlmostEqual(stats["gate_wait_seconds_total"], 0.25) + self.assertAlmostEqual(stats["gate_wait_seconds_max"], 0.25) + self.assertAlmostEqual(stats["execute_seconds_total"], 1.5) + self.assertAlmostEqual(stats["execute_seconds_max"], 1.5) + # Neither timeout counter fires on a read that completed. + self.assertEqual(stats["gate_timeouts_total"], 0) + self.assertEqual(stats["execute_timeouts_total"], 0) + + def test_admission_timeout_is_a_gate_timeout_with_no_execution_sample( + self, + ) -> None: + """A budget spent queueing is never charged to PostgreSQL. + + The gate is genuinely held by another holder, so the expiry is a real + admission expiry rather than a scripted one, and it names the + primitive the caller was waiting for. + """ + for operation, gate_name, result, call in read_cases(): + with self.subTest(operation=operation): + ledger = _AttributedReadLedger([result], gate_wait_seconds=4.0) + held = ( + ledger.writer_gate if gate_name == WRITER_LOCK else ledger.read_gate + ) + self.assertTrue(held.primitive.acquire(timeout=0)) + self.addCleanup(held.primitive.release) + ledger.arm_deadline(0.01) + + with self.assertRaises(LedgerOperationTimeout) as raised: + call(ledger) + + self.assertIn(gate_name, str(raised.exception)) + # No statement was ever sent, so there is nothing to charge to + # the server -- not even a zero-length execution sample. + self.assertEqual(ledger.queries, []) + stats = ledger.ledger_read_gate_stats()[operation] + self.assertEqual(stats["calls_total"], 1) + self.assertEqual(stats["gate_timeouts_total"], 1) + self.assertEqual(stats["execute_timeouts_total"], 0) + self.assertEqual(stats["execute_seconds_total"], 0.0) + self.assertEqual(stats["execute_seconds_max"], 0.0) + self.assertAlmostEqual(stats["gate_wait_seconds_max"], 4.0) + # A failed admission releases nothing, because it took nothing. + self.assertEqual(held.admissions, 0) + self.assertEqual(held.releases, 0) + + def test_execution_timeout_is_an_execution_timeout(self) -> None: + """A deadline that expired inside the statement belongs to the server. + + Including the tail a server-cancelled statement spends returning: the + elapsed execution is recorded, not discarded, so the operator sees how + long PostgreSQL actually held the budget. + """ + for operation, _gate_name, _result, call in read_cases(): + with self.subTest(operation=operation): + ledger = _AttributedReadLedger( + [LedgerOperationTimeout("postgres operation exceeded 5s")], + gate_wait_seconds=0.1, + execute_seconds=5.0, + ) + + with self.assertRaisesRegex(LedgerOperationTimeout, "exceeded 5s"): + call(ledger) + + stats = ledger.ledger_read_gate_stats()[operation] + self.assertEqual(stats["calls_total"], 1) + self.assertEqual(stats["gate_timeouts_total"], 0) + self.assertEqual(stats["execute_timeouts_total"], 1) + self.assertAlmostEqual(stats["execute_seconds_max"], 5.0) + self.assertAlmostEqual(stats["gate_wait_seconds_max"], 0.1) + + def test_failing_reads_release_the_primitive_and_keep_the_error(self) -> None: + """A raising statement leaks nothing and masks nothing. + + Attribution runs in a ``finally``; a version of it that recorded + before releasing, or that swallowed the statement's exception into a + bookkeeping one, would strand the coordinator's writer lock or hide + the failure that mattered. + """ + for operation, gate_name, _result, call in read_cases(): + with self.subTest(operation=operation): + failure = RuntimeError("postgres query failed: connection reset") + ledger = _AttributedReadLedger([failure]) + gate = ( + ledger.writer_gate if gate_name == WRITER_LOCK else ledger.read_gate + ) + + with self.assertRaises(RuntimeError) as raised: + call(ledger) + + # The original exception object, not a wrapper and not a + # timeout misclassification. + self.assertIs(raised.exception, failure) + self.assertEqual(gate.admissions, 1) + self.assertEqual(gate.releases, 1) + self.assertEqual(gate.outstanding, 0) + self.assertTrue(gate.is_free()) + # A failure is still a call, and still not a timeout. + stats = ledger.ledger_read_gate_stats()[operation] + self.assertEqual(stats["calls_total"], 1) + self.assertEqual(stats["gate_timeouts_total"], 0) + self.assertEqual(stats["execute_timeouts_total"], 0) + + def test_a_released_gate_admits_the_next_read(self) -> None: + """The proof that release was real: the same gate serves again. + + Run against a read slot of one and a single writer lock, so a read + that failed to give its primitive back cannot be admitted a second + time (see ``STRANDED_GATE_SECONDS``). + """ + for operation, _gate_name, result, call in read_cases(): + with self.subTest(operation=operation): + ledger = _AttributedReadLedger( + [RuntimeError("first attempt failed"), result], + read_concurrency=1, + ) + + with self.assertRaises(RuntimeError): + call(ledger) + second = call(ledger) + + self.assertEqual(len(second), len(result)) + self.assertEqual( + ledger.ledger_read_gate_stats()[operation]["calls_total"], 2 + ) + + +class UnchangedBehaviorTests(unittest.TestCase): + """Everything except the bookkeeping must be byte-for-byte what it was.""" + + def test_sql_and_results_match_the_unattributed_path(self) -> None: + """Differential against the pre-#224 admission body. + + Both ledgers run the shipped method; only the read helper differs -- + one attributes, one reproduces the old ``_operation_gate`` + + ``_run_retry_safe_read_json`` pair. Identical SQL text and identical + return values is the statement that this wave added a measurement and + nothing else. + """ + for operation, _gate_name, result, call in read_cases(): + with self.subTest(operation=operation): + # Deep copies: both balance reads convert ``balance_sats`` + # in place, and a shared row would let one ledger's parsing + # feed the other's. + attributed = _AttributedReadLedger([copy.deepcopy(result)]) + legacy = _UnattributedReadLedger([copy.deepcopy(result)]) + + new_value = call(attributed) + old_value = call(legacy) + + self.assertEqual(len(attributed.queries), 1) + self.assertEqual(attributed.queries, legacy.queries) + self.assertEqual(new_value, old_value) + # The legacy path records nothing, so equal results cannot be + # an artifact of both sides sharing the new helper. + self.assertEqual(legacy.ledger_read_gate_stats(), {}) + + def test_snapshot_records_parse_exactly_as_before(self) -> None: + """Result parsing is untouched: same fields, same types, same order.""" + ledger = _AttributedReadLedger([[share_row(1), share_row(2, miner="miner-b")]]) + + records = ledger.snapshot_at_job_issue(1_005, window_weight=64) + + self.assertEqual([type(record) for record in records], [AcceptedShareRecord] * 2) + self.assertEqual([record.share_seq for record in records], [1, 2]) + self.assertEqual([record.miner_id for record in records], ["miner-a", "miner-b"]) + self.assertEqual([record.share_difficulty for record in records], [4096, 4096]) + self.assertIsNone(records[0].credit_policy) + + def test_balances_are_still_returned_as_integers(self) -> None: + """Both balance reads keep converting ``balance_sats`` out of text.""" + for call in ( + lambda ledger: ledger.prior_balances_after_pool_block(block_hash=BLOCK_HASH), + lambda ledger: ledger.current_prior_balances(), + ): + with self.subTest(call=call): + ledger = _AttributedReadLedger([[balance_row()]]) + + balances = call(ledger) + + self.assertEqual( + balances, + [{**balance_row(), "balance_sats": 2500}], + ) + + def test_snapshot_sql_still_carries_its_window_bound(self) -> None: + """The statements themselves are the ones the audit path depends on.""" + ledger = _AttributedReadLedger([[share_row(1)], [share_row(2)]]) + + ledger.snapshot_at_job_issue(1_005, window_weight=64) + ledger.snapshot_between_job_issues(1_005, 1_006) + + snapshot_sql, delta_sql = ledger.queries + self.assertIn("WITH RECURSIVE pages AS", snapshot_sql) + self.assertIn("cumulative_difficulty - share_difficulty < 64::numeric", snapshot_sql) + self.assertIn("ORDER BY share_seq ASC", snapshot_sql) + # The delta's two disjoint branches are what keep a share from being + # credited twice across consecutive anchors. + self.assertIn("UNION ALL", delta_sql) + self.assertIn("ledger.accepted_at > to_timestamp", delta_sql) + self.assertIn("ledger.job_issued_at > to_timestamp", delta_sql) + + def test_empty_delta_still_short_circuits_without_a_gate_or_a_statement( + self, + ) -> None: + """An unmoved anchor reads nothing, so it records nothing. + + A sample here would be a lie about work that never happened, and would + inflate the delta operation's call count on every idle rebuild. + """ + ledger = _AttributedReadLedger([]) + + self.assertEqual(ledger.snapshot_between_job_issues(1_005, 1_005), []) + + self.assertEqual(ledger.queries, []) + self.assertEqual(ledger.journal, []) + self.assertEqual(ledger.ledger_read_gate_stats(), {}) + + def test_backwards_anchor_is_still_rejected_before_admission(self) -> None: + ledger = _AttributedReadLedger([]) + + with self.assertRaisesRegex(ValueError, "snapshot anchor moved backwards"): + ledger.snapshot_between_job_issues(1_006, 1_005) + + self.assertEqual(ledger.journal, []) + + def test_legacy_prior_balances_aggregate_is_preserved(self) -> None: + """``qbit_prism_prior_balances_*`` keeps its old meaning. + + It still counts only successful reads and still covers admission and + execution together. The new per-operation sample duplicates part of it + on purpose: an operator comparing the existing series across this + upgrade must not find it silently redefined. + """ + ledger = _AttributedReadLedger( + [[balance_row()], RuntimeError("postgres query failed")], + gate_wait_seconds=0.5, + execute_seconds=2.0, + ) + + ledger.current_prior_balances() + with self.assertRaises(RuntimeError): + ledger.current_prior_balances() + + legacy = ledger.prior_balances_read_stats() + # One success, one failure: the failure is absent from the legacy + # aggregate, exactly as before, and present in the new one. + self.assertEqual(legacy["reads_total"], 1) + self.assertGreaterEqual(legacy["last_seconds"], 0.0) + self.assertGreaterEqual(legacy["max_seconds"], legacy["last_seconds"]) + self.assertEqual( + ledger.ledger_read_gate_stats()["current_prior_balances"]["calls_total"], + 2, + ) + + +class LabelBoundednessTests(unittest.TestCase): + def test_every_attributed_call_site_names_a_contract_operation(self) -> None: + """The literals in the module are the closed vocabulary, and all of it. + + ``tests.test_prism_accepted_preview_telemetry`` already fails on a + literal outside the contract. This is the other direction: the four + names this wave was asked for are actually present, so a lane that + wired up only some of them cannot pass. + """ + source = (REPO_ROOT / "lab" / "prism" / "share_ledger.py").read_text() + literals = set(re.findall(r'operation="([^"]+)"', source)) + + self.assertEqual( + literals, + { + "pending_block_candidate_rows", + "observe_block_candidate_outbox_row", + "pending_block_candidate_headers", + "read_block_candidate_body_manifest", + "read_block_candidate_body_page", + "read_block_candidate_body_pages", + "read_block_candidate_body_spans", + "payout_window_snapshot", + "payout_window_delta", + "current_prior_balances", + "prior_balances_after_pool_block", + }, + ) + self.assertEqual(sorted(literals - set(PRISM_LEDGER_READ_OPERATIONS)), []) + + def test_an_out_of_contract_name_folds_to_other_before_rendering(self) -> None: + """A drifted call site costs an attribution, never a new series. + + The recorder itself keeps whatever name it is handed -- it is a plain + dict, and rejecting there would turn a metrics mistake into a failed + ledger read. Boundedness is the renderer's job, and this is where the + two meet. + """ + ledger = _AttributedReadLedger([[balance_row()], [balance_row()]]) + + ledger.current_prior_balances() + ledger._run_attributed_read_json( + "SELECT json_build_object('ok', true);", + operation="block_hash=" + BLOCK_HASH, + gate=ledger._read_semaphore, + gate_name=READ_SLOT, + ) + + raw = ledger.ledger_read_gate_stats() + self.assertEqual(len(raw), 2) + folded = fold_ledger_read_stats(raw) + self.assertEqual( + sorted(folded), + ["current_prior_balances", "other"], + ) + for operation in folded: + self.assertIn(operation, PRISM_LEDGER_READ_OPERATIONS) + + def test_many_unknown_names_still_render_as_one_series(self) -> None: + """Unbounded cardinality at the call site stays one label at the edge.""" + ledger = _AttributedReadLedger([[] for _ in range(8)]) + + for index in range(8): + ledger._run_attributed_read_json( + "SELECT json_build_object('ok', true);", + operation=f"snapshot_at_job_issue_{index}", + gate=ledger._read_semaphore, + gate_name=READ_SLOT, + ) + + folded = fold_ledger_read_stats(ledger.ledger_read_gate_stats()) + self.assertEqual(list(folded), ["other"]) + self.assertEqual(folded["other"]["calls_total"], 8) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_prism_job_build_exception_retention.py b/tests/test_prism_job_build_exception_retention.py new file mode 100644 index 000000000..6acc3badc --- /dev/null +++ b/tests/test_prism_job_build_exception_retention.py @@ -0,0 +1,604 @@ +#!/usr/bin/env python3 +"""Cancelled job builds must release producer and waiter references. + +A build that observes supersession or timeout raises from inside the executor +task. The stored error's traceback reached the request through the finished +producer frames (and, on Python 3.14, through the executor context frame that +still owned the task's argument tuple), and every waiter that re-raised the +stored instance appended its own frame owning the request and promise. Both +halves formed request -> promise -> error -> traceback -> request cycles that +only cyclic GC reclaimed. + +These regressions drive the real coordinator facade, scheduler, executor +callback, builder checkpoint, and waiter loop with automatic collection +disabled, and assert release by reference counting alone. Threads are +synchronized with events; nothing inspects frame locals or referrers. +""" + +from __future__ import annotations + +import gc +import threading +import time +import unittest +import weakref +from concurrent.futures import Future +from typing import Callable + +from lab.prism.job_bundle import ( + JobBuildCancelled, + JobBuildSuperseded, + _await_job_build_promise, +) +from tests.prism_coordinator_test_support import ( + FakeLedger, + coordinator, + install_fake_bundle_builder, +) + +# The builder checkpoints ledger conversion every 256 rows. Cancelling while +# row CANCEL_ROW converts lets the next checkpoint raise from the real builder +# frame while the ledger snapshot and the converted rows are still live. +WINDOW_ROWS = 1024 +CANCEL_ROW = 300 +WAIT_SECONDS = 20.0 +WAITER_FRAME = "_shared_job_bundle_after_priority_admission" +PRODUCER_FRAMES = ( + "_execute_job_build_request", + "build_shared_job_bundle", + "raise_if_cancelled", +) + + +def wait_until(predicate: Callable[[], bool], *, timeout: float = WAIT_SECONDS) -> None: + deadline = time.monotonic() + timeout + while time.monotonic() < deadline: + if predicate(): + return + time.sleep(0.005) + raise AssertionError("condition did not become true before the deadline") + + +def traceback_names(error: BaseException | None) -> list[str]: + """Function names along a traceback, retaining no frame.""" + + names: list[str] = [] + entry = error.__traceback__ if error is not None else None + while entry is not None: + names.append(entry.tb_frame.f_code.co_name) + entry = entry.tb_next + return names + + +def describe(error: BaseException | None) -> tuple[type, tuple, list[str]] | None: + if error is None: + return None + return type(error), error.args, traceback_names(error) + + +class ShareRow(dict): + """dict subclass so a converted share row is weakref-able.""" + + +class Snapshot(list): + """list subclass so a ledger snapshot is weakref-able.""" + + +class LedgerRecord: + def __init__(self, ledger: "WindowLedger", snapshot_index: int, index: int) -> None: + self.ledger = ledger + self.snapshot_index = snapshot_index + self.index = index + self.payload = { + "miner_id": f"miner-{index % 3}", + "share_seq": index + 1, + "blob": "x" * 64, + } + + def to_prism_json(self) -> ShareRow: + self.ledger.on_convert(self.snapshot_index, self.index) + row = ShareRow(self.payload) + if self.index == 0: + self.ledger.first_rows.append(weakref.ref(row)) + return row + + +class WindowLedger(FakeLedger): + """A ledger whose snapshot rows call back into the test as they convert.""" + + def __init__(self) -> None: + super().__init__(miners=["miner-0", "miner-1", "miner-2"]) + self.on_convert: Callable[[int, int], None] = lambda snapshot_index, index: None + self.snapshots: list[weakref.ref] = [] + self.first_rows: list[weakref.ref] = [] + + def accepted_share_stats(self) -> dict[str, int]: + self.stats_calls += 1 + return {"accepted_share_count": WINDOW_ROWS, "distinct_miner_count": 3} + + def snapshot_at_job_issue( + self, + anchor_job_issued_at_ms: int, + *, + window_weight: int | None = None, + ) -> Snapshot: + self.snapshot_calls += 1 + snapshot_index = self.snapshot_calls + snapshot = Snapshot( + LedgerRecord(self, snapshot_index, index) for index in range(WINDOW_ROWS) + ) + self.snapshots.append(weakref.ref(snapshot)) + return snapshot + + +class _Payload: + pass + + +class JobBuildExceptionRetentionTests(unittest.TestCase): + def setUp(self) -> None: + gc.collect() + self.gc_enabled = gc.isenabled() + gc.disable() + self.ledger = WindowLedger() + self.server, rpc = coordinator(ledger=self.ledger) + install_fake_bundle_builder(self.server) + self.service = self.server._ensure_job_bundle_service() + self.artifacts = self.server.store_template_artifacts(dict(rpc.template)) + self.assertIsNotNone(self.artifacts) + self.requests: list[weakref.ref] = [] + self.promises: list[weakref.ref] = [] + self.flight_promise: weakref.ref | None = None + original_new_request = self.server._new_job_build_request + + def observe_new_request(*args: object, **kwargs: object) -> object: + request = original_new_request(*args, **kwargs) # type: ignore[arg-type] + self.requests.append(weakref.ref(request)) + self.promises.append(weakref.ref(request.promise)) + return request + + self.server._new_job_build_request = observe_new_request # type: ignore[method-assign] + + def tearDown(self) -> None: + self.server.shutdown_job_build_executor() + if self.gc_enabled: + gc.enable() + gc.collect() + + # -- harness ----------------------------------------------------------- + + def hold_first_conversion( + self, + after: Callable[[int], None] | None = None, + ) -> tuple[threading.Event, threading.Event]: + """Pause the first snapshot's conversion at CANCEL_ROW until released.""" + + reached = threading.Event() + proceed = threading.Event() + + def on_convert(snapshot_index: int, index: int) -> None: + if snapshot_index == 1 and index == CANCEL_ROW: + reached.set() + if not proceed.wait(WAIT_SECONDS): + raise TimeoutError("test never released the paused build") + if after is not None: + after(index) + + self.ledger.on_convert = on_convert + return reached, proceed + + def run_waiters( + self, + count: int, + *, + retry_superseded: bool = False, + ) -> tuple[list[threading.Thread], list[dict[str, object]]]: + """Run real bundle waiters, recording only plain data about outcomes.""" + + outcomes: list[dict[str, object]] = [{} for _ in range(count)] + + def stored_error() -> BaseException | None: + reference = self.flight_promise or self.promises[0] + promise = reference() + return None if promise is None else promise.exception(timeout=WAIT_SECONDS) + + def wait(slot: int) -> None: + outcome = outcomes[slot] + try: + self.server.shared_job_bundle( + self.artifacts, + mode="ready", + retry_superseded=retry_superseded, + ) + except JobBuildCancelled as error: + stored = stored_error() + outcome.update( + kind="cancelled", + type=type(error), + args=error.args, + names=traceback_names(error), + notes=list(getattr(error, "__notes__", [])), + attributes={ + key: value + for key, value in vars(error).items() + if key != "__notes__" + }, + cause=describe(error.__cause__), + context=describe(error.__context__), + suppress_context=error.__suppress_context__, + stored_is_error=stored is error, + stored_names=traceback_names(stored), + ) + except BaseException as error: # noqa: BLE001 - asserted below + stored = stored_error() + outcome.update( + kind="unexpected", + type=type(error), + args=error.args, + stored_is_error=stored is error, + ) + else: + outcome.update(kind="bundle") + + threads = [ + threading.Thread(target=wait, args=(slot,), name=f"bundle-waiter-{slot}") + for slot in range(count) + ] + for thread in threads: + thread.start() + return threads, outcomes + + def await_joined_flight(self, waiters: int) -> None: + """Every waiter must have joined the paused flight before it is cancelled.""" + + wait_until( + lambda: self.service.job_build_scheduler_counts["requests"] == waiters + ) + with self.service._job_build_scheduler_lock: + flight = self.service._job_build_active + self.assertIsNotNone(flight) + self.flight_promise = weakref.ref(flight.request.promise) + self.assertEqual(self.ledger.snapshot_calls, 1) + + def cancel_active_flight(self, reason: str) -> None: + with self.service._job_build_scheduler_lock: + flight = self.service._job_build_active + self.assertIsNotNone(flight) + self.assertTrue(self.service._cancel_job_build_flight_locked(flight, reason)) + self.assertEqual(flight.request.cancellation.reason, reason) + + def join_all(self, threads: list[threading.Thread]) -> None: + for thread in threads: + thread.join(WAIT_SECONDS) + self.assertFalse(thread.is_alive(), f"{thread.name} did not finish") + # Joining the executor thread drops the finished work item and the + # task argument tuple it still holds between completion and exit. + self.server.shutdown_job_build_executor() + + def tracked_references(self) -> dict[str, weakref.ref]: + references: dict[str, weakref.ref] = {} + for index, reference in enumerate(self.requests): + references[f"request[{index}]"] = reference + for index, reference in enumerate(self.promises): + references[f"promise[{index}]"] = reference + for index, reference in enumerate(self.ledger.snapshots): + references[f"snapshot[{index}]"] = reference + for index, reference in enumerate(self.ledger.first_rows): + references[f"first_row[{index}]"] = reference + return references + + def assert_released(self, references: dict[str, weakref.ref]) -> None: + self.assertFalse(gc.isenabled()) + self.assertEqual( + {name: reference() is not None for name, reference in references.items()}, + dict.fromkeys(references, False), + ) + + def assert_cancelled_outcome( + self, + outcome: dict[str, object], + error_type: type[JobBuildCancelled], + message_prefix: str, + ) -> None: + self.assertEqual(outcome["kind"], "cancelled", outcome) + self.assertIs(outcome["type"], error_type) + args = outcome["args"] + assert isinstance(args, tuple) + self.assertEqual(len(args), 1) + self.assertTrue(str(args[0]).startswith(message_prefix), args) + names = outcome["names"] + assert isinstance(names, list) + self.assertIn(WAITER_FRAME, names) + self.assertFalse(outcome["stored_is_error"]) + stored_names = outcome["stored_names"] + assert isinstance(stored_names, list) + self.assertNotIn(WAITER_FRAME, stored_names) + self.assertIn("_execute_job_build_request", stored_names) + + # -- supersession and timeout through the real checkpoint -------------- + + def check_cancelled_flight_releases_for_every_waiter( + self, + reason: str, + error_type: type[JobBuildCancelled], + message_prefix: str, + ) -> None: + reached, proceed = self.hold_first_conversion() + threads, outcomes = self.run_waiters(3) + self.assertTrue(reached.wait(WAIT_SECONDS)) + self.await_joined_flight(3) + self.cancel_active_flight(reason) + proceed.set() + self.join_all(threads) + for outcome in outcomes: + self.assert_cancelled_outcome(outcome, error_type, message_prefix) + names = outcome["names"] + assert isinstance(names, list) + for producer_frame in PRODUCER_FRAMES: + self.assertIn(producer_frame, names) + self.assertEqual(outcome["cause"], None) + self.assertEqual(outcome["context"], None) + self.assertEqual(len(self.requests), 3) + self.assertEqual(self.ledger.snapshot_calls, 1) + # Scheduler accounting is unchanged: supersession is an obsolete + # result, a timeout is a failed build. + superseded = reason != "timeout" + self.assertEqual( + self.service.job_build_scheduler_counts["obsolete_results"], + 1 if superseded else 0, + ) + self.assertEqual( + self.service.shared_bundle_build_counts["superseded"], + 1 if superseded else 0, + ) + self.assertEqual( + self.service.shared_bundle_build_counts["failed"], + 0 if superseded else 1, + ) + self.assert_released(self.tracked_references()) + + def test_superseded_flight_releases_window_for_every_waiter(self) -> None: + self.check_cancelled_flight_releases_for_every_waiter( + "superseded", + JobBuildSuperseded, + "job build superseded at ledger_snapshot_conversion", + ) + + def test_timed_out_flight_releases_window_for_every_waiter(self) -> None: + self.check_cancelled_flight_releases_for_every_waiter( + "timeout", + JobBuildCancelled, + "job build timeout at ledger_snapshot_conversion", + ) + + def test_in_frame_retry_releases_the_superseded_build(self) -> None: + reached, proceed = self.hold_first_conversion() + threads, outcomes = self.run_waiters(1, retry_superseded=True) + self.assertTrue(reached.wait(WAIT_SECONDS)) + self.await_joined_flight(1) + self.cancel_active_flight("superseded") + proceed.set() + self.join_all(threads) + self.assertEqual(outcomes[0], {"kind": "bundle"}) + self.assertEqual(self.ledger.snapshot_calls, 2) + self.assertEqual(len(self.requests), 2) + # The retry's cached bundle legitimately keeps its converted rows; + # everything the superseded build owned must be gone. + self.assert_released( + { + "superseded_request": self.requests[0], + "superseded_promise": self.promises[0], + "superseded_snapshot": self.ledger.snapshots[0], + "superseded_first_row": self.ledger.first_rows[0], + "retry_request": self.requests[1], + "retry_promise": self.promises[1], + "retry_snapshot": self.ledger.snapshots[1], + } + ) + + # -- orphan sweep resolves under the scheduler lock -------------------- + + def test_orphan_sweep_resolves_wedged_flight_and_releases_window(self) -> None: + self.server.job_build_orphan_sweep_grace_seconds = 0.0 # type: ignore[attr-defined] + flights: list[weakref.ref] = [] + + def never_arm(flight: object) -> None: + # A done callback that died before resolving leaves the flight + # wedged in its slot; the admission sweep must settle it. + flights.append(weakref.ref(flight)) + + self.server._arm_job_build_locked = never_arm # type: ignore[method-assign] + reached, proceed = self.hold_first_conversion() + threads, outcomes = self.run_waiters(2) + self.assertTrue(reached.wait(WAIT_SECONDS)) + self.await_joined_flight(2) + self.cancel_active_flight("superseded") + proceed.set() + wait_until(lambda: flights[0]().future.done()) + with self.service._job_build_scheduler_lock: + self.assertEqual(self.service._evict_orphaned_job_build_flights_locked(), []) + logs = self.service._evict_orphaned_job_build_flights_locked() + self.assertEqual(len(logs), 1) + self.assertIn("slot=active", logs[0]) + self.assertIn("cancelled=True", logs[0]) + self.assertEqual(self.service.job_build_scheduler_counts["orphan_evicted"], 1) + self.join_all(threads) + for outcome in outcomes: + self.assert_cancelled_outcome( + outcome, + JobBuildSuperseded, + "job build superseded at ledger_snapshot_conversion", + ) + references = self.tracked_references() + references["flight"] = flights[0] + self.assert_released(references) + + # -- chained producer failures reported as supersession ----------------- + + def check_chained_cancellation_releases_nested_frames(self, chaining: str) -> None: + def reject_row(index: int) -> None: + raise ValueError(f"ledger row {index} rejected") + + reached, proceed = self.hold_first_conversion(after=reject_row) + coordinator_entry = self.server._execute_job_build_request + + def _execute_job_build_request(request: object) -> object: + # Stand-in for an executor entry that reports a nested producer + # failure as supersession, as the bundle compiler does with a + # subprocess failure. Like the real entry, its frame owns the + # request while the nested frames own the payout window. + try: + return coordinator_entry(request) # type: ignore[arg-type] + except ValueError as failure: + error = JobBuildSuperseded( + "job build superseded at ledger_snapshot_conversion; row rejected" + ) + error.add_note("rejected row converts on the next build") + error.phase = "ledger_snapshot_conversion" # type: ignore[attr-defined] + if chaining == "cause": + raise error from failure + if chaining == "suppressed": + raise error from None + raise error + + self.server._execute_job_build_request = _execute_job_build_request # type: ignore[method-assign] + threads, outcomes = self.run_waiters(2) + self.assertTrue(reached.wait(WAIT_SECONDS)) + self.await_joined_flight(2) + proceed.set() + self.join_all(threads) + failure = (ValueError, (f"ledger row {CANCEL_ROW} rejected",)) + for outcome in outcomes: + self.assert_cancelled_outcome( + outcome, + JobBuildSuperseded, + "job build superseded at ledger_snapshot_conversion; row rejected", + ) + self.assertEqual(outcome["notes"], ["rejected row converts on the next build"]) + self.assertEqual(outcome["attributes"], {"phase": "ledger_snapshot_conversion"}) + cause = outcome["cause"] + context = outcome["context"] + if chaining == "cause": + self.assertTrue(outcome["suppress_context"]) + self.assertIsNotNone(cause) + chained = cause + else: + self.assertIsNone(cause) + self.assertEqual(outcome["suppress_context"], chaining == "suppressed") + chained = context + assert chained is not None + self.assertEqual(chained[:2], failure) + for producer_frame in ("_execute_job_build_request", "build_shared_job_bundle", "to_prism_json"): + self.assertIn(producer_frame, chained[2]) + self.assertEqual(len(self.requests), 2) + self.assertEqual(self.ledger.snapshot_calls, 1) + self.assert_released(self.tracked_references()) + + def test_cancellation_with_cause_releases_nested_producer_frames(self) -> None: + self.check_chained_cancellation_releases_nested_frames("cause") + + def test_cancellation_with_context_releases_nested_producer_frames(self) -> None: + self.check_chained_cancellation_releases_nested_frames("context") + + def test_cancellation_with_suppressed_context_releases_nested_frames(self) -> None: + self.check_chained_cancellation_releases_nested_frames("suppressed") + + # -- unexpected failures keep their behavior ---------------------------- + + def test_unexpected_build_failure_still_raises_the_stored_instance(self) -> None: + def reject_row(index: int) -> None: + raise ValueError(f"ledger row {index} rejected") + + reached, proceed = self.hold_first_conversion(after=reject_row) + threads, outcomes = self.run_waiters(1) + self.assertTrue(reached.wait(WAIT_SECONDS)) + self.await_joined_flight(1) + proceed.set() + self.join_all(threads) + self.assertEqual(outcomes[0]["kind"], "unexpected") + self.assertIs(outcomes[0]["type"], ValueError) + self.assertEqual(outcomes[0]["args"], (f"ledger row {CANCEL_ROW} rejected",)) + self.assertTrue(outcomes[0]["stored_is_error"]) + self.assertEqual(self.service.shared_bundle_build_counts["failed"], 1) + # An unexpected failure keeps its full traceback for post-mortem + # inspection, so only collection reclaims that build. + gc.collect() + self.assert_released(self.tracked_references()) + + +class WaiterCopyTests(unittest.TestCase): + def setUp(self) -> None: + gc.collect() + self.gc_enabled = gc.isenabled() + gc.disable() + + def tearDown(self) -> None: + if self.gc_enabled: + gc.enable() + gc.collect() + + def test_waiter_copy_preserves_metadata_and_leaves_stored_traceback_alone(self) -> None: + payload = _Payload() + promise: Future[object] = Future() + try: + raise ValueError("compiler exited 137") + except ValueError as failure: + stored = JobBuildSuperseded("job build superseded at bundle_assembly") + stored.add_note("compiler exit 137") + stored.phase = "bundle_assembly" # type: ignore[attr-defined] + stored.__cause__ = failure + stored.__suppress_context__ = True + promise.set_exception(stored) + references = {"payload": weakref.ref(payload), "promise": weakref.ref(promise)} + + def waiter(owned_payload: _Payload, owned_promise: Future[object]) -> None: + _await_job_build_promise(owned_promise, 1.0) + + try: + waiter(payload, promise) + except JobBuildSuperseded as error: + self.assertIsNot(error, stored) + self.assertIs(type(error), JobBuildSuperseded) + self.assertEqual(error.args, stored.args) + self.assertEqual(error.__notes__, ["compiler exit 137"]) # type: ignore[attr-defined] + self.assertEqual(error.phase, "bundle_assembly") # type: ignore[attr-defined] + self.assertIs(error.__cause__, stored.__cause__) + self.assertIs(error.__context__, stored.__context__) + self.assertTrue(error.__suppress_context__) + self.assertEqual( + traceback_names(error)[-2:], + ["waiter", "_await_job_build_promise"], + ) + else: + self.fail("waiter did not observe the cancellation") + # The stored instance was never raised: no waiter frame reached it. + self.assertIsNone(stored.__traceback__) + self.assertIs(promise.exception(), stored) + del stored, payload, promise + self.assertFalse(gc.isenabled()) + self.assertEqual( + {name: reference() is not None for name, reference in references.items()}, + dict.fromkeys(references, False), + ) + + def test_waiter_keeps_future_result_semantics_for_other_outcomes(self) -> None: + pending: Future[object] = Future() + with self.assertRaises(TimeoutError): + _await_job_build_promise(pending, 0.001) + completed: Future[object] = Future() + result = _Payload() + completed.set_result(result) + self.assertIs(_await_job_build_promise(completed, 1.0), result) + failed: Future[object] = Future() + failure = RuntimeError("build failed") + failed.set_exception(failure) + try: + _await_job_build_promise(failed, 1.0) + except RuntimeError as error: + self.assertIs(error, failure) + else: + self.fail("unexpected failure was not re-raised") + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_prism_job_build_promises.py b/tests/test_prism_job_build_promises.py index 708dd151b..cf21a5698 100644 --- a/tests/test_prism_job_build_promises.py +++ b/tests/test_prism_job_build_promises.py @@ -19,11 +19,10 @@ JobBuildCancelled, JobBuildSuperseded, PrismCoordinator, - WorkerIdentity, _JobBuildFlight, _JobBuildRequest, ) -from tests.test_prism_coordinator_job_cache import ( +from tests.prism_coordinator_test_support import ( coordinator, install_fake_bundle_builder, synthetic_manifest_coinbase_hex, diff --git a/tests/test_prism_job_build_retention.py b/tests/test_prism_job_build_retention.py new file mode 100644 index 000000000..145f902e0 --- /dev/null +++ b/tests/test_prism_job_build_retention.py @@ -0,0 +1,144 @@ +#!/usr/bin/env python3 +"""Completed job flights must release their windows without cyclic GC.""" + +from __future__ import annotations + +import gc +import unittest +import weakref +from concurrent.futures import CancelledError, Future +from types import SimpleNamespace +from unittest.mock import patch + +from lab.prism.job_bundle import JobBuildFlight, JobBundleService + + +class _Payload: + pass + + +class JobBuildCallbackRetentionTests(unittest.TestCase): + def setUp(self) -> None: + gc.collect() + self.gc_enabled = gc.isenabled() + gc.disable() + self.completions = 0 + self.fail_completion = False + self.owner = SimpleNamespace(_runtime=self) + + def tearDown(self) -> None: + if self.gc_enabled: + gc.enable() + gc.collect() + + def _job_build_done(self, flight: JobBuildFlight, future: Future) -> None: + self.assertIs(flight.future, future) + self.assertTrue(future.done()) + self.completions += 1 + if self.fail_completion: + raise RuntimeError("injected completion bookkeeping failure") + + def arm(self, *, already_finished: bool = False): + request = _Payload() + result = _Payload() + future = Future() + flight = JobBuildFlight(request=request, future=future) + references = { + "request": weakref.ref(request), + "result": weakref.ref(result), + "flight": weakref.ref(flight), + "future": weakref.ref(future), + } + if already_finished: + future.set_result(result) + JobBundleService._arm_job_build_locked(self.owner, flight) + return future, result, references + + def assert_released(self, references) -> None: + self.assertFalse(gc.isenabled()) + self.assertEqual( + {name: ref() is not None for name, ref in references.items()}, + dict.fromkeys(references, False), + ) + + def test_completed_flights_and_results_release_without_collection(self) -> None: + for _ in range(10): + future, result, references = self.arm() + future.set_result(result) + del future, result + self.assert_released(references) + self.assertEqual(self.completions, 10) + + def test_callback_owns_pending_flight_until_completion(self) -> None: + future, result, references = self.arm() + self.assertIsNotNone(references["flight"]()) + self.assertIsNotNone(references["request"]()) + self.assertEqual(self.completions, 0) + future.set_result(result) + self.assertIsNone(references["flight"]()) + self.assertIsNone(references["request"]()) + self.assertIs(future.result(), result) + del future, result + self.assert_released(references) + self.assertEqual(self.completions, 1) + + def test_already_finished_future_completes_inline(self) -> None: + future, result, references = self.arm(already_finished=True) + self.assertEqual(self.completions, 1) + self.assertIs(future.result(), result) + del future, result + self.assert_released(references) + + def test_cancelled_future_releases_flight(self) -> None: + future, result, references = self.arm() + self.assertTrue(future.cancel()) + del future, result + self.assert_released(references) + self.assertEqual(self.completions, 1) + + def test_exceptional_future_releases_callback_ownership(self) -> None: + future, result, references = self.arm() + error = RuntimeError("build failed") + future.set_exception(error) + self.assertIs(future.exception(), error) + del future, result + self.assert_released(references) + self.assertEqual(self.completions, 1) + + def test_bookkeeping_failure_releases_callback_ownership(self) -> None: + future, result, references = self.arm() + self.fail_completion = True + # Capturing logging records here would itself retain the callback's + # traceback and flight, masking the ownership under test. + with patch("concurrent.futures._base.LOGGER.exception", lambda *a, **kw: None): + future.set_result(result) + self.assertEqual(self.completions, 1) + self.assertIs(future.result(), result) + del future, result + self.assert_released(references) + + def check_failed_outcome_releases_request(self, *, cancelled: bool) -> None: + request = _Payload() + request.cancellation = SimpleNamespace(is_set=lambda: False) + future = Future() + references = {"request": weakref.ref(request), "future": weakref.ref(future)} + if cancelled: + future.cancel() + else: + future.set_exception(RuntimeError("build failed")) + result, error = JobBundleService._job_build_flight_outcome(request, future) + self.assertIsNone(result) + self.assertIsInstance(error, CancelledError if cancelled else RuntimeError) + self.assertIsNone(error.__traceback__) + del request, future, result, error + self.assert_released(references) + + def test_failed_outcome_does_not_capture_request_in_traceback(self) -> None: + self.check_failed_outcome_releases_request(cancelled=False) + + def test_cancelled_outcome_does_not_capture_request_in_traceback(self) -> None: + self.check_failed_outcome_releases_request(cancelled=True) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_prism_job_builder.py b/tests/test_prism_job_builder.py new file mode 100644 index 000000000..f806bef3f --- /dev/null +++ b/tests/test_prism_job_builder.py @@ -0,0 +1,4810 @@ +#!/usr/bin/env python3 +"""Focused PRISM coordinator job builder tests.""" +# ruff: noqa: F403, F405 + +from __future__ import annotations + +import json +import unittest +from tests.prism_coordinator_test_support import * + +from lab.prism.bundle_compiler import ( + _compact_share_tail_chunks, + _iter_prepare_window_request_chunks, +) +from lab.prism.share_ledger import DaemonShareWindowMirror + + +class SnapshotAnchorFloorTests(unittest.TestCase): + def _hold_floor(self, server: PrismCoordinator, share: PendingShare) -> None: + server._ensure_pending_share_commit_state() + with server._pending_share_commit_lock: + server._pending_share_commit_floor[id(share)] = [ + share, + time.monotonic(), + False, + ] + + def test_job_bundle_anchor_clamps_below_pending_share_commit(self) -> None: + # The issued snapshot must be reproducible from the durable ledger: + # while a stamped share's commit is pending, the job anchor (which the + # bundle declares as anchor_job_issued_at_ms) has to predate it, or + # qbit_audit_share_window at the declared anchor would include a share + # the published window omitted. + ledger = AnchorRecordingLedger() + server, _rpc = coordinator(ledger=ledger) + install_fake_bundle_builder(server) + stamped_ms = now_ms() - 5 + share = stamped_pending_share(stamped_ms) + self._hold_floor(server, share) + + bundle = server.build_shared_job_bundle( + server.current_template_artifacts(), + worker(), + ) + self.assertEqual(ledger.anchors[-1], stamped_ms - 1) + self.assertEqual( + bundle.found_block["anchor_job_issued_at_ms"], stamped_ms - 1 + ) + self.assertEqual(bundle.issued_at_ms, stamped_ms - 1) + + server._finish_pending_share_commit(share) + # The issued time is frozen per template generation; drop the frozen + # entry so the rebuild stamps a fresh anchor now that no commit is + # pending. + with server._job_cache_lock: + server._job_build_issued_at_ms.clear() + rebuilt = server.build_shared_job_bundle( + server.current_template_artifacts(), + worker(), + ) + self.assertGreaterEqual(ledger.anchors[-1], stamped_ms) + self.assertGreaterEqual( + int(rebuilt.found_block["anchor_job_issued_at_ms"]), stamped_ms + ) + + def test_payout_artifact_declares_its_own_snapshot_anchor(self) -> None: + # An artifact snapshot is taken at its own earlier anchor. A bundle + # reusing the artifact must declare that anchor rather than the + # fresher job-issue time: a share stamped between the two anchors is + # excluded from the artifact by construction, yet a re-derivation at + # the job-issue anchor would include it. + ledger = AnchorRecordingLedger() + server, _rpc = coordinator(ledger=ledger) + install_fake_bundle_builder(server) + artifacts = server.current_template_artifacts() + clamp_now_ms = now_ms() - 6 + # Anchor selection sits strictly below the clamp instant so a share + # stamped in the same millisecond can never tie the anchor. + artifact_anchor_ms = clamp_now_ms - 1 + with patch( + "lab.prism.prism_coordinator.now_ms", + return_value=clamp_now_ms, + ): + artifact = server._build_payout_ledger_artifact( + 0, 0, artifacts.network_difficulty + ) + assert artifact is not None + self.assertEqual(artifact.snapshot_anchor_ms, artifact_anchor_ms) + self.assertEqual(ledger.anchors[-1], artifact_anchor_ms) + + # Construction re-validates that the passed artifact is the installed + # current one. + with server._job_cache_lock: + server._payout_ledger_artifact = artifact + bundle = server.build_shared_job_bundle( + artifacts, + worker(), + payout_artifact=artifact, + ) + self.assertEqual( + bundle.found_block["anchor_job_issued_at_ms"], + artifact.snapshot_anchor_ms, + ) + self.assertGreater(bundle.issued_at_ms, int(artifact.snapshot_anchor_ms)) + + def test_artifact_paths_proceed_at_the_clamped_anchor_floor( + self, + ) -> None: + # The pending-commit clamp is anchor selection, not a fence: every + # share stamped at or below the clamped anchor is already durable, + # so the window read at that anchor is exact and reproducible, and + # both artifact producers publish it. Shares stamped above the + # anchor deterministically belong to the next window. + ledger = AnchorRecordingLedger() + server, _rpc = coordinator(ledger=ledger) + install_fake_bundle_builder(server) + artifacts = server.current_template_artifacts() + stamped_ms = now_ms() - 5 + share = stamped_pending_share(stamped_ms) + self._hold_floor(server, share) + try: + artifact = server._build_payout_ledger_artifact( + 0, 0, artifacts.network_difficulty + ) + self.assertIsNotNone(artifact) + assert artifact is not None + self.assertEqual(artifact.snapshot_anchor_ms, stamped_ms - 1) + self.assertEqual(ledger.anchors[-1], stamped_ms - 1) + + # The synchronous build proceeds at the clamped anchor and seeds + # its window for reuse under that same anchor; cache publication + # installs the seed. + bundle = server.build_shared_job_bundle(artifacts, worker()) + self.assertEqual(bundle.issued_at_ms, stamped_ms - 1) + seeded = bundle.prepared_ledger_artifact + self.assertIsNotNone(seeded) + assert seeded is not None + self.assertEqual(seeded.snapshot_anchor_ms, stamped_ms - 1) + server._install_payout_ledger_artifact(seeded) + with server._job_cache_lock: + published = server._payout_ledger_artifact + self.assertIsNotNone(published) + assert published is not None + self.assertEqual(published.snapshot_anchor_ms, stamped_ms - 1) + finally: + server._finish_pending_share_commit(share) + + def test_background_build_refuses_a_pathologically_old_floor(self) -> None: + # A floor held further below now than the audit anchor ceiling (a + # wedged writer or leaked release) would arm an artifact that is + # born expired; refuse before paying the window walk so the + # re-arm backoff paces retries. + ledger = AnchorRecordingLedger() + server, _rpc = coordinator(ledger=ledger) + install_fake_bundle_builder(server) + artifacts = server.current_template_artifacts() + share = stamped_pending_share(now_ms() - 400_000) + self._hold_floor(server, share) + try: + self.assertIsNone( + server._build_payout_ledger_artifact( + 0, 0, artifacts.network_difficulty + ) + ) + self.assertEqual(ledger.snapshot_calls, 0) + finally: + server._finish_pending_share_commit(share) + + def test_anchor_selection_excludes_same_millisecond_stamps(self) -> None: + # Millisecond granularity can hand a share stamped right after + # anchor selection the same accepted_at_ms as the clamp instant, + # and such a share is never protected by the pending floor. The + # window predicate is anchor-inclusive, so the anchor must sit + # strictly below the clamp-time millisecond or that share would + # join audit replays of a window that never contained it. + server, _rpc = coordinator(ledger=AnchorRecordingLedger()) + clamp_now = now_ms() + self.assertEqual( + server._job_snapshot_anchor_ms(clamp_now), + clamp_now - 1, + ) + + share = stamped_pending_share(clamp_now - 7) + self._hold_floor(server, share) + try: + self.assertEqual( + server._job_snapshot_anchor_ms(clamp_now), + clamp_now - 8, + ) + finally: + server._finish_pending_share_commit(share) + + +class JobBundleCacheTests(unittest.TestCase): + def test_tip_template_snapshot_stays_coherent_across_tip_transition(self) -> None: + old_tip = "11" * 32 + new_tip = "22" * 32 + server, rpc = coordinator(template=base_template(prevhash=old_tip)) + new_template = base_template(height=11, prevhash=new_tip) + original_call = rpc.call + + def transition_during_template_fetch( + method: str, + params: list[object] | None = None, + ) -> object: + if method == "getblocktemplate": + rpc.tip = new_tip + rpc.template = new_template + return original_call(method, params) + + rpc.call = transition_during_template_fetch # type: ignore[method-assign] + + snapshot = server.fetch_qbit_tip_template_snapshot() + + self.assertEqual(snapshot.bestblockhash, new_tip) + self.assertEqual(snapshot.previousblockhash, new_tip) + self.assertEqual( + snapshot.template_fingerprint, + qbit_template_fingerprint(new_template), + ) + self.assertEqual(rpc.calls[:2], ["getblocktemplate", "getbestblockhash"]) + + def test_one_heavy_build_shared_across_clients_with_per_client_stamping(self) -> None: + server, rpc = coordinator() + recorded = install_fake_bundle_builder(server) + clients = [client(1), client(2), client(3)] + + contexts = [server.build_job_for_client(c, clean_jobs=True) for c in clients] + + self.assertEqual(recorded["calls"], 1) + self.assertEqual(rpc.count("getblocktemplate"), 1) + self.assertEqual(server.ledger.snapshot_calls, 1) + # The heavy build uses the placeholder extranonce1, never a client's. + self.assertEqual( + recorded["suffixes"], + [ + server.coinbase_tag_hex + + PRISM_JOB_EXTRANONCE1_PLACEHOLDER_HEX + + "00" * EXTRANONCE2_SIZE + ], + ) + job_ids = {context.job.job_id for context in contexts} + self.assertEqual(len(job_ids), 3) + self.assertEqual( + [context.job.extranonce1_hex for context in contexts], + [c.extranonce1_hex for c in clients], + ) + # coinb1/coinb2 exclude the extranonce window entirely, so the shared + # split is byte-identical for every client. + self.assertEqual(len({context.job.coinb1 for context in contexts}), 1) + self.assertEqual(len({context.job.coinb2 for context in contexts}), 1) + self.assertTrue(all(not hasattr(context, "bundle") for context in contexts)) + self.assertTrue( + all(context.prospective_prior_balances == () for context in contexts) + ) + cached = next(iter(server._job_bundle_cache.values())) + self.assertFalse(hasattr(cached, "bundle")) + self.assertEqual(cached.prospective_prior_balances, ()) + self.assertEqual( + cached.coinbase_manifest["coinbase_tx_hex"], + synthetic_manifest_coinbase_hex(recorded["suffixes"][0]), + ) + self.assertIs(contexts[0].shares_json, contexts[1].shares_json) + + def test_latest_wins_scheduler_preserves_synchronous_builder_output(self) -> None: + server, rpc = coordinator() + recorded = install_fake_bundle_builder(server) + artifacts = server.store_template_artifacts(dict(rpc.template)) + assert artifacts is not None + identity = worker() + cache_key = server._job_bundle_key( + artifacts, + mode="ready", + payout_state_generation=0, + worker=identity, + ) + + with patch("lab.prism.prism_coordinator.now_ms", return_value=1_700_000_001_000): + direct_request = server._new_job_build_request( + artifacts, + identity, + mode="ready", + payout_state_generation=0, + cache_key=cache_key, + ) + direct = server.build_shared_job_bundle( + artifacts, + identity, + mode="ready", + payout_state_generation=0, + key=cache_key, + build_request=direct_request, + ) + scheduled_request = server._new_job_build_request( + artifacts, + identity, + mode="ready", + payout_state_generation=0, + cache_key=cache_key, + ) + scheduled = server._request_job_build(scheduled_request).result(5) + + server.shutdown_job_build_executor() + self.assertEqual(recorded["calls"], 2) + self.assertEqual(scheduled.key, direct.key) + self.assertEqual(scheduled.template, direct.template) + self.assertEqual(scheduled.template_fingerprint, direct.template_fingerprint) + self.assertEqual(scheduled.coinbase_manifest, direct.coinbase_manifest) + self.assertEqual(scheduled.shares_json, direct.shares_json) + self.assertEqual(scheduled.prior_balances, direct.prior_balances) + self.assertEqual(scheduled.found_block, direct.found_block) + self.assertEqual(scheduled.collection_only, direct.collection_only) + self.assertEqual(scheduled.issued_at_ms, direct.issued_at_ms) + self.assertEqual(scheduled.base_job, direct.base_job) + self.assertEqual(scheduled.build_key, direct.build_key) + + def test_stamped_job_reassembles_coinbase_with_client_extranonce(self) -> None: + server, _ = coordinator() + install_fake_bundle_builder(server) + state = client(0x2A) + + context = server.build_job_for_client(state, clean_jobs=True) + extranonce2_hex = "11" * EXTRANONCE2_SIZE + submission = direct_stratum.assemble_submission( + context.job, + extranonce2_hex=extranonce2_hex, + ntime_hex="65000000", + nonce_hex="00000001", + ) + + expected_suffix = server.coinbase_tag_hex + state.extranonce1_hex + extranonce2_hex + coinbase = bytes.fromhex(submission.coinbase_tx_hex) + script_start, script_len = direct_stratum.coinbase_scriptsig_span( + coinbase, field_name="stamped coinbase" + ) + script_sig_hex = coinbase[script_start : script_start + script_len].hex() + self.assertTrue(script_sig_hex.endswith(expected_suffix)) + self.assertNotIn(PRISM_JOB_EXTRANONCE1_PLACEHOLDER_HEX, script_sig_hex[len("03aabbcc") :]) + + def test_template_fingerprint_change_invalidates_bundle_cache(self) -> None: + server, rpc = coordinator() + recorded = install_fake_bundle_builder(server) + server.build_job_for_client(client(1), clean_jobs=True) + + new_template = base_template(height=11, prevhash="22" * 32) + rpc.template = new_template + rpc.tip = str(new_template["previousblockhash"]) + server.store_template_artifacts(dict(new_template)) + + context = server.build_job_for_client(client(2), clean_jobs=True) + + self.assertEqual(recorded["calls"], 2) + self.assertEqual(context.template_fingerprint, qbit_template_fingerprint(new_template)) + # Bundles for the old fingerprint are evicted. + self.assertEqual( + {entry.template_fingerprint for entry in server._job_bundle_cache.values()}, + {qbit_template_fingerprint(new_template)}, + ) + + def test_bundle_cache_ttl_expiry_rebuilds(self) -> None: + server, _ = coordinator() + recorded = install_fake_bundle_builder(server) + server.job_bundle_cache_seconds = 0.05 + + server.build_job_for_client(client(1), clean_jobs=True) + time.sleep(0.06) + server.build_job_for_client(client(2), clean_jobs=True) + + self.assertEqual(recorded["calls"], 2) + + def test_bundle_cache_lookup_prunes_every_expired_snapshot(self) -> None: + server, _ = coordinator() + install_fake_bundle_builder(server) + artifacts = server.store_template_artifacts(base_template()) + assert artifacts is not None + current = server.shared_job_bundle(artifacts, worker()) + expired_key = ("expired-template", "ready") + expired = dataclass_replace( + current, + key=expired_key, + template_fingerprint="expired-template", + built_monotonic=time.monotonic() - 60, + ) + server._job_bundle_cache[expired_key] = expired + + looked_up = server._lookup_job_bundle(current.key) + + self.assertIs(looked_up, current) + self.assertNotIn(expired_key, server._job_bundle_cache) + self.assertEqual(list(server._job_bundle_cache.values()), [current]) + + def test_zero_ttl_disables_bundle_cache(self) -> None: + server, _ = coordinator() + recorded = install_fake_bundle_builder(server) + server.job_bundle_cache_seconds = 0.0 + + server.build_job_for_client(client(1), clean_jobs=True) + server.build_job_for_client(client(2), clean_jobs=True) + + self.assertEqual(recorded["calls"], 2) + + def test_payout_state_change_during_build_retries_before_cache_or_return(self) -> None: + server, rpc = coordinator() + recorded = install_fake_bundle_builder(server) + artifacts = server.store_template_artifacts(dict(rpc.template)) + self.assertIsNotNone(artifacts) + assert artifacts is not None + identity = worker() + original_build = server.build_shared_job_bundle + built_generations: list[int] = [] + + def mutate_after_first_build( + build_artifacts: object, + build_worker: WorkerIdentity | None, + **kwargs: object, + ) -> object: + bundle = original_build( # type: ignore[arg-type] + build_artifacts, + build_worker, + **kwargs, + ) + built_generations.append(bundle.payout_state_generation) + if len(built_generations) == 1: + server._advance_payout_state_generation() + return bundle + + server.build_shared_job_bundle = mutate_after_first_build # type: ignore[method-assign] + + bundle = server.shared_job_bundle(artifacts, identity) + cached = server.shared_job_bundle(artifacts, identity) + + self.assertEqual(built_generations, [0, 1]) + self.assertEqual(recorded["calls"], 2) + self.assertEqual(bundle.payout_state_generation, 1) + self.assertIs(cached, bundle) + + def test_escaped_stale_bundle_is_rejected_before_direct_delivery(self) -> None: + server, _rpc = coordinator() + install_fake_bundle_builder(server) + server._ensure_tip_refresh_state() + state = client(1) + sent: list[dict[str, object]] = [] + state.send = sent.append # type: ignore[method-assign] + original_shared_job_bundle = server.shared_job_bundle + advanced = False + + def advance_after_bundle(*args: object, **kwargs: object) -> object: + nonlocal advanced + bundle = original_shared_job_bundle(*args, **kwargs) # type: ignore[arg-type] + if not advanced: + advanced = True + server._advance_payout_state_generation() + return bundle + + server.shared_job_bundle = advance_after_bundle # type: ignore[method-assign] + + self.assertFalse(server.maybe_send_job(state, clean_jobs=True)) + self.assertEqual(sent, []) + self.assertIsNone(state.active_job) + self.assertEqual(server._payout_state_generation, 1) + self.assertTrue(server._tip_refresh_retry.is_set()) + + self.assertTrue(server.maybe_send_job(state, clean_jobs=True)) + self.assertIsNotNone(state.active_job) + self.assertEqual(state.active_job.payout_state_generation, 1) + self.assertEqual( + [payload["method"] for payload in sent], + ["mining.set_difficulty", "mining.notify"], + ) + + def test_priority_decision_uses_one_publication_snapshot(self) -> None: + server, _rpc = coordinator() + state = client(1) + context = SimpleNamespace( + payout_state_generation=0, + template={"previousblockhash": "11" * 32}, + ) + server.ensure_reorg_reconciled_for_current_tip = ( # type: ignore[method-assign] + lambda **_kwargs: True + ) + server.build_job_for_client = ( # type: ignore[method-assign] + lambda *_args, **_kwargs: context + ) + original_lock = server._job_cache_lock + + class PublishAfterPrioritySnapshot: + advanced = False + + def __enter__(self) -> object: + original_lock.acquire() + return self + + def __exit__( + self, + _exc_type: object, + _exc: object, + _traceback: object, + ) -> None: + original_lock.release() + if not self.advanced: + self.advanced = True + server._payout_state_generation = 1 + + priorities: list[bool] = [] + + class RecordingGate: + @contextmanager + def delivery_cancelable( + self, + _cancelled: object, + *, + priority: bool, + **_kwargs: object, + ) -> object: + priorities.append(priority) + yield False + + server._job_cache_lock = PublishAfterPrioritySnapshot() # type: ignore[assignment] + server._payout_state_delivery_gate = RecordingGate() # type: ignore[assignment] + + self.assertFalse(server.maybe_send_job(state, clean_jobs=True)) + self.assertEqual(priorities, [True]) + self.assertEqual(server._payout_state_generation, 1) + + def test_zero_template_ttl_fetches_template_per_build(self) -> None: + server, rpc = coordinator() + install_fake_bundle_builder(server) + server.template_cache_seconds = 0.0 + + server.build_job_for_client(client(1), clean_jobs=True) + server.build_job_for_client(client(2), clean_jobs=True) + + self.assertEqual(rpc.count("getblocktemplate"), 2) + + def test_late_stale_template_fetch_cannot_replace_newer_artifacts(self) -> None: + server, rpc = coordinator() + server.template_cache_seconds = 0.0 + stale_template = dict(rpc.template) + current_template = base_template(height=11, prevhash="22" * 32) + fetch_started = threading.Event() + release_fetch = threading.Event() + results: list[object] = [] + errors: list[BaseException] = [] + original_call = rpc.call + thread: threading.Thread + + def blocking_call( + method: str, + params: list[object] | None = None, + ) -> object: + if method == "getblocktemplate" and threading.current_thread() is thread: + fetch_started.set() + if not release_fetch.wait(5): + raise AssertionError("stale template fetch was not released") + return dict(stale_template) + return original_call(method, params) + + def fetch_stale_artifacts() -> None: + try: + results.append(server.current_template_artifacts()) + except BaseException as exc: # noqa: BLE001 - surface to the test + errors.append(exc) + + rpc.call = blocking_call # type: ignore[method-assign] + thread = threading.Thread(target=fetch_stale_artifacts) + thread.start() + try: + self.assertTrue(fetch_started.wait(5)) + current_artifacts = server.store_template_artifacts(current_template) + self.assertIsNotNone(current_artifacts) + assert current_artifacts is not None + self.assertGreater(current_artifacts.generation, 1) + finally: + release_fetch.set() + thread.join(5) + + self.assertFalse(thread.is_alive()) + self.assertEqual(errors, []) + self.assertEqual(results, [current_artifacts]) + self.assertIs(server._template_artifacts, current_artifacts) + self.assertEqual( + current_artifacts.fingerprint, + qbit_template_fingerprint(current_template), + ) + + def test_collection_mode_bundles_are_keyed_per_worker(self) -> None: + server, _ = coordinator(ledger=FakeLedger(miners=["solo"])) + recorded = install_fake_bundle_builder(server) + server.min_ready_miners = 3 + + worker_a = worker(payout="tq1worker-a") + worker_b = worker(payout="tq1worker-b") + context_a1 = server.build_job_for_client(client(1, worker_a), clean_jobs=True) + context_a2 = server.build_job_for_client(client(2, worker_a), clean_jobs=True) + context_b = server.build_job_for_client(client(3, worker_b), clean_jobs=True) + + self.assertTrue(context_a1.collection_only) + self.assertTrue(context_b.collection_only) + self.assertEqual(recorded["calls"], 2) + self.assertTrue( + all(not hasattr(context, "bundle") for context in (context_a1, context_a2, context_b)) + ) + self.assertIs(context_a1.shares_json, context_a2.shares_json) + self.assertIsNot(context_a1.shares_json, context_b.shares_json) + + def test_collection_bundle_cache_rebuilds_when_pool_becomes_ready(self) -> None: + ledger = FakeLedger(miners=["solo"]) + server, _ = coordinator(ledger=ledger) + recorded = install_fake_bundle_builder(server) + state = client(1) + + collection_context = server.build_job_for_client(state, clean_jobs=True) + ledger.miners = ["miner-a", "miner-b", "miner-c"] + ready_context = server.build_job_for_client(state, clean_jobs=True) + + self.assertTrue(collection_context.collection_only) + self.assertFalse(ready_context.collection_only) + self.assertEqual(recorded["calls"], 2) + self.assertEqual(ledger.snapshot_calls, 1) + + def test_ready_empty_snapshot_does_not_fall_back_to_worker_collection(self) -> None: + ledger = ReadyLedgerWithEmptyFirstSnapshot() + server, _ = coordinator(ledger=ledger) + recorded = install_fake_bundle_builder(server) + state = client(1) + + with self.assertRaisesRegex( + RuntimeError, + "ready-pool ledger snapshot contained no payout shares", + ): + server.build_job_for_client(state, clean_jobs=True) + ready_context = server.build_job_for_client(state, clean_jobs=True) + + self.assertFalse(ready_context.collection_only) + self.assertEqual(recorded["calls"], 1) + self.assertEqual(ledger.snapshot_calls, 2) + + def test_vardiff_difficulty_is_stamped_per_client(self) -> None: + server, _ = coordinator() + install_fake_bundle_builder(server) + easy = client(1) + hard = client(2) + hard.pending_share_difficulty = Decimal("512") + + easy_context = server.build_job_for_client(easy, clean_jobs=True) + hard_context = server.build_job_for_client(hard, clean_jobs=True) + + self.assertEqual(easy_context.job.coinb1, hard_context.job.coinb1) + self.assertGreater(easy_context.job.share_target, hard_context.job.share_target) + self.assertEqual(hard_context.job.share_difficulty, Decimal("512")) + + def test_template_artifacts_reuse_derivations_when_fingerprint_unchanged(self) -> None: + server, _ = coordinator() + template = base_template() + first = server.store_template_artifacts(dict(template)) + refreshed_template = dict(template) + refreshed_template["curtime"] = int(template["curtime"]) + 30 + second = server.store_template_artifacts(refreshed_template) + + self.assertIsNotNone(first) + self.assertIsNotNone(second) + assert first is not None and second is not None + self.assertEqual(first.fingerprint, second.fingerprint) + self.assertIs(first.transaction_hexes, second.transaction_hexes) + self.assertIs(first.witness_merkle_leaves_hex, second.witness_merkle_leaves_hex) + + def test_poll_seeds_template_cache_for_client_builds(self) -> None: + server, rpc = coordinator() + install_fake_bundle_builder(server) + + refreshed = server.poll_qbit_tip_template_once() + self.assertEqual(refreshed, 0) + self.assertEqual(rpc.count("getblocktemplate"), 1) + + server.build_job_for_client(client(1), clean_jobs=True) + self.assertEqual(rpc.count("getblocktemplate"), 1) + + def test_reorg_reconciliation_cached_per_tip(self) -> None: + server, rpc = coordinator() + server.reorg_reconciler_enabled = True + reconcile_calls: list[str | None] = [] + + def fake_reconcile(*, tip_hash: str | None = None) -> dict[str, object]: + reconcile_calls.append(tip_hash) + server._note_reorg_reconcile_outcome(tip_hash, trusted=True) + return {"untrusted": False} + + server.reconcile_prism_pool_blocks_once = fake_reconcile # type: ignore[method-assign] + + self.assertTrue(server.ensure_reorg_reconciled_for_current_tip()) + self.assertTrue(server.ensure_reorg_reconciled_for_current_tip()) + self.assertEqual(len(reconcile_calls), 1) + + rpc.tip = "33" * 32 + self.assertTrue(server.ensure_reorg_reconciled_for_current_tip()) + self.assertEqual(len(reconcile_calls), 2) + self.assertEqual(reconcile_calls[-1], "33" * 32) + + def test_reorg_cache_rechecks_chain_view_before_reuse(self) -> None: + server, rpc = coordinator() + server.reorg_reconciler_enabled = True + rpc.blockchain_info["headers"] = 101 + server._note_reorg_reconcile_outcome(rpc.tip, trusted=True) + reconcile_calls: list[str | None] = [] + + def fake_reconcile(*, tip_hash: str | None = None) -> dict[str, object]: + reconcile_calls.append(tip_hash) + server._note_reorg_reconcile_outcome(tip_hash, trusted=False) + return {"untrusted": True} + + server.reconcile_prism_pool_blocks_once = fake_reconcile # type: ignore[method-assign] + + self.assertFalse(server.ensure_reorg_reconciled_for_current_tip()) + self.assertEqual(rpc.count("getblockchaininfo"), 1) + self.assertEqual(reconcile_calls, [rpc.tip]) + self.assertFalse(server.last_reorg_reconciled_trusted) + + def test_single_flight_builds_once_under_concurrency(self) -> None: + server, _ = coordinator() + recorded = install_fake_bundle_builder(server) + original_builder = server.build_audit_bundle + build_started = threading.Event() + + def slow_builder(**kwargs: object) -> dict[str, object]: + build_started.set() + time.sleep(0.05) + return original_builder(**kwargs) + + server.build_audit_bundle = slow_builder # type: ignore[method-assign] + errors: list[BaseException] = [] + + def build(connection_id: int) -> None: + try: + server.build_job_for_client(client(connection_id), clean_jobs=True) + except BaseException as exc: # noqa: BLE001 - surface to the test + errors.append(exc) + + threads = [threading.Thread(target=build, args=(index + 1,)) for index in range(6)] + for thread in threads: + thread.start() + for thread in threads: + thread.join() + + self.assertEqual(errors, []) + self.assertEqual(recorded["calls"], 1) + + def test_observed_tip_change_rejects_stale_cached_bundle_delivery(self) -> None: + old_tip = "11" * 32 + new_tip = "22" * 32 + server, rpc = coordinator(template=base_template(prevhash=old_tip)) + install_fake_bundle_builder(server) + state = client(1) + sent: list[dict[str, object]] = [] + state.send = sent.append # type: ignore[method-assign] + server.clients = {state} + server.observe_tip_first_seen(old_tip, observation_sequence=1) + + self.assertTrue(server.maybe_send_job(state, clean_jobs=True)) + self.assertEqual(len(sent), 2) + sent.clear() + + rpc.tip = new_tip + server.observe_tip_first_seen(new_tip, observation_sequence=2) + + self.assertFalse(server.maybe_send_job(state, clean_jobs=True)) + self.assertEqual(sent, []) + + def test_ready_ledger_snapshot_holds_payout_mutation_lock(self) -> None: + server, rpc = coordinator() + install_fake_bundle_builder(server) + artifacts = server.store_template_artifacts(dict(rpc.template)) + assert artifacts is not None + entered_snapshot = threading.Event() + release_snapshot = threading.Event() + original_snapshot = server.ledger.snapshot_at_job_issue + + def blocked_snapshot(*args: object, **kwargs: object) -> object: + entered_snapshot.set() + self.assertTrue(release_snapshot.wait(2)) + return original_snapshot(*args, **kwargs) + + server.ledger.snapshot_at_job_issue = blocked_snapshot # type: ignore[method-assign] + errors: list[BaseException] = [] + build_thread = threading.Thread( + target=lambda: self._capture_error( + errors, + lambda: server.shared_job_bundle(artifacts, mode="ready"), + ) + ) + build_thread.start() + try: + self.assertTrue(entered_snapshot.wait(2)) + mutation_acquired = server._payout_state_prepare_lock.acquire( + blocking=False + ) + if mutation_acquired: + server._payout_state_prepare_lock.release() + self.assertFalse(mutation_acquired) + finally: + release_snapshot.set() + build_thread.join(2) + + self.assertFalse(build_thread.is_alive()) + self.assertEqual(errors, []) + + def test_ready_build_identity_separates_clock_only_generations(self) -> None: + server, rpc = coordinator() + first = server.store_template_artifacts(dict(rpc.template)) + second_template = dict(rpc.template) + second_template["curtime"] = int(second_template["curtime"]) + 1 + second = server.store_template_artifacts(second_template) + assert first is not None and second is not None + self.assertEqual(first.fingerprint, second.fingerprint) + self.assertNotEqual(first.generation, second.generation) + payout_generation = server._payout_state_generation + + with patch( + "lab.prism.prism_coordinator.now_ms", + side_effect=[1_700_000_000_000, 1_700_000_001_000], + ): + first_request = server._new_job_build_request( + first, + None, + mode="ready", + payout_state_generation=payout_generation, + cache_key=server._job_bundle_key( + first, + mode="ready", + payout_state_generation=payout_generation, + worker=None, + ), + ) + second_request = server._new_job_build_request( + second, + None, + mode="ready", + payout_state_generation=payout_generation, + cache_key=server._job_bundle_key( + second, + mode="ready", + payout_state_generation=payout_generation, + worker=None, + ), + ) + + self.assertNotEqual( + first_request.equivalence_key, + second_request.equivalence_key, + ) + self.assertNotEqual( + first_request.key.issued_at_ms, + second_request.key.issued_at_ms, + ) + + def test_precomputed_payout_artifact_matches_inline_output_exactly(self) -> None: + server, rpc = coordinator() + install_fake_bundle_builder(server) + artifacts = server.store_template_artifacts(dict(rpc.template)) + assert artifacts is not None + + with patch("lab.prism.prism_coordinator.now_ms", return_value=1_700_000_123_000): + inline = server.shared_job_bundle(artifacts, mode="ready") + with server._job_cache_lock: + server._job_bundle_cache.clear() + server._prepare_payout_ledger_artifact( + server._payout_state_generation, + artifacts.network_difficulty, + ) + prepared = server.shared_job_bundle(artifacts, mode="ready") + + self.assertEqual(prepared.base_job, inline.base_job) + self.assertEqual(prepared.coinbase_manifest, inline.coinbase_manifest) + self.assertEqual(prepared.shares_json, inline.shares_json) + self.assertEqual(prepared.prior_balances, inline.prior_balances) + self.assertEqual(prepared.found_block, inline.found_block) + self.assertGreater(prepared.payout_artifact_generation, 0) + + def test_accepted_preview_patches_artifact_across_normal_clear(self) -> None: + class CountingBalanceLedger(FakeLedger): + def __init__(self) -> None: + super().__init__() + self.prior_balance_reads = 0 + self.database_balances = [ + { + "recipient_id": "stale-miner", + "order_key": "stale-miner", + "p2mr_program_hex": "22" * 32, + "balance_sats": 1, + } + ] + + def current_prior_balances(self) -> list[dict[str, object]]: + self.prior_balance_reads += 1 + return [dict(balance) for balance in self.database_balances] + + ledger = CountingBalanceLedger() + server, rpc = coordinator(ledger=ledger) + recorded = install_fake_bundle_builder(server) + artifacts = server.store_template_artifacts(dict(rpc.template)) + assert artifacts is not None + server._pool_ready_latched = True + parent_hash = str(rpc.template["previousblockhash"]) + parent_height = int(rpc.template["height"]) - 1 + preview = [ + { + "recipient_id": "miner-a", + "order_key": "miner-a", + "p2mr_program_hex": "11" * 32, + "balance_sats": 25, + } + ] + + server._begin_accepted_block_payout_preview( + parent_hash, + block_height=parent_height, + ) + server._publish_accepted_block_payout_preview(parent_hash, preview) + + artifact = server._payout_ledger_artifact + self.assertIsNotNone(artifact) + assert artifact is not None + self.assertEqual(artifact.prior_balances, tuple(preview)) + self.assertEqual( + artifact.payout_state_generation, + server._payout_state_generation, + ) + self.assertGreater(artifact.generation, 0) + + ledger.prior_balance_reads = 0 + preview_bundle = server.shared_job_bundle(artifacts, mode="ready") + + self.assertEqual(preview_bundle.prior_balances, preview) + self.assertEqual(recorded["last_kwargs"]["prior_balances"], preview) # type: ignore[index] + self.assertEqual( + preview_bundle.payout_artifact_generation, + artifact.generation, + ) + self.assertEqual(ledger.prior_balance_reads, 0) + + payout_generation = server._payout_state_generation + ledger.database_balances = [dict(balance) for balance in preview] + server._clear_accepted_block_payout_preview(parent_hash) + self.assertEqual(server._payout_state_generation, payout_generation) + self.assertIs(server._payout_ledger_artifact, artifact) + self.assertNotIn(parent_hash, server._accepted_block_payout_previews) + self.assertNotIn( + parent_hash, + server._invalidated_accepted_block_payout_previews, + ) + with server._job_cache_lock: + server._job_bundle_cache.clear() + + post_clear_bundle = server.shared_job_bundle(artifacts, mode="ready") + + self.assertEqual(post_clear_bundle.prior_balances, preview) + self.assertEqual(recorded["last_kwargs"]["prior_balances"], preview) # type: ignore[index] + self.assertEqual( + post_clear_bundle.payout_artifact_generation, + artifact.generation, + ) + self.assertEqual(ledger.prior_balance_reads, 0) + + def test_valid_precomputed_artifact_skips_tip_path_ledger_snapshot(self) -> None: + server, rpc = coordinator() + install_fake_bundle_builder(server) + snapshot = server.fetch_qbit_tip_template_snapshot() + assert snapshot.template_artifacts is not None + server._prepare_payout_ledger_artifact( + server._payout_state_generation, + snapshot.template_artifacts.network_difficulty, + ) + server.ledger.snapshot_calls = 0 + + bundle = server.prepare_tip_refresh_bundle(snapshot) + + self.assertFalse(bundle.collection_only) + self.assertGreater(bundle.payout_artifact_generation, 0) + self.assertEqual(server.ledger.snapshot_calls, 0) + + def test_mismatched_precomputed_artifact_falls_back_to_inline_snapshot(self) -> None: + server, rpc = coordinator() + install_fake_bundle_builder(server) + artifacts = server.store_template_artifacts(dict(rpc.template)) + assert artifacts is not None + server._current_payout_state_artifact() + server._prepare_payout_ledger_artifact( + server._payout_state_generation, + artifacts.network_difficulty, + ) + with server._job_cache_lock: + published = server._published_payout_state + assert published.artifact is not None + changed_balances = [{"miner_id": "miner-z", "balance_sats": 1}] + server._published_payout_state = dataclass_replace( + published, + artifact=dataclass_replace( + published.artifact, + prior_balances_json=canonical_json_text(changed_balances), + prior_balances_sha256=canonical_json_sha256(changed_balances), + ), + ) + + self.assertIsNone( + server._usable_payout_ledger_artifact( + server._payout_state_generation, + artifacts.network_difficulty, + ) + ) + self.assertIsNone(server._payout_ledger_artifact) + server.ledger.snapshot_calls = 0 + + bundle = server.shared_job_bundle(artifacts, mode="ready") + + self.assertEqual(bundle.payout_artifact_generation, 0) + self.assertEqual(server.ledger.snapshot_calls, 1) + + def test_new_tip_cancels_blocked_old_bundle_without_publication(self) -> None: + old_tip = "11" * 32 + new_tip = "22" * 32 + server, rpc = coordinator(template=base_template(prevhash=old_tip)) + recorded = install_fake_bundle_builder(server) + original_builder = server.build_audit_bundle + build_started = threading.Event() + + def cancelable_builder(**kwargs: object) -> dict[str, object]: + control = server._job_build_phase_local.bundle_build_control + build_started.set() + self.assertTrue(control.cancel_event.wait(2)) + return original_builder(**kwargs) + + server.build_audit_bundle = cancelable_builder # type: ignore[method-assign] + artifacts = server.store_template_artifacts(dict(rpc.template)) + assert artifacts is not None + server.observe_tip_first_seen(old_tip, observation_sequence=1) + errors: list[BaseException] = [] + thread = threading.Thread( + target=lambda: self._capture_error( + errors, + lambda: server.shared_job_bundle(artifacts, mode="ready"), + ) + ) + thread.start() + self.assertTrue(build_started.wait(2)) + + server.observe_tip_first_seen(new_tip, observation_sequence=2) + thread.join(2) + + self.assertFalse(thread.is_alive()) + self.assertEqual(len(errors), 1) + self.assertIsInstance(errors[0], TemplateRefreshBlocked) + self.assertEqual(recorded["calls"], 1) + self.assertEqual(server._active_job_bundle_builds, {}) + self.assertEqual(server.tip_refresh_build_inflight, 0) + self.assertFalse(any( + entry.template_fingerprint == artifacts.fingerprint + for entry in server._job_bundle_cache.values() + )) + self.assertEqual(server.tip_refresh_superseded_results, 1) + + def test_builder_crash_and_timeout_fail_closed_then_recover(self) -> None: + server, _rpc = coordinator() + server.prism_ctv_settlement_config = lambda **_kwargs: None # type: ignore[method-assign] + server.signing_seed_hex = "42" * 32 + server.ledger_attestation_signing_seed_hex = "43" * 32 + build_kwargs = { + "shares": [], + "found_block": { + "block_height": 10, + "coinbase_value_sats": 50_00000000, + "network_difficulty": 1, + "anchor_job_issued_at_ms": 1_700_000_000_000, + }, + "prior_balances": [], + "coinbase_script_sig_suffix_hex": "00", + } + + with patch( + "lab.prism.prism_coordinator.prism_tool_command", + return_value=[sys.executable, "-c", "raise SystemExit(7)"], + ): + with self.assertRaisesRegex(RuntimeError, "failed"): + server.build_audit_bundle(**build_kwargs) + + server.bundle_build_timeout_seconds = 0.01 + with patch( + "lab.prism.prism_coordinator.prism_tool_command", + return_value=[ + sys.executable, + "-c", + "import time; time.sleep(5)", + ], + ): + with self.assertRaisesRegex(RuntimeError, "timed out"): + server.build_audit_bundle(**build_kwargs) + + server.bundle_build_timeout_seconds = 1.0 + recovery_script = ( + "import json,sys; json.load(sys.stdin); " + "json.dump({'recovered': True}, sys.stdout)" + ) + with patch( + "lab.prism.prism_coordinator.prism_tool_command", + return_value=[sys.executable, "-c", recovery_script], + ): + recovered = server.build_audit_bundle(**build_kwargs) + self.assertEqual(recovered, {"recovered": True}) + + def test_audit_builder_child_does_not_inherit_open_socket(self) -> None: + server, _rpc = coordinator() + server.prism_ctv_settlement_config = lambda **_kwargs: None # type: ignore[method-assign] + server.signing_seed_hex = "42" * 32 + server.ledger_attestation_signing_seed_hex = "43" * 32 + probe_script = ( + "import json,os,sys; json.load(sys.stdin); fd=int(sys.argv[1]); " + "inherited=True; " + "\ntry: os.fstat(fd)" + "\nexcept OSError: inherited=False" + "\njson.dump({'inherited_socket': inherited}, sys.stdout)" + ) + with socket.socket() as parent_socket: + parent_socket.set_inheritable(True) + with patch( + "lab.prism.prism_coordinator.prism_tool_command", + return_value=[ + sys.executable, + "-c", + probe_script, + str(parent_socket.fileno()), + ], + ): + result = server.build_audit_bundle( + shares=[], + found_block={ + "block_height": 10, + "coinbase_value_sats": 50_00000000, + "network_difficulty": 1, + "anchor_job_issued_at_ms": 1_700_000_000_000, + }, + prior_balances=[], + coinbase_script_sig_suffix_hex="00", + ) + self.assertEqual(result, {"inherited_socket": False}) + + def test_repeated_superseded_builds_leave_state_bounded(self) -> None: + server, rpc = coordinator() + install_fake_bundle_builder(server) + original_builder = server.build_audit_bundle + starts: queue.Queue[None] = queue.Queue() + + def cancelable_builder(**kwargs: object) -> dict[str, object]: + control = server._job_build_phase_local.bundle_build_control + starts.put(None) + self.assertTrue(control.cancel_event.wait(2)) + return original_builder(**kwargs) + + server.build_audit_bundle = cancelable_builder # type: ignore[method-assign] + current_tip = str(rpc.tip) + server.observe_tip_first_seen(current_tip, observation_sequence=1) + errors: list[BaseException] = [] + for index in range(8): + rpc.template = base_template(height=10 + index, prevhash=current_tip) + artifacts = server.store_template_artifacts(dict(rpc.template)) + assert artifacts is not None + thread = threading.Thread( + target=lambda current=artifacts: self._capture_error( + errors, + lambda: server.shared_job_bundle(current, mode="ready"), + ) + ) + thread.start() + starts.get(timeout=2) + current_tip = f"{index + 2:064x}" + rpc.tip = current_tip + server.observe_tip_first_seen( + current_tip, + observation_sequence=index + 2, + ) + thread.join(2) + self.assertFalse(thread.is_alive()) + + self.assertEqual(len(errors), 8) + self.assertTrue(all(isinstance(exc, TemplateRefreshBlocked) for exc in errors)) + self.assertEqual(server._active_job_bundle_builds, {}) + self.assertEqual(server.tip_refresh_build_inflight, 0) + self.assertEqual(server.tip_refresh_build_queue_depth, 0) + self.assertLessEqual( + len(server._job_bundle_cache), + MAX_PRISM_JOB_BUNDLE_CACHE_ENTRIES, + ) + self.assertEqual(server.tip_refresh_superseded_results, 8) + + @staticmethod + def _capture_error( + errors: list[BaseException], + operation: object, + ) -> None: + try: + operation() # type: ignore[operator] + except BaseException as exc: # noqa: BLE001 - test thread handoff + errors.append(exc) + + def test_same_fingerprint_bundle_rebinds_exact_template_observation(self) -> None: + server, _ = coordinator() + install_fake_bundle_builder(server) + identity = worker() + first = server.store_template_artifacts(base_template()) + assert first is not None + original = server.shared_job_bundle(first, identity) + updated_template = dict(first.template) + updated_template["curtime"] = int(updated_template["curtime"]) + 30 + second = server.store_template_artifacts(updated_template) + assert second is not None + + rebound = server.shared_job_bundle(second, identity) + + self.assertEqual(first.fingerprint, second.fingerprint) + self.assertIs(rebound.template, second.template) + self.assertIsNot(rebound.template, original.template) + self.assertEqual(rebound.template_generation, second.generation) + self.assertEqual(rebound.base_job.ntime, f'{updated_template["curtime"]:08x}') + + def test_clock_only_refresh_does_not_discard_inflight_ready_build(self) -> None: + server, rpc = coordinator() + recorded = install_fake_bundle_builder(server) + first = server.store_template_artifacts(dict(rpc.template)) + assert first is not None + build_entered = threading.Event() + release_build = threading.Event() + original_build = server.build_shared_job_bundle + build_calls = 0 + build_calls_lock = threading.Lock() + + def blocking_build(*args: object, **kwargs: object) -> object: + nonlocal build_calls + with build_calls_lock: + build_calls += 1 + build_entered.set() + self.assertTrue(release_build.wait(5)) + return original_build(*args, **kwargs) # type: ignore[arg-type] + + server.build_shared_job_bundle = blocking_build # type: ignore[method-assign] + results: list[list[object]] = [[], []] + errors: list[list[BaseException]] = [[], []] + + def build(index: int, build_artifacts: object) -> None: + try: + results[index].append( + server.shared_job_bundle( # type: ignore[arg-type] + build_artifacts, + mode="ready", + ) + ) + except BaseException as exc: # noqa: BLE001 - surfaced below + errors[index].append(exc) + + first_thread = threading.Thread(target=build, args=(0, first)) + second_thread: threading.Thread | None = None + first_thread.start() + try: + self.assertTrue(build_entered.wait(5)) + updated_template = dict(first.template) + updated_template["curtime"] = int(updated_template["curtime"]) + 30 + second = server.store_template_artifacts(updated_template) + assert second is not None + second_thread = threading.Thread(target=build, args=(1, second)) + second_thread.start() + time.sleep(0.05) + with build_calls_lock: + self.assertEqual(build_calls, 1) + finally: + release_build.set() + first_thread.join(5) + if second_thread is not None: + second_thread.join(5) + + self.assertFalse(first_thread.is_alive()) + assert second_thread is not None + self.assertFalse(second_thread.is_alive()) + self.assertEqual(errors, [[], []]) + self.assertEqual([len(items) for items in results], [1, 1]) + built = results[0][0] + self.assertIs(built.template, first.template) # type: ignore[union-attr] + rebound = results[1][0] + self.assertIs(rebound.template, second.template) + self.assertEqual(rebound.template_generation, second.generation) + self.assertEqual(recorded["calls"], 1) + + def test_same_fingerprint_collection_bundle_rebuilds_exact_observation(self) -> None: + server, _ = coordinator(ledger=FakeLedger(miners=["miner-a"])) + recorded = install_fake_bundle_builder(server) + identity = worker() + first = server.store_template_artifacts(base_template()) + assert first is not None + original = server.shared_job_bundle(first, identity) + updated_template = dict(first.template) + updated_template["curtime"] = int(updated_template["curtime"]) + 30 + second = server.store_template_artifacts(updated_template) + assert second is not None + + rebuilt = server.shared_job_bundle(second, identity) + + self.assertTrue(original.collection_only) + self.assertTrue(rebuilt.collection_only) + self.assertEqual(first.fingerprint, second.fingerprint) + self.assertEqual(recorded["calls"], 2) + self.assertIsNot(rebuilt.coinbase_manifest, original.coinbase_manifest) + self.assertIs(rebuilt.template, second.template) + self.assertEqual(rebuilt.template_generation, second.generation) + + def test_job_bundle_cache_is_bounded(self) -> None: + server, _ = coordinator() + install_fake_bundle_builder(server) + artifacts = server.store_template_artifacts(base_template()) + assert artifacts is not None + bundle = server.shared_job_bundle(artifacts, worker()) + + for index in range(MAX_PRISM_JOB_BUNDLE_CACHE_ENTRIES + 5): + candidate = dataclass_replace( + bundle, + key=(artifacts.fingerprint, "test", index), + ) + server._cache_job_bundle_if_current(candidate, artifacts) + + self.assertEqual( + len(server._job_bundle_cache), + MAX_PRISM_JOB_BUNDLE_CACHE_ENTRIES, + ) + self.assertNotIn( + (artifacts.fingerprint, "test", 0), + server._job_bundle_cache, + ) + + def test_job_bundle_cache_preserves_coordinator_lock_order(self) -> None: + server, _ = coordinator() + install_fake_bundle_builder(server) + artifacts = server.store_template_artifacts(base_template()) + assert artifacts is not None + bundle = server.shared_job_bundle(artifacts, mode="ready") + observed_cache_lock = ObservedRLock() + server._job_cache_lock = observed_cache_lock # type: ignore[assignment] + errors: list[BaseException] = [] + + def cache_bundle() -> None: + try: + server._cache_job_bundle_if_current(bundle, artifacts) + except BaseException as exc: # noqa: BLE001 - surfaced below + errors.append(exc) + + observed_cache_lock.acquire() + observed_cache_lock.observe_acquires = True + cache_thread = threading.Thread(target=cache_bundle) + coordinator_lock_acquired = False + try: + cache_thread.start() + self.assertTrue(observed_cache_lock.acquire_attempted.wait(5)) + coordinator_lock_acquired = server.lock.acquire(timeout=0.25) + if coordinator_lock_acquired: + server.lock.release() + finally: + observed_cache_lock.release() + cache_thread.join(5) + + self.assertTrue(coordinator_lock_acquired) + self.assertFalse(cache_thread.is_alive()) + self.assertEqual(errors, []) + + def test_active_gap_replaces_older_pending_job_build(self) -> None: + server, rpc = coordinator() + first = server.store_template_artifacts(dict(rpc.template)) + second_template = dict(rpc.template) + second_template["coinbasevalue"] = int(second_template["coinbasevalue"]) + 1 + second = server.store_template_artifacts(second_template) + assert first is not None and second is not None + payout_generation = server._payout_state_generation + + def request_for(artifacts: object) -> object: + return server._new_job_build_request( + artifacts, # type: ignore[arg-type] + None, + mode="ready", + payout_state_generation=payout_generation, + cache_key=server._job_bundle_key( + artifacts, # type: ignore[arg-type] + mode="ready", + payout_state_generation=payout_generation, + worker=None, + ), + ) + + pending = request_for(first) + newest = request_for(second) + server._job_build_active = None + server._job_build_retiring = SimpleNamespace(request=pending) + server._job_build_pending = pending + server._start_job_build_locked = ( # type: ignore[method-assign] + lambda request: SimpleNamespace(request=request, future=None) + ) + server._arm_job_build_locked = lambda _flight: None # type: ignore[method-assign] + + promise = server._request_job_build(newest) # type: ignore[arg-type] + + self.assertIs(promise, newest.promise) # type: ignore[union-attr] + self.assertIsNone(server._job_build_pending) + assert server._job_build_active is not None + self.assertIs(server._job_build_active.request, newest) + self.assertTrue(pending.promise.done()) # type: ignore[union-attr] + self.assertIsInstance( + pending.promise.exception(), # type: ignore[union-attr] + JobBuildSuperseded, + ) + + def test_publication_critical_build_cannot_be_displaced_by_initial_work( + self, + ) -> None: + server, rpc = coordinator() + old_artifacts = server.store_template_artifacts(dict(rpc.template)) + new_artifacts = server.store_template_artifacts( + base_template(height=11, prevhash="22" * 32) + ) + assert old_artifacts is not None and new_artifacts is not None + payout_generation = server._payout_state_generation + + def request_for( + artifacts: object, + *, + publication_critical: bool, + request_source: str, + ) -> object: + return server._new_job_build_request( + artifacts, # type: ignore[arg-type] + None, + mode="ready", + payout_state_generation=payout_generation, + cache_key=server._job_bundle_key( + artifacts, # type: ignore[arg-type] + mode="ready", + payout_state_generation=payout_generation, + worker=None, + ), + publication_critical=publication_critical, + request_source=request_source, + ) + + latest = request_for( + new_artifacts, + publication_critical=True, + request_source="tip_refresh", + ) + reconnect = request_for( + old_artifacts, + publication_critical=False, + request_source="initial", + ) + same_tip_reconnect = request_for( + new_artifacts, + publication_critical=False, + request_source="initial", + ) + latest_flight = SimpleNamespace(request=latest) + server._job_build_active = latest_flight + server._job_build_retiring = None + server._job_build_pending = None + + deferred = server._request_job_build(reconnect) # type: ignore[arg-type] + coalesced = server._request_job_build( # type: ignore[arg-type] + same_tip_reconnect + ) + + self.assertIs(server._job_build_active, latest_flight) + self.assertIs(coalesced, latest.promise) # type: ignore[union-attr] + self.assertFalse(latest.cancellation.is_set()) # type: ignore[union-attr] + self.assertFalse(deferred.done()) + self.assertEqual( + server.job_build_priority_counts["routine_deferred"], + 1, + ) + self.assertEqual( + server.initial_job_prepared_work_counts["deferred"], + 1, + ) + self.assertEqual( + server.initial_job_prepared_work_counts["singleflight"], + 1, + ) + + latest.promise.set_result(object()) # type: ignore[union-attr] + with self.assertRaises(JobBuildSuperseded): + deferred.result() + + metrics = "\n".join(server.job_build_metrics_lines()) + self.assertIn( + 'qbit_prism_job_build_priority_events_total{result="routine_deferred"} 1', + metrics, + ) + self.assertIn( + 'qbit_prism_initial_job_prepared_work_total{result="deferred"} 1', + metrics, + ) + self.assertIn( + 'qbit_prism_initial_job_prepared_work_total{result="singleflight"} 1', + metrics, + ) + + def test_publication_critical_build_preempts_routine_builder_capacity( + self, + ) -> None: + server, rpc = coordinator() + old_artifacts = server.store_template_artifacts(dict(rpc.template)) + new_artifacts = server.store_template_artifacts( + base_template(height=11, prevhash="22" * 32) + ) + assert old_artifacts is not None and new_artifacts is not None + payout_generation = server._payout_state_generation + + def request_for( + artifacts: object, + *, + publication_critical: bool, + request_source: str, + ) -> object: + return server._new_job_build_request( + artifacts, # type: ignore[arg-type] + None, + mode="ready", + payout_state_generation=payout_generation, + cache_key=server._job_bundle_key( + artifacts, # type: ignore[arg-type] + mode="ready", + payout_state_generation=payout_generation, + worker=None, + ), + publication_critical=publication_critical, + request_source=request_source, + ) + + routine = request_for( + old_artifacts, + publication_critical=False, + request_source="initial", + ) + latest = request_for( + new_artifacts, + publication_critical=True, + request_source="tip_refresh", + ) + routine_flight = SimpleNamespace(request=routine) + server._job_build_active = routine_flight + server._job_build_retiring = None + server._job_build_pending = None + server._start_job_build_locked = ( # type: ignore[method-assign] + lambda request: SimpleNamespace(request=request, future=None) + ) + server._arm_job_build_locked = lambda _flight: None # type: ignore[method-assign] + + promise = server._request_job_build(latest) # type: ignore[arg-type] + + self.assertIs(promise, latest.promise) # type: ignore[union-attr] + self.assertTrue(routine.cancellation.is_set()) # type: ignore[union-attr] + self.assertIs(server._job_build_retiring, routine_flight) + assert server._job_build_active is not None + self.assertIs(server._job_build_active.request, latest) + self.assertEqual( + server.job_build_priority_counts["routine_preempted"], + 1, + ) + + def test_publication_critical_build_restarts_unhealthy_exact_flight( + self, + ) -> None: + for unhealthy in ("almost_expired", "stalled"): + with self.subTest(unhealthy=unhealthy): + server, rpc = coordinator() + artifacts = server.store_template_artifacts(dict(rpc.template)) + assert artifacts is not None + payout_generation = server._payout_state_generation + + def request_for( + *, + publication_critical: bool, + priority_requested_monotonic: float | None = None, + ) -> object: + return server._new_job_build_request( + artifacts, + None, + mode="ready", + payout_state_generation=payout_generation, + cache_key=server._job_bundle_key( + artifacts, + mode="ready", + payout_state_generation=payout_generation, + worker=None, + ), + publication_critical=publication_critical, + request_source=( + "tip_refresh" + if publication_critical + else "initial" + ), + priority_requested_monotonic=( + priority_requested_monotonic + ), + ) + + routine = request_for(publication_critical=False) + now = time.monotonic() + if unhealthy == "almost_expired": + routine.cancellation.started_monotonic = now - 59.99 + routine.cancellation.deadline_monotonic = now + 0.01 + routine.cancellation.last_checkpoint_monotonic = now + else: + routine.cancellation.started_monotonic = now - 1.0 + routine.cancellation.deadline_monotonic = now + 59.0 + routine.cancellation.last_checkpoint_monotonic = ( + now - server.job_build_cancel_grace_seconds - 0.01 + ) + priority_requested = now - 59.0 + latest = request_for( + publication_critical=True, + priority_requested_monotonic=priority_requested, + ) + routine_flight = SimpleNamespace(request=routine) + server._job_build_active = routine_flight + server._job_build_retiring = None + server._job_build_pending = None + server._start_job_build_locked = ( # type: ignore[method-assign] + lambda request: SimpleNamespace(request=request, future=None) + ) + server._arm_job_build_locked = lambda _flight: None # type: ignore[method-assign] + + promise = server._request_job_build(latest) # type: ignore[arg-type] + + self.assertIs(promise, latest.promise) + self.assertIsNot(promise, routine.promise) + self.assertTrue(routine.cancellation.is_set()) + self.assertIs(server._job_build_retiring, routine_flight) + assert server._job_build_active is not None + self.assertIs(server._job_build_active.request, latest) + self.assertEqual( + latest.requested_monotonic, + priority_requested, + ) + self.assertGreater( + latest.cancellation.deadline_monotonic + - time.monotonic(), + server.job_build_timeout_seconds - 1.0, + ) + self.assertEqual( + server.job_build_priority_counts["routine_preempted"], + 1, + ) + + def test_publication_priority_precedes_immutable_request_preparation( + self, + ) -> None: + server, rpc = coordinator() + install_fake_bundle_builder(server) + artifacts = server.store_template_artifacts(dict(rpc.template)) + assert artifacts is not None + critical_preparation_entered = threading.Event() + release_critical_preparation = threading.Event() + initial_preparation_entered = threading.Event() + original_new_request = server._new_job_build_request + + def observed_new_request(*args: object, **kwargs: object) -> object: + request_source = str(kwargs.get("request_source", "routine")) + if request_source == "tip_refresh": + critical_preparation_entered.set() + if not release_critical_preparation.wait(5): + raise AssertionError("test did not release priority preparation") + elif request_source == "initial": + initial_preparation_entered.set() + return original_new_request(*args, **kwargs) # type: ignore[arg-type] + + server._new_job_build_request = observed_new_request # type: ignore[method-assign] + results: dict[str, list[object]] = {"critical": [], "initial": []} + errors: list[BaseException] = [] + + def build(label: str, *, publication_critical: bool) -> None: + try: + results[label].append( + server.shared_job_bundle( + artifacts, + mode="ready", + publication_critical=publication_critical, + request_source=( + "tip_refresh" if publication_critical else "initial" + ), + ) + ) + except BaseException as exc: # noqa: BLE001 - surfaced below + errors.append(exc) + + critical_thread = threading.Thread( + target=build, + args=("critical",), + kwargs={"publication_critical": True}, + ) + initial_thread = threading.Thread( + target=build, + args=("initial",), + kwargs={"publication_critical": False}, + ) + critical_thread.start() + try: + self.assertTrue(critical_preparation_entered.wait(5)) + initial_thread.start() + self.assertFalse(initial_preparation_entered.wait(0.1)) + metrics = "\n".join(server.job_build_metrics_lines()) + self.assertIn("qbit_prism_job_build_priority_active 1", metrics) + finally: + release_critical_preparation.set() + critical_thread.join(5) + initial_thread.join(5) + + self.assertFalse(critical_thread.is_alive()) + self.assertFalse(initial_thread.is_alive()) + self.assertEqual(errors, []) + self.assertEqual([len(results[label]) for label in results], [1, 1]) + self.assertIs(results["critical"][0], results["initial"][0]) + self.assertFalse(initial_preparation_entered.is_set()) + self.assertEqual( + server.initial_job_prepared_work_counts["deferred"], + 1, + ) + self.assertEqual( + server.initial_job_prepared_work_counts["cache_hit"], + 1, + ) + self.assertEqual( + server.job_build_priority_admission_seconds["count"], + 1, + ) + self.assertGreaterEqual( + server.job_build_priority_admission_seconds["sum"], + 0.1, + ) + + def test_priority_reservation_cancels_admitted_routine_preparation( + self, + ) -> None: + server, rpc = coordinator() + install_fake_bundle_builder(server) + artifacts = server.store_template_artifacts(dict(rpc.template)) + assert artifacts is not None + routine_in_payout_lookup = threading.Event() + release_routine_lookup = threading.Event() + routine_request_constructed = threading.Event() + original_usable_artifact = server._usable_payout_ledger_artifact + original_new_request = server._new_job_build_request + routine_thread: threading.Thread + + def blocked_usable_artifact(*args: object, **kwargs: object) -> object: + if threading.current_thread() is routine_thread: + # The pre-admission cache probe also consults the payout + # artifact; only the lookup made under an admitted routine + # preparation exercises the cancellation contract here. + with server._job_build_scheduler_lock: + admitted = bool(server._job_build_routine_preparations) + if admitted: + routine_in_payout_lookup.set() + if not release_routine_lookup.wait(5): + raise AssertionError("test did not release payout lookup") + return original_usable_artifact(*args, **kwargs) # type: ignore[arg-type] + + def observed_new_request(*args: object, **kwargs: object) -> object: + if kwargs.get("request_source") == "initial": + routine_request_constructed.set() + return original_new_request(*args, **kwargs) # type: ignore[arg-type] + + server._usable_payout_ledger_artifact = blocked_usable_artifact # type: ignore[method-assign] + server._new_job_build_request = observed_new_request # type: ignore[method-assign] + errors: list[BaseException] = [] + + def build_routine() -> None: + try: + server.shared_job_bundle( + artifacts, + mode="ready", + retry_superseded=False, + request_source="initial", + ) + except BaseException as exc: # noqa: BLE001 - surfaced below + errors.append(exc) + + routine_thread = threading.Thread(target=build_routine) + routine_thread.start() + priority_token: int | None = None + try: + self.assertTrue(routine_in_payout_lookup.wait(5)) + with server._job_build_scheduler_lock: + routine_cancellations = tuple( + cancellation_ref() + for cancellation_ref in ( + server._job_build_routine_preparations.values() + ) + ) + self.assertEqual(len(routine_cancellations), 1) + self.assertIsNotNone(routine_cancellations[0]) + priority_token, _requested = ( + server._begin_job_build_priority_preparation() + ) + assert routine_cancellations[0] is not None + self.assertTrue(routine_cancellations[0].is_set()) + finally: + release_routine_lookup.set() + routine_thread.join(5) + if priority_token is not None: + server._finish_job_build_priority_preparation(priority_token) + + self.assertFalse(routine_thread.is_alive()) + self.assertEqual(len(errors), 1) + self.assertIsInstance(errors[0], JobBuildSuperseded) + self.assertFalse(routine_request_constructed.is_set()) + with server._job_build_scheduler_lock: + self.assertEqual(len(server._job_build_routine_preparations), 0) + + def test_publication_critical_collection_promotes_past_ready_work(self) -> None: + server, rpc = coordinator(ledger=FakeLedger(miners=["miner-a"])) + old_artifacts = server.store_template_artifacts(dict(rpc.template)) + new_artifacts = server.store_template_artifacts( + base_template(height=11, prevhash="22" * 32) + ) + assert old_artifacts is not None and new_artifacts is not None + payout_generation = server._payout_state_generation + ready = server._new_job_build_request( + old_artifacts, + None, + mode="ready", + payout_state_generation=payout_generation, + cache_key=server._job_bundle_key( + old_artifacts, + mode="ready", + payout_state_generation=payout_generation, + worker=None, + ), + request_source="initial", + ) + latest_identity = worker("tq1latest", "tq1latest.rig") + latest = server._new_job_build_request( + new_artifacts, + latest_identity, + mode="collection", + payout_state_generation=payout_generation, + cache_key=server._job_bundle_key( + new_artifacts, + mode="collection", + payout_state_generation=payout_generation, + worker=latest_identity, + ), + publication_critical=True, + request_source="tip_refresh", + ) + ready_flight = SimpleNamespace(request=ready) + server._job_build_active = ready_flight + server._job_build_retiring = None + server._job_build_pending = latest + armed: list[object] = [] + server._start_job_build_locked = ( # type: ignore[method-assign] + lambda request: SimpleNamespace(request=request, future=None) + ) + server._arm_job_build_locked = armed.append # type: ignore[method-assign] + + server._promote_pending_job_build_locked() + + self.assertTrue(ready.cancellation.is_set()) + self.assertIs(server._job_build_retiring, ready_flight) + self.assertIsNone(server._job_build_pending) + assert server._job_build_active is not None + self.assertIs(server._job_build_active.request, latest) + self.assertEqual(armed, [server._job_build_active]) + self.assertEqual( + server.job_build_priority_counts["routine_preempted"], + 1, + ) + + def test_routine_pending_never_promotes_over_publication_critical_flight( + self, + ) -> None: + for placement in ("active", "retiring"): + with self.subTest(placement=placement): + server, rpc = coordinator() + critical_artifacts = server.store_template_artifacts( + dict(rpc.template) + ) + routine_artifacts = server.store_template_artifacts( + base_template(height=11, prevhash="22" * 32) + ) + assert critical_artifacts is not None + assert routine_artifacts is not None + payout_generation = server._payout_state_generation + + def request_for( + artifacts: object, + *, + publication_critical: bool, + ) -> object: + return server._new_job_build_request( + artifacts, # type: ignore[arg-type] + None, + mode="ready", + payout_state_generation=payout_generation, + cache_key=server._job_bundle_key( + artifacts, # type: ignore[arg-type] + mode="ready", + payout_state_generation=payout_generation, + worker=None, + ), + publication_critical=publication_critical, + request_source=( + "tip_refresh" if publication_critical else "initial" + ), + ) + + critical = request_for( + critical_artifacts, + publication_critical=True, + ) + routine = request_for( + routine_artifacts, + publication_critical=False, + ) + critical_flight = SimpleNamespace(request=critical) + server._job_build_active = ( + critical_flight if placement == "active" else None + ) + server._job_build_retiring = ( + critical_flight if placement == "retiring" else None + ) + server._job_build_pending = routine + server._start_job_build_locked = ( # type: ignore[method-assign] + lambda _request: self.fail( + "routine pending work displaced publication-critical work" + ) + ) + + server._promote_pending_job_build_locked() + + self.assertIs(server._job_build_pending, routine) + self.assertFalse(critical.cancellation.is_set()) + self.assertIs( + ( + server._job_build_active + if placement == "active" + else server._job_build_retiring + ), + critical_flight, + ) + + def test_cancelled_ready_does_not_block_collection_promotion(self) -> None: + for placement in ("active", "retiring"): + with self.subTest(placement=placement): + server, rpc = coordinator(ledger=FakeLedger(miners=["miner-a"])) + artifacts = server.store_template_artifacts(dict(rpc.template)) + assert artifacts is not None + payout_generation = server._payout_state_generation + + def request_for( + mode: str, + identity: WorkerIdentity | None, + ) -> object: + return server._new_job_build_request( + artifacts, + identity, + mode=mode, + payout_state_generation=payout_generation, + cache_key=server._job_bundle_key( + artifacts, + mode=mode, + payout_state_generation=payout_generation, + worker=identity, + ), + ) + + ready = request_for("ready", None) + collection = request_for( + "collection", + worker("tq1collection", "tq1collection.rig"), + ) + self.assertTrue( # type: ignore[union-attr] + ready.cancellation.cancel("superseded") + ) + ready_flight = SimpleNamespace(request=ready) + if placement == "active": + server._job_build_active = ready_flight + server._job_build_retiring = None + else: + server._job_build_active = None + server._job_build_retiring = ready_flight + server._job_build_pending = collection + armed: list[object] = [] + server._start_job_build_locked = ( # type: ignore[method-assign] + lambda request: SimpleNamespace(request=request, future=None) + ) + server._arm_job_build_locked = armed.append # type: ignore[method-assign] + + server._promote_pending_job_build_locked() + + self.assertIsNone(server._job_build_pending) + assert server._job_build_active is not None + self.assertIs(server._job_build_active.request, collection) + self.assertIs(server._job_build_retiring, ready_flight) + self.assertEqual(armed, [server._job_build_active]) + + def test_immediate_collection_completion_does_not_reoccupy_slot(self) -> None: + server, rpc = coordinator(ledger=FakeLedger(miners=["miner-a"])) + artifacts = server.store_template_artifacts(dict(rpc.template)) + assert artifacts is not None + payout_generation = server._payout_state_generation + identities = [ + worker(f"tq1worker-{index}", f"tq1worker-{index}.rig") + for index in range(2) + ] + + def request_for(identity: WorkerIdentity) -> object: + return server._new_job_build_request( + artifacts, + identity, + mode="collection", + payout_state_generation=payout_generation, + cache_key=server._job_bundle_key( + artifacts, + mode="collection", + payout_state_generation=payout_generation, + worker=identity, + ), + ) + + pending = request_for(identities[0]) + incoming = request_for(identities[1]) + results: dict[int, object] = {} + + def completed_flight(request: object) -> object: + result = SimpleNamespace(request=request) + results[id(request)] = result + future: Future[object] = Future() + future.set_result(result) + return SimpleNamespace(request=request, future=future) + + server._job_build_pending = pending + server._start_job_build_locked = completed_flight # type: ignore[method-assign] + + promise = server._request_job_build(incoming) # type: ignore[arg-type] + + self.assertIs(promise.result(), results[id(incoming)]) + self.assertIs( # type: ignore[union-attr] + pending.promise.result(), + results[id(pending)], + ) + self.assertIsNone(server._job_build_active) + self.assertIsNone(server._job_build_retiring) + self.assertIsNone(server._job_build_pending) + + def test_independent_collection_workers_do_not_supersede_each_other(self) -> None: + server, rpc = coordinator(ledger=FakeLedger(miners=["miner-a"])) + recorded = install_fake_bundle_builder(server) + artifacts = server.store_template_artifacts(dict(rpc.template)) + assert artifacts is not None + identities = [ + worker(f"tq1worker-{index}", f"tq1worker-{index}.rig") + for index in range(4) + ] + entered = [threading.Event() for _identity in identities] + releases = [threading.Event() for _identity in identities] + original_build = server.build_shared_job_bundle + active_builds = 0 + max_active_builds = 0 + active_lock = threading.Lock() + + def blocking_build( + build_artifacts: object, + identity: WorkerIdentity, + **kwargs: object, + ) -> object: + nonlocal active_builds, max_active_builds + index = identities.index(identity) + with active_lock: + active_builds += 1 + max_active_builds = max(max_active_builds, active_builds) + entered[index].set() + try: + self.assertTrue(releases[index].wait(5)) + return original_build( + build_artifacts, # type: ignore[arg-type] + identity, + **kwargs, + ) + finally: + with active_lock: + active_builds -= 1 + + server.build_shared_job_bundle = blocking_build # type: ignore[method-assign] + results: list[list[object]] = [[] for _identity in identities] + errors: list[list[BaseException]] = [[] for _identity in identities] + + def build(index: int) -> None: + try: + results[index].append( + server.shared_job_bundle( + artifacts, + identities[index], + mode="collection", + ) + ) + except BaseException as exc: # noqa: BLE001 - surfaced below + errors[index].append(exc) + + threads = [ + threading.Thread(target=build, args=(index,)) + for index in range(len(identities)) + ] + threads[0].start() + try: + self.assertTrue(entered[0].wait(5)) + threads[1].start() + self.assertTrue(entered[1].wait(5)) + threads[2].start() + pending_deadline = time.monotonic() + 5 + while time.monotonic() < pending_deadline: + with server._job_build_scheduler_lock: + pending = server._job_build_pending + if pending is not None and pending.worker == identities[2]: + break + time.sleep(0.01) + else: + self.fail("third collection build was not queued") + threads[3].start() + self.assertEqual(server.job_build_scheduler_counts["starts"], 2) + releases[0].set() + self.assertTrue(entered[2].wait(5)) + releases[1].set() + self.assertTrue(entered[3].wait(5)) + self.assertEqual(server.job_build_scheduler_counts["starts"], 4) + finally: + for release in releases: + release.set() + for thread in threads: + thread.join(5) + + self.assertTrue(all(not thread.is_alive() for thread in threads)) + self.assertEqual(errors, [[], [], [], []]) + self.assertEqual([len(items) for items in results], [1, 1, 1, 1]) + self.assertEqual(recorded["calls"], 4) + self.assertLessEqual(max_active_builds, 2) + self.assertEqual(server.job_build_scheduler_counts["supersessions"], 0) + + def test_collection_independence_requires_one_immutable_cohort(self) -> None: + server, rpc = coordinator(ledger=FakeLedger(miners=["miner-a"])) + first = server.store_template_artifacts(dict(rpc.template)) + assert first is not None + second_template = dict(rpc.template) + second_template["curtime"] = int(second_template["curtime"]) + 1 + second = server.store_template_artifacts(second_template) + assert second is not None + payout_generation = server._payout_state_generation + first_worker = worker("tq1worker-1", "tq1worker-1.rig") + second_worker = worker("tq1worker-2", "tq1worker-2.rig") + + def request_for( + build_artifacts: object, + identity: WorkerIdentity, + ) -> object: + return server._new_job_build_request( + build_artifacts, # type: ignore[arg-type] + identity, + mode="collection", + payout_state_generation=payout_generation, + cache_key=server._job_bundle_key( + build_artifacts, # type: ignore[arg-type] + mode="collection", + payout_state_generation=payout_generation, + worker=identity, + ), + ) + + first_request = request_for(first, first_worker) + peer_request = request_for(first, second_worker) + newer_request = request_for(second, first_worker) + + self.assertTrue( + server._collection_job_builds_are_independent( + first_request, # type: ignore[arg-type] + peer_request, # type: ignore[arg-type] + ) + ) + self.assertFalse( + server._collection_job_builds_are_independent( + first_request, # type: ignore[arg-type] + newer_request, # type: ignore[arg-type] + ) + ) + + def test_ready_build_cancels_both_live_collection_flights(self) -> None: + server, rpc = coordinator(ledger=FakeLedger(miners=["miner-a"])) + server._ensure_tip_refresh_state() + install_fake_bundle_builder(server) + artifacts = server.store_template_artifacts(dict(rpc.template)) + assert artifacts is not None + identities = [ + worker(f"tq1worker-{index}", f"tq1worker-{index}.rig") + for index in range(2) + ] + entered = [threading.Event(), threading.Event()] + cancellation_observed = [threading.Event(), threading.Event()] + release_cancelled = threading.Event() + original_build = server.build_shared_job_bundle + + def blocking_build( + build_artifacts: object, + identity: WorkerIdentity | None, + **kwargs: object, + ) -> object: + request = kwargs["build_request"] + if identity in identities: + index = identities.index(identity) + entered[index].set() + while not request.cancellation.is_set(): # type: ignore[union-attr] + if release_cancelled.wait(0.01): + raise AssertionError("collection build was not cancelled") + cancellation_observed[index].set() + release_cancelled.wait(5) + request.cancellation.raise_if_cancelled( # type: ignore[union-attr] + "test collection hold" + ) + return original_build( + build_artifacts, # type: ignore[arg-type] + identity, + **kwargs, + ) + + server.build_shared_job_bundle = blocking_build # type: ignore[method-assign] + payout_generation = server._payout_state_generation + + def request_for( + mode: str, + identity: WorkerIdentity | None, + ) -> object: + return server._new_job_build_request( + artifacts, + identity, + mode=mode, + payout_state_generation=payout_generation, + cache_key=server._job_bundle_key( + artifacts, + mode=mode, + payout_state_generation=payout_generation, + worker=identity, + ), + ) + + collection_requests = [ + request_for("collection", identity) for identity in identities + ] + collection_promises = [] + ready_promise = None + try: + collection_promises.append( + server._request_job_build(collection_requests[0]) # type: ignore[arg-type] + ) + self.assertTrue(entered[0].wait(5)) + collection_promises.append( + server._request_job_build(collection_requests[1]) # type: ignore[arg-type] + ) + self.assertTrue(entered[1].wait(5)) + + ready_request = request_for("ready", None) + ready_promise = server._request_job_build( # type: ignore[arg-type] + ready_request + ) + self.assertTrue(cancellation_observed[0].wait(5)) + self.assertTrue(cancellation_observed[1].wait(5)) + self.assertFalse(collection_promises[0].done()) + self.assertFalse(collection_promises[1].done()) + self.assertFalse(ready_promise.done()) + self.assertEqual(server.job_build_scheduler_counts["starts"], 2) + finally: + release_cancelled.set() + + assert ready_promise is not None + ready_bundle = ready_promise.result(timeout=5) + self.assertFalse(ready_bundle.collection_only) + for request, promise in zip(collection_requests, collection_promises): + self.assertTrue(request.cancellation.is_set()) # type: ignore[union-attr] + self.assertIsInstance( + promise.exception(timeout=5), + JobBuildSuperseded, + ) + self.assertEqual(server.job_build_scheduler_counts["starts"], 3) + + def test_ready_build_cancels_retiring_only_collection_flight(self) -> None: + server, rpc = coordinator(ledger=FakeLedger(miners=["miner-a"])) + server._ensure_tip_refresh_state() + install_fake_bundle_builder(server) + artifacts = server.store_template_artifacts(dict(rpc.template)) + assert artifacts is not None + identities = [ + worker(f"tq1worker-{index}", f"tq1worker-{index}.rig") + for index in range(2) + ] + entered = [threading.Event(), threading.Event()] + release_active = threading.Event() + retiring_cancelled = threading.Event() + release_retiring = threading.Event() + original_build = server.build_shared_job_bundle + + def blocking_build( + build_artifacts: object, + identity: WorkerIdentity | None, + **kwargs: object, + ) -> object: + request = kwargs["build_request"] + if identity == identities[0]: + entered[0].set() + while not request.cancellation.is_set(): # type: ignore[union-attr] + if release_retiring.wait(0.01): + raise AssertionError("retiring build was not cancelled") + retiring_cancelled.set() + release_retiring.wait(5) + request.cancellation.raise_if_cancelled( # type: ignore[union-attr] + "test retiring-only hold" + ) + elif identity == identities[1]: + entered[1].set() + self.assertTrue(release_active.wait(5)) + return original_build( + build_artifacts, # type: ignore[arg-type] + identity, + **kwargs, + ) + + server.build_shared_job_bundle = blocking_build # type: ignore[method-assign] + payout_generation = server._payout_state_generation + + def request_for( + mode: str, + identity: WorkerIdentity | None, + ) -> object: + return server._new_job_build_request( + artifacts, + identity, + mode=mode, + payout_state_generation=payout_generation, + cache_key=server._job_bundle_key( + artifacts, + mode=mode, + payout_state_generation=payout_generation, + worker=identity, + ), + ) + + first_request = request_for("collection", identities[0]) + second_request = request_for("collection", identities[1]) + first_promise = server._request_job_build( # type: ignore[arg-type] + first_request + ) + self.assertTrue(entered[0].wait(5)) + second_promise = server._request_job_build( # type: ignore[arg-type] + second_request + ) + self.assertTrue(entered[1].wait(5)) + try: + release_active.set() + second_bundle = second_promise.result(timeout=5) + self.assertTrue(second_bundle.collection_only) + with server._job_build_scheduler_lock: + self.assertIsNone(server._job_build_active) + assert server._job_build_retiring is not None + self.assertIs( + server._job_build_retiring.request, + first_request, + ) + + ready_request = request_for("ready", None) + ready_promise = server._request_job_build( # type: ignore[arg-type] + ready_request + ) + self.assertTrue(retiring_cancelled.wait(5)) + ready_bundle = ready_promise.result(timeout=5) + self.assertFalse(ready_bundle.collection_only) + self.assertTrue(first_request.cancellation.is_set()) # type: ignore[union-attr] + finally: + release_active.set() + release_retiring.set() + + self.assertIsInstance( + first_promise.exception(timeout=5), + JobBuildSuperseded, + ) + + def test_collection_retries_do_not_supersede_ready_build(self) -> None: + server, rpc = coordinator(ledger=FakeLedger(miners=["miner-a"])) + server._ensure_tip_refresh_state() + install_fake_bundle_builder(server) + artifacts = server.store_template_artifacts(dict(rpc.template)) + assert artifacts is not None + identities = [ + worker(f"tq1worker-{index}", f"tq1worker-{index}.rig") + for index in range(2) + ] + collection_entered = [threading.Event(), threading.Event()] + collection_cancelled = [threading.Event(), threading.Event()] + release_cancelled = threading.Event() + ready_entered = threading.Event() + release_ready = threading.Event() + stop_collections = threading.Event() + ready_requests: list[object] = [] + collection_requests: list[object | None] = [None, None] + original_build = server.build_shared_job_bundle + + def blocking_build( + build_artifacts: object, + identity: WorkerIdentity | None, + **kwargs: object, + ) -> object: + request = kwargs["build_request"] + if request.mode == "ready": # type: ignore[union-attr] + ready_requests.append(request) + ready_entered.set() + self.assertTrue(release_ready.wait(5)) + elif identity in identities: + index = identities.index(identity) + collection_requests[index] = request + collection_entered[index].set() + while not request.cancellation.is_set(): # type: ignore[union-attr] + if release_cancelled.wait(0.01): + raise AssertionError("collection build was not cancelled") + collection_cancelled[index].set() + release_cancelled.wait(5) + request.cancellation.raise_if_cancelled( # type: ignore[union-attr] + "test collection retry hold" + ) + return original_build( + build_artifacts, # type: ignore[arg-type] + identity, + **kwargs, + ) + + server.build_shared_job_bundle = blocking_build # type: ignore[method-assign] + collection_results: list[list[object]] = [[], []] + collection_errors: list[list[BaseException]] = [[], []] + ready_results: list[object] = [] + ready_errors: list[BaseException] = [] + + def build_collection(index: int) -> None: + try: + collection_results[index].append( + server.shared_job_bundle( + artifacts, + identities[index], + mode="collection", + cancelled=stop_collections.is_set, + ) + ) + except BaseException as exc: # noqa: BLE001 - surfaced below + collection_errors[index].append(exc) + + def build_ready() -> None: + try: + ready_results.append( + server.shared_job_bundle( + artifacts, + mode="ready", + retry_superseded=False, + ) + ) + except BaseException as exc: # noqa: BLE001 - surfaced below + ready_errors.append(exc) + + collection_threads = [ + threading.Thread(target=build_collection, args=(index,)) + for index in range(2) + ] + ready_thread = threading.Thread(target=build_ready) + try: + collection_threads[0].start() + self.assertTrue(collection_entered[0].wait(5)) + collection_threads[1].start() + self.assertTrue(collection_entered[1].wait(5)) + ready_thread.start() + self.assertTrue(collection_cancelled[0].wait(5)) + self.assertTrue(collection_cancelled[1].wait(5)) + self.assertEqual(server.job_build_scheduler_counts["starts"], 2) + + release_cancelled.set() + self.assertTrue(ready_entered.wait(5)) + retry_deadline = time.monotonic() + 5 + while ( + server.job_build_scheduler_counts["requests"] < 5 + and time.monotonic() < retry_deadline + ): + time.sleep(0.01) + self.assertGreaterEqual( + server.job_build_scheduler_counts["requests"], + 5, + ) + self.assertEqual(server.job_build_scheduler_counts["starts"], 3) + self.assertEqual(len(ready_requests), 1) + self.assertFalse( + ready_requests[0].cancellation.is_set() # type: ignore[union-attr] + ) + + stop_collections.set() + for thread in collection_threads: + thread.join(2) + release_ready.set() + ready_thread.join(5) + finally: + stop_collections.set() + release_cancelled.set() + release_ready.set() + for thread in collection_threads: + if thread.ident is not None: + thread.join(5) + if ready_thread.ident is not None: + ready_thread.join(5) + + self.assertTrue(all(not thread.is_alive() for thread in collection_threads)) + self.assertFalse(ready_thread.is_alive()) + self.assertEqual(collection_results, [[], []]) + self.assertEqual([len(errors) for errors in collection_errors], [1, 1]) + self.assertEqual(ready_errors, []) + self.assertEqual(len(ready_results), 1) + self.assertFalse(ready_results[0].collection_only) # type: ignore[union-attr] + for request in collection_requests: + assert request is not None + self.assertTrue(request.cancellation.is_set()) # type: ignore[union-attr] + + def test_shutdown_cancels_builder_with_full_helper_input_pipe(self) -> None: + server, rpc = coordinator() + server.signing_seed_hex = "42" * 32 + server.ledger_attestation_signing_seed_hex = "43" * 32 + artifacts = server.store_template_artifacts(dict(rpc.template)) + assert artifacts is not None + payout_generation = server._payout_state_generation + request = server._new_job_build_request( + artifacts, + None, + mode="ready", + payout_state_generation=payout_generation, + cache_key=server._job_bundle_key( + artifacts, + mode="ready", + payout_state_generation=payout_generation, + worker=None, + ), + ) + helper_started = threading.Event() + helper_processes: list[subprocess.Popen[str]] = [] + real_popen = subprocess.Popen + + def capture_popen(*args: object, **kwargs: object) -> subprocess.Popen[str]: + process = real_popen(*args, **kwargs) # type: ignore[arg-type] + helper_processes.append(process) + helper_started.set() + return process + + def fill_helper_pipe(*_args: object, **kwargs: object) -> object: + build_request = kwargs["build_request"] + return server.build_audit_bundle( + shares=[], + found_block={ + "block_height": 10, + "coinbase_value_sats": 50_00000000, + "network_difficulty": 1, + "anchor_job_issued_at_ms": 1_700_000_000_000, + }, + prior_balances=[ + { + "miner_id": "pipe-filler", + "balance_sats": 1, + "padding": "x" * (4 * 1024 * 1024), + } + ], + coinbase_script_sig_suffix_hex="00", + cancellation=build_request.cancellation, # type: ignore[union-attr] + ) + + server.build_shared_job_bundle = fill_helper_pipe # type: ignore[method-assign] + shutdown_finished = threading.Event() + shutdown_errors: list[BaseException] = [] + + def shutdown() -> None: + try: + server.shutdown_job_build_executor() + except BaseException as exc: # noqa: BLE001 - surfaced below + shutdown_errors.append(exc) + finally: + shutdown_finished.set() + + with patch( + "lab.prism.prism_coordinator.prism_tool_command", + return_value=[sys.executable, "-c", "import time; time.sleep(30)"], + ), patch( + "lab.prism.prism_coordinator.subprocess.Popen", + side_effect=capture_popen, + ): + promise = server._request_job_build(request) + self.assertTrue( + helper_started.wait(5), + repr(promise.exception(timeout=1)) if promise.done() else None, + ) + time.sleep(0.1) + self.assertFalse(promise.done()) + shutdown_thread = threading.Thread(target=shutdown) + shutdown_thread.start() + shutdown_returned = shutdown_finished.wait(2) + if not shutdown_returned: + for process in helper_processes: + if process.poll() is None: + process.kill() + shutdown_thread.join(5) + + self.assertTrue(shutdown_returned) + self.assertFalse(shutdown_thread.is_alive()) + self.assertEqual(shutdown_errors, []) + self.assertIsInstance(promise.exception(timeout=1), JobBuildSuperseded) + + def test_control_cancel_during_serialization_is_supersession(self) -> None: + server, rpc = coordinator() + server.signing_seed_hex = "42" * 32 + server.ledger_attestation_signing_seed_hex = "43" * 32 + artifacts = server.store_template_artifacts(dict(rpc.template)) + assert artifacts is not None + helper_started = threading.Event() + helper_processes: list[subprocess.Popen[str]] = [] + real_popen = subprocess.Popen + + def capture_popen(*args: object, **kwargs: object) -> subprocess.Popen[str]: + process = real_popen(*args, **kwargs) # type: ignore[arg-type] + helper_processes.append(process) + helper_started.set() + return process + + def fill_helper_pipe(*_args: object, **kwargs: object) -> object: + build_request = kwargs["build_request"] + return server.build_audit_bundle( + shares=[], + found_block={ + "block_height": 10, + "coinbase_value_sats": 50_00000000, + "network_difficulty": 1, + "anchor_job_issued_at_ms": 1_700_000_000_000, + }, + prior_balances=[ + { + "miner_id": "pipe-filler", + "balance_sats": 1, + "padding": "x" * (4 * 1024 * 1024), + } + ], + coinbase_script_sig_suffix_hex="00", + cancellation=build_request.cancellation, # type: ignore[union-attr] + ) + + server.build_shared_job_bundle = fill_helper_pipe # type: ignore[method-assign] + errors: list[BaseException] = [] + + def build() -> None: + try: + server.shared_job_bundle( + artifacts, + mode="ready", + retry_superseded=False, + ) + except BaseException as exc: # noqa: BLE001 - surfaced below + errors.append(exc) + + with patch( + "lab.prism.prism_coordinator.prism_tool_command", + return_value=[sys.executable, "-c", "import time; time.sleep(30)"], + ), patch( + "lab.prism.prism_coordinator.subprocess.Popen", + side_effect=capture_popen, + ): + build_thread = threading.Thread(target=build) + build_thread.start() + self.assertTrue(helper_started.wait(5)) + with server._job_cache_lock: + controls = list(server._active_job_bundle_builds.values()) + self.assertEqual(len(controls), 1) + controls[0].cancel_event.set() + build_thread.join(2) + if build_thread.is_alive(): + for process in helper_processes: + if process.poll() is None: + process.kill() + build_thread.join(5) + + self.assertFalse(build_thread.is_alive()) + self.assertEqual(len(errors), 1) + self.assertIsInstance(errors[0], JobBuildSuperseded) + self.assertEqual(server.shared_bundle_build_counts["superseded"], 1) + self.assertEqual(server.shared_bundle_build_counts["failed"], 0) + self.assertEqual(server.tip_refresh_superseded_results, 1) + + def test_full_helper_input_pipe_obeys_builder_timeout(self) -> None: + server, _rpc = coordinator() + server.signing_seed_hex = "42" * 32 + server.ledger_attestation_signing_seed_hex = "43" * 32 + server.bundle_build_timeout_seconds = 0.05 + helper_started = threading.Event() + helper_processes: list[subprocess.Popen[str]] = [] + real_popen = subprocess.Popen + + def capture_popen(*args: object, **kwargs: object) -> subprocess.Popen[str]: + process = real_popen(*args, **kwargs) # type: ignore[arg-type] + helper_processes.append(process) + helper_started.set() + return process + + errors: list[BaseException] = [] + + def build() -> None: + try: + server.build_audit_bundle( + shares=[], + found_block={ + "block_height": 10, + "coinbase_value_sats": 50_00000000, + "network_difficulty": 1, + "anchor_job_issued_at_ms": 1_700_000_000_000, + }, + prior_balances=[ + { + "miner_id": "pipe-filler", + "balance_sats": 1, + "padding": "x" * (4 * 1024 * 1024), + } + ], + coinbase_script_sig_suffix_hex="00", + ) + except BaseException as exc: # noqa: BLE001 - surfaced below + errors.append(exc) + + with patch( + "lab.prism.prism_coordinator.prism_tool_command", + return_value=[sys.executable, "-c", "import time; time.sleep(30)"], + ), patch( + "lab.prism.prism_coordinator.subprocess.Popen", + side_effect=capture_popen, + ): + build_thread = threading.Thread(target=build) + build_thread.start() + self.assertTrue(helper_started.wait(5)) + build_thread.join(2) + if build_thread.is_alive(): + for process in helper_processes: + if process.poll() is None: + process.kill() + build_thread.join(5) + + self.assertFalse(build_thread.is_alive()) + self.assertEqual(len(errors), 1) + self.assertIsInstance(errors[0], RuntimeError) + self.assertIn("timed out", str(errors[0])) + + def test_untrusted_outcome_for_other_tip_keeps_memo_armed(self) -> None: + server, rpc = coordinator() + server.reorg_reconciler_enabled = True + server._note_reorg_reconcile_outcome(rpc.tip, trusted=True) + reconcile_calls: list[str | None] = [] + + def fake_reconcile(*, tip_hash: str | None = None) -> dict[str, object]: + reconcile_calls.append(tip_hash) + server._note_reorg_reconcile_outcome(tip_hash, trusted=True) + return {"untrusted": False} + + server.reconcile_prism_pool_blocks_once = fake_reconcile # type: ignore[method-assign] + + # A superseded/untrusted pass for a different tip must not unarm the + # cached trusted outcome for the current tip. + server._note_reorg_reconcile_outcome("44" * 32, trusted=False) + self.assertTrue(server.ensure_reorg_reconciled_for_current_tip()) + self.assertEqual(reconcile_calls, []) + + # An untrusted outcome for the current tip itself unarms only it. + server._note_reorg_reconcile_outcome(rpc.tip, trusted=False) + self.assertTrue(server.ensure_reorg_reconciled_for_current_tip()) + self.assertEqual(reconcile_calls, [rpc.tip]) + + def test_reconcile_error_outcome_clears_every_memo_entry(self) -> None: + server, rpc = coordinator() + server.reorg_reconciler_enabled = True + server._note_reorg_reconcile_outcome(rpc.tip, trusted=True) + server._note_reorg_reconcile_outcome("55" * 32, trusted=True) + reconcile_calls: list[str | None] = [] + + def fake_reconcile(*, tip_hash: str | None = None) -> dict[str, object]: + reconcile_calls.append(tip_hash) + server._note_reorg_reconcile_outcome(tip_hash, trusted=True) + return {"untrusted": False} + + server.reconcile_prism_pool_blocks_once = fake_reconcile # type: ignore[method-assign] + + server._note_reorg_reconcile_outcome( + "55" * 32, + trusted=False, + clear_memo=True, + ) + + self.assertTrue(server.ensure_reorg_reconciled_for_current_tip()) + self.assertEqual(reconcile_calls, [rpc.tip]) + + def test_tip_flip_back_within_ttl_requires_fresh_reconcile(self) -> None: + server, rpc = coordinator() + server.reorg_reconciler_enabled = True + tip_a = rpc.tip + self.addCleanup(server.shutdown_tip_refresh_executor) + server.observe_tip_for_refresh(tip_a) + server._note_reorg_reconcile_outcome(tip_a, trusted=True) + reconcile_calls: list[str | None] = [] + + def fake_reconcile(*, tip_hash: str | None = None) -> dict[str, object]: + reconcile_calls.append(tip_hash) + return {"untrusted": False} + + server.reconcile_prism_pool_blocks_once = fake_reconcile # type: ignore[method-assign] + + self.assertTrue(server.ensure_reorg_reconciled_for_current_tip()) + self.assertEqual(reconcile_calls, []) + + # Detecting an intervening tip ends tip A's memo epoch: when the + # chain flips back to A within the cache TTL, pool-block chain state + # must be re-proven by a fresh pass, not assumed from the pre-flip + # outcome. + server.observe_tip_for_refresh("bb" * 32) + + self.assertTrue(server.ensure_reorg_reconciled_for_current_tip()) + self.assertEqual(reconcile_calls, [tip_a]) + + def test_stale_pass_cannot_rearm_memo_after_newer_tip_detected(self) -> None: + server, rpc = coordinator() + server.reorg_reconciler_enabled = True + tip_a = rpc.tip + self.addCleanup(server.shutdown_tip_refresh_executor) + server.observe_tip_for_refresh(tip_a) + server.observe_tip_for_refresh("bb" * 32) + + # A pass for tip A finishing after tip B was detected must not arm + # the memo for A: its epoch already ended at the newer observation. + server._note_reorg_reconcile_outcome(tip_a, trusted=True) + reconcile_calls: list[str | None] = [] + + def fake_reconcile(*, tip_hash: str | None = None) -> dict[str, object]: + reconcile_calls.append(tip_hash) + return {"untrusted": False} + + server.reconcile_prism_pool_blocks_once = fake_reconcile # type: ignore[method-assign] + + self.assertTrue(server.ensure_reorg_reconciled_for_current_tip()) + self.assertEqual(reconcile_calls, [tip_a]) + + def test_rearmed_artifact_is_not_shadowed_by_stale_cached_bundle( + self, + ) -> None: + server, rpc = coordinator() + recorded = install_fake_bundle_builder(server) + artifacts = server.store_template_artifacts(dict(rpc.template)) + assert artifacts is not None + + stale = server.shared_job_bundle(artifacts, mode="ready") + self.assertEqual(recorded["calls"], 1) + self.assertEqual(len(stale.shares_json), 3) + + # A share lands and the debounced re-arm rebuilds a fresher window. + server.ledger.miners = [*server.ledger.miners, "late-share"] + server._prepare_payout_ledger_artifact( + server._payout_state_generation, + artifacts.network_difficulty, + ) + with server._job_cache_lock: + fresh_artifact = server._payout_ledger_artifact + assert fresh_artifact is not None + self.assertEqual(fresh_artifact.accepted_share_count, 4) + + rebuilt = server.shared_job_bundle(artifacts, mode="ready") + + # The pre-re-arm bundle is still inside its cache TTL under the + # no-artifact key; its older window must not shadow the fresher + # artifact. + self.assertEqual( + rebuilt.payout_artifact_generation, + fresh_artifact.generation, + ) + self.assertEqual(len(rebuilt.shares_json), 4) + self.assertEqual(recorded["calls"], 2) + + def test_in_flight_build_keeps_aged_artifact_it_already_selected( + self, + ) -> None: + server, rpc = coordinator() + install_fake_bundle_builder(server) + artifacts = server.store_template_artifacts(dict(rpc.template)) + assert artifacts is not None + server._prepare_payout_ledger_artifact( + server._payout_state_generation, + artifacts.network_difficulty, + ) + server.payout_artifact_max_anchor_age_seconds = 5.0 + with server._job_cache_lock: + armed = server._payout_ledger_artifact + assert armed is not None + assert armed.snapshot_anchor_ms is not None + aged = dataclass_replace( + armed, + snapshot_anchor_ms=int(armed.snapshot_anchor_ms) - 6_000, + ) + server._payout_ledger_artifact = aged + + # New reuse decisions reject a window past the audit ceiling... + self.assertIsNone( + server._usable_payout_ledger_artifact( + server._payout_state_generation, + artifacts.network_difficulty, + rearm_on_fence_failure=False, + ) + ) + # ...but a build that selected it while fresh completes with it: the + # in-build re-validation checks supersession and the balances fence + # only, so queue delay past the bound cannot scrap the reuse into + # the full snapshot it was armed to avoid. + server.ledger.snapshot_calls = 0 + bundle = server.build_shared_job_bundle( + artifacts, + worker(), + payout_artifact=aged, + ) + self.assertEqual(server.ledger.snapshot_calls, 0) + self.assertEqual( + bundle.found_block["anchor_job_issued_at_ms"], + aged.snapshot_anchor_ms, + ) + + def test_in_flight_build_survives_same_window_anchor_refresh(self) -> None: + server, rpc = coordinator() + install_fake_bundle_builder(server) + artifacts = server.store_template_artifacts(dict(rpc.template)) + assert artifacts is not None + server._prepare_payout_ledger_artifact( + server._payout_state_generation, + artifacts.network_difficulty, + ) + with server._job_cache_lock: + armed = server._payout_ledger_artifact + assert armed is not None + assert armed.snapshot_anchor_ms is not None + # An in-flight build selected the armed artifact while it + # carried an older anchor. + selected = dataclass_replace( + armed, + snapshot_anchor_ms=int(armed.snapshot_anchor_ms) - 5, + ) + server._payout_ledger_artifact = selected + + # A same-window rebuild refreshes the anchor in place: the stored + # instance swaps while the generation -- the re-key authority -- + # and the window bytes stay identical. + server._prepare_payout_ledger_artifact( + server._payout_state_generation, + artifacts.network_difficulty, + ) + with server._job_cache_lock: + refreshed = server._payout_ledger_artifact + assert refreshed is not None + self.assertIsNot(refreshed, selected) + self.assertEqual(refreshed.generation, selected.generation) + self.assertEqual( + refreshed.share_snapshot_sha256, + selected.share_snapshot_sha256, + ) + + # The build holding the pre-refresh instance completes its reuse + # instead of being scrapped into a full snapshot by the swap. + server.ledger.snapshot_calls = 0 + bundle = server.build_shared_job_bundle( + artifacts, + worker(), + payout_artifact=selected, + ) + self.assertEqual(server.ledger.snapshot_calls, 0) + self.assertEqual( + bundle.found_block["anchor_job_issued_at_ms"], + selected.snapshot_anchor_ms, + ) + + def test_generation_current_cached_bundle_serves_past_reanchor_floor( + self, + ) -> None: + server, rpc = coordinator() + recorded = install_fake_bundle_builder(server) + first = server.store_template_artifacts(dict(rpc.template)) + assert first is not None + with server._payout_artifact_executor_lock: + server._payout_artifact_executor_shutdown = True + # Re-anchor floor below the bundle-cache TTL so the two gates are + # distinguishable inside the TTL window. + server.payout_artifact_reanchor_seconds = 5.0 + + server.shared_job_bundle(first, mode="ready") + self.assertEqual(recorded["calls"], 1) + with server._job_cache_lock: + armed = server._payout_ledger_artifact + assert armed is not None + + # A bundle keyed to the currently armed artifact generation carries + # exactly the armed window: it keeps serving past the re-anchor + # floor (its window is as fresh as reuse itself), bounded upstream + # by the cache TTL. + second = server.store_template_artifacts( + base_template(height=11, prevhash="22" * 32) + ) + assert second is not None + reused = server.shared_job_bundle(second, mode="ready") + self.assertEqual(reused.payout_artifact_generation, armed.generation) + self.assertEqual(recorded["calls"], 2) + with server._job_cache_lock: + for key, entry in list(server._job_bundle_cache.items()): + if entry.payout_artifact_generation == armed.generation: + server._job_bundle_cache[key] = dataclass_replace( + entry, + built_monotonic=float(entry.built_monotonic) - 7.0, + ) + server.shared_job_bundle(second, mode="ready") + self.assertEqual(recorded["calls"], 2) + + def test_cached_no_artifact_bundle_gates_on_build_age_not_anchor( + self, + ) -> None: + # 2026-07-29 regression companion: the gate must follow build age + # (freshness of the work) inside the audit ceiling, never the + # wall-clock anchor, which is frozen per template generation. + server, rpc = coordinator() + recorded = install_fake_bundle_builder(server) + artifacts = server.store_template_artifacts(dict(rpc.template)) + assert artifacts is not None + with server._payout_artifact_executor_lock: + server._payout_artifact_executor_shutdown = True + server.payout_artifact_reanchor_seconds = 5.0 + + seeded = server.shared_job_bundle(artifacts, mode="ready") + self.assertEqual(recorded["calls"], 1) + + # Fresh build age serves from cache once the armed artifact is + # gone. + with server._job_cache_lock: + server._payout_ledger_artifact = None + server.shared_job_bundle(artifacts, mode="ready") + self.assertEqual(recorded["calls"], 1) + + # Build age past the re-anchor floor rebuilds even though the + # bundle-cache TTL has not lapsed. + with server._job_cache_lock: + server._payout_ledger_artifact = None + for key, entry in list(server._job_bundle_cache.items()): + server._job_bundle_cache[key] = dataclass_replace( + entry, + built_monotonic=float(entry.built_monotonic) - 7.0, + ) + server.shared_job_bundle(artifacts, mode="ready") + self.assertEqual(recorded["calls"], 2) + + # The audit ceiling on the declared anchor stays as the backstop + # even for a freshly built entry. + aged_anchor_ms = ( + int(seeded.found_block["anchor_job_issued_at_ms"]) - 400_000 + ) + with server._job_cache_lock: + server._payout_ledger_artifact = None + for key, entry in list(server._job_bundle_cache.items()): + aged_found_block = dict(entry.found_block) + aged_found_block["anchor_job_issued_at_ms"] = aged_anchor_ms + server._job_bundle_cache[key] = dataclass_replace( + entry, + found_block=aged_found_block, + ) + server.shared_job_bundle(artifacts, mode="ready") + self.assertEqual(recorded["calls"], 3) + + def test_generation_bump_does_not_scrap_in_flight_build(self) -> None: + # 2026-07-29 regression: every differing window bumps the armed + # artifact generation, and the in-build re-validation scrapped any + # build whose selected artifact lost the slot -- a rebuild storm at + # production share rates. A build must finish on the copy it + # selected: its window stays audit-reproducible at its declared + # anchor while the payout generation and published balances hold. + server, rpc = coordinator() + install_fake_bundle_builder(server) + artifacts = server.store_template_artifacts(dict(rpc.template)) + assert artifacts is not None + server._prepare_payout_ledger_artifact( + server._payout_state_generation, + artifacts.network_difficulty, + ) + with server._job_cache_lock: + selected = server._payout_ledger_artifact + assert selected is not None + assert selected.snapshot_anchor_ms is not None + + fresher_window = [ + {"share_seq": seq, "miner_id": "miner-a"} for seq in (1, 2, 3, 4) + ] + fresher = dataclass_replace( + selected, + generation=0, + accepted_share_count=4, + shares_json=tuple(fresher_window), + snapshot_anchor_ms=int(selected.snapshot_anchor_ms) + 5, + share_snapshot_sha256=canonical_json_sha256(fresher_window), + ) + self.assertTrue(server._install_payout_ledger_artifact(fresher)) + with server._job_cache_lock: + bumped = server._payout_ledger_artifact + assert bumped is not None + self.assertGreater(bumped.generation, selected.generation) + + server.ledger.snapshot_calls = 0 + bundle = server.build_shared_job_bundle( + artifacts, + worker(), + payout_artifact=selected, + ) + self.assertEqual(server.ledger.snapshot_calls, 0) + self.assertEqual( + bundle.found_block["anchor_job_issued_at_ms"], + selected.snapshot_anchor_ms, + ) + self.assertEqual(bundle.shares_json, list(selected.shares_json)) + + def test_rebuilt_bundle_serves_under_aged_template_generation( + self, + ) -> None: + # 2026-07-29 regression: issued_at_ms is frozen per template + # generation and predates the walk, so a wall-clock anchor gate on + # cached bundles declared every rebuilt bundle dead on arrival once + # its template generation outlived the bound. Below the audit + # ceiling, cache service must follow build age, not anchor age. + server, rpc = coordinator() + recorded = install_fake_bundle_builder(server) + artifacts = server.store_template_artifacts(dict(rpc.template)) + assert artifacts is not None + with server._payout_artifact_executor_lock: + server._payout_artifact_executor_shutdown = True + with server._job_cache_lock: + server._job_build_issued_at_ms[artifacts.generation] = ( + now_ms() - 20_000 + ) + + built = server.shared_job_bundle(artifacts, mode="ready") + self.assertEqual(recorded["calls"], 1) + self.assertEqual( + built.found_block["anchor_job_issued_at_ms"], + built.issued_at_ms, + ) + + served = server.shared_job_bundle(artifacts, mode="ready") + self.assertEqual(recorded["calls"], 1) + self.assertEqual(served.issued_at_ms, built.issued_at_ms) + + def test_admission_deadline_fails_fast_and_visibly(self) -> None: + server, _rpc = coordinator() + server.routine_admission_deadline_seconds = 0.05 + server._publication_priority_scheduled_locked = ( # type: ignore[method-assign] + lambda: True + ) + + with self.assertRaises(JobBuildAdmissionDeadlineExceeded) as raised: + server._begin_routine_job_build_preparation( + request_source="initial", + cancelled=None, + ) + + # The deadline is coordination churn, not a failure budget event: + # callers retry with their normal pacing. + self.assertIsInstance(raised.exception, TemplateRefreshBlocked) + with server._job_build_scheduler_lock: + self.assertEqual( + server.initial_job_prepared_work_counts["admission_deadline"], + 1, + ) + + +class ShareWindowSpoolTests(unittest.TestCase): + """Spool-file handoff of the serialized share window to the builder.""" + + def _coordinator(self) -> PrismCoordinator: + server, _rpc = coordinator() + server.signing_seed_hex = "42" * 32 + server.ledger_attestation_signing_seed_hex = "43" * 32 + self.addCleanup(server.retire_share_window_spool) + return server + + def _ledger_artifact( + self, + shares: list[dict[str, object]], + *, + generation: int, + ) -> PayoutLedgerArtifact: + return PayoutLedgerArtifact( + generation=generation, + payout_state_generation=0, + network_difficulty=1, + accepted_share_count=len(shares), + shares_json=tuple(shares), + prior_balances=(), + prepared_monotonic=time.monotonic(), + snapshot_anchor_ms=None, + ) + + def _build_with_echo_builder( + self, + server: PrismCoordinator, + shares: list[dict[str, object]], + serialization: object, + *, + height: int, + ) -> dict[str, object]: + # These tests cover the one-shot spool handoff; pin the persistent + # builder off so the echo helper is never spawned as a daemon. + with patch.dict(os.environ, {"PRISM_BUILDER_SERVE": "0"}), patch( + "lab.prism.prism_coordinator.prism_tool_command", + return_value=list(ECHO_BUILDER_COMMAND), + ): + return server.build_audit_bundle( + shares=shares, + found_block={ + "block_height": height, + "coinbase_value_sats": 50_00000000, + "network_difficulty": 1, + "anchor_job_issued_at_ms": 1_700_000_000_000, + }, + prior_balances=[], + coinbase_script_sig_suffix_hex="00", + summary_only=True, + payout_policy={"policy": "day-one"}, + share_serialization=serialization, # type: ignore[arg-type] + ) + + def _expected_payload( + self, + shares: list[dict[str, object]], + *, + height: int, + ) -> dict[str, object]: + identities, compact_shares = _compact_share_payload(shares) + return { + "found_block": { + "block_height": height, + "coinbase_value_sats": 50_00000000, + "network_difficulty": 1, + "anchor_job_issued_at_ms": 1_700_000_000_000, + }, + "prior_balances": [], + "payout_policy": {"policy": "day-one"}, + "coinbase_script_sig_suffix_hex": "00", + "witness_merkle_leaves_hex": [], + "compact_share_identities": [ + list(identity) for identity in identities + ], + "compact_shares": [list(share) for share in compact_shares], + } + + def _awkward_shares(self, count: int) -> list[dict[str, object]]: + shares = [] + for seq in range(1, count + 1): + share = spool_share(seq) + share["share_id"] = f"miner-\u00e9.rig{seq}:\U0001F600,}}{{\"\\" + share["miner_id"] = f"miner-{seq % 4}-\u2603" + share["order_key"] = f"{seq % 4:02d}:miner-{seq % 4}" + share["share_difficulty"] = (1 << 130) + seq + share["credit_policy"] = "policy-\u00fc" if seq % 3 else None + shares.append(share) + return shares + + def _historical_tail(self, shares: list[dict[str, object]]) -> str: + identities, compact_shares = _compact_share_payload(shares) + return ( + ',"compact_share_identities":' + + json.dumps(identities, separators=(",", ":")) + + ',"compact_shares":' + + json.dumps(compact_shares, separators=(",", ":")) + + "}" + ) + + def test_compact_tail_chunks_match_the_historical_fragment_bytes(self) -> None: + for count in (0, 1, 2, 1_300): + shares = self._awkward_shares(count) + expected = self._historical_tail(shares) + for batch, chunk in ((512, 64 * 1024), (1, 1), (7, 100), (5_000, 10)): + with self.subTest(count=count, batch=batch, chunk=chunk): + chunks = _compact_share_tail_chunks( + shares, + batch_records=batch, + chunk_chars=chunk, + ) + self.assertEqual("".join(chunks), expected) + self.assertTrue(all(piece.isascii() for piece in chunks)) + + def test_spool_and_in_memory_tail_hold_identical_bounded_bytes(self) -> None: + server = self._coordinator() + shares = self._awkward_shares(1_300) + serialization = server._share_window_serialization_for_artifact( + self._ledger_artifact(shares, generation=1), + shares, + ) + expected = self._historical_tail(shares) + chunks = serialization.compact_tail_chunks(shares) + self.assertGreater(len(chunks), 4) + self.assertEqual("".join(chunks), expected) + self.assertIs(serialization.compact_tail_chunks(shares), chunks) + self.assertEqual(serialization.compact_json_bytes, len(expected)) + lease = serialization.acquire_spooled_tail(shares) + assert lease is not None + spool_file, spool_size = lease + try: + spool_file.seek(0) + self.assertEqual(spool_file.read(), expected.encode("utf-8")) + self.assertEqual(spool_size, len(expected)) + finally: + serialization.release_spooled_tail() + # Both transports deliver the same parsed window. + first = self._build_with_echo_builder(server, shares, serialization, height=10) + with patch( + "lab.prism.prism_coordinator._share_window_spool_file", + side_effect=OSError("temp filesystem unavailable"), + ): + fresh = server._share_window_serialization_for_artifact( + self._ledger_artifact(shares, generation=2), + shares, + ) + fresh.retire_spool() + second = self._build_with_echo_builder(server, shares, fresh, height=10) + self.assertEqual(first["received"], self._expected_payload(shares, height=10)) + self.assertEqual(second["received"], first["received"]) + + def test_spool_feeds_builder_and_is_written_once_per_generation( + self, + ) -> None: + server = self._coordinator() + shares = [spool_share(seq) for seq in range(1, 4)] + serialization = server._share_window_serialization_for_artifact( + self._ledger_artifact(shares, generation=1), + shares, + ) + spool_creations: list[int] = [] + real_spool_file = prism_coordinator_module._share_window_spool_file + + def counting_spool_file() -> object: + spool_creations.append(1) + return real_spool_file() + + with patch( + "lab.prism.prism_coordinator._share_window_spool_file", + side_effect=counting_spool_file, + ): + first = self._build_with_echo_builder( + server, + shares, + serialization, + height=10, + ) + second = self._build_with_echo_builder( + server, + shares, + serialization, + height=11, + ) + + self.assertEqual( + first["received"], + self._expected_payload(shares, height=10), + ) + self.assertEqual( + second["received"], + self._expected_payload(shares, height=11), + ) + self.assertEqual(len(spool_creations), 1) + + def test_spool_creation_failure_falls_back_to_pipe_writes(self) -> None: + server = self._coordinator() + shares = [spool_share(seq) for seq in range(1, 4)] + serialization = server._share_window_serialization_for_artifact( + self._ledger_artifact(shares, generation=1), + shares, + ) + + with patch( + "lab.prism.prism_coordinator._share_window_spool_file", + side_effect=OSError("temp filesystem unavailable"), + ) as spool_factory: + first = self._build_with_echo_builder( + server, + shares, + serialization, + height=10, + ) + second = self._build_with_echo_builder( + server, + shares, + serialization, + height=11, + ) + + self.assertEqual( + first["received"], + self._expected_payload(shares, height=10), + ) + self.assertEqual( + second["received"], + self._expected_payload(shares, height=11), + ) + # The failure is sticky: a broken temp filesystem is not retried on + # every subsequent build of the same generation. + self.assertEqual(spool_factory.call_count, 1) + + def test_splice_failure_poisons_spool_and_falls_back_to_pipe(self) -> None: + server = self._coordinator() + shares = [spool_share(seq) for seq in range(1, 4)] + serialization = server._share_window_serialization_for_artifact( + self._ledger_artifact(shares, generation=1), + shares, + ) + + with patch( + "lab.prism.prism_coordinator.os.splice", + side_effect=OSError("splice unsupported"), + ): + first = self._build_with_echo_builder( + server, + shares, + serialization, + height=10, + ) + second = self._build_with_echo_builder( + server, + shares, + serialization, + height=11, + ) + + self.assertEqual( + first["received"], + self._expected_payload(shares, height=10), + ) + self.assertEqual( + second["received"], + self._expected_payload(shares, height=11), + ) + self.assertTrue(serialization._spool_failed) + self.assertIsNone(serialization._spool_file) + + def test_mid_stream_splice_failure_poisons_spool_for_later_builds( + self, + ) -> None: + server = self._coordinator() + shares = [spool_share(seq) for seq in range(1, 4)] + serialization = server._share_window_serialization_for_artifact( + self._ledger_artifact(shares, generation=1), + shares, + ) + + # The first chunk moves and the second fails: this build is + # unrepairable (partial tail already in the pipe), but the spool must + # be poisoned so later builds stream the cached fragments instead of + # repeating the failure. + with patch( + "lab.prism.prism_coordinator.os.splice", + side_effect=[16, OSError("splice io error")], + ): + with self.assertRaises(OSError): + self._build_with_echo_builder( + server, + shares, + serialization, + height=10, + ) + self.assertTrue(serialization._spool_failed) + + recovered = self._build_with_echo_builder( + server, + shares, + serialization, + height=11, + ) + self.assertEqual( + recovered["received"], + self._expected_payload(shares, height=11), + ) + + def test_content_rotation_retires_spool_after_last_lease(self) -> None: + server = self._coordinator() + shares = [spool_share(seq) for seq in range(1, 4)] + first = server._share_window_serialization_for_artifact( + self._ledger_artifact(shares, generation=1), + shares, + ) + lease = first.acquire_spooled_tail(shares) + self.assertIsNotNone(lease) + assert lease is not None + spool_file, spool_size = lease + self.assertGreater(spool_size, 0) + + changed_shares = [*shares, spool_share(4)] + second = server._share_window_serialization_for_artifact( + self._ledger_artifact(changed_shares, generation=2), + changed_shares, + ) + self.assertIsNot(second, first) + # Retired: no new leases, but the in-flight transfer keeps the + # descriptor alive until it releases. + self.assertIsNone(first.acquire_spooled_tail(shares)) + self.assertFalse(spool_file.closed) + first.release_spooled_tail() + self.assertTrue(spool_file.closed) + + def test_generation_retag_reuses_content_keyed_serialization(self) -> None: + server = self._coordinator() + shares = [spool_share(seq) for seq in range(1, 4)] + + first = server._share_window_serialization_for_artifact( + self._ledger_artifact(shares, generation=1), + shares, + ) + second = server._share_window_serialization_for_artifact( + self._ledger_artifact(shares, generation=2), + shares, + ) + + self.assertIs(second, first) + + def test_shutdown_retires_current_spool(self) -> None: + server = self._coordinator() + shares = [spool_share(seq) for seq in range(1, 4)] + serialization = server._share_window_serialization_for_artifact( + self._ledger_artifact(shares, generation=1), + shares, + ) + lease = serialization.acquire_spooled_tail(shares) + assert lease is not None + serialization.release_spooled_tail() + self.assertFalse(lease[0].closed) + + server.retire_share_window_spool() + + self.assertTrue(lease[0].closed) + self.assertIsNone(serialization.acquire_spooled_tail(shares)) + + +class ServeBuilderTests(unittest.TestCase): + """Persistent --serve builder client, its fallback, and supersession.""" + + def _coordinator(self) -> PrismCoordinator: + server, _rpc = coordinator() + server.signing_seed_hex = "42" * 32 + server.ledger_attestation_signing_seed_hex = "43" * 32 + self.addCleanup(server.shutdown_serve_builder) + self.addCleanup(server.retire_share_window_spool) + return server + + def _serialization( + self, + server: PrismCoordinator, + shares: list[dict[str, object]], + *, + generation: int = 1, + ) -> object: + return server._share_window_serialization_for_artifact( + PayoutLedgerArtifact( + generation=generation, + payout_state_generation=0, + network_difficulty=1, + accepted_share_count=len(shares), + shares_json=tuple(shares), + prior_balances=(), + prepared_monotonic=time.monotonic(), + snapshot_anchor_ms=None, + ), + shares, + ) + + def _build( + self, + server: PrismCoordinator, + shares: list[dict[str, object]], + serialization: object, + *, + height: int, + mode: str = "ok", + cancellation: object = None, + ) -> dict[str, object]: + with patch.dict( + os.environ, + {"FAKE_SERVE_BUILDER_MODE": mode}, + ), patch( + "lab.prism.prism_coordinator.prism_tool_command", + return_value=list(FAKE_SERVE_BUILDER_COMMAND), + ): + return server.build_audit_bundle( + shares=shares, + found_block={ + "block_height": height, + "coinbase_value_sats": 50_00000000, + "network_difficulty": 1, + "anchor_job_issued_at_ms": 1_700_000_000_000, + }, + prior_balances=[], + coinbase_script_sig_suffix_hex="00", + summary_only=True, + payout_policy={"policy": "day-one"}, + share_serialization=serialization, # type: ignore[arg-type] + cancellation=cancellation, # type: ignore[arg-type] + ) + + @staticmethod + def _prepare_record(seq: int, **overrides: object) -> dict[str, object]: + record: dict[str, object] = { + "share_seq": seq, + "share_id": f"share-\u00e9-{seq}:\U0001F600", + "miner_id": f"miner-{seq % 3}", + "order_key": f"{seq % 3:02d}:miner-{seq % 3}", + "p2mr_program_hex": "22" * 32, + "share_difficulty": 1, + "network_difficulty": 1, + "template_height": 9, + "job_id": f"job-{seq}", + "job_issued_at_ms": 1_700_000_000_000 + seq, + "accepted_at_ms": 1_700_000_000_100 + seq, + "ntime": 1_700_000_000 + seq, + } + record.update(overrides) + return record + + def _prepare( + self, + server: PrismCoordinator, + records: list[dict[str, object]], + *, + mode: str = "ok", + ) -> object: + with patch.dict( + os.environ, + {"FAKE_SERVE_BUILDER_MODE": mode}, + ), patch( + "lab.prism.prism_coordinator.prism_tool_command", + return_value=list(FAKE_SERVE_BUILDER_COMMAND), + ): + # An anchor past every stamp and a weight equal to the record + # count: the fold retains the whole snapshot. + return server.prepare_payout_window( + mode="full", + records_json=records, + anchor_job_issued_at_ms=1_700_000_100_000, + append_invalidation_epoch=0, + window_weight=len(records) or 1, + page_size=512, + ) + + def test_prepare_request_streams_the_same_bytes_as_one_json_dumps(self) -> None: + records = [ + self._prepare_record(seq, share_difficulty=(1 << 130) + seq) + for seq in range(1, 1_301) + ] + for fields in ( + { + "request": "prepare_window", + "mode": "full", + "append_invalidation_epoch": 3, + "anchor_job_issued_at_ms": 1_700_000_001_000, + "records": records, + "window_weight": 1 << 100, + "page_size": 512, + }, + { + "request": "prepare_window", + "mode": "advance", + "append_invalidation_epoch": 3, + "anchor_job_issued_at_ms": 1_700_000_001_000, + "records": [], + "base_digest": "ab" * 32, + }, + ): + with self.subTest(mode=fields["mode"]): + chunks = list(_iter_prepare_window_request_chunks(fields)) + self.assertEqual( + b"".join(chunks), + json.dumps(fields, separators=(",", ":")).encode("utf-8") + + b"\n", + ) + self.assertEqual(chunks[-1], b"\n") + if fields["records"]: + self.assertGreater(len(chunks), 3) + + def test_prepare_request_reaches_the_daemon_in_bounded_chunks(self) -> None: + server = self._coordinator() + # Several record batches and past the pipe capacity, so the request + # is written as many chunks and the daemon folds it byte-faithfully. + records = [self._prepare_record(seq) for seq in range(1, 2_001)] + try: + outcome = self._prepare(server, records) + self.assertEqual(outcome.status, "prepared") + self.assertEqual(outcome.record_count, len(records)) + mirror = DaemonShareWindowMirror.from_full_items( + anchor_job_issued_at_ms=1_700_000_100_000, + window_weight=len(records), + page_size=512, + record_count=outcome.record_count, + canonical_items=outcome.window_items, + share_snapshot_sha256=outcome.share_snapshot_sha256, + ) + self.assertEqual( + [share["share_id"] for share in mirror.json_records()], + [record["share_id"] for record in records], + ) + client = server._serve_builder + assert client is not None + self.assertFalse(client.request_incomplete) + with server._serve_builder_metrics_lock: + counts = dict(server.serve_builder_counts) + self.assertEqual(counts["window_prepares"], 1) + self.assertEqual(counts["fallbacks"], 0) + finally: + server.shutdown_serve_builder() + + def test_prepare_request_that_fails_to_encode_mid_stream_retires_daemon( + self, + ) -> None: + server = self._coordinator() + records = [self._prepare_record(seq) for seq in range(1, 1_301)] + # The failure lands in the third batch: two batches are already on + # the daemon's stdin as an unterminated line when it surfaces. + records[1_200] = self._prepare_record(1_201, share_id=object()) + try: + outcome = self._prepare(server, records) + self.assertIsNone(outcome) + self.assertIsNone(server._serve_builder) + with server._serve_builder_metrics_lock: + counts = dict(server.serve_builder_counts) + self.assertEqual(counts["fallbacks"], 1) + self.assertEqual(counts["window_prepares"], 0) + # The replacement daemon starts clean and serves the next + # preparation; nothing of the partial line survived. + recovered = self._prepare(server, records[:1_200]) + self.assertEqual(recovered.status, "prepared") + self.assertEqual(recovered.record_count, 1_200) + with server._serve_builder_metrics_lock: + counts = dict(server.serve_builder_counts) + self.assertEqual(counts["spawns"], 2) + finally: + server.shutdown_serve_builder() + + def test_prepare_request_daemon_death_mid_request_degrades_in_process( + self, + ) -> None: + server = self._coordinator() + records = [self._prepare_record(seq) for seq in range(1, 3_001)] + try: + outcome = self._prepare( + server, + records, + mode="crash-during-prepare", + ) + self.assertIsNone(outcome) + self.assertIsNone(server._serve_builder) + with server._serve_builder_metrics_lock: + counts = dict(server.serve_builder_counts) + self.assertEqual(counts["fallbacks"], 1) + finally: + server.shutdown_serve_builder() + + def test_cache_miss_uploads_window_then_hits_on_next_request(self) -> None: + server = self._coordinator() + shares = [spool_share(seq) for seq in range(1, 4)] + serialization = self._serialization(server, shares) + identities, compact_shares = _compact_share_payload(shares) + expected_window = { + "compact_share_identities": [ + list(identity) for identity in identities + ], + "compact_shares": [list(share) for share in compact_shares], + } + try: + first = self._build(server, shares, serialization, height=10) + second = self._build(server, shares, serialization, height=11) + + self.assertEqual(first["transport"], "serve") + self.assertTrue(first["request_had_window"]) + self.assertEqual(first["window"], expected_window) + self.assertEqual(first["found_block"]["block_height"], 10) + + self.assertEqual(second["transport"], "serve") + self.assertFalse(second["request_had_window"]) + self.assertEqual(second["window"], expected_window) + self.assertEqual(second["found_block"]["block_height"], 11) + + with server._serve_builder_metrics_lock: + counts = dict(server.serve_builder_counts) + window_counts = dict(server.serve_builder_window_cache_counts) + self.assertEqual(counts["requests"], 2) + self.assertEqual(counts["spawns"], 1) + self.assertEqual(counts["fallbacks"], 0) + self.assertEqual(counts["window_uploads"], 1) + self.assertEqual(window_counts, {"hits": 1, "misses": 1}) + finally: + server.shutdown_serve_builder() + + def test_needs_window_bounce_reuploads_and_succeeds(self) -> None: + server = self._coordinator() + shares = [spool_share(seq) for seq in range(1, 4)] + serialization = self._serialization(server, shares) + try: + first = self._build(server, shares, serialization, height=10) + self.assertTrue(first["request_had_window"]) + client = server._serve_builder + assert client is not None + second_shares = [spool_share(seq) for seq in range(1, 5)] + second_serialization = self._serialization( + server, + second_shares, + generation=2, + ) + # Pretend this window was already uploaded: the daemon's + # needs_window bounce must repair the divergence transparently. + client.note_uploaded_window( + second_serialization.share_snapshot_sha256 # type: ignore[attr-defined] + ) + + server._ensure_tip_refresh_state() + server._job_build_phase_local.tip_refresh_metrics = True + try: + second = self._build( + server, + second_shares, + second_serialization, + height=11, + ) + finally: + server._job_build_phase_local.tip_refresh_metrics = False + + self.assertEqual(second["transport"], "serve") + self.assertTrue(second["request_had_window"]) + self.assertEqual(second["found_block"]["block_height"], 11) + with server._serve_builder_metrics_lock: + counts = dict(server.serve_builder_counts) + self.assertEqual(counts["requests"], 2) + self.assertEqual(counts["fallbacks"], 0) + # One build observes serialization_copy exactly three times -- + # fragment precompose, the accumulated transport writes, and the + # daemon's reported timings -- matching the one-shot transport + # even when a needs_window bounce re-sent the request. + histogram = server.tip_refresh_build_phase_histograms[ + "serialization_copy" + ] + self.assertEqual(histogram["count"], 3) + finally: + server.shutdown_serve_builder() + + def test_hit_promotion_keeps_mirror_and_daemon_lru_aligned(self) -> None: + server = self._coordinator() + shares_a = [spool_share(seq) for seq in range(1, 4)] + shares_b = [spool_share(seq) for seq in range(1, 5)] + shares_c = [spool_share(seq) for seq in range(1, 6)] + ser_a = self._serialization(server, shares_a, generation=1) + ser_b = self._serialization(server, shares_b, generation=2) + ser_c = self._serialization(server, shares_c, generation=3) + try: + self.assertTrue( + self._build(server, shares_a, ser_a, height=10)[ + "request_had_window" + ] + ) + self.assertTrue( + self._build(server, shares_b, ser_b, height=11)[ + "request_had_window" + ] + ) + # The hit promotes window A on both sides; uploading a third + # window must therefore evict B everywhere. + self.assertFalse( + self._build(server, shares_a, ser_a, height=12)[ + "request_had_window" + ] + ) + self.assertTrue( + self._build(server, shares_c, ser_c, height=13)[ + "request_had_window" + ] + ) + + aligned = self._build(server, shares_a, ser_a, height=14) + + self.assertEqual(aligned["transport"], "serve") + self.assertFalse(aligned["request_had_window"]) + with server._serve_builder_metrics_lock: + counts = dict(server.serve_builder_counts) + self.assertEqual(counts["requests"], 5) + self.assertEqual(counts["window_uploads"], 3) + self.assertEqual(counts["fallbacks"], 0) + finally: + server.shutdown_serve_builder() + + def test_daemon_crash_falls_back_to_one_shot(self) -> None: + server = self._coordinator() + shares = [spool_share(seq) for seq in range(1, 4)] + serialization = self._serialization(server, shares) + try: + result = self._build( + server, + shares, + serialization, + height=10, + mode="crash-before-response", + ) + + self.assertEqual(result["transport"], "one-shot") + self.assertEqual( + result["received"]["found_block"]["block_height"], + 10, + ) + self.assertIn("compact_shares", result["received"]) + with server._serve_builder_metrics_lock: + counts = dict(server.serve_builder_counts) + self.assertIsNone(server._serve_builder) + self.assertEqual(counts["fallbacks"], 1) + self.assertEqual(counts["requests"], 0) + finally: + server.shutdown_serve_builder() + + def test_daemon_splice_failure_poisons_spool_and_one_shot_recovers( + self, + ) -> None: + server = self._coordinator() + shares = [spool_share(seq) for seq in range(1, 4)] + serialization = self._serialization(server, shares) + try: + # The daemon upload dies mid-splice; the spool is poisoned before + # falling back, so the fresh one-shot subprocess streams the + # cached fragments instead of retrying the same failing transfer. + with patch( + "lab.prism.prism_coordinator.os.splice", + side_effect=[16, OSError("splice io error")], + ): + result = self._build(server, shares, serialization, height=10) + + self.assertEqual(result["transport"], "one-shot") + self.assertIn("compact_shares", result["received"]) + self.assertTrue(serialization._spool_failed) # type: ignore[attr-defined] + with server._serve_builder_metrics_lock: + counts = dict(server.serve_builder_counts) + self.assertEqual(counts["fallbacks"], 1) + finally: + server.shutdown_serve_builder() + + def test_protocol_mismatch_falls_back_to_one_shot(self) -> None: + server = self._coordinator() + shares = [spool_share(seq) for seq in range(1, 4)] + serialization = self._serialization(server, shares) + try: + result = self._build( + server, + shares, + serialization, + height=10, + mode="protocol-mismatch", + ) + + self.assertEqual(result["transport"], "one-shot") + with server._serve_builder_metrics_lock: + counts = dict(server.serve_builder_counts) + self.assertIsNone(server._serve_builder) + self.assertEqual(counts["fallbacks"], 1) + # The mismatched daemon was a real worker launch and must appear + # in the lifecycle totals alongside the one-shot fallback. + with server._job_build_scheduler_lock: + worker_counts = dict(server.job_build_worker_counts) + self.assertEqual(worker_counts["starts"], 2) + finally: + server.shutdown_serve_builder() + + def test_disabled_serve_builder_uses_one_shot_directly(self) -> None: + server = self._coordinator() + shares = [spool_share(seq) for seq in range(1, 4)] + serialization = self._serialization(server, shares) + with patch.dict(os.environ, {"PRISM_BUILDER_SERVE": "0"}): + result = self._build(server, shares, serialization, height=10) + + self.assertEqual(result["transport"], "one-shot") + with server._serve_builder_metrics_lock: + counts = dict(server.serve_builder_counts) + self.assertEqual(counts["spawns"], 0) + + def test_daemon_timeout_fallback_shares_the_build_deadline(self) -> None: + server = self._coordinator() + # An already-exhausted budget: the daemon attempt must not grant the + # one-shot fallback a fresh full deadline, and the timeout is counted + # as one worker failure, not two. + server.bundle_build_timeout_seconds = 0.0 + shares = [spool_share(seq) for seq in range(1, 4)] + serialization = self._serialization(server, shares) + + with self.assertRaisesRegex( + RuntimeError, + "qbit-prism-build-audit-bundle timed out", + ): + self._build( + server, + shares, + serialization, + height=10, + mode="hang-after-request", + ) + + with server._serve_builder_metrics_lock: + counts = dict(server.serve_builder_counts) + self.assertEqual(counts["fallbacks"], 1) + self.assertEqual(counts["spawns"], 0) + self.assertEqual(server.tip_refresh_worker_failures, 1) + # The live daemon killed at the handshake deadline is a recorded + # termination, and the one-shot replacement consumes the pending + # restart: two launches, one termination, one restart. + with server._job_build_scheduler_lock: + worker_counts = dict(server.job_build_worker_counts) + self.assertFalse(server._job_build_worker_restart_pending) + self.assertEqual(worker_counts["starts"], 2) + self.assertEqual(worker_counts["terminations"], 1) + self.assertEqual(worker_counts["restarts"], 1) + + def test_idle_daemon_death_counts_crash_and_restart(self) -> None: + server = self._coordinator() + shares = [spool_share(seq) for seq in range(1, 4)] + serialization = self._serialization(server, shares) + try: + first = self._build(server, shares, serialization, height=10) + self.assertEqual(first["transport"], "serve") + client = server._serve_builder + assert client is not None + client.process.kill() + client.process.wait(timeout=5) + + second = self._build(server, shares, serialization, height=11) + + self.assertEqual(second["transport"], "serve") + with server._job_build_scheduler_lock: + counts = dict(server.job_build_worker_counts) + self.assertFalse(server._job_build_worker_restart_pending) + self.assertEqual(counts["crashes"], 1) + self.assertEqual(counts["restarts"], 1) + self.assertEqual(server.tip_refresh_worker_restarts, 1) + with server._serve_builder_metrics_lock: + serve_counts = dict(server.serve_builder_counts) + self.assertEqual(serve_counts["spawns"], 2) + self.assertEqual(serve_counts["fallbacks"], 0) + finally: + server.shutdown_serve_builder() + + def test_daemon_respawn_counts_as_worker_restart(self) -> None: + server = self._coordinator() + shares = [spool_share(seq) for seq in range(1, 4)] + serialization = self._serialization(server, shares) + try: + # A prior worker ended abnormally (crash or supersession kill) + # with no immediate fallback consuming the flag; the replacement + # daemon spawn must claim the restart instead of leaking it to + # an unrelated later one-shot build. + server._ensure_tip_refresh_state() + with server._job_build_scheduler_lock: + server._job_build_worker_restart_pending = True + + result = self._build(server, shares, serialization, height=10) + + self.assertEqual(result["transport"], "serve") + with server._job_build_scheduler_lock: + counts = dict(server.job_build_worker_counts) + self.assertFalse(server._job_build_worker_restart_pending) + self.assertEqual(counts["restarts"], 1) + self.assertGreaterEqual(counts["starts"], 1) + self.assertEqual(server.tip_refresh_worker_restarts, 1) + finally: + server.shutdown_serve_builder() + + def test_daemon_detaches_from_build_control_after_request(self) -> None: + server = self._coordinator() + shares = [spool_share(seq) for seq in range(1, 4)] + serialization = self._serialization(server, shares) + control = prism_coordinator_module._JobBundleBuildControl( + key=("serve-detach",), + previousblockhash="11" * 32, + payout_state_generation=0, + payout_artifact_generation=1, + ) + with server._job_cache_lock: + server._active_job_bundle_builds[control.key] = control + server._job_build_phase_local.bundle_build_control = control + try: + result = self._build(server, shares, serialization, height=10) + + self.assertEqual(result["transport"], "serve") + # The completed request detached the daemon: a late supersession + # of this build has no process reference left to terminate. + self.assertIsNone(control.process) + control.cancel_event.set() + client = server._serve_builder + assert client is not None + self.assertIsNone(client.process.poll()) + finally: + server._job_build_phase_local.bundle_build_control = None + control.cancel_event.set() + with server._job_cache_lock: + server._active_job_bundle_builds.pop(control.key, None) + server.shutdown_serve_builder() + + def test_supersession_cancels_in_flight_daemon_request(self) -> None: + server = self._coordinator() + shares = [spool_share(seq) for seq in range(1, 4)] + serialization = self._serialization(server, shares) + control = prism_coordinator_module._JobBundleBuildControl( + key=("serve-test",), + previousblockhash="11" * 32, + payout_state_generation=0, + payout_artifact_generation=1, + ) + with server._job_cache_lock: + server._active_job_bundle_builds[control.key] = control + errors: list[BaseException] = [] + results: list[object] = [] + + def build() -> None: + # The bundle-build control travels on the builder thread's local + # state, exactly as _execute_job_build installs it. + server._job_build_phase_local.bundle_build_control = control + try: + results.append( + self._build( + server, + shares, + serialization, + height=10, + mode="hang-after-request", + ) + ) + except BaseException as exc: # noqa: BLE001 - surfaced below + errors.append(exc) + finally: + server._job_build_phase_local.bundle_build_control = None + + thread = threading.Thread(target=build) + thread.start() + try: + deadline = time.monotonic() + 5.0 + while time.monotonic() < deadline: + if control.process is not None: + break + time.sleep(0.005) + self.assertIsNotNone(control.process) + + control.cancel_event.set() + thread.join(5.0) + finally: + control.cancel_event.set() + thread.join(1.0) + server.shutdown_serve_builder() + + self.assertFalse(thread.is_alive()) + self.assertEqual(results, []) + self.assertEqual(len(errors), 1) + self.assertIsInstance(errors[0], JobBuildSuperseded) + self.assertIsNone(server._serve_builder) + + +class ReorgReconcileRefreshPathTests(unittest.TestCase): + """The tip-refresh reconcile stage: memo reuse and fetch overlap.""" + + def test_poll_memo_hit_skips_full_reconcile_pass(self) -> None: + server, rpc = coordinator() + install_fake_bundle_builder(server) + server.reorg_reconciler_enabled = True + state = client(1) + state.send = lambda _payload: None # type: ignore[method-assign] + server.clients = {state} + with server.lock: + server._ensure_reorg_reconciler_service()._reorg_reconcile_trusted_memo[rpc.tip] = time.monotonic() + + def unexpected_pass(**_kwargs: object) -> dict[str, object]: + raise AssertionError( + "memo-armed poll must not queue a serialized reconcile pass" + ) + + server.reconcile_prism_pool_blocks_once = ( # type: ignore[method-assign] + unexpected_pass + ) + + try: + refreshed = server.poll_qbit_tip_template_once() + finally: + server.shutdown_tip_refresh_executor() + + self.assertEqual(refreshed, 1) + # A memo hit never needs the prefetch worker. + self.assertIsNone(server._ensure_reorg_reconciler_service()._reconcile_prefetch_executor) + metrics = server.metrics_payload() + self.assertIn( + 'qbit_prism_reorg_reconcile_lookups_total{path="tip_refresh",source="memo_hit"} 1', + metrics, + ) + self.assertIn( + 'qbit_prism_tip_refresh_bundle_phase_seconds_count{phase="reorg_reconcile"} 1', + metrics, + ) + + def test_poll_overlaps_reconcile_with_template_fetch(self) -> None: + # Deadlock-free only when the reconcile pass and the template fetch + # genuinely run concurrently: the fetch refuses to return before the + # reconcile starts, and the reconcile refuses to return before the + # fetch starts. + fetch_started = threading.Event() + reconcile_running = threading.Event() + + class OverlapProbeRpc(FakeRpc): + def call( + self, + method: str, + params: list[object] | None = None, + ) -> object: + if method == "getblocktemplate": + fetch_started.set() + assert reconcile_running.wait(5.0), ( + "reconcile prefetch did not start while the template " + "fetch was in flight" + ) + return super().call(method, params) + + template = base_template() + rpc = OverlapProbeRpc(template, tip=str(template["previousblockhash"])) + server, _ = coordinator(template=template) + server.rpc = rpc + install_fake_bundle_builder(server) + server.reorg_reconciler_enabled = True + state = client(1) + state.send = lambda _payload: None # type: ignore[method-assign] + server.clients = {state} + reconciled_tips: list[str] = [] + + def fake_ensure(tip_hash: str) -> bool: + reconciled_tips.append(tip_hash) + reconcile_running.set() + assert fetch_started.wait(5.0), ( + "template fetch did not start while the reconcile pass was " + "in flight" + ) + # A real trusted pass arms the per-tip memo; the join validates + # the overlapped result against that armed entry. + with server.lock: + server._ensure_reorg_reconciler_service()._reorg_reconcile_trusted_memo[tip_hash] = time.monotonic() + return True + + server.ensure_reorg_reconciled_for_tip = ( # type: ignore[method-assign] + fake_ensure + ) + + try: + refreshed = server.poll_qbit_tip_template_once() + finally: + server.shutdown_tip_refresh_executor() + + self.assertEqual(refreshed, 1) + self.assertEqual(reconciled_tips, [rpc.tip]) + metrics = server.metrics_payload() + self.assertIn( + 'qbit_prism_reorg_reconcile_lookups_total{path="tip_refresh",source="overlap"} 1', + metrics, + ) + + def test_poll_reconciles_snapshot_tip_serially_when_tip_moves_before_fetch( + self, + ) -> None: + # The cheap probe observed a tip that was replaced before the + # template fetch. The prefetched pass proved the superseded hash, so + # the poll must reconcile the snapshot's newer tip on its own thread. + template = base_template() + new_tip = str(template["previousblockhash"]) + old_tip = "ab" * 32 + + class MovingTipRpc(FakeRpc): + def __init__(self) -> None: + super().__init__(template, tip=new_tip) + self.best_tip_calls = 0 + + def call( + self, + method: str, + params: list[object] | None = None, + ) -> object: + if method == "getbestblockhash": + self.best_tip_calls += 1 + if self.best_tip_calls == 1: + self.calls.append(method) + return old_tip + return super().call(method, params) + + rpc = MovingTipRpc() + server, _ = coordinator(template=template) + server.rpc = rpc + install_fake_bundle_builder(server) + server.reorg_reconciler_enabled = True + state = client(1) + state.send = lambda _payload: None # type: ignore[method-assign] + server.clients = {state} + reconcile_calls: list[str] = [] + prefetch_drained = threading.Event() + + def fake_ensure(tip_hash: str) -> bool: + reconcile_calls.append(tip_hash) + if tip_hash == old_tip: + prefetch_drained.set() + return True + + server.ensure_reorg_reconciled_for_tip = ( # type: ignore[method-assign] + fake_ensure + ) + + try: + refreshed = server.poll_qbit_tip_template_once() + finally: + server.shutdown_tip_refresh_executor() + + self.assertEqual(refreshed, 1) + self.assertTrue(prefetch_drained.wait(5.0)) + self.assertIn(new_tip, reconcile_calls) + self.assertEqual( + sorted(reconcile_calls), + sorted([old_tip, new_tip]), + ) + metrics = server.metrics_payload() + self.assertIn( + 'qbit_prism_reorg_reconcile_lookups_total{path="tip_refresh",source="serial"} 1', + metrics, + ) + + def test_poll_reproves_when_memo_evicted_during_template_fetch(self) -> None: + # A tip that flips away and back during the template fetch evicts + # the memo entry so post-flip pool-block state is re-proven. The + # join must judge the memo at join time, never the pre-fetch probe. + template = base_template() + server, _ = coordinator(template=template) + server.reorg_reconciler_enabled = True + tip = str(template["previousblockhash"]) + + class EvictingRpc(FakeRpc): + def call( + self, + method: str, + params: list[object] | None = None, + ) -> object: + if method == "getblocktemplate": + # Models a detection during the fetch unarming the memo. + with server.lock: + server._ensure_reorg_reconciler_service()._reorg_reconcile_trusted_memo.clear() + return super().call(method, params) + + rpc = EvictingRpc(template, tip=tip) + server.rpc = rpc + install_fake_bundle_builder(server) + state = client(1) + state.send = lambda _payload: None # type: ignore[method-assign] + server.clients = {state} + with server.lock: + server._ensure_reorg_reconciler_service()._reorg_reconcile_trusted_memo[tip] = time.monotonic() + reconcile_calls: list[str] = [] + + def fake_ensure(tip_hash: str) -> bool: + reconcile_calls.append(tip_hash) + return True + + server.ensure_reorg_reconciled_for_tip = ( # type: ignore[method-assign] + fake_ensure + ) + + try: + refreshed = server.poll_qbit_tip_template_once() + finally: + server.shutdown_tip_refresh_executor() + + self.assertEqual(refreshed, 1) + self.assertEqual(reconcile_calls, [tip]) + metrics = server.metrics_payload() + self.assertIn( + 'qbit_prism_reorg_reconcile_lookups_total{path="tip_refresh",source="serial"} 1', + metrics, + ) + self.assertIn( + 'qbit_prism_reorg_reconcile_lookups_total{path="tip_refresh",source="memo_hit"} 0', + metrics, + ) + + def test_poll_reproves_when_memo_rearmed_across_detection_epoch(self) -> None: + # A pass whose execution straddles a flip away and back can re-arm + # the memo after the flip-back (the latest detected hash matches + # again), but its proof belongs to the closed epoch. The join must + # refuse memo entries when the detection epoch moved mid-fetch. + template = base_template() + server, _ = coordinator(template=template) + server.reorg_reconciler_enabled = True + tip = str(template["previousblockhash"]) + + class FlipBackRpc(FakeRpc): + def call( + self, + method: str, + params: list[object] | None = None, + ) -> object: + if method == "getblocktemplate": + # Models a flip away and back detected during the fetch + # (two epoch bumps) with a straddling pass re-arming the + # memo for the same hash afterwards. + with server.lock: + server.tip_detection_epoch = ( + int(getattr(server, "tip_detection_epoch", 0)) + 2 + ) + server._ensure_reorg_reconciler_service()._reorg_reconcile_trusted_memo[tip] = ( + time.monotonic() + ) + return super().call(method, params) + + rpc = FlipBackRpc(template, tip=tip) + server.rpc = rpc + install_fake_bundle_builder(server) + state = client(1) + state.send = lambda _payload: None # type: ignore[method-assign] + server.clients = {state} + with server.lock: + server._ensure_reorg_reconciler_service()._reorg_reconcile_trusted_memo[tip] = time.monotonic() + reconcile_calls: list[str] = [] + + def fake_ensure(tip_hash: str) -> bool: + reconcile_calls.append(tip_hash) + return True + + server.ensure_reorg_reconciled_for_tip = ( # type: ignore[method-assign] + fake_ensure + ) + + try: + refreshed = server.poll_qbit_tip_template_once() + finally: + server.shutdown_tip_refresh_executor() + + self.assertEqual(refreshed, 1) + self.assertEqual(reconcile_calls, [tip]) + metrics = server.metrics_payload() + self.assertIn( + 'qbit_prism_reorg_reconcile_lookups_total{path="tip_refresh",source="serial"} 1', + metrics, + ) + + def test_overlap_join_blocks_when_trust_flips_after_prefetch(self) -> None: + # The prefetched pass proves trust when it runs, up to a fetch + # window earlier. Headers running ahead without a best-hash change + # (an arriving reorg, no detection) must still block the refresh at + # the join, matching the memo branch's live trust check. + template = base_template() + server, _ = coordinator(template=template) + server.reorg_reconciler_enabled = True + tip = str(template["previousblockhash"]) + + class TrustFlipRpc(FakeRpc): + def call( + self, + method: str, + params: list[object] | None = None, + ) -> object: + if method == "getblocktemplate": + # Headers run ahead after the fetch: untrusted view with + # no tip detection (epoch unchanged). + self.blockchain_info["headers"] = 101 + return super().call(method, params) + + rpc = TrustFlipRpc(template, tip=tip) + server.rpc = rpc + install_fake_bundle_builder(server) + state = client(1) + state.send = lambda _payload: None # type: ignore[method-assign] + server.clients = {state} + server.ensure_reorg_reconciled_for_tip = ( # type: ignore[method-assign] + lambda _tip: True + ) + + try: + with self.assertRaises(TemplateRefreshBlocked) as raised: + server.poll_qbit_tip_template_once() + finally: + server.shutdown_tip_refresh_executor() + + self.assertIn("untrusted", str(raised.exception)) + self.assertIsNone(state.active_job) + + def test_tip_refresh_join_bounds_a_stalled_reconcile_prefetch(self) -> None: + # The overlap join must never park the poll loop on a crawling + # prefetched pass: it times out into the normal blocked-retry path + # within the CONFIGURED budget (not merely eventually), retries + # re-join the identical still-running future without starting + # another pass, and the pass completing unblocks the next poll. + server, rpc = coordinator() + server.reorg_reconciler_enabled = True + server.reconcile_prefetch_join_timeout_seconds = 0.05 + started = threading.Event() + release = threading.Event() + pass_calls = [0] + + def slow_ensure(tip_hash: str) -> bool: + pass_calls[0] += 1 + started.set() + assert release.wait(10.0) + return True + + server.ensure_reorg_reconciled_for_tip = ( # type: ignore[method-assign] + slow_ensure + ) + try: + join_started = time.monotonic() + with self.assertRaises(TemplateRefreshBlocked): + server.poll_qbit_tip_template_once() + # Generous scheduler tolerance, but far below any hardcoded + # multi-second budget: the configured 0.05s must be what fired. + self.assertLess(time.monotonic() - join_started, 1.0) + self.assertTrue(started.is_set()) + with server._ensure_reorg_reconciler_service()._reconcile_prefetch_executor_lock: + pending = server._ensure_reorg_reconciler_service()._reconcile_prefetch_pending + self.assertIsNotNone(pending) + assert pending is not None + first_future = pending[1] + self.assertFalse(first_future.done()) + + # A retry re-joins the same running future: same slot identity, + # no additional pass started. + with self.assertRaises(TemplateRefreshBlocked): + server.poll_qbit_tip_template_once() + with server._ensure_reorg_reconciler_service()._reconcile_prefetch_executor_lock: + pending_again = server._ensure_reorg_reconciler_service()._reconcile_prefetch_pending + assert pending_again is not None + self.assertIs(pending_again[1], first_future) + self.assertEqual(pass_calls[0], 1) + + release.set() + self.assertTrue(first_future.result(5.0)) + server.poll_qbit_tip_template_once() + finally: + release.set() + server.shutdown_reconcile_prefetch_executor() + + def test_serial_reconcile_branch_uses_the_bounded_join(self) -> None: + # With the memo disabled no overlap prefetch exists, so the pass + # lands in the serial re-prove branch -- which must route through + # the same prefetch slot and bounded join instead of parking the + # poll loop synchronously inside a crawling pass. + server, rpc = coordinator() + server.reorg_reconciler_enabled = True + server.reorg_reconcile_cache_seconds = 0.0 + server.reconcile_prefetch_join_timeout_seconds = 0.05 + started = threading.Event() + release = threading.Event() + + def slow_ensure(tip_hash: str) -> bool: + started.set() + assert release.wait(10.0) + return True + + server.ensure_reorg_reconciled_for_tip = ( # type: ignore[method-assign] + slow_ensure + ) + try: + join_started = time.monotonic() + with self.assertRaises(TemplateRefreshBlocked): + server.poll_qbit_tip_template_once() + self.assertLess(time.monotonic() - join_started, 1.0) + self.assertTrue(started.is_set()) + with server._ensure_reorg_reconciler_service()._reconcile_prefetch_executor_lock: + pending = server._ensure_reorg_reconciler_service()._reconcile_prefetch_pending + self.assertIsNotNone(pending) + assert pending is not None + self.assertFalse(pending[1].done()) + + release.set() + self.assertTrue(pending[1].result(5.0)) + server.poll_qbit_tip_template_once() + finally: + release.set() + server.shutdown_reconcile_prefetch_executor() + + def test_pass_raised_timeout_is_not_logged_as_join_expiry(self) -> None: + # socket.timeout IS builtins.TimeoutError: a pass that raises it + # completes the future exceptionally and must surface as an ordinary + # blocked pass -- not as a join expiry, which would misdirect + # operators toward the join while nothing is running. + server, rpc = coordinator() + server.reorg_reconciler_enabled = True + server.reorg_reconcile_cache_seconds = 0.0 + server.reconcile_prefetch_join_timeout_seconds = 5.0 + + def timing_out_ensure(tip_hash: str) -> bool: + raise TimeoutError("pass-owned timeout") + + server.ensure_reorg_reconciled_for_tip = ( # type: ignore[method-assign] + timing_out_ensure + ) + try: + captured = io.StringIO() + join_started = time.monotonic() + with contextlib.redirect_stdout(captured): + with self.assertRaises(TemplateRefreshBlocked): + server.poll_qbit_tip_template_once() + self.assertLess(time.monotonic() - join_started, 2.0) + self.assertNotIn("join exceeded", captured.getvalue()) + finally: + server.shutdown_reconcile_prefetch_executor() + + def test_reconcile_prefetch_slot_is_reused_across_failed_attempts(self) -> None: + # A refresh attempt that dies before its join (template-RPC outage) + # must not queue another serialized pass per retry: the slot holds + # at most one outstanding prefetch, reused for the same tip and + # replaced (with the queued task cancelled) on a tip change. + server, _ = coordinator() + server.reorg_reconciler_enabled = True + started = threading.Event() + release = threading.Event() + + def slow_ensure(tip_hash: str) -> bool: + started.set() + assert release.wait(5.0) + return True + + server.ensure_reorg_reconciled_for_tip = ( # type: ignore[method-assign] + slow_ensure + ) + try: + first = server._submit_reconcile_prefetch("aa" * 32) + assert first is not None + self.assertTrue(started.wait(5.0)) + second = server._submit_reconcile_prefetch("aa" * 32) + self.assertIs(second, first) + third = server._submit_reconcile_prefetch("bb" * 32) + assert third is not None + self.assertIsNot(third, first) + fourth = server._submit_reconcile_prefetch("cc" * 32) + assert fourth is not None + self.assertIsNot(fourth, third) + self.assertTrue(third.cancelled()) + release.set() + self.assertTrue(first.result(5.0)) + self.assertTrue(fourth.result(5.0)) + finally: + release.set() + server.shutdown_reconcile_prefetch_executor() + + def test_job_build_memo_lookups_are_counted(self) -> None: + server, rpc = coordinator() + server.reorg_reconciler_enabled = True + with server.lock: + server._ensure_reorg_reconciler_service()._reorg_reconcile_trusted_memo[rpc.tip] = time.monotonic() + + self.assertTrue(server.ensure_reorg_reconciled_for_current_tip()) + + with server.lock: + server._ensure_reorg_reconciler_service()._reorg_reconcile_trusted_memo[rpc.tip] = ( + time.monotonic() - 100.0 + ) + serial_tips: list[str] = [] + + def fake_ensure(tip_hash: str) -> bool: + serial_tips.append(tip_hash) + return True + + server.ensure_reorg_reconciled_for_tip = ( # type: ignore[method-assign] + fake_ensure + ) + + self.assertTrue(server.ensure_reorg_reconciled_for_current_tip()) + + self.assertEqual(serial_tips, [rpc.tip]) + metrics = server.metrics_payload() + self.assertIn( + 'qbit_prism_reorg_reconcile_lookups_total{path="job_build",source="memo_hit"} 1', + metrics, + ) + self.assertIn( + 'qbit_prism_reorg_reconcile_lookups_total{path="job_build",source="serial"} 1', + metrics, + ) + + def test_poll_falls_back_to_serial_reconcile_after_prefetch_shutdown( + self, + ) -> None: + server, rpc = coordinator() + install_fake_bundle_builder(server) + server.reorg_reconciler_enabled = True + state = client(1) + state.send = lambda _payload: None # type: ignore[method-assign] + server.clients = {state} + server.shutdown_reconcile_prefetch_executor() + reconcile_calls: list[str] = [] + + def fake_ensure(tip_hash: str) -> bool: + reconcile_calls.append(tip_hash) + return True + + server.ensure_reorg_reconciled_for_tip = ( # type: ignore[method-assign] + fake_ensure + ) + + try: + refreshed = server.poll_qbit_tip_template_once() + finally: + server.shutdown_tip_refresh_executor() + + self.assertEqual(refreshed, 1) + self.assertEqual(reconcile_calls, [rpc.tip]) + self.assertIsNone(server._ensure_reorg_reconciler_service()._reconcile_prefetch_executor) + metrics = server.metrics_payload() + self.assertIn( + 'qbit_prism_reorg_reconcile_lookups_total{path="tip_refresh",source="serial"} 1', + metrics, + ) + + +class JobContextStampTests(unittest.TestCase): + def test_job_context_stamps_client_version_mask(self) -> None: + # Cross-connection resumes validate version bits against the mask + # negotiated on the delivering connection, so the context must + # record it at stamp time. + server, _ = coordinator() + install_fake_bundle_builder(server) + state = client(1) + state.version_mask = 0x1FFFE000 + + context = server.build_job_for_client(state, clean_jobs=True) + + self.assertEqual(context.version_mask, 0x1FFFE000) + + +class ExternallyTerminatedBuilderTests(unittest.TestCase): + """A build superseded while its helper runs terminates without a crash.""" + + def test_superseded_running_builder_terminates_without_crash(self) -> None: + server, rpc = coordinator() + server.signing_seed_hex = "42" * 32 + server.ledger_attestation_signing_seed_hex = "43" * 32 + artifacts = server.store_template_artifacts(dict(rpc.template)) + assert artifacts is not None + real_popen = subprocess.Popen + + def build_with_control(*_args: object, **kwargs: object) -> object: + build_request = kwargs["build_request"] + return server.build_audit_bundle( + shares=[], + found_block={ + "block_height": 10, + "coinbase_value_sats": 50_00000000, + "network_difficulty": 1, + "anchor_job_issued_at_ms": 1_700_000_000_000, + }, + prior_balances=[], + coinbase_script_sig_suffix_hex="00", + cancellation=build_request.cancellation, # type: ignore[union-attr] + ) + + server.build_shared_job_bundle = build_with_control # type: ignore[method-assign] + + def cancel_then_exit( + *args: object, + **kwargs: object, + ) -> subprocess.Popen[str]: + process = real_popen(*args, **kwargs) # type: ignore[arg-type] + original_poll = process.poll + original_wait = process.wait + first_poll = True + + def poll() -> int | None: + nonlocal first_poll + if not first_poll: + return original_poll() + first_poll = False + with server._job_cache_lock: + controls = list(server._active_job_bundle_builds.values()) + self.assertEqual(len(controls), 1) + # Supersession lands while the helper is still running; the + # compiler owns the terminate/kill sequence itself, so this + # never counts as a worker crash. + controls[0].cancel_event.set() + return None + + process.poll = poll # type: ignore[method-assign] + del original_wait + return process + + with patch( + "lab.prism.prism_coordinator.prism_tool_command", + return_value=[sys.executable, "-c", "import time; time.sleep(30)"], + ), patch( + "lab.prism.prism_coordinator.subprocess.Popen", + side_effect=cancel_then_exit, + ): + with self.assertRaises(JobBuildSuperseded): + server.shared_job_bundle( + artifacts, + mode="ready", + retry_superseded=False, + ) + + service = server._ensure_job_bundle_service() + self.assertEqual(service.shared_bundle_build_counts["superseded"], 1) + self.assertEqual(service.shared_bundle_build_counts["failed"], 0) + self.assertEqual(server.job_build_worker_counts["terminations"], 1) + self.assertEqual(server.job_build_worker_counts["crashes"], 0) + self.assertEqual(server.job_build_failure_count, 0) diff --git a/tests/test_prism_job_delivery.py b/tests/test_prism_job_delivery.py new file mode 100644 index 000000000..1c41f360b --- /dev/null +++ b/tests/test_prism_job_delivery.py @@ -0,0 +1,207 @@ +#!/usr/bin/env python3 +"""Direct contract tests for the coordinator-free S2 delivery boundary. + +The current owner keeps one call-time ``JobDeliveryRuntime`` typed port and +descriptor-routed compatibility state (the old stack's per-capability port +dataclasses and authority objects were deliberately not built). These cases +pin the boundary itself: symbol identity, descriptor routing and adoption, +the narrow ``next_job_id`` / ``complete_delivery`` adapters, and legacy +retained-state conversion. Delivery behavior (stamping, retained jobs, +initial jobs, difficulty adjacency) is asserted by the upstream suites in +``tests/test_prism_retained_jobs.py``, ``tests/test_prism_initial_job_delivery.py``, +``tests/test_prism_tip_refresh_delivery.py`` and +``tests/test_prism_job_builder.py`` (``JobContextStampTests``). +""" + +from __future__ import annotations + +from collections import OrderedDict +import time +from types import SimpleNamespace +import unittest + +import lab.prism.job_delivery as job_delivery_module +from lab.prism import prism_coordinator +from lab.prism.job_delivery import ( + EvictedJobEntry, + JobBuildFailed, + PendingInitialJob, + PrismJobContext, +) +from lab.prism.prism_coordinator import ( + EvictedJobEntry as FacadeEvictedJobEntry, + PendingInitialJob as FacadePendingInitialJob, + PrismJobContext as FacadePrismJobContext, +) +from tests.prism_vardiff_test_support import client, coordinator + +TIP_A = "11" * 32 + + +class S2FacadeIdentityTests(unittest.TestCase): + def test_facade_reexports_exact_s2_identities(self) -> None: + self.assertIs(FacadePrismJobContext, PrismJobContext) + self.assertIs(FacadeEvictedJobEntry, EvictedJobEntry) + self.assertIs(FacadePendingInitialJob, PendingInitialJob) + self.assertIs(prism_coordinator._JobBuildFailed, JobBuildFailed) + # PR 80 removed the admission-poll compatibility re-export. + self.assertFalse( + hasattr(prism_coordinator, "PRISM_TIP_REFRESH_ADMISSION_POLL_SECONDS") + ) + # The owner is a leaf: it never imports the coordinator module. + self.assertFalse(hasattr(job_delivery_module, "PrismCoordinator")) + + +class S2DescriptorRoutingTests(unittest.TestCase): + def test_coordinator_descriptors_route_s2_state_to_owner(self) -> None: + server = coordinator() + service = server._ensure_job_delivery_service() + + self.assertIs(server.jobs, service.jobs) + self.assertIs(server.pending_initial_jobs, service.pending_initial_jobs) + self.assertIs(server.evicted_job_graveyard, service.evicted_job_graveyard) + + server.job_counter = 5 + self.assertEqual(service.job_counter, 5) + service.initial_job_sent_count = 3 + self.assertEqual(server.initial_job_sent_count, 3) + + def test_coordinator_adopts_replaced_jobs_and_pending_initial_mappings( + self, + ) -> None: + server = coordinator() + service = server._ensure_job_delivery_service() + + jobs_replacement: dict[str, PrismJobContext] = {} + server.jobs = jobs_replacement + self.assertIs(service.jobs, jobs_replacement) + + # The admission map is private to the owner and every access to it is + # taken under the admission lock, so assigning through the legacy name + # adopts the mapping's contents rather than retaining (and exposing) + # the caller's mutable object (#159). + state = client() + request = PendingInitialJob( + client=state, + authorization_generation=1, + worker=state.worker, + requested_monotonic=time.monotonic(), + deadline_monotonic=None, + connection_id=state.connection_id, + difficulty_generation=0, + ) + pending_replacement: dict[object, PendingInitialJob] = {state: request} + server.pending_initial_jobs = pending_replacement + + self.assertIsNot(service.pending_initial_jobs, pending_replacement) + self.assertIs(service.pending_initial_jobs[state], request) + self.assertEqual(len(server.pending_initial_jobs), 1) + # Adoption is a copy: later admission changes are not written back + # into the caller's dict, and the caller cannot mutate admission state + # behind the lock by holding on to it. + server.cancel_initial_job_delivery(state) + self.assertEqual(len(server.pending_initial_jobs), 0) + self.assertEqual(pending_replacement, {state: request}) + + +class S2NarrowAdapterTests(unittest.TestCase): + def test_next_job_id_allocates_through_the_owner_counter(self) -> None: + server = coordinator() + service = server._ensure_job_delivery_service() + server.job_counter = 41 + + self.assertEqual(service.next_job_id(), "prism-42") + # The coordinator adapter is a pure delegate over the same counter. + self.assertEqual(server._next_job_delivery_id(), "prism-43") + self.assertEqual(server.job_counter, 43) + + def test_complete_delivery_notes_tip_work_before_progress_proof(self) -> None: + server = coordinator() + service = server._ensure_job_delivery_service() + state = client() + events: list[tuple[str, object]] = [] + server.note_tip_work_delivered = ( # type: ignore[method-assign] + lambda delivered_client, parent: events.append( + ("tip", (delivered_client, parent)) + ) + ) + server._record_progress_delivery = ( # type: ignore[method-assign] + lambda delivered_client, context, monotonic: events.append( + ("progress", (delivered_client, context, monotonic)) + ) + ) + context = SimpleNamespace(template={"previousblockhash": TIP_A}) + + service.complete_delivery(state, context, 3.5) + + self.assertEqual( + [name for name, _payload in events], ["tip", "progress"] + ) + self.assertEqual(events[0][1], (state, TIP_A)) + self.assertEqual(events[1][1], (state, context, 3.5)) + + # The coordinator adapter routes through the same owner method. + events.clear() + server._complete_job_delivery(state, context, 4.5) + self.assertEqual( + [name for name, _payload in events], ["tip", "progress"] + ) + + +class RetainedStateAdoptionTests(unittest.TestCase): + def test_ensure_evicted_job_state_converts_legacy_tuple_entries(self) -> None: + server = coordinator() + service = server._ensure_job_delivery_service() + live_client = SimpleNamespace(connection_id=7) + server.clients = [live_client] + context = SimpleNamespace(template={"previousblockhash": TIP_A}) + evicted_at = time.monotonic() + # A legacy plain-dict graveyard with tuple entries, seeded through the + # compatibility field exactly as pre-extraction tests did. + server.evicted_job_graveyard = { + "prism-legacy": (context, 7, evicted_at), + } + + server._ensure_evicted_job_state() + + graveyard = service.evicted_job_graveyard + self.assertIsInstance(graveyard, OrderedDict) + entry = graveyard["prism-legacy"] + self.assertIsInstance(entry, EvictedJobEntry) + self.assertIs(entry.context, context) + self.assertEqual(entry.connection_id, 7) + self.assertEqual(entry.evicted_monotonic, evicted_at) + self.assertEqual(entry.previousblockhash, TIP_A) + self.assertIs(entry.client, live_client) + # Conversion rebuilds the per-connection retained indexes. + self.assertIn(7, service.evicted_jobs_by_connection) + self.assertIn("prism-legacy", service.evicted_jobs_by_connection[7]) + + def test_ensure_evicted_job_state_rebuilds_indexes_on_tip_change(self) -> None: + server = coordinator() + service = server._ensure_job_delivery_service() + server.clients = set() + context = SimpleNamespace(template={"previousblockhash": TIP_A}) + entry = EvictedJobEntry( + context=context, + connection_id=3, + evicted_monotonic=time.monotonic(), + previousblockhash=TIP_A, + client=None, + ) + service.evicted_job_graveyard = OrderedDict({"prism-kept": entry}) + service.evicted_jobs_by_connection = {} + service.evicted_job_index_tip_hash = "stale-tip" + + server._ensure_evicted_job_state() + + self.assertIn(3, service.evicted_jobs_by_connection) + self.assertIn("prism-kept", service.evicted_jobs_by_connection[3]) + self.assertEqual( + service.evicted_job_index_tip_hash, + server._current_published_tip_hash_locked(), + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_prism_landing_append_epoch.py b/tests/test_prism_landing_append_epoch.py new file mode 100644 index 000000000..8534b9d27 --- /dev/null +++ b/tests/test_prism_landing_append_epoch.py @@ -0,0 +1,515 @@ +#!/usr/bin/env python3 +"""The append-invalidation epoch, asked per anchor instead of per counter. + +Issue #126: the epoch is one global counter, so a landing that compares its +own baseline against the live value learns only that *somebody's* payout +window was invalidated. A share that becomes durable stamped later than this +candidate's declared anchor -- but early enough to predate some newer exposed +window -- bumps that counter without touching this candidate's window. On the +fallback submit lane, where nothing has been offered yet, the pre-offer fence +read the inequality and terminally abandoned the block as +``append_epoch_stale``: it never reached qbitd at all, and a valid block's +reward was forfeited. + +What the fix may not do is forgive a window that genuinely was invalidated, +so these scenarios are written in pairs. The same interleaving, at the same +fence, differs only in the stamp of the appended row: one stamp that cannot +predate the candidate's declared anchor must land, and one that can must +still abandon with nothing submitted. The boundary itself (a stamp exactly +equal to the anchor) is on the terminal side, because +``_pending_share_predates_anchor`` treats an equal stamp as predating. + +Both landing fences are covered. The pre-offer one is advisory -- it reads +the epoch under the job-cache lock and releases it -- so its pair drains the +appender before the landing starts. The authoritative one holds +``_payout_append_landing_fence_lock`` across the read and ``submitblock``, so +its pair parks the accounting tail at ``phase:tip-height-rpc`` (past the +advisory read, before the fence arms) and lets the appender commit its bump +into exactly that gap. + +The two fail-closed cases that have nothing to do with stamps are here too: +a baseline epoch older than the retained stamp history cannot be answered, +and neither can a candidate that declares no anchor. Both abandon, which is +what the bare counter did for every candidate. + +The scoped predicate reads recorded bumps, so it is only as good as the +guarantee that every predating append bumps. The disarm-gap scenario pins +that guarantee at its one weak point: a seeded window's anchor is exposed +only by the armed artifact, and the bump that consults it also disarms it, +so without the watermark fold a second append could predate the candidate's +window with the anchor set empty -- no bump, no stamp, and a fence that +reads only harmless history submits an invalidated window. + +Finally, the recorded stamp itself is pinned. A row predates an anchor iff +BOTH its stamps do -- iff their max does -- and every equal-stamp scenario +above records the same number under max(), min(), or either stamp alone. +The straddling pair pulls the stamps apart so only max() survives it. +""" + +from __future__ import annotations + +import sys +import unittest +from pathlib import Path +from types import SimpleNamespace +from typing import Any + +sys.path.insert(0, str(Path(__file__).resolve().parents[1])) + +from lab.prism.payout_state import ( # noqa: E402 + PRISM_PAYOUT_APPEND_INVALIDATION_STAMP_HISTORY, +) +from tests.prism_concurrency_harness import assert_deterministic # noqa: E402 +from tests.prism_landing_harness import LandingHarness # noqa: E402 + +BLOCK = "cc" * 32 + +# The candidate's own window. Every share stamp below is placed relative to +# this: at or before it invalidates the candidate, after it does not. +ANCHOR_MS = 12_000 +# A newer published window, live while this candidate lands. It is what makes +# an append that misses the candidate's window still bump the global epoch -- +# without it there would be no defect to scope, because nothing would predate +# a live anchor at all. +NEWER_WINDOW_ANCHOR_MS = 20_000 +# Stamped after the candidate's anchor and before the newer window's: it +# invalidates that newer window and nothing of this candidate's. +LATER_THAN_ANCHOR_MS = 15_000 +# Stamped inside the candidate's own window, and on its exact boundary. +INSIDE_ANCHOR_MS = 9_000 +AT_ANCHOR_MS = ANCHOR_MS +# A newer SEEDED window. Unlike NEWER_WINDOW_ANCHOR_MS it is never handed +# to the published-window watermark: its anchor is exposed only by the +# armed payout-ledger artifact, which the first bump disarms -- the +# precondition for the disarm gap. +NEWER_SEEDED_ANCHOR_MS = 20_000 +# The acceptance stamp of a row whose two stamps differ while both sit +# inside the candidate's window. +ALSO_INSIDE_ANCHOR_MS = 11_000 + +# The last phase the landing names before the fallback lane's authoritative +# fence. Stopping here puts a tail past the advisory pre-offer read, with no +# fence lock held and ``submitblock`` not yet reached -- the one gap in which +# an append-side bump can still overtake a landing that already checked. +BEFORE_AUTHORITATIVE_FENCE = "phase:tip-height-rpc" + + +def _landing_outcome(harness: LandingHarness, found: Any) -> dict[str, object]: + """The durable verdict on one candidate, as a comparable digest.""" + row = harness.pool_block(found.block_hash) + return { + "submitted": list(harness.rpc.submitted), + "abandoned": dict(harness.pool.block_candidate_abandoned_counts), + "stale_classes": dict( + getattr(harness.pool, "stale_job_abandon_counts", None) or {} + ), + "outbox": harness.outbox_state(found), + "chain_state": None if row is None else row.chain_state, + "envelope_written": harness.envelope_written(found), + "live_epoch": harness.live_append_epoch(), + "summary": harness.landing_summary(), + } + + +def _append_then_land( + harness: LandingHarness, + *, + stamp_ms: int, +) -> dict[str, object]: + """The reproduction from #126, parameterised by the appended stamp. + + The append is fully drained before the landing starts, so the epoch has + already moved when the pre-offer fence takes its advisory read. Which + verdict that fence reaches is then a question about ``stamp_ms`` alone. + """ + harness.boot() + found = harness.found_block( + BLOCK, + height=10, + anchor_job_issued_at_ms=ANCHOR_MS, + ) + harness.publish_window_anchor(NEWER_WINDOW_ANCHOR_MS) + appended = harness.append_late_visible_share(stamp_ms=stamp_ms) + harness.drain([harness.appender]) + + harness.land_on_accounting_tail(found) + harness.drain([harness.accounting, harness.appender]) + + outcome = _landing_outcome(harness, found) + # Evidence that the premise held: the row really did advance the global + # epoch. A scenario where it did not would prove nothing about scoping. + outcome["bumped_to"] = appended.value() + return outcome + + +def _land_across_the_authoritative_fence( + harness: LandingHarness, + *, + stamp_ms: int, +) -> dict[str, object]: + """Commit the bump between the advisory read and the fence-held one. + + The accounting tail parks past the pre-offer fence, which therefore saw + an unmoved epoch; the appender then commits its bump into that gap and + the tail resumes into the authoritative fence. ``epoch_at_advisory_read`` + is recorded so a scenario cannot silently degenerate into the drained + shape above. + """ + harness.boot() + harness.break_at(BEFORE_AUTHORITATIVE_FENCE) + found = harness.found_block( + BLOCK, + height=10, + anchor_job_issued_at_ms=ANCHOR_MS, + ) + harness.publish_window_anchor(NEWER_WINDOW_ANCHOR_MS) + + harness.land_on_accounting_tail(found) + harness.run_until(harness.accounting, BEFORE_AUTHORITATIVE_FENCE) + epoch_at_advisory_read = harness.live_append_epoch() + + appended = harness.append_late_visible_share(stamp_ms=stamp_ms) + harness.drain([harness.appender]) + harness.drain([harness.accounting, harness.appender]) + + outcome = _landing_outcome(harness, found) + outcome["epoch_at_advisory_read"] = epoch_at_advisory_read + outcome["bumped_to"] = appended.value() + return outcome + + +def _silent_append_after_disarm(harness: LandingHarness) -> dict[str, object]: + """The disarm gap: a predating append that leaves no bump at all. + + A newer SEEDED window's anchor lives only on the armed artifact -- + deliberately NOT ``publish_window_anchor()``, whose seedless watermark + is immortal and would hide the gap. The first append predates only that + newer window, so it bumps; the bump also disarms the artifact, and with + it the window's only anchor exposure. The second append genuinely + predates the candidate's declared anchor, but if the disarm emptied the + anchor set it records nothing: no bump, no stamp, and the landing's + anchor-scoped fences read only the first bump's harmless stamp and + submit a window a durable share invalidated. The fold of the armed + artifact's anchor into the published-window watermark is what closes + this: the second append still finds an anchor to predate and leaves the + stamp the fences then refuse. + """ + harness.boot() + found = harness.found_block( + BLOCK, + height=10, + anchor_job_issued_at_ms=ANCHOR_MS, + ) + # A staging liberty: production arms a full PayoutLedgerArtifact, but + # the anchor-set arithmetic under test reads only snapshot_anchor_ms. + harness.pool._payout_ledger_artifact = SimpleNamespace( + snapshot_anchor_ms=NEWER_SEEDED_ANCHOR_MS + ) + bump = harness.append_late_visible_share(stamp_ms=LATER_THAN_ANCHOR_MS) + harness.drain([harness.appender]) + silent = harness.append_late_visible_share( + stamp_ms=INSIDE_ANCHOR_MS, + share_id="appender:silent", + ) + harness.drain([harness.appender]) + + harness.land_on_accounting_tail(found) + harness.drain([harness.accounting, harness.appender]) + + outcome = _landing_outcome(harness, found) + outcome["bumped_to"] = bump.value() + outcome["silent_bump"] = silent.value() + outcome["artifact_disarmed"] = harness.pool._payout_ledger_artifact is None + return outcome + + +def _append_with_distinct_stamps_then_land( + harness: LandingHarness, + *, + job_issued_at_ms: int, + accepted_at_ms: int, +) -> dict[str, object]: + """The drained shape above, with the row's two stamps pulled apart. + + The recorded bump stamp is ``max(job_issued_at_ms, accepted_at_ms)``: a + row predates an anchor iff BOTH stamps do, i.e. iff that max does. The + equal-stamp scenarios in this file cannot tell that choice from + ``min()`` or either stamp alone. A straddling row -- issued at or + before the candidate's anchor, accepted after -- is where they part + ways: it does not predate the candidate's window, so recording anything + but the max would abandon a block whose window was never invalidated, + which is #126's over-abandon back again. + """ + harness.boot() + found = harness.found_block( + BLOCK, + height=10, + anchor_job_issued_at_ms=ANCHOR_MS, + ) + harness.publish_window_anchor(NEWER_WINDOW_ANCHOR_MS) + appended = harness.append_late_visible_share( + stamp_ms=job_issued_at_ms, + accepted_at_ms=accepted_at_ms, + ) + harness.drain([harness.appender]) + + harness.land_on_accounting_tail(found) + harness.drain([harness.accounting, harness.appender]) + + outcome = _landing_outcome(harness, found) + outcome["bumped_to"] = appended.value() + return outcome + + +def _land_after_pruned_history(harness: LandingHarness) -> dict[str, object]: + """Push the candidate's baseline off the end of the retained stamps. + + Every append here misses the candidate's window, so an unbounded history + would land the block. The point is that a bounded one must not: once the + baseline epoch is older than the oldest retained bump, the landing cannot + prove anything about the bumps in between and has to abandon. + """ + harness.boot() + found = harness.found_block( + BLOCK, + height=10, + anchor_job_issued_at_ms=ANCHOR_MS, + ) + harness.publish_window_anchor(NEWER_WINDOW_ANCHOR_MS) + bumps = PRISM_PAYOUT_APPEND_INVALIDATION_STAMP_HISTORY + 2 + for index in range(bumps): + harness.append_late_visible_share( + stamp_ms=LATER_THAN_ANCHOR_MS, + share_id=f"appender:pruned-{index}", + ) + # One drain for the whole queue: the appends are independent of each + # other and of the landing, which has not started. + harness.drain([harness.appender], limit=4 * bumps) + + harness.land_on_accounting_tail(found) + harness.drain([harness.accounting, harness.appender]) + + outcome = _landing_outcome(harness, found) + outcome["bumps"] = bumps + return outcome + + +def _land_without_a_declared_anchor(harness: LandingHarness) -> dict[str, object]: + """Land a candidate whose job declared no payout window anchor. + + Nothing scopes the epoch question for such a candidate -- the landing + cannot even expose an anchor for the append side to bump against -- so a + moved epoch stays as terminal as it was before #126. + """ + harness.boot() + found = harness.found_block( + BLOCK, + height=10, + anchor_job_issued_at_ms=ANCHOR_MS, + ) + found.candidate.context.found_block.pop("anchor_job_issued_at_ms") + harness.publish_window_anchor(NEWER_WINDOW_ANCHOR_MS) + appended = harness.append_late_visible_share(stamp_ms=LATER_THAN_ANCHOR_MS) + harness.drain([harness.appender]) + + harness.land_on_accounting_tail(found) + harness.drain([harness.accounting, harness.appender]) + + outcome = _landing_outcome(harness, found) + outcome["bumped_to"] = appended.value() + return outcome + + +class AppendEpochScopedToTheDeclaredAnchorTests(unittest.TestCase): + """#126: only an append that predates *this* anchor may abandon.""" + + def _deterministic(self, scenario: Any) -> dict[str, object]: + outcome = assert_deterministic( + self, + scenario, + harness_factory=LandingHarness, + ).outcome + assert isinstance(outcome, dict) + return outcome + + def _assert_landed(self, outcome: dict[str, object]) -> None: + """The block reached qbitd, confirmed durably, and was not abandoned.""" + self.assertEqual(outcome["submitted"], [f"block:{BLOCK}"]) + self.assertEqual(outcome["chain_state"], "confirmed") + self.assertEqual(outcome["outbox"], "submitted") + self.assertTrue(outcome["envelope_written"]) + self.assertEqual(outcome["abandoned"], {}) + self.assertEqual(outcome["stale_classes"], {}) + + def _assert_abandoned_as_append_epoch_stale( + self, + outcome: dict[str, object], + ) -> None: + """The block was withheld from qbitd and terminally abandoned.""" + self.assertEqual(outcome["submitted"], []) + self.assertIsNone(outcome["chain_state"]) + self.assertEqual(outcome["outbox"], "abandoned") + self.assertFalse(outcome["envelope_written"]) + self.assertEqual(outcome["abandoned"], {"stale-job": 1}) + self.assertEqual( + outcome["stale_classes"], + {"tip_moved": 0, "balance_stale": 0, "append_epoch_stale": 1}, + ) + + # -- the over-abandon --------------------------------------------------- + + def test_an_append_that_misses_this_window_lands_the_block(self) -> None: + """#126's reproduction: the block must reach qbitd and confirm. + + Before the fix this abandoned with ``submitted == []`` and + ``block_candidate_abandoned_counts == {"stale-job": 1}`` -- the + forfeited reward the issue is about. + """ + outcome = self._deterministic( + lambda harness: _append_then_land( + harness, + stamp_ms=LATER_THAN_ANCHOR_MS, + ) + ) + # The premise: the append really did move the global epoch, so the + # bare comparison this fence used to make would have fired. + self.assertEqual(outcome["bumped_to"], 1) + self.assertEqual(outcome["live_epoch"], 1) + self._assert_landed(outcome) + + # -- fail-closed, preserved -------------------------------------------- + + def test_an_append_inside_this_window_still_abandons(self) -> None: + outcome = self._deterministic( + lambda harness: _append_then_land(harness, stamp_ms=INSIDE_ANCHOR_MS) + ) + self.assertEqual(outcome["bumped_to"], 1) + self._assert_abandoned_as_append_epoch_stale(outcome) + + def test_an_append_exactly_at_the_declared_anchor_still_abandons(self) -> None: + """The boundary belongs to the terminal side. + + A row predates an anchor when *both* its stamps are at or before it, + so a share stamped exactly at the anchor is inside the candidate's + window and its coinbase would omit a durable share. + """ + outcome = self._deterministic( + lambda harness: _append_then_land(harness, stamp_ms=AT_ANCHOR_MS) + ) + self.assertEqual(outcome["bumped_to"], 1) + self._assert_abandoned_as_append_epoch_stale(outcome) + + # -- the disarm gap ------------------------------------------------------ + + def test_a_predating_append_after_the_seeded_anchor_disarms_still_abandons( + self, + ) -> None: + """A bump that disarms the artifact must not blind the next append. + + Before the watermark fold this submitted: the second append + returned ``None`` (no live anchor covered it), the fences saw only + the first bump's harmless stamp, and ``submitted`` held the block + -- a coinbase whose window omitted a durable share. + """ + outcome = self._deterministic(_silent_append_after_disarm) + # The premise: the first append moved the epoch by predating the + # seeded window, and doing so disarmed the artifact that was that + # window's only anchor exposure. + self.assertEqual(outcome["bumped_to"], 1) + self.assertTrue(outcome["artifact_disarmed"]) + # The fix: the second append still finds an anchor to predate -- + # the disarm folded the seeded anchor into the watermark -- so its + # bump and stamp exist for the fences to refuse. + self.assertEqual(outcome["silent_bump"], 2) + self.assertEqual(outcome["live_epoch"], 2) + self._assert_abandoned_as_append_epoch_stale(outcome) + + # -- the recorded stamp is the max of the row's two ---------------------- + + def test_a_straddling_append_lands_the_block(self) -> None: + """Issued at or before the anchor, accepted after: not predating. + + The row invalidates the newer published window (both stamps are at + or before its anchor), so the epoch moves -- but the candidate's + own window is intact, and only a recorded stamp of + ``max(job_issued_at_ms, accepted_at_ms)`` says so. ``min()`` or the + issue stamp alone records 9_000 here and abandons a valid block. + """ + outcome = self._deterministic( + lambda harness: _append_with_distinct_stamps_then_land( + harness, + job_issued_at_ms=INSIDE_ANCHOR_MS, + accepted_at_ms=LATER_THAN_ANCHOR_MS, + ) + ) + self.assertEqual(outcome["bumped_to"], 1) + self._assert_landed(outcome) + + def test_an_append_with_distinct_stamps_inside_the_window_still_abandons( + self, + ) -> None: + """And the same distinct-stamp row is terminal once both predate.""" + outcome = self._deterministic( + lambda harness: _append_with_distinct_stamps_then_land( + harness, + job_issued_at_ms=INSIDE_ANCHOR_MS, + accepted_at_ms=ALSO_INSIDE_ANCHOR_MS, + ) + ) + self.assertEqual(outcome["bumped_to"], 1) + self._assert_abandoned_as_append_epoch_stale(outcome) + + # -- the authoritative fence ------------------------------------------- + + def test_a_bump_after_the_advisory_read_is_terminal_when_it_predates( + self, + ) -> None: + """The fence-held check still catches what the advisory one missed.""" + outcome = self._deterministic( + lambda harness: _land_across_the_authoritative_fence( + harness, + stamp_ms=INSIDE_ANCHOR_MS, + ) + ) + # The interleaving is the whole point: the pre-offer fence read an + # unmoved epoch, so this verdict can only come from the fence-held + # comparison that runs after it. + self.assertEqual(outcome["epoch_at_advisory_read"], 0) + self.assertEqual(outcome["bumped_to"], 1) + self._assert_abandoned_as_append_epoch_stale(outcome) + + def test_a_bump_after_the_advisory_read_is_harmless_when_it_does_not( + self, + ) -> None: + """And forgives the same interleaving when the stamp cannot predate.""" + outcome = self._deterministic( + lambda harness: _land_across_the_authoritative_fence( + harness, + stamp_ms=LATER_THAN_ANCHOR_MS, + ) + ) + self.assertEqual(outcome["epoch_at_advisory_read"], 0) + self.assertEqual(outcome["bumped_to"], 1) + self._assert_landed(outcome) + + # -- what cannot be answered ------------------------------------------- + + def test_a_baseline_older_than_the_retained_history_abandons(self) -> None: + """Pruning narrows acceptance; it never widens it.""" + outcome = self._deterministic(_land_after_pruned_history) + self.assertEqual( + outcome["live_epoch"], + PRISM_PAYOUT_APPEND_INVALIDATION_STAMP_HISTORY + 2, + ) + # Every one of those appends missed the candidate's window, so this + # abandon is the retention bound talking and nothing else. + self._assert_abandoned_as_append_epoch_stale(outcome) + + def test_a_candidate_without_a_declared_anchor_abandons(self) -> None: + outcome = self._deterministic(_land_without_a_declared_anchor) + self.assertEqual(outcome["bumped_to"], 1) + self._assert_abandoned_as_append_epoch_stale(outcome) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_prism_landing_budget_watchdog.py b/tests/test_prism_landing_budget_watchdog.py new file mode 100644 index 000000000..caf5f9d27 --- /dev/null +++ b/tests/test_prism_landing_budget_watchdog.py @@ -0,0 +1,710 @@ +#!/usr/bin/env python3 +"""One landing step's budget against the watchdog that is timing it. + +Issue #125: a landing-class ledger step is two waits, not one. The caller +first queues for the ledger's writer lock -- no statement has been sent, so +neither ``statement_timeout`` nor any server-side cancellation bounds it -- +and only then runs the statement it was queuing for, under its own full +budget. Both are spent on the block-work thread the watchdog monitors, and +before ``49f04c3`` both were heartbeat-silent. At the reviewed 120s +escalation cap that is ~240s of silence against a 120s tolerance: the +watchdog hard-exits mid-landing, the in-memory escalation counter dies with +the process, and the restart comes back at the 30s base to ride the same +cycle again. No accounting is lost -- the outbox row stays pending and +replays -- but found-block settlement is deferred behind a restart loop. + +``49f04c3`` answers it in two halves, and these scenarios are shaped around +them: + +* the admission wait is served in slices whenever a progress hook is + installed, so a queued landing stamps its heartbeat instead of going + quiet -- while the caller's own deadline stays the sole authority on when + the wait ends; +* every configured landing budget, base and escalated cap alike, is clamped + to a fraction of the watchdog tolerance, so admission plus statement + cannot outrun the monitor timing them. + +These pass on ``2.x.x``. Their evidence is not that the fix is correct -- +its own PR argued that against unit-level doubles -- but that the +interleaving #125 describes can now be *driven*: a real landing tail queuing +on the real ledger writer lock, escalating its real budget across real +timed-out attempts, on a clock that moves only when this file says so. +""" + +from __future__ import annotations + +import sys +import threading +import unittest +from collections.abc import Callable +from pathlib import Path +from typing import Any + +sys.path.insert(0, str(Path(__file__).resolve().parents[1])) + +from tests.prism_concurrency_harness import Actor, assert_deterministic # noqa: E402 +from tests.prism_landing_harness import ( # noqa: E402 + PARENT_HASH, + LandingHarness, +) + +BLOCK_A = "aa" * 32 +BLOCK_B = "bb" * 32 + +# The landing names its own boundaries; these are the two it names that this +# file turns on. ``phase:accounting`` is the last stamp before the accounting +# tail opens its landing-class budget scope, so an actor parked there is +# parked exactly one step short of the wait #125 is about. +# ``wait-ledger-admission:landing`` is the stamp the fix added *inside* that +# wait -- arming it turns every heartbeat into a checkpoint, which makes the +# stamps part of the order-stable trace rather than something counted on the +# side. +LANDING_SCOPE = "phase:accounting" +ADMISSION_STAMP = "phase:wait-ledger-admission:landing" + +WRITER_LOCK_HELD = "writer-lock-held" +CLAMP_LINE = "prism coordinator: landing db budget clamped by watchdog" +GATE_TIMEOUT = "timed out waiting for postgres writer lock" + + +# -------------------------------------------------------------------------- +# Driving +# -------------------------------------------------------------------------- + + +def _own_block_work(harness: LandingHarness) -> None: + """Let the accounting actor stand in for the block_accounting thread. + + Block-work stamps are gated to the two threads that own a heartbeat slot + (``_block_work_heartbeat_owner``), which is the whole reason a client + connection thread cannot refresh the accounting thread's liveness budget + on its behalf. ``block_accounting_loop`` claims that slot by stamping its + own ident; the harness dequeues nothing and drives + ``_run_block_accounting_task`` directly, so the claim has to be made + here. Without it the admission stamps still fire -- the fix does not + depend on ownership -- but they land on the unowned fallback slot with no + phase attached, which is not the shape production runs and not the shape + #125 describes. + """ + service = harness.pool._ensure_block_candidate_service() + + def claim() -> None: + service._block_accounting_thread_ident = threading.get_ident() + + harness.accounting.submit(claim, label="own-block-work") + harness.run_until(harness.accounting, "done:own-block-work") + + +def _pump( + harness: LandingHarness, + actor: Actor, + label: str, + *, + on_stop: Callable[[str, float], None] | None = None, + limit: int = 20_000, +) -> None: + """Step ``actor`` to ``label``, moving virtual time when it waits on it. + + ``run_until`` cannot drive a sliced wait: an actor parked on a slice + deadline is not runnable until the clock reaches it, and only the + controller can move a virtual clock. Advancing to the *next* pending + deadline rather than by a fixed step is what keeps the schedule a + function of the scenario -- every second that passes here is a second + some deadline in the code under test asked for, and an actor parked with + no deadline at all is reported rather than waited on, because nothing + would ever release it. + """ + for _ in range(limit): + if not actor.runnable(): + if harness.advance_to_next_deadline() is None: + raise AssertionError( + f"actor {actor.name!r} is parked at {actor.stop!r} with no " + "deadline; nothing bounds this wait" + ) + continue + stop = harness.step(actor) + if on_stop is not None: + on_stop(stop, harness.clock.monotonic()) + if stop == label: + return + raise AssertionError( + f"actor {actor.name!r} did not reach {label!r} within {limit} steps; " + f"last stop {actor.stop!r}" + ) + + +def _hold_writer_lock(harness: LandingHarness) -> None: + """Park the client actor holding the ledger's writer lock. + + A landing queues on admission only if something else is already inside + the gate. The client tail is the honest holder: in production it is the + other thread that lands blocks, and a share append on it takes the same + lock. Parking it at a checkpoint rather than blocking it means the + controller decides exactly when the queue clears. + """ + lock = harness.pool.ledger._lock + + def hold() -> None: + lock.acquire() + harness.scheduler.checkpoint(WRITER_LOCK_HELD) + lock.release() + + harness.client.submit(hold, label="hold-writer-lock") + harness.run_until(harness.client, WRITER_LOCK_HELD) + + +def _release_writer_lock(harness: LandingHarness) -> None: + harness.run_until(harness.client, "done:hold-writer-lock") + + +def _granted_budget(harness: LandingHarness, block_hash: str | None) -> float: + """What production would actually grant this hash's next landing step. + + Read through the coordinator rather than recomputed here, because the + clamp arithmetic is the thing under test; a second copy of it in the + test would agree with itself and prove nothing. It runs on an actor + because it takes the coordinator lock, which is a harness lock. + """ + call = harness.accounting.submit( + lambda: harness.pool._block_landing_db_timeout(block_hash), + label="landing-budget", + ) + harness.run_until(harness.accounting, "done:landing-budget") + return float(call.value()) + + +def _wait_slice(harness: LandingHarness) -> float: + service = harness.pool._ensure_block_candidate_service() + return float(service._block_work_wait_slice()) + + +def _timeout_count(harness: LandingHarness, block_hash: str) -> int: + service = harness.pool._ensure_block_candidate_service() + counts = getattr(service, "_block_landing_timeout_counts", None) or {} + return int(counts.get(block_hash, 0)) + + +def _land_while_queued( + harness: LandingHarness, + found: Any, + *, + release_after_stamps: int | None = None, +) -> dict[str, Any]: + """Run one accounting-tail landing attempt against a held writer lock. + + The tail is stopped at the last boundary before its landing-class scope + opens, the lock is taken there, and the tail is then let go: everything + from that point on -- the admission wait, its stamps, the deadline that + ends it -- is the real landing running under its real budget. + + ``release_after_stamps`` hands the lock back mid-wait, which is the only + way to get a landing that queues *and then still runs*; leaving it held + can only ever produce the timing-out half of the story. + """ + harness.break_at(LANDING_SCOPE) + call = harness.land_on_accounting_tail(found) + harness.run_until(harness.accounting, LANDING_SCOPE) + started = harness.clock.monotonic() + _hold_writer_lock(harness) + + stamps: list[float] = [] + after_admission: list[float] = [] + queued = True + + def observe(stop: str, when: float) -> None: + nonlocal queued + if stop != ADMISSION_STAMP: + return + # The same phase covers both reports the fix added: the slices of an + # admission wait, and the round trip a gate-holding step completes + # between two statements it cannot release the gate between. Which + # one a stamp is depends only on whether this landing is still + # outside the gate, so they are split on that rather than the label. + if not queued: + after_admission.append(when) + return + stamps.append(when) + if release_after_stamps is not None and len(stamps) == release_after_stamps: + _release_writer_lock(harness) + queued = False + + _pump( + harness, + harness.accounting, + f"done:accounting-land:{found.block_hash[:4]}", + on_stop=observe, + ) + finished = harness.clock.monotonic() + # Admission ends when the lock is handed over -- the last stamp taken + # while it was still shut -- or, when it never is, at the deadline one + # slice past the last stamp. Nothing else in a landing consumes virtual + # time, so the tail's own completion pins the timing-out case exactly. + admitted_at = stamps[-1] if release_after_stamps is not None else finished + return { + "started": started, + "stamps": stamps, + "after_admission": after_admission, + "admitted_at": admitted_at, + "finished": finished, + "admission_seconds": admitted_at - started, + "call": call, + } + + +def _silent_gaps(started: float, stamps: list[float], ended: float) -> list[float]: + """Every interval the block-work heartbeat went unstamped, in order.""" + marks = [started, *stamps, ended] + return [round(later - earlier, 6) for earlier, later in zip(marks, marks[1:])] + + +def _prepared_harness( + harness: LandingHarness, + *, + configured_max_seconds: float, +) -> None: + """Boot, claim the block-work slot, and make every admission stamp visible. + + ``block_landing_db_timeout_max_seconds`` is the escalation cap an + operator configures; ``LandingHarness`` exposes the base budget and the + watchdog tolerance as constructor arguments but not the cap, so it is + set here before anything can read it. + """ + harness.pool.block_landing_db_timeout_max_seconds = float(configured_max_seconds) + harness.boot() + _own_block_work(harness) + harness.break_at(ADMISSION_STAMP) + + +# -------------------------------------------------------------------------- +# 1. The admission wait is no longer silent +# -------------------------------------------------------------------------- + + +def _stamped_admission_wait(harness: LandingHarness) -> dict[str, Any]: + """A landing that queues on the writer lock until its budget runs out.""" + _prepared_harness(harness, configured_max_seconds=120.0) + found = harness.found_block(BLOCK_A, height=10, parent_hash=PARENT_HASH) + budget = _granted_budget(harness, BLOCK_A) + + attempt = _land_while_queued(harness, found) + _release_writer_lock(harness) + + return { + "budget": budget, + "slice": _wait_slice(harness), + "stamps": len(attempt["stamps"]), + "waited": round(attempt["admission_seconds"], 6), + "silent_gaps": sorted( + set( + _silent_gaps( + attempt["started"], + attempt["stamps"], + attempt["admitted_at"], + ) + ) + ), + # The tail swallows the timeout and leaves the row for replay, which + # is the behaviour #125 says keeps accounting correct. The gate error + # itself reaches stderr with the traceback. + "tail_error": ( + type(attempt["call"].error).__name__ if attempt["call"].error else None + ), + "gate_timed_out": GATE_TIMEOUT in harness.errors, + "escalation_armed": _timeout_count(harness, BLOCK_A), + "outbox": harness.outbox_state(found), + "summary": harness.landing_summary(), + } + + +def _sliced_and_unsliced_wait(harness: LandingHarness) -> dict[str, Any]: + """The same gate, the same budget, once with the hook and once without. + + Slicing is only safe if it is invisible to the caller's deadline, so the + claim needs a control rather than an argument: one landing-class step + under the production scope that installs the hook, and one bare + ``statement_timeout`` at the same budget, both queued behind the same + held lock on the same actor. Anything the slicing changed about *when* + the wait ends would show up as a difference between the two. + """ + _prepared_harness(harness, configured_max_seconds=120.0) + ledger = harness.pool.ledger + budget = _granted_budget(harness, BLOCK_B) + _hold_writer_lock(harness) + + def queue(scope: Callable[[], Any], label: str) -> dict[str, Any]: + def run() -> None: + with scope(): + # A landing-class read that takes the writer lock, so the + # wait under test is the ledger's own admission gate rather + # than something the harness invented. + ledger.current_prior_balances() + + call = harness.accounting.submit(run, label=label) + started = harness.clock.monotonic() + stamps: list[float] = [] + _pump( + harness, + harness.accounting, + f"done:{label}", + on_stop=lambda stop, when: ( + stamps.append(when) if stop == ADMISSION_STAMP else None + ), + ) + return { + "stamps": len(stamps), + "waited": round(harness.clock.monotonic() - started, 6), + "error": type(call.error).__name__ if call.error else None, + "message": str(call.error) if call.error else None, + } + + hooked = queue( + lambda: harness.pool._block_landing_ledger_statement_timeout_scope(BLOCK_B), + "hooked-admission", + ) + unhooked = queue(lambda: ledger.statement_timeout(budget), "unhooked-admission") + _release_writer_lock(harness) + + return {"budget": budget, "hooked": hooked, "unhooked": unhooked} + + +class AdmissionWaitStampTests(unittest.TestCase): + """#125, first half: a queued landing must not go quiet.""" + + def test_a_queued_landing_stamps_its_heartbeat_while_it_waits(self) -> None: + outcome = assert_deterministic( + self, + _stamped_admission_wait, + harness_factory=LandingHarness, + ).outcome + assert isinstance(outcome, dict) + + # The default deployment: a 120s watchdog grants 30s to a first + # landing attempt, and the wait is served in watchdog-sized slices. + self.assertEqual(outcome["budget"], 30.0) + self.assertEqual(outcome["slice"], 0.25) + + # The assertion the fix exists for. Before it, this wait was one + # unbroken silence; now every slice boundary reports liveness, and + # the longest gap the watchdog could observe is one slice. + self.assertEqual(outcome["stamps"], 119) + self.assertEqual(outcome["silent_gaps"], [0.25]) + + # ...and the slicing bought that without lengthening the wait: the + # caller's own deadline still ends it, to the microsecond. + self.assertEqual(outcome["waited"], 30.0) + + # The landing failed the way #125 says it does: the gate deadline + # fires, the tail keeps the durable row for replay, and the next + # attempt for this hash is armed to escalate. + self.assertTrue(outcome["gate_timed_out"]) + self.assertIsNone(outcome["tail_error"]) + self.assertEqual(outcome["outbox"], "pending") + self.assertEqual(outcome["escalation_armed"], 1) + + def test_slicing_the_wait_does_not_lengthen_it(self) -> None: + outcome = assert_deterministic( + self, + _sliced_and_unsliced_wait, + harness_factory=LandingHarness, + ).outcome + assert isinstance(outcome, dict) + + hooked = outcome["hooked"] + unhooked = outcome["unhooked"] + + # One wait reports 119 times, the other not once... + self.assertEqual(hooked["stamps"], 119) + self.assertEqual(unhooked["stamps"], 0) + + # ...and they end at the same instant, with the same error. The + # deadline is the authority; the hook only decides how loudly the + # caller waits for it. + self.assertEqual(hooked["waited"], outcome["budget"]) + self.assertEqual(unhooked["waited"], outcome["budget"]) + self.assertEqual(hooked["error"], "LedgerOperationTimeout") + self.assertEqual(unhooked["error"], "LedgerOperationTimeout") + self.assertEqual(hooked["message"], unhooked["message"]) + self.assertIn(GATE_TIMEOUT, hooked["message"]) + + +# -------------------------------------------------------------------------- +# 2. The escalation ladder cannot outrun the watchdog +# -------------------------------------------------------------------------- + + +def _escalation_ladder( + harness: LandingHarness, + *, + configured_max_seconds: float, + attempts: int, +) -> dict[str, Any]: + """Time out ``attempts`` landings for one hash and read the ladder off. + + The escalation counter is keyed by block hash and moved only by an + observed landing-class timeout, so the ladder is driven the way + production climbs it -- one genuinely timed-out attempt per rung -- + rather than by writing the counter. + """ + _prepared_harness(harness, configured_max_seconds=configured_max_seconds) + found = harness.found_block(BLOCK_A, height=10, parent_hash=PARENT_HASH) + watchdog = float(harness.pool.watchdog_timeout_seconds) + + rungs: list[dict[str, Any]] = [] + for _ in range(attempts): + granted = _granted_budget(harness, BLOCK_A) + attempt = _land_while_queued(harness, found) + _release_writer_lock(harness) + rungs.append( + { + "granted": granted, + "waited": round(attempt["admission_seconds"], 6), + "stamps": len(attempt["stamps"]), + # #125's arithmetic: admission and statement each receive a + # full budget, so this is what one landing step can cost the + # heartbeat before the watchdog is entitled to give up. + "admission_plus_statement": 2.0 * granted, + "fits_watchdog": 2.0 * granted <= watchdog, + } + ) + + return { + "watchdog": watchdog, + "configured_base": float(harness.pool.block_landing_db_timeout_seconds), + "configured_max": float(harness.pool.block_landing_db_timeout_max_seconds), + "rungs": rungs, + "clamped": harness.output.count(CLAMP_LINE), + "escalations": _timeout_count(harness, BLOCK_A), + "outbox": harness.outbox_state(found), + } + + +# ``assert_deterministic`` takes a one-argument scenario, so each +# configuration under test is a named binding of the ladder above rather +# than a parameter the test method passes in. + + +def _clamped_cap_ladder(harness: LandingHarness) -> dict[str, Any]: + return _escalation_ladder(harness, configured_max_seconds=120.0, attempts=3) + + +def _clamped_base_ladder(harness: LandingHarness) -> dict[str, Any]: + return _escalation_ladder(harness, configured_max_seconds=120.0, attempts=2) + + +def _unclamped_ladder(harness: LandingHarness) -> dict[str, Any]: + return _escalation_ladder(harness, configured_max_seconds=60.0, attempts=3) + + +class EscalatedBudgetCeilingTests(unittest.TestCase): + """#125, second half: escalation stops below the watchdog, and says so.""" + + def test_the_reviewed_cap_is_clamped_to_the_default_watchdog(self) -> None: + """The exact deployment #125 was reported against. + + A 120s tolerance with the reviewed 120s escalation cap is the + configuration that produces the ~240s heartbeat gap: two full + budgets, one monitor, and no slack at all between them. + """ + outcome = assert_deterministic( + self, + _clamped_cap_ladder, + harness_factory=LandingHarness, + ).outcome + assert isinstance(outcome, dict) + + self.assertEqual(outcome["watchdog"], 120.0) + self.assertEqual(outcome["configured_base"], 30.0) + self.assertEqual(outcome["configured_max"], 120.0) + + # What #125 measured: the configured cap alone is the whole watchdog + # tolerance, so admission plus statement is twice it. + self.assertGreater(2.0 * outcome["configured_max"], outcome["watchdog"]) + + granted = [rung["granted"] for rung in outcome["rungs"]] + self.assertEqual(granted, [30.0, 60.0, 60.0]) + + # Every rung, including the exhausted one, leaves the watchdog able + # to tell a slow landing from a wedged one. + for rung in outcome["rungs"]: + self.assertTrue(rung["fits_watchdog"], rung) + self.assertLessEqual(rung["admission_plus_statement"], outcome["watchdog"]) + + # The ladder was climbed by real timed-out landings, not asserted: + # each rung spent its whole granted budget queuing, and stamped + # every slice of it. + self.assertEqual([rung["waited"] for rung in outcome["rungs"]], granted) + self.assertEqual([rung["stamps"] for rung in outcome["rungs"]], [119, 239, 239]) + self.assertEqual(outcome["escalations"], 3) + self.assertEqual(outcome["outbox"], "pending") + + # A reduced budget is never silent -- and it is said once, not on + # every budget read, of which this scenario made several per rung. + self.assertEqual(outcome["clamped"], 1) + + def test_a_tight_watchdog_clamps_the_base_budget_too(self) -> None: + """The ceiling is not a cap-only rule. + + An operator who tightens the watchdog without touching the landing + budgets moves the ceiling below the *base*, and the first attempt -- + the one that never escalated -- is already the one that would outrun + the monitor. Escalation then has nowhere to climb, which is the + correct answer: a budget that cannot be spent safely is not a budget. + """ + outcome = assert_deterministic( + self, + _clamped_base_ladder, + harness_factory=LandingHarness, + watchdog_timeout_seconds=40.0, + ).outcome + assert isinstance(outcome, dict) + + self.assertEqual(outcome["watchdog"], 40.0) + self.assertEqual(outcome["configured_base"], 30.0) + self.assertGreater(2.0 * outcome["configured_base"], outcome["watchdog"]) + + granted = [rung["granted"] for rung in outcome["rungs"]] + self.assertEqual(granted, [20.0, 20.0]) + for rung in outcome["rungs"]: + self.assertTrue(rung["fits_watchdog"], rung) + + self.assertEqual([rung["waited"] for rung in outcome["rungs"]], granted) + self.assertEqual([rung["stamps"] for rung in outcome["rungs"]], [79, 79]) + self.assertEqual(outcome["clamped"], 1) + + def test_a_budget_the_watchdog_can_afford_is_left_alone(self) -> None: + """The clamp is a ceiling, not a policy. + + A 300s tolerance with a 60s cap has room for both halves of a + landing step, so nothing is reduced and nothing is logged. This is + the case that keeps the ceiling honest: if it lowered budgets it did + not need to, the reported clamp would stop meaning anything. + """ + outcome = assert_deterministic( + self, + _unclamped_ladder, + harness_factory=LandingHarness, + watchdog_timeout_seconds=300.0, + ).outcome + assert isinstance(outcome, dict) + + self.assertEqual(outcome["watchdog"], 300.0) + self.assertEqual(outcome["configured_max"], 60.0) + + granted = [rung["granted"] for rung in outcome["rungs"]] + self.assertEqual(granted, [30.0, 60.0, 60.0]) + for rung in outcome["rungs"]: + self.assertTrue(rung["fits_watchdog"], rung) + + self.assertEqual([rung["waited"] for rung in outcome["rungs"]], granted) + self.assertEqual(outcome["clamped"], 0) + + +# -------------------------------------------------------------------------- +# 3. Both halves at once +# -------------------------------------------------------------------------- + + +def _queue_then_land_at_the_escalated_budget( + harness: LandingHarness, +) -> dict[str, Any]: + """Burn a whole escalated admission budget, then land on what is left. + + #125's worst case is not a landing that times out -- that one merely + retries. It is a landing that queues for almost everything admission + will give it and *then* starts a statement with a fresh full budget, + because that is the pair the watchdog sees back to back. So the first + attempt is spent to arm the escalation, and the second is handed the + lock one slice before its own deadline: the longest admission wait that + still ends in a landing. + """ + _prepared_harness(harness, configured_max_seconds=120.0) + found = harness.found_block(BLOCK_A, height=10, parent_hash=PARENT_HASH) + watchdog = float(harness.pool.watchdog_timeout_seconds) + slice_seconds = _wait_slice(harness) + + # Rung one: a full base budget spent queuing, which is what escalates. + base = _granted_budget(harness, BLOCK_A) + _land_while_queued(harness, found) + _release_writer_lock(harness) + + # Rung two: the escalated budget, released with one slice to spare. + escalated = _granted_budget(harness, BLOCK_A) + attempt = _land_while_queued( + harness, + found, + release_after_stamps=int(escalated / slice_seconds) - 1, + ) + + admission = attempt["admission_seconds"] + return { + "watchdog": watchdog, + "base": base, + "escalated": escalated, + "admission": round(admission, 6), + "stamps": len(attempt["stamps"]), + "stamps_after_admission": len(attempt["after_admission"]), + "longest_silence": max( + _silent_gaps( + attempt["started"], + attempt["stamps"], + attempt["admitted_at"], + ) + ), + # The quantity #125 is about: what this one landing step can still + # cost the block-work heartbeat, counting the admission it actually + # spent plus the untouched statement budget waiting on the far side + # of the gate. + "worst_case": round(admission + escalated, 6), + "landed": attempt["call"].error is None, + "outbox": harness.outbox_state(found), + "sequence": harness.publication_sequence(found), + "envelope": harness.envelope_written(found), + "summary": harness.landing_summary(), + } + + +class CombinedWorstCaseTests(unittest.TestCase): + """#125 end to end: queue on admission, then spend the escalated budget.""" + + def test_a_queued_escalated_landing_stays_inside_the_watchdog(self) -> None: + outcome = assert_deterministic( + self, + _queue_then_land_at_the_escalated_budget, + harness_factory=LandingHarness, + ).outcome + assert isinstance(outcome, dict) + + self.assertEqual(outcome["base"], 30.0) + self.assertEqual(outcome["escalated"], 60.0) + + # The landing queued for all but one slice of its escalated budget. + self.assertEqual(outcome["admission"], 59.75) + self.assertEqual(outcome["stamps"], 239) + + # Admission plus a fresh statement budget -- the exact pair that read + # as ~240s of silence at the unclamped cap -- now fits the tolerance. + self.assertEqual(outcome["worst_case"], 119.75) + self.assertLessEqual(outcome["worst_case"], outcome["watchdog"]) + + # And the watchdog never had to infer any of that: the longest stretch + # it saw without a stamp was one slice. + self.assertEqual(outcome["longest_silence"], 0.25) + + # Past the gate the same contract holds one level down. A landing + # step that must issue two statements without releasing the writer + # lock reports the first one's completed round trip, so the pair + # costs the monitor one budget of silence rather than two -- the + # same doubling #125 found on the other side of the gate. + self.assertEqual(outcome["stamps_after_admission"], 1) + + # The point of staying alive: the block lands. Nearly the whole + # escalated budget went to queuing and the landing still completed, + # published its ordinal and wrote its envelope. + self.assertTrue(outcome["landed"]) + self.assertEqual(outcome["outbox"], "submitted") + self.assertEqual(outcome["sequence"], 1) + self.assertTrue(outcome["envelope"]) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_prism_landing_harness_model.py b/tests/test_prism_landing_harness_model.py new file mode 100644 index 000000000..ea765cb0f --- /dev/null +++ b/tests/test_prism_landing_harness_model.py @@ -0,0 +1,1388 @@ +#!/usr/bin/env python3 +"""Self-tests for the landing half of the deterministic concurrency harness. + +``tests/test_prism_concurrency_harness.py`` defends the lease half. This file +defends the half that models block candidate landing, for the same reason: a +fake that answers an unrecognised statement plausibly drifts away from +production without anyone noticing, and every scenario built on it then +inherits the drift. When a landing scenario fails, these tests are what makes +"the code under test is wrong" the first explanation rather than "the model +is wrong". + +Four claims are pinned here, each one something a landing scenario relies on +without restating: + +1. **The publication-ordinal allocator follows ``qbit_confirm_pool_block``.** + The ordinal comes from a sequence rather than from ``MAX + 1``, so a fresh + confirmation burns a value, an exact replay burns nothing, a terminally + disposed row reports the distinct superseded disposition, and the durable + floor — a ``MAX`` over the table — tolerates the gap an aborted + confirmation leaves. #133 turns on precisely the difference between the + sequence and the floor, so the model owes the SQL an exact account of it. + +2. **Transaction scope is real.** A deadline-scoped landing statement sends + its COMMIT as a separate message, and the window in between is where the + interesting interleavings live. A model that published a landing write at + statement time would close that window and hide the defects inside it. + +3. **Classification is anchored to production SQL.** Every landing statement + is driven through the shipped ``PsqlShareLedger`` method that emits it, so + a change to that SQL breaks classification loudly instead of quietly + changing what the fake models. + +4. **The synchronisation primitives behave like the ones they replace.** The + landing topology is the first one where two threads of a single process + contend a process-local lock, so the harness locks are load-bearing: an + ordering they can express but a real lock cannot would be an ordering a + scenario could "prove" and production could never reach. +""" + +from __future__ import annotations + +import contextlib +import sys +import unittest +from collections.abc import Callable, Iterator +from dataclasses import replace +from pathlib import Path +from typing import Any + +sys.path.insert(0, str(Path(__file__).resolve().parents[1])) + +from tests.prism_concurrency_harness import ( # noqa: E402 + Actor, + Call, + HarnessError, + LandingOp, + PoolBlockRow, + UnsupportedStatement, + assert_deterministic, + classify, +) +from tests.prism_landing_harness import ( # noqa: E402 + PARENT_HASH, + LandingHarness, +) + +BLOCK_A = "aa" * 32 +BLOCK_B = "bb" * 32 +BLOCK_C = "cc" * 32 +BLOCK_D = "dd" * 32 + +# The top of the publish step, named by the landing itself. A tail stopped +# here has confirmed durably and released every lock it took to do so. +PUBLISH_BOUNDARY = "progress:evidence-write" + +# A successor's session token. Nothing but the token has to differ: the +# landing statements fence on the whole identity triple. +SUCCESSOR_TOKEN = "heartbeat-v1:successor" + + +def _run( + harness: LandingHarness, + actor: Actor, + work: Callable[[], Any], + label: str, +) -> Call: + """Run ``work`` to completion on ``actor`` and hand back its call record. + + The call record rather than its value, because half the assertions here + are about the error a statement raised. + """ + call = actor.submit(work, label=label) + harness.run_until(actor, f"done:{label}") + return call + + +def _prepared( + harness: LandingHarness, + block_hash: str, + *, + height: int, + parent_hash: str = PARENT_HASH, + chain_state: str = "prepared", + maturity_state: str = "immature", +) -> PoolBlockRow: + """Seed one committed ``qbit_pool_blocks`` row. + + Landing a whole block to reach a given row state would put the landing + state machine between the test and the allocator, which is the wrong + thing under test here: these cases are about what the allocator does with + a row, not about how the row got there. + """ + row = PoolBlockRow( + block_hash=block_hash, + block_height=height, + parent_hash=parent_hash, + chain_state=chain_state, + maturity_state=maturity_state, + ) + harness.server.pool_blocks[block_hash] = row + return row + + +@contextlib.contextmanager +def _deposed_writer(harness: LandingHarness) -> Iterator[None]: + """Hand the writer lease to a successor for the duration of the block. + + A landing statement carries its writer identity inline, so moving the + committed lease row is the whole of a takeover as far as one statement is + concerned. Building a second coordinator to produce the successor would + add a lease lifecycle to a test that is not about lease lifecycles. + """ + live = harness.server.lease + assert live is not None + harness.server.lease = replace(live, writer_session_token=SUCCESSOR_TOKEN) + try: + yield + finally: + harness.server.lease = live + + +def _captured_sql( + harness: LandingHarness, + actor: Actor, + work: Callable[[], Any], + label: str, +) -> str: + """Production's SQL for one landing statement, without its effects. + + Every landing statement fences on the writer lease before it writes + anything, so running one under a deposed writer records the statement — + classified, with its payload recovered — while leaving every table + untouched. That is what lets a test replay genuine production SQL into a + transaction it intends to abort, instead of hand-writing the statement + whose fidelity is the thing being tested. + """ + with _deposed_writer(harness): + call = _run(harness, actor, work, label) + assert call.error is not None, "a deposed writer's landing statement must fail" + return harness.server.statements[-1].sql + + +def _read_back_sql(harness: LandingHarness, ledger: Any) -> str: + """The pool-block state read, as ``pool_block_state`` emits it. + + A read takes no lease CTE, so it needs none of ``_captured_sql``'s + deposition: running it is already free of effects. + """ + _run( + harness, + harness.accounting, + lambda: ledger.pool_block_state(block_hash=BLOCK_A), + "capture-state", + ).value() + return harness.server.statements[-1].sql + + +class PublicationOrdinalAllocatorTests(unittest.TestCase): + """``qbit_confirm_pool_block``, as the model claims to implement it.""" + + def test_a_fresh_confirmation_assigns_the_next_sequence_value(self) -> None: + with LandingHarness() as harness: + ledger = harness.boot() + _prepared(harness, BLOCK_A, height=10) + _prepared(harness, BLOCK_B, height=11, parent_hash=BLOCK_A) + + first = _run( + harness, + harness.client, + lambda: ledger.confirm_accepted_block( + block_hash=BLOCK_A, + active_tip_height=10, + ), + "confirm-a", + ).value() + second = _run( + harness, + harness.client, + lambda: ledger.confirm_accepted_block( + block_hash=BLOCK_B, + active_tip_height=11, + ), + "confirm-b", + ).value() + + self.assertEqual(first["confirmed_count"], 1) + self.assertEqual(first["audit_publication_sequence"], 1) + self.assertEqual(second["audit_publication_sequence"], 2) + + row = harness.pool_block(BLOCK_A) + assert row is not None + self.assertEqual(row.chain_state, "confirmed") + # Confirmation moves the chain state only. Maturity is the reorg + # window's business and 100 blocks away. + self.assertEqual(row.maturity_state, "immature") + + def test_an_exact_replay_returns_one_without_burning_a_sequence_value( + self, + ) -> None: + """The half of the allocator's contract a counter model would get wrong. + + A replay that burned a value would leave the row's ordinal behind the + sequence, so the next block would jump — and the durable floor would + rise for a confirmation that changed nothing. The proof is the *next* + block's ordinal, not the replay's own return value. + """ + with LandingHarness() as harness: + ledger = harness.boot() + _prepared(harness, BLOCK_A, height=10) + _prepared(harness, BLOCK_B, height=11, parent_hash=BLOCK_A) + + _run( + harness, + harness.client, + lambda: ledger.confirm_accepted_block( + block_hash=BLOCK_A, + active_tip_height=10, + ), + "confirm-a", + ).value() + replay = _run( + harness, + harness.client, + lambda: ledger.confirm_accepted_block( + block_hash=BLOCK_A, + active_tip_height=10, + ), + "replay-a", + ).value() + + self.assertEqual(replay["confirmed_count"], 1) + self.assertEqual(replay["audit_publication_sequence"], 1) + + after_replay = _run( + harness, + harness.client, + lambda: ledger.confirm_accepted_block( + block_hash=BLOCK_B, + active_tip_height=11, + ), + "confirm-b", + ).value() + self.assertEqual( + after_replay["audit_publication_sequence"], + 2, + "the replay burned a sequence value it must not have burned", + ) + + def test_a_terminally_disposed_row_reports_the_superseded_disposition( + self, + ) -> None: + """-1 is a distinct answer, not a variant of 0. + + A caller reading -1 abandons the candidate knowing the ledger + explained itself; a caller reading 0 has to treat the state as + unexplained. Collapsing the two would let a routine reorg race look + like corruption, so every terminal shape the SQL names is checked + here rather than one representative. + """ + terminal = { + "reorg-quarantined": ("11" * 32, {"chain_state": "inactive"}), + "rejected": ("22" * 32, {"chain_state": "rejected"}), + "reversed": ("33" * 32, {"chain_state": "reversed"}), + # Confirmed but reversed: the maturity arm of the same test, and + # the one a chain_state-only model would answer as a replay. + "confirmed-then-reversed": ( + "44" * 32, + {"chain_state": "confirmed", "maturity_state": "reversed"}, + ), + } + with LandingHarness() as harness: + ledger = harness.boot() + for name, (block_hash, state) in terminal.items(): + with self.subTest(disposition=name): + _prepared(harness, block_hash, height=10, **state) + result = _run( + harness, + harness.client, + lambda h=block_hash: ledger.confirm_accepted_block( + block_hash=h, + active_tip_height=10, + ), + f"confirm-{name}", + ).value() + self.assertEqual(result["confirmed_count"], -1) + self.assertNotIn("audit_publication_sequence", result) + + # None of them burned an ordinal, so nothing was allocated at all. + self.assertEqual( + _run( + harness, + harness.client, + ledger.audit_publication_sequence_floor, + "floor", + ).value(), + 0, + ) + + def test_a_height_mismatch_and_a_missing_row_both_report_no_match(self) -> None: + """0 keeps its own meaning: no row, or a live row this does not match.""" + with LandingHarness() as harness: + ledger = harness.boot() + _prepared(harness, BLOCK_A, height=10) + + mismatched = _run( + harness, + harness.client, + lambda: ledger.confirm_accepted_block( + block_hash=BLOCK_A, + active_tip_height=11, + ), + "wrong-height", + ).value() + missing = _run( + harness, + harness.client, + lambda: ledger.confirm_accepted_block( + block_hash=BLOCK_C, + active_tip_height=10, + ), + "no-row", + ).value() + + self.assertEqual(mismatched["confirmed_count"], 0) + self.assertEqual(missing["confirmed_count"], 0) + row = harness.pool_block(BLOCK_A) + assert row is not None + self.assertEqual(row.chain_state, "prepared") + self.assertIsNone(row.audit_publication_sequence) + + def test_the_durable_floor_is_a_max_and_tolerates_a_burned_gap(self) -> None: + """``nextval`` is non-transactional, so an abort leaves a hole. + + The floor read is a ``MAX`` over the table for exactly this reason: it + follows the ordinals that became durable, not the ones the sequence + handed out. A model that counted allocations instead would put the + floor ahead of every published block after the first aborted + confirmation, and #133's whole argument is about how a block's ordinal + compares to that floor. + """ + with LandingHarness() as harness: + ledger = harness.boot() + _prepared(harness, BLOCK_A, height=10) + _prepared(harness, BLOCK_C, height=12, parent_hash=BLOCK_A) + _prepared(harness, BLOCK_D, height=13, parent_hash=BLOCK_C) + + _run( + harness, + harness.client, + lambda: ledger.confirm_accepted_block( + block_hash=BLOCK_A, + active_tip_height=10, + ), + "confirm-a", + ).value() + self.assertEqual(harness.publication_floor(), 1) + + confirm_c = _captured_sql( + harness, + harness.client, + lambda: ledger.confirm_accepted_block( + block_hash=BLOCK_C, + active_tip_height=12, + ), + "capture-confirm-c", + ) + + def aborted_confirmation() -> int | None: + backend = harness.server.connect(application_name="aborted") + transaction = harness.server._begin(backend, explicit=True) + harness.server.execute( + backend, + confirm_c, + transaction=transaction, + timeout_seconds=30.0, + tag="aborted", + ) + burned = transaction.staged_pool_blocks[ + BLOCK_C + ].audit_publication_sequence + harness.server._rollback(transaction) + return burned + + burned = _run( + harness, + harness.client, + aborted_confirmation, + "abort-confirm-c", + ).value() + self.assertEqual(burned, 2) + + floor_after_abort = _run( + harness, + harness.client, + ledger.audit_publication_sequence_floor, + "floor-after-abort", + ).value() + self.assertEqual( + floor_after_abort, + 1, + "an aborted confirmation must not raise the durable floor", + ) + + after = _run( + harness, + harness.client, + lambda: ledger.confirm_accepted_block( + block_hash=BLOCK_D, + active_tip_height=13, + ), + "confirm-d", + ).value() + self.assertEqual( + after["audit_publication_sequence"], + 3, + "the burned value must never be handed out again", + ) + self.assertEqual(harness.publication_floor(), 3) + + +class LandingTransactionVisibilityTests(unittest.TestCase): + """The confirm→COMMIT window, which is where landing defects live.""" + + def test_a_staged_landing_write_is_invisible_to_another_session(self) -> None: + with LandingHarness() as harness: + ledger = harness.boot() + _prepared(harness, BLOCK_A, height=10) + # Captured up front, so the reader replays one identical statement + # on both sides of the COMMIT. + state_sql = _read_back_sql(harness, ledger) + reader = harness.server.connect(application_name="reader") + + def read_pool_block() -> Any: + return harness.server.execute( + reader, + state_sql, + transaction=None, + timeout_seconds=None, + tag="reader", + ) + + def confirm_under_deadline() -> Any: + # A caller deadline is what makes _NativePostgresClient wrap + # the statement in BEGIN / statement / COMMIT, so it is also + # what opens the window this test is about. + with ledger.operation_timeout(30.0): + return ledger.confirm_accepted_block( + block_hash=BLOCK_A, + active_tip_height=10, + ) + + landing = harness.client.submit(confirm_under_deadline, label="confirm") + harness.run_until(harness.client, "ledger.precommit") + + assert harness.sql_backend is not None + transaction = harness.sql_backend._in_use[0].transaction + assert transaction is not None + staged = transaction.staged_pool_blocks[BLOCK_A] + self.assertEqual(staged.chain_state, "confirmed") + self.assertEqual(staged.audit_publication_sequence, 1) + + before = _run( + harness, + harness.accounting, + read_pool_block, + "read-before-commit", + ).value() + self.assertEqual(before["state"]["chain_state"], "prepared") + self.assertIsNone(before["state"]["audit_publication_sequence"]) + + harness.drain([harness.client]) + landing.value() + + after = _run( + harness, + harness.accounting, + read_pool_block, + "read-after-commit", + ).value() + self.assertEqual(after["state"]["chain_state"], "confirmed") + self.assertEqual(after["state"]["audit_publication_sequence"], 1) + + def test_a_transaction_sees_its_own_landing_write_before_commit(self) -> None: + """READ COMMITTED: committed rows, plus this transaction's own writes. + + The other half of the window. A landing reads its own confirmation + back — that is how it learns its publication ordinal — so a model that + staged writes where even their own session could not see them would + break every landing rather than only the interleaved ones. + """ + with LandingHarness() as harness: + ledger = harness.boot() + _prepared(harness, BLOCK_A, height=10) + state_sql = _read_back_sql(harness, ledger) + confirm_sql = _captured_sql( + harness, + harness.client, + lambda: ledger.confirm_accepted_block( + block_hash=BLOCK_A, + active_tip_height=10, + ), + "capture-confirm", + ) + + def confirm_then_read() -> Any: + backend = harness.server.connect(application_name="own-session") + transaction = harness.server._begin(backend, explicit=True) + harness.server.execute( + backend, + confirm_sql, + transaction=transaction, + timeout_seconds=30.0, + tag="own", + ) + own = harness.server.execute( + backend, + state_sql, + transaction=transaction, + timeout_seconds=30.0, + tag="own", + ) + harness.server._commit(transaction) + return own + + own = _run( + harness, + harness.client, + confirm_then_read, + "own-session-read", + ).value() + self.assertEqual(own["state"]["chain_state"], "confirmed") + self.assertEqual(own["state"]["audit_publication_sequence"], 1) + # And it really was the same transaction's write that it saw. + row = harness.pool_block(BLOCK_A) + assert row is not None + self.assertEqual(row.chain_state, "confirmed") + + def test_a_rolled_back_landing_leaves_no_row_but_keeps_the_burned_ordinal( + self, + ) -> None: + """Rollback undoes rows. It does not undo ``nextval``. + + Both halves matter to a landing scenario: a durable-looking row that + an abort left behind would make a scenario assert a landing that never + happened, and an ordinal handed back on abort would make the sequence + look like the counter it deliberately is not. + """ + with LandingHarness() as harness: + ledger = harness.boot() + _prepared(harness, BLOCK_A, height=10) + _prepared(harness, BLOCK_C, height=12, parent_hash=BLOCK_A) + _prepared(harness, BLOCK_D, height=13, parent_hash=BLOCK_C) + + _run( + harness, + harness.client, + lambda: ledger.confirm_accepted_block( + block_hash=BLOCK_A, + active_tip_height=10, + ), + "confirm-a", + ).value() + + found = harness.found_block(BLOCK_C, height=12, parent_hash=BLOCK_A) + intent = harness.pool.block_candidate_intent(found.candidate) + outbox_sql = _captured_sql( + harness, + harness.client, + lambda: ledger.persist_block_candidate_intent(intent), + "capture-outbox", + ) + confirm_sql = _captured_sql( + harness, + harness.client, + lambda: ledger.confirm_accepted_block( + block_hash=BLOCK_C, + active_tip_height=12, + ), + "capture-confirm", + ) + + def aborted_landing() -> None: + backend = harness.server.connect(application_name="aborted") + transaction = harness.server._begin(backend, explicit=True) + for sql in (outbox_sql, confirm_sql): + harness.server.execute( + backend, + sql, + transaction=transaction, + timeout_seconds=30.0, + tag="aborted", + ) + self.assertIn(BLOCK_C, transaction.staged_outbox) + self.assertNotIn(BLOCK_C, harness.server.outbox) + harness.server._rollback(transaction) + + _run(harness, harness.client, aborted_landing, "abort").value() + + self.assertNotIn(BLOCK_C, harness.server.outbox) + row = harness.pool_block(BLOCK_C) + assert row is not None + self.assertEqual(row.chain_state, "prepared") + self.assertIsNone(row.audit_publication_sequence) + + next_ordinal = _run( + harness, + harness.client, + lambda: ledger.confirm_accepted_block( + block_hash=BLOCK_D, + active_tip_height=13, + ), + "confirm-d", + ).value() + self.assertEqual(next_ordinal["audit_publication_sequence"], 3) + + +class LandingStatementClassificationTests(unittest.TestCase): + """The classifier is pinned to the SQL PsqlShareLedger actually emits.""" + + def test_every_modelled_landing_statement_is_recognised_from_real_calls( + self, + ) -> None: + """Drive every ``LandingOp`` through production code and classify it. + + The fragments in ``_LANDING_SIGNATURES`` were lifted from + ``share_ledger.py``; this is what proves they still match what that + module generates rather than what it generated when the landing model + was written. A full landing covers most of them on its own — that it + does is itself part of the claim, because a scenario that never + reaches a statement never checks it either. + """ + with LandingHarness() as harness: + ledger = harness.boot() + found = harness.found_block(BLOCK_A, height=10, parent_hash=PARENT_HASH) + landed = harness.land_on_client_tail(found) + harness.drain([harness.client]) + self.assertIsNone(landed.error) + + # The four a successful landing never emits: nothing was + # rejected, neither the share history nor the pool-block state + # is read on the happy path, and the height-bounded as-of balance + # read belongs to confirmed-ancestor replay only (issue #209). + _prepared(harness, BLOCK_B, height=11, parent_hash=BLOCK_A) + _run( + harness, + harness.client, + lambda: ledger.reject_prepared_block( + block_hash=BLOCK_B, + active_tip_height=11, + ), + "reject", + ).value() + _run(harness, harness.client, ledger.all_shares, "shares").value() + _run( + harness, + harness.client, + lambda: ledger.pool_block_state(block_hash=BLOCK_A), + "state", + ).value() + _run( + harness, + harness.client, + lambda: ledger.prior_balances_after_pool_block( + block_hash=BLOCK_A, + ), + "as-of", + ).value() + + # The replay path's two statements (issues #196 and #211). Neither + # is on a landing's own critical path -- the enumeration recovers + # missed wakeups and the batch abandon collapses superseded + # siblings -- so both are driven explicitly here. + _prepared(harness, BLOCK_C, height=12, parent_hash=BLOCK_B) + _run( + harness, + harness.client, + lambda: ledger.persist_block_candidate_intent( + { + "schema": "qbit.prism.block-candidate-intent.v1", + "block_hash_hex": BLOCK_C, + "block_hex": "00", + } + ), + "intent-c", + ).value() + page = _run( + harness, + harness.client, + lambda: ledger._legacy_pending_block_candidate_rows(limit=32), + "pending-page", + ).value() + self.assertEqual([row["block_hash"] for row in page], [BLOCK_C]) + # BLOCK_C has a prepared pool-block row, so the fenced batch write + # re-checks it and reports nothing: the page fact and the write's + # own veto are exercised together. + self.assertTrue(page[0]["pool_block_exists"]) + self.assertEqual( + _run( + harness, + harness.client, + lambda: ledger.mark_block_candidates_abandoned( + block_hashes=[BLOCK_C], + error="superseded by a decided height", + ), + "batch-abandon", + ).value(), + (), + ) + + # Issue #255: the chunked-body statements. A candidate with a + # share array wider than the fast path gets a span and page + # index; the metadata-first page, a hydration (manifest, spans, + # pages, chunk pages), a retired staging body and one janitor + # step drive the rest. + block_d = "dd" * 32 + self.assertEqual( + _run( + harness, + harness.client, + ledger.verify_candidate_schema, + "schema-check", + ).value(), + {"declared": 2, "has_body_table": True}, + ) + _run( + harness, + harness.client, + lambda: ledger.persist_block_candidate_intent( + { + "schema": "qbit.prism.block-candidate-intent.v1", + "block_hash_hex": block_d, + "block_hex": "00", + "parent_hash": BLOCK_C, + "expected_height": 13, + "template": {"previousblockhash": BLOCK_C, "height": 13}, + "found_block": {"network_difficulty": 1}, + "pending_share": {"job_id": "job-d"}, + "credit_share_on_accept": False, + "username": "miner-d", + "shares_json": [{"share_seq": index, "share_id": f"s{index}"} for index in range(4000)], + } + ), + "intent-d", + ).value() + headers = _run( + harness, + harness.client, + lambda: ledger.pending_block_candidate_headers(limit=32), + "header-page", + ).value() + # BLOCK_C stayed pending (its pool-block row vetoed the abandon). + self.assertEqual([row["block_hash"] for row in headers.rows], [BLOCK_C, block_d]) + self.assertTrue(headers.exhausted) + hydrated = _run( + harness, + harness.client, + lambda: ledger.hydrate_block_candidate_intent(headers.rows[1]), + "hydrate-d", + ).value() + self.assertEqual(len(hydrated["shares_json"]), 4000) + self.assertEqual(hydrated["shares_json"][3999]["share_id"], "s3999") + hydrated.body.close() + _run( + harness, + harness.client, + lambda: ledger._retire_staging_body("ab" * 16), + "retire-staging", + ).value() + _run( + harness, + harness.client, + ledger.reap_retired_candidate_bodies, + "janitor", + ).value() + + observed = { + kind + for kind in harness.statement_kinds() + if kind in {op.value for op in LandingOp} + } + self.assertEqual(observed, {op.value for op in LandingOp}) + + def test_a_landing_table_alone_is_not_a_classification(self) -> None: + """A signature match, not a table mention, is what answers a statement. + + The deferred markers for the landing tables are gone now that landing + is modelled, so a statement against one of them that matches no + signature has to reach the unmodelled-statement error rather than + being answered by whichever evaluator matched loosest. + """ + for sql in ( + "SELECT count(*) FROM qbit_pool_blocks;", + "DELETE FROM qbit_block_candidate_outbox WHERE block_hash = 'ab';", + "UPDATE qbit_pool_blocks SET maturity_state = 'mature';", + ): + with self.subTest(sql=sql): + with self.assertRaises(UnsupportedStatement) as caught: + classify(sql) + self.assertIn("does not model this statement", str(caught.exception)) + self.assertIn("_LANDING_SIGNATURES", str(caught.exception)) + + def test_deferred_state_machines_still_name_their_follow_up(self) -> None: + """Landing borders two state machines #128 defers, and says so. + + A landing scenario that wanders into share submission or CTV fanout + gets an error naming the follow-up rather than a bare model gap, and + that remains true now that landing statements are recognised ahead of + the deferred markers. + """ + for sql, expected in ( + ("INSERT INTO qbit_share_ledger (share_id)", "share submission"), + ( + "UPDATE qbit_ctv_fanouts SET state = 'broadcast'", + "CTV fanout", + ), + ): + with self.subTest(sql=sql): + with self.assertRaises(UnsupportedStatement) as caught: + classify(sql) + self.assertIn(expected, str(caught.exception)) + self.assertIn("#128", str(caught.exception)) + + def test_the_writer_identity_is_recovered_from_the_statement(self) -> None: + """The model is never told who a session belongs to; it reads the SQL. + + The two shapes recover it differently — the pool-block functions take + it as inlined arguments, the outbox statements inline it into their + lease CTE — so both are checked. If either recovery broke, the fence + below would fail open rather than closed. + """ + with LandingHarness() as harness: + ledger = harness.boot() + _prepared(harness, BLOCK_A, height=10) + found = harness.found_block(BLOCK_A, height=10, parent_hash=PARENT_HASH) + + _run( + harness, + harness.client, + lambda: ledger.confirm_accepted_block( + block_hash=BLOCK_A, + active_tip_height=10, + ), + "confirm", + ).value() + confirm = next( + statement + for statement in reversed(harness.server.statements) + if statement.kind is LandingOp.CONFIRM_BLOCK + ) + self.assertEqual(confirm.payload["block_hash"], BLOCK_A) + self.assertEqual(confirm.payload["active_tip_height"], 10) + + _run( + harness, + harness.client, + lambda: ledger.persist_block_candidate_intent( + harness.pool.block_candidate_intent(found.candidate) + ), + "outbox", + ).value() + outbox = harness.server.statements[-1] + self.assertEqual(outbox.kind, LandingOp.OUTBOX_RECORD) + self.assertEqual(outbox.payload["block_hash"], BLOCK_A) + + for statement in (confirm, outbox): + with self.subTest(kind=statement.kind.value): + self.assertEqual( + statement.payload["writer_id"], + "prism-coordinator", + ) + self.assertEqual(statement.payload["writer_epoch"], 1) + self.assertEqual( + statement.payload["writer_session_token"], + harness.session_token, + ) + + def test_a_deposed_writer_fails_the_lease_cte_in_the_shape_sql_dictates( + self, + ) -> None: + """Same fence, two observable shapes, and the difference is real. + + ``qbit_confirm_pool_block`` and ``qbit_reject_prepared_pool_block`` + ``RAISE`` when their lease UPDATE affects no rows, so the client sees + an error. The outbox statements are plain CTEs whose ``CASE`` reports + the same condition as a value, so the client sees a result and its + caller is what turns it into an error. A model that flattened the two + would let a scenario put a fenced-out landing on the wrong side of a + ``try``. + """ + with LandingHarness() as harness: + ledger = harness.boot() + _prepared(harness, BLOCK_A, height=10) + found = harness.found_block(BLOCK_A, height=10, parent_hash=PARENT_HASH) + + confirm_sql = _captured_sql( + harness, + harness.client, + lambda: ledger.confirm_accepted_block( + block_hash=BLOCK_A, + active_tip_height=10, + ), + "capture-confirm", + ) + reject_sql = _captured_sql( + harness, + harness.client, + lambda: ledger.reject_prepared_block( + block_hash=BLOCK_A, + active_tip_height=10, + ), + "capture-reject", + ) + outbox_sql = _captured_sql( + harness, + harness.client, + lambda: ledger.persist_block_candidate_intent( + harness.pool.block_candidate_intent(found.candidate) + ), + "capture-outbox", + ) + + assert harness.sql_backend is not None + backend = harness.sql_backend + with _deposed_writer(harness): + for name, sql in ( + ("confirm", confirm_sql), + ("reject_prepared", reject_sql), + ): + with self.subTest(statement=name): + call = _run( + harness, + harness.client, + lambda s=sql: backend.run_json(s), + f"fenced-{name}", + ) + self.assertIsInstance(call.error, RuntimeError) + self.assertEqual( + str(call.error), + "writer lease is not active", + ) + + outbox = _run( + harness, + harness.client, + lambda: backend.run_json(outbox_sql), + "fenced-outbox", + ).value() + self.assertEqual( + outbox, + {"error": "writer lease is not active"}, + ) + + # Fenced out means fenced out: nothing reached a table either way. + row = harness.pool_block(BLOCK_A) + assert row is not None + self.assertEqual(row.chain_state, "prepared") + self.assertEqual(harness.server.outbox, {}) + + +class HarnessLockTests(unittest.TestCase): + """A contended process-local lock is an interleaving point, not a stall.""" + + def test_two_actors_are_mutually_excluded_and_granted_in_arrival_order( + self, + ) -> None: + """FIFO, because the schedule is an input here. + + Without a queue the model could grant in an order neither a real + futex-backed lock nor the scheduler would produce, and a scenario + could then "prove" an interleaving production cannot reach. + """ + with LandingHarness() as harness: + lock = harness.lock("payout-balance-mutation") + granted: list[str] = [] + + def hold(name: str) -> Callable[[], None]: + def run() -> None: + lock.acquire() + granted.append(name) + harness.scheduler.checkpoint(f"{name}:holding") + lock.release() + + return run + + harness.client.submit(hold("client"), label="client-hold") + harness.accounting.submit(hold("accounting"), label="accounting-hold") + harness.appender.submit(hold("appender"), label="appender-hold") + + harness.run_until(harness.client, "client:holding") + # Arrival order is chosen by the controller, not by thread timing. + parked = "lock:payout-balance-mutation" + harness.run_until_blocked(harness.accounting, parked) + harness.run_until_blocked(harness.appender, parked) + self.assertEqual(granted, ["client"]) + self.assertIs(lock.owner, harness.client) + + harness.step(harness.client) + self.assertTrue(harness.accounting.runnable()) + self.assertFalse( + harness.appender.runnable(), + "a later arrival must not overtake a queued waiter", + ) + + harness.run_until(harness.accounting, "accounting:holding") + self.assertEqual(granted, ["client", "accounting"]) + self.assertFalse(harness.appender.runnable()) + + harness.step(harness.accounting) + harness.run_until(harness.appender, "appender:holding") + self.assertEqual(granted, ["client", "accounting", "appender"]) + harness.drain([harness.client, harness.accounting, harness.appender]) + self.assertIsNone(lock.owner) + + def test_a_non_blocking_acquire_refuses_rather_than_parking(self) -> None: + with LandingHarness() as harness: + lock = harness.lock("tip-refresh") + + def hold() -> None: + lock.acquire() + harness.scheduler.checkpoint("holding") + lock.release() + + harness.client.submit(hold, label="hold") + harness.run_until(harness.client, "holding") + + probe = _run( + harness, + harness.accounting, + lambda: lock.acquire(blocking=False), + "try-acquire", + ) + self.assertFalse(probe.value()) + # It refused instead of queueing, so nothing is waiting on the + # holder and the holder owes it nothing. + self.assertFalse(harness.accounting.blocked) + harness.drain([harness.client, harness.accounting]) + + def test_a_timed_acquire_expires_against_the_virtual_clock(self) -> None: + with LandingHarness() as harness: + lock = harness.lock("job-cache") + + def hold() -> None: + lock.acquire() + harness.scheduler.checkpoint("holding") + lock.release() + + harness.client.submit(hold, label="hold") + harness.run_until(harness.client, "holding") + + started = harness.clock.monotonic() + timed = harness.accounting.submit( + lambda: lock.acquire(timeout=5.0), + label="timed-acquire", + ) + harness.run_until_blocked(harness.accounting, "lock:job-cache") + self.assertEqual(harness.accounting.wake_at, started + 5.0) + + harness.advance_to_next_deadline() + harness.run_until(harness.accounting, "done:timed-acquire") + self.assertFalse(timed.value()) + self.assertEqual(harness.clock.monotonic(), started + 5.0) + # Nothing slept: the wait was measured, not endured. + self.assertEqual(harness.sleeps, []) + harness.drain([harness.client, harness.accounting]) + + def test_re_acquiring_a_non_reentrant_lock_is_reported_not_hidden(self) -> None: + """A real mutex deadlocks there, and a harness that hid it would lie. + + Silently granting the second acquire would turn a production deadlock + into a green scenario, which is the single most expensive thing this + harness could get wrong. + """ + with LandingHarness() as harness: + lock = harness.lock("payout-append-landing-fence") + + def double_acquire() -> None: + lock.acquire() + lock.acquire() + + call = _run(harness, harness.client, double_acquire, "double") + with self.assertRaises(HarnessError) as caught: + call.value() + self.assertIn("production would deadlock here", str(caught.exception)) + + +class HarnessRLockTests(unittest.TestCase): + def test_the_owner_re_enters_and_a_waiter_waits_for_depth_zero(self) -> None: + with LandingHarness() as harness: + lock = harness.rlock("coordinator") + + def hold() -> None: + lock.acquire() + lock.acquire() + harness.scheduler.checkpoint("depth-two") + lock.release() + harness.scheduler.checkpoint("depth-one") + lock.release() + + harness.client.submit(hold, label="hold") + waiter = harness.accounting.submit(lock.acquire, label="waiter") + harness.run_until(harness.client, "depth-two") + harness.run_until_blocked(harness.accounting, "lock:coordinator") + + harness.run_until(harness.client, "depth-one") + self.assertFalse( + harness.accounting.runnable(), + "an outer release still leaves the lock held", + ) + + harness.step(harness.client) + self.assertTrue(harness.accounting.runnable()) + harness.run_until(harness.accounting, "done:waiter") + self.assertTrue(waiter.value()) + self.assertIs(lock.owner, harness.accounting) + + +class HarnessConditionTests(unittest.TestCase): + """``wait`` must drop the lock, or the notifier could never take it.""" + + def test_a_wait_releases_the_lock_and_a_notify_from_elsewhere_wakes_it( + self, + ) -> None: + with LandingHarness() as harness: + condition = harness.condition("accepted-block-payout-preview") + notifier_held_the_lock: list[bool] = [] + reacquired: list[bool] = [] + + def wait_for_notify() -> bool: + with condition: + notified = condition.wait() + # Held again before wait returns, because a predicate loop + # written against a real condition re-reads shared state + # on this line. + reacquired.append(condition._lock.held_by_current()) + return notified + + def notify() -> None: + with condition: + notifier_held_the_lock.append(condition._lock.locked()) + condition.notify_all() + + waiter = harness.accounting.submit(wait_for_notify, label="wait") + harness.run_until_blocked( + harness.accounting, + "cond:accepted-block-payout-preview", + ) + self.assertIsNone( + condition._lock.owner, + "a parked waiter must not still hold the condition's lock", + ) + + _run(harness, harness.client, notify, "notify").value() + self.assertEqual(notifier_held_the_lock, [True]) + self.assertTrue(harness.accounting.runnable()) + + harness.run_until(harness.accounting, "done:wait") + self.assertTrue(waiter.value()) + self.assertEqual(reacquired, [True]) + self.assertIsNone(condition._lock.owner) + + def test_a_timed_wait_expires_against_the_virtual_clock(self) -> None: + with LandingHarness() as harness: + condition = harness.condition("unfenced-append-drained") + started = harness.clock.monotonic() + + def wait_briefly() -> bool: + with condition: + return condition.wait(7.5) + + timed = harness.appender.submit(wait_briefly, label="timed-wait") + harness.run_until_blocked(harness.appender, "cond:unfenced-append-drained") + self.assertEqual(harness.appender.wake_at, started + 7.5) + + harness.advance_to_next_deadline() + harness.run_until(harness.appender, "done:timed-wait") + self.assertFalse(timed.value(), "an expiry is not a notification") + self.assertEqual(harness.clock.monotonic(), started + 7.5) + self.assertEqual(harness.sleeps, []) + + def test_re_entrant_depth_survives_a_wait(self) -> None: + """The coordinator's condition sits on a reentrant lock. + + A wait that re-acquired to depth one would leave the caller's outer + ``with`` releasing a lock it no longer holds — a failure a long way + from its cause. Proved behaviourally: after one release the lock is + still held, which can only be true if the depth came back as two. + """ + with LandingHarness() as harness: + condition = harness.condition("accepted-block-payout-preview") + depth_after_wait: list[bool] = [] + + def nested_wait() -> None: + condition.acquire() + condition.acquire() + condition.wait() + condition.release() + depth_after_wait.append(condition._lock.locked()) + condition.release() + + harness.accounting.submit(nested_wait, label="nested") + harness.run_until_blocked( + harness.accounting, + "cond:accepted-block-payout-preview", + ) + _run(harness, harness.client, condition.notify_all, "notify").value() + harness.run_until(harness.accounting, "done:nested") + + self.assertEqual(depth_after_wait, [True]) + self.assertIsNone(condition._lock.owner) + + +class HarnessSemaphoreTests(unittest.TestCase): + """The ledger's read slots, which a landing scenario can exhaust on purpose.""" + + def test_exhaustion_parks_and_a_release_admits_one_waiter(self) -> None: + with LandingHarness() as harness: + semaphore = harness.semaphore("ledger-read-slot", 1) + + def hold() -> None: + semaphore.acquire() + harness.scheduler.checkpoint("holding-slot") + semaphore.release() + + harness.client.submit(hold, label="hold") + harness.run_until(harness.client, "holding-slot") + self.assertEqual(semaphore.value, 0) + + waiter = harness.accounting.submit(semaphore.acquire, label="wait-slot") + harness.run_until_blocked(harness.accounting, "semaphore:ledger-read-slot") + + harness.step(harness.client) + self.assertTrue(harness.accounting.runnable()) + harness.run_until(harness.accounting, "done:wait-slot") + self.assertTrue(waiter.value()) + self.assertEqual(semaphore.value, 0) + + def test_releasing_above_the_initial_value_is_refused(self) -> None: + """``BoundedSemaphore``'s check, kept: an over-release is a real bug. + + A slot handed back twice inflates read concurrency for the rest of the + run, and the scenario that then passes is measuring a pool production + never had. + """ + with LandingHarness() as harness: + semaphore = harness.semaphore("ledger-read-slot", 2) + call = _run(harness, harness.client, semaphore.release, "over-release") + with self.assertRaises(HarnessError) as caught: + call.value() + self.assertIn("released too many times", str(caught.exception)) + + +class HarnessEventTests(unittest.TestCase): + def test_a_wait_parks_until_the_event_is_set(self) -> None: + with LandingHarness() as harness: + event = harness.event("block-landed") + waiter = harness.appender.submit(event.wait, label="await-event") + harness.run_until_blocked(harness.appender, "event:block-landed") + self.assertFalse(event.is_set()) + + _run(harness, harness.client, event.set, "set-event").value() + self.assertTrue(harness.appender.runnable()) + harness.run_until(harness.appender, "done:await-event") + self.assertTrue(waiter.value()) + + def test_a_timed_wait_expires_against_the_virtual_clock(self) -> None: + with LandingHarness() as harness: + event = harness.event("block-landed") + started = harness.clock.monotonic() + timed = harness.appender.submit( + lambda: event.wait(3.0), + label="timed-event", + ) + harness.run_until_blocked(harness.appender, "event:block-landed") + self.assertEqual(harness.appender.wake_at, started + 3.0) + + harness.advance_to_next_deadline() + harness.run_until(harness.appender, "done:timed-event") + self.assertFalse(timed.value()) + self.assertEqual(harness.clock.monotonic(), started + 3.0) + self.assertEqual(harness.sleeps, []) + + +def _landing_with_a_replay_in_the_publish_window( + harness: LandingHarness, +) -> dict[str, object]: + """One landing, with the allocator driven inside its confirm→publish gap. + + The gap is the landing model's one genuinely concurrent window — the + landing has confirmed durably and released every lock it took to do so — + so it is the right place to prove the allocator's answers do not depend on + which actor the controller steps next. + """ + ledger = harness.boot() + harness.break_at(PUBLISH_BOUNDARY) + found = harness.found_block(BLOCK_A, height=10, parent_hash=PARENT_HASH) + landed = harness.land_on_client_tail(found) + harness.run_until(harness.client, PUBLISH_BOUNDARY) + + _prepared(harness, BLOCK_B, height=11, parent_hash=BLOCK_A) + _prepared(harness, BLOCK_C, height=12, parent_hash=BLOCK_B) + replay = harness.accounting.submit( + lambda: ledger.confirm_accepted_block( + block_hash=BLOCK_A, + active_tip_height=10, + ), + label="replay-a", + ) + rejected = harness.accounting.submit( + lambda: ledger.reject_prepared_block( + block_hash=BLOCK_B, + active_tip_height=11, + ), + label="reject-b", + ) + harness.run_until(harness.accounting, "done:reject-b") + harness.drain([harness.client, harness.accounting]) + + # After the window rather than inside it, and deliberately so. A bare + # confirmation allocates an ordinal that no publication ever follows, and + # a landing still holding a lower one then fails its own publish against a + # floor nothing published raised. That is the landing refusing to publish + # behind the ledger, not a model artefact — but it belongs to a scenario + # about the landing, and this one is about the allocator. + confirmed = _run( + harness, + harness.accounting, + lambda: ledger.confirm_accepted_block( + block_hash=BLOCK_C, + active_tip_height=12, + ), + "confirm-c", + ) + + return { + "landed": landed.error is None and landed.result is True, + "a_sequence": harness.publication_sequence(found), + "replay": replay.value(), + "rejected": rejected.value(), + "confirmed": confirmed.value(), + "floor": harness.publication_floor(), + "summary": harness.landing_summary(), + } + + +class LandingModelDeterminismTests(unittest.TestCase): + """A scenario that is merely usually right is worse than no scenario.""" + + def test_the_landing_model_is_order_stable_across_repeated_runs(self) -> None: + outcome = assert_deterministic( + self, + _landing_with_a_replay_in_the_publish_window, + harness_factory=LandingHarness, + ).outcome + assert isinstance(outcome, dict) + + self.assertTrue(outcome["landed"]) + self.assertEqual(outcome["a_sequence"], 1) + # The replay ran in the gap and reported the ordinal already assigned. + self.assertEqual(outcome["replay"]["confirmed_count"], 1) + self.assertEqual(outcome["replay"]["audit_publication_sequence"], 1) + self.assertEqual(outcome["rejected"]["rejected_count"], 1) + # And it burned nothing: the next fresh confirmation takes 2. + self.assertEqual(outcome["confirmed"]["audit_publication_sequence"], 2) + self.assertEqual(outcome["floor"], 2) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_prism_landing_interleavings.py b/tests/test_prism_landing_interleavings.py new file mode 100644 index 000000000..2eb542aad --- /dev/null +++ b/tests/test_prism_landing_interleavings.py @@ -0,0 +1,206 @@ +#!/usr/bin/env python3 +"""Two distinct found blocks interleaved across PRISM's two landing tails. + +Issue #133: when two *distinct* found blocks land inside one confirm→publish +window and one of them lands on the synchronous client-thread route, the +earlier block's live audit envelope was silently and permanently never +written. Nothing was lost but the published evidence pointer — the block was +mined, paid and audited correctly — which is exactly why no error appeared +and no test caught it. The reachable case was untested because the per-hash +disposition lease provides no mutual exclusion across hashes, and no test +interleaved two hashes. + +That defect is fixed on ``2.x.x`` by ``03e733e``, so these scenarios pass. +The evidence they carry is not that the fix works — its own PR asserted that +against a hand-built store — but that the *interleaving* can now be driven +end to end, through the real landing tails, the real publication-ordinal +allocator and a real audit root. Nothing could do that before. +""" + +from __future__ import annotations + +import sys +import unittest +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).resolve().parents[1])) + +from tests.prism_concurrency_harness import assert_deterministic # noqa: E402 +from tests.prism_landing_harness import ( # noqa: E402 + PARENT_HASH, + LandingHarness, +) + +BLOCK_A = "aa" * 32 +BLOCK_B = "bb" * 32 + +# The top of the publish step, named by the landing itself. Every tail stops +# here having confirmed durably and having released the payout-balance +# serializer, the publication order guard and the ledger's stats +# single-flight — which is precisely the window #133 describes as open to a +# descendant. +PUBLISH_BOUNDARY = "progress:evidence-write" + + +def _two_tail_landing(harness: LandingHarness) -> dict[str, object]: + """Land A on the client tail and B on the accounting tail, interleaved. + + Block B is A's child: #133's trigger is explicit that the window is open + to descendants because A clears its accepted-block payout preview — the + child-deferral gate — before it publishes. + """ + harness.boot() + harness.break_at(PUBLISH_BOUNDARY) + block_a = harness.found_block(BLOCK_A, height=10, parent_hash=PARENT_HASH) + block_b = harness.found_block(BLOCK_B, height=11, parent_hash=BLOCK_A) + + landed_a = harness.land_on_client_tail(block_a) + landed_b = harness.land_on_accounting_tail(block_b) + + # A confirms and stops at the top of its publish step. + harness.run_until(harness.client, PUBLISH_BOUNDARY) + a_sequence = harness.publication_sequence(block_a) + floor_before_b = harness.publication_floor() + + # B lands entirely inside that window: confirm, ordinal, publish. + harness.run_until(harness.accounting, PUBLISH_BOUNDARY) + harness.drain([harness.accounting]) + b_sequence = harness.publication_sequence(block_b) + + # A resumes and publishes against a floor B has already raised. + harness.drain([harness.client, harness.accounting]) + + return { + "a_sequence": a_sequence, + "b_sequence": b_sequence, + "floor_before_b": floor_before_b, + "floor_after": harness.publication_floor(), + "a_landed": landed_a.error is None and landed_a.result is True, + "b_landed": landed_b.error is None, + "a_envelope_written": harness.envelope_written(block_a), + "b_envelope_written": harness.envelope_written(block_b), + "a_outbox": harness.outbox_state(block_a), + "b_outbox": harness.outbox_state(block_b), + "summary": harness.landing_summary(), + } + + +class TwoDistinctHashLandingTests(unittest.TestCase): + """#133: the earlier ordinal must keep its own published evidence.""" + + def test_harness_drives_the_two_distinct_hash_interleaving(self) -> None: + outcome = assert_deterministic( + self, + _two_tail_landing, + harness_factory=LandingHarness, + ).outcome + assert isinstance(outcome, dict) + + # The interleaving actually happened: A took the lower ordinal, B + # took the next one and raised the durable floor past A while A was + # still between its confirmation and its publication. + self.assertEqual(outcome["a_sequence"], 1) + self.assertEqual(outcome["b_sequence"], 2) + self.assertEqual(outcome["floor_before_b"], 1) + self.assertEqual(outcome["floor_after"], 2) + + # Both blocks land, and neither is abandoned by the race. + self.assertTrue(outcome["a_landed"]) + self.assertTrue(outcome["b_landed"]) + self.assertEqual(outcome["a_outbox"], "submitted") + self.assertEqual(outcome["b_outbox"], "submitted") + + # The assertion #133 exists for. A published at sequence 1 against a + # floor of 2, so it is superseded on the evidence pointer and never + # becomes the current reference — but its own height+hash envelope is + # its only published evidence, it competes with nothing, and it must + # be on disk. Before 03e733e this file was silently absent. + self.assertTrue(outcome["a_envelope_written"]) + self.assertTrue(outcome["b_envelope_written"]) + + def test_the_superseded_publication_is_reported_not_silent(self) -> None: + """A superseded envelope write must leave a trace an operator can find. + + The original defect's whole character was silence: no error, no log, + a permanent gap in the public audit trail. A fix that restored the + file but stayed quiet would still leave nobody able to tell the + interleaving had happened. + """ + with LandingHarness() as harness: + _two_tail_landing(harness) + # Audit-path diagnostics go to stderr: worker processes reserve + # stdout for a strict JSON-lines protocol. + self.assertIn( + "wrote live envelope for superseded publication", + harness.errors, + ) + self.assertIn("sequence=1 floor_sequence=2", harness.errors) + + def test_restore_authority_is_gated_on_a_fresh_lease_proof(self) -> None: + """Withheld authority degrades to the old behaviour, not to a failure. + + Only the live writer for a landing may mutate the shared audit root. + A process whose writer lease was taken over inside the confirm→publish + gap reaches the same call site with stale reads behind it, and + publishing a stale report as a block's public evidence is worse than + leaving the envelope missing. So the restore is gated — and when the + gate refuses, the publication must still succeed, because failing a + found-block landing over a lease hiccup would be a live regression. + """ + + def scenario(harness: LandingHarness) -> dict[str, object]: + harness.withheld_lease_components = { + "superseded_audit_envelope_restore" + } + outcome = _two_tail_landing(harness) + return { + "a_envelope_written": outcome["a_envelope_written"], + "b_envelope_written": outcome["b_envelope_written"], + "a_landed": outcome["a_landed"], + "b_landed": outcome["b_landed"], + } + + outcome = assert_deterministic( + self, + scenario, + harness_factory=LandingHarness, + ).outcome + assert isinstance(outcome, dict) + # The landing still succeeds on both tails. + self.assertTrue(outcome["a_landed"]) + self.assertTrue(outcome["b_landed"]) + # B is unaffected: it is not superseded, so it never asks. + self.assertTrue(outcome["b_envelope_written"]) + # A degrades to the pre-fix behaviour rather than failing. + self.assertFalse(outcome["a_envelope_written"]) + + def test_the_disposition_lease_does_not_serialise_distinct_hashes(self) -> None: + """The topology the defect needs, asserted rather than assumed. + + If the per-hash disposition lease did serialise across hashes, this + whole scenario would be unreachable and the tests above would be + proving nothing. The claim is load-bearing, so it is checked: with A + holding its own disposition and parked mid-landing, B's tail claims + its disposition and completes a full landing. + """ + with LandingHarness() as harness: + harness.boot() + harness.break_at(PUBLISH_BOUNDARY) + block_a = harness.found_block(BLOCK_A, height=10, parent_hash=PARENT_HASH) + block_b = harness.found_block(BLOCK_B, height=11, parent_hash=BLOCK_A) + harness.land_on_client_tail(block_a) + landed_b = harness.land_on_accounting_tail(block_b) + + harness.run_until(harness.client, PUBLISH_BOUNDARY) + self.assertIsNone(harness.publication_sequence(block_b)) + + # A is mid-landing and holds its hash's disposition. B proceeds + # anyway, all the way through its own durable confirmation. + harness.run_until(harness.accounting, PUBLISH_BOUNDARY) + harness.drain([harness.accounting]) + self.assertIsNone(landed_b.error) + self.assertEqual(harness.publication_sequence(block_b), 2) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_prism_lease_heartbeat_tail_latency.py b/tests/test_prism_lease_heartbeat_tail_latency.py new file mode 100644 index 000000000..871a5dad5 --- /dev/null +++ b/tests/test_prism_lease_heartbeat_tail_latency.py @@ -0,0 +1,978 @@ +#!/usr/bin/env python3 +"""False lease-heartbeat exits under normal tail latency (issue #212). + +What could not be tested before +------------------------------- +``union-mainnet`` restarted 69 times during one rapid-block burst while +PostgreSQL, qbitd, the RAID array, CPU, memory and disk were all healthy and +the coordinator still held its PostgreSQL advisory guard. The exits were +real protective exits: the heartbeat's server-proven envelope had genuinely +elapsed. It had elapsed because the envelope was sized for an idle interval +and not for the tail of a lawful verification. + +Reproducing that needs three things at once — a guard statement running near +its server-side timeout, the coordinator's own threads being descheduled +behind the burst, and a fenced write holding the lease tuple — and needs them +in a fixed order. Running the coordinator and hoping produces the schedule +by accident at best. This module puts the real +:class:`LedgerLeaseHeartbeatService` loops on the #128 deterministic harness: +its heartbeat and monitor become two harness actors, every interval they +measure reads the harness's virtual clock, and modelled server time is +charged per statement. Nothing sleeps. + +What is proved +-------------- +``RapidBlockBurstTests`` + A production-shaped burst — near-bound guard statements, repeated + scheduler stalls, an in-flight fenced write on the lease tuple, and + external-effect fence callers competing for the guard's query slot — + produces no hard exit, while the peak staleness it reaches is above + what the pre-#212 policy tolerated. The regression is anchored to the + old bounds, so a future retune that reintroduces them fails here. + +``ProofRenewalSplitTests`` + The frequent statement is the cheap ownership proof, renewal escalates + on the same beat the committed row enters the authority margin, and + neither ever queues on the lease tuple's row lock. + +``FailClosedTests`` + The safety half. A lost guard session, a wedged proof that never + answers, and a guarded statement cancelled by its own statement timeout + each terminate this coordinator's authority strictly before a successor + becomes eligible to adopt — measured against the successor's real + guard-acquisition edge, not against a constant. +""" + +from __future__ import annotations + +from typing import Callable +import unittest + +from lab.prism.background_services import ( + LedgerLeaseHeartbeatPorts, + LedgerLeaseHeartbeatService, +) +from lab.prism.writer_lease_timing import ( + DEFAULT_WRITER_LEASE_HEARTBEAT_POLICY, + LEASE_HEARTBEAT_MODE_FENCE, + WriterLeaseHeartbeatPolicy, + WriterLeaseVerificationAttempt, +) +from tests.prism_concurrency_harness import ( + GUARD_STATEMENT_TIMEOUT_SECONDS, + LeaseHarness, +) +from tests.test_prism_writer_lease_policy import PRE_212_POLICY + +# The shipped policy, used verbatim: these scenarios exist to judge the +# defaults operators actually run, not a scaled-down stand-in. +POLICY = DEFAULT_WRITER_LEASE_HEARTBEAT_POLICY + +# Guard statements run just inside their server-side ceiling — the tail the +# burst produced, not a pathological value. Anything above the ceiling is +# cancelled by PostgreSQL and is a different scenario (see FailClosedTests). +TAIL_STATEMENT_SECONDS = 0.45 + +# One descheduling of the coordinator's threads behind the burst. Well inside +# the policy's scheduler-slack allowance and far outside what the pre-#212 +# envelope could absorb. +SCHEDULER_STALL_SECONDS = 0.35 + +# Virtual seconds of burst. Long enough to cross the own-write authority +# margin (half the 60s lease TTL) and prove the renewal escalation fires on +# its own, rather than only on the first beat. +BURST_SECONDS = 40.0 + +LEASE_TTL_SECONDS = 60.0 + +# A fenced write's caller deadline. Arming one is what makes the ledger wrap +# the statement in an explicit transaction whose COMMIT is a separate client +# message, which is how the lease tuple stays row-locked across the burst. +FENCED_WRITE_BUDGET_SECONDS = 600.0 + + +class HeartbeatUnderTest: + """The real heartbeat service, wired onto the deterministic harness. + + Substitution goes through ports, exactly as the harness does for the + ledger: the loops, the fail-closed bookkeeping, the phase attribution + and the monitor's arithmetic are the shipped implementations. Only the + clock, the synchronisation primitives and the process-exit seam are + supplied by the harness, so what a scenario observes is what production + would do on the same schedule. + """ + + def __init__( + self, + harness: LeaseHarness, + coordinator, + *, + policy: WriterLeaseHeartbeatPolicy = POLICY, + name: str = "hb", + ) -> None: + self.harness = harness + self.policy = policy + self.exits: list[tuple[float, str, str | None]] = [] + service: LedgerLeaseHeartbeatService + + def lease_hard_exit(message: str, *, include_traceback: bool) -> None: + # The coordinator's delegate, spelled the same way: the real + # fail-closed bookkeeping (single-arming, exit-thread exemption, + # failure reason) has to run for the assertions to mean anything. + service.hard_exit(message, include_traceback=include_traceback) + + def watchdog_hard_exit(reason: str, *, timeout_seconds: float) -> None: + # Stands in for os._exit: record when the process would have + # gone, and stop both loops, because a dead process runs nothing. + self.exits.append( + (harness.clock.monotonic(), reason, service.failure_reason) + ) + service.stop_event.set() + + ports = LedgerLeaseHeartbeatPorts( + ledger=lambda: coordinator.ledger, + heartbeat_seconds=lambda: policy.heartbeat_interval_seconds, + failure_seconds=lambda: policy.failure_budget_seconds, + monitor_seconds=lambda: policy.monitor_interval_seconds, + exit_timeout_seconds=lambda: policy.exit_margin_seconds, + scheduler_slack_seconds=lambda: policy.scheduler_slack_seconds, + external_fence_timeout_seconds=lambda: 0.5, + lease_hard_exit=lease_hard_exit, + watchdog_hard_exit=watchdog_hard_exit, + heartbeat_loop=lambda: service.heartbeat_loop(), + monitor_loop=lambda: service.monitor_loop(), + monotonic=harness.clock.monotonic, + ) + service = LedgerLeaseHeartbeatService(ports) + # The freshness and failure locks stay real: the service never + # yields while holding either, and under the baton scheduler no two + # actors run at once, so neither can be contended. The stop/ready/ + # failed events do have to be scheduler-visible, because the loops + # wait on them with a timeout and that wait is what makes the + # heartbeat interval and the monitor poll observable. + service.stop_event = harness.event(f"{name}.stop") + service.ready = harness.event(f"{name}.ready") + service.failed = harness.event(f"{name}.failed") + armed = harness.clock.monotonic() + service.last_success_monotonic = armed + service.last_server_proven_monotonic = armed + self.service = service + self.armed_monotonic = armed + self.heartbeat = harness.scheduler.actor(f"{name}-heartbeat") + self.monitor = harness.scheduler.actor(f"{name}-monitor") + self.heartbeat.submit(service.heartbeat_loop, label="heartbeat-loop") + self.monitor.submit(service.monitor_loop, label="monitor-loop") + self.peak_server_proven_age = 0.0 + self.peak_activity_age = 0.0 + + # -- observation ------------------------------------------------------- + + @property + def exited(self) -> bool: + return bool(self.exits) + + @property + def exit_monotonic(self) -> float: + if not self.exits: + raise AssertionError("the heartbeat did not hard-exit") + return self.exits[0][0] + + @property + def exit_reason(self) -> str: + if not self.exits: + raise AssertionError("the heartbeat did not hard-exit") + return self.exits[0][2] or "" + + def observe(self) -> None: + """Sample staleness the way the monitor sees it, for peak reporting.""" + self.peak_server_proven_age = max( + self.peak_server_proven_age, + self.service.server_proven_age_seconds(), + ) + self.peak_activity_age = max( + self.peak_activity_age, + self.service.activity_age_seconds(), + ) + + # -- driving ----------------------------------------------------------- + + def fence_call(self, ledger) -> Callable[[], object]: + """The external-effect fence's guard call, minus its worker thread. + + ``require_fresh_lease_for_external_side_effect`` runs the + verification on a daemon thread the baton scheduler cannot drive, + so a scenario calls the same shipped verification with the same two + callbacks wired the same way. What is under test through this seam + is the guard's single query slot and the marks a fence leaves + behind, both of which are identical either way — and leaving the + marks out would model a guard caller that does not exist. + """ + attempt = WriterLeaseVerificationAttempt( + LEASE_HEARTBEAT_MODE_FENCE, + monotonic=self.harness.clock.monotonic, + ) + + def run() -> object: + return ledger.verify_writer_lease_guard_session( + on_query_start=attempt.slot_acquired, + on_statement_end=lambda: self.service.record_server_proven( + attempt.statement_completed() + ), + ) + + return run + + def advance_until_beat(self, others: list) -> None: + """Jump forward until the heartbeat's next beat is due. + + Other actors keep their own turns as their timers come up, so the + heartbeat is not privileged by the fast-forward. + """ + while not self.heartbeat.runnable(): + if self.harness.advance_to_next_deadline() is None: + raise AssertionError("nothing bounds the heartbeat's idle wait") + for actor in others: + while actor.runnable(): + self.harness.step(actor) + + def run_monitor_maximally_late( + self, + *, + rounds: int, + first_poll_monotonic: float | None = None, + ) -> None: + """Drive only the monitor, each poll as late as the policy allows. + + The worst schedule the safety inequality claims to survive: the + heartbeat thread gets no turn at all, and the monitor — the thread + that has to notice and exit — wakes ``monitor_interval + + scheduler_slack`` after its previous poll every single time. + + ``first_poll_monotonic`` pins the phase of that grid. Lateness + alone is not the worst case: a poll that happens to land just after + a bound elapses notices immediately, and the scenario then proves + nothing about the term being tested. Placing one poll immediately + *before* the bound forces the next one to arrive a full late + interval after it, which is the case the envelope has to cover. + """ + late_by = ( + self.policy.monitor_interval_seconds + + self.policy.scheduler_slack_seconds + ) + if first_poll_monotonic is not None: + self.harness.clock.advance_to( + max(first_poll_monotonic, self.harness.clock.monotonic()) + ) + while self.monitor.runnable(): + self.harness.step(self.monitor) + self.observe() + for _ in range(rounds): + if self.exited: + return + self.harness.clock.advance(late_by) + while self.monitor.runnable(): + self.harness.step(self.monitor) + self.observe() + + def stall_heartbeat(self, seconds: float) -> None: + """Deschedule the coordinator's heartbeat thread for ``seconds``. + + Only the monitor runs, in its own poll steps, which is what a GIL + stall behind a rapid-block burst looks like from the monitor's side: + time passes, no new activity mark appears, and the decision to + hard-exit or not is taken on the policy alone. + """ + target = self.harness.clock.monotonic() + seconds + while self.harness.clock.monotonic() < target and not self.exited: + self.harness.clock.advance_to( + min( + target, + self.harness.clock.monotonic() + + self.policy.monitor_interval_seconds, + ) + ) + while self.monitor.runnable(): + self.harness.step(self.monitor) + self.observe() + + def run( + self, + actors: list, + *, + seconds: float, + stall_every: int = 0, + stall_seconds: float = SCHEDULER_STALL_SECONDS, + limit: int = 8000, + ) -> None: + """Drain every runnable actor, then jump to the next pending timer.""" + started = self.harness.clock.monotonic() + rounds = 0 + while ( + self.harness.clock.monotonic() - started < seconds + and rounds < limit + and not self.exited + ): + self.harness.drain(actors) + self.observe() + if stall_every and rounds % stall_every == stall_every - 1: + self.stall_heartbeat(stall_seconds) + if self.harness.advance_to_next_deadline() is None: + break + rounds += 1 + + def quiesce(self, actors: list) -> None: + self.service.stop_event.set() + self.harness.drain(actors) + + +def guard_statement_latency(seconds: float): + """Charge ``seconds`` of modelled server time to each guarded statement.""" + + def latency(statement) -> float: + return seconds if statement.kind.value in {"prove", "verify"} else 0.0 + + return latency + + +def statement_counts(harness: LeaseHarness) -> dict[str, int]: + kinds = harness.statement_kinds() + return {kind: kinds.count(kind) for kind in set(kinds)} + + +class RapidBlockBurstTests(unittest.TestCase): + def test_burst_with_tail_latency_and_stalls_never_hard_exits(self) -> None: + """The regression, on the schedule that produced it. + + Guard statements run just under their server-side ceiling, the + coordinator's threads are descheduled repeatedly, a fenced write + holds the lease tuple for the whole burst, and external-effect + fences compete for the guard's single query slot. Nothing here is + a fault; every restart the burst produced was a healthy + coordinator being told its own session was gone. + """ + with LeaseHarness( + lease_ttl_seconds=LEASE_TTL_SECONDS, + capture_output=True, + ) as harness: + alpha = harness.coordinator("alpha") + alpha.start() + harness.run_until(alpha, "done:startup") + heartbeat = HeartbeatUnderTest(harness, alpha) + harness.server.statement_latency_seconds = guard_statement_latency( + TAIL_STATEMENT_SECONDS + ) + + # A long fenced write parked mid-statement: PostgreSQL holds the + # lease tuple's row lock and the COMMIT has not been sent. This + # is the shape persist_accepted_block leaves behind. + def fenced_write() -> object: + with alpha.ledger.operation_timeout(FENCED_WRITE_BUDGET_SECONDS): + return alpha.ledger.renew_writer_lease() + + alpha.submit(fenced_write, label="fenced-write") + harness.run_until(alpha, "alpha.done:renew") + + fence = harness.scheduler.actor("external-fence") + actors = [heartbeat.heartbeat, heartbeat.monitor, fence] + heartbeat.run( + actors, + seconds=BURST_SECONDS, + stall_every=7, + ) + heartbeat.quiesce(actors) + + self.assertFalse( + heartbeat.exited, + f"healthy coordinator hard-exited: {heartbeat.exits}", + ) + # The burst really did produce the staleness the issue reported: + # both pre-#212 bounds are crossed by a coordinator that never + # stopped holding its guard. + self.assertGreaterEqual( + heartbeat.peak_server_proven_age, + PRE_212_POLICY.server_proven_cap_seconds, + ) + self.assertLess( + heartbeat.peak_server_proven_age, + POLICY.server_proven_cap_seconds, + ) + self.assertLess( + heartbeat.peak_activity_age, + POLICY.failure_budget_seconds, + ) + # The lease tuple was locked throughout, and no guarded + # statement ever queued for it. + self.assertIsNotNone(harness.server.lease_lock_holder) + self.assertEqual( + [stop for stop in harness.trace if "guard" in stop and "lockwait" in stop], + [], + ) + + def test_monitor_lateness_is_attributed_and_survived(self) -> None: + """A stalled monitor thread spends envelope like a slow statement. + + The monitor decides the exit, so its own lateness has to be + measurable separately from database latency — otherwise an operator + reading an exit cannot tell "PostgreSQL was slow" from "this + process could not get scheduled". + """ + with LeaseHarness(lease_ttl_seconds=LEASE_TTL_SECONDS) as harness: + alpha = harness.coordinator("alpha") + alpha.start() + harness.run_until(alpha, "done:startup") + heartbeat = HeartbeatUnderTest(harness, alpha) + actors = [heartbeat.heartbeat, heartbeat.monitor] + heartbeat.run(actors, seconds=1.0) + + # Deschedule the monitor alone: the heartbeat keeps proving. + late_by = 0.4 + harness.advance(heartbeat.policy.monitor_interval_seconds + late_by) + while heartbeat.monitor.runnable(): + harness.step(heartbeat.monitor) + heartbeat.run(actors, seconds=1.0) + heartbeat.quiesce(actors) + + self.assertFalse(heartbeat.exited) + self.assertGreaterEqual( + heartbeat.service.monitor_wake_delay_seconds, + late_by, + ) + snapshot = heartbeat.service.snapshot() + self.assertGreaterEqual( + float(snapshot["monitor_wake_delay_seconds"]), + late_by, + ) + + +class ProofRenewalSplitTests(unittest.TestCase): + def test_beats_prove_ownership_and_escalate_only_when_renewal_is_due( + self, + ) -> None: + """Frequent ownership proof, infrequent TTL renewal — same guarantees. + + The first beat renews, because startup readiness must prove the + renewing path works. After that every beat asks the cheap ownership + question until the committed row falls inside the own-write + authority margin (half the 60s TTL), at which point renewal + escalates on that same beat. + """ + with LeaseHarness(lease_ttl_seconds=LEASE_TTL_SECONDS) as harness: + alpha = harness.coordinator("alpha") + alpha.start() + harness.run_until(alpha, "done:startup") + heartbeat = HeartbeatUnderTest(harness, alpha) + harness.server.statement_latency_seconds = guard_statement_latency( + TAIL_STATEMENT_SECONDS + ) + actors = [heartbeat.heartbeat, heartbeat.monitor] + heartbeat.run(actors, seconds=BURST_SECONDS) + heartbeat.quiesce(actors) + + counts = statement_counts(harness) + self.assertFalse(heartbeat.exited) + # Proof dominates by a wide margin: renewal runs on the first + # beat and once more when half the TTL has eroded. + self.assertGreater(counts["prove"], 20) + self.assertEqual(counts["verify"], 2) + self.assertGreater(counts["prove"], 10 * counts["verify"]) + + attempts = heartbeat.service.attempt_counts + outcomes = heartbeat.service.outcome_counts + self.assertEqual(attempts["renew"], counts["verify"]) + self.assertEqual(attempts["proof"], counts["prove"]) + # Exactly one proof reported the row renewal-due; that beat + # escalated immediately rather than waiting for the next one. + self.assertEqual(outcomes["renewal_due"], 1) + self.assertEqual(outcomes["renewed"], counts["verify"]) + self.assertEqual(outcomes["failed"], 0) + + def test_proof_never_queues_behind_an_in_flight_fenced_write(self) -> None: + """The block-39416 property, preserved across the split. + + The proof takes no row lock at all — it asks two ``EXISTS`` + questions — so a fenced write holding the lease tuple for the whole + of its transaction cannot make a beat queue and die on the guard's + statement timeout. + """ + with LeaseHarness(lease_ttl_seconds=LEASE_TTL_SECONDS) as harness: + alpha = harness.coordinator("alpha") + alpha.start() + harness.run_until(alpha, "done:startup") + + def fenced_write() -> object: + with alpha.ledger.operation_timeout(FENCED_WRITE_BUDGET_SECONDS): + return alpha.ledger.renew_writer_lease() + + alpha.submit(fenced_write, label="fenced-write") + harness.run_until(alpha, "alpha.done:renew") + self.assertIsNotNone(harness.server.lease_lock_holder) + + heartbeat = HeartbeatUnderTest(harness, alpha) + actors = [heartbeat.heartbeat, heartbeat.monitor] + heartbeat.run(actors, seconds=2.0) + heartbeat.quiesce(actors) + + self.assertFalse(heartbeat.exited) + self.assertGreater(statement_counts(harness)["prove"], 1) + self.assertEqual( + [stop for stop in harness.trace if "lockwait" in stop], + [], + ) + + def test_guard_slot_contention_is_attributed_to_queue_wait(self) -> None: + """A beat that queued behind a fence must not look like slow SQL. + + The guard session is one connection with one query slot, so an + external-effect fence and the heartbeat serialize. Charging that + wait to the database would send an operator to PostgreSQL for a + problem that is in this process. + """ + with LeaseHarness(lease_ttl_seconds=LEASE_TTL_SECONDS) as harness: + alpha = harness.coordinator("alpha") + alpha.start() + harness.run_until(alpha, "done:startup") + heartbeat = HeartbeatUnderTest(harness, alpha) + harness.server.statement_latency_seconds = guard_statement_latency( + TAIL_STATEMENT_SECONDS + ) + actors = [heartbeat.heartbeat, heartbeat.monitor] + heartbeat.run(actors, seconds=1.0) + + # Park a fence caller inside the guard's slot, then let the + # heartbeat's next beat arrive behind it. + fence = harness.scheduler.actor("external-fence") + fence.submit( + heartbeat.fence_call(alpha.ledger), + label="submitblock-fence", + ) + harness.run_until(fence, "alpha.guard.begin:verify") + heartbeat.advance_until_beat([heartbeat.monitor]) + harness.run_until_blocked( + heartbeat.heartbeat, + "alpha.guard.slot:wait", + ) + # Only now does the fence's statement spend its server time, so + # every second of it is queue wait for the heartbeat. + harness.drain([fence] + actors) + heartbeat.quiesce(actors + [fence]) + + self.assertFalse(heartbeat.exited) + queued = [ + phases + for phases in (heartbeat.service.worst_phases,) + if phases.slot_wait_seconds > 0.0 + ] + self.assertTrue( + queued, + "a beat queued behind the fence but recorded no slot wait", + ) + self.assertGreaterEqual( + heartbeat.service.worst_phases.slot_wait_seconds, + TAIL_STATEMENT_SECONDS, + ) + + +class FailClosedTests(unittest.TestCase): + """Losing authority must still be fatal, and fatal early enough.""" + + def takeover_deadline(self, harness: LeaseHarness, successor) -> float: + """Earliest virtual instant the successor may compare-and-swap. + + Measured from the instant the successor actually took the writer + advisory guard, because that is the edge + ``_writer_lease_adoption_wait_seconds`` counts its silence from — + deliberately, so a predecessor that just lost the guard always gets + its whole self-fencing budget however stale its lease row looks. + """ + successor.start() + return self.step_until_guard_held(harness, successor) + + def step_until_guard_held(self, harness: LeaseHarness, successor) -> float: + for _ in range(50): + if successor.guard is not None and successor.guard.held: + return harness.clock.monotonic() + POLICY.adoption_silence_seconds + if not successor.actor.runnable(): + break + harness.step(successor.actor) + raise AssertionError("the successor never acquired the writer guard") + + def assert_guard_withheld(self, harness: LeaseHarness, successor) -> None: + """A successor cannot even start its silence while we hold the guard. + + This is the first fence, ahead of any timing argument: the advisory + lock is a session lock, so a predecessor that is wedged — rather + than dead — keeps it, and the successor's adoption clock has not + begun. + """ + successor.start() + for _ in range(20): + if not successor.actor.runnable(): + break + harness.step(successor.actor) + self.assertFalse( + successor.guard is not None and successor.guard.held, + "a successor took the writer guard while the predecessor held it", + ) + + def test_lost_guard_session_exits_before_takeover_eligibility(self) -> None: + """An injected guard-session loss fences the old coordinator first.""" + with LeaseHarness(lease_ttl_seconds=LEASE_TTL_SECONDS) as harness: + alpha = harness.coordinator("alpha") + alpha.start() + harness.run_until(alpha, "done:startup") + heartbeat = HeartbeatUnderTest(harness, alpha) + harness.server.statement_latency_seconds = guard_statement_latency( + TAIL_STATEMENT_SECONDS + ) + actors = [heartbeat.heartbeat, heartbeat.monitor] + heartbeat.run(actors, seconds=1.0) + self.assertFalse(heartbeat.exited) + + # The guard connection drops: the advisory lock is released + # server-side and this coordinator no longer owns anything. + assert alpha.guard is not None + alpha.guard.close() + lost_at = harness.clock.monotonic() + + beta = harness.coordinator("beta") + deadline = self.takeover_deadline(harness, beta) + heartbeat.run( + actors + [beta.actor], + seconds=POLICY.adoption_silence_seconds, + ) + + self.assertTrue(heartbeat.exited, "a lost guard did not fence") + self.assertLess(heartbeat.exit_monotonic, deadline) + self.assertLess( + heartbeat.exit_monotonic - lost_at, + POLICY.adoption_silence_seconds, + ) + self.assertIn("ledger lease heartbeat failed", heartbeat.exit_reason) + # The exit names the phase breakdown of the attempt that failed. + self.assertIn("mode=proof", heartbeat.exit_reason) + self.assertIn("outcome=failed", heartbeat.exit_reason) + self.assertEqual(heartbeat.service.outcome_counts["failed"], 1) + + def test_wedged_proof_exits_before_takeover_eligibility(self) -> None: + """A statement that never answers still terminates authority. + + A wedged beat stamps nothing at all after its attempt start, so the + liveness budget is the bound that fires first and the server-proven + edge — which no client-side mark can move — is stale behind it. The + exit has to land inside the adoption silence, and the reason has to + say which of the two bounds decided it. + """ + with LeaseHarness(lease_ttl_seconds=LEASE_TTL_SECONDS) as harness: + alpha = harness.coordinator("alpha") + alpha.start() + harness.run_until(alpha, "done:startup") + heartbeat = HeartbeatUnderTest(harness, alpha) + actors = [heartbeat.heartbeat, heartbeat.monitor] + heartbeat.run(actors, seconds=1.0) + wedged_at = harness.clock.monotonic() + + def wedge(statement) -> float: + if statement.kind.value in {"prove", "verify"}: + # No answer, no error: the beat is abandoned mid + # statement exactly as a black-holed session leaves it. + harness.scheduler.park_forever("alpha.guard.wedged") + return 0.0 + + harness.server.statement_latency_seconds = wedge + + beta = harness.coordinator("beta") + # A wedged coordinator still holds the session advisory lock, so + # the successor cannot even begin its silence window yet. + self.assert_guard_withheld(harness, beta) + heartbeat.run( + actors + [beta.actor], + seconds=POLICY.adoption_silence_seconds, + ) + + self.assertTrue(heartbeat.exited, "a wedged proof did not fence") + self.assertLess( + heartbeat.exit_monotonic - wedged_at, + POLICY.adoption_silence_seconds, + ) + # The process is gone, which is what finally drops the guard. + # Only then does the successor's full silence window start, so + # the old coordinator's authority ended strictly first. + assert alpha.guard is not None + alpha.guard.close() + deadline = self.step_until_guard_held(harness, beta) + self.assertLess(heartbeat.exit_monotonic, deadline) + reason = heartbeat.exit_reason + self.assertIn("stopped making progress", reason) + # The server-proven edge is the bound that decides a wedge: it + # is a whole idle interval staler than the client-side activity + # mark the abandoned beat left behind, so it reaches its cap + # first. That ordering is the point of measuring the adoption + # envelope from completed round trips. + self.assertIn("Tripped: server-proven", reason) + self.assertIn("adoption envelope cap", reason) + self.assertGreater( + heartbeat.service.server_proven_age_seconds(), + heartbeat.service.activity_age_seconds(), + ) + # Every term of the policy that produced the decision is in the + # exit, so an operator never has to guess which bound tripped. + self.assertIn("Last attempt:", reason) + self.assertIn("Monitor wake delay:", reason) + self.assertIn("server_proven_cap=", reason) + + def test_statement_cancelled_by_its_own_timeout_fails_closed(self) -> None: + """A guarded statement PostgreSQL cancels is loss, not latency. + + The guard session's statement_timeout is the ceiling the whole + envelope is sized against. A statement that reaches it produces an + error, and an error on the guarded session is loss of the guarded + session — never something to retry into the next beat. + """ + with LeaseHarness(lease_ttl_seconds=LEASE_TTL_SECONDS) as harness: + alpha = harness.coordinator("alpha") + alpha.start() + harness.run_until(alpha, "done:startup") + heartbeat = HeartbeatUnderTest(harness, alpha) + actors = [heartbeat.heartbeat, heartbeat.monitor] + heartbeat.run(actors, seconds=1.0) + self.assertFalse(heartbeat.exited) + + harness.server.statement_latency_seconds = guard_statement_latency( + GUARD_STATEMENT_TIMEOUT_SECONDS * 2 + ) + beta = harness.coordinator("beta") + self.assert_guard_withheld(harness, beta) + heartbeat.run( + actors + [beta.actor], + seconds=POLICY.adoption_silence_seconds, + ) + + self.assertTrue(heartbeat.exited) + self.assertIn("ledger lease heartbeat failed", heartbeat.exit_reason) + assert alpha.guard is not None + alpha.guard.close() + self.assertLess( + heartbeat.exit_monotonic, + self.step_until_guard_held(harness, beta), + ) + + def assert_exit_precedes_eligibility( + self, + harness: LeaseHarness, + heartbeat: HeartbeatUnderTest, + alpha, + successor, + *, + authority_lost_monotonic: float, + ) -> None: + """The whole safety claim, measured rather than assumed. + + Three things have to hold together: the old coordinator exits, it + exits within one adoption silence of losing authority, and the + successor's own eligibility edge — its real guard acquisition plus + the silence — is strictly later than that exit. + """ + self.assertTrue(heartbeat.exited, "authority was lost without an exit") + # The recorded instant is when the monitor *decided*; the process is + # gone at most one exit margin later, and that is what has to beat + # the adoption edge. + self.assertLess( + heartbeat.exit_monotonic + + POLICY.exit_margin_seconds + - authority_lost_monotonic, + POLICY.adoption_silence_seconds, + ) + if alpha.guard is not None and alpha.guard.held: + # The process is gone; that is what finally drops the session + # advisory lock and lets any successor start its silence. + alpha.guard.close() + eligible_at = self.step_until_guard_held(harness, successor) + self.assertLess( + heartbeat.exit_monotonic + POLICY.exit_margin_seconds, + eligible_at, + ) + + def test_wedged_proof_and_maximally_late_monitor_still_exit_in_time( + self, + ) -> None: + """The combined worst case the exit envelope is sized for. + + Authority is gone (the guard session answers nothing), the thread + that would notice by raising is descheduled entirely, and the + monitor — the only thread left that can act — wakes as late as the + policy's scheduler slack permits on every poll. This is the + schedule that made the first cut of the envelope unsafe: reserving + only the poll intervals, a maximally late monitor observed the + stale edge after the successor's adoption edge. + """ + with LeaseHarness(lease_ttl_seconds=LEASE_TTL_SECONDS) as harness: + alpha = harness.coordinator("alpha") + alpha.start() + harness.run_until(alpha, "done:startup") + heartbeat = HeartbeatUnderTest(harness, alpha) + # Beats run their statements at the guard's server-side ceiling. + # That maximises the gap between the newest completed round trip + # and the client-side marks the next beat leaves before it + # wedges, which is exactly when the envelope cap — not the + # liveness budget — is the bound that has to carry the + # guarantee. + harness.server.statement_latency_seconds = guard_statement_latency( + GUARD_STATEMENT_TIMEOUT_SECONDS + ) + actors = [heartbeat.heartbeat, heartbeat.monitor] + heartbeat.run(actors, seconds=2.0) + self.assertFalse(heartbeat.exited) + + # The last instant this coordinator's authority was provable. + # The guarded session cannot have died before it, so a + # successor's guard acquisition — and therefore its silence + # window — cannot start earlier either. Measuring from here is + # conservative in the direction that matters. + proven_at = float(heartbeat.service.last_server_proven_monotonic) + + def wedge(statement) -> float: + if statement.kind.value in {"prove", "verify"}: + harness.scheduler.park_forever("alpha.guard.wedged") + return 0.0 + + harness.server.statement_latency_seconds = wedge + # Let one beat reach the wedge — leaving its attempt-start and + # slot-acquisition marks behind, which is what makes the + # client-side liveness clock look healthier than the session + # is — then abandon the heartbeat thread: from here only the + # monitor can save the deployment. + heartbeat.advance_until_beat([heartbeat.monitor]) + while heartbeat.heartbeat.runnable(): + harness.step(heartbeat.heartbeat) + + beta = harness.coordinator("beta") + self.assert_guard_withheld(harness, beta) + # Worst phase: one poll lands a hair before the cap elapses, so + # the poll that can actually notice is a full late interval + # after it. + heartbeat.run_monitor_maximally_late( + rounds=200, + first_poll_monotonic=( + proven_at + POLICY.server_proven_cap_seconds - 0.001 + ), + ) + + # A precondition, not a bound: the point is that the monitor + # really was at least as late as the policy budgets for, so a + # lower bound is what this checks. + self.assertGreaterEqual( + heartbeat.service.monitor_wake_delay_seconds, + POLICY.scheduler_slack_seconds - 1e-6, + "the monitor was not actually driven at maximum lateness", + ) + self.assertTrue(heartbeat.exited, "a wedged proof did not fence") + # The whole safety claim, at the worst schedule the policy + # tolerates: gone before the silence window — which starts no + # earlier than the last provable instant — can elapse. Asserted + # before anything about wording, so a regression in the envelope + # reports the breach rather than a changed message. + self.assertLess( + heartbeat.exit_monotonic + + POLICY.exit_margin_seconds + - proven_at, + POLICY.adoption_silence_seconds, + "the old coordinator was still exiting after a successor " + "became eligible to adopt", + ) + # The envelope cap is the bound that fired, not the liveness + # budget the fresh client marks were still inside. + self.assertIn("Tripped: server-proven", heartbeat.exit_reason) + self.assert_exit_precedes_eligibility( + harness, + heartbeat, + alpha, + beta, + authority_lost_monotonic=proven_at, + ) + + def test_lost_guard_with_stalled_heartbeat_and_late_monitor_exits_in_time( + self, + ) -> None: + """The same worst case, entered through guard-session loss. + + The advisory lock is genuinely gone, so this coordinator has no + authority at all — and its heartbeat thread, which would normally + raise on the next beat, never gets scheduled again. The monitor + has to carry the exit alone, maximally late, and still finish + before the successor that can now take the guard becomes eligible. + """ + with LeaseHarness(lease_ttl_seconds=LEASE_TTL_SECONDS) as harness: + alpha = harness.coordinator("alpha") + alpha.start() + harness.run_until(alpha, "done:startup") + heartbeat = HeartbeatUnderTest(harness, alpha) + actors = [heartbeat.heartbeat, heartbeat.monitor] + heartbeat.run(actors, seconds=1.0) + self.assertFalse(heartbeat.exited) + + assert alpha.guard is not None + alpha.guard.close() + authority_lost = harness.clock.monotonic() + + # The successor may take the guard immediately — the silence + # window it must then wait out is the only thing standing + # between it and the lease row. + beta = harness.coordinator("beta") + eligible_at = self.takeover_deadline(harness, beta) + + # From here the heartbeat thread never runs again. + heartbeat.run_monitor_maximally_late(rounds=200) + + self.assertTrue(heartbeat.exited, "a lost guard did not fence") + self.assertGreaterEqual( + heartbeat.service.monitor_wake_delay_seconds, + POLICY.scheduler_slack_seconds - 1e-6, + ) + self.assertLess( + heartbeat.exit_monotonic + + POLICY.exit_margin_seconds + - authority_lost, + POLICY.adoption_silence_seconds, + ) + self.assertLess( + heartbeat.exit_monotonic + POLICY.exit_margin_seconds, + eligible_at, + ) + + def test_unsafe_policy_refuses_to_arm_the_heartbeat(self) -> None: + """Startup rejection, on the real service. + + A policy whose failure budget cannot fit inside the adoption + silence alongside the exit envelope is a double-writer hazard. The + service must refuse to start rather than run with a broken + exit-before-adoption argument. + """ + with LeaseHarness(lease_ttl_seconds=LEASE_TTL_SECONDS) as harness: + alpha = harness.coordinator("alpha") + alpha.start() + harness.run_until(alpha, "done:startup") + unsafe = WriterLeaseHeartbeatPolicy( + adoption_silence_seconds=POLICY.adoption_silence_seconds, + heartbeat_interval_seconds=0.25, + failure_budget_seconds=POLICY.adoption_silence_seconds, + monitor_interval_seconds=0.05, + exit_margin_seconds=0.1, + ) + heartbeat = HeartbeatUnderTest(harness, alpha, policy=unsafe) + starter = harness.scheduler.actor("serve") + started = starter.submit(heartbeat.service.start, label="start") + harness.run_until(starter, "done:start") + + self.assertIsNone(started.value()) + self.assertTrue(heartbeat.exited) + self.assertIn( + "refusing to start the ledger lease heartbeat", + heartbeat.exit_reason, + ) + self.assertIn("adoption silence", heartbeat.exit_reason) + # Nothing was armed: no guarded statement ran. + self.assertNotIn("prove", statement_counts(harness)) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_prism_lease_heartbeat_under_fenced_write.py b/tests/test_prism_lease_heartbeat_under_fenced_write.py new file mode 100644 index 000000000..5f94c0357 --- /dev/null +++ b/tests/test_prism_lease_heartbeat_under_fenced_write.py @@ -0,0 +1,631 @@ +#!/usr/bin/env python3 +"""The lease heartbeat self-fencing shape: a heartbeat queued behind our own write. + +Every fenced write this coordinator performs — a share append, +``persist_accepted_block`` — row-locks the singleton ``qbit_ledger_writer_lease`` +tuple for the whole of its transaction. The pre-fix heartbeat renewed the lease +with a plain ``UPDATE``, which queues for that tuple. On the guard connection, +which carries a server-side ``statement_timeout``, queueing is fatal: the +heartbeat's own statement is cancelled, the coordinator reads that as loss of +its guarded session, and a perfectly healthy writer hard-exits because it was +briefly busy writing. The coordinator fenced itself out with its own success. + +The fix made the heartbeat's renewal non-blocking — +``FOR NO KEY UPDATE SKIP LOCKED`` — and proved liveness by reads instead: the +session answers, the guard advisory lock is still held by this backend, and +the last committed lease row still names this exact session. A skipped renewal +is harmless while that committed row is unexpired, because the fenced write +that skipped it refreshes the TTL when it commits. + +Nothing could interleave a real heartbeat with a real in-flight fenced write +before the deterministic harness existed: the fenced write has to be parked +*inside* its statement, after PostgreSQL has taken the tuple lock and before +the client's COMMIT, and no amount of running the coordinator reliably lands +there. This module parks it there on purpose and drives the shipped +``verify_writer_lease_guard_session`` across the gap. + +Four things are proved, and one of them is the reason the others mean anything: + +``HeartbeatUnderFencedWriteTests`` + The heartbeat survives an in-flight fenced write, skipping renewal + without ever queueing, and renewal resumes the moment that write commits. + ``test_the_queueing_renewal_spelling_dies_where_the_heartbeat_survives`` + is the control: it runs the *pre-fix* spelling into the same lock and + watches it die on the guard's statement timeout. Without that control, + ``renewed_count == 0`` would be equally consistent with a heartbeat that + never tried. + +``OwnWriteRenewalDeferralTests`` + The one lock-blocked-and-expired case that must not fail closed: this + process's own fenced write outlasting the TTL. The locker is attributable + through ``pg_stat_activity`` to this process's pool ``application_name``, + so the verification defers rather than self-fencing — and the same + expired row locked by anyone else still fails closed. + +``GuardQuerySlotTests`` + The guard connection is one session, so its callers serialize on one + query slot rather than pipelining onto the same backend. +""" + +from __future__ import annotations + +import threading +import unittest + +from lab.prism import share_ledger as share_ledger_module +from tests.prism_concurrency_harness import ( + LeaseHarness, + LockTimeout, + assert_deterministic, +) + +# Lease TTL for every scenario here. The own-write authority margin defaults +# to half the TTL (``_resolve_lease_authority_margin_seconds``), so a freshly +# acquired row at 60s is 30s clear of the margin: a renewal skipped over it +# defers nothing, which is what makes scenario 1's `renewal_deferred_to_own_write` +# assertion distinguishable from scenario 4's. +LEASE_TTL_SECONDS = 60.0 + +# The caller deadline the fenced write runs under. A slow +# ``persist_accepted_block`` is the real shape; the value only has to be a +# deadline, because arming one is what makes _NativePostgresClient wrap the +# statement in an explicit transaction whose COMMIT is a separate client +# message — the gap this module parks in. Chosen well beyond every clock +# advance below so the write's own budget never explains what we observe. +FENCED_WRITE_BUDGET_SECONDS = 600.0 + +# Past LEASE_TTL_SECONDS, so the committed row the heartbeat reads has lapsed +# while the fenced write that would have refreshed it is still in flight. +TTL_LAPSE_SECONDS = 70.0 + +# Three passes, because one is not a heartbeat. The point is that the guarded +# session keeps answering for the whole span of the fenced write, not that it +# answered once. +HEARTBEAT_PASSES = 3 + + +def drive_heartbeat_under_fenced_write(harness: LeaseHarness) -> dict[str, object]: + """Park a fenced write mid-statement and heartbeat across it. + + 1. ``alpha`` starts and takes the writer lease. + 2. ``alpha`` begins a deadline-scoped fenced statement. Stopping at + ``alpha.done:renew`` leaves PostgreSQL exactly where a long fenced + write leaves it: the statement has executed and holds the lease tuple + lock, and the COMMIT has not been sent. + 3. A second actor runs the shipped heartbeat verification on ``alpha``'s + guard connection, repeatedly. It has to be a second actor: ``alpha`` + is parked inside its own statement and also holds the ledger's writer + lock, so a heartbeat submitted to ``alpha`` could never run — which is + precisely the production shape, where the heartbeat is a separate + thread from whatever is writing. + """ + alpha = harness.coordinator("alpha") + alpha.start() + harness.run_until(alpha, "done:startup") + + def fenced_write() -> object: + # Renewal is the smallest fenced write that touches the lease tuple, + # and it is shipped code rather than a stand-in. Any fenced write has + # this shape: deadline armed, tuple locked, COMMIT still to come. + with alpha.ledger.operation_timeout(FENCED_WRITE_BUDGET_SECONDS): + return alpha.ledger.renew_writer_lease() + + alpha.submit(fenced_write, label="fenced-write") + harness.run_until(alpha, "alpha.done:renew") + + heartbeat = harness.scheduler.actor("heartbeat") + verifications = [] + for pass_number in range(1, HEARTBEAT_PASSES + 1): + label = f"heartbeat-{pass_number}" + call = heartbeat.submit( + alpha.ledger.verify_writer_lease_guard_session, + label=label, + ) + harness.run_until(heartbeat, f"done:{label}") + verifications.append(call.value()) + + return { + "verifications": verifications, + "fenced_write_still_holds_lease_lock": ( + harness.server.lease_lock_holder is not None + ), + "fenced_write_finished": alpha.actor.calls[-1].done, + "lockwait_stops": [stop for stop in harness.trace if "lockwait" in stop], + "statements": harness.statement_kinds(), + "lease_holder_session": harness.lease_holder_session(), + } + + +def drive_own_write_outlasting_the_ttl(harness: LeaseHarness) -> dict[str, object]: + """Let the TTL lapse under this coordinator's own in-flight fenced write. + + Same park as above, then the virtual clock runs past the committed row's + ``lease_expires_at``. The renewal that would have refreshed it is the very + write holding the lock, so the heartbeat now reads a row that is expired, + locked, and unrenewable — the shape that must fail closed when the locker + is a competing claimant, and must not when it is us. + """ + alpha = harness.coordinator("alpha") + alpha.start() + harness.run_until(alpha, "done:startup") + + def fenced_write() -> object: + with alpha.ledger.operation_timeout(FENCED_WRITE_BUDGET_SECONDS): + return alpha.ledger.renew_writer_lease() + + alpha.submit(fenced_write, label="fenced-write") + harness.run_until(alpha, "alpha.done:renew") + + harness.advance(TTL_LAPSE_SECONDS) + + heartbeat = harness.scheduler.actor("heartbeat") + call = heartbeat.submit( + alpha.ledger.verify_writer_lease_guard_session, + label="heartbeat", + ) + harness.run_until(heartbeat, "done:heartbeat") + + row = harness.lease_row() + return { + "verification": call.value(), + "committed_row_expired": ( + row is not None and row.lease_expires_at <= harness.clock.now() + ), + "committed_row_locked": row is not None and row.xmax is not None, + "lockwait_stops": [stop for stop in harness.trace if "lockwait" in stop], + "statements": harness.statement_kinds(), + } + + +def drive_foreign_locker_over_the_expired_row( + harness: LeaseHarness, +) -> dict[str, object]: + """The same expired-and-lock-blocked read, with the lock held by someone else. + + ``beta`` is a different-epoch claimant, so it has a different advisory + guard key and is not held off by ``alpha``'s guard. Parking it at + ``beta.done:acquire`` stops it in the gap its expiry CAS opens: it holds + the tuple lock and has staged a takeover that has not committed, so the + committed row ``alpha``'s heartbeat reads still names ``alpha`` and is + still expired. That stale read is exactly what the queueing renewal used + to catch by waiting, and what the non-blocking spelling has to catch by + failing closed. + """ + alpha = harness.coordinator("alpha", writer_epoch=1) + alpha.start() + harness.run_until(alpha, "done:startup") + + harness.advance(TTL_LAPSE_SECONDS) + + beta = harness.coordinator("beta", writer_epoch=2) + beta.start() + harness.run_until(beta, "beta.done:acquire") + + heartbeat = harness.scheduler.actor("heartbeat") + call = heartbeat.submit( + alpha.ledger.verify_writer_lease_guard_session, + label="heartbeat", + ) + harness.run_until(heartbeat, "done:heartbeat") + + row = harness.lease_row() + return { + "error": None if call.error is None else str(call.error), + "error_type": None if call.error is None else type(call.error).__name__, + "committed_row_session": None if row is None else row.writer_session_token, + "committed_row_expired": ( + row is not None and row.lease_expires_at <= harness.clock.now() + ), + "lockwait_stops": [stop for stop in harness.trace if "lockwait" in stop], + "statements": harness.statement_kinds(), + } + + +class HeartbeatUnderFencedWriteTests(unittest.TestCase): + """A heartbeat crossing an in-flight fenced write, and the control that dates it.""" + + def test_heartbeat_survives_the_whole_fenced_write_without_queueing(self) -> None: + """Skipped, never queued, never raised — for every pass of the write. + + ``renewed_count == 0`` on its own is only half the claim. The other + half is ``lockwait_stops``: the harness gives a statement that queues + for the lease tuple a named checkpoint, so an empty list is positive + evidence that the verification declined the lock rather than waiting + on it and getting lucky. ``verified_count == 1`` alongside it says the + session was proven live by reads while the renewal was skipped, which + is the whole substance of the fix. + """ + with LeaseHarness(lease_ttl_seconds=LEASE_TTL_SECONDS) as harness: + observed = drive_heartbeat_under_fenced_write(harness) + + self.assertTrue( + observed["fenced_write_still_holds_lease_lock"], + "the fenced write must still be holding the lease tuple lock, " + "or the heartbeat had nothing to survive", + ) + self.assertFalse(observed["fenced_write_finished"]) + self.assertEqual(len(observed["verifications"]), HEARTBEAT_PASSES) + for pass_number, result in enumerate(observed["verifications"], start=1): + with self.subTest(heartbeat_pass=pass_number): + self.assertEqual(result["verified_count"], 1) + self.assertEqual(result["renewed_count"], 0) + # The committed row is a full TTL fresh and the margin is + # half a TTL, so this skip is over a row with plenty of + # runway: liveness *and* authority, no deferral. + self.assertFalse(result["renewal_deferred_to_own_write"]) + self.assertEqual( + observed["lockwait_stops"], + [], + "a verification that stopped at a lockwait checkpoint queued " + "for the lease tuple, which is the self-fencing bug itself", + ) + self.assertEqual( + observed["statements"], + ["acquire", "renew", "verify", "verify", "verify"], + "each heartbeat must be one non-blocking statement; a second " + "statement would mean the attribution recheck fired, and this " + "row is not ambiguous", + ) + + def test_renewal_resumes_once_the_fenced_write_commits(self) -> None: + """The skip is deferral to the write, not a heartbeat that stopped renewing. + + The fenced write refreshes the TTL itself when it commits, so the + renewal the heartbeat skipped was never lost. Proving that the very + next verification renews is what separates "declined the lock" from + "silently gave up on renewing". + """ + with LeaseHarness(lease_ttl_seconds=LEASE_TTL_SECONDS) as harness: + drive_heartbeat_under_fenced_write(harness) + alpha = harness.coordinators["alpha"] + heartbeat = harness.scheduler.actors["heartbeat"] + + harness.run_until(alpha, "alpha.precommit") + harness.run_until(alpha, "done:fenced-write") + self.assertIsNone( + harness.server.lease_lock_holder, + "the committed fenced write must have released the tuple lock", + ) + + call = heartbeat.submit( + alpha.ledger.verify_writer_lease_guard_session, + label="heartbeat-after-commit", + ) + harness.run_until(heartbeat, "done:heartbeat-after-commit") + + self.assertEqual( + call.value(), + { + "backend": "postgres-psql", + "verified_count": 1, + "renewed_count": 1, + "renewal_deferred_to_own_write": False, + }, + ) + + def test_the_queueing_renewal_spelling_dies_where_the_heartbeat_survives( + self, + ) -> None: + """Control: the pre-fix renewal, run into the same lock, is cancelled. + + ``_renew_writer_lease_with`` is the plain locking ``UPDATE`` the + heartbeat used to run. Handed the guard's ``run_json`` it becomes the + pre-fix heartbeat exactly, and the guard session's server-side + ``statement_timeout`` is what turns its queue into a hard failure — + which the coordinator reads as loss of its guarded session. + + This test is what makes ``renewed_count == 0`` above mean "declined a + lock that was really there" rather than "nothing was attempted". + """ + with LeaseHarness(lease_ttl_seconds=LEASE_TTL_SECONDS) as harness: + drive_heartbeat_under_fenced_write(harness) + alpha = harness.coordinators["alpha"] + heartbeat = harness.scheduler.actors["heartbeat"] + guard = alpha.guard + assert guard is not None + + call = heartbeat.submit( + lambda: alpha.ledger._renew_writer_lease_with(guard.run_json), + label="queueing-renewal", + ) + # The pre-fix spelling does what the fixed one refuses to: it + # queues on the tuple the fenced write holds. + harness.run_until_blocked(heartbeat, "alpha.guard.lockwait:renew") + + self.assertIsNotNone( + heartbeat.wake_at, + "the guard session's statement timeout must bound this wait; " + "an unbounded one would be a different defect", + ) + self.assertIsNotNone( + harness.advance_to_next_deadline(), + "the guard statement timeout is the only pending deadline", + ) + harness.run_until(heartbeat, "done:queueing-renewal") + + # Specifically the guard connection's server-side + # statement_timeout, not a caller-armed lock_timeout: the + # heartbeat runs on a session whose bound it cannot widen, which + # is why queueing there is fatal rather than merely slow. + with self.assertRaisesRegex( + LockTimeout, + "canceling statement due to statement timeout", + ): + call.value() + self.assertTrue( + harness.server.lease_lock_holder is not None, + "the fenced write still holds the lock it queued for; the " + "renewal died, the write did not", + ) + + def test_interleaving_is_order_stable_across_repeated_runs(self) -> None: + """A flaky concurrency test is worse than none; prove this one is not.""" + run = assert_deterministic( + self, + drive_heartbeat_under_fenced_write, + repeats=25, + lease_ttl_seconds=LEASE_TTL_SECONDS, + ) + + self.assertEqual( + list(run.trace), + [ + "alpha@alpha.begin:acquire", + "alpha@alpha.done:acquire", + # Since #123 the acquire/adopt statements carry an acquisition + # deadline, so each commits explicitly and offers a precommit stop. + "alpha@alpha.precommit", + "alpha@done:startup", + "alpha@alpha.begin:renew", + "alpha@alpha.done:renew", + "heartbeat@alpha.guard.begin:verify", + "heartbeat@alpha.guard.snapshot:verify", + "heartbeat@alpha.guard.done:verify", + "heartbeat@done:heartbeat-1", + "heartbeat@alpha.guard.begin:verify", + "heartbeat@alpha.guard.snapshot:verify", + "heartbeat@alpha.guard.done:verify", + "heartbeat@done:heartbeat-2", + "heartbeat@alpha.guard.begin:verify", + "heartbeat@alpha.guard.snapshot:verify", + "heartbeat@alpha.guard.done:verify", + "heartbeat@done:heartbeat-3", + ], + ) + + +class OwnWriteRenewalDeferralTests(unittest.TestCase): + """Expired, lock-blocked, and ours: defer. Expired and someone else's: fail closed.""" + + def test_own_write_outlasting_the_ttl_defers_instead_of_self_fencing(self) -> None: + """Raising here would roll back a valid write and restart-loop on the next one. + + The locker is this process's own fenced write, so no competing expiry + claim can be in flight behind it — the tuple's exclusive lock is the + proof — and its commit refreshes ``lease_expires_at`` before any + queued claimant re-evaluates its CAS. The session provably survives, + so the verification must not raise. It survives as a *process* only: + the argument assumes the write commits, and a rollback would hand the + expired row straight to a queued claimant, so the same result carries + ``renewal_deferred_to_own_write`` to tell external-side-effect fences + to withhold their RPC. + """ + with LeaseHarness(lease_ttl_seconds=LEASE_TTL_SECONDS) as harness: + observed = drive_own_write_outlasting_the_ttl(harness) + + self.assertTrue( + observed["committed_row_expired"], + "the committed row must have lapsed for this to be the " + "fail-closed shape at all", + ) + self.assertTrue(observed["committed_row_locked"]) + self.assertEqual( + observed["verification"], + { + "backend": "postgres-psql", + "verified_count": 1, + "renewed_count": 0, + "renewal_deferred_to_own_write": True, + }, + ) + self.assertEqual(observed["lockwait_stops"], []) + self.assertEqual( + observed["statements"], + ["acquire", "renew", "verify"], + "the locker was attributable on the first statement, so no " + "attribution recheck was warranted", + ) + + def test_an_expired_row_locked_by_a_foreign_application_fails_closed(self) -> None: + """The contrast that gives the deferral its meaning. + + Identical read — expired committed row, renewal skipped over a held + lock — and the only difference is who holds the lock. A locker that + cannot be attributed to this process's pool ``application_name`` may + be a different-identity expiry claim whose commit lands right after + this snapshot, and the stale token read would name us until it does. + A skipped renewal is not evidence of liveness there, so it raises. + """ + with LeaseHarness(lease_ttl_seconds=LEASE_TTL_SECONDS) as harness: + observed = drive_foreign_locker_over_the_expired_row(harness) + + self.assertEqual( + observed["committed_row_session"], + "heartbeat-v1:alpha-1", + "the claimant's takeover must still be uncommitted, or the " + "verification would fail on identity instead", + ) + self.assertTrue(observed["committed_row_expired"]) + self.assertEqual(observed["error_type"], "RuntimeError") + self.assertIsNotNone(observed["error"]) + self.assertIn( + "expired and its renewal was lock-blocked", + str(observed["error"]), + ) + self.assertEqual( + observed["lockwait_stops"], + [], + "failing closed must still be non-blocking; the point of the " + "fix is that the guard never queues, not that it never raises", + ) + self.assertEqual( + observed["statements"], + ["acquire", "acquire", "verify", "verify"], + "an unattributable locker over an expired row is the one " + "ambiguous shape, so it earns exactly one recheck before the " + "fail-closed raise", + ) + + def test_own_write_deferral_is_order_stable_across_repeated_runs(self) -> None: + """The TTL-lapse interleaving pins a schedule too, not just an outcome.""" + run = assert_deterministic( + self, + drive_own_write_outlasting_the_ttl, + repeats=25, + lease_ttl_seconds=LEASE_TTL_SECONDS, + ) + + self.assertEqual( + list(run.trace), + [ + "alpha@alpha.begin:acquire", + "alpha@alpha.done:acquire", + "alpha@alpha.precommit", + "alpha@done:startup", + "alpha@alpha.begin:renew", + "alpha@alpha.done:renew", + "heartbeat@alpha.guard.begin:verify", + "heartbeat@alpha.guard.snapshot:verify", + "heartbeat@alpha.guard.done:verify", + "heartbeat@done:heartbeat", + ], + ) + + +class GuardQuerySlotTests(unittest.TestCase): + """One guard connection, one statement at a time.""" + + def test_production_guard_serializes_callers_on_one_query_slot(self) -> None: + """The shipped guard, not the harness's model of it. + + The scheduler test below drives ``FakeLeaseGuard``, whose slot is a + reimplementation: deleting ``with self._query_lock:`` from + ``_NativePostgresLeaseGuard.run_json`` would leave every assertion + there green. This one holds the production lock and shows the + production method waiting for it, which is the claim the module + actually wants to make. It needs no server — the serialization is a + plain lock around the statement, above the connection. + """ + guard = share_ledger_module._NativePostgresLeaseGuard.__new__( + share_ledger_module._NativePostgresLeaseGuard + ) + guard._advisory_lock_key = 0 + guard._query_lock = threading.Lock() + guard._closed = False + guard._held = True + guard._connection = None + + entered = threading.Event() + arrived = threading.Event() + released = threading.Event() + + def second_caller() -> None: + entered.set() + # Reaching the statement requires the slot; `held` is True and + # `_connection` is None, so arrival past the slot is observable as + # the AttributeError the statement itself then raises. Catching + # that exact type is what makes this evidence: any other outcome + # means run_json did not get where this test claims it got. + try: + guard.run_json("SELECT 1") + except AttributeError: + arrived.set() + released.set() + + guard._query_lock.acquire() + caller = threading.Thread(target=second_caller, daemon=True) + caller.start() + self.assertTrue(entered.wait(2.0)) + self.assertFalse( + released.wait(0.2), + "run_json entered the statement while another caller held the " + "query slot; the guard session is not serialized", + ) + guard._query_lock.release() + self.assertTrue( + released.wait(2.0), + "releasing the slot must let the queued caller proceed", + ) + self.assertTrue( + arrived.is_set(), + "the queued caller must have reached the statement itself, not " + "merely returned", + ) + caller.join(timeout=2.0) + + def test_a_second_caller_parks_on_the_guard_query_slot(self) -> None: + """The harness models that serialization as an interleaving point. + + Both callers run the non-blocking verification, so neither can be + queued on the lease tuple; the only thing that can hold one up is the + other's occupancy of the single guard connection. This pins the + harness's rendering of it — that the second caller stops at a named + checkpoint and is released by the holder rather than by a timeout — + so scenarios can schedule around it. The production serialization + itself is pinned by the test above. + """ + with LeaseHarness(lease_ttl_seconds=LEASE_TTL_SECONDS) as harness: + alpha = harness.coordinator("alpha") + alpha.start() + harness.run_until(alpha, "done:startup") + + first = harness.scheduler.actor("hb-first") + second = harness.scheduler.actor("hb-second") + first_call = first.submit( + alpha.ledger.verify_writer_lease_guard_session, + label="first", + ) + second_call = second.submit( + alpha.ledger.verify_writer_lease_guard_session, + label="second", + ) + + harness.run_until(first, "alpha.guard.begin:verify") + harness.run_until_blocked(second, "alpha.guard.slot:wait") + self.assertIsNone( + second.wake_at, + "the slot wait is released by the holder, not by a timeout", + ) + + harness.run_until(first, "done:first") + self.assertTrue( + second.runnable(), + "releasing the slot must be what makes the second caller " + "runnable again", + ) + harness.run_until(second, "done:second") + + self.assertEqual(first_call.value()["verified_count"], 1) + self.assertEqual(second_call.value()["verified_count"], 1) + self.assertEqual( + harness.trace, + [ + "alpha@alpha.begin:acquire", + "alpha@alpha.done:acquire", + "alpha@alpha.precommit", + "alpha@done:startup", + "hb-first@alpha.guard.begin:verify", + "hb-second@alpha.guard.slot:wait", + "hb-first@alpha.guard.snapshot:verify", + "hb-first@alpha.guard.done:verify", + "hb-first@done:first", + "hb-second@alpha.guard.begin:verify", + "hb-second@alpha.guard.snapshot:verify", + "hb-second@alpha.guard.done:verify", + "hb-second@done:second", + ], + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_prism_lease_monitor_stalls.py b/tests/test_prism_lease_monitor_stalls.py new file mode 100644 index 000000000..10015180b --- /dev/null +++ b/tests/test_prism_lease_monitor_stalls.py @@ -0,0 +1,1019 @@ +#!/usr/bin/env python3 +"""In-process lease-monitor stalls: attribution and response (issue #227). + +What production showed +---------------------- +Three days after PR #214 shipped the writer-lease timing policy, +``union-mainnet`` recorded a heartbeat-monitor wake delay of 0.648s while +the host was 93% idle. The stall was in-process — a 5.1s payout-window +rescan holding the interpreter — and two things about the telemetry made +it undiagnosable: + +* ``guard_sql`` was measured client-side, so the worst attempt showed + 0.714s of "database time" against a server-enforced 0.50s statement + timeout while ``scheduler_delay`` never exceeded 22 microseconds in 645k + attempts. The attribution literally could not see the stall. +* The monitor's lateness was exported only as a lifetime high-water mark, + so every smaller stall was invisible and the alert could fire once per + process lifetime. + +What is proved here +------------------- +``InRoundTripStallAttributionTests`` + The required scenario: on the #128 deterministic harness (virtual + clock, baton scheduler, nothing sleeps) a 0.65s scheduler stall is + injected *inside* a guard round trip — PostgreSQL has answered, the + heartbeat thread does not get the interpreter back — and the new + attribution reports it as scheduler time, not ``guard_sql``. Plus the + accumulator-level arithmetic of the split. + +``MonitorWakeTelemetryTests`` + The point of #227: the rolling-window maximum falls back down once a + stall ages out and the record's age advances, unlike the lifetime + gauge. + +``GcPauseTelemetryTests`` + Pause durations land in the right generation's series. + +``StallProbeTests`` + Repeated over-half-slack wakes produce at most the configured number + of stack samples per window — a count, never a duration. + +``StallProbeFrameOwnershipTests`` + Completed samples, sampling caps, and formatting errors release sampled + workers' locals without waiting for cyclic GC (issue #247). + +``LatenessBeyondSlackResponseTests`` + The decided response (option (b) in the ``writer_lease_timing`` + docstring): the monitor does not hard-exit a healthy coordinator, the + breach is counted with its overrun past the exit-guarantee bound, and + one structured warning names it. +""" + +from __future__ import annotations + +from dataclasses import replace +import gc +import re +import threading +import unittest +from unittest import mock +import weakref + +from lab.prism.background_services import ( + GcPauseTelemetry, + LedgerLeaseHeartbeatPorts, + LedgerLeaseHeartbeatService, + MonitorWakeTelemetry, + StallProbe, + verifier_callbacks, +) +from lab.prism.process_telemetry import PRISM_GC_GENERATIONS +from lab.prism.writer_lease_timing import ( + DEFAULT_WRITER_LEASE_HEARTBEAT_POLICY, + LEASE_HEARTBEAT_MODE_PROOF, + LEASE_HEARTBEAT_OUTCOME_PROVEN, + LEASE_MONITOR_LATE_WAKE_SLACK_FRACTIONS, + LEASE_MONITOR_STALL_PROBE_MAX_SAMPLES_PER_WINDOW, + LEASE_MONITOR_STALL_PROBE_TRIGGER_SLACK_FRACTION, + LEASE_MONITOR_STALL_PROBE_WINDOW_SECONDS, + LEASE_MONITOR_WAKE_DELAY_BUCKETS, + LEASE_MONITOR_WAKE_DELAY_WINDOW_SECONDS, + OBSERVED_MONITOR_WAKE_DELAY_SECONDS_227, + WriterLeaseHeartbeatPolicy, + WriterLeaseVerificationAttempt, +) +from tests.prism_concurrency_harness import LeaseHarness +from tests.test_prism_lease_heartbeat_tail_latency import ( + HeartbeatUnderTest, + guard_statement_latency, +) + +POLICY = DEFAULT_WRITER_LEASE_HEARTBEAT_POLICY + +# The stall issue #227's acceptance criterion names: a scheduler stall of +# this length inside one guard round trip. Above the 0.50s slack and above +# the 0.55s bound at which exit-before-adoption stops being guaranteed. +IN_ROUND_TRIP_STALL_SECONDS = 0.65 + +# Modelled server time per guard statement in the harness scenario: a +# perfectly ordinary proof, so the whole stall is unambiguously not SQL. +SERVER_EXECUTION_SECONDS = 0.05 + +LEASE_TTL_SECONDS = 60.0 + + +class Clock: + """A settable virtual clock for the accumulator- and telemetry-level tests.""" + + def __init__(self, now: float = 1_000.0) -> None: + self.now = now + + def __call__(self) -> float: + return self.now + + def advance(self, seconds: float) -> float: + self.now += seconds + return self.now + + +class InRoundTripStallAttributionTests(unittest.TestCase): + def test_scheduler_stall_inside_a_guard_round_trip_is_not_charged_to_guard_sql( + self, + ) -> None: + """The required test (issue #227, acceptance criterion 1). + + The heartbeat is driven to the checkpoint at which the fake + PostgreSQL has finished executing the ownership proof — the + answer has left the server — and the virtual clock then advances + 0.65s before the heartbeat actor is stepped again, which is what a + GIL stall between the answer arriving and the heartbeat thread + resuming looks like. The statement reported 0.05s of server + execution, so the attribution must put the 0.65s in scheduler time + (and in its in-round-trip sub-phase, client resume), not in + ``guard_sql``. On the merge base this scenario reports + ``guard_sql`` of 0.70s and a scheduler delay of zero. + """ + with LeaseHarness(lease_ttl_seconds=LEASE_TTL_SECONDS) as harness: + alpha = harness.coordinator("alpha") + alpha.start() + harness.run_until(alpha, "done:startup") + heartbeat = HeartbeatUnderTest(harness, alpha) + warnings: list[str] = [] + heartbeat.service._ports = replace( + heartbeat.service._ports, log=warnings.append + ) + harness.server.statement_latency_seconds = guard_statement_latency( + SERVER_EXECUTION_SECONDS + ) + # The shipped ledger accepts the server-timing callback, so + # the wiring under test is production's, not a stub's. + self.assertIn( + "on_statement_server_seconds", + verifier_callbacks(alpha.ledger.prove_writer_lease_guard_session), + ) + actors = [heartbeat.heartbeat, heartbeat.monitor] + # The first beat renews; settle into the proof cadence. + heartbeat.run(actors, seconds=1.0) + self.assertFalse(heartbeat.exited) + proofs_before = heartbeat.service.attempt_counts[LEASE_HEARTBEAT_MODE_PROOF] + + # Fast-forward to the next beat, then run the heartbeat actor + # exactly up to the point where the fake server has finished + # the proof statement (its modelled 0.05s already charged). + heartbeat.advance_until_beat([heartbeat.monitor]) + harness.run_until(heartbeat.heartbeat, "alpha.guard.done:prove") + answered_at = harness.clock.monotonic() + + # PostgreSQL has answered. This process does not get back to + # the heartbeat thread for 0.65s. + harness.advance(IN_ROUND_TRIP_STALL_SECONDS) + harness.drain(actors) + heartbeat.quiesce(actors) + + self.assertFalse( + heartbeat.exited, + f"healthy coordinator hard-exited: {heartbeat.exits}", + ) + self.assertEqual( + heartbeat.service.attempt_counts[LEASE_HEARTBEAT_MODE_PROOF], + proofs_before + 1, + ) + phases = heartbeat.service.last_phases + self.assertEqual(phases.mode, LEASE_HEARTBEAT_MODE_PROOF) + self.assertEqual(phases.outcome, LEASE_HEARTBEAT_OUTCOME_PROVEN) + self.assertTrue(phases.server_reported) + # The database is charged only what it reported. + self.assertAlmostEqual( + phases.guard_sql_seconds, SERVER_EXECUTION_SECONDS, places=6 + ) + self.assertLess(phases.guard_sql_seconds, IN_ROUND_TRIP_STALL_SECONDS) + # The stall is scheduler time, and specifically the part of it + # that fell inside the round trip. + self.assertGreaterEqual( + phases.scheduler_delay_seconds, IN_ROUND_TRIP_STALL_SECONDS - 1e-6 + ) + self.assertAlmostEqual( + phases.client_resume_seconds, IN_ROUND_TRIP_STALL_SECONDS, places=6 + ) + self.assertAlmostEqual( + phases.slot_wait_seconds + + phases.guard_sql_seconds + + phases.scheduler_delay_seconds, + phases.total_seconds, + places=9, + ) + # The metrics view carries the same split under fixed keys. + snapshot = heartbeat.service.snapshot() + last = snapshot["last_phase_seconds"] + assert isinstance(last, dict) + self.assertAlmostEqual(last["guard_sql"], SERVER_EXECUTION_SECONDS, places=6) + self.assertAlmostEqual( + last["guard_client_resume"], IN_ROUND_TRIP_STALL_SECONDS, places=6 + ) + # The server-proven edge is still the conservative send edge: + # the stall neither made the session look fresher nor older + # than the instant the statement left. + self.assertAlmostEqual( + heartbeat.service.last_server_proven_monotonic, + answered_at - SERVER_EXECUTION_SECONDS, + places=6, + ) + self.assertIn("client_resume=0.650s", phases.summary()) + + def test_guard_sql_splits_into_server_execution_and_client_resume(self) -> None: + """Known server time plus a known client gap land where they should. + + ``slot_wait + guard_sql + scheduler_delay == total`` still holds, + and ``client_resume`` is the in-round-trip share of the residual. + """ + clock = Clock() + attempt = WriterLeaseVerificationAttempt( + LEASE_HEARTBEAT_MODE_PROOF, monotonic=clock + ) + clock.advance(0.02) # queue wait for the guard's slot + attempt.slot_acquired() + clock.advance(0.10) # the server executes + clock.advance(0.65) # ... and this process does not resume + attempt.statement_completed() + attempt.statement_server_seconds(0.10) + clock.advance(0.03) # result handling after the round trip + phases = attempt.finish(LEASE_HEARTBEAT_OUTCOME_PROVEN) + + self.assertTrue(phases.server_reported) + self.assertAlmostEqual(phases.slot_wait_seconds, 0.02) + self.assertAlmostEqual(phases.guard_sql_seconds, 0.10) + self.assertAlmostEqual(phases.client_resume_seconds, 0.65) + self.assertAlmostEqual(phases.scheduler_delay_seconds, 0.68) + self.assertAlmostEqual(phases.total_seconds, 0.80) + self.assertAlmostEqual( + phases.slot_wait_seconds + + phases.guard_sql_seconds + + phases.scheduler_delay_seconds, + phases.total_seconds, + ) + self.assertLessEqual(phases.client_resume_seconds, phases.scheduler_delay_seconds) + self.assertIn("server-timed", phases.summary()) + + def test_server_time_accumulates_across_a_two_statement_verification(self) -> None: + """The recheck's server time adds to the first statement's.""" + clock = Clock() + attempt = WriterLeaseVerificationAttempt("renew", monotonic=clock) + attempt.slot_acquired() + clock.advance(0.20) + attempt.statement_completed() + attempt.statement_server_seconds(0.15) + clock.advance(0.30) + attempt.statement_completed() + attempt.statement_server_seconds(0.25) + phases = attempt.finish("renewed") + self.assertEqual(phases.statement_count, 2) + self.assertAlmostEqual(phases.guard_sql_seconds, 0.40) + self.assertAlmostEqual(phases.client_resume_seconds, 0.10) + self.assertAlmostEqual(phases.scheduler_delay_seconds, 0.10) + + def test_unreported_or_malformed_server_time_keeps_the_conservative_split( + self, + ) -> None: + for reported in (None, "not a number", float("nan"), -1.0): + with self.subTest(reported=reported): + clock = Clock() + attempt = WriterLeaseVerificationAttempt( + LEASE_HEARTBEAT_MODE_PROOF, monotonic=clock + ) + attempt.slot_acquired() + clock.advance(0.70) + attempt.statement_completed() + attempt.statement_server_seconds(reported) # type: ignore[arg-type] + phases = attempt.finish(LEASE_HEARTBEAT_OUTCOME_PROVEN) + self.assertFalse(phases.server_reported) + self.assertAlmostEqual(phases.guard_sql_seconds, 0.70) + self.assertAlmostEqual(phases.client_resume_seconds, 0.0) + self.assertAlmostEqual(phases.scheduler_delay_seconds, 0.0) + + def test_server_time_cannot_exceed_the_span_it_fits_inside(self) -> None: + """A server figure larger than the client span is clamped, not trusted.""" + clock = Clock() + attempt = WriterLeaseVerificationAttempt( + LEASE_HEARTBEAT_MODE_PROOF, monotonic=clock + ) + attempt.slot_acquired() + clock.advance(0.30) + attempt.statement_completed() + attempt.statement_server_seconds(0.45) + phases = attempt.finish(LEASE_HEARTBEAT_OUTCOME_PROVEN) + self.assertAlmostEqual(phases.guard_sql_seconds, 0.30) + self.assertAlmostEqual(phases.client_resume_seconds, 0.0) + + +class MonitorWakeTelemetryTests(unittest.TestCase): + def test_rolling_window_maximum_decays_and_record_age_advances(self) -> None: + """Unlike the lifetime gauge, the window maximum comes back down. + + The lifetime record stays at 0.648s forever (that is the dashboard + continuity the existing gauge keeps), while the rolling maximum + reports the stall only for one window and the record age says how + long ago the record was set. + """ + telemetry = MonitorWakeTelemetry() + slack = POLICY.scheduler_slack_seconds + now = 10_000.0 + telemetry.observe(0.001, slack_seconds=slack, now=now) + stall_at = now + 20.0 + telemetry.observe( + OBSERVED_MONITOR_WAKE_DELAY_SECONDS_227, + slack_seconds=slack, + now=stall_at, + ) + self.assertAlmostEqual( + telemetry.lifetime_max_seconds, OBSERVED_MONITOR_WAKE_DELAY_SECONDS_227 + ) + self.assertAlmostEqual( + telemetry.window_max_seconds(stall_at), + OBSERVED_MONITOR_WAKE_DELAY_SECONDS_227, + ) + self.assertAlmostEqual(telemetry.record_age_seconds(stall_at), 0.0) + + # Quiet wakes for the rest of the window: the stall still dominates. + later = stall_at + LEASE_MONITOR_WAKE_DELAY_WINDOW_SECONDS / 2.0 + telemetry.observe(0.002, slack_seconds=slack, now=later) + self.assertAlmostEqual( + telemetry.window_max_seconds(later), + OBSERVED_MONITOR_WAKE_DELAY_SECONDS_227, + ) + self.assertAlmostEqual( + telemetry.record_age_seconds(later), + LEASE_MONITOR_WAKE_DELAY_WINDOW_SECONDS / 2.0, + ) + + # Once the stall ages out, the window maximum is the recent quiet + # wake, the lifetime record is unchanged, and its age keeps growing. + aged_out = stall_at + LEASE_MONITOR_WAKE_DELAY_WINDOW_SECONDS + 10.0 + telemetry.observe(0.003, slack_seconds=slack, now=aged_out) + self.assertAlmostEqual(telemetry.window_max_seconds(aged_out), 0.003) + self.assertAlmostEqual( + telemetry.lifetime_max_seconds, OBSERVED_MONITOR_WAKE_DELAY_SECONDS_227 + ) + self.assertAlmostEqual( + telemetry.record_age_seconds(aged_out), + LEASE_MONITOR_WAKE_DELAY_WINDOW_SECONDS + 10.0, + ) + # And with no wakes at all inside the window, it is zero. + self.assertAlmostEqual( + telemetry.window_max_seconds( + aged_out + 2.0 * LEASE_MONITOR_WAKE_DELAY_WINDOW_SECONDS + ), + 0.0, + ) + + snapshot = telemetry.snapshot(aged_out) + self.assertEqual(snapshot["count"], 4) + self.assertEqual( + set(snapshot["late_wakes"]), set(LEASE_MONITOR_LATE_WAKE_SLACK_FRACTIONS) + ) + self.assertEqual(tuple(snapshot["buckets"]), LEASE_MONITOR_WAKE_DELAY_BUCKETS) + + def test_histogram_is_cumulative_and_threshold_counters_are_exact(self) -> None: + telemetry = MonitorWakeTelemetry() + slack = 0.5 + for delay in (0.004, 0.26, 0.41, 0.5, 0.648): + telemetry.observe(delay, slack_seconds=slack, now=1.0) + snapshot = telemetry.snapshot(1.0) + buckets = snapshot["buckets"] + assert isinstance(buckets, dict) + self.assertEqual(buckets[0.005], 1) + self.assertEqual(buckets[0.25], 1) + self.assertEqual(buckets[0.4], 2) + self.assertEqual(buckets[0.5], 4) + self.assertEqual(buckets[0.75], 5) + self.assertEqual(buckets[2.0], 5) + self.assertEqual(snapshot["count"], 5) + self.assertAlmostEqual(snapshot["sum"], 0.004 + 0.26 + 0.41 + 0.5 + 0.648) + late = snapshot["late_wakes"] + assert isinstance(late, dict) + # >= 0.25, >= 0.4, >= 0.5 respectively. + self.assertEqual(late, {"0.5": 4, "0.8": 3, "1.0": 2}) + + +class GcPauseTelemetryTests(unittest.TestCase): + def test_gc_pause_histogram_records_per_generation(self) -> None: + """A pause lands in the series of the generation that paused.""" + clock = Clock() + telemetry = GcPauseTelemetry(clock=clock) + telemetry._callback("start", {"generation": 2}) + clock.advance(0.30) + telemetry._callback("stop", {"generation": 2, "collected": 5, "uncollectable": 0}) + telemetry._callback("start", {"generation": 0}) + clock.advance(0.002) + telemetry._callback("stop", {"generation": 0, "collected": 1, "uncollectable": 0}) + + snapshot = telemetry.snapshot() + self.assertEqual(set(snapshot), set(PRISM_GC_GENERATIONS)) + self.assertEqual(snapshot["2"]["count"], 1) + self.assertAlmostEqual(snapshot["2"]["last_seconds"], 0.30) + self.assertAlmostEqual(snapshot["2"]["max_seconds"], 0.30) + self.assertEqual(snapshot["0"]["count"], 1) + self.assertAlmostEqual(snapshot["0"]["last_seconds"], 0.002) + self.assertEqual(snapshot["1"]["count"], 0) + buckets_2 = snapshot["2"]["buckets"] + assert isinstance(buckets_2, dict) + self.assertEqual(buckets_2[0.25], 0) + self.assertEqual(buckets_2[0.5], 1) + worst_generation, worst_seconds = telemetry.worst() + self.assertEqual(worst_generation, "2") + self.assertAlmostEqual(worst_seconds, 0.30) + # An unknown generation never grows the label set. + telemetry.record("3", 5.0) + self.assertEqual(set(telemetry.snapshot()), set(PRISM_GC_GENERATIONS)) + + def test_installed_collector_callbacks_record_a_real_collection(self) -> None: + telemetry = GcPauseTelemetry() + telemetry.install() + telemetry.install() # idempotent + self.addCleanup(telemetry.uninstall) + self.assertEqual(gc.callbacks.count(telemetry._callback), 1) + gc.collect() # a full collection reports generation 2 + self.assertGreaterEqual(telemetry.snapshot()["2"]["count"], 1) + telemetry.uninstall() + self.assertNotIn(telemetry._callback, gc.callbacks) + self.assertFalse(telemetry.installed) + + +class StallProbeTests(unittest.TestCase): + def test_stall_probe_is_rate_limited(self) -> None: + """Repeated over-half-slack wakes yield at most N samples per window.""" + probe = StallProbe() + self.assertEqual( + probe.max_samples_per_window, LEASE_MONITOR_STALL_PROBE_MAX_SAMPLES_PER_WINDOW + ) + self.assertEqual(probe.window_seconds, LEASE_MONITOR_STALL_PROBE_WINDOW_SECONDS) + triggers = 10 * LEASE_MONITOR_STALL_PROBE_MAX_SAMPLES_PER_WINDOW + sampled = [ + probe.sample(now=100.0 + index, wake_delay_seconds=0.3) + for index in range(triggers) + ] + self.assertEqual(probe.samples_total, LEASE_MONITOR_STALL_PROBE_MAX_SAMPLES_PER_WINDOW) + self.assertEqual( + probe.suppressed_total, + triggers - LEASE_MONITOR_STALL_PROBE_MAX_SAMPLES_PER_WINDOW, + ) + self.assertEqual( + sum(1 for text in sampled if text is not None), + LEASE_MONITOR_STALL_PROBE_MAX_SAMPLES_PER_WINDOW, + ) + # The next window re-arms the budget. + self.assertIsNotNone( + probe.sample( + now=100.0 + LEASE_MONITOR_STALL_PROBE_WINDOW_SECONDS, + wake_delay_seconds=0.3, + ) + ) + self.assertEqual( + probe.samples_total, LEASE_MONITOR_STALL_PROBE_MAX_SAMPLES_PER_WINDOW + 1 + ) + + def test_probe_samples_other_threads_not_itself(self) -> None: + parked = threading.Event() + release = threading.Event() + + def park() -> None: + parked.set() + release.wait(5.0) + + worker = threading.Thread(target=park, name="stall-probe-target", daemon=True) + worker.start() + self.addCleanup(release.set) + self.assertTrue(parked.wait(5.0)) + stacks = StallProbe().capture_stacks() + self.assertIn("stall-probe-target", stacks) + self.assertIn("park@", stacks) + self.assertNotIn(threading.current_thread().name + " (", stacks) + # Bounded output: every frame names a bare file, never a path. + frame_files = re.findall(r"@([^:\s]+):\d+", stacks) + self.assertTrue(frame_files) + for frame_file in frame_files: + self.assertNotIn("/", frame_file) + + def test_service_level_probe_holds_the_limit_under_a_stall_storm(self) -> None: + """Through the monitor's own entry point, the count still holds.""" + clock = Clock() + service, logged, exits = _service_with_fake_ports(clock) + trigger = LEASE_MONITOR_STALL_PROBE_TRIGGER_SLACK_FRACTION * POLICY.scheduler_slack_seconds + for _ in range(50): + clock.advance(0.05) + _wake(service, trigger) + self.assertEqual( + service.stall_probe.samples_total, + LEASE_MONITOR_STALL_PROBE_MAX_SAMPLES_PER_WINDOW, + ) + self.assertEqual( + service.stall_probe.suppressed_total, + 50 - LEASE_MONITOR_STALL_PROBE_MAX_SAMPLES_PER_WINDOW, + ) + # One probe line per sample, none for suppressed triggers, no exit. + self.assertEqual(len(logged), LEASE_MONITOR_STALL_PROBE_MAX_SAMPLES_PER_WINDOW) + self.assertEqual(exits, []) + self.assertEqual(service.exit_guarantee_breaches, 0) + + +class StallProbeFrameOwnershipTests(unittest.TestCase): + """A diagnostic must not keep finished workers alive until cyclic GC (#247).""" + + def _assert_worker_released( + self, probe: StallProbe, *, fail_formatting: bool = False + ) -> None: + class Payload: + pass + + parked = threading.Event() + release = threading.Event() + payload_refs: list[weakref.ReferenceType[Payload]] = [] + + def park() -> None: + payload = Payload() + payload_refs.append(weakref.ref(payload)) + parked.set() + release.wait(10.0) + + worker = threading.Thread(target=park, name="stall-probe-owner", daemon=True) + gc_was_enabled = gc.isenabled() + gc.disable() + failure = None + try: + worker.start() + try: + self.assertTrue(parked.wait(5.0)) + if fail_formatting: + expected = RuntimeError("stack formatting failed") + with mock.patch( + "lab.prism.background_services.os", + ) as probe_os: + probe_os.path.basename.side_effect = expected + try: + probe.capture(now=100.0, wake_delay_seconds=0.3) + except RuntimeError as error: + # Keep the traceback alive too: its capture_stacks + # frame must no longer own the sampled frames. + failure = error + self.assertIs(failure, expected) + else: + stacks = probe.capture(now=100.0, wake_delay_seconds=0.3) + self.assertEqual(probe.last_sample["stacks"], stacks) + finally: + release.set() + worker.join(5.0) + self.assertFalse(worker.is_alive()) + self.assertEqual(len(payload_refs), 1) + # No gc.collect() before this assertion. Keeping the probe, its + # last_sample, and any formatting traceback must retain only text. + self.assertIsNone(payload_refs[0]()) + finally: + failure = None + gc.collect() + if gc_was_enabled: + gc.enable() + + def test_completed_sample_releases_worker_without_cyclic_gc(self) -> None: + self._assert_worker_released(StallProbe()) + + def test_sampling_limits_release_worker_without_cyclic_gc(self) -> None: + for limits in ( + {"max_threads": 0}, + {"max_threads": 1}, + {"max_frames_per_thread": 0}, + {"max_frames_per_thread": 1}, + ): + with self.subTest(**limits): + self._assert_worker_released(StallProbe(**limits)) + + def test_formatting_error_releases_worker_with_traceback_alive(self) -> None: + self._assert_worker_released(StallProbe(), fail_formatting=True) + + def test_empty_snapshot(self) -> None: + with mock.patch("lab.prism.background_services.sys._current_frames", return_value={}): + self.assertEqual(StallProbe().capture_stacks(), "") + + +def _service_with_fake_ports( + clock: Clock, + *, + policy: WriterLeaseHeartbeatPolicy = POLICY, +) -> tuple[LedgerLeaseHeartbeatService, list[str], list[str]]: + """A heartbeat service on a virtual clock with a recording log and exit.""" + logged: list[str] = [] + exits: list[str] = [] + + class Ledger: + _lease_adoption_silence_seconds = policy.adoption_silence_seconds + + ports = LedgerLeaseHeartbeatPorts( + ledger=lambda: Ledger(), + heartbeat_seconds=lambda: policy.heartbeat_interval_seconds, + failure_seconds=lambda: policy.failure_budget_seconds, + monitor_seconds=lambda: policy.monitor_interval_seconds, + exit_timeout_seconds=lambda: policy.exit_margin_seconds, + external_fence_timeout_seconds=lambda: 0.5, + lease_hard_exit=lambda message, *, include_traceback: exits.append(message), + watchdog_hard_exit=lambda reason, *, timeout_seconds: exits.append(reason), + heartbeat_loop=lambda: None, + monitor_loop=lambda: None, + monotonic=clock, + scheduler_slack_seconds=lambda: policy.scheduler_slack_seconds, + log=logged.append, + ) + service = LedgerLeaseHeartbeatService(ports) + service.last_success_monotonic = clock() + service.last_server_proven_monotonic = clock() + return service, logged, exits + + +def _wake( + service: LedgerLeaseHeartbeatService, + wake_delay: float, + policy: WriterLeaseHeartbeatPolicy = POLICY, +): + """One monitor wake through the shipped accounting, diagnostics run inline. + + The monitor loop hands the returned job to a diagnostics thread after + the exit decision; the tier tests here run it synchronously so their + assertions are deterministic. The threading itself is pinned by + ``InstrumentCostAndOrderingTests``. + """ + job = service._observe_monitor_wake(wake_delay, policy) + if job is not None: + job() + return job + + +class LatenessBeyondSlackResponseTests(unittest.TestCase): + def test_lateness_beyond_slack_triggers_the_documented_response(self) -> None: + """Option (b), pinned: no exit, a counted breach, one structured warning. + + Driven at the production number. A 0.648s wake is beyond the + 0.50s slack *and* beyond the 0.55s bound of inequality (4), so it + is an exit-guarantee breach with a 0.098s overrun. The monitor + accepts it — the fences carry the residual — and says so. + """ + clock = Clock() + service, logged, exits = _service_with_fake_ports(clock) + clock.advance(1.0) + _wake(service, OBSERVED_MONITOR_WAKE_DELAY_SECONDS_227) + + self.assertEqual(exits, []) + self.assertEqual(service.exit_guarantee_breaches, 1) + self.assertAlmostEqual( + service.worst_exit_guarantee_overrun_seconds, + OBSERVED_MONITOR_WAKE_DELAY_SECONDS_227 + - POLICY.max_guaranteed_monitor_lateness_seconds, + ) + self.assertEqual(len(logged), 1) + warning = logged[0] + self.assertIn("WARNING", warning) + self.assertIn("monitor wake delay 0.648s exceeded the 0.500s scheduler slack", warning) + self.assertIn("issue #227", warning) + self.assertIn("exit-before-adoption was NOT guaranteed for this beat", warning) + self.assertIn("0.098s after the adoption edge", warning) + self.assertIn("session-token fenced", warning) + self.assertIn("Last attempt:", warning) + self.assertIn("Policy:", warning) + # A stack sample rode along with the first warning. + self.assertIn("Threads:", warning) + self.assertEqual(service.stall_probe.samples_total, 1) + breach = service.last_slack_breach + assert breach is not None + self.assertAlmostEqual(breach["wake_delay_seconds"], 0.648) + self.assertAlmostEqual(breach["overrun_seconds"], 0.098) + + # The metric surface reports it under fixed keys. + snapshot = service.snapshot() + self.assertEqual(snapshot["exit_guarantee_breaches"], 1) + self.assertAlmostEqual(snapshot["worst_exit_guarantee_overrun_seconds"], 0.098) + wakes = snapshot["monitor_wakes"] + assert isinstance(wakes, dict) + self.assertEqual(wakes["late_wakes"], {"0.5": 1, "0.8": 1, "1.0": 1}) + self.assertAlmostEqual(wakes["lifetime_max_seconds"], 0.648) + self.assertAlmostEqual(wakes["window_max_seconds"], 0.648) + policy_seconds = snapshot["policy_seconds"] + assert isinstance(policy_seconds, dict) + self.assertAlmostEqual(policy_seconds["max_guaranteed_monitor_lateness"], 0.55) + + def test_lateness_inside_the_reserve_warns_without_a_guarantee_breach(self) -> None: + clock = Clock() + service, logged, exits = _service_with_fake_ports(clock) + clock.advance(1.0) + _wake(service, 0.52) + self.assertEqual(exits, []) + self.assertEqual(service.exit_guarantee_breaches, 0) + self.assertEqual(service.worst_exit_guarantee_overrun_seconds, 0.0) + self.assertEqual(len(logged), 1) + self.assertIn("exit-before-adoption still held for this beat", logged[0]) + self.assertIn("strictness reserve consumed", logged[0]) + + def test_wakes_within_slack_are_counted_but_never_warned(self) -> None: + clock = Clock() + service, logged, exits = _service_with_fake_ports(clock) + for delay in (0.0, 0.01, 0.2, 0.24): + clock.advance(0.05) + self.assertIsNone(_wake(service, delay)) + self.assertEqual(logged, []) + self.assertEqual(exits, []) + self.assertEqual(service.exit_guarantee_breaches, 0) + self.assertEqual(service.stall_probe.samples_total, 0) + self.assertEqual(service.snapshot()["monitor_wakes"]["count"], 4) + + def test_breach_warnings_are_rate_limited_like_the_probe(self) -> None: + """A stall storm produces bounded warnings, every breach still counted.""" + clock = Clock() + service, logged, exits = _service_with_fake_ports(clock) + storm = 4 * LEASE_MONITOR_STALL_PROBE_MAX_SAMPLES_PER_WINDOW + for _ in range(storm): + clock.advance(0.7) + _wake(service, 0.7) + self.assertEqual(service.exit_guarantee_breaches, storm) + self.assertEqual(len(logged), LEASE_MONITOR_STALL_PROBE_MAX_SAMPLES_PER_WINDOW) + self.assertEqual( + service.suppressed_slack_breach_warnings, + storm - LEASE_MONITOR_STALL_PROBE_MAX_SAMPLES_PER_WINDOW, + ) + # Suppressed warnings are exported, so a quiet period and a + # rate-limited flood do not look the same on /metrics. + self.assertEqual( + service.snapshot()["slack_breach_warnings_suppressed"], + storm - LEASE_MONITOR_STALL_PROBE_MAX_SAMPLES_PER_WINDOW, + ) + self.assertEqual(exits, []) + + def test_late_monitor_beyond_slack_survives_on_the_harness(self) -> None: + """The real loops: a 0.648s-late monitor over a healthy heartbeat. + + The tail-latency suite proves survival at 0.4s (inside the slack). + This is the #227 case, beyond the slack: the monitor must not + hard-exit — the heartbeat kept proving, so nothing tripped a bound + — and must have recorded the exit-guarantee breach. + """ + with LeaseHarness(lease_ttl_seconds=LEASE_TTL_SECONDS) as harness: + alpha = harness.coordinator("alpha") + alpha.start() + harness.run_until(alpha, "done:startup") + heartbeat = HeartbeatUnderTest(harness, alpha) + warnings: list[str] = [] + heartbeat.service._ports = replace( + heartbeat.service._ports, log=warnings.append + ) + actors = [heartbeat.heartbeat, heartbeat.monitor] + heartbeat.run(actors, seconds=1.0) + + harness.advance( + heartbeat.policy.monitor_interval_seconds + + OBSERVED_MONITOR_WAKE_DELAY_SECONDS_227 + ) + while heartbeat.monitor.runnable(): + harness.step(heartbeat.monitor) + heartbeat.run(actors, seconds=1.0) + heartbeat.quiesce(actors) + + self.assertFalse(heartbeat.exited, heartbeat.exits) + service = heartbeat.service + self.assertGreaterEqual(service.monitor_wake_delay_seconds, 0.648) + self.assertEqual(service.exit_guarantee_breaches, 1) + self.assertGreaterEqual(service.worst_exit_guarantee_overrun_seconds, 0.098) + # The warning is written by the diagnostics thread, after the + # monitor's decision; wait for it before reading the sink. + diagnostics = service.diagnostics_thread + self.assertIsNotNone(diagnostics) + assert diagnostics is not None + diagnostics.join(5.0) + self.assertFalse(diagnostics.is_alive()) + self.assertEqual(len(warnings), 1) + self.assertIn("NOT guaranteed", warnings[0]) + snapshot = service.snapshot() + self.assertEqual(snapshot["monitor_wakes"]["late_wakes"]["1.0"], 1) + + def test_hard_exit_reason_quotes_window_max_record_age_and_gc(self) -> None: + """The exit message no longer quotes only a possibly hours-old record.""" + with LeaseHarness(lease_ttl_seconds=LEASE_TTL_SECONDS) as harness: + alpha = harness.coordinator("alpha") + alpha.start() + harness.run_until(alpha, "done:startup") + heartbeat = HeartbeatUnderTest(harness, alpha) + heartbeat.service._ports = replace( + heartbeat.service._ports, log=lambda line: None + ) + heartbeat.service.gc_pauses.record("2", 0.123) + actors = [heartbeat.heartbeat, heartbeat.monitor] + heartbeat.run(actors, seconds=1.0) + # Deschedule the heartbeat for good: the monitor must age it + # out and exit, quoting the new figures. + heartbeat.stall_heartbeat(heartbeat.policy.server_proven_cap_seconds + 1.0) + self.assertTrue(heartbeat.exited) + reason = heartbeat.exit_reason + self.assertIn("Monitor wake delay:", reason) + self.assertIn("lifetime record", reason) + self.assertIn(f"in the last {LEASE_MONITOR_WAKE_DELAY_WINDOW_SECONDS:g}s", reason) + self.assertIn("exit-guarantee breach(es)", reason) + self.assertIn("Worst GC pause: 0.123s (generation 2)", reason) + + +class _SnapshotOnWrite(list): + """A bucket list that takes a snapshot the instant any bucket is bumped. + + Stands in for a scrape interleaving with a writer between two of the + writer's stores; the invariant under test is that no interleaving can + render a bucket above the histogram's count. + """ + + def __init__(self, values, on_write) -> None: + super().__init__(values) + self._on_write = on_write + + def __setitem__(self, index, value) -> None: + super().__setitem__(index, value) + self._on_write() + + +DIAGNOSTICS_THREAD_NAME = "prism-ledger-lease-monitor-diagnostics" + + +class InstrumentCostAndOrderingTests(unittest.TestCase): + """The instrument must not extend the path it measures. + + The monitor thread decides the hard exit. Everything this package adds + to that thread has to be O(1) and has to come *after* the decision, or + it consumes the very envelope it reports on — and work done between + polls is not measured as lateness at all. These tests pin the + mechanism (which thread, in which order, which calls are never made), + not a duration. + """ + + def test_observing_a_wake_does_no_capture_or_logging_itself(self) -> None: + """Accounting is O(1); the frame walk and the log line are deferred.""" + clock = Clock() + service, logged, exits = _service_with_fake_ports(clock) + captures: list[str] = [] + + def capture_stacks() -> str: + captures.append(threading.current_thread().name) + return "stacks" + + service.stall_probe.capture_stacks = capture_stacks # type: ignore[method-assign] + clock.advance(1.0) + job = service._observe_monitor_wake(OBSERVED_MONITOR_WAKE_DELAY_SECONDS_227, POLICY) + # Counted and admitted, but nothing walked and nothing written yet. + self.assertEqual(service.exit_guarantee_breaches, 1) + self.assertEqual(captures, []) + self.assertEqual(logged, []) + self.assertEqual(service.stall_probe.samples_total, 0) + self.assertIsNotNone(job) + assert job is not None + # The shipped runner hands the job to a named daemon thread. + service._run_diagnostics(job) + thread = service.diagnostics_thread + assert thread is not None + thread.join(5.0) + self.assertFalse(thread.is_alive()) + self.assertEqual(captures, [DIAGNOSTICS_THREAD_NAME]) + self.assertEqual(len(logged), 1) + self.assertIn("Threads:\nstacks", logged[0]) + self.assertEqual(service.stall_probe.samples_total, 1) + self.assertEqual(exits, []) + + def test_probe_and_warning_run_after_the_exit_decision_and_off_the_monitor_thread( + self, + ) -> None: + """On the real loops: decide first, diagnose after, never on the monitor. + + First a healthy heartbeat with a 0.648s-late monitor: the sample and + the warning both happen, on the diagnostics thread, after the + monitor has decided not to exit. Then a dead heartbeat with a wake + far beyond the slack: the monitor exits, on its own thread, and no + capture or log line runs ahead of that exit (nor after it — the + exit reason carries the figures). + """ + with LeaseHarness(lease_ttl_seconds=LEASE_TTL_SECONDS) as harness: + alpha = harness.coordinator("alpha") + alpha.start() + harness.run_until(alpha, "done:startup") + heartbeat = HeartbeatUnderTest(harness, alpha) + service = heartbeat.service + monitor_thread_name = heartbeat.monitor._thread.name + events: list[tuple[str, str]] = [] + + def capture_stacks() -> str: + events.append(("capture", threading.current_thread().name)) + return "stacks" + + service.stall_probe.capture_stacks = capture_stacks # type: ignore[method-assign] + original_exit = service._ports.lease_hard_exit + + def lease_hard_exit(message: str, *, include_traceback: bool) -> None: + events.append(("exit", threading.current_thread().name)) + original_exit(message, include_traceback=include_traceback) + + service._ports = replace( + service._ports, + log=lambda line: events.append(("log", threading.current_thread().name)), + lease_hard_exit=lease_hard_exit, + ) + actors = [heartbeat.heartbeat, heartbeat.monitor] + heartbeat.run(actors, seconds=1.0) + + # Healthy heartbeat, late monitor. + harness.advance( + heartbeat.policy.monitor_interval_seconds + + OBSERVED_MONITOR_WAKE_DELAY_SECONDS_227 + ) + while heartbeat.monitor.runnable(): + harness.step(heartbeat.monitor) + self.assertFalse(heartbeat.exited) + diagnostics = service.diagnostics_thread + self.assertIsNotNone(diagnostics) + assert diagnostics is not None + diagnostics.join(5.0) + self.assertFalse(diagnostics.is_alive()) + self.assertEqual([kind for kind, _ in events], ["capture", "log"]) + self.assertEqual( + {name for _, name in events}, {DIAGNOSTICS_THREAD_NAME} + ) + self.assertNotEqual(DIAGNOSTICS_THREAD_NAME, monitor_thread_name) + self.assertEqual(service.exit_guarantee_breaches, 1) + events.clear() + + # Dead heartbeat: only the monitor is stepped, after sleeping + # through the whole cap. Its wake is far beyond the slack, so + # both tiers are due — and neither may run ahead of the exit. + harness.advance(heartbeat.policy.server_proven_cap_seconds + 1.0) + while heartbeat.monitor.runnable() and not heartbeat.exited: + harness.step(heartbeat.monitor) + self.assertTrue(heartbeat.exited) + self.assertEqual(events, [("exit", monitor_thread_name)]) + # The wake was still accounted (O(1), before the decision). + self.assertEqual(service.exit_guarantee_breaches, 2) + self.assertIn("exit-guarantee breach(es)", heartbeat.exit_reason) + heartbeat.quiesce(actors) + + def test_stack_capture_never_touches_linecache(self) -> None: + """No stat, no source read: the walk uses frame objects only. + + ``traceback.extract_stack`` stats every file and reads every line; + even ``StackSummary.extract(lookup_lines=False)`` still calls + ``linecache.checkcache`` per file. The sample prints function + names, basenames and line numbers, none of which need the source. + """ + parked = threading.Event() + release = threading.Event() + + def park() -> None: + parked.set() + release.wait(5.0) + + worker = threading.Thread(target=park, name="linecache-probe-target", daemon=True) + worker.start() + self.addCleanup(release.set) + self.assertTrue(parked.wait(5.0)) + with mock.patch("linecache.checkcache") as checkcache, mock.patch( + "linecache.lazycache" + ) as lazycache, mock.patch("linecache.getline") as getline, mock.patch( + "linecache.getlines" + ) as getlines, mock.patch("linecache.updatecache") as updatecache: + stacks = StallProbe().capture_stacks() + for patched in (checkcache, lazycache, getline, getlines, updatecache): + patched.assert_not_called() + self.assertIn("linecache-probe-target", stacks) + self.assertIn("park@", stacks) + + def test_gc_pause_buckets_never_exceed_count_mid_record(self) -> None: + """A scrape between two of record()'s stores sees bucket <= count.""" + telemetry = GcPauseTelemetry(clock=Clock()) + seen: list[dict[str, object]] = [] + telemetry._bucket_counts["2"] = _SnapshotOnWrite( + telemetry._bucket_counts["2"], + lambda: seen.append(telemetry.snapshot()["2"]), + ) + telemetry.record("2", 0.3) + self.assertTrue(seen) + for mid in seen: + buckets = mid["buckets"] + assert isinstance(buckets, dict) + self.assertTrue( + all(value <= mid["count"] for value in buckets.values()), mid + ) + + def test_wake_buckets_never_exceed_count_mid_observe(self) -> None: + telemetry = MonitorWakeTelemetry() + seen: list[dict[str, object]] = [] + telemetry._bucket_counts = _SnapshotOnWrite( + telemetry._bucket_counts, + lambda: seen.append(telemetry.snapshot(1.0)), + ) + telemetry.observe(0.3, slack_seconds=0.5, now=1.0) + self.assertTrue(seen) + for mid in seen: + buckets = mid["buckets"] + assert isinstance(buckets, dict) + self.assertTrue( + all(value <= mid["count"] for value in buckets.values()), mid + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_prism_lease_orphaned_lock.py b/tests/test_prism_lease_orphaned_lock.py new file mode 100644 index 000000000..9a793cf6c --- /dev/null +++ b/tests/test_prism_lease_orphaned_lock.py @@ -0,0 +1,384 @@ +#!/usr/bin/env python3 +"""Issue #123: an orphaned lease-tuple lock blocking writer failover. + +#123 describes a coordinator that vanishes mid-statement — a network +partition, a SIGSTOP, a VM pause — after PostgreSQL has finished executing a +deadline-scoped landing statement but before the client's COMMIT arrives. The +server keeps the transaction open and keeps the ``qbit_ledger_writer_lease`` +tuple lock. Before the fix nothing bounded how long it held them: +``idle_in_transaction_session_timeout`` was set nowhere, server-side TCP +keepalives were configured nowhere, and the startup lease upsert ran with no +``lock_timeout``, so a successor coordinator waited inside +``PsqlShareLedger.__init__`` until TCP keepalive teardown — hours at OS +defaults. + +The issue was written as a hypothesis with a hand-derived trigger sequence +because no test could express it: reproducing the interleaving meant a live +PostgreSQL, a partition, and luck. This module drives it deterministically +against the harness instead, with the successor running the shipped +acquisition path — the real retry loop, the real CAS SQL, the real +``_run_json`` transaction shape. + +Two tests, doing two different jobs: + +``OrphanedLeaseLockInterleavingTests`` + Proves the harness still expresses the interleaving: the schedule is + stable across repeated runs, the orphan really does hold the tuple lock, + and the successor really does park on it. That evidence is what #128 + asked for, and the fix does not remove any of it — the successor still + queues behind an orphan it cannot clear. What changed is that the wait is + now bounded, so the tests pin the bound instead of its absence. + +``OrphanedLeaseLockFailoverBoundTests`` + Asserts the behaviour #123's fix must produce: a successor's startup + finishes, one way or the other, within a bounded time. + + That assertion was authored against the fixed behaviour while 2.x.x was + still unfixed, and carried an ``expectedFailure`` marker so the suite + would go red the day it started passing. It now passes, so the marker is + gone — removed by the commit that fixed #123, exactly as this file + demanded. The assertion itself is unchanged and is the strongest + evidence the fix works: it was written by someone who had not seen the + fix, against a model of PostgreSQL that knows nothing about it. +""" + +from __future__ import annotations + +import unittest + +from lab.prism.share_ledger import ( + DEFAULT_LEASE_ACQUIRE_ATTEMPTS, + DEFAULT_LEASE_ACQUIRE_LOCK_TIMEOUT_SECONDS, + LedgerOperationTimeout, +) +from tests.prism_concurrency_harness import ( + LeaseHarness, + assert_deterministic, +) + +# How long the vanished client's landing statement was budgeted for. #120's +# escalated landing budget tops out at 120s; the exact value does not matter +# to the interleaving, only that a deadline is armed, because that is what +# makes _NativePostgresClient wrap the statement in an explicit transaction +# whose COMMIT is a separate client message. +LANDING_BUDGET_SECONDS = 120.0 + +# Well past any plausible watchdog, restart or retry interval, and still far +# short of the "on the order of hours" TCP keepalive teardown #123 describes. +FAILOVER_HORIZON_SECONDS = 6 * 60 * 60 + +# PsqlShareLedger's floor between acquisition attempts (_lease_retry_sleep is +# called with _lease_retry_min_sleep_seconds, which is min(0.25, max sleep)). +# Not a tunable of this fix; named so the elapsed-time assertion below can be +# exact rather than approximate. +RETRY_SLEEP_SECONDS = 0.25 + + +def drive_orphaned_lock_interleaving(harness: LeaseHarness) -> dict[str, object]: + """Run #123's trigger sequence and report what the successor sees. + + 1. ``alpha`` starts and takes the writer lease. + 2. ``alpha`` runs a deadline-scoped fenced statement, which PostgreSQL + executes inside an explicit transaction holding the lease tuple lock. + 3. ``alpha`` vanishes in the gap between the statement completing and its + COMMIT — no RST reaches the server, so the transaction stays open. + 4. ``beta``, a failover twin at the next writer epoch, starts up. Its + advisory-guard key differs from ``alpha``'s, so it is not held off + there and reaches the lease upsert, which is where #123 bites. + """ + alpha = harness.coordinator("alpha", writer_epoch=1) + alpha.start() + harness.run_until(alpha, "done:startup") + + def fenced_landing_statement() -> object: + # Any deadline-scoped fenced write has this shape; renewal is the + # smallest one that touches the lease tuple, and it is real shipped + # code rather than a stand-in. + with alpha.ledger.operation_timeout(LANDING_BUDGET_SECONDS): + return alpha.ledger.renew_writer_lease() + + alpha.submit(fenced_landing_statement, label="landing") + # Stop in the gap PostgreSQL is left in: statement executed, COMMIT not + # sent. This is the exact instant #123 names. + harness.run_until(alpha, "alpha.precommit") + alpha.vanish() + + orphan = harness.server.lease_lock_holder + + beta = harness.coordinator("beta", writer_epoch=2) + beta.start() + harness.run_until_blocked(beta, "beta.lockwait:acquire") + + return { + "orphan_open": orphan is not None and orphan.orphaned, + "orphan_holds_lease_lock": harness.server.lease_lock_holder is orphan, + "beta_blocked_at": beta.actor.block_reason, + "beta_parked_at": harness.clock.monotonic(), + "beta_wake_at": beta.actor.wake_at, + "scheduler_next_deadline": harness.scheduler.next_deadline(), + "lease_holder_session": harness.lease_holder_session(), + "statements": harness.statement_kinds(), + } + + +def drive_successor_to_quiescence( + harness: LeaseHarness, + successor: object, + *, + horizon_seconds: float = FAILOVER_HORIZON_SECONDS, +) -> bool: + """Run the successor until it finishes or nothing can move it again. + + Draining once is not enough to decide "did the wait end". A fix built + from ``lock_timeout`` plus a bounded retry — which is what #123 proposes + — parks the successor on a succession of short deadlines, and a single + ``advance`` followed by a single ``drain`` would leave it stopped at the + first of them and report a wait that never ended. Advancing to each + pending deadline in turn asks the question the assertion actually means: + is there any sequence of timeouts that gets this coordinator out? + + Returns True when startup finished, by acquiring or by raising. Returns + False when the successor is stuck with no deadline left to reach, which + is the state ``2.x.x`` produces. + """ + startup = successor.actor.calls[0] # type: ignore[attr-defined] + deadline = harness.clock.monotonic() + horizon_seconds + while not startup.done and harness.clock.monotonic() < deadline: + harness.drain([successor]) + if startup.done: + break + if harness.advance_to_next_deadline() is None: + # Nothing in the system is waiting on time, so no amount of + # further waiting changes anything. + break + return startup.done + + +class OrphanedLeaseLockInterleavingTests(unittest.TestCase): + """The harness can express and reproduce #123's interleaving.""" + + def test_successor_parks_on_the_orphaned_lease_tuple_lock(self) -> None: + with LeaseHarness() as harness: + observed = drive_orphaned_lock_interleaving(harness) + + self.assertTrue( + observed["orphan_open"], + "the vanished client's transaction must still be open server-side", + ) + self.assertTrue( + observed["orphan_holds_lease_lock"], + "the orphaned transaction must still hold the lease tuple lock", + ) + self.assertEqual(observed["beta_blocked_at"], "beta.lockwait:acquire") + # The interleaving #123 names is intact: the orphan holds the + # tuple lock and the successor is parked on it. What the fix + # changes is the only thing that was ever wrong with it — the + # wait now ends. _run_lease_acquisition_json arms + # lock_timeout/statement_timeout on every attempt, so the + # successor parks with a wake time rather than forever. + self.assertEqual( + observed["beta_wake_at"], + observed["beta_parked_at"] + + DEFAULT_LEASE_ACQUIRE_LOCK_TIMEOUT_SECONDS, + "the successor must park under the configured acquisition " + "lock deadline, not indefinitely", + ) + self.assertEqual( + observed["scheduler_next_deadline"], + observed["beta_wake_at"], + "and that deadline must be the thing the whole system is " + "waiting on: nothing else is pending, so it is what releases " + "the successor", + ) + # The predecessor's renewal never committed, so the committed row + # still names its session: the successor is queued behind a write + # whose effect will never land. + self.assertEqual( + observed["lease_holder_session"], + "heartbeat-v1:alpha-1", + ) + self.assertEqual( + observed["statements"], + ["acquire", "renew", "acquire"], + ) + + def test_wait_ends_in_bounded_attempts_and_a_visible_failure(self) -> None: + """How the wait ends, now that it ends. + + This test used to assert ``finished is False``: six virtual hours + passed and the successor was still parked, because nothing bounded + the wait. The fix inverts that premise outright, and simply flipping + the assertion would duplicate + ``OrphanedLeaseLockFailoverBoundTests``, which already pins *that* + the wait ends. + + So it pins *how* it ends instead, which nothing else covers: the + successor spends exactly its configured retry budget on the lock, + each attempt bounded by the configured deadline, and then fails + visibly with a ``RuntimeError`` that chains the underlying + ``LedgerOperationTimeout``. Retrying matters because an orphan is + often reaped partway through the budget; giving up visibly matters + because a coordinator that cannot take the lease must exit for its + supervisor rather than hang in ``__init__``. The evidence the old + assertion carried — that nothing releases the orphan's lock, so the + successor never actually starts — is kept below. + """ + with LeaseHarness() as harness: + drive_orphaned_lock_interleaving(harness) + beta = harness.coordinators["beta"] + before = harness.clock.monotonic() + + finished = drive_successor_to_quiescence(harness, beta) + + self.assertTrue(finished, "the bounded retry must terminate") + self.assertFalse( + beta.started, + "and it must not have started: the orphan still holds the " + "lease lock, so there was never a lease to take", + ) + # The orphan is untouched — this is still #123's interleaving, + # not a scenario that quietly resolved itself. + orphan = harness.server.lease_lock_holder + self.assertIsNotNone(orphan) + assert orphan is not None + self.assertTrue(orphan.orphaned) + + # Exactly the configured budget of attempts, no more and no + # fewer: one attempt would abandon orphans that clear on their + # own, and an unbounded count is the outage this fix removes. + self.assertEqual( + harness.statement_kinds().count("acquire"), + 1 + DEFAULT_LEASE_ACQUIRE_ATTEMPTS, + "alpha's successful acquire plus beta's full retry budget", + ) + # Every attempt ran the deadline out, so the wall time is the + # budget itself rather than an accident of scheduling. + self.assertEqual( + harness.clock.monotonic() - before, + DEFAULT_LEASE_ACQUIRE_ATTEMPTS + * DEFAULT_LEASE_ACQUIRE_LOCK_TIMEOUT_SECONDS + + (DEFAULT_LEASE_ACQUIRE_ATTEMPTS - 1) * RETRY_SLEEP_SECONDS, + ) + + with self.assertRaisesRegex( + RuntimeError, + "did not complete within its .*deadline on any of " + f"{DEFAULT_LEASE_ACQUIRE_ATTEMPTS} attempts", + ) as raised: + beta.actor.calls[0].value() + # Chained, not swallowed: the operator diagnoses from the cause + # rather than from this layer's guess at one. + self.assertIsInstance( + raised.exception.__cause__, + LedgerOperationTimeout, + ) + + def test_interleaving_is_order_stable_across_repeated_runs(self) -> None: + """A flaky concurrency test is worse than none; prove this one is not.""" + run = assert_deterministic( + self, + drive_orphaned_lock_interleaving, + repeats=25, + ) + + self.assertEqual( + list(run.trace), + [ + "alpha@alpha.begin:acquire", + "alpha@alpha.done:acquire", + # New since the fix: the startup acquire is deadline-scoped, + # so it runs in an explicit transaction with its own COMMIT. + # docs/prism-ledger-ops.md records what that costs. + "alpha@alpha.precommit", + "alpha@done:startup", + "alpha@alpha.begin:renew", + "alpha@alpha.done:renew", + "alpha@alpha.precommit", + "alpha@vanished", + "beta@beta.begin:acquire", + "beta@beta.lockwait:acquire", + ], + ) + + def test_committing_the_predecessor_releases_the_successor(self) -> None: + """Control: the successor is blocked by the lock, not by the harness. + + If the vanished client's transaction commits after all, the successor + wakes on the next step and its upsert proceeds. That is what makes the + parked state above attributable to the orphaned lock rather than to a + scenario that simply never scheduled the successor. + """ + with LeaseHarness() as harness: + drive_orphaned_lock_interleaving(harness) + beta = harness.coordinators["beta"] + orphan = harness.server.lease_lock_holder + assert orphan is not None + + harness.server._commit(orphan) + + self.assertTrue(beta.actor.runnable()) + harness.run_until(beta, "done:startup") + + # Once the lock is gone the twin gets to read the row and reaches + # its decision immediately. A different epoch cannot wait on + # another identity's unexpired lease, so it refuses loudly. That + # is the shape #123's fix should produce within a bounded time: + # the twin has no useful work to do behind that lock, and every + # second it spends queued for it is pure outage. + with self.assertRaisesRegex( + RuntimeError, + "qbit ledger writer lease is held by prism-coordinator epoch=1", + ): + beta.actor.calls[0].value() + self.assertEqual( + harness.lease_holder_session(), + "heartbeat-v1:alpha-1", + ) + + +class OrphanedLeaseLockFailoverBoundTests(unittest.TestCase): + """The bound #123's fix must establish. Does not hold until #123 is fixed.""" + + def test_successor_startup_must_not_wait_unboundedly(self) -> None: + """A successor must fail fast and visibly, not queue for hours. + + #123's proposed fix runs the startup lease upsert with + ``SET LOCAL lock_timeout`` and a bounded retry. Under that fix the + successor's acquisition ends within a bounded time — by acquiring, or + by raising something an operator can see — rather than waiting on a + lock nobody will ever release. + + The successor is driven through every deadline it reaches rather than + drained once, so a fix whose bounded wait spans several short + timeouts satisfies this as readily as one that fails on the first. + Asserting anything narrower would leave the person who lands #123 + looking at a red test their correct fix did not turn green, and + reaching for the same weakening this test exists to refuse. + + This assertion was deliberately written against the fixed behaviour + rather than against the behaviour of the day. It carried an + ``expectedFailure`` marker until #123 was fixed; the marker is gone + because the bound now holds, which is the removal this docstring + asked the fixing commit to make. The assertion is otherwise + untouched. + """ + with LeaseHarness() as harness: + drive_orphaned_lock_interleaving(harness) + beta = harness.coordinators["beta"] + + finished = drive_successor_to_quiescence(harness, beta) + + self.assertTrue( + finished, + "the successor's PsqlShareLedger.__init__ is still waiting on " + f"the orphaned lease-tuple lock after " + f"{FAILOVER_HORIZON_SECONDS / 3600:g} virtual hours, with no " + "deadline left anywhere that could release it. #123 has " + "regressed: the startup lease upsert is reaching the tuple " + "lock without a lock_timeout again, so nothing bounds the " + "wait.", + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_prism_lease_renewal_vs_expiry_claim.py b/tests/test_prism_lease_renewal_vs_expiry_claim.py new file mode 100644 index 000000000..1a4534f32 --- /dev/null +++ b/tests/test_prism_lease_renewal_vs_expiry_claim.py @@ -0,0 +1,564 @@ +#!/usr/bin/env python3 +"""Writer-lease heartbeat renewal racing a competing expiry claim. + +``PsqlShareLedger.verify_writer_lease_guard_session`` renews the lease TTL +with ``FOR NO KEY UPDATE SKIP LOCKED`` so a heartbeat never queues behind a +fenced write holding the ``qbit_ledger_writer_lease`` tuple lock. Declining +to queue buys liveness but loses the re-evaluation the old locking renewal +performed after a contending transaction committed: once the committed row's +TTL has lapsed, the lock the heartbeat skipped may be a different-identity +``_try_acquire_writer_lease`` taking the row through its expiry CAS, and the +stale committed snapshot would keep naming this session right up until that +claim commits. The defect shape this module covers is that gap: a heartbeat +that reads its own stale token off an expired, foreign-locked row and calls +itself alive would authorize a fenced-out writer. + +The production docstring states four invariants, and each gets a scenario: + +1. Uncontended, the heartbeat renews: ``renewed_count == 1`` and the + committed row's ``lease_expires_at`` moves a full TTL ahead of the + virtual clock. +2. Expired row plus a lock held by a different-identity claimant fails + closed ("expired and its renewal was lock-blocked"), and it is the + *combination* that raises: the same foreign lock over an unexpired row + verifies fine with ``renewed_count == 0``. +3. The one ambiguous result shape — expired, renewal skipped, no + attributable locker — earns exactly one recheck on a fresh snapshot, + because this coordinator's own fenced write committing between the first + statement's MVCC snapshot and its live ``pg_stat_activity`` probe + presents identically to a competing claim. The recheck resolves it and + the verification returns instead of restart-looping a healthy writer. +4. A claim that commits wins: the next verification raises "writer lease is + not active" instead of reporting a renewal. + +The competing claim is a real second coordinator at the next writer epoch +running the shipped startup acquisition. Its expiry CAS executes and takes +the tuple lock, and the harness's ``done:acquire`` checkpoint sits between +statement execution and the transaction's commit, so parking the claimant +there holds the claim in flight — executed, locking, uncommitted — for as +long as the scenario needs. +""" + +from __future__ import annotations + +import unittest +from datetime import timedelta + +from tests.prism_concurrency_harness import ( + LeaseHarness, + assert_deterministic, +) + +LEASE_TTL_SECONDS = 60.0 + +# Push the virtual clock just past lease_expires_at. The exact overshoot is +# irrelevant; the fail-closed branch triggers on `lease_expires_at <= +# clock_timestamp()`, not on how stale the row is. +EXPIRY_LAPSE_SECONDS = LEASE_TTL_SECONDS + 1.0 + +# For the unexpired control: far enough in that time visibly passed, far +# enough out that the row is neither expired nor inside the authority margin +# (TTL/2 = 30s), so the success path exercised is plain "skip and trust the +# committed row" with no deferral semantics in play. +UNEXPIRED_ADVANCE_SECONDS = 20.0 + +# Budget for the coordinator's own fenced write in the recheck scenario. Any +# armed deadline works; a deadline is what makes the client wrap the +# statement in an explicit transaction whose COMMIT is a separate message, +# giving the scenario a precommit stop inside the snapshot-to-probe window. +LANDING_BUDGET_SECONDS = 30.0 + + +def drive_expired_lock_blocked_renewal(harness: LeaseHarness) -> dict[str, object]: + """Scenario 2's raise: expired row, tuple lock held by a foreign claim. + + 1. ``alpha`` starts and holds the lease; the clock then lapses the TTL. + 2. ``claimant``, at the next writer epoch, starts up. Its advisory-guard + key differs from ``alpha``'s, so it reaches the lease upsert, whose + expiry CAS executes and takes the tuple lock. It is parked at + ``done:acquire``: the claim is in flight and uncommitted. + 3. ``alpha``'s heartbeat verification runs. Its ``SKIP LOCKED`` renewal + skips, the committed snapshot still names ``alpha``, and the row is + expired — the shape whose meaning one statement cannot decide, so the + verification spends its one recheck on a fresh snapshot, sees the + same contended expiry, and fails closed. + """ + alpha = harness.coordinator("alpha", writer_epoch=1) + alpha.start() + harness.run_until(alpha, "done:startup") + row_at_startup = harness.lease_row() + assert row_at_startup is not None + + harness.advance(EXPIRY_LAPSE_SECONDS) + + claimant = harness.coordinator("claimant", writer_epoch=2) + claimant.start() + harness.run_until(claimant, "claimant.begin:acquire") + harness.run_until(claimant, "claimant.done:acquire") + claim_tx = harness.server.lease_lock_holder + + verify = alpha.submit( + alpha.ledger.verify_writer_lease_guard_session, + label="verify", + ) + harness.run_until(alpha, "done:verify") + + row = harness.lease_row() + assert row is not None + return { + "claim_took_lock": claim_tx is not None, + "claim_still_holds_lock": harness.server.lease_lock_holder is claim_tx, + "claim_uncommitted": not claimant.started, + "verify_raised": verify.error is not None, + "verify_error": str(verify.error), + "committed_row_session": row.writer_session_token, + "committed_row_expired": row.lease_expires_at <= harness.clock.now(), + "committed_row_unrenewed": ( + row.lease_expires_at == row_at_startup.lease_expires_at + ), + "statement_kinds": harness.statement_kinds(), + } + + +def drive_unexpired_lock_blocked_renewal(harness: LeaseHarness) -> dict[str, object]: + """Scenario 2's control: the identical foreign lock over a live row. + + Same claimant, same parked uncommitted acquire holding the tuple lock; + the only difference is that the committed row has not expired (the + claimant's upsert therefore observed the holder instead of running the + expiry CAS — either way its statement locks the row until commit). The + skipped renewal must be trusted on the committed row's own validity. + """ + alpha = harness.coordinator("alpha", writer_epoch=1) + alpha.start() + harness.run_until(alpha, "done:startup") + row_at_startup = harness.lease_row() + assert row_at_startup is not None + + harness.advance(UNEXPIRED_ADVANCE_SECONDS) + + claimant = harness.coordinator("claimant", writer_epoch=2) + claimant.start() + harness.run_until(claimant, "claimant.begin:acquire") + harness.run_until(claimant, "claimant.done:acquire") + claim_tx = harness.server.lease_lock_holder + + verify = alpha.submit( + alpha.ledger.verify_writer_lease_guard_session, + label="verify", + ) + harness.run_until(alpha, "done:verify") + + row = harness.lease_row() + assert row is not None + return { + "claim_still_holds_lock": ( + claim_tx is not None + and harness.server.lease_lock_holder is claim_tx + ), + "verify_raised": verify.error is not None, + "result": verify.result, + "committed_row_session": row.writer_session_token, + "committed_row_unrenewed": ( + row.lease_expires_at == row_at_startup.lease_expires_at + ), + "statement_kinds": harness.statement_kinds(), + } + + +def drive_own_write_attribution_recheck(harness: LeaseHarness) -> dict[str, object]: + """Scenario 3: the coordinator's own commit lands inside the probe gap. + + ``alpha``'s deadline-scoped fenced write has executed its statement — + holding the tuple lock, its renewal staged — but has not yet sent + COMMIT. The heartbeat verification's first statement takes its MVCC + snapshot in that window: expired row, renewal skipped. The scenario then + commits the fenced write at the ``guard.snapshot:verify`` checkpoint, + exactly between that snapshot and the statement's live + ``pg_stat_activity`` probe, so the probe finds the locker's + ``backend_xid`` already cleared: an own write that just landed is now + indistinguishable from a competing claim. The verification must resolve + this with one recheck, not a raise. + """ + alpha = harness.coordinator("alpha", writer_epoch=1) + alpha.start() + harness.run_until(alpha, "done:startup") + + harness.advance(EXPIRY_LAPSE_SECONDS) + + def fenced_landing() -> object: + # Renewal under an armed deadline: the smallest shipped statement + # with the fenced-write transaction shape (BEGIN / statement / + # separate COMMIT) that locks and rewrites the lease tuple. + with alpha.ledger.operation_timeout(LANDING_BUDGET_SECONDS): + return alpha.ledger.renew_writer_lease() + + alpha.submit(fenced_landing, label="landing") + harness.run_until(alpha, "alpha.precommit") + landing_tx = harness.server.lease_lock_holder + row_in_snapshot = harness.lease_row() + assert row_in_snapshot is not None + statements_before = len(harness.server.statements) + + # The heartbeat runs on its own thread inside the coordinator process; + # alpha's actor is parked inside the fenced write, so the verification + # needs its own actor to interleave with it. + progress: list[str] = [] + heartbeat = harness.scheduler.actor("heartbeat") + verify = heartbeat.submit( + lambda: alpha.ledger.verify_writer_lease_guard_session( + on_statement_progress=lambda: progress.append("round-trip"), + ), + label="verify", + ) + harness.run_until(heartbeat, "alpha.guard.snapshot:verify") + + # What the first statement's snapshot saw: the documented ambiguous + # inputs, minus the locker attribution the commit is about to spoil. + first_statement_shape = { + "row_expired": row_in_snapshot.lease_expires_at <= harness.clock.now(), + "row_names_this_session": ( + row_in_snapshot.writer_session_token == alpha.session_token + ), + "renewal_lock_blocked": ( + landing_tx is not None + and harness.server.lease_lock_holder is landing_tx + ), + "row_xmax_is_own_write": ( + landing_tx is not None and row_in_snapshot.xmax == landing_tx.xid + ), + } + + # The misfire: the own fenced write commits between the snapshot and the + # pg_stat_activity probe, clearing its backend_xid while the snapshot + # still shows the expired row that write had locked. + harness.run_until(alpha, "done:landing") + harness.run_until(heartbeat, "done:verify") + + row = harness.lease_row() + assert row is not None + return { + "first_statement_shape": first_statement_shape, + "verify_raised": verify.error is not None, + "result": verify.result, + "progress_calls": len(progress), + "verify_statements_run": ( + len(harness.server.statements) - statements_before + ), + "statement_kinds": harness.statement_kinds(), + "lease_holder_session": harness.lease_holder_session(), + "lease_unexpired_after": ( + row.lease_expires_at > harness.clock.now() + ), + } + + +def drive_committed_expiry_claim(harness: LeaseHarness) -> dict[str, object]: + """Scenario 4: the competing claim commits and this session is fenced.""" + alpha = harness.coordinator("alpha", writer_epoch=1) + alpha.start() + harness.run_until(alpha, "done:startup") + + harness.advance(EXPIRY_LAPSE_SECONDS) + + claimant = harness.coordinator("claimant", writer_epoch=2) + claimant.start() + # Run the claimant's startup to completion: its expiry CAS commits and + # the singleton row now names the claimant's session. + harness.run_until(claimant, "done:startup") + + verify = alpha.submit( + alpha.ledger.verify_writer_lease_guard_session, + label="verify", + ) + harness.run_until(alpha, "done:verify") + + return { + "claimant_started": claimant.started, + "verify_raised": verify.error is not None, + "verify_error": str(verify.error), + "lease_holder_session": harness.lease_holder_session(), + "statement_kinds": harness.statement_kinds(), + } + + +class UncontendedRenewalTests(unittest.TestCase): + """Scenario 1: a healthy heartbeat renews without contention.""" + + def test_heartbeat_renews_a_full_ttl_ahead_of_the_clock(self) -> None: + """The committed row, not just the return value, must move. + + The return value alone could report a renewal whose UPDATE matched + zero rows; asserting ``lease_expires_at`` and ``updated_at`` against + the virtual clock proves the renewal landed on the singleton row and + pushed the TTL a full interval ahead of *now*, not ahead of the old + expiry. + """ + with LeaseHarness(lease_ttl_seconds=LEASE_TTL_SECONDS) as harness: + alpha = harness.coordinator("alpha", writer_epoch=1) + alpha.start() + harness.run_until(alpha, "done:startup") + + # Half a TTL of virtual time passes so the renewed expiry is + # visibly different from the acquisition's. + harness.advance(LEASE_TTL_SECONDS / 2) + + result = alpha.call( + alpha.ledger.verify_writer_lease_guard_session, + label="verify", + ) + self.assertEqual( + result, + { + "backend": "postgres-psql", + "verified_count": 1, + "renewed_count": 1, + "renewal_deferred_to_own_write": False, + }, + ) + + row = harness.lease_row() + assert row is not None + self.assertEqual(row.writer_session_token, "heartbeat-v1:alpha-1") + self.assertEqual( + row.lease_expires_at, + harness.clock.now() + timedelta(seconds=LEASE_TTL_SECONDS), + ) + self.assertEqual(row.updated_at, harness.clock.now()) + self.assertIsNone(row.xmax, "the renewal's transaction committed") + + +class ExpiryClaimFailClosedTests(unittest.TestCase): + """Scenario 2: expired + foreign-locked raises; either alone does not.""" + + def test_expired_row_with_foreign_lock_fails_closed(self) -> None: + with LeaseHarness(lease_ttl_seconds=LEASE_TTL_SECONDS) as harness: + observed = drive_expired_lock_blocked_renewal(harness) + + self.assertTrue(observed["claim_took_lock"]) + self.assertTrue( + observed["claim_still_holds_lock"], + "the competing claim must still be in flight when the " + "heartbeat verifies; a committed claim is scenario 4", + ) + self.assertTrue(observed["claim_uncommitted"]) + self.assertTrue(observed["verify_raised"]) + self.assertRegex( + str(observed["verify_error"]), + "expired and its renewal was lock-blocked", + ) + # The stale committed snapshot is exactly what makes the raise + # necessary: it still names this session, so a token read alone + # would have called the fenced-out writer alive. + self.assertEqual( + observed["committed_row_session"], + "heartbeat-v1:alpha-1", + ) + self.assertTrue(observed["committed_row_expired"]) + self.assertTrue(observed["committed_row_unrenewed"]) + # Two verify statements: the ambiguous shape (the claimant's + # backend is not attributable to alpha's pool) earns the one + # recheck, whose fresh snapshot sees the same contended expiry + # and still fails closed — the docstring's residual outcome. + self.assertEqual( + observed["statement_kinds"], + ["acquire", "acquire", "verify", "verify"], + ) + + def test_same_lock_over_an_unexpired_row_verifies_clean(self) -> None: + """The control that pins the raise on the *combination*. + + If the foreign lock alone raised, every heartbeat racing an ordinary + fenced write would kill a healthy coordinator; the non-blocking + renewal is only allowed to distrust a skip once the committed row's + own validity has lapsed. Identical claimant, identical held lock, + unexpired row: the verification must succeed, renewing nothing. + """ + with LeaseHarness(lease_ttl_seconds=LEASE_TTL_SECONDS) as harness: + observed = drive_unexpired_lock_blocked_renewal(harness) + + self.assertTrue(observed["claim_still_holds_lock"]) + self.assertFalse(observed["verify_raised"]) + self.assertEqual( + observed["result"], + { + "backend": "postgres-psql", + "verified_count": 1, + "renewed_count": 0, + "renewal_deferred_to_own_write": False, + }, + ) + self.assertEqual( + observed["committed_row_session"], + "heartbeat-v1:alpha-1", + ) + self.assertTrue( + observed["committed_row_unrenewed"], + "a skipped renewal must leave the committed expiry alone", + ) + # Unexpired means unambiguous: no recheck, one verify statement. + self.assertEqual( + observed["statement_kinds"], + ["acquire", "acquire", "verify"], + ) + + def test_fail_closed_interleaving_is_order_stable(self) -> None: + """A fail-closed path that only usually triggers is a split brain. + + The exact schedule is the assertion: the claim executes and parks + uncommitted before the verification starts, and the verification's + two statements (first read, one recheck) both run against the still + contended row. Any drift in this trace means the scenario stopped + testing the interleaving it claims to test. + """ + run = assert_deterministic( + self, + drive_expired_lock_blocked_renewal, + repeats=25, + lease_ttl_seconds=LEASE_TTL_SECONDS, + ) + + self.assertEqual( + list(run.trace), + [ + "alpha@alpha.begin:acquire", + "alpha@alpha.done:acquire", + # Since #123 the acquire/adopt statements carry an acquisition + # deadline, so each commits explicitly and offers a precommit stop. + "alpha@alpha.precommit", + "alpha@done:startup", + "claimant@claimant.begin:acquire", + "claimant@claimant.done:acquire", + "alpha@alpha.guard.begin:verify", + "alpha@alpha.guard.snapshot:verify", + "alpha@alpha.guard.done:verify", + "alpha@alpha.guard.begin:verify", + "alpha@alpha.guard.snapshot:verify", + "alpha@alpha.guard.done:verify", + "alpha@done:verify", + ], + ) + + +class AttributionRecheckTests(unittest.TestCase): + """Scenario 3: the ambiguous shape earns exactly one fresh-snapshot look.""" + + def test_own_commit_in_probe_window_resolves_by_recheck(self) -> None: + with LeaseHarness(lease_ttl_seconds=LEASE_TTL_SECONDS) as harness: + observed = drive_own_write_attribution_recheck(harness) + + # The first statement really produced the ambiguous shape: the + # snapshot row was expired, still named this session, and its + # locker was this coordinator's own in-flight write — which the + # live probe could no longer attribute after the commit. + self.assertEqual( + observed["first_statement_shape"], + { + "row_expired": True, + "row_names_this_session": True, + "renewal_lock_blocked": True, + "row_xmax_is_own_write": True, + }, + ) + # The verification returned; raising here is the misfire the + # recheck exists to prevent — it would roll back nothing (the + # write already committed) and restart-loop a healthy writer on + # every fenced write slow enough to outlast the TTL. + self.assertFalse(observed["verify_raised"]) + self.assertEqual( + observed["result"], + { + "backend": "postgres-psql", + "verified_count": 1, + "renewed_count": 1, + "renewal_deferred_to_own_write": False, + }, + ) + # WRITER_LEASE_VERIFICATION_MAX_STATEMENTS is 2: exactly one + # recheck ran, and it announced the first statement's completed + # round trip to the liveness monitor exactly once. + self.assertEqual(observed["progress_calls"], 1) + self.assertEqual(observed["verify_statements_run"], 2) + self.assertEqual( + observed["statement_kinds"], + ["acquire", "renew", "verify", "verify"], + ) + # The recheck's fresh snapshot saw the committed refresh and the + # second statement's renewal landed on the uncontended row. + self.assertEqual( + observed["lease_holder_session"], + "heartbeat-v1:alpha-1", + ) + self.assertTrue(observed["lease_unexpired_after"]) + + def test_recheck_interleaving_is_order_stable(self) -> None: + """The misfire window is two checkpoints wide; pin the schedule. + + The own write must commit after ``guard.snapshot:verify`` (the first + statement's MVCC snapshot is taken) and before that statement's live + probe runs. If the trace drifts — the commit landing before the + snapshot, or the recheck not running — the test would silently stop + covering the one-direction misfire the production docstring names. + """ + run = assert_deterministic( + self, + drive_own_write_attribution_recheck, + repeats=25, + lease_ttl_seconds=LEASE_TTL_SECONDS, + ) + + self.assertEqual( + list(run.trace), + [ + "alpha@alpha.begin:acquire", + "alpha@alpha.done:acquire", + "alpha@alpha.precommit", + "alpha@done:startup", + "alpha@alpha.begin:renew", + "alpha@alpha.done:renew", + "alpha@alpha.precommit", + "heartbeat@alpha.guard.begin:verify", + "heartbeat@alpha.guard.snapshot:verify", + "alpha@done:landing", + "heartbeat@alpha.guard.done:verify", + "heartbeat@alpha.guard.begin:verify", + "heartbeat@alpha.guard.snapshot:verify", + "heartbeat@alpha.guard.done:verify", + "heartbeat@done:verify", + ], + ) + + +class CommittedExpiryClaimTests(unittest.TestCase): + """Scenario 4: a claim that wins fences this session outright.""" + + def test_committed_claim_fences_the_session(self) -> None: + """Losing the row must read as fencing, not as a lock-blocked retry. + + Once the claimant's CAS commits, the committed row no longer names + this session, which is conclusive — not the ambiguous shape — so + the verification raises the identity failure in one statement + rather than spending its recheck or, worse, reporting a renewal + against a row it no longer owns. + """ + with LeaseHarness(lease_ttl_seconds=LEASE_TTL_SECONDS) as harness: + observed = drive_committed_expiry_claim(harness) + + self.assertTrue(observed["claimant_started"]) + self.assertEqual( + observed["lease_holder_session"], + "heartbeat-v1:claimant-2", + ) + self.assertTrue(observed["verify_raised"]) + self.assertRegex( + str(observed["verify_error"]), + "writer lease is not active", + ) + self.assertEqual( + observed["statement_kinds"], + ["acquire", "acquire", "verify"], + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_prism_lease_takeover.py b/tests/test_prism_lease_takeover.py new file mode 100644 index 000000000..2f7171c9f --- /dev/null +++ b/tests/test_prism_lease_takeover.py @@ -0,0 +1,746 @@ +#!/usr/bin/env python3 +"""Writer-lease takeover: the proof fast adoption owes before it fences a peer. + +Fast adoption is the only path by which one PRISM coordinator takes the +writer lease from a same-identity predecessor *before* the TTL has run out, +and therefore the only path that can hand two live processes overlapping +authority if its proof is wrong. The proof has three independent parts, and +each one fails in a way that is invisible to a single-process test: + +``_initialize_writer_lease_guard`` + The successor must hold the predecessor's PostgreSQL session advisory + lock. Two coordinators sharing ``(writer_id, writer_epoch)`` hash to the + same key, so a live predecessor keeps its successor out of the lease + upsert entirely: the successor never observes the row, never runs a CAS, + and spins. A guard that failed open would not change any *outcome* a + functional test can see -- adoption would still work -- so the only + executable statement of it is that no statement reaches the server. + +``_writer_lease_adoption_wait_seconds`` + Silence is measured from two edges and the later one wins: the lease + row's ``updated_at`` age, and the time since *this* process acquired the + guard. The guard edge is the one that is easy to drop and impossible to + notice, because dropping it is only wrong when the row is *already* + stale: a long fenced transaction such as ``persist_accepted_block`` + withholds ``updated_at`` refreshes, so the row can look minutes silent at + the instant the predecessor dies. A row-only implementation adopts + instantly in exactly that case and gives the dying predecessor none of + its heartbeat failure budget to self-fence. The second scenario here + constructs precisely that state -- a row far staler than the silence + interval -- and requires a full interval of waiting anyway. + +``_try_adopt_writer_lease`` + The CAS is the final database fence. Winning it rewrites the session + token, which fences every later predecessor mutation. Losing it must not + be terminal: the renewal that beat it may have been the predecessor's + final act before dying, so refusing that token permanently would recreate + the whole-TTL outage adoption exists to avoid. The successor must + re-observe and wait a *fresh* full silence interval instead. + +The last scenario covers the other half of the identity rule. Fast adoption +is deliberately unavailable across writer epochs -- ``_can_wait_for_writer_lease`` +requires an exact ``(writer_id, writer_epoch)`` match, and a different epoch +hashes to a different advisory key, so nothing in this process can prove +anything about the holder. A different epoch must therefore refuse loudly +while the lease is live, and take it through the ordinary expiry CAS once it +has lapsed. + +Every scenario is driven through the shipped code: the real retry loop, the +real adoption arithmetic, the real CAS SQL. Nothing is patched; the harness +substitutes PostgreSQL, the clock and the retry sleep at the constructor +seams ``PsqlShareLedger`` already exposes. +""" + +from __future__ import annotations + +import unittest + +from tests.prism_concurrency_harness import ( + Coordinator, + LeaseHarness, + assert_deterministic, +) + +# A long TTL keeps every scenario inside the *unexpired* half of the lease +# lifecycle, which is the only half where adoption is reachable at all: an +# expired row is taken by the ordinary expiry CAS in `_try_acquire_writer_lease` +# and never consults the silence arithmetic. +LEASE_TTL_SECONDS = 600.0 + +# Small and explicit so the waiting arithmetic below is readable as arithmetic +# rather than taken on trust. Production defaults to 1.0s +# (`DEFAULT_WRITER_LEASE_ADOPTION_SILENCE_SECONDS`); the mechanism does not +# depend on the value, only on which edge it is measured from. +ADOPTION_SILENCE_SECONDS = 5.0 + +# How stale the lease row is made before the successor ever starts. Chosen to +# be far larger than the silence interval, so a row-edge-only implementation +# would compute a wait of zero and adopt on its first observation. +PREDECESSOR_SILENT_SECONDS = 120.0 + +# `_lease_retry_min_sleep_seconds` is `min(0.25, lease_retry_max_sleep_seconds)`, +# and the guard spin sleeps exactly that between attempts. +GUARD_SPIN_SLEEP_SECONDS = 0.25 + +# How many times the gated successor is made to go round the guard spin. Any +# number above one works; more than one is the point, because the warning is +# asserted to be printed once for the whole wait rather than once per lap. +GUARD_SPIN_LAPS = 4 + +GUARD_HELD_WARNING = ( + "prism ledger writer guard held by a live same-identity coordinator; " + "waiting before lease acquisition" +) + +HARNESS_KWARGS = { + "lease_ttl_seconds": LEASE_TTL_SECONDS, + "lease_adoption_silence_seconds": ADOPTION_SILENCE_SECONDS, +} + + +def start_predecessor(harness: LeaseHarness) -> Coordinator: + """Bring up the lease holder every scenario takes over from.""" + alpha = harness.coordinator("alpha", writer_epoch=1) + alpha.start() + harness.run_until(alpha, "done:startup") + return alpha + + +def drive_guard_gated_successor(harness: LeaseHarness) -> dict[str, object]: + """Start a same-identity successor while the predecessor is alive. + + ``beta`` shares ``alpha``'s ``(writer_id, writer_epoch)``, so + ``_writer_lease_advisory_lock_key`` hands both the same key and ``beta`` + cannot get past ``_initialize_writer_lease_guard``. It is stepped round + the spin several times to show the wait is a loop rather than a single + stall, and that the loop reaches the server not at all. + """ + alpha = start_predecessor(harness) + statements_before = harness.statement_kinds() + + beta = harness.coordinator("beta", writer_epoch=1) + beta.start() + harness.run_until_blocked(beta, f"sleep:{GUARD_SPIN_SLEEP_SECONDS:g}") + for _ in range(GUARD_SPIN_LAPS - 1): + harness.advance_to_next_deadline() + harness.run_until_blocked(beta, f"sleep:{GUARD_SPIN_SLEEP_SECONDS:g}") + + return { + "statements_before": statements_before, + "statements_after": harness.statement_kinds(), + "successor_stops": [ + entry.split("@", 1)[1] + for entry in harness.trace + if entry.startswith(f"{beta.name}@") + ], + "successor_started": beta.started, + "sleeps": list(harness.sleeps), + "same_advisory_key": ( + beta.guard.advisory_lock_key == alpha.guard.advisory_lock_key + ), + "predecessor_guard_held": alpha.guard.held, + "successor_guard_held": beta.guard.held, + "warning_count": harness.output.count(GUARD_HELD_WARNING), + } + + +def drive_fast_adoption(harness: LeaseHarness) -> dict[str, object]: + """Adopt from a predecessor whose lease row is already long stale. + + The clock is advanced ``PREDECESSOR_SILENT_SECONDS`` *before* the + predecessor's guard is released, which is the shape a long fenced + transaction produces: the row has not been refreshed for minutes, yet the + predecessor only stopped being able to act at the instant the guard went. + Everything the successor may conclude about silence therefore has to be + measured from its own guard acquisition. + """ + alpha = start_predecessor(harness) + harness.advance(PREDECESSOR_SILENT_SECONDS) + alpha.guard.close() + + beta = harness.coordinator("beta", writer_epoch=1) + beta.start() + # The successor takes the guard, observes the row, and finds it cannot + # adopt yet. This is the first moment it could have adopted if silence + # were read off the row alone. + harness.run_until_blocked(beta, f"sleep:{ADOPTION_SILENCE_SECONDS:g}") + observed_row = harness.lease_row() + row_age_at_first_observation = ( + harness.clock.now() - observed_row.updated_at + ).total_seconds() + statements_at_first_observation = harness.statement_kinds() + monotonic_at_first_observation = harness.clock.monotonic() + + harness.advance_to_next_deadline() + harness.run_until(beta, "done:startup") + + adopted_row = harness.lease_row() + return { + "row_age_at_first_observation": row_age_at_first_observation, + "statements_at_first_observation": statements_at_first_observation, + "waited_seconds": ( + harness.clock.monotonic() - monotonic_at_first_observation + ), + "sleeps": list(harness.sleeps), + "statements": harness.statement_kinds(), + "session_token": adopted_row.writer_session_token, + "predecessor_token": alpha.session_token, + "successor_token": beta.session_token, + "lease_expires_at": str(adopted_row.lease_expires_at), + "ttl_ahead_seconds": ( + adopted_row.lease_expires_at - harness.clock.now() + ).total_seconds(), + "adoption_announced": ( + "adopted from same-identity predecessor" in harness.output + ), + } + + +def drive_lost_adoption_cas(harness: LeaseHarness) -> dict[str, object]: + """Land a predecessor renewal between the successor's observation and CAS. + + The interleaving is forced by PostgreSQL's own ordering rather than by + scheduling luck. At ``beta.done:acquire`` the successor's transaction + still holds the lease tuple lock, so the predecessor's renewal queues + behind it; letting the successor run on to its COMMIT releases the lock, + at which point the renewal lands and changes ``updated_at`` -- the exact + column the adoption CAS compares. The successor is then at + ``beta.begin:adopt`` with a stale observation and no way to know it. + + Since #123's fix the observation is deadline-scoped, so that COMMIT is an + explicit message and reaching it takes a ``beta.precommit`` stop and then + a step, where an autocommit statement committed itself in one. The + interleaving is the same one and lands at the same place; only the number + of scheduler steps needed to reach it changed. + """ + alpha = start_predecessor(harness) + harness.advance(PREDECESSOR_SILENT_SECONDS) + alpha.guard.close() + + beta = harness.coordinator("beta", writer_epoch=1) + beta.start() + harness.run_until_blocked(beta, f"sleep:{ADOPTION_SILENCE_SECONDS:g}") + harness.advance_to_next_deadline() + + # Re-observation: adoption is now permitted, and the successor holds the + # tuple lock until this statement's autocommit COMMIT. + harness.run_until(beta, "beta.done:acquire") + renewal = alpha.submit(alpha.ledger.renew_writer_lease, label="renew") + harness.run_until_blocked(alpha, "alpha.lockwait:renew") + + # Commit the observation and carry the successor to the CAS statement, + # which has begun a transaction but not yet taken the lock. + harness.run_until(beta, "beta.precommit") + harness.step(beta) + stop_before_cas = beta.actor.stop + harness.run_until(alpha, "done:renew") + + # The CAS now runs against a row the renewal moved out from under it. + harness.run_until_blocked(beta, f"sleep:{ADOPTION_SILENCE_SECONDS:g}") + statements_after_lost_cas = harness.statement_kinds() + sleeps_after_lost_cas = list(harness.sleeps) + monotonic_after_lost_cas = harness.clock.monotonic() + + harness.advance_to_next_deadline() + harness.run_until(beta, "done:startup") + + adopted_row = harness.lease_row() + return { + "stop_before_cas": stop_before_cas, + "renewal": renewal.value(), + "statements_after_lost_cas": statements_after_lost_cas, + "sleeps_after_lost_cas": sleeps_after_lost_cas, + "waited_after_lost_cas": ( + harness.clock.monotonic() - monotonic_after_lost_cas + ), + "sleeps": list(harness.sleeps), + "statements": harness.statement_kinds(), + "session_token": adopted_row.writer_session_token, + "successor_token": beta.session_token, + } + + +class WriterGuardGatesTakeoverTests(unittest.TestCase): + """A live same-identity predecessor keeps its successor out entirely.""" + + def test_successor_never_reaches_the_lease_upsert(self) -> None: + """Not "does not adopt" -- does not *observe*. + + The advisory guard is the only part of the proof that speaks to what + the predecessor can still do outside the database, so it has to hold + before any lease statement runs, not merely before the CAS. Asserting + that no statement reached the server is the difference between a + guard that gates takeover and one that only delays it. + """ + with LeaseHarness(**HARNESS_KWARGS) as harness: + observed = drive_guard_gated_successor(harness) + + self.assertTrue( + observed["same_advisory_key"], + "two coordinators at the same (writer_id, writer_epoch) must " + "contend for one advisory key, or the guard proves nothing", + ) + self.assertTrue(observed["predecessor_guard_held"]) + self.assertFalse(observed["successor_guard_held"]) + + self.assertEqual( + observed["statements_after"], + observed["statements_before"], + "the gated successor must not have executed any lease " + "statement; it is held off before the upsert, not at it", + ) + self.assertEqual(observed["statements_after"], ["acquire"]) + self.assertNotIn( + "beta.begin:acquire", + observed["successor_stops"], + "reaching begin:acquire would mean the successor observed the " + "lease row while its predecessor was still able to act", + ) + self.assertEqual( + observed["successor_stops"], + [f"sleep:{GUARD_SPIN_SLEEP_SECONDS:g}"] * GUARD_SPIN_LAPS, + ) + self.assertEqual( + observed["sleeps"], + [("beta", GUARD_SPIN_SLEEP_SECONDS)] * GUARD_SPIN_LAPS, + ) + self.assertFalse( + observed["successor_started"], + "PsqlShareLedger.__init__ has not returned", + ) + + def test_guard_contention_is_reported_once_not_once_per_retry(self) -> None: + """The spin is unbounded; the log line must not be. + + ``_initialize_writer_lease_guard`` retries every 0.25s for as long as + the predecessor lives, so a per-attempt warning would emit four lines + a second for the whole of a healthy coordinator's life and bury the + one line an operator needs. The ``warned`` latch is the behaviour, and + a spin of several laps producing one line is the only way to see it. + """ + with LeaseHarness(**HARNESS_KWARGS) as harness: + observed = drive_guard_gated_successor(harness) + + self.assertEqual(len(observed["sleeps"]), GUARD_SPIN_LAPS) + self.assertEqual(observed["warning_count"], 1) + + def test_releasing_the_guard_lets_the_gated_successor_proceed(self) -> None: + """Control: the successor is held by the guard, not by the schedule. + + Without this, the parked state above is equally consistent with a + scenario that simply never gave the successor a chance to run. Closing + the predecessor's guard and doing nothing else must be enough to carry + the same successor all the way through adoption. + """ + with LeaseHarness(**HARNESS_KWARGS) as harness: + drive_guard_gated_successor(harness) + alpha = harness.coordinators["alpha"] + beta = harness.coordinators["beta"] + + alpha.guard.close() + + harness.advance_to_next_deadline() + harness.run_until_blocked( + beta, f"sleep:{ADOPTION_SILENCE_SECONDS:g}" + ) + self.assertTrue( + beta.guard.held, + "the successor took the guard on its next lap", + ) + harness.advance_to_next_deadline() + harness.run_until(beta, "done:startup") + + self.assertTrue(beta.started) + self.assertEqual( + harness.lease_holder_session(), + beta.session_token, + ) + + +class AdoptionSilenceIsMeasuredFromGuardAcquisitionTests(unittest.TestCase): + """The guard edge, not the row edge, is what bounds the wait.""" + + def test_successor_waits_a_full_interval_after_taking_the_guard(self) -> None: + """A row that is already minutes stale buys the successor nothing. + + This is the case the two-edge rule exists for. The predecessor's + ``updated_at`` is ``PREDECESSOR_SILENT_SECONDS`` old -- far past the + silence interval -- when the successor takes the guard, so the row + edge contributes a wait of zero. If the row edge were the only one, + the successor would adopt on its first observation and the dying + predecessor would get none of its heartbeat failure budget to + self-fence. The wait must be a full interval measured from this + process's own guard acquisition instead. + """ + with LeaseHarness(**HARNESS_KWARGS) as harness: + observed = drive_fast_adoption(harness) + + self.assertGreater( + observed["row_age_at_first_observation"], + ADOPTION_SILENCE_SECONDS, + "the row must already be staler than the silence interval, or " + "this scenario is not testing the guard edge at all", + ) + self.assertEqual( + observed["row_age_at_first_observation"], + PREDECESSOR_SILENT_SECONDS, + ) + self.assertEqual( + observed["sleeps"], + [("beta", ADOPTION_SILENCE_SECONDS)], + "exactly one wait, of exactly one full silence interval", + ) + self.assertEqual( + observed["waited_seconds"], + ADOPTION_SILENCE_SECONDS, + ) + + def test_observation_precedes_the_cas_by_a_whole_interval(self) -> None: + """The statement order is the executable form of the arithmetic. + + A wait is only a fence if nothing was written during it. The successor + must observe, wait, re-observe, and only then run the CAS -- so every + statement before the first ``adopt`` is a read-only ``acquire`` + attempt, and there are two of them: one either side of the interval. + """ + with LeaseHarness(**HARNESS_KWARGS) as harness: + observed = drive_fast_adoption(harness) + + self.assertEqual( + observed["statements_at_first_observation"], + ["acquire", "acquire"], + "the predecessor's startup acquire, then the successor's " + "first observation -- and no adopt", + ) + self.assertEqual( + observed["statements"], + ["acquire", "acquire", "acquire", "adopt"], + ) + statements = observed["statements"] + self.assertEqual( + set(statements[: statements.index("adopt")]), + {"acquire"}, + ) + + def test_adoption_schedule_is_order_stable_across_repeated_runs(self) -> None: + """The wait is a property of the schedule, so pin the schedule. + + Everything this class asserts is a claim about *when* statements ran + relative to one another. That claim is only worth as much as the + reproducibility of the interleaving that produced it. + """ + run = assert_deterministic( + self, + drive_fast_adoption, + repeats=25, + **HARNESS_KWARGS, + ) + + self.assertEqual( + list(run.trace), + [ + "alpha@alpha.begin:acquire", + "alpha@alpha.done:acquire", + # Since #123 the acquire/adopt statements carry an acquisition + # deadline, so each commits explicitly and offers a precommit stop. + "alpha@alpha.precommit", + "alpha@done:startup", + "beta@beta.begin:acquire", + "beta@beta.done:acquire", + "beta@beta.precommit", + "beta@sleep:5", + "beta@beta.begin:acquire", + "beta@beta.done:acquire", + "beta@beta.precommit", + "beta@beta.begin:adopt", + "beta@beta.done:adopt", + "beta@beta.precommit", + "beta@done:startup", + ], + ) + + +class AdoptionRewritesTheSessionTokenTests(unittest.TestCase): + """What a won CAS leaves behind, and what it takes away.""" + + def test_adoption_installs_the_successor_for_a_full_ttl(self) -> None: + with LeaseHarness(**HARNESS_KWARGS) as harness: + observed = drive_fast_adoption(harness) + + self.assertEqual( + observed["session_token"], + observed["successor_token"], + ) + self.assertNotEqual( + observed["session_token"], + observed["predecessor_token"], + ) + self.assertEqual( + observed["ttl_ahead_seconds"], + LEASE_TTL_SECONDS, + "adoption installs a full fresh TTL, not the remainder of the " + "predecessor's", + ) + self.assertTrue(observed["adoption_announced"]) + self.assertIn( + "prism ledger writer lease adopted from same-identity " + "predecessor session=heartbeat-v1:alpha-1", + harness.output, + ) + + def test_predecessor_is_fenced_by_the_rewritten_token(self) -> None: + """The CAS is the fence; nothing tells the predecessor it lost. + + A replaced predecessor is still a running process holding a ledger + object that believes it owns the lease. Its renewal must fail the same + way a fenced write does, so it fails fast rather than continuing to + act, and its release must decline to touch a row it no longer owns -- + expiring a successor's live lease would open the window adoption just + closed. + """ + with LeaseHarness(**HARNESS_KWARGS) as harness: + drive_fast_adoption(harness) + alpha = harness.coordinators["alpha"] + beta = harness.coordinators["beta"] + + with self.assertRaisesRegex( + RuntimeError, + "writer lease is not active", + ): + alpha.call(alpha.ledger.renew_writer_lease, label="renew") + + self.assertFalse( + alpha.call(alpha.ledger.release_writer_lease, label="release"), + "a lease already reassigned must not be expired by its " + "predecessor", + ) + self.assertEqual( + harness.lease_holder_session(), + beta.session_token, + ) + row = harness.lease_row() + self.assertGreater(row.lease_expires_at, harness.clock.now()) + self.assertEqual( + harness.statement_kinds(), + ["acquire", "acquire", "acquire", "adopt", "renew", "release"], + "both fenced calls really did reach the server and were " + "refused by the row, not short-circuited in the client", + ) + + +class LostAdoptionCasTests(unittest.TestCase): + """A CAS loss re-observes; it never refuses the token permanently.""" + + def test_lost_cas_re_observes_after_a_fresh_silence_interval(self) -> None: + """Neither give up nor retry immediately -- both are unsafe. + + The renewal that beat the CAS proves the predecessor was alive + moments ago, so retrying straight away would adopt from a process + that had just demonstrated it can still act. Giving up instead would + recreate the whole-TTL outage fast adoption exists to avoid, because + that renewal may equally have been the predecessor's final act. The + comment in ``_ensure_writer_lease`` picks the third option, and this + is it: re-observe, and require a fresh full silence interval -- which + the renewal's own ``updated_at`` now supplies through the row edge. + """ + with LeaseHarness(**HARNESS_KWARGS) as harness: + observed = drive_lost_adoption_cas(harness) + + self.assertEqual(observed["stop_before_cas"], "beta.begin:adopt") + self.assertEqual(observed["renewal"]["renewed_count"], 1) + + self.assertEqual( + observed["statements_after_lost_cas"], + [ + "acquire", # predecessor startup + "acquire", # successor's first observation + "acquire", # successor re-observes after the guard wait + "renew", # predecessor's renewal wins the tuple lock + "adopt", # the CAS, now against a moved updated_at + ], + "the renewal must land between the observation and the CAS, " + "or the CAS was never actually contested", + ) + self.assertEqual( + observed["sleeps_after_lost_cas"], + [ + ("beta", ADOPTION_SILENCE_SECONDS), + ("beta", ADOPTION_SILENCE_SECONDS), + ], + "a second full interval, not a shortened one and not none", + ) + self.assertEqual( + observed["waited_after_lost_cas"], + ADOPTION_SILENCE_SECONDS, + ) + + self.assertEqual( + observed["statements"], + [ + "acquire", + "acquire", + "acquire", + "renew", + "adopt", + "acquire", + "adopt", + ], + "re-observe before the second CAS: adopting on the first " + "observation's token would be adopting on stale evidence", + ) + self.assertEqual( + observed["session_token"], + observed["successor_token"], + "the successor still ends up holding the lease", + ) + + def test_interleaving_is_order_stable_across_repeated_runs(self) -> None: + """A flaky concurrency test is worse than none; prove this one is not.""" + run = assert_deterministic( + self, + drive_lost_adoption_cas, + repeats=25, + **HARNESS_KWARGS, + ) + + self.assertEqual( + list(run.trace), + [ + "alpha@alpha.begin:acquire", + "alpha@alpha.done:acquire", + "alpha@alpha.precommit", + "alpha@done:startup", + "beta@beta.begin:acquire", + "beta@beta.done:acquire", + "beta@beta.precommit", + "beta@sleep:5", + "beta@beta.begin:acquire", + "beta@beta.done:acquire", + "alpha@alpha.begin:renew", + "alpha@alpha.lockwait:renew", + "beta@beta.precommit", + "beta@beta.begin:adopt", + "alpha@alpha.done:renew", + "alpha@done:renew", + "beta@beta.done:adopt", + "beta@beta.precommit", + "beta@sleep:5", + "beta@beta.begin:acquire", + "beta@beta.done:acquire", + "beta@beta.precommit", + "beta@beta.begin:adopt", + "beta@beta.done:adopt", + "beta@beta.precommit", + "beta@done:startup", + ], + ) + + +class WriterEpochFencingTests(unittest.TestCase): + """A different epoch gets the TTL and nothing else.""" + + def test_different_epoch_refuses_an_unexpired_lease(self) -> None: + """Refusing loudly is the only safe answer, and it must be immediate. + + A different epoch hashes to a different advisory key, so it is not + held off by the holder's guard and does reach the lease upsert -- and + having reached it, it can prove nothing at all about whether the + holder is alive. ``_can_wait_for_writer_lease`` therefore requires an + exact identity match, which makes every different-epoch outcome + against a live lease a raise rather than a wait or a CAS. + """ + with LeaseHarness(**HARNESS_KWARGS) as harness: + alpha = start_predecessor(harness) + + gamma = harness.coordinator("gamma", writer_epoch=2) + gamma.start() + harness.run_until(gamma, "done:startup") + + self.assertNotEqual( + gamma.guard.advisory_lock_key, + alpha.guard.advisory_lock_key, + "different epochs must not share an advisory key, or this " + "coordinator would have been gated instead of refused", + ) + self.assertIn( + "gamma@gamma.begin:acquire", + harness.trace, + "the different-epoch guard was acquired and the upsert was " + "reached, so the refusal comes from the identity check and " + "not from guard contention", + ) + # `PsqlShareLedger.__init__` closes what it opened when it raises. + # A refused claimant that kept its advisory lock would gate the + # legitimate holder's own successor for the life of the process. + self.assertFalse(gamma.guard.held) + self.assertFalse( + gamma.started, + "PsqlShareLedger.__init__ must not return", + ) + with self.assertRaisesRegex( + RuntimeError, + "qbit ledger writer lease is held by prism-coordinator epoch=1", + ): + gamma.actor.calls[0].value() + + self.assertEqual( + harness.statement_kinds(), + ["acquire", "acquire"], + "one observation and no adopt: fast adoption is unreachable " + "across epochs", + ) + self.assertEqual(harness.sleeps, []) + self.assertEqual( + harness.lease_holder_session(), + alpha.session_token, + ) + + def test_different_epoch_takes_an_expired_lease_immediately(self) -> None: + """The expiry CAS is the different-epoch path, and it does not wait.""" + with LeaseHarness(**HARNESS_KWARGS) as harness: + alpha = start_predecessor(harness) + + gamma = harness.coordinator("gamma", writer_epoch=2) + gamma.start() + harness.run_until(gamma, "done:startup") + self.assertIsNotNone(gamma.actor.calls[0].error) + + row = harness.lease_row() + remaining = ( + row.lease_expires_at - harness.clock.now() + ).total_seconds() + self.assertEqual(remaining, LEASE_TTL_SECONDS) + harness.advance(remaining + 1.0) + + delta = harness.coordinator("delta", writer_epoch=2) + delta.start() + harness.run_until(delta, "done:startup") + + self.assertTrue(delta.started) + adopted = harness.lease_row() + self.assertEqual(adopted.writer_epoch, 2) + self.assertEqual(adopted.writer_id, alpha.writer_id) + self.assertEqual( + adopted.writer_session_token, + delta.session_token, + ) + self.assertEqual( + harness.statement_kinds(), + ["acquire", "acquire", "acquire"], + "the expiry CAS lands on the first attempt; no adopt " + "statement is involved", + ) + self.assertEqual( + harness.sleeps, + [], + "an expired lease is taken without waiting at all", + ) + self.assertEqual( + adopted.lease_expires_at - harness.clock.now(), + row.lease_expires_at - row.updated_at, + "the claimant installs a full fresh TTL", + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_prism_metrics.py b/tests/test_prism_metrics.py new file mode 100644 index 000000000..b85ff3b97 --- /dev/null +++ b/tests/test_prism_metrics.py @@ -0,0 +1,3480 @@ +#!/usr/bin/env python3 +"""PRISM metrics renderer extraction tests. + +The frozen reference implementations below are byte-copied from the +coordinator formatter bodies as they stood immediately before the PR 80 +metrics extraction (``self`` renamed to ``server``). The full-render parity +test proves the extracted ``MetricsRenderer`` reproduces the pre-extraction +Prometheus document byte-for-byte for identical inputs. +""" + +from __future__ import annotations + +from collections import OrderedDict +from contextlib import redirect_stdout +from decimal import Decimal +import gc +import hashlib +from io import StringIO +from pathlib import Path +import re +import tempfile +import time +import tracemalloc +import unittest +from types import SimpleNamespace +from unittest import mock + +from lab.prism.coordinator_config import ( + DEFAULT_ACCEPTED_PARENT_UNRESOLVED_DEPTH_MAX, + DEFAULT_BLOCK_CANDIDATE_CLEANUP_RETRY_BACKLOG_MAX, + DEFAULT_PRISM_INITIAL_JOB_MAX_WORKERS, + DEFAULT_PRISM_JOB_BUILD_EXECUTOR_WORKERS, +) +from lab.prism.accepted_preview_telemetry import ( + PRISM_ACCEPTED_LANDING_PHASES, + PRISM_PAYOUT_WINDOW_FULL_RESCAN_PATHS, + PRISM_PAYOUT_WINDOW_FULL_RESCAN_REASONS, + PRISM_REORG_RECONCILE_CALLERS, + PRISM_REORG_RECONCILE_STEPS, + AcceptedPreviewTelemetry, + ensure_accepted_preview_telemetry, + fold_ledger_read_stats, +) +from lab.prism.block_candidates import ( + PRISM_ACCEPTED_BLOCK_PREVIEW_PUBLICATION_RESULTS, + PRISM_ACCEPTED_BLOCK_PREVIEW_PUBLICATION_SECONDS_BUCKETS, + PRISM_BLOCK_CANDIDATE_COLLAPSE_OUTCOMES, + PRISM_STALE_JOB_ABANDON_CLASSES, +) +from lab.prism.bundle_compiler import _ServeBuilderClient, _ShareWindowSerialization +from lab.prism.job_bundle import ( + PRISM_JOB_BUILD_PHASES, + PRISM_JOB_BUILD_SECONDS_BUCKETS, + PRISM_JOB_CACHE_KINDS, +) +from lab.prism.job_delivery import ( + PRISM_EVICTED_JOB_CAPACITY_SCOPES, + PRISM_EVICTED_JOB_CLASSES, + PRISM_EVICTED_JOB_SUBMIT_OUTCOMES, +) +from lab.prism.metrics import ( + PRISM_COMPONENT_BYTE_KINDS, + PRISM_COMPONENT_ENTRY_KINDS, + MetricsRenderer, +) +from lab.prism.payout_state import ( + AcceptedBlockPayoutTransition, + PRISM_PAYOUT_WINDOW_BUILD_OUTCOMES, + PRISM_PAYOUT_WINDOW_BUILD_PHASES, + PayoutLedgerArtifact, + _IncrementalPayoutArtifactWindow, +) +from lab.prism.prism_coordinator import PRISM_REJECTION_REASON_IDS +from lab.prism.process_telemetry import ( + PRISM_GC_GENERATIONS, + MallInfo2, + ProcessHeapSample, + ProcessHeapTelemetry, +) +from lab.prism.reorg_reconciler import ( + PRISM_REORG_RECONCILE_LOOKUP_PATHS, + PRISM_REORG_RECONCILE_LOOKUP_SOURCES, + _ReconcileFlight, +) +from lab.prism.share_ledger import ( + DaemonShareWindowMirror, + IncrementalShareJsonSequence, + IncrementalShareWindow, + _IncrementalShareWindowPage, +) +from lab.prism.share_submission import ( + PRISM_SHARE_ACK_RESULTS, + empty_share_ack_histograms, +) +from lab.prism.vardiff_service import PRISM_VARDIFF_RESUME_OUTCOMES +from lab.prism.writer_lease_timing import ( + LEASE_HEARTBEAT_MODES, + LEASE_HEARTBEAT_OUTCOMES, + LEASE_HEARTBEAT_PHASES, + LEASE_HEARTBEAT_POLICY_TERMS, + LEASE_MONITOR_LATE_WAKE_SLACK_FRACTIONS, + LEASE_MONITOR_WAKE_DELAY_BUCKETS, +) +from lab.prism.background_services import PRISM_GC_PAUSE_SECONDS_BUCKETS +from tests import prism_vardiff_test_support as support + + +def reference_share_ack_metrics_lines(server) -> list[str]: + service = server.__dict__.get("_share_submission_service") + if service is not None: + histograms = service.share_ack_snapshot() + else: + legacy = server.__dict__.get("share_ack_histograms") + histograms = ( + { + result: { + "buckets": dict(histogram["buckets"]), + "sum": float(histogram["sum"]), + "count": int(histogram["count"]), + } + for result, histogram in legacy.items() + } + if legacy is not None + else empty_share_ack_histograms() + ) + lines = [ + "# HELP qbit_prism_share_ack_seconds mining.submit line arrival to Stratum response, by outcome.", + "# TYPE qbit_prism_share_ack_seconds histogram", + ] + for result in PRISM_SHARE_ACK_RESULTS: + histogram = histograms[result] + buckets = histogram["buckets"] + lines.extend( + f'qbit_prism_share_ack_seconds_bucket{{result="{result}",le="{bucket:g}"}} {int(buckets.get(bucket, 0))}' + for bucket in PRISM_JOB_BUILD_SECONDS_BUCKETS + ) + lines.append( + f'qbit_prism_share_ack_seconds_bucket{{result="{result}",le="+Inf"}} {histogram["count"]}' + ) + lines.append( + f'qbit_prism_share_ack_seconds_sum{{result="{result}"}} {histogram["sum"]:.6f}' + ) + lines.append( + f'qbit_prism_share_ack_seconds_count{{result="{result}"}} {histogram["count"]}' + ) + return lines + + +def reference_coordinator_lock_metrics_lines(server) -> list[str]: + snapshot = getattr(server.lock, "contention_snapshot", None) + if callable(snapshot): + acquisition_count, contention_count, wait_sum, wait_max = snapshot() + else: + acquisition_count, contention_count, wait_sum, wait_max = 0, 0, 0.0, 0.0 + return [ + "# HELP qbit_prism_coordinator_lock_acquisitions_total Coordinator control-plane lock acquisitions granted, contended or not; divide qbit_prism_coordinator_lock_contentions_total by this for the contended fraction. A timed-out wait is counted as contention without an acquisition, so that fraction is an upper bound where a caller passes a timeout.", + "# TYPE qbit_prism_coordinator_lock_acquisitions_total counter", + f"qbit_prism_coordinator_lock_acquisitions_total {int(acquisition_count)}", + "# HELP qbit_prism_coordinator_lock_contentions_total Coordinator control-plane lock acquisitions that had to wait.", + "# TYPE qbit_prism_coordinator_lock_contentions_total counter", + f"qbit_prism_coordinator_lock_contentions_total {int(contention_count)}", + "# HELP qbit_prism_coordinator_lock_wait_seconds Coordinator control-plane lock wait duration for contended acquisitions.", + "# TYPE qbit_prism_coordinator_lock_wait_seconds summary", + f"qbit_prism_coordinator_lock_wait_seconds_sum {float(wait_sum):.6f}", + f"qbit_prism_coordinator_lock_wait_seconds_count {int(contention_count)}", + "# HELP qbit_prism_coordinator_lock_wait_seconds_max Longest observed coordinator control-plane lock wait.", + "# TYPE qbit_prism_coordinator_lock_wait_seconds_max gauge", + f"qbit_prism_coordinator_lock_wait_seconds_max {float(wait_max):.6f}", + ] + + +def reference_block_submitter_metrics_lines(server) -> list[str]: + pending_metrics = { + "pending_count": -1, + "oldest_pending_age_seconds": -1.0, + "oldest_unattempted_age_seconds": -1.0, + } + pending_snapshot = getattr(server.ledger, "block_candidate_pending_metrics", None) + if callable(pending_snapshot): + try: + pending_metrics.update(pending_snapshot()) + except Exception: + pass + service = server._ensure_block_candidate_service() + backoff_active, backoff_remaining, backoff_delay = ( + service.backoff_snapshot() + ) + submit_buckets, submit_sum, submit_count = ( + service.block_submit_seconds_snapshot() + ) + collapse_counts = service.block_candidate_collapse_snapshot() + preview_publication = service.accepted_block_preview_publication_snapshot() + preview_publication_lines: list[str] = [] + for result in PRISM_ACCEPTED_BLOCK_PREVIEW_PUBLICATION_RESULTS: + histogram = preview_publication[result] + buckets = histogram["buckets"] + count = int(histogram["count"]) + preview_publication_lines.extend( + f"qbit_prism_accepted_block_preview_publication_seconds_bucket" + f'{{result="{result}",le="{bucket:g}"}} {int(buckets.get(bucket, 0))}' + for bucket in PRISM_ACCEPTED_BLOCK_PREVIEW_PUBLICATION_SECONDS_BUCKETS + ) + preview_publication_lines.append( + f"qbit_prism_accepted_block_preview_publication_seconds_bucket" + f'{{result="{result}",le="+Inf"}} {count}' + ) + preview_publication_lines.append( + f"qbit_prism_accepted_block_preview_publication_seconds_sum" + f'{{result="{result}"}} {float(histogram["sum"]):.6f}' + ) + preview_publication_lines.append( + f"qbit_prism_accepted_block_preview_publication_seconds_count" + f'{{result="{result}"}} {count}' + ) + return [ + "# HELP qbit_prism_block_submit_seconds Seconds from a block candidate landing in this process to its submitblock RPC returning.", + "# TYPE qbit_prism_block_submit_seconds histogram", + *[ + f'qbit_prism_block_submit_seconds_bucket{{le="{bucket:g}"}} {int(submit_buckets.get(bucket, 0))}' + for bucket in PRISM_JOB_BUILD_SECONDS_BUCKETS + ], + f'qbit_prism_block_submit_seconds_bucket{{le="+Inf"}} {submit_count}', + f"qbit_prism_block_submit_seconds_sum {submit_sum:.6f}", + f"qbit_prism_block_submit_seconds_count {submit_count}", + "# HELP qbit_prism_accepted_block_preview_publication_seconds Seconds from definitive qbitd acceptance of a block candidate to its payout preview becoming visible to waiting child work, by publication result.", + "# TYPE qbit_prism_accepted_block_preview_publication_seconds histogram", + *preview_publication_lines, + "# HELP qbit_prism_block_candidates_pending Durable block candidates awaiting a terminal outcome, or -1 if unavailable.", + "# TYPE qbit_prism_block_candidates_pending gauge", + f"qbit_prism_block_candidates_pending {int(pending_metrics['pending_count'])}", + "# HELP qbit_prism_block_candidate_oldest_pending_seconds Age of the oldest durable pending block candidate, or -1 if unavailable.", + "# TYPE qbit_prism_block_candidate_oldest_pending_seconds gauge", + f"qbit_prism_block_candidate_oldest_pending_seconds {float(pending_metrics['oldest_pending_age_seconds']):.6f}", + "# HELP qbit_prism_block_candidate_oldest_unattempted_seconds Age of the oldest durable candidate that has never entered processing, or -1 if unavailable.", + "# TYPE qbit_prism_block_candidate_oldest_unattempted_seconds gauge", + f"qbit_prism_block_candidate_oldest_unattempted_seconds {float(pending_metrics['oldest_unattempted_age_seconds']):.6f}", + "# HELP qbit_prism_block_submitter_retry_backoff_active Whether the submitter is in an intentional interruptible retry wait.", + "# TYPE qbit_prism_block_submitter_retry_backoff_active gauge", + f"qbit_prism_block_submitter_retry_backoff_active {1 if backoff_active else 0}", + "# HELP qbit_prism_block_submitter_retry_backoff_remaining_seconds Remaining intentional submitter retry wait.", + "# TYPE qbit_prism_block_submitter_retry_backoff_remaining_seconds gauge", + f"qbit_prism_block_submitter_retry_backoff_remaining_seconds {backoff_remaining:.6f}", + "# HELP qbit_prism_block_submitter_retry_backoff_seconds Current intentional submitter retry delay.", + "# TYPE qbit_prism_block_submitter_retry_backoff_seconds gauge", + f"qbit_prism_block_submitter_retry_backoff_seconds {backoff_delay:.6f}", + "# HELP qbit_prism_block_candidate_collapse_total Durable pending block candidates handled by the decided-height collapse, by bounded outcome. The outcome label set is closed: no block hash, parent hash, or job ID is ever a label.", + "# TYPE qbit_prism_block_candidate_collapse_total counter", + *[ + f'qbit_prism_block_candidate_collapse_total{{outcome="{outcome}"}} {int(collapse_counts.get(outcome, 0))}' + for outcome in PRISM_BLOCK_CANDIDATE_COLLAPSE_OUTCOMES + ], + ] + + +def reference_block_candidate_cleanup_backlog_metrics_lines(server) -> list[str]: + # Issue #198. Pinned deliberately: seven unlabelled single-series + # families over the B1 owner's fixed-key backlog snapshot. Adding a + # label, a key, or a family here is a metrics-contract change and must + # be mirrored in docs/prism-overload-alerts.md. + snapshot = ( + server._ensure_block_candidate_service() + .collapsed_candidate_cleanup_backlog_snapshot() + ) + return [ + "# HELP qbit_prism_block_candidate_cleanup_retry_backlog Durably terminal collapsed block candidates whose in-memory cleanup is still owed and retried by the accounting lane.", + "# TYPE qbit_prism_block_candidate_cleanup_retry_backlog gauge", + f"qbit_prism_block_candidate_cleanup_retry_backlog {int(snapshot['depth'])}", + "# HELP qbit_prism_block_candidate_cleanup_retry_backlog_max Configured cleanup-retry backlog depth at which the decided-height collapse stops admitting rows to bulk terminalization.", + "# TYPE qbit_prism_block_candidate_cleanup_retry_backlog_max gauge", + f"qbit_prism_block_candidate_cleanup_retry_backlog_max {int(snapshot['backlog_max'])}", + "# HELP qbit_prism_block_candidate_cleanup_retry_oldest_seconds Age of the oldest owed collapse cleanup since it was first deferred, or -1 when none is owed.", + "# TYPE qbit_prism_block_candidate_cleanup_retry_oldest_seconds gauge", + f"qbit_prism_block_candidate_cleanup_retry_oldest_seconds {float(snapshot['oldest_age_seconds']):.6f}", + "# HELP qbit_prism_block_candidate_cleanup_retry_pending_share_holders Pending-share floor holders retained by owed collapse cleanups, by exact object identity.", + "# TYPE qbit_prism_block_candidate_cleanup_retry_pending_share_holders gauge", + f"qbit_prism_block_candidate_cleanup_retry_pending_share_holders {int(snapshot['pending_share_holders'])}", + "# HELP qbit_prism_block_candidate_cleanup_retry_terminal_outcome_pins Terminal-outcome fences the cleanup-retry backlog pins against eviction.", + "# TYPE qbit_prism_block_candidate_cleanup_retry_terminal_outcome_pins gauge", + f"qbit_prism_block_candidate_cleanup_retry_terminal_outcome_pins {int(snapshot['terminal_outcome_pins'])}", + "# HELP qbit_prism_block_candidate_cleanup_backpressure_active Whether the cleanup-retry backlog is at its bound and bulk terminalization is refusing new rows.", + "# TYPE qbit_prism_block_candidate_cleanup_backpressure_active gauge", + f"qbit_prism_block_candidate_cleanup_backpressure_active {1 if snapshot['backpressure_active'] else 0}", + "# HELP qbit_prism_block_candidate_cleanup_backpressure_total Occasions on which the cleanup-retry backlog bound preserved at least one row from bulk terminalization.", + "# TYPE qbit_prism_block_candidate_cleanup_backpressure_total counter", + f"qbit_prism_block_candidate_cleanup_backpressure_total {int(snapshot['backpressure_engagements'])}", + ] + + +# The closed family set issue #198 exports; a test pins the renderer to it. +BLOCK_CANDIDATE_CLEANUP_BACKLOG_METRIC_FAMILIES = ( + ("qbit_prism_block_candidate_cleanup_retry_backlog", "gauge"), + ("qbit_prism_block_candidate_cleanup_retry_backlog_max", "gauge"), + ("qbit_prism_block_candidate_cleanup_retry_oldest_seconds", "gauge"), + ("qbit_prism_block_candidate_cleanup_retry_pending_share_holders", "gauge"), + ("qbit_prism_block_candidate_cleanup_retry_terminal_outcome_pins", "gauge"), + ("qbit_prism_block_candidate_cleanup_backpressure_active", "gauge"), + ("qbit_prism_block_candidate_cleanup_backpressure_total", "counter"), +) + + +# Issue #226 part 1: the closed family sets of the always-on heap telemetry +# and the component-cardinality gauges; tests pin the renderer to both. +PROCESS_HEAP_METRIC_FAMILIES = ( + ("qbit_prism_process_allocated_blocks", "gauge"), + ("qbit_prism_process_gc_trigger_count", "gauge"), + ("qbit_prism_process_gc_collections_total", "counter"), + ("qbit_prism_process_gc_collected_objects_total", "counter"), + ("qbit_prism_process_gc_uncollectable_objects_total", "counter"), + ("qbit_prism_process_threads", "gauge"), + ("qbit_prism_process_malloc_info_available", "gauge"), + ("qbit_prism_process_malloc_arena_bytes", "gauge"), + ("qbit_prism_process_malloc_in_use_bytes", "gauge"), + ("qbit_prism_process_malloc_free_bytes", "gauge"), + ("qbit_prism_process_malloc_mmapped_bytes", "gauge"), +) +# The only labelled heap families; their sole label is the closed +# three-value generation set. +PROCESS_HEAP_GENERATION_FAMILIES = frozenset( + { + "qbit_prism_process_gc_trigger_count", + "qbit_prism_process_gc_collections_total", + "qbit_prism_process_gc_collected_objects_total", + "qbit_prism_process_gc_uncollectable_objects_total", + } +) +COMPONENT_CARDINALITY_METRIC_FAMILIES = ( + ("qbit_prism_component_entries", "gauge"), + ("qbit_prism_component_bytes", "gauge"), +) +# The parity fixture pins the collector to this sample: allocation and GC +# counters move between the reference and the actual render. +PINNED_HEAP_SAMPLE = ProcessHeapSample( + allocated_blocks=123_456, + gc_trigger_count=(5, 6, 7), + gc_collections=(100, 10, 1), + gc_collected=(1_000, 100, 10), + gc_uncollectable=(0, 0, 2), + threads=64, + malloc_info_available=True, + malloc_arena_bytes=9_000, + malloc_in_use_bytes=6_000, + malloc_free_bytes=3_000, + malloc_mmapped_bytes=500, +) + + +def reference_process_heap_metrics_lines(sample: ProcessHeapSample) -> list[str]: + lines = [ + "# HELP qbit_prism_process_allocated_blocks Memory blocks currently allocated by the CPython allocator (sys.getallocatedblocks); a live-object proxy that rises with retention and stays flat under pure allocator fragmentation.", + "# TYPE qbit_prism_process_allocated_blocks gauge", + f"qbit_prism_process_allocated_blocks {int(sample.allocated_blocks)}", + "# HELP qbit_prism_process_gc_trigger_count gc.get_count(): the cyclic collector's per-generation trigger counters, compared against gc.get_threshold() to decide when to collect. Generation 0 is allocations minus deallocations, NOT a count of retained objects. CPython 3.13 made the collector incremental, so the third entry is unused and reads 0; the series is kept for a fixed family set.", + "# TYPE qbit_prism_process_gc_trigger_count gauge", + ] + lines.extend( + f'qbit_prism_process_gc_trigger_count{{generation="{generation}"}} {int(count)}' + for generation, count in zip(PRISM_GC_GENERATIONS, sample.gc_trigger_count) + ) + lines.extend( + [ + "# HELP qbit_prism_process_gc_collections_total Cyclic collector passes completed per generation since process start (gc.get_stats).", + "# TYPE qbit_prism_process_gc_collections_total counter", + ] + ) + lines.extend( + f'qbit_prism_process_gc_collections_total{{generation="{generation}"}} {int(count)}' + for generation, count in zip(PRISM_GC_GENERATIONS, sample.gc_collections) + ) + lines.extend( + [ + "# HELP qbit_prism_process_gc_collected_objects_total Objects the cyclic collector freed per generation since process start.", + "# TYPE qbit_prism_process_gc_collected_objects_total counter", + ] + ) + lines.extend( + f'qbit_prism_process_gc_collected_objects_total{{generation="{generation}"}} {int(count)}' + for generation, count in zip(PRISM_GC_GENERATIONS, sample.gc_collected) + ) + lines.extend( + [ + "# HELP qbit_prism_process_gc_uncollectable_objects_total Objects the cyclic collector found unreachable but could not free per generation since process start; growth is a leak the collector already knows about.", + "# TYPE qbit_prism_process_gc_uncollectable_objects_total counter", + ] + ) + lines.extend( + f'qbit_prism_process_gc_uncollectable_objects_total{{generation="{generation}"}} {int(count)}' + for generation, count in zip(PRISM_GC_GENERATIONS, sample.gc_uncollectable) + ) + lines.extend( + [ + "# HELP qbit_prism_process_threads Live Python threads (threading.active_count); each one is a candidate glibc malloc arena.", + "# TYPE qbit_prism_process_threads gauge", + f"qbit_prism_process_threads {int(sample.threads)}", + "# HELP qbit_prism_process_malloc_info_available Whether glibc mallinfo2 is bound and PRISM_MALLOC_TELEMETRY is on (1), or the allocator byte gauges are unavailable (0): the symbol is glibc 2.33+ only and absent on musl and macOS. While 0 the byte gauges render -1, never 0.", + "# TYPE qbit_prism_process_malloc_info_available gauge", + f"qbit_prism_process_malloc_info_available {1 if sample.malloc_info_available else 0}", + "# HELP qbit_prism_process_malloc_arena_bytes Heap space glibc malloc holds for its arenas across every thread arena, excluding mmapped chunks (mallinfo2.arena), or -1 when unavailable.", + "# TYPE qbit_prism_process_malloc_arena_bytes gauge", + f"qbit_prism_process_malloc_arena_bytes {int(sample.malloc_arena_bytes)}", + "# HELP qbit_prism_process_malloc_in_use_bytes Bytes in allocated chunks across every glibc malloc arena (mallinfo2.uordblks), or -1 when unavailable; arena minus this is the fragmentation glibc has not returned to the kernel.", + "# TYPE qbit_prism_process_malloc_in_use_bytes gauge", + f"qbit_prism_process_malloc_in_use_bytes {int(sample.malloc_in_use_bytes)}", + "# HELP qbit_prism_process_malloc_free_bytes Bytes in free chunks glibc malloc retains across every arena (mallinfo2.fordblks), or -1 when unavailable.", + "# TYPE qbit_prism_process_malloc_free_bytes gauge", + f"qbit_prism_process_malloc_free_bytes {int(sample.malloc_free_bytes)}", + "# HELP qbit_prism_process_malloc_mmapped_bytes Bytes in chunks glibc malloc served directly through mmap (mallinfo2.hblkhd), or -1 when unavailable.", + "# TYPE qbit_prism_process_malloc_mmapped_bytes gauge", + f"qbit_prism_process_malloc_mmapped_bytes {int(sample.malloc_mmapped_bytes)}", + ] + ) + return lines + + +def reference_component_cardinality_metrics_lines(server) -> list[str]: + server._ensure_job_cache_state() + bundles = server._ensure_job_bundle_service() + payout = server._ensure_payout_state_service() + writer = server._ensure_share_writer_service() + compiler = server._ensure_bundle_compiler() + candidates = server._ensure_block_candidate_service() + candidates._ensure_block_replay_state() + candidates._ensure_block_candidate_disposition_state() + reconciler = server.__dict__.get("_reorg_reconciler_service") + serialization = bundles._share_window_serialization + cached_window = payout._incremental_payout_artifact_window + artifact = payout._payout_ledger_artifact + client = compiler._serve_builder + window = cached_window.window if cached_window is not None else None + pages = records = canonical = mirror_records = mirror_bytes = 0 + if isinstance(window, DaemonShareWindowMirror): + # The mirror is the cached window on the Rust path, so it is accounted + # once, under daemon_window_mirror_*; payout_window_* stays zero. + mirror_records = int(window.record_count) + mirror_bytes = len(window.canonical_items) + elif window is not None: + pages = len(window.pages) + records = sum(len(page.records) for page in window.pages) + canonical = sum(len(page.canonical_json_items) for page in window.pages) + with server.lock: + server._ensure_evicted_job_state() + locked = { + "evicted_job_graveyard": len(server.evicted_job_graveyard), + "evicted_same_tip_job_ids": len(server.evicted_same_tip_job_ids), + "outstanding_block_candidate_hashes": len( + candidates._outstanding_block_candidate_hashes + ), + "tip_observed_accepted_block_hashes": len( + candidates._tip_observed_accepted_block_hashes + ), + "counted_block_candidate_abandonments": len( + candidates._counted_block_candidate_abandonments + ), + "ancestor_redrive_records": len(candidates._ancestor_redrive_records), + "block_candidate_terminal_outcomes": len( + candidates._block_candidate_terminal_outcomes + ), + "block_disposition_waiting_retries": len( + candidates._block_disposition_waiting_retries + ), + "block_candidate_dequeued_hashes": len( + candidates._block_candidate_dequeued_hashes + ), + "accounted_accepted_block_hashes": len( + server._accounted_accepted_block_hashes + ), + "reconcile_trusted_memo": ( + len(reconciler._reorg_reconcile_trusted_memo) + if reconciler is not None + else 0 + ), + } + entries = { + "payout_window_pages": pages, + "payout_window_records": records, + "payout_ledger_artifact_shares": ( + len(artifact.shares_json) if artifact is not None else 0 + ), + "accepted_block_payout_previews": len(payout._accepted_block_payout_previews), + "invalidated_accepted_block_payout_previews": len( + payout._invalidated_accepted_block_payout_previews + ), + "payout_append_invalidation_stamps": len( + payout._payout_append_invalidation_stamps + ), + "payout_window_inflight_scan_anchors": len( + payout._payout_window_inflight_scan_anchors + ), + "payout_unfenced_append_inflight_stamps": len( + payout._payout_unfenced_append_inflight_stamps + ), + "share_window_serialization_shares": ( + int(serialization.share_count) if serialization is not None else 0 + ), + "job_contexts": len(getattr(server, "jobs", {})), + "job_bundle_cache": len(bundles._job_bundle_cache), + "job_build_issued_stamps": len(bundles._job_build_issued_at_ms), + "bundle_preparation_flights": len(bundles._bundle_preparation_flights), + "active_job_bundle_builds": len(bundles._active_job_bundle_builds), + "block_candidate_queue": candidates.candidate_queue.qsize(), + "block_replay_queue": candidates._block_replay_candidate_queue.qsize(), + "block_replay_inflight_hashes": len(candidates._block_replay_inflight_hashes), + "block_quarantine_queue": candidates._block_quarantine_queue.qsize(), + "block_quarantine_hashes": len(candidates._block_quarantine_hashes), + "accepted_block_preview_stamps": len( + candidates._accepted_block_preview_acceptance_monotonic + ), + "block_candidate_disposition_flights": len( + candidates._block_candidate_disposition_flights + ), + "reconcile_flights": ( + len(reconciler._reconcile_flights) if reconciler is not None else 0 + ), + "pending_share_commit_floor": len(writer._pending_share_commit_floor), + "daemon_window_mirror_records": mirror_records, + "daemon_uploaded_windows": ( + len(client.uploaded_windows) if client is not None else 0 + ), + **locked, + } + byte_sizes = { + "payout_window_canonical_json": canonical, + "daemon_window_mirror_canonical_items": mirror_bytes, + "share_window_serialization_spool": ( + int(serialization._spool_size) if serialization is not None else 0 + ), + "share_window_serialization_compact_json": ( + int(serialization.compact_json_bytes) + if serialization is not None + else 0 + ), + } + return [ + "# HELP qbit_prism_component_entries Entries retained by each in-process coordinator component; the component label set is closed and carries no job id, tip hash, generation, or worker name.", + "# TYPE qbit_prism_component_entries gauge", + *[ + f'qbit_prism_component_entries{{component="{kind}"}} {int(entries[kind])}' + for kind in PRISM_COMPONENT_ENTRY_KINDS + ], + "# HELP qbit_prism_component_bytes Bytes retained by each byte-sized coordinator payload (the cached payout window's canonical JSON, the daemon window mirror, and the share-window serialization compact strings); the component label set is closed. All are resident bytes except share_window_serialization_spool, which is an on-disk temporary file and is not part of RSS.", + "# TYPE qbit_prism_component_bytes gauge", + *[ + f'qbit_prism_component_bytes{{component="{kind}"}} {int(byte_sizes[kind])}' + for kind in PRISM_COMPONENT_BYTE_KINDS + ], + ] + + +def reference_landing_observability_metrics_lines(server) -> list[str]: + lines: list[str] = [ + "# HELP qbit_prism_block_ledger_calls_total Submitter ledger calls by deadline class.", + "# TYPE qbit_prism_block_ledger_calls_total counter", + "# HELP qbit_prism_block_ledger_call_timeouts_total Submitter ledger deadline expiries by deadline class.", + "# TYPE qbit_prism_block_ledger_call_timeouts_total counter", + "# HELP qbit_prism_block_ledger_call_budget_seconds Most recent statement budget applied per deadline class (escalates for retried landings).", + "# TYPE qbit_prism_block_ledger_call_budget_seconds gauge", + "# HELP qbit_prism_block_ledger_call_last_duration_seconds Duration of the most recent call per deadline class.", + "# TYPE qbit_prism_block_ledger_call_last_duration_seconds gauge", + "# HELP qbit_prism_block_ledger_call_max_duration_seconds Longest observed call per deadline class since process start.", + "# TYPE qbit_prism_block_ledger_call_max_duration_seconds gauge", + ] + for call_class, stats in sorted( + server.block_ledger_call_class_metrics().items() + ): + label = server.prometheus_label_value(call_class) + lines.extend( + [ + f'qbit_prism_block_ledger_calls_total{{call_class="{label}"}} {int(stats["calls_total"])}', + f'qbit_prism_block_ledger_call_timeouts_total{{call_class="{label}"}} {int(stats["timeouts_total"])}', + f'qbit_prism_block_ledger_call_budget_seconds{{call_class="{label}"}} {float(stats["last_budget_seconds"]):.6f}', + f'qbit_prism_block_ledger_call_last_duration_seconds{{call_class="{label}"}} {float(stats["last_duration_seconds"]):.6f}', + f'qbit_prism_block_ledger_call_max_duration_seconds{{call_class="{label}"}} {float(stats["max_duration_seconds"]):.6f}', + ] + ) + unresolved_depth = server._accepted_parent_unresolved_depth() + unresolved_depth_cap = server._accepted_parent_unresolved_depth_cap() + unresolved_ages = server.accepted_parent_unresolved_ages_seconds() + oldest_unresolved = max(unresolved_ages) if unresolved_ages else -1.0 + with server.lock: + preview_wait_timeouts = int( + getattr(server, "_accepted_parent_preview_wait_timeouts", 0) + ) + lines.extend( + [ + "# HELP qbit_prism_accepted_parent_unresolved_transitions Landed accepted-block transitions whose durable bookkeeping is unresolved.", + "# TYPE qbit_prism_accepted_parent_unresolved_transitions gauge", + f"qbit_prism_accepted_parent_unresolved_transitions {unresolved_depth}", + "# HELP qbit_prism_accepted_parent_unresolved_depth_max Configured unresolved accepted-parent depth at which job admission fails closed.", + "# TYPE qbit_prism_accepted_parent_unresolved_depth_max gauge", + f"qbit_prism_accepted_parent_unresolved_depth_max {unresolved_depth_cap}", + "# HELP qbit_prism_accepted_parent_unresolved_oldest_seconds Age of the oldest unresolved accepted-parent transition, or -1 when none.", + "# TYPE qbit_prism_accepted_parent_unresolved_oldest_seconds gauge", + f"qbit_prism_accepted_parent_unresolved_oldest_seconds {oldest_unresolved:.6f}", + "# HELP qbit_prism_accepted_parent_preview_wait_timeouts_total Child job builds that timed out waiting for an accepted-parent payout preview.", + "# TYPE qbit_prism_accepted_parent_preview_wait_timeouts_total counter", + f"qbit_prism_accepted_parent_preview_wait_timeouts_total {preview_wait_timeouts}", + "# HELP qbit_prism_accepted_parent_redrive_attempts_total In-process ancestor re-drives armed after repeated finalization deferrals on one pending accepted-parent transition.", + "# TYPE qbit_prism_accepted_parent_redrive_attempts_total counter", + "qbit_prism_accepted_parent_redrive_attempts_total " + f"{int(getattr(server, 'accepted_parent_redrive_attempt_count', 0))}", + "# HELP qbit_prism_accepted_parent_redrive_resolved_total Pending accepted-parent transitions that resolved after at least one in-process re-drive attempt.", + "# TYPE qbit_prism_accepted_parent_redrive_resolved_total counter", + "qbit_prism_accepted_parent_redrive_resolved_total " + f"{int(getattr(server, 'accepted_parent_redrive_resolved_count', 0))}", + "# HELP qbit_prism_accepted_parent_redrive_exhausted_total Ancestors whose re-drive attempt cap was exhausted with the transition still pending; the publication-progress watchdog remains the backstop.", + "# TYPE qbit_prism_accepted_parent_redrive_exhausted_total counter", + "qbit_prism_accepted_parent_redrive_exhausted_total " + f"{int(getattr(server, 'accepted_parent_redrive_exhausted_count', 0))}", + ] + ) + prior_stats_fn = getattr(server.ledger, "prior_balances_read_stats", None) + if callable(prior_stats_fn): + prior_stats = prior_stats_fn() + lines.extend( + [ + "# HELP qbit_prism_prior_balances_reads_total Prior-balances reads served by the ledger.", + "# TYPE qbit_prism_prior_balances_reads_total counter", + f"qbit_prism_prior_balances_reads_total {int(prior_stats['reads_total'])}", + "# HELP qbit_prism_prior_balances_read_last_seconds Duration of the most recent prior-balances read.", + "# TYPE qbit_prism_prior_balances_read_last_seconds gauge", + f"qbit_prism_prior_balances_read_last_seconds {float(prior_stats['last_seconds']):.6f}", + "# HELP qbit_prism_prior_balances_read_max_seconds Longest prior-balances read since process start.", + "# TYPE qbit_prism_prior_balances_read_max_seconds gauge", + f"qbit_prism_prior_balances_read_max_seconds {float(prior_stats['max_seconds']):.6f}", + ] + ) + read_gate_fn = getattr(server.ledger, "ledger_read_gate_stats", None) + read_gate_stats = ( + fold_ledger_read_stats(read_gate_fn() or {}) + if callable(read_gate_fn) + else {} + ) + if read_gate_stats: + families = ( + ( + "qbit_prism_ledger_read_calls_total", + "counter", + "Ledger read-slot operations completed or failed, by operation.", + "calls_total", + True, + ), + ( + "qbit_prism_ledger_read_gate_wait_seconds_total", + "counter", + "Cumulative coordinator-local read-slot admission wait, by operation.", + "gate_wait_seconds_total", + False, + ), + ( + "qbit_prism_ledger_read_gate_wait_seconds_max", + "gauge", + "Longest coordinator-local read-slot admission wait since process start, by operation.", + "gate_wait_seconds_max", + False, + ), + ( + "qbit_prism_ledger_read_gate_timeouts_total", + "counter", + "Read-slot operations whose deadline expired before admission, so no statement was ever sent.", + "gate_timeouts_total", + True, + ), + ( + "qbit_prism_ledger_read_execute_seconds_total", + "counter", + "Cumulative PostgreSQL execution time for admitted read-slot statements, by operation.", + "execute_seconds_total", + False, + ), + ( + "qbit_prism_ledger_read_execute_seconds_max", + "gauge", + "Longest PostgreSQL execution time for an admitted read-slot statement since process start, by operation.", + "execute_seconds_max", + False, + ), + ( + "qbit_prism_ledger_read_execute_timeouts_total", + "counter", + "Admitted read-slot statements whose deadline expired inside PostgreSQL, cancel lag included.", + "execute_timeouts_total", + True, + ), + ) + operations = [ + (server.prometheus_label_value(str(operation)), stats) + for operation, stats in sorted(read_gate_stats.items()) + ] + for name, metric_type, help_text, field, integral in families: + lines.append(f"# HELP {name} {help_text}") + lines.append(f"# TYPE {name} {metric_type}") + for label, stats in operations: + value = stats.get(field, 0) + rendered = f"{int(value)}" if integral else f"{float(value):.6f}" + lines.append(f'{name}{{operation="{label}"}} {rendered}') + startup_phases = server.startup_phase_seconds() + if startup_phases: + lines.extend( + [ + "# HELP qbit_prism_startup_phase_seconds Seconds from serve() start to each startup phase, recorded once.", + "# TYPE qbit_prism_startup_phase_seconds gauge", + ] + ) + for phase, seconds in sorted(startup_phases.items()): + label = server.prometheus_label_value(phase) + lines.append( + f'qbit_prism_startup_phase_seconds{{phase="{label}"}} {float(seconds):.6f}' + ) + return lines + + +def reference_accepted_stats_reconcile_metric_lines(server) -> list[str]: + status_fn = getattr(server.ledger, "accepted_stats_reconcile_status", None) + if not callable(status_fn): + return [] + status = status_fn() + lines = [ + "# HELP qbit_prism_accepted_stats_reconcile_failures_total Failed background accepted-stats reconcile passes.", + "# TYPE qbit_prism_accepted_stats_reconcile_failures_total counter", + f"qbit_prism_accepted_stats_reconcile_failures_total {int(status.get('failures') or 0)}", + ] + age = status.get("age_seconds") + if age is not None: + lines.extend( + [ + "# HELP qbit_prism_accepted_stats_reconcile_age_seconds Seconds since the accepted-share counters were last reconciled against the ledger.", + "# TYPE qbit_prism_accepted_stats_reconcile_age_seconds gauge", + f"qbit_prism_accepted_stats_reconcile_age_seconds {float(age):.6f}", + ] + ) + return lines + + +def reference_lease_heartbeat_metrics_lines(server) -> list[str]: + snapshot = server._ensure_lease_heartbeat_service().snapshot() + attempts = snapshot["attempts"] + outcomes = snapshot["outcomes"] + last_phases = snapshot["last_phase_seconds"] + worst_phases = snapshot["worst_phase_seconds"] + policy_seconds = snapshot["policy_seconds"] + wakes = snapshot["monitor_wakes"] + probe = snapshot["stall_probe"] + assert isinstance(attempts, dict) + assert isinstance(outcomes, dict) + assert isinstance(last_phases, dict) + assert isinstance(worst_phases, dict) + assert isinstance(policy_seconds, dict) + assert isinstance(wakes, dict) + assert isinstance(probe, dict) + wake_buckets = wakes["buckets"] + late_wakes = wakes["late_wakes"] + assert isinstance(wake_buckets, dict) + assert isinstance(late_wakes, dict) + return [ + "# HELP qbit_prism_lease_heartbeat_running Whether the writer-lease heartbeat thread is alive.", + "# TYPE qbit_prism_lease_heartbeat_running gauge", + f"qbit_prism_lease_heartbeat_running {int(bool(snapshot['running']))}", + "# HELP qbit_prism_lease_heartbeat_attempts_total Guard verifications by mode.", + "# TYPE qbit_prism_lease_heartbeat_attempts_total counter", + *[ + f'qbit_prism_lease_heartbeat_attempts_total{{mode="{mode}"}} {int(attempts.get(mode, 0))}' + for mode in LEASE_HEARTBEAT_MODES + ], + "# HELP qbit_prism_lease_heartbeat_outcomes_total Guard verification outcomes.", + "# TYPE qbit_prism_lease_heartbeat_outcomes_total counter", + *[ + f'qbit_prism_lease_heartbeat_outcomes_total{{outcome="{outcome}"}} {int(outcomes.get(outcome, 0))}' + for outcome in LEASE_HEARTBEAT_OUTCOMES + ], + "# HELP qbit_prism_lease_heartbeat_phase_seconds Phase breakdown of the latest guard verification.", + "# TYPE qbit_prism_lease_heartbeat_phase_seconds gauge", + *[ + f'qbit_prism_lease_heartbeat_phase_seconds{{phase="{phase}"}} {float(last_phases.get(phase, 0.0)):.6f}' + for phase in LEASE_HEARTBEAT_PHASES + ], + "# HELP qbit_prism_lease_heartbeat_worst_phase_seconds Phase breakdown of the slowest guard verification observed.", + "# TYPE qbit_prism_lease_heartbeat_worst_phase_seconds gauge", + *[ + f'qbit_prism_lease_heartbeat_worst_phase_seconds{{phase="{phase}"}} {float(worst_phases.get(phase, 0.0)):.6f}' + for phase in LEASE_HEARTBEAT_PHASES + ], + "# HELP qbit_prism_lease_heartbeat_activity_age_seconds Age of the newest monitor-visible heartbeat activity mark.", + "# TYPE qbit_prism_lease_heartbeat_activity_age_seconds gauge", + f"qbit_prism_lease_heartbeat_activity_age_seconds {float(snapshot['activity_age_seconds']):.6f}", + "# HELP qbit_prism_lease_heartbeat_server_proven_age_seconds Age of the newest completed guard round trip.", + "# TYPE qbit_prism_lease_heartbeat_server_proven_age_seconds gauge", + f"qbit_prism_lease_heartbeat_server_proven_age_seconds {float(snapshot['server_proven_age_seconds']):.6f}", + "# HELP qbit_prism_lease_heartbeat_monitor_wake_delay_seconds Worst observed lateness of the heartbeat monitor's own poll.", + "# TYPE qbit_prism_lease_heartbeat_monitor_wake_delay_seconds gauge", + f"qbit_prism_lease_heartbeat_monitor_wake_delay_seconds {float(snapshot['monitor_wake_delay_seconds']):.6f}", + "# HELP qbit_prism_lease_heartbeat_policy_seconds Terms of the validated writer-lease heartbeat timing policy.", + "# TYPE qbit_prism_lease_heartbeat_policy_seconds gauge", + *[ + f'qbit_prism_lease_heartbeat_policy_seconds{{term="{term}"}} {float(policy_seconds.get(term, 0.0)):.6f}' + for term in LEASE_HEARTBEAT_POLICY_TERMS + ], + "# HELP qbit_prism_lease_heartbeat_monitor_wake_lateness_seconds Lateness of every heartbeat-monitor poll wake (issue #227); express the wake-delay alert on this or on the late-wake counters, not on the lifetime gauge.", + "# TYPE qbit_prism_lease_heartbeat_monitor_wake_lateness_seconds histogram", + *[ + f'qbit_prism_lease_heartbeat_monitor_wake_lateness_seconds_bucket{{le="{bucket:g}"}} {int(wake_buckets.get(bucket, 0))}' + for bucket in LEASE_MONITOR_WAKE_DELAY_BUCKETS + ], + f'qbit_prism_lease_heartbeat_monitor_wake_lateness_seconds_bucket{{le="+Inf"}} {int(wakes["count"])}', + f'qbit_prism_lease_heartbeat_monitor_wake_lateness_seconds_sum {float(wakes["sum"]):.6f}', + f'qbit_prism_lease_heartbeat_monitor_wake_lateness_seconds_count {int(wakes["count"])}', + "# HELP qbit_prism_lease_heartbeat_monitor_late_wakes_total Monitor wakes at least the labelled fraction of the policy's scheduler slack late.", + "# TYPE qbit_prism_lease_heartbeat_monitor_late_wakes_total counter", + *[ + f'qbit_prism_lease_heartbeat_monitor_late_wakes_total{{slack_fraction="{fraction}"}} {int(late_wakes.get(fraction, 0))}' + for fraction in LEASE_MONITOR_LATE_WAKE_SLACK_FRACTIONS + ], + "# HELP qbit_prism_lease_heartbeat_monitor_wake_delay_window_max_seconds Worst monitor wake lateness inside the trailing 300s window; falls back once a stall ages out, unlike the lifetime gauge.", + "# TYPE qbit_prism_lease_heartbeat_monitor_wake_delay_window_max_seconds gauge", + f"qbit_prism_lease_heartbeat_monitor_wake_delay_window_max_seconds {float(wakes['window_max_seconds']):.6f}", + "# HELP qbit_prism_lease_heartbeat_monitor_wake_delay_record_age_seconds Seconds since the lifetime monitor wake-delay record was last raised.", + "# TYPE qbit_prism_lease_heartbeat_monitor_wake_delay_record_age_seconds gauge", + f"qbit_prism_lease_heartbeat_monitor_wake_delay_record_age_seconds {float(wakes['record_age_seconds']):.6f}", + "# HELP qbit_prism_lease_heartbeat_monitor_exit_guarantee_breaches_total Monitor wakes later than the policy's max_guaranteed_monitor_lateness: beats on which exit-before-adoption was not guaranteed (issue #227).", + "# TYPE qbit_prism_lease_heartbeat_monitor_exit_guarantee_breaches_total counter", + f"qbit_prism_lease_heartbeat_monitor_exit_guarantee_breaches_total {int(snapshot['exit_guarantee_breaches'])}", + "# HELP qbit_prism_lease_heartbeat_monitor_worst_exit_guarantee_overrun_seconds Worst amount by which a late monitor wake could have placed the hard exit after the successor's adoption edge.", + "# TYPE qbit_prism_lease_heartbeat_monitor_worst_exit_guarantee_overrun_seconds gauge", + f"qbit_prism_lease_heartbeat_monitor_worst_exit_guarantee_overrun_seconds {float(snapshot['worst_exit_guarantee_overrun_seconds']):.6f}", + "# HELP qbit_prism_lease_heartbeat_stall_probe_samples_total Stack samples the stall probe took on monitor wakes at least half a scheduler slack late.", + "# TYPE qbit_prism_lease_heartbeat_stall_probe_samples_total counter", + f"qbit_prism_lease_heartbeat_stall_probe_samples_total {int(probe['samples'])}", + "# HELP qbit_prism_lease_heartbeat_stall_probe_suppressed_total Stall-probe triggers refused by its rate limit (at most 3 samples per 60s).", + "# TYPE qbit_prism_lease_heartbeat_stall_probe_suppressed_total counter", + f"qbit_prism_lease_heartbeat_stall_probe_suppressed_total {int(probe['suppressed'])}", + "# HELP qbit_prism_lease_heartbeat_monitor_breach_warnings_suppressed_total Lateness-beyond-slack warnings the monitor refused under its rate limit (at most 3 per 60s); the breaches themselves are still counted.", + "# TYPE qbit_prism_lease_heartbeat_monitor_breach_warnings_suppressed_total counter", + f"qbit_prism_lease_heartbeat_monitor_breach_warnings_suppressed_total {int(snapshot['slack_breach_warnings_suppressed'])}", + ] + + +def reference_gc_pause_metrics_lines(server) -> list[str]: + pauses = server._ensure_lease_heartbeat_service().snapshot()["gc_pauses"] + assert isinstance(pauses, dict) + lines = [ + "# HELP qbit_prism_process_gc_pause_seconds Cyclic collector pause duration per generation, from gc.callbacks; the qbit_prism_process_gc_* count families say how many passes ran, this says how long each held the interpreter.", + "# TYPE qbit_prism_process_gc_pause_seconds histogram", + ] + for generation in PRISM_GC_GENERATIONS: + entry = pauses[generation] + buckets = entry["buckets"] + lines.extend( + f'qbit_prism_process_gc_pause_seconds_bucket{{generation="{generation}",le="{bucket:g}"}} {int(buckets.get(bucket, 0))}' + for bucket in PRISM_GC_PAUSE_SECONDS_BUCKETS + ) + lines.append( + f'qbit_prism_process_gc_pause_seconds_bucket{{generation="{generation}",le="+Inf"}} {int(entry["count"])}' + ) + lines.append( + f'qbit_prism_process_gc_pause_seconds_sum{{generation="{generation}"}} {float(entry["sum"]):.6f}' + ) + lines.append( + f'qbit_prism_process_gc_pause_seconds_count{{generation="{generation}"}} {int(entry["count"])}' + ) + lines.extend( + [ + "# HELP qbit_prism_process_gc_last_pause_seconds Duration of the most recent cyclic collector pass per generation.", + "# TYPE qbit_prism_process_gc_last_pause_seconds gauge", + *[ + f'qbit_prism_process_gc_last_pause_seconds{{generation="{generation}"}} {float(pauses[generation]["last_seconds"]):.6f}' + for generation in PRISM_GC_GENERATIONS + ], + "# HELP qbit_prism_process_gc_max_pause_seconds Longest cyclic collector pass observed per generation since the lease heartbeat armed.", + "# TYPE qbit_prism_process_gc_max_pause_seconds gauge", + *[ + f'qbit_prism_process_gc_max_pause_seconds{{generation="{generation}"}} {float(pauses[generation]["max_seconds"]):.6f}' + for generation in PRISM_GC_GENERATIONS + ], + ] + ) + return lines + + +def reference_shutdown_metrics_lines(server) -> list[str]: + snapshot = server._ensure_shutdown_controller().snapshot() + quiescence = snapshot["writer_quiescence_outcomes"] + release = snapshot["lease_release_outcomes"] + active = snapshot["active_writers"] + assert isinstance(quiescence, dict) + assert isinstance(release, dict) + assert isinstance(active, dict) + return [ + "# HELP qbit_prism_shutdowns_total Controlled coordinator shutdown sequences started.", + "# TYPE qbit_prism_shutdowns_total counter", + f"qbit_prism_shutdowns_total {int(snapshot['shutdowns_total'])}", + "# HELP qbit_prism_shutdown_writer_operations Active admitted ledger-mutating operations by component.", + "# TYPE qbit_prism_shutdown_writer_operations gauge", + *[ + f'qbit_prism_shutdown_writer_operations{{component="{server.prometheus_label_value(str(component))}"}} {int(count)}' + for component, count in sorted(active.items()) + ], + "# HELP qbit_prism_shutdown_writer_quiescence_total Writer-quiescence outcomes.", + "# TYPE qbit_prism_shutdown_writer_quiescence_total counter", + *[ + f'qbit_prism_shutdown_writer_quiescence_total{{outcome="{outcome}"}} {int(quiescence.get(outcome, 0))}' + for outcome in ("success", "timeout") + ], + "# HELP qbit_prism_shutdown_writer_quiescence_seconds Duration of the latest writer-quiescence barrier.", + "# TYPE qbit_prism_shutdown_writer_quiescence_seconds gauge", + f"qbit_prism_shutdown_writer_quiescence_seconds {float(snapshot['writer_quiescence_seconds']):.6f}", + "# HELP qbit_prism_shutdown_lease_release_attempts_total Writer-lease release attempts.", + "# TYPE qbit_prism_shutdown_lease_release_attempts_total counter", + f"qbit_prism_shutdown_lease_release_attempts_total {int(snapshot['lease_release_attempts_total'])}", + "# HELP qbit_prism_shutdown_lease_release_total Writer-lease release outcomes.", + "# TYPE qbit_prism_shutdown_lease_release_total counter", + *[ + f'qbit_prism_shutdown_lease_release_total{{outcome="{outcome}"}} {int(release.get(outcome, 0))}' + for outcome in ("success", "not_held", "unsupported", "failure") + ], + "# HELP qbit_prism_shutdown_lease_release_seconds Duration of the latest writer-lease release attempt.", + "# TYPE qbit_prism_shutdown_lease_release_seconds gauge", + f"qbit_prism_shutdown_lease_release_seconds {float(snapshot['lease_release_seconds']):.6f}", + "# HELP qbit_prism_shutdown_sigterm_to_lease_release_seconds Time from SIGTERM admission close to safe lease release, or -1 if unobserved.", + "# TYPE qbit_prism_shutdown_sigterm_to_lease_release_seconds gauge", + "qbit_prism_shutdown_sigterm_to_lease_release_seconds " + + ( + f"{float(snapshot['sigterm_to_lease_release_seconds']):.6f}" + if snapshot["sigterm_release_observed"] + else "-1" + ), + "# HELP qbit_prism_shutdown_release_withheld_total Shutdowns that withheld lease release because a writer did not quiesce.", + "# TYPE qbit_prism_shutdown_release_withheld_total counter", + f"qbit_prism_shutdown_release_withheld_total {int(snapshot['release_withheld_total'])}", + "# HELP qbit_prism_shutdown_non_writer_drain_seconds Duration of cleanup after writer lease handling.", + "# TYPE qbit_prism_shutdown_non_writer_drain_seconds gauge", + f"qbit_prism_shutdown_non_writer_drain_seconds {float(snapshot['non_writer_drain_seconds']):.6f}", + ] + + +def reference_initial_delivery_metrics_lines(server) -> list[str]: + server._ensure_initial_job_state() + mining = server.mining_delivery_snapshot() + with server.lock: + counts = { + "sent": server.initial_job_sent_count, + "cancelled": server.initial_job_cancelled_count, + "coalesced": server.initial_job_coalesced_count, + "failed": server.initial_job_failed_count, + "superseded": server.initial_job_superseded_count, + } + latency_sum = server.initial_job_delivery_latency_seconds_sum + latency_count = server.initial_job_delivery_latency_count + queue_capacity_reclaimed = ( + server.initial_job_queue_capacity_reclaimed_count + ) + executor = getattr(server, "_initial_job_executor", None) + queued, slots = executor.stats() if executor is not None else (0, 0) + configured_workers = int( + getattr( + server, + "initial_job_max_workers", + DEFAULT_PRISM_INITIAL_JOB_MAX_WORKERS, + ) + ) + with server._bundle_preparation_lock: + build_counts = dict(server.shared_bundle_build_counts) + preparation_sum = server.shared_bundle_preparation_seconds_sum + preparation_count = server.shared_bundle_preparation_count + waiters = server.shared_bundle_preparation_waiters + return [ + "# HELP qbit_prism_stratum_subscribed_clients Subscribed Stratum clients.", + "# TYPE qbit_prism_stratum_subscribed_clients gauge", + f'qbit_prism_stratum_subscribed_clients {mining["subscribed_clients"]}', + "# HELP qbit_prism_stratum_authorized_clients Subscribed and authorized Stratum clients.", + "# TYPE qbit_prism_stratum_authorized_clients gauge", + f'qbit_prism_stratum_authorized_clients {mining["authorized_clients"]}', + "# HELP qbit_prism_clients_without_current_tip_job Authorized clients without usable current-tip work.", + "# TYPE qbit_prism_clients_without_current_tip_job gauge", + f'qbit_prism_clients_without_current_tip_job {mining["clients_without_current_tip_job"]}', + "# HELP qbit_prism_clients_with_no_active_job Authorized clients with no active job at all.", + "# TYPE qbit_prism_clients_with_no_active_job gauge", + f'qbit_prism_clients_with_no_active_job {mining["clients_with_no_active_job"]}', + "# HELP qbit_prism_clients_with_current_tip_job Authorized clients with usable current-tip work.", + "# TYPE qbit_prism_clients_with_current_tip_job gauge", + f'qbit_prism_clients_with_current_tip_job {mining["clients_with_current_tip_job"]}', + "# HELP qbit_prism_current_tip_job_coverage_ratio Fraction of authorized clients with current-tip work.", + "# TYPE qbit_prism_current_tip_job_coverage_ratio gauge", + f'qbit_prism_current_tip_job_coverage_ratio {float(mining["current_tip_job_coverage_ratio"]):.12g}', + "# HELP qbit_prism_initial_job_deliveries_pending Coalesced initial deliveries queued or running.", + "# TYPE qbit_prism_initial_job_deliveries_pending gauge", + f'qbit_prism_initial_job_deliveries_pending {mining["clients_pending_initial_job"]}', + "# HELP qbit_prism_initial_job_delivery_tasks_inflight Bounded shared delivery slots currently occupied.", + "# TYPE qbit_prism_initial_job_delivery_tasks_inflight gauge", + f"qbit_prism_initial_job_delivery_tasks_inflight {slots}", + "# HELP qbit_prism_initial_job_delivery_queue_depth Initial-job tasks waiting for a dedicated worker.", + "# TYPE qbit_prism_initial_job_delivery_queue_depth gauge", + f"qbit_prism_initial_job_delivery_queue_depth {queued}", + "# HELP qbit_prism_initial_job_delivery_active_workers Dedicated initial-job workers currently running tasks.", + "# TYPE qbit_prism_initial_job_delivery_active_workers gauge", + f"qbit_prism_initial_job_delivery_active_workers {slots}", + "# HELP qbit_prism_initial_job_delivery_configured_workers Configured dedicated initial-job worker count.", + "# TYPE qbit_prism_initial_job_delivery_configured_workers gauge", + f"qbit_prism_initial_job_delivery_configured_workers {configured_workers}", + "# HELP qbit_prism_initial_job_delivery_seconds Authorization-to-current-job latency.", + "# TYPE qbit_prism_initial_job_delivery_seconds summary", + f"qbit_prism_initial_job_delivery_seconds_sum {latency_sum:.6f}", + f"qbit_prism_initial_job_delivery_seconds_count {latency_count}", + "# HELP qbit_prism_initial_job_requests_total Initial delivery outcomes.", + "# TYPE qbit_prism_initial_job_requests_total counter", + *[ + f'qbit_prism_initial_job_requests_total{{result="{result}"}} {count}' + for result, count in sorted(counts.items()) + ], + "# HELP qbit_prism_initial_job_queue_capacity_reclaimed_total Queued initial-job slots reclaimed immediately by cancellation.", + "# TYPE qbit_prism_initial_job_queue_capacity_reclaimed_total counter", + f"qbit_prism_initial_job_queue_capacity_reclaimed_total {queue_capacity_reclaimed}", + "# HELP qbit_prism_shared_bundle_preparation_seconds Heavy shared bundle preparation wall time.", + "# TYPE qbit_prism_shared_bundle_preparation_seconds summary", + f"qbit_prism_shared_bundle_preparation_seconds_sum {preparation_sum:.6f}", + f"qbit_prism_shared_bundle_preparation_seconds_count {preparation_count}", + "# HELP qbit_prism_shared_bundle_preparation_waiters Callers waiting on the keyed shared preparation flight.", + "# TYPE qbit_prism_shared_bundle_preparation_waiters gauge", + f"qbit_prism_shared_bundle_preparation_waiters {waiters}", + "# HELP qbit_prism_shared_bundle_builds_total Shared bundle builds by terminal outcome.", + "# TYPE qbit_prism_shared_bundle_builds_total counter", + *[ + f'qbit_prism_shared_bundle_builds_total{{result="{result}"}} {count}' + for result, count in sorted(build_counts.items()) + ], + ] + + +def reference_job_build_metrics_lines(server) -> list[str]: + server._ensure_job_cache_state() + with server._job_cache_lock: + bucket_counts = dict(server.job_build_seconds_bucket_counts) + build_sum = server.job_build_seconds_sum + build_count = server.job_build_count + phase_seconds = dict(server.job_build_phase_seconds) + hit_counts = dict(server.job_cache_hit_counts) + miss_counts = dict(server.job_cache_miss_counts) + health_refresh_failures = ( + server._ensure_observability_service() + .state() + .health_snapshot_refresh_failure_count + ) + with server._job_build_scheduler_lock: + scheduler_counts = dict(server.job_build_scheduler_counts) + priority_counts = dict(server.job_build_priority_counts) + priority_admission_seconds = dict( + server.job_build_priority_admission_seconds + ) + initial_prepared_counts = dict( + server.initial_job_prepared_work_counts + ) + cancellation_seconds = dict(server.job_build_cancellation_seconds) + replacement_seconds = dict(server.job_build_replacement_start_seconds) + worker_counts = dict(server.job_build_worker_counts) + active_builds = int(server._job_build_active is not None) + pending_builds = int(server._job_build_pending is not None) + priority_requests = tuple( + request + for request in ( + ( + server._job_build_active.request + if server._job_build_active is not None + else None + ), + ( + server._job_build_retiring.request + if server._job_build_retiring is not None + else None + ), + server._job_build_pending, + ) + if request is not None + and not request.cancellation.is_set() + and server._job_build_is_publication_critical(request) + ) + priority_preparations = tuple( + server._job_build_priority_preparations.values() + ) + priority_active = int( + bool(priority_requests or priority_preparations) + ) + priority_age_seconds = max( + ( + time.monotonic() - request.requested_monotonic + for request in priority_requests + ), + default=0.0, + ) + priority_age_seconds = max( + priority_age_seconds, + max( + ( + time.monotonic() - started + for started in priority_preparations + ), + default=0.0, + ), + ) + lock = getattr(server, "lock", None) + if lock is not None: + with lock: + connected_clients = len(getattr(server, "clients", ())) + else: + connected_clients = len(getattr(server, "clients", ())) + lines = [ + "# HELP qbit_prism_job_build_seconds Wall time from client job build or prepared submission to completion, including skipped prepared tasks.", + "# TYPE qbit_prism_job_build_seconds histogram", + ] + for bucket in PRISM_JOB_BUILD_SECONDS_BUCKETS: + lines.append( + f'qbit_prism_job_build_seconds_bucket{{le="{bucket:g}"}} {bucket_counts.get(bucket, 0)}' + ) + lines.extend( + [ + f'qbit_prism_job_build_seconds_bucket{{le="+Inf"}} {build_count}', + f"qbit_prism_job_build_seconds_sum {build_sum:.6f}", + f"qbit_prism_job_build_seconds_count {build_count}", + "# HELP qbit_prism_job_build_phase_seconds_total Cumulative job build wall time by phase.", + "# TYPE qbit_prism_job_build_phase_seconds_total counter", + *[ + f'qbit_prism_job_build_phase_seconds_total{{phase="{phase}"}} {phase_seconds.get(phase, 0.0):.6f}' + for phase in PRISM_JOB_BUILD_PHASES + ], + "# HELP qbit_prism_job_cache_hits_total Job build cache hits by cache kind.", + "# TYPE qbit_prism_job_cache_hits_total counter", + *[ + f'qbit_prism_job_cache_hits_total{{cache="{kind}"}} {int(hit_counts.get(kind, 0))}' + for kind in PRISM_JOB_CACHE_KINDS + ], + "# HELP qbit_prism_job_cache_misses_total Job build cache misses by cache kind.", + "# TYPE qbit_prism_job_cache_misses_total counter", + *[ + f'qbit_prism_job_cache_misses_total{{cache="{kind}"}} {int(miss_counts.get(kind, 0))}' + for kind in PRISM_JOB_CACHE_KINDS + ], + "# HELP qbit_prism_health_snapshot_refresh_failures_total Background health snapshot refreshes that raised.", + "# TYPE qbit_prism_health_snapshot_refresh_failures_total counter", + f"qbit_prism_health_snapshot_refresh_failures_total {health_refresh_failures}", + "# HELP qbit_prism_connected_clients Currently connected Stratum clients.", + "# TYPE qbit_prism_connected_clients gauge", + f"qbit_prism_connected_clients {connected_clients}", + "# HELP qbit_prism_job_build_requests_total Immutable job build requests admitted to the latest-wins scheduler.", + "# TYPE qbit_prism_job_build_requests_total counter", + f'qbit_prism_job_build_requests_total {int(scheduler_counts.get("requests", 0))}', + "# HELP qbit_prism_job_build_starts_total Immutable job builds started by the bounded executor.", + "# TYPE qbit_prism_job_build_starts_total counter", + f'qbit_prism_job_build_starts_total {int(scheduler_counts.get("starts", 0))}', + "# HELP qbit_prism_job_build_completions_total Immutable job build executions completed.", + "# TYPE qbit_prism_job_build_completions_total counter", + f'qbit_prism_job_build_completions_total {int(scheduler_counts.get("completions", 0))}', + "# HELP qbit_prism_job_build_supersessions_total Active or pending builds replaced by a newer immutable key.", + "# TYPE qbit_prism_job_build_supersessions_total counter", + f'qbit_prism_job_build_supersessions_total {int(scheduler_counts.get("supersessions", 0))}', + "# HELP qbit_prism_job_build_obsolete_results_total Obsolete build results discarded before cache or delivery.", + "# TYPE qbit_prism_job_build_obsolete_results_total counter", + f'qbit_prism_job_build_obsolete_results_total {int(scheduler_counts.get("obsolete_results", 0))}', + "# HELP qbit_prism_job_build_orphan_evicted_total Finished flights evicted from scheduler slots after their completion never released them.", + "# TYPE qbit_prism_job_build_orphan_evicted_total counter", + f'qbit_prism_job_build_orphan_evicted_total {int(scheduler_counts.get("orphan_evicted", 0))}', + "# HELP qbit_prism_job_build_active Current latest-generation build executions.", + "# TYPE qbit_prism_job_build_active gauge", + f"qbit_prism_job_build_active {active_builds}", + "# HELP qbit_prism_job_build_pending Newest build request waiting for a bounded executor slot.", + "# TYPE qbit_prism_job_build_pending gauge", + f"qbit_prism_job_build_pending {pending_builds}", + "# HELP qbit_prism_job_build_cancellation_seconds Cancellation signal to obsolete execution completion.", + "# TYPE qbit_prism_job_build_cancellation_seconds summary", + f'qbit_prism_job_build_cancellation_seconds_sum {float(cancellation_seconds.get("sum", 0.0)):.6f}', + f'qbit_prism_job_build_cancellation_seconds_count {int(cancellation_seconds.get("count", 0))}', + "# HELP qbit_prism_job_build_replacement_start_seconds Supersession signal to replacement build start.", + "# TYPE qbit_prism_job_build_replacement_start_seconds summary", + f'qbit_prism_job_build_replacement_start_seconds_sum {float(replacement_seconds.get("sum", 0.0)):.6f}', + f'qbit_prism_job_build_replacement_start_seconds_count {int(replacement_seconds.get("count", 0))}', + "# HELP qbit_prism_job_build_priority_events_total Publication-critical scheduler admissions and routine-work displacement.", + "# TYPE qbit_prism_job_build_priority_events_total counter", + *[ + f'qbit_prism_job_build_priority_events_total{{result="{result}"}} {int(priority_counts.get(result, 0))}' + for result in ( + "started", + "coalesced", + "queued", + "routine_deferred", + "routine_preempted", + ) + ], + "# HELP qbit_prism_job_build_priority_admission_seconds Publication-priority reservation to builder start or exact-flight coalescing.", + "# TYPE qbit_prism_job_build_priority_admission_seconds summary", + f'qbit_prism_job_build_priority_admission_seconds_sum {float(priority_admission_seconds.get("sum", 0.0)):.6f}', + f'qbit_prism_job_build_priority_admission_seconds_count {int(priority_admission_seconds.get("count", 0))}', + "# HELP qbit_prism_job_build_priority_active Whether publication-critical build work is preparing, running, retiring, or pending.", + "# TYPE qbit_prism_job_build_priority_active gauge", + f"qbit_prism_job_build_priority_active {priority_active}", + "# HELP qbit_prism_job_build_priority_age_seconds Age of the oldest admitted publication-critical build request.", + "# TYPE qbit_prism_job_build_priority_age_seconds gauge", + f"qbit_prism_job_build_priority_age_seconds {priority_age_seconds:.6f}", + "# HELP qbit_prism_initial_job_prepared_work_total Initial jobs that reused, coalesced behind, or deferred to prepared shared work.", + "# TYPE qbit_prism_initial_job_prepared_work_total counter", + *[ + f'qbit_prism_initial_job_prepared_work_total{{result="{result}"}} {int(initial_prepared_counts.get(result, 0))}' + for result in ( + "cache_hit", + "singleflight", + "deferred", + "subscribed", + "admission_deadline", + ) + ], + "# HELP qbit_prism_job_build_worker_events_total Pure builder subprocess lifecycle events.", + "# TYPE qbit_prism_job_build_worker_events_total counter", + *[ + f'qbit_prism_job_build_worker_events_total{{event="{event}"}} {int(worker_counts.get(event, 0))}' + for event in ("starts", "terminations", "crashes", "restarts") + ], + "# HELP qbit_prism_job_build_configured_workers Configured bounded job-build executor worker count.", + "# TYPE qbit_prism_job_build_configured_workers gauge", + ( + "qbit_prism_job_build_configured_workers " + f"{int(getattr(server, 'job_build_executor_workers', DEFAULT_PRISM_JOB_BUILD_EXECUTOR_WORKERS))}" + ), + ] + ) + return lines + + +def reference_vardiff_convergence_metrics_lines(server) -> list[str]: + snapshot = server.vardiff_convergence_snapshot() + lane_accepted = snapshot["lane_accepted_shares"] + assert isinstance(lane_accepted, dict) + resume_outcomes = snapshot["resume_outcomes"] + assert isinstance(resume_outcomes, dict) + lanes = [ + str(profile.name) + for profile in getattr(server, "listener_profiles", ()) or () + ] + lanes.extend(sorted(lane for lane in lane_accepted if lane not in lanes)) + elapsed = max(0.001, time.monotonic() - server.started_monotonic) + return [ + "# HELP qbit_prism_vardiff_sessions_at_max_difficulty Sessions whose current difficulty sits at their effective vardiff ceiling.", + "# TYPE qbit_prism_vardiff_sessions_at_max_difficulty gauge", + f"qbit_prism_vardiff_sessions_at_max_difficulty {int(snapshot['sessions_at_max_difficulty'])}", + "# HELP qbit_prism_vardiff_lane_accepted_shares_total Accepted shares by Stratum lane.", + "# TYPE qbit_prism_vardiff_lane_accepted_shares_total counter", + *[ + f'qbit_prism_vardiff_lane_accepted_shares_total{{lane="{server.prometheus_label_value(lane)}"}} {int(lane_accepted.get(lane, 0))}' + for lane in lanes + ], + "# HELP qbit_prism_vardiff_lane_accepted_shares_per_second Accepted shares per second by Stratum lane, averaged since coordinator start; a long-run average cannot show a transient reconnect storm -- use rate(qbit_prism_vardiff_lane_accepted_shares_total[5m]) for that. The numerator counts shares this process newly committed on the live submission path, not the ledger lifetime total published by qbit_prism_accepted_shares_total: an exact replay of an already-durable share and a startup recovery-journal replay are both acked accepted without being lane-counted, so the lanes sum to this coordinator's own commit rate and need not reconcile against an accepted-ack count.", + "# TYPE qbit_prism_vardiff_lane_accepted_shares_per_second gauge", + *[ + f'qbit_prism_vardiff_lane_accepted_shares_per_second{{lane="{server.prometheus_label_value(lane)}"}} {int(lane_accepted.get(lane, 0)) / elapsed:.12g}' + for lane in lanes + ], + "# HELP qbit_prism_vardiff_resume_total Reconnect difficulty resume attempts by outcome; clamped means the retained value was pulled into the lane's plausibility bounds, overridden means an adopted value was superseded by an explicit difficulty request in the same authorize, so resumed + clamped - overridden is the number that stuck.", + "# TYPE qbit_prism_vardiff_resume_total counter", + *[ + f'qbit_prism_vardiff_resume_total{{outcome="{outcome}"}} {int(resume_outcomes.get(outcome, 0))}' + for outcome in PRISM_VARDIFF_RESUME_OUTCOMES + ], + "# HELP qbit_prism_vardiff_resume_retained_sessions Worker sessions holding a retained difficulty eligible for resume.", + "# TYPE qbit_prism_vardiff_resume_retained_sessions gauge", + f"qbit_prism_vardiff_resume_retained_sessions {int(snapshot['retained_sessions'])}", + ] + + +def reference_accepted_preview_attribution_metrics_lines(server) -> list[str]: + snapshot = ensure_accepted_preview_telemetry(server).snapshot() + landing = snapshot["landing_phases"] + passes = snapshot["reconcile_passes"] + steps = snapshot["reconcile_steps"] + rescans = snapshot["full_rescans"] + + def summary(name: str, labels: str, stats: dict[str, object]) -> list[str]: + return [ + f'{name}_sum{{{labels}}} {float(stats["sum"]):.6f}', + f'{name}_count{{{labels}}} {int(stats["count"])}', + f'{name}_max{{{labels}}} {float(stats["max"]):.6f}', + ] + + lines = [ + "# HELP qbit_prism_accepted_block_landing_phase_seconds Accepted-block landing wall time between definitive node acceptance and payout preview publication, by bounded sub-phase; prior_balances_check is a prior-balances reread, never a payout-window rescan.", + "# TYPE qbit_prism_accepted_block_landing_phase_seconds summary", + ] + for phase in PRISM_ACCEPTED_LANDING_PHASES: + lines.extend( + summary( + "qbit_prism_accepted_block_landing_phase_seconds", + f'phase="{phase}"', + landing[phase], + ) + ) + lines.extend( + [ + "# HELP qbit_prism_reorg_reconcile_pass_seconds Reorg reconciliation passes by bounded caller; _count is the pass count, and a pass minus the sum of its steps is the unattributed remainder.", + "# TYPE qbit_prism_reorg_reconcile_pass_seconds summary", + ] + ) + for caller in PRISM_REORG_RECONCILE_CALLERS: + lines.extend( + summary( + "qbit_prism_reorg_reconcile_pass_seconds", + f'caller="{caller}"', + passes[caller], + ) + ) + lines.extend( + [ + "# HELP qbit_prism_reorg_reconcile_step_seconds Reorg reconciliation wall time by bounded caller and step; admission_wait is writer admission plus the payout-balance mutation lock, and candidate_prepare is where a reconcile_invalidation full rescan executes.", + "# TYPE qbit_prism_reorg_reconcile_step_seconds summary", + ] + ) + for caller in PRISM_REORG_RECONCILE_CALLERS: + for step in PRISM_REORG_RECONCILE_STEPS: + lines.extend( + summary( + "qbit_prism_reorg_reconcile_step_seconds", + f'caller="{caller}",step="{step}"', + steps[(caller, step)], + ) + ) + lines.extend( + [ + "# HELP qbit_prism_payout_window_full_rescan_seconds Full payout-window oracle rescans by bounded reason and window pipeline path; a prior-balances reread is a ledger read operation, not an observation here.", + "# TYPE qbit_prism_payout_window_full_rescan_seconds summary", + ] + ) + for reason in PRISM_PAYOUT_WINDOW_FULL_RESCAN_REASONS: + for path in PRISM_PAYOUT_WINDOW_FULL_RESCAN_PATHS: + lines.extend( + summary( + "qbit_prism_payout_window_full_rescan_seconds", + f'reason="{reason}",path="{path}"', + rescans[(reason, path)], + ) + ) + return lines + + +def reference_render_metrics_payload(server) -> str: + ledger_metrics = server.ledger.metrics() + audit_metrics = server.audit_artifact_metrics() + mining_metrics = server.mining_delivery_snapshot() + process_rss_bytes, process_open_fds = server.process_resource_metrics() + accepted_share_count = server.accepted_share_stats()[0] + server._ensure_share_hot_path_state() + with server._share_accounting_lock: + submitted_share_count = int(getattr(server, "submitted_share_count", 0)) + stale_share_count = int(getattr(server, "stale_share_count", 0)) + duplicate_share_count = int(getattr(server, "duplicate_share_count", 0)) + low_difficulty_share_count = int( + getattr(server, "low_difficulty_share_count", 0) + ) + collection_block_submission_count = int( + getattr(server, "collection_block_submission_count", 0) + ) + rejection_counts = dict( + getattr(server, "rejection_counts_by_reason", {}) + ) + grace_credited_share_count = int( + getattr(server, "grace_credited_share_count", 0) + ) + block_solves_dropped_counts = dict( + getattr( + server, + "block_solves_dropped_counts", + {"stale_grace": 0}, + ) + ) + stale_percent = 0.0 + if submitted_share_count > 0: + stale_percent = (stale_share_count / submitted_share_count) * 100.0 + idle_retarget_count = int(getattr(server, "idle_retarget_count", 0)) + with server.lock: + server._ensure_connection_capacity_state() + active_connection_count = len(server.clients) + connection_limit_rejection_counts = dict( + server.connection_limit_rejection_counts + ) + accept_resource_exhaustion_count = int( + getattr(server, "accept_resource_exhaustion_count", 0) + ) + connection_setup_failure_count = int( + getattr(server, "connection_setup_failure_count", 0) + ) + server._ensure_evicted_job_state() + server.prune_evicted_job_graveyard(force=False) + same_tip_context_count = len(server.evicted_same_tip_job_ids) + evicted_job_context_counts = { + "same_tip": same_tip_context_count, + "stale_grace": len(server.evicted_job_graveyard) - same_tip_context_count, + } + evicted_job_submit_counts = dict(server.evicted_job_submit_counts) + evicted_job_expiration_counts = dict(server.evicted_job_expiration_counts) + evicted_job_capacity_eviction_counts = dict( + server.evicted_job_capacity_eviction_counts + ) + stale_job_abandon_counts = dict( + getattr( + server, + "stale_job_abandon_counts", + { + abandon_class: 0 + for abandon_class in PRISM_STALE_JOB_ABANDON_CLASSES + }, + ) + ) + server._ensure_worker_metrics_state() + with server.worker_metrics_lock: + worker_share_counts = { + label: dict(counts) + for label, counts in server.worker_share_counts.items() + } + worker_rejection_counts = dict(server.worker_rejection_counts) + coinbase_weight_headroom = 2_000_000 + latest_coinbase_size_bytes = getattr(server, "latest_coinbase_size_bytes", None) + if latest_coinbase_size_bytes is not None: + coinbase_weight_headroom = 2_000_000 - int(latest_coinbase_size_bytes) + ctv_pending = 0 + ctv_broadcastable = 0 + ctv_failed = 0 + pending_ctv_fanouts = getattr(server.ledger, "pending_ctv_fanout_statuses", None) + if callable(pending_ctv_fanouts): + try: + for fanout in pending_ctv_fanouts(limit=1_000): + ctv_pending += 1 + status = str(fanout.get("settlement_status", "")) + if status == "broadcastable": + ctv_broadcastable += 1 + elif status == "failed": + ctv_failed += 1 + except Exception: + ctv_pending = -1 + ctv_broadcastable = -1 + ctv_failed = -1 + if ctv_failed >= 0: + ctv_failed = int(ledger_metrics.get("ctv_fanouts_failed", ctv_failed)) + ibd = 0 + peers = 0 + try: + blockchain_info = server.rpc.call("getblockchaininfo") + if isinstance(blockchain_info, dict) and blockchain_info.get("initialblockdownload"): + ibd = 1 + except Exception: + ibd = -1 + try: + network_info = server.rpc.call("getnetworkinfo") + if isinstance(network_info, dict): + peers = int(network_info.get("connections", 0)) + except Exception: + peers = -1 + lines = [ + "# HELP qbit_prism_accepted_shares_total Accepted shares recorded by the canonical PRISM ledger.", + "# TYPE qbit_prism_accepted_shares_total counter", + f"qbit_prism_accepted_shares_total {accepted_share_count}", + *reference_accepted_stats_reconcile_metric_lines(server), + *reference_landing_observability_metrics_lines(server), + "# HELP qbit_prism_submitted_shares_total Stratum share submissions seen by the PRISM coordinator.", + "# TYPE qbit_prism_submitted_shares_total counter", + f"qbit_prism_submitted_shares_total {submitted_share_count}", + "# HELP qbit_prism_stratum_active_connections Active admitted Stratum connections across all listeners.", + "# TYPE qbit_prism_stratum_active_connections gauge", + f"qbit_prism_stratum_active_connections {active_connection_count}", + "# HELP qbit_prism_stratum_connection_limit Configured global Stratum connection ceiling; zero means unlimited.", + "# TYPE qbit_prism_stratum_connection_limit gauge", + f"qbit_prism_stratum_connection_limit {mining_metrics['connection_capacity']}", + "# HELP qbit_prism_stratum_peak_active_connections Peak admitted Stratum connections since process start.", + "# TYPE qbit_prism_stratum_peak_active_connections gauge", + f"qbit_prism_stratum_peak_active_connections {mining_metrics['peak_active_connections']}", + "# HELP qbit_prism_stratum_subscribed_connections Active subscribed Stratum connections.", + "# TYPE qbit_prism_stratum_subscribed_connections gauge", + f"qbit_prism_stratum_subscribed_connections {mining_metrics['subscribed_connections']}", + "# HELP qbit_prism_stratum_authorized_connections Active subscribed and authorized Stratum connections.", + "# TYPE qbit_prism_stratum_authorized_connections gauge", + f"qbit_prism_stratum_authorized_connections {mining_metrics['authorized_connections']}", + "# HELP qbit_prism_stratum_pending_initial_jobs Authorized clients awaiting their first usable current-tip job.", + "# TYPE qbit_prism_stratum_pending_initial_jobs gauge", + f"qbit_prism_stratum_pending_initial_jobs {mining_metrics['pending_initial_jobs']}", + "# HELP qbit_prism_stratum_pending_initial_job_limit Configured bound for clients awaiting their first usable job.", + "# TYPE qbit_prism_stratum_pending_initial_job_limit gauge", + f"qbit_prism_stratum_pending_initial_job_limit {mining_metrics['pending_initial_job_capacity']}", + "# HELP qbit_prism_stratum_oldest_pending_initial_job_seconds Age of the oldest pending first-job request.", + "# TYPE qbit_prism_stratum_oldest_pending_initial_job_seconds gauge", + f"qbit_prism_stratum_oldest_pending_initial_job_seconds {mining_metrics['oldest_pending_initial_job_age_seconds']}", + "# HELP qbit_prism_stratum_oldest_genuinely_pending_initial_job_seconds Age of the oldest authorized client that has never received usable work.", + "# TYPE qbit_prism_stratum_oldest_genuinely_pending_initial_job_seconds gauge", + f"qbit_prism_stratum_oldest_genuinely_pending_initial_job_seconds {mining_metrics['oldest_genuinely_pending_initial_job_age_seconds']}", + "# HELP qbit_prism_stratum_current_tip_coverage_gap_seconds Continuous age of current-tip job coverage below 95 percent.", + "# TYPE qbit_prism_stratum_current_tip_coverage_gap_seconds gauge", + f"qbit_prism_stratum_current_tip_coverage_gap_seconds {mining_metrics['current_tip_coverage_gap_age_seconds']}", + "# HELP qbit_prism_stratum_initial_job_queue_rejections_total Sessions closed because bounded first-job delivery was full.", + "# TYPE qbit_prism_stratum_initial_job_queue_rejections_total counter", + f"qbit_prism_stratum_initial_job_queue_rejections_total {mining_metrics['initial_job_queue_rejections']}", + "# HELP qbit_prism_stratum_initial_job_timeouts_total Sessions disconnected after first-job delivery timed out.", + "# TYPE qbit_prism_stratum_initial_job_timeouts_total counter", + f"qbit_prism_stratum_initial_job_timeouts_total {mining_metrics['initial_job_timeout_disconnects']}", + "# HELP qbit_prism_stratum_initial_job_tasks_total First-job tasks canceled or coalesced before duplicate work.", + "# TYPE qbit_prism_stratum_initial_job_tasks_total counter", + f'qbit_prism_stratum_initial_job_tasks_total{{result="cancelled"}} {mining_metrics["initial_job_cancelled_tasks"]}', + f'qbit_prism_stratum_initial_job_tasks_total{{result="coalesced"}} {mining_metrics["initial_job_coalesced_tasks"]}', + "# HELP qbit_prism_stratum_initial_job_queue_capacity_reclaimed_total Queued first-job admission slots reclaimed immediately by cancellation.", + "# TYPE qbit_prism_stratum_initial_job_queue_capacity_reclaimed_total counter", + f'qbit_prism_stratum_initial_job_queue_capacity_reclaimed_total {mining_metrics["initial_job_queue_capacity_reclaimed"]}', + "# HELP qbit_prism_stratum_clients_with_current_tip_jobs Authorized clients holding usable current-tip work.", + "# TYPE qbit_prism_stratum_clients_with_current_tip_jobs gauge", + f"qbit_prism_stratum_clients_with_current_tip_jobs {mining_metrics['clients_with_current_tip_jobs']}", + "# HELP qbit_prism_stratum_current_tip_job_coverage Ratio of authorized clients holding current-tip work.", + "# TYPE qbit_prism_stratum_current_tip_job_coverage gauge", + f"qbit_prism_stratum_current_tip_job_coverage {mining_metrics['current_tip_job_coverage']}", + "# HELP qbit_prism_stratum_semantic_current_work_ratio Ratio of authorized clients whose work matches the current template fingerprint, payout generation, and client session.", + "# TYPE qbit_prism_stratum_semantic_current_work_ratio gauge", + f"qbit_prism_stratum_semantic_current_work_ratio {mining_metrics['semantic_current_work_ratio']}", + "# HELP qbit_prism_stratum_handler_threads Active per-connection Stratum handler threads.", + "# TYPE qbit_prism_stratum_handler_threads gauge", + f"qbit_prism_stratum_handler_threads {mining_metrics['handler_threads']}", + "# HELP qbit_prism_job_delivery_queue_depth Current bounded delivery executor queue depth.", + "# TYPE qbit_prism_job_delivery_queue_depth gauge", + f"qbit_prism_job_delivery_queue_depth {mining_metrics['delivery_executor_queue_depth']}", + "# HELP qbit_prism_job_delivery_active_workers Delivery executor workers currently running tasks.", + "# TYPE qbit_prism_job_delivery_active_workers gauge", + f"qbit_prism_job_delivery_active_workers {mining_metrics['delivery_executor_active_workers']}", + "# HELP qbit_prism_process_resident_memory_bytes Current process RSS bytes, or -1 when unavailable.", + "# TYPE qbit_prism_process_resident_memory_bytes gauge", + f"qbit_prism_process_resident_memory_bytes {process_rss_bytes}", + "# HELP qbit_prism_process_open_file_descriptors Current process open descriptor count, or -1 when unavailable.", + "# TYPE qbit_prism_process_open_file_descriptors gauge", + f"qbit_prism_process_open_file_descriptors {process_open_fds}", + "# HELP qbit_prism_stratum_connection_limit_rejections_total Stratum connections rejected by an explicitly configured admission limit.", + "# TYPE qbit_prism_stratum_connection_limit_rejections_total counter", + *[ + f'qbit_prism_stratum_connection_limit_rejections_total{{scope="{scope}"}} {int(connection_limit_rejection_counts.get(scope, 0))}' + for scope in ("global", "username") + ], + "# HELP qbit_prism_stratum_accept_resource_exhaustions_total Recoverable Stratum accept or client-setup failures caused by process or system descriptor exhaustion.", + "# TYPE qbit_prism_stratum_accept_resource_exhaustions_total counter", + f"qbit_prism_stratum_accept_resource_exhaustions_total {accept_resource_exhaustion_count}", + "# HELP qbit_prism_stratum_connection_setup_failures_total Admitted Stratum connections cleaned up after socket or handler-thread setup failure.", + "# TYPE qbit_prism_stratum_connection_setup_failures_total counter", + f"qbit_prism_stratum_connection_setup_failures_total {connection_setup_failure_count}", + "# HELP qbit_prism_stale_shares_total Stratum shares rejected or ignored as stale.", + "# TYPE qbit_prism_stale_shares_total counter", + f"qbit_prism_stale_shares_total {stale_share_count}", + "# HELP qbit_prism_duplicate_shares_total Duplicate Stratum shares rejected.", + "# TYPE qbit_prism_duplicate_shares_total counter", + f"qbit_prism_duplicate_shares_total {duplicate_share_count}", + "# HELP qbit_prism_low_difficulty_shares_total Low-difficulty Stratum shares rejected.", + "# TYPE qbit_prism_low_difficulty_shares_total counter", + f"qbit_prism_low_difficulty_shares_total {low_difficulty_share_count}", + "# HELP qbit_prism_collection_block_submissions_total Solver-pays-all block candidates submitted from collection-mode jobs.", + "# TYPE qbit_prism_collection_block_submissions_total counter", + f"qbit_prism_collection_block_submissions_total {collection_block_submission_count}", + "# HELP qbit_prism_grace_credited_shares_total Accepted shares credited by the stale-grace policy.", + "# TYPE qbit_prism_grace_credited_shares_total counter", + f"qbit_prism_grace_credited_shares_total {grace_credited_share_count}", + "# HELP qbit_prism_block_solves_dropped_total Block-passing submissions intentionally excluded from block submission by a bounded policy reason.", + "# TYPE qbit_prism_block_solves_dropped_total counter", + *[ + f'qbit_prism_block_solves_dropped_total{{reason="{reason}"}} {int(block_solves_dropped_counts.get(reason, 0))}' + for reason in ("stale_grace",) + ], + "# HELP qbit_prism_rejections_total PRISM share or block rejections by canonical reason ID.", + "# TYPE qbit_prism_rejections_total counter", + *[ + f'qbit_prism_rejections_total{{reason_id="{reason}"}} {int(rejection_counts.get(reason, 0))}' + for reason in PRISM_REJECTION_REASON_IDS + ], + "# HELP qbit_prism_worker_submitted_shares_total Stratum share submissions by bounded worker label.", + "# TYPE qbit_prism_worker_submitted_shares_total counter", + *[ + f'qbit_prism_worker_submitted_shares_total{{worker="{server.prometheus_label_value(label)}"}} {int(counts.get("submitted", 0))}' + for label, counts in sorted(worker_share_counts.items()) + ], + "# HELP qbit_prism_worker_accepted_shares_total Accepted shares by bounded worker label.", + "# TYPE qbit_prism_worker_accepted_shares_total counter", + *[ + f'qbit_prism_worker_accepted_shares_total{{worker="{server.prometheus_label_value(label)}"}} {int(counts.get("accepted", 0))}' + for label, counts in sorted(worker_share_counts.items()) + ], + "# HELP qbit_prism_worker_grace_credited_shares_total Stale-grace credited shares by bounded worker label.", + "# TYPE qbit_prism_worker_grace_credited_shares_total counter", + *[ + f'qbit_prism_worker_grace_credited_shares_total{{worker="{server.prometheus_label_value(label)}"}} {int(counts.get("grace", 0))}' + for label, counts in sorted(worker_share_counts.items()) + ], + "# HELP qbit_prism_worker_rejections_total PRISM share or block rejections by bounded worker label and reason ID.", + "# TYPE qbit_prism_worker_rejections_total counter", + *[ + f'qbit_prism_worker_rejections_total{{worker="{server.prometheus_label_value(label)}",reason_id="{reason}"}} {int(count)}' + for (label, reason), count in sorted(worker_rejection_counts.items()) + ], + "# HELP qbit_prism_job_build_failures_total Job builds skipped after a template/coinbase error without dropping the client.", + "# TYPE qbit_prism_job_build_failures_total counter", + f"qbit_prism_job_build_failures_total {server.job_build_failure_count}", + "# HELP qbit_prism_block_candidates_dropped_total Legacy counter; durable candidate outbox rows are never dropped on queue overflow.", + "# TYPE qbit_prism_block_candidates_dropped_total counter", + f"qbit_prism_block_candidates_dropped_total {int(getattr(server, 'block_candidates_dropped', 0))}", + "# HELP qbit_prism_block_candidate_wakeups_coalesced_total Candidate queue wakeups coalesced while the durable outbox retained the work.", + "# TYPE qbit_prism_block_candidate_wakeups_coalesced_total counter", + f"qbit_prism_block_candidate_wakeups_coalesced_total {int(getattr(server, 'block_candidate_wakeups_coalesced', 0))}", + "# HELP qbit_prism_block_candidate_retries_total Transient candidate outcomes retained for durable retry.", + "# TYPE qbit_prism_block_candidate_retries_total counter", + f"qbit_prism_block_candidate_retries_total {int(getattr(server, 'block_candidate_retry_count', 0))}", + "# HELP qbit_prism_block_candidate_accept_pending_defers_total Terminal abandonments refused because the candidate is (or was recently observed as) an active chain block; the candidate retries until its accepted success tail finalizes it as submitted.", + "# TYPE qbit_prism_block_candidate_accept_pending_defers_total counter", + f"qbit_prism_block_candidate_accept_pending_defers_total {int(getattr(server, 'block_candidate_accept_pending_defer_count', 0))}", + "# HELP qbit_prism_block_candidate_poisoned_total Invalid durable candidate intents quarantined from replay.", + "# TYPE qbit_prism_block_candidate_poisoned_total counter", + f"qbit_prism_block_candidate_poisoned_total {int(getattr(server, 'block_candidate_poisoned_count', 0))}", + "# HELP qbit_prism_block_candidates_abandoned_total Block candidates that did not land (lost tip race or failed submit), by reason. Not share rejections: the underlying share was accepted.", + "# TYPE qbit_prism_block_candidates_abandoned_total counter", + *[ + f'qbit_prism_block_candidates_abandoned_total{{reason_id="{reason}"}} {int(count)}' + for reason, count in sorted(getattr(server, "block_candidate_abandoned_counts", {}).items()) + ], + "# HELP qbit_prism_stale_job_abandons_total Terminal stale-job block candidate abandonments by bounded cause.", + "# TYPE qbit_prism_stale_job_abandons_total counter", + *[ + f'qbit_prism_stale_job_abandons_total{{class="{abandon_class}"}} {int(stale_job_abandon_counts.get(abandon_class, 0))}' + for abandon_class in PRISM_STALE_JOB_ABANDON_CLASSES + ], + "# HELP qbit_prism_share_append_queue_depth Accepted shares waiting on the ledger writer thread.", + "# TYPE qbit_prism_share_append_queue_depth gauge", + f"qbit_prism_share_append_queue_depth {server.share_append_queue.qsize() if getattr(server, 'share_append_queue', None) is not None else 0}", + "# HELP qbit_prism_share_append_failures_total Shares in group commits that failed before acknowledgement.", + "# TYPE qbit_prism_share_append_failures_total counter", + f"qbit_prism_share_append_failures_total {int(getattr(server, 'share_append_failure_count', 0))}", + "# HELP qbit_prism_shares_recovered_to_disk_total Legacy pre-commit-ACK shares written to the upgrade recovery file.", + "# TYPE qbit_prism_shares_recovered_to_disk_total counter", + f"qbit_prism_shares_recovered_to_disk_total {int(getattr(server, 'shares_recovered_to_disk', 0))}", + "# HELP qbit_prism_shares_replayed_total Recovery-file shares replayed into the ledger at startup.", + "# TYPE qbit_prism_shares_replayed_total counter", + f"qbit_prism_shares_replayed_total {int(getattr(server, 'shares_replayed', 0))}", + "# HELP qbit_prism_share_replay_conflicts_total Recovery-file rows quarantined because the durable row disagrees with the journal payload.", + "# TYPE qbit_prism_share_replay_conflicts_total counter", + f"qbit_prism_share_replay_conflicts_total {int(getattr(server, 'share_replay_conflicts', 0))}", + "# HELP qbit_prism_tip_refresh_jobs_total Client jobs refreshed after qbit tip/template changes.", + "# TYPE qbit_prism_tip_refresh_jobs_total counter", + f"qbit_prism_tip_refresh_jobs_total {server.tip_refresh_job_count}", + "# HELP qbit_prism_active_job_contexts Current retained PRISM job contexts.", + "# TYPE qbit_prism_active_job_contexts gauge", + f"qbit_prism_active_job_contexts {len(getattr(server, 'jobs', {}))}", + "# HELP qbit_prism_evicted_job_contexts Evicted job contexts retained by safety class.", + "# TYPE qbit_prism_evicted_job_contexts gauge", + *[ + f'qbit_prism_evicted_job_contexts{{class="{job_class}"}} {evicted_job_context_counts[job_class]}' + for job_class in PRISM_EVICTED_JOB_CLASSES + ], + "# HELP qbit_prism_evicted_job_submits_total Accepted submits validated against an evicted job context.", + "# TYPE qbit_prism_evicted_job_submits_total counter", + *[ + f'qbit_prism_evicted_job_submits_total{{outcome="{outcome}"}} {int(evicted_job_submit_counts.get(outcome, 0))}' + for outcome in PRISM_EVICTED_JOB_SUBMIT_OUTCOMES + ], + "# HELP qbit_prism_evicted_job_expirations_total Retained job contexts removed after their class TTL.", + "# TYPE qbit_prism_evicted_job_expirations_total counter", + *[ + f'qbit_prism_evicted_job_expirations_total{{class="{job_class}"}} {int(evicted_job_expiration_counts.get(job_class, 0))}' + for job_class in PRISM_EVICTED_JOB_CLASSES + ], + "# HELP qbit_prism_evicted_job_capacity_evictions_total Same-tip retained contexts removed by a configured count limit.", + "# TYPE qbit_prism_evicted_job_capacity_evictions_total counter", + *[ + f'qbit_prism_evicted_job_capacity_evictions_total{{scope="{scope}"}} {int(evicted_job_capacity_eviction_counts.get(scope, 0))}' + for scope in PRISM_EVICTED_JOB_CAPACITY_SCOPES + ], + "# HELP qbit_prism_post_accept_refresh_failures_total Immediate clean-job refreshes that failed after direct block acceptance.", + "# TYPE qbit_prism_post_accept_refresh_failures_total counter", + f"qbit_prism_post_accept_refresh_failures_total {server.post_accept_refresh_failure_count}", + "# HELP qbit_prism_reorg_inactive_blocks_total PRISM pool blocks quarantined after leaving the active chain.", + "# TYPE qbit_prism_reorg_inactive_blocks_total counter", + f"qbit_prism_reorg_inactive_blocks_total {server.reorg_inactive_block_count}", + "# HELP qbit_prism_reorg_reactivated_blocks_total Quarantined PRISM pool blocks restored after returning to the active chain.", + "# TYPE qbit_prism_reorg_reactivated_blocks_total counter", + f"qbit_prism_reorg_reactivated_blocks_total {server.reorg_reactivated_block_count}", + "# HELP qbit_prism_reorg_reconcile_skips_total Reorg reconciliation passes skipped because qbitd chain view was not trusted.", + "# TYPE qbit_prism_reorg_reconcile_skips_total counter", + f"qbit_prism_reorg_reconcile_skips_total {server.reorg_reconcile_skip_count}", + "# HELP qbit_prism_reorg_reconcile_errors_total Reorg reconciliation errors that prevented ordered job issuance.", + "# TYPE qbit_prism_reorg_reconcile_errors_total counter", + f"qbit_prism_reorg_reconcile_errors_total {server.reorg_reconcile_error_count}", + "# HELP qbit_prism_reorg_reconcile_lookups_total Reconcile demand by caller path and the source that satisfied it.", + "# TYPE qbit_prism_reorg_reconcile_lookups_total counter", + *[ + f'qbit_prism_reorg_reconcile_lookups_total{{path="{path}",source="{source}"}} ' + f"{int(getattr(server, 'reorg_reconcile_lookup_counts', {}).get((path, source), 0))}" + for path in PRISM_REORG_RECONCILE_LOOKUP_PATHS + for source in PRISM_REORG_RECONCILE_LOOKUP_SOURCES + ], + "# HELP qbit_prism_matured_payouts_total Payout entries marked mature by the coordinator tip reconciliation path.", + "# TYPE qbit_prism_matured_payouts_total counter", + f"qbit_prism_matured_payouts_total {server.matured_payout_count}", + "# HELP qbit_prism_vardiff_idle_retargets_total Vardiff retargets triggered by the idle zero-accepted-share sweep.", + "# TYPE qbit_prism_vardiff_idle_retargets_total counter", + f"qbit_prism_vardiff_idle_retargets_total {idle_retarget_count}", + "# HELP qbit_prism_stale_share_percent Percent of submitted shares classified stale.", + "# TYPE qbit_prism_stale_share_percent gauge", + f"qbit_prism_stale_share_percent {stale_percent:.12g}", + "# HELP qbit_prism_blocks_accepted_total Blocks accepted through the PRISM coordinator.", + "# TYPE qbit_prism_blocks_accepted_total counter", + f"qbit_prism_blocks_accepted_total {server.accepted_block_count}", + "# HELP qbit_prism_persisted_blocks Persisted PRISM pool block rows.", + "# TYPE qbit_prism_persisted_blocks gauge", + f"qbit_prism_persisted_blocks {ledger_metrics['blocks']}", + "# HELP qbit_prism_inactive_pool_blocks PRISM pool block rows currently quarantined as inactive.", + "# TYPE qbit_prism_inactive_pool_blocks gauge", + f"qbit_prism_inactive_pool_blocks {ledger_metrics.get('inactive_blocks', 0)}", + "# HELP qbit_prism_reversed_pool_blocks PRISM pool block rows terminally reversed.", + "# TYPE qbit_prism_reversed_pool_blocks gauge", + f"qbit_prism_reversed_pool_blocks {ledger_metrics.get('reversed_blocks', 0)}", + "# HELP qbit_prism_rejected_pool_blocks PRISM pool block rows rejected before confirmation.", + "# TYPE qbit_prism_rejected_pool_blocks gauge", + f"qbit_prism_rejected_pool_blocks {ledger_metrics.get('rejected_blocks', 0)}", + "# HELP qbit_prism_owed_accounts Current accounts with positive carried owed balances.", + "# TYPE qbit_prism_owed_accounts gauge", + f"qbit_prism_owed_accounts {ledger_metrics['owed_accounts']}", + "# HELP qbit_prism_coinbase_weight_headroom_bytes Remaining qbit block weight bytes after the latest pool coinbase.", + "# TYPE qbit_prism_coinbase_weight_headroom_bytes gauge", + f"qbit_prism_coinbase_weight_headroom_bytes {coinbase_weight_headroom}", + "# HELP qbit_prism_ctv_fanouts_pending Pending non-terminal CTV fanouts known to the ledger, or -1 if unavailable.", + "# TYPE qbit_prism_ctv_fanouts_pending gauge", + f"qbit_prism_ctv_fanouts_pending {ctv_pending}", + "# HELP qbit_prism_ctv_fanouts_broadcastable CTV fanouts that are mature enough to broadcast, or -1 if unavailable.", + "# TYPE qbit_prism_ctv_fanouts_broadcastable gauge", + f"qbit_prism_ctv_fanouts_broadcastable {ctv_broadcastable}", + "# HELP qbit_prism_ctv_fanouts_failed CTV fanouts with failed or rejected broadcast state, or -1 if unavailable.", + "# TYPE qbit_prism_ctv_fanouts_failed gauge", + f"qbit_prism_ctv_fanouts_failed {ctv_failed}", + "# HELP qbit_prism_vardiff_enabled Whether PRISM Stratum vardiff is enabled.", + "# TYPE qbit_prism_vardiff_enabled gauge", + f"qbit_prism_vardiff_enabled {1 if server.vardiff_config.enabled else 0}", + "# HELP qbit_prism_qbitd_initial_block_download qbitd initialblockdownload status, or -1 if unavailable.", + "# TYPE qbit_prism_qbitd_initial_block_download gauge", + f"qbit_prism_qbitd_initial_block_download {ibd}", + "# HELP qbit_prism_qbitd_peers qbitd peer count, or -1 if unavailable.", + "# TYPE qbit_prism_qbitd_peers gauge", + f"qbit_prism_qbitd_peers {peers}", + "# HELP qbit_prism_audit_artifact_bytes Bytes used by PRISM audit artifacts in PRISM_AUDIT_DIR by artifact kind.", + "# TYPE qbit_prism_audit_artifact_bytes gauge", + *[ + f'qbit_prism_audit_artifact_bytes{{kind="{kind}"}} {audit_metrics[kind]["bytes"]}' + for kind in ("body", "share_segment", "live_bundle", "candidate", "other") + ], + "# HELP qbit_prism_audit_artifact_files PRISM audit artifact file count in PRISM_AUDIT_DIR by artifact kind.", + "# TYPE qbit_prism_audit_artifact_files gauge", + *[ + f'qbit_prism_audit_artifact_files{{kind="{kind}"}} {audit_metrics[kind]["files"]}' + for kind in ("body", "share_segment", "live_bundle", "candidate", "other") + ], + "# HELP qbit_prism_audit_artifact_scan_error Whether the latest PRISM_AUDIT_DIR metric scan failed.", + "# TYPE qbit_prism_audit_artifact_scan_error gauge", + f"qbit_prism_audit_artifact_scan_error {audit_metrics['scan_error']}", + ] + lines.extend(reference_lease_heartbeat_metrics_lines(server)) + lines.extend(reference_shutdown_metrics_lines(server)) + lines.extend(reference_coordinator_lock_metrics_lines(server)) + lines.extend(reference_block_submitter_metrics_lines(server)) + lines.extend(reference_block_candidate_cleanup_backlog_metrics_lines(server)) + lines.extend(reference_share_ack_metrics_lines(server)) + lines.extend(server.ctv_fanout_broadcaster_metrics_lines()) + lines.extend(server.vardiff_idle_metrics_lines()) + lines.extend(reference_vardiff_convergence_metrics_lines(server)) + lines.extend(server.block_finalization_metrics_lines()) + lines.extend(reference_job_build_metrics_lines(server)) + lines.extend(server.tip_refresh_metrics_lines()) + lines.extend(server.payout_state_metrics_lines()) + lines.extend(reference_initial_delivery_metrics_lines(server)) + lines.extend(server.progress_health_metrics_lines()) + lines.extend(reference_accepted_preview_attribution_metrics_lines(server)) + lines.extend(reference_gc_pause_metrics_lines(server)) + lines.extend(reference_process_heap_metrics_lines(PINNED_HEAP_SAMPLE)) + lines.extend(reference_component_cardinality_metrics_lines(server)) + return "\n".join(lines) + "\n" + + +class MetricsRenderParityTests(unittest.TestCase): + """Byte-for-byte parity between the renderer and the frozen reference.""" + + def _seeded_coordinator(self): + server = support.coordinator() + temporary = tempfile.TemporaryDirectory() + server.audit_dir = Path(temporary.name) / "audit" + server.evidence_path = Path(temporary.name) / "state" / "evidence.json" + + def cleanup() -> None: + store = server.__dict__.get("_audit_artifact_store") + if store is not None: + store.close() + temporary.cleanup() + + self.addCleanup(cleanup) + # Pin the /proc-derived gauges: RSS moves between the two renders. + server.process_resource_metrics = lambda: (123_456_789, 42) + # Pin issue #226's heap collector the same way: allocation and GC + # counters move between the reference and the actual render. The + # pinned renderer is the one metrics_payload() resolves. + server.__dict__["_metrics_renderer"] = MetricsRenderer( + server, + process_telemetry=SimpleNamespace(sample=lambda: PINNED_HEAP_SAMPLE), + ) + # Issue #226's component gauges, seeded through the shipped owners + # so the parity document carries non-zero values: a bundle-cache + # entry, a cached two-page payout window, a serialization slot with + # a spool, a reconcile flight and memo entry, a pending-share floor + # holder, a terminal outcome, and one daemon-uploaded window. + bundles = server._ensure_job_bundle_service() + bundles._job_bundle_cache[("parity-bundle",)] = object() + bundles._share_window_serialization = _ShareWindowSerialization( + key=("parity-digest", 3, 0), + share_count=3, + share_snapshot_sha256="parity-digest", + ) + bundles._share_window_serialization._spool_size = 4_096 + parity_pages = ( + _IncrementalShareWindowPage( + records=(object(), object()), + total_difficulty=0, + prism_json_records=({}, {}), + canonical_json_items=b'{"a":1},{"b":2}', + ), + _IncrementalShareWindowPage( + records=(object(),), + total_difficulty=0, + prism_json_records=({},), + canonical_json_items=b'{"c":3}', + ), + ) + payout = server._ensure_payout_state_service() + payout._incremental_payout_artifact_window = _IncrementalPayoutArtifactWindow( + window=IncrementalShareWindow( + anchor_job_issued_at_ms=1, + window_weight=1, + page_size=512, + pages=parity_pages, + total_difficulty=0, + ), + shares_json=IncrementalShareJsonSequence(pages=parity_pages, record_count=3), + share_snapshot_sha256="parity-digest", + refreshed_monotonic=0.0, + full_rescan_monotonic=0.0, + full_rescan_attempt_monotonic=0.0, + ) + reconciler = server._ensure_reorg_reconciler_service() + reconciler._reconcile_flights["parity-tip"] = _ReconcileFlight() + reconciler._reorg_reconcile_trusted_memo["parity-tip"] = 1.0 + server._ensure_share_writer_service().adopt_pending_share( + SimpleNamespace(share_id="parity-share") + ) + candidate_service = server._ensure_block_candidate_service() + candidate_service._ensure_block_candidate_disposition_state() + candidate_service._block_candidate_terminal_outcomes["ef" * 32] = True + daemon_client = _ServeBuilderClient(process=SimpleNamespace()) + daemon_client.note_uploaded_window("parity-digest") + server._ensure_bundle_compiler()._serve_builder = daemon_client + # Share accounting and rejection counters. + server.submitted_share_count = 10 + server.stale_share_count = 2 + server.duplicate_share_count = 1 + server.low_difficulty_share_count = 3 + server.collection_block_submission_count = 4 + server.grace_credited_share_count = 6 + server.idle_retarget_count = 7 + server.rejection_counts_by_reason["stale-job"] = 2 + server.rejection_counts_by_reason["duplicate-share"] = 1 + # Worker-labeled counters. + server.worker_share_counts = { + "alice": {"submitted": 5, "accepted": 4, "grace": 1}, + "bob": {"submitted": 2, "accepted": 1}, + } + server.worker_rejection_counts = {("alice", "stale-job"): 2} + # Connection admission counters. + server.connection_limit_rejection_counts = {"global": 2, "username": 3} + server.accept_resource_exhaustion_count = 4 + server.connection_setup_failure_count = 5 + # Share-ACK histograms via the submission owner. + server._observe_share_ack_seconds("accepted", 0.05) + server._observe_share_ack_seconds("rejected", 0.2) + # Block-candidate producer state (routed to the B1 owner). + server.block_candidate_retry_count = 2 + server.block_candidate_accept_pending_defer_count = 1 + server.block_candidate_poisoned_count = 1 + server.block_candidate_wakeups_coalesced = 3 + server.block_candidate_abandoned_counts = {"stale-job": 2} + server.stale_job_abandon_counts = { + abandon_class: index + for index, abandon_class in enumerate(PRISM_STALE_JOB_ABANDON_CLASSES) + } + server.block_solves_dropped_counts = {"stale_grace": 2} + server._observe_block_submit_seconds(0.5) + # Acceptance-to-preview-publication needs both halves of the interval: + # the submitter-thread acceptance stamp and the publication that + # closes it. Seeding through the shipped B1 entry points keeps the + # parity document exercising the real observer, not a hand-built + # histogram. + service = server._ensure_block_candidate_service() + service._note_accepted_block_preview_acceptance("ab" * 32) + server._observe_accepted_block_preview_publication( + "ab" * 32, + result="published", + ) + # Issue #198's cleanup-retry backlog, seeded through the shipped + # deferral and backpressure entry points: one owed hash carrying one + # retained holder and a published fence, and one engagement that + # preserved two rows, so every new family renders a non-zero value. + service._record_block_candidate_terminal_outcome("cd" * 32, accepted=False) + service._defer_collapsed_candidate_cleanup( + "cd" * 32, + ("retry-state",), + shares=(SimpleNamespace(share_id="miner-a:" + "cd" * 32),), + ) + with redirect_stdout(StringIO()): + service._note_block_candidate_cleanup_backpressure( + caller="replay-page", + rows=2, + admitted=0, + depth=1, + maximum=1, + ) + server._record_block_ledger_call( + call_class="landing", + budget_seconds=30.0, + duration_seconds=0.25, + timed_out=False, + ) + server._record_block_ledger_call( + call_class="fast", + budget_seconds=1.0, + duration_seconds=1.2, + timed_out=True, + ) + # Reorg reconciler outcome and lookup counters. + server.reorg_inactive_block_count = 2 + server.reorg_reactivated_block_count = 1 + server.reorg_reconcile_skip_count = 3 + server.reorg_reconcile_error_count = 1 + server.matured_payout_count = 4 + server._record_reorg_reconcile_lookup("job_build", "memo_hit") + server._record_reorg_reconcile_lookup("job_build", "serial") + server._record_reorg_reconcile_lookup("tip_refresh", "overlap") + # Landing observability inputs. + server._accepted_parent_preview_wait_timeouts = 2 + server._startup_phase_origin_monotonic = time.monotonic() - 1.0 + server._record_startup_phase_once("audit_listener_bound") + server._record_startup_phase_once("node_rpc_ready") + # Ledger-side optional snapshots. + server.ledger.accepted_stats_reconcile_status = lambda: { + "failures": 1, + "age_seconds": 2.5, + } + server.ledger.prior_balances_read_stats = lambda: { + "reads_total": 3, + "last_seconds": 0.01, + "max_seconds": 0.02, + } + # Issue #211's attribution: local admission and server execution are + # separate series so a budget exhaustion names the half it went to. + # Issue #224 Wave 0: a contract operation renders under its own + # name, an operation outside PRISM_LEDGER_READ_OPERATIONS folds + # into ``other`` instead of opening a series. + server.ledger.ledger_read_gate_stats = lambda: { + "pending_block_candidate_rows": { + "calls_total": 7, + "gate_wait_seconds_total": 0.125, + "gate_wait_seconds_max": 0.1, + "gate_timeouts_total": 1, + "execute_seconds_total": 4.5, + "execute_seconds_max": 2.25, + "execute_timeouts_total": 2, + }, + "current_prior_balances": { + "calls_total": 3, + "gate_wait_seconds_total": 0.5, + "gate_wait_seconds_max": 0.4, + "gate_timeouts_total": 0, + "execute_seconds_total": 0.75, + "execute_seconds_max": 0.5, + "execute_timeouts_total": 0, + }, + "legacy_probe": { + "calls_total": 1, + "gate_wait_seconds_total": 0.01, + "gate_wait_seconds_max": 0.01, + "gate_timeouts_total": 0, + "execute_seconds_total": 0.02, + "execute_seconds_max": 0.02, + "execute_timeouts_total": 0, + }, + } + # Issue #224 Wave 0's attribution families, seeded through the + # shared owner every lane will record into, including one + # out-of-vocabulary rescan reason that must fold into ``other``. + telemetry = ensure_accepted_preview_telemetry(server) + telemetry.observe_landing_phase("reconcile", 0.25) + telemetry.observe_landing_phase("reconcile", 0.75) + telemetry.observe_landing_phase("preview_publish", 0.05) + telemetry.observe_reconcile_pass("landing", 0.5) + telemetry.observe_reconcile_step("landing", "watch_query", 0.125) + telemetry.observe_reconcile_step("post_confirm", "candidate_prepare", 3.5) + telemetry.observe_payout_window_full_rescan( + "reconcile_invalidation", "daemon", 3.5 + ) + telemetry.observe_payout_window_full_rescan( + "not-a-contract-reason", "in_process", 0.5 + ) + server.ledger.block_candidate_pending_metrics = lambda: { + "pending_count": 2, + "oldest_pending_age_seconds": 1.5, + "oldest_unattempted_age_seconds": 0.5, + } + server.ledger.pending_ctv_fanout_statuses = lambda limit: [ + {"settlement_status": "broadcastable"}, + {"settlement_status": "failed"}, + {"settlement_status": "pending"}, + ] + # Job-build histogram and phase state via the J1 owner. + server._ensure_job_cache_state() + server.observe_job_build_elapsed(0.05, {"assemble": 0.02, "payout": 0.01}) + server._record_job_cache_event("template", hit=True) + server._record_job_cache_event("bundle", hit=False) + # Vardiff convergence/resume state via the vardiff owner. + server.listener_profiles = [ + SimpleNamespace(name="default"), + SimpleNamespace(name="highdiff"), + ] + vardiff_service = server._ensure_vardiff_service() + with vardiff_service._vardiff_convergence_lock: + vardiff_service.vardiff_lane_accepted_counts["default"] = 9 + vardiff_service.vardiff_lane_accepted_counts["highdiff"] = 2 + vardiff_service.vardiff_resume_outcome_counts["resumed"] = 3 + vardiff_service.vardiff_resume_outcome_counts["clamped"] = 1 + vardiff_service.vardiff_resume_outcome_counts["overridden"] = 2 + vardiff_service.session_difficulty_store.record( + ("default", "alice"), + Decimal("32768"), + now=time.monotonic(), + share_backed=True, + ) + return server + + def test_full_render_parity_with_frozen_reference(self) -> None: + server = self._seeded_coordinator() + frozen_now = time.monotonic() + with mock.patch.object(time, "monotonic", return_value=frozen_now): + expected = reference_render_metrics_payload(server) + actual = server.metrics_payload() + self.assertEqual(expected, actual) + # The parity document must exercise the families the acceptance + # checklist names; an accidentally empty fixture would prove nothing. + for needle in ( + "qbit_prism_block_candidate_accept_pending_defers_total 1", + 'qbit_prism_share_ack_seconds_count{result="accepted"} 1', + "qbit_prism_block_submit_seconds_count 1", + 'qbit_prism_accepted_block_preview_publication_seconds_count{result="published"} 1', + 'qbit_prism_reorg_reconcile_lookups_total{path="job_build",source="memo_hit"} 1', + 'qbit_prism_block_solves_dropped_total{reason="stale_grace"} 2', + 'qbit_prism_stale_job_abandons_total{class="', + "qbit_prism_stratum_semantic_current_work_ratio", + "qbit_prism_job_build_configured_workers", + 'qbit_prism_initial_job_prepared_work_total{result="admission_deadline"}', + "qbit_prism_job_build_orphan_evicted_total", + "qbit_prism_accepted_stats_reconcile_failures_total 1", + "qbit_prism_accepted_stats_reconcile_age_seconds 2.500000", + 'qbit_prism_block_ledger_calls_total{call_class="fast"} 1', + 'qbit_prism_block_ledger_call_timeouts_total{call_class="fast"} 1', + "qbit_prism_prior_balances_reads_total 3", + 'qbit_prism_ledger_read_calls_total{operation="pending_block_candidate_rows"} 7', + 'qbit_prism_ledger_read_gate_wait_seconds_max{operation="pending_block_candidate_rows"} 0.100000', + 'qbit_prism_ledger_read_gate_timeouts_total{operation="pending_block_candidate_rows"} 1', + 'qbit_prism_ledger_read_execute_seconds_max{operation="pending_block_candidate_rows"} 2.250000', + 'qbit_prism_ledger_read_execute_timeouts_total{operation="pending_block_candidate_rows"} 2', + 'qbit_prism_startup_phase_seconds{phase="audit_listener_bound"}', + "qbit_prism_accepted_parent_preview_wait_timeouts_total 2", + "qbit_prism_tip_refresh_wave_outcomes_total", + "qbit_prism_payout_artifact_events_total", + 'qbit_prism_payout_window_build_phase_seconds_count{phase="ledger_read",outcome="completed"}', + 'qbit_prism_payout_window_build_phase_seconds_count{phase="lock_wait",outcome="failed"}', + "qbit_prism_vardiff_sessions_at_max_difficulty", + 'qbit_prism_vardiff_resume_total{outcome="clamped"}', + 'qbit_prism_vardiff_resume_total{outcome="overridden"} 2', + 'qbit_prism_vardiff_lane_accepted_shares_per_second{lane="default"}', + 'qbit_prism_lease_heartbeat_attempts_total{mode="proof"}', + 'qbit_prism_lease_heartbeat_phase_seconds{phase="guard_slot_wait"}', + 'qbit_prism_lease_heartbeat_phase_seconds{phase="guard_client_resume"}', + 'qbit_prism_lease_heartbeat_policy_seconds{term="scheduler_slack"}', + 'qbit_prism_lease_heartbeat_policy_seconds{term="exit_envelope"}', + 'qbit_prism_lease_heartbeat_policy_seconds{term="server_proven_cap"}', + 'qbit_prism_lease_heartbeat_policy_seconds{term="max_guaranteed_monitor_lateness"} 0.550000', + 'qbit_prism_lease_heartbeat_monitor_wake_lateness_seconds_bucket{le="+Inf"}', + 'qbit_prism_lease_heartbeat_monitor_late_wakes_total{slack_fraction="1.0"}', + "qbit_prism_lease_heartbeat_monitor_wake_delay_window_max_seconds", + "qbit_prism_lease_heartbeat_monitor_wake_delay_record_age_seconds", + "qbit_prism_lease_heartbeat_monitor_exit_guarantee_breaches_total", + "qbit_prism_lease_heartbeat_stall_probe_samples_total", + "qbit_prism_lease_heartbeat_monitor_breach_warnings_suppressed_total", + 'qbit_prism_process_gc_pause_seconds_bucket{generation="2",le="+Inf"}', + 'qbit_prism_process_gc_max_pause_seconds{generation="0"}', + "qbit_prism_block_candidate_cleanup_retry_backlog 1", + f"qbit_prism_block_candidate_cleanup_retry_backlog_max {DEFAULT_BLOCK_CANDIDATE_CLEANUP_RETRY_BACKLOG_MAX}", + "qbit_prism_block_candidate_cleanup_retry_pending_share_holders 1", + "qbit_prism_block_candidate_cleanup_retry_terminal_outcome_pins 1", + "qbit_prism_block_candidate_cleanup_backpressure_active 0", + "qbit_prism_block_candidate_cleanup_backpressure_total 1", + 'qbit_prism_block_candidate_collapse_total{outcome="backlog_deferred"} 2', + 'qbit_prism_ledger_read_calls_total{operation="current_prior_balances"} 3', + 'qbit_prism_ledger_read_calls_total{operation="other"} 1', + 'qbit_prism_accepted_block_landing_phase_seconds_count{phase="reconcile"} 2', + 'qbit_prism_accepted_block_landing_phase_seconds_max{phase="reconcile"} 0.750000', + 'qbit_prism_accepted_block_landing_phase_seconds_sum{phase="lane_wait"} 0.000000', + 'qbit_prism_reorg_reconcile_pass_seconds_count{caller="landing"} 1', + 'qbit_prism_reorg_reconcile_step_seconds_sum{caller="landing",step="watch_query"} 0.125000', + 'qbit_prism_reorg_reconcile_step_seconds_max{caller="post_confirm",step="candidate_prepare"} 3.500000', + 'qbit_prism_payout_window_full_rescan_seconds_count{reason="reconcile_invalidation",path="daemon"} 1', + 'qbit_prism_payout_window_full_rescan_seconds_count{reason="other",path="in_process"} 1', + 'qbit_prism_payout_window_full_rescan_seconds_count{reason="window_daemon_state_lost",path="daemon"} 0', + "qbit_prism_process_allocated_blocks 123456", + 'qbit_prism_process_gc_trigger_count{generation="2"} 7', + 'qbit_prism_process_gc_uncollectable_objects_total{generation="2"} 2', + "qbit_prism_process_threads 64", + "qbit_prism_process_malloc_info_available 1", + "qbit_prism_process_malloc_arena_bytes 9000", + "qbit_prism_process_malloc_mmapped_bytes 500", + 'qbit_prism_component_entries{component="payout_window_pages"} 2', + 'qbit_prism_component_entries{component="payout_window_records"} 3', + 'qbit_prism_component_entries{component="share_window_serialization_shares"} 3', + 'qbit_prism_component_entries{component="job_bundle_cache"} 1', + # The #198 seed above publishes one terminal outcome of its own. + 'qbit_prism_component_entries{component="block_candidate_terminal_outcomes"} 2', + 'qbit_prism_component_entries{component="reconcile_flights"} 1', + 'qbit_prism_component_entries{component="reconcile_trusted_memo"} 1', + 'qbit_prism_component_entries{component="pending_share_commit_floor"} 1', + 'qbit_prism_component_entries{component="daemon_uploaded_windows"} 1', + 'qbit_prism_component_bytes{component="payout_window_canonical_json"} 22', + 'qbit_prism_component_bytes{component="share_window_serialization_spool"} 4096', + ): + self.assertIn(needle, actual) + self.assertNotIn('operation="legacy_probe"', actual) + self.assertNotIn("not-a-contract-reason", actual) + # The oldest-age gauge is a real elapsed interval, not the -1 + # sentinel, once a record is owed. + oldest = [ + line + for line in actual.splitlines() + if line.startswith("qbit_prism_block_candidate_cleanup_retry_oldest_seconds ") + ] + self.assertEqual(len(oldest), 1) + self.assertGreaterEqual(float(oldest[0].split()[1]), 0.0) + + def test_payout_window_build_phase_family_is_fixed_cardinality(self) -> None: + """Issue #228: one histogram family over closed phase and outcome sets.""" + server = self._seeded_coordinator() + payload = server.metrics_payload() + family = "qbit_prism_payout_window_build_phase_seconds" + self.assertEqual( + PRISM_PAYOUT_WINDOW_BUILD_PHASES, + ("ledger_read", "record_conversion", "daemon_prepare", "lock_wait"), + ) + self.assertEqual( + PRISM_PAYOUT_WINDOW_BUILD_OUTCOMES, + ("completed", "debounced", "aborted", "failed"), + ) + self.assertEqual(payload.count(f"# TYPE {family} histogram\n"), 1) + series = [ + line + for line in payload.splitlines() + if line.startswith(f"{family}_") + ] + self.assertTrue(series) + cells_seen: set[tuple[str, str]] = set() + for line in series: + labels = line.split("{", 1)[1].split("}", 1)[0] + keys = {pair.split("=", 1)[0] for pair in labels.split(",")} + # Only the closed phase and outcome labels and the histogram's + # own ``le``: no per-tip, per-generation or per-reason label + # may ever ride this family. + self.assertTrue(keys <= {"phase", "outcome", "le"}, line) + phase = labels.split('phase="', 1)[1].split('"', 1)[0] + outcome = labels.split('outcome="', 1)[1].split('"', 1)[0] + cells_seen.add((phase, outcome)) + self.assertEqual( + cells_seen, + { + (phase, outcome) + for phase in PRISM_PAYOUT_WINDOW_BUILD_PHASES + for outcome in PRISM_PAYOUT_WINDOW_BUILD_OUTCOMES + }, + ) + for phase, outcome in cells_seen: + labels = f'phase="{phase}",outcome="{outcome}"' + self.assertEqual( + sum(1 for line in series if line.startswith(f"{family}_count{{{labels}}}")), + 1, + ) + self.assertEqual( + sum(1 for line in series if line.startswith(f"{family}_sum{{{labels}}}")), + 1, + ) + + def test_renderer_bypasses_cached_metrics_path(self) -> None: + server = self._seeded_coordinator() + server.refresh_metrics_snapshot = mock.Mock( # type: ignore[method-assign] + side_effect=AssertionError("renderer must not re-enter the cache") + ) + server.cached_metrics_payload = mock.Mock( # type: ignore[method-assign] + side_effect=AssertionError("renderer must not re-enter the cache") + ) + payload = MetricsRenderer(server).render() + self.assertIn("qbit_prism_accepted_shares_total", payload) + + +class RemovedCompatibilitySurfaceTests(unittest.TestCase): + """PR 80's deleted re-exports and health properties must stay absent.""" + + REMOVED_MODULE_EXPORTS = ( + # coordinator_config re-exports + "DEFAULT_HIGHDIFF_DIFFICULTY", + "DEFAULT_HIGHDIFF_MAX_DIFFICULTY", + "DEFAULT_MIN_OUTPUT_FEERATE_SATS_PER_BYTE", + "DEFAULT_MIN_OUTPUT_SAFETY_MULTIPLIER", + "DEFAULT_P2MR_SPEND_INPUT_BYTES", + "DEFAULT_PRISM_COINBASE_TAG", + "DEFAULT_PRISM_VARDIFF_IDLE_SWEEP_SECONDS", + "DEFAULT_TESTNET_USERNAME_FALLBACK_ADDRESS", + "MAX_PRISM_COINBASE_TAG_BYTES", + "default_prism_coinbase_tag_hex", + "env_decimal", + "env_nonnegative_int_with_legacy", + "env_optional_bool", + "env_seed_hex", + "load_prism_highdiff_listener", + "load_prism_vardiff_config", + "production_mode", + "require_production_env", + "validate_prism_production_gate", + "validate_same_tip_job_retention_limits", + # stratum_session re-exports + "parse_stratum_password_options", + "parse_worker_username", + "split_worker_username", + # ctv re-exports + "MAX_CTV_FANOUT_BROADCASTER_CHUNK_SIZE", + "PRISM_CTV_BROADCASTER_CHUNK_ROWS_BUCKETS", + "PRISM_CTV_BROADCASTER_CHUNK_SECONDS_BUCKETS", + "PRISM_CTV_BROADCASTER_SECONDS_BUCKETS", + # job_bundle re-exports + "CollectionIdentityUnavailable", + "JobBuildKey", + "_JobBuildCancelled", + # job_delivery re-exports + "DEFAULT_PRISM_EVICTED_JOB_PRUNE_INTERVAL_SECONDS", + "InitialJobSnapshot", + "InitialJobTracker", + "MAX_ACTIVE_PRISM_JOBS_PER_CLIENT", + "PRISM_TIP_REFRESH_ADMISSION_POLL_SECONDS", + # tip_refresh re-exports + "PRISM_TIP_REFRESH_CANCELLATION_STAGES", + "PRISM_TIP_REFRESH_RESULTS", + "PublishedTipSnapshot", + # progress_health re-exports + "BundleBuildToken", + "RefreshActivityToken", + # payout_state aliases and re-export + "_AcceptedBlockPayoutTransition", + "_PayoutDeliveryAdmission", + "_PayoutStateDeliveryGate", + "_PayoutStatePublicationBlocked", + "PublishedPayoutState", + # coordinator_shutdown re-export + "_WriterOperationToken", + # duplicate job-build registries (canonical in job_bundle) + "PRISM_JOB_BUILD_PHASES", + "PRISM_JOB_CACHE_KINDS", + # reorg machinery moved to lab.prism.reorg_reconciler + "_ReconcileFlight", + "PRISM_REORG_RECONCILE_MEMO_MAX_TIPS", + "PRISM_REORG_RECONCILE_LOOKUP_PATHS", + "PRISM_REORG_RECONCILE_LOOKUP_SOURCES", + "PRISM_RECONCILE_PREFETCH_JOIN_TIMEOUT_CEILING_SECONDS", + ) + + KEPT_MODULE_EXPORTS = ( + "BackgroundServiceRegistry", + "BackgroundServiceSpec", + "CoordinatorShutdownController", + "ShutdownInProgress", + "ledger_writer_operation", + "DEFAULT_SHARE_COMMIT_BATCH_SIZE", + "DEFAULT_SHARE_COMMIT_LINGER_MILLISECONDS", + "DEFAULT_SHARE_COMMIT_TIMEOUT_SECONDS", + "_BoundedPriorityExecutor", + "_DeliveryQueueFull", + ) + + REMOVED_HEALTH_PROPERTIES = ( + "_health_snapshot", + "_health_snapshot_monotonic", + "_health_refresh_loop_running", + "_health_snapshot_lock", + "health_snapshot_refresh_failure_count", + ) + + def test_removed_reexports_are_absent_from_the_coordinator(self) -> None: + import lab.prism.prism_coordinator as prism_coordinator + + for name in self.REMOVED_MODULE_EXPORTS: + self.assertFalse(hasattr(prism_coordinator, name), name) + for name in self.KEPT_MODULE_EXPORTS: + self.assertTrue(hasattr(prism_coordinator, name), name) + # The buckets survive only as an import of the canonical owner. + self.assertIs( + prism_coordinator.PRISM_JOB_BUILD_SECONDS_BUCKETS, + PRISM_JOB_BUILD_SECONDS_BUCKETS, + ) + + def test_health_compatibility_properties_are_gone(self) -> None: + from lab.prism.observability import ObservabilityService + from lab.prism.prism_coordinator import PrismCoordinator + + for name in self.REMOVED_HEALTH_PROPERTIES: + self.assertFalse(hasattr(PrismCoordinator, name), name) + for name in ( + "lock", + "set_health_snapshot_for_compatibility", + "set_health_snapshot_monotonic_for_compatibility", + "set_loop_running_for_compatibility", + "set_refresh_failure_count_for_compatibility", + ): + self.assertFalse(hasattr(ObservabilityService, name), name) + for name in ( + "replace_lock_for_test", + "clear_health_snapshot_for_test", + "set_health_snapshot_monotonic_for_test", + "set_loop_running_for_test", + ): + self.assertTrue(hasattr(ObservabilityService, name), name) + + +class MetricsRendererTests(unittest.TestCase): + def test_shutdown_formatter_consumes_one_owner_snapshot(self) -> None: + snapshot_calls = 0 + + def snapshot() -> dict[str, object]: + nonlocal snapshot_calls + snapshot_calls += 1 + return { + "shutdowns_total": 1, + "writer_quiescence_outcomes": {"success": 1, "timeout": 0}, + "lease_release_outcomes": { + "success": 1, + "not_held": 0, + "unsupported": 0, + "failure": 0, + }, + "active_writers": {"candidate": 2}, + "writer_quiescence_seconds": 0.25, + "lease_release_attempts_total": 1, + "lease_release_seconds": 0.125, + "sigterm_release_observed": True, + "sigterm_to_lease_release_seconds": 0.5, + "release_withheld_total": 0, + "non_writer_drain_seconds": 0.75, + } + + port = SimpleNamespace( + _ensure_shutdown_controller=lambda: SimpleNamespace( + snapshot=snapshot + ), + prometheus_label_value=lambda value: value, + ) + lines = MetricsRenderer(port).shutdown_metrics_lines() # type: ignore[arg-type] + + self.assertEqual(snapshot_calls, 1) + self.assertIn("qbit_prism_shutdowns_total 1", lines) + self.assertIn( + 'qbit_prism_shutdown_writer_operations{component="candidate"} 2', + lines, + ) + self.assertIn( + "qbit_prism_shutdown_sigterm_to_lease_release_seconds 0.500000", + lines, + ) + + def test_block_submitter_formatter_consumes_one_merged_snapshot(self) -> None: + snapshot_calls = 0 + + def snapshot() -> dict[str, object]: + nonlocal snapshot_calls + snapshot_calls += 1 + return { + "pending_count": 2, + "oldest_pending_age_seconds": 1.5, + "oldest_unattempted_age_seconds": 0.5, + "backoff_active": True, + "backoff_remaining_seconds": 0.25, + "backoff_delay_seconds": 1.0, + "submit_seconds_buckets": {0.05: 1}, + "submit_seconds_sum": 0.04, + "submit_seconds_count": 1, + "accepted_preview_publication": { + "published": { + "buckets": { + bucket: (1 if bucket >= 0.5 else 0) + for bucket in ( + PRISM_ACCEPTED_BLOCK_PREVIEW_PUBLICATION_SECONDS_BUCKETS + ) + }, + "sum": 0.4, + "count": 1, + }, + "degraded": { + "buckets": { + bucket: 0 + for bucket in ( + PRISM_ACCEPTED_BLOCK_PREVIEW_PUBLICATION_SECONDS_BUCKETS + ) + }, + "sum": 0.0, + "count": 0, + }, + }, + } + + port = SimpleNamespace( + block_submitter_snapshot=snapshot, + block_candidate_collapse_snapshot=lambda: { + outcome: 0 for outcome in PRISM_BLOCK_CANDIDATE_COLLAPSE_OUTCOMES + }, + ) + lines = MetricsRenderer(port).block_submitter_metrics_lines() # type: ignore[arg-type] + + self.assertEqual(snapshot_calls, 1) + self.assertIn("qbit_prism_block_candidates_pending 2", lines) + self.assertIn("qbit_prism_block_submitter_retry_backoff_active 1", lines) + self.assertIn("qbit_prism_block_submit_seconds_count 1", lines) + self.assertIn( + 'qbit_prism_block_submit_seconds_bucket{le="0.05"} 1', + lines, + ) + self.assertIn( + "qbit_prism_accepted_block_preview_publication_seconds_bucket" + '{result="published",le="0.5"} 1', + lines, + ) + # The 5 s child wait budget must be an exact bucket boundary: the + # acceptance criterion is a p95 below it, which a Prometheus + # histogram can only answer at a boundary it carries. + self.assertIn( + "qbit_prism_accepted_block_preview_publication_seconds_bucket" + '{result="published",le="5"} 1', + lines, + ) + self.assertIn( + "qbit_prism_accepted_block_preview_publication_seconds_count" + '{result="published"} 1', + lines, + ) + self.assertIn( + "qbit_prism_accepted_block_preview_publication_seconds_count" + '{result="degraded"} 0', + lines, + ) + + def test_cleanup_backlog_formatter_consumes_one_owner_snapshot(self) -> None: + """Issue #198: one snapshot read, seven unlabelled families.""" + snapshot_calls = 0 + + def snapshot() -> dict[str, object]: + nonlocal snapshot_calls + snapshot_calls += 1 + return { + "depth": 3, + "backlog_max": 4096, + "oldest_age_seconds": 12.5, + "pending_share_holders": 2, + "terminal_outcome_pins": 3, + "backpressure_active": False, + "backpressure_engagements": 4, + } + + port = SimpleNamespace( + _ensure_block_candidate_service=lambda: SimpleNamespace( + collapsed_candidate_cleanup_backlog_snapshot=snapshot, + ), + ) + lines = MetricsRenderer(port).block_candidate_cleanup_backlog_metrics_lines() # type: ignore[arg-type] + + self.assertEqual(snapshot_calls, 1) + self.assertIn("qbit_prism_block_candidate_cleanup_retry_backlog 3", lines) + self.assertIn("qbit_prism_block_candidate_cleanup_retry_backlog_max 4096", lines) + self.assertIn( + "qbit_prism_block_candidate_cleanup_retry_oldest_seconds 12.500000", + lines, + ) + self.assertIn( + "qbit_prism_block_candidate_cleanup_retry_pending_share_holders 2", + lines, + ) + self.assertIn( + "qbit_prism_block_candidate_cleanup_retry_terminal_outcome_pins 3", + lines, + ) + self.assertIn("qbit_prism_block_candidate_cleanup_backpressure_active 0", lines) + self.assertIn("qbit_prism_block_candidate_cleanup_backpressure_total 4", lines) + # Fixed cardinality: no series in the family carries a label. + for line in lines: + if not line.startswith("#"): + self.assertNotIn("{", line) + self.assertEqual( + [ + (line.split()[2], line.split()[3]) + for line in lines + if line.startswith("# TYPE ") + ], + list(BLOCK_CANDIDATE_CLEANUP_BACKLOG_METRIC_FAMILIES), + ) + + def test_cleanup_backlog_families_render_from_the_shipped_owner(self) -> None: + """An empty backlog renders the -1 age sentinel and the default bound.""" + server = support.coordinator() + lines = MetricsRenderer(server).block_candidate_cleanup_backlog_metrics_lines() + self.assertIn("qbit_prism_block_candidate_cleanup_retry_backlog 0", lines) + self.assertIn( + "qbit_prism_block_candidate_cleanup_retry_backlog_max " + f"{DEFAULT_BLOCK_CANDIDATE_CLEANUP_RETRY_BACKLOG_MAX}", + lines, + ) + self.assertIn( + "qbit_prism_block_candidate_cleanup_retry_oldest_seconds -1.000000", + lines, + ) + self.assertIn("qbit_prism_block_candidate_cleanup_backpressure_active 0", lines) + # A pinned coordinator bound is the one exported, so an alert can be + # written as a ratio against the running configuration. + server.block_candidate_cleanup_retry_backlog_max = 12 + lines = MetricsRenderer(server).block_candidate_cleanup_backlog_metrics_lines() + self.assertIn("qbit_prism_block_candidate_cleanup_retry_backlog_max 12", lines) + self.assertEqual(lines, reference_block_candidate_cleanup_backlog_metrics_lines(server)) + + def test_collapse_counter_carries_the_backlog_deferred_outcome(self) -> None: + self.assertIn("backlog_deferred", PRISM_BLOCK_CANDIDATE_COLLAPSE_OUTCOMES) + server = support.coordinator() + lines = MetricsRenderer(server).block_submitter_metrics_lines() + self.assertIn( + 'qbit_prism_block_candidate_collapse_total{outcome="backlog_deferred"} 0', + lines, + ) + + def test_accepted_preview_attribution_formatter_consumes_one_owner_snapshot( + self, + ) -> None: + """Issue #224 Wave 0: one snapshot read, four closed-product families.""" + snapshot_calls = 0 + telemetry = AcceptedPreviewTelemetry() + telemetry.observe_landing_phase("balance_lock_wait", 1.5) + telemetry.observe_reconcile_pass("tip_refresh", 0.25) + telemetry.observe_reconcile_step("tip_refresh", "admission_wait", 0.2) + telemetry.observe_payout_window_full_rescan( + "window_daemon_state_lost", "in_process", 2.0 + ) + real_snapshot = telemetry.snapshot + + def snapshot() -> dict[str, object]: + nonlocal snapshot_calls + snapshot_calls += 1 + return real_snapshot() + + telemetry.snapshot = snapshot # type: ignore[method-assign] + port = SimpleNamespace(_accepted_preview_telemetry=telemetry) + lines = MetricsRenderer(port).accepted_preview_attribution_metrics_lines() # type: ignore[arg-type] + + self.assertEqual(snapshot_calls, 1) + self.assertIn( + 'qbit_prism_accepted_block_landing_phase_seconds_max{phase="balance_lock_wait"} 1.500000', + lines, + ) + self.assertIn( + 'qbit_prism_reorg_reconcile_pass_seconds_count{caller="tip_refresh"} 1', + lines, + ) + self.assertIn( + 'qbit_prism_reorg_reconcile_step_seconds_sum{caller="tip_refresh",step="admission_wait"} 0.200000', + lines, + ) + self.assertIn( + 'qbit_prism_payout_window_full_rescan_seconds_count{reason="window_daemon_state_lost",path="in_process"} 1', + lines, + ) + # Every cell of every closed product renders, at zero when unobserved. + expected_samples = 3 * ( + len(PRISM_ACCEPTED_LANDING_PHASES) + + len(PRISM_REORG_RECONCILE_CALLERS) + + len(PRISM_REORG_RECONCILE_CALLERS) * len(PRISM_REORG_RECONCILE_STEPS) + + len(PRISM_PAYOUT_WINDOW_FULL_RESCAN_REASONS) + * len(PRISM_PAYOUT_WINDOW_FULL_RESCAN_PATHS) + ) + self.assertEqual( + len([line for line in lines if not line.startswith("#")]), + expected_samples, + ) + + def test_ledger_read_operations_outside_the_contract_fold_into_other( + self, + ) -> None: + """Issue #224 Wave 0: the operation label cannot grow past the vocabulary.""" + stats = { + "calls_total": 2, + "gate_wait_seconds_total": 0.25, + "gate_wait_seconds_max": 0.2, + "gate_timeouts_total": 1, + "execute_seconds_total": 1.0, + "execute_seconds_max": 0.75, + "execute_timeouts_total": 0, + } + port = SimpleNamespace( + ledger=SimpleNamespace( + ledger_read_gate_stats=lambda: { + "zeta_probe": dict(stats), + "pending_block_candidate_rows": dict(stats), + "alpha_probe": dict(stats, gate_wait_seconds_max=0.9), + } + ), + prometheus_label_value=lambda value: value, + ) + lines = MetricsRenderer(port)._ledger_read_gate_metric_lines() # type: ignore[arg-type] + + operations = [ + line.split('operation="', 1)[1].split('"', 1)[0] + for line in lines + if line.startswith("qbit_prism_ledger_read_calls_total{") + ] + self.assertEqual(operations, ["other", "pending_block_candidate_rows"]) + self.assertIn('qbit_prism_ledger_read_calls_total{operation="other"} 4', lines) + self.assertIn( + 'qbit_prism_ledger_read_gate_wait_seconds_max{operation="other"} 0.900000', + lines, + ) + self.assertIn( + 'qbit_prism_ledger_read_gate_timeouts_total{operation="other"} 2', + lines, + ) + self.assertIn( + 'qbit_prism_ledger_read_calls_total{operation="pending_block_candidate_rows"} 2', + lines, + ) + + def test_share_ack_formatter_renders_owner_snapshot(self) -> None: + histograms = empty_share_ack_histograms() + histograms["accepted"]["buckets"][PRISM_JOB_BUILD_SECONDS_BUCKETS[0]] = 2 + histograms["accepted"]["sum"] = 0.01 + histograms["accepted"]["count"] = 2 + port = SimpleNamespace(share_ack_snapshot=lambda: histograms) + lines = MetricsRenderer(port).share_ack_metrics_lines() # type: ignore[arg-type] + self.assertIn( + f'qbit_prism_share_ack_seconds_bucket{{result="accepted",le="{PRISM_JOB_BUILD_SECONDS_BUCKETS[0]:g}"}} 2', + lines, + ) + self.assertIn( + 'qbit_prism_share_ack_seconds_count{result="accepted"} 2', + lines, + ) + for result in PRISM_SHARE_ACK_RESULTS: + self.assertIn( + f'qbit_prism_share_ack_seconds_count{{result="{result}"}} ' + f'{histograms[result]["count"]}', + lines, + ) + + +class AcceptedParentUnresolvedDepthGaugeTests(unittest.TestCase): + """The depth gauge must read exactly what the admission fence counts.""" + + def _coordinator(self): + server = support.coordinator() + server._ensure_job_cache_state() + return server + + def _landing_lines(self, server) -> list[str]: + return MetricsRenderer(server).landing_observability_metrics_lines() + + def _arm_landed_without_timestamp(self, server, block_hash: str) -> None: + """Reproduce the landed transitions that carry no monotonic stamp. + + Both the degraded preview-publication fallback and the durable + candidate replay reconstruct a transition from the dataclass default, + so ``landed`` is armed while ``landed_monotonic`` stays ``None``. + """ + with server._accepted_block_payout_preview_condition: + server._accepted_block_payout_previews[block_hash] = ( + AcceptedBlockPayoutTransition(block_height=11, landed=True) + ) + + def test_depth_gauge_counts_a_landed_transition_with_no_timestamp(self) -> None: + server = self._coordinator() + self._arm_landed_without_timestamp(server, "cd" * 32) + # The age list cannot see this transition, so a gauge derived from it + # would report zero while the fence already counts depth one. + self.assertEqual(server.accepted_parent_unresolved_ages_seconds(), []) + self.assertEqual(server._accepted_parent_unresolved_depth(), 1) + lines = self._landing_lines(server) + self.assertIn("qbit_prism_accepted_parent_unresolved_transitions 1", lines) + # No timestamp means no age to invent: the oldest gauge reports -1 + # rather than fabricating an age for the transition depth just counted. + self.assertIn( + "qbit_prism_accepted_parent_unresolved_oldest_seconds -1.000000", + lines, + ) + + def test_depth_gauge_matches_the_fence_across_mixed_transitions(self) -> None: + server = self._coordinator() + timestamped = "ce" * 32 + server._begin_accepted_block_payout_preview(timestamped, block_height=10) + server._mark_accepted_block_payout_landed(timestamped, block_height=10) + self._arm_landed_without_timestamp(server, "cf" * 32) + # A pending (not yet landed) transition stays out of both signals. + server._begin_accepted_block_payout_preview("d0" * 32, block_height=12) + self.assertEqual(len(server.accepted_parent_unresolved_ages_seconds()), 1) + self.assertEqual(server._accepted_parent_unresolved_depth(), 2) + lines = self._landing_lines(server) + self.assertIn("qbit_prism_accepted_parent_unresolved_transitions 2", lines) + self.assertNotIn( + "qbit_prism_accepted_parent_unresolved_oldest_seconds -1.000000", + lines, + ) + + def test_depth_gauge_is_zero_without_landed_transitions(self) -> None: + server = self._coordinator() + lines = self._landing_lines(server) + self.assertIn("qbit_prism_accepted_parent_unresolved_transitions 0", lines) + self.assertIn( + "qbit_prism_accepted_parent_unresolved_oldest_seconds -1.000000", + lines, + ) + + def test_exported_cap_is_the_runtime_configured_cap(self) -> None: + server = self._coordinator() + # The unconfigured runtime falls back to the shipped default. + self.assertEqual( + server._accepted_parent_unresolved_depth_cap(), + DEFAULT_ACCEPTED_PARENT_UNRESOLVED_DEPTH_MAX, + ) + self.assertIn( + "qbit_prism_accepted_parent_unresolved_depth_max " + f"{DEFAULT_ACCEPTED_PARENT_UNRESOLVED_DEPTH_MAX}", + self._landing_lines(server), + ) + # A non-default configured cap must reach the exposition too, so an + # alert can compare the depth against the limit actually enforced. + non_default = DEFAULT_ACCEPTED_PARENT_UNRESOLVED_DEPTH_MAX + 3 + server.accepted_parent_unresolved_depth_max = non_default + self.assertEqual(server._accepted_parent_unresolved_depth_cap(), non_default) + self.assertIn( + f"qbit_prism_accepted_parent_unresolved_depth_max {non_default}", + self._landing_lines(server), + ) + + +_SERIES_LINE = re.compile(r"^(?P[a-z_]+)(?:\{(?P[^}]*)\})? (?P\S+)$") + + +def _parse_series(line: str) -> tuple[str, dict[str, str], str]: + """Split one exposition sample into (family, labels, value).""" + match = _SERIES_LINE.match(line) + if match is None: + raise AssertionError(f"not a sample line: {line!r}") + labels: dict[str, str] = {} + if match.group("labels"): + for pair in match.group("labels").split(","): + key, _, quoted = pair.partition("=") + labels[key] = quoted.strip('"') + return match.group("family"), labels, match.group("value") + + +def _type_families(lines: list[str]) -> list[tuple[str, str]]: + return [ + (line.split()[2], line.split()[3]) + for line in lines + if line.startswith("# TYPE ") + ] + + +def _series_value(lines: list[str], series: str) -> int: + matches = [line for line in lines if line.startswith(series + " ")] + if len(matches) != 1: + raise AssertionError(f"{series!r} rendered {len(matches)} times") + return int(matches[0].split()[-1]) + + +class HeapAndComponentCardinalityTelemetryTests(unittest.TestCase): + """Issue #226 part 1: always-on heap telemetry and component gauges. + + The five behaviours the work package pins: the exact family set, the + mallinfo2-absent path rendering as unavailable rather than zero, every + component gauge reading its shipped owner, no unbounded label anywhere + in the component families, and a scrape that never walks the heap. + """ + + TIP = "aa" * 32 + + def _seeded_owners(self) -> tuple[object, dict[str, int], dict[str, int]]: + """Seed every component through its shipped owner; return the expectations. + + Each seed goes through the real owning structure -- the service + field, the queue, the LRU, the floor -- never a stub on the port, + so a renamed owner field breaks the read instead of reporting 0. + """ + server = support.coordinator() + server._ensure_job_cache_state() + server.jobs = {"job-1": object(), "job-2": object()} + bundles = server._ensure_job_bundle_service() + bundles._job_bundle_cache[("bundle-a",)] = object() + bundles._job_bundle_cache[("bundle-b",)] = object() + bundles._job_build_issued_at_ms[7] = 7_000 + bundles._bundle_preparation_flights[("flight",)] = object() + bundles._active_job_bundle_builds[("build",)] = object() + serialization = _ShareWindowSerialization( + key=("digest", 5, 0), + share_count=5, + share_snapshot_sha256="digest", + ) + serialization._spool_size = 4_096 + serialization._compact_tail_chunks = ("[" + "x" * 98 + "]", "y" * 20) + serialization._compact_tail_chars = 120 + bundles._share_window_serialization = serialization + pages = ( + _IncrementalShareWindowPage( + records=(object(), object()), + total_difficulty=0, + prism_json_records=({}, {}), + canonical_json_items=b'{"a":1},{"b":2}', + ), + _IncrementalShareWindowPage( + records=(object(),), + total_difficulty=0, + prism_json_records=({},), + canonical_json_items=b'{"c":3}', + ), + ) + payout = server._ensure_payout_state_service() + payout._incremental_payout_artifact_window = _IncrementalPayoutArtifactWindow( + window=IncrementalShareWindow( + anchor_job_issued_at_ms=1, + window_weight=1, + page_size=512, + pages=pages, + total_difficulty=0, + ), + shares_json=IncrementalShareJsonSequence(pages=pages, record_count=3), + share_snapshot_sha256="digest", + refreshed_monotonic=0.0, + full_rescan_monotonic=0.0, + full_rescan_attempt_monotonic=0.0, + ) + payout._payout_ledger_artifact = PayoutLedgerArtifact( + generation=1, + payout_state_generation=1, + network_difficulty=1, + accepted_share_count=3, + shares_json=IncrementalShareJsonSequence(pages=pages, record_count=3), + prior_balances=(), + prepared_monotonic=0.0, + ) + payout._accepted_block_payout_previews["b1" * 32] = AcceptedBlockPayoutTransition( + block_height=11, + landed=True, + ) + payout._invalidated_accepted_block_payout_previews["b2" * 32] = 5 + payout._payout_append_invalidation_stamps[1] = 100 + payout._payout_append_invalidation_stamps[2] = 200 + payout._payout_window_inflight_scan_anchors[1] = 100 + payout._payout_unfenced_append_inflight_stamps[1] = 100 + server.evicted_job_graveyard = { + "job-old": ( + SimpleNamespace( + job_id="job-old", + template={"previousblockhash": "bb" * 32}, + ), + 1, + time.monotonic(), + ), + } + candidates = server._ensure_block_candidate_service() + candidates._ensure_block_replay_state() + candidates._ensure_block_candidate_disposition_state() + candidates.candidate_queue.put_nowait(object()) + candidates._block_replay_candidate_queue.put_nowait(object()) + candidates._block_replay_candidate_queue.put_nowait(object()) + candidates._block_replay_inflight_hashes.add("c1" * 32) + candidates._block_quarantine_queue.put_nowait(object()) + candidates._block_quarantine_hashes.add("c2" * 32) + candidates._outstanding_block_candidate_hashes.update({"c3" * 32, "c4" * 32}) + candidates._tip_observed_accepted_block_hashes["c5" * 32] = 1.0 + candidates._counted_block_candidate_abandonments.add("c6" * 32) + candidates._accepted_block_preview_acceptance_monotonic["c7" * 32] = None + candidates._ancestor_redrive_records["c8" * 32] = object() + candidates._block_candidate_terminal_outcomes["c9" * 32] = True + candidates._block_candidate_terminal_outcomes["ca" * 32] = False + candidates._block_candidate_disposition_flights["cb" * 32] = object() + candidates._block_disposition_waiting_retries["cc" * 32] = object() + candidates._block_candidate_dequeued_hashes["cd" * 32] = 1 + server._accounted_accepted_block_hashes.add("ce" * 32) + reconciler = server._ensure_reorg_reconciler_service() + reconciler._reconcile_flights[self.TIP] = _ReconcileFlight() + reconciler._reorg_reconcile_trusted_memo[self.TIP] = 1.0 + reconciler._reorg_reconcile_trusted_memo["dd" * 32] = 2.0 + writer = server._ensure_share_writer_service() + writer.adopt_pending_share(SimpleNamespace(share_id="miner-a:s1")) + writer.adopt_pending_share(SimpleNamespace(share_id="miner-a:s2")) + daemon_client = _ServeBuilderClient(process=SimpleNamespace()) + daemon_client.note_uploaded_window("digest") + daemon_client.note_uploaded_window("digest-2") + server._ensure_bundle_compiler()._serve_builder = daemon_client + expected_entries = { + "payout_window_pages": 2, + "payout_window_records": 3, + "payout_ledger_artifact_shares": 3, + "accepted_block_payout_previews": 1, + "invalidated_accepted_block_payout_previews": 1, + "payout_append_invalidation_stamps": 2, + "payout_window_inflight_scan_anchors": 1, + "payout_unfenced_append_inflight_stamps": 1, + "share_window_serialization_shares": 5, + "job_contexts": 2, + "job_bundle_cache": 2, + "job_build_issued_stamps": 1, + "bundle_preparation_flights": 1, + "active_job_bundle_builds": 1, + "evicted_job_graveyard": 1, + # With no published tip the delivery owner classes every retained + # context as same-tip, so its same-tip index holds the entry too. + "evicted_same_tip_job_ids": 1, + "block_candidate_queue": 1, + "block_replay_queue": 2, + "block_replay_inflight_hashes": 1, + "block_quarantine_queue": 1, + "block_quarantine_hashes": 1, + "outstanding_block_candidate_hashes": 2, + "tip_observed_accepted_block_hashes": 1, + "counted_block_candidate_abandonments": 1, + "accepted_block_preview_stamps": 1, + "ancestor_redrive_records": 1, + "block_candidate_terminal_outcomes": 2, + "block_candidate_disposition_flights": 1, + "block_disposition_waiting_retries": 1, + "block_candidate_dequeued_hashes": 1, + "accounted_accepted_block_hashes": 1, + "reconcile_flights": 1, + "reconcile_trusted_memo": 2, + "pending_share_commit_floor": 2, + "daemon_window_mirror_records": 0, + "daemon_uploaded_windows": 2, + } + expected_bytes = { + "payout_window_canonical_json": 22, + "daemon_window_mirror_canonical_items": 0, + "share_window_serialization_spool": 4_096, + "share_window_serialization_compact_json": 120, + } + return server, expected_entries, expected_bytes + + def test_process_heap_telemetry_families_are_pinned(self) -> None: + """The exact (family, type) sequence renders, in order, fixed cardinality.""" + server = support.coordinator() + renderer = MetricsRenderer(server) + heap = renderer.process_heap_metrics_lines() + self.assertEqual(_type_families(heap), list(PROCESS_HEAP_METRIC_FAMILIES)) + rendered_generations: dict[str, list[str]] = {} + for line in heap: + if line.startswith("#"): + continue + family, labels, _value = _parse_series(line) + if family in PROCESS_HEAP_GENERATION_FAMILIES: + # The one permitted label, drawn from the closed set. + self.assertEqual(set(labels), {"generation"}) + self.assertIn(labels["generation"], PRISM_GC_GENERATIONS) + rendered_generations.setdefault(family, []).append(labels["generation"]) + else: + self.assertNotIn("{", line) + for family in PROCESS_HEAP_GENERATION_FAMILIES: + self.assertEqual(rendered_generations[family], list(PRISM_GC_GENERATIONS)) + components = renderer.component_cardinality_metrics_lines() + self.assertEqual( + _type_families(components), + list(COMPONENT_CARDINALITY_METRIC_FAMILIES), + ) + entry_kinds = [ + _parse_series(line)[1]["component"] + for line in components + if line.startswith("qbit_prism_component_entries{") + ] + byte_kinds = [ + _parse_series(line)[1]["component"] + for line in components + if line.startswith("qbit_prism_component_bytes{") + ] + self.assertEqual(entry_kinds, list(PRISM_COMPONENT_ENTRY_KINDS)) + self.assertEqual(byte_kinds, list(PRISM_COMPONENT_BYTE_KINDS)) + self.assertEqual(len(set(PRISM_COMPONENT_ENTRY_KINDS)), len(PRISM_COMPONENT_ENTRY_KINDS)) + # Every component the issue names has a gauge. + for named in ( + "payout_window_pages", + "share_window_serialization_shares", + "job_contexts", + "job_bundle_cache", + "evicted_job_graveyard", + "block_candidate_queue", + "block_replay_queue", + "reconcile_flights", + "pending_share_commit_floor", + "daemon_window_mirror_records", + ): + self.assertIn(named, PRISM_COMPONENT_ENTRY_KINDS) + # The full document carries both blocks, in this order, at its tail. + type_lines = [ + line for line in server.metrics_payload().splitlines() + if line.startswith("# TYPE ") + ] + expected_tail = [ + f"# TYPE {family} {kind}" + for family, kind in ( + PROCESS_HEAP_METRIC_FAMILIES + COMPONENT_CARDINALITY_METRIC_FAMILIES + ) + ] + self.assertEqual(type_lines[-len(expected_tail):], expected_tail) + + def test_mallinfo_absence_renders_unavailable_not_zero(self) -> None: + """A failed ctypes lookup renders availability 0 and -1 bytes, never 0.""" + server = support.coordinator() + resolver = mock.Mock(side_effect=AttributeError("mallinfo2: symbol not found")) + telemetry = ProcessHeapTelemetry(mallinfo_resolver=resolver) + renderer = MetricsRenderer(server, process_telemetry=telemetry) + first = renderer.process_heap_metrics_lines() + second = renderer.process_heap_metrics_lines() + self.assertEqual(_series_value(first, "qbit_prism_process_malloc_info_available"), 0) + for gauge in ("arena", "in_use", "free", "mmapped"): + self.assertEqual( + _series_value(first, f"qbit_prism_process_malloc_{gauge}_bytes"), + -1, + ) + # The interpreter readings are real even when glibc's are not. + self.assertGreater(_series_value(first, "qbit_prism_process_allocated_blocks"), 0) + self.assertGreaterEqual(_series_value(first, "qbit_prism_process_threads"), 1) + # Resolved once and remembered: no per-scrape lookup, no per-scrape log. + self.assertEqual(resolver.call_count, 1) + self.assertEqual(_type_families(second), _type_families(first)) + self.assertEqual(_series_value(second, "qbit_prism_process_malloc_info_available"), 0) + # The family set is identical to the available case, so a dashboard + # built on either platform reads the same series. + self.assertEqual(_type_families(first), list(PROCESS_HEAP_METRIC_FAMILIES)) + # A symbol that binds but fails at call time degrades the same way + # and is not retried on the next scrape. + broken = mock.Mock(side_effect=OSError("mallinfo2 call failed")) + renderer = MetricsRenderer( + server, + process_telemetry=ProcessHeapTelemetry(mallinfo_resolver=lambda: broken), + ) + lines = renderer.process_heap_metrics_lines() + renderer.process_heap_metrics_lines() + self.assertEqual(_series_value(lines, "qbit_prism_process_malloc_info_available"), 0) + self.assertEqual(_series_value(lines, "qbit_prism_process_malloc_arena_bytes"), -1) + self.assertEqual(broken.call_count, 1) + # Where the symbol binds, the struct's fields render; the fake makes + # this branch platform-independent too. + def fake_mallinfo2() -> MallInfo2: + info = MallInfo2() + info.arena = 9_000 + info.uordblks = 6_000 + info.fordblks = 3_000 + info.hblkhd = 500 + return info + + lines = MetricsRenderer( + server, + process_telemetry=ProcessHeapTelemetry(mallinfo_resolver=lambda: fake_mallinfo2), + ).process_heap_metrics_lines() + self.assertEqual(_series_value(lines, "qbit_prism_process_malloc_info_available"), 1) + self.assertEqual(_series_value(lines, "qbit_prism_process_malloc_arena_bytes"), 9_000) + self.assertEqual(_series_value(lines, "qbit_prism_process_malloc_in_use_bytes"), 6_000) + self.assertEqual(_series_value(lines, "qbit_prism_process_malloc_free_bytes"), 3_000) + self.assertEqual(_series_value(lines, "qbit_prism_process_malloc_mmapped_bytes"), 500) + # The operator switch renders unavailable without probing at all. + resolver = mock.Mock() + lines = MetricsRenderer( + server, + process_telemetry=ProcessHeapTelemetry( + malloc_enabled=False, + mallinfo_resolver=resolver, + ), + ).process_heap_metrics_lines() + self.assertEqual(_series_value(lines, "qbit_prism_process_malloc_info_available"), 0) + self.assertEqual(_series_value(lines, "qbit_prism_process_malloc_in_use_bytes"), -1) + resolver.assert_not_called() + + def test_component_cardinality_gauges_render_from_the_shipped_owner(self) -> None: + """Each gauge reads the real owning structure, not a stub.""" + server, expected_entries, expected_bytes = self._seeded_owners() + self.assertEqual(set(expected_entries), set(PRISM_COMPONENT_ENTRY_KINDS)) + self.assertEqual(set(expected_bytes), set(PRISM_COMPONENT_BYTE_KINDS)) + lines = MetricsRenderer(server).component_cardinality_metrics_lines() + for kind, value in expected_entries.items(): + self.assertIn( + f'qbit_prism_component_entries{{component="{kind}"}} {value}', + lines, + ) + for kind, value in expected_bytes.items(): + self.assertIn( + f'qbit_prism_component_bytes{{component="{kind}"}} {value}', + lines, + ) + self.assertEqual(lines, reference_component_cardinality_metrics_lines(server)) + # A mirror-backed window reports the daemon's bytes and records and + # no in-process pages: the same slot, the other shipped window type. + items = b'{"a":1},{"b":2}' + digest = hashlib.sha256(b"[" + items + b"]").hexdigest() + mirror = DaemonShareWindowMirror.from_full_items( + anchor_job_issued_at_ms=1, + window_weight=1, + page_size=512, + record_count=2, + canonical_items=items, + share_snapshot_sha256=digest, + ) + payout = server._ensure_payout_state_service() + payout._incremental_payout_artifact_window = _IncrementalPayoutArtifactWindow( + window=mirror, + shares_json=mirror.json_records(), + share_snapshot_sha256=digest, + refreshed_monotonic=0.0, + full_rescan_monotonic=0.0, + full_rescan_attempt_monotonic=0.0, + ) + lines = MetricsRenderer(server).component_cardinality_metrics_lines() + # Accounted exactly once. The mirror *is* the cached window here, so + # its size appears only under daemon_window_mirror_*; reporting it + # under payout_window_* as well would make an operator summing + # qbit_prism_component_bytes for RSS attribution count it twice. + self.assertIn('qbit_prism_component_entries{component="payout_window_pages"} 0', lines) + self.assertIn('qbit_prism_component_entries{component="payout_window_records"} 0', lines) + self.assertIn( + 'qbit_prism_component_entries{component="daemon_window_mirror_records"} 2', + lines, + ) + self.assertIn( + 'qbit_prism_component_bytes{component="payout_window_canonical_json"} 0', + lines, + ) + self.assertIn( + f'qbit_prism_component_bytes{{component="daemon_window_mirror_canonical_items"}} {len(items)}', + lines, + ) + # The same property stated as a sum, so a future backing that reports + # under both slots fails here rather than in an operator's dashboard. + self.assertEqual( + sum( + int(line.rsplit(" ", 1)[1]) + for line in lines + if line.startswith("qbit_prism_component_bytes{") + and ( + 'component="payout_window_canonical_json"' in line + or 'component="daemon_window_mirror_canonical_items"' in line + ) + ), + len(items), + ) + # An empty window slot and an absent daemon client are true zeros. + payout._incremental_payout_artifact_window = None + server._ensure_bundle_compiler()._serve_builder = None + lines = MetricsRenderer(server).component_cardinality_metrics_lines() + self.assertIn('qbit_prism_component_entries{component="payout_window_records"} 0', lines) + self.assertIn('qbit_prism_component_entries{component="daemon_uploaded_windows"} 0', lines) + # The reads are direct: renaming an owner field breaks the scrape + # loudly instead of reporting a silent zero. + reconciler = server._ensure_reorg_reconciler_service() + flights = reconciler._reconcile_flights + del reconciler._reconcile_flights + try: + with self.assertRaises(AttributeError): + MetricsRenderer(server).component_cardinality_metrics_lines() + finally: + reconciler._reconcile_flights = flights + candidates = server._ensure_block_candidate_service() + outstanding = candidates._outstanding_block_candidate_hashes + del candidates._outstanding_block_candidate_hashes + try: + with self.assertRaises(AttributeError): + MetricsRenderer(server).component_cardinality_metrics_lines() + finally: + candidates._outstanding_block_candidate_hashes = outstanding + + def test_component_cardinality_gauges_carry_no_unbounded_label(self) -> None: + """No series carries a label that varies per job, tip, hash, or generation.""" + server, _entries, _bytes = self._seeded_owners() + + def series_set(lines: list[str]) -> list[tuple[str, str]]: + observed: list[tuple[str, str]] = [] + for line in lines: + if line.startswith("#"): + continue + family, labels, _value = _parse_series(line) + self.assertEqual(set(labels), {"component"}, line) + closed = ( + PRISM_COMPONENT_ENTRY_KINDS + if family == "qbit_prism_component_entries" + else PRISM_COMPONENT_BYTE_KINDS + ) + self.assertIn(labels["component"], closed, line) + observed.append((family, labels["component"])) + return observed + + before = MetricsRenderer(server).component_cardinality_metrics_lines() + baseline = series_set(before) + self.assertEqual(len(baseline), len(set(baseline))) + self.assertEqual( + len(baseline), + len(PRISM_COMPONENT_ENTRY_KINDS) + len(PRISM_COMPONENT_BYTE_KINDS), + ) + # Grow every unbounded population the issue names -- more jobs, more + # tips, more candidate hashes, more generations, more workers -- and + # the series set must not move by one. + seeded_identifiers = [] + for index in range(5): + job_id = f"job-growth-{index}" + tip_hash = f"{index:02x}" * 32 + block_hash = f"e{index}" * 32 + seeded_identifiers.extend((job_id, tip_hash, block_hash, f"worker-{index}")) + server.jobs[job_id] = object() + reconciler = server._ensure_reorg_reconciler_service() + reconciler._reorg_reconcile_trusted_memo[tip_hash] = float(index) + reconciler._reconcile_flights[tip_hash] = _ReconcileFlight() + candidates = server._ensure_block_candidate_service() + candidates._outstanding_block_candidate_hashes.add(block_hash) + candidates._block_candidate_terminal_outcomes[block_hash] = True + server._ensure_payout_state_service()._payout_append_invalidation_stamps[ + 10 + index + ] = index + server._ensure_share_writer_service().adopt_pending_share( + SimpleNamespace(share_id=f"worker-{index}:{block_hash}") + ) + after = MetricsRenderer(server).component_cardinality_metrics_lines() + self.assertEqual(series_set(after), baseline) + self.assertEqual(_type_families(after), _type_families(before)) + rendered = "\n".join(after) + for identifier in seeded_identifiers: + self.assertNotIn(identifier, rendered) + # The values moved; only the label set stayed put. + self.assertGreater( + _series_value(after, 'qbit_prism_component_entries{component="job_contexts"}'), + _series_value(before, 'qbit_prism_component_entries{component="job_contexts"}'), + ) + + def test_heap_telemetry_scrape_does_not_walk_the_heap(self) -> None: + """The collector never calls gc.get_objects or starts tracemalloc.""" + server, _entries, _bytes = self._seeded_owners() + self.assertFalse(tracemalloc.is_tracing()) + with ( + mock.patch.object( + gc, + "get_objects", + side_effect=AssertionError("gc.get_objects walked the heap on a scrape"), + ) as get_objects, + mock.patch.object( + gc, + "collect", + side_effect=AssertionError("gc.collect ran on a scrape"), + ) as collect, + mock.patch.object( + tracemalloc, + "start", + side_effect=AssertionError("tracemalloc started on a scrape"), + ) as start, + mock.patch.object( + tracemalloc, + "take_snapshot", + side_effect=AssertionError("tracemalloc snapshot on a scrape"), + ) as take_snapshot, + ): + renderer = MetricsRenderer(server) + heap = renderer.process_heap_metrics_lines() + components = renderer.component_cardinality_metrics_lines() + document = renderer.render() + self.assertEqual(_type_families(heap), list(PROCESS_HEAP_METRIC_FAMILIES)) + self.assertEqual( + _type_families(components), + list(COMPONENT_CARDINALITY_METRIC_FAMILIES), + ) + self.assertIn("qbit_prism_process_allocated_blocks ", document) + get_objects.assert_not_called() + collect.assert_not_called() + start.assert_not_called() + take_snapshot.assert_not_called() + self.assertFalse(tracemalloc.is_tracing()) + + +# Issue #227: the closed family sets of the monitor-lateness signals and the +# GC pause durations; tests pin the renderer to both. +LEASE_MONITOR_STALL_METRIC_FAMILIES = ( + ("qbit_prism_lease_heartbeat_monitor_wake_lateness_seconds", "histogram"), + ("qbit_prism_lease_heartbeat_monitor_late_wakes_total", "counter"), + ("qbit_prism_lease_heartbeat_monitor_wake_delay_window_max_seconds", "gauge"), + ("qbit_prism_lease_heartbeat_monitor_wake_delay_record_age_seconds", "gauge"), + ("qbit_prism_lease_heartbeat_monitor_exit_guarantee_breaches_total", "counter"), + ("qbit_prism_lease_heartbeat_monitor_worst_exit_guarantee_overrun_seconds", "gauge"), + ("qbit_prism_lease_heartbeat_stall_probe_samples_total", "counter"), + ("qbit_prism_lease_heartbeat_stall_probe_suppressed_total", "counter"), + ("qbit_prism_lease_heartbeat_monitor_breach_warnings_suppressed_total", "counter"), +) +GC_PAUSE_METRIC_FAMILIES = ( + ("qbit_prism_process_gc_pause_seconds", "histogram"), + ("qbit_prism_process_gc_last_pause_seconds", "gauge"), + ("qbit_prism_process_gc_max_pause_seconds", "gauge"), +) + + +class LeaseMonitorStallTelemetryTests(unittest.TestCase): + """Issue #227: the re-armable monitor-lateness signals and GC pauses.""" + + @staticmethod + def _series_keys(lines: list[str]) -> list[tuple[str, tuple[tuple[str, str], ...]]]: + return [ + (family, tuple(sorted(labels.items()))) + for family, labels, _value in ( + _parse_series(line) for line in lines if not line.startswith("#") + ) + ] + + def test_monitor_wake_delay_histogram_and_threshold_counters_are_pinned( + self, + ) -> None: + """Family set, bucket boundaries and the slack-fraction counters. + + All fixed cardinality: the series set after two thousand more + observations is identical to the series set before them. The + lifetime ``..._monitor_wake_delay_seconds`` gauge still renders. + """ + server = support.coordinator() + service = server._ensure_lease_heartbeat_service() + slack = service.policy().scheduler_slack_seconds + # The renderer's snapshot reads the service clock (time.monotonic + # here), so the seeded wakes are stamped on that clock too; the + # rolling window and the record age are then read at "now". + now = time.monotonic() - 10.0 + for delay in [0.0] * 100 + [0.3] * 40 + [0.45] * 30 + [0.6] * 20: + now += 0.05 + service.monitor_wakes.observe(delay, slack_seconds=slack, now=now) + service.monitor_wakes.observe(0.648, slack_seconds=slack, now=now + 0.05) + # The lifetime gauge is the monitor loop's own high-water mark. + service.monitor_wake_delay_seconds = 0.648 + renderer = MetricsRenderer(server) + lines = renderer.lease_heartbeat_metrics_lines() + + families = _type_families(lines) + self.assertIn( + ("qbit_prism_lease_heartbeat_monitor_wake_delay_seconds", "gauge"), families + ) + self.assertIn("qbit_prism_lease_heartbeat_monitor_wake_delay_seconds 0.648000", lines) + self.assertEqual( + families[-len(LEASE_MONITOR_STALL_METRIC_FAMILIES):], + list(LEASE_MONITOR_STALL_METRIC_FAMILIES), + ) + # Bucket boundaries are the closed set plus +Inf, cumulative. + histogram = "qbit_prism_lease_heartbeat_monitor_wake_lateness_seconds" + bucket_lines = [line for line in lines if line.startswith(histogram + "_bucket{")] + self.assertEqual( + [_parse_series(line)[1] for line in bucket_lines], + [{"le": f"{bucket:g}"} for bucket in LEASE_MONITOR_WAKE_DELAY_BUCKETS] + + [{"le": "+Inf"}], + ) + counts = [int(_parse_series(line)[2]) for line in bucket_lines] + self.assertEqual(counts, sorted(counts)) + self.assertEqual(counts[-1], 191) + self.assertEqual(_series_value(lines, histogram + "_count"), 191) + by_bound = dict(zip(LEASE_MONITOR_WAKE_DELAY_BUCKETS, counts)) + self.assertEqual(by_bound[0.001], 100) + self.assertEqual(by_bound[0.4], 140) + self.assertEqual(by_bound[0.5], 170) + self.assertEqual(by_bound[0.75], 191) + # The 0.5 / 0.8 / 1.0 x slack counters, exactly that label set. + counter = "qbit_prism_lease_heartbeat_monitor_late_wakes_total" + fraction_lines = [line for line in lines if line.startswith(counter + "{")] + self.assertEqual( + [_parse_series(line)[1]["slack_fraction"] for line in fraction_lines], + list(LEASE_MONITOR_LATE_WAKE_SLACK_FRACTIONS), + ) + self.assertEqual(_series_value(lines, counter + '{slack_fraction="0.5"}'), 91) + self.assertEqual(_series_value(lines, counter + '{slack_fraction="0.8"}'), 51) + self.assertEqual(_series_value(lines, counter + '{slack_fraction="1.0"}'), 21) + # Rolling maximum, record age, the breach counters and the probe. + self.assertIn( + "qbit_prism_lease_heartbeat_monitor_wake_delay_window_max_seconds 0.648000", + lines, + ) + self.assertTrue( + any( + line.startswith( + "qbit_prism_lease_heartbeat_monitor_wake_delay_record_age_seconds " + ) + for line in lines + ) + ) + self.assertEqual( + _series_value(lines, "qbit_prism_lease_heartbeat_monitor_exit_guarantee_breaches_total"), + 0, + ) + self.assertEqual( + _series_value(lines, "qbit_prism_lease_heartbeat_stall_probe_samples_total"), 0 + ) + # Suppressed breach warnings are exported alongside the probe's. + service.suppressed_slack_breach_warnings = 4 + self.assertEqual( + _series_value( + renderer.lease_heartbeat_metrics_lines(), + "qbit_prism_lease_heartbeat_monitor_breach_warnings_suppressed_total", + ), + 4, + ) + # The policy term alerts compare against, and the new phase. + self.assertIn( + 'qbit_prism_lease_heartbeat_policy_seconds{term="max_guaranteed_monitor_lateness"} 0.550000', + lines, + ) + self.assertEqual( + [ + _parse_series(line)[1]["phase"] + for line in lines + if line.startswith("qbit_prism_lease_heartbeat_phase_seconds{") + ], + list(LEASE_HEARTBEAT_PHASES), + ) + self.assertIn("guard_client_resume", LEASE_HEARTBEAT_PHASES) + # Fixed cardinality: traffic moves values, never the series set. + before = self._series_keys(lines) + for _ in range(2_000): + now += 0.05 + service.monitor_wakes.observe(1.7, slack_seconds=slack, now=now) + self.assertEqual(self._series_keys(renderer.lease_heartbeat_metrics_lines()), before) + + def test_gc_pause_families_are_pinned_to_the_closed_generation_set(self) -> None: + server = support.coordinator() + service = server._ensure_lease_heartbeat_service() + service.gc_pauses.record("2", 0.3) + service.gc_pauses.record("0", 0.0004) + service.gc_pauses.record("7", 9.0) # not a CPython generation: dropped + renderer = MetricsRenderer(server) + lines = renderer.gc_pause_metrics_lines() + self.assertEqual(_type_families(lines), list(GC_PAUSE_METRIC_FAMILIES)) + rendered: dict[str, list[str]] = {} + for line in lines: + if line.startswith("#"): + continue + family, labels, _value = _parse_series(line) + self.assertEqual(set(labels) - {"le"}, {"generation"}) + self.assertIn(labels["generation"], PRISM_GC_GENERATIONS) + if family == "qbit_prism_process_gc_pause_seconds_bucket": + rendered.setdefault(labels["generation"], []).append(labels["le"]) + for generation in PRISM_GC_GENERATIONS: + self.assertEqual( + rendered[generation], + [f"{bucket:g}" for bucket in PRISM_GC_PAUSE_SECONDS_BUCKETS] + ["+Inf"], + ) + self.assertEqual( + _series_value(lines, 'qbit_prism_process_gc_pause_seconds_count{generation="2"}'), + 1, + ) + self.assertEqual( + _series_value(lines, 'qbit_prism_process_gc_pause_seconds_count{generation="1"}'), + 0, + ) + self.assertIn('qbit_prism_process_gc_last_pause_seconds{generation="2"} 0.300000', lines) + self.assertIn('qbit_prism_process_gc_max_pause_seconds{generation="0"} 0.000400', lines) + # In the full document the block sits immediately before the issue + # #226 heap block it complements. + type_lines = [ + line for line in server.metrics_payload().splitlines() + if line.startswith("# TYPE ") + ] + index = type_lines.index("# TYPE qbit_prism_process_gc_pause_seconds histogram") + self.assertEqual( + type_lines[index : index + len(GC_PAUSE_METRIC_FAMILIES)], + [f"# TYPE {family} {kind}" for family, kind in GC_PAUSE_METRIC_FAMILIES], + ) + self.assertEqual( + type_lines[index + len(GC_PAUSE_METRIC_FAMILIES)], + "# TYPE qbit_prism_process_allocated_blocks gauge", + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_prism_observability.py b/tests/test_prism_observability.py new file mode 100644 index 000000000..8de991ce6 --- /dev/null +++ b/tests/test_prism_observability.py @@ -0,0 +1,1119 @@ +#!/usr/bin/env python3 +"""Direct ownership and safety tests for PRISM health observability.""" + +from __future__ import annotations + +from dataclasses import replace +import threading +import time +import unittest +from types import SimpleNamespace + +from lab.prism.observability import ( + HEALTH_SCHEMA, + METRICS_STALE_WARNING, + METRICS_STATES, + METRICS_STATE_FRESH, + METRICS_STATE_HEADER, + METRICS_STATE_STALE, + METRICS_STATE_UNAVAILABLE, + MiningDeliveryInputs, + ObservabilityService, +) +from lab.prism.prism_coordinator import ( + ClientState, + PrismCoordinator, + WorkerIdentity, +) + + +def healthy_inputs() -> MiningDeliveryInputs: + return MiningDeliveryInputs( + active_connections=0, + connection_capacity=64, + peak_active_connections=0, + subscribed_connections=0, + authorized_connections=0, + pending_initial_jobs=0, + pending_initial_job_capacity=32, + oldest_pending_initial_job_age_seconds=0.0, + oldest_genuinely_pending_initial_job_age_seconds=0.0, + clients_with_current_tip_jobs=0, + clients_with_semantically_current_work=0, + clients_with_no_active_job=0, + last_initial_job_delivery_monotonic=None, + initial_job_timeout_seconds=5.0, + initial_job_queue_rejections=0, + initial_job_timeout_disconnects=0, + initial_job_cancelled_tasks=0, + initial_job_coalesced_tasks=0, + initial_job_queue_capacity_reclaimed=0, + handler_threads=0, + delivery_executor_queue_depth=0, + delivery_executor_active_workers=0, + started_monotonic=0.0, + startup_grace_seconds=0.0, + stale_unknown_rejections=0, + submitted_shares=0, + job_preparation_pending=False, + current_observed_tip=None, + prepared_bundle_current=False, + prepared_bundle_tip=None, + prepared_bundle_template_generation=None, + prepared_bundle_payout_generation=None, + ) + + +def incident_inputs() -> MiningDeliveryInputs: + """The 2026-08-31 fleet: 4 of 33 miners on the exact observed tip while + all 33 held semantically current work and nothing was genuinely pending.""" + + return replace( + healthy_inputs(), + active_connections=33, + subscribed_connections=33, + authorized_connections=33, + clients_with_current_tip_jobs=4, + clients_with_semantically_current_work=33, + last_initial_job_delivery_monotonic=99.5, + ) + + +class FakeObservabilityPort: + def __init__(self) -> None: + self.now = 100.0 + self.inputs = healthy_inputs() + self.progress: dict[str, object] = { + "ok": True, + "reason": None, + "reasons": [], + } + self.raise_on_stats = False + self.stats_calls = 0 + self.metrics_payload = "qbit_prism_fixture 1\n" + self.metrics_error: Exception | None = None + self.metrics_render_count = 0 + self.startup_phases: list[str] = [] + self.log_messages: list[str] = [] + self.exception_count = 0 + + def monotonic(self) -> float: + return self.now + + def mining_delivery_inputs(self, now: float) -> MiningDeliveryInputs: + self.assert_current_time(now) + return self.inputs + + def assert_current_time(self, now: float) -> None: + if now != self.now: + raise AssertionError(f"unexpected monotonic time: {now}") + + def accepted_share_stats(self) -> tuple[int, int]: + self.stats_calls += 1 + if self.raise_on_stats: + raise RuntimeError("stats unavailable") + return 3, 2 + + def ledger_backend(self) -> str: + return "memory" + + def block_counts(self) -> tuple[int, int]: + return 1, 2 + + def progress_health(self) -> dict[str, object]: + return dict(self.progress) + + def health_refresh_seconds(self) -> float: + return 1.0 + + def render_metrics_payload(self) -> str: + self.metrics_render_count += 1 + if self.metrics_error is not None: + raise self.metrics_error + return self.metrics_payload + + def metrics_refresh_seconds(self) -> float: + return 1.0 + + def record_startup_phase(self, phase: str) -> None: + self.startup_phases.append(phase) + + def stop_requested(self) -> bool: + return False + + def wait_for_stop(self, timeout: float) -> bool: + if timeout != 1.0: + raise AssertionError(f"unexpected wait: {timeout}") + return True + + def log(self, message: str) -> None: + self.log_messages.append(message) + + def log_exception(self) -> None: + self.exception_count += 1 + + +class ObservabilityServiceTests(unittest.TestCase): + def test_cached_base_health_uses_fresh_progress_overlay(self) -> None: + port = FakeObservabilityPort() + service = ObservabilityService(port) + service.refresh_health_snapshot() + self.assertNotIn("reason", service.state().health_snapshot or {}) + + port.progress = { + "ok": False, + "reason": "tip_poll_stale", + "reasons": ["tip_poll_stale"], + } + status, payload = service.cached_health_payload() + + self.assertEqual(status, 503) + self.assertFalse(payload["ok"]) + self.assertEqual(payload["reason"], "tip_poll_stale") + + def test_refresh_stamps_health_snapshot_warm_startup_phase(self) -> None: + port = FakeObservabilityPort() + service = ObservabilityService(port) + + service.refresh_health_snapshot() + + self.assertEqual(port.startup_phases, ["health_snapshot_warm"]) + + def test_fresh_progress_cannot_mask_cached_base_failure(self) -> None: + port = FakeObservabilityPort() + port.inputs = replace( + port.inputs, + subscribed_connections=1, + authorized_connections=1, + oldest_genuinely_pending_initial_job_age_seconds=6.0, + ) + service = ObservabilityService(port) + service.refresh_health_snapshot() + + port.inputs = healthy_inputs() + status, payload = service.cached_health_payload() + + self.assertEqual(status, 503) + self.assertFalse(payload["ok"]) + self.assertIn("initial-delivery-stalled", payload["unhealthy_reasons"]) + + def test_running_loop_without_snapshot_reports_starting_state(self) -> None: + # Upstream #120: while the background warm-up runs, the endpoint + # must fail closed with an explicit starting state instead of + # collecting ledger aggregates inline on the handler thread. + port = FakeObservabilityPort() + service = ObservabilityService(port) + self.assertTrue(service.begin_refresh_loop()) + + status, payload = service.cached_health_payload() + + self.assertEqual(status, 503) + self.assertFalse(payload["ok"]) + self.assertEqual(payload["state"], "starting") + self.assertEqual(payload["schema"], HEALTH_SCHEMA) + self.assertEqual(port.stats_calls, 0) + + def test_stale_snapshot_fails_closed_with_current_progress(self) -> None: + port = FakeObservabilityPort() + service = ObservabilityService(port) + service.refresh_health_snapshot() + port.now += 16.0 + port.progress = { + "ok": False, + "reason": "refresh_pending_too_long", + "reasons": ["refresh_pending_too_long"], + } + + status, payload = service.cached_health_payload() + + self.assertEqual(status, 503) + self.assertFalse(payload["ok"]) + self.assertEqual(payload["schema"], HEALTH_SCHEMA) + self.assertEqual(payload["error"], "health snapshot is stale") + self.assertEqual(payload["reason"], "refresh_pending_too_long") + + def test_delivery_timers_recover_after_sustained_overload(self) -> None: + port = FakeObservabilityPort() + port.inputs = replace( + port.inputs, + active_connections=1, + subscribed_connections=1, + authorized_connections=1, + pending_initial_jobs=32, + ) + service = ObservabilityService(port) + + self.assertTrue(service.mining_delivery_snapshot()["mining_ready"]) + port.now += 6.0 + stalled = service.mining_delivery_snapshot() + self.assertFalse(stalled["mining_ready"]) + self.assertEqual( + stalled["unhealthy_reasons"], + [ + "initial-delivery-stalled", + "pending-initial-jobs-saturated", + ], + ) + + port.inputs = replace( + port.inputs, + pending_initial_jobs=0, + clients_with_current_tip_jobs=1, + clients_with_semantically_current_work=1, + ) + recovered = service.mining_delivery_snapshot() + self.assertTrue(recovered["mining_ready"]) + state = service.state() + self.assertIsNone(state.mining_overload_started_monotonic) + self.assertIsNone(state.mining_delivery_failure_started_monotonic) + self.assertIsNone(state.mining_semantic_work_gap_started_monotonic) + + def test_semantic_gauges_report_count_and_six_decimal_ratio(self) -> None: + port = FakeObservabilityPort() + port.inputs = replace( + port.inputs, + active_connections=3, + subscribed_connections=3, + authorized_connections=3, + clients_with_current_tip_jobs=3, + clients_with_semantically_current_work=1, + ) + service = ObservabilityService(port) + + snapshot = service.mining_delivery_snapshot() + + self.assertEqual(snapshot["clients_with_semantically_current_work"], 1) + self.assertEqual(snapshot["semantic_current_work_ratio"], round(1 / 3, 6)) + + def test_semantic_ratio_is_one_with_zero_authorized_clients(self) -> None: + port = FakeObservabilityPort() + service = ObservabilityService(port) + + snapshot = service.mining_delivery_snapshot() + + self.assertEqual(snapshot["clients_with_semantically_current_work"], 0) + self.assertEqual(snapshot["semantic_current_work_ratio"], 1.0) + self.assertTrue(snapshot["mining_ready"]) + + def test_delivery_timer_observations_are_applied_in_capture_order(self) -> None: + first_captured = threading.Event() + release_first = threading.Event() + second_captured = threading.Event() + + class OrderedPort(FakeObservabilityPort): + def mining_delivery_inputs(self, now: float) -> MiningDeliveryInputs: + if now == 100.0: + first_captured.set() + if not release_first.wait(1.0): + raise AssertionError("first health capture was not released") + return replace( + self.inputs, + authorized_connections=1, + clients_with_current_tip_jobs=0, + ) + second_captured.set() + return replace( + self.inputs, + authorized_connections=1, + clients_with_current_tip_jobs=1, + ) + + port = OrderedPort() + service = ObservabilityService(port) + snapshots: list[dict[str, object]] = [] + first = threading.Thread( + target=lambda: snapshots.append( + service.mining_delivery_snapshot(now=100.0) + ) + ) + second = threading.Thread( + target=lambda: snapshots.append( + service.mining_delivery_snapshot(now=101.0) + ) + ) + first.start() + self.assertTrue(first_captured.wait(1.0)) + second.start() + self.assertFalse(second_captured.wait(0.05)) + release_first.set() + for thread in (first, second): + thread.join(1.0) + self.assertFalse(thread.is_alive()) + + self.assertEqual(len(snapshots), 2) + self.assertTrue(second_captured.is_set()) + state = service.state() + self.assertIsNone(state.mining_overload_started_monotonic) + self.assertIsNone(state.mining_delivery_failure_started_monotonic) + + def test_refresh_loop_counts_failure_and_keeps_one_shot_latch(self) -> None: + port = FakeObservabilityPort() + port.raise_on_stats = True + service = ObservabilityService(port) + self.assertTrue(service.begin_refresh_loop()) + self.assertFalse(service.begin_refresh_loop()) + + service.health_snapshot_loop() + + state = service.state() + # Upstream #120: the running flag is a one-shot latch. Loop exit must + # not re-open the legacy inline collection path for late requests. + self.assertTrue(state.health_refresh_loop_running) + self.assertEqual(state.health_snapshot_refresh_failure_count, 1) + self.assertEqual( + port.log_messages, + ["prism coordinator: health snapshot refresh failed"], + ) + self.assertEqual(port.exception_count, 1) + + def test_metrics_snapshot_preserves_renderer_bytes_and_adds_diagnostics(self) -> None: + port = FakeObservabilityPort() + service = ObservabilityService(port) + + rendered = service.refresh_metrics_snapshot() + status, cached = service.cached_metrics_payload() + + self.assertEqual(rendered, port.metrics_payload) + self.assertEqual(status, 200) + self.assertTrue(cached.startswith(port.metrics_payload)) + self.assertEqual(port.metrics_render_count, 1) + self.assertIn("qbit_prism_metrics_snapshot_available 1\n", cached) + self.assertIn("qbit_prism_metrics_snapshot_stale 0\n", cached) + self.assertIn("qbit_prism_metrics_snapshot_generation 1\n", cached) + + def test_metrics_failure_preserves_prior_complete_generation(self) -> None: + port = FakeObservabilityPort() + service = ObservabilityService(port) + service.refresh_metrics_snapshot() + port.metrics_payload = "partial" + + with self.assertRaises(ValueError): + service.refresh_metrics_snapshot() + + state = service.metrics_state() + self.assertEqual(state.metrics_snapshot, "qbit_prism_fixture 1\n") + self.assertEqual(state.metrics_collection_generation, 1) + self.assertEqual(state.metrics_collection_failure_count, 1) + self.assertEqual(state.metrics_failure_invalid_payload_count, 1) + status, cached = service.cached_metrics_payload() + self.assertEqual(status, 200) + self.assertTrue(cached.startswith("qbit_prism_fixture 1\n")) + + port.now += 16.0 + response = service.cached_metrics_payload() + + # The failed refresh left the last complete generation in place and + # #184 keeps serving it. Refusing here discarded that payload along + # with the failure counters that explain it, because Prometheus drops + # the body of a non-200 response. + self.assertEqual(response.status, 200) + self.assertEqual(response.state, METRICS_STATE_STALE) + self.assertTrue(response.body.startswith("qbit_prism_fixture 1\n")) + self.assertIn("qbit_prism_metrics_snapshot_stale 1\n", response.body) + self.assertIn( + 'qbit_prism_metrics_collection_failures_total{class="invalid_payload"} 1', + response.body, + ) + + def test_metrics_without_snapshot_fails_closed_without_collecting(self) -> None: + port = FakeObservabilityPort() + service = ObservabilityService(port) + + response = service.cached_metrics_payload() + + # Warm-up is the one case that still refuses: there is no complete + # payload to serve, so the body is diagnostics only and a scraper must + # not store it as this process's metrics. + self.assertEqual(response.status, 503) + self.assertEqual(response.state, METRICS_STATE_UNAVAILABLE) + self.assertIsNone(response.age_seconds) + self.assertEqual(port.metrics_render_count, 0) + self.assertIn("qbit_prism_metrics_snapshot_available 0\n", response.body) + self.assertIn( + "qbit_prism_metrics_snapshot_age_seconds -1.000\n", + response.body, + ) + self.assertNotIn("qbit_prism_fixture", response.body) + + def test_stale_complete_snapshot_is_served_as_200_with_truthful_age( + self, + ) -> None: + port = FakeObservabilityPort() + service = ObservabilityService(port) + service.refresh_metrics_snapshot() + + port.now += 47.5 + + response = service.cached_metrics_payload() + + self.assertEqual(response.status, 200) + self.assertEqual(response.state, METRICS_STATE_STALE) + # Byte-for-byte the generation that was published, with the current + # diagnostics appended -- nothing re-rendered on this thread. + self.assertTrue(response.body.startswith(port.metrics_payload)) + self.assertEqual(port.metrics_render_count, 1) + self.assertIn("qbit_prism_metrics_snapshot_available 1\n", response.body) + self.assertIn("qbit_prism_metrics_snapshot_stale 1\n", response.body) + self.assertIn( + "qbit_prism_metrics_snapshot_age_seconds 47.500\n", + response.body, + ) + # Floored, not rounded: Age must never overstate freshness. + self.assertEqual(response.age_seconds, 47) + + def test_blocked_refresh_past_budget_serves_prior_document_promptly( + self, + ) -> None: + port = FakeObservabilityPort() + service = ObservabilityService(port) + service.refresh_metrics_snapshot() + entered = threading.Event() + release = threading.Event() + render_calls: list[str] = [] + + def blocked_render() -> str: + render_calls.append("render") + entered.set() + release.wait(5.0) + return "qbit_prism_fixture 2\n" + + port.render_metrics_payload = blocked_render # type: ignore[method-assign] + collector = threading.Thread(target=service.refresh_metrics_snapshot) + collector.start() + try: + self.assertTrue(entered.wait(2.0)) + # Far past the staleness budget, with the collector still wedged + # inside the renderer holding the collection lock. + port.now += 600.0 + + started = time.monotonic() + response = service.cached_metrics_payload() + elapsed = time.monotonic() - started + finally: + release.set() + collector.join(5.0) + + # Never waited on the stuck collector, and never re-entered the + # renderer itself: the only render is the collector's own. + self.assertLess(elapsed, 1.0) + self.assertEqual(render_calls, ["render"]) + self.assertEqual(response.status, 200) + self.assertEqual(response.state, METRICS_STATE_STALE) + self.assertTrue(response.body.startswith("qbit_prism_fixture 1\n")) + self.assertEqual(response.age_seconds, 600) + self.assertFalse(collector.is_alive()) + + def test_metrics_response_metadata_is_exact_and_bounded(self) -> None: + port = FakeObservabilityPort() + service = ObservabilityService(port) + + unavailable = service.cached_metrics_payload() + service.refresh_metrics_snapshot() + port.now += 2.0 + fresh = service.cached_metrics_payload() + port.now += 20.0 + stale = service.cached_metrics_payload() + + self.assertEqual( + unavailable.response_headers(), + { + "Cache-Control": "no-store", + METRICS_STATE_HEADER: "unavailable", + }, + ) + self.assertEqual( + fresh.response_headers(), + { + "Cache-Control": "no-store", + METRICS_STATE_HEADER: "fresh", + "Age": "2", + }, + ) + self.assertEqual( + stale.response_headers(), + { + "Cache-Control": "no-store", + METRICS_STATE_HEADER: "stale", + "Age": "22", + "Warning": METRICS_STALE_WARNING, + }, + ) + # Bounded vocabulary, and the warn-code line is the registered 110. + self.assertEqual(METRICS_STATES, ("fresh", "stale", "unavailable")) + for response in (unavailable, fresh, stale): + self.assertIn(response.state, METRICS_STATES) + self.assertEqual( + METRICS_STALE_WARNING, + '110 qbit-prism "metrics snapshot is stale; ' + 'serving last complete payload"', + ) + # Still destructures as the pair every pre-#184 caller unpacks. + status, body = fresh + self.assertEqual((status, body), (fresh.status, fresh.body)) + self.assertEqual(fresh.state, METRICS_STATE_FRESH) + + def test_metrics_exception_is_bounded_and_recovery_replaces_snapshot( + self, + ) -> None: + port = FakeObservabilityPort() + service = ObservabilityService(port) + service.refresh_metrics_snapshot() + port.metrics_error = RuntimeError("dynamic backend detail") + + with self.assertRaises(RuntimeError): + service.refresh_metrics_snapshot() + + state = service.metrics_state() + self.assertEqual(state.metrics_failure_exception_count, 1) + self.assertEqual(state.metrics_last_failure_class, "exception") + _, failed = service.cached_metrics_payload() + self.assertNotIn("dynamic backend detail", failed) + + port.metrics_error = None + port.metrics_payload = "qbit_prism_fixture 2\n" + service.refresh_metrics_snapshot() + + recovered = service.metrics_state() + self.assertEqual(recovered.metrics_collection_generation, 2) + self.assertEqual(recovered.metrics_collection_success_count, 2) + self.assertEqual(recovered.metrics_collection_failure_count, 1) + self.assertIsNone(recovered.metrics_last_failure_class) + status, cached = service.cached_metrics_payload() + self.assertEqual(status, 200) + self.assertTrue(cached.startswith("qbit_prism_fixture 2\n")) + self.assertIn( + 'qbit_prism_metrics_collection_failures_total{class="exception"} 1', + cached, + ) + + def test_slow_metrics_collection_does_not_block_or_partially_replace_cache( + self, + ) -> None: + port = FakeObservabilityPort() + service = ObservabilityService(port) + service.refresh_metrics_snapshot() + entered = threading.Event() + release = threading.Event() + + def slow_render() -> str: + entered.set() + release.wait(2.0) + return "qbit_prism_fixture 2\n" + + port.render_metrics_payload = slow_render # type: ignore[method-assign] + collector = threading.Thread(target=service.refresh_metrics_snapshot) + collector.start() + self.assertTrue(entered.wait(1.0)) + + status, during = service.cached_metrics_payload() + + self.assertEqual(status, 200) + self.assertTrue(during.startswith("qbit_prism_fixture 1\n")) + release.set() + collector.join(2.0) + self.assertFalse(collector.is_alive()) + _, after = service.cached_metrics_payload() + self.assertTrue(after.startswith("qbit_prism_fixture 2\n")) + + def test_metrics_and_health_loop_state_are_independent(self) -> None: + port = FakeObservabilityPort() + service = ObservabilityService(port) + + self.assertTrue(service.begin_refresh_loop()) + self.assertTrue(service.begin_metrics_refresh_loop()) + self.assertTrue(service.state().health_refresh_loop_running) + self.assertTrue(service.metrics_state().metrics_refresh_loop_running) + + service.metrics_snapshot_loop() + + self.assertTrue(service.state().health_refresh_loop_running) + self.assertFalse(service.metrics_state().metrics_refresh_loop_running) + + +class SemanticCoverageDeliveryHealthTests(unittest.TestCase): + """Issue #216: the delivery stall is timed on semantic coverage.""" + + def test_rapid_tip_churn_with_complete_semantic_coverage_stays_ready( + self, + ) -> None: + port = FakeObservabilityPort() + port.inputs = incident_inputs() + service = ObservabilityService(port) + + # Exact coverage churns below 95% for four times the 5s deadline + # while every miner keeps semantically current work, nothing is + # genuinely pending, and deliveries keep landing. The refresher's + # cached /healthz answer must hold 200 on every sample, not flap. + for step, exact in enumerate((4, 12, 1, 20, 7, 30)): + port.now = 100.0 + 4.0 * step + port.inputs = replace( + port.inputs, + clients_with_current_tip_jobs=exact, + last_initial_job_delivery_monotonic=port.now - 0.5, + ) + service.refresh_health_snapshot() + status, payload = service.cached_health_payload() + + self.assertEqual(status, 200, payload) + self.assertTrue(payload["ok"]) + self.assertTrue(payload["mining_ready"]) + self.assertFalse(payload["initial_delivery_stalled"]) + self.assertEqual(payload["unhealthy_reasons"], []) + self.assertEqual(payload["pending_initial_jobs"], 0) + self.assertEqual( + payload["oldest_genuinely_pending_initial_job_age_seconds"], + 0.0, + ) + self.assertLess(payload["current_tip_job_coverage"], 0.95) + self.assertEqual(payload["semantic_current_work_ratio"], 1.0) + # The exact gap keeps reporting the churn as telemetry. + self.assertEqual( + payload["current_tip_coverage_gap_age_seconds"], + 4.0 * step, + ) + self.assertEqual(payload["semantic_current_work_gap_age_seconds"], 0.0) + + state = service.state() + self.assertEqual(state.mining_delivery_failure_started_monotonic, 100.0) + self.assertIsNone(state.mining_semantic_work_gap_started_monotonic) + + def test_genuine_first_job_starvation_fails_closed_after_deadline( + self, + ) -> None: + port = FakeObservabilityPort() + port.inputs = replace( + healthy_inputs(), + active_connections=3, + subscribed_connections=3, + authorized_connections=3, + pending_initial_jobs=3, + oldest_pending_initial_job_age_seconds=4.999, + oldest_genuinely_pending_initial_job_age_seconds=4.999, + clients_with_no_active_job=3, + ) + service = ObservabilityService(port) + + self.assertTrue(service.mining_delivery_snapshot()["mining_ready"]) + + # Only the genuine pending age crosses the deadline here: both + # coverage gaps are a millisecond old, so starvation fires on its own. + port.now += 0.001 + port.inputs = replace( + port.inputs, + oldest_pending_initial_job_age_seconds=5.0, + oldest_genuinely_pending_initial_job_age_seconds=5.0, + ) + starved = service.mining_delivery_snapshot() + + self.assertFalse(starved["mining_ready"]) + self.assertTrue(starved["initial_delivery_stalled"]) + self.assertIn("initial-delivery-stalled", starved["unhealthy_reasons"]) + self.assertEqual(starved["semantic_current_work_gap_age_seconds"], 0.001) + + def test_sustained_semantic_work_loss_fails_closed_after_deadline( + self, + ) -> None: + # Every miner holds work, so nothing is genuinely pending, but none + # of it matches the current template content and none is on the + # observed tip: fanout has stopped. This must still fail closed. + port = FakeObservabilityPort() + port.inputs = replace( + healthy_inputs(), + active_connections=4, + subscribed_connections=4, + authorized_connections=4, + clients_with_current_tip_jobs=0, + clients_with_semantically_current_work=0, + ) + service = ObservabilityService(port) + + self.assertTrue(service.mining_delivery_snapshot()["mining_ready"]) + port.now = 104.999 + grace = service.mining_delivery_snapshot() + self.assertTrue(grace["mining_ready"]) + self.assertEqual(grace["semantic_current_work_gap_age_seconds"], 4.999) + self.assertEqual(grace["current_tip_coverage_gap_age_seconds"], 4.999) + + port.now = 105.0 + stalled = service.mining_delivery_snapshot() + self.assertFalse(stalled["mining_ready"]) + self.assertTrue(stalled["initial_delivery_stalled"]) + self.assertEqual(stalled["unhealthy_reasons"], ["initial-delivery-stalled"]) + self.assertEqual( + stalled["oldest_genuinely_pending_initial_job_age_seconds"], + 0.0, + ) + + # Semantic recovery alone ends the stall and closes the semantic gap + # while the strict gauge is still lagging the observed tip. + port.now = 106.0 + port.inputs = replace(port.inputs, clients_with_semantically_current_work=4) + recovered = service.mining_delivery_snapshot() + self.assertTrue(recovered["mining_ready"]) + self.assertFalse(recovered["initial_delivery_stalled"]) + self.assertEqual(recovered["semantic_current_work_gap_age_seconds"], 0.0) + self.assertEqual(recovered["current_tip_coverage_gap_age_seconds"], 6.0) + state = service.state() + self.assertIsNone(state.mining_semantic_work_gap_started_monotonic) + self.assertEqual(state.mining_delivery_failure_started_monotonic, 100.0) + + def test_semantic_gap_must_be_sustained_not_sampled(self) -> None: + # A long-running exact gap plus one poor semantic sample is not a + # stall: the semantic gap has its own timer and must itself last a + # full deadline before health fails. + port = FakeObservabilityPort() + port.inputs = incident_inputs() + service = ObservabilityService(port) + service.mining_delivery_snapshot() + + port.now = 120.0 + port.inputs = replace(port.inputs, clients_with_semantically_current_work=20) + dip = service.mining_delivery_snapshot() + self.assertTrue(dip["mining_ready"]) + self.assertEqual(dip["current_tip_coverage_gap_age_seconds"], 20.0) + self.assertEqual(dip["semantic_current_work_gap_age_seconds"], 0.0) + + port.now = 124.0 + port.inputs = replace(port.inputs, clients_with_semantically_current_work=33) + back = service.mining_delivery_snapshot() + self.assertTrue(back["mining_ready"]) + self.assertIsNone(service.state().mining_semantic_work_gap_started_monotonic) + + port.now = 128.0 + port.inputs = replace(port.inputs, clients_with_semantically_current_work=20) + self.assertTrue(service.mining_delivery_snapshot()["mining_ready"]) + port.now = 132.999 + self.assertTrue(service.mining_delivery_snapshot()["mining_ready"]) + port.now = 133.0 + stalled = service.mining_delivery_snapshot() + self.assertFalse(stalled["mining_ready"]) + self.assertEqual(stalled["unhealthy_reasons"], ["initial-delivery-stalled"]) + self.assertEqual(stalled["semantic_current_work_gap_age_seconds"], 5.0) + + def test_exact_tip_work_without_semantic_evidence_keeps_health_ready( + self, + ) -> None: + # An embedder that publishes only the observed tip (no template + # snapshot) reads semantic coverage 0 for every client while the + # strict gauge is complete. In production strict currency is checked + # against the same fingerprint and payout generation, so this state + # means "no semantic evidence", never "semantically stale", and the + # strict gauge must still count as delivered work. + port = FakeObservabilityPort() + port.inputs = replace( + healthy_inputs(), + active_connections=4, + subscribed_connections=4, + authorized_connections=4, + clients_with_current_tip_jobs=4, + clients_with_semantically_current_work=0, + ) + service = ObservabilityService(port) + + service.mining_delivery_snapshot() + port.now += 10.0 + payload = service.mining_delivery_snapshot() + + self.assertTrue(payload["mining_ready"]) + self.assertFalse(payload["initial_delivery_stalled"]) + self.assertEqual(payload["semantic_current_work_gap_age_seconds"], 10.0) + self.assertEqual(payload["current_tip_coverage_gap_age_seconds"], 0.0) + + def test_incident_payload_retains_exact_and_semantic_telemetry(self) -> None: + port = FakeObservabilityPort() + port.inputs = incident_inputs() + service = ObservabilityService(port) + service.mining_delivery_snapshot() + + port.now += 6.0 + payload = service.mining_delivery_snapshot() + + self.assertEqual(payload["authorized_connections"], 33) + self.assertEqual(payload["clients_with_current_tip_jobs"], 4) + self.assertEqual(payload["current_tip_job_coverage"], 0.121212) + self.assertEqual(payload["clients_with_semantically_current_work"], 33) + self.assertEqual(payload["semantic_current_work_ratio"], 1.0) + self.assertEqual(payload["pending_initial_jobs"], 0) + self.assertEqual( + payload["oldest_genuinely_pending_initial_job_age_seconds"], + 0.0, + ) + self.assertEqual(payload["clients_with_no_active_job"], 0) + self.assertEqual(payload["current_tip_coverage_gap_age_seconds"], 6.0) + self.assertEqual(payload["semantic_current_work_gap_age_seconds"], 0.0) + # Compatibility aliases keep their exact-tip meaning. + self.assertEqual(payload["clients_with_current_tip_job"], 4) + self.assertEqual(payload["clients_without_current_tip_job"], 29) + self.assertEqual(payload["current_tip_job_coverage_ratio"], 4 / 33) + self.assertEqual(payload["oldest_initial_job_pending_seconds"], 0.0) + self.assertTrue(payload["mining_ready"]) + self.assertTrue(payload["mining_delivery_healthy"]) + self.assertFalse(payload["initial_delivery_stalled"]) + self.assertFalse(payload["overload"]) + self.assertEqual(payload["unhealthy_reasons"], []) + + def test_overload_now_keeps_using_exact_coverage(self) -> None: + port = FakeObservabilityPort() + port.inputs = replace( + incident_inputs(), + active_connections=64, + subscribed_connections=64, + authorized_connections=64, + clients_with_current_tip_jobs=8, + clients_with_semantically_current_work=64, + ) + service = ObservabilityService(port) + + strict_pressure = service.mining_delivery_snapshot() + + self.assertTrue(strict_pressure["connection_capacity_saturated"]) + self.assertTrue(strict_pressure["overload"]) + self.assertTrue(strict_pressure["mining_ready"]) + self.assertEqual( + service.state().mining_overload_started_monotonic, + 100.0, + ) + + # Semantic loss with complete exact coverage is not overload. + port.now += 1.0 + port.inputs = replace( + port.inputs, + clients_with_current_tip_jobs=64, + clients_with_semantically_current_work=0, + ) + semantic_only = service.mining_delivery_snapshot() + + self.assertTrue(semantic_only["connection_capacity_saturated"]) + self.assertFalse(semantic_only["overload"]) + self.assertEqual(semantic_only["overload_age_seconds"], 0.0) + self.assertIsNone(service.state().mining_overload_started_monotonic) + + def test_reject_storm_keeps_using_exact_coverage(self) -> None: + port = FakeObservabilityPort() + port.inputs = replace( + incident_inputs(), + submitted_shares=100, + stale_unknown_rejections=95, + ) + service = ObservabilityService(port) + + storm = service.mining_delivery_snapshot() + + self.assertTrue(storm["overload"]) + self.assertFalse(storm["mining_ready"]) + self.assertEqual( + storm["unhealthy_reasons"], + ["stale-unknown-rejection-storm"], + ) + + # The same rejection ratio with complete exact coverage and no + # semantic coverage is not a storm: the semantic gauge does not + # corroborate it. + port.now += 1.0 + port.inputs = replace( + port.inputs, + clients_with_current_tip_jobs=33, + clients_with_semantically_current_work=0, + ) + quiet = service.mining_delivery_snapshot() + + self.assertFalse(quiet["overload"]) + self.assertTrue(quiet["mining_ready"]) + self.assertEqual(quiet["unhealthy_reasons"], []) + + def test_cap_saturated_reason_keeps_using_exact_coverage(self) -> None: + port = FakeObservabilityPort() + port.inputs = replace( + incident_inputs(), + active_connections=64, + subscribed_connections=64, + authorized_connections=64, + clients_with_current_tip_jobs=8, + clients_with_semantically_current_work=64, + ) + service = ObservabilityService(port) + service.mining_delivery_snapshot() + + port.now += 5.0 + persistent = service.mining_delivery_snapshot() + + # Strict-tip pressure under a full cap is surfaced on its own; the + # delivery stall does not ride along on the exact gap. + self.assertFalse(persistent["mining_ready"]) + self.assertEqual( + persistent["unhealthy_reasons"], + ["connection-capacity-saturated"], + ) + self.assertFalse(persistent["initial_delivery_stalled"]) + self.assertEqual(persistent["current_tip_coverage_gap_age_seconds"], 5.0) + + # Exact coverage restored under the same full cap clears the reason + # even though semantic coverage is now gone. + port.now += 5.0 + port.inputs = replace( + port.inputs, + clients_with_current_tip_jobs=64, + clients_with_semantically_current_work=0, + ) + cleared = service.mining_delivery_snapshot() + + self.assertTrue(cleared["connection_capacity_saturated"]) + self.assertTrue(cleared["mining_ready"]) + self.assertEqual(cleared["unhealthy_reasons"], []) + + +def bare_semantic_coordinator() -> PrismCoordinator: + server = PrismCoordinator.__new__(PrismCoordinator) + server.lock = threading.RLock() + server.stop_event = threading.Event() + server.clients = set() + server.jobs = {} + server.connection_counter = 0 + server.job_counter = 0 + server.stratum_max_connections = 8 + server.stratum_max_connections_per_username = 0 + server.stratum_max_pending_initial_jobs = 4 + server.stratum_initial_job_timeout_seconds = 30.0 + server.mining_health_startup_grace_seconds = 30.0 + server.tip_refresh_max_workers = 1 + server.tip_template_snapshot = None + server.started_monotonic = time.monotonic() + server.submitted_share_count = 0 + server.rejection_counts_by_reason = {} + server._ensure_job_cache_state() + server._ensure_tip_refresh_state() + server._ensure_initial_job_state() + return server + + +def semantic_client(server: PrismCoordinator, connection_id: int) -> ClientState: + state = ClientState( + sock=SimpleNamespace(), + address=("127.0.0.1", 40_000 + connection_id), + connection_id=connection_id, + extranonce1_hex=f"{connection_id:08x}", + ) + state.subscribed = True + state.authorized = True + state.authorization_generation = 1 + state.worker = WorkerIdentity( + username=f"miner-{connection_id}", + payout_address=f"miner-{connection_id}", + worker_name=None, + script_pubkey_hex="5220" + "11" * 32, + p2mr_program_hex="11" * 32, + ) + state.username = state.worker.username + server.clients.add(state) + return state + + +class SemanticCurrencyAdapterTests(unittest.TestCase): + def test_template_generation_and_identity_do_not_affect_the_count( + self, + ) -> None: + # Upstream #107: template generation and template object identity do + # not participate in semantic currency. Client-session generations do, + # and the strict fail-closed gauge stays untouched by semantic matches. + server = bare_semantic_coordinator() + fingerprint = "ff" * 32 + server.tip_template_snapshot = SimpleNamespace( + bestblockhash="aa" * 32, + template_fingerprint=fingerprint, + template_generation=7, + template_artifacts=None, + ) + matching = semantic_client(server, 1) + matching.active_job = SimpleNamespace( + template={"previousblockhash": "aa" * 32}, + template_fingerprint=fingerprint, + # Deliberately different template generation and a distinct + # template object: neither may disqualify semantic currency. + template_generation=3, + payout_state_generation=0, + connection_id=matching.connection_id, + authorization_generation=matching.authorization_generation, + difficulty_generation=matching.difficulty_generation, + ) + mismatched = semantic_client(server, 2) + mismatched.active_job = SimpleNamespace( + template={"previousblockhash": "aa" * 32}, + template_fingerprint="ee" * 32, + template_generation=7, + payout_state_generation=0, + connection_id=mismatched.connection_id, + authorization_generation=mismatched.authorization_generation, + difficulty_generation=mismatched.difficulty_generation, + ) + + snapshot = server.mining_delivery_snapshot() + + self.assertEqual(snapshot["clients_with_semantically_current_work"], 1) + self.assertEqual(snapshot["semantic_current_work_ratio"], 0.5) + self.assertEqual(snapshot["clients_with_current_tip_jobs"], 0) + self.assertEqual(snapshot["current_tip_job_coverage"], 0.0) + self.assertTrue(snapshot["mining_ready"]) + + def test_stale_payout_generation_disqualifies_semantic_currency(self) -> None: + server = bare_semantic_coordinator() + fingerprint = "ff" * 32 + server.tip_template_snapshot = SimpleNamespace( + bestblockhash="aa" * 32, + template_fingerprint=fingerprint, + template_generation=7, + template_artifacts=None, + ) + stale = semantic_client(server, 1) + stale.active_job = SimpleNamespace( + template={"previousblockhash": "aa" * 32}, + template_fingerprint=fingerprint, + template_generation=7, + payout_state_generation=99, + connection_id=stale.connection_id, + authorization_generation=stale.authorization_generation, + difficulty_generation=stale.difficulty_generation, + ) + + snapshot = server.mining_delivery_snapshot() + + self.assertEqual(snapshot["clients_with_semantically_current_work"], 0) + self.assertEqual(snapshot["semantic_current_work_ratio"], 0.0) + + def test_session_generation_changes_disqualify_semantic_currency(self) -> None: + server = bare_semantic_coordinator() + fingerprint = "ff" * 32 + server.tip_template_snapshot = SimpleNamespace( + bestblockhash="aa" * 32, + template_fingerprint=fingerprint, + template_generation=7, + template_artifacts=None, + ) + reauthorized = semantic_client(server, 1) + reauthorized.active_job = SimpleNamespace( + template={"previousblockhash": "aa" * 32}, + template_fingerprint=fingerprint, + payout_state_generation=0, + connection_id=reauthorized.connection_id, + authorization_generation=reauthorized.authorization_generation, + difficulty_generation=reauthorized.difficulty_generation, + ) + retargeted = semantic_client(server, 2) + retargeted.active_job = SimpleNamespace( + template={"previousblockhash": "aa" * 32}, + template_fingerprint=fingerprint, + payout_state_generation=0, + connection_id=retargeted.connection_id, + authorization_generation=retargeted.authorization_generation, + difficulty_generation=retargeted.difficulty_generation, + ) + + reauthorized.authorization_generation += 1 + retargeted.difficulty_generation += 1 + snapshot = server.mining_delivery_snapshot() + + self.assertEqual(snapshot["clients_with_semantically_current_work"], 0) + self.assertEqual(snapshot["semantic_current_work_ratio"], 0.0) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_prism_payout_state.py b/tests/test_prism_payout_state.py new file mode 100644 index 000000000..93ba78546 --- /dev/null +++ b/tests/test_prism_payout_state.py @@ -0,0 +1,6043 @@ +#!/usr/bin/env python3 +"""Focused PRISM coordinator payout state tests.""" +# ruff: noqa: F403, F405 + +from __future__ import annotations + +import json +import unittest + +from lab.prism.payout_state import AcceptedParentPayoutPreviewPending +from lab.prism.payout_state import _snapshot_window_weight_within_band +from lab.prism.payout_state import PRISM_PAYOUT_WINDOW_BUILD_OUTCOMES +from lab.prism.payout_state import PRISM_PAYOUT_WINDOW_BUILD_PHASES +from lab.prism.payout_state import PRISM_REWARD_WINDOW_MULTIPLIER +from lab.prism.payout_state import PRISM_SNAPSHOT_WINDOW_MARGIN +from lab.prism.payout_state import PRISM_TIP_REFRESH_WINDOW_STALENESS_SECONDS +from tests.prism_coordinator_test_support import * + + +class JobBundleCacheTests(unittest.TestCase): + def test_child_bundle_waits_for_pending_parent_preview_without_confirmed_read( + self, + ) -> None: + class PreviewLedger(FakeLedger): + def __init__(self) -> None: + super().__init__() + self.current_balance_reads = 0 + + def current_prior_balances(self) -> list[dict[str, object]]: + self.current_balance_reads += 1 + return [] + + class ObservedCondition(threading.Condition): + def __init__(self) -> None: + super().__init__() + self.wait_entered = threading.Event() + + def wait(self, timeout: float | None = None) -> bool: + self.wait_entered.set() + return super().wait(timeout) + + ledger = PreviewLedger() + server, rpc = coordinator(ledger=ledger) + recorded = install_fake_bundle_builder(server) + artifacts = server.store_template_artifacts(dict(rpc.template)) + self.assertIsNotNone(artifacts) + assert artifacts is not None + parent_hash = str(rpc.template["previousblockhash"]) + preview_condition = ObservedCondition() + server._accepted_block_payout_preview_condition = preview_condition + server.accepted_block_payout_preview_wait_seconds = 10 + preview = [ + { + "recipient_id": "miner-a", + "order_key": "miner-a", + "p2mr_program_hex": "11" * 32, + "balance_sats": 25, + } + ] + bundles: list[object] = [] + errors: list[BaseException] = [] + + def build_child_bundle() -> None: + try: + bundles.append(server.shared_job_bundle(artifacts, worker())) + except BaseException as exc: # pragma: no cover - asserted below + errors.append(exc) + + server._begin_accepted_block_payout_preview(parent_hash) + thread = threading.Thread(target=build_child_bundle, daemon=True) + thread.start() + try: + preview_wait_reached = preview_condition.wait_entered.wait(2) + waiting_for_preview = thread.is_alive() + confirmed_reads_while_pending = ledger.current_balance_reads + builds_while_pending = int(recorded["calls"]) + finally: + server._publish_accepted_block_payout_preview(parent_hash, preview) + thread.join(5) + server._clear_accepted_block_payout_preview(parent_hash) + + self.assertTrue(preview_wait_reached) + self.assertTrue(waiting_for_preview) + self.assertEqual(confirmed_reads_while_pending, 0) + self.assertEqual(builds_while_pending, 0) + self.assertFalse(thread.is_alive()) + self.assertEqual(errors, []) + self.assertEqual(len(bundles), 1) + bundle = bundles[0] + self.assertEqual(bundle.prior_balances, preview) # type: ignore[union-attr] + self.assertEqual(recorded["last_kwargs"]["prior_balances"], preview) # type: ignore[index] + # Preview publication prepares an artifact from one confirmed snapshot; + # the zero count captured above proves the waiting child did not read it. + self.assertEqual(ledger.current_balance_reads, 1) + self.assertEqual( # type: ignore[union-attr] + bundle.payout_state_generation, + server._payout_state_generation, + ) + + self.assertEqual(server._prior_balances_for_job_parent(parent_hash), []) + self.assertEqual(ledger.current_balance_reads, 2) + + def test_parent_preview_publication_is_idempotent_and_withdrawal_invalidates( + self, + ) -> None: + server, _rpc = coordinator() + parent_hash = "ab" * 32 + preview = [ + { + "recipient_id": "miner-a", + "order_key": "miner-a", + "p2mr_program_hex": "11" * 32, + "balance_sats": 25, + } + ] + + server._begin_accepted_block_payout_preview(parent_hash) + self.assertEqual( + server._publish_accepted_block_payout_preview(parent_hash, preview), + preview, + ) + self.assertEqual(server._payout_state_generation, 1) + + self.assertEqual( + server._publish_accepted_block_payout_preview(parent_hash, preview), + preview, + ) + self.assertEqual(server._payout_state_generation, 1) + with self.assertRaisesRegex(RuntimeError, "changed during retry"): + server._publish_accepted_block_payout_preview( + parent_hash, + [{**preview[0], "balance_sats": 26}], + ) + + server._clear_accepted_block_payout_preview( + parent_hash, + invalidate_published=True, + ) + self.assertEqual(server._payout_state_generation, 2) + self.assertEqual(server._accepted_block_payout_previews, {}) + self.assertEqual( + server._invalidated_accepted_block_payout_previews, + {parent_hash: None}, + ) + with self.assertRaisesRegex(TemplateRefreshBlocked, "was withdrawn"): + server._prior_balances_for_job_parent(parent_hash) + + server._begin_accepted_block_payout_preview(parent_hash) + self.assertEqual(server._invalidated_accepted_block_payout_previews, {}) + server._clear_accepted_block_payout_preview(parent_hash) + + def test_unpublished_parent_preview_retries_and_reopens_delivery(self) -> None: + server, _rpc = coordinator() + server.payout_reconcile_supersession_retries = 2 + parent_hash = "ac" * 32 + preview = [ + { + "recipient_id": "miner-a", + "order_key": "miner-a", + "p2mr_program_hex": "11" * 32, + "balance_sats": 25, + } + ] + + server._begin_accepted_block_payout_preview(parent_hash) + with patch.object( + server, + "_publish_payout_state_candidate", + return_value=None, + ) as publish_candidate: + self.assertEqual( + server._publish_accepted_block_payout_preview(parent_hash, preview), + preview, + ) + + transition = server._accepted_block_payout_previews[parent_hash] + self.assertEqual(publish_candidate.call_count, 3) + self.assertIsNotNone(transition.preview) + self.assertIsNone(transition.published_generation) + self.assertEqual(server._payout_state_generation, 0) + self.assertTrue(server._payout_state_publication_fenced()) + self.assertTrue(server._payout_state_delivery_gate._delivery_blocked) + + self.assertEqual( + server._publish_accepted_block_payout_preview(parent_hash, preview), + preview, + ) + + transition = server._accepted_block_payout_previews[parent_hash] + self.assertEqual(transition.published_generation, 1) + self.assertEqual(server._payout_state_generation, 1) + self.assertFalse(server._payout_state_publication_fenced()) + self.assertFalse(server._payout_state_delivery_gate._delivery_blocked) + + def test_ancestor_scan_probes_each_height_once(self) -> None: + # Every offered or replay-adopted candidate arms a preview barrier, and + # siblings of one solved block share a height -- so without dedupe this + # scan issues one getblockhash per barrier for a single answer. The + # 2026-08-20 load test reached 3,120 barriers at one height. + server, rpc = coordinator() + original_rpc_call = rpc.call + heights_probed: list[int] = [] + + def counting_call( + method: str, + params: list[object] | None = None, + ) -> object: + if method == "getblockhash": + assert params is not None + heights_probed.append(int(params[0])) + return "ff" * 32 + return original_rpc_call(method, params) + + rpc.call = counting_call # type: ignore[method-assign] + for index in range(50): + server._begin_accepted_block_payout_preview( + f"{index:064x}", + block_height=10, + ) + for index in range(50, 60): + server._begin_accepted_block_payout_preview( + f"{index:064x}", + block_height=11, + ) + + server._accepted_block_payout_transition_for_parent( + "ab" * 32, + parent_height=12, + ) + + # Two distinct heights, sixty barriers: two probes, not sixty. + self.assertEqual(sorted(heights_probed), [10, 11]) + + def test_ancestor_scan_dedupe_is_scoped_to_one_scan(self) -> None: + # The active block at a height changes under reorgs, so the memo must + # not outlive the scan that built it. + server, rpc = coordinator() + original_rpc_call = rpc.call + probe_count = 0 + + def counting_call( + method: str, + params: list[object] | None = None, + ) -> object: + nonlocal probe_count + if method == "getblockhash": + probe_count += 1 + return "ff" * 32 + return original_rpc_call(method, params) + + rpc.call = counting_call # type: ignore[method-assign] + server._begin_accepted_block_payout_preview("cd" * 32, block_height=10) + + for _ in range(3): + server._accepted_block_payout_transition_for_parent( + "ab" * 32, + parent_height=12, + ) + + self.assertEqual(probe_count, 3) + + def test_withdrawn_landed_transition_blocks_active_descendant_fallback( + self, + ) -> None: + class CountingLedger(FakeLedger): + def __init__(self) -> None: + super().__init__() + self.current_balance_reads = 0 + + def current_prior_balances(self) -> list[dict[str, object]]: + self.current_balance_reads += 1 + return [] + + ledger = CountingLedger() + server, rpc = coordinator(ledger=ledger) + accepted_hash = "bc" * 32 + descendant_hash = "bd" * 32 + original_rpc_call = rpc.call + + def active_chain_call( + method: str, + params: list[object] | None = None, + ) -> object: + if method == "getblockhash": + self.assertEqual(params, [10]) + return accepted_hash + return original_rpc_call(method, params) + + rpc.call = active_chain_call # type: ignore[method-assign] + server._begin_accepted_block_payout_preview( + accepted_hash, + block_height=10, + ) + server._mark_accepted_block_payout_landed( + accepted_hash, + block_height=10, + ) + server._clear_accepted_block_payout_preview( + accepted_hash, + invalidate_published=True, + ) + + with self.assertRaisesRegex(TemplateRefreshBlocked, "was withdrawn"): + server._prior_balances_for_job_parent( + descendant_hash, + parent_height=11, + ) + self.assertEqual(ledger.current_balance_reads, 0) + self.assertTrue(server._tip_refresh_retry.is_set()) + + def test_inactive_landed_ancestor_rejects_preview_patched_artifact( + self, + ) -> None: + accepted_hash = "c0" * 32 + alternate_tip = "c1" * 32 + server, rpc = coordinator( + template=base_template(height=12, prevhash=alternate_tip) + ) + recorded = install_fake_bundle_builder(server) + artifacts = server.store_template_artifacts(dict(rpc.template)) + assert artifacts is not None + self.assertTrue(server.pool_readiness_latched()) + preview = [ + { + "recipient_id": "miner-a", + "order_key": "miner-a", + "p2mr_program_hex": "11" * 32, + "balance_sats": 25, + } + ] + server._begin_accepted_block_payout_preview( + accepted_hash, + block_height=10, + ) + server._mark_accepted_block_payout_landed( + accepted_hash, + block_height=10, + ) + server._publish_accepted_block_payout_preview(accepted_hash, preview) + with server._job_cache_lock: + artifact = server._payout_ledger_artifact + self.assertIsNotNone(artifact) + assert artifact is not None + self.assertEqual(list(artifact.prior_balances), preview) + + original_rpc_call = rpc.call + + def alternate_chain_call( + method: str, + params: list[object] | None = None, + ) -> object: + if method == "getblockhash": + self.assertEqual(params, [10]) + return "c2" * 32 + return original_rpc_call(method, params) + + rpc.call = alternate_chain_call # type: ignore[method-assign] + + with self.assertRaisesRegex( + TemplateRefreshBlocked, + "no longer active", + ): + server.shared_job_bundle(artifacts, mode="ready") + + self.assertEqual(recorded["calls"], 0) + self.assertTrue(server._tip_refresh_retry.is_set()) + + def test_waiting_child_does_not_fall_back_after_transition_withdrawal( + self, + ) -> None: + class CountingLedger(FakeLedger): + def __init__(self) -> None: + super().__init__() + self.current_balance_reads = 0 + + def current_prior_balances(self) -> list[dict[str, object]]: + self.current_balance_reads += 1 + return [] + + class ObservedCondition(threading.Condition): + def __init__(self) -> None: + super().__init__() + self.wait_entered = threading.Event() + + def wait(self, timeout: float | None = None) -> bool: + self.wait_entered.set() + return super().wait(timeout) + + ledger = CountingLedger() + server, _rpc = coordinator(ledger=ledger) + parent_hash = "be" * 32 + preview_condition = ObservedCondition() + server._accepted_block_payout_preview_condition = preview_condition + server.accepted_block_payout_preview_wait_seconds = 10 + server._begin_accepted_block_payout_preview( + parent_hash, + block_height=10, + ) + server._mark_accepted_block_payout_landed( + parent_hash, + block_height=10, + ) + errors: list[BaseException] = [] + + def read_parent_balances() -> None: + try: + server._prior_balances_for_job_parent( + parent_hash, + parent_height=10, + ) + except BaseException as exc: # pragma: no cover - asserted below + errors.append(exc) + + thread = threading.Thread(target=read_parent_balances, daemon=True) + thread.start() + try: + self.assertTrue(preview_condition.wait_entered.wait(2)) + self.assertTrue(thread.is_alive()) + finally: + server._clear_accepted_block_payout_preview( + parent_hash, + invalidate_published=True, + ) + thread.join(5) + + self.assertFalse(thread.is_alive()) + self.assertEqual(len(errors), 1) + self.assertIsInstance(errors[0], TemplateRefreshBlocked) + self.assertIn("was withdrawn", str(errors[0])) + self.assertEqual(ledger.current_balance_reads, 0) + + def test_pending_parent_preview_wait_is_bounded_and_retryable(self) -> None: + server, _rpc = coordinator() + parent_hash = "ac" * 32 + server.accepted_block_payout_preview_wait_seconds = 0.01 + server._begin_accepted_block_payout_preview(parent_hash) + + with self.assertRaisesRegex(TemplateRefreshBlocked, "not ready"): + server._prior_balances_for_job_parent(parent_hash) + + self.assertTrue(server._tip_refresh_retry.is_set()) + server._clear_accepted_block_payout_preview(parent_hash) + + def test_replayed_active_ancestor_blocks_descendant_until_preview(self) -> None: + class PreviewLedger(FakeLedger): + def __init__(self) -> None: + super().__init__() + self.current_balance_reads = 0 + + def current_prior_balances(self) -> list[dict[str, object]]: + self.current_balance_reads += 1 + return [] + + class ObservedCondition(threading.Condition): + def __init__(self) -> None: + super().__init__() + self.wait_entered = threading.Event() + + def wait(self, timeout: float | None = None) -> bool: + self.wait_entered.set() + return super().wait(timeout) + + ledger = PreviewLedger() + server, rpc = coordinator(ledger=ledger) + accepted_hash = "ad" * 32 + descendant_hash = "ae" * 32 + preview = [ + { + "recipient_id": "miner-a", + "order_key": "miner-a", + "p2mr_program_hex": "11" * 32, + "balance_sats": 25, + } + ] + original_rpc_call = rpc.call + + def active_chain_call( + method: str, + params: list[object] | None = None, + ) -> object: + if method == "getblockhash": + self.assertEqual(params, [10]) + return accepted_hash + return original_rpc_call(method, params) + + rpc.call = active_chain_call # type: ignore[method-assign] + preview_condition = ObservedCondition() + server._accepted_block_payout_preview_condition = preview_condition + server.accepted_block_payout_preview_wait_seconds = 10 + server._begin_accepted_block_payout_preview(accepted_hash, block_height=10) + balances: list[list[dict[str, object]]] = [] + errors: list[BaseException] = [] + + def read_descendant_balances() -> None: + try: + balances.append( + server._prior_balances_for_job_parent( + descendant_hash, + parent_height=11, + ) + ) + except BaseException as exc: # pragma: no cover - asserted below + errors.append(exc) + + thread = threading.Thread(target=read_descendant_balances, daemon=True) + thread.start() + try: + self.assertTrue(preview_condition.wait_entered.wait(2)) + self.assertTrue(thread.is_alive()) + self.assertEqual(ledger.current_balance_reads, 0) + finally: + server._publish_accepted_block_payout_preview(accepted_hash, preview) + thread.join(5) + server._clear_accepted_block_payout_preview(accepted_hash) + + self.assertFalse(thread.is_alive()) + self.assertEqual(errors, []) + self.assertEqual(balances, [preview]) + self.assertEqual(ledger.current_balance_reads, 0) + + def test_landed_transition_bars_reconciliation_before_preview(self) -> None: + server, _rpc = coordinator() + block_hash = "af" * 32 + server._begin_accepted_block_payout_preview(block_hash, block_height=10) + server._mark_accepted_block_payout_landed(block_hash, block_height=10) + + with self.assertRaisesRegex( + _PayoutStatePublicationBlocked, + "confirmation is still pending", + ): + with server._payout_balance_mutation(): + self.fail("landed transition must bar payout reconciliation") + with self.assertRaisesRegex( + _PayoutStatePublicationBlocked, + "confirmation is still pending", + ): + server.reconcile_prism_pool_blocks_once() + + server._clear_accepted_block_payout_preview(block_hash) + with server._payout_balance_mutation(): + pass + + def test_readiness_latch_during_preparation_admission_reselects_ready_mode(self) -> None: + ledger = FakeLedger(miners=["solo"]) + server, rpc = coordinator(ledger=ledger) + recorded = install_fake_bundle_builder(server) + artifacts = server.store_template_artifacts(dict(rpc.template)) + self.assertIsNotNone(artifacts) + assert artifacts is not None + first_lookup_entered = threading.Event() + release_first_lookup = threading.Event() + original_lookup = server._lookup_job_bundle + lookup_calls = 0 + + def block_first_lookup(key: tuple[object, ...]) -> object: + nonlocal lookup_calls + lookup_calls += 1 + if lookup_calls == 1: + first_lookup_entered.set() + self.assertTrue(release_first_lookup.wait(2.0)) + return original_lookup(key) + + server._lookup_job_bundle = block_first_lookup # type: ignore[method-assign] + bundles: list[object] = [] + errors: list[BaseException] = [] + + def build_bundle() -> None: + try: + bundles.append(server.shared_job_bundle(artifacts, worker())) + except BaseException as exc: # pragma: no cover - assertion reports it + errors.append(exc) + + build_thread = threading.Thread(target=build_bundle) + build_thread.start() + self.assertTrue(first_lookup_entered.wait(2.0)) + ledger.miners = ["miner-a", "miner-b", "miner-c"] + release_first_lookup.set() + + build_thread.join(2.0) + + self.assertFalse(build_thread.is_alive()) + self.assertEqual(errors, []) + self.assertEqual(len(bundles), 1) + bundle = bundles[0] + self.assertFalse(bundle.collection_only) # type: ignore[union-attr] + self.assertEqual(recorded["calls"], 1) + self.assertEqual(ledger.snapshot_calls, 1) + + def test_payout_generation_advance_does_not_cancel_new_generation_fanout(self) -> None: + server, _rpc = coordinator() + server._ensure_tip_refresh_state() + cancellations: list[str] = [] + + class CurrentToken: + payout_state_generation = 1 + + class Cancellation: + def cancel(self) -> None: + cancellations.append("cancelled") + + class InjectCurrentFanoutLock: + def __enter__(self) -> object: + server._active_tip_refresh = (CurrentToken(), Cancellation()) + return self + + def __exit__(self, *_args: object) -> None: + return None + + # Model a new-generation refresh registering after the cache-state + # increment but before the invalidator reaches the coordinator lock. + server.lock = InjectCurrentFanoutLock() # type: ignore[assignment] + + self.assertEqual(server._advance_payout_state_generation(), 1) + self.assertEqual(cancellations, []) + self.assertFalse(server.tip_refresh_is_pending()) + self.assertFalse(server._tip_refresh_retry.is_set()) + + def test_payout_generation_retry_marks_tip_refresh_pending(self) -> None: + server, _rpc = coordinator() + server._ensure_tip_refresh_state() + + self.assertFalse(server.tip_refresh_is_pending()) + self.assertEqual(server._advance_payout_state_generation(), 1) + + self.assertTrue(server.tip_refresh_is_pending()) + self.assertTrue(server._tip_refresh_retry.is_set()) + + self.assertEqual(server.poll_qbit_tip_template_once(), 0) + self.assertFalse(server.tip_refresh_is_pending()) + + def test_payout_only_advance_bounds_publish_supersession(self) -> None: + server, _rpc = coordinator() + server.payout_reconcile_supersession_retries = 2 + real_publish = server._publish_payout_state_candidate + publish_attempts = 0 + + def supersede_before_publish(candidate: object) -> int | None: + nonlocal publish_attempts + publish_attempts += 1 + server._reserve_payout_state_source( + "external_tip", + tip_hash=f"{publish_attempts + 30:064x}", + ) + return real_publish(candidate) # type: ignore[arg-type] + + server._publish_payout_state_candidate = supersede_before_publish # type: ignore[method-assign] + + with self.assertRaisesRegex( + TemplateRefreshBlocked, + "payout-only invalidation was superseded", + ): + server._advance_payout_state_generation() + + self.assertEqual(publish_attempts, 3) + self.assertEqual(server._payout_state_generation, 0) + self.assertTrue(server._payout_state_publication_blocked) + self.assertTrue(server._payout_state_delivery_gate._delivery_blocked) + self.assertTrue(server.tip_refresh_is_pending()) + + def test_payout_publication_fence_is_not_a_job_build_failure(self) -> None: + server, _rpc = coordinator() + install_fake_bundle_builder(server) + state = client(1) + state.send = lambda _payload: None # type: ignore[method-assign] + server.clients = {state} + server._pool_ready_latched = True + server._reserve_payout_state_source("payout_only") + server._block_payout_state_publication() + + self.assertFalse(server.maybe_send_job(state, clean_jobs=True)) + with self.assertRaisesRegex(TemplateRefreshBlocked, "pending publication"): + server.poll_qbit_tip_template_once() + + self.assertEqual(server.job_build_failure_count, 0) + self.assertEqual(server.tip_refresh_client_counts["failed"], 0) + self.assertEqual(server.tip_refresh_client_counts["skipped"], 1) + + def test_successful_poll_clears_payout_pending_created_during_reconcile(self) -> None: + server, _rpc = coordinator() + server._ensure_tip_refresh_state() + reconcile_entered = threading.Event() + allow_reconcile = threading.Event() + results: list[int] = [] + errors: list[BaseException] = [] + + def reconcile(_tip_hash: str) -> bool: + reconcile_entered.set() + if not allow_reconcile.wait(5): + raise AssertionError("test did not release reconciliation") + return True + + def poll() -> None: + try: + results.append(server.poll_qbit_tip_template_once()) + except BaseException as exc: # pragma: no cover - asserted below + errors.append(exc) + + server.ensure_reorg_reconciled_for_tip = reconcile # type: ignore[method-assign] + server._mark_tip_refresh_pending("seed") + poll_thread = threading.Thread(target=poll) + poll_thread.start() + try: + self.assertTrue(reconcile_entered.wait(5)) + self.assertEqual(server._advance_payout_state_generation(), 1) + finally: + allow_reconcile.set() + poll_thread.join(5) + + self.assertFalse(poll_thread.is_alive()) + self.assertEqual(errors, []) + self.assertEqual(results, [0]) + self.assertEqual(server._payout_state_generation, 1) + self.assertIsNotNone(server.last_successful_template_refresh_monotonic) + self.assertFalse(server.tip_refresh_is_pending()) + + def test_failed_poll_preserves_pending_signal_until_successful_retry(self) -> None: + server, _rpc = coordinator() + server._ensure_tip_refresh_state() + pending_token = server._mark_tip_refresh_pending("seed") + + def fail_reconciliation(_tip_hash: str) -> bool: + server._schedule_tip_refresh_retry() + raise RuntimeError("ledger unavailable") + + server.ensure_reorg_reconciled_for_tip = fail_reconciliation # type: ignore[method-assign] + + with self.assertRaisesRegex( + TemplateRefreshBlocked, + "qbit reorg reconciliation failed", + ): + server.poll_qbit_tip_template_once() + + self.assertTrue(server._tip_refresh_retry.is_set()) + self.assertTrue(server.tip_refresh_is_pending()) + self.assertEqual(server._tip_refresh_pending_token, pending_token) + + # Model blockpoll claiming the immediate wake and completing the retry. + server._tip_refresh_retry.clear() + server.ensure_reorg_reconciled_for_tip = lambda _tip: True # type: ignore[method-assign] + + self.assertEqual(server.poll_qbit_tip_template_once(), 0) + self.assertFalse(server.tip_refresh_is_pending()) + + def test_shutdown_during_reconciliation_stops_poll_without_refresh_failure(self) -> None: + server, _rpc = coordinator() + + def rejected_reconciliation(_tip_hash: str) -> bool: + server.stop_event.set() + raise ShutdownInProgress("PRISM coordinator is shutting down") + + server.ensure_reorg_reconciled_for_tip = ( # type: ignore[method-assign] + rejected_reconciliation + ) + + self.assertEqual(server.poll_qbit_tip_template_once(), 0) + self.assertIsNone( + getattr(server, "last_successful_template_refresh_monotonic", None) + ) + + def test_completed_refresh_cannot_clear_newer_payout_pending(self) -> None: + server, _rpc = coordinator() + snapshot = server.fetch_qbit_tip_template_snapshot() + sequence = server._reserve_tip_observation_sequence() + self.assertTrue( + server.observe_tip_first_seen( + snapshot.bestblockhash, + observation_sequence=sequence, + publish_refresh_observation=True, + ) + ) + with server.lock: + server.tip_template_snapshot = snapshot + completed_generation = server._payout_state_generation + self.assertEqual(server._advance_payout_state_generation(), 1) + newer_token = server._tip_refresh_pending_token + + self.assertFalse( + server._clear_tip_refresh_pending_for_completed_refresh( + snapshot, + sequence, + completed_generation, + ) + ) + self.assertEqual(server._tip_refresh_pending_token, newer_token) + self.assertTrue(server.tip_refresh_is_pending()) + + @staticmethod + def _capture_error( + errors: list[BaseException], + operation: object, + ) -> None: + try: + operation() # type: ignore[operator] + except BaseException as exc: # noqa: BLE001 - test thread handoff + errors.append(exc) + + def test_preview_publication_degrades_when_artifact_preparation_times_out( + self, + ) -> None: + server, _rpc = coordinator() + parent_hash = "ab" * 32 + preview = [ + { + "recipient_id": "miner-a", + "order_key": "miner-a", + "p2mr_program_hex": "11" * 32, + "balance_sats": 25, + } + ] + + def timing_out_preparation(*_args: object, **_kwargs: object) -> object: + raise TimeoutError("postgres operation exceeded 30s") + + server._prepared_payout_state_candidate = ( # type: ignore[method-assign] + timing_out_preparation + ) + server._begin_accepted_block_payout_preview(parent_hash) + with patch("builtins.print"): + self.assertEqual( + server._publish_accepted_block_payout_preview(parent_hash, preview), + preview, + ) + + # Children of the pending parent receive the immutable compact + # preview even though no payout-state generation was published. + transition = server._accepted_block_payout_previews[parent_hash] + self.assertIsNone(transition.published_generation) + self.assertTrue(transition.landed) + self.assertEqual( + server._prior_balances_for_job_parent(parent_hash), + preview, + ) + # Payout-state publication itself stays gated until the landing + # reaches a terminal durable state. + with self.assertRaisesRegex( + _PayoutStatePublicationBlocked, "still pending" + ): + with server._payout_balance_mutation(): + pass + server._clear_accepted_block_payout_preview(parent_hash) + + def test_unresolved_accepted_parent_depth_cap_fails_closed(self) -> None: + server, _rpc = coordinator() + server.accepted_parent_unresolved_depth_max = 2 + server.accepted_block_payout_preview_wait_seconds = 0.01 + hashes = ["a1" * 32, "a2" * 32, "a3" * 32] + for height, block_hash in enumerate(hashes, start=10): + server._begin_accepted_block_payout_preview( + block_hash, block_height=height + ) + server._mark_accepted_block_payout_landed( + block_hash, block_height=height + ) + self.assertEqual(len(server.accepted_parent_unresolved_ages_seconds()), 3) + with self.assertRaisesRegex(TemplateRefreshBlocked, "exceeds cap"): + server._await_pending_parent_payout_preview(hashes[0]) + # At the cap issuance still blocks: a child issued now could land + # and create a (cap + 1)th unresolved transition, so the configured + # maximum would no longer bound the prospective balance chain. + server._clear_accepted_block_payout_preview(hashes[2]) + with self.assertRaisesRegex(TemplateRefreshBlocked, "exceeds cap"): + server._await_pending_parent_payout_preview(hashes[0]) + # Below the cap the ordinary bounded wait applies instead. + server._clear_accepted_block_payout_preview(hashes[1]) + with self.assertRaisesRegex(TemplateRefreshBlocked, "not ready yet"): + server._await_pending_parent_payout_preview(hashes[0]) + server._clear_accepted_block_payout_preview(hashes[0]) + self.assertEqual(server.accepted_parent_unresolved_ages_seconds(), []) + + def test_landing_observability_metrics_exposition(self) -> None: + import contextlib as _contextlib + import time as _time + + server, _rpc = coordinator() + server.block_landing_db_timeout_seconds = 45.0 + server.block_landing_db_timeout_max_seconds = 90.0 + + @_contextlib.contextmanager + def statement_timeout(seconds: float): + yield + + original_ledger = server.ledger + server.ledger = SimpleNamespace( + statement_timeout=statement_timeout, + prior_balances_read_stats=lambda: { + "reads_total": 3, + "last_seconds": 0.004, + "max_seconds": 0.021, + }, + ) + block_hash = "ab" * 32 + with self.assertRaises(TimeoutError): + with server._block_landing_ledger_statement_timeout_scope(block_hash): + raise LedgerOperationTimeout("postgres operation exceeded 45s") + server.ledger = original_ledger + + server._begin_accepted_block_payout_preview(block_hash, block_height=10) + server._mark_accepted_block_payout_landed(block_hash, block_height=10) + server.accepted_block_payout_preview_wait_seconds = 0.0 + with self.assertRaisesRegex(TemplateRefreshBlocked, "not ready yet"): + server._await_pending_parent_payout_preview(block_hash) + server._startup_phase_origin_monotonic = _time.monotonic() + server._record_startup_phase_once("audit_listener_bound") + + prior_stats = { + "reads_total": 3, + "last_seconds": 0.004, + "max_seconds": 0.021, + } + server.ledger.prior_balances_read_stats = ( # type: ignore[attr-defined] + lambda: prior_stats + ) + lines = "\n".join(server.landing_observability_metrics_lines()) + self.assertIn( + 'qbit_prism_block_ledger_call_timeouts_total{call_class="landing"} 1', + lines, + ) + self.assertIn( + 'qbit_prism_block_ledger_call_budget_seconds{call_class="landing"} 45.000000', + lines, + ) + self.assertIn( + "qbit_prism_accepted_parent_unresolved_transitions 1", + lines, + ) + self.assertIn( + "qbit_prism_accepted_parent_preview_wait_timeouts_total 1", + lines, + ) + self.assertNotIn( + "qbit_prism_accepted_parent_unresolved_oldest_seconds -1", + lines, + ) + self.assertIn("qbit_prism_prior_balances_reads_total 3", lines) + self.assertIn( + "qbit_prism_prior_balances_read_max_seconds 0.021000", + lines, + ) + self.assertIn( + 'qbit_prism_startup_phase_seconds{phase="audit_listener_bound"}', + lines, + ) + server._clear_accepted_block_payout_preview(block_hash) + + def test_startup_replay_gate_blocks_job_builds_until_enumeration(self) -> None: + server, _rpc = coordinator() + enumerated: list[int] = [] + + class EnumLedger(FakeLedger): + def pending_block_candidate_rows( + self, *, limit: int = 32 + ) -> list[dict[str, object]]: + enumerated.append(limit) + return [] + + server.ledger = EnumLedger() + server._note_block_replay_enumeration_owed() + with self.assertRaisesRegex(TemplateRefreshBlocked, "has not enumerated"): + server._await_pending_parent_payout_preview("ab" * 32) + self.assertEqual(server.replay_pending_block_candidates(), 0) + self.assertEqual(enumerated, [32]) + self.assertFalse(server._block_replay_enumeration_owed()) + self.assertIsNone(server._await_pending_parent_payout_preview("ab" * 32)) + + def test_owed_enumeration_bypasses_live_candidate_short_circuits(self) -> None: + server, _rpc = coordinator() + enumerated: list[int] = [] + + class EnumLedger(FakeLedger): + def pending_block_candidate_rows( + self, *, limit: int = 32 + ) -> list[dict[str, object]]: + enumerated.append(limit) + return [] + + server.ledger = EnumLedger() + server.block_landing_db_timeout_seconds = 7.5 + server._retry_block_candidate = object() + # Steady state: a retained retry short-circuits the outbox poll. + self.assertEqual(server.replay_pending_block_candidates(), 0) + self.assertEqual(enumerated, []) + # Owed enumeration is correctness-critical and bypasses the + # short-circuits, running with the landing-class budget and + # recording under the landing metrics class. + server._note_block_replay_enumeration_owed() + self.assertEqual(server.replay_pending_block_candidates(), 0) + self.assertEqual(enumerated, [32]) + self.assertFalse(server._block_replay_enumeration_owed()) + metrics = server.block_ledger_call_class_metrics() + self.assertEqual(metrics["landing"]["last_budget_seconds"], 7.5) + self.assertNotIn("fast", metrics) + + def test_worker_reported_deadline_counts_as_landing_timeout(self) -> None: + """A server-side cancellation completes the ledger worker with a + timeout error before the coordinator-side wait expires; the + completion path must still count it for the landing-timeout alert.""" + server, _rpc = coordinator() + + class CancelledLedger(FakeLedger): + def pending_block_candidate_rows( + self, *, limit: int = 32 + ) -> list[dict[str, object]]: + raise LedgerOperationTimeout("postgres statement deadline expired") + + server.ledger = CancelledLedger() + server._note_block_replay_enumeration_owed() + with patch("builtins.print"): + self.assertTrue(server._run_startup_block_candidate_replay()) + self.assertTrue(server._block_replay_enumeration_owed()) + metrics = server.block_ledger_call_class_metrics() + self.assertEqual(metrics["landing"]["calls_total"], 1) + self.assertEqual(metrics["landing"]["timeouts_total"], 1) + self.assertNotIn("fast", metrics) + + def test_startup_enumeration_escalates_past_truncated_batches(self) -> None: + """A full first batch must not end enumeration: rows beyond the batch + window (potentially the active parent) still need their payout + barriers armed before job builds unblock.""" + server, _rpc = coordinator() + requested: list[int] = [] + total_pending = 33 + + class TruncatingLedger(FakeLedger): + def pending_block_candidate_rows( + self, *, limit: int = 32 + ) -> list[dict[str, object]]: + requested.append(limit) + return [ + durable_candidate_row(index) + for index in range(min(limit, total_pending)) + ] + + server.ledger = TruncatingLedger() + server._note_block_replay_enumeration_owed() + self.assertEqual(server.replay_pending_block_candidates(), total_pending) + self.assertEqual(requested, [32, 64]) + self.assertFalse(server._block_replay_enumeration_owed()) + self.assertEqual( + server._block_replay_candidate_queue.qsize(), total_pending + ) + # Every restored row armed its payout barrier before the gate opened. + self.assertEqual( + len(server._accepted_block_payout_previews), total_pending + ) + + def test_startup_enumeration_stays_owed_at_truncation_cap(self) -> None: + """An outbox larger than the enumeration cap keeps job builds blocked + instead of silently declaring enumeration complete.""" + server, _rpc = coordinator() + requested: list[int] = [] + + class EndlessLedger(FakeLedger): + def pending_block_candidate_rows( + self, *, limit: int = 32 + ) -> list[dict[str, object]]: + requested.append(limit) + return [durable_candidate_row(index) for index in range(limit)] + + server.ledger = EndlessLedger() + server._note_block_replay_enumeration_owed() + with patch("builtins.print"): + queued = server.replay_pending_block_candidates() + self.assertEqual(queued, 1024) + self.assertEqual(requested, [32, 64, 128, 256, 512, 1024]) + self.assertTrue(server._block_replay_enumeration_owed()) + with self.assertRaisesRegex(TemplateRefreshBlocked, "has not enumerated"): + server._await_pending_parent_payout_preview("ab" * 32) + + def test_steady_state_poll_restores_a_single_batch(self) -> None: + """Without an owed enumeration, a full batch stays a single query; + the next poll picks up later rows after the queue drains.""" + server, _rpc = coordinator() + requested: list[int] = [] + + class FullBatchLedger(FakeLedger): + def pending_block_candidate_rows( + self, *, limit: int = 32 + ) -> list[dict[str, object]]: + requested.append(limit) + return [durable_candidate_row(index) for index in range(limit)] + + server.ledger = FullBatchLedger() + self.assertEqual(server.replay_pending_block_candidates(), 32) + self.assertEqual(requested, [32]) + self.assertFalse(server._block_replay_enumeration_owed()) + + def test_startup_replay_timeout_keeps_job_builds_blocked(self) -> None: + server, _rpc = coordinator() + release = threading.Event() + + class StallLedger(FakeLedger): + def pending_block_candidate_rows( + self, *, limit: int = 32 + ) -> list[dict[str, object]]: + release.wait(5) + return [] + + server.ledger = StallLedger() + server.block_landing_db_timeout_seconds = 0.01 + server.block_landing_db_timeout_max_seconds = 0.01 + try: + with patch("builtins.print"): + self.assertTrue(server._run_startup_block_candidate_replay()) + self.assertTrue(server._block_replay_enumeration_owed()) + with self.assertRaisesRegex( + TemplateRefreshBlocked, "has not enumerated" + ): + server._await_pending_parent_payout_preview("ab" * 32) + # The startup enumeration ran with the landing budget, so its + # timeout fires the documented landing-timeout alert instead of + # hiding inside the fast-call series. + metrics = server.block_ledger_call_class_metrics() + self.assertEqual(metrics["landing"]["timeouts_total"], 1) + self.assertNotIn("fast", metrics) + finally: + release.set() + + def test_startup_phase_seconds_records_first_occurrence_only(self) -> None: + import time as _time + + server, _rpc = coordinator() + self.assertEqual(server.startup_phase_seconds(), {}) + server._record_startup_phase_once("audit_listener_bound") + self.assertEqual(server.startup_phase_seconds(), {}) + server._startup_phase_origin_monotonic = _time.monotonic() - 1.0 + server._record_startup_phase_once("audit_listener_bound") + first = server.startup_phase_seconds()["audit_listener_bound"] + self.assertGreaterEqual(first, 1.0) + server._record_startup_phase_once("audit_listener_bound") + self.assertEqual( + server.startup_phase_seconds()["audit_listener_bound"], first + ) + + def test_job_issuance_continues_during_padded_landing(self) -> None: + """Acceptance shape for issue #188: a landing padded past its normal + duration must not stop child job issuance, and payout-state + publication stays gated until the landing reaches a terminal state. + The pad is logical (the barrier simply stays unresolved) so the test + is instant while pinning the same ordering a 10s landing produces.""" + server, _rpc = coordinator() + parent_hash = "ab" * 32 + preview = [ + { + "recipient_id": "miner-a", + "order_key": "miner-a", + "p2mr_program_hex": "11" * 32, + "balance_sats": 25, + } + ] + server._begin_accepted_block_payout_preview(parent_hash, block_height=10) + server._mark_accepted_block_payout_landed(parent_hash, block_height=10) + server._publish_accepted_block_payout_preview(parent_hash, preview) + # The landing bookkeeping is still unresolved; every child build keeps + # receiving the immutable preview instead of blocking or reading + # confirmed balances. + for _ in range(5): + self.assertEqual( + server._prior_balances_for_job_parent(parent_hash), + preview, + ) + ages = server.accepted_parent_unresolved_ages_seconds() + self.assertEqual(len(ages), 1) + with self.assertRaisesRegex( + _PayoutStatePublicationBlocked, "still pending" + ): + with server._payout_balance_mutation(): + pass + # Landing completes: the transition clears and confirmed reads resume. + server._clear_accepted_block_payout_preview(parent_hash) + self.assertEqual( + server._prior_balances_for_job_parent( + parent_hash, fallback_balances=[] + ), + [], + ) + self.assertEqual(server.accepted_parent_unresolved_ages_seconds(), []) + + def test_unmark_landed_clears_unresolved_age(self) -> None: + server, _rpc = coordinator() + block_hash = "cd" * 32 + server._begin_accepted_block_payout_preview(block_hash, block_height=10) + self.assertEqual(server.accepted_parent_unresolved_ages_seconds(), []) + server._mark_accepted_block_payout_landed(block_hash, block_height=10) + ages = server.accepted_parent_unresolved_ages_seconds() + self.assertEqual(len(ages), 1) + self.assertGreaterEqual(ages[0], 0.0) + server._unmark_accepted_block_payout_landed(block_hash) + self.assertEqual(server.accepted_parent_unresolved_ages_seconds(), []) + server._clear_accepted_block_payout_preview(block_hash) + + def test_preview_publication_survives_window_build_timeout(self) -> None: + """Issue #188 acceptance: when the payout window build dies with the + slow ledger during found-block publication, the preview must still + cross the atomic publication boundary. Admission then reopens and + child jobs build against the verified compact preview instead of + stalling behind the global publication fence until the ledger + recovers.""" + accepted_hash = "c4" * 32 + server, rpc = coordinator( + template=base_template(height=12, prevhash=accepted_hash) + ) + recorded = install_fake_bundle_builder(server) + artifacts = server.store_template_artifacts(dict(rpc.template)) + assert artifacts is not None + self.assertTrue(server.pool_readiness_latched()) + preview = [ + { + "recipient_id": "miner-a", + "order_key": "miner-a", + "p2mr_program_hex": "11" * 32, + "balance_sats": 25, + } + ] + + def timing_out_window_build( + *_args: object, **_kwargs: object + ) -> PayoutLedgerArtifact: + raise LedgerOperationTimeout("postgres statement deadline expired") + + server._build_payout_ledger_artifact = ( # type: ignore[method-assign] + timing_out_window_build + ) + server._begin_accepted_block_payout_preview(accepted_hash, block_height=11) + server._mark_accepted_block_payout_landed(accepted_hash, block_height=11) + generation_before = server._payout_state_generation + with patch("builtins.print"): + published = server._publish_accepted_block_payout_preview( + accepted_hash, preview + ) + self.assertEqual(published, preview) + # The publication crossed the atomic boundary: a generation was + # installed and delivery admission reopened despite the dead build. + self.assertFalse(server._payout_state_publication_fenced()) + self.assertEqual(server._payout_state_generation, generation_before + 1) + transition = server._accepted_block_payout_previews[accepted_hash] + self.assertEqual(transition.published_generation, generation_before + 1) + # Child job construction is admitted and consumes the preview. + bundle = server.shared_job_bundle(artifacts, mode="ready") + self.assertIsNotNone(bundle) + self.assertEqual(recorded["calls"], 1) + last_kwargs = recorded["last_kwargs"] + assert isinstance(last_kwargs, dict) + self.assertEqual(list(last_kwargs["prior_balances"]), preview) + server._clear_accepted_block_payout_preview(accepted_hash) + + def test_landed_transition_poll_uses_coordination_blocked_budget(self) -> None: + server, _rpc = coordinator() + server.reorg_reconciler_enabled = True + block_hash = "af" * 32 + server._begin_accepted_block_payout_preview(block_hash, block_height=10) + server._mark_accepted_block_payout_landed(block_hash, block_height=10) + + with ( + patch( + "lab.prism.prism_coordinator.time.monotonic", + return_value=100.0, + ), + self.assertRaisesRegex( + _PayoutStatePublicationBlocked, + "confirmation is still pending", + ), + ): + server.poll_qbit_tip_template_once() + + self.assertIsNone( + getattr(server, "template_refresh_failure_started_monotonic", None) + ) + self.assertEqual( + server.coordination_blocked_streak_age_seconds(105.0), + 5.0, + ) + + def test_sync_ledger_snapshot_publishes_reusable_artifact(self) -> None: + server, rpc = coordinator() + install_fake_bundle_builder(server) + first = server.store_template_artifacts(dict(rpc.template)) + assert first is not None + # Pin distinct per-generation anchors near the live clock so the + # armed window stays anchored inside the audit ceiling. + first_anchor_ms = now_ms() - 20 + with server._job_cache_lock: + server._job_build_issued_at_ms[first.generation] = first_anchor_ms + + inline = server.shared_job_bundle(first, mode="ready") + + self.assertEqual(inline.payout_artifact_generation, 0) + self.assertEqual(server.ledger.snapshot_calls, 1) + with server._job_cache_lock: + artifact = server._payout_ledger_artifact + self.assertIsNotNone(artifact) + assert artifact is not None + self.assertGreater(artifact.generation, 0) + self.assertEqual( + artifact.payout_state_generation, + server._payout_state_generation, + ) + self.assertEqual( + artifact.accepted_share_count, + len(server.ledger.miners), + ) + self.assertEqual(list(artifact.shares_json), inline.shares_json) + self.assertEqual(artifact.snapshot_anchor_ms, inline.issued_at_ms) + + second = server.store_template_artifacts( + base_template(height=11, prevhash="22" * 32) + ) + assert second is not None + with server._job_cache_lock: + server._job_build_issued_at_ms[second.generation] = ( + first_anchor_ms + 10 + ) + + reused = server.shared_job_bundle(second, mode="ready") + + self.assertEqual(reused.payout_artifact_generation, artifact.generation) + self.assertEqual(server.ledger.snapshot_calls, 1) + self.assertEqual(reused.shares_json, inline.shares_json) + self.assertNotEqual(reused.issued_at_ms, inline.issued_at_ms) + self.assertEqual( + reused.found_block["anchor_job_issued_at_ms"], + artifact.snapshot_anchor_ms, + ) + + def test_sync_artifact_publication_survives_share_commit_mid_read( + self, + ) -> None: + class MidReadCommitLedger(FakeLedger): + def snapshot_at_job_issue( + self, + anchor_job_issued_at_ms: int, + *, + window_weight: int | None = None, + ) -> list[FakeShare]: + result = super().snapshot_at_job_issue( + anchor_job_issued_at_ms, + window_weight=window_weight, + ) + # A share becomes durable while the window read is in + # flight. The read is scoped by the frozen anchor -- the + # commit lands above it and belongs to the next window -- so + # anchor-scoped publication proceeds. + self.miners = [*self.miners, f"mid-read-{self.snapshot_calls}"] + return result + + server, rpc = coordinator(ledger=MidReadCommitLedger()) + install_fake_bundle_builder(server) + artifacts = server.store_template_artifacts(dict(rpc.template)) + assert artifacts is not None + + bundle = server.shared_job_bundle(artifacts, mode="ready") + + self.assertFalse(bundle.collection_only) + self.assertEqual(bundle.payout_artifact_generation, 0) + with server._job_cache_lock: + artifact = server._payout_ledger_artifact + self.assertIsNotNone(artifact) + assert artifact is not None + self.assertEqual(artifact.snapshot_anchor_ms, bundle.issued_at_ms) + self.assertEqual(list(artifact.shares_json), bundle.shares_json) + + # The armed window keeps serving under continuous commits: the live + # durable count has moved past the artifact's, and reuse must not + # care. + second = server.store_template_artifacts( + base_template(height=11, prevhash="22" * 32) + ) + assert second is not None + reused = server.shared_job_bundle(second, mode="ready") + self.assertEqual(reused.payout_artifact_generation, artifact.generation) + self.assertEqual(server.ledger.snapshot_calls, 1) + self.assertEqual( + reused.found_block["anchor_job_issued_at_ms"], + artifact.snapshot_anchor_ms, + ) + + def test_sync_publication_allowed_when_anchor_predates_durable_share( + self, + ) -> None: + server, rpc = coordinator() + install_fake_bundle_builder(server) + artifacts = server.store_template_artifacts(dict(rpc.template)) + assert artifacts is not None + + first = server.shared_job_bundle(artifacts, mode="ready") + self.assertFalse(first.collection_only) + with server._job_cache_lock: + self.assertIsNotNone(server._payout_ledger_artifact) + # A share becomes durable after this generation's anchor was + # frozen. Force the synchronous path to run again for the same + # frozen anchor. + server._payout_ledger_artifact = None + server._job_bundle_cache.clear() + server.ledger.miners = [*server.ledger.miners, "post-anchor-share"] + + second = server.shared_job_bundle(artifacts, mode="ready") + + self.assertEqual(second.payout_artifact_generation, 0) + # The window read at the frozen anchor deterministically excludes + # the later durable share -- it belongs to the next window -- so the + # anchor-scoped publication proceeds even though the live count has + # moved past the frozen anchor. + with server._job_cache_lock: + republished = server._payout_ledger_artifact + self.assertIsNotNone(republished) + assert republished is not None + self.assertEqual(republished.snapshot_anchor_ms, second.issued_at_ms) + self.assertEqual(second.issued_at_ms, first.issued_at_ms) + + def test_racing_commit_does_not_disable_sync_artifact_seeding(self) -> None: + class RacingStatsLedger(FakeLedger): + def __init__(self) -> None: + super().__init__( + miners=["miner-a", "miner-b", "miner-c", "miner-d"] + ) + self._pending_counts = iter([3, 4]) + + def accepted_share_stats(self) -> dict[str, int]: + self.stats_calls += 1 + try: + count = next(self._pending_counts) + except StopIteration: + count = 4 + return { + "accepted_share_count": count, + "distinct_miner_count": 4, + } + + server, rpc = coordinator(ledger=RacingStatsLedger()) + install_fake_bundle_builder(server) + artifacts = server.store_template_artifacts(dict(rpc.template)) + assert artifacts is not None + + # A commit races the informational count read. The window is scoped + # by the frozen anchor regardless of which side of the read the + # commit landed on, so seeding proceeds; the count is diagnostics, + # not a fence. + bundle = server.shared_job_bundle(artifacts, mode="ready") + + self.assertFalse(bundle.collection_only) + with server._job_cache_lock: + artifact = server._payout_ledger_artifact + self.assertIsNotNone(artifact) + assert artifact is not None + self.assertEqual(artifact.snapshot_anchor_ms, bundle.issued_at_ms) + self.assertEqual(list(artifact.shares_json), bundle.shares_json) + + def test_same_window_background_rebuild_keeps_artifact_generation( + self, + ) -> None: + server, rpc = coordinator() + install_fake_bundle_builder(server) + artifacts = server.store_template_artifacts(dict(rpc.template)) + assert artifacts is not None + + server.shared_job_bundle(artifacts, mode="ready") + with server._job_cache_lock: + installed = server._payout_ledger_artifact + assert installed is not None + + # No share committed since: the speculative rebuild reads the same + # window under a fresh anchor and must not spin the generation, + # which would re-key bundle lookups for nothing -- but it is still a + # successful preparation, so an accumulated re-arm backoff releases + # and the fresher anchor advances the staleness clock in place. + with server._payout_artifact_executor_lock: + server._payout_artifact_rearm_backoff = 4 + server._prepare_payout_ledger_artifact( + server._payout_state_generation, + artifacts.network_difficulty, + ) + with server._job_cache_lock: + refreshed = server._payout_ledger_artifact + assert refreshed is not None + self.assertEqual(refreshed.generation, installed.generation) + self.assertEqual( + refreshed.share_snapshot_sha256, + installed.share_snapshot_sha256, + ) + assert installed.snapshot_anchor_ms is not None + assert refreshed.snapshot_anchor_ms is not None + self.assertGreaterEqual( + int(refreshed.snapshot_anchor_ms), + int(installed.snapshot_anchor_ms), + ) + with server._payout_artifact_executor_lock: + self.assertEqual(server._payout_artifact_rearm_backoff, 1) + + # A new durable share makes the rebuild a genuine replacement. + server.ledger.miners = [*server.ledger.miners, "late-share"] + server._prepare_payout_ledger_artifact( + server._payout_state_generation, + artifacts.network_difficulty, + ) + with server._job_cache_lock: + replaced = server._payout_ledger_artifact + assert replaced is not None + self.assertGreater(replaced.generation, installed.generation) + self.assertEqual(replaced.accepted_share_count, 4) + + def test_delayed_older_snapshot_does_not_regress_artifact(self) -> None: + server, rpc = coordinator() + install_fake_bundle_builder(server) + artifacts = server.store_template_artifacts(dict(rpc.template)) + assert artifacts is not None + server.shared_job_bundle(artifacts, mode="ready") + with server._job_cache_lock: + current = server._payout_ledger_artifact + assert current is not None + assert current.snapshot_anchor_ms is not None + + # A snapshot taken at an earlier anchor finishes its window + # conversion late, so its preparation timestamp is newer than the + # installed artifact's; the anchor still proves it is the older + # window. + older_window = [ + {"share_seq": seq, "miner_id": "miner-a"} for seq in (1, 2) + ] + delayed = PayoutLedgerArtifact( + generation=0, + payout_state_generation=server._payout_state_generation, + network_difficulty=int(current.network_difficulty), + accepted_share_count=2, + shares_json=tuple(older_window), + prior_balances=(), + prepared_monotonic=time.monotonic(), + snapshot_anchor_ms=int(current.snapshot_anchor_ms) - 5, + share_snapshot_sha256=canonical_json_sha256(older_window), + ) + server._install_payout_ledger_artifact(delayed) + with server._job_cache_lock: + self.assertIs(server._payout_ledger_artifact, current) + + # A window snapshotted at a fresher anchor replaces regardless of + # its preparation timestamp ordering. + newer_window = [ + {"share_seq": seq, "miner_id": "miner-a"} for seq in (1, 2, 3, 4) + ] + fresher = dataclass_replace( + delayed, + accepted_share_count=4, + shares_json=tuple(newer_window), + prepared_monotonic=current.prepared_monotonic - 1.0, + snapshot_anchor_ms=int(current.snapshot_anchor_ms) + 5, + share_snapshot_sha256=canonical_json_sha256(newer_window), + ) + server._install_payout_ledger_artifact(fresher) + with server._job_cache_lock: + replaced = server._payout_ledger_artifact + assert replaced is not None + self.assertGreater(replaced.generation, current.generation) + self.assertEqual(replaced.accepted_share_count, 4) + + def test_difficulty_change_replaces_same_window_artifact(self) -> None: + server, rpc = coordinator() + install_fake_bundle_builder(server) + artifacts = server.store_template_artifacts(dict(rpc.template)) + assert artifacts is not None + server.shared_job_bundle(artifacts, mode="ready") + with server._job_cache_lock: + installed = server._payout_ledger_artifact + assert installed is not None + + # A retarget moves the live template to a new difficulty while the + # share window stays identical. The rebuild at the live difficulty + # must replace the artifact: keeping the old-difficulty artifact + # would fail the reuse fence on every build while the same-window + # skip kept discarding its replacement. + retarget_difficulty = int(installed.network_difficulty) + 1 + with server._job_cache_lock: + live = server._template_artifacts + assert live is not None + server._template_artifacts = dataclass_replace( + live, + network_difficulty=retarget_difficulty, + ) + server._prepare_payout_ledger_artifact( + server._payout_state_generation, + retarget_difficulty, + ) + with server._job_cache_lock: + replaced = server._payout_ledger_artifact + assert replaced is not None + self.assertGreater(replaced.generation, installed.generation) + self.assertEqual(replaced.network_difficulty, retarget_difficulty) + + # A pre-retarget build delayed after its snapshot carries the same + # window at the old difficulty; whether its anchor trails or ties, + # the live-difficulty artifact must stay. + delayed = dataclass_replace( + installed, + generation=0, + prepared_monotonic=time.monotonic(), + ) + server._install_payout_ledger_artifact(delayed) + with server._job_cache_lock: + self.assertIs(server._payout_ledger_artifact, replaced) + + # Even a delayed pre-retarget build that clamped a fresher anchor + # must not displace it: a wrong-difficulty install would fail every + # reuse probe on the difficulty check, which never re-arms. + assert replaced.snapshot_anchor_ms is not None + leading = dataclass_replace( + installed, + generation=0, + prepared_monotonic=time.monotonic(), + snapshot_anchor_ms=int(replaced.snapshot_anchor_ms) + 5, + ) + server._install_payout_ledger_artifact(leading) + with server._job_cache_lock: + self.assertIs(server._payout_ledger_artifact, replaced) + + def test_fence_failure_rearms_artifact_preparation_with_debounce(self) -> None: + server, rpc = coordinator() + install_fake_bundle_builder(server) + artifacts = server.store_template_artifacts(dict(rpc.template)) + assert artifacts is not None + server.payout_artifact_rearm_min_seconds = 5.0 + server._prepare_payout_ledger_artifact( + server._payout_state_generation, + artifacts.network_difficulty, + ) + + class RecordingExecutor: + def __init__(self) -> None: + self.submissions = 0 + + def submit(self, _fn: object) -> Future[None]: + # Recorded but never run: the request slot stays observable. + self.submissions += 1 + return Future() + + executor = RecordingExecutor() + with server._payout_artifact_executor_lock: + server._payout_artifact_executor = executor # type: ignore[assignment] + + usable = server._usable_payout_ledger_artifact( + server._payout_state_generation, + artifacts.network_difficulty, + ) + self.assertIsNotNone(usable) + with server._payout_artifact_executor_lock: + self.assertIsNone(server._payout_artifact_requested) + + # Age the armed window past the re-anchor floor: the probe keeps + # serving the artifact while the debounced background re-anchor is + # scheduled. + server.payout_artifact_reanchor_seconds = 10.0 + with server._job_cache_lock: + armed = server._payout_ledger_artifact + assert armed is not None + assert armed.snapshot_anchor_ms is not None + server._payout_ledger_artifact = dataclass_replace( + armed, + snapshot_anchor_ms=int(armed.snapshot_anchor_ms) - 11_000, + ) + + self.assertIsNotNone( + server._usable_payout_ledger_artifact( + server._payout_state_generation, + artifacts.network_difficulty, + ) + ) + with server._payout_artifact_executor_lock: + self.assertEqual( + server._payout_artifact_requested, + ( + server._payout_state_generation, + int(artifacts.network_difficulty), + ), + ) + self.assertEqual(executor.submissions, 1) + # Every past-floor probe counts, even when the rearm itself is + # debounced -- the canary needs "serving past the floor" to be + # visible independently of rearm_scheduled. + self.assertEqual( + server.payout_artifact_event_counts["probe_past_floor"], + 1, + ) + # The one-slot worker dequeues the request; a served probe inside + # the interval must not slip a duplicate into the emptied slot. + server._payout_artifact_requested = None + + self.assertIsNotNone( + server._usable_payout_ledger_artifact( + server._payout_state_generation, + artifacts.network_difficulty, + ) + ) + with server._payout_artifact_executor_lock: + self.assertIsNone(server._payout_artifact_requested) + + # Once the interval has elapsed the next served probe re-arms. + with server._payout_artifact_executor_lock: + server._payout_artifact_last_schedule_monotonic = ( + time.monotonic() - 6.0 + ) + self.assertIsNotNone( + server._usable_payout_ledger_artifact( + server._payout_state_generation, + artifacts.network_difficulty, + ) + ) + with server._payout_artifact_executor_lock: + self.assertEqual( + server._payout_artifact_requested, + ( + server._payout_state_generation, + int(artifacts.network_difficulty), + ), + ) + + def test_aborted_speculative_rebuilds_back_off_and_reset(self) -> None: + ledger = AnchorRecordingLedger() + server, rpc = coordinator(ledger=ledger) + install_fake_bundle_builder(server) + artifacts = server.store_template_artifacts(dict(rpc.template)) + assert artifacts is not None + server.payout_artifact_rearm_min_seconds = 5.0 + + # A pending-commit floor older than the audit anchor ceiling aborts + # the rebuild before the window walk (the artifact would arm born + # expired); each abort doubles the re-arm interval instead of + # retrying the reward-window walk at the floor forever. + share = stamped_pending_share(now_ms() - 400_000) + server._ensure_pending_share_commit_state() + with server._pending_share_commit_lock: + server._pending_share_commit_floor[id(share)] = [ + share, + time.monotonic(), + False, + ] + server._prepare_payout_ledger_artifact(0, artifacts.network_difficulty) + with server._payout_artifact_executor_lock: + self.assertEqual(server._payout_artifact_rearm_backoff, 2) + server._prepare_payout_ledger_artifact(0, artifacts.network_difficulty) + with server._payout_artifact_executor_lock: + self.assertEqual(server._payout_artifact_rearm_backoff, 4) + self.assertEqual(ledger.snapshot_calls, 0) + + # Elapsed time beyond the floor but inside the scaled interval must + # not re-arm. + with server._payout_artifact_executor_lock: + server._payout_artifact_last_schedule_monotonic = ( + time.monotonic() - 10.0 + ) + server._rearm_payout_ledger_artifact_after_fence_failure( + 0, + artifacts.network_difficulty, + ) + with server._payout_artifact_executor_lock: + self.assertIsNone(server._payout_artifact_requested) + + # A rebuild that finally arms resets the backoff to the floor. + server._finish_pending_share_commit(share) + server._prepare_payout_ledger_artifact(0, artifacts.network_difficulty) + with server._job_cache_lock: + self.assertIsNotNone(server._payout_ledger_artifact) + with server._payout_artifact_executor_lock: + self.assertEqual(server._payout_artifact_rearm_backoff, 1) + + def test_payout_publication_resets_rearm_backoff(self) -> None: + server, rpc = coordinator() + install_fake_bundle_builder(server) + artifacts = server.store_template_artifacts(dict(rpc.template)) + assert artifacts is not None + server._pool_ready_latched = True + with server._payout_artifact_executor_lock: + server._payout_artifact_rearm_backoff = 8 + + # The accepted-block preview publishes a new payout generation whose + # candidate artifact installs through the atomic publication pointer + # swap; the accumulated backoff must release with it. + parent_hash = str(rpc.template["previousblockhash"]) + server._begin_accepted_block_payout_preview( + parent_hash, + block_height=int(rpc.template["height"]) - 1, + ) + server._publish_accepted_block_payout_preview( + parent_hash, + [ + { + "recipient_id": "miner-a", + "order_key": "miner-a", + "p2mr_program_hex": "11" * 32, + "balance_sats": 25, + } + ], + ) + + self.assertGreater(server._payout_state_generation, 0) + with server._job_cache_lock: + self.assertIsNotNone(server._payout_ledger_artifact) + with server._payout_artifact_executor_lock: + self.assertEqual(server._payout_artifact_rearm_backoff, 1) + + def test_landed_preview_suppresses_fence_failure_rearm(self) -> None: + server, rpc = coordinator() + install_fake_bundle_builder(server) + artifacts = server.store_template_artifacts(dict(rpc.template)) + assert artifacts is not None + server._prepare_payout_ledger_artifact( + server._payout_state_generation, + artifacts.network_difficulty, + ) + scheduled: list[tuple[int, int]] = [] + server._schedule_payout_ledger_artifact_preparation = ( # type: ignore[method-assign] + lambda generation, difficulty, **_kwargs: scheduled.append( + (generation, difficulty) + ) + ) + accepted_hash = "d0" * 32 + server._begin_accepted_block_payout_preview( + accepted_hash, + block_height=int(rpc.template["height"]) - 1, + ) + server._mark_accepted_block_payout_landed( + accepted_hash, + block_height=int(rpc.template["height"]) - 1, + ) + + # Age the armed window past the re-anchor floor: the probe keeps + # serving while attempting the background re-anchor. + server.payout_artifact_reanchor_seconds = 10.0 + with server._job_cache_lock: + armed = server._payout_ledger_artifact + assert armed is not None + assert armed.snapshot_anchor_ms is not None + server._payout_ledger_artifact = dataclass_replace( + armed, + snapshot_anchor_ms=int(armed.snapshot_anchor_ms) - 11_000, + ) + + self.assertIsNotNone( + server._usable_payout_ledger_artifact( + server._payout_state_generation, + artifacts.network_difficulty, + ) + ) + # A speculative rebuild would read database balances the landed + # prospective state supersedes; preparation resumes only through the + # durable-confirmation call site. + self.assertEqual(scheduled, []) + + def test_artifact_reuse_serves_past_install_age_to_anchor_ceiling( + self, + ) -> None: + server, rpc = coordinator() + install_fake_bundle_builder(server) + artifacts = server.store_template_artifacts(dict(rpc.template)) + assert artifacts is not None + server._prepare_payout_ledger_artifact( + server._payout_state_generation, + artifacts.network_difficulty, + ) + with server._job_cache_lock: + armed = server._payout_ledger_artifact + assert armed is not None + assert armed.snapshot_anchor_ms is not None + # The balances digest is memoized at construction so the serving + # probe and the reused-build fences never re-canonicalize the + # tuple on the request path. + self.assertEqual( + armed.prior_balances_sha256, + canonical_json_sha256(armed.prior_balances), + ) + + # A window whose anchor is past the re-anchor floor still serves -- + # post-anchor shares belong to the next window by construction, and + # validity is carried by the generation/balances fences, never by + # wall-clock age below the ceiling. (The re-anchor and + # anchor-ceiling env knobs wire these same attributes in __init__.) + server.payout_artifact_reanchor_seconds = 10.0 + server.ledger.miners = [*server.ledger.miners, "late-1", "late-2"] + with server._job_cache_lock: + server._payout_ledger_artifact = dataclass_replace( + armed, + snapshot_anchor_ms=int(armed.snapshot_anchor_ms) - 15_000, + ) + self.assertIsNotNone( + server._usable_payout_ledger_artifact( + server._payout_state_generation, + artifacts.network_difficulty, + rearm_on_fence_failure=False, + ) + ) + + # 2026-07-30 brownout regression: install age must never retire the + # window. A 10s install budget at the production build cadence (a + # shared build every 10-14s, a 4-8s walk) expired between + # consecutive builds, sending every build back through the + # synchronous reward-window walk. Only payout events and the audit + # ceiling end reuse. + with server._job_cache_lock: + server._payout_ledger_artifact = dataclass_replace( + armed, + prepared_monotonic=float(armed.prepared_monotonic) - 3600.0, + ) + self.assertIsNotNone( + server._usable_payout_ledger_artifact( + server._payout_state_generation, + artifacts.network_difficulty, + rearm_on_fence_failure=False, + ) + ) + + # The wall-clock anchor ceiling stays as the audit-facing backstop. + server.payout_artifact_max_anchor_age_seconds = 5.0 + with server._job_cache_lock: + server._payout_ledger_artifact = dataclass_replace( + armed, + snapshot_anchor_ms=int(armed.snapshot_anchor_ms) - 6_000, + ) + self.assertIsNone( + server._usable_payout_ledger_artifact( + server._payout_state_generation, + artifacts.network_difficulty, + rearm_on_fence_failure=False, + ) + ) + + def test_reuse_kill_switch_disables_probe_and_background_walks( + self, + ) -> None: + server, rpc = coordinator() + install_fake_bundle_builder(server) + artifacts = server.store_template_artifacts(dict(rpc.template)) + assert artifacts is not None + server._prepare_payout_ledger_artifact( + server._payout_state_generation, + artifacts.network_difficulty, + ) + with server._job_cache_lock: + assert server._payout_ledger_artifact is not None + + # Off: the probe refuses a perfectly valid armed artifact and the + # scheduler runs no background reward-window walks -- the disabled + # state restores pre-reuse delivery economics (the synchronous + # path keeps the re-landed anchor-selection semantics), so a + # delivery regression is an env flip instead of a rollback. + server.payout_artifact_reuse_enabled = False + self.assertIsNone( + server._usable_payout_ledger_artifact( + server._payout_state_generation, + artifacts.network_difficulty, + ) + ) + server._schedule_payout_ledger_artifact_preparation( + server._payout_state_generation, + artifacts.network_difficulty, + ) + with server._payout_artifact_executor_lock: + self.assertIsNone(server._payout_artifact_requested) + self.assertIsNone(server._payout_artifact_future) + + # Back on: the armed artifact serves again without a rebuild. + server.payout_artifact_reuse_enabled = True + self.assertIsNotNone( + server._usable_payout_ledger_artifact( + server._payout_state_generation, + artifacts.network_difficulty, + ) + ) + + # Off also gates arming itself: a prepared window is discarded at + # install, so nothing churns install events or re-keys the idle + # bundle fast path to an unusable generation while disabled. + server.payout_artifact_reuse_enabled = False + with server._job_cache_lock: + server._payout_ledger_artifact = None + server._prepare_payout_ledger_artifact( + server._payout_state_generation, + artifacts.network_difficulty, + ) + with server._job_cache_lock: + self.assertIsNone(server._payout_ledger_artifact) + + def test_anchor_ceiling_must_clear_reanchor_floor(self) -> None: + # One env var used to be able to re-create the 2026-07-30 incident: + # a ceiling the background re-anchor cannot beat (floor + debounce + + # the window walk) rejects every probe while the rebuild runs + # debounced in parallel -- walk-per-build. __init__ refuses the + # configuration at startup through this validation. + with self.assertRaises(SystemExit): + prism_coordinator_module.validate_payout_artifact_age_bounds( + 60.0, + 60.0, + ) + with self.assertRaises(SystemExit): + prism_coordinator_module.validate_payout_artifact_age_bounds( + 120.0, + 60.0, + ) + # The same margin applies to a custom build cadence; validating only + # the re-anchor floor allowed 299s builds under a 300s ceiling. + with self.assertRaises(SystemExit): + prism_coordinator_module.validate_payout_artifact_age_bounds( + 300.0, + 299.0, + ) + # The default split (300s ceiling, 60s floor) passes. + prism_coordinator_module.validate_payout_artifact_age_bounds( + 300.0, + 60.0, + ) + + def test_slow_window_walk_still_arms_reusable_artifact(self) -> None: + # 2026-07-29 regression: the reward-window walk takes multiple + # seconds at production volume, and freshness measured from the + # wall-clock anchor rejected every slow build on arrival. The walk + # here advances the wall clock past the freshness budget; the + # artifact must still arm, serve reuse, and reset the re-arm + # backoff. + clock = [now_ms()] + + class SlowWalkLedger(FakeLedger): + def snapshot_at_job_issue( + self, + anchor_job_issued_at_ms: int, + *, + window_weight: int | None = None, + ) -> list[FakeShare]: + result = super().snapshot_at_job_issue( + anchor_job_issued_at_ms, + window_weight=window_weight, + ) + clock[0] += 15_000 + return result + + server, rpc = coordinator(ledger=SlowWalkLedger()) + install_fake_bundle_builder(server) + artifacts = server.store_template_artifacts(dict(rpc.template)) + assert artifacts is not None + with server._payout_artifact_executor_lock: + server._payout_artifact_rearm_backoff = 4 + with patch( + "lab.prism.prism_coordinator.now_ms", + side_effect=lambda: clock[0], + ): + server._prepare_payout_ledger_artifact( + server._payout_state_generation, + artifacts.network_difficulty, + ) + with server._job_cache_lock: + armed = server._payout_ledger_artifact + self.assertIsNotNone(armed) + assert armed is not None + assert armed.snapshot_anchor_ms is not None + self.assertGreater( + clock[0] - int(armed.snapshot_anchor_ms), + 10_000, + ) + self.assertIs( + server._usable_payout_ledger_artifact( + server._payout_state_generation, + artifacts.network_difficulty, + ), + armed, + ) + with server._payout_artifact_executor_lock: + self.assertEqual(server._payout_artifact_rearm_backoff, 1) + metrics = server.metrics_payload() + self.assertIn( + 'qbit_prism_payout_artifact_events_total{event="installed"} 1', + metrics, + ) + self.assertIn( + 'qbit_prism_payout_artifact_events_total{event="built"} 1', + metrics, + ) + self.assertIn( + 'qbit_prism_payout_artifact_events_total{event="incremental"}', + metrics, + ) + self.assertIn( + 'qbit_prism_payout_artifact_events_total{event="self_check_failed"}', + metrics, + ) + self.assertIn( + 'qbit_prism_payout_artifact_events_total{event="found_block_cached"}', + metrics, + ) + + def test_born_expired_install_paces_backoff_instead_of_resetting( + self, + ) -> None: + # A walk that outlives the audit ceiling arms nothing. Crediting it + # as a success reset the re-arm backoff to the floor and re-walked + # the reward window continuously -- the 2026-07-29 livelock. It + # must pace exactly like a failed preparation. + clock = [now_ms()] + + class GlacialWalkLedger(FakeLedger): + def snapshot_at_job_issue( + self, + anchor_job_issued_at_ms: int, + *, + window_weight: int | None = None, + ) -> list[FakeShare]: + result = super().snapshot_at_job_issue( + anchor_job_issued_at_ms, + window_weight=window_weight, + ) + clock[0] += 400_000 + return result + + server, rpc = coordinator(ledger=GlacialWalkLedger()) + install_fake_bundle_builder(server) + artifacts = server.store_template_artifacts(dict(rpc.template)) + assert artifacts is not None + with patch( + "lab.prism.prism_coordinator.now_ms", + side_effect=lambda: clock[0], + ): + server._prepare_payout_ledger_artifact( + server._payout_state_generation, + artifacts.network_difficulty, + ) + with server._job_cache_lock: + self.assertIsNone(server._payout_ledger_artifact) + with server._payout_artifact_executor_lock: + self.assertEqual(server._payout_artifact_rearm_backoff, 2) + server._prepare_payout_ledger_artifact( + server._payout_state_generation, + artifacts.network_difficulty, + ) + with server._payout_artifact_executor_lock: + self.assertEqual(server._payout_artifact_rearm_backoff, 4) + with server._payout_artifact_executor_lock: + self.assertEqual( + server.payout_artifact_event_counts["born_expired"], + 2, + ) + + def test_equal_anchor_reprove_credits_freshness(self) -> None: + # A pending-commit floor pins the snapshot anchor, so a fence-failure + # rebuild re-proves the same window at the SAME anchor. That re-prove + # must advance the freshness clock: no fresher window is + # constructible while the floor holds, and rejecting the credit + # would re-walk the reward window (with backoff reset each round) + # until the audit ceiling -- the livelock shape this re-land + # removes. + server, rpc = coordinator() + install_fake_bundle_builder(server) + artifacts = server.store_template_artifacts(dict(rpc.template)) + assert artifacts is not None + share = stamped_pending_share(now_ms() - 5) + server._ensure_pending_share_commit_state() + with server._pending_share_commit_lock: + server._pending_share_commit_floor[id(share)] = [ + share, + time.monotonic(), + False, + ] + try: + server._prepare_payout_ledger_artifact( + server._payout_state_generation, + artifacts.network_difficulty, + ) + with server._job_cache_lock: + armed = server._payout_ledger_artifact + assert armed is not None + server._payout_ledger_artifact = dataclass_replace( + armed, + prepared_monotonic=float(armed.prepared_monotonic) - 11.0, + ) + # Install age no longer gates the probe (2026-07-30 brownout + # regression); the armed window keeps serving while pinned. + self.assertIsNotNone( + server._usable_payout_ledger_artifact( + server._payout_state_generation, + artifacts.network_difficulty, + rearm_on_fence_failure=False, + ) + ) + + server._prepare_payout_ledger_artifact( + server._payout_state_generation, + artifacts.network_difficulty, + ) + with server._job_cache_lock: + reproved = server._payout_ledger_artifact + assert reproved is not None + self.assertEqual(reproved.generation, armed.generation) + self.assertEqual( + reproved.snapshot_anchor_ms, + armed.snapshot_anchor_ms, + ) + self.assertIsNotNone( + server._usable_payout_ledger_artifact( + server._payout_state_generation, + artifacts.network_difficulty, + ) + ) + finally: + server._finish_pending_share_commit(share) + + def test_publication_restamps_candidate_artifact_freshness(self) -> None: + # A payout-state candidate builds its artifact before the atomic + # publication, and the delivery-gate drain between the two can + # outlive the reuse budget. The install must restamp freshness so a + # freshly published generation never arms an already-stale artifact. + server, rpc = coordinator() + install_fake_bundle_builder(server) + artifacts = server.store_template_artifacts(dict(rpc.template)) + assert artifacts is not None + server._pool_ready_latched = True + real_build = server._build_payout_ledger_artifact + + def slow_publication_build(*args: object) -> object: + built = real_build(*args) + if built is None: + return None + return dataclass_replace( + built, + prepared_monotonic=time.monotonic() - 11.0, + ) + + server._build_payout_ledger_artifact = slow_publication_build # type: ignore[method-assign] + parent_hash = str(rpc.template["previousblockhash"]) + server._begin_accepted_block_payout_preview( + parent_hash, + block_height=int(rpc.template["height"]) - 1, + ) + server._publish_accepted_block_payout_preview( + parent_hash, + [ + { + "recipient_id": "miner-a", + "order_key": "miner-a", + "p2mr_program_hex": "11" * 32, + "balance_sats": 25, + } + ], + ) + + self.assertGreater(server._payout_state_generation, 0) + usable = server._usable_payout_ledger_artifact( + server._payout_state_generation, + artifacts.network_difficulty, + ) + self.assertIsNotNone(usable) + + def test_usable_probe_survives_equal_window_freshness_restamp( + self, + ) -> None: + # An equal-window re-prove restamps the armed artifact by replacing + # the object while a reuse probe is hashing balances outside the + # cache lock. The probe must treat the restamped copy as the same + # armed window instead of failing closed into a synchronous + # reward-window walk: under a pinned pending-commit floor the + # restamp IS the intentional recovery path, so the race is routine. + server, rpc = coordinator() + install_fake_bundle_builder(server) + artifacts = server.store_template_artifacts(dict(rpc.template)) + assert artifacts is not None + server._prepare_payout_ledger_artifact( + server._payout_state_generation, + artifacts.network_difficulty, + ) + with server._job_cache_lock: + armed = server._payout_ledger_artifact + assert armed is not None + # Strip the memoized balances digest: the race window this test + # drives opens while the probe hashes balances outside the cache + # lock, which only happens on the fallback path for artifacts + # without a memo (legacy or test-constructed). The restamp + # admission below must keep protecting that path. + armed = dataclass_replace(armed, prior_balances_sha256=None) + server._payout_ledger_artifact = armed + + real_sha256 = prism_coordinator_module.canonical_json_sha256 + restamped_once = [False] + + def restamp_during_hash(value: object) -> str: + if not restamped_once[0]: + restamped_once[0] = True + self.assertTrue( + server._install_payout_ledger_artifact( + dataclass_replace( + armed, + generation=0, + prepared_monotonic=time.monotonic(), + ) + ) + ) + return real_sha256(value) + + with patch.object( + prism_coordinator_module, + "canonical_json_sha256", + restamp_during_hash, + ): + usable = server._usable_payout_ledger_artifact( + server._payout_state_generation, + artifacts.network_difficulty, + ) + self.assertTrue(restamped_once[0]) + with server._job_cache_lock: + restamped = server._payout_ledger_artifact + assert restamped is not None + self.assertIsNot(restamped, armed) + self.assertIsNotNone(usable) + self.assertIs(usable, restamped) + self.assertEqual(restamped.generation, armed.generation) + + def test_publication_discards_born_expired_candidate_artifact( + self, + ) -> None: + # Candidate construction plus the delivery-gate drain can push the + # candidate artifact's declared anchor past the audit ceiling. The + # atomic publication must apply the same born-expired admission rule + # as _install_payout_ledger_artifact: arming the artifact would fail + # every reuse probe on anchor age -- with the re-arm suppressed + # while the accepted preview awaits durability -- instead of letting + # the post-publication probe schedule recovery. + server, rpc = coordinator() + install_fake_bundle_builder(server) + artifacts = server.store_template_artifacts(dict(rpc.template)) + assert artifacts is not None + server._pool_ready_latched = True + real_build = server._build_payout_ledger_artifact + ceiling_ms = server._payout_artifact_max_anchor_age_ms() + + def expired_anchor_build(*args: object) -> object: + built = real_build(*args) + if built is None: + return None + assert built.snapshot_anchor_ms is not None + return dataclass_replace( + built, + snapshot_anchor_ms=int(built.snapshot_anchor_ms) + - int(ceiling_ms) + - 1_000, + ) + + server._build_payout_ledger_artifact = expired_anchor_build # type: ignore[method-assign] + parent_hash = str(rpc.template["previousblockhash"]) + server._begin_accepted_block_payout_preview( + parent_hash, + block_height=int(rpc.template["height"]) - 1, + ) + generation_before = server._payout_state_generation + server._publish_accepted_block_payout_preview( + parent_hash, + [ + { + "recipient_id": "miner-a", + "order_key": "miner-a", + "p2mr_program_hex": "11" * 32, + "balance_sats": 25, + } + ], + ) + self.assertGreater(server._payout_state_generation, generation_before) + with server._job_cache_lock: + self.assertIsNone(server._payout_ledger_artifact) + with server._payout_artifact_executor_lock: + born_expired = int( + server.payout_artifact_event_counts.get("born_expired", 0) + ) + self.assertGreaterEqual(born_expired, 1) + self.assertIsNone( + server._usable_payout_ledger_artifact( + server._payout_state_generation, + artifacts.network_difficulty, + ) + ) + + def test_publication_records_installed_event(self) -> None: + # The atomic publication arms the candidate artifact through its own + # pointer swap, not _install_payout_ledger_artifact. It must still + # count in the installed lifecycle event family, or every + # publication-path install is invisible to the observability the + # event counter exists for. + server, rpc = coordinator() + install_fake_bundle_builder(server) + artifacts = server.store_template_artifacts(dict(rpc.template)) + assert artifacts is not None + server._pool_ready_latched = True + with server._payout_artifact_executor_lock: + installed_before = int( + server.payout_artifact_event_counts.get("installed", 0) + ) + parent_hash = str(rpc.template["previousblockhash"]) + server._begin_accepted_block_payout_preview( + parent_hash, + block_height=int(rpc.template["height"]) - 1, + ) + server._publish_accepted_block_payout_preview( + parent_hash, + [ + { + "recipient_id": "miner-a", + "order_key": "miner-a", + "p2mr_program_hex": "11" * 32, + "balance_sats": 25, + } + ], + ) + with server._job_cache_lock: + self.assertIsNotNone(server._payout_ledger_artifact) + with server._payout_artifact_executor_lock: + installed_after = int( + server.payout_artifact_event_counts.get("installed", 0) + ) + self.assertEqual(installed_after, installed_before + 1) + + def test_reused_anchor_bundle_preview_matches_fresh_build_through_guard( + self, + ) -> None: + # The #50 payout-preview guard compares the preview computed from + # the issued job against one recomputed at landing. A reused-anchor + # bundle must therefore reproduce, byte for byte, what a fresh + # ledger read at the same anchor produces -- the audit + # reproducibility contract the artifact's snapshot_anchor_ms + # declaration documents. + programs = { + "miner-a": "aa" * 32, + "miner-b": "bb" * 32, + "miner-c": "cc" * 32, + "miner-d": "dd" * 32, + } + + class AnchorScopedLedger(FakeLedger): + """Window reads respect the anchor like the real ledger.""" + + def __init__(self) -> None: + super().__init__(miners=[]) + self.stamped: list[tuple[int, FakeShare]] = [] + + def add_share(self, miner_id: str, accepted_at_ms: int) -> None: + self.stamped.append( + ( + int(accepted_at_ms), + FakeShare( + miner_id=miner_id, + share_seq=len(self.stamped) + 1, + ), + ) + ) + + def accepted_share_stats(self) -> dict[str, int]: + self.stats_calls += 1 + return { + "accepted_share_count": len(self.stamped), + "distinct_miner_count": len( + {share.miner_id for _, share in self.stamped} + ), + } + + def snapshot_at_job_issue( + self, + anchor_job_issued_at_ms: int, + *, + window_weight: int | None = None, + ) -> list[FakeShare]: + self.snapshot_calls += 1 + return [ + share + for stamp, share in self.stamped + if stamp <= int(anchor_job_issued_at_ms) + ] + + anchor_ms = now_ms() - 50 + ledger = AnchorScopedLedger() + ledger.add_share("miner-a", anchor_ms - 30) + ledger.add_share("miner-b", anchor_ms - 20) + ledger.add_share("miner-c", anchor_ms - 10) + server, rpc = coordinator(ledger=ledger) + + def preview_bundle_builder(**kwargs: object) -> dict[str, object]: + suffix_hex = str(kwargs["coinbase_script_sig_suffix_hex"]) + weights: dict[str, int] = {} + for share in kwargs["shares"]: # type: ignore[union-attr] + miner = str(share["miner_id"]) # type: ignore[index] + weights[miner] = weights.get(miner, 0) + 1 + return { + "found_block": dict(kwargs["found_block"]), # type: ignore[call-overload] + "payout_policy_manifest": { + "accounts": [ + { + "account_type": "miner", + "recipient_id": miner, + "order_key": miner, + "p2mr_program_hex": programs[miner], + "carry_forward_balance_sats": 1_000 * weight, + } + for miner, weight in sorted(weights.items()) + ] + }, + "signed_coinbase_manifest": { + "manifest": { + "coinbase_tx_hex": synthetic_manifest_coinbase_hex( + suffix_hex + ), + } + }, + } + + server.build_audit_bundle = preview_bundle_builder # type: ignore[method-assign] + + first = server.store_template_artifacts(dict(rpc.template)) + assert first is not None + with server._job_cache_lock: + server._job_build_issued_at_ms[first.generation] = anchor_ms + + seeded_bundle = server.shared_job_bundle(first, mode="ready") + self.assertEqual(seeded_bundle.payout_artifact_generation, 0) + with server._job_cache_lock: + artifact = server._payout_ledger_artifact + assert artifact is not None + self.assertEqual(artifact.snapshot_anchor_ms, anchor_ms) + + # Shares keep landing after the anchor; the durable count moves past + # the artifact's, which must not matter for reuse. + ledger.add_share("miner-a", anchor_ms + 20) + ledger.add_share("miner-d", anchor_ms + 25) + + second = server.store_template_artifacts( + base_template(height=11, prevhash="22" * 32) + ) + assert second is not None + reused = server.shared_job_bundle(second, mode="ready") + self.assertEqual(reused.payout_artifact_generation, artifact.generation) + self.assertEqual( + reused.found_block["anchor_job_issued_at_ms"], + anchor_ms, + ) + + # Control: a fresh synchronous build pinned to the same anchor pays + # its own window read and must reproduce the reused output exactly, + # excluding the post-anchor shares deterministically. + with server._job_cache_lock: + server._payout_ledger_artifact = None + server._job_bundle_cache.clear() + third = server.store_template_artifacts( + base_template(height=12, prevhash="33" * 32) + ) + assert third is not None + with server._job_cache_lock: + server._job_build_issued_at_ms[third.generation] = anchor_ms + snapshot_calls_before = ledger.snapshot_calls + fresh = server.shared_job_bundle(third, mode="ready") + self.assertEqual(ledger.snapshot_calls, snapshot_calls_before + 1) + self.assertEqual(fresh.payout_artifact_generation, 0) + + self.assertEqual(reused.shares_json, fresh.shares_json) + self.assertEqual( + fresh.found_block["anchor_job_issued_at_ms"], + anchor_ms, + ) + assert reused.prospective_prior_balances is not None + assert fresh.prospective_prior_balances is not None + self.assertEqual( + canonical_json_text(list(reused.prospective_prior_balances)), + canonical_json_text(list(fresh.prospective_prior_balances)), + ) + + # A window at a fresher anchor covers the post-anchor shares and + # produces a different preview; built now to prove the guard + # equality below is load-bearing. + with server._job_cache_lock: + server._payout_ledger_artifact = None + server._job_bundle_cache.clear() + fourth = server.store_template_artifacts( + base_template(height=13, prevhash="44" * 32) + ) + assert fourth is not None + with server._job_cache_lock: + server._job_build_issued_at_ms[fourth.generation] = anchor_ms + 30 + divergent = server.shared_job_bundle(fourth, mode="ready") + assert divergent.prospective_prior_balances is not None + self.assertNotEqual( + canonical_json_text(list(divergent.prospective_prior_balances)), + canonical_json_text(list(reused.prospective_prior_balances)), + ) + + # The #50 guard sequence at landing: the issued preview (from the + # reused-anchor job) publishes first; the verified preview + # recomputed at the same anchor must then publish idempotently. + block_hash = "d1" * 32 + server._begin_accepted_block_payout_preview( + block_hash, + block_height=int(rpc.template["height"]), + ) + issued = server._materialize_prior_balance_preview( + reused.prospective_prior_balances + ) + server._publish_accepted_block_payout_preview(block_hash, issued) + generation_after_issued = server._payout_state_generation + verified = server._materialize_prior_balance_preview( + fresh.prospective_prior_balances + ) + server._publish_accepted_block_payout_preview(block_hash, verified) + self.assertEqual( + server._payout_state_generation, + generation_after_issued, + ) + with server._accepted_block_payout_preview_condition: + transition = server._accepted_block_payout_previews[block_hash] + self.assertEqual( + transition.preview, + server._serialize_prior_balance_preview(issued), + ) + + # A preview from any other anchor's window trips the guard exactly + # as submit_block_candidate's landing rebuild would. + with self.assertRaisesRegex(RuntimeError, "changed during retry"): + server._publish_accepted_block_payout_preview( + block_hash, + server._materialize_prior_balance_preview( + divergent.prospective_prior_balances + ), + ) + + +class IncrementalPayoutArtifactTests(unittest.TestCase): + def configured_server( + self, + ) -> tuple[PrismCoordinator, IncrementalRecordingLedger, object]: + ledger = IncrementalRecordingLedger() + append_incremental_share(ledger, share_seq=1, accepted_at_ms=999_900) + append_incremental_share(ledger, share_seq=2, accepted_at_ms=999_910) + append_incremental_share(ledger, share_seq=3, accepted_at_ms=999_920) + server, rpc = coordinator(ledger=ledger) + artifacts = server.store_template_artifacts(dict(rpc.template)) + assert artifacts is not None + server._pool_ready_latched = True + server.payout_artifact_min_build_interval_seconds = 60.0 + server.payout_artifact_full_rescan_seconds = 3_600.0 + return server, ledger, artifacts + + @staticmethod + def replay_shaped_append_entry( + *, + accepted_at_ms: int, + block_hash_hex: str, + ) -> PendingShareAppend: + persisted = stamped_pending_share(accepted_at_ms) + pending = PendingShare(**dict(vars(persisted))) + return PendingShareAppend( + pending_share=pending, + username="miner-a", + job_id=pending.job_id, + block_hash_hex=block_hash_hex, + collection_only=False, + credit_policy=None, + candidate_intent={ + "block_hash_hex": block_hash_hex, + "pending_share": dict(vars(pending)), + "credit_share_on_accept": True, + }, + ) + + def test_difficulty_drift_within_band_keeps_incremental_window( + self, + ) -> None: + # ASERT retargets every block, so the snapshot weight drifts a + # fraction of a percent per tip change. Within the retained margin + # that drift must ride the incremental window: the pre-band behavior + # ran the full oracle (a multi-second writer-lock hold) on every + # block, which starved landing-window ledger reads (mainnet alert + # #236). + server, ledger, artifacts = self.configured_server() + clock_ms = [1_000_000] + with patch( + "lab.prism.prism_coordinator.now_ms", + side_effect=lambda: clock_ms[0], + ): + initial = server._build_payout_ledger_artifact( + 0, 0, artifacts.network_difficulty + ) + assert initial is not None + self.assertEqual(initial.window_build_mode, "full_rescan") + self.assertEqual(ledger.full_snapshot_calls, 1) + + server.payout_artifact_min_build_interval_seconds = 0.0 + clock_ms[0] = 1_000_020 + drifted_difficulty = ( + int(artifacts.network_difficulty) * 101 + 99 + ) // 100 + drifted = server._build_payout_ledger_artifact( + 0, 0, drifted_difficulty + ) + assert drifted is not None + self.assertEqual(drifted.window_build_mode, "incremental") + self.assertEqual(ledger.full_snapshot_calls, 1) + # The artifact still stamps the live difficulty: the exact + # 8 x difficulty reward cutoff is re-derived downstream, so the + # cached superset only has to cover it. + self.assertEqual( + drifted.network_difficulty, drifted_difficulty + ) + + def test_difficulty_drift_beyond_band_floor_forces_full_rescan( + self, + ) -> None: + server, ledger, artifacts = self.configured_server() + clock_ms = [1_000_000] + with patch( + "lab.prism.prism_coordinator.now_ms", + side_effect=lambda: clock_ms[0], + ): + initial = server._build_payout_ledger_artifact( + 0, 0, artifacts.network_difficulty + ) + assert initial is not None + server.payout_artifact_min_build_interval_seconds = 0.0 + clock_ms[0] = 1_000_020 + # Cached weight below 5/8 of the live snapshot weight: the + # retained superset no longer covers the live reward window + # with headroom, so the oracle must re-center. + surged = server._build_payout_ledger_artifact( + 0, 0, int(artifacts.network_difficulty) * 2 + ) + assert surged is not None + self.assertEqual(surged.window_build_mode, "full_rescan") + self.assertEqual( + surged.window_full_rescan_reason, + "snapshot_window_weight_out_of_band", + ) + self.assertEqual(ledger.full_snapshot_calls, 2) + + def test_difficulty_drop_beyond_band_ceiling_forces_full_rescan( + self, + ) -> None: + server, ledger, artifacts = self.configured_server() + clock_ms = [1_000_000] + with patch( + "lab.prism.prism_coordinator.now_ms", + side_effect=lambda: clock_ms[0], + ): + initial = server._build_payout_ledger_artifact( + 0, 0, artifacts.network_difficulty + ) + assert initial is not None + server.payout_artifact_min_build_interval_seconds = 0.0 + clock_ms[0] = 1_000_020 + # Cached weight more than twice the live snapshot weight: the + # stale superset is bounded above, so a difficulty collapse + # re-centers rather than serving an oversized window forever. + collapsed = server._build_payout_ledger_artifact( + 0, 0, max(1, int(artifacts.network_difficulty) // 3) + ) + assert collapsed is not None + self.assertEqual(collapsed.window_build_mode, "full_rescan") + self.assertEqual( + collapsed.window_full_rescan_reason, + "snapshot_window_weight_out_of_band", + ) + self.assertEqual(ledger.full_snapshot_calls, 2) + + def test_self_check_under_small_drift_matches_and_keeps_weight( + self, + ) -> None: + # The periodic runtime-check compares like-for-like at the cached + # weight (in-band ASERT drift is not a pipeline mismatch) and, on + # ordinary drift, adopts the oracle window still at the cached + # weight: adopting a digest the daemon never prepared would answer + # needs_full on the next advance and re-pay the full oracle read + # at every runtime-check. + server, ledger, artifacts = self.configured_server() + clock_ms = [1_000_000] + with patch( + "lab.prism.prism_coordinator.now_ms", + side_effect=lambda: clock_ms[0], + ): + initial = server._build_payout_ledger_artifact( + 0, 0, artifacts.network_difficulty + ) + assert initial is not None + server.payout_artifact_min_build_interval_seconds = 0.0 + server.payout_artifact_full_rescan_seconds = 0.0 + clock_ms[0] = 1_000_020 + drifted_difficulty = ( + int(artifacts.network_difficulty) * 101 + 99 + ) // 100 + checked = server._build_payout_ledger_artifact( + 0, 0, drifted_difficulty + ) + assert checked is not None + self.assertEqual(checked.window_build_mode, "self_check_match") + window = server._incremental_payout_artifact_window + assert window is not None + self.assertEqual( + int(window.window.window_weight), + PRISM_REWARD_WINDOW_MULTIPLIER + * PRISM_SNAPSHOT_WINDOW_MARGIN + * int(artifacts.network_difficulty), + ) + self.assertEqual(ledger.full_snapshot_calls, 2) + + def test_self_check_recenters_only_a_well_oversized_weight( + self, + ) -> None: + # When difficulty falls far enough that the cached weight sits more + # than 25% above the live snapshot weight, the runtime-check adopts + # the oracle window at the live weight so a falling market cannot + # pin an oversized oracle scan; the match verdict still compares at + # the cached weight. + server, ledger, artifacts = self.configured_server() + clock_ms = [1_000_000] + with patch( + "lab.prism.prism_coordinator.now_ms", + side_effect=lambda: clock_ms[0], + ): + initial = server._build_payout_ledger_artifact( + 0, 0, artifacts.network_difficulty + ) + assert initial is not None + server.payout_artifact_min_build_interval_seconds = 0.0 + server.payout_artifact_full_rescan_seconds = 0.0 + clock_ms[0] = 1_000_020 + # cached/live = 4/3 > 5/4, still inside the 2x band ceiling. + collapsed_difficulty = max( + 1, int(artifacts.network_difficulty) * 3 // 4 + ) + checked = server._build_payout_ledger_artifact( + 0, 0, collapsed_difficulty + ) + assert checked is not None + self.assertEqual(checked.window_build_mode, "self_check_match") + window = server._incremental_payout_artifact_window + assert window is not None + self.assertEqual( + int(window.window.window_weight), + PRISM_REWARD_WINDOW_MULTIPLIER + * PRISM_SNAPSHOT_WINDOW_MARGIN + * collapsed_difficulty, + ) + self.assertEqual(ledger.full_snapshot_calls, 2) + + def test_late_visible_replay_append_forces_next_build_to_full_oracle( + self, + ) -> None: + server, ledger, artifacts = self.configured_server() + clock_ms = [1_000_000] + with patch( + "lab.prism.prism_coordinator.now_ms", + side_effect=lambda: clock_ms[0], + ): + initial = server._build_payout_ledger_artifact( + 0, 0, artifacts.network_difficulty + ) + assert initial is not None + self.assertEqual(initial.window_build_mode, "full_rescan") + + server.payout_artifact_min_build_interval_seconds = 0.0 + clock_ms[0] = 1_000_020 + advanced = server._build_payout_ledger_artifact( + 0, 0, artifacts.network_difficulty + ) + assert advanced is not None + self.assertEqual(advanced.window_build_mode, "incremental") + self.assertTrue(server._install_payout_ledger_artifact(advanced)) + anchor_ms = int(advanced.snapshot_anchor_ms) + + entry = self.replay_shaped_append_entry( + accepted_at_ms=anchor_ms - 10, + block_hash_hex="44" * 32, + ) + self.assertTrue(server._append_share_batch([entry])) + self.assertIsNone(server._incremental_payout_artifact_window) + with server._job_cache_lock: + self.assertIsNone(server._payout_ledger_artifact) + self.assertFalse(server._install_payout_ledger_artifact(advanced)) + self.assertIsNone( + server._cached_found_block_payout_artifact( + base_generation=0, + artifact_payout_state_generation=1, + network_difficulty=artifacts.network_difficulty, + fallback_reason="prepare_lock_busy", + ) + ) + + clock_ms[0] = 1_000_040 + rebuilt = server._build_payout_ledger_artifact( + 0, 0, artifacts.network_difficulty + ) + + assert rebuilt is not None + self.assertEqual(rebuilt.window_build_mode, "full_rescan") + self.assertEqual( + rebuilt.window_full_rescan_reason, + "late_visible_append", + ) + self.assertEqual(ledger.full_snapshot_calls, 2) + self.assertIn( + entry.pending_share.share_id, + {str(share["share_id"]) for share in rebuilt.shares_json}, + ) + + def test_replay_shaped_batch_commits_under_the_landing_fence(self) -> None: + # The durable append and its epoch bump must share the landing fence + # boundary: a bump that only starts after append_batch() returned + # leaves a gap where a landing verifies the pre-bump epoch and enters + # submitblock with the row already durable but unpaid. A batch holding + # a replay-shaped row therefore holds the fence across the ledger + # commit itself; an ordinary batch stays off the lock entirely. + server, ledger, artifacts = self.configured_server() + clock_ms = [1_000_000] + with patch( + "lab.prism.prism_coordinator.now_ms", + side_effect=lambda: clock_ms[0], + ): + initial = server._build_payout_ledger_artifact( + 0, 0, artifacts.network_difficulty + ) + assert initial is not None + server.payout_artifact_min_build_interval_seconds = 0.0 + clock_ms[0] = 1_000_020 + advanced = server._build_payout_ledger_artifact( + 0, 0, artifacts.network_difficulty + ) + assert advanced is not None + self.assertTrue(server._install_payout_ledger_artifact(advanced)) + anchor_ms = int(advanced.snapshot_anchor_ms) + + fence_held_during_commit: list[bool] = [] + original_append_batch = ledger.append_batch + + def recording_append_batch(entries: list[object]) -> list[object]: + fence_held_during_commit.append( + server._payout_append_landing_fence_lock.locked() + ) + return original_append_batch(entries) + + ledger.append_batch = recording_append_batch # type: ignore[method-assign] + + fresh = self.replay_shaped_append_entry( + accepted_at_ms=anchor_ms + 50, + block_hash_hex="88" * 32, + ) + self.assertTrue(server._append_share_batch([fresh])) + self.assertEqual(fence_held_during_commit, [False]) + with server._job_cache_lock: + self.assertEqual( + server._payout_ledger_append_invalidation_epoch, 0 + ) + + predating = self.replay_shaped_append_entry( + accepted_at_ms=anchor_ms - 10, + block_hash_hex="99" * 32, + ) + self.assertTrue(server._append_share_batch([predating])) + self.assertEqual(fence_held_during_commit, [False, True]) + with server._job_cache_lock: + self.assertEqual( + server._payout_ledger_append_invalidation_epoch, 1 + ) + self.assertIsNone(server._payout_ledger_artifact) + self.assertIsNone(server._incremental_payout_artifact_window) + + def test_replay_append_during_cold_scan_invalidates_inflight_walk( + self, + ) -> None: + # A cold-start walk exposes no cached window or armed artifact while + # its ledger read runs. A replayed row committing mid-walk with + # timestamps below the walk's anchor must still record an + # invalidation, or the walk installs a window that can never + # rediscover the row through delta reads. + server, ledger, artifacts = self.configured_server() + clock_ms = [1_000_000] + entry = self.replay_shaped_append_entry( + accepted_at_ms=999_950, + block_hash_hex="55" * 32, + ) + original_snapshot = ledger.snapshot_at_job_issue + raced = threading.Event() + append_threads: list[threading.Thread] = [] + + def racing_snapshot( + anchor_job_issued_at_ms: int, + *, + window_weight: int | None = None, + ) -> list[object]: + # Capture the read result first: the replayed row committing + # afterwards models a database snapshot taken before the commit. + records = original_snapshot( + anchor_job_issued_at_ms, + window_weight=window_weight, + ) + if not raced.is_set(): + raced.set() + thread = threading.Thread( + target=server._append_share_batch, + args=([entry],), + ) + thread.start() + append_threads.append(thread) + deadline = time.monotonic() + 2.0 + while time.monotonic() < deadline: + with server._job_cache_lock: + if server._payout_ledger_append_invalidation_epoch: + break + time.sleep(0.001) + return records + + ledger.snapshot_at_job_issue = racing_snapshot # type: ignore[method-assign] + with patch( + "lab.prism.prism_coordinator.now_ms", + side_effect=lambda: clock_ms[0], + ): + initial = server._build_payout_ledger_artifact( + 0, 0, artifacts.network_difficulty + ) + assert initial is not None + self.assertEqual(initial.window_build_mode, "full_rescan") + self.assertNotIn( + entry.pending_share.share_id, + {str(share["share_id"]) for share in initial.shares_json}, + ) + self.assertTrue(append_threads) + append_threads[0].join(timeout=2.0) + self.assertFalse(append_threads[0].is_alive()) + with server._job_cache_lock: + self.assertEqual( + server._payout_ledger_append_invalidation_epoch, 1 + ) + self.assertFalse(server._install_payout_ledger_artifact(initial)) + with server._job_cache_lock: + self.assertIsNone(server._payout_ledger_artifact) + + server.payout_artifact_min_build_interval_seconds = 0.0 + clock_ms[0] = 1_000_040 + rebuilt = server._build_payout_ledger_artifact( + 0, 0, artifacts.network_difficulty + ) + + assert rebuilt is not None + self.assertEqual(rebuilt.window_build_mode, "full_rescan") + self.assertEqual( + rebuilt.window_full_rescan_reason, + "late_visible_append", + ) + self.assertIn( + entry.pending_share.share_id, + {str(share["share_id"]) for share in rebuilt.shares_json}, + ) + + def test_inflight_scan_anchor_records_invalidation_without_windows( + self, + ) -> None: + # Directly exercise the published in-flight anchor: with no cached + # window and no armed artifact, only a predating append may advance + # the epoch, and fresh shares above the anchor must leave the walk + # undisturbed. + server, _ledger, _artifacts = self.configured_server() + server._ensure_job_cache_state() + predating = stamped_pending_share(999_950) + fresh = stamped_pending_share(1_000_050) + + server._invalidate_incremental_payout_window_for_append(predating) + with server._job_cache_lock: + self.assertEqual(server._payout_ledger_append_invalidation_epoch, 0) + + with patch( + "lab.prism.prism_coordinator.now_ms", + side_effect=lambda: 1_000_000, + ): + token = server._expose_inflight_scan_anchor(999_999) + try: + server._invalidate_incremental_payout_window_for_append(fresh) + with server._job_cache_lock: + self.assertEqual( + server._payout_ledger_append_invalidation_epoch, 0 + ) + server._invalidate_incremental_payout_window_for_append(predating) + finally: + server._retire_inflight_scan_anchor(token) + with server._job_cache_lock: + self.assertEqual(server._payout_ledger_append_invalidation_epoch, 1) + self.assertEqual(server._payout_window_inflight_scan_anchors, {}) + self.assertEqual( + server._incremental_payout_artifact_window_invalidation_reason, + "late_visible_append", + ) + + def test_replay_append_during_bundle_construction_invalidates_build( + self, + ) -> None: + # A cold synchronous ready build exposes no cached window or armed + # artifact once its ledger read returns, while manifest construction + # and cache publication still lie ahead. A replayed row committing in + # that gap with timestamps below the read's anchor must still advance + # the epoch -- the exposure outlives the read -- so the publication + # fence supersedes the bundle instead of seeding a window that delta + # reads can never rediscover the row from. + server, _ledger, _artifacts = self.configured_server() + install_fake_bundle_builder(server) + clock_ms = [1_000_000] + entry = self.replay_shaped_append_entry( + accepted_at_ms=999_950, + block_hash_hex="66" * 32, + ) + raced = threading.Event() + original_build = server.build_audit_bundle + + def racing_build(**kwargs: object) -> dict[str, object]: + # The ledger read has already returned; the replayed row commits + # while the coinbase manifest is being constructed. + if not raced.is_set(): + raced.set() + server._append_share_batch([entry]) + return original_build(**kwargs) + + server.build_audit_bundle = racing_build # type: ignore[method-assign] + with patch( + "lab.prism.prism_coordinator.now_ms", + side_effect=lambda: clock_ms[0], + ): + with self.assertRaises(JobBuildSuperseded): + server.build_shared_job_bundle( + server.current_template_artifacts(), + worker(), + ) + with server._job_cache_lock: + self.assertEqual( + server._payout_ledger_append_invalidation_epoch, 1 + ) + self.assertIsNone(server._payout_ledger_artifact) + + # A fresh build keyed to the advanced epoch re-reads the ledger + # and rediscovers the replayed row. + with server._job_cache_lock: + server._job_build_issued_at_ms.clear() + clock_ms[0] = 1_000_040 + rebuilt = server.build_shared_job_bundle( + server.current_template_artifacts(), + worker(), + ) + self.assertIn( + entry.pending_share.share_id, + {str(share["share_id"]) for share in rebuilt.shares_json}, + ) + + def test_seeded_artifact_exposure_survives_until_install_fence( + self, + ) -> None: + # The synchronous build's exposed scan anchor rides its seeded + # artifact: a replayed row committing after the bundle returned but + # before cache publication installs the seed must advance the epoch, + # and the doomed seed's install fence must discard it (retiring the + # exposure) instead of arming the pre-append window. + server, _ledger, _artifacts = self.configured_server() + install_fake_bundle_builder(server) + with patch( + "lab.prism.prism_coordinator.now_ms", + side_effect=lambda: 1_000_000, + ): + bundle = server.build_shared_job_bundle( + server.current_template_artifacts(), + worker(), + ) + seed = bundle.prepared_ledger_artifact + assert seed is not None + self.assertIsNotNone(seed.inflight_scan_anchor_token) + + entry = self.replay_shaped_append_entry( + accepted_at_ms=999_950, + block_hash_hex="77" * 32, + ) + self.assertTrue(server._append_share_batch([entry])) + with server._job_cache_lock: + self.assertEqual( + server._payout_ledger_append_invalidation_epoch, 1 + ) + + self.assertFalse(server._install_payout_ledger_artifact(seed)) + with server._job_cache_lock: + self.assertIsNone(server._payout_ledger_artifact) + self.assertEqual(server._payout_window_inflight_scan_anchors, {}) + + def test_late_append_disarms_before_prepare_lock_is_available(self) -> None: + server, _ledger, artifacts = self.configured_server() + clock_ms = [1_000_000] + with patch( + "lab.prism.prism_coordinator.now_ms", + side_effect=lambda: clock_ms[0], + ): + artifact = server._build_payout_ledger_artifact( + 0, 0, artifacts.network_difficulty + ) + assert artifact is not None + self.assertTrue(server._install_payout_ledger_artifact(artifact)) + anchor_ms = int(artifact.snapshot_anchor_ms) + pending = stamped_pending_share(anchor_ms - 10) + + server._payout_state_prepare_lock.acquire() + # Exercise the artifact anchor independently: an already-cold + # incremental cache must not hide an unsafe armed artifact. + server._incremental_payout_artifact_window = None + invalidation_finished = threading.Event() + + def invalidate() -> None: + server._invalidate_incremental_payout_window_for_append( + pending + ) + invalidation_finished.set() + + thread = threading.Thread(target=invalidate) + thread.start() + try: + deadline = time.monotonic() + 1.0 + while time.monotonic() < deadline: + with server._job_cache_lock: + if server._payout_ledger_artifact is None: + break + time.sleep(0.001) + with server._job_cache_lock: + self.assertIsNone(server._payout_ledger_artifact) + self.assertGreater( + server._payout_ledger_append_invalidation_epoch, + artifact.append_invalidation_epoch, + ) + self.assertFalse(invalidation_finished.is_set()) + self.assertFalse( + server._install_payout_ledger_artifact(artifact) + ) + finally: + server._payout_state_prepare_lock.release() + thread.join(timeout=1.0) + self.assertFalse(thread.is_alive()) + self.assertTrue(invalidation_finished.is_set()) + + def test_atomic_publication_rejects_pre_append_artifact(self) -> None: + server, _ledger, artifacts = self.configured_server() + clock_ms = [1_000_000] + with patch( + "lab.prism.prism_coordinator.now_ms", + side_effect=lambda: clock_ms[0], + ): + candidate = server._current_payout_state_candidate() + stale = candidate.ledger_artifact + assert stale is not None + anchor_ms = int(stale.snapshot_anchor_ms) + server._invalidate_incremental_payout_window_for_append( + stamped_pending_share(anchor_ms - 10) + ) + with server._payout_artifact_executor_lock: + server._payout_artifact_executor_shutdown = True + published = server._publish_payout_state_candidate(candidate) + + self.assertIsNotNone(published) + with server._job_cache_lock: + self.assertIsNone(server._payout_ledger_artifact) + self.assertGreater( + server._payout_ledger_append_invalidation_epoch, + stale.append_invalidation_epoch, + ) + + def test_late_append_supersedes_in_flight_sync_seed_bundle(self) -> None: + server, _ledger, artifacts = self.configured_server() + clock_ms = [1_000_000] + with patch( + "lab.prism.prism_coordinator.now_ms", + side_effect=lambda: clock_ms[0], + ): + artifact = server._build_payout_ledger_artifact( + 0, 0, artifacts.network_difficulty + ) + assert artifact is not None + self.assertTrue(server._install_payout_ledger_artifact(artifact)) + pending = stamped_pending_share( + int(artifact.snapshot_anchor_ms) - 10 + ) + + def invalidating_builder(**kwargs: object) -> dict[str, object]: + server._invalidate_incremental_payout_window_for_append( + pending + ) + suffix_hex = str(kwargs["coinbase_script_sig_suffix_hex"]) + return { + "found_block": dict(kwargs["found_block"]), + "payout_policy_manifest": {"accounts": []}, + "signed_coinbase_manifest": { + "manifest": { + "coinbase_tx_hex": synthetic_manifest_coinbase_hex( + suffix_hex + ), + } + }, + } + + server.build_audit_bundle = invalidating_builder # type: ignore[method-assign] + with self.assertRaises(JobBuildSuperseded): + server.build_shared_job_bundle( + artifacts, + worker(), + payout_artifact=None, + ) + + with server._job_cache_lock: + self.assertIsNone(server._payout_ledger_artifact) + + def test_normal_append_preserves_incremental_window(self) -> None: + server, ledger, artifacts = self.configured_server() + clock_ms = [1_000_000] + with patch( + "lab.prism.prism_coordinator.now_ms", + side_effect=lambda: clock_ms[0], + ): + initial = server._build_payout_ledger_artifact( + 0, 0, artifacts.network_difficulty + ) + assert initial is not None + cached = server._incremental_payout_artifact_window + self.assertIsNotNone(cached) + self.assertTrue(server._install_payout_ledger_artifact(initial)) + with server._job_cache_lock: + armed = server._payout_ledger_artifact + append_epoch = ( + server._payout_ledger_append_invalidation_epoch + ) + self.assertIsNotNone(armed) + + entry = self.replay_shaped_append_entry( + accepted_at_ms=1_000_010, + block_hash_hex="55" * 32, + ) + self.assertTrue(server._append_share_entry(entry)) + self.assertIs(server._incremental_payout_artifact_window, cached) + with server._job_cache_lock: + self.assertIs(server._payout_ledger_artifact, armed) + self.assertEqual( + server._payout_ledger_append_invalidation_epoch, + append_epoch, + ) + self.assertIs( + server._usable_payout_ledger_artifact( + 0, + artifacts.network_difficulty, + ), + armed, + ) + + server.payout_artifact_min_build_interval_seconds = 0.0 + clock_ms[0] = 1_000_020 + advanced = server._build_payout_ledger_artifact( + 0, 0, artifacts.network_difficulty + ) + + assert advanced is not None + self.assertEqual(advanced.window_build_mode, "incremental") + self.assertIsNone(advanced.window_full_rescan_reason) + self.assertEqual(advanced.window_touched_pages, 1) + self.assertEqual(ledger.full_snapshot_calls, 1) + self.assertEqual(ledger.delta_snapshot_calls, 1) + self.assertIn( + entry.pending_share.share_id, + {str(share["share_id"]) for share in advanced.shares_json}, + ) + + def test_debounce_then_delta_and_forced_full_rescan(self) -> None: + server, ledger, artifacts = self.configured_server() + clock_ms = [1_000_000] + with patch( + "lab.prism.prism_coordinator.now_ms", + side_effect=lambda: clock_ms[0], + ): + initial = server._build_payout_ledger_artifact( + 0, 0, artifacts.network_difficulty + ) + assert initial is not None + self.assertEqual(initial.window_build_mode, "full_rescan") + self.assertEqual(ledger.full_snapshot_calls, 1) + + append_incremental_share( + ledger, + share_seq=4, + accepted_at_ms=1_000_010, + ) + clock_ms[0] = 1_000_020 + debounced = server._build_payout_ledger_artifact( + 0, 0, artifacts.network_difficulty + ) + assert debounced is not None + self.assertEqual(debounced.window_build_mode, "debounced") + self.assertEqual(ledger.full_snapshot_calls, 1) + self.assertEqual(ledger.delta_snapshot_calls, 0) + self.assertEqual(len(debounced.shares_json), 3) + + server.payout_artifact_min_build_interval_seconds = 0.0 + advanced = server._build_payout_ledger_artifact( + 0, 0, artifacts.network_difficulty + ) + assert advanced is not None + self.assertEqual(advanced.window_build_mode, "incremental") + self.assertEqual(advanced.window_delta_rows, 1) + self.assertEqual(ledger.delta_snapshot_calls, 1) + self.assertEqual(len(advanced.shares_json), 4) + + # A payout mutation (the reconciler's #224 intent) re-reads the + # balances and advances the append-only window from its delta; + # it is never a full rescan. + mutated = server._build_payout_ledger_artifact( + 0, + 0, + artifacts.network_difficulty, + force_prior_balances_read=True, + ) + assert mutated is not None + self.assertEqual(mutated.window_build_mode, "incremental") + self.assertIsNone(mutated.window_full_rescan_reason) + self.assertEqual(ledger.full_snapshot_calls, 1) + self.assertEqual(ledger.delta_snapshot_calls, 2) + # The positional flag keeps its full fail-closed meaning. + forced = server._build_payout_ledger_artifact( + 0, + 0, + artifacts.network_difficulty, + True, + ) + assert forced is not None + self.assertEqual(forced.window_build_mode, "full_rescan") + self.assertEqual( + forced.window_full_rescan_reason, + "reconcile_invalidation", + ) + self.assertEqual(ledger.full_snapshot_calls, 2) + self.assertEqual( + forced.share_snapshot_sha256, mutated.share_snapshot_sha256 + ) + + def test_periodic_self_check_compares_delta_to_full_oracle(self) -> None: + server, ledger, artifacts = self.configured_server() + clock_ms = [1_000_000] + with patch( + "lab.prism.prism_coordinator.now_ms", + side_effect=lambda: clock_ms[0], + ): + self.assertIsNotNone( + server._build_payout_ledger_artifact( + 0, 0, artifacts.network_difficulty + ) + ) + append_incremental_share( + ledger, + share_seq=4, + accepted_at_ms=1_000_010, + ) + clock_ms[0] = 1_000_020 + server.payout_artifact_min_build_interval_seconds = 0.0 + server.payout_artifact_full_rescan_seconds = 0.0 + + checked = server._build_payout_ledger_artifact( + 0, 0, artifacts.network_difficulty + ) + + assert checked is not None + self.assertEqual(checked.window_build_mode, "self_check_match") + self.assertEqual(ledger.delta_snapshot_calls, 1) + self.assertEqual(ledger.full_snapshot_calls, 2) + + def test_routine_build_runs_overdue_self_check_despite_preview_refresh( + self, + ) -> None: + # Accepted-block previews bypass the build interval and refresh + # refreshed_monotonic on every block, yet never run the periodic + # oracle themselves. Under a sustained sub-interval block cadence + # every routine build used to return from the debounce branch + # before reaching the self-check condition, so the configured + # full-rescan window and balance oracle could be postponed + # indefinitely. An overdue self-check now disarms the debounce + # for routine builds while the urgent preview path stays + # debounce-free and oracle-free. + server, ledger, artifacts = self.configured_server() + clock_ms = [1_000_000] + with patch( + "lab.prism.prism_coordinator.now_ms", + side_effect=lambda: clock_ms[0], + ): + self.assertIsNotNone( + server._build_payout_ledger_artifact( + 0, 0, artifacts.network_difficulty + ) + ) + append_incremental_share( + ledger, + share_seq=4, + accepted_at_ms=1_000_010, + ) + clock_ms[0] = 1_000_020 + preview = server._build_payout_ledger_artifact( + 0, 0, artifacts.network_difficulty, False, True + ) + assert preview is not None + self.assertEqual(preview.window_build_mode, "incremental") + # The next preview arrives with the self-check long overdue; + # the urgent path must still not pay the oracle walk. + server.payout_artifact_full_rescan_seconds = 0.0 + append_incremental_share( + ledger, + share_seq=5, + accepted_at_ms=1_000_030, + ) + clock_ms[0] = 1_000_040 + urgent = server._build_payout_ledger_artifact( + 0, 0, artifacts.network_difficulty, False, True + ) + assert urgent is not None + self.assertEqual(urgent.window_build_mode, "incremental") + self.assertEqual(ledger.full_snapshot_calls, 1) + # A routine build inside the min build interval used to + # debounce here -- forever, while blocks kept refreshing the + # window above -- and the self-check never ran. + checked = server._build_payout_ledger_artifact( + 0, 0, artifacts.network_difficulty + ) + + assert checked is not None + self.assertEqual(checked.window_build_mode, "self_check_match") + self.assertEqual(ledger.full_snapshot_calls, 2) + + def test_periodic_self_check_replaces_divergent_delta_with_full_bytes( + self, + ) -> None: + class OmittingDeltaLedger(IncrementalRecordingLedger): + def snapshot_between_job_issues( + self, + previous_anchor_job_issued_at_ms: int, + anchor_job_issued_at_ms: int, + ) -> list[object]: + del previous_anchor_job_issued_at_ms, anchor_job_issued_at_ms + self.delta_snapshot_calls += 1 + return [] + + ledger = OmittingDeltaLedger() + for share_seq in range(1, 4): + append_incremental_share( + ledger, + share_seq=share_seq, + accepted_at_ms=999_900 + share_seq, + ) + server, rpc = coordinator(ledger=ledger) + artifacts = server.store_template_artifacts(dict(rpc.template)) + assert artifacts is not None + server._pool_ready_latched = True + clock_ms = [1_000_000] + with patch( + "lab.prism.prism_coordinator.now_ms", + side_effect=lambda: clock_ms[0], + ): + self.assertIsNotNone( + server._build_payout_ledger_artifact( + 0, 0, artifacts.network_difficulty + ) + ) + append_incremental_share( + ledger, + share_seq=4, + accepted_at_ms=1_000_010, + ) + clock_ms[0] = 1_000_020 + server.payout_artifact_min_build_interval_seconds = 0.0 + server.payout_artifact_full_rescan_seconds = 0.0 + repaired = server._build_payout_ledger_artifact( + 0, 0, artifacts.network_difficulty + ) + forced = server._build_payout_ledger_artifact( + 0, + 0, + artifacts.network_difficulty, + True, + ) + + assert repaired is not None and forced is not None + self.assertEqual(forced.window_build_mode, "full_rescan") + self.assertEqual(repaired.window_build_mode, "self_check_mismatch") + self.assertEqual(list(repaired.shares_json), list(forced.shares_json)) + self.assertEqual( + repaired.share_snapshot_sha256, + forced.share_snapshot_sha256, + ) + + def test_periodic_self_check_invalidates_drifted_published_balances( + self, + ) -> None: + class DriftingCarryLedger(IncrementalRecordingLedger): + def __init__(self) -> None: + super().__init__() + self.balance_sats = 546 + self.prior_balance_reads = 0 + + def current_prior_balances(self) -> list[dict[str, object]]: + self.prior_balance_reads += 1 + return [ + { + "recipient_id": "carry", + "order_key": "01:carry", + "p2mr_program_hex": "66" * 32, + "balance_sats": self.balance_sats, + } + ] + + ledger = DriftingCarryLedger() + for share_seq in range(1, 4): + append_incremental_share( + ledger, + share_seq=share_seq, + accepted_at_ms=999_900 + share_seq, + ) + server, rpc = coordinator(ledger=ledger) + artifacts = server.store_template_artifacts(dict(rpc.template)) + assert artifacts is not None + server._pool_ready_latched = True + server.payout_artifact_min_build_interval_seconds = 0.0 + server.payout_artifact_full_rescan_seconds = 3_600.0 + clock_ms = [1_000_000] + monotonic_seconds = [100.0] + build_logs: list[dict[str, object]] = [] + server._payout_artifact_log = ( # type: ignore[method-assign] + lambda event, **fields: build_logs.append( + {"event": event, **fields} + ) + ) + + with ( + patch( + "lab.prism.prism_coordinator.now_ms", + side_effect=lambda: clock_ms[0], + ), + patch( + "lab.prism.prism_coordinator.time.monotonic", + side_effect=lambda: monotonic_seconds[0], + ), + ): + published = server._current_payout_state_artifact() + initial = server._build_payout_ledger_artifact( + 0, + 0, + artifacts.network_difficulty, + ) + assert initial is not None + self.assertTrue(server._install_payout_ledger_artifact(initial)) + + ledger.balance_sats = 777 + append_incremental_share( + ledger, + share_seq=4, + accepted_at_ms=1_000_010, + ) + clock_ms[0] = 1_000_020 + monotonic_seconds[0] = 3_701.0 + checked = server._build_payout_ledger_artifact( + 0, + 0, + artifacts.network_difficulty, + ) + + with server._job_cache_lock: + self.assertIsNone(server._payout_ledger_artifact) + + # The mismatch build publishes the repaired balances for the same + # generation, so the next build resumes byte reuse from the + # repaired snapshot instead of paying another carry read against + # a still-stale publication. + clock_ms[0] = 1_000_030 + monotonic_seconds[0] = 3_702.0 + following = server._build_payout_ledger_artifact( + 0, + 0, + artifacts.network_difficulty, + ) + + assert checked is not None and following is not None + self.assertEqual(checked.window_build_mode, "self_check_match") + self.assertEqual(checked.prior_balances[0]["balance_sats"], 777) + self.assertEqual(following.prior_balances[0]["balance_sats"], 777) + self.assertEqual(ledger.prior_balance_reads, 2) + self.assertEqual(ledger.full_snapshot_calls, 2) + repaired = server._published_payout_state.artifact + assert repaired is not None + self.assertNotEqual( + repaired.prior_balances_sha256, + published.prior_balances_sha256, + ) + self.assertEqual( + repaired.prior_balances_sha256, + checked.prior_balances_sha256, + ) + self.assertIsNone( + server._payout_prior_balances_reuse_invalidated_sha256 + ) + self.assertEqual( + server.payout_artifact_event_counts["balance_check_mismatch"], + 1, + ) + self.assertEqual( + server.payout_artifact_event_counts["balance_repair_published"], + 1, + ) + self.assertEqual( + server.payout_artifact_event_counts["self_check_match"], + 1, + ) + built_sources = [ + entry["prior_balances_source"] + for entry in build_logs + if entry["event"] == "payout_artifact_built" + ] + self.assertEqual(built_sources, ["published", "ledger", "published"]) + mismatch_logs = [ + entry + for entry in build_logs + if entry["event"] == "payout_artifact_balance_check_mismatch" + ] + self.assertEqual(len(mismatch_logs), 1) + repair_logs = [ + entry + for entry in build_logs + if entry["event"] == "payout_artifact_balance_repair_published" + ] + self.assertEqual(len(repair_logs), 1) + + def test_balance_backstop_failure_retains_oracle_repaired_window( + self, + ) -> None: + # The share-window oracle and the carry-balance backstop are + # independent reads. A backstop failure after a successful oracle + # must not resurrect the delta window the oracle just refuted. + class FailingCarryOmittingDeltaLedger(IncrementalRecordingLedger): + def __init__(self) -> None: + super().__init__() + self.fail_balance_read = False + self.balance_read_attempts = 0 + + def snapshot_between_job_issues( + self, + previous_anchor_job_issued_at_ms: int, + anchor_job_issued_at_ms: int, + ) -> list[object]: + del previous_anchor_job_issued_at_ms, anchor_job_issued_at_ms + self.delta_snapshot_calls += 1 + return [] + + def current_prior_balances(self) -> list[dict[str, object]]: + self.balance_read_attempts += 1 + if self.fail_balance_read: + raise RuntimeError("carry aggregate unavailable") + return [ + { + "recipient_id": "carry", + "order_key": "01:carry", + "p2mr_program_hex": "66" * 32, + "balance_sats": 546, + } + ] + + ledger = FailingCarryOmittingDeltaLedger() + for share_seq in range(1, 4): + append_incremental_share( + ledger, + share_seq=share_seq, + accepted_at_ms=999_900 + share_seq, + ) + server, rpc = coordinator(ledger=ledger) + artifacts = server.store_template_artifacts(dict(rpc.template)) + assert artifacts is not None + server._pool_ready_latched = True + server.payout_artifact_min_build_interval_seconds = 0.0 + server.payout_artifact_full_rescan_seconds = 3_600.0 + clock_ms = [1_000_000] + monotonic_seconds = [100.0] + with ( + patch( + "lab.prism.prism_coordinator.now_ms", + side_effect=lambda: clock_ms[0], + ), + patch( + "lab.prism.prism_coordinator.time.monotonic", + side_effect=lambda: monotonic_seconds[0], + ), + ): + server._current_payout_state_artifact() + initial = server._build_payout_ledger_artifact( + 0, + 0, + artifacts.network_difficulty, + ) + assert initial is not None + self.assertTrue(server._install_payout_ledger_artifact(initial)) + + ledger.fail_balance_read = True + append_incremental_share( + ledger, + share_seq=4, + accepted_at_ms=1_000_010, + ) + clock_ms[0] = 1_000_020 + monotonic_seconds[0] = 3_701.0 + checked = server._build_payout_ledger_artifact( + 0, + 0, + artifacts.network_difficulty, + ) + + append_incremental_share( + ledger, + share_seq=5, + accepted_at_ms=1_000_021, + ) + clock_ms[0] = 1_000_030 + monotonic_seconds[0] = 3_702.0 + following = server._build_payout_ledger_artifact( + 0, + 0, + artifacts.network_difficulty, + ) + + assert checked is not None and following is not None + self.assertEqual(checked.window_build_mode, "self_check_mismatch") + self.assertEqual( + checked.window_full_rescan_reason, + "periodic_self_check_balance_check_failed", + ) + # The omitting delta produced three retained shares; the oracle's + # four-share window must be the one served and cached. + self.assertEqual(len(checked.shares_json), 4) + # One read published the priming state; the only other attempt is + # the backstop that failed. + self.assertEqual(ledger.balance_read_attempts, 2) + # The unverified balances keep reusing the published snapshot. + self.assertEqual( + checked.prior_balances_sha256, + initial.prior_balances_sha256, + ) + self.assertIsNone( + server._payout_prior_balances_reuse_invalidated_sha256 + ) + self.assertEqual( + server.payout_artifact_event_counts["self_check_mismatch"], + 1, + ) + self.assertEqual( + server.payout_artifact_event_counts["self_check_failed"], + 0, + ) + self.assertEqual( + server.payout_artifact_event_counts["balance_check_mismatch"], + 0, + ) + # The failed backstop still stamps the attempt: the next build stays + # on the delta path instead of retrying the oracle immediately. + self.assertEqual(following.window_build_mode, "incremental") + self.assertEqual(ledger.full_snapshot_calls, 2) + + def test_periodic_self_check_repair_arms_and_serves(self) -> None: + # The repaired publication is what lets the mismatch build's fresh + # balances actually reach miners: its artifact must arm against the + # repaired digest and the reuse probe must serve it, instead of the + # armed-artifact fence clearing the repair against a stale published + # snapshot until an unrelated generation change. + class DriftingCarryLedger(IncrementalRecordingLedger): + def __init__(self) -> None: + super().__init__() + self.balance_sats = 546 + + def current_prior_balances(self) -> list[dict[str, object]]: + return [ + { + "recipient_id": "carry", + "order_key": "01:carry", + "p2mr_program_hex": "66" * 32, + "balance_sats": self.balance_sats, + } + ] + + ledger = DriftingCarryLedger() + for share_seq in range(1, 4): + append_incremental_share( + ledger, + share_seq=share_seq, + accepted_at_ms=999_900 + share_seq, + ) + server, rpc = coordinator(ledger=ledger) + artifacts = server.store_template_artifacts(dict(rpc.template)) + assert artifacts is not None + server._pool_ready_latched = True + server.payout_artifact_min_build_interval_seconds = 0.0 + server.payout_artifact_full_rescan_seconds = 3_600.0 + clock_ms = [1_000_000] + monotonic_seconds = [100.0] + + with ( + patch( + "lab.prism.prism_coordinator.now_ms", + side_effect=lambda: clock_ms[0], + ), + patch( + "lab.prism.prism_coordinator.time.monotonic", + side_effect=lambda: monotonic_seconds[0], + ), + ): + server._current_payout_state_artifact() + initial = server._build_payout_ledger_artifact( + 0, 0, artifacts.network_difficulty + ) + assert initial is not None + self.assertTrue(server._install_payout_ledger_artifact(initial)) + + ledger.balance_sats = 777 + clock_ms[0] = 1_000_020 + monotonic_seconds[0] = 3_701.0 + checked = server._build_payout_ledger_artifact( + 0, 0, artifacts.network_difficulty + ) + assert checked is not None + self.assertEqual(checked.prior_balances[0]["balance_sats"], 777) + + self.assertTrue(server._install_payout_ledger_artifact(checked)) + served = server._usable_payout_ledger_artifact( + 0, + artifacts.network_difficulty, + ) + + assert served is not None + self.assertEqual(served.prior_balances[0]["balance_sats"], 777) + self.assertEqual( + served.prior_balances_sha256, + checked.prior_balances_sha256, + ) + with server._job_cache_lock: + self.assertIsNotNone(server._payout_ledger_artifact) + + def test_self_check_repair_supersedes_active_jobs(self) -> None: + # The in-place repair changes neither the payout generation nor the + # append epoch, so an already-stamped job passes every admission + # fence while carrying the refuted balances. The repair must + # schedule a refresh wave, and the wave's reselection must identify + # such jobs by their payout digest, or they stay mineable (and a + # block solve commits the stale allocation) until an unrelated + # generation change. + server, _ledger, _artifacts = self.configured_server() + install_fake_bundle_builder(server) + server._ensure_tip_refresh_state() + state = client(1) + with patch( + "lab.prism.prism_coordinator.now_ms", + side_effect=lambda: 1_000_000, + ): + bundle = server.build_shared_job_bundle( + server.current_template_artifacts(), + worker(), + ) + context = server.stamp_job_for_client( + state, + bundle, + clean_jobs=True, + ) + state.active_job = context + snapshot = SimpleNamespace( + bestblockhash=str(bundle.template["previousblockhash"]), + previousblockhash=str(bundle.template["previousblockhash"]), + template_fingerprint=bundle.template_fingerprint, + ) + with server._job_cache_lock: + published = server._published_payout_state.artifact + assert published is not None + self.assertEqual( + context.payout_artifact_sha256, + published.prior_balances_sha256, + ) + self.assertFalse( + server.client_needs_tip_template_refresh(state, snapshot) + ) + + self.assertTrue( + server._publish_self_check_repaired_balances( + 0, + stale_prior_balances_sha256=published.prior_balances_sha256, + balances=[ + { + "recipient_id": "carry", + "order_key": "01:carry", + "p2mr_program_hex": "66" * 32, + "balance_sats": 999, + } + ], + ) + ) + + self.assertTrue( + server.client_needs_tip_template_refresh(state, snapshot) + ) + self.assertTrue(server._tip_refresh_pending_event.is_set()) + self.assertTrue(server._tip_refresh_retry.is_set()) + + def test_self_check_repair_retires_refuted_jobs_clean(self) -> None: + # Reselection alone is not enough: the wave stamps the replacement + # with clean_jobs from client_tip_changed_for_snapshot, and a + # clean_jobs=False delivery leaves the refuted job ID admissible in + # active_job_ids. The digest mismatch must force clean-job + # retirement, or a late block submission on the old job still + # commits the refuted allocation after corrected work is delivered. + server, _ledger, _artifacts = self.configured_server() + install_fake_bundle_builder(server) + server._ensure_tip_refresh_state() + state = client(1) + with patch( + "lab.prism.prism_coordinator.now_ms", + side_effect=lambda: 1_000_000, + ): + bundle = server.build_shared_job_bundle( + server.current_template_artifacts(), + worker(), + ) + context = server.stamp_job_for_client( + state, + bundle, + clean_jobs=True, + ) + state.active_job = context + snapshot = SimpleNamespace( + bestblockhash=str(bundle.template["previousblockhash"]), + previousblockhash=str(bundle.template["previousblockhash"]), + template_fingerprint=bundle.template_fingerprint, + ) + with server._job_cache_lock: + published = server._published_payout_state.artifact + assert published is not None + self.assertFalse( + server.client_tip_changed_for_snapshot(state, snapshot) + ) + + self.assertTrue( + server._publish_self_check_repaired_balances( + 0, + stale_prior_balances_sha256=published.prior_balances_sha256, + balances=[ + { + "recipient_id": "carry", + "order_key": "01:carry", + "p2mr_program_hex": "66" * 32, + "balance_sats": 999, + } + ], + ) + ) + + self.assertTrue( + server.client_tip_changed_for_snapshot(state, snapshot) + ) + + def test_late_append_invalidation_supersedes_active_jobs(self) -> None: + # A late-visible append advances only the append epoch: the payout + # generation and published digest both survive, so an active job + # mining the pre-append window passes every other fence. The + # invalidation must schedule a refresh wave, the reselection must + # identify the job by its stamped epoch, and the replacement must + # retire it clean -- otherwise miners keep working (and can solve a + # block from) a window that omitted the late share until some + # unrelated tip event. + server, _ledger, _artifacts = self.configured_server() + install_fake_bundle_builder(server) + server._ensure_tip_refresh_state() + state = client(1) + with patch( + "lab.prism.prism_coordinator.now_ms", + side_effect=lambda: 1_000_000, + ): + bundle = server.build_shared_job_bundle( + server.current_template_artifacts(), + worker(), + ) + context = server.stamp_job_for_client( + state, + bundle, + clean_jobs=True, + ) + state.active_job = context + snapshot = SimpleNamespace( + bestblockhash=str(bundle.template["previousblockhash"]), + previousblockhash=str(bundle.template["previousblockhash"]), + template_fingerprint=bundle.template_fingerprint, + ) + self.assertFalse( + server.client_needs_tip_template_refresh(state, snapshot) + ) + self.assertFalse( + server.client_tip_changed_for_snapshot(state, snapshot) + ) + + with patch( + "lab.prism.prism_coordinator.now_ms", + side_effect=lambda: 1_000_000, + ): + token = server._expose_inflight_scan_anchor(999_999) + try: + server._invalidate_incremental_payout_window_for_append( + stamped_pending_share(999_950) + ) + finally: + server._retire_inflight_scan_anchor(token) + with server._job_cache_lock: + self.assertEqual( + server._payout_ledger_append_invalidation_epoch, 1 + ) + + self.assertTrue( + server.client_needs_tip_template_refresh(state, snapshot) + ) + self.assertTrue( + server.client_tip_changed_for_snapshot(state, snapshot) + ) + self.assertTrue(server._tip_refresh_pending_event.is_set()) + self.assertTrue(server._tip_refresh_retry.is_set()) + + def test_direct_delivery_rechecks_published_digest_at_commit(self) -> None: + # A periodic self-check repair swaps the published balance snapshot + # in place: neither the payout generation nor the append epoch moves, + # so only the digest identifies a direct-delivery context keyed to + # the refuted balances. When the repair lands after the build but + # before the commit boundary, the delivery must re-check the digest + # or the miner's active job carries the refuted allocation. + server, _ledger, _artifacts = self.configured_server() + install_fake_bundle_builder(server) + server._ensure_tip_refresh_state() + state = client(1) + sent: list[dict[str, object]] = [] + state.send = sent.append # type: ignore[method-assign] + original_build_job_for_client = server.build_job_for_client + repaired = False + + def repair_after_build(*args: object, **kwargs: object) -> object: + nonlocal repaired + context = original_build_job_for_client(*args, **kwargs) # type: ignore[arg-type] + if not repaired: + repaired = True + with server._job_cache_lock: + published = server._published_payout_state.artifact + assert published is not None + self.assertEqual( + context.payout_artifact_sha256, + published.prior_balances_sha256, + ) + self.assertTrue( + server._publish_self_check_repaired_balances( + 0, + stale_prior_balances_sha256=( + published.prior_balances_sha256 + ), + balances=[ + { + "recipient_id": "carry", + "order_key": "01:carry", + "p2mr_program_hex": "66" * 32, + "balance_sats": 999, + } + ], + ) + ) + return context + + server.build_job_for_client = repair_after_build # type: ignore[method-assign] + + with patch( + "lab.prism.prism_coordinator.now_ms", + side_effect=lambda: 1_000_000, + ): + self.assertFalse(server.maybe_send_job(state, clean_jobs=True)) + self.assertEqual(sent, []) + self.assertIsNone(state.active_job) + self.assertEqual(state.active_job_ids, set()) + self.assertEqual(server.jobs, {}) + + def test_incremental_and_forced_full_artifacts_match_with_carry_boundaries( + self, + ) -> None: + class CarryLedger(IncrementalRecordingLedger): + def current_prior_balances(self) -> list[dict[str, object]]: + return [ + { + "recipient_id": "below", + "order_key": "01:below", + "p2mr_program_hex": "11" * 32, + "balance_sats": 545, + }, + { + "recipient_id": "at", + "order_key": "02:at", + "p2mr_program_hex": "22" * 32, + "balance_sats": 546, + }, + { + "recipient_id": "above", + "order_key": "03:above", + "p2mr_program_hex": "33" * 32, + "balance_sats": 547, + }, + ] + + ledger = CarryLedger() + for share_seq in range(1, 4): + append_incremental_share( + ledger, + share_seq=share_seq, + accepted_at_ms=999_900 + share_seq, + ) + server, rpc = coordinator(ledger=ledger) + artifacts = server.store_template_artifacts(dict(rpc.template)) + assert artifacts is not None + server._pool_ready_latched = True + server.payout_artifact_min_build_interval_seconds = 0.0 + clock_ms = [1_000_000] + with patch( + "lab.prism.prism_coordinator.now_ms", + side_effect=lambda: clock_ms[0], + ): + self.assertIsNotNone( + server._build_payout_ledger_artifact( + 0, 0, artifacts.network_difficulty + ) + ) + append_incremental_share( + ledger, + share_seq=4, + accepted_at_ms=1_000_010, + ) + clock_ms[0] = 1_000_020 + incremental = server._build_payout_ledger_artifact( + 0, 0, artifacts.network_difficulty + ) + forced = server._build_payout_ledger_artifact( + 0, + 0, + artifacts.network_difficulty, + True, + ) + + assert incremental is not None and forced is not None + self.assertEqual(incremental.window_build_mode, "incremental") + self.assertEqual(forced.window_build_mode, "full_rescan") + incremental_bytes = canonical_json_text( + { + "shares": list(incremental.shares_json), + "prior_balances": list(incremental.prior_balances), + } + ).encode() + forced_bytes = canonical_json_text( + { + "shares": list(forced.shares_json), + "prior_balances": list(forced.prior_balances), + } + ).encode() + self.assertEqual(incremental_bytes, forced_bytes) + self.assertEqual( + incremental.share_snapshot_sha256, + forced.share_snapshot_sha256, + ) + self.assertEqual( + incremental.prior_balances_sha256, + forced.prior_balances_sha256, + ) + + def test_normal_refresh_reuses_generation_owned_prior_balances(self) -> None: + class CountingCarryLedger(IncrementalRecordingLedger): + prior_balance_reads = 0 + + def current_prior_balances(self) -> list[dict[str, object]]: + self.prior_balance_reads += 1 + return [ + { + "recipient_id": "carry", + "order_key": "01:carry", + "p2mr_program_hex": "66" * 32, + "balance_sats": 545, + } + ] + + ledger = CountingCarryLedger() + for share_seq in range(1, 4): + append_incremental_share( + ledger, + share_seq=share_seq, + accepted_at_ms=999_900 + share_seq, + ) + server, rpc = coordinator(ledger=ledger) + artifacts = server.store_template_artifacts(dict(rpc.template)) + assert artifacts is not None + server._pool_ready_latched = True + server.payout_artifact_min_build_interval_seconds = 0.0 + clock_ms = [1_000_000] + with patch( + "lab.prism.prism_coordinator.now_ms", + side_effect=lambda: clock_ms[0], + ): + published = server._current_payout_state_artifact() + initial = server._build_payout_ledger_artifact( + 0, 0, artifacts.network_difficulty + ) + append_incremental_share( + ledger, + share_seq=4, + accepted_at_ms=1_000_010, + ) + clock_ms[0] = 1_000_020 + incremental = server._build_payout_ledger_artifact( + 0, 0, artifacts.network_difficulty + ) + forced = server._build_payout_ledger_artifact( + 0, + 0, + artifacts.network_difficulty, + True, + ) + + assert initial is not None and incremental is not None and forced is not None + self.assertEqual(ledger.prior_balance_reads, 2) + self.assertEqual( + initial.prior_balances_sha256, + published.prior_balances_sha256, + ) + self.assertEqual( + incremental.prior_balances_sha256, + published.prior_balances_sha256, + ) + + def test_fifteen_second_generation_bumps_refresh_once_per_minute(self) -> None: + server, ledger, artifacts = self.configured_server() + clock_ms = [1_000_000] + monotonic_seconds = [100.0] + modes: list[str | None] = [] + with ( + patch( + "lab.prism.prism_coordinator.now_ms", + side_effect=lambda: clock_ms[0], + ), + patch( + "lab.prism.prism_coordinator.time.monotonic", + side_effect=lambda: monotonic_seconds[0], + ), + ): + initial = server._build_payout_ledger_artifact( + 0, 0, artifacts.network_difficulty + ) + assert initial is not None + modes.append(initial.window_build_mode) + for bump in range(1, 5): + monotonic_seconds[0] = 100.0 + (15.0 * bump) + clock_ms[0] = 1_000_000 + (15_000 * bump) + append_incremental_share( + ledger, + share_seq=3 + bump, + accepted_at_ms=clock_ms[0] - 2, + ) + built = server._build_payout_ledger_artifact( + 0, 0, artifacts.network_difficulty + ) + assert built is not None + modes.append(built.window_build_mode) + + self.assertEqual( + modes, + ["full_rescan", "debounced", "debounced", "debounced", "incremental"], + ) + self.assertEqual(ledger.full_snapshot_calls, 1) + self.assertEqual(ledger.delta_snapshot_calls, 1) + + def test_found_block_candidate_bypasses_normal_interval(self) -> None: + server, ledger, artifacts = self.configured_server() + clock_ms = [1_000_000] + with patch( + "lab.prism.prism_coordinator.now_ms", + side_effect=lambda: clock_ms[0], + ): + self.assertIsNotNone( + server._build_payout_ledger_artifact( + 0, 0, artifacts.network_difficulty + ) + ) + append_incremental_share( + ledger, + share_seq=4, + accepted_at_ms=1_000_010, + ) + # Even an overdue routine oracle must be deferred from the + # immediate found-block publication path. + server.payout_artifact_full_rescan_seconds = 0.0 + clock_ms[0] = 1_000_020 + candidate = server._prepared_payout_state_candidate( + ( + 0, + 1, + str(server.rpc.tip), + "accepted_block_preview", + time.monotonic(), + ) + ) + candidate = server._accepted_block_preview_candidate( + candidate, + block_hash="44" * 32, + preview=(("winner", "01:winner", "55" * 32, 546),), + ) + + artifact = candidate.ledger_artifact + assert artifact is not None + self.assertEqual(artifact.window_build_mode, "incremental") + self.assertEqual(artifact.window_delta_rows, 1) + self.assertEqual(ledger.delta_snapshot_calls, 1) + self.assertEqual(ledger.full_snapshot_calls, 1) + self.assertEqual(len(artifact.shares_json), 4) + + def test_failed_periodic_oracle_is_spaced_from_next_delta_build(self) -> None: + class FailingOracleLedger(IncrementalRecordingLedger): + fail_full_snapshot = False + + def snapshot_at_job_issue( + self, + anchor_job_issued_at_ms: int, + *, + window_weight: int | None = None, + ) -> list[object]: + if self.fail_full_snapshot: + self.full_snapshot_calls += 1 + raise RuntimeError("oracle unavailable") + return super().snapshot_at_job_issue( + anchor_job_issued_at_ms, + window_weight=window_weight, + ) + + ledger = FailingOracleLedger() + for share_seq in range(1, 4): + append_incremental_share( + ledger, + share_seq=share_seq, + accepted_at_ms=999_900 + share_seq, + ) + server, rpc = coordinator(ledger=ledger) + artifacts = server.store_template_artifacts(dict(rpc.template)) + assert artifacts is not None + server._pool_ready_latched = True + server.payout_artifact_min_build_interval_seconds = 0.0 + server.payout_artifact_full_rescan_seconds = 10.0 + clock_ms = [1_000_000] + monotonic_seconds = [100.0] + with ( + patch( + "lab.prism.prism_coordinator.now_ms", + side_effect=lambda: clock_ms[0], + ), + patch( + "lab.prism.prism_coordinator.time.monotonic", + side_effect=lambda: monotonic_seconds[0], + ), + ): + self.assertIsNotNone( + server._build_payout_ledger_artifact( + 0, 0, artifacts.network_difficulty + ) + ) + ledger.fail_full_snapshot = True + append_incremental_share( + ledger, + share_seq=4, + accepted_at_ms=1_000_010, + ) + clock_ms[0] = 1_000_020 + monotonic_seconds[0] = 110.0 + failed_check = server._build_payout_ledger_artifact( + 0, 0, artifacts.network_difficulty + ) + assert failed_check is not None + self.assertEqual( + failed_check.window_build_mode, + "incremental_self_check_failed", + ) + + append_incremental_share( + ledger, + share_seq=5, + accepted_at_ms=1_000_021, + ) + clock_ms[0] = 1_000_030 + monotonic_seconds[0] = 111.0 + next_delta = server._build_payout_ledger_artifact( + 0, 0, artifacts.network_difficulty + ) + + assert next_delta is not None + self.assertEqual(next_delta.window_build_mode, "incremental") + self.assertEqual(ledger.full_snapshot_calls, 2) + + def test_found_block_reuses_armed_window_when_prepare_lock_is_busy( + self, + ) -> None: + class BusyPrepareLock: + def acquire(self, blocking: bool = True) -> bool: + self.assert_nonblocking(blocking) + return False + + @staticmethod + def assert_nonblocking(blocking: bool) -> None: + if blocking: + raise AssertionError("found-block path attempted to wait") + + def release(self) -> None: + raise AssertionError("unacquired lock was released") + + server, ledger, artifacts = self.configured_server() + clock_ms = [1_000_000] + with patch( + "lab.prism.prism_coordinator.now_ms", + side_effect=lambda: clock_ms[0], + ): + initial = server._build_payout_ledger_artifact( + 0, 0, artifacts.network_difficulty + ) + assert initial is not None + self.assertTrue(server._install_payout_ledger_artifact(initial)) + server._payout_state_prepare_lock = BusyPrepareLock() # type: ignore[assignment] + candidate = server._prepared_payout_state_candidate( + ( + 0, + 1, + str(server.rpc.tip), + "accepted_block_preview", + time.monotonic(), + ) + ) + candidate = server._accepted_block_preview_candidate( + candidate, + block_hash="44" * 32, + preview=(("winner", "01:winner", "55" * 32, 546),), + ) + + artifact = candidate.ledger_artifact + assert artifact is not None + self.assertEqual(artifact.window_build_mode, "found_block_cached") + self.assertEqual(artifact.payout_state_generation, 1) + self.assertEqual( + list(artifact.prior_balances), + [ + { + "recipient_id": "winner", + "order_key": "01:winner", + "p2mr_program_hex": "55" * 32, + "balance_sats": 546, + } + ], + ) + self.assertEqual(ledger.full_snapshot_calls, 1) + self.assertEqual(ledger.delta_snapshot_calls, 0) + self.assertEqual( + server.payout_artifact_event_counts["found_block_cached"], + 1, + ) + + def test_found_block_build_failure_is_visible_and_reuses_exact_window( + self, + ) -> None: + class FailingDeltaLedger(IncrementalRecordingLedger): + fail_delta = False + + def snapshot_between_job_issues( + self, + previous_anchor_job_issued_at_ms: int, + anchor_job_issued_at_ms: int, + ) -> list[object]: + if self.fail_delta: + self.delta_snapshot_calls += 1 + raise RuntimeError("delta unavailable") + return super().snapshot_between_job_issues( + previous_anchor_job_issued_at_ms, + anchor_job_issued_at_ms, + ) + + ledger = FailingDeltaLedger() + for share_seq in range(1, 4): + append_incremental_share( + ledger, + share_seq=share_seq, + accepted_at_ms=999_900 + share_seq, + ) + server, rpc = coordinator(ledger=ledger) + artifacts = server.store_template_artifacts(dict(rpc.template)) + assert artifacts is not None + server._pool_ready_latched = True + clock_ms = [1_000_000] + with patch( + "lab.prism.prism_coordinator.now_ms", + side_effect=lambda: clock_ms[0], + ): + initial = server._build_payout_ledger_artifact( + 0, 0, artifacts.network_difficulty + ) + assert initial is not None + self.assertTrue(server._install_payout_ledger_artifact(initial)) + ledger.fail_delta = True + clock_ms[0] = 1_000_020 + candidate = server._prepared_payout_state_candidate( + ( + 0, + 1, + str(server.rpc.tip), + "accepted_block_preview", + time.monotonic(), + ) + ) + + artifact = candidate.ledger_artifact + assert artifact is not None + self.assertEqual(artifact.window_build_mode, "found_block_cached") + self.assertEqual( + server.payout_artifact_event_counts["build_aborted"], + 1, + ) + self.assertEqual( + server.payout_artifact_event_counts["found_block_cached"], + 1, + ) + + def test_old_anchor_does_not_rearm_before_window_cadence(self) -> None: + class RecordingExecutor: + def __init__(self) -> None: + self.submissions = 0 + + def submit(self, _fn: object) -> Future[None]: + self.submissions += 1 + return Future() + + server, _ledger, artifacts = self.configured_server() + clock_ms = [1_000_000] + monotonic_seconds = [100.0] + with ( + patch( + "lab.prism.prism_coordinator.now_ms", + side_effect=lambda: clock_ms[0], + ), + patch( + "lab.prism.prism_coordinator.time.monotonic", + side_effect=lambda: monotonic_seconds[0], + ), + ): + initial = server._build_payout_ledger_artifact( + 0, 0, artifacts.network_difficulty + ) + assert initial is not None + self.assertTrue(server._install_payout_ledger_artifact(initial)) + executor = RecordingExecutor() + with server._payout_artifact_executor_lock: + server._payout_artifact_executor = executor # type: ignore[assignment] + + # A pending-share floor can leave the declared anchor old even + # though its exact pages were just re-proved. Past-floor probes + # must not enqueue five-second debounced balance-only builds. + clock_ms[0] = 1_061_000 + monotonic_seconds[0] = 105.0 + self.assertIsNotNone( + server._usable_payout_ledger_artifact( + 0, + artifacts.network_difficulty, + ) + ) + with server._payout_artifact_executor_lock: + self.assertIsNone(server._payout_artifact_requested) + + monotonic_seconds[0] = 160.0 + self.assertIsNotNone( + server._usable_payout_ledger_artifact( + 0, + artifacts.network_difficulty, + ) + ) + + with server._payout_artifact_executor_lock: + self.assertEqual( + server._payout_artifact_requested, + (0, int(artifacts.network_difficulty)), + ) + self.assertEqual(executor.submissions, 1) + + +@contextmanager +def _captured_payout_artifact_logs(): + """Collect the JSON payout-artifact lifecycle lines a build prints.""" + lines: list[dict[str, object]] = [] + prefix = "prism coordinator: " + + def fake_print(*args: object, **_kwargs: object) -> None: + text = " ".join(str(arg) for arg in args) + if text.startswith(prefix + "{"): + try: + lines.append(json.loads(text[len(prefix):])) + except ValueError: + pass + + with patch("builtins.print", side_effect=fake_print): + yield lines + + +class PayoutWindowRescanBoundTests(unittest.TestCase): + """Issue #228: forced full rescans are rare, attributable, and bounded.""" + + # Wall-clock ceiling a routine (tip-refresh path) candidate preparation + # may take while another build holds the prepare lock and a window inside + # PRISM_TIP_REFRESH_WINDOW_STALENESS_SECONDS is armed. The preparation + # must not queue behind the lock at all, so this is generous for a + # non-blocking path and far below any build the lock holder could be + # running; a join that times out means the path blocked. + NON_BLOCKING_BUDGET_SECONDS = 2.0 + + def configured_server( + self, + ) -> tuple[PrismCoordinator, IncrementalRecordingLedger, object]: + ledger = IncrementalRecordingLedger() + append_incremental_share(ledger, share_seq=1, accepted_at_ms=999_900) + append_incremental_share(ledger, share_seq=2, accepted_at_ms=999_910) + append_incremental_share(ledger, share_seq=3, accepted_at_ms=999_920) + server, rpc = coordinator(ledger=ledger) + artifacts = server.store_template_artifacts(dict(rpc.template)) + assert artifacts is not None + server._pool_ready_latched = True + server.payout_artifact_min_build_interval_seconds = 60.0 + server.payout_artifact_full_rescan_seconds = 3_600.0 + return server, ledger, artifacts + + @staticmethod + def _built_lines( + lines: list[dict[str, object]], + ) -> list[dict[str, object]]: + return [line for line in lines if line.get("event") == "payout_artifact_built"] + + def test_sticky_invalidation_cause_does_not_force_later_full_rescans( + self, + ) -> None: + # The defect: _capture_payout_state_source re-reads the sticky + # source tuple without consuming it, so every later capture at the + # same source generation inherited a withdrawal's cause and forced + # the full oracle again. The withdrawal's own preparation is the + # only one entitled to act on the cause it reserved. + server, ledger, artifacts = self.configured_server() + clock_ms = [1_000_000] + with patch( + "lab.prism.prism_coordinator.now_ms", + side_effect=lambda: clock_ms[0], + ), _captured_payout_artifact_logs() as lines: + initial = server._build_payout_ledger_artifact( + 0, 0, artifacts.network_difficulty + ) + assert initial is not None + self.assertEqual(ledger.full_snapshot_calls, 1) + + captured = server._capture_payout_state_source() + reserved = server._reserve_payout_state_source_if_current( + captured[1], + "accepted_block_preview_withdrawn", + tip_hash=captured[2], + ) + assert reserved is not None + # The withdrawal path's own preparation: explicit flag plus the + # reserved cause. Balances must come from the ledger; the + # append-only share window is advanced, not rescanned. + own = server._prepared_payout_state_candidate( + reserved, + force_full_window_rescan=True, + bypass_build_interval=True, + ) + assert own.ledger_artifact is not None + full_after_own = ledger.full_snapshot_calls + own_line = self._built_lines(lines)[-1] + self.assertEqual( + own_line["invalidation_cause"], + "accepted_block_preview_withdrawn", + ) + self.assertEqual( + own_line["force_origin"], "caller_flag+reserved_cause" + ) + self.assertFalse(own_line["cause_inherited"]) + self.assertTrue(own_line["payout_mutated"]) + self.assertEqual(own_line["prior_balances_source"], "ledger") + + # A routine build through the capture path at the same, still + # current, source generation must not rescan. + server.payout_artifact_min_build_interval_seconds = 0.0 + append_incremental_share( + ledger, share_seq=4, accepted_at_ms=1_000_010 + ) + clock_ms[0] = 1_000_020 + routine = server._current_payout_state_candidate() + assert routine.ledger_artifact is not None + self.assertNotEqual( + routine.ledger_artifact.window_build_mode, "full_rescan" + ) + self.assertEqual(ledger.full_snapshot_calls, full_after_own) + self.assertEqual(routine.ledger_artifact.window_delta_rows, 1) + routine_line = self._built_lines(lines)[-1] + self.assertEqual( + routine_line["invalidation_cause"], + "accepted_block_preview_withdrawn", + ) + self.assertTrue(routine_line["cause_inherited"]) + self.assertEqual(routine_line["force_origin"], "none") + self.assertFalse(routine_line["payout_mutated"]) + # The tuple itself is untouched: supersession readers still see it. + self.assertEqual( + server._payout_state_source[2], + "accepted_block_preview_withdrawn", + ) + + def test_payout_state_source_supersession_still_detected_after_the_fix( + self, + ) -> None: + server, ledger, artifacts = self.configured_server() + clock_ms = [1_000_000] + with patch( + "lab.prism.prism_coordinator.now_ms", + side_effect=lambda: clock_ms[0], + ), _captured_payout_artifact_logs(): + captured = server._capture_payout_state_source() + stale = server._prepared_payout_state_candidate(captured) + # A newer source is reserved before publication: the candidate + # prepared against generation N is discarded, not published. + server._reserve_payout_state_source( + "external_tip", tip_hash="ab" * 32 + ) + server._block_payout_state_publication(force=True) + self.assertIsNone(server._publish_payout_state_candidate(stale)) + self.assertEqual(server.payout_state_candidates_discarded, 1) + self.assertEqual(server._payout_state_generation, 0) + # Reserve-if-current refuses a superseded expectation exactly as + # before, so a withdrawal cannot attach its cause to a source + # generation that already moved. + self.assertIsNone( + server._reserve_payout_state_source_if_current( + captured[1], + "accepted_block_preview_withdrawn", + tip_hash=captured[2], + ) + ) + # The current source still publishes. + fresh = server._current_payout_state_candidate() + self.assertEqual(fresh.source_generation, server._payout_state_source[0]) + self.assertEqual(server._publish_payout_state_candidate(fresh), 1) + self.assertEqual(server._payout_state_generation, 1) + self.assertEqual( + server._published_payout_state.source_generation, + fresh.source_generation, + ) + + def test_window_build_phase_attribution_pins_the_split(self) -> None: + class SlowOracleLedger(IncrementalRecordingLedger): + oracle_delay_seconds = 0.3 + + def snapshot_at_job_issue( + self, + anchor_job_issued_at_ms: int, + *, + window_weight: int | None = None, + ) -> list[object]: + time.sleep(self.oracle_delay_seconds) + return super().snapshot_at_job_issue( + anchor_job_issued_at_ms, + window_weight=window_weight, + ) + + ledger = SlowOracleLedger() + for share_seq in range(1, 4): + append_incremental_share( + ledger, + share_seq=share_seq, + accepted_at_ms=999_900 + share_seq, + ) + server, rpc = coordinator(ledger=ledger) + artifacts = server.store_template_artifacts(dict(rpc.template)) + assert artifacts is not None + server._pool_ready_latched = True + with patch( + "lab.prism.prism_coordinator.now_ms", + return_value=1_000_000, + ), _captured_payout_artifact_logs() as lines: + built = server._build_payout_ledger_artifact( + 0, 0, artifacts.network_difficulty + ) + assert built is not None + self.assertEqual(built.window_build_mode, "full_rescan") + line = self._built_lines(lines)[-1] + phases = line["phase_seconds"] + assert isinstance(phases, dict) + self.assertEqual( + set(phases), set(PRISM_PAYOUT_WINDOW_BUILD_PHASES) + ) + self.assertEqual( + PRISM_PAYOUT_WINDOW_BUILD_PHASES, + ("ledger_read", "record_conversion", "daemon_prepare", "lock_wait"), + ) + # The known-slow ledger read lands in the ledger-read phase and not + # in conversion, daemon, or lock time. + self.assertGreaterEqual( + float(phases["ledger_read"]), ledger.oracle_delay_seconds + ) + other = ( + float(phases["record_conversion"]) + + float(phases["daemon_prepare"]) + + float(phases["lock_wait"]) + ) + self.assertLess(other, 0.1) + self.assertEqual(float(phases["daemon_prepare"]), 0.0) + # The phases account for the reported build within 0.1 s (the + # unattributed remainder is artifact construction and the cheap + # accepted-share count, both microseconds on this ledger). + tolerance_seconds = 0.1 + self.assertLessEqual( + abs(sum(float(value) for value in phases.values()) - float(line["duration_seconds"])), + tolerance_seconds, + ) + # Exported as one fixed-cardinality histogram family over the + # closed phase and outcome sets. + metrics = server.payout_state_metrics_lines() + family = "qbit_prism_payout_window_build_phase_seconds" + count_lines = [ + entry for entry in metrics if entry.startswith(f"{family}_count{{") + ] + exported_phases = { + entry.split('phase="', 1)[1].split('"', 1)[0] for entry in count_lines + } + exported_outcomes = { + entry.split('outcome="', 1)[1].split('"', 1)[0] for entry in count_lines + } + self.assertEqual(exported_phases, set(PRISM_PAYOUT_WINDOW_BUILD_PHASES)) + self.assertEqual(exported_outcomes, set(PRISM_PAYOUT_WINDOW_BUILD_OUTCOMES)) + completed = 'phase="ledger_read",outcome="completed"' + self.assertIn(f"{family}_count{{{completed}}} 1", metrics) + self.assertIn(f"# TYPE {family} histogram", metrics) + ledger_sum = [ + entry for entry in metrics if entry.startswith(f"{family}_sum{{{completed}}}") + ] + self.assertEqual(len(ledger_sum), 1) + self.assertGreaterEqual( + float(ledger_sum[0].split()[-1]), ledger.oracle_delay_seconds + ) + + def test_tip_refresh_wave_does_not_wait_for_a_full_rescan_when_a_recent_window_exists( + self, + ) -> None: + server, ledger, artifacts = self.configured_server() + recorded = install_fake_bundle_builder(server) + clock_ms = [1_000_000] + with patch( + "lab.prism.prism_coordinator.now_ms", + side_effect=lambda: clock_ms[0], + ), _captured_payout_artifact_logs() as lines: + initial = server._build_payout_ledger_artifact( + 0, 0, artifacts.network_difficulty + ) + assert initial is not None + self.assertTrue(server._install_payout_ledger_artifact(initial)) + # Publish the generation's balance artifact up front, as every + # publication does in production, so the job build below never + # needs the prepare lock for a balance read. + server._current_payout_state_artifact() + + # An invalidating build (a forced reorg or withdrawal rescan) + # owns the prepare lock for longer than the whole budget. + lock_held = threading.Event() + release_lock = threading.Event() + + def hold_prepare_lock() -> None: + with server._payout_state_prepare_lock: + lock_held.set() + release_lock.wait(30.0) + + holder = threading.Thread(target=hold_prepare_lock, daemon=True) + holder.start() + self.assertTrue(lock_held.wait(5.0)) + try: + # The routine candidate (what the tip-refresh path prepares) + # comes back inside the budget with the armed window. + candidates: list[PayoutStateCandidate] = [] + routine = threading.Thread( + target=lambda: candidates.append( + server._current_payout_state_candidate() + ), + daemon=True, + ) + routine.start() + routine.join(self.NON_BLOCKING_BUDGET_SECONDS) + self.assertFalse( + routine.is_alive(), + "routine candidate preparation queued behind the rescan", + ) + candidate = candidates[0] + artifact = candidate.ledger_artifact + assert artifact is not None + self.assertEqual(artifact.window_build_mode, "tip_refresh_cached") + self.assertEqual( + artifact.window_full_rescan_reason, "prepare_lock_busy" + ) + self.assertEqual( + artifact.share_snapshot_sha256, + initial.share_snapshot_sha256, + ) + self.assertEqual(artifact.payout_state_generation, 1) + self.assertEqual(ledger.full_snapshot_calls, 1) + self.assertEqual(ledger.delta_snapshot_calls, 0) + cached_line = [ + line + for line in lines + if line.get("event") == "payout_artifact_tip_refresh_cached" + ][-1] + self.assertLessEqual( + float(cached_line["anchor_age_ms"]), + PRISM_TIP_REFRESH_WINDOW_STALENESS_SECONDS * 1000.0, + ) + self.assertEqual( + server.payout_artifact_event_counts["tip_refresh_cached"], + 1, + ) + + # Fanout itself: a ready job build during the same window + # completes on the armed window without touching the lock. + bundles: list[object] = [] + build = threading.Thread( + target=lambda: bundles.append( + server.shared_job_bundle(artifacts, mode="ready") + ), + daemon=True, + ) + build.start() + build.join(self.NON_BLOCKING_BUDGET_SECONDS) + self.assertFalse(build.is_alive(), "job build queued behind the rescan") + self.assertEqual(len(bundles), 1) + self.assertEqual(recorded["calls"], 1) + self.assertEqual(ledger.full_snapshot_calls, 1) + + # Past the accepted staleness bound the routine path must + # queue for a fresh build instead of propagating a window + # the re-anchor has already flagged stale. + clock_ms[0] = 1_000_000 + int( + PRISM_TIP_REFRESH_WINDOW_STALENESS_SECONDS * 1000.0 + ) + 1_000 + stale_candidates: list[PayoutStateCandidate] = [] + stale_routine = threading.Thread( + target=lambda: stale_candidates.append( + server._current_payout_state_candidate() + ), + daemon=True, + ) + stale_routine.start() + stale_routine.join(0.5) + self.assertTrue( + stale_routine.is_alive(), + "a window past the staleness bound was reused", + ) + finally: + release_lock.set() + holder.join(5.0) + stale_routine.join(10.0) + self.assertFalse(stale_routine.is_alive()) + stale_artifact = stale_candidates[0].ledger_artifact + assert stale_artifact is not None + self.assertNotEqual( + stale_artifact.window_build_mode, "tip_refresh_cached" + ) + self.assertEqual( + server.payout_artifact_event_counts["tip_refresh_cached"], 1 + ) + + def test_production_shaped_cadence_keeps_full_rescans_rare(self) -> None: + # Two simulated hours at the mainnet cadence (75 s blocks), with + # accepted-block confirmations, one reorg withdrawal, and maturity + # mutations at the observed 48 h ratio (55 confirmations and 8 + # reorgs per 48 h, rounded up per two hours). The assertion is a + # count, a property of the invalidation logic, never a duration. + server, ledger, artifacts = self.configured_server() + server.payout_artifact_min_build_interval_seconds = 0.0 + difficulty = int(artifacts.network_difficulty) + block_seconds = 75 + simulated_hours = 2 + blocks = simulated_hours * 3600 // block_seconds + confirmations_at = {20, 60} + withdrawal_at = {40} + maturity_at = {30, 70} + preview = [ + { + "recipient_id": "miner-a", + "order_key": "miner-a", + "p2mr_program_hex": "11" * 32, + "balance_sats": 25, + } + ] + clock_ms = [1_000_000] + share_seq = 3 + + def drain_background() -> None: + future = server._payout_artifact_future + if future is not None: + future.result(timeout=30.0) + + def reconciler_pass(*, payout_changed: bool) -> None: + # The reorg reconciler's candidate path, minus the RPC and + # ledger mutations: prepare under the lock with the #224 + # intent (a confirmed mutation rereads balances, never the + # window), fence, publish. + with server._payout_state_prepare_lock: + captured = server._capture_payout_state_source() + if not ( + payout_changed + or server._captured_payout_source_requires_publication(captured) + ): + return + candidate = server._prepared_payout_state_candidate( + captured, + force_full_window_rescan=False, + force_prior_balances_read=payout_changed, + ) + server._block_payout_state_publication(force=True) + self.assertIsNotNone(server._publish_payout_state_candidate(candidate)) + drain_background() + + with patch( + "lab.prism.prism_coordinator.now_ms", + side_effect=lambda: clock_ms[0], + ), _captured_payout_artifact_logs() as lines: + self.assertIsNotNone( + server._build_payout_ledger_artifact(0, 0, difficulty) + ) + pending_own_block: str | None = None + next_reanchor_ms = clock_ms[0] + 60_000 + for block in range(1, blocks + 1): + # Shares land between blocks. + for _ in range(5): + share_seq += 1 + clock_ms[0] += block_seconds * 1000 // 6 + append_incremental_share( + ledger, + share_seq=share_seq, + accepted_at_ms=clock_ms[0] - 5, + ) + clock_ms[0] += block_seconds * 1000 // 6 + if pending_own_block is not None: + # Our accepted block becomes the tip: the source tip + # already names it, so no external_tip is reserved. + tip_hash = pending_own_block + pending_own_block = None + else: + tip_hash = f"{block:064x}" + if server._payout_state_source[1] != tip_hash: + server._reserve_payout_state_source( + "external_tip", tip_hash=tip_hash + ) + reconciler_pass(payout_changed=block in maturity_at) + if block in confirmations_at or block in withdrawal_at: + own_hash = f"{0xA000 + block:064x}" + server._begin_accepted_block_payout_preview( + own_hash, block_height=100 + block + ) + server._mark_accepted_block_payout_landed( + own_hash, block_height=100 + block + ) + server._publish_accepted_block_payout_preview(own_hash, preview) + drain_background() + if block in withdrawal_at: + server._clear_accepted_block_payout_preview( + own_hash, invalidate_published=True + ) + drain_background() + else: + server._clear_accepted_block_payout_preview(own_hash) + pending_own_block = own_hash + if clock_ms[0] >= next_reanchor_ms: + # The background re-anchor cadence. + server._prepare_payout_ledger_artifact( + server._payout_state_generation, difficulty + ) + next_reanchor_ms = clock_ms[0] + 60_000 + + built = self._built_lines(lines) + full_rescans = [ + line for line in built if line["window_build_mode"] == "full_rescan" + ] + reasons = sorted(str(line.get("full_rescan_reason")) for line in full_rescans) + self.assertLessEqual( + len(full_rescans), + 1 * simulated_hours, + f"full rescans per simulated hour exceeded 1: {reasons}", + ) + # The withdrawal path keeps its fail-closed full rescan (#224); the + # maturity passes never produce one. + self.assertLessEqual( + reasons.count("reconcile_invalidation"), len(withdrawal_at) + ) + self.assertEqual( + server.payout_artifact_event_counts["full_rescan"], len(full_rescans) + ) + # Every payout mutation re-read its balances from the ledger, and + # every one that was not an explicit fail-closed rescan advanced the + # append-only window instead of rescanning it. + mutated = [line for line in built if line.get("payout_mutated")] + self.assertGreaterEqual(len(mutated), len(maturity_at) + len(withdrawal_at)) + for line in mutated: + self.assertEqual(line["prior_balances_source"], "ledger") + if not line["window_oracle_forced"]: + self.assertNotEqual(line["window_build_mode"], "full_rescan") + self.assertGreaterEqual( + sum(1 for line in mutated if line["prior_balances_read_forced"]), + len(maturity_at), + ) + # Each build is attributable from its own log line. + for line in built: + self.assertIn("invalidation_cause", line) + self.assertIn("force_origin", line) + self.assertIn("cause_inherited", line) + self.assertGreater(len(built), blocks) + + def test_preview_reservation_ownership_survives_interleaved_foreign_reservation( + self, + ) -> None: + # Review finding: ownership was a single newest-wins slot, so a + # foreign reservation at a later generation displaced the preview's + # record and its own preparation fell through to the routine path, + # losing the found-block immediacy and degrade of issue #188. + class BusyPrepareLock: + def acquire(self, blocking: bool = True) -> bool: + if blocking: + raise AssertionError("found-block path attempted to wait") + return False + + def release(self) -> None: + raise AssertionError("unacquired lock was released") + + class FailingDeltaLedger(IncrementalRecordingLedger): + fail_delta = False + + def snapshot_between_job_issues( + self, + previous_anchor_job_issued_at_ms: int, + anchor_job_issued_at_ms: int, + ) -> list[object]: + if self.fail_delta: + raise RuntimeError("delta unavailable") + return super().snapshot_between_job_issues( + previous_anchor_job_issued_at_ms, + anchor_job_issued_at_ms, + ) + + ledger = FailingDeltaLedger() + for share_seq in range(1, 4): + append_incremental_share( + ledger, + share_seq=share_seq, + accepted_at_ms=999_900 + share_seq, + ) + server, rpc = coordinator(ledger=ledger) + artifacts = server.store_template_artifacts(dict(rpc.template)) + assert artifacts is not None + server._pool_ready_latched = True + server.payout_artifact_min_build_interval_seconds = 0.0 + server.payout_artifact_full_rescan_seconds = 3_600.0 + real_lock = server._payout_state_prepare_lock + clock_ms = [1_000_000] + + def interleave_foreign_work(tip_hash: str) -> None: + # Another thread captures and prepares at the preview's + # generation, then reserves the next generation, before the + # preview path reaches its own preparation. + failures: list[BaseException] = [] + + def foreign() -> None: + try: + server._prepared_payout_state_candidate( + server._capture_payout_state_source() + ) + server._reserve_payout_state_source( + "external_tip", tip_hash=tip_hash + ) + except BaseException as exc: # pragma: no cover - asserted + failures.append(exc) + + thread = threading.Thread(target=foreign, daemon=True) + thread.start() + thread.join(10.0) + self.assertFalse(thread.is_alive()) + self.assertEqual(failures, []) + + with patch( + "lab.prism.prism_coordinator.now_ms", + side_effect=lambda: clock_ms[0], + ), patch("builtins.print"): + initial = server._build_payout_ledger_artifact( + 0, 0, artifacts.network_difficulty + ) + assert initial is not None + self.assertTrue(server._install_payout_ledger_artifact(initial)) + server._current_payout_state_artifact() + + # Cycle one: a busy prepare lock. The preview's preparation must + # not wait and must degrade to the armed window as a found-block + # candidate, not as a routine one. + captured = server._capture_payout_state_source() + reserved = server._reserve_payout_state_source_if_current( + captured[1], "accepted_block_preview", tip_hash="44" * 32 + ) + assert reserved is not None + interleave_foreign_work("55" * 32) + server._payout_state_prepare_lock = BusyPrepareLock() # type: ignore[assignment] + try: + candidate = server._prepared_payout_state_candidate(reserved) + finally: + server._payout_state_prepare_lock = real_lock + artifact = candidate.ledger_artifact + assert artifact is not None + self.assertEqual(artifact.window_build_mode, "found_block_cached") + self.assertEqual(artifact.window_full_rescan_reason, "prepare_lock_busy") + self.assertEqual(candidate.cause, "accepted_block_preview") + self.assertEqual(server.payout_artifact_event_counts["tip_refresh_cached"], 0) + + # Cycle two: a build that dies with the ledger. The preview's + # preparation must degrade instead of raising. + captured = server._capture_payout_state_source() + reserved = server._reserve_payout_state_source_if_current( + captured[1], "accepted_block_preview", tip_hash="66" * 32 + ) + assert reserved is not None + interleave_foreign_work("77" * 32) + ledger.fail_delta = True + clock_ms[0] = 1_000_020 + try: + candidate = server._prepared_payout_state_candidate(reserved) + finally: + ledger.fail_delta = False + artifact = candidate.ledger_artifact + assert artifact is not None + self.assertEqual(artifact.window_build_mode, "found_block_cached") + self.assertEqual(artifact.window_full_rescan_reason, "build_failed") + self.assertEqual(server.payout_artifact_event_counts["found_block_cached"], 2) + + def test_stale_preparation_cannot_rewind_cause_consumption(self) -> None: + # Review finding: a preparation carrying an older source generation + # could reset the consumed marker, after which the newer + # generation's reserved cause was honoured a second time and forced + # a duplicate payout-mutated build. + server, ledger, artifacts = self.configured_server() + clock_ms = [1_000_000] + with patch( + "lab.prism.prism_coordinator.now_ms", + side_effect=lambda: clock_ms[0], + ), _captured_payout_artifact_logs() as lines: + # Publish the generation's balance artifact so a routine build + # can reuse the published bytes: a build that honours the + # withdrawal cause a second time would show up as a ledger read. + server._current_payout_state_artifact() + initial = server._build_payout_ledger_artifact( + 0, 0, artifacts.network_difficulty + ) + assert initial is not None + self.assertEqual( + self._built_lines(lines)[-1]["prior_balances_source"], + "published", + ) + stale_captured = server._capture_payout_state_source() + reserved = server._reserve_payout_state_source_if_current( + stale_captured[1], + "accepted_block_preview_withdrawn", + tip_hash=stale_captured[2], + ) + assert reserved is not None + own = server._prepared_payout_state_candidate( + reserved, + force_full_window_rescan=True, + bypass_build_interval=True, + ) + assert own.ledger_artifact is not None + own_line = self._built_lines(lines)[-1] + self.assertEqual(own_line["force_origin"], "caller_flag+reserved_cause") + self.assertEqual(own_line["prior_balances_source"], "ledger") + # The stale capture, taken before the reservation, is prepared + # afterwards (a slow reconciler pass would do this). It acts on + # nothing, and its candidate is discarded at publication. + server.payout_artifact_min_build_interval_seconds = 0.0 + stale = server._prepared_payout_state_candidate(stale_captured) + self.assertEqual(stale.source_generation, stale_captured[1]) + stale_line = self._built_lines(lines)[-1] + self.assertEqual(stale_line["force_origin"], "none") + self.assertFalse(stale_line["payout_mutated"]) + server._block_payout_state_publication(force=True) + self.assertIsNone(server._publish_payout_state_candidate(stale)) + # The withdrawal generation's cause is not honoured again. + append_incremental_share( + ledger, share_seq=4, accepted_at_ms=1_000_010 + ) + clock_ms[0] = 1_000_020 + routine = server._current_payout_state_candidate() + assert routine.ledger_artifact is not None + routine_line = self._built_lines(lines)[-1] + self.assertEqual( + routine_line["invalidation_cause"], + "accepted_block_preview_withdrawn", + ) + self.assertTrue(routine_line["cause_inherited"]) + self.assertEqual(routine_line["force_origin"], "none") + self.assertFalse(routine_line["payout_mutated"]) + self.assertEqual(routine_line["prior_balances_source"], "published") + + def test_owner_entitlement_survives_a_foreign_preparation_at_a_later_generation( + self, + ) -> None: + # Review finding: pruning below the consumption high-water mark + # deleted the reserver's own record once a foreign preparation at + # a later generation moved the mark, so the preview's preparation + # fell to the routine path. The entitlement now travels with the + # reserved tuple, where no bookkeeping rule can reach it: after a + # burst of foreign reservations and a foreign preparation at the + # newest of them, the owner's preparation still takes the + # immediate path, non-blocking on a busy lock and degrading when + # its build dies with the ledger. + class BusyPrepareLock: + def acquire(self, blocking: bool = True) -> bool: + if blocking: + raise AssertionError("found-block path attempted to wait") + return False + + def release(self) -> None: + raise AssertionError("unacquired lock was released") + + class FailingDeltaLedger(IncrementalRecordingLedger): + fail_delta = False + + def snapshot_between_job_issues( + self, + previous_anchor_job_issued_at_ms: int, + anchor_job_issued_at_ms: int, + ) -> list[object]: + if self.fail_delta: + raise RuntimeError("delta unavailable") + return super().snapshot_between_job_issues( + previous_anchor_job_issued_at_ms, + anchor_job_issued_at_ms, + ) + + ledger = FailingDeltaLedger() + for share_seq in range(1, 4): + append_incremental_share( + ledger, + share_seq=share_seq, + accepted_at_ms=999_900 + share_seq, + ) + server, rpc = coordinator(ledger=ledger) + artifacts = server.store_template_artifacts(dict(rpc.template)) + assert artifacts is not None + server._pool_ready_latched = True + server.payout_artifact_min_build_interval_seconds = 0.0 + server.payout_artifact_full_rescan_seconds = 3_600.0 + service = server._ensure_payout_state_service() + real_lock = server._payout_state_prepare_lock + clock_ms = [1_000_000] + + def foreign_burst_then_preparation(burst: int, tip_base: int) -> None: + failures: list[BaseException] = [] + + def foreign() -> None: + try: + for index in range(burst): + server._reserve_payout_state_source( + "external_tip", tip_hash=f"{tip_base + index:064x}" + ) + # A foreign preparation at the newest generation moves + # the consumption mark past the owner's generation. + server._prepared_payout_state_candidate( + server._capture_payout_state_source() + ) + except BaseException as exc: # pragma: no cover - asserted + failures.append(exc) + + thread = threading.Thread(target=foreign, daemon=True) + thread.start() + thread.join(10.0) + self.assertFalse(thread.is_alive()) + self.assertEqual(failures, []) + + with patch( + "lab.prism.prism_coordinator.now_ms", + side_effect=lambda: clock_ms[0], + ), patch("builtins.print"): + initial = server._build_payout_ledger_artifact( + 0, 0, artifacts.network_difficulty + ) + assert initial is not None + self.assertTrue(server._install_payout_ledger_artifact(initial)) + server._current_payout_state_artifact() + + # Cycle one: busy prepare lock after a burst and a foreign + # preparation at a later generation. + captured = server._capture_payout_state_source() + reserved = server._reserve_payout_state_source_if_current( + captured[1], "accepted_block_preview", tip_hash="44" * 32 + ) + assert reserved is not None + owner_generation = reserved[1] + foreign_burst_then_preparation(200, 0x1000) + self.assertGreater( + service._payout_source_cause_high_water or 0, owner_generation + ) + server._payout_state_prepare_lock = BusyPrepareLock() # type: ignore[assignment] + try: + candidate = server._prepared_payout_state_candidate(reserved) + finally: + server._payout_state_prepare_lock = real_lock + artifact = candidate.ledger_artifact + assert artifact is not None + self.assertEqual(artifact.window_build_mode, "found_block_cached") + self.assertEqual(artifact.window_full_rescan_reason, "prepare_lock_busy") + self.assertEqual(server.payout_artifact_event_counts["tip_refresh_cached"], 0) + # The entitlement is one-shot: presenting the reserved tuple + # again inherits, exactly like a capture would. + self.assertFalse(service._consume_payout_source_cause(reserved)) + # And the mark was not rewound by the owner's older generation. + self.assertGreater( + service._payout_source_cause_high_water or 0, owner_generation + ) + + # Cycle two: a build that dies with the ledger, same + # interleaving. The preview's preparation degrades instead of + # raising. + captured = server._capture_payout_state_source() + reserved = server._reserve_payout_state_source_if_current( + captured[1], "accepted_block_preview", tip_hash="66" * 32 + ) + assert reserved is not None + foreign_burst_then_preparation(3, 0x2000) + ledger.fail_delta = True + clock_ms[0] = 1_000_020 + try: + candidate = server._prepared_payout_state_candidate(reserved) + finally: + ledger.fail_delta = False + artifact = candidate.ledger_artifact + assert artifact is not None + self.assertEqual(artifact.window_build_mode, "found_block_cached") + self.assertEqual(artifact.window_full_rescan_reason, "build_failed") + self.assertEqual(server.payout_artifact_event_counts["found_block_cached"], 2) + + def test_aborted_and_failed_builds_still_contribute_their_phases(self) -> None: + # Review finding: phases were observed only when a build finished, + # which biased the family toward fast builds. Every exit observes + # now, under a closed outcome, so a slow read that ends in an abort + # or a failure keeps its measured time. + class SlowThenEmptyLedger(IncrementalRecordingLedger): + oracle_delay_seconds = 0.2 + raise_after_read = False + + def snapshot_at_job_issue( + self, + anchor_job_issued_at_ms: int, + *, + window_weight: int | None = None, + ) -> list[object]: + # A slow oracle that comes back empty (or dies): the build + # pays for the read and then aborts on the empty window. + self.full_snapshot_calls += 1 + time.sleep(self.oracle_delay_seconds) + if self.raise_after_read: + raise RuntimeError("postgres statement deadline expired") + return [] + + # Shares exist (so the anchor clamp is ordinary and the build gets + # as far as the read), but the oracle answers with an empty window. + ledger = SlowThenEmptyLedger() + for share_seq in range(1, 4): + append_incremental_share( + ledger, + share_seq=share_seq, + accepted_at_ms=999_900 + share_seq, + ) + server, rpc = coordinator(ledger=ledger) + artifacts = server.store_template_artifacts(dict(rpc.template)) + assert artifacts is not None + server._pool_ready_latched = True + family = "qbit_prism_payout_window_build_phase_seconds" + + def cell(metrics: list[str], phase: str, outcome: str, product: str) -> float: + prefix = f'{family}_{product}{{phase="{phase}",outcome="{outcome}"}} ' + values = [entry for entry in metrics if entry.startswith(prefix)] + self.assertEqual(len(values), 1, prefix) + return float(values[0].split()[-1]) + + with patch( + "lab.prism.prism_coordinator.now_ms", + return_value=1_000_000, + ), patch("builtins.print"): + self.assertIsNone( + server._build_payout_ledger_artifact( + 0, 0, artifacts.network_difficulty + ) + ) + metrics = server.payout_state_metrics_lines() + self.assertEqual(cell(metrics, "ledger_read", "aborted", "count"), 1) + self.assertGreaterEqual( + cell(metrics, "ledger_read", "aborted", "sum"), + ledger.oracle_delay_seconds, + ) + self.assertEqual(cell(metrics, "ledger_read", "completed", "count"), 0) + + # The read itself dies: a failed build keeps its time too. + ledger.raise_after_read = True + self.assertIsNone( + server._build_payout_ledger_artifact( + 0, 0, artifacts.network_difficulty + ) + ) + metrics = server.payout_state_metrics_lines() + self.assertEqual(cell(metrics, "ledger_read", "failed", "count"), 1) + self.assertGreaterEqual( + cell(metrics, "ledger_read", "failed", "sum"), + ledger.oracle_delay_seconds, + ) + # A lost generation race aborts before any read and is counted + # as such, with no ledger time. + ledger.raise_after_read = False + self.assertIsNone( + server._build_payout_ledger_artifact( + 7, 7, artifacts.network_difficulty + ) + ) + metrics = server.payout_state_metrics_lines() + self.assertEqual(cell(metrics, "ledger_read", "aborted", "count"), 2) + self.assertLess( + cell(metrics, "ledger_read", "aborted", "sum"), + 2 * ledger.oracle_delay_seconds, + ) + for outcome in PRISM_PAYOUT_WINDOW_BUILD_OUTCOMES: + for phase in PRISM_PAYOUT_WINDOW_BUILD_PHASES: + cell(metrics, phase, outcome, "count") + + def test_deferred_self_check_is_bounded_under_frequent_builds(self) -> None: + # Review finding: the deferral measured freshness against the last + # build, so candidates arriving faster than the staleness bound + # deferred the hourly self-check every time it came due. It is now + # bounded against the last self-check attempt, and a deferral hands + # the check to the background walker at the next publication. + server, ledger, artifacts = self.configured_server() + # Part one: no background walker at all (reuse off), builds every + # 20 s against a 100 s check interval and a 60 s bound, across two + # check intervals. The check must run on the critical path once the + # grace expires, and never later than that. + server.payout_artifact_reuse_enabled = False + server.payout_artifact_min_build_interval_seconds = 0.0 + server.payout_artifact_full_rescan_seconds = 100.0 + self.assertEqual(PRISM_TIP_REFRESH_WINDOW_STALENESS_SECONDS, 60.0) + clock_ms = [1_000_000] + monotonic_seconds = [1_000.0] + share_seq = 3 + outcomes: list[tuple[int, str, bool]] = [] + with patch( + "lab.prism.prism_coordinator.now_ms", + side_effect=lambda: clock_ms[0], + ), patch( + "lab.prism.prism_coordinator.time.monotonic", + side_effect=lambda: monotonic_seconds[0], + ), _captured_payout_artifact_logs() as lines: + initial = server._build_payout_ledger_artifact( + 0, 0, artifacts.network_difficulty + ) + assert initial is not None + for step in range(1, 17): + elapsed = 20 * step + monotonic_seconds[0] = 1_000.0 + elapsed + clock_ms[0] = 1_000_000 + elapsed * 1_000 + share_seq += 1 + append_incremental_share( + ledger, share_seq=share_seq, accepted_at_ms=clock_ms[0] - 5 + ) + candidate = server._current_payout_state_candidate() + assert candidate.ledger_artifact is not None + line = self._built_lines(lines)[-1] + outcomes.append( + ( + elapsed, + str(line["window_build_mode"]), + bool(line["self_check_deferred"]), + ) + ) + self.assertEqual( + [elapsed for elapsed, mode, _ in outcomes if mode == "self_check_match"], + [160, 320], + ) + self.assertEqual( + [elapsed for elapsed, _, deferred in outcomes if deferred], + [100, 120, 140, 260, 280, 300], + ) + self.assertEqual(ledger.full_snapshot_calls, 3) + self.assertIsNone(server._payout_artifact_future) + + # Part two: with the walker available, a deferral is handed to it at + # the next publication and the check runs there, off the critical + # path, before the grace expires. + server, ledger, artifacts = self.configured_server() + server.payout_artifact_min_build_interval_seconds = 0.0 + server.payout_artifact_full_rescan_seconds = 100.0 + clock_ms = [1_000_000] + monotonic_seconds = [1_000.0] + share_seq = 3 + with patch( + "lab.prism.prism_coordinator.now_ms", + side_effect=lambda: clock_ms[0], + ), patch( + "lab.prism.prism_coordinator.time.monotonic", + side_effect=lambda: monotonic_seconds[0], + ), _captured_payout_artifact_logs() as lines: + initial = server._build_payout_ledger_artifact( + 0, 0, artifacts.network_difficulty + ) + assert initial is not None + self.assertTrue(server._install_payout_ledger_artifact(initial)) + server._current_payout_state_artifact() + candidate: PayoutStateCandidate | None = None + for step in range(1, 6): + elapsed = 20 * step + monotonic_seconds[0] = 1_000.0 + elapsed + clock_ms[0] = 1_000_000 + elapsed * 1_000 + share_seq += 1 + append_incremental_share( + ledger, share_seq=share_seq, accepted_at_ms=clock_ms[0] - 5 + ) + candidate = server._current_payout_state_candidate() + assert candidate is not None and candidate.ledger_artifact is not None + deferred_line = self._built_lines(lines)[-1] + self.assertTrue(deferred_line["self_check_deferred"]) + self.assertEqual(deferred_line["window_build_mode"], "incremental") + self.assertEqual(ledger.full_snapshot_calls, 1) + server._block_payout_state_publication(force=True) + self.assertEqual(server._publish_payout_state_candidate(candidate), 1) + future = server._payout_artifact_future + self.assertIsNotNone(future, "publication did not hand the owed check to the walker") + assert future is not None + future.result(timeout=30.0) + background = [ + line + for line in self._built_lines(lines) + if line["invalidation_cause"] is None + and line["window_build_mode"] == "self_check_match" + ] + self.assertEqual(len(background), 1) + self.assertEqual(background[0]["payout_state_generation"], 1) + self.assertEqual(ledger.full_snapshot_calls, 2) + # The next routine build is not overdue and defers nothing. + monotonic_seconds[0] = 1_120.0 + clock_ms[0] = 1_120_000 + share_seq += 1 + append_incremental_share( + ledger, share_seq=share_seq, accepted_at_ms=clock_ms[0] - 5 + ) + later = server._current_payout_state_candidate() + assert later.ledger_artifact is not None + later_line = self._built_lines(lines)[-1] + self.assertFalse(later_line["self_check_deferred"]) + self.assertEqual(later_line["window_build_mode"], "incremental") + self.assertEqual(ledger.full_snapshot_calls, 2) + + +class UnfencedAppendDrainTests(unittest.TestCase): + def test_drain_waits_in_stamped_slices_without_returning_early(self) -> None: + """The drain stays unbounded; only its silence is broken up. + + A landing calls this inline on the watchdog-monitored block-work + thread right after exposing its declared anchor. Bounding the wait is + not an option -- it enforces the fencing contract, and returning + early would let the landing arm its epoch fences ahead of an append + that must be drained first -- so the fix is to wait in + watchdog-sized slices and stamp each one. The ``while`` re-check + makes the timed wait semantically identical to the untimed one. + """ + server, _rpc = coordinator() + server._ensure_job_cache_state() + stamps: list[tuple[str, object]] = [] + server._record_heartbeat = ( # type: ignore[method-assign] + lambda name, phase=None: stamps.append((name, phase)) + ) + server._block_work_wait_slice = lambda: 0.01 # type: ignore[method-assign] + anchor = 1000 + with server._payout_unfenced_append_drained: + server._payout_unfenced_append_inflight_stamps[1] = anchor - 1 + returned = threading.Event() + + def drain() -> None: + server._block_submitter_thread_ident = threading.get_ident() + server._await_unfenced_appends_predating_anchor(anchor) + returned.set() + + worker_thread = threading.Thread(target=drain) + worker_thread.start() + try: + # The predating append is still in flight: no number of expired + # slices may let the drain return. + self.assertFalse(returned.wait(0.2)) + drain_stamps = [ + stamp + for stamp in stamps + if stamp == ("block_submitter", "wait-unfenced-append-drain") + ] + self.assertGreaterEqual(len(drain_stamps), 2) + with server._payout_unfenced_append_drained: + server._payout_unfenced_append_inflight_stamps.pop(1, None) + server._payout_unfenced_append_drained.notify_all() + self.assertTrue(returned.wait(5)) + finally: + with server._payout_unfenced_append_drained: + server._payout_unfenced_append_inflight_stamps.pop(1, None) + server._payout_unfenced_append_drained.notify_all() + worker_thread.join(5) + + def test_drain_with_nothing_in_flight_stamps_nothing(self) -> None: + """The common path never enters the loop, so it never stamps.""" + server, _rpc = coordinator() + server._ensure_job_cache_state() + stamps: list[object] = [] + server._record_heartbeat = ( # type: ignore[method-assign] + lambda name, phase=None: stamps.append((name, phase)) + ) + server._block_submitter_thread_ident = threading.get_ident() + server._await_unfenced_appends_predating_anchor(1000) + self.assertEqual(stamps, []) + + +class AcceptedParentPreviewBackpressureTests(unittest.TestCase): + """#182: the bounded accepted-parent preview wait is typed backpressure.""" + + def test_bounded_wait_timeout_raises_typed_preview_pending(self) -> None: + server, _rpc = coordinator() + server.accepted_block_payout_preview_wait_seconds = 0.0 + block_hash = "b1" * 32 + server._begin_accepted_block_payout_preview(block_hash, block_height=10) + server._mark_accepted_block_payout_landed(block_hash, block_height=10) + before = int(getattr(server, "_accepted_parent_preview_wait_timeouts", 0)) + + with self.assertRaises(AcceptedParentPayoutPreviewPending) as caught: + server._await_pending_parent_payout_preview(block_hash) + + exc = caught.exception + # Compatibility: every caller that catches either parent still catches + # this, so no existing handler changes behavior. + self.assertIsInstance(exc, _PayoutStatePublicationBlocked) + self.assertIsInstance(exc, TemplateRefreshBlocked) + # An explicit, machine-readable retry/log reason. + self.assertEqual(exc.retry_reason, "accepted_parent_preview_pending") + self.assertEqual(exc.parent_hash, block_hash) + self.assertEqual(exc.waited_seconds, 0.0) + self.assertIn("not ready yet", str(exc)) + # The existing timeout counter stays meaningful and is carried on the + # exception so a retry log can report it without a second read. + self.assertEqual(exc.timeout_count, before + 1) + self.assertEqual( + server._accepted_parent_preview_wait_timeouts, before + 1 + ) + server._clear_accepted_block_payout_preview(block_hash) + + def test_published_preview_and_terminal_transition_do_not_raise(self) -> None: + server, _rpc = coordinator() + server.accepted_block_payout_preview_wait_seconds = 0.0 + block_hash = "b2" * 32 + preview = [ + { + "recipient_id": "miner-a", + "order_key": "miner-a", + "p2mr_program_hex": "11" * 32, + "balance_sats": 25, + } + ] + server._begin_accepted_block_payout_preview(block_hash, block_height=10) + server._mark_accepted_block_payout_landed(block_hash, block_height=10) + with patch("builtins.print"): + server._publish_accepted_block_payout_preview(block_hash, preview) + + # A published preview is the success path: no backpressure, no timeout. + self.assertEqual( + server._await_pending_parent_payout_preview(block_hash), preview + ) + self.assertEqual( + int(getattr(server, "_accepted_parent_preview_wait_timeouts", 0)), 0 + ) + # A parent with no governing transition returns None untouched. + server._clear_accepted_block_payout_preview(block_hash) + self.assertIsNone(server._await_pending_parent_payout_preview(block_hash)) + self.assertEqual( + int(getattr(server, "_accepted_parent_preview_wait_timeouts", 0)), 0 + ) + + def test_withdrawn_preview_keeps_its_existing_type(self) -> None: + server, _rpc = coordinator() + server.accepted_block_payout_preview_wait_seconds = 0.0 + block_hash = "b3" * 32 + server._begin_accepted_block_payout_preview(block_hash, block_height=10) + server._mark_accepted_block_payout_landed(block_hash, block_height=10) + server._clear_accepted_block_payout_preview( + block_hash, + invalidate_published=True, + ) + + with self.assertRaisesRegex( + TemplateRefreshBlocked, "was withdrawn" + ) as caught: + server._await_pending_parent_payout_preview(block_hash) + + self.assertNotIsInstance( + caught.exception, AcceptedParentPayoutPreviewPending + ) + # A withdrawal is not a bounded-wait expiry, so it never touches the + # preview-wait timeout counter. + self.assertEqual( + int(getattr(server, "_accepted_parent_preview_wait_timeouts", 0)), 0 + ) + + def test_unresolved_depth_cap_keeps_its_existing_type(self) -> None: + server, _rpc = coordinator() + server.accepted_parent_unresolved_depth_max = 1 + server.accepted_block_payout_preview_wait_seconds = 0.0 + hashes = ["b4" * 32, "b5" * 32] + for height, block_hash in enumerate(hashes, start=10): + server._begin_accepted_block_payout_preview( + block_hash, block_height=height + ) + server._mark_accepted_block_payout_landed( + block_hash, block_height=height + ) + + with self.assertRaisesRegex( + TemplateRefreshBlocked, "exceeds cap" + ) as caught: + server._await_pending_parent_payout_preview(hashes[0]) + + self.assertNotIsInstance( + caught.exception, AcceptedParentPayoutPreviewPending + ) + self.assertEqual( + int(getattr(server, "_accepted_parent_preview_wait_timeouts", 0)), 0 + ) + for block_hash in hashes: + server._clear_accepted_block_payout_preview(block_hash) + + def test_owed_replay_enumeration_keeps_its_existing_type(self) -> None: + server, _rpc = coordinator() + + class EnumLedger(FakeLedger): + def pending_block_candidate_rows( + self, *, limit: int = 32 + ) -> list[dict[str, object]]: + return [] + + server.ledger = EnumLedger() + server._note_block_replay_enumeration_owed() + + with self.assertRaisesRegex( + TemplateRefreshBlocked, "has not enumerated" + ) as caught: + server._await_pending_parent_payout_preview("b6" * 32) + + self.assertNotIsInstance( + caught.exception, AcceptedParentPayoutPreviewPending + ) + + def test_preview_pending_uses_the_coordination_blocked_budget(self) -> None: + """The typed raise reaches the refresh poller as coordination churn. + + Sustained pendency accrues the separate coordination-blocked streak + (which the publication-progress watchdog enforces) instead of arming + the ordinary template-refresh failure budget, so it neither retries + silently forever nor exits on the wrong clock. + """ + server, _rpc = coordinator() + server.reorg_reconciler_enabled = True + server.accepted_block_payout_preview_wait_seconds = 0.0 + block_hash = "b7" * 32 + server._begin_accepted_block_payout_preview(block_hash, block_height=10) + server._mark_accepted_block_payout_landed(block_hash, block_height=10) + server.fetch_qbit_tip_template_snapshot = ( # type: ignore[method-assign] + lambda: (_ for _ in ()).throw( + AcceptedParentPayoutPreviewPending( + "accepted parent payout preview is not ready yet" + ) + ) + ) + + with ( + patch( + "lab.prism.prism_coordinator.time.monotonic", + return_value=100.0, + ), + self.assertRaises(AcceptedParentPayoutPreviewPending), + ): + server.poll_qbit_tip_template_once() + + self.assertIsNone( + getattr(server, "template_refresh_failure_started_monotonic", None) + ) + self.assertFalse(server.template_refresh_failure_expired(10_000.0)) + # The long-lived coordination escalation still runs on its own clock. + self.assertEqual( + server.coordination_blocked_streak_age_seconds(105.0), 5.0 + ) + server._clear_accepted_block_payout_preview(block_hash) + + def test_plain_template_refresh_blocked_still_uses_generic_budget(self) -> None: + """Negative control: only the typed subclass is exempt.""" + server, _rpc = coordinator() + server.reorg_reconciler_enabled = True + server.template_refresh_failure_exit_seconds = 10.0 + server.fetch_qbit_tip_template_snapshot = ( # type: ignore[method-assign] + lambda: (_ for _ in ()).throw( + TemplateRefreshBlocked( + "unable to derive exact artifacts for observed qbit template" + ) + ) + ) + + with ( + patch( + "lab.prism.prism_coordinator.time.monotonic", + return_value=100.0, + ), + self.assertRaises(TemplateRefreshBlocked), + ): + server.poll_qbit_tip_template_once() + + self.assertEqual( + server.template_refresh_failure_started_monotonic, 100.0 + ) + self.assertTrue(server.template_refresh_failure_expired(110.0)) + self.assertEqual( + server.coordination_blocked_streak_age_seconds(110.0), 0.0 + ) + + +class SnapshotWindowWeightBandTests(unittest.TestCase): + def test_equal_weights_are_within_band(self) -> None: + self.assertTrue(_snapshot_window_weight_within_band(1_000, 1_000)) + + def test_floor_boundary_is_exact(self) -> None: + # cached * 8 >= live * 5 holds at exactly 5/8 and fails one below. + self.assertTrue(_snapshot_window_weight_within_band(625, 1_000)) + self.assertFalse(_snapshot_window_weight_within_band(624, 1_000)) + + def test_ceiling_boundary_is_exact(self) -> None: + self.assertTrue(_snapshot_window_weight_within_band(2_000, 1_000)) + self.assertFalse(_snapshot_window_weight_within_band(2_001, 1_000)) + + def test_non_positive_weights_fail_closed(self) -> None: + self.assertFalse(_snapshot_window_weight_within_band(0, 1_000)) + self.assertFalse(_snapshot_window_weight_within_band(1_000, 0)) + self.assertFalse(_snapshot_window_weight_within_band(-16, -16)) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_prism_payout_window_daemon_recenter.py b/tests/test_prism_payout_window_daemon_recenter.py new file mode 100644 index 000000000..f52de2bf6 --- /dev/null +++ b/tests/test_prism_payout_window_daemon_recenter.py @@ -0,0 +1,501 @@ +#!/usr/bin/env python3 +"""Issue #207: the periodic self-check must leave the Rust daemon prepared. + +The daemon here is an in-process fake that serves ``prepare_window`` with +the real Python fold, exactly like the subprocess fixture the window-pipeline +suite uses, but addressable from the test: it counts full preparations and +advances, answers ``needs_full`` for a digest it does not hold, and can be +evicted on demand. That makes three coordinator contracts checkable without +a Rust build: + +- ordinary in-band drift keeps the cached weight and digest warm, so the + next build advances through the daemon without a second oracle read; +- an oversized re-center adopts the live-weight digest AND prepares the + daemon for it before the next advance, so the deliberate re-center never + pays a second full ``snapshot_at_job_issue`` under the writer lock; +- a genuine eviction still reports ``window_daemon_state_lost`` and takes + the existing recovery path, and the strings stay distinguishable from a + deliberate re-center that could not prepare the daemon. +""" +# ruff: noqa: F403, F405 + +from __future__ import annotations + +import os +import unittest + +from lab.prism.bundle_compiler import PreparedWindowOutcome +from lab.prism.payout_state import PRISM_REWARD_WINDOW_MULTIPLIER +from lab.prism.payout_state import PRISM_SNAPSHOT_WINDOW_MARGIN +from lab.prism.share_ledger import ( + AcceptedShareRecord, + DaemonShareWindowMirror, + IncrementalShareWindow, + IncrementalWindowFallback, + PendingShare, +) +from tests.prism_coordinator_test_support import * + + +def _canonical_items(window: IncrementalShareWindow) -> bytes: + return b",".join( + page.canonical_json_items + for page in window.pages + if page.canonical_json_items + ) + + +class InProcessFakeDaemon: + """``prepare_payout_window`` served by the shipped Python fold. + + Held windows are addressed by digest alone, like the real daemon; an + advance against a digest that is not held answers ``needs_full``. + """ + + def __init__(self) -> None: + self.held: dict[str, IncrementalShareWindow] = {} + self.full_prepares = 0 + self.advances = 0 + self.needs_full_answers = 0 + self.full_prepare_statuses: list[str] = [] + # Statuses to answer the next full preparations with instead of + # folding (fault injection for the unprepared-adoption test). + self.inject_full_statuses: list[str] = [] + + def evict(self) -> None: + self.held.clear() + + def __call__( + self, + *, + mode: str, + records_json: list[dict[str, object]], + anchor_job_issued_at_ms: int, + append_invalidation_epoch: int, + base_digest: str | None = None, + window_weight: int | None = None, + page_size: int | None = None, + wait_for_daemon: bool = True, + ) -> PreparedWindowOutcome | None: + records = [ + AcceptedShareRecord( + **{ + key: value + for key, value in record_json.items() + if value is not None or key == "credit_policy" + } + ) + for record_json in records_json + ] + anchor = int(anchor_job_issued_at_ms) + if mode == "full": + self.full_prepares += 1 + if self.inject_full_statuses: + status = self.inject_full_statuses.pop(0) + self.full_prepare_statuses.append(status) + return PreparedWindowOutcome(status=status) + assert window_weight is not None + window = IncrementalShareWindow.from_full_snapshot( + records, + anchor_job_issued_at_ms=anchor, + window_weight=int(window_weight), + page_size=int(page_size or 512), + ) + digest = window.json_records().canonical_json_sha256() + self.held = {digest: window} + self.full_prepare_statuses.append("prepared") + return PreparedWindowOutcome( + status="prepared", + share_snapshot_sha256=digest, + record_count=window.record_count, + window_items=_canonical_items(window), + ) + assert mode == "advance", mode + base = self.held.get(str(base_digest)) + if base is None: + self.needs_full_answers += 1 + return PreparedWindowOutcome( + status="needs_full", + error=f"prepared window {base_digest} is not held", + ) + old_items = _canonical_items(base) + try: + advanced, stats = base.advance( + records, + anchor_job_issued_at_ms=anchor, + ) + except IncrementalWindowFallback as error: + return PreparedWindowOutcome(status="fallback", error=str(error)) + digest = advanced.json_records().canonical_json_sha256() + new_items = _canonical_items(advanced) + drop = 0 + while drop <= len(old_items): + retained = old_items[drop:] + if new_items[: len(retained)] == retained: + break + drop += 1 + appended = new_items[len(old_items) - drop :] + self.held = {digest: advanced} + self.advances += 1 + return PreparedWindowOutcome( + status="prepared", + share_snapshot_sha256=digest, + record_count=advanced.record_count, + added_rows=stats.added_rows, + expired_rows=stats.expired_rows, + touched_pages=stats.touched_pages, + retained_drop_bytes=drop, + appended_items=appended, + ) + + +def _append_heavy_share( + ledger: SingleWriterShareLedger, + *, + share_seq: int, + accepted_at_ms: int, + share_difficulty: int, +) -> None: + """A share heavy enough that a handful of them fill the reward window.""" + ledger.append( + PendingShare( + share_id=f"heavy-{share_seq}", + miner_id=f"miner-{share_seq % 3}", + order_key=f"miner-{share_seq % 3}", + p2mr_program_hex=f"{share_seq % 256:02x}" * 32, + share_difficulty=share_difficulty, + network_difficulty=share_difficulty, + template_height=9, + job_id=f"job-{share_seq}", + job_issued_at_ms=accepted_at_ms - 1, + accepted_at_ms=accepted_at_ms, + ntime=1_700_000_000 + share_seq, + ) + ) + + +class DaemonRecenterTests(unittest.TestCase): + """Issue #207 regression tests against the in-process fake daemon.""" + + def setUp(self) -> None: + patcher = patch.dict(os.environ, {"PRISM_WINDOW_PIPELINE_RUST": "1"}) + patcher.start() + self.addCleanup(patcher.stop) + + def _server( + self, + ) -> tuple[PrismCoordinator, IncrementalRecordingLedger, object, InProcessFakeDaemon]: + ledger = IncrementalRecordingLedger() + server, rpc = coordinator(ledger=ledger) + artifacts = server.store_template_artifacts(dict(rpc.template)) + assert artifacts is not None + difficulty = int(artifacts.network_difficulty) + # Twenty shares each worth one unit of network difficulty: the + # window at the live weight (16 * difficulty) retains sixteen, so a + # re-center to a smaller weight changes the retained set and hence + # the digest the daemon must hold. + for share_seq in range(1, 21): + _append_heavy_share( + ledger, + share_seq=share_seq, + accepted_at_ms=999_000 + share_seq * 10, + share_difficulty=difficulty, + ) + server._pool_ready_latched = True + server.payout_artifact_min_build_interval_seconds = 60.0 + server.payout_artifact_full_rescan_seconds = 3_600.0 + daemon = InProcessFakeDaemon() + server.prepare_payout_window = daemon # type: ignore[method-assign] + return server, ledger, artifacts, daemon + + @staticmethod + def _oracle_digest( + ledger: IncrementalRecordingLedger, + *, + anchor_ms: int, + window_weight: int, + ) -> str: + records = list( + SingleWriterShareLedger.snapshot_at_job_issue( + ledger, + anchor_ms, + window_weight=None, + ) + ) + window = IncrementalShareWindow.from_full_snapshot( + records, + anchor_job_issued_at_ms=anchor_ms, + window_weight=window_weight, + ) + return window.json_records().canonical_json_sha256() + + def test_oversized_recenter_leaves_the_daemon_prepared(self) -> None: + server, ledger, artifacts, daemon = self._server() + difficulty = int(artifacts.network_difficulty) + clock_ms = [1_000_000] + with patch( + "lab.prism.prism_coordinator.now_ms", + side_effect=lambda: clock_ms[0], + ): + initial = server._build_payout_ledger_artifact(0, 0, difficulty) + assert initial is not None + self.assertEqual(initial.window_build_mode, "full_rescan") + self.assertEqual(daemon.full_prepares, 1) + self.assertEqual(ledger.full_snapshot_calls, 1) + + # Difficulty collapses far enough (cached/live = 4/3 > 5/4) that + # the periodic self-check re-centers the window at the live + # weight; the adopted digest differs from the daemon's. + server.payout_artifact_min_build_interval_seconds = 0.0 + server.payout_artifact_full_rescan_seconds = 0.0 + clock_ms[0] = 1_000_020 + collapsed_difficulty = difficulty * 3 // 4 + live_weight = ( + PRISM_REWARD_WINDOW_MULTIPLIER + * PRISM_SNAPSHOT_WINDOW_MARGIN + * collapsed_difficulty + ) + checked = server._build_payout_ledger_artifact( + 0, 0, collapsed_difficulty + ) + assert checked is not None + self.assertEqual(checked.window_build_mode, "self_check_match") + self.assertEqual( + checked.window_full_rescan_reason, "periodic_self_check" + ) + # The deliberate re-center is flagged on the build itself. + self.assertTrue(checked.window_self_check_recentered) + self.assertFalse(initial.window_self_check_recentered) + self.assertEqual(ledger.full_snapshot_calls, 2) + cached = server._incremental_payout_artifact_window + assert cached is not None + self.assertIsInstance(cached.window, DaemonShareWindowMirror) + self.assertEqual(int(cached.window.window_weight), live_weight) + self.assertIsNone(cached.daemon_unprepared_reason) + # Byte-exact parity with the in-process oracle at the live + # weight, and the daemon holds exactly that digest. + self.assertNotEqual( + checked.share_snapshot_sha256, initial.share_snapshot_sha256 + ) + self.assertEqual( + checked.share_snapshot_sha256, + self._oracle_digest( + ledger, + anchor_ms=int(checked.snapshot_anchor_ms or 0), + window_weight=live_weight, + ), + ) + self.assertEqual(daemon.full_prepares, 2) + self.assertIn(checked.share_snapshot_sha256, daemon.held) + + # The next build advances incrementally: no second full ledger + # snapshot, no needs_full, no window_daemon_state_lost. + server.payout_artifact_full_rescan_seconds = 3_600.0 + _append_heavy_share( + ledger, + share_seq=21, + accepted_at_ms=1_000_030, + share_difficulty=difficulty, + ) + clock_ms[0] = 1_000_040 + advanced = server._build_payout_ledger_artifact( + 0, 0, collapsed_difficulty + ) + assert advanced is not None + self.assertEqual(advanced.window_build_mode, "incremental") + self.assertIsNone(advanced.window_full_rescan_reason) + self.assertEqual(advanced.window_delta_rows, 1) + self.assertEqual(ledger.full_snapshot_calls, 2) + self.assertEqual(daemon.needs_full_answers, 0) + self.assertEqual(daemon.full_prepares, 2) + self.assertEqual( + advanced.share_snapshot_sha256, + self._oracle_digest( + ledger, + anchor_ms=int(advanced.snapshot_anchor_ms or 0), + window_weight=live_weight, + ), + ) + + def test_in_band_drift_keeps_the_cached_weight_and_digest_warm(self) -> None: + server, ledger, artifacts, daemon = self._server() + difficulty = int(artifacts.network_difficulty) + cached_weight = ( + PRISM_REWARD_WINDOW_MULTIPLIER + * PRISM_SNAPSHOT_WINDOW_MARGIN + * difficulty + ) + clock_ms = [1_000_000] + with patch( + "lab.prism.prism_coordinator.now_ms", + side_effect=lambda: clock_ms[0], + ): + initial = server._build_payout_ledger_artifact(0, 0, difficulty) + assert initial is not None + self.assertEqual(daemon.full_prepares, 1) + + # ASERT-sized drift: inside the band and far from the 1.25x + # re-center threshold. The self-check compares and adopts at + # the cached weight, whose digest the daemon already holds. + server.payout_artifact_min_build_interval_seconds = 0.0 + server.payout_artifact_full_rescan_seconds = 0.0 + clock_ms[0] = 1_000_020 + drifted_difficulty = (difficulty * 101 + 99) // 100 + checked = server._build_payout_ledger_artifact( + 0, 0, drifted_difficulty + ) + assert checked is not None + self.assertEqual(checked.window_build_mode, "self_check_match") + self.assertEqual( + checked.window_full_rescan_reason, "periodic_self_check" + ) + self.assertFalse(checked.window_self_check_recentered) + self.assertEqual( + checked.share_snapshot_sha256, initial.share_snapshot_sha256 + ) + cached = server._incremental_payout_artifact_window + assert cached is not None + self.assertIsInstance(cached.window, DaemonShareWindowMirror) + self.assertEqual(int(cached.window.window_weight), cached_weight) + self.assertIsNone(cached.daemon_unprepared_reason) + # Warm: no second full preparation was needed. + self.assertEqual(daemon.full_prepares, 1) + self.assertIn(checked.share_snapshot_sha256, daemon.held) + + server.payout_artifact_full_rescan_seconds = 3_600.0 + _append_heavy_share( + ledger, + share_seq=21, + accepted_at_ms=1_000_030, + share_difficulty=difficulty, + ) + clock_ms[0] = 1_000_040 + advanced = server._build_payout_ledger_artifact( + 0, 0, drifted_difficulty + ) + assert advanced is not None + self.assertEqual(advanced.window_build_mode, "incremental") + self.assertEqual(advanced.window_delta_rows, 1) + self.assertEqual(ledger.full_snapshot_calls, 2) + self.assertEqual(daemon.needs_full_answers, 0) + self.assertEqual(daemon.full_prepares, 1) + + def test_genuine_daemon_eviction_still_reports_window_daemon_state_lost( + self, + ) -> None: + server, ledger, artifacts, daemon = self._server() + difficulty = int(artifacts.network_difficulty) + clock_ms = [1_000_000] + with patch( + "lab.prism.prism_coordinator.now_ms", + side_effect=lambda: clock_ms[0], + ): + initial = server._build_payout_ledger_artifact(0, 0, difficulty) + assert initial is not None + # A respawn or LRU eviction: the daemon forgets its prepared + # state while the coordinator mirror stays digest-verified. + daemon.evict() + _append_heavy_share( + ledger, + share_seq=21, + accepted_at_ms=1_000_010, + share_difficulty=difficulty, + ) + server.payout_artifact_min_build_interval_seconds = 0.0 + clock_ms[0] = 1_000_020 + rebuilt = server._build_payout_ledger_artifact(0, 0, difficulty) + assert rebuilt is not None + self.assertEqual(rebuilt.window_build_mode, "full_rescan") + self.assertEqual( + rebuilt.window_full_rescan_reason, + "window_daemon_state_lost", + ) + # The existing recovery path: one oracle read re-seeds the + # daemon through the full preparation and the cache stays a + # daemon mirror the next build can advance. + self.assertEqual(daemon.needs_full_answers, 1) + self.assertEqual(daemon.full_prepares, 2) + self.assertEqual(ledger.full_snapshot_calls, 2) + cached = server._incremental_payout_artifact_window + assert cached is not None + self.assertIsInstance(cached.window, DaemonShareWindowMirror) + self.assertIn(rebuilt.share_snapshot_sha256, daemon.held) + self.assertEqual( + rebuilt.share_snapshot_sha256, + self._oracle_digest( + ledger, + anchor_ms=int(rebuilt.snapshot_anchor_ms or 0), + window_weight=PRISM_REWARD_WINDOW_MULTIPLIER + * PRISM_SNAPSHOT_WINDOW_MARGIN + * difficulty, + ), + ) + # A genuine eviction is never flagged as a re-center, so + # telemetry can tell the two apart. + self.assertFalse(rebuilt.window_self_check_recentered) + + def test_unprepared_recenter_is_distinguishable_from_daemon_state_loss( + self, + ) -> None: + # When the daemon cannot be prepared at re-center time (busy under + # another build), the adopted mirror is tagged with the daemon's own + # outcome and the next build's rebuild names that outcome rather + # than blaming a daemon eviction; recovery is the existing + # full-oracle path. The reason stays inside the closed vocabulary + # the #224 telemetry owner exports. + server, ledger, artifacts, daemon = self._server() + difficulty = int(artifacts.network_difficulty) + clock_ms = [1_000_000] + with patch( + "lab.prism.prism_coordinator.now_ms", + side_effect=lambda: clock_ms[0], + ): + initial = server._build_payout_ledger_artifact(0, 0, difficulty) + assert initial is not None + server.payout_artifact_min_build_interval_seconds = 0.0 + server.payout_artifact_full_rescan_seconds = 0.0 + clock_ms[0] = 1_000_020 + collapsed_difficulty = difficulty * 3 // 4 + daemon.inject_full_statuses = ["busy"] + checked = server._build_payout_ledger_artifact( + 0, 0, collapsed_difficulty + ) + assert checked is not None + self.assertEqual(checked.window_build_mode, "self_check_match") + self.assertEqual( + checked.window_full_rescan_reason, "periodic_self_check" + ) + self.assertTrue(checked.window_self_check_recentered) + cached = server._incremental_payout_artifact_window + assert cached is not None + self.assertEqual(cached.daemon_unprepared_reason, "window_daemon_busy") + self.assertEqual(daemon.full_prepare_statuses, ["prepared", "busy"]) + + server.payout_artifact_full_rescan_seconds = 3_600.0 + _append_heavy_share( + ledger, + share_seq=21, + accepted_at_ms=1_000_030, + share_difficulty=difficulty, + ) + clock_ms[0] = 1_000_040 + rebuilt = server._build_payout_ledger_artifact( + 0, 0, collapsed_difficulty + ) + assert rebuilt is not None + self.assertEqual(rebuilt.window_build_mode, "full_rescan") + self.assertEqual(rebuilt.window_full_rescan_reason, "window_daemon_busy") + self.assertNotEqual( + rebuilt.window_full_rescan_reason, "window_daemon_state_lost" + ) + self.assertFalse(rebuilt.window_self_check_recentered) + self.assertEqual(daemon.needs_full_answers, 1) + self.assertEqual(daemon.full_prepares, 3) + recovered = server._incremental_payout_artifact_window + assert recovered is not None + self.assertIsNone(recovered.daemon_unprepared_reason) + self.assertIn(rebuilt.share_snapshot_sha256, daemon.held) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_prism_pending_block_recovery.py b/tests/test_prism_pending_block_recovery.py new file mode 100644 index 000000000..93c0e425f --- /dev/null +++ b/tests/test_prism_pending_block_recovery.py @@ -0,0 +1,520 @@ +"""Focused recovery safety tests; native cases run in the PostgreSQL gate.""" + +import hashlib +import json +import os +import tempfile +import threading +import unittest +import uuid +from contextlib import nullcontext +from pathlib import Path +from types import SimpleNamespace +from unittest.mock import Mock, patch + +from lab.auxpow.stratum_codec import header_hash_hex +from lab.prism import direct_stratum +from lab.prism import recover_pending_blocks as recovery +from lab.prism.block_candidates import block_candidate_from_intent +from lab.prism.coordinator_config import load_coordinator_config +from lab.prism.recovery_json import cooperative_json +from lab.prism.share_ledger import PsqlShareLedger, block_candidate_identity_sha256 +from tests.prism_coordinator_test_support import durable_candidate_row + + +def fixture(index=0, parent="11" * 32, height=10): + row = durable_candidate_row(index) + intent = row["candidate"] + header = bytes([index + 1]) * 80 + block_hash = header_hash_hex(header) + intent.update( + block_hex=header.hex(), + block_hash_hex=block_hash, + parent_hash=parent, + expected_height=height, + ) + intent["template"].update(previousblockhash=parent, height=height) + row.update(block_hash=block_hash, candidate_sha256=block_candidate_identity_sha256(intent)) + block = recovery.RecoveryBlock(block_hash, height, parent, "pending") + return block, row + + +def metadata(block, state="pending"): + return dict( + block_hash=block.block_hash, + state=state, + block_height=block.height if state == "submitted" else None, + parent_hash=block.parent_hash if state == "submitted" else None, + chain_state="confirmed" if state == "submitted" else None, + has_audit=state == "submitted", + ) + + +def fake_rpc(blocks): + def call(method, params): + if method == "getblockheader": + block = next(block for block in blocks if block.block_hash == params[0]) + return dict(height=block.height, previousblockhash=block.parent_hash, confirmations=1) + if method == "getblockhash": + return next(block.block_hash for block in blocks if block.height == params[0]) + raise AssertionError(f"unexpected RPC {method}") + + return SimpleNamespace(call=Mock(side_effect=call)) + + +def test_config(root, database_url="postgresql://qbit:qbit@localhost/qbit"): + return load_coordinator_config( + { + "QBIT_RPC_HOST": "localhost", + "QBIT_RPC_USER": "test", + "QBIT_RPC_PASSWORD": "test", + "PRISM_DATABASE_URL": database_url, + "PRISM_ALLOW_TEST_SIGNING_SEEDS": "1", + "PRISM_ALLOW_BUNDLE_EMBEDDED_LEDGER_KEY": "1", + "PRISM_AUDIT_DIR": str(root), + "PRISM_EVIDENCE_PATH": str(root / "evidence.json"), + } + ) + + +class RecoveryTests(unittest.TestCase): + def test_plan_orders_parent_before_child_without_loading_payloads(self): + parent, _ = fixture() + child, _ = fixture(1, parent.block_hash, 11) + rows = {block.block_hash: metadata(block) for block in [parent, child]} + reader = Mock(metadata=lambda value: rows.get(value)) + planned = recovery.plan_recovery( + reader, fake_rpc([parent, child]), [child.block_hash, parent.block_hash] + ) + self.assertEqual(planned, [parent, child]) + reader.candidate.assert_not_called() + + def test_missing_pending_parent_rejects_plan(self): + parent, _ = fixture() + child, _ = fixture(1, parent.block_hash, 11) + rows = {block.block_hash: metadata(block) for block in [parent, child]} + with self.assertRaisesRegex(recovery.RecoveryError, "pending parent"): + recovery.plan_recovery( + Mock(metadata=lambda value: rows.get(value)), + fake_rpc([parent, child]), + [child.block_hash], + ) + + def test_duplicate_missing_abandoned_and_inactive_fail_closed(self): + block, _ = fixture() + for hashes, row in [ + ([], None), + ([block.block_hash] * 2, metadata(block)), + ([block.block_hash], None), + ([block.block_hash], metadata(block, "abandoned")), + ]: + with self.subTest(hashes=hashes, row=row), self.assertRaises(recovery.RecoveryError): + recovery.plan_recovery(Mock(metadata=lambda _: row), fake_rpc([block]), hashes) + rpc = fake_rpc([block]) + rpc.call = Mock( + side_effect=[dict(height=10, previousblockhash=block.parent_hash), "22" * 32] + ) + with self.assertRaisesRegex(recovery.RecoveryError, "active chain"): + recovery.require_active_block(rpc, block.block_hash) + + def test_submitted_without_complete_accounting_is_not_skipped(self): + block, _ = fixture() + row = metadata(block, "submitted") + row["has_audit"] = False + with self.assertRaisesRegex(recovery.RecoveryError, "completion"): + recovery.plan_recovery( + Mock(metadata=lambda _: row), fake_rpc([block]), [block.block_hash] + ) + + def test_payload_identity_height_and_header_are_verified(self): + block, row = fixture() + coordinator = SimpleNamespace(block_candidate_from_intent=block_candidate_from_intent) + candidate = recovery.decode_candidate(coordinator, block, row) + self.assertTrue(candidate.durable_replay) + for field, value in [ + ("shares_json", [{"changed": True}]), + ("expected_height", 12), + ("block_hex", "00" * 80), + ]: + modified = json.loads(json.dumps(row)) + modified["candidate"][field] = value + if field != "shares_json": + modified["candidate_sha256"] = block_candidate_identity_sha256( + modified["candidate"] + ) + with self.subTest(field=field), self.assertRaises(recovery.RecoveryError): + recovery.decode_candidate(coordinator, block, modified) + + def test_chunked_candidate_row_is_hydrated_not_misclassified(self): + """Issue #255: a version-2 row has no jsonb; it decodes via the ledger.""" + from lab.prism.candidate_codec import prepare_candidate_intent + + block, row = fixture() + prepared = prepare_candidate_intent(row["candidate"]) + v2_row = { + "candidate": None, + "candidate_sha256": row["candidate_sha256"], + "storage_version": 2, + "body_id": "ab" * 16, + "replay_header": prepared.replay_header(), + "byte_count": prepared.manifest.byte_count, + "chunk_count": prepared.manifest.chunk_count, + "chunk_bytes": prepared.manifest.chunk_bytes, + "share_count": prepared.manifest.share_count, + "body_state": "sealed", + } + hydrations = [] + + def hydrate(header_row, *, cancelled): + hydrations.append(header_row) + self.assertEqual(header_row["body"]["body_id"], "ab" * 16) + self.assertFalse(cancelled()) + return prepared + + coordinator = SimpleNamespace( + block_candidate_from_intent=block_candidate_from_intent, + ledger=SimpleNamespace(hydrate_block_candidate_intent=hydrate), + stop_event=threading.Event(), + ) + candidate = recovery.decode_candidate(coordinator, block, v2_row) + self.assertTrue(candidate.durable_replay) + self.assertEqual(candidate.submission.block_hash_hex, block.block_hash) + self.assertEqual(len(hydrations), 1) + # A digest that disagrees with the hydrated body is still refused. + with self.assertRaisesRegex(recovery.RecoveryError, "digest mismatch"): + recovery.decode_candidate(coordinator, block, {**v2_row, "candidate_sha256": "0" * 64}) + + def test_unsupported_storage_version_and_unsealed_body_fail_closed(self): + block, row = fixture() + coordinator = SimpleNamespace( + block_candidate_from_intent=block_candidate_from_intent, + ledger=SimpleNamespace(hydrate_block_candidate_intent=Mock()), + stop_event=threading.Event(), + ) + base = {"candidate": None, "candidate_sha256": row["candidate_sha256"]} + for label, v2_row in { + "future version": {**base, "storage_version": 3, "body_id": "ab" * 16, "body_state": "sealed"}, + "unsealed body": {**base, "storage_version": 2, "body_id": "ab" * 16, "body_state": "staging"}, + "missing body": {**base, "storage_version": 2, "body_id": None, "body_state": None}, + }.items(): + with self.subTest(label=label), self.assertRaises(recovery.RecoveryError): + recovery.decode_candidate(coordinator, block, v2_row) + coordinator.ledger.hydrate_block_candidate_intent.assert_not_called() + + def test_apply_finalizes_only_after_normal_accounting_and_checks_durable_result(self): + block, payload = fixture() + events = [] + row = metadata(block) + reader = Mock(metadata=lambda _: row, candidate=lambda _: payload) + + def submit(_candidate, *, node_submission): + self.assertFalse(node_submission.attempted) + events.append("accounting") + return True + + def finalize(*_args, **kwargs): + self.assertTrue(kwargs["accepted"]) + events.append("outbox") + row.update(metadata(block, "submitted")) + + coordinator = Mock( + rpc=fake_rpc([block]), + stop_event=threading.Event(), + block_candidate_from_intent=block_candidate_from_intent, + _writer_operation=lambda _: nullcontext(), + _block_landing_ledger_statement_timeout_scope=lambda _: nullcontext(), + submit_block_candidate=submit, + _finalize_block_candidate=finalize, + ) + recovery.recover_block(coordinator, reader, block) + self.assertEqual(events, ["accounting", "outbox"]) + recovery.recover_block(coordinator, reader, block) + self.assertEqual(events, ["accounting", "outbox"]) + coordinator.serve.assert_not_called() + + def test_accounting_failure_and_lost_lease_leave_outbox_pending(self): + block, payload = fixture() + for failure in ("accounting", "lease", "terminal_update"): + reader = Mock(metadata=lambda _: metadata(block), candidate=lambda _: payload) + coordinator = Mock( + rpc=fake_rpc([block]), + stop_event=threading.Event(), + block_candidate_from_intent=block_candidate_from_intent, + _writer_operation=lambda _: nullcontext(), + _block_landing_ledger_statement_timeout_scope=lambda _: nullcontext(), + ) + coordinator.submit_block_candidate.return_value = failure != "accounting" + if failure == "lease": + coordinator._require_fresh_ledger_lease_for_external_side_effect.side_effect = ( + recovery.RecoveryError("lease lost") + ) + with self.subTest(failure=failure), self.assertRaises(recovery.RecoveryError): + recovery.recover_block(coordinator, reader, block) + if failure != "terminal_update": + coordinator._finalize_block_candidate.assert_not_called() + + def test_recovery_config_preserves_lease_policy_and_signing_identity(self): + with tempfile.TemporaryDirectory() as directory: + before = test_config(Path(directory)) + after = recovery.recovery_config(before) + self.assertEqual(after.lifecycle, before.lifecycle) + self.assertEqual(after.ledger.writer_id, before.ledger.writer_id) + self.assertEqual(after.ledger.writer_epoch, before.ledger.writer_epoch) + self.assertEqual(after.ledger.signing_seed_hex, before.ledger.signing_seed_hex) + self.assertFalse(after.ledger.initialize_schema) + self.assertEqual(after.ledger.native_client_mode, "on") + self.assertIsNone(after.ledger.writer_session_token) + + def test_runner_never_offers_new_blocks(self): + with self.assertRaisesRegex(recovery.RecoveryError, "refuses to submit"): + recovery.RecoveryCoordinator.__new__( + recovery.RecoveryCoordinator + )._submit_block_candidate_to_node(None) + + +class CooperativeJsonTests(unittest.TestCase): + def test_canonical_bytes_identity_and_parser_behavior_match_stdlib(self): + values = [None, True, -(2**90), 1.25, '\u03bb\U0001f680\n\\"', {"z": [1, None], "a": "x"}] + expected = [json.dumps(value, sort_keys=True, separators=(",", ":")) for value in values] + _, payload = fixture() + digest = block_candidate_identity_sha256(payload["candidate"]) + original = json._default_decoder + with cooperative_json(): + for value, text in zip(values, expected): + self.assertEqual(json.dumps(value, sort_keys=True, separators=(",", ":")), text) + self.assertEqual(json.loads(text), value) + self.assertEqual(json.loads(text.encode()), value) + self.assertEqual(block_candidate_identity_sha256(payload["candidate"]), digest) + for malformed in ('{"x":', "[1,]", '"\\q"', "{} garbage"): + with self.assertRaises(json.JSONDecodeError): + json.loads(malformed) + self.assertEqual(json.loads('{"x":1,"x":2}'), {"x": 2}) + self.assertIs(json._default_decoder, original) + + def test_codec_restores_after_error(self): + original = json.scanner.make_scanner + with self.assertRaises(RuntimeError), cooperative_json(): + raise RuntimeError("test") + self.assertIs(json.scanner.make_scanner, original) + + +@unittest.skipUnless( + os.environ.get("PRISM_RECOVERY_TEST_DATABASE_URL"), "requires the native PostgreSQL test gate" +) +class NativeRecoveryTests(unittest.TestCase): + @classmethod + def setUpClass(cls): + import psycopg + from psycopg.conninfo import make_conninfo + + cls.schema = "recovery_" + uuid.uuid4().hex + cls.admin = psycopg.connect(os.environ["PRISM_RECOVERY_TEST_DATABASE_URL"], autocommit=True) + cls.admin.execute(f'CREATE SCHEMA "{cls.schema}"') + # Private schema only: with public on the path, 001's DROP FUNCTION + # statements would remove the shared database's public functions + # before creating the private replacements. Builtins stay implicitly + # available. + cls.url = make_conninfo( + os.environ["PRISM_RECOVERY_TEST_DATABASE_URL"], + options=f"-csearch_path={cls.schema}", + ) + with psycopg.connect(cls.url, autocommit=True) as connection: + connection.execute(Path("crates/qbit-prism/sql/001_share_ledger.sql").read_text()) + # Issue #255: the chunked-body migration the ledger applies + # after 001 on every initialized start. + connection.execute(Path("crates/qbit-prism/sql/002_candidate_bodies.sql").read_text()) + + @classmethod + def tearDownClass(cls): + cls.admin.execute(f'DROP SCHEMA "{cls.schema}" CASCADE') + cls.admin.close() + + def ledger(self, **kwargs): + ledger = PsqlShareLedger( + psql_command="psql", database_url=self.url, native_client_mode="on", **kwargs + ) + self.addCleanup(ledger.close) + return ledger + + def test_exclusive_guard_refuses_live_writer_and_reacquires_after_release(self): + owner = self.ledger(writer_session_token="heartbeat-v1:" + uuid.uuid4().hex) + token = owner._writer_session_token + with self.assertRaisesRegex(recovery.RecoveryError, "exclusive writer lease"): + self.ledger( + writer_session_token="heartbeat-v1:" + uuid.uuid4().hex, + lease_retry_sleep=recovery.refuse_lease_wait, + ) + self.assertEqual(owner._writer_session_token, token) + self.assertTrue(owner.release_writer_lease()) + owner.close() + successor = self.ledger( + writer_session_token="heartbeat-v1:" + uuid.uuid4().hex, + lease_retry_sleep=recovery.refuse_lease_wait, + ) + self.assertTrue(successor.release_writer_lease()) + + def test_reader_is_read_only_and_loads_only_selected_candidate_with_cooperative_codec(self): + import psycopg + from psycopg.types.json import Jsonb + + block, payload = fixture() + other, other_payload = fixture(1) + with psycopg.connect(self.url, autocommit=True) as connection: + for target, row in [(block, payload), (other, other_payload)]: + connection.execute( + "INSERT INTO qbit_block_candidate_outbox (block_hash, candidate, candidate_sha256) VALUES (%s,%s,%s)", + (target.block_hash, Jsonb(row["candidate"]), row["candidate_sha256"]), + ) + with cooperative_json(): + reader = recovery.RecoveryReader(self.url) + try: + self.assertEqual(reader.metadata(block.block_hash)["state"], "pending") + self.assertNotIn("candidate", reader.metadata(block.block_hash)) + self.assertEqual( + reader.candidate(block.block_hash)["candidate"], payload["candidate"] + ) + with self.assertRaises(psycopg.errors.ReadOnlySqlTransaction): + reader.connection.execute( + "UPDATE qbit_block_candidate_outbox SET attempt_count=1" + ) + finally: + reader.close() + + def test_legacy_only_schema_is_refused_by_the_reader_and_the_ledger(self): + # A 001-only database: the staged-upgrade case with + # PRISM_POSTGRES_INIT_SCHEMA=0, which storage version 1 cannot rescue. + import psycopg + from psycopg.conninfo import make_conninfo + + from lab.prism.candidate_store import IncompatibleCandidateSchema + + schema = "recovery_legacy_" + uuid.uuid4().hex + self.admin.execute(f'CREATE SCHEMA "{schema}"') + self.addCleanup(self.admin.execute, f'DROP SCHEMA "{schema}" CASCADE') + url = make_conninfo( + os.environ["PRISM_RECOVERY_TEST_DATABASE_URL"], options=f"-csearch_path={schema}" + ) + with psycopg.connect(url, autocommit=True) as connection: + connection.execute(Path("crates/qbit-prism/sql/001_share_ledger.sql").read_text()) + reader = recovery.RecoveryReader(url) + self.addCleanup(reader.close) + with self.assertRaisesRegex(recovery.RecoveryError, "002_candidate_bodies.sql"): + reader.require_candidate_schema() + ledger = PsqlShareLedger( + psql_command="psql", + database_url=url, + native_client_mode="on", + candidate_storage_version=1, + ) + self.addCleanup(ledger.close) + with self.assertRaisesRegex(IncompatibleCandidateSchema, "002_candidate_bodies.sql"): + ledger.verify_candidate_schema() + + def test_real_finalizer_persists_confirms_publishes_and_completes_outbox(self): + # Real coordinator, PostgreSQL, lease heartbeat, finalizer and audit + # publication. Only the node and cryptographic builder/verifier use + # deterministic fixtures; their own suites cover consensus proofs. + import psycopg + from psycopg.types.json import Jsonb + + from tests.prism_vardiff_test_support import ( + verified_audit_report, + verified_block_bundle, + ) + + parent, parent_payload = fixture(3, height=20) + child, child_payload = fixture(4, parent.block_hash, 21) + with psycopg.connect(self.url, autocommit=True) as connection: + for block, payload in [(parent, parent_payload), (child, child_payload)]: + payload["candidate"]["prospective_prior_balances"] = [] + payload["candidate_sha256"] = block_candidate_identity_sha256(payload["candidate"]) + connection.execute( + "INSERT INTO qbit_block_candidate_outbox (block_hash,candidate,candidate_sha256) VALUES (%s,%s,%s)", + (block.block_hash, Jsonb(payload["candidate"]), payload["candidate_sha256"]), + ) + rpc = fake_rpc([parent, child]) + chain_call = rpc.call.side_effect + + def call(method, params=None, **_kwargs): + if method == "getbestblockhash": + return child.block_hash + if method == "getblockcount": + return child.height + if method == "getblockchaininfo": + return dict( + chain="regtest", + blocks=child.height, + headers=child.height, + initialblockdownload=False, + ) + if method == "getblockhash" and params == [parent.height - 1]: + return parent.parent_hash + return chain_call(method, params) + + rpc.call.side_effect = call + bundle = verified_block_bundle("00") + + def canonical(value): + return json.dumps(value, sort_keys=True, separators=(",", ":")).encode() + + audit_report = verified_audit_report("00") + audit_report.update(audit_bundle_sha256_hex=hashlib.sha256(canonical(bundle)).hexdigest()) + + def verify(*_args, **kwargs): + return {**audit_report, "block_height": kwargs["expected_block_height"]} + + selection = direct_stratum.VersionRollingMaskSelection( + direct_stratum.QBIT_VERSION_ROLLING_MASK, "fallback", "test" + ) + with ( + tempfile.TemporaryDirectory() as directory, + cooperative_json(), + patch("lab.prism.prism_coordinator.JsonRpc", return_value=rpc), + patch.object( + recovery.RecoveryCoordinator, "resolve_version_rolling_mask", return_value=selection + ), + patch.object(recovery.RecoveryCoordinator, "validate_live_chain_identity"), + patch.object(recovery.RecoveryCoordinator, "validate_live_template_and_fee_policy"), + patch.object(recovery.RecoveryCoordinator, "build_audit_bundle", return_value=bundle), + patch("lab.prism.audit_artifacts.AuditArtifactStore.verify_bundle", side_effect=verify), + patch("lab.prism.prism_coordinator.canonical_bundle_bytes", canonical), + patch("lab.prism.share_ledger._default_bundle_canonicalizer", return_value=canonical), + ): + reader = recovery.RecoveryReader(self.url) + self.addCleanup(reader.close) + config = test_config(Path(directory), self.url) + events = [] + blocks = recovery.plan_recovery(reader, rpc, [child.block_hash, parent.block_hash]) + with patch( + "lab.prism.audit_artifacts.AuditArtifactStore.publish_success", + side_effect=RuntimeError("injected publication failure"), + ): + with self.assertRaisesRegex(RuntimeError, "injected publication failure"): + recovery.apply_recovery( + config, reader, blocks, timeout_seconds=30, report=events.append + ) + self.assertEqual(reader.metadata(parent.block_hash)["chain_state"], "confirmed") + self.assertEqual(reader.metadata(parent.block_hash)["state"], "pending") + self.assertIsNone(reader.metadata(child.block_hash)["chain_state"]) + events.clear() + recovery.apply_recovery( + config, reader, blocks, timeout_seconds=30, report=events.append + ) + for block in blocks: + recovery.require_completed(reader.metadata(block.block_hash), block) + self.assertEqual( + [event["block_hash"] for event in events if event["event"] == "completed"], + [parent.block_hash, child.block_hash], + ) + # Re-entry is a durable no-op; no payload or accounting is rebuilt. + with patch.object( + reader, "candidate", side_effect=AssertionError("terminal payload read") + ): + recovery.apply_recovery( + config, reader, blocks, timeout_seconds=30, report=events.append + ) + self.assertFalse(any(call.args[0] == "submitblock" for call in rpc.call.call_args_list)) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_prism_process_telemetry.py b/tests/test_prism_process_telemetry.py new file mode 100644 index 000000000..041e78b0b --- /dev/null +++ b/tests/test_prism_process_telemetry.py @@ -0,0 +1,309 @@ +"""Issue #226 part 1: the process heap collector and its configuration switch. + +The renderer-level contracts (family set, unavailable rendering, owner +reads, closed labels, no heap walk) live in tests/test_prism_metrics.py. +This module pins the collector itself: the glibc struct layout, the +resolve-once behaviour, the closed generation set, and the strict +PRISM_MALLOC_TELEMETRY switch through the coordinator config loader. +""" + +from __future__ import annotations + +import ctypes +import gc +import os +from pathlib import Path +import sys +import tempfile +import unittest +from types import SimpleNamespace +from unittest import mock + +from lab.prism.coordinator_config import ( + DEFAULT_PRISM_MALLOC_TELEMETRY, + env_malloc_telemetry, + load_coordinator_config, +) +from lab.prism.metrics import MetricsRenderer +from lab.prism.process_telemetry import ( + PRISM_GC_GENERATIONS, + MallInfo2, + ProcessHeapTelemetry, + resolve_mallinfo2, +) + + +def _host_has_mallinfo2() -> bool: + try: + resolve_mallinfo2() + except Exception: # noqa: BLE001 - any failure means "not here" + return False + return True + + +class MallInfo2LayoutTests(unittest.TestCase): + def test_struct_matches_glibc_field_order(self) -> None: + """Ten size_t fields in glibc's declared order; a wrong layout would + read the wrong field as the arena on every scrape.""" + names = [name for name, _ctype in MallInfo2._fields_] + self.assertEqual( + names, + [ + "arena", + "ordblks", + "smblks", + "hblks", + "hblkhd", + "usmblks", + "fsmblks", + "uordblks", + "fordblks", + "keepcost", + ], + ) + for _name, ctype in MallInfo2._fields_: + self.assertIs(ctype, ctypes.c_size_t) + self.assertEqual(ctypes.sizeof(MallInfo2), 10 * ctypes.sizeof(ctypes.c_size_t)) + + @unittest.skipUnless( + sys.platform.startswith("linux") and _host_has_mallinfo2(), + "glibc mallinfo2 is not available on this host", + ) + def test_real_binding_reads_a_consistent_struct(self) -> None: + """On a glibc 2.33+ host the bound symbol returns arena = in-use + free.""" + info = resolve_mallinfo2()() + self.assertIsInstance(info, MallInfo2) + self.assertGreater(int(info.arena), 0) + self.assertGreaterEqual(int(info.uordblks), 0) + self.assertGreaterEqual(int(info.fordblks), 0) + self.assertEqual(int(info.arena), int(info.uordblks) + int(info.fordblks)) + + +class ProcessHeapTelemetryTests(unittest.TestCase): + def test_sample_carries_every_interpreter_reading(self) -> None: + sample = ProcessHeapTelemetry(malloc_enabled=False).sample() + self.assertGreater(sample.allocated_blocks, 0) + self.assertGreaterEqual(sample.threads, 1) + for reading in ( + sample.gc_trigger_count, + sample.gc_collections, + sample.gc_collected, + sample.gc_uncollectable, + ): + self.assertEqual(len(reading), len(PRISM_GC_GENERATIONS)) + for value in reading: + self.assertGreaterEqual(value, 0) + self.assertFalse(sample.malloc_info_available) + self.assertEqual(sample.malloc_arena_bytes, -1) + self.assertEqual(sample.malloc_in_use_bytes, -1) + self.assertEqual(sample.malloc_free_bytes, -1) + self.assertEqual(sample.malloc_mmapped_bytes, -1) + + def test_generation_set_stays_closed_when_the_interpreter_reports_fewer(self) -> None: + """A missing generation renders -1 rather than shrinking the series set.""" + with ( + mock.patch.object(gc, "get_count", return_value=(4, 2)), + mock.patch.object( + gc, + "get_stats", + return_value=[ + {"collections": 9, "collected": 8, "uncollectable": 1}, + {"collections": 3, "collected": 2, "uncollectable": 0}, + ], + ), + ): + sample = ProcessHeapTelemetry(malloc_enabled=False).sample() + self.assertEqual(sample.gc_trigger_count, (4, 2, -1)) + self.assertEqual(sample.gc_collections, (9, 3, -1)) + self.assertEqual(sample.gc_collected, (8, 2, -1)) + self.assertEqual(sample.gc_uncollectable, (1, 0, -1)) + + def test_resolver_runs_once_and_its_failure_is_remembered(self) -> None: + resolver = mock.Mock(side_effect=AttributeError("no mallinfo2")) + telemetry = ProcessHeapTelemetry(mallinfo_resolver=resolver) + for _ in range(3): + sample = telemetry.sample() + self.assertFalse(sample.malloc_info_available) + self.assertEqual(sample.malloc_arena_bytes, -1) + self.assertEqual(resolver.call_count, 1) + + def test_bound_symbol_that_fails_at_call_time_is_not_retried(self) -> None: + bound = mock.Mock(side_effect=OSError("call failed")) + telemetry = ProcessHeapTelemetry(mallinfo_resolver=lambda: bound) + first = telemetry.sample() + second = telemetry.sample() + self.assertFalse(first.malloc_info_available) + self.assertFalse(second.malloc_info_available) + self.assertEqual(bound.call_count, 1) + + def test_bound_symbol_fields_map_to_the_sample(self) -> None: + def fake_mallinfo2() -> MallInfo2: + info = MallInfo2() + info.arena = 20_000 + info.uordblks = 12_000 + info.fordblks = 8_000 + info.hblkhd = 1_024 + return info + + sample = ProcessHeapTelemetry(mallinfo_resolver=lambda: fake_mallinfo2).sample() + self.assertTrue(sample.malloc_info_available) + self.assertEqual(sample.malloc_arena_bytes, 20_000) + self.assertEqual(sample.malloc_in_use_bytes, 12_000) + self.assertEqual(sample.malloc_free_bytes, 8_000) + self.assertEqual(sample.malloc_mmapped_bytes, 1_024) + + @unittest.skipUnless( + sys.platform.startswith("linux") and _host_has_mallinfo2(), + "glibc mallinfo2 is not available on this host", + ) + def test_default_collector_binds_glibc_on_this_host(self) -> None: + sample = ProcessHeapTelemetry().sample() + self.assertTrue(sample.malloc_info_available) + self.assertGreater(sample.malloc_arena_bytes, 0) + self.assertGreaterEqual(sample.malloc_in_use_bytes, 0) + self.assertGreaterEqual(sample.malloc_free_bytes, 0) + self.assertGreaterEqual(sample.malloc_mmapped_bytes, 0) + + +class MallocTelemetrySwitchTests(unittest.TestCase): + """PRISM_MALLOC_TELEMETRY: default on, strict values, renderer resolution.""" + + def test_switch_defaults_to_on(self) -> None: + self.assertTrue(DEFAULT_PRISM_MALLOC_TELEMETRY) + self.assertTrue(env_malloc_telemetry(environ={})) + + def test_switch_accepts_boolean_tokens(self) -> None: + for raw in ("1", "true", "YES", "On"): + with self.subTest(raw=raw): + self.assertTrue( + env_malloc_telemetry(environ={"PRISM_MALLOC_TELEMETRY": raw}) + ) + for raw in ("0", "false", "NO", "Off"): + with self.subTest(raw=raw): + self.assertFalse( + env_malloc_telemetry(environ={"PRISM_MALLOC_TELEMETRY": raw}) + ) + + def test_switch_fails_closed_on_non_boolean_values(self) -> None: + for raw in ("2", "banana", "tru"): + with self.subTest(raw=raw): + with self.assertRaisesRegex( + SystemExit, "PRISM_MALLOC_TELEMETRY must be a boolean" + ): + env_malloc_telemetry(environ={"PRISM_MALLOC_TELEMETRY": raw}) + + def test_coordinator_config_load_validates_the_switch_at_startup(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + base = { + "QBIT_RPC_HOST": "qbit.example", + "QBIT_RPC_USER": "rpc-user", + "QBIT_RPC_PASSWORD": "rpc-password", + "PRISM_ALLOW_MEMORY_LEDGER": "1", + "PRISM_ALLOW_TEST_SIGNING_SEEDS": "1", + "PRISM_ALLOW_BUNDLE_EMBEDDED_LEDGER_KEY": "1", + "PRISM_AUDIT_DIR": str(root), + "PRISM_EVIDENCE_PATH": str(root / "evidence.json"), + } + config = load_coordinator_config(base) + self.assertTrue(config.lifecycle.malloc_telemetry_enabled) + disabled = load_coordinator_config( + {**base, "PRISM_MALLOC_TELEMETRY": "0"} + ) + self.assertFalse(disabled.lifecycle.malloc_telemetry_enabled) + with self.assertRaisesRegex( + SystemExit, "PRISM_MALLOC_TELEMETRY must be a boolean" + ): + load_coordinator_config( + {**base, "PRISM_MALLOC_TELEMETRY": "maybe"} + ) + + def test_renderer_resolves_the_switch_once_at_construction(self) -> None: + # A port that pins the switch wins over the environment. + pinned = MetricsRenderer(SimpleNamespace(malloc_telemetry_enabled=False)) + self.assertFalse(pinned._process_telemetry.malloc_enabled) + # Otherwise the strict environment switch is read, once, at + # construction; a later environment change does not reach a built + # renderer. + with mock.patch.dict(os.environ, {"PRISM_MALLOC_TELEMETRY": "0"}): + from_environment = MetricsRenderer(SimpleNamespace()) + self.assertFalse(from_environment._process_telemetry.malloc_enabled) + with mock.patch.dict(os.environ, {"PRISM_MALLOC_TELEMETRY": "1"}): + self.assertFalse(from_environment._process_telemetry.malloc_enabled) + self.assertTrue( + MetricsRenderer(SimpleNamespace())._process_telemetry.malloc_enabled + ) + with mock.patch.dict(os.environ, {"PRISM_MALLOC_TELEMETRY": ""}, clear=False): + self.assertEqual( + MetricsRenderer(SimpleNamespace())._process_telemetry.malloc_enabled, + DEFAULT_PRISM_MALLOC_TELEMETRY, + ) + + def test_validated_config_snapshot_beats_the_ambient_environment(self) -> None: + """A coordinator built with an explicit config is not overridden by env. + + The lifecycle snapshot is the value the config loader validated at + startup. A caller that constructed the coordinator with telemetry + disabled must not have mallinfo2 run anyway because the ambient + environment happens to say otherwise, and a valid environment change + after startup must not silently override the snapshot either. + """ + disabled = SimpleNamespace( + config=SimpleNamespace( + lifecycle=SimpleNamespace(malloc_telemetry_enabled=False) + ) + ) + with mock.patch.dict(os.environ, {"PRISM_MALLOC_TELEMETRY": "1"}): + self.assertFalse( + MetricsRenderer(disabled)._process_telemetry.malloc_enabled + ) + enabled = SimpleNamespace( + config=SimpleNamespace( + lifecycle=SimpleNamespace(malloc_telemetry_enabled=True) + ) + ) + with mock.patch.dict(os.environ, {"PRISM_MALLOC_TELEMETRY": "0"}): + self.assertTrue( + MetricsRenderer(enabled)._process_telemetry.malloc_enabled + ) + # An explicit port pin still outranks the snapshot. + pinned = SimpleNamespace( + malloc_telemetry_enabled=False, + config=SimpleNamespace( + lifecycle=SimpleNamespace(malloc_telemetry_enabled=True) + ), + ) + self.assertFalse(MetricsRenderer(pinned)._process_telemetry.malloc_enabled) + # With no snapshot at all the environment is still the fallback. + with mock.patch.dict(os.environ, {"PRISM_MALLOC_TELEMETRY": "0"}): + self.assertFalse( + MetricsRenderer( + SimpleNamespace(config=SimpleNamespace()) + )._process_telemetry.malloc_enabled + ) + + def test_gc_trigger_count_is_not_described_as_retained_objects(self) -> None: + """gc.get_count() is a trigger counter, and the family must say so. + + Naming it after retained objects sent the runbook's leak guidance at + internal collector state; an operator reading it as a backlog during an + incident would draw the wrong conclusion. + """ + lines = MetricsRenderer( + SimpleNamespace(), process_telemetry=ProcessHeapTelemetry() + ).process_heap_metrics_lines() + help_line = next( + line + for line in lines + if line.startswith("# HELP qbit_prism_process_gc_trigger_count") + ) + self.assertIn("trigger counters", help_line) + self.assertIn("NOT a count of retained objects", help_line) + self.assertFalse( + [line for line in lines if "gc_pending_objects" in line], + "the misleading family name must not come back", + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_prism_progress_health.py b/tests/test_prism_progress_health.py index ae20956c4..0038ce134 100644 --- a/tests/test_prism_progress_health.py +++ b/tests/test_prism_progress_health.py @@ -8,8 +8,15 @@ from types import SimpleNamespace from unittest.mock import patch +from concurrent.futures import CancelledError + from lab.prism.prism_coordinator import QbitTipTemplateSnapshot -from tests.test_prism_coordinator_job_cache import client, coordinator +from lab.prism.progress_health import ( + ProgressHealthConfig, + ProgressHealthService, + WorkGeneration, +) +from tests.prism_coordinator_test_support import client, coordinator class FakeMonotonicClock: @@ -76,9 +83,6 @@ def progress_coordinator() -> tuple[object, FakeMonotonicClock]: server._progress_last_refresh_activity_monotonic = None server._progress_bundle_build_counter = 0 server._progress_bundle_builds.clear() - server._health_snapshot = None - server._health_snapshot_monotonic = None - server._health_refresh_loop_running = False return server, clock @@ -655,7 +659,7 @@ def test_stale_cached_ok_cannot_mask_a_progress_failure(self) -> None: publish(server, original) server.health_refresh_seconds = 60.0 server.refresh_health_snapshot() - server._health_refresh_loop_running = True + server._ensure_observability_service().set_loop_running_for_test(True) changed = snapshot(generation=2, fingerprint="bb" * 32, tip="22" * 32) server._record_progress_tip_poll(changed) @@ -805,5 +809,146 @@ def test_progress_metrics_have_bounded_state_and_age_gauges(self) -> None: ) +def owner_progress_service( + *, bundle_build_deadline_seconds: float = 60.0 +) -> tuple[ProgressHealthService, FakeMonotonicClock, ProgressHealthConfig, float]: + clock = FakeMonotonicClock() + service = ProgressHealthService( + started_monotonic=clock.now, + monotonic=clock, + ) + config = ProgressHealthConfig( + pending_refresh_deadline_seconds=15.0, + tip_poll_deadline_seconds=15.0, + bundle_build_deadline_seconds=bundle_build_deadline_seconds, + ) + return service, clock, config, clock.now + + +def work_generation( + *, generation: int, fingerprint: str, payout_generation: int = 0 +) -> WorkGeneration: + return WorkGeneration( + template_generation=generation, + template_fingerprint=fingerprint, + payout_generation=payout_generation, + ) + + +def owner_service_health( + service: ProgressHealthService, + config: ProgressHealthConfig, + started_monotonic: float, + *, + payout_generation: int = 0, +) -> dict[str, object]: + return service.snapshot( + lambda _fingerprint, _payout_generation: (0, 0), + payout_generation, + config, + started_monotonic, + ).as_mapping() + + +class ProgressTokenOwnershipTests(unittest.TestCase): + """Direct R1/J1 token contracts on the extracted G1 owner service.""" + + def test_refresh_token_finishes_on_exception(self) -> None: + service, _clock, config, started = owner_progress_service() + + with self.assertRaisesRegex(RuntimeError, "boom"): + with service.start_refresh(): + raise RuntimeError("boom") + + health = owner_service_health(service, config, started) + self.assertFalse(health["tip_refresh_in_progress"]) + + def test_bundle_token_finishes_on_cancellation(self) -> None: + service, clock, config, started = owner_progress_service( + bundle_build_deadline_seconds=1.0 + ) + + with self.assertRaises(CancelledError): + with service.start_bundle_build(): + raise CancelledError() + clock.advance(2) + + health = owner_service_health(service, config, started) + self.assertEqual(health["bundle_build_oldest_age_seconds"], 0.0) + + def test_token_finish_is_idempotent(self) -> None: + service, _clock, config, started = owner_progress_service() + refresh = service.start_refresh() + build = service.start_bundle_build() + + refresh.finish() + refresh.finish() + build.finish() + build.finish() + + health = owner_service_health(service, config, started) + self.assertFalse(health["tip_refresh_in_progress"]) + self.assertEqual(health["bundle_build_oldest_age_seconds"], 0.0) + + def test_overlapping_refresh_tokens_finish_independently(self) -> None: + service, clock, config, started = owner_progress_service() + first = service.start_refresh() + clock.advance(5) + second = service.start_refresh() + + second.finish() + health = owner_service_health(service, config, started) + self.assertTrue(health["tip_refresh_in_progress"]) + first.note_activity() + first.finish() + + health = owner_service_health(service, config, started) + self.assertFalse(health["tip_refresh_in_progress"]) + + def test_oldest_overlapping_bundle_controls_health(self) -> None: + service, clock, config, started = owner_progress_service( + bundle_build_deadline_seconds=60.0 + ) + current = work_generation(generation=1, fingerprint="aa" * 32) + service.observe_tip(current) + self.assertTrue(service.publish_work(current)) + first = service.start_bundle_build() + clock.advance(10) + second = service.start_bundle_build() + clock.advance(51) + service.observe_tip( + work_generation(generation=2, fingerprint="aa" * 32) + ) + + health = owner_service_health(service, config, started) + self.assertIn("bundle_build_stuck", health["reasons"]) + self.assertEqual(health["bundle_build_oldest_age_seconds"], 61.0) + + first.finish() + health = owner_service_health(service, config, started) + self.assertNotIn("bundle_build_stuck", health["reasons"]) + self.assertEqual(health["bundle_build_oldest_age_seconds"], 51.0) + second.finish() + + def test_multiple_failures_keep_the_fixed_reason_order(self) -> None: + service, clock, config, started = owner_progress_service( + bundle_build_deadline_seconds=60.0 + ) + build = service.start_bundle_build() + clock.advance(61) + + health = owner_service_health(service, config, started) + + self.assertEqual( + health["reasons"], + [ + "tip_poll_stale", + "bundle_build_stuck", + "current_generation_not_published", + ], + ) + build.finish() + + if __name__ == "__main__": unittest.main() diff --git a/tests/test_prism_public_dashboard_api.py b/tests/test_prism_public_dashboard_api.py index e5b654478..f3094c4bb 100644 --- a/tests/test_prism_public_dashboard_api.py +++ b/tests/test_prism_public_dashboard_api.py @@ -11,16 +11,38 @@ import unittest import urllib.error import urllib.request +from contextlib import contextmanager from datetime import datetime, timezone from decimal import Decimal from http.server import ThreadingHTTPServer from pathlib import Path from types import SimpleNamespace +from typing import Iterator from unittest.mock import patch -from lab.prism import direct_stratum, public_api -from lab.prism.prism_coordinator import make_audit_handler -from lab.prism.share_ledger import PendingShare, PsqlShareLedger, SingleWriterShareLedger +from lab.prism import direct_stratum, public_api, public_read_service +from lab.prism.share_ledger import ( + LedgerOperationTimeout, + PendingShare, + PsqlShareLedger, + SingleWriterShareLedger, + canonical_json_text, + sha256_json_hex, +) + + +def make_public_handler(coordinator: object) -> type: + """Serve /public/v1 through the process that owns it. + + These routes left the coordinator's audit listener in issue #145, so the + handler under test here is the extracted public read service's. It wraps + the same public_api.dispatch behind the same PublicResponseCache, which is + what keeps these assertions meaningful across the move; + tests/test_prism_public_read_service.py pins that equivalence directly. + """ + + service = public_read_service.PublicReadService(coordinator) # type: ignore[arg-type] + return public_read_service.make_handler(service) class FanoutPublicRowTests(unittest.TestCase): @@ -61,11 +83,16 @@ def __init__( template_bits: str | None = "207fffff", blockchain_info: dict[str, object] | None = None, network_info: dict[str, object] | None = None, + network_hashps: object = 2_500_000_000_000_000_000, ) -> None: self.template_difficulty = template_difficulty self.template_bits = template_bits self.blockchain_info = blockchain_info or {} self.network_info = network_info or {} + # H/s, as qbitd's getnetworkhashps reports. Pass an exception instance + # to simulate a node whose hashrate estimate is unavailable. + self.network_hashps = network_hashps + self.network_hashps_calls: list[list[object] | None] = [] def call(self, method: str, params: list[object] | None = None) -> object: if method == "getblockchaininfo": @@ -85,6 +112,11 @@ def call(self, method: str, params: list[object] | None = None) -> object: return template if method == "getnetworkinfo": return {"connections": 8, **self.network_info} + if method == "getnetworkhashps": + self.network_hashps_calls.append(params) + if isinstance(self.network_hashps, BaseException): + raise self.network_hashps + return self.network_hashps raise AssertionError(f"unexpected RPC method {method}") @@ -92,6 +124,19 @@ def _bucket_timestamp(epoch: int) -> str: return public_api.iso_datetime(datetime.fromtimestamp(epoch, timezone.utc)) +def _epoch_from_timestamp(timestamp: str) -> int: + parsed = datetime.strptime(timestamp, "%Y-%m-%dT%H:%M:%SZ") + return int(parsed.replace(tzinfo=timezone.utc).timestamp()) + + +FIXTURE_DIR = Path(__file__).resolve().parents[1] / "docs" / "public-dashboard-api" / "fixtures" + + +def _load_fixture(name: str) -> dict[str, object]: + with (FIXTURE_DIR / name).open(encoding="utf-8") as fixture_file: + return json.load(fixture_file) + + class FakePublicLedger: backend_name = "fake-ledger" block_hash = "a" * 64 @@ -108,6 +153,26 @@ def __init__(self) -> None: self.leaderboard_calls = 0 self.reward_leaderboard_calls: list[dict[str, object]] = [] self.pool_snapshot_calls = 0 + self.hashrate_series_calls = 0 + self.blocks_calls: list[dict[str, object]] = [] + self.block_marker_calls = 0 + self.last_block_markers_anchor: int | None = None + # Found blocks across two recent 1h buckets plus one ancient one, as + # (height, hash, found_at_epoch, chain_state). The newer bucket holds + # four accepted blocks (so the top-3 cap truncates), two of them tied + # on found_at (so the height tiebreak is observable), plus a reversed + # block that must never appear. The ancient block sits far outside + # every bounded range and is only reachable through range=all. + base_hour = int(time.time()) // 3600 * 3600 + self.block_marker_records: list[tuple[int, str, int, str]] = [ + (67440, "1a" * 32, base_hour - 7200 + 300, "confirmed"), + (67441, "1b" * 32, base_hour - 3600 + 60, "confirmed"), + (67442, "1c" * 32, base_hour - 3600 + 120, "confirmed"), + (67443, "1d" * 32, base_hour - 3600 + 240, "confirmed"), + (67444, "1e" * 32, base_hour - 3600 + 240, "confirmed"), + (67439, "1f" * 32, base_hour - 3600 + 240, "reversed"), + (60000, "2a" * 32, base_hour - 400 * 86400, "confirmed"), + ] def dashboard_pool_snapshot(self, *, current_network_difficulty: object, generated_at: str) -> dict[str, object]: self.pool_snapshot_calls += 1 @@ -118,6 +183,8 @@ def dashboard_pool_snapshot(self, *, current_network_difficulty: object, generat "participants_3h": 2, "blocks_found_total": 3, "prism_blocks_total": 3, + "blocks_reversed_total": 1, + "blocks_inactive_total": 2, "total_mined_bits": 600, "latest_block": { "height": 123450, @@ -136,26 +203,52 @@ def dashboard_pool_snapshot(self, *, current_network_difficulty: object, generat }, } - def dashboard_blocks(self, *, page: int, limit: int) -> dict[str, object]: + def dashboard_blocks(self, *, page: int, limit: int, chain_state: str | None = None) -> dict[str, object]: + # chain_state=None records that the caller omitted the argument, which + # is what the default public /blocks path must do. + self.blocks_calls.append({"page": page, "limit": limit, "chain_state": chain_state}) + active_row = { + "height": 123450, + "hash": "a" * 64, + "found_at": "2026-06-26T19:55:00Z", + "network_difficulty": "1000", + "bits": "207fffff", + "solver_recipient_id": "miner-a", + "solver_worker_name": None, + "solver_share_difficulty": "99", + "reward_window_weight": "792", + "coinbase_value_bits": 600, + "audit_bundle_sha256": "b" * 64, + "payout_manifest_sha256": "c" * 64, + "explorer_url": None, + } + if chain_state in (None, "active"): + rows = [active_row] + else: + reversed_row = { + **active_row, + "height": 123449, + "hash": "f" * 64, + "found_at": "2026-06-26T18:55:00Z", + "solver_recipient_id": "miner-b", + "solver_share_difficulty": None, + "audit_bundle_sha256": None, + "payout_manifest_sha256": None, + "chain_state": "reversed", + "disconnected_at": "2026-06-26T19:05:00Z", + } + if chain_state == "all": + rows = [ + {**active_row, "chain_state": "confirmed", "disconnected_at": None}, + reversed_row, + ] + elif chain_state == "reversed": + rows = [reversed_row] + else: + raise ValueError("chain_state must be one of active, all, reversed") return { - "pagination": {"page": page, "limit": limit, "total_count": 1, "total_pages": 1}, - "rows": [ - { - "height": 123450, - "hash": "a" * 64, - "found_at": "2026-06-26T19:55:00Z", - "network_difficulty": "1000", - "bits": "207fffff", - "solver_recipient_id": "miner-a", - "solver_worker_name": None, - "solver_share_difficulty": "99", - "reward_window_weight": "792", - "coinbase_value_bits": 600, - "audit_bundle_sha256": "b" * 64, - "payout_manifest_sha256": "c" * 64, - "explorer_url": None, - } - ], + "pagination": {"page": page, "limit": limit, "total_count": len(rows), "total_pages": 1}, + "rows": rows, } def dashboard_leaderboard(self, *, page: int, limit: int, search: str | None = None) -> dict[str, object]: @@ -353,6 +446,7 @@ def dashboard_hashrate_series( lookback_seconds: int = 0, range_anchor_epoch: int | None = None, ) -> list[dict[str, object]]: + self.hashrate_series_calls += 1 self.last_hashrate_series_lookback = lookback_seconds self.last_hashrate_series_anchor = range_anchor_epoch if bucket == "5m": @@ -382,6 +476,56 @@ def dashboard_hashrate_series( } ] + def dashboard_block_markers( + self, + *, + range_id: str, + bucket: str, + range_anchor_epoch: int | None = None, + ) -> dict[str, object]: + # Mirrors the Postgres read model over the fixture records: reversed + # blocks excluded, floor bucketing on the hashrate grid, the + # hashrate_series_min_epoch lower bound for bounded ranges, per-bucket + # top-3 by found_at DESC then height DESC. + self.block_marker_calls += 1 + self.last_block_markers_anchor = range_anchor_epoch + bucket_seconds = public_api.HASHRATE_SERIES_BUCKET_SECONDS[bucket] + range_seconds = public_api.HASHRATE_SERIES_RANGE_SECONDS[range_id] + min_epoch = None + if range_seconds is not None: + anchor = range_anchor_epoch if range_anchor_epoch is not None else int(time.time()) + min_epoch = public_api.hashrate_series_min_epoch(anchor, range_seconds, bucket_seconds) + buckets: dict[int, list[tuple[int, str, int]]] = {} + total = 0 + for height, block_hash, found_epoch, chain_state in self.block_marker_records: + if chain_state == "reversed": + continue + if min_epoch is not None and found_epoch < min_epoch: + continue + bucket_epoch = found_epoch // bucket_seconds * bucket_seconds + buckets.setdefault(bucket_epoch, []).append((height, block_hash, found_epoch)) + total += 1 + points: list[dict[str, object]] = [] + for bucket_epoch in sorted(buckets): + records = sorted(buckets[bucket_epoch], key=lambda record: (-record[2], -record[0])) + points.append( + { + "timestamp": _bucket_timestamp(bucket_epoch), + "block_count": len(records), + "blocks": [ + { + "height": height, + "hash": block_hash, + "found_at": _bucket_timestamp(found_epoch), + } + for height, block_hash, found_epoch in records[ + : public_api.BLOCK_MARKERS_MAX_BLOCKS_PER_BUCKET + ] + ], + } + ) + return {"total_blocks": total, "points": points} + def all_shares(self) -> list[object]: return [ SimpleNamespace( @@ -494,23 +638,69 @@ def dashboard_hashrate_series( # Derive the boundary from the caller-provided anchor, exactly as the # endpoint's min_epoch trim does, so this stub is deterministic. anchor_epoch = range_anchor_epoch if range_anchor_epoch is not None else int(time.time()) - cutoff_epoch = public_api.hashrate_series_min_epoch(anchor_epoch, 7 * 86400, 300) + bucket_seconds = public_api.HASHRATE_SERIES_BUCKET_SECONDS[bucket] + cutoff_epoch = public_api.hashrate_series_min_epoch( + anchor_epoch, + 7 * 86400, + bucket_seconds, + ) return [ { - "timestamp": _bucket_timestamp(cutoff_epoch - 300), - "hashrate_ths": public_api.hashrate_ths_from_difficulty(600, 300), + "timestamp": _bucket_timestamp(cutoff_epoch - bucket_seconds), + "hashrate_ths": public_api.hashrate_ths_from_difficulty( + 600, + bucket_seconds, + ), "accepted_share_count": 1, "accepted_share_difficulty": "600", }, { - "timestamp": _bucket_timestamp(cutoff_epoch + 1200), - "hashrate_ths": public_api.hashrate_ths_from_difficulty(1200, 300), + "timestamp": _bucket_timestamp(cutoff_epoch + 4 * bucket_seconds), + "hashrate_ths": public_api.hashrate_ths_from_difficulty( + 1200, + bucket_seconds, + ), "accepted_share_count": 1, "accepted_share_difficulty": "1200", }, ] +class RangeBoundaryBlockMarkerLedger(FakePublicLedger): + """Serves one block just before the 1w marker cutoff and one just after. + + The records are derived from the caller-provided anchor with the same + hashrate_series_min_epoch arithmetic the read models apply, so the test + can pin the trim boundary deterministically: the pre-cutoff block sits in + the straddling bucket that must be dropped whole, and the post-cutoff + block sits in the first fully covered bucket. + """ + + def dashboard_block_markers( + self, + *, + range_id: str, + bucket: str, + range_anchor_epoch: int | None = None, + ) -> dict[str, object]: + anchor_epoch = range_anchor_epoch if range_anchor_epoch is not None else int(time.time()) + bucket_seconds = public_api.HASHRATE_SERIES_BUCKET_SECONDS[bucket] + cutoff_epoch = public_api.hashrate_series_min_epoch( + anchor_epoch, + 7 * 86400, + bucket_seconds, + ) + self.block_marker_records = [ + (100, "aa" * 32, cutoff_epoch - 1, "confirmed"), + (101, "bb" * 32, cutoff_epoch + 60, "confirmed"), + ] + return super().dashboard_block_markers( + range_id=range_id, + bucket=bucket, + range_anchor_epoch=range_anchor_epoch, + ) + + class BrokenPublicLedger(FakePublicLedger): def dashboard_leaderboard(self, *, page: int, limit: int, search: str | None = None) -> dict[str, object]: self.leaderboard_calls += 1 @@ -528,6 +718,112 @@ def dashboard_public_artifact(self, *, sha256: str) -> dict[str, object] | None: return super().dashboard_public_artifact(sha256=sha256) +def sample_ctv_manifest_set() -> dict[str, object]: + # Mirrors the tests/test_prism_share_ledger.py fixture: the minimal + # manifest set that passes ctv_fanout_recovery_payload validation. + parent_coinbase_txid = "11" * 32 + precommitment = { + "chunk_index": 0, + "chunk_count": 1, + "block_height": 123450, + "settlement_mode": "ctv_fanout", + "fanout_tx_template_hex": "0300000001", + "fanout_output_sum_sats": 25_000, + "anchor_vout": 1, + "ctv_hash_hex": "33" * 32, + } + manifest = { + "schema": "qbit.prism.ctv-fanout-manifest.v1", + "precommitment": precommitment, + "precommitment_sha256_hex": "44" * 32, + "commitment_witness_leaf_hex": "55" * 32, + "parent_coinbase_txid": parent_coinbase_txid, + "parent_coinbase_tx_hex": "0200000001", + "parent_coinbase_vout": 2, + "covenant_output_value_sats": 25_000, + "fanout_tx_hex": "0300000002", + "fanout_txid": "22" * 32, + } + return { + "schema": "qbit.prism.ctv-fanout-manifest-set.v1", + "block_height": 123450, + "settlement_mode": "ctv_fanout", + "parent_coinbase_txid": parent_coinbase_txid, + "fanout_count": 1, + "fanout_output_sum_sats": 25_000, + "covenant_output_value_sats": 25_000, + "manifests": [manifest], + } + + +class CanonicalArtifactPublicLedger(FakePublicLedger): + """Artifacts whose canonical text and advertised sha256 are genuine.""" + + def __init__(self) -> None: + super().__init__() + self.manifest_set = sample_ctv_manifest_set() + self.manifest = self.manifest_set["manifests"][0] # type: ignore[index] + self.manifest_sha256 = sha256_json_hex(self.manifest) + self.manifest_set_sha256 = sha256_json_hex(self.manifest_set) + self.audit_bundle = { + "schema": "qbit.prism.audit-bundle.v1", + "found_block": {"block_height": 123450}, + "accepted_shares": [{"share_seq": 1}], + } + # Audit bundles are hashed over these exact canonical bytes. Production + # persists the same sequence beside the compact external-body pointer. + self.audit_bundle_sha256 = hashlib.sha256( + json.dumps(self.audit_bundle, separators=(",", ":")).encode() + ).hexdigest() + + def dashboard_public_artifact(self, *, sha256: str) -> dict[str, object] | None: + if sha256 == self.audit_bundle_sha256: + return self.audit_bundle + if sha256 == self.manifest_set_sha256: + return self.manifest_set + if sha256 == self.manifest_sha256: + return self.manifest + return None + + def dashboard_public_artifact_document(self, *, sha256: str) -> dict[str, object] | None: + payload = self.dashboard_public_artifact(sha256=sha256) + if payload is None: + return None + canonical_json = None + if sha256 == self.audit_bundle_sha256: + canonical_json = json.dumps(self.audit_bundle, separators=(",", ":")) + if sha256 == self.manifest_set_sha256: + canonical_json = canonical_json_text(self.manifest_set) + if sha256 == self.manifest_sha256: + canonical_json = canonical_json_text(self.manifest) + return {"payload": payload, "canonical_json": canonical_json} + + +class TamperedCanonicalArtifactPublicLedger(CanonicalArtifactPublicLedger): + def dashboard_public_artifact_document(self, *, sha256: str) -> dict[str, object] | None: + document = super().dashboard_public_artifact_document(sha256=sha256) + if document is None or document.get("canonical_json") is None: + return document + # Same document, different bytes: must never be served raw. + return {**document, "canonical_json": str(document["canonical_json"]) + " "} + + +class MissingCanonicalArtifactPublicLedger(CanonicalArtifactPublicLedger): + def __init__(self) -> None: + super().__init__() + self.audit_document_calls = 0 + + def dashboard_public_artifact_document(self, *, sha256: str) -> dict[str, object] | None: + if sha256 != self.audit_bundle_sha256: + return super().dashboard_public_artifact_document(sha256=sha256) + self.audit_document_calls += 1 + return { + "payload": self.audit_bundle, + "canonical_json": None, + "canonical_fallback_reason": "missing", + } + + class DirectCoinbasePublicLedger(FakePublicLedger): block_hash = "00000000000026e9383a5aeae5fe5c3297f24884f29c4cf1585f71829491f0d9" block_height = 23342 @@ -648,6 +944,33 @@ def dashboard_leaderboard(self, *, page: int, limit: int, search: str | None = N } +class StatementTimeoutRecordingLedger(FakePublicLedger): + """Duck-typed operation_timeout scope, recording what the handler arms. + + Mirrors PsqlShareLedger's contextmanager signature so the read service's + getattr + callable probe takes the same branch it takes in production. + The operation scope, not the statement scope: one budget must cover the + whole dispatch, however many ledger reads it performs. + """ + + def __init__(self) -> None: + super().__init__() + self.statement_timeout_budgets: list[float] = [] + + @contextmanager + def operation_timeout(self, timeout_seconds: float) -> Iterator[None]: + self.statement_timeout_budgets.append(timeout_seconds) + yield + + +class TimingOutPublicLedger(StatementTimeoutRecordingLedger): + """Every hashrate query dies the way a server-cancelled statement does.""" + + def dashboard_hashrate_series(self, **kwargs: object) -> list[dict[str, object]]: + self.hashrate_series_calls += 1 + raise LedgerOperationTimeout("postgres statement deadline expired") + + class FakeCoordinator: def __init__(self, ledger: FakePublicLedger | None = None, rpc: FakeRpc | None = None) -> None: self.ledger = ledger or FakePublicLedger() @@ -664,7 +987,7 @@ def __init__(self) -> None: class PrismPublicDashboardApiTests(unittest.TestCase): def setUp(self) -> None: - handler = make_audit_handler(FakeCoordinator()) # type: ignore[arg-type] + handler = make_public_handler(FakeCoordinator()) # type: ignore[arg-type] self.server = ThreadingHTTPServer(("127.0.0.1", 0), handler) self.thread = threading.Thread(target=self.server.serve_forever, daemon=True) self.thread.start() @@ -748,7 +1071,7 @@ def test_network_summary_uses_valid_blockchain_bits_when_template_bits_are_missi def test_missing_compact_bits_fail_endpoints_before_reward_ledger_reads(self) -> None: ledger = FakePublicLedger() - handler = make_audit_handler( + handler = make_public_handler( FakeCoordinator(ledger=ledger, rpc=FakeRpc(template_bits=None)) ) # type: ignore[arg-type] server = ThreadingHTTPServer(("127.0.0.1", 0), handler) @@ -809,6 +1132,266 @@ def test_blocks_leaderboard_and_hashrate_series(self) -> None: # rates pass through untouched. self.assertEqual(series["points"][0]["hashrate_ths"], "2.5") + BLOCK_ROW_V1_KEYS = { + "height", + "hash", + "found_at", + "network_difficulty", + "bits", + "solver_recipient_id", + "solver_worker_name", + "solver_share_difficulty", + "reward_window_weight", + "coinbase_value_bits", + "audit_bundle_sha256", + "payout_manifest_sha256", + "explorer_url", + } + + def test_blocks_default_response_keeps_v1_shape_and_omits_state_fields(self) -> None: + ledger = FakePublicLedger() + coordinator = FakeCoordinator(ledger=ledger) + + for query in ({}, {"chain_state": ["active"]}): + with self.subTest(query=query): + status, payload = public_api.dispatch(coordinator, "/public/v1/blocks", query) + + self.assertEqual(status, 200) + self.assertEqual(payload["schema"], "prism.dashboard.blocks.v1") + for row in payload["rows"]: + self.assertEqual(set(row), self.BLOCK_ROW_V1_KEYS) + # Both the implicit default and an explicit chain_state=active reach + # the read model without any chain_state argument, so a ledger that + # predates the filter keeps serving the default contract. + self.assertEqual( + ledger.blocks_calls, + [ + {"page": 1, "limit": 15, "chain_state": None}, + {"page": 1, "limit": 15, "chain_state": None}, + ], + ) + + def test_blocks_chain_state_filter_returns_v2_rows_and_filtered_pagination(self) -> None: + ledger = FakePublicLedger() + coordinator = FakeCoordinator(ledger=ledger) + + status, all_payload = public_api.dispatch( + coordinator, "/public/v1/blocks", {"chain_state": ["all"]} + ) + self.assertEqual(status, 200) + self.assertEqual(all_payload["schema"], "prism.dashboard.blocks.v2") + self.assertEqual(all_payload["pagination"]["total_count"], 2) + for row in all_payload["rows"]: + self.assertEqual(set(row), self.BLOCK_ROW_V1_KEYS | {"chain_state", "disconnected_at"}) + self.assertEqual(all_payload["rows"][0]["chain_state"], "confirmed") + self.assertIsNone(all_payload["rows"][0]["disconnected_at"]) + self.assertEqual(all_payload["rows"][1]["chain_state"], "reversed") + self.assertEqual(all_payload["rows"][1]["disconnected_at"], "2026-06-26T19:05:00Z") + + status, reversed_payload = public_api.dispatch( + coordinator, "/public/v1/blocks", {"chain_state": ["reversed"]} + ) + self.assertEqual(status, 200) + self.assertEqual(reversed_payload["schema"], "prism.dashboard.blocks.v2") + self.assertEqual(reversed_payload["pagination"]["total_count"], 1) + self.assertEqual( + [row["chain_state"] for row in reversed_payload["rows"]], + ["reversed"], + ) + self.assertEqual( + ledger.blocks_calls, + [ + {"page": 1, "limit": 15, "chain_state": "all"}, + {"page": 1, "limit": 15, "chain_state": "reversed"}, + ], + ) + + def test_blocks_rejects_unknown_chain_state(self) -> None: + with self.assertRaises(urllib.error.HTTPError) as raised: + self.get_json("/public/v1/blocks?chain_state=orphaned") + + self.assertEqual(raised.exception.code, 400) + payload = json.loads(raised.exception.read()) + raised.exception.close() + self.assertEqual(payload["error"]["code"], "bad_request") + + def test_origin_cache_separates_blocks_chain_states(self) -> None: + ledger = FakePublicLedger() + handler = make_public_handler(FakeCoordinator(ledger=ledger)) # type: ignore[arg-type] + server = ThreadingHTTPServer(("127.0.0.1", 0), handler) + thread = threading.Thread(target=server.serve_forever, daemon=True) + thread.start() + try: + with patch.dict(os.environ, {"PRISM_PUBLIC_CACHE_TTL_SECONDS": "30"}, clear=True): + base = f"http://127.0.0.1:{server.server_port}/public/v1/blocks" + for url in (base, base, f"{base}?chain_state=all"): + with urllib.request.urlopen(url, timeout=5) as response: + payload = json.loads(response.read()) + with urllib.request.urlopen(base, timeout=5) as response: + default_payload = json.loads(response.read()) + finally: + server.shutdown() + server.server_close() + thread.join(timeout=5) + + # chain_state partitions the origin cache automatically because the + # cache key carries the sorted query string: the repeated default + # requests share one entry while chain_state=all computes its own. + self.assertEqual(payload["schema"], "prism.dashboard.blocks.v2") + self.assertEqual(default_payload["schema"], "prism.dashboard.blocks.v1") + self.assertEqual( + [call["chain_state"] for call in ledger.blocks_calls], + [None, "all"], + ) + + def test_pool_summary_reports_reorg_counts_and_network_hashrate(self) -> None: + rpc = FakeRpc(network_hashps=2_500_000_000_000_000_000) + payload = public_api.pool_summary(FakeCoordinator(rpc=rpc)) + + self.assertEqual(payload["pool"]["blocks_reversed_total"], 1) + self.assertEqual(payload["pool"]["blocks_inactive_total"], 2) + # 2.5e18 H/s reported by the node is 2,500,000 TH/s, and the lookup + # asks for the permissionless lane the pool's shares are credited + # against, over the RPC's own default 120-block window at the tip. + self.assertEqual(payload["network"]["hashrate_ths"], "2500000") + self.assertEqual(rpc.network_hashps_calls, [[120, -1, "permissionless"]]) + + def test_pool_summary_network_hashrate_is_null_when_rpc_fails(self) -> None: + rpc = FakeRpc(network_hashps=RuntimeError("getnetworkhashps unavailable")) + payload = public_api.pool_summary(FakeCoordinator(rpc=rpc)) + + # The field is nullable and non-fatal: pool-summary still answers 200 + # with everything else populated, and only compact-bits loss may 503. + self.assertIsNone(payload["network"]["hashrate_ths"]) + self.assertEqual(payload["schema"], "prism.dashboard.pool-summary.v1") + self.assertEqual(payload["pool"]["blocks_found_total"], 3) + + def test_network_hashrate_ths_rejects_non_numeric_estimates(self) -> None: + for network_hashps in (float("nan"), float("inf"), -1, True, "not-a-rate", None, {}): + with self.subTest(network_hashps=network_hashps): + coordinator = FakeCoordinator(rpc=FakeRpc(network_hashps=network_hashps)) + self.assertIsNone(public_api.network_hashrate_ths(coordinator)) + fractional = FakeCoordinator(rpc=FakeRpc(network_hashps=1_500_000_000_000.5)) + self.assertEqual(public_api.network_hashrate_ths(fractional), "1.5000000000005") + + def test_block_markers_shape_alignment_and_truncation(self) -> None: + markers = self.get_json("/public/v1/block-markers?range=1m&bucket=auto") + + self.assertEqual(markers["schema"], "prism.dashboard.block-markers.v1") + self.assertEqual(markers["range"], "1m") + # auto resolves exactly as it does for hashrate-series: 1h for 1m. + self.assertEqual(markers["bucket"], "1h") + self.assertEqual(markers["bucket_seconds"], 3600) + # The reversed block and the out-of-range ancient block are excluded + # from the total; empty buckets are omitted rather than zero-filled. + self.assertEqual(markers["total_blocks"], 5) + points = markers["points"] + self.assertEqual(len(points), 2) + epochs = [_epoch_from_timestamp(str(point["timestamp"])) for point in points] + self.assertEqual(epochs, sorted(epochs)) + for point, epoch in zip(points, epochs): + # Marker buckets sit on the hashrate chart's grid: the same floor + # arithmetic hashrate-series buckets use, so every timestamp is + # bucket-aligned and every block floors into its point's bucket. + self.assertEqual(epoch % 3600, 0) + for block in point["blocks"]: + found_epoch = _epoch_from_timestamp(str(block["found_at"])) + self.assertEqual(found_epoch // 3600 * 3600, epoch) + self.assertEqual(points[0]["block_count"], 1) + self.assertFalse(points[0]["truncated"]) + self.assertEqual(points[0]["blocks"][0]["height"], 67440) + # Four accepted blocks in the newer bucket: only the three most recent + # survive, ordered found_at DESC with height DESC breaking the tie. + self.assertEqual(points[1]["block_count"], 4) + self.assertTrue(points[1]["truncated"]) + self.assertEqual( + [block["height"] for block in points[1]["blocks"]], + [67444, 67443, 67442], + ) + found_ats = [ + _epoch_from_timestamp(str(block["found_at"])) + for block in points[1]["blocks"] + ] + self.assertEqual(found_ats, sorted(found_ats, reverse=True)) + self.assertNotIn("1f" * 32, json.dumps(markers)) + + def test_block_markers_all_range_reaches_unbounded_history(self) -> None: + markers = self.get_json("/public/v1/block-markers?range=all&bucket=1d") + + self.assertEqual(markers["bucket"], "1d") + self.assertEqual(markers["bucket_seconds"], 86400) + # `all` has no lower bound, so the ancient block joins the total. + self.assertEqual(markers["total_blocks"], 6) + points = markers["points"] + self.assertEqual( + sum(int(point["block_count"]) for point in points), + markers["total_blocks"], + ) + self.assertEqual(points[0]["blocks"][0]["hash"], "2a" * 32) + for point in points: + self.assertEqual(_epoch_from_timestamp(str(point["timestamp"])) % 86400, 0) + # An unbounded range carries no anchor: there is no lower bound for + # the ledger to derive from it. + ledger = FakePublicLedger() + ledger.last_block_markers_anchor = -1 + public_api.block_markers(FakeCoordinator(ledger=ledger), range_id="all", bucket="1d") + self.assertIsNone(ledger.last_block_markers_anchor) + + def test_block_markers_pass_range_anchor_and_trim_straddling_bucket(self) -> None: + ledger = RangeBoundaryBlockMarkerLedger() + handler = make_public_handler(FakeCoordinator(ledger=ledger)) # type: ignore[arg-type] + server = ThreadingHTTPServer(("127.0.0.1", 0), handler) + thread = threading.Thread(target=server.serve_forever, daemon=True) + thread.start() + try: + url = f"http://127.0.0.1:{server.server_port}/public/v1/block-markers?range=1w&bucket=5m" + with urllib.request.urlopen(url, timeout=5) as response: + markers = json.loads(response.read()) + finally: + server.shutdown() + server.server_close() + thread.join(timeout=5) + + # The endpoint anchors the range on its own clock and the ledger keeps + # only fully covered buckets, so the block in the straddling bucket is + # dropped whole while the first in-range bucket survives -- the same + # boundary the hashrate series trims against. + self.assertIsNotNone(ledger.last_block_markers_anchor) + self.assertEqual(markers["total_blocks"], 1) + self.assertEqual(len(markers["points"]), 1) + self.assertEqual(markers["points"][0]["blocks"][0]["hash"], "bb" * 32) + self.assertNotIn("aa" * 32, json.dumps(markers)) + cutoff_epoch = public_api.hashrate_series_min_epoch( + int(ledger.last_block_markers_anchor), + 7 * 86400, + 300, + ) + self.assertEqual( + _epoch_from_timestamp(str(markers["points"][0]["timestamp"])), + cutoff_epoch, + ) + + def test_block_markers_reject_invalid_range_bucket_and_clamp_combos(self) -> None: + for path in ( + "/public/v1/block-markers?range=2y", + "/public/v1/block-markers?range=1w&bucket=7m", + # Every clamped combination: finer buckets than the range's chart + # renders are rejected rather than served unboundedly long. + "/public/v1/block-markers?range=1m&bucket=5m", + "/public/v1/block-markers?range=6m&bucket=5m", + "/public/v1/block-markers?range=6m&bucket=1h", + "/public/v1/block-markers?range=all&bucket=5m", + "/public/v1/block-markers?range=all&bucket=1h", + ): + with self.subTest(path=path): + with self.assertRaises(urllib.error.HTTPError) as raised: + self.get_json(path) + self.assertEqual(raised.exception.code, 400) + payload = json.loads(raised.exception.read()) + raised.exception.close() + self.assertEqual(payload["schema"], "prism.dashboard.error.v1") + self.assertEqual(payload["error"]["code"], "bad_request") + def assert_hashrate_ths(self, actual: object, difficulty: int, seconds: int) -> None: # Hashrate strings are Decimal-context dependent (direct_stratum pins # prec=40 in the importing thread while HTTP handler threads use the @@ -845,7 +1428,7 @@ def test_hashrate_series_smoothing_disabled_by_env(self) -> None: def test_hashrate_series_fetches_lookback_and_trims_pre_range_points(self) -> None: ledger = RangeBoundaryPublicLedger() - handler = make_audit_handler(FakeCoordinator(ledger=ledger)) # type: ignore[arg-type] + handler = make_public_handler(FakeCoordinator(ledger=ledger)) # type: ignore[arg-type] server = ThreadingHTTPServer(("127.0.0.1", 0), handler) thread = threading.Thread(target=server.serve_forever, daemon=True) thread.start() @@ -947,6 +1530,330 @@ def test_smooth_hashrate_series_points_drops_malformed_points(self) -> None: self.assertEqual(len(smoothed), 1) self.assertEqual(smoothed[0]["timestamp"], _bucket_timestamp(cutoff_epoch)) + V1_SERIES_KEYS = {"schema", "generated_at", "subject", "range", "bucket", "unit", "points"} + V1_POINT_KEYS = {"timestamp", "hashrate_ths", "accepted_share_count", "accepted_share_difficulty"} + V2_POINT_KEYS = { + "timestamp", + "raw_hashrate_ths", + "smoothed_hashrate_ths", + "accepted_share_count", + "accepted_share_difficulty", + "complete", + } + + def test_hashrate_series_default_view_retains_exact_v1_shape(self) -> None: + fixture = _load_fixture("hashrate-series.json") + for suffix in ("",): + with urllib.request.urlopen( + f"{self.base_url}/public/v1/hashrate-series?subject=pool&range=1w&bucket=5m{suffix}", + timeout=5, + ) as response: + body = response.read() + series = json.loads(body) + + self.assertEqual(series["schema"], "prism.dashboard.hashrate-series.v1") + self.assertEqual(set(series), self.V1_SERIES_KEYS) + self.assertEqual(set(series), set(fixture)) + self.assertEqual(len(series["points"]), 2) + for point in series["points"]: + self.assertEqual(set(point), self.V1_POINT_KEYS) + self.assertEqual(set(point), set(fixture["points"][0])) + # The legacy body carries none of the dual-rate vocabulary. + for token in (b"raw_hashrate_ths", b"smoothed_hashrate_ths", b"bucket_seconds", b"rate_basis", b"smoothing", b"complete"): + self.assertNotIn(token, body) + # And still serves the trailing-window rate as hashrate_ths. + self.assert_hashrate_ths(series["points"][0]["hashrate_ths"], 600, 1800) + self.assert_hashrate_ths(series["points"][1]["hashrate_ths"], 1800, 1800) + + def test_hashrate_series_both_view_returns_raw_and_smoothed_rates(self) -> None: + fixture = _load_fixture("hashrate-series-dual-rate.json") + series = self.get_json("/public/v1/hashrate-series?subject=pool&range=1w&bucket=5m&view=both") + + self.assertEqual(series["schema"], "prism.dashboard.hashrate-series.v2") + self.assertEqual(set(series), set(fixture)) + self.assertEqual(series["subject"], {"type": "pool", "id": None}) + self.assertEqual(series["range"], "1w") + self.assertEqual(series["bucket"], "5m") + self.assertEqual(series["bucket_seconds"], 300) + self.assertEqual(series["unit"], "ths") + self.assertEqual(series["rate_basis"], "accepted_share_difficulty") + self.assertEqual(series["smoothing"], {"method": "trailing", "window_seconds": 1800}) + + points = series["points"] + self.assertEqual(len(points), 2) + for point in points: + self.assertEqual(set(point), self.V2_POINT_KEYS) + self.assertEqual(set(point), set(fixture["points"][0])) + self.assertEqual([point["accepted_share_difficulty"] for point in points], ["600", "1200"]) + self.assertEqual([point["accepted_share_count"] for point in points], [2, 1]) + # Raw divides the bucket's credit by the full 5m bucket; smoothed is the + # 30m trailing estimate v1 serves, so the two differ for the same point. + self.assert_hashrate_ths(points[0]["raw_hashrate_ths"], 600, 300) + self.assert_hashrate_ths(points[0]["smoothed_hashrate_ths"], 600, 1800) + self.assert_hashrate_ths(points[1]["raw_hashrate_ths"], 1200, 300) + self.assert_hashrate_ths(points[1]["smoothed_hashrate_ths"], 1800, 1800) + for point in points: + self.assertNotEqual(Decimal(point["raw_hashrate_ths"]), Decimal(point["smoothed_hashrate_ths"])) + # The older bucket ended before generated_at; the current one is still + # accumulating credit. + self.assertEqual([point["complete"] for point in points], [True, False]) + generated_at_epoch = int( + datetime.strptime(series["generated_at"], "%Y-%m-%dT%H:%M:%SZ").replace(tzinfo=timezone.utc).timestamp() + ) + for point in points: + point_epoch = int( + datetime.strptime(point["timestamp"], "%Y-%m-%dT%H:%M:%SZ").replace(tzinfo=timezone.utc).timestamp() + ) + self.assertEqual(point["complete"], generated_at_epoch >= point_epoch + 300) + + def test_hashrate_series_both_view_reports_no_smoothing_when_disabled(self) -> None: + with patch.dict(os.environ, {"PRISM_PUBLIC_HASHRATE_SMOOTHING_SECONDS": "0"}): + series = self.get_json("/public/v1/hashrate-series?subject=pool&range=1w&bucket=5m&view=both") + + self.assertEqual(series["schema"], "prism.dashboard.hashrate-series.v2") + self.assertEqual(series["smoothing"], {"method": "none", "window_seconds": 300}) + points = series["points"] + self.assertEqual(len(points), 2) + self.assert_hashrate_ths(points[0]["raw_hashrate_ths"], 600, 300) + self.assert_hashrate_ths(points[1]["raw_hashrate_ths"], 1200, 300) + for point in points: + self.assertEqual(point["smoothed_hashrate_ths"], point["raw_hashrate_ths"]) + + def test_hashrate_series_both_view_reports_no_smoothing_for_coarse_buckets(self) -> None: + # The default 30m window is shorter than two 1h buckets, so the v1 + # series passes ledger rates through; v2 says so explicitly. + ledger = RangeBoundaryPublicLedger() + series = public_api.hashrate_series( + FakeCoordinator(ledger=ledger), # type: ignore[arg-type] + subject="miner:miner-a", + range_id="1w", + bucket="auto", + view="both", + ) + + self.assertEqual(series["schema"], "prism.dashboard.hashrate-series.v2") + self.assertEqual(series["subject"], {"type": "miner", "id": "miner-a"}) + self.assertEqual(series["bucket"], "1h") + self.assertEqual(series["bucket_seconds"], 3600) + self.assertEqual(series["smoothing"], {"method": "none", "window_seconds": 3600}) + self.assertEqual(len(series["points"]), 1) + point = series["points"][0] + self.assertEqual(point["accepted_share_count"], 1) + self.assertEqual(point["accepted_share_difficulty"], "1200") + self.assert_hashrate_ths(point["raw_hashrate_ths"], 1200, 3600) + self.assertEqual(point["smoothed_hashrate_ths"], point["raw_hashrate_ths"]) + self.assertTrue(point["complete"]) + + def test_hashrate_series_both_view_trims_partial_leading_coarse_bucket(self) -> None: + ledger = RangeBoundaryPublicLedger() + series = public_api.hashrate_series( + FakeCoordinator(ledger=ledger), # type: ignore[arg-type] + subject="pool", + range_id="1w", + bucket="auto", + view="both", + ) + + self.assertEqual(series["smoothing"], {"method": "none", "window_seconds": 3600}) + self.assertEqual(ledger.last_hashrate_series_lookback, 0) + self.assertIsNotNone(ledger.last_hashrate_series_anchor) + self.assertEqual(len(series["points"]), 1) + self.assertEqual(series["points"][0]["accepted_share_difficulty"], "1200") + self.assertTrue(series["points"][0]["complete"]) + + def test_hashrate_series_both_view_trims_lookback_but_keeps_its_smoothing(self) -> None: + ledger = RangeBoundaryPublicLedger() + handler = make_public_handler(FakeCoordinator(ledger=ledger)) # type: ignore[arg-type] + server = ThreadingHTTPServer(("127.0.0.1", 0), handler) + thread = threading.Thread(target=server.serve_forever, daemon=True) + thread.start() + try: + url = ( + f"http://127.0.0.1:{server.server_port}/public/v1/hashrate-series" + "?subject=pool&range=1w&bucket=5m&view=both" + ) + with urllib.request.urlopen(url, timeout=5) as response: + series = json.loads(response.read()) + finally: + server.shutdown() + server.server_close() + thread.join(timeout=5) + + self.assertEqual(ledger.last_hashrate_series_lookback, 1800) + self.assertIsNotNone(ledger.last_hashrate_series_anchor) + points = series["points"] + # The pre-range bucket is trimmed from the response, but its credit + # still feeds the kept point's trailing window while the raw rate + # describes only the kept bucket. + self.assertEqual(len(points), 1) + self.assertEqual(points[0]["accepted_share_difficulty"], "1200") + self.assert_hashrate_ths(points[0]["raw_hashrate_ths"], 1200, 300) + self.assert_hashrate_ths(points[0]["smoothed_hashrate_ths"], 1800, 1800) + self.assertTrue(points[0]["complete"]) + + def test_hashrate_series_rejects_unknown_view(self) -> None: + for view in ("raw", "smoothed", "BOTH", "both,raw"): + with self.assertRaises(urllib.error.HTTPError) as raised: + self.get_json(f"/public/v1/hashrate-series?subject=pool&range=1w&bucket=5m&view={view}") + + self.assertEqual(raised.exception.code, 400, view) + payload = json.loads(raised.exception.read()) + raised.exception.close() + self.assertEqual(payload["schema"], "prism.dashboard.error.v1") + self.assertEqual( + payload["error"], + {"code": "bad_request", "message": "view must be both or omitted", "request_id": None}, + ) + + def test_origin_cache_separates_hashrate_series_views(self) -> None: + path = "/public/v1/hashrate-series" + base_query = {"subject": ["pool"], "range": ["1w"], "bucket": ["5m"]} + self.assertNotEqual( + public_api.public_cache_key(path, base_query), + public_api.public_cache_key(path, {**base_query, "view": ["both"]}), + ) + + ledger = FakePublicLedger() + handler = make_public_handler(FakeCoordinator(ledger=ledger)) # type: ignore[arg-type] + server = ThreadingHTTPServer(("127.0.0.1", 0), handler) + thread = threading.Thread(target=server.serve_forever, daemon=True) + thread.start() + try: + with patch.dict(os.environ, {"PRISM_PUBLIC_AGGREGATE_CACHE_TTL_SECONDS": "30"}, clear=True): + schemas = [] + for suffix in ("", "&view=both", ""): + with urllib.request.urlopen( + f"http://127.0.0.1:{server.server_port}{path}?subject=pool&range=1w&bucket=5m{suffix}", + timeout=5, + ) as response: + schemas.append(json.loads(response.read())["schema"]) + finally: + server.shutdown() + server.server_close() + thread.join(timeout=5) + + self.assertEqual( + schemas, + [ + "prism.dashboard.hashrate-series.v1", + "prism.dashboard.hashrate-series.v2", + "prism.dashboard.hashrate-series.v1", + ], + ) + # One origin computation per view; the repeated v1 request is a hit. + self.assertEqual(ledger.hashrate_series_calls, 2) + + def test_hashrate_series_smoothing_descriptor_matches_smoother_threshold(self) -> None: + self.assertEqual( + public_api.hashrate_series_smoothing(bucket_seconds=300, window_seconds=1800), + {"method": "trailing", "window_seconds": 1800}, + ) + # Partial buckets are dropped from the effective window. + self.assertEqual( + public_api.hashrate_series_smoothing(bucket_seconds=300, window_seconds=899), + {"method": "trailing", "window_seconds": 600}, + ) + for bucket_seconds, window_seconds in ((300, 0), (300, 599), (3600, 1800), (86400, 86400)): + self.assertEqual( + public_api.hashrate_series_smoothing(bucket_seconds=bucket_seconds, window_seconds=window_seconds), + {"method": "none", "window_seconds": bucket_seconds}, + (bucket_seconds, window_seconds), + ) + + def test_dual_rate_points_complete_flag_follows_bucket_end(self) -> None: + epoch = 1_750_000_000 // 300 * 300 + point = { + "timestamp": _bucket_timestamp(epoch), + "hashrate_ths": "0", + "accepted_share_count": 4, + "accepted_share_difficulty": "900", + } + for generated_at_epoch, complete in ((epoch, False), (epoch + 299, False), (epoch + 300, True), (epoch + 301, True)): + points = public_api.dual_rate_hashrate_series_points( + [point], + bucket_seconds=300, + window_seconds=1800, + generated_at_epoch=generated_at_epoch, + ) + self.assertEqual(len(points), 1) + self.assertEqual(points[0]["complete"], complete, generated_at_epoch - epoch) + self.assertEqual(points[0]["timestamp"], _bucket_timestamp(epoch)) + self.assertEqual(points[0]["accepted_share_count"], 4) + self.assertEqual(points[0]["accepted_share_difficulty"], "900") + self.assertEqual(points[0]["raw_hashrate_ths"], public_api.hashrate_ths_from_difficulty(900, 300)) + + def test_dual_rate_points_match_v1_smoothing_and_trim_lookback(self) -> None: + base_epoch = 1_750_000_000 // 300 * 300 + series = [ + (base_epoch, 100), + (base_epoch + 300, 250), + (base_epoch + 900, 75), + (base_epoch + 1500, 500), + (base_epoch + 1800, 125), + (base_epoch + 3600, 900), + ] + malformed = { + "timestamp": "not-a-timestamp", + "hashrate_ths": "2.5", + "accepted_share_count": 1, + "accepted_share_difficulty": "100", + } + points = [ + { + "timestamp": _bucket_timestamp(epoch), + "hashrate_ths": "0", + "accepted_share_count": 1, + "accepted_share_difficulty": str(difficulty), + } + for epoch, difficulty in series + ] + generated_at_epoch = base_epoch + 3600 + 150 + smoothed = public_api.smooth_hashrate_series_points( + list(points), bucket_seconds=300, window_seconds=1800 + ) + dual_rate = public_api.dual_rate_hashrate_series_points( + [malformed, *reversed(points)], + bucket_seconds=300, + window_seconds=1800, + generated_at_epoch=generated_at_epoch, + ) + self.assertEqual(len(dual_rate), len(series)) + for (epoch, difficulty), v1_point, v2_point in zip(series, smoothed, dual_rate): + self.assertEqual(v2_point["timestamp"], _bucket_timestamp(epoch)) + self.assertEqual(v2_point["smoothed_hashrate_ths"], v1_point["hashrate_ths"]) + self.assertEqual(v2_point["raw_hashrate_ths"], public_api.hashrate_ths_from_difficulty(difficulty, 300)) + self.assertEqual(v2_point["complete"], epoch != base_epoch + 3600) + + # Trimming keeps the lookback credit inside the first kept window. + trimmed = public_api.dual_rate_hashrate_series_points( + points, + bucket_seconds=300, + window_seconds=1800, + min_epoch=base_epoch + 900, + generated_at_epoch=generated_at_epoch, + ) + self.assertEqual([point["timestamp"] for point in trimmed], [_bucket_timestamp(epoch) for epoch, _ in series[2:]]) + self.assertEqual(trimmed[0]["raw_hashrate_ths"], public_api.hashrate_ths_from_difficulty(75, 300)) + self.assertEqual(trimmed[0]["smoothed_hashrate_ths"], public_api.hashrate_ths_from_difficulty(425, 1800)) + + # Below the two-bucket threshold the smoothed rate is the raw rate. + unsmoothed = public_api.dual_rate_hashrate_series_points( + points, + bucket_seconds=300, + window_seconds=300, + generated_at_epoch=generated_at_epoch, + ) + for point in unsmoothed: + self.assertEqual(point["smoothed_hashrate_ths"], point["raw_hashrate_ths"]) + + count_as_decimal = public_api.dual_rate_hashrate_series_points( + [{**points[0], "accepted_share_count": "3.0"}], + bucket_seconds=300, + window_seconds=1800, + generated_at_epoch=generated_at_epoch, + ) + self.assertEqual(count_as_decimal[0]["accepted_share_count"], 3) + def test_omitted_leaderboard_window_retains_exact_legacy_v1_keys(self) -> None: payload = self.get_json("/public/v1/leaderboard") @@ -1031,7 +1938,7 @@ def test_legacy_leaderboard_rejects_reward_recipient_filter(self) -> None: def test_origin_cache_separates_legacy_and_reward_leaderboards(self) -> None: ledger = FakePublicLedger() - handler = make_audit_handler(FakeCoordinator(ledger=ledger)) # type: ignore[arg-type] + handler = make_public_handler(FakeCoordinator(ledger=ledger)) # type: ignore[arg-type] server = ThreadingHTTPServer(("127.0.0.1", 0), handler) thread = threading.Thread(target=server.serve_forever, daemon=True) thread.start() @@ -1099,7 +2006,7 @@ def test_public_cache_policy_aggregate_ttl_honors_env_and_kill_switch(self) -> N def test_public_api_successes_emit_cache_headers_and_hit_origin_cache(self) -> None: ledger = FakePublicLedger() - handler = make_audit_handler(FakeCoordinator(ledger=ledger)) # type: ignore[arg-type] + handler = make_public_handler(FakeCoordinator(ledger=ledger)) # type: ignore[arg-type] server = ThreadingHTTPServer(("127.0.0.1", 0), handler) thread = threading.Thread(target=server.serve_forever, daemon=True) thread.start() @@ -1136,13 +2043,13 @@ def test_public_api_successes_emit_cache_headers_and_hit_origin_cache(self) -> N self.assertEqual(first_age, "0") self.assertEqual(second_age, "0") self.assertEqual(first_browser_cache, "public, max-age=0, must-revalidate") - self.assertEqual(first_cdn_cache, "public, max-age=30, stale-while-revalidate=30") + self.assertEqual(first_cdn_cache, "public, max-age=15") self.assertEqual(second_cdn_cache, first_cdn_cache) self.assertEqual(ledger.leaderboard_calls, 1) def test_public_api_errors_are_no_store_and_not_cached(self) -> None: ledger = BrokenPublicLedger() - handler = make_audit_handler(FakeCoordinator(ledger)) # type: ignore[arg-type] + handler = make_public_handler(FakeCoordinator(ledger)) # type: ignore[arg-type] server = ThreadingHTTPServer(("127.0.0.1", 0), handler) thread = threading.Thread(target=server.serve_forever, daemon=True) thread.start() @@ -1164,7 +2071,7 @@ def test_public_api_errors_are_no_store_and_not_cached(self) -> None: def test_public_api_coalesces_concurrent_origin_cache_misses(self) -> None: ledger = SlowPublicLedger() - handler = make_audit_handler(FakeCoordinator(ledger)) # type: ignore[arg-type] + handler = make_public_handler(FakeCoordinator(ledger)) # type: ignore[arg-type] server = ThreadingHTTPServer(("127.0.0.1", 0), handler) thread = threading.Thread(target=server.serve_forever, daemon=True) thread.start() @@ -1205,7 +2112,7 @@ def test_public_api_coalesces_concurrent_misses_when_response_not_cacheable(self # An oversize (non-cacheable) response must still coalesce concurrent # waiters onto the owner's single origin call instead of re-running it. ledger = SlowPublicLedger() - handler = make_audit_handler(FakeCoordinator(ledger)) # type: ignore[arg-type] + handler = make_public_handler(FakeCoordinator(ledger)) # type: ignore[arg-type] server = ThreadingHTTPServer(("127.0.0.1", 0), handler) thread = threading.Thread(target=server.serve_forever, daemon=True) thread.start() @@ -1315,7 +2222,7 @@ def recompute() -> tuple[int, object]: def test_public_api_cache_headers_use_route_specific_ttls(self) -> None: ledger = FakePublicLedger() - handler = make_audit_handler(FakeCoordinator(ledger=ledger)) # type: ignore[arg-type] + handler = make_public_handler(FakeCoordinator(ledger=ledger)) # type: ignore[arg-type] server = ThreadingHTTPServer(("127.0.0.1", 0), handler) thread = threading.Thread(target=server.serve_forever, daemon=True) thread.start() @@ -1345,7 +2252,7 @@ def test_public_api_cache_headers_use_route_specific_ttls(self) -> None: server.server_close() thread.join(timeout=5) - self.assertEqual(config_cache, "public, max-age=600, stale-while-revalidate=3600") + self.assertEqual(config_cache, "public, max-age=600, stale-while-revalidate=300") self.assertEqual(artifact_cache, "public, max-age=7200, stale-while-revalidate=86400, immutable") def test_public_api_errors_use_public_error_schema(self) -> None: @@ -1425,7 +2332,7 @@ def test_scaled_network_difficulty_rejects_non_positive_targets(self) -> None: public_api.scaled_network_difficulty("00000000") def test_unexpected_public_api_errors_do_not_leak_internal_details(self) -> None: - handler = make_audit_handler(FakeCoordinator(BrokenPublicLedger())) # type: ignore[arg-type] + handler = make_public_handler(FakeCoordinator(BrokenPublicLedger())) # type: ignore[arg-type] server = ThreadingHTTPServer(("127.0.0.1", 0), handler) thread = threading.Thread(target=server.serve_forever, daemon=True) thread.start() @@ -1450,7 +2357,7 @@ def test_unexpected_public_api_errors_do_not_leak_internal_details(self) -> None self.assertEqual(payload["error"], {"code": "internal_error", "message": "internal server error", "request_id": None}) def test_public_value_errors_do_not_leak_internal_details(self) -> None: - handler = make_audit_handler(FakeCoordinator(ValueErrorPublicLedger())) # type: ignore[arg-type] + handler = make_public_handler(FakeCoordinator(ValueErrorPublicLedger())) # type: ignore[arg-type] server = ThreadingHTTPServer(("127.0.0.1", 0), handler) thread = threading.Thread(target=server.serve_forever, daemon=True) thread.start() @@ -1503,7 +2410,7 @@ def test_settlement_fanout_and_artifact_endpoints(self) -> None: def test_settlement_artifacts_returns_direct_coinbase_bundle_without_fanouts(self) -> None: ledger = DirectCoinbasePublicLedger() - handler = make_audit_handler(FakeCoordinator(ledger=ledger)) # type: ignore[arg-type] + handler = make_public_handler(FakeCoordinator(ledger=ledger)) # type: ignore[arg-type] server = ThreadingHTTPServer(("127.0.0.1", 0), handler) thread = threading.Thread(target=server.serve_forever, daemon=True) thread.start() @@ -1541,7 +2448,7 @@ def test_settlement_artifacts_uses_metadata_only_artifact_availability_when_pres def test_settlement_artifacts_supports_externalized_direct_coinbase_audit_bundle(self) -> None: with tempfile.TemporaryDirectory() as tmp: ledger = ExternalizedDirectCoinbasePublicLedger(tmp) - handler = make_audit_handler(FakeCoordinator(ledger=ledger)) # type: ignore[arg-type] + handler = make_public_handler(FakeCoordinator(ledger=ledger)) # type: ignore[arg-type] server = ThreadingHTTPServer(("127.0.0.1", 0), handler) thread = threading.Thread(target=server.serve_forever, daemon=True) thread.start() @@ -1568,7 +2475,7 @@ def test_settlement_artifacts_returns_404_when_externalized_audit_body_is_unread with tempfile.TemporaryDirectory() as tmp: ledger = ExternalizedDirectCoinbasePublicLedger(tmp) ledger.body_uri.unlink() - handler = make_audit_handler(FakeCoordinator(ledger=ledger)) # type: ignore[arg-type] + handler = make_public_handler(FakeCoordinator(ledger=ledger)) # type: ignore[arg-type] server = ThreadingHTTPServer(("127.0.0.1", 0), handler) thread = threading.Thread(target=server.serve_forever, daemon=True) thread.start() @@ -2039,6 +2946,496 @@ def recipient_payout_history(self, *, recipient_id: str, limit: int = 50) -> lis self.assertEqual(ledger.request, ("miner-a", 1_000)) +class PublicHandlerTestCase(unittest.TestCase): + """One extracted-handler server per ledger, torn down with the test.""" + + def serve(self, ledger: FakePublicLedger) -> str: + handler = make_public_handler(FakeCoordinator(ledger=ledger)) # type: ignore[arg-type] + server = ThreadingHTTPServer(("127.0.0.1", 0), handler) + thread = threading.Thread(target=server.serve_forever, daemon=True) + thread.start() + + def stop() -> None: + server.shutdown() + server.server_close() + thread.join(timeout=5) + + self.addCleanup(stop) + return f"http://127.0.0.1:{server.server_port}" + + def get_error(self, url: str) -> tuple[int, dict[str, object], dict[str, str]]: + with self.assertRaises(urllib.error.HTTPError) as raised: + urllib.request.urlopen(url, timeout=5) + with raised.exception as error: + return error.code, json.loads(error.read()), dict(error.headers.items()) + + +# Every range/bucket pair outside HASHRATE_SERIES_ALLOWED_BUCKETS, and every +# concrete pair inside it. Derived by hand rather than from the constant so a +# vocabulary regression cannot rewrite the test that guards it. +FORBIDDEN_BUCKET_COMBOS = ( + ("1m", "5m"), + ("6m", "5m"), + ("6m", "1h"), + ("all", "5m"), + ("all", "1h"), +) +ALLOWED_BUCKET_COMBOS = ( + ("1w", "5m"), + ("1w", "1h"), + ("1w", "1d"), + ("1m", "1h"), + ("1m", "1d"), + ("6m", "1d"), + ("all", "1d"), +) + + +class HashrateBucketClampTests(PublicHandlerTestCase): + """The static bucket-per-range vocabulary closes the unbounded-cost combos.""" + + def test_forbidden_combinations_are_rejected_before_any_ledger_call(self) -> None: + ledger = FakePublicLedger() + base = self.serve(ledger) + for range_id, bucket in FORBIDDEN_BUCKET_COMBOS: + with self.subTest(range=range_id, bucket=bucket): + status, payload, headers = self.get_error( + f"{base}/public/v1/hashrate-series?range={range_id}&bucket={bucket}" + ) + self.assertEqual(400, status) + self.assertEqual("prism.dashboard.error.v1", payload["schema"]) + self.assertEqual("bad_request", payload["error"]["code"]) + self.assertIn( + f"bucket {bucket} is not allowed for range {range_id}", + payload["error"]["message"], + ) + self.assertEqual("no-store", headers.get("Cache-Control")) + self.assertEqual(0, ledger.hashrate_series_calls) + + def test_the_refusal_names_the_allowed_buckets(self) -> None: + base = self.serve(FakePublicLedger()) + _status, payload, _headers = self.get_error( + f"{base}/public/v1/hashrate-series?range=1m&bucket=5m" + ) + self.assertIn("allowed: 1h, 1d", payload["error"]["message"]) + + def test_allowed_combinations_still_serve(self) -> None: + ledger = FakePublicLedger() + base = self.serve(ledger) + combos = ALLOWED_BUCKET_COMBOS + tuple( + (range_id, "auto") for range_id in public_api.HASHRATE_SERIES_RANGE_SECONDS + ) + for range_id, bucket in combos: + with self.subTest(range=range_id, bucket=bucket): + url = f"{base}/public/v1/hashrate-series?range={range_id}&bucket={bucket}" + with urllib.request.urlopen(url, timeout=5) as response: + payload = json.loads(response.read()) + self.assertEqual("prism.dashboard.hashrate-series.v1", payload["schema"]) + self.assertEqual(len(combos), ledger.hashrate_series_calls) + + def test_every_auto_resolution_is_inside_the_allowed_vocabulary(self) -> None: + for range_id in public_api.HASHRATE_SERIES_RANGE_SECONDS: + resolved = public_api.auto_bucket(range_id) + self.assertIn(resolved, public_api.HASHRATE_SERIES_ALLOWED_BUCKETS[range_id]) + # And the helper itself agrees: a resolved auto bucket never raises. + public_api.allowed_hashrate_bucket(range_id, resolved) + + +class PublicReadStatementTimeoutTests(PublicHandlerTestCase): + """The per-request database deadline and its read_timeout refusal.""" + + def test_a_ledger_deadline_becomes_a_503_read_timeout_and_is_never_cached(self) -> None: + ledger = TimingOutPublicLedger() + base = self.serve(ledger) + # Patch only the deadline knob: clearing the whole environment also + # drops NO_PROXY, and on a proxy-configured machine urllib then routes + # this localhost request through the proxy instead of the test server. + with patch.dict(os.environ): + os.environ.pop("PRISM_PUBLIC_READ_STATEMENT_TIMEOUT_SECONDS", None) + for expected_origin_calls in (1, 2): + status, payload, headers = self.get_error( + f"{base}/public/v1/hashrate-series" + ) + self.assertEqual(503, status) + self.assertEqual("prism.dashboard.error.v1", payload["schema"]) + self.assertEqual("read_timeout", payload["error"]["code"]) + self.assertEqual("no-store", headers.get("Cache-Control")) + self.assertIsNone(headers.get("CDN-Cache-Control")) + # The second request re-ran the origin: a timeout stored no + # cache entry, so one expensive miss cannot poison the cache + # with a refusal. + self.assertEqual(expected_origin_calls, ledger.hashrate_series_calls) + self.assertEqual([20.0, 20.0], ledger.statement_timeout_budgets) + + def test_the_scope_is_entered_with_the_configured_budget(self) -> None: + ledger = StatementTimeoutRecordingLedger() + base = self.serve(ledger) + with patch.dict( + os.environ, + {"PRISM_PUBLIC_READ_STATEMENT_TIMEOUT_SECONDS": "7"}, + ): + with urllib.request.urlopen( + f"{base}/public/v1/hashrate-series", timeout=5 + ) as response: + json.loads(response.read()) + self.assertEqual([7.0], ledger.statement_timeout_budgets) + self.assertEqual(1, ledger.hashrate_series_calls) + + def test_env_zero_disables_the_scope_entirely(self) -> None: + ledger = StatementTimeoutRecordingLedger() + base = self.serve(ledger) + with patch.dict( + os.environ, + {"PRISM_PUBLIC_READ_STATEMENT_TIMEOUT_SECONDS": "0"}, + ): + with urllib.request.urlopen( + f"{base}/public/v1/hashrate-series", timeout=5 + ) as response: + json.loads(response.read()) + self.assertEqual([], ledger.statement_timeout_budgets) + self.assertEqual(1, ledger.hashrate_series_calls) + + +class PublicResponseCacheStaleWhileRevalidateTests(unittest.TestCase): + """In-process stale-while-revalidate on PublicResponseCache. + + Entry expiry is keyed on time.monotonic(), which is not injectable, so + these tests age entries by editing the stored bounds directly -- the + precedent tests/test_prism_public_read_service.py sets for the outage + contract -- rather than sleeping out real TTLs. + """ + + KEY = ("/swr", ()) + TTL = 60 + + def prime(self, cache: public_api.PublicResponseCache, payload: object) -> None: + status, served, state, _age = cache.get_or_compute( + key=self.KEY, ttl_seconds=self.TTL, compute=lambda: (200, payload) + ) + self.assertEqual((200, payload, "MISS"), (status, served, state)) + + def age_entry( + self, + cache: public_api.PublicResponseCache, + *, + age_seconds: float, + key: tuple[str, tuple[tuple[str, tuple[str, ...]], ...]] | None = None, + stale_while_revalidate_seconds: int = 30, + ) -> None: + entry = cache._entries[key if key is not None else self.KEY] + entry.stored_at = time.monotonic() - age_seconds + entry.expires_at = entry.stored_at + self.TTL + entry.stale_until = entry.expires_at + stale_while_revalidate_seconds + + def lookup( + self, + cache: public_api.PublicResponseCache, + compute, + ) -> tuple[int, object, str, int]: + return cache.get_or_compute( + key=self.KEY, + ttl_seconds=self.TTL, + stale_while_revalidate_seconds=30, + compute=compute, + ) + + def wait_for_refresh_slot_to_clear(self, cache: public_api.PublicResponseCache) -> None: + # Bounded: the refresh thread frees the inflight slot in its finally + # clause, so a few milliseconds suffice and 5s is a hard failure. + deadline = time.monotonic() + 5 + while self.KEY in cache._inflight and time.monotonic() < deadline: + time.sleep(0.01) + self.assertNotIn(self.KEY, cache._inflight) + + def test_capacity_reaping_spares_stale_servable_entries(self) -> None: + """Reaping removes entries past their whole window, not merely expired. + + A full cache receiving a new key must shed dead weight first: an entry + past TTL *and* its recorded revalidate window is gone, but one still + inside the window stays exactly as servable as the stale-serve path + promises -- reaping it would turn its next visitor's instant STALE + answer into a blocking recompute. + """ + cache = public_api.PublicResponseCache() + keep_key = ("/swr-keep", ()) + dead_key = ("/swr-dead", ()) + for key in (keep_key, dead_key): + cache.get_or_compute( + key=key, + ttl_seconds=self.TTL, + stale_while_revalidate_seconds=30, + compute=lambda: (200, {"k": key[0]}), + ) + # keep: expired 5s ago, inside its 30s window. dead: past the window. + self.age_entry(cache, age_seconds=65, key=keep_key) + self.age_entry(cache, age_seconds=100, key=dead_key) + with patch.dict(os.environ, {"PRISM_PUBLIC_CACHE_MAX_ENTRIES": "2"}): + cache.get_or_compute( + key=("/swr-new", ()), + ttl_seconds=self.TTL, + stale_while_revalidate_seconds=30, + compute=lambda: (200, {"k": "new"}), + ) + self.assertNotIn(dead_key, cache._entries) + self.assertIn(keep_key, cache._entries) + status, payload, state, age = cache.get_or_compute( + key=keep_key, + ttl_seconds=self.TTL, + stale_while_revalidate_seconds=30, + compute=lambda: (200, {"k": "refreshed"}), + ) + self.assertEqual((200, {"k": "/swr-keep"}, "STALE", 65), (status, payload, state, age)) + deadline = time.monotonic() + 5 + while keep_key in cache._inflight and time.monotonic() < deadline: + time.sleep(0.01) + self.assertNotIn(keep_key, cache._inflight) + + def test_a_failed_refresh_thread_start_frees_the_slot(self) -> None: + """Thread creation failure must not strand the registered slot. + + The refresh slot is registered before Thread.start() runs, and under + resource pressure start() can raise. Without cleanup no thread ever + frees the slot: later stale hits skip their refresh forever and, + past the window, blocking requests coalesce onto an event nobody can + set. The stale answer is still served; the slot is freed and the + event set so the next request proceeds normally. + """ + cache = public_api.PublicResponseCache() + self.prime(cache, {"v": "stale"}) + self.age_entry(cache, age_seconds=65) + + with patch.object( + threading.Thread, "start", side_effect=RuntimeError("can't start new thread") + ): + status, payload, state, age = self.lookup( + cache, lambda: (200, {"v": "unused"}) + ) + + self.assertEqual((200, {"v": "stale"}, "STALE", 65), (status, payload, state, age)) + self.assertNotIn(self.KEY, cache._inflight) + # The next stale hit retries the refresh with a working thread... + refreshed = threading.Event() + + def refresh() -> tuple[int, object]: + refreshed.set() + return 200, {"v": "fresh"} + + status, payload, state, _age = self.lookup(cache, refresh) + self.assertEqual((200, {"v": "stale"}, "STALE"), (status, payload, state)) + self.assertTrue(refreshed.wait(timeout=5)) + self.wait_for_refresh_slot_to_clear(cache) + status, payload, state, _age = self.lookup(cache, lambda: (200, {"v": "wrong"})) + self.assertEqual((200, {"v": "fresh"}, "HIT"), (status, payload, state)) + + def test_a_fresh_entry_is_a_hit_with_the_window_armed(self) -> None: + cache = public_api.PublicResponseCache() + self.prime(cache, {"v": "fresh"}) + + status, payload, state, age = self.lookup(cache, lambda: (200, {"v": "wrong"})) + + self.assertEqual((200, {"v": "fresh"}, "HIT", 0), (status, payload, state, age)) + + def test_a_stale_entry_is_served_while_one_background_refresh_recomputes(self) -> None: + cache = public_api.PublicResponseCache() + self.prime(cache, {"v": "stale"}) + self.age_entry(cache, age_seconds=65) # expired 5s ago, inside the 30s window + refreshed = threading.Event() + + def refresh() -> tuple[int, object]: + refreshed.set() + return 200, {"v": "fresh"} + + status, payload, state, age = self.lookup(cache, refresh) + + # Served immediately, from the expired entry, with its honest age. + self.assertEqual((200, {"v": "stale"}, "STALE", 65), (status, payload, state, age)) + self.assertTrue(refreshed.wait(timeout=5)) + self.wait_for_refresh_slot_to_clear(cache) + status, payload, state, age = self.lookup(cache, lambda: (200, {"v": "wrong"})) + self.assertEqual((200, {"v": "fresh"}, "HIT", 0), (status, payload, state, age)) + + def test_a_failed_refresh_keeps_the_stale_entry_and_frees_the_slot(self) -> None: + cache = public_api.PublicResponseCache() + self.prime(cache, {"v": "stale"}) + self.age_entry(cache, age_seconds=61) + failed = threading.Event() + + def failing_refresh() -> tuple[int, object]: + failed.set() + raise RuntimeError("origin down") + + status, payload, state, _age = self.lookup(cache, failing_refresh) + + self.assertEqual((200, {"v": "stale"}, "STALE"), (status, payload, state)) + self.assertTrue(failed.wait(timeout=5)) + self.wait_for_refresh_slot_to_clear(cache) + # Still servable -- the failure deleted nothing -- and the freed slot + # lets this later stale hit start a second refresh, which succeeds. + recovered = threading.Event() + + def recovering_refresh() -> tuple[int, object]: + recovered.set() + return 200, {"v": "fresh"} + + status, payload, state, _age = self.lookup(cache, recovering_refresh) + self.assertEqual((200, {"v": "stale"}, "STALE"), (status, payload, state)) + self.assertTrue(recovered.wait(timeout=5)) + self.wait_for_refresh_slot_to_clear(cache) + _status, payload, state, _age = self.lookup(cache, lambda: (200, {"v": "wrong"})) + self.assertEqual(({"v": "fresh"}, "HIT"), (payload, state)) + + def test_beyond_the_window_the_next_request_blocks_and_recomputes(self) -> None: + cache = public_api.PublicResponseCache() + self.prime(cache, {"v": "stale"}) + self.age_entry(cache, age_seconds=95) # expired 35s ago, past the 30s window + + status, payload, state, age = self.lookup(cache, lambda: (200, {"v": "fresh"})) + + self.assertEqual((200, {"v": "fresh"}, "MISS", 0), (status, payload, state, age)) + + def test_concurrent_stale_hits_share_exactly_one_refresh(self) -> None: + cache = public_api.PublicResponseCache() + self.prime(cache, {"v": "stale"}) + self.age_entry(cache, age_seconds=61) + release = threading.Event() + compute_calls: list[int] = [] + compute_lock = threading.Lock() + + def slow_refresh() -> tuple[int, object]: + with compute_lock: + compute_calls.append(1) + self.assertTrue(release.wait(timeout=5)) + return 200, {"v": "fresh"} + + results: list[tuple[int, object, str, int]] = [] + results_lock = threading.Lock() + + def stale_hit() -> None: + result = self.lookup(cache, slow_refresh) + with results_lock: + results.append(result) + + workers = [threading.Thread(target=stale_hit) for _ in range(5)] + for worker in workers: + worker.start() + for worker in workers: + worker.join(timeout=5) + + # Every requester was answered immediately from the stale entry; none + # blocked behind the (still running) refresh, and none started one of + # its own. + self.assertEqual(5, len(results)) + for status, payload, state, _age in results: + self.assertEqual((200, {"v": "stale"}, "STALE"), (status, payload, state)) + release.set() + self.wait_for_refresh_slot_to_clear(cache) + self.assertEqual([1], compute_calls) + _status, payload, state, _age = self.lookup(cache, lambda: (200, {"v": "wrong"})) + self.assertEqual(({"v": "fresh"}, "HIT"), (payload, state)) + + +class PublicArtifactByteExactnessTests(unittest.TestCase): + """GET /public/v1/artifacts/{sha256} must return the exact bytes the + advertised sha256 was computed over, or no external verifier can confirm + any artifact. This was the missing regression coverage: the hash side + serializes with canonical_json_text (compact separators) while the + generic write_json path re-serializes with default separators plus a + trailing newline.""" + + def serve(self, ledger: FakePublicLedger) -> str: + handler = make_public_handler(FakeCoordinator(ledger=ledger)) # type: ignore[arg-type] + server = ThreadingHTTPServer(("127.0.0.1", 0), handler) + thread = threading.Thread(target=server.serve_forever, daemon=True) + thread.start() + + def stop() -> None: + server.shutdown() + server.server_close() + thread.join(timeout=5) + + self.addCleanup(stop) + return f"http://127.0.0.1:{server.server_port}" + + def fetch_artifact(self, base_url: str, sha256: str) -> tuple[bytes, dict[str, str]]: + with urllib.request.urlopen(f"{base_url}/public/v1/artifacts/{sha256}", timeout=5) as response: + return response.read(), dict(response.headers) + + def test_manifest_artifact_bytes_hash_to_advertised_sha256(self) -> None: + ledger = CanonicalArtifactPublicLedger() + base_url = self.serve(ledger) + for sha256, document in ( + (ledger.manifest_sha256, ledger.manifest), + (ledger.manifest_set_sha256, ledger.manifest_set), + ): + with self.subTest(sha256=sha256): + body, headers = self.fetch_artifact(base_url, sha256) + + self.assertEqual(hashlib.sha256(body).hexdigest(), sha256) + self.assertEqual(json.loads(body), document) + self.assertEqual(headers.get("Content-Type"), "application/json") + self.assertEqual(int(headers["Content-Length"]), len(body)) + + def test_artifact_bytes_stay_exact_across_cache_hits(self) -> None: + ledger = CanonicalArtifactPublicLedger() + base_url = self.serve(ledger) + + first, first_headers = self.fetch_artifact(base_url, ledger.manifest_set_sha256) + second, second_headers = self.fetch_artifact(base_url, ledger.manifest_set_sha256) + + self.assertEqual(first, second) + self.assertEqual(hashlib.sha256(second).hexdigest(), ledger.manifest_set_sha256) + for headers in (first_headers, second_headers): + self.assertIn("immutable", headers.get("CDN-Cache-Control", "")) + + def test_uppercase_artifact_path_serves_the_same_exact_bytes(self) -> None: + ledger = CanonicalArtifactPublicLedger() + base_url = self.serve(ledger) + + body, _headers = self.fetch_artifact(base_url, ledger.manifest_sha256.upper()) + + self.assertEqual(hashlib.sha256(body).hexdigest(), ledger.manifest_sha256) + + def test_tampered_canonical_text_falls_back_to_reserialized_document(self) -> None: + # Stored canonical text that no longer hashes to its content address + # must never be served as the exact bytes; the endpoint keeps the + # legacy re-serialized response instead. + ledger = TamperedCanonicalArtifactPublicLedger() + base_url = self.serve(ledger) + + body, _headers = self.fetch_artifact(base_url, ledger.manifest_sha256) + + self.assertEqual(body, json.dumps(ledger.manifest, sort_keys=True).encode() + b"\n") + self.assertEqual(json.loads(body), ledger.manifest) + + def test_audit_bundle_artifact_bytes_hash_to_advertised_sha256(self) -> None: + ledger = CanonicalArtifactPublicLedger() + base_url = self.serve(ledger) + + body, _headers = self.fetch_artifact(base_url, ledger.audit_bundle_sha256) + + self.assertEqual(hashlib.sha256(body).hexdigest(), ledger.audit_bundle_sha256) + + def test_missing_canonical_audit_bundle_fallback_is_never_cached(self) -> None: + ledger = MissingCanonicalArtifactPublicLedger() + base_url = self.serve(ledger) + + first, first_headers = self.fetch_artifact(base_url, ledger.audit_bundle_sha256) + second, second_headers = self.fetch_artifact(base_url, ledger.audit_bundle_sha256) + + self.assertEqual(json.loads(first), ledger.audit_bundle) + self.assertEqual(second, first) + self.assertEqual(ledger.audit_document_calls, 2) + for headers in (first_headers, second_headers): + self.assertEqual(headers.get("Cache-Control"), "no-store") + self.assertEqual( + headers.get("X-Prism-Artifact-Canonical-State"), + "missing", + ) + self.assertNotIn("CDN-Cache-Control", headers) + self.assertNotIn("Vercel-CDN-Cache-Control", headers) + self.assertNotIn("Age", headers) + + class PrismPublicDashboardMemoryLedgerTests(unittest.TestCase): def test_memory_ledger_public_read_models_are_empty_safe(self) -> None: coordinator = MemoryCoordinator() @@ -2058,7 +3455,7 @@ def test_memory_ledger_public_read_models_are_empty_safe(self) -> None: ntime=1, ) ) - handler = make_audit_handler(coordinator) # type: ignore[arg-type] + handler = make_public_handler(coordinator) # type: ignore[arg-type] server = ThreadingHTTPServer(("127.0.0.1", 0), handler) thread = threading.Thread(target=server.serve_forever, daemon=True) thread.start() @@ -2077,6 +3474,81 @@ def test_memory_ledger_public_read_models_are_empty_safe(self) -> None: self.assertEqual(payload["pagination"]["total_count"], 1) self.assertEqual(payload["rows"][0]["recipient_id"], "miner-a") + def test_memory_ledger_supports_blocks_chain_state_filter(self) -> None: + ledger = SingleWriterShareLedger() + + for chain_state in ("active", "all", "reversed"): + with self.subTest(chain_state=chain_state): + payload = ledger.dashboard_blocks(page=1, limit=15, chain_state=chain_state) + self.assertEqual(payload["rows"], []) + self.assertEqual(payload["pagination"]["total_count"], 0) + with self.assertRaises(ValueError): + ledger.dashboard_blocks(page=1, limit=15, chain_state="orphaned") + + def test_memory_ledger_pool_snapshot_reports_zero_reorg_counts(self) -> None: + snapshot = SingleWriterShareLedger().dashboard_pool_snapshot( + current_network_difficulty="1", + generated_at=public_api.utc_now_iso(), + ) + + self.assertEqual(snapshot["blocks_reversed_total"], 0) + self.assertEqual(snapshot["blocks_inactive_total"], 0) + + def test_memory_ledger_block_markers_are_empty_safe(self) -> None: + # The in-memory backend records no found_at for its pool blocks, so + # the marker series degrades to an empty response rather than 500ing, + # the way its other dashboard read models degrade. + coordinator = MemoryCoordinator() + handler = make_public_handler(coordinator) # type: ignore[arg-type] + server = ThreadingHTTPServer(("127.0.0.1", 0), handler) + thread = threading.Thread(target=server.serve_forever, daemon=True) + thread.start() + try: + with urllib.request.urlopen( + f"http://127.0.0.1:{server.server_port}/public/v1/block-markers?range=1m&bucket=1h", + timeout=5, + ) as response: + payload = json.loads(response.read()) + finally: + server.shutdown() + server.server_close() + thread.join(timeout=5) + + self.assertEqual(payload["schema"], "prism.dashboard.block-markers.v1") + self.assertEqual(payload["total_blocks"], 0) + self.assertEqual(payload["points"], []) + + def test_memory_ledger_artifact_responses_verify_end_to_end(self) -> None: + # Full path: the real record path persists the canonical text, and the + # HTTP response returns those exact bytes for every manifest kind. + coordinator = MemoryCoordinator() + manifest_set = sample_ctv_manifest_set() + manifest = manifest_set["manifests"][0] # type: ignore[index] + manifest_set_sha256 = sha256_json_hex(manifest_set) + manifest_sha256 = sha256_json_hex(manifest) + coordinator.ledger.persist_ctv_fanout_manifest_set( + block_hash="aa" * 32, + manifest_set=manifest_set, + manifest_set_sha256=manifest_set_sha256, + ) + handler = make_public_handler(coordinator) # type: ignore[arg-type] + server = ThreadingHTTPServer(("127.0.0.1", 0), handler) + thread = threading.Thread(target=server.serve_forever, daemon=True) + thread.start() + try: + for sha256 in (manifest_set_sha256, manifest_sha256): + with self.subTest(sha256=sha256): + with urllib.request.urlopen( + f"http://127.0.0.1:{server.server_port}/public/v1/artifacts/{sha256}", + timeout=5, + ) as response: + body = response.read() + self.assertEqual(hashlib.sha256(body).hexdigest(), sha256) + finally: + server.shutdown() + server.server_close() + thread.join(timeout=5) + if __name__ == "__main__": unittest.main() diff --git a/tests/test_prism_public_http_limits.py b/tests/test_prism_public_http_limits.py new file mode 100644 index 000000000..f3eb02b64 --- /dev/null +++ b/tests/test_prism_public_http_limits.py @@ -0,0 +1,151 @@ +"""Socket-level admission and request-header deadlines for the public API.""" + +import gc +import http.client +import socket +import threading +import time +import unittest +import weakref +from unittest.mock import patch + +from lab.prism import public_read_service as public + + +class PublicHttpLimitsTests(unittest.TestCase): + def setUp(self): + self.admitted = threading.Event() + handler = public.make_handler(public.PublicReadService(object())) + admitted = self.admitted + self.handler_refs = [] + handler_refs = self.handler_refs + + class ObservedHandler(handler): + def setup(self): + super().setup() + handler_refs.append(weakref.ref(self)) + admitted.set() + + self.server = public.BoundedPublicHTTPServer( + ("127.0.0.1", 0), ObservedHandler, max_connections=1, + timeout_seconds=0.3, + ) + self.thread = threading.Thread( + target=lambda: self.server.serve_forever(poll_interval=0.01), daemon=True, + ) + self.thread.start() + self.addCleanup(self.stop_server) + + def stop_server(self): + self.server.shutdown() + self.server.server_close() + self.thread.join(2) + + def connect(self): + client = socket.create_connection(self.server.server_address, timeout=2) + self.addCleanup(client.close) + return client + + def assert_closed(self, client): + try: + self.assertEqual(client.recv(4096), b"") + except ConnectionResetError: + pass + + def wait_for_capacity(self): + # Wait on the actual admission boundary; no arbitrary recovery sleep. + self.assertTrue(self.server._connection_slots.acquire(timeout=2)) + self.server._connection_slots.release() + + def test_excess_connections_close_and_idle_worker_retires(self): + idle = self.connect() + self.assertTrue(self.admitted.wait(2)) + self.assert_closed(self.connect()) + self.assert_closed(idle) + self.wait_for_capacity() + client = http.client.HTTPConnection(*self.server.server_address, timeout=2) + self.addCleanup(client.close) + client.request("GET", "/missing") + response = client.getresponse() + self.assertEqual(response.status, 404) + self.assertEqual(response.getheader("Connection"), "close") + response.read() + + def test_slow_headers_cannot_extend_total_deadline(self): + client = self.connect() + self.assertTrue(self.admitted.wait(2)) + dispatched = threading.Event() + handler = self.server.RequestHandlerClass + original = handler.do_GET + + def observe_dispatch(request): + dispatched.set() + original(request) + + patcher = patch.object(handler, "do_GET", observe_dispatch) + patcher.start() + self.addCleanup(patcher.stop) + client.sendall(b"GET /missing HTTP/1.1\r\nX-Slow: ") + stopped = threading.Event() + + def drip(): + while not stopped.wait(0.03): + try: + client.sendall(b"x") + except OSError: + return + + writer = threading.Thread(target=drip, daemon=True) + writer.start() + started = time.monotonic() + try: + self.assert_closed(client) + self.assertLess(time.monotonic() - started, 1.5) + finally: + stopped.set() + writer.join(2) + self.wait_for_capacity() + self.assertFalse(dispatched.is_set(), "expired headers reached application dispatch") + + def test_keep_alive_request_cannot_retain_worker(self): + client = self.connect() + client.sendall(b"GET /missing HTTP/1.1\r\nHost: localhost\r\nConnection: keep-alive\r\n\r\n") + body = b"" + while chunk := client.recv(4096): + body += chunk + self.assertIn(b"404", body) + self.assertIn(b"Connection: close", body) + self.wait_for_capacity() + + def test_completed_handler_retires_without_cyclic_gc(self): + was_enabled = gc.isenabled() + gc.disable() + try: + client = self.connect() + client.sendall(b"GET /missing HTTP/1.1\r\nHost: localhost\r\n\r\n") + while client.recv(4096): + pass + self.wait_for_capacity() + self.assertEqual(len(self.handler_refs), 1) + self.assertIsNone(self.handler_refs[0]()) + finally: + if was_enabled: + gc.enable() + + def test_worker_start_failure_returns_admission_slot(self): + request, peer = socket.socketpair() + self.addCleanup(request.close) + self.addCleanup(peer.close) + with patch.object(public.ThreadingHTTPServer, "process_request", side_effect=RuntimeError): + with self.assertRaises(RuntimeError): + self.server.process_request(request, ("127.0.0.1", 1)) + self.wait_for_capacity() + + def test_invalid_limits_refuse_before_binding(self): + for kwargs in ( + {"max_connections": 0}, {"max_connections": 1025}, + {"timeout_seconds": 0}, {"timeout_seconds": float("nan")}, + {"timeout_seconds": float("inf")}, + ): + with self.subTest(kwargs=kwargs), self.assertRaises(public.PublicReadConfigurationError): + public.BoundedPublicHTTPServer(self.server.server_address, object, **kwargs) diff --git a/tests/test_prism_public_read_service.py b/tests/test_prism_public_read_service.py new file mode 100644 index 000000000..501432b96 --- /dev/null +++ b/tests/test_prism_public_read_service.py @@ -0,0 +1,2571 @@ +#!/usr/bin/env python3 +"""Pin the extracted PRISM public read tier against the surface it replaced. + +The extraction step of issue #145 moved exactly the 13 original ``/public/v1`` +routes out of the coordinator's audit/ops listener into +``lab/prism/public_read_service.py``; ``/public/v1/block-markers`` joined the +extracted surface later, bringing it to 14 routes. The issue's read-replica back-end is a +separate follow-up -- this service still reads the primary Postgres, through +bounded read slots rather than the writer path. An extraction is +only worth doing if it changes nothing a client can observe and everything the +coordinator can feel, so these tests assert both halves: + +* **Contract equality** -- every extracted route's status, body and cache + headers still come from ``public_api`` unchanged. ``ContractEqualityTests`` + drives the new handler and ``public_api.dispatch`` against the *same* + coordinator and compares. The concrete pre-extraction values are pinned + separately and unchanged by ``tests/test_prism_public_dashboard_api.py``, + whose 70 assertions were written against the coordinator's listener and now + run against this handler. +* **The point of the move** -- ``NoWriterLockReadTests`` drives all 14 routes + against a ledger that raises if any writer-lock read is reached. Moving the + GIL contention while leaving the lock contention would have been motion + without progress. +* **The staleness contract** -- budgets published on every response, 503 rather + than a silently stale body, and no staleness refusal for immutable + content-addressed artifacts. +* **Fail-closed startup** and **coordinator subtraction**. +""" + +from __future__ import annotations + +import json +import os +import threading +import time +from contextlib import contextmanager +from threading import BoundedSemaphore, Lock +import unittest +import urllib.error +import urllib.parse +import urllib.request +from pathlib import Path +from typing import Iterator +from unittest.mock import patch + +from lab.prism import endpoint_registry, public_api, public_read_service +from lab.prism.audit_http import AuditHttpConfig, AuditHttpFacade +from lab.prism.share_ledger import PsqlShareLedger, ReadOnlyLedgerError + +# The public fixtures these routes are exercised against already exist, written +# against the coordinator's listener before the extraction. Reusing them rather +# than restating them is deliberate: a second copy could drift, and then the +# contract test would be comparing the new service to a fixture that no longer +# describes the old behaviour. +from tests.test_prism_public_dashboard_api import ( + DirectCoinbasePublicLedger, + FakeCoordinator, + FakePublicLedger, +) + + +ROOT = Path(__file__).resolve().parents[1] +AUDIT_HTTP_PATH = ROOT / "lab" / "prism" / "audit_http.py" + +FROZEN_NOW = "2026-01-01T00:00:00Z" + +RECIPIENT_ID = "miner-a" +BLOCK_HASH = FakePublicLedger.block_hash +FANOUT_TXID = FakePublicLedger.fanout_txid +ARTIFACT_SHA256 = FakePublicLedger.audit_bundle_sha256 + +# One concrete request per extracted route. EXTRACTED_ROUTE_COUNT is asserted +# against the registry so a new public route cannot be added without landing +# here too. +EXTRACTED_ROUTES: tuple[tuple[str, str], ...] = ( + ("/public/v1/pool-summary", ""), + ("/public/v1/blocks", ""), + ("/public/v1/leaderboard", ""), + ("/public/v1/hashrate-series", ""), + ("/public/v1/block-markers", ""), + ("/public/v1/mining-configuration", ""), + (f"/public/v1/miners/{RECIPIENT_ID}", ""), + (f"/public/v1/miners/{RECIPIENT_ID}/earnings", ""), + (f"/public/v1/miners/{RECIPIENT_ID}/payouts", ""), + (f"/public/v1/miners/{RECIPIENT_ID}/workers", ""), + (f"/public/v1/blocks/{BLOCK_HASH}/settlement-artifacts", ""), + ("/public/v1/fanouts/pending", ""), + (f"/public/v1/fanouts/{FANOUT_TXID}", ""), + (f"/public/v1/artifacts/{ARTIFACT_SHA256}", ""), +) + +EXTRACTED_ROUTE_COUNT = 14 + +CACHE_HEADERS = ("Cache-Control", "CDN-Cache-Control", "Vercel-CDN-Cache-Control", "Age") + +WRITER_LOCK_LEDGER_METHODS = ( + "current_owed_balances", + "recipient_payout_history", + "audit_bundle", + "audit_bundle_by_commitment", + "audit_block_payouts", + "audit_share_window", + "carry_forward_integrity_report", + # public_api.worker_rows falls back to this full-table read when + # dashboard_miner_worker_rows is absent; on Postgres it is a writer-lock + # read like the rest, so the public tier must never reach it either. + "all_shares", +) + + +class WriterLockReached(AssertionError): + """Raised when a public route reaches a read that fences on the writer.""" + + +class ServedResponse: + """One HTTP response, reduced to what the contract actually covers.""" + + def __init__(self, status: int, body: bytes, headers: dict[str, str]) -> None: + self.status = status + self.body = body + self.headers = headers + + @property + def payload(self) -> object: + return json.loads(self.body) + + def cache_headers(self) -> dict[str, str | None]: + return {name: self.headers.get(name) for name in CACHE_HEADERS} + + +class ServiceHarness: + """A bound public read service over one coordinator, for the test's lifetime.""" + + def __init__( + self, + coordinator: object, + *, + response_cache: object | None = None, + readiness: object | None = None, + metrics: object | None = None, + replica: object | None = None, + ) -> None: + self.service = public_read_service.PublicReadService( + coordinator, + response_cache=response_cache, # type: ignore[arg-type] + metrics=metrics, # type: ignore[arg-type] + readiness=readiness, # type: ignore[arg-type] + replica=replica, # type: ignore[arg-type] + ) + handler = public_read_service.make_handler(self.service) + self.server = public_read_service.BoundedPublicHTTPServer(("127.0.0.1", 0), handler) + self.thread = threading.Thread(target=self.server.serve_forever, daemon=True) + self.thread.start() + self.base_url = f"http://127.0.0.1:{self.server.server_port}" + # Loopback fixtures must never inherit a cached process-wide proxy. + self.opener = urllib.request.build_opener(urllib.request.ProxyHandler({})) + + def get(self, path_and_query: str) -> ServedResponse: + url = self.base_url + path_and_query + try: + with self.opener.open(url, timeout=5) as response: + return ServedResponse( + response.status, + response.read(), + dict(response.headers.items()), + ) + except urllib.error.HTTPError as error: + with error: + return ServedResponse( + error.code, + error.read(), + dict(error.headers.items()), + ) + + def close(self) -> None: + self.server.shutdown() + self.server.server_close() + self.thread.join(timeout=5) + + +def request_path(route: tuple[str, str]) -> str: + path, query = route + return f"{path}?{query}" if query else path + + +class RegistryAgreementTests(unittest.TestCase): + """The routes exercised here must be exactly the routes the registry extracts.""" + + def test_every_extracted_registry_path_is_covered(self) -> None: + covered = set() + for path, _query in EXTRACTED_ROUTES: + endpoint = endpoint_registry.endpoint_for_request_path(path) + self.assertIsNotNone(endpoint, f"{path} resolves to no registry endpoint") + assert endpoint is not None + covered.add(endpoint.primary_path) + self.assertEqual(set(endpoint_registry.extracted_paths()), covered) + + def test_the_extracted_surface_is_still_fourteen_routes(self) -> None: + self.assertEqual( + EXTRACTED_ROUTE_COUNT, + len(endpoint_registry.extracted_paths()), + ) + self.assertEqual(EXTRACTED_ROUTE_COUNT, len(EXTRACTED_ROUTES)) + + +class ContractEqualityTests(unittest.TestCase): + """Same status, same body, same cache headers as dispatch() produces. + + Time is frozen for both sides so ``generated_at`` cannot make two + structurally identical payloads compare unequal. + """ + + def setUp(self) -> None: + self.coordinator = FakeCoordinator(ledger=FullReadModelLedger()) + self.harness = ServiceHarness(self.coordinator) + self.addCleanup(self.harness.close) + patcher = patch.object(public_api, "utc_now_iso", return_value=FROZEN_NOW) + patcher.start() + self.addCleanup(patcher.stop) + + def expected_success(self, path: str, query: str) -> tuple[int, object, dict[str, str]]: + parsed_query = urllib.parse.parse_qs(query) + status, payload = public_api.dispatch(self.coordinator, path, parsed_query) + policy = public_api.public_cache_policy(path) + budget = public_read_service.staleness_budget_for_path(path) + if budget is not None: + ttl = min(policy.ttl_seconds, int(budget)) + policy = public_api.PublicCachePolicy( + ttl, max(0, min(policy.stale_while_revalidate_seconds, int(budget) - ttl)), + policy.immutable, + ) + # A fresh service cache serves the first request as a MISS at age 0, + # which is exactly what the coordinator's handle_public did. + headers = public_api.public_cache_headers( + policy, + cache_state="MISS", + age_seconds=0, + ) + return status, payload, headers + + def test_every_extracted_route_matches_dispatch(self) -> None: + for route in EXTRACTED_ROUTES: + path, query = route + with self.subTest(path=path): + served = self.harness.get(request_path(route)) + status, payload, headers = self.expected_success(path, query) + + self.assertEqual(status, served.status) + self.assertEqual(payload, served.payload) + # Byte-for-byte, not just structurally: the framing (sorted + # keys, trailing newline) is part of what clients receive. + self.assertEqual( + json.dumps(payload, sort_keys=True).encode() + b"\n", + served.body, + ) + for name in CACHE_HEADERS: + self.assertEqual( + headers.get(name), + served.headers.get(name), + f"{path}: {name} changed across the extraction", + ) + + def test_mining_configuration_highdiff_endpoint_survives_the_extraction(self) -> None: + # This process runs no Stratum listener, so the High-diff endpoint in + # the rendered body comes only from PRISM_STRATUM_HIGHDIFF_PORT in this + # process's own environment (compose passes it through; see + # tests/test_prism_compose_profile.py). With the env set the served + # body must still match dispatch and must carry the High-diff endpoint. + route = ("/public/v1/mining-configuration", "") + with patch.dict( + os.environ, + { + "PRISM_PUBLIC_STRATUM_URL": "stratum+tcp://public-pool.example:3335", + "PRISM_STRATUM_HIGHDIFF_PORT": "4334", + }, + clear=True, + ): + served = self.harness.get(request_path(route)) + status, payload, headers = self.expected_success(*route) + + self.assertEqual(status, served.status) + self.assertEqual(payload, served.payload) + for name in CACHE_HEADERS: + self.assertEqual(headers.get(name), served.headers.get(name)) + + endpoints = served.payload["configurations"][0]["stratum_endpoints"] + self.assertEqual( + ["Primary", "High-diff"], + [endpoint["label"] for endpoint in endpoints], + ) + self.assertEqual( + "stratum+tcp://public-pool.example:4334", + endpoints[1]["url"], + ) + self.assertEqual(4334, endpoints[1]["default_port"]) + + def test_harness_bypasses_a_cached_proxy_with_no_proxy_environment_cleared(self) -> None: + proxy_opener = urllib.request.build_opener( + urllib.request.ProxyHandler({"http": "http://127.0.0.1:9"}) + ) + with patch.object(urllib.request, "_opener", proxy_opener), patch.dict(os.environ, {}, clear=True): + self.assertEqual(self.harness.get("/public/v1/blocks").status, 200) + + def test_content_type_is_unchanged(self) -> None: + for route in EXTRACTED_ROUTES: + with self.subTest(path=route[0]): + served = self.harness.get(request_path(route)) + self.assertEqual("application/json", served.headers.get("Content-Type")) + + def test_not_found_error_matches_dispatch(self) -> None: + # A block with no settlement rows: dispatch raises PublicApiError(404). + unknown_block = "f" * 64 + path = f"/public/v1/blocks/{unknown_block}/settlement-artifacts" + with self.assertRaises(public_api.PublicApiError) as raised: + public_api.dispatch(self.coordinator, path, {}) + expected = raised.exception + + served = self.harness.get(path) + + self.assertEqual(404, expected.status) + self.assertEqual(expected.status, served.status) + self.assertEqual( + public_api.error_payload(expected.code, expected.message), + served.payload, + ) + self.assertEqual("no-store", served.headers.get("Cache-Control")) + # Errors are not cached, so they carry no shared-cache or Age headers. + self.assertIsNone(served.headers.get("CDN-Cache-Control")) + self.assertIsNone(served.headers.get("Age")) + + def test_public_api_error_matches_dispatch(self) -> None: + # An explicit PublicApiError raised by dispatch's own validation. + path, query = "/public/v1/leaderboard", "window=bogus" + with self.assertRaises(public_api.PublicApiError) as raised: + public_api.dispatch(self.coordinator, path, urllib.parse.parse_qs(query)) + expected = raised.exception + + served = self.harness.get(f"{path}?{query}") + + self.assertEqual(400, expected.status) + self.assertEqual("invalid_window", expected.code) + self.assertEqual(expected.status, served.status) + self.assertEqual( + public_api.error_payload(expected.code, expected.message), + served.payload, + ) + self.assertEqual("prism.dashboard.error.v1", served.payload["schema"]) + self.assertEqual("no-store", served.headers.get("Cache-Control")) + + def test_reward_filter_conflict_matches_dispatch(self) -> None: + path, query = "/public/v1/leaderboard", "window=reward&search=a&recipient_id=miner-a" + with self.assertRaises(public_api.PublicApiError) as raised: + public_api.dispatch(self.coordinator, path, urllib.parse.parse_qs(query)) + + served = self.harness.get(f"{path}?{query}") + + self.assertEqual(raised.exception.status, served.status) + self.assertEqual( + public_api.error_payload(raised.exception.code, raised.exception.message), + served.payload, + ) + + def test_unknown_public_route_is_a_public_schema_404(self) -> None: + served = self.harness.get("/public/v1/not-a-route") + + self.assertEqual(404, served.status) + self.assertEqual("prism.dashboard.error.v1", served.payload["schema"]) + self.assertEqual("not_found", served.payload["error"]["code"]) + + def test_a_second_request_is_served_from_cache_with_a_real_age(self) -> None: + # The response cache moved with the routes; a HIT must still report Age. + route = ("/public/v1/blocks", "") + first = self.harness.get(request_path(route)) + second = self.harness.get(request_path(route)) + + self.assertEqual(200, first.status) + self.assertEqual(first.body, second.body) + self.assertEqual("0", second.headers.get("Age")) + + +class FullReadModelLedger(FakePublicLedger): + """A ledger implementing every ``dashboard_*`` read model, as Postgres does. + + ``FakePublicLedger`` deliberately leaves some read models out so the + pre-extraction tests could exercise ``public_api``'s duck-typed fallbacks. + The extracted service never runs those fallbacks in production -- + ``PsqlShareLedger`` implements every read model -- and two of the fallbacks + derive hashrate from ``all_shares()`` relative to wall-clock now, which + makes them unusable for an equality comparison. This subclass fills the + gaps with fixed rows so a route's answer depends only on its inputs. + """ + + def dashboard_miner_share_summary(self, *, recipient_id: str) -> dict[str, object]: + return { + "hashrate_ths": {"h1": "1.5", "h3": "2.5", "h24": "3.5"}, + "accepted_3h": 3, + "accepted_difficulty_3h": "600000", + "last_share_at": "2026-06-26T20:44:53Z", + "share_percent": "60", + } + + def dashboard_miner_worker_rows( + self, + *, + recipient_id: str, + page: int, + limit: int, + search: str | None, + hide_inactive: bool, + ) -> dict[str, object]: + rows = [ + { + "worker_name": "rig-1", + "status": "active", + "last_share_at": "2026-06-26T20:44:53Z", + "hashrate_ths_60s": "1.5", + "hashrate_ths_3h": "2.5", + } + ] + if search: + rows = [row for row in rows if search.lower() in str(row["worker_name"]).lower()] + offset = (page - 1) * limit + return { + "pagination": public_api.pagination(page, limit, len(rows)), + "rows": rows[offset : offset + limit], + "active_count": len(rows), + } + + def dashboard_miner_owed_balance_bits(self, *, recipient_id: str) -> int: + return sum( + int(balance["balance_sats"]) + for balance in self.current_owed_balances() + if balance["recipient_id"] == recipient_id + ) + + def dashboard_direct_coinbase_settlement( + self, + *, + block_hash: str, + ) -> dict[str, object] | None: + return None + + # The Postgres implementations of these two are independent SQL. The base + # fixture builds them from recipient_payout_history(), which is itself a + # writer-lock read, so they are re-based here on the raw fixture rows. + def dashboard_miner_payout_rows( + self, + *, + recipient_id: str, + page: int, + limit: int, + ) -> dict[str, object]: + rows = [public_api.miner_payout_row(row) for row in self._payout_history()] + offset = (page - 1) * limit + return { + "pagination": public_api.pagination(page, limit, len(rows)), + "rows": rows[offset : offset + limit], + } + + def dashboard_miner_earning_rows( + self, + *, + recipient_id: str, + page: int, + limit: int, + ) -> dict[str, object]: + rows = [public_api.miner_earning_row(row) for row in self._payout_history()] + offset = (page - 1) * limit + return { + "pagination": public_api.pagination(page, limit, len(rows)), + "rows": rows[offset : offset + limit], + } + + +class NoWriterLockLedger(FullReadModelLedger): + """A public ledger whose writer-lock reads are all landmines. + + Every method in WRITER_LOCK_LEDGER_METHODS is one ``public_api`` would + reach only by falling back off a ``dashboard_*`` read model. On the real + Postgres ledger each of them runs under + ``_operation_gate(self._lock, "writer lock")``, so reaching one from a + public route serializes a dashboard poll against the block-landing writer. + The extraction's whole purpose is that none of them are reachable. + """ + + def __init__(self) -> None: + super().__init__() + self.settlement_calls = 0 + self.owed_balance_calls = 0 + + def dashboard_miner_owed_balance_bits(self, *, recipient_id: str) -> int: + self.owed_balance_calls += 1 + return 1234 + + def dashboard_direct_coinbase_settlement( + self, + *, + block_hash: str, + ) -> dict[str, object] | None: + self.settlement_calls += 1 + return None + + def dashboard_miner_share_summary(self, *, recipient_id: str) -> dict[str, object]: + # FakePublicLedger has no share-summary read model, so public_api would + # fall back to all_shares(). PsqlShareLedger does have one, and it is a + # read-slot query; supply it so the fallback is genuinely unreachable. + return { + "hashrate_ths": {"m1": "0", "m5": "0", "m10": "0", "h3": "2.5", "h24": "2.5"}, + "accepted_3h": 2, + "accepted_difficulty_3h": "180", + "last_share_at": "2026-06-26T20:44:53Z", + "share_percent": "60", + } + + def dashboard_miner_payout_rows( + self, + *, + recipient_id: str, + page: int, + limit: int, + ) -> dict[str, object]: + # The inherited version routes through recipient_payout_history, which + # is a landmine here. On PsqlShareLedger this read model is its own + # read-slot query and touches no fenced path. + rows = [public_api.miner_payout_row(row) for row in self._payout_history()] + offset = (page - 1) * limit + return { + "pagination": public_api.pagination(page, limit, len(rows)), + "rows": rows[offset : offset + limit], + } + + def dashboard_miner_earning_rows( + self, + *, + recipient_id: str, + page: int, + limit: int, + ) -> dict[str, object]: + rows = [public_api.miner_earning_row(row) for row in self._payout_history()] + offset = (page - 1) * limit + return { + "pagination": public_api.pagination(page, limit, len(rows)), + "rows": rows[offset : offset + limit], + } + + +def _install_writer_lock_landmines(ledger: object) -> None: + for name in WRITER_LOCK_LEDGER_METHODS: + def explode(*_args: object, _name: str = name, **_kwargs: object) -> object: + raise WriterLockReached( + f"public route reached the writer-lock read {_name}()" + ) + + setattr(ledger, name, explode) + + +class NoWriterLockReadTests(unittest.TestCase): + """No extracted route may reach a read that fences on the writer lock. + + This is the test that proves the extraction achieved its purpose. If it + fails, the public tier still serializes against the lease-holding writer + and the move bought only GIL separation. + """ + + def setUp(self) -> None: + self.ledger = NoWriterLockLedger() + _install_writer_lock_landmines(self.ledger) + self.coordinator = FakeCoordinator(ledger=self.ledger) + self.harness = ServiceHarness(self.coordinator) + self.addCleanup(self.harness.close) + + def test_no_extracted_route_touches_a_writer_lock_read(self) -> None: + for route in EXTRACTED_ROUTES: + with self.subTest(path=route[0]): + served = self.harness.get(request_path(route)) + # A landmine surfaces as a 500 in the public error schema, + # because the handler converts every unexpected exception. Any + # 5xx here means a writer-lock read was reached. + self.assertLess( + served.status, + 500, + f"{route[0]} reached a writer-lock read (status {served.status})", + ) + + def test_the_landmines_are_actually_armed(self) -> None: + # Guard against the previous test passing vacuously because the stub + # stopped raising. + for name in WRITER_LOCK_LEDGER_METHODS: + with self.subTest(method=name): + with self.assertRaises(WriterLockReached): + getattr(self.ledger, name)() + + def test_the_miner_page_uses_the_owed_balance_read_model(self) -> None: + # /public/v1/miners/{recipient_id} is the most-polled route on the + # surface; it must reach the per-recipient read model, not the + # every-recipient writer-lock dump. + served = self.harness.get(f"/public/v1/miners/{RECIPIENT_ID}") + + self.assertEqual(200, served.status) + self.assertEqual(1, self.ledger.owed_balance_calls) + + def test_direct_coinbase_settlement_uses_the_read_model(self) -> None: + # A block the CTV manifest-set read does not know: this is the branch + # that used to fall through to audit_bundle() under the writer lock. + served = self.harness.get( + f"/public/v1/blocks/{'0e' * 32}/settlement-artifacts" + ) + + self.assertEqual(404, served.status) + self.assertEqual(1, self.ledger.settlement_calls) + + +class StubCache: + """A response cache that answers with a chosen age, to drive staleness.""" + + def __init__(self, age_seconds: int, cache_state: str = "HIT") -> None: + self.age_seconds = age_seconds + self.cache_state = cache_state + self.stale_windows: list[int] = [] + + def get_or_compute( + self, + *, + key: object, + ttl_seconds: int, + compute, + stale_while_revalidate_seconds: int = 0, + ) -> tuple: + del key, ttl_seconds + self.stale_windows.append(stale_while_revalidate_seconds) + status, payload = compute() + return status, payload, self.cache_state, self.age_seconds + + +class StalenessContractTests(unittest.TestCase): + """Publish the budget, refuse past it, and never refuse immutable content.""" + + def harness_at_age(self, age_seconds: int) -> ServiceHarness: + harness = ServiceHarness( + FakeCoordinator(ledger=FullReadModelLedger()), + response_cache=StubCache(age_seconds), + ) + self.addCleanup(harness.close) + return harness + + def test_every_extracted_response_publishes_its_budget(self) -> None: + harness = ServiceHarness(FakeCoordinator(ledger=FullReadModelLedger())) + self.addCleanup(harness.close) + for route in EXTRACTED_ROUTES: + path = route[0] + with self.subTest(path=path): + served = harness.get(request_path(route)) + budget = served.headers.get( + public_read_service.STALENESS_BUDGET_HEADER + ) + self.assertIsNotNone( + budget, + f"{path} served no staleness budget", + ) + endpoint = endpoint_registry.endpoint_for_request_path(path) + assert endpoint is not None + if endpoint.immutable_content: + self.assertEqual( + public_read_service.UNBOUNDED_STALENESS_BUDGET, + budget, + ) + else: + self.assertEqual( + str(int(endpoint.max_staleness_seconds or 0)), + budget, + ) + + def test_a_response_past_its_budget_is_refused(self) -> None: + # One second past the largest budget on the surface, so every + # non-immutable route is over its own. + budgets = [ + endpoint.max_staleness_seconds or 0 + for endpoint in endpoint_registry.ENDPOINTS + if endpoint.disposition is endpoint_registry.Disposition.EXTRACT + ] + harness = self.harness_at_age(int(max(budgets)) + 1) + + for route in EXTRACTED_ROUTES: + path = route[0] + endpoint = endpoint_registry.endpoint_for_request_path(path) + assert endpoint is not None + if endpoint.immutable_content: + continue + with self.subTest(path=path): + served = harness.get(request_path(route)) + + self.assertEqual(503, served.status) + self.assertEqual("no-store", served.headers.get("Cache-Control")) + self.assertEqual( + "prism.dashboard.error.v1", + served.payload["schema"], + ) + message = served.payload["error"]["message"] + budget = endpoint.max_staleness_seconds or 0 + self.assertIn(f"{int(budget)}", message) + self.assertIn(f"{int(max(budgets)) + 1}", message) + + def test_a_refusal_still_reports_the_budget_and_the_observed_age(self) -> None: + harness = self.harness_at_age(10_000) + served = harness.get("/public/v1/pool-summary") + + self.assertEqual(503, served.status) + self.assertEqual( + "90", + served.headers.get(public_read_service.STALENESS_BUDGET_HEADER), + ) + self.assertEqual("10000", served.headers.get("Age")) + + def test_a_response_inside_its_budget_is_served(self) -> None: + # 60s is inside pool-summary's 90s budget and outside the 15s floor + # routes, so this also pins that the check is per-route. + harness = self.harness_at_age(60) + + self.assertEqual(200, harness.get("/public/v1/pool-summary").status) + self.assertEqual(503, harness.get("/public/v1/blocks").status) + + def test_immutable_artifacts_never_refuse_for_staleness(self) -> None: + # Content-addressed: a body that hashes to the requested sha256 is + # correct however old it is, so age must not turn it into a 503. + harness = self.harness_at_age(10_000_000) + served = harness.get(f"/public/v1/artifacts/{ARTIFACT_SHA256}") + + self.assertEqual(200, served.status) + self.assertEqual( + public_read_service.UNBOUNDED_STALENESS_BUDGET, + served.headers.get(public_read_service.STALENESS_BUDGET_HEADER), + ) + + def test_refusals_are_counted_in_metrics(self) -> None: + harness = self.harness_at_age(10_000) + harness.get("/public/v1/pool-summary") + + metrics = harness.get("/metrics") + self.assertEqual(200, metrics.status) + self.assertIn( + "qbit_prism_public_staleness_refusals_total 1", + metrics.body.decode(), + ) + + +class StatementTimeoutScopeLedger(FullReadModelLedger): + """FullReadModelLedger plus PsqlShareLedger's operation_timeout scope shape.""" + + def __init__(self) -> None: + super().__init__() + self.statement_timeout_budgets: list[float] = [] + + @contextmanager + def operation_timeout(self, timeout_seconds: float) -> Iterator[None]: + self.statement_timeout_budgets.append(timeout_seconds) + yield + + +class StatementTimeoutWiringTests(unittest.TestCase): + """The per-request database deadline arms exactly where a read slot is held. + + The deadline's own semantics (budget value, env kill switch, the + read_timeout 503, cache non-poisoning) are pinned in + tests/test_prism_public_dashboard_api.py; what belongs to this service is + the classification -- path_occupies_read_slot(), which is deliberately + wider than the staleness gates' path_reads_database(): a cold immutable + artifact lookup holds a Postgres read slot exactly like any other cold + read even though its body, once found, is correct at any age. + """ + + def test_the_scope_wraps_every_read_slot_route_and_nothing_else(self) -> None: + ledger = StatementTimeoutScopeLedger() + harness = ServiceHarness(FakeCoordinator(ledger=ledger)) + self.addCleanup(harness.close) + + for route in EXTRACTED_ROUTES: + path = route[0] + with self.subTest(path=path): + armed_before = len(ledger.statement_timeout_budgets) + served = harness.get(request_path(route)) + self.assertEqual(200, served.status) + armed = len(ledger.statement_timeout_budgets) > armed_before + # mining-configuration touches no read slot; every other + # extracted route -- the immutable artifact route included -- + # occupies one on a cold request and owes the deadline. + self.assertEqual( + public_read_service.path_occupies_read_slot(path), armed + ) + + def test_artifacts_owe_the_deadline_but_not_the_staleness_gates(self) -> None: + path = f"/public/v1/artifacts/{ARTIFACT_SHA256}" + self.assertTrue(public_read_service.path_occupies_read_slot(path)) + # Deliberately split from path_reads_database: immutable content is + # exempt from replica/outage refusals, never from the deadline. + self.assertFalse(public_read_service.path_reads_database(path)) + + def test_mining_configuration_arms_nothing(self) -> None: + path = "/public/v1/mining-configuration" + self.assertFalse(public_read_service.path_occupies_read_slot(path)) + self.assertFalse(public_read_service.path_reads_database(path)) + + def test_one_budget_covers_a_multi_read_dispatch(self) -> None: + """The scope is entered once around the whole dispatch, not per read. + + The miner detail route performs several sequential ledger reads; a + per-statement scope would hand each a fresh budget and let the + request run many multiples of the configured limit. Exactly one + operation_timeout entry proves the deadline spans them all. + """ + ledger = StatementTimeoutScopeLedger() + harness = ServiceHarness(FakeCoordinator(ledger=ledger)) + self.addCleanup(harness.close) + + served = harness.get(f"/public/v1/miners/{RECIPIENT_ID}") + + self.assertEqual(200, served.status) + self.assertEqual(1, len(ledger.statement_timeout_budgets)) + + +class StaleServeTests(unittest.TestCase): + """Read-service wiring for the in-process stale-while-revalidate window.""" + + def setUp(self) -> None: + self.cache = public_api.PublicResponseCache() + self.metrics = public_read_service.ServiceMetrics() + self.harness = ServiceHarness( + FakeCoordinator(ledger=FullReadModelLedger()), + response_cache=self.cache, + metrics=self.metrics, + ) + self.addCleanup(self.harness.close) + + def age_blocks_entry(self, age_seconds: float) -> None: + """Age the warm /public/v1/blocks entry to a chosen served age. + + The route's default TTL is 5s, so any age beyond that is expired; + the bounds are edited directly for the reason cached_entry() states. + """ + entry = cached_entry(self.cache, "/public/v1/blocks") + assert entry is not None + entry.stored_at = time.monotonic() - age_seconds # type: ignore[union-attr] + entry.expires_at = entry.stored_at + 5.0 # type: ignore[union-attr] + + def test_a_stale_entry_inside_the_budget_is_served_with_its_age(self) -> None: + warm = self.harness.get("/public/v1/blocks") + self.age_blocks_entry(8.0) # 3s past the 5s TTL, inside the 15s budget + + served = self.harness.get("/public/v1/blocks") + + self.assertEqual(200, served.status) + self.assertEqual(warm.body, served.body) + self.assertEqual("8", served.headers.get("Age")) + + def test_cdn_policy_cannot_exceed_the_route_budget(self) -> None: + for ttl, swr, expected in ( + (5, 30, "public, max-age=5, stale-while-revalidate=10"), + (60, 30, "public, max-age=15"), + (5, 0, "public, max-age=5"), + ): + with self.subTest(ttl=ttl, swr=swr), patch.object( + public_api, "public_cache_policy", + return_value=public_api.PublicCachePolicy(ttl, swr), + ): + served = self.harness.get(f"/public/v1/blocks?limit=1&case={ttl}-{swr}") + self.assertEqual(served.status, 200) + self.assertEqual(served.headers["X-Prism-Staleness-Budget-Seconds"], "15") + self.assertEqual(served.headers["CDN-Cache-Control"], expected) + self.assertEqual(served.headers["Vercel-CDN-Cache-Control"], expected) + + def test_stale_serves_are_counted_in_cache_metrics(self) -> None: + self.harness.get("/public/v1/blocks") + self.age_blocks_entry(8.0) + self.harness.get("/public/v1/blocks") + + body = self.harness.get("/metrics").body.decode() + + self.assertIn('qbit_prism_public_cache_total{state="stale"} 1', body) + + def test_the_window_is_clamped_to_the_route_staleness_budget(self) -> None: + """A stale serve must never turn an age the blocking path handled into a 503. + + Age 20 is inside the raw 30s stale-while-revalidate window but past + the 15s budget /public/v1/blocks publishes. Unclamped, the cache + would serve it STALE and the unconditional budget check would refuse + it; clamped to budget minus TTL, the request falls through to the + blocking recompute exactly as it did before the window existed. + """ + self.harness.get("/public/v1/blocks") + self.age_blocks_entry(20.0) + + served = self.harness.get("/public/v1/blocks") + + self.assertEqual(200, served.status) + self.assertEqual("0", served.headers.get("Age")) + + def test_an_outage_withdraws_the_window(self) -> None: + """While degraded, only unexpired entries are served (the #164 contract). + + The stale-while-revalidate window must not widen what an outage may + answer with, and its background refresh must not become a second + origin path the outage gate does not see. + """ + probe = ScriptedReadiness() + readiness = public_read_service.ReadinessProbe(probe, interval_seconds=3600) + readiness.check_once() + cache = public_api.PublicResponseCache() + harness = ServiceHarness( + FakeCoordinator(ledger=FullReadModelLedger()), + response_cache=cache, + readiness=readiness, + ) + self.addCleanup(harness.close) + harness.get("/public/v1/blocks") + entry = cached_entry(cache, "/public/v1/blocks") + assert entry is not None + entry.stored_at = time.monotonic() - 8.0 # type: ignore[union-attr] + entry.expires_at = entry.stored_at + 5.0 # type: ignore[union-attr] + probe.healthy = False + readiness.check_once() + + served = harness.get("/public/v1/blocks") + + self.assertEqual(503, served.status) + self.assertEqual("upstream_unavailable", served.payload["error"]["code"]) + + +class FakeClock: + """Monotonic time the test moves by hand.""" + + def __init__(self) -> None: + self.now = 1_000.0 + + def __call__(self) -> float: + return self.now + + def advance(self, seconds: float) -> None: + self.now += seconds + + +class ScriptedReplica: + """A read_replica_status() whose answers -- and failures -- the test picks.""" + + def __init__(self, *answers: object) -> None: + self.answers = list(answers) + self.calls = 0 + + def __call__(self) -> dict[str, object]: + self.calls += 1 + answer = self.answers[min(self.calls, len(self.answers)) - 1] + if isinstance(answer, Exception): + raise answer + assert isinstance(answer, dict) + return answer + + +def standby(heartbeat: float | None = 1.0, replay_lag: float | None = 2.0) -> dict: + return { + "in_recovery": True, + "receiver_heartbeat_age_seconds": heartbeat, + "replay_lag_seconds": replay_lag, + "apply_backlog_bytes": 4096, + } + + +PRIMARY_STATUS = { + "in_recovery": False, + "receiver_heartbeat_age_seconds": None, + "replay_lag_seconds": None, + "apply_backlog_bytes": None, +} + + +class ReplicaContractTests(unittest.TestCase): + """Serve from a live standby, or refuse and say which part failed. + + The bound is on the walreceiver heartbeat rather than replay lag; see + ReplicaProbe. These tests drive the probe directly with a fake clock, so + they pin the freshness rules without a Postgres. + """ + + def probe( + self, + *answers: object, + max_lag_seconds: float = 60.0, + ) -> tuple[public_read_service.ReplicaProbe, FakeClock]: + clock = FakeClock() + probe = public_read_service.ReplicaProbe( + ScriptedReplica(*answers), + max_lag_seconds=max_lag_seconds, + monotonic=clock, + ) + return probe, clock + + def harness(self, probe: object, **kwargs: object) -> ServiceHarness: + harness = ServiceHarness( + FakeCoordinator(ledger=FullReadModelLedger()), + replica=probe, + **kwargs, # type: ignore[arg-type] + ) + self.addCleanup(harness.close) + return harness + + def test_before_the_first_probe_replica_routes_are_warming_up(self) -> None: + probe, _ = self.probe(standby()) + served = self.harness(probe).get("/public/v1/blocks") + + self.assertEqual(503, served.status) + self.assertIn("warming up", served.payload["error"]["message"]) + + def test_a_live_standby_is_served(self) -> None: + probe, _ = self.probe(standby()) + probe.check() + served = self.harness(probe).get("/public/v1/blocks") + + self.assertEqual(200, served.status) + + def test_a_writable_primary_is_refused(self) -> None: + probe, _ = self.probe(PRIMARY_STATUS) + self.assertFalse(probe.check()) + served = self.harness(probe).get("/public/v1/blocks") + + self.assertEqual(503, served.status) + self.assertIn("not in recovery", served.payload["error"]["message"]) + + def test_a_refusal_uses_the_documented_public_error_enum(self) -> None: + # PUBLIC_ERROR_CODES silently rewrites an unknown code to + # internal_error, so an undocumented one would ship a 503 that says + # "internal error" and never fail a test that only reads the message. + probe, _ = self.probe(PRIMARY_STATUS) + probe.check() + served = self.harness(probe).get("/public/v1/blocks") + + self.assertEqual("prism.dashboard.error.v1", served.payload["schema"]) + self.assertEqual("upstream_unavailable", served.payload["error"]["code"]) + + def test_a_disconnected_walreceiver_is_refused(self) -> None: + probe, _ = self.probe(standby(heartbeat=None)) + probe.check() + served = self.harness(probe).get("/public/v1/blocks") + + self.assertEqual(503, served.status) + self.assertIn("not connected", served.payload["error"]["message"]) + + def test_a_heartbeat_past_the_bound_is_refused(self) -> None: + probe, _ = self.probe(standby(heartbeat=61.0), max_lag_seconds=60.0) + probe.check() + served = self.harness(probe).get("/public/v1/blocks") + + self.assertEqual(503, served.status) + self.assertIn("silent for", served.payload["error"]["message"]) + + def test_a_failing_probe_ages_its_last_good_answer_out(self) -> None: + """The gate fails closed: a dead probe must not pin a fresh verdict. + + This is the whole reason freshness is computed as "heartbeat age when + last probed, plus how long ago that was". Without the second term a + replica that stopped answering entirely keeps serving 200 forever off + its last good snapshot. + """ + probe, clock = self.probe( + standby(heartbeat=1.0), + RuntimeError("connection refused"), + max_lag_seconds=60.0, + ) + probe.check() + harness = self.harness(probe) + + # Probe starts failing; the snapshot is still inside the bound. + for _ in range(100): + with self.assertRaises(RuntimeError): + probe.check() + clock.advance(30.0) + self.assertEqual(200, harness.get("/public/v1/blocks").status) + + # Past the bound, the same unchanged snapshot must now refuse. + clock.advance(30.0) + served = harness.get("/public/v1/blocks") + self.assertEqual(503, served.status) + self.assertIn("silent for", served.payload["error"]["message"]) + self.assertIn("connection refused", served.payload["error"]["message"]) + + def test_a_refusal_is_not_masked_by_a_warm_cache(self) -> None: + """The gate runs before the cache, not after. + + A replica outage does not expire cache entries, so a check placed after + the cache would keep answering 200 from the last good snapshot for as + long as traffic kept it warm. + """ + probe, clock = self.probe(standby(heartbeat=1.0), max_lag_seconds=60.0) + probe.check() + harness = self.harness(probe, response_cache=StubCache(0)) + + self.assertEqual(200, harness.get("/public/v1/blocks").status) + clock.advance(120.0) + self.assertEqual(503, harness.get("/public/v1/blocks").status) + + def test_routes_that_read_no_replica_state_keep_serving(self) -> None: + probe, _ = self.probe(PRIMARY_STATUS) + probe.check() + harness = self.harness(probe) + + self.assertEqual(503, harness.get("/public/v1/blocks").status) + self.assertEqual( + 200, harness.get("/public/v1/mining-configuration").status + ) + + def test_immutable_artifacts_are_not_replica_gated(self) -> None: + # Content-addressed: the body either hashes to the requested sha256 or + # it does not, which replication lag cannot change. The registry gives + # these no staleness budget for the same reason. + probe, _ = self.probe(PRIMARY_STATUS) + probe.check() + harness = self.harness(probe) + + self.assertFalse( + public_read_service.path_reads_database( + f"/public/v1/artifacts/{ARTIFACT_SHA256}" + ) + ) + self.assertEqual( + 200, harness.get(f"/public/v1/artifacts/{ARTIFACT_SHA256}").status + ) + + def test_every_replica_backed_route_is_gated(self) -> None: + """Exhaustive, not a sample: every extracted route that reads a read + slot must refuse, and the classification must come from the registry.""" + + probe, _ = self.probe(PRIMARY_STATUS) + probe.check() + harness = self.harness(probe) + + gated = 0 + for route in EXTRACTED_ROUTES: + path = route[0] + endpoint = endpoint_registry.endpoint_for_request_path(path) + assert endpoint is not None + expected_gated = ( + not endpoint.immutable_content + and endpoint_registry.LedgerAccess.READ_SLOT in endpoint.access + ) + with self.subTest(path=path): + self.assertEqual( + expected_gated, + public_read_service.path_reads_database(path), + ) + served = harness.get(request_path(route)) + if expected_gated: + gated += 1 + self.assertEqual(503, served.status) + self.assertIn( + "not in recovery", served.payload["error"]["message"] + ) + else: + self.assertNotEqual(503, served.status) + self.assertGreater(gated, 0, "no route exercised the replica gate") + + def test_observed_replay_lag_travels_on_the_response(self) -> None: + probe, clock = self.probe(standby(heartbeat=1.0, replay_lag=2.0)) + probe.check() + clock.advance(3.0) + served = self.harness(probe).get("/public/v1/blocks") + + self.assertEqual(200, served.status) + self.assertEqual( + "5.000", + served.headers.get(public_read_service.REPLICA_LAG_HEADER), + ) + + def test_healthz_carries_the_replication_facts(self) -> None: + probe, _ = self.probe(standby(heartbeat=1.0)) + probe.check() + served = self.harness(probe).get("/healthz") + + self.assertEqual(200, served.status) + self.assertEqual(True, served.payload["replica"]["in_recovery"]) + self.assertEqual( + 1.0, served.payload["replica"]["receiver_heartbeat_age_seconds"] + ) + self.assertEqual(60.0, served.payload["replica"]["max_lag_seconds"]) + + def test_healthz_goes_unready_when_the_contract_fails(self) -> None: + probe, _ = self.probe(PRIMARY_STATUS) + probe.check() + served = self.harness(probe).get("/healthz") + + self.assertEqual(503, served.status) + self.assertFalse(served.payload["ok"]) + self.assertIn("not in recovery", served.payload["error"]) + + def test_metrics_publish_the_replica_gauges_and_refusals(self) -> None: + probe, _ = self.probe(standby(heartbeat=1.0, replay_lag=2.0)) + probe.check() + harness = self.harness(probe) + harness.get("/public/v1/blocks") + + body = harness.get("/metrics").body.decode() + self.assertIn("qbit_prism_public_replica_in_recovery 1", body) + self.assertIn( + "qbit_prism_public_replica_heartbeat_age_seconds 1.000", body + ) + self.assertIn("qbit_prism_public_replica_replay_lag_seconds 2.000", body) + self.assertIn("qbit_prism_public_replica_apply_backlog_bytes 4096.000", body) + self.assertIn("qbit_prism_public_replica_max_lag_seconds 60.000", body) + self.assertIn("qbit_prism_public_replica_refusals_total 0", body) + + def test_replica_refusals_are_counted(self) -> None: + probe, _ = self.probe(PRIMARY_STATUS) + probe.check() + harness = self.harness(probe) + harness.get("/public/v1/blocks") + + self.assertIn( + "qbit_prism_public_replica_refusals_total 1", + harness.get("/metrics").body.decode(), + ) + + def test_unknown_numbers_report_minus_one_rather_than_zero(self) -> None: + probe, _ = self.probe(standby(heartbeat=None, replay_lag=None)) + probe.check() + harness = self.harness(probe) + + body = harness.get("/metrics").body.decode() + self.assertIn("qbit_prism_public_replica_heartbeat_age_seconds -1", body) + self.assertIn("qbit_prism_public_replica_replay_lag_seconds -1", body) + + +class ReplicaModeTests(unittest.TestCase): + """PRISM_PUBLIC_REPLICA_MODE decides whether the contract is enforced.""" + + class Ledger: + def read_replica_status(self) -> dict[str, object]: + return standby() + + def test_absent_mode_enforces_nothing(self) -> None: + # The default must not 503 a deployment that merged this before it + # provisioned a standby; the shipped compose sets require explicitly. + self.assertEqual( + public_read_service.REPLICA_MODE_OFF, + public_read_service.resolve_replica_mode({}), + ) + self.assertIsNone(public_read_service.build_replica_probe(self.Ledger(), {})) + + def test_require_builds_the_probe(self) -> None: + probe = public_read_service.build_replica_probe( + self.Ledger(), {"PRISM_PUBLIC_REPLICA_MODE": "require"} + ) + self.assertIsInstance(probe, public_read_service.ReplicaProbe) + + def test_an_unrecognised_mode_refuses_startup(self) -> None: + # Neither default is safe to guess: the two modes disagree about + # whether this process may read the coordinator's primary. + with self.assertRaises( + public_read_service.PublicReadConfigurationError + ) as raised: + public_read_service.resolve_replica_mode( + {"PRISM_PUBLIC_REPLICA_MODE": "yes"} + ) + self.assertIn("PRISM_PUBLIC_REPLICA_MODE", str(raised.exception)) + + +class DatabaseReached(AssertionError): + """Raised when a route consulted the database while readiness said it was down.""" + + +class ScriptedReadiness: + """A readiness probe callable whose answer the test flips. + + Raises rather than returning False, because that is what an unreachable + Postgres actually does to ``dashboard_readiness_probe``, and the two take + different arms inside ReadinessProbe.check_once(). + """ + + def __init__(self, *, healthy: bool = True) -> None: + self.healthy = healthy + self.calls = 0 + + def __call__(self) -> bool: + self.calls += 1 + if not self.healthy: + raise RuntimeError("connection refused") + return True + + +def cached_entry( + cache: public_api.PublicResponseCache, + path: str, +) -> object | None: + """The live cache entry for a path, or None. + + Reaches into PublicResponseCache's storage deliberately. Expiry there is + keyed on time.monotonic(), which is not injectable, so the alternatives for + driving the expired-entry case are sleeping out a real TTL or replacing the + cache with a double -- and a double cannot prove the thing under test, + which is that a degraded request leaves the *real* entry's bounds alone. + """ + return cache._entries.get(public_api.public_cache_key(path, {})) + + +class CoalescingCache: + """A cache that hands back a result some other request computed. + + This is PublicResponseCache's waiter path: a request that arrives while + another is already computing the same key never calls compute() itself and + receives the owner's result. Modelled here rather than raced, because the + property under test is what the handler does with a result it did not + produce, not the cache's own locking. + """ + + def __init__(self, result: tuple) -> None: + self.result = result + self.computes = 0 + + def get_or_compute( + self, + *, + key: object, + ttl_seconds: int, + compute, + stale_while_revalidate_seconds: int = 0, + ) -> tuple: + # a waiter never runs compute() + del key, ttl_seconds, compute, stale_while_revalidate_seconds + return self.result + + +class DatabaseOutageTests(unittest.TestCase): + """Bounded-stale availability while Postgres is unreachable (#164). + + The endpoint is public and the database is not: an outage that turns every + dashboard into an error page is an outage the public sees in full. What the + surface owes instead is its own already-published bound -- serve the warm + entry if there is one inside the route's staleness budget, say plainly that + it is doing so, and refuse otherwise. + + Two properties make that honest rather than a way to hide an outage, and + each has a test below: a degraded request never calls the database (so the + outage is not amplified by the retry traffic it causes), and it never + extends the entry it serves (so ordinary polling cannot walk a stale + dashboard past the bound, one hit at a time, for as long as the outage + lasts). + """ + + def setUp(self) -> None: + self.ledger = FullReadModelLedger() + self.coordinator = FakeCoordinator(ledger=self.ledger) + self.cache = public_api.PublicResponseCache() + self.probe_source = ScriptedReadiness() + self.readiness = public_read_service.ReadinessProbe( + self.probe_source, + interval_seconds=3600, + ) + self.readiness.check_once() + self.metrics = public_read_service.ServiceMetrics() + self.reached: list[str] = [] + self.armed: list[str] = [] + self.harness = ServiceHarness( + self.coordinator, + response_cache=self.cache, + readiness=self.readiness, + metrics=self.metrics, + ) + self.addCleanup(self.harness.close) + + def take_database_down(self, *, arm_landmines: bool = True) -> None: + """Readiness starts failing, and every database read becomes a landmine. + + Everything public_api asks a ledger for is a database read, so arming + all of them turns "the handler went to the origin anyway" into a + recorded method name rather than a subtle difference in a body. + """ + self.probe_source.healthy = False + self.readiness.check_once() + self.assertEqual(503, self.readiness.snapshot()[0]) + if not arm_landmines: + return + names = {name for name in dir(self.ledger) if name.startswith("dashboard_")} + names.update( + name for name in WRITER_LOCK_LEDGER_METHODS if hasattr(self.ledger, name) + ) + for name in sorted(names): + def explode(*_args: object, _name: str = name, **_kwargs: object) -> object: + self.reached.append(_name) + raise DatabaseReached(f"the origin reached {_name}() during an outage") + + setattr(self.ledger, name, explode) + self.armed.append(name) + + def bring_database_up(self) -> None: + for name in self.armed: + delattr(self.ledger, name) + self.armed.clear() + self.probe_source.healthy = True + self.readiness.check_once() + self.assertEqual(200, self.readiness.snapshot()[0]) + + def assert_degraded(self, served: ServedResponse) -> None: + self.assertEqual( + public_read_service.DEGRADED_WARNING, + served.headers.get(public_read_service.DEGRADED_WARNING_HEADER), + ) + self.assertEqual( + public_read_service.DATABASE_STATE_UNAVAILABLE, + served.headers.get(public_read_service.DATABASE_STATE_HEADER), + ) + + def test_a_warm_entry_is_served_and_says_so(self) -> None: + warm = self.harness.get("/public/v1/blocks") + self.assertEqual(200, warm.status) + self.take_database_down() + + served = self.harness.get("/public/v1/blocks") + + self.assertEqual(200, served.status) + self.assertEqual(warm.body, served.body) + self.assertEqual([], self.reached) + # Pinned literally, not through the constant: this string is the + # contract a dashboard frontend parses. + self.assertEqual( + '110 qbit-prism "database unavailable; serving cached response"', + served.headers.get("Warning"), + ) + self.assertEqual( + "unavailable", served.headers.get("X-Prism-Database-State") + ) + # The outage fact and the numerical bound travel together: without the + # second, "stale" has no size. + self.assertEqual("0", served.headers.get("Age")) + self.assertEqual( + "15", + served.headers.get(public_read_service.STALENESS_BUDGET_HEADER), + ) + + def test_a_cold_cache_refuses_without_calling_the_database(self) -> None: + self.take_database_down() + + served = self.harness.get("/public/v1/blocks") + + self.assertEqual(503, served.status) + self.assertEqual([], self.reached) + self.assertEqual("no-store", served.headers.get("Cache-Control")) + self.assertEqual("prism.dashboard.error.v1", served.payload["schema"]) + # PUBLIC_ERROR_CODES rewrites an undocumented code to internal_error, + # so a 503 that only reads right in the message is not enough. + self.assertEqual("upstream_unavailable", served.payload["error"]["code"]) + self.assertIn( + "cannot reach the database", served.payload["error"]["message"] + ) + self.assertEqual( + "unavailable", served.headers.get("X-Prism-Database-State") + ) + # 110 is "Response is Stale". Nothing was served, stale or otherwise. + self.assertIsNone(served.headers.get("Warning")) + self.assertEqual( + "15", + served.headers.get(public_read_service.STALENESS_BUDGET_HEADER), + ) + + def test_a_refusal_stores_nothing(self) -> None: + # Otherwise the first refusal would become a warm entry and the next + # request would serve a 503 body as a cached 503 -- or worse, count as + # the warm entry the outage path is allowed to serve. + self.take_database_down() + self.harness.get("/public/v1/blocks") + + self.assertIsNone(cached_entry(self.cache, "/public/v1/blocks")) + + def test_an_expired_entry_refuses_and_traffic_cannot_refill_it(self) -> None: + self.harness.get("/public/v1/blocks") + entry = cached_entry(self.cache, "/public/v1/blocks") + self.assertIsNotNone(entry) + self.take_database_down() + # Expire it where the cache actually looks, rather than sleeping out a + # real TTL. + entry.expires_at = 0.0 # type: ignore[union-attr] + + for _ in range(3): + self.assertEqual(503, self.harness.get("/public/v1/blocks").status) + + self.assertEqual([], self.reached) + self.assertIsNone(cached_entry(self.cache, "/public/v1/blocks")) + + def test_an_entry_past_the_route_budget_refuses_though_it_is_warm(self) -> None: + """TTL and budget are separate numbers, and the budget is the promise. + + Nothing stops an operator raising PRISM_PUBLIC_*_CACHE_TTL_SECONDS + above the registry budget the same route publishes. When that happens + the entry is unexpired and past the bound at once, and the bound wins. + """ + self.harness.get("/public/v1/blocks") + entry = cached_entry(self.cache, "/public/v1/blocks") + entry.stored_at -= 3600.0 # type: ignore[union-attr] + entry.expires_at += 3600.0 # type: ignore[union-attr] + self.take_database_down() + + for _ in range(3): + served = self.harness.get("/public/v1/blocks") + self.assertEqual(503, served.status) + + self.assertEqual([], self.reached) + self.assertIn( + "exceeds the staleness budget", served.payload["error"]["message"] + ) + self.assertEqual("no-store", served.headers.get("Cache-Control")) + self.assertEqual("3600", served.headers.get("Age")) + self.assertEqual( + "15", + served.headers.get(public_read_service.STALENESS_BUDGET_HEADER), + ) + self.assertEqual( + "unavailable", served.headers.get("X-Prism-Database-State") + ) + self.assertIsNone(served.headers.get("Warning")) + + def test_degraded_traffic_never_extends_the_entry_it_serves(self) -> None: + """The bound has to survive the traffic, or it is not a bound. + + A degraded hit that refreshed stored_at or expires_at would let + ordinary polling walk one snapshot forward indefinitely, and the + staleness budget on every one of those responses would be a number + describing nothing. + """ + self.harness.get("/public/v1/blocks") + entry = cached_entry(self.cache, "/public/v1/blocks") + assert entry is not None + bounds = (entry.stored_at, entry.expires_at) + self.take_database_down() + + for _ in range(5): + self.assert_degraded(self.harness.get("/public/v1/blocks")) + + self.assertEqual(bounds, (entry.stored_at, entry.expires_at)) + # And when those bounds are reached, the outage cannot renew them. + entry.expires_at = 0.0 + self.assertEqual(503, self.harness.get("/public/v1/blocks").status) + + def test_recovery_restores_the_origin_and_drops_the_degraded_headers(self) -> None: + self.harness.get("/public/v1/blocks") + self.take_database_down() + self.assert_degraded(self.harness.get("/public/v1/blocks")) + + self.bring_database_up() + # Expire the warm entry so the answer has to come from the origin. + cached_entry(self.cache, "/public/v1/blocks").expires_at = 0.0 # type: ignore[union-attr] + served = self.harness.get("/public/v1/blocks") + + self.assertEqual(200, served.status) + self.assertIsNone(served.headers.get("Warning")) + self.assertIsNone(served.headers.get("X-Prism-Database-State")) + self.assertEqual("0", served.headers.get("Age")) + # Refilled, which only the origin path can do. + self.assertIsNotNone(cached_entry(self.cache, "/public/v1/blocks")) + + def test_every_database_backed_route_degrades_and_the_rest_do_not(self) -> None: + """Exhaustive and registry-derived, so a new route is classified once.""" + + for route in EXTRACTED_ROUTES: + self.assertEqual( + 200, self.harness.get(request_path(route)).status, route[0] + ) + self.take_database_down() + + degraded = 0 + for route in EXTRACTED_ROUTES: + path = route[0] + endpoint = endpoint_registry.endpoint_for_request_path(path) + assert endpoint is not None + expected = ( + not endpoint.immutable_content + and endpoint_registry.LedgerAccess.READ_SLOT in endpoint.access + ) + with self.subTest(path=path): + self.assertEqual( + expected, public_read_service.path_reads_database(path) + ) + served = self.harness.get(request_path(route)) + self.assertEqual(200, served.status) + if expected: + degraded += 1 + self.assert_degraded(served) + self.assertIsNotNone(served.headers.get("Age")) + self.assertIsNotNone( + served.headers.get( + public_read_service.STALENESS_BUDGET_HEADER + ) + ) + else: + self.assertIsNone(served.headers.get("Warning")) + self.assertIsNone( + served.headers.get("X-Prism-Database-State") + ) + self.assertEqual([], self.reached) + self.assertGreater(degraded, 0, "no route exercised the outage gate") + + def test_the_exempt_routes_stay_available_from_a_cold_cache(self) -> None: + """Mining configuration and immutable artifacts owe answers regardless. + + The first is assembled from the environment and has no database answer + to lose; the second is content-addressed, so its body either hashes to + the requested sha256 or it does not, which an outage cannot change. + Landmines stay disarmed here precisely because these two are expected + to reach their origin. + """ + self.take_database_down(arm_landmines=False) + + refused = 0 + for route in EXTRACTED_ROUTES: + path = route[0] + with self.subTest(path=path): + served = self.harness.get(request_path(route)) + if public_read_service.path_reads_database(path): + refused += 1 + self.assertEqual(503, served.status) + self.assertEqual( + "upstream_unavailable", served.payload["error"]["code"] + ) + self.assertEqual( + "no-store", served.headers.get("Cache-Control") + ) + else: + self.assertEqual(200, served.status) + self.assertIsNone(served.headers.get("Warning")) + self.assertIsNone( + served.headers.get("X-Prism-Database-State") + ) + self.assertEqual(len(EXTRACTED_ROUTES) - 2, refused) + + def test_a_bypassed_cache_refuses_rather_than_calling_the_database(self) -> None: + # With the response cache off there is no warm entry to serve and no + # way to make one, so every request is the cold case. + with patch.dict(os.environ, {"PRISM_PUBLIC_CACHE_ENABLED": "false"}): + self.harness.get("/public/v1/blocks") + self.take_database_down() + served = self.harness.get("/public/v1/blocks") + + self.assertEqual(503, served.status) + self.assertEqual([], self.reached) + self.assertEqual( + "unavailable", served.headers.get("X-Prism-Database-State") + ) + + def test_a_route_that_reads_no_database_is_never_degraded(self) -> None: + self.take_database_down() + + served = self.harness.get("/public/v1/mining-configuration") + + self.assertEqual(200, served.status) + self.assertIsNone(served.headers.get("Warning")) + self.assertIsNone(served.headers.get("X-Prism-Database-State")) + + def test_degraded_responses_and_refusals_are_counted(self) -> None: + self.harness.get("/public/v1/blocks") + self.take_database_down() + self.harness.get("/public/v1/blocks") # warm: degraded 200 + self.harness.get("/public/v1/leaderboard") # cold: refused + + body = self.harness.get("/metrics").body.decode() + + self.assertIn("qbit_prism_public_degraded_responses_total 1", body) + self.assertIn("qbit_prism_public_database_outage_refusals_total 1", body) + + def test_a_healthy_readiness_snapshot_changes_nothing(self) -> None: + # The gate must be invisible while the database is up, including on the + # first, uncached request. + served = self.harness.get("/public/v1/blocks") + + self.assertEqual(200, served.status) + self.assertIsNone(served.headers.get("Warning")) + self.assertIsNone(served.headers.get("X-Prism-Database-State")) + body = self.harness.get("/metrics").body.decode() + self.assertIn("qbit_prism_public_degraded_responses_total 0", body) + self.assertIn("qbit_prism_public_database_outage_refusals_total 0", body) + + def test_a_warming_up_probe_is_treated_as_unavailable(self) -> None: + # Before the first probe completes there is no evidence the database is + # reachable, and this gate fails closed on absent evidence like every + # other freshness check in this service. + readiness = public_read_service.ReadinessProbe( + ScriptedReadiness(), interval_seconds=3600 + ) + harness = ServiceHarness( + FakeCoordinator(ledger=FullReadModelLedger()), readiness=readiness + ) + self.addCleanup(harness.close) + + self.assertEqual(503, harness.get("/public/v1/blocks").status) + self.assertEqual(200, harness.get("/public/v1/mining-configuration").status) + + def test_a_coalesced_refusal_is_never_shared_cacheable(self) -> None: + """A waiter must not turn another request's refusal into a CDN entry. + + get_or_compute() coalesces: a request arriving while another is + computing the same key gets that one's result, whatever its own view of + readiness was. So a request whose probe had already recovered can + receive the outage refusal -- and if the handler decided "is this the + refusal" from the request instead of from the response, it would send + that 503 out under the ordinary shared-cache headers and a CDN would + hold it for the route's TTL. + """ + harness = ServiceHarness( + self.coordinator, + response_cache=CoalescingCache( + (503, public_read_service.DATABASE_UNAVAILABLE_PAYLOAD, "MISS", 0) + ), + readiness=self.readiness, # healthy: this request never degraded + ) + self.addCleanup(harness.close) + + served = harness.get("/public/v1/blocks") + + self.assertEqual(503, served.status) + self.assertEqual("no-store", served.headers.get("Cache-Control")) + self.assertIsNone(served.headers.get("CDN-Cache-Control")) + self.assertEqual( + "unavailable", served.headers.get("X-Prism-Database-State") + ) + + def test_a_coalesced_fresh_body_is_not_labelled_stale(self) -> None: + """The mirror case: the waiter is the degraded one, the body is fresh. + + Warning 110 and X-Prism-Database-State are claims about the body being + served. A response another request just computed from a database that + answered it is not a cached response, whatever this request's probe + said a moment later. + """ + fresh = {"schema": "prism.dashboard.blocks.v1", "rows": []} + self.take_database_down() + harness = ServiceHarness( + self.coordinator, + response_cache=CoalescingCache((200, fresh, "MISS", 0)), + readiness=self.readiness, + ) + self.addCleanup(harness.close) + + served = harness.get("/public/v1/blocks") + + self.assertEqual(200, served.status) + self.assertEqual(fresh, served.payload) + self.assertIsNone(served.headers.get("Warning")) + self.assertIsNone(served.headers.get("X-Prism-Database-State")) + + def test_replica_mode_require_keeps_its_stricter_refusal(self) -> None: + """A warm entry does not soften the standby contract. + + In require mode the replica probe is the readiness probe, so both gates + see the same failure. They do not owe the same answer: reading the + coordinator's primary is refused because of *where* the answer would + come from, which a cached body from that same wrong place does not fix. + """ + clock = FakeClock() + replica = public_read_service.ReplicaProbe( + ScriptedReplica(standby(heartbeat=1.0), PRIMARY_STATUS), + max_lag_seconds=60.0, + monotonic=clock, + ) + readiness = public_read_service.ReadinessProbe( + replica.check, interval_seconds=3600, monotonic=clock + ) + readiness.check_once() + harness = ServiceHarness( + FakeCoordinator(ledger=FullReadModelLedger()), + readiness=readiness, + replica=replica, + ) + self.addCleanup(harness.close) + self.assertEqual(200, harness.get("/public/v1/blocks").status) + + readiness.check_once() + served = harness.get("/public/v1/blocks") + + self.assertEqual(503, served.status) + self.assertIn("not in recovery", served.payload["error"]["message"]) + self.assertIsNone(served.headers.get("Warning")) + + +class FailClosedStartupTests(unittest.TestCase): + """Refuse to start rather than serve wrong facts.""" + + def test_missing_public_stratum_url_refuses_startup(self) -> None: + for environ in ({}, {"PRISM_PUBLIC_STRATUM_URL": ""}, {"PRISM_PUBLIC_STRATUM_URL": " "}): + with self.subTest(environ=environ): + with self.assertRaises( + public_read_service.PublicReadConfigurationError + ) as raised: + public_read_service.require_public_stratum_url(environ) + self.assertIn("PRISM_PUBLIC_STRATUM_URL", str(raised.exception)) + + def test_a_configured_public_stratum_url_is_accepted(self) -> None: + value = public_read_service.require_public_stratum_url( + {"PRISM_PUBLIC_STRATUM_URL": "stratum+tcp://pool.example:3340"} + ) + self.assertEqual("stratum+tcp://pool.example:3340", value) + + def test_invalid_stratum_urls_refuse_before_database_construction(self) -> None: + for url in ( + "pool.example:3340", "https://pool.example:3340", + "stratum+tcp://pool.example", "stratum+tcp://pool.example:not-a-port", + "stratum+tcp://pool.example:0", "stratum+tcp://pool.example:65536", + "stratum+tcp://:3340", "stratum+tcp://[broken:3340", + "stratum+tcp://user:password@pool.example:3340", + "stratum+tcp://pool.example:3340/path", "stratum+tcp://pool.example:3340?q=1", + "stratum+tcp://pool.example:3340#fragment", "stratum+tcp://bad host:3340", + "stratum+tcp://pool.example:33\t40", " stratum+tcp://pool.example:3340 ", + "stratum+tcp://bad%host:3340", "stratum+tcp://bad\\host:3340", + ): + with self.subTest(url=url), patch.object( + public_read_service, "build_ledger_from_env" + ) as ledger: + with self.assertRaises(public_read_service.PublicReadConfigurationError): + public_read_service.build_service({"PRISM_PUBLIC_STRATUM_URL": url}) + ledger.assert_not_called() + + def test_highdiff_url_is_validated_only_when_advertised(self) -> None: + base = {"PRISM_PUBLIC_STRATUM_URL": "stratum+tcp://pool.example:3340"} + for bad in ("https://host/path", "stratum+tcp://host:not-a-port", " "): + with self.subTest(url=bad), patch.object(public_read_service, "build_ledger_from_env") as ledger: + with self.assertRaisesRegex(public_read_service.PublicReadConfigurationError, "PRISM_PUBLIC_STRATUM_HIGHDIFF_URL"): + public_read_service.build_service({**base, "PRISM_STRATUM_HIGHDIFF_PORT": "4334", "PRISM_PUBLIC_STRATUM_HIGHDIFF_URL": bad}) + ledger.assert_not_called() + self.assertEqual(public_read_service.require_public_stratum_url( + {**base, "PRISM_PUBLIC_STRATUM_HIGHDIFF_URL": bad}), base["PRISM_PUBLIC_STRATUM_URL"]) + for explicit in ("", "stratum+ssl://[2001:db8::1]:443"): + self.assertEqual(public_read_service.require_public_stratum_url( + {**base, "PRISM_STRATUM_HIGHDIFF_PORT": "4334", "PRISM_PUBLIC_STRATUM_HIGHDIFF_URL": explicit}), base["PRISM_PUBLIC_STRATUM_URL"]) + + def test_ipv6_and_tls_stratum_urls_are_accepted(self) -> None: + for url in ("stratum+tcp://[2001:db8::1]:3340", "stratum+ssl://pool.example:443"): + self.assertEqual(public_read_service.require_public_stratum_url( + {"PRISM_PUBLIC_STRATUM_URL": url}), url) + + def test_memory_ledger_is_refused(self) -> None: + with self.assertRaises( + public_read_service.PublicReadConfigurationError + ) as raised: + public_read_service.build_ledger_from_env( + { + "PRISM_ALLOW_MEMORY_LEDGER": "1", + "PRISM_DATABASE_URL": "postgresql:///prism", + } + ) + message = str(raised.exception) + self.assertIn("PRISM_ALLOW_MEMORY_LEDGER", message) + # The reason matters as much as the refusal: the memory ledger would + # reintroduce writer-lock reads through public_api's fallbacks. + self.assertIn("dashboard_", message) + + def test_missing_database_configuration_refuses_startup(self) -> None: + with self.assertRaises( + public_read_service.PublicReadConfigurationError + ) as raised: + public_read_service.build_ledger_from_env({}) + self.assertIn("PRISM_DATABASE_URL", str(raised.exception)) + + def test_build_service_refuses_before_touching_the_database(self) -> None: + # The stratum-url check must come first, so a service missing both + # reports the configuration error rather than a connection failure. + with self.assertRaises( + public_read_service.PublicReadConfigurationError + ) as raised: + public_read_service.build_service({}) + self.assertIn("PRISM_PUBLIC_STRATUM_URL", str(raised.exception)) + + def test_the_stand_in_defines_no_stratum_endpoint_attributes(self) -> None: + # public_api.mining_configuration reads getattr(coordinator, "port"/ + # "bind") with an environment fallback. The stand-in must not define + # them, or the fallback -- and with it the required-env check -- would + # be dead code. + coordinator = public_read_service.PublicReadCoordinator( + ledger=object(), + rpc=object(), + ) + self.assertFalse(hasattr(coordinator, "port")) + self.assertFalse(hasattr(coordinator, "bind")) + + +class ReadinessAndMetricsTests(unittest.TestCase): + """The service's own liveness and counters, which are new rather than moved.""" + + def test_healthz_reports_ready_when_the_probe_succeeds(self) -> None: + probe = public_read_service.ReadinessProbe(lambda: True, interval_seconds=60) + probe.check_once() + harness = ServiceHarness(FakeCoordinator(), readiness=probe) + self.addCleanup(harness.close) + + served = harness.get("/healthz") + + self.assertEqual(200, served.status) + self.assertTrue(served.payload["ok"]) + + def test_healthz_reports_unready_when_postgres_is_unreachable(self) -> None: + def failing_probe() -> bool: + raise RuntimeError("connection refused") + + probe = public_read_service.ReadinessProbe(failing_probe, interval_seconds=60) + probe.check_once() + harness = ServiceHarness(FakeCoordinator(), readiness=probe) + self.addCleanup(harness.close) + + served = harness.get("/healthz") + + self.assertEqual(503, served.status) + self.assertFalse(served.payload["ok"]) + self.assertIn("connection refused", str(served.payload["error"])) + + def test_healthz_refuses_a_stale_probe(self) -> None: + clock = [1000.0] + probe = public_read_service.ReadinessProbe( + lambda: True, + interval_seconds=5, + monotonic=lambda: clock[0], + ) + probe.check_once() + clock[0] += probe.stale_after_seconds + 1 + harness = ServiceHarness(FakeCoordinator(), readiness=probe) + self.addCleanup(harness.close) + + served = harness.get("/healthz") + + self.assertEqual(503, served.status) + self.assertIn("stale", str(served.payload["error"])) + + def test_healthz_does_not_query_on_the_request_thread(self) -> None: + # The compose healthcheck polls this forever; a database round trip per + # poll would make liveness checking a source of load. + calls: list[int] = [] + probe = public_read_service.ReadinessProbe( + lambda: (calls.append(1), True)[1], + interval_seconds=3600, + ) + probe.check_once() + harness = ServiceHarness(FakeCoordinator(), readiness=probe) + self.addCleanup(harness.close) + + for _ in range(5): + self.assertEqual(200, harness.get("/healthz").status) + + self.assertEqual(1, len(calls)) + + def test_metrics_are_prometheus_text_for_this_process(self) -> None: + harness = ServiceHarness(FakeCoordinator()) + self.addCleanup(harness.close) + harness.get("/public/v1/blocks") + + served = harness.get("/metrics") + + self.assertEqual(200, served.status) + self.assertTrue( + served.headers.get("Content-Type", "").startswith("text/plain") + ) + body = served.body.decode() + for metric in ( + "qbit_prism_public_requests_total", + "qbit_prism_public_responses_total", + "qbit_prism_public_cache_total", + "qbit_prism_public_staleness_refusals_total", + ): + self.assertIn(metric, body) + + def test_cache_hits_and_misses_are_counted(self) -> None: + harness = ServiceHarness(FakeCoordinator()) + self.addCleanup(harness.close) + harness.get("/public/v1/blocks") + harness.get("/public/v1/blocks") + + body = harness.get("/metrics").body.decode() + + self.assertIn('qbit_prism_public_cache_total{state="miss"} 1', body) + self.assertIn('qbit_prism_public_cache_total{state="hit"} 1', body) + + def test_unknown_paths_outside_the_public_surface_are_plain_404s(self) -> None: + harness = ServiceHarness(FakeCoordinator()) + self.addCleanup(harness.close) + + served = harness.get("/audit/latest") + + self.assertEqual(404, served.status) + self.assertEqual({"error": "unknown endpoint"}, served.payload) + + +class _MinimalAuditPort: + """Answers nothing: an unknown path never reaches a port method.""" + + def ledger_backend(self) -> str: + return "fixture" + + +class CoordinatorSubtractionTests(unittest.TestCase): + """The coordinator's listener no longer serves the public surface.""" + + @classmethod + def setUpClass(cls) -> None: + cls.facade = AuditHttpFacade( + _MinimalAuditPort(), # type: ignore[arg-type] + AuditHttpConfig("127.0.0.1", 0, join_timeout_seconds=1.0), + ) + state = cls.facade.start() + assert state.bound_address is not None + cls.host, cls.port = state.bound_address + + @classmethod + def tearDownClass(cls) -> None: + cls.facade.stop() + + def get(self, path: str) -> tuple[int, object]: + url = f"http://{self.host}:{self.port}{path}" + try: + with urllib.request.urlopen(url, timeout=5) as response: + return response.status, json.loads(response.read()) + except urllib.error.HTTPError as error: + with error: + return error.code, json.loads(error.read()) + + def test_public_routes_are_unknown_endpoints_on_the_coordinator(self) -> None: + for path in ( + "/public/v1/pool-summary", + "/public/v1/blocks", + f"/public/v1/miners/{RECIPIENT_ID}", + "/public/v1", + ): + with self.subTest(path=path): + status, body = self.get(path) + self.assertEqual(404, status) + self.assertEqual({"error": "unknown endpoint"}, body) + + def test_audit_http_no_longer_imports_public_api(self) -> None: + # The public error schema and cache policy left with the routes. An + # import here would mean some public handling stayed behind. + source = AUDIT_HTTP_PATH.read_text(encoding="utf-8") + self.assertNotIn("import public_api", source) + self.assertNotIn("public_api.", source) + + def test_the_audit_http_port_no_longer_exposes_public_payload(self) -> None: + from lab.prism import audit_http + + self.assertFalse(hasattr(audit_http.AuditHttpPort, "public_payload")) + + +class DirectCoinbaseSettlementContractTests(unittest.TestCase): + """The severed writer-lock settlement read must answer identically.""" + + def test_read_model_and_audit_bundle_fallback_agree(self) -> None: + # The read model added in share_ledger returns exactly what + # public_api.direct_coinbase_settlement_payload built from audit_bundle. + fallback_ledger = DirectCoinbasePublicLedger() + expected = public_api.direct_coinbase_settlement_payload( + fallback_ledger, + block_hash=DirectCoinbasePublicLedger.block_hash, + ) + + class ReadModelLedger(DirectCoinbasePublicLedger): + def dashboard_direct_coinbase_settlement( + self, + *, + block_hash: str, + ) -> dict[str, object] | None: + # Stand in for the Postgres read-slot implementation by + # reusing the same source row the fenced path would have read. + return expected + + def audit_bundle(self, *, block_hash: str) -> dict[str, object] | None: + raise WriterLockReached("audit_bundle() is a writer-lock read") + + actual = public_api.direct_coinbase_settlement_payload( + ReadModelLedger(), + block_hash=DirectCoinbasePublicLedger.block_hash, + ) + + self.assertEqual(expected, actual) + self.assertEqual( + { + "block_hash", + "block_height", + "settlement_mode", + "audit_bundle_sha256", + "payout_manifest_sha256", + "artifacts", + }, + set(actual or {}), + ) + + def test_owed_balance_read_model_matches_the_python_sum(self) -> None: + balances = [ + {"recipient_id": "miner-a", "balance_sats": 700}, + {"recipient_id": "miner-b", "balance_sats": 11}, + {"recipient_id": "miner-a", "balance_sats": 300}, + ] + + class SummingLedger: + def current_owed_balances(self) -> list[dict[str, object]]: + return balances + + class ReadModelLedger: + def dashboard_miner_owed_balance_bits(self, *, recipient_id: str) -> int: + return sum( + int(balance["balance_sats"]) + for balance in balances + if balance["recipient_id"] == recipient_id + ) + + def current_owed_balances(self) -> list[dict[str, object]]: + raise WriterLockReached("current_owed_balances() is a writer-lock read") + + self.assertEqual( + public_api.owed_balance_for_recipient(SummingLedger(), "miner-a"), + public_api.owed_balance_for_recipient(ReadModelLedger(), "miner-a"), + ) + self.assertEqual( + 1000, + public_api.owed_balance_for_recipient(ReadModelLedger(), "miner-a"), + ) + + def test_owed_balance_falls_back_for_a_ledger_without_the_read_model(self) -> None: + # The in-memory ledger has no dashboard_* read models; the fallback + # must stay for it. + class MemoryStyleLedger: + def current_owed_balances(self) -> list[dict[str, object]]: + return [{"recipient_id": "miner-a", "balance_sats": 42}] + + self.assertEqual( + 42, + public_api.owed_balance_for_recipient(MemoryStyleLedger(), "miner-a"), + ) + + +class StubbedReadSlotLedger(PsqlShareLedger): + """A PsqlShareLedger with a real writer lock and a canned SQL backend. + + Bypasses __init__ (as the other share-ledger tests do) so the gates under + test are real threading primitives while no database is involved: the + property being asserted is which gate a method takes, and that is decided + entirely by the Python code around the statement. + """ + + def __init__(self, results: dict[str, object]) -> None: + self._lock = threading.Lock() + self._read_semaphore = threading.BoundedSemaphore(4) + self._results = results + self.statements: list[str] = [] + + def _run_json(self, sql: str) -> object: + self.statements.append(sql) + for marker, result in self._results.items(): + if marker in sql: + return result + raise AssertionError(f"unexpected statement: {sql[:120]}") + + +OWED_BALANCE_SQL_RESULT = {"qbit_current_owed_balances": {"owed_balance_bits": "1000"}} +SETTLEMENT_SQL_RESULT = { + "qbit_pool_audit_bundles": { + "block_hash": "ab" * 32, + "block_height": 23342, + "payout_manifest_sha256": "8c" * 32, + "audit_bundle_sha256": "92" * 32, + "audit_bundle": { + "schema": "qbit.prism.audit-bundle.v1", + "found_block": {"block_height": 23342}, + "settlement_mode_decision": {"mode": "direct_coinbase"}, + }, + "body_uri": None, + } +} + + +class ReadSlotNotWriterLockTests(unittest.TestCase): + """The severed reads must take the read slot, structurally. + + Asserted by holding the writer lock from another thread for the whole + call: a method that fences would block until the holder released, so a + call that returns under a held lock cannot have taken it. The two + control cases at the end show the same setup does block the reads these + replaced, so a passing assertion here is not vacuous. + """ + + HOLD_SECONDS = 30.0 + RETURN_TIMEOUT = 5.0 + + def call_with_writer_lock_held(self, ledger: PsqlShareLedger, call) -> object: + """Run call() while another thread holds ledger's writer lock.""" + + acquired = threading.Event() + release = threading.Event() + + def hold() -> None: + with ledger._lock: + acquired.set() + release.wait(self.HOLD_SECONDS) + + holder = threading.Thread(target=hold, name="writer-lock-holder", daemon=True) + holder.start() + self.assertTrue(acquired.wait(5.0), "writer lock holder never started") + try: + result: list[object] = [] + error: list[BaseException] = [] + + def run() -> None: + try: + result.append(call()) + except BaseException as exc: # noqa: BLE001 - reported below + error.append(exc) + + worker = threading.Thread(target=run, name="read-under-lock", daemon=True) + worker.start() + worker.join(timeout=self.RETURN_TIMEOUT) + returned = not worker.is_alive() + finally: + release.set() + holder.join(timeout=5.0) + if error: + raise error[0] + if not returned: + raise AssertionError("call did not return while the writer lock was held") + return result[0] + + def blocks_while_writer_lock_held(self, ledger: PsqlShareLedger, call) -> bool: + try: + self.call_with_writer_lock_held(ledger, call) + except AssertionError as exc: + return "did not return" in str(exc) + return False + + def test_owed_balance_read_model_returns_under_a_held_writer_lock(self) -> None: + ledger = StubbedReadSlotLedger(OWED_BALANCE_SQL_RESULT) + + value = self.call_with_writer_lock_held( + ledger, + lambda: ledger.dashboard_miner_owed_balance_bits(recipient_id="miner-a"), + ) + + self.assertEqual(1000, value) + + def test_settlement_read_model_returns_under_a_held_writer_lock(self) -> None: + ledger = StubbedReadSlotLedger(SETTLEMENT_SQL_RESULT) + + payload = self.call_with_writer_lock_held( + ledger, + lambda: ledger.dashboard_direct_coinbase_settlement(block_hash="ab" * 32), + ) + + assert payload is not None + self.assertEqual("direct_coinbase", payload["settlement_mode"]) + self.assertEqual(23342, payload["block_height"]) + + def test_owed_balance_sql_filters_by_recipient_rather_than_in_python(self) -> None: + ledger = StubbedReadSlotLedger(OWED_BALANCE_SQL_RESULT) + + ledger.dashboard_miner_owed_balance_bits(recipient_id="miner-a") + + statement = ledger.statements[0] + self.assertIn("qbit_current_owed_balances()", statement) + # The same two predicates the Python sum applied, now in SQL. + self.assertIn("owed_balance_sats > 0", statement) + self.assertIn("miner_id = 'miner-a'", statement) + + def test_a_quoted_recipient_id_cannot_escape_its_literal(self) -> None: + ledger = StubbedReadSlotLedger(OWED_BALANCE_SQL_RESULT) + + ledger.dashboard_miner_owed_balance_bits(recipient_id="mi'ner") + + self.assertIn("miner_id = 'mi''ner'", ledger.statements[0]) + + def test_an_empty_recipient_id_is_rejected(self) -> None: + ledger = StubbedReadSlotLedger(OWED_BALANCE_SQL_RESULT) + + with self.assertRaises(ValueError): + ledger.dashboard_miner_owed_balance_bits(recipient_id="") + + def test_settlement_read_model_reports_non_direct_coinbase_as_none(self) -> None: + ledger = StubbedReadSlotLedger( + { + "qbit_pool_audit_bundles": { + **SETTLEMENT_SQL_RESULT["qbit_pool_audit_bundles"], + "audit_bundle": { + "settlement_mode_decision": { + "mode": "hybrid_coinbase_ctv_fanout" + } + }, + } + } + ) + + self.assertIsNone( + ledger.dashboard_direct_coinbase_settlement(block_hash="ab" * 32) + ) + + def test_settlement_read_model_reports_a_missing_bundle_as_none(self) -> None: + ledger = StubbedReadSlotLedger({"qbit_pool_audit_bundles": None}) + + self.assertIsNone( + ledger.dashboard_direct_coinbase_settlement(block_hash="ab" * 32) + ) + + def test_an_unreadable_external_body_is_none_rather_than_an_error(self) -> None: + # public_api.audit_bundle_body_read_failed() tolerated exactly these + # three failures on the fenced path; the read model must keep doing so. + ledger = StubbedReadSlotLedger( + { + "qbit_pool_audit_bundles": { + **SETTLEMENT_SQL_RESULT["qbit_pool_audit_bundles"], + "audit_bundle": None, + "body_uri": "/nonexistent/prism-audit-bundle-body.json", + } + } + ) + + self.assertIsNone( + ledger.dashboard_direct_coinbase_settlement(block_hash="ab" * 32) + ) + + def test_the_replaced_reads_really_do_block_on_the_writer_lock(self) -> None: + # The control: without this, the assertions above could pass because + # nothing in the harness ever takes the writer lock at all. + owed = StubbedReadSlotLedger( + {"qbit_current_owed_balances": [], "count(*)": {"count": 0}} + ) + self.assertTrue( + self.blocks_while_writer_lock_held(owed, owed.current_owed_balances), + "current_owed_balances() no longer fences, so the contrast is void", + ) + + bundle = StubbedReadSlotLedger(SETTLEMENT_SQL_RESULT) + self.assertTrue( + self.blocks_while_writer_lock_held( + bundle, + lambda: bundle.audit_bundle(block_hash="ab" * 32), + ), + "audit_bundle() no longer fences, so the contrast is void", + ) + + def test_a_read_only_ledger_cannot_take_the_writer_lock_at_all(self) -> None: + ledger = PsqlShareLedger( + psql_command="psql postgresql://example.invalid/qbit", + native_client_mode="psql", + read_only=True, + ) + + with self.assertRaises(ReadOnlyLedgerError): + ledger.current_owed_balances() + with self.assertRaises(ReadOnlyLedgerError): + ledger.audit_bundle(block_hash="ab" * 32) + + def test_a_read_only_ledger_refuses_schema_initialization(self) -> None: + with self.assertRaises(ValueError): + PsqlShareLedger( + psql_command="psql postgresql://example.invalid/qbit", + native_client_mode="psql", + read_only=True, + initialize_schema=True, + ) + + +class _GatedLedger(PsqlShareLedger): + """A PsqlShareLedger with real gates and a stubbed SQL layer. + + Built without __init__ (the pattern tests/test_prism_public_dashboard_api.py + already uses) so the writer lock and the read semaphore are the genuine + objects while no database is involved. + """ + + def __init__(self) -> None: + self._lock = Lock() + self._read_semaphore = BoundedSemaphore(4) + self.statements: list[str] = [] + self.row: object = None + + def _text_literal(self, value: object) -> str: + return "'" + str(value).replace("'", "''") + "'" + + def _run_retry_safe_read_json(self, sql: str) -> object: + self.statements.append(sql) + return self.row + + +class WriterLockIsNotTakenTests(unittest.TestCase): + """Structural proof: the new read models never touch the writer gate. + + Each call is driven with the writer lock already held by another thread. A + method that fenced would block there until the test timed out; these must + return, because they take the bounded read slot instead. + """ + + def hold_writer_lock(self, ledger: _GatedLedger) -> None: + acquired = threading.Event() + release = threading.Event() + + def holder() -> None: + ledger._lock.acquire() + acquired.set() + release.wait(10) + ledger._lock.release() + + thread = threading.Thread(target=holder, daemon=True) + thread.start() + self.assertTrue(acquired.wait(5), "helper thread never took the writer lock") + self.addCleanup(thread.join, 5) + self.addCleanup(release.set) + + def call_with_timeout(self, call) -> object: + result: list[object] = [] + error: list[BaseException] = [] + + def run() -> None: + try: + result.append(call()) + except BaseException as exc: # noqa: BLE001 - reported below + error.append(exc) + + thread = threading.Thread(target=run, daemon=True) + thread.start() + thread.join(10) + self.assertFalse( + thread.is_alive(), + "the call blocked on the writer lock instead of taking a read slot", + ) + if error: + raise error[0] + return result[0] + + def test_owed_balance_read_model_does_not_fence(self) -> None: + ledger = _GatedLedger() + ledger.row = {"owed_balance_bits": "1000"} + self.hold_writer_lock(ledger) + + balance = self.call_with_timeout( + lambda: ledger.dashboard_miner_owed_balance_bits(recipient_id=RECIPIENT_ID) + ) + + self.assertEqual(1000, balance) + self.assertEqual(1, len(ledger.statements)) + # Filtered in SQL, not in Python: the point is to stop reading every + # recipient's balance to answer for one. + self.assertIn("miner_id", ledger.statements[0]) + self.assertIn(RECIPIENT_ID, ledger.statements[0]) + self.assertIn("owed_balance_sats > 0", ledger.statements[0]) + + def test_owed_balance_read_model_returns_zero_without_rows(self) -> None: + ledger = _GatedLedger() + ledger.row = {"owed_balance_bits": "0"} + self.hold_writer_lock(ledger) + + self.assertEqual( + 0, + self.call_with_timeout( + lambda: ledger.dashboard_miner_owed_balance_bits( + recipient_id="nobody" + ) + ), + ) + + def test_direct_coinbase_settlement_read_model_does_not_fence(self) -> None: + ledger = _GatedLedger() + ledger.row = { + "block_hash": DirectCoinbasePublicLedger.block_hash, + "block_height": DirectCoinbasePublicLedger.block_height, + "payout_manifest_sha256": DirectCoinbasePublicLedger.payout_manifest_sha256, + "audit_bundle_sha256": DirectCoinbasePublicLedger.audit_bundle_sha256, + "audit_bundle": { + "schema": "qbit.prism.audit-bundle.v1", + "found_block": {"block_height": DirectCoinbasePublicLedger.block_height}, + "settlement_mode_decision": {"mode": "direct_coinbase"}, + }, + "body_uri": None, + } + self.hold_writer_lock(ledger) + + payload = self.call_with_timeout( + lambda: ledger.dashboard_direct_coinbase_settlement( + block_hash=DirectCoinbasePublicLedger.block_hash + ) + ) + + # Identical to what the audit_bundle() path builds for the same row. + expected = public_api.direct_coinbase_settlement_payload( + DirectCoinbasePublicLedger(), + block_hash=DirectCoinbasePublicLedger.block_hash, + ) + self.assertEqual(expected, payload) + + def test_direct_coinbase_settlement_ignores_other_settlement_modes(self) -> None: + ledger = _GatedLedger() + ledger.row = { + "block_hash": DirectCoinbasePublicLedger.block_hash, + "block_height": 1, + "payout_manifest_sha256": None, + "audit_bundle_sha256": None, + "audit_bundle": { + "settlement_mode_decision": {"mode": "ctv_fanout"}, + }, + "body_uri": None, + } + self.hold_writer_lock(ledger) + + self.assertIsNone( + self.call_with_timeout( + lambda: ledger.dashboard_direct_coinbase_settlement( + block_hash=DirectCoinbasePublicLedger.block_hash + ) + ) + ) + + def test_direct_coinbase_settlement_is_none_without_a_bundle(self) -> None: + ledger = _GatedLedger() + ledger.row = None + self.hold_writer_lock(ledger) + + self.assertIsNone( + self.call_with_timeout( + lambda: ledger.dashboard_direct_coinbase_settlement( + block_hash=DirectCoinbasePublicLedger.block_hash + ) + ) + ) + + def test_an_unreadable_external_body_is_reported_as_none(self) -> None: + # public_api.audit_bundle_body_read_failed() tolerates exactly these + # three; the read model must keep that behaviour rather than 500. + for message in ( + "audit bundle body is not retrievable", + "audit bundle body hash mismatch", + "audit bundle body is not valid JSON", + ): + with self.subTest(message=message): + ledger = _GatedLedger() + ledger.row = { + "block_hash": DirectCoinbasePublicLedger.block_hash, + "audit_bundle_sha256": DirectCoinbasePublicLedger.audit_bundle_sha256, + "audit_bundle": None, + "body_uri": "file:///missing", + } + + def explode(_body_uri: object, *, expected_sha256: object = None) -> object: + raise RuntimeError(message) + + ledger._read_external_body = explode # type: ignore[assignment] + + self.assertIsNone( + ledger.dashboard_direct_coinbase_settlement( + block_hash=DirectCoinbasePublicLedger.block_hash + ) + ) + + def test_an_unexpected_body_error_still_raises(self) -> None: + ledger = _GatedLedger() + ledger.row = { + "block_hash": DirectCoinbasePublicLedger.block_hash, + "audit_bundle_sha256": DirectCoinbasePublicLedger.audit_bundle_sha256, + "audit_bundle": None, + "body_uri": "file:///missing", + } + + def explode(_body_uri: object, *, expected_sha256: object = None) -> object: + raise RuntimeError("postgres connection lost") + + ledger._read_external_body = explode # type: ignore[assignment] + + with self.assertRaises(RuntimeError): + ledger.dashboard_direct_coinbase_settlement( + block_hash=DirectCoinbasePublicLedger.block_hash + ) + + def test_the_readiness_probe_does_not_fence(self) -> None: + ledger = _GatedLedger() + ledger.row = {"ok": True} + self.hold_writer_lock(ledger) + + self.assertTrue( + self.call_with_timeout(ledger.dashboard_readiness_probe) + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_prism_public_refresh_limits.py b/tests/test_prism_public_refresh_limits.py new file mode 100644 index 000000000..38e88a9a8 --- /dev/null +++ b/tests/test_prism_public_refresh_limits.py @@ -0,0 +1,76 @@ +"""Global background refresh admission across cache keys and instances.""" + +import threading +import time +import unittest +from unittest.mock import patch + +from lab.prism import public_api + + +class PublicRefreshLimitsTests(unittest.TestCase): + def test_saturated_refreshes_serve_stale_without_registering_waiters(self): + slots = threading.BoundedSemaphore(1) + entered, release = threading.Event(), threading.Event() + first, second = public_api.PublicResponseCache(), public_api.PublicResponseCache() + key_a, key_b = ("/first", ()), ("/second", ()) + for cache, key in ((first, key_a), (second, key_b)): + cache.get_or_compute(key=key, ttl_seconds=5, compute=lambda: (200, {"old": True})) + cache._entries[key].stored_at = time.monotonic() - 6 + cache._entries[key].expires_at = time.monotonic() - 1 + + def blocked(): + entered.set() + self.assertTrue(release.wait(5)) + return 200, {"new": True} + + def stale(cache, key, compute): + return cache.get_or_compute( + key=key, ttl_seconds=5, stale_while_revalidate_seconds=30, compute=compute, + ) + + with patch.object(public_api, "_PUBLIC_REFRESH_SLOTS", slots): + try: + self.assertEqual(stale(first, key_a, blocked)[2], "STALE") + self.assertTrue(entered.wait(2)) + with patch.object(threading.Thread, "start") as start: + for _ in range(20): + self.assertEqual(stale(second, key_b, lambda: self.fail("unadmitted refresh"))[2], "STALE") + start.assert_not_called() + self.assertNotIn(key_b, second._inflight) + finally: + release.set() + self.assertTrue(slots.acquire(timeout=5)) + slots.release() + # Once capacity returns, the previously deferred key can refresh. + done = threading.Event() + + def recovered(): + done.set() + return 200, {"recovered": True} + + self.assertEqual(stale(second, key_b, recovered)[2], "STALE") + self.assertTrue(done.wait(2)) + self.assertTrue(slots.acquire(timeout=5)) + slots.release() + + def test_start_failure_and_compute_failure_both_return_capacity(self): + slots = threading.BoundedSemaphore(1) + cache = public_api.PublicResponseCache() + key = ("/failure", ()) + cache.get_or_compute(key=key, ttl_seconds=5, compute=lambda: (200, {})) + cache._entries[key].expires_at = time.monotonic() - 1 + + def fail(): + raise RuntimeError("refresh failed") + + with patch.object(public_api, "_PUBLIC_REFRESH_SLOTS", slots): + with patch.object(threading.Thread, "start", side_effect=RuntimeError("start failed")): + cache.get_or_compute(key=key, ttl_seconds=5, stale_while_revalidate_seconds=30, compute=fail) + self.assertTrue(slots.acquire(timeout=2)) + slots.release() + self.assertNotIn(key, cache._inflight) + cache.get_or_compute(key=key, ttl_seconds=5, stale_while_revalidate_seconds=30, compute=fail) + self.assertTrue(slots.acquire(timeout=2)) + slots.release() + self.assertNotIn(key, cache._inflight) diff --git a/tests/test_prism_reconnect_backpressure.py b/tests/test_prism_reconnect_backpressure.py index 865fa0758..5f6631dc1 100644 --- a/tests/test_prism_reconnect_backpressure.py +++ b/tests/test_prism_reconnect_backpressure.py @@ -13,12 +13,14 @@ from unittest.mock import patch from lab.auxpow import vardiff -from lab.prism.prism_coordinator import ( - ClientState, - PendingInitialJob, +from lab.prism.job_delivery import ( PRISM_DELIVERY_PRIORITY_INITIAL, PRISM_DELIVERY_PRIORITY_NEW_TIP, PRISM_DELIVERY_PRIORITY_SAME_TIP, +) +from lab.prism.prism_coordinator import ( + ClientState, + PendingInitialJob, PrismCoordinator, StratumError, StratumListenerProfile, @@ -295,6 +297,14 @@ def test_tip_churn_with_successful_deliveries_does_not_inherit_old_gap(self) -> payout_state_generation=0, ) server.note_tip_work_delivered(state, "bb" * 32) + # The health refresh owns the fleet coverage census; delivery no + # longer repeats it per job (#159). The refresh that follows the + # delivery observes restored coverage and closes the gap, so the next + # tip's gap starts fresh instead of inheriting this one. + restored = server.mining_delivery_snapshot(now=101.0) + self.assertTrue(restored["mining_ready"]) + self.assertEqual(restored["current_tip_coverage_gap_age_seconds"], 0.0) + server.current_tip_first_seen = ("cc" * 32, 106.0) second_gap = server.mining_delivery_snapshot(now=106.0) self.assertTrue(second_gap["mining_ready"]) @@ -305,6 +315,10 @@ def test_tip_churn_with_successful_deliveries_does_not_inherit_old_gap(self) -> payout_state_generation=0, ) server.note_tip_work_delivered(state, "cc" * 32) + recovered = server.mining_delivery_snapshot(now=107.0) + self.assertTrue(recovered["mining_ready"]) + self.assertEqual(recovered["current_tip_coverage_gap_age_seconds"], 0.0) + server.current_tip_first_seen = ("dd" * 32, 112.0) third_gap = server.mining_delivery_snapshot(now=112.0) self.assertTrue(third_gap["mining_ready"]) @@ -560,7 +574,7 @@ def test_expired_initial_request_is_fenced_before_preparation(self) -> None: requested_monotonic=time.monotonic() - 2, deadline_monotonic=time.monotonic() - 1, ) - server.pending_initial_jobs[state] = request + server.pending_initial_jobs = {state: request} server.ensure_reorg_reconciled_for_current_tip = ( # type: ignore[method-assign] lambda: preparation_started.set() or True ) @@ -594,7 +608,7 @@ def test_reauthorize_schedules_current_work_after_stale_direct_delivery(self) -> requested_monotonic=0.0, deadline_monotonic=30.0, ) - server.pending_initial_jobs[state] = old_request + server.pending_initial_jobs = {state: old_request} release = threading.Event() server._run_initial_job = lambda _request: release.wait(5) # type: ignore[method-assign] server.resolve_worker = lambda username: worker(username) # type: ignore[method-assign] @@ -718,7 +732,7 @@ def test_timeout_commits_closing_before_disconnect_handoff(self) -> None: requested_monotonic=0.0, deadline_monotonic=1.0, ) - server.pending_initial_jobs[state] = request + server.pending_initial_jobs = {state: request} disconnect_client = server.disconnect_client rescheduled: list[bool] = [] @@ -748,7 +762,7 @@ def test_overload_rejection_never_evicts_current_work(self) -> None: requested_monotonic=10.0, deadline_monotonic=40.0, ) - server.pending_initial_jobs[waiting] = request + server.pending_initial_jobs = {waiting: request} self.assertFalse(server.schedule_initial_job(excess)) @@ -1083,14 +1097,16 @@ def test_health_grace_failure_and_recovery_follow_job_coverage(self) -> None: server = coordinator(connection_limit=2, pending_limit=2) server.started_monotonic = 0.0 first, second = client(server, 1), client(server, 2) - for state in (first, second): - server.pending_initial_jobs[state] = PendingInitialJob( + server.pending_initial_jobs = { + state: PendingInitialJob( client=state, authorization_generation=1, worker=state.worker, requested_monotonic=0.0, deadline_monotonic=30.0, ) + for state in (first, second) + } startup = server.mining_delivery_snapshot(now=10.0) failed = server.mining_delivery_snapshot(now=31.0) diff --git a/tests/test_prism_refresh_retry_pacing.py b/tests/test_prism_refresh_retry_pacing.py index 259262c3a..383a881a7 100644 --- a/tests/test_prism_refresh_retry_pacing.py +++ b/tests/test_prism_refresh_retry_pacing.py @@ -16,7 +16,7 @@ from unittest.mock import patch from lab.prism.prism_coordinator import PrismCoordinator, TemplateRefreshBlocked -from tests.test_prism_coordinator_job_cache import ( +from tests.prism_coordinator_test_support import ( base_template, client, coordinator, diff --git a/tests/test_prism_reorg_reconciler.py b/tests/test_prism_reorg_reconciler.py new file mode 100644 index 000000000..78a14c34d --- /dev/null +++ b/tests/test_prism_reorg_reconciler.py @@ -0,0 +1,633 @@ +from __future__ import annotations + +import inspect +import threading +import time +import unittest +from unittest import mock +from contextlib import contextmanager, nullcontext +from dataclasses import replace as dataclass_replace +from types import SimpleNamespace + +from lab.prism.prism_coordinator import PrismCoordinator +from lab.prism.reorg_reconciler import ( + STRANDED_PREPARED_REJECT_MIN_DEPTH, + STRANDED_PREPARED_SWEEP_LIMIT, + ReorgPorts, + ReorgReconcilerService, + qbit_chain_view_untrusted, +) +from tests import prism_vardiff_test_support as support + + +class FakeLedger: + def __init__(self) -> None: + self.inactivated: list[str] = [] + self.reactivated: list[str] = [] + + def reorg_watch_blocks(self, *, active_tip_height: int) -> list[dict[str, object]]: + assert active_tip_height == 10 + return [ + {"block_height": 8, "block_hash": "aa", "chain_state": "confirmed"}, + {"block_height": 9, "block_hash": "bb", "chain_state": "inactive"}, + ] + + def mark_pool_block_inactive(self, **kwargs: object) -> dict[str, int]: + self.inactivated.append(str(kwargs["block_hash"])) + return {"inactive_count": 1} + + def reactivate_pool_block(self, **kwargs: object) -> dict[str, int]: + self.reactivated.append(str(kwargs["block_hash"])) + return {"reactivated_count": 1} + + def mark_mature_pool_payouts(self, **_kwargs: object) -> dict[str, int]: + return {"matured_count": 2} + + +def make_ports( + *, + ledger: object | None = None, + publish: object = 7, + chain_untrusted: bool = False, + ensure_tip: object = None, + tip_height: int = 10, + block_hashes: dict[int, str] | None = None, +) -> tuple[ReorgPorts, list[tuple[str, object]]]: + events: list[tuple[str, object]] = [] + active_ledger = FakeLedger() if ledger is None else ledger + state_lock = threading.RLock() + active_chain = {8: "not-aa", 9: "bb"} if block_hashes is None else block_hashes + + def rpc_call(method: str, params: object = None) -> object: + if method == "getbestblockhash": + return "tip" + if method == "getblockcount": + return tip_height + if method == "getblockhash": + height = int(params[0]) # type: ignore[index] + events.append(("getblockhash", height)) + return active_chain[height] + raise AssertionError(method) + + @contextmanager + def publication_guard(): + events.append(("guard", "enter")) + try: + yield + finally: + events.append(("guard", "exit")) + + def prepared_candidate(captured, **kwargs): + events.append(("prepare", kwargs)) + return SimpleNamespace(captured=captured) + + return ( + ReorgPorts( + rpc_call=rpc_call, + ledger=lambda: active_ledger, + ensure_job_cache_state=lambda: events.append(("ensure", None)), + state_lock=lambda: state_lock, + source_tip=lambda: "tip", + reserve_external_tip=lambda tip: events.append(("reserve", tip)), + max_supersession_retries=lambda: 1, + prepare_lock=nullcontext, + capture_source=lambda: (0, 0, "tip", "test", 0.0), + prepared_candidate=prepared_candidate, + captured_publication_required=lambda _captured: False, + block_publication=lambda **kwargs: events.append(("block", kwargs)), + publication_guard=publication_guard, + publish_candidate=lambda _candidate: publish, # type: ignore[return-value] + observe_preparation=lambda elapsed: events.append(("observe", elapsed)), + chain_view_untrusted=lambda: chain_untrusted, + reorg_proof_snapshot=lambda: ("tip", 0), + flight_wait_seconds=lambda: 30.0, + prefetch_join_timeout_seconds=lambda: 20.0, + reconcile_with_admission=lambda **kwargs: { + "tip": kwargs.get("tip_hash"), + "untrusted": chain_untrusted, + }, + reconcile_serialized=lambda **kwargs: { + "serialized": kwargs, + "untrusted": False, + "superseded": False, + }, + ensure_tip=( + ensure_tip + if ensure_tip is not None + else lambda tip: events.append(("ensure_tip", tip)) or True + ), + ), + events, + ) + + +class ReorgReconcilerServiceTests(unittest.TestCase): + def test_fresh_memo_still_checks_live_chain_trust(self) -> None: + ports, events = make_ports() + service = ReorgReconcilerService(ports, cache_seconds=5.0) + service._reorg_reconcile_trusted_memo["tip"] = time.monotonic() + self.assertTrue(service.ensure_current()) + # The memo satisfied the caller; no serialized pass ran. + self.assertNotIn(("ensure_tip", "tip"), events) + self.assertEqual( + service.reorg_reconcile_lookup_snapshot()[("job_build", "memo_hit")], + 1, + ) + + untrusted_ports, untrusted_events = make_ports(chain_untrusted=True) + untrusted = ReorgReconcilerService(untrusted_ports, cache_seconds=5.0) + untrusted._reorg_reconcile_trusted_memo["tip"] = time.monotonic() + # An arriving reorg (headers ahead of the validated tip) must force + # the full pass immediately, memo freshness notwithstanding. + untrusted.ensure_current() + self.assertIn(("ensure_tip", "tip"), untrusted_events) + self.assertEqual( + untrusted.reorg_reconcile_lookup_snapshot()[("job_build", "serial")], + 1, + ) + + def test_reconcile_owns_chain_mutations_counts_and_publication(self) -> None: + ledger = FakeLedger() + ports, events = make_ports(ledger=ledger) + service = ReorgReconcilerService(ports) + + summary = service.reconcile(tip_hash="tip", force_publish=True) + + self.assertEqual(ledger.inactivated, ["aa"]) + self.assertEqual(ledger.reactivated, ["bb"]) + self.assertEqual(summary["published_generation"], 7) + self.assertEqual(summary["inactive_blocks"], 1) + self.assertEqual(summary["reactivated_blocks"], 1) + self.assertEqual(summary["matured_payouts"], 2) + self.assertIn(("block", {"force": True}), events) + # PR 75 publication-order guard wraps reactivation. + guard_enter = events.index(("guard", "enter")) + guard_exit = events.index(("guard", "exit")) + self.assertLess(guard_enter, guard_exit) + self.assertEqual(ledger.reactivated, ["bb"]) + # A confirmed mutation moves payout balances but never the share + # ledger, so preparation asks for a live prior-balances reread over + # the still-exact window instead of the #112 full rescan (#224). + self.assertIn( + ( + "prepare", + { + "force_full_window_rescan": False, + "force_prior_balances_read": True, + }, + ), + events, + ) + state = service.snapshot() + self.assertEqual(state.inactive_block_count, 1) + self.assertEqual(state.reactivated_block_count, 1) + self.assertEqual(state.matured_payout_count, 2) + self.assertTrue(state.last_trusted) + # A trusted pass for the latest detected tip arms the per-tip memo. + self.assertIn("tip", service._reorg_reconcile_trusted_memo) + + def test_reconcile_stamps_progress_through_its_unbounded_chain_walk(self) -> None: + """One pass is many statements, so one bracket around it is not enough. + + An accepted-block landing runs ``ensure_tip`` inline on the + watchdog-monitored block-work thread. The pass walks every watched + pool block, spending a getblockhash and up to two ledger statements + per row, so its heartbeat-silent span grows with the reorg window. + Bracketing the call from the outside would leave that growth + unreported; the stamps have to follow the work (issue #125). + """ + ledger = FakeLedger() + ports, _events = make_ports(ledger=ledger) + phases: list[str] = [] + service = ReorgReconcilerService( + dataclass_replace(ports, record_progress=phases.append) + ) + + service.reconcile(tip_hash="tip", force_publish=True) + + # FakeLedger watches two blocks; each row gets its own stamp. + self.assertEqual(phases.count("reorg-reconcile:watch-block"), 2) + self.assertEqual( + phases[:3], + [ + "reorg-reconcile:tip-height", + "reorg-reconcile:watch-blocks", + "reorg-reconcile:watch-block", + ], + ) + for phase in ( + "reorg-reconcile:mature-payouts", + "reorg-reconcile:prepare-candidate", + "reorg-reconcile:publish", + ): + self.assertIn(phase, phases) + + def test_reconcile_ports_without_a_recorder_stamp_nothing(self) -> None: + """Background passes and focused tests keep their existing ports.""" + ledger = FakeLedger() + ports, _events = make_ports(ledger=ledger) + summary = ReorgReconcilerService(ports).reconcile( + tip_hash="tip", + force_publish=True, + ) + self.assertEqual(summary["watched_blocks"], 2) + + def test_untrusted_pass_records_skip_and_does_not_arm_memo(self) -> None: + ports, _events = make_ports(chain_untrusted=True) + service = ReorgReconcilerService(ports) + summary = service.reconcile(tip_hash="tip") + self.assertTrue(summary["untrusted"]) + self.assertEqual(service.snapshot().reconcile_skip_count, 1) + self.assertNotIn("tip", service._reorg_reconcile_trusted_memo) + self.assertFalse(service.snapshot().last_trusted) + + def test_stale_proof_epoch_refuses_to_arm_memo(self) -> None: + ports, _events = make_ports() + service = ReorgReconcilerService(ports) + # The pass started in epoch 3; detection has since moved to epoch 0 + # (make_ports pins reorg_proof_snapshot at ("tip", 0)). + service.note_outcome("tip", trusted=True, proof_epoch=3) + self.assertNotIn("tip", service._reorg_reconcile_trusted_memo) + service.note_outcome("tip", trusted=True, proof_epoch=0) + self.assertIn("tip", service._reorg_reconcile_trusted_memo) + + def test_mutating_pass_evicts_other_tips_and_error_clears_memo(self) -> None: + ports, _events = make_ports() + service = ReorgReconcilerService(ports) + service._reorg_reconcile_trusted_memo["other"] = time.monotonic() + service.note_outcome("tip", trusted=True, evict_others=True, proof_epoch=0) + self.assertNotIn("other", service._reorg_reconcile_trusted_memo) + self.assertIn("tip", service._reorg_reconcile_trusted_memo) + service.note_outcome("tip", trusted=False, clear_memo=True) + self.assertEqual(dict(service._reorg_reconcile_trusted_memo), {}) + + def test_flight_admission_runs_before_the_serialized_adapter(self) -> None: + ports, _events = make_ports() + service = ReorgReconcilerService(ports) + summary = service.reconcile_with_flights(tip_hash="tip") + self.assertEqual( + summary["serialized"], + {"tip_hash": "tip"}, + ) + # Forced/source-reserved callers bypass flight registration entirely. + forced = service.reconcile_with_flights(tip_hash="tip", _force_publish=True) + self.assertEqual( + forced["serialized"], + {"tip_hash": "tip", "_force_publish": True, "_source_reserved": False}, + ) + self.assertEqual(service._reconcile_flights, {}) + + def test_proving_prefetch_satisfies_both_memo_pass_neither(self) -> None: + release = threading.Event() + + def blocking_ensure_tip(_tip: str) -> bool: + release.wait(5.0) + return True + + ports, _events = make_ports(ensure_tip=blocking_ensure_tip) + service = ReorgReconcilerService(ports) + try: + memo_pass = service.submit_prefetch("aa" * 32) + self.assertIsNotNone(memo_pass) + # A memo-honoring pending pass may not satisfy a proving caller; + # the prove request replaces it like a tip change. + proving = service.submit_prefetch("aa" * 32, prove=True) + self.assertIsNotNone(proving) + self.assertIsNot(proving, memo_pass) + # A proving pending pass satisfies either kind of caller. + reused = service.submit_prefetch("aa" * 32) + self.assertIs(reused, proving) + reused_proving = service.submit_prefetch("aa" * 32, prove=True) + self.assertIs(reused_proving, proving) + finally: + release.set() + service.shutdown_prefetch_executor() + self.assertIsNone(service.submit_prefetch("bb" * 32)) + + def test_chain_view_validation_fails_closed(self) -> None: + self.assertTrue( + qbit_chain_view_untrusted( + lambda _method: { + "initialblockdownload": False, + "blocks": 9, + "headers": 10, + }, + "main", + ) + ) + with self.assertRaisesRegex(RuntimeError, "non-object"): + qbit_chain_view_untrusted(lambda _method: [], "regtest") + + +class CoordinatorReorgIntegrationTests(unittest.TestCase): + def test_compat_fields_adopt_pre_service_writes_and_route_after(self) -> None: + server = support.coordinator() + # The bare fixture wrote the legacy names before any service exists; + # the descriptors parked them in backing slots. + self.assertIn("_reorg_compat_enabled", server.__dict__) + self.assertNotIn("_reorg_reconciler_service", server.__dict__) + server.reorg_inactive_block_count = 5 + + service = server._ensure_reorg_reconciler_service() + + # Adoption seeds the service and removes every backing key. + self.assertFalse(service.enabled) + self.assertEqual(service.inactive_block_count, 5) + for key in list(server.__dict__): + self.assertFalse(key.startswith("_reorg_compat_"), key) + # Post-service reads and writes route to the single owner. + server.reorg_reconcile_skip_count = 9 + self.assertEqual(service.reconcile_skip_count, 9) + service.matured_payout_count = 4 + self.assertEqual(server.matured_payout_count, 4) + + def test_lookup_counts_property_returns_a_copy(self) -> None: + server = support.coordinator() + server._record_reorg_reconcile_lookup("job_build", "serial") + snapshot = server.reorg_reconcile_lookup_counts + self.assertEqual(snapshot[("job_build", "serial")], 1) + snapshot[("job_build", "serial")] = 99 + self.assertEqual( + server.reorg_reconcile_lookup_counts[("job_build", "serial")], + 1, + ) + + def test_serialized_adapter_keeps_writer_payout_preview_order(self) -> None: + source = inspect.getsource( + PrismCoordinator._reconcile_prism_pool_blocks_serialized + ) + # Writer admission decorates the adapter; the direct payout-balance + # mutation lock and the landed-preview fail-closed check run inside + # it, before the service core. The old _payout_balance_mutation() + # wrapper must not reappear. + self.assertIn("_payout_balance_mutation_lock", source) + self.assertIn("_accepted_block_payout_preview_condition", source) + self.assertIn("PayoutStatePublicationBlocked", source) + self.assertIn("_ensure_reorg_reconciler_service().reconcile(", source) + self.assertNotIn("with self._payout_balance_mutation()", source) + wrappers = getattr( + PrismCoordinator._reconcile_prism_pool_blocks_serialized, + "__wrapped__", + None, + ) + self.assertIsNotNone(wrappers, "writer-admission decorator missing") + + def test_service_wiring_uses_heartbeat_aware_preparation_lock(self) -> None: + source = inspect.getsource( + PrismCoordinator._ensure_reorg_reconciler_service + ) + self.assertIn("_block_submitter_lock", source) + self.assertIn('"payout-state-prepare"', source) + # Live config-override reads stay callable ports. + self.assertIn("reconcile_flight_wait_seconds", source) + self.assertIn("reconcile_prefetch_join_timeout_seconds", source) + # The atomic tip/epoch proof comes from the tip-refresh owner. + self.assertIn("reorg_proof_snapshot", source) + + +if __name__ == "__main__": + unittest.main() + + +class StrandedPreparedLedger: + """A ledger whose only outstanding reorg work is a stranded prepared row. + + ``reorg_watch_blocks`` returns nothing because it structurally cannot see + prepared rows -- that is exactly why the sweep exists. The depth filter + lives here, in the read, so the tests exercise the real constant the + reconciler passes rather than a value they choose themselves. + """ + + def __init__(self, rows: list[dict[str, object]]) -> None: + self.rows = rows + self.requests: list[dict[str, int]] = [] + self.rejected: list[str] = [] + + def reorg_watch_blocks(self, *, active_tip_height: int) -> list[dict[str, object]]: + return [] + + def stranded_prepared_blocks( + self, + *, + active_tip_height: int, + min_depth: int, + limit: int, + ) -> list[dict[str, object]]: + self.requests.append( + { + "active_tip_height": int(active_tip_height), + "min_depth": int(min_depth), + "limit": int(limit), + } + ) + return [ + row + for row in self.rows + if int(row["block_height"]) <= int(active_tip_height) - int(min_depth) + ][:limit] + + def reject_prepared_block( + self, + *, + block_hash: str, + active_tip_height: int, + ) -> dict[str, int]: + self.rejected.append(str(block_hash)) + return {"rejected_count": 1} + + +class StrandedPreparedSweepTests(unittest.TestCase): + """The reconciler's heal for prepared rows that lost their owner. + + A prepared pool block is resolved by the live submit/replay path that + holds its outbox row. When that row is gone -- quarantined, or completed + by a process that died before confirming -- nothing re-examines the + ledger row again: reorg_watch_blocks selects confirmed/inactive only. + Testnet4 carried one such row (height 30822) for six weeks with its + payout entries, carry forward, and fanout artifacts pinned immature. + """ + + def orphan_row(self, block_height: int = 30822) -> dict[str, object]: + return { + "block_hash": "19" * 32, + "block_height": block_height, + "parent_hash": "20" * 32, + } + + def test_orphaned_stranded_row_is_rejected_and_republished(self) -> None: + row = self.orphan_row() + ledger = StrandedPreparedLedger([row]) + ports, events = make_ports( + ledger=ledger, + tip_height=31000, + # The canonical block at that height is a different hash: the + # pool block was orphaned and never rejected. + block_hashes={30822: "aa" * 32}, + ) + service = ReorgReconcilerService(ports) + + with mock.patch("builtins.print"): + summary = service.reconcile(tip_hash="tip") + + self.assertEqual(ledger.rejected, ["19" * 32]) + self.assertEqual(summary["stranded_prepared_rejected"], 1) + self.assertEqual(summary["stranded_prepared_canonical"], 0) + # The rejection cascades payout entries, carry forward, and fanout + # artifacts, so the pass must republish rather than serve balances + # computed before it -- rereading balances over the exact window + # (#224), since the cascade never touches the share ledger. + self.assertIn( + ( + "prepare", + { + "force_full_window_rescan": False, + "force_prior_balances_read": True, + }, + ), + events, + ) + self.assertEqual(summary["published_generation"], 7) + self.assertIn(("block", {"force": True}), events) + # A row mutation invalidates proofs memoized for every other tip. + service._reorg_reconcile_trusted_memo["other"] = time.monotonic() + self.assertEqual( + ledger.requests, + [ + { + "active_tip_height": 31000, + "min_depth": STRANDED_PREPARED_REJECT_MIN_DEPTH, + "limit": STRANDED_PREPARED_SWEEP_LIMIT, + } + ], + ) + + def test_canonical_stranded_row_is_counted_but_never_mutated(self) -> None: + """The confirm path owns audit publication sequencing, not the sweep.""" + row = self.orphan_row() + ledger = StrandedPreparedLedger([row]) + ports, events = make_ports( + ledger=ledger, + tip_height=31000, + # The pool block *is* the canonical block at its height; it was + # simply never confirmed. + block_hashes={30822: "19" * 32}, + ) + service = ReorgReconcilerService(ports) + + printed: list[str] = [] + with mock.patch( + "builtins.print", + side_effect=lambda *args, **_kwargs: printed.append( + " ".join(str(value) for value in args) + ), + ): + summary = service.reconcile(tip_hash="tip") + + self.assertEqual(ledger.rejected, []) + self.assertEqual(summary["stranded_prepared_canonical"], 1) + self.assertEqual(summary["stranded_prepared_rejected"], 0) + # Nothing changed, so nothing is prepared or published. + self.assertNotIn(("block", {"force": True}), events) + self.assertIsNone(summary["published_generation"]) + self.assertTrue( + any( + "canonical and needs operator review" in message + and "19" * 32 in message + for message in printed + ), + printed, + ) + + def test_rows_inside_the_depth_floor_are_never_examined(self) -> None: + """A finalization still in flight owns its own row; depth is the proxy.""" + shallow = self.orphan_row(block_height=31000 - 1) + ledger = StrandedPreparedLedger([shallow]) + ports, events = make_ports( + ledger=ledger, + tip_height=31000, + block_hashes={}, + ) + service = ReorgReconcilerService(ports) + + summary = service.reconcile(tip_hash="tip") + + self.assertEqual(ledger.rejected, []) + self.assertEqual(summary["stranded_prepared_rejected"], 0) + self.assertEqual(summary["stranded_prepared_canonical"], 0) + # Not even a getblockhash: the row never left the ledger read. + self.assertNotIn( + "getblockhash", + [name for name, _payload in events], + ) + self.assertGreaterEqual(STRANDED_PREPARED_REJECT_MIN_DEPTH, 100) + + def test_sweep_stamps_progress_for_every_row_it_walks(self) -> None: + """Each row costs a getblockhash plus a fenced mutation, like watch rows.""" + rows = [ + self.orphan_row(block_height=30820), + self.orphan_row(block_height=30822), + ] + rows[0]["block_hash"] = "18" * 32 + ledger = StrandedPreparedLedger(rows) + ports, _events = make_ports( + ledger=ledger, + tip_height=31000, + block_hashes={30820: "aa" * 32, 30822: "aa" * 32}, + ) + phases: list[str] = [] + service = ReorgReconcilerService( + dataclass_replace(ports, record_progress=phases.append) + ) + + with mock.patch("builtins.print"): + summary = service.reconcile(tip_hash="tip") + + self.assertEqual(summary["stranded_prepared_rejected"], 2) + self.assertEqual(phases.count("reorg-reconcile:stranded-prepared"), 2) + self.assertIn("reorg-reconcile:stranded-prepared-blocks", phases) + + def test_partial_sweep_that_raises_still_accounts_its_mutations(self) -> None: + """A pass that dies mid-sweep must not lose the rows it already moved.""" + rows = [ + self.orphan_row(block_height=30820), + self.orphan_row(block_height=30822), + ] + rows[0]["block_hash"] = "18" * 32 + ledger = StrandedPreparedLedger(rows) + original_reject = ledger.reject_prepared_block + + def failing_reject(**kwargs: object) -> dict[str, int]: + if kwargs["block_hash"] == "19" * 32: + raise RuntimeError("postgres unavailable") + return original_reject(**kwargs) # type: ignore[arg-type] + + ledger.reject_prepared_block = failing_reject # type: ignore[method-assign] + ports, events = make_ports( + ledger=ledger, + tip_height=31000, + block_hashes={30820: "aa" * 32, 30822: "aa" * 32}, + ) + service = ReorgReconcilerService(ports) + + with mock.patch("builtins.print"), self.assertRaisesRegex( + RuntimeError, "postgres unavailable" + ): + service.reconcile(tip_hash="tip") + + # The first rejection committed; the error path republishes against a + # forced full rescan rather than leaving stale balances served. + self.assertEqual(ledger.rejected, ["18" * 32]) + self.assertIn(("prepare", {"force_full_window_rescan": True}), events) + self.assertEqual(service.snapshot().reconcile_error_count, 1) + self.assertEqual(dict(service._reorg_reconcile_trusted_memo), {}) + + def test_ledger_without_the_sweep_read_is_left_alone(self) -> None: + """Feature-detected like every other reconcile step.""" + ledger = FakeLedger() + ports, _events = make_ports(ledger=ledger) + service = ReorgReconcilerService(ports) + + summary = service.reconcile(tip_hash="tip", force_publish=True) + + self.assertEqual(summary["stranded_prepared_rejected"], 0) + self.assertEqual(summary["stranded_prepared_canonical"], 0) diff --git a/tests/test_prism_replay_enumeration_gate.py b/tests/test_prism_replay_enumeration_gate.py new file mode 100644 index 000000000..c18676da1 --- /dev/null +++ b/tests/test_prism_replay_enumeration_gate.py @@ -0,0 +1,526 @@ +#!/usr/bin/env python3 +"""Replay enumeration must not queue behind the coordinator's writer convoy. + +Issue #211. ``PsqlShareLedger.pending_block_candidate_rows()`` is one +read-only, single-snapshot statement, but it used to take +``_operation_gate(self._lock, "writer lock")`` before sending it. During +accepted-block accounting that put a *read* in the same coordinator-local +convoy as the accounting write, and on ``union-mainnet`` the read spent most +of its bounded budget there: the outer call reported +``replay-outbox-query exceeded 5s`` while the inner PostgreSQL deadline still +had ~0.4-1.3s left, PostgreSQL showed zero blocked backends, and the host was +~96% idle. Accepted candidates still converged, but in 79-186s, and +``qbit_prism_accepted_parent_unresolved_oldest_seconds`` reached ~101s. + +These scenarios drive that interleaving deterministically. The writer gate is +genuinely held by a fenced write that has sent its statement and not yet +committed -- the shape a long accounting write has -- and the enumeration runs +against it. The evidence is not merely "the read returned": the same held gate, +under the same budget, is shown to *fail* writer admission on another actor, so +a run cannot pass by accident of a gate that happened to be free. + +The harness is the #128 deterministic one, extended here with the pending-page +statement (``LandingOp.OUTBOX_PENDING_PAGE``); the code under test is the +shipped ``PsqlShareLedger``, reached through its real ports. +""" + +from __future__ import annotations + +import sys +import unittest +from pathlib import Path +from typing import Any + +sys.path.insert(0, str(Path(__file__).resolve().parents[1])) + +from lab.prism.share_ledger import LedgerOperationTimeout # noqa: E402 +from tests.prism_concurrency_harness import ( # noqa: E402 + LandingOp, + LeaseHarness, + assert_deterministic, +) +from tests.prism_landing_harness import PARENT_HASH, LandingHarness # noqa: E402 + +BLOCK_A = "aa" * 32 +BLOCK_B = "bb" * 32 + +# The production fast-call budget the incident ran with +# (PRISM_BLOCK_SUBMIT_DB_TIMEOUT_SECONDS=5). Every enumeration below is armed +# with exactly this, and every writer hold outlasts it. +FAST_CALL_BUDGET_SECONDS = 5.0 + +# How long the fenced write stays parked mid-statement. Deliberately longer +# than the fast-call budget: that gap is the whole defect. +WRITER_HOLD_SECONDS = 12.0 + +# Statement checkpoints FakePostgres offers, tagged with the coordinator name. +FENCED_WRITE_PRECOMMIT = "writer.precommit" +PAGE_STATEMENT_BEGIN = "writer.begin:outbox_header_page" + + +def _intent(block_hash: str) -> dict[str, Any]: + return { + "schema": "qbit.prism.block-candidate-intent.v1", + "block_hash_hex": block_hash, + "block_hex": "00", + } + + +def _booted(harness: LeaseHarness) -> Any: + """Boot one coordinator and make its ledger gates schedulable. + + The gates are the subject, so they have to be visible to the scheduler: + a real ``threading.Lock`` would block an actor thread outside the baton + and read as a harness stall rather than as the contention it is. These + are the same substitutions the landing harness makes, and they change how + a waiter blocks, never what the ledger decides. + """ + coordinator = harness.coordinator("writer") + coordinator.start() + harness.run_until(coordinator.actor, "done:startup") + ledger = coordinator.ledger + ledger._lock = harness.lock("ledger-writer") + ledger._read_semaphore = harness.semaphore("ledger-read-slot", 4) + return ledger + + +def _park_a_fenced_write(harness: LeaseHarness, ledger: Any, block_hash: str) -> Any: + """Leave a fenced outbox write in flight, holding the writer gate. + + ``persist_block_candidate_intent`` goes through ``_run_fenced_json``, so + the writer gate is held for the whole statement. Arming a deadline is what + makes the backend run BEGIN / statement / COMMIT rather than autocommit, + which gives the scenario a checkpoint *inside* the held gate -- the + accounting write's own shape, and the only place a read can observe the + convoy. + """ + writer = harness.scheduler.actor("long-write") + + def run() -> Any: + with ledger.operation_timeout(WRITER_HOLD_SECONDS * 4): + return ledger.persist_block_candidate_intent(_intent(block_hash)) + + call = writer.submit(run, label=f"fenced-write:{block_hash[:4]}") + # Issue #255: the intent's body is staged and sealed first, in + # statements that hold no writer gate at all; only the final outbox + # insert is the fenced write this scenario parks. Run through the + # staging checkpoints until the parked statement is that insert. + for _ in range(64): + harness.run_until(writer, FENCED_WRITE_PRECOMMIT) + if harness.server.statements[-1].kind is LandingOp.OUTBOX_RECORD: + break + else: + raise AssertionError("the fenced outbox insert never reached its precommit") + return writer, call + + +def _enumerate(ledger: Any) -> list[dict[str, Any]]: + """One replay enumeration under the incident's own fast-call budget.""" + with ledger.operation_timeout(FAST_CALL_BUDGET_SECONDS): + return list(ledger.pending_block_candidate_headers(limit=32).rows) + + +def _take_writer_gate(ledger: Any) -> None: + """Ask for writer admission on the same budget: the control.""" + with ledger.operation_timeout(FAST_CALL_BUDGET_SECONDS): + ledger._acquire_operation_gate(ledger._lock, "writer lock") + ledger._lock.release() + + +def _step_to_the_stale_page_window( + harness: LandingHarness, + ledger: Any, + found: Any, +) -> None: + """Step the landing to the window a stale page is acted on in. + + Three conditions, deliberately named instead of a checkpoint: the + accepted-block write has committed so a durable ``qbit_pool_blocks`` row + exists, the candidate's outbox row is still pending so the terminal + write's ordinary state predicate cannot be what refuses it, and the writer + gate is free so that write can run at all. + + A checkpoint name would have been the shorter spelling and the wrong one. + This scenario originally stopped at ``ledger.begin:prior_balances_as_of``, + and issue #210 then routed fresh confirmations through the + current-balance summary instead of the height-bounded as-of read -- which + deleted that checkpoint from the landing while leaving the window itself + exactly where it was. Asking for the state the proof depends on survives + that kind of change; asking for the statement that happened to sit next to + it did not. + """ + block_hash = found.block_hash + for _ in range(200): + if not harness.accounting.runnable(): + raise AssertionError( + "the landing stalled before reaching the stale-page window " + f"(blocked at {harness.accounting.block_reason!r})" + ) + label = harness.step(harness.accounting) + if ( + harness.pool_block(block_hash) is not None + and harness.outbox_state(found) == "pending" + and not ledger._lock.locked() + ): + return + if label.startswith("done:"): + break + raise AssertionError( + "the landing never offered a landed-but-pending window with the " + "writer gate free" + ) + + +class EnumerationOutsideTheWriterConvoyTests(unittest.TestCase): + def _scenario(self, harness: LeaseHarness) -> dict[str, Any]: + """Enumerate a durable backlog while a fenced write holds the gate.""" + ledger = _booted(harness) + + # A durable pending candidate exists before the convoy forms, so the + # enumeration has something real to return. + setup = harness.scheduler.actor("setup") + setup.submit( + lambda: ledger.persist_block_candidate_intent(_intent(BLOCK_A)), + label="seed", + ) + harness.run_until(setup, "done:seed") + + # A second candidate's fenced write is now in flight and parked: its + # statement has been sent, its COMMIT has not, and the writer gate is + # held for the duration. + long_write, long_call = _park_a_fenced_write(harness, ledger, BLOCK_B) + gate_held_during_read = ledger._lock.locked() + + # The replay enumeration, under the incident's own budget. + replay = harness.scheduler.actor("replay") + page_call = replay.submit( + lambda: _enumerate(ledger), + label="replay-outbox-query", + ) + + # A second reader asks for *writer* admission under the same budget + # against the same held gate. It is the control: it proves the hold is + # real and outlasts the budget, so the enumeration's success below + # cannot be a gate that happened to be free. + contender = harness.scheduler.actor("writer-admission") + admission_call = contender.submit( + lambda: _take_writer_gate(ledger), + label="writer-admission", + ) + harness.run_until_blocked(contender, "lock:ledger-writer") + + # The enumeration reaches PostgreSQL with the gate still held... + harness.run_until(replay, PAGE_STATEMENT_BEGIN) + gate_held_at_statement = ledger._lock.locked() + # ...and spends measurable time in the server rather than in the gate. + harness.advance(0.75) + harness.run_until(replay, "done:replay-outbox-query") + + # Only now does the hold outlast the fast-call budget. The contender + # gives up; the fenced write is still parked. + harness.advance(WRITER_HOLD_SECONDS) + harness.drain([contender]) + + harness.drain([long_write, contender, replay]) + return { + # Random durable body IDs are irrelevant to this scheduler proof. + "page": [{key: value for key, value in row.items() if key != "body"} + for row in page_call.value()], + "gate_held_during_read": gate_held_during_read, + "gate_held_at_statement": gate_held_at_statement, + "admission_error": type(admission_call.error).__name__, + "admission_message": str(admission_call.error), + "long_write_result": ( + type(long_call.error).__name__ + if long_call.error is not None + else str(long_call.value()) + ), + "stats": ledger.ledger_read_gate_stats()[ + "pending_block_candidate_headers" + ], + } + + def test_enumeration_completes_through_the_read_path(self) -> None: + with LeaseHarness() as harness: + result = self._scenario(harness) + + # The gate really was held, both when the read started and when its + # statement reached the server. + self.assertTrue(result["gate_held_during_read"]) + self.assertTrue(result["gate_held_at_statement"]) + # The control proves the hold outlasted the identical budget. + self.assertEqual(result["admission_error"], "LedgerOperationTimeout") + self.assertIn("writer lock", result["admission_message"]) + # And the enumeration returned the durable backlog anyway. + # + # Exactly the durable backlog: BLOCK_B's row is staged inside the + # parked write's uncommitted transaction, and the page -- on its own + # pooled session, at its own snapshot -- does not see it. That is the + # commit-before-wakeup ordering the outbox relies on, stated from the + # reader's side: enumeration never surfaces a candidate before the row + # that makes it durable exists. + self.assertEqual( + [row["block_hash"] for row in result["page"]], + [BLOCK_A], + ) + self.assertEqual( + [row["pool_block_exists"] for row in result["page"]], + [False], + ) + # The long write is untouched by any of it: it commits when it resumes. + self.assertEqual( + result["long_write_result"], + "BlockCandidateIntentPersistResult(inserted=True, state='pending')", + ) + + def test_no_time_is_charged_to_local_admission(self) -> None: + """The attribution says database, because the database is where it went. + + The incident was unattributable from ``/metrics``: one duration + covered both admission and execution, so "exceeded 5s" could not be + told apart from a slow query. These two fields are that distinction. + """ + with LeaseHarness() as harness: + stats = self._scenario(harness)["stats"] + + self.assertEqual(stats["calls_total"], 1) + self.assertEqual(stats["gate_wait_seconds_max"], 0.0) + self.assertEqual(stats["gate_timeouts_total"], 0) + self.assertEqual(stats["execute_timeouts_total"], 0) + # The 0.75s advanced while the statement was in flight lands on the + # execution series, not the gate series. + self.assertAlmostEqual(stats["execute_seconds_max"], 0.75) + self.assertAlmostEqual(stats["execute_seconds_total"], 0.75) + + def test_the_interleaving_is_deterministic(self) -> None: + assert_deterministic(self, self._scenario) + + def test_enumeration_still_queues_on_the_bounded_read_slot(self) -> None: + """Bounded, not unbounded: the read slot is still the admission limit. + + Removing the writer gate must not mean removing every gate. A read + semaphore of one admits one enumeration; the second waits for a slot + rather than opening a connection or starting a thread of its own, and + it names the read slot when its own budget expires. + """ + with LeaseHarness() as harness: + ledger = _booted(harness) + ledger._read_semaphore = harness.semaphore("ledger-read-slot", 1) + + setup = harness.scheduler.actor("setup") + setup.submit( + lambda: ledger.persist_block_candidate_intent(_intent(BLOCK_A)), + label="seed", + ) + harness.run_until(setup, "done:seed") + + first = harness.scheduler.actor("replay-a") + second = harness.scheduler.actor("replay-b") + first_call = first.submit( + lambda: _enumerate(ledger), + label="page-a", + ) + second_call = second.submit( + lambda: _enumerate(ledger), + label="page-b", + ) + + # The first holds the only slot, parked inside its statement. + harness.run_until(first, PAGE_STATEMENT_BEGIN) + harness.run_until_blocked(second, "semaphore:ledger-read-slot") + + # The second's own budget expires while it waits for admission. + harness.advance(FAST_CALL_BUDGET_SECONDS * 2) + harness.drain([second]) + harness.drain([first, second]) + + self.assertIsNone(first_call.error) + self.assertEqual( + [row["block_hash"] for row in first_call.value()], + [BLOCK_A], + ) + self.assertIsInstance(second_call.error, LedgerOperationTimeout) + self.assertIn("read slot", str(second_call.error)) + + # An admission expiry is counted as one, and never as a statement + # that PostgreSQL was slow to answer. + stats = ledger.ledger_read_gate_stats()[ + "pending_block_candidate_headers" + ] + self.assertEqual(stats["calls_total"], 2) + self.assertEqual(stats["gate_timeouts_total"], 1) + self.assertEqual(stats["execute_timeouts_total"], 0) + self.assertGreaterEqual( + stats["gate_wait_seconds_max"], + FAST_CALL_BUDGET_SECONDS, + ) + + def test_no_extra_connection_or_thread_is_taken(self) -> None: + """The page read borrows a pooled connection, as it always did. + + The fix must not buy latency with a connection per call: the pool is + sized ``read_concurrency + 1`` precisely so the read slots and the + single writer cannot together outrun it. + """ + with LeaseHarness() as harness: + ledger = _booted(harness) + backend = harness.server + pool = ledger._native + + setup = harness.scheduler.actor("setup") + setup.submit( + lambda: ledger.persist_block_candidate_intent(_intent(BLOCK_A)), + label="seed", + ) + harness.run_until(setup, "done:seed") + sessions_before = len(backend.live_backends()) + + replay = harness.scheduler.actor("replay") + replay.submit(lambda: _enumerate(ledger), label="page") + harness.run_until(replay, "done:page") + + self.assertEqual(len(backend.live_backends()), sessions_before) + self.assertEqual( + pool.pool_size, + ledger._read_semaphore.value + len(pool._in_use) + 1, + ) + + +class StalePageAgainstALandingWriteTests(unittest.TestCase): + """The production landing topology, with the enumeration reading across it. + + Class one proves the gate is no longer taken. This one proves that not + taking it is *safe*: the page a caller now gets can be stale about a + candidate that lands while the read is in flight, and every terminal + decision built on that page must still refuse the landed row. The refusal + is the production ``mark_block_candidates_abandoned`` statement, whose + pool-block re-check sits inside the same fenced UPDATE that transitions + the rows. + """ + + def _scenario(self, harness: LandingHarness) -> dict[str, Any]: + ledger = harness.boot() + block_a = harness.found_block(BLOCK_A, height=10, parent_hash=PARENT_HASH) + block_b = harness.found_block(BLOCK_B, height=11, parent_hash=BLOCK_A) + + # Both candidates are durable before either lands: closely spaced + # found blocks, which is the population #211 measured converging in + # 79-186s. + harness.appender.submit( + lambda: harness._persist_intent(block_b), + label="seed-b", + ) + harness.run_until(harness.appender, "done:seed-b") + + # A's accepted-block accounting write is in flight: the statement has + # been sent, its COMMIT has not, and the writer gate is held for the + # whole of it. + harness.land_on_accounting_tail(block_a) + harness.run_until(harness.accounting, "ledger.done:persist_block") + harness.step(harness.accounting) + gate_held_during_read = ledger._lock.locked() + + # Writer admission under the fast-call budget cannot get through that + # hold -- the control for everything below. + contender = harness.scheduler.actor("writer-admission") + admission_call = contender.submit( + lambda: _take_writer_gate(ledger), + label="writer-admission", + ) + harness.run_until_blocked(contender, "lock:ledger-writer") + + # The enumeration does, on the same budget. + replay = harness.scheduler.actor("replay") + page_call = replay.submit( + lambda: _enumerate(ledger), + label="replay-outbox-query", + ) + harness.run_until(replay, "done:replay-outbox-query") + page = page_call.value() + + harness.advance(FAST_CALL_BUDGET_SECONDS * 2) + harness.drain([contender]) + + # A's write commits. Its pool-block row is now durable while its + # outbox row is still pending, and the writer gate is free again -- + # the exact window in which a caller holding the stale page acts. + _step_to_the_stale_page_window(harness, ledger, block_a) + landed = harness.pool_block(BLOCK_A) is not None + still_pending = harness.outbox_state(block_a) + + terminal = harness.scheduler.actor("collapse") + abandon_call = terminal.submit( + lambda: ledger.mark_block_candidates_abandoned( + block_hashes=[str(row["block_hash"]) for row in page], + error="superseded by a decided height", + ), + label="batch-abandon", + ) + harness.run_until(terminal, "done:batch-abandon") + + harness.drain([harness.accounting, harness.appender, terminal]) + return { + "gate_held_during_read": gate_held_during_read, + "pending_at_the_terminal_write": still_pending, + "admission_error": type(admission_call.error).__name__, + "page": [ + (str(row["block_hash"]), bool(row["pool_block_exists"])) + for row in page + ], + "landed_before_the_terminal_write": landed, + "abandoned": list(abandon_call.value()), + "outbox_a": harness.outbox_state(block_a), + "outbox_b": harness.outbox_state(block_b), + } + + def test_enumeration_crosses_a_landing_write_and_stays_advisory(self) -> None: + with LandingHarness() as harness: + result = self._scenario(harness) + + # The gate was genuinely held, and genuinely impassable on the budget. + self.assertTrue(result["gate_held_during_read"]) + self.assertEqual(result["admission_error"], "LedgerOperationTimeout") + + # The page came back, and it is stale about A: A's pool-block row + # exists only inside the uncommitted landing transaction, so this + # snapshot cannot see it. That staleness is allowed -- it is what + # "advisory" means -- and it is exactly the input the next assertion + # feeds to a terminal write. + # + # The order is A then B even though B was persisted first: both rows + # carry the same creation stamp here, and the total order's block-hash + # tiebreak decides. That tiebreak is the reason a cursor cannot replay + # or skip a colliding group. + self.assertEqual( + result["page"], + [(BLOCK_A, False), (BLOCK_B, False)], + ) + + def test_a_landed_candidate_is_refused_by_the_fence_not_the_page(self) -> None: + with LandingHarness() as harness: + result = self._scenario(harness) + + # By the time the terminal write runs, A has landed and is still + # pending -- both halves matter, or the row would be excluded by the + # ordinary state predicate and prove nothing about the re-check. + self.assertTrue(result["landed_before_the_terminal_write"]) + self.assertEqual(result["pending_at_the_terminal_write"], "pending") + + # The caller asked to abandon both hashes on the strength of a page + # that said neither had landed. The fenced UPDATE re-asked + # qbit_pool_blocks and returned only the row that had not. + self.assertEqual(result["abandoned"], [BLOCK_B]) + self.assertEqual(result["outbox_b"], "abandoned") + # A survived the stale page and finished its landing normally. + self.assertEqual(result["outbox_a"], "submitted") + + def test_the_interleaving_is_deterministic(self) -> None: + assert_deterministic( + self, + self._scenario, + harness_factory=LandingHarness, + ) + + +if __name__ == "__main__": # pragma: no cover - direct invocation + unittest.main() diff --git a/tests/test_prism_representative_independence.py b/tests/test_prism_representative_independence.py index 9a97eb5b6..58e583339 100644 --- a/tests/test_prism_representative_independence.py +++ b/tests/test_prism_representative_independence.py @@ -7,15 +7,14 @@ import time import unittest +from lab.prism.job_bundle import CachedJobBundle, CollectionIdentityUnavailable from lab.prism.prism_coordinator import ( - CachedJobBundle, CachedTemplateArtifacts, - CollectionIdentityUnavailable, StratumError, TemplateRefreshBlocked, WorkerIdentity, ) -from tests.test_prism_coordinator_job_cache import ( +from tests.prism_coordinator_test_support import ( FakeLedger, base_template, client, diff --git a/tests/test_prism_retained_jobs.py b/tests/test_prism_retained_jobs.py new file mode 100644 index 000000000..7b0a216e4 --- /dev/null +++ b/tests/test_prism_retained_jobs.py @@ -0,0 +1,1248 @@ +#!/usr/bin/env python3 +"""Focused PRISM coordinator retained jobs tests.""" +# ruff: noqa: F403, F405 + +from __future__ import annotations + +import unittest +from tests.prism_vardiff_test_support import * + + +class PrismCoordinatorVardiffTests(unittest.TestCase): + def test_clean_job_prunes_previous_active_prism_job(self) -> None: + server = coordinator() + state = client() + state.worker = WorkerIdentity( + username="miner-a", + payout_address="miner-a", + worker_name=None, + script_pubkey_hex="5220" + "11" * 32, + p2mr_program_hex="11" * 32, + ) + server.jobs = {} + counter = {"value": 0} + + def build_context(client: ClientState, *, clean_jobs: bool) -> object: + counter["value"] += 1 + return SimpleNamespace( + job=SimpleNamespace(job_id=f"job-{counter['value']}", share_difficulty=Decimal("1")), + template={"previousblockhash": "00" * 32}, + collection_only=False, + ) + + server.build_job_for_client = build_context # type: ignore[method-assign] + server.send_difficulty = lambda *args, **kwargs: None + server.send_job = lambda *args, **kwargs: None + server.apply_job_difficulty = lambda *args, **kwargs: None + + server.maybe_send_job(state, clean_jobs=True) + first_job_id = next(iter(state.active_job_ids)) + server.maybe_send_job(state, clean_jobs=True) + second_job_id = next(iter(state.active_job_ids)) + + self.assertNotEqual(first_job_id, second_job_id) + self.assertNotIn(first_job_id, server.jobs) + self.assertIn(second_job_id, server.jobs) + + state.sock = SimpleNamespace(shutdown=lambda *_args: None, close=lambda: None) + server.disconnect_client(state) + self.assertNotIn(second_job_id, server.jobs) + self.assertEqual(state.active_job_ids, set()) + + def test_non_clean_job_retention_caps_previous_active_prism_jobs(self) -> None: + server = coordinator() + state = client() + state.worker = WorkerIdentity( + username="miner-a", + payout_address="miner-a", + worker_name=None, + script_pubkey_hex="5220" + "11" * 32, + p2mr_program_hex="11" * 32, + ) + server.jobs = {} + counter = {"value": 0} + + def build_context(client: ClientState, *, clean_jobs: bool) -> object: + counter["value"] += 1 + return SimpleNamespace( + job=SimpleNamespace(job_id=f"job-{counter['value']}", share_difficulty=Decimal("1")), + template={"previousblockhash": "00" * 32}, + collection_only=False, + ) + + server.build_job_for_client = build_context # type: ignore[method-assign] + server.send_difficulty = lambda *args, **kwargs: None + server.send_job = lambda *args, **kwargs: None + server.apply_job_difficulty = lambda *args, **kwargs: None + + total_jobs = MAX_ACTIVE_PRISM_JOBS_PER_CLIENT + 3 + for _ in range(total_jobs): + server.maybe_send_job(state, clean_jobs=False) + + retained_ids = { + f"job-{index}" + for index in range(4, total_jobs + 1) + } + self.assertEqual(state.active_job_ids, retained_ids) + self.assertEqual(set(server.jobs), retained_ids) + self.assertNotIn("job-1", server.jobs) + self.assertEqual(state.active_job.job.job_id, f"job-{total_jobs}") + + def test_normal_accepted_share_does_not_close_client(self) -> None: + server, state, ledger = submit_coordinator() + submission = SimpleNamespace( + header_hex="aa" * 80, + block_hash_hex="bb" * 32, + share_pass=True, + block_pass=False, + ) + + with patch( + "lab.prism.prism_coordinator.direct_stratum.assemble_submission", + return_value=submission, + ): + should_close = server.handle_submit( + state, + ["miner-a", "job-1", "00" * 8, "00000001", "00000002"], + ) + + self.assertFalse(should_close) + self.assertEqual(len(ledger.pending), 1) + self.assertEqual(ledger.pending[0].share_id, "miner-a:" + "bb" * 32) + + def test_prior_tip_share_inside_grace_is_credited_without_submitblock(self) -> None: + old_tip = "00" * 32 + new_tip = "11" * 32 + server, state, ledger = submit_coordinator(tip=old_tip) + rpc = ParentTipRpc(tip=new_tip, parent=old_tip) + server.rpc = rpc + server.current_tip_first_seen = (new_tip, time.monotonic()) + server.stale_grace_seconds = 3 + submission = SimpleNamespace( + header_hex="aa" * 80, + block_hash_hex="cc" * 32, + share_pass=True, + block_pass=True, + ) + + with ( + patch("builtins.print") as emitted, + patch( + "lab.prism.prism_coordinator.direct_stratum.assemble_submission", + return_value=submission, + ), + ): + should_close = server.handle_submit( + state, + ["miner-a", "job-1", "00" * 8, "00000001", "00000002"], + ) + + self.assertFalse(should_close) + self.assertEqual(rpc.submitblock_calls, 0) + self.assertEqual(len(ledger.pending), 1) + self.assertEqual(ledger.pending[0].credit_policy, PRISM_CREDIT_POLICY_STALE_GRACE) + self.assertEqual(server.grace_credited_share_count, 1) + self.assertEqual(server.worker_share_counts["miner-a"]["grace"], 1) + self.assertEqual(server.block_solves_dropped_counts, {"stale_grace": 1}) + self.assertIn( + 'qbit_prism_block_solves_dropped_total{reason="stale_grace"} 1', + server.metrics_payload(), + ) + self.assertEqual(emitted.call_count, 1) + log_line = " ".join(str(value) for value in emitted.call_args.args) + self.assertIn(submission.block_hash_hex, log_line) + self.assertIn(old_tip, log_line) + + def test_evicted_prior_tip_share_inside_grace_is_credited(self) -> None: + old_tip = "00" * 32 + new_tip = "11" * 32 + server, state, ledger = submit_coordinator(tip=old_tip) + rpc = ParentTipRpc(tip=new_tip, parent=old_tip) + server.rpc = rpc + server.current_tip_first_seen = (new_tip, time.monotonic()) + server.bury_evicted_job(state, "job-1") + server.jobs.pop("job-1") + state.active_job_ids.clear() + submission = SimpleNamespace( + header_hex="ad" * 80, + block_hash_hex="cd" * 32, + share_pass=True, + block_pass=False, + ) + + with patch( + "lab.prism.prism_coordinator.direct_stratum.assemble_submission", + return_value=submission, + ): + should_close = server.handle_submit( + state, + ["miner-a", "job-1", "00" * 8, "00000001", "00000002"], + ) + + self.assertFalse(should_close) + self.assertEqual(len(ledger.pending), 1) + self.assertEqual(ledger.pending[0].credit_policy, PRISM_CREDIT_POLICY_STALE_GRACE) + + def test_evicted_same_tip_share_is_credited_without_stale_grace_policy(self) -> None: + tip = "00" * 32 + server, state, ledger = submit_coordinator(tip=tip) + server.bury_evicted_job(state, "job-1") + server.jobs.pop("job-1") + state.active_job_ids.clear() + submission = SimpleNamespace( + header_hex="af" * 80, + block_hash_hex="cf" * 32, + share_pass=True, + block_pass=False, + ) + + with patch( + "lab.prism.prism_coordinator.direct_stratum.assemble_submission", + return_value=submission, + ): + should_close = server.handle_submit( + state, + ["miner-a", "job-1", "00" * 8, "00000001", "00000002"], + ) + + self.assertFalse(should_close) + self.assertEqual(len(ledger.pending), 1) + self.assertIsNone(ledger.pending[0].credit_policy) + + def test_retained_share_dedup_uses_original_worker_after_reauthorization(self) -> None: + tip = "00" * 32 + server, state, ledger = submit_coordinator(tip=tip) + server.bury_evicted_job(state, "job-1") + server.jobs.pop("job-1") + state.active_job_ids.clear() + submission = SimpleNamespace( + header_hex="af" * 80, + block_hash_hex="cf" * 32, + share_pass=True, + block_pass=False, + ) + + with patch( + "lab.prism.prism_coordinator.direct_stratum.assemble_submission", + return_value=submission, + ): + self.assertFalse( + server.handle_submit( + state, + ["miner-a", "job-1", "00" * 8, "00000001", "00000002"], + ) + ) + state.username = "miner-b" + state.worker = worker_identity("miner-b") + with self.assertRaises(StratumError) as raised: + server.handle_submit( + state, + ["miner-b", "job-1", "00" * 8, "00000001", "00000002"], + ) + + self.assertEqual(raised.exception.reason, PRISM_REJECTION_DUPLICATE_SHARE) + self.assertEqual(len(ledger.pending), 1) + self.assertEqual(ledger.pending[0].share_id, "miner-a:" + "cf" * 32) + self.assertEqual(server.worker_share_counts["miner-a"]["accepted"], 1) + self.assertEqual(server.worker_share_counts["miner-b"]["accepted"], 0) + + def test_evicted_same_tip_share_survives_beyond_legacy_one_second_floor(self) -> None: + tip = "00" * 32 + server, state, ledger = submit_coordinator(tip=tip) + server.current_tip_first_seen = (tip, None) + server.same_tip_job_retention_seconds = 30 + server.bury_evicted_job(state, "job-1", now=100.0) + server.jobs.pop("job-1") + state.active_job_ids.clear() + submission = SimpleNamespace( + header_hex="a1" * 80, + block_hash_hex="c1" * 32, + share_pass=True, + block_pass=False, + ) + + with patch( + "lab.prism.prism_coordinator.time.monotonic", + return_value=102.0, + ), patch( + "lab.prism.prism_coordinator.direct_stratum.assemble_submission", + return_value=submission, + ): + self.assertFalse( + server.handle_submit( + state, + ["miner-a", "job-1", "00" * 8, "00000001", "00000002"], + ) + ) + + self.assertEqual(len(ledger.pending), 1) + self.assertIsNone(ledger.pending[0].credit_policy) + self.assertIn("job-1", server.evicted_job_graveyard) + self.assertEqual(server.evicted_job_submit_counts["accepted_same_tip"], 1) + + def test_evicted_same_tip_submit_uses_original_job_difficulty(self) -> None: + tip = "00" * 32 + server, state, ledger = submit_coordinator(tip=tip) + server.current_tip_first_seen = (tip, None) + original_context = server.jobs["job-1"] + original_context.job.share_difficulty = Decimal("2") + state.share_difficulty = Decimal("32") + server.bury_evicted_job(state, "job-1") + server.jobs.pop("job-1") + state.active_job_ids.clear() + submission = SimpleNamespace( + header_hex="a2" * 80, + block_hash_hex="c2" * 32, + share_pass=True, + block_pass=False, + ) + + def assemble(job: object, **_kwargs: object) -> object: + self.assertIs(job, original_context.job) + self.assertEqual(job.share_difficulty, Decimal("2")) + return submission + + with patch( + "lab.prism.prism_coordinator.direct_stratum.assemble_submission", + side_effect=assemble, + ): + self.assertFalse( + server.handle_submit( + state, + ["miner-a", "job-1", "00" * 8, "00000001", "00000002"], + ) + ) + + self.assertEqual(len(ledger.pending), 1) + self.assertIsNone(ledger.pending[0].credit_policy) + + def test_same_tip_retention_ttl_and_capacity_are_bounded(self) -> None: + tip = "00" * 32 + server, state, _ledger = submit_coordinator(tip=tip) + server.current_tip_first_seen = (tip, None) + server.same_tip_job_retention_seconds = 30 + server.same_tip_job_retention_per_connection = 2 + identity = state.worker + for index in range(3): + job_id = f"job-{index + 1}" + server.jobs[job_id] = prism_context(job_id, tip, worker=identity) + server.bury_evicted_job(state, job_id, now=100.0 + index) + + self.assertNotIn("job-1", server.evicted_job_graveyard) + self.assertEqual( + list(server.evicted_job_graveyard), + ["job-2", "job-3"], + ) + self.assertEqual(server.evicted_job_capacity_eviction_counts["connection"], 1) + + server.prune_evicted_job_graveyard(now=133.1) + self.assertEqual(server.evicted_job_graveyard, {}) + self.assertEqual(server.evicted_job_expiration_counts["same_tip"], 2) + + def test_evicted_job_hit_is_constant_work_in_large_graveyard(self) -> None: + tip = "00" * 32 + server, state, _ledger = submit_coordinator(tip=tip) + server.current_tip_first_seen = (tip, None) + server.same_tip_job_retention_seconds = 30 + server.same_tip_job_retention_per_connection = 4_096 + for index in range(4_096): + job_id = f"retained-{index}" + server.jobs[job_id] = prism_context(job_id, tip, worker=state.worker) + server.bury_evicted_job(state, job_id, now=100.0, prune=False) + + self.assertEqual(len(server.evicted_job_graveyard), 4_096) + self.assertEqual(len(server.evicted_same_tip_job_ids), 4_096) + classify_calls = 0 + original_classify = server._evicted_job_class_locked + + def counted_classify(entry: object) -> str: + nonlocal classify_calls + classify_calls += 1 + return original_classify(entry) + + server._evicted_job_class_locked = counted_classify # type: ignore[method-assign] + with patch("lab.prism.prism_coordinator.time.monotonic", return_value=101.0): + for _ in range(100): + self.assertIsNotNone( + server.evicted_job_entry(state, "retained-2048") + ) + + self.assertEqual(classify_calls, 100) + + def test_pool_width_does_not_evict_other_connections_retained_jobs(self) -> None: + tip = "00" * 32 + server, _state, _ledger = submit_coordinator(tip=tip) + server.current_tip_first_seen = (tip, None) + server.same_tip_job_retention_seconds = 30 + server.same_tip_job_retention_per_connection = 1 + clients: list[ClientState] = [] + for index in range(4_097): + state = client() + state.connection_id = index + 1 + state.worker = worker_identity(f"miner-{index}") + clients.append(state) + job_id = f"wide-{index}" + server.jobs[job_id] = prism_context(job_id, tip, worker=state.worker) + server.bury_evicted_job(state, job_id, now=100.0, prune=False) + + self.assertEqual(len(server.evicted_job_graveyard), 4_097) + self.assertIn("wide-0", server.evicted_job_graveyard) + self.assertIn("wide-4096", server.evicted_job_graveyard) + + replacement_id = "wide-0-replacement" + server.jobs[replacement_id] = prism_context( + replacement_id, + tip, + worker=clients[0].worker, + ) + server.bury_evicted_job( + clients[0], + replacement_id, + now=101.0, + prune=False, + ) + + self.assertNotIn("wide-0", server.evicted_job_graveyard) + self.assertIn("wide-1", server.evicted_job_graveyard) + self.assertIn(replacement_id, server.evicted_job_graveyard) + self.assertEqual(len(server.evicted_job_graveyard), 4_097) + self.assertEqual(server.evicted_job_capacity_eviction_counts["connection"], 1) + + def test_tip_flip_reanchors_retained_job_grace_to_client_delivery(self) -> None: + old_tip = "00" * 32 + new_tip = "11" * 32 + server, state, _ledger = submit_coordinator(tip=old_tip) + server.clients = {state} + server.current_tip_first_seen = (old_tip, None) + server.stale_grace_seconds = 3 + server.bury_evicted_job(state, "job-1", now=100.0) + + with patch("lab.prism.prism_coordinator.time.monotonic", return_value=120.0): + server.observe_tip_first_seen(new_tip) + + # Burial predates the flip by twenty seconds, but grace does not begin + # until this connection actually receives replacement work. + server.prune_evicted_job_graveyard(now=130.0) + self.assertIn("job-1", server.evicted_job_graveyard) + + state.tip_work_delivered = (new_tip, 130.0) + server.prune_evicted_job_graveyard(now=132.9) + self.assertIn("job-1", server.evicted_job_graveyard) + server.prune_evicted_job_graveyard(now=133.1) + self.assertNotIn("job-1", server.evicted_job_graveyard) + self.assertEqual(server.evicted_job_expiration_counts["stale_grace"], 1) + + def test_tip_flip_prunes_by_chain_parent_when_poller_skips_observed_tip(self) -> None: + observed_tip = "00" * 32 + intermediate_tip = "11" * 32 + current_tip = "22" * 32 + server, state, _ledger = submit_coordinator(tip=intermediate_tip) + server.clients = {state} + server.current_tip_first_seen = (observed_tip, None) + server.stale_grace_seconds = 3 + server.jobs["older-job"] = prism_context( + "older-job", + observed_tip, + worker=state.worker, + ) + server.bury_evicted_job(state, "older-job", now=100.0, prune=False) + server.bury_evicted_job(state, "job-1", now=110.0, prune=False) + server.rpc = ParentTipRpc(tip=current_tip, parent=intermediate_tip) + + with patch("lab.prism.prism_coordinator.time.monotonic", return_value=120.0): + server.observe_tip_first_seen(current_tip) + + # The poller's previous observation is not authoritative. Tip + # observation proactively loads the actual parent, drops older work, + # and preserves the intermediate-tip context that submit can credit. + self.assertNotIn("older-job", server.evicted_job_graveyard) + self.assertIn("job-1", server.evicted_job_graveyard) + self.assertEqual( + server.current_tip_parent_hash(current_tip), + intermediate_tip, + ) + entry = server.evicted_job_entry(state, "job-1") + self.assertIsNotNone(entry) + assert entry is not None + self.assertEqual( + server.evicted_submit_context(state, entry, current_tip), + (entry.context, PRISM_CREDIT_POLICY_STALE_GRACE), + ) + + def test_slow_parent_lookup_cannot_overwrite_newer_tip_parent_cache(self) -> None: + old_tip = "00" * 32 + old_parent = "ff" * 32 + new_tip = "11" * 32 + new_parent = old_tip + server = coordinator() + server.current_tip_first_seen = (old_tip, None) + server.current_tip_observation_sequence = 1 + server.current_tip_parent = None + + def overtake_parent_lookup(tip_hash: str) -> str: + self.assertEqual(tip_hash, old_tip) + with server.lock: + server.current_tip_first_seen = (new_tip, 100.0) + server.current_tip_observation_sequence = 2 + server.current_tip_parent = (new_tip, new_parent) + return old_parent + + server._fetch_tip_parent_hash = overtake_parent_lookup # type: ignore[method-assign] + + self.assertEqual(server.current_tip_parent_hash(old_tip), old_parent) + self.assertEqual(server.current_tip_parent, (new_tip, new_parent)) + + def test_retained_same_tip_duplicate_remains_duplicate_share(self) -> None: + tip = "00" * 32 + server, state, ledger = submit_coordinator(tip=tip) + server.current_tip_first_seen = (tip, None) + server.bury_evicted_job(state, "job-1") + server.jobs.pop("job-1") + state.active_job_ids.clear() + submission = SimpleNamespace( + header_hex="a3" * 80, + block_hash_hex="c3" * 32, + share_pass=True, + block_pass=False, + ) + params = ["miner-a", "job-1", "00" * 8, "00000001", "00000002"] + + with patch( + "lab.prism.prism_coordinator.direct_stratum.assemble_submission", + return_value=submission, + ): + self.assertFalse(server.handle_submit(state, params)) + with self.assertRaises(StratumError) as raised: + server.handle_submit(state, params) + + self.assertEqual(raised.exception.reason, PRISM_REJECTION_DUPLICATE_SHARE) + self.assertEqual(len(ledger.pending), 1) + self.assertIn("job-1", server.evicted_job_graveyard) + metrics = server.metrics_payload() + self.assertIn('qbit_prism_evicted_job_contexts{class="same_tip"} 1', metrics) + self.assertIn( + 'qbit_prism_evicted_job_submits_total{outcome="accepted_same_tip"} 1', + metrics, + ) + + def test_pool_closed_submit_rejects_before_any_share_accounting(self) -> None: + # Post-close submits must not inflate submitted totals (the + # stale-percent denominator), per-worker submitted counters, or the + # vardiff window; only the pool-closed rejection itself is recorded. + server, state, ledger = submit_coordinator() + server.accepted_block_count = 1 + server.max_blocks = 1 + state.vardiff_config = SimpleNamespace(enabled=True) + submitted_before = server.submitted_share_count + + with self.assertRaises(StratumError) as raised: + server.handle_submit( + state, + ["miner-a", "job-1", "00" * 8, "00000001", "00000002"], + ) + + self.assertEqual(raised.exception.reason, PRISM_REJECTION_POOL_CLOSED) + self.assertEqual(server.submitted_share_count, submitted_before) + # The rejection itself may admit the label, but no submission counted. + self.assertEqual(server.worker_share_counts["miner-a"]["submitted"], 0) + self.assertEqual(state.vardiff_window_submitted, 0) + self.assertEqual(len(ledger.pending), 0) + self.assertEqual( + server.worker_rejection_counts[("miner-a", PRISM_REJECTION_POOL_CLOSED)], 1 + ) + + def test_malformed_submit_does_not_diverge_worker_and_aggregate_submitted(self) -> None: + # A malformed-ntime submit must count identically in the per-worker and + # aggregate submitted counters (i.e. not at all) so the two never drift. + server, state, _ledger = submit_coordinator() + submitted_before = server.submitted_share_count + + with self.assertRaises(StratumError) as raised: + server.handle_submit( + state, + ["miner-a", "job-1", "00" * 8, "bad-ntime", "00000002"], + ) + + self.assertEqual(raised.exception.reason, PRISM_REJECTION_INVALID_NTIME_OR_NONCE) + self.assertEqual(server.submitted_share_count, submitted_before) + self.assertEqual(server.worker_share_counts["miner-a"]["submitted"], 0) + self.assertEqual( + server.worker_rejection_counts[("miner-a", PRISM_REJECTION_INVALID_NTIME_OR_NONCE)], + 1, + ) + + def test_stale_grace_closed_when_refresh_path_has_not_observed_tip(self) -> None: + # Only blockpoll/blockwait may open the grace window. If the refresh path + # has not anchored the new tip (current_tip_first_seen is None) and only + # this submit's getbestblockhash sees it, the prior-tip share must reject + # as stale-job -- not get credited from a submit-anchored window. + old_tip = "00" * 32 + new_tip = "11" * 32 + server, state, ledger = submit_coordinator(tip=old_tip) + rpc = ParentTipRpc(tip=new_tip, parent=old_tip) + server.rpc = rpc + server.current_tip_first_seen = None + server.stale_grace_seconds = 3 + submission = SimpleNamespace( + header_hex="aa" * 80, + block_hash_hex="cc" * 32, + share_pass=True, + block_pass=False, + ) + + with patch( + "lab.prism.prism_coordinator.direct_stratum.assemble_submission", + return_value=submission, + ): + with self.assertRaises(StratumError) as raised: + server.handle_submit( + state, + ["miner-a", "job-1", "00" * 8, "00000001", "00000002"], + ) + + self.assertEqual(raised.exception.reason, PRISM_REJECTION_STALE_JOB) + self.assertEqual(len(ledger.pending), 0) + self.assertEqual(rpc.submitblock_calls, 0) + # The submit must not have anchored the window either. + self.assertIsNone(server.current_tip_first_seen) + + def test_stale_grace_rejected_after_window_expires(self) -> None: + # This connection received current-tip work well outside the grace + # window; a prior-tip share arriving now must reject rather than be + # credited late. + old_tip = "00" * 32 + new_tip = "11" * 32 + server, state, ledger = submit_coordinator(tip=old_tip) + rpc = ParentTipRpc(tip=new_tip, parent=old_tip) + server.rpc = rpc + server.stale_grace_seconds = 3 + server.current_tip_first_seen = (new_tip, time.monotonic() - 10) + state.tip_work_delivered = (new_tip, time.monotonic() - 10) + submission = SimpleNamespace( + header_hex="ab" * 80, + block_hash_hex="ce" * 32, + share_pass=True, + block_pass=False, + ) + + with patch( + "lab.prism.prism_coordinator.direct_stratum.assemble_submission", + return_value=submission, + ): + with self.assertRaises(StratumError) as raised: + server.handle_submit( + state, + ["miner-a", "job-1", "00" * 8, "00000001", "00000002"], + ) + + self.assertEqual(raised.exception.reason, PRISM_REJECTION_STALE_JOB) + self.assertEqual(len(ledger.pending), 0) + self.assertEqual(rpc.submitblock_calls, 0) + + def test_stale_grace_open_until_connection_receives_new_tip_work(self) -> None: + # The refresh pass may be slow or aborted (reorg reconcile failure, + # transient build errors). Until THIS connection is sent current-tip + # work, its prior-tip shares are still in flight and must stay + # creditable even after the global first-seen stamp ages past the + # grace window. + old_tip = "00" * 32 + new_tip = "11" * 32 + server, state, ledger = submit_coordinator(tip=old_tip) + rpc = ParentTipRpc(tip=new_tip, parent=old_tip) + server.rpc = rpc + server.stale_grace_seconds = 3 + server.current_tip_first_seen = (new_tip, time.monotonic() - 10) + state.tip_work_delivered = (old_tip, time.monotonic() - 60) + submission = SimpleNamespace( + header_hex="ac" * 80, + block_hash_hex="cd" * 32, + share_pass=True, + block_pass=False, + ) + + with patch( + "lab.prism.prism_coordinator.direct_stratum.assemble_submission", + return_value=submission, + ): + should_close = server.handle_submit( + state, + ["miner-a", "job-1", "00" * 8, "00000001", "00000002"], + ) + + self.assertFalse(should_close) + self.assertEqual(rpc.submitblock_calls, 0) + self.assertEqual(len(ledger.pending), 1) + self.assertEqual(ledger.pending[0].credit_policy, PRISM_CREDIT_POLICY_STALE_GRACE) + + def test_stale_grace_window_runs_from_per_connection_delivery_not_first_seen(self) -> None: + # A slow refresh pass can deliver current-tip work to a connection + # after the global first-seen stamp has already aged past the grace + # window. The window for that connection runs from ITS delivery. + old_tip = "00" * 32 + new_tip = "11" * 32 + server, state, ledger = submit_coordinator(tip=old_tip) + rpc = ParentTipRpc(tip=new_tip, parent=old_tip) + server.rpc = rpc + server.stale_grace_seconds = 3 + server.current_tip_first_seen = (new_tip, time.monotonic() - 10) + state.tip_work_delivered = (new_tip, time.monotonic() - 1) + submission = SimpleNamespace( + header_hex="ae" * 80, + block_hash_hex="cb" * 32, + share_pass=True, + block_pass=False, + ) + + with patch( + "lab.prism.prism_coordinator.direct_stratum.assemble_submission", + return_value=submission, + ): + should_close = server.handle_submit( + state, + ["miner-a", "job-1", "00" * 8, "00000001", "00000002"], + ) + + self.assertFalse(should_close) + self.assertEqual(len(ledger.pending), 1) + self.assertEqual(ledger.pending[0].credit_policy, PRISM_CREDIT_POLICY_STALE_GRACE) + + def test_startup_baseline_tip_does_not_open_stale_grace_window(self) -> None: + # The first tip observed after process start is a baseline, not a tip + # flip: it must not open the grace window. A later real flip must. + old_tip = "00" * 32 + new_tip = "11" * 32 + server, state, _ledger = submit_coordinator(tip=old_tip) + server.stale_grace_seconds = 3 + server.current_tip_first_seen = None + + server.observe_tip_first_seen(new_tip) + self.assertEqual(server.current_tip_first_seen, (new_tip, None)) + self.assertFalse(server.stale_grace_deadline_open(state, new_tip)) + + # A change away from the observed baseline is a real flip and opens + # the window for connections that have not yet received the new work. + flip_tip = "22" * 32 + server.observe_tip_first_seen(flip_tip) + self.assertIsNotNone(server.current_tip_first_seen[1]) + self.assertTrue(server.stale_grace_deadline_open(state, flip_tip)) + + def test_note_tip_work_delivered_keeps_first_delivery_per_tip(self) -> None: + # Same-tip template refreshes must not slide the grace anchor forward. + server, state, _ledger = submit_coordinator() + tip = "11" * 32 + + server.note_tip_work_delivered(state, tip) + first = state.tip_work_delivered + self.assertEqual(first[0], tip) + server.note_tip_work_delivered(state, tip) + self.assertEqual(state.tip_work_delivered, first) + + # A new tip re-anchors. + server.note_tip_work_delivered(state, "22" * 32) + self.assertEqual(state.tip_work_delivered[0], "22" * 32) + self.assertGreaterEqual(state.tip_work_delivered[1], first[1]) + + def test_evicted_graveyard_keeps_unexpired_entries_above_previous_cap_for_grace_credit(self) -> None: + old_tip = "00" * 32 + new_tip = "11" * 32 + server, state, ledger = submit_coordinator(tip=old_tip) + rpc = ParentTipRpc(tip=new_tip, parent=old_tip) + server.rpc = rpc + server.current_tip_first_seen = (new_tip, time.monotonic()) + context = server.jobs["job-1"] + evicted_at = time.monotonic() + server.evicted_job_graveyard = { + "job-1": (context, state.connection_id, evicted_at), + } + previous_hard_cap = 512 + for index in range(previous_hard_cap): + server.evicted_job_graveyard[f"filler-{index}"] = ( + context, + state.connection_id, + evicted_at + 0.001 + (index / 1_000_000), + ) + server.prune_evicted_job_graveyard(now=evicted_at + 0.5) + self.assertIn("job-1", server.evicted_job_graveyard) + server.jobs.pop("job-1") + state.active_job_ids.clear() + submission = SimpleNamespace( + header_hex="ae" * 80, + block_hash_hex="ce" * 32, + share_pass=True, + block_pass=False, + ) + + with patch( + "lab.prism.prism_coordinator.direct_stratum.assemble_submission", + return_value=submission, + ): + should_close = server.handle_submit( + state, + ["miner-a", "job-1", "00" * 8, "00000001", "00000002"], + ) + + self.assertFalse(should_close) + self.assertEqual(len(ledger.pending), 1) + self.assertEqual(ledger.pending[0].credit_policy, PRISM_CREDIT_POLICY_STALE_GRACE) + + def test_stale_grace_parent_rpc_failure_rejects_as_backend_unavailable(self) -> None: + old_tip = "00" * 32 + new_tip = "11" * 32 + server, state, _ledger = submit_coordinator(tip=old_tip) + server.rpc = TipRpc(new_tip) + server.current_tip_first_seen = (new_tip, time.monotonic()) + + with self.assertRaises(StratumError) as raised: + server.handle_submit( + state, + ["miner-a", "job-1", "00" * 8, "00000001", "00000002"], + ) + + self.assertEqual(raised.exception.reason, PRISM_REJECTION_BACKEND_RPC_UNAVAILABLE) + + def test_unknown_job_rejects_before_getbestblockhash_rpc(self) -> None: + class CountingTipRpc(TipRpc): + def __init__(self, tip: str) -> None: + super().__init__(tip) + self.getbest_calls = 0 + + def call(self, method: str, params: list[object] | None = None) -> object: + if method == "getbestblockhash": + self.getbest_calls += 1 + return super().call(method, params) + + server, state, _ledger = submit_coordinator() + rpc = CountingTipRpc("00" * 32) + server.rpc = rpc + + with self.assertRaises(StratumError) as raised: + server.handle_submit( + state, + ["miner-a", "garbage-job", "00" * 8, "00000001", "00000002"], + ) + + self.assertEqual(raised.exception.reason, PRISM_REJECTION_UNKNOWN_JOB) + self.assertEqual(rpc.getbest_calls, 0) + + def test_submit_passes_negotiated_version_bits_and_mask_to_stratum_assembly(self) -> None: + server, state, _ledger = submit_coordinator() + state.version_mask = 0x1FFFE000 + submission = SimpleNamespace( + header_hex="ac" * 80, + block_hash_hex="ba" * 32, + share_pass=True, + block_pass=False, + ) + + with patch( + "lab.prism.prism_coordinator.direct_stratum.assemble_submission", + return_value=submission, + ) as assemble_submission: + server.handle_submit( + state, + ["miner-a", "job-1", "00" * 8, "00000001", "00000002", "00002000"], + ) + + self.assertEqual(assemble_submission.call_args.kwargs["version_bits_hex"], "00002000") + self.assertEqual(assemble_submission.call_args.kwargs["version_mask"], 0x1FFFE000) + + def test_address_worker_submit_accrues_to_base_payout_address(self) -> None: + server, state, ledger = submit_coordinator() + username = f"{PAYOUT_ADDRESS}.rig-a" + worker = WorkerIdentity( + username=username, + payout_address=PAYOUT_ADDRESS, + worker_name="rig-a", + script_pubkey_hex="5220" + "44" * 32, + p2mr_program_hex="44" * 32, + ) + state.username = username + state.worker = worker + server.jobs["job-1"].worker = worker + server.share_weights_by_username = {PAYOUT_ADDRESS: 9} + submission = SimpleNamespace( + header_hex="aa" * 80, + block_hash_hex="bc" * 32, + share_pass=True, + block_pass=False, + ) + + with patch( + "lab.prism.prism_coordinator.direct_stratum.assemble_submission", + return_value=submission, + ): + should_close = server.handle_submit( + state, + [username, "job-1", "00" * 8, "00000001", "00000002"], + ) + + self.assertFalse(should_close) + self.assertEqual(len(ledger.pending), 1) + self.assertEqual(ledger.pending[0].share_id, username + ":" + "bc" * 32) + self.assertEqual(ledger.pending[0].miner_id, PAYOUT_ADDRESS) + self.assertEqual(ledger.pending[0].order_key, PAYOUT_ADDRESS) + self.assertEqual(ledger.pending[0].share_difficulty, 9) + + def test_address_worker_submit_still_requires_authorized_full_username(self) -> None: + server, state, ledger = submit_coordinator() + username = f"{PAYOUT_ADDRESS}.rig-a" + worker = WorkerIdentity( + username=username, + payout_address=PAYOUT_ADDRESS, + worker_name="rig-a", + script_pubkey_hex="5220" + "44" * 32, + p2mr_program_hex="44" * 32, + ) + state.username = username + state.worker = worker + server.jobs["job-1"].worker = worker + + with self.assertRaises(StratumError) as raised: + server.handle_submit( + state, + [PAYOUT_ADDRESS, "job-1", "00" * 8, "00000001", "00000002"], + ) + + self.assertEqual(raised.exception.code, 20) + self.assertEqual(len(ledger.pending), 0) + + def test_stale_tip_rejects_without_appending_share(self) -> None: + old_tip = "00" * 32 + new_tip = "11" * 32 + server, state, ledger = submit_coordinator(tip=old_tip) + server.rpc = ParentTipRpc(tip=new_tip, parent="22" * 32) + + with self.assertRaises(StratumError) as raised: + server.handle_submit( + state, + ["miner-a", "job-1", "00" * 8, "00000001", "00000002"], + ) + + self.assertEqual(raised.exception.code, 21) + self.assertEqual(raised.exception.reason, PRISM_REJECTION_STALE_JOB) + self.assertEqual(len(ledger.pending), 0) + self.assertEqual(server.stale_share_count, 1) + self.assertEqual(server.rejection_counts_by_reason[PRISM_REJECTION_STALE_JOB], 1) + + def _pending_append(self, tag: str, accepted_at_ms: int = 2) -> PendingShareAppend: + from lab.prism.share_ledger import PendingShare + + return PendingShareAppend( + pending_share=PendingShare( + share_id=f"miner-a:{tag}", + miner_id="miner-a", + order_key="miner-a", + p2mr_program_hex="11" * 32, + share_difficulty=1, + network_difficulty=1, + template_height=10, + job_id="job-1", + job_issued_at_ms=1, + accepted_at_ms=accepted_at_ms, + ntime=1_700_000_000, + ), + username="miner-a", + job_id="job-1", + block_hash_hex=tag * 32, + collection_only=False, + credit_policy=None, + ) + + def test_reconnected_same_username_submits_against_disconnected_job(self) -> None: + # A proxy flap leaves devices mining the dead connection's jobs; + # their shares arrive on the replacement connection and must credit + # against the retained context instead of rejecting unknown-job. + tip = "00" * 32 + server, state, ledger = submit_coordinator(tip=tip) + server.clients = {state} + state.close = lambda: None # type: ignore[method-assign] + server.disconnect_client(state) + retained = server.evicted_job_graveyard.get("job-1") + self.assertIsNotNone(retained) + assert retained is not None + self.assertIsNone(retained.client) + + reconnected = ClientState( + sock=object(), + address=("127.0.0.1", 2), + connection_id=2, + extranonce1_hex="00000002", + ) + reconnected.subscribed = True + reconnected.authorized = True + reconnected.username = "miner-a" + reconnected.worker = worker_identity("miner-a") + submission = SimpleNamespace( + header_hex="af" * 80, + block_hash_hex="cf" * 32, + share_pass=True, + block_pass=False, + ) + + with patch( + "lab.prism.prism_coordinator.direct_stratum.assemble_submission", + return_value=submission, + ): + should_close = server.handle_submit( + reconnected, + ["miner-a", "job-1", "00" * 8, "00000001", "00000002"], + ) + + self.assertFalse(should_close) + self.assertEqual(len(ledger.pending), 1) + self.assertIsNone(ledger.pending[0].credit_policy) + # Credit follows the retained context's original worker identity. + self.assertEqual(ledger.pending[0].share_id, "miner-a:" + "cf" * 32) + self.assertEqual( + server.evicted_job_submit_counts[ + "accepted_same_tip_cross_connection" + ], + 1, + ) + + def test_cross_connection_submit_requires_same_username(self) -> None: + tip = "00" * 32 + server, state, _ledger = submit_coordinator(tip=tip) + server.clients = {state} + state.close = lambda: None # type: ignore[method-assign] + server.disconnect_client(state) + self.assertIn("job-1", server.evicted_job_graveyard) + + intruder = ClientState( + sock=object(), + address=("127.0.0.1", 3), + connection_id=3, + extranonce1_hex="00000003", + ) + intruder.subscribed = True + intruder.authorized = True + intruder.username = "miner-b" + intruder.worker = worker_identity("miner-b") + submission = SimpleNamespace( + header_hex="af" * 80, + block_hash_hex="cf" * 32, + share_pass=True, + block_pass=False, + ) + + with patch( + "lab.prism.prism_coordinator.direct_stratum.assemble_submission", + return_value=submission, + ): + with self.assertRaises(StratumError) as raised: + server.handle_submit( + intruder, + ["miner-b", "job-1", "00" * 8, "00000001", "00000002"], + ) + + self.assertEqual(raised.exception.reason, PRISM_REJECTION_UNKNOWN_JOB) + + def test_cross_connection_submit_uses_retained_version_mask(self) -> None: + # In-flight work from the dead connection was version-rolled under + # the mask negotiated there; the replacement's own mask (0 before + # mining.configure) must not judge those version bits. + tip = "00" * 32 + server, state, _ledger = submit_coordinator(tip=tip) + server.jobs["job-1"].version_mask = 0x1FFFE000 + server.clients = {state} + state.close = lambda: None # type: ignore[method-assign] + server.disconnect_client(state) + + reconnected = ClientState( + sock=object(), + address=("127.0.0.1", 2), + connection_id=2, + extranonce1_hex="00000002", + ) + reconnected.subscribed = True + reconnected.authorized = True + reconnected.username = "miner-a" + reconnected.worker = worker_identity("miner-a") + self.assertEqual(reconnected.version_mask, 0) + submission = SimpleNamespace( + header_hex="af" * 80, + block_hash_hex="cf" * 32, + share_pass=True, + block_pass=False, + ) + + with patch( + "lab.prism.prism_coordinator.direct_stratum.assemble_submission", + return_value=submission, + ) as assemble: + self.assertFalse( + server.handle_submit( + reconnected, + [ + "miner-a", + "job-1", + "00" * 8, + "00000001", + "00000002", + "1fffe000", + ], + ) + ) + + self.assertEqual( + assemble.call_args.kwargs["version_mask"], + 0x1FFFE000, + ) + + def test_cross_connection_block_candidate_keeps_original_extranonce(self) -> None: + # The mined coinbase embeds the extranonce1 the retained job was + # stamped with; a candidate recording the replacement connection's + # extranonce would fail the audit fence after submitblock. + tip = "00" * 32 + server, state, _ledger = submit_coordinator(tip=tip) + server.stop_after_block = False + server.max_blocks = 10 + server.jobs["job-1"].job.extranonce1_hex = state.extranonce1_hex + server.clients = {state} + state.close = lambda: None # type: ignore[method-assign] + server.disconnect_client(state) + + reconnected = ClientState( + sock=object(), + address=("127.0.0.1", 2), + connection_id=2, + extranonce1_hex="00000002", + ) + reconnected.subscribed = True + reconnected.authorized = True + reconnected.username = "miner-a" + reconnected.worker = worker_identity("miner-a") + submission = SimpleNamespace( + header_hex="af" * 80, + block_hash_hex="cf" * 32, + share_pass=True, + block_pass=True, + coinbase_tx_hex="c0ffee", + block_hex="00", + ) + + with patch( + "lab.prism.prism_coordinator.direct_stratum.assemble_submission", + return_value=submission, + ): + self.assertFalse( + server.handle_submit( + reconnected, + ["miner-a", "job-1", "00" * 8, "00000001", "00000002"], + ) + ) + + self.assertEqual(server.block_candidate_queue.qsize(), 1) + candidate = server.block_candidate_queue.get_nowait() + self.assertEqual(candidate.extranonce1_hex, "00000001") + self.assertIs(candidate.client, reconnected) + + def test_detached_entry_never_falls_back_to_stale_grace(self) -> None: + # A tip moving between the graveyard lookup and submit + # classification must not let a detached cross-connection entry + # borrow the reconnect's stale-grace window: cross-connection + # resumes are same-tip only. + tip_old = "00" * 32 + tip_new = "11" * 32 + server, state, _ledger = submit_coordinator(tip=tip_old) + server.clients = {state} + state.close = lambda: None # type: ignore[method-assign] + server.disconnect_client(state) + detached = server.evicted_job_graveyard.get("job-1") + self.assertIsNotNone(detached) + assert detached is not None + self.assertIsNone(detached.client) + + reconnected = ClientState( + sock=object(), + address=("127.0.0.1", 2), + connection_id=2, + extranonce1_hex="00000002", + ) + reconnected.subscribed = True + reconnected.authorized = True + reconnected.username = "miner-a" + reconnected.worker = worker_identity("miner-a") + eligibility_calls: list[str] = [] + + def recording_eligibility( + _client: ClientState, + _context: object, + current_tip: str, + ) -> bool: + eligibility_calls.append(current_tip) + return True + + server.context_eligible_for_stale_grace = ( # type: ignore[method-assign] + recording_eligibility + ) + + self.assertIsNone( + server.evicted_submit_context(reconnected, detached, tip_new) + ) + self.assertEqual(eligibility_calls, []) + + # Same-connection entries keep today's stale-grace classification. + live_state = client() + server.jobs["job-2"] = prism_context( + "job-2", + tip_old, + worker=live_state.worker, + ) + server.bury_evicted_job(live_state, "job-2") + live_entry = server.evicted_job_graveyard["job-2"] + result = server.evicted_submit_context(live_state, live_entry, tip_new) + self.assertIsNotNone(result) + assert result is not None + self.assertEqual(result[1], PRISM_CREDIT_POLICY_STALE_GRACE) + self.assertEqual(eligibility_calls, [tip_new]) + + def test_disconnected_retention_capacity_is_bounded(self) -> None: + tip = "00" * 32 + server, state, _ledger = submit_coordinator(tip=tip) + server.disconnected_job_retention = 1 + server.jobs["job-2"] = prism_context("job-2", tip, worker=state.worker) + state.active_job_ids = {"job-1", "job-2"} + server.clients = {state} + state.close = lambda: None # type: ignore[method-assign] + + server.disconnect_client(state) + + self.assertEqual(len(server._disconnected_evicted_job_ids), 1) + self.assertEqual(len(server.evicted_job_graveyard), 1) + self.assertEqual( + server.evicted_job_capacity_eviction_counts["disconnected"], + 1, + ) + + def test_tip_change_prunes_and_disconnect_detaches_retained_contexts(self) -> None: + old_tip = "00" * 32 + new_tip = "11" * 32 + server, state, _ledger = submit_coordinator(tip=old_tip) + server.current_tip_first_seen = (old_tip, None) + server.stale_grace_seconds = 0 + server.bury_evicted_job(state, "job-1") + self.assertIn("job-1", server.evicted_job_graveyard) + + server.observe_tip_first_seen(new_tip) + self.assertNotIn("job-1", server.evicted_job_graveyard) + + server.current_tip_first_seen = (new_tip, None) + server.jobs["job-2"] = prism_context("job-2", new_tip, worker=state.worker) + server.bury_evicted_job(state, "job-2") + server.clients = {state} + state.close = lambda: None # type: ignore[method-assign] + server.disconnect_client(state) + # Same-tip work survives the disconnect for a same-username + # reconnect; the dead connection's entry is detached and ages out on + # the normal same-tip TTL instead of vanishing with the socket. + retained = server.evicted_job_graveyard.get("job-2") + self.assertIsNotNone(retained) + assert retained is not None + self.assertIsNone(retained.client) + server.prune_evicted_job_graveyard( + now=time.monotonic() + + float(server.same_tip_job_retention_seconds) + + 1.0 + ) + self.assertEqual(server.evicted_job_graveyard, {}) diff --git a/tests/test_prism_self_check.py b/tests/test_prism_self_check.py index a0cffe9ab..bbb05b9a9 100644 --- a/tests/test_prism_self_check.py +++ b/tests/test_prism_self_check.py @@ -38,7 +38,7 @@ def authorize_mainnet_prelaunch(env: dict[str, str]) -> dict[str, str]: "CKPOOL_NON_TEST_READINESS_GATE": "0", "QBIT_MAINNET_LAUNCH_READINESS_CHECKS_ENABLED": "0", "PRISM_POSTGRES_PASSWORD": "not-default", - "PRISM_STRATUM_STALE_GRACE_SECONDS": "0", + "PRISM_STRATUM_STALE_GRACE_SECONDS": "3", } ) return env @@ -360,7 +360,7 @@ def test_mainnet_implies_production_safeguards(self) -> None: "QBIT_CHAIN_FLAG": "-chain=main", "QBIT_EXPECTED_GENESIS_HASH": "11" * 32, "QBIT_GIT_COMMIT": "41" * 20, - "PRISM_STRATUM_STALE_GRACE_SECONDS": "0", + "PRISM_STRATUM_STALE_GRACE_SECONDS": "3", "PRISM_ALLOW_TEST_SIGNING_SEEDS": "1", } ) @@ -395,7 +395,7 @@ def test_static_checks_accept_mainnet_chain_selector_and_genesis(self) -> None: "QBIT_CHAIN_FLAG": "-chain=main", "QBIT_EXPECTED_GENESIS_HASH": "AB" * 32, "QBIT_GIT_COMMIT": "41" * 20, - "PRISM_STRATUM_STALE_GRACE_SECONDS": "0", + "PRISM_STRATUM_STALE_GRACE_SECONDS": "3", } ) reporter = self.self_check.Reporter() @@ -419,7 +419,7 @@ def test_static_checks_require_immutable_production_git_commit(self) -> None: { "QBIT_PRODUCTION": "1", "QBIT_GIT_COMMIT": commit, - "PRISM_STRATUM_STALE_GRACE_SECONDS": "0", + "PRISM_STRATUM_STALE_GRACE_SECONDS": "3", } ) reporter = self.self_check.Reporter() @@ -437,7 +437,7 @@ def test_static_checks_accept_immutable_production_git_commit(self) -> None: { "QBIT_PRODUCTION": "1", "QBIT_GIT_COMMIT": "AB" * 20, - "PRISM_STRATUM_STALE_GRACE_SECONDS": "0", + "PRISM_STRATUM_STALE_GRACE_SECONDS": "3", } ) reporter = self.self_check.Reporter() @@ -460,7 +460,7 @@ def test_static_checks_do_not_include_capacity_evidence(self) -> None: "QBIT_PRODUCTION": "1", "QBIT_GIT_COMMIT": "41" * 20, "PRISM_POSTGRES_PASSWORD": "not-default", - "PRISM_STRATUM_STALE_GRACE_SECONDS": "0", + "PRISM_STRATUM_STALE_GRACE_SECONDS": "3", "PRISM_STRATUM_SHARE_DIFF": "1024", "PRISM_STRATUM_VARDIFF_MIN_DIFF": "1024", "PRISM_STRATUM_VARDIFF_START_DIFF": "4096", @@ -484,7 +484,7 @@ def test_static_checks_mainnet_does_not_require_capacity_evidence(self) -> None: "QBIT_EXPECTED_GENESIS_HASH": "AB" * 32, "QBIT_GIT_COMMIT": "41" * 20, "QBIT_REQUIRE_RELEASE_PROVENANCE": "0", - "PRISM_STRATUM_STALE_GRACE_SECONDS": "0", + "PRISM_STRATUM_STALE_GRACE_SECONDS": "3", "PRISM_STRATUM_SHARE_DIFF": "1024", "PRISM_STRATUM_VARDIFF_MIN_DIFF": "1024", "PRISM_STRATUM_VARDIFF_START_DIFF": "4096", @@ -503,7 +503,7 @@ def test_static_checks_reject_lab_production_difficulty(self) -> None: { "QBIT_PRODUCTION": "1", "QBIT_GIT_COMMIT": "41" * 20, - "PRISM_STRATUM_STALE_GRACE_SECONDS": "0", + "PRISM_STRATUM_STALE_GRACE_SECONDS": "3", } ) reporter = self.self_check.Reporter() @@ -515,7 +515,10 @@ def test_static_checks_reject_lab_production_difficulty(self) -> None: {(row.status, row.name) for row in reporter.rows}, ) - def test_static_checks_require_zero_mainnet_stale_grace(self) -> None: + def test_static_checks_accept_bounded_mainnet_stale_grace(self) -> None: + # Mainnet follows the same bounded-grace rule as every other chain: + # zero grace rejects every in-flight prior-tip share at each block, + # which miners read as pool failure. env = self.valid_env() env.update( { @@ -530,10 +533,9 @@ def test_static_checks_require_zero_mainnet_stale_grace(self) -> None: self.self_check.static_checks(env, reporter) - self.assertIn( - ("FAIL", "mining.stale_grace"), - {(row.status, row.name) for row in reporter.rows}, - ) + rows = {(row.status, row.name) for row in reporter.rows} + self.assertIn(("PASS", "mining.stale_grace"), rows) + self.assertNotIn(("FAIL", "mining.stale_grace"), rows) def test_static_checks_accept_bounded_stale_grace_off_mainnet(self) -> None: env = self.valid_env() @@ -552,13 +554,38 @@ def test_static_checks_accept_bounded_stale_grace_off_mainnet(self) -> None: self.assertIn(("PASS", "mining.stale_grace"), rows) self.assertNotIn(("FAIL", "mining.stale_grace"), rows) + def test_static_checks_parse_stale_grace_with_runtime_float_semantics(self) -> None: + # The coordinator parses the window with env_nonnegative_float(), so a + # fractional window is valid while negative or non-finite values are not. + for grace, expected in ( + ("0.5", "PASS"), + ("0", "PASS"), + ("1e-1", "PASS"), + (".5", "PASS"), + ("3.", "PASS"), + ("-1", "FAIL"), + ("nan", "FAIL"), + ("inf", "FAIL"), + ("1" + "0" * 400, "FAIL"), + ("abc", "FAIL"), + ): + with self.subTest(grace=grace): + env = self.valid_env() + env["PRISM_STRATUM_STALE_GRACE_SECONDS"] = grace + reporter = self.self_check.Reporter() + + self.self_check.static_checks(env, reporter) + + rows = {(row.status, row.name) for row in reporter.rows} + self.assertIn((expected, "mining.stale_grace"), rows) + def test_static_checks_require_explicit_production_auxpow_payouts(self) -> None: env = self.valid_env() env.update( { "QBIT_PRODUCTION": "1", "QBIT_GIT_COMMIT": "41" * 20, - "PRISM_STRATUM_STALE_GRACE_SECONDS": "0", + "PRISM_STRATUM_STALE_GRACE_SECONDS": "3", "BITCOIN_CHAIN": "mainnet", "BITCOIN_CHAIN_FLAG": "-chain=main", "QBIT_MINER_ADDRESS": "auto", @@ -579,7 +606,7 @@ def test_static_checks_accept_explicit_production_auxpow_payouts(self) -> None: { "QBIT_PRODUCTION": "1", "QBIT_GIT_COMMIT": "41" * 20, - "PRISM_STRATUM_STALE_GRACE_SECONDS": "0", + "PRISM_STRATUM_STALE_GRACE_SECONDS": "3", "BITCOIN_CHAIN": "mainnet", "BITCOIN_CHAIN_FLAG": "-chain=main", "QBIT_MINER_ADDRESS": "qb1explicit", diff --git a/tests/test_prism_share_ledger.py b/tests/test_prism_share_ledger.py index 62e3a2339..4c5a9faf0 100644 --- a/tests/test_prism_share_ledger.py +++ b/tests/test_prism_share_ledger.py @@ -4,14 +4,18 @@ import contextlib import gc +import gzip import hashlib import io import json +import os +import subprocess import sys import tempfile import threading import time import tracemalloc +import types import unittest import unittest.mock @@ -23,22 +27,35 @@ from lab.prism.backfill_ctv_fanouts import backfill_input_from_path, backfill_input_from_payload, infer_block_hash_from_path from lab.prism import public_api from lab.prism import share_ledger as share_ledger_module +from lab.prism.audit_artifacts import ( + AuditArtifactConfig, + AuditArtifactStore, + CanonicalAuditBundleCorrupt, +) from lab.prism.share_ledger import ( database_url_from_psql_command, parse_single_json_value, AUDIT_BODY_REF_SCHEMA, AUDIT_BUNDLE_V2_SCHEMA, + AcceptedShareRecord, PendingShare, PsqlShareLedger, + ShareReplayConflict, + ShareReplayResult, LedgerOperationTimeout, + PAYOUT_WINDOW_ROW_BATCH_SIZE, AUDIT_WINDOW_COMPLETENESS_PROOF_SCHEMA, DEFAULT_WRITER_LEASE_ADOPTION_SILENCE_SECONDS, SingleWriterShareLedger, + WRITER_LEASE_ACQUIRE_RETRY_ATTEMPTS, + WRITER_LEASE_ACQUIRE_RETRY_KEY, + WRITER_LEASE_ACQUIRE_RETRY_SUBJECT, WRITER_LEASE_HEARTBEAT_SESSION_PREFIX, _NativePostgresClient, _NativePostgresLeaseGuard, _prism_window_shares, _writer_lease_advisory_lock_key, + canonical_json_text, sha256_json_hex, ) @@ -161,6 +178,24 @@ def held_lease( return result +def snapshot_retry_lease() -> dict[str, object]: + """The acquisition statement's third arm, as PostgreSQL renders it. + + Built from the same constants the statement interpolates, so a change to + either the key or the subject shows up as a failure in the retry tests + rather than as a sentinel the ledger silently stops recognising. + """ + return { + "acquired": False, + WRITER_LEASE_ACQUIRE_RETRY_KEY: True, + "lease": WRITER_LEASE_ACQUIRE_RETRY_SUBJECT, + "retry_reason": ( + f"the {WRITER_LEASE_ACQUIRE_RETRY_SUBJECT} row was committed by a " + "concurrent first acquisition after this statement snapshot" + ), + } + + class FakeMonotonicClock: """Deterministic stand-in for time.monotonic in lease-adoption tests. @@ -179,6 +214,29 @@ def sleep(self, seconds: float) -> None: self.now += seconds +# Fixed hashes for the bulk terminal-update cases, named for the role each +# row plays in a page rather than for its bytes. +BULK_PENDING_A = "1a" * 32 +BULK_PENDING_B = "1b" * 32 +BULK_SUBMITTED = "1c" * 32 +BULK_MISSING = "99" * 32 + + +class CountingLock: + """Context-manager lock proxy that counts how often it is entered.""" + + def __init__(self, inner: Any) -> None: + self._inner = inner + self.acquisitions = 0 + + def __enter__(self) -> Any: + self.acquisitions += 1 + return self._inner.__enter__() + + def __exit__(self, *exc_info: Any) -> Any: + return self._inner.__exit__(*exc_info) + + class FakeLeasePsqlShareLedger(PsqlShareLedger): def __init__(self, lease_results: list[dict[str, object]], **kwargs: Any): self.lease_results = list(lease_results) @@ -189,7 +247,15 @@ def __init__(self, lease_results: list[dict[str, object]], **kwargs: Any): super().__init__( psql_command="psql postgresql://example.invalid/qbit", lease_retry_sleep=retry_sleep, - lease_retry_max_sleep_seconds=1.0, + # One retry sleep must be able to cover a whole adoption silence, + # or a scenario that waits out the silence is split into several + # capped sleeps and consumes lease observations the fixture never + # scripted. Derived from the policy so retuning the silence does + # not silently change what these scenarios exercise. + lease_retry_max_sleep_seconds=max( + 1.0, + DEFAULT_WRITER_LEASE_ADOPTION_SILENCE_SECONDS, + ), **kwargs, ) @@ -225,6 +291,39 @@ def _run_json(self, sql: str) -> Any: return [] +class GateStatementRecordingPsqlShareLedger(PsqlShareLedger): + """Ledger whose gate-held statements and progress stamps interleave. + + Built for the multi-statement gate bodies (confirm/reactivate/carry-forward + integrity), where the ordering of the stamp relative to each statement is + the property under test, not merely that a stamp happened. + """ + + def __init__(self, results: list[object]) -> None: + self._lock = threading.Lock() + self._read_semaphore = threading.BoundedSemaphore(1) + self._operation_timeout_local = threading.local() + self._statement_timeout_local = threading.local() + self._operation_progress_local = threading.local() + self._writer_id = "writer-a" + self._writer_epoch = 1 + self._writer_session_token = "session-a" + self._lease_interval_sql = "make_interval(secs => 30)" + self._pending_results = list(results) + self.events: list[str] = [] + self.statements = 0 + + def note_progress(self) -> None: + self.events.append("progress") + + def _run_json(self, sql: str) -> Any: + self.statements += 1 + self.events.append(f"statement-{self.statements}") + if not self._pending_results: + raise AssertionError(f"unexpected extra statement: {sql[:80]}") + return self._pending_results.pop(0) + + class BlockingReadPsqlShareLedger(PsqlShareLedger): def __init__(self, *, read_concurrency: int) -> None: self._lock = threading.Lock() @@ -260,6 +359,212 @@ def release_reads(self) -> None: class PrismShareLedgerTests(unittest.TestCase): + def test_psql_private_audit_delegates_and_subclass_override_route_through_a1(self) -> None: + ledger = PsqlShareLedger.__new__(PsqlShareLedger) + store = unittest.mock.Mock() + ledger._audit_artifact_store = store + ledger._audit_bundle_canonicalizer = lambda _bundle: b"legacy" + bundle = {"schema": "test"} + body_path = Path("body.json") + part = {"kind": "segment"} + recovery = {"load_missing_range": ledger._load_audit_share_ledger_range} + cases = ( + ( + "_externalize_audit_body", + ("aa" * 32, "bb" * 32, bundle), + {}, + "externalize_audit_body", + {}, + ), + ( + "_canonical_audit_body_bytes_for_sha", + (bundle, "bb" * 32), + {}, + "canonical_audit_body_bytes_for_sha", + {}, + ), + ("_audit_body_ref", (), {"payload": bundle}, "audit_body_ref", {}), + ("_audit_bundle_v2", (), {"payload": bundle}, "audit_bundle_v2", recovery), + ("_audit_share_parts", ([{"share_seq": 1}],), {}, "audit_share_parts", {}), + ( + "_audit_share_range_parts", + ([{"share_seq": 1}],), + {}, + "audit_share_range_parts", + recovery, + ), + ( + "_audit_share_segment_payload", + (), + {"shares": [{"share_seq": 1}]}, + "audit_share_segment_payload", + {}, + ), + ( + "_write_audit_share_segment", + (), + {"shares": [{"share_seq": 1}]}, + "write_audit_share_segment", + {}, + ), + ( + "_write_audit_share_segment_range", + (), + {"shares": [{"share_seq": 1}]}, + "write_audit_share_segment_range", + recovery, + ), + ( + "_merge_audit_share_ranges", + ([{"share_seq": 1}], [{"share_seq": 2}]), + {"segment_path": body_path}, + "merge_audit_share_ranges", + recovery, + ), + ( + "_audit_shares_by_seq", + ([{"share_seq": 1}],), + {"segment_path": body_path}, + "audit_shares_by_seq", + {"require_contiguous": True}, + ), + ("_storage_json_bytes", (bundle,), {}, "storage_json_bytes", {}), + ( + "_canonical_audit_bundle_bytes", + (bundle,), + {}, + "canonical_audit_bundle_bytes", + {}, + ), + ( + "_audit_body_path", + ("aa" * 32, "bb" * 32), + {}, + "body_path", + {}, + ), + ("_resolve_audit_body_path", (body_path,), {}, "resolve_owned_path", {}), + ( + "_audit_body_byte_len", + (body_path, bundle, body_path), + {}, + "audit_body_byte_len", + {}, + ), + ( + "_read_external_body", + (body_path,), + {"expected_sha256": "bb" * 32}, + "read_external_body", + {}, + ), + ( + "_external_body_matches_sha", + (body_path, "bb" * 32), + {}, + "external_body_matches_sha", + {}, + ), + ( + "_external_body_available_for_sha", + (body_path, "bb" * 32), + {}, + "external_body_available_for_sha", + {}, + ), + ( + "_resolve_audit_body_ref", + (bundle,), + {"expected_sha256": "bb" * 32, "body_uri": body_path}, + "resolve_audit_body_ref", + {}, + ), + ( + "_resolve_audit_bundle_v2", + (bundle,), + {"expected_sha256": "bb" * 32, "body_uri": body_path}, + "resolve_audit_bundle_v2", + {}, + ), + ( + "_read_audit_share_segment", + (part,), + {"parent_body_uri": body_path}, + "read_audit_share_segment", + {}, + ), + ( + "_select_audit_share_segment_range", + ([{"share_seq": 1}],), + { + "first_share_seq": 1, + "last_share_seq": 1, + "parent_body_uri": body_path, + "body_uri": body_path, + }, + "select_audit_share_segment_range", + {}, + ), + ) + token = object() + for wrapper_name, args, kwargs, target_name, extra_kwargs in cases: + with self.subTest(wrapper=wrapper_name): + target = getattr(store, target_name) + target.reset_mock() + target.return_value = token + self.assertIs(getattr(ledger, wrapper_name)(*args, **kwargs), token) + target.assert_called_once_with(*args, **{**kwargs, **extra_kwargs}) + self.assertIs(ledger._audit_store(), store) + + store.read_audit_share_segment.reset_mock() + self.assertTrue( + ledger._audit_share_segment_available( + part, + parent_body_uri=body_path, + ) + ) + store.read_audit_share_segment.assert_called_once_with( + part, + parent_body_uri=body_path, + ) + + override_calls: list[tuple[object, object]] = [] + ledger._read_external_body = ( # type: ignore[method-assign] + lambda body_uri, *, expected_sha256=None: ( + override_calls.append((body_uri, expected_sha256)) + or {"resolved": True} + ) + ) + self.assertEqual( + ledger._resolve_audit_bundle_row( + { + "audit_bundle": None, + "body_uri": "owned-body", + "audit_bundle_sha256": "bb" * 32, + } + ), + { + "audit_bundle": {"resolved": True}, + "audit_bundle_sha256": "bb" * 32, + }, + ) + self.assertEqual(override_calls, [("owned-body", "bb" * 32)]) + + def test_memory_recovery_append_distinguishes_insert_exact_and_conflict(self) -> None: + ledger = SingleWriterShareLedger() + share = pending_share(1) + + inserted = ledger.append_recovered_share(share) + exact = ledger.append_recovered_share(share) + + self.assertIsInstance(inserted, ShareReplayResult) + self.assertEqual(inserted.disposition, "inserted") + self.assertEqual(exact.disposition, "exact_existing") + with self.assertRaises(ShareReplayConflict): + ledger.append_recovered_share( + PendingShare(**{**share.__dict__, "ntime": share.ntime + 1}) + ) + def test_writer_lease_advisory_guard_is_stable_and_session_scoped(self) -> None: statements: list[str] = [] connect_kwargs: dict[str, object] = {} @@ -452,10 +757,23 @@ def test_share_and_block_candidate_intent_commit_atomically(self) -> None: self.assertTrue(records[0].newly_inserted) self.assertEqual(records[0].candidate_outbox_state, "pending") self.assertEqual(ledger.pending_block_candidates(), [intent]) + rows = ledger.pending_block_candidate_rows() self.assertEqual( - ledger.pending_block_candidate_rows(), - [{"block_hash": "ab" * 32, "candidate": intent}], + [ + {key: value for key, value in row.items() if key != "cursor"} + for row in rows + ], + [ + { + "block_hash": "ab" * 32, + "candidate": intent, + "pool_block_exists": False, + } + ], ) + # The pagination cursor is opaque to callers but always keyed on the + # row it came from. + self.assertEqual(rows[0]["cursor"][1], "ab" * 32) # Exact replay returns the original row and does not duplicate outbox # work. A changed intent with the same hash is rejected as corruption. replay = ledger.append_batch([(share, intent)])[0] @@ -503,6 +821,337 @@ def test_pending_candidate_age_distinguishes_first_attempt(self) -> None: 1, ) + def _seeded_bulk_abandon_ledger(self) -> SingleWriterShareLedger: + """One pending pair, one already-terminal row, one never persisted.""" + ledger = SingleWriterShareLedger() + for block_hash in (BULK_PENDING_A, BULK_PENDING_B, BULK_SUBMITTED): + ledger.persist_block_candidate_intent( + { + "schema": "qbit.prism.block-candidate-intent.v1", + "block_hash_hex": block_hash, + "block_hex": "00", + } + ) + self.assertTrue( + ledger.mark_block_candidate_submitted(block_hash=BULK_SUBMITTED) + ) + return ledger + + def test_bulk_candidate_abandon_reports_only_the_rows_it_transitioned( + self, + ) -> None: + """The report is the caller's cleanup authority, not an echo. + + A page handed to the bulk update is a candidate set, not a decided + set: rows can already be terminal, or absent entirely. Returning the + rows this call actually moved -- rather than a count or the request + -- is what lets the caller confine its follow-up cleanup to rows it + won, so it is asserted against a page deliberately containing both + kinds of non-transition. + """ + ledger = self._seeded_bulk_abandon_ledger() + + abandoned = ledger.mark_block_candidates_abandoned( + block_hashes=[ + BULK_PENDING_B, + BULK_PENDING_A.upper(), + BULK_PENDING_A, + BULK_SUBMITTED, + BULK_MISSING, + ], + error="superseded by decided height", + ) + + # Duplicates and case collapse; ordering is the normalized set's, not + # the caller's, so one input set always reports one identical result. + self.assertEqual(abandoned, (BULK_PENDING_A, BULK_PENDING_B)) + for block_hash in (BULK_PENDING_A, BULK_PENDING_B): + row = ledger._block_candidate_outbox[block_hash] + self.assertEqual(row["state"], "abandoned") + self.assertEqual(row["last_error"], "superseded by decided height") + self.assertIsNone(row["candidate"]) + # An already-terminal row is neither reported nor rewritten, so its + # own terminal reason survives a page that happens to name it. + submitted = ledger._block_candidate_outbox[BULK_SUBMITTED] + self.assertEqual(submitted["state"], "submitted") + self.assertIsNone(submitted["last_error"]) + self.assertNotIn(BULK_MISSING, ledger._block_candidate_outbox) + self.assertEqual(ledger.pending_block_candidates(), []) + + # Re-running the same page wins nothing: the rows are terminal now. + self.assertEqual( + ledger.mark_block_candidates_abandoned( + block_hashes=[BULK_PENDING_A, BULK_PENDING_B], + error="second attempt", + ), + (), + ) + self.assertEqual( + ledger._block_candidate_outbox[BULK_PENDING_A]["last_error"], + "superseded by decided height", + ) + + def test_bulk_candidate_abandon_on_an_empty_page_touches_nothing(self) -> None: + ledger = self._seeded_bulk_abandon_ledger() + + self.assertEqual( + ledger.mark_block_candidates_abandoned(block_hashes=[], error="unused"), + (), + ) + self.assertEqual( + ledger._block_candidate_outbox[BULK_PENDING_A]["state"], + "pending", + ) + self.assertEqual(len(ledger.pending_block_candidates()), 2) + + def test_pending_candidate_page_reports_pool_block_existence(self) -> None: + """The page read answers the landed-block question, not a per-row call. + + Both backends carry the fact on the row itself so a caller reading a + page never has to ask once per hash. The memory mirror reads the same + durable evidence the Postgres ``EXISTS`` does: a pool-block row exists + only for a candidate that reached ``persist_accepted_block``. + """ + ledger = self._seeded_bulk_abandon_ledger() + ledger._memory_pool_blocks[BULK_PENDING_B] = (10, "prepared", "00" * 32) + + rows = ledger.pending_block_candidate_rows(limit=32) + + self.assertEqual( + {row["block_hash"]: row["pool_block_exists"] for row in rows}, + {BULK_PENDING_A: False, BULK_PENDING_B: True}, + ) + + def test_bulk_candidate_abandon_spares_a_landed_candidate(self) -> None: + """A pool-block row vetoes the transition even inside the page. + + The caller's page could have been read before the candidate landed, + so the veto has to live in the write itself. A row that acquired one + is simply absent from the report, which is exactly how the caller + learns not to clean it up. + """ + ledger = self._seeded_bulk_abandon_ledger() + ledger._memory_pool_blocks[BULK_PENDING_B] = (10, "prepared", "00" * 32) + + abandoned = ledger.mark_block_candidates_abandoned( + block_hashes=[BULK_PENDING_A, BULK_PENDING_B], + error="superseded by decided height", + ) + + self.assertEqual(abandoned, (BULK_PENDING_A,)) + landed = ledger._block_candidate_outbox[BULK_PENDING_B] + self.assertEqual(landed["state"], "pending") + self.assertIsNone(landed["last_error"]) + self.assertIsNotNone(landed["candidate"]) + + def test_bulk_candidate_abandon_decides_a_page_under_one_lock_hold(self) -> None: + """One page is one atomic decision, matching the Postgres statement. + + The Postgres backend resolves a whole page inside a single fenced + statement, so a memory backend that released and re-took the lock per + hash would let a concurrent writer split the reported set across two + views of the outbox -- a parity gap the returned set would then hide. + """ + ledger = self._seeded_bulk_abandon_ledger() + counting = CountingLock(ledger._lock) + ledger._lock = counting + + abandoned = ledger.mark_block_candidates_abandoned( + block_hashes=[BULK_PENDING_A, BULK_PENDING_B, BULK_MISSING], + error="superseded by decided height", + ) + + self.assertEqual(abandoned, (BULK_PENDING_A, BULK_PENDING_B)) + self.assertEqual(counting.acquisitions, 1) + + def test_pending_candidate_cursor_pages_walk_a_stable_total_order(self) -> None: + """Startup enumeration must complete for a backlog of any size. + + The doubling window it replaces fails closed once one query would + have to hold the whole backlog, which blocks every job build until + the outbox drains on its own. Pagination needs three properties for + that gate to be safe: one stable total order, a cursor that resumes + strictly after its own row, and a short page that proves nothing + followed it. + """ + ledger = SingleWriterShareLedger() + hashes = [f"{index:02x}" * 32 for index in (0xA1, 0xA2, 0xB1, 0xB2, 0xC1)] + # Inserted out of order so the assertions below prove the ordering + # comes from the row keys and not from the outbox's insertion order. + for index in (2, 4, 0, 3, 1): + ledger.persist_block_candidate_intent( + { + "schema": "qbit.prism.block-candidate-intent.v1", + "block_hash_hex": hashes[index], + "block_hex": "00", + } + ) + # Colliding creation stamps are the interesting case: time.monotonic + # (like one transaction's clock_timestamp) repeats, so ordering on + # the stamp alone would let a cursor replay or skip a whole group. + stamps = { + hashes[0]: 100.0, + hashes[1]: 100.0, + hashes[2]: 100.0, + hashes[3]: 200.0, + hashes[4]: 300.0, + } + for block_hash, stamp in stamps.items(): + ledger._block_candidate_outbox[block_hash]["created_monotonic"] = stamp + + walked: list[str] = [] + pages: list[int] = [] + after_cursor: object | None = None + while True: + page = ledger.pending_block_candidate_rows( + limit=2, + after_cursor=after_cursor, + ) + pages.append(len(page)) + walked.extend(str(row["block_hash"]) for row in page) + if len(page) < 2: + break + after_cursor = page[-1]["cursor"] + + # Total order: creation stamp first, block hash as the tiebreak. + self.assertEqual(walked, sorted(hashes, key=lambda h: (stamps[h], h))) + # Full pages until one short page proves the walk reached the end. + self.assertEqual(pages, [2, 2, 1]) + # No row was served twice, and the walk saw every pending row. + self.assertEqual(len(set(walked)), len(hashes)) + + # A cursor resumes strictly after its own row, including when its + # stamp collides with the next row's. + tied_cursor = [100.0, hashes[0]] + self.assertEqual( + [ + str(row["block_hash"]) + for row in ledger.pending_block_candidate_rows( + limit=32, + after_cursor=tied_cursor, + ) + ], + [hashes[1], hashes[2], hashes[3], hashes[4]], + ) + # The cursor round-trips: handing back a returned cursor verbatim is + # equivalent to reconstructing it from the row's own ordering parts. + last_first_page = ledger.pending_block_candidate_rows(limit=1)[0] + self.assertEqual(last_first_page["cursor"], [100.0, hashes[0]]) + # A cursor past every row proves completion with an empty page. + self.assertEqual( + ledger.pending_block_candidate_rows( + limit=32, + after_cursor=[300.0, hashes[4]], + ), + [], + ) + # Terminal rows leave the pending order entirely. + self.assertTrue( + ledger.mark_block_candidate_submitted(block_hash=hashes[1]) + ) + self.assertNotIn( + hashes[1], + [ + str(row["block_hash"]) + for row in ledger.pending_block_candidate_rows(limit=32) + ], + ) + + def test_pending_candidate_cursor_rejects_unusable_shapes(self) -> None: + """A cursor crosses process and JSON boundaries, so it is validated.""" + ledger = SingleWriterShareLedger() + for cursor in ( + [], + [1.0], + [1.0, "ab" * 32, "extra"], + [1.0, ""], + ["not-a-stamp", "ab" * 32], + "1.0,ab", + ): + with self.assertRaises(ValueError): + ledger.pending_block_candidate_rows(after_cursor=cursor) + + def test_stranded_prepared_blocks_filter_by_depth_and_state(self) -> None: + """The sweep's read only surfaces rows an operator-free heal may touch. + + reorg_watch_blocks selects confirmed/inactive rows only, so a + prepared row whose outbox entry is gone has nothing left to + re-examine it. The depth floor is what separates that from a + finalization still legitimately in flight. + """ + ledger = SingleWriterShareLedger() + heights = { + "deep": ("a1" * 32, 100), + "at_floor": ("a2" * 32, 900), + "shallow": ("a3" * 32, 901), + "confirmed": ("a4" * 32, 200), + } + for block_hash, block_height in heights.values(): + ledger.persist_accepted_block( + block_hash=block_hash, + block_height=block_height, + parent_hash="bb" * 32, + final_bundle={}, + audit_report={}, + ) + confirmed_hash, confirmed_height = heights["confirmed"] + self.assertEqual( + ledger.confirm_accepted_block( + block_hash=confirmed_hash, + active_tip_height=confirmed_height, + )["confirmed_count"], + 1, + ) + + stranded = ledger.stranded_prepared_blocks( + active_tip_height=1000, + min_depth=100, + ) + + self.assertEqual( + [(row["block_hash"], row["block_height"]) for row in stranded], + [ + (heights["deep"][0], heights["deep"][1]), + (heights["at_floor"][0], heights["at_floor"][1]), + ], + ) + self.assertEqual(stranded[0]["parent_hash"], "bb" * 32) + # A confirmed row belongs to the watch loop, not this sweep, and a + # row shallower than the floor may still be finalizing. + returned = {str(row["block_hash"]) for row in stranded} + self.assertNotIn(heights["confirmed"][0], returned) + self.assertNotIn(heights["shallow"][0], returned) + # The limit bounds one pass; the remainder waits for the next one. + self.assertEqual( + [ + row["block_hash"] + for row in ledger.stranded_prepared_blocks( + active_tip_height=1000, + min_depth=100, + limit=1, + ) + ], + [heights["deep"][0]], + ) + # Rejecting through the fenced method retires the row from the sweep. + self.assertEqual( + ledger.reject_prepared_block( + block_hash=heights["deep"][0], + active_tip_height=1000, + )["rejected_count"], + 1, + ) + self.assertEqual( + [ + row["block_hash"] + for row in ledger.stranded_prepared_blocks( + active_tip_height=1000, + min_depth=100, + ) + ], + [heights["at_floor"][0]], + ) + def test_batch_validation_is_all_or_nothing(self) -> None: ledger = SingleWriterShareLedger() first = pending_share(1) @@ -565,8 +1214,17 @@ def test_candidate_retry_with_new_acknowledgment_stamp_is_idempotent(self) -> No ledger.append_batch([(retry_share, retry_intent)])[0].share_seq, 1 ) self.assertEqual( - ledger.pending_block_candidate_rows(), - [{"block_hash": "ef" * 32, "candidate": intent}], + [ + {key: value for key, value in row.items() if key != "cursor"} + for row in ledger.pending_block_candidate_rows() + ], + [ + { + "block_hash": "ef" * 32, + "candidate": intent, + "pool_block_exists": False, + } + ], ) self.assertTrue(ledger.mark_block_candidate_submitted(block_hash="ef" * 32)) terminal_persist = ledger.persist_block_candidate_intent(retry_intent) @@ -772,6 +1430,42 @@ def test_memory_ledger_public_artifact_omits_missing_ctv_audit_bundle_body(self) self.assertIsNone(ledger.dashboard_public_artifact(sha256=audit_bundle_sha256)) + def test_memory_ledger_public_artifact_document_returns_hashed_text(self) -> None: + # The canonical text is the exact byte sequence the advertised sha256 + # was computed over; serving anything else breaks external + # verification of content-addressed artifacts. + ledger = SingleWriterShareLedger() + block_hash = "aa" * 32 + audit_bundle_sha256 = "77" * 32 + manifest_set = { + **sample_ctv_manifest_set(), + "audit_bundle_sha256": audit_bundle_sha256, + "audit_bundle": {"schema": "qbit.prism.audit-bundle.v1"}, + } + manifest = manifest_set["manifests"][0] # type: ignore[index] + manifest_set_sha256 = sha256_json_hex(manifest_set) + + ledger.persist_ctv_fanout_manifest_set( + block_hash=block_hash, + manifest_set=manifest_set, + manifest_set_sha256=manifest_set_sha256, + ) + + set_document = ledger.dashboard_public_artifact_document(sha256=manifest_set_sha256) + manifest_document = ledger.dashboard_public_artifact_document(sha256=sha256_json_hex(manifest)) + bundle_document = ledger.dashboard_public_artifact_document(sha256=audit_bundle_sha256) + + self.assertEqual(set_document["canonical_json"], canonical_json_text(manifest_set)) # type: ignore[index] + self.assertEqual(set_document["payload"], manifest_set) # type: ignore[index] + self.assertEqual(manifest_document["canonical_json"], canonical_json_text(manifest)) # type: ignore[index] + self.assertEqual( + hashlib.sha256(str(set_document["canonical_json"]).encode()).hexdigest(), # type: ignore[index] + manifest_set_sha256, + ) + self.assertEqual(bundle_document["payload"], {"schema": "qbit.prism.audit-bundle.v1"}) # type: ignore[index] + self.assertIsNone(bundle_document["canonical_json"]) # type: ignore[index] + self.assertIsNone(ledger.dashboard_public_artifact_document(sha256="99" * 32)) + def test_reorg_watch_blocks_keeps_height_mature_immature_rows(self) -> None: ledger = QueryCapturePsqlShareLedger() @@ -793,6 +1487,7 @@ def test_pool_block_state_wraps_nullable_row_in_json(self) -> None: "parent_hash": "bb" * 32, "chain_state": "confirmed", "maturity_state": "immature", + "audit_publication_sequence": None, } }, ] @@ -807,10 +1502,37 @@ def test_pool_block_state_wraps_nullable_row_in_json(self) -> None: "parent_hash": "bb" * 32, "chain_state": "confirmed", "maturity_state": "immature", + "audit_publication_sequence": None, }, ) self.assertIn("SELECT json_build_object(\n 'state'", ledger.queries[1]) + def test_postgres_publication_floor_reads_max_durable_row_ordinal(self) -> None: + ledger = CannedQueryPsqlShareLedger( + [ + acquired_lease_result(), + {"audit_publication_sequence_floor": 7}, + ] + ) + + self.assertEqual(ledger.audit_publication_sequence_floor(), 7) + query = ledger.queries[1] + self.assertIn("MAX(audit_publication_sequence)", query) + self.assertIn("COALESCE", query) + self.assertIn("FROM qbit_pool_blocks", query) + self.assertNotIn("last_value", query) + + for invalid in (None, True, "7", -1): + with self.subTest(invalid=invalid): + invalid_ledger = CannedQueryPsqlShareLedger( + [ + acquired_lease_result(), + {"audit_publication_sequence_floor": invalid}, + ] + ) + with self.assertRaisesRegex(RuntimeError, "floor is invalid"): + invalid_ledger.audit_publication_sequence_floor() + def test_prior_balances_after_pool_block_is_height_bounded(self) -> None: balance = { "recipient_id": "miner-a", @@ -1017,6 +1739,9 @@ def test_postgres_batch_sql_fences_share_and_candidate_in_one_statement(self) -> ledger = FakeLeasePsqlShareLedger( [acquired_lease(), {"records": [record]}] ) + # This pins the legacy whole-jsonb statement (storage version 1); + # the chunked-body statement is covered by the #255 suites. + ledger._candidate_storage_version = 1 intent = { "schema": "qbit.prism.block-candidate-intent.v1", "block_hash_hex": "ab" * 32, @@ -1033,10 +1758,55 @@ def test_postgres_batch_sql_fences_share_and_candidate_in_one_statement(self) -> self.assertNotIn("ledger.accepted_at IS DISTINCT", query) self.assertEqual(query.count("SELECT CASE"), 1) - def test_postgres_candidate_only_intent_forces_durable_fenced_commit(self) -> None: + def test_postgres_recovery_append_returns_exact_existing_from_batch_comparator(self) -> None: + share = pending_share(2) + record = { + "share_seq": 8, + "share_id": share.share_id, + "miner_id": share.miner_id, + "order_key": share.order_key, + "p2mr_program_hex": share.p2mr_program_hex, + "share_difficulty": str(share.share_difficulty), + "network_difficulty": str(share.network_difficulty), + "template_height": share.template_height, + "job_id": share.job_id, + "job_issued_at_ms": share.job_issued_at_ms, + "accepted_at_ms": share.accepted_at_ms, + "ntime": share.ntime, + "credit_policy": share.credit_policy, + "newly_inserted": False, + } ledger = FakeLeasePsqlShareLedger( - [acquired_lease(), {"inserted": 1}] + [acquired_lease(), {"records": [record]}] + ) + + outcome = ledger.append_recovered_share(share) + + self.assertEqual(outcome.disposition, "exact_existing") + self.assertEqual(outcome.record.share_seq, 8) + self.assertIn("share_mismatch AS", ledger.lease_queries[-1]) + + def test_postgres_recovery_append_raises_typed_payload_conflict(self) -> None: + ledger = FakeLeasePsqlShareLedger( + [ + acquired_lease(), + { + "error": "duplicate share_id payload mismatch", + "error_kind": "share_replay_conflict", + }, + ] + ) + + with self.assertRaises(ShareReplayConflict): + ledger.append_recovered_share(pending_share(3)) + + def test_postgres_candidate_only_intent_forces_durable_fenced_commit(self) -> None: + ledger = FakeLeasePsqlShareLedger( + [acquired_lease(), {"inserted": 1}] ) + # Legacy whole-jsonb statement (storage version 1); see #255 suites + # for the chunked-body route. + ledger._candidate_storage_version = 1 intent = { "schema": "qbit.prism.block-candidate-intent.v1", "block_hash_hex": "cd" * 32, @@ -1056,20 +1826,134 @@ def test_postgres_pending_candidate_rows_keep_authoritative_outbox_key(self) -> "schema": "unsupported", "block_hex": "00", } + cursor = ["2026-07-08T21:02:03.123456Z", "ef" * 32] ledger = FakeLeasePsqlShareLedger( [ acquired_lease(), - [{"block_hash": "ef" * 32, "candidate": intent}], + [ + { + "block_hash": "ef" * 32, + "candidate": intent, + "pool_block_exists": False, + "cursor": cursor, + } + ], ] ) self.assertEqual( ledger.pending_block_candidate_rows(), - [{"block_hash": "ef" * 32, "candidate": intent}], + [ + { + "block_hash": "ef" * 32, + "candidate": intent, + "pool_block_exists": False, + "cursor": cursor, + } + ], ) query = ledger.lease_queries[-1] - self.assertIn("json_build_object('block_hash', block_hash, 'candidate', candidate)", query) + self.assertIn("'block_hash', pending.block_hash,", query) + self.assertIn("'candidate', pending.candidate,", query) + # The cursor is the two ordering columns, and its stamp keeps the + # microsecond precision the column stores. + self.assertIn( + "'cursor', json_build_array(\n pending.cursor_created_at,\n pending.block_hash\n )", + query, + ) + self.assertIn( + "'YYYY-MM-DD\"T\"HH24:MI:SS.US\"Z\"'", + query, + ) self.assertIn("ORDER BY created_at, block_hash", query) + # A first page carries no keyset predicate at all. + self.assertNotIn("(created_at, block_hash) >", query) + + def test_postgres_pending_candidate_page_uses_an_indexable_keyset(self) -> None: + """Each page must stay one bounded range scan of the partial index. + + qbit_block_candidate_outbox_pending_idx is (created_at, block_hash) + WHERE state = 'pending'. A row-comparison predicate on exactly those + columns is what keeps page N of a 5,000-row backlog as cheap as page + one; an OFFSET walk, or a predicate on either column alone, would + not be. + """ + ledger = FakeLeasePsqlShareLedger([acquired_lease(), []]) + + self.assertEqual( + ledger.pending_block_candidate_rows( + limit=1024, + after_cursor=["2026-07-08T21:02:03.123456Z", "ef" * 32], + ), + [], + ) + + query = ledger.lease_queries[-1] + self.assertIn( + "AND (created_at, block_hash) > " + "('2026-07-08T21:02:03.123456Z'::timestamptz, " + f"'{'ef' * 32}')", + query, + ) + self.assertIn("WHERE state = 'pending'", query) + self.assertIn("ORDER BY created_at, block_hash", query) + self.assertIn("LIMIT 1024", query) + # A hostile cursor is a literal like any other and is escaped, not + # interpolated raw. + ledger.lease_results.append([]) + ledger.pending_block_candidate_rows( + after_cursor=["2026-07-08T21:02:03.123456Z", "ef' OR true --"], + ) + self.assertIn("'ef'' OR true --'", ledger.lease_queries[-1]) + + def test_postgres_stranded_prepared_blocks_bound_depth_and_page(self) -> None: + """The sweep read is depth-floored, state-filtered, and bounded.""" + ledger = FakeLeasePsqlShareLedger( + [ + acquired_lease(), + [ + { + "block_hash": "19" * 32, + "block_height": "30822", + "parent_hash": "20" * 32, + } + ], + ] + ) + + self.assertEqual( + ledger.stranded_prepared_blocks( + active_tip_height=31000, + min_depth=100, + limit=64, + ), + [ + { + "block_hash": "19" * 32, + "block_height": 30822, + "parent_hash": "20" * 32, + } + ], + ) + + query = ledger.lease_queries[-1] + self.assertIn("FROM qbit_pool_blocks", query) + # Leads with the qbit_pool_blocks_maturity_idx columns. + self.assertIn("WHERE maturity_state = 'immature'", query) + self.assertIn("AND block_height <= 30900", query) + self.assertIn("AND chain_state = 'prepared'", query) + self.assertIn("ORDER BY block_height ASC, block_hash ASC", query) + self.assertIn("LIMIT 64", query) + # Nothing is fetched at all when the caller allows no rows. + self.assertEqual( + ledger.stranded_prepared_blocks( + active_tip_height=31000, + min_depth=100, + limit=0, + ), + [], + ) + self.assertIs(ledger.lease_queries[-1], query) def test_postgres_pending_candidate_metrics_are_aggregate_and_indexable(self) -> None: ledger = FakeLeasePsqlShareLedger( @@ -1112,6 +1996,499 @@ def test_postgres_candidate_attempt_is_writer_fenced(self) -> None: self.assertIn("writer_epoch = 7", query) self.assertIn("attempt_count = attempt_count + 1", query) + def _bulk_abandon_query( + self, + *, + block_hashes: list[str], + error: str = "superseded by decided height", + abandoned: list[str] | None = None, + ) -> tuple[FakeLeasePsqlShareLedger, tuple[str, ...], str]: + ledger = FakeLeasePsqlShareLedger( + [acquired_lease(), {"abandoned": abandoned or []}], + writer_id="writer-a", + writer_epoch=7, + ) + result = ledger.mark_block_candidates_abandoned( + block_hashes=block_hashes, + error=error, + ) + return ledger, result, ledger.lease_queries[-1] + + def test_postgres_bulk_candidate_abandon_is_one_fenced_set_statement( + self, + ) -> None: + """One statement per page, carrying the same fence as the per-row path. + + The point of the batch form is that a storm-sized page costs one + fenced write rather than one per hash, so the shape assertions are + the behaviour: exactly one outbox ``UPDATE``, and a set predicate + rather than a disjunction of equalities. + """ + _, _, query = self._bulk_abandon_query( + block_hashes=[BULK_PENDING_A, BULK_PENDING_B], + ) + + self.assertEqual(query.count("UPDATE qbit_block_candidate_outbox"), 1) + self.assertIn("qbit_ledger_writer_lease", query) + self.assertIn("writer_id = 'writer-a'", query) + self.assertIn("writer_epoch = 7", query) + self.assertIn("writer_session_token = ", query) + self.assertIn( + f"block_hash = ANY(ARRAY['{BULK_PENDING_A}', '{BULK_PENDING_B}']::text[])", + query, + ) + self.assertIn("AND state = 'pending'", query) + self.assertIn("SET state = 'abandoned'", query) + self.assertIn("last_error = 'superseded by decided height'", query) + self.assertIn("updated_at = clock_timestamp()", query) + self.assertIn("completed_at = clock_timestamp()", query) + self.assertIn("candidate = NULL", query) + self.assertIn("RETURNING block_hash", query) + self.assertIn("'writer lease is not active'", query) + + def test_postgres_bulk_candidate_abandon_re_checks_pool_blocks(self) -> None: + """The landed-block fact is re-asked inside the write, not trusted. + + ``pool_block_exists`` from a page read is a fact about the moment of + that read. A candidate can land between the read and the write, and + the pool-block row is what records that it did, so the terminal + statement carries its own ``NOT EXISTS`` arm under the writer fence. + """ + _, _, query = self._bulk_abandon_query(block_hashes=[BULK_PENDING_A]) + + self.assertIn("AND NOT EXISTS (", query) + self.assertIn("FROM qbit_pool_blocks pool", query) + self.assertIn( + "WHERE pool.block_hash = qbit_block_candidate_outbox.block_hash", + query, + ) + + def test_postgres_bulk_candidate_abandon_returns_the_rows_it_won(self) -> None: + """``RETURNING`` decides the report, so the request cannot inflate it.""" + _, abandoned, _ = self._bulk_abandon_query( + block_hashes=[BULK_PENDING_A, BULK_PENDING_B, BULK_SUBMITTED], + abandoned=[BULK_PENDING_A], + ) + + self.assertEqual(abandoned, (BULK_PENDING_A,)) + + def test_postgres_bulk_candidate_abandon_reports_an_empty_win(self) -> None: + _, abandoned, query = self._bulk_abandon_query( + block_hashes=[BULK_PENDING_A], + abandoned=[], + ) + + self.assertEqual(abandoned, ()) + # The empty case is the aggregate's, not a missing key: an outer + # json_agg over zero rows is NULL and would decode as None. + self.assertIn("'[]'::json", query) + + def test_postgres_bulk_candidate_abandon_normalizes_the_target_set( + self, + ) -> None: + """Mixed case and duplicates resolve to one deterministic target set. + + Rows are keyed on the lowercase hash, so a page naming the same row + twice in two cases must ask for it once. Sorting is what makes the + generated statement a function of the set rather than of the caller's + iteration order. + """ + _, _, query = self._bulk_abandon_query( + block_hashes=[ + BULK_PENDING_B, + BULK_PENDING_A.upper(), + BULK_PENDING_A, + BULK_PENDING_B.upper(), + ], + ) + + self.assertIn( + f"ANY(ARRAY['{BULK_PENDING_A}', '{BULK_PENDING_B}']::text[])", + query, + ) + self.assertEqual(query.count(BULK_PENDING_A), 1) + self.assertNotIn(BULK_PENDING_A.upper(), query) + + def test_postgres_bulk_candidate_abandon_quotes_the_reason(self) -> None: + _, _, query = self._bulk_abandon_query( + block_hashes=[BULK_PENDING_A], + error="height 'H' decided by another block", + ) + + self.assertIn( + "last_error = 'height ''H'' decided by another block'", + query, + ) + + def test_postgres_bulk_candidate_abandon_on_an_empty_page_issues_no_query( + self, + ) -> None: + """An empty page must not reach the server at all. + + ``ANY(ARRAY[])`` has no inferable element type and would fail to + parse, so the guard is not an optimization: the alternative is a + malformed statement. + """ + ledger = FakeLeasePsqlShareLedger([acquired_lease()]) + queries_before = len(ledger.lease_queries) + + self.assertEqual( + ledger.mark_block_candidates_abandoned(block_hashes=[], error="unused"), + (), + ) + + self.assertEqual(len(ledger.lease_queries), queries_before) + + def test_postgres_bulk_candidate_abandon_fails_on_inactive_writer_lease( + self, + ) -> None: + ledger = FakeLeasePsqlShareLedger( + [acquired_lease(), {"error": "writer lease is not active"}] + ) + + with self.assertRaisesRegex(RuntimeError, "writer lease is not active"): + ledger.mark_block_candidates_abandoned( + block_hashes=[BULK_PENDING_A, BULK_PENDING_B], + error="superseded by decided height", + ) + + def test_postgres_pending_candidate_page_carries_pool_block_existence( + self, + ) -> None: + """One page read answers the landed-block question for every row. + + Asking per row is one round trip per row, and the page read exists + precisely for backlogs where that is the whole cost. The probe sits + outside the limited subquery so it runs at most ``limit`` times + rather than once per pending row in the backlog. + """ + ledger = FakeLeasePsqlShareLedger( + [ + acquired_lease(), + [ + { + "block_hash": BULK_PENDING_A, + "candidate": {"block_hash_hex": BULK_PENDING_A}, + "pool_block_exists": False, + "cursor": ["2026-08-20T00:00:00.000001Z", BULK_PENDING_A], + }, + { + "block_hash": BULK_PENDING_B, + "candidate": {"block_hash_hex": BULK_PENDING_B}, + "pool_block_exists": True, + "cursor": ["2026-08-20T00:00:00.000002Z", BULK_PENDING_B], + }, + ], + ] + ) + + rows = ledger.pending_block_candidate_rows(limit=32) + + self.assertEqual( + [row["pool_block_exists"] for row in rows], + [False, True], + ) + query = ledger.lease_queries[-1] + self.assertIn("'pool_block_exists', EXISTS (", query) + self.assertIn("FROM qbit_pool_blocks pool", query) + self.assertIn("WHERE pool.block_hash = pending.block_hash", query) + # Outside the LIMITed subquery: the probe is bounded by the page, not + # by the backlog behind it. + self.assertLess( + query.index("'pool_block_exists', EXISTS ("), + query.index("FROM (\n SELECT"), + ) + self.assertIn("LIMIT 32", query) + + def test_postgres_pending_candidate_page_without_existence_fails_closed( + self, + ) -> None: + """A page that cannot report the fact raises instead of implying false.""" + ledger = FakeLeasePsqlShareLedger( + [ + acquired_lease(), + [{"block_hash": BULK_PENDING_A, "candidate": {}, "cursor": [1, "a"]}], + ] + ) + + with self.assertRaisesRegex(RuntimeError, "pool block existence"): + ledger.pending_block_candidate_rows(limit=32) + + def test_postgres_pending_candidate_page_propagates_read_failure(self) -> None: + """A failed page read must not read as an empty backlog.""" + + class FailingPageLedger(FakeLeasePsqlShareLedger): + def _run_retry_safe_read_json( + self, + sql: str, + *, + on_statement_start: Any = None, + ) -> Any: + if on_statement_start is not None: + on_statement_start() + raise RuntimeError("connection reset by peer") + + ledger = FailingPageLedger([acquired_lease()]) + + with self.assertRaisesRegex(RuntimeError, "connection reset by peer"): + ledger.pending_block_candidate_rows(limit=32) + + def test_postgres_pending_candidate_page_never_touches_the_writer_gate( + self, + ) -> None: + """The page is a read, so it takes the read slot and nothing else. + + Issue #211: waiting for writer admission bought this statement + nothing and cost it its whole fast-call budget whenever accepted-block + accounting held the gate. A gate that refuses every acquisition is the + strongest available statement of "this path does not go there" -- + stronger than timing a wait, because it fails whether the gate is + contended or free. + """ + ledger = FakeLeasePsqlShareLedger( + [ + acquired_lease(), + [ + { + "block_hash": BULK_PENDING_A, + "candidate": {"block_hash_hex": BULK_PENDING_A}, + "pool_block_exists": False, + "cursor": ["2026-08-31T00:00:00.000001Z", BULK_PENDING_A], + } + ], + ] + ) + ledger._lock = share_ledger_module._RefusingWriterGate() + + rows = ledger.pending_block_candidate_rows(limit=32) + + self.assertEqual([row["block_hash"] for row in rows], [BULK_PENDING_A]) + # The read slot is still taken and still handed back: bounded, not + # ungated. + self.assertEqual(ledger._read_semaphore._value, 4) + + def test_postgres_pending_candidate_page_splits_gate_wait_from_execution( + self, + ) -> None: + """Attribution, not a single opaque duration. + + The incident reported ``exceeded 5s`` for a call whose statement + deadline was barely touched, and nothing exported said where the + budget went. These two fields are that answer, so a scenario drives + the clock across each half separately and reads them back. + """ + ledger = FakeLeasePsqlShareLedger([acquired_lease(), []]) + # Installed after construction so the script covers exactly this + # call's four reads -- admission start, admission end, statement + # start, statement end -- and an unexpected fifth fails loudly rather + # than silently re-using a stamp. + clock = iter([100.0, 100.25, 100.25, 101.75]) + ledger._monotonic = lambda: next(clock) + + ledger.pending_block_candidate_rows(limit=32) + + self.assertEqual(list(clock), []) + + stats = ledger.ledger_read_gate_stats()["pending_block_candidate_rows"] + self.assertEqual(stats["calls_total"], 1) + self.assertAlmostEqual(stats["gate_wait_seconds_total"], 0.25) + self.assertAlmostEqual(stats["gate_wait_seconds_max"], 0.25) + self.assertAlmostEqual(stats["execute_seconds_total"], 1.5) + self.assertAlmostEqual(stats["execute_seconds_max"], 1.5) + self.assertEqual(stats["gate_timeouts_total"], 0) + self.assertEqual(stats["execute_timeouts_total"], 0) + + def test_postgres_pending_candidate_page_attributes_a_statement_deadline( + self, + ) -> None: + """A deadline that expires inside PostgreSQL is PostgreSQL's. + + Including the tail a cancelled statement spends returning: cancel lag + is server time, and charging it to local admission would point the + next investigation at the wrong half of the system. + """ + + class TimingOutPageLedger(FakeLeasePsqlShareLedger): + def _run_retry_safe_read_json( + self, + sql: str, + *, + on_statement_start: Any = None, + ) -> Any: + if on_statement_start is not None: + on_statement_start() + raise LedgerOperationTimeout("postgres operation exceeded 5s") + + ledger = TimingOutPageLedger([acquired_lease()]) + + with self.assertRaises(LedgerOperationTimeout): + ledger.pending_block_candidate_rows(limit=32) + + stats = ledger.ledger_read_gate_stats()["pending_block_candidate_rows"] + self.assertEqual(stats["calls_total"], 1) + self.assertEqual(stats["execute_timeouts_total"], 1) + self.assertEqual(stats["gate_timeouts_total"], 0) + # The failed read still handed its slot back, or the next enumeration + # would queue behind a slot nobody holds. + self.assertEqual(ledger._read_semaphore._value, 4) + + def test_postgres_pending_candidate_page_attributes_pre_send_expiry_locally( + self, + ) -> None: + """A deadline exhausted after admission is still not server time.""" + + class NeverRunNative: + def run_json(self, sql: str, **kwargs: Any) -> Any: + raise AssertionError("deadline-expired read must not reach PostgreSQL") + + ledger = PsqlShareLedger.__new__(PsqlShareLedger) + ledger._read_semaphore = threading.BoundedSemaphore(1) + ledger._native = NeverRunNative() + deadline_local = threading.local() + deadline_local.deadline = 1.0 + ledger._operation_timeout_local = deadline_local + # admission start, deadline check while acquiring, admission end, + # deadline check immediately before the native statement + clock = iter([0.0, 0.5, 0.75, 1.01]) + ledger._monotonic = lambda: next(clock) + + with self.assertRaisesRegex(LedgerOperationTimeout, "deadline expired"): + ledger._run_attributed_read_json( + "SELECT json_build_object('ok', true);", + operation="pending_block_candidate_rows", + gate=ledger._read_semaphore, + gate_name="read slot", + ) + + self.assertEqual(list(clock), []) + stats = ledger.ledger_read_gate_stats()["pending_block_candidate_rows"] + self.assertEqual(stats["calls_total"], 1) + self.assertEqual(stats["gate_timeouts_total"], 1) + self.assertEqual(stats["execute_timeouts_total"], 0) + self.assertEqual(stats["execute_seconds_total"], 0.0) + self.assertEqual(ledger._read_semaphore._value, 1) + + def test_psql_pending_candidate_page_attributes_pre_spawn_expiry_locally( + self, + ) -> None: + """The subprocess backend validates its deadline before execution.""" + + ledger = PsqlShareLedger.__new__(PsqlShareLedger) + ledger._read_semaphore = threading.BoundedSemaphore(1) + ledger._native = None + ledger._command = ["psql"] + ledger._session_guards = None + deadline_local = threading.local() + deadline_local.deadline = 1.0 + ledger._operation_timeout_local = deadline_local + # admission start, deadline check while acquiring, admission end, + # deadline check in _run_sql immediately before subprocess setup + clock = iter([0.0, 0.5, 0.75, 1.01]) + ledger._monotonic = lambda: next(clock) + + with unittest.mock.patch.object( + share_ledger_module.subprocess, + "run", + side_effect=AssertionError("deadline-expired read must not spawn psql"), + ) as run: + with self.assertRaisesRegex(LedgerOperationTimeout, "deadline expired"): + ledger._run_attributed_read_json( + "SELECT json_build_object('ok', true);", + operation="pending_block_candidate_rows", + gate=ledger._read_semaphore, + gate_name="read slot", + ) + + run.assert_not_called() + self.assertEqual(list(clock), []) + stats = ledger.ledger_read_gate_stats()["pending_block_candidate_rows"] + self.assertEqual(stats["calls_total"], 1) + self.assertEqual(stats["gate_timeouts_total"], 1) + self.assertEqual(stats["execute_timeouts_total"], 0) + self.assertEqual(stats["execute_seconds_total"], 0.0) + self.assertEqual(ledger._read_semaphore._value, 1) + + def test_run_json_preserves_legacy_run_sql_override_signature(self) -> None: + """Private one-argument subprocess seams remain supported.""" + + calls: list[str] = [] + starts: list[str] = [] + + class LegacyRunSqlLedger(PsqlShareLedger): + def _run_sql(self, sql: str) -> str: + calls.append(sql) + return '{"ok": true}' + + ledger = LegacyRunSqlLedger.__new__(LegacyRunSqlLedger) + ledger._native = None + + result = ledger._run_json( + "SELECT json_build_object('ok', true);", + on_statement_start=lambda: starts.append("started"), + ) + + self.assertEqual(result, {"ok": True}) + self.assertEqual(calls, ["SELECT json_build_object('ok', true);"]) + self.assertEqual(starts, ["started"]) + + def test_postgres_pending_candidate_page_attributes_an_admission_expiry( + self, + ) -> None: + """An expiry before admission is a gate timeout with no SQL sample. + + No statement was sent, so recording an execution duration for it + would invent server time that never happened. + """ + ledger = FakeLeasePsqlShareLedger([acquired_lease()]) + ledger._read_semaphore = threading.BoundedSemaphore(1) + self.assertTrue(ledger._read_semaphore.acquire()) + + with self.assertRaisesRegex(LedgerOperationTimeout, "read slot"): + with ledger.operation_timeout(0.01): + ledger.pending_block_candidate_rows(limit=32) + + stats = ledger.ledger_read_gate_stats()["pending_block_candidate_rows"] + self.assertEqual(stats["calls_total"], 1) + self.assertEqual(stats["gate_timeouts_total"], 1) + self.assertEqual(stats["execute_timeouts_total"], 0) + self.assertEqual(stats["execute_seconds_total"], 0.0) + self.assertGreater(stats["gate_wait_seconds_total"], 0.0) + + def test_ledger_read_gate_stats_start_empty_and_are_per_instance(self) -> None: + """No operation, no series: an unused ledger exports nothing. + + The lazy bootstrap exists for ledgers built through ``__new__``; it + must not hand two of them the same dict, or one instance's admission + wait would appear on another's series. + """ + first = PsqlShareLedger.__new__(PsqlShareLedger) + second = PsqlShareLedger.__new__(PsqlShareLedger) + + self.assertEqual(first.ledger_read_gate_stats(), {}) + first._note_ledger_read_timing( + "pending_block_candidate_rows", + gate_wait_seconds=0.5, + execute_seconds=1.0, + timed_out=False, + ) + + self.assertEqual(second.ledger_read_gate_stats(), {}) + self.assertEqual( + first.ledger_read_gate_stats()["pending_block_candidate_rows"][ + "calls_total" + ], + 1, + ) + # The accessor hands back a copy, so a caller cannot edit the record. + snapshot = first.ledger_read_gate_stats() + snapshot["pending_block_candidate_rows"]["calls_total"] = 99 + self.assertEqual( + first.ledger_read_gate_stats()["pending_block_candidate_rows"][ + "calls_total" + ], + 1, + ) + def test_writer_lease_ttl_defaults_to_sixty_seconds(self) -> None: ledger = FakeLeasePsqlShareLedger([acquired_lease()]) @@ -1941,15 +3318,30 @@ def test_block_state_functions_refresh_configured_lease_after_sql_function(self) ) for method_name, result, function_name, count_key in cases: with self.subTest(method_name=method_name): + canned = [acquired_lease(), {"backend": "postgres-psql", **result}] + if method_name in { + "confirm_accepted_block", + "reactivate_pool_block", + }: + canned.append({"audit_publication_sequence": 7}) ledger = FakeLeasePsqlShareLedger( - [acquired_lease(), {"backend": "postgres-psql", **result}], + canned, lease_ttl_seconds=42, ) payload = getattr(ledger, method_name)(block_hash="aa" * 32, active_tip_height=10) - query = ledger.lease_queries[-1] + query = next( + query + for query in ledger.lease_queries + if function_name in query + ) self.assertEqual(payload[count_key], 1) + if method_name in { + "confirm_accepted_block", + "reactivate_pool_block", + }: + self.assertEqual(payload["audit_publication_sequence"], 7) self.assertIn(function_name, query) self.assertNotIn("lease_refresh AS", query) self.assertIn("make_interval(secs => 42.0)", query) @@ -2262,6 +3654,8 @@ def test_postgres_pool_snapshot_reward_window_timestamps_come_from_window_rows(s "participants_3h": 0, "blocks_found_total": 0, "prism_blocks_total": 0, + "blocks_reversed_total": 0, + "blocks_inactive_total": 0, "total_mined_bits": 0, "latest_block": None, "oldest_share_accepted_at": None, @@ -2346,6 +3740,10 @@ def test_postgres_dashboard_pending_fanout_rows_include_broadcast_attempts(self) self.assertIn("'broadcast_attempt_summary'", query) self.assertIn("broadcast_attempt_count", query) self.assertIn("qbit_ctv_fanout_broadcast_attempts", query) + # Unqualified, the outer fanout_txid resolves to attempt itself and every + # row would carry every attempt in the table. + self.assertIn("WHERE attempt.fanout_txid = page_rows.fanout_txid", query) + self.assertNotIn("WHERE attempt.fanout_txid = fanout_txid", query) self.assertIn("settlement_status NOT IN ('confirmed', 'reorged', 'failed')", query) self.assertIn("artifact.next_broadcast_attempt_at IS NULL", query) self.assertIn("artifact.next_broadcast_attempt_at <= clock_timestamp()", query) @@ -2533,6 +3931,8 @@ def test_postgres_pool_snapshot_reward_window_allows_zero_difficulty(self) -> No "participants_3h": 0, "blocks_found_total": 0, "prism_blocks_total": 0, + "blocks_reversed_total": 0, + "blocks_inactive_total": 0, "total_mined_bits": 0, "latest_block": None, "oldest_share_accepted_at": None, @@ -2746,6 +4146,8 @@ def test_postgres_public_block_solver_queries_use_suffix_index_expression(self) "participants_3h": 0, "blocks_found_total": 0, "prism_blocks_total": 0, + "blocks_reversed_total": 0, + "blocks_inactive_total": 0, "total_mined_bits": 0, "latest_block": None, "oldest_share_accepted_at": None, @@ -2798,6 +4200,98 @@ def test_postgres_dashboard_blocks_reads_bits_from_promoted_column_then_bundle(s self.assertIn("'bits', COALESCE(rows.audit_bits, '00000000')", query) self.assertNotIn("rows.audit_bundle#>>", query) + def test_postgres_dashboard_blocks_chain_state_filters_partition_count_and_rows(self) -> None: + ledger = FakeLeasePsqlShareLedger( + [ + acquired_lease(), + {"total_count": 0, "rows": []}, + {"total_count": 0, "rows": []}, + {"total_count": 0, "rows": []}, + ] + ) + + ledger.dashboard_blocks(page=1, limit=15) + default_query = ledger.lease_queries[-1] + ledger.dashboard_blocks(page=1, limit=15, chain_state="all") + all_query = ledger.lease_queries[-1] + ledger.dashboard_blocks(page=1, limit=15, chain_state="reversed") + reversed_query = ledger.lease_queries[-1] + + # The default emits exactly the pre-filter SQL: reversed rows hidden + # from both the page and the total, and no state fields in the rows. + self.assertIn("WHERE chain_state <> 'reversed'", default_query) + self.assertIn("WHERE block.chain_state <> 'reversed'", default_query) + self.assertNotIn("'chain_state'", default_query) + self.assertNotIn("disconnected_at", default_query) + + # chain_state=all drops the filter from the page and the pagination + # count alike, and surfaces the public state fields. + self.assertIn("WHERE true", all_query) + self.assertNotIn("WHERE chain_state", all_query) + self.assertNotIn("WHERE block.chain_state", all_query) + self.assertIn("'chain_state', rows.chain_state", all_query) + # disconnected_at is a reorg disconnect time: rejected rows carry the + # column too (their maturity flip requires it), so the public row + # masks it to reversed chain states. + self.assertIn( + "'disconnected_at', CASE WHEN rows.chain_state = 'reversed'" + " THEN to_char(rows.disconnected_at AT TIME ZONE 'UTC'", + all_query, + ) + + # chain_state=reversed inverts the default filter in both places. + self.assertIn("WHERE chain_state = 'reversed'", reversed_query) + self.assertIn("WHERE block.chain_state = 'reversed'", reversed_query) + self.assertIn("'chain_state', rows.chain_state", reversed_query) + + with self.assertRaisesRegex(ValueError, "chain_state"): + ledger.dashboard_blocks(page=1, limit=15, chain_state="orphaned") + + def test_postgres_pool_snapshot_reports_reorg_state_counts(self) -> None: + ledger = FakeLeasePsqlShareLedger( + [ + acquired_lease(), + { + "h1_difficulty": "0", + "h3_difficulty": "0", + "h24_difficulty": "0", + "participants_3h": 0, + "blocks_found_total": 7, + "prism_blocks_total": 7, + "blocks_reversed_total": 4, + "blocks_inactive_total": 2, + "total_mined_bits": 0, + "latest_block": None, + "oldest_share_accepted_at": None, + "newest_share_accepted_at": None, + "included_share_count": 0, + }, + ] + ) + + snapshot = ledger.dashboard_pool_snapshot( + current_network_difficulty="1", generated_at=public_api.utc_now_iso() + ) + query = ledger.lease_queries[-1] + + # The public reorg counters reuse the audit summary's per-state count + # shape, and blocks_found_total keeps excluding only reversed rows. + self.assertIn( + "'blocks_reversed_total', (SELECT count(*) FROM qbit_pool_blocks WHERE chain_state = 'reversed')", + query, + ) + self.assertIn( + "'blocks_inactive_total', (SELECT count(*) FROM qbit_pool_blocks WHERE chain_state = 'inactive')", + query, + ) + self.assertIn( + "'blocks_found_total', (SELECT count(*) FROM qbit_pool_blocks WHERE chain_state <> 'reversed')", + query, + ) + self.assertEqual(snapshot["blocks_reversed_total"], 4) + self.assertEqual(snapshot["blocks_inactive_total"], 2) + self.assertEqual(snapshot["blocks_found_total"], 7) + def test_psql_externalizes_audit_body_and_resolves_round_trip(self) -> None: with tempfile.TemporaryDirectory() as tmp: ledger = FakeLeasePsqlShareLedger( @@ -2871,18 +4365,20 @@ def test_psql_reuses_complete_10k_share_slot_without_parse_merge_or_memory_ampli expected_bytes = segment_path.read_bytes() expected_file_sha256 = hashlib.sha256(expected_bytes).hexdigest() expected_mtime_ns = segment_path.stat().st_mtime_ns + store = ledger._audit_artifact_store + assert store is not None gc.collect() tracemalloc.start() try: with ( unittest.mock.patch( - "lab.prism.share_ledger.json.loads", + "lab.prism.audit_artifacts.json.loads", side_effect=AssertionError("complete share slot must not be parsed"), ), unittest.mock.patch.object( - ledger, - "_merge_audit_share_ranges", + store, + "merge_audit_share_ranges", side_effect=AssertionError("complete share slot must not be merged"), ), ): @@ -3152,7 +4648,8 @@ def test_psql_canonical_bundle_path_skips_canonicalizer_and_is_retry_safe(self) with tempfile.TemporaryDirectory() as tmp: root = Path(tmp) body_dir = root / "body-store" - candidate_path = root / "canonical-candidate.json" + body_dir.mkdir() + candidate_path = body_dir / "canonical-candidate.json" bundle = { "schema": "qbit.prism.audit-bundle.v1", "shares": [{"share_seq": 1, "share_id": "s1"}], @@ -3213,11 +4710,12 @@ def test_psql_canonical_bundle_path_skips_canonicalizer_and_is_retry_safe(self) canonicalizer.assert_not_called() self.assertEqual(list(body_dir.glob(".*.tmp")), []) - def test_psql_compact_v2_retry_stays_bounded_and_skips_reconstruction(self) -> None: + def test_psql_compact_v2_retry_reconstructs_without_recanonicalizing(self) -> None: with tempfile.TemporaryDirectory() as tmp: root = Path(tmp) body_dir = root / "body-store" - candidate_path = root / "canonical-candidate.json" + body_dir.mkdir() + candidate_path = body_dir / "canonical-candidate.json" bundle = { "schema": "qbit.prism.audit-bundle.v1", "shares": [ @@ -3269,18 +4767,28 @@ def test_psql_compact_v2_retry_stays_bounded_and_skips_reconstruction(self) -> N json.loads(body_path.read_text(encoding="utf-8"))["schema"], "qbit.prism.audit-bundle.v2", ) + store = ledger._audit_artifact_store + assert store is not None with ( unittest.mock.patch.object( - ledger, - "_external_body_matches_sha", + store, + "external_body_matches_sha", side_effect=AssertionError("same-version compact retry must compare bounded storage"), ), + # The retry verifies the existing compact body by streaming + # its segments against the logical window (issue #255); it + # never reconstructs a whole-window bundle. unittest.mock.patch.object( - ledger, - "_resolve_audit_bundle_v2", - side_effect=AssertionError("compact retry must not reconstruct every segment"), + store, + "resolve_audit_bundle_v2", + side_effect=AssertionError("compact retry must not reconstruct the whole window"), ), + unittest.mock.patch.object( + store, + "_compact_body_reconstructs_to", + wraps=store._compact_body_reconstructs_to, + ) as verified, ): second_uri = ledger._prepare_external_audit_body( payload, @@ -3296,13 +4804,72 @@ def test_psql_compact_v2_retry_stays_bounded_and_skips_reconstruction(self) -> N self.assertEqual(path.stat().st_ino, first_stat.st_ino) self.assertEqual(path.stat().st_mtime_ns, first_stat.st_mtime_ns) canonicalizer.assert_not_called() + verified.assert_called_once() + self.assertEqual(verified.call_args.kwargs["expected"], body_sha) self.assertEqual(list(body_dir.glob(".*.tmp")), []) + def test_psql_compact_body_rejects_canonical_and_logical_bundle_mismatch(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + body_dir = root / "body-store" + body_dir.mkdir() + candidate_path = body_dir / "canonical-candidate.json" + canonical_bundle = { + "schema": "qbit.prism.audit-bundle.v1", + "shares": [ + {"share_seq": 1, "share_id": "canonical-a"}, + {"share_seq": 2, "share_id": "canonical-a-2"}, + ], + } + logical_bundle = { + "schema": "qbit.prism.audit-bundle.v1", + "shares": [ + {"share_seq": 1, "share_id": "logical-b"}, + {"share_seq": 2, "share_id": "logical-b-2"}, + ], + } + canonical_bytes = fake_audit_bundle_bytes(canonical_bundle) + candidate_path.write_bytes(canonical_bytes) + body_sha = hashlib.sha256(canonical_bytes).hexdigest() + ledger = FakeLeasePsqlShareLedger( + [acquired_lease()], + audit_body_dir=body_dir, + audit_bundle_canonicalizer=unittest.mock.Mock( + side_effect=AssertionError("literal candidate should be authoritative") + ), + audit_share_segment_size=1, + ) + payload = { + "block_hash": "aa" * 32, + "audit_bundle_sha256": body_sha, + "body_uri": str( + body_dir + / f"prism-audit-bundle-body-{'aa' * 32}-{body_sha}.json" + ), + } + + with self.assertRaisesRegex(RuntimeError, "does not match logical bundle"): + ledger._prepare_external_audit_body( + payload, + logical_bundle, + canonical_bundle_path=candidate_path, + ) + + self.assertEqual( + [ + path + for path in body_dir.iterdir() + if path.name != ".prism-audit-publication.lock" + ], + [candidate_path], + ) + def test_psql_canonical_bundle_path_hash_mismatch_publishes_nothing(self) -> None: with tempfile.TemporaryDirectory() as tmp: root = Path(tmp) body_dir = root / "body-store" - candidate_path = root / "canonical-candidate.json" + body_dir.mkdir() + candidate_path = body_dir / "canonical-candidate.json" bundle = {"schema": "qbit.prism.audit-bundle.v1", "shares": []} candidate_path.write_bytes(fake_audit_bundle_bytes(bundle)) canonicalizer = unittest.mock.Mock( @@ -3339,7 +4906,14 @@ def test_psql_canonical_bundle_path_hash_mismatch_publishes_nothing(self) -> Non canonicalizer.assert_not_called() self.assertEqual(len(ledger.lease_results), 1) - self.assertFalse(body_dir.exists()) + self.assertEqual( + [ + path + for path in body_dir.iterdir() + if path.name != ".prism-audit-publication.lock" + ], + [candidate_path], + ) def test_psql_compact_audit_body_writes_v2_range_proof_and_resolves_v1_bundle(self) -> None: with tempfile.TemporaryDirectory() as tmp: @@ -3395,8 +4969,195 @@ def test_psql_compact_audit_body_writes_v2_range_proof_and_resolves_v1_bundle(se audit_body_dir=tmp, audit_bundle_canonicalizer=fake_audit_bundle_bytes, ) - resolved = reader._read_external_body(body_uri, expected_sha256=body_sha) - self.assertEqual(resolved, bundle) + resolved = reader._read_external_body(body_uri, expected_sha256=body_sha) + self.assertEqual(resolved, bundle) + + def test_psql_v1_and_v2_body_tamper_fail_read_and_availability(self) -> None: + def assert_tamper_rejected( + ledger: PsqlShareLedger, + body_path: Path, + digest: str, + original: dict[str, Any], + mutate: Any, + ) -> None: + tampered = json.loads(json.dumps(original)) + mutate(tampered) + body_path.write_text( + json.dumps(tampered, separators=(",", ":")), + encoding="utf-8", + ) + self.assertFalse( + ledger._external_body_available_for_sha(str(body_path), digest) + ) + with self.assertRaises(RuntimeError): + ledger._read_external_body( + str(body_path), + expected_sha256=digest, + ) + + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + ledger = FakeLeasePsqlShareLedger( + [acquired_lease()], + audit_body_dir=root, + audit_bundle_canonicalizer=fake_audit_bundle_bytes, + ) + v1_bundle = { + "schema": "qbit.prism.audit-bundle.v1", + "shares": [ + {"share_seq": 1, "share_id": "s1"}, + {"share_seq": 2, "share_id": "s2"}, + ], + "found_block": {"bits": "207fffff"}, + } + v1_digest = fake_audit_bundle_sha256(v1_bundle) + v1_body = { + "schema": AUDIT_BODY_REF_SCHEMA, + "block_hash": "aa" * 32, + "audit_bundle_sha256": v1_digest, + "bundle_without_shares": { + "schema": "qbit.prism.audit-bundle.v1", + "found_block": {"bits": "207fffff"}, + }, + "share_count": 2, + "shares_key_index": 1, + "share_parts": [ + { + "kind": "inline", + "first_share_seq": 1, + "last_share_seq": 1, + "share_count": 1, + "shares": [{"share_seq": 1, "share_id": "s1"}], + }, + { + "kind": "inline", + "first_share_seq": 2, + "last_share_seq": 2, + "share_count": 1, + "shares": [{"share_seq": 2, "share_id": "s2"}], + }, + ], + } + v1_path = root / ( + f"prism-audit-bundle-body-{'aa' * 32}-{v1_digest}.json" + ) + v1_path.write_text( + json.dumps(v1_body, separators=(",", ":")), + encoding="utf-8", + ) + self.assertEqual( + ledger._read_external_body( + str(v1_path), + expected_sha256=v1_digest, + ), + v1_bundle, + ) + v1_mutations = ( + lambda body: body.__setitem__("block_hash", "bb" * 32), + lambda body: body["bundle_without_shares"]["found_block"].__setitem__( + "bits", + "1d00ffff", + ), + lambda body: body.__setitem__("shares_key_index", 0), + lambda body: body.__setitem__( + "share_parts", + list(reversed(body["share_parts"])), + ), + lambda body: body["share_parts"][0].__setitem__( + "first_share_seq", + 2, + ), + ) + for index, mutate in enumerate(v1_mutations): + with self.subTest(schema="v1", mutation=index): + assert_tamper_rejected( + ledger, + v1_path, + v1_digest, + v1_body, + mutate, + ) + + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + v2_bundle = { + "schema": "qbit.prism.audit-bundle.v1", + "shares": [ + {"share_seq": 1, "share_id": "s1"}, + {"share_seq": 2, "share_id": "s2"}, + ], + "reward_manifest": { + "anchor_job_issued_at_ms": 100, + "anchor_share_seq": 1, + "newest_share_seq": 2, + "oldest_share_seq": 1, + "included_share_count": 2, + "requested_window_weight": 20, + "counted_window_weight": 20, + "share_slice_digest_hex": "44" * 32, + }, + } + v2_digest = fake_audit_bundle_sha256(v2_bundle) + writer = FakeLeasePsqlShareLedger( + [ + acquired_lease(), + {"existing_block": False, "existing_body_uri": None}, + ], + audit_body_dir=root, + audit_bundle_canonicalizer=fake_audit_bundle_bytes, + audit_share_segment_size=1, + ) + body_uri = writer._prepare_external_audit_body( + { + "block_hash": "cc" * 32, + "audit_bundle_sha256": v2_digest, + "coinbase_tx_hex": "00", + "coinbase_txid": "11" * 32, + "payout_manifest_sha256": "22" * 32, + "block_height": 10, + "parent_hash": "bb" * 32, + "writer_id": writer._writer_id, + "writer_epoch": writer._writer_epoch, + "writer_session_token": writer._writer_session_token, + }, + v2_bundle, + ) + assert body_uri is not None + v2_path = Path(body_uri) + v2_body = json.loads(v2_path.read_text(encoding="utf-8")) + self.assertEqual(v2_body["schema"], AUDIT_BUNDLE_V2_SCHEMA) + self.assertEqual( + writer._read_external_body( + body_uri, + expected_sha256=v2_digest, + ), + v2_bundle, + ) + v2_mutations = ( + lambda body: body.__setitem__("block_hash", "dd" * 32), + lambda body: body["bundle_without_shares"]["reward_manifest"].__setitem__( + "included_share_count", + 3, + ), + lambda body: body.__setitem__("shares_key_index", 0), + lambda body: body["share_window_proof"].__setitem__( + "share_parts", + list(reversed(body["share_window_proof"]["share_parts"])), + ), + lambda body: body["share_window_proof"].__setitem__( + "included_share_count", + 3, + ), + ) + for index, mutate in enumerate(v2_mutations): + with self.subTest(schema="v2", mutation=index): + assert_tamper_rejected( + writer, + v2_path, + v2_digest, + v2_body, + mutate, + ) def test_psql_v2_range_segments_grow_without_breaking_old_refs(self) -> None: with tempfile.TemporaryDirectory() as tmp: @@ -3499,12 +5260,14 @@ def test_psql_public_artifact_resolves_external_audit_bundle(self) -> None: "has_audit_row": True, "fallback": None, }, - ] + ], + audit_body_dir=tmp, + audit_bundle_canonicalizer=fake_audit_bundle_bytes, ) self.assertEqual(ledger.dashboard_public_artifact(sha256=body_sha), bundle) query = ledger.lease_queries[-1] - self.assertIn("SELECT audit_bundle, audit_bundle_sha256, body_uri", query) + self.assertIn("SELECT block_hash, audit_bundle, audit_bundle_sha256, body_uri", query) def test_psql_public_artifact_exists_uses_metadata_only(self) -> None: ledger = FakeLeasePsqlShareLedger( @@ -3524,6 +5287,264 @@ def test_psql_public_artifact_exists_uses_metadata_only(self) -> None: self.assertIn("FROM qbit_pool_audit_bundles", query) self.assertIn("body_uri", query) + def test_psql_public_artifact_document_reads_persisted_text_in_one_query(self) -> None: + manifest_set = sample_ctv_manifest_set() + manifest_set_json = canonical_json_text(manifest_set) + ledger = FakeLeasePsqlShareLedger( + [ + acquired_lease(), + { + "has_audit_row": False, + "audit_bundle": None, + "audit_bundle_sha256": None, + "body_uri": None, + "fallback": manifest_set, + "fallback_canonical": manifest_set_json, + }, + ] + ) + + document = ledger.dashboard_public_artifact_document(sha256=sha256_json_hex(manifest_set)) + + self.assertEqual(document, {"payload": manifest_set, "canonical_json": manifest_set_json}) + # The payload and its canonical text come from the same statement so a + # request touches each artifact table at most once. + query = ledger.lease_queries[-1] + self.assertIn("SELECT block_hash, audit_bundle, audit_bundle_sha256, body_uri", query) + self.assertIn("SELECT manifest_set, manifest_set_json", query) + self.assertIn("FROM qbit_ctv_fanout_sets", query) + self.assertIn("SELECT manifest, manifest_json", query) + self.assertIn("FROM qbit_ctv_fanout_artifacts", query) + self.assertEqual(query.count("FROM qbit_ctv_fanout_sets"), 1) + self.assertEqual(query.count("FROM qbit_ctv_fanout_artifacts"), 1) + + def test_psql_public_artifact_document_prefers_stored_canonical_bundle(self) -> None: + canonical_bytes = b'{"schema":"qbit.prism.audit-bundle.v1","accepted_shares":[]}' + digest = hashlib.sha256(canonical_bytes).hexdigest() + block_hash = "ab" * 32 + calls: list[tuple[str, str]] = [] + + def read_canonical(requested_block: str, requested_digest: str) -> bytes: + calls.append((requested_block, requested_digest)) + return canonical_bytes + + ledger = FakeLeasePsqlShareLedger( + [ + acquired_lease(), + { + "has_audit_row": True, + "block_hash": block_hash, + "audit_bundle": {"schema": "legacy-reconstructed"}, + "audit_bundle_sha256": digest, + "body_uri": None, + "fallback": None, + }, + ], + audit_artifact_store=types.SimpleNamespace( + read_canonical_audit_bundle=read_canonical, + ), + ) + + document = ledger.dashboard_public_artifact_document(sha256=digest) + + self.assertEqual( + document, + { + "payload": json.loads(canonical_bytes), + "canonical_json": canonical_bytes.decode(), + }, + ) + self.assertEqual(calls, [(block_hash, digest)]) + + def test_psql_public_artifact_serves_real_stored_bytes_without_reserialization(self) -> None: + canonical_bytes = b'{"schema":"qbit.prism.audit-bundle.v1","z":0,"a":1}' + digest = hashlib.sha256(canonical_bytes).hexdigest() + block_hash = "ab" * 32 + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + store = AuditArtifactStore( + AuditArtifactConfig( + root=root, + evidence_path=root / "evidence.json", + ) + ) + try: + store.write_canonical_audit_bundle( + block_hash, + digest, + canonical_bytes, + ) + ledger = FakeLeasePsqlShareLedger( + [ + acquired_lease(), + { + "has_audit_row": True, + "block_hash": block_hash, + "audit_bundle": {"schema": "legacy-reconstructed"}, + "audit_bundle_sha256": digest, + "body_uri": None, + "fallback": None, + }, + ], + audit_artifact_store=store, + ) + + response = public_api.artifact( + types.SimpleNamespace(ledger=ledger), + sha256=digest, + ) + + self.assertIsInstance(response, public_api.RawJsonBody) + self.assertEqual(response.body, canonical_bytes) # type: ignore[union-attr] + self.assertEqual(hashlib.sha256(response.body).hexdigest(), digest) # type: ignore[union-attr] + finally: + store.close() + + def test_psql_public_artifact_visible_row_missing_file_uses_legacy_body(self) -> None: + legacy = {"schema": "qbit.prism.audit-bundle.v1", "legacy": True} + ledger = FakeLeasePsqlShareLedger( + [ + acquired_lease(), + { + "has_audit_row": True, + "block_hash": "ab" * 32, + "audit_bundle": legacy, + "audit_bundle_sha256": "cd" * 32, + "body_uri": None, + "fallback": None, + }, + ], + audit_artifact_store=types.SimpleNamespace( + read_canonical_audit_bundle=lambda _block, _digest: None, + ), + ) + + with self.assertLogs("lab.prism.share_ledger", level="WARNING") as logs: + document = ledger.dashboard_public_artifact_document(sha256="cd" * 32) + + self.assertEqual( + document, + { + "payload": legacy, + "canonical_json": None, + "canonical_fallback_reason": "missing", + }, + ) + self.assertIn("reason=missing", "\n".join(logs.output)) + + def test_psql_public_artifact_corrupt_file_fails_closed(self) -> None: + canonical_bytes = b'{"schema":"qbit.prism.audit-bundle.v1"}' + digest = hashlib.sha256(canonical_bytes).hexdigest() + block_hash = "ab" * 32 + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + store = AuditArtifactStore( + AuditArtifactConfig( + root=root, + evidence_path=root / "evidence.json", + ) + ) + try: + path = store.write_canonical_audit_bundle( + block_hash, + digest, + canonical_bytes, + ) + path.chmod(0o600) + path.write_bytes(b"not a gzip stream") + ledger = FakeLeasePsqlShareLedger( + [ + acquired_lease(), + { + "has_audit_row": True, + "block_hash": block_hash, + "audit_bundle": {"schema": "legacy-reconstructed"}, + "audit_bundle_sha256": digest, + "body_uri": None, + "fallback": None, + }, + ], + audit_artifact_store=store, + ) + + with self.assertLogs( + "lab.prism.share_ledger", + level="ERROR", + ) as logs: + with self.assertRaises(CanonicalAuditBundleCorrupt): + ledger.dashboard_public_artifact_document(sha256=digest) + + output = "\n".join(logs.output) + self.assertIn("reason=corrupt", output) + self.assertIn(block_hash, output) + self.assertIn(digest, output) + finally: + store.close() + + def test_psql_public_artifact_file_before_replica_row_is_not_visible(self) -> None: + def unexpected_read(_block: str, _digest: str) -> bytes: + self.fail("canonical storage must not bypass replica row visibility") + + ledger = FakeLeasePsqlShareLedger( + [ + acquired_lease(), + { + "has_audit_row": False, + "block_hash": None, + "audit_bundle": None, + "audit_bundle_sha256": None, + "body_uri": None, + "fallback": None, + "fallback_canonical": None, + }, + ], + audit_artifact_store=types.SimpleNamespace( + read_canonical_audit_bundle=unexpected_read, + ), + ) + + self.assertIsNone(ledger.dashboard_public_artifact_document(sha256="ef" * 32)) + + def test_psql_public_artifact_exists_accepts_verified_canonical_file(self) -> None: + canonical_bytes = b'{"schema":"qbit.prism.audit-bundle.v1"}' + digest = hashlib.sha256(canonical_bytes).hexdigest() + block_hash = "ab" * 32 + ledger = FakeLeasePsqlShareLedger( + [ + acquired_lease(), + { + "has_audit_row": True, + "block_hash": block_hash, + "audit_bundle_sha256": digest, + "audit_bundle_inline": False, + "body_uri": "/missing/legacy-body.json", + "fallback_exists": False, + }, + ], + audit_artifact_store=types.SimpleNamespace( + read_canonical_audit_bundle=lambda _block, _digest: canonical_bytes, + ), + ) + + self.assertTrue(ledger.dashboard_public_artifact_exists(sha256=digest)) + + def test_psql_public_artifact_document_returns_none_when_unmatched(self) -> None: + ledger = FakeLeasePsqlShareLedger( + [ + acquired_lease(), + { + "has_audit_row": False, + "audit_bundle": None, + "audit_bundle_sha256": None, + "body_uri": None, + "fallback": None, + "fallback_canonical": None, + }, + ] + ) + + self.assertIsNone(ledger.dashboard_public_artifact_document(sha256="aa" * 32)) + def test_psql_public_artifact_exists_rejects_missing_external_body(self) -> None: with tempfile.TemporaryDirectory() as tmp: ledger = FakeLeasePsqlShareLedger( @@ -3544,6 +5565,11 @@ def test_psql_public_artifact_exists_rejects_missing_external_body(self) -> None def test_psql_public_artifact_exists_validates_compact_body_segments(self) -> None: with tempfile.TemporaryDirectory() as tmp: root = Path(tmp) + logical_bundle = { + "schema": "qbit.prism.audit-bundle.v1", + "shares": [{"share_seq": 1, "share_id": "s1"}], + } + body_sha = fake_audit_bundle_sha256(logical_bundle) segment = { "schema": "qbit.prism.audit-share-segment.v1", "first_share_seq": 1, @@ -3557,9 +5583,11 @@ def test_psql_public_artifact_exists_validates_compact_body_segments(self) -> No segment_path.write_bytes(segment_bytes) body_ref = { "schema": AUDIT_BODY_REF_SCHEMA, - "audit_bundle_sha256": "aa" * 32, + "block_hash": "bb" * 32, + "audit_bundle_sha256": body_sha, "bundle_without_shares": {"schema": "qbit.prism.audit-bundle.v1"}, "share_count": 1, + "shares_key_index": 1, "share_parts": [ { "kind": "segment", @@ -3571,7 +5599,7 @@ def test_psql_public_artifact_exists_validates_compact_body_segments(self) -> No } ], } - body_path = root / f"prism-audit-bundle-body-{'bb' * 32}-{'aa' * 32}.json" + body_path = root / f"prism-audit-bundle-body-{'bb' * 32}-{body_sha}.json" body_path.write_text(json.dumps(body_ref, separators=(",", ":")), encoding="utf-8") ledger = FakeLeasePsqlShareLedger( [ @@ -3584,9 +5612,10 @@ def test_psql_public_artifact_exists_validates_compact_body_segments(self) -> No }, ], audit_body_dir=tmp, + audit_bundle_canonicalizer=fake_audit_bundle_bytes, ) - self.assertTrue(ledger.dashboard_public_artifact_exists(sha256="aa" * 32)) + self.assertTrue(ledger.dashboard_public_artifact_exists(sha256=body_sha)) segment_path.unlink() ledger = FakeLeasePsqlShareLedger( [ @@ -3599,8 +5628,9 @@ def test_psql_public_artifact_exists_validates_compact_body_segments(self) -> No }, ], audit_body_dir=tmp, + audit_bundle_canonicalizer=fake_audit_bundle_bytes, ) - self.assertFalse(ledger.dashboard_public_artifact_exists(sha256="aa" * 32)) + self.assertFalse(ledger.dashboard_public_artifact_exists(sha256=body_sha)) def test_psql_public_artifact_exists_rejects_overstated_inline_share_count(self) -> None: body_ref = { @@ -3619,7 +5649,9 @@ def test_psql_public_artifact_exists_rejects_overstated_inline_share_count(self) ], } with tempfile.TemporaryDirectory() as tmp: - body_path = Path(tmp) / "body-ref.json" + body_path = Path(tmp) / ( + f"prism-audit-bundle-body-{'11' * 32}-{'aa' * 32}.json" + ) body_path.write_text(json.dumps(body_ref, separators=(",", ":")), encoding="utf-8") ledger = FakeLeasePsqlShareLedger([acquired_lease()], audit_body_dir=tmp) @@ -3634,6 +5666,7 @@ def test_psql_body_ref_respects_zero_shares_key_index(self) -> None: body_sha = fake_audit_bundle_sha256(bundle) body_ref = { "schema": AUDIT_BODY_REF_SCHEMA, + "block_hash": "11" * 32, "audit_bundle_sha256": body_sha, "share_count": 1, "shares_key_index": 0, @@ -3652,7 +5685,9 @@ def test_psql_body_ref_respects_zero_shares_key_index(self) -> None: ], } with tempfile.TemporaryDirectory() as tmp: - body_path = Path(tmp) / "body-ref.json" + body_path = Path(tmp) / ( + f"prism-audit-bundle-body-{'11' * 32}-{body_sha}.json" + ) body_path.write_text(json.dumps(body_ref, separators=(",", ":")), encoding="utf-8") ledger = FakeLeasePsqlShareLedger( [acquired_lease()], @@ -3689,7 +5724,9 @@ def test_psql_external_body_hash_mismatch_fails_readers(self) -> None: acquired_lease(), audit_row, {**audit_row, "audit_commitment_leaf_hex": "ab" * 32}, - ] + ], + audit_body_dir=tmp, + audit_bundle_canonicalizer=fake_audit_bundle_bytes, ) with self.assertRaisesRegex(RuntimeError, "hash mismatch"): ledger.audit_bundle(block_hash="aa" * 32) @@ -3706,7 +5743,9 @@ def test_psql_external_body_hash_mismatch_fails_readers(self) -> None: "has_audit_row": True, "fallback": None, }, - ] + ], + audit_body_dir=tmp, + audit_bundle_canonicalizer=fake_audit_bundle_bytes, ) with self.assertRaisesRegex(RuntimeError, "hash mismatch"): public_ledger.dashboard_public_artifact(sha256=body_sha) @@ -3825,6 +5864,10 @@ def test_psql_persist_rejects_report_digest_mismatch_before_body_write(self) -> audit_report=report, ) self.assertEqual(list(Path(tmp).glob("prism-audit-bundle-body-*.json")), []) + self.assertEqual( + list(Path(tmp).glob("prism-audit-bundle-canonical-*.json.gz")), + [], + ) def test_psql_persist_requires_lease_preflight_before_external_body_write(self) -> None: with tempfile.TemporaryDirectory() as tmp: @@ -3847,16 +5890,230 @@ def test_psql_persist_requires_lease_preflight_before_external_body_write(self) "coinbase_tx_hex": "00", } - with self.assertRaisesRegex(RuntimeError, "writer lease is not active"): - ledger.persist_accepted_block( - block_hash="aa" * 32, - block_height=10, - parent_hash="bb" * 32, - final_bundle=bundle, - audit_report=report, - ) - self.assertEqual(list(Path(tmp).glob("prism-audit-bundle-body-*.json")), []) - self.assertEqual(list(Path(tmp).glob("prism-audit-share-segment-*.json")), []) + with self.assertRaisesRegex(RuntimeError, "writer lease is not active"): + ledger.persist_accepted_block( + block_hash="aa" * 32, + block_height=10, + parent_hash="bb" * 32, + final_bundle=bundle, + audit_report=report, + ) + self.assertEqual(list(Path(tmp).glob("prism-audit-bundle-body-*.json")), []) + self.assertEqual(list(Path(tmp).glob("prism-audit-share-segment-*.json")), []) + # The canonical copy is a filesystem side effect like any other and + # must stay behind the same lease fence. + self.assertEqual( + list(Path(tmp).glob("prism-audit-bundle-canonical-*.json.gz")), + [], + ) + + def test_psql_persist_publishes_the_canonical_bundle_bytes(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + bundle = { + "schema": "qbit.prism.audit-bundle.v1", + "shares": [ + {"share_seq": 1, "share_id": "s1"}, + {"share_seq": 2, "share_id": "s2"}, + ], + "signed_coinbase_manifest": {"manifest": {"payout_count": 0}}, + "payout_policy_manifest": {"accounts": []}, + } + digest = fake_audit_bundle_sha256(bundle) + report = { + "coinbase_txid": "ee" * 32, + "coinbase_manifest_sha256_hex": "11" * 32, + "audit_bundle_sha256_hex": digest, + "coinbase_tx_hex": "00", + } + insert_result = { + "backend": "postgres-psql", + "share_count": 0, + "block_count": 1, + "bundle_count": 1, + "payout_entry_count": 0, + "carry_forward_count": 0, + "onchain_output_count": 0, + } + ledger = FakeLeasePsqlShareLedger( + [ + acquired_lease(), + {"existing_block": False, "existing_body_uri": None}, + insert_result, + ], + audit_body_dir=tmp, + audit_bundle_canonicalizer=fake_audit_bundle_bytes, + audit_share_segment_size=2, + ) + ledger.persist_accepted_block( + block_hash="aa" * 32, + block_height=10, + parent_hash="bb" * 32, + final_bundle=bundle, + audit_report=report, + ) + canonical_files = sorted( + Path(tmp).glob("prism-audit-bundle-canonical-*.json.gz") + ) + self.assertEqual(len(canonical_files), 1) + self.assertEqual( + canonical_files[0].name, + f"prism-audit-bundle-canonical-{'aa' * 32}-{digest}.json.gz", + ) + # The compact body stays the pointer the row references; the + # canonical copy carries the exact bytes the digest is taken over. + compressed = canonical_files[0].read_bytes() + self.assertEqual(gzip.decompress(compressed), fake_audit_bundle_bytes(bundle)) + body_files = sorted(Path(tmp).glob("prism-audit-bundle-body-*.json")) + self.assertEqual(len(body_files), 1) + self.assertNotEqual(body_files[0].read_bytes(), fake_audit_bundle_bytes(bundle)) + self.assertEqual( + ledger._audit_store().read_canonical_audit_bundle("aa" * 32, digest), + fake_audit_bundle_bytes(bundle), + ) + + # Re-persisting the same block republishes byte-for-byte. + ledger.lease_results.extend( + [{"existing_block": True, "existing_body_uri": str(body_files[0])}, insert_result] + ) + ledger.persist_accepted_block( + block_hash="aa" * 32, + block_height=10, + parent_hash="bb" * 32, + final_bundle=bundle, + audit_report=report, + ) + self.assertEqual(canonical_files[0].read_bytes(), compressed) + self.assertEqual( + sorted(Path(tmp).glob("prism-audit-bundle-canonical-*.json.gz")), + canonical_files, + ) + + def test_backfill_seams_bind_the_ledger_loader_and_the_owned_store(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + bundle = { + "schema": "qbit.prism.audit-bundle.v1", + "shares": [ + {"share_seq": 1, "share_id": "s1"}, + {"share_seq": 2, "share_id": "s2"}, + ], + "signed_coinbase_manifest": {"manifest": {"payout_count": 0}}, + "payout_policy_manifest": {"accounts": []}, + } + digest = fake_audit_bundle_sha256(bundle) + ledger = FakeLeasePsqlShareLedger( + [acquired_lease(), {"existing_block": False, "existing_body_uri": None}], + audit_body_dir=tmp, + audit_bundle_canonicalizer=fake_audit_bundle_bytes, + audit_share_segment_size=2, + ) + body_uri = ledger._prepare_external_audit_body( + {"block_hash": "aa" * 32, "audit_bundle_sha256": digest}, + bundle, + ) + assert body_uri is not None + canonical_path = Path( + ledger.publish_canonical_bundle_bytes( + block_hash="aa" * 32, + audit_bundle_sha256=digest, + canonical_bytes=fake_audit_bundle_bytes(bundle), + ) + ) + self.assertTrue(canonical_path.exists()) + + # Drop the segments; the seam must repair them from the ledger. + for segment in Path(tmp).glob("prism-audit-share-segment-*"): + segment.unlink() + loaded: list[tuple[int, int]] = [] + + def load_range(*, first_share_seq: int, last_share_seq: int) -> list[Any]: + loaded.append((first_share_seq, last_share_seq)) + return [ + {"share_seq": share_seq, "share_id": f"s{share_seq}"} + for share_seq in range(first_share_seq, last_share_seq + 1) + ] + + with unittest.mock.patch.object( + ledger, + "_load_audit_share_ledger_range", + side_effect=load_range, + ): + recovered = ledger.canonical_bundle_bytes_for_backfill( + block_hash="aa" * 32, + audit_bundle_sha256=digest, + body_uri=body_uri, + ) + self.assertEqual(recovered, fake_audit_bundle_bytes(bundle)) + self.assertEqual(loaded, [(1, 2)]) + + def test_preflight_canonical_bundle_publication_fences_each_backfill_write(self) -> None: + ledger = FakeLeasePsqlShareLedger( + [ + acquired_lease(), + { + "block_hash": "aa" * 32, + "audit_bundle_sha256": "22" * 32, + "body_uri": "prism-audit-bundle-body-x.json", + "has_inline_bundle": False, + }, + ] + ) + result = ledger.preflight_canonical_bundle_publication( + block_hash="aa" * 32, + audit_bundle_sha256="22" * 32, + ) + self.assertEqual( + result, + { + "block_hash": "aa" * 32, + "audit_bundle_sha256": "22" * 32, + "body_uri": "prism-audit-bundle-body-x.json", + "has_inline_bundle": False, + }, + ) + query = ledger.lease_queries[-1] + # Refreshes the lease this writer already holds, and reads the audit + # row it is about to answer for. No schema change, no row rewrite. + self.assertIn("UPDATE qbit_ledger_writer_lease", query) + self.assertIn("lease_expires_at = clock_timestamp() + make_interval", query) + self.assertIn("qbit_ledger_writer_lease.writer_session_token = data->>", query) + self.assertIn("FROM qbit_pool_audit_bundles", query) + self.assertIn("'writer lease is not active'", query) + self.assertIn("'audit bundle digest does not match the stored row'", query) + for forbidden in ( + "ALTER TABLE", + "CREATE ", + "INSERT INTO", + "DELETE FROM", + "UPDATE qbit_pool_audit_bundles", + ): + with self.subTest(forbidden=forbidden): + self.assertNotIn(forbidden, query) + + def test_preflight_canonical_bundle_publication_fails_closed(self) -> None: + for failure, expected in ( + ({"error": "writer lease is not active"}, "writer lease is not active"), + ({"error": "audit bundle row does not exist"}, "row does not exist"), + ( + {"error": "audit bundle digest does not match the stored row"}, + "digest does not match", + ), + (None, "returned no row"), + ): + with self.subTest(failure=failure): + ledger = FakeLeasePsqlShareLedger([acquired_lease(), failure]) + with self.assertRaisesRegex(RuntimeError, expected): + ledger.preflight_canonical_bundle_publication( + block_hash="aa" * 32, + audit_bundle_sha256="22" * 32, + ) + ledger = FakeLeasePsqlShareLedger([acquired_lease()]) + for block_hash, digest in (("aa" * 31, "22" * 32), ("aa" * 32, "zz" * 32)): + with self.subTest(block_hash=block_hash, digest=digest): + with self.assertRaises(ValueError): + ledger.preflight_canonical_bundle_publication( + block_hash=block_hash, + audit_bundle_sha256=digest, + ) def test_psql_external_body_path_must_stay_under_store(self) -> None: with tempfile.TemporaryDirectory() as tmp: @@ -3918,10 +6175,14 @@ def test_postgres_startup_waits_for_same_writer_predecessor_lease(self) -> None: writer_epoch=1, ) - self.assertEqual(ledger.sleeps, [1.0, 0.25]) + # The first wait is clamped by the fixture's retry ceiling (the + # adoption silence), the second by the retry floor. + retry_ceiling = max(1.0, DEFAULT_WRITER_LEASE_ADOPTION_SILENCE_SECONDS) + self.assertEqual(ledger.sleeps, [retry_ceiling, 0.25]) self.assertEqual(len(ledger.lease_queries), 3) self.assertIn( - "prism ledger writer lease held until 2026-06-26 19:50:22.233718+00; waiting 1s before retry", + "prism ledger writer lease held until 2026-06-26 19:50:22.233718+00; " + f"waiting {retry_ceiling:.3g}s before retry", stdout.getvalue(), ) self.assertIn("holder writer=writer-a epoch=1 session=old-session", stdout.getvalue()) @@ -4160,10 +6421,16 @@ def stop_after_second_sleep(seconds: float) -> None: lease_retry_sleep=stop_after_second_sleep, ) - # First wait is floored by the guard-acquisition silence (1.0s); once - # that elapsed, the renewing twin's fresh updated_at keeps gating the - # CAS through the row-silence edge (0.9s remaining). - self.assertEqual(sleeps, [1.0, 0.9]) + # First wait is floored by the guard-acquisition silence; once that + # elapsed, the renewing twin's fresh updated_at keeps gating the CAS + # through the row-silence edge (the silence less the row's 0.1s age). + self.assertEqual( + sleeps, + [ + DEFAULT_WRITER_LEASE_ADOPTION_SILENCE_SECONDS, + round(DEFAULT_WRITER_LEASE_ADOPTION_SILENCE_SECONDS - 0.1, 6), + ], + ) self.assertEqual(len(ledger.lease_queries), 2) self.assertTrue( all("observed_writer_session_token" not in query for query in ledger.lease_queries) @@ -4191,12 +6458,17 @@ def sleep_and_advance(seconds: float) -> None: held_lease( session=old_session, updated_at=first_updated_at, - age_seconds=1.1, + age_seconds=( + DEFAULT_WRITER_LEASE_ADOPTION_SILENCE_SECONDS + 0.1 + ), ), held_lease( session=old_session, updated_at=first_updated_at, - age_seconds=2.1, + age_seconds=( + 2 * DEFAULT_WRITER_LEASE_ADOPTION_SILENCE_SECONDS + + 0.1 + ), ), held_lease( session=old_session, @@ -4206,7 +6478,9 @@ def sleep_and_advance(seconds: float) -> None: held_lease( session=old_session, updated_at=renewed_updated_at, - age_seconds=1.1, + age_seconds=( + DEFAULT_WRITER_LEASE_ADOPTION_SILENCE_SECONDS + 0.1 + ), ), acquired_lease(session=new_session) | {"adopted": True}, ], @@ -4219,7 +6493,10 @@ def sleep_and_advance(seconds: float) -> None: # Guard-acquisition silence first, then a lost CAS re-observes the # renewed row and requires a fresh full row-silence interval before # the second CAS. - self.assertEqual(sleeps, [1.0, 1.0]) + self.assertEqual( + sleeps, + [DEFAULT_WRITER_LEASE_ADOPTION_SILENCE_SECONDS] * 2, + ) self.assertEqual(len(ledger.lease_queries), 5) self.assertIn("observed_writer_session_token", ledger.lease_queries[2]) self.assertIn("observed_writer_session_token", ledger.lease_queries[4]) @@ -4283,6 +6560,158 @@ def test_postgres_startup_acquires_expired_lease_immediately(self) -> None: self.assertEqual(stdout.getvalue(), "") +class FirstAcquireWriterLeaseRaceTests(unittest.TestCase): + """The first-ever concurrent acquisition, from the loser's side. + + Under READ COMMITTED the loser's ``ON CONFLICT DO UPDATE`` re-reads only + the conflicting tuple after its lock wait, so the upsert affects zero rows + while the holder ``SELECT`` still reads the pre-wait snapshot and finds + nothing. Both arms empty used to make the whole statement SQL NULL and kill + startup with a driver-level ``postgres query returned no JSON``. These + tests drive that interleaving through canned statement results, so they + pin the loser's experience without a live server. + """ + + def test_statement_carries_a_named_retry_arm_for_the_empty_snapshot(self) -> None: + ledger = FakeLeasePsqlShareLedger( + [acquired_lease()], + writer_id="writer-a", + writer_epoch=1, + ) + + acquire_sql = ledger.lease_queries[0] + # A third COALESCE arm makes the statement total: it is a constant, so + # the expression cannot evaluate to SQL NULL however the first two arms + # resolve. + self.assertIn(f"'{WRITER_LEASE_ACQUIRE_RETRY_KEY}', true", acquire_sql) + self.assertIn(f"'lease', '{WRITER_LEASE_ACQUIRE_RETRY_SUBJECT}'", acquire_sql) + self.assertIn("'acquired', false", acquire_sql) + + def test_first_acquire_loser_retries_into_the_named_lease_refusal(self) -> None: + stdout = io.StringIO() + + with contextlib.redirect_stdout(stdout): + with self.assertRaises(RuntimeError) as caught: + FakeLeasePsqlShareLedger( + [ + snapshot_retry_lease(), + held_lease(writer_id="writer-b", wait_seconds=10.0), + ], + writer_id="writer-a", + writer_epoch=1, + ) + + message = str(caught.exception) + # The operator-facing difference this change exists for: the loser now + # names the lease and its holder instead of a driver-level parse error. + self.assertIn( + "qbit ledger writer lease is held by writer-b epoch=1", + message, + ) + self.assertNotIn("postgres query returned no JSON", message) + + def test_first_acquire_loser_can_retry_into_an_ordinary_acquisition(self) -> None: + # The winner's row is visible on the fresh snapshot and has already + # expired, so the second statement takes the lease through the ordinary + # expiry CAS. Startup completes without operator action. + stdout = io.StringIO() + + with contextlib.redirect_stdout(stdout): + ledger = FakeLeasePsqlShareLedger( + [snapshot_retry_lease(), acquired_lease()], + writer_id="writer-a", + writer_epoch=1, + ) + + self.assertEqual(len(ledger.lease_queries), 2) + self.assertEqual(ledger.sleeps, []) + self.assertIn("fresh statement snapshot", stdout.getvalue()) + + def test_retry_result_never_reaches_the_wait_or_adopt_decisions(self) -> None: + # The sentinel carries no holder identity, so letting it out of the + # acquisition path would be read as a holder of None. It is consumed + # where the remedy is -- a fresh statement snapshot -- and the caller + # only ever sees an ordinary arm. + ledger = PsqlShareLedger.__new__(PsqlShareLedger) + ledger._writer_id = "writer-a" + ledger._writer_epoch = 1 + ledger._writer_session_token = ( + f"{WRITER_LEASE_HEARTBEAT_SESSION_PREFIX}session-a" + ) + retry = snapshot_retry_lease() + + self.assertFalse(PsqlShareLedger._can_wait_for_writer_lease(ledger, retry)) + self.assertIsNone( + PsqlShareLedger._writer_lease_adoption_wait_seconds(ledger, retry) + ) + self.assertFalse(retry["acquired"]) + + def test_unconverging_snapshot_retry_fails_naming_the_lease(self) -> None: + # A caller whose transaction pinned one snapshot for its whole life + # cannot advance it by re-running. The bound turns that into a visible + # failure that still names the lease, never a raw parser error and + # never an unbounded loop. + stdout = io.StringIO() + + with contextlib.redirect_stdout(stdout): + with self.assertRaises(RuntimeError) as caught: + FakeLeasePsqlShareLedger( + [snapshot_retry_lease()] * WRITER_LEASE_ACQUIRE_RETRY_ATTEMPTS, + writer_id="writer-a", + writer_epoch=1, + ) + + message = str(caught.exception) + self.assertIn(WRITER_LEASE_ACQUIRE_RETRY_SUBJECT, message) + self.assertNotIn("postgres query returned no JSON", message) + + def test_unrelated_empty_json_statement_still_raises_the_parser_error(self) -> None: + # The lease statement is what became total; the parser is unchanged, so + # any other statement returning SQL NULL still fails loudly rather than + # being reinterpreted as a retry. + with self.assertRaisesRegex( + RuntimeError, + "postgres query returned no JSON", + ): + parse_single_json_value(None) + + class FakeCursor: + @staticmethod + def fetchone() -> tuple[object]: + return (None,) + + class FakeConnection: + closed = False + + @staticmethod + def execute(sql: str) -> FakeCursor: + return FakeCursor() + + def close(self) -> None: + pass + + class FakeOperationalError(Exception): + pass + + fake_psycopg = types.ModuleType("psycopg") + fake_psycopg.connect = lambda conninfo, **kwargs: FakeConnection() # type: ignore[attr-defined] + fake_psycopg.conninfo = types.SimpleNamespace( # type: ignore[attr-defined] + conninfo_to_dict=lambda conninfo: {"dbname": "qbit"} + ) + fake_psycopg.OperationalError = FakeOperationalError # type: ignore[attr-defined] + + with unittest.mock.patch.dict(sys.modules, {"psycopg": fake_psycopg}): + client = _NativePostgresClient( + "postgresql://example.invalid/qbit", + pool_size=1, + ) + with self.assertRaisesRegex( + RuntimeError, + "postgres query returned no JSON", + ): + client.run_json("SELECT to_json(NULL);") + + class CannedQueryPsqlShareLedger(PsqlShareLedger): """Subprocess-mode ledger whose _run_json pops canned results. @@ -4370,6 +6799,417 @@ def test_operation_timeout_bounds_local_writer_lock_admission(self) -> None: gate.release() self.assertLess(time.monotonic() - started, 0.5) + def test_operation_gate_stamps_progress_while_admission_is_blocked(self) -> None: + """A blocked admission wait must report liveness to its caller. + + Landing-class callers run this wait on a watchdog-monitored thread. + No statement has been sent yet, so nothing on the server can bound or + report the wait; without the progress hook the whole admission budget + is silence and the watchdog kills a coordinator that is merely queued + behind another writer. Slicing must not extend the wait either: the + caller's deadline still ends it with the same error. + """ + ledger = PsqlShareLedger.__new__(PsqlShareLedger) + ledger._operation_timeout_local = threading.local() + ledger._statement_timeout_local = threading.local() + gate = threading.Lock() + gate.acquire() + stamps: list[float] = [] + started = time.monotonic() + try: + with ledger.statement_timeout(0.4): + with ledger.operation_progress( + lambda: stamps.append(time.monotonic()), + slice_seconds=0.05, + ): + with self.assertRaisesRegex( + LedgerOperationTimeout, + "writer lock", + ): + with ledger._operation_gate(gate, "writer lock"): + self.fail("contended writer lock unexpectedly acquired") + finally: + gate.release() + elapsed = time.monotonic() - started + self.assertGreaterEqual(len(stamps), 2) + # No per-gap upper bound: a loaded CI host can stall a 0.05s slice + # wakeup well past any tight cutoff, and cadence is already proven + # deterministically under a virtual clock elsewhere in this module. + self.assertGreaterEqual(elapsed, 0.4) + self.assertLess(elapsed, 3.0) + + def test_operation_gate_slicing_times_out_on_the_virtual_clock(self) -> None: + """The sliced admission wait is driven by the injected clock. + + With a virtual clock installed and a gate that consumes each slice + without yielding, the deadline must expire exactly when the virtual + clock crosses the admission budget — never on wall time — so the + slicing loop can be exercised deterministically. + """ + clock = FakeMonotonicClock() + started = clock.now + + class SliceConsumingGate: + def __init__(self) -> None: + self.timeouts: list[float] = [] + + def acquire(self, timeout: float = -1) -> bool: + self.timeouts.append(timeout) + if len(self.timeouts) > 50: + raise AssertionError( + "admission wait ignored the virtual-clock deadline" + ) + clock.sleep(timeout) + return False + + def release(self) -> None: + raise AssertionError("gate released without being acquired") + + ledger = PsqlShareLedger.__new__(PsqlShareLedger) + ledger._operation_timeout_local = threading.local() + ledger._statement_timeout_local = threading.local() + ledger._monotonic = clock.monotonic + gate = SliceConsumingGate() + stamps: list[float] = [] + with ledger.statement_timeout(1.0): + with ledger.operation_progress( + lambda: stamps.append(clock.now), + slice_seconds=0.25, + ): + with self.assertRaisesRegex(LedgerOperationTimeout, "writer lock"): + ledger._acquire_operation_gate(gate, "writer lock") + self.assertEqual(clock.now, started + 1.0) + self.assertEqual(gate.timeouts, [0.25, 0.25, 0.25, 0.25]) + self.assertEqual( + stamps, + [started + 0.25, started + 0.5, started + 0.75], + ) + + def test_operation_gate_without_progress_hook_waits_once(self) -> None: + """The unhooked path is still a single blocking acquire. + + Ordinary ledger callers have no liveness monitor to satisfy and must + not start paying for wakeups they cannot use. + """ + acquires: list[float] = [] + + class RecordingGate: + def acquire(self, timeout: float = -1) -> bool: + acquires.append(timeout) + return False + + def release(self) -> None: + raise AssertionError("gate released without being acquired") + + ledger = PsqlShareLedger.__new__(PsqlShareLedger) + ledger._operation_timeout_local = threading.local() + ledger._statement_timeout_local = threading.local() + with ledger.statement_timeout(0.02): + with self.assertRaisesRegex(LedgerOperationTimeout, "writer lock"): + with ledger._operation_gate(RecordingGate(), "writer lock"): + self.fail("contended writer lock unexpectedly acquired") + self.assertEqual(acquires, [0.02]) + + def test_operation_progress_validates_and_restores_its_slice(self) -> None: + """The hook scope matches its sibling timeout scopes exactly.""" + ledger = PsqlShareLedger.__new__(PsqlShareLedger) + for slice_seconds in (0.0, -1.0, float("inf"), float("nan")): + with self.subTest(slice_seconds=slice_seconds): + with self.assertRaisesRegex(ValueError, "finite and positive"): + with ledger.operation_progress( + lambda: None, + slice_seconds=slice_seconds, + ): + self.fail("invalid progress slice unexpectedly accepted") + self.assertIsNone(ledger._operation_progress_hook()) + def outer() -> None: + return None + + def inner() -> None: + return None + + with ledger.operation_progress(outer, slice_seconds=1.0): + self.assertEqual(ledger._operation_progress_hook(), (outer, 1.0)) + with ledger.operation_progress(inner, slice_seconds=0.25): + self.assertEqual(ledger._operation_progress_hook(), (inner, 0.25)) + self.assertEqual(ledger._operation_progress_hook(), (outer, 1.0)) + self.assertIsNone(ledger._operation_progress_hook()) + + def test_operation_progress_local_is_created_before_any_scope(self) -> None: + """The production path must never lazily create the thread-local. + + ``operation_progress``'s ``getattr``-then-assign is not atomic, so a + ledger that reaches its first scope without the attribute already in + place can lose one caller's hook to a concurrent installer. + """ + with contextlib.redirect_stdout(io.StringIO()): + ledger = FakeLeasePsqlShareLedger( + [acquired_lease()], + writer_id="writer-a", + writer_epoch=1, + ) + self.assertIn("_operation_progress_local", vars(ledger)) + self.assertIsInstance( + vars(ledger)["_operation_progress_local"], + threading.local, + ) + + def test_concurrent_first_scopes_share_one_progress_local(self) -> None: + """Two first-ever scopes at once must both keep their own hook. + + The lazy check-then-set lets both callers build a ``local()`` and one + assignment win; the loser's hook then lives on an object the ledger + no longer references, so its admission wait silently reverts to the + heartbeat-silent path -- the exact failure this hook exists to + remove. The patched ``local`` below makes that interleaving + deterministic rather than a sleep race: it parks every constructing + thread on a two-party barrier, so neither can assign before the other + has decided to construct. A ledger that built the thread-local in + ``__init__`` never constructs one here at all, and the barrier is + simply never reached. + """ + with contextlib.redirect_stdout(io.StringIO()): + ledger = FakeLeasePsqlShareLedger( + [acquired_lease()], + writer_id="writer-a", + writer_epoch=1, + ) + constructing = threading.Barrier(2, timeout=10) + installed = threading.Barrier(2, timeout=10) + + def racing_local() -> threading.local: + # Neither thread may leave the constructor until both have + # decided to build one, so the check-then-set cannot serialize + # itself by luck. + constructing.wait() + return threading.local() + + def first_stamp() -> None: + return None + + def second_stamp() -> None: + return None + + observed: dict[str, object] = {} + errors: list[BaseException] = [] + + def install(name: str, hook: Any) -> None: + try: + with ledger.operation_progress(hook, slice_seconds=0.5): + # Both hooks are installed before either is read, so a + # lost assignment cannot hide behind thread ordering. + installed.wait() + observed[name] = ledger._operation_progress_hook() + except BaseException as exc: # pragma: no cover - asserted below + errors.append(exc) + + with unittest.mock.patch.object( + share_ledger_module, + "local", + racing_local, + ): + threads = [ + threading.Thread(target=install, args=("first", first_stamp)), + threading.Thread(target=install, args=("second", second_stamp)), + ] + for thread in threads: + thread.start() + for thread in threads: + thread.join(10) + self.assertEqual(errors, []) + self.assertEqual(observed.get("first"), (first_stamp, 0.5)) + self.assertEqual(observed.get("second"), (second_stamp, 0.5)) + + def test_operation_progress_nesting_keeps_the_tighter_slice(self) -> None: + """Nesting tightens the stamp cadence, exactly like its siblings. + + ``operation_timeout`` and ``statement_timeout`` merge by minimum, so + a nested scope can only narrow what the enclosing caller allowed. A + progress scope that replaced the slice outright would let an inner + caller widen the heartbeat gaps an outer caller had already sized + against its own monitor. + """ + ledger = PsqlShareLedger.__new__(PsqlShareLedger) + + def outer() -> None: + return None + + def inner() -> None: + return None + + with ledger.operation_progress(outer, slice_seconds=0.25): + self.assertEqual(ledger._operation_progress_hook(), (outer, 0.25)) + # A larger inner slice does not widen the cadence; the callback + # is still the inner caller's, because it is the one whose + # liveness the wait now belongs to. + with ledger.operation_progress(inner, slice_seconds=4.0): + self.assertEqual( + ledger._operation_progress_hook(), + (inner, 0.25), + ) + self.assertEqual(ledger._operation_progress_hook(), (outer, 0.25)) + self.assertIsNone(ledger._operation_progress_hook()) + + def test_operation_gate_propagates_progress_hook_failures(self) -> None: + """A liveness stamp that cannot be taken is a failure, not noise. + + Swallowing it would leave the caller blocked inside a lock wait it + believes is being reported, which is the exact silence this hook + exists to remove. + """ + + class Stamped(RuntimeError): + pass + + def on_progress() -> None: + raise Stamped("heartbeat unavailable") + + ledger = PsqlShareLedger.__new__(PsqlShareLedger) + ledger._operation_timeout_local = threading.local() + ledger._statement_timeout_local = threading.local() + gate = threading.Lock() + gate.acquire() + try: + with ledger.statement_timeout(5.0): + with ledger.operation_progress(on_progress, slice_seconds=0.01): + with self.assertRaises(Stamped): + with ledger._operation_gate(gate, "writer lock"): + self.fail("contended writer lock unexpectedly acquired") + finally: + gate.release() + + def test_note_operation_progress_reports_only_when_a_hook_is_installed(self) -> None: + """The between-statements reporter is opt-in, exactly like the wait. + + Ordinary ledger callers have no liveness monitor, so an operation + that reports unconditionally would be paying for stamps nobody + reads. The hook is installed only by callers whose thread is + watchdog-monitored. + """ + ledger = PsqlShareLedger.__new__(PsqlShareLedger) + ledger._operation_timeout_local = threading.local() + ledger._statement_timeout_local = threading.local() + # No hook installed: reporting must be a silent no-op, not an error. + ledger._note_operation_progress() + stamps: list[int] = [] + with ledger.operation_progress( + lambda: stamps.append(len(stamps)), + slice_seconds=0.05, + ): + ledger._note_operation_progress() + ledger._note_operation_progress() + self.assertEqual(stamps, [0, 1]) + # The scope restored the previous (absent) hook, so this is silent. + ledger._note_operation_progress() + self.assertEqual(stamps, [0, 1]) + + def test_note_operation_progress_propagates_hook_failures(self) -> None: + """A stamp that cannot be taken is a failure, as inside the gate wait.""" + + class Stamped(RuntimeError): + pass + + def on_progress() -> None: + raise Stamped("heartbeat unavailable") + + ledger = PsqlShareLedger.__new__(PsqlShareLedger) + ledger._operation_timeout_local = threading.local() + ledger._statement_timeout_local = threading.local() + with ledger.operation_progress(on_progress, slice_seconds=0.05): + with self.assertRaises(Stamped): + ledger._note_operation_progress() + + def test_confirm_accepted_block_reports_between_its_two_statements(self) -> None: + """The publication-ordinal read must not extend one silent span. + + confirm_accepted_block holds the writer lock across a mutating + statement and the follow-up read its command snapshot could not see. + No admission slice runs between them, so without a report the caller's + monitor is asked to sit through two statement budgets in a row -- at + the 120s default that is the whole watchdog tolerance with no margin + at all (issue #125). + """ + ledger = GateStatementRecordingPsqlShareLedger( + [ + {"backend": "postgres-psql", "confirmed_count": 1}, + {"audit_publication_sequence": 7}, + ] + ) + with ledger.operation_progress( + ledger.note_progress, + slice_seconds=0.05, + ): + response = ledger.confirm_accepted_block( + block_hash="ab" * 32, + active_tip_height=10, + ) + self.assertEqual(response["audit_publication_sequence"], 7) + self.assertEqual( + ledger.events, + ["statement-1", "progress", "statement-2"], + ) + + def test_confirmed_count_zero_stays_a_single_silent_statement(self) -> None: + """A one-statement gate body must not add stamps it does not owe.""" + ledger = GateStatementRecordingPsqlShareLedger( + [{"backend": "postgres-psql", "confirmed_count": 0}] + ) + with ledger.operation_progress( + ledger.note_progress, + slice_seconds=0.05, + ): + ledger.confirm_accepted_block( + block_hash="ab" * 32, + active_tip_height=10, + ) + self.assertEqual(ledger.events, ["statement-1"]) + + def test_reactivate_pool_block_reports_between_its_two_statements(self) -> None: + """The reorg-walk twin of confirm_accepted_block's ordinal read.""" + ledger = GateStatementRecordingPsqlShareLedger( + [ + {"backend": "postgres-psql", "reactivated_count": 1}, + {"audit_publication_sequence": 4}, + ] + ) + with ledger.operation_progress( + ledger.note_progress, + slice_seconds=0.05, + ): + response = ledger.reactivate_pool_block( + block_hash="cd" * 32, + active_tip_height=10, + ) + self.assertEqual(response["audit_publication_sequence"], 4) + self.assertEqual( + ledger.events, + ["statement-1", "progress", "statement-2"], + ) + + def test_carry_forward_integrity_report_reports_between_its_reads(self) -> None: + """Two reads that must see one another cannot release the lock. + + The report and its audit head are consistent only because they run + under one held writer lock, so the report between them is the only + liveness a caller's monitor can get. + """ + ledger = GateStatementRecordingPsqlShareLedger( + [ + {"checked_active_rows": 0, "mismatch_count": 0}, + [], + ] + ) + with ledger.operation_progress( + ledger.note_progress, + slice_seconds=0.05, + ): + report = ledger.carry_forward_integrity_report() + self.assertEqual(report["audit_row_count"], 0) + self.assertEqual( + ledger.events, + ["statement-1", "progress", "statement-2"], + ) + def test_statement_timeout_refreshes_for_each_database_step(self) -> None: ledger = PsqlShareLedger.__new__(PsqlShareLedger) ledger._operation_timeout_local = threading.local() @@ -4496,6 +7336,49 @@ def connection(*, timeout_seconds: float | None = None) -> Any: self.assertRegex(executions[1], r"^SET LOCAL lock_timeout = '\d+ms'$") self.assertEqual(executions[2], "SELECT json_build_object('ok', true)") + def test_native_connection_setup_expiry_does_not_start_statement_timing( + self, + ) -> None: + """A budget exhausted while connecting never becomes server time.""" + + class OperationalError(Exception): + pass + + class FakePsycopg: + pass + + FakePsycopg.OperationalError = OperationalError # type: ignore[attr-defined] + + client = _NativePostgresClient.__new__(_NativePostgresClient) + client._psycopg = FakePsycopg + starts: list[str] = [] + + @contextlib.contextmanager + def connection(*, timeout_seconds: float | None = None) -> Any: + self.assertGreater(float(timeout_seconds), 0.0) + yield unittest.mock.Mock() + + client.connection = connection # type: ignore[method-assign] + + # Establish deadline, calculate the connection budget, then discover + # that connection setup consumed it before SET LOCAL or the query. + with unittest.mock.patch.object( + share_ledger_module.time, + "monotonic", + side_effect=[0.0, 0.1, 0.6], + ): + with self.assertRaisesRegex( + LedgerOperationTimeout, + "statement deadline expired", + ): + client.run_json( + "SELECT json_build_object('ok', true)", + timeout_seconds=0.5, + on_statement_start=lambda: starts.append("started"), + ) + + self.assertEqual(starts, []) + def test_native_server_deadline_raises_operation_timeout(self) -> None: class OperationalError(Exception): pass @@ -4588,6 +7471,39 @@ def test_forced_native_without_psycopg_raises(self) -> None: native_client_mode="1", ) + def test_auto_native_psycopg_import_fallback_warns(self) -> None: + stdout = io.StringIO() + with unittest.mock.patch.dict(sys.modules, {"psycopg": None}): + with contextlib.redirect_stdout(stdout): + ledger = CannedQueryPsqlShareLedger( + [acquired_lease_result()], + native_client_mode="auto", + ) + self.assertIsNone(ledger._native) + self.assertEqual(ledger.execution_backend, "psql-subprocess") + self.assertIn( + "psycopg import failed; falling back to the psql subprocess backend", + stdout.getvalue(), + ) + + def test_psql_run_sql_uses_single_transaction(self) -> None: + recorded: dict[str, object] = {} + + def fake_run(cmd, **kwargs): # type: ignore[no-untyped-def] + recorded["cmd"] = list(cmd) + return types.SimpleNamespace(returncode=0, stdout="{}\n", stderr="") + + ledger = CannedQueryPsqlShareLedger([acquired_lease_result()]) + with unittest.mock.patch.object( + share_ledger_module.subprocess, "run", side_effect=fake_run + ): + output = ledger._run_sql("SELECT '{}'::json;") + self.assertEqual(output, "{}\n") + cmd = recorded["cmd"] + self.assertIn("--single-transaction", cmd) + self.assertIn("--no-psqlrc", cmd) + self.assertIn("ON_ERROR_STOP=1", cmd) + def test_parse_single_json_value_contract(self) -> None: self.assertEqual(parse_single_json_value('{"a": 1}'), {"a": 1}) self.assertEqual(parse_single_json_value({"a": 1}), {"a": 1}) @@ -4697,44 +7613,703 @@ def fetchone(self) -> tuple[object]: def connection() -> Any: yield FakeConnection() - client.connection = connection # type: ignore[method-assign] + client.connection = connection # type: ignore[method-assign] + + result = client.run_json("SELECT count(*)", retry_safe=True) + + self.assertEqual(result, {"count": 11}) + self.assertEqual(executions, ["SELECT count(*)", "SELECT count(*)"]) + + def test_ctv_attempt_journal_uses_non_retrying_native_execution(self) -> None: + class FakeNative: + def __init__(self) -> None: + self.calls: list[tuple[str, bool]] = [] + + def run_json(self, sql: str, *, retry_safe: bool = False) -> Any: + self.calls.append((sql, retry_safe)) + raise RuntimeError("ambiguous connection loss") + + ledger = PsqlShareLedger.__new__(PsqlShareLedger) + ledger._lock = threading.Lock() + ledger._ctv_broadcast_attempt_detail_limit = 20 + ledger._ctv_broadcast_retry_backoff_seconds = 300 + ledger._writer_id = "writer-a" + ledger._writer_epoch = 1 + ledger._writer_session_token = "session-a" + ledger._lease_interval_sql = "make_interval(secs => 30)" + native = FakeNative() + ledger._native = native + + with self.assertRaisesRegex(RuntimeError, "ambiguous connection loss"): + ledger.record_ctv_fanout_broadcast_attempt( + fanout_txid="22" * 32, + attempt_status="submitted", + ) + + self.assertEqual(len(native.calls), 1) + sql, retry_safe = native.calls[0] + self.assertFalse(retry_safe) + self.assertIn("INSERT INTO qbit_ctv_fanout_broadcast_attempts", sql) + self.assertIn("broadcast_attempt_count = artifact.broadcast_attempt_count + 1", sql) + + +def make_session_guards(**overrides: Any) -> Any: + """Build a PostgresSessionGuards, resolving the class at call time. + + Resolved through the module object so a tree without the session guards + fails each guard test individually with a clear AttributeError instead + of breaking the whole module import. + """ + kwargs: dict[str, Any] = { + "idle_in_transaction_timeout_seconds": 15.0, + "tcp_keepalives_idle_seconds": 30, + "tcp_keepalives_interval_seconds": 10, + "tcp_keepalives_count": 3, + } + kwargs.update(overrides) + return share_ledger_module.PostgresSessionGuards(**kwargs) + + +class PostgresSessionGuardTests(unittest.TestCase): + def test_native_pool_connect_carries_session_guard_options(self) -> None: + connect_kwargs: list[dict[str, Any]] = [] + + class FakeConninfo: + @staticmethod + def conninfo_to_dict(conninfo: str) -> dict[str, Any]: + return {"dbname": "qbit"} + + class FakePsycopg: + conninfo = FakeConninfo + + @staticmethod + def connect(conninfo: str, **kwargs: Any) -> object: + connect_kwargs.append(kwargs) + return object() + + client = _NativePostgresClient.__new__(_NativePostgresClient) + client._psycopg = FakePsycopg + client._conninfo = "postgresql://example.invalid/qbit" + client._application_name = "qbit-prism-writer-test" + client._session_guards = make_session_guards( + idle_in_transaction_timeout_seconds=2.5, + tcp_keepalives_idle_seconds=31, + tcp_keepalives_interval_seconds=7, + tcp_keepalives_count=4, + ) + + client._connect() + + self.assertEqual(len(connect_kwargs), 1) + options = connect_kwargs[0]["options"] + self.assertIn("-c idle_in_transaction_session_timeout=2500ms", options) + self.assertIn("-c tcp_keepalives_idle=31", options) + self.assertIn("-c tcp_keepalives_interval=7", options) + self.assertIn("-c tcp_keepalives_count=4", options) + + def test_native_pool_connect_preserves_operator_conninfo_options(self) -> None: + connect_kwargs: list[dict[str, Any]] = [] + operator_options = "-c geqo=off -c idle_in_transaction_session_timeout=0" + + class FakeConninfo: + @staticmethod + def conninfo_to_dict(conninfo: str) -> dict[str, Any]: + return {"dbname": "qbit", "options": operator_options} + + class FakePsycopg: + conninfo = FakeConninfo + + @staticmethod + def connect(conninfo: str, **kwargs: Any) -> object: + connect_kwargs.append(kwargs) + return object() + + client = _NativePostgresClient.__new__(_NativePostgresClient) + client._psycopg = FakePsycopg + client._conninfo = ( + "postgresql://example.invalid/qbit?options=-c%20geqo%3Doff" + ) + client._application_name = None + guards = make_session_guards() + client._session_guards = guards + + client._connect() + + options = connect_kwargs[0]["options"] + # Both the operator's DSN-level options and the guards are present, + # and the guard fragment comes last so its -c duplicates win. + self.assertIn("-c geqo=off", options) + self.assertTrue(options.endswith(guards.options_fragment())) + self.assertLess( + options.index("geqo"), + options.index("tcp_keepalives_idle"), + ) + + def test_lease_guard_keeps_statement_timeout_and_gains_session_guards(self) -> None: + connect_calls: list[dict[str, Any]] = [] + + class FakeGuardConnection: + closed = False + + def close(self) -> None: + pass + + def fake_connect(conninfo: str, **kwargs: Any) -> FakeGuardConnection: + connect_calls.append(kwargs) + return FakeGuardConnection() + + fake_psycopg = types.ModuleType("psycopg") + fake_psycopg.connect = fake_connect # type: ignore[attr-defined] + fake_psycopg.conninfo = types.SimpleNamespace( # type: ignore[attr-defined] + conninfo_to_dict=lambda conninfo: {"dbname": "qbit"} + ) + + with unittest.mock.patch.dict(sys.modules, {"psycopg": fake_psycopg}): + _NativePostgresLeaseGuard( + "postgresql://example.invalid/qbit", + 1234, + session_guards=make_session_guards(), + ) + + self.assertEqual(len(connect_calls), 1) + options = connect_calls[0]["options"] + self.assertIn("-c statement_timeout=500", options) + self.assertIn("-c idle_in_transaction_session_timeout=15000ms", options) + self.assertIn("-c tcp_keepalives_idle=30", options) + self.assertIn("-c tcp_keepalives_interval=10", options) + self.assertIn("-c tcp_keepalives_count=3", options) + + def test_subprocess_backend_sets_session_guards_without_a_deadline(self) -> None: + ledger = PsqlShareLedger.__new__(PsqlShareLedger) + ledger._command = ["psql"] + ledger._native = None + ledger._operation_timeout_local = threading.local() + ledger._statement_timeout_local = threading.local() + ledger._session_guards = make_session_guards() + completed = unittest.mock.Mock(returncode=0, stdout="{}\n", stderr="") + + with unittest.mock.patch.dict( + "os.environ", + {"PGOPTIONS": "-c geqo=off"}, + ), unittest.mock.patch( + "lab.prism.share_ledger.subprocess.run", + return_value=completed, + ) as run: + self.assertIsNone(ledger._remaining_operation_timeout()) + self.assertEqual(ledger._run_sql("SELECT '{}'::json;"), "{}\n") + + kwargs = run.call_args.kwargs + pgoptions = kwargs["env"]["PGOPTIONS"] + self.assertIn("-c idle_in_transaction_session_timeout=15000ms", pgoptions) + self.assertIn("-c tcp_keepalives_idle=30", pgoptions) + self.assertIn("-c tcp_keepalives_interval=10", pgoptions) + self.assertIn("-c tcp_keepalives_count=3", pgoptions) + # Operator-supplied PGOPTIONS survive, ahead of the guard fragment. + self.assertTrue(pgoptions.startswith("-c geqo=off")) + # No armed deadline: no per-statement bounds, no subprocess timeout. + self.assertNotIn("statement_timeout", pgoptions) + self.assertNotIn("lock_timeout", pgoptions) + self.assertNotIn("timeout", kwargs) + + +class ReadOnlySessionEnforcementTests(unittest.TestCase): + """``read_only=True`` must be a promise PostgreSQL keeps, not a gate. + + The refusing writer gate only covers paths that take the gate. Any + statement issued outside it — the lease upsert, a one-shot psql release — + reaches SQL exactly as it would on a read-write ledger, and until #157 + pointed the public tier at a hot standby, nothing but the deployment + topology stopped it. These tests pin the connect-time GUC that makes the + server refuse instead, on every connection the instance can create. + """ + + READ_ONLY_GUC = "-c default_transaction_read_only=on" + + def test_read_only_ledger_builds_read_only_session_guards(self) -> None: + ledger = PsqlShareLedger( + psql_command="psql postgresql://example.invalid/qbit", + native_client_mode="0", + read_only=True, + ) + self.addCleanup(ledger.close) + + self.assertTrue(ledger._session_guards.read_only) + fragment = ledger._session_guards.options_fragment() + self.assertIn(self.READ_ONLY_GUC, fragment) + # The orphan-reaping guards #137 established are preserved, not + # displaced: one options carrier, both guarantees. + self.assertIn("-c idle_in_transaction_session_timeout=15000ms", fragment) + self.assertIn("-c tcp_keepalives_idle=30", fragment) + + def test_writable_ledger_keeps_a_writable_session(self) -> None: + # The coordinator's own ledger must be untouched by this: it commits + # every share, block and settlement the pool has. + ledger = FakeLeasePsqlShareLedger( + [acquired_lease()], + writer_id="writer-a", + writer_epoch=1, + ) + self.addCleanup(ledger.close) + + self.assertFalse(ledger._session_guards.read_only) + self.assertNotIn( + "default_transaction_read_only", + ledger._session_guards.options_fragment(), + ) + + def test_read_only_native_pool_connections_are_read_only(self) -> None: + connect_kwargs: list[dict[str, Any]] = [] + + class FakeConninfo: + @staticmethod + def conninfo_to_dict(conninfo: str) -> dict[str, Any]: + return {"dbname": "qbit"} + + class FakePsycopg: + conninfo = FakeConninfo + + @staticmethod + def connect(conninfo: str, **kwargs: Any) -> object: + connect_kwargs.append(kwargs) + return object() + + client = _NativePostgresClient.__new__(_NativePostgresClient) + client._psycopg = FakePsycopg + client._conninfo = "postgresql://example.invalid/qbit" + client._application_name = "qbit-prism-public-read" + client._session_guards = make_session_guards(read_only=True) + + client._connect() + + options = connect_kwargs[0]["options"] + self.assertIn(self.READ_ONLY_GUC, options) + self.assertIn("-c idle_in_transaction_session_timeout=15000ms", options) + + def test_read_only_native_options_preserve_operator_conninfo_options(self) -> None: + connect_kwargs: list[dict[str, Any]] = [] + operator_options = "-c geqo=off -c default_transaction_read_only=off" + + class FakeConninfo: + @staticmethod + def conninfo_to_dict(conninfo: str) -> dict[str, Any]: + return {"dbname": "qbit", "options": operator_options} + + class FakePsycopg: + conninfo = FakeConninfo + + @staticmethod + def connect(conninfo: str, **kwargs: Any) -> object: + connect_kwargs.append(kwargs) + return object() + + client = _NativePostgresClient.__new__(_NativePostgresClient) + client._psycopg = FakePsycopg + client._conninfo = "postgresql://example.invalid/qbit" + client._application_name = None + client._session_guards = make_session_guards(read_only=True) + + client._connect() + + options = connect_kwargs[0]["options"] + # The operator's own options survive, but libpq applies -c settings + # left to right with the last duplicate winning, so a DSN that tries + # to turn the session writable again cannot. + self.assertIn("-c geqo=off", options) + self.assertLess( + options.index("-c default_transaction_read_only=off"), + options.index(self.READ_ONLY_GUC), + ) + self.assertTrue(options.endswith(self.READ_ONLY_GUC)) + + def test_read_only_lease_guard_session_is_read_only(self) -> None: + connect_calls: list[dict[str, Any]] = [] + + class FakeGuardConnection: + closed = False + + def close(self) -> None: + pass + + fake_psycopg = types.ModuleType("psycopg") + fake_psycopg.connect = lambda conninfo, **kwargs: ( # type: ignore[attr-defined] + connect_calls.append(kwargs) or FakeGuardConnection() + ) + fake_psycopg.conninfo = types.SimpleNamespace( # type: ignore[attr-defined] + conninfo_to_dict=lambda conninfo: {"dbname": "qbit"} + ) + + with unittest.mock.patch.dict(sys.modules, {"psycopg": fake_psycopg}): + _NativePostgresLeaseGuard( + "postgresql://example.invalid/qbit", + 1234, + session_guards=make_session_guards(read_only=True), + ) + + options = connect_calls[0]["options"] + # A read-only ledger never builds a lease guard, but the guarantee is + # a property of the carrier rather than of which paths happen to use + # it, so the dedicated session carries it too. + self.assertIn(self.READ_ONLY_GUC, options) + self.assertIn("-c statement_timeout=500", options) + + def test_read_only_subprocess_backend_is_read_only(self) -> None: + ledger = PsqlShareLedger.__new__(PsqlShareLedger) + ledger._command = ["psql"] + ledger._native = None + ledger._operation_timeout_local = threading.local() + ledger._statement_timeout_local = threading.local() + ledger._session_guards = make_session_guards(read_only=True) + completed = unittest.mock.Mock(returncode=0, stdout="{}\n", stderr="") + + with unittest.mock.patch.dict( + "os.environ", + {"PGOPTIONS": "-c geqo=off"}, + ), unittest.mock.patch( + "lab.prism.share_ledger.subprocess.run", + return_value=completed, + ) as run: + ledger._run_sql("SELECT '{}'::json;") + + pgoptions = run.call_args.kwargs["env"]["PGOPTIONS"] + self.assertIn(self.READ_ONLY_GUC, pgoptions) + self.assertTrue(pgoptions.startswith("-c geqo=off")) + + def test_read_only_covers_a_write_method_that_never_takes_the_gate(self) -> None: + # release_writer_lease_fresh_connection issues an UPDATE through a + # one-shot psql connection, deliberately touching neither the writer + # gate nor the pool. It is the shape #163 is about: the gate cannot + # refuse it, so the session it opens must be the thing that does. + ledger = PsqlShareLedger( + psql_command="psql postgresql://example.invalid/qbit", + native_client_mode="0", + read_only=True, + ) + self.addCleanup(ledger.close) + completed = unittest.mock.Mock(returncode=0, stdout='{"released": 0}\n', stderr="") + + with unittest.mock.patch( + "lab.prism.share_ledger.subprocess.run", + return_value=completed, + ) as run: + ledger.release_writer_lease_fresh_connection() + + self.assertIn("UPDATE qbit_ledger_writer_lease", run.call_args.kwargs["input"]) + self.assertIn( + self.READ_ONLY_GUC, + run.call_args.kwargs["env"]["PGOPTIONS"], + ) + + def test_read_only_ledger_still_refuses_the_writer_gate(self) -> None: + # The server-side refusal is added to the in-process gate, not + # substituted for it. + ledger = PsqlShareLedger( + psql_command="psql postgresql://example.invalid/qbit", + native_client_mode="0", + read_only=True, + ) + self.addCleanup(ledger.close) + + with self.assertRaises(share_ledger_module.ReadOnlyLedgerError): + ledger.append(pending_share(0)) + + +class BoundedLeaseAcquisitionTests(unittest.TestCase): + class DeadlineRecordingLeaseLedger(FakeLeasePsqlShareLedger): + def __init__(self, lease_results: list[dict[str, object]], **kwargs: Any): + self.recorded_deadlines: list[float | None] = [] + super().__init__(lease_results, **kwargs) + + def _run_json(self, sql: str) -> Any: + self.recorded_deadlines.append(self._remaining_operation_timeout()) + return super()._run_json(sql) + + def test_startup_lease_upsert_runs_under_bounded_deadline(self) -> None: + ledger = self.DeadlineRecordingLeaseLedger( + [acquired_lease()], + writer_id="writer-a", + writer_epoch=1, + ) + + self.assertEqual(len(ledger.recorded_deadlines), 1) + # None here is precisely the issue #123 defect: the startup upsert + # queueing on the lease row lock with no deadline at all. + self.assertIsNotNone(ledger.recorded_deadlines[0]) + # DEFAULT_LEASE_ACQUIRE_LOCK_TIMEOUT_SECONDS bounds the statement. + self.assertGreater(ledger.recorded_deadlines[0], 0.0) + self.assertLessEqual(ledger.recorded_deadlines[0], 5.0) + + configured = self.DeadlineRecordingLeaseLedger( + [acquired_lease()], + writer_id="writer-a", + writer_epoch=1, + lease_acquire_lock_timeout_seconds=2.5, + ) + self.assertLessEqual(configured.recorded_deadlines[0], 2.5) + + def test_lock_blocked_acquisition_fails_visibly_after_bounded_retries(self) -> None: + attempts: list[str] = [] + sleeps: list[float] = [] + + class LockedLeaseLedger(PsqlShareLedger): + def _run_json(self, sql: str) -> Any: + attempts.append(sql) + raise LedgerOperationTimeout( + "canceling statement due to lock timeout" + ) + + stdout = io.StringIO() + with contextlib.redirect_stdout(stdout): + with self.assertRaises(RuntimeError) as caught: + LockedLeaseLedger( + psql_command="psql postgresql://example.invalid/qbit", + native_client_mode="0", + lease_retry_sleep=sleeps.append, + ) + + # Exactly DEFAULT_LEASE_ACQUIRE_ATTEMPTS bounded attempts, with a + # retry sleep between consecutive attempts, then a visible failure — + # never LedgerOperationTimeout leaking out, never an unbounded wait. + self.assertEqual(len(attempts), 5) + self.assertEqual(len(sleeps), 4) + self.assertIn("attempt 5/5", stdout.getvalue()) + # The diagnostic names the lock conflict as the likely cause without + # asserting it — an unreachable or overloaded server expires the same + # deadline — and quotes the underlying error on every line. + message = str(caught.exception) + self.assertIn("did not complete within its 5s deadline", message) + self.assertIn("commonly", message) + self.assertIn("lock-blocked", message) + self.assertIn("canceling statement due to lock timeout", message) + self.assertIn("lock-blocked", stdout.getvalue()) + self.assertIn("canceling statement due to lock timeout", stdout.getvalue()) + # The originating timeout is chained, not discarded: it is the only + # record of which of the several causes actually fired. + self.assertIsInstance(caught.exception.__cause__, LedgerOperationTimeout) + + def test_acquisition_recovers_once_the_orphaned_lock_is_reaped(self) -> None: + outcomes: list[Any] = [ + LedgerOperationTimeout("canceling statement due to lock timeout"), + LedgerOperationTimeout("canceling statement due to lock timeout"), + acquired_lease(), + ] + + class EventuallyReapedLeaseLedger(PsqlShareLedger): + def _run_json(self, sql: str) -> Any: + outcome = outcomes.pop(0) + if isinstance(outcome, Exception): + raise outcome + return outcome + + stdout = io.StringIO() + with contextlib.redirect_stdout(stdout): + ledger = EventuallyReapedLeaseLedger( + psql_command="psql postgresql://example.invalid/qbit", + native_client_mode="0", + lease_retry_sleep=lambda _seconds: None, + ) + + # The retry budget outlasting idle_in_transaction_session_timeout is + # what makes the ordinary orphaned-lock case self-heal: the server + # reaps the orphan mid-budget and a later attempt lands the lease. + self.assertEqual(outcomes, []) + self.assertIn("attempt 2/5", stdout.getvalue()) + ledger.close() + + @staticmethod + def _adoptable_predecessor_lease_results( + *, + old_session: str, + new_session: str, + updated_at: str, + ) -> list[dict[str, object]]: + """Two observations of a silent predecessor, then a won CAS. + + Mirrors test_postgres_startup_adopts_after_one_guard_acquisition_silence: + the row is already silent for a full interval, so the second + observation lands once the guard-acquisition edge has elapsed too and + _ensure_writer_lease proceeds to the adoption CAS. + """ + return [ + held_lease( + session=old_session, + updated_at=updated_at, + age_seconds=DEFAULT_WRITER_LEASE_ADOPTION_SILENCE_SECONDS, + ), + held_lease( + session=old_session, + updated_at=updated_at, + age_seconds=DEFAULT_WRITER_LEASE_ADOPTION_SILENCE_SECONDS * 2, + ), + acquired_lease(session=new_session) | {"adopted": True}, + ] + + def test_adoption_cas_runs_under_bounded_deadline(self) -> None: + """The adoption CAS row-locks the lease too, so it must be bounded. + + Reached through the real _ensure_writer_lease adoption path rather + than by calling the CAS helper directly: an unbounded + _try_adopt_writer_lease reintroduces the issue #123 startup hang on + exactly this branch, where the predecessor left the row locked. + """ + updated_at = "2026-06-26 19:49:22.233718+00" + old_session = f"{WRITER_LEASE_HEARTBEAT_SESSION_PREFIX}old-session" + new_session = f"{WRITER_LEASE_HEARTBEAT_SESSION_PREFIX}new-session" + clock = FakeMonotonicClock() + + with contextlib.redirect_stdout(io.StringIO()), unittest.mock.patch.object( + share_ledger_module.time, + "monotonic", + clock.monotonic, + ): + ledger = self.DeadlineRecordingLeaseLedger( + self._adoptable_predecessor_lease_results( + old_session=old_session, + new_session=new_session, + updated_at=updated_at, + ), + writer_id="writer-a", + writer_epoch=1, + writer_session_token=new_session, + lease_acquire_lock_timeout_seconds=2.5, + lease_retry_sleep=clock.sleep, + ) + + self.assertEqual(len(ledger.lease_queries), 3) + # The third query is the CAS, not another observation. + self.assertIn("observed_writer_session_token", ledger.lease_queries[2]) + adoption_deadline = ledger.recorded_deadlines[2] + # None here is the issue #123 defect on the adoption branch: the CAS + # queueing on the lease row lock with no deadline at all. + self.assertIsNotNone(adoption_deadline) + self.assertGreater(adoption_deadline, 0.0) + self.assertLessEqual(adoption_deadline, 2.5) + + def test_lock_blocked_adoption_fails_visibly_after_bounded_retries(self) -> None: + updated_at = "2026-06-26 19:49:22.233718+00" + old_session = f"{WRITER_LEASE_HEARTBEAT_SESSION_PREFIX}old-session" + new_session = f"{WRITER_LEASE_HEARTBEAT_SESSION_PREFIX}new-session" + clock = FakeMonotonicClock() + adoption_deadlines: list[float | None] = [] + + class BlockedAdoptionLeaseLedger(FakeLeasePsqlShareLedger): + def _run_json(self, sql: str) -> Any: + if "observed_writer_session_token" not in sql: + return super()._run_json(sql) + adoption_deadlines.append(self._remaining_operation_timeout()) + raise LedgerOperationTimeout( + "canceling statement due to lock timeout" + ) - result = client.run_json("SELECT count(*)", retry_safe=True) + stdout = io.StringIO() + with contextlib.redirect_stdout(stdout), unittest.mock.patch.object( + share_ledger_module.time, + "monotonic", + clock.monotonic, + ): + with self.assertRaises(RuntimeError) as caught: + BlockedAdoptionLeaseLedger( + self._adoptable_predecessor_lease_results( + old_session=old_session, + new_session=new_session, + updated_at=updated_at, + ), + writer_id="writer-a", + writer_epoch=1, + writer_session_token=new_session, + lease_retry_sleep=clock.sleep, + ) - self.assertEqual(result, {"count": 11}) - self.assertEqual(executions, ["SELECT count(*)", "SELECT count(*)"]) + # Every CAS attempt ran under the bound, and the budget stopped at + # DEFAULT_LEASE_ACQUIRE_ATTEMPTS instead of retrying forever. + self.assertEqual(len(adoption_deadlines), 5) + for deadline in adoption_deadlines: + self.assertIsNotNone(deadline) + self.assertLessEqual(deadline, 5.0) + message = str(caught.exception) + self.assertIn("writer lease adoption", message) + self.assertIn("did not complete within its 5s deadline", message) + self.assertIsInstance(caught.exception.__cause__, LedgerOperationTimeout) + self.assertIn("adoption: attempt 5/5", stdout.getvalue()) + + def test_non_timeout_lease_failure_propagates_without_retry(self) -> None: + """Only deadline expiries are retried; everything else propagates. + + Swallowing a broader exception class into the retry loop would spend + the whole budget on an error no retry can fix and then relabel it as + a possible lock conflict, hiding the real fault from the operator. + """ - def test_ctv_attempt_journal_uses_non_retrying_native_execution(self) -> None: - class FakeNative: - def __init__(self) -> None: - self.calls: list[tuple[str, bool]] = [] + class FakeIdleInTransactionSessionTimeout(Exception): + """Shaped like psycopg.errors.IdleInTransactionSessionTimeout. - def run_json(self, sql: str, *, retry_safe: bool = False) -> Any: - self.calls.append((sql, retry_safe)) - raise RuntimeError("ambiguous connection loss") + sqlstate 25P03 is class 25 (invalid transaction state), which + psycopg raises as an InternalError, not an OperationalError, so + _NativePostgresClient.run_json's handler never wraps it and it + reaches the lease helper raw. The stand-in keeps this case + meaningful on interpreters without psycopg installed. + """ - ledger = PsqlShareLedger.__new__(PsqlShareLedger) - ledger._lock = threading.Lock() - ledger._ctv_broadcast_attempt_detail_limit = 20 - ledger._ctv_broadcast_retry_backoff_seconds = 300 - ledger._writer_id = "writer-a" - ledger._writer_epoch = 1 - ledger._writer_session_token = "session-a" - ledger._lease_interval_sql = "make_interval(secs => 30)" - native = FakeNative() - ledger._native = native + sqlstate = "25P03" - with self.assertRaisesRegex(RuntimeError, "ambiguous connection loss"): - ledger.record_ctv_fanout_broadcast_attempt( - fanout_txid="22" * 32, - attempt_status="submitted", + errors: list[Exception] = [ + # What run_json raises for a wrapped, non-deadline OperationalError. + RuntimeError("postgres query failed: server closed the connection"), + FakeIdleInTransactionSessionTimeout( + "terminating connection due to idle-in-transaction timeout" + ), + ] + try: + import psycopg + except ImportError: + psycopg = None # type: ignore[assignment] + if psycopg is not None: + real_idle_timeout = psycopg.errors.IdleInTransactionSessionTimeout + self.assertFalse(issubclass(real_idle_timeout, psycopg.OperationalError)) + errors.append( + real_idle_timeout( + "terminating connection due to idle-in-transaction timeout" + ) ) - self.assertEqual(len(native.calls), 1) - sql, retry_safe = native.calls[0] - self.assertFalse(retry_safe) - self.assertIn("INSERT INTO qbit_ctv_fanout_broadcast_attempts", sql) - self.assertIn("broadcast_attempt_count = artifact.broadcast_attempt_count + 1", sql) + for error in errors: + with self.subTest(error=type(error).__name__): + attempts: list[str] = [] + sleeps: list[float] = [] + + class FailingLeaseLedger(PsqlShareLedger): + def _run_json(self, sql: str) -> Any: + attempts.append(sql) + raise error + + stdout = io.StringIO() + with contextlib.redirect_stdout(stdout): + with self.assertRaises(type(error)) as caught: + FailingLeaseLedger( + psql_command="psql postgresql://example.invalid/qbit", + native_client_mode="0", + lease_retry_sleep=sleeps.append, + ) + + # Unchanged, on the first attempt, with no retry and none of + # the deadline wording. + self.assertIs(caught.exception, error) + self.assertEqual(len(attempts), 1) + self.assertEqual(sleeps, []) + self.assertEqual(stdout.getvalue(), "") + + def test_init_rejects_disarmed_session_guard_and_lease_bounds(self) -> None: + cases = ( + {"lease_acquire_lock_timeout_seconds": 0}, + {"lease_acquire_lock_timeout_seconds": -1}, + {"lease_acquire_attempts": 0}, + {"postgres_idle_in_transaction_timeout_seconds": 0}, + {"postgres_tcp_keepalives_idle_seconds": 0}, + {"postgres_tcp_keepalives_interval_seconds": -1}, + {"postgres_tcp_keepalives_count": 0}, + ) + for overrides in cases: + with self.subTest(**overrides), self.assertRaises(ValueError): + FakeLeasePsqlShareLedger([acquired_lease()], **overrides) class AcceptedStatsCacheTests(unittest.TestCase): @@ -5150,5 +8725,945 @@ def test_cold_seed_still_runs_exact_aggregate_synchronously(self) -> None: self.assertIsNone(ledger._stats_background_refresh_thread) +# A stand-in psql for the subprocess backend's row-result path (#236). It is +# a real child process -- spawned, fed the statement on stdin, killed on a +# deadline and reaped -- so the tests below exercise the shipped spool, +# timeout and exit translation rather than a mock of subprocess. The mode +# argument selects its behaviour; the ledger's own flags follow it and are +# echoed back by the ``env`` mode so the invocation can be asserted on. +FAKE_PSQL_SCRIPT = """ +import json +import os +import sys +import time + +mode = sys.argv[1] +sys.stdin.read() + + +def row(index): + return json.dumps({ + "share_seq": index, + "share_id": f"share-{index}", + "miner_id": f"miner-{index % 3}", + "order_key": f"{index % 3:04d}", + "p2mr_program_hex": "11" * 32, + "share_difficulty": "4096", + "network_difficulty": "1048576", + "template_height": 800000 + index, + "job_id": f"job-{index}", + "job_issued_at_ms": 1000 + index, + "accepted_at_ms": 1001 + index, + "ntime": 1700000000 + index, + "credit_policy": None, + }) + + +if mode == "rows": + count = int(os.environ.get("FAKE_PSQL_ROWS", "0")) + for index in range(1, count + 1): + sys.stdout.write(row(index) + "\\n") + sys.stdout.write("\\n") +elif mode == "empty": + pass +elif mode == "fail": + sys.stderr.write('ERROR: 42P01: relation "missing" does not exist\\n') + sys.exit(3) +elif mode == "partial-then-fail": + for index in range(1, 4): + sys.stdout.write(row(index) + "\\n") + sys.stdout.flush() + sys.stderr.write("ERROR: XX000: server closed the connection unexpectedly\\n") + sys.exit(2) +elif mode == "deadline": + sys.stderr.write("ERROR: 57014: canceling statement due to statement timeout\\n") + sys.exit(3) +elif mode == "hang": + time.sleep(30) +elif mode == "malformed": + sys.stdout.write(row(1) + "\\n") + sys.stdout.write("{not json\\n") +elif mode == "env": + sys.stdout.write(json.dumps({ + "pgoptions": os.environ.get("PGOPTIONS", ""), + "pgconnect_timeout": os.environ.get("PGCONNECT_TIMEOUT", ""), + "argv": sys.argv[2:], + }) + "\\n") +else: + sys.stderr.write(f"unknown mode {mode}\\n") + sys.exit(4) +""" + + +def row_payload(index: int) -> dict[str, Any]: + """One accepted-share payload in the shape the per-row projection returns.""" + return { + "share_seq": index, + "share_id": f"share-{index}", + "miner_id": f"miner-{index % 3}", + "order_key": f"{index % 3:04d}", + "p2mr_program_hex": "11" * 32, + "share_difficulty": "4096", + "network_difficulty": "1048576", + "template_height": 800_000 + index, + "job_id": f"job-{index}", + "job_issued_at_ms": 1_000 + index, + "accepted_at_ms": 1_001 + index, + "ntime": 1_700_000_000 + index, + "credit_policy": None, + } + + +class PayoutWindowRowDecodingTests(unittest.TestCase): + """Issue #236: the payout-window reads decode one row at a time. + + The snapshot and delta used to return one ``json_agg`` value the size of + the whole window, decoded in one C call that never released the GIL. + Both production backends now project one JSON object per row and + convert the rows in bounded batches, rechecking the caller's deadline + and stamping liveness between batches. These tests pin that contract on + the psql subprocess backend (through a real child process), on the + native client (through a fake driver), on the ledger plumbing that + joins them, and on the supported subclass seams. + """ + + @classmethod + def setUpClass(cls) -> None: + cls._script_dir = tempfile.TemporaryDirectory() + cls.script_path = str(Path(cls._script_dir.name) / "fake_psql.py") + Path(cls.script_path).write_text(FAKE_PSQL_SCRIPT, encoding="utf-8") + + @classmethod + def tearDownClass(cls) -> None: + cls._script_dir.cleanup() + + # -- fixtures ----------------------------------------------------------- + + def psql_ledger( + self, + mode: str, + *, + batch_size: int | None = None, + session_guards: Any = None, + ) -> PsqlShareLedger: + ledger = PsqlShareLedger.__new__(PsqlShareLedger) + ledger._native = None + ledger._command = [sys.executable, self.script_path, mode] + ledger._session_guards = session_guards + ledger._lock = threading.Lock() + ledger._read_semaphore = threading.BoundedSemaphore(1) + if batch_size is not None: + ledger._json_row_batch_size = batch_size + return ledger + + @staticmethod + def arm_deadline(ledger: PsqlShareLedger, deadline: float) -> None: + deadline_local = threading.local() + deadline_local.deadline = deadline + ledger._operation_timeout_local = deadline_local + + # -- psql subprocess backend -------------------------------------------- + + def test_psql_snapshot_decodes_one_json_object_per_line(self) -> None: + ledger = self.psql_ledger("rows", batch_size=512) + + with unittest.mock.patch.dict(os.environ, {"FAKE_PSQL_ROWS": "1200"}): + records = ledger.snapshot_at_job_issue(1_005, window_weight=64) + + self.assertEqual(len(records), 1200) + self.assertEqual([type(record) for record in records[:2]], [AcceptedShareRecord] * 2) + self.assertEqual([record.share_seq for record in records[:3]], [1, 2, 3]) + self.assertEqual(records[-1].share_id, "share-1200") + self.assertEqual(records[0].share_difficulty, 4096) + self.assertIsNone(records[0].credit_policy) + stats = ledger.ledger_read_gate_stats()["payout_window_snapshot"] + self.assertEqual(stats["calls_total"], 1) + self.assertEqual(stats["execute_timeouts_total"], 0) + self.assertEqual(ledger._read_semaphore._value, 1) + + def test_psql_empty_window_is_an_empty_list(self) -> None: + """psql prints nothing for zero rows; that is a window, not an error.""" + ledger = self.psql_ledger("empty") + + self.assertEqual(ledger.snapshot_at_job_issue(1_005, window_weight=64), []) + self.assertEqual(ledger.snapshot_between_job_issues(1_005, 1_006), []) + + def test_psql_failure_after_partial_output_publishes_nothing(self) -> None: + ledger = self.psql_ledger("partial-then-fail", batch_size=1) + stamped: list[str] = [] + + with ( + ledger.operation_progress(lambda: stamped.append("x"), slice_seconds=1.0), + self.assertRaisesRegex( + RuntimeError, + r"psql command failed \(exit 2\): ERROR: XX000", + ), + ): + ledger.snapshot_at_job_issue(1_005, window_weight=64) + + # The spool was never read: exit status is validated first, so not + # a single row of the partial output was decoded or stamped. + self.assertEqual(stamped, []) + self.assertEqual(ledger._read_semaphore._value, 1) + stats = ledger.ledger_read_gate_stats()["payout_window_snapshot"] + self.assertEqual(stats["calls_total"], 1) + self.assertEqual(stats["execute_timeouts_total"], 0) + + def test_psql_generic_failure_keeps_its_stderr(self) -> None: + ledger = self.psql_ledger("fail") + + with self.assertRaisesRegex(RuntimeError, r"\(exit 3\): ERROR: 42P01"): + ledger.snapshot_between_job_issues(1_005, 1_006) + + def test_psql_deadline_cancellation_is_a_timeout(self) -> None: + ledger = self.psql_ledger("deadline") + + with ledger.operation_timeout(5.0): + with self.assertRaisesRegex(LedgerOperationTimeout, "psql operation exceeded"): + ledger.snapshot_at_job_issue(1_005, window_weight=64) + + stats = ledger.ledger_read_gate_stats()["payout_window_snapshot"] + self.assertEqual(stats["execute_timeouts_total"], 1) + self.assertEqual(ledger._read_semaphore._value, 1) + + def test_psql_hang_is_killed_and_reaped_at_the_deadline(self) -> None: + ledger = self.psql_ledger("hang") + spawned: list[subprocess.Popen[bytes]] = [] + real_popen = share_ledger_module.subprocess.Popen + + def spy(*args: Any, **kwargs: Any) -> subprocess.Popen[bytes]: + process = real_popen(*args, **kwargs) + spawned.append(process) + return process + + started = time.monotonic() + with unittest.mock.patch.object(share_ledger_module.subprocess, "Popen", spy): + with ledger.operation_timeout(0.5): + with self.assertRaisesRegex(LedgerOperationTimeout, "psql operation exceeded"): + ledger.snapshot_at_job_issue(1_005, window_weight=64) + elapsed = time.monotonic() - started + + self.assertLess(elapsed, 10.0) + self.assertEqual(len(spawned), 1) + # Reaped, and by our signal rather than its own exit. + self.assertIsNotNone(spawned[0].returncode) + self.assertLess(spawned[0].returncode, 0) + + def test_psql_malformed_line_is_an_error_not_a_partial_snapshot(self) -> None: + ledger = self.psql_ledger("malformed") + + with self.assertRaisesRegex(RuntimeError, "malformed JSON on output line 2"): + ledger.snapshot_at_job_issue(1_005, window_weight=64) + + def test_psql_row_invocation_carries_guards_flags_and_deadline(self) -> None: + """The row path and the single-value path share one invocation.""" + guards = make_session_guards(idle_in_transaction_timeout_seconds=2.5) + ledger = self.psql_ledger("env", session_guards=guards) + + with ledger.operation_timeout(2.0): + rows = ledger._run_retry_safe_read_json_rows("SELECT 1;") + + self.assertEqual(len(rows), 1) + echoed = rows[0] + self.assertIn("-c idle_in_transaction_session_timeout=2500ms", echoed["pgoptions"]) + self.assertRegex(echoed["pgoptions"], r"-c statement_timeout=\d+ms -c lock_timeout=\d+ms") + self.assertEqual(echoed["pgconnect_timeout"], "2") + for flag in ( + "--no-psqlrc", + "--single-transaction", + "--tuples-only", + "--no-align", + "--quiet", + "ON_ERROR_STOP=1", + ): + self.assertIn(flag, echoed["argv"]) + + def test_psql_decode_deadline_expires_between_batches(self) -> None: + """A budget that runs out mid-decode fails at the next batch boundary.""" + clock = FakeMonotonicClock(start=0.0) + ledger = self.psql_ledger("rows", batch_size=500) + ledger._monotonic = clock.monotonic + self.arm_deadline(ledger, deadline=10.0) + stamps: list[float] = [] + + def on_progress() -> None: + stamps.append(clock.now) + clock.sleep(6.0) + + with unittest.mock.patch.dict(os.environ, {"FAKE_PSQL_ROWS": "2000"}): + with ledger.operation_progress(on_progress, slice_seconds=1.0): + with self.assertRaisesRegex(LedgerOperationTimeout, "deadline expired"): + ledger.snapshot_at_job_issue(1_005, window_weight=64) + + # Batch 1 decoded at t=0 and stamped; batch 2 decoded at t=6 (still + # inside the 10s budget) and stamped; the check before batch 3 sees + # t=12 and fails. Nothing was returned for the 1000 rows decoded. + self.assertEqual(stamps, [0.0, 6.0]) + self.assertEqual(ledger._read_semaphore._value, 1) + stats = ledger.ledger_read_gate_stats()["payout_window_snapshot"] + self.assertEqual(stats["execute_timeouts_total"], 1) + + def test_psql_progress_is_stamped_between_batches(self) -> None: + ledger = self.psql_ledger("rows", batch_size=500) + stamped: list[str] = [] + + with unittest.mock.patch.dict(os.environ, {"FAKE_PSQL_ROWS": "2000"}): + with ledger.operation_progress(lambda: stamped.append("x"), slice_seconds=1.0): + records = ledger.snapshot_at_job_issue(1_005, window_weight=64) + + self.assertEqual(len(records), 2000) + self.assertEqual(len(stamped), 4) + + def test_psql_deadline_expiring_in_the_final_partial_batch_publishes_nothing(self) -> None: + """A short trailing batch is still deadline-checked before the result is returned.""" + clock = FakeMonotonicClock(start=0.0) + ledger = self.psql_ledger("rows", batch_size=500) + ledger._monotonic = clock.monotonic + self.arm_deadline(ledger, deadline=10.0) + stamps: list[float] = [] + + def on_progress() -> None: + stamps.append(clock.now) + clock.sleep(12.0) + + with unittest.mock.patch.dict(os.environ, {"FAKE_PSQL_ROWS": "700"}): + with ledger.operation_progress(on_progress, slice_seconds=1.0): + with self.assertRaisesRegex(LedgerOperationTimeout, "deadline expired"): + ledger.snapshot_at_job_issue(1_005, window_weight=64) + + # One full batch stamped at t=0; the 200-row tail converted at t=12 + # and the final check refused to publish it. + self.assertEqual(stamps, [0.0]) + self.assertEqual(ledger._read_semaphore._value, 1) + stats = ledger.ledger_read_gate_stats()["payout_window_snapshot"] + self.assertEqual(stats["execute_timeouts_total"], 1) + + def test_psql_deadline_expiring_before_decoding_publishes_nothing(self) -> None: + """A result smaller than one batch is deadline-checked before any row is decoded. + + The clock advances 3s on every read, so the budget that was still + open when psql was dispatched (a 13s deadline read at t=9) has run + out by the time the child has exited and decoding would start. + Whichever of the two deadlines the pre-decode check trips first, + the read must fail as a timeout: the fixture's second line is not + JSON, so a missing pre-check would surface as a parse error instead. + """ + ledger = self.psql_ledger("malformed", batch_size=512) + now = 0.0 + + def stepping() -> float: + nonlocal now + value = now + now += 3.0 + return value + + ledger._monotonic = stepping + self.arm_deadline(ledger, deadline=13.0) + + with self.assertRaisesRegex(LedgerOperationTimeout, "deadline expired"): + ledger.snapshot_at_job_issue(1_005, window_weight=64) + + self.assertEqual(ledger._read_semaphore._value, 1) + + def test_legacy_run_sql_seam_checks_the_deadline_before_decoding(self) -> None: + clock = FakeMonotonicClock(start=0.0) + + class SlowTextLedger(PsqlShareLedger): + def __init__(self) -> None: + self._native = None + self._lock = threading.Lock() + self._read_semaphore = threading.BoundedSemaphore(1) + self._monotonic = clock.monotonic + + def _run_sql(self, sql: str) -> str: + clock.sleep(11.0) + # Not a share row: converting it would raise KeyError. + return json.dumps({"share_seq": 1}) + + ledger = SlowTextLedger() + self.arm_deadline(ledger, deadline=10.0) + + with self.assertRaisesRegex(LedgerOperationTimeout, "statement deadline expired"): + ledger.snapshot_between_job_issues(1_005, 1_006) + + def test_legacy_run_json_seam_checks_the_deadline_around_the_final_batch(self) -> None: + clock = FakeMonotonicClock(start=0.0) + stamps: list[float] = [] + + class LegacyJsonLedger(PsqlShareLedger): + def __init__(self, rows: list[Any], *, slow: bool) -> None: + self._native = None + self._lock = threading.Lock() + self._read_semaphore = threading.BoundedSemaphore(1) + self._json_row_batch_size = 2 + self._monotonic = clock.monotonic + self.rows = rows + self.slow = slow + + def _run_json(self, sql: str) -> Any: + if self.slow: + clock.sleep(11.0) + return self.rows + + def on_progress() -> None: + stamps.append(clock.now) + clock.sleep(12.0) + + # Final partial batch: two rows convert and stamp at t=0, the third + # converts after the stamp moved the clock to t=12, and the final + # check refuses to publish. + clock.now = 0.0 + ledger = LegacyJsonLedger([row_payload(1), row_payload(2), row_payload(3)], slow=False) + self.arm_deadline(ledger, deadline=10.0) + with ledger.operation_progress(on_progress, slice_seconds=1.0): + with self.assertRaisesRegex(LedgerOperationTimeout, "deadline expired"): + ledger.snapshot_at_job_issue(1_005, window_weight=64) + self.assertEqual(stamps, [0.0]) + + # Before decoding: the seam answered after the budget, with a row + # that would raise KeyError if it were converted. + clock.now = 0.0 + ledger = LegacyJsonLedger([{"share_seq": 1}], slow=True) + self.arm_deadline(ledger, deadline=10.0) + with self.assertRaisesRegex(LedgerOperationTimeout, "statement deadline expired"): + ledger.snapshot_at_job_issue(1_005, window_weight=64) + self.assertEqual(ledger._read_semaphore._value, 1) + + def test_native_rows_deadline_expiring_in_the_final_partial_batch(self) -> None: + clock = FakeMonotonicClock(start=0.0) + rows = [row_payload(index) for index in range(1, 6)] + client, events, cursors = self.native_client(results=[{"rows": rows}], clock=clock) + + with self.assertRaisesRegex(LedgerOperationTimeout, "expired while decoding rows"): + client.run_json_rows( + "SELECT rows", + retry_safe=True, + timeout_seconds=9.0, + batch_size=2, + on_batch=lambda: clock.sleep(4.0), + ) + + # Checks at t=0, t=4 and t=8 admit three fetches (2, 2 and the + # trailing 1); the check before the terminating fetch, at t=12, + # refuses to publish the converted rows. + self.assertEqual(events.count("fetch:2"), 2) + self.assertEqual(events.count("fetch:1"), 1) + self.assertNotIn("fetch:0", events) + self.assertTrue(cursors[0].closed) + + def test_native_rows_deadline_expiring_during_execute_decodes_nothing(self) -> None: + clock = FakeMonotonicClock(start=0.0) + events: list[str] = [] + + class OperationalError(Exception): + pass + + class FakePsycopg: + pass + + FakePsycopg.OperationalError = OperationalError # type: ignore[attr-defined] + cursor = self.FakeCursor([], events) + + class SlowConnection: + def execute(self, sql: str) -> Any: + events.append(f"execute:{sql[:24]}") + if not sql.startswith("SET LOCAL"): + clock.sleep(6.0) + return cursor + return None + + @contextlib.contextmanager + def transaction(self) -> Any: + yield + + client = _NativePostgresClient.__new__(_NativePostgresClient) + client._psycopg = FakePsycopg + client._monotonic = clock.monotonic + + @contextlib.contextmanager + def connection(*, timeout_seconds: float | None = None) -> Any: + yield SlowConnection() + + client.connection = connection # type: ignore[method-assign] + + with self.assertRaisesRegex(LedgerOperationTimeout, "expired while decoding rows"): + client.run_json_rows("SELECT rows", timeout_seconds=5.0, batch_size=2) + + # An empty result is not published either: no fetch happened. + self.assertNotIn("fetch:0", events) + self.assertTrue(cursor.closed) + + # -- supported subclass seams ----------------------------------------------- + + def test_legacy_run_json_override_answers_with_the_row_list(self) -> None: + """A ``_run_json`` fake keeps working: it returns what json_agg decoded to.""" + stamped: list[str] = [] + + class LegacyJsonLedger(PsqlShareLedger): + def __init__(self) -> None: + self._native = None + self._lock = threading.Lock() + self._read_semaphore = threading.BoundedSemaphore(1) + self._json_row_batch_size = 2 + self.queries: list[str] = [] + + def _run_json(self, sql: str) -> Any: + self.queries.append(sql) + return [row_payload(1), row_payload(2), row_payload(3)] + + ledger = LegacyJsonLedger() + with ledger.operation_progress(lambda: stamped.append("x"), slice_seconds=1.0): + records = ledger.snapshot_at_job_issue(1_005, window_weight=64) + + self.assertEqual([record.share_seq for record in records], [1, 2, 3]) + self.assertEqual(len(ledger.queries), 1) + self.assertNotIn("json_agg", ledger.queries[0]) + # One boundary between the two batches of two and one. + self.assertEqual(stamped, ["x"]) + + def test_legacy_run_json_override_returning_a_scalar_is_rejected(self) -> None: + class ScalarLedger(PsqlShareLedger): + def __init__(self) -> None: + self._native = None + self._lock = threading.Lock() + self._read_semaphore = threading.BoundedSemaphore(1) + + def _run_json(self, sql: str) -> Any: + return {"count": 3} + + with self.assertRaisesRegex(RuntimeError, "non-list value: dict"): + ScalarLedger().snapshot_at_job_issue(1_005, window_weight=64) + + def test_legacy_run_sql_override_is_parsed_line_by_line(self) -> None: + """The A1 gate's text seam: one JSON value per psql output line.""" + calls: list[str] = [] + + class LegacyRunSqlLedger(PsqlShareLedger): + def __init__(self) -> None: + self._native = None + self._lock = threading.Lock() + self._read_semaphore = threading.BoundedSemaphore(1) + + def _run_sql(self, sql: str) -> str: + calls.append(sql) + return "\n".join( + [json.dumps(row_payload(1)), "", json.dumps(row_payload(2)), ""] + ) + + records = LegacyRunSqlLedger().snapshot_between_job_issues(1_005, 1_006) + + self.assertEqual([record.share_seq for record in records], [1, 2]) + self.assertEqual(len(calls), 1) + self.assertIn("UNION ALL", calls[0]) + + # -- SQL shape ----------------------------------------------------------------- + + def test_payout_window_reads_project_one_json_object_per_row(self) -> None: + """Neither production statement builds a whole-window aggregate.""" + ledger = QueryCapturePsqlShareLedger() + + ledger.snapshot_at_job_issue(1_000, window_weight=8) + ledger.snapshot_at_job_issue(1_000) + ledger.snapshot_between_job_issues(1_000, 2_000) + + self.assertEqual(len(ledger.queries), 3) + for sql in ledger.queries: + self.assertNotIn("json_agg", sql) + self.assertIn("json_build_object(", sql) + self.assertTrue(sql.rstrip().endswith("FROM rows\nORDER BY share_seq ASC;"), sql[-120:]) + bounded, unbounded, delta = ledger.queries + self.assertIn("WITH RECURSIVE pages AS", bounded) + self.assertIn("cumulative_difficulty - share_difficulty < 8::numeric", bounded) + self.assertNotIn("RECURSIVE", unbounded) + self.assertIn("UNION ALL", delta) + + # -- native client ----------------------------------------------------------- + + class FakeCursor: + def __init__(self, rows: list[Any], events: list[str], fail_at_batch: int | None = None, + error: Exception | None = None) -> None: + self._rows = rows + self._pos = 0 + self.events = events + self.fail_at_batch = fail_at_batch + self.error = error + self.batches = 0 + self.closed = False + + def fetchmany(self, size: int) -> list[tuple[Any]]: + self.batches += 1 + if self.fail_at_batch is not None and self.batches == self.fail_at_batch: + assert self.error is not None + raise self.error + batch = self._rows[self._pos : self._pos + size] + self._pos += size + self.events.append(f"fetch:{len(batch)}") + return [(row,) for row in batch] + + def close(self) -> None: + self.closed = True + self.events.append("close") + + def native_client( + self, + *, + results: list[Any], + clock: FakeMonotonicClock | None = None, + ) -> tuple[_NativePostgresClient, list[str], list[Any]]: + """A client over a fake driver; ``results`` are per-execute outcomes. + + The client's ``outcomes`` attribute is the live per-execute queue, so + a test can seed it with instances of the client's own + ``OperationalError`` after construction. + """ + events: list[str] = [] + cursors: list[Any] = [] + outcomes = list(results) + + class OperationalError(Exception): + def __init__(self, message: str, sqlstate: str | None = None) -> None: + super().__init__(message) + self.sqlstate = sqlstate + + class FakePsycopg: + pass + + FakePsycopg.OperationalError = OperationalError # type: ignore[attr-defined] + + tests = self + + class FakeConnection: + def execute(self, sql: str) -> Any: + events.append(f"execute:{sql[:24]}") + if sql.startswith("SET LOCAL"): + return None + outcome = outcomes.pop(0) + if isinstance(outcome, Exception): + raise outcome + cursor = tests.FakeCursor( + outcome["rows"], + events, + fail_at_batch=outcome.get("fail_at_batch"), + error=outcome.get("error"), + ) + cursors.append(cursor) + return cursor + + @contextlib.contextmanager + def transaction(self) -> Any: + events.append("begin") + try: + yield + finally: + events.append("commit") + + client = _NativePostgresClient.__new__(_NativePostgresClient) + client._psycopg = FakePsycopg + if clock is not None: + client._monotonic = clock.monotonic + + @contextlib.contextmanager + def connection(*, timeout_seconds: float | None = None) -> Any: + events.append(f"borrow:{timeout_seconds is not None}") + try: + yield FakeConnection() + except BaseException: + events.append("discard") + raise + else: + events.append("return") + + client.connection = connection # type: ignore[method-assign] + client.OperationalError = OperationalError # type: ignore[attr-defined] + client.outcomes = outcomes # type: ignore[attr-defined] + return client, events, cursors + + def test_native_rows_are_fetched_in_batches_after_the_deadline_commit(self) -> None: + rows = [row_payload(index) for index in range(1, 1201)] + client, events, cursors = self.native_client(results=[{"rows": rows}]) + batches: list[int] = [] + + records = client.run_json_rows( + "SELECT json_build_object(...)", + retry_safe=True, + timeout_seconds=5.0, + row_converter=PsqlShareLedger._record_from_json, + batch_size=512, + on_batch=lambda: batches.append(len(events)), + ) + + self.assertEqual([record.share_seq for record in records[:2]], [1, 2]) + self.assertEqual(len(records), 1200) + self.assertEqual(type(records[0]), AcceptedShareRecord) + # One SELECT inside the SET LOCAL transaction, committed before the + # first fetch; then three bounded fetches, the empty terminator, and + # the cursor closed before the connection goes back to the pool. + self.assertEqual( + events, + [ + "borrow:True", + "begin", + "execute:SET LOCAL statement_time", + "execute:SET LOCAL lock_timeout =", + "execute:SELECT json_build_object", + "commit", + "fetch:512", + "fetch:512", + "fetch:176", + "fetch:0", + "close", + "return", + ], + ) + self.assertEqual(len(batches), 3) + self.assertTrue(cursors[0].closed) + + def test_native_rows_without_a_deadline_run_in_autocommit(self) -> None: + client, events, _cursors = self.native_client( + results=[{"rows": [row_payload(1)]}] + ) + + rows = client.run_json_rows("SELECT 1", batch_size=4) + + self.assertEqual(rows, [row_payload(1)]) + self.assertEqual( + events, + ["borrow:False", "execute:SELECT 1", "fetch:1", "fetch:0", "close", "return"], + ) + + def test_native_retry_safe_rows_restart_the_whole_statement(self) -> None: + """A connection loss mid-fetch discards the partial rows and re-runs.""" + rows = [row_payload(index) for index in range(1, 7)] + client, events, cursors = self.native_client( + results=[ + { + "rows": rows, + "fail_at_batch": 2, + "error": None, + }, + {"rows": rows}, + ] + ) + # The first cursor's second fetch raises the driver's OperationalError. + first_error = client.OperationalError("SSL connection has been closed unexpectedly") + cursors_seen: list[Any] = [] + original_fetchmany = self.FakeCursor.fetchmany + + def fetchmany(cursor: Any, size: int) -> Any: + if cursor not in cursors_seen: + cursors_seen.append(cursor) + if len(cursors_seen) == 1 and cursor.batches == 1: + cursor.batches += 1 + raise first_error + return original_fetchmany(cursor, size) + + with unittest.mock.patch.object(self.FakeCursor, "fetchmany", fetchmany): + records = client.run_json_rows( + "SELECT rows", + retry_safe=True, + row_converter=PsqlShareLedger._record_from_json, + batch_size=2, + ) + + self.assertEqual([record.share_seq for record in records], [1, 2, 3, 4, 5, 6]) + self.assertEqual(events.count("execute:SELECT rows"), 2) + # First attempt: borrowed, one batch fetched, the failure discarded + # the connection and closed the cursor; second attempt complete. + self.assertEqual(events.index("discard"), events.index("close") + 1) + self.assertTrue(all(cursor.closed for cursor in cursors)) + + def test_native_rows_without_retry_fail_after_the_first_attempt(self) -> None: + client, events, _cursors = self.native_client(results=[]) + client.outcomes[:] = [client.OperationalError("connection reset"), {"rows": []}] + + with self.assertRaisesRegex(RuntimeError, "postgres query failed: connection reset"): + client.run_json_rows("SELECT rows", retry_safe=False) + + self.assertEqual(events.count("execute:SELECT rows"), 1) + + def test_native_rows_deadline_expires_between_batches(self) -> None: + clock = FakeMonotonicClock(start=0.0) + rows = [row_payload(index) for index in range(1, 7)] + client, events, cursors = self.native_client(results=[{"rows": rows}], clock=clock) + + def on_batch() -> None: + clock.sleep(4.0) + + with self.assertRaisesRegex(LedgerOperationTimeout, "expired while decoding rows"): + client.run_json_rows( + "SELECT rows", + retry_safe=True, + timeout_seconds=5.0, + batch_size=2, + on_batch=on_batch, + ) + + # Two batches fit inside the budget (checks at t=0 and t=4); the + # check before the third, at t=8, fails. No retry: the deadline is + # not an OperationalError, and the cursor is closed on the way out. + self.assertEqual(events.count("fetch:2"), 2) + self.assertEqual(events.count("execute:SELECT rows"), 1) + self.assertTrue(cursors[0].closed) + self.assertIn("discard", events) + + def test_native_rows_deadline_cancellation_is_not_retried(self) -> None: + client, events, _cursors = self.native_client(results=[]) + client.outcomes[:] = [ + client.OperationalError("canceling statement due to statement timeout", "57014"), + {"rows": []}, + ] + + with self.assertRaisesRegex(LedgerOperationTimeout, "postgres operation exceeded 2s"): + client.run_json_rows("SELECT rows", retry_safe=True, timeout_seconds=2.0) + + self.assertEqual(events.count("execute:SELECT rows"), 1) + + def test_native_null_row_is_rejected(self) -> None: + client, _events, _cursors = self.native_client(results=[{"rows": [None]}]) + + with self.assertRaisesRegex(RuntimeError, "returned no JSON"): + client.run_json_rows("SELECT NULL") + + def test_native_text_rows_are_decoded(self) -> None: + client, _events, _cursors = self.native_client( + results=[{"rows": [json.dumps(row_payload(7))]}] + ) + + self.assertEqual(client.run_json_rows("SELECT ...::text"), [row_payload(7)]) + + # -- ledger plumbing over the native port ------------------------------- + + def test_ledger_hands_the_native_port_the_full_rows_contract(self) -> None: + calls: list[dict[str, Any]] = [] + + class FakeNative: + def run_json_rows(self, sql: str, **kwargs: Any) -> list[Any]: + calls.append({"sql": sql, **kwargs}) + if kwargs.get("on_statement_start") is not None: + kwargs["on_statement_start"]() + converter = kwargs["row_converter"] + return [converter(row_payload(1)), converter(row_payload(2))] + + def close(self) -> None: + return None + + ledger = PsqlShareLedger.__new__(PsqlShareLedger) + ledger._native = FakeNative() + ledger._lock = threading.Lock() + ledger._read_semaphore = threading.BoundedSemaphore(2) + + with ledger.operation_timeout(3.0): + records = ledger.snapshot_at_job_issue(1_005, window_weight=64) + + self.assertEqual([record.share_seq for record in records], [1, 2]) + self.assertEqual(len(calls), 1) + call = calls[0] + self.assertTrue(call["retry_safe"]) + self.assertEqual(call["batch_size"], PAYOUT_WINDOW_ROW_BATCH_SIZE) + self.assertEqual(call["on_batch"], ledger._note_json_row_batch) + self.assertEqual(call["row_converter"], ledger._record_from_json) + self.assertGreater(call["timeout_seconds"], 0.0) + self.assertLessEqual(call["timeout_seconds"], 3.0) + self.assertNotIn("json_agg", call["sql"]) + stats = ledger.ledger_read_gate_stats()["payout_window_snapshot"] + self.assertEqual(stats["calls_total"], 1) + self.assertGreaterEqual(stats["execute_seconds_total"], 0.0) + self.assertEqual(ledger._read_semaphore._value, 2) + + def test_ledger_batch_hook_enforces_the_operation_deadline(self) -> None: + """The ledger's own absolute deadline is rechecked between batches.""" + clock = FakeMonotonicClock(start=0.0) + stamped: list[float] = [] + + class FakeNative: + def run_json_rows(self, sql: str, **kwargs: Any) -> list[Any]: + kwargs["on_statement_start"]() + converter = kwargs["row_converter"] + results = [] + for index in range(1, 4): + results.append(converter(row_payload(index))) + clock.sleep(4.0) + kwargs["on_batch"]() + return results + + def close(self) -> None: + return None + + ledger = PsqlShareLedger.__new__(PsqlShareLedger) + ledger._native = FakeNative() + ledger._monotonic = clock.monotonic + ledger._lock = threading.Lock() + ledger._read_semaphore = threading.BoundedSemaphore(1) + self.arm_deadline(ledger, deadline=10.0) + + with ledger.operation_progress(lambda: stamped.append(clock.now), slice_seconds=1.0): + with self.assertRaisesRegex(LedgerOperationTimeout, "operation deadline expired"): + ledger.snapshot_between_job_issues(1_005, 1_006) + + # Stamped at t=4 and t=8 (both inside the budget); the hook at t=12 + # raised before stamping, so the decode never completed late. + self.assertEqual(stamped, [4.0, 8.0]) + self.assertEqual(ledger._read_semaphore._value, 1) + stats = ledger.ledger_read_gate_stats()["payout_window_delta"] + self.assertEqual(stats["execute_timeouts_total"], 1) + + def test_native_client_satisfies_the_row_result_port(self) -> None: + client = _NativePostgresClient.__new__(_NativePostgresClient) + self.assertIsInstance(client, share_ledger_module.LedgerSqlPort) + self.assertTrue(callable(getattr(client, "run_json_rows", None))) + + +class BlockCandidateIdentityDigestTests(unittest.TestCase): + """The candidate identity digest streams its share window (#236).""" + + def _candidate(self, share_count: int, accepted_at_ms: int) -> dict[str, Any]: + shares = [ + { + "share_seq": seq, + "share_id": f"miner-\u00e9.rig{seq}:\U0001F600,}}{{", + "miner_id": f"miner-{seq % 7}", + "order_key": f"{seq % 7:02d}:miner-{seq % 7}", + "p2mr_program_hex": "ab" * 32, + "share_difficulty": (1 << 130) + seq, + "network_difficulty": 5, + "template_height": 9, + "job_id": f"job-{seq}", + "job_issued_at_ms": 1_000 + seq, + "accepted_at_ms": 1_100 + seq, + "ntime": 1_700_000_000 + seq, + } + for seq in range(1, share_count + 1) + ] + return { + "schema": "qbit.prism.block-candidate-intent.v1", + "block_hash_hex": "ab" * 32, + "pending_share": {"share_id": "s", "accepted_at_ms": accepted_at_ms}, + "shares_json": shares, + "prior_balances": [["m", "o", "p", 5]], + "found_block": {"block_height": 9, "z": None}, + } + + def test_streamed_identity_digest_matches_whole_document_digest(self) -> None: + for count in (0, 1, 1_300): + with self.subTest(count=count): + candidate = self._candidate(count, 5) + self.assertEqual( + share_ledger_module.block_candidate_identity_sha256(candidate), + share_ledger_module.sha256_json_hex( + share_ledger_module.block_candidate_identity(candidate) + ), + ) + # The acknowledgment stamp stays neutralized: two resubmissions of + # the same solved block hash identically. + self.assertEqual( + share_ledger_module.block_candidate_identity_sha256(self._candidate(9, 5)), + share_ledger_module.block_candidate_identity_sha256(self._candidate(9, 6)), + ) + self.assertNotEqual( + share_ledger_module.block_candidate_identity_sha256(self._candidate(9, 5)), + share_ledger_module.block_candidate_identity_sha256(self._candidate(8, 5)), + ) + + if __name__ == "__main__": unittest.main() diff --git a/tests/test_prism_share_submission.py b/tests/test_prism_share_submission.py new file mode 100644 index 000000000..755143714 --- /dev/null +++ b/tests/test_prism_share_submission.py @@ -0,0 +1,374 @@ +#!/usr/bin/env python3 +"""Direct tests for pure PRISM share-submission decisions.""" + +from __future__ import annotations + +from dataclasses import FrozenInstanceError +from types import SimpleNamespace +import threading +import unittest +from unittest.mock import patch + +from lab.prism.job_delivery import PRISM_CREDIT_POLICY_STALE_GRACE +from lab.prism.prism_coordinator import PrismCoordinator +from lab.prism import prism_coordinator +from lab.prism.share_submission import ( + PRISM_REJECTION_INVALID_EXTRANONCE, + PRISM_REJECTION_LOW_DIFFICULTY, + PRISM_REJECTION_MALFORMED_SUBMIT, + PRISM_REJECTION_STALE_JOB, + PRISM_REJECTION_UNAUTHORIZED_WORKER, + PRISM_REJECTION_UNKNOWN_JOB, + PRISM_SHARE_ACK_RESULTS, + RecentShareIndex, + SubmitContextInput, + SubmitRejected, + classify_submit_context, + classify_submit_work, + parse_submit_request, + validate_submit_request, +) + + +def context(*, parent: str = "tip", username: str = "miner-a") -> object: + return SimpleNamespace( + template={"previousblockhash": parent}, + worker=SimpleNamespace(username=username), + ) + + +class SubmitRequestTests(unittest.TestCase): + def test_parse_returns_immutable_wire_values(self) -> None: + request = parse_submit_request( + ["miner-a", "job-1", "00" * 8, "01020304", "05060708", "1fffe000"] + ) + + self.assertEqual(request.job_id, "job-1") + self.assertEqual(request.version_bits_hex, "1fffe000") + with self.assertRaises(FrozenInstanceError): + request.job_id = "job-2" # type: ignore[misc] + + def test_parse_rejects_incomplete_params(self) -> None: + with self.assertRaises(SubmitRejected) as raised: + parse_submit_request(["miner-a", "job-1"]) + + self.assertEqual(raised.exception.reason, PRISM_REJECTION_MALFORMED_SUBMIT) + + def test_validation_preserves_protocol_order(self) -> None: + request = parse_submit_request( + ["miner-b", "job-1", "00", "01020304", "05060708"] + ) + with self.assertRaises(SubmitRejected) as unauthorized: + validate_submit_request( + request, + authorized_username="miner-a", + pool_open=True, + extranonce2_size=8, + ) + self.assertEqual( + unauthorized.exception.reason, + PRISM_REJECTION_UNAUTHORIZED_WORKER, + ) + + request = parse_submit_request( + ["miner-a", "job-1", "00", "01020304", "05060708"] + ) + with self.assertRaises(SubmitRejected) as invalid_width: + validate_submit_request( + request, + authorized_username="miner-a", + pool_open=True, + extranonce2_size=8, + ) + self.assertEqual( + invalid_width.exception.reason, + PRISM_REJECTION_INVALID_EXTRANONCE, + ) + + +class RecentShareIndexTests(unittest.TestCase): + def test_coordinator_initialization_is_single_flight(self) -> None: + server = PrismCoordinator.__new__(PrismCoordinator) + server.extranonce2_size = 8 + seed = {("miner-a", "header-a")} + server.recent_share_keys = seed + constructor_started = threading.Event() + release_constructor = threading.Event() + constructor_calls = 0 + results: list[object] = [] + errors: list[BaseException] = [] + real_constructor = prism_coordinator.ShareSubmissionService + + def delayed_constructor(*args: object, **kwargs: object) -> object: + nonlocal constructor_calls + constructor_calls += 1 + constructor_started.set() + release_constructor.wait(2) + return real_constructor(*args, **kwargs) # type: ignore[arg-type] + + def initialize() -> None: + try: + results.append(server._ensure_share_submission_service()) + except BaseException as exc: # noqa: BLE001 - surfaced below + errors.append(exc) + + with patch.object( + prism_coordinator, + "ShareSubmissionService", + side_effect=delayed_constructor, + ): + first = threading.Thread(target=initialize) + second = threading.Thread(target=initialize) + first.start() + self.assertTrue(constructor_started.wait(1)) + second.start() + release_constructor.set() + first.join(2) + second.join(2) + + self.assertFalse(first.is_alive()) + self.assertFalse(second.is_alive()) + self.assertEqual(errors, []) + self.assertEqual(constructor_calls, 1) + self.assertEqual(len(results), 2) + self.assertIs(results[0], results[1]) + self.assertEqual(results[0].recent_shares.snapshot(), tuple(seed)) + + def test_capacity_evicts_only_the_oldest_insertion(self) -> None: + index = RecentShareIndex(capacity=2) + first = ("miner", "header-a") + second = ("miner", "header-b") + third = ("miner", "header-c") + + self.assertTrue(index.reserve(first)) + self.assertTrue(index.reserve(second)) + self.assertFalse(index.reserve(first)) + self.assertTrue(index.reserve(third)) + + self.assertEqual(index.snapshot(), (second, third)) + self.assertFalse(index.reserve(second)) + self.assertTrue(index.reserve(first)) + self.assertEqual(index.snapshot(), (third, first)) + + def test_release_allows_exact_retry_without_disturbing_order(self) -> None: + index = RecentShareIndex(capacity=2) + first = ("miner", "header-a") + second = ("miner", "header-b") + index.reserve(first) + index.reserve(second) + + index.release(first) + + self.assertTrue(index.reserve(first)) + self.assertEqual(index.snapshot(), (second, first)) + + +class SubmitClassificationTests(unittest.TestCase): + def test_active_and_retained_current_tip_are_normal_credit(self) -> None: + active = context() + active_decision = classify_submit_context( + SubmitContextInput(active, None, "tip", False) # type: ignore[arg-type] + ) + self.assertEqual(active_decision.source, "active") + self.assertIsNone(active_decision.credit_policy) + + retained = SimpleNamespace(context=context()) + retained_decision = classify_submit_context( + SubmitContextInput(None, retained, "tip", False) # type: ignore[arg-type] + ) + self.assertEqual(retained_decision.source, "retained") + self.assertIsNone(retained_decision.credit_policy) + + def test_missing_and_stale_contexts_have_distinct_rejections(self) -> None: + with self.assertRaises(SubmitRejected) as missing: + classify_submit_context(SubmitContextInput(None, None, "tip", False)) + self.assertEqual(missing.exception.reason, PRISM_REJECTION_UNKNOWN_JOB) + + with self.assertRaises(SubmitRejected) as stale: + classify_submit_context( + SubmitContextInput(context(parent="old"), None, "tip", False) # type: ignore[arg-type] + ) + self.assertEqual(stale.exception.reason, PRISM_REJECTION_STALE_JOB) + + def test_cross_connection_retained_context_is_same_tip_only(self) -> None: + # A detached cross-connection entry must never borrow the + # reconnect's stale-grace window, even when a caller (wrongly) + # classified the window as open for it. + retained = SimpleNamespace(context=context(parent="old")) + with self.assertRaises(SubmitRejected) as stale: + classify_submit_context( + SubmitContextInput( + None, + retained, # type: ignore[arg-type] + "tip", + True, + cross_connection=True, + ) + ) + self.assertEqual(stale.exception.reason, PRISM_REJECTION_STALE_JOB) + + # A same-tip cross-connection resume stays normal credit and keeps + # its cross-connection tag for mask/outcome handling. + same_tip = SimpleNamespace(context=context()) + decision = classify_submit_context( + SubmitContextInput( + None, + same_tip, # type: ignore[arg-type] + "tip", + False, + cross_connection=True, + ) + ) + self.assertEqual(decision.source, "retained") + self.assertIsNone(decision.credit_policy) + self.assertTrue(decision.cross_connection) + + def test_stale_grace_suppresses_block_route(self) -> None: + decision = classify_submit_context( + SubmitContextInput(context(parent="old"), None, "tip", True) # type: ignore[arg-type] + ) + self.assertEqual(decision.credit_policy, PRISM_CREDIT_POLICY_STALE_GRACE) + + submission = SimpleNamespace( + share_pass=True, + block_pass=True, + header_hex="aa", + ) + work = classify_submit_work( + decision.context, + submission, + credit_policy=decision.credit_policy, + ) + self.assertEqual(work.route, "share") + self.assertFalse(work.block_worthy) + + def test_work_routes_cover_share_and_both_block_paths(self) -> None: + value = context() + ordinary = classify_submit_work( + value, # type: ignore[arg-type] + SimpleNamespace(share_pass=True, block_pass=False, header_hex="a"), + credit_policy=None, + ) + asynchronous = classify_submit_work( + value, # type: ignore[arg-type] + SimpleNamespace(share_pass=True, block_pass=True, header_hex="b"), + credit_policy=None, + ) + synchronous = classify_submit_work( + value, # type: ignore[arg-type] + SimpleNamespace(share_pass=False, block_pass=True, header_hex="c"), + credit_policy=None, + ) + + self.assertEqual(ordinary.route, "share") + self.assertEqual(asynchronous.route, "async_block") + self.assertEqual(synchronous.route, "synchronous_block") + self.assertTrue(synchronous.credit_share_on_accept) + + with self.assertRaises(SubmitRejected) as low: + classify_submit_work( + value, # type: ignore[arg-type] + SimpleNamespace(share_pass=False, block_pass=False, header_hex="d"), + credit_policy=None, + ) + self.assertEqual(low.exception.reason, PRISM_REJECTION_LOW_DIFFICULTY) + + def test_every_block_route_requires_block_pass(self) -> None: + """The precondition ``assemble_submission`` gates serialization on. + + ``block_hex`` is materialized only when ``block_pass`` is set, on the + strength of this classifier: no route that builds a block candidate + can be reached without it. If a future route reaches the candidate + path on ``share_pass`` alone, that submission would carry no block + bytes -- so pin the coupling here, next to the routing it constrains. + """ + value = context() + + for share_pass in (False, True): + for block_pass in (False, True): + for credit_policy in (None, PRISM_CREDIT_POLICY_STALE_GRACE): + with self.subTest( + share_pass=share_pass, + block_pass=block_pass, + credit_policy=credit_policy, + ): + try: + work = classify_submit_work( + value, # type: ignore[arg-type] + SimpleNamespace( + share_pass=share_pass, + block_pass=block_pass, + header_hex="e", + ), + credit_policy=credit_policy, + ) + except SubmitRejected: + continue + if work.route != "share": + self.assertTrue(block_pass) + self.assertEqual(work.block_worthy, work.route != "share") + + +class SubmissionMetricsOwnerTests(unittest.TestCase): + def make_server(self) -> PrismCoordinator: + server = PrismCoordinator.__new__(PrismCoordinator) + server.extranonce2_size = 8 + return server + + def test_share_ack_observation_lives_with_the_owner(self) -> None: + server = self.make_server() + server._observe_share_ack_seconds("accepted", 0.02) + server._observe_share_ack_seconds("rejected", 100.0) + + snapshot = server._ensure_share_submission_service().share_ack_snapshot() + self.assertEqual(tuple(snapshot), PRISM_SHARE_ACK_RESULTS) + self.assertEqual(snapshot["accepted"]["count"], 1) + self.assertEqual(snapshot["accepted"]["buckets"][0.025], 1) + self.assertEqual(snapshot["accepted"]["buckets"][0.01], 0) + self.assertEqual(snapshot["rejected"]["count"], 1) + # Above the last finite bucket: only sum/count observe it. + self.assertEqual(snapshot["rejected"]["buckets"][30.0], 0) + self.assertEqual(snapshot["rejected"]["sum"], 100.0) + metrics = "\n".join(server.share_ack_metrics_lines()) + self.assertIn( + 'qbit_prism_share_ack_seconds_count{result="accepted"} 1', + metrics, + ) + + with self.assertRaises(ValueError): + server._observe_share_ack_seconds("weird", 0.1) + + def test_share_ack_metrics_render_zeroed_without_the_owner(self) -> None: + server = PrismCoordinator.__new__(PrismCoordinator) + metrics = "\n".join(server.share_ack_metrics_lines()) + self.assertIn( + 'qbit_prism_share_ack_seconds_count{result="accepted"} 0', + metrics, + ) + self.assertIn( + 'qbit_prism_share_ack_seconds_count{result="rejected"} 0', + metrics, + ) + self.assertNotIn("_share_submission_service", server.__dict__) + + def test_dropped_solve_counts_route_to_the_owner_without_duplicates( + self, + ) -> None: + server = self.make_server() + server.block_solves_dropped_counts = {"stale_grace": 3} + + service = server._ensure_share_submission_service() + # Adopted at construction; the instance dict no longer holds a + # second copy of the mutable mapping. + self.assertNotIn("block_solves_dropped_counts", server.__dict__) + self.assertEqual(server.block_solves_dropped_counts, {"stale_grace": 3}) + + service._count_dropped_block_solve("stale_grace") + self.assertEqual(server.block_solves_dropped_counts, {"stale_grace": 4}) + + server.block_solves_dropped_counts = {"stale_grace": 0} + self.assertEqual(service.dropped_solves_snapshot(), {"stale_grace": 0}) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_prism_share_writer.py b/tests/test_prism_share_writer.py new file mode 100644 index 000000000..65720708a --- /dev/null +++ b/tests/test_prism_share_writer.py @@ -0,0 +1,839 @@ +#!/usr/bin/env python3 +"""Focused PRISM coordinator share writer tests.""" +# ruff: noqa: F403, F405 + +from __future__ import annotations + +import unittest +from tests.prism_vardiff_test_support import * + + +class PrismCoordinatorVardiffTests(unittest.TestCase): + def test_share_ack_and_counters_wait_for_group_commit(self) -> None: + server, state, ledger = submit_coordinator() + server.share_writer_active = True + server.share_commit_timeout_seconds = 2.0 + server.share_commit_linger_seconds = 0.0 + commit_started = threading.Event() + release_commit = threading.Event() + + class BlockingBatchLedger(type(ledger)): + def append_batch(self, entries: object) -> list[object]: + commit_started.set() + release_commit.wait(timeout=2) + return [self.append(pending) for pending, _candidate in entries] + + ledger = BlockingBatchLedger() + server.ledger = ledger + submission = SimpleNamespace( + header_hex="aa" * 80, + block_hash_hex="cc" * 32, + share_pass=True, + block_pass=False, + ) + + writer = threading.Thread(target=server.share_append_loop, daemon=True) + writer.start() + outcome: list[object] = [] + + def submit() -> None: + with patch( + "lab.prism.prism_coordinator.direct_stratum.assemble_submission", + return_value=submission, + ): + outcome.append( + server.handle_submit( + state, + ["miner-a", "job-1", "00" * 8, "00000001", "00000002"], + ) + ) + + submitter = threading.Thread(target=submit) + submitter.start() + self.assertTrue(commit_started.wait(timeout=1)) + self.assertTrue(submitter.is_alive()) + self.assertEqual(len(ledger.pending), 0) + self.assertEqual(server.worker_share_counts["miner-a"]["accepted"], 0) + + release_commit.set() + submitter.join(timeout=2) + self.assertFalse(submitter.is_alive()) + self.assertEqual(outcome, [False]) + self.assertEqual(len(ledger.pending), 1) + self.assertEqual(ledger.pending[0].share_id, "miner-a:" + "cc" * 32) + self.assertEqual(server.worker_share_counts["miner-a"]["accepted"], 1) + server.request_shutdown() + writer.join(timeout=2) + + def test_job_snapshot_anchor_precedes_stamped_uncommitted_share(self) -> None: + # A share is stamped accepted_at_ms at validation time, before its + # group commit. Anchors chosen while that commit is pending must + # predate the stamp, or the issued window would omit a share that a + # later re-derivation at the same anchor includes. + server, state, ledger = submit_coordinator() + server.share_writer_active = True + server.share_commit_timeout_seconds = 2.0 + server.share_commit_linger_seconds = 0.0 + commit_started = threading.Event() + release_commit = threading.Event() + + class BlockingBatchLedger(type(ledger)): + def append_batch(self, entries: object) -> list[object]: + commit_started.set() + release_commit.wait(timeout=2) + return [self.append(pending) for pending, _candidate in entries] + + ledger = BlockingBatchLedger() + server.ledger = ledger + submission = SimpleNamespace( + header_hex="aa" * 80, + block_hash_hex="cc" * 32, + share_pass=True, + block_pass=False, + ) + + writer = threading.Thread(target=server.share_append_loop, daemon=True) + writer.start() + + def submit() -> None: + with patch( + "lab.prism.prism_coordinator.direct_stratum.assemble_submission", + return_value=submission, + ): + server.handle_submit( + state, + ["miner-a", "job-1", "00" * 8, "00000001", "00000002"], + ) + + submitter = threading.Thread(target=submit) + submitter.start() + self.assertTrue(commit_started.wait(timeout=1)) + + floor = dict(server._pending_share_commit_floor) + self.assertEqual(len(floor), 1) + (floor_entry,) = floor.values() + stamped_ms = int(floor_entry[0].accepted_at_ms) + self.assertEqual( + server._job_snapshot_anchor_ms(stamped_ms + 60_000), + stamped_ms - 1, + ) + + release_commit.set() + submitter.join(timeout=2) + self.assertFalse(submitter.is_alive()) + self.assertEqual(server._pending_share_commit_floor, {}) + # Drained floor: the anchor still sits strictly below the clamp + # instant so a same-millisecond stamp can never tie it. + self.assertEqual( + server._job_snapshot_anchor_ms(stamped_ms + 60_000), + stamped_ms + 60_000 - 1, + ) + server.request_shutdown() + writer.join(timeout=2) + + def test_share_batch_failure_still_restores_snapshot_anchor(self) -> None: + server, _state, _ledger = submit_coordinator() + entry = self._pending_append("aa", accepted_at_ms=2) + server._ensure_pending_share_commit_state() + with server._pending_share_commit_lock: + server._pending_share_commit_floor[id(entry.pending_share)] = [ + entry.pending_share, + time.monotonic(), + False, + ] + self.assertEqual(server._job_snapshot_anchor_ms(10_000), 1) + + class FailingLedger(FakeLedger): + def append(self, pending: object) -> object: + raise RuntimeError("ledger unavailable") + + server.ledger = FailingLedger() + self.assertFalse(server._append_share_batch([entry])) + self.assertTrue(entry.committed.is_set()) + self.assertEqual(server._pending_share_commit_floor, {}) + self.assertEqual(server._job_snapshot_anchor_ms(10_000), 9_999) + + def test_failed_commit_releases_duplicate_key_for_exact_retry(self) -> None: + server, state, healthy = submit_coordinator() + server.share_writer_active = True + server.share_commit_linger_seconds = 0.0 + server.share_commit_timeout_seconds = 1.0 + + class FailedLedger: + def append_batch(self, _entries: object) -> list[object]: + raise RuntimeError("postgres unavailable") + + server.ledger = FailedLedger() + writer = threading.Thread(target=server.share_append_loop, daemon=True) + writer.start() + submission = SimpleNamespace( + header_hex="aa" * 80, + block_hash_hex="dd" * 32, + share_pass=True, + block_pass=False, + ) + with patch( + "lab.prism.prism_coordinator.direct_stratum.assemble_submission", + return_value=submission, + ): + with self.assertRaisesRegex(StratumError, "commit failed"): + server.handle_submit( + state, + ["miner-a", "job-1", "00" * 8, "00000001", "00000002"], + ) + + server.share_writer_active = False + server.ledger = healthy + with patch( + "lab.prism.prism_coordinator.direct_stratum.assemble_submission", + return_value=submission, + ): + self.assertFalse( + server.handle_submit( + state, + ["miner-a", "job-1", "00" * 8, "00000001", "00000002"], + ) + ) + self.assertEqual(len(healthy.pending), 1) + self.assertEqual(server.worker_share_counts["miner-a"]["accepted"], 1) + server.request_shutdown() + writer.join(timeout=2) + + def test_share_writer_retries_failed_append_until_it_lands(self) -> None: + server, state, ledger = submit_coordinator() + + class FlakyLedger(type(ledger)): + def __init__(self) -> None: + super().__init__() + self.failures_remaining = 2 + + def append(self, pending: object) -> object: + if self.failures_remaining > 0: + self.failures_remaining -= 1 + raise RuntimeError("ledger briefly unavailable") + return super().append(pending) + + flaky = FlakyLedger() + server.ledger = flaky + entry = PendingShareAppend( + pending_share=SimpleNamespace( + share_id="miner-a:" + "ee" * 32, + job_issued_at_ms=0, + accepted_at_ms=0, + ), + username="miner-a", + job_id="job-1", + block_hash_hex="ee" * 32, + collection_only=False, + credit_policy=None, + ) + + with patch.object(server.stop_event, "wait", return_value=False) as waited: + server._append_share_entry(entry, retry_until_stopped=True) + + self.assertEqual(len(flaky.pending), 1) + self.assertEqual(server.share_append_failure_count, 2) + self.assertIn( + "qbit_prism_share_append_failures_total 2", + server.metrics_payload(), + ) + self.assertEqual(waited.call_count, 2) + + def _pending_append(self, tag: str, accepted_at_ms: int = 2) -> PendingShareAppend: + from lab.prism.share_ledger import PendingShare + + return PendingShareAppend( + pending_share=PendingShare( + share_id=f"miner-a:{tag}", + miner_id="miner-a", + order_key="miner-a", + p2mr_program_hex="11" * 32, + share_difficulty=1, + network_difficulty=1, + template_height=10, + job_id="job-1", + job_issued_at_ms=1, + accepted_at_ms=accepted_at_ms, + ntime=1_700_000_000, + ), + username="miner-a", + job_id="job-1", + block_hash_hex=tag * 32, + collection_only=False, + credit_policy=None, + ) + + def test_share_append_backlog_overflow_never_reports_success(self) -> None: + server, _state, _ledger = submit_coordinator() + server.share_append_queue = queue.Queue(maxsize=2) + with tempfile.TemporaryDirectory() as tempdir: + server.share_recovery_path = Path(tempdir) / "recovery.jsonl" + + server.enqueue_share_append(self._pending_append("aa")) + server.enqueue_share_append(self._pending_append("bb")) + with self.assertRaisesRegex(StratumError, "queue is full"): + server.enqueue_share_append(self._pending_append("cc")) + + self.assertEqual(server.shares_recovered_to_disk, 0) + remaining = [ + server.share_append_queue.get_nowait().pending_share.share_id + for _ in range(2) + ] + self.assertEqual(remaining, ["miner-a:aa", "miner-a:bb"]) + self.assertFalse(server.share_recovery_path.exists()) + + def test_writer_recovers_acked_share_on_shutdown_during_outage(self) -> None: + server, _state, ledger = submit_coordinator() + with tempfile.TemporaryDirectory() as tempdir: + server.share_recovery_path = Path(tempdir) / "recovery.jsonl" + + class DownLedger(type(ledger)): + def append(self, pending: object) -> object: + raise RuntimeError("postgres unavailable") + + server.ledger = DownLedger() + entry = self._pending_append("ee") + # First backoff wait returns True (stop requested): the share must + # be recovered, not silently dropped. + with patch.object(server.stop_event, "wait", return_value=True): + server._append_share_entry(entry, retry_until_stopped=True) + + self.assertEqual(server.shares_recovered_to_disk, 1) + recovered = json.loads(server.share_recovery_path.read_text().strip()) + self.assertEqual(recovered["share_id"], "miner-a:ee") + + def test_replay_recovered_shares_appends_and_clears_file(self) -> None: + server, _state, ledger = submit_coordinator() + with tempfile.TemporaryDirectory() as tempdir: + server.share_recovery_path = Path(tempdir) / "recovery.jsonl" + server._recover_share_to_disk(self._pending_append("f1"), "test") + server._recover_share_to_disk(self._pending_append("f2"), "test") + + replayed = server.replay_recovered_shares() + + self.assertEqual(replayed, 2) + self.assertEqual(server.shares_replayed, 2) + self.assertEqual( + [p.share_id for p in ledger.pending], ["miner-a:f1", "miner-a:f2"] + ) + # File is cleared after a clean replay so shares are not re-added. + self.assertFalse(server.share_recovery_path.exists()) + self.assertEqual(server.replay_recovered_shares(), 0) + + def test_replay_recovered_shares_orders_by_accepted_at(self) -> None: + # A share can be recovered out of FIFO order (overflow of the newest, or + # a ledger flap during the shutdown drain). Replay must reorder by + # accepted_at_ms so share_seq reflects acceptance order, keeping the + # reward window correctly ordered. + server, _state, ledger = submit_coordinator() + with tempfile.TemporaryDirectory() as tempdir: + server.share_recovery_path = Path(tempdir) / "recovery.jsonl" + server._recover_share_to_disk(self._pending_append("late", accepted_at_ms=300), "test") + server._recover_share_to_disk(self._pending_append("early", accepted_at_ms=100), "test") + server._recover_share_to_disk(self._pending_append("mid", accepted_at_ms=200), "test") + + replayed = server.replay_recovered_shares() + + self.assertEqual(replayed, 3) + self.assertEqual( + [p.share_id for p in ledger.pending], + ["miner-a:early", "miner-a:mid", "miner-a:late"], + ) + + def test_replay_skips_torn_line_and_keeps_file(self) -> None: + # A crash mid-append can leave the last line torn. That one line must + # not block the intact shares before it, and the file is kept so the + # torn line is preserved rather than silently discarded. + server, _state, ledger = submit_coordinator() + with tempfile.TemporaryDirectory() as tempdir: + server.share_recovery_path = Path(tempdir) / "recovery.jsonl" + server._recover_share_to_disk(self._pending_append("g1", accepted_at_ms=100), "test") + server._recover_share_to_disk(self._pending_append("g2", accepted_at_ms=200), "test") + with open(server.share_recovery_path, "a", encoding="utf-8") as handle: + handle.write('{"share_id": "miner-a:torn", "miner_i') # truncated, no newline + + replayed = server.replay_recovered_shares() + + self.assertEqual(replayed, 2) + self.assertEqual( + [p.share_id for p in ledger.pending], ["miner-a:g1", "miner-a:g2"] + ) + # File kept because a line could not be parsed. + self.assertTrue(server.share_recovery_path.exists()) + # Re-running classifies the intact rows as exact-existing. They are + # not inserted or counted again, while the torn line keeps the + # journal for inspection. + self.assertEqual(server.replay_recovered_shares(), 0) + self.assertEqual( + [p.share_id for p in ledger.pending], ["miner-a:g1", "miner-a:g2"] + ) + self.assertTrue(server.share_recovery_path.exists()) + + def test_replay_is_idempotent_across_partial_replay(self) -> None: + # Finding: a partial replay (A commits, B fails transiently) kept the + # whole file; on retry, replay hit A's duplicate and stopped, stranding + # B forever. Replay must skip the already-committed A and reach B. + server, _state, _ledger = submit_coordinator() + with tempfile.TemporaryDirectory() as tempdir: + server.share_recovery_path = Path(tempdir) / "recovery.jsonl" + server._recover_share_to_disk(self._pending_append("A", accepted_at_ms=100), "test") + server._recover_share_to_disk(self._pending_append("B", accepted_at_ms=200), "test") + + class DedupLedger: + def __init__(self) -> None: + self.ids: list[str] = [] + self.pending_by_id: dict[str, object] = {} + self.fail_b_once = True + + def append_recovered_share(self, pending: object) -> object: + if pending.share_id == "miner-a:B" and self.fail_b_once: + self.fail_b_once = False + raise RuntimeError("postgres unavailable") + if pending.share_id in self.ids: + if self.pending_by_id[pending.share_id] != pending: + raise ShareReplayConflict(pending.share_id) + return ShareReplayResult( + "exact_existing", + SimpleNamespace( + share_seq=self.ids.index(pending.share_id) + 1 + ), + ) + self.ids.append(pending.share_id) + self.pending_by_id[pending.share_id] = pending + return ShareReplayResult( + "inserted", + SimpleNamespace(share_seq=len(self.ids)), + ) + + server.ledger = DedupLedger() + + # Pass 1: A commits, B raises a transient (non-duplicate) error, so + # the pass stops and keeps the file. + self.assertEqual(server.replay_recovered_shares(), 1) + self.assertTrue(server.share_recovery_path.exists()) + self.assertEqual(server.ledger.ids, ["miner-a:A"]) + + # Pass 2: A is now a duplicate (skipped, not fatal); B commits and + # the file is cleared. + self.assertEqual(server.replay_recovered_shares(), 1) + self.assertEqual(server.ledger.ids, ["miner-a:A", "miner-a:B"]) + self.assertFalse(server.share_recovery_path.exists()) + + def test_replay_quarantines_conflicting_row_and_replays_the_rest(self) -> None: + # A ShareReplayConflict is keyed to one share_id; it must not strand + # the journal rows behind it, and the conflict must be visible as a + # counter, not only in stdout. + server, _state, _ledger = submit_coordinator() + with tempfile.TemporaryDirectory() as tempdir: + server.share_recovery_path = Path(tempdir) / "recovery.jsonl" + server._recover_share_to_disk(self._pending_append("A", accepted_at_ms=100), "test") + server._recover_share_to_disk(self._pending_append("B", accepted_at_ms=200), "test") + server._recover_share_to_disk(self._pending_append("C", accepted_at_ms=300), "test") + + class ConflictingLedger(RecordingLedger): + def __init__(self) -> None: + super().__init__() + self.ids: list[str] = [] + + def append_recovered_share(self, pending: object) -> object: + if pending.share_id == "miner-a:A": + raise ShareReplayConflict(pending.share_id) + if pending.share_id in self.ids: + return ShareReplayResult( + "exact_existing", + SimpleNamespace( + share_seq=self.ids.index(pending.share_id) + 1 + ), + ) + self.ids.append(pending.share_id) + return ShareReplayResult( + "inserted", + SimpleNamespace(share_seq=len(self.ids)), + ) + + server.ledger = ConflictingLedger() + + self.assertEqual(server.replay_recovered_shares(), 2) + self.assertTrue(server.share_recovery_path.exists()) + self.assertEqual(server.ledger.ids, ["miner-a:B", "miner-a:C"]) + self.assertEqual(server.share_replay_conflicts, 1) + self.assertIn( + "qbit_prism_share_replay_conflicts_total 1", + server.metrics_payload(), + ) + + # Re-running the retained journal credits nothing twice: B and C + # are exact duplicates, A conflicts again, the file stays. + self.assertEqual(server.replay_recovered_shares(), 0) + self.assertTrue(server.share_recovery_path.exists()) + self.assertEqual(server.ledger.ids, ["miner-a:B", "miner-a:C"]) + self.assertEqual(server.share_replay_conflicts, 2) + + def test_append_share_entry_reports_persisted_vs_recovered(self) -> None: + server, _state, ledger = submit_coordinator() + with tempfile.TemporaryDirectory() as tempdir: + server.share_recovery_path = Path(tempdir) / "recovery.jsonl" + # Healthy ledger: reports persisted. + self.assertTrue( + server._append_share_entry(self._pending_append("ok"), retry_until_stopped=True) + ) + + class DownLedger(type(ledger)): + def append(self, pending: object) -> object: + raise RuntimeError("postgres unavailable") + + server.ledger = DownLedger() + with patch.object(server.stop_event, "wait", return_value=True): + # Shutdown mid-outage: reports recovered, not persisted. + self.assertFalse( + server._append_share_entry(self._pending_append("down"), retry_until_stopped=True) + ) + + def test_group_commit_failure_releases_all_waiters_without_recovery_file(self) -> None: + server, _state, ledger = submit_coordinator() + with tempfile.TemporaryDirectory() as tempdir: + server.share_recovery_path = Path(tempdir) / "recovery.jsonl" + server.share_append_queue = queue.Queue(maxsize=MAX_PENDING_SHARE_APPENDS) + + append_calls: list[str] = [] + + class DownLedger(type(ledger)): + def append(self, pending: object) -> object: + append_calls.append(pending.share_id) + raise RuntimeError("postgres unavailable") + + server.ledger = DownLedger() + entries = [] + for tag in ("s1", "s2", "s3"): + entry = self._pending_append(tag) + entries.append(entry) + server.enqueue_share_append(entry) + + server.request_shutdown() + server.share_append_loop() + + # The compatibility ledger fails on the first row; the whole batch + # is reported failed and no uncommitted share is called durable. + self.assertEqual(append_calls, ["miner-a:s1"]) + self.assertEqual(ledger.pending, []) + self.assertTrue(all(entry.committed.is_set() for entry in entries)) + self.assertTrue(all(entry.error is not None for entry in entries)) + self.assertFalse(server.share_recovery_path.exists()) + + def test_predating_append_commits_while_fast_lane_offer_is_in_flight( + self, + ) -> None: + # The other side of the fence boundary flipped with the node fast + # lane: submitblock no longer holds the landing fence, so a + # predating append's durable commit does NOT wait for the RPC to + # return. The row may become durable mid-offer with its epoch bump + # landing under the fence; the landing's post-offer accounting + # fences (not the offer itself) are what observe the invalidation. + # Here the node rejects the block outright, so the rejection -- not + # the epoch race -- terminally abandons the candidate. + server, state, _recording = submit_coordinator() + ledger = SingleWriterShareLedger() + server.ledger = ledger + context = server.jobs["job-1"] + context.found_block = { + "network_difficulty": 1, + "anchor_job_issued_at_ms": 12000, + } + # Keep an anchor the late row predates exposed for the whole test, + # independent of the landing's own declared-anchor lifetime. + anchor_token = server._expose_inflight_scan_anchor(12000) + self.addCleanup(server._retire_inflight_scan_anchor, anchor_token) + + late_entry = self._pending_append("mid-rpc-append") + durable_mid_rpc: list[bool] = [] + append_threads: list[threading.Thread] = [] + + class MidRpcAppendRpc(TipRpc): + def call( + rpc_self, + method: str, + params: list[object] | None = None, + ) -> object: + if method == "getblockcount": + return 9 + if method == "submitblock": + thread = threading.Thread( + target=server._append_share_entry, + args=(late_entry,), + daemon=True, + ) + thread.start() + append_threads.append(thread) + deadline = time.monotonic() + 0.25 + became_durable = False + while time.monotonic() < deadline: + if late_entry.pending_share.share_id in ledger._share_ids: + became_durable = True + break + time.sleep(0.005) + durable_mid_rpc.append(became_durable) + return "rejected-by-test" + return super().call(method, params) + + server.rpc = MidRpcAppendRpc("00" * 32) + submission = SimpleNamespace( + coinbase_tx_hex="c0ffee", + block_hash_hex="dd" * 32, + block_hex="00", + ) + pending = self._pending_append("landing-holds-fence").pending_share + candidate = block_candidate( + server, + state, + submission, + pending_share=pending, + ) + ledger.append_batch( + [(pending, server.block_candidate_intent(candidate))] + ) + server.enqueue_block_candidate(candidate) + + self.assertTrue(server.submit_next_block_candidate()) + + # The fast-lane RPC holds no fence, so the predating row became + # durable while the offer was still in flight. + self.assertEqual(durable_mid_rpc, [True]) + self.assertTrue(append_threads) + append_threads[0].join(timeout=10.0) + self.assertFalse(append_threads[0].is_alive()) + self.assertIn(late_entry.pending_share.share_id, ledger._share_ids) + with server._job_cache_lock: + self.assertEqual(server._payout_ledger_append_invalidation_epoch, 1) + self.assertEqual( + server.block_candidate_abandoned_counts[ + PRISM_REJECTION_SUBMITBLOCK_REJECTED + ], + 1, + ) + + def test_completed_append_predating_seedless_published_window_fences( + self, + ) -> None: + # A build that seeds no artifact (the documented + # PRISM_PAYOUT_ARTIFACT_REUSE=0 rollback mode) retires its walk + # exposure at the publication fence, yet jobs stamped from it keep + # serving that window until they retire. A replay-shaped append + # that started AND finished in that gap used to see no live + # anchor: no epoch bump, and its registry entry was popped on + # completion -- so a later landing drained nothing, its + # nonnegative context epoch skipped durable revalidation, and + # submitblock accepted a coinbase omitting the durable share. + # Publication now hands the declared anchor to the + # published-window watermark, so the append commits under the + # landing fence with its epoch bump -- and the landing, whose + # offer already landed, keeps the as-issued snapshot for + # accounting. + server, state, _recording = submit_coordinator() + ledger = SingleWriterShareLedger() + server.ledger = ledger + bundle_builds: list[dict[str, object]] = [] + + def recording_build_audit_bundle(**kwargs: object) -> dict[str, object]: + bundle_builds.append(dict(kwargs)) + return verified_block_bundle() + + server.build_audit_bundle = recording_build_audit_bundle # type: ignore[method-assign] + server.verify_bundle = lambda *_args, **_kwargs: verified_audit_report() # type: ignore[method-assign] + tail_dir = tempfile.TemporaryDirectory() + self.addCleanup(tail_dir.cleanup) + server.audit_dir = Path(tail_dir.name) + server.evidence_path = Path(tail_dir.name) / "evidence.json" + server.ledger_writer_public_key_hex = "aa" * 32 + context = server.jobs["job-1"] + context.found_block = { + "network_difficulty": 1, + "anchor_job_issued_at_ms": 12000, + } + submission = SimpleNamespace( + coinbase_tx_hex="c0ffee", + block_hash_hex="dd" * 32, + block_hex="00", + ) + pending = self._pending_append("seedless-window").pending_share + candidate = block_candidate( + server, + state, + submission, + pending_share=pending, + ) + ledger.append_batch( + [(pending, server.block_candidate_intent(candidate))] + ) + server.enqueue_block_candidate(candidate) + + # The publication fence of a seedless build hands the bundle's + # declared anchor to the watermark when it retires the walk + # exposure. + server._ensure_job_cache_state() + with server._job_cache_lock: + server._publish_seedless_job_window_anchor_locked(12000) + + # The ordinary share hot path stays off the fence: a row stamped + # above the watermark commits unfenced and bumps nothing. + ordinary_entry = self._pending_append( + "ordinary-above-watermark", accepted_at_ms=13000 + ) + server._append_share_entry(ordinary_entry) + with server._job_cache_lock: + self.assertEqual(server._payout_ledger_append_invalidation_epoch, 0) + self.assertEqual( + server._payout_unfenced_append_inflight_stamps, {} + ) + + # The replay-shaped append starts and finishes entirely before the + # landing begins: nothing is in flight for the landing to drain. + late_entry = self._pending_append("completed-before-landing") + server._append_share_entry(late_entry) + self.assertIn(late_entry.pending_share.share_id, ledger._share_ids) + with server._job_cache_lock: + self.assertEqual(server._payout_ledger_append_invalidation_epoch, 1) + self.assertEqual( + server._payout_unfenced_append_inflight_stamps, {} + ) + + submitblock_calls: list[object] = [] + + class RecordingSubmitRpc(TipRpc): + def call( + rpc_self, + method: str, + params: list[object] | None = None, + ) -> object: + if method == "getblockcount": + return 9 + if method == "submitblock": + submitblock_calls.append(params) + return None + if method == "getblockhash": + return "dd" * 32 + return super().call(method, params) + + server.rpc = RecordingSubmitRpc("00" * 32) + + self.assertTrue(server.submit_next_block_candidate()) + + # The bump already happened at commit time; the landing's epoch + # fence observes it and, with the node offer already accepted, + # accounts the as-issued coinbase (the durable predating share + # rides the next window). + self.assertEqual(submitblock_calls, [["00"]]) + # The landing observed the bump but the node already accepted + # the block: accounting keeps the as-issued snapshot instead of + # terminally abandoning a live block's payouts. + self.assertEqual(len(bundle_builds), 1) + self.assertNotIn( + PRISM_REJECTION_STALE_JOB, + server.block_candidate_abandoned_counts, + ) + self.assertEqual( + getattr(server, "stale_job_abandon_counts", {}).get( + "append_epoch_stale", 0 + ), + 0, + ) + outbox_row = ledger._block_candidate_outbox[submission.block_hash_hex] + self.assertEqual(outbox_row["state"], "submitted") + self.assertIsNone(outbox_row["last_error"]) + self.assertEqual(server.accepted_block_count, 1) + + def test_replay_recovered_shares_commit_under_the_landing_fence(self) -> None: + # Recovered rows reconstruct pre-crash timestamps, so they predate + # live anchors. Each replay append must hold the landing fence across + # the ledger commit and its epoch bump -- never bump after the row is + # already durable, where a landing could verify the pre-bump epoch + # and submit a coinbase omitting the replayed share. + server, _state, ledger = submit_coordinator() + anchor_token = server._expose_inflight_scan_anchor(12000) + self.addCleanup(server._retire_inflight_scan_anchor, anchor_token) + with tempfile.TemporaryDirectory() as tempdir: + server.share_recovery_path = Path(tempdir) / "recovery.jsonl" + server._recover_share_to_disk(self._pending_append("h1"), "test") + + fence_held_during_commit: list[bool] = [] + original_append = ledger.append + + def recording_append(pending: object) -> object: + fence_held_during_commit.append( + server._payout_append_landing_fence_lock.locked() + ) + return original_append(pending) + + ledger.append = recording_append # type: ignore[method-assign] + replayed = server.replay_recovered_shares() + + self.assertEqual(replayed, 1) + self.assertEqual(fence_held_during_commit, [True]) + with server._job_cache_lock: + self.assertEqual(server._payout_ledger_append_invalidation_epoch, 1) + + def test_sync_append_commits_predating_row_under_the_landing_fence( + self, + ) -> None: + # The synchronous (no-writer) append path takes the same fence + # boundary as the group-commit writer for a row that predates a + # live anchor. + server, _state, ledger = submit_coordinator() + anchor_token = server._expose_inflight_scan_anchor(12000) + self.addCleanup(server._retire_inflight_scan_anchor, anchor_token) + + fence_held_during_commit: list[bool] = [] + original_append = ledger.append + + def recording_append(pending: object) -> object: + fence_held_during_commit.append( + server._payout_append_landing_fence_lock.locked() + ) + return original_append(pending) + + ledger.append = recording_append # type: ignore[method-assign] + self.assertTrue(server._append_share_entry(self._pending_append("h2"))) + self.assertEqual(fence_held_during_commit, [True]) + with server._job_cache_lock: + self.assertEqual(server._payout_ledger_append_invalidation_epoch, 1) + + +class PrismCoordinatorReliabilityTests(unittest.TestCase): + def _bare_coordinator(self) -> PrismCoordinator: + server = PrismCoordinator.__new__(PrismCoordinator) + server.lock = threading.RLock() + server.stop_event = threading.Event() + server._heartbeats = {} + server._watchdog_pauses = {} + server._heartbeats_lock = threading.Lock() + server.watchdog_timeout_seconds = 120.0 + server.watchdog_interval_seconds = 15.0 + return server + + def test_release_ledger_lease_is_noop_without_lease_support(self) -> None: + server = self._bare_coordinator() + server.ledger = SimpleNamespace() + + # In-memory/regtest ledgers have no release_writer_lease; must not raise. + server.release_ledger_lease() + + def test_release_ledger_lease_invokes_ledger_release(self) -> None: + server = self._bare_coordinator() + calls: list[bool] = [] + server.ledger = SimpleNamespace( + release_writer_lease=lambda: (calls.append(True), True)[1] + ) + + server.release_ledger_lease() + + self.assertEqual(calls, [True]) + + def test_release_ledger_lease_swallows_release_errors(self) -> None: + server = self._bare_coordinator() + + def _boom() -> bool: + raise RuntimeError("db unreachable during shutdown") + + server.ledger = SimpleNamespace(release_writer_lease=_boom) + + # Shutdown must not raise even if the lease release fails. + server.release_ledger_lease() diff --git a/tests/test_prism_share_writer_service.py b/tests/test_prism_share_writer_service.py new file mode 100644 index 000000000..7b6c7b79e --- /dev/null +++ b/tests/test_prism_share_writer_service.py @@ -0,0 +1,654 @@ +#!/usr/bin/env python3 +"""Direct tests for the coordinator-free S3 share writer service. + +The service resolves the coordinator through a call-time ``ShareWriterRuntime`` +typed port, so these tests drive it with a small recording runtime plus the +real shutdown controller and (mostly) the real in-memory ledger. +""" + +from __future__ import annotations + +from contextlib import contextmanager +from pathlib import Path +import queue +import tempfile +import threading +import time +from types import SimpleNamespace +import unittest +from unittest.mock import patch + +from lab.prism.coordinator_shutdown import ( + CoordinatorShutdownController, + ShutdownInProgress, +) +from lab.prism.share_ledger import ( + PendingShare, + ShareReplayResult, + SingleWriterShareLedger, +) +from lab.prism.share_writer import ( + PENDING_SHARE_COMMIT_WARN_SECONDS, + PendingShareAppend, + ShareWriter, +) +from lab.prism.stratum_session import StratumError + + +class RecordingRuntime: + """Minimal call-time runtime port mirroring the coordinator facades.""" + + def __init__( + self, + *, + ledger: object, + controller: CoordinatorShutdownController, + stop_event: threading.Event, + ) -> None: + self.ledger = ledger + self.hot_path_log_enabled = False + self.stop_event = stop_event + self._controller = controller + self._share_accounting_lock = threading.Lock() + self.writer: ShareWriter | None = None + self.worker_notes: list[tuple[str, str | None]] = [] + self.vardiff_notes: list[tuple[object, object]] = [] + self.retired: list[tuple[PendingShare, int]] = [] + + def _ensure_shutdown_controller(self) -> CoordinatorShutdownController: + return self._controller + + def _record_heartbeat(self, _name: str, *, phase: str | None = None) -> None: + return None + + def _ensure_share_hot_path_state(self) -> None: + return None + + def _finish_pending_share_commit(self, pending: PendingShare) -> None: + assert self.writer is not None + self.writer.finish_pending_share(pending) + + @contextmanager + def _landing_fence_for_predating_append(self, _pendings: list[PendingShare]): + yield False + + def _record_late_visible_payout_append( + self, + _pending: PendingShare, + *, + landing_fence_owned: bool, + ) -> int | None: + return None + + def _retire_payout_windows_for_late_append( + self, pending: PendingShare, epoch: int + ) -> None: + self.retired.append((pending, epoch)) + + def _recover_share_to_disk(self, entry: PendingShareAppend, reason: str) -> None: + assert self.writer is not None + self.writer.recover_share_to_disk(entry, reason) + + def accepted_share_difficulty(self, _context: object) -> int: + return 1 + + def enqueue_share_append( + self, entry: PendingShareAppend, *, wait: bool = False + ) -> None: + assert self.writer is not None + self.writer.enqueue_share_append(entry, wait=wait) + + def _append_share_entry(self, entry: PendingShareAppend) -> bool: + assert self.writer is not None + return self.writer.append_share_entry(entry) + + def note_worker_accepted_share( + self, username: str, credit_policy: str | None + ) -> None: + self.worker_notes.append((username, credit_policy)) + + def note_vardiff_accepted_share(self, client: object, job: object) -> None: + self.vardiff_notes.append((client, job)) + + +class ShareWriterServiceTests(unittest.TestCase): + def _service( + self, + *, + ledger: object | None = None, + wall_times: list[int] | None = None, + controller: CoordinatorShutdownController | None = None, + ) -> tuple[ShareWriter, object, CoordinatorShutdownController, RecordingRuntime]: + ledger = ledger if ledger is not None else SingleWriterShareLedger() + controller = controller or CoordinatorShutdownController(1.0) + stop = threading.Event() + times = list(wall_times or [100]) + last = times[-1] + + def wall_time_ms() -> int: + return times.pop(0) if times else last + + runtime = RecordingRuntime( + ledger=ledger, + controller=controller, + stop_event=stop, + ) + service = ShareWriter( + runtime, + internal_error_reason="internal_error", + now_ms=wall_time_ms, + ) + runtime.writer = service + return service, ledger, controller, runtime + + @staticmethod + def _context(username: str = "miner-a") -> SimpleNamespace: + return SimpleNamespace( + worker=SimpleNamespace( + username=username, + payout_address="miner-a", + p2mr_program_hex="11" * 32, + ), + found_block={"network_difficulty": 1}, + template={"height": 10}, + job=SimpleNamespace(job_id="job-1"), + issued_at_ms=1, + collection_only=False, + ) + + @staticmethod + def _submission(block_hash_hex: str = "ab" * 32) -> SimpleNamespace: + return SimpleNamespace(block_hash_hex=block_hash_hex) + + def _stamp(self, service: ShareWriter, block_hash_hex: str = "ab" * 32) -> PendingShare: + return service.pending_share_from_submission( + context=self._context(), + submission=self._submission(block_hash_hex), + ntime_hex="65000000", + ) + + @staticmethod + def _entry(pending: PendingShare) -> PendingShareAppend: + return PendingShareAppend( + pending_share=pending, + username="miner-a", + job_id="job-1", + block_hash_hex=pending.share_id.rsplit(":", 1)[-1], + collection_only=False, + credit_policy=None, + ) + + @staticmethod + def _pending(share_id: str, *, accepted_at_ms: int = 100, ntime: int = 1_700_000_000) -> PendingShare: + return PendingShare( + share_id=share_id, + miner_id="miner-a", + order_key="miner-a", + p2mr_program_hex="11" * 32, + share_difficulty=1, + network_difficulty=1, + template_height=9, + job_id="job-1", + job_issued_at_ms=1, + accepted_at_ms=accepted_at_ms, + ntime=ntime, + ) + + # -- compatibility identity and descriptor routing --------------------- + + def test_coordinator_reexports_exact_pending_append_identity(self) -> None: + from lab.prism.prism_coordinator import PendingShareAppend as compatibility + + self.assertIs(compatibility, PendingShareAppend) + + def test_legacy_coordinator_fields_route_to_the_service_owner(self) -> None: + from lab.prism.prism_coordinator import PrismCoordinator + + server = PrismCoordinator.__new__(PrismCoordinator) + service = server._ensure_share_writer_service() + + self.assertIs(server.share_append_queue, service.share_append_queue) + self.assertIs( + server._pending_share_commit_floor, + service._pending_share_commit_floor, + ) + + server.share_commit_batch_size = 7 + self.assertEqual(service.share_commit_batch_size, 7) + + adopted: queue.Queue[PendingShareAppend] = queue.Queue(maxsize=3) + server.share_append_queue = adopted + self.assertIs(service.share_append_queue, adopted) + + server.share_writer_active = True + self.assertTrue(service.share_writer_active) + + # -- pending-commit floor ---------------------------------------------- + + def test_submission_stamp_registers_floor_and_anchor_clamps_below(self) -> None: + service, _ledger, _controller, _runtime = self._service(wall_times=[100]) + + pending = self._stamp(service) + + self.assertEqual(pending.accepted_at_ms, 100) + entry = service._pending_share_commit_floor[id(pending)] + self.assertIs(entry[0], pending) + self.assertEqual(len(entry), 3) + self.assertFalse(entry[2]) + self.assertEqual(service.snapshot_anchor_ms(1_000), 99) + + service.finish_pending_share(pending) + self.assertEqual(service._pending_share_commit_floor, {}) + self.assertEqual(service.snapshot_anchor_ms(1_000), 999) + + def test_floor_release_is_idempotent_and_keyed_by_object_identity(self) -> None: + # Layer note: the current tree keeps the id-keyed holder (the old + # stack's durable share-id lease was deliberately not built), so a + # reconstructed object with the same durable identity does not + # release the original stamp's floor entry. + service, _ledger, _controller, _runtime = self._service(wall_times=[100]) + pending = self._stamp(service) + + reconstructed = PendingShare(**pending.__dict__) + service.finish_pending_share(reconstructed) + self.assertIn(id(pending), service._pending_share_commit_floor) + self.assertEqual(service.snapshot_anchor_ms(1_000), 99) + + service.finish_pending_share(pending) + self.assertEqual(service._pending_share_commit_floor, {}) + service.finish_pending_share(pending) + self.assertEqual(service._pending_share_commit_floor, {}) + + def test_attempt_and_candidate_release_share_one_current_holder(self) -> None: + # finish_pending_attempt / finish_pending_candidate are the staged + # future split seams; at this layer both release the single holder. + service, _ledger, _controller, _runtime = self._service(wall_times=[100, 200]) + + attempt = self._stamp(service) + service.finish_pending_attempt(attempt) + self.assertEqual(service._pending_share_commit_floor, {}) + + candidate = self._stamp(service, "cd" * 32) + service.finish_pending_candidate(candidate) + self.assertEqual(service._pending_share_commit_floor, {}) + + def test_stale_floor_entry_warns_once_and_keeps_holding_the_anchor(self) -> None: + service, _ledger, _controller, _runtime = self._service() + pending = self._pending("miner-a:stale", accepted_at_ms=100) + service._pending_share_commit_floor[id(pending)] = [ + pending, + time.monotonic() - PENDING_SHARE_COMMIT_WARN_SECONDS - 1.0, + False, + ] + + with patch("builtins.print") as first_pass: + self.assertEqual(service.snapshot_anchor_ms(1_000), 99) + warned = [ + call + for call in first_pass.call_args_list + if "holding the job snapshot anchor floor" in str(call) + ] + self.assertEqual(len(warned), 1) + self.assertIn("miner-a:stale", str(warned[0])) + self.assertTrue(service._pending_share_commit_floor[id(pending)][2]) + + with patch("builtins.print") as second_pass: + self.assertEqual(service.snapshot_anchor_ms(1_000), 99) + self.assertEqual(second_pass.call_args_list, []) + + # -- append admission and floor release -------------------------------- + + def test_full_queue_admission_refusal_releases_token_and_floor(self) -> None: + service, _ledger, controller, _runtime = self._service() + service.share_writer_active = True + service.share_commit_timeout_seconds = 0.05 + service.share_append_queue = queue.Queue(maxsize=1) + blocker = self._entry(self._pending("miner-a:blocker")) + blocker.writer_token = controller.reserve_writer("share_persistence") + service.share_append_queue.put_nowait(blocker) + pending = self._stamp(service) + + with self.assertRaisesRegex(StratumError, "commit queue is full"): + service.append_accepted_share( + object(), + self._context(), + self._submission(), + pending, + ) + + self.assertEqual(service._pending_share_commit_floor, {}) + self.assertEqual( + controller.snapshot()["active_writers"], + {"share_persistence": 1}, + ) + blocker.writer_token.finish() + self.assertEqual(controller.snapshot()["active_writers"], {}) + + def test_closed_writer_admission_refusal_releases_unhanded_attempt(self) -> None: + controller = CoordinatorShutdownController(1.0) + service, ledger, _controller, runtime = self._service(controller=controller) + service.share_writer_active = True + pending = self._stamp(service) + controller.begin_shutdown("test shutdown") + + with self.assertRaises(ShutdownInProgress): + service.append_accepted_share( + object(), + self._context(), + self._submission(), + pending, + ) + + self.assertEqual(service._pending_share_commit_floor, {}) + self.assertEqual(len(ledger), 0) + self.assertEqual(runtime.worker_notes, []) + self.assertEqual(controller.snapshot()["active_writers"], {}) + + def test_nested_append_inherits_outer_admission_after_shutdown_closes(self) -> None: + controller = CoordinatorShutdownController(1.0) + service, ledger, _controller, _runtime = self._service(controller=controller) + service.share_writer_active = True + outer = controller.enter_writer("share_submission") + try: + pending = self._stamp(service) + controller.begin_shutdown("test shutdown") + entry = self._entry(pending) + service.enqueue_share_append(entry) + finally: + controller.exit_writer(outer) + + self.assertIsNotNone(entry.writer_token) + queued = service.share_append_queue.get_nowait() + self.assertIs(queued, entry) + self.assertTrue(service.append_share_batch([queued])) + self.assertEqual(len(ledger), 1) + self.assertTrue(entry.committed.is_set()) + self.assertIsNone(entry.writer_token) + self.assertEqual(service._pending_share_commit_floor, {}) + self.assertEqual(controller.snapshot()["active_writers"], {}) + + def test_synchronous_append_releases_floor_and_never_recounts_replays(self) -> None: + service, ledger, _controller, runtime = self._service() + service.share_writer_active = False + pending = self._stamp(service) + + service.append_accepted_share( + object(), + self._context(), + self._submission(), + pending, + ) + + self.assertEqual(len(ledger), 1) + self.assertEqual(service._pending_share_commit_floor, {}) + self.assertEqual(runtime.worker_notes, [("miner-a", None)]) + self.assertEqual(len(runtime.vardiff_notes), 1) + + # An exact ledger replay returns the original record; process-local + # worker/vardiff counters must not increment again. + service.append_accepted_share( + object(), + self._context(), + self._submission(), + pending, + ) + self.assertEqual(len(ledger), 1) + self.assertEqual(runtime.worker_notes, [("miner-a", None)]) + self.assertEqual(len(runtime.vardiff_notes), 1) + + def test_batch_success_sets_records_and_releases_every_waiter(self) -> None: + service, ledger, controller, _runtime = self._service(wall_times=[100, 200]) + first = self._entry(self._stamp(service, "aa" * 32)) + second = self._entry(self._stamp(service, "bb" * 32)) + for entry in (first, second): + entry.writer_token = controller.reserve_writer("share_persistence") + + self.assertTrue(service.append_share_batch([first, second])) + + self.assertEqual(len(ledger), 2) + for entry in (first, second): + self.assertTrue(entry.committed.is_set()) + self.assertIsNone(entry.error) + self.assertIsNotNone(entry.record) + self.assertIsNone(entry.writer_token) + self.assertEqual(service._pending_share_commit_floor, {}) + self.assertEqual(controller.snapshot()["active_writers"], {}) + + def test_batch_failure_marks_entries_and_still_releases_everything(self) -> None: + class FailingLedger: + def append_batch(self, _entries: object) -> list[object]: + raise RuntimeError("ledger unavailable") + + service, _ledger, controller, _runtime = self._service( + ledger=FailingLedger(), + wall_times=[100, 200], + ) + first = self._entry(self._stamp(service, "aa" * 32)) + second = self._entry(self._stamp(service, "bb" * 32)) + for entry in (first, second): + entry.writer_token = controller.reserve_writer("share_persistence") + + with patch("builtins.print"), patch("traceback.print_exc"): + self.assertFalse(service.append_share_batch([first, second])) + + self.assertEqual(service.share_append_failure_count, 2) + for entry in (first, second): + self.assertTrue(entry.committed.is_set()) + self.assertIsInstance(entry.error, RuntimeError) + self.assertIsNone(entry.writer_token) + self.assertEqual(service._pending_share_commit_floor, {}) + self.assertEqual(controller.snapshot()["active_writers"], {}) + + def test_append_loop_exits_only_when_admission_closed_and_writers_drained( + self, + ) -> None: + service, _ledger, controller, runtime = self._service() + runtime.stop_event.set() + exited = threading.Event() + + def run_loop() -> None: + service.share_append_loop() + exited.set() + + loop = threading.Thread(target=run_loop, daemon=True) + loop.start() + # Admission still open: the stopping loop keeps polling for late + # writer handoffs instead of abandoning queued shares. + self.assertFalse(exited.wait(0.4)) + + token = controller.reserve_writer("share_submission") + controller.begin_shutdown("test shutdown") + # Admission is closed but a submitting writer is still active. + self.assertFalse(exited.wait(0.4)) + + token.finish() + self.assertTrue(exited.wait(2.0)) + loop.join(timeout=2.0) + self.assertFalse(loop.is_alive()) + + # -- recovery journal and typed replay --------------------------------- + + def test_recovered_share_replays_inserted_and_unlinks_clean_journal(self) -> None: + with tempfile.TemporaryDirectory() as tempdir: + path = Path(tempdir) / "recovery.jsonl" + service, ledger, _controller, _runtime = self._service() + service.share_recovery_path = path + pending = self._stamp(service) + with patch("builtins.print"): + service.recover_share_to_disk(self._entry(pending), "test") + self.assertEqual(service.shares_recovered_to_disk, 1) + + with patch("builtins.print"): + self.assertEqual(service.replay_recovered_shares(), 1) + + self.assertFalse(path.exists()) + self.assertEqual(service.shares_replayed, 1) + self.assertEqual(len(ledger), 1) + self.assertEqual( + [record.share_id for record in ledger.all_shares()], + [pending.share_id], + ) + + def test_recovery_exact_existing_is_typed_skip_and_clears_clean_journal( + self, + ) -> None: + with tempfile.TemporaryDirectory() as tempdir: + path = Path(tempdir) / "recovery.jsonl" + service, ledger, _controller, _runtime = self._service() + service.share_recovery_path = path + pending = self._stamp(service) + with patch("builtins.print"): + service.recover_share_to_disk(self._entry(pending), "test") + replay = ledger.append_recovered_share(pending) + self.assertEqual(replay.disposition, "inserted") + + with patch("builtins.print"): + self.assertEqual(service.replay_recovered_shares(), 0) + + self.assertFalse(path.exists()) + self.assertEqual(len(ledger), 1) + + def test_recovery_payload_conflict_is_typed_and_retains_journal(self) -> None: + with tempfile.TemporaryDirectory() as tempdir: + path = Path(tempdir) / "recovery.jsonl" + service, ledger, _controller, _runtime = self._service() + service.share_recovery_path = path + pending = self._stamp(service) + with patch("builtins.print"): + service.recover_share_to_disk(self._entry(pending), "test") + ledger.append_recovered_share( + PendingShare(**{**pending.__dict__, "ntime": pending.ntime + 1}) + ) + + with patch("builtins.print"): + self.assertEqual(service.replay_recovered_shares(), 0) + + self.assertTrue(path.exists()) + self.assertEqual(service.shares_replayed, 0) + self.assertEqual(service.share_replay_conflicts, 1) + + def test_conflicting_row_is_quarantined_and_rest_of_journal_replays(self) -> None: + with tempfile.TemporaryDirectory() as tempdir: + path = Path(tempdir) / "recovery.jsonl" + service, ledger, _controller, _runtime = self._service() + service.share_recovery_path = path + conflicting = self._stamp(service, "ab" * 32) + later_a = self._stamp(service, "ac" * 32) + later_b = self._stamp(service, "ad" * 32) + with patch("builtins.print"): + service.recover_share_to_disk(self._entry(conflicting), "test") + service.recover_share_to_disk(self._entry(later_a), "test") + service.recover_share_to_disk(self._entry(later_b), "test") + # The durable copy of the first row disagrees with the journal + # payload, so its replay raises ShareReplayConflict. + ledger.append_recovered_share( + PendingShare( + **{**conflicting.__dict__, "ntime": conflicting.ntime + 1} + ) + ) + + with patch("builtins.print"): + self.assertEqual(service.replay_recovered_shares(), 2) + + # The rows behind the conflict are credited; the conflicting row + # is quarantined -- counted and preserved in the retained file -- + # instead of stranding the rest of the journal. + self.assertTrue(path.exists()) + self.assertEqual(service.shares_replayed, 2) + self.assertEqual(service.share_replay_conflicts, 1) + self.assertEqual(len(ledger), 3) + replayed_ids = {record.share_id for record in ledger.all_shares()} + self.assertIn(later_a.share_id, replayed_ids) + self.assertIn(later_b.share_id, replayed_ids) + + # Re-running the retained journal double-credits nothing: the + # replayed rows are exact duplicates now and the conflicting row + # conflicts again. + with patch("builtins.print"): + self.assertEqual(service.replay_recovered_shares(), 0) + + self.assertTrue(path.exists()) + self.assertEqual(service.shares_replayed, 2) + self.assertEqual(service.share_replay_conflicts, 2) + self.assertEqual(len(ledger), 3) + + def test_unknown_recovery_disposition_is_conservative_and_retains_journal( + self, + ) -> None: + class FutureLedger: + def append_recovered_share(self, pending: PendingShare) -> ShareReplayResult: + return ShareReplayResult( + "future-disposition", + SimpleNamespace(share_seq=1, share_id=pending.share_id), + ) + + with tempfile.TemporaryDirectory() as tempdir: + path = Path(tempdir) / "recovery.jsonl" + service, _ledger, _controller, _runtime = self._service( + ledger=FutureLedger(), + ) + service.share_recovery_path = path + pending = self._stamp(service) + with patch("builtins.print"): + service.recover_share_to_disk(self._entry(pending), "test") + + with patch("builtins.print"): + self.assertEqual(service.replay_recovered_shares(), 0) + + self.assertTrue(path.exists()) + + def test_untyped_append_only_ledger_keeps_legacy_duplicate_fallback(self) -> None: + class LegacyLedger: + def __init__(self) -> None: + self.appended: list[PendingShare] = [] + + def append(self, pending: PendingShare) -> SimpleNamespace: + for existing in self.appended: + if existing.share_id == pending.share_id: + raise RuntimeError( + f"duplicate share_id: {pending.share_id}" + ) + self.appended.append(pending) + return SimpleNamespace(share_seq=len(self.appended)) + + with tempfile.TemporaryDirectory() as tempdir: + path = Path(tempdir) / "recovery.jsonl" + ledger = LegacyLedger() + service, _ledger, _controller, _runtime = self._service(ledger=ledger) + service.share_recovery_path = path + first = self._stamp(service, "aa" * 32) + duplicate = self._stamp(service, "aa" * 32) + second = self._stamp(service, "bb" * 32) + with patch("builtins.print"): + for pending in (first, duplicate, second): + service.recover_share_to_disk(self._entry(pending), "test") + + with patch("builtins.print"): + self.assertEqual(service.replay_recovered_shares(), 2) + + self.assertFalse(path.exists()) + self.assertEqual( + [pending.share_id for pending in ledger.appended], + [first.share_id, second.share_id], + ) + + def test_torn_journal_line_is_preserved_while_intact_lines_replay(self) -> None: + with tempfile.TemporaryDirectory() as tempdir: + path = Path(tempdir) / "recovery.jsonl" + service, ledger, _controller, _runtime = self._service() + service.share_recovery_path = path + pending = self._stamp(service) + with patch("builtins.print"): + service.recover_share_to_disk(self._entry(pending), "test") + with open(path, "a", encoding="utf-8") as handle: + handle.write('{"share_id": "miner-a:torn", "miner_') + + with patch("builtins.print"), patch("traceback.print_exc"): + self.assertEqual(service.replay_recovered_shares(), 1) + + self.assertTrue(path.exists()) + self.assertEqual(len(ledger), 1) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_prism_singleflight_exception_retention.py b/tests/test_prism_singleflight_exception_retention.py new file mode 100644 index 000000000..f64efb62e --- /dev/null +++ b/tests/test_prism_singleflight_exception_retention.py @@ -0,0 +1,571 @@ +"""Failed singleflight owners must not pin their flight until cyclic GC (#251). + +``PublicResponseCache`` and ``P2mrAddressValidator`` record an owner's failure +on the in-flight entry so coalesced waiters can re-raise it. The recorded +exception carries a traceback, the traceback holds the owner's frame, and a +frame that still names the flight closes the cycle +``flight -> exception -> traceback -> frame -> flight``. Popping the flight +from its map removes the only external reference, but nothing in that cycle +is freed by reference counting: the failed origin/RPC frame and every local it +holds stay allocated until a cyclic collection happens to run. + +These tests exercise the real owner methods with automatic collection +disabled, so an object that is still alive when asserted is being held by a +cycle, not by a pending collection. Weak references are the only handles +kept: no ``f_locals`` or referrer inspection, which would itself retain the +frames under test. Threads are ordered with events and barriers, never +sleeps, and every wait is bounded. Test-only GC policy is restored on exit. +""" + +from __future__ import annotations + +import gc +import threading +import unittest +import weakref +from collections.abc import Callable, Iterator +from contextlib import contextmanager +from typing import Any + +from lab.prism.public_api import PublicResponseCache, _PublicInflight +from lab.prism.stratum_session import P2mrAddressValidator, StratumError + +CACHE_KEY: tuple[str, tuple[tuple[str, tuple[str, ...]], ...]] = ("/issue-251", ()) +ADDRESS = "qbit1issue251payoutaddress" +P2MR_SCRIPT = "5220" + "ab" * 32 +WAITER_COUNT = 3 +WAIT_SECONDS = 5.0 + + +class Payload: + """Weak-referenceable stand-in for a failed origin call's locals.""" + + +@contextmanager +def _without_cyclic_gc() -> Iterator[None]: + """Leave only reference counting to free objects inside the block. + + Assertions inside run before any ``gc.collect()``; whatever is still alive + there is held by a reference cycle. The prior policy is restored on exit, + after one full collection reclaims anything a failing case left behind. + """ + gc.collect() + was_enabled = gc.isenabled() + gc.disable() + try: + yield + finally: + gc.collect() + if was_enabled: + gc.enable() + + +def _traceback_names(error: BaseException) -> list[str]: + """Function names along ``error.__traceback__``, outermost first. + + Reads code objects only, never ``f_locals``, so checking that the + diagnostics survived does not itself keep the frames alive. + """ + names: list[str] = [] + traceback = error.__traceback__ + while traceback is not None: + names.append(traceback.tb_frame.f_code.co_name) + traceback = traceback.tb_next + return names + + +class _Origin: + """A ``compute()`` / ``rpc_call`` that creates a weak-referenceable local. + + With ``park=True`` the call blocks until ``release`` is set, so concurrent + requests can register as waiters while the owner is in flight. Failing + calls raise after creating the local; successful calls return ``success``. + """ + + def __init__(self, *, park: bool = False, success: object | None = None) -> None: + self.park = park + self.success = success + self.calls = 0 + self.entered = threading.Event() + self.release = threading.Event() + self.payloads: list[weakref.ReferenceType[Payload]] = [] + self._lock = threading.Lock() + + def __call__(self, *args: object) -> object: + payload = Payload() + with self._lock: + self.calls += 1 + self.payloads.append(weakref.ref(payload)) + self.entered.set() + if self.park and not self.release.wait(timeout=WAIT_SECONDS): + raise TimeoutError("test did not release the parked origin call") + if self.success is not None: + return self.success + raise RuntimeError("injected origin failure") + + +class _GatedEvent(threading.Event): + """Flight event whose waiters announce themselves and can be held. + + A waiter reaching the singleflight's ``event.wait()`` joins ``arrived`` + (the test thread is the extra party) and then blocks on ``proceed`` before + entering the real wait. The test therefore knows when every waiter is + inside the owner method and decides whether the owner finishes before or + after they observe the recorded result. + + The real wait is this class's own condition variable rather than the + inherited one, so the guarantee below depends on nothing private to the + standard library. Each waiter counts itself in ``all_inside`` while + holding the condition's lock and only releases that lock by entering + ``Condition.wait``; ``set()`` needs the same lock. Once ``all_inside`` is + set, every waiter is therefore blocked in the real wait before any + ``set()`` can complete. Waits are bounded: a flight that is never set + fails the test instead of hanging it. + """ + + def __init__(self, *, waiters: int) -> None: + super().__init__() + self.arrived = threading.Barrier(waiters + 1) + self.proceed = threading.Event() + self.all_inside = threading.Event() + self._expected_waiters = waiters + self._waiting = 0 + self._set = False + self._condition = threading.Condition() + + def is_set(self) -> bool: + with self._condition: + return self._set + + def set(self) -> None: + with self._condition: + self._set = True + self._condition.notify_all() + + def clear(self) -> None: + with self._condition: + self._set = False + + def wait(self, timeout: float | None = None) -> bool: + # The owner methods wait without a timeout; bound it here so a + # regression that forgets to wake waiters fails this call instead of + # leaving non-daemon test threads blocked after the join times out. + bound = WAIT_SECONDS if timeout is None else min(timeout, WAIT_SECONDS) + self.arrived.wait(timeout=WAIT_SECONDS) + if not self.proceed.wait(timeout=WAIT_SECONDS): + raise TimeoutError("test did not release the gated waiters") + with self._condition: + self._waiting += 1 + if self._waiting == self._expected_waiters: + self.all_inside.set() + if not self._set and not self._condition.wait(timeout=bound): + raise TimeoutError(f"the flight was not set within {bound}s") + return True + + +class _InflightMap(dict[Any, Any]): + """Inflight map that weakly records each registered flight. + + When ``event`` is given, the flight's event is swapped for it at + registration, which is the only point where the owner method exposes the + flight it just created. + """ + + def __init__(self, *, event: threading.Event | None = None) -> None: + super().__init__() + self.event = event + self.flights: list[weakref.ReferenceType[Any]] = [] + + def __setitem__(self, key: Any, flight: Any) -> None: + if self.event is not None: + flight.event = self.event + self.flights.append(weakref.ref(flight)) + super().__setitem__(key, flight) + + +class _ReleaseAssertions(unittest.TestCase): + def assert_released(self, refs: list[weakref.ReferenceType[Any]], what: str) -> None: + self.assertTrue(refs, f"no {what} was ever registered") + self.assertEqual( + [None] * len(refs), + [ref() for ref in refs], + f"{what} still alive without a cyclic collection", + ) + + def observe_failure(self, call: Callable[[], object]) -> list[str]: + """Run ``call``, require the injected failure, and return its frames. + + The exception is dropped before returning, exactly as a request + handler that logged and answered the error would drop it. + """ + try: + call() + except RuntimeError as error: + self.assertEqual(str(error), "injected origin failure") + return _traceback_names(error) + self.fail("expected the injected origin failure") + + def assert_shared_failure( + self, + outcomes: list[tuple[str, BaseException | None]], + *, + chained: bool, + ) -> None: + """Check every thread saw the injected failure with diagnostics intact. + + Runs in its own frame so no local keeps an exception, and with it the + failed origin frame, alive once the caller clears ``outcomes``. + """ + leader_errors = [error for name, error in outcomes if name == "owner"] + waiter_errors = [error for name, error in outcomes if name != "owner"] + self.assertEqual(len(leader_errors), 1) + self.assertEqual(len(waiter_errors), WAITER_COUNT) + leader_error = leader_errors[0] + self.assertIsInstance(leader_error, RuntimeError) + self.assertEqual(str(leader_error), "injected origin failure") + self.assertEqual(_traceback_names(leader_error)[-1], "__call__") + for waiter_error in waiter_errors: + self.assertIsInstance(waiter_error, RuntimeError) + self.assertEqual(str(waiter_error), "injected origin failure") + if chained: + # The documented wrapper: a fresh error chained to the leader's + # original, which still carries the RPC frame. + self.assertIs(waiter_error.__cause__, leader_error) + else: + # The owner's own exception object, re-raised by each waiter; + # its traceback grew by the waiter frames and kept the origin. + self.assertIs(waiter_error, leader_error) + + def run_concurrently( + self, + request: Callable[[], object], + *, + origin: _Origin, + gate: _GatedEvent, + owner_finishes_first: bool, + flight_released: Callable[[], None], + waiters_arrived: Callable[[], None] | None = None, + ) -> list[tuple[str, BaseException | None]]: + """Run one owner and WAITER_COUNT coalesced waiters through ``request``. + + Returns ``(thread name, error)`` per thread, ``None`` for a normal + return. The owner is parked inside the origin call until every waiter + has entered the flight's ``event.wait()``, when ``waiters_arrived`` + runs. With ``owner_finishes_first`` the owner completes, drops out of + the inflight map (checked by ``flight_released``), and unwinds + entirely before any waiter reads the recorded outcome. Otherwise the + owner is released only after every waiter is blocked in the real + wait, which the gate guarantees rather than merely makes likely. + """ + outcomes: list[tuple[str, BaseException | None]] = [] + outcomes_lock = threading.Lock() + + def run() -> None: + error: BaseException | None = None + try: + request() + except BaseException as exc: + error = exc + with outcomes_lock: + outcomes.append((threading.current_thread().name, error)) + del error + + owner = threading.Thread(target=run, name="owner") + waiters = [ + threading.Thread(target=run, name=f"waiter-{index}") for index in range(WAITER_COUNT) + ] + started: list[threading.Thread] = [] + try: + owner.start() + started.append(owner) + self.assertTrue(origin.entered.wait(timeout=WAIT_SECONDS)) + for waiter in waiters: + waiter.start() + started.append(waiter) + gate.arrived.wait(timeout=WAIT_SECONDS) + if waiters_arrived is not None: + waiters_arrived() + if owner_finishes_first: + origin.release.set() + owner.join(timeout=WAIT_SECONDS) + self.assertFalse(owner.is_alive()) + flight_released() + gate.proceed.set() + else: + gate.proceed.set() + self.assertTrue(gate.all_inside.wait(timeout=WAIT_SECONDS)) + origin.release.set() + finally: + # Unblock whatever is still parked, then join only the threads + # that were actually started so an earlier assertion failure is + # reported as itself rather than masked by joining unstarted ones. + origin.release.set() + gate.proceed.set() + for thread in started: + thread.join(timeout=WAIT_SECONDS) + self.assertEqual(len(started), 1 + WAITER_COUNT) + self.assertFalse(any(thread.is_alive() for thread in started)) + self.assertEqual(origin.calls, 1) + self.assertEqual(len(outcomes), 1 + WAITER_COUNT) + return outcomes + + +class PublicResponseCacheFailedFlightTests(_ReleaseAssertions): + """Foreground owners, background refreshes, and waiters release the flight.""" + + def lookup(self, cache: PublicResponseCache, compute: Callable[[], Any]) -> Any: + return cache.get_or_compute(key=CACHE_KEY, ttl_seconds=60, compute=compute) + + def test_foreground_owner_failure_releases_flight_and_compute_frame(self) -> None: + origin = _Origin() + with _without_cyclic_gc(): + cache = PublicResponseCache() + flights = _InflightMap() + cache._inflight = flights + + frames = self.observe_failure(lambda: self.lookup(cache, origin)) + + self.assertNotIn(CACHE_KEY, cache._inflight) + self.assertEqual(origin.calls, 1) + # The owner re-raised its own exception with the origin frame in + # the traceback; the diagnostics were not trimmed to break the cycle. + self.assertEqual(frames[-2:], ["get_or_compute", "__call__"]) + self.assert_released(flights.flights, "foreground flight") + self.assert_released(origin.payloads, "failed compute frame local") + + def test_background_refresh_failure_releases_flight_and_compute_frame(self) -> None: + origin = _Origin() + with _without_cyclic_gc(): + cache = PublicResponseCache() + flights = _InflightMap() + cache._inflight = flights + flight = _PublicInflight(event=threading.Event()) + cache._inflight[CACHE_KEY] = flight + + cache._refresh_entry(CACHE_KEY, 60, 30, origin, flight) + + self.assertNotIn(CACHE_KEY, cache._inflight) + self.assertTrue(flight.event.is_set()) + self.assertIsInstance(flight.exception, RuntimeError) + self.assertIsNone(flight.result) + # The swallowed error keeps its origin frame for whoever coalesced. + self.assertEqual( + _traceback_names(flight.exception)[-2:], ["_refresh_entry", "__call__"] + ) + del flight + self.assert_released(flights.flights, "background refresh flight") + self.assert_released(origin.payloads, "failed refresh compute frame local") + + def test_stale_hit_refresh_thread_failure_releases_flight_and_keeps_entry(self) -> None: + origin = _Origin() + with _without_cyclic_gc(): + cache = PublicResponseCache() + flights = _InflightMap() + cache._inflight = flights + status, payload, state, _age = cache.get_or_compute( + key=CACHE_KEY, ttl_seconds=60, compute=lambda: (200, {"v": "stale"}) + ) + self.assertEqual((200, {"v": "stale"}, "MISS"), (status, payload, state)) + entry = cache._entries[CACHE_KEY] + entry.expires_at = entry.stored_at - 1 # expired, inside the window below + entry.stale_until = entry.expires_at + 30 + del entry + + status, payload, state, _age = cache.get_or_compute( + key=CACHE_KEY, + ttl_seconds=60, + compute=origin, + stale_while_revalidate_seconds=30, + ) + + self.assertEqual((200, {"v": "stale"}, "STALE"), (status, payload, state)) + self.assertTrue(origin.entered.wait(timeout=WAIT_SECONDS)) + refreshers = [ + thread + for thread in threading.enumerate() + if thread.name == "prism-public-cache-refresh" + ] + for thread in refreshers: + thread.join(timeout=WAIT_SECONDS) + self.assertFalse(any(thread.is_alive() for thread in refreshers)) + self.assertNotIn(CACHE_KEY, cache._inflight) + self.assertIn(CACHE_KEY, cache._entries) # the failure deleted nothing + self.assertEqual(origin.calls, 1) + self.assert_released(flights.flights, "stale refresh flight") + self.assert_released(origin.payloads, "failed refresh compute frame local") + + def assert_waiters_release_failed_flight(self, *, owner_finishes_first: bool) -> None: + origin = _Origin(park=True) + gate = _GatedEvent(waiters=WAITER_COUNT) + with _without_cyclic_gc(): + cache = PublicResponseCache() + flights = _InflightMap(event=gate) + cache._inflight = flights + + outcomes = self.run_concurrently( + lambda: self.lookup(cache, origin), + origin=origin, + gate=gate, + owner_finishes_first=owner_finishes_first, + flight_released=lambda: self.assertNotIn(CACHE_KEY, cache._inflight), + ) + + self.assertNotIn(CACHE_KEY, cache._inflight) + self.assert_shared_failure(outcomes, chained=False) + outcomes.clear() + self.assert_released(flights.flights, "shared flight") + self.assert_released(origin.payloads, "failed compute frame local") + + def test_waiters_release_flight_when_owner_fails_while_they_wait(self) -> None: + self.assert_waiters_release_failed_flight(owner_finishes_first=False) + + def test_waiters_release_flight_when_owner_failed_before_they_observe(self) -> None: + self.assert_waiters_release_failed_flight(owner_finishes_first=True) + + def test_successful_flight_is_released_after_the_last_observer(self) -> None: + origin = _Origin(park=True, success=(200, {"v": "fresh"})) + gate = _GatedEvent(waiters=WAITER_COUNT) + with _without_cyclic_gc(): + cache = PublicResponseCache() + flights = _InflightMap(event=gate) + cache._inflight = flights + results: list[Any] = [] + results_lock = threading.Lock() + + def request() -> None: + result = self.lookup(cache, origin) + with results_lock: + results.append(result) + + outcomes = self.run_concurrently( + request, + origin=origin, + gate=gate, + owner_finishes_first=True, + flight_released=lambda: self.assertNotIn(CACHE_KEY, cache._inflight), + ) + + self.assertEqual([error for _name, error in outcomes], [None] * (1 + WAITER_COUNT)) + self.assertEqual(len(results), 1 + WAITER_COUNT) + self.assertEqual( + {(status, state) for status, _payload, state, _age in results}, {(200, "MISS")} + ) + self.assertEqual(cache._entries[CACHE_KEY].payload, {"v": "fresh"}) + self.assert_released(flights.flights, "successful flight") + + +class P2mrAddressValidatorFailedFlightTests(_ReleaseAssertions): + """The validation leader and its waiters release the pending flight.""" + + def validator(self, rpc_call: Callable[..., object], flights: _InflightMap) -> P2mrAddressValidator: + return P2mrAddressValidator( + rpc_call=rpc_call, + max_entries=lambda: 4, + ttl_seconds=lambda: 60.0, + inflight=flights, + ) + + def test_leader_rpc_failure_releases_flight_and_rpc_frame(self) -> None: + origin = _Origin() + with _without_cyclic_gc(): + flights = _InflightMap() + validator = self.validator(origin, flights) + + frames = self.observe_failure(lambda: validator.validate(ADDRESS, label="test")) + + self.assertEqual(validator.inflight, {}) + self.assertEqual(validator.cache, {}) + self.assertEqual(origin.calls, 1) + self.assertEqual(frames[-2:], ["validate", "__call__"]) + self.assert_released(flights.flights, "validation flight") + self.assert_released(origin.payloads, "failed RPC frame local") + + def test_leader_validation_failure_releases_flight(self) -> None: + with _without_cyclic_gc(): + flights = _InflightMap() + validator = self.validator(lambda *_args: {"isvalid": False}, flights) + + try: + validator.validate(ADDRESS, label="payout") + except StratumError as error: + self.assertEqual(error.code, 20) + self.assertIn("payout is not a valid qbit address", error.message) + else: + self.fail("expected the address to be rejected") + + self.assertEqual(validator.inflight, {}) + self.assert_released(flights.flights, "rejected validation flight") + + def assert_waiters_release_failed_flight(self, *, owner_finishes_first: bool) -> None: + origin = _Origin(park=True) + gate = _GatedEvent(waiters=WAITER_COUNT) + with _without_cyclic_gc(): + flights = _InflightMap(event=gate) + validator = self.validator(origin, flights) + + def flight_released() -> None: + with validator.cache_lock: + self.assertNotIn(ADDRESS, validator.inflight) + + def waiters_counted() -> None: + # The existing accounting contract: every coalesced request + # is counted on the leader's flight while it is in flight. + with validator.cache_lock: + self.assertEqual(validator.inflight[ADDRESS].waiters, WAITER_COUNT) + + outcomes = self.run_concurrently( + lambda: validator.validate(ADDRESS, label="test"), + origin=origin, + gate=gate, + owner_finishes_first=owner_finishes_first, + flight_released=flight_released, + waiters_arrived=waiters_counted, + ) + + self.assertEqual(validator.inflight, {}) + self.assertEqual(validator.cache, {}) + self.assert_shared_failure(outcomes, chained=True) + outcomes.clear() + self.assert_released(flights.flights, "shared validation flight") + self.assert_released(origin.payloads, "failed RPC frame local") + + def test_waiters_release_flight_when_leader_fails_while_they_wait(self) -> None: + self.assert_waiters_release_failed_flight(owner_finishes_first=False) + + def test_waiters_release_flight_when_leader_failed_before_they_observe(self) -> None: + self.assert_waiters_release_failed_flight(owner_finishes_first=True) + + def test_successful_flight_is_released_after_the_last_observer(self) -> None: + origin = _Origin(park=True, success={"isvalid": True, "scriptPubKey": P2MR_SCRIPT}) + gate = _GatedEvent(waiters=WAITER_COUNT) + with _without_cyclic_gc(): + flights = _InflightMap(event=gate) + validator = self.validator(origin, flights) + results: list[tuple[str, str]] = [] + results_lock = threading.Lock() + + def request() -> None: + result = validator.validate(ADDRESS, label="test") + with results_lock: + results.append(result) + + def flight_released() -> None: + with validator.cache_lock: + self.assertNotIn(ADDRESS, validator.inflight) + + outcomes = self.run_concurrently( + request, + origin=origin, + gate=gate, + owner_finishes_first=True, + flight_released=flight_released, + ) + + self.assertEqual([error for _name, error in outcomes], [None] * (1 + WAITER_COUNT)) + self.assertEqual(results, [(P2MR_SCRIPT, P2MR_SCRIPT[4:])] * (1 + WAITER_COUNT)) + self.assertIn(ADDRESS, validator.cache) + self.assert_released(flights.flights, "successful validation flight") + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_prism_statement_spool.py b/tests/test_prism_statement_spool.py new file mode 100644 index 000000000..12d433fc7 --- /dev/null +++ b/tests/test_prism_statement_spool.py @@ -0,0 +1,106 @@ +"""Native-ledger finalization transport: bytes, deadlines, and exact SQL.""" + +from __future__ import annotations + +import os +import subprocess +import threading +import time +import tracemalloc +import unittest +from unittest import mock +import uuid + +from lab.prism.share_ledger import LedgerOperationTimeout, PsqlShareLedger, WRITER_LEASE_HEARTBEAT_SESSION_PREFIX +from lab.prism.statement_spool import run_fenced_statement + + +@unittest.skipUnless(os.environ.get("PRISM_TEST_DATABASE_URL"), "requires disposable PostgreSQL") +class StatementSpoolPostgresTests(unittest.TestCase): + def setUp(self): + self.ledger = PsqlShareLedger( + database_url=os.environ["PRISM_TEST_DATABASE_URL"], psql_command="psql", + native_client_mode="native", writer_id="candidate-statement-tests", + writer_session_token=WRITER_LEASE_HEARTBEAT_SESSION_PREFIX + uuid.uuid4().hex, + ) + self.addCleanup(self.ledger.close) + self.addCleanup(self.ledger.release_writer_lease) + + def test_large_statement_never_enters_coordinator_native_buffer(self): + def pieces(): + yield "SELECT json_build_object('length', octet_length($payload$" + for _ in range(4096): + yield "x" * 4096 + yield "$payload$::text));" + + tracemalloc.start() + try: + with mock.patch.object(self.ledger._native, "run_json", side_effect=AssertionError("whole query")): + result = run_fenced_statement(self.ledger, pieces()) + _, peak = tracemalloc.get_traced_memory() + finally: + tracemalloc.stop() + self.assertEqual(result, {"length": 16 * 1024 * 1024}) + self.assertLess(peak, 3 * 1024 * 1024) + + def test_server_deadline_releases_gate_and_allows_retry(self): + with self.ledger.operation_timeout(0.15): + with self.assertRaises(LedgerOperationTimeout): + run_fenced_statement(self.ledger, ["SELECT pg_sleep(10);"]) + self.assertTrue(self.ledger._lock.acquire(blocking=False)) + self.ledger._lock.release() + self.assertEqual(run_fenced_statement(self.ledger, ["SELECT '{\"ok\":true}'::json;"]), {"ok": True}) + + def test_spooled_connection_matches_pool_options_and_keeps_password_out_of_argv(self): + from psycopg.conninfo import conninfo_to_dict + + sql = "SELECT json_build_object('schema', current_setting('search_path'), 'work_mem', current_setting('work_mem'), 'app', current_setting('application_name'));" + expected = self.ledger._run_json(sql) + with mock.patch.dict(os.environ, {"PGOPTIONS": "-c search_path=wrong_schema -c work_mem=1MB", "PGAPPNAME": "foreign"}), \ + mock.patch("lab.prism.statement_spool.subprocess.Popen", wraps=subprocess.Popen) as launch: + self.assertEqual(run_fenced_statement(self.ledger, [sql]), expected) + command = launch.call_args.args[0] + dsn = command[command.index("--dbname") + 1] + self.assertNotIn("password", conninfo_to_dict(dsn)) + + def test_expired_lease_attributes_spooled_write_to_its_owner(self): + self.ledger._run_json("UPDATE qbit_ledger_writer_lease SET lease_expires_at = clock_timestamp() - interval '1 second' RETURNING '{}'::json;") + results, errors = [], [] + + def write(): + try: + results.append(run_fenced_statement(self.ledger, [ + "BEGIN; UPDATE qbit_ledger_writer_lease SET lease_expires_at = clock_timestamp() + interval '60 seconds'; " + "SELECT json_build_object('ok', true) FROM pg_sleep(2); COMMIT;", + ])) + except Exception as exc: + errors.append(exc) + + thread = threading.Thread(target=write) + thread.start() + deadline = time.monotonic() + 5 + locked = False + try: + while thread.is_alive() and time.monotonic() < deadline: + locked = self.ledger._run_json(""" +SELECT json_build_object('locked', EXISTS ( + SELECT 1 FROM qbit_ledger_writer_lease lease, pg_stat_activity activity + WHERE activity.backend_xid = lease.xmax AND activity.backend_xid IS NOT NULL +)); +""")["locked"] + if locked: + break + time.sleep(0.01) + self.assertTrue(locked) + proof = self.ledger.verify_writer_lease_guard_session() + self.assertTrue(proof["renewal_deferred_to_own_write"]) + finally: + thread.join(10) + self.assertFalse(thread.is_alive()) + self.assertEqual(errors, []) + self.assertEqual(results, [{"ok": True}]) + self.assertFalse(self.ledger.verify_writer_lease_guard_session()["renewal_deferred_to_own_write"]) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_prism_stratum_session.py b/tests/test_prism_stratum_session.py new file mode 100644 index 000000000..e575db550 --- /dev/null +++ b/tests/test_prism_stratum_session.py @@ -0,0 +1,507 @@ +#!/usr/bin/env python3 +"""Deterministic tests for the extracted PRISM Stratum session boundary. + +The current service resolves the coordinator through one call-time +``SessionRuntime`` typed port (not the old per-seam collaborator objects), so +these tests drive :class:`StratumSessionService` with a recording runtime and +exercise :class:`SessionRegistry` directly where the contract is registry-owned. +""" + +from __future__ import annotations + +import socket +import threading +import unittest +from decimal import Decimal +from types import SimpleNamespace +from unittest.mock import patch + +from lab.auxpow import vardiff +from lab.prism import stratum_session +from lab.prism.coordinator_config import StratumListenerProfile +from lab.prism.prism_coordinator import ( + ClientState as FacadeClientState, + PrismCoordinator, + StratumError as FacadeStratumError, + WorkerIdentity as FacadeWorkerIdentity, +) +from lab.prism.stratum_session import ( + ClientState, + SessionRegistry, + StratumError, + StratumSessionService, + WorkerIdentity, + client_can_receive_jobs, + error_payload, + parse_stratum_password_options, + result_payload, +) + + +def listener() -> StratumListenerProfile: + config = vardiff.VardiffConfig( + enabled=False, + target_share_interval_seconds=Decimal("15"), + min_difficulty=Decimal("1"), + max_difficulty=Decimal("1024"), + retarget_interval_seconds=Decimal("90"), + max_step_factor=Decimal("4"), + startup_difficulty=Decimal("1"), + max_step_down_factor=Decimal("4"), + ewma_alpha=Decimal("0.4"), + retarget_tolerance=Decimal("0.25"), + ) + return StratumListenerProfile( + name="default", + bind="127.0.0.1", + port=3340, + share_difficulty=Decimal("1"), + vardiff_config=config, + heartbeat_name="stratum_accept", + ) + + +def worker(username: str) -> WorkerIdentity: + return WorkerIdentity( + username=username, + payout_address=username, + worker_name=None, + script_pubkey_hex="5220" + "11" * 32, + p2mr_program_hex="11" * 32, + ) + + +class FakeSocket: + def __init__(self) -> None: + self.closed = threading.Event() + self.sent: list[bytes] = [] + + def settimeout(self, _timeout: object) -> None: + return + + def setsockopt(self, *_args: object) -> None: + return + + def shutdown(self, _how: object) -> None: + self.closed.set() + + def close(self) -> None: + self.closed.set() + + def sendall(self, data: bytes) -> None: + self.sent.append(data) + + +def client_state(connection_id: int) -> ClientState: + return ClientState( + FakeSocket(), # type: ignore[arg-type] + ("127.0.0.1", connection_id), + connection_id, + f"{connection_id:08x}", + ) + + +class FakeProgressService: + """G1 stand-in that records callback-time registry proof visibility.""" + + def __init__(self, registry: SessionRegistry) -> None: + self.registry = registry + self.proofs: list[object] = [] + self.registry_had_proof_at_callback: list[bool] = [] + + def record_delivery( + self, + proof: object, + ready_mode_required: bool, + *, + matches_latest_tip: bool, + ) -> bool: + snapshot = self.registry.eligible_snapshot() + delivered = snapshot.get(proof.connection_id) # type: ignore[union-attr] + self.registry_had_proof_at_callback.append( + delivered is not None and delivered.delivered is not None + ) + self.proofs.append(proof) + return True + + +class FakeSessionRuntime: + """Recording call-time SessionRuntime port for the session service.""" + + def __init__(self) -> None: + self.lock = threading.RLock() + self.stop_event = threading.Event() + self.stratum_max_connections = 8 + self.stratum_max_connections_per_username = 1 + self._pool_ready_latched = False + self.latest_detected_tip = None + self.rpc = SimpleNamespace( + call=lambda _method, _params: { + "isvalid": True, + "scriptPubKey": "5220" + "11" * 32, + } + ) + self.service: StratumSessionService | None = None + self.progress: FakeProgressService | None = None + self.cancelled: list[ClientState] = [] + self.cleaned: list[ClientState] = [] + self.retained = 0 + self.activity: list[float] = [] + self.reconciles: list[float] = [] + self.coverage: list[object] = [] + self.send_timeout_error: BaseException | None = None + self._controller = SimpleNamespace(phase="running") + + # -- lifecycle plumbing ------------------------------------------------- + + def _record_heartbeat(self, _name: str, *, phase: str | None = None) -> None: + return None + + def _ensure_shutdown_controller(self) -> object: + return self._controller + + def _ensure_initial_job_state(self) -> None: + return None + + def client_startup_difficulty(self, _profile: StratumListenerProfile) -> Decimal: + return Decimal("1") + + def apply_stratum_send_timeout(self, _sock: object) -> None: + if self.send_timeout_error is not None: + raise self.send_timeout_error + + def handle_client(self, _client: ClientState) -> None: + return None + + def disconnect_client(self, client: ClientState) -> None: + assert self.service is not None + self.service.disconnect_client(client) + + def _cancel_pending_initial_job_locked( + self, client: ClientState, *, count: bool + ) -> None: + self.cancelled.append(client) + + def _ensure_job_delivery_service(self) -> object: + return SimpleNamespace(cleanup_disconnected_client=self.cleaned.append) + + def _retain_current_collection_refresh_if_unrepresented(self) -> None: + self.retained += 1 + + def _record_stratum_resource_exhaustion(self, **_kwargs: object) -> None: + return None + + def _wait_after_stratum_resource_failure(self, _heartbeat_name: str) -> None: + return None + + # -- delivery/progress plumbing ---------------------------------------- + + def _ensure_job_cache_state(self) -> None: + return None + + def _ensure_progress_health_service(self) -> object: + assert self.progress is not None + return self.progress + + def _tip_refresh_epoch_coverage_reached_locked( + self, + _client: ClientState, + _context: object, + _delivered_monotonic: float, + ) -> list[tuple[str, float]]: + return [] + + def _progress_note_refresh_activity(self, delivered_monotonic: float) -> None: + self.activity.append(delivered_monotonic) + + def _progress_reconcile_pending(self, *, now: float) -> None: + self.reconciles.append(now) + + def _record_tip_refresh_epoch_coverage(self, coverage: object) -> None: + self.coverage.append(coverage) + + def _handle_request(self, _client: ClientState, _request: dict[str, object]) -> None: + raise _DispatchShutdown("writer admission closed") + + +class _DispatchShutdown(RuntimeError): + pass + + +def service_fixture() -> tuple[StratumSessionService, SessionRegistry, FakeSessionRuntime]: + runtime = FakeSessionRuntime() + service = StratumSessionService( + runtime, + shutdown_error=_DispatchShutdown, + pool_closed_reason="pool-closed", + ) + runtime.service = service + runtime.progress = FakeProgressService(service.registry) + return service, service.registry, runtime + + +class SessionRegistryTests(unittest.TestCase): + def test_registry_adopts_seeded_clients_and_derives_generation_and_counts( + self, + ) -> None: + early = client_state(3) + late = client_state(7) + late.handler_thread_registered = True + + registry = SessionRegistry(clients={early, late}) + + self.assertEqual(registry.connection_generation, 7) + self.assertEqual(registry.handler_thread_count, 1) + self.assertEqual(registry.peak_active_connections, 2) + self.assertEqual(registry.note_rejection_locked("global"), 1) + self.assertEqual(registry.note_rejection_locked("global"), 2) + self.assertEqual(registry.rejection_counts["global"], 2) + + def test_eligibility_snapshot_is_immutable_exact_and_uses_delivered_context( + self, + ) -> None: + current = client_state(1) + current.subscribed = current.authorized = True + current.worker = worker("miner") + current.username = "miner" + idle = client_state(2) + registry = SessionRegistry(clients=[current, idle]) + delivered = SimpleNamespace(name="delivered") + current.active_job = SimpleNamespace(name="registered-before-send") + registry.record_delivery_locked(current, delivered, 42.0) + current.active_job = SimpleNamespace(name="newer-unsent") + + snapshot = registry.eligible_snapshot() + + self.assertEqual(tuple(snapshot), (1,)) + self.assertIs(snapshot[1].delivered.context, delivered) # type: ignore[union-attr] + self.assertTrue(client_can_receive_jobs(current)) + with self.assertRaises(TypeError): + snapshot[3] = snapshot[1] # type: ignore[index] + + # Reauthorization does not erase a valid socket-delivery proof. + current.worker = worker("replacement") + self.assertIs( + registry.eligible_snapshot()[1].delivered.context, # type: ignore[union-attr] + delivered, + ) + registry.clients = set(registry.clients) + self.assertTrue(registry.begin_retirement_locked(current)) + self.assertFalse(registry.eligible_snapshot()) + + def test_retirement_claim_is_single_owner_and_clears_delivery_proof(self) -> None: + state = client_state(1) + registry = SessionRegistry(clients={state}) + registry.record_delivery_locked(state, SimpleNamespace(name="sent"), 1.0) + self.assertIn(1, registry._delivered_by_connection) + + self.assertTrue(registry.begin_retirement_locked(state)) + self.assertNotIn(state, registry.clients) + self.assertEqual(registry._delivered_by_connection, {}) + self.assertFalse(registry.begin_retirement_locked(state)) + + def test_ordered_membership_can_be_adopted_before_and_after_use(self) -> None: + first = client_state(1) + second = client_state(2) + registry = SessionRegistry(clients=[first, second]) + self.assertIsInstance(registry.clients, list) + self.assertEqual(registry.clients, [first, second]) + + replacement = [second, first] + registry.adopt_clients(replacement) + + self.assertIs(registry.clients, replacement) + self.assertEqual(registry.clients, [second, first]) + + def test_coordinator_registry_adopts_ordered_membership_replacement(self) -> None: + first = client_state(1) + second = client_state(2) + coordinator = PrismCoordinator.__new__(PrismCoordinator) + coordinator.lock = threading.RLock() + coordinator.connection_counter = 0 + coordinator.clients = [first, second] # type: ignore[assignment] + + registry = coordinator._ensure_stratum_session_service().registry + self.assertIs(registry.clients, coordinator.clients) + self.assertEqual(registry.connection_generation, 2) + first.handler_thread_registered = True + replacement = [second, first] + coordinator.clients = replacement # type: ignore[assignment] + + self.assertIs( + coordinator._ensure_stratum_session_service().registry, registry + ) + self.assertIs(registry.clients, replacement) + self.assertEqual(registry.clients, [second, first]) + self.assertEqual(registry.handler_thread_count, 1) + self.assertEqual(coordinator.connection_counter, 2) + self.assertEqual(coordinator.handler_thread_count, 1) + + +class SessionLifecycleTests(unittest.TestCase): + def test_username_reservation_enforces_capacity_and_preserves_identity( + self, + ) -> None: + service, registry, _runtime = service_fixture() + first = client_state(1) + occupant = client_state(2) + registry.clients.update({first, occupant}) + original = worker("original") + full = worker("full") + + self.assertTrue(service.reserve_client_username(first, original)) + self.assertTrue(service.reserve_client_username(occupant, full)) + + with patch("builtins.print"): + accepted = service.reserve_client_username(first, full) + + self.assertFalse(accepted) + self.assertIs(first.worker, original) + self.assertEqual(first.username, "original") + self.assertEqual(registry.rejection_counts["username"], 1) + + def test_handler_setup_failure_rolls_back_membership_and_socket(self) -> None: + service, registry, runtime = service_fixture() + runtime.send_timeout_error = RuntimeError("socket setup unavailable") + accepted = FakeSocket() + + class Listener: + calls = 0 + + def accept(self) -> tuple[FakeSocket, tuple[str, int]]: + self.calls += 1 + if self.calls == 1: + return accepted, ("127.0.0.1", 1) + runtime.stop_event.set() + raise socket.timeout + + with patch("builtins.print"): + service.accept_loop(Listener(), listener()) # type: ignore[arg-type] + + self.assertTrue(accepted.closed.is_set()) + self.assertFalse(registry.clients) + self.assertEqual(registry.handler_thread_count, 0) + self.assertEqual(service.connection_setup_failure_count, 1) + + def test_disconnect_closes_socket_before_waiting_for_job_update_lock(self) -> None: + service, registry, runtime = service_fixture() + state = client_state(1) + sock = state.sock + registry.clients.add(state) + state.job_update_lock.acquire() + finished = threading.Event() + + def disconnect() -> None: + service.disconnect_client(state) + finished.set() + + thread = threading.Thread(target=disconnect) + thread.start() + self.assertTrue(sock.closed.wait(1.0)) # type: ignore[union-attr] + self.assertFalse(finished.is_set()) + self.assertNotIn(state, registry.clients) + state.job_update_lock.release() + thread.join(1.0) + + self.assertTrue(finished.is_set()) + self.assertEqual(runtime.cancelled, [state]) + self.assertEqual(runtime.cleaned, [state]) + self.assertEqual(runtime.retained, 1) + # Only the first caller owns socket close and final cleanup. + service.disconnect_client(state) + self.assertEqual(runtime.cleaned, [state]) + + def test_successful_delivery_records_registry_proof_before_health_callback( + self, + ) -> None: + service, registry, runtime = service_fixture() + progress = runtime.progress + assert progress is not None + state = client_state(1) + state.subscribed = state.authorized = True + state.worker = worker("miner") + state.username = "miner" + registry.clients.add(state) + context = SimpleNamespace( + name="sent", + template={"previousblockhash": "00" * 32}, + template_fingerprint="fp-1", + template_generation=4, + payout_state_generation=2, + collection_only=False, + ) + + service.record_successful_delivery(state, context, 12.5) + + self.assertIs( + registry.eligible_snapshot()[1].delivered.context, # type: ignore[union-attr] + context, + ) + self.assertEqual(progress.registry_had_proof_at_callback, [True]) + self.assertEqual(len(progress.proofs), 1) + proof = progress.proofs[0] + self.assertEqual(proof.connection_id, 1) # type: ignore[union-attr] + self.assertEqual(proof.delivered_monotonic, 12.5) # type: ignore[union-attr] + # The staged-split compatibility mirrors advance in the same critical + # section as the registry proof. + self.assertIs(state._progress_delivered_context, context) + self.assertEqual(state._progress_delivered_template_generation, 4) + self.assertEqual(runtime.activity, [12.5]) + self.assertEqual(runtime.reconciles, [12.5]) + self.assertEqual(runtime.coverage, [[]]) + + def test_retired_client_leaves_no_eligible_snapshot_entry(self) -> None: + service, registry, _runtime = service_fixture() + state = client_state(1) + state.subscribed = state.authorized = True + state.worker = worker("miner") + registry.clients.add(state) + self.assertTrue(registry.begin_retirement_locked(state)) + + registry.record_delivery_locked( + state, SimpleNamespace(name="sent-before-retirement-won"), 12.5 + ) + + # A retired connection is invisible to fanout eligibility even if a + # racing delivery commits proof after retirement claimed the client. + self.assertFalse(registry.eligible_snapshot()) + + def test_shutdown_race_in_dispatch_maps_to_pool_closed_stratum_error(self) -> None: + service, _registry, _runtime = service_fixture() + state = client_state(1) + + with self.assertRaises(StratumError) as caught: + service.handle_request(state, {"id": 1, "method": "mining.submit"}) + + error = caught.exception + self.assertEqual(error.code, 20) + self.assertEqual(error.reason, "pool-closed") + self.assertTrue(error.disconnect) + self.assertIsInstance(error.__cause__, _DispatchShutdown) + + +class CompatibilityTests(unittest.TestCase): + def test_coordinator_reexports_exact_session_model_identities(self) -> None: + self.assertIs(FacadeClientState, ClientState) + self.assertIs(FacadeWorkerIdentity, WorkerIdentity) + self.assertIs(FacadeStratumError, StratumError) + self.assertFalse(hasattr(stratum_session, "PrismCoordinator")) + + def test_protocol_helpers_preserve_payload_and_password_behavior(self) -> None: + self.assertEqual(result_payload(1, True), {"id": 1, "result": True, "error": None}) + self.assertEqual( + error_payload(2, 21, "stale", reason="stale-job"), + { + "id": 2, + "result": None, + "error": [21, "stale", {"reason_id": "stale-job"}], + }, + ) + self.assertEqual( + parse_stratum_password_options("x,md=4,d=8,bad=1"), + (Decimal("8"), Decimal("4")), + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_prism_streamed_block_hex_consumers.py b/tests/test_prism_streamed_block_hex_consumers.py new file mode 100644 index 000000000..7624a4b13 --- /dev/null +++ b/tests/test_prism_streamed_block_hex_consumers.py @@ -0,0 +1,120 @@ +"""Streamed strings from a hydrated body survive every replay consumer (#255).""" + +from dataclasses import replace +import json +import threading +import unittest +from types import SimpleNamespace + +from lab.auxpow.stratum_codec import header_hash_hex +from lab.prism import candidate_codec as codec +from lab.prism import candidate_spool_view as views +from lab.prism import recover_pending_blocks as recovery +from lab.prism.block_candidates import block_candidate_from_intent, block_candidate_intent +from lab.prism.share_ledger import PsqlShareLedger, block_candidate_identity_sha256 +from tests.test_prism_candidate_spool_view import SpoolFixture, intent_for + +THRESHOLD = 32 * 1024 +HEADER = bytes([7]) * 80 +PENDING_SHARE = { + "share_id": "u:" + "ab" * 32, + "miner_id": "m", + "order_key": "m", + "p2mr_program_hex": "ab" * 32, + "share_difficulty": 100, + "network_difficulty": 1000, + "template_height": 10, + "job_id": "job", + "job_issued_at_ms": 100, + "accepted_at_ms": 123, + "ntime": 1_700_000_000, + "credit_policy": None, +} + + +def credited_fields() -> dict: + # Above the lowered threshold the hydrated block_hex is a streamed view, + # as a raw block of roughly 512 KiB is at the production threshold. + return intent_for( + 3, + block_hash_hex=header_hash_hex(HEADER), + block_hex=HEADER.hex() + "ab" * 40_000, + pending_share=dict(PENDING_SHARE), + credit_share_on_accept=True, + ) + + +class StreamedBlockHexConsumerTests(SpoolFixture): + def test_replayed_credit_intent_re_encodes_a_streamed_block(self): + fields = credited_fields() + with self.bounded(THRESHOLD): + hydrated, _body = self.hydrate(fields) + self.assertIsInstance(hydrated["block_hex"], views.SpoolStringView) + replayed = replace(block_candidate_from_intent(hydrated), durable_replay=True) + plain = replace(block_candidate_from_intent(fields), durable_replay=True) + self.assertEqual( + block_candidate_intent(replayed).candidate_sha256, + block_candidate_intent(plain).candidate_sha256, + ) + + def test_streamed_username_is_decoded_for_the_worker_identity(self): + fields = {**credited_fields(), "username": "q" * 40_000 + ".worker"} + with self.bounded(THRESHOLD): + hydrated, _body = self.hydrate(fields) + self.assertIsInstance(hydrated["username"], views.SpoolStringView) + replayed = replace(block_candidate_from_intent(hydrated), durable_replay=True) + self.assertEqual(replayed.context.worker.username, fields["username"]) + self.assertEqual(replayed.client.username, fields["username"]) + plain = replace(block_candidate_from_intent(fields), durable_replay=True) + self.assertEqual( + block_candidate_intent(replayed).candidate_sha256, + block_candidate_intent(plain).candidate_sha256, + ) + + def test_offline_recovery_reads_the_header_of_a_streamed_block(self): + fields = credited_fields() + block = recovery.RecoveryBlock(fields["block_hash_hex"], 10, fields["parent_hash"], "pending") + with self.bounded(THRESHOLD): + hydrated, _body = self.hydrate(fields) + coordinator = SimpleNamespace( + block_candidate_from_intent=block_candidate_from_intent, + ledger=SimpleNamespace(hydrate_block_candidate_intent=lambda _row, **_kwargs: hydrated), + stop_event=threading.Event(), + ) + row = { + "candidate": None, + "candidate_sha256": hydrated.candidate_sha256, + "storage_version": 2, + "body_id": "ab" * 16, + "replay_header": {}, + "byte_count": 0, + "chunk_count": 0, + "chunk_bytes": 0, + "share_count": 0, + "body_state": "sealed", + } + candidate = recovery.decode_candidate(coordinator, block, row) + self.assertEqual(candidate.submission.block_hash_hex, block.block_hash) + self.assertIsInstance(candidate.submission.block_hex, views.SpoolStringView) + + def test_legacy_document_omits_absent_shares_and_materializes_views(self): + candidate_only = intent_for(0) + del candidate_only["shares_json"] + prepared = codec.prepare_candidate_intent(candidate_only) + self.assertFalse(prepared.has_shares) + document = PsqlShareLedger._legacy_candidate_document(prepared) + self.assertNotIn("shares_json", document) + self.assertEqual(block_candidate_identity_sha256(document), prepared.candidate_sha256) + + fields = credited_fields() + with self.bounded(THRESHOLD): + hydrated, _body = self.hydrate(fields) + document = PsqlShareLedger._legacy_candidate_document(hydrated) + self.assertIsInstance(document["block_hex"], str) + self.assertEqual(document["block_hex"], fields["block_hex"]) + json.dumps(document) + self.assertEqual(block_candidate_identity_sha256(document), hydrated.candidate_sha256) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_prism_streamed_submitblock.py b/tests/test_prism_streamed_submitblock.py new file mode 100644 index 000000000..fbfdfe059 --- /dev/null +++ b/tests/test_prism_streamed_submitblock.py @@ -0,0 +1,85 @@ +"""Real HTTP transport for a replayed block whose hex stays on disk.""" + +import hashlib +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer +import json +import threading +import tracemalloc +import unittest + +from lab.prism.rpc import JsonRpc + + +class StreamedSubmitblockTests(unittest.TestCase): + def test_large_block_uses_content_length_and_bounded_stream(self): + class Block: + byte_length = 2 + 256 * 65536 + + def iter_byte_chunks(self): + yield b'"' + for _ in range(256): + yield b"ab" * 32768 + yield b'"' + + def __str__(self): + raise AssertionError("block was materialized") + + observed = [] + + class Handler(BaseHTTPRequestHandler): + def do_POST(self): + length = int(self.headers["Content-Length"]) + digest = hashlib.sha256() + remaining = length + while remaining: + chunk = self.rfile.read(min(65536, remaining)) + if not chunk: + break + digest.update(chunk) + remaining -= len(chunk) + observed.append((length, remaining, digest.hexdigest(), self.headers.get("Transfer-Encoding"))) + payload = b'{"result":null,"error":null,"id":"submitblock"}' + self.send_response(200) + self.send_header("Content-Length", str(len(payload))) + self.end_headers() + try: + self.wfile.write(payload) + except OSError: + pass # A deliberately interrupted upload closed its socket. + + def log_message(self, *args): + pass + + server = ThreadingHTTPServer(("127.0.0.1", 0), Handler) + thread = threading.Thread(target=server.serve_forever) + thread.start() + rpc = JsonRpc(host="127.0.0.1", port=server.server_port, user="u", password="p") + try: + tracemalloc.start() + try: + self.assertIsNone(rpc.call("submitblock", [Block()])) + _, peak = tracemalloc.get_traced_memory() + finally: + tracemalloc.stop() + expected = json.dumps({"jsonrpc": "1.0", "id": "submitblock", "method": "submitblock", + "params": ["ab" * (256 * 32768)]}, separators=(",", ":")).encode() + self.assertEqual(observed, [(len(expected), 0, hashlib.sha256(expected).hexdigest(), None)]) + self.assertLess(peak, 2 * 1024 * 1024) + + class FailedBlock(Block): + def iter_byte_chunks(self): + yield b'"ab' + raise ValueError("source closed") + + with self.assertRaisesRegex(ValueError, "source closed"): + rpc.call("submitblock", [FailedBlock()]) + self.assertIsNone(rpc._connections.conn) + finally: + rpc._drop_connection() + server.shutdown() + server.server_close() + thread.join() + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_prism_tip_publication_boundary.py b/tests/test_prism_tip_publication_boundary.py index ace8a9b02..3b7bcc8a6 100644 --- a/tests/test_prism_tip_publication_boundary.py +++ b/tests/test_prism_tip_publication_boundary.py @@ -12,7 +12,7 @@ TemplateRefreshBlocked, _JobBuildCancellation, ) -from tests.test_prism_coordinator_job_cache import ( +from tests.prism_coordinator_test_support import ( base_template, client, coordinator, diff --git a/tests/test_prism_tip_refresh.py b/tests/test_prism_tip_refresh.py new file mode 100644 index 000000000..9dec17020 --- /dev/null +++ b/tests/test_prism_tip_refresh.py @@ -0,0 +1,185 @@ +#!/usr/bin/env python3 +"""Owner-boundary tests for the extracted R1 tip-refresh and P1 payout owners. + +The current services resolve the coordinator through call-time Runtime +Protocols (the old stack's standalone ``*Ports``/``*Config`` fixtures were +deliberately not built), so this suite pins the boundary itself: exact symbol +identity, private-alias compatibility, descriptor-routed legacy fields, and +the self-contained fanout delivery gate. The scheduler/trigger/publication +behavior is asserted by the upstream suites in +``tests/test_prism_tip_refresh_validation.py``, +``tests/test_prism_tip_refresh_epoch.py``, +``tests/test_prism_tip_publication_boundary.py``, +``tests/test_prism_refresh_retry_pacing.py`` and +``tests/test_prism_payout_state.py``. +""" + +from __future__ import annotations + +import threading +import unittest + +from lab.prism import ( + payout_state, + prism_coordinator, + template_artifacts, + tip_refresh, +) +from lab.prism.prism_coordinator import PrismCoordinator +from lab.prism.tip_refresh import FanoutCancellation +from tests.prism_vardiff_test_support import coordinator + +TIP_A = "11" * 32 + + +class FanoutCancellationTests(unittest.TestCase): + def test_fanout_cancellation_closes_admission_then_drains(self) -> None: + cancellation = FanoutCancellation() + self.assertTrue(cancellation.begin_delivery()) + drained = threading.Event() + + def cancel_and_drain() -> None: + cancellation.set() + drained.set() + + thread = threading.Thread(target=cancel_and_drain) + thread.start() + self.assertFalse(drained.wait(0.05)) + self.assertFalse(cancellation.begin_delivery()) + cancellation.end_delivery() + self.assertTrue(drained.wait(1.0)) + thread.join(1.0) + self.assertFalse(thread.is_alive()) + + def test_delivery_gate_release_without_admission_fails_closed(self) -> None: + cancellation = FanoutCancellation() + + with self.assertRaisesRegex(RuntimeError, "without admission"): + cancellation.end_delivery() + + # cancel() closes admission without waiting so workers may call it + # while holding a client lock; is_set reports the closed gate. + cancellation.cancel() + self.assertTrue(cancellation.is_set()) + self.assertFalse(cancellation.begin_delivery()) + + +class R1FacadeIdentityTests(unittest.TestCase): + def test_facade_reexports_exact_r1_identities(self) -> None: + self.assertIs( + prism_coordinator.RetainedCollectionRefresh, + tip_refresh.RetainedCollectionRefresh, + ) + self.assertIs(prism_coordinator.RefreshResult, tip_refresh.RefreshResult) + self.assertIs( + prism_coordinator.TipRefreshValidationToken, + tip_refresh.TipRefreshValidationToken, + ) + self.assertIs( + prism_coordinator._FanoutCancellation, tip_refresh.FanoutCancellation + ) + self.assertIs( + prism_coordinator.PRISM_TIP_REFRESH_SECONDS_BUCKETS, + tip_refresh.PRISM_TIP_REFRESH_SECONDS_BUCKETS, + ) + self.assertIs( + prism_coordinator.PRISM_TIP_REFRESH_BUILD_PHASES, + tip_refresh.PRISM_TIP_REFRESH_BUILD_PHASES, + ) + # PR 80 removed the result/cancellation-stage re-exports. + self.assertFalse(hasattr(prism_coordinator, "PRISM_TIP_REFRESH_RESULTS")) + self.assertFalse( + hasattr(prism_coordinator, "PRISM_TIP_REFRESH_CANCELLATION_STAGES") + ) + # The owner is a leaf: it never imports the coordinator module. + self.assertFalse(hasattr(tip_refresh, "PrismCoordinator")) + + def test_p1_reexports_and_private_aliases_preserve_identity(self) -> None: + self.assertIs( + prism_coordinator.PayoutStateArtifact, payout_state.PayoutStateArtifact + ) + self.assertIs( + prism_coordinator.PayoutStateCandidate, payout_state.PayoutStateCandidate + ) + # PR 80 removed the PublishedPayoutState compatibility re-export. + self.assertFalse(hasattr(prism_coordinator, "PublishedPayoutState")) + self.assertIs( + prism_coordinator.PayoutLedgerArtifact, payout_state.PayoutLedgerArtifact + ) + self.assertIs( + prism_coordinator.TemplateRefreshBlocked, + payout_state.TemplateRefreshBlocked, + ) + self.assertIs( + prism_coordinator.TemplateRefreshSuperseded, + payout_state.TemplateRefreshSuperseded, + ) + # PR 80 removed the four underscore payout aliases; the coordinator + # keeps only the unaliased exception it still raises. + self.assertFalse(hasattr(prism_coordinator, "_AcceptedBlockPayoutTransition")) + self.assertFalse(hasattr(prism_coordinator, "_PayoutDeliveryAdmission")) + self.assertFalse(hasattr(prism_coordinator, "_PayoutStateDeliveryGate")) + self.assertFalse(hasattr(prism_coordinator, "_PayoutStatePublicationBlocked")) + self.assertIs( + prism_coordinator.PayoutStatePublicationBlocked, + payout_state.PayoutStatePublicationBlocked, + ) + # template_artifacts re-imports the exception trio from the final P1 + # owner so the interim import path stays identity-compatible. + self.assertIs( + template_artifacts.TemplateRefreshBlocked, + payout_state.TemplateRefreshBlocked, + ) + self.assertIs( + template_artifacts.TemplateRefreshSuperseded, + payout_state.TemplateRefreshSuperseded, + ) + + +class R1DescriptorRoutingTests(unittest.TestCase): + def test_tip_state_descriptors_route_to_the_r1_owner(self) -> None: + server = coordinator() + service = server._ensure_tip_refresh_service() + self.assertIs(server._ensure_tip_refresh_service(), service) + + server.tip_refresh_job_count = 7 + self.assertEqual(service.tip_refresh_job_count, 7) + self.assertEqual(server.tip_refresh_job_count, 7) + + server.current_tip_first_seen = (TIP_A, 5.0) + self.assertEqual(service.current_tip_first_seen, (TIP_A, 5.0)) + with server.lock: + self.assertEqual(server._current_published_tip_hash_locked(), TIP_A) + + # The refresh lock is owner state reached through the legacy name. + self.assertIs(server._tip_refresh_lock, service._tip_refresh_lock) + + def test_unset_tip_state_reads_raise_attribute_error_like_lazy_fields( + self, + ) -> None: + server = PrismCoordinator.__new__(PrismCoordinator) + server.lock = threading.RLock() + + with self.assertRaises(AttributeError): + _ = server.current_tip_first_seen + with self.assertRaises(AttributeError): + _ = server.tip_refresh_job_count + + server.tip_refresh_job_count = 3 + self.assertEqual(server.tip_refresh_job_count, 3) + + def test_payout_state_descriptors_route_to_the_p1_owner(self) -> None: + server = coordinator() + service = server._ensure_payout_state_service() + + server._payout_state_generation = 9 + self.assertEqual(server._payout_state_generation, 9) + current = server._published_payout_state + self.assertIs(current, server._published_payout_state) + # The generation adapter reads the same owner state. + self.assertEqual(server._current_payout_generation(), 9) + self.assertIsNotNone(service) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_prism_tip_refresh_delivery.py b/tests/test_prism_tip_refresh_delivery.py new file mode 100644 index 000000000..d1e148158 --- /dev/null +++ b/tests/test_prism_tip_refresh_delivery.py @@ -0,0 +1,3180 @@ +#!/usr/bin/env python3 +"""Focused PRISM coordinator tip refresh delivery tests.""" +# ruff: noqa: F821 + +from __future__ import annotations + +from pathlib import Path +import tempfile +import threading +import unittest +from lab.prism.job_delivery import JobBuildFailed +from lab.prism.payout_state import AcceptedParentPayoutPreviewPending +from tests import prism_coordinator_test_support as _job_support +from tests import prism_vardiff_test_support as _vardiff_support + + +class _VardiffSupportTestCase(unittest.TestCase): + def setUp(self) -> None: + globals().update( + {name: getattr(_vardiff_support, name) for name in _vardiff_support.__all__} + ) + + +class _JobSupportTestCase(unittest.TestCase): + def setUp(self) -> None: + globals().update( + {name: getattr(_job_support, name) for name in _job_support.__all__} + ) + + +class PrismCoordinatorVardiffTests(_VardiffSupportTestCase): + def test_tip_change_refreshes_clean_job_and_old_job_becomes_stale_without_reconnect(self) -> None: + old_tip = "00" * 32 + new_tip = "11" * 32 + server = coordinator() + server.accepted_block_count = 0 + server.max_blocks = 1 + server.stop_after_block = True + server.stale_grace_seconds = 0 + server.jobs = {} + server.recent_share_keys = set() + server.share_weights_by_username = {} + ledger = RecordingLedger() + server.ledger = ledger + worker = worker_identity() + state = client() + state.username = worker.username + state.worker = worker + state.share_difficulty = Decimal("1") + sent: list[dict[str, object]] = [] + state.send = lambda payload: sent.append(payload) # type: ignore[method-assign] + server.clients = {state} + + old_context = prism_context("old-job", old_tip, worker=worker) + state.active_job = old_context + state.active_job_ids = {"old-job"} + server.jobs["old-job"] = old_context + server.tip_template_snapshot = QbitTipTemplateSnapshot( + bestblockhash=old_tip, + previousblockhash=old_tip, + template_fingerprint=qbit_template_fingerprint(old_context.template), + ) + server.rpc = TipTemplateRpc(tip=new_tip, template=gbt_template(new_tip, height=11)) + + def build_fresh_job(client: ClientState, *, clean_jobs: bool) -> object: + self.assertIs(client, state) + return prism_context( + "fresh-job", + new_tip, + worker=worker, + difficulty=client.pending_share_difficulty or client.share_difficulty, + clean_jobs=clean_jobs, + ) + + server.build_job_for_client = build_fresh_job # type: ignore[method-assign] + + refreshed = server.poll_qbit_tip_template_once() + + self.assertEqual(refreshed, 1) + self.assertEqual(server.tip_refresh_job_count, 1) + self.assertIn(state, server.clients) + self.assertNotIn("old-job", server.jobs) + self.assertEqual(state.active_job_ids, {"fresh-job"}) + self.assertIn("fresh-job", server.jobs) + self.assertEqual([payload["method"] for payload in sent], ["mining.set_difficulty", "mining.notify"]) + self.assertEqual(sent[1]["params"][0], "fresh-job") + self.assertTrue(sent[1]["params"][8]) + + with self.assertRaises(StratumError) as raised: + server.handle_submit( + state, + ["miner-a", "old-job", "00" * 8, "00000001", "00000002"], + ) + self.assertEqual(raised.exception.code, 21) + self.assertEqual(raised.exception.reason, PRISM_REJECTION_UNKNOWN_JOB) + self.assertEqual(server.stale_share_count, 1) + self.assertEqual(server.rejection_counts_by_reason[PRISM_REJECTION_UNKNOWN_JOB], 1) + self.assertEqual(len(ledger.pending), 0) + + submission = SimpleNamespace( + header_hex="aa" * 80, + block_hash_hex="bb" * 32, + share_pass=True, + block_pass=False, + ) + with patch( + "lab.prism.prism_coordinator.direct_stratum.assemble_submission", + return_value=submission, + ): + should_close = server.handle_submit( + state, + ["miner-a", "fresh-job", "00" * 8, "00000001", "00000002"], + ) + + self.assertFalse(should_close) + self.assertEqual(len(ledger.pending), 1) + self.assertEqual(ledger.pending[0].job_id, "fresh-job") + self.assertIn(state, server.clients) + + def test_tip_refresh_rpc_race_blocks_mismatched_tip_template_snapshot(self) -> None: + old_tip = "00" * 32 + new_tip = "11" * 32 + server = coordinator() + server.jobs = {} + worker = worker_identity() + state = client() + state.username = worker.username + state.worker = worker + sent: list[dict[str, object]] = [] + state.send = lambda payload: sent.append(payload) # type: ignore[method-assign] + server.clients = {state} + + old_context = prism_context("old-job", old_tip, worker=worker) + state.active_job = old_context + state.active_job_ids = {"old-job"} + server.jobs["old-job"] = old_context + server.tip_template_snapshot = QbitTipTemplateSnapshot( + bestblockhash=old_tip, + previousblockhash=old_tip, + template_fingerprint=qbit_template_fingerprint(old_context.template), + ) + server.rpc = TipTemplateRpc(tip=old_tip, template=gbt_template(new_tip, height=11)) + + with self.assertRaisesRegex( + TemplateRefreshBlocked, + "tip changed while fetching block template", + ): + server.poll_qbit_tip_template_once() + + self.assertIs(state.active_job, old_context) + self.assertEqual(server.jobs, {"old-job": old_context}) + self.assertEqual(state.active_job_ids, {"old-job"}) + self.assertEqual(sent, []) + # Tip observation is recorded as a detection before expensive template + # construction so obsolete builders can be cancelled immediately, but + # submit authority is published only alongside a coherent snapshot. + # The incoherent template never enters the artifact cache, client job + # maps, or the published tip state. + self.assertEqual(server.latest_detected_tip[0], old_tip) + self.assertIsNone(server.current_tip_first_seen) + self.assertIsNone(server._template_artifacts) + + def test_slow_tip_poll_cannot_regress_newer_blockwait_observation(self) -> None: + old_tip = "00" * 32 + new_tip = "11" * 32 + server = coordinator() + old_snapshot = QbitTipTemplateSnapshot( + bestblockhash=old_tip, + previousblockhash=old_tip, + template_fingerprint="22" * 32, + ) + server.rpc = TipRpc(old_tip) + self.assertTrue(server.observe_tip_first_seen(old_tip)) + + def overtake_poll() -> QbitTipTemplateSnapshot: + self.assertTrue(server.observe_tip_for_refresh(new_tip)) + return old_snapshot + + server.fetch_qbit_tip_template_snapshot = overtake_poll # type: ignore[method-assign] + + with self.assertRaisesRegex( + TemplateRefreshSuperseded, + "tip/template poll was superseded during template fetch", + ): + server.poll_qbit_tip_template_once() + + self.assertEqual(server.latest_detected_tip[0], new_tip) + self.assertEqual(server.current_tip_first_seen[0], old_tip) + self.assertIsNone(server.tip_template_snapshot) + self.assertIsNone( + getattr(server, "template_refresh_failure_started_monotonic", None) + ) + self.assertTrue(server._consume_tip_refresh_retry()) + self.assertFalse(server._consume_tip_refresh_retry()) + + def test_same_tip_template_refresh_sends_non_clean_job_and_keeps_old_job_submittable(self) -> None: + tip = "00" * 32 + server = coordinator() + server.accepted_block_count = 0 + server.max_blocks = 1 + server.stop_after_block = True + server.jobs = {} + server.recent_share_keys = set() + server.share_weights_by_username = {} + ledger = RecordingLedger() + server.ledger = ledger + worker = worker_identity() + state = client() + state.username = worker.username + state.worker = worker + state.share_difficulty = Decimal("1") + sent: list[dict[str, object]] = [] + state.send = lambda payload: sent.append(payload) # type: ignore[method-assign] + server.clients = {state} + + old_context = prism_context("old-job", tip, worker=worker) + state.active_job = old_context + state.active_job_ids = {"old-job"} + server.jobs["old-job"] = old_context + server.tip_template_snapshot = QbitTipTemplateSnapshot( + bestblockhash=tip, + previousblockhash=tip, + template_fingerprint=qbit_template_fingerprint(old_context.template), + ) + refreshed_template = gbt_template(tip, height=10, coinbasevalue=50_00000001) + server.rpc = TipTemplateRpc(tip=tip, template=refreshed_template) + + def build_fresh_job(client: ClientState, *, clean_jobs: bool) -> object: + self.assertIs(client, state) + self.assertFalse(clean_jobs) + fresh_context = prism_context( + "fresh-job", + tip, + worker=worker, + difficulty=client.pending_share_difficulty or client.share_difficulty, + clean_jobs=clean_jobs, + ) + fresh_context.template["coinbasevalue"] = refreshed_template["coinbasevalue"] + fresh_context.template_fingerprint = qbit_template_fingerprint(fresh_context.template) + return fresh_context + + server.build_job_for_client = build_fresh_job # type: ignore[method-assign] + + refreshed = server.poll_qbit_tip_template_once() + + self.assertEqual(refreshed, 1) + self.assertEqual(server.tip_refresh_job_count, 1) + self.assertIn(state, server.clients) + self.assertIn("old-job", server.jobs) + self.assertIn("fresh-job", server.jobs) + self.assertEqual(state.active_job_ids, {"old-job", "fresh-job"}) + self.assertEqual([payload["method"] for payload in sent], ["mining.set_difficulty", "mining.notify"]) + self.assertEqual(sent[1]["params"][0], "fresh-job") + self.assertFalse(sent[1]["params"][8]) + self.assertIn("qbit_prism_active_job_contexts 2", server.metrics_payload()) + + submission = SimpleNamespace( + header_hex="aa" * 80, + block_hash_hex="bb" * 32, + share_pass=True, + block_pass=False, + ) + with patch( + "lab.prism.prism_coordinator.direct_stratum.assemble_submission", + return_value=submission, + ): + should_close = server.handle_submit( + state, + ["miner-a", "old-job", "00" * 8, "00000001", "00000002"], + ) + + self.assertFalse(should_close) + self.assertEqual(len(ledger.pending), 1) + self.assertEqual(ledger.pending[0].job_id, "old-job") + self.assertEqual(server.stale_share_count, 0) + self.assertEqual(server.rejection_counts_by_reason[PRISM_REJECTION_UNKNOWN_JOB], 0) + self.assertEqual(server.rejection_counts_by_reason[PRISM_REJECTION_STALE_JOB], 0) + + def test_tip_refresh_uses_pending_vardiff_difficulty_for_consistent_pair(self) -> None: + old_tip = "00" * 32 + new_tip = "22" * 32 + server = coordinator() + server.jobs = {} + worker = worker_identity() + state = client() + state.username = worker.username + state.worker = worker + state.share_difficulty = Decimal("1") + state.pending_share_difficulty = Decimal("8") + sent: list[dict[str, object]] = [] + state.send = lambda payload: sent.append(payload) # type: ignore[method-assign] + server.clients = {state} + old_context = prism_context("old-job", old_tip, worker=worker, difficulty=Decimal("1")) + state.active_job = old_context + state.active_job_ids = {"old-job"} + server.jobs["old-job"] = old_context + server.tip_template_snapshot = QbitTipTemplateSnapshot( + bestblockhash=old_tip, + previousblockhash=old_tip, + template_fingerprint=qbit_template_fingerprint(old_context.template), + ) + server.rpc = TipTemplateRpc(tip=new_tip, template=gbt_template(new_tip, height=11)) + + def build_fresh_job(client: ClientState, *, clean_jobs: bool) -> object: + return prism_context( + "fresh-vardiff-job", + new_tip, + worker=worker, + difficulty=server.desired_client_share_difficulty(client), + clean_jobs=clean_jobs, + ) + + server.build_job_for_client = build_fresh_job # type: ignore[method-assign] + + refreshed = server.poll_qbit_tip_template_once() + + self.assertEqual(refreshed, 1) + self.assertEqual(sent[0]["method"], "mining.set_difficulty") + self.assertEqual(sent[0]["params"], [8.0]) + self.assertEqual(sent[1]["method"], "mining.notify") + self.assertEqual(sent[1]["params"][0], "fresh-vardiff-job") + self.assertTrue(sent[1]["params"][8]) + self.assertEqual(state.share_difficulty, Decimal("8")) + self.assertIsNone(state.pending_share_difficulty) + self.assertEqual(server.jobs["fresh-vardiff-job"].job.share_difficulty, Decimal("8")) + + def test_tip_refresh_build_failure_keeps_client_connected_and_old_job_registered(self) -> None: + old_tip = "00" * 32 + new_tip = "33" * 32 + server = coordinator() + server.jobs = {} + worker = worker_identity() + state = client() + state.username = worker.username + state.worker = worker + sent: list[dict[str, object]] = [] + state.send = lambda payload: sent.append(payload) # type: ignore[method-assign] + server.clients = {state} + old_context = prism_context("old-job", old_tip, worker=worker) + state.active_job = old_context + state.active_job_ids = {"old-job"} + server.jobs["old-job"] = old_context + server.tip_template_snapshot = QbitTipTemplateSnapshot( + bestblockhash=old_tip, + previousblockhash=old_tip, + template_fingerprint=qbit_template_fingerprint(old_context.template), + ) + server.rpc = TipTemplateRpc(tip=new_tip, template=gbt_template(new_tip, height=11)) + + def failing_build(client: ClientState, *, clean_jobs: bool) -> object: + raise RuntimeError("transient getblocktemplate failure") + + server.build_job_for_client = failing_build # type: ignore[method-assign] + + with self.assertRaisesRegex(TemplateRefreshBlocked, "no refreshed work was issued"): + server.poll_qbit_tip_template_once() + + self.assertEqual(server.job_build_failure_count, 1) + self.assertEqual(server.tip_refresh_job_count, 0) + self.assertIn(state, server.clients) + self.assertEqual(state.active_job_ids, {"old-job"}) + self.assertIn("old-job", server.jobs) + self.assertEqual(sent, []) + + def test_tip_refresh_build_failure_is_not_masked_by_disconnected_client(self) -> None: + old_tip = "00" * 32 + new_tip = "33" * 32 + server = coordinator() + server.jobs = {} + + build_failed = client() + build_failed.worker = worker_identity("miner-build-failed") + build_failed.username = build_failed.worker.username + build_failed.active_job = prism_context( + "old-build-failed-job", old_tip, worker=build_failed.worker + ) + build_failed.active_job_ids = {"old-build-failed-job"} + + disconnected = client() + disconnected.connection_id = 2 + disconnected.worker = worker_identity("miner-disconnected") + disconnected.username = disconnected.worker.username + disconnected.active_job = prism_context( + "old-disconnected-job", old_tip, worker=disconnected.worker + ) + disconnected.active_job_ids = {"old-disconnected-job"} + + def disconnect_on_send(_payload: object) -> None: + raise OSError("socket closed") + + disconnected.send = disconnect_on_send # type: ignore[method-assign] + server.clients = {build_failed, disconnected} + server.jobs = { + "old-build-failed-job": build_failed.active_job, + "old-disconnected-job": disconnected.active_job, + } + server.tip_template_snapshot = QbitTipTemplateSnapshot( + bestblockhash=old_tip, + previousblockhash=old_tip, + template_fingerprint=qbit_template_fingerprint(build_failed.active_job.template), + ) + server.rpc = TipTemplateRpc(tip=new_tip, template=gbt_template(new_tip, height=11)) + + def mixed_build(state: ClientState, *, clean_jobs: bool) -> object: + if state is build_failed: + raise RuntimeError("template build unavailable") + return prism_context( + "disconnected-fresh-job", + new_tip, + worker=state.worker, + clean_jobs=clean_jobs, + ) + + disconnected_clients: list[ClientState] = [] + server.build_job_for_client = mixed_build # type: ignore[method-assign] + server.disconnect_client = disconnected_clients.append # type: ignore[method-assign] + + with self.assertRaisesRegex(TemplateRefreshBlocked, "no refreshed work was issued"): + server.poll_qbit_tip_template_once() + + self.assertEqual(server.job_build_failure_count, 1) + self.assertEqual(disconnected_clients, [disconnected]) + self.assertIn(build_failed, server.clients) + + def test_tip_reconciliation_quarantines_disconnected_block_before_refresh_job(self) -> None: + old_tip = "00" * 32 + new_tip = "44" * 32 + pool_block_hash = "aa" * 32 + server = coordinator() + server.reorg_reconciler_enabled = True + server.jobs = {} + worker = worker_identity() + state = client() + state.username = worker.username + state.worker = worker + state.share_difficulty = Decimal("1") + sent: list[dict[str, object]] = [] + state.send = lambda payload: sent.append(payload) # type: ignore[method-assign] + server.clients = {state} + old_context = prism_context("old-job", old_tip, worker=worker) + state.active_job = old_context + state.active_job_ids = {"old-job"} + server.jobs["old-job"] = old_context + server.tip_template_snapshot = QbitTipTemplateSnapshot( + bestblockhash=old_tip, + previousblockhash=old_tip, + template_fingerprint=qbit_template_fingerprint(old_context.template), + ) + ledger = ReorgLedger( + [ + { + "block_hash": pool_block_hash, + "block_height": 10, + "chain_state": "confirmed", + "maturity_state": "immature", + } + ] + ) + server.ledger = ledger + server.rpc = ReorgRpc( + tip=new_tip, + template=gbt_template(new_tip, height=11), + height=10, + block_hashes={10: "bb" * 32}, + ) + + def build_fresh_job(client: ClientState, *, clean_jobs: bool) -> object: + self.assertIn(("inactive", pool_block_hash, 10), ledger.events) + ledger.events.append(("build", client.connection_id)) + return prism_context("fresh-job", new_tip, worker=worker, clean_jobs=clean_jobs) + + server.build_job_for_client = build_fresh_job # type: ignore[method-assign] + + refreshed = server.poll_qbit_tip_template_once() + + self.assertEqual(refreshed, 1) + self.assertLess( + ledger.events.index(("inactive", pool_block_hash, 10)), + ledger.events.index(("build", state.connection_id)), + ) + self.assertEqual(server.reorg_inactive_block_count, 1) + self.assertEqual(ledger.rows[0]["chain_state"], "inactive") + self.assertEqual(sent[1]["params"][0], "fresh-job") + + def test_reconciliation_quarantines_confirmed_block_above_shortened_tip(self) -> None: + pool_block_hash = "af" * 32 + server = coordinator() + server.reorg_reconciler_enabled = True + ledger = ReorgLedger( + [ + { + "block_hash": pool_block_hash, + "block_height": 12, + "chain_state": "confirmed", + "maturity_state": "immature", + } + ] + ) + server.ledger = ledger + server.rpc = ReorgRpc( + tip="77" * 32, + template=gbt_template("77" * 32, height=11), + height=10, + block_hashes={}, + ) + + summary = server.reconcile_prism_pool_blocks_once(tip_hash="77" * 32) + + self.assertEqual(summary["inactive_blocks"], 1) + self.assertEqual(server.reorg_inactive_block_count, 1) + self.assertEqual(ledger.rows[0]["chain_state"], "inactive") + + def test_tip_reconciliation_skips_jobs_when_qbit_chain_view_is_untrusted(self) -> None: + old_tip = "00" * 32 + new_tip = "55" * 32 + server = coordinator() + server.reorg_reconciler_enabled = True + server.jobs = {} + worker = worker_identity() + state = client() + state.username = worker.username + state.worker = worker + state.active_job = prism_context("old-job", old_tip, worker=worker) + state.active_job_ids = {"old-job"} + server.jobs["old-job"] = state.active_job + server.clients = {state} + server.tip_template_snapshot = QbitTipTemplateSnapshot( + bestblockhash=old_tip, + previousblockhash=old_tip, + template_fingerprint=qbit_template_fingerprint(state.active_job.template), + ) + ledger = ReorgLedger([]) + server.ledger = ledger + server.rpc = ReorgRpc( + tip=new_tip, + template=gbt_template(new_tip, height=11), + height=10, + block_hashes={10: new_tip}, + initialblockdownload=True, + ) + + def unexpected_build(client: ClientState, *, clean_jobs: bool) -> object: + raise AssertionError("job build should be skipped while qbitd is in IBD") + + server.build_job_for_client = unexpected_build # type: ignore[method-assign] + + with self.assertRaisesRegex(TemplateRefreshBlocked, "chain view remained untrusted"): + server.poll_qbit_tip_template_once() + + self.assertEqual(server.reorg_reconcile_skip_count, 1) + self.assertEqual(ledger.events, []) + self.assertEqual(state.active_job_ids, {"old-job"}) + self.assertEqual(server.tip_template_snapshot.bestblockhash, old_tip) + + def test_reconciliation_error_before_job_build_is_not_counted_as_build_failure(self) -> None: + tip = "59" * 32 + server = coordinator() + server.reorg_reconciler_enabled = True + state = client() + state.username = "miner-a" + state.worker = worker_identity() + state.share_difficulty = Decimal("1") + server.rpc = ReorgRpc( + tip=tip, + template=gbt_template(tip, height=11), + height=10, + block_hashes={10: tip}, + ) + + class FailingReorgLedger(FakeLedger): + def reorg_watch_blocks(self, *, active_tip_height: int) -> list[dict[str, object]]: + raise RuntimeError("ledger unavailable") + + server.ledger = FailingReorgLedger() + server.build_job_for_client = lambda _client, *, clean_jobs: (_ for _ in ()).throw( # type: ignore[method-assign] + AssertionError("job build should not run after reconcile failure") + ) + + sent_job = server.maybe_send_job(state, clean_jobs=True) + + self.assertFalse(sent_job) + self.assertEqual(server.reorg_reconcile_error_count, 1) + self.assertEqual(server.job_build_failure_count, 0) + + def test_reconciliation_runs_again_for_same_tip_hash(self) -> None: + tip = "5a" * 32 + server = coordinator() + server.reorg_reconciler_enabled = True + ledger = ReorgLedger([]) + server.ledger = ledger + server.rpc = ReorgRpc( + tip=tip, + template=gbt_template(tip, height=11), + height=10, + block_hashes={10: tip}, + ) + + self.assertTrue(server.ensure_reorg_reconciled_for_tip(tip)) + self.assertTrue(server.ensure_reorg_reconciled_for_tip(tip)) + + self.assertEqual(ledger.events, [("watch", 10), ("mature", 10), ("watch", 10), ("mature", 10)]) + self.assertEqual(server._payout_state_generation, 1) + self.assertEqual(server._payout_state_source[0], 1) + + def test_reconciliation_reactivates_inactive_block_that_returns_to_active_chain(self) -> None: + pool_block_hash = "cc" * 32 + server = coordinator() + server.reorg_reconciler_enabled = True + ledger = ReorgLedger( + [ + { + "block_hash": pool_block_hash, + "block_height": 12, + "chain_state": "inactive", + "maturity_state": "immature", + } + ] + ) + server.ledger = ledger + temporary_audit = tempfile.TemporaryDirectory() + self.addCleanup(temporary_audit.cleanup) + server.audit_dir = Path(temporary_audit.name) / "audit" + server.evidence_path = ( + Path(temporary_audit.name) / "state" / "evidence.json" + ) + audit_store = server._ensure_audit_artifact_store() + self.addCleanup(audit_store.close) + payout_service = server._ensure_payout_state_service() + real_reactivate = ledger.reactivate_pool_block + + def guarded_reactivate(**kwargs: object) -> dict[str, object]: + self.assertTrue( + payout_service._payout_balance_mutation_lock._is_owned(), # type: ignore[attr-defined] + "P1 balance lock must be outermost during reactivation", + ) + self.assertEqual( + audit_store._publication_guard_owner, + threading.get_ident(), + ) + return real_reactivate(**kwargs) + + ledger.reactivate_pool_block = guarded_reactivate # type: ignore[method-assign] + server.rpc = ReorgRpc( + tip=pool_block_hash, + template=gbt_template(pool_block_hash, height=13), + height=12, + block_hashes={12: pool_block_hash}, + ) + + summary = server.reconcile_prism_pool_blocks_once(tip_hash=pool_block_hash) + + self.assertEqual(summary["reactivated_blocks"], 1) + self.assertEqual(server.reorg_reactivated_block_count, 1) + self.assertEqual(ledger.rows[0]["chain_state"], "confirmed") + self.assertIn(("mature", 12), ledger.events) + + def test_maybe_send_job_reconciles_before_direct_job_build(self) -> None: + tip = "66" * 32 + pool_block_hash = "dd" * 32 + server = coordinator() + server.reorg_reconciler_enabled = True + server.jobs = {} + worker = worker_identity() + state = client() + state.username = worker.username + state.worker = worker + state.share_difficulty = Decimal("1") + sent: list[dict[str, object]] = [] + state.send = lambda payload: sent.append(payload) # type: ignore[method-assign] + ledger = ReorgLedger( + [ + { + "block_hash": pool_block_hash, + "block_height": 20, + "chain_state": "confirmed", + "maturity_state": "immature", + } + ] + ) + server.ledger = ledger + server.rpc = ReorgRpc( + tip=tip, + template=gbt_template(tip, height=21), + height=20, + block_hashes={20: "ee" * 32}, + ) + + def build_direct_job(client: ClientState, *, clean_jobs: bool) -> object: + self.assertIn(("inactive", pool_block_hash, 20), ledger.events) + ledger.events.append(("build", client.connection_id)) + return prism_context("direct-job", tip, worker=worker, clean_jobs=clean_jobs) + + server.build_job_for_client = build_direct_job # type: ignore[method-assign] + + sent_job = server.maybe_send_job(state, clean_jobs=True) + + self.assertTrue(sent_job) + self.assertLess( + ledger.events.index(("inactive", pool_block_hash, 20)), + ledger.events.index(("build", state.connection_id)), + ) + self.assertEqual(sent[1]["params"][0], "direct-job") + + def test_template_refresh_failure_budget_starts_at_first_failure(self) -> None: + server = PrismCoordinator.__new__(PrismCoordinator) + server.template_refresh_failure_exit_seconds = 120 + server.last_successful_template_refresh_monotonic = 100.0 + + server._record_template_refresh_failure(500.0) + self.assertFalse(server.template_refresh_failure_expired(500.0)) + self.assertEqual(server.template_refresh_failure_started_monotonic, 500.0) + self.assertFalse(server.template_refresh_failure_expired(619.999)) + self.assertTrue(server.template_refresh_failure_expired(620.0)) + + def test_disabled_template_refresh_failure_budget_does_not_start_or_expire(self) -> None: + server = PrismCoordinator.__new__(PrismCoordinator) + server.template_refresh_failure_exit_seconds = 0 + + server._record_template_refresh_failure(500.0) + self.assertFalse(server.template_refresh_failure_expired(500.0)) + self.assertFalse(hasattr(server, "template_refresh_failure_started_monotonic")) + + def test_coordination_blocked_refresh_does_not_start_failure_budget(self) -> None: + for blocked_error in ( + TemplateRefreshSuperseded("qbit tip changed during sequential refresh"), + _PayoutStatePublicationBlocked( + "accepted block payout confirmation is still pending" + ), + ): + with self.subTest(blocked=type(blocked_error).__name__): + server = coordinator() + server.template_refresh_failure_exit_seconds = 10.0 + server.coordination_blocked_exit_seconds = 30.0 + server._record_heartbeat = lambda _name: None # type: ignore[method-assign] + server.rpc = TipRpc("11" * 32) + + def raise_blocked(error: Exception = blocked_error) -> QbitTipTemplateSnapshot: + raise error + + server.fetch_qbit_tip_template_snapshot = raise_blocked # type: ignore[method-assign] + with ( + patch( + "lab.prism.prism_coordinator.time.monotonic", + return_value=100.0, + ), + self.assertRaises(type(blocked_error)), + ): + server.poll_qbit_tip_template_once() + + self.assertIsNone( + getattr(server, "template_refresh_failure_started_monotonic", None) + ) + self.assertFalse(server.template_refresh_failure_expired(10_000.0)) + self.assertAlmostEqual( + server.coordination_blocked_streak_age_seconds(109.999), + 9.999, + ) + self.assertFalse( + server.coordination_blocked_streak_expired(129.999) + ) + + def test_non_coordination_blocked_refresh_still_starts_failure_budget(self) -> None: + # Plain TemplateRefreshBlocked also wraps genuine failures (malformed + # template artifacts, job builds failing, untrusted chain views); only + # the TemplateRefreshSuperseded/payout-fence subclasses are exempt. + server = coordinator() + server.template_refresh_failure_exit_seconds = 10.0 + server.coordination_blocked_exit_seconds = 10.0 + server._record_coordination_blocked_refresh(90.0) + server._record_heartbeat = lambda _name: None # type: ignore[method-assign] + server.rpc = TipRpc("11" * 32) + + def raise_blocked() -> QbitTipTemplateSnapshot: + raise TemplateRefreshBlocked( + "unable to derive exact artifacts for observed qbit template" + ) + + server.fetch_qbit_tip_template_snapshot = raise_blocked # type: ignore[method-assign] + with ( + patch("lab.prism.prism_coordinator.time.monotonic", return_value=100.0), + self.assertRaises(TemplateRefreshBlocked), + ): + server.poll_qbit_tip_template_once() + + self.assertEqual(server.template_refresh_failure_started_monotonic, 100.0) + self.assertEqual(server.coordination_blocked_streak_age_seconds(110.0), 0.0) + self.assertTrue(server.template_refresh_failure_expired(110.0)) + + def test_armed_budget_is_not_fired_by_coordination_blocked_refresh(self) -> None: + # A transient budgeted failure armed the clock, qbitd recovered, and + # only coordination churn follows. Blocked attempts must not trip the + # armed budget in blockpoll_loop: the exit is reserved for the next + # budgeted failure, and the clock clears on the next completed refresh. + server = coordinator() + server.blockpoll_seconds = 0 + server.template_refresh_failure_exit_seconds = 10.0 + server.template_refresh_failure_started_monotonic = 100.0 + server._record_heartbeat = lambda _name: None # type: ignore[method-assign] + server.rpc = TipRpc("11" * 32) + clock = {"now": 100.0} + blocked_polls = 0 + + def blocked_fetch() -> QbitTipTemplateSnapshot: + nonlocal blocked_polls + blocked_polls += 1 + clock["now"] += 6.0 + if blocked_polls >= 4: + server.stop_event.set() + raise TemplateRefreshSuperseded( + "qbit tip changed during sequential refresh; immediate retry scheduled" + ) + + server.fetch_qbit_tip_template_snapshot = blocked_fetch # type: ignore[method-assign] + with ( + patch( + "lab.prism.prism_coordinator.time.monotonic", + side_effect=lambda: clock["now"], + ), + patch("lab.prism.prism_coordinator.traceback.print_exc"), + patch("lab.prism.prism_coordinator.os._exit", side_effect=SystemExit(1)) as exit_process, + ): + server.blockpoll_loop() + + self.assertEqual(blocked_polls, 4) + self.assertGreater( + clock["now"], + server.template_refresh_failure_started_monotonic + + server.template_refresh_failure_exit_seconds, + ) + exit_process.assert_not_called() + self.assertEqual(server.template_refresh_failure_started_monotonic, 100.0) + + def test_rpc_outage_arms_and_exhausts_failure_budget(self) -> None: + server = coordinator() + server.blockpoll_seconds = 0 + server.template_refresh_failure_exit_seconds = 10.0 + server._record_heartbeat = lambda _name: None # type: ignore[method-assign] + clock = {"now": 100.0} + + class OutageRpc: + def call(self, method: str, params: list[object] | None = None, **_kwargs: object) -> object: + clock["now"] += 6.0 + raise ConnectionError("qbitd unreachable") + + server.rpc = OutageRpc() + with ( + patch( + "lab.prism.prism_coordinator.time.monotonic", + side_effect=lambda: clock["now"], + ), + patch("lab.prism.prism_coordinator.traceback.print_exc"), + patch("lab.prism.prism_coordinator.os._exit", side_effect=SystemExit(1)) as exit_process, + self.assertRaises(SystemExit), + ): + server.blockpoll_loop() + + exit_process.assert_called_once_with(1) + self.assertIsNotNone(server.template_refresh_failure_started_monotonic) + + def test_persistent_blocked_template_derivation_arms_and_exhausts_failure_budget(self) -> None: + # Top-of-poll RPCs stay healthy, but every refresh fails with plain + # TemplateRefreshBlocked (e.g. malformed template artifacts, all job + # builds failing). A fresh process must still arm the budget from its + # first such failure and take the budgeted restart path. + server = coordinator() + server.blockpoll_seconds = 0 + server.template_refresh_failure_exit_seconds = 10.0 + server._record_heartbeat = lambda _name: None # type: ignore[method-assign] + server.rpc = TipRpc("11" * 32) + clock = {"now": 100.0} + + def blocked_fetch() -> QbitTipTemplateSnapshot: + clock["now"] += 6.0 + raise TemplateRefreshBlocked( + "unable to derive exact artifacts for observed qbit template" + ) + + server.fetch_qbit_tip_template_snapshot = blocked_fetch # type: ignore[method-assign] + with ( + patch( + "lab.prism.prism_coordinator.time.monotonic", + side_effect=lambda: clock["now"], + ), + patch("lab.prism.prism_coordinator.traceback.print_exc"), + patch("lab.prism.prism_coordinator.os._exit", side_effect=SystemExit(1)) as exit_process, + self.assertRaises(SystemExit), + ): + server.blockpoll_loop() + + exit_process.assert_called_once_with(1) + self.assertIsNotNone(server.template_refresh_failure_started_monotonic) + + def test_healthy_noop_template_poll_resets_refresh_failure_clock(self) -> None: + server = coordinator() + server.blockpoll_seconds = 0 + server.last_successful_template_refresh_monotonic = 100.0 + server.template_refresh_failure_started_monotonic = 190.0 + server.template_refresh_failure_exit_seconds = 10.0 + server.coordination_blocked_exit_seconds = 10.0 + server._record_coordination_blocked_refresh(190.0) + server._record_heartbeat = lambda _name: None # type: ignore[method-assign] + snapshot = QbitTipTemplateSnapshot( + bestblockhash="11" * 32, + previousblockhash="11" * 32, + template_fingerprint="22" * 32, + ) + server.tip_template_snapshot = snapshot + server.rpc = TipRpc(snapshot.bestblockhash) + server.fetch_qbit_tip_template_snapshot = lambda: snapshot # type: ignore[method-assign] + + def trusted_chain_view(_tip: str) -> bool: + server.stop_event.set() + return True + + server.ensure_reorg_reconciled_for_tip = trusted_chain_view # type: ignore[method-assign] + with patch("lab.prism.prism_coordinator.time.monotonic", return_value=200.0): + server.blockpoll_loop() + + self.assertEqual(server.last_successful_template_refresh_monotonic, 200.0) + self.assertIsNone(server.template_refresh_failure_started_monotonic) + self.assertEqual(server.coordination_blocked_streak_age_seconds(300.0), 0.0) + self.assertFalse(server.coordination_blocked_streak_expired(300.0)) + self.assertFalse(server.template_refresh_failure_expired(300.0)) + self.assertIsNone(server.template_refresh_failure_started_monotonic) + server._record_template_refresh_failure(300.0) + self.assertEqual(server.template_refresh_failure_started_monotonic, 300.0) + + def test_shared_template_poll_records_success_for_blockwait_callers(self) -> None: + server = coordinator() + server.last_successful_template_refresh_monotonic = 100.0 + snapshot = QbitTipTemplateSnapshot( + bestblockhash="11" * 32, + previousblockhash="11" * 32, + template_fingerprint="22" * 32, + ) + server.rpc = TipRpc(snapshot.bestblockhash) + server.fetch_qbit_tip_template_snapshot = lambda: snapshot # type: ignore[method-assign] + server.ensure_reorg_reconciled_for_tip = lambda _tip: True # type: ignore[method-assign] + + with patch("lab.prism.prism_coordinator.time.monotonic", return_value=200.0): + refreshed = server.poll_qbit_tip_template_once(heartbeat_name="qbit_blockwait") + + self.assertEqual(refreshed, 0) + self.assertEqual(server.last_successful_template_refresh_monotonic, 200.0) + + def test_untrusted_reconciliation_exhausts_template_refresh_failure_budget(self) -> None: + server = coordinator() + server.blockpoll_seconds = 0 + server.last_successful_template_refresh_monotonic = 100.0 + server.template_refresh_failure_started_monotonic = 100.0 + server.template_refresh_failure_exit_seconds = 10.0 + server._record_heartbeat = lambda _name: None # type: ignore[method-assign] + snapshot = QbitTipTemplateSnapshot( + bestblockhash="11" * 32, + previousblockhash="11" * 32, + template_fingerprint="22" * 32, + ) + server.rpc = TipRpc(snapshot.bestblockhash) + server.fetch_qbit_tip_template_snapshot = lambda: snapshot # type: ignore[method-assign] + server.ensure_reorg_reconciled_for_tip = lambda _tip: False # type: ignore[method-assign] + + with ( + patch("lab.prism.prism_coordinator.time.monotonic", return_value=110.0), + patch("lab.prism.prism_coordinator.traceback.print_exc"), + patch("lab.prism.prism_coordinator.os._exit", side_effect=SystemExit(1)) as exit_process, + self.assertRaises(SystemExit), + ): + server.blockpoll_loop() + + exit_process.assert_called_once_with(1) + self.assertEqual(server.last_successful_template_refresh_monotonic, 100.0) + + def test_all_refresh_job_builds_failing_exhausts_failure_budget(self) -> None: + old_tip = "00" * 32 + new_tip = "33" * 32 + server = coordinator() + server.blockpoll_seconds = 0 + server.last_successful_template_refresh_monotonic = 100.0 + server.template_refresh_failure_started_monotonic = 100.0 + server.template_refresh_failure_exit_seconds = 10.0 + server._record_heartbeat = lambda _name: None # type: ignore[method-assign] + state = client() + state.username = "miner-a" + state.worker = worker_identity() + state.active_job = prism_context("old-job", old_tip, worker=state.worker) + state.active_job_ids = {"old-job"} + server.clients = {state} + server.jobs = {"old-job": state.active_job} + server.tip_template_snapshot = QbitTipTemplateSnapshot( + bestblockhash=old_tip, + previousblockhash=old_tip, + template_fingerprint=qbit_template_fingerprint(state.active_job.template), + ) + snapshot = QbitTipTemplateSnapshot( + bestblockhash=new_tip, + previousblockhash=new_tip, + template_fingerprint="44" * 32, + ) + server.rpc = TipRpc(new_tip) + server.fetch_qbit_tip_template_snapshot = lambda: snapshot # type: ignore[method-assign] + server.build_job_for_client = lambda *_args, **_kwargs: (_ for _ in ()).throw( # type: ignore[method-assign] + RuntimeError("template build unavailable") + ) + + with ( + patch("lab.prism.prism_coordinator.time.monotonic", return_value=110.0), + patch("lab.prism.prism_coordinator.traceback.print_exc"), + patch("lab.prism.prism_coordinator.os._exit", side_effect=SystemExit(1)) as exit_process, + self.assertRaises(SystemExit), + ): + server.blockpoll_loop() + + exit_process.assert_called_once_with(1) + self.assertEqual(server.job_build_failure_count, 1) + self.assertEqual(server.last_successful_template_refresh_monotonic, 100.0) + + def test_guarded_sequential_build_supersession_does_not_arm_failure_budget(self) -> None: + # Non-ready/collection mode: the sequential loop's guarded client + # build detects a superseded snapshot inside maybe_send_job. That is + # coordination churn, not template unhealthiness -- the real raise + # site must carry TemplateRefreshSuperseded so sustained pre-ready + # churn neither arms nor fires the budget. + old_tip = "00" * 32 + new_tip = "33" * 32 + server = coordinator() + server.blockpoll_seconds = 0 + server.template_refresh_failure_exit_seconds = 10.0 + server._record_heartbeat = lambda _name: None # type: ignore[method-assign] + state = client() + state.username = "miner-a" + state.worker = worker_identity() + state.active_job = prism_context("old-job", old_tip, worker=state.worker) + state.active_job_ids = {"old-job"} + server.clients = {state} + server.jobs = {"old-job": state.active_job} + server.tip_template_snapshot = QbitTipTemplateSnapshot( + bestblockhash=old_tip, + previousblockhash=old_tip, + template_fingerprint=qbit_template_fingerprint(state.active_job.template), + ) + snapshot = QbitTipTemplateSnapshot( + bestblockhash=new_tip, + previousblockhash=new_tip, + template_fingerprint="44" * 32, + ) + server.rpc = TipRpc(new_tip) + clock = {"now": 100.0} + fetches = 0 + + def fetch_snapshot() -> QbitTipTemplateSnapshot: + nonlocal fetches + fetches += 1 + clock["now"] += 6.0 + if fetches >= 4: + server.stop_event.set() + return snapshot + + server.fetch_qbit_tip_template_snapshot = fetch_snapshot # type: ignore[method-assign] + # The guarded pre-build currency check loses the race on every pass. + server._tip_refresh_snapshot_current_locked = ( # type: ignore[method-assign] + lambda *_args, **_kwargs: False + ) + + with ( + patch( + "lab.prism.prism_coordinator.time.monotonic", + side_effect=lambda: clock["now"], + ), + patch("lab.prism.prism_coordinator.traceback.print_exc"), + patch("lab.prism.prism_coordinator.os._exit", side_effect=SystemExit(1)) as exit_process, + ): + server.blockpoll_loop() + + self.assertEqual(fetches, 4) + self.assertGreater(clock["now"] - 100.0, server.template_refresh_failure_exit_seconds) + exit_process.assert_not_called() + self.assertIsNone( + getattr(server, "template_refresh_failure_started_monotonic", None) + ) + + def test_transient_template_refresh_failure_recovers_on_healthy_noop(self) -> None: + server = coordinator() + server.blockpoll_seconds = 0 + server.last_successful_template_refresh_monotonic = 100.0 + server.template_refresh_failure_exit_seconds = 10.0 + server._record_heartbeat = lambda _name: None # type: ignore[method-assign] + poll_count = 0 + + def fail_then_noop() -> int: + nonlocal poll_count + poll_count += 1 + if poll_count == 1: + raise RuntimeError("transient RPC failure") + server.last_successful_template_refresh_monotonic = time.monotonic() + server.stop_event.set() + return 0 + + server.poll_qbit_tip_template_once = fail_then_noop # type: ignore[method-assign] + with ( + patch("lab.prism.prism_coordinator.time.monotonic", side_effect=[105.0, 106.0]), + patch("lab.prism.prism_coordinator.traceback.print_exc"), + ): + server.blockpoll_loop() + + self.assertEqual(poll_count, 2) + self.assertEqual(server.last_successful_template_refresh_monotonic, 106.0) + + def _pending_append(self, tag: str, accepted_at_ms: int = 2) -> PendingShareAppend: + from lab.prism.share_ledger import PendingShare + + return PendingShareAppend( + pending_share=PendingShare( + share_id=f"miner-a:{tag}", + miner_id="miner-a", + order_key="miner-a", + p2mr_program_hex="11" * 32, + share_difficulty=1, + network_difficulty=1, + template_height=10, + job_id="job-1", + job_issued_at_ms=1, + accepted_at_ms=accepted_at_ms, + ntime=1_700_000_000, + ), + username="miner-a", + job_id="job-1", + block_hash_hex=tag * 32, + collection_only=False, + credit_policy=None, + ) + + def test_lost_mutation_response_publishes_fresh_balances_with_forced_rescan( + self, + ) -> None: + tip = "5e" * 32 + pool_block_hash = "af" * 32 + + class LostResponseLedger(ReorgLedger): + def __init__(self) -> None: + super().__init__([]) + self.prior_balance_reads = 0 + + def current_prior_balances(self) -> list[dict[str, object]]: + self.prior_balance_reads += 1 + if any(row["chain_state"] == "confirmed" for row in self.rows): + return [ + { + "recipient_id": "stale-carry", + "order_key": "01:stale-carry", + "p2mr_program_hex": "44" * 32, + "balance_sats": 546, + } + ] + return [] + + def mark_pool_block_inactive( + self, + *, + block_hash: str, + active_tip_height: int, + ) -> dict[str, object]: + result = super().mark_pool_block_inactive( + block_hash=block_hash, + active_tip_height=active_tip_height, + ) + if int(result["inactive_count"]): + raise ConnectionError("mutation response lost") + return result + + ledger = LostResponseLedger() + server = coordinator() + server.reorg_reconciler_enabled = True + server.ledger = ledger + server.rpc = ReorgRpc( + tip=tip, + template=gbt_template(tip, height=11), + height=10, + block_hashes={10: tip}, + ) + forced_rescans: list[bool] = [] + original_prepare = server._prepared_payout_state_candidate + + def record_prepare( + captured: tuple[int, int, str | None, str, float], + *, + force_full_window_rescan: bool = False, + bypass_build_interval: bool = False, + ) -> object: + forced_rescans.append(force_full_window_rescan) + return original_prepare( + captured, + force_full_window_rescan=force_full_window_rescan, + bypass_build_interval=bypass_build_interval, + ) + + server._prepared_payout_state_candidate = record_prepare # type: ignore[method-assign] + + first = server.reconcile_prism_pool_blocks_once(tip_hash=tip) + self.assertEqual(first["published_generation"], 1) + with server._job_cache_lock: + initially_published = server._published_payout_state.artifact + assert initially_published is not None + self.assertEqual(len(initially_published.prior_balances()), 0) + + ledger.rows.append( + { + "block_hash": pool_block_hash, + "block_height": 12, + "chain_state": "confirmed", + "maturity_state": "immature", + } + ) + # Rebuild the current generation's immutable state so the test starts + # from balances that predate the lost-response mutation. + with server._job_cache_lock: + server._published_payout_state = dataclass_replace( + server._published_payout_state, + artifact=None, + ) + stale_published = server._current_payout_state_artifact() + self.assertEqual(len(stale_published.prior_balances()), 1) + reads_before_error = ledger.prior_balance_reads + forced_rescans.clear() + + with self.assertRaisesRegex(ConnectionError, "mutation response lost"): + server.reconcile_prism_pool_blocks_once(tip_hash=tip) + + with server._job_cache_lock: + healed_state = server._published_payout_state + assert healed_state.artifact is not None + self.assertEqual(healed_state.generation, 2) + self.assertEqual(healed_state.artifact.prior_balances(), []) + self.assertEqual(ledger.prior_balance_reads, reads_before_error + 1) + self.assertEqual(forced_rescans, [True]) + self.assertEqual(ledger.rows[0]["chain_state"], "inactive") + + def test_read_phase_reconcile_failure_does_not_force_balance_rescan( + self, + ) -> None: + tip = "5f" * 32 + + class PrefetchFailureLedger(ReorgLedger): + fail_prefetch = False + + def __init__(self) -> None: + super().__init__([]) + self.prior_balance_reads = 0 + + def current_prior_balances(self) -> list[dict[str, object]]: + self.prior_balance_reads += 1 + return [] + + def reorg_watch_blocks( + self, + *, + active_tip_height: int, + ) -> list[dict[str, object]]: + if self.fail_prefetch: + raise TimeoutError("reconcile prefetch join exceeded 20s") + return super().reorg_watch_blocks( + active_tip_height=active_tip_height + ) + + ledger = PrefetchFailureLedger() + server = coordinator() + server.reorg_reconciler_enabled = True + server.ledger = ledger + server.rpc = ReorgRpc( + tip=tip, + template=gbt_template(tip, height=11), + height=10, + block_hashes={10: tip}, + ) + first = server.reconcile_prism_pool_blocks_once(tip_hash=tip) + self.assertEqual(first["published_generation"], 1) + reads_before_failure = ledger.prior_balance_reads + with server._job_cache_lock: + generation_before_failure = server._published_payout_state.generation + + forced_rescans: list[bool] = [] + original_prepare = server._prepared_payout_state_candidate + + def record_prepare( + captured: tuple[int, int, str | None, str, float], + *, + force_full_window_rescan: bool = False, + bypass_build_interval: bool = False, + ) -> object: + forced_rescans.append(force_full_window_rescan) + return original_prepare( + captured, + force_full_window_rescan=force_full_window_rescan, + bypass_build_interval=bypass_build_interval, + ) + + server._prepared_payout_state_candidate = record_prepare # type: ignore[method-assign] + ledger.fail_prefetch = True + + with self.assertRaisesRegex( + TimeoutError, + "reconcile prefetch join exceeded 20s", + ): + server.reconcile_prism_pool_blocks_once(tip_hash=tip) + + with server._job_cache_lock: + generation_after_failure = server._published_payout_state.generation + self.assertEqual(forced_rescans, []) + self.assertEqual(ledger.prior_balance_reads, reads_before_failure) + self.assertEqual(generation_after_failure, generation_before_failure) + + def test_concurrent_same_tip_reconciles_share_one_pass(self) -> None: + tip = "5c" * 32 + entered = threading.Event() + release = threading.Event() + + class BlockingReorgLedger(ReorgLedger): + def reorg_watch_blocks( + self, + *, + active_tip_height: int, + ) -> list[dict[str, object]]: + entered.set() + if not release.wait(timeout=5.0): + raise AssertionError("reconcile release was never signaled") + return super().reorg_watch_blocks( + active_tip_height=active_tip_height + ) + + ledger = BlockingReorgLedger([]) + server = coordinator() + server.reorg_reconciler_enabled = True + server.ledger = ledger + server.rpc = ReorgRpc( + tip=tip, + template=gbt_template(tip, height=11), + height=10, + block_hashes={10: tip}, + ) + server._ensure_job_cache_state() + + results: list[dict[str, object] | None] = [None, None] + + def reconcile(slot: int) -> None: + results[slot] = server.reconcile_prism_pool_blocks_once(tip_hash=tip) + + leader = threading.Thread(target=reconcile, args=(0,)) + leader.start() + self.assertTrue(entered.wait(timeout=5.0)) + with server._ensure_reorg_reconciler_service()._reconcile_flight_lock: + flight = server._ensure_reorg_reconciler_service()._reconcile_flights[tip] + + follower_waiting = threading.Event() + + class SignalingEvent: + def __init__(self, inner: threading.Event) -> None: + self._inner = inner + + def wait(self, timeout: float | None = None) -> bool: + follower_waiting.set() + return self._inner.wait(timeout) + + def set(self) -> None: + self._inner.set() + + flight.event = SignalingEvent(flight.event) # type: ignore[assignment] + follower = threading.Thread(target=reconcile, args=(1,)) + follower.start() + self.assertTrue(follower_waiting.wait(timeout=5.0)) + + release.set() + leader.join(timeout=5.0) + follower.join(timeout=5.0) + self.assertFalse(leader.is_alive()) + self.assertFalse(follower.is_alive()) + + self.assertEqual( + ledger.events, + [("watch", 10), ("mature", 10)], + "the follower must reuse the leader's pass, not run its own", + ) + self.assertEqual(results[0], results[1]) + self.assertIsNotNone(results[0]) + self.assertIsNot(results[0], results[1]) + with server._ensure_reorg_reconciler_service()._reconcile_flight_lock: + self.assertEqual(server._ensure_reorg_reconciler_service()._reconcile_flights, {}) + + def test_lock_owner_reconcile_bypasses_same_tip_flight(self) -> None: + tip = "5f" * 32 + ledger = ReorgLedger([]) + server = coordinator() + server.reorg_reconciler_enabled = True + server.ledger = ledger + server.rpc = ReorgRpc( + tip=tip, + template=gbt_template(tip, height=11), + height=10, + block_hashes={10: tip}, + ) + server._ensure_job_cache_state() + + leader_lock_attempted = threading.Event() + flight_waited = threading.Event() + flight_completed = threading.Event() + underlying_lock = threading.RLock() + leader: threading.Thread | None = None + + class ObservedBalanceLock: + def __enter__(self) -> ObservedBalanceLock: + if threading.current_thread() is leader: + if underlying_lock.acquire(blocking=False): + underlying_lock.release() + raise AssertionError( + "reconcile leader unexpectedly acquired the owned lock" + ) + leader_lock_attempted.set() + underlying_lock.acquire() + else: + underlying_lock.acquire() + return self + + def __exit__(self, *_args: object) -> None: + underlying_lock.release() + + class WaitForbiddenEvent: + def __init__(self, inner: threading.Event) -> None: + self._inner = inner + + def wait(self, timeout: float | None = None) -> bool: + flight_waited.set() + raise AssertionError( + "lock-owning reconciliation joined the same-tip flight" + ) + + def set(self) -> None: + flight_completed.set() + self._inner.set() + + server._payout_balance_mutation_lock = ObservedBalanceLock() # type: ignore[assignment] + leader_results: list[dict[str, object]] = [] + leader_errors: list[BaseException] = [] + + def lead_reconcile() -> None: + try: + leader_results.append( + server.reconcile_prism_pool_blocks_once(tip_hash=tip) + ) + except BaseException as exc: # noqa: BLE001 - asserted below + leader_errors.append(exc) + + leader = threading.Thread(target=lead_reconcile) + try: + with server._payout_balance_mutation_lock: + leader.start() + self.assertTrue(leader_lock_attempted.wait(timeout=5.0)) + with server._ensure_reorg_reconciler_service()._reconcile_flight_lock: + flight = server._ensure_reorg_reconciler_service()._reconcile_flights[tip] + flight.event = WaitForbiddenEvent( # type: ignore[assignment] + flight.event + ) + + bypassed = server.ensure_reorg_reconciled_for_tip( + tip, + _coalesce_same_tip=False, + ) + + self.assertTrue(bypassed) + self.assertFalse(flight_waited.is_set()) + self.assertTrue(leader.is_alive()) + self.assertEqual( + ledger.events, + [("watch", 10), ("mature", 10)], + ) + finally: + leader.join(timeout=5.0) + + self.assertFalse(leader.is_alive()) + if leader_errors: + raise leader_errors[0] + self.assertEqual(len(leader_results), 1) + self.assertTrue(flight_completed.is_set()) + self.assertFalse(flight_waited.is_set()) + self.assertEqual( + ledger.events, + [ + ("watch", 10), + ("mature", 10), + ("watch", 10), + ("mature", 10), + ], + ) + with server._ensure_reorg_reconciler_service()._reconcile_flight_lock: + self.assertEqual(server._ensure_reorg_reconciler_service()._reconcile_flights, {}) + + def test_lock_owner_reconcile_leads_visible_flight_when_absent(self) -> None: + tip = "60" * 32 + entered = threading.Event() + release = threading.Event() + + class BlockingReorgLedger(ReorgLedger): + def reorg_watch_blocks( + self, + *, + active_tip_height: int, + ) -> list[dict[str, object]]: + entered.set() + if not release.wait(timeout=5.0): + raise AssertionError("reconcile release was never signaled") + return super().reorg_watch_blocks( + active_tip_height=active_tip_height + ) + + ledger = BlockingReorgLedger([]) + server = coordinator() + server.reorg_reconciler_enabled = True + server.ledger = ledger + server.rpc = ReorgRpc( + tip=tip, + template=gbt_template(tip, height=11), + height=10, + block_hashes={10: tip}, + ) + server._ensure_job_cache_state() + + lock_owner_result: list[bool] = [] + follower_result: list[dict[str, object]] = [] + errors: list[BaseException] = [] + follower_waiting = threading.Event() + + class SignalingEvent: + def __init__(self, inner: threading.Event) -> None: + self._inner = inner + + def wait(self, timeout: float | None = None) -> bool: + follower_waiting.set() + return self._inner.wait(timeout) + + def set(self) -> None: + self._inner.set() + + def reconcile_while_owning_lock() -> None: + try: + with server._payout_balance_mutation_lock: + lock_owner_result.append( + server.ensure_reorg_reconciled_for_tip( + tip, + _coalesce_same_tip=False, + ) + ) + except BaseException as exc: # noqa: BLE001 - asserted below + errors.append(exc) + + def follow_reconcile() -> None: + try: + follower_result.append( + server.reconcile_prism_pool_blocks_once(tip_hash=tip) + ) + except BaseException as exc: # noqa: BLE001 - asserted below + errors.append(exc) + + lock_owner = threading.Thread(target=reconcile_while_owning_lock) + follower = threading.Thread(target=follow_reconcile) + lock_owner.start() + try: + self.assertTrue(entered.wait(timeout=5.0)) + with server._ensure_reorg_reconciler_service()._reconcile_flight_lock: + flight = server._ensure_reorg_reconciler_service()._reconcile_flights[tip] + flight.event = SignalingEvent(flight.event) # type: ignore[assignment] + + follower.start() + self.assertTrue(follower_waiting.wait(timeout=5.0)) + self.assertTrue(lock_owner.is_alive()) + self.assertTrue(follower.is_alive()) + self.assertEqual(ledger.events, []) + finally: + release.set() + lock_owner.join(timeout=5.0) + if follower.ident is not None: + follower.join(timeout=5.0) + + self.assertFalse(lock_owner.is_alive()) + self.assertFalse(follower.is_alive()) + if errors: + raise errors[0] + self.assertEqual(lock_owner_result, [True]) + self.assertEqual(len(follower_result), 1) + self.assertEqual( + ledger.events, + [("watch", 10), ("mature", 10)], + "the ordinary caller must reuse the lock owner's visible pass", + ) + with server._ensure_reorg_reconciler_service()._reconcile_flight_lock: + self.assertEqual(server._ensure_reorg_reconciler_service()._reconcile_flights, {}) + + def test_forced_and_reserved_reconciles_bypass_flight_reuse(self) -> None: + tip = "5e" * 32 + ledger = ReorgLedger([]) + server = coordinator() + server.reorg_reconciler_enabled = True + server.ledger = ledger + server.rpc = ReorgRpc( + tip=tip, + template=gbt_template(tip, height=11), + height=10, + block_hashes={10: tip}, + ) + server._ensure_job_cache_state() + + sentinel_summary: dict[str, object] = {"sentinel": True} + flight = _ReconcileFlight() + flight.summary = sentinel_summary + flight.event.set() + with server._ensure_reorg_reconciler_service()._reconcile_flight_lock: + server._ensure_reorg_reconciler_service()._reconcile_flights[tip] = flight + + joined = server.reconcile_prism_pool_blocks_once(tip_hash=tip) + self.assertEqual(joined, sentinel_summary) + self.assertIsNot(joined, sentinel_summary) + self.assertEqual(ledger.events, []) + + forced = server.reconcile_prism_pool_blocks_once( + tip_hash=tip, + _force_publish=True, + ) + self.assertEqual(forced["published_generation"], 1) + self.assertEqual(ledger.events, [("watch", 10), ("mature", 10)]) + + reserved = server.reconcile_prism_pool_blocks_once( + tip_hash=tip, + _source_reserved=True, + ) + self.assertIsNone(reserved["published_generation"]) + self.assertEqual( + ledger.events, + [("watch", 10), ("mature", 10), ("watch", 10), ("mature", 10)], + ) + + with server._ensure_reorg_reconciler_service()._reconcile_flight_lock: + server._ensure_reorg_reconciler_service()._reconcile_flights.pop(tip, None) + + def test_pass_spanning_detection_cycle_does_not_arm_memo(self) -> None: + # A pass whose reads straddle a flip away and back finishes with the + # latest detected hash matching its tip again, but its proof belongs + # to the closed epoch: arming must be refused so every memo consumer + # (refresh joins, initial-job and vardiff-idle builds) re-proves. + tip = "5c" * 32 + server = coordinator() + server.reorg_reconciler_enabled = True + ledger = ReorgLedger([]) + + def epoch_bumping_watch(*, active_tip_height: int) -> list[dict[str, object]]: + with server.lock: + server.tip_detection_epoch = ( + int(getattr(server, "tip_detection_epoch", 0)) + 2 + ) + return [] + + ledger.reorg_watch_blocks = epoch_bumping_watch # type: ignore[method-assign] + server.ledger = ledger + server.rpc = ReorgRpc( + tip=tip, + template=gbt_template(tip, height=11), + height=10, + block_hashes={10: tip}, + ) + + self.assertTrue(server.ensure_reorg_reconciled_for_tip(tip)) + + with server.lock: + self.assertNotIn(tip, server._ensure_reorg_reconciler_service()._reorg_reconcile_trusted_memo) + + def test_row_mutating_pass_evicts_other_tip_memo_entries(self) -> None: + tip_a = "a1" * 32 + tip_b = "b2" * 32 + pool_block_hash = "cd" * 32 + server = coordinator() + server.reorg_reconciler_enabled = True + ledger = ReorgLedger([]) + server.ledger = ledger + server.rpc = ReorgRpc( + tip=tip_b, + template=gbt_template(tip_b, height=11), + height=10, + block_hashes={10: tip_b}, + ) + server._ensure_job_cache_state() + server._note_reorg_reconcile_outcome(tip_a, trusted=True) + + # A pass for another tip that mutates no rows leaves tip A's cached + # proof valid: rows and tip-A chain state are both unchanged. + first = server.reconcile_prism_pool_blocks_once(tip_hash=tip_b) + self.assertEqual(first["published_generation"], 1) + with server.lock: + self.assertIn(tip_a, server._ensure_reorg_reconciler_service()._reorg_reconcile_trusted_memo) + self.assertIn(tip_b, server._ensure_reorg_reconciler_service()._reorg_reconcile_trusted_memo) + + # A pass that applies orphan/maturity row mutations ends every other + # tip's memo epoch, even though tip B was never observed by + # observe_tip_for_refresh. + ledger.rows.append( + { + "block_hash": pool_block_hash, + "block_height": 12, + "chain_state": "confirmed", + "maturity_state": "immature", + } + ) + second = server.reconcile_prism_pool_blocks_once(tip_hash=tip_b) + self.assertEqual(second["inactive_blocks"], 1) + with server.lock: + self.assertNotIn(tip_a, server._ensure_reorg_reconciler_service()._reorg_reconcile_trusted_memo) + self.assertIn(tip_b, server._ensure_reorg_reconciler_service()._reorg_reconcile_trusted_memo) + + def test_coordination_blocked_default_budget_is_fifteen_minutes(self) -> None: + server = PrismCoordinator.__new__(PrismCoordinator) + server._record_coordination_blocked_refresh(100.0) + + self.assertFalse(server.coordination_blocked_streak_expired(999.999)) + self.assertTrue(server.coordination_blocked_streak_expired(1000.0)) + + def test_sustained_blocked_refresh_storm_exits_via_coordination_budget(self) -> None: + server = coordinator() + server.blockpoll_seconds = 0 + server.template_refresh_failure_exit_seconds = 10.0 + server.coordination_blocked_exit_seconds = 10.0 + server._record_heartbeat = lambda _name: None # type: ignore[method-assign] + server.rpc = TipRpc("11" * 32) + clock = {"now": 100.0} + blocked_polls = 0 + + def blocked_fetch() -> QbitTipTemplateSnapshot: + nonlocal blocked_polls + blocked_polls += 1 + clock["now"] += 6.0 + if blocked_polls >= 4: + server.stop_event.set() + if blocked_polls % 2: + raise TemplateRefreshSuperseded( + "qbit tip changed during sequential refresh; immediate retry scheduled" + ) + raise _PayoutStatePublicationBlocked( + "payout state invalidation is pending publication" + ) + + server.fetch_qbit_tip_template_snapshot = blocked_fetch # type: ignore[method-assign] + with ( + patch( + "lab.prism.prism_coordinator.time.monotonic", + side_effect=lambda: clock["now"], + ), + patch("lab.prism.prism_coordinator.traceback.print_exc"), + ): + server.blockpoll_loop() + + self.assertEqual(blocked_polls, 4) + self.assertTrue(server.coordination_blocked_streak_expired(clock["now"])) + self.assertIsNone( + getattr(server, "template_refresh_failure_started_monotonic", None) + ) + server.stop_event.clear() + server.watchdog_enabled = False + server.watchdog_interval_seconds = 0.001 + with ( + patch( + "lab.prism.prism_coordinator.time.monotonic", + return_value=clock["now"], + ), + patch( + "lab.prism.prism_coordinator.os._exit", + side_effect=SystemExit(1), + ) as exit_process, + patch("builtins.print"), + self.assertRaises(SystemExit), + ): + server.watchdog_loop() + + exit_process.assert_called_once_with(1) + + +class AcceptedParentPreviewDeliveryBoundaryTests(_VardiffSupportTestCase): + """#182: the three job_delivery TemplateRefreshBlocked boundaries. + + The typed preview-pending exception must coalesce onto the ordinary + coordination retry at each boundary while staying outside + ``job_build_failure_count`` and the ordinary template-refresh failure + budget. + """ + + def _delivery_client(self, server: PrismCoordinator) -> ClientState: + state = client() + state.username = "miner-a" + state.worker = worker_identity() + state.send = lambda _payload: None # type: ignore[method-assign] + server.clients = {state} + return state + + @staticmethod + def _preview_pending() -> AcceptedParentPayoutPreviewPending: + return AcceptedParentPayoutPreviewPending( + "accepted parent payout preview is not ready yet", + parent_hash="cc" * 32, + waited_seconds=0.25, + timeout_count=3, + ) + + def test_reorg_boundary_propagates_preview_pending_unrelabelled(self) -> None: + # Boundary 2: the pre-build reorg-reconciliation guard. The generic + # clause below it converts failures into a plain TemplateRefreshBlocked + # (which is budgeted); the typed backpressure must pass through it + # unchanged instead. + server = coordinator() + state = self._delivery_client(server) + server.job_build_failure_count = 0 + raised = self._preview_pending() + + def blocked_reconcile() -> bool: + raise raised + + server.ensure_reorg_reconciled_for_current_tip = blocked_reconcile # type: ignore[method-assign] + server.build_job_for_client = lambda *_a, **_k: ( # type: ignore[method-assign] + self.fail("build must not run once the reorg guard blocks") + ) + + with ( + patch("builtins.print"), + self.assertRaises(AcceptedParentPayoutPreviewPending) as caught, + ): + server.maybe_send_job(state, clean_jobs=True) + + self.assertIs(caught.exception, raised) + self.assertEqual(server.job_build_failure_count, 0) + self.assertIsNone( + getattr(server, "template_refresh_failure_started_monotonic", None) + ) + + def test_build_boundary_retries_preview_pending_without_counting_it( + self, + ) -> None: + # Boundary 3: the client job-build boundary. Non-raising callers skip + # the job and retry; raising callers get the typed exception rather + # than the generic JobBuildFailed, and neither arms the failure count. + server = coordinator() + state = self._delivery_client(server) + server.job_build_failure_count = 0 + retries: list[int] = [] + server._schedule_tip_refresh_retry = lambda: retries.append(1) # type: ignore[method-assign] + server.ensure_reorg_reconciled_for_current_tip = lambda: True # type: ignore[method-assign] + raised = self._preview_pending() + + def blocked_build(*_args: object, **_kwargs: object) -> object: + raise raised + + server.build_job_for_client = blocked_build # type: ignore[method-assign] + + with patch("builtins.print") as logged: + self.assertFalse(server.maybe_send_job(state, clean_jobs=True)) + + self.assertEqual(server.job_build_failure_count, 0) + self.assertEqual(len(retries), 1) + # An explicit retry reason is logged instead of a build-failure + # traceback. + self.assertTrue( + any( + "accepted_parent_preview_pending" in str(call.args[0]) + for call in logged.call_args_list + if call.args + ) + ) + + # raise_on_build_failure must surface the typed backpressure, not the + # generic JobBuildFailed wrapper reserved for real build failures. + with ( + patch("builtins.print"), + self.assertRaises(AcceptedParentPayoutPreviewPending) as caught, + ): + server.maybe_send_job( + state, + clean_jobs=True, + raise_on_build_failure=True, + ) + + self.assertIs(caught.exception, raised) + self.assertEqual(server.job_build_failure_count, 0) + self.assertEqual(len(retries), 2) + + def test_build_boundary_still_counts_plain_build_failures(self) -> None: + # Negative control for boundary 3: an ordinary build failure keeps the + # generic accounting and the JobBuildFailed wrapper. + server = coordinator() + state = self._delivery_client(server) + server.job_build_failure_count = 0 + server.ensure_reorg_reconciled_for_current_tip = lambda: True # type: ignore[method-assign] + + def failing_build(*_args: object, **_kwargs: object) -> object: + raise RuntimeError("template build unavailable") + + server.build_job_for_client = failing_build # type: ignore[method-assign] + + with ( + patch("builtins.print"), + patch("lab.prism.job_delivery.traceback.print_exc"), + ): + self.assertFalse(server.maybe_send_job(state, clean_jobs=True)) + self.assertEqual(server.job_build_failure_count, 1) + + with ( + patch("builtins.print"), + patch("lab.prism.job_delivery.traceback.print_exc"), + self.assertRaises(JobBuildFailed), + ): + server.maybe_send_job( + state, + clean_jobs=True, + raise_on_build_failure=True, + ) + self.assertEqual(server.job_build_failure_count, 2) + + def test_sustained_preview_pending_exits_via_coordination_budget(self) -> None: + # The escalation that keeps this from retrying silently forever: the + # ordinary template-refresh budget stays disarmed, while the separate + # coordination-blocked streak expires and the watchdog exits. + server = coordinator() + server.blockpoll_seconds = 0 + server.template_refresh_failure_exit_seconds = 10.0 + server.coordination_blocked_exit_seconds = 10.0 + server._record_heartbeat = lambda _name: None # type: ignore[method-assign] + server.rpc = TipRpc("11" * 32) + clock = {"now": 100.0} + blocked_polls = 0 + + def blocked_fetch() -> QbitTipTemplateSnapshot: + nonlocal blocked_polls + blocked_polls += 1 + clock["now"] += 6.0 + if blocked_polls >= 4: + server.stop_event.set() + raise AcceptedParentPayoutPreviewPending( + "accepted parent payout preview is not ready yet" + ) + + server.fetch_qbit_tip_template_snapshot = blocked_fetch # type: ignore[method-assign] + with ( + patch( + "lab.prism.prism_coordinator.time.monotonic", + side_effect=lambda: clock["now"], + ), + patch("lab.prism.prism_coordinator.traceback.print_exc"), + patch("builtins.print"), + ): + server.blockpoll_loop() + + self.assertEqual(blocked_polls, 4) + self.assertIsNone( + getattr(server, "template_refresh_failure_started_monotonic", None) + ) + self.assertTrue(server.coordination_blocked_streak_expired(clock["now"])) + + server.stop_event.clear() + server.watchdog_enabled = False + server.watchdog_interval_seconds = 0.001 + with ( + patch( + "lab.prism.prism_coordinator.time.monotonic", + return_value=clock["now"], + ), + patch( + "lab.prism.prism_coordinator.os._exit", + side_effect=SystemExit(1), + ) as exit_process, + patch("builtins.print"), + self.assertRaises(SystemExit), + ): + server.watchdog_loop() + + exit_process.assert_called_once_with(1) + + +class PrismCoordinatorReliabilityTests(_VardiffSupportTestCase): + def _bare_coordinator(self) -> PrismCoordinator: + server = PrismCoordinator.__new__(PrismCoordinator) + server.lock = threading.RLock() + server.stop_event = threading.Event() + server._heartbeats = {} + server._watchdog_pauses = {} + server._heartbeats_lock = threading.Lock() + server.watchdog_timeout_seconds = 120.0 + server.watchdog_interval_seconds = 15.0 + return server + + def test_blockwait_failure_loop_reuses_existing_stop_event(self) -> None: + server = self._bare_coordinator() + tip = "aa" * 32 + server.rpc = SimpleNamespace(call=lambda method: tip) + server.blockpoll_seconds = 0.0 + server.observe_tip_for_refresh = lambda _tip: None # type: ignore[method-assign] + attempts = 0 + + def fail_blockwait(_known_tip: str) -> str: + nonlocal attempts + attempts += 1 + if attempts == 3: + server.stop_event.set() + raise RuntimeError("transient waitfornewblock failure") + + server.blockwait_once = fail_blockwait # type: ignore[method-assign] + # Construct the R1 owner up front: its own event allocations are + # owner state, while the loop itself must reuse the existing + # stop_event rather than allocating a fallback. + server._ensure_tip_refresh_service() + + with ( + patch("builtins.print"), + patch("lab.prism.tip_refresh.traceback.print_exc"), + patch( + "lab.prism.tip_refresh.threading.Event", + side_effect=AssertionError("allocated fallback stop event"), + ), + patch( + "lab.prism.prism_coordinator.threading.Event", + side_effect=AssertionError("allocated fallback stop event"), + ), + ): + server.blockwait_loop() + + self.assertEqual(attempts, 3) + + def test_blockwait_parameter_mismatch_is_treated_as_unsupported(self) -> None: + self.assertTrue( + PrismCoordinator._blockwait_unsupported( + RuntimeError("RPC error -32602: invalid params: wrong number of parameters") + ) + ) + + def test_blockwait_advances_known_tip_before_notification_failure(self) -> None: + server = self._bare_coordinator() + tip_a = "aa" * 32 + tip_b = "bb" * 32 + server.rpc = SimpleNamespace(call=lambda method: tip_a) + server.blockpoll_seconds = 1.0 + known_tips: list[str] = [] + detected_tips: list[str] = [] + + def blockwait_once(known_tip: str) -> str: + known_tips.append(known_tip) + if len(known_tips) == 2: + server.stop_event.set() + return tip_b + + server.blockwait_once = blockwait_once # type: ignore[method-assign] + + def observe_tip_for_refresh(tip_hash: str, **_kwargs: object) -> bool: + detected_tips.append(tip_hash) + if tip_hash == tip_b: + raise TemplateRefreshBlocked("notification failed") + return True + + def reject_premature_publication(*_args: object, **_kwargs: object) -> bool: + raise AssertionError("blockwait must not publish submit authority") + + server.observe_tip_for_refresh = observe_tip_for_refresh # type: ignore[method-assign] + server.observe_tip_first_seen = reject_premature_publication # type: ignore[method-assign] + + with patch("builtins.print"), patch( + "lab.prism.prism_coordinator.traceback.print_exc" + ), patch.object( + server.stop_event, + "wait", + side_effect=lambda _timeout: server.stop_event.is_set(), + ): + server.blockwait_loop() + + self.assertEqual(known_tips, [tip_a, tip_b]) + self.assertEqual(detected_tips, [tip_a, tip_b]) + + def test_blockwait_unsupported_removes_watchdog_heartbeat(self) -> None: + server = coordinator() + server.rpc = UnsupportedBlockwaitRpc("00" * 32) + server._record_heartbeat("qbit_blockwait") + + server.blockwait_loop() + + self.assertNotIn("qbit_blockwait", server._heartbeats) + self.assertNotIn("qbit_blockwait", server._watchdog_pauses) + + +class PrismStampedJobFloorTests(_VardiffSupportTestCase): + """The listener floor must hold on the wire, not just in vardiff policy. + + Stamped jobs are the single choke point for every mining.set_difficulty + the coordinator sends, and marketplace verification judges the first one. + The regression here is a young chain: qbit network difficulty below the + high-diff floor used to drag the advertised difficulty down with it. + """ + + def stamp_coordinator(self) -> PrismCoordinator: + server = coordinator() + server.job_counter = 0 + server.share_weights_by_username = {} + server.default_share_weight = 1 + return server + + def cached_bundle(self) -> CachedJobBundle: + # bits 207fffff: regtest-grade network difficulty (~4.7e-10), far + # below the 500k marketplace floor. + qbit_target = target_from_compact("207fffff") + base_job = direct_stratum.DirectQbitStratumJob( + job_id="prism-template-base", + previousblockhash_display="00" * 32, + prevhash="00" * 32, + coinb1="", + coinb2="", + full_coinbase_prefix="", + full_coinbase_suffix="", + merkle_branch=(), + transaction_hexes=(), + version="20000000", + nbits="207fffff", + ntime="6553f100", + qbit_target=qbit_target, + share_target=qbit_target, + share_difficulty=Decimal("1"), + extranonce1_hex="ffffffff", + extranonce2_size=8, + clean_jobs=True, + ) + return CachedJobBundle( + key=("test",), + template=gbt_template("00" * 32), + template_fingerprint="fp", + coinbase_manifest={}, + shares_json=[], + prior_balances=[], + found_block={"network_difficulty": 1}, + collection_only=False, + issued_at_ms=12345, + base_job=base_job, + built_monotonic=time.monotonic(), + ) + + def highdiff_client(self) -> ClientState: + state = client() + state.worker = worker_identity() + state.listener_vardiff_config = highdiff_vardiff_config() + state.minimum_advertised_difficulty = Decimal("500000") + state.share_difficulty = Decimal("500000") + return state + + def test_ready_pool_refreshes_clients_left_on_collection_jobs(self) -> None: + # Once the pool crosses min_ready_miners, the poller must replace + # collection jobs with windowed work even when the template snapshot + # is otherwise unchanged -- readiness itself is invisible to the + # template fingerprint. + server, state, _ledger = submit_coordinator() + tip = "00" * 32 + snapshot = SimpleNamespace( + bestblockhash=tip, + previousblockhash=tip, + template_fingerprint="fp", + ) + context = SimpleNamespace( + template={"previousblockhash": tip}, + template_fingerprint="fp", + collection_only=True, + ) + state.active_job = context + + server.min_ready_miners = 1 + server.accepted_share_stats = lambda: (0, 0) # type: ignore[method-assign] + self.assertFalse(server.pool_readiness_latched()) + self.assertFalse(server.client_needs_tip_template_refresh(state, snapshot)) + + server.accepted_share_stats = lambda: (1, 1) # type: ignore[method-assign] + self.assertTrue(server.pool_readiness_latched()) + self.assertTrue(server.client_needs_tip_template_refresh(state, snapshot)) + + # Readiness is monotonic: once latched the ledger is never consulted + # again, and ready (non-collection) jobs still need no refresh. + server.accepted_share_stats = None # type: ignore[assignment] + self.assertTrue(server.pool_readiness_latched()) + context.collection_only = False + self.assertFalse(server.client_needs_tip_template_refresh(state, snapshot)) + + +class JobBundleCacheTests(_JobSupportTestCase): + @staticmethod + def _capture_error( + errors: list[BaseException], + operation: object, + ) -> None: + try: + operation() # type: ignore[operator] + except BaseException as exc: # noqa: BLE001 - test thread handoff + errors.append(exc) + + def test_ready_tip_refresh_builds_once_and_stamps_every_client(self) -> None: + server, _ = coordinator() + recorded = install_fake_bundle_builder(server) + clients = [client(1), client(2), client(3)] + clients[1].pending_share_difficulty = Decimal("8") + sent: dict[int, list[dict[str, object]]] = {state.connection_id: [] for state in clients} + for state in clients: + state.send = ( # type: ignore[method-assign] + lambda payload, connection_id=state.connection_id: sent[connection_id].append(payload) + ) + server.clients = set(clients) + + try: + refreshed = server.poll_qbit_tip_template_once() + finally: + server.shutdown_tip_refresh_executor() + + self.assertEqual(refreshed, 3) + self.assertEqual(recorded["calls"], 1) + contexts = [state.active_job for state in clients] + self.assertEqual(len({context.job.job_id for context in contexts}), 3) + self.assertEqual( + [context.job.extranonce1_hex for context in contexts], + [state.extranonce1_hex for state in clients], + ) + self.assertEqual(contexts[1].job.share_difficulty, Decimal("8")) + self.assertEqual( + [payload["method"] for payload in sent[2]], + ["mining.set_difficulty", "mining.notify"], + ) + metrics = server.metrics_payload() + self.assertIn('qbit_prism_tip_refresh_clients_total{result="sent"} 3', metrics) + self.assertIn("qbit_prism_tip_refresh_first_delivery_seconds_count 1", metrics) + self.assertIn("qbit_prism_tip_refresh_last_delivery_seconds_count 1", metrics) + + def test_ready_tip_refresh_shares_one_bundle_across_250_clients(self) -> None: + server, rpc = coordinator() + recorded = install_fake_bundle_builder(server) + server.reorg_reconciler_enabled = True + reconciled: list[str] = [] + trust_checks = 0 + + def reconcile_once(tip_hash: str) -> bool: + reconciled.append(tip_hash) + return True + + def chain_view_untrusted() -> bool: + nonlocal trust_checks + trust_checks += 1 + return False + + server.ensure_reorg_reconciled_for_tip = reconcile_once # type: ignore[method-assign] + server.qbit_chain_view_untrusted = chain_view_untrusted # type: ignore[method-assign] + server.ensure_reorg_reconciled_for_current_tip = ( # type: ignore[method-assign] + lambda **_kwargs: self.fail("fanout repeated current-tip validation") + ) + clients = [client(index + 1) for index in range(250)] + sent: dict[int, list[dict[str, object]]] = { + state.connection_id: [] for state in clients + } + for state in clients: + state.send = ( # type: ignore[method-assign] + lambda payload, connection_id=state.connection_id: sent[ + connection_id + ].append(payload) + ) + server.clients = set(clients) + + try: + refreshed = server.poll_qbit_tip_template_once() + finally: + server.shutdown_tip_refresh_executor() + + self.assertEqual(refreshed, 250) + self.assertEqual(recorded["calls"], 1) + cached = next(iter(server._job_bundle_cache.values())) + self.assertEqual( + len({id(state.active_job.shares_json) for state in clients}), + 1, + ) + self.assertTrue( + all(state.active_job.shares_json is cached.shares_json for state in clients) + ) + self.assertEqual(reconciled, [rpc.tip]) + self.assertEqual(trust_checks, 2) + # The early priority probe, snapshot coherence, pre-fanout validation, + # and post-fanout detection are each constant-cost regardless of + # client count. + self.assertEqual(rpc.count("getbestblockhash"), 4) + self.assertTrue(all(len(payloads) == 2 for payloads in sent.values())) + fingerprints = {state.active_job.template_fingerprint for state in clients} + self.assertEqual(len(fingerprints), 1) + + def test_supersession_retry_wakes_blockpoll_without_full_interval(self) -> None: + server, _ = coordinator() + server.blockpoll_seconds = 60.0 + server._ensure_tip_refresh_state() + poll_called = threading.Event() + + def poll_once() -> int: + poll_called.set() + server.stop_event.set() + return 0 + + server.poll_qbit_tip_template_once = poll_once # type: ignore[method-assign] + thread = threading.Thread(target=server.blockpoll_loop) + thread.start() + try: + server._schedule_tip_refresh_retry() + self.assertTrue(poll_called.wait(1)) + finally: + server.stop_event.set() + server._schedule_tip_refresh_retry() + thread.join(1) + + self.assertFalse(thread.is_alive()) + + def test_ready_tip_refresh_respects_executor_bound(self) -> None: + server, _ = coordinator() + install_fake_bundle_builder(server) + server.tip_refresh_max_workers = 2 + clients = [client(index + 1) for index in range(6)] + server.clients = set(clients) + release = threading.Event() + two_started = threading.Event() + counter_lock = threading.Lock() + active = 0 + maximum = 0 + + def send(payload: dict[str, object]) -> None: + nonlocal active, maximum + if payload["method"] != "mining.notify": + return + with counter_lock: + active += 1 + maximum = max(maximum, active) + if active == 2: + two_started.set() + try: + self.assertTrue(release.wait(5)) + finally: + with counter_lock: + active -= 1 + + for state in clients: + state.send = send # type: ignore[method-assign] + result: list[int] = [] + thread = threading.Thread(target=lambda: result.append(server.poll_qbit_tip_template_once())) + thread.start() + try: + self.assertTrue(two_started.wait(5)) + self.assertLessEqual(maximum, 2) + finally: + release.set() + thread.join(5) + server.shutdown_tip_refresh_executor() + + self.assertFalse(thread.is_alive()) + self.assertEqual(result, [6]) + self.assertEqual(maximum, 2) + + def test_blocked_socket_does_not_delay_another_client(self) -> None: + server, _ = coordinator() + install_fake_bundle_builder(server) + server.tip_refresh_max_workers = 2 + blocked = client(1) + healthy = client(2) + server.clients = {blocked, healthy} + blocked_started = threading.Event() + healthy_delivered = threading.Event() + release = threading.Event() + + def blocked_send(payload: dict[str, object]) -> None: + if payload["method"] == "mining.notify": + blocked_started.set() + self.assertTrue(release.wait(5)) + + def healthy_send(payload: dict[str, object]) -> None: + if payload["method"] == "mining.notify": + healthy_delivered.set() + + blocked.send = blocked_send # type: ignore[method-assign] + healthy.send = healthy_send # type: ignore[method-assign] + result: list[int] = [] + thread = threading.Thread(target=lambda: result.append(server.poll_qbit_tip_template_once())) + thread.start() + try: + self.assertTrue(blocked_started.wait(5)) + self.assertTrue(healthy_delivered.wait(5)) + finally: + release.set() + thread.join(5) + server.shutdown_tip_refresh_executor() + + self.assertEqual(result, [2]) + + def test_shutdown_drains_inflight_tip_refresh_worker(self) -> None: + server, _ = coordinator() + install_fake_bundle_builder(server) + server.tip_refresh_max_workers = 1 + state = client(1) + server.clients = {state} + worker_started = threading.Event() + worker_send_finished = threading.Event() + release_worker = threading.Event() + shutdown_complete = threading.Event() + poll_errors: list[BaseException] = [] + + def blocked_send(payload: dict[str, object]) -> None: + if payload["method"] != "mining.notify": + return + worker_started.set() + try: + self.assertTrue(release_worker.wait(5)) + finally: + worker_send_finished.set() + + def poll() -> None: + try: + server.poll_qbit_tip_template_once() + except BaseException as exc: # noqa: BLE001 - surface to the test + poll_errors.append(exc) + + def shutdown() -> None: + server.shutdown_tip_refresh_executor() + shutdown_complete.set() + + state.send = blocked_send # type: ignore[method-assign] + poll_thread = threading.Thread(target=poll) + shutdown_thread = threading.Thread(target=shutdown) + poll_thread.start() + try: + self.assertTrue(worker_started.wait(5)) + server.stop_event.set() + shutdown_thread.start() + self.assertFalse(shutdown_complete.wait(0.05)) + self.assertFalse(worker_send_finished.is_set()) + finally: + release_worker.set() + shutdown_thread.join(5) + poll_thread.join(5) + + self.assertFalse(shutdown_thread.is_alive()) + self.assertFalse(poll_thread.is_alive()) + self.assertTrue(shutdown_complete.is_set()) + self.assertTrue(worker_send_finished.is_set()) + self.assertEqual(poll_errors, []) + self.assertEqual(server.tip_refresh_inflight, 0) + with self.assertRaisesRegex(RuntimeError, "executor is shut down"): + server.tip_refresh_executor() + + def test_queued_fanout_stops_when_chain_view_becomes_untrusted(self) -> None: + server, _ = coordinator() + install_fake_bundle_builder(server) + server.reorg_reconciler_enabled = True + server.ensure_reorg_reconciled_for_tip = lambda _tip: True # type: ignore[method-assign] + trust_checks = 0 + + def chain_view_untrusted() -> bool: + nonlocal trust_checks + trust_checks += 1 + return True + + server.qbit_chain_view_untrusted = chain_view_untrusted # type: ignore[method-assign] + first = client(1) + second = client(2) + server.clients = [first, second] # type: ignore[assignment] + sent: list[dict[str, object]] = [] + first.send = sent.append # type: ignore[method-assign] + second.send = sent.append # type: ignore[method-assign] + try: + with self.assertRaisesRegex(TemplateRefreshBlocked, "became untrusted"): + server.poll_qbit_tip_template_once() + finally: + server.shutdown_tip_refresh_executor() + + self.assertEqual(trust_checks, 1) + self.assertEqual(sent, []) + self.assertIsNone(first.active_job) + self.assertIsNone(second.active_job) + + def test_queued_fanout_stops_when_live_tip_changes(self) -> None: + server, rpc = coordinator() + install_fake_bundle_builder(server) + server.tip_refresh_max_workers = 1 + first = client(1) + second = client(2) + server.clients = [first, second] # type: ignore[assignment] + first_blocked = threading.Event() + release_first = threading.Event() + second_sent: list[dict[str, object]] = [] + + def first_send(payload: dict[str, object]) -> None: + if payload["method"] == "mining.notify": + first_blocked.set() + self.assertTrue(release_first.wait(5)) + + first.send = first_send # type: ignore[method-assign] + second.send = second_sent.append # type: ignore[method-assign] + refreshed: list[int] = [] + errors: list[BaseException] = [] + + def poll() -> None: + try: + refreshed.append(server.poll_qbit_tip_template_once()) + except BaseException as exc: # noqa: BLE001 - surface to the test + errors.append(exc) + + thread = threading.Thread(target=poll) + thread.start() + try: + self.assertTrue(first_blocked.wait(5)) + rpc.tip = "33" * 32 + finally: + release_first.set() + thread.join(5) + server.shutdown_tip_refresh_executor() + + self.assertFalse(thread.is_alive()) + self.assertEqual(refreshed, []) + self.assertEqual(len(errors), 1) + self.assertIsInstance(errors[0], TemplateRefreshBlocked) + self.assertIn("immediate retry scheduled", str(errors[0])) + self.assertTrue(server._tip_refresh_retry.is_set()) + self.assertIsNotNone(first.active_job) + self.assertIsNotNone(second.active_job) + self.assertEqual( + [payload["method"] for payload in second_sent], + ["mining.set_difficulty", "mining.notify"], + ) + + def test_multiworker_cancel_releases_client_lock_while_draining_peer(self) -> None: + server, _ = coordinator() + install_fake_bundle_builder(server) + server.tip_refresh_max_workers = 1 + admitted = client(1) + queued = client(2) + server.clients = [admitted, queued] # type: ignore[assignment] + admitted_send_started = threading.Event() + release_admitted_send = threading.Event() + queued_sent: list[dict[str, object]] = [] + + def admitted_send(payload: dict[str, object]) -> None: + if payload["method"] == "mining.notify": + admitted_send_started.set() + self.assertTrue(release_admitted_send.wait(5)) + + admitted.send = admitted_send # type: ignore[method-assign] + queued.send = queued_sent.append # type: ignore[method-assign] + refreshed: list[int] = [] + errors: list[BaseException] = [] + + def poll() -> None: + try: + refreshed.append(server.poll_qbit_tip_template_once()) + except BaseException as exc: # noqa: BLE001 - surface to the test + errors.append(exc) + + thread = threading.Thread(target=poll) + thread.start() + try: + self.assertTrue(admitted_send_started.wait(5)) + server.observe_tip_for_refresh("33" * 32) + lock_acquired = queued.job_update_lock.acquire(timeout=0.1) + self.assertTrue(lock_acquired) + if lock_acquired: + queued.job_update_lock.release() + # The coordinator still waits for the admitted peer delivery, but + # queued workers observe cancellation without taking client state. + self.assertTrue(thread.is_alive()) + self.assertIsNone(queued.active_job) + finally: + release_admitted_send.set() + thread.join(5) + server.shutdown_tip_refresh_executor() + + self.assertFalse(thread.is_alive()) + self.assertEqual(refreshed, []) + self.assertEqual(len(errors), 1) + self.assertIsInstance(errors[0], TemplateRefreshBlocked) + self.assertIsNotNone(admitted.active_job) + self.assertIsNone(queued.active_job) + self.assertEqual(queued_sent, []) + + def test_same_tip_cache_refresh_during_fanout_does_not_abort(self) -> None: + server, rpc = coordinator() + install_fake_bundle_builder(server) + server.tip_refresh_max_workers = 1 + first = client(1) + second = client(2) + # Preserve task order so the cache replacement happens after one + # delivery and before the next worker task starts. + server.clients = [first, second] # type: ignore[assignment] + first_blocked = threading.Event() + release_first = threading.Event() + second_sent: list[dict[str, object]] = [] + + def first_send(payload: dict[str, object]) -> None: + if payload["method"] == "mining.notify": + first_blocked.set() + self.assertTrue(release_first.wait(5)) + + first.send = first_send # type: ignore[method-assign] + second.send = second_sent.append # type: ignore[method-assign] + refreshed: list[int] = [] + errors: list[BaseException] = [] + + def poll() -> None: + try: + refreshed.append(server.poll_qbit_tip_template_once()) + except BaseException as exc: # noqa: BLE001 - surface to the test + errors.append(exc) + + thread = threading.Thread(target=poll) + thread.start() + try: + self.assertTrue(first_blocked.wait(5)) + replacement = dict(rpc.template) + replacement["coinbasevalue"] = int(replacement["coinbasevalue"]) + 1 + replacement_artifacts = server.store_template_artifacts(replacement) + self.assertIsNotNone(replacement_artifacts) + self.assertNotEqual( + replacement_artifacts.fingerprint, + first.active_job.template_fingerprint, + ) + finally: + release_first.set() + thread.join(5) + server.shutdown_tip_refresh_executor() + + self.assertFalse(thread.is_alive()) + self.assertEqual(errors, []) + self.assertEqual(refreshed, [2]) + self.assertIsNotNone(server.last_successful_template_refresh_monotonic) + self.assertEqual( + [payload["method"] for payload in second_sent], + ["mining.set_difficulty", "mining.notify"], + ) + + def test_queued_fanout_does_not_overwrite_intervening_job(self) -> None: + server, rpc = coordinator() + install_fake_bundle_builder(server) + server.tip_refresh_max_workers = 1 + first = client(1) + second = client(2) + server.clients = [first, second] # type: ignore[assignment] + first_blocked = threading.Event() + release_first = threading.Event() + second_sent: list[dict[str, object]] = [] + + def first_send(payload: dict[str, object]) -> None: + if payload["method"] == "mining.notify": + first_blocked.set() + self.assertTrue(release_first.wait(5)) + + first.send = first_send # type: ignore[method-assign] + second.send = second_sent.append # type: ignore[method-assign] + refreshed: list[int] = [] + errors: list[BaseException] = [] + + def poll() -> None: + try: + refreshed.append(server.poll_qbit_tip_template_once()) + except BaseException as exc: # noqa: BLE001 - surface to the test + errors.append(exc) + + thread = threading.Thread(target=poll) + thread.start() + try: + self.assertTrue(first_blocked.wait(5)) + replacement = dict(rpc.template) + replacement["coinbasevalue"] = int(replacement["coinbasevalue"]) + 1 + replacement_artifacts = server.store_template_artifacts(replacement) + self.assertIsNotNone(replacement_artifacts) + self.assertTrue(server.maybe_send_job(second, clean_jobs=False)) + intervening_job = second.active_job + self.assertEqual( + intervening_job.template_fingerprint, + replacement_artifacts.fingerprint, + ) + self.assertGreater( + intervening_job.template_generation, + first.active_job.template_generation, + ) + finally: + release_first.set() + thread.join(5) + server.shutdown_tip_refresh_executor() + + self.assertFalse(thread.is_alive()) + self.assertEqual(errors, []) + self.assertEqual(refreshed, [1]) + self.assertIs(second.active_job, intervening_job) + self.assertEqual( + [payload["method"] for payload in second_sent], + ["mining.set_difficulty", "mining.notify"], + ) + + def test_queued_fanout_replaces_stale_intervening_job(self) -> None: + server, rpc = coordinator() + install_fake_bundle_builder(server) + server.tip_refresh_max_workers = 1 + # The poll must observe the same-tip template rotation immediately for + # the queued-fanout replacement race below; disable the same-tip + # template reuse window so the rotation is fetched, not deferred. + server.template_cache_seconds = 0.0 + first = client(1) + second = client(2) + server.clients = [first, second] # type: ignore[assignment] + old_artifacts = server.store_template_artifacts(dict(rpc.template)) + self.assertIsNotNone(old_artifacts) + assert old_artifacts is not None + assert second.worker is not None + old_bundle = server.shared_job_bundle(old_artifacts, second.worker) + refreshed_template = dict(rpc.template) + refreshed_template["coinbasevalue"] = int( + refreshed_template["coinbasevalue"] + ) + 1 + rpc.template = refreshed_template + first_blocked = threading.Event() + release_first = threading.Event() + second_sent: list[dict[str, object]] = [] + + def first_send(payload: dict[str, object]) -> None: + if payload["method"] == "mining.notify": + first_blocked.set() + self.assertTrue(release_first.wait(5)) + + first.send = first_send # type: ignore[method-assign] + second.send = second_sent.append # type: ignore[method-assign] + refreshed: list[int] = [] + errors: list[BaseException] = [] + + def poll() -> None: + try: + refreshed.append(server.poll_qbit_tip_template_once()) + except BaseException as exc: # noqa: BLE001 - surface to the test + errors.append(exc) + + thread = threading.Thread(target=poll) + thread.start() + try: + self.assertTrue(first_blocked.wait(5)) + with second.job_update_lock, server.lock: + stale_intervening_job = server.stamp_job_for_client( + second, + old_bundle, + clean_jobs=False, + ) + second.active_job = stale_intervening_job + second.active_job_ids.add(stale_intervening_job.job.job_id) + server.jobs[stale_intervening_job.job.job_id] = stale_intervening_job + finally: + release_first.set() + thread.join(5) + server.shutdown_tip_refresh_executor() + + self.assertFalse(thread.is_alive()) + self.assertEqual(errors, []) + self.assertEqual(refreshed, [2]) + self.assertIsNot(second.active_job, stale_intervening_job) + self.assertEqual( + second.active_job.template_fingerprint, + qbit_template_fingerprint(refreshed_template), + ) + self.assertGreater( + second.active_job.template_generation, + stale_intervening_job.template_generation, + ) + self.assertEqual( + [payload["method"] for payload in second_sent], + ["mining.set_difficulty", "mining.notify"], + ) + + def test_newer_template_does_not_supersede_current_payout_refresh(self) -> None: + server, rpc = coordinator() + install_fake_bundle_builder(server) + state = client(1) + artifacts = server.store_template_artifacts(dict(rpc.template)) + self.assertIsNotNone(artifacts) + assert artifacts is not None and state.worker is not None + stale_bundle = server.shared_job_bundle(artifacts, state.worker) + snapshot = server.fetch_qbit_tip_template_snapshot() + stale_intervening_job = dataclass_replace( + server.stamp_job_for_client( + state, + stale_bundle, + clean_jobs=False, + ), + template_generation=snapshot.template_generation + 1, + ) + server._advance_payout_state_generation() + + self.assertFalse( + server.intervening_job_supersedes_snapshot( + stale_intervening_job, + None, + snapshot, + ) + ) + current_intervening_job = dataclass_replace( + stale_intervening_job, + payout_state_generation=server._payout_state_generation, + ) + self.assertTrue( + server.intervening_job_supersedes_snapshot( + current_intervening_job, + None, + snapshot, + ) + ) + + def test_higher_generation_old_tip_does_not_supersede_new_tip_snapshot(self) -> None: + server, rpc = coordinator() + install_fake_bundle_builder(server) + state = client(1) + old_artifacts = server.store_template_artifacts(dict(rpc.template)) + self.assertIsNotNone(old_artifacts) + assert old_artifacts is not None and state.worker is not None + old_bundle = server.shared_job_bundle(old_artifacts, state.worker) + + new_tip = "22" * 32 + rpc.tip = new_tip + rpc.template = base_template(height=11, prevhash=new_tip) + snapshot = server.fetch_qbit_tip_template_snapshot() + old_tip_job = dataclass_replace( + server.stamp_job_for_client( + state, + old_bundle, + clean_jobs=False, + ), + template_generation=snapshot.template_generation + 1, + ) + + self.assertNotEqual( + old_tip_job.template_fingerprint, + snapshot.template_fingerprint, + ) + self.assertFalse( + server.intervening_job_supersedes_snapshot( + old_tip_job, + None, + snapshot, + ) + ) + + def test_broken_socket_disconnects_only_that_client(self) -> None: + server, _ = coordinator() + install_fake_bundle_builder(server) + broken = client(1) + healthy = client(2) + server.clients = {broken, healthy} + healthy_sent: list[dict[str, object]] = [] + disconnected: list[ClientState] = [] + broken.send = lambda _payload: (_ for _ in ()).throw(OSError("closed")) # type: ignore[method-assign] + healthy.send = healthy_sent.append # type: ignore[method-assign] + server.disconnect_client = disconnected.append # type: ignore[method-assign] + + try: + refreshed = server.poll_qbit_tip_template_once() + finally: + server.shutdown_tip_refresh_executor() + + self.assertEqual(refreshed, 1) + self.assertEqual(disconnected, [broken]) + self.assertEqual( + [payload["method"] for payload in healthy_sent], + ["mining.set_difficulty", "mining.notify"], + ) + + def test_client_removed_before_pending_task_runs_is_skipped(self) -> None: + server, _ = coordinator() + install_fake_bundle_builder(server) + server.tip_refresh_max_workers = 1 + first = client(1) + removed = client(2) + clients = [first, removed] + server.clients = set(clients) + snapshot = server.fetch_qbit_tip_template_snapshot() + server.observe_tip_first_seen(snapshot.bestblockhash) + server.pool_readiness_latched() + server.tip_template_snapshot = snapshot + bundle = server.prepare_tip_refresh_bundle(snapshot) + blocked = threading.Event() + release = threading.Event() + + def first_send(payload: dict[str, object]) -> None: + if payload["method"] == "mining.notify": + blocked.set() + self.assertTrue(release.wait(5)) + + first.send = first_send # type: ignore[method-assign] + removed.send = lambda _payload: self.fail("removed client received a job") # type: ignore[method-assign] + result: list[tuple[int, float | None, float | None, int]] = [] + thread = threading.Thread( + target=lambda: result.append( + server._fanout_prepared_tip_refresh( + clients, + bundle, + snapshot, + heartbeat_name="qbit_blockpoll", + ) + ) + ) + thread.start() + try: + self.assertTrue(blocked.wait(5)) + with server.lock: + server.clients.remove(removed) + finally: + release.set() + thread.join(5) + server.shutdown_tip_refresh_executor() + + self.assertEqual(result[0][0], 1) + self.assertIsNone(removed.active_job) + self.assertEqual(removed.active_job_ids, set()) + + def test_template_fingerprint_race_uses_snapshot_owned_artifacts(self) -> None: + server, _ = coordinator() + install_fake_bundle_builder(server) + states = [client(1), client(2)] + sent: list[dict[str, object]] = [] + for state in states: + state.send = sent.append # type: ignore[method-assign] + server.clients = set(states) + original_shared_job_bundle = server.shared_job_bundle + race_calls = 0 + + def race_artifacts( + artifacts: object, + identity: WorkerIdentity | None = None, + **kwargs: object, + ) -> object: + nonlocal race_calls + race_calls += 1 + bundle = original_shared_job_bundle(artifacts, identity, **kwargs) + with server._job_cache_lock: + server._template_artifacts = dataclass_replace( + server._template_artifacts, + fingerprint="ff" * 32, + ) + return bundle + + server.shared_job_bundle = race_artifacts # type: ignore[method-assign] + + refreshed = server.poll_qbit_tip_template_once() + + self.assertEqual(race_calls, 1) + self.assertEqual(refreshed, 2) + self.assertEqual(len(sent), 4) + self.assertIsNotNone(server.tip_template_snapshot) + snapshot = server.tip_template_snapshot + assert snapshot is not None and snapshot.template_artifacts is not None + for state in states: + self.assertIs(state.active_job.template, snapshot.template_artifacts.template) + self.assertEqual( + state.active_job.template_fingerprint, + snapshot.template_fingerprint, + ) + + +class ClientCleanupTests(_JobSupportTestCase): + def test_disconnect_retires_and_closes_before_job_lock_cleanup(self) -> None: + server, _ = coordinator() + state = client(1) + server.clients = {state} + socket_closed = threading.Event() + state.close = socket_closed.set # type: ignore[method-assign] + state.job_update_lock.acquire() + disconnect = threading.Thread(target=server.disconnect_client, args=(state,)) + try: + disconnect.start() + self.assertTrue(socket_closed.wait(5)) + with server.lock: + self.assertNotIn(state, server.clients) + self.assertTrue(state.closing) + self.assertTrue(disconnect.is_alive()) + finally: + state.job_update_lock.release() + disconnect.join(5) + + self.assertFalse(disconnect.is_alive()) + + def test_disconnect_during_prepared_refresh_skips_without_job_state(self) -> None: + server, _ = coordinator() + install_fake_bundle_builder(server) + state = client(1) + observed_lock = ObservedRLock() + state.job_update_lock = observed_lock # type: ignore[assignment] + server.clients = {state} + snapshot = server.fetch_qbit_tip_template_snapshot() + server.observe_tip_first_seen(snapshot.bestblockhash) + server.pool_readiness_latched() + server.tip_template_snapshot = snapshot + bundle = server.prepare_tip_refresh_bundle(snapshot) + state.send = lambda _payload: self.fail( # type: ignore[method-assign] + "disconnected client received prepared work" + ) + socket_closed = threading.Event() + state.close = socket_closed.set # type: ignore[method-assign] + results: list[tuple[int, float | None, float | None, int]] = [] + errors: list[BaseException] = [] + + def refresh() -> None: + try: + results.append( + server._fanout_prepared_tip_refresh( + [state], + bundle, + snapshot, + heartbeat_name="qbit_blockpoll", + ) + ) + except BaseException as exc: # noqa: BLE001 - surface thread failures + errors.append(exc) + + observed_lock.acquire() + observed_lock.observe_acquires = True + refresh_thread = threading.Thread(target=refresh) + disconnect_thread = threading.Thread( + target=server.disconnect_client, + args=(state,), + ) + try: + refresh_thread.start() + self.assertTrue(observed_lock.acquire_attempted.wait(5)) + disconnect_thread.start() + self.assertTrue(socket_closed.wait(5)) + refresh_thread.join(5) + self.assertFalse(refresh_thread.is_alive()) + self.assertTrue(disconnect_thread.is_alive()) + finally: + observed_lock.release() + refresh_thread.join(5) + disconnect_thread.join(5) + server.shutdown_tip_refresh_executor() + + self.assertFalse(disconnect_thread.is_alive()) + self.assertEqual(errors, []) + self.assertEqual(results[0][0], 0) + self.assertIsNone(state.active_job) + self.assertEqual(state.active_job_ids, set()) + self.assertEqual(server.jobs, {}) + + def test_mass_disconnect_releases_active_connection_accounting(self) -> None: + server, _ = coordinator() + states = [client(index) for index in range(1, 129)] + for state in states: + state.close = lambda: None # type: ignore[method-assign] + server.clients = set(states) + + for state in states: + server.disconnect_client(state) + + with server.lock: + self.assertEqual(len(server.clients), 0) + self.assertTrue(all(state.closing for state in states)) + + def test_concurrent_disconnect_is_idempotent_and_deadlock_free(self) -> None: + server, _ = coordinator() + state = client(1) + server.clients = {state} + close_count = 0 + close_count_lock = threading.Lock() + caller_count = 16 + start = threading.Barrier(caller_count + 1) + errors: list[BaseException] = [] + + def close() -> None: + nonlocal close_count + with close_count_lock: + close_count += 1 + + def disconnect() -> None: + try: + start.wait() + server.disconnect_client(state) + except BaseException as exc: # noqa: BLE001 - surface thread failures + errors.append(exc) + + state.close = close # type: ignore[method-assign] + callers = [threading.Thread(target=disconnect) for _ in range(caller_count)] + for caller in callers: + caller.start() + start.wait() + for caller in callers: + caller.join(5) + + self.assertTrue(all(not caller.is_alive() for caller in callers)) + self.assertEqual(errors, []) + self.assertEqual(close_count, 1) + self.assertNotIn(state, server.clients) + + def test_disconnect_removes_active_and_evicted_job_contexts(self) -> None: + server, _ = coordinator() + install_fake_bundle_builder(server) + state = client(1) + active = server.build_job_for_client(state, clean_jobs=True) + evicted = server.build_job_for_client(state, clean_jobs=False) + active_id = active.job.job_id + evicted_id = evicted.job.job_id + state.active_job = active + state.active_job_ids = {active_id} + server.jobs = {active_id: active, evicted_id: evicted} + server.bury_evicted_job(state, evicted_id) + server.jobs.pop(evicted_id) + server.clients = {state} + state.close = lambda: None # type: ignore[method-assign] + + server.disconnect_client(state) + + self.assertIsNone(state.active_job) + self.assertEqual(state.active_job_ids, set()) + self.assertNotIn(active_id, server.jobs) + # Same-tip contexts survive the disconnect (detached from the dead + # connection object) so a same-username reconnect can still submit + # in-flight work; they age out on the same-tip retention TTL. + for job_id in (active_id, evicted_id): + entry = server.evicted_job_graveyard.get(job_id) + self.assertIsNotNone(entry) + assert entry is not None + self.assertIsNone(entry.client) + self.assertIn(job_id, server._disconnected_evicted_job_ids) + + def test_reconnect_storm_leaves_no_handler_threads_or_ghost_clients(self) -> None: + server, _ = coordinator() + connection_count = 32 + start = threading.Barrier(connection_count + 1) + peers: list[socket.socket] = [] + handlers: list[threading.Thread] = [] + + def handle(state: ClientState) -> None: + start.wait() + server.handle_client(state) + + for connection_id in range(1, connection_count + 1): + coordinator_socket, peer_socket = socket.socketpair() + state = client(connection_id) + state.sock = coordinator_socket + server.clients.add(state) + peers.append(peer_socket) + handler = threading.Thread( + target=handle, + args=(state,), + name=f"prism-test-handler-{connection_id}", + ) + handlers.append(handler) + handler.start() + + start.wait() + for peer in peers: + peer.close() + for handler in handlers: + handler.join(5) + + self.assertTrue(all(not handler.is_alive() for handler in handlers)) + with server.lock: + self.assertEqual(server.clients, set()) + + def test_disconnect_purges_evicted_contexts_when_retention_disabled( + self, + ) -> None: + server, _ = coordinator() + install_fake_bundle_builder(server) + server.disconnected_job_retention = 0 + state = client(1) + active = server.build_job_for_client(state, clean_jobs=True) + evicted = server.build_job_for_client(state, clean_jobs=False) + active_id = active.job.job_id + evicted_id = evicted.job.job_id + state.active_job = active + state.active_job_ids = {active_id} + server.jobs = {active_id: active, evicted_id: evicted} + server.bury_evicted_job(state, evicted_id) + server.jobs.pop(evicted_id) + server.clients = {state} + state.close = lambda: None # type: ignore[method-assign] + + server.disconnect_client(state) + + self.assertIsNone(state.active_job) + self.assertEqual(state.active_job_ids, set()) + self.assertNotIn(active_id, server.jobs) + self.assertNotIn(active_id, server.evicted_job_graveyard) + self.assertNotIn(evicted_id, server.evicted_job_graveyard) + self.assertNotIn(state.connection_id, server.evicted_jobs_by_connection) diff --git a/tests/test_prism_tip_refresh_epoch.py b/tests/test_prism_tip_refresh_epoch.py index 6af78e945..d0e48b899 100644 --- a/tests/test_prism_tip_refresh_epoch.py +++ b/tests/test_prism_tip_refresh_epoch.py @@ -18,17 +18,19 @@ PendingInitialJob, PrismCoordinator, TipRefreshValidationToken, - _PayoutStatePublicationBlocked, _TipRefreshFanoutSuperseded, _TipRefreshTrustBlocked, ) -from tests.test_prism_coordinator_job_cache import ( +from lab.prism.payout_state import ( + PayoutStatePublicationBlocked as _PayoutStatePublicationBlocked, +) +from tests.prism_coordinator_test_support import ( base_template, client, coordinator, install_fake_bundle_builder, ) -from tests.test_prism_coordinator_vardiff import RecordingLedger +from tests.prism_vardiff_test_support import RecordingLedger TIP_A = "11" * 32 @@ -933,7 +935,7 @@ def test_initial_fence_miss_completes_request_with_newer_work(self) -> None: connection_id=state.connection_id, difficulty_generation=0, ) - server.pending_initial_jobs[state] = request + server.pending_initial_jobs = {state: request} self.assertIs( server._deliver_initial_bundle(request, artifacts, bundle), diff --git a/tests/test_prism_tip_refresh_validation.py b/tests/test_prism_tip_refresh_validation.py index eb03034e5..074b7f792 100644 --- a/tests/test_prism_tip_refresh_validation.py +++ b/tests/test_prism_tip_refresh_validation.py @@ -16,9 +16,11 @@ TemplateRefreshSuperseded, TipRefreshValidationToken, _FanoutCancellation, - _PayoutStateDeliveryGate, ) -from tests.test_prism_coordinator_job_cache import ( +from lab.prism.payout_state import ( + PayoutStateDeliveryGate as _PayoutStateDeliveryGate, +) +from tests.prism_coordinator_test_support import ( FakeLedger, FakeRpc, base_template, diff --git a/tests/test_prism_vardiff.py b/tests/test_prism_vardiff.py new file mode 100644 index 000000000..eb4ab501a --- /dev/null +++ b/tests/test_prism_vardiff.py @@ -0,0 +1,1416 @@ +#!/usr/bin/env python3 +"""Focused PRISM coordinator vardiff tests.""" +# ruff: noqa: F403, F405 + +from __future__ import annotations + +import unittest +from lab.prism.vardiff_service import VardiffService +from tests.prism_vardiff_test_support import * + + +class PrismCoordinatorVardiffTests(unittest.TestCase): + def test_vardiff_owner_initialization_is_single_flight(self) -> None: + server = coordinator() + server.__dict__.pop("_vardiff_service", None) + server.vardiff_idle_queue_depth = 7 + real_service_type = VardiffService + constructor_entered = threading.Event() + release_constructor = threading.Event() + calls = 0 + calls_lock = threading.Lock() + + def construct(runtime: object) -> VardiffService: + nonlocal calls + with calls_lock: + calls += 1 + constructor_entered.set() + self.assertTrue(release_constructor.wait(1.0)) + return real_service_type(runtime) # type: ignore[arg-type] + + services: list[VardiffService] = [] + descriptor_write_finished = threading.Event() + + def write_compatibility_field() -> None: + server.idle_retarget_count = 9 + descriptor_write_finished.set() + + with patch( + "lab.prism.prism_coordinator.VardiffService", + side_effect=construct, + ): + threads = [ + threading.Thread( + target=lambda: services.append( + server._ensure_vardiff_service() + ) + ) + for _ in range(2) + ] + threads[0].start() + self.assertTrue(constructor_entered.wait(1.0)) + threads[1].start() + descriptor_writer = threading.Thread(target=write_compatibility_field) + descriptor_writer.start() + time.sleep(0.05) + with calls_lock: + self.assertEqual(calls, 1) + self.assertFalse(descriptor_write_finished.is_set()) + release_constructor.set() + for thread in threads: + thread.join(1.0) + self.assertFalse(thread.is_alive()) + descriptor_writer.join(1.0) + self.assertFalse(descriptor_writer.is_alive()) + + self.assertEqual(len(services), 2) + self.assertIs(services[0], services[1]) + self.assertEqual(services[0].vardiff_idle_queue_depth, 7) + self.assertEqual(services[0].idle_retarget_count, 9) + self.assertNotIn("idle_retarget_count", server.__dict__) + + def test_load_prism_vardiff_config_defaults_to_small_miner_vardiff(self) -> None: + names = [name for name in os.environ if name.startswith("PRISM_STRATUM_VARDIFF")] + with patch.dict(os.environ, {}, clear=False): + for name in names: + os.environ.pop(name, None) + config = load_prism_vardiff_config(Decimal("0.000000001")) + + self.assertTrue(config.enabled) + self.assertEqual(config.target_share_interval_seconds, Decimal("15")) + self.assertEqual(config.min_difficulty, Decimal("1E-9")) + self.assertEqual(config.startup_difficulty, Decimal("1E-9")) + self.assertEqual(config.max_step_factor, Decimal("4")) + self.assertEqual(config.max_step_down_factor, Decimal("4")) + + def test_vardiff_retarget_build_failure_keeps_consistent_difficulty_and_job(self) -> None: + # If the job build is skipped during a retarget, the client must stay on its + # existing job at its existing difficulty -- never advertise a new difficulty + # for a job it never received. Otherwise its easier shares miss the old + # target, nothing is accepted, and (since retargets only fire on accepted + # shares) it cannot self-heal without reconnecting. + server = coordinator() + server.jobs = {"old-job": SimpleNamespace(job=SimpleNamespace(job_id="old-job"))} + server.extranonce2_size = 8 + state = client() + state.username = "miner-a" + state.worker = WorkerIdentity( + username="miner-a", + payout_address="miner-a", + worker_name=None, + script_pubkey_hex="5220" + "11" * 32, + p2mr_program_hex="11" * 32, + ) + state.share_difficulty = Decimal("1") + state.active_job_ids = {"old-job"} + state.vardiff_window_started_monotonic = time.monotonic() - 2 + advertised: list[object] = [] + state.send = lambda payload: advertised.append(payload) # type: ignore[method-assign] + + def failing_build(client: ClientState, *, clean_jobs: bool) -> object: + raise ValueError("transient getblocktemplate failure") + + server.build_job_for_client = failing_build # type: ignore[method-assign] + + server.note_vardiff_submitted_share(state) + server.note_vardiff_accepted_share(state, FakeJob(Decimal("1"))) # type: ignore[arg-type] + + self.assertEqual(server.job_build_failure_count, 1) + self.assertIsNone(state.pending_share_difficulty) # rolled back, not left at the new value + self.assertEqual(state.share_difficulty, Decimal("1")) # unchanged + self.assertEqual(state.active_job_ids, {"old-job"}) # old job retained, still submittable + self.assertEqual(set(server.jobs), {"old-job"}) + self.assertEqual(advertised, []) # no set_difficulty / notify advertised for the skipped build + + def test_idle_vardiff_success_sends_paired_job_and_resets_window(self) -> None: + server = coordinator() + state = client() + prepare_idle_client(server, state) + install_idle_job_cache(server) + sent: list[dict[str, object]] = [] + delivered = threading.Event() + window_started = state.vardiff_window_started_monotonic + + def record(payload: dict[str, object]) -> None: + sent.append(payload) + if payload.get("method") == "mining.notify": + delivered.set() + + state.send = record # type: ignore[method-assign] + + self.assertEqual(server.vardiff_idle_sweep_once(), 1) + self.assertTrue(delivered.wait(timeout=1)) + server.shutdown_vardiff_idle_executor() + + self.assertEqual(server.idle_retarget_count, 1) + self.assertEqual( + [payload["method"] for payload in sent], + ["mining.set_difficulty", "mining.notify"], + ) + self.assertEqual(sent[0]["params"], [4.0]) + self.assertTrue(sent[1]["params"][8]) + self.assertEqual(state.share_difficulty, Decimal("4")) + self.assertIsNone(state.pending_share_difficulty) + self.assertGreater(state.vardiff_window_started_monotonic, window_started) + self.assertEqual(state.vardiff_window_accepted, 0) + self.assertEqual(state.vardiff_window_submitted, 0) + self.assertEqual(state.vardiff_window_work, Decimal("0")) + + def test_idle_vardiff_shutdown_after_delivery_keeps_committed_window(self) -> None: + server = coordinator() + state = client() + prepare_idle_client(server, state) + install_idle_job_cache(server) + sent: list[dict[str, object]] = [] + delivered = threading.Event() + window_started = state.vardiff_window_started_monotonic + + def stop_after_delivery(payload: dict[str, object]) -> None: + sent.append(payload) + if payload.get("method") == "mining.notify": + server.stop_event.set() + delivered.set() + + state.send = stop_after_delivery # type: ignore[method-assign] + + self.assertEqual(server.vardiff_idle_sweep_once(), 1) + self.assertTrue(delivered.wait(timeout=1)) + server.shutdown_vardiff_idle_executor() + + self.assertEqual( + [payload["method"] for payload in sent], + ["mining.set_difficulty", "mining.notify"], + ) + self.assertEqual(server.idle_retarget_count, 1) + self.assertEqual(state.share_difficulty, Decimal("4")) + self.assertIsNone(state.pending_share_difficulty) + self.assertGreater(state.vardiff_window_started_monotonic, window_started) + self.assertEqual(state.vardiff_window_accepted, 0) + self.assertEqual(state.vardiff_window_submitted, 0) + self.assertEqual(state.vardiff_window_work, Decimal("0")) + + def test_idle_vardiff_sweep_skips_submitted_reject_storm_window(self) -> None: + server = coordinator() + state = client() + state.worker = worker_identity() + state.active_job = prism_context("job-1", "00" * 32, worker=state.worker) + state.share_difficulty = Decimal("16") + state.vardiff_window_started_monotonic = time.monotonic() - 2 + state.vardiff_window_submitted = 3 + server.clients = {state} + + def fail_send_job(client: ClientState, *, clean_jobs: bool) -> bool: + raise AssertionError("reject-storm windows must not idle-retarget") + + server.maybe_send_job = fail_send_job # type: ignore[method-assign] + + retargeted = server.vardiff_idle_sweep_once() + + self.assertEqual(retargeted, 0) + self.assertEqual(state.pending_share_difficulty, None) + self.assertEqual(state.vardiff_window_submitted, 3) + + def test_idle_vardiff_share_after_snapshot_is_not_stepped_down(self) -> None: + server = coordinator() + blockers = [client(), client()] + for connection_id, blocker in enumerate(blockers, start=1): + prepare_idle_client(server, blocker, connection_id=connection_id) + install_idle_job_cache(server) + workers_started = threading.Barrier(3) + release_workers = threading.Event() + + def block_delivery(payload: dict[str, object]) -> None: + if payload.get("method") == "mining.set_difficulty": + workers_started.wait(timeout=1) + release_workers.wait(timeout=1) + + for blocker in blockers: + blocker.send = block_delivery # type: ignore[method-assign] + + target = client() + target.connection_id = 3 + sent: list[dict[str, object]] = [] + target.send = sent.append # type: ignore[method-assign] + try: + self.assertEqual(server.vardiff_idle_sweep_once(), 2) + workers_started.wait(timeout=1) + prepare_idle_client(server, target, connection_id=3) + self.assertEqual(server.vardiff_idle_sweep_once(), 1) + + # The task is queued from an idle snapshot. A later submit changes + # that exact window before any worker can commit the step-down. + server.note_vardiff_submitted_share(target) + finally: + release_workers.set() + server.shutdown_vardiff_idle_executor() + + self.assertEqual(sent, []) + self.assertIsNone(target.pending_share_difficulty) + self.assertEqual(target.share_difficulty, Decimal("16")) + self.assertEqual(target.vardiff_window_submitted, 1) + self.assertGreaterEqual(server.vardiff_idle_skip_counts["not_idle"], 1) + + def test_idle_vardiff_failure_restores_pending_and_idle_window(self) -> None: + server = coordinator() + state = client() + prepare_idle_client(server, state) + install_idle_job_cache(server) + window_state = ( + state.vardiff_window_started_monotonic, + state.vardiff_window_accepted, + state.vardiff_window_submitted, + state.vardiff_window_work, + ) + disconnected: list[ClientState] = [] + failure_finished = threading.Event() + + def failing_send(_payload: dict[str, object]) -> None: + self.assertEqual(state.pending_share_difficulty, Decimal("4")) + raise OSError("socket send failed") + + def fake_disconnect(client: ClientState) -> None: + disconnected.append(client) + server.clients.discard(client) + failure_finished.set() + + state.send = failing_send # type: ignore[method-assign] + server.disconnect_client = fake_disconnect # type: ignore[method-assign] + + self.assertEqual(server.vardiff_idle_sweep_once(), 1) + self.assertTrue(failure_finished.wait(timeout=1)) + server.shutdown_vardiff_idle_executor() + + self.assertEqual(disconnected, [state]) + self.assertIsNone(state.pending_share_difficulty) + self.assertEqual(state.share_difficulty, Decimal("16")) + self.assertEqual( + ( + state.vardiff_window_started_monotonic, + state.vardiff_window_accepted, + state.vardiff_window_submitted, + state.vardiff_window_work, + ), + window_state, + ) + self.assertEqual(server.vardiff_idle_task_failures, 1) + + def test_idle_vardiff_stamp_failure_restores_speculative_state(self) -> None: + server = coordinator() + state = client() + prepare_idle_client(server, state) + install_idle_job_cache(server) + window_state = ( + state.vardiff_window_started_monotonic, + state.vardiff_window_accepted, + state.vardiff_window_submitted, + state.vardiff_window_work, + ) + + def fail_stamp(*_args: object, **_kwargs: object) -> None: + self.assertEqual(state.pending_share_difficulty, Decimal("4")) + raise RuntimeError("cached job stamping failed") + + state.send = lambda payload: self.fail( # type: ignore[method-assign] + f"unexpected delivery: {payload}" + ) + server.stamp_job_for_client = fail_stamp # type: ignore[method-assign] + + self.assertEqual(server.vardiff_idle_sweep_once(), 1) + server.shutdown_vardiff_idle_executor() + + self.assertIsNone(state.pending_share_difficulty) + self.assertEqual(state.share_difficulty, Decimal("16")) + self.assertEqual( + ( + state.vardiff_window_started_monotonic, + state.vardiff_window_accepted, + state.vardiff_window_submitted, + state.vardiff_window_work, + ), + window_state, + ) + self.assertEqual(server.job_build_failure_count, 1) + self.assertEqual(server.vardiff_idle_task_failures, 1) + + def test_idle_cached_bundle_requires_live_reorg_trust(self) -> None: + server = coordinator() + state = client() + prepare_idle_client(server, state) + install_idle_job_cache(server) + trust_checked = threading.Event() + sent: list[dict[str, object]] = [] + window_started = state.vardiff_window_started_monotonic + + def reject_untrusted_tip() -> bool: + trust_checked.set() + return False + + state.send = sent.append # type: ignore[method-assign] + server.ensure_reorg_reconciled_for_current_tip = ( # type: ignore[method-assign] + reject_untrusted_tip + ) + + self.assertEqual(server.vardiff_idle_sweep_once(), 1) + server.shutdown_vardiff_idle_executor() + + self.assertTrue(trust_checked.is_set()) + self.assertEqual(sent, []) + self.assertEqual(server.idle_retarget_count, 0) + self.assertEqual(state.share_difficulty, Decimal("16")) + self.assertIsNone(state.pending_share_difficulty) + self.assertEqual(state.vardiff_window_started_monotonic, window_started) + self.assertGreaterEqual(server.vardiff_idle_skip_counts["superseded"], 1) + + def test_idle_retarget_defers_detected_payout_source_during_tip_divergence( + self, + ) -> None: + old_tip = "00" * 32 + new_tip = "11" * 32 + server = coordinator() + state = client() + prepare_idle_client(server, state, tip=old_tip) + install_idle_job_cache(server, tip=old_tip) + with server._job_cache_lock: + published_artifacts = server._template_artifacts + assert published_artifacts is not None + detected_template = gbt_template(new_tip, height=11) + detected_artifacts = CachedTemplateArtifacts( + template=detected_template, + fingerprint=qbit_template_fingerprint(detected_template), + previousblockhash=new_tip, + transaction_hexes=(), + witness_merkle_leaves_hex=(), + network_difficulty=1, + fetched_monotonic=time.monotonic(), + generation=2, + ) + now = time.monotonic() + server.current_tip_first_seen = (old_tip, now) + server.current_tip_observed_monotonic = now + server.latest_detected_tip = (new_tip, 2) + server.tip_refresh_divergence_started_monotonic = now + server.tip_template_snapshot = QbitTipTemplateSnapshot( + bestblockhash=old_tip, + previousblockhash=old_tip, + template_fingerprint=published_artifacts.fingerprint, + template_generation=published_artifacts.generation, + template_artifacts=published_artifacts, + ) + with server._job_cache_lock: + server._template_artifacts = detected_artifacts + server._published_payout_state = dataclass_replace( + server._published_payout_state, + source_tip_hash=new_tip, + ) + window_state = ( + state.vardiff_window_started_monotonic, + state.vardiff_window_accepted, + state.vardiff_window_submitted, + state.vardiff_window_work, + ) + sent: list[dict[str, object]] = [] + + server._build_idle_job_bundle = lambda _request: self.fail( # type: ignore[method-assign] + "idle divergence entered the shared build scheduler" + ) + state.send = sent.append # type: ignore[method-assign] + + self.assertEqual(server.vardiff_idle_sweep_once(), 0) + server.shutdown_vardiff_idle_executor() + + self.assertEqual(sent, []) + self.assertEqual(state.share_difficulty, Decimal("16")) + self.assertIsNone(state.pending_share_difficulty) + self.assertEqual( + ( + state.vardiff_window_started_monotonic, + state.vardiff_window_accepted, + state.vardiff_window_submitted, + state.vardiff_window_work, + ), + window_state, + ) + self.assertEqual(server.vardiff_idle_skip_counts["superseded"], 1) + + def test_replacement_tip_build_survives_repeated_idle_sweeps(self) -> None: + old_tip = "00" * 32 + new_tip = "11" * 32 + server = coordinator() + state = client() + prepare_idle_client(server, state, tip=old_tip) + install_idle_job_cache(server, tip=old_tip) + with server._job_cache_lock: + published_artifacts = server._template_artifacts + assert published_artifacts is not None + detected_template = gbt_template(new_tip, height=11) + detected_artifacts = CachedTemplateArtifacts( + template=detected_template, + fingerprint=qbit_template_fingerprint(detected_template), + previousblockhash=new_tip, + transaction_hexes=(), + witness_merkle_leaves_hex=(), + network_difficulty=1, + fetched_monotonic=time.monotonic(), + generation=2, + ) + now = time.monotonic() + server.current_tip_first_seen = (old_tip, now) + server.current_tip_observed_monotonic = now + server.latest_detected_tip = (new_tip, 2) + server.tip_refresh_divergence_started_monotonic = now + server.tip_template_snapshot = QbitTipTemplateSnapshot( + bestblockhash=old_tip, + previousblockhash=old_tip, + template_fingerprint=published_artifacts.fingerprint, + template_generation=published_artifacts.generation, + template_artifacts=published_artifacts, + ) + with server._job_cache_lock: + server._template_artifacts = detected_artifacts + server._ensure_job_cache_state() + replacement_cancellation = _JobBuildCancellation(timeout_seconds=60.0) + replacement_request = SimpleNamespace( + artifacts=detected_artifacts, + cancellation=replacement_cancellation, + ) + replacement_flight = SimpleNamespace(request=replacement_request) + with server._job_build_scheduler_lock: + server._job_build_active = replacement_flight + build_called = threading.Event() + window_started = state.vardiff_window_started_monotonic + + def unexpected_idle_build(_request: object) -> CachedJobBundle: + build_called.set() + raise AssertionError("idle sweep displaced the replacement build") + + server._build_idle_job_bundle = unexpected_idle_build # type: ignore[method-assign] + + for _sweep in range(4): + self.assertEqual(server.vardiff_idle_sweep_once(), 0) + + # Close the race where a worker passed its first divergence check just + # before detection. Scheduler admission must reject that idle request + # without superseding the active replacement build. + racing_idle_cancellation = _JobBuildCancellation(timeout_seconds=60.0) + racing_idle_request = SimpleNamespace( + idle_retarget=True, + cancellation=racing_idle_cancellation, + promise=Future(), + ) + racing_idle_promise = server._request_job_build(racing_idle_request) # type: ignore[arg-type] + with self.assertRaises(JobBuildCancelled): + racing_idle_promise.result() + + with server._job_build_scheduler_lock: + self.assertIs(server._job_build_active, replacement_flight) + self.assertFalse(replacement_cancellation.is_set()) + self.assertTrue(racing_idle_cancellation.is_set()) + self.assertFalse(build_called.is_set()) + self.assertEqual(state.vardiff_window_started_monotonic, window_started) + self.assertEqual(server.vardiff_idle_skip_counts["superseded"], 4) + with server._job_build_scheduler_lock: + server._job_build_active = None + server.shutdown_vardiff_idle_executor() + + def test_idle_shared_build_does_not_retry_scheduler_divergence_race( + self, + ) -> None: + old_tip = "00" * 32 + new_tip = "11" * 32 + server = coordinator() + state = client() + prepare_idle_client(server, state, tip=old_tip) + install_idle_job_cache(server, tip=old_tip) + with server._job_cache_lock: + old_artifacts = server._template_artifacts + server._job_bundle_cache.clear() + assert old_artifacts is not None + now = time.monotonic() + server.current_tip_first_seen = (old_tip, now) + server.current_tip_observed_monotonic = now + server.latest_detected_tip = (old_tip, 1) + server.tip_template_snapshot = QbitTipTemplateSnapshot( + bestblockhash=old_tip, + previousblockhash=old_tip, + template_fingerprint=old_artifacts.fingerprint, + template_generation=old_artifacts.generation, + template_artifacts=old_artifacts, + ) + server._ensure_job_cache_state() + replacement_cancellation = _JobBuildCancellation(timeout_seconds=60.0) + replacement_flight = SimpleNamespace( + request=SimpleNamespace( + artifacts=SimpleNamespace(previousblockhash=new_tip), + cancellation=replacement_cancellation, + ) + ) + with server._job_build_scheduler_lock: + server._job_build_active = replacement_flight + request_builds = 0 + admission_attempts = 0 + idle_cancellation: _JobBuildCancellation | None = None + original_request_job_build = server._request_job_build + + def make_idle_request( + _artifacts: CachedTemplateArtifacts, + _worker: WorkerIdentity | None, + **kwargs: object, + ) -> object: + nonlocal request_builds, idle_cancellation + request_builds += 1 + self.assertTrue(kwargs["idle_retarget"]) + idle_cancellation = _JobBuildCancellation(timeout_seconds=60.0) + return SimpleNamespace( + idle_retarget=True, + cancellation=idle_cancellation, + promise=Future(), + ) + + def detect_before_admission(request: object) -> Future[CachedJobBundle]: + nonlocal admission_attempts + admission_attempts += 1 + with server.lock: + server.latest_detected_tip = (new_tip, 2) + server.tip_refresh_divergence_started_monotonic = time.monotonic() + return original_request_job_build(request) # type: ignore[arg-type] + + server._new_job_build_request = make_idle_request # type: ignore[method-assign] + server._request_job_build = detect_before_admission # type: ignore[method-assign] + server._schedule_tip_refresh_retry = lambda: None # type: ignore[method-assign] + assert state.worker is not None + + with self.assertRaises(JobBuildSuperseded): + server._build_idle_job_bundle(SimpleNamespace(worker=state.worker)) # type: ignore[arg-type] + + self.assertEqual(request_builds, 1) + self.assertEqual(admission_attempts, 1) + self.assertIs(server._template_artifacts, old_artifacts) + self.assertIsNotNone(idle_cancellation) + assert idle_cancellation is not None + self.assertTrue(idle_cancellation.is_set()) + self.assertFalse(replacement_cancellation.is_set()) + with server._job_build_scheduler_lock: + self.assertIs(server._job_build_active, replacement_flight) + server._job_build_active = None + server.shutdown_vardiff_idle_executor() + + def test_idle_cached_collection_bundle_refreshes_readiness(self) -> None: + server = coordinator() + state = client() + prepare_idle_client(server, state) + ready_bundle = install_idle_job_cache(server) + assert state.worker is not None + with server._job_cache_lock: + artifacts = server._template_artifacts + assert artifacts is not None + collection_key = server._job_bundle_key( + artifacts, + mode="collection", + payout_state_generation=server._payout_state_generation, + payout_artifact_generation=0, + worker=state.worker, + ) + collection_bundle = dataclass_replace( + ready_bundle, + key=collection_key, + collection_only=True, + collection_identity=( + state.worker.payout_address, + state.worker.p2mr_program_hex, + ), + ) + server._job_bundle_cache.clear() + server._job_bundle_cache[collection_key] = collection_bundle + server._pool_ready_latched = False + server.min_ready_miners = 3 + server.accepted_share_stats = lambda: (3, 3) # type: ignore[method-assign] + rebuilt = threading.Event() + sent: list[dict[str, object]] = [] + + def build_ready(_request: object) -> CachedJobBundle: + rebuilt.set() + with server._job_cache_lock: + server._job_bundle_cache[ready_bundle.key] = ready_bundle + return ready_bundle + + state.send = sent.append # type: ignore[method-assign] + server._build_idle_job_bundle = build_ready # type: ignore[method-assign] + + self.assertEqual(server.vardiff_idle_sweep_once(), 1) + server.shutdown_vardiff_idle_executor() + + self.assertTrue(server._pool_ready_latched) + self.assertTrue(rebuilt.is_set()) + self.assertEqual( + [payload["method"] for payload in sent], + ["mining.set_difficulty", "mining.notify"], + ) + self.assertIsNotNone(state.active_job) + self.assertFalse(state.active_job.collection_only) + + def test_idle_cached_ready_bundle_rebinds_same_tip_observation(self) -> None: + server = coordinator() + state = client() + prepare_idle_client(server, state) + cached = install_idle_job_cache(server) + updated_template = dict(cached.template) + updated_template["curtime"] = int(updated_template["curtime"]) + 30 + current = server.store_template_artifacts(updated_template, generation=2) + assert current is not None + delivered = threading.Event() + rebound = threading.Event() + sent: list[dict[str, object]] = [] + + def bind_current_observation( + bundle: CachedJobBundle, + artifacts: CachedTemplateArtifacts, + ) -> CachedJobBundle: + rebound.set() + return dataclass_replace( + bundle, + template=artifacts.template, + base_job=dataclass_replace( + bundle.base_job, + ntime=f'{artifacts.template["curtime"]:08x}', + ), + template_generation=artifacts.generation, + ) + + def record(payload: dict[str, object]) -> None: + sent.append(payload) + if payload.get("method") == "mining.notify": + delivered.set() + + state.send = record # type: ignore[method-assign] + server._bind_cached_bundle_to_artifacts = ( # type: ignore[method-assign] + bind_current_observation + ) + + self.assertEqual(server.vardiff_idle_sweep_once(), 1) + self.assertTrue(delivered.wait(timeout=1)) + server.shutdown_vardiff_idle_executor() + + self.assertIsNotNone(state.active_job) + self.assertTrue(rebound.is_set()) + self.assertIs(state.active_job.template, current.template) + self.assertEqual(state.active_job.template_generation, current.generation) + expected_ntime = f'{updated_template["curtime"]:08x}' + self.assertEqual(state.active_job.job.ntime, expected_ntime) + notify = next( + payload + for payload in sent + if payload.get("method") == "mining.notify" + ) + self.assertEqual(notify["params"][7], expected_ntime) + + def test_repeated_idle_sweeps_do_not_enqueue_duplicate_connection_work(self) -> None: + server = coordinator() + state = client() + prepare_idle_client(server, state) + install_idle_job_cache(server) + delivery_started = threading.Event() + release_delivery = threading.Event() + sent: list[dict[str, object]] = [] + + def blocking_send(payload: dict[str, object]) -> None: + if payload.get("method") == "mining.set_difficulty": + delivery_started.set() + release_delivery.wait(timeout=1) + sent.append(payload) + + state.send = blocking_send # type: ignore[method-assign] + try: + self.assertEqual(server.vardiff_idle_sweep_once(), 1) + self.assertTrue(delivery_started.wait(timeout=1)) + self.assertEqual(server.vardiff_idle_sweep_once(), 0) + with server._vardiff_idle_lock: + self.assertEqual(len(server._vardiff_idle_pending), 1) + finally: + release_delivery.set() + server.shutdown_vardiff_idle_executor() + + self.assertEqual( + [payload["method"] for payload in sent], + ["mining.set_difficulty", "mining.notify"], + ) + self.assertGreaterEqual( + server.vardiff_idle_skip_counts["superseded"] + + server.vardiff_idle_skip_counts["not_idle"], + 1, + ) + + def test_idle_sweep_skips_busy_client_lock_and_returns_promptly(self) -> None: + server = coordinator() + state = client() + prepare_idle_client(server, state) + install_idle_job_cache(server) + lock_held = threading.Event() + release_lock = threading.Event() + + def hold_client_lock() -> None: + state.job_update_lock.acquire() + try: + lock_held.set() + release_lock.wait(timeout=1) + finally: + state.job_update_lock.release() + + holder = threading.Thread(target=hold_client_lock) + holder.start() + self.assertTrue(lock_held.wait(timeout=1)) + started = time.monotonic() + try: + self.assertEqual(server.vardiff_idle_sweep_once(), 0) + finally: + elapsed = time.monotonic() - started + release_lock.set() + holder.join(timeout=1) + server.shutdown_vardiff_idle_executor() + + self.assertLess(elapsed, 0.25) + self.assertEqual(server.vardiff_idle_skip_counts["busy"], 1) + + def test_stuck_bundle_builder_does_not_stale_idle_sweep_heartbeat(self) -> None: + server = coordinator() + state = client() + prepare_idle_client(server, state) + bundle = install_idle_job_cache(server) + with server._job_cache_lock: + server._job_bundle_cache.clear() + state.send = lambda _payload: None # type: ignore[method-assign] + build_started = threading.Event() + release_build = threading.Event() + + def blocked_build( + *_args: object, + **_kwargs: object, + ) -> CachedJobBundle: + build_started.set() + if not release_build.wait(timeout=1): + raise AssertionError("idle retarget bundle build was not released") + return bundle + + server.build_shared_job_bundle = blocked_build # type: ignore[method-assign] + server._record_heartbeat("vardiff_idle_sweep") + heartbeat_before = server._heartbeats["vardiff_idle_sweep"] + started = time.monotonic() + try: + self.assertEqual(server.vardiff_idle_sweep_once(), 1) + elapsed = time.monotonic() - started + self.assertTrue(build_started.wait(timeout=0.25)) + self.assertLess(elapsed, 0.25) + self.assertGreater( + server._heartbeats["vardiff_idle_sweep"], + heartbeat_before, + ) + finally: + release_build.set() + server.shutdown_vardiff_idle_executor() + + self.assertEqual(server.vardiff_idle_skip_counts["cache_miss"], 1) + + def test_idle_sweep_cache_miss_builds_only_on_bounded_worker(self) -> None: + server = coordinator() + state = client() + prepare_idle_client(server, state) + bundle = install_idle_job_cache(server) + server.job_bundle_cache_seconds = 10.0 + expired_bundle = dataclass_replace( + bundle, + built_monotonic=time.monotonic() - 11.0, + ) + with server._job_cache_lock: + server._job_bundle_cache[bundle.key] = expired_bundle + build_started = threading.Event() + release_build = threading.Event() + build_thread_ids: list[int] = [] + sent: list[dict[str, object]] = [] + + def blocked_build( + *_args: object, + **_kwargs: object, + ) -> CachedJobBundle: + build_thread_ids.append(threading.get_ident()) + build_started.set() + if not release_build.wait(timeout=1): + raise AssertionError("idle retarget bundle build was not released") + return bundle + + state.send = sent.append # type: ignore[method-assign] + server.build_shared_job_bundle = blocked_build # type: ignore[method-assign] + sweep_thread_id = threading.get_ident() + started = time.monotonic() + try: + self.assertEqual(server.vardiff_idle_sweep_once(), 1) + elapsed = time.monotonic() - started + self.assertTrue(build_started.wait(timeout=0.25)) + self.assertLess(elapsed, 0.25) + self.assertEqual(server.vardiff_idle_sweep_once(), 0) + self.assertEqual(len(build_thread_ids), 1) + self.assertNotEqual(build_thread_ids[0], sweep_thread_id) + with server._vardiff_idle_lock: + self.assertEqual(len(server._vardiff_idle_pending), 1) + finally: + release_build.set() + server.shutdown_vardiff_idle_executor() + + self.assertEqual(server.vardiff_idle_skip_counts["cache_miss"], 1) + self.assertEqual( + [payload["method"] for payload in sent], + ["mining.set_difficulty", "mining.notify"], + ) + self.assertEqual(server.idle_retarget_count, 1) + self.assertIsNone(state.pending_share_difficulty) + self.assertEqual(state.share_difficulty, Decimal("4")) + self.assertEqual(state.vardiff_window_accepted, 0) + self.assertEqual(state.vardiff_window_submitted, 0) + self.assertEqual(state.vardiff_window_work, Decimal("0")) + + def test_idle_retarget_delivers_fresh_bundle_when_cache_is_disabled( + self, + ) -> None: + server = coordinator() + state = client() + prepare_idle_client(server, state) + bundle = install_idle_job_cache(server) + server.job_bundle_cache_seconds = 0.0 + with server._job_cache_lock: + server._job_bundle_cache.clear() + built = threading.Event() + sent: list[dict[str, object]] = [] + + def build_uncached(_request: object) -> CachedJobBundle: + built.set() + return bundle + + server._build_idle_job_bundle = build_uncached # type: ignore[method-assign] + state.send = sent.append # type: ignore[method-assign] + + self.assertEqual(server.vardiff_idle_sweep_once(), 1) + server.shutdown_vardiff_idle_executor() + + self.assertTrue(built.is_set()) + self.assertEqual(server.vardiff_idle_skip_counts["cache_miss"], 1) + self.assertEqual( + [payload["method"] for payload in sent], + ["mining.set_difficulty", "mining.notify"], + ) + self.assertEqual(server.idle_retarget_count, 1) + self.assertEqual(state.share_difficulty, Decimal("4")) + self.assertIsNone(state.pending_share_difficulty) + with server._job_cache_lock: + self.assertEqual(server._job_bundle_cache, {}) + + def test_idle_preparation_oserror_keeps_client_connected(self) -> None: + for failure_phase in ("bundle", "reorg"): + with self.subTest(failure_phase=failure_phase): + server = coordinator() + state = client() + prepare_idle_client(server, state) + install_idle_job_cache(server) + disconnected: list[ClientState] = [] + window_started = state.vardiff_window_started_monotonic + + def fail_bundle(_request: object) -> CachedJobBundle: + raise OSError("qbit RPC transport unavailable") + + def fail_reorg() -> bool: + raise OSError("qbit trust RPC transport unavailable") + + def unexpected_send(payload: dict[str, object]) -> None: + self.fail(f"unexpected idle delivery: {payload}") + + def record_disconnect(client_state: ClientState) -> None: + disconnected.append(client_state) + + if failure_phase == "bundle": + server._build_idle_job_bundle = ( # type: ignore[method-assign] + fail_bundle + ) + else: + server.ensure_reorg_reconciled_for_current_tip = ( # type: ignore[method-assign] + fail_reorg + ) + state.send = unexpected_send # type: ignore[method-assign] + server.disconnect_client = record_disconnect # type: ignore[method-assign] + + self.assertEqual(server.vardiff_idle_sweep_once(), 1) + server.shutdown_vardiff_idle_executor() + + self.assertEqual(disconnected, []) + self.assertIn(state, server.clients) + self.assertFalse(state.closing) + self.assertEqual(server.vardiff_idle_task_failures, 1) + self.assertIsNone(state.pending_share_difficulty) + self.assertEqual(state.share_difficulty, Decimal("16")) + self.assertEqual( + state.vardiff_window_started_monotonic, + window_started, + ) + + def test_disconnect_while_idle_retarget_pending_prevents_delivery(self) -> None: + server = coordinator() + blockers = [client(), client()] + for connection_id, blocker in enumerate(blockers, start=1): + prepare_idle_client(server, blocker, connection_id=connection_id) + install_idle_job_cache(server) + workers_started = threading.Barrier(3) + release_workers = threading.Event() + + def block_delivery(payload: dict[str, object]) -> None: + if payload.get("method") == "mining.set_difficulty": + workers_started.wait(timeout=1) + release_workers.wait(timeout=1) + + for blocker in blockers: + blocker.send = block_delivery # type: ignore[method-assign] + + target = client() + target_sent: list[dict[str, object]] = [] + target.send = target_sent.append # type: ignore[method-assign] + target.close = lambda: None # type: ignore[method-assign] + disconnected_skip = threading.Event() + record_skip = server._record_vardiff_idle_skip + + def record_and_signal(reason: str) -> None: + record_skip(reason) + if reason == "disconnected": + disconnected_skip.set() + + server._record_vardiff_idle_skip = record_and_signal # type: ignore[method-assign] + try: + self.assertEqual(server.vardiff_idle_sweep_once(), 2) + workers_started.wait(timeout=1) + prepare_idle_client(server, target, connection_id=3) + self.assertEqual(server.vardiff_idle_sweep_once(), 1) + server.disconnect_client(target) + finally: + release_workers.set() + try: + self.assertTrue(disconnected_skip.wait(timeout=1)) + finally: + server.shutdown_vardiff_idle_executor() + + self.assertEqual(target_sent, []) + self.assertTrue(target.closing) + self.assertNotIn(target, server.clients) + self.assertGreaterEqual(server.vardiff_idle_skip_counts["disconnected"], 1) + + def test_hundreds_of_busy_and_dead_clients_cannot_stall_idle_sweep(self) -> None: + server = coordinator() + + class BusyLock: + def acquire(self, blocking: bool = True) -> bool: + self.assert_nonblocking = blocking + return False + + def release(self) -> None: + raise AssertionError("unacquired busy lock released") + + for connection_id in range(1, 201): + state = client() + prepare_idle_client(server, state, connection_id=connection_id) + state.job_update_lock = BusyLock() # type: ignore[assignment] + for connection_id in range(201, 401): + state = client() + prepare_idle_client(server, state, connection_id=connection_id) + state.closing = True + + server.vardiff_idle_sweep_seconds = 0.5 + started = time.monotonic() + self.assertEqual(server.vardiff_idle_sweep_once(), 0) + elapsed = time.monotonic() - started + server.shutdown_vardiff_idle_executor() + + self.assertLess(elapsed, server.vardiff_idle_sweep_seconds) + self.assertEqual(server.vardiff_idle_clients_inspected, 400) + self.assertEqual(server.vardiff_idle_skip_counts["busy"], 200) + self.assertEqual(server.vardiff_idle_skip_counts["disconnected"], 200) + + def test_idle_retarget_queue_is_globally_bounded(self) -> None: + server = coordinator() + states = [client() for _ in range(9)] + for connection_id, state in enumerate(states, start=1): + prepare_idle_client(server, state, connection_id=connection_id) + install_idle_job_cache(server) + release_workers = threading.Event() + two_workers_started = threading.Event() + started_lock = threading.Lock() + started_count = 0 + + def block_delivery(payload: dict[str, object]) -> None: + nonlocal started_count + if payload.get("method") != "mining.set_difficulty": + return + with started_lock: + started_count += 1 + if started_count == 2: + two_workers_started.set() + release_workers.wait(timeout=1) + + for state in states: + state.send = block_delivery # type: ignore[method-assign] + + try: + self.assertEqual(server.vardiff_idle_sweep_once(), 8) + self.assertTrue(two_workers_started.wait(timeout=1)) + with server._vardiff_idle_lock: + self.assertEqual(len(server._vardiff_idle_pending), 8) + self.assertEqual(server.vardiff_idle_inflight, 2) + self.assertEqual(server.vardiff_idle_queue_depth, 6) + self.assertEqual(server.vardiff_idle_skip_counts["queue_full"], 1) + finally: + release_workers.set() + server.shutdown_vardiff_idle_executor() + + def test_maybe_send_job_isolates_build_failure_and_keeps_client_connected(self) -> None: + server = coordinator() + server.jobs = {} + server.extranonce2_size = 8 + state = client() + state.username = "miner-a" + state.worker = WorkerIdentity( + username="miner-a", + payout_address="miner-a", + worker_name=None, + script_pubkey_hex="5220" + "11" * 32, + p2mr_program_hex="11" * 32, + ) + sent: list[object] = [] + state.send = lambda payload: sent.append(payload) # type: ignore[method-assign] + + def boom(client: ClientState, *, clean_jobs: bool) -> None: + raise ValueError( + "full coinbase transaction does not end its coinbase scriptSig " + "with the extranonce placeholder" + ) + + server.build_job_for_client = boom # type: ignore[method-assign] + + # The bug: this used to propagate out of handle_client and drop the miner. + # It must now be swallowed so the client thread survives a single bad template. + server.maybe_send_job(state, clean_jobs=True) + + self.assertEqual(server.job_build_failure_count, 1) + self.assertEqual(state.active_job_ids, set()) + self.assertEqual(server.jobs, {}) + self.assertEqual(sent, []) # no difficulty / mining.notify pushed for the failed build + + # A subsequent good template still issues a job (skip, do not permanently break). + server.build_job_for_client = lambda client, *, clean_jobs: SimpleNamespace( # type: ignore[method-assign] + job=SimpleNamespace( + job_id="job-ok", + share_difficulty=Decimal("1"), + share_target=target_from_compact("207fffff"), + ), + template={"previousblockhash": "00" * 32}, + collection_only=False, + ) + server.send_difficulty = lambda client, job: None # type: ignore[method-assign] + server.send_job = lambda client, job: sent.append("notify") # type: ignore[method-assign] + server.apply_job_difficulty = lambda client, job: None # type: ignore[method-assign] + + server.maybe_send_job(state, clean_jobs=True) + + self.assertEqual(server.job_build_failure_count, 1) + self.assertEqual(state.active_job_ids, {"job-ok"}) + self.assertEqual(sent, ["notify"]) + + def test_maybe_send_job_does_not_swallow_send_failures_as_build_failures(self) -> None: + # Only the job build is isolated. A Stratum send failure (a dead socket) + # must propagate so handle_client disconnects and cleans up, rather than + # being miscounted as a build failure or leaving the client wedged. + server = coordinator() + server.jobs = {} + server.extranonce2_size = 8 + state = client() + state.username = "miner-a" + state.worker = WorkerIdentity( + username="miner-a", + payout_address="miner-a", + worker_name=None, + script_pubkey_hex="5220" + "11" * 32, + p2mr_program_hex="11" * 32, + ) + + server.build_job_for_client = lambda client, *, clean_jobs: SimpleNamespace( # type: ignore[method-assign] + job=SimpleNamespace( + job_id="job-dead", + share_difficulty=Decimal("1"), + share_target=target_from_compact("207fffff"), + ), + collection_only=False, + ) + server.send_difficulty = lambda client, job: None # type: ignore[method-assign] + + def dead_socket(client: ClientState, job: object) -> None: + raise OSError("broken pipe") + + server.send_job = dead_socket # type: ignore[method-assign] + + with self.assertRaises(OSError): + server.maybe_send_job(state, clean_jobs=True) + + # The send failure is not a build failure, and handle_client (not us) owns + # the disconnect/cleanup of the registered job for the dead connection. + self.assertEqual(server.job_build_failure_count, 0) + + def _pending_append(self, tag: str, accepted_at_ms: int = 2) -> PendingShareAppend: + from lab.prism.share_ledger import PendingShare + + return PendingShareAppend( + pending_share=PendingShare( + share_id=f"miner-a:{tag}", + miner_id="miner-a", + order_key="miner-a", + p2mr_program_hex="11" * 32, + share_difficulty=1, + network_difficulty=1, + template_height=10, + job_id="job-1", + job_issued_at_ms=1, + accepted_at_ms=accepted_at_ms, + ntime=1_700_000_000, + ), + username="miner-a", + job_id="job-1", + block_hash_hex=tag * 32, + collection_only=False, + credit_policy=None, + ) + + def test_vardiff_step_up_retarget_sends_new_difficulty_without_flush(self) -> None: + server = coordinator() + state = client() + state.share_difficulty = Decimal("1") + state.vardiff_window_started_monotonic = time.monotonic() - 2 + sent: dict[str, object] = {"jobs": 0} + + def fake_send_job(client: object, clean_jobs: bool) -> bool: + sent.update({"jobs": sent["jobs"] + 1, "clean": clean_jobs}) + return True + + server.maybe_send_job = fake_send_job # type: ignore[method-assign] + + server.note_vardiff_submitted_share(state) + server.note_vardiff_accepted_share(state, FakeJob(Decimal("1"))) # type: ignore[arg-type] + + # Difficulty is advertised by maybe_send_job alongside the job (gated on + # a successful build). A step-up must not flush the miner's in-flight + # work: old jobs validate at their own stamped share_target, so the + # retarget requests a single non-clean job. + self.assertEqual(state.pending_share_difficulty, Decimal("4")) + self.assertEqual(sent["jobs"], 1) + self.assertFalse(sent["clean"]) + + def test_vardiff_step_down_retarget_flushes_with_clean_job(self) -> None: + # Firmware that applies mining.set_difficulty retroactively would + # submit sub-target shares against the old job after a step-down, so + # the step-down keeps flushing in-flight work. + server = coordinator() + state = client() + state.share_difficulty = Decimal("16") + state.vardiff_window_started_monotonic = time.monotonic() - 65 + sent: dict[str, object] = {"jobs": 0} + + def fake_send_job(client: object, clean_jobs: bool) -> bool: + sent.update({"jobs": sent["jobs"] + 1, "clean": clean_jobs}) + return True + + server.maybe_send_job = fake_send_job # type: ignore[method-assign] + + server.note_vardiff_submitted_share(state) + server.note_vardiff_accepted_share(state, FakeJob(Decimal("16"))) # type: ignore[arg-type] + + self.assertEqual(state.pending_share_difficulty, Decimal("4")) + self.assertEqual(sent["jobs"], 1) + self.assertTrue(sent["clean"]) + + def test_step_up_retarget_keeps_old_job_submittable_at_old_difficulty(self) -> None: + tip = "00" * 32 + server = coordinator() + server.accepted_block_count = 0 + server.max_blocks = 1 + server.stop_after_block = True + server.jobs = {} + server.recent_share_keys = set() + server.share_weights_by_username = {} + ledger = RecordingLedger() + server.ledger = ledger + worker = worker_identity() + state = client() + state.username = worker.username + state.worker = worker + state.share_difficulty = Decimal("1") + state.vardiff_window_started_monotonic = time.monotonic() - 2 + sent: list[dict[str, object]] = [] + state.send = lambda payload: sent.append(payload) # type: ignore[method-assign] + server.clients = {state} + server.rpc = TipTemplateRpc(tip=tip, template=gbt_template(tip)) + + old_context = prism_context("old-job", tip, worker=worker, difficulty=Decimal("1")) + state.active_job = old_context + state.active_job_ids = {"old-job"} + server.jobs["old-job"] = old_context + + def build_fresh_job(client: ClientState, *, clean_jobs: bool) -> object: + return prism_context( + "fresh-job", + tip, + worker=worker, + difficulty=client.pending_share_difficulty or client.share_difficulty, + clean_jobs=clean_jobs, + ) + + server.build_job_for_client = build_fresh_job # type: ignore[method-assign] + + server.note_vardiff_submitted_share(state) + server.note_vardiff_accepted_share(state, FakeJob(Decimal("1"))) # type: ignore[arg-type] + + # The 1 -> 4 step-up pairs the new difficulty with a non-clean job and + # keeps the old job registered and submittable. + self.assertEqual(state.share_difficulty, Decimal("4")) + self.assertIsNone(state.pending_share_difficulty) + self.assertEqual( + [payload["method"] for payload in sent], + ["mining.set_difficulty", "mining.notify"], + ) + self.assertEqual(sent[0]["params"], [4.0]) + self.assertEqual(sent[1]["params"][0], "fresh-job") + self.assertFalse(sent[1]["params"][8]) + self.assertIn("old-job", server.jobs) + self.assertIn("fresh-job", server.jobs) + self.assertEqual(state.active_job_ids, {"old-job", "fresh-job"}) + + # In-flight work against the old job still lands, validated at the old + # job's own stamped difficulty: this share passes the diff-1 target it + # was mined against, not the diff-4 target advertised afterward. + def assemble_at_stamped_difficulty(job: object, **_kwargs: object) -> SimpleNamespace: + return SimpleNamespace( + header_hex="aa" * 80, + block_hash_hex="bb" * 32, + share_pass=job.share_difficulty <= Decimal("1"), + block_pass=False, + ) + + with patch( + "lab.prism.prism_coordinator.direct_stratum.assemble_submission", + side_effect=assemble_at_stamped_difficulty, + ): + should_close = server.handle_submit( + state, + ["miner-a", "old-job", "00" * 8, "00000001", "00000002"], + ) + + self.assertFalse(should_close) + self.assertEqual(len(ledger.pending), 1) + self.assertEqual(ledger.pending[0].job_id, "old-job") + self.assertEqual(server.low_difficulty_share_count, 0) + self.assertEqual(server.rejection_counts_by_reason[PRISM_REJECTION_LOW_DIFFICULTY], 0) + self.assertEqual(server.rejection_counts_by_reason[PRISM_REJECTION_UNKNOWN_JOB], 0) + self.assertEqual(server.rejection_counts_by_reason[PRISM_REJECTION_STALE_JOB], 0) + + +class PrismStampedJobFloorTests(unittest.TestCase): + """The listener floor must hold on the wire, not just in vardiff policy. + + Stamped jobs are the single choke point for every mining.set_difficulty + the coordinator sends, and marketplace verification judges the first one. + The regression here is a young chain: qbit network difficulty below the + high-diff floor used to drag the advertised difficulty down with it. + """ + + def stamp_coordinator(self) -> PrismCoordinator: + server = coordinator() + server.job_counter = 0 + server.share_weights_by_username = {} + server.default_share_weight = 1 + return server + + def cached_bundle(self) -> CachedJobBundle: + # bits 207fffff: regtest-grade network difficulty (~4.7e-10), far + # below the 500k marketplace floor. + qbit_target = target_from_compact("207fffff") + base_job = direct_stratum.DirectQbitStratumJob( + job_id="prism-template-base", + previousblockhash_display="00" * 32, + prevhash="00" * 32, + coinb1="", + coinb2="", + full_coinbase_prefix="", + full_coinbase_suffix="", + merkle_branch=(), + transaction_hexes=(), + version="20000000", + nbits="207fffff", + ntime="6553f100", + qbit_target=qbit_target, + share_target=qbit_target, + share_difficulty=Decimal("1"), + extranonce1_hex="ffffffff", + extranonce2_size=8, + clean_jobs=True, + ) + return CachedJobBundle( + key=("test",), + template=gbt_template("00" * 32), + template_fingerprint="fp", + coinbase_manifest={}, + shares_json=[], + prior_balances=[], + found_block={"network_difficulty": 1}, + collection_only=False, + issued_at_ms=12345, + base_job=base_job, + built_monotonic=time.monotonic(), + ) + + def highdiff_client(self) -> ClientState: + state = client() + state.worker = worker_identity() + state.listener_vardiff_config = highdiff_vardiff_config() + state.minimum_advertised_difficulty = Decimal("500000") + state.share_difficulty = Decimal("500000") + return state + + def test_stamped_job_enforces_floor_below_network_difficulty(self) -> None: + server = self.stamp_coordinator() + state = self.highdiff_client() + + context = server.stamp_job_for_client(state, self.cached_bundle(), clean_jobs=True) + + self.assertEqual( + context.job.share_target, + direct_stratum.difficulty_target(Decimal("500000")), + ) + # Decimal round-tripping can land within 1e-27 of the floor; the wire + # value is float(difficulty), which is what marketplaces judge. + self.assertGreaterEqual(float(context.job.share_difficulty), 500000.0) + + def test_stamped_job_keeps_network_cap_without_listener_floor(self) -> None: + server = self.stamp_coordinator() + state = client() + state.worker = worker_identity() + # Even an absurd desired difficulty stays capped at the network + # target on the default listener: shares are never required to be + # harder than blocks there. + state.share_difficulty = Decimal("500000") + + context = server.stamp_job_for_client(state, self.cached_bundle(), clean_jobs=True) + + self.assertEqual(context.job.share_target, target_from_compact("207fffff")) + self.assertLess(context.job.share_difficulty, Decimal("1")) + + def test_stamped_job_honors_md_raised_floor_on_highdiff_listener(self) -> None: + server = self.stamp_coordinator() + state = self.highdiff_client() + state.requested_min_difficulty = Decimal("2000000") + server.apply_client_difficulty_requests(state) + + context = server.stamp_job_for_client(state, self.cached_bundle(), clean_jobs=True) + + self.assertEqual( + context.job.share_target, + direct_stratum.difficulty_target(Decimal("2000000")), + ) + self.assertGreaterEqual(float(context.job.share_difficulty), 2000000.0) diff --git a/tests/test_prism_vardiff_durable_resume.py b/tests/test_prism_vardiff_durable_resume.py new file mode 100644 index 000000000..7f463df96 --- /dev/null +++ b/tests/test_prism_vardiff_durable_resume.py @@ -0,0 +1,1045 @@ +#!/usr/bin/env python3 +"""Durable worker-difficulty wiring at the vardiff ownership boundary.""" + +from __future__ import annotations + +from decimal import Decimal +import inspect +from itertools import product +from types import SimpleNamespace +import threading +import time +import unittest +from unittest import mock + +from lab.auxpow import vardiff +from lab.prism.prism_coordinator import PrismCoordinator +from lab.prism.stratum_session import ClientState +from lab.prism.vardiff_service import ( + MAX_PENDING_VARDIFF_DURABLE_WRITES, + PRISM_VARDIFF_DURABLE_PRUNE_BATCH, + PRISM_VARDIFF_DURABLE_PRUNE_MAX_BATCHES, + VardiffService, + _PendingDurableWrite, +) +from lab.prism.worker_difficulty_store import ( + MemoryWorkerDifficultyStore, + PostgresWorkerDifficultyStore, +) +from tests.prism_vardiff_test_support import worker_identity + + +def lane_config() -> vardiff.VardiffConfig: + return vardiff.VardiffConfig( + enabled=True, + target_share_interval_seconds=Decimal("15"), + min_difficulty=Decimal("1024"), + max_difficulty=Decimal("4294967296"), + retarget_interval_seconds=Decimal("300"), + max_step_factor=Decimal("4"), + startup_difficulty=Decimal("16384"), + max_step_down_factor=Decimal("4"), + ) + + +def runtime( + store: MemoryWorkerDifficultyStore, + *, + ttl_seconds: float = 900.0, + enabled: bool = True, + max_entries: int = 32, +) -> SimpleNamespace: + return SimpleNamespace( + worker_difficulty_store=store, + vardiff_resume_enabled=enabled, + vardiff_resume_ttl_seconds=ttl_seconds, + vardiff_resume_max_entries=max_entries, + vardiff_resume_max_start_factor=Decimal("1024"), + vardiff_config=lane_config(), + share_difficulty=Decimal("16384"), + lock=threading.RLock(), + clients=set(), + ) + + +def client( + *, + listener: str = "default", + username: str = "miner-a.rig-1", + difficulty: Decimal = Decimal("16384"), +) -> ClientState: + state = ClientState( + sock=object(), + address=("127.0.0.1", 1), + connection_id=1, + extranonce1_hex="00000001", + listener_name=listener, + listener_vardiff_config=lane_config(), + share_difficulty=difficulty, + ) + state.worker = worker_identity(username) + state.username = username + return state + + +class DurableVardiffResumeTests(unittest.TestCase): + def service(self, runtime_namespace: SimpleNamespace) -> VardiffService: + """A service whose persistence lane is closed when the test ends.""" + built = VardiffService(runtime_namespace) + self.addCleanup(built.shutdown_durable_executor) + return built + + def drained(self, built: VardiffService) -> VardiffService: + """Wait out the asynchronous persistence lane. + + Durable writes are deliberately fire-and-forget so mining never waits + on them; tests that assert on the store have to say so explicitly. + """ + self.assertTrue(built.flush_durable_writes(timeout=5.0)) + return built + + def test_durable_cache_is_loaded_before_any_listener_accepts(self) -> None: + source = inspect.getsource(PrismCoordinator._serve_with_listener_stack) + preload = source.index("self._ensure_vardiff_service()") + + self.assertLess(preload, source.index("self._start_secondary_accept_service")) + self.assertLess(preload, source.index("self.accept_loop")) + + def test_share_backed_retarget_survives_service_restart(self) -> None: + durable = MemoryWorkerDifficultyStore() + first = self.service(runtime(durable)) + state = client() + state.vardiff_last_accepted_difficulty = Decimal("16384") + state.vardiff_last_accepted_wall_ms = 1_000_000 + + with mock.patch("lab.prism.vardiff_service.time.time", return_value=1000.1): + first.record_session_difficulty( + state, + Decimal("500000"), + share_backed=True, + ) + self.drained(first) + + with ( + mock.patch("lab.prism.vardiff_service.time.time", return_value=1001.0), + mock.patch( + "lab.prism.vardiff_service.time.monotonic", + return_value=2000.0, + ), + ): + restarted = self.service(runtime(durable)) + retained, outcome = restarted.session_difficulty_store.lookup( + ("default", "miner-a.rig-1"), + now=2000.0, + ) + + self.assertEqual((retained, outcome), (Decimal("500000"), "hit")) + self.assertEqual(restarted.vardiff_durable_preload_records, 1) + self.assertEqual(first.vardiff_durable_write_outcome_counts["applied"], 1) + + def test_unproven_disconnect_step_up_is_not_persisted(self) -> None: + durable = MemoryWorkerDifficultyStore() + service = self.service(runtime(durable)) + state = client(difficulty=Decimal("500000")) + state.vardiff_last_accepted_difficulty = Decimal("16384") + state.vardiff_last_accepted_wall_ms = 1_000_000 + + self.drained(service).record_session_difficulty(state, share_backed=True) + self.drained(service) + + self.assertEqual(len(durable), 0) + self.assertEqual( + service.session_difficulty_store.lookup( + ("default", "miner-a.rig-1"), + now=state.vardiff_window_started_monotonic, + )[0], + Decimal("500000"), + ) + + def test_fast_initial_step_is_not_durable_until_proven(self) -> None: + durable = MemoryWorkerDifficultyStore() + service = self.service(runtime(durable)) + state = client() + state.vardiff_last_accepted_difficulty = Decimal("16384") + state.vardiff_last_accepted_wall_ms = 1_000_000 + + service.record_session_difficulty( + state, + Decimal("1048576"), + share_backed=True, + previous_difficulty=Decimal("16384"), + initial_convergence=True, + ) + self.drained(service) + self.assertEqual(len(durable), 0) + + state.share_difficulty = Decimal("1048576") + state.vardiff_last_accepted_difficulty = Decimal("1048576") + state.vardiff_last_accepted_wall_ms = 1_000_100 + service.record_session_difficulty(state, share_backed=True) + self.drained(service) + self.assertEqual( + durable.load_recent(evidence_after_ms=1_000_000, limit=1)[0].difficulty, + Decimal("1048576"), + ) + + def test_idle_step_down_replaces_higher_durable_row_across_restart(self) -> None: + durable = MemoryWorkerDifficultyStore() + service = self.service(runtime(durable)) + state = client(difficulty=Decimal("1048576")) + state.vardiff_last_accepted_difficulty = Decimal("1048576") + state.vardiff_last_accepted_wall_ms = 1_000_000 + + with mock.patch("lab.prism.vardiff_service.time.time", return_value=1000.1): + service.record_session_difficulty(state, share_backed=True) + with mock.patch("lab.prism.vardiff_service.time.time", return_value=1000.2): + service.record_session_difficulty( + state, + Decimal("65536"), + share_backed=False, + previous_difficulty=Decimal("1048576"), + ) + self.drained(service) + with ( + mock.patch("lab.prism.vardiff_service.time.time", return_value=1000.3), + mock.patch("lab.prism.vardiff_service.time.monotonic", return_value=2000.0), + ): + restarted = self.service(runtime(durable)) + + self.assertEqual( + restarted.session_difficulty_store.lookup( + ("default", "miner-a.rig-1"), + now=2000.0, + )[0], + Decimal("65536"), + ) + + def test_retention_kill_switch_prevents_durable_writes(self) -> None: + durable = MemoryWorkerDifficultyStore() + service = self.service(runtime(durable, enabled=False)) + state = client() + state.vardiff_last_accepted_difficulty = Decimal("16384") + state.vardiff_last_accepted_wall_ms = 1_000_000 + + service.record_session_difficulty( + state, + Decimal("500000"), + share_backed=True, + ) + self.drained(service) + + self.assertEqual(len(durable), 0) + self.assertEqual( + sum(service.vardiff_durable_write_outcome_counts.values()), + 0, + ) + + def test_startup_prune_removes_all_expired_rows_not_only_cache_limit(self) -> None: + durable = MemoryWorkerDifficultyStore() + for index in range(5): + durable.upsert( + listener="default", + worker_username=f"expired-{index}", + difficulty=Decimal("16384"), + evidence_at_ms=1_000, + now_ms=1_000, + ) + with mock.patch("lab.prism.vardiff_service.time.time", return_value=1000.0): + service = VardiffService( + runtime(durable, ttl_seconds=1.0, max_entries=2) + ) + + self.assertEqual(len(durable), 0) + self.assertEqual(service.vardiff_durable_pruned_records, 5) + + def test_exact_difficulty_share_evidence_persists_on_disconnect(self) -> None: + durable = MemoryWorkerDifficultyStore() + service = self.service(runtime(durable)) + state = client(difficulty=Decimal("500000")) + state.vardiff_last_accepted_difficulty = Decimal("500000") + state.vardiff_last_accepted_wall_ms = 1_000_000 + + with mock.patch("lab.prism.vardiff_service.time.time", return_value=1000.2): + service.record_session_difficulty(state, share_backed=True) + self.drained(service) + + recent = durable.load_recent(evidence_after_ms=999_999, limit=10) + self.assertEqual(len(recent), 1) + self.assertEqual(recent[0].difficulty, Decimal("500000")) + + def test_preload_preserves_remaining_ttl(self) -> None: + durable = MemoryWorkerDifficultyStore() + durable.upsert( + listener="default", + worker_username="miner-a.rig-1", + difficulty=Decimal("500000"), + evidence_at_ms=100_000, + now_ms=100_000, + ) + with ( + mock.patch("lab.prism.vardiff_service.time.time", return_value=109.0), + mock.patch( + "lab.prism.vardiff_service.time.monotonic", + return_value=1000.0, + ), + ): + service = self.service(runtime(durable, ttl_seconds=10.0)) + + self.assertEqual( + service.session_difficulty_store.lookup( + ("default", "miner-a.rig-1"), + now=1000.5, + )[0], + Decimal("500000"), + ) + self.assertEqual( + service.session_difficulty_store.lookup( + ("default", "miner-a.rig-1"), + now=1001.1, + ), + (None, "expired"), + ) + + def test_preload_is_lane_and_exact_username_scoped(self) -> None: + durable = MemoryWorkerDifficultyStore() + for listener, username, difficulty in ( + ("default", "miner-a.rig-1", Decimal("500000")), + ("highdiff", "miner-a.rig-1", Decimal("750000")), + ("default", "miner-a.RIG-1", Decimal("250000")), + ): + durable.upsert( + listener=listener, + worker_username=username, + difficulty=difficulty, + evidence_at_ms=1_000_000, + now_ms=1_000_000, + ) + with ( + mock.patch("lab.prism.vardiff_service.time.time", return_value=1000.1), + mock.patch( + "lab.prism.vardiff_service.time.monotonic", + return_value=2000.0, + ), + ): + service = self.service(runtime(durable)) + + for key, expected in ( + (("default", "miner-a.rig-1"), Decimal("500000")), + (("highdiff", "miner-a.rig-1"), Decimal("750000")), + (("default", "miner-a.RIG-1"), Decimal("250000")), + ): + self.assertEqual( + service.session_difficulty_store.lookup(key, now=2000.0)[0], + expected, + ) + + def test_store_failure_falls_back_without_breaking_retention(self) -> None: + class FailingStore(MemoryWorkerDifficultyStore): + def upsert(self, **kwargs: object): # type: ignore[no-untyped-def] + raise RuntimeError("database unavailable") + + durable = FailingStore() + service = self.service(runtime(durable)) + state = client() + state.vardiff_last_accepted_difficulty = Decimal("16384") + state.vardiff_last_accepted_wall_ms = 1_000_000 + + service.record_session_difficulty( + state, + Decimal("500000"), + share_backed=True, + ) + self.drained(service) + + self.assertEqual(service.vardiff_durable_write_outcome_counts["failed"], 1) + self.assertEqual( + service.session_difficulty_store.lookup( + ("default", "miner-a.rig-1"), + now=state.vardiff_window_started_monotonic, + )[0], + Decimal("500000"), + ) + + def test_durable_write_never_waits_on_the_client_job_lock(self) -> None: + # The payload is snapshotted by record_session_difficulty, so the lane + # has no reason to touch client state -- and a lane that waited on + # job_update_lock would let one client's slow job send stall every + # other client's write behind it on the single worker. + invoked = threading.Event() + + def run_json(_sql: str) -> dict[str, object]: + invoked.set() + return {"applied": True, "stored": None} + + service = self.service(runtime(MemoryWorkerDifficultyStore())) + service.worker_difficulty_store = PostgresWorkerDifficultyStore(run_json) + state = client() + state.vardiff_last_accepted_difficulty = Decimal("16384") + state.vardiff_last_accepted_wall_ms = 1_000_000 + + with state.job_update_lock: + service.record_session_difficulty( + state, + Decimal("500000"), + share_backed=True, + ) + self.assertTrue(invoked.wait(5.0)) + + self.assertEqual( + service.vardiff_durable_write_outcome_counts["applied"], + 1, + ) + + def test_one_clients_held_job_lock_cannot_stall_another_clients_write( + self, + ) -> None: + service = self.service(runtime(MemoryWorkerDifficultyStore())) + blocked = client(username="blocked.rig") + blocked.vardiff_last_accepted_difficulty = Decimal("16384") + blocked.vardiff_last_accepted_wall_ms = 1_000_000 + other = client(username="other.rig") + other.vardiff_last_accepted_difficulty = Decimal("16384") + other.vardiff_last_accepted_wall_ms = 1_000_000 + + with blocked.job_update_lock: + service.record_session_difficulty( + blocked, Decimal("500000"), share_backed=True + ) + service.record_session_difficulty( + other, Decimal("500000"), share_backed=True + ) + self.assertTrue(service.flush_durable_writes(timeout=5.0)) + + self.assertEqual( + service.vardiff_durable_write_outcome_counts["applied"], + 2, + ) + + def test_saturated_lane_evicts_resume_hints_before_step_downs(self) -> None: + durable = MemoryWorkerDifficultyStore() + service = self.service(runtime(durable, max_entries=4096)) + + # Park the lane so the queue shape is deterministic: no worker is + # concurrently draining entries out from under the assertions. + with mock.patch.object(service, "_ensure_durable_worker_locked"): + capacity = MAX_PENDING_VARDIFF_DURABLE_WRITES + for index in range(capacity): + service._enqueue_durable_write( + _PendingDurableWrite( + key=("default", f"hint-{index}"), + downward_only=False, + difficulty=Decimal("16384"), + evidence_at_ms=1_000_000, + now_ms=1_000_000, + ) + ) + self.assertEqual(service.durable_pending_depth(), capacity) + # Turn one hint into a composite whose correction matters if its + # evidence is stale. It must receive the same queue priority as a + # pure step-down even though its primary operation is an upsert. + service._enqueue_durable_write( + _PendingDurableWrite( + key=("default", "hint-0"), + downward_only=True, + difficulty=Decimal("1024"), + evidence_at_ms=None, + now_ms=1_000_100, + ) + ) + # New step-downs must still get in by evicting the remaining pure + # hints rather than any safety-critical correction. + for index in range(8): + service._enqueue_durable_write( + _PendingDurableWrite( + key=("default", f"stepdown-{index}"), + downward_only=True, + difficulty=Decimal("1024"), + evidence_at_ms=None, + now_ms=1_000_000, + ) + ) + self.assertEqual(service.durable_pending_depth(), capacity) + with service._vardiff_durable_lock: + queued = list(service._vardiff_durable_pending.values()) + + self.assertEqual(sum(1 for entry in queued if entry.downward_only), 8) + self.assertEqual( + sum(1 for entry in queued if entry.has_downward_correction), + 9, + ) + self.assertEqual(service.vardiff_durable_downward_dropped, 0) + self.assertEqual( + service.vardiff_durable_write_outcome_counts["dropped"], + 8, + ) + + def test_a_step_down_the_lane_cannot_keep_is_counted_and_logged(self) -> None: + service = self.service(runtime(MemoryWorkerDifficultyStore())) + + with mock.patch.object(service, "_ensure_durable_worker_locked"): + for index in range(MAX_PENDING_VARDIFF_DURABLE_WRITES + 4): + service._enqueue_durable_write( + _PendingDurableWrite( + key=("default", f"stepdown-{index}"), + downward_only=True, + difficulty=Decimal("1024"), + evidence_at_ms=None, + now_ms=1_000_000, + ) + ) + depth = service.durable_pending_depth() + + # Bounded, and the losses are loud rather than silent. + self.assertEqual(depth, MAX_PENDING_VARDIFF_DURABLE_WRITES) + self.assertGreaterEqual(service.vardiff_durable_downward_dropped, 4) + self.assertIn( + "qbit_prism_vardiff_durable_downward_dropped_total", + "\n".join(service.metrics_lines()), + ) + + def test_queued_writes_coalesce_by_worker_identity(self) -> None: + durable = MemoryWorkerDifficultyStore() + service = self.service(runtime(durable)) + + with mock.patch.object(service, "_ensure_durable_worker_locked"): + for difficulty in ("1048576", "262144", "65536"): + service._enqueue_durable_write( + _PendingDurableWrite( + key=("default", "miner-a.rig-1"), + downward_only=True, + difficulty=Decimal(difficulty), + evidence_at_ms=None, + now_ms=1_000_000, + ) + ) + self.assertEqual(service.durable_pending_depth(), 1) + with service._vardiff_durable_lock: + merged = service._vardiff_durable_pending[ + ("default", "miner-a.rig-1") + ] + + # Three step-downs collapse to the lowest, which is what applying all + # three in order would have left behind. + self.assertEqual(merged.difficulty, Decimal("65536")) + self.assertTrue(merged.downward_only) + self.assertEqual(service.vardiff_durable_coalesced, 2) + + def test_a_later_step_down_clamps_a_queued_evidence_write(self) -> None: + service = self.service(runtime(MemoryWorkerDifficultyStore())) + upsert = _PendingDurableWrite( + key=("default", "miner-a.rig-1"), + downward_only=False, + difficulty=Decimal("1048576"), + evidence_at_ms=1_000_000, + now_ms=1_000_000, + ) + lower = _PendingDurableWrite( + key=("default", "miner-a.rig-1"), + downward_only=True, + difficulty=Decimal("65536"), + evidence_at_ms=None, + now_ms=1_000_100, + ) + + merged = upsert.merged_with(lower) + + # upsert-then-lower must leave the minimum difficulty AND the upsert's + # evidence, so the merge cannot discard the share evidence. + self.assertFalse(merged.downward_only) + self.assertEqual(merged.difficulty, Decimal("65536")) + self.assertEqual(merged.evidence_at_ms, 1_000_000) + # A later evidence write supersedes the step-down if it applies, but + # must retain the correction for the branch where the store already + # has still-newer evidence and rejects the queued upsert. + lower_then_upsert = lower.merged_with(upsert) + self.assertEqual(lower_then_upsert.difficulty, upsert.difficulty) + self.assertEqual( + lower_then_upsert.stale_downward_difficulty, + lower.difficulty, + ) + + def test_coalescing_keeps_newer_evidence_when_it_was_queued_first(self) -> None: + newer = _PendingDurableWrite( + key=("default", "miner-a.rig-1"), + downward_only=False, + difficulty=Decimal("1048576"), + evidence_at_ms=1_000_100, + now_ms=1_000_100, + ) + older = _PendingDurableWrite( + key=newer.key, + downward_only=False, + difficulty=Decimal("65536"), + evidence_at_ms=1_000_000, + now_ms=1_000_200, + ) + + merged = newer.merged_with(older) + + # Applying newer then older leaves the newer row standing because the + # store rejects the second write as stale. Coalescing must preserve the + # same evidence and difficulty. + self.assertEqual(merged.evidence_at_ms, newer.evidence_at_ms) + self.assertEqual(merged.difficulty, newer.difficulty) + self.assertEqual(merged.now_ms, newer.now_ms) + + def test_coalesced_step_down_still_applies_when_upsert_is_stale(self) -> None: + durable = MemoryWorkerDifficultyStore() + service = self.service(runtime(durable)) + durable.upsert( + listener="default", + worker_username="miner-a.rig-1", + difficulty=Decimal("1048576"), + evidence_at_ms=1_000_200, + now_ms=1_000_200, + ) + stale_upsert = _PendingDurableWrite( + key=("default", "miner-a.rig-1"), + downward_only=False, + difficulty=Decimal("262144"), + evidence_at_ms=1_000_100, + now_ms=1_000_300, + ) + lower = _PendingDurableWrite( + key=stale_upsert.key, + downward_only=True, + difficulty=Decimal("65536"), + evidence_at_ms=None, + now_ms=1_000_400, + ) + + service._apply_durable_write(stale_upsert.merged_with(lower)) + + stored = durable._entries[stale_upsert.key] + self.assertEqual(stored.difficulty, Decimal("65536")) + self.assertEqual(stored.evidence_at_ms, 1_000_200) + self.assertEqual(stored.updated_at_ms, 1_000_400) + self.assertEqual(service.vardiff_durable_write_outcome_counts["stale"], 1) + self.assertEqual(service.vardiff_durable_write_outcome_counts["lowered"], 1) + + def test_coalescing_is_order_equivalent_for_upserts_and_step_downs(self) -> None: + key = ("default", "miner-a.rig-1") + operation_specs = ( + (False, Decimal("16"), 1_000_000), + (False, Decimal("64"), 1_000_100), + (False, Decimal("8"), 1_000_100), + (True, Decimal("8"), None), + (True, Decimal("32"), None), + (True, Decimal("128"), None), + ) + seeds = ( + None, + (999_900, Decimal("4")), + (999_900, Decimal("256")), + (1_000_050, Decimal("32")), + (1_000_200, Decimal("256")), + ) + + def apply(store, pending: _PendingDurableWrite) -> None: + if pending.downward_only: + store.apply_downward( + listener=key[0], + worker_username=key[1], + difficulty=pending.difficulty, + now_ms=pending.now_ms, + ) + return + result = store.upsert( + listener=key[0], + worker_username=key[1], + difficulty=pending.difficulty, + evidence_at_ms=int(pending.evidence_at_ms or 0), + now_ms=pending.now_ms, + ) + stale_downward = pending._stale_downward() + if not result.applied and stale_downward is not None: + store.apply_downward( + listener=key[0], + worker_username=key[1], + difficulty=stale_downward[0], + now_ms=stale_downward[1], + ) + + for length in range(1, 5): + for specs in product(operation_specs, repeat=length): + operations = [ + _PendingDurableWrite( + key=key, + downward_only=downward_only, + difficulty=difficulty, + evidence_at_ms=evidence_at_ms, + now_ms=1_001_000 + index, + ) + for index, (downward_only, difficulty, evidence_at_ms) in enumerate( + specs + ) + ] + merged = operations[0] + for operation in operations[1:]: + merged = merged.merged_with(operation) + for seed in seeds: + sequential = MemoryWorkerDifficultyStore() + coalesced = MemoryWorkerDifficultyStore() + if seed is not None: + for store in (sequential, coalesced): + store.upsert( + listener=key[0], + worker_username=key[1], + difficulty=seed[1], + evidence_at_ms=seed[0], + now_ms=999_000, + ) + for operation in operations: + apply(sequential, operation) + apply(coalesced, merged) + self.assertEqual( + coalesced._entries, + sequential._entries, + (specs, seed, merged), + ) + + def test_shutdown_drains_step_downs_instead_of_cancelling_them(self) -> None: + durable = MemoryWorkerDifficultyStore() + # Live evidence, so the service's startup prune leaves these standing. + live_ms = max(0, int(time.time() * 1000)) + for index in range(6): + durable.upsert( + listener="default", + worker_username=f"stepdown-{index}", + difficulty=Decimal("1048576"), + evidence_at_ms=live_ms, + now_ms=live_ms, + ) + service = self.service(runtime(durable)) + + # Queue with the lane parked, so every entry is still pending when + # shutdown begins -- the case where cancelling would lose them. + with mock.patch.object(service, "_ensure_durable_worker_locked"): + for index in range(6): + service._enqueue_durable_write( + _PendingDurableWrite( + key=("default", f"stepdown-{index}"), + downward_only=True, + difficulty=Decimal("65536"), + evidence_at_ms=None, + now_ms=1_000_100, + ) + ) + self.assertEqual(service.durable_pending_depth(), 6) + service.shutdown_durable_executor() + + self.assertEqual(service.durable_pending_depth(), 0) + self.assertEqual(service.vardiff_durable_downward_dropped, 0) + for index in range(6): + self.assertEqual( + durable._entries[("default", f"stepdown-{index}")].difficulty, + Decimal("65536"), + ) + + def test_shutdown_drain_stays_bounded_when_the_store_hangs(self) -> None: + service = self.service(runtime(MemoryWorkerDifficultyStore())) + service._vardiff_durable_operation_timeout_seconds = 0.3 + + with mock.patch.object(service, "_ensure_durable_worker_locked"): + for index in range(40): + service._enqueue_durable_write( + _PendingDurableWrite( + key=("default", f"stepdown-{index}"), + downward_only=True, + difficulty=Decimal("65536"), + evidence_at_ms=None, + now_ms=1_000_100, + ) + ) + # Each statement is bounded by the store's own operation deadline; the + # drain's job is to bound how many of them shutdown will wait for. + with mock.patch.object( + service, + "_apply_durable_write", + side_effect=lambda pending: time.sleep(0.02), + ): + started = time.monotonic() + service.shutdown_durable_executor() + elapsed = time.monotonic() - started + + # Bounded by the ledger operation budget, and whatever the budget + # could not cover is accounted for rather than vanishing. + self.assertLess(elapsed, 3.0) + self.assertEqual(service.durable_pending_depth(), 0) + self.assertGreater(service.vardiff_durable_downward_dropped, 0) + + def test_step_down_never_suppresses_later_share_backed_evidence(self) -> None: + durable = MemoryWorkerDifficultyStore() + service = self.service(runtime(durable)) + state = client(difficulty=Decimal("1048576")) + state.vardiff_last_accepted_difficulty = Decimal("1048576") + state.vardiff_last_accepted_wall_ms = 1_000_000 + + # A share-backed write, then an idle step-down whose wall clock is + # LATER than the share evidence that follows it. + with mock.patch("lab.prism.vardiff_service.time.time", return_value=1000.1): + service.record_session_difficulty(state, share_backed=True) + with mock.patch("lab.prism.vardiff_service.time.time", return_value=1200.0): + service.record_session_difficulty( + state, + Decimal("65536"), + share_backed=False, + previous_difficulty=Decimal("1048576"), + ) + self.drained(service) + stored = durable._entries[("default", "miner-a.rig-1")] + self.assertEqual(stored.difficulty, Decimal("65536")) + # The correction carries no evidence of its own, so evidence_at still + # points at the last accepted share. + self.assertEqual(stored.evidence_at_ms, 1_000_000) + + # Genuine share evidence recorded a moment after the last accepted + # share -- but before the step-down's wall clock -- must still apply. + state.share_difficulty = Decimal("500000") + state.vardiff_last_accepted_difficulty = Decimal("500000") + state.vardiff_last_accepted_wall_ms = 1_000_050 + with mock.patch("lab.prism.vardiff_service.time.time", return_value=1300.0): + service.record_session_difficulty(state, share_backed=True) + self.drained(service) + + self.assertEqual( + durable._entries[("default", "miner-a.rig-1")].difficulty, + Decimal("500000"), + ) + self.assertEqual( + service.vardiff_durable_write_outcome_counts["stale"], + 0, + ) + + def test_step_down_never_inserts_a_row_for_an_unknown_worker(self) -> None: + durable = MemoryWorkerDifficultyStore() + service = self.service(runtime(durable)) + state = client(difficulty=Decimal("1048576")) + + service.record_session_difficulty( + state, + Decimal("65536"), + share_backed=False, + previous_difficulty=Decimal("1048576"), + ) + self.drained(service) + + # Nothing stored means nothing a restart could resurrect. + self.assertEqual(len(durable), 0) + self.assertEqual( + service.vardiff_durable_write_outcome_counts["unchanged"], + 1, + ) + + def test_periodic_prune_runs_without_any_successful_write(self) -> None: + durable = MemoryWorkerDifficultyStore() + with mock.patch("lab.prism.vardiff_service.time.time", return_value=1000.0): + service = self.service(runtime(durable, ttl_seconds=1.0)) + self.assertEqual(service.vardiff_durable_pruned_records, 0) + + # Rows this process never wrote -- inherited from a previous run. + for index in range(3): + durable.upsert( + listener="default", + worker_username=f"expired-{index}", + difficulty=Decimal("16384"), + evidence_at_ms=1_000, + now_ms=1_000, + ) + + # Not yet due: the interval is what drives it, not write traffic. + self.assertFalse(service.request_durable_prune_if_due()) + self.assertEqual(len(durable), 3) + + service._vardiff_durable_next_prune_monotonic = time.monotonic() - 1.0 + with mock.patch("lab.prism.vardiff_service.time.time", return_value=2000.0): + self.assertTrue(service.request_durable_prune_if_due()) + self.assertTrue(service.flush_durable_writes(timeout=5.0)) + for _ in range(500): + if not durable._entries: + break + time.sleep(0.005) + + self.assertEqual(len(durable), 0) + self.assertEqual(service.vardiff_durable_pruned_records, 3) + + def test_periodic_prune_runs_when_idle_sweeps_are_disabled(self) -> None: + pruned = threading.Event() + + class RecordingStore(MemoryWorkerDifficultyStore): + def prune(self, **kwargs: object) -> int: # type: ignore[override] + deleted = super().prune(**kwargs) # type: ignore[arg-type] + pruned.set() + return deleted + + durable = RecordingStore() + service = self.service(runtime(durable, ttl_seconds=1.0)) + pruned.clear() # Ignore the service's empty startup-prune call. + durable.upsert( + listener="default", + worker_username="expired", + difficulty=Decimal("16384"), + evidence_at_ms=1_000, + now_ms=1_000, + ) + service._vardiff_durable_next_prune_monotonic = time.monotonic() - 1.0 + + # No idle-sweep runtime fields or thread are involved: the durable + # lane's scheduler must wake and prune on its own. + service.start_durable_prune_scheduler() + service._vardiff_durable_wake.set() + + self.assertTrue(pruned.wait(5.0)) + self.assertEqual(len(durable), 0) + + def test_serve_starts_durable_pruning_before_accepting(self) -> None: + source = inspect.getsource(PrismCoordinator._serve_with_listener_stack) + scheduler = source.index("start_durable_prune_scheduler") + + self.assertLess(scheduler, source.index("self.accept_loop")) + + def test_prune_metric_is_a_counter_covering_every_prune(self) -> None: + service = self.service(runtime(MemoryWorkerDifficultyStore())) + text = "\n".join(service.metrics_lines()) + + self.assertIn( + "# TYPE qbit_prism_vardiff_durable_pruned_total counter", + text, + ) + self.assertIn("qbit_prism_vardiff_durable_pruned_total 0", text) + self.assertNotIn("qbit_prism_vardiff_durable_pruned gauge", text) + self.assertIn( + "# TYPE qbit_prism_vardiff_durable_prune_failures_total counter", + text, + ) + + def test_step_down_after_shutdown_is_applied_inline_not_dropped(self) -> None: + durable = MemoryWorkerDifficultyStore() + live_ms = max(0, int(time.time() * 1000)) + durable.upsert( + listener="default", + worker_username="miner-a.rig-1", + difficulty=Decimal("1048576"), + evidence_at_ms=live_ms, + now_ms=live_ms, + ) + service = self.service(runtime(durable)) + service.shutdown_durable_executor() + + # A disconnect landing after the lane closed still has to correct the + # stored row; queueing it would strand it past the drain. + service._enqueue_durable_write( + _PendingDurableWrite( + key=("default", "miner-a.rig-1"), + downward_only=True, + difficulty=Decimal("65536"), + evidence_at_ms=None, + now_ms=live_ms + 100, + ) + ) + + self.assertEqual( + durable._entries[("default", "miner-a.rig-1")].difficulty, + Decimal("65536"), + ) + self.assertEqual(service.vardiff_durable_downward_dropped, 0) + self.assertEqual( + service.vardiff_durable_write_outcome_counts["lowered"], + 1, + ) + # An optional resume hint arriving after the drain is still dropped. + service._enqueue_durable_write( + _PendingDurableWrite( + key=("default", "other.rig"), + downward_only=False, + difficulty=Decimal("500000"), + evidence_at_ms=live_ms, + now_ms=live_ms, + ) + ) + self.assertEqual( + service.vardiff_durable_write_outcome_counts["dropped"], + 1, + ) + + def test_every_prune_statement_carries_a_finite_limit(self) -> None: + limits: list[object] = [] + + class RecordingStore(MemoryWorkerDifficultyStore): + def prune(self, **kwargs: object) -> int: # type: ignore[override] + limits.append(kwargs.get("limit")) + return super().prune(**kwargs) # type: ignore[arg-type] + + durable = RecordingStore() + with mock.patch("lab.prism.vardiff_service.time.time", return_value=1000.0): + service = self.service(runtime(durable, ttl_seconds=1.0)) + service._vardiff_durable_next_prune_monotonic = time.monotonic() - 1.0 + service.request_durable_prune_if_due() + service._prune_durable_worker_difficulties_if_requested() + + self.assertTrue(limits) + for limit in limits: + self.assertIsInstance(limit, int) + self.assertGreater(limit, 0) + + def test_prune_batches_until_the_backlog_drains(self) -> None: + durable = MemoryWorkerDifficultyStore() + with mock.patch("lab.prism.vardiff_service.time.time", return_value=1000.0): + service = self.service(runtime(durable, ttl_seconds=1.0)) + batches: list[int] = [] + + def batched(**kwargs: object) -> int: + batches.append(int(kwargs["limit"])) # type: ignore[arg-type] + # Two full batches, then a short one: the loop must stop there. + return ( + PRISM_VARDIFF_DURABLE_PRUNE_BATCH + if len(batches) <= 2 + else 7 + ) + + with mock.patch.object(durable, "prune", side_effect=batched): + pruned = service._prune_expired_rows(1_000) + + self.assertEqual(len(batches), 3) + self.assertEqual(pruned, 2 * PRISM_VARDIFF_DURABLE_PRUNE_BATCH + 7) + self.assertEqual( + batches, + [PRISM_VARDIFF_DURABLE_PRUNE_BATCH] * 3, + ) + + def test_prune_batching_is_capped_per_pass(self) -> None: + durable = MemoryWorkerDifficultyStore() + with mock.patch("lab.prism.vardiff_service.time.time", return_value=1000.0): + service = self.service(runtime(durable, ttl_seconds=1.0)) + + with mock.patch.object( + durable, + "prune", + side_effect=lambda **kwargs: PRISM_VARDIFF_DURABLE_PRUNE_BATCH, + ) as always_full: + service._prune_expired_rows(1_000) + + # An endless backlog must not hold the lane for an unbounded pass. + self.assertEqual( + always_full.call_count, + PRISM_VARDIFF_DURABLE_PRUNE_MAX_BATCHES, + ) + + def test_metrics_expose_only_bounded_durable_outcomes(self) -> None: + service = self.service(runtime(MemoryWorkerDifficultyStore())) + text = "\n".join(service.metrics_lines()) + + self.assertIn("qbit_prism_vardiff_durable_preloaded 0", text) + for outcome in ("applied", "stale", "failed", "dropped"): + self.assertIn( + f'qbit_prism_vardiff_durable_writes_total{{outcome="{outcome}"}} 0', + text, + ) + self.assertIn( + 'qbit_prism_vardiff_durable_backend{backend="memory"} 1', + text, + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_prism_vardiff_fast_arrival.py b/tests/test_prism_vardiff_fast_arrival.py new file mode 100644 index 000000000..46d576829 --- /dev/null +++ b/tests/test_prism_vardiff_fast_arrival.py @@ -0,0 +1,451 @@ +#!/usr/bin/env python3 +"""Fast-arrival initial vardiff convergence (#132). + +Credit is target-based: a share is worth the difficulty its job was stamped +at. A rental router that filters submissions at its own, much higher +difficulty therefore under-credits its miner for as long as the stamped +target lags the difficulty the rig is really working at. These tests pin the +incident shape -- a standard-lane session started at 16384 behind a router +filtering at 500000 -- and the guards that keep the fast path from touching +anyone else. +""" +# ruff: noqa: F403, F405 + +from __future__ import annotations + +import unittest + +from lab.prism.vardiff_service import ( + PRISM_VARDIFF_HIGH_DIFF_ARRIVAL_SECONDS_BUCKETS, + PRISM_VARDIFF_HIGH_DIFF_ARRIVAL_SHARES_BUCKETS, + PRISM_VARDIFF_INITIAL_RETARGET_OUTCOMES, +) +from tests.prism_vardiff_test_support import * + + +STANDARD_LANE_START = Decimal("16384") +ROUTER_FILTER_DIFFICULTY = Decimal("500000") +# 9.4 PH/s clearing a 500000 filter: 9.4e15 / (500000 * 2**32) accepted +# shares per second, so one every ~0.2285s. +ROUTER_SHARE_INTERVAL_SECONDS = 0.2285 + + +def standard_lane_config(**overrides: object) -> vardiff.VardiffConfig: + """The deployed standard lane: 16384 start, 4x steps, 300s retargets.""" + base = dict( + enabled=True, + target_share_interval_seconds=Decimal("15"), + min_difficulty=STANDARD_LANE_START, + max_difficulty=Decimal("4294967296"), + retarget_interval_seconds=Decimal("300"), + max_step_factor=Decimal("4"), + startup_difficulty=STANDARD_LANE_START, + max_step_down_factor=Decimal("4"), + ewma_alpha=Decimal("0.4"), + retarget_tolerance=Decimal("0.25"), + ) + base.update(overrides) + return vardiff.VardiffConfig(**base) # type: ignore[arg-type] + + +class FastArrivalHarness: + """A coordinator, a client, and a clock the test drives by hand.""" + + def __init__( + self, + config: vardiff.VardiffConfig, + *, + difficulty: Decimal = STANDARD_LANE_START, + ) -> None: + self.server = coordinator() + self.server.vardiff_config = config + self.client = client() + self.client.share_difficulty = difficulty + self.now = 1000.0 + self.client.vardiff_window_started_monotonic = self.now + self.client.vardiff_session_started_monotonic = self.now + self.sent: list[dict[str, object]] = [] + self.server.maybe_send_job = self._send # type: ignore[method-assign] + + def _send(self, client: object, clean_jobs: bool) -> bool: + self.sent.append({"clean": clean_jobs, "at": self.now}) + return True + + @property + def service(self) -> object: + return self.server._ensure_vardiff_service() + + @property + def difficulty(self) -> Decimal: + return self.client.pending_share_difficulty or self.client.share_difficulty + + def deliver(self, count: int, *, interval: float, difficulty: Decimal | None = None) -> None: + """Feed ``count`` accepted shares stamped at the client's target.""" + for _ in range(count): + self.now += interval + stamped = difficulty if difficulty is not None else self.difficulty + with patch( + "lab.prism.vardiff_service.time.monotonic", + side_effect=lambda: self.now, + ): + self.server.note_vardiff_submitted_share(self.client) + self.server.note_vardiff_accepted_share( + self.client, + FakeJob(stamped), # type: ignore[arg-type] + ) + + def metrics(self) -> dict[str, str]: + lines = self.server.vardiff_idle_metrics_lines() + return { + line.rsplit(" ", 1)[0]: line.rsplit(" ", 1)[1] + for line in lines + if not line.startswith("#") + } + + +class PrismVardiffFastArrivalTests(unittest.TestCase): + def test_router_filtered_session_reaches_router_difficulty_in_the_initial_burst( + self, + ) -> None: + # The incident: a 9.4 PH/s rental order behind a router that only + # forwards shares clearing 500000, into a standard lane that stamps + # 16384. Every share the pool sees is worth ~30x what it is credited, + # and the ordinary ladder needs three 300s retargets to close that. + harness = FastArrivalHarness(standard_lane_config()) + self.assertEqual(harness.difficulty, STANDARD_LANE_START) + + harness.deliver(8, interval=ROUTER_SHARE_INTERVAL_SECONDS) + + self.assertGreaterEqual(harness.difficulty, ROUTER_FILTER_DIFFICULTY) + self.assertEqual(harness.difficulty, STANDARD_LANE_START * 64) + # Arrival is bounded by the burst, not by the retarget interval. + elapsed = harness.now - 1000.0 + self.assertLess(elapsed, 2.0) + self.assertLess( + Decimal(str(elapsed)), + standard_lane_config().retarget_interval_seconds, + ) + # The step up keeps the miner's in-flight work: one non-clean job. + self.assertEqual(len(harness.sent), 1) + self.assertFalse(harness.sent[0]["clean"]) + + def test_ordinary_ladder_would_still_be_short_after_the_same_burst(self) -> None: + # Same burst with the fast path off: the ordinary 4x bound lands at + # 65536, still 7.6x below what the router is filtering at, and the + # next move cannot come until the 300s interval elapses. + harness = FastArrivalHarness( + standard_lane_config(initial_convergence_enabled=False) + ) + + harness.deliver(8, interval=ROUTER_SHARE_INTERVAL_SECONDS) + + self.assertEqual(harness.difficulty, STANDARD_LANE_START) + self.assertEqual(harness.sent, []) + + # Only a full ordinary window of that same cadence retargets, and the + # 4x bound lands at 65536 -- still 7.6x below what the router filters + # at, with another 300s before the next move. Three intervals of + # under-credited shares is what the fast path removes. + harness.deliver(1305, interval=ROUTER_SHARE_INTERVAL_SECONDS) + + self.assertGreaterEqual(harness.now - 1000.0, 300.0) + self.assertEqual(harness.difficulty, STANDARD_LANE_START * 4) + self.assertLess(harness.difficulty, ROUTER_FILTER_DIFFICULTY) + + def test_on_cadence_small_miner_is_not_driven_upward(self) -> None: + # One accepted share per target interval, for well past the point + # where the share threshold is met. The early path exists but never + # engages, and no difficulty change is advertised. + harness = FastArrivalHarness(standard_lane_config()) + + harness.deliver(20, interval=15.0) + + self.assertEqual(harness.difficulty, STANDARD_LANE_START) + self.assertEqual(harness.sent, []) + self.assertTrue(self.client_initial_pending(harness)) + metrics = harness.metrics() + self.assertEqual( + metrics["qbit_prism_vardiff_initial_retarget_attempts_total"], "0" + ) + + def test_burst_shorter_than_the_elapsed_floor_waits(self) -> None: + # Eight shares read off the socket back to back. The work is real but + # the window is not: dividing by ~0 would observe ~1e9. The elapsed + # floor holds the session at its lane start until a window long + # enough to mean something has passed. + harness = FastArrivalHarness(standard_lane_config()) + + harness.deliver(8, interval=0.001) + + self.assertEqual(harness.difficulty, STANDARD_LANE_START) + self.assertEqual(harness.sent, []) + + # One more share carries the window past the floor, and now the + # accumulated burst converges in a single bounded move. + harness.deliver(1, interval=1.0) + + self.assertEqual(harness.difficulty, STANDARD_LANE_START * 64) + + def test_initial_step_obeys_the_lane_ceiling(self) -> None: + harness = FastArrivalHarness( + standard_lane_config(max_difficulty=Decimal("262144")) + ) + + harness.deliver(8, interval=ROUTER_SHARE_INTERVAL_SECONDS) + + self.assertEqual(harness.difficulty, Decimal("262144")) + + def test_initial_step_stays_within_high_diff_lane_bounds(self) -> None: + # A rental-scale rig on the high-diff lane, whose floor is a wire + # guarantee. The initial move is bounded above by the lane ceiling + # and can never land below the floor. + high_diff = standard_lane_config( + min_difficulty=ROUTER_FILTER_DIFFICULTY, + startup_difficulty=ROUTER_FILTER_DIFFICULTY, + max_difficulty=Decimal("8000000"), + ) + harness = FastArrivalHarness(high_diff, difficulty=ROUTER_FILTER_DIFFICULTY) + harness.client.listener_name = "highdiff" + harness.client.minimum_advertised_difficulty = ROUTER_FILTER_DIFFICULTY + + harness.deliver(8, interval=ROUTER_SHARE_INTERVAL_SECONDS) + + self.assertEqual(harness.difficulty, Decimal("8000000")) + self.assertGreaterEqual(harness.difficulty, ROUTER_FILTER_DIFFICULTY) + self.assertEqual( + harness.server.client_minimum_advertised_difficulty(harness.client), + ROUTER_FILTER_DIFFICULTY, + ) + + def test_only_the_first_qualifying_phase_gets_the_larger_bound(self) -> None: + harness = FastArrivalHarness(standard_lane_config()) + + harness.deliver(8, interval=ROUTER_SHARE_INTERVAL_SECONDS) + converged = harness.difficulty + self.assertEqual(converged, STANDARD_LANE_START * 64) + self.assertFalse(self.client_initial_pending(harness)) + + # Commit the advertised difficulty as delivered, then present an + # equally decisive window: 600 accepted shares over a full 300s + # interval observes 30x the current difficulty. The relaxation is + # spent, so this move is bounded by the ordinary 4x, not by 30x. + harness.client.share_difficulty = converged + harness.client.pending_share_difficulty = None + harness.deliver(600, interval=0.5) + + self.assertEqual(harness.difficulty, converged * 4) + metrics = harness.metrics() + self.assertEqual( + metrics["qbit_prism_vardiff_initial_retarget_attempts_total"], "1" + ) + + def test_a_failed_send_does_not_spend_the_relaxation(self) -> None: + # The relaxation is spent at the paired-send commit point, so a build + # the coordinator skipped cannot consume it while leaving the miner + # on its old difficulty. + harness = FastArrivalHarness(standard_lane_config()) + harness.server.maybe_send_job = lambda client, clean_jobs: False # type: ignore[method-assign] + + harness.deliver(8, interval=ROUTER_SHARE_INTERVAL_SECONDS) + + self.assertEqual(harness.difficulty, STANDARD_LANE_START) + self.assertIsNone(harness.client.pending_share_difficulty) + self.assertTrue(self.client_initial_pending(harness)) + metrics = harness.metrics() + self.assertEqual( + metrics['qbit_prism_vardiff_initial_retargets_total{outcome="superseded"}'], + "1", + ) + self.assertEqual( + metrics['qbit_prism_vardiff_initial_retargets_total{outcome="applied"}'], + "0", + ) + + # Delivery recovers, and the session still converges in one move. + harness.server.maybe_send_job = harness._send # type: ignore[method-assign] + harness.deliver(8, interval=ROUTER_SHARE_INTERVAL_SECONDS) + + self.assertEqual(harness.difficulty, STANDARD_LANE_START * 64) + + def test_explicit_request_outranks_a_converged_initial_difficulty(self) -> None: + # The early path measures a session against the difficulty it is + # currently stamped at, so an explicit d= raises the baseline the + # evidence is judged against rather than being bypassed by it. And + # once vardiff has converged, a d= still resolves to exactly what + # was asked for: the request outranks the converged value. + harness = FastArrivalHarness(standard_lane_config()) + harness.deliver(8, interval=ROUTER_SHARE_INTERVAL_SECONDS) + self.assertEqual(harness.difficulty, STANDARD_LANE_START * 64) + + harness.client.requested_difficulty = Decimal("262144") + target = harness.server.apply_client_difficulty_requests(harness.client) + + self.assertEqual(target, Decimal("262144")) + self.assertEqual( + harness.client.vardiff_config.startup_difficulty, + Decimal("262144"), + ) + + def test_md_floor_is_never_undercut_by_an_initial_step(self) -> None: + # md= raises the client's own minimum. The initial step clamps to the + # specialized policy's bounds like any other step. + harness = FastArrivalHarness(standard_lane_config()) + harness.client.requested_min_difficulty = Decimal("262144") + harness.server.apply_client_difficulty_requests(harness.client) + self.assertEqual( + harness.client.vardiff_config.min_difficulty, + Decimal("262144"), + ) + + harness.deliver(8, interval=ROUTER_SHARE_INTERVAL_SECONDS) + + self.assertGreaterEqual(harness.difficulty, Decimal("262144")) + + def test_request_landing_mid_retarget_wins_over_the_initial_step(self) -> None: + # The race: the early retarget computed its step from 16384, but an + # explicit request moved the client before the paired send. The + # retarget yields rather than overriding the request. + harness = FastArrivalHarness(standard_lane_config()) + harness.client.pending_share_difficulty = ROUTER_FILTER_DIFFICULTY + + applied = harness.service.retarget_locked( + harness.client, + current_difficulty=STANDARD_LANE_START, + accepted_shares=8, + submitted_shares=8, + accepted_difficulty=STANDARD_LANE_START * 8, + elapsed_seconds=Decimal("1.828"), + initial_convergence=True, + ) + + self.assertFalse(applied) + self.assertEqual(harness.client.pending_share_difficulty, ROUTER_FILTER_DIFFICULTY) + self.assertEqual(harness.sent, []) + metrics = harness.metrics() + self.assertEqual( + metrics['qbit_prism_vardiff_initial_retargets_total{outcome="superseded"}'], + "1", + ) + + def test_idle_step_down_never_takes_the_initial_bound(self) -> None: + # An idle window carries no accepted shares, so it can never be the + # evidence the relaxation is granted for, even if a caller asks. + harness = FastArrivalHarness(standard_lane_config(), difficulty=Decimal("1048576")) + # The client is never admitted, so the idle preconditions reject this + # retarget outright. What matters is that require_idle stripped the + # relaxation before any of that: a zero-share window is not evidence. + applied = harness.service.retarget_locked( + harness.client, + current_difficulty=Decimal("1048576"), + accepted_shares=0, + submitted_shares=0, + accepted_difficulty=Decimal("0"), + elapsed_seconds=Decimal("300"), + initial_convergence=True, + require_idle=True, + prepared_bundle=object(), # type: ignore[arg-type] + ) + + self.assertFalse(applied) + metrics = harness.metrics() + self.assertEqual( + metrics["qbit_prism_vardiff_initial_retarget_attempts_total"], "0" + ) + + def test_arrival_metrics_are_bounded_and_observed_once(self) -> None: + harness = FastArrivalHarness(standard_lane_config()) + + harness.deliver(8, interval=ROUTER_SHARE_INTERVAL_SECONDS) + + metrics = harness.metrics() + self.assertEqual( + metrics["qbit_prism_vardiff_initial_retarget_attempts_total"], "1" + ) + self.assertEqual( + metrics['qbit_prism_vardiff_initial_retargets_total{outcome="applied"}'], + "1", + ) + self.assertEqual( + metrics["qbit_prism_vardiff_high_diff_arrival_seconds_count"], "1" + ) + self.assertEqual( + metrics["qbit_prism_vardiff_high_diff_arrival_shares_count"], "1" + ) + # Eight accepted shares over ~1.8s of connection life. + self.assertEqual( + metrics["qbit_prism_vardiff_high_diff_arrival_shares_sum"], "8.000000" + ) + self.assertEqual( + metrics['qbit_prism_vardiff_high_diff_arrival_shares_bucket{le="8"}'], "1" + ) + self.assertEqual( + metrics['qbit_prism_vardiff_high_diff_arrival_shares_bucket{le="4"}'], "0" + ) + self.assertEqual( + metrics['qbit_prism_vardiff_high_diff_arrival_seconds_bucket{le="2.5"}'], + "1", + ) + + # A later retarget across the same threshold does not re-observe. + harness.client.share_difficulty = harness.difficulty + harness.client.pending_share_difficulty = None + harness.deliver(8, interval=ROUTER_SHARE_INTERVAL_SECONDS) + + self.assertEqual( + harness.metrics()["qbit_prism_vardiff_high_diff_arrival_shares_count"], "1" + ) + + def test_session_starting_above_the_threshold_never_contributes(self) -> None: + high_diff = standard_lane_config( + min_difficulty=ROUTER_FILTER_DIFFICULTY, + startup_difficulty=ROUTER_FILTER_DIFFICULTY, + ) + harness = FastArrivalHarness(high_diff, difficulty=ROUTER_FILTER_DIFFICULTY) + + harness.deliver(8, interval=ROUTER_SHARE_INTERVAL_SECONDS) + + self.assertGreater(harness.difficulty, ROUTER_FILTER_DIFFICULTY) + self.assertEqual( + harness.metrics()["qbit_prism_vardiff_high_diff_arrival_shares_count"], "0" + ) + + def test_metric_label_sets_are_fixed(self) -> None: + harness = FastArrivalHarness(standard_lane_config()) + harness.deliver(8, interval=ROUTER_SHARE_INTERVAL_SECONDS) + + rendered = harness.metrics() + outcomes = [ + name + for name in rendered + if name.startswith("qbit_prism_vardiff_initial_retargets_total") + ] + self.assertEqual(len(outcomes), len(PRISM_VARDIFF_INITIAL_RETARGET_OUTCOMES)) + for outcome in PRISM_VARDIFF_INITIAL_RETARGET_OUTCOMES: + self.assertIn( + f'qbit_prism_vardiff_initial_retargets_total{{outcome="{outcome}"}}', + rendered, + ) + for metric, buckets in ( + ( + "qbit_prism_vardiff_high_diff_arrival_seconds", + PRISM_VARDIFF_HIGH_DIFF_ARRIVAL_SECONDS_BUCKETS, + ), + ( + "qbit_prism_vardiff_high_diff_arrival_shares", + PRISM_VARDIFF_HIGH_DIFF_ARRIVAL_SHARES_BUCKETS, + ), + ): + emitted = [ + name for name in rendered if name.startswith(f"{metric}_bucket") + ] + self.assertEqual(len(emitted), len(buckets) + 1) + # No worker or username ever reaches a label. + self.assertNotIn("miner", " ".join(rendered)) + + @staticmethod + def client_initial_pending(harness: FastArrivalHarness) -> bool: + return bool(harness.client.vardiff_initial_convergence_pending) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_prism_vardiff_resume.py b/tests/test_prism_vardiff_resume.py new file mode 100644 index 000000000..a0aa86f1a --- /dev/null +++ b/tests/test_prism_vardiff_resume.py @@ -0,0 +1,889 @@ +#!/usr/bin/env python3 +"""Reconnect difficulty resume: retention store, adoption, and observability. + +A reconnecting worker (same listener + exact username) resumes at its last +converged difficulty instead of the lane start, so a mass reconnect does not +re-flood the share path while every session re-climbs (the 2026-07-16 +amplifier). These tests pin the store semantics, the authorize-time adoption +ordering, the plausibility clamp, the kill switch, and the new metrics. +""" + +from __future__ import annotations + +from contextlib import contextmanager +from decimal import Decimal +from pathlib import Path +from types import SimpleNamespace +import tempfile +import threading +import time +import unittest +from typing import Iterator +from unittest import mock + +from lab.auxpow import vardiff +from lab.prism.coordinator_config import load_coordinator_config +from lab.prism.metrics import MetricsRenderer +from lab.prism.prism_coordinator import PrismCoordinator +from lab.prism.rpc import JsonRpc +from lab.prism.stratum_session import ClientState, StratumError +from lab.prism.vardiff_service import ( + PRISM_VARDIFF_RESUME_OUTCOMES, + SessionDifficultyStore, + VardiffService, + session_difficulty_key, +) +from tests.prism_vardiff_test_support import ( + PAYOUT_ADDRESS, + AddressValidationRpc, + coordinator, + highdiff_vardiff_config, + worker_identity, +) + + +STANDARD_LANE_START = Decimal("16384") + + +def standard_lane_config() -> vardiff.VardiffConfig: + """The deployed standard lane from issue #132's policy table.""" + return vardiff.VardiffConfig( + enabled=True, + target_share_interval_seconds=Decimal("15"), + min_difficulty=Decimal("1024"), + max_difficulty=Decimal("4294967296"), + retarget_interval_seconds=Decimal("300"), + max_step_factor=Decimal("4"), + startup_difficulty=STANDARD_LANE_START, + max_step_down_factor=Decimal("4"), + ) + + +class SessionDifficultyStoreTests(unittest.TestCase): + def test_lru_bound_evicts_oldest_and_counts(self) -> None: + store = SessionDifficultyStore(max_entries=2, ttl_seconds=900.0) + now = time.monotonic() + store.record(("default", "a"), Decimal("512"), now=now, share_backed=True) + store.record(("default", "b"), Decimal("256"), now=now, share_backed=True) + store.record(("default", "c"), Decimal("128"), now=now, share_backed=True) + + self.assertEqual(len(store), 2) + self.assertEqual(store.lookup(("default", "a"), now=now), (None, "miss")) + self.assertEqual( + store.lookup(("default", "c"), now=now), (Decimal("128"), "hit") + ) + self.assertEqual(store.snapshot()["evicted"], 1) + + def test_hit_refreshes_recency_but_not_ttl_and_keeps_the_entry(self) -> None: + store = SessionDifficultyStore(max_entries=2, ttl_seconds=10.0) + now = time.monotonic() + store.record(("default", "a"), Decimal("512"), now=now, share_backed=True) + store.record(("default", "b"), Decimal("1"), now=now + 1, share_backed=True) + + # A hit does not remove the entry (a miner may re-authorize) and it + # refreshes LRU recency: the next capacity eviction takes "b". + self.assertEqual( + store.lookup(("default", "a"), now=now + 2), (Decimal("512"), "hit") + ) + store.record(("default", "c"), Decimal("2"), now=now + 2, share_backed=True) + self.assertEqual(store.lookup(("default", "b"), now=now + 2), (None, "miss")) + self.assertEqual( + store.lookup(("default", "a"), now=now + 6), (Decimal("512"), "hit") + ) + # But the TTL still measures the age of the VALUE, not of the last + # access: the entry recorded at `now` is gone at now + 11. + self.assertEqual( + store.lookup(("default", "a"), now=now + 11), (None, "expired") + ) + self.assertEqual(store.snapshot()["expired"], 1) + + def test_record_silently_ignores_unusable_difficulties(self) -> None: + store = SessionDifficultyStore(max_entries=4, ttl_seconds=10.0) + now = time.monotonic() + store.record(("default", "a"), Decimal("0"), now=now, share_backed=True) + store.record(("default", "a"), Decimal("-4"), now=now, share_backed=True) + store.record(("default", "a"), Decimal("NaN"), now=now, share_backed=True) + store.record( + ("default", "a"), Decimal("Infinity"), now=now, share_backed=True + ) + self.assertEqual(len(store), 0) + + def test_non_positive_bounds_disable_retention_entirely(self) -> None: + now = time.monotonic() + for store in ( + SessionDifficultyStore(max_entries=0, ttl_seconds=10.0), + SessionDifficultyStore(max_entries=8, ttl_seconds=0.0), + ): + store.record(("default", "a"), Decimal("512"), now=now, share_backed=True) + self.assertEqual(len(store), 0) + self.assertEqual( + store.lookup(("default", "a"), now=now), (None, "disabled") + ) + + def test_prune_drops_only_expired_entries(self) -> None: + store = SessionDifficultyStore(max_entries=4, ttl_seconds=10.0) + now = time.monotonic() + store.record(("default", "old"), Decimal("512"), now=now, share_backed=True) + store.record( + ("default", "new"), Decimal("256"), now=now + 8, share_backed=True + ) + + self.assertEqual(store.prune(now=now + 11), 1) + self.assertEqual(len(store), 1) + self.assertEqual( + store.lookup(("default", "new"), now=now + 11), + (Decimal("256"), "hit"), + ) + + def test_unchanged_value_without_share_evidence_keeps_ageing(self) -> None: + # The disconnect seam records on EVERY disconnect. If that refreshed + # the timestamp, a session that resumes a retained value, submits + # nothing and disconnects would keep the value alive forever on + # reconnect cycles shorter than the TTL. + store = SessionDifficultyStore(max_entries=4, ttl_seconds=10.0) + now = time.monotonic() + store.record(("default", "a"), Decimal("512"), now=now, share_backed=True) + store.record( + ("default", "a"), Decimal("512"), now=now + 8, share_backed=False + ) + + self.assertEqual( + store.lookup(("default", "a"), now=now + 9), (Decimal("512"), "hit") + ) + self.assertEqual( + store.lookup(("default", "a"), now=now + 11), (None, "expired") + ) + + def test_ttl_refreshes_on_re_validation_only(self) -> None: + store = SessionDifficultyStore(max_entries=4, ttl_seconds=10.0) + now = time.monotonic() + # An unchanged value backed by accepted shares was re-validated. + store.record(("default", "a"), Decimal("512"), now=now, share_backed=True) + store.record( + ("default", "a"), Decimal("512"), now=now + 8, share_backed=True + ) + self.assertEqual( + store.lookup(("default", "a"), now=now + 17), (Decimal("512"), "hit") + ) + # A value that actually moved is new information, share-backed or not + # (an idle step-down carries no shares but does lower the value). + store.record(("default", "b"), Decimal("512"), now=now, share_backed=True) + store.record( + ("default", "b"), Decimal("128"), now=now + 8, share_backed=False + ) + self.assertEqual( + store.lookup(("default", "b"), now=now + 17), (Decimal("128"), "hit") + ) + + def test_non_refreshing_record_still_moves_lru_recency(self) -> None: + # Keeping the old timestamp must not turn record() into a no-op: + # recency and the TTL stay independent, so the LRU bound keeps + # evicting genuinely cold entries. + store = SessionDifficultyStore(max_entries=2, ttl_seconds=100.0) + now = time.monotonic() + store.record(("default", "a"), Decimal("512"), now=now, share_backed=True) + store.record(("default", "b"), Decimal("256"), now=now + 1, share_backed=True) + store.record(("default", "a"), Decimal("512"), now=now + 2, share_backed=False) + store.record(("default", "c"), Decimal("128"), now=now + 3, share_backed=True) + + self.assertEqual(store.lookup(("default", "b"), now=now + 3), (None, "miss")) + self.assertEqual( + store.lookup(("default", "a"), now=now + 3), (Decimal("512"), "hit") + ) + + def test_session_difficulty_key_requires_a_reserved_worker(self) -> None: + state = SimpleNamespace(listener_name="highdiff", worker=None) + self.assertIsNone(session_difficulty_key(state)) # type: ignore[arg-type] + state.worker = worker_identity("miner-a.rig-1") + self.assertEqual( + session_difficulty_key(state), # type: ignore[arg-type] + ("highdiff", "miner-a.rig-1"), + ) + + +def resume_coordinator(**attributes: object): + """A no-environment coordinator wired for authorize round trips. + + Resume attributes must land before the first vardiff-service touch, so + they are applied here, ahead of any authorize call. + """ + server = coordinator() + for name, value in attributes.items(): + setattr(server, name, value) + server.rpc = AddressValidationRpc() + server.username_fallback_address = None + server.maybe_send_job = lambda client, *, clean_jobs: True + server.jobs = {} + return server + + +def connect_client( + server, + connection_id: int, + *, + listener_name: str = "default", + lane_config: vardiff.VardiffConfig | None = None, +) -> ClientState: + """A freshly accepted connection sitting at its lane's start difficulty.""" + config = lane_config if lane_config is not None else standard_lane_config() + state = ClientState( + sock=object(), + address=("127.0.0.1", connection_id), + connection_id=connection_id, + extranonce1_hex=f"{connection_id:08x}", + listener_name=listener_name, + listener_vardiff_config=config, + share_difficulty=config.startup_difficulty, + ) + state.subscribed = True + state.send = lambda payload: None # type: ignore[method-assign] + state.close = lambda: None # type: ignore[method-assign] + return state + + +def authorize(server, state: ClientState, password: str = "x") -> None: + server.handle_request( + state, + { + "id": 1, + "method": "mining.authorize", + "params": [PAYOUT_ADDRESS, password], + }, + ) + + +def converge_and_disconnect(server, state: ClientState, difficulty: Decimal) -> None: + with server._client_vardiff_lock(state): + state.share_difficulty = difficulty + state.pending_share_difficulty = None + server.disconnect_client(state) + + +@contextmanager +def frozen_clock(now: float) -> Iterator[None]: + """Run a reconnect cycle at an exact monotonic instant. + + Retention timestamps come from time.monotonic() inside the vardiff + service, so the whole record/lookup path has to see the injected value. + """ + with mock.patch.object(time, "monotonic", return_value=now): + yield + + +def silent_reconnect_cycle(server, connection_id: int, now: float) -> ClientState: + """Connect, authorize, submit nothing, disconnect -- all at ``now``.""" + with frozen_clock(now): + state = connect_client(server, connection_id) + authorize(server, state) + server.disconnect_client(state) + return state + + +def retained_difficulty(server, now: float | None = None) -> Decimal | None: + store = server._ensure_vardiff_service().session_difficulty_store + retained, _outcome = store.lookup( + ("default", PAYOUT_ADDRESS), + now=time.monotonic() if now is None else now, + ) + return retained + + +def resume_outcomes(server) -> dict[str, int]: + outcomes = server.vardiff_convergence_snapshot()["resume_outcomes"] + assert isinstance(outcomes, dict) + return outcomes + + +class ReconnectResumeTests(unittest.TestCase): + def test_reconnect_resumes_last_converged_difficulty(self) -> None: + server = resume_coordinator() + first = connect_client(server, 1) + authorize(server, first) + self.assertEqual(first.share_difficulty, STANDARD_LANE_START) + converge_and_disconnect(server, first, Decimal("262144")) + + second = connect_client(server, 2) + authorize(server, second) + + self.assertEqual(second.share_difficulty, Decimal("262144")) + self.assertIsNone(second.pending_share_difficulty) + self.assertEqual(second.difficulty_generation, 1) + outcomes = resume_outcomes(server) + self.assertEqual(outcomes["resumed"], 1) + # The very first authorize of the worker found nothing retained. + self.assertEqual(outcomes["miss"], 1) + + def test_expired_retention_falls_back_to_lane_start(self) -> None: + server = resume_coordinator(vardiff_resume_ttl_seconds=60.0) + first = connect_client(server, 1) + authorize(server, first) + converge_and_disconnect(server, first, Decimal("262144")) + + second = connect_client(server, 2) + frozen_now = time.monotonic() + 61.0 + with mock.patch.object(time, "monotonic", return_value=frozen_now): + authorize(server, second) + + self.assertEqual(second.share_difficulty, STANDARD_LANE_START) + self.assertEqual(resume_outcomes(server)["expired"], 1) + + def test_implausible_retained_value_is_clamped_to_the_lane_ceiling(self) -> None: + server = resume_coordinator() + service = server._ensure_vardiff_service() + service.session_difficulty_store.record( + ("default", PAYOUT_ADDRESS), + Decimal("1e15"), + now=time.monotonic(), + share_backed=True, + ) + + state = connect_client(server, 1) + authorize(server, state) + + # min(lane max, lane start * default 1024x factor). + config = standard_lane_config() + expected = min( + config.max_difficulty, + config.startup_difficulty * Decimal("1024"), + ) + self.assertEqual(expected, Decimal("16777216")) + self.assertEqual(state.share_difficulty, expected) + self.assertNotEqual(state.share_difficulty, Decimal("1e15")) + self.assertEqual(resume_outcomes(server)["clamped"], 1) + + def test_retention_is_scoped_per_lane(self) -> None: + server = resume_coordinator() + first = connect_client( + server, + 1, + listener_name="highdiff", + lane_config=highdiff_vardiff_config(), + ) + authorize(server, first) + converge_and_disconnect(server, first, Decimal("2000000")) + + second = connect_client(server, 2, listener_name="default") + authorize(server, second) + + # A lane switch is a miss that behaves exactly like today. + self.assertEqual(second.share_difficulty, STANDARD_LANE_START) + outcomes = resume_outcomes(server) + self.assertEqual(outcomes["miss"], 2) + self.assertEqual(outcomes["resumed"], 0) + self.assertEqual(outcomes["clamped"], 0) + + def test_explicit_password_difficulty_outranks_the_resume(self) -> None: + server = resume_coordinator() + service = server._ensure_vardiff_service() + service.session_difficulty_store.record( + ("default", PAYOUT_ADDRESS), + Decimal("262144"), + now=time.monotonic(), + share_backed=True, + ) + + state = connect_client(server, 1) + authorize(server, state, password="d=32768") + + self.assertEqual(state.share_difficulty, Decimal("32768")) + self.assertIsNone(state.pending_share_difficulty) + + def test_md_only_password_clamps_the_resumed_value(self) -> None: + server = resume_coordinator() + service = server._ensure_vardiff_service() + service.session_difficulty_store.record( + ("default", PAYOUT_ADDRESS), + Decimal("262144"), + now=time.monotonic(), + share_backed=True, + ) + + state = connect_client(server, 1) + authorize(server, state, password="md=524288") + + # The md= floor applies to the just-resumed value, not the lane start. + self.assertEqual(state.share_difficulty, Decimal("524288")) + self.assertIsNone(state.pending_share_difficulty) + + def test_reauthorize_of_a_live_session_never_re_resumes(self) -> None: + server = resume_coordinator() + state = connect_client(server, 1) + authorize(server, state) + with server._client_vardiff_lock(state): + state.share_difficulty = Decimal("65536") + service = server._ensure_vardiff_service() + service.session_difficulty_store.record( + ("default", PAYOUT_ADDRESS), + Decimal("262144"), + now=time.monotonic(), + share_backed=True, + ) + + authorize(server, state) + + self.assertEqual(state.share_difficulty, Decimal("65536")) + self.assertEqual(resume_outcomes(server)["resumed"], 0) + + def test_silent_reconnect_cycles_never_extend_the_retention_ttl(self) -> None: + # The review repro: with the idle step-down sweep disabled (as + # deployed), a session resumed above its true hashrate produces no + # accepted shares and cannot step itself down. If each disconnect + # re-stamped the entry, cycles shorter than the TTL would keep that + # value adoptable forever and the short TTL would protect nothing. + server = resume_coordinator(vardiff_resume_ttl_seconds=900.0) + # The +900.0s cycle probes the TTL boundary exactly, where the store + # keeps the entry (expiry is strictly age > TTL). A real time. + # monotonic() reading makes that comparison depend on float + # rounding: (start + 900.0) - start exceeds 900.0 for roughly one in + # five thousand captured clock values, flipping the boundary cycle + # to "expired" and failing the test. Every record and lookup below + # runs under frozen_clock, so an exactly-representable constant + # makes the boundary arithmetic exact instead of lucky. + start = 1_000_000.0 + with frozen_clock(start): + first = connect_client(server, 1) + authorize(server, first) + converge_and_disconnect(server, first, Decimal("262144")) + + # Three reconnect cycles inside the TTL, each submitting nothing. + for index, offset in enumerate((300.0, 600.0, 900.0), start=2): + state = silent_reconnect_cycle(server, index, start + offset) + self.assertEqual( + state.share_difficulty, + Decimal("262144"), + f"cycle at +{offset}s should still resume", + ) + + # The entry ages from the original convergence at `start`, not from + # the last silent disconnect: it is gone one cycle later. + with frozen_clock(start + 1200.0): + last = connect_client(server, 5) + authorize(server, last) + + self.assertEqual(last.share_difficulty, STANDARD_LANE_START) + outcomes = resume_outcomes(server) + self.assertEqual(outcomes["resumed"], 3) + self.assertEqual(outcomes["expired"], 1) + self.assertIsNone(retained_difficulty(server, now=start + 1200.0)) + + def test_reconnect_with_an_accepted_share_refreshes_the_ttl(self) -> None: + # The counterpart: an accepted share re-validates the retained value, + # so its TTL restarts and the session keeps resuming. + server = resume_coordinator(vardiff_resume_ttl_seconds=900.0) + service = server._ensure_vardiff_service() + start = time.monotonic() + with frozen_clock(start): + first = connect_client(server, 1) + authorize(server, first) + converge_and_disconnect(server, first, Decimal("262144")) + + with frozen_clock(start + 800.0): + second = connect_client(server, 2) + authorize(server, second) + self.assertEqual(second.share_difficulty, Decimal("262144")) + # Open the vardiff window at the injected instant so the share + # below only marks the connection, leaving the retarget path + # (covered separately) out of this test. + with server._client_vardiff_lock(second): + second.vardiff_window_started_monotonic = start + 800.0 + service.note_accepted(second, Decimal("262144")) + self.assertTrue(second.vardiff_accepted_any) + self.assertIsNone(second.pending_share_difficulty) + server.disconnect_client(second) + + with frozen_clock(start + 1600.0): + third = connect_client(server, 3) + authorize(server, third) + + self.assertEqual(third.share_difficulty, Decimal("262144")) + outcomes = resume_outcomes(server) + self.assertEqual(outcomes["resumed"], 2) + self.assertEqual(outcomes["expired"], 0) + + def test_disconnect_retains_the_delivered_difficulty_not_a_pending_one( + self, + ) -> None: + # pending_share_difficulty is advertised with a FUTURE job, so a + # disconnect racing an in-flight retarget must not retain a value the + # miner was never given. A retarget that did land records itself at + # its own commit point. + server = resume_coordinator() + state = connect_client(server, 1) + authorize(server, state) + with server._client_vardiff_lock(state): + state.share_difficulty = Decimal("262144") + state.pending_share_difficulty = Decimal("1048576") + + server.disconnect_client(state) + + self.assertEqual(retained_difficulty(server), Decimal("262144")) + + def test_committed_retarget_retains_the_new_difficulty(self) -> None: + # The `if sent:` commit point inside retarget_locked: a delivered + # retarget retains its own value, with share evidence taken from the + # window that drove it. + server = resume_coordinator() + service = server._ensure_vardiff_service() + state = connect_client(server, 1) + authorize(server, state) + recorded: list[tuple[Decimal, bool]] = [] + store = service.session_difficulty_store + store_record = store.record + + def spy(key, difficulty, *, now, share_backed): # type: ignore[no-untyped-def] + recorded.append((difficulty, share_backed)) + store_record(key, difficulty, now=now, share_backed=share_backed) + + store.record = spy # type: ignore[method-assign] + + # A share-driven window at four times the target share rate. + server.retarget_client( + state, + current_difficulty=STANDARD_LANE_START, + accepted_shares=80, + submitted_shares=80, + accepted_difficulty=STANDARD_LANE_START * 80, + elapsed_seconds=Decimal("300"), + ) + + self.assertEqual(state.pending_share_difficulty, Decimal("65536")) + self.assertEqual(recorded, [(Decimal("65536"), True)]) + self.assertEqual(retained_difficulty(server), Decimal("65536")) + + # A zero-share window steps back down; it carries no share evidence, + # but it moves the value, so the store refreshes the TTL anyway. + server.retarget_client( + state, + current_difficulty=Decimal("65536"), + accepted_shares=0, + submitted_shares=0, + accepted_difficulty=Decimal("0"), + elapsed_seconds=Decimal("300"), + ) + + self.assertEqual(recorded[-1], (STANDARD_LANE_START, False)) + self.assertEqual(retained_difficulty(server), STANDARD_LANE_START) + + def test_explicit_password_difficulty_counts_an_overridden_resume(self) -> None: + server = resume_coordinator() + service = server._ensure_vardiff_service() + service.session_difficulty_store.record( + ("default", PAYOUT_ADDRESS), + Decimal("262144"), + now=time.monotonic(), + share_backed=True, + ) + + state = connect_client(server, 1) + authorize(server, state, password="d=32768") + + self.assertEqual(state.share_difficulty, Decimal("32768")) + outcomes = resume_outcomes(server) + # resumed stays an attempt counter; overridden subtracts the ones an + # explicit request replaced, so resumed + clamped - overridden is the + # number of resumes that actually stuck. + self.assertEqual(outcomes["resumed"], 1) + self.assertEqual(outcomes["overridden"], 1) + + def test_a_resume_that_sticks_is_not_counted_as_overridden(self) -> None: + server = resume_coordinator() + service = server._ensure_vardiff_service() + service.session_difficulty_store.record( + ("default", PAYOUT_ADDRESS), + Decimal("262144"), + now=time.monotonic(), + share_backed=True, + ) + + # No password request at all, then a request that resolves to exactly + # the resumed value: neither supersedes the resume. + first = connect_client(server, 1) + authorize(server, first) + second = connect_client(server, 2) + authorize(server, second, password="d=262144") + + self.assertEqual(first.share_difficulty, Decimal("262144")) + self.assertEqual(second.share_difficulty, Decimal("262144")) + outcomes = resume_outcomes(server) + self.assertEqual(outcomes["resumed"], 2) + self.assertEqual(outcomes["overridden"], 0) + + def test_failed_username_reservation_neither_resumes_nor_retains(self) -> None: + server = resume_coordinator() + service = server._ensure_vardiff_service() + service.session_difficulty_store.record( + ("default", PAYOUT_ADDRESS), + Decimal("262144"), + now=time.monotonic(), + share_backed=True, + ) + server.reserve_client_username = lambda client, worker: False + state = connect_client(server, 1) + + with self.assertRaises(StratumError): + authorize(server, state) + + self.assertFalse(state.authorized) + self.assertEqual(state.share_difficulty, STANDARD_LANE_START) + outcomes = resume_outcomes(server) + self.assertEqual(outcomes["resumed"], 0) + self.assertEqual(outcomes["clamped"], 0) + # No reserved worker means no retention identity, so the disconnect + # that follows the rejection writes nothing either. + server.disconnect_client(state) + self.assertEqual(service.session_difficulty_store.snapshot()["records"], 1) + self.assertEqual(retained_difficulty(server), Decimal("262144")) + + def test_kill_switch_restores_lane_start_reconnects(self) -> None: + server = resume_coordinator(vardiff_resume_enabled=False) + first = connect_client(server, 1) + authorize(server, first) + converge_and_disconnect(server, first, Decimal("262144")) + + second = connect_client(server, 2) + authorize(server, second) + + self.assertEqual(second.share_difficulty, STANDARD_LANE_START) + snapshot = server.vardiff_convergence_snapshot() + self.assertEqual(snapshot["retained_sessions"], 0) + outcomes = snapshot["resume_outcomes"] + assert isinstance(outcomes, dict) + self.assertEqual(outcomes["disabled"], 2) + self.assertEqual(outcomes["resumed"], 0) + + +class ResumeConfigTests(unittest.TestCase): + BASE_ENV = { + "QBIT_RPC_HOST": "127.0.0.1", + "QBIT_RPC_USER": "user", + "QBIT_RPC_PASSWORD": "password", + "PRISM_ALLOW_BUNDLE_EMBEDDED_LEDGER_KEY": "1", + "PRISM_ALLOW_TEST_SIGNING_SEEDS": "1", + } + + def test_defaults_and_overrides(self) -> None: + stratum = load_coordinator_config(self.BASE_ENV).stratum + self.assertTrue(stratum.vardiff_resume_enabled) + self.assertEqual(stratum.vardiff_resume_ttl_seconds, 900.0) + self.assertEqual(stratum.vardiff_resume_max_entries, 8192) + self.assertEqual(stratum.vardiff_resume_max_start_factor, Decimal("1024")) + + tuned = load_coordinator_config( + { + **self.BASE_ENV, + "PRISM_STRATUM_VARDIFF_RESUME": "0", + "PRISM_STRATUM_VARDIFF_RESUME_TTL_SECONDS": "120", + "PRISM_STRATUM_VARDIFF_RESUME_MAX_ENTRIES": "16", + "PRISM_STRATUM_VARDIFF_RESUME_MAX_START_FACTOR": "64", + } + ).stratum + self.assertFalse(tuned.vardiff_resume_enabled) + self.assertEqual(tuned.vardiff_resume_ttl_seconds, 120.0) + self.assertEqual(tuned.vardiff_resume_max_entries, 16) + self.assertEqual(tuned.vardiff_resume_max_start_factor, Decimal("64")) + + def test_env_sizes_the_coordinator_store_end_to_end(self) -> None: + # env -> StratumConfig -> PrismCoordinator attributes -> + # SessionDifficultyStore. The service reads those attributes through + # getattr defaults, so a rename anywhere in the chain would silently + # fall back to the defaults instead of failing; this asserts the + # configured values actually reach the store. + with tempfile.TemporaryDirectory() as temp_dir: + root = Path(temp_dir) + source = { + **self.BASE_ENV, + "PRISM_ALLOW_MEMORY_LEDGER": "1", + "PRISM_AUDIT_DIR": str(root), + "PRISM_EVIDENCE_PATH": str(root / "evidence.json"), + "PRISM_STRATUM_VARDIFF_RESUME_TTL_SECONDS": "123", + "PRISM_STRATUM_VARDIFF_RESUME_MAX_ENTRIES": "17", + } + config = load_coordinator_config(source) + with mock.patch.object( + JsonRpc, "call", side_effect=RuntimeError("offline") + ): + server = PrismCoordinator(config) + + self.assertEqual(server.vardiff_resume_ttl_seconds, 123.0) + self.assertEqual(server.vardiff_resume_max_entries, 17) + store = server._ensure_vardiff_service().session_difficulty_store + self.assertEqual(store.ttl_seconds, 123.0) + self.assertEqual(store.max_entries, 17) + self.assertTrue(store.enabled) + + def test_kill_switch_env_disables_the_coordinator_store(self) -> None: + with tempfile.TemporaryDirectory() as temp_dir: + root = Path(temp_dir) + source = { + **self.BASE_ENV, + "PRISM_ALLOW_MEMORY_LEDGER": "1", + "PRISM_AUDIT_DIR": str(root), + "PRISM_EVIDENCE_PATH": str(root / "evidence.json"), + "PRISM_STRATUM_VARDIFF_RESUME": "0", + } + config = load_coordinator_config(source) + with mock.patch.object( + JsonRpc, "call", side_effect=RuntimeError("offline") + ): + server = PrismCoordinator(config) + + self.assertFalse(server.vardiff_resume_enabled) + store = server._ensure_vardiff_service().session_difficulty_store + self.assertEqual(store.max_entries, 0) + self.assertFalse(store.enabled) + + def test_start_factor_below_one_fails_startup(self) -> None: + with self.assertRaises(SystemExit) as raised: + load_coordinator_config( + { + **self.BASE_ENV, + "PRISM_STRATUM_VARDIFF_RESUME_MAX_START_FACTOR": "0.5", + } + ) + self.assertEqual( + str(raised.exception), + "PRISM_STRATUM_VARDIFF_RESUME_MAX_START_FACTOR must be at least 1", + ) + + +class ServiceRuntime: + """Lightweight VardiffRuntime fake mirroring test_prism_vardiff_service.""" + + def __init__(self) -> None: + self.lock = threading.RLock() + # A list rather than a set: SimpleNamespace clients are unhashable, + # and convergence_snapshot only needs an iterable membership view. + self.clients: list[object] = [] + self.stop_event = threading.Event() + self.vardiff_config = standard_lane_config() + self.share_difficulty = STANDARD_LANE_START + self.vardiff_idle_sweep_seconds = 1.0 + + +def service_client(**overrides: object) -> SimpleNamespace: + values: dict[str, object] = dict( + vardiff_config=None, + listener_vardiff_config=None, + listener_name="default", + pending_share_difficulty=None, + share_difficulty=STANDARD_LANE_START, + vardiff_window_started_monotonic=time.monotonic(), + vardiff_window_accepted=0, + vardiff_window_submitted=0, + vardiff_window_work=Decimal("0"), + vardiff_difficulty_estimate=None, + ) + values.update(overrides) + return SimpleNamespace(**values) + + +class ConvergenceObservabilityTests(unittest.TestCase): + def test_snapshot_counts_lanes_ceiling_sessions_and_retention(self) -> None: + runtime = ServiceRuntime() + service = VardiffService(runtime) # type: ignore[arg-type] + at_ceiling = service_client( + share_difficulty=runtime.vardiff_config.max_difficulty, + listener_name="highdiff", + ) + converging = service_client() + runtime.clients.extend([at_ceiling, converging]) + + service.note_accepted(at_ceiling, Decimal("4")) # type: ignore[arg-type] + service.note_accepted(converging, Decimal("4")) # type: ignore[arg-type] + service.note_accepted(converging, Decimal("4")) # type: ignore[arg-type] + service.session_difficulty_store.record( + ("default", "miner-a"), + Decimal("262144"), + now=time.monotonic(), + share_backed=True, + ) + + snapshot = service.convergence_snapshot() + + self.assertEqual(snapshot["sessions_at_max_difficulty"], 1) + self.assertEqual( + snapshot["lane_accepted_shares"], {"highdiff": 1, "default": 2} + ) + self.assertEqual(snapshot["retained_sessions"], 1) + self.assertEqual( + snapshot["resume_outcomes"], + {outcome: 0 for outcome in PRISM_VARDIFF_RESUME_OUTCOMES}, + ) + + def test_lane_accepted_counts_where_vardiff_is_disabled(self) -> None: + runtime = ServiceRuntime() + runtime.vardiff_config = vardiff.VardiffConfig( + enabled=False, + target_share_interval_seconds=Decimal("15"), + min_difficulty=Decimal("1024"), + max_difficulty=Decimal("4294967296"), + retarget_interval_seconds=Decimal("300"), + max_step_factor=Decimal("4"), + startup_difficulty=STANDARD_LANE_START, + ) + service = VardiffService(runtime) # type: ignore[arg-type] + + service.note_accepted(service_client(), Decimal("4")) # type: ignore[arg-type] + + snapshot = service.convergence_snapshot() + self.assertEqual(snapshot["lane_accepted_shares"], {"default": 1}) + self.assertEqual(snapshot["sessions_at_max_difficulty"], 0) + + def test_renderer_emits_deterministic_lane_and_outcome_series(self) -> None: + started_monotonic = time.monotonic() + port = SimpleNamespace( + vardiff_convergence_snapshot=lambda: { + "sessions_at_max_difficulty": 2, + "lane_accepted_shares": {"default": 30, "extra": 5}, + "resume_outcomes": {"resumed": 4}, + "retained_sessions": 3, + }, + listener_profiles=[ + SimpleNamespace(name="default"), + SimpleNamespace(name="highdiff"), + ], + prometheus_label_value=lambda value: value, + started_monotonic=started_monotonic, + ) + renderer = MetricsRenderer(port) # type: ignore[arg-type] + + with mock.patch.object( + time, "monotonic", return_value=started_monotonic + 10.0 + ): + lines = renderer.vardiff_convergence_metrics_lines() + text = "\n".join(lines) + + self.assertIn("qbit_prism_vardiff_sessions_at_max_difficulty 2", text) + self.assertIn( + 'qbit_prism_vardiff_lane_accepted_shares_total{lane="default"} 30', + text, + ) + self.assertIn( + 'qbit_prism_vardiff_lane_accepted_shares_total{lane="highdiff"} 0', + text, + ) + self.assertIn( + 'qbit_prism_vardiff_lane_accepted_shares_total{lane="extra"} 5', + text, + ) + self.assertIn( + 'qbit_prism_vardiff_lane_accepted_shares_per_second{lane="default"} 3', + text, + ) + for outcome in PRISM_VARDIFF_RESUME_OUTCOMES: + self.assertIn( + f'qbit_prism_vardiff_resume_total{{outcome="{outcome}"}}', + text, + ) + self.assertIn('qbit_prism_vardiff_resume_total{outcome="resumed"} 4', text) + self.assertIn("qbit_prism_vardiff_resume_retained_sessions 3", text) + # Configured lanes lead in profile order; observed-only lanes follow. + default_index = text.index('shares_total{lane="default"}') + highdiff_index = text.index('shares_total{lane="highdiff"}') + extra_index = text.index('shares_total{lane="extra"}') + self.assertLess(default_index, highdiff_index) + self.assertLess(highdiff_index, extra_index) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_prism_vardiff_service.py b/tests/test_prism_vardiff_service.py new file mode 100644 index 000000000..38bc4688e --- /dev/null +++ b/tests/test_prism_vardiff_service.py @@ -0,0 +1,108 @@ +#!/usr/bin/env python3 +"""Direct tests for the PRISM vardiff owner.""" + +from __future__ import annotations + +from decimal import Decimal +from types import SimpleNamespace +import threading +import time +import unittest + +from lab.auxpow import vardiff +from lab.prism.vardiff_service import VardiffService + + +def config(*, interval: str = "1") -> vardiff.VardiffConfig: + return vardiff.VardiffConfig( + enabled=True, + target_share_interval_seconds=Decimal("15"), + min_difficulty=Decimal("1"), + max_difficulty=Decimal("1024"), + retarget_interval_seconds=Decimal(interval), + max_step_factor=Decimal("4"), + startup_difficulty=Decimal("4"), + max_step_down_factor=Decimal("4"), + ewma_alpha=Decimal("0.4"), + retarget_tolerance=Decimal("0.25"), + ) + + +class Runtime: + def __init__(self) -> None: + self.lock = threading.RLock() + self.clients: set[object] = set() + self.stop_event = threading.Event() + self.vardiff_config = config() + self.share_difficulty = Decimal("2") + self.vardiff_idle_sweep_seconds = 1.0 + self.retargets: list[dict[str, object]] = [] + + def retarget_client(self, client: object, **kwargs: object) -> bool: + self.retargets.append({"client": client, **kwargs}) + return True + + +def client() -> SimpleNamespace: + return SimpleNamespace( + vardiff_config=None, + listener_vardiff_config=None, + minimum_advertised_difficulty=Decimal("0"), + pending_share_difficulty=None, + share_difficulty=Decimal("4"), + vardiff_window_started_monotonic=time.monotonic() - 2, + vardiff_window_accepted=0, + vardiff_window_submitted=1, + vardiff_window_work=Decimal("0"), + vardiff_difficulty_estimate=None, + ) + + +class VardiffServiceTests(unittest.TestCase): + def test_accepted_window_is_captured_and_reset_before_retarget(self) -> None: + runtime = Runtime() + service = VardiffService(runtime) # type: ignore[arg-type] + state = client() + + service.note_accepted(state, Decimal("3")) # type: ignore[arg-type] + + self.assertEqual(len(runtime.retargets), 1) + retarget = runtime.retargets[0] + self.assertEqual(retarget["accepted_shares"], 1) + self.assertEqual(retarget["submitted_shares"], 1) + self.assertEqual(retarget["accepted_difficulty"], Decimal("3")) + self.assertEqual(state.vardiff_window_accepted, 0) + self.assertEqual(state.vardiff_window_submitted, 0) + self.assertEqual(state.vardiff_window_work, Decimal("0")) + + def test_speculative_idle_rollback_requires_unchanged_reset_stamp(self) -> None: + state = client() + original = (10.0, 0, 0, Decimal("0")) + state.vardiff_window_started_monotonic = 20.0 + state.vardiff_window_submitted = 0 + + VardiffService.restore_idle_window_state(state, original, 20.0) # type: ignore[arg-type] + self.assertEqual(state.vardiff_window_started_monotonic, 10.0) + + state.vardiff_window_started_monotonic = 30.0 + state.vardiff_window_submitted = 1 + VardiffService.restore_idle_window_state(state, original, 30.0) # type: ignore[arg-type] + self.assertEqual(state.vardiff_window_started_monotonic, 30.0) + self.assertEqual(state.vardiff_window_submitted, 1) + + def test_idle_metrics_are_service_owned(self) -> None: + service = VardiffService(Runtime()) # type: ignore[arg-type] + service.record_idle_skip("busy") + service.observe_idle_seconds("sweep", 0.01) + + metrics = "\n".join(service.metrics_lines()) + + self.assertIn( + 'qbit_prism_vardiff_idle_skips_total{reason="busy"} 1', + metrics, + ) + self.assertIn("qbit_prism_vardiff_idle_sweep_seconds_count 1", metrics) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_prism_window_pipeline_rust.py b/tests/test_prism_window_pipeline_rust.py new file mode 100644 index 000000000..a00366346 --- /dev/null +++ b/tests/test_prism_window_pipeline_rust.py @@ -0,0 +1,1472 @@ +#!/usr/bin/env python3 +"""Coordinator-side tests for the Rust payout-window pipeline slice. + +Why these tests exist: the byte-level equivalence of the Rust fold is proven +by the parity oracle (``tests.test_window_pipeline_parity`` with the +``rust-daemon`` adapter); what still needs proof is the coordinator's side of +the contract -- that the ``PRISM_WINDOW_PIPELINE_RUST`` switch defaults to +Python and fails closed, that the daemon path preserves the append-epoch and +scan-anchor fences, and that every non-success shape (daemon death, +``needs_full`` after a respawn, an ``advance`` invariant failure, a protocol +mismatch, a value outside the daemon's declared integer widths) degrades +exactly like its in-process counterpart instead of inventing a new failure +mode. The daemon here is the fake serve builder, which serves prepare_window +with the real Python fold so mirror digests verify byte-exactly without a +Rust build, and which declares the real daemon's integer widths. +""" +# ruff: noqa: F403, F405 + +from __future__ import annotations + +import dataclasses +import hashlib +import json +import os +import tempfile +import threading +import time +import unittest +from pathlib import Path + +from lab.prism import share_ledger as share_ledger_module +from lab.prism.coordinator_config import ( + env_window_pipeline_rust, + load_coordinator_config, +) +from lab.prism.share_ledger import ( + DaemonShareJsonSequence, + DaemonShareWindowMirror, + DaemonWindowMirrorDivergence, + IncrementalShareWindow, +) +from tests.prism_coordinator_test_support import * + + +def _canonical_items(window: IncrementalShareWindow) -> bytes: + return b",".join( + page.canonical_json_items + for page in window.pages + if page.canonical_json_items + ) + + +def _python_window_for( + ledger: IncrementalRecordingLedger, + *, + anchor_ms: int, + network_difficulty: int, +) -> IncrementalShareWindow: + """The in-process oracle over the same ledger rows, counters untouched.""" + records = list( + SingleWriterShareLedger.snapshot_at_job_issue( + ledger, + anchor_ms, + window_weight=None, + ) + ) + return IncrementalShareWindow.from_full_snapshot( + records, + anchor_job_issued_at_ms=anchor_ms, + window_weight=8 * 2 * int(network_difficulty), + ) + + +class WindowPipelineSwitchTests(unittest.TestCase): + """PRISM_WINDOW_PIPELINE_RUST: default Python, strict values, inert off.""" + + def test_switch_defaults_to_python(self) -> None: + self.assertFalse(env_window_pipeline_rust(environ={})) + + def test_switch_accepts_boolean_tokens(self) -> None: + for raw in ("1", "true", "YES", "On"): + with self.subTest(raw=raw): + self.assertTrue( + env_window_pipeline_rust( + environ={"PRISM_WINDOW_PIPELINE_RUST": raw} + ) + ) + for raw in ("0", "false", "NO", "Off"): + with self.subTest(raw=raw): + self.assertFalse( + env_window_pipeline_rust( + environ={"PRISM_WINDOW_PIPELINE_RUST": raw} + ) + ) + + def test_switch_fails_closed_on_non_boolean_values(self) -> None: + for raw in ("2", "banana", "tru"): + with self.subTest(raw=raw): + with self.assertRaisesRegex( + SystemExit, "PRISM_WINDOW_PIPELINE_RUST must be a boolean" + ): + env_window_pipeline_rust( + environ={"PRISM_WINDOW_PIPELINE_RUST": raw} + ) + + def test_coordinator_config_load_validates_the_switch_at_startup(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + base = { + "QBIT_RPC_HOST": "qbit.example", + "QBIT_RPC_USER": "rpc-user", + "QBIT_RPC_PASSWORD": "rpc-password", + "PRISM_ALLOW_MEMORY_LEDGER": "1", + "PRISM_ALLOW_TEST_SIGNING_SEEDS": "1", + "PRISM_ALLOW_BUNDLE_EMBEDDED_LEDGER_KEY": "1", + "PRISM_AUDIT_DIR": str(root), + "PRISM_EVIDENCE_PATH": str(root / "evidence.json"), + } + config = load_coordinator_config(base) + self.assertFalse(config.jobs.window_pipeline_rust_enabled) + enabled = load_coordinator_config( + {**base, "PRISM_WINDOW_PIPELINE_RUST": "1"} + ) + self.assertTrue(enabled.jobs.window_pipeline_rust_enabled) + with self.assertRaisesRegex( + SystemExit, "PRISM_WINDOW_PIPELINE_RUST must be a boolean" + ): + load_coordinator_config( + {**base, "PRISM_WINDOW_PIPELINE_RUST": "maybe"} + ) + + +class DaemonShareWindowMirrorTests(unittest.TestCase): + """The digest-verified byte mirror and its lazy JSON sequence.""" + + def _window(self) -> IncrementalShareWindow: + records = [ + stamped_pending_share(999_900 + seq) for seq in range(3) + ] + ledger = SingleWriterShareLedger() + appended = [ledger.append(record) for record in records] + return IncrementalShareWindow.from_full_snapshot( + appended, + anchor_job_issued_at_ms=1_000_000, + window_weight=1_000, + ) + + def test_full_mirror_verifies_and_parses_lazily(self) -> None: + window = self._window() + items = _canonical_items(window) + digest = window.json_records().canonical_json_sha256() + mirror = DaemonShareWindowMirror.from_full_items( + anchor_job_issued_at_ms=1_000_000, + window_weight=1_000, + page_size=512, + record_count=window.record_count, + canonical_items=items, + share_snapshot_sha256=digest, + ) + sequence = mirror.json_records() + self.assertIsInstance(sequence, DaemonShareJsonSequence) + self.assertEqual(len(sequence), window.record_count) + self.assertIsNone(sequence._parsed) + self.assertEqual(sequence.canonical_json_sha256(), digest) + # The digest never forces a parse; iteration does, exactly once. + self.assertIsNone(sequence._parsed) + self.assertEqual(list(sequence), list(window.json_records())) + self.assertEqual(sequence[0], window.json_records()[0]) + + def _awkward_window(self, count: int = 40) -> IncrementalShareWindow: + """Records whose canonical bytes carry every escape the parser meets.""" + ledger = SingleWriterShareLedger() + appended = [] + for seq in range(count): + # Stamped well inside the anchor so every record is eligible + # whatever the count. + pending = stamped_pending_share(900_000 + seq) + pending = dataclasses.replace( + pending, + share_id=f"miner-\u00e9.rig{seq}:\U0001F600,}}{{\"\\", + miner_id=f"miner-{seq % 3}-\u2603", + share_difficulty=(1 << 130) + seq, + credit_policy='stale-grace' if seq % 2 else None, + ) + appended.append(ledger.append(pending)) + # A weight past the whole snapshot's total, so nothing expires. + return IncrementalShareWindow.from_full_snapshot( + appended, + anchor_job_issued_at_ms=1_000_000, + window_weight=(1 << 131) * count, + ) + + def test_lazy_parse_walks_records_across_decode_chunks(self) -> None: + window = self._awkward_window() + items = _canonical_items(window) + expected = json.loads(b"[" + items + b"]") + self.assertEqual(expected, list(window.json_records())) + # Chunk sizes that cut records, escapes and separators everywhere; + # the last is the shipped default. + for chunk_bytes in (1, 2, 3, 5, 64, 1_000, None): + with self.subTest(chunk_bytes=chunk_bytes), patch.object( + share_ledger_module, + "CANONICAL_ITEMS_DECODE_CHUNK_BYTES", + chunk_bytes or share_ledger_module.CANONICAL_ITEMS_DECODE_CHUNK_BYTES, + ): + sequence = DaemonShareJsonSequence(items, len(expected)) + self.assertIsNone(sequence._parsed) + self.assertEqual(list(sequence), expected) + self.assertEqual(sequence[0], expected[0]) + self.assertEqual(sequence[-1], expected[-1]) + self.assertEqual(sequence[3:7], tuple(expected[3:7])) + self.assertEqual(len(sequence), len(expected)) + # The parse ran exactly once and was published whole. + self.assertIs(sequence._records(), sequence._parsed) + # Keys are shared across records like a whole-array + # json.loads shares them: one string object per key. + key_ids = {id(key) for record in sequence for key in record} + self.assertLessEqual(len(key_ids), 13) + self.assertEqual( + share_ledger_module._canonical_items_layout(items), + (len(expected), False), + ) + self.assertEqual( + share_ledger_module._canonical_items_layout(items + b","), + (len(expected), True), + ) + + def test_lazy_parse_decodes_raw_utf8_split_across_chunks(self) -> None: + # Not canonical (the daemon escapes non-ASCII), but valid JSON: the + # incremental decoder must never split a multi-byte sequence and + # the digest check, not the parser, is what pins canonicality. + items = b'{"k":"\xc3\xa9\xf0\x9f\x98\x80"},{"k":"\xe2\x98\x83"}' + expected = json.loads(b"[" + items + b"]") + for chunk_bytes in (1, 2, 3, 4, 5, 7, 100): + with self.subTest(chunk_bytes=chunk_bytes), patch.object( + share_ledger_module, + "CANONICAL_ITEMS_DECODE_CHUNK_BYTES", + chunk_bytes, + ): + self.assertEqual( + list(DaemonShareJsonSequence(items, 2)), + expected, + ) + + def test_lazy_parse_rejects_malformed_streams_without_publishing(self) -> None: + window = self._awkward_window(6) + items = _canonical_items(window) + count = window.record_count + broken = { + "truncated record": items[:-1], + "truncated escape": items[: items.index(b"\\ud83d") + 3], + "trailing separator": items + b",", + "leading separator": b"," + items, + "doubled separator": items.replace(b"},{", b"},,{", 1), + "missing separator": items.replace(b"},{", b"}{", 1), + "junk between records": items.replace(b"},{", b"}x{", 1), + "whitespace between records": items.replace(b"},{", b"}, {", 1), + "trailing whitespace": items + b" ", + "non-object item": items.replace(b"},{", b"},1,{", 1), + "invalid utf-8 mid-stream": items.replace(b"},{", b"},\xff{", 1), + "invalid utf-8 in a string": items.replace(b"miner", b"mi\xc3ner", 1), + "truncated multi-byte tail": items + b",{\"k\":\"\xc3", + } + for label, stream in broken.items(): + for chunk_bytes in (1, 4, 64 * 1024): + with self.subTest(label=label, chunk_bytes=chunk_bytes), patch.object( + share_ledger_module, + "CANONICAL_ITEMS_DECODE_CHUNK_BYTES", + chunk_bytes, + ): + with self.assertRaises(DaemonWindowMirrorDivergence): + share_ledger_module._canonical_items_record_count(stream) + sequence = DaemonShareJsonSequence(stream, count) + with self.assertRaises(DaemonWindowMirrorDivergence): + list(sequence) + # Nothing partial is ever published. + self.assertIsNone(sequence._parsed) + # A count the bytes refute is refused after the whole walk, and + # the parse is not memoized either. + miscounted = DaemonShareJsonSequence(items, count + 1) + with self.assertRaisesRegex( + DaemonWindowMirrorDivergence, + f"parsed {count} records where {count + 1}", + ): + miscounted[0] + self.assertIsNone(miscounted._parsed) + self.assertEqual(list(DaemonShareJsonSequence(b"", 0)), []) + + def test_large_mirror_is_counted_and_parsed_chunk_by_chunk(self) -> None: + # Past the decode chunk (64 KiB) several times over, so record + # boundaries land inside chunk boundaries throughout the stream. + window = self._awkward_window(3_000) + items = _canonical_items(window) + self.assertGreater( + len(items), + 4 * share_ledger_module.CANONICAL_ITEMS_DECODE_CHUNK_BYTES, + ) + self.assertEqual(window.record_count, 3_000) + mirror = DaemonShareWindowMirror.from_full_items( + anchor_job_issued_at_ms=1_000_000, + window_weight=(1 << 131) * 3_000, + page_size=512, + record_count=3_000, + canonical_items=items, + share_snapshot_sha256=window.json_records().canonical_json_sha256(), + ) + self.assertEqual( + list(mirror.json_records()), + list(window.json_records()), + ) + + def test_full_mirror_rejects_bytes_that_do_not_hash_to_the_digest(self) -> None: + window = self._window() + digest = window.json_records().canonical_json_sha256() + with self.assertRaises(DaemonWindowMirrorDivergence): + DaemonShareWindowMirror.from_full_items( + anchor_job_issued_at_ms=1_000_000, + window_weight=1_000, + page_size=512, + record_count=window.record_count, + canonical_items=_canonical_items(window) + b" ", + share_snapshot_sha256=digest, + ) + + def test_full_mirror_rejects_a_count_the_bytes_refute(self) -> None: + window = self._window() + items = _canonical_items(window) + digest = window.json_records().canonical_json_sha256() + for declared in (window.record_count - 1, window.record_count + 1): + with self.subTest(declared=declared): + # Eager, beside the digest check: the bytes are right, only + # the count beside them is wrong, and that must not be left + # for whichever consumer first parses the stream. + with self.assertRaisesRegex( + DaemonWindowMirrorDivergence, + f"holds {window.record_count} records where {declared}", + ): + DaemonShareWindowMirror.from_full_items( + anchor_job_issued_at_ms=1_000_000, + window_weight=1_000, + page_size=512, + record_count=declared, + canonical_items=items, + share_snapshot_sha256=digest, + ) + + def test_full_mirror_rejects_a_stream_that_is_not_whole_records( + self, + ) -> None: + window = self._window() + items = _canonical_items(window) + for broken in (items + b",", b"," + items, items[:-1]): + with self.subTest(broken=broken[:8]): + digest = hashlib.sha256(b"[" + broken + b"]").hexdigest() + with self.assertRaises(DaemonWindowMirrorDivergence): + DaemonShareWindowMirror.from_full_items( + anchor_job_issued_at_ms=1_000_000, + window_weight=1_000, + page_size=512, + record_count=window.record_count, + canonical_items=broken, + share_snapshot_sha256=digest, + ) + + def test_advanced_applies_drop_and_append_surgery_exactly(self) -> None: + window = self._window() + mirror = DaemonShareWindowMirror.from_full_items( + anchor_job_issued_at_ms=1_000_000, + window_weight=1_000, + page_size=512, + record_count=window.record_count, + canonical_items=_canonical_items(window), + share_snapshot_sha256=window.json_records().canonical_json_sha256(), + ) + ledger = SingleWriterShareLedger(first_share_seq=10) + delta = [ledger.append(stamped_pending_share(1_000_010))] + advanced_window, _stats = window.advance( + delta, + anchor_job_issued_at_ms=1_000_020, + ) + old_items = _canonical_items(window) + new_items = _canonical_items(advanced_window) + self.assertTrue(new_items.startswith(old_items)) + appended = new_items[len(old_items):] + advanced = mirror.advanced( + anchor_job_issued_at_ms=1_000_020, + record_count=advanced_window.record_count, + retained_drop_bytes=0, + appended_items=appended, + share_snapshot_sha256=( + advanced_window.json_records().canonical_json_sha256() + ), + ) + self.assertEqual(advanced.canonical_items, new_items) + self.assertEqual(advanced.anchor_job_issued_at_ms, 1_000_020) + # The advance's count is counted from the surgery, not accepted. + with self.assertRaisesRegex( + DaemonWindowMirrorDivergence, + "records where", + ): + mirror.advanced( + anchor_job_issued_at_ms=1_000_020, + record_count=advanced_window.record_count + 1, + retained_drop_bytes=0, + appended_items=appended, + share_snapshot_sha256=( + advanced_window.json_records().canonical_json_sha256() + ), + ) + with self.assertRaises(DaemonWindowMirrorDivergence): + mirror.advanced( + anchor_job_issued_at_ms=1_000_020, + record_count=advanced_window.record_count, + retained_drop_bytes=1, + appended_items=appended, + share_snapshot_sha256=( + advanced_window.json_records().canonical_json_sha256() + ), + ) + + def test_anchor_only_advance_reuses_bytes_and_checks_the_digest(self) -> None: + window = self._window() + digest = window.json_records().canonical_json_sha256() + mirror = DaemonShareWindowMirror.from_full_items( + anchor_job_issued_at_ms=1_000_000, + window_weight=1_000, + page_size=512, + record_count=window.record_count, + canonical_items=_canonical_items(window), + share_snapshot_sha256=digest, + ) + advanced = mirror.advanced( + anchor_job_issued_at_ms=1_000_050, + record_count=window.record_count, + retained_drop_bytes=0, + appended_items=b"", + share_snapshot_sha256=digest, + ) + self.assertIs(advanced.canonical_items, mirror.canonical_items) + with self.assertRaisesRegex( + DaemonWindowMirrorDivergence, + "records where", + ): + mirror.advanced( + anchor_job_issued_at_ms=1_000_050, + record_count=window.record_count + 1, + retained_drop_bytes=0, + appended_items=b"", + share_snapshot_sha256=digest, + ) + with self.assertRaises(DaemonWindowMirrorDivergence): + mirror.advanced( + anchor_job_issued_at_ms=1_000_050, + record_count=window.record_count, + retained_drop_bytes=0, + appended_items=b"", + share_snapshot_sha256="0" * 64, + ) + + +class DaemonWindowPipelineTestCase(unittest.TestCase): + """Shared coordinator/fake-daemon fixture for the window-pipeline tests.""" + + def _server(self) -> tuple[PrismCoordinator, IncrementalRecordingLedger, object]: + ledger = IncrementalRecordingLedger() + append_incremental_share(ledger, share_seq=1, accepted_at_ms=999_900) + append_incremental_share(ledger, share_seq=2, accepted_at_ms=999_910) + append_incremental_share(ledger, share_seq=3, accepted_at_ms=999_920) + server, rpc = coordinator(ledger=ledger) + server.signing_seed_hex = "42" * 32 + server.ledger_attestation_signing_seed_hex = "43" * 32 + artifacts = server.store_template_artifacts(dict(rpc.template)) + assert artifacts is not None + server._pool_ready_latched = True + server.payout_artifact_min_build_interval_seconds = 60.0 + server.payout_artifact_full_rescan_seconds = 3_600.0 + self.addCleanup(server.shutdown_serve_builder) + return server, ledger, artifacts + + def _rust_env(self, mode: str = "ok", **extra: str): + values = { + "PRISM_WINDOW_PIPELINE_RUST": "1", + "FAKE_SERVE_BUILDER_MODE": mode, + **extra, + } + return patch.dict(os.environ, values) + + def _fake_daemon_command(self): + return patch( + "lab.prism.prism_coordinator.prism_tool_command", + return_value=list(FAKE_SERVE_BUILDER_COMMAND), + ) + + def _serve_counts(self, server: PrismCoordinator) -> dict[str, int]: + with server._serve_builder_metrics_lock: + return dict(server.serve_builder_counts) + + +class WindowPipelineRustMaterializationTests(DaemonWindowPipelineTestCase): + """The daemon-backed materialization path against the fake serve builder.""" + + def test_full_then_advance_materializes_daemon_windows_byte_exactly( + self, + ) -> None: + server, ledger, artifacts = self._server() + clock_ms = [1_000_000] + with self._rust_env(), self._fake_daemon_command(), patch( + "lab.prism.prism_coordinator.now_ms", + side_effect=lambda: clock_ms[0], + ): + initial = server._build_payout_ledger_artifact( + 0, 0, artifacts.network_difficulty + ) + assert initial is not None + self.assertEqual(initial.window_build_mode, "full_rescan") + cached = server._incremental_payout_artifact_window + assert cached is not None + self.assertIsInstance(cached.window, DaemonShareWindowMirror) + self.assertIsInstance(initial.shares_json, DaemonShareJsonSequence) + # The routine path must not have parsed the mirror's dicts. + self.assertIsNone(initial.shares_json._parsed) + oracle = _python_window_for( + ledger, + anchor_ms=int(initial.snapshot_anchor_ms or 0), + network_difficulty=int(artifacts.network_difficulty), + ) + self.assertEqual( + initial.share_snapshot_sha256, + oracle.json_records().canonical_json_sha256(), + ) + self.assertEqual(ledger.full_snapshot_calls, 1) + + append_incremental_share(ledger, share_seq=4, accepted_at_ms=1_000_010) + server.payout_artifact_min_build_interval_seconds = 0.0 + clock_ms[0] = 1_000_020 + advanced = server._build_payout_ledger_artifact( + 0, 0, artifacts.network_difficulty + ) + assert advanced is not None + self.assertEqual(advanced.window_build_mode, "incremental") + self.assertEqual(advanced.window_delta_rows, 1) + self.assertEqual(ledger.full_snapshot_calls, 1) + self.assertEqual(ledger.delta_snapshot_calls, 1) + advanced_oracle = _python_window_for( + ledger, + anchor_ms=int(advanced.snapshot_anchor_ms or 0), + network_difficulty=int(artifacts.network_difficulty), + ) + self.assertEqual( + advanced.share_snapshot_sha256, + advanced_oracle.json_records().canonical_json_sha256(), + ) + # Forcing the rare-path parse must reproduce the oracle's dicts. + self.assertEqual( + list(advanced.shares_json), + list(advanced_oracle.json_records()), + ) + counts = self._serve_counts(server) + self.assertEqual(counts["window_prepares"], 2) + self.assertEqual(counts["spawns"], 1) + self.assertEqual(counts["fallbacks"], 0) + + def test_prepared_window_serves_builds_without_upload_or_parse(self) -> None: + server, ledger, artifacts = self._server() + clock_ms = [1_000_000] + with self._rust_env(), self._fake_daemon_command(), patch( + "lab.prism.prism_coordinator.now_ms", + side_effect=lambda: clock_ms[0], + ): + artifact = server._build_payout_ledger_artifact( + 0, 0, artifacts.network_difficulty + ) + assert artifact is not None + serialization = server._share_window_serialization_for_artifact( + artifact, + artifact.shares_json, + ) + summary = server.build_audit_bundle( + shares=artifact.shares_json, + found_block={ + "block_height": 101, + "coinbase_value_sats": 50_00000000, + "network_difficulty": int(artifacts.network_difficulty), + "anchor_job_issued_at_ms": int( + artifact.snapshot_anchor_ms or 0 + ), + }, + prior_balances=[], + coinbase_script_sig_suffix_hex="00", + summary_only=True, + payout_policy={"policy": "day-one"}, + share_serialization=serialization, # type: ignore[arg-type] + append_invalidation_epoch=artifact.append_invalidation_epoch, + ) + self.assertEqual(summary["transport"], "serve") + # The prepared window is the build cache entry: no upload rode + # along, and the routine build never materialized the dicts. + self.assertFalse(summary["request_had_window"]) + self.assertIsNone(artifact.shares_json._parsed) + counts = self._serve_counts(server) + self.assertEqual(counts["window_uploads"], 0) + self.assertEqual(counts["requests"], 1) + + def test_epoch_bump_between_daemon_call_and_install_discards_artifact( + self, + ) -> None: + server, _ledger, artifacts = self._server() + clock_ms = [1_000_000] + with self._rust_env(), self._fake_daemon_command(), patch( + "lab.prism.prism_coordinator.now_ms", + side_effect=lambda: clock_ms[0], + ): + artifact = server._build_payout_ledger_artifact( + 0, 0, artifacts.network_difficulty + ) + assert artifact is not None + with server._job_cache_lock: + server._payout_ledger_append_invalidation_epoch += 1 + self.assertFalse(server._install_payout_ledger_artifact(artifact)) + with server._job_cache_lock: + self.assertIsNone(server._payout_ledger_artifact) + + def test_inflight_scan_anchor_spans_the_daemon_round_trip(self) -> None: + server, _ledger, artifacts = self._server() + observed_anchors: list[dict[int, int]] = [] + real_prepare = server.prepare_payout_window + + def observing_prepare(**kwargs: object) -> object: + with server._job_cache_lock: + observed_anchors.append( + dict(server._payout_window_inflight_scan_anchors) + ) + return real_prepare(**kwargs) + + clock_ms = [1_000_000] + with self._rust_env(), self._fake_daemon_command(), patch( + "lab.prism.prism_coordinator.now_ms", + side_effect=lambda: clock_ms[0], + ), patch.object(server, "prepare_payout_window", observing_prepare): + artifact = server._build_payout_ledger_artifact( + 0, 0, artifacts.network_difficulty + ) + assert artifact is not None + self.assertEqual(len(observed_anchors), 1) + self.assertIn( + int(artifact.snapshot_anchor_ms or 0), + observed_anchors[0].values(), + ) + with server._job_cache_lock: + self.assertEqual(server._payout_window_inflight_scan_anchors, {}) + + def test_daemon_death_mid_prepare_degrades_to_python_same_build(self) -> None: + server, ledger, artifacts = self._server() + clock_ms = [1_000_000] + with self._rust_env(mode="crash-during-prepare"), ( + self._fake_daemon_command() + ), patch( + "lab.prism.prism_coordinator.now_ms", + side_effect=lambda: clock_ms[0], + ): + artifact = server._build_payout_ledger_artifact( + 0, 0, artifacts.network_difficulty + ) + assert artifact is not None + self.assertEqual(artifact.window_build_mode, "full_rescan") + cached = server._incremental_payout_artifact_window + assert cached is not None + self.assertIsInstance(cached.window, IncrementalShareWindow) + oracle = _python_window_for( + ledger, + anchor_ms=int(artifact.snapshot_anchor_ms or 0), + network_difficulty=int(artifacts.network_difficulty), + ) + self.assertEqual( + artifact.share_snapshot_sha256, + oracle.json_records().canonical_json_sha256(), + ) + self.assertTrue(server._install_payout_ledger_artifact(artifact)) + counts = self._serve_counts(server) + self.assertEqual(counts["fallbacks"], 1) + self.assertEqual(counts["window_prepares"], 0) + + def test_needs_full_after_respawn_reprepares_identical_digest(self) -> None: + server, ledger, artifacts = self._server() + clock_ms = [1_000_000] + with self._rust_env(), self._fake_daemon_command(), patch( + "lab.prism.prism_coordinator.now_ms", + side_effect=lambda: clock_ms[0], + ): + initial = server._build_payout_ledger_artifact( + 0, 0, artifacts.network_difficulty + ) + assert initial is not None + # The daemon dies between generations; its prepared state dies + # with it, while the coordinator's mirror stays digest-verified. + client = server._serve_builder + assert client is not None + client.process.kill() + client.process.wait(timeout=5.0) + + append_incremental_share(ledger, share_seq=4, accepted_at_ms=1_000_010) + server.payout_artifact_min_build_interval_seconds = 0.0 + clock_ms[0] = 1_000_020 + rebuilt = server._build_payout_ledger_artifact( + 0, 0, artifacts.network_difficulty + ) + assert rebuilt is not None + self.assertEqual(rebuilt.window_build_mode, "full_rescan") + self.assertEqual( + rebuilt.window_full_rescan_reason, + "window_daemon_state_lost", + ) + oracle = _python_window_for( + ledger, + anchor_ms=int(rebuilt.snapshot_anchor_ms or 0), + network_difficulty=int(artifacts.network_difficulty), + ) + self.assertEqual( + rebuilt.share_snapshot_sha256, + oracle.json_records().canonical_json_sha256(), + ) + cached = server._incremental_payout_artifact_window + assert cached is not None + self.assertIsInstance(cached.window, DaemonShareWindowMirror) + counts = self._serve_counts(server) + self.assertEqual(counts["spawns"], 2) + self.assertEqual(ledger.full_snapshot_calls, 2) + + def test_advance_invariant_failure_full_rescans_without_retiring_daemon( + self, + ) -> None: + server, ledger, artifacts = self._server() + clock_ms = [1_000_000] + with self._rust_env(mode="prepare-fallback"), ( + self._fake_daemon_command() + ), patch( + "lab.prism.prism_coordinator.now_ms", + side_effect=lambda: clock_ms[0], + ): + initial = server._build_payout_ledger_artifact( + 0, 0, artifacts.network_difficulty + ) + assert initial is not None + append_incremental_share(ledger, share_seq=4, accepted_at_ms=1_000_010) + server.payout_artifact_min_build_interval_seconds = 0.0 + clock_ms[0] = 1_000_020 + rebuilt = server._build_payout_ledger_artifact( + 0, 0, artifacts.network_difficulty + ) + assert rebuilt is not None + self.assertEqual(rebuilt.window_build_mode, "full_rescan") + self.assertEqual( + rebuilt.window_full_rescan_reason, + "incremental_invariant_failed", + ) + client = server._serve_builder + assert client is not None + self.assertIsNone(client.process.poll()) + counts = self._serve_counts(server) + self.assertEqual(counts["fallbacks"], 0) + self.assertEqual(counts["spawns"], 1) + self.assertEqual(ledger.full_snapshot_calls, 2) + + def test_protocol_mismatch_retires_daemon_and_degrades_to_python(self) -> None: + server, _ledger, artifacts = self._server() + clock_ms = [1_000_000] + with self._rust_env(FAKE_SERVE_BUILDER_PROTOCOL="1"), ( + self._fake_daemon_command() + ), patch( + "lab.prism.prism_coordinator.now_ms", + side_effect=lambda: clock_ms[0], + ): + artifact = server._build_payout_ledger_artifact( + 0, 0, artifacts.network_difficulty + ) + assert artifact is not None + cached = server._incremental_payout_artifact_window + assert cached is not None + self.assertIsInstance(cached.window, IncrementalShareWindow) + self.assertIsNone(server._serve_builder) + counts = self._serve_counts(server) + self.assertEqual(counts["fallbacks"], 1) + self.assertEqual(counts["window_prepares"], 0) + + def test_switch_defaults_to_python_pipeline(self) -> None: + server, _ledger, artifacts = self._server() + clock_ms = [1_000_000] + with patch.dict(os.environ) as _env, self._fake_daemon_command(), patch( + "lab.prism.prism_coordinator.now_ms", + side_effect=lambda: clock_ms[0], + ): + os.environ.pop("PRISM_WINDOW_PIPELINE_RUST", None) + artifact = server._build_payout_ledger_artifact( + 0, 0, artifacts.network_difficulty + ) + assert artifact is not None + cached = server._incremental_payout_artifact_window + assert cached is not None + self.assertIsInstance(cached.window, IncrementalShareWindow) + counts = self._serve_counts(server) + self.assertEqual(counts["spawns"], 0) + self.assertEqual(counts["window_prepares"], 0) + + def test_switch_is_inert_without_the_daemon_transport(self) -> None: + server, _ledger, artifacts = self._server() + clock_ms = [1_000_000] + with self._rust_env(PRISM_BUILDER_SERVE="0"), ( + self._fake_daemon_command() + ), patch( + "lab.prism.prism_coordinator.now_ms", + side_effect=lambda: clock_ms[0], + ): + artifact = server._build_payout_ledger_artifact( + 0, 0, artifacts.network_difficulty + ) + assert artifact is not None + cached = server._incremental_payout_artifact_window + assert cached is not None + self.assertIsInstance(cached.window, IncrementalShareWindow) + counts = self._serve_counts(server) + self.assertEqual(counts["spawns"], 0) + self.assertEqual(counts["window_prepares"], 0) + + def test_flipping_the_switch_off_rebuilds_from_the_oracle(self) -> None: + server, ledger, artifacts = self._server() + clock_ms = [1_000_000] + with self._fake_daemon_command(), patch( + "lab.prism.prism_coordinator.now_ms", + side_effect=lambda: clock_ms[0], + ): + with self._rust_env(): + initial = server._build_payout_ledger_artifact( + 0, 0, artifacts.network_difficulty + ) + assert initial is not None + cached = server._incremental_payout_artifact_window + assert cached is not None + self.assertIsInstance(cached.window, DaemonShareWindowMirror) + server.payout_artifact_min_build_interval_seconds = 0.0 + clock_ms[0] = 1_000_020 + with patch.dict( + os.environ, {"PRISM_WINDOW_PIPELINE_RUST": "0"} + ): + rebuilt = server._build_payout_ledger_artifact( + 0, 0, artifacts.network_difficulty + ) + assert rebuilt is not None + self.assertEqual(rebuilt.window_build_mode, "full_rescan") + self.assertEqual( + rebuilt.window_full_rescan_reason, + "window_pipeline_mode_changed", + ) + cached = server._incremental_payout_artifact_window + assert cached is not None + self.assertIsInstance(cached.window, IncrementalShareWindow) + self.assertEqual(ledger.full_snapshot_calls, 2) + + +class DaemonWindowContentionTests(DaemonWindowPipelineTestCase): + """A daemon another build owns is degraded around, never retired. + + The prepare path used to stamp its deadline before an untimed blocking + acquire, so a preparation that queued behind a slow build arrived inside + the lock with its whole budget already spent: the first write read as a + daemon timeout, and the anomaly handler SIGKILLed a daemon that had done + nothing wrong. Waiting out a lock is not a daemon fault. + """ + + def _lock_holder(self, server: PrismCoordinator, hold_seconds: float): + """Own the serve lock from another thread for a bounded window.""" + taken = threading.Event() + release = threading.Event() + + def hold() -> None: + server._serve_builder_lock.acquire() + try: + taken.set() + release.wait(hold_seconds) + finally: + server._serve_builder_lock.release() + + thread = threading.Thread(target=hold, daemon=True) + thread.start() + self.assertTrue(taken.wait(5.0)) + self.addCleanup(thread.join, 10.0) + self.addCleanup(release.set) + return release + + def _worker_counts(self, server: PrismCoordinator) -> dict[str, int]: + with server._job_build_scheduler_lock: + return dict(server.job_build_worker_counts) + + def test_contended_daemon_degrades_without_being_retired(self) -> None: + server, ledger, artifacts = self._server() + clock_ms = [1_000_000] + with self._rust_env(), self._fake_daemon_command(), patch( + "lab.prism.prism_coordinator.now_ms", + side_effect=lambda: clock_ms[0], + ): + initial = server._build_payout_ledger_artifact( + 0, 0, artifacts.network_difficulty + ) + assert initial is not None + self.assertIsInstance( + server._incremental_payout_artifact_window.window, + DaemonShareWindowMirror, + ) + daemon = server._serve_builder + assert daemon is not None + + append_incremental_share(ledger, share_seq=4, accepted_at_ms=1_000_010) + server.payout_artifact_min_build_interval_seconds = 0.0 + server.bundle_build_timeout_seconds = 0.2 + clock_ms[0] = 1_000_020 + # Held past this preparation's whole budget: the pre-fix code + # blocked here, then read its own exhausted deadline as a sick + # daemon and killed it. + self._lock_holder(server, 3.0) + rebuilt = server._build_payout_ledger_artifact( + 0, 0, artifacts.network_difficulty + ) + assert rebuilt is not None + self.assertEqual(rebuilt.window_build_mode, "full_rescan") + self.assertEqual( + rebuilt.window_full_rescan_reason, + "window_daemon_busy", + ) + # Degraded to the in-process fold for this materialization, and + # still byte-exact against the oracle. + cached = server._incremental_payout_artifact_window + assert cached is not None + self.assertIsInstance(cached.window, IncrementalShareWindow) + oracle = _python_window_for( + ledger, + anchor_ms=int(rebuilt.snapshot_anchor_ms or 0), + network_difficulty=int(artifacts.network_difficulty), + ) + self.assertEqual( + rebuilt.share_snapshot_sha256, + oracle.json_records().canonical_json_sha256(), + ) + # The daemon is the same live process it was before the contest. + self.assertIs(server._serve_builder, daemon) + self.assertIsNone(daemon.process.poll()) + counts = self._serve_counts(server) + self.assertEqual(counts["fallbacks"], 0) + self.assertEqual(counts["spawns"], 1) + # One refusal for the advance, one for the full re-preparation. + self.assertEqual(counts["window_prepare_contended"], 2) + self.assertEqual(self._worker_counts(server)["terminations"], 0) + self.assertEqual(ledger.full_snapshot_calls, 2) + + def test_deadline_is_stamped_after_the_daemon_lock_is_taken(self) -> None: + server, ledger, artifacts = self._server() + clock_ms = [1_000_000] + server.bundle_build_timeout_seconds = 1.0 + with self._rust_env( + mode="prepare-slow", + FAKE_SERVE_BUILDER_PREPARE_DELAY_SECONDS="0.6", + ), self._fake_daemon_command(), patch( + "lab.prism.prism_coordinator.now_ms", + side_effect=lambda: clock_ms[0], + ): + initial = server._build_payout_ledger_artifact( + 0, 0, artifacts.network_difficulty + ) + assert initial is not None + self.assertIsInstance( + server._incremental_payout_artifact_window.window, + DaemonShareWindowMirror, + ) + + append_incremental_share(ledger, share_seq=4, accepted_at_ms=1_000_010) + server.payout_artifact_min_build_interval_seconds = 0.0 + clock_ms[0] = 1_000_020 + # Most of the budget is spent waiting; the round trip that + # follows needs more than what would be left of it. Charging the + # wait against the exchange is what used to retire the daemon. + self._lock_holder(server, 0.7) + advanced = server._build_payout_ledger_artifact( + 0, 0, artifacts.network_difficulty + ) + assert advanced is not None + self.assertEqual(advanced.window_build_mode, "incremental") + self.assertIsNone(advanced.window_full_rescan_reason) + self.assertIsInstance( + server._incremental_payout_artifact_window.window, + DaemonShareWindowMirror, + ) + oracle = _python_window_for( + ledger, + anchor_ms=int(advanced.snapshot_anchor_ms or 0), + network_difficulty=int(artifacts.network_difficulty), + ) + self.assertEqual( + advanced.share_snapshot_sha256, + oracle.json_records().canonical_json_sha256(), + ) + counts = self._serve_counts(server) + self.assertEqual(counts["fallbacks"], 0) + self.assertEqual(counts["spawns"], 1) + self.assertEqual(counts["window_prepares"], 2) + self.assertEqual(ledger.full_snapshot_calls, 1) + + def test_urgent_preview_materialization_never_queues_for_the_daemon( + self, + ) -> None: + server, _ledger, artifacts = self._server() + clock_ms = [1_000_000] + server.bundle_build_timeout_seconds = 30.0 + with self._rust_env(), self._fake_daemon_command(), patch( + "lab.prism.prism_coordinator.now_ms", + side_effect=lambda: clock_ms[0], + ): + initial = server._build_payout_ledger_artifact( + 0, 0, artifacts.network_difficulty + ) + assert initial is not None + clock_ms[0] = 1_000_020 + # An accepted-block preview publishes on the critical path. It + # takes _payout_state_prepare_lock non-blocking for exactly that + # reason, so its materialization must not then wait on the serve + # lock either. + self._lock_holder(server, 30.0) + started = time.monotonic() + preview = server._build_payout_ledger_artifact( + 0, + 0, + artifacts.network_difficulty, + bypass_build_interval=True, + ) + waited = time.monotonic() - started + assert preview is not None + self.assertLess(waited, 5.0) + self.assertEqual(preview.window_build_mode, "full_rescan") + self.assertEqual( + preview.window_full_rescan_reason, + "window_daemon_busy", + ) + counts = self._serve_counts(server) + self.assertEqual(counts["fallbacks"], 0) + self.assertEqual(counts["window_prepare_contended"], 2) + + +class DaemonWindowRecordCountTests(DaemonWindowPipelineTestCase): + """A declared record count is reconciled with the bytes, eagerly. + + The construction-time digest pins the bytes and says nothing about the + count beside them, so a miscount used to surface only at whichever + consumer first parsed the stream -- possibly inside a serve request that + had already written its prefix, leaving the daemon's stdin holding an + unterminated line for the next request to concatenate onto. + """ + + def test_miscounted_window_is_refused_at_materialization(self) -> None: + server, ledger, artifacts = self._server() + clock_ms = [1_000_000] + with self._rust_env(mode="prepare-miscount"), ( + self._fake_daemon_command() + ), patch( + "lab.prism.prism_coordinator.now_ms", + side_effect=lambda: clock_ms[0], + ): + artifact = server._build_payout_ledger_artifact( + 0, 0, artifacts.network_difficulty + ) + assert artifact is not None + # Refused where it was built: the mirror never reaches the cache, + # so no consumer can be the first to learn of the divergence. + cached = server._incremental_payout_artifact_window + assert cached is not None + self.assertIsInstance(cached.window, IncrementalShareWindow) + self.assertNotIsInstance( + artifact.shares_json, + DaemonShareJsonSequence, + ) + oracle = _python_window_for( + ledger, + anchor_ms=int(artifact.snapshot_anchor_ms or 0), + network_difficulty=int(artifacts.network_difficulty), + ) + self.assertEqual( + artifact.share_snapshot_sha256, + oracle.json_records().canonical_json_sha256(), + ) + # A wrong count is the daemon's answer being wrong, not the + # daemon being sick: it stays registered and alive. + client = server._serve_builder + assert client is not None + self.assertIsNone(client.process.poll()) + with server._payout_artifact_executor_lock: + events = dict(server.payout_artifact_event_counts) + self.assertEqual(events.get("window_mirror_divergence"), 1) + self.assertEqual(self._serve_counts(server)["fallbacks"], 0) + + def test_half_written_request_retires_the_poisoned_daemon(self) -> None: + server, _ledger, artifacts = self._server() + clock_ms = [1_000_000] + with self._rust_env(), self._fake_daemon_command(), patch( + "lab.prism.prism_coordinator.now_ms", + side_effect=lambda: clock_ms[0], + ): + artifact = server._build_payout_ledger_artifact( + 0, 0, artifacts.network_difficulty + ) + assert artifact is not None + assert isinstance(artifact.shares_json, DaemonShareJsonSequence) + # A sequence that only refuses itself once iterated, standing in + # for whatever could still reach a build holding the serve lock. + poisoned = DaemonShareJsonSequence( + canonical_items=artifact.shares_json.canonical_items, + record_count=len(artifact.shares_json) + 1, + ) + found_block = { + "block_height": 101, + "coinbase_value_sats": 50_00000000, + "network_difficulty": int(artifacts.network_difficulty), + "anchor_job_issued_at_ms": int(artifact.snapshot_anchor_ms or 0), + } + serialization = server._share_window_serialization_for_artifact( + artifact, + poisoned, + ) + daemon = server._serve_builder + assert daemon is not None + # Force the window to ride along: the divergence has to be + # raised after the request prefix is already on the daemon's + # stdin, which is the case the narrow catch used to miss. + daemon.uploaded_windows.clear() + with self.assertRaises(DaemonWindowMirrorDivergence): + server.build_audit_bundle( + shares=poisoned, + found_block=found_block, + prior_balances=[], + coinbase_script_sig_suffix_hex="00", + summary_only=True, + payout_policy={"policy": "day-one"}, + share_serialization=serialization, # type: ignore[arg-type] + append_invalidation_epoch=artifact.append_invalidation_epoch, + ) + # The request never reached its terminating newline, so the + # daemon went with it rather than answering "malformed serve + # request" to the next innocent build. + self.assertIsNone(server._serve_builder) + self.assertIsNotNone(daemon.process.poll()) + self.assertIsNone(server._incremental_payout_artifact_window) + + # The next honest build still gets the daemon transport. + rebuilt = server._build_payout_ledger_artifact( + 0, 0, artifacts.network_difficulty + ) + assert rebuilt is not None + summary = server.build_audit_bundle( + shares=rebuilt.shares_json, + found_block=found_block, + prior_balances=[], + coinbase_script_sig_suffix_hex="00", + summary_only=True, + payout_policy={"policy": "day-one"}, + share_serialization=( + server._share_window_serialization_for_artifact( + rebuilt, + rebuilt.shares_json, + ) + ), # type: ignore[arg-type] + append_invalidation_epoch=rebuilt.append_invalidation_epoch, + ) + self.assertEqual(summary["transport"], "serve") + with server._payout_artifact_executor_lock: + events = dict(server.payout_artifact_event_counts) + self.assertEqual(events.get("window_mirror_divergence"), 1) + + +class DaemonWindowIntegerDomainTests(DaemonWindowPipelineTestCase): + """A value outside the daemon's declared widths degrades to Python; the daemon lives. + + The daemon carries ``share_difficulty`` in u128 (and the other integers + in u64/u32/i64); Python carries them unbounded. A window holding a value + above those widths used to come back as a malformed-request error, which + the coordinator read as an anomaly: it SIGKILLed a healthy daemon, + degraded once, and respawned on the next materialization only to kill + it again -- a spawn-and-kill loop, once per materialization, for as long + as the window held the value. The daemon now declines such a request as + ``out_of_range`` and keeps its state; the coordinator folds the window + in-process and never touches the daemon. + """ + + def _append_wide_share( + self, + ledger: SingleWriterShareLedger, + *, + share_seq: int, + accepted_at_ms: int, + ) -> None: + """A share whose difficulty is one above u128: inside Python, outside the daemon.""" + ledger.append( + PendingShare( + share_id=f"wide-{share_seq}", + miner_id=f"miner-{share_seq % 3}", + order_key=f"miner-{share_seq % 3}", + p2mr_program_hex=f"{share_seq % 256:02x}" * 32, + share_difficulty=2**128 + 1, + network_difficulty=1, + template_height=9, + job_id=f"job-{share_seq}", + job_issued_at_ms=accepted_at_ms - 1, + accepted_at_ms=accepted_at_ms, + ntime=1_700_000_000 + share_seq, + ) + ) + + def _worker_counts(self, server: PrismCoordinator) -> dict[str, int]: + with server._job_build_scheduler_lock: + return dict(server.job_build_worker_counts) + + def test_out_of_range_snapshot_folds_in_python_and_keeps_the_daemon(self) -> None: + server, ledger, artifacts = self._server() + self._append_wide_share(ledger, share_seq=4, accepted_at_ms=999_930) + clock_ms = [1_000_000] + with self._rust_env(), self._fake_daemon_command(), patch( + "lab.prism.prism_coordinator.now_ms", + side_effect=lambda: clock_ms[0], + ): + artifact = server._build_payout_ledger_artifact( + 0, 0, artifacts.network_difficulty + ) + assert artifact is not None + self.assertEqual(artifact.window_build_mode, "full_rescan") + # Declined by the daemon, folded in-process: a Python window is + # cached, byte-exact against the oracle, with the wide digits. + cached = server._incremental_payout_artifact_window + assert cached is not None + self.assertIsInstance(cached.window, IncrementalShareWindow) + oracle = _python_window_for( + ledger, + anchor_ms=int(artifact.snapshot_anchor_ms or 0), + network_difficulty=int(artifacts.network_difficulty), + ) + self.assertEqual( + artifact.share_snapshot_sha256, + oracle.json_records().canonical_json_sha256(), + ) + self.assertTrue( + any( + int(record["share_difficulty"]) == 2**128 + 1 + for record in artifact.shares_json + ) + ) + self.assertTrue(server._install_payout_ledger_artifact(artifact)) + # The daemon was never retired: same live process, registered. + daemon = server._serve_builder + assert daemon is not None + self.assertIsNone(daemon.process.poll()) + + # The next materialization still holds the value. Before the fix + # this is where the loop turned: a fresh spawn, declined again, + # killed again. Now the degraded Python cache advances in + # process and the daemon is left alone. + append_incremental_share(ledger, share_seq=5, accepted_at_ms=1_000_010) + server.payout_artifact_min_build_interval_seconds = 0.0 + clock_ms[0] = 1_000_020 + advanced = server._build_payout_ledger_artifact( + 0, 0, artifacts.network_difficulty + ) + assert advanced is not None + self.assertEqual(advanced.window_build_mode, "incremental") + self.assertIs(server._serve_builder, daemon) + self.assertIsNone(daemon.process.poll()) + advanced_oracle = _python_window_for( + ledger, + anchor_ms=int(advanced.snapshot_anchor_ms or 0), + network_difficulty=int(artifacts.network_difficulty), + ) + self.assertEqual( + advanced.share_snapshot_sha256, + advanced_oracle.json_records().canonical_json_sha256(), + ) + counts = self._serve_counts(server) + self.assertEqual(counts["spawns"], 1) + self.assertEqual(counts["fallbacks"], 0) + # window_prepares counts completed round trips, declined ones + # included: one, and it was the decline. The Python-advancing + # second materialization never asked the daemon at all. + self.assertEqual(counts["window_prepares"], 1) + self.assertEqual(counts["window_prepare_out_of_range"], 1) + self.assertEqual(self._worker_counts(server)["terminations"], 0) + + def test_out_of_range_delta_rebuilds_in_python_and_keeps_the_daemon(self) -> None: + server, ledger, artifacts = self._server() + clock_ms = [1_000_000] + with self._rust_env(), self._fake_daemon_command(), patch( + "lab.prism.prism_coordinator.now_ms", + side_effect=lambda: clock_ms[0], + ): + initial = server._build_payout_ledger_artifact( + 0, 0, artifacts.network_difficulty + ) + assert initial is not None + cached = server._incremental_payout_artifact_window + assert cached is not None + self.assertIsInstance(cached.window, DaemonShareWindowMirror) + daemon = server._serve_builder + assert daemon is not None + + # The wide share arrives as a delta against a daemon-held mirror: + # the daemon declines the advance, the full re-preparation is + # declined the same way, and the in-process fold takes over. + self._append_wide_share(ledger, share_seq=4, accepted_at_ms=1_000_010) + server.payout_artifact_min_build_interval_seconds = 0.0 + clock_ms[0] = 1_000_020 + rebuilt = server._build_payout_ledger_artifact( + 0, 0, artifacts.network_difficulty + ) + assert rebuilt is not None + self.assertEqual(rebuilt.window_build_mode, "full_rescan") + self.assertEqual( + rebuilt.window_full_rescan_reason, + "window_value_out_of_range", + ) + cached = server._incremental_payout_artifact_window + assert cached is not None + self.assertIsInstance(cached.window, IncrementalShareWindow) + oracle = _python_window_for( + ledger, + anchor_ms=int(rebuilt.snapshot_anchor_ms or 0), + network_difficulty=int(artifacts.network_difficulty), + ) + self.assertEqual( + rebuilt.share_snapshot_sha256, + oracle.json_records().canonical_json_sha256(), + ) + self.assertIs(server._serve_builder, daemon) + self.assertIsNone(daemon.process.poll()) + counts = self._serve_counts(server) + self.assertEqual(counts["spawns"], 1) + self.assertEqual(counts["fallbacks"], 0) + # Three completed round trips: the initial full preparation + # succeeded, then the advance and the full re-preparation were + # both declined. + self.assertEqual(counts["window_prepares"], 3) + self.assertEqual(counts["window_prepare_out_of_range"], 2) + self.assertEqual(self._worker_counts(server)["terminations"], 0) + self.assertEqual(ledger.full_snapshot_calls, 2) + + def test_rejection_envelopes_carry_the_category_not_just_the_message(self) -> None: + # The fold_invalid/fallback outcomes name their condition in a stable + # category beside the diagnostic message, so the coordinator (and the + # parity adapter) never have to match on prose that may drift. + server, ledger, artifacts = self._server() + clock_ms = [1_000_000] + with self._rust_env(mode="prepare-fallback"), ( + self._fake_daemon_command() + ), patch( + "lab.prism.prism_coordinator.now_ms", + side_effect=lambda: clock_ms[0], + ): + initial = server._build_payout_ledger_artifact( + 0, 0, artifacts.network_difficulty + ) + assert initial is not None + outcome = server.prepare_payout_window( + mode="advance", + records_json=[], + anchor_job_issued_at_ms=1_000_010, + append_invalidation_epoch=0, + base_digest=initial.share_snapshot_sha256, + ) + assert outcome is not None + self.assertEqual(outcome.status, "fallback") + self.assertEqual(outcome.rejection, "delta_not_append") + duplicate = dict(initial.shares_json[0]) + declined = server.prepare_payout_window( + mode="full", + records_json=[duplicate, duplicate], + anchor_job_issued_at_ms=1_000_000, + append_invalidation_epoch=0, + window_weight=1_000, + page_size=512, + ) + assert declined is not None + self.assertEqual(declined.status, "fold_invalid") + self.assertEqual(declined.rejection, "duplicate_share_seq") + + +class DaemonWindowRetagTests(DaemonWindowPipelineTestCase): + """A retagged window advances in place, Rust switch on or off. + + _retire_payout_windows_for_late_append clears the cache only when the + late row predates the cached anchor, and otherwise retags it precisely so + the next delta build advances the window it already has. A daemon that + refused an older-tagged base turned every such retag into a full DB walk, + fold and upload -- the common case made worse, under exactly the replay + bursts that produce the bumps. + """ + + def _retag_after_late_append(self, server: PrismCoordinator) -> int: + cached = server._incremental_payout_artifact_window + assert cached is not None + anchor_ms = int(cached.window.anchor_job_issued_at_ms) + with server._job_cache_lock: + server._payout_ledger_append_invalidation_epoch += 1 + epoch = int(server._payout_ledger_append_invalidation_epoch) + # Stamped above the cached anchor: the armed artifact was affected, + # this incremental window was not, so the coordinator retags it. + server._retire_payout_windows_for_late_append( + stamped_pending_share(anchor_ms + 1_000), + epoch, + ) + retagged = server._incremental_payout_artifact_window + assert retagged is not None + self.assertEqual(retagged.append_invalidation_epoch, epoch) + return epoch + + def test_retagged_window_advances_incrementally_on_both_pipelines( + self, + ) -> None: + for rust in (False, True): + with self.subTest(rust=rust): + server, ledger, artifacts = self._server() + clock_ms = [1_000_000] + env = ( + self._rust_env() + if rust + else patch.dict( + os.environ, + {"PRISM_WINDOW_PIPELINE_RUST": "0"}, + ) + ) + with env, self._fake_daemon_command(), patch( + "lab.prism.prism_coordinator.now_ms", + side_effect=lambda: clock_ms[0], + ): + initial = server._build_payout_ledger_artifact( + 0, 0, artifacts.network_difficulty + ) + assert initial is not None + self.assertEqual(ledger.full_snapshot_calls, 1) + self._retag_after_late_append(server) + + append_incremental_share( + ledger, share_seq=4, accepted_at_ms=1_000_010 + ) + server.payout_artifact_min_build_interval_seconds = 0.0 + clock_ms[0] = 1_000_020 + advanced = server._build_payout_ledger_artifact( + 0, 0, artifacts.network_difficulty + ) + assert advanced is not None + self.assertEqual(advanced.window_build_mode, "incremental") + self.assertIsNone(advanced.window_full_rescan_reason) + self.assertEqual(advanced.window_delta_rows, 1) + # The retag survived the advance: no oracle was paid. + self.assertEqual(ledger.full_snapshot_calls, 1) + oracle = _python_window_for( + ledger, + anchor_ms=int(advanced.snapshot_anchor_ms or 0), + network_difficulty=int(artifacts.network_difficulty), + ) + self.assertEqual( + advanced.share_snapshot_sha256, + oracle.json_records().canonical_json_sha256(), + ) + cached = server._incremental_payout_artifact_window + assert cached is not None + self.assertIsInstance( + cached.window, + DaemonShareWindowMirror if rust else IncrementalShareWindow, + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_prism_worker_difficulty_store.py b/tests/test_prism_worker_difficulty_store.py new file mode 100644 index 000000000..091274139 --- /dev/null +++ b/tests/test_prism_worker_difficulty_store.py @@ -0,0 +1,700 @@ +#!/usr/bin/env python3 +"""Focused tests for the durable worker-difficulty storage seam. + +Everything here runs without a database: the in-memory back end is exercised +directly, the PostgreSQL back end through query capture and canned results, +and the schema through string contracts against the monolithic SQL file. +Real-PostgreSQL parity for the same contract lives in +``tests/prism_worker_difficulty_store_gate.py``. +""" + +from __future__ import annotations + +import unittest + +from decimal import Decimal +from pathlib import Path +from typing import Any + +from lab.prism.worker_difficulty_store import ( + MemoryWorkerDifficultyStore, + PostgresWorkerDifficultyStore, + WORKER_DIFFICULTY_SCHEMA_SQL, + WorkerDifficultyRecord, + WorkerDifficultyStorePort, + WorkerDifficultyUpsertResult, +) + + +ROOT = Path(__file__).resolve().parents[1] +LEDGER_SCHEMA = ROOT / "crates" / "qbit-prism" / "sql" / "001_share_ledger.sql" + +BASE_MS = 1_756_000_000_000 + + +def record( + *, + listener: str = "default", + worker_username: str = "miner.worker1", + difficulty: str = "8192", + evidence_at_ms: int = BASE_MS, + updated_at_ms: int = BASE_MS, +) -> WorkerDifficultyRecord: + return WorkerDifficultyRecord( + listener=listener, + worker_username=worker_username, + difficulty=Decimal(difficulty), + evidence_at_ms=evidence_at_ms, + updated_at_ms=updated_at_ms, + ) + + +class QueryCaptureStore(PostgresWorkerDifficultyStore): + """Postgres store whose runner records SQL and replays canned results.""" + + def __init__(self, results: list[Any] | None = None) -> None: + self.queries: list[str] = [] + self._results = list(results or []) + super().__init__(self._capture) + + def _capture(self, sql: str) -> Any: + self.queries.append(sql) + if self._results: + return self._results.pop(0) + return {"applied": True, "stored": None} + + +class SchemaContractTests(unittest.TestCase): + def test_module_schema_block_matches_monolithic_file_verbatim(self) -> None: + schema = LEDGER_SCHEMA.read_text(encoding="utf-8") + self.assertIn(WORKER_DIFFICULTY_SCHEMA_SQL.strip(), schema) + # Exactly once, inside the file's single BEGIN/COMMIT transaction. + self.assertEqual(schema.count("CREATE TABLE IF NOT EXISTS qbit_worker_difficulty"), 1) + self.assertLess( + schema.index("CREATE TABLE IF NOT EXISTS qbit_worker_difficulty"), + schema.rindex("COMMIT;"), + ) + + def test_schema_block_is_idempotent_and_scoped(self) -> None: + # Idempotent reapply: every statement in the block tolerates the + # object already existing. + for statement in WORKER_DIFFICULTY_SCHEMA_SQL.split(";"): + statement = statement.strip() + if not statement: + continue + self.assertIn("IF NOT EXISTS", statement) + # Primary key covers listener plus exact username; the preload/prune + # index leads on the evidence timestamp and matches the preload + # ORDER BY column-for-column. + self.assertIn("PRIMARY KEY (listener, worker_username)", WORKER_DIFFICULTY_SCHEMA_SQL) + self.assertIn( + "ON qbit_worker_difficulty (evidence_at DESC, listener, worker_username)", + WORKER_DIFFICULTY_SCHEMA_SQL, + ) + # Positive finite difficulty: NaN and Infinity both compare greater + # than zero in PostgreSQL numeric, so the finite guard is load-bearing. + self.assertIn("difficulty > 0 AND difficulty < 'Infinity'::numeric", WORKER_DIFFICULTY_SCHEMA_SQL) + # The decimal wire value must not be forced through an integer typmod. + self.assertIn("difficulty numeric NOT NULL", WORKER_DIFFICULTY_SCHEMA_SQL) + self.assertNotIn("numeric(", WORKER_DIFFICULTY_SCHEMA_SQL) + # Evidence and ordinary-update timestamps are separate columns. + self.assertIn("evidence_at timestamptz NOT NULL", WORKER_DIFFICULTY_SCHEMA_SQL) + self.assertIn("updated_at timestamptz NOT NULL", WORKER_DIFFICULTY_SCHEMA_SQL) + # Tie-breaks are byte order in every deployment and match the + # in-memory store's code-point ordering. + self.assertIn('listener text COLLATE "C" NOT NULL', WORKER_DIFFICULTY_SCHEMA_SQL) + self.assertIn('worker_username text COLLATE "C" NOT NULL', WORKER_DIFFICULTY_SCHEMA_SQL) + + def test_schema_touches_no_canonical_share_or_payout_tables(self) -> None: + for name in ( + "qbit_share_ledger", + "qbit_pool_blocks", + "qbit_pool_payout_entries", + "qbit_payout_carry_forward", + ): + self.assertNotIn(name, WORKER_DIFFICULTY_SCHEMA_SQL) + + +class StorePortTests(unittest.TestCase): + def test_port_exposes_no_per_key_read(self) -> None: + # Reconnect authorization must never block on a synchronous + # database read, so the contract is upsert + bounded batch preload + + # prune, and nothing key-addressed to read. + for store_type in (MemoryWorkerDifficultyStore, PostgresWorkerDifficultyStore): + self.assertTrue(issubclass(store_type, WorkerDifficultyStorePort)) + for forbidden in ("get", "lookup", "load_one", "fetch"): + self.assertFalse( + hasattr(store_type, forbidden), + f"{store_type.__name__} must not expose {forbidden}", + ) + + +class MemoryStoreTests(unittest.TestCase): + def setUp(self) -> None: + self.store = MemoryWorkerDifficultyStore() + + def test_insert_then_update(self) -> None: + first = self.store.upsert( + listener="default", + worker_username="miner.worker1", + difficulty=Decimal("4096"), + evidence_at_ms=BASE_MS, + now_ms=BASE_MS, + ) + self.assertTrue(first.applied) + self.assertEqual( + first.stored, + record(difficulty="4096"), + ) + second = self.store.upsert( + listener="default", + worker_username="miner.worker1", + difficulty=Decimal("8192"), + evidence_at_ms=BASE_MS + 1_000, + now_ms=BASE_MS + 1_500, + ) + self.assertTrue(second.applied) + self.assertEqual( + second.stored, + record( + difficulty="8192", + evidence_at_ms=BASE_MS + 1_000, + updated_at_ms=BASE_MS + 1_500, + ), + ) + self.assertEqual(len(self.store), 1) + + def test_older_evidence_cannot_replace_newer_evidence(self) -> None: + self.store.upsert( + listener="default", + worker_username="miner.worker1", + difficulty=Decimal("8192"), + evidence_at_ms=BASE_MS + 5_000, + now_ms=BASE_MS + 5_000, + ) + stale = self.store.upsert( + listener="default", + worker_username="miner.worker1", + difficulty=Decimal("1"), + evidence_at_ms=BASE_MS + 4_999, + now_ms=BASE_MS + 6_000, + ) + self.assertFalse(stale.applied) + self.assertEqual( + stale.stored, + record( + difficulty="8192", + evidence_at_ms=BASE_MS + 5_000, + updated_at_ms=BASE_MS + 5_000, + ), + ) + loaded = self.store.load_recent(evidence_after_ms=0, limit=10) + self.assertEqual(loaded[0].difficulty, Decimal("8192")) + self.assertEqual(loaded[0].updated_at_ms, BASE_MS + 5_000) + + def test_equal_evidence_is_deterministic_last_write_wins(self) -> None: + self.store.upsert( + listener="default", + worker_username="miner.worker1", + difficulty=Decimal("8192"), + evidence_at_ms=BASE_MS, + now_ms=BASE_MS, + ) + rewrite = self.store.upsert( + listener="default", + worker_username="miner.worker1", + difficulty=Decimal("2048"), + evidence_at_ms=BASE_MS, + now_ms=BASE_MS + 100, + ) + self.assertTrue(rewrite.applied) + self.assertEqual( + rewrite.stored, + record(difficulty="2048", updated_at_ms=BASE_MS + 100), + ) + + def test_lane_separation(self) -> None: + self.store.upsert( + listener="stratum-public", + worker_username="miner.worker1", + difficulty=Decimal("16"), + evidence_at_ms=BASE_MS + 1, + now_ms=BASE_MS + 1, + ) + self.store.upsert( + listener="stratum-highdiff", + worker_username="miner.worker1", + difficulty=Decimal("65536"), + evidence_at_ms=BASE_MS + 2, + now_ms=BASE_MS + 2, + ) + loaded = self.store.load_recent(evidence_after_ms=0, limit=10) + self.assertEqual(len(loaded), 2) + by_lane = {entry.listener: entry.difficulty for entry in loaded} + self.assertEqual(by_lane["stratum-public"], Decimal("16")) + self.assertEqual(by_lane["stratum-highdiff"], Decimal("65536")) + + def test_exact_username_separation(self) -> None: + # Case, surrounding whitespace and lookalikes are distinct identities: + # exact byte match only, no normalization. + usernames = ["miner.worker1", "Miner.worker1", " miner.worker1", "miner.worker1 "] + for index, username in enumerate(usernames): + outcome = self.store.upsert( + listener="default", + worker_username=username, + difficulty=Decimal(2 ** (index + 1)), + evidence_at_ms=BASE_MS + index, + now_ms=BASE_MS + index, + ) + self.assertTrue(outcome.applied) + loaded = self.store.load_recent(evidence_after_ms=0, limit=10) + self.assertEqual(len(loaded), len(usernames)) + self.assertEqual( + {entry.worker_username for entry in loaded}, + set(usernames), + ) + + def test_load_recent_is_bounded_cutoff_exclusive_and_ordered(self) -> None: + for index in range(5): + self.store.upsert( + listener="default", + worker_username=f"miner.worker{index}", + difficulty=Decimal("64"), + evidence_at_ms=BASE_MS + index * 1_000, + now_ms=BASE_MS + index * 1_000, + ) + # Tie on the evidence timestamp to pin the (listener, username) + # tie-break. + self.store.upsert( + listener="alpha-lane", + worker_username="miner.workerZ", + difficulty=Decimal("64"), + evidence_at_ms=BASE_MS + 4_000, + now_ms=BASE_MS + 4_000, + ) + loaded = self.store.load_recent(evidence_after_ms=BASE_MS + 1_000, limit=3) + self.assertEqual( + [ + (entry.evidence_at_ms, entry.listener, entry.worker_username) + for entry in loaded + ], + [ + (BASE_MS + 4_000, "alpha-lane", "miner.workerZ"), + (BASE_MS + 4_000, "default", "miner.worker4"), + (BASE_MS + 3_000, "default", "miner.worker3"), + ], + ) + # The cutoff is strict: an entry exactly at the cutoff is excluded. + cutoff_hits = self.store.load_recent(evidence_after_ms=BASE_MS + 4_000, limit=10) + self.assertEqual(cutoff_hits, []) + + def test_prune_is_cutoff_inclusive_and_complements_load(self) -> None: + for index in range(4): + self.store.upsert( + listener="default", + worker_username=f"miner.worker{index}", + difficulty=Decimal("64"), + evidence_at_ms=BASE_MS + index * 1_000, + now_ms=BASE_MS + index * 1_000, + ) + deleted = self.store.prune(evidence_cutoff_ms=BASE_MS + 1_000) + self.assertEqual(deleted, 2) + survivors = self.store.load_recent(evidence_after_ms=BASE_MS + 1_000, limit=10) + self.assertEqual(len(survivors), 2) + self.assertEqual(len(self.store), 2) + + def test_bounded_prune_removes_oldest_first_deterministically(self) -> None: + entries = [ + ("default", "miner.workerB", BASE_MS), + ("default", "miner.workerA", BASE_MS), + ("alpha-lane", "miner.workerA", BASE_MS), + ("default", "miner.workerC", BASE_MS + 1_000), + ] + for listener, username, evidence_ms in entries: + self.store.upsert( + listener=listener, + worker_username=username, + difficulty=Decimal("64"), + evidence_at_ms=evidence_ms, + now_ms=evidence_ms, + ) + deleted = self.store.prune(evidence_cutoff_ms=BASE_MS + 1_000, limit=2) + self.assertEqual(deleted, 2) + remaining = self.store.load_recent(evidence_after_ms=0, limit=10) + # Oldest evidence first, ties broken by (listener, username): the + # alpha-lane row and default/workerA go, workerB and workerC stay. + self.assertEqual( + {(entry.listener, entry.worker_username) for entry in remaining}, + {("default", "miner.workerB"), ("default", "miner.workerC")}, + ) + + def test_decimal_values_round_trip_exactly(self) -> None: + difficulties = [ + Decimal("0.000030517578125"), # 2**-15 + Decimal("123456789012345678901234567890.5"), + Decimal("1E-12"), + Decimal("8192.000"), + ] + for index, difficulty in enumerate(difficulties): + self.store.upsert( + listener="default", + worker_username=f"miner.worker{index}", + difficulty=difficulty, + evidence_at_ms=BASE_MS + index, + now_ms=BASE_MS + index, + ) + loaded = self.store.load_recent(evidence_after_ms=0, limit=10) + by_user = {entry.worker_username: entry.difficulty for entry in loaded} + for index, difficulty in enumerate(difficulties): + self.assertEqual(by_user[f"miner.worker{index}"], difficulty) + + def test_input_validation(self) -> None: + good = dict( + listener="default", + worker_username="miner.worker1", + difficulty=Decimal("64"), + evidence_at_ms=BASE_MS, + now_ms=BASE_MS, + ) + rejected = [ + {**good, "difficulty": Decimal("NaN")}, + {**good, "difficulty": Decimal("Infinity")}, + {**good, "difficulty": Decimal("0")}, + {**good, "difficulty": Decimal("-1")}, + {**good, "difficulty": 64}, + {**good, "difficulty": 64.0}, + {**good, "difficulty": Decimal("1E+1000")}, + {**good, "difficulty": Decimal("1E-1000")}, + {**good, "listener": ""}, + {**good, "listener": "bad\x00lane"}, + {**good, "worker_username": ""}, + {**good, "evidence_at_ms": -1}, + {**good, "evidence_at_ms": 1.5}, + {**good, "now_ms": True}, + ] + for kwargs in rejected: + with self.assertRaises(ValueError, msg=repr(kwargs)): + self.store.upsert(**kwargs) + with self.assertRaises(ValueError): + self.store.load_recent(evidence_after_ms=0, limit=0) + with self.assertRaises(ValueError): + self.store.prune(evidence_cutoff_ms=0, limit=-1) + self.assertEqual(len(self.store), 0) + + +class PostgresStoreSqlTests(unittest.TestCase): + def test_upsert_sql_is_one_guarded_atomic_statement(self) -> None: + store = QueryCaptureStore( + results=[ + { + "applied": True, + "stored": { + "listener": "default", + "worker_username": "miner.worker1", + "difficulty": "8192.5", + "evidence_at_ms": BASE_MS, + "updated_at_ms": BASE_MS + 1, + }, + } + ] + ) + outcome = store.upsert( + listener="default", + worker_username="o'brien.worker1", + difficulty=Decimal("8192.5"), + evidence_at_ms=BASE_MS, + now_ms=BASE_MS + 1, + ) + self.assertEqual(len(store.queries), 1) + sql = store.queries[0] + self.assertIn("INSERT INTO qbit_worker_difficulty", sql) + self.assertIn("ON CONFLICT (listener, worker_username) DO UPDATE", sql) + # The evidence guard is the only thing standing between older and + # newer evidence; pin it verbatim. + self.assertIn("WHERE EXCLUDED.evidence_at >= existing.evidence_at", sql) + # Values are inlined with quoting; the embedded quote must be doubled. + self.assertIn("'o''brien.worker1'", sql) + self.assertIn("'8192.5'::numeric", sql) + self.assertIn(f"to_timestamp(({BASE_MS}::double precision / 1000.0))", sql) + # Difficulty travels back as text, never as a JSON float. + self.assertIn("difficulty::text", sql) + self.assertEqual( + outcome, + WorkerDifficultyUpsertResult( + applied=True, + stored=record(difficulty="8192.5", updated_at_ms=BASE_MS + 1), + ), + ) + + def test_refused_upsert_parses_standing_row(self) -> None: + store = QueryCaptureStore( + results=[ + { + "applied": False, + "stored": { + "listener": "default", + "worker_username": "miner.worker1", + "difficulty": "8192", + "evidence_at_ms": BASE_MS + 5_000, + "updated_at_ms": BASE_MS + 5_000, + }, + } + ] + ) + outcome = store.upsert( + listener="default", + worker_username="miner.worker1", + difficulty=Decimal("1"), + evidence_at_ms=BASE_MS, + now_ms=BASE_MS + 6_000, + ) + self.assertFalse(outcome.applied) + assert outcome.stored is not None + self.assertEqual(outcome.stored.difficulty, Decimal("8192")) + self.assertEqual(outcome.stored.evidence_at_ms, BASE_MS + 5_000) + + def test_load_recent_sql_pins_bound_order_and_cutoff(self) -> None: + store = QueryCaptureStore(results=[[]]) + loaded = store.load_recent(evidence_after_ms=BASE_MS, limit=250) + self.assertEqual(loaded, []) + sql = store.queries[0] + self.assertIn( + f"WHERE evidence_at > to_timestamp(({BASE_MS}::double precision / 1000.0))", + sql, + ) + self.assertIn("ORDER BY evidence_at DESC, listener, worker_username", sql) + self.assertIn("LIMIT 250", sql) + self.assertIn( + "ORDER BY recent.evidence_at DESC, recent.listener, recent.worker_username", + sql, + ) + + def test_prune_sql_is_store_side_and_boundable(self) -> None: + store = QueryCaptureStore(results=[{"deleted": 3}, {"deleted": 2}]) + self.assertEqual(store.prune(evidence_cutoff_ms=BASE_MS), 3) + self.assertEqual(store.prune(evidence_cutoff_ms=BASE_MS, limit=2), 2) + unbounded, bounded = store.queries + self.assertIn("DELETE FROM qbit_worker_difficulty", unbounded) + self.assertIn( + f"WHERE evidence_at <= to_timestamp(({BASE_MS}::double precision / 1000.0))", + unbounded, + ) + self.assertIn("ORDER BY evidence_at ASC, listener, worker_username", bounded) + self.assertIn("LIMIT 2", bounded) + # The candidate scan and DELETE share one statement, but a concurrent + # upsert can refresh a selected row while DELETE waits on its lock. + # Recheck eligibility on the tuple DELETE actually reaches. + self.assertIn( + f"target.evidence_at <= to_timestamp(({BASE_MS}::double precision / 1000.0))", + bounded, + ) + + def test_statements_touch_only_the_worker_difficulty_table(self) -> None: + store = QueryCaptureStore( + results=[ + {"applied": True, "stored": None}, + {"applied": True, "stored": None}, + [], + {"deleted": 0}, + ] + ) + store.upsert( + listener="default", + worker_username="miner.worker1", + difficulty=Decimal("64"), + evidence_at_ms=BASE_MS, + now_ms=BASE_MS, + ) + store.apply_downward( + listener="default", + worker_username="miner.worker1", + difficulty=Decimal("32"), + now_ms=BASE_MS, + ) + store.load_recent(evidence_after_ms=0, limit=1) + store.prune(evidence_cutoff_ms=0) + for sql in store.queries: + for name in ( + "qbit_share_ledger", + "qbit_pool_blocks", + "qbit_pool_payout_entries", + "qbit_payout_carry_forward", + "qbit_ledger_writer_lease", + ): + self.assertNotIn(name, sql) + + +class LowerOnlyCorrectionTests(unittest.TestCase): + """The atomic downward correction, on the in-memory reference back end. + + The PostgreSQL twin is driven through the same sequence by + tests/prism_worker_difficulty_store_gate.py. + """ + + def setUp(self) -> None: + self.store = MemoryWorkerDifficultyStore() + self.key = {"listener": "default", "worker_username": "miner.worker1"} + + def seed(self, difficulty: str, *, evidence_at_ms: int = BASE_MS) -> None: + self.store.upsert( + **self.key, + difficulty=Decimal(difficulty), + evidence_at_ms=evidence_at_ms, + now_ms=evidence_at_ms, + ) + + def test_lowers_a_higher_row_and_preserves_its_evidence(self) -> None: + self.seed("1048576") + + result = self.store.apply_downward( + **self.key, + difficulty=Decimal("65536"), + now_ms=BASE_MS + 5_000, + ) + + self.assertTrue(result.applied) + assert result.stored is not None + self.assertEqual(result.stored.difficulty, Decimal("65536")) + # The correction is not evidence, so it must not re-stamp the clock + # the TTL and the write ordering both read. + self.assertEqual(result.stored.evidence_at_ms, BASE_MS) + self.assertEqual(result.stored.updated_at_ms, BASE_MS + 5_000) + + def test_never_raises_a_stored_difficulty(self) -> None: + self.seed("65536") + + result = self.store.apply_downward( + **self.key, + difficulty=Decimal("1048576"), + now_ms=BASE_MS + 5_000, + ) + + self.assertFalse(result.applied) + assert result.stored is not None + self.assertEqual(result.stored.difficulty, Decimal("65536")) + self.assertEqual(result.stored.updated_at_ms, BASE_MS) + + def test_equal_difficulty_is_a_no_op(self) -> None: + self.seed("65536") + + result = self.store.apply_downward( + **self.key, + difficulty=Decimal("65536.0"), + now_ms=BASE_MS + 5_000, + ) + + self.assertFalse(result.applied) + self.assertEqual(len(self.store), 1) + + def test_missing_row_is_a_no_op_and_never_an_insert(self) -> None: + result = self.store.apply_downward( + **self.key, + difficulty=Decimal("65536"), + now_ms=BASE_MS, + ) + + # Nothing stored means nothing a restart could resurrect, so there is + # no value to write and no evidence to invent. + self.assertFalse(result.applied) + self.assertIsNone(result.stored) + self.assertEqual(len(self.store), 0) + + def test_a_correction_cannot_suppress_later_share_backed_evidence(self) -> None: + self.seed("1048576", evidence_at_ms=BASE_MS) + self.store.apply_downward( + **self.key, + difficulty=Decimal("65536"), + now_ms=BASE_MS + 60_000, + ) + + # Share evidence recorded after the seed but BEFORE the correction's + # wall clock still has to win: the correction wrote no evidence. + result = self.store.upsert( + **self.key, + difficulty=Decimal("500000"), + evidence_at_ms=BASE_MS + 50, + now_ms=BASE_MS + 70_000, + ) + + self.assertTrue(result.applied) + assert result.stored is not None + self.assertEqual(result.stored.difficulty, Decimal("500000")) + + def test_correction_is_scoped_to_lane_and_exact_username(self) -> None: + self.seed("1048576") + self.store.upsert( + listener="highdiff", + worker_username="miner.worker1", + difficulty=Decimal("1048576"), + evidence_at_ms=BASE_MS, + now_ms=BASE_MS, + ) + self.store.upsert( + listener="default", + worker_username="miner.worker10", + difficulty=Decimal("1048576"), + evidence_at_ms=BASE_MS, + now_ms=BASE_MS, + ) + + self.store.apply_downward( + **self.key, + difficulty=Decimal("65536"), + now_ms=BASE_MS + 1, + ) + + untouched = { + (record.listener, record.worker_username): record.difficulty + for record in self.store.load_recent(evidence_after_ms=0, limit=10) + } + self.assertEqual(untouched[("default", "miner.worker1")], Decimal("65536")) + self.assertEqual( + untouched[("highdiff", "miner.worker1")], Decimal("1048576") + ) + self.assertEqual( + untouched[("default", "miner.worker10")], Decimal("1048576") + ) + + def test_input_validation_matches_the_upsert_path(self) -> None: + with self.assertRaises(ValueError): + self.store.apply_downward( + **self.key, difficulty=Decimal("0"), now_ms=BASE_MS + ) + with self.assertRaises(ValueError): + self.store.apply_downward( + **self.key, difficulty=Decimal("1"), now_ms=-1 + ) + with self.assertRaises(ValueError): + self.store.apply_downward( + listener="", + worker_username="miner.worker1", + difficulty=Decimal("1"), + now_ms=BASE_MS, + ) + + +class LowerOnlySqlTests(unittest.TestCase): + def test_correction_is_one_guarded_atomic_statement(self) -> None: + store = QueryCaptureStore(results=[{"applied": False, "stored": None}]) + + store.apply_downward( + listener="default", + worker_username="miner.worker1", + difficulty=Decimal("65536"), + now_ms=BASE_MS, + ) + + self.assertEqual(len(store.queries), 1) + sql = store.queries[0] + self.assertEqual(sql.count("UPDATE qbit_worker_difficulty"), 1) + # The guard is what makes the correction lower-only... + self.assertIn("AND difficulty > ", sql) + # ...and evidence_at is never on the SET list, which is what stops it + # suppressing later share-backed evidence. + self.assertNotIn("evidence_at =", sql) + self.assertIn("SET difficulty = ", sql) + self.assertIn("updated_at = ", sql) + self.assertNotIn("INSERT", sql) + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_prism_writer_lease_policy.py b/tests/test_prism_writer_lease_policy.py new file mode 100644 index 000000000..ebdae4f8a --- /dev/null +++ b/tests/test_prism_writer_lease_policy.py @@ -0,0 +1,660 @@ +#!/usr/bin/env python3 +"""The writer-lease heartbeat timing policy, as one inequality (issue #212). + +The five heartbeat timings were previously five constants derived from the +adoption silence by hand (``silence / 4``, ``silence * 0.75``) and checked at +startup by a print statement. That is how ``union-mainnet`` came to restart +69 times in one rapid-block burst while every component was healthy: the +derived envelope covered a heartbeat's *idle interval* but not the *tail* of +a lawful verification, and nothing in the code said so. + +These tests hold the policy to three separate obligations: + +``PolicyInequalityTests`` + Safety — the terms that make "this process is gone before a replacement + may adopt" true — and the refusal of combinations that break it. + +``ProductionEvidenceTests`` + The shipped defaults, checked against the numbers issue #212 actually + reported from production. The old policy is *shown* to reject those + observations and the new one to accept them, so the regression is + pinned to evidence rather than to a hand-picked constant. + +``VerificationAttributionTests`` + The phase accumulator: what an operator reads when a heartbeat does + exit. Driven by an injected clock, so every duration in the assertions + is exact rather than approximate. +""" + +from __future__ import annotations + +import unittest + +from lab.prism.coordinator_config import ( + DEFAULT_PRISM_LEDGER_LEASE_HEARTBEAT_EXIT_TIMEOUT_SECONDS, + DEFAULT_PRISM_LEDGER_LEASE_HEARTBEAT_FAILURE_SECONDS, + DEFAULT_PRISM_LEDGER_LEASE_HEARTBEAT_MONITOR_SECONDS, + DEFAULT_PRISM_LEDGER_LEASE_HEARTBEAT_SECONDS, +) +import lab.prism.writer_lease_timing as writer_lease_timing +from lab.prism.writer_lease_timing import ( + DEFAULT_WRITER_LEASE_ADOPTION_SILENCE_SECONDS, + DEFAULT_WRITER_LEASE_HEARTBEAT_POLICY, + LEASE_HEARTBEAT_MODE_PROOF, + LEASE_HEARTBEAT_OUTCOME_PROVEN, + LEASE_HEARTBEAT_PHASES, + LEASE_HEARTBEAT_POLICY_TERMS, + OBSERVED_MONITOR_WAKE_DELAY_SECONDS_227, + WRITER_LEASE_GUARD_STATEMENT_TIMEOUT_SECONDS, + WriterLeaseHeartbeatPolicy, + WriterLeaseHeartbeatPolicyError, + WriterLeaseVerificationAttempt, +) + +# The policy that shipped before this change, reconstructed from the +# constants it derived. Referenced rather than described so the regression +# assertions below are about a real prior configuration. +PRE_212_POLICY = WriterLeaseHeartbeatPolicy( + adoption_silence_seconds=1.0, + heartbeat_interval_seconds=0.25, + failure_budget_seconds=0.75, + monitor_interval_seconds=0.05, + exit_margin_seconds=0.1, +) + +# The staleness cap the pre-#212 code actually enforced, spelled out rather +# than recomputed: it reserved only `exit_margin + 2 * monitor_interval` +# inside the silence and made no room at all for the monitor thread's own +# lateness. Evaluating PRE_212_POLICY through today's corrected envelope +# would report a different (and safer) number than production ran, so the +# evidence assertions below use this constant. +PRE_212_SERVER_PROVEN_CAP_SECONDS = 0.8 + +# The activity budget the same code enforced. +PRE_212_FAILURE_BUDGET_SECONDS = 0.75 + +# The downstream operational mitigation issue #212 describes as temporary: +# it stopped the restart loop but added ~3s to genuine writer failover and +# explained nothing. +DOWNSTREAM_MITIGATION_POLICY = WriterLeaseHeartbeatPolicy( + adoption_silence_seconds=4.0, + heartbeat_interval_seconds=1.0, + failure_budget_seconds=3.0, + monitor_interval_seconds=0.05, + exit_margin_seconds=0.1, +) + +# Server-proven ages, in seconds, reported by representative protective +# exits during the union-mainnet burst on 2026-08-31 (bootstrap pin +# c27a1f5336e661360a86f1df70f2a769e7bcaa14). Every one of these came from a +# coordinator that still held its PostgreSQL advisory guard. +OBSERVED_HEALTHY_SERVER_PROVEN_AGES = (0.76, 0.81, 0.87, 0.91) + +# Activity ages from the same exits. +OBSERVED_HEALTHY_ACTIVITY_AGES = (0.54, 0.61, 0.70, 0.78) + + +class PolicyInequalityTests(unittest.TestCase): + def test_shipped_defaults_satisfy_every_term(self) -> None: + policy = DEFAULT_WRITER_LEASE_HEARTBEAT_POLICY + self.assertEqual(policy.violations(), ()) + self.assertEqual(policy.advisories(), ()) + # (1) exit before adoption, stated as the worst-case timeline + # rather than as a rearranged constant: the newest server-proven + # edge cannot postdate the session's death, the monitor observes + # the stale edge at most one poll plus its whole budgeted lateness + # after the cap elapses, and the hard exit then spends its margin. + # All of that must finish strictly before a successor — which + # cannot start its silence until the guard session is released — + # becomes eligible. + worst_case_exit = ( + policy.server_proven_cap_seconds + + policy.monitor_interval_seconds + + policy.scheduler_slack_seconds + + policy.exit_margin_seconds + ) + self.assertLess(worst_case_exit, policy.adoption_silence_seconds) + # The reserve that makes it strict is one monitor poll. + self.assertAlmostEqual( + policy.adoption_silence_seconds - worst_case_exit, + policy.monitor_interval_seconds, + ) + self.assertLessEqual( + policy.server_proven_cap_seconds + policy.exit_envelope_seconds, + policy.adoption_silence_seconds, + ) + # The startup check is stated on the failure budget, which must + # leave the same envelope strictly free. + self.assertLess( + policy.failure_budget_seconds + policy.exit_envelope_seconds, + policy.adoption_silence_seconds, + ) + # (2) one idle wait cannot exhaust the liveness budget. + self.assertLess( + policy.heartbeat_interval_seconds, + policy.failure_budget_seconds, + ) + # (3) the cap covers the largest gap a healthy coordinator produces. + self.assertGreaterEqual( + policy.server_proven_cap_seconds, + policy.max_healthy_server_gap_seconds, + ) + + def test_configuration_defaults_are_the_policy(self) -> None: + """The coordinator config must not carry a second copy of the numbers.""" + policy = DEFAULT_WRITER_LEASE_HEARTBEAT_POLICY + self.assertEqual( + ( + DEFAULT_PRISM_LEDGER_LEASE_HEARTBEAT_SECONDS, + DEFAULT_PRISM_LEDGER_LEASE_HEARTBEAT_FAILURE_SECONDS, + DEFAULT_PRISM_LEDGER_LEASE_HEARTBEAT_MONITOR_SECONDS, + DEFAULT_PRISM_LEDGER_LEASE_HEARTBEAT_EXIT_TIMEOUT_SECONDS, + DEFAULT_WRITER_LEASE_ADOPTION_SILENCE_SECONDS, + ), + ( + policy.heartbeat_interval_seconds, + policy.failure_budget_seconds, + policy.monitor_interval_seconds, + policy.exit_margin_seconds, + policy.adoption_silence_seconds, + ), + ) + + def test_exit_envelope_reserves_the_monitors_own_lateness(self) -> None: + """The monitor is a thread in the same process as the heartbeat. + + Regression for the review finding on the first cut of this policy: + the envelope reserved only poll granularity and the exit margin, so + a monitor delayed by exactly the slack the policy budgets for the + heartbeat observed the stale edge after the successor's adoption + edge — a stall the policy called acceptable produced two live + writers. + """ + policy = DEFAULT_WRITER_LEASE_HEARTBEAT_POLICY + self.assertAlmostEqual( + policy.exit_envelope_seconds, + policy.exit_margin_seconds + + 2.0 * policy.monitor_interval_seconds + + policy.scheduler_slack_seconds, + ) + # The unreserved spelling is unsafe at the shipped numbers, which + # is why it is not the spelling. + unreserved_envelope = ( + policy.exit_margin_seconds + 2.0 * policy.monitor_interval_seconds + ) + unreserved_cap = ( + policy.adoption_silence_seconds - unreserved_envelope + ) + self.assertGreater( + unreserved_cap + + policy.monitor_interval_seconds + + policy.scheduler_slack_seconds + + policy.exit_margin_seconds, + policy.adoption_silence_seconds, + ) + + def test_healthy_gap_is_derived_from_the_guard_statement_bound(self) -> None: + """The dominant term is the guard session's own statement timeout. + + The monitor can only measure from completed round trips, and + PostgreSQL will let one guarded statement run for the whole + statement_timeout. Any envelope shorter than that plus one idle + interval is a false-exit generator, whatever else is tuned. + """ + policy = DEFAULT_WRITER_LEASE_HEARTBEAT_POLICY + self.assertEqual( + policy.guard_statement_timeout_seconds, + WRITER_LEASE_GUARD_STATEMENT_TIMEOUT_SECONDS, + ) + self.assertEqual( + policy.max_healthy_server_gap_seconds, + policy.heartbeat_interval_seconds + + policy.guard_statement_timeout_seconds + + policy.scheduler_slack_seconds, + ) + + def test_silence_that_cannot_contain_the_exit_envelope_is_unsafe(self) -> None: + policy = WriterLeaseHeartbeatPolicy( + adoption_silence_seconds=1.0, + heartbeat_interval_seconds=0.25, + failure_budget_seconds=1.25, + monitor_interval_seconds=0.05, + exit_margin_seconds=0.1, + ) + violations = policy.violations() + self.assertEqual(len(violations), 1) + self.assertIn("adoption silence", violations[0]) + with self.assertRaises(WriterLeaseHeartbeatPolicyError): + policy.validate() + + def test_interval_at_or_above_the_budget_is_unsafe(self) -> None: + policy = WriterLeaseHeartbeatPolicy( + adoption_silence_seconds=10.0, + heartbeat_interval_seconds=1.0, + failure_budget_seconds=1.0, + monitor_interval_seconds=0.05, + exit_margin_seconds=0.1, + ) + self.assertIn( + "must exceed the heartbeat interval", + "".join(policy.violations()), + ) + + def test_non_finite_and_negative_terms_are_unsafe(self) -> None: + for term in ( + "adoption_silence_seconds", + "heartbeat_interval_seconds", + "failure_budget_seconds", + "monitor_interval_seconds", + "exit_margin_seconds", + ): + for bad in (float("nan"), float("inf"), -1.0): + with self.subTest(term=term, value=bad): + policy = WriterLeaseHeartbeatPolicy( + **{ + **{ + "adoption_silence_seconds": 2.0, + "heartbeat_interval_seconds": 0.25, + "failure_budget_seconds": 1.25, + "monitor_interval_seconds": 0.05, + "exit_margin_seconds": 0.1, + }, + term: bad, + } + ) + self.assertTrue(policy.violations()) + + def test_safe_but_tight_policy_advises_instead_of_refusing(self) -> None: + """Stability is a warning; safety is a refusal. + + A lab or test policy may legitimately run a silence far below the + guard's statement timeout. That cannot produce two writers — the + exit still precedes adoption — so refusing to start would turn a + deliberate trade into an outage. Its scheduler slack is scaled with + the rest of it: the slack is a term of the safety inequality, so a + scaled policy that left it at the production value would be + genuinely unsafe rather than merely tight. + """ + policy = WriterLeaseHeartbeatPolicy( + adoption_silence_seconds=0.5, + heartbeat_interval_seconds=0.01, + failure_budget_seconds=0.2, + monitor_interval_seconds=0.005, + exit_margin_seconds=0.01, + scheduler_slack_seconds=0.005, + ) + self.assertEqual(policy.violations(), ()) + self.assertEqual(len(policy.advisories()), 1) + self.assertIn("issue #212", policy.advisories()[0]) + self.assertLess(policy.stability_surplus_seconds, 0.0) + + def test_describe_names_every_term_of_the_inequality(self) -> None: + described = DEFAULT_WRITER_LEASE_HEARTBEAT_POLICY.describe() + for term in ( + "adoption_silence=", + "interval=", + "failure_budget=", + "monitor=", + "exit_margin=", + "guard_statement_timeout=", + "scheduler_slack=", + "server_proven_cap=", + "healthy_gap=", + "surplus=", + ): + self.assertIn(term, described) + + +class ProductionEvidenceTests(unittest.TestCase): + """The shipped policy, judged against issue #212's production numbers.""" + + def test_pre_212_policy_bounds_fall_inside_healthy_production_ranges( + self, + ) -> None: + """The old envelope hard-exits coordinators that were fine. + + This is the defect, stated as arithmetic. Both of the old bounds + sit *inside* the ranges the burst reported from coordinators that + still held their PostgreSQL advisory guard, so healthy operation + crosses them: a protective exit stops being evidence of a problem + and becomes a function of ordinary latency. The observations are + reported as ranges rather than paired samples, so the claim under + test is about the ranges reaching the bounds, not about which of + the two bounds any individual exit tripped. + """ + self.assertGreaterEqual( + max(OBSERVED_HEALTHY_SERVER_PROVEN_AGES), + PRE_212_SERVER_PROVEN_CAP_SECONDS, + ) + self.assertGreaterEqual( + max(OBSERVED_HEALTHY_ACTIVITY_AGES), + PRE_212_FAILURE_BUDGET_SECONDS, + ) + self.assertEqual( + PRE_212_POLICY.failure_budget_seconds, + PRE_212_FAILURE_BUDGET_SECONDS, + ) + # And the old policy could not have been fixed by tuning alone. It + # does not merely lack tail-latency headroom: judged by the + # corrected exit envelope it never satisfied exit-before-adoption + # at all, because a one-second silence cannot hold a 0.75s budget + # plus the monitor's own budgeted lateness. + self.assertTrue(PRE_212_POLICY.violations()) + self.assertLess(PRE_212_POLICY.stability_surplus_seconds, 0.0) + + def test_shipped_policy_accepts_every_healthy_observation(self) -> None: + policy = DEFAULT_WRITER_LEASE_HEARTBEAT_POLICY + self.assertGreater( + policy.server_proven_cap_seconds, + PRE_212_SERVER_PROVEN_CAP_SECONDS, + ) + for age in OBSERVED_HEALTHY_SERVER_PROVEN_AGES: + with self.subTest(server_proven_age=age): + self.assertLess(age, policy.server_proven_cap_seconds) + for age in OBSERVED_HEALTHY_ACTIVITY_AGES: + with self.subTest(activity_age=age): + self.assertLess(age, policy.failure_budget_seconds) + + def test_shipped_policy_fails_over_faster_than_the_mitigation(self) -> None: + """The point of the upstream fix over the 4-second pin. + + Both are safe; the shipped one costs half the failover latency and + keeps a heartbeat interval short enough to prove ownership several + times inside one silence window. + """ + policy = DEFAULT_WRITER_LEASE_HEARTBEAT_POLICY + self.assertEqual(DOWNSTREAM_MITIGATION_POLICY.violations(), ()) + self.assertLess( + policy.adoption_silence_seconds, + DOWNSTREAM_MITIGATION_POLICY.adoption_silence_seconds, + ) + self.assertGreaterEqual( + policy.adoption_silence_seconds / policy.heartbeat_interval_seconds, + 4.0, + ) + + def test_shipped_silence_is_the_smallest_round_safe_value(self) -> None: + """No unexplained slack: one step down breaks the inequality. + + Guards the derivation itself. The silence is not a comfort value — + 1.95s is the exact floor for these phase budgets, so anything at or + below it must be refused, and the shipped 2.0s is the next round + value above it. + """ + policy = DEFAULT_WRITER_LEASE_HEARTBEAT_POLICY + floor = policy.failure_budget_seconds + policy.exit_envelope_seconds + self.assertAlmostEqual(floor, 1.95) + at_the_floor = WriterLeaseHeartbeatPolicy( + adoption_silence_seconds=floor, + heartbeat_interval_seconds=policy.heartbeat_interval_seconds, + failure_budget_seconds=policy.failure_budget_seconds, + monitor_interval_seconds=policy.monitor_interval_seconds, + exit_margin_seconds=policy.exit_margin_seconds, + ) + self.assertTrue(at_the_floor.violations()) + + +class VerificationAttributionTests(unittest.TestCase): + """Phase attribution: the answer to "which phase consumed the envelope?".""" + + class Clock: + def __init__(self) -> None: + self.now = 1_000.0 + + def __call__(self) -> float: + return self.now + + def advance(self, seconds: float) -> float: + self.now += seconds + return self.now + + def test_phases_separate_queue_wait_sql_and_scheduler_delay(self) -> None: + clock = self.Clock() + attempt = WriterLeaseVerificationAttempt( + LEASE_HEARTBEAT_MODE_PROOF, + monotonic=clock, + ) + clock.advance(0.30) # queued behind another guard caller + attempt.slot_acquired() + clock.advance(0.40) # the statement itself + attempt.statement_completed() + clock.advance(0.05) # this process could not get scheduled + phases = attempt.finish(LEASE_HEARTBEAT_OUTCOME_PROVEN) + + self.assertAlmostEqual(phases.slot_wait_seconds, 0.30) + self.assertAlmostEqual(phases.guard_sql_seconds, 0.40) + self.assertAlmostEqual(phases.scheduler_delay_seconds, 0.05) + self.assertAlmostEqual(phases.total_seconds, 0.75) + self.assertEqual(phases.statement_count, 1) + # No server-reported time: the conservative reading, with the + # whole statement span in guard_sql and nothing in client resume. + self.assertFalse(phases.server_reported) + self.assertAlmostEqual(phases.client_resume_seconds, 0.0) + self.assertEqual( + set(phases.phase_seconds()), + { + "guard_slot_wait", + "guard_sql", + "guard_client_resume", + "scheduler_delay", + "total", + }, + ) + self.assertEqual(tuple(phases.phase_seconds()), LEASE_HEARTBEAT_PHASES) + + def test_summary_names_the_phase_that_consumed_the_time(self) -> None: + clock = self.Clock() + attempt = WriterLeaseVerificationAttempt( + LEASE_HEARTBEAT_MODE_PROOF, + monotonic=clock, + ) + attempt.slot_acquired() + clock.advance(0.45) + attempt.statement_completed() + summary = attempt.finish(LEASE_HEARTBEAT_OUTCOME_PROVEN).summary() + self.assertIn("mode=proof", summary) + self.assertIn("outcome=proven", summary) + self.assertIn("guard_sql=0.450s(1 stmt)", summary) + + def test_proven_edge_is_the_send_time_not_the_receipt_time(self) -> None: + """A response proves liveness no later than when the request left. + + Stamping receipt time would let scheduler delay between the answer + arriving and the stamp make the session look fresher than + PostgreSQL proved, spending adoption envelope the exit ordering + argument budgets for the monitor's own poll. + """ + clock = self.Clock() + attempt = WriterLeaseVerificationAttempt( + LEASE_HEARTBEAT_MODE_PROOF, + monotonic=clock, + ) + attempt.slot_acquired() + sent_at = clock.now + clock.advance(0.40) + proven = attempt.statement_completed() + self.assertEqual(proven, sent_at) + self.assertEqual(attempt.proven_edge_monotonic, sent_at) + + def test_second_statement_proves_from_the_first_response(self) -> None: + """The attribution recheck's send edge is the previous answer. + + A followup leaves for PostgreSQL as soon as the previous result is + in hand, so that instant — not the recheck's completion — is the + conservative edge it proves. + """ + clock = self.Clock() + attempt = WriterLeaseVerificationAttempt("renew", monotonic=clock) + attempt.slot_acquired() + clock.advance(0.20) + attempt.statement_completed() + first_response_at = clock.now + clock.advance(0.30) + self.assertEqual(attempt.statement_completed(), first_response_at) + phases = attempt.finish("renewed") + self.assertEqual(phases.statement_count, 2) + self.assertAlmostEqual(phases.guard_sql_seconds, 0.50) + self.assertAlmostEqual(phases.scheduler_delay_seconds, 0.0) + + def test_unreported_attempt_lands_entirely_in_scheduler_delay(self) -> None: + """A verifier with no callbacks still yields a truthful total. + + Nothing is invented: with no slot mark and no round trip, the whole + duration is unattributed to the database, which is itself the + signal that the attempt never demonstrably reached PostgreSQL. + """ + clock = self.Clock() + attempt = WriterLeaseVerificationAttempt("renew", monotonic=clock) + clock.advance(0.9) + phases = attempt.finish("renewed") + self.assertAlmostEqual(phases.slot_wait_seconds, 0.0) + self.assertAlmostEqual(phases.guard_sql_seconds, 0.0) + self.assertAlmostEqual(phases.scheduler_delay_seconds, 0.9) + self.assertEqual(phases.statement_count, 0) + + +class ProofDocstringTests(unittest.TestCase): + """The module docstring is the proof; it must not drift from the code. + + Issue #227 found the proof presenting ``1.95 < 2.00`` as though the + slack term bounded reality, three days after production had exceeded + it. These tests hold the docstring to the constants, hold the stated + monitor-lateness bound (inequality (4)) to the policy, and pin the + production observation the docstring now discusses to the number the + code carries. + """ + + DOCSTRING = writer_lease_timing.__doc__ or "" + + def _documented_number(self, term: str) -> float: + """The value a ``term = ... = `` line in the defaults table states.""" + import re + + heading = "The shipped defaults\n" + self.assertIn(heading, self.DOCSTRING) + table = self.DOCSTRING.split(heading, 1)[1] + pattern = re.compile( + r"^\s+(?:=>\s+)?" + re.escape(term) + r"\s+=\s+(?P[^(\n]+)", + re.MULTILINE, + ) + match = pattern.search(table) + self.assertIsNotNone(match, f"defaults table has no line for {term}") + assert match is not None + numbers = re.findall(r"[0-9]+\.[0-9]+", match.group("expr")) + self.assertTrue(numbers, f"no number on the {term} line") + # A derived line spells the arithmetic and ends with its result. + return float(numbers[-1]) + + def test_documented_shipped_numbers_match_the_constants(self) -> None: + policy = DEFAULT_WRITER_LEASE_HEARTBEAT_POLICY + for term, actual in ( + ("guard_statement_timeout", policy.guard_statement_timeout_seconds), + ("heartbeat_interval", policy.heartbeat_interval_seconds), + ("scheduler_slack", policy.scheduler_slack_seconds), + ("max_healthy_server_gap", policy.max_healthy_server_gap_seconds), + ("failure_budget", policy.failure_budget_seconds), + ("monitor_interval", policy.monitor_interval_seconds), + ("exit_margin", policy.exit_margin_seconds), + ("exit_envelope", policy.exit_envelope_seconds), + ("adoption_silence", policy.adoption_silence_seconds), + ("server_proven_cap", policy.server_proven_cap_seconds), + ("stability surplus", policy.stability_surplus_seconds), + ( + "max_guaranteed_monitor_lateness", + policy.max_guaranteed_monitor_lateness_seconds, + ), + ): + with self.subTest(term=term): + self.assertAlmostEqual(self._documented_number(term), actual) + + def test_inequality_one_is_still_rejected_when_violated(self) -> None: + """Guarding the proof: an unsafe silence is refused, not advised.""" + policy = DEFAULT_WRITER_LEASE_HEARTBEAT_POLICY + floor = policy.failure_budget_seconds + policy.exit_envelope_seconds + for silence in (floor, floor - 0.01, 1.0): + with self.subTest(adoption_silence=silence): + unsafe = WriterLeaseHeartbeatPolicy( + adoption_silence_seconds=silence, + heartbeat_interval_seconds=policy.heartbeat_interval_seconds, + failure_budget_seconds=policy.failure_budget_seconds, + monitor_interval_seconds=policy.monitor_interval_seconds, + exit_margin_seconds=policy.exit_margin_seconds, + ) + self.assertTrue(unsafe.violations()) + with self.assertRaises(WriterLeaseHeartbeatPolicyError): + unsafe.validate() + + def test_monitor_lateness_bound_is_inequality_four(self) -> None: + """(4): exit precedes adoption iff L < slack + monitor_interval. + + Stated from the worst-case timeline rather than from the property: + the poll that first sees the stale edge is at most one interval + plus L after the cap elapsed, the exit spends its margin, and the + successor is eligible one silence after the proven edge. + """ + policy = DEFAULT_WRITER_LEASE_HEARTBEAT_POLICY + bound = policy.max_guaranteed_monitor_lateness_seconds + self.assertAlmostEqual( + bound, + policy.scheduler_slack_seconds + policy.monitor_interval_seconds, + ) + self.assertAlmostEqual(bound, 0.55) + for lateness, guaranteed in ( + (policy.scheduler_slack_seconds, True), + (bound - 1e-9, True), + (bound + 1e-9, False), + (OBSERVED_MONITOR_WAKE_DELAY_SECONDS_227, False), + ): + with self.subTest(lateness=lateness): + worst_case_exit = ( + policy.server_proven_cap_seconds + + policy.monitor_interval_seconds + + lateness + + policy.exit_margin_seconds + ) + self.assertEqual( + worst_case_exit < policy.adoption_silence_seconds, + guaranteed, + ) + self.assertEqual( + policy.exit_guarantee_overrun_seconds(lateness) == 0.0, + guaranteed, + ) + + def test_production_observation_exceeds_the_bound_and_is_documented( + self, + ) -> None: + """The #227 record breaks the assumption, and the docstring says so.""" + policy = DEFAULT_WRITER_LEASE_HEARTBEAT_POLICY + observed = OBSERVED_MONITOR_WAKE_DELAY_SECONDS_227 + self.assertGreater(observed, policy.scheduler_slack_seconds) + self.assertGreater(observed, policy.max_guaranteed_monitor_lateness_seconds) + self.assertAlmostEqual( + policy.exit_guarantee_overrun_seconds(observed), + observed - 0.55, + ) + # The sum the spec quotes, with the real lateness in place of slack. + self.assertAlmostEqual( + policy.server_proven_cap_seconds + + policy.monitor_interval_seconds + + observed + + policy.exit_margin_seconds, + 2.098, + ) + self.assertIn(f"{observed:.3f}s", self.DOCSTRING) + self.assertIn("2.098 > 2.00", self.DOCSTRING) + self.assertIn("issue #227", self.DOCSTRING) + # The decision is recorded, not just the observation. + self.assertIn("(b)", self.DOCSTRING) + self.assertIn("the shipped response is (b)", self.DOCSTRING) + + def test_policy_terms_and_describe_include_the_lateness_bound(self) -> None: + self.assertIn("max_guaranteed_monitor_lateness", LEASE_HEARTBEAT_POLICY_TERMS) + self.assertIn( + "max_monitor_lateness=0.55s", + DEFAULT_WRITER_LEASE_HEARTBEAT_POLICY.describe(), + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_public_dashboard_api_contract.py b/tests/test_public_dashboard_api_contract.py index 97f64a596..709798424 100644 --- a/tests/test_public_dashboard_api_contract.py +++ b/tests/test_public_dashboard_api_contract.py @@ -22,9 +22,12 @@ EXPECTED_FIXTURES = { "pool-summary.json": "prism.dashboard.pool-summary.v1", "hashrate-series.json": "prism.dashboard.hashrate-series.v1", + "hashrate-series-dual-rate.json": "prism.dashboard.hashrate-series.v2", "leaderboard.json": "prism.dashboard.leaderboard.v2", "leaderboard-legacy.json": "prism.dashboard.leaderboard.v1", "blocks.json": "prism.dashboard.blocks.v1", + "blocks-chain-states.json": "prism.dashboard.blocks.v2", + "block-markers.json": "prism.dashboard.block-markers.v1", "settlement-artifacts.json": "prism.dashboard.settlement-artifacts.v1", "settlement-artifacts-direct-coinbase.json": "prism.dashboard.settlement-artifacts.v1", "pending-fanouts.json": "prism.dashboard.pending-fanouts.v1", @@ -40,6 +43,7 @@ EXPECTED_ROUTES = ( "/pool-summary:", "/hashrate-series:", + "/block-markers:", "/leaderboard:", "/blocks:", "/blocks/{block_hash}/settlement-artifacts:", @@ -129,6 +133,7 @@ def test_pagination_fixtures_are_consistent(self) -> None: "leaderboard.json", "leaderboard-legacy.json", "blocks.json", + "blocks-chain-states.json", "pending-fanouts.json", "miner-earnings.json", "miner-payouts.json", @@ -267,13 +272,238 @@ def test_legacy_leaderboard_fixture_retains_exact_v1_shape(self) -> None: ) def test_hashrate_subject_fixture_matches_subject_type(self) -> None: - subject = self.load_fixture("hashrate-series.json")["subject"] - if subject["type"] == "pool": - self.assertIsNone(subject["id"]) + for fixture_name in ("hashrate-series.json", "hashrate-series-dual-rate.json"): + subject = self.load_fixture(fixture_name)["subject"] + if subject["type"] == "pool": + self.assertIsNone(subject["id"], fixture_name) + else: + self.assertEqual(subject["type"], "miner", fixture_name) + self.assertIsInstance(subject["id"], str, fixture_name) + self.assertGreater(len(subject["id"]), 0, fixture_name) + + def test_legacy_hashrate_series_fixture_retains_exact_v1_shape(self) -> None: + series = self.load_fixture("hashrate-series.json") + + self.assertEqual(set(series), {"schema", "generated_at", "subject", "range", "bucket", "unit", "points"}) + self.assertGreater(len(series["points"]), 0) + for point in series["points"]: + self.assertEqual( + set(point), + {"timestamp", "hashrate_ths", "accepted_share_count", "accepted_share_difficulty"}, + ) + + def test_dual_rate_hashrate_series_fixture_is_internally_consistent(self) -> None: + series = self.load_fixture("hashrate-series-dual-rate.json") + + self.assertEqual( + set(series), + { + "schema", + "generated_at", + "subject", + "range", + "bucket", + "bucket_seconds", + "unit", + "rate_basis", + "smoothing", + "points", + }, + ) + self.assertEqual(series["unit"], "ths") + self.assertEqual(series["rate_basis"], "accepted_share_difficulty") + bucket_seconds = series["bucket_seconds"] + self.assertEqual(bucket_seconds, public_api.HASHRATE_SERIES_BUCKET_SECONDS[series["bucket"]]) + smoothing = series["smoothing"] + self.assertEqual(set(smoothing), {"method", "window_seconds"}) + window_seconds = smoothing["window_seconds"] + self.assertIsInstance(window_seconds, int) + self.assertEqual(window_seconds % bucket_seconds, 0) + if smoothing["method"] == "none": + self.assertEqual(window_seconds, bucket_seconds) else: - self.assertEqual(subject["type"], "miner") - self.assertIsInstance(subject["id"], str) - self.assertGreater(len(subject["id"]), 0) + self.assertEqual(smoothing["method"], "trailing") + self.assertGreaterEqual(window_seconds // bucket_seconds, 2) + self.assertEqual( + smoothing, + public_api.hashrate_series_smoothing(bucket_seconds=bucket_seconds, window_seconds=window_seconds), + ) + + generated_at_epoch = self.epoch(series["generated_at"]) + points = series["points"] + self.assertGreater(len(points), 0) + epochs = [self.epoch(point["timestamp"]) for point in points] + self.assertEqual(epochs, sorted(set(epochs))) + for point, epoch in zip(points, epochs): + self.assertEqual( + set(point), + { + "timestamp", + "raw_hashrate_ths", + "smoothed_hashrate_ths", + "accepted_share_count", + "accepted_share_difficulty", + "complete", + }, + ) + self.assertEqual(epoch % bucket_seconds, 0, point["timestamp"]) + self.assertIsInstance(point["complete"], bool) + self.assertEqual(point["complete"], generated_at_epoch >= epoch + bucket_seconds, point["timestamp"]) + # Every raw rate is credit over the full bucket, so the trailing + # estimate is the window's raw rates summed and re-scaled from + # bucket duration to window duration, with missing buckets as zero. + window_raw_total = sum( + Decimal(other["raw_hashrate_ths"]) + for other, other_epoch in zip(points, epochs) + if epoch - window_seconds < other_epoch <= epoch + ) + # Each public rate is independently rounded by decimal_string, so + # their re-scaled totals may differ by the final decimal place. + smoothed_total = Decimal(point["smoothed_hashrate_ths"]) * Decimal( + window_seconds + ) + raw_total = window_raw_total * Decimal(bucket_seconds) + self.assertLessEqual( + abs(smoothed_total - raw_total), + max(abs(raw_total) * Decimal("1e-30"), Decimal("1e-30")), + point["timestamp"], + ) + if smoothing["method"] == "none": + self.assertEqual(Decimal(point["smoothed_hashrate_ths"]), Decimal(point["raw_hashrate_ths"])) + + def test_block_markers_fixture_is_internally_consistent(self) -> None: + markers = self.load_fixture("block-markers.json") + + self.assertEqual( + set(markers), + {"schema", "generated_at", "range", "bucket", "bucket_seconds", "total_blocks", "points"}, + ) + bucket_seconds = markers["bucket_seconds"] + self.assertEqual(bucket_seconds, public_api.HASHRATE_SERIES_BUCKET_SECONDS[markers["bucket"]]) + self.assertIn(markers["bucket"], public_api.HASHRATE_SERIES_ALLOWED_BUCKETS[markers["range"]]) + points = markers["points"] + self.assertGreater(len(points), 0) + epochs = [self.epoch(point["timestamp"]) for point in points] + self.assertEqual(epochs, sorted(set(epochs))) + # total_blocks counts every non-reversed block in range and every such + # block lands in some listed bucket, so the two totals must agree. + self.assertEqual( + markers["total_blocks"], + sum(point["block_count"] for point in points), + ) + for point, epoch in zip(points, epochs): + self.assertEqual(set(point), {"timestamp", "block_count", "blocks", "truncated"}) + # Marker buckets sit on the hashrate chart's floor-aligned grid. + self.assertEqual(epoch % bucket_seconds, 0, point["timestamp"]) + self.assertGreater(point["block_count"], 0) + blocks = point["blocks"] + self.assertEqual( + len(blocks), + min(point["block_count"], public_api.BLOCK_MARKERS_MAX_BLOCKS_PER_BUCKET), + ) + self.assertEqual( + point["truncated"], + point["block_count"] > public_api.BLOCK_MARKERS_MAX_BLOCKS_PER_BUCKET, + ) + found_epochs = [self.epoch(block["found_at"]) for block in blocks] + self.assertEqual(found_epochs, sorted(found_epochs, reverse=True)) + for block, found_epoch in zip(blocks, found_epochs): + self.assertEqual(set(block), {"height", "hash", "found_at"}) + self.assertEqual(found_epoch // bucket_seconds * bucket_seconds, epoch, block["hash"]) + + def test_block_markers_fixture_shares_the_blocks_fixture_pool(self) -> None: + # The newest marker is blocks.json's latest block, so the two fixtures + # describe one pool and a dashboard can render them together. + markers = self.load_fixture("block-markers.json") + latest = self.load_fixture("blocks.json")["rows"][0] + newest_marker = markers["points"][-1]["blocks"][0] + self.assertEqual(newest_marker["hash"], latest["hash"]) + self.assertEqual(newest_marker["height"], latest["height"]) + self.assertEqual(newest_marker["found_at"], latest["found_at"]) + + def test_openapi_block_markers_declares_marker_contract(self) -> None: + text = OPENAPI_PATH.read_text(encoding="utf-8") + self.assertIn( + " BlockMarkerPoint:\n" + " type: object\n" + " additionalProperties: false\n" + " required: [timestamp, block_count, blocks, truncated]\n", + text, + ) + self.assertIn( + " BlockMarkerBlock:\n" + " type: object\n" + " additionalProperties: false\n" + " required: [height, hash, found_at]\n", + text, + ) + self.assertIn("const: prism.dashboard.block-markers.v1", text) + self.assertIn("maxItems: 3", text) + + readme_text = (CONTRACT_DIR / "README.md").read_text(encoding="utf-8") + self.assertIn("/public/v1/block-markers", readme_text) + self.assertIn("block-markers.json", readme_text) + + def test_openapi_hashrate_series_declares_dual_rate_view(self) -> None: + text = OPENAPI_PATH.read_text(encoding="utf-8") + self.assertIn("- name: view\n in: query", text) + self.assertIn("enum: [both]", text) + self.assertIn('- $ref: "#/components/schemas/HashrateSeriesResponse"', text) + self.assertIn('- $ref: "#/components/schemas/HashrateSeriesDualRateResponse"', text) + self.assertIn("HashrateSmoothing:", text) + self.assertIn("enum: [trailing, none]", text) + self.assertIn("const: accepted_share_difficulty", text) + self.assertIn("credited hashrate", text) + # The documented required fields are exactly the fixture's keys. + self.assertIn( + " DualRateHashratePoint:\n" + " type: object\n" + " additionalProperties: false\n" + " required:\n" + " - timestamp\n" + " - raw_hashrate_ths\n" + " - smoothed_hashrate_ths\n" + " - accepted_share_count\n" + " - accepted_share_difficulty\n" + " - complete\n", + text, + ) + self.assertIn( + " HashrateSeriesDualRateResponse:\n" + " type: object\n" + " additionalProperties: false\n" + " required:\n" + " - schema\n" + " - generated_at\n" + " - subject\n" + " - range\n" + " - bucket\n" + " - bucket_seconds\n" + " - unit\n" + " - rate_basis\n" + " - smoothing\n" + " - points\n", + text, + ) + # The legacy point schema is untouched. + self.assertIn( + " HashratePoint:\n" + " type: object\n" + " additionalProperties: false\n" + " required:\n" + " - timestamp\n" + " - hashrate_ths\n" + " - accepted_share_count\n" + " - accepted_share_difficulty\n", + text, + ) + + readme_text = (CONTRACT_DIR / "README.md").read_text(encoding="utf-8") + self.assertIn("credited hashrate", readme_text) + self.assertIn("`view=both`", readme_text) + self.assertIn("prism.dashboard.hashrate-series.v2", readme_text) + self.assertIn("PRISM_PUBLIC_HASHRATE_SMOOTHING_SECONDS", readme_text) + self.assertIn("router telemetry", readme_text) def test_fanout_fixture_status_matches_tip_height(self) -> None: tip_height = self.load_fixture("pool-summary.json")["network"]["height"] @@ -322,11 +552,97 @@ def test_current_fixture_difficulty_is_derived_from_compact_bits(self) -> None: ) def test_historical_block_fixture_difficulty_is_derived_from_compact_bits(self) -> None: + for fixture_name in ("blocks.json", "blocks-chain-states.json"): + for row in self.load_fixture(fixture_name)["rows"]: + self.assertEqual( + Decimal(row["network_difficulty"]), + public_api.scaled_network_difficulty(row["bits"]), + fixture_name, + ) + + BLOCK_ROW_V1_KEYS = { + "height", + "hash", + "found_at", + "network_difficulty", + "bits", + "solver_recipient_id", + "solver_worker_name", + "solver_share_difficulty", + "reward_window_weight", + "coinbase_value_bits", + "audit_bundle_sha256", + "payout_manifest_sha256", + "explorer_url", + } + + def test_default_blocks_fixture_keeps_exact_v1_row_shape(self) -> None: + # The chain_state filter must not leak into the default response: the + # v1 fixture stays byte-shape identical to the pre-filter contract. for row in self.load_fixture("blocks.json")["rows"]: + self.assertEqual(set(row), self.BLOCK_ROW_V1_KEYS) + + def test_chain_state_blocks_fixture_adds_only_reorg_visibility_fields(self) -> None: + fixture = self.load_fixture("blocks-chain-states.json") + rows = fixture["rows"] + + self.assertGreater(len(rows), 0) + reversed_rows = [row for row in rows if row["chain_state"] == "reversed"] + self.assertGreater(len(reversed_rows), 0) + for row in rows: self.assertEqual( - Decimal(row["network_difficulty"]), - public_api.scaled_network_difficulty(row["bits"]), + set(row), + self.BLOCK_ROW_V1_KEYS | {"chain_state", "disconnected_at"}, + ) + self.assertIn( + row["chain_state"], + {"prepared", "confirmed", "inactive", "rejected", "reversed"}, ) + if row["chain_state"] == "reversed": + self.assertIsNotNone(row["disconnected_at"]) + else: + self.assertIsNone(row["disconnected_at"]) + + def test_pool_summary_fixture_reports_reorg_counts_and_network_hashrate(self) -> None: + pool_summary = self.load_fixture("pool-summary.json") + pool = pool_summary["pool"] + + for key in ("blocks_reversed_total", "blocks_inactive_total"): + self.assertIsInstance(pool[key], int) + self.assertGreaterEqual(pool[key], 0) + network_hashrate = pool_summary["network"]["hashrate_ths"] + self.assertIsInstance(network_hashrate, str) + self.assertRegex(network_hashrate, DECIMAL_PATTERN) + # The pool's 3h credited rate divided by the network estimate is the + # documented share-of-network ratio; the fixture keeps it plausible. + share = Decimal(pool["hashrate_ths"]["h3"]) / Decimal(network_hashrate) + self.assertGreater(share, 0) + self.assertLess(share, 1) + + def test_openapi_declares_blocks_chain_state_filter_and_summary_reorg_fields(self) -> None: + text = OPENAPI_PATH.read_text(encoding="utf-8") + self.assertIn("- name: chain_state\n in: query", text) + self.assertIn("enum: [active, all, reversed]", text) + self.assertIn('- $ref: "#/components/schemas/BlocksResponse"', text) + self.assertIn('- $ref: "#/components/schemas/BlocksChainStateResponse"', text) + self.assertIn("ChainStateBlockRow:", text) + self.assertIn("const: prism.dashboard.blocks.v2", text) + self.assertIn("ChainState:", text) + self.assertIn("- disconnected_at", text) + self.assertIn("- blocks_reversed_total", text) + self.assertIn("- blocks_inactive_total", text) + # network.hashrate_ths is required-but-nullable and documents the + # credited-work vs chain-estimate caveat. + self.assertIn("- hashrate_ths\n - initial_block_download", text) + self.assertIn("ratio is approximate", text) + + readme_text = (CONTRACT_DIR / "README.md").read_text(encoding="utf-8") + self.assertIn("`chain_state=all`", readme_text) + self.assertIn("`chain_state=reversed`", readme_text) + self.assertIn("prism.dashboard.blocks.v2", readme_text) + self.assertIn("blocks_reversed_total", readme_text) + self.assertIn("blocks_inactive_total", readme_text) + self.assertIn("network.hashrate_ths", readme_text) def test_miner_summary_embeds_only_bounded_previews(self) -> None: miner = self.load_fixture("miner.json") @@ -430,6 +746,10 @@ def is_decimal_key(self, key: str, path: tuple[str, ...] = ()) -> bool: def is_hex_hash_key(self, key: str) -> bool: return key in HEX_HASH_KEYS or key.endswith("_sha256") or key.endswith("_txid") + def epoch(self, timestamp: str) -> int: + self.assertTrue(timestamp.endswith("Z"), timestamp) + return int(datetime.fromisoformat(timestamp.removesuffix("Z") + "+00:00").timestamp()) + def assert_iso_timestamp(self, value: object, source: str, path: tuple[str, ...]) -> None: self.assertIsInstance(value, str, ".".join(path)) text = str(value) diff --git a/tests/test_qbit_image_runtime_smoke.py b/tests/test_qbit_image_runtime_smoke.py new file mode 100644 index 000000000..692bbb4e9 --- /dev/null +++ b/tests/test_qbit_image_runtime_smoke.py @@ -0,0 +1,444 @@ +#!/usr/bin/env python3 +"""Contract tests for the real-qbitd runtime image smoke script. + +The smoke itself only ever runs against a freshly built qbitd image in CI, so +these tests drive it against a deterministic stand-in for the docker CLI. That +keeps the assertions it makes -- in particular the ``getblockheader.bits`` +contract block-candidate collapse depends on -- from silently disappearing +without a red test. +""" + +from __future__ import annotations + +import json +import os +import subprocess +import sys +import tempfile +import unittest +from pathlib import Path + + +ROOT = Path(__file__).resolve().parents[1] +SCRIPT = ROOT / ".github" / "scripts" / "test-qbit-image-runtime.sh" +CI_WORKFLOW = ROOT / ".github" / "workflows" / "ci.yml" +IMAGE = "qbit-runtime-smoke-fixture:test" +GENESIS_HASH = "0f9188f13cb7b2c71f2a335e3a4fc328bf5beb436012afca590b1a11466e2206" + +sys.path.insert(0, str(ROOT)) + +from lab.prism.block_candidates import _collapse_scaled_difficulty # noqa: E402 +from lab.prism.template_artifacts import scaled_network_difficulty # noqa: E402 + + +# A stand-in for the docker CLI covering exactly the subcommands the smoke +# issues. Anything else exits 97 so an unmodelled call fails loudly instead of +# being mistaken for a passing smoke. +FAKE_DOCKER = r'''#!/usr/bin/env python3 +from __future__ import annotations + +import json +import os +import sys +from pathlib import Path + +STATE = Path(os.environ["FAKE_DOCKER_STATE"]) +IMAGE = os.environ["QBIT_IMAGE"] +ENTRYPOINT = '["/usr/bin/tini","--","/usr/local/bin/qbit-entrypoint.sh"]' + + +def record_path(name): + return STATE / ("container-" + name + ".json") + + +def load(name): + path = record_path(name) + if not path.exists(): + return None + return json.loads(path.read_text(encoding="utf-8")) + + +def save(name, record): + record_path(name).write_text(json.dumps(record), encoding="utf-8") + + +def out(text): + sys.stdout.write(text if text.endswith("\n") else text + "\n") + + +def run(argv): + args = argv[1:] + if "--rm" in args: + tail = args[args.index("--rm") + 1:] + daemon_args = tail[1:] + if "-maxtipage=1" in daemon_args: + sys.stderr.write( + "qbit-entrypoint: caller-provided -maxtipage=1 is not allowed\n" + ) + return 1 + if "-version" in daemon_args: + out("qbit daemon version v0.0.0-fixture") + return 0 + + name = args[args.index("--name") + 1] + env = {} + index = 0 + while index < len(args): + if args[index] == "--env": + key, _, value = args[index + 1].partition("=") + env[key] = value + index += 2 + continue + index += 1 + daemon_args = args[args.index(IMAGE) + 1:] + save( + name, + { + "env": env, + "args": daemon_args, + "chain": "regtest" if "-regtest" in daemon_args else "main", + "running": True, + "exit_code": 0, + }, + ) + out("f" * 64) + return 0 + + +def exec_(argv): + name = argv[1] + record = load(name) + if record is None or not record["running"]: + return 1 + rest = argv[2:] + if rest[0] == "sh": + lines = ["qbitd"] + list(record["args"]) + if record["env"].get("QBIT_MAINNET_LAUNCH_READINESS_CHECKS_ENABLED") == "0": + lines.append( + "-maxtipage=" + + record["env"]["QBIT_MAINNET_PRELAUNCH_MAX_TIP_AGE_SECONDS"] + ) + out("\n".join(lines)) + return 0 + if rest[0] != "qbit-cli": + return 97 + positional = [token for token in rest[1:] if not token.startswith("-")] + method, params = positional[0], positional[1:] + if method == "getblockchaininfo": + out(json.dumps({"chain": record["chain"], "blocks": 0}, indent=2)) + return 0 + if method == "getbestblockhash": + out(os.environ["FAKE_DOCKER_BEST_HASH"]) + return 0 + if method == "getblockheader": + header = json.loads(os.environ["FAKE_DOCKER_HEADER"]) + header["hash"] = params[0] + out(json.dumps(header, indent=2)) + return 0 + return 97 + + +def container(argv): + if argv[1] != "inspect": + return 97 + rest = argv[2:] + template = None + if rest[0] == "--format": + template = rest[1] + rest = rest[2:] + record = load(rest[0]) + if record is None: + sys.stderr.write("Error: No such container: " + rest[0] + "\n") + return 1 + if template is None: + out("[]") + elif "Running" in template: + out("true" if record["running"] else "false") + elif "ExitCode" in template: + out(str(record["exit_code"])) + else: + return 97 + return 0 + + +def main(argv): + with (STATE / "commands.log").open("a", encoding="utf-8") as handle: + handle.write(json.dumps(argv) + "\n") + command = argv[0] + if command == "image": + if argv[1] != "inspect": + return 97 + if "--format" in argv: + out(ENTRYPOINT) + else: + out("[]") + return 0 + if command == "run": + return run(argv) + if command == "exec": + return exec_(argv) + if command == "container": + return container(argv) + if command == "top": + if load(argv[1]) is None: + return 1 + out("PID COMMAND\n1 tini\n7 qbitd") + return 0 + if command == "stop": + name = argv[-1] + record = load(name) + if record is None: + return 1 + record["running"] = False + record["exit_code"] = int(os.environ.get("FAKE_DOCKER_EXIT_CODE", "0")) + save(name, record) + out(name) + return 0 + if command == "rm": + path = record_path(argv[-1]) + if not path.exists(): + return 1 + path.unlink() + out(argv[-1]) + return 0 + if command == "logs": + return 0 + return 97 + + +sys.exit(main(sys.argv[1:])) +''' + + +class QbitImageRuntimeSmokeTests(unittest.TestCase): + def setUp(self) -> None: + self.temp_dir = tempfile.TemporaryDirectory() + self.addCleanup(self.temp_dir.cleanup) + root = Path(self.temp_dir.name) + self.state = root / "state" + self.state.mkdir() + bin_dir = root / "bin" + bin_dir.mkdir() + fake_docker = bin_dir / "docker" + fake_docker.write_text(FAKE_DOCKER, encoding="utf-8") + fake_docker.chmod(0o755) + self.bin_dir = bin_dir + + def run_smoke( + self, + *, + bits: object = "207fffff", + header: dict[str, object] | None = None, + best_hash: str = GENESIS_HASH, + ) -> subprocess.CompletedProcess[str]: + if header is None: + header = {"height": 0, "version": 1} + if bits is not None: + header["bits"] = bits + env = os.environ.copy() + env.update( + { + "PATH": f"{self.bin_dir}{os.pathsep}{env['PATH']}", + "QBIT_IMAGE": IMAGE, + "FAKE_DOCKER_STATE": str(self.state), + "FAKE_DOCKER_HEADER": json.dumps(header), + "FAKE_DOCKER_BEST_HASH": best_hash, + } + ) + env.pop("QBIT_CI_IMAGE", None) + return subprocess.run( + ["bash", str(SCRIPT)], + env=env, + capture_output=True, + text=True, + timeout=120, + cwd=self.temp_dir.name, + ) + + def docker_commands(self) -> list[list[str]]: + log = self.state / "commands.log" + if not log.exists(): + return [] + return [ + json.loads(line) + for line in log.read_text(encoding="utf-8").splitlines() + if line + ] + + def assert_smoke_failed( + self, result: subprocess.CompletedProcess[str], needle: str + ) -> None: + self.assertNotEqual(result.returncode, 0, result.stdout + result.stderr) + self.assertNotIn("smoke: PASS", result.stdout) + self.assertIn("qbit runtime image smoke: FAIL", result.stderr) + self.assertIn(needle, result.stderr) + + def test_smoke_passes_and_reports_production_scaled_difficulty(self) -> None: + result = self.run_smoke(bits="207fffff") + + self.assertEqual(result.returncode, 0, result.stdout + result.stderr) + self.assertIn("qbit runtime image smoke: PASS", result.stdout) + expected = _collapse_scaled_difficulty("207fffff") + self.assertIsNotNone(expected) + self.assertIn( + f"bits 207fffff scaled difficulty {expected}", + result.stdout, + ) + + def test_smoke_reads_the_header_of_the_running_regtest_container(self) -> None: + result = self.run_smoke() + self.assertEqual(result.returncode, 0, result.stdout + result.stderr) + + commands = self.docker_commands() + header_reads = [ + command + for command in commands + if command[0] == "exec" and "getblockheader" in command + ] + self.assertEqual(len(header_reads), 1, commands) + (header_read,) = header_reads + self.assertTrue(header_read[1].endswith("-regtest"), header_read) + self.assertIn(GENESIS_HASH, header_read) + # The header must be read while the node is up: its stop follows. + stop_index = next( + index + for index, command in enumerate(commands) + if command[0] == "stop" and command[-1] == header_read[1] + ) + self.assertLess(commands.index(header_read), stop_index) + + def test_smoke_derives_mainnet_scale_difficulty_without_shell_overflow(self) -> None: + # 1b0404cb scales past 2**63, where bash's signed arithmetic would wrap + # negative. The reported value must still be the production integer. + result = self.run_smoke(bits="1b0404cb") + + self.assertEqual(result.returncode, 0, result.stdout + result.stderr) + expected = scaled_network_difficulty("1b0404cb") + self.assertGreater(expected, 2**63) + self.assertEqual(expected, _collapse_scaled_difficulty("1b0404cb")) + self.assertIn(f"bits 1b0404cb scaled difficulty {expected}", result.stdout) + + def test_smoke_fails_when_header_bits_are_too_short(self) -> None: + self.assert_smoke_failed( + self.run_smoke(bits="7fffff"), "is not eight hex characters: 7fffff" + ) + + def test_smoke_fails_when_header_bits_are_too_long(self) -> None: + self.assert_smoke_failed( + self.run_smoke(bits="207fffff0"), "is not eight hex characters: 207fffff0" + ) + + def test_smoke_fails_when_header_bits_are_not_hexadecimal(self) -> None: + self.assert_smoke_failed( + self.run_smoke(bits="207fzzzz"), "is not eight hex characters: 207fzzzz" + ) + + def test_smoke_fails_when_header_omits_bits(self) -> None: + self.assert_smoke_failed( + self.run_smoke(bits=None), "is not eight hex characters: " + ) + + def test_smoke_fails_when_header_bits_are_not_a_string(self) -> None: + self.assert_smoke_failed( + self.run_smoke(header={"height": 0, "bits": 545259519}), + "is not eight hex characters: ", + ) + + def test_smoke_fails_when_header_bits_carry_a_zero_target(self) -> None: + # Eight hex characters, but PRISM derives no positive work from them, + # which is exactly the shape collapse fails its page closed on. + result = self.run_smoke(bits="00000000") + + self.assertIsNone(_collapse_scaled_difficulty("00000000")) + self.assert_smoke_failed(result, "did not derive PRISM's scaled difficulty") + self.assertIn("collapse rejected the compact bits", result.stderr) + + def test_smoke_fails_when_best_block_hash_is_not_a_block_hash(self) -> None: + self.assert_smoke_failed( + self.run_smoke(best_hash="not-a-block-hash"), + "getbestblockhash is not a block hash", + ) + + def test_smoke_preserves_the_existing_lifecycle_checks(self) -> None: + result = self.run_smoke() + self.assertEqual(result.returncode, 0, result.stdout + result.stderr) + + commands = self.docker_commands() + self.assertTrue( + any(command[0] == "top" for command in commands), + "process-chain check disappeared", + ) + self.assertTrue( + any( + command[0] == "run" and "-maxtipage=1" in command + for command in commands + ), + "caller-provided -maxtipage rejection check disappeared", + ) + argv_probes = [ + command + for command in commands + if command[0] == "exec" and "sh" in command + ] + self.assertEqual(len(argv_probes), 2, commands) + started = [ + command[command.index("--name") + 1] + for command in commands + if command[0] == "run" and "--name" in command + ] + self.assertEqual(len(started), 3, commands) + for name in started: + self.assertTrue( + any(command[0] == "rm" and command[-1] == name for command in commands), + f"{name} was never removed after a clean exit", + ) + + def test_smoke_fails_when_a_container_exits_uncleanly(self) -> None: + env_backup = os.environ.get("FAKE_DOCKER_EXIT_CODE") + os.environ["FAKE_DOCKER_EXIT_CODE"] = "3" + try: + result = self.run_smoke() + finally: + if env_backup is None: + os.environ.pop("FAKE_DOCKER_EXIT_CODE", None) + else: + os.environ["FAKE_DOCKER_EXIT_CODE"] = env_backup + self.assert_smoke_failed(result, "exited with status 3 after SIGTERM") + + +class QbitImageRuntimeSmokeSourceTests(unittest.TestCase): + """Static guards keeping the assertion wired up and formula-free.""" + + def setUp(self) -> None: + self.script = SCRIPT.read_text(encoding="utf-8") + + def test_regtest_container_is_asserted_against_the_header_contract(self) -> None: + self.assertIn( + 'assert_header_bits_contract "${REGTEST_CONTAINER}" -regtest', + self.script, + ) + + def test_scaled_difficulty_comes_from_the_production_helpers(self) -> None: + self.assertIn( + "from lab.prism.block_candidates import _collapse_scaled_difficulty", + self.script, + ) + self.assertIn( + "from lab.prism.template_artifacts import scaled_network_difficulty", + self.script, + ) + # The scale and the powLimit belong to the production helpers. A copy + # of either here could drift away from collapse with no test noticing. + for constant in ("1000000", "1_000_000", "207fffff"): + with self.subTest(constant=constant): + self.assertNotIn(constant, self.script) + + def test_ci_runs_the_smoke_with_an_interpreter_for_the_helpers(self) -> None: + workflow = CI_WORKFLOW.read_text(encoding="utf-8") + job = workflow.split("Build and smoke real qbitd image", 1)[1] + self.assertIn("bash .github/scripts/test-qbit-image-runtime.sh", job) + self.assertIn("uses: actions/setup-python@v6", job) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_run_python_test_shard.py b/tests/test_run_python_test_shard.py new file mode 100644 index 000000000..584f1154a --- /dev/null +++ b/tests/test_run_python_test_shard.py @@ -0,0 +1,110 @@ +from __future__ import annotations + +import contextlib +import io +from pathlib import Path +import tempfile +import unittest +from unittest import mock + +from scripts.run_python_test_shard import main, select_shard + + +def test_ids(suite): + for test in suite: + if isinstance(test, unittest.TestSuite): + yield from test_ids(test) + else: + yield test.id() + + +class PythonTestShardTests(unittest.TestCase): + def fixture_suite(self, directory): + for index in range(7): + Path(directory, f"test_shard_fixture_{index}.py").write_text( + "import unittest\n" + "active = False\n" + "def setUpModule():\n" + " global active\n" + " active = True\n" + "def tearDownModule():\n" + " global active\n" + " active = False\n" + "class Fixture(unittest.TestCase):\n" + " @classmethod\n" + " def setUpClass(cls):\n" + " cls.ready = True\n" + " def test_a(self):\n" + " self.assertTrue(active and self.ready)\n" + " def test_b(self):\n" + " self.assertTrue(active and self.ready)\n" + ) + return unittest.TestLoader().discover(directory) + + def test_every_discovered_test_runs_once_with_module_fixtures_intact(self): + with tempfile.TemporaryDirectory() as directory: + suite = self.fixture_suite(directory) + expected = list(test_ids(suite)) + shards = [select_shard(suite, index, 4) for index in range(4)] + rediscovered = unittest.TestLoader().discover(directory) + self.assertEqual( + [list(test_ids(shard)) for shard in shards], + [list(test_ids(select_shard(rediscovered, i, 4))) for i in range(4)], + ) + actual = [test_id for shard in shards for test_id in test_ids(shard)] + self.assertEqual(len(expected), 14) + self.assertCountEqual(actual, expected) + module_shards = {} + for index, shard in enumerate(shards): + for test_id in test_ids(shard): + module = test_id.split(".")[0] + self.assertEqual(module_shards.setdefault(module, index), index) + result = unittest.TextTestRunner(stream=io.StringIO()).run(shard) + self.assertTrue(result.wasSuccessful()) + + def test_one_shard_preserves_discovery_order(self): + suite = unittest.defaultTestLoader.loadTestsFromTestCase(type(self)) + self.assertEqual(list(test_ids(select_shard(suite, 0, 1))), list(test_ids(suite))) + + def test_invalid_coordinates_and_empty_shards_fail_closed(self): + suite = unittest.defaultTestLoader.loadTestsFromTestCase(type(self)) + for index, count in [(-1, 4), (4, 4), (0, 0), (0, -1)]: + with self.subTest(index=index, count=count), self.assertRaises(ValueError): + select_shard(suite, index, count) + with self.assertRaises(ValueError): + select_shard(unittest.TestSuite(), 0, 1) + with self.assertRaises(ValueError): + select_shard(suite, suite.countTestCases(), suite.countTestCases() + 1) + + def test_discovery_import_errors_fail_the_assigned_shard(self): + with tempfile.TemporaryDirectory() as directory: + Path(directory, "test_broken_shard_fixture.py").write_text( + "raise ImportError('broken fixture dependency')\n" + ) + suite = unittest.TestLoader().discover(directory) + selected = select_shard(suite, 0, 1) + result = unittest.TextTestRunner(stream=io.StringIO()).run(selected) + self.assertFalse(result.wasSuccessful()) + self.assertEqual(len(result.errors), 1) + + def test_runner_exit_status_tracks_test_result(self): + for passes in [True, False]: + with self.subTest(passes=passes): + class Fixture(unittest.TestCase): + def runTest(self): + self.assertTrue(passes) + + suite = unittest.TestSuite([unittest.TestSuite([Fixture()])]) + with ( + mock.patch.object(unittest.TestLoader, "discover", return_value=suite), + contextlib.redirect_stdout(io.StringIO()), + contextlib.redirect_stderr(io.StringIO()), + ): + self.assertEqual( + main(["--shard-index", "0", "--shard-count", "1"]), + 0 if passes else 1, + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_window_paging_allocation_harness.py b/tests/test_window_paging_allocation_harness.py new file mode 100644 index 000000000..15331016c --- /dev/null +++ b/tests/test_window_paging_allocation_harness.py @@ -0,0 +1,337 @@ +#!/usr/bin/env python3 +"""Bounded safety tests for ``tests/perf/window_paging_allocation.py``. + +The measurement driver talks to a disposable ``--serve`` daemon over blocking +pipes. A stalled daemon must never hang a benchmark: every exchange, the +handshake included, runs under a wall-clock deadline that kills and reaps the +child and records the failure, and every exit path closes the pipes and the +stderr capture. These tests exercise that with a fake daemon that stalls in +three ways -- no handshake, after consuming the request, and without ever +reading it (so the request writer is blocked on a full pipe) -- and stay +well under a few seconds each. They also pin the provenance of the reported +peak resident set: every observation made before process exit, including +kernel ``VmHWM`` before a kill or normal close, is a lower bound rendered +with ``≥``; nothing observed is ``unavailable``. +""" + +from __future__ import annotations + +import os +import io +import json +import stat +import tempfile +import time +import unittest +from pathlib import Path +from contextlib import redirect_stdout +from unittest import mock + +from tests.perf import window_paging_allocation as harness + + +FAKE_DAEMON = """#!/usr/bin/env python3 +import json, os, sys, time +mode = os.environ.get("FAKE_DAEMON_MODE", "stall_after_handshake") +if mode != "no_handshake": + sys.stdout.write(json.dumps({ + "event": "handshake", + "tool": "qbit-prism-build-audit-bundle", + "protocol": %d, + }) + "\\n") + sys.stdout.flush() +if mode == "exit_after_handshake": + sys.exit(0) +if mode == "stall_after_handshake": + sys.stdin.readline() +time.sleep(120) +""" % harness.PRISM_SERVE_BUILDER_PROTOCOL_VERSION + + +def _assert_reaped(test: unittest.TestCase, pid: int) -> None: + try: + os.kill(pid, 0) + except ProcessLookupError: + return + except PermissionError: # pragma: no cover - pid reused by another user + return + test.fail(f"fake daemon pid {pid} is still present after close()") + + +class StalledDaemonTests(unittest.TestCase): + @classmethod + def setUpClass(cls) -> None: + cls.tmp = tempfile.TemporaryDirectory() + cls.binary = Path(cls.tmp.name) / "fake-daemon" + cls.binary.write_text(FAKE_DAEMON) + cls.binary.chmod(cls.binary.stat().st_mode | stat.S_IXUSR) + + @classmethod + def tearDownClass(cls) -> None: + cls.tmp.cleanup() + + def _daemon(self, mode: str, timeout: float = 0.5) -> harness.Daemon: + os.environ["FAKE_DAEMON_MODE"] = mode + try: + return harness.Daemon( + self.binary, + memory_limit_mb=None, + stderr_path=Path(self.tmp.name) / f"{mode}.stderr", + exchange_timeout=timeout, + shutdown_timeout=2.0, + ) + finally: + os.environ.pop("FAKE_DAEMON_MODE", None) + + def test_exchange_that_stalls_after_reading_times_out_and_is_reaped(self) -> None: + daemon = self._daemon("stall_after_handshake") + started = time.perf_counter() + exchange = daemon.exchange(b'{"request":"prepare_window"}\n', label="full") + elapsed = time.perf_counter() - started + self.assertLess(elapsed, 5.0) + self.assertIsNone(exchange.envelope) + self.assertIn("full exceeded 0.5s; daemon killed", exchange.error or "") + outcome = daemon.close() + self.assertEqual(outcome["timed_out"], "full") + self.assertEqual(outcome["signal"], 9) + # The watchdog sampled the kernel high-water mark before killing. + if Path("/proc/self/status").exists(): + self.assertEqual(outcome["peak_rss_source"], "vmhwm_before_kill") + self.assertIsInstance(outcome["peak_rss_mb"], float) + else: + self.assertEqual(outcome["peak_rss_source"], "unavailable") + self.assertIsNone(outcome["peak_rss_mb"]) + self.assertTrue(daemon.process.stdin.closed) + self.assertTrue(daemon.process.stdout.closed) + self.assertTrue(daemon._stderr.closed) + _assert_reaped(self, outcome["pid"]) + # Idempotent, and a later exchange never touches the dead child. + self.assertIs(daemon.close(), outcome) + self.assertEqual(daemon.exchange(b"{}\n").error, "daemon is not running") + + def test_stall_without_reading_unblocks_the_request_writer(self) -> None: + daemon = self._daemon("stall_without_reading") + request = b'{"records":"' + b"x" * (4 << 20) + b'"}\n' # far beyond one pipe capacity + started = time.perf_counter() + exchange = daemon.exchange(request, label="full") + elapsed = time.perf_counter() - started + self.assertLess(elapsed, 5.0) + self.assertIn("daemon killed", exchange.error or "") + self.assertIn("write:", exchange.error or "") + outcome = daemon.close() + self.assertEqual(outcome["signal"], 9) + _assert_reaped(self, outcome["pid"]) + + def test_missing_handshake_times_out_and_is_reaped(self) -> None: + started = time.perf_counter() + with self.assertRaises(harness.DaemonTimeout) as caught: + self._daemon("no_handshake") + self.assertLess(time.perf_counter() - started, 5.0) + outcome = caught.exception.daemon_outcome # type: ignore[attr-defined] + self.assertEqual(outcome["timed_out"], "handshake") + self.assertEqual(outcome["signal"], 9) + _assert_reaped(self, outcome["pid"]) + + def test_run_variant_records_the_timeout_and_reaps(self) -> None: + fixture = harness.build_fixture(64, miners=2, page_size=16, small=2, large=3) + os.environ["FAKE_DAEMON_MODE"] = "stall_after_handshake" + try: + started = time.perf_counter() + result = harness.run_variant( + "fake", + self.binary, + fixture, + oracle=None, + memory_limit_mb=None, + memory_margin_mb=0, + sample_interval=0.05, + exchange_timeout=0.5, + stderr_dir=Path(self.tmp.name), + log=lambda _text: None, + ) + finally: + os.environ.pop("FAKE_DAEMON_MODE", None) + self.assertLess(time.perf_counter() - started, 10.0) + self.assertIsNone(result.skipped) + self.assertEqual([phase.phase for phase in result.phases], ["full"]) + self.assertEqual(result.phases[0].status, "timeout") + self.assertIsNone(result.phases[0].residual_wait_seconds) + self.assertEqual(result.outcome["timed_out"], "full") + self.assertEqual(result.outcome["signal"], 9) + _assert_reaped(self, result.outcome["pid"]) + # The tables render a killed run rather than choking on it. + self.assertIn("killed on full timeout", harness.render_summary([result])) + self.assertIn("| timeout", harness.render_runs([result])) + + def test_main_skips_before_parent_allocations_when_memory_is_low(self) -> None: + fixture = harness.build_fixture(64, miners=2, page_size=16, small=2, large=3) + for available, stage, builds in ( + ([0], "fixture construction", 0), + ([10000, 0], "Python oracle", 1), + ): + with self.subTest(stage=stage), \ + mock.patch.object(harness, "read_meminfo_mb", side_effect=available), \ + mock.patch.object(harness, "machine_info", return_value={}), \ + mock.patch.object(harness, "git_describe", return_value="test"), \ + mock.patch.object(harness, "build_fixture", return_value=fixture) as build, \ + mock.patch.object(harness, "python_oracle") as oracle, \ + mock.patch.object(harness, "run_variant") as variant, \ + redirect_stdout(io.StringIO()): + output = Path(self.tmp.name) / "low-memory.json" + self.assertEqual(harness.main([ + "--daemon-binary", str(self.binary), "--sizes", "400000", + "--stderr-dir", self.tmp.name, "--json", str(output), "--quiet", + ]), 0) + result = json.loads(output.read_text()) + self.assertEqual(build.call_count, builds) + oracle.assert_not_called() + variant.assert_not_called() + self.assertEqual(result["oracles"], {}) + self.assertEqual(len(result["runs"]), 1) + self.assertEqual(result["runs"][0]["size"], 400000) + self.assertIn(f"before {stage}", result["runs"][0]["skipped"]) + + def test_handshake_timeout_preserves_outcome_in_result_and_summary(self) -> None: + fixture = harness.build_fixture(64, miners=2, page_size=16, small=2, large=3) + os.environ["FAKE_DAEMON_MODE"] = "no_handshake" + try: + result = harness.run_variant( + "fake", self.binary, fixture, oracle=None, + memory_limit_mb=None, memory_margin_mb=0, + sample_interval=0.01, exchange_timeout=0.5, + stderr_dir=Path(self.tmp.name), log=lambda _text: None, + ) + finally: + os.environ.pop("FAKE_DAEMON_MODE", None) + self.assertEqual(result.phases, []) + self.assertIn("daemon failed to start", result.error or "") + self.assertEqual(result.outcome["timed_out"], "handshake") + self.assertEqual(result.outcome["signal"], 9) + self.assertIn("stderr_tail", result.outcome) + self.assertIn("peak_rss_source", result.outcome) + _assert_reaped(self, result.outcome["pid"]) + summary = harness.render_summary([result]) + self.assertIn("signal 9, killed on handshake timeout", summary) + self.assertIn("failed: daemon failed to start", summary) + self.assertIn(harness.peak_source_label(result.outcome["peak_rss_source"]), summary) + + def test_close_on_a_live_daemon_reports_the_kernel_high_water_mark(self) -> None: + daemon = self._daemon("stall_after_handshake") + outcome = daemon.close() # alive at close: VmHWM read live, then killed + if Path("/proc/self/status").exists(): + self.assertEqual(outcome["peak_rss_source"], "vmhwm") + self.assertIsInstance(outcome["peak_rss_mb"], float) + else: + self.assertEqual(outcome["peak_rss_source"], "unavailable") + self.assertIsNone(outcome["peak_rss_mb"]) + self.assertEqual(outcome["signal"], 9) + _assert_reaped(self, outcome["pid"]) + + def test_self_exited_daemon_peak_is_a_lower_bound_or_unavailable(self) -> None: + daemon = self._daemon("exit_after_handshake") + daemon.process.wait(timeout=5.0) + outcome = daemon.close() + self.assertEqual(outcome["exit_code"], 0) + self.assertEqual(outcome["peak_rss_source"], "unavailable") + self.assertIsNone(outcome["peak_rss_mb"]) + _assert_reaped(self, outcome["pid"]) + + fixture = harness.build_fixture(64, miners=2, page_size=16, small=2, large=3) + os.environ["FAKE_DAEMON_MODE"] = "exit_after_handshake" + try: + result = harness.run_variant( + "fake", + self.binary, + fixture, + oracle=None, + memory_limit_mb=None, + memory_margin_mb=0, + sample_interval=0.01, + exchange_timeout=5.0, + stderr_dir=Path(self.tmp.name), + log=lambda _text: None, + ) + finally: + os.environ.pop("FAKE_DAEMON_MODE", None) + self.assertEqual(result.outcome["exit_code"], 0) + self.assertEqual(result.phases[0].status, "no_response") + # Whatever the sampler caught before the exit, the figure is never + # presented as a kernel high-water mark. + source = result.outcome["peak_rss_source"] + self.assertIn(source, ("lower_bound", "unavailable")) + summary = harness.render_summary([result]) + if source == "lower_bound": + self.assertIsInstance(result.outcome["peak_rss_mb"], float) + self.assertIn("| ≥ ", summary) + self.assertIn("lower bound", summary) + else: + self.assertIsNone(result.outcome["peak_rss_mb"]) + self.assertIn("| - | unavailable |", summary) + self.assertNotIn("kernel VmHWM", summary) + + def test_resolve_peak_rss_provenance(self) -> None: + exact = harness.resolve_peak_rss({"peak_rss_mb": 601.0, "peak_rss_source": "vmhwm"}, [900.0]) + self.assertEqual((exact["peak_rss_mb"], exact["peak_rss_source"]), (601.0, "vmhwm")) + before_kill = harness.resolve_peak_rss( + {"peak_rss_mb": 601.0, "peak_rss_source": "vmhwm_before_kill"}, [900.0] + ) + self.assertEqual(before_kill["peak_rss_source"], "vmhwm_before_kill") + legacy = harness.resolve_peak_rss({"peak_rss_mb": 601.0}, []) + self.assertEqual(legacy["peak_rss_source"], "vmhwm") + bound = harness.resolve_peak_rss({"peak_rss_mb": None}, [435.0, None, 0.0, 601.0, 12.5]) + self.assertEqual((bound["peak_rss_mb"], bound["peak_rss_source"]), (601.0, "lower_bound")) + nothing = harness.resolve_peak_rss({"peak_rss_mb": None}, [None, 0.0]) + self.assertEqual((nothing["peak_rss_mb"], nothing["peak_rss_source"]), (None, "unavailable")) + + def test_peak_rendering_marks_all_live_observations_as_lower_bounds(self) -> None: + self.assertEqual(harness.format_peak(601.4, "vmhwm"), "≥ 601") + self.assertEqual(harness.format_peak(601.4, "vmhwm_before_kill"), "≥ 601") + self.assertEqual(harness.format_peak(601.4, "lower_bound"), "≥ 601") + self.assertEqual(harness.format_peak(2.857, "lower_bound", 2), "≥ 2.86") + self.assertEqual(harness.format_peak(None, "unavailable"), "-") + + def run(source: str | None, peak: float | None) -> harness.RunResult: + return harness.RunResult( + "fixed", "bin", 210_000, 512, 16_000.0, None, 6144, + [ + harness.PhaseResult( + phase="full", records_sent=210_000, request_bytes=1, + request_write_seconds=0.15, response_wait_seconds=1.29, + response_read_seconds=0.12, daemon_metrics={"fold_seconds": 0.377}, + status="prepared", + ) + ], + {"exit_code": 0, "signal": None, "timed_out": None, "peak_rss_mb": peak, + "peak_rss_source": source, "peak_vsize_mb": 640.0}, + 0.0, [], + ) + + exact_row = harness.render_summary([run("vmhwm", 601.0)]).splitlines()[-1] + self.assertIn("| ≥ 601 | lower bound (kernel VmHWM at close) | ≥ 640 | ≥ 2.86 |", exact_row) + kill_row = harness.render_summary([run("vmhwm_before_kill", 601.0)]).splitlines()[-1] + self.assertIn("| ≥ 601 | lower bound (kernel VmHWM before watchdog kill) | ≥ 640 | ≥ 2.86 |", kill_row) + bound_row = harness.render_summary([run("lower_bound", 601.0)]).splitlines()[-1] + self.assertIn("| ≥ 601 | lower bound (highest earlier VmHWM/RSS observation) | ≥ 640 | ≥ 2.86 |", bound_row) + none_row = harness.render_summary([run("unavailable", None)]).splitlines()[-1] + self.assertIn("| - | unavailable | ≥ 640 | - |", none_row) + header = harness.render_summary([]).splitlines()[0] + self.assertIn("| peak RSS MiB | peak source |", header) + self.assertNotIn("VmHWM", header) + + def test_residual_wait_is_an_approximation_clamped_at_zero(self) -> None: + phase = harness.PhaseResult( + phase="full", + records_sent=1, + request_bytes=1, + request_write_seconds=0.0, + response_wait_seconds=1.0, + response_read_seconds=0.0, + daemon_metrics={"input_deserialization_seconds": 0.4, "fold_seconds": 0.7, "output_serialization_seconds": 0.2}, + ) + self.assertEqual(phase.residual_wait_seconds, 0.0) + phase.daemon_metrics = {} + self.assertIsNone(phase.residual_wait_seconds) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_window_pipeline_parity.py b/tests/test_window_pipeline_parity.py new file mode 100644 index 000000000..5ef8c3ac3 --- /dev/null +++ b/tests/test_window_pipeline_parity.py @@ -0,0 +1,1155 @@ +#!/usr/bin/env python3 +"""Hold the window pipeline to its frozen byte-exact reference. + +Why these tests exist: a second implementation of the payout-window pipeline +can only be adopted if it is byte-identical to the shipped Python one, and +that comparison is only trustworthy if (a) the shipped pipeline reproduces a +frozen reference committed at a known revision, (b) the corpus demonstrably +exercises the fold's edge cases rather than just its bulk behavior, and +(c) the harness has been *seen to fail* on known divergence classes. The +first two are the parity tests below; the third is the divergence-detection +suite, which drives deliberately perturbed backends -- wrong field order, +wrong digest framing, null-instead-of-absent, a dropped crossing row, spool +whitespace, narrow or saturating integers, a fold that never refuses or +misfiles its refusals, advance expiry with the wrong comparison or confined +to pre-existing pages, an equal-anchor rejection, a raw DEL or an escaped +solidus, advance stats derived from the wrong thing, and a backend that +declares Unsupported inside its own domain or returns wrong bytes outside it +-- and asserts the harness catches each one exactly where it should and +nowhere else. Green alone is not evidence here: every class from the +integer-width one onward was green against the original fourteen-case +corpus, which could not see it, and the width class is re-checked against +the retuned ``wide-integers`` case so the declared-domain split could not +have quietly taken that protection away. +""" + +from __future__ import annotations + +import json +import os +import unittest +from dataclasses import replace +from unittest import mock + +from lab.prism.bundle_compiler import _compact_share_payload +from lab.prism.share_ledger import ( + DEFAULT_INCREMENTAL_SHARE_WINDOW_PAGE_SIZE, + IncrementalShareWindow, + IncrementalWindowFallback, +) +from tests import window_pipeline_parity as parity + + +class WindowPipelineAdapterRegistryTests(unittest.TestCase): + def test_selection_defaults_to_the_shipped_python_pipeline(self) -> None: + with mock.patch.dict(os.environ): + os.environ.pop(parity.ADAPTER_ENV_VAR, None) + adapter = parity.resolve_adapter() + self.assertEqual(adapter.name, "python") + self.assertIsInstance(adapter, parity.PythonWindowPipelineAdapter) + + def test_environment_variable_selects_a_registered_backend(self) -> None: + class _RegisteredBackend: + name = "registry-selection-probe" + + def run(self, case: parity.WindowPipelineCase) -> parity.WindowPipelineResult: + raise NotImplementedError + + parity.register_adapter("registry-selection-probe", _RegisteredBackend) + try: + with mock.patch.dict( + os.environ, + {parity.ADAPTER_ENV_VAR: "registry-selection-probe"}, + ): + self.assertIsInstance(parity.resolve_adapter(), _RegisteredBackend) + finally: + parity._ADAPTER_FACTORIES.pop("registry-selection-probe", None) + + def test_duplicate_registration_is_rejected(self) -> None: + with self.assertRaisesRegex(ValueError, "already registered"): + parity.register_adapter("python", parity.PythonWindowPipelineAdapter) + + def test_unknown_backend_selection_is_a_clear_error(self) -> None: + with self.assertRaisesRegex( + ValueError, "unknown window-pipeline parity adapter" + ): + parity.resolve_adapter("no-such-backend") + + def test_rejection_vocabulary_is_closed(self) -> None: + # A backend cannot invent a category, and the vocabulary is exactly + # the two entry points' condition lists: a rejection the shipped + # pipeline cannot express cannot be frozen as parity either. + with self.assertRaisesRegex(ValueError, "unknown window-pipeline rejection reason"): + parity.WindowPipelineRejection("ValueError") + self.assertEqual( + parity.REJECTION_REASONS, + parity.FULL_SNAPSHOT_REJECTIONS + parity.ADVANCE_REJECTIONS, + ) + self.assertEqual(len(set(parity.REJECTION_REASONS)), 11) + + +class WindowPipelineParityOracleTests(unittest.TestCase): + """The selected backend must reproduce every frozen corpus case.""" + + corpus: tuple[parity.WindowPipelineCase, ...] + cases: dict[str, parity.WindowPipelineCase] + reference_cases: dict[str, dict[str, object]] + results: dict[str, parity.WindowPipelineResult] + + @classmethod + def setUpClass(cls) -> None: + cls.corpus = parity.build_corpus() + cls.cases = {case.name: case for case in cls.corpus} + cls.reference_cases = parity.load_reference_document()["cases"] + cls.adapter = parity.resolve_adapter() + cls.results = {case.name: cls.adapter.run(case) for case in cls.corpus} + + def _outputs(self, name: str) -> parity.WindowPipelineOutputs: + result = self.results[name] + self.assertIsInstance(result, parity.WindowPipelineOutputs, name) + assert isinstance(result, parity.WindowPipelineOutputs) + return result + + def _declared_unsupported(self, name: str) -> bool: + """Whether the backend legitimately declined this case. + + True only for an Unsupported result on a case outside the backend's + declared domain; the main parity test fails any other Unsupported, so + the output-shape tests may skip exactly these cases and no others. + """ + result = self.results[name] + if not isinstance(result, parity.WindowPipelineUnsupported): + return False + self.assertIsNotNone( + parity.domain_exclusion(self.adapter, self.cases[name]), + f"{name}: Unsupported inside the declared domain", + ) + return True + + def test_backend_reproduces_every_frozen_case(self) -> None: + self.assertEqual(set(self.cases), set(self.reference_cases)) + for name, case in self.cases.items(): + with self.subTest(case=name): + entry = self.reference_cases[name] + self.assertEqual( + parity.case_input_sha256(case), + entry["input_sha256"], + "corpus inputs drifted from the ones the reference froze", + ) + pinned_input = (entry.get("pinned_literals") or {}).get("input") + if pinned_input is not None: + # The literal makes input drift legible in the fixture + # diff; here it also localizes which field moved. + self.assertEqual(parity.case_input_document(case), pinned_input) + self.assertEqual( + parity.diverging_outputs( + entry, + self.results[name], + domain_exclusion=parity.domain_exclusion(self.adapter, case), + ), + [], + ) + + def test_unsupported_is_declared_only_outside_the_declared_domain(self) -> None: + # Every Unsupported the selected backend returned names a case its + # declared domain excludes, the reference still holds Python's bytes + # for it, and the shipped pipeline itself declines nothing. + for name in self.cases: + result = self.results[name] + if not isinstance(result, parity.WindowPipelineUnsupported): + continue + with self.subTest(case=name): + self.assertTrue(result.reason) + self.assertIsNotNone(parity.domain_exclusion(self.adapter, self.cases[name])) + self.assertIn("record_count", self.reference_cases[name]) + if isinstance(self.adapter, parity.PythonWindowPipelineAdapter): + self.assertFalse( + any(isinstance(result, parity.WindowPipelineUnsupported) for result in self.results.values()) + ) + + def test_outputs_are_internally_consistent(self) -> None: + # The digest must hash exactly the framed canonical bytes, and the + # page-fragment concatenation must equal the flat per-record encoding + # -- the identity that makes page layout invisible on the wire. + for name, case in self.cases.items(): + if case.expected_rejection is not None or self._declared_unsupported(name): + continue + with self.subTest(case=name): + outputs = self._outputs(name) + self.assertEqual( + parity.sha256_hex(outputs.canonical_bytes), + outputs.canonical_digest, + ) + self.assertEqual( + b"[" + b",".join(outputs.record_jsons) + b"]", + outputs.canonical_bytes, + ) + self.assertEqual(len(outputs.advance_stats), len(case.advances)) + + def test_incremental_advances_land_on_full_rebuild_bytes(self) -> None: + # advance() folding deltas must produce the same byte outputs as + # from_full_snapshot over the union at the final anchor. The rebuild + # side always runs the shipped Python pipeline, so a selected + # non-default backend is compared against it here as well. Only + # accepted advances have a rebuild to land on: a rejected advance's + # rebuild is the coordinator's recovery, which is deliberately not + # pinned (see the module docstring). + python_adapter = parity.PythonWindowPipelineAdapter() + advancing = [ + case for case in self.corpus if case.advances and case.expected_rejection is None + ] + self.assertGreaterEqual(len(advancing), 8) + for case in advancing: + if self._declared_unsupported(case.name): + continue + with self.subTest(case=case.name): + rebuilt = python_adapter.run(parity.union_rebuild_case(case)) + self.assertEqual( + replace(self._outputs(case.name), advance_stats=()), + rebuilt, + ) + + def test_rejections_are_pinned_for_every_condition(self) -> None: + # Every category in the vocabulary is frozen by at least one case, + # each rejection case froze exactly the category it was built for + # (so it cannot drift into passing for a different reason), and no + # accepted case froze a rejection. + pinned: dict[str, list[str]] = {} + for name, case in self.cases.items(): + entry = self.reference_cases[name] + with self.subTest(case=name): + if case.expected_rejection is None: + self.assertNotIn("rejected", entry) + continue + self.assertEqual(entry.get("rejected"), case.expected_rejection) + self.assertNotIn("record_count", entry) + self.assertEqual(self.results[name], parity.WindowPipelineRejection(case.expected_rejection)) + pinned.setdefault(case.expected_rejection, []).append(name) + self.assertEqual(set(pinned), set(parity.REJECTION_REASONS)) + # Both eligibility exclusions are pinned separately, and the repeat + # case repeats a share still inside the window. + self.assertEqual(len(pinned["delta_ineligible_at_anchor"]), 2) + repeat = self.cases["reject-delta-repeats-retained-share"] + retained_seqs = { + record.share_seq + for record in parity.fold_case(replace(repeat, advances=())).records() + } + self.assertTrue( + all(record.share_seq in retained_seqs for record in repeat.advances[0].delta_records) + ) + + def test_frozen_reference_regeneration_is_a_no_op(self) -> None: + committed = parity.REFERENCE_FIXTURE_PATH.read_bytes() + regenerated = parity.encode_reference_document(parity.reference_document()) + if regenerated != committed: + self.fail( + "the frozen reference no longer matches the shipped pipeline" + " and corpus; if the byte contract changed deliberately," + f" regenerate it with: {parity.REGENERATE_COMMAND}" + ) + + def test_corpus_exercises_the_declared_edge_cases(self) -> None: + # An accidentally hollow corpus would prove nothing, so pin the + # structural properties each case exists for. + self.assertEqual(DEFAULT_INCREMENTAL_SHARE_WINDOW_PAGE_SIZE, 512) + reference = self.reference_cases + self.assertEqual(reference["empty-window"]["record_count"], 0) + self.assertEqual(reference["single-record"]["record_count"], 1) + for count in (511, 512, 513): + self.assertEqual(reference[f"page-boundary-{count}"]["record_count"], count) + self.assertEqual( + len(parity.fold_case(self.cases["page-boundary-512"]).pages), 1 + ) + self.assertEqual( + len(parity.fold_case(self.cases["page-boundary-513"]).pages), 2 + ) + + multi_page_case = self.cases["multi-page-interior-cutoff"] + multi_page = parity.fold_case(multi_page_case) + self.assertGreaterEqual(len(multi_page.pages), 3) + self.assertGreater(multi_page.total_difficulty, multi_page_case.window_weight) + + bulk = parity.fold_case(self.cases["bulk-seeded"]) + self.assertGreaterEqual(reference["bulk-seeded"]["record_count"], 2_500) + self.assertGreaterEqual(len(bulk.pages), 4) + self.assertEqual(len(self.cases["bulk-seeded"].advances), 1) + self.assertGreaterEqual(len(self.cases["incremental-two-advances"].advances), 2) + + crossing = parity.fold_case(self.cases["crossing-row-retained"]) + self.assertGreater( + crossing.total_difficulty, + self.cases["crossing-row-retained"].window_weight, + ) + exact_fit = parity.fold_case(self.cases["crossing-row-exact-fit"]) + self.assertEqual( + exact_fit.total_difficulty, + self.cases["crossing-row-exact-fit"].window_weight, + ) + + self.assertEqual(reference["eligibility-filtering"]["record_count"], 2) + + unsorted_case = self.cases["unsorted-input"] + input_seqs = [record.share_seq for record in unsorted_case.snapshot_records] + self.assertNotEqual(input_seqs, sorted(input_seqs)) + output_seqs = [ + record.share_seq for record in parity.fold_case(unsorted_case).records() + ] + self.assertEqual(output_seqs, sorted(input_seqs)) + + # The escape alphabet: everything Python's ensure_ascii encoder + # treats specially, plus the printable characters that must pass + # through raw (solidus, space, uppercase). DEL is the separating + # case -- escaped by Python, raw in most other encoders -- so it is + # asserted both escaped and never raw, in the spool identities too. + non_ascii = reference["non-ascii-strings"]["pinned_literals"]["canonical_bytes"] + for needle in ( + "\\ud83d\\ude80", "\\u00e9", "\\u0001", "\\t", "\\n", '\\"', "\\\\", + "\\u007f", "\\r", "\\b", "\\f", "\\u001f", "\\u2028", "\\u2029", + "MINER-A/worker 1", + ): + self.assertIn(needle, non_ascii) + self.assertNotIn("\\/", non_ascii) + self.assertNotIn("\x7f", non_ascii) + non_ascii_spool = reference["non-ascii-strings"]["pinned_literals"]["spool_tail"] + self.assertIn("MINER-A/worker 1\\u007f", non_ascii_spool) + self.assertNotIn("\x7f", non_ascii_spool) + + # Null-vs-absent: prism JSON omits an absent credit_policy while the + # compact share tuple carries an explicit null. + single = reference["single-record"]["pinned_literals"] + self.assertNotIn("credit_policy", single["canonical_bytes"]) + self.assertIn(",null]", single["spool_tail"]) + + reuse_tail = reference["credit-policy-identity-reuse"]["pinned_literals"]["spool_tail"] + spool = json.loads("{" + reuse_tail[1:]) + identities = spool["compact_share_identities"] + compact_shares = spool["compact_shares"] + self.assertLess(len(identities), len(compact_shares)) + self.assertEqual( + sorted({row[2] for row in compact_shares}), + list(range(len(identities))), + ) + + # Integer width: wide-integers cannot be quietly shrunk back under + # 32/53/64 bits, must stay inside every declared Rust width (so both + # backends are held to its bytes), and its retention cutoff genuinely + # depends on bits above 2^64 (the three newest rows fall short of the + # weight only because of its 2^127 term; truncated to 64 bits they + # would not). + wide_case = self.cases["wide-integers"] + wide_inputs = wide_case.snapshot_records + self.assertTrue(any(record.share_difficulty.bit_length() > 64 for record in wide_inputs)) + self.assertTrue(any(record.network_difficulty.bit_length() > 64 for record in wide_inputs)) + self.assertTrue(any(record.network_difficulty == 2**128 - 1 for record in wide_inputs)) + self.assertTrue(any(record.share_seq.bit_length() > 32 for record in wide_inputs)) + self.assertTrue(any(record.share_seq == 2**64 - 1 for record in wide_inputs)) + self.assertTrue(any(record.template_height.bit_length() > 32 for record in wide_inputs)) + # ntime: above i32 (top bit set) and at u32's maximum, never beyond. + self.assertTrue(any(record.ntime == 2**32 - 1 for record in wide_inputs)) + self.assertTrue(any(record.ntime.bit_length() == 32 for record in wide_inputs)) + self.assertTrue(any(record.template_height == 0 and record.ntime == 0 for record in wide_inputs)) + self.assertGreater(wide_case.anchor_job_issued_at_ms.bit_length(), 40) + self.assertGreater(wide_case.window_weight.bit_length(), 64) + self.assertIsNone(parity.RUST_INTEGER_DOMAIN.exclusion(wide_case)) + self.assertEqual(reference["wide-integers"]["record_count"], 4) + wide = parity.fold_case(wide_case) + self.assertGreater(wide.total_difficulty, wide_case.window_weight) + newest_three = sum(record.share_difficulty for record in wide.records()[1:]) + self.assertLess(newest_three, wide_case.window_weight) + self.assertGreaterEqual(newest_three % 2**64, wide_case.window_weight % 2**64) + wide_literal = reference["wide-integers"]["pinned_literals"]["canonical_bytes"] + for digits in (2**127 + 3, 2**128 - 1, 2**64 - 1, 2**62, 2**32 - 1, 2**53 + 1): + self.assertIn(str(digits), wide_literal) + + # The genuinely out-of-domain values, one case per width, each + # excluded from the Rust domain by exactly the field it exists for, + # with Python's bytes frozen. + beyond_case = self.cases["difficulty-beyond-u128"] + self.assertTrue( + any(record.share_difficulty.bit_length() > 128 for record in beyond_case.snapshot_records) + ) + self.assertGreater(beyond_case.window_weight.bit_length(), 128) + self.assertIn("u128", parity.RUST_INTEGER_DOMAIN.exclusion(beyond_case) or "") + self.assertEqual(reference["difficulty-beyond-u128"]["record_count"], 3) + beyond = parity.fold_case(beyond_case) + self.assertGreater(beyond.total_difficulty.bit_length(), 128) + self.assertEqual(beyond.records()[0].share_difficulty, 2**128 + 1) + beyond_literal = reference["difficulty-beyond-u128"]["pinned_literals"]["canonical_bytes"] + for digits in (2**128 + 1, 2**128, 2**128 - 1, 10**78 - 1): + self.assertIn(str(digits), beyond_literal) + + ntime_case = self.cases["ntime-beyond-u32"] + self.assertTrue(any(record.ntime.bit_length() > 32 for record in ntime_case.snapshot_records)) + self.assertTrue(any(record.ntime == 2**63 - 1 for record in ntime_case.snapshot_records)) + self.assertEqual( + parity.RUST_INTEGER_DOMAIN.exclusion(ntime_case), + "snapshot record[0].ntime 4294967296 is outside u32", + ) + self.assertIsNone( + parity.RUST_INTEGER_DOMAIN.exclusion( + replace( + ntime_case, + snapshot_records=tuple( + replace(record, ntime=2**32 - 1) for record in ntime_case.snapshot_records + ), + ) + ) + ) + self.assertEqual(reference["ntime-beyond-u32"]["record_count"], 3) + ntime_literal = reference["ntime-beyond-u32"]["pinned_literals"]["canonical_bytes"] + for digits in (2**32, 2**63 - 1): + self.assertIn(str(digits), ntime_literal) + + # Advance-path shapes and their stats, pinned at the values the + # shipped fold reports so touched_pages' subtle definition cannot be + # re-frozen to a plausible wrong one without a visible diff here. + exact_case = self.cases["advance-exact-fit-expiry"] + self.assertEqual(reference["advance-exact-fit-expiry"]["record_count"], 3) + self.assertEqual(parity.fold_case(exact_case).total_difficulty, exact_case.window_weight) + self.assertEqual( + reference["advance-exact-fit-expiry"]["advance_stats"], + [{"added_rows": 1, "expired_rows": 1, "touched_pages": 1}], + ) + exceeds_case = self.cases["advance-delta-exceeds-window"] + self.assertGreater( + sum(record.share_difficulty for record in exceeds_case.advances[0].delta_records), + exceeds_case.window_weight, + ) + self.assertEqual(reference["advance-delta-exceeds-window"]["record_count"], 2) + self.assertEqual( + reference["advance-delta-exceeds-window"]["advance_stats"], + [{"added_rows": 5, "expired_rows": 6, "touched_pages": 1}], + ) + self.assertEqual(reference["advance-partial-expiry-in-appended-page"]["record_count"], 1) + self.assertEqual( + reference["advance-partial-expiry-in-appended-page"]["advance_stats"], + [{"added_rows": 5, "expired_rows": 7, "touched_pages": 1}], + ) + self.assertEqual(self.cases["advance-from-empty-window"].snapshot_records, ()) + self.assertEqual(reference["advance-from-empty-window"]["record_count"], 1) + self.assertEqual( + reference["advance-from-empty-window"]["advance_stats"], + [{"added_rows": 1, "expired_rows": 0, "touched_pages": 0}], + ) + for name in ("advance-empty-delta", "advance-equal-anchor"): + case = self.cases[name] + self.assertEqual(case.advances[0].delta_records, ()) + self.assertEqual( + reference[name]["advance_stats"], + [{"added_rows": 0, "expired_rows": 0, "touched_pages": 0}], + ) + # Bytes identical to the snapshot alone: the advance is a no-op. + self.assertEqual( + reference[name]["canonical_bytes_sha256"], + parity.sha256_hex(parity.outputs_from_window(parity.fold_case(replace(case, advances=()))).canonical_bytes), + ) + equal_anchor = self.cases["advance-equal-anchor"] + self.assertEqual( + equal_anchor.advances[0].anchor_job_issued_at_ms, + equal_anchor.anchor_job_issued_at_ms, + ) + self.assertGreater( + self.cases["advance-empty-delta"].advances[0].anchor_job_issued_at_ms, + self.cases["advance-empty-delta"].anchor_job_issued_at_ms, + ) + + +# --- perturbed backends ----------------------------------------------------- + + +def _window_holding( + records: tuple[parity.AcceptedShareRecord, ...], + case: parity.WindowPipelineCase, + anchor_ms: int, +) -> IncrementalShareWindow: + """A window retaining exactly these records, with no cutoff applied.""" + window = IncrementalShareWindow.from_full_snapshot( + list(records), + anchor_job_issued_at_ms=anchor_ms, + window_weight=sum(int(record.share_difficulty) for record in records) + 1, + page_size=case.page_size, + ) + return replace(window, window_weight=case.window_weight) + + +_INTEGER_RECORD_FIELDS = ( + "share_seq", + "share_difficulty", + "network_difficulty", + "template_height", + "job_issued_at_ms", + "accepted_at_ms", + "ntime", +) + + +def _map_case_integers( + case: parity.WindowPipelineCase, + transform, + *, + record_fields: tuple[str, ...] = _INTEGER_RECORD_FIELDS, +) -> parity.WindowPipelineCase: + """The case as a backend carrying its integers in some narrower type sees it.""" + + def mapped(record: parity.AcceptedShareRecord) -> parity.AcceptedShareRecord: + return replace(record, **{name: transform(getattr(record, name)) for name in record_fields}) + + return replace( + case, + anchor_job_issued_at_ms=transform(case.anchor_job_issued_at_ms), + window_weight=transform(case.window_weight), + snapshot_records=tuple(mapped(record) for record in case.snapshot_records), + advances=tuple( + replace( + step, + anchor_job_issued_at_ms=transform(step.anchor_job_issued_at_ms), + delta_records=tuple(mapped(record) for record in step.delta_records), + ) + for step in case.advances + ), + ) + + +def _reencoded(result: parity.WindowPipelineResult, old: bytes, new: bytes) -> parity.WindowPipelineResult: + """The backend's native byte outputs re-encoded with one escape rule changed.""" + if isinstance(result, parity.WindowPipelineRejection): + return result + record_jsons = tuple(record.replace(old, new) for record in result.record_jsons) + canonical_bytes = result.canonical_bytes.replace(old, new) + return replace( + result, + record_jsons=record_jsons, + canonical_bytes=canonical_bytes, + canonical_digest=parity.sha256_hex(canonical_bytes), + ) + + +class _FieldOrderPerturbedAdapter: + """Encodes record keys in insertion order instead of sorted order.""" + + name = "perturbed-field-order" + + def run(self, case: parity.WindowPipelineCase) -> parity.WindowPipelineResult: + window = parity.fold_case(case) + base = parity.outputs_from_window(window) + record_jsons = tuple( + json.dumps(record, sort_keys=False, separators=(",", ":"), default=str).encode() + for record in window.json_records() + ) + canonical_bytes = b"[" + b",".join(record_jsons) + b"]" + return replace( + base, + record_jsons=record_jsons, + canonical_bytes=canonical_bytes, + canonical_digest=parity.sha256_hex(canonical_bytes), + ) + + +class _MissingPageSeparatorAdapter: + """Streams the digest without the comma between page fragments.""" + + name = "perturbed-missing-page-separator" + + def run(self, case: parity.WindowPipelineCase) -> parity.WindowPipelineResult: + window = parity.fold_case(case) + base = parity.outputs_from_window(window) + fragments = [ + page.canonical_json_items + for page in window.pages + if page.canonical_json_items + ] + return replace( + base, + canonical_digest=parity.sha256_hex(b"[" + b"".join(fragments) + b"]"), + ) + + +class _NullCreditPolicyAdapter: + """Emits credit_policy as an explicit null instead of omitting the key.""" + + name = "perturbed-null-credit-policy" + + def run(self, case: parity.WindowPipelineCase) -> parity.WindowPipelineResult: + window = parity.fold_case(case) + base = parity.outputs_from_window(window) + record_jsons = tuple( + parity.canonical_json_bytes({**record, "credit_policy": record.get("credit_policy")}) + for record in window.json_records() + ) + canonical_bytes = b"[" + b",".join(record_jsons) + b"]" + return replace( + base, + record_jsons=record_jsons, + canonical_bytes=canonical_bytes, + canonical_digest=parity.sha256_hex(canonical_bytes), + ) + + +class _CrossingRowDroppedAdapter: + """Applies a wrong strict cutoff: the crossing row is dropped.""" + + name = "perturbed-crossing-row-dropped" + + def run(self, case: parity.WindowPipelineCase) -> parity.WindowPipelineResult: + window = parity.fold_case(case) + records = window.records() + if records and window.total_difficulty > case.window_weight: + records = records[1:] + rebuilt = IncrementalShareWindow.from_full_snapshot( + list(records), + anchor_job_issued_at_ms=parity.final_anchor_job_issued_at_ms(case), + window_weight=case.window_weight, + page_size=case.page_size, + ) + return parity.outputs_from_window(rebuilt) + + +class _SpoolWhitespaceAdapter: + """Writes the spool fragments with json.dumps's default spaced separators.""" + + name = "perturbed-spool-whitespace" + + def run(self, case: parity.WindowPipelineCase) -> parity.WindowPipelineResult: + window = parity.fold_case(case) + base = parity.outputs_from_window(window) + identities, compact_shares = _compact_share_payload(list(window.json_records())) + spool_tail = ( + b',"compact_share_identities":' + + json.dumps(identities).encode() + + b',"compact_shares":' + + json.dumps(compact_shares).encode() + + b"}" + ) + return replace(base, spool_tail=spool_tail) + + +class _WrappingU64Adapter: + """Carries every integer in a wrapping unsigned 64-bit type.""" + + name = "perturbed-u64-wrap" + + def run(self, case: parity.WindowPipelineCase) -> parity.WindowPipelineResult: + return parity.run_python_pipeline(_map_case_integers(case, lambda value: value % 2**64)) + + +class _WrappingU32Adapter: + """Carries every integer in a wrapping unsigned 32-bit type.""" + + name = "perturbed-u32-wrap" + + def run(self, case: parity.WindowPipelineCase) -> parity.WindowPipelineResult: + return parity.run_python_pipeline(_map_case_integers(case, lambda value: value % 2**32)) + + +class _SaturatingU64AccumulatorAdapter: + """Clamps the difficulties and window_weight at u64::MAX: a saturating u64 accumulator.""" + + name = "perturbed-u64-saturating-accumulator" + + def run(self, case: parity.WindowPipelineCase) -> parity.WindowPipelineResult: + return parity.run_python_pipeline( + _map_case_integers( + case, + lambda value: min(value, 2**64 - 1), + record_fields=("share_difficulty",), + ) + ) + + +class _SaturatingU128DifficultyAdapter: + """Clamps share_difficulty and window_weight at u128::MAX: a saturating u128 accumulator. + + Declares the Rust domain, so where it answers the oracle must hold it to + the bytes and not excuse it: a declared domain permits Unsupported, never + wrong digits. + """ + + name = "perturbed-u128-saturating-difficulty" + integer_domain = parity.RUST_INTEGER_DOMAIN + + def run(self, case: parity.WindowPipelineCase) -> parity.WindowPipelineResult: + return parity.run_python_pipeline( + _map_case_integers( + case, + lambda value: min(value, 2**128 - 1), + record_fields=("share_difficulty",), + ) + ) + + +class _DeclaredDomainDecliningAdapter: + """Declares the Rust domain and declines exactly the cases outside it. + + The honest shape of a fixed-width backend: Unsupported with a declared + reason where the domain excludes the case, Python's bytes everywhere else. + """ + + name = "perturbed-declared-domain-declining" + integer_domain = parity.RUST_INTEGER_DOMAIN + + def run(self, case: parity.WindowPipelineCase) -> parity.WindowPipelineResult: + exclusion = self.integer_domain.exclusion(case) + if exclusion is not None: + return parity.WindowPipelineUnsupported(exclusion) + return parity.run_python_pipeline(case) + + +class _OverDecliningAdapter: + """Declares the Rust domain but declines anything above 64 bits. + + A backend whose real width is narrower than the one it declares: it + answers Unsupported for wide-integers, which its declared domain includes. + """ + + name = "perturbed-over-declining" + integer_domain = parity.RUST_INTEGER_DOMAIN + + def run(self, case: parity.WindowPipelineCase) -> parity.WindowPipelineResult: + records = case.snapshot_records + tuple( + record for step in case.advances for record in step.delta_records + ) + if any(record.share_difficulty.bit_length() > 64 for record in records): + return parity.WindowPipelineUnsupported("difficulty above u64") + return parity.run_python_pipeline(case) + + +class _UndeclaredDecliningAdapter: + """Declares no domain at all yet declines the beyond-u128 case.""" + + name = "perturbed-undeclared-declining" + + def run(self, case: parity.WindowPipelineCase) -> parity.WindowPipelineResult: + if case.name == "difficulty-beyond-u128": + return parity.WindowPipelineUnsupported("cannot carry 2^128+1") + return parity.run_python_pipeline(case) + + +class _NeverRefusingAdapter: + """Folds whatever it is given: where the shipped fold refuses, it recovers by itself. + + Models a backend with every check removed, or one that performs the + coordinator's full-rescan recovery internally and returns its bytes as + if nothing had happened -- the bytes a lenient fold or the recovery + would produce are exactly what the oracle must not accept as parity. + """ + + name = "perturbed-never-refusing" + + def run(self, case: parity.WindowPipelineCase) -> parity.WindowPipelineResult: + result = parity.run_python_pipeline(case) + if isinstance(result, parity.WindowPipelineOutputs): + return result + union = case.snapshot_records + tuple( + record for step in case.advances for record in step.delta_records + ) + seen_seqs: set[int] = set() + seen_ids: set[str] = set() + kept = [] + for record in union: + if ( + record.share_seq in seen_seqs + or record.share_id in seen_ids + or record.share_difficulty <= 0 + ): + continue + seen_seqs.add(record.share_seq) + seen_ids.add(record.share_id) + kept.append(record) + folded = IncrementalShareWindow.from_full_snapshot( + kept, + anchor_job_issued_at_ms=parity.final_anchor_job_issued_at_ms(case), + window_weight=max(case.window_weight, 1), + page_size=max(case.page_size, 1), + ) + return parity.outputs_from_window( + folded, + tuple( + parity.WindowPipelineAdvanceStats(len(step.delta_records), 0, 0) + for step in case.advances + ), + ) + + +class _MisfiledRejectionAdapter: + """Refuses where the shipped fold does, but files every refusal under one category.""" + + name = "perturbed-misfiled-rejection" + + def run(self, case: parity.WindowPipelineCase) -> parity.WindowPipelineResult: + result = parity.run_python_pipeline(case) + if isinstance(result, parity.WindowPipelineRejection): + return parity.WindowPipelineRejection("anchor_regression") + return result + + +class _StrictAdvanceExpiryAdapter: + """Expires on > instead of >= in the advance loops (modelled as weight+1 while advancing).""" + + name = "perturbed-strict-advance-expiry" + + def run(self, case: parity.WindowPipelineCase) -> parity.WindowPipelineResult: + try: + window = IncrementalShareWindow.from_full_snapshot( + list(case.snapshot_records), + anchor_job_issued_at_ms=case.anchor_job_issued_at_ms, + window_weight=case.window_weight, + page_size=case.page_size, + ) + stats = [] + for step in case.advances: + window, step_stats = replace(window, window_weight=case.window_weight + 1).advance( + list(step.delta_records), + anchor_job_issued_at_ms=step.anchor_job_issued_at_ms, + ) + stats.append(parity.WindowPipelineAdvanceStats.from_shipped(step_stats)) + except (ValueError, IncrementalWindowFallback): + return parity.run_python_pipeline(case) + return parity.outputs_from_window(replace(window, window_weight=case.window_weight), tuple(stats)) + + +class _OldPagesOnlyExpiryAdapter: + """Expires only among pre-existing rows; the appended delta is immune. + + The natural shape of a drop-prefix / append-suffix protocol: when the + shipped fold exhausts every old row and goes on into the delta, this + backend stops at the old rows and keeps the delta whole. + """ + + name = "perturbed-old-pages-only-expiry" + + def run(self, case: parity.WindowPipelineCase) -> parity.WindowPipelineResult: + try: + window = IncrementalShareWindow.from_full_snapshot( + list(case.snapshot_records), + anchor_job_issued_at_ms=case.anchor_job_issued_at_ms, + window_weight=case.window_weight, + page_size=case.page_size, + ) + stats = [] + for step in case.advances: + advanced, step_stats = window.advance( + list(step.delta_records), + anchor_job_issued_at_ms=step.anchor_job_issued_at_ms, + ) + retained_seqs = {record.share_seq for record in advanced.records()} + if all(record.share_seq in retained_seqs for record in step.delta_records): + window = advanced + else: + step_stats = replace(step_stats, expired_rows=len(window.records())) + window = _window_holding(step.delta_records, case, step.anchor_job_issued_at_ms) + stats.append(parity.WindowPipelineAdvanceStats.from_shipped(step_stats)) + except (ValueError, IncrementalWindowFallback): + return parity.run_python_pipeline(case) + return parity.outputs_from_window(window, tuple(stats)) + + +class _EqualAnchorRejectingAdapter: + """Treats an advance at the same anchor as a regression (<= instead of <).""" + + name = "perturbed-equal-anchor-rejecting" + + def run(self, case: parity.WindowPipelineCase) -> parity.WindowPipelineResult: + anchor_ms = case.anchor_job_issued_at_ms + for step in case.advances: + if step.anchor_job_issued_at_ms == anchor_ms: + return parity.WindowPipelineRejection("anchor_regression") + anchor_ms = step.anchor_job_issued_at_ms + return parity.run_python_pipeline(case) + + +class _RawDelAdapter: + """An encoder that escapes only below 0x20 plus quote and backslash: DEL stays raw.""" + + name = "perturbed-raw-del" + + def run(self, case: parity.WindowPipelineCase) -> parity.WindowPipelineResult: + return _reencoded(parity.run_python_pipeline(case), b"\\u007f", b"\x7f") + + +class _EscapedSolidusAdapter: + """An encoder that emits the optional JSON escape for solidus.""" + + name = "perturbed-escaped-solidus" + + def run(self, case: parity.WindowPipelineCase) -> parity.WindowPipelineResult: + return _reencoded(parity.run_python_pipeline(case), b"/", b"\\/") + + +class _NoAdvanceStatsAdapter: + """Produces the bytes but reports no per-advance stats.""" + + name = "perturbed-no-advance-stats" + + def run(self, case: parity.WindowPipelineCase) -> parity.WindowPipelineResult: + result = parity.run_python_pipeline(case) + if isinstance(result, parity.WindowPipelineRejection): + return result + return replace(result, advance_stats=()) + + +class _SurvivorTouchedPagesAdapter: + """Derives touched_pages from which pre-existing pages survive, not from the work done. + + Counts a pre-existing page when it is partially retained, and the + append-into page only if any of its rows survive -- so a page appended + into and then expired wholesale in the same advance is not counted, where + the shipped fold counts it because its records were rewritten. + """ + + name = "perturbed-survivor-touched-pages" + + def run(self, case: parity.WindowPipelineCase) -> parity.WindowPipelineResult: + try: + window = IncrementalShareWindow.from_full_snapshot( + list(case.snapshot_records), + anchor_job_issued_at_ms=case.anchor_job_issued_at_ms, + window_weight=case.window_weight, + page_size=case.page_size, + ) + stats = [] + for step in case.advances: + advanced, step_stats = window.advance( + list(step.delta_records), + anchor_job_issued_at_ms=step.anchor_job_issued_at_ms, + ) + retained_seqs = {record.share_seq for record in advanced.records()} + touched: set[int] = set() + last = window.pages[-1] if window.pages else None + if step.delta_records and last is not None and len(last.records) < case.page_size: + if any(record.share_seq in retained_seqs for record in last.records): + touched.add(len(window.pages) - 1) + for index, page in enumerate(window.pages): + surviving = sum(1 for record in page.records if record.share_seq in retained_seqs) + if 0 < surviving < len(page.records): + touched.add(index) + stats.append( + parity.WindowPipelineAdvanceStats( + added_rows=step_stats.added_rows, + expired_rows=step_stats.expired_rows, + touched_pages=len(touched), + ) + ) + window = advanced + except (ValueError, IncrementalWindowFallback): + return parity.run_python_pipeline(case) + return parity.outputs_from_window(window, tuple(stats)) + + +class WindowPipelineDivergenceDetectionTests(unittest.TestCase): + """Prove the harness fails on known divergence classes, and only there.""" + + cases: dict[str, parity.WindowPipelineCase] + reference_cases: dict[str, dict[str, object]] + + @classmethod + def setUpClass(cls) -> None: + cls.cases = {case.name: case for case in parity.build_corpus()} + cls.reference_cases = parity.load_reference_document()["cases"] + + def _diverging(self, adapter: parity.WindowPipelineAdapter, case_name: str) -> list[str]: + return parity.diverging_outputs( + self.reference_cases[case_name], + adapter.run(self.cases[case_name]), + ) + + def test_record_field_reordering_is_detected(self) -> None: + problems = self._diverging(_FieldOrderPerturbedAdapter(), "single-record") + self.assertTrue(problems) + self.assertTrue(any(problem.startswith("record_jsons[0]") for problem in problems)) + + def test_missing_page_separator_is_detected_only_on_multi_page_windows(self) -> None: + adapter = _MissingPageSeparatorAdapter() + problems = self._diverging(adapter, "page-boundary-513") + self.assertTrue(problems) + self.assertTrue(all(problem.startswith("canonical_digest") for problem in problems)) + # On a single page the missing separator never fires, so this framing + # bug is invisible there -- the reason multi-page corpora must exist. + self.assertEqual(self._diverging(adapter, "page-boundary-512"), []) + + def test_null_credit_policy_instead_of_absent_is_detected(self) -> None: + problems = self._diverging(_NullCreditPolicyAdapter(), "single-record") + self.assertTrue(problems) + self.assertTrue(any(problem.startswith("record_jsons[0]") for problem in problems)) + + def test_dropped_crossing_row_is_detected_only_where_one_exists(self) -> None: + adapter = _CrossingRowDroppedAdapter() + problems = self._diverging(adapter, "crossing-row-retained") + self.assertTrue(any(problem.startswith("record_count") for problem in problems)) + # The exact-fit sibling has no crossing row, so the wrong cutoff + # passes there -- the pair is what localizes the off-by-one. + self.assertEqual(self._diverging(adapter, "crossing-row-exact-fit"), []) + + def test_spool_separator_whitespace_is_detected(self) -> None: + problems = self._diverging(_SpoolWhitespaceAdapter(), "credit-policy-identity-reuse") + self.assertTrue(problems) + self.assertTrue(all(problem.startswith("spool_tail") for problem in problems)) + + def _diverging_declared(self, adapter: parity.WindowPipelineAdapter, case_name: str) -> list[str]: + """As _diverging, but telling the harness the backend's declared domain.""" + case = self.cases[case_name] + return parity.diverging_outputs( + self.reference_cases[case_name], + adapter.run(case), + domain_exclusion=parity.domain_exclusion(adapter, case), + ) + + def test_wrapping_64_bit_integers_are_detected_only_above_64_bits(self) -> None: + adapter = _WrappingU64Adapter() + # Truncating window_weight and the difficulties stops retention one + # row early, so the count itself diverges, not only the digits. + problems = self._diverging(adapter, "wide-integers") + self.assertTrue(any(problem.startswith("record_count") for problem in problems)) + self.assertTrue(self._diverging(adapter, "difficulty-beyond-u128")) + # Every other case lives below 2^31, where a 64-bit backend is exact: + # the width holes were invisible until the wide cases existed. + for name in ("single-record", "incremental-two-advances", "bulk-seeded"): + self.assertEqual(self._diverging(adapter, name), []) + + def test_retuned_wide_integers_still_refuses_every_narrow_backend(self) -> None: + # The protection wide-integers was built for, re-checked after the + # values were brought inside the declared Rust widths: a u32-wrapping, + # a u64-wrapping and a u64-saturating backend each still fail it (the + # count itself, not only the digits, for the two that touch the + # accumulator), while passing the sub-2^31 corpus exactly as before. + # Declaring the Rust domain could not excuse any of them, because the + # case lies inside that domain. + for adapter in (_WrappingU32Adapter(), _WrappingU64Adapter(), _SaturatingU64AccumulatorAdapter()): + with self.subTest(adapter=adapter.name): + problems = self._diverging(adapter, "wide-integers") + self.assertTrue(problems, adapter.name) + if adapter.name != "perturbed-u32-wrap": + self.assertTrue(any(problem.startswith("record_count") for problem in problems)) + declared = type(adapter.name, (object,), {"name": adapter.name, "integer_domain": parity.RUST_INTEGER_DOMAIN, "run": staticmethod(adapter.run)})() + self.assertEqual(self._diverging_declared(declared, "wide-integers"), problems) + for name in ("single-record", "incremental-two-advances", "crossing-row-retained"): + self.assertEqual(self._diverging(adapter, name), []) + self.assertIsNone(parity.RUST_INTEGER_DOMAIN.exclusion(self.cases["wide-integers"])) + + def test_saturating_u128_difficulty_is_refused_by_the_beyond_u128_case_alone(self) -> None: + # 128-bit difficulty accumulation passes wide-integers (inside the + # declared domain) and is refused where numeric(78,0) exceeds it, by + # the clamped crossing row's digits (retained count still agrees). + # The backend declares the Rust domain, and that excuses nothing: it + # produced bytes, so it is held to the frozen bytes. + adapter = _SaturatingU128DifficultyAdapter() + self.assertEqual(self._diverging_declared(adapter, "wide-integers"), []) + self.assertIsNotNone(parity.domain_exclusion(adapter, self.cases["difficulty-beyond-u128"])) + problems = self._diverging_declared(adapter, "difficulty-beyond-u128") + self.assertTrue(any(problem.startswith("record_jsons[0]") for problem in problems)) + self.assertFalse(any(problem.startswith("record_count") for problem in problems)) + + def test_unsupported_is_parity_outside_the_declared_domain_and_divergence_inside(self) -> None: + honest = _DeclaredDomainDecliningAdapter() + for name in ("difficulty-beyond-u128", "ntime-beyond-u32"): + with self.subTest(case=name): + # Outside the declared domain: Unsupported is parity, while + # the frozen reference keeps Python's bytes for the case. + self.assertEqual(self._diverging_declared(honest, name), []) + self.assertIn("record_count", self.reference_cases[name]) + result = honest.run(self.cases[name]) + self.assertIsInstance(result, parity.WindowPipelineUnsupported) + # Inside it, the same backend produces bytes and is held to them. + self.assertEqual(self._diverging_declared(honest, "wide-integers"), []) + self.assertIsInstance(honest.run(self.cases["wide-integers"]), parity.WindowPipelineOutputs) + + # A backend narrower than it declares: Unsupported inside the declared + # domain is a divergence, named as such. + over = _OverDecliningAdapter() + self.assertEqual( + self._diverging_declared(over, "wide-integers"), + ["unsupported: backend declared 'difficulty above u64' but the case is inside its declared integer domain"], + ) + self.assertEqual(self._diverging_declared(over, "difficulty-beyond-u128"), []) + + # A backend that declares nothing claims the whole domain, so any + # Unsupported it returns is a divergence -- even on the beyond case. + undeclared = _UndeclaredDecliningAdapter() + self.assertIs(parity.adapter_integer_domain(undeclared), parity.UNBOUNDED_INTEGER_DOMAIN) + self.assertEqual( + self._diverging_declared(undeclared, "difficulty-beyond-u128"), + ["unsupported: backend declared 'cannot carry 2^128+1' but the case is inside its declared integer domain"], + ) + # And the shipped pipeline never declines: the reference cannot even + # freeze an Unsupported. + with self.assertRaisesRegex(RuntimeError, "declared the case unsupported"): + parity.reference_case_entry( + self.cases["single-record"], + parity.WindowPipelineUnsupported("never"), + ) + + def test_a_backend_that_never_refuses_is_detected_on_every_rejection_case(self) -> None: + adapter = _NeverRefusingAdapter() + rejection_cases = [case for case in self.cases.values() if case.expected_rejection] + self.assertEqual(len(rejection_cases), 12) + for case in rejection_cases: + with self.subTest(case=case.name): + problems = self._diverging(adapter, case.name) + self.assertEqual(len(problems), 1) + self.assertTrue(problems[0].startswith("rejected: backend produced outputs")) + # Where the shipped fold accepts, the lenient backend is identical. + self.assertEqual(self._diverging(adapter, "incremental-two-advances"), []) + + def test_misfiled_rejection_category_is_detected(self) -> None: + adapter = _MisfiledRejectionAdapter() + self.assertEqual(self._diverging(adapter, "reject-anchor-regression"), []) + self.assertEqual( + self._diverging(adapter, "reject-delta-not-append"), + ["rejected: 'anchor_regression' != frozen 'delta_not_append'"], + ) + + def test_strict_advance_expiry_is_detected_only_at_exact_fit(self) -> None: + adapter = _StrictAdvanceExpiryAdapter() + problems = self._diverging(adapter, "advance-exact-fit-expiry") + self.assertTrue(any(problem.startswith("record_count") for problem in problems)) + # No other advance in the corpus lands an expiry exactly on the + # weight, and the snapshot-side twin is separate code. + for name in ("incremental-two-advances", "bulk-seeded", "advance-delta-exceeds-window", "crossing-row-exact-fit"): + self.assertEqual(self._diverging(adapter, name), []) + + def test_expiry_confined_to_old_pages_is_detected_only_when_the_delta_outweighs_the_window(self) -> None: + adapter = _OldPagesOnlyExpiryAdapter() + for name in ("advance-delta-exceeds-window", "advance-partial-expiry-in-appended-page"): + with self.subTest(case=name): + problems = self._diverging(adapter, name) + self.assertTrue(any(problem.startswith("record_count") for problem in problems)) + for name in ("incremental-two-advances", "bulk-seeded", "advance-exact-fit-expiry"): + self.assertEqual(self._diverging(adapter, name), []) + + def test_equal_anchor_rejection_is_detected(self) -> None: + adapter = _EqualAnchorRejectingAdapter() + self.assertEqual( + self._diverging(adapter, "advance-equal-anchor"), + ["rejected: backend rejected as 'anchor_regression' but the frozen reference has outputs"], + ) + self.assertEqual(self._diverging(adapter, "reject-anchor-regression"), []) + self.assertEqual(self._diverging(adapter, "advance-empty-delta"), []) + + def test_raw_del_and_escaped_solidus_are_detected(self) -> None: + for adapter in (_RawDelAdapter(), _EscapedSolidusAdapter()): + with self.subTest(adapter=adapter.name): + problems = self._diverging(adapter, "non-ascii-strings") + self.assertTrue(any(problem.startswith("record_jsons[3]") for problem in problems)) + self.assertEqual(self._diverging(adapter, "single-record"), []) + + def test_missing_advance_stats_are_detected_on_advancing_cases(self) -> None: + adapter = _NoAdvanceStatsAdapter() + problems = self._diverging(adapter, "advance-from-empty-window") + self.assertEqual(len(problems), 1) + self.assertTrue(problems[0].startswith("advance_stats")) + self.assertEqual(self._diverging(adapter, "single-record"), []) + + def test_touched_pages_from_survivors_is_detected_only_where_an_appended_page_expires(self) -> None: + adapter = _SurvivorTouchedPagesAdapter() + for name in ("advance-delta-exceeds-window", "advance-partial-expiry-in-appended-page"): + with self.subTest(case=name): + problems = self._diverging(adapter, name) + self.assertTrue(problems) + self.assertTrue(all(problem.startswith("advance_stats") for problem in problems)) + # Where the appended page survives (partially or whole), the two + # definitions agree -- which is why those cases alone proved nothing. + for name in ("advance-exact-fit-expiry", "incremental-two-advances", "advance-from-empty-window"): + self.assertEqual(self._diverging(adapter, name), []) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_window_pipeline_parity_gate.py b/tests/test_window_pipeline_parity_gate.py new file mode 100644 index 000000000..af72800c6 --- /dev/null +++ b/tests/test_window_pipeline_parity_gate.py @@ -0,0 +1,84 @@ +#!/usr/bin/env python3 +"""Fail-closed contracts for the payout-window CI parity entry point.""" + +from __future__ import annotations + +import os +import sys +import tempfile +import unittest +from pathlib import Path +from unittest import mock + +from tests import window_pipeline_parity as parity +from tests import window_pipeline_parity_gate as gate + + +class WindowPipelineParityGateTests(unittest.TestCase): + def test_adapter_selection_must_be_explicit(self) -> None: + with mock.patch.dict(os.environ, {}, clear=True): + with self.assertRaisesRegex( + RuntimeError, + f"{parity.ADAPTER_ENV_VAR} must explicitly select", + ): + gate.resolve_gate_adapter() + + def test_python_selection_resolves_the_python_adapter(self) -> None: + with mock.patch.dict( + os.environ, + {parity.ADAPTER_ENV_VAR: "python"}, + clear=True, + ): + adapter, command = gate.resolve_gate_adapter() + self.assertIsInstance(adapter, parity.PythonWindowPipelineAdapter) + self.assertEqual(adapter.name, "python") + self.assertEqual(command, ()) + + def test_rust_selection_resolves_the_prebuilt_real_daemon(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + binary = Path(tmp) / gate.RUST_DAEMON_BINARY + binary.touch(mode=0o755) + binary.chmod(0o755) + with mock.patch.dict( + os.environ, + { + parity.ADAPTER_ENV_VAR: "rust-daemon", + "PRISM_TOOL_BIN_DIR": tmp, + }, + clear=True, + ): + adapter, command = gate.resolve_gate_adapter() + self.assertIsInstance(adapter, parity.RustDaemonWindowPipelineAdapter) + self.assertEqual(adapter.name, "rust-daemon") + self.assertEqual(command, (str(binary),)) + + def test_rust_selection_refuses_a_missing_binary_and_cargo_fallback(self) -> None: + with tempfile.TemporaryDirectory() as tmp, mock.patch.dict( + os.environ, + { + parity.ADAPTER_ENV_VAR: "rust-daemon", + "PRISM_TOOL_BIN_DIR": tmp, + }, + clear=True, + ): + with self.assertRaisesRegex(RuntimeError, "refusing cargo fallback"): + gate.resolve_gate_adapter() + + def test_rust_adapter_refuses_a_protocol_mismatch(self) -> None: + fake_daemon = Path(__file__).parent / "fixtures" / "fake_serve_builder.py" + case = parity.build_corpus()[0] + with mock.patch.object( + parity, + "prism_tool_command", + return_value=[sys.executable, str(fake_daemon)], + ), mock.patch.dict( + os.environ, + {"FAKE_SERVE_BUILDER_PROTOCOL": "1"}, + clear=True, + ): + with self.assertRaisesRegex(RuntimeError, "daemon announced handshake"): + parity.RustDaemonWindowPipelineAdapter().run(case) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/window_pipeline_parity.py b/tests/window_pipeline_parity.py new file mode 100644 index 000000000..203fb762f --- /dev/null +++ b/tests/window_pipeline_parity.py @@ -0,0 +1,2147 @@ +#!/usr/bin/env python3 +"""Differential parity oracle for the PRISM payout-window pipeline. + +Why this exists: the window pipeline -- accepted-share records in, folded +window plus canonical digest plus audit-builder spool fragments out -- is +slated to gain a second implementation that must be proven byte-identical to +the shipped Python one before it can take traffic (issue #131's method: run +both against the same inputs and assert byte-identical outputs). That proof +needs three things to exist *before* the second implementation does: a corpus +derived from the fold's own invariants, frozen byte-exact reference outputs +produced by the shipped pipeline at a known revision, and a runner that can +drive more than one backend over the same corpus. This module provides all +three, composing the two house precedents: the seeded-corpus discipline of +``tests.test_prism_incremental_payout_window`` and the frozen-reference +discipline of the metrics render-parity fixture. + +What is pinned per corpus case, all inside the migration's scope: + +1. ``record_jsons`` -- the folded window as the ordered + ``AcceptedShareRecord.to_prism_json()`` sequence, one canonical JSON + encoding per record (sorted keys, ``(",", ":")`` separators, ASCII + escapes), exactly the encoding each window page pre-computes. +2. ``canonical_bytes`` / ``canonical_digest`` -- the canonical JSON array + exactly as ``IncrementalShareJsonSequence.canonical_json_sha256`` streams + it (non-empty page fragments joined with ``,`` inside ``[``/``]``) and its + SHA-256 hex digest. The digest is *not* a hash of a monolithic re-encode; + an implementation with wrong framing produces a different digest, so the + framed bytes are pinned alongside the digest. +3. ``spool_tail`` -- the audit-builder spool payload tail written by + ``_ShareWindowSerialization.acquire_spooled_tail``, byte-for-byte, + including the ``,"compact_share_identities":`` / ``,"compact_shares":`` / + ``}`` framing, because that framing is the wire contract. +4. ``advance_stats`` -- one ``IncrementalWindowAdvanceStats`` triple + (``added_rows``, ``expired_rows``, ``touched_pages``) per advance step, in + order. These are not wire bytes, but the coordinator exports them to + metrics on every incremental materialization, so a backend that owns + ``advance()`` reports them and they must agree. ``touched_pages`` has a + definition (see :class:`WindowPipelineAdvanceStats`) that is easy to + re-derive wrongly from the retained result instead of from the work done. +5. A rejection, when the shipped pipeline refuses the case. ``from_full_ + snapshot`` refuses five shapes and ``advance()`` six; a backend that + accepts an input the shipped pipeline rejects, or rejects one it accepts, + ships a different payout window. A rejected case freezes only the + implementation-neutral reason category (``REJECTION_REASONS``), never a + Python exception class or message, and ``diverging_outputs`` treats + rejected-vs-outputs in either direction as a divergence and + rejected-vs-rejected with different categories as one too. + +The oracle pins the *pipeline's rejection*, not the coordinator's recovery +from it. In production ``payout_state`` catches the advance fallback, drops +its cached window and re-materializes from a full rescan; that recovery is +coordinator behaviour and is tested there (``tests.test_prism_payout_state``). +Pinning recovery bytes here would let a backend that silently recovers inside +itself -- returning the rescan's bytes where the shipped pipeline refuses -- +pass as parity, which is exactly the divergence class the rejection cases +exist to catch. + +Integer width. The ledger schema is wider than any fixed machine type: +``share_difficulty`` and ``network_difficulty`` are ``numeric(78,0)``, +``share_seq``, ``template_height`` and ``ntime`` are ``bigint``, and the +millisecond fields are epoch milliseconds above 2^40; Python carries all of +them exactly and renders every digit. The contract the corpus enforces is +narrower, and the decision is made here explicitly: the pipeline's declared +integer widths are the Rust widths -- + + share_seq u64 + share_difficulty u128 + network_difficulty u128 + template_height u64 + ntime u32 + job_issued_at_ms, accepted_at_ms i64 (and every anchor) + window_weight u128 + page_size u64 + +-- with the fold's difficulty accumulators u128 as well, so a window whose +difficulty total leaves u128 is outside the domain too. Why u128 and not the +schema's 78 digits: ``share_difficulty`` is ``(pow_limit_target*1e6)//target``, +so at a Bitcoin-mainnet-scale target a share is ~2^98 and a 16x window totals +~2^102 -- a factor of roughly 6*10^7 short of 2^128, unreachable on any chain +this code will see. ``numeric(78,0)`` is a generic wide-numeric column +choice, not a considered statement that difficulties exceed 2^128; widening +the Rust fold to arbitrary precision would put bignum arithmetic on the exact +path the migration exists to make faster. + +A narrower domain is modelled as exactly that, not as a divergence. A +backend declares its domain (:class:`WindowPipelineIntegerDomain`: the +shipped Python pipeline declares ``UNBOUNDED_INTEGER_DOMAIN``, the daemon +``RUST_INTEGER_DOMAIN``, and a backend that declares nothing claims all of +it) and may answer a case outside that domain with the third adapter +outcome, :class:`WindowPipelineUnsupported`, carrying a declared reason. The +frozen reference keeps Python's bytes for such cases; the harness accepts +``Unsupported`` only where the case really lies outside the backend's +declared domain (:func:`diverging_outputs` is told so by +:func:`domain_exclusion`), still fails a backend that returns wrong *bytes* +there (a saturating u128 accumulator, say), and fails a backend that +declares ``Unsupported`` for a case inside its declared domain. This is what +the system really does: the Rust daemon answers ``out_of_range`` instead of +folding, the coordinator degrades to the in-process fold for that +materialization without retiring the daemon, and Python's unbounded integers +produce the right answer. + +The width cases are split accordingly. ``wide-integers`` carries every +integer field far above the 2^31/2^53/2^64 boundaries at which a wrapping or +saturating 32/53/64-bit backend breaks, yet inside every declared width, so +both backends produce bytes and full parity is enforced there: the +protection it was built for (nine narrow-integer backends passed the +original fourteen-case corpus) survives the domain restriction, and the +divergence suite re-checks it against u32-wrapping, u64-wrapping and +u64-saturating backends. The genuinely out-of-domain values live in +``difficulty-beyond-u128`` (``share_difficulty`` 2^128+1 as the retained +crossing row, ``network_difficulty`` at 2^128-1, 2^128 and numeric(78,0)'s +maximum, ``window_weight`` and the retained total above 2^128) and +``ntime-beyond-u32`` (``ntime`` at 2^32 and bigint's maximum): Python's +bytes are frozen, a u128/u32 backend answers ``Unsupported``, and one that +clamps or truncates renders different digits and fails. + +Adapter contract (the seam a future backend implements): inputs in, one +result out. Inputs are one :class:`WindowPipelineCase`, available as an +implementation-neutral JSON document via :func:`case_input_document` -- the +snapshot records with every durable field explicit (``credit_policy`` is +null when absent), the snapshot anchor, ``window_weight``, ``page_size``, and +zero or more append-only advance steps that the backend must fold through +its incremental path. The backend returns :class:`WindowPipelineOutputs`, +:class:`WindowPipelineRejection`, or -- only for a case outside its declared +integer domain -- :class:`WindowPipelineUnsupported`. Register it with +:func:`register_adapter` and select it by setting +``QBIT_WINDOW_PIPELINE_PARITY_ADAPTER``; selection defaults to the shipped +Python pipeline. There is deliberately no production feature switch here -- +the per-component switch belongs to the migration slice, not to the oracle. + +The corpus owns its record factory (:func:`corpus_record`) rather than +importing the sibling golden test's, so an edit there cannot move this +corpus; and literal-pinned cases freeze the input document itself alongside +the outputs, so when the fixture is regenerated the diff shows whether +inputs moved, outputs moved, or both. Without that, any input-only change +fails the suite with "regenerate" and the resulting diff cannot tell the +two apart. + +Regenerating the frozen reference is reproducible and a no-op on an +unchanged tree: + + python3 -m tests.window_pipeline_parity regenerate +""" + +from __future__ import annotations + +import hashlib +import io +import json +import os +import random +import subprocess +import sys +from dataclasses import dataclass, replace +from pathlib import Path +from typing import Callable, Protocol + +from lab.prism.bundle_compiler import ( + PRISM_SERVE_BUILDER_PROTOCOL_VERSION, + _ShareWindowSerialization, +) +from lab.prism.prism_tools import prism_tool_command +from lab.prism.share_ledger import ( + DEFAULT_INCREMENTAL_SHARE_WINDOW_PAGE_SIZE, + AcceptedShareRecord, + IncrementalShareWindow, + IncrementalWindowAdvanceStats, + IncrementalWindowFallback, +) + + +# Distinct from the sibling test's RANDOM_MASTER_SEED so the two corpora can +# never silently alias; derived per-case seeds use fixed documented offsets. +PARITY_RANDOM_MASTER_SEED = 0xB17E_5EED + +REFERENCE_SCHEMA = "qbit-prism-window-pipeline-parity-reference/v2" +CASE_INPUT_SCHEMA = "qbit-prism-window-pipeline-parity-input/v1" +REGENERATE_COMMAND = "python3 -m tests.window_pipeline_parity regenerate" +REFERENCE_FIXTURE_RELPATH = "tests/fixtures/window_pipeline_parity/reference.json" +REFERENCE_FIXTURE_PATH = ( + Path(__file__).resolve().parent / "fixtures" / "window_pipeline_parity" / "reference.json" +) + +# Cases whose canonical document fits under this bound additionally pin the +# literal bytes in the fixture, so a divergence shows *what* diverged, not +# just that something did; cases whose canonical *input* document fits under +# it pin that document too, so an input-only change is visible as such in the +# fixture diff. Larger cases are pinned by length + SHA-256, which is still +# byte-exact; every framing and encoding rule those cases exercise is also +# covered by a literal-pinned case. +LITERAL_PIN_MAX_CANONICAL_BYTES = 16_384 + +ADAPTER_ENV_VAR = "QBIT_WINDOW_PIPELINE_PARITY_ADAPTER" +DEFAULT_ADAPTER_NAME = "python" + +# The durable/public share identity, in declaration order. Deliberately +# excludes ``newly_inserted`` and ``candidate_outbox_state``: both are +# process-local append metadata (compare=False on the dataclass) and are +# outside the pipeline's byte contract. +_RECORD_INPUT_FIELDS = ( + "share_seq", + "share_id", + "miner_id", + "order_key", + "p2mr_program_hex", + "share_difficulty", + "network_difficulty", + "template_height", + "job_id", + "job_issued_at_ms", + "accepted_at_ms", + "ntime", + "credit_policy", +) + +# The implementation-neutral rejection vocabulary, one category per condition +# the shipped pipeline refuses. Categories name the *condition*, never the +# Python exception or its message: a backend in another language reports the +# same category from its own error type. The two groups mirror the two entry +# points so a reader can tell which phase refused without knowing the case. +FULL_SNAPSHOT_REJECTIONS = ( + "duplicate_share_seq", + "duplicate_share_id", + "non_positive_difficulty", + "non_positive_window_weight", + "non_positive_page_size", +) +ADVANCE_REJECTIONS = ( + "anchor_regression", + "delta_non_positive_difficulty", + "delta_ineligible_at_anchor", + "delta_repeats_eligible_share", + "delta_not_append", + "delta_order_not_increasing", +) +REJECTION_REASONS = FULL_SNAPSHOT_REJECTIONS + ADVANCE_REJECTIONS + +# How the shipped pipeline's rejections map onto the vocabulary. The shipped +# fold signals each condition only through an exception message, so the +# Python adapter classifies by exact message; a message not in this table is +# re-raised rather than guessed, so a new or reworded rejection path surfaces +# as a harness error instead of being quietly filed under a wrong category. +_REJECTION_MESSAGE_CATEGORIES = { + "full payout window contains duplicate share_seq": "duplicate_share_seq", + "full payout window contains duplicate share_id": "duplicate_share_id", + "full payout window contains non-positive difficulty": "non_positive_difficulty", + "window_weight must be positive": "non_positive_window_weight", + "page_size must be positive": "non_positive_page_size", + "snapshot anchor moved backwards": "anchor_regression", + "delta contains non-positive share difficulty": "delta_non_positive_difficulty", + "delta contains a share ineligible at the new anchor": "delta_ineligible_at_anchor", + "delta repeats a share eligible at the previous anchor": "delta_repeats_eligible_share", + "newly eligible share is not an append": "delta_not_append", + "delta share_seq order is not increasing": "delta_order_not_increasing", +} +assert set(_REJECTION_MESSAGE_CATEGORIES.values()) == set(REJECTION_REASONS) + + +def rejection_category_for_message(message: str) -> str | None: + """The vocabulary category for one shipped-pipeline rejection message. + + None for a message the table does not know, which callers must treat as + a harness fault rather than guess at. Shared with the fake serve builder + so its ``rejection`` field is classified exactly as the Python adapter + classifies. + """ + return _REJECTION_MESSAGE_CATEGORIES.get(message) + + +# --- declared integer domains ------------------------------------------------ +# +# The integer fields a record carries, in declaration order; the strings are +# outside the width contract. +_RECORD_INTEGER_FIELDS = ( + "share_seq", + "share_difficulty", + "network_difficulty", + "template_height", + "job_issued_at_ms", + "accepted_at_ms", + "ntime", +) + + +@dataclass(frozen=True) +class IntegerWidth: + """One declared integer width: an inclusive range, or unbounded.""" + + name: str + minimum: int | None + maximum: int | None + + def admits(self, value: int) -> bool: + return (self.minimum is None or value >= self.minimum) and ( + self.maximum is None or value <= self.maximum + ) + + +WIDTH_U32 = IntegerWidth("u32", 0, 2**32 - 1) +WIDTH_U64 = IntegerWidth("u64", 0, 2**64 - 1) +WIDTH_U128 = IntegerWidth("u128", 0, 2**128 - 1) +WIDTH_I64 = IntegerWidth("i64", -(2**63), 2**63 - 1) +WIDTH_UNBOUNDED = IntegerWidth("unbounded", None, None) + + +@dataclass(frozen=True) +class WindowPipelineIntegerDomain: + """The integer widths one backend declares for the pipeline's inputs. + + Every record integer, the anchors, ``window_weight`` and ``page_size`` + carry a declared width; ``difficulty_total`` bounds the fold's + accumulator -- the sum of ``share_difficulty`` over every record a case + feeds the backend (snapshot plus every delta) must fit it, which is a + conservative envelope of every partial sum the fold forms. A case with + any value outside its width is outside the domain, and only then may the + backend answer :class:`WindowPipelineUnsupported`. + """ + + share_seq: IntegerWidth + share_difficulty: IntegerWidth + network_difficulty: IntegerWidth + template_height: IntegerWidth + ntime: IntegerWidth + timestamps_ms: IntegerWidth + window_weight: IntegerWidth + page_size: IntegerWidth + difficulty_total: IntegerWidth + + def record_field_width(self, name: str) -> IntegerWidth: + if name in ("job_issued_at_ms", "accepted_at_ms"): + return self.timestamps_ms + if name not in _RECORD_INTEGER_FIELDS: + raise KeyError(f"{name!r} is not an integer record field") + return getattr(self, name) + + def exclusion(self, case: WindowPipelineCase) -> str | None: + """Why the case lies outside this domain, or None when it is inside. + + Walks the inputs in document order -- the top-level fields, the + snapshot records, then each advance's anchor and records -- and names + the first value outside its width, so the reason is stable and reads + as the field that caused it; the accumulator envelope is checked last. + """ + + def outside(label: str, value: int, width: IntegerWidth) -> str | None: + if width.admits(int(value)): + return None + return f"{label} {int(value)} is outside {width.name}" + + for label, value, width in ( + ("window_weight", case.window_weight, self.window_weight), + ("page_size", case.page_size, self.page_size), + ("snapshot anchor_job_issued_at_ms", case.anchor_job_issued_at_ms, self.timestamps_ms), + ): + problem = outside(label, value, width) + if problem is not None: + return problem + groups: list[tuple[str, int | None, tuple[AcceptedShareRecord, ...]]] = [ + ("snapshot", None, case.snapshot_records) + ] + groups.extend( + (f"advance[{index}]", step.anchor_job_issued_at_ms, step.delta_records) + for index, step in enumerate(case.advances) + ) + total = 0 + for label, anchor_ms, records in groups: + if anchor_ms is not None: + problem = outside(f"{label} anchor_job_issued_at_ms", anchor_ms, self.timestamps_ms) + if problem is not None: + return problem + for index, record in enumerate(records): + for name in _RECORD_INTEGER_FIELDS: + problem = outside( + f"{label} record[{index}].{name}", + getattr(record, name), + self.record_field_width(name), + ) + if problem is not None: + return problem + total += int(record.share_difficulty) + return outside("difficulty total over every record", total, self.difficulty_total) + + +# The shipped Python pipeline: arbitrary-precision integers throughout. +UNBOUNDED_INTEGER_DOMAIN = WindowPipelineIntegerDomain( + share_seq=WIDTH_UNBOUNDED, + share_difficulty=WIDTH_UNBOUNDED, + network_difficulty=WIDTH_UNBOUNDED, + template_height=WIDTH_UNBOUNDED, + ntime=WIDTH_UNBOUNDED, + timestamps_ms=WIDTH_UNBOUNDED, + window_weight=WIDTH_UNBOUNDED, + page_size=WIDTH_UNBOUNDED, + difficulty_total=WIDTH_UNBOUNDED, +) + +# The Rust window pipeline, mirroring the declared-width table in +# crates/qbit-prism/src/window.rs (the AcceptedShare field types plus the +# prepare_window request fields and the u128 accumulators). Changing either +# side without the other is a contract change and must show up here. +RUST_INTEGER_DOMAIN = WindowPipelineIntegerDomain( + share_seq=WIDTH_U64, + share_difficulty=WIDTH_U128, + network_difficulty=WIDTH_U128, + template_height=WIDTH_U64, + ntime=WIDTH_U32, + timestamps_ms=WIDTH_I64, + window_weight=WIDTH_U128, + page_size=WIDTH_U64, + difficulty_total=WIDTH_U128, +) + + +def corpus_record( + share_seq: int, + *, + share_difficulty: int, + job_issued_at_ms: int, + accepted_at_ms: int, + credit_policy: str | None = None, +) -> AcceptedShareRecord: + """The corpus's own deterministic record factory. + + Owned here rather than imported from the sibling golden test so that an + edit to that module's factory can never move this corpus: the frozen + reference is only as stable as the inputs that produced it, and the + fixture stores those inputs' hash (and, for small cases, the document). + The derivations deliberately match the sibling factory as of the first + freeze, so the reference needed no regeneration when ownership moved. + """ + return AcceptedShareRecord( + share_seq=share_seq, + share_id=f"share-{share_seq}", + miner_id=f"miner-{share_seq % 11}", + order_key=f"{share_seq % 11:02d}:miner-{share_seq % 11}", + p2mr_program_hex=f"{share_seq % 256:02x}" * 32, + share_difficulty=share_difficulty, + network_difficulty=1_000_000 + (share_seq % 17), + template_height=800_000 + (share_seq // 32), + job_id=f"job-{share_seq}", + job_issued_at_ms=job_issued_at_ms, + accepted_at_ms=accepted_at_ms, + ntime=1_700_000_000 + share_seq, + credit_policy=credit_policy, + ) + + +@dataclass(frozen=True) +class WindowPipelineAdvance: + """One append-only delta step folded through the incremental path.""" + + anchor_job_issued_at_ms: int + delta_records: tuple[AcceptedShareRecord, ...] + + +@dataclass(frozen=True) +class WindowPipelineCase: + """One corpus case: a full snapshot plus optional incremental advances. + + ``expected_rejection`` is the category the case was *built* to trigger + (None for a case the pipeline accepts). It is not input -- the reference + freezes what the shipped pipeline actually reports -- but the structural + test holds the two equal so a rejection case cannot quietly start passing + for a different reason, or none. + """ + + name: str + why: str + anchor_job_issued_at_ms: int + window_weight: int + page_size: int + snapshot_records: tuple[AcceptedShareRecord, ...] + advances: tuple[WindowPipelineAdvance, ...] = () + expected_rejection: str | None = None + + +@dataclass(frozen=True) +class WindowPipelineAdvanceStats: + """The bounded-work triple one advance step reports, seam-owned. + + Mirrors ``IncrementalWindowAdvanceStats`` without importing its type into + the contract. ``touched_pages`` counts pre-existing pages whose records + the advance inspected or rewrote: the last pre-existing page when the + delta was appended into it, and the pre-existing page that was partially + expired, deduplicated -- so a page both appended into and partially + expired counts once, a page appended into and then expired wholesale in + the same advance still counts (its records were rewritten), a freshly + allocated append page never counts even when partial expiry lands in it, + and pages expired wholesale without being appended into never count. + That is the shipped definition; it is derived from the work performed, + not from the retained result, which is why a re-implementation that + computes it from survivors gets the expired-append page wrong. + """ + + added_rows: int + expired_rows: int + touched_pages: int + + @classmethod + def from_shipped(cls, stats: IncrementalWindowAdvanceStats) -> WindowPipelineAdvanceStats: + return cls( + added_rows=int(stats.added_rows), + expired_rows=int(stats.expired_rows), + touched_pages=int(stats.touched_pages), + ) + + def as_document(self) -> dict[str, int]: + return { + "added_rows": self.added_rows, + "expired_rows": self.expired_rows, + "touched_pages": self.touched_pages, + } + + +@dataclass(frozen=True) +class WindowPipelineOutputs: + """The pipeline's outputs for one accepted corpus case. + + ``advance_stats`` carries one entry per advance step the case folded, in + order; it is empty for snapshot-only cases. + """ + + record_jsons: tuple[bytes, ...] + canonical_bytes: bytes + canonical_digest: str + spool_tail: bytes + advance_stats: tuple[WindowPipelineAdvanceStats, ...] = () + + +@dataclass(frozen=True) +class WindowPipelineRejection: + """The pipeline refused the case, for one of ``REJECTION_REASONS``.""" + + reason: str + + def __post_init__(self) -> None: + if self.reason not in REJECTION_REASONS: + raise ValueError( + f"unknown window-pipeline rejection reason {self.reason!r};" + f" the vocabulary is {list(REJECTION_REASONS)}" + ) + + +@dataclass(frozen=True) +class WindowPipelineUnsupported: + """The backend's declared integer domain does not include this input. + + Neither a rejection (the pipeline did not refuse the case; this backend + cannot represent it) nor a divergence (the frozen reference keeps the + shipped pipeline's result): it is the outcome the coordinator really + sees when the daemon answers ``out_of_range`` and the window is folded + in-process instead. ``reason`` is the backend's declared explanation, + carried for the report and never matched on -- whether the outcome is + legitimate is decided from the backend's declared domain alone, see + :func:`diverging_outputs`. + """ + + reason: str + + +WindowPipelineResult = ( + WindowPipelineOutputs | WindowPipelineRejection | WindowPipelineUnsupported +) + + +class WindowPipelineAdapter(Protocol): + """One backend driven over the corpus: inputs in, a result out. + + ``integer_domain`` is the domain the backend declares; a backend without + the attribute claims the unbounded domain, so any ``Unsupported`` it + returns is a divergence. + """ + + name: str + integer_domain: WindowPipelineIntegerDomain + + def run(self, case: WindowPipelineCase) -> WindowPipelineResult: ... + + +def adapter_integer_domain(adapter: object) -> WindowPipelineIntegerDomain: + """The domain a backend declares; one that declares nothing claims all of it.""" + domain = getattr(adapter, "integer_domain", UNBOUNDED_INTEGER_DOMAIN) + if not isinstance(domain, WindowPipelineIntegerDomain): + raise TypeError( + f"adapter {getattr(adapter, 'name', adapter)!r} declares an" + " integer_domain that is not a WindowPipelineIntegerDomain" + ) + return domain + + +def domain_exclusion(adapter: object, case: WindowPipelineCase) -> str | None: + """Why the case is outside the backend's declared domain, or None.""" + return adapter_integer_domain(adapter).exclusion(case) + + +def sha256_hex(data: bytes) -> str: + return hashlib.sha256(data).hexdigest() + + +def canonical_json_bytes(value: object) -> bytes: + """The pipeline's canonical encoder, byte-identical to the window pages.""" + return json.dumps(value, sort_keys=True, separators=(",", ":"), default=str).encode() + + +def record_input_json(record: AcceptedShareRecord) -> dict[str, object]: + """One record as implementation-neutral input data, every field explicit.""" + return {name: getattr(record, name) for name in _RECORD_INPUT_FIELDS} + + +def case_input_document(case: WindowPipelineCase) -> dict[str, object]: + """The full case as the JSON document a non-Python backend consumes. + + Record order is preserved verbatim: unsorted snapshot input is part of + what several cases exercise, so normalizing here would erase it. + """ + return { + "schema": CASE_INPUT_SCHEMA, + "name": case.name, + "window_weight": case.window_weight, + "page_size": case.page_size, + "snapshot": { + "anchor_job_issued_at_ms": case.anchor_job_issued_at_ms, + "records": [record_input_json(record) for record in case.snapshot_records], + }, + "advances": [ + { + "anchor_job_issued_at_ms": step.anchor_job_issued_at_ms, + "records": [record_input_json(record) for record in step.delta_records], + } + for step in case.advances + ], + } + + +def case_input_bytes(case: WindowPipelineCase) -> bytes: + return canonical_json_bytes(case_input_document(case)) + + +def case_input_sha256(case: WindowPipelineCase) -> str: + return sha256_hex(case_input_bytes(case)) + + +def final_anchor_job_issued_at_ms(case: WindowPipelineCase) -> int: + if case.advances: + return case.advances[-1].anchor_job_issued_at_ms + return case.anchor_job_issued_at_ms + + +def union_rebuild_case(case: WindowPipelineCase) -> WindowPipelineCase: + """The snapshot-only case whose bytes an incremental fold must land on.""" + union_records = case.snapshot_records + tuple( + record for step in case.advances for record in step.delta_records + ) + return WindowPipelineCase( + name=f"{case.name}-union-rebuild", + why=( + "from_full_snapshot over the union of the snapshot and every" + " advance delta, at the final anchor; the incremental path must" + " land on these exact bytes" + ), + anchor_job_issued_at_ms=final_anchor_job_issued_at_ms(case), + window_weight=case.window_weight, + page_size=case.page_size, + snapshot_records=union_records, + ) + + +def fold_case_with_stats( + case: WindowPipelineCase, +) -> tuple[IncrementalShareWindow, tuple[WindowPipelineAdvanceStats, ...]]: + """Drive the shipped fold: full snapshot, then each advance in order. + + Raises exactly what the shipped pipeline raises; :func:`run_python_pipeline` + is the seam-facing wrapper that classifies those into rejections. + """ + window = IncrementalShareWindow.from_full_snapshot( + list(case.snapshot_records), + anchor_job_issued_at_ms=case.anchor_job_issued_at_ms, + window_weight=case.window_weight, + page_size=case.page_size, + ) + stats: list[WindowPipelineAdvanceStats] = [] + for step in case.advances: + window, step_stats = window.advance( + list(step.delta_records), + anchor_job_issued_at_ms=step.anchor_job_issued_at_ms, + ) + stats.append(WindowPipelineAdvanceStats.from_shipped(step_stats)) + return window, tuple(stats) + + +def fold_case(case: WindowPipelineCase) -> IncrementalShareWindow: + """The folded window of a case the shipped pipeline accepts.""" + return fold_case_with_stats(case)[0] + + +def spool_tail_bytes(shares: list[dict[str, object]]) -> bytes: + """The exact spool payload tail the shipped serialization writes. + + Drives the real ``acquire_spooled_tail`` against an in-memory spool file + so the pinned bytes are the shipped write path's output, framing + included, not a reconstruction of it. + """ + serialization = _ShareWindowSerialization( + key=("window-pipeline-parity", len(shares), 0), + share_count=len(shares), + share_snapshot_sha256="window-pipeline-parity", + _spool_factory=io.BytesIO, + ) + leased = serialization.acquire_spooled_tail(shares) + if leased is None: + raise RuntimeError("in-memory spool lease unexpectedly unavailable") + spool, size = leased + spool.seek(0) + tail = spool.read() + if len(tail) != size: + raise RuntimeError(f"spool reported {size} bytes but holds {len(tail)}") + serialization.release_spooled_tail() + serialization.retire_spool() + return tail + + +def outputs_from_window( + window: IncrementalShareWindow, + advance_stats: tuple[WindowPipelineAdvanceStats, ...] = (), +) -> WindowPipelineOutputs: + """Capture the byte outputs from one folded window.""" + json_records = window.json_records() + record_jsons = tuple(canonical_json_bytes(record) for record in json_records) + fragments = [ + page.canonical_json_items + for page in json_records.pages + if page.canonical_json_items + ] + canonical_bytes = b"[" + b",".join(fragments) + b"]" + return WindowPipelineOutputs( + record_jsons=record_jsons, + canonical_bytes=canonical_bytes, + canonical_digest=json_records.canonical_json_sha256(), + spool_tail=spool_tail_bytes(list(json_records)), + advance_stats=advance_stats, + ) + + +def run_python_pipeline(case: WindowPipelineCase) -> WindowPipelineResult: + """The shipped pipeline over one case: outputs, or its rejection classified. + + Only the fold is guarded: a failure while capturing outputs (spool, JSON) + is a harness fault and propagates, never a rejection. + """ + try: + window, advance_stats = fold_case_with_stats(case) + except (ValueError, IncrementalWindowFallback) as exc: + category = rejection_category_for_message(str(exc)) + if category is None: + raise + return WindowPipelineRejection(category) + return outputs_from_window(window, advance_stats) + + +def record_stream_bytes(record_jsons: tuple[bytes, ...]) -> bytes: + # Injective: canonical encodings ASCII-escape every control character, so + # a raw newline can never occur inside one record's bytes. + return b"\n".join(record_jsons) + + +@dataclass(frozen=True) +class PythonWindowPipelineAdapter: + """The shipped Python pipeline, registered as the default backend.""" + + name: str = "python" + integer_domain: WindowPipelineIntegerDomain = UNBOUNDED_INTEGER_DOMAIN + + def run(self, case: WindowPipelineCase) -> WindowPipelineResult: + return run_python_pipeline(case) + + +def _split_record_fragments(items: bytes) -> tuple[bytes, ...]: + """Split one canonical items stream into its per-record byte spans. + + The spans come straight from the backend's bytes (``raw_decode`` only + reports where each JSON object ends), so a backend encoding divergence is + still visible in the returned fragments rather than being papered over by + a re-encode. + """ + if not items: + return () + text = items.decode("ascii") + decoder = json.JSONDecoder() + fragments: list[bytes] = [] + position = 0 + while position < len(text): + _, end = decoder.raw_decode(text, position) + fragments.append(text[position:end].encode("ascii")) + if end < len(text) and text[end] != ",": + raise ValueError( + f"canonical items stream has {text[end]!r} where a record" + " separator was expected" + ) + position = end + 1 + return tuple(fragments) + + +@dataclass(frozen=True) +class RustDaemonWindowPipelineAdapter: + """The Rust --serve daemon's prepare_window pipeline as a backend. + + Drives the real daemon binary over its JSONL protocol: one ``full`` + preparation for the snapshot, then one ``advance`` per delta step, holding + the canonical items stream exactly the way the coordinator's opaque + mirror does (drop the reported prefix, append the returned suffix) and + taking each advance's stats triple from its envelope. The three byte + outputs are taken from the daemon's bytes; only the spool tail runs the + shipped Python serialization over the lazily parsed records, which is + precisely the production fallback path under the Rust switch. + + Non-success envelopes are classified on their structure, never on the + diagnostic ``error`` text: ``fold_invalid`` and ``fallback`` carry the + daemon's ``rejection`` category, which is the oracle's own vocabulary and + is filed as :class:`WindowPipelineRejection` (an unknown category is a + harness error, not a guess); ``out_of_range`` -- an integer or difficulty + total outside the widths this adapter declares as ``RUST_INTEGER_DOMAIN`` + -- is :class:`WindowPipelineUnsupported`. Anything else is a harness + error, exactly as the coordinator treats it as a daemon anomaly. + + Selected with ``QBIT_WINDOW_PIPELINE_PARITY_ADAPTER=rust-daemon``; the + daemon binary resolves through ``prism_tool_command`` (``cargo run`` + unless ``PRISM_TOOL_BIN_DIR`` provides a prebuilt binary). + """ + + name: str = "rust-daemon" + integer_domain: WindowPipelineIntegerDomain = RUST_INTEGER_DOMAIN + + @staticmethod + def daemon_command() -> list[str]: + """Return the real deployment tool invocation used by this adapter.""" + return prism_tool_command("qbit-prism-build-audit-bundle") + [ + "--serve", + "--signing-key-seed-hex", + "42" * 32, + "--ledger-signing-key-seed-hex", + "43" * 32, + ] + + @staticmethod + def _wire_record(record: AcceptedShareRecord) -> dict[str, object]: + return record_input_json(record) + + @staticmethod + def _declined(envelope: dict[str, object]) -> WindowPipelineRejection | WindowPipelineUnsupported: + """Classify one non-``ok`` prepare_window envelope by its structure.""" + if envelope.get("request") != "prepare_window": + raise RuntimeError(f"prepare_window failed: {envelope}") + if envelope.get("out_of_range") is True: + return WindowPipelineUnsupported( + str(envelope.get("error") or "value outside the daemon's declared widths") + ) + if envelope.get("fold_invalid") is True or envelope.get("fallback") is True: + category = envelope.get("rejection") + if not isinstance(category, str): + raise RuntimeError( + f"daemon rejection carries no machine-readable category: {envelope}" + ) + # A category outside the vocabulary raises ValueError here: a + # harness error, never a quiet filing under the wrong reason. + return WindowPipelineRejection(category) + raise RuntimeError(f"prepare_window failed: {envelope}") + + def run(self, case: WindowPipelineCase) -> WindowPipelineResult: + process = subprocess.Popen( + self.daemon_command(), + stdin=subprocess.PIPE, + stdout=subprocess.PIPE, + ) + advance_stats: list[WindowPipelineAdvanceStats] = [] + try: + assert process.stdin is not None and process.stdout is not None + handshake = json.loads(process.stdout.readline()) + expected_handshake = { + "event": "handshake", + "tool": "qbit-prism-build-audit-bundle", + "protocol": PRISM_SERVE_BUILDER_PROTOCOL_VERSION, + } + announced_handshake = { + key: handshake.get(key) for key in expected_handshake + } + if announced_handshake != expected_handshake: + raise RuntimeError( + f"daemon announced handshake {announced_handshake!r}," + f" expected {expected_handshake!r}" + ) + + def exchange(request: dict[str, object]) -> tuple[dict[str, object], bytes | None]: + """One round trip; the payload is None when the daemon declined.""" + assert process.stdin is not None and process.stdout is not None + process.stdin.write( + json.dumps(request, separators=(",", ":")).encode("ascii") + b"\n" + ) + process.stdin.flush() + envelope = json.loads(process.stdout.readline()) + if envelope.get("ok") is not True: + return envelope, None + length_key = ( + "window_items_len" + if "window_items_len" in envelope + else "appended_items_len" + ) + payload = process.stdout.read(int(envelope[length_key])) + process.stdout.read(1) # the terminating newline + return envelope, payload + + envelope, items = exchange( + { + "request": "prepare_window", + "mode": "full", + "append_invalidation_epoch": 0, + "anchor_job_issued_at_ms": case.anchor_job_issued_at_ms, + "window_weight": case.window_weight, + "page_size": case.page_size, + "records": [ + self._wire_record(record) for record in case.snapshot_records + ], + } + ) + if items is None: + return self._declined(envelope) + for step in case.advances: + envelope, appended = exchange( + { + "request": "prepare_window", + "mode": "advance", + "append_invalidation_epoch": 0, + "anchor_job_issued_at_ms": step.anchor_job_issued_at_ms, + "base_digest": envelope["share_snapshot_sha256"], + "records": [ + self._wire_record(record) for record in step.delta_records + ], + } + ) + if appended is None: + return self._declined(envelope) + items = items[int(envelope["retained_drop_bytes"]) :] + appended + advance_stats.append( + WindowPipelineAdvanceStats( + added_rows=int(envelope["added_rows"]), + expired_rows=int(envelope["expired_rows"]), + touched_pages=int(envelope["touched_pages"]), + ) + ) + finally: + if process.stdin is not None: + process.stdin.close() + process.wait(timeout=30.0) + if process.stdout is not None: + process.stdout.close() + + record_jsons = _split_record_fragments(items) + if len(record_jsons) != int(envelope["record_count"]): + raise RuntimeError( + f"daemon reported {envelope['record_count']} records but the" + f" items stream holds {len(record_jsons)}" + ) + shares = [json.loads(fragment) for fragment in record_jsons] + return WindowPipelineOutputs( + record_jsons=record_jsons, + canonical_bytes=b"[" + items + b"]", + canonical_digest=str(envelope["share_snapshot_sha256"]), + spool_tail=spool_tail_bytes(shares), + advance_stats=tuple(advance_stats), + ) + + +_ADAPTER_FACTORIES: dict[str, Callable[[], WindowPipelineAdapter]] = { + DEFAULT_ADAPTER_NAME: PythonWindowPipelineAdapter, + "rust-daemon": RustDaemonWindowPipelineAdapter, +} + + +def register_adapter(name: str, factory: Callable[[], WindowPipelineAdapter]) -> None: + if name in _ADAPTER_FACTORIES: + raise ValueError(f"window-pipeline parity adapter {name!r} is already registered") + _ADAPTER_FACTORIES[name] = factory + + +def resolve_adapter(name: str | None = None) -> WindowPipelineAdapter: + """Resolve one backend; defaults to the shipped Python pipeline.""" + resolved = name or os.environ.get(ADAPTER_ENV_VAR) or DEFAULT_ADAPTER_NAME + factory = _ADAPTER_FACTORIES.get(resolved) + if factory is None: + raise ValueError( + f"unknown window-pipeline parity adapter {resolved!r};" + f" registered adapters: {sorted(_ADAPTER_FACTORIES)}" + ) + return factory() + + +# --- corpus ----------------------------------------------------------------- +# +# Every case is derived from an invariant of ``from_full_snapshot``/``advance`` +# or of the byte encodings, and says which. Seeded and deterministic: the +# builders below always produce the same records, so the frozen reference is +# reproducible from an unchanged tree. + + +def _empty_window_case() -> WindowPipelineCase: + return WindowPipelineCase( + name="empty-window", + why="zero eligible records: [] framing, digest of [], empty compact arrays", + anchor_job_issued_at_ms=1_000, + window_weight=10, + page_size=DEFAULT_INCREMENTAL_SHARE_WINDOW_PAGE_SIZE, + snapshot_records=(), + ) + + +def _single_record_case() -> WindowPipelineCase: + record = corpus_record( + 1, + share_difficulty=5, + job_issued_at_ms=1_999, + accepted_at_ms=2_000, + ) + return WindowPipelineCase( + name="single-record", + why=( + "one record with credit_policy None: to_prism_json omits the key" + " while the compact share tuple carries an explicit null -- the" + " null-vs-absent divergence surface" + ), + anchor_job_issued_at_ms=2_000, + window_weight=10, + page_size=DEFAULT_INCREMENTAL_SHARE_WINDOW_PAGE_SIZE, + snapshot_records=(record,), + ) + + +def _crossing_row_retained_case() -> WindowPipelineCase: + anchor_ms = 10_000 + records = ( + corpus_record(1, share_difficulty=2, job_issued_at_ms=anchor_ms - 4, accepted_at_ms=anchor_ms - 3), + corpus_record( + 2, + share_difficulty=3, + job_issued_at_ms=anchor_ms - 2, + accepted_at_ms=anchor_ms - 1, + credit_policy="stale-grace", + ), + corpus_record(3, share_difficulty=8, job_issued_at_ms=anchor_ms, accepted_at_ms=anchor_ms), + ) + return WindowPipelineCase( + name="crossing-row-retained", + why=( + "difficulties (2,3,8) at weight 10: the final whole share crossing" + " window_weight (seq 2) is deliberately retained, total 11 > 10;" + " an implementation that drops it diverges here" + ), + anchor_job_issued_at_ms=anchor_ms, + window_weight=10, + page_size=DEFAULT_INCREMENTAL_SHARE_WINDOW_PAGE_SIZE, + snapshot_records=records, + ) + + +def _crossing_row_exact_fit_case() -> WindowPipelineCase: + anchor_ms = 10_000 + records = ( + corpus_record(1, share_difficulty=2, job_issued_at_ms=anchor_ms - 4, accepted_at_ms=anchor_ms - 3), + corpus_record(2, share_difficulty=3, job_issued_at_ms=anchor_ms - 2, accepted_at_ms=anchor_ms - 1), + corpus_record(3, share_difficulty=7, job_issued_at_ms=anchor_ms, accepted_at_ms=anchor_ms), + ) + return WindowPipelineCase( + name="crossing-row-exact-fit", + why=( + "difficulties (2,3,7) at weight 10: cumulative weight lands" + " exactly on window_weight, so seq 1 is dropped and there is no" + " crossing row; the pair with crossing-row-retained separates the" + " >= cutoff from a wrong > cutoff" + ), + anchor_job_issued_at_ms=anchor_ms, + window_weight=10, + page_size=DEFAULT_INCREMENTAL_SHARE_WINDOW_PAGE_SIZE, + snapshot_records=records, + ) + + +def _eligibility_filtering_case() -> WindowPipelineCase: + anchor_ms = 30_000 + records = ( + corpus_record(1, share_difficulty=4, job_issued_at_ms=anchor_ms - 2, accepted_at_ms=anchor_ms - 1), + corpus_record(2, share_difficulty=5, job_issued_at_ms=anchor_ms + 1, accepted_at_ms=anchor_ms - 1), + corpus_record(3, share_difficulty=6, job_issued_at_ms=anchor_ms - 1, accepted_at_ms=anchor_ms + 1), + corpus_record(4, share_difficulty=7, job_issued_at_ms=anchor_ms + 2, accepted_at_ms=anchor_ms + 3), + corpus_record(5, share_difficulty=8, job_issued_at_ms=anchor_ms, accepted_at_ms=anchor_ms), + ) + return WindowPipelineCase( + name="eligibility-filtering", + why=( + "one record excluded by job_issued_at_ms alone, one by" + " accepted_at_ms alone, one by both; seq 5 sits exactly on the" + " anchor, pinning the inclusive <= comparisons" + ), + anchor_job_issued_at_ms=anchor_ms, + window_weight=100, + page_size=DEFAULT_INCREMENTAL_SHARE_WINDOW_PAGE_SIZE, + snapshot_records=records, + ) + + +def _unsorted_input_case() -> WindowPipelineCase: + rng = random.Random(PARITY_RANDOM_MASTER_SEED + 1) + anchor_ms = 40_000 + share_seqs = rng.sample(range(1, 100), 9) + records = [ + corpus_record( + share_seq, + share_difficulty=rng.choice((1, 2, 3, 7, 19)), + job_issued_at_ms=anchor_ms - rng.randint(0, 30), + accepted_at_ms=anchor_ms - rng.randint(0, 30), + credit_policy="stale-grace" if rng.randrange(4) == 0 else None, + ) + for share_seq in share_seqs + ] + return WindowPipelineCase( + name="unsorted-input", + why=( + "snapshot input arrives in non-ascending share_seq order with" + " non-contiguous seqs; the fold's sorted() is load-bearing and the" + " output ordering is pinned" + ), + anchor_job_issued_at_ms=anchor_ms, + window_weight=1_000_000, + page_size=DEFAULT_INCREMENTAL_SHARE_WINDOW_PAGE_SIZE, + snapshot_records=tuple(records), + ) + + +def _identity_reuse_case() -> WindowPipelineCase: + anchor_ms = 50_000 + base = [ + corpus_record( + share_seq, + share_difficulty=share_seq, + job_issued_at_ms=anchor_ms - share_seq, + accepted_at_ms=anchor_ms - share_seq, + credit_policy="stale-grace" if share_seq in (2, 5) else None, + ) + for share_seq in range(1, 7) + ] + # Force full (miner_id, order_key, p2mr_program_hex) identity collisions: + # the factory varies p2mr with share_seq, so collisions must be explicit. + base[3] = replace( + base[3], + miner_id=base[0].miner_id, + order_key=base[0].order_key, + p2mr_program_hex=base[0].p2mr_program_hex, + ) + base[5] = replace( + base[5], + miner_id=base[1].miner_id, + order_key=base[1].order_key, + p2mr_program_hex=base[1].p2mr_program_hex, + ) + return WindowPipelineCase( + name="credit-policy-identity-reuse", + why=( + "duplicate (miner_id, order_key, p2mr_program_hex) triples pin" + " identity deduplication and identity_index reuse in the compact" + " spool fragments, alongside mixed credit_policy values" + ), + anchor_job_issued_at_ms=anchor_ms, + window_weight=1_000_000, + page_size=DEFAULT_INCREMENTAL_SHARE_WINDOW_PAGE_SIZE, + snapshot_records=tuple(base), + ) + + +def _non_ascii_strings_case() -> WindowPipelineCase: + # share_id embeds the raw Stratum username, which survives payout-address + # fallback un-sanitized, and the ledger validates none of the string + # fields, so non-ASCII (including non-BMP), quotes, backslashes and + # control characters can all reach the durable record. json.dumps here + # runs with ensure_ascii=True, so every such character is \\u-escaped + # (non-BMP as surrogate pairs) -- a default a different JSON encoder is + # likely to get wrong. The fourth record widens the alphabet to the + # characters that separate Python's encoder from a "below 0x20 plus quote + # and backslash" re-encoder: DEL (0x7f, escaped by Python, raw in + # serde_json and most hand-rolled encoders), solidus (raw in Python, + # escaped by some encoders), the short escapes \\r \\b \\f, U+001F (the + # last short-less control), U+2028/U+2029 (escaped by Python, raw in + # encoders that only escape JavaScript-unsafe text), and space and + # uppercase, which must pass through untouched. They sit in the identity + # fields too so the spool's compact identities carry the same escapes. + anchor_ms = 60_000 + records = ( + replace( + corpus_record(1, share_difficulty=3, job_issued_at_ms=anchor_ms - 3, accepted_at_ms=anchor_ms - 2), + share_id="minér-é.workér:0a1b", + miner_id="minér-é", + order_key="00:minér-é", + ), + replace( + corpus_record( + 2, + share_difficulty=4, + job_issued_at_ms=anchor_ms - 2, + accepted_at_ms=anchor_ms - 1, + credit_policy="stale-grace", + ), + share_id='\U0001f680-share-"quoted"\\back\\slash', + job_id="job-\t\n\x01", + ), + corpus_record(3, share_difficulty=5, job_issued_at_ms=anchor_ms - 1, accepted_at_ms=anchor_ms), + replace( + corpus_record(4, share_difficulty=6, job_issued_at_ms=anchor_ms - 1, accepted_at_ms=anchor_ms), + share_id="MINER-A/worker 1:\x7f\r\b\x0c\x1f\u2028\u2029", + miner_id="MINER-A/worker 1\x7f", + order_key="04:MINER-A/worker 1\x7f", + ), + ) + return WindowPipelineCase( + name="non-ascii-strings", + why=( + "string fields carrying non-ASCII, a non-BMP code point, quotes," + " backslashes, control characters, DEL, solidus, CR/BS/FF, U+001F," + " U+2028/U+2029, space and uppercase: pins the ensure_ascii escape" + " rules -- surrogate pairs, \\u007f for DEL, raw solidus, short" + " escapes, and pass-through of printable ASCII" + ), + anchor_job_issued_at_ms=anchor_ms, + window_weight=1_000_000, + page_size=DEFAULT_INCREMENTAL_SHARE_WINDOW_PAGE_SIZE, + snapshot_records=records, + ) + + +def _page_boundary_case(retained_count: int, seed_offset: int) -> WindowPipelineCase: + rng = random.Random(PARITY_RANDOM_MASTER_SEED + seed_offset) + anchor_ms = 70_000 + retained_count + records = tuple( + corpus_record( + share_seq, + share_difficulty=rng.choice((1, 2, 3, 7, 19)), + job_issued_at_ms=anchor_ms - rng.randint(0, 30), + accepted_at_ms=anchor_ms - rng.randint(0, 30), + credit_policy="stale-grace" if rng.randrange(4) == 0 else None, + ) + for share_seq in range(1, retained_count + 1) + ) + return WindowPipelineCase( + name=f"page-boundary-{retained_count}", + why=( + f"exactly {retained_count} retained records around the default" + f" page size {DEFAULT_INCREMENTAL_SHARE_WINDOW_PAGE_SIZE}: pins" + " page splitting and the digest's page-fragment framing at the" + " boundary" + ), + anchor_job_issued_at_ms=anchor_ms, + window_weight=10**9, + page_size=DEFAULT_INCREMENTAL_SHARE_WINDOW_PAGE_SIZE, + snapshot_records=records, + ) + + +def _multi_page_case() -> WindowPipelineCase: + rng = random.Random(PARITY_RANDOM_MASTER_SEED + 11) + anchor_ms = 90_000 + record_count = 1_800 + records = [ + corpus_record( + share_seq, + share_difficulty=rng.choice((1, 2, 3, 7, 19)), + job_issued_at_ms=anchor_ms - rng.randint(0, 30), + accepted_at_ms=anchor_ms - rng.randint(0, 30), + credit_policy="stale-grace" if rng.randrange(4) == 0 else None, + ) + for share_seq in range(1, record_count + 1) + ] + # Weight chosen so the newest 1699 shares miss the weight by one and the + # 1700th (by seq: 101) is the retained crossing row, landing the cutoff in + # a page interior across four pages. + window_weight = sum(record.share_difficulty for record in records[101:]) + 1 + shuffled = list(records) + rng.shuffle(shuffled) + return WindowPipelineCase( + name="multi-page-interior-cutoff", + why=( + "1800 unsorted input records with the retention cutoff landing" + " inside a page interior: pins multi-page fragment concatenation" + " and a crossing row far from page edges" + ), + anchor_job_issued_at_ms=anchor_ms, + window_weight=window_weight, + page_size=DEFAULT_INCREMENTAL_SHARE_WINDOW_PAGE_SIZE, + snapshot_records=tuple(shuffled), + ) + + +def _incremental_small_case() -> WindowPipelineCase: + anchor_ms = 80_000 + difficulties = {1: 5, 2: 1, 3: 1, 4: 2, 5: 1, 6: 3, 7: 1} + snapshot = [ + corpus_record( + share_seq, + share_difficulty=difficulty, + job_issued_at_ms=anchor_ms - share_seq, + accepted_at_ms=anchor_ms - share_seq, + credit_policy="stale-grace" if share_seq == 6 else None, + ) + for share_seq, difficulty in difficulties.items() + ] + # Never eligible at any anchor this case reaches: the incremental path + # never sees it in a delta, and the union rebuild must exclude it too. + snapshot.append( + corpus_record(8, share_difficulty=100, job_issued_at_ms=10**9, accepted_at_ms=10**9) + ) + first_advance = WindowPipelineAdvance( + anchor_job_issued_at_ms=anchor_ms + 1, + delta_records=( + corpus_record(9, share_difficulty=2, job_issued_at_ms=anchor_ms + 1, accepted_at_ms=anchor_ms - 1), + corpus_record(10, share_difficulty=4, job_issued_at_ms=anchor_ms - 10, accepted_at_ms=anchor_ms + 1), + ), + ) + second_advance = WindowPipelineAdvance( + anchor_job_issued_at_ms=anchor_ms + 10, + delta_records=( + corpus_record(11, share_difficulty=1, job_issued_at_ms=anchor_ms + 5, accepted_at_ms=anchor_ms + 2), + corpus_record( + 12, + share_difficulty=1, + job_issued_at_ms=anchor_ms + 10, + accepted_at_ms=anchor_ms - 5, + credit_policy="stale-grace", + ), + corpus_record(13, share_difficulty=1, job_issued_at_ms=anchor_ms, accepted_at_ms=anchor_ms + 10), + corpus_record( + 14, + share_difficulty=1, + job_issued_at_ms=anchor_ms + 9, + accepted_at_ms=anchor_ms + 9, + credit_policy="stale-grace", + ), + ), + ) + return WindowPipelineCase( + name="incremental-two-advances", + why=( + "two advance() steps over a page_size-3 window: append into a" + " partially filled page, spill into new pages, expire a whole head" + " page and partially expire the next; the folded bytes must equal" + " a full rebuild over the union" + ), + anchor_job_issued_at_ms=anchor_ms, + window_weight=9, + page_size=3, + snapshot_records=tuple(snapshot), + advances=(first_advance, second_advance), + ) + + +def _bulk_seeded_case() -> WindowPipelineCase: + rng = random.Random(PARITY_RANDOM_MASTER_SEED + 12) + anchor_ms = 1_000_000 + advance_anchor_ms = anchor_ms + 10 + snapshot: list[AcceptedShareRecord] = [] + for share_seq in range(1, 4_001): + if rng.random() < 0.1: + # Ineligible at the snapshot anchor and at every later anchor in + # this case, since the incremental path never re-delivers it. + lateness = rng.choice(("job", "accepted", "both")) + job_ms = ( + anchor_ms + rng.randint(11, 500) + if lateness in ("job", "both") + else anchor_ms - rng.randint(0, 30) + ) + accepted_ms = ( + anchor_ms + rng.randint(11, 500) + if lateness in ("accepted", "both") + else anchor_ms - rng.randint(0, 30) + ) + else: + job_ms = anchor_ms - rng.randint(0, 30) + accepted_ms = anchor_ms - rng.randint(0, 30) + snapshot.append( + corpus_record( + share_seq, + share_difficulty=rng.choice((1, 2, 3, 7, 19, 101, 1_000)), + job_issued_at_ms=job_ms, + accepted_at_ms=accepted_ms, + credit_policy="stale-grace" if rng.randrange(5) == 0 else None, + ) + ) + rng.shuffle(snapshot) + delta: list[AcceptedShareRecord] = [] + for share_seq in range(4_001, 4_701): + if rng.randrange(2): + job_ms = rng.randint(anchor_ms + 1, advance_anchor_ms) + accepted_ms = advance_anchor_ms - rng.randint(0, 40) + else: + job_ms = advance_anchor_ms - rng.randint(0, 40) + accepted_ms = rng.randint(anchor_ms + 1, advance_anchor_ms) + delta.append( + corpus_record( + share_seq, + share_difficulty=rng.choice((1, 2, 3, 7, 19, 101, 1_000)), + job_issued_at_ms=job_ms, + accepted_at_ms=accepted_ms, + credit_policy="stale-grace" if rng.randrange(5) == 0 else None, + ) + ) + return WindowPipelineCase( + name="bulk-seeded", + why=( + "4000 seeded snapshot records (roughly a tenth ineligible) plus a" + " 700-record advance: representative page-fragment concatenation," + " identity deduplication at natural factory collisions, and head" + " expiry at scale" + ), + anchor_job_issued_at_ms=anchor_ms, + window_weight=450_000, + page_size=DEFAULT_INCREMENTAL_SHARE_WINDOW_PAGE_SIZE, + snapshot_records=tuple(snapshot), + advances=( + WindowPipelineAdvance( + anchor_job_issued_at_ms=advance_anchor_ms, + delta_records=tuple(delta), + ), + ), + ) + + +# A realistic production anchor: epoch milliseconds in 2025, above 2^40. The +# previous cases all sit below 2^31, which is exactly why a 32-bit backend +# could pass them; every width case below runs at this scale. +_REALISTIC_ANCHOR_MS = 1_755_000_000_000 + + +def _wide_integers_case() -> WindowPipelineCase: + anchor_ms = _REALISTIC_ANCHOR_MS + # Every value sits far above the 2^31 / 2^53 / 2^64 boundaries at which a + # wrapping or saturating 32/53/64-bit backend breaks, yet inside every + # declared Rust width (share_seq and template_height u64, difficulties + # and window_weight u128, ntime u32), so both backends must produce these + # exact bytes. Retention runs newest-first until the accumulated weight + # reaches window_weight. The three newest (2^53+1, 2^63, 2^64+1) sum to + # 2^64+2^63+2^53+2, below window_weight only because of its 2^127 term, + # so the oldest (2^127+3) is the retained crossing row and all four + # survive: a backend that wraps or saturates at 64 bits (or truncates + # window_weight to 64 bits) stops after three, and one that clamps the + # inputs renders different digits even where it keeps the count. The + # difficulty total over every record stays below 2^128, inside the u128 + # accumulator, so the case is inside the declared domain entirely. + records = ( + replace( + corpus_record(1, share_difficulty=2**127 + 3, job_issued_at_ms=anchor_ms - 3, accepted_at_ms=anchor_ms - 2), + network_difficulty=2**128 - 1, + template_height=2**62, + ntime=2**32 - 1, + ), + replace( + corpus_record(2**31, share_difficulty=2**64 + 1, job_issued_at_ms=anchor_ms - 2, accepted_at_ms=anchor_ms - 1), + network_difficulty=2**64 + 1, + template_height=2**31 + 1, + ntime=2**31, + ), + replace( + corpus_record(2**32 + 5, share_difficulty=2**63, job_issued_at_ms=anchor_ms - 1, accepted_at_ms=anchor_ms - 1), + network_difficulty=2**64 - 1, + template_height=0, + ntime=0, + ), + # share_seq at u64's maximum (the factory's derived template_height + # lands near 2^59); ntime must be set explicitly because the + # factory's default would add the seq to it. + replace( + corpus_record(2**64 - 1, share_difficulty=2**53 + 1, job_issued_at_ms=anchor_ms, accepted_at_ms=anchor_ms), + network_difficulty=2**53 + 1, + ntime=2**31 + 1, + ), + # Excluded by a one-millisecond margin at 2^40 scale: a backend + # comparing truncated or float-rounded timestamps includes it. + corpus_record(7, share_difficulty=1, job_issued_at_ms=anchor_ms + 1, accepted_at_ms=anchor_ms), + ) + return WindowPipelineCase( + name="wide-integers", + why=( + "every integer field above 2^32 at a realistic epoch-ms anchor," + " all inside the declared Rust widths: share_difficulty across" + " 2^53+1, 2^63, 2^64+1 and 2^127+3, network_difficulty up to" + " 2^128-1, template_height 2^31+1, 2^62 and near 2^59, ntime" + " 2^31 to 2^32-1, share_seq up to 2^64-1, a zero" + " template_height/ntime, and window_weight 2^127+2^64+2^63+4 so" + " the retention cutoff depends on exact arithmetic above 2^64;" + " pins full-digit rendering and non-wrapping, non-saturating," + " non-float accumulation, with full parity enforced on every" + " backend" + ), + anchor_job_issued_at_ms=anchor_ms, + window_weight=2**127 + 2**64 + 2**63 + 4, + page_size=DEFAULT_INCREMENTAL_SHARE_WINDOW_PAGE_SIZE, + snapshot_records=records, + ) + + +def _difficulty_beyond_u128_case() -> WindowPipelineCase: + anchor_ms = _REALISTIC_ANCHOR_MS + # Newest-first: 2^63, then 2^64-1, then 2^128+1 crosses window_weight and + # is retained; the oldest (2^127) is dropped. The retained total is above + # 2^128 and so is window_weight itself, so neither the inputs nor the + # accumulator fit a 128-bit integer: a backend that clamps at u128::MAX + # renders different digits for the crossing row and for the numeric(78,0) + # maximum network_difficulty even though its retained count agrees. + records = ( + replace( + corpus_record(1, share_difficulty=2**127, job_issued_at_ms=anchor_ms - 3, accepted_at_ms=anchor_ms - 3), + network_difficulty=2**128 - 1, + ), + replace( + corpus_record(2, share_difficulty=2**128 + 1, job_issued_at_ms=anchor_ms - 2, accepted_at_ms=anchor_ms - 2), + network_difficulty=10**78 - 1, + ), + replace( + corpus_record(3, share_difficulty=2**64 - 1, job_issued_at_ms=anchor_ms - 1, accepted_at_ms=anchor_ms - 1), + network_difficulty=2**128, + ), + replace( + corpus_record(4, share_difficulty=2**63, job_issued_at_ms=anchor_ms, accepted_at_ms=anchor_ms), + network_difficulty=2**128 - 1, + ), + ) + return WindowPipelineCase( + name="difficulty-beyond-u128", + why=( + "numeric(78,0) width taken literally: share_difficulty 2^128+1 as" + " the retained crossing row, network_difficulty at 2^128-1, 2^128" + " and 10^78-1, window_weight and the retained total above 2^128" + " -- outside the declared u128 widths, so a backend declaring" + " them answers Unsupported here while the frozen bytes stay" + " Python's; one that clamps at u128::MAX, saturating or not," + " renders different digits and fails -- see the module docstring" + ), + anchor_job_issued_at_ms=anchor_ms, + window_weight=2**128 + 2**64, + page_size=DEFAULT_INCREMENTAL_SHARE_WINDOW_PAGE_SIZE, + snapshot_records=records, + ) + + +def _ntime_beyond_u32_case() -> WindowPipelineCase: + anchor_ms = _REALISTIC_ANCHOR_MS + # ntime is bigint in the ledger but a 32-bit header field and u32 in the + # declared Rust domain: 2^32 (one above the width) and bigint's maximum + # both lie outside it while every other field is ordinary, so this is + # the one width the case exercises. + records = ( + replace( + corpus_record(1, share_difficulty=3, job_issued_at_ms=anchor_ms - 2, accepted_at_ms=anchor_ms - 1), + ntime=2**32, + ), + replace( + corpus_record( + 2, + share_difficulty=4, + job_issued_at_ms=anchor_ms - 1, + accepted_at_ms=anchor_ms - 1, + credit_policy="stale-grace", + ), + ntime=2**63 - 1, + ), + corpus_record(3, share_difficulty=5, job_issued_at_ms=anchor_ms, accepted_at_ms=anchor_ms), + ) + return WindowPipelineCase( + name="ntime-beyond-u32", + why=( + "ntime at 2^32 and 2^63-1 (bigint's maximum) on otherwise" + " ordinary records at a realistic anchor: outside the declared" + " u32 width alone, so a backend declaring it answers Unsupported" + " here while the frozen bytes stay Python's full-digit rendering;" + " one that truncates ntime to 32 bits renders different digits" + " and fails" + ), + anchor_job_issued_at_ms=anchor_ms, + window_weight=1_000, + page_size=DEFAULT_INCREMENTAL_SHARE_WINDOW_PAGE_SIZE, + snapshot_records=records, + ) + + +def _advance_exact_fit_expiry_case() -> WindowPipelineCase: + anchor_ms = 100_000 + snapshot = tuple( + corpus_record(share_seq, share_difficulty=4, job_issued_at_ms=anchor_ms - share_seq, accepted_at_ms=anchor_ms - share_seq) + for share_seq in (1, 2, 3) + ) + return WindowPipelineCase( + name="advance-exact-fit-expiry", + why=( + "three records of difficulty 4 at weight 10 (all retained, total" + " 12) plus one delta of difficulty 2: 14-4 = 10 >= 10 expires the" + " head row exactly at the weight, retaining seqs 2..4 at total 10;" + " an advance-side expiry using > instead of >= retains all four." + " The crossing-row-exact-fit sibling pins the snapshot-side twin" + " of this comparison; the advance loops are separate code in any" + " re-implementation" + ), + anchor_job_issued_at_ms=anchor_ms, + window_weight=10, + page_size=DEFAULT_INCREMENTAL_SHARE_WINDOW_PAGE_SIZE, + snapshot_records=snapshot, + advances=( + WindowPipelineAdvance( + anchor_job_issued_at_ms=anchor_ms + 1, + delta_records=( + corpus_record(4, share_difficulty=2, job_issued_at_ms=anchor_ms + 1, accepted_at_ms=anchor_ms + 1), + ), + ), + ), + ) + + +def _advance_delta_exceeds_window_case() -> WindowPipelineCase: + anchor_ms = 110_000 + snapshot = tuple( + corpus_record(share_seq, share_difficulty=3, job_issued_at_ms=anchor_ms - share_seq, accepted_at_ms=anchor_ms - share_seq) + for share_seq in (1, 2, 3) + ) + delta = tuple( + corpus_record(share_seq, share_difficulty=5, job_issued_at_ms=anchor_ms + 1, accepted_at_ms=anchor_ms + 1) + for share_seq in (4, 5, 6, 7, 8) + ) + return WindowPipelineCase( + name="advance-delta-exceeds-window", + why=( + "weight 8, page_size 2, three retained records of difficulty 3 in" + " pages [1,2],[3], then one advance of five records of difficulty" + " 5 (25 alone outweighs the window): the delta fills page [3,4]" + " and allocates [5,6],[7,8]; expiry must run across the appended" + " pages too, expiring every old page and three of the five new" + " rows to retain [7,8] at total 10. A backend that expires only" + " among pre-existing pages -- the natural shape of a drop-prefix /" + " append-suffix protocol -- retains four. touched_pages is 1: the" + " page appended into counts although this same advance then" + " expires it wholesale" + ), + anchor_job_issued_at_ms=anchor_ms, + window_weight=8, + page_size=2, + snapshot_records=snapshot, + advances=(WindowPipelineAdvance(anchor_job_issued_at_ms=anchor_ms + 1, delta_records=delta),), + ) + + +def _advance_partial_expiry_in_appended_page_case() -> WindowPipelineCase: + anchor_ms = 120_000 + snapshot = tuple( + corpus_record(share_seq, share_difficulty=3, job_issued_at_ms=anchor_ms - share_seq, accepted_at_ms=anchor_ms - share_seq) + for share_seq in (1, 2, 3) + ) + delta = tuple( + corpus_record(share_seq, share_difficulty=difficulty, job_issued_at_ms=anchor_ms + 1, accepted_at_ms=anchor_ms + 1) + for share_seq, difficulty in zip((4, 5, 6, 7, 8), (5, 5, 5, 2, 9)) + ) + return WindowPipelineCase( + name="advance-partial-expiry-in-appended-page", + why=( + "as advance-delta-exceeds-window but with delta difficulties" + " (5,5,5,2,9): whole-page expiry stops at the freshly allocated" + " page [7,8] (total 11) and the partial loop then expires seq 7" + " inside it, retaining [8] at total 9; touched_pages stays 1" + " because a freshly allocated append page never counts even when" + " partial expiry lands in it" + ), + anchor_job_issued_at_ms=anchor_ms, + window_weight=8, + page_size=2, + snapshot_records=snapshot, + advances=(WindowPipelineAdvance(anchor_job_issued_at_ms=anchor_ms + 1, delta_records=delta),), + ) + + +def _advance_from_empty_window_case() -> WindowPipelineCase: + anchor_ms = 130_000 + return WindowPipelineCase( + name="advance-from-empty-window", + why=( + "an empty window (no pages, no last page to append into) advanced" + " with a one-record delta: the record lands in a freshly allocated" + " page, nothing expires, touched_pages is 0; a backend that" + " unwraps the last page fails here" + ), + anchor_job_issued_at_ms=anchor_ms, + window_weight=10, + page_size=DEFAULT_INCREMENTAL_SHARE_WINDOW_PAGE_SIZE, + snapshot_records=(), + advances=( + WindowPipelineAdvance( + anchor_job_issued_at_ms=anchor_ms + 1, + delta_records=( + corpus_record(1, share_difficulty=5, job_issued_at_ms=anchor_ms + 1, accepted_at_ms=anchor_ms + 1), + ), + ), + ), + ) + + +def _advance_empty_delta_case() -> WindowPipelineCase: + anchor_ms = 140_000 + return WindowPipelineCase( + name="advance-empty-delta", + why=( + "one retained record advanced to a later anchor with an empty" + " delta: bytes identical to the snapshot, stats all zero; a backend" + " that allocates an empty page and frames a trailing separator, or" + " re-runs expiry with a stale total, diverges" + ), + anchor_job_issued_at_ms=anchor_ms, + window_weight=10, + page_size=DEFAULT_INCREMENTAL_SHARE_WINDOW_PAGE_SIZE, + snapshot_records=( + corpus_record(1, share_difficulty=5, job_issued_at_ms=anchor_ms, accepted_at_ms=anchor_ms), + ), + advances=(WindowPipelineAdvance(anchor_job_issued_at_ms=anchor_ms + 1, delta_records=()),), + ) + + +def _advance_equal_anchor_case() -> WindowPipelineCase: + anchor_ms = 150_000 + return WindowPipelineCase( + name="advance-equal-anchor", + why=( + "an advance at the same anchor as the snapshot: the regression" + " check is strictly <, so this is accepted (only an empty delta" + " can be, since no record is both newly eligible and eligible at" + " the same anchor); a backend rejecting on <= diverges" + ), + anchor_job_issued_at_ms=anchor_ms, + window_weight=10, + page_size=DEFAULT_INCREMENTAL_SHARE_WINDOW_PAGE_SIZE, + snapshot_records=( + corpus_record(1, share_difficulty=5, job_issued_at_ms=anchor_ms, accepted_at_ms=anchor_ms), + ), + advances=(WindowPipelineAdvance(anchor_job_issued_at_ms=anchor_ms, delta_records=()),), + ) + + +# --- rejection cases -------------------------------------------------------- +# +# One case per condition the shipped pipeline refuses, each built to trigger +# exactly that condition and nothing earlier in the check order. The three +# from_full_snapshot duplicate/non-positive shapes are unreachable from the +# database (PK, UNIQUE, CHECK), but the in-memory ledger and the daemon +# protocol are not the database, so they are pinned too. What is pinned is +# the rejection itself, not the coordinator's recovery from it. + +_REJECTION_ANCHOR_MS = 200_000 + + +def _rejection_case( + name: str, + category: str, + why: str, + *, + snapshot_records: tuple[AcceptedShareRecord, ...], + window_weight: int = 100, + page_size: int = DEFAULT_INCREMENTAL_SHARE_WINDOW_PAGE_SIZE, + advances: tuple[WindowPipelineAdvance, ...] = (), +) -> WindowPipelineCase: + return WindowPipelineCase( + name=name, + why=why, + anchor_job_issued_at_ms=_REJECTION_ANCHOR_MS, + window_weight=window_weight, + page_size=page_size, + snapshot_records=snapshot_records, + advances=advances, + expected_rejection=category, + ) + + +def _eligible(share_seq: int, *, share_difficulty: int = 1, share_id: str | None = None) -> AcceptedShareRecord: + """A record eligible at the rejection anchor, at one ms per seq of age.""" + record = corpus_record( + share_seq, + share_difficulty=share_difficulty, + job_issued_at_ms=_REJECTION_ANCHOR_MS - share_seq, + accepted_at_ms=_REJECTION_ANCHOR_MS - share_seq, + ) + return record if share_id is None else replace(record, share_id=share_id) + + +def _newly_eligible(share_seq: int, *, share_difficulty: int = 1, job_offset_ms: int = 1, accepted_offset_ms: int = 1) -> AcceptedShareRecord: + """A delta record first eligible at anchor+1 (both stamps above the old anchor).""" + return corpus_record( + share_seq, + share_difficulty=share_difficulty, + job_issued_at_ms=_REJECTION_ANCHOR_MS + job_offset_ms, + accepted_at_ms=_REJECTION_ANCHOR_MS + accepted_offset_ms, + ) + + +def _full_snapshot_rejection_cases() -> tuple[WindowPipelineCase, ...]: + return ( + _rejection_case( + "reject-duplicate-share-seq", + "duplicate_share_seq", + "two eligible records share share_seq 1 under distinct share_ids;" + " the sorted-order duplicate check fires before the share_id one", + snapshot_records=(_eligible(1), _eligible(1, share_difficulty=2, share_id="share-1-again")), + ), + _rejection_case( + "reject-duplicate-share-id", + "duplicate_share_id", + "seqs 1 and 2 carry the same share_id; distinct seqs so only the" + " share_id check can fire", + snapshot_records=(_eligible(1), _eligible(2, share_id="share-1")), + ), + _rejection_case( + "reject-non-positive-difficulty", + "non_positive_difficulty", + "one eligible record with share_difficulty 0 (the check runs on" + " eligible records only, so it must be eligible to be refused)", + snapshot_records=(_eligible(1, share_difficulty=0),), + ), + _rejection_case( + "reject-non-positive-window-weight", + "non_positive_window_weight", + "window_weight 0 with one ordinary eligible record; refused before" + " any record is examined", + snapshot_records=(_eligible(1),), + window_weight=0, + ), + _rejection_case( + "reject-non-positive-page-size", + "non_positive_page_size", + "page_size 0 with one ordinary eligible record; refused before any" + " record is examined", + snapshot_records=(_eligible(1),), + page_size=0, + ), + ) + + +def _advance_rejection_cases() -> tuple[WindowPipelineCase, ...]: + # Seqs 1 and 3 retained at the anchor; seq 2 is the gap a late-visible + # row would fill. + retained = (_eligible(1), _eligible(3)) + + def advance(*delta: AcceptedShareRecord, anchor_offset_ms: int = 1) -> tuple[WindowPipelineAdvance, ...]: + return ( + WindowPipelineAdvance( + anchor_job_issued_at_ms=_REJECTION_ANCHOR_MS + anchor_offset_ms, + delta_records=delta, + ), + ) + + return ( + _rejection_case( + "reject-anchor-regression", + "anchor_regression", + "an advance whose anchor is one ms earlier than the window's, with" + " an empty delta: the anchor check fires before any record check", + snapshot_records=retained, + advances=advance(anchor_offset_ms=-1), + ), + _rejection_case( + "reject-delta-non-positive-difficulty", + "delta_non_positive_difficulty", + "a newly eligible delta record of difficulty 0; the difficulty" + " check is the first per-record check", + snapshot_records=retained, + advances=advance(_newly_eligible(4, share_difficulty=0)), + ), + _rejection_case( + "reject-delta-ineligible-job-only", + "delta_ineligible_at_anchor", + "a delta record whose job_issued_at_ms is one ms past the new" + " anchor while accepted_at_ms is on it: ineligible by job alone", + snapshot_records=retained, + advances=advance(_newly_eligible(4, job_offset_ms=2, accepted_offset_ms=1)), + ), + _rejection_case( + "reject-delta-ineligible-accepted-only", + "delta_ineligible_at_anchor", + "a delta record whose accepted_at_ms is one ms past the new anchor" + " while job_issued_at_ms is on it: ineligible by accepted alone", + snapshot_records=retained, + advances=advance(_newly_eligible(4, job_offset_ms=1, accepted_offset_ms=2)), + ), + _rejection_case( + "reject-delta-repeats-retained-share", + "delta_repeats_eligible_share", + "the delta re-delivers seq 3, a share eligible at the previous" + " anchor and still inside the window: refused as a repeat before" + " the append-order check could see its seq", + snapshot_records=retained, + advances=advance(_eligible(3)), + ), + _rejection_case( + "reject-delta-not-append", + "delta_not_append", + "a newly eligible record with seq 2, below the window's last seq" + " 3: the late-visible row that cannot be folded as an append and" + " in production forces the full rescan", + snapshot_records=retained, + advances=advance(_newly_eligible(2)), + ), + _rejection_case( + "reject-delta-order-not-increasing", + "delta_order_not_increasing", + "two newly eligible records delivered as seqs 5 then 4, both" + " above the window's last seq so only the intra-delta order check" + " can fire", + snapshot_records=retained, + advances=advance(_newly_eligible(5), _newly_eligible(4)), + ), + ) + + +def build_corpus() -> tuple[WindowPipelineCase, ...]: + """Every corpus case, deterministic and in a stable order.""" + return ( + _empty_window_case(), + _single_record_case(), + _crossing_row_retained_case(), + _crossing_row_exact_fit_case(), + _eligibility_filtering_case(), + _unsorted_input_case(), + _identity_reuse_case(), + _non_ascii_strings_case(), + _page_boundary_case(DEFAULT_INCREMENTAL_SHARE_WINDOW_PAGE_SIZE - 1, seed_offset=8), + _page_boundary_case(DEFAULT_INCREMENTAL_SHARE_WINDOW_PAGE_SIZE, seed_offset=9), + _page_boundary_case(DEFAULT_INCREMENTAL_SHARE_WINDOW_PAGE_SIZE + 1, seed_offset=10), + _multi_page_case(), + _incremental_small_case(), + _bulk_seeded_case(), + _wide_integers_case(), + _difficulty_beyond_u128_case(), + _ntime_beyond_u32_case(), + _advance_exact_fit_expiry_case(), + _advance_delta_exceeds_window_case(), + _advance_partial_expiry_in_appended_page_case(), + _advance_from_empty_window_case(), + _advance_empty_delta_case(), + _advance_equal_anchor_case(), + *_full_snapshot_rejection_cases(), + *_advance_rejection_cases(), + ) + + +# --- frozen reference ------------------------------------------------------- + + +def reference_case_entry(case: WindowPipelineCase, result: WindowPipelineResult) -> dict[str, object]: + entry: dict[str, object] = { + "why": case.why, + "input_sha256": case_input_sha256(case), + } + literals: dict[str, object] = {} + if len(case_input_bytes(case)) <= LITERAL_PIN_MAX_CANONICAL_BYTES: + literals["input"] = case_input_document(case) + if isinstance(result, WindowPipelineUnsupported): + raise RuntimeError( + f"case {case.name}: the shipped pipeline declared the case" + f" unsupported ({result.reason}); the reference freezes only the" + " shipped pipeline's outputs or rejections" + ) + if isinstance(result, WindowPipelineRejection): + entry["rejected"] = result.reason + else: + outputs = result + stream = record_stream_bytes(outputs.record_jsons) + if outputs.canonical_digest != sha256_hex(outputs.canonical_bytes): + raise RuntimeError( + f"case {case.name}: canonical digest does not hash the framed" + " canonical bytes; the streaming model no longer matches the" + " shipped pipeline" + ) + entry.update( + { + "record_count": len(outputs.record_jsons), + "record_stream_len": len(stream), + "record_stream_sha256": sha256_hex(stream), + "canonical_bytes_len": len(outputs.canonical_bytes), + "canonical_bytes_sha256": sha256_hex(outputs.canonical_bytes), + "canonical_digest": outputs.canonical_digest, + "spool_tail_len": len(outputs.spool_tail), + "spool_tail_sha256": sha256_hex(outputs.spool_tail), + "advance_stats": [stats.as_document() for stats in outputs.advance_stats], + } + ) + if len(outputs.canonical_bytes) <= LITERAL_PIN_MAX_CANONICAL_BYTES: + literals.update( + { + "record_jsons": [record.decode("ascii") for record in outputs.record_jsons], + "canonical_bytes": outputs.canonical_bytes.decode("ascii"), + "spool_tail": outputs.spool_tail.decode("ascii"), + } + ) + if literals: + entry["pinned_literals"] = literals + return entry + + +def reference_document() -> dict[str, object]: + """The frozen-reference document, regenerated from the shipped pipeline.""" + adapter = PythonWindowPipelineAdapter() + return { + "schema": REFERENCE_SCHEMA, + "regenerate_with": REGENERATE_COMMAND, + "master_seed": f"0x{PARITY_RANDOM_MASTER_SEED:08x}", + "default_page_size": DEFAULT_INCREMENTAL_SHARE_WINDOW_PAGE_SIZE, + "cases": {case.name: reference_case_entry(case, adapter.run(case)) for case in build_corpus()}, + } + + +def encode_reference_document(document: dict[str, object]) -> bytes: + # sort_keys plus the default ensure_ascii keeps the fixture byte-stable, + # so regeneration on an unchanged tree is a no-op. + return json.dumps(document, indent=2, sort_keys=True).encode("ascii") + b"\n" + + +def load_reference_document() -> dict[str, object]: + try: + raw = REFERENCE_FIXTURE_PATH.read_bytes() + except FileNotFoundError: + raise AssertionError( + f"frozen reference {REFERENCE_FIXTURE_RELPATH} is missing;" + f" regenerate it with: {REGENERATE_COMMAND}" + ) from None + document = json.loads(raw) + if document.get("schema") != REFERENCE_SCHEMA: + raise AssertionError( + f"frozen reference {REFERENCE_FIXTURE_RELPATH} has schema" + f" {document.get('schema')!r}, expected {REFERENCE_SCHEMA!r};" + f" regenerate it with: {REGENERATE_COMMAND}" + ) + return document + + +def diverging_outputs( + reference_case: dict[str, object], + result: WindowPipelineResult, + *, + domain_exclusion: str | None = None, +) -> list[str]: + """Every way one backend's result diverges from one frozen case. + + An empty list is parity. A rejection matches only a frozen rejection of + the same category; outputs match only a frozen outputs entry, where + length and SHA-256 comparisons hold for every case, literal comparisons + additionally localize the first divergence when the case pins literals, + and the per-advance stats must agree exactly. + + ``domain_exclusion`` is why the case lies outside the backend's declared + integer domain (:func:`domain_exclusion`), or None when it lies inside. + An ``Unsupported`` result is parity exactly when the case is outside: + the frozen reference keeps the shipped pipeline's result and the backend + has declared, rather than mis-rendered, the input it cannot carry. A + backend that answers ``Unsupported`` for a case inside its declared + domain diverges, and a backend that produces bytes for a case outside + its domain is held to the frozen bytes like any other. + """ + expected_rejection = reference_case.get("rejected") + if isinstance(result, WindowPipelineUnsupported): + if domain_exclusion is None: + return [ + f"unsupported: backend declared {result.reason!r} but the" + " case is inside its declared integer domain" + ] + return [] + if isinstance(result, WindowPipelineRejection): + if expected_rejection is None: + return [ + f"rejected: backend rejected as {result.reason!r} but the" + " frozen reference has outputs" + ] + if result.reason != expected_rejection: + return [f"rejected: {result.reason!r} != frozen {expected_rejection!r}"] + return [] + outputs = result + if expected_rejection is not None: + return [ + f"rejected: backend produced outputs but the frozen reference is" + f" rejected as {expected_rejection!r}" + ] + + problems: list[str] = [] + + def check_stream(label: str, actual: bytes, literal: str | None) -> None: + expected_len = reference_case[f"{label}_len"] + expected_sha = reference_case[f"{label}_sha256"] + if len(actual) != expected_len: + problems.append(f"{label}: {len(actual)} bytes != frozen {expected_len}") + if sha256_hex(actual) != expected_sha: + problems.append(f"{label}: sha256 diverges from the frozen reference") + if literal is not None and literal.encode("ascii") != actual: + problems.append(f"{label}: bytes diverge from the frozen literal") + + literals = reference_case.get("pinned_literals") or {} + literal_records = literals.get("record_jsons") + if len(outputs.record_jsons) != reference_case["record_count"]: + problems.append( + f"record_count: {len(outputs.record_jsons)} != frozen {reference_case['record_count']}" + ) + check_stream( + "record_stream", + record_stream_bytes(outputs.record_jsons), + "\n".join(literal_records) if literal_records is not None else None, + ) + if literal_records is not None: + expected_records = [text.encode("ascii") for text in literal_records] + for index, (expected, actual) in enumerate(zip(expected_records, outputs.record_jsons)): + if expected != actual: + problems.append( + f"record_jsons[{index}]: {actual!r} != frozen {expected!r}" + ) + break + check_stream("canonical_bytes", outputs.canonical_bytes, literals.get("canonical_bytes")) + if outputs.canonical_digest != reference_case["canonical_digest"]: + problems.append( + f"canonical_digest: {outputs.canonical_digest} != frozen" + f" {reference_case['canonical_digest']}" + ) + check_stream("spool_tail", outputs.spool_tail, literals.get("spool_tail")) + actual_stats = [stats.as_document() for stats in outputs.advance_stats] + if actual_stats != reference_case["advance_stats"]: + problems.append( + f"advance_stats: {actual_stats} != frozen {reference_case['advance_stats']}" + ) + return problems + + +def _main(argv: list[str]) -> int: + if argv != ["regenerate"]: + print(f"usage: {REGENERATE_COMMAND}", file=sys.stderr) + return 2 + payload = encode_reference_document(reference_document()) + REFERENCE_FIXTURE_PATH.parent.mkdir(parents=True, exist_ok=True) + if REFERENCE_FIXTURE_PATH.exists() and REFERENCE_FIXTURE_PATH.read_bytes() == payload: + print(f"unchanged: {REFERENCE_FIXTURE_RELPATH}") + return 0 + REFERENCE_FIXTURE_PATH.write_bytes(payload) + print(f"wrote {len(payload)} bytes: {REFERENCE_FIXTURE_RELPATH}") + return 0 + + +if __name__ == "__main__": + sys.exit(_main(sys.argv[1:])) diff --git a/tests/window_pipeline_parity_gate.py b/tests/window_pipeline_parity_gate.py new file mode 100644 index 000000000..010212217 --- /dev/null +++ b/tests/window_pipeline_parity_gate.py @@ -0,0 +1,79 @@ +#!/usr/bin/env python3 +"""Strict CI entry point for the frozen payout-window parity corpus.""" + +from __future__ import annotations + +import os +import sys +import unittest +from pathlib import Path + +from lab.prism.prism_tools import prism_tool_command +from tests import window_pipeline_parity as parity + + +PARITY_TEST_TARGET = ( + "tests.test_window_pipeline_parity.WindowPipelineParityOracleTests" +) +RUST_DAEMON_BINARY = "qbit-prism-build-audit-bundle" + + +def resolve_gate_adapter() -> tuple[parity.WindowPipelineAdapter, tuple[str, ...]]: + """Resolve the explicitly selected adapter and its required prebuilt tool.""" + requested = os.environ.get(parity.ADAPTER_ENV_VAR) + if not requested: + raise RuntimeError( + f"{parity.ADAPTER_ENV_VAR} must explicitly select a parity adapter" + ) + adapter = parity.resolve_adapter(requested) + if adapter.name != requested: + raise RuntimeError( + f"resolved parity adapter {adapter.name!r}, expected {requested!r}" + ) + + if requested != "rust-daemon": + return adapter, () + + bin_dir = os.environ.get("PRISM_TOOL_BIN_DIR") + if not bin_dir: + raise RuntimeError( + "rust-daemon parity requires PRISM_TOOL_BIN_DIR to select the" + " prebuilt deployment binary" + ) + expected = Path(bin_dir) / RUST_DAEMON_BINARY + command = tuple(prism_tool_command(RUST_DAEMON_BINARY)) + if command != (str(expected),): + raise RuntimeError( + f"rust-daemon parity requires executable prebuilt binary {expected};" + " refusing cargo fallback" + ) + return adapter, command + + +def main() -> int: + try: + adapter, command = resolve_gate_adapter() + except (RuntimeError, ValueError) as error: + print(f"payout-window parity gate: {error}", file=sys.stderr) + return 2 + + corpus = parity.build_corpus() + advance_cases = sum(bool(case.advances) for case in corpus) + advance_steps = sum(len(case.advances) for case in corpus) + print(f"payout-window parity adapter: {adapter.name}", flush=True) + if command: + print(f"payout-window parity daemon: {command[0]} --serve", flush=True) + print( + "payout-window frozen corpus:" + f" {len(corpus)} full preparations," + f" {advance_steps} advance() calls across {advance_cases} cases", + flush=True, + ) + + suite = unittest.defaultTestLoader.loadTestsFromName(PARITY_TEST_TARGET) + result = unittest.TextTestRunner(verbosity=2).run(suite) + return 0 if result.wasSuccessful() else 1 + + +if __name__ == "__main__": + sys.exit(main()) From e1436af94ed0290b24d9858f348cd35065095c40 Mon Sep 17 00:00:00 2001 From: Dan Hepworth Date: Mon, 28 Sep 2026 15:52:16 -0400 Subject: [PATCH 03/10] main: run the PRISM load-harness nightly from the default branch (#521) (#538) GitHub fires schedule triggers, and offers workflow_dispatch, only for workflows on the default branch. This adds prism-load-nightly.yml to main, byte-identical to the copy merged to 3.x.x in #536. Its jobs check out the ref input, which defaults to 3.x.x. The presets, scripts, gate and composite actions they run therefore come from 3.x.x, and nothing else changes on main. Refs #521 Co-authored-by: Claude Opus 5.5 --- .github/workflows/prism-load-nightly.yml | 225 +++++++++++++++++++++++ 1 file changed, 225 insertions(+) create mode 100644 .github/workflows/prism-load-nightly.yml diff --git a/.github/workflows/prism-load-nightly.yml b/.github/workflows/prism-load-nightly.yml new file mode 100644 index 000000000..4d3ae7c21 --- /dev/null +++ b/.github/workflows/prism-load-nightly.yml @@ -0,0 +1,225 @@ +name: PRISM load harness (nightly) + +# #521 item 6: the load harness (qbit-prism-load) on checked-in presets, every +# night and on demand. Each preset in crates/qbit-prism-load/presets pins +# every flag its result depends on (a unit test refuses one that does not), +# names the runner it needs and states its gates, so what runs, where, and +# what it is held to are all reviewed in one file. +# +# A run builds qbit-prism-server and qbit-prism-load in release mode at the +# chosen ref, records pg_test_fsync on the filesystem the harness puts its +# PostgreSQL cluster on, runs the preset, and gates it with +# qbit-prism-load-gate: exit 0 (completed and reconciled exactly), no +# durability finding, shortfall == 0 in each gated phase, #473's D1 rule +# where the preset asks for it, and the tip-to-last-notify p99 budget where +# the preset sets one. The verdict table goes to the job summary and the +# harness's reports to an artifact. +# +# Runners: every nightly preset runs on an 8 vCPU runner, the size #271's +# 20k matrix was taken on; that includes one 400k D1 cell of #473's +# (d1-473-400k-fe1-async, the production-window baseline) and the 400k +# realism and #275 presets. The heavier manual presets (#473's multi-frontend +# and 500k cells, growth-20x's 8,000 sessions on four frontends) take a +# 16 vCPU runner so the client and the database do not starve the frontends +# being measured. Rates are only comparable run to run on the same runner +# class, which is why the runner is part of the preset. +# +# Schedules run from the default branch's copy of this file; the build and +# the presets come from `ref` (3.x.x unless dispatched otherwise). + +on: + schedule: + - cron: "17 2 * * *" + workflow_dispatch: + inputs: + preset: + description: >- + Preset name(s), comma separated, from crates/qbit-prism-load/presets; + `nightly` for the nightly set, `all` for nightly plus manual + (growth-20x and #473's cells other than 400k fe1 async). + required: true + default: nightly + ref: + description: Git ref to build and run. + required: true + default: 3.x.x + tip_last_notify_p99_budget_ms: + description: >- + Override every selected preset's tip-to-last-notify p99 budget, in + ms. Empty keeps each preset's own (gates.tip_last_notify_p99_budget_ms: + 2,500 for tip-275*, 3,000 for mainnet-floor and growth-5x, 5,000 for + growth-20x; ungated for the D1 presets, whose rule is #473's). + required: false + default: "" + max_shortfall: + description: >- + Override every selected preset's per-phase shortfall budget. Empty + keeps each preset's own (0 for every checked-in preset). + required: false + default: "" + live: + description: >- + Also run the opt-in live regtest variants listed in + test/prism-nightly-gated-tests.txt (the schedule always does). + type: boolean + required: false + default: true + +permissions: + contents: read + +concurrency: + group: prism-load-nightly + cancel-in-progress: false + +jobs: + plan: + name: Plan the preset matrix + runs-on: blacksmith-2vcpu-ubuntu-2404 + outputs: + matrix: ${{ steps.plan.outputs.matrix }} + steps: + - name: Check out the ref under test + uses: actions/checkout@v7 + with: + ref: ${{ inputs.ref || '3.x.x' }} + - name: Select presets + id: plan + env: + SELECTION: ${{ inputs.preset || 'nightly' }} + run: | + set -euo pipefail + matrix="$(python3 scripts/prism_load_matrix.py "${SELECTION}")" + echo "${matrix}" + printf 'matrix=%s\n' "${matrix}" >> "${GITHUB_OUTPUT}" + + run: + name: ${{ matrix.preset }} + needs: plan + runs-on: ${{ matrix.runner }} + timeout-minutes: ${{ matrix.timeout_minutes }} + strategy: + fail-fast: false + matrix: ${{ fromJSON(needs.plan.outputs.matrix) }} + steps: + - name: Check out the ref under test + uses: actions/checkout@v7 + with: + ref: ${{ inputs.ref || '3.x.x' }} + - name: Set up Rust + uses: ./.github/actions/rust-toolchain + - name: Install PostgreSQL 16 server binaries + uses: ./.github/actions/apt-install + with: + packages: postgresql-16 + - name: Build the server, the harness and the gate in release mode + run: cargo build --locked --release -p qbit-prism-server -p qbit-prism-load + - name: Record the host + env: + PRESET: ${{ matrix.preset }} + RUNNER_LABEL: ${{ matrix.runner }} + run: | + set -euo pipefail + { + echo "### ${PRESET} on ${RUNNER_LABEL}" + echo + echo '```' + git rev-parse HEAD + nproc + free -m + uname -r + echo '```' + } >> "${GITHUB_STEP_SUMMARY}" + - name: Run and gate the preset + env: + PRISM_LOAD_TIP_LAST_NOTIFY_P99_BUDGET_MS: ${{ inputs.tip_last_notify_p99_budget_ms }} + PRISM_LOAD_MAX_SHORTFALL: ${{ inputs.max_shortfall }} + PRESET: ${{ matrix.preset }} + run: bash .github/scripts/prism-load-run.sh "${PRESET}" "${RUNNER_TEMP}/prism-load-out" + - name: Upload the harness reports + if: ${{ !cancelled() }} + uses: actions/upload-artifact@v7 + with: + name: prism-load-${{ matrix.preset }} + overwrite: true + retention-days: 30 + path: ${{ runner.temp }}/prism-load-out + if-no-files-found: error + + # The opt-in #[ignore] live_regtest variants (#521): each id in + # test/prism-nightly-gated-tests.txt, run explicitly with --ignored --exact + # in required integration mode against a real qbitd and PostgreSQL 16, and + # proven executed with the same gate manifest check the PR suite uses. + # Today that is #523's qbitd -reindex crash variant and #524's 130-payee + # weighted-recipients variant. TODO(#521): add scenario 5's dense-cadence + # soak to the list when it lands on 3.x.x. + live-nightly: + name: Live regtest nightly variants + if: ${{ github.event_name == 'schedule' || inputs.live }} + runs-on: blacksmith-8vcpu-ubuntu-2404 + timeout-minutes: 60 + services: + postgres: + image: postgres:16 + env: + POSTGRES_DB: prism_test + POSTGRES_USER: prism_test + POSTGRES_PASSWORD: prism_test + ports: + - 5432:5432 + options: >- + --health-cmd "pg_isready -U prism_test -d prism_test" + --health-interval 5s --health-timeout 5s --health-retries 20 + env: + PRISM_TEST_DATABASE_URL: postgresql://prism_test:prism_test@127.0.0.1:5432/prism_test + PRISM_TEST_PG_BIN_DIR: /usr/lib/postgresql/16/bin + PRISM_TEST_REQUIRE_INTEGRATION: "1" + steps: + - name: Check out the ref under test + uses: actions/checkout@v7 + with: + ref: ${{ inputs.ref || '3.x.x' }} + - name: Name the gate manifest and the test log + run: | + printf 'PRISM_TEST_GATE_MANIFEST=%s/prism-nightly-gate-manifest.txt\n' "${RUNNER_TEMP}" >> "${GITHUB_ENV}" + printf 'PRISM_TEST_LOG=%s/prism-nightly-tests.log\n' "${RUNNER_TEMP}" >> "${GITHUB_ENV}" + - name: Set up Rust + uses: ./.github/actions/rust-toolchain + - name: Install PostgreSQL 16 server binaries + uses: ./.github/actions/apt-install + with: + packages: postgresql-16 + - name: Restore the pinned qbit release from the cache + uses: actions/cache/restore@v6 + with: + path: ~/.cache/prism-qbit + key: prism-qbit-v1-${{ runner.os }}-${{ runner.arch }}-${{ hashFiles('.github/scripts/install-prism-qbit.sh') }} + - name: Install pinned qbit + run: | + set -euo pipefail + PRISM_QBIT_CACHE_DIR="${HOME}/.cache/prism-qbit" \ + bash .github/scripts/install-prism-qbit.sh "${RUNNER_TEMP}/prism-qbit" + - name: Run the listed variants + run: | + set -euo pipefail + mapfile -t tests < <(sed -e 's/#.*//' -e 's/[[:space:]]//g' test/prism-nightly-gated-tests.txt \ + | grep -v '^$' | sed 's/^qbit-prism-server::live_regtest:://') + cargo test --locked -p qbit-prism-server --test live_regtest -- \ + --ignored --exact --test-threads=1 --nocapture "${tests[@]}" 2>&1 | tee "${PRISM_TEST_LOG}" + - name: Prove every listed variant executed + run: | + python3 scripts/check_gate_manifest.py \ + --manifest "${PRISM_TEST_GATE_MANIFEST}" \ + --expected test/prism-nightly-gated-tests.txt \ + --log "${PRISM_TEST_LOG}" + - name: Upload the gate manifest and the test log + if: ${{ !cancelled() }} + uses: actions/upload-artifact@v7 + with: + name: prism-live-nightly + overwrite: true + retention-days: 30 + path: | + ${{ env.PRISM_TEST_GATE_MANIFEST }} + ${{ env.PRISM_TEST_LOG }} + if-no-files-found: error From 348c28f3a580132d5e31d021ef707c6e64c990a1 Mon Sep 17 00:00:00 2001 From: Dan Hepworth Date: Mon, 28 Sep 2026 16:53:00 -0400 Subject: [PATCH 04/10] main: sync the PRISM load-harness nightly workflow from 3.x.x (#559, #540) (#567) Brings prism-load-nightly.yml on main up to date with its copy on 3.x.x. The two files are now byte-identical: - #559: the plan job pins one commit for the whole matrix, a single build job shares the release binaries, and the preset names change; - #540: scenario 5's dense-cadence soak joins the live-nightly list, with a comment-only change here. Refs #521 Co-authored-by: Claude Opus 5.5 --- .github/workflows/prism-load-nightly.yml | 94 ++++++++++++++++++------ 1 file changed, 71 insertions(+), 23 deletions(-) diff --git a/.github/workflows/prism-load-nightly.yml b/.github/workflows/prism-load-nightly.yml index 4d3ae7c21..1dba53f86 100644 --- a/.github/workflows/prism-load-nightly.yml +++ b/.github/workflows/prism-load-nightly.yml @@ -6,20 +6,21 @@ name: PRISM load harness (nightly) # names the runner it needs and states its gates, so what runs, where, and # what it is held to are all reviewed in one file. # -# A run builds qbit-prism-server and qbit-prism-load in release mode at the -# chosen ref, records pg_test_fsync on the filesystem the harness puts its -# PostgreSQL cluster on, runs the preset, and gates it with -# qbit-prism-load-gate: exit 0 (completed and reconciled exactly), no -# durability finding, shortfall == 0 in each gated phase, #473's D1 rule -# where the preset asks for it, and the tip-to-last-notify p99 budget where -# the preset sets one. The verdict table goes to the job summary and the +# The plan job pins the chosen ref to one commit for the whole matrix, and +# one job builds qbit-prism-server and qbit-prism-load in release mode there. +# Each preset job downloads those binaries, records pg_test_fsync on the +# filesystem the harness puts its PostgreSQL cluster on, runs the preset, +# and gates it with qbit-prism-load-gate: exit 0 (completed and reconciled +# exactly), no acknowledged share missing, no durability finding, +# shortfall == 0 in each gated phase, #473's D1 rule where the preset asks +# for it, and the tip-to-last-notify p99 budget where the preset sets one. The verdict table goes to the job summary and the # harness's reports to an artifact. # # Runners: every nightly preset runs on an 8 vCPU runner, the size #271's # 20k matrix was taken on; that includes one 400k D1 cell of #473's -# (d1-473-400k-fe1-async, the production-window baseline) and the 400k +# (throughput-400k-window-1fe-async, the production-window baseline) and the 400k # realism and #275 presets. The heavier manual presets (#473's multi-frontend -# and 500k cells, growth-20x's 8,000 sessions on four frontends) take a +# and 500k cells, mainnet-shape-2600-addresses's 8,000 sessions on four frontends) take a # 16 vCPU runner so the client and the database do not starve the frontends # being measured. Rates are only comparable run to run on the same runner # class, which is why the runner is part of the preset. @@ -36,7 +37,7 @@ on: description: >- Preset name(s), comma separated, from crates/qbit-prism-load/presets; `nightly` for the nightly set, `all` for nightly plus manual - (growth-20x and #473's cells other than 400k fe1 async). + (mainnet-shape-2600-addresses and #473's cells other than 400k fe1 async). required: true default: nightly ref: @@ -47,8 +48,8 @@ on: description: >- Override every selected preset's tip-to-last-notify p99 budget, in ms. Empty keeps each preset's own (gates.tip_last_notify_p99_budget_ms: - 2,500 for tip-275*, 3,000 for mainnet-floor and growth-5x, 5,000 for - growth-20x; ungated for the D1 presets, whose rule is #473's). + 2,500 for tip-delivery-2000-miners-400k-2fe-retarget*, 3,000 for mainnet-shape-130-addresses and mainnet-shape-650-addresses, 5,000 for + mainnet-shape-2600-addresses; ungated for the D1 presets, whose rule is #473's). required: false default: "" max_shortfall: @@ -78,12 +79,13 @@ jobs: runs-on: blacksmith-2vcpu-ubuntu-2404 outputs: matrix: ${{ steps.plan.outputs.matrix }} + sha: ${{ steps.plan.outputs.sha }} steps: - name: Check out the ref under test uses: actions/checkout@v7 with: ref: ${{ inputs.ref || '3.x.x' }} - - name: Select presets + - name: Select presets and pin the commit id: plan env: SELECTION: ${{ inputs.preset || 'nightly' }} @@ -92,28 +94,75 @@ jobs: matrix="$(python3 scripts/prism_load_matrix.py "${SELECTION}")" echo "${matrix}" printf 'matrix=%s\n' "${matrix}" >> "${GITHUB_OUTPUT}" + # Every later job checks out this commit, so the binaries built + # once and the presets each run reads are the same revision even + # if the ref moves while the matrix runs. + printf 'sha=%s\n' "$(git rev-parse HEAD)" >> "${GITHUB_OUTPUT}" + + build: + name: Build the release binaries once + needs: plan + runs-on: blacksmith-8vcpu-ubuntu-2404 + timeout-minutes: 45 + steps: + - name: Check out the pinned commit + uses: actions/checkout@v7 + with: + ref: ${{ needs.plan.outputs.sha }} + - name: Set up Rust + uses: ./.github/actions/rust-toolchain + - name: Build the server, the harness and the gate in release mode + run: cargo build --locked --release -p qbit-prism-server -p qbit-prism-load + # The Cargo dep-info files go with the binaries: the harness ties the + # server to the checkout through them, as it does for a local build. + # A tar keeps the executable bits an artifact upload would drop. + - name: Pack the binaries and their dep-info + run: | + set -euo pipefail + tar -C target/release -cf "${RUNNER_TEMP}/prism-load-binaries.tar" \ + qbit-prism-server qbit-prism-server.d \ + qbit-prism-load qbit-prism-load.d qbit-prism-load-gate + - name: Upload the binaries + uses: actions/upload-artifact@v7 + with: + name: prism-load-binaries + overwrite: true + retention-days: 3 + path: ${{ runner.temp }}/prism-load-binaries.tar + if-no-files-found: error run: name: ${{ matrix.preset }} - needs: plan + needs: [plan, build] runs-on: ${{ matrix.runner }} timeout-minutes: ${{ matrix.timeout_minutes }} strategy: fail-fast: false matrix: ${{ fromJSON(needs.plan.outputs.matrix) }} steps: - - name: Check out the ref under test + - name: Check out the pinned commit uses: actions/checkout@v7 with: - ref: ${{ inputs.ref || '3.x.x' }} - - name: Set up Rust - uses: ./.github/actions/rust-toolchain + ref: ${{ needs.plan.outputs.sha }} - name: Install PostgreSQL 16 server binaries uses: ./.github/actions/apt-install with: packages: postgresql-16 - - name: Build the server, the harness and the gate in release mode - run: cargo build --locked --release -p qbit-prism-server -p qbit-prism-load + - name: Download the binaries built once + uses: actions/download-artifact@v8 + with: + name: prism-load-binaries + path: ${{ runner.temp }} + # Unpacked where a local release build puts them, at the same workspace + # path they were built at, and stamped after this checkout, so the + # harness can show the server was built from the sources beside it. + - name: Unpack the binaries into target/release + run: | + set -euo pipefail + mkdir -p target/release + tar -C target/release -xf "${RUNNER_TEMP}/prism-load-binaries.tar" + touch target/release/qbit-prism-server target/release/qbit-prism-load \ + target/release/qbit-prism-load-gate - name: Record the host env: PRESET: ${{ matrix.preset }} @@ -150,9 +199,8 @@ jobs: # test/prism-nightly-gated-tests.txt, run explicitly with --ignored --exact # in required integration mode against a real qbitd and PostgreSQL 16, and # proven executed with the same gate manifest check the PR suite uses. - # Today that is #523's qbitd -reindex crash variant and #524's 130-payee - # weighted-recipients variant. TODO(#521): add scenario 5's dense-cadence - # soak to the list when it lands on 3.x.x. + # That is #523's qbitd -reindex crash variant, #524's 130-payee + # weighted-recipients variant and scenario 5's dense-cadence soak. live-nightly: name: Live regtest nightly variants if: ${{ github.event_name == 'schedule' || inputs.live }} From 308c7c4ea1cdce3201e64acfd0a0a63bb305f832 Mon Sep 17 00:00:00 2001 From: Dan Hepworth Date: Tue, 29 Sep 2026 15:01:10 -0400 Subject: [PATCH 05/10] main: sync the PRISM load-harness nightly and runner probe workflows from 3.x.x (#565, #561) (#583) * main: sync the PRISM load-harness nightly workflow from 3.x.x (#565) Schedules run from the default branch's copy of this workflow, so #565's weekly 2.x.x migration-lifecycle schedule (Sunday 03:43 UTC) and its schedule-specific job conditions only fire once main carries them. The file is copied byte for byte from 3.x.x; jobs still build and test the `ref` input (3.x.x by default). Co-Authored-By: Claude Opus 5.5 (1M context) * main: copy the PRISM load-harness runner probe workflow from 3.x.x (#561) GitHub dispatches a workflow only when its file is on the default branch, so #561's dispatch-only runner probe can't run until main carries it, and #541's runs are blocked on that. The file is copied byte for byte from 3.x.x at e9611133; every job that uses a repository file checks out the `ref` input (3.x.x by default) first. Refs #589 Co-Authored-By: Claude Opus 5.5 (1M context) --------- Co-authored-by: Claude Opus 5.5 (1M context) Co-authored-by: Anatolie --- .github/workflows/prism-load-nightly.yml | 94 +++- .github/workflows/prism-load-runner-probe.yml | 409 ++++++++++++++++++ 2 files changed, 502 insertions(+), 1 deletion(-) create mode 100644 .github/workflows/prism-load-runner-probe.yml diff --git a/.github/workflows/prism-load-nightly.yml b/.github/workflows/prism-load-nightly.yml index 1dba53f86..a30a3d2e6 100644 --- a/.github/workflows/prism-load-nightly.yml +++ b/.github/workflows/prism-load-nightly.yml @@ -25,12 +25,19 @@ name: PRISM load harness (nightly) # being measured. Rates are only comparable run to run on the same runner # class, which is why the runner is part of the preset. # +# Once a week the `live-weekly` job runs the longer live regtest scenarios +# listed in test/prism-weekly-gated-tests.txt instead (#487's L4 weekly set): +# today #545's 2.x.x migration lifecycle. The weekly schedule runs only that +# job; the nightly jobs keep their own schedule. +# # Schedules run from the default branch's copy of this file; the build and # the presets come from `ref` (3.x.x unless dispatched otherwise). on: schedule: - cron: "17 2 * * *" + # Weekly, on Sunday. The jobs' `if` conditions compare against both crons. + - cron: "43 3 * * 0" workflow_dispatch: inputs: preset: @@ -65,6 +72,13 @@ on: type: boolean required: false default: true + weekly: + description: >- + Also run the weekly live regtest scenarios listed in + test/prism-weekly-gated-tests.txt (the weekly schedule always does). + type: boolean + required: false + default: false permissions: contents: read @@ -76,6 +90,7 @@ concurrency: jobs: plan: name: Plan the preset matrix + if: ${{ github.event.schedule != '43 3 * * 0' }} runs-on: blacksmith-2vcpu-ubuntu-2404 outputs: matrix: ${{ steps.plan.outputs.matrix }} @@ -203,7 +218,7 @@ jobs: # weighted-recipients variant and scenario 5's dense-cadence soak. live-nightly: name: Live regtest nightly variants - if: ${{ github.event_name == 'schedule' || inputs.live }} + if: ${{ github.event.schedule == '17 2 * * *' || inputs.live }} runs-on: blacksmith-8vcpu-ubuntu-2404 timeout-minutes: 60 services: @@ -271,3 +286,80 @@ jobs: ${{ env.PRISM_TEST_GATE_MANIFEST }} ${{ env.PRISM_TEST_LOG }} if-no-files-found: error + + # The weekly #[ignore] live_regtest scenarios (#487 L4): each id in + # test/prism-weekly-gated-tests.txt, run as live-nightly runs its list, with + # PostgreSQL's client tools for the scenarios' pg_dump and psql. Today that + # is #545's migration lifecycle: a scaled 2.x.x ledger through migrate, + # import-audits, native mining on the real node, exact reconciliation and + # the isolated-restore rollback of docs/prism-rust-migration.md. + live-weekly: + name: Live regtest weekly scenarios + if: ${{ github.event.schedule == '43 3 * * 0' || inputs.weekly }} + runs-on: blacksmith-8vcpu-ubuntu-2404 + timeout-minutes: 60 + services: + postgres: + image: postgres:16 + env: + POSTGRES_DB: prism_test + POSTGRES_USER: prism_test + POSTGRES_PASSWORD: prism_test + ports: + - 5432:5432 + options: >- + --health-cmd "pg_isready -U prism_test -d prism_test" + --health-interval 5s --health-timeout 5s --health-retries 20 + env: + PRISM_TEST_DATABASE_URL: postgresql://prism_test:prism_test@127.0.0.1:5432/prism_test + PRISM_TEST_PG_BIN_DIR: /usr/lib/postgresql/16/bin + PRISM_TEST_REQUIRE_INTEGRATION: "1" + steps: + - name: Check out the ref under test + uses: actions/checkout@v7 + with: + ref: ${{ inputs.ref || '3.x.x' }} + - name: Name the gate manifest and the test log + run: | + printf 'PRISM_TEST_GATE_MANIFEST=%s/prism-weekly-gate-manifest.txt\n' "${RUNNER_TEMP}" >> "${GITHUB_ENV}" + printf 'PRISM_TEST_LOG=%s/prism-weekly-tests.log\n' "${RUNNER_TEMP}" >> "${GITHUB_ENV}" + - name: Set up Rust + uses: ./.github/actions/rust-toolchain + - name: Install PostgreSQL 16 server binaries + uses: ./.github/actions/apt-install + with: + packages: postgresql-16 + - name: Restore the pinned qbit release from the cache + uses: actions/cache/restore@v6 + with: + path: ~/.cache/prism-qbit + key: prism-qbit-v1-${{ runner.os }}-${{ runner.arch }}-${{ hashFiles('.github/scripts/install-prism-qbit.sh') }} + - name: Install pinned qbit + run: | + set -euo pipefail + PRISM_QBIT_CACHE_DIR="${HOME}/.cache/prism-qbit" \ + bash .github/scripts/install-prism-qbit.sh "${RUNNER_TEMP}/prism-qbit" + - name: Run the listed scenarios + run: | + set -euo pipefail + mapfile -t tests < <(sed -e 's/#.*//' -e 's/[[:space:]]//g' test/prism-weekly-gated-tests.txt \ + | grep -v '^$' | sed 's/^qbit-prism-server::live_regtest:://') + cargo test --locked -p qbit-prism-server --test live_regtest -- \ + --ignored --exact --test-threads=1 --nocapture "${tests[@]}" 2>&1 | tee "${PRISM_TEST_LOG}" + - name: Prove every listed scenario executed + run: | + python3 scripts/check_gate_manifest.py \ + --manifest "${PRISM_TEST_GATE_MANIFEST}" \ + --expected test/prism-weekly-gated-tests.txt \ + --log "${PRISM_TEST_LOG}" + - name: Upload the gate manifest and the test log + if: ${{ !cancelled() }} + uses: actions/upload-artifact@v7 + with: + name: prism-live-weekly + overwrite: true + retention-days: 30 + path: | + ${{ env.PRISM_TEST_GATE_MANIFEST }} + ${{ env.PRISM_TEST_LOG }} + if-no-files-found: error diff --git a/.github/workflows/prism-load-runner-probe.yml b/.github/workflows/prism-load-runner-probe.yml new file mode 100644 index 000000000..4a657178e --- /dev/null +++ b/.github/workflows/prism-load-runner-probe.yml @@ -0,0 +1,409 @@ +name: PRISM load harness runner probe + +# #541 (S1 of #487): measure what the Blacksmith runners are before any lane +# depends on them. Dispatch only: there is no schedule, so no standing spend. +# +# Each probe job builds qbit-prism-server and qbit-prism-load in release mode +# and runs the short-plan-20k-window-1fe and throughput-20k-window-1fe presets in the same job and checkout, so +# the harness's provenance checks see the binary its own tree built (#487 +# lesson 4); no --allow-* override is passed, and the row says whether every +# check passed. It records, per runner class: +# - pg_test_fsync on the filesystem the harness puts its PostgreSQL +# clusters on (lesson 1), with the kernel's view of the block device +# under it (write cache, FUA) as the evidence on whether fsync reaches +# stable storage; +# - wall time for the build, seeding and each run; +# - peak host memory (MemTotal - MemAvailable, sampled every second) over +# the build and each run, the harness, frontends and both clusters +# together (lesson 3); +# - shortfall per phase and the tip-to-last-notify p99, which #521 +# scenario 6's gates read. +# The Cargo target directory is kept either on a Blacksmith sticky disk or +# in the Actions cache, one job per backend per class; dispatch twice to +# compare a warm restore against a cold build. The fsync-spread jobs run +# pg_test_fsync alone on several VMs of each class, for VM-to-VM disk +# consistency, and the optional duration job holds a 2 vCPU runner for the +# given minutes to find the maximum job duration. The collate job writes the +# per-class results table to its summary and to an artifact; the billing +# multiplier is read off the usage dashboard against these jobs' minutes. +# +# The rates here are for this probe only: CI numbers are not compared with +# another host class's (lesson 1) and are not D1 verdicts (lesson 2). + +on: + workflow_dispatch: + inputs: + ref: + description: Git ref to build and run. + required: true + default: 3.x.x + classes: + description: Runner classes in vCPU, comma separated (2, 4, 8, 16, 32). + required: true + default: "8,16,32" + target_cache: + description: Where the Cargo target directory is kept between runs. + type: choice + required: true + default: both + options: + - both + - actions-cache + - sticky-disk + fsync_vms: + description: >- + pg_test_fsync-only VMs per class, for VM-to-VM disk consistency + (0 to skip, at most 10). + required: true + default: "3" + duration_probe_minutes: + description: >- + Hold one 2 vCPU runner this many minutes to find the maximum job + duration (0 to skip; 1500 tests a 24 h soak). + required: true + default: "0" + +permissions: + contents: read + +concurrency: + group: prism-load-runner-probe + cancel-in-progress: false + +jobs: + plan: + name: Plan the probe matrix + runs-on: blacksmith-2vcpu-ubuntu-2404 + outputs: + probe: ${{ steps.plan.outputs.probe }} + fsync: ${{ steps.plan.outputs.fsync }} + fsync_enabled: ${{ steps.plan.outputs.fsync_enabled }} + duration_minutes: ${{ steps.duration.outputs.minutes }} + sha: ${{ steps.pin.outputs.sha }} + steps: + - name: Check out the ref under test + uses: actions/checkout@v7 + with: + ref: ${{ inputs.ref }} + - name: Pin the ref to one commit + id: pin + # Every later job checks out this commit, so a push to the ref while + # runners are queued cannot make classes measure different code. + run: printf 'sha=%s\n' "$(git rev-parse HEAD)" >> "${GITHUB_OUTPUT}" + - name: Select runner classes and target caches + id: plan + env: + CLASSES: ${{ inputs.classes }} + TARGET_CACHE: ${{ inputs.target_cache }} + FSYNC_VMS: ${{ inputs.fsync_vms }} + run: | + set -euo pipefail + python3 scripts/prism_load_probe.py matrix \ + --classes "${CLASSES}" --target-cache "${TARGET_CACHE}" --fsync-vms "${FSYNC_VMS}" \ + | tee -a "${GITHUB_OUTPUT}" + - name: Check the duration probe's length + id: duration + env: + MINUTES: ${{ inputs.duration_probe_minutes }} + run: | + set -euo pipefail + # The duration job's own timeout is 4,320 minutes (72 h), so a + # longer hold would measure that and not the runner's limit. + # Read as decimal before the bound check: a leading zero would + # otherwise make bash compare the value as octal. + if [[ ! "${MINUTES}" =~ ^[0-9]{1,6}$ ]]; then + echo "duration_probe_minutes ${MINUTES@Q} is not a whole number 0..4200" >&2 + exit 2 + fi + minutes=$((10#${MINUTES})) + if (( minutes > 4200 )); then + echo "duration_probe_minutes ${MINUTES@Q} is not a whole number 0..4200" >&2 + exit 2 + fi + printf 'minutes=%d\n' "${minutes}" >> "${GITHUB_OUTPUT}" + + probe: + name: ${{ matrix.class }} vCPU, ${{ matrix.target_cache }} + needs: plan + runs-on: ${{ matrix.runner }} + timeout-minutes: 150 + strategy: + fail-fast: false + matrix: ${{ fromJSON(needs.plan.outputs.probe) }} + steps: + - name: Check out the ref under test + uses: actions/checkout@v7 + with: + ref: ${{ needs.plan.outputs.sha }} + - name: Name the probe's directories + run: | + set -euo pipefail + # The target directory sits beside the checkout, not in it, so a + # restore or a mount never touches the tree the harness checks. + printf 'CARGO_TARGET_DIR=%s\n' "$(realpath -m "${GITHUB_WORKSPACE}/../prism-probe-target")" >> "${GITHUB_ENV}" + printf 'PROBE=%s/probe\n' "${RUNNER_TEMP}" >> "${GITHUB_ENV}" + - name: Set up Rust + uses: ./.github/actions/rust-toolchain + - name: Install PostgreSQL 16 server binaries + uses: ./.github/actions/apt-install + with: + packages: postgresql-16 + - name: Name the target cache + id: key + env: + SIZE: ${{ matrix.class }} + HASH: ${{ hashFiles('Cargo.lock', 'rust-toolchain.toml') }} + run: | + set -euo pipefail + mkdir -p "${PROBE}" "${CARGO_TARGET_DIR}" + # One key per class and backend, so a class's warm run restores + # its own previous build and never another class's. + printf 'prefix=prism-probe-target-v1-%svcpu-%s-\n' "${SIZE}" "${HASH}" >> "${GITHUB_OUTPUT}" + printf 'sticky=prism-probe-target-v1-%svcpu\n' "${SIZE}" >> "${GITHUB_OUTPUT}" + date +%s.%N > "${PROBE}/restore-started" + - name: Restore the target directory from the Actions cache + if: ${{ matrix.target_cache == 'actions-cache' }} + uses: actions/cache/restore@v6 + with: + path: ${{ env.CARGO_TARGET_DIR }} + key: ${{ steps.key.outputs.prefix }}${{ github.run_id }} + restore-keys: ${{ steps.key.outputs.prefix }} + - name: Mount the target directory's sticky disk + if: ${{ matrix.target_cache == 'sticky-disk' }} + uses: useblacksmith/stickydisk@v1 + with: + key: ${{ steps.key.outputs.sticky }} + path: ${{ env.CARGO_TARGET_DIR }} + - name: Record the host and the restored target directory + run: | + set -euo pipefail + awk -v now="$(date +%s.%N)" -v then="$(cat "${PROBE}/restore-started")" \ + 'BEGIN { printf "%.1f\n", now - then }' > "${PROBE}/restore-seconds" + # Anything in the target directory before the build came from the + # backend. A fresh sticky disk is not empty (lost+found), so a warm + # target is one with an earlier release build in it. + du -sb "${CARGO_TARGET_DIR}" | cut -f1 > "${PROBE}/target-bytes-before" + if [[ -d "${CARGO_TARGET_DIR}/release" ]]; then echo true; else echo false; fi \ + > "${PROBE}/target-restored" + mkdir -p "${RUNNER_TEMP}/pload" + python3 scripts/prism_load_probe.py host --out "${PROBE}/host.json" --dir "${RUNNER_TEMP}/pload" + { + echo "### ${{ matrix.class }} vCPU, ${{ matrix.target_cache }}" + echo + echo '```' + git rev-parse HEAD + cat "${PROBE}/host.json" + echo "target directory before build: $(cat "${PROBE}/target-bytes-before") bytes" + echo '```' + } >> "${GITHUB_STEP_SUMMARY}" + - name: Build the server, the harness and the gate in release mode + run: | + python3 scripts/prism_load_probe.py measure --out "${PROBE}/build.json" -- \ + cargo build --locked --release -p qbit-prism-server -p qbit-prism-load + - name: Measure the target directory after the build + run: du -sb "${CARGO_TARGET_DIR}" | cut -f1 > "${PROBE}/target-bytes-after" + - name: Start timing the Actions cache save + if: ${{ matrix.target_cache == 'actions-cache' }} + run: date +%s.%N > "${PROBE}/save-started" + - name: Save the target directory to the Actions cache + if: ${{ matrix.target_cache == 'actions-cache' }} + uses: actions/cache/save@v6 + with: + path: ${{ env.CARGO_TARGET_DIR }} + key: ${{ steps.key.outputs.prefix }}${{ github.run_id }} + - name: Time the Actions cache save + if: ${{ matrix.target_cache == 'actions-cache' }} + run: | + awk -v now="$(date +%s.%N)" -v then="$(cat "${PROBE}/save-started")" \ + 'BEGIN { printf "%.1f\n", now - then }' > "${PROBE}/save-seconds" + # The sticky disk is committed in the action's post step, after every + # step here, so its save time is in that step's log, not in the row. + - name: Run short-plan-20k-window-1fe + timeout-minutes: 60 + env: + TARGET_DIR: ${{ env.CARGO_TARGET_DIR }} + run: | + set -uo pipefail + out="${PROBE}/short-plan-20k-window-1fe" + python3 scripts/prism_load_probe.py measure --out "${out}/probe-measure.json" -- \ + bash .github/scripts/prism-load-run.sh short-plan-20k-window-1fe "${out}" || true + - name: Run throughput-20k-window-1fe + if: ${{ !cancelled() }} + timeout-minutes: 60 + env: + TARGET_DIR: ${{ env.CARGO_TARGET_DIR }} + run: | + set -uo pipefail + out="${PROBE}/throughput-20k-window-1fe" + python3 scripts/prism_load_probe.py measure --out "${out}/probe-measure.json" -- \ + bash .github/scripts/prism-load-run.sh throughput-20k-window-1fe "${out}" || true + - name: Write the class's row + if: ${{ !cancelled() }} + env: + SIZE: ${{ matrix.class }} + RUNNER_LABEL: ${{ matrix.runner }} + TARGET_CACHE: ${{ matrix.target_cache }} + CACHE_ROW: ${{ matrix.class }}-${{ matrix.target_cache }} + run: | + set -euo pipefail + read_or_empty() { cat "$1" 2>/dev/null || true; } + before="$(read_or_empty "${PROBE}/target-bytes-before")" + restored="$(read_or_empty "${PROBE}/target-restored")" + mkdir -p "${PROBE}/row" + python3 scripts/prism_load_probe.py row --out "${PROBE}/row/${CACHE_ROW}.json" \ + --class "${SIZE}" --runner "${RUNNER_LABEL}" --target-cache "${TARGET_CACHE}" \ + --target-restored "${restored}" \ + --target-bytes-before "${before}" \ + --target-bytes-after "$(read_or_empty "${PROBE}/target-bytes-after")" \ + --restore-seconds "$(read_or_empty "${PROBE}/restore-seconds")" \ + --save-seconds "$(read_or_empty "${PROBE}/save-seconds")" \ + --build "${PROBE}/build.json" --host "${PROBE}/host.json" \ + --run "short-plan-20k-window-1fe=${PROBE}/short-plan-20k-window-1fe" --run "throughput-20k-window-1fe=${PROBE}/throughput-20k-window-1fe" + python3 scripts/prism_load_probe.py table "${PROBE}/row" >> "${GITHUB_STEP_SUMMARY}" + - name: Upload the class's row + if: ${{ !cancelled() }} + uses: actions/upload-artifact@v7 + with: + name: prism-probe-row-${{ matrix.class }}-${{ matrix.target_cache }} + overwrite: true + retention-days: 90 + path: ${{ env.PROBE }}/row + if-no-files-found: error + - name: Upload the harness reports + if: ${{ !cancelled() }} + uses: actions/upload-artifact@v7 + with: + name: prism-probe-${{ matrix.class }}-${{ matrix.target_cache }} + overwrite: true + retention-days: 90 + path: | + ${{ env.PROBE }} + !${{ env.PROBE }}/row + if-no-files-found: error + - name: Fail on the build's or a gate's exit code + # #487 lesson 5: the exit code is the verdict, after the reports are + # uploaded either way. + run: | + set -euo pipefail + python3 - "${PROBE}" <<'EOF' + import json, sys + from pathlib import Path + probe = Path(sys.argv[1]) + failed = [] + for name in ("build", "short-plan-20k-window-1fe/probe-measure", "throughput-20k-window-1fe/probe-measure"): + path = probe / f"{name}.json" + code = json.loads(path.read_text())["exit_code"] if path.exists() else None + if code != 0: + failed.append(f"{name}: exit {code}") + if failed: + sys.exit("; ".join(failed)) + EOF + + fsync-spread: + name: pg_test_fsync, ${{ matrix.class }} vCPU, VM ${{ matrix.vm }} + needs: plan + if: ${{ needs.plan.outputs.fsync_enabled == 'true' }} + runs-on: ${{ matrix.runner }} + timeout-minutes: 20 + strategy: + fail-fast: false + matrix: ${{ fromJSON(needs.plan.outputs.fsync) }} + steps: + - name: Check out the ref under test + uses: actions/checkout@v7 + with: + ref: ${{ needs.plan.outputs.sha }} + - name: Install PostgreSQL 16 server binaries + uses: ./.github/actions/apt-install + with: + packages: postgresql-16 + - name: Run pg_test_fsync where the harness puts its clusters + env: + SIZE: ${{ matrix.class }} + RUNNER_LABEL: ${{ matrix.runner }} + VM: ${{ matrix.vm }} + run: | + set -euo pipefail + dir="${RUNNER_TEMP}/pload" + out="${RUNNER_TEMP}/fsync" + mkdir -p "${dir}" "${out}/row" + python3 scripts/prism_load_probe.py host --out "${out}/host.json" --dir "${dir}" + /usr/lib/postgresql/16/bin/pg_test_fsync -s 2 -f "${dir}/pg_test_fsync.out" \ + > "${out}/pg_test_fsync.txt" 2>&1 || echo "pg_test_fsync failed" >> "${out}/pg_test_fsync.txt" + python3 scripts/prism_load_probe.py fsync-row --out "${out}/row/fsync-${SIZE}-${VM}.json" \ + --class "${SIZE}" --runner "${RUNNER_LABEL}" --vm "${VM}" \ + --host "${out}/host.json" --fsync "${out}/pg_test_fsync.txt" + cat "${out}/pg_test_fsync.txt" + - name: Upload the VM's row + uses: actions/upload-artifact@v7 + with: + name: prism-probe-row-fsync-${{ matrix.class }}-${{ matrix.vm }} + overwrite: true + retention-days: 90 + path: | + ${{ runner.temp }}/fsync/row + if-no-files-found: error + + duration: + name: Hold a runner for ${{ needs.plan.outputs.duration_minutes }} minutes + needs: plan + if: ${{ needs.plan.outputs.duration_minutes != '0' }} + runs-on: blacksmith-2vcpu-ubuntu-2404 + # Well above any hold the plan accepts, so a runner limit ends the job + # before this does. + timeout-minutes: 4320 + steps: + - name: Hold the runner + env: + MINUTES: ${{ needs.plan.outputs.duration_minutes }} + run: | + set -euo pipefail + started="$(date +%s)" + for ((minute = 1; minute <= MINUTES; minute++)); do + sleep 60 + echo "held $(( ($(date +%s) - started) / 60 )) of ${MINUTES} minutes at $(date -u +%FT%TZ)" + done + echo "Held a 2 vCPU runner for ${MINUTES} minutes without the runner ending the job." \ + >> "${GITHUB_STEP_SUMMARY}" + + collate: + name: Collate the results table + needs: [plan, probe, fsync-spread] + if: ${{ !cancelled() && needs.plan.result == 'success' }} + runs-on: blacksmith-2vcpu-ubuntu-2404 + steps: + - name: Check out the ref under test + uses: actions/checkout@v7 + with: + ref: ${{ needs.plan.outputs.sha }} + - name: Download every row + # A run in which every job failed before its row has nothing to + # download; the table then names each job as missing. + continue-on-error: true + uses: actions/download-artifact@v8 + with: + pattern: prism-probe-row-* + path: ${{ runner.temp }}/rows + merge-multiple: true + - name: Write the table + env: + PROBE_MATRIX: ${{ needs.plan.outputs.probe }} + FSYNC_MATRIX: ${{ needs.plan.outputs.fsync }} + FSYNC_ENABLED: ${{ needs.plan.outputs.fsync_enabled }} + run: | + set -euo pipefail + mkdir -p "${RUNNER_TEMP}/rows" + # What the plan asked for, so a job that wrote no row is counted. + python3 -c 'import json, os; print(json.dumps({"probe": json.loads(os.environ["PROBE_MATRIX"]), "fsync": json.loads(os.environ["FSYNC_MATRIX"]), "fsync_enabled": os.environ["FSYNC_ENABLED"] == "true"}))' \ + > "${RUNNER_TEMP}/expected.json" + python3 scripts/prism_load_probe.py table "${RUNNER_TEMP}/rows" --expected "${RUNNER_TEMP}/expected.json" \ + | tee "${RUNNER_TEMP}/rows/table.md" \ + >> "${GITHUB_STEP_SUMMARY}" + - name: Upload the table and the rows + uses: actions/upload-artifact@v7 + with: + name: prism-probe-table + overwrite: true + retention-days: 90 + path: ${{ runner.temp }}/rows + if-no-files-found: error From e9083619aa60feaef1cb557187c8ae287eb94eb7 Mon Sep 17 00:00:00 2001 From: Dan Hepworth Date: Tue, 29 Sep 2026 16:08:35 -0400 Subject: [PATCH 06/10] main: sync the PRISM load-harness nightly workflow from 3.x.x (#586, #587, #593, #594) (#595) Adds #586's disk-full and clock-jump tool installs, #587's weekly cutover rehearsal settings, #593's comment and #594's nightly Stratum fuzz job. Byte-identical to 3.x.x. Co-authored-by: Claude Opus 5.5 (1M context) --- .github/workflows/prism-load-nightly.yml | 75 ++++++++++++++++++++++-- 1 file changed, 70 insertions(+), 5 deletions(-) diff --git a/.github/workflows/prism-load-nightly.yml b/.github/workflows/prism-load-nightly.yml index a30a3d2e6..c7aaa2283 100644 --- a/.github/workflows/prism-load-nightly.yml +++ b/.github/workflows/prism-load-nightly.yml @@ -27,8 +27,9 @@ name: PRISM load harness (nightly) # # Once a week the `live-weekly` job runs the longer live regtest scenarios # listed in test/prism-weekly-gated-tests.txt instead (#487's L4 weekly set): -# today #545's 2.x.x migration lifecycle. The weekly schedule runs only that -# job; the nightly jobs keep their own schedule. +# today #545's 2.x.x migration lifecycle and #575's measured cutover of a +# mainnet-shaped 2.x.x ledger. The weekly schedule runs only that job; the +# nightly jobs keep their own schedule. # # Schedules run from the default branch's copy of this file; the build and # the presets come from `ref` (3.x.x unless dispatched otherwise). @@ -79,6 +80,13 @@ on: type: boolean required: false default: false + fuzz: + description: >- + Also run the time-boxed Stratum fuzz targets (#575; the nightly + schedule always does). + type: boolean + required: false + default: false permissions: contents: read @@ -215,7 +223,8 @@ jobs: # in required integration mode against a real qbitd and PostgreSQL 16, and # proven executed with the same gate manifest check the PR suite uses. # That is #523's qbitd -reindex crash variant, #524's 130-payee - # weighted-recipients variant and scenario 5's dense-cadence soak. + # weighted-recipients variant, scenario 5's dense-cadence soak, and #575's + # PostgreSQL disk-exhaustion and wall-clock-jump variants. live-nightly: name: Live regtest nightly variants if: ${{ github.event.schedule == '17 2 * * *' || inputs.live }} @@ -252,6 +261,12 @@ jobs: uses: ./.github/actions/apt-install with: packages: postgresql-16 + # #575: the disk-exhaustion variant mounts a small ext4 image with + # fuse2fs (no root), the clock-jump variant preloads libfaketime. + - name: Install the disk-full and clock-jump tools + uses: ./.github/actions/apt-install + with: + packages: fuse2fs fuse3 faketime libfaketime - name: Restore the pinned qbit release from the cache uses: actions/cache/restore@v6 with: @@ -297,7 +312,14 @@ jobs: name: Live regtest weekly scenarios if: ${{ github.event.schedule == '43 3 * * 0' || inputs.weekly }} runs-on: blacksmith-8vcpu-ubuntu-2404 - timeout-minutes: 60 + # #575's scenario writes mainnet's 13,566 blocks and about 2M of its 65.5M + # share rows (1x at 1/32 until #582 lands, then 1/16 at 4.1M rows), keeps + # two copies of that ledger (about 8 GB at 1/32, 15 GB at 1/16) + # in the service database, and cuts it over. It takes 7.6 minutes at 1/64 + # of the rows on a 22-core host in release, and its row-proportional + # phases put 1/16 near 20 minutes there, so the job builds in release and + # allows two hours. + timeout-minutes: 120 services: postgres: image: postgres:16 @@ -344,7 +366,7 @@ jobs: set -euo pipefail mapfile -t tests < <(sed -e 's/#.*//' -e 's/[[:space:]]//g' test/prism-weekly-gated-tests.txt \ | grep -v '^$' | sed 's/^qbit-prism-server::live_regtest:://') - cargo test --locked -p qbit-prism-server --test live_regtest -- \ + cargo test --locked --release -p qbit-prism-server --test live_regtest -- \ --ignored --exact --test-threads=1 --nocapture "${tests[@]}" 2>&1 | tee "${PRISM_TEST_LOG}" - name: Prove every listed scenario executed run: | @@ -363,3 +385,46 @@ jobs: ${{ env.PRISM_TEST_GATE_MANIFEST }} ${{ env.PRISM_TEST_LOG }} if-no-files-found: error + + # #575: the Stratum cargo-fuzz targets in crates/qbit-prism-server/fuzz + # (line codec, per-connection session state machine, codec parsers), all + # three at once for 20 minutes, from the checked-in seeds plus the corpus + # the previous night kept. Gated on no crash, leak, timeout or OOM input; + # the corpus is saved either way, and any finding is uploaded with the logs. + # .github/scripts/prism-stratum-fuzz.sh pins the nightly toolchain and + # cargo-fuzz and says how to reproduce a finding. + stratum-fuzz: + name: Stratum fuzzing (20 min) + if: ${{ github.event.schedule == '17 2 * * *' || inputs.fuzz }} + runs-on: blacksmith-8vcpu-ubuntu-2404 + timeout-minutes: 45 + steps: + - name: Check out the ref under test + uses: actions/checkout@v7 + with: + ref: ${{ inputs.ref || '3.x.x' }} + - name: Restore the kept corpus + uses: actions/cache/restore@v6 + with: + path: ${{ runner.temp }}/stratum-fuzz-corpus + key: prism-stratum-fuzz-corpus-v1-${{ github.run_id }} + restore-keys: prism-stratum-fuzz-corpus-v1- + - name: Run the fuzz targets and gate on no crash + run: | + bash .github/scripts/prism-stratum-fuzz.sh \ + "${RUNNER_TEMP}/stratum-fuzz-corpus" "${RUNNER_TEMP}/stratum-fuzz-out" 1200 + - name: Keep the corpus for the next night + if: ${{ !cancelled() }} + uses: actions/cache/save@v6 + with: + path: ${{ runner.temp }}/stratum-fuzz-corpus + key: prism-stratum-fuzz-corpus-v1-${{ github.run_id }} + - name: Upload the logs and any crashing inputs + if: ${{ !cancelled() }} + uses: actions/upload-artifact@v7 + with: + name: prism-stratum-fuzz + overwrite: true + retention-days: 30 + path: ${{ runner.temp }}/stratum-fuzz-out + if-no-files-found: error From 93cf49124c075b5438f51a477a18793392b036d9 Mon Sep 17 00:00:00 2001 From: Anatolie Date: Fri, 16 Oct 2026 17:24:44 -0400 Subject: [PATCH 07/10] main: sync the PRISM load-harness nightly workflow from 3.x.x at c95febb3 (#590, #596, #588, #597, #552, #551, #553, #554) (#601) * main: sync the PRISM load-harness nightly workflow from 3.x.x (#590, #596) Co-Authored-By: Claude Opus 5.5 * main: sync the PRISM load-harness nightly workflow from 3.x.x (#590, #596, #588, #597) Adds #588's Saturday 5.5 h soak schedule and its live-nightly step that runs every listed test binary, so qbit-prism-load::soak_smoke runs beside the live_regtest variants, and #597's L6 shipped-images lane on the Sunday schedule. Byte-identical to 3.x.x at 1932cb30. Co-Authored-By: Claude Opus 5.5 * main: re-sync the PRISM load-harness nightly workflow from 3.x.x (#552, #551, #553, #554, #582) The file is 3.x.x's copy byte for byte again, as of 3.x.x c95febb3; the last change to it there is bd1ddeb0 (#663). Since 1932cb30 (#597), five 3.x.x changes landed in it: - #552 (#613): the `bridging` job runs the short plan on the fake and the real node on one runner, nightly, on the run-load label and on a dispatch with the new `bridging` input. - #551 (#606): every preset job builds a trend row; on a schedule the `evidence` job (the one job with `contents: write`) appends the rows to the orphan `ci-evidence` branch, applies the report-only regression rule, promotes the soak's and flagged runs' bundles to the rolling `ci-evidence/YYYY-MM` pre-release, and keeps one `prism-load-regression` issue. PR and dispatch runs get the read-only `evidence-preview` instead. - #553 (#618): the live nightly and weekly lists gain the chain-event scenarios under 2,000 sessions; the workflow's comments say so. - #554 (#635): the run job installs fuse2fs and fuse3 for a preset whose faults fill the WAL volume (faults-long-real-node, Saturday). - #582 (#663): comment only. Without this, main's scheduled copy (#595) still has no qbitd install step, so every `--node qbitd` preset fails with "--node qbitd needs --qbitd-bin or QBITD_BIN" (scheduled runs 37188928798 and 37287515779). Co-Authored-By: Claude Opus 5.5 --------- Co-authored-by: Claude Opus 5.5 Co-authored-by: Robert Clarke --- .github/workflows/prism-load-nightly.yml | 863 ++++++++++++++++++++++- 1 file changed, 832 insertions(+), 31 deletions(-) diff --git a/.github/workflows/prism-load-nightly.yml b/.github/workflows/prism-load-nightly.yml index c7aaa2283..f25d73e1e 100644 --- a/.github/workflows/prism-load-nightly.yml +++ b/.github/workflows/prism-load-nightly.yml @@ -25,27 +25,123 @@ name: PRISM load harness (nightly) # being measured. Rates are only comparable run to run on the same runner # class, which is why the runner is part of the preset. # +# The bridging pair (short-plan-real-node and short-plan-fake-node, #547, +# #552) runs the same short plan on a managed regtest qbitd and on the fake +# node, on the same runner class, so their difference is what the real node +# costs. The matrix runs each half on its own VM; the `bridging` job (#552) +# also runs both on one VM and writes the difference as a row, so the cost is +# not blurred by the spread between two VMs. +# # Once a week the `live-weekly` job runs the longer live regtest scenarios # listed in test/prism-weekly-gated-tests.txt instead (#487's L4 weekly set): -# today #545's 2.x.x migration lifecycle and #575's measured cutover of a -# mainnet-shaped 2.x.x ledger. The weekly schedule runs only that job; the +# today #545's 2.x.x migration lifecycle, #575's measured cutover of a +# mainnet-shaped 2.x.x ledger and #553's long set of chain events under +# 2,000 sessions. The weekly schedule runs only that job (and L6); the # nightly jobs keep their own schedule. # +# #575 item 2: the long soak has a third schedule, Saturday 05:41 UTC, whose +# plan selects `weekly`: the soak-weekly preset, one server lifetime of about +# 5.5 h looping the mainnet-shape-130-addresses, rental-churn-bursts-and-storms +# and tip-delivery workloads, with forced share-partition rollovers and the +# share-archive retention cycle under the load, sampled every minute and gated +# on RSS and descriptor trends, connection drift, bounded WAL, zero lost +# shares and exact reconciliation (docs/prism-soak.md). The preset's timeout +# is 355 min, under the 6 h job limit. It has its own concurrency group, so it +# never holds the nightly queue; neither live job nor the fuzzing runs on its +# schedule. #549's guard never skips it: the soak measures drift over a week +# of commits and its bounds are still being calibrated, so a second lifetime +# on an unchanged commit is evidence, not waste, and the commit the guard +# compares against is the one the nightly presets tested, not the soak. A +# failed soak opens or updates the failure issue like any scheduled run. Cost: +# one 8 vCPU runner for about 5 h 40 min plus the shared release build, about +# 350 runner-minutes a week, about $5.60 at $0.016 per 8 vCPU minute. +# `preset: soak-weekly` (or `weekly`) dispatches it by hand. +# +# #554: the same Saturday selection runs faults-long-real-node on its own +# 16 vCPU runner beside the soak: every fault under load with 2,000 sessions +# on the real node, the primary's WAL on a fuse2fs volume the `wal-disk-full` +# fault fills, about 60 minutes with the ramp (timeout 100). Cost: about +# $2 a week at $0.032 per 16 vCPU minute. The run job installs fuse2fs +# and fuse3 only for a preset whose --faults lists wal-disk-full. +# +# The `shipped-images` job is #487's lane L6 (#544): the images and the +# Compose wiring operators run, under the `prism` profile with the HA overlay +# on a real regtest node, mined by cpuminer-opt and qbit-prism-miner. It runs +# with the Sunday schedule, on demand (`images`), and on a pull request into +# 3.x.x or a 3.x.x push that changes a Dockerfile, a Compose file or what they +# build from: the `changes` job applies that list, WATCHED_PATHS in +# scripts/prism_shipped_image_lane.py, because a trigger-level `paths` filter +# would also gate the `run-load` label below. Those events run nothing else. +# It is not a required check. +# # Schedules run from the default branch's copy of this file; the build and # the presets come from `ref` (3.x.x unless dispatched otherwise). +# +# #549 (S3 of #487): +# - A maintainer applying the `run-load` label to a PR runs the nightly +# preset set against the PR's merge result, on `pull_request` (never +# `pull_request_target`), in a concurrency group of its own so it can +# never cancel a queued nightly. Only the `labeled` event starts it: a +# later push needs the label applied again. Other labels start nothing. +# - The guard job ends the nightly schedule early when 3.x.x's head is the +# commit the last scheduled nightly tested, and cancels the run so the +# skip shows as cancelled, never as passed. A nightly records the commit +# it tested in its `prism-load-tested-commit` artifact once every preset, +# and both halves of the bridging job (#552), reached its gate, pass or +# fail. Neither weekly schedule (the Sunday +# live scenarios, the Saturday soak) nor a dispatch is ever skipped. +# - Each preset's artifact carries host.json (runner label, filesystem mount +# options, block device write cache and FUA) and is kept for 90 days +# (#487 decision 3). +# - A failed scheduled run opens, or comments on, the one open issue +# labelled `prism-load-nightly-failure`. GitHub itself only emails +# whoever last edited the cron. +# +# #551 (S6 of #487, decision 3): +# - Every preset job builds its trend row (scripts/prism_load_trend.py +# row: commit, lane, preset, runner class, fsync cost and band, the +# headline numbers, the artifact URL) and uploads it as an artifact, on +# every event. +# - Only a scheduled run records anything: the `evidence` job, the one job +# here with `contents: write`, adds a row per preset, per non-preset job +# and per planned preset that left none to the orphan `ci-evidence` +# branch (trend//YYYY-MM.jsonl, append-only; it creates the branch +# on its first write), applies the report-only regression rule +# (scripts/prism_load_regress.py, test/prism-load-regression.toml, +# provisional until #542), promotes the weekly soak's and every flagged +# L2 run's bundle to the rolling `ci-evidence/YYYY-MM` pre-release, and +# opens or updates the one `prism-load-regression` issue with the commit +# range and both runs' numbers. That issue never pages and never blocks +# a merge. All of it is .github/actions/prism-load-evidence, which the +# L3 tag job (#550) calls too. +# - A `run-load` pull request or a dispatch gets `evidence-preview` +# instead: the same rule against the recorded trend, read-only, in the +# job summary. PR and dispatch runs write nothing. on: schedule: - cron: "17 2 * * *" # Weekly, on Sunday. The jobs' `if` conditions compare against both crons. - cron: "43 3 * * 0" + # The weekly soak (#575), on Saturday: the plan, build and run jobs, with + # the plan selecting `weekly` by this exact string; the guard lets it + # through and the report job reports a failure. + - cron: "41 5 * * 6" + # `labeled` is #549's run-load trigger; the others let L6 see a PR's + # changes (the `changes` job decides whether it runs). + pull_request: + types: [opened, synchronize, reopened, labeled] + push: + branches: [3.x.x] workflow_dispatch: inputs: preset: description: >- Preset name(s), comma separated, from crates/qbit-prism-load/presets; `nightly` for the nightly set, `all` for nightly plus manual - (mainnet-shape-2600-addresses and #473's cells other than 400k fe1 async). + (mainnet-shape-2600-addresses and #473's cells other than 400k fe1 async), + `weekly` for the 5.5 h soak (soak-weekly) and the long fault set + (faults-long-real-node). required: true default: nightly ref: @@ -87,31 +183,135 @@ on: type: boolean required: false default: false + images: + description: >- + Also run L6, the shipped images under the Compose prism profile + with the HA overlay (#544; the weekly schedule always does). + type: boolean + required: false + default: false + bridging: + description: >- + Also run the bridging lane, the short plan on the fake and the real + node on one runner (#552; the nightly schedule and the run-load + label always do). + type: boolean + required: false + default: false permissions: contents: read +# The weekly soak holds its runner for most of six hours, so it queues only +# behind another soak, never in front of a nightly. A labelled PR run gets a +# group per PR and label, so it never waits on or cancels a nightly or a soak, +# and another label's (empty) run never cancels a pending run-load run. A PR's +# other events (L6 only) get a group per PR, where a newer commit cancels the +# older run; a 3.x.x push (L6 only) gets its own. concurrency: - group: prism-load-nightly - cancel-in-progress: false + group: >- + ${{ github.event_name == 'pull_request' + && (github.event.action == 'labeled' + && format('prism-load-pr-{0}-{1}', github.event.pull_request.number, github.event.label.name) + || format('prism-shipped-images-pr-{0}', github.event.pull_request.number)) + || github.event_name == 'push' && 'prism-shipped-images-push' + || format('prism-load-{0}', (github.event.schedule == '41 5 * * 6' + || contains(inputs.preset || '', 'soak-weekly') + || inputs.preset == 'weekly') && 'soak' || 'nightly') }} + cancel-in-progress: ${{ github.event_name == 'pull_request' && github.event.action != 'labeled' }} jobs: + # Every job but live-weekly and report needs this one, so on a PR it is + # the one place the label is checked, and on the nightly schedule it is the + # P4 cost lever: nothing starts when 3.x.x has not moved. Only the nightly + # cron compares; the Saturday soak (#575) passes straight through. + guard: + name: Skip an unchanged night + if: >- + ${{ github.event.schedule != '43 3 * * 0' && github.event_name != 'push' + && (github.event_name != 'pull_request' || github.event.label.name == 'run-load') }} + runs-on: blacksmith-2vcpu-ubuntu-2404 + permissions: + actions: write + contents: read + outputs: + skip: ${{ steps.last.outputs.skip }} + steps: + - name: Compare 3.x.x's head with the commit the last nightly tested + id: last + if: ${{ github.event.schedule == '17 2 * * *' }} + env: + GH_TOKEN: ${{ github.token }} + REPO: ${{ github.repository }} + RUN_ID: ${{ github.run_id }} + DEFAULT_BRANCH: ${{ github.event.repository.default_branch }} + run: | + set -euo pipefail + head="$(gh api "repos/${REPO}/commits/3.x.x" --jq .sha)" || head="" + # Only a scheduled nightly's report job uploads the artifact, and + # the artifacts API lists it newest first however many skipped + # (cancelled) or weekly runs came since, so the first unexpired one + # from the default branch is what the last nightly tested. Any + # failure to find or read it runs the night: a skip needs positive + # evidence. + tested="" tested_run="" + tested_run="$(gh api "repos/${REPO}/actions/artifacts?name=prism-load-tested-commit&per_page=100" \ + --jq "[.artifacts[] | select(.expired == false + and .workflow_run.head_branch == \"${DEFAULT_BRANCH}\" + and .workflow_run.id != ${RUN_ID})][0].workflow_run.id // empty")" || tested_run="" + dir="${RUNNER_TEMP}/tested" + if [[ -n "${tested_run}" ]] \ + && gh run download "${tested_run}" --repo "${REPO}" -n prism-load-tested-commit -D "${dir}" \ + && [[ -s "${dir}/commit" ]]; then + tested="$(tr -d '[:space:]' < "${dir}/commit")" + fi + echo "3.x.x is at ${head}; the last nightly tested ${tested:-nothing found} (run ${tested_run:-none})" + if [[ -n "${head}" && "${tested}" == "${head}" ]]; then + printf 'skip=true\n' >> "${GITHUB_OUTPUT}" + { + echo "### Skipped: 3.x.x has not moved" + echo + echo "3.x.x is still at \`${head}\`, which [run ${tested_run}](${GITHUB_SERVER_URL}/${REPO}/actions/runs/${tested_run}) tested." + echo "This run is cancelled so the skip does not show as a pass." + } >> "${GITHUB_STEP_SUMMARY}" + else + printf 'skip=false\n' >> "${GITHUB_OUTPUT}" + fi + - name: Cancel the run + if: ${{ steps.last.outputs.skip == 'true' }} + env: + GH_TOKEN: ${{ github.token }} + REPO: ${{ github.repository }} + RUN_ID: ${{ github.run_id }} + run: | + set -euo pipefail + gh run cancel "${RUN_ID}" --repo "${REPO}" + # The cancellation ends this job; if it has not within five + # minutes, fail rather than let the skip read as a pass. The other + # jobs are skipped on this job's output either way. + for _ in $(seq 30); do sleep 10; done + echo "the run was not cancelled within five minutes" >&2 + exit 1 + plan: name: Plan the preset matrix - if: ${{ github.event.schedule != '43 3 * * 0' }} + needs: guard + if: ${{ needs.guard.outputs.skip != 'true' }} runs-on: blacksmith-2vcpu-ubuntu-2404 outputs: matrix: ${{ steps.plan.outputs.matrix }} sha: ${{ steps.plan.outputs.sha }} steps: + # A labelled PR runs its merge result (refs/pull/N/merge), which is + # what would land; everything else runs `ref`, 3.x.x by default. - name: Check out the ref under test uses: actions/checkout@v7 with: - ref: ${{ inputs.ref || '3.x.x' }} + ref: ${{ github.event_name == 'pull_request' && github.ref || inputs.ref || '3.x.x' }} - name: Select presets and pin the commit id: plan env: - SELECTION: ${{ inputs.preset || 'nightly' }} + SELECTION: ${{ inputs.preset || (github.event.schedule == '41 5 * * 6' && 'weekly') || 'nightly' }} run: | set -euo pipefail matrix="$(python3 scripts/prism_load_matrix.py "${SELECTION}")" @@ -171,6 +371,37 @@ jobs: uses: ./.github/actions/apt-install with: packages: postgresql-16 + # #554: a preset whose faults fill the primary's WAL volume mounts a + # small ext4 image with fuse2fs, without root, as #575's disk-full + # scenario does. + - name: Check whether the preset fills a WAL volume + id: wal_volume + env: + PRESET: ${{ matrix.preset }} + run: | + set -euo pipefail + faults="$(jq -r '.args["--faults"] // ""' "crates/qbit-prism-load/presets/${PRESET}.json")" + if [[ "${faults}" == *wal-disk-full* ]]; then + printf 'needed=true\n' >> "${GITHUB_OUTPUT}" + fi + - name: Install the disk-full tools + if: ${{ steps.wal_volume.outputs.needed == 'true' }} + uses: ./.github/actions/apt-install + with: + packages: fuse2fs fuse3 + # A `--node qbitd` preset (#547) drives the pinned qbit release; the + # install script exports QBITD_BIN, which the harness reads when the + # command line gives no --qbitd-bin. A fake-node preset ignores it. + - name: Restore the pinned qbit release from the cache + uses: actions/cache/restore@v6 + with: + path: ~/.cache/prism-qbit + key: prism-qbit-v1-${{ runner.os }}-${{ runner.arch }}-${{ hashFiles('.github/scripts/install-prism-qbit.sh') }} + - name: Install pinned qbit + run: | + set -euo pipefail + PRISM_QBIT_CACHE_DIR="${HOME}/.cache/prism-qbit" \ + bash .github/scripts/install-prism-qbit.sh "${RUNNER_TEMP}/prism-qbit" - name: Download the binaries built once uses: actions/download-artifact@v8 with: @@ -186,7 +417,7 @@ jobs: tar -C target/release -xf "${RUNNER_TEMP}/prism-load-binaries.tar" touch target/release/qbit-prism-server target/release/qbit-prism-load \ target/release/qbit-prism-load-gate - - name: Record the host + - name: Name the run in the summary env: PRESET: ${{ matrix.preset }} RUNNER_LABEL: ${{ matrix.runner }} @@ -195,41 +426,250 @@ jobs: { echo "### ${PRESET} on ${RUNNER_LABEL}" echo - echo '```' - git rev-parse HEAD - nproc - free -m - uname -r - echo '```' + echo "Commit \`$(git rev-parse HEAD)\`; the host, its disk and the runner label are in host.json below." + echo } >> "${GITHUB_STEP_SUMMARY}" - name: Run and gate the preset env: PRISM_LOAD_TIP_LAST_NOTIFY_P99_BUDGET_MS: ${{ inputs.tip_last_notify_p99_budget_ms }} PRISM_LOAD_MAX_SHORTFALL: ${{ inputs.max_shortfall }} PRESET: ${{ matrix.preset }} + RUNNER_LABEL: ${{ matrix.runner }} run: bash .github/scripts/prism-load-run.sh "${PRESET}" "${RUNNER_TEMP}/prism-load-out" + # prism-load-run.sh writes gate-exit-code after the gate returns; 0 + # (pass) or 1 (fail) is a verdict on the harness's result. The report + # job records a nightly's commit as tested only if every planned preset + # left this marker, so a setup or runner failure, or a gate that could + # not read its inputs or did not run, is retried the next night. + - name: Mark the preset as gated + if: ${{ !cancelled() }} + env: + PRESET: ${{ matrix.preset }} + run: | + set -euo pipefail + mkdir -p "${RUNNER_TEMP}/gated" + code="$(cat "${RUNNER_TEMP}/prism-load-out/gate-exit-code" 2>/dev/null || true)" + if [[ "${code}" == 0 || "${code}" == 1 ]]; then + touch "${RUNNER_TEMP}/gated/${PRESET}" + fi + - name: Upload the gated marker + if: ${{ !cancelled() }} + uses: actions/upload-artifact@v7 + with: + name: prism-load-gated-${{ matrix.preset }} + overwrite: true + retention-days: 3 + path: ${{ runner.temp }}/gated + if-no-files-found: ignore - name: Upload the harness reports + id: reports if: ${{ !cancelled() }} uses: actions/upload-artifact@v7 with: name: prism-load-${{ matrix.preset }} overwrite: true - retention-days: 30 + # #487 decision 3: the load evidence is kept 90 days. + retention-days: 90 path: ${{ runner.temp }}/prism-load-out if-no-files-found: error + # #551: the preset's trend row, on every event; only the scheduled + # `evidence` job records it. What the run did not measure is null. + - name: Build the trend row + if: ${{ !cancelled() }} + env: + PRESET: ${{ matrix.preset }} + RUNNER_LABEL: ${{ matrix.runner }} + COMMIT: ${{ needs.plan.outputs.sha }} + REF: ${{ github.event_name == 'pull_request' && github.ref || inputs.ref || '3.x.x' }} + ARTIFACT_URL: ${{ steps.reports.outputs.artifact-url }} + ARTIFACT_ID: ${{ steps.reports.outputs.artifact-id }} + run: | + set -euo pipefail + python3 scripts/prism_load_trend.py row --dir "${RUNNER_TEMP}/prism-load-out" \ + --preset "${PRESET}" --lane auto --runner-class "${RUNNER_LABEL}" \ + --commit "${COMMIT}" --ref "${REF}" \ + --run-id "${GITHUB_RUN_ID}" --run-attempt "${GITHUB_RUN_ATTEMPT}" \ + --event "${GITHUB_EVENT_NAME}" --repository "${GITHUB_REPOSITORY}" \ + --server-url "${GITHUB_SERVER_URL}" \ + --artifact-name "prism-load-${PRESET}" --artifact-url "${ARTIFACT_URL}" \ + --artifact-id "${ARTIFACT_ID}" \ + --out "${RUNNER_TEMP}/prism-load-trend/${PRESET}.json" + - name: Upload the trend row + if: ${{ !cancelled() }} + uses: actions/upload-artifact@v7 + with: + name: prism-load-trend-${{ matrix.preset }} + overwrite: true + retention-days: 7 + path: ${{ runner.temp }}/prism-load-trend + if-no-files-found: error + + # #552 (S12 of #487): the bridging lane. The same short plan on the fake + # node and on the real node, one after the other on one runner, so the + # difference is what the real node costs and not the spread between two + # VMs. It runs in addition to the matrix's short-plan-fake-node and + # short-plan-real-node jobs, which gate each half on its own runner. The + # order alternates with the run number, so whatever going first or second + # costs (a cold page cache, say) does not always land on one side; the row + # records it. scripts/prism_load_bridge.py writes the row and the summary + # table: the real node's cost on this runner, not a D1 verdict (#487 + # decisions 1 and 5), with anything a side could not measure left unknown. + # Both halves always run and the row is always written; the job then fails + # if either half failed its gate. + bridging: + name: Bridging (short plan, fake and real node on one runner) + needs: [guard, plan, build] + if: >- + ${{ needs.guard.outputs.skip != 'true' && (github.event.schedule == '17 2 * * *' + || github.event_name == 'pull_request' || inputs.bridging) }} + runs-on: blacksmith-8vcpu-ubuntu-2404 + # Each half runs under its preset's timeout_minutes (30), so this is + # both of them plus the setup, the row and the upload. + timeout-minutes: 75 + # #551: the commit and the attempt the job's trend row names (it checks + # out this commit; a re-run of another job leaves these as they were). + outputs: + commit: ${{ needs.plan.outputs.sha }} + attempt: ${{ github.run_attempt }} + steps: + - name: Check out the pinned commit + uses: actions/checkout@v7 + with: + ref: ${{ needs.plan.outputs.sha }} + - name: Install PostgreSQL 16 server binaries + uses: ./.github/actions/apt-install + with: + packages: postgresql-16 + - name: Restore the pinned qbit release from the cache + uses: actions/cache/restore@v6 + with: + path: ~/.cache/prism-qbit + key: prism-qbit-v1-${{ runner.os }}-${{ runner.arch }}-${{ hashFiles('.github/scripts/install-prism-qbit.sh') }} + - name: Install pinned qbit + run: | + set -euo pipefail + PRISM_QBIT_CACHE_DIR="${HOME}/.cache/prism-qbit" \ + bash .github/scripts/install-prism-qbit.sh "${RUNNER_TEMP}/prism-qbit" + - name: Download the binaries built once + uses: actions/download-artifact@v8 + with: + name: prism-load-binaries + path: ${{ runner.temp }} + - name: Unpack the binaries into target/release + run: | + set -euo pipefail + mkdir -p target/release + tar -C target/release -xf "${RUNNER_TEMP}/prism-load-binaries.tar" + touch target/release/qbit-prism-server target/release/qbit-prism-load \ + target/release/qbit-prism-load-gate + - name: Run both halves on this runner + id: sides + env: + PRISM_LOAD_TIP_LAST_NOTIFY_P99_BUDGET_MS: ${{ inputs.tip_last_notify_p99_budget_ms }} + PRISM_LOAD_MAX_SHORTFALL: ${{ inputs.max_shortfall }} + RUNNER_LABEL: blacksmith-8vcpu-ubuntu-2404 + RUN_NUMBER: ${{ github.run_number }} + run: | + set -euo pipefail + declare -A preset=([fake]=short-plan-fake-node [real]=short-plan-real-node) + if (( RUN_NUMBER % 2 )); then order=(real fake); else order=(fake real); fi + printf 'order=%s,%s\n' "${order[@]}" >> "${GITHUB_OUTPUT}" + for side in "${order[@]}"; do + { + echo "### ${preset[${side}]} on ${RUNNER_LABEL} (bridging, ${side} side)" + echo + echo "Commit \`$(git rev-parse HEAD)\`; order ${order[*]}." + echo + } >> "${GITHUB_STEP_SUMMARY}" + # Each half gets its preset's own timeout_minutes (the matrix + # enforces it as the job's), so a hung half cannot use up the + # job's and leave the other half and the row unrun. timeout + # signals the half's whole process group. A failing or timed-out + # half is its gate's verdict (or its missing gate-exit-code), + # read in the last step; the other half still runs. + minutes="$(jq -er .timeout_minutes "crates/qbit-prism-load/presets/${preset[${side}]}.json")" + timeout --kill-after=60 "${minutes}m" \ + bash .github/scripts/prism-load-run.sh "${preset[${side}]}" \ + "${RUNNER_TEMP}/prism-load-bridging/${side}" || true + done + - name: Write the comparison row + if: ${{ !cancelled() && steps.sides.outputs.order != '' }} + env: + ORDER: ${{ steps.sides.outputs.order }} + SHA: ${{ needs.plan.outputs.sha }} + RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} + run: | + set -euo pipefail + out="${RUNNER_TEMP}/prism-load-bridging" + python3 scripts/prism_load_bridge.py --fake "${out}/fake" --real "${out}/real" \ + --order "${ORDER}" --out "${out}/bridge-row.json" --summary "${GITHUB_STEP_SUMMARY}" \ + --commit "${SHA}" --run-url "${RUN_URL}" + # As the matrix's presets do: the report job records a nightly's commit + # as tested only if this marker says both halves reached a verdict + # (gate exit 0 or 1), so a bridge that failed before that, or timed + # out, is retried the next night rather than skipped as unchanged. + - name: Mark the bridge as gated + if: ${{ !cancelled() }} + run: | + set -euo pipefail + mkdir -p "${RUNNER_TEMP}/gated" + for side in fake real; do + code="$(cat "${RUNNER_TEMP}/prism-load-bridging/${side}/gate-exit-code" 2>/dev/null || true)" + [[ "${code}" == 0 || "${code}" == 1 ]] || exit 0 + done + touch "${RUNNER_TEMP}/gated/bridging" + - name: Upload the gated marker + if: ${{ !cancelled() }} + uses: actions/upload-artifact@v7 + with: + name: prism-load-gated-bridging + overwrite: true + retention-days: 3 + path: ${{ runner.temp }}/gated + if-no-files-found: ignore + - name: Upload both runs and the row + if: ${{ !cancelled() }} + uses: actions/upload-artifact@v7 + with: + name: prism-load-bridging + overwrite: true + # #487 decision 3: the load evidence is kept 90 days. + retention-days: 90 + path: ${{ runner.temp }}/prism-load-bridging + if-no-files-found: error + - name: Fail if either half failed its gate + run: | + set -euo pipefail + failed=0 + for side in fake real; do + code="$(cat "${RUNNER_TEMP}/prism-load-bridging/${side}/gate-exit-code" 2>/dev/null || echo missing)" + if [[ "${code}" != 0 ]]; then + echo "the ${side} half's gate exit code is ${code}; see its gate.md in the summary" >&2 + failed=1 + fi + done + exit "${failed}" # The opt-in #[ignore] live_regtest variants (#521): each id in # test/prism-nightly-gated-tests.txt, run explicitly with --ignored --exact # in required integration mode against a real qbitd and PostgreSQL 16, and # proven executed with the same gate manifest check the PR suite uses. # That is #523's qbitd -reindex crash variant, #524's 130-payee - # weighted-recipients variant, scenario 5's dense-cadence soak, and #575's - # PostgreSQL disk-exhaustion and wall-clock-jump variants. + # weighted-recipients variant, scenario 5's dense-cadence soak, #575's + # PostgreSQL disk-exhaustion and wall-clock-jump variants and soak smoke, + # and #553's short set of chain events under 2,000 sessions, #598's guard + # included (about 13 minutes on a 22-core host in debug). live-nightly: name: Live regtest nightly variants - if: ${{ github.event.schedule == '17 2 * * *' || inputs.live }} + needs: guard + if: ${{ needs.guard.outputs.skip != 'true' && (github.event.schedule == '17 2 * * *' || inputs.live) }} runs-on: blacksmith-8vcpu-ubuntu-2404 timeout-minutes: 60 + # #551: the commit and the attempt a trend row names for this job. + outputs: + commit: ${{ steps.head.outputs.commit }} + # Job-level, so it is set even when a step before `head` failed. + attempt: ${{ github.run_attempt }} services: postgres: image: postgres:16 @@ -251,6 +691,9 @@ jobs: uses: actions/checkout@v7 with: ref: ${{ inputs.ref || '3.x.x' }} + - name: Record the commit under test + id: head + run: printf 'commit=%s\n' "$(git rev-parse HEAD)" >> "${GITHUB_OUTPUT}" - name: Name the gate manifest and the test log run: | printf 'PRISM_TEST_GATE_MANIFEST=%s/prism-nightly-gate-manifest.txt\n' "${RUNNER_TEMP}" >> "${GITHUB_ENV}" @@ -277,13 +720,25 @@ jobs: set -euo pipefail PRISM_QBIT_CACHE_DIR="${HOME}/.cache/prism-qbit" \ bash .github/scripts/install-prism-qbit.sh "${RUNNER_TEMP}/prism-qbit" + # Each :: in the list runs once with its own ids + # (qbit-prism-server::live_regtest and #575's qbit-prism-load::soak_smoke + # today). Every binary runs even if an earlier one failed; the step + # fails if any did. - name: Run the listed variants run: | set -euo pipefail - mapfile -t tests < <(sed -e 's/#.*//' -e 's/[[:space:]]//g' test/prism-nightly-gated-tests.txt \ - | grep -v '^$' | sed 's/^qbit-prism-server::live_regtest:://') - cargo test --locked -p qbit-prism-server --test live_regtest -- \ - --ignored --exact --test-threads=1 --nocapture "${tests[@]}" 2>&1 | tee "${PRISM_TEST_LOG}" + mapfile -t ids < <(sed -e 's/#.*//' -e 's/[[:space:]]//g' test/prism-nightly-gated-tests.txt | grep -v '^$') + : > "${PRISM_TEST_LOG}" + failed=0 + for target in $(printf '%s\n' "${ids[@]}" | awk -F'::' '{print $1 "::" $2}' | sort -u); do + package="${target%%::*}" + binary="${target#*::}" + mapfile -t tests < <(printf '%s\n' "${ids[@]}" | grep "^${target}::" | sed "s/^${target}:://") + cargo test --locked -p "${package}" --test "${binary}" -- \ + --ignored --exact --test-threads=1 --nocapture "${tests[@]}" 2>&1 | tee -a "${PRISM_TEST_LOG}" \ + || failed=1 + done + exit "${failed}" - name: Prove every listed variant executed run: | python3 scripts/check_gate_manifest.py \ @@ -307,19 +762,28 @@ jobs: # PostgreSQL's client tools for the scenarios' pg_dump and psql. Today that # is #545's migration lifecycle: a scaled 2.x.x ledger through migrate, # import-audits, native mining on the real node, exact reconciliation and - # the isolated-restore rollback of docs/prism-rust-migration.md. + # the isolated-restore rollback of docs/prism-rust-migration.md. And #553's + # long set of chain events under 2,000 sessions: #521 scenario 4's other + # four cases, scenario 5's 20-minute soak and 45 minutes of external tips + # at 400 shares/s. live-weekly: name: Live regtest weekly scenarios if: ${{ github.event.schedule == '43 3 * * 0' || inputs.weekly }} runs-on: blacksmith-8vcpu-ubuntu-2404 - # #575's scenario writes mainnet's 13,566 blocks and about 2M of its 65.5M - # share rows (1x at 1/32 until #582 lands, then 1/16 at 4.1M rows), keeps - # two copies of that ledger (about 8 GB at 1/32, 15 GB at 1/16) - # in the service database, and cuts it over. It takes 7.6 minutes at 1/64 + # #575's scenario writes mainnet's 13,566 blocks and about 4.1M of its + # 65.5M share rows (1x at 1/16), keeps two copies of that ledger (about + # 15 GB) in the service database, and cuts it over at the server's + # default statement timeout (#582). It takes 7.6 minutes at 1/64 # of the rows on a 22-core host in release, and its row-proportional # phases put 1/16 near 20 minutes there, so the job builds in release and - # allows two hours. - timeout-minutes: 120 + # allowed two hours; #553's long set adds about 75 minutes (45 of + # external tips, a 20-minute soak and four shorter cases, measured in + # debug), so it allows three. + timeout-minutes: 180 + outputs: + commit: ${{ steps.head.outputs.commit }} + # Job-level, so it is set even when a step before `head` failed. + attempt: ${{ github.run_attempt }} services: postgres: image: postgres:16 @@ -341,6 +805,9 @@ jobs: uses: actions/checkout@v7 with: ref: ${{ inputs.ref || '3.x.x' }} + - name: Record the commit under test + id: head + run: printf 'commit=%s\n' "$(git rev-parse HEAD)" >> "${GITHUB_OUTPUT}" - name: Name the gate manifest and the test log run: | printf 'PRISM_TEST_GATE_MANIFEST=%s/prism-weekly-gate-manifest.txt\n' "${RUNNER_TEMP}" >> "${GITHUB_ENV}" @@ -395,14 +862,22 @@ jobs: # cargo-fuzz and says how to reproduce a finding. stratum-fuzz: name: Stratum fuzzing (20 min) - if: ${{ github.event.schedule == '17 2 * * *' || inputs.fuzz }} + needs: guard + if: ${{ needs.guard.outputs.skip != 'true' && (github.event.schedule == '17 2 * * *' || inputs.fuzz) }} runs-on: blacksmith-8vcpu-ubuntu-2404 timeout-minutes: 45 + outputs: + commit: ${{ steps.head.outputs.commit }} + # Job-level, so it is set even when a step before `head` failed. + attempt: ${{ github.run_attempt }} steps: - name: Check out the ref under test uses: actions/checkout@v7 with: ref: ${{ inputs.ref || '3.x.x' }} + - name: Record the commit under test + id: head + run: printf 'commit=%s\n' "$(git rev-parse HEAD)" >> "${GITHUB_OUTPUT}" - name: Restore the kept corpus uses: actions/cache/restore@v6 with: @@ -428,3 +903,329 @@ jobs: retention-days: 30 path: ${{ runner.temp }}/stratum-fuzz-out if-no-files-found: error + + # #549: what a scheduled run leaves behind. The nightly records the commit + # its presets ran to a verdict on, for the guard's next comparison, and any + # failed scheduled run, nightly, weekly or the Saturday soak, opens or + # updates the one open `prism-load-nightly-failure` issue. A cancelled run + # (a skipped night) does neither. The soak never records a tested commit. + report: + name: Record the tested commit and report a failure + # `evidence` (#551) is here so its failure is reported like any job's. + needs: [guard, plan, build, run, bridging, live-nightly, live-weekly, stratum-fuzz, shipped-images, evidence] + if: ${{ !cancelled() && github.event_name == 'schedule' }} + runs-on: blacksmith-2vcpu-ubuntu-2404 + permissions: + contents: read + issues: write + steps: + - name: Download the gated markers + id: gated + if: >- + ${{ github.event.schedule == '17 2 * * *' + && (needs.run.result == 'success' || needs.run.result == 'failure') }} + # No preset reaching its gate means no markers to download. + continue-on-error: true + uses: actions/download-artifact@v8 + with: + pattern: prism-load-gated-* + path: ${{ runner.temp }}/gated + merge-multiple: true + - name: Write the tested commit if every preset reached its gate + id: tested + if: ${{ steps.gated.conclusion == 'success' }} + env: + SHA: ${{ needs.plan.outputs.sha }} + MATRIX: ${{ needs.plan.outputs.matrix }} + run: | + set -euo pipefail + mkdir -p "${RUNNER_TEMP}/gated" "${RUNNER_TEMP}/tested" + # `bridging` is #552's job, which runs on every nightly. + missing="$( { jq -r '.include[].preset' <<<"${MATRIX}"; echo bridging; } | while read -r preset; do + [[ -f "${RUNNER_TEMP}/gated/${preset}" ]] || echo "${preset}" + done)" + if [[ -n "${missing}" ]]; then + echo "Not recorded as tested; these presets never reached their gate:" "${missing}" + exit 0 + fi + printf '%s\n' "${SHA}" > "${RUNNER_TEMP}/tested/commit" + printf 'recorded=true\n' >> "${GITHUB_OUTPUT}" + - name: Upload the tested commit + if: ${{ steps.tested.outputs.recorded == 'true' }} + uses: actions/upload-artifact@v7 + with: + name: prism-load-tested-commit + overwrite: true + retention-days: 90 + path: ${{ runner.temp }}/tested/commit + if-no-files-found: error + - name: Open or update the failure issue + if: ${{ contains(needs.*.result, 'failure') }} + env: + GH_TOKEN: ${{ github.token }} + REPO: ${{ github.repository }} + RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} + SCHEDULE: >- + ${{ github.event.schedule == '43 3 * * 0' && 'weekly' + || github.event.schedule == '41 5 * * 6' && 'weekly soak' + || 'nightly' }} + SHA: ${{ needs.plan.outputs.sha }} + RESULTS: ${{ toJSON(needs) }} + run: | + set -euo pipefail + label=prism-load-nightly-failure + gh label create "${label}" --repo "${REPO}" --color B60205 \ + --description "A scheduled prism-load-nightly.yml run failed (#549)" 2>/dev/null || true + # The backticks are Markdown, not command substitution. + # shellcheck disable=SC2016 + failed="$(jq -r 'to_entries[] | select(.value.result == "failure") | "- `\(.key)`"' <<<"${RESULTS}")" + # shellcheck disable=SC2016 + body="$(printf 'The %s scheduled run failed: %s\n\nCommit: `%s`\n\nFailed jobs:\n%s\n' \ + "${SCHEDULE}" "${RUN_URL}" "${SHA:-3.x.x (the weekly job checks out the branch)}" "${failed}")" + issue="$(gh issue list --repo "${REPO}" --label "${label}" --state open --limit 1 --json number --jq '.[0].number // empty')" + if [[ -n "${issue}" ]]; then + gh issue comment "${issue}" --repo "${REPO}" --body "${body}" + else + gh issue create --repo "${REPO}" --label "${label}" \ + --title "PRISM load nightly: a scheduled run failed" \ + --body "${body} + + Opened by the report job of .github/workflows/prism-load-nightly.yml (#549). Later failures comment here while it is open; close it once the cause is fixed." + fi + + # #551 (S6 of #487): what a scheduled run records, and the one job in this + # workflow with `contents: write`. It adds this run's rows to ci-evidence: + # one per preset (built by the preset job), one per non-preset job that ran + # (the bridging pair as L2, the live scenarios as L4, the fuzzing, the + # shipped images as L6) and a + # `kind: missing` row for each planned preset whose job left none, so every + # scheduled run has rows. The prism-load-evidence action applies the + # report-only regression rule, promotes the soak's and every flagged L2 + # run's bundle, appends the rows (retrying a rejected push from a fresh + # fetch, never forcing) and keeps the `prism-load-regression` issue. + evidence: + name: Record the trend rows and apply the regression rule + needs: [guard, plan, build, run, bridging, live-nightly, live-weekly, stratum-fuzz, shipped-images] + if: ${{ !cancelled() && github.event_name == 'schedule' }} + runs-on: blacksmith-2vcpu-ubuntu-2404 + # Promotion downloads and uploads the soak's bundle; the append and the + # promotion each stop at their own deadline inside this. + timeout-minutes: 45 + permissions: + actions: read + contents: write + issues: write + steps: + # The scripts, the action and the rule's config are the tested + # commit's; a Sunday run has no plan and reads 3.x.x's. + - name: Check out the tested commit + uses: actions/checkout@v7 + with: + ref: ${{ needs.plan.outputs.sha || '3.x.x' }} + - name: Download the preset rows + # A run with no plan (the Sunday schedule) has none; when presets + # were planned a failed download fails the job, to be re-run, rather + # than recording the night without them. + continue-on-error: ${{ needs.plan.outputs.matrix == '' }} + uses: actions/download-artifact@v8 + with: + pattern: prism-load-trend-* + path: ${{ runner.temp }}/prism-load-trend + merge-multiple: true + - name: Add the job rows and the missing preset rows + env: + GH_TOKEN: ${{ github.token }} + NEEDS: ${{ toJSON(needs) }} + MATRIX: ${{ needs.plan.outputs.matrix }} + run: | + set -euo pipefail + # Each job's latest attempt, so a re-run preset job that left no + # row is missing at that attempt, not stood in for by its earlier + # row (the matrix jobs are named after their preset). + jobs="${RUNNER_TEMP}/prism-load-jobs.jsonl" + gh api --paginate "repos/${GITHUB_REPOSITORY}/actions/runs/${GITHUB_RUN_ID}/jobs?filter=latest&per_page=100" \ + --jq '.jobs[] | {name, run_attempt} | tojson' > "${jobs}" + python3 scripts/prism_load_trend.py job-rows --needs "${NEEDS}" --matrix "${MATRIX}" \ + --rows "${RUNNER_TEMP}/prism-load-trend" --jobs "${jobs}" \ + --run-id "${GITHUB_RUN_ID}" --run-attempt "${GITHUB_RUN_ATTEMPT}" \ + --event "${GITHUB_EVENT_NAME}" --repository "${GITHUB_REPOSITORY}" \ + --server-url "${GITHUB_SERVER_URL}" --ref 3.x.x + - name: Record the evidence + uses: ./.github/actions/prism-load-evidence + with: + rows-dir: ${{ runner.temp }}/prism-load-trend + write: "true" + github-token: ${{ github.token }} + + # #551: a `run-load` pull request or a dispatch compares its rows with the + # recorded trend and says so in the summary, with a read-only token. + evidence-preview: + name: Compare with the recorded trend (read-only) + needs: [plan, run] + if: ${{ !cancelled() && github.event_name != 'schedule' && needs.plan.result == 'success' }} + runs-on: blacksmith-2vcpu-ubuntu-2404 + timeout-minutes: 15 + permissions: + actions: read + contents: read + steps: + - name: Check out the tested commit + uses: actions/checkout@v7 + with: + ref: ${{ needs.plan.outputs.sha }} + - name: Download the preset rows + continue-on-error: true + uses: actions/download-artifact@v8 + with: + pattern: prism-load-trend-* + path: ${{ runner.temp }}/prism-load-trend + merge-multiple: true + # As the scheduled job does: a planned preset that left no row (at its + # latest attempt) is shown as missing, not silently absent. + - name: Add the missing preset rows + env: + GH_TOKEN: ${{ github.token }} + MATRIX: ${{ needs.plan.outputs.matrix }} + run: | + set -euo pipefail + jobs="${RUNNER_TEMP}/prism-load-jobs.jsonl" + gh api --paginate "repos/${GITHUB_REPOSITORY}/actions/runs/${GITHUB_RUN_ID}/jobs?filter=latest&per_page=100" \ + --jq '.jobs[] | {name, run_attempt} | tojson' > "${jobs}" + python3 scripts/prism_load_trend.py job-rows --needs '{}' --matrix "${MATRIX}" \ + --rows "${RUNNER_TEMP}/prism-load-trend" --jobs "${jobs}" \ + --run-id "${GITHUB_RUN_ID}" --run-attempt "${GITHUB_RUN_ATTEMPT}" \ + --event "${GITHUB_EVENT_NAME}" --repository "${GITHUB_REPOSITORY}" \ + --server-url "${GITHUB_SERVER_URL}" + - name: Compare + uses: ./.github/actions/prism-load-evidence + with: + rows-dir: ${{ runner.temp }}/prism-load-trend + write: "false" + github-token: ${{ github.token }} + + # L6's path filter (see the header). A PR's other events and a 3.x.x push + # land here; only a change to a watched path runs `shipped-images`. PRs into + # other branches (main's copy of this file) never run L6: their checkout + # has no lane to run. + changes: + name: Detect Dockerfile or Compose changes + if: >- + ${{ (github.event_name == 'pull_request' && github.event.action != 'labeled' + && github.base_ref == '3.x.x') + || github.event_name == 'push' }} + runs-on: blacksmith-2vcpu-ubuntu-2404 + outputs: + images: ${{ steps.match.outputs.images }} + steps: + - name: Check out the lane driver + uses: actions/checkout@v7 + with: + sparse-checkout: scripts/prism_shipped_image_lane.py + sparse-checkout-cone-mode: false + - name: Match the changed files against the watched paths + id: match + env: + GH_TOKEN: ${{ github.token }} + REPO: ${{ github.repository }} + PR: ${{ github.event.pull_request.number }} + BEFORE: ${{ github.event.before }} + AFTER: ${{ github.sha }} + run: | + set -euo pipefail + files="${RUNNER_TEMP}/changed-files" + # The APIs list at most 3,000 files for a PR and 300 for a + # comparison. A list at its cap may omit a watched path, so it runs + # the lane, as does a new branch with nothing to compare. + if [[ -n "${PR}" ]]; then + cap=3000 + gh api --paginate "repos/${REPO}/pulls/${PR}/files" --jq '.[] | .filename' > "${files}.names" + gh api --paginate "repos/${REPO}/pulls/${PR}/files" --jq '.[] | .previous_filename // empty' > "${files}.renames" + elif [[ "${BEFORE}" =~ ^0+$ ]]; then + cap=0 + : > "${files}.names" + : > "${files}.renames" + else + cap=300 + gh api "repos/${REPO}/compare/${BEFORE}...${AFTER}" --jq '.files[] | .filename' > "${files}.names" + gh api "repos/${REPO}/compare/${BEFORE}...${AFTER}" --jq '.files[] | .previous_filename // empty' > "${files}.renames" + fi + count="$(wc -l < "${files}.names")" + if (( count >= cap )); then + images=true + echo "The change list has ${count} files (cap ${cap}); running L6 in case it is truncated." + else + images="$(cat "${files}.names" "${files}.renames" | python3 scripts/prism_shipped_image_lane.py changed)" + fi + echo "L6 runs: ${images}" + printf 'images=%s\n' "${images}" >> "${GITHUB_OUTPUT}" + + # #487 L6 (#544): scripts/prism_shipped_image_lane.py builds the images with + # `docker compose build` (Blacksmith's builder keeps the layer cache between + # runs), brings up compose.yaml + compose.prism-ha.yaml with the `prism` + # profile, and holds it to the checks its docstring lists: both frontends + # ready, #281 criterion 1 across them, a verified found block for each + # client, exact ledger reconciliation and the public API's block list. A + # cold build is most of the 45 minutes; the scenario itself takes about ten. + shipped-images: + name: Shipped images (Compose prism profile, HA overlay) + needs: changes + # `changes` is skipped on a schedule or dispatch, so accept that too. + if: >- + ${{ !cancelled() && (needs.changes.outputs.images == 'true' + || github.event.schedule == '43 3 * * 0' || inputs.images) }} + runs-on: blacksmith-8vcpu-ubuntu-2404 + timeout-minutes: 45 + outputs: + commit: ${{ steps.head.outputs.commit }} + # Job-level, so it is set even when a step before `head` failed. + attempt: ${{ github.run_attempt }} + steps: + - name: Check out the ref under test + uses: actions/checkout@v7 + with: + ref: ${{ (github.event_name == 'schedule' || github.event_name == 'workflow_dispatch') && (inputs.ref || '3.x.x') || '' }} + - name: Record the commit under test + id: head + run: printf 'commit=%s\n' "$(git rev-parse HEAD)" >> "${GITHUB_OUTPUT}" + # The runner context is not available to a job's env, so name the + # directories here. + - name: Name the work and output directories + run: | + printf 'L6_WORK=%s/generated/l6\n' "${GITHUB_WORKSPACE}" >> "${GITHUB_ENV}" + printf 'L6_OUT=%s/prism-shipped-images\n' "${RUNNER_TEMP}" >> "${GITHUB_ENV}" + - name: Set up Blacksmith Docker builder + uses: useblacksmith/setup-docker-builder@v1 + - name: Record the host and the Compose version + run: | + set -euo pipefail + { + echo "### L6 on blacksmith-8vcpu-ubuntu-2404" + echo + echo '```' + git rev-parse HEAD + nproc + free -m + docker compose version + echo '```' + } >> "${GITHUB_STEP_SUMMARY}" + # The qbit clone is the slow part of staging; key it on the pin. + - name: Restore the qbit source cache + uses: actions/cache@v6 + with: + path: generated/qbit-git-cache + key: prism-shipped-images-qbit-src-v1-${{ hashFiles('config/upstream.env', 'config/upstream.env.example') }} + - name: Stage the qbit source and write the lane's env file + run: python3 scripts/prism_shipped_image_lane.py prepare --work "${L6_WORK}" + - name: Build the images with Compose + run: python3 scripts/prism_shipped_image_lane.py build --work "${L6_WORK}" + - name: Run the lane + run: python3 scripts/prism_shipped_image_lane.py run --work "${L6_WORK}" --out "${L6_OUT}" + - name: Upload the report, client logs and Compose logs + if: ${{ !cancelled() }} + uses: actions/upload-artifact@v7 + with: + name: prism-shipped-images + overwrite: true + retention-days: 30 + path: ${{ runner.temp }}/prism-shipped-images + if-no-files-found: error From 937e8277184dc4e68f356878d3c6391e33f93991 Mon Sep 17 00:00:00 2001 From: Alex Date: Thu, 22 Oct 2026 16:32:51 -0400 Subject: [PATCH 08/10] v3.0.0 chore: consolidate release for main (#778) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Freeze money-path vectors from 2.x.x and record the D2 payout differences (#302) * Freeze money-path vectors from 2.x.x and record D2 payout differences (#269) Export 48 money-path cases from 2.x.x (504846cc) into crates/qbit-prism/fixtures/vectors/. Topics 1-7 (window clipping, fractional oldest share, carry-only recipients, pool fee, dust recompute, largest-remainder ties, CTV chunks) come from the 2.x.x qbit-prism entry points. The bootstrap transition and below-target credit come from the 2.x.x Python rule code, with each rule's payout computed by the same engine. The exporter source and its regeneration command sit under fixtures/vectors/exporter/, outside any cargo target. tests/money_path_vectors.rs replays every case through the 3.x.x engine and reports all mismatches at once. Topics 1-7 match 2.x.x exactly. The five differing cases carry expected_2xx, expected_3xx and a d2_entry whose anchor must exist in docs/prism-rust-migration.md. That doc now records D2a bootstrap pooling, D2b below-target credit, and D2c prior balances during bootstrap, the last pending the umbrella owner's approval. checked_in_hand_worked_fixtures_match_engine now also asserts the power-law-accrual fixture's floor, on-chain count and accrued recipients. Co-Authored-By: Claude Opus 5 * Pin the money-path vectors and freeze the zero-fee boundary (#269) The vector test now pins the sha256 of every vector file next to SOURCE_COMMIT, so a vector changes only through a re-export from 2.x.x in a reviewed commit that updates the pin. It also requires the set of d2_entry anchors used by the cases to equal the entries documented in the "Payout differences from 2.x.x (decision D2)" section, which must be exactly D2a, D2b and D2c. Deleting a D2 case, or rewriting one into a plain matching case, now fails. pool_fee.json gains three cases where a 1-bps fee on a 9999-sat coinbase floors to zero. With a 1000-sat fixed floor, 2.x.x records the pool-fee account at 0 sats and emits no fee output, under both canonical and pool-fee-first ordering; with the day-one floor the coinbase is below 14720 and the policy fails. 3.x.x reproduces all three. The exporter's implemented_at citations now point at the lines that implement each rule (coordinator.rs:747 and 759,773; job_bundle.py:3400; accepted_preview_telemetry.py:144). The D2c reason labels build_collection_bundle as 2.x.x code, and D2b states the confirmation timing assumption behind its next-block payout. A real 1160-recipient TRUC chunk is left as a documented gap: it would add about 1.6 MB. Co-Authored-By: Claude Opus 5 * Note the 2.x.x share-weight settings 3.x.x does not carry (#269) 2.x.x read PRISM_STRATUM_SHARE_WEIGHT and PRISM_STRATUM_SHARE_WEIGHTS_JSON. A per-worker entry in the JSON object credited max(1, override) instead of the share target's difficulty; the default weight only filled the job's delivered share_weight field. 3.x.x supports neither. With the 2.x.x defaults crediting is unchanged, so this is not a D2 payout difference, but a deployment that set either variable must remove it before cutover. Co-Authored-By: Claude Opus 5 --------- Co-authored-by: Claude Opus 5 * Add a native SQL oracle for the payout-window read (#306) * Add a native SQL oracle for the payout-window read (#269) Item 3 of #269 asks for an independent proof that `Ledger::snapshot` returns exactly the eligible payout window. The 2.x.x oracle was a Python module written against the old per-row read; 3.x.x pages the window backwards through a different query, so this is a new oracle written natively in Rust. Each test gets a throwaway schema, drives the read through the public `Ledger` API only, and recomputes the expected window in one unpaged statement: an eligibility filter, a running-sum window function standing in for the subtract-until-zero loop, and a single `json_agg`. The oracle shares no SQL text, constants or private items with `ledger.rs`, so it survives the pending split of that module. It then compares membership, order and every field of every share, plus the captured cutoff. The anchor barrier is expressed with exact integer millisecond arithmetic rather than the read's `to_timestamp(double precision/1000)`, so the two sides are genuinely independent. Scenarios cover the 4096-row page boundary from either side and three pages deep, the row that straddles the remaining weight, rows sitting exactly on the anchor and one millisecond past it on each timestamp, rejected rows interleaved with accepted ones, difficulties at the top of the `u128` range, anchors ending in 999/000/001 near 2^41 and 2^43 milliseconds, and empty windows. Each read is given a ceiling so a cursor that stops advancing fails with a named error instead of hanging the run. The database URL comes from `PRISM_TEST_DATABASE_URL`. One guard decides the outcome: run when it is set, fail when it is missing while `PRISM_TEST_REQUIRE_INTEGRATION=1` or `GITHUB_JOB=prism-native-postgres` demands the integration suite, and otherwise print a skip line. Keying on `CI` would break `rust-tests`, which runs the whole workspace with no database. Co-Authored-By: Claude Opus 5 * Derive oracle anchors from the clock and assert the append path at the anchor (#269) Two follow-ups to the payout-window oracle. The suite pinned fixed anchors in 2030 and 2039. Because a pinned anchor has to sit above the wall clock, those were dated fuses: from January 2030 every test would have failed in CI for no reason in the code. Anchors at an ordinary magnitude are now derived from the database clock at run time -- a day of margin, rounded up to a round millisecond, plus the scenario's own `999`, `000` or `001` suffix -- so the millisecond coverage is unchanged and the date is gone. Fixed anchors are kept only for the far-future magnitude cases, and the 2^41 anchors move to 2^42. The second change turns a comment into an assertion. `Ledger::append` stamps `accepted_at` with `GREATEST(ledger_clock_ms, floor(now))` and requires `job_issued_at_ms <= accepted_at_ms`, so pinning the ledger clock lets the public API alone place a real share exactly on an anchor and another a millisecond past it. Every anchor in the barrier scenarios now does that, and asserts the first is credited and the second is not. Doing so surfaced why those two changes belong together. Past roughly 2^43 epoch-milliseconds a `double precision` no longer resolves a microsecond, so the read's `to_timestamp(double precision/1000)` barrier lands up to a microsecond either side of the true millisecond. A fixture storing the exact millisecond and a share written through the writer's own conversion then fall on opposite sides, and no single barrier makes both agree. The exact-time oracle therefore runs only where the conversion is exact -- the derived anchors and 2^42 -- with a runtime guard that refuses an anchor which is not, so the fixture cannot silently pick one. Behaviour past 2^43 gets its own test, in the integer milliseconds `AcceptedShare` carries and with no oracle and no direct inserts: three shares per anchor, all through `Ledger::append`. Membership follows the contract at every anchor. The read-back does not: `share_from_row` floors, so a millisecond whose conversion lands low reads back one millisecond early -- 480000 of every 2000001 milliseconds measured near 2^43, none near 2^42 or at any realistic date. Rather than skip that check, the test pins it from both sides: a millisecond converting at or above itself must round-trip unchanged, one converting below must be early by exactly one millisecond and no more. Nothing known-wrong is asserted as right, and a regression in either direction fails. Co-Authored-By: Claude Opus 5 * Bound the far-future read-back drift instead of pinning it (#269) `writer_rows_follow_integer_milliseconds_past_2_43` required the read-back drift to be exactly one millisecond wherever the stored timestamp converts below its own millisecond. That asserted a known defect as expected behaviour: the day `share_from_row` rounds instead of flooring, the drift becomes zero and the test would have failed the fix. The branch now bounds the drift rather than requiring it -- at most one millisecond early, never late -- and pushes to the printed `read_back_early` list only when a millisecond actually drifted. The defect stays visible in the per-scenario output, a silent change of behaviour is still obvious, and a fix passes instead of failing. The other side of the classification is unchanged: a millisecond the conversion represents at or above itself must still round-trip unchanged. Membership is still asserted exactly at every anchor, and both timestamps must still drift alike. Co-Authored-By: Claude Opus 5 * Tolerate the read-back convention for off-grid fixture rows and tidy the oracle (#269) The oracle held the read's millisecond fields to its own `floor(extract(epoch …)*1000)`. Every timestamp a writer can produce lands on a whole millisecond, so that was exact for real rows -- but scenarios c and e deliberately insert rows 400 microseconds either side of the anchor, to prove the barrier compares instants rather than milliseconds, and `AcceptedShare` cannot say which side of a millisecond such a row fell on. Flooring one is no more correct than rounding it, yet the oracle demanded the floor, so the separately-reported `share_from_row` rounding fix would have failed `c:anchor-barrier` and every `e:` sub-case on `accepted_at_ms` and `job_issued_at_ms`. The oracle now reports, per share and per timestamp, whether the stored value lies on a whole millisecond. On the grid the read-back must still be exact, which covers every row the writer can write. Off the grid the read may return the floor or the floor plus one and nothing else, so a read that lost a whole millisecond or gained two still fails. Each timestamp is judged on its own flag, so a read that treated the two fields differently is still caught, and membership, order and every other field stay exact. Every test now drops its throwaway schema on the failing path too, through a helper that runs the body and then always closes the ledger and drops the schema. The body's error is what a failing test reports; a cleanup failure is attached to it as context rather than replacing it, so a leaked schema can never hide the assertion that actually failed. That covers the per-sub-case schemas in `check_anchor_boundary` and in the 2^43 loop, which previously leaked one schema per sub-case whenever a run went red. Three comments were also wrong. The module header claimed the oracle shares no SQL text with `ledger.rs`, but `oracle_cutoff` repeats the read's cutoff query and `compare_conversion` embeds the read's own `to_timestamp`; both are deliberate and the header now says which, and why each is acceptable. The guard said the repository sets both "required" signals, when it sets only `PRISM_TEST_DATABASE_URL` -- GitHub sets `GITHUB_JOB`, and nothing sets `PRISM_TEST_REQUIRE_INTEGRATION`, the opt-in switch proposed by #286. And the 2^43 measurement spans 2000001 milliseconds, about 33 minutes, not two seconds. Co-Authored-By: Claude Opus 5 --------- Co-authored-by: Claude Opus 5 * Retry CI apt installs through Ubuntu mirror blips (#311) (#312) Three CI jobs install Ubuntu packages with a bare `apt-get update` and `apt-get install`. On 2026-09-11 the Ubuntu archive refused connections and served hash-sum mismatches to the runners. Each of those jobs failed with exit 100 before any test ran. Add `.github/scripts/apt-install.sh` and use it in all three jobs. apt retries each download (Acquire::Retries=5, with 30 s timeouts). If the update-and-install still fails, the helper clears the package lists, waits 30 s, then 60 s, then 90 s, and tries again, for up to four attempts. Every attempt but the last treats a partially failed `apt-get update` as a failure, instead of installing from stale indexes. The last keeps apt's default leniency, so the step is never stricter than before. Each failed attempt is annotated as a warning. Co-authored-by: Claude Opus 5 (1M context) * Record the approved D2 payout decision in the migration guide (#310) Decision D2 in #260 was approved on 2026-09-11: keep the 3.x.x behaviour for D2a (bootstrap pooling), D2b (below-target block credit) and D2c (prior balances during bootstrap). Mark D2c as approved, and point the configuration note at the operator release notes. Co-authored-by: Claude Opus 5 (1M context) * 3.x.x: native metrics registry with the cutover-minimum series (#308) * feat(metrics): add native cutover registry and producer hooks * fix(metrics): enforce registry ownership and live observation contracts * docs(metrics): align operator guidance and collector validation * test(metrics): pass registry to merged admission fixture * test(metrics): reset whole socket before failed-write assertions * fix(metrics): validate listeners before coordinator startup * fix(metrics): bound progress guard to health publication * fix(metrics): render registry before first publication * Survive Ubuntu mirror outages in CI package installs (#311) (#314) #312 made the three CI apt steps retry, which rides out short blips. An outage like the one on 2026-09-11, about 35 minutes of refused connections and hash-sum mismatches from Canonical's archive, still fails them. - Pin fallback mirrors in .github/apt/ubuntu-mirrors.txt: Azure's cloud mirror, kernel.org and Princeton, with Canonical's archive last. After the first failed attempt, apt-install.sh replaces the runner's Ubuntu sources with that list. Priority tags make apt try the mirrors in order, and retries skip apt's growing delay so a refused mirror doesn't hold up every download. Healthy runs keep the runner's own sources. - Cache the downloaded packages. The new composite action .github/actions/apt-install restores and saves them with actions/cache, keyed on the package set and the month. The helper downloads into its own directory, keeps exactly the packages a successful install used, seeds the next install from them, and if every online attempt fails, installs them directly with dpkg. It never reinstalls or downgrades anything the runner already has. - Stop retrying after a deadline (480 s by default), so the cache still gets its turn well inside the jobs' timeouts. The three CI apt steps now use the composite action. Co-authored-by: Claude Opus 5 (1M context) * 3.x.x: ratchet the payout window against the PostgreSQL JSONB ceiling (#298) * 3.x.x: ratchet the payout window against the PostgreSQL JSONB ceiling (#264) At the production window the native server stores the whole share window inside single JSONB documents. PostgreSQL refuses any container whose elements exceed 268,435,455 bytes. CI never saw this, because the largest native fixture had 800 shares. Add a deterministic production-shaped window fixture, an in-process fake node, and a gate that drives refresh, enqueue, claim, landing and import against PostgreSQL 16. After each phase the gate measures every JSONB column, and it fits bytes per share at two reduced sizes to project to 400,000 shares. Four known violations are checked in as a ratchet. The gate fails on a new crossing and on a listed write that stops crossing, and it never skips when CI is set without a database. It runs in the prism-native-postgres job; the database-free rust-tests job skips it by name. The full-size run and the baseline sweep are #[ignore]d and documented. test/prism-native-tests.sh gains a cargo-args mode, and the measurement and capacity docs record the baselines. Part of #261. Co-Authored-By: Claude Opus 5 (1M context) * 3.x.x: attribute refused JSONB writes and check the sweep's fit stability (#264) Two accepted review findings on the JSONB ceiling gate, both in test code and the measurement document. A refused write was attributed to a hard-coded column. PostgreSQL's ceiling error names neither the table nor the column, and the server adds no context naming the write, so each of the four `ceiling_rejection` sites was recording an assumption: a phase that ever wrote a second oversized JSONB value would have had the refusal blamed on the expected column, and the new offender would never have been seen. Every refusal now goes through `attribute_refusal`, which takes the refused phase, the refused share count and what that phase's accepted measurements project for each of its columns there. Exactly one column must project past `ATTRIBUTION_FLOOR`, 90% of the hard limit - seven times the 1.4% the measured fits disagree by - and it must be the write the call site expects; no candidate, two candidates, or the wrong one fails the gate with both names. The full-size run and the baseline sweep now run the reduced pair first for those projections; the reduced run attributes a refusal at `n2` from the `n1` measurement scaled to it, and a refusal at `n1` fails outright, since nothing smaller was accepted to attribute it from. The ratchet row says how it was attributed and still never shows a byte count. The baseline sweep never checked fit stability. Its comment claimed the CI-pair projection had to agree with the large-pair one, but it only computed the large pair, and with the default sizes refresh is refused there and has no projection at all. The sweep now also runs the reduced pair for the CI projection, skipping a size its list already has, and compares the two per write: the large pair is the two largest sweep sizes at which that write was accepted, since the writes are refused at different sizes and no single pair fits them all. A disagreement over `FIT_STABILITY_TOLERANCE` fails the sweep, naming the write, both projections and both pairs; a write accepted at fewer than two sweep sizes fails rather than going unchecked; a CI projection below 1 MiB is printed as not compared, because relative noise on a write that small says nothing about the window's growth. The table has the columns of the document's own. Unit tests cover attribution (unique, ambiguous, unexplained, another phase's column, and a candidate that is not the expected key) and fit stability (agreement, a 10% divergence, pair selection skipping refused sizes, fewer than two accepted sizes, and the 1 MiB floor), all without a database. Verified on PostgreSQL 16: the reduced gate ends with the ratchet holding at exactly the four known violations, and the sweep at 10000,20000,40000 prints the comparison table with every compared write inside 0.22%. Co-Authored-By: Claude Opus 5 (1M context) * 3.x.x: keep the n1-scaled projection for a write refused at n2 (#264) A write PostgreSQL refuses at n2 gets a Rejected verdict in the reduced fit, so projections_at gave it no projection. With PRISM_JSONB_GATE_N2 equal to the target, the full-size run then met the same refusal with nothing to attribute it from, and failed as unexplained although the write is a known violation. The sweep had the same gap at larger sizes. reduced_projections keeps the projection that attributed the n2 refusal, the n1 measurement scaled to the size in question, for every write the reduced pair refused. The full-size run and the sweep now use it. A unit test covers the refused write, the fitted write it leaves alone, and the attribution at the target. Part of #261. Co-Authored-By: Claude Opus 5 (1M context) * 3.x.x: attribute refusals before any table names their column (#264) The full-size run, the reduced run's n2 half and the baseline sweep printed their measurement tables before attribute_refusals ran, and write_table_lines named each refusal's table and column as the call site recorded it. A moved or new oversized write would have been shown under the old column before the attribution check disputed it. Every path now attributes first and still prints its table whatever the outcome. write_table_lines shows an unattributed refusal by phase only, as (refused) (not attributed). attribute_refusals no longer empties the map before every refusal has attributed, so a failure keeps the refusals in the report. The sweep attributes through a new attribute_sweep before printing any size. A unit test covers both rows. Part of #261. Co-Authored-By: Claude Opus 5 (1M context) * 3.x.x: fail on an unreadable database URL instead of skipping the gate (#264) database_url read PRISM_TEST_DATABASE_URL and CI with .ok(), which turns VarError::NotUnicode into "unset". Outside CI, a deliberately configured but malformed URL therefore skipped the ratchet silently, and a non-Unicode CI value counted as unset too. Both reads now go through database_url_from, where an unreadable variable is an error, as every other gate setting already is. The existing rule is unchanged: with CI set and no URL, the gate fails. A unit test covers a set URL, unset and blank values, CI without a URL, and non-UTF-8 values of both variables on Unix. Part of #261. Co-Authored-By: Claude Opus 5 (1M context) --------- Co-authored-by: Claude Opus 5 (1M context) * 3.x.x: Set one 3.0.0 release identity across VERSION and every crate (#315) * 3.x.x: set one 3.0.0 release identity across VERSION and every crate After #283 the tree carried three version strings on the 2.0.x line: VERSION and the two library crates read 2.0.2 by inheritance from the 2.x.x merge, and the rewritten server still read 2.0.0 from #244. None of them named a 3.x.x release. Declare the version once, as [workspace.package].version = "3.0.0" in the root Cargo.toml, and have every crate inherit it with version.workspace = true. VERSION reads 3.0.0 to match. Cargo.lock only moves the three workspace-member entries; no third-party dependency changes, and `cargo metadata --locked` and `cargo check --workspace --locked` pass. The toolchain pin and builder-image digest stay with #286; the 3.0.0 release notes stay with #291. Part of #284. Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_01YRVX7BZXPMWRNnbY9QLm1H * 3.x.x: gate the release identity with a version-skew check scripts/check_version_skew.py asserts that VERSION equals the version Cargo resolves for every workspace crate. It reads crate versions from `cargo metadata --no-deps` rather than the manifests, so a crate that later drops version.workspace = true for its own literal is still caught. It names each mismatching crate with both versions and exits non-zero on any mismatch, on a missing, empty, or malformed VERSION, and when cargo itself cannot describe the workspace. CI runs it as the first step of the rust-tests job, which already has a Rust toolchain and is in the required checks aggregator's needs list, so skew fails the merge gate without a new job. `make check-version-skew` runs the same command locally. tests/test_check_version_skew.py drives the script against temporary workspaces: matching versions pass; VERSION behind the workspace names every crate; one crate with a literal version is named alone; empty, missing, and malformed VERSION files fail; an unusable cargo is reported rather than passing. It also asserts the checked-in tree passes, that the CI step lives in a job the gate needs, and that the make target exists. The 3.x.x development-line note records the single source and points the real release notes at #291. Part of #284. Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_01YRVX7BZXPMWRNnbY9QLm1H --------- Co-authored-by: Claude Fable 5.1 * 3.x.x: assert the server-side D2 payout rules (#316) * Cover the D2b below-target block credit against PostgreSQL Decision D2 on #260 keeps 3.x.x's payout behaviour, and entry d2b-below-target-credit of the migration guide is the one that changes a miner's money: a proof that meets the network target but misses its assigned share target is credited at network difficulty, and only once the block it solved is observed on the active chain. 2.x.x credited the assigned share difficulty the moment the node accepted the block. Until now the only in-tree coverage of that shape ran against an in-memory stratum backend, which can see the frame being forwarded but neither the credited amount nor when it lands. Add coordinator::d2_below_target_tests, which drives the real Coordinator against a throwaway PostgreSQL schema and a scripted qbit JSON-RPC node, one test per case of the frozen below_target_credit vectors. Each test seeds the vector's ledger window, builds the found-block work, mines a proof of the shape the case names, and starts a real submission, then asserts: * the credit is published only inside the confirmation transaction. While the candidate is pending there is no credit row, exactly one deferred share, and no acknowledgement to the miner; * the credited difficulty is the proven network work and not the 2.x.x assigned difficulty, read from both sides of the vector rather than written here; * a lost submitblock reply ends in exactly one credit, published by reconciliation, and repeating that observation does not credit twice; * a reorg after publication leaves the credit intact while the block goes inactive; * a block the node refuses is abandoned, credits nothing, and fails the submission with reason ledger-confirmation-failed, with the internal reason the miner never sees captured from the log; * a share-passing proof is still credited its assigned difficulty at once, with nothing deferred; * the block after the solved one pays exactly what the vector projects, compared leaf by leaf, which is where a wrong credit would become a wrong payout. The assigned share target is reached through the stratum difficulty floor, because reassign clamps a target easier than the network target back to the network target and the floor is the only way to get a share target harder than the block target. The two difficulty units differ, so the floor is converted from the vector's scaled unit and stepped up until it is at least as hard as the vector's target; no f64 divides to that exact 256-bit value. The tests carry the same integration guard as the window read oracle: they run when PRISM_TEST_DATABASE_URL is set, fail by name when PRISM_TEST_REQUIRE_INTEGRATION=1 or GITHUB_JOB=prism-native-postgres demand the suite, and otherwise print a skip line, so a database outage in that job cannot pass silently. Each test takes its own schema, serialises against the cluster-wide advisory locks the other coordinator tests share, bounds every wait, and drops its schema and aborts its tasks on the error paths too. Also point the stratum protocol test at this module, so the boundary between the two is written down where a reader meets it. Co-Authored-By: Claude Opus 5 * Pin the D2a bootstrap-pooling selection with coordinator tests Decision D2 on #260 keeps 3.x.x's payout behaviour during pool bootstrap, and nothing yet fails if that behaviour drifts back towards 2.x.x. This adds an in-crate test module that drives Coordinator::refresh_once against a fake qbit JSON-RPC node and asserts on the work it actually publishes. What is asserted, and why it is asserted this way: * D2a, bootstrap pooling. With two accepted shares from two miners -- below 2.x.x's readiness gate of three, above 3.x.x's rule of "any share at all" -- refresh_once must select the ledger window, and the payout must be the proportional split the bootstrap_transition vectors record for 3.x.x, and must not be the 2.x.x whole-coinbase-to-the-finder outcome. Restoring any readiness gate of three or more fails this test. * The one-share boundary the vectors do not cover: a single share from a miner who is not the solver takes the whole coinbase. A restored gate of two or more fails this test. * An empty ledger and no balances still pays the solver, through the one synthetic bootstrap-share the bootstrap path builds. Every expectation is read out of the vector file by case name, checked against the D2 anchor the case carries, so a renamed, re-anchored or weakened case fails here rather than passing quietly. The assertions read the Prepared work refresh_once published and the bundle build_job hands a solver; build_bundle is never called directly, because that would compute the answer the selection under test was supposed to make. The harness template's network difficulty is 1_000_000 rather than the vectors' 100, so payouts that turn on the synthetic share are compared in units of each side's own network difficulty, with exact divisibility asserted rather than rounded away. Seeded-share payouts are compared exactly: the 8x window counts every seeded share at either scale. The integration guard is the one from the payout-window oracle: it runs when PRISM_TEST_DATABASE_URL is set, fails and names the variable under PRISM_TEST_REQUIRE_INTEGRATION=1 or GITHUB_JOB=prism-native-postgres, and otherwise prints a skip line. Keying on CI would be wrong, because the no-database job sets it too. The D2c carry-only case is not covered yet: an empty payout window alongside a non-zero carry-forward balance is not reachable from 3.x.x's own write paths, and how to reach it is still being decided. Co-Authored-By: Claude Opus 5 * Pin D2c: a bootstrap block still pays carried-forward balances Decision D2 on #260 keeps carried-forward balances in the payout of a bootstrap block, where 2.x.x dropped them from its collection bundle and left the account unpaid. Nothing failed if that regressed. This adds the carry-only case from the bootstrap_transition vectors. With an empty share window and one carry-only account whose balance is above the day-one floor, refresh_once must publish no window bundle, the bundle build_job hands the solver must carry the ledger's prior balances, and its payout policy manifest must match the recorded 3.x.x outcome exactly -- the solver paid the coinbase less the carried account's settlement, and the carried account paid out of this block. It must also differ from the 2.x.x outcome, which pays the solver everything. Passing an empty prior-balance list on the bootstrap path fails this test and no other. Reaching that state takes an explanation, because 3.x.x cannot produce it. An empty window beside a non-zero carry is the migrated, or archived, state of a pool that inherited balances without the share history behind them, and every route to it from a running 3.x.x is closed: a block that accrues must carry real ledger shares, because the landing path re-reads the bundle's share range out of the ledger and requires a match; a bootstrap-shaped block cannot accrue at all, because its single entitlement always takes the whole miner reward and so always clears the floor that was just checked against it; and shares can never be removed afterwards, because the share ledger refuses every update, delete and truncate at the database level, and one accepted row is enough to keep the window non-empty forever. So the test records the carrying parent block's rows directly, and does it without inventing a single figure. The parent's payout is built by the engine and verified through the public audit verifier, the test asserts the engine accrued exactly the balance the vector expects before anything is written, and then only that manifest's own accounts are recorded -- through the same statements, the same account-type filter and the same order the landing path uses, the block row prepared first and flipped to confirmed separately so the summary triggers count its carry once. Nothing writes a summary table by hand. A schema change that moves the real landing path leaves this copy failing loudly rather than quietly seeding some other state. The fake node's tip at that height is that parent block, so reconciliation sees it on the active chain, and the block under test is its child. Co-Authored-By: Claude Opus 5 * Share the D2 test harness and apply the review findings Both decision-D2 coordinator test modules carried their own copy of the same harness. Move it into coordinator::d2_test_support: the integration guard, a throwaway schema that is removed on success, on failure and when a fixture fails to open, the coordinator configuration both run, the payout projection with its leaf-level diff, and the shared constants. Also apply what review of the two modules found: * the bootstrap tests' fake node served a fixed template time, so every one of them would have started failing as stale in January 2027; it now serves the wall clock, as the below-target node already did; * each test now reports its own failure ahead of any cleanup failure, instead of letting a cleanup error replace it; * the below-target fixture now drops its schema when setup fails; * assertions that could no longer fail once the exact 3.x.x match had passed now check that the vector itself still records a difference from 2.x.x, which is what they were meant to guard; * the carry-seeding comment now says exactly which landing statements it repeats verbatim and which parts of landing it leaves out. No production code changes. Co-Authored-By: Claude Opus 5 (1M context) --------- Co-authored-by: Claude Opus 5 * 3.x.x: Make gated tests fail loudly, prove CI execution, add Rust hygiene jobs (#322) * Fix every clippy warning without changing behaviour `cargo clippy --locked --workspace --all-targets -- -D warnings` now passes on the 1.98.1 toolchain. Each change keeps the code's behaviour: - `(n + 1) / 2` becomes `n.div_ceil(2)` in both Merkle root loops. - The two `Default` impls that picked one variant become `#[derive(Default)]` with `#[default]` on that variant. - The eligible-share sort keys on `Reverse(share_seq)` instead of a comparator that spelled the same order. - `&[x.clone()]` becomes `slice::from_ref(&x)` in the CTV manifest test. - `seq % 4 == 0` becomes `seq.is_multiple_of(4)` in the window fixture. - `chunks_exact(_mut)` with a constant size becomes `as_chunks(_mut)`, which ignores the same remainder. - The adversarial settlement case that looped over four blocks and broke after the first now states that it checks block 0 only, which is what it always did. - `tools.rs` keeps its `live_instance_tests` module after the last item. - The two owning audit-bundle entry points allow `too_many_arguments` with the reason that they mirror the borrowed forms they forward to. `cargo fmt --all --check` already passed on the base commit, so there is no formatting-only change to land. Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_01DzhiDBXX9hveszStKkyFt1 * Route every environment-gated test through one shared gate Every test whose execution depends on PRISM_TEST_DATABASE_URL, PRISM_TEST_PG_BIN_DIR or QBITD_BIN used to open with its own copy of a skip: read the variable, print a line libtest hides, return Ok. The new qbit-prism-test-gate crate holds the one decision table that window_read_oracle.rs already followed, generalised per input: present, run; missing while PRISM_TEST_REQUIRE_INTEGRATION=1 or GITHUB_JOB=prism-native-postgres, fail naming the variable and the test; missing otherwise, print one line with a fixed prefix past libtest's capture and return. A variable that is not valid Unicode is an error, never absent, and a switch value other than 1 or 0 is an error too. The table is a pure function over injected values with a unit test per row, and a child-process test drives the runtime path. Every site is converted: the ledger, 2.x upgrade, rollups, API, readiness, authorization, migration-rollback, failover, replica, live regtest, candidate-lease, public-service and live-instance tests, and the window oracle, which drops its private guard. The two explicitly selected #[ignore] tests use the gate's required entry point, which never skips. The JSONB ceiling gate moves off its CI keying (GitHub sets CI=true in rust-tests, which has no database), so rust-tests no longer skips it by name; its unreadable-variable test now exercises the shared table. The live fixture keeps the qbitd path it was opened with for node restarts. The crate is a dev-dependency of qbit-prism-server and qbit-prism, so a gated test under either crate's tests, or in a #[cfg(test)] module, uses the same gate. scripts/check_gate_env_reads.py, run in the lint job, fails when any Rust file outside the gate crate names one of the gate variables in a string literal, so no new test can read them directly. Measured gated tests: 74 in the tree, 72 of which the native job runs (the two JSONB #[ignore] measurement runs are gated but never selected in CI). Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_01DzhiDBXX9hveszStKkyFt1 * Prove in CI that every gated test executed With PRISM_TEST_GATE_MANIFEST set, the gate appends one line per decision, executed, skipped or failed, followed by ::::, in a single O_APPEND write so parallel test threads and separate test binaries share the file. test/prism-gated-tests.txt is the checked-in list of the 72 gated tests the native job must execute, including the two #[ignore] runs it selects explicitly; its length is the minimum. scripts/check_gate_manifest.py fails when any expected test did not execute, any skipped or failed line appears, fewer distinct tests executed than the list holds, a test executed that the list does not name, or the log shows the gate's skip prefix or the wording of the skip messages the gate replaced. It prints the manifest to the log and to the step summary. The prism-native-postgres job keeps its id (the gate keys on it), sets PRISM_TEST_REQUIRE_INTEGRATION=1 and the manifest path, runs the whole workspace with --all-targets and --nocapture through tee, then the two explicit --ignored runs, then the proof, and uploads the manifest as an artifact. A gated test added under crates/qbit-prism is therefore covered by a required job. test/prism-native-tests.sh mirrors the job: its default mode runs the workspace, both explicit runs and the proof, and exports the switch once the PostgreSQL server tools and qbitd are present; the live, replica and cargo-args modes always export it, since they refuse to start without their inputs or name their own tests. Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_01DzhiDBXX9hveszStKkyFt1 * Add fmt, clippy, cargo-deny and hadolint as required checks Four new jobs join the checks aggregator: cargo fmt --all --check, cargo clippy --locked --workspace --all-targets -- -D warnings, cargo deny --locked check, and hadolint over every Dockerfile. deny.toml checks advisories (nothing ignored), licenses (an allow-list of what the tree uses, plus CDLA-Permissive-2.0 for the CA bundle that webpki-roots ships), bans (duplicate versions warn; wildcard requirements are refused except the workspace's own path dependencies, which needs qbit-prism and qbit-prism-server to be publish = false, as they are in practice) and sources (crates.io only). cargo-deny 0.20.2 and hadolint 2.15.1 are installed by .github/scripts/install-pinned-tool.sh from their GitHub releases after a sha256 check. hadolint reads .hadolint.yaml, which ignores nothing globally: lab/prism/Dockerfile pins its two apt packages, and the other images, whose behaviour is not this workstream's to change, carry inline ignores with reasons (unpinned apt versions, the named qbit-src build context that DL3022 mistakes for a stage, the deliberate separate cmake layers, and the checked pipes in the Bitcoin Core download). .github/actionlint.yaml names the Blacksmith runner label so actionlint runs clean locally. Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_01DzhiDBXX9hveszStKkyFt1 * Pin the Rust toolchain to 1.98.1 in one place rust-toolchain.toml pins the exact stable version with rustfmt and clippy under the minimal profile. [workspace.package].rust-version carries the same version and every crate inherits it. The lab/prism/Dockerfile builder is rust:1.98.1-bookworm at the multi-arch index digest Docker Hub serves for that tag (resolved through the registry API; both the 1.98.1-bookworm and 1.98-bookworm tags point at it today). scripts/check_toolchain_pin.py fails when the three disagree, when a crate stops inheriting rust-version, when the builder line lacks a digest, or, with --installed, when the active rustc or cargo is not the pin. The lint job runs the static check; the new .github/actions/rust-toolchain composite action replaces the floating dtolnay/rust-toolchain@stable in every Rust-using job, installing the toolchain rustup reads from the file and asserting the installed version. docker-builds builds the builder stage on its own first and asserts that its cargo and rustc report the pin. VERSION and scripts/check_version_skew.py from #315 are unchanged. Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_01DzhiDBXX9hveszStKkyFt1 * Keep one make target per native test suite test-prism-postgres-ledger, test-prism-postgres-scale, test-prism-postgres-native-ledger and test-prism-postgres-seed-guard all ran test/prism-native-tests.sh; they become test-prism-postgres. test-prism-stratum-regtest-live, test-prism-stratum-postgres-regtest-live and test-prism-combined-regtest all ran the live mode that test-prism-regtest already names; they are removed. test-prism-public-read-replica keeps its name and calls the script's replica mode directly. The eight wrapper scripts under test/ that only exec'd prism-native-tests.sh are deleted; test-prism-postgres-throughput and its script are untouched (#271 owns the throughput floor). README, the coordinator-refactor validation notes and the overload alert spec now name the surviving targets; the historical release notes under doc/ keep the names they shipped with. Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_01DzhiDBXX9hveszStKkyFt1 * Document the integration test gate and the deleted-test map docs/prism-integration-test-gate.md is the gate contract: the inputs, the decision table, the switch, the skip prefix, test identity, the manifest, the expected list and the proof, local runs, and how to add a gated test. docs/prism-deleted-test-map.md rebuilds the readiness review's map from git: every test file under tests/ and test/ on origin/2.x.x (504846c) absent on 3.x.x, 142 files grouped by area, each with its native replacement, or the open issue that closes the gap, or a needs-triage note where neither exists. Both are linked from docs/README.md, and the wrapper descriptions in PRISM.md, the server README and the ledger operations guide now describe required mode and the manifest. Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_01DzhiDBXX9hveszStKkyFt1 * Route the D2 payout tests through the shared gate #316 landed nine PostgreSQL-gated coordinator tests with a hand-written copy of the old guard in d2_test_support.rs, which read PRISM_TEST_DATABASE_URL, PRISM_TEST_REQUIRE_INTEGRATION and GITHUB_JOB directly and so failed scripts/check_gate_env_reads.py. The guard now takes the URL through qbit_prism_test_gate like every other #[cfg(test)] site, the nine callers drop the test name the gate derives itself, and the tests join test/prism-gated-tests.txt, which the native job's manifest is checked against: 81 expected gated tests, 83 in the tree. Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_01DzhiDBXX9hveszStKkyFt1 * Require every explicit #[ignore] run and write manifest lines in one write The two JSONB measurement runs, jsonb_ceiling_ratchet_at_full_size and jsonb_ceiling_baseline_sweep, are #[ignore] tests someone asks for by name, yet they reached the gate through the non-required entry point, so a manual --ignored run with no database printed a skip line and reported ok. They now use the gate's required entry point, as the observability_database and stratum_admission_postgres runs already did, and the contract says that every explicit #[ignore] run is required. They stay out of test/prism-gated-tests.txt because CI never selects them. The gate's record_to used write_all, which can split a line across two writes after a short write and let another thread's line land between them. It now hands the whole line to one write and treats a short write as an error, so each manifest line stays a single O_APPEND write; the skip line on stderr goes out the same way. Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_01DzhiDBXX9hveszStKkyFt1 * Leave the runtime image's apt packages unpinned The PRISM runtime stage floats on debian:bookworm-slim, and Debian drops superseded package versions from its archive, so the exact ca-certificates and curl pins would have broken the build at the next point or security release, with no Docker here to build-test a bump. The stage carries the same inline DL3008 ignore and reason as the other images; pinning its base and packages belongs to #288. The builder stage keeps its digest pin. Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_01DzhiDBXX9hveszStKkyFt1 * Re-check the deleted-test map against the D2 payout tests The map is now built against 8dccc74, the merge of #316, and pins both ends of its file list (504846c and 8dccc74), since origin/2.x.x has moved on and this branch removes wrapper scripts of its own. The nine D2 tests were read against every row they could touch: ten partial rows now cite them (below-target credit, lost-reply reconciliation, reorg survival, bootstrap selection, the retired readiness gate), and the settlement balance comparison row states why D2c does not cover it. No status changes; the counts stay 5 full, 67 partial, 8 open gaps, 7 needs triage, 55 retired. Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_01DzhiDBXX9hveszStKkyFt1 --------- Co-authored-by: Claude Fable 5.1 * 3.x.x: Assert stale-grace anchoring, retained-job and hot-path submit semantics (#313) * fix: preserve PRISM issued-work and stale-grace semantics Separate observed and published tip authority, retain original issued worker context through replacement and reconnects, and preserve bounded delivery-based grace without renewing expiry after awaited work. Exercise real Coordinator decisions and socket interleavings, including constant node-RPC load across 1, 32 and 128 clients. Refs #280 * docs: reference PRISM miner parity settings Co-Authored-By: OpenAI Codex * fix(prism): bind share admission to one published payout view * fix(prism): verify changed block notifications with fresh chain state * docs(prism): record B8 review regressions and rebased qualification * test(prism): adapt merged admission fixture to stale grace contract * fix(prism): classify retained capacity by current published tip * fix(prism): retain prepared dependencies for issued work * test(prism): verify issued job dependency persistence * fix(prism): keep sessions alive while retained work is usable * docs(prism): record timer coverage and final qualification procedure * docs: point D2 credit explanation to miner submit module * test: align candidate lease tip fixture with coordinator state * fix: integrate miner parity with native metrics registry * fix: integrate miner parity with metrics registry API * ci: run issued job dependency acceptance explicitly * test: keep miner parity fixture publication reachable * test: complete JSONB gate fixture config * test: restore D2 coordinator coverage * fix: bound configurable tip freshness durations * test: retain tokio time controls under pinned toolchain * chore: satisfy pinned toolchain checks * test: route issued job dependency through integration gate --------- Co-authored-by: OpenAI Codex * docs: design record for WindowRef and Ledger::read_window (#297) * docs: design record for WindowRef and Ledger::read_window (#264) Add docs/prism-coordinator-refactor/window-ref.md, the design record that #265 (migration 007) and #273 (migration 008) implement against, and link it from the directory README. The record specifies: - WindowRef { anchor_ms, prior_balances_digest, shares: Option } - the six window_* columns and their three-state CHECK - Ledger::read_window: one REPEATABLE READ READ ONLY transaction, mandatory ascending keyset paging, a streaming digest, typed errors and never a partial window - per-caller revision fences that keep today's observe_candidate rule for claims - the non-window bundle inputs a rebuild stores instead of reading config - the upgrade procedure, retention floor and expected cost at 400k shares - the borrowed qbit-prism parts API merged in #296 It went through two adversarial reviews. Part of #261. Co-Authored-By: Claude Opus 5 (1M context) * docs: hex-serialize WindowRef digests and clamp the resume timeout (#264) Two fixes to the WindowRef design record after review: - A derived Serialize writes a [u8; 32] as a JSON array of 32 integers, not the lowercase 64-hex string the payload and column contract requires. Both digest fields now carry #[serde(with = "hex32")], a small adapter in ledger/window.rs that writes lowercase hex and reads exactly 64 lowercase hex characters. The server's hex dependency has no serde feature. - The proposed resume timeout, initial_job_timeout_seconds minus 5 s, is zero or negative for any setting at or below 5 s, which validation allows. It is now outer - min(5 s, outer / 2), computed from the Duration the caller already builds: 25 s at the 30 s default, always positive, strictly shorter than the outer timeout, and free of Duration underflow. Part of #261. Co-Authored-By: Claude Opus 5 (1M context) * docs: bootstrap resumes use stored policy inputs (#264) An empty-window resume rebuilt each miner's bootstrap bundle through Coordinator::build_bundle. That function reads config.ctv_enabled, payout_policy, ctv_direct_floor and ctv_config, so after a config or fingerprint reset a resumed job could get a different signed bundle under the same job ID. That is the drift the stored-inputs invariant forbids. The record now has the per-miner bootstrap rebuild call the borrowing builders directly. It takes its own build_slots permit after the single-flight entry's permit is released. Its inputs are the synthetic share build_bundle fabricates today, the template, anchor, payout_policy and ctv stored in StoredPrepared, and the balances and revision read_window returned. #273's must-not row now forbids build_bundle for any resume rebuild. Part of #261. Co-Authored-By: Claude Opus 5 (1M context) * docs: readable imported audits and a race-free upgrade order (#264) Two fixes to the WindowRef design record after review. Imported legacy audits: #265 stores canonical_audit_bytes and no inline body. Only audit_canonical_bytes reads those bytes today. Ledger::audit_bundle returns just the JSON column, so backfill-ctv would stop on imported rows, and the bundle endpoint's fallback reads only body_uri. The record now requires the same PR to make both readers decode the stored canonical bytes first, digest-checked and under spawn_blocking. Leaving an imported audit readable only through body_uri is now forbidden. Upgrade order: a zero pending count taken while the pre-007 frontends still run proves nothing, because a block can be found before they stop. The count is now taken after every frontend is stopped. A non-zero count means restarting the pre-007 frontends to drain and repeating, and 007's refusal remains the backstop. Part of #261. Co-Authored-By: Claude Opus 5 (1M context) * docs: one owned window per Prepared and the established lock order (#264) Two fixes to the WindowRef design record after review. Window ownership: Prepared and JobContext would have held an owned AuditBundle beside the Snapshot. into_bundle takes the share Vec by value, and two Arcs don't share storage, so every resume would keep a second window: about 0.75 GB per 400k rebuild instead of 0.5 GB. Both structs now hold Arc, the only owned window, next to Arc. Canonical bytes and verification come from the parts functions, and an owned bundle is assembled only at claim. Lock order: the D6 prune now takes SETTLEMENT_LOCK and then ORDER_LOCK, the order every existing dual-lock site uses, so it can't form an advisory-lock cycle with a refresh snapshot or a candidate finalization. Part of #261. Co-Authored-By: Claude Opus 5 (1M context) * docs: version the audit builder behind durable references (#264) Stored inputs reproduce a bundle only under the builder that first built it. If a release changes builder output, a pending candidate that survived the upgrade would rebuild into a different bundle than its coinbase commits to, and fail on every retry or land changed audit data. qbit-prism gains AUDIT_BUILDER_VERSION, which both the 007 candidate and 008's StoredPrepared store. A rebuild checks it before read_window: a mismatched candidate is retried with an alert and never rebuilt with the current builder, and a mismatched job is a cache miss. A release that bumps the constant ships a migration step that refuses while pending outbox rows carry another version; they drain through the 007 procedure. Keeping frozen old builders is recorded as the rejected alternative. Part of #261. Co-Authored-By: Claude Opus 5 (1M context) * docs: pin the signing keys behind durable references (#264) The bundle embeds both public keys and their signatures, so a rebuild reproduces the original bytes only under the original keys. Rotating a key takes an operator resetting the cluster fingerprint, and a candidate still pending across that reset would be re-signed into bytes its coinbase does not commit to. Both documents now store signer_keys, the two public keys the building frontend signed with. A rebuild checks them next to the builder version, before read_window, and never re-signs a mismatch with the local keys. #265 extends configure to refuse a new fingerprint while a pending outbox row stores other keys, so the operator drains with the old keys first. Keeping retired seeds online to reproduce old signatures is recorded as the rejected alternative. Part of #261. Co-Authored-By: Claude Opus 5 (1M context) * docs: reuse the refresh-time WindowRef at submit (#264) Submit built the candidate's reference from the prepared Snapshot, which re-hashes and re-serializes the whole 233 to 260 MB window whenever a block is found. That puts 0.7 to 1.4 s of digest CPU on the latency-critical share path, on top of the once-per-refresh cost the cost table budgets. refresh_once already computes the identical reference, so Prepared now keeps it as window: WindowRef. Submit clones that small value into the candidate, a resume takes it from StoredPrepared, and re-digesting at submit is a must-not for #265. Part of #261. Co-Authored-By: Claude Opus 5 (1M context) * docs: sort rebuilt prior balances canonically (#264) prior_balances_digest sorts internally, but AuditBundle.prior_balances serializes in vector order. read_prior_balances has no outer ORDER BY, and SQL doesn't guarantee the function's internal order through a table expression. Balances returned in a different order at claim or resume would pass the digest check and still change the canonical bundle. read_prior_balances now sorts with the digest's comparator (order_key, recipient_id, p2mr_program_hex, byte order). Refresh and every rebuild share that function, so the builder gets the same vector. A SQL ORDER BY alone would follow the database collation, not the digest's byte order. A #265 unit test requires permutations to give identical bytes. Part of #261. Co-Authored-By: Claude Opus 5 (1M context) * docs: finish recovered claims from landed digest columns (#264) A claim that finds its block's audit already landed finished through materialize_audit_row. That function reads the whole range unpaged, serializes and clones it on the async runtime, and builds an owned AuditBundle: two share arrays at 400k on the submit loop, which the cooperative 60 s deadline cannot interrupt. The recovery branch now checks the landed row's columns alone, as today's landing does for an existing audit. share_snapshot_sha256 must equal window_snapshot_sha256, found_block_bits must match the header, and the claim then finishes. materialize_audit_row on the submit loop is a #265 must-not. If that path ever needs the body, #267's paged, parts-based reader becomes a prerequisite. Part of #261. Co-Authored-By: Claude Opus 5 (1M context) * docs: re-pin the WindowRef record to 3.x.x after #283 (#264) #283 landed through #300: the 2.x.x tip merge and the split of ledger.rs into candidates, jobs, window and connect submodules. The record's code citations were pinned to 1398bbc, and many pointed into the old ledger.rs. Every citation now resolves at 3.x.x 41a2afd: 76 of 252 changed, and all 247 cited line numbers were checked against that tree. "Where the code lives" now describes what #300 landed. ledger/window.rs exists under workstream B and already holds share_from_row and read_prior_balances, and SELECT_SHARE stays in the module root. The only question left for Anatolie is whether read_range moves out of A-owned ledger/audit.rs. Two review fixes in the landed-audit recovery path. A retried claim authenticates the landed audit against the block it found: the landed coinbase must match block_hex, and the root of the landed commitment leaves must equal the coinbase's witness reserved value. For an empty window, the snapshot comparison uses the digest of [bootstrap_share], because window_snapshot_sha256 is NULL there. Part of #261. Co-Authored-By: Claude Opus 5 (1M context) * docs: keep pool connections free during window rebuilds (#264) build_slots bounds concurrent rebuilds at build_workers, but that doesn't protect the database pool. PRISM_DATABASE_MAX_CONNECTIONS may be as low as 4, build_workers may exceed it, and share appends and candidate-lease heartbeats share the pool with a 15 s acquire timeout. Rebuilds of distinct jobs could hold every connection for their whole multi-page reads, and a heartbeat that fails to get a connection drops recoverable claim work. read_window now also takes a permit from a window_reads semaphore sized clamp(database_max_connections - 2, 1, build_workers). The caller takes it after build_slots and releases it before the rebuild, so two connections always stay free. Calling read_window without that permit is a must-not for #265 and #273. A dedicated pool was rejected because it would exceed the configured connection budget. Part of #261. Co-Authored-By: Claude Opus 5 (1M context) * docs: continue to submission after recognizing a landed audit (#264) The recovery path finished a claim as soon as it authenticated a landed audit. But an inactive block lands before the lease renewal and submitblock, so an earlier claim can land and lose its lease before it submits. Finishing then would abandon a valid found block that was never submitted. A recognized landed audit now means landing is done. The claim skips read_window, the rebuild and land_candidate, and continues through today's observe, renew and submitblock flow, finishing only where today's code does: an active block, or an inactive one whose revision or parent moved. Finishing a claim only because its audit landed is a #265 must-not. Part of #261. Co-Authored-By: Claude Opus 5 (1M context) * docs: reserve in-flight windows and scope the claim deadline (#264) Two review fixes. The retention floor covered written references only. Ledger::snapshot releases its locks when its first transaction commits, and the scan, the bundle and save_job follow with no row referencing the window yet. A prune in that gap, after a fall in difficulty, could delete rows of the window being built. D6 now adds a reservation. The snapshot's locked first transaction inserts it, the refresh fills in the window start after the scan, and save_job deletes it (publishing nothing if it expired or the rows are gone). The prune skips while any reservation is unresolved and otherwise keeps every reserved window. An age-based horizon was rejected because a window's reach in time depends on hashrate. The claim's 60 s timeout covered the whole call, including submitblock, whose RPC timeout may be up to 86,400 s. It now covers read_window plus the rebuild only, and the later steps keep their existing deadlines. Part of #261. Co-Authored-By: Claude Opus 5 (1M context) * docs: answer djh58's four B contract points (#264) djh58's review on #273 (comment 5624891010) raised four points against the resume contract. The record now: - lets B's published-identity and lease check decide resume eligibility instead of revision equality, and rebuilds eligible work with its as-issued balances from an immutable qbit_prism_balance_snapshots row keyed by prior_balances_digest; - keeps reconstruction failures (Incomplete, digest mismatch, decode, database, expiry) distinct from missing work, under one end-to-end deadline, the caller's existing initial_job_timeout_seconds, with no inner timeout; - names both share digests, the native snapshot_sha256 and #274's canonical_digest_hex, amends #274's reuse instruction, and adds a cost model for a sliding window; - moves the template out of the prepared payload into an immutable qbit_prism_templates row, keeping #273's 1 MB criterion whole-value, and moves #265's block_hex into a bytea column. It also records the permit-ownership and coalesced-waiter rules from the same review. Part of #261. Co-Authored-By: Claude Opus 5 (1M context) * docs: add the balance source to read_window and authenticate block bytes (#264) The resume contract needs as-issued balances, but read_window's declared API had no way to ask for them, and the read algorithm always used the current set. Implemented literally, it would have returned PriorBalancesChanged for the published work it was meant to rebuild. read_window now takes a BalanceSource. Current reads the current view as before. AsIssued reads the qbit_prism_balance_snapshots row keyed by the reference's digest, sorts it with the digest's comparator and verifies it. A missing row is the new BalanceSnapshotMissing, and a mismatching one is Decode. Claims pass Current, and resumes of published work pass AsIssued. Moving block_hex to a bytea column took it out of candidate_sha256. The candidate JSON now keeps block_sha256. Claim decode checks it and that the block header hashes to block_hash before any witness derivation, landing or submission. Part of #261. Co-Authored-By: Claude Opus 5 (1M context) * docs: prune externalized templates and balance snapshots with their jobs (#264) Moving each full template into an immutable qbit_prism_templates row left no cleanup for those rows. Templates change on every new transaction set and at every reanchor, so each is a new, possibly multi-megabyte row, and prune_expired_jobs deletes only job rows. The balance snapshots had the same gap. prune_expired_jobs becomes one transaction under SETTLEMENT_LOCK, the lock save_job holds while it inserts the template, the balance snapshot and the job row. It deletes the expired batch, then the template and balance rows that batch referenced and no remaining job row references, in two statements, because a data-modifying CTE's NOT EXISTS would still see the deleted jobs. The job row gains an indexed template_sha256. Balance snapshots follow job rows only, since claims read the current balances. Part of #261. Co-Authored-By: Claude Opus 5 (1M context) * docs: keep the issued revision on resume and ratchet D6's horizon (#264) Two review fixes. A resumed Snapshot now keeps StoredPrepared.payout_revision, the revision the job was issued with. read_window's current revision is only B's eligibility input and the claim's probe hint. The job wire and any candidate take their revision from the snapshot, so a block found on R0 work carries R0, and observe_candidate takes the superseded path instead of retrying PriorBalancesChanged forever. A D6 horizon computed from today's difficulty fails when difficulty rises after a prune: Ledger::snapshot stops when rows run out, so it would publish an underweight window. D6 now needs a ratcheted horizon kept at a durable maximum difficulty in qbit_prism_cluster. It must also record the share_seq it deleted below, so snapshot fails when its scan reaches that floor before the requested weight. Part of #261. Co-Authored-By: Claude Opus 5 (1M context) * docs: drop rebuilt windows from resumed jobs once the coinbase is built (#264) build_workers bounds how many rebuilds run at once, not how many rebuilt windows stay resident. Each resumed JobContext kept its Prepared, with Arc and Arc, and a session keeps up to 64 jobs per connection. Sequential resumes of different jobs could therefore pin about 0.5 GB each, and the stated k x 0.5 GB peak didn't hold. Once the rebuild has produced the job's coinbase, a resumed Prepared keeps only what submit reads: the WindowRef, the snapshot scalars, the template, the fee and the body's non-window fields. Submit reads nothing else, and the whole-bundle clone into the candidate goes with #265. Snapshot.shares and reward_manifest.shares are dropped before the job reaches the session, so windows exist only inside rebuilds. Part of #261. Co-Authored-By: Claude Opus 5 (1M context) * docs: stop old signers before a reset and fence writes on the fingerprint (#264) configure's one-time pending-row check could pass while an old-key frontend was still running. That frontend could then find a block and enqueue a candidate with the old signer_keys, which no new-key frontend can rebuild, and enqueue is not fenced by the fingerprint today. A rotation now follows the 007 order: drain, stop every old-key frontend, then check that nothing is pending, reset and start the new frontends. #265 and #273 also re-read config_fingerprint FOR SHARE inside every enqueue and save_job transaction and refuse writes under a foreign fingerprint. FOR SHARE conflicts with configure's FOR UPDATE, so a write and a reset cannot interleave, and a procedure violation fails loudly instead of leaving rows nobody can rebuild. Part of #261. Co-Authored-By: Claude Opus 5 (1M context) * docs: keep every stored input in slim resumed jobs (#264) The slim resumed Prepared kept the WindowRef, the snapshot scalars, the template, the fee and the body's non-window fields, but not the stored payout_policy and nested ctv. A candidate found on that job must persist those exact inputs, and neither the fee nor the coinbase manifest can recover them, so submit would have had to read local configuration. The slim form now keeps the decoded StoredPrepared itself: payout_policy, the whole ctv, audit_builder_version, signer_keys, coinbase_suffix, template_sha256, the WindowRef and the snapshot scalars, all O(1). For an empty window it also keeps the miner's own bootstrap_share. Only the window is dropped. Dropping a stored input before enqueue is a #273 must-not. Part of #261. Co-Authored-By: Claude Opus 5 (1M context) * docs: let the published-work lease reach submission (#264) Keeping R0 on a resumed snapshot doesn't help if its found block never reaches observe_candidate. Today submit rejects work whose frontend or job revision differs from the ledger's, apart from parent-stale work inside the stale grace. That grace work is credited as a share, but a candidate is enqueued only when the work isn't stale, so its block is dropped. B8's lease now decides those submit checks the way it decides resume eligibility. Work the lease covers is not stale because the revision changed, and a block found on it is enqueued as a candidate carrying R0, with its as-issued window and inputs. observe_candidate then decides at claim whether it lands. Work outside the lease keeps today's handling. Part of #261. Co-Authored-By: Claude Opus 5 (1M context) * docs: submit leased candidates first and clear terminal block bytes (#264) Two review fixes. A candidate enqueued from leased R0 work still met the unchanged claim fence, which would finish it as superseded before submitblock and discard the block. The candidate now records a leased flag. A leased candidate skips the pre-submit supersession checks and the pre-submit landing, renews its lease, calls submitblock, and only then meets observe_candidate. Whether an active leased block lands is B8's rule with #289. Moving block_hex into the block_bytes bytea column meant the terminal UPDATE no longer removed the block. It now NULLs block_bytes too, so the outbox doesn't keep every finished block forever. Part of #261. Co-Authored-By: Claude Opus 5 (1M context) * docs: correct claim-flow line citations to 3.x.x 41a2afd (#264) The landed-audit continuation and leased-candidate paragraphs cited coordinator.rs claim-flow lines read from a later 3.x.x, where #308 had shifted them by five. They now point at the record's pin, 41a2afd: the post-rebuild observe at :1039, the active branch at :1040-1048, the inactive supersession at :1050-1063, the landing at :1067-1069, the post-land check at :1070-1080, the lease renewal and submitblock at :1084-1092, and the post-submit observe at :1095. Part of #261. Co-Authored-By: Claude Opus 5 (1M context) * docs: submit leased candidates before any rebuild (#264) A leased candidate skipped the pre-submit fences, but the #265 claim rebuild still called read_window with BalanceSource::Current before them. For an R0 candidate that returns PriorBalancesChanged and retries forever, so the block never reached submitblock. A leased candidate now submits before any rebuild, because submitblock needs only the stored block. If the block is active, its audit rebuild uses BalanceSource::AsIssued. A non-terminal leased outbox row therefore keeps its balance snapshot. The prune checks for such rows and takes SETTLEMENT_LOCK then ORDER_LOCK, so it cannot interleave with a leased enqueue. Part of #261. Co-Authored-By: Claude Opus 5 (1M context) * docs: release rebuilt windows off the runtime; hold the lease to unchanged balances (#264) Two review fixes. Freeing a completed rebuild's last Arc and Arc drops about 0.5 GB on whichever Tokio worker releases it, which can stall lease heartbeats and share processing. The rebuild's spawn_blocking closure now builds the coinbase and the slim resumed form and drops the window there. Every other release of a rebuilt window goes through spawn_blocking(drop). An active leased block rebuilt with R0's balances could not land, because landing requires the current balances. Until B8 and #289 define how such a block lands, the lease covers only revision changes that leave prior_balances_digest unchanged, and a balance change ends it at once. An active leased block then lands through today's path, and the open question asks djh58 and #289 for the full rule. Part of #261. Co-Authored-By: Claude Opus 5 (1M context) * docs: land first-time claims from parts; sweep orphaned balance snapshots (#264) Two review fixes. A first-time landing assembled the rebuilt window into an AuditBundle, then cloned it, serialized it whole for the stored body and the canonical length, and re-read the range unpaged, all on a Tokio worker. At 400k shares that is several half-gigabyte representations on the runtime. CandidateClaim now holds the parts. Landing verifies, serializes and digests them in one spawn_blocking closure before its transaction opens. It keeps the range fence, but as a paged re-read. #267 is not a prerequisite. The balance-snapshot prune only considered digests from the expired batch, so a snapshot kept alive by a leased candidate could never be reclaimed once its job had gone. Snapshots now get their own statement over every row with no remaining reference. Part of #261. Co-Authored-By: Claude Opus 5 (1M context) * docs: keep the bootstrap share in local jobs; correct local window residency (#264) Three review fixes. An empty-window local job held its only full AcceptedShare in JobContext.bundle. Once JobContext holds a body, that share would be gone and submit could not fill Candidate.bootstrap_share. JobContext now keeps the synthetic share beside the body. The record claimed the local frontend keeps one full window. Retired local jobs keep their generation's Prepared for Stratum's retention period, as they do today. The record now says so. Bounding that is logged as a #273 acceptance note. The landing bind still copies the stored body into SQLx's argument buffer on the runtime. The record now names that copy. Measuring it is logged as a #265 acceptance note. Part of #261. Co-Authored-By: Claude Opus 5 (1M context) * docs: keep null-response leased candidates pending; carry empty-window balances (#264) Two review fixes. The record finished an inactive leased candidate as superseded after submission. Today's wrapper treats a null submitblock result on an inactive block as unresolved, because it can still be a known side-chain block, and retries with the candidate intact. Leased candidates now keep that outcome, so a block that becomes active later still has its audit inputs. For an empty window, resume waiters build their own bootstrap bundles from the as-issued balances. The single-flight entry now carries those balances until every waiter has built, instead of only the rebuilding caller holding them. Part of #261. Co-Authored-By: Claude Opus 5 (1M context) * docs: land a submitted leased candidate before any terminal outcome (#264) After a null submitblock result on an inactive leased block, the retry resubmitted before landing. A node can then answer "duplicate", which the string branch treats as terminal, clearing the unlanded audit inputs. Today's flow avoids that by landing an inactive block before its RPC, so a late acceptance stays reconcilable. A leased candidate still submits first, so delivery never waits on a rebuild. It then rebuilds with AsIssued and lands at the observed revision before any terminal outcome, whether or not the block is active, and only then takes today's three outcomes. Part of #261. Co-Authored-By: Claude Opus 5 (1M context) * docs: re-establish candidate references inside the enqueue transaction (#264) Taking locks in order stops a prune and an enqueue from interleaving, but it doesn't decide which runs first. A submission that passed its expiry check could lose to a prune that deleted the job row and its balance snapshot, and then enqueue a leased candidate pointing at missing data. Enqueue now re-inserts a leased candidate's balance snapshot from the as-issued balances the job carries (the slim resumed form keeps them), and probes the window's first share row, both inside its ORDER_LOCK transaction. Shares can't be written back, so an expired job row now holds the retention floor until the prune deletes it, and the prune waits a grace past expiry. A failed probe still publishes the candidate, with an alert. Part of #261. Co-Authored-By: Claude Opus 5 (1M context) --------- Co-authored-by: Claude Opus 5 (1M context) * 3.x.x: make Prism session sequence wrap-safe (#319) * fix(prism): reserve session extranonces safely across sequence wrap * fix(prism): fence stopped session owners by process incarnation * fix(prism): drain session guards before shutdown cleanup * fix(prism): scope migration readiness to mining startup * test(prism): build complete legacy schema for read-only connect * fix(tests): route integration databases through test gate * test(prism): register all gated session coverage * fix(tests): adapt fixtures to shared session backend * 3.x.x: Generate native metric inventory and migrate deployed alerts (#320) * docs(prism): generate metric inventory and migrate deployed alerts * fix(prism): preserve the network-wide connected client alert floor * fix(prism): evaluate standby lag from observed durable WAL progress * fix(prism): delete provisioned native rules when gates disable them * fix(prism): alert on invalid synchronous standby configuration * docs(prism): record local review evidence and prepare PR draft * fix(prism): reject stale alert and metric metadata * fix(prism): alert when public scrape fails * fix(prism): preserve native failure and paging alerts * fix(prism): preserve native alert tuning overrides * Revert "fix(prism): preserve native alert tuning overrides" This reverts commit 31b26812ef133a6cfb7e76deefb11e50a2bf306e. * fix(prism): preserve configurable alert tuning * fix(prism): render tuning dwells as JSON * fix(prism): validate rendered alert dwell overrides * 3.x.x: finish native health contract evidence (#329) * docs(prism): pin intentional legacy health differences * docs(prism): correct legacy max blocks semantics * style(prism): format health snapshot assertions * v3.0.0 feat: add prepared-window reference foundation (#327) * fix: preserve PRISM issued-work and stale-grace semantics Separate observed and published tip authority, retain original issued worker context through replacement and reconnects, and preserve bounded delivery-based grace without renewing expiry after awaited work. Exercise real Coordinator decisions and socket interleavings, including constant node-RPC load across 1, 32 and 128 clients. Refs #280 * docs: reference PRISM miner parity settings Co-Authored-By: OpenAI Codex * fix(prism): bind share admission to one published payout view * fix(prism): verify changed block notifications with fresh chain state * docs(prism): record B8 review regressions and rebased qualification * test(prism): adapt merged admission fixture to stale grace contract * fix(prism): classify retained capacity by current published tip * fix(prism): retain prepared dependencies for issued work * test(prism): verify issued job dependency persistence * fix(prism): keep sessions alive while retained work is usable * docs(prism): record timer coverage and final qualification procedure * docs: point D2 credit explanation to miner submit module * test: align candidate lease tip fixture with coordinator state * fix: integrate miner parity with native metrics registry * fix: integrate miner parity with metrics registry API * ci: run issued job dependency acceptance explicitly * test: keep miner parity fixture publication reachable * test: complete JSONB gate fixture config * docs: specify prepared window reference integration plan * test: restore D2 coordinator coverage * feat(prism): add window reference schema and authenticated reader Add the independently approved B273 schema and reader foundation on PR313, with typed reconstruction errors, immutable blob tables, per-version migrations, and PostgreSQL qualification. Runtime prepared/resume, candidate, atomic repair and GC integration remain gated on A265 and shared owner agreement. * feat(prism): add coherent payout state for lease eligibility * Keep cancelled window cleanup off runtime workers and qualify current base * Qualify landed 008 and 009 migration orders on current base * Preserve legacy balance order and reject fatal window reads * Prove legacy balance order preservation in PostgreSQL * Run all gated suites before manifest proof * Revert "Run all gated suites before manifest proof" This reverts commit b11cbbb183927db9b51daf6bf0176a09c4afc0b8. --------- Co-authored-by: OpenAI Codex * 3.x.x: store imported legacy audits as canonical bytes and decode them on read (#325) * 3.x.x: store imported legacy audits as canonical bytes and decode them on read The legacy-audit import wrote the parsed bundle back into the audit_bundle JSONB column. That value holds two copies of the payout window (shares and reward_manifest.shares), so at 400k shares it crosses PostgreSQL's JSONB container limit. The import now stores only canonical_audit_bytes and the non-share metadata. It keeps its idempotency and optimistic guards, verifies off the runtime, and drops the parsed window before the write transaction opens. Rows that already hold an inline body keep it unchanged, because the body-present CHECK needs one of audit_bundle and body_uri. found_block_bits stays as stored: no audit body carries the compact bits. A shared decode_canonical_audit_body helper checks the stored bytes against audit_bundle_sha256 and parses them with parse_audit_bundle_value. It returns the logical body the import used to store inline, and all of that work runs under spawn_blocking. Ledger::audit_bundle serves native snapshot rows first, then imported canonical bytes, then inline bodies, so backfill-ctv works on imported rows. The bundle endpoint serves imported bytes before any body_uri fallback, on the same public pool. A digest mismatch or undecodable bytes refuses the row, even when the legacy file still exists. The settlement-artifacts audit link now also accepts stored canonical bytes, so it does not decode a whole body just to show a link. The JSONB ceiling gate drops its import-phase known violation, and the measurement and capacity-readiness docs now count three window-carrying writes. Part of #265. Co-Authored-By: Claude Opus 5 (1M context) * 3.x.x: load only the served audit body and bound imported-audit decodes The bundle queries now project no inline audit_bundle for a row whose canonical bytes supersede it, the same rule Ledger::audit_bundle uses, and report whether canonical bytes exist so bundle() skips the bytes query for rows without them. The flag is removed before responding. Imported-audit decodes run after their read connection is released, so ApiState now carries a semaphore sized from the same read concurrency as the public read pool. bundle() takes an owned permit inside the request's existing deadline and moves it into the blocking decode, so a dropped request cannot free it before the decode ends. Ledger::audit_bundle takes no permit and releases its fetched row before decoding. The decode refuses body-ref and v2 envelope schemas, since canonical bytes are always a flat bundle. The storage-sizing doc describes the canonical-bytes import, and the measurement doc gains a post-#265 note. Tests pin the pre-change row shape (inline body, canonical bytes and body_uri, no snapshot), bound the decode across a dropped request, and refuse a digest-valid body-ref envelope. Part of #265. Co-Authored-By: Claude Opus 5 (1M context) --------- Co-authored-by: Claude Opus 5 (1M context) * 3.x.x: add deterministic HA readiness probe harness (#331) * test: add deterministic HA readiness probe harness * test: qualify readiness probe timing * test: preserve probe state in jsonl harness * test: harden readiness probe input boundaries * test: reject overlapping readiness probes * docs: use generic HA readiness contract wording * test: classify malformed probe durations as failures * 3.x.x: reconcile share-commit deadlines instead of dropping the append (#333) * ledger: let append callers veto COMMIT at the last moment append_at_revision_gated takes a non-optional pre-commit hook. It runs at most once, after every statement including persist_candidate has succeeded, while ORDER_LOCK is held and immediately before COMMIT. A false return rolls the transaction back explicitly and fails with the typed CommitGateClosed error, so COMMIT is never sent. append and append_at_revision keep their signatures and pass no hook. Co-Authored-By: Claude Opus 5 (1M context) * metrics: count late share confirmations and unknown outcomes A share whose commit is confirmed after the acknowledgement deadline is still accepted, and a commit whose outcome cannot be established is no longer reported as a clean rejection. Both need their own signal. Add the late_confirmed_shares_total family, recorded through record_late_confirmation and zero-registered like the other counters, and the ledger-outcome-unknown reject reason. Without the new reason, an unknown outcome would fold into internal-error. PrismShareAppendFailures paged on any ledger-confirmation-failed, so it now matches the unknown reason too and keeps its low-volume total-write- failure coverage. The deployment patch is regenerated from the reviewed snapshot, and a scenario covers the new reason alone. Document the narrowed meaning of ledger-confirmation-failed, the new reason, the statement-timeout guard, and the block-only acknowledgement bound. Co-Authored-By: Claude Opus 5 (1M context) * coordinator: reconcile share-commit deadlines instead of dropping the append (#324) submit wrapped the whole persistence future in a timeout and dropped it when the deadline fired, answering ledger-confirmation-failed. If COMMIT had already been flushed, sqlx drained the reply and reused the connection, so the share was committed and credited without an acknowledgement. A load harness reproduced that: 164 such rejections, two of them committed rows. The block-only path committed its candidate first, so its timeout always led to a later credit with no acknowledgement. Run the append in a task behind a one-shot commit gate. At the deadline the acknowledgement path either closes the gate, which proves COMMIT was never sent and aborts the doomed append, or finds the commit already in flight and awaits its reply for share_commit_grace. Only a server error of severity ERROR is a definite rejection; every other error, and a commit that outlasts the grace, is answered ledger-outcome-unknown and logged with its eventual outcome. A confirmation whose commit took at least the effective statement_timeout is also unknown, because a cancelled synchronous-replication wait commits only locally. An append carrying a found-block candidate is never refused, since that would discard the block; it waits for block_only_ack_timeout instead. Block-only proofs always finish their enqueue and then follow the candidate's disposition within the same bound, and poll errors after the enqueue never become a rejection. The new bounds are Config fields rather than environment variables, and statement_timeout is read from the session the ledger pool actually uses. Co-Authored-By: Claude Opus 5 (1M context) * coordinator: bound the block-only enqueue wait and time COMMIT in the task Review feedback on PR #333. enqueue_candidate_once was awaited directly, outside the bound that covers every other step of a block-only submission, so a degraded database could hold the acknowledgement open well past block_only_ack_timeout. Wait for it in a task until the bound instead. The enqueue still runs to completion, because cutting it off could strand a COMMIT that was already sent, but the miner is answered ledger-outcome-unknown on time and the enqueue's eventual outcome is logged. The synchronous-replication guard compared statement_timeout against a COMMIT duration measured when the coordinator classified the finished task. Scheduler delay after a successful COMMIT therefore counted as database time and could turn a durable accepted share into an unknown one. Measure the duration inside the append task, where COMMIT actually finishes. Co-Authored-By: Claude Opus 5 (1M context) * metrics: count a late confirmation from when the append finished Review feedback on PR #333. late_confirmed_shares_total was decided by reading the clock while the acknowledgement was being processed, which can be arbitrarily later than the append itself. A coordinator task scheduled late therefore counted an on-time confirmation as a late one. Decide it from the instant the append finished, which the task already reports, and record it where that instant is known. The help text also claimed every counted share was confirmed within the grace period. That was never true of a share-pass submission carrying a found-block candidate: those are deliberately never refused, so they can be confirmed later still, up to the block-only bound, and they are counted. Say what the counter measures instead, and spell the found-block case out in the metrics documentation. Co-Authored-By: Claude Opus 5 (1M context) --------- Co-authored-by: Claude Opus 5 (1M context) * perf(metrics): remove allocations from closed-label event recording (#344) * 3.x.x: add bounded PostgreSQL WAL qualification helper (#330) * test(prism): add bounded PostgreSQL WAL measurement helper * fix(prism): harden WAL qualification failure semantics * fix(prism): harden WAL helper validation and outcomes * fix(prism): sanitize WAL helper connection inputs * test(prism): cover WAL helper outcomes and scope * fix(prism): preserve WAL operation outcome uncertainty * fix(prism): bound WAL measurement subprocesses * fix(prism): complete WAL helper diagnostics and test coverage * test(prism): repair WAL helper test layout * test(prism): stabilize WAL diagnostic mocks * 3.x.x: Measure the share-append throughput floor against PostgreSQL (#323) * 3.x.x: measure the share-append throughput floor against PostgreSQL test/test-prism-postgres-throughput.sh measured `cargo run -- benchmark`, a CPU-only run of the audit builder with no database at all, and the 2.x.x report it replaced measured one bulk INSERT ... SELECT ... FROM generate_series. Neither says anything about the path a share actually takes. Every native share append is one transaction that takes the cluster-wide ORDER_LOCK advisory lock, checks writability, bumps the single-row ledger clock and writes two rows, so share appends are serialized cluster-wide, and nobody had measured that with more than one writer. tests/throughput_floor.rs seeds a payout window with the #264 fixture generator and then appends through Ledger::append at 1, 2 and 4 concurrent appenders, each appender holding its own Ledger with its own pool and its own instance_id, as separate production frontends do. A connection outside every appender pool samples pg_locks for backends queued behind ORDER_LOCK while a level runs, and the report says how many waiters it saw and for how long -- labelled throughout as sampled estimates, with the episode count a lower bound, because a wait shorter than the 10 ms interval can be missed entirely. A sampler that fails reports null and the error, never zero. The report is qbit.prism.postgres-throughput.v2, a deliberately new schema name: its numbers must never be read against v1's bulk-insert rate. It is written before the floor is asserted, so a run that fails the floor still leaves its evidence behind. Every QBIT_PRISM_* input is validated at entry, before a connection is opened: nan, inf, a negative or an empty floor are configuration errors rather than extreme floors, and a window size that does not divide the fixture's window weight would not produce the window it claims to. CI_MIN_SHARES_PER_SEC is provisional -- 25% of the slowest level rate measured in a debug build on the writer's host. Debug is the right base because CI runs this test in debug, and the wide margin is honest about the fact that nothing has calibrated it on the 2-vCPU runner yet. Part of #271 Co-Authored-By: Claude Opus 5 * 3.x.x: point the throughput script and target at the share-append floor test-prism-postgres-throughput ran `cargo run -- benchmark`, which measures the audit builder's CPU cost and never opens a database, so the target named after PostgreSQL throughput was not measuring PostgreSQL at all. It now runs the full-size share-append floor: the ignored throughput_floor test, in release, against PRISM_TEST_DATABASE_URL when one is set and otherwise through prism-native-tests.sh, which initdb's a disposable cluster on a random loopback port and removes it on exit. The builder benchmark has not gone anywhere and is still worth running; the script says so, and gives the exact command, because it answers a different question rather than a worse version of this one. `set -e` is suspended around the test so the report path is printed on a red run too -- the test writes the report before it asserts the floor, so a run that fails the floor is exactly the run whose report someone wants to read -- and the test's exit status is re-raised immediately afterwards. Every QBIT_PRISM_* variable reaches the test process unchanged; the script reads none of them except to say where the report landed. Part of #271 Co-Authored-By: Claude Opus 5 * 3.x.x: resolve pg_stat_statements by schema, not by search path The pg_stat_statements probe was `to_regclass('pg_stat_statements')`, and the read and the reset named the view unqualified. Every connection in this run carries `options=-csearch_path=`, which replaces the search path rather than prepending to it, so `public` is not on it. On a server with the extension installed in `public` the probe therefore returned NULL and the report said "unavailable" about a view that was sitting right there -- an EP-OBSERVABILITY failure of exactly the kind the rest of this file is careful about, because it reads as "this server cannot tell us" when the truth was "this query looked in the wrong place". The extension's schema is now resolved from pg_extension and the server's own quote_ident, and both the reset and the read qualify with it. Verified against PostgreSQL 16.15 with `shared_preload_libraries=pg_stat_statements` and the extension created in public: the block now reports 2000 calls per level, and its total_exec_time corroborates the sampler closely -- 4254 ms against a sampled 3.97-4.31 waiter-seconds at two appenders, and 17387 ms against a sampled 16.9-17.3 at four. Two independent methods agreeing to within a few percent is worth more than either alone, which is the whole reason the block is there. Part of #271 Co-Authored-By: Claude Opus 5 * 3.x.x: attribute ORDER_LOCK waits to the run that caused them The adversarial review found that the per-run schema isolates the tables but not the lock. ORDER_LOCK is a PostgreSQL advisory lock and an advisory lock is scoped to the database, so any other backend appending shares into the same database serialises against this run's appenders, and the sampler -- filtered only by database -- counted those foreign waits as this level's. Measured on a quiet server against the same server with one other appending process, the rates fell 40-57% and nothing in the report said why. A run like that either fails a calibrated floor for a reason that is not the append path, or, once the margin narrows, passes on a number describing two workloads. Every connection this run opens now carries a per-run application_name, `throughput-`, pushed through the URL -- which is the only thing Ledger::connect is handed, so the seeding ledger, every appender ledger and the sampler all announce themselves without any production code knowing this test exists. sqlx 0.8 parses the parameter out of the URL and sends it in the startup packet, but the run does not take that on trust: it reads current_setting('application_name') back on its first connection and refuses to measure anything if the server disagrees, because a silently ignored parameter would make every foreign backend look like ours and every contaminated run look clean. The sampler then reports this run's waiters as before, and foreign_waiters_max and foreign_backends_seen beside them. A foreign backend counts whether it was waiting on the lock or holding it: either way it is serialising against these appends. A backend whose pg_stat_activity row this role may not read counts as foreign, which is the safe direction. A level during which any foreign backend was seen is marked contaminated, in the level, at the top of the report and in the floor's failure message; it is still measured and still gated, because the rate is what the appends really achieved, but the report now says plainly that the number describes a shared database and must not be carried forward. contaminated is null, not false, when the sampler could not look. "Nobody checked" is not "nothing was there", and the same reasoning fixes the smaller version of that bug next door: a level that ended before the sampler completed a poll used to emit mean_waiters 0.0, max_waiters 0 and estimated_waiter_seconds 0.0, which reads as "nobody ever waited". Those are null now, samples stays 0, and the error says the level ended first. The sampler's shutdown and its interval are fixed together. It tested the stop flag only at the top of its loop and then blocked in sleep(interval), so every level burned up to a whole interval of dead time after its appends had finished -- 65 seconds of wall clock for 0.43 seconds of measurement at a 20-second interval. The flag is now paired with a Notify signalled alongside it, so the sleep is interruptible and a level ends when the appends end; notify_one stores the permit, so the signal cannot be lost in the window between a poll and a sleep. The interval itself is bounded to 1..=1000 ms at entry, before any connection, with a message saying that an interval coarser than a second cannot resolve a lock wait that lasts milliseconds. Two other gaps the review left open. A run that dies connecting, seeding or appending now writes a report anyway -- identity, configuration, the full error chain, passed_minimum false and an empty results array -- so the newest file on disk belongs to this run rather than to some earlier one; a failure to write it is attached to the original error as context and never replaces it. And the default report path is per test, target/prism-postgres-throughput-.json, with a "test" field in the JSON to match, because the two floor tests run in parallel threads under --include-ignored and the survivor of a shared path was whichever finished last, unlabelled. QBIT_PRISM_THROUGHPUT_REPORT is still the explicit override. Cancellation is documented rather than fixed: SIGINT kills the process before the schema drop runs, and handling it properly needs a signal handler, which means a dependency this test may not add. The module docs now say so and give the query that finds and drops the leftovers, next to the standing requirement that this run wants a database nothing else is appending to. Part of #271 Co-Authored-By: Claude Opus 5 * 3.x.x: make the throughput script point at a report that exists Two things the script got wrong, both of which send a reader looking for evidence that is not there. It printed the report path unconditionally, after the status had been captured. On the fallback path with PRISM_TEST_DATABASE_URL unset and no local server binaries, the run dies before the test starts and writes nothing, yet the script still announced a file. It now prints the path only when the file is actually on disk, and otherwise says that no report was written and where it would have been. The exit status is unchanged either way. It also pointed at the old shared default path. The test writes target/prism-postgres-throughput-.json now, one per test, so the script derives its path from the test it runs rather than from a constant that has moved. The header gains the two warnings the test itself carries. ORDER_LOCK is database-wide, not schema-wide, so this must run against a database nothing else is appending to; a run that shares one is marked contaminated and its number describes two workloads. And a cancelled run leaves its prism_throughput_ schema behind, so the header gives the query that finds the leftovers and the statement that drops them -- unnecessary on a disposable cluster, which is thrown away whole, and that is the reason to prefer one. Part of #271 Co-Authored-By: Claude Opus 5 * 3.x.x: print the floor's per-level rates where CI can see them The floor test prints its rates with println!, and libtest throws away the captured output of a passing test. CI keeps no report artifact either, so a green prism-native-postgres run recorded nothing about how far above the floor it was, and the provisional CI_MIN_SHARES_PER_SEC could not be calibrated from CI at all. After the report is written, the test now emits one summary line through std::io::stderr() directly, which the test harness does not capture: the test name, build profile, window, shares per level, the rate at each appender count, the floor, the verdict and the contamination flag. Every CI log then carries the margin the floor is set from. A failed write is ignored, because the report file already holds everything the line summarises. Part of #271 Co-Authored-By: Claude Opus 5 (1M context) * 3.x.x: move the throughput floor onto the shared integration gate #322 replaced every test's private copy of the database guard with one shared gate, qbit-prism-test-gate, and made the prism-native-postgres job prove that each gated test executed against test/prism-gated-tests.txt. The floor still carried its own copy of the old guard, which the new check_gate_env_reads lint rejects, and it was not on the list, so the proof step would have failed the moment the test ran. The floor now asks the gate for its database URL, so it follows the same decision table as every other gated test and records its decision in the execution manifest. share_append_throughput_floor joins the list; the full-size test is not selected in CI and stays off it. Part of #271 Co-Authored-By: Claude Opus 5 (1M context) * 3.x.x: set the CI throughput floor from the CI runner's own numbers CI_MIN_SHARES_PER_SEC was a provisional 71 shares/s, a quarter of what an 8-vCPU devbox managed, because nothing had measured the runner that enforces it. Four runs of prism-native-postgres on blacksmith-2vcpu-ubuntu-2404, read from the summary line the test now writes, put the slowest level of each run at 1,290, 847, 1,242 and 1,200 shares/s (debug build, the job's PostgreSQL 16 service, a 20k-share window and 2,000 appends per level). The floor is now 250, about 30% of the slowest level seen. Runs on that runner differ by about a third, so the margin keeps a noisy run green, while a 3-4x collapse of the append path still fails the build. The report's source label no longer calls the constant provisional. Part of #271 Co-Authored-By: Claude Opus 5 (1M context) * 3.x.x: run the floor the disposable-cluster path was skipping Four findings from the review of this PR, all in the paths that are hardest to exercise by hand. The wrapper script passed cargo_args beginning with 'test' to prism-native-tests.sh, which runs 'cargo test "$@"' itself, so the second word became a test-name filter. With --exact alongside it the disposable-cluster path selected nothing, printed nothing and exited 0: the script that exists to measure the floor measured nothing whenever PRISM_TEST_DATABASE_URL was unset. The array now carries only the arguments, and the direct call adds 'test' itself. The full-size test is selected by name, so a missing database must fail it rather than skip it; it now takes the gate's required entry point, and the CI-sized test keeps the skipping one. Resetting pg_stat_statements cleared the whole server's counters, including other databases; each level now reports the difference of two reads instead, and says so where the numbers are. The level clock started after the barrier opened, so an appender the runtime resumed first could work outside the measured window and inflate the rate; it starts before. Part of #271 * 3.x.x: scope the lock statistics to this database and stop announcing stale reports Two findings from review, both about a number that looks right and is not. `pg_stat_statements` is server-wide. The level's advisory-lock figures were the difference of two reads filtered only by query text, so an advisory lock taken in any other database on the same server landed in the level's count. A lock in another database cannot contend with this run -- an advisory lock is scoped to its database -- so the level was correctly marked uncontaminated while its lock figures described someone else's work. Filter on `dbid`, and say so in the report's caveat. Measured on a PostgreSQL 16 with `pg_stat_statements` loaded, over a level during which another database took 3,000 advisory locks: the old query attributed 3,000 calls to the level, the filtered one attributes 0. With the fix and a concurrent foreign load of about 24,000 locks, each level reports exactly 300 calls for 300 appended shares -- one lock per share, which is what the append path takes. The wrapper announced any file at the report path as this run's evidence, so a run that died during compilation or cluster setup pointed the reader at a previous run's report while returning a failure. The earlier file is now moved aside before the run, which makes the report's presence afterwards proof that this run wrote it, and it is put back if this run wrote nothing. Deleting it instead would have destroyed the previous run's data on a run that produced none of its own. Timestamps were tried first and rejected: a warm build and a small window can finish inside one filesystem timestamp tick, and a fresh report then compares as no newer than the moment the run started, which reports a good run as stale. Verified end to end through the wrapper against a real PostgreSQL 16, and for all three announcement cases: a failed run with an earlier report present, a successful run that writes one, and no report at all. Part of #271 --------- Co-authored-by: Claude Opus 5 * 3.x.x: list the new reject reason in the metric exposition fixture (#348) `3.x.x` has been red since #344 landed. Two pull requests collided semantically while each was green on its own: - #333 (e13051cf) added `ledger-outcome-unknown` to `RejectReason`, so `/metrics` now emits one more `qbit_prism_rejections_total` series; - #344 (037b04e4) added `event_exposition_matches_the_pre_allocation_change_byte_for_byte`, whose fixture was captured at 2914a629, before that reason existed. Neither branch could see the problem, because it only appears in the merged tree. The whole difference is one line, so the fixture gains it. The comment above the assertion is refreshed at the same time: it claimed provenance the fixture no longer had, and it now says what the pin implies -- adding a value to a closed label set has to be recorded here. Verified on this branch: the four metrics_allocation tests pass, and the failure reproduces on a6fe3fd9 without this change. Part of #338 * v3.0.0 fix: record cancelled collector pool acquisitions (#345) * fix: record cancelled collector pool acquisitions * docs: align collector cancellation timing contract * docs: qualify cancelled collector waits in alert contracts * test(metrics): include approved unknown ledger outcome in fixture PR #333 intentionally added the ledger-outcome-unknown rejection label and a producer contract test. Add exactly that sample to the older exposition fixture from #344, retaining the byte-for-byte equality assertion and leaving runtime behavior unchanged. (cherry picked from commit 4e5b4c41e980a3ebd956f39d5ed4d4676083d04e) * build: update rustls to 0.23.45 for RUSTSEC-2026-0285 (#354) rustls 0.23.44 accepted TLS 1.3 handshake messages sent at the wrong encryption level when they followed a key-changing message in the same record, instead of terminating the connection as RFC 8446 section 5.1 requires. The transcript stays authenticated, so a network-position attacker cannot alter or complete a handshake; the effect is that a peer can send handshake messages in plaintext that should have been encrypted without rustls rejecting the connection. It is the same bug as Go's CVE-2025-61730. The advisory names 0.23.45 as the fix. This is a lockfile-only change: `cargo update -p rustls`, no manifest or source change. The advisory database picked this up today, so the dependency audit now fails on 3.x.x and on every open pull request. * 3.x.x: Normalize stored audit bodies so they stop growing with the window (#353) * 3.x.x: add PrismRewardManifestHeader and a checked reward-manifest rebuild (#267) A native audit row is about to stop storing `reward_manifest.shares`. The counted window is a pure fold over `(shares, found_block)`, so the row only needs the manifest header; `restore_reward_manifest` reruns the fold and returns its window only after proving the rebuilt header is identical to the stored one, failing as the existing `reward_manifest` audit mismatch otherwise. There is no unchecked variant. `PrismRewardManifestHeader` denies unknown fields, so a legacy manifest that still carries `shares` can never be read as a header; and the full manifest still requires `shares`, so an older reader meeting a normalized row fails explicitly instead of serving an empty window. Tests prove the rebuilt bundle is byte-identical to the original (both schemas, credit_policy included), that one changed field in a rebuilt share moves the canonical sha256, that a tampered header is refused by name, and that neither shape can be mistaken for the other. Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_01LXvnfUC8CvPxskAXLgd1xx * 3.x.x: store native audit bodies without reward_manifest.shares and verify the range before the settlement lock (#267) The landed native audit body kept `reward_manifest.shares`, one record per counted share, so it grew with the payout window: about 235 MB in one JSONB value at the production window, 88% of PostgreSQL's container limit. The landing write now removes that copy alongside the top-level `shares`; the stored body is O(distinct miners and payout accounts). Nothing else is added or removed, `audit_body_byte_len` stays the full canonical artifact length, and the GIN-indexed leaf arrays stay in the body. Reads branch on the stored shape. A body whose `reward_manifest` has no `shares` key is decoded as `PrismRewardManifestHeader` and its window rebuilt through `restore_reward_manifest`, which refuses a header the fold does not reproduce; a body that still has the key (landed before this change) is served exactly as before. Both arms then run the unchanged canonical digest check, so a rebuilt body is proven byte-identical, not merely equivalent. Imported rows never enter this path. The snapshot digest, the rebuild and the canonical hash run on one blocking thread, and the public API hands that job the same permit imported decodes hold, so a dropped request cannot free the limit before the rebuild ends. The durable-range proof no longer runs inside the settlement transaction. `verify_durable_range` runs before the lock, page by page (4096 rows, the window reader's page), decoding and comparing each page on a blocking thread against the same positions of the candidate window: same predicate, same ordering, same full equality, same bootstrap exemption, same error text, and the first mismatched page fails the landing. An already-landed block skips it and takes the existing-row digest path as before. Under the lock the snapshot now only counts the same anchored predicate and fails the landing if the count differs from the window. The JSONB ceiling gate's landing entry is deleted: that write no longer crosses the threshold. Gated tests prove the stored shape and identical served bytes, that a tampered stored header is refused by name, that pre-change native rows still serve identical bytes, and that imported rows serve from their canonical bytes even with a poisoned inline body. An ignored harness measures the stored body, the settlement-lock hold and the landing peak RSS at a configurable window size. Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_01LXvnfUC8CvPxskAXLgd1xx * 3.x.x: document the measured native audit body before and after normalization (#267) State the measured stored bytes per block at 20,000 and 100,000 shares before and after `reward_manifest.shares` left the stored body, say that the residual is O(distinct miners and payout accounts) rather than a fixed bound, and separate `audit_body_byte_len` (the full canonical artifact) from `sum(pg_column_size(audit_bundle))` (the stored, compressed non-share body). Also record the measured cost of rebuilding a native audit on read. Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_01LXvnfUC8CvPxskAXLgd1xx * 3.x.x: correct three statements the audit-body normalization made untrue (#267) The JSONB ceiling gate's module doc and its landing phase note both still described the landing write as embedding the payout window, and the gate printed that note on every run. The payout-artifact measurement guide counted three native window-carrying writes. After normalization there are two, and landing is flat in the window. `d2_bootstrap_tests` cited `persist_audit_snapshot` re-reading the share range inside the transaction as one of the three closed exits for an empty payout window beside a non-zero carry. The exit is still closed, but by `verify_durable_range` page by page before the settlement lock plus the in-transaction count of the same anchored range, so the citation now names both. Co-Authored-By: Claude Opus 5 (1M context) * 3.x.x: give the paged durable-range proof negative multi-page coverage (#267) `verify_durable_range` had no negative coverage: its body could be replaced with `Ok(())`, together with the in-transaction count guard, and every reachable test still passed. The gated tests only ever drove it with a one-share window, so no test had executed a second page of the loop. A new gated test loads a 10,000-share window, three pages of the proof's 4096 rows, and lands four candidates that disagree with the ledger from the claim side, since the share ledger is immutable by trigger. Each window is built and signed as it is, so the audit signature check that precedes the proof accepts it and the proof is what refuses it, asserted by name: one `ntime` differs on the first share of the second page, one on the last share of the final partial page, one share is missing from the final page so the durable page outruns the window slice, and one share the ledger accepted after the anchor is claimed, so every page matches and the final count refuses it. A refused landing writes nothing. The unaltered window then lands and serves identical bytes, so the same fixture proves the loop accepts a correct multi-page window. The in-transaction count guard has no independent coverage. The anchored set is frozen once the anchor is issued and the ledger is append-only, so a window that passes the paged proof cannot count differently under the lock without a writer that defeats the immutability trigger between the two reads; it is defence in depth, and the test names the proof's message, not the guard's. `land_small_block` now appends a second share after landing, so present-day ledger state differs from the block's anchored snapshot in every test that uses it. A read that rebuilt the window from current ledger state instead of the snapshot range passed all of them before; it now fails the snapshot digest. Two comments are corrected without a behaviour change: the negative block in `reward_manifest_header.rs` now mutates the restored window it claimed to, and the permit comment in `read_models.rs` says the permit also spans the snapshot lookup and window read that precede the blocking job. Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_016P2UWk2rh2R1W8yqTdLZ4k --------- Co-authored-by: Claude Fable 5.1 * 3.x.x: Record advisory-lock and pool-acquire waits in the ledger (#334) * feat(prism): record ledger advisory-lock and pool-acquire waits The native registry declares two database wait families, but nothing in the ledger observed either one: advisory-lock waits had no observations at all, and pool acquisition counted only the metrics collector's own. ORDER_LOCK contention, which serializes every share acknowledgement, was therefore invisible in production and in qualification. Give Ledger an optional metrics handle, supplied by the new connect_with_metrics; connect keeps its signature and passes None, so the tools and tests that connect without a registry are unchanged and record nothing. The coordinator passes its own handle. Route every advisory lock through the existing lock() helper, now timing the pg_advisory_xact_lock statement alone, and every ledger transaction through a new begin() helper that times pool acquisition alone. begin() is Pool::begin split in two around the timer, down to its sqlx::Error return type, so both helpers are drop-in replacements and every call site's `?` converts exactly as before. Leaving the BEGIN round trip outside the timer keeps the pool series meaning what the collector already records. Errors, including a lock_timeout expiry, propagate unchanged and record as failure. A drop guard records a wait abandoned by cancellation as a failure with its elapsed time, so a cancelled wait stays distinct from one that never happened. Completing the wait takes the handle out of the guard, so a wait is recorded exactly once. The metrics mutex is taken only after the wait has ended, never across an await, so rendering cannot queue behind a contended lock; a guard held across the await would also make the future !Send and fail to compile in the spawned share path. The CPFP funding lock has no LockKind value and stays unobserved; an unmapped key trips a debug assertion rather than borrowing another lock's label. Refs #328. * test(prism): cover ledger wait metrics by lock and outcome Two gated PostgreSQL tests and one source invariant. The first attaches a registry and drives each outcome against a real database: the migration lock from schema creation, a share append taking ORDER_LOCK once, a second session holding ORDER_LOCK past a 200 ms lock_timeout so the append fails with SQLSTATE 55P03 and records one failure of at least that duration, an append cancelled mid-wait recording its partial wait and no more, the settlement lock through save_job, five transactions each observing one pool acquisition plus the collector's own unchanged, a pool acquisition cancelled while every connection is held, and a closed pool recording an acquisition failure. The second runs the same sequence on a ledger with no handle and asserts that results and error codes, 55P03 included, are what they are at the base. It deliberately makes no assertion about a registry: one no ledger can reach has nothing to record into and could not fail the test. The third walks the ledger sources recursively and fails if a transaction begins outside the timed helper or a second pg_advisory_xact_lock statement appears, so a new call site, including one in a nested module directory, cannot silently stop being observed. Refs #328. * docs(prism): describe the recorded database wait metrics The two database wait families now say what they measure and who records them: pool acquisition excludes transaction BEGIN and the queries that follow, and advisory-lock timing covers one pg_advisory_xact_lock round trip for the migration, order and settlement locks. Both record a wait abandoned by cancellation as a failure, except the collector's own acquisitions, which keep reporting cancellation through collector status rather than inventing a completed wait. The follow-up ownership and alert-migration notes no longer describe either family as unpopulated, and record that PrismDatabasePoolWaitHigh now evaluates ledger acquisitions while its description still predates this change. No alert rule or digest input changes. Refs #328. * docs(prism): record the pool alert's collector-availability gate PrismDatabasePoolWaitHigh is gated on collector_available for the database collector. That collector acquires from the same pool as the ledger, so sustained pool exhaustion times out its own acquisition, zeroes the gauge and suppresses the rule exactly while the ledger histogram is recording the waits that should fire it. The gate cost nothing before this branch, because the family carried no samples during such an outage. It does now. The rule itself cannot change here: its file feeds the alert specification digest embedded in the deployment patch, which can only be regenerated from the frozen template. #336 tracks that work; this records the interaction where operators read it. Refs #328. * v3.0.0 docs: correct pool-acquire alert measurement boundaries (#343) * docs: correct pool-acquire alert measurement boundaries Describe metrics collection and instrumented ledger transaction pool acquisition, retaining query/advisory-lock/read-gate exclusions and the five no-replacement dispositions. Regenerate migration tables and deployment patch from the verified frozen snapshot; all alert behavior is unchanged. * test(metrics): include approved unknown ledger outcome in fixture PR #333 intentionally added the ledger-outcome-unknown rejection label and a producer contract test. Add exactly that sample to the older exposition fixture from #344, retaining the byte-for-byte equality assertion and leaving runtime behavior unchanged. * feat(prism): record ledger advisory-lock and pool-acquire waits The native registry declares two database wait families, but nothing in the ledger observed either one: advisory-lock waits had no observations at all, and pool acquisition counted only the metrics collector's own. ORDER_LOCK contention, which serializes every share acknowledgement, was therefore invisible in production and in qualification. Give Ledger an optional metrics handle, supplied by the new connect_with_metrics; connect keeps its signature and passes None, so the tools and tests that connect without a registry are unchanged and record nothing. The coordinator passes its own handle. Route every advisory lock through the existing lock() helper, now timing the pg_advisory_xact_lock statement alone, and every ledger transaction through a new begin() helper that times pool acquisition alone. begin() is Pool::begin split in two around the timer, down to its sqlx::Error return type, so both helpers are drop-in replacements and every call site's `?` converts exactly as before. Leaving the BEGIN round trip outside the timer keeps the pool series meaning what the collector already records. Errors, including a lock_timeout expiry, propagate unchanged and record as failure. A drop guard records a wait abandoned by cancellation as a failure with its elapsed time, so a cancelled wait stays distinct from one that never happened. Completing the wait takes the handle out of the guard, so a wait is recorded exactly once. The metrics mutex is taken only after the wait has ended, never across an await, so rendering cannot queue behind a contended lock; a guard held across the await would also make the future !Send and fail to compile in the spawned share path. The CPFP funding lock has no LockKind value and stays unobserved; an unmapped key trips a debug assertion rather than borrowing another lock's label. Refs #328. * test(prism): cover ledger wait metrics by lock and outcome Two gated PostgreSQL tests and one source invariant. The first attaches a registry and drives each outcome against a real database: the migration lock from schema creation, a share append taking ORDER_LOCK once, a second session holding ORDER_LOCK past a 200 ms lock_timeout so the append fails with SQLSTATE 55P03 and records one failure of at least that duration, an append cancelled mid-wait recording its partial wait and no more, the settlement lock through save_job, five transactions each observing one pool acquisition plus the collector's own unchanged, a pool acquisition cancelled while every connection is held, and a closed pool recording an acquisition failure. The second runs the same sequence on a ledger with no handle and asserts that results and error codes, 55P03 included, are what they are at the base. It deliberately makes no assertion about a registry: one no ledger can reach has nothing to record into and could not fail the test. The third walks the ledger sources recursively and fails if a transaction begins outside the timed helper or a second pg_advisory_xact_lock statement appears, so a new call site, including one in a nested module directory, cannot silently stop being observed. Refs #328. * docs(prism): describe the recorded database wait metrics The two database wait families now say what they measure and who records them: pool acquisition excludes transaction BEGIN and the queries that follow, and advisory-lock timing covers one pg_advisory_xact_lock round trip for the migration, order and settlement locks. Both record a wait abandoned by cancellation as a failure, except the collector's own acquisitions, which keep reporting cancellation through collector status rather than inventing a completed wait. The follow-up ownership and alert-migration notes no longer describe either family as unpopulated, and record that PrismDatabasePoolWaitHigh now evaluates ledger acquisitions while its description still predates this change. No alert rule or digest input changes. Refs #328. * docs(prism): record the pool alert's collector-availability gate PrismDatabasePoolWaitHigh is gated on collector_available for the database collector. That collector acquires from the same pool as the ledger, so sustained pool exhaustion times out its own acquisition, zeroes the gauge and suppresses the rule exactly while the ledger histogram is recording the waits that should fire it. The gate cost nothing before this branch, because the family carried no samples during such an outage. It does now. The rule itself cannot change here: its file feeds the alert specification digest embedded in the deployment patch, which can only be regenerated from the frozen template. #336 tracks that work; this records the interaction where operators read it. Refs #328. * feat(prism): record ledger advisory-lock and pool-acquire waits The native registry declares two database wait families, but nothing in the ledger observed either one: advisory-lock waits had no observations at all, and pool acquisition counted only the metrics collector's own. ORDER_LOCK contention, which serializes every share acknowledgement, was therefore invisible in production and in qualification. Give Ledger an optional metrics handle, supplied by the new connect_with_metrics; connect keeps its signature and passes None, so the tools and tests that connect without a registry are unchanged and record nothing. The coordinator passes its own handle. Route every advisory lock through the existing lock() helper, now timing the pg_advisory_xact_lock statement alone, and every ledger transaction through a new begin() helper that times pool acquisition alone. begin() is Pool::begin split in two around the timer, down to its sqlx::Error return type, so both helpers are drop-in replacements and every call site's `?` converts exactly as before. Leaving the BEGIN round trip outside the timer keeps the pool series meaning what the collector already records. Errors, including a lock_timeout expiry, propagate unchanged and record as failure. A drop guard records a wait abandoned by cancellation as a failure with its elapsed time, so a cancelled wait stays distinct from one that never happened. Completing the wait takes the handle out of the guard, so a wait is recorded exactly once. The metrics mutex is taken only after the wait has ended, never across an await, so rendering cannot queue behind a contended lock; a guard held across the await would also make the future !Send and fail to compile in the spawned share path. The CPFP funding lock has no LockKind value and stays unobserved; an unmapped key trips a debug assertion rather than borrowing another lock's label. Refs #328. * test(prism): cover ledger wait metrics by lock and outcome Two gated PostgreSQL tests and one source invariant. The first attaches a registry and drives each outcome against a real database: the migration lock from schema creation, a share append taking ORDER_LOCK once, a second session holding ORDER_LOCK past a 200 ms lock_timeout so the append fails with SQLSTATE 55P03 and records one failure of at least that duration, an append cancelled mid-wait recording its partial wait and no more, the settlement lock through save_job, five transactions each observing one pool acquisition plus the collector's own unchanged, a pool acquisition cancelled while every connection is held, and a closed pool recording an acquisition failure. The second runs the same sequence on a ledger with no handle and asserts that results and error codes, 55P03 included, are what they are at the base. It deliberately makes no assertion about a registry: one no ledger can reach has nothing to record into and could not fail the test. The third walks the ledger sources recursively and fails if a transaction begins outside the timed helper or a second pg_advisory_xact_lock statement appears, so a new call site, including one in a nested module directory, cannot silently stop being observed. Refs #328. * docs(prism): describe the recorded database wait metrics The two database wait families now say what they measure and who records them: pool acquisition excludes transaction BEGIN and the queries that follow, and advisory-lock timing covers one pg_advisory_xact_lock round trip for the migration, order and settlement locks. Both record a wait abandoned by cancellation as a failure, except the collector's own acquisitions, which keep reporting cancellation through collector status rather than inventing a completed wait. The follow-up ownership and alert-migration notes no longer describe either family as unpopulated, and record that PrismDatabasePoolWaitHigh now evaluates ledger acquisitions while its description still predates this change. No alert rule or digest input changes. Refs #328. * docs(prism): record the pool alert's collector-availability gate PrismDatabasePoolWaitHigh is gated on collector_available for the database collector. That collector acquires from the same pool as the ledger, so sustained pool exhaustion times out its own acquisition, zeroes the gauge and suppresses the rule exactly while the ledger histogram is recording the waits that should fire it. The gate cost nothing before this branch, because the family carried no samples during such an outage. It does now. The rule itself cannot change here: its file feeds the alert specification digest embedded in the deployment patch, which can only be regenerated from the frozen template. #336 tracks that work; this records the interaction where operators read it. Refs #328. --------- Co-authored-by: Alex * v3.0.0 test: cover native ledger deadlines and execution safety (#355) * test: cover native ledger deadlines and execution safety Add 14 PostgreSQL regressions for late issued-job expiry, atomic publication, migration restart, incomplete snapshots, and claim-fenced execution. Use a test-only protocol observer to detect sensitive write replay after timeouts or lost acknowledgements, and document the native retry and snapshot contracts. * test: make candidate backoff eligibility deterministic Verify retry scheduling before controlling eligibility through fixture timestamps. Use future and past infinity to exercise claim fencing without racing the one-second retry deadline or waiting for it to elapse. Addresses the timing-race review comment on #355. * test: remove fanout backoff timing race and clarify locks Compare the fanout retry schedule with its durable write timestamp, then control claim eligibility through fixture timestamps. Document settlement and order locking and the scope of the sibling-row independence test. Addresses the two follow-up review comments on #355. * test: isolate ledger deadline and fanout eligibility checks Freeze all three completed fanouts before the global pre-deadline claim assertion. Use a dedicated fixture pool without competing server timeouts for the issued-job absolute-deadline regression, retaining bounded barriers and cleanup. Production behavior and other fixture defaults are unchanged. Addresses the two follow-up review comments on #355. * v3.0.0 fix: preserve pool-wait alert during collector failure (#351) * docs: correct pool-acquire alert measurement boundaries Describe metrics collection and instrumented ledger transaction pool acquisition, retaining query/advisory-lock/read-gate exclusions and the five no-replacement dispositions. Regenerate migration tables and deployment patch from the verified frozen snapshot; all alert behavior is unchanged. * test(metrics): include approved unknown ledger outcome in fixture PR #333 intentionally added the ledger-outcome-unknown rejection label and a producer contract test. Add exactly that sample to the older exposition fixture from #344, retaining the byte-for-byte equality assertion and leaving runtime behavior unchanged. * fix: keep pool-wait alert active when database collector fails Remove only the collector-availability gate; preserve snapshot freshness, sample sufficiency, thresholds, duration and labels. Add bounded PromQL scenarios with per-instance assertions, recovery and the old-expression negative control. Refs #336. Landing depends on #334 and reconciliation with #343 and #345. * feat(prism): record ledger advisory-lock and pool-acquire waits The native registry declares two database wait families, but nothing in the ledger observed either one: advisory-lock waits had no observations at all, and pool acquisition counted only the metrics collector's own. ORDER_LOCK contention, which serializes every share acknowledgement, was therefore invisible in production and in qualification. Give Ledger an optional metrics handle, supplied by the new connect_with_metrics; connect keeps its signature and passes None, so the tools and tests that connect without a registry are unchanged and record nothing. The coordinator passes its own handle. Route every advisory lock through the existing lock() helper, now timing the pg_advisory_xact_lock statement alone, and every ledger transaction through a new begin() helper that times pool acquisition alone. begin() is Pool::begin split in two around the timer, down to its sqlx::Error return type, so both helpers are drop-in replacements and every call site's `?` converts exactly as before. Leaving the BEGIN round trip outside the timer keeps the pool series meaning what the collector already records. Errors, including a lock_timeout expiry, propagate unchanged and record as failure. A drop guard records a wait abandoned by cancellation as a failure with its elapsed time, so a cancelled wait stays distinct from one that never happened. Completing the wait takes the handle out of the guard, so a wait is recorded exactly once. The metrics mutex is taken only after the wait has ended, never across an await, so rendering cannot queue behind a contended lock; a guard held across the await would also make the future !Send and fail to compile in the spawned share path. The CPFP funding lock has no LockKind value and stays unobserved; an unmapped key trips a debug assertion rather than borrowing another lock's label. Refs #328. * test(prism): cover ledger wait metrics by lock and outcome Two gated PostgreSQL tests and one source invariant. The first attaches a registry and drives each outcome against a real database: the migration lock from schema creation, a share append taking ORDER_LOCK once, a second session holding ORDER_LOCK past a 200 ms lock_timeout so the append fails with SQLSTATE 55P03 and records one failure of at least that duration, an append cancelled mid-wait recording its partial wait and no more, the settlement lock through save_job, five transactions each observing one pool acquisition plus the collector's own unchanged, a pool acquisition cancelled while every connection is held, and a closed pool recording an acquisition failure. The second runs the same sequence on a ledger with no handle and asserts that results and error codes, 55P03 included, are what they are at the base. It deliberately makes no assertion about a registry: one no ledger can reach has nothing to record into and could not fail the test. The third walks the ledger sources recursively and fails if a transaction begins outside the timed helper or a second pg_advisory_xact_lock statement appears, so a new call site, including one in a nested module directory, cannot silently stop being observed. Refs #328. * docs(prism): describe the recorded database wait metrics The two database wait families now say what they measure and who records them: pool acquisition excludes transaction BEGIN and the queries that follow, and advisory-lock timing covers one pg_advisory_xact_lock round trip for the migration, order and settlement locks. Both record a wait abandoned by cancellation as a failure, except the collector's own acquisitions, which keep reporting cancellation through collector status rather than inventing a completed wait. The follow-up ownership and alert-migration notes no longer describe either family as unpopulated, and record that PrismDatabasePoolWaitHigh now evaluates ledger acquisitions while its description still predates this change. No alert rule or digest input changes. Refs #328. * docs(prism): record the pool alert's collector-availability gate PrismDatabasePoolWaitHigh is gated on collector_available for the database collector. That collector acquires from the same pool as the ledger, so sustained pool exhaustion times out its own acquisition, zeroes the gauge and suppresses the rule exactly while the ledger histogram is recording the waits that should fire it. The gate cost nothing before this branch, because the family carried no samples during such an outage. It does now. The rule itself cannot change here: its file feeds the alert specification digest embedded in the deployment patch, which can only be regenerated from the frozen template. #336 tracks that work; this records the interaction where operators read it. Refs #328. * docs: distinguish pool-wait guard from database gauge guards * feat(prism): record ledger advisory-lock and pool-acquire waits The native registry declares two database wait families, but nothing in the ledger observed either one: advisory-lock waits had no observations at all, and pool acquisition counted only the metrics collector's own. ORDER_LOCK contention, which serializes every share acknowledgement, was therefore invisible in production and in qualification. Give Ledger an optional metrics handle, supplied by the new connect_with_metrics; connect keeps its signature and passes None, so the tools and tests that connect without a registry are unchanged and record nothing. The coordinator passes its own handle. Route every advisory lock through the existing lock() helper, now timing the pg_advisory_xact_lock statement alone, and every ledger transaction through a new begin() helper that times pool acquisition alone. begin() is Pool::begin split in two around the timer, down to its sqlx::Error return type, so both helpers are drop-in replacements and every call site's `?` converts exactly as before. Leaving the BEGIN round trip outside the timer keeps the pool series meaning what the collector already records. Errors, including a lock_timeout expiry, propagate unchanged and record as failure. A drop guard records a wait abandoned by cancellation as a failure with its elapsed time, so a cancelled wait stays distinct from one that never happened. Completing the wait takes the handle out of the guard, so a wait is recorded exactly once. The metrics mutex is taken only after the wait has ended, never across an await, so rendering cannot queue behind a contended lock; a guard held across the await would also make the future !Send and fail to compile in the spawned share path. The CPFP funding lock has no LockKind value and stays unobserved; an unmapped key trips a debug assertion rather than borrowing another lock's label. Refs #328. * test(prism): cover ledger wait metrics by lock and outcome Two gated PostgreSQL tests and one source invariant. The first attaches a registry and drives each outcome against a real database: the migration lock from schema creation, a share append taking ORDER_LOCK once, a second session holding ORDER_LOCK past a 200 ms lock_timeout so the append fails with SQLSTATE 55P03 and records one failure of at least that duration, an append cancelled mid-wait recording its partial wait and no more, the settlement lock through save_job, five transactions each observing one pool acquisition plus the collector's own unchanged, a pool acquisition cancelled while every connection is held, and a closed pool recording an acquisition failure. The second runs the same sequence on a ledger with no handle and asserts that results and error codes, 55P03 included, are what they are at the base. It deliberately makes no assertion about a registry: one no ledger can reach has nothing to record into and could not fail the test. The third walks the ledger sources recursively and fails if a transaction begins outside the timed helper or a second pg_advisory_xact_lock statement appears, so a new call site, including one in a nested module directory, cannot silently stop being observed. Refs #328. * docs(prism): describe the recorded database wait metrics The two database wait families now say what they measure and who records them: pool acquisition excludes transaction BEGIN and the queries that follow, and advisory-lock timing covers one pg_advisory_xact_lock round trip for the migration, order and settlement locks. Both record a wait abandoned by cancellation as a failure, except the collector's own acquisitions, which keep reporting cancellation through collector status rather than inventing a completed wait. The follow-up ownership and alert-migration notes no longer describe either family as unpopulated, and record that PrismDatabasePoolWaitHigh now evaluates ledger acquisitions while its description still predates this change. No alert rule or digest input changes. Refs #328. * docs(prism): record the pool alert's collector-availability gate PrismDatabasePoolWaitHigh is gated on collector_available for the database collector. That collector acquires from the same pool as the ledger, so sustained pool exhaustion times out its own acquisition, zeroes the gauge and suppresses the rule exactly while the ledger histogram is recording the waits that should fire it. The gate cost nothing before this branch, because the family carried no samples during such an outage. It does now. The rule itself cannot change here: its file feeds the alert specification digest embedded in the deployment patch, which can only be regenerated from the frozen template. #336 tracks that work; this records the interaction where operators read it. Refs #328. * Expose pool acquisition waits independently of health publication --------- Co-authored-by: Alex * v3.0.0 feat: add audited fatal-state recovery (#360) * docs: add PRISM fatal-state recovery runbook (#290) Document the operator procedure for a shared fatal state: evidence first, graceful stop of every frontend with stale-versus-stopped instance guidance, separately reviewed reconciliation before clearing, the audited fatal-state clear command, and restart verification. Includes read-only, bounded SELECT examples only. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_013RXEBLAaU5LEGU9Px59q19 * Add transactional PRISM fatal-state recovery and regression tests (#290) --------- Co-authored-by: Claude Opus 5 * Unify heartbeat status writers and self-check reader (#362) * v3.0.0 fix: harden native PRISM configuration and containers (#361) * v3.0.0 fix: harden native PRISM configuration and containers * fix: align heartbeat freshness with health cadence * fix: decouple job pruning from health publication cadence * 3.x.x: Pin the migration source schema, refuse unknown storage versions, gate startup (#321) * 3.x.x: pin the migration source schema and gate startup on it The migrator accepted any database that had a share ledger, tested the outbox with a JSONB predicate that a #258 v2 row slips past, never read qbit_prism_schema_capabilities, and skipped the schema version entirely when PRISM_POSTGRES_INIT_SCHEMA=0, the native default. Its "frozen 2.x.x schema" in the upgrade tests was include_str! of the live 001, which #244 already edits. Freeze the 2.x.x release SQL as byte-exact fixtures taken from the release commits (001 from v2.0.2, identical at v2.0.1 and v2.0.0; 002 from v2.0.2), pin their digests, pin the live 001's digest, and compare the live 001 to the release statement for statement, since #244 changed only its comments. Editing either live file fails the test with the instruction the editor needs. Migration 006 pins the accepted source states as data: fresh, pre-#258 (001 only), #258 applied (candidate_storage_version = 2 and every 002 object), partial 002 (refused, naming the missing object) and newer (refused before any DDL). The inventory comes from the catalog, so the drain predicate is built only from outbox columns that exist: the storage_version and body_id clauses on a #258 source, the v1 form on a pre-#258 one, where a fixed predicate errored instead of refusing. The check reads rows, never the capability row, which only proves 002 ran. The refusal names the blocking rows and the 2.x.x drain: the pinned coordinator's submitter, or lab.prism.recover_pending_blocks from the 2.x.x image. Every start now reads qbit_prism_schema_migrations and the capability rows, with or without initialize, and refuses a schema other than version 6 or a capability the binary does not understand, naming the required version. A successful migration records the accepted source state, its 2.x.x release and commit, the declared capability value and the prior schema version in qbit_prism_migration_source; migrate prints it, every start logs it, and a repeated migrate never rewrites it. 006 also declares candidate_storage_version = 1 on sources that had no capability row and adds the outbox storage_version column so one claim statement serves every source; the chunk tables are not carried. The API database test hand-applies migrations up to 003 and then connected without initialize, which the gate now refuses; it initializes. Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_01AauTYgQEqxzDE13xLjYTR9 * 3.x.x: park candidates the claim lane cannot decode The claim lane returned the outbox row's candidate column alone. A #258 v2 row returns NULL with the claim token set, no CandidateClaim can be built, the submit loop logs "candidate polling failed", and the row re-enters the due lane at every lease expiry, forever. Return storage_version and an Option candidate. A version 1 row with a body is decoded as before. Any other row, a v2 body this server does not import or a version it does not know, is parked inside the claim transaction: the claim is released, last_error says why, and next_attempt_at becomes infinity, so no lease expiry offers it again and ordinary v1 work keeps flowing. The row stays pending with its body untouched; resetting next_attempt_at re-offers it, which is how an operator hands it back after draining it with the 2.x.x image. The edit is deliberately small, since #265 reshapes candidate storage. Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_01AauTYgQEqxzDE13xLjYTR9 * Document the supported 2.x.x source schemas and the drain requirement The migration guide names the minimum supported 2.x.x release (v2.0.1, whose offline recovery command is part of the drain; v2.0.0 has the same schema), the newest (v2.0.2), #258's rollback floor, the source-state table the migrator pins, the drain requirement for v1 and v2 rows with the 2.x.x commands, what qbit_prism_migration_source records, the startup gate, and how a parked row is found and re-offered. The server README points at it. Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_01AauTYgQEqxzDE13xLjYTR9 * 3.x.x: accept a newer schema at start and let capabilities fence formats The startup gate refused any schema version other than the one this binary migrates to. That silently withdrew the rollout the guide promises, where compatible Rust releases drain and replace one frontend at a time, because a release that adds a migration would have kept every frontend on the previous release from starting once its migrate ran. Refuse only a schema below the required version, as the issue asks. A newer schema is accepted with a warning that names both versions: native migrations are additive, and a release whose format an older binary must not touch declares a capability, which require_known_capabilities still refuses exactly as before. The guide's rollout paragraph is restored with the order that makes it hold: the new release's migrate runs first, frontends on the previous release keep starting while its migrations are additive and it declares no capability they do not understand, and a release that declares a new capability needs every frontend stopped first. The startup-gate test now asserts that a start on the next schema version succeeds and rewrites nothing. Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_01AauTYgQEqxzDE13xLjYTR9 * Record the migrator and schema freeze in the 3.x.x development-line note The note listed how 002_candidate_bodies.sql is handled as frozen by a pending issue. It is now carried byte-identical to v2.0.2 and digest-pinned, never applied, and a #258 source is accepted after the drain check while a partial or newer source is refused before any DDL. The matching "not done yet" bullet is gone. Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_01AauTYgQEqxzDE13xLjYTR9 * 3.x.x: verify the frozen 001 schema after it has run, refusing drift transactionally A database with no 002 objects was accepted as the pre-#258 source on the strength of qbit_share_ledger existing. 001 is idempotent, but its IF NOT EXISTS leaves an existing table, column, index or named constraint as it is, so an older, partially applied or locally modified schema passed the pin, was recorded as v2.0.1, and its incompatible definitions survived a successful migration. The migrator now applies 001 to the source as before and then, before 002_multi_instance.sql or any later native DDL touches those tables, checks that what 001 left alone is the release definition. The expected definitions are not a stored fingerprint: under a savepoint in the migration transaction it creates a scratch schema, applies the same 001 there (plus 002_candidate_bodies.sql for a #258 source), reads every table, column, constraint, index, trigger and function that produced, and rolls the savepoint back, so the comparison is exact for the running PostgreSQL version and nothing from the scratch apply survives. Each object then needs an equivalent in the schema the migrator runs in: column type, NOT NULL, default, identity, generated status and collation; constraints per table by definition; index definitions and validity; trigger definitions and enabled state; function arguments, result, language, body, volatility and settings. Schema qualification, column order, comments, auto-generated constraint names and the NOT VALID mark 001 itself leaves on an upgraded table are ignored; extra objects are kept and logged; a missing or different one fails the migration, which rolls back whole, naming each object and what differs. A migrate role without CREATE on the database is refused with the grant it needs. The source-state table gains the "drifted 001" row in SOURCE_STATES, migration 006's comment and the migration guide; the migration record is unchanged. Tests: a pre-#258 source with a dropped foreign key, a dropped NOT NULL, a changed column type and a same-named index with a different definition is refused naming all four and left untouched; a #258 source with a replaced 002 function body and a disabled 002 trigger is refused; an extra table, column and index, a 001 column moved to a different physical position with its data, and a NOT VALID credit-policy CHECK still migrate; a role without CREATE on the database is told the grant. Replays of the real 0.1.2 -> 1.x -> 2.x history (every 001 revision in order, then the frozen v2.0.2 file, with and without 002) all migrate. Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_01EVxRdWsnzBhrsgiEX2kvKR * 3.x.x: register the source-schema gated tests in the gate manifest The branch was rebased onto #322, which fails the prism-native-postgres job for any gated test that executes without being listed in test/prism-gated-tests.txt. None of the ten Postgres tests this PR adds in tests/support/ledger_2x.rs were listed, so the job would have failed on its manifest check. List them, sorted, and update the count the gate guide quotes. The other #322 checks (formatting, clippy with -D warnings, the gate env-read lint, the toolchain pin, version skew) already pass on this branch and need no change. Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_019mWgeEnsyHLZSiGaAj8bZi * 3.x.x: compare release sequences in the source-schema check The post-001 fingerprint read tables, columns, constraints, indexes, triggers and functions, but never the sequences behind 001's bigserial columns or its explicit qbit_audit_publication_sequence_seq. CREATE TABLE IF NOT EXISTS leaves an existing sequence exactly as it is, so a lowered MAXVALUE or a changed INCREMENT on qbit_share_ledger_share_seq_seq survived the idempotent apply, compared equal to the frozen release, and migrated; the next share inserts would then fail or take unexpected identifiers. The fingerprint now records every sequence of the schema from pg_sequence: data type, start, increment, minimum, maximum, cache and cycle. The value a sequence has reached (last_value, is_called) is the source's data and is not read, so an advanced ledger still migrates and keeps its position. A release sequence that is missing or whose structure differs is drift, named in the refusal the way columns and functions are, for example sequence qbit_share_ledger_share_seq_seq differs: maximum expected 9223372036854775807, found 1000000 and an extra sequence is logged with the other extras. Tests: the pre-#258 drift test also lowers the share sequence's maximum and doubles the payout entry sequence's increment, and the refusal names both and rolls back; the tolerated-differences test advances the share sequence with setval and the first native share takes the next value; a unit test covers the comparison. The real 0.1.2 -> 1.x -> 2.x histories (every 001 revision in order, then the frozen v2.0.2 001, with and without 002) all still migrate, and each leaves the same sequence structure as a fresh apply. The migration guide and the 006 comment list what is compared. Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_019mWgeEnsyHLZSiGaAj8bZi * 3.x.x: treat a database as fresh only when no release object exists A database without qbit_share_ledger and without any 002 object was classified as fresh on that evidence alone, and the post-001 comparison was skipped for it. A selective restore or a hand-installed piece of the schema, qbit_pool_blocks on its own for instance, therefore passed: 001's CREATE ... IF NOT EXISTS kept whatever those objects held, the migration recorded success, and the accounting code ran over a schema nobody had checked. The release definitions are now taken once, before any DDL touches the source: the frozen release SQL (001, plus 002 for a #258 source) is applied to a scratch schema under a savepoint and rolled back, and that one fingerprint serves both checks. A database without qbit_share_ledger is fresh only if it has no table, sequence, index, trigger or function the release 001 creates. Otherwise it is the new source state "partial 001", refused before any DDL and naming the objects present, capped as the drift report is: refusing to migrate a partial 001 source before any DDL: the database has no qbit_share_ledger but holds 4 object(s) that the 2.x.x release's 001_share_ledger.sql creates (table qbit_pool_blocks; index ...), so it is neither an empty database nor a 2.x.x ledger, and 001's IF NOT EXISTS would keep those objects whatever they hold. Nothing was changed. Restore the full pre-migration backup, or migrate into an empty database Objects the release does not create, an operator's own table, do not disqualify a fresh database and are logged as extras. The post-001 comparison now runs for every source state, fresh included, where it passes trivially: 001 has just created everything. SOURCE_STATES gains the partial 001 row after fresh. The rows are cited by name through constants and looked up by name, never by position, and a unit test checks every cited name is exactly one row, so reordering the table cannot silently change a refusal again. The pinned-data test, the 006 comment, the guide's table and the development-line note carry the new row. Tests (Postgres): a leftover 001 table, sequence or function alone, each created from the frozen release definition, is refused as partial 001 naming it, with no native table, no scratch schema and the leftover untouched; an empty database with only an operator's table still migrates as fresh and keeps the table; a truly empty database migrates as fresh. The real 0.1.2 -> 1.x -> 2.x history replays, with and without 002, still migrate. Both new gated tests are in the manifest. Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_019mWgeEnsyHLZSiGaAj8bZi * 3.x.x: recheck the outbox before 006 on a native schema an earlier build migrated A database an earlier 3.x.x build brought to native schema 3, 4 or 5 went straight to 004, 005 and 006 without the drain check, and that build's v1-only predicate never counted a v2 row (`candidate ?& ...` is NULL for a NULL body). A pending v2 candidate could therefore be carried into version 6 and parked by the claim lane instead of refusing the cutover, leaving a possibly valid found block unsubmitted. `migrate_schema` now runs the column-aware inventory and `refuse_undrained_outbox` on a recorded version 3, 4 or 5 before 004, 005 and 006 run, so a refusal on that path is before any DDL as well. Native pending rows carry the native payout_revision, bundle and block_hash fields and are not counted; only a v2 body, a body_id, a storage_version other than 1, or a v1 body without those fields is refused. The refusal has its own wording: it names the native schema version, names the rows the same way as the 2.x.x refusal, and gives the remedy the guide backs, restoring the pre-migration 2.x.x backup and draining there with the pinned image. The 2.x.x release is not supported against a native schema (its revert script refuses one), so the message says not to point it at the database. Postgres tests build the earlier build's schema 5 faithfully: the frozen 001 (and 002), the current migration, then 006 undone, deleting the version 6 row and dropping qbit_prism_migration_source, and on a pre-#258 source also the storage_version column and the capability table 006 created. A pending v2 row on the #258 simulation is refused naming the row, with max(version) still 5 and no 006 object; a native pending candidate on either simulation migrates to 6 and is still claimable; an undrained 2.x.x v1 row on the pre-#258 simulation exercises the v1-only form of the predicate. The guide's drain text and the 006 comment describe the check; the gated-test manifest and its documented count are updated. Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_01QuEYH8xHQRo7C4SnexM3cP * 3.x.x: compare table and sequence persistence in the source-schema check The release-schema fingerprint recorded a table by name and compared its columns, constraints and indexes, so a release table an operator had changed to UNLOGGED compared equal to the logged table the scratch apply creates, and the migration accepted a ledger or outbox whose rows PostgreSQL truncates after a crash. The fingerprint now records pg_class.relpersistence for every table and every sequence (PostgreSQL 15 and later have unlogged sequences, and a sequence owned by an unlogged table is unlogged with it). A mismatch is drift, named the way the other differences are: `table qbit_share_ledger differs: expected logged, found UNLOGGED`, and for a sequence the persistence joins its other properties. Extra objects are unaffected. A Postgres test makes qbit_payout_carry_forward UNLOGGED on a pre-#258 source: it holds the carried balances and has no foreign key in either direction, so SET UNLOGGED is accepted as it is, and the sequence behind its bigserial column follows the table. The migration is refused naming the table and its sequence, rolls back with the relations still unlogged and no native table, and accepts the same source once it is logged again. (qbit_audit_publication_sequence_seq is not used for this: 001's own guard already refuses that sequence when it is not logged.) A unit test covers the comparison for tables, sequences and a temporary table. Replays of the real 0.1.2 -> 1.x -> 2.x schema history, with and without 002, still migrate: no revision creates an unlogged relation. The guide's release-schema paragraph, its drifted 001 row and the comment in migration 006 name persistence; the gated-test manifest and its count are updated. Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_01QuEYH8xHQRo7C4SnexM3cP * 3.x.x: compare constraint validation, exempting only the release's own NOT VALID check The source-schema check stripped NOT VALID from every constraint definition before comparing, so a foreign key or CHECK an operator had dropped and re-added NOT VALID compared equal to the validated release constraint, and a source whose rows were never checked against it, orphaned outbox rows or invalid accounting rows for instance, still migrated. The blanket strip is replaced by a pinned exemption list, NOT_VALID_EXEMPT: the constraints the frozen release 001 itself adds NOT VALID on an upgraded table and never validates. Today that is qbit_share_ledger_credit_policy_check alone (001 adds it NOT VALID to a qbit_share_ledger upgraded from before the column existed; a fresh apply creates it validated inside CREATE TABLE). An exempt constraint is accepted in either state on either side. Every other constraint is compared by pg_constraint.convalidated as well as by definition: a release constraint that is NOT VALID in the source is drift, `constraint qbit_block_candidate_outbox_share_id_fkey on qbit_block_candidate_outbox is NOT VALID; the release validates it`. The definition without its NOT VALID suffix is still the lookup key, so an unvalidated constraint finds its release counterpart and is named. A test parses the frozen 001 and 002 fixtures for constraints added NOT VALID and not validated later in the same file (plain statements and the format() form alike) and asserts the set equals NOT_VALID_EXEMPT, so the list cannot drift from the release. A Postgres test drops and re-adds the outbox share_id foreign key and a carry-forward CHECK NOT VALID: the migration is refused naming both, rolls back with nothing validated, and accepts the source once they are validated. The existing tolerated- differences case, with the credit-policy check NOT VALID, still migrates. Unit tests cover the comparison and the exemption. Replays of the real 0.1.2 -> 1.x -> 2.x schema history, with and without 002, still migrate: the only constraint a genuine history leaves NOT VALID is the exempt one. The guide's release-schema paragraph, its drifted 001 row and the comment in migration 006 describe the rule; the gated-test manifest and its count are updated. Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_01QuEYH8xHQRo7C4SnexM3cP * 3.x.x: require every schema migration at start, 009 included and may land after 009, and the rebase moved its 009 step into migrate_schema with the rest of the chain. The startup gate still compared max(version) with 6, which a database at 2, 3, 4, 5 and 9 passes without 006, and its "newer than required" warning would have fired on every start once 009 was in the chain. REQUIRED_SCHEMA_VERSIONS names 2, 3, 4, 5, 6 and 9. A start refuses a database missing any of them, naming the gap, and warns only about migrations this binary does not know. The pre-006 outbox refusal names the applied set instead of a high-water mark; prior_schema_version in the source record stays the highest migration recorded. Tests: a database at 2, 3, 4, 5 and 9 applies 006 on the next migrate and leaves 009 as it found it; a start without PRISM_POSTGRES_INIT_SCHEMA on that database is refused naming 6; the newest-migration case now undoes 009. #319's read-only pre-009 connect test is removed, since the gate refuses that database by design, and the gate manifest follows. Co-Authored-By: Claude Fable 5.1 * 3.x.x: refuse unknown capabilities before 004 to 006 run on a native schema On a database an earlier 3.x.x build migrated to native schema 3, 4 or 5, migrate_schema ran the drain check and then applied 004, 005 and 006 without reading the capability rows, so a database a newer release wrote (candidate_storage_version = 3, or a capability this binary does not know) was altered and recorded at 6, and only Ledger::connect refused it, after the commit. Every native path now takes the inventory first and refuses those rows before any DDL, on a database already at 6 too, before 009; the 006 record reuses the same rows instead of reading them again, and the recorded versions are read in order so the refusal names them deterministically. A pre-006 test on a #258 source covers both refusals and the migration once the row is gone; the migration guide and the 006 comment describe the order. Co-Authored-By: Claude Fable 5.1 * 3.x.x: compare row-level security and policies in the source-schema check The release-schema fingerprint read a table's name, persistence and columns only, so a release table with row-level security enabled or forced, or a policy on it, compared equal to the frozen release. For a migrate role that does not bypass row-level security, a policy hiding pending rows (or forced security with no policy) makes the drain check see an empty outbox, so the migration commits with legacy candidates stranded where the native claim lane never sees them. The fingerprint now reads relrowsecurity and relforcerowsecurity from pg_class and every policy from pg_policy (command, permissive or restrictive, roles, USING and WITH CHECK, rendered without schema qualification), both for the scratch release apply and the source, and the comparison refuses a release table whose flags differ or that carries a policy the release does not have, naming the table, the flags found and each policy. Policies on tables the release does not create are not reported. A Postgres test migrates as a non-superuser owner role and shows the row hidden, the refusal, the rollback, and the migration once security is off and the row drained; the docs and the 006 header say what is compared and why. Co-Authored-By: Claude Fable 5.1 * Quote the pre-006 outbox refusal as the migrator words it since 009 joined the chain The guide's example still read "a native schema 5 database", the wording from before the applied migrations were tracked as a set; the migrator names the recorded set ("a native database at schema migrations 2, 3, 4, 5, 9") since the startup gate stopped comparing a high-water mark. Co-Authored-By: Claude Fable 5.1 * 3.x.x: refuse a 2.x.x or empty source that already holds a native object or column Codex P1 on #321: a source that already contains an object a native migration creates (a qbit_prism_cluster(singleton boolean PRIMARY KEY), for instance) passed the fresh and release checks as a harmless extra. 002's CREATE TABLE IF NOT EXISTS then kept it as it was, the migration recorded its versions, and only after commit did writable() fail on the missing fatal_error column, with every later migrate skipping the recorded version. A column a native migration adds with ADD COLUMN IF NOT EXISTS (002's claim columns on the outbox and the fanout artifacts, 003's canonical_audit_bytes and inactive_since, 006's storage_version on a pre-#258 outbox) is the same bug: a source release table that already has a column of that name keeps it whatever its type or default. The scratch apply release_fingerprint runs now continues, after the release SQL, with the native migrations in the order migrate_schema applies them (NATIVE_MIGRATIONS, pinned to REQUIRED_SCHEMA_VERSIONS by a unit test) and fingerprints the scratch schema again. The second reading minus the first is the reserved set (ReservedObjects): every table, sequence, index, trigger and function a native migration creates and the release does not, and, per release table, every column the native chain adds to it. qbit_prism_schema_migrations, the migrator's own, is never in it, and an object or column in both readings (the capability table on a #258 source, the outbox's storage_version there, which the release 002 added; on a pre-#258 source 006 adds it, so it is reserved) stays governed by the release checks. Reserved columns are kept in their own field: objects_present also serves the partial-001 check, where a present table already refuses and listing its columns would be noise, so only require_no_native_collision looks at them. On the 2.x.x path, before any DDL, the source is fingerprinted once, require_fresh_source (now synchronous, on that reading) runs on a fresh source, and a source that already has a reserved object or column is refused as the new "native collision" state, naming each (capped by named_objects; a column as column .), saying nothing was changed, and telling the operator to restore the backup or check and remove the objects; nothing is dropped and no DROP is suggested. release_objects_present is renamed objects_present. The migrator applies each native migration from NATIVE_MIGRATIONS, so the scratch apply and the real one cannot drift apart. SOURCE_STATES, its pinned-table test, the comment in 006 and the guide's table carry the new row; the guide shows the refusal and names the column case. Tests, against PostgreSQL 16: an empty database with a stray qbit_prism_cluster is refused naming it, with no migrator table and the stray table unaltered, and migrates as fresh once the table is removed; a pre-#258 and a #258 source with a stray native table, function and sequence are refused naming all three, unchanged, and migrate once they are removed; a pre-#258 and a #258 source with a stray claim_token integer on the outbox are refused naming column qbit_block_candidate_outbox.claim_token, unchanged (the column keeps its type), and migrate once it is dropped, after which 002 adds the text column it wanted. The unit test on the reserved set pins the column derivation, including storage_version reserved on a pre-#258 outbox and not on a #258 one. The api_database fixture, which hand-applies 001 to 003 and then lets connect bring the schema forward, now records versions 2 and 3 the way the migrator does: native objects with no record are the collision this commit refuses. The genuine 0.1.2 -> 1.x -> 2.x history replays (the 12 distinct 001 revisions reachable from the release tags, main, 2.x.x and 3.x.x, applied in order into one database and each on its own, then the frozen v2.0.2 001, with and without 002, then migrate: 26 runs) are all accepted, so no real history contains a native object or column. The new Postgres tests are registered in test/prism-gated-tests.txt, and the "N today" count in docs/prism-integration-test-gate.md, which had fallen behind the manifest at an earlier rebase, is set to the manifest's id count, 114. Co-Authored-By: Claude Fable 5.1 * 3.x.x: fold migration 008 into the native chain after rebasing on 3.x.x #327 added 008_prepared_window_reference.sql to the runner in connect.rs, which this branch had moved into migrate_schema. Put 008 in NATIVE_MIGRATIONS, so the scratch apply reserves what it creates, and in REQUIRED_SCHEMA_VERSIONS, so the startup gate requires it; apply it after 006 and before 009. 007 is the only reserved number now. The pre-006 fixtures the tests undo from the current chain therefore model the database today's 3.x.x leaves, at 2, 3, 4, 5, 8 and 9, and the messages and docs name that set. * 3.x.x: refuse a native record with 3 and not 2 before any DDL Every native build has recorded 2 and 3 in the one transaction that applied them, so a record with 3 and not 2 was edited or restored selectively. migrate_schema's native path applied the rest of the chain above it and only the startup gate then refused the gap, at every start, and no later migrate could repair it: 002 is not re-run on a database at 3, and recording it unseen would vouch for objects the run never checked. Refuse the record before any DDL, naming the remedy; a start without initialize keeps refusing the gap. * 3.x.x: require the capability declaration once migration 006 is recorded require_known_capabilities accepted a database at 6 whose capability table or candidate_storage_version row was gone, as if absence were the legacy state it was before 006; after a selective restore or a deletion the database could no longer say which release wrote it and the connect gate let it through. 006 created both and nothing native removes them, so require them at connect once require_schema_version has established 6, naming the remedy, and refuse the same database on the migrate path before 008 or 009 run above the missing declaration. * 3.x.x: compare the enforcement triggers of release constraints The trigger reading skipped every internal trigger and the constraint comparison looked at the definition and convalidated only, so a release foreign key whose referential triggers a superuser had disabled (ALTER TABLE ... DISABLE TRIGGER ALL during a bulk load, say) compared equal: no new row was checked against it, 001 never re-enables a trigger it did not create, and the native ledger could go on accumulating orphaned rows. Read the enabled state of each internal trigger that enforces a constraint into its definition, on both tables of a foreign key, and refuse a release constraint whose states differ from the scratch apply, naming them. * 3.x.x: refuse inheritance involving a release table The table reading never looked at pg_inherits, so a child created with INHERITS (qbit_share_ledger) was an extra ordinary table, accepted with a warning, while PostgreSQL includes its rows in every query of the ledger, the unqualified share reads included, without the release constraints ever checking them. Read each table's parents and children into its definition and refuse a release table that has either, naming them; a release table attached as a partition of or inheriting from another table is drift the same way, and inheritance among the operator's own tables stays theirs. * 3.x.x: count a reserved name held by a relation of any kind as a collision The index reading left every index backing a primary, unique or exclusion constraint to the constraint comparison, and the collision check looked up reserved index names in that reading only, so an operator table whose unique constraint was named qbit_prism_candidate_claim_idx owned that name unseen: 002's CREATE INDEX IF NOT EXISTS would skip the outbox index it meant to create, record the migration anyway, and leave candidate polling to scan the outbox. IF NOT EXISTS looks at the name alone, for a view or a composite type as much as for such an index, so read every relation of the kinds no other map models into the fingerprint, reserve the native ones the release lacks, and count a reserved table, sequence or index name held by any of them, naming what holds it. An index backing a constraint of an expected table goes with that table and is not named twice. * 3.x.x: refuse extra user triggers on release tables The release-schema check classified a user trigger the release does not create on a release table as a tolerated extra, so require_release_schema logged it and migrated over it. Such a trigger can be valid for the legacy writer and incompatible with the native one: a guard that rejects writer_epoch = 0 lets the 2.x.x writer's leased epochs through and fails every native share insert, which writes epoch 0, and a trigger can just as well rewrite what the migrator and the native writers put in the accounting tables. Report a found user trigger on a release table whose (table, name) the release lacks as drift, naming the trigger, its table and its definition, and whether the release has no trigger on that table or simply does not create this one, so the migration refuses transactionally and the database is unchanged. Triggers on tables the release does not create stay the operator's own; the expected-trigger definition and enabled-state comparison and the enforcement-trigger check of release constraints are unchanged. Regression: a comparator unit test for both reasons, a missing release trigger and an operator table's trigger; and a two_x PostgreSQL test on the frozen pre-#258 and #258 sources that shows the guard accepting a legacy epoch and refusing epoch 0, the refusal naming the trigger and rolling back with the trigger, the legacy rows and no native table or version in place, then the same source migrating once the operator drops the guard, with the operator table's trigger kept and firing and the native writers inserting their epoch-0 shares. Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_01N2if7arQVKaZ1yMupJiprr * fix: reject extra constraints on migration source tables * fix: reject unsatisfied extra migration source columns * fix: reject extra indexes that affect native writes * fix: update rustls to address RUSTSEC-2026-0285 * fix: reject executable expressions on extra source columns * fix: fingerprint table rewrite rules before migration * fix: reject incoming foreign keys from extra source tables * fix: validate storage version before native migration 006 * fix: reject domain-typed extra migration columns * fix: reject migration sources resolved outside the current schema * fix: reject all extra indexes on release tables * fix: validate source metadata before later native migrations * fix: reject preexisting source metadata before migration 006 * fix: reject row security on capability metadata * fix: separate legacy source versions from native capabilities * fix: reject outbox row security before native migration drain * fix: validate migration history before trusting version records * fix: require an ordinary table for capability declarations * fix: bind the capability declaration to the current schema The connect-time gate probed qbit_prism_schema_capabilities with an unqualified to_regclass, which follows search_path. When the migrated schema had lost its capability table, the probe could resolve a later schema's table, and its version-1 row cleared a database whose own declaration was gone, while require_migration_history had bound the history to current_schema(). Migrate was already safe through require_source_schema_resolution; connect was not. read_capabilities now looks the visible relation up in the catalog and refuses one outside current_schema(), naming both schemas, before any row is read. Both callers share the path: the connect gate drops its separate probe and the migrate inventory reads through the same function. A new gated test drops the migrated schema's declaration, declares version 1 in a later schema on search_path, and asserts that connect and migrate both refuse it, that the loss is still named as a loss without the later schema, and that the restored declaration is read with the later schema still present. Co-Authored-By: Claude Fable 5.1 * fix: bind migration provenance to the verified schema * fix: reject object collisions in native migration gaps --------- Co-authored-by: Claude Fable 5.1 * docs: reference authoritative gated test count (#364) * 3.x.x: Reference the window from candidates and move the found-block write off the ordering lock (#350) * 3.x.x: add migration 007 and refuse pre-007 candidate outbox rows Migration 007 mirrors 008 onto qbit_block_candidate_outbox: the same six window-reference columns, and a three-state CHECK whose body is identical to 008's apart from the table and constraint names. It also adds block_bytes, the nullable bytea the block moves into so the candidate document stays small, and a partial index on window_prior_balances_sha256 for the balance-snapshot GC. Nothing writes any of them yet; this is additive schema only. The runner applies 007 by membership, in its own block between 5 and 8, so it assumes neither 006 nor 008 has run. Before any DDL it refuses while the outbox still holds a pending row in a pre-007 shape: a native inline candidate, a NULL candidate, or storage_version other than 1 on the 2.x.x schemas that have that column, which is checked for first so the predicate does not error on a native schema. The refusal names every offending block_hash and the drain procedure, and the whole migration transaction rolls back, so a refused start leaves the schema untouched. The applied set is now [2, 3, 4, 5, 7, 8, 9], and the four exact migration assertions move with it rather than being weakened to a superset. Part of #265 Co-Authored-By: Claude Opus 5 * 3.x.x: retain the cluster fingerprint configure pinned or verified The candidate enqueue and save_job must fence against a fingerprint reset by re-reading qbit_prism_cluster.config_fingerprint FOR SHARE in their own transaction and refusing a write that is not this frontend's own. That needs the value configure settled at startup, so Ledger keeps it in a shared OnceLock, set only once the pin or the match has committed, and exposes it read-only through config_fingerprint(). configure keeps its signature and its mismatch refusal, and a refused call retains nothing. The rotation refusal in configure, and the writer fence itself, belong to the enqueue change and are not here. Part of #265 Co-Authored-By: Claude Opus 5 * 3.x.x: share the window reference, balance writer and page primitives WindowRef::from_snapshot is the synchronous reference builder both writers need: anchor_ms copied, the semantic balances digest, and for a non-empty snapshot the first and last share_seq, the length, and the share array's SHA-256 streamed through the existing DigestWriter, so the 233 to 260 MB serialized window at 400,000 shares is never built and the digest equals the one qbit_prism_audit_snapshots already stores. An empty snapshot gives the empty-window reference. Callers run it inside their own spawn_blocking. put_balance_snapshot is the one canonical writer of qbit_prism_balance_snapshots, on the caller's transaction: the sort by (order_key, recipient_id, p2mr_program_hex), the semantic digest that keys the row, and compact JSON of the sorted set, all off the runtime, then INSERT ... ON CONFLICT DO NOTHING. On a conflict it reads the stored bytes back and compares them with its own, because these rows are immutable and can only disagree through corruption. The enqueue and save_job and its repair all write through it. read_prior_balances and the current-balance read still keep their SQL order; the writer's sort defines the stored encoding, which is the order read_window(..., AsIssued) returns. probe_share_rows and read_range_paged are read_window's endpoint probe and page loop, lifted onto a caller's connection so the enqueue probe and the landing re-read share one implementation. The predicate, the ascending keyset order, the 4096-row page bound and the per-page count bound are unchanged, and read_window now calls them, with its signature and behaviour unchanged. The paged reader takes the caller's state and a per-page consumer, both of which travel to the blocking thread with the rows, so the landing re-read can compare each page with its slice of a vector it already holds without any whole-window work on the runtime; a dropped read still releases the accumulated state on a blocking thread. read_window_with_permit holds the caller's window_reads permit across the whole read. The permit is moved into a BlockingDrop declared before the read starts, so a cancellation releases it on a blocking thread after the page state queued its own release there, and never with the future; a return releases it inline, before the caller's rebuild. WindowError::TaskFailed separates a cancelled or panicking blocking hand-off from corruption. Every hand-off maps a JoinError to it instead of to Decode, so a cancellation is a retryable failure with its own alert and never takes the abandon path. Part of #265 Co-Authored-By: Claude Opus 5 * 3.x.x: derive the witness merkle leaves from an assembled block A claim rebuilds from stored inputs, and the witness leaves are the one builder input that scales with the block, so they are re-derived rather than stored: one 64-hex string per transaction beside the candidate would grow the document with the block. witness_merkle_leaves_from_block skips the 80-byte header, reads the compact-size transaction count, walks past the coinbase and returns double_sha256 of every remaining transaction's full serialized bytes. That is sound because assemble_submission appends the template's transactions verbatim after the coinbase, so the result equals witness_merkle_leaves_hex(&transactions_from_template(template)) for the block built from that template, with or without segwit transactions. The segwit-aware walk is strip_witness_transaction's own, factored into a helper that also reports where the transaction ends, which is what a block walker needs; strip_witness_transaction keeps its trailing-byte check and its output. A truncated block, trailing bytes, a zero count and a count that disagrees with the transactions present are all errors, never a partial list of leaves. Part of #265 Co-Authored-By: Claude Opus 5 * 3.x.x: version the audit builders and key the CC5 goldens by it A stored window reference reproduces its bundle only under the builder that first built it. The schema strings do not capture that, because the share shape chooses them rather than the builder logic, so qbit-prism gains AUDIT_BUILDER_VERSION, starting at 1. A candidate and a prepared job store it, and a rebuild refuses a reference at another version instead of rebuilding it with the current builder. The bump rule is on the constant: any change that alters canonical_audit_bundle_bytes for the same inputs bumps it in the same commit. The CC5 golden test now selects its expected digests from a table keyed by the constant, and fails when no entry exists, so a new output is recorded as a new entry beside the old one and review sees both. Version 1 holds the existing literals, unchanged. Part of #265 Co-Authored-By: Claude Opus 5 * 3.x.x: cover 007, the fingerprint and the window primitives on PostgreSQL Ten gated database cases, all listed in test/prism-gated-tests.txt. They go in a new file rather than the shared 2.x.x support module, which another open change appends to. Migration 007: a frozen 2.x.x schema carrying only terminal outbox rows upgrades with every row and field intact and the seven new columns NULL; each of the three pre-007 pending shapes is refused by name, with nothing from 007 applied and no version 7 row, and the same binary starts once the row is drained; a database at [2, 3, 4, 5, 8, 9] applies 007 and only 007, leaving every other version's applied_at untouched; the CHECK accepts exactly the three states across all 64 null combinations and rejects a short or uppercase digest, a zero or out-of-range count and a zero sequence; and the retention index is the partial one on window_prior_balances_sha256. config_fingerprint() returns what configure pinned or verified, is shared by clones, and stays empty after a refused configure. from_snapshot's streamed digest equals sha256(serde_json::to_vec) and the snapshot_sha256 landing writes for the same window. The balance writer stores the canonical sort, is idempotent under any permutation, errors rather than accepting a different encoding of the same digest, and read_window(..., AsIssued) hands back the stored order. The probe misses a missing endpoint, the paged reader compares 4,999 rows page by page with a vector held inside the caller's transaction, and a row the predicate skips inside a range is Incomplete with no partial window. The permit case builds a runtime with one blocking thread, occupies it, and cancels a reader parked mid-page on a database gate: no permit is available while the cleanup is queued, and one is once it has run. A failed blocking hand-off surfaces as TaskFailed, never as Decode. Part of #265 Co-Authored-By: Claude Opus 5 * 3.x.x: carry the window reference on prepared work, behind a read permit `Prepared` gains `window: WindowRef`, computed once per non-cached refresh beside `build_bundle` under `tokio::join!`, so submit can clone it instead of re-digesting the window on the share path when a block is found. An empty snapshot gets the empty-window reference on the runtime thread; a resume computes it inside the blocking decode it already runs. `Coordinator` gains the `window_reads` semaphore, sized `clamp(database_max_connections - 2, 1, build_workers)`, and the one entry point callers use: it takes an owned permit after the caller's `build_slots` permit and hands it to `Ledger::read_window_with_permit`, which holds it across the whole read, including the off-runtime cleanup a cancellation leaves behind, and releases it before the rebuild. `Ledger::read_window` is awaited on the runtime, never wrapped in `spawn_blocking`; the test with a single blocking thread pins that. The four gated `window_ref_tests` are listed in the manifest. Part of #265 Co-Authored-By: Claude Fable 5.1 * 3.x.x: switch the candidate outbox to the window reference `Candidate` drops the embedded `AuditBundle` for `window: WindowRef` and every builder input except the window: `found_block`, `payout_policy`, the nested `ctv` triple (whose presence selects the CTV builder), the stored `bootstrap_share` of an empty window, `audit_builder_version`, `signer_keys`, a `leased` flag and the now-required `coinbase_suffix_hex`. The block leaves the document for the `block_bytes` column, authenticated by `block_sha256`, which `candidate_sha256` covers. The write serializes, digests and checks the candidate before the append or enqueue transaction opens; under `ORDER_LOCK` the transaction re-reads `config_fingerprint FOR SHARE` and refuses a value the writer did not pin, re-inserts a `leased` candidate's as-issued balance snapshot through the shared writer, probes the window's first row (an absent row alerts and still publishes), and inserts only the prepared bytes with the six typed columns. Submit builds the slim candidate from `Prepared.window`, the `BundleInputs` captured where `build_bundle` read configuration, and the job's own bootstrap share; it clones no bundle and reads no config. The claim's `UPDATE … RETURNING` returns the columns and the block, and the decode, off the runtime, checks the document against them, the digest, `sha256(block_bytes)`, the header hash, the required fields and the two reference invariants. A pending row with a NULL anchor is a hard error naming the pre-007 frontend; an inline document is never decoded. Before any read the claim retries, with an alert naming both values, a stored builder version or signer pair that is not this binary's; a landed audit is authenticated against the block (coinbase, audit root as the witness reserved value, share snapshot, bits) and the claim then skips the rebuild and landing and continues to observe, renew and submit. Otherwise it takes `build_slots`, then `window_reads` inside one 60 s deadline around the awaited `read_window(Current)` and the borrowing builder run directly in `spawn_blocking`, and keeps `Arc` and the shares as `ClaimParts`. `Incomplete`, `SnapshotDigestMismatch`, `Decode`, `BalanceSnapshotMissing`, `TaskFailed` and the deadline retry with their own alerts; `PriorBalancesChanged` finishes only a block `observe_candidate` proves inactive and superseded; `Database` propagates. A `leased` candidate submits from `block_bytes` first, then rebuilds `AsIssued` and lands before any terminal outcome. Rebuilt parts are released off the runtime on every path. Landing takes the parts and does all whole-window work in one blocking closure: `verify_audit_parts`, the raw body, the counted canonical bytes, the leaves and accounts, and the share snapshot digest, which must equal the reference's. The balance fence compares the current set and the parts' through the semantic digest, so an as-issued set compares equal in any order. `persist_audit_snapshot` re-reads the range with the paged reader inside the landing transaction, page against slice on the blocking thread; `read_range` stays for the API. The terminal UPDATE NULLs the six window columns and `block_bytes` with `candidate`. `configure` refuses to pin a fingerprint onto a reset one while a pending row stores other signer keys, and the startup gate requires 007 with 009. The JSONB gate drops its enqueue entry, and every helper that built a candidate builds the slim one, handing the ledger-level landings the bundle's parts through `CandidateClaim::with_bundle`. Part of #265 Co-Authored-By: Claude Fable 5.1 * 3.x.x: cover the candidate switch on PostgreSQL Ledger-level, in `candidate_window_switch`: a two-page window enqueues a document with no array and no share row, under 1 MB, whose six typed columns equal its reference and whose block is stored as bytes; the writer fence refuses a reset or different fingerprint in the same transaction, share included; a missing window prefix row alerts and still publishes, and its read then meets `Incomplete`; a `leased` enqueue re-inserts the as-issued balance snapshot in the writer's canonical order, writes none when not leased, and refuses balances that do not hash to the reference; every preparation failure is refused without waiting for `ORDER_LOCK`; every column, digest, block, header, NULL-anchor, missing-field, invariant and inline-document disagreement is refused at claim; landing refuses parts that are not the referenced window and a claim without parts; the terminal UPDATE NULLs the six columns and the block for submitted and abandoned rows; the landed-audit authentication rejects a forged coinbase, root, snapshot or bits; and `configure` refuses a rotation over a foreign-signed pending candidate. Coordinator-level, in `window_switch_tests` behind a fake node: a stored builder version or signer pair that is not this binary's retries, naming both values, while the build slot is held; a landed audit is authenticated and the claim continues to `submitblock` without a build slot, and a forged landed coinbase or root is refused with the row left pending; `Incomplete`, a digest mismatch, changed balances on a current block and the rebuild deadline all retry with an alert and release their permits; `classify_window_error` keeps `TaskFailed` apart from corruption and propagates `Database`; the read, rebuild and landing complete while `ORDER_LOCK` is held, with one build worker; a `leased` candidate submits first, then lands from `AsIssued`, and finishes submitted, abandoned with the node's reason, or pending when its balance snapshot is missing; and `OffRuntime` releases on a blocking thread, on cancellation too. The fifteen gated tests are listed in the manifest. Part of #265 Co-Authored-By: Claude Fable 5.1 * 3.x.x: migrate the imported-audit tests to the slim candidate Slice 1 (#325) merged after this stack branched, and its tests in tests/support/ledger_2x.rs still built the pre-007 candidate: they read block.bundle from a Candidate and called candidate_with_bundle without a window reference or settlement inputs. The rebase merged them cleanly and the tree then failed to compile. The three helpers now take the TestCandidate the constructors return, so they reach the bundle beside the slim candidate, and land_confirmed attaches the claim's parts the way the other ledger tests do. The three constructor calls pass the window reference the bundle was built on, and the settlement inputs for the CTV case. No assertion changes. Part of #265. * 3.x.x: qualify lease renewal and the claim's canonical bytes at scale Incident 1: a candidate whose window holds N shares is claimed and processed under the short lease, and every renewal is read back from claim_expires_at. The largest gap stays below interval + 2 x timeout, the lease never lapses, and a successor ledger cannot take the claim. The bundle a claim rebuilds through the window-read entry point and the borrowing builder has the canonical bytes refresh built for the same job, and the landed audit carries that bundle's digest. Both run at PRISM_WINDOW_QUALIFY_SHARES (20,000 by default); the 400,000-share variants are ignored and run on a larger host. Part of #265 Co-Authored-By: Claude Opus 5 * 3.x.x: measure ACK latency and ORDER_LOCK hold across window sizes Incident 2: two frontends on one database submit ordinary shares at a fixed rate while one of them solves a block on an N-share window and its own submit loop claims, rebuilds, lands and submits it. The solving share's ACK and the bystander's p99 stay within max(2 x baseline p99 + 50 ms, 250 ms), every ACK is well under the share-commit timeout, and the block is enqueued exactly once. Latency is the in-process MiningBackend::submit wall clock; the socket-level harness (#271) does not exist yet. A pg_locks sampler on its own connection measures how long a solving append holds ORDER_LOCK at two window sizes with no other traffic: the larger size stays within a fixed slack of the smaller, and the candidate document is the same size at both and under 1 MiB. Part of #265 Co-Authored-By: Claude Opus 5 * 3.x.x: hold the JSONB gate's enqueue to a window-free row under 1 MiB At every measured size the enqueue must be accepted and every JSONB value it writes, the outbox candidate document included, must stay under 1,048,576 bytes. At the reduced sizes the enqueue, claim and landing phases must all run. The ratchet and its expected values are unchanged. Part of #265 Co-Authored-By: Claude Opus 5 * 3.x.x: make the window-read permit test actually wait before it looks The delay that gives an unpermitted read time to finish was an empty async block, so the check that the read had not finished ran at once and passed whether or not the entry point took a permit. It now sleeps 500 ms inside a 5 s bound, and fails when the permit acquisition is removed from the entry point. Part of #265 Co-Authored-By: Claude Opus 5 * 3.x.x: rebuild a CTV candidate from its stored settlement inputs No host-runnable test reached the CTV branch of the claim rebuild. A CTV-enabled frontend now claims a candidate whose stored direct floor differs from its configuration, and builds a different coinbase than the configured floor would, and the claim must land that candidate's own audit and submit it. Part of #265 Co-Authored-By: Claude Opus 5 * 3.x.x: pin the SQL order of the current-balance reads Nothing defended the ruling that the current-balance path keeps its SQL order: every fixture's balances already arrived in byte-wise canonical order. The carry table's text columns now take an ICU collation, under which the SQL order and the canonical sort disagree, and the snapshot and the current-balance window read must both return the SQL order. Part of #265 Co-Authored-By: Claude Opus 5 * 3.x.x: record what incident 2 must also assert once #324 is in the base After #324 a slow solve is acknowledged late rather than refused, so acceptance proves nothing and the latency percentiles stay the assertion. The doc comment and a TODO at the outcome checks name the counter and outcome to assert at rebase. Part of #265 Co-Authored-By: Claude Opus 5 * 3.x.x: assert incident 2's late-confirm counter now that #324 is in the base W3c wrote the qualification tests against a base that predated #324 and left a marked place for the assertions that only make sense once the commit gate is there. The base now carries it, so apply them: incident 2 asserts `qbit_prism_late_confirmed_shares_total` is zero on both frontends and that no acknowledgement, the block-solving share included, is `ledger-outcome-unknown`. Acceptance alone is no longer proof. #324 gives a share-pass append carrying a candidate its own `block_only_ack_timeout`, so a slow solve is never refused at the share deadline; it is answered late, or answered unknown. Either would have satisfied the acceptance checks with the serialization stall still present. Also set #324's two new `Config` fields in the qualification fixture. * 3.x.x: pin a refused candidate as a definite rejection, not an unknown outcome #324 decides what a miner is told when a block-only enqueue fails: `enqueue_failed_before_commit` answers a provably uncommitted failure with `ledger-confirmation-failed`, and anything else with `ledger-outcome-unknown`, which asks an operator to go and follow the outbox row. Slice 3 added the checks that guard the reference: the block's length, its header hash, its digest against `block_sha256`, the coinbase suffix, the reference invariants, and a leased candidate's balances. All of them are raised by `prepare_candidate` before the enqueue takes a connection, so none carries an `sqlx::Error` and all classify as definite today. That is a property of where the checks sit, not something the classifier knows about them, so it is pinned here rather than left to hold by accident: a corrupt candidate must never send a miner chasing a row that was never written. Verified by mutation: classifying a non-sqlx failure as unknown fails this test. * 3.x.x: say why the full-size qualification variants cannot reach 400k yet Their ignore reasons blamed RAM and the host. The real limit is #273: the job payload still holds three copies of the window (2 x AcceptedShare + 1 x CountedShare), so publishing a job for a 400,000-share window needs about 670 MiB and PostgreSQL refuses the write above its 256 MiB jsonb object limit. Every full-size variant fails at fixture set-up, before it measures anything. The count must also divide the window weight exactly, so the largest usable size today is 125,000, where all four variants pass in release: incident 1 50 renewals, max gap 118.25 ms against a 700 ms bound incident 2 phase-A p99 2.60 ms, phase-B p99 95.79 ms, bound 250 ms lock hold median 0.75 ms at 5,000 against 1.32 ms at 125,000 canonical refresh and claim rebuild both 139,040,610 B, same sha256 The candidate document stays flat at 1103 B and 1107 B, which is the property this slice exists to establish. * 3.x.x: migrate the audit-body normalization tests to the slim candidate #267 landed its own test file against the candidate that still carried an assembled bundle. This slice replaced that with a window reference, so the file keeps a `TestCandidate` pairing the stored candidate with the bundle a test still needs, as `tests/ledger_postgres.rs` does, and attaches the claim's parts through `with_bundle`. One case needed more than a shape change. The four negative cases hand the claim a window the ledger does not hold and require the durable-range proof to be what refuses it. A candidate now commits to a snapshot digest, so a reference taken from the unaltered snapshot would be refused earlier, at the reference check, and the proof under test would never run. `window_ref_for` therefore derives each altered window's reference from that window, leaving the proof as the only thing that can notice, which is what #267 wrote them to prove. * 3.x.x: pin the durable-range proof outside the settlement lock #267 moved the full-equality proof of a candidate's window out of the settlement transaction so the lock is not held for a window-sized read. This slice made that read paged, so it does not occupy a runtime thread either. Reconciling the two is the step that can quietly undo the first: keep this slice's `persist_audit_snapshot` through the rebase and the full read goes back under the lock. The landing stays correct, every other test still passes, and only the lock hold grows, so nothing in the suite can tell. Measured at 100,000 shares on #267's branch that was a 34% longer hold. The guard is deterministic and needs no large window. A second connection holds `SETTLEMENT_LOCK`, so the landing blocks on it once its proof has run. While it waits, one share's `ntime` is altered: full equality sees that, a count over the same anchored predicate cannot. The landing must then succeed, which it can only do if it had already read and compared the shares before taking the lock. Accepted history is immutable and a trigger enforces it, so the test lifts that trigger for the single statement, inside its own disposable schema, and puts it straight back. Verified by mutation: restoring the in-transaction range read fails this test with "the landing read share payloads under the settlement lock". * 3.x.x: serialise the coordinator test modules that share the ledger's locks The ledger's advisory locks are cluster-wide constants, not schema-scoped, so two coordinator test modules running in parallel contend for ORDER_LOCK even though each works in its own throwaway schema. Every module held a lock of its own, which serialised it against itself and left cross-module waits to `lock_timeout`, 5 s by default. #324's reconciliation tests hold ORDER_LOCK for up to about 1.5 s, and this slice adds three more modules that open a ledger. Once enough of them ran at once those waits stacked past the timeout: a full `--lib` run failed with "canceling statement due to lock timeout" in `candidate_lease_tests` and `window_switch_tests`, and passed on the next run. A suite that fails for a reason unrelated to what it tests is worse than a slow one. All seven modules now take one shared lock. The run goes from about 35 s to about 62 s, and three consecutive runs pass where the flake had reproduced. * 3.x.x: pass the signer keys #334's wait-metrics tests now need `Ledger::configure` takes the frontend's signing keys since this slice, so the fingerprint contention tests #334 added call it with a fixed pair. That suite measures advisory-lock and pool-acquire waits, not signing, so the only thing the keys have to be is the same on every call. * 3.x.x: resume a job's bundle only while the current inputs still describe it A resumed job reuses the bundle it was issued with, but captures its inputs from this frontend's configuration now. Before this slice that was harmless: the candidate carried the assembled bundle, and landing used it directly. Now the candidate stores the inputs and the claim rebuilds the audit from them, so an old bundle labelled with today's configuration rebuilds to a coinbase the block does not commit to. The claim fails its comparison before `submitblock` and retries until the job expires, which can lose a found block. The stored bundle is now reused only when the captured inputs still describe it, comparing what the bundle itself records about how it was built: the payout policy, and the public keys that actually signed its coinbase manifest and its ledger attestation. A difference is a cache miss, so the bundle is rebuilt rather than mislabelled. Both directions are covered, since over-tightening would silently defeat the job cache and rebuild on every resume: unchanged inputs must still describe the bundle, and a payout-policy change, a full signer rotation and a rotation of either key alone must each refuse it. Known residue, recorded in the doc comment: a bundle records neither the CTV settlement inputs nor `audit_builder_version`, so a change to either between issue and resume is still undetectable here. Closing that needs the issued job to store its inputs, which is #273's format. * 3.x.x: migrate #355's ledger tests to the slim candidate #355 landed deadline and single-execution coverage against the candidate that still carried an assembled bundle, and against `configure` before it took the frontend's signing keys. Adapted, the same way #325's and #267's suites were: a `TestCandidate` pairs the stored candidate with the bundle a landing still needs, claims attach their parts through `with_bundle`, and the conflicting claim that is never landed carries none. `ledger_deadline_atomicity` builds its candidate from a bundle rather than a snapshot, so its window reference is derived from that bundle: the anchor and prior-balances digest it already records, and the range and digest of the shares it carries. Its signer keys come from the public keys that actually signed it, rather than from a local pair, so the stored inputs describe the bundle. * 3.x.x: make the landed-audit forgery always alter the row, and migrate #360's fixture The forged-coinbase case placed a literal '0' at a fixed offset. Whenever the digit already held that value the forgery changed nothing, so the claim was right to accept it and the case failed only by luck. The coinbase varies per run, because the bundle's anchor comes from the ledger clock, so this passed repeatedly and then failed once as "the forgery was accepted" — an alarming message for a case that was in fact a no-op. A security check that can quietly become vacuous is worse than one that is absent. The digit is now flipped to a value it does not currently hold, and the row is read back either side of the statement so a forgery that altered nothing fails the case for that reason instead. Also passes the signer keys #360's `tests/support/fatal_state.rs` now needs, since `configure` takes them in this slice. * 3.x.x: drop the pre-#362 path for the heartbeat SQL gated test #362 moved `live_instance_tests` from `tools` to `ledger::instances`. Merging the gated-test manifest during the rebase kept both paths, because a rename upstream looks like one id removed and another added, and a union keeps the removed one. The old `tools::` path then never executed and the gate refused the run: "194 distinct gated tests executed, below the minimum of 195". The manifest is a superset of the base's ids, and every id this branch adds over the base resolves to a test function that exists in the tree. * v3.0.0 feat: add compact prepared storage and immutable template blobs (#359) * 3.x.x: add migration 007 and refuse pre-007 candidate outbox rows Migration 007 mirrors 008 onto qbit_block_candidate_outbox: the same six window-reference columns, and a three-state CHECK whose body is identical to 008's apart from the table and constraint names. It also adds block_bytes, the nullable bytea the block moves into so the candidate document stays small, and a partial index on window_prior_balances_sha256 for the balance-snapshot GC. Nothing writes any of them yet; this is additive schema only. The runner applies 007 by membership, in its own block between 5 and 8, so it assumes neither 006 nor 008 has run. Before any DDL it refuses while the outbox still holds a pending row in a pre-007 shape: a native inline candidate, a NULL candidate, or storage_version other than 1 on the 2.x.x schemas that have that column, which is checked for first so the predicate does not error on a native schema. The refusal names every offending block_hash and the drain procedure, and the whole migration transaction rolls back, so a refused start leaves the schema untouched. The applied set is now [2, 3, 4, 5, 7, 8, 9], and the four exact migration assertions move with it rather than being weakened to a superset. Part of #265 Co-Authored-By: Claude Opus 5 * 3.x.x: retain the cluster fingerprint configure pinned or verified The candidate enqueue and save_job must fence against a fingerprint reset by re-reading qbit_prism_cluster.config_fingerprint FOR SHARE in their own transaction and refusing a write that is not this frontend's own. That needs the value configure settled at startup, so Ledger keeps it in a shared OnceLock, set only once the pin or the match has committed, and exposes it read-only through config_fingerprint(). configure keeps its signature and its mismatch refusal, and a refused call retains nothing. The rotation refusal in configure, and the writer fence itself, belong to the enqueue change and are not here. Part of #265 Co-Authored-By: Claude Opus 5 * 3.x.x: share the window reference, balance writer and page primitives WindowRef::from_snapshot is the synchronous reference builder both writers need: anchor_ms copied, the semantic balances digest, and for a non-empty snapshot the first and last share_seq, the length, and the share array's SHA-256 streamed through the existing DigestWriter, so the 233 to 260 MB serialized window at 400,000 shares is never built and the digest equals the one qbit_prism_audit_snapshots already stores. An empty snapshot gives the empty-window reference. Callers run it inside their own spawn_blocking. put_balance_snapshot is the one canonical writer of qbit_prism_balance_snapshots, on the caller's transaction: the sort by (order_key, recipient_id, p2mr_program_hex), the semantic digest that keys the row, and compact JSON of the sorted set, all off the runtime, then INSERT ... ON CONFLICT DO NOTHING. On a conflict it reads the stored bytes back and compares them with its own, because these rows are immutable and can only disagree through corruption. The enqueue and save_job and its repair all write through it. read_prior_balances and the current-balance read still keep their SQL order; the writer's sort defines the stored encoding, which is the order read_window(..., AsIssued) returns. probe_share_rows and read_range_paged are read_window's endpoint probe and page loop, lifted onto a caller's connection so the enqueue probe and the landing re-read share one implementation. The predicate, the ascending keyset order, the 4096-row page bound and the per-page count bound are unchanged, and read_window now calls them, with its signature and behaviour unchanged. The paged reader takes the caller's state and a per-page consumer, both of which travel to the blocking thread with the rows, so the landing re-read can compare each page with its slice of a vector it already holds without any whole-window work on the runtime; a dropped read still releases the accumulated state on a blocking thread. read_window_with_permit holds the caller's window_reads permit across the whole read. The permit is moved into a BlockingDrop declared before the read starts, so a cancellation releases it on a blocking thread after the page state queued its own release there, and never with the future; a return releases it inline, before the caller's rebuild. WindowError::TaskFailed separates a cancelled or panicking blocking hand-off from corruption. Every hand-off maps a JoinError to it instead of to Decode, so a cancellation is a retryable failure with its own alert and never takes the abandon path. Part of #265 Co-Authored-By: Claude Opus 5 * 3.x.x: derive the witness merkle leaves from an assembled block A claim rebuilds from stored inputs, and the witness leaves are the one builder input that scales with the block, so they are re-derived rather than stored: one 64-hex string per transaction beside the candidate would grow the document with the block. witness_merkle_leaves_from_block skips the 80-byte header, reads the compact-size transaction count, walks past the coinbase and returns double_sha256 of every remaining transaction's full serialized bytes. That is sound because assemble_submission appends the template's transactions verbatim after the coinbase, so the result equals witness_merkle_leaves_hex(&transactions_from_template(template)) for the block built from that template, with or without segwit transactions. The segwit-aware walk is strip_witness_transaction's own, factored into a helper that also reports where the transaction ends, which is what a block walker needs; strip_witness_transaction keeps its trailing-byte check and its output. A truncated block, trailing bytes, a zero count and a count that disagrees with the transactions present are all errors, never a partial list of leaves. Part of #265 Co-Authored-By: Claude Opus 5 * 3.x.x: version the audit builders and key the CC5 goldens by it A stored window reference reproduces its bundle only under the builder that first built it. The schema strings do not capture that, because the share shape chooses them rather than the builder logic, so qbit-prism gains AUDIT_BUILDER_VERSION, starting at 1. A candidate and a prepared job store it, and a rebuild refuses a reference at another version instead of rebuilding it with the current builder. The bump rule is on the constant: any change that alters canonical_audit_bundle_bytes for the same inputs bumps it in the same commit. The CC5 golden test now selects its expected digests from a table keyed by the constant, and fails when no entry exists, so a new output is recorded as a new entry beside the old one and review sees both. Version 1 holds the existing literals, unchanged. Part of #265 Co-Authored-By: Claude Opus 5 * 3.x.x: cover 007, the fingerprint and the window primitives on PostgreSQL Ten gated database cases, all listed in test/prism-gated-tests.txt. They go in a new file rather than the shared 2.x.x support module, which another open change appends to. Migration 007: a frozen 2.x.x schema carrying only terminal outbox rows upgrades with every row and field intact and the seven new columns NULL; each of the three pre-007 pending shapes is refused by name, with nothing from 007 applied and no version 7 row, and the same binary starts once the row is drained; a database at [2, 3, 4, 5, 8, 9] applies 007 and only 007, leaving every other version's applied_at untouched; the CHECK accepts exactly the three states across all 64 null combinations and rejects a short or uppercase digest, a zero or out-of-range count and a zero sequence; and the retention index is the partial one on window_prior_balances_sha256. config_fingerprint() returns what configure pinned or verified, is shared by clones, and stays empty after a refused configure. from_snapshot's streamed digest equals sha256(serde_json::to_vec) and the snapshot_sha256 landing writes for the same window. The balance writer stores the canonical sort, is idempotent under any permutation, errors rather than accepting a different encoding of the same digest, and read_window(..., AsIssued) hands back the stored order. The probe misses a missing endpoint, the paged reader compares 4,999 rows page by page with a vector held inside the caller's transaction, and a row the predicate skips inside a range is Incomplete with no partial window. The permit case builds a runtime with one blocking thread, occupies it, and cancels a reader parked mid-page on a database gate: no permit is available while the cleanup is queued, and one is once it has run. A failed blocking hand-off surfaces as TaskFailed, never as Decode. Part of #265 Co-Authored-By: Claude Opus 5 * 3.x.x: carry the window reference on prepared work, behind a read permit `Prepared` gains `window: WindowRef`, computed once per non-cached refresh beside `build_bundle` under `tokio::join!`, so submit can clone it instead of re-digesting the window on the share path when a block is found. An empty snapshot gets the empty-window reference on the runtime thread; a resume computes it inside the blocking decode it already runs. `Coordinator` gains the `window_reads` semaphore, sized `clamp(database_max_connections - 2, 1, build_workers)`, and the one entry point callers use: it takes an owned permit after the caller's `build_slots` permit and hands it to `Ledger::read_window_with_permit`, which holds it across the whole read, including the off-runtime cleanup a cancellation leaves behind, and releases it before the rebuild. `Ledger::read_window` is awaited on the runtime, never wrapped in `spawn_blocking`; the test with a single blocking thread pins that. The four gated `window_ref_tests` are listed in the manifest. Part of #265 Co-Authored-By: Claude Fable 5.1 * 3.x.x: switch the candidate outbox to the window reference `Candidate` drops the embedded `AuditBundle` for `window: WindowRef` and every builder input except the window: `found_block`, `payout_policy`, the nested `ctv` triple (whose presence selects the CTV builder), the stored `bootstrap_share` of an empty window, `audit_builder_version`, `signer_keys`, a `leased` flag and the now-required `coinbase_suffix_hex`. The block leaves the document for the `block_bytes` column, authenticated by `block_sha256`, which `candidate_sha256` covers. The write serializes, digests and checks the candidate before the append or enqueue transaction opens; under `ORDER_LOCK` the transaction re-reads `config_fingerprint FOR SHARE` and refuses a value the writer did not pin, re-inserts a `leased` candidate's as-issued balance snapshot through the shared writer, probes the window's first row (an absent row alerts and still publishes), and inserts only the prepared bytes with the six typed columns. Submit builds the slim candidate from `Prepared.window`, the `BundleInputs` captured where `build_bundle` read configuration, and the job's own bootstrap share; it clones no bundle and reads no config. The claim's `UPDATE … RETURNING` returns the columns and the block, and the decode, off the runtime, checks the document against them, the digest, `sha256(block_bytes)`, the header hash, the required fields and the two reference invariants. A pending row with a NULL anchor is a hard error naming the pre-007 frontend; an inline document is never decoded. Before any read the claim retries, with an alert naming both values, a stored builder version or signer pair that is not this binary's; a landed audit is authenticated against the block (coinbase, audit root as the witness reserved value, share snapshot, bits) and the claim then skips the rebuild and landing and continues to observe, renew and submit. Otherwise it takes `build_slots`, then `window_reads` inside one 60 s deadline around the awaited `read_window(Current)` and the borrowing builder run directly in `spawn_blocking`, and keeps `Arc` and the shares as `ClaimParts`. `Incomplete`, `SnapshotDigestMismatch`, `Decode`, `BalanceSnapshotMissing`, `TaskFailed` and the deadline retry with their own alerts; `PriorBalancesChanged` finishes only a block `observe_candidate` proves inactive and superseded; `Database` propagates. A `leased` candidate submits from `block_bytes` first, then rebuilds `AsIssued` and lands before any terminal outcome. Rebuilt parts are released off the runtime on every path. Landing takes the parts and does all whole-window work in one blocking closure: `verify_audit_parts`, the raw body, the counted canonical bytes, the leaves and accounts, and the share snapshot digest, which must equal the reference's. The balance fence compares the current set and the parts' through the semantic digest, so an as-issued set compares equal in any order. `persist_audit_snapshot` re-reads the range with the paged reader inside the landing transaction, page against slice on the blocking thread; `read_range` stays for the API. The terminal UPDATE NULLs the six window columns and `block_bytes` with `candidate`. `configure` refuses to pin a fingerprint onto a reset one while a pending row stores other signer keys, and the startup gate requires 007 with 009. The JSONB gate drops its enqueue entry, and every helper that built a candidate builds the slim one, handing the ledger-level landings the bundle's parts through `CandidateClaim::with_bundle`. Part of #265 Co-Authored-By: Claude Fable 5.1 * 3.x.x: cover the candidate switch on PostgreSQL Ledger-level, in `candidate_window_switch`: a two-page window enqueues a document with no array and no share row, under 1 MB, whose six typed columns equal its reference and whose block is stored as bytes; the writer fence refuses a reset or different fingerprint in the same transaction, share included; a missing window prefix row alerts and still publishes, and its read then meets `Incomplete`; a `leased` enqueue re-inserts the as-issued balance snapshot in the writer's canonical order, writes none when not leased, and refuses balances that do not hash to the reference; every preparation failure is refused without waiting for `ORDER_LOCK`; every column, digest, block, header, NULL-anchor, missing-field, invariant and inline-document disagreement is refused at claim; landing refuses parts that are not the referenced window and a claim without parts; the terminal UPDATE NULLs the six columns and the block for submitted and abandoned rows; the landed-audit authentication rejects a forged coinbase, root, snapshot or bits; and `configure` refuses a rotation over a foreign-signed pending candidate. Coordinator-level, in `window_switch_tests` behind a fake node: a stored builder version or signer pair that is not this binary's retries, naming both values, while the build slot is held; a landed audit is authenticated and the claim continues to `submitblock` without a build slot, and a forged landed coinbase or root is refused with the row left pending; `Incomplete`, a digest mismatch, changed balances on a current block and the rebuild deadline all retry with an alert and release their permits; `classify_window_error` keeps `TaskFailed` apart from corruption and propagates `Database`; the read, rebuild and landing complete while `ORDER_LOCK` is held, with one build worker; a `leased` candidate submits first, then lands from `AsIssued`, and finishes submitted, abandoned with the node's reason, or pending when its balance snapshot is missing; and `OffRuntime` releases on a blocking thread, on cancellation too. The fifteen gated tests are listed in the manifest. Part of #265 Co-Authored-By: Claude Fable 5.1 * 3.x.x: migrate the imported-audit tests to the slim candidate Slice 1 (#325) merged after this stack branched, and its tests in tests/support/ledger_2x.rs still built the pre-007 candidate: they read block.bundle from a Candidate and called candidate_with_bundle without a window reference or settlement inputs. The rebase merged them cleanly and the tree then failed to compile. The three helpers now take the TestCandidate the constructors return, so they reach the bundle beside the slim candidate, and land_confirmed attaches the claim's parts the way the other ledger tests do. The three constructor calls pass the window reference the bundle was built on, and the settlement inputs for the CTV case. No assertion changes. Part of #265. * 3.x.x: qualify lease renewal and the claim's canonical bytes at scale Incident 1: a candidate whose window holds N shares is claimed and processed under the short lease, and every renewal is read back from claim_expires_at. The largest gap stays below interval + 2 x timeout, the lease never lapses, and a successor ledger cannot take the claim. The bundle a claim rebuilds through the window-read entry point and the borrowing builder has the canonical bytes refresh built for the same job, and the landed audit carries that bundle's digest. Both run at PRISM_WINDOW_QUALIFY_SHARES (20,000 by default); the 400,000-share variants are ignored and run on a larger host. Part of #265 Co-Authored-By: Claude Opus 5 * 3.x.x: measure ACK latency and ORDER_LOCK hold across window sizes Incident 2: two frontends on one database submit ordinary shares at a fixed rate while one of them solves a block on an N-share window and its own submit loop claims, rebuilds, lands and submits it. The solving share's ACK and the bystander's p99 stay within max(2 x baseline p99 + 50 ms, 250 ms), every ACK is well under the share-commit timeout, and the block is enqueued exactly once. Latency is the in-process MiningBackend::submit wall clock; the socket-level harness (#271) does not exist yet. A pg_locks sampler on its own connection measures how long a solving append holds ORDER_LOCK at two window sizes with no other traffic: the larger size stays within a fixed slack of the smaller, and the candidate document is the same size at both and under 1 MiB. Part of #265 Co-Authored-By: Claude Opus 5 * 3.x.x: hold the JSONB gate's enqueue to a window-free row under 1 MiB At every measured size the enqueue must be accepted and every JSONB value it writes, the outbox candidate document included, must stay under 1,048,576 bytes. At the reduced sizes the enqueue, claim and landing phases must all run. The ratchet and its expected values are unchanged. Part of #265 Co-Authored-By: Claude Opus 5 * 3.x.x: make the window-read permit test actually wait before it looks The delay that gives an unpermitted read time to finish was an empty async block, so the check that the read had not finished ran at once and passed whether or not the entry point took a permit. It now sleeps 500 ms inside a 5 s bound, and fails when the permit acquisition is removed from the entry point. Part of #265 Co-Authored-By: Claude Opus 5 * 3.x.x: rebuild a CTV candidate from its stored settlement inputs No host-runnable test reached the CTV branch of the claim rebuild. A CTV-enabled frontend now claims a candidate whose stored direct floor differs from its configuration, and builds a different coinbase than the configured floor would, and the claim must land that candidate's own audit and submit it. Part of #265 Co-Authored-By: Claude Opus 5 * 3.x.x: pin the SQL order of the current-balance reads Nothing defended the ruling that the current-balance path keeps its SQL order: every fixture's balances already arrived in byte-wise canonical order. The carry table's text columns now take an ICU collation, under which the SQL order and the canonical sort disagree, and the snapshot and the current-balance window read must both return the SQL order. Part of #265 Co-Authored-By: Claude Opus 5 * 3.x.x: record what incident 2 must also assert once #324 is in the base After #324 a slow solve is acknowledged late rather than refused, so acceptance proves nothing and the latency percentiles stay the assertion. The doc comment and a TODO at the outcome checks name the counter and outcome to assert at rebase. Part of #265 Co-Authored-By: Claude Opus 5 * 3.x.x: assert incident 2's late-confirm counter now that #324 is in the base W3c wrote the qualification tests against a base that predated #324 and left a marked place for the assertions that only make sense once the commit gate is there. The base now carries it, so apply them: incident 2 asserts `qbit_prism_late_confirmed_shares_total` is zero on both frontends and that no acknowledgement, the block-solving share included, is `ledger-outcome-unknown`. Acceptance alone is no longer proof. #324 gives a share-pass append carrying a candidate its own `block_only_ack_timeout`, so a slow solve is never refused at the share deadline; it is answered late, or answered unknown. Either would have satisfied the acceptance checks with the serialization stall still present. Also set #324's two new `Config` fields in the qualification fixture. * 3.x.x: pin a refused candidate as a definite rejection, not an unknown outcome #324 decides what a miner is told when a block-only enqueue fails: `enqueue_failed_before_commit` answers a provably uncommitted failure with `ledger-confirmation-failed`, and anything else with `ledger-outcome-unknown`, which asks an operator to go and follow the outbox row. Slice 3 added the checks that guard the reference: the block's length, its header hash, its digest against `block_sha256`, the coinbase suffix, the reference invariants, and a leased candidate's balances. All of them are raised by `prepare_candidate` before the enqueue takes a connection, so none carries an `sqlx::Error` and all classify as definite today. That is a property of where the checks sit, not something the classifier knows about them, so it is pinned here rather than left to hold by accident: a corrupt candidate must never send a miner chasing a row that was never written. Verified by mutation: classifying a non-sqlx failure as unknown fails this test. * 3.x.x: say why the full-size qualification variants cannot reach 400k yet Their ignore reasons blamed RAM and the host. The real limit is #273: the job payload still holds three copies of the window (2 x AcceptedShare + 1 x CountedShare), so publishing a job for a 400,000-share window needs about 670 MiB and PostgreSQL refuses the write above its 256 MiB jsonb object limit. Every full-size variant fails at fixture set-up, before it measures anything. The count must also divide the window weight exactly, so the largest usable size today is 125,000, where all four variants pass in release: incident 1 50 renewals, max gap 118.25 ms against a 700 ms bound incident 2 phase-A p99 2.60 ms, phase-B p99 95.79 ms, bound 250 ms lock hold median 0.75 ms at 5,000 against 1.32 ms at 125,000 canonical refresh and claim rebuild both 139,040,610 B, same sha256 The candidate document stays flat at 1103 B and 1107 B, which is the property this slice exists to establish. * 3.x.x: migrate the audit-body normalization tests to the slim candidate #267 landed its own test file against the candidate that still carried an assembled bundle. This slice replaced that with a window reference, so the file keeps a `TestCandidate` pairing the stored candidate with the bundle a test still needs, as `tests/ledger_postgres.rs` does, and attaches the claim's parts through `with_bundle`. One case needed more than a shape change. The four negative cases hand the claim a window the ledger does not hold and require the durable-range proof to be what refuses it. A candidate now commits to a snapshot digest, so a reference taken from the unaltered snapshot would be refused earlier, at the reference check, and the proof under test would never run. `window_ref_for` therefore derives each altered window's reference from that window, leaving the proof as the only thing that can notice, which is what #267 wrote them to prove. * 3.x.x: pin the durable-range proof outside the settlement lock #267 moved the full-equality proof of a candidate's window out of the settlement transaction so the lock is not held for a window-sized read. This slice made that read paged, so it does not occupy a runtime thread either. Reconciling the two is the step that can quietly undo the first: keep this slice's `persist_audit_snapshot` through the rebase and the full read goes back under the lock. The landing stays correct, every other test still passes, and only the lock hold grows, so nothing in the suite can tell. Measured at 100,000 shares on #267's branch that was a 34% longer hold. The guard is deterministic and needs no large window. A second connection holds `SETTLEMENT_LOCK`, so the landing blocks on it once its proof has run. While it waits, one share's `ntime` is altered: full equality sees that, a count over the same anchored predicate cannot. The landing must then succeed, which it can only do if it had already read and compared the shares before taking the lock. Accepted history is immutable and a trigger enforces it, so the test lifts that trigger for the single statement, inside its own disposable schema, and puts it straight back. Verified by mutation: restoring the in-transaction range read fails this test with "the landing read share payloads under the settlement lock". * 3.x.x: serialise the coordinator test modules that share the ledger's locks The ledger's advisory locks are cluster-wide constants, not schema-scoped, so two coordinator test modules running in parallel contend for ORDER_LOCK even though each works in its own throwaway schema. Every module held a lock of its own, which serialised it against itself and left cross-module waits to `lock_timeout`, 5 s by default. #324's reconciliation tests hold ORDER_LOCK for up to about 1.5 s, and this slice adds three more modules that open a ledger. Once enough of them ran at once those waits stacked past the timeout: a full `--lib` run failed with "canceling statement due to lock timeout" in `candidate_lease_tests` and `window_switch_tests`, and passed on the next run. A suite that fails for a reason unrelated to what it tests is worse than a slow one. All seven modules now take one shared lock. The run goes from about 35 s to about 62 s, and three consecutive runs pass where the flake had reproduced. * 3.x.x: add migration 007 and refuse pre-007 candidate outbox rows Migration 007 mirrors 008 onto qbit_block_candidate_outbox: the same six window-reference columns, and a three-state CHECK whose body is identical to 008's apart from the table and constraint names. It also adds block_bytes, the nullable bytea the block moves into so the candidate document stays small, and a partial index on window_prior_balances_sha256 for the balance-snapshot GC. Nothing writes any of them yet; this is additive schema only. The runner applies 007 by membership, in its own block between 5 and 8, so it assumes neither 006 nor 008 has run. Before any DDL it refuses while the outbox still holds a pending row in a pre-007 shape: a native inline candidate, a NULL candidate, or storage_version other than 1 on the 2.x.x schemas that have that column, which is checked for first so the predicate does not error on a native schema. The refusal names every offending block_hash and the drain procedure, and the whole migration transaction rolls back, so a refused start leaves the schema untouched. The applied set is now [2, 3, 4, 5, 7, 8, 9], and the four exact migration assertions move with it rather than being weakened to a superset. Part of #265 Co-Authored-By: Claude Opus 5 * 3.x.x: retain the cluster fingerprint configure pinned or verified The candidate enqueue and save_job must fence against a fingerprint reset by re-reading qbit_prism_cluster.config_fingerprint FOR SHARE in their own transaction and refusing a write that is not this frontend's own. That needs the value configure settled at startup, so Ledger keeps it in a shared OnceLock, set only once the pin or the match has committed, and exposes it read-only through config_fingerprint(). configure keeps its signature and its mismatch refusal, and a refused call retains nothing. The rotation refusal in configure, and the writer fence itself, belong to the enqueue change and are not here. Part of #265 Co-Authored-By: Claude Opus 5 * 3.x.x: share the window reference, balance writer and page primitives WindowRef::from_snapshot is the synchronous reference builder both writers need: anchor_ms copied, the semantic balances digest, and for a non-empty snapshot the first and last share_seq, the length, and the share array's SHA-256 streamed through the existing DigestWriter, so the 233 to 260 MB serialized window at 400,000 shares is never built and the digest equals the one qbit_prism_audit_snapshots already stores. An empty snapshot gives the empty-window reference. Callers run it inside their own spawn_blocking. put_balance_snapshot is the one canonical writer of qbit_prism_balance_snapshots, on the caller's transaction: the sort by (order_key, recipient_id, p2mr_program_hex), the semantic digest that keys the row, and compact JSON of the sorted set, all off the runtime, then INSERT ... ON CONFLICT DO NOTHING. On a conflict it reads the stored bytes back and compares them with its own, because these rows are immutable and can only disagree through corruption. The enqueue and save_job and its repair all write through it. read_prior_balances and the current-balance read still keep their SQL order; the writer's sort defines the stored encoding, which is the order read_window(..., AsIssued) returns. probe_share_rows and read_range_paged are read_window's endpoint probe and page loop, lifted onto a caller's connection so the enqueue probe and the landing re-read share one implementation. The predicate, the ascending keyset order, the 4096-row page bound and the per-page count bound are unchanged, and read_window now calls them, with its signature and behaviour unchanged. The paged reader takes the caller's state and a per-page consumer, both of which travel to the blocking thread with the rows, so the landing re-read can compare each page with its slice of a vector it already holds without any whole-window work on the runtime; a dropped read still releases the accumulated state on a blocking thread. read_window_with_permit holds the caller's window_reads permit across the whole read. The permit is moved into a BlockingDrop declared before the read starts, so a cancellation releases it on a blocking thread after the page state queued its own release there, and never with the future; a return releases it inline, before the caller's rebuild. WindowError::TaskFailed separates a cancelled or panicking blocking hand-off from corruption. Every hand-off maps a JoinError to it instead of to Decode, so a cancellation is a retryable failure with its own alert and never takes the abandon path. Part of #265 Co-Authored-By: Claude Opus 5 * 3.x.x: derive the witness merkle leaves from an assembled block A claim rebuilds from stored inputs, and the witness leaves are the one builder input that scales with the block, so they are re-derived rather than stored: one 64-hex string per transaction beside the candidate would grow the document with the block. witness_merkle_leaves_from_block skips the 80-byte header, reads the compact-size transaction count, walks past the coinbase and returns double_sha256 of every remaining transaction's full serialized bytes. That is sound because assemble_submission appends the template's transactions verbatim after the coinbase, so the result equals witness_merkle_leaves_hex(&transactions_from_template(template)) for the block built from that template, with or without segwit transactions. The segwit-aware walk is strip_witness_transaction's own, factored into a helper that also reports where the transaction ends, which is what a block walker needs; strip_witness_transaction keeps its trailing-byte check and its output. A truncated block, trailing bytes, a zero count and a count that disagrees with the transactions present are all errors, never a partial list of leaves. Part of #265 Co-Authored-By: Claude Opus 5 * 3.x.x: version the audit builders and key the CC5 goldens by it A stored window reference reproduces its bundle only under the builder that first built it. The schema strings do not capture that, because the share shape chooses them rather than the builder logic, so qbit-prism gains AUDIT_BUILDER_VERSION, starting at 1. A candidate and a prepared job store it, and a rebuild refuses a reference at another version instead of rebuilding it with the current builder. The bump rule is on the constant: any change that alters canonical_audit_bundle_bytes for the same inputs bumps it in the same commit. The CC5 golden test now selects its expected digests from a table keyed by the constant, and fails when no entry exists, so a new output is recorded as a new entry beside the old one and review sees both. Version 1 holds the existing literals, unchanged. Part of #265 Co-Authored-By: Claude Opus 5 * 3.x.x: cover 007, the fingerprint and the window primitives on PostgreSQL Ten gated database cases, all listed in test/prism-gated-tests.txt. They go in a new file rather than the shared 2.x.x support module, which another open change appends to. Migration 007: a frozen 2.x.x schema carrying only terminal outbox rows upgrades with every row and field intact and the seven new columns NULL; each of the three pre-007 pending shapes is refused by name, with nothing from 007 applied and no version 7 row, and the same binary starts once the row is drained; a database at [2, 3, 4, 5, 8, 9] applies 007 and only 007, leaving every other version's applied_at untouched; the CHECK accepts exactly the three states across all 64 null combinations and rejects a short or uppercase digest, a zero or out-of-range count and a zero sequence; and the retention index is the partial one on window_prior_balances_sha256. config_fingerprint() returns what configure pinned or verified, is shared by clones,… * ci: shard Rust PRISM integration tests (#371) * v3.0.0 feat: prepare compact coordinator input adapters (#363) * 3.x.x: add migration 007 and refuse pre-007 candidate outbox rows Migration 007 mirrors 008 onto qbit_block_candidate_outbox: the same six window-reference columns, and a three-state CHECK whose body is identical to 008's apart from the table and constraint names. It also adds block_bytes, the nullable bytea the block moves into so the candidate document stays small, and a partial index on window_prior_balances_sha256 for the balance-snapshot GC. Nothing writes any of them yet; this is additive schema only. The runner applies 007 by membership, in its own block between 5 and 8, so it assumes neither 006 nor 008 has run. Before any DDL it refuses while the outbox still holds a pending row in a pre-007 shape: a native inline candidate, a NULL candidate, or storage_version other than 1 on the 2.x.x schemas that have that column, which is checked for first so the predicate does not error on a native schema. The refusal names every offending block_hash and the drain procedure, and the whole migration transaction rolls back, so a refused start leaves the schema untouched. The applied set is now [2, 3, 4, 5, 7, 8, 9], and the four exact migration assertions move with it rather than being weakened to a superset. Part of #265 Co-Authored-By: Claude Opus 5 * 3.x.x: retain the cluster fingerprint configure pinned or verified The candidate enqueue and save_job must fence against a fingerprint reset by re-reading qbit_prism_cluster.config_fingerprint FOR SHARE in their own transaction and refusing a write that is not this frontend's own. That needs the value configure settled at startup, so Ledger keeps it in a shared OnceLock, set only once the pin or the match has committed, and exposes it read-only through config_fingerprint(). configure keeps its signature and its mismatch refusal, and a refused call retains nothing. The rotation refusal in configure, and the writer fence itself, belong to the enqueue change and are not here. Part of #265 Co-Authored-By: Claude Opus 5 * 3.x.x: share the window reference, balance writer and page primitives WindowRef::from_snapshot is the synchronous reference builder both writers need: anchor_ms copied, the semantic balances digest, and for a non-empty snapshot the first and last share_seq, the length, and the share array's SHA-256 streamed through the existing DigestWriter, so the 233 to 260 MB serialized window at 400,000 shares is never built and the digest equals the one qbit_prism_audit_snapshots already stores. An empty snapshot gives the empty-window reference. Callers run it inside their own spawn_blocking. put_balance_snapshot is the one canonical writer of qbit_prism_balance_snapshots, on the caller's transaction: the sort by (order_key, recipient_id, p2mr_program_hex), the semantic digest that keys the row, and compact JSON of the sorted set, all off the runtime, then INSERT ... ON CONFLICT DO NOTHING. On a conflict it reads the stored bytes back and compares them with its own, because these rows are immutable and can only disagree through corruption. The enqueue and save_job and its repair all write through it. read_prior_balances and the current-balance read still keep their SQL order; the writer's sort defines the stored encoding, which is the order read_window(..., AsIssued) returns. probe_share_rows and read_range_paged are read_window's endpoint probe and page loop, lifted onto a caller's connection so the enqueue probe and the landing re-read share one implementation. The predicate, the ascending keyset order, the 4096-row page bound and the per-page count bound are unchanged, and read_window now calls them, with its signature and behaviour unchanged. The paged reader takes the caller's state and a per-page consumer, both of which travel to the blocking thread with the rows, so the landing re-read can compare each page with its slice of a vector it already holds without any whole-window work on the runtime; a dropped read still releases the accumulated state on a blocking thread. read_window_with_permit holds the caller's window_reads permit across the whole read. The permit is moved into a BlockingDrop declared before the read starts, so a cancellation releases it on a blocking thread after the page state queued its own release there, and never with the future; a return releases it inline, before the caller's rebuild. WindowError::TaskFailed separates a cancelled or panicking blocking hand-off from corruption. Every hand-off maps a JoinError to it instead of to Decode, so a cancellation is a retryable failure with its own alert and never takes the abandon path. Part of #265 Co-Authored-By: Claude Opus 5 * 3.x.x: derive the witness merkle leaves from an assembled block A claim rebuilds from stored inputs, and the witness leaves are the one builder input that scales with the block, so they are re-derived rather than stored: one 64-hex string per transaction beside the candidate would grow the document with the block. witness_merkle_leaves_from_block skips the 80-byte header, reads the compact-size transaction count, walks past the coinbase and returns double_sha256 of every remaining transaction's full serialized bytes. That is sound because assemble_submission appends the template's transactions verbatim after the coinbase, so the result equals witness_merkle_leaves_hex(&transactions_from_template(template)) for the block built from that template, with or without segwit transactions. The segwit-aware walk is strip_witness_transaction's own, factored into a helper that also reports where the transaction ends, which is what a block walker needs; strip_witness_transaction keeps its trailing-byte check and its output. A truncated block, trailing bytes, a zero count and a count that disagrees with the transactions present are all errors, never a partial list of leaves. Part of #265 Co-Authored-By: Claude Opus 5 * 3.x.x: version the audit builders and key the CC5 goldens by it A stored window reference reproduces its bundle only under the builder that first built it. The schema strings do not capture that, because the share shape chooses them rather than the builder logic, so qbit-prism gains AUDIT_BUILDER_VERSION, starting at 1. A candidate and a prepared job store it, and a rebuild refuses a reference at another version instead of rebuilding it with the current builder. The bump rule is on the constant: any change that alters canonical_audit_bundle_bytes for the same inputs bumps it in the same commit. The CC5 golden test now selects its expected digests from a table keyed by the constant, and fails when no entry exists, so a new output is recorded as a new entry beside the old one and review sees both. Version 1 holds the existing literals, unchanged. Part of #265 Co-Authored-By: Claude Opus 5 * 3.x.x: cover 007, the fingerprint and the window primitives on PostgreSQL Ten gated database cases, all listed in test/prism-gated-tests.txt. They go in a new file rather than the shared 2.x.x support module, which another open change appends to. Migration 007: a frozen 2.x.x schema carrying only terminal outbox rows upgrades with every row and field intact and the seven new columns NULL; each of the three pre-007 pending shapes is refused by name, with nothing from 007 applied and no version 7 row, and the same binary starts once the row is drained; a database at [2, 3, 4, 5, 8, 9] applies 007 and only 007, leaving every other version's applied_at untouched; the CHECK accepts exactly the three states across all 64 null combinations and rejects a short or uppercase digest, a zero or out-of-range count and a zero sequence; and the retention index is the partial one on window_prior_balances_sha256. config_fingerprint() returns what configure pinned or verified, is shared by clones, and stays empty after a refused configure. from_snapshot's streamed digest equals sha256(serde_json::to_vec) and the snapshot_sha256 landing writes for the same window. The balance writer stores the canonical sort, is idempotent under any permutation, errors rather than accepting a different encoding of the same digest, and read_window(..., AsIssued) hands back the stored order. The probe misses a missing endpoint, the paged reader compares 4,999 rows page by page with a vector held inside the caller's transaction, and a row the predicate skips inside a range is Incomplete with no partial window. The permit case builds a runtime with one blocking thread, occupies it, and cancels a reader parked mid-page on a database gate: no permit is available while the cleanup is queued, and one is once it has run. A failed blocking hand-off surfaces as TaskFailed, never as Decode. Part of #265 Co-Authored-By: Claude Opus 5 * 3.x.x: carry the window reference on prepared work, behind a read permit `Prepared` gains `window: WindowRef`, computed once per non-cached refresh beside `build_bundle` under `tokio::join!`, so submit can clone it instead of re-digesting the window on the share path when a block is found. An empty snapshot gets the empty-window reference on the runtime thread; a resume computes it inside the blocking decode it already runs. `Coordinator` gains the `window_reads` semaphore, sized `clamp(database_max_connections - 2, 1, build_workers)`, and the one entry point callers use: it takes an owned permit after the caller's `build_slots` permit and hands it to `Ledger::read_window_with_permit`, which holds it across the whole read, including the off-runtime cleanup a cancellation leaves behind, and releases it before the rebuild. `Ledger::read_window` is awaited on the runtime, never wrapped in `spawn_blocking`; the test with a single blocking thread pins that. The four gated `window_ref_tests` are listed in the manifest. Part of #265 Co-Authored-By: Claude Fable 5.1 * 3.x.x: switch the candidate outbox to the window reference `Candidate` drops the embedded `AuditBundle` for `window: WindowRef` and every builder input except the window: `found_block`, `payout_policy`, the nested `ctv` triple (whose presence selects the CTV builder), the stored `bootstrap_share` of an empty window, `audit_builder_version`, `signer_keys`, a `leased` flag and the now-required `coinbase_suffix_hex`. The block leaves the document for the `block_bytes` column, authenticated by `block_sha256`, which `candidate_sha256` covers. The write serializes, digests and checks the candidate before the append or enqueue transaction opens; under `ORDER_LOCK` the transaction re-reads `config_fingerprint FOR SHARE` and refuses a value the writer did not pin, re-inserts a `leased` candidate's as-issued balance snapshot through the shared writer, probes the window's first row (an absent row alerts and still publishes), and inserts only the prepared bytes with the six typed columns. Submit builds the slim candidate from `Prepared.window`, the `BundleInputs` captured where `build_bundle` read configuration, and the job's own bootstrap share; it clones no bundle and reads no config. The claim's `UPDATE … RETURNING` returns the columns and the block, and the decode, off the runtime, checks the document against them, the digest, `sha256(block_bytes)`, the header hash, the required fields and the two reference invariants. A pending row with a NULL anchor is a hard error naming the pre-007 frontend; an inline document is never decoded. Before any read the claim retries, with an alert naming both values, a stored builder version or signer pair that is not this binary's; a landed audit is authenticated against the block (coinbase, audit root as the witness reserved value, share snapshot, bits) and the claim then skips the rebuild and landing and continues to observe, renew and submit. Otherwise it takes `build_slots`, then `window_reads` inside one 60 s deadline around the awaited `read_window(Current)` and the borrowing builder run directly in `spawn_blocking`, and keeps `Arc` and the shares as `ClaimParts`. `Incomplete`, `SnapshotDigestMismatch`, `Decode`, `BalanceSnapshotMissing`, `TaskFailed` and the deadline retry with their own alerts; `PriorBalancesChanged` finishes only a block `observe_candidate` proves inactive and superseded; `Database` propagates. A `leased` candidate submits from `block_bytes` first, then rebuilds `AsIssued` and lands before any terminal outcome. Rebuilt parts are released off the runtime on every path. Landing takes the parts and does all whole-window work in one blocking closure: `verify_audit_parts`, the raw body, the counted canonical bytes, the leaves and accounts, and the share snapshot digest, which must equal the reference's. The balance fence compares the current set and the parts' through the semantic digest, so an as-issued set compares equal in any order. `persist_audit_snapshot` re-reads the range with the paged reader inside the landing transaction, page against slice on the blocking thread; `read_range` stays for the API. The terminal UPDATE NULLs the six window columns and `block_bytes` with `candidate`. `configure` refuses to pin a fingerprint onto a reset one while a pending row stores other signer keys, and the startup gate requires 007 with 009. The JSONB gate drops its enqueue entry, and every helper that built a candidate builds the slim one, handing the ledger-level landings the bundle's parts through `CandidateClaim::with_bundle`. Part of #265 Co-Authored-By: Claude Fable 5.1 * 3.x.x: cover the candidate switch on PostgreSQL Ledger-level, in `candidate_window_switch`: a two-page window enqueues a document with no array and no share row, under 1 MB, whose six typed columns equal its reference and whose block is stored as bytes; the writer fence refuses a reset or different fingerprint in the same transaction, share included; a missing window prefix row alerts and still publishes, and its read then meets `Incomplete`; a `leased` enqueue re-inserts the as-issued balance snapshot in the writer's canonical order, writes none when not leased, and refuses balances that do not hash to the reference; every preparation failure is refused without waiting for `ORDER_LOCK`; every column, digest, block, header, NULL-anchor, missing-field, invariant and inline-document disagreement is refused at claim; landing refuses parts that are not the referenced window and a claim without parts; the terminal UPDATE NULLs the six columns and the block for submitted and abandoned rows; the landed-audit authentication rejects a forged coinbase, root, snapshot or bits; and `configure` refuses a rotation over a foreign-signed pending candidate. Coordinator-level, in `window_switch_tests` behind a fake node: a stored builder version or signer pair that is not this binary's retries, naming both values, while the build slot is held; a landed audit is authenticated and the claim continues to `submitblock` without a build slot, and a forged landed coinbase or root is refused with the row left pending; `Incomplete`, a digest mismatch, changed balances on a current block and the rebuild deadline all retry with an alert and release their permits; `classify_window_error` keeps `TaskFailed` apart from corruption and propagates `Database`; the read, rebuild and landing complete while `ORDER_LOCK` is held, with one build worker; a `leased` candidate submits first, then lands from `AsIssued`, and finishes submitted, abandoned with the node's reason, or pending when its balance snapshot is missing; and `OffRuntime` releases on a blocking thread, on cancellation too. The fifteen gated tests are listed in the manifest. Part of #265 Co-Authored-By: Claude Fable 5.1 * 3.x.x: migrate the imported-audit tests to the slim candidate Slice 1 (#325) merged after this stack branched, and its tests in tests/support/ledger_2x.rs still built the pre-007 candidate: they read block.bundle from a Candidate and called candidate_with_bundle without a window reference or settlement inputs. The rebase merged them cleanly and the tree then failed to compile. The three helpers now take the TestCandidate the constructors return, so they reach the bundle beside the slim candidate, and land_confirmed attaches the claim's parts the way the other ledger tests do. The three constructor calls pass the window reference the bundle was built on, and the settlement inputs for the CTV case. No assertion changes. Part of #265. * 3.x.x: qualify lease renewal and the claim's canonical bytes at scale Incident 1: a candidate whose window holds N shares is claimed and processed under the short lease, and every renewal is read back from claim_expires_at. The largest gap stays below interval + 2 x timeout, the lease never lapses, and a successor ledger cannot take the claim. The bundle a claim rebuilds through the window-read entry point and the borrowing builder has the canonical bytes refresh built for the same job, and the landed audit carries that bundle's digest. Both run at PRISM_WINDOW_QUALIFY_SHARES (20,000 by default); the 400,000-share variants are ignored and run on a larger host. Part of #265 Co-Authored-By: Claude Opus 5 * 3.x.x: measure ACK latency and ORDER_LOCK hold across window sizes Incident 2: two frontends on one database submit ordinary shares at a fixed rate while one of them solves a block on an N-share window and its own submit loop claims, rebuilds, lands and submits it. The solving share's ACK and the bystander's p99 stay within max(2 x baseline p99 + 50 ms, 250 ms), every ACK is well under the share-commit timeout, and the block is enqueued exactly once. Latency is the in-process MiningBackend::submit wall clock; the socket-level harness (#271) does not exist yet. A pg_locks sampler on its own connection measures how long a solving append holds ORDER_LOCK at two window sizes with no other traffic: the larger size stays within a fixed slack of the smaller, and the candidate document is the same size at both and under 1 MiB. Part of #265 Co-Authored-By: Claude Opus 5 * 3.x.x: hold the JSONB gate's enqueue to a window-free row under 1 MiB At every measured size the enqueue must be accepted and every JSONB value it writes, the outbox candidate document included, must stay under 1,048,576 bytes. At the reduced sizes the enqueue, claim and landing phases must all run. The ratchet and its expected values are unchanged. Part of #265 Co-Authored-By: Claude Opus 5 * 3.x.x: make the window-read permit test actually wait before it looks The delay that gives an unpermitted read time to finish was an empty async block, so the check that the read had not finished ran at once and passed whether or not the entry point took a permit. It now sleeps 500 ms inside a 5 s bound, and fails when the permit acquisition is removed from the entry point. Part of #265 Co-Authored-By: Claude Opus 5 * 3.x.x: rebuild a CTV candidate from its stored settlement inputs No host-runnable test reached the CTV branch of the claim rebuild. A CTV-enabled frontend now claims a candidate whose stored direct floor differs from its configuration, and builds a different coinbase than the configured floor would, and the claim must land that candidate's own audit and submit it. Part of #265 Co-Authored-By: Claude Opus 5 * 3.x.x: pin the SQL order of the current-balance reads Nothing defended the ruling that the current-balance path keeps its SQL order: every fixture's balances already arrived in byte-wise canonical order. The carry table's text columns now take an ICU collation, under which the SQL order and the canonical sort disagree, and the snapshot and the current-balance window read must both return the SQL order. Part of #265 Co-Authored-By: Claude Opus 5 * 3.x.x: record what incident 2 must also assert once #324 is in the base After #324 a slow solve is acknowledged late rather than refused, so acceptance proves nothing and the latency percentiles stay the assertion. The doc comment and a TODO at the outcome checks name the counter and outcome to assert at rebase. Part of #265 Co-Authored-By: Claude Opus 5 * 3.x.x: assert incident 2's late-confirm counter now that #324 is in the base W3c wrote the qualification tests against a base that predated #324 and left a marked place for the assertions that only make sense once the commit gate is there. The base now carries it, so apply them: incident 2 asserts `qbit_prism_late_confirmed_shares_total` is zero on both frontends and that no acknowledgement, the block-solving share included, is `ledger-outcome-unknown`. Acceptance alone is no longer proof. #324 gives a share-pass append carrying a candidate its own `block_only_ack_timeout`, so a slow solve is never refused at the share deadline; it is answered late, or answered unknown. Either would have satisfied the acceptance checks with the serialization stall still present. Also set #324's two new `Config` fields in the qualification fixture. * 3.x.x: pin a refused candidate as a definite rejection, not an unknown outcome #324 decides what a miner is told when a block-only enqueue fails: `enqueue_failed_before_commit` answers a provably uncommitted failure with `ledger-confirmation-failed`, and anything else with `ledger-outcome-unknown`, which asks an operator to go and follow the outbox row. Slice 3 added the checks that guard the reference: the block's length, its header hash, its digest against `block_sha256`, the coinbase suffix, the reference invariants, and a leased candidate's balances. All of them are raised by `prepare_candidate` before the enqueue takes a connection, so none carries an `sqlx::Error` and all classify as definite today. That is a property of where the checks sit, not something the classifier knows about them, so it is pinned here rather than left to hold by accident: a corrupt candidate must never send a miner chasing a row that was never written. Verified by mutation: classifying a non-sqlx failure as unknown fails this test. * 3.x.x: say why the full-size qualification variants cannot reach 400k yet Their ignore reasons blamed RAM and the host. The real limit is #273: the job payload still holds three copies of the window (2 x AcceptedShare + 1 x CountedShare), so publishing a job for a 400,000-share window needs about 670 MiB and PostgreSQL refuses the write above its 256 MiB jsonb object limit. Every full-size variant fails at fixture set-up, before it measures anything. The count must also divide the window weight exactly, so the largest usable size today is 125,000, where all four variants pass in release: incident 1 50 renewals, max gap 118.25 ms against a 700 ms bound incident 2 phase-A p99 2.60 ms, phase-B p99 95.79 ms, bound 250 ms lock hold median 0.75 ms at 5,000 against 1.32 ms at 125,000 canonical refresh and claim rebuild both 139,040,610 B, same sha256 The candidate document stays flat at 1103 B and 1107 B, which is the property this slice exists to establish. * 3.x.x: migrate the audit-body normalization tests to the slim candidate #267 landed its own test file against the candidate that still carried an assembled bundle. This slice replaced that with a window reference, so the file keeps a `TestCandidate` pairing the stored candidate with the bundle a test still needs, as `tests/ledger_postgres.rs` does, and attaches the claim's parts through `with_bundle`. One case needed more than a shape change. The four negative cases hand the claim a window the ledger does not hold and require the durable-range proof to be what refuses it. A candidate now commits to a snapshot digest, so a reference taken from the unaltered snapshot would be refused earlier, at the reference check, and the proof under test would never run. `window_ref_for` therefore derives each altered window's reference from that window, leaving the proof as the only thing that can notice, which is what #267 wrote them to prove. * 3.x.x: pin the durable-range proof outside the settlement lock #267 moved the full-equality proof of a candidate's window out of the settlement transaction so the lock is not held for a window-sized read. This slice made that read paged, so it does not occupy a runtime thread either. Reconciling the two is the step that can quietly undo the first: keep this slice's `persist_audit_snapshot` through the rebase and the full read goes back under the lock. The landing stays correct, every other test still passes, and only the lock hold grows, so nothing in the suite can tell. Measured at 100,000 shares on #267's branch that was a 34% longer hold. The guard is deterministic and needs no large window. A second connection holds `SETTLEMENT_LOCK`, so the landing blocks on it once its proof has run. While it waits, one share's `ntime` is altered: full equality sees that, a count over the same anchored predicate cannot. The landing must then succeed, which it can only do if it had already read and compared the shares before taking the lock. Accepted history is immutable and a trigger enforces it, so the test lifts that trigger for the single statement, inside its own disposable schema, and puts it straight back. Verified by mutation: restoring the in-transaction range read fails this test with "the landing read share payloads under the settlement lock". * 3.x.x: serialise the coordinator test modules that share the ledger's locks The ledger's advisory locks are cluster-wide constants, not schema-scoped, so two coordinator test modules running in parallel contend for ORDER_LOCK even though each works in its own throwaway schema. Every module held a lock of its own, which serialised it against itself and left cross-module waits to `lock_timeout`, 5 s by default. #324's reconciliation tests hold ORDER_LOCK for up to about 1.5 s, and this slice adds three more modules that open a ledger. Once enough of them ran at once those waits stacked past the timeout: a full `--lib` run failed with "canceling statement due to lock timeout" in `candidate_lease_tests` and `window_switch_tests`, and passed on the next run. A suite that fails for a reason unrelated to what it tests is worse than a slow one. All seven modules now take one shared lock. The run goes from about 35 s to about 62 s, and three consecutive runs pass where the flake had reproduced. * 3.x.x: add migration 007 and refuse pre-007 candidate outbox rows Migration 007 mirrors 008 onto qbit_block_candidate_outbox: the same six window-reference columns, and a three-state CHECK whose body is identical to 008's apart from the table and constraint names. It also adds block_bytes, the nullable bytea the block moves into so the candidate document stays small, and a partial index on window_prior_balances_sha256 for the balance-snapshot GC. Nothing writes any of them yet; this is additive schema only. The runner applies 007 by membership, in its own block between 5 and 8, so it assumes neither 006 nor 008 has run. Before any DDL it refuses while the outbox still holds a pending row in a pre-007 shape: a native inline candidate, a NULL candidate, or storage_version other than 1 on the 2.x.x schemas that have that column, which is checked for first so the predicate does not error on a native schema. The refusal names every offending block_hash and the drain procedure, and the whole migration transaction rolls back, so a refused start leaves the schema untouched. The applied set is now [2, 3, 4, 5, 7, 8, 9], and the four exact migration assertions move with it rather than being weakened to a superset. Part of #265 Co-Authored-By: Claude Opus 5 * 3.x.x: retain the cluster fingerprint configure pinned or verified The candidate enqueue and save_job must fence against a fingerprint reset by re-reading qbit_prism_cluster.config_fingerprint FOR SHARE in their own transaction and refusing a write that is not this frontend's own. That needs the value configure settled at startup, so Ledger keeps it in a shared OnceLock, set only once the pin or the match has committed, and exposes it read-only through config_fingerprint(). configure keeps its signature and its mismatch refusal, and a refused call retains nothing. The rotation refusal in configure, and the writer fence itself, belong to the enqueue change and are not here. Part of #265 Co-Authored-By: Claude Opus 5 * 3.x.x: share the window reference, balance writer and page primitives WindowRef::from_snapshot is the synchronous reference builder both writers need: anchor_ms copied, the semantic balances digest, and for a non-empty snapshot the first and last share_seq, the length, and the share array's SHA-256 streamed through the existing DigestWriter, so the 233 to 260 MB serialized window at 400,000 shares is never built and the digest equals the one qbit_prism_audit_snapshots already stores. An empty snapshot gives the empty-window reference. Callers run it inside their own spawn_blocking. put_balance_snapshot is the one canonical writer of qbit_prism_balance_snapshots, on the caller's transaction: the sort by (order_key, recipient_id, p2mr_program_hex), the semantic digest that keys the row, and compact JSON of the sorted set, all off the runtime, then INSERT ... ON CONFLICT DO NOTHING. On a conflict it reads the stored bytes back and compares them with its own, because these rows are immutable and can only disagree through corruption. The enqueue and save_job and its repair all write through it. read_prior_balances and the current-balance read still keep their SQL order; the writer's sort defines the stored encoding, which is the order read_window(..., AsIssued) returns. probe_share_rows and read_range_paged are read_window's endpoint probe and page loop, lifted onto a caller's connection so the enqueue probe and the landing re-read share one implementation. The predicate, the ascending keyset order, the 4096-row page bound and the per-page count bound are unchanged, and read_window now calls them, with its signature and behaviour unchanged. The paged reader takes the caller's state and a per-page consumer, both of which travel to the blocking thread with the rows, so the landing re-read can compare each page with its slice of a vector it already holds without any whole-window work on the runtime; a dropped read still releases the accumulated state on a blocking thread. read_window_with_permit holds the caller's window_reads permit across the whole read. The permit is moved into a BlockingDrop declared before the read starts, so a cancellation releases it on a blocking thread after the page state queued its own release there, and never with the future; a return releases it inline, before the caller's rebuild. WindowError::TaskFailed separates a cancelled or panicking blocking hand-off from corruption. Every hand-off maps a JoinError to it instead of to Decode, so a cancellation is a retryable failure with its own alert and never takes the abandon path. Part of #265 Co-Authored-By: Claude Opus 5 * 3.x.x: derive the witness merkle leaves from an assembled block A claim rebuilds from stored inputs, and the witness leaves are the one builder input that scales with the block, so they are re-derived rather than stored: one 64-hex string per transaction beside the candidate would grow the document with the block. witness_merkle_leaves_from_block skips the 80-byte header, reads the compact-size transaction count, walks past the coinbase and returns double_sha256 of every remaining transaction's full serialized bytes. That is sound because assemble_submission appends the template's transactions verbatim after the coinbase, so the result equals witness_merkle_leaves_hex(&transactions_from_template(template)) for the block built from that template, with or without segwit transactions. The segwit-aware walk is strip_witness_transaction's own, factored into a helper that also reports where the transaction ends, which is what a block walker needs; strip_witness_transaction keeps its trailing-byte check and its output. A truncated block, trailing bytes, a zero count and a count that disagrees with the transactions present are all errors, never a partial list of leaves. Part of #265 Co-Authored-By: Claude Opus 5 * 3.x.x: version the audit builders and key the CC5 goldens by it A stored window reference reproduces its bundle only under the builder that first built it. The schema strings do not capture that, because the share shape chooses them rather than the builder logic, so qbit-prism gains AUDIT_BUILDER_VERSION, starting at 1. A candidate and a prepared job store it, and a rebuild refuses a reference at another version instead of rebuilding it with the current builder. The bump rule is on the constant: any change that alters canonical_audit_bundle_bytes for the same inputs bumps it in the same commit. The CC5 golden test now selects its expected digests from a table keyed by the constant, and fails when no entry exists, so a new output is recorded as a new entry beside the old one and review sees both. Version 1 holds the existing literals, unchanged. Part of #265 Co-Authored-By: Claude Opus 5 * 3.x.x: cover 007, the fingerprint and the window primitives on PostgreSQL Ten gated database cases, all listed in test/prism-gated-tests.txt. They go in a new file rather than the shared 2.x.x support module, which another open change appends to. Migration 007: a frozen 2.x.x schema carrying only terminal outbox rows upgrades with every row and field intact and the seven new columns NULL; each of the three pre-007 pending shapes is refused by name, with nothing from 007 applied and no version 7 row, and the same binary starts once the row is drained; a database at [2, 3, 4, 5, 8, 9] applies 007 and only 007, leaving every other version's applied_at untouched; the CHECK accepts exactly the three states across all 64 null combinations and rejects a short or uppercase digest, a zero or out-of-range count and a zero sequence; and the retention index is the partial one on window_prior_balances_sha256. config_fingerprint() returns what configure pinned or verified, is shared by clones, and stays empty after a refused configure. from_snapshot's streamed digest equals sha256(serde_json::to_vec) and the snapshot_sha256 landing writes for the same window. The balance writer stores the canonical sort, is idempotent under any permutation, errors rather than accepting a different encoding of the same digest, and read_window(..., AsIssued) hands back the stored order. The probe misses a missing endpoint, the paged reader compares 4,999 rows page by page with a vector held inside the caller's transaction, and a row the predicate skips inside a range is Incomplete with no partial window. The permit case builds a runtime with one blocking thread, occupies it, and cancels a reader parked mid-page on a database gate: no permit is available while the cleanup is queued, and one is once it has run. A failed blocking hand-off surfaces as TaskFailed, never as Decode. Part of #265 Co-Authored-By: Claude Opus 5 * 3.x.x: carry the window reference on prepared work, behind a read permit `Prepared` gains `window: WindowRef`, computed once per non-cached refresh beside `build_bundle` under `tokio::join!`, so submit can clone it instead of re-digesting the window on the share path when a block is found. An empty snapshot gets the empty-window reference on the runtime thread; a resume computes it inside the blocking decode it already runs. `Coordinator` gains the `window_reads` semaphore, sized `clamp(database_max_connections - 2, 1, build_workers)`, and the one entry point callers use: it takes an owned permit after the caller's `build_slots` permit and hands it to `Ledger::read_window_with_permit`, which holds it across the whole read, including the off-runtime cleanup a cancellation leaves behind, and releases it before the rebuild. `Ledger::read_window` is awaited on the runtime, never wrapped in `spawn_blocking`; the test with a single blocking thread pins that. The four gated `window_ref_tests` are listed in the manifest. Part of #265 Co-Authored-By: Claude Fable 5.1 * 3.x.x: switch the candidate outbox to the window reference `Candidate` drops the embedded `AuditBundle` for `window: WindowRef` and every builder input except the window: `found_block`, `payout_policy`, the nested `ctv` triple (whose presence selects the CTV builder), the stored `bootstrap_share` of an empty window, `audit_builder_version`, `signer_keys`, a `leased` flag and the now-required `coinbase_suffix_hex`. The block leaves the document for the `block_bytes` column, authenticated by `block_sha256`, which `candidate_sha256` covers. The write serializes, digests and checks the candidate before the append or enqueue transaction opens; under `ORDER_LOCK` the transaction re-reads `config_fingerprint FOR SHARE` and refuses a value the writer did not pin, re-inserts a `leased` candidate's as-issued balance snapshot through the shared writer, probes the window's first row (an absent row alerts and still publishes), and inserts only the prepared bytes with the six typed columns. Submit builds the slim candidate from `Prepared.window`, the `BundleInputs` captured where `build_bundle` read configuration, and the job's own bootstrap share; it clones no bundle and reads no config. The claim's `UPDATE … RETURNING` returns the columns and the block, and the decode, off the runtime, checks the document against them, the digest, `sha256(block_bytes)`, the header hash, the required fields and the two reference invariants. A pending row with a NULL anchor is a hard error naming the pre-007 frontend; an inline document is never decoded. Before any read the claim retries, with an alert naming both values, a stored builder version or signer pair that is not this binary's; a landed audit is authenticated against the block (coinbase, audit root as the witness reserved value, share snapshot, bits) and the claim then skips the rebuild and landing and continues to observe, renew and submit. Otherwise it takes `build_slots`, then `window_reads` inside one 60 s deadline around the awaited `read_window(Current)` and the borrowing builder run directly in `spawn_blocking`, and keeps `Arc` and the shares as `ClaimParts`. `Incomplete`, `SnapshotDigestMismatch`, `Decode`, `BalanceSnapshotMissing`, `TaskFailed` and the deadline retry with their own alerts; `PriorBalancesChanged` finishes only a block `observe_candidate` proves inactive and superseded; `Database` propagates. A `leased` candidate submits from `block_bytes` first, then rebuilds `AsIssued` and lands before any terminal outcome. Rebuilt parts are released off the runtime on every path. Landing takes the parts and does all whole-window work in one blocking closure: `verify_audit_parts`, the raw body, the counted canonical bytes, the leaves and accounts, and the share snapshot digest, which must equal the reference's. The balance fence compares the current set and the parts' through the semantic digest, so an as-issued set compares equal in any order. `persist_audit_snapshot` re-reads the range with the paged reader inside the landing transaction, page against slice on the blocking thread; `read_range` stays for the API. The terminal UPDATE NULLs the six window columns and `block_bytes` with `candidate`. `configure` refuses to pin a fingerprint onto a reset one while a pending row stores other signer keys, and the startup gate requires 007 with 009. The JSONB gate drops its enqueue entry, and every helper that built a candidate builds the slim one, handing the ledger-level landings the bundle's parts through `CandidateClaim::with_bundle`. Part of #265 Co-Authored-By: Claude Fable 5.1 * 3.x.x: cover the candidate switch on PostgreSQL Ledger-level, in `candidate_window_switch`: a two-page window enqueues a document with no array and no share row, under 1 MB, whose six typed columns equal its reference and whose block is stored as bytes; the writer fence refuses a reset or different fingerprint in the same transaction, share included; a missing window prefix row alerts and still publishes, and its read then meets `Incomplete`; a `leased` enqueue re-inserts the as-issued balance snapshot in the writer's canonical order, writes none when not leased, and refuses balances that do not hash to the reference; every preparation failure is refused without waiting for `ORDER_LOCK`; every column, digest, block, header, NULL-anchor, missing-field, invariant and inline-document disagreement is refused at claim; landing refuses parts that are not the referenced window and a claim without parts; the terminal UPDATE NULLs the six columns and the block for submitted and abandoned rows; the landed-audit authentication rejects a forged coinbase, root, snapshot or bits; and `configure` refuses a rotation over a foreign-signed pending candidate. Coordinator-level, in `window_switch_tests` behind a fake node: a stored builder version or signer pair that is not this binary's retries, naming both values, while the build slot is held; a landed audit is authenticated and the claim continues to `submitblock` without a build slot, and a forged landed coinbase or root is refused with the row left pending; `Incomplete`, a digest mismatch, changed balances on a current block and the rebuild deadline all retry with an alert and release their permits; `classify_window_error` keeps `TaskFailed` apart from corruption and propagates `Database`; the read, rebuild and landing complete while `ORDER_LOCK` is held, with one build worker; a `leased` candidate submits first, then lands from `AsIssued`, and finishes submitted, abandoned with the node's reason, or pending when its balance snapshot is missing; and `OffRuntime` releases on a blocking thread, on cancellation too. The fifteen gated tests are listed in the manifest. Part of #265 Co-Authored-By: Claude Fable 5.1 * 3.x.x: migrate the imported-audit tests to the slim candidate Slice 1 (#325) merged after this stack branched, and its tests in tests/support/ledger_2x.rs still built the pre-007 candidate: they read block.bundle from a Candidate and called candidate_with_bundle without a window reference or settlement inputs. The rebase merged them cleanly and the tree then failed to compile. The three helpers now take the TestCandidate the constructors return, so they reach the bundle beside the slim candidate, and land_confirmed attaches the claim's parts the way the other ledger tests do. The three constructor calls pass the window reference the bundle was built on, and the settlement inputs for the CTV case. No assertion changes. Part of #265. * 3.x.x: qualify lease renewal and the claim's canonical bytes at scale Incident 1: a candidate whose window holds N shares is claimed and processed under the short lease, and every renewal is read back from claim_expires_at. The largest gap stays below interval + 2 x timeout, the lease never lapses, and a successor ledger cannot take the claim. The bundle a claim rebuilds through the window-read entry point and the borrowing builder has the canonical bytes refresh built for the same job, and the landed audit carries that bundle's digest. Both run at PRISM_WINDOW_QUALIFY_SHARES (20,000 by default); the 400,000-share variants are ignored and run on a larger host. Part of #265 Co-Authored-By: Claude Opus 5 * 3.x.x: measure ACK latency and ORDER_LOCK hold across window sizes Incident 2: two frontends on one database submit ordinary shares at a fixed rate while one of them solves a block on an N-share window and its own submit loop claims, rebuilds, lands and submits it. The solving share's ACK and the bystander's p99 stay within max(2 x baseline p99 + 50 ms, 250 ms), every ACK is well under the share-commit timeout, and the block is enqueued exactly once. Latency is the in-process MiningBackend::submit wall clock; the socket-level harness (#271) does not exist yet. A pg_locks sampler on its own connection measures how long a solving append holds ORDER_LOCK at two window sizes with no other traffic: the larger size stays within a fixed slack of the smaller, and the candidate document is the same size at both and under 1 MiB. Part of #265 Co-Authored-By: Claude Opus 5 * 3.x.x: hold the JSONB gate's enqueue to a window-free row under 1 MiB At every measured size the enqueue must be accepted and every JSONB value it writes, the outbox candidate document included, must stay under 1,048,576 bytes. At the reduced sizes the enqueue, claim and landing phases must all run. The ratchet and its expected values are unchanged. Part of #265 Co-Authored-By: Claude Opus 5 * 3.x.x: make the window-read permit test actually wait before it looks The delay that gives an unpermitted read time to finish was an empty async block, so the check that the read had not finished ran at once and passed whether or not the entry point took a permit. It now sleeps 500 ms inside a 5 s bound, and fails when the permit acquisition is removed from the entry point. Part of #265 Co-Authored-By: Claude Opus 5 * 3.x.x: rebuild a CTV candidate from its stored settlement inputs No host-runnable test reached the CTV branch of the claim rebuild. A CTV-enabled frontend now claims a candidate whose stored direct floor differs from its configuration, and builds a different coinbase than the configured floor would, and the claim must land that candidate's own audit and submit it. Part of #265 Co-Authored-By: Claude Opus 5 * 3.x.x: pin the SQL order of the current-balance reads Nothing defended the ruling that the current-balance path keeps its SQL order: every fixture's balances already arrived in byte-wise canonical order. The carry table's text columns now take an ICU collation, under which the SQL order and the canonical sort disagree, and the snapshot and the current-balance window read must both return the SQL order. Part of #265 Co-Authored-By: Claude Opus 5 * 3.x.x: record what incident 2 must also assert once #324 is in the base After #324 a slow solve is acknowledged late rather than refused, so acceptance proves nothing and the latency percentiles stay the assertion. The doc comment and a TODO at the outcome checks name the counter and outcome to assert at rebase. Part of #265 Co-Authored-By: Claude Opus 5 * 3.x.x: assert incident 2's late-confirm counter now that #324 is in the base W3c wrote the qualification tests against a base that predated #324 and left a marked place for the assertions that only make sense once the commit gate is there. The base now carries it, so apply them: incident 2 asserts `qbit_prism_late_confirmed_shares_total` is zero on both frontends and that no acknowledgement, the block-solving share included, is `ledger-outcome-unknown`. Acceptance alone is no longer proof. #324 gives a share-pass append carrying a candidate its own `block_only_ack_timeout`, so a slow solve is never refused at the share deadline; it is answered late, or answered unknown. Either would have satisfied the acceptance checks with the serialization stall still present. Also set #324's two new `Config` fields in the qualification fixture. * 3.x.x: pin a refused candidate as a definite rejection, not an unknown outcome #324 decides what a miner is told when a block-only enqueue fails: `enqueue_failed_before_commit` answers a provably uncommitted failure with `ledger-confirmation-failed`, and anything else with `ledger-outcome-unknown`, which asks an operator to go and follow the outbox row. Slice 3 added the checks that guard the reference: the block's length, its header hash, its digest against `block_sha256`, the coinbase suffix, the reference invariants, and a leased candidate's balances. All of them are raised by `prepare_candidate` before the enqueue takes a connection, so none carries an `sqlx::Error` and all classify as definite today. That is a property of where the checks sit, not something the classifier knows about them, so it is pinned here rather than left to hold by accident: a corrupt candidate must never send a miner chasing a row that was never written. Verified by mutation: classifying a non-sqlx failure as unknown fails this test. * 3.x.x: say why the full-size qualification variants cannot reach 400k yet Their ignore reasons blamed RAM and the host. The real limit is #273: the job payload still holds three copies of the window (2 x AcceptedShare + 1 x CountedShare), so publishing a job for a 400,000-share window needs about 670 MiB and PostgreSQL refuses the write above its 256 MiB jsonb object limit. Every full-size variant fails at fixture set-up, before it measures anything. The count must also divide the window weight exactly, so the largest usable size today is 125,000, where all four variants pass in release: incident 1 50 renewals, max gap 118.25 ms against a 700 ms bound incident 2 phase-A p99 2.60 ms, phase-B p99 95.79 ms, bound 250 ms lock hold median 0.75 ms at 5,000 against 1.32 ms at 125,000 canonical refresh and claim rebuild both 139,040,610 B, same sha256 The candidate document stays flat at 1103 B and 1107 B, which is the property this slice exists to establish. * 3.x.x: migrate the audit-body normalization tests to the slim candidate #267 landed its own test file against the candidate that still carried an assembled bundle. This slice replaced that with a window reference, so the file keeps a `TestCandidate` pairing the stored candidate with the bundle a test still needs, as `tests/ledger_postgres.rs` does, and attaches the claim's parts through `with_bundle`. One case needed more than a shape change. The four negative cases hand the claim a window the ledger does not hold and require the durable-range proof to be what refuses it. A candidate now commits to a snapshot digest, so a reference taken from the unaltered snapshot would be refused earlier, at the reference check, and the proof under test would never run. `window_ref_for` therefore derives each altered window's reference from that window, leaving the proof as the only thing that can notice, which is what #267 wrote them to prove. * 3.x.x: pin the durable-range proof outside the settlement lock #267 moved the full-equality proof of a candidate's window out of the settlement transaction so the lock is not held for a window-sized read. This slice made that read paged, so it does not occupy a runtime thread either. Reconciling the two is the step that can quietly undo the first: keep this slice's `persist_audit_snapshot` through the rebase and the full read goes back under the lock. The landing stays correct, every other test still passes, and only the lock hold grows, so nothing in the suite can tell. Measured at 100,000 shares on #267's branch that was a 34% longer hold. The guard is deterministic and needs no large window. A second connection holds `SETTLEMENT_LOCK`, so the landing blocks on it once its proof has run. While it waits, one share's `ntime` is altered: full equality sees that, a count over the same anchored predicate cannot. The landing must then succeed, which it can only do if it had already read and compared the shares before taking the lock. Accepted history is immutable and a trigger enforces it, so the test lifts that trigger for the single statement, inside its own disposable schema, and puts it straight back. Verified by mutation: restoring the in-transaction range read fails this test with "the landing read share payloads under the settlement lock". * 3.x.x: serialise the coordinator test modules that share the ledger's locks The ledger's advisory locks are cluster-wide constants, not schema-scoped, so two coordinator test modules running in parallel contend for ORDER_LOCK even though each works in its own throwaway schema. Every module held a lock of its own, which serialised it against itself and left cross-module waits to `lock_timeout`, 5 s by default. #324's reconciliation tests hold ORDER_LOCK for up to about 1.5 s, and this slice adds three more modules that open a ledger. Once enough of them ran at once those waits stacked past the timeout: a full `--lib` run failed with "canceling statement due to lock timeout" in `candidate_lease_tests` and `window_switch_tests`, and passed on the next run. A suite that fails for a reason unrelated to what it tests is worse than a slow one. All seven modules now take one shared lock. The run goes from about 35 s to about 62 s, and three consecutive runs pass where the flake had reproduced. * 3.x.x: pass the signer keys #334's wait-metrics tests now need `Ledger::configure` takes the frontend's signing keys since this slice, so the fingerprint contention tests #334 added call it with a fixed pair. That suite measures advisory-lock and pool-acquire waits, not signing, so the only thing the keys have to be is the same on every call. * feat: add compact prepared storage and immutable template blobs * Validate compact prepared endpoints and document caller contracts * 3.x.x: add migration 007 and refuse pre-007 candidate outbox rows Migration 007 mirrors 008 onto qbit_block_candidate_outbox: the same six window-reference columns, and a three-state CHECK whose body is identical to 008's apart from the table and constraint names. It also adds block_bytes, the nullable bytea the block moves into so the candidate document stays small, and a partial index on window_prior_balances_sha256 for the balance-snapshot GC. Nothing writes any of them yet; this is additive schema only. The runner applies 007 by membership, in its own block between 5 and 8, so it assumes neither 006 nor 008 has run. Before any DDL it refuses while the outbox still holds a pending row in a pre-007 shape: a native inline candidate, a NULL candidate, or storage_version other than 1 on the 2.x.x schemas that have that column, which is checked for first so the predicate does not error on a native schema. The refusal names every offending block_hash and the drain procedure, and the whole migration transaction rolls back, so a refused start leaves the schema untouched. The applied set is now [2, 3, 4, 5, 7, 8, 9], and the four exact migration assertions move with it rather than being weakened to a superset. Part of #265 Co-Authored-By: Claude Opus 5 * 3.x.x: retain the cluster fingerprint configure pinned or verified The candidate enqueue and save_job must fence against a fingerprint reset by re-reading qbit_prism_cluster.config_fingerprint FOR SHARE in their own transaction and refusing a write that is not this frontend's own. That needs the value configure settled at startup, so Ledger keeps it in a shared OnceLock, set only once the pin or the match has committed, and exposes it read-only through config_fingerprint(). configure keeps its signature and its mismatch refusal, and a refused call retains nothing. The rotation refusal in configure, and the writer fence itself, belong to the enqueue change and are not here. Part of #265 Co-Authored-By: Claude Opus 5 * 3.x.x: share the window reference, balance writer and page primitives WindowRef::from_snapshot is the synchronous reference builder both writers need: anchor_ms copied, the semantic balances digest, and for a non-empty snapshot the first and last share_seq, the length, and the share array's SHA-256 streamed through the existing DigestWriter, so the 233 to 260 MB serialized window at 400,000 shares is never built and the digest equals the one qbit_prism_audit_snapshots already stores. An empty snapshot gives the empty-window reference. Callers run it inside their own spawn_blocking. put_balance_snapshot is the one canonical writer of qbit_prism_balance_snapshots, on the caller's transaction: the sort by (order_key, recipient_id, p2mr_program_hex), the semantic digest that keys the row, and compact JSON of the sorted set, all off the runtime, then INSERT ... ON CONFLICT DO NOTHING. On a conflict it reads the stored bytes back and compares them with its own, because these rows are immutable and can only disagree through corruption. The enqueue and save_job and its repair all write through it. read_prior_balances and the current-balance read still keep their SQL order; the writer's sort defines the stored encoding, which is the order read_window(..., AsIssued) returns. probe_share_rows and read_range_paged are read_window's endpoint probe and page loop, lifted onto a caller's connection so the enqueue probe and the landing re-read share one implementation. The predicate, the ascending keyset order, the 4096-row page bound and the per-page count bound are unchanged, and read_window now calls them, with its signature and behaviour unchanged. The paged reader takes the caller's state and a per-page consumer, both of which travel to the blocking thread with the rows, so the landing re-read can compare each page with its slice of a vector it already holds without any whole-window work on the runtime; a dropped read still releases the accumulated state on a blocking thread. read_window_with_permit holds the caller's window_reads permit across the whole read. The permit is moved into a BlockingDrop declared before the read starts, so a cancellation releases it on a blocking thread after the page state queued its own release there, and never with the future; a return releases it inline, before the caller's rebuild. WindowError::TaskFailed separates a cancelled or panicking blocking hand-off from corruption. Every hand-off maps a JoinError to it instead of to Decode, so a cancellation is a retryable failure with its own alert and never takes the abandon path. Part of #265 Co-Authored-By: Claude Opus 5 * 3.x.x: derive the witness merkle leaves from an assembled block A claim rebuilds from stored inputs, and the witness leaves are the one builder input that scales with the block, so they are re-derived rather than stored: one 64-hex string per transaction beside the candidate would grow the document with the block. witness_merkle_leaves_from_block skips the 80-byte header, reads the compact-size transaction count, walks past the coinbase and returns double_sha256 of every remaining transaction's full serialized bytes. That is sound because assemble_submission appends the template's transactions verbatim after the coinbase, so the result equals witness_merkle_leaves_hex(&transactions_from_template(template)) for the block built from that template, with or without segwit transactions. The segwit-aware walk is strip_witness_transaction's own, factored into a helper that also reports where the transaction ends, which is what a block walker needs; strip_witness_transaction keeps its trailing-byte check and its output. A truncated block, trailing bytes, a zero count and a count that disagrees with the transactions present are all errors, never a partial list of leaves. Part of #265 Co-Authored-By: Claude Opus 5 * 3.x.x: version the audit builders and key the CC5 goldens by it A stored window reference reproduces its bundle only under the builder that first built it. The schema strings do not capture that, because the share shape chooses them rather than the builder logic, so qbit-prism gains AUDIT_BUILDER_VERSION, starting at 1. A candidate and a prepared job store it, and a rebuild refuses a reference at another version instead of rebuilding it with the current builder. The bump rule is on the constant: any change that alters canonical_audit_bundle_bytes for the same inputs bumps it in the same commit. The CC5 golden test now selects its expected digests from a table keyed by the constant, and fails when no entry exists, so a new output is recorded as a new entry beside the old one and review sees both. Version 1 holds the existing literals, unchanged. Part of #265 Co-Authored-By: Claude Opus 5 * 3.x.x: cover 007, the fingerprint and the window primitives on PostgreSQL Ten gated database cases, all listed in test/prism-gated-tests.txt. They go in a new file rather than the shared 2.x.x support module, which another open change appends to. Migration 007: a frozen 2.x.x schema carrying only terminal outbox rows upgrades with every row and field intact and the seven new columns NULL; each of the three pre-007 pending shapes is refused by name, with nothing from 007 applied and no version 7 row, and the same binary starts once the row is drained; a database at [2, 3, 4, 5, 8, 9] applies 007 and only 007, leaving every other version's applied_at untouched; the CHECK accepts exactly the three states across all 64 null combinations and rejects a short or uppercase digest, a zero or out-of-range count and a zero sequence; and the retention index is the partial one on window_prior_balances_sha256. config_fingerprint() returns what configure pinned or verified, is shared by clones, and stays empty after a refused configure. from_snapshot's streamed digest equals sha256(serde_json::to_vec) and the snapshot_sha256 landing writes for the same window. The balance writer stores the canonical sort, is idempotent under any permutation, errors rather than accepting a different encoding of the same digest, and read_window(..., AsIssued) hands back the stored order. The probe misses a missing endpoint, the paged reader compares 4,999 rows page by page with a vector held inside the caller's transaction, and a row the predicate skips inside a range is Incomplete with no partial window. The permit case builds a runtime with one blocking thread, occupies it, and cancels a reader parked mid-page on a database gate: no permit is available while the cleanup is queued, and one is once it has run. A failed blocking hand-off surfaces as TaskFailed, never as Decode. Part of #265 Co-Authored-By: Claude Opus 5 * 3.x.x: carry the window reference on prepared work, behind a read permit `Prepared` gains `window: WindowRef`, computed once per non-cached refresh beside `build_bundle` under `tokio::join!`, so submit can clone it instead of re-digesting the window on the share path when a block is found. An empty snapshot gets the empty-window reference on the runtime thread; a resume computes it inside the blocking decode it already runs. `Coordinator` gains the `window_reads` semaphore, sized `clamp(database_max_connections - 2, 1, build_workers)`, and the one entry point callers use: it takes an owned permit after the caller's `build_slots` permit and hands it to `Ledger::read_window_with_permit`, which holds it across the whole read, including the off-runtime cleanup a cancellation leaves behind, and releases it before the rebuild. `Ledger::read_window` is awaited on the runtime, never wrapped in `spawn_blocking`; the test with a single blocking thread pins that. The four gated `window_ref_tests` are listed in the manifest. Part of #265 Co-Authored-By: Claude Fable 5.1 * 3.x.x: switch the candidate outbox to the window reference `Candidate` drops the embedded `AuditBundle` for `window: WindowRef` and every builder input except the window: `found_block`, `payout_policy`, the nested `ctv` triple (whose presence selects the CTV builder), the stored `bootstrap_share` of an empty window, `audit_builder_version`, `signer_keys`, a `leased` flag and the now-required `coinbase_suffix_hex`. The block leaves the document for the `block_bytes` column, authenticated by `block_sha256`, which `candidate_sha256` covers. The write serializes, digests and checks the candidate before the append or enqueue transaction opens; under `ORDER_LOCK` the transaction re-reads `config_fingerprint FOR SHARE` and refuses a value the writer did not pin, re-inserts a `leased` candidate's as-issued balance snapshot through the shared writer, probes the window's first row (an absent row alerts and still publishes), and inserts only the prepared bytes with the six typed columns. Submit builds the slim candidate from `Prepared.window`, the `BundleInputs` captured where `build_bundle` read configuration, and the job's own bootstrap share; it clones no bundle and reads no config. The claim's `UPDATE … RETURNING` returns the columns and the block, and the decode, off the runtime, checks the document against them, the digest, `sha256(block_bytes)`, the header hash, the required fields and the two reference invariants. A pending row with a NULL anchor is a hard error naming the pre-007 frontend; an inline document is never decoded. Before any read the claim retries, with an alert naming both values, a stored builder version or signer pair that is not this binary's; a landed audit is authenticated against the block (coinbase, audit root as the witness reserved value, share snapshot, … * 3.x.x: Prepare compact work qualification inputs and guards (#367) * 3.x.x: add migration 007 and refuse pre-007 candidate outbox rows Migration 007 mirrors 008 onto qbit_block_candidate_outbox: the same six window-reference columns, and a three-state CHECK whose body is identical to 008's apart from the table and constraint names. It also adds block_bytes, the nullable bytea the block moves into so the candidate document stays small, and a partial index on window_prior_balances_sha256 for the balance-snapshot GC. Nothing writes any of them yet; this is additive schema only. The runner applies 007 by membership, in its own block between 5 and 8, so it assumes neither 006 nor 008 has run. Before any DDL it refuses while the outbox still holds a pending row in a pre-007 shape: a native inline candidate, a NULL candidate, or storage_version other than 1 on the 2.x.x schemas that have that column, which is checked for first so the predicate does not error on a native schema. The refusal names every offending block_hash and the drain procedure, and the whole migration transaction rolls back, so a refused start leaves the schema untouched. The applied set is now [2, 3, 4, 5, 7, 8, 9], and the four exact migration assertions move with it rather than being weakened to a superset. Part of #265 Co-Authored-By: Claude Opus 5 * 3.x.x: retain the cluster fingerprint configure pinned or verified The candidate enqueue and save_job must fence against a fingerprint reset by re-reading qbit_prism_cluster.config_fingerprint FOR SHARE in their own transaction and refusing a write that is not this frontend's own. That needs the value configure settled at startup, so Ledger keeps it in a shared OnceLock, set only once the pin or the match has committed, and exposes it read-only through config_fingerprint(). configure keeps its signature and its mismatch refusal, and a refused call retains nothing. The rotation refusal in configure, and the writer fence itself, belong to the enqueue change and are not here. Part of #265 Co-Authored-By: Claude Opus 5 * 3.x.x: share the window reference, balance writer and page primitives WindowRef::from_snapshot is the synchronous reference builder both writers need: anchor_ms copied, the semantic balances digest, and for a non-empty snapshot the first and last share_seq, the length, and the share array's SHA-256 streamed through the existing DigestWriter, so the 233 to 260 MB serialized window at 400,000 shares is never built and the digest equals the one qbit_prism_audit_snapshots already stores. An empty snapshot gives the empty-window reference. Callers run it inside their own spawn_blocking. put_balance_snapshot is the one canonical writer of qbit_prism_balance_snapshots, on the caller's transaction: the sort by (order_key, recipient_id, p2mr_program_hex), the semantic digest that keys the row, and compact JSON of the sorted set, all off the runtime, then INSERT ... ON CONFLICT DO NOTHING. On a conflict it reads the stored bytes back and compares them with its own, because these rows are immutable and can only disagree through corruption. The enqueue and save_job and its repair all write through it. read_prior_balances and the current-balance read still keep their SQL order; the writer's sort defines the stored encoding, which is the order read_window(..., AsIssued) returns. probe_share_rows and read_range_paged are read_window's endpoint probe and page loop, lifted onto a caller's connection so the enqueue probe and the landing re-read share one implementation. The predicate, the ascending keyset order, the 4096-row page bound and the per-page count bound are unchanged, and read_window now calls them, with its signature and behaviour unchanged. The paged reader takes the caller's state and a per-page consumer, both of which travel to the blocking thread with the rows, so the landing re-read can compare each page with its slice of a vector it already holds without any whole-window work on the runtime; a dropped read still releases the accumulated state on a blocking thread. read_window_with_permit holds the caller's window_reads permit across the whole read. The permit is moved into a BlockingDrop declared before the read starts, so a cancellation releases it on a blocking thread after the page state queued its own release there, and never with the future; a return releases it inline, before the caller's rebuild. WindowError::TaskFailed separates a cancelled or panicking blocking hand-off from corruption. Every hand-off maps a JoinError to it instead of to Decode, so a cancellation is a retryable failure with its own alert and never takes the abandon path. Part of #265 Co-Authored-By: Claude Opus 5 * 3.x.x: derive the witness merkle leaves from an assembled block A claim rebuilds from stored inputs, and the witness leaves are the one builder input that scales with the block, so they are re-derived rather than stored: one 64-hex string per transaction beside the candidate would grow the document with the block. witness_merkle_leaves_from_block skips the 80-byte header, reads the compact-size transaction count, walks past the coinbase and returns double_sha256 of every remaining transaction's full serialized bytes. That is sound because assemble_submission appends the template's transactions verbatim after the coinbase, so the result equals witness_merkle_leaves_hex(&transactions_from_template(template)) for the block built from that template, with or without segwit transactions. The segwit-aware walk is strip_witness_transaction's own, factored into a helper that also reports where the transaction ends, which is what a block walker needs; strip_witness_transaction keeps its trailing-byte check and its output. A truncated block, trailing bytes, a zero count and a count that disagrees with the transactions present are all errors, never a partial list of leaves. Part of #265 Co-Authored-By: Claude Opus 5 * 3.x.x: version the audit builders and key the CC5 goldens by it A stored window reference reproduces its bundle only under the builder that first built it. The schema strings do not capture that, because the share shape chooses them rather than the builder logic, so qbit-prism gains AUDIT_BUILDER_VERSION, starting at 1. A candidate and a prepared job store it, and a rebuild refuses a reference at another version instead of rebuilding it with the current builder. The bump rule is on the constant: any change that alters canonical_audit_bundle_bytes for the same inputs bumps it in the same commit. The CC5 golden test now selects its expected digests from a table keyed by the constant, and fails when no entry exists, so a new output is recorded as a new entry beside the old one and review sees both. Version 1 holds the existing literals, unchanged. Part of #265 Co-Authored-By: Claude Opus 5 * 3.x.x: cover 007, the fingerprint and the window primitives on PostgreSQL Ten gated database cases, all listed in test/prism-gated-tests.txt. They go in a new file rather than the shared 2.x.x support module, which another open change appends to. Migration 007: a frozen 2.x.x schema carrying only terminal outbox rows upgrades with every row and field intact and the seven new columns NULL; each of the three pre-007 pending shapes is refused by name, with nothing from 007 applied and no version 7 row, and the same binary starts once the row is drained; a database at [2, 3, 4, 5, 8, 9] applies 007 and only 007, leaving every other version's applied_at untouched; the CHECK accepts exactly the three states across all 64 null combinations and rejects a short or uppercase digest, a zero or out-of-range count and a zero sequence; and the retention index is the partial one on window_prior_balances_sha256. config_fingerprint() returns what configure pinned or verified, is shared by clones, and stays empty after a refused configure. from_snapshot's streamed digest equals sha256(serde_json::to_vec) and the snapshot_sha256 landing writes for the same window. The balance writer stores the canonical sort, is idempotent under any permutation, errors rather than accepting a different encoding of the same digest, and read_window(..., AsIssued) hands back the stored order. The probe misses a missing endpoint, the paged reader compares 4,999 rows page by page with a vector held inside the caller's transaction, and a row the predicate skips inside a range is Incomplete with no partial window. The permit case builds a runtime with one blocking thread, occupies it, and cancels a reader parked mid-page on a database gate: no permit is available while the cleanup is queued, and one is once it has run. A failed blocking hand-off surfaces as TaskFailed, never as Decode. Part of #265 Co-Authored-By: Claude Opus 5 * 3.x.x: carry the window reference on prepared work, behind a read permit `Prepared` gains `window: WindowRef`, computed once per non-cached refresh beside `build_bundle` under `tokio::join!`, so submit can clone it instead of re-digesting the window on the share path when a block is found. An empty snapshot gets the empty-window reference on the runtime thread; a resume computes it inside the blocking decode it already runs. `Coordinator` gains the `window_reads` semaphore, sized `clamp(database_max_connections - 2, 1, build_workers)`, and the one entry point callers use: it takes an owned permit after the caller's `build_slots` permit and hands it to `Ledger::read_window_with_permit`, which holds it across the whole read, including the off-runtime cleanup a cancellation leaves behind, and releases it before the rebuild. `Ledger::read_window` is awaited on the runtime, never wrapped in `spawn_blocking`; the test with a single blocking thread pins that. The four gated `window_ref_tests` are listed in the manifest. Part of #265 Co-Authored-By: Claude Fable 5.1 * 3.x.x: switch the candidate outbox to the window reference `Candidate` drops the embedded `AuditBundle` for `window: WindowRef` and every builder input except the window: `found_block`, `payout_policy`, the nested `ctv` triple (whose presence selects the CTV builder), the stored `bootstrap_share` of an empty window, `audit_builder_version`, `signer_keys`, a `leased` flag and the now-required `coinbase_suffix_hex`. The block leaves the document for the `block_bytes` column, authenticated by `block_sha256`, which `candidate_sha256` covers. The write serializes, digests and checks the candidate before the append or enqueue transaction opens; under `ORDER_LOCK` the transaction re-reads `config_fingerprint FOR SHARE` and refuses a value the writer did not pin, re-inserts a `leased` candidate's as-issued balance snapshot through the shared writer, probes the window's first row (an absent row alerts and still publishes), and inserts only the prepared bytes with the six typed columns. Submit builds the slim candidate from `Prepared.window`, the `BundleInputs` captured where `build_bundle` read configuration, and the job's own bootstrap share; it clones no bundle and reads no config. The claim's `UPDATE … RETURNING` returns the columns and the block, and the decode, off the runtime, checks the document against them, the digest, `sha256(block_bytes)`, the header hash, the required fields and the two reference invariants. A pending row with a NULL anchor is a hard error naming the pre-007 frontend; an inline document is never decoded. Before any read the claim retries, with an alert naming both values, a stored builder version or signer pair that is not this binary's; a landed audit is authenticated against the block (coinbase, audit root as the witness reserved value, share snapshot, bits) and the claim then skips the rebuild and landing and continues to observe, renew and submit. Otherwise it takes `build_slots`, then `window_reads` inside one 60 s deadline around the awaited `read_window(Current)` and the borrowing builder run directly in `spawn_blocking`, and keeps `Arc` and the shares as `ClaimParts`. `Incomplete`, `SnapshotDigestMismatch`, `Decode`, `BalanceSnapshotMissing`, `TaskFailed` and the deadline retry with their own alerts; `PriorBalancesChanged` finishes only a block `observe_candidate` proves inactive and superseded; `Database` propagates. A `leased` candidate submits from `block_bytes` first, then rebuilds `AsIssued` and lands before any terminal outcome. Rebuilt parts are released off the runtime on every path. Landing takes the parts and does all whole-window work in one blocking closure: `verify_audit_parts`, the raw body, the counted canonical bytes, the leaves and accounts, and the share snapshot digest, which must equal the reference's. The balance fence compares the current set and the parts' through the semantic digest, so an as-issued set compares equal in any order. `persist_audit_snapshot` re-reads the range with the paged reader inside the landing transaction, page against slice on the blocking thread; `read_range` stays for the API. The terminal UPDATE NULLs the six window columns and `block_bytes` with `candidate`. `configure` refuses to pin a fingerprint onto a reset one while a pending row stores other signer keys, and the startup gate requires 007 with 009. The JSONB gate drops its enqueue entry, and every helper that built a candidate builds the slim one, handing the ledger-level landings the bundle's parts through `CandidateClaim::with_bundle`. Part of #265 Co-Authored-By: Claude Fable 5.1 * 3.x.x: cover the candidate switch on PostgreSQL Ledger-level, in `candidate_window_switch`: a two-page window enqueues a document with no array and no share row, under 1 MB, whose six typed columns equal its reference and whose block is stored as bytes; the writer fence refuses a reset or different fingerprint in the same transaction, share included; a missing window prefix row alerts and still publishes, and its read then meets `Incomplete`; a `leased` enqueue re-inserts the as-issued balance snapshot in the writer's canonical order, writes none when not leased, and refuses balances that do not hash to the reference; every preparation failure is refused without waiting for `ORDER_LOCK`; every column, digest, block, header, NULL-anchor, missing-field, invariant and inline-document disagreement is refused at claim; landing refuses parts that are not the referenced window and a claim without parts; the terminal UPDATE NULLs the six columns and the block for submitted and abandoned rows; the landed-audit authentication rejects a forged coinbase, root, snapshot or bits; and `configure` refuses a rotation over a foreign-signed pending candidate. Coordinator-level, in `window_switch_tests` behind a fake node: a stored builder version or signer pair that is not this binary's retries, naming both values, while the build slot is held; a landed audit is authenticated and the claim continues to `submitblock` without a build slot, and a forged landed coinbase or root is refused with the row left pending; `Incomplete`, a digest mismatch, changed balances on a current block and the rebuild deadline all retry with an alert and release their permits; `classify_window_error` keeps `TaskFailed` apart from corruption and propagates `Database`; the read, rebuild and landing complete while `ORDER_LOCK` is held, with one build worker; a `leased` candidate submits first, then lands from `AsIssued`, and finishes submitted, abandoned with the node's reason, or pending when its balance snapshot is missing; and `OffRuntime` releases on a blocking thread, on cancellation too. The fifteen gated tests are listed in the manifest. Part of #265 Co-Authored-By: Claude Fable 5.1 * 3.x.x: migrate the imported-audit tests to the slim candidate Slice 1 (#325) merged after this stack branched, and its tests in tests/support/ledger_2x.rs still built the pre-007 candidate: they read block.bundle from a Candidate and called candidate_with_bundle without a window reference or settlement inputs. The rebase merged them cleanly and the tree then failed to compile. The three helpers now take the TestCandidate the constructors return, so they reach the bundle beside the slim candidate, and land_confirmed attaches the claim's parts the way the other ledger tests do. The three constructor calls pass the window reference the bundle was built on, and the settlement inputs for the CTV case. No assertion changes. Part of #265. * 3.x.x: qualify lease renewal and the claim's canonical bytes at scale Incident 1: a candidate whose window holds N shares is claimed and processed under the short lease, and every renewal is read back from claim_expires_at. The largest gap stays below interval + 2 x timeout, the lease never lapses, and a successor ledger cannot take the claim. The bundle a claim rebuilds through the window-read entry point and the borrowing builder has the canonical bytes refresh built for the same job, and the landed audit carries that bundle's digest. Both run at PRISM_WINDOW_QUALIFY_SHARES (20,000 by default); the 400,000-share variants are ignored and run on a larger host. Part of #265 Co-Authored-By: Claude Opus 5 * 3.x.x: measure ACK latency and ORDER_LOCK hold across window sizes Incident 2: two frontends on one database submit ordinary shares at a fixed rate while one of them solves a block on an N-share window and its own submit loop claims, rebuilds, lands and submits it. The solving share's ACK and the bystander's p99 stay within max(2 x baseline p99 + 50 ms, 250 ms), every ACK is well under the share-commit timeout, and the block is enqueued exactly once. Latency is the in-process MiningBackend::submit wall clock; the socket-level harness (#271) does not exist yet. A pg_locks sampler on its own connection measures how long a solving append holds ORDER_LOCK at two window sizes with no other traffic: the larger size stays within a fixed slack of the smaller, and the candidate document is the same size at both and under 1 MiB. Part of #265 Co-Authored-By: Claude Opus 5 * 3.x.x: hold the JSONB gate's enqueue to a window-free row under 1 MiB At every measured size the enqueue must be accepted and every JSONB value it writes, the outbox candidate document included, must stay under 1,048,576 bytes. At the reduced sizes the enqueue, claim and landing phases must all run. The ratchet and its expected values are unchanged. Part of #265 Co-Authored-By: Claude Opus 5 * 3.x.x: make the window-read permit test actually wait before it looks The delay that gives an unpermitted read time to finish was an empty async block, so the check that the read had not finished ran at once and passed whether or not the entry point took a permit. It now sleeps 500 ms inside a 5 s bound, and fails when the permit acquisition is removed from the entry point. Part of #265 Co-Authored-By: Claude Opus 5 * 3.x.x: rebuild a CTV candidate from its stored settlement inputs No host-runnable test reached the CTV branch of the claim rebuild. A CTV-enabled frontend now claims a candidate whose stored direct floor differs from its configuration, and builds a different coinbase than the configured floor would, and the claim must land that candidate's own audit and submit it. Part of #265 Co-Authored-By: Claude Opus 5 * 3.x.x: pin the SQL order of the current-balance reads Nothing defended the ruling that the current-balance path keeps its SQL order: every fixture's balances already arrived in byte-wise canonical order. The carry table's text columns now take an ICU collation, under which the SQL order and the canonical sort disagree, and the snapshot and the current-balance window read must both return the SQL order. Part of #265 Co-Authored-By: Claude Opus 5 * 3.x.x: record what incident 2 must also assert once #324 is in the base After #324 a slow solve is acknowledged late rather than refused, so acceptance proves nothing and the latency percentiles stay the assertion. The doc comment and a TODO at the outcome checks name the counter and outcome to assert at rebase. Part of #265 Co-Authored-By: Claude Opus 5 * 3.x.x: assert incident 2's late-confirm counter now that #324 is in the base W3c wrote the qualification tests against a base that predated #324 and left a marked place for the assertions that only make sense once the commit gate is there. The base now carries it, so apply them: incident 2 asserts `qbit_prism_late_confirmed_shares_total` is zero on both frontends and that no acknowledgement, the block-solving share included, is `ledger-outcome-unknown`. Acceptance alone is no longer proof. #324 gives a share-pass append carrying a candidate its own `block_only_ack_timeout`, so a slow solve is never refused at the share deadline; it is answered late, or answered unknown. Either would have satisfied the acceptance checks with the serialization stall still present. Also set #324's two new `Config` fields in the qualification fixture. * 3.x.x: pin a refused candidate as a definite rejection, not an unknown outcome #324 decides what a miner is told when a block-only enqueue fails: `enqueue_failed_before_commit` answers a provably uncommitted failure with `ledger-confirmation-failed`, and anything else with `ledger-outcome-unknown`, which asks an operator to go and follow the outbox row. Slice 3 added the checks that guard the reference: the block's length, its header hash, its digest against `block_sha256`, the coinbase suffix, the reference invariants, and a leased candidate's balances. All of them are raised by `prepare_candidate` before the enqueue takes a connection, so none carries an `sqlx::Error` and all classify as definite today. That is a property of where the checks sit, not something the classifier knows about them, so it is pinned here rather than left to hold by accident: a corrupt candidate must never send a miner chasing a row that was never written. Verified by mutation: classifying a non-sqlx failure as unknown fails this test. * 3.x.x: say why the full-size qualification variants cannot reach 400k yet Their ignore reasons blamed RAM and the host. The real limit is #273: the job payload still holds three copies of the window (2 x AcceptedShare + 1 x CountedShare), so publishing a job for a 400,000-share window needs about 670 MiB and PostgreSQL refuses the write above its 256 MiB jsonb object limit. Every full-size variant fails at fixture set-up, before it measures anything. The count must also divide the window weight exactly, so the largest usable size today is 125,000, where all four variants pass in release: incident 1 50 renewals, max gap 118.25 ms against a 700 ms bound incident 2 phase-A p99 2.60 ms, phase-B p99 95.79 ms, bound 250 ms lock hold median 0.75 ms at 5,000 against 1.32 ms at 125,000 canonical refresh and claim rebuild both 139,040,610 B, same sha256 The candidate document stays flat at 1103 B and 1107 B, which is the property this slice exists to establish. * 3.x.x: migrate the audit-body normalization tests to the slim candidate #267 landed its own test file against the candidate that still carried an assembled bundle. This slice replaced that with a window reference, so the file keeps a `TestCandidate` pairing the stored candidate with the bundle a test still needs, as `tests/ledger_postgres.rs` does, and attaches the claim's parts through `with_bundle`. One case needed more than a shape change. The four negative cases hand the claim a window the ledger does not hold and require the durable-range proof to be what refuses it. A candidate now commits to a snapshot digest, so a reference taken from the unaltered snapshot would be refused earlier, at the reference check, and the proof under test would never run. `window_ref_for` therefore derives each altered window's reference from that window, leaving the proof as the only thing that can notice, which is what #267 wrote them to prove. * 3.x.x: pin the durable-range proof outside the settlement lock #267 moved the full-equality proof of a candidate's window out of the settlement transaction so the lock is not held for a window-sized read. This slice made that read paged, so it does not occupy a runtime thread either. Reconciling the two is the step that can quietly undo the first: keep this slice's `persist_audit_snapshot` through the rebase and the full read goes back under the lock. The landing stays correct, every other test still passes, and only the lock hold grows, so nothing in the suite can tell. Measured at 100,000 shares on #267's branch that was a 34% longer hold. The guard is deterministic and needs no large window. A second connection holds `SETTLEMENT_LOCK`, so the landing blocks on it once its proof has run. While it waits, one share's `ntime` is altered: full equality sees that, a count over the same anchored predicate cannot. The landing must then succeed, which it can only do if it had already read and compared the shares before taking the lock. Accepted history is immutable and a trigger enforces it, so the test lifts that trigger for the single statement, inside its own disposable schema, and puts it straight back. Verified by mutation: restoring the in-transaction range read fails this test with "the landing read share payloads under the settlement lock". * 3.x.x: serialise the coordinator test modules that share the ledger's locks The ledger's advisory locks are cluster-wide constants, not schema-scoped, so two coordinator test modules running in parallel contend for ORDER_LOCK even though each works in its own throwaway schema. Every module held a lock of its own, which serialised it against itself and left cross-module waits to `lock_timeout`, 5 s by default. #324's reconciliation tests hold ORDER_LOCK for up to about 1.5 s, and this slice adds three more modules that open a ledger. Once enough of them ran at once those waits stacked past the timeout: a full `--lib` run failed with "canceling statement due to lock timeout" in `candidate_lease_tests` and `window_switch_tests`, and passed on the next run. A suite that fails for a reason unrelated to what it tests is worse than a slow one. All seven modules now take one shared lock. The run goes from about 35 s to about 62 s, and three consecutive runs pass where the flake had reproduced. * 3.x.x: pass the signer keys #334's wait-metrics tests now need `Ledger::configure` takes the frontend's signing keys since this slice, so the fingerprint contention tests #334 added call it with a fixed pair. That suite measures advisory-lock and pool-acquire waits, not signing, so the only thing the keys have to be is the same on every call. * 3.x.x: resume a job's bundle only while the current inputs still describe it A resumed job reuses the bundle it was issued with, but captures its inputs from this frontend's configuration now. Before this slice that was harmless: the candidate carried the assembled bundle, and landing used it directly. Now the candidate stores the inputs and the claim rebuilds the audit from them, so an old bundle labelled with today's configuration rebuilds to a coinbase the block does not commit to. The claim fails its comparison before `submitblock` and retries until the job expires, which can lose a found block. The stored bundle is now reused only when the captured inputs still describe it, comparing what the bundle itself records about how it was built: the payout policy, and the public keys that actually signed its coinbase manifest and its ledger attestation. A difference is a cache miss, so the bundle is rebuilt rather than mislabelled. Both directions are covered, since over-tightening would silently defeat the job cache and rebuild on every resume: unchanged inputs must still describe the bundle, and a payout-policy change, a full signer rotation and a rotation of either key alone must each refuse it. Known residue, recorded in the doc comment: a bundle records neither the CTV settlement inputs nor `audit_builder_version`, so a change to either between issue and resume is still undetectable here. Closing that needs the issued job to store its inputs, which is #273's format. * 3.x.x: migrate #355's ledger tests to the slim candidate #355 landed deadline and single-execution coverage against the candidate that still carried an assembled bundle, and against `configure` before it took the frontend's signing keys. Adapted, the same way #325's and #267's suites were: a `TestCandidate` pairs the stored candidate with the bundle a landing still needs, claims attach their parts through `with_bundle`, and the conflicting claim that is never landed carries none. `ledger_deadline_atomicity` builds its candidate from a bundle rather than a snapshot, so its window reference is derived from that bundle: the anchor and prior-balances digest it already records, and the range and digest of the shares it carries. Its signer keys come from the public keys that actually signed it, rather than from a local pair, so the stored inputs describe the bundle. * 3.x.x: make the landed-audit forgery always alter the row, and migrate #360's fixture The forged-coinbase case placed a literal '0' at a fixed offset. Whenever the digit already held that value the forgery changed nothing, so the claim was right to accept it and the case failed only by luck. The coinbase varies per run, because the bundle's anchor comes from the ledger clock, so this passed repeatedly and then failed once as "the forgery was accepted" — an alarming message for a case that was in fact a no-op. A security check that can quietly become vacuous is worse than one that is absent. The digit is now flipped to a value it does not currently hold, and the row is read back either side of the statement so a forgery that altered nothing fails the case for that reason instead. Also passes the signer keys #360's `tests/support/fatal_state.rs` now needs, since `configure` takes them in this slice. * 3.x.x: drop the pre-#362 path for the heartbeat SQL gated test #362 moved `live_instance_tests` from `tools` to `ledger::instances`. Merging the gated-test manifest during the rebase kept both paths, because a rename upstream looks like one id removed and another added, and a union keeps the removed one. The old `tools::` path then never executed and the gate refused the run: "194 distinct gated tests executed, below the minimum of 195". The manifest is a superset of the base's ids, and every id this branch adds over the base resolves to a test function that exists in the tree. * v3.0.0 feat: add compact prepared storage and immutable template blobs (#359) * 3.x.x: add migration 007 and refuse pre-007 candidate outbox rows Migration 007 mirrors 008 onto qbit_block_candidate_outbox: the same six window-reference columns, and a three-state CHECK whose body is identical to 008's apart from the table and constraint names. It also adds block_bytes, the nullable bytea the block moves into so the candidate document stays small, and a partial index on window_prior_balances_sha256 for the balance-snapshot GC. Nothing writes any of them yet; this is additive schema only. The runner applies 007 by membership, in its own block between 5 and 8, so it assumes neither 006 nor 008 has run. Before any DDL it refuses while the outbox still holds a pending row in a pre-007 shape: a native inline candidate, a NULL candidate, or storage_version other than 1 on the 2.x.x schemas that have that column, which is checked for first so the predicate does not error on a native schema. The refusal names every offending block_hash and the drain procedure, and the whole migration transaction rolls back, so a refused start leaves the schema untouched. The applied set is now [2, 3, 4, 5, 7, 8, 9], and the four exact migration assertions move with it rather than being weakened to a superset. Part of #265 Co-Authored-By: Claude Opus 5 * 3.x.x: retain the cluster fingerprint configure pinned or verified The candidate enqueue and save_job must fence against a fingerprint reset by re-reading qbit_prism_cluster.config_fingerprint FOR SHARE in their own transaction and refusing a write that is not this frontend's own. That needs the value configure settled at startup, so Ledger keeps it in a shared OnceLock, set only once the pin or the match has committed, and exposes it read-only through config_fingerprint(). configure keeps its signature and its mismatch refusal, and a refused call retains nothing. The rotation refusal in configure, and the writer fence itself, belong to the enqueue change and are not here. Part of #265 Co-Authored-By: Claude Opus 5 * 3.x.x: share the window reference, balance writer and page primitives WindowRef::from_snapshot is the synchronous reference builder both writers need: anchor_ms copied, the semantic balances digest, and for a non-empty snapshot the first and last share_seq, the length, and the share array's SHA-256 streamed through the existing DigestWriter, so the 233 to 260 MB serialized window at 400,000 shares is never built and the digest equals the one qbit_prism_audit_snapshots already stores. An empty snapshot gives the empty-window reference. Callers run it inside their own spawn_blocking. put_balance_snapshot is the one canonical writer of qbit_prism_balance_snapshots, on the caller's transaction: the sort by (order_key, recipient_id, p2mr_program_hex), the semantic digest that keys the row, and compact JSON of the sorted set, all off the runtime, then INSERT ... ON CONFLICT DO NOTHING. On a conflict it reads the stored bytes back and compares them with its own, because these rows are immutable and can only disagree through corruption. The enqueue and save_job and its repair all write through it. read_prior_balances and the current-balance read still keep their SQL order; the writer's sort defines the stored encoding, which is the order read_window(..., AsIssued) returns. probe_share_rows and read_range_paged are read_window's endpoint probe and page loop, lifted onto a caller's connection so the enqueue probe and the landing re-read share one implementation. The predicate, the ascending keyset order, the 4096-row page bound and the per-page count bound are unchanged, and read_window now calls them, with its signature and behaviour unchanged. The paged reader takes the caller's state and a per-page consumer, both of which travel to the blocking thread with the rows, so the landing re-read can compare each page with its slice of a vector it already holds without any whole-window work on the runtime; a dropped read still releases the accumulated state on a blocking thread. read_window_with_permit holds the caller's window_reads permit across the whole read. The permit is moved into a BlockingDrop declared before the read starts, so a cancellation releases it on a blocking thread after the page state queued its own release there, and never with the future; a return releases it inline, before the caller's rebuild. WindowError::TaskFailed separates a cancelled or panicking blocking hand-off from corruption. Every hand-off maps a JoinError to it instead of to Decode, so a cancellation is a retryable failure with its own alert and never takes the abandon path. Part of #265 Co-Authored-By: Claude Opus 5 * 3.x.x: derive the witness merkle leaves from an assembled block A claim rebuilds from stored inputs, and the witness leaves are the one builder input that scales with the block, so they are re-derived rather than stored: one 64-hex string per transaction beside the candidate would grow the document with the block. witness_merkle_leaves_from_block skips the 80-byte header, reads the compact-size transaction count, walks past the coinbase and returns double_sha256 of every remaining transaction's full serialized bytes. That is sound because assemble_submission appends the template's transactions verbatim after the coinbase, so the result equals witness_merkle_leaves_hex(&transactions_from_template(template)) for the block built from that template, with or without segwit transactions. The segwit-aware walk is strip_witness_transaction's own, factored into a helper that also reports where the transaction ends, which is what a block walker needs; strip_witness_transaction keeps its trailing-byte check and its output. A truncated block, trailing bytes, a zero count and a count that disagrees with the transactions present are all errors, never a partial list of leaves. Part of #265 Co-Authored-By: Claude Opus 5 * 3.x.x: version the audit builders and key the CC5 goldens by it A stored window reference reproduces its bundle only under the builder that first built it. The schema strings do not capture that, because the share shape chooses them rather than the builder logic, so qbit-prism gains AUDIT_BUILDER_VERSION, starting at 1. A candidate and a prepared job store it, and a rebuild refuses a reference at another version instead of rebuilding it with the current builder. The bump rule is on the constant: any change that alters canonical_audit_bundle_bytes for the same inputs bumps it in the same commit. The CC5 golden test now selects its expected digests from a table keyed by the constant, and fails when no entry exists, so a new output is recorded as a new entry beside the old one and review sees both. Version 1 holds the existing literals, unchanged. Part of #265 Co-Authored-By: Claude Opus 5 * 3.x.x: cover 007, the fingerprint and the window primitives on PostgreSQL Ten gated database cases, all listed in test/prism-gated-tests.txt. They go in a new file rather than the shared 2.x.x support module, which another open change appends to. Migration 007: a frozen 2.x.x schema carrying only terminal outbox rows upgrades with every row and field intact and the seven new columns NULL; each of the three pre-007 pending shapes is refused by name, with nothing from 007 applied and no version 7 row, and the same binary starts once the row is drained; a database at [2, 3, 4, 5, 8, 9] applies 007 and only 007, leaving every other version's applied_at untouched; the CHECK accepts exactly the three states across all 64 null combinations and rejects a short or uppercase digest, a zero or out-of-range count and a zero sequence; and the retention index is the partial one on window_prior_balances_sha256. config_fingerprint() returns what configure pinned or verified, is shared by clones, and stays empty after a refused configure. from_snapshot's streamed digest equals sha256(serde_json::to_vec) and the snapshot_sha256 landing writes for the same window. The balance writer stores the canonical sort, is idempotent under any permutation, errors rather than accepting a different encoding of the same digest, and read_window(..., AsIssued) hands back the stored order. The probe misses a missing endpoint, the paged reader compares 4,999 rows page by page with a vector held inside the caller's transaction, and a row the predicate skips inside a range is Incomplete with no partial window. The permit case builds a runtime with one blocking thread, occupies it, and cancels a reader parked mid-page on a database gate: no permit is available while the cleanup is queued, and one is once it has run. A failed blocking hand-off surfaces as TaskFailed, never as Decode. Part of #265 Co-Authored-By: Claude Opus 5 * 3.x.x: carry the window reference on prepared work, behind a read permit `Prepared` gains `window: WindowRef`, computed once per non-cached refresh beside `build_bundle` under `tokio::join!`, so submit can clone it instead of re-digesting the window on the share path when a block is found. An empty snapshot gets the empty-window reference on the runtime thread; a resume computes it inside the blocking decode it already runs. `Coordinator` gains the `window_reads` semaphore, sized `clamp(database_max_connections - 2, 1, build_workers)`, and the one entry point callers use: it takes an owned permit after the caller's `build_slots` permit and hands it to `Ledger::read_window_with_permit`, which holds it across the whole read, including the off-runtime cleanup a cancellation leaves behind, and releases it before the rebuild. `Ledger::read_window` is awaited on the runtime, never wrapped in `spawn_blocking`; the test with a single blocking thread pins that. The four gated `window_ref_tests` are listed in the manifest. Part of #265 Co-Authored-By: Claude Fable 5.1 * 3.x.x: switch the candidate outbox to the window reference `Candidate` drops the embedded `AuditBundle` for `window: WindowRef` and every builder input except the window: `found_block`, `payout_policy`, the nested `ctv` triple (whose presence selects the CTV builder), the stored `bootstrap_share` of an empty window, `audit_builder_version`, `signer_keys`, a `leased` flag and the now-required `coinbase_suffix_hex`. The block leaves the document for the `block_bytes` column, authenticated by `block_sha256`, which `candidate_sha256` covers. The write serializes, digests and checks the candidate before the append or enqueue transaction opens; under `ORDER_LOCK` the transaction re-reads `config_fingerprint FOR SHARE` and refuses a value the writer did not pin, re-inserts a `leased` candidate's as-issued balance snapshot through the shared writer, probes the window's first row (an absent row alerts and still publishes), and inserts only the prepared bytes with the six typed columns. Submit builds the slim candidate from `Prepared.window`, the `BundleInputs` captured where `build_bundle` read configuration, and the job's own bootstrap share; it clones no bundle and reads no config. The claim's `UPDATE … RETURNING` returns the columns and the block, and the decode, off the runtime, checks the document against them, the digest, `sha256(block_bytes)`, the header hash, the required fields and the two reference invariants. A pending row with a NULL anchor is a hard error naming the pre-007 frontend; an inline document is never decoded. Before any read the claim retries, with an alert naming both values, a stored builder version or signer pair that is not this binary's; a landed audit is authenticated against the block (coinbase, audit root as the witness reserved value, share snapshot, bits) and the claim then skips the rebuild and landing and continues to observe, renew and submit. Otherwise it takes `build_slots`, then `window_reads` inside one 60 s deadline around the awaited `read_window(Current)` and the borrowing builder run directly in `spawn_blocking`, and keeps `Arc` and the shares as `ClaimParts`. `Incomplete`, `SnapshotDigestMismatch`, `Decode`, `BalanceSnapshotMissing`, `TaskFailed` and the deadline retry with their own alerts; `PriorBalancesChanged` finishes only a block `observe_candidate` proves inactive and superseded; `Database` propagates. A `leased` candidate submits from `block_bytes` first, then rebuilds `AsIssued` and lands before any terminal outcome. Rebuilt parts are released off the runtime on every path. Landing takes the parts and does all whole-window work in one blocking closure: `verify_audit_parts`, the raw body, the counted canonical bytes, the leaves and accounts, and the share snapshot digest, which must equal the reference's. The balance fence compares the current set and the parts' through the semantic digest, so an as-issued set compares equal in any order. `persist_audit_snapshot` re-reads the range with the paged reader inside the landing transaction, page against slice on the blocking thread; `read_range` stays for the API. The terminal UPDATE NULLs the six window columns and `block_bytes` with `candidate`. `configure` refuses to pin a fingerprint onto a reset one while a pending row stores other signer keys, and the startup gate requires 007 with 009. The JSONB gate drops its enqueue entry, and every helper that built a candidate builds the slim one, handing the ledger-level landings the bundle's parts through `CandidateClaim::with_bundle`. Part of #265 Co-Authored-By: Claude Fable 5.1 * 3.x.x: cover the candidate switch on PostgreSQL Ledger-level, in `candidate_window_switch`: a two-page window enqueues a document with no array and no share row, under 1 MB, whose six typed columns equal its reference and whose block is stored as bytes; the writer fence refuses a reset or different fingerprint in the same transaction, share included; a missing window prefix row alerts and still publishes, and its read then meets `Incomplete`; a `leased` enqueue re-inserts the as-issued balance snapshot in the writer's canonical order, writes none when not leased, and refuses balances that do not hash to the reference; every preparation failure is refused without waiting for `ORDER_LOCK`; every column, digest, block, header, NULL-anchor, missing-field, invariant and inline-document disagreement is refused at claim; landing refuses parts that are not the referenced window and a claim without parts; the terminal UPDATE NULLs the six columns and the block for submitted and abandoned rows; the landed-audit authentication rejects a forged coinbase, root, snapshot or bits; and `configure` refuses a rotation over a foreign-signed pending candidate. Coordinator-level, in `window_switch_tests` behind a fake node: a stored builder version or signer pair that is not this binary's retries, naming both values, while the build slot is held; a landed audit is authenticated and the claim continues to `submitblock` without a build slot, and a forged landed coinbase or root is refused with the row left pending; `Incomplete`, a digest mismatch, changed balances on a current block and the rebuild deadline all retry with an alert and release their permits; `classify_window_error` keeps `TaskFailed` apart from corruption and propagates `Database`; the read, rebuild and landing complete while `ORDER_LOCK` is held, with one build worker; a `leased` candidate submits first, then lands from `AsIssued`, and finishes submitted, abandoned with the node's reason, or pending when its balance snapshot is missing; and `OffRuntime` releases on a blocking thread, on cancellation too. The fifteen gated tests are listed in the manifest. Part of #265 Co-Authored-By: Claude Fable 5.1 * 3.x.x: migrate the imported-audit tests to the slim candidate Slice 1 (#325) merged after this stack branched, and its tests in tests/support/ledger_2x.rs still built the pre-007 candidate: they read block.bundle from a Candidate and called candidate_with_bundle without a window reference or settlement inputs. The rebase merged them cleanly and the tree then failed to compile. The three helpers now take the TestCandidate the constructors return, so they reach the bundle beside the slim candidate, and land_confirmed attaches the claim's parts the way the other ledger tests do. The three constructor calls pass the window reference the bundle was built on, and the settlement inputs for the CTV case. No assertion changes. Part of #265. * 3.x.x: qualify lease renewal and the claim's canonical bytes at scale Incident 1: a candidate whose window holds N shares is claimed and processed under the short lease, and every renewal is read back from claim_expires_at. The largest gap stays below interval + 2 x timeout, the lease never lapses, and a successor ledger cannot take the claim. The bundle a claim rebuilds through the window-read entry point and the borrowing builder has the canonical bytes refresh built for the same job, and the landed audit carries that bundle's digest. Both run at PRISM_WINDOW_QUALIFY_SHARES (20,000 by default); the 400,000-share variants are ignored and run on a larger host. Part of #265 Co-Authored-By: Claude Opus 5 * 3.x.x: measure ACK latency and ORDER_LOCK hold across window sizes Incident 2: two frontends on one database submit ordinary shares at a fixed rate while one of them solves a block on an N-share window and its own submit loop claims, rebuilds, lands and submits it. The solving share's ACK and the bystander's p99 stay within max(2 x baseline p99 + 50 ms, 250 ms), every ACK is well under the share-commit timeout, and the block is enqueued exactly once. Latency is the in-process MiningBackend::submit wall clock; the socket-level harness (#271) does not exist yet. A pg_locks sampler on its own connection measures how long a solving append holds ORDER_LOCK at two window sizes with no other traffic: the larger size stays within a fixed slack of the smaller, and the candidate document is the same size at both and under 1 MiB. Part of #265 Co-Authored-By: Claude Opus 5 * 3.x.x: hold the JSONB gate's enqueue to a window-free row under 1 MiB At every measured size the enqueue must be accepted and every JSONB value it writes, the outbox candidate document included, must stay under 1,048,576 bytes. At the reduced sizes the enqueue, claim and landing phases must all run. The ratchet and its expected values are unchanged. Part of #265 Co-Authored-By: Claude Opus 5 * 3.x.x: make the window-read permit test actually wait before it looks The delay that gives an unpermitted read time to finish was an empty async block, so the check that the read had not finished ran at once and passed whether or not the entry point took a permit. It now sleeps 500 ms inside a 5 s bound, and fails when the permit acquisition is removed from the entry point. Part of #265 Co-Authored-By: Claude Opus 5 * 3.x.x: rebuild a CTV candidate from its stored settlement inputs No host-runnable test reached the CTV branch of the claim rebuild. A CTV-enabled frontend now claims a candidate whose stored direct floor differs from its configuration, and builds a different coinbase than the configured floor would, and the claim must land that candidate's own audit and submit it. Part of #265 Co-Authored-By: Claude Opus 5 * 3.x.x: pin the SQL order of the current-balance reads Nothing defended the ruling that the current-balance path keeps its SQL order: every fixture's balances already arrived in byte-wise canonical order. The carry table's text columns now take an ICU collation, under which the SQL order and the canonical sort disagree, and the snapshot and the current-balance window read must both return the SQL order. Part of #265 Co-Authored-By: Claude Opus 5 * 3.x.x: record what incident 2 must also assert once #324 is in the base After #324 a slow solve is acknowledged late rather than refused, so acceptance proves nothing and the latency percentiles stay the assertion. The doc comment and a TODO at the outcome checks name the counter and outcome to assert at rebase. Part of #265 Co-Authored-By: Claude Opus 5 * 3.x.x: assert incident 2's late-confirm counter now that #324 is in the base W3c wrote the qualification tests against a base that predated #324 and left a marked place for the assertions that only make sense once the commit gate is there. The base now carries it, so apply them: incident 2 asserts `qbit_prism_late_confirmed_shares_total` is zero on both frontends and that no acknowledgement, the block-solving share included, is `ledger-outcome-unknown`. Acceptance alone is no longer proof. #324 gives a share-pass append carrying a candidate its own `block_only_ack_timeout`, so a slow solve is never refused at the share deadline; it is answered late, or answered unknown. Either would have satisfied the acceptance checks with the serialization stall still present. Also set #324's two new `Config` fields in the qualification fixture. * 3.x.x: pin a refused candidate as a definite rejection, not an unknown outcome #324 decides what a miner is told when a block-only enqueue fails: `enqueue_failed_before_commit` answers a provably uncommitted failure with `ledger-confirmation-failed`, and anything else with `ledger-outcome-unknown`, which asks an operator to go and follow the outbox row. Slice 3 added the checks that guard the reference: the block's length, its header hash, its digest against `block_sha256`, the coinbase suffix, the reference invariants, and a leased candidate's balances. All of them are raised by `prepare_candidate` before the enqueue takes a connection, so none carries an `sqlx::Error` and all classify as definite today. That is a property of where the checks sit, not something the classifier knows about them, so it is pinned here rather than left to hold by accident: a corrupt candidate must never send a miner chasing a row that was never written. Verified by mutation: classifying a non-sqlx failure as unknown fails this test. * 3.x.x: say why the full-size qualification variants cannot reach 400k yet Their ignore reasons blamed RAM and the host. The real limit is #273: the job payload still holds three copies of the window (2 x AcceptedShare + 1 x CountedShare), so publishing a job for a 400,000-share window needs about 670 MiB and PostgreSQL refuses the write above its 256 MiB jsonb object limit. Every full-size variant fails at fixture set-up, before it measures anything. The count must also divide the window weight exactly, so the largest usable size today is 125,000, where all four variants pass in release: incident 1 50 renewals, max gap 118.25 ms against a 700 ms bound incident 2 phase-A p99 2.60 ms, phase-B p99 95.79 ms, bound 250 ms lock hold median 0.75 ms at 5,000 against 1.32 ms at 125,000 canonical refresh and claim rebuild both 139,040,610 B, same sha256 The candidate document stays flat at 1103 B and 1107 B, which is the property this slice exists to establish. * 3.x.x: migrate the audit-body normalization tests to the slim candidate #267 landed its own test file against the candidate that still carried an assembled bundle. This slice replaced that with a window reference, so the file keeps a `TestCandidate` pairing the stored candidate with the bundle a test still needs, as `tests/ledger_postgres.rs` does, and attaches the claim's parts through `with_bundle`. One case needed more than a shape change. The four negative cases hand the claim a window the ledger does not hold and require the durable-range proof to be what refuses it. A candidate now commits to a snapshot digest, so a reference taken from the unaltered snapshot would be refused earlier, at the reference check, and the proof under test would never run. `window_ref_for` therefore derives each altered window's reference from that window, leaving the proof as the only thing that can notice, which is what #267 wrote them to prove. * 3.x.x: pin the durable-range proof outside the settlement lock #267 moved the full-equality proof of a candidate's window out of the settlement transaction so the lock is not held for a window-sized read. This slice made that read paged, so it does not occupy a runtime thread either. Reconciling the two is the step that can quietly undo the first: keep this slice's `persist_audit_snapshot` through the rebase and the full read goes back under the lock. The landing stays correct, every other test still passes, and only the lock hold grows, so nothing in the suite can tell. Measured at 100,000 shares on #267's branch that was a 34% longer hold. The guard is deterministic and needs no large window. A second connection holds `SETTLEMENT_LOCK`, so the landing blocks on it once its proof has run. While it waits, one share's `ntime` is altered: full equality sees that, a count over the same anchored predicate cannot. The landing must then succeed, which it can only do if it had already read and compared the shares before taking the lock. Accepted history is immutable and a trigger enforces it, so the test lifts that trigger for the single statement, inside its own disposable schema, and puts it straight back. Verified by mutation: restoring the in-transaction range read fails this test with "the landing read share payloads under the settlement lock". * 3.x.x: serialise the coordinator test modules that share the ledger's locks The ledger's advisory locks are cluster-wide constants, not schema-scoped, so two coordinator test modules running in parallel contend for ORDER_LOCK even though each works in its own throwaway schema. Every module held a lock of its own, which serialised it against itself and left cross-module waits to `lock_timeout`, 5 s by default. #324's reconciliation tests hold ORDER_LOCK for up to about 1.5 s, and this slice adds three more modules that open a ledger. Once enough of them ran at once those waits stacked past the timeout: a full `--lib` run failed with "canceling statement due to lock timeout" in `candidate_lease_tests` and `window_switch_tests`, and passed on the next run. A suite that fails for a reason unrelated to what it tests is worse than a slow one. All seven modules now take one shared lock. The run goes from about 35 s to about 62 s, and three consecutive runs pass where the flake had reproduced. * 3.x.x: add migration 007 and refuse pre-007 candidate outbox rows Migration 007 mirrors 008 onto qbit_block_candidate_outbox: the same six window-reference columns, and a three-state CHECK whose body is identical to 008's apart from the table and constraint names. It also adds block_bytes, the nullable bytea the block moves into so the candidate document stays small, and a partial index on window_prior_balances_sha256 for the balance-snapshot GC. Nothing writes any of them yet; this is additive schema only. The runner applies 007 by membership, in its own block between 5 and 8, so it assumes neither 006 nor 008 has run. Before any DDL it refuses while the outbox still holds a pending row in a pre-007 shape: a native inline candidate, a NULL candidate, or storage_version other than 1 on the 2.x.x schemas that have that column, which is checked for first so the predicate does not error on a native schema. The refusal names every offending block_hash and the drain procedure, and the whole migration transaction rolls back, so a refused start leaves the schema untouched. The applied set is now [2, 3, 4, 5, 7, 8, 9], and the four exact migration assertions move with it rather than being weakened to a superset. Part of #265 Co-Authored-By: Claude Opus 5 * 3.x.x: retain the cluster fingerprint configure pinned or verified The candidate enqueue and save_job must fence against a fingerprint reset by re-reading qbit_prism_cluster.config_fingerprint FOR SHARE in their own transaction and refusing a write that is not this frontend's own. That needs the value configure settled at startup, so Ledger keeps it in a shared OnceLock, set only once the pin or the match has committed, and exposes it read-only through config_fingerprint(). configure keeps its signature and its mismatch refusal, and a refused call retains nothing. The rotation refusal in configure, and the writer fence itself, belong to the enqueue change and are not here. Part of #265 Co-Authored-By: Claude Opus 5 * 3.x.x: share the window reference, balance writer and page primitives WindowRef::from_snapshot is the synchronous reference builder both writers need: anchor_ms copied, the semantic balances digest, and for a non-empty snapshot the first and last share_seq, the length, and the share array's SHA-256 streamed through the existing DigestWriter, so the 233 to 260 MB serialized window at 400,000 shares is never built and the digest equals the one qbit_prism_audit_snapshots already stores. An empty snapshot gives the empty-window reference. Callers run it inside their own spawn_blocking. put_balance_snapshot is the one canonical writer of qbit_prism_balance_snapshots, on the caller's transaction: the sort by (order_key, recipient_id, p2mr_program_hex), the semantic digest that keys the row, and compact JSON of the sorted set, all off the runtime, then INSERT ... ON CONFLICT DO NOTHING. On a conflict it reads the stored bytes back and compares them with its own, because these rows are immutable and can only disagree through corruption. The enqueue and save_job and its repair all write through it. read_prior_balances and the current-balance read still keep their SQL order; the writer's sort defines the stored encoding, which is the order read_window(..., AsIssued) returns. probe_share_rows and read_range_paged are read_window's endpoint probe and page loop, lifted onto a caller's connection so the enqueue probe and the landing re-read share one implementation. The predicate, the ascending keyset order, the 4096-row page bound and the per-page count bound are unchanged, and read_window now calls them, with its signature and behaviour unchanged. The paged reader takes the caller's state and a per-page consumer, both of which travel to the blocking thread with the rows, so the landing re-read can compare each page with its slice of a vector it already holds without any whole-window work on the runtime; a dropped read still releases the accumulated state on a blocking thread. read_window_with_permit holds the caller's window_reads permit across the whole read. The permit is moved into a BlockingDrop declared before the read starts, so a cancellation releases it on a blocking thread after the page state queued its own release there, and never with the future; a return releases it inline, before the caller's rebuild. WindowError::TaskFailed separates a cancelled or panicking blocking hand-off from corruption. Every hand-off maps a JoinError to it instead of to Decode, so a cancellation is a retryable failure with its own alert and never takes the abandon path. Part of #265 Co-Authored-By: Claude Opus 5 * 3.x.x: derive the witness merkle leaves from an assembled block A claim rebuilds from stored inputs, and the witness leaves are the one builder input that scales with the block, so they are re-derived rather than stored: one 64-hex string per transaction beside the candidate would grow the document with the block. witness_merkle_leaves_from_block skips the 80-byte header, reads the compact-size transaction count, walks past the coinbase and returns double_sha256 of every remaining transaction's full serialized bytes. That is sound because assemble_submission appends the template's transactions verbatim after the coinbase, so the result equals witness_merkle_leaves_hex(&transactions_from_template(template)) for the block built from that template, with or without segwit transactions. The segwit-aware walk is strip_witness_transaction's own, factored into a helper that also reports where the transaction ends, which is what a block walker needs; strip_witness_transaction keeps its trailing-byte check and its output. A truncated block, trailing bytes, a zero count and a count that disagrees with the transactions present are all errors, never a partial list of leaves. Part of #265 Co-Authored-By: Claude Opus 5 * 3.x.x: version the audit builders and key the CC5 goldens by it A stored window reference reproduces its bundle only under the builder that first built it. The schema strings do not capture that, because the share shape chooses them rather than the builder logic, so qbit-prism gains AUDIT_BUILDER_VERSION, starting at 1. A candidate and a prepared job store it, and a rebuild refuses a reference at another version instead of rebuilding it with the current builder. The bump rule is on the constant: any change that alters canonical_audit_bundle_bytes for the same inputs bumps it in the same commit. The CC5 golden test now selects its expected digests from a table keyed by the constant, and fails when no entry exists, so a new output is recorded as a new entry beside the old one and review sees both. Version 1 holds the existing literals, unchanged. Part of #265 Co-Authored-By: Claude Opus 5 * 3.x.x: cover 007, the fingerprint and the window primitives on PostgreSQL Ten gated database cases, all listed in test/prism-gated-tests.txt. They go in a new file rather than the shared 2.x.x support module, which another open change appends to. Migration 007: a frozen 2.x.x schema carrying only terminal outbox rows upgrades with every row and field intact and the seven new columns NULL; each of the three pre-007 pending shapes is refused by name, with nothing from 007 applied and no version 7 row, and the same binary starts once the row is drained; a database at [2, 3, 4, 5, 8, 9] applies 007 and only 007, leaving every other version's applied_at untouched; the CHECK accepts exactly the three states across all 64 null combinations and rejects a short or uppercase digest, a zero or out-of-range count and a zero sequence; and the retention index is the partial one on window_prior_balances_sha256. config_fingerprint() returns what configure pinned or verified, is shared by clones, and stays empty after a refused conf… * v3.0.0 feat: time selected direct ledger pool acquisitions (#368) * feat: extend pool acquisition timing to direct ledger queries Reuse the existing cancellation-safe wait guard for a shared Ledger::acquire boundary and eight direct query paths. Preserve SQL and transaction behavior, and qualify elapsed timing, cancellation, and checkout release against PostgreSQL. Refs #352. * test: include publisher acquisitions in live pool observations The HTTP exhaustion test observes the aggregate pool histogram, now including payout-revision and heartbeat waits. Require every explicit cancellation to remain visible while preserving exact per-attempt assertions in the isolated tests. * docs: describe partial direct-query pool timing coverage Refresh canonical metric and alert guidance and regenerate the inventory and review-only provisioning patch from the checksum-pinned source snapshot. Alert expressions, thresholds and role exports are unchanged. * docs: complete direct-query timing coverage guidance Co-Authored-By: OpenAI Codex * docs: align acquisition coverage in compatibility explanations --------- Co-authored-by: OpenAI Codex * v3.0.0 refactor: share pool acquisition observation (#376) * refactor: share pool acquisition observation Use one optional-metrics guard for ledger and collector checkout waits. Choose Tokio monotonic time explicitly while retaining the existing advisory-lock clock and all checkout, error, and cancellation boundaries. Add first-poll, unattached cleanup, and SQLx BEGIN boundary regressions. Refs #352 * refactor: place checkout timing beside metrics events Rename the async timing boundary to time_pool_acquire and expose it through metrics while leaving the recording hook name unchanged. Consolidate helper tests under events, retaining unique result, clock, cancellation cleanup and concurrency assertions, and remove the ledger free forwarding function. Preserve advisory-lock timing and every PostgreSQL checkout/BEGIN regression. Refs #352 * v3.0.0 ci: make pinned tool downloads resilient to release outages (#380) * 3.x.x: Ride out release CDN outages when installing pinned CI tools (#374) curl --retry 3 gave cargo-deny and hadolint about seven seconds of retries, so a GitHub release CDN 504 failed both required jobs. - Retry transient failures every 15 seconds for up to 180 seconds (12 retries, 10-second connect and 30-second per-attempt limits) and kill the download at a hard 210 seconds. A 404 still fails at once. - With PINNED_TOOL_CACHE_DIR set, keep the original release asset under tool/version/sha256. A cached asset must be a regular file and is copied to private scratch and checked against the pin before anything is extracted, installed or run; a bad entry fails without refetching. A new download is checked, then published through a checked temporary file renamed into place. - --cache-metadata writes the actions/cache key and path, so both jobs restore and save the asset with actions/cache while the pins stay in the script. * ci: cancel pinned tool downloads promptly * v3.0.0 fix: persist settlement inputs before resuming stored jobs (#372) * fix: persist issued settlement inputs for job resume Refs #370 * refactor: centralize stored settlement input compatibility Keep compact input conversions together and extract the inline resume compatibility decision without moving fee admission, cancellation owners, or the original expiry calculation. Document why Prepared keeps a typed input view and why resumed work cannot authorize original compact capture. Require an explicit ctv key in direct compact decoding, matching the storage reader that already rejects omitted keys via canonical re-encoding. Preserve explicit null, flat compact fields, field order and legacy bytes. Incompatible inputs still return a miss before the live fee check; compatible work retains that check. Cover the policy directly and exercise missing-CTV rejection through the real PostgreSQL reader and retry path. Refs #370 * v3.0.0 fix: retain window read admission through blocking cleanup (#373) * fix: retain window admission through blocking work and cleanup Share one completion owner across mapping and payload destruction so cancellation cannot release a reader slot before its work finishes. Preserve the public read APIs and caller deadline. * fix: distinguish payout task failure from balance corruption Keep blocking JoinError as TaskFailed and reserve Decode for malformed balances. Add focused error regressions and record reader completion, caller compatibility, and local validation. Refs #273 * refactor: encapsulate window paging and blocking handoffs Keep the paging cursor private and return admission-owned state, reuse BlockingDrop mapping for payout digests and balance snapshots, and consolidate cleanup tests. Retain whole cleanup ownership through runtime shutdown and cover queued destructor panic; real PostgreSQL corruption coverage remains authoritative. Refs #273. Verified 15 focused tests, 267 library tests, 62 PostgreSQL integration tests, Clippy, and formatting without changing clocks, stored bytes, version, or Git signing configuration. * 3.x.x: re-anchor the capacity-readiness runbook to native telemetry and guard lab/prism doc references (#318) * 3.x.x: re-anchor the capacity-readiness runbook to native telemetry (#303) docs/prism-capacity-readiness.md documented and told the operator to run lab/prism/metrics.py and lab/prism/process_telemetry.py, which #244 deleted; lab/prism/ holds only Dockerfile on this line. Decide per section instead of deleting wholesale: - Retire the Python-specific material: the interpreter/allocator and component-cardinality families, the heap census, malloc_trim, and the MALLOC_ARENA_MAX storm instrument. Each keeps a short past-tense note that it applied to the retired Python coordinator and names what replaces it or who owns the gap (#278 registry, #279 inventory). - Re-anchor what is runtime-independent on the #308 registry: the process block (process_resident_memory_bytes, the collector_* state, runtime_* gauges, the #277 snapshot block), the RSS reading order, the 2.0x resident-set bound as one awk pass over the sample CSV, and the 24 h soak runbook using docker exec, /proc/1/status, and curl against the audit listener on 3341. Every command parses and resolves on this branch. - Mark the Validate Qualification Evidence gate deferred: the native validator exists, but no producer on this branch emits the v2 file and the strict key set still requires three retired settings. #288 re-keys the validator; #291 produces the first native artifact. tests/test_docs_lab_prism_references.py guards the recurrence: no runnable python -m lab.* / python lab/*.py command in docs/ may target a module absent from the branch, and every lab/prism path or lab.prism module reference must resolve against git ls-files, except GitHub URLs pinned to a commit SHA. The pre-existing residue in prism-overload-alerts.md, prism-coordinator-refactor/ and prism-rust-migration.md is ratcheted at today's counts; the test fails when a file grows or when it improves without lowering its ceiling. Discovery picks the module up in the existing CI shards. Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_014VHcDK2anZJPx8Gu1LZvuN * 3.x.x: correct the signal and #278 scope claims in the capacity runbook (#303) The native server is PID 1 in the shipped container with no init and installs handlers for SIGTERM and SIGINT only, so the kernel drops SIGUSR1 rather than terminating the process. Thread and open-file-descriptor gauges fell outside #278's cutover-minimum scope, and #288's docs CI check is still to land. Co-Authored-By: Claude Opus 5 (1M context) * 3.x.x: join shell continuations before scanning doc commands (#318 review) `dead_commands` matched each physical line on its own, so a documented `python3 -m \` + `lab.prism.deleted` (or `python3 \` + `-m lab.x`, `python3 -u \` + `-m lab.x`, `python3 \` + `lab/x.py`) slipped past the guard even though the fence runs the deleted module. Join backslash continuations into logical shell lines first and report the command at its first physical line. `dangling_references` and the RATCHET ceilings are untouched; no wrapped `python ... lab` command exists in docs/ today. Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_01YH4dPMQC6xqq9dreB1puTb * 3.x.x: keep the warm-up boundary sample in the RSS-bound baseline (#318 review) The Python evaluator this awk pass replaced put a sample taken exactly warmup seconds after the first one into the warm-up (`t <= warmup_seconds`) and judged only later samples against the bound (`t > warmup_seconds`). The awk used a strict `<`, so with the runbook's regular 300 s cadence a transient peak landing exactly at hour one fell out of the baseline and could turn a pass into a fail. Use `<=` so the verdict is the Python tool's, and say so in the prose. Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_016PioJBmGucLpZZ9HhQgUj1 * 3.x.x: state the RSS-bound span floor the check actually enforces (#318 review) The resident-set bound prose said the awk command "refuses a series shorter than the soak". It does not: `last - t0 < 82800` refuses a series spanning less than 23 hours from its first sample to its last, exactly as the retired Python evaluator's RSS_BOUND_DEFAULT_MIN_SPAN_SECONDS did (23 * 3600, measured from the first sample after re-basing). The threshold and the expression are unchanged; a 23 h floor is parity with the tool this runbook qualified under, and 86,400 s from t0 would reject an honest 24 h soak whose last 5-minute sample lands just short. The one-hour slack is the reason the Python source recorded on the constant ("a 24 h soak with an hour of tolerance for a late first sample", #234); the prose now says so and keeps step 2's 24 h run. Verified with the unchanged awk program on synthetic 300 s-cadence series: 82,800 s exits 0, 82,500 s exits 2. Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_013yFEcAtVPzfzMiwYmFskLD * 3.x.x: parse real CPython option forms in the doc-command guard (#318 review) The command regexes accepted only single-letter switches between the interpreter and `-m` or a script path, so `python3 -OO -m lab.prism.deleted` or `python3 -X dev -m lab.prism.deleted` ran deleted code while passing contract (a). Share one interpreter-plus-options sub-pattern that follows `python3 --help`: clustered flag letters, `-W`/`-X` with an attached or following argument, and `--check-hash-based-pycs `. `-c cmd` stays excluded because it runs its argument and ends the option list; CPython rejects the `=` spelling of the long option, so none is accepted. Also scan version-suffixed interpreters (`python3.12`), as the CI images name them. Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_01FcDKnhKaUD45rfBYeYTGH4 * 3.x.x: record the image Compose resolved from the soak env files (#318 review) Step 1 of the testnet soak runbook built the coordinator image through Compose with the deployment's env files, then inspected `${PRISM_COORDINATOR_IMAGE:-qbit-lab-prism-coordinator:local}` as a standalone shell expansion. An operator who sets `PRISM_COORDINATOR_IMAGE` only in an env file, without exporting it, would build the configured image and inspect the default one instead. Substitute the env files once, in a `compose` function, and let the same invocation build the image and resolve its name through `config --format json`, reading the `prism-coordinator` service record with `python3`. `config --images prism-coordinator` is not used because it prints the images of the services the coordinator depends on as well (`postgres:16-alpine` and the qbitd image), one per line and in varying order, so `docker image inspect` would receive one multi-line argument and fail; `docker compose config` has no `--no-deps`. Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_01JWTB41CrR7dJ353fGub6HL * 3.x.x: match quoted and ./-relative targets in the doc-command guard (#318 review) A ratcheted doc could swap one dangling `lab/prism/...` mention for a runnable `python3 -m 'lab.prism.deleted'` or `python3 ./lab/prism/deleted.py` command: rule (b) still counted the path, so the ceiling held, and rule (a) required an unquoted module or `lab/` straight after whitespace, so it saw no command. Both command rules now share one `quoted` sub-pattern that accepts a target in matching single or double quotes, the script rule accepts `./` prefixes, and the reported missing path is the normalised one. `-m` may also close a flag cluster and take its module attached. On CPython 3.12 `-mjson.tool`, `-Im json.tool`, `-OOm json.tool`, `-Imjson.tool` and `-m'json.tool'` all run the module, so those spellings are matched; `-Wm` and `-Xm` hand `m` to `-W`/`-X` and stay unmatched, as does `-c`. A mismatched quote is a shell syntax error, runs nothing and matches nothing; the prose rule still sees the reference. The module docstring now states the contract's limit: rule (a) is a lexical check of direct python invocations and does not follow `cd`, `PYTHONPATH`, aliases or shell variables. Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_01YL7EVncHtFRE74z559dGFf * 3.x.x: reject malformed RSS rows in the bound check (#318 review) The `awk` port of the resident-set bound skipped every row with fewer than two fields under the same rule that skipped `#` comments, so a sample lost during an RSS excursion (a truncated line in the CSV) disappeared silently and a valid late sample could still pass the soak. The deleted `read_rss_samples` raised on any non-blank, non-comment line that was not `seconds,rss_bytes`, and the `rss-bound` CLI turned that into exit 2. A non-numeric RSS field was worse than skipped: `abc > n` compared as strings and reported a breach. Skip only comment and blank lines, require a numeric first and second field on every other row (extra trailing fields stay tolerated, as they were in the Python tool) and report the offending row as unusable input with exit 2. `exit` inside a main rule still runs `END` in every awk, so the failure is recorded in a variable that `END` checks before any verdict logic; verified on the macOS awk (20200816). `-1` samples are still ignored, after validation. Co-Authored-By: Claude Fable 5.1 * 3.x.x: sort RSS samples by time before the bound check (#318 review) The deleted `evaluate_rss_bound` sorted the series by time before it re-based on the first sample; the `awk` port took the origin from the first physical row. A nonmonotonic file, such as two partial soak logs concatenated with the later one first, put a late sample at the origin, so the warm-up window swallowed every earlier sample (including a post-warm-up excursion), the baseline rose to the excursion and the verdict flipped or the span check refused the file. Sort the file numerically on the first field with `sort -t, -k1,1n` before the `awk` pass, so the origin is the earliest sample as it was in the Python tool. Comment and blank lines sort as zero and are skipped wherever they land, a malformed row is still caught after sorting, and the pipeline exits with awk's status. Verified with the macOS sort (2.3-Apple) and awk (20200816) against the deleted evaluator: a breaching series with its hour-5 sample moved before the hour-0 rows now fails as the Python tool failed it, where the previous command refused a 24 h file for its span and passed a 28 h one. Co-Authored-By: Claude Fable 5.1 * 3.x.x: capture the runtime and pool series the RSS reading order cites (#318 review) Item 3 of "What to look at first when RSS climbs" names `qbit_prism_runtime_task_stalled{task}`, `qbit_prism_runtime_poll_lag_seconds{task}` and `qbit_prism_database_pool_acquire_seconds` as the evidence that a climb is a blocked worker holding buffers rather than a leak. The five-minute capture loop kept only the stall gauge, so the poll lag and pool wait at a breach could not be read back: the full snapshots are taken at hour 1, hour 24 and the breach itself, and cannot recover what those series showed at an earlier sample. Add both families to the `grep -E` filter. The poll-lag gauge renders with its `task` label and matches the existing `[ {]` anchor as it is. The pool histogram never renders under its bare family name: the registry emits `..._bucket{result=...,le=...}`, `..._sum{result=...}` and `..._count{result=...}`, as the observability tests assert, so the pattern names those three suffixes explicitly and keeps the anchor after them. The sentence after the loop now says the log carries the reading order's runtime and pool series, so a breach is read at its own sample rather than the nearest hourly snapshot. Co-Authored-By: Claude Fable 5.1 * 3.x.x: match quoted interpreter names in the doc-command guard (#318 review) A ratcheted doc could swap one dangling `lab/prism/...` mention for a runnable `"python3" -m lab.prism.deleted` command: the shell strips the matching quotes and runs Python, rule (b) still counted the module so the ceiling held, and rule (a) required whitespace straight after a bare interpreter name, so it saw no command. The interpreter now goes through the same `quoted` sub-pattern as the target, bare or in matching single or double quotes, with the quote in its own named group because `re` rejects one group name used twice in a pattern. The word boundary before `python` is kept, so `/usr/bin/python3` and `env python3` still match. Inside `sh -c "python3 -m lab.x"` no quote closes right after the name, so the interpreter quote backtracks to empty and the inner command is read as before; a mismatched quote around the name is a shell syntax error, runs nothing and matches nothing, and the prose rule still sees the reference. No ratchet ceiling changed: the tracked docs contain no such command today. Co-Authored-By: Claude Fable 5.1 * 3.x.x: pin the deleted-test map's retired module paths to their last commit (#318 CI) `docs/prism-deleted-test-map.md` landed on 3.x.x with #322 after this branch's merge-base, and it names three `lab/prism/*.py` modules that #244 removed: `candidate_window.py`, `bundle_compiler.py` and `endpoint_registry.py`. The reference guard this branch adds scans every tracked doc on the merge ref, so CI's "Python unit tests (shard 1)" failed on those three lines while the same test passed on the branch alone. The guard's contract offers two ways to name retired code: describe it in words or link a commit-pinned GitHub URL, which is stable history and exempt. The three mentions are now links to the files at `700fae0`, the last commit before #244 deleted them, with the module basename as the link text so the visible path no longer claims to exist on the branch. The table's wording is otherwise unchanged. Co-Authored-By: Claude Fable 5.1 * 3.x.x: pin candidate_window.py to the commit that last carried it (#318 CI) `lab/prism/candidate_window.py` was removed before #244, so the link 4d6d0c8 added at `700fae0` pointed at a commit without the file. The other two modules did exist there; this one now links to `^`, the last commit before its own deletion. Verified with `git cat-file -e` on the pinned tree. Co-Authored-By: Claude Fable 5.1 * 3.x.x: pin candidate_window.py to v2.0.2, where the module lives (#318 CI) bd35536 rewrote the `candidate_window.py` link with a malformed revision (`blob/^/`), which the reference guard no longer recognised as a commit-pinned URL. The module never existed on the 3.x.x line; it was added on the 2.x line by the v2.0.0 consolidation (#246) and is present at v2.0.2, `504846c`, the same pinned commit the guard's own tests use as their example. The link now points there, checked with `git cat-file -e` against that tree before committing. The guard passes on the merged tree. Co-Authored-By: Claude Fable 5.1 * 3.x.x: gate the soak pre-check on a fresh metrics body (#318 review) The soak pre-check matched any x-prism-metrics-state value, so a stale or unavailable body whose cached process gauge still read 1 printed two lines, grep exited 0, and the prose told the operator the RSS series was trustworthy. Every gauge in the body is a scrape-time render of cached observations, and the header is what says whether that render is current. Replace the grep with an awk verdict in the style of the RSS bound check: it sets a flag on the exact `x-prism-metrics-state: fresh` line and on the process collector gauge at 1, prints `ready` and exits 0 only when both are present, and otherwise names the failed condition and exits 1. The sentence after it now says both lines are required, that a stale or unavailable body can still carry a cached gauge at 1, and that the run must not start until the check passes. The step-3 capture loop is unchanged; it records the state header whatever its value on purpose. Co-Authored-By: Claude Fable 5.1 * 3.x.x: require a runnable entry point for -m targets in the doc-command guard (#318 review) The doc-command guard accepted a `python -m lab.pkg` command when only `lab/pkg/__init__.py` was tracked, because it reused the prose contract's candidate list and took its first two entries. CPython does not run such a package: on 3.14, `python3 -m lab.pkg` against an `__init__.py`-only package, or against a bare directory, prints "No module named lab.pkg.__main__; 'lab.pkg' is a package and cannot be directly executed" and exits 1. It runs the package only when `lab/pkg/__main__.py` exists, and it runs `lab/auxpow/vardiff.py` even though `lab/auxpow` has no `__init__.py`. Commands now resolve through a separate `runnable_candidates` helper that admits only `.py` or `/__main__.py`; the prose contract keeps `module_candidates`, so a `lab.prism.pkg` mention of an `__init__.py`-only package is still not dangling. A dead `-m` command reports both paths joined by `or`, so a reader is not told to create `lab/pkg.py` beside a tracked package that merely lacks `__main__.py`. The docstring states the new rule, the passing fixtures gain `lab/pkg/__main__.py`, and tests cover the four package shapes and the prose exemption. Co-Authored-By: Claude Fable 5.1 * 3.x.x: pin the B8 parity record's retired module paths to the commit it cites (#318 CI) `docs/prism-b8-miner-parity.md` landed on 3.x.x with #313 after this branch's last merge of the base, and its reference line names `lab/prism/tip_refresh.py` and `lab/prism/job_delivery.py`, which #244 removed. The reference guard this branch adds scans every tracked doc on the merge ref, so CI's "Python unit tests (shard 1)" failed on that line while the test passed on the branch alone. The line already anchors itself to a 2.x.x commit; the two module mentions are now links to those files at `95ffe06`, checked with `git cat-file -e` against that tree, with the basenames as link text so the visible path no longer claims to exist on the branch. The guard treats a commit-pinned GitHub URL as stable history. The sentence is otherwise unchanged. Co-Authored-By: Claude Fable 5.1 * 3.x.x: parse quoted option arguments in the doc-command guard (#318 review) The doc-command guard read a `-W`, `-X` or `--check-hash-based-pycs` argument only as a bare non-whitespace word, so `python3 --check-hash-based-pycs 'always' -m lab.prism.deleted` and `python3 -X 'dev mode' -m lab.prism.deleted` ran and yet were not reported: the quoted mode broke the `always|default|never` match, and the space inside the quotes ended the `-X` argument early. The bare word also swallowed an unterminated quote, so `python3 -X 'dev" -m lab.prism.x`, which bash rejects before Python starts, was reported as a dead command. On CPython 3.14, `-X 'dev'`, `-X "dev mode"`, `-X'dev'`, `-X"dev mode"`, `-W 'error'`, `-W"error"`, `-W 'error'::DeprecationWarning` and `--check-hash-based-pycs "always"` all run `json.tool` exactly as their bare spellings, because the shell strips matching quotes, attached or not, before CPython sees them. An option argument is now a shell word: runs of unquoted characters alternating with matching-quoted strings, non-empty, ending at whitespace. The runs and strings alternate rather than nest, so one word has a single parse and a long argument cannot backtrack exponentially. `-W` and `-X` take that word attached or after whitespace; `--check-hash-based-pycs` reuses `quoted()` around its three modes with its own group name, since the command patterns already use that group for the interpreter and the target. An unquoted run holds no quote character, so a mismatched quote matches no option and the line is left to the prose contract. Tests cover each quoted form against missing and existing targets, a wrapped form, and the mismatched forms. Co-Authored-By: Claude Fable 5.1 * 3.x.x: accept -R and the -- terminator in the doc-command guard (#318 review) `PYTHON_FLAG` listed every single-letter option but `-R`, so `python3 -R -m lab.prism.deleted` ran and was not reported. On CPython 3.14 the option lines of `python3 --help` are `-b -B -c -d -E -h -i -I -m -O -OO -P -q -R -s -S -u -v -V -W -x -X`, with `-h : print this help message and exit (also -? or --help)`; `-R` was the only letter missing, and `python3 -R -m json.tool --help` and `python3 -Rm json.tool --help` both run the module, so `-Rm` joins the adjacent `-m` spellings. Separately, the standard `--` terminator before a script path evaded `SCRIPT_COMMAND`: `python3 -- s.py`, `python3 -O -- s.py` and `python3 -OO -- ./s.py` run the script. `R` is in the flag class and `SCRIPT_COMMAND` accepts an optional `--` followed by whitespace before the script target. `MODULE_COMMAND` does not: after `--` CPython takes `-m` as a script name, and `python3 -- -m json.tool` fails with "can't open file '.../-m'", so that line runs nothing and stays with the prose contract, which a test asserts. Tests cover `-R` and `-ER -X dev` in the option table, `-Rm`, and `--` before a script with missing and existing targets, in a wrapped form, and with `./`. Co-Authored-By: Claude Fable 5.1 * 3.x.x: invalidate the soak when the coordinator restarts (#318 review) `compose.yaml` gives `prism-coordinator` `restart: on-failure`, so a coordinator that exits mid-soak is started again by Compose inside the same container, and the step-3 capture loop, which only read `/proc/1/status`, would go on sampling the new PID 1 without a word. The CSV would then splice several short lifetimes into one 23 h span, each restart's low RSS and reset share counters would hide the growth, and the bound, which assumes one process, could pass. The loop now reads `docker inspect` for the container's status, `StartedAt` and `RestartCount` once before it starts and again before every sample, appends each reading to `soak-process.log`, and on any change prints both readings with the time to stderr and stops. The container is the same across a Compose restart, so `$c` stays the handle and `docker inspect` on it is the right probe. The prose says the bound is evidence about one process, why the restart policy would otherwise hide a restart, that the log lets a later reader show the run was one process, and that a change invalidates the run: keep the message and the log, attach `docker logs "$c"`, and start over from step 2. Step 6 gets one sentence saying such a run is not judged. Verified against a throwaway container with the same restart policy, where two policy restarts moved `StartedAt` and took `RestartCount` from 0 to 2, and with a stub `docker` that changes `StartedAt` on the second iteration. Co-Authored-By: Claude Fable 5.1 * 3.x.x: invalidate the soak when an RSS sample is missing (#318 review) The step-3 capture loop piped `/proc/1/status` straight into `awk`, so when `docker exec` failed transiently, or the body had no `VmRSS:` line, the awk printed nothing and exited 0 and the loop went on without a word. The bound check only enforces the overall 23 h span from the first sample to the last, so two successful samples 82,800 s apart pass it while the missing five-minute samples between them can hide an excursion, and the judge's malformed-row rule cannot catch this because no row is written at all. Each iteration now writes exactly one `seconds,bytes` row or stops the run. The body is read into a variable first and its exit status kept, because the block is pasted into an operator's bash or zsh and a pipeline's status is its last command's without `pipefail`; the awk prints a row only when the body carries exactly one `VmRSS:` line in kB; and an empty or malformed result prints the time, the exec's exit status when it was non-zero, and the body that was read to stderr, then breaks, in the restart guard's style. The variable is `body`, not `status`, because `status` is read-only in zsh. The prose says a missing sample invalidates the run for the same reason a restart does, that a transient `docker exec` failure is therefore not skipped, and that the operator starts over from step 2; step 6 now says a run stopped on a missing sample is not judged. Verified with `sh -n`, `bash -n` and `zsh -n` on the extracted loop, with a stub `docker` in sh, bash and zsh that returns a normal body, then a body without `VmRSS:` or a non-zero exit, where each run wrote one row and stopped with the message, and with a live throwaway container, where the exec line produced one row and, after the container was removed, an empty body with exit status 1. Co-Authored-By: Claude Fable 5.1 * 3.x.x: tokenize shell-quoted targets in the doc-command guard (#318 review) The command contract accepted a `-m` target or script path that was entirely bare or entirely inside one pair of quotes, so a target the shell assembles from several pieces slipped through: `python3 -m "lab.prism."deleted`, `python3 -m lab."prism".deleted` and `python3 "lab/prism/"deleted.py` all run the missing target (bash 3.2 and CPython 3.14 run `python3 -m "json."tool` exactly as the bare spelling), yet the scanner reported nothing, and the prose rule saw only `lab.prism`, which resolves to the tracked directory. The target is now read as a whole shell word: the tokenizer that already parsed quoted option arguments is generalised to `SHELL_WORD` and used for the `-m` target and the script path as well. `dead_commands` removes the matching quotes, strips any `./` prefixes from a script path, and only then reads the word against the target grammar and resolves it, reporting the normalised target. A word that then names no `lab` module or script, such as `$VAR`, another package, or `"lab.prism."` with nothing after the dot (which CPython answers with "No module named lab.prism."), is not a `lab` command. An unterminated opening quote still matches nothing. `quoted()` stays for the interpreter name and the `--check-hash-based-pycs` mode, whose values are fixed words with nothing to concatenate. A word ends at whitespace, at a bash metacharacter (`;`, `|`, `&`, `(`, `)`, `<`, `>`), at a backtick, or at a quote that opens no string, so the forms the previous `\b` boundary caught keep being caught: `` `python3 -m lab.x` `` in inline code, `python3 -m lab.x; ...`, `$(python3 lab/x.py)` and the inner command of `sh -c "python3 -m lab.x"`. Backslash escapes are out of scope and the docstring now lists them with the other things the lexical check does not follow. `dangling_references` is unchanged: the prose rule is about mentions, and `"lab.prism."deleted` in prose is not a mention of a module, while the runnable form is now caught by the command rule. Co-Authored-By: Claude Fable 5.1 * 3.x.x: skip patch and diff files under docs/ in the reference guard (#318 CI) The guard read every tracked file under `docs/`, and #320 (22bd1a2) added `docs/prism-alert-rules-qbit-tools.patch`, a unified diff of qbit-tools' Ansible monitoring role. Its line 991 is a `-` line quoting a comment that used to name `lab/prism/block_candidates.py` in that other tree, so the prose contract reported it as a dangling reference on this branch. The mention is quoted history, not this repository's prose: the patch records another tree's text before and after a change, editing a line of it would break the patch, and neither contract has anything to say about it. Both contracts now skip tracked files under `docs/` whose suffix is in `QUOTED_HISTORY_SUFFIXES` (`.patch` and `.diff`). The selection is factored into `documented(paths)`, which `setUpClass` filters to files, so the rule is testable without git: a new test checks that a patch and a diff are left out while `.md`, `.json` and `.yaml` paths stay in, and that no selected doc carries a skipped suffix. The scope is not narrowed to Markdown: every other suffix is still read. The module docstring states the exemption. With the skip in place the real-docs tests pass again, `dead_commands` over the 59 documents in scope is empty, and every RATCHET count equals its ceiling. Co-Authored-By: Claude Fable 5.1 * 3.x.x: tokenize the interpreter word in the doc-command guard (#318 review) The command contract accepted an interpreter name that was wholly bare or wholly inside one pair of quotes, so a name the shell assembles from pieces slipped through: bash 3.2 runs `"python"3 -m json.tool`, `'pyth'on3 -m json.tool`, `python"3" -m json.tool` and `"/usr/bin/pyth"on3 -m json.tool` exactly as `python3`, yet the scanner reported nothing for `"python"3 -m lab.prism.deleted`, `'pyth'on3 -m lab.prism.deleted` and `python"3" lab/prism/deleted.py`, while `/usr/bin/python3 …` and `env python3 …` were caught and `mypython3 …` was correctly not. The interpreter is now read as a whole shell word, the way 43609d1 reads the target. `dead_commands` strips the matching quotes, takes the basename after the last `/`, so `/usr/bin/python3` keeps matching, and accepts the word only if it is wholly `python`, `python3` or `python3.N`; `mypython3`, `cpython3`, `python2` and `python3x` are other programs and are not commands, while the prose contract still sees their reference. Option parsing, the `-m` and script handling and the target tokenizer are untouched: `SHELL_WORD`, `PYTHON_OPTION` and `MODULE_OPTION` are the same strings as before, and the same 21 commands are found across the tracked documents by the old and the new grammar. Two details make reading a word from every position of a line sound. First, a word may start right after a closing quote, so `"/usr/bin/"python3` is one candidate and its tail `python3` another: the command patterns are lookaheads that consume nothing, and `python_commands` drops a candidate that starts inside the interpreter word of the one before it, whether or not that word named Python, so `"/usr/bin/"python3 lab/prism/x.py` is reported once and `"my"python3 -m lab.prism.x` (`mypython3` to bash) not at all. Second, the interpreter word's quoted pieces hold no whitespace: a program name is one word, and with arbitrary quoted pieces `"$c" sh -c "python3 …` read from `$c` would pair the closing quote of `"$c"` with the opening quote of the `sh -c` string and hide the `python3` inside it. A word starts where no unquoted word character precedes it, so `env python3 …`, `docker exec "$c" python3 …`, `sh -c "python3 …"`, inline-code `python3 …` and `$(python3 …)` are caught as before, and a mismatched interpreter quote still matches nothing. The docstring and the grammar comments now cover the interpreter alongside the target. Co-Authored-By: Claude Fable 5.1 * 3.x.x: invalidate the soak when a metrics sample fails or is stale (#318 review) The step-3 capture loop piped the `/metrics` scrape through `tr`, `grep` and `sed` straight into `soak-metrics.log`, so when the `curl` failed, the body carried none of the expected families, or its header said `stale` or `unavailable`, the trailing `sed` still exited 0 and the loop went on without a word. The correlated series is what reads an RSS excursion back at its own five-minute sample, so the log could omit exactly the interval where RSS breached while the run stayed eligible for judgment, and the step-2 gate could not catch it because it proves the body fresh once, at the start. Each iteration now appends the filtered lines or stops the run, in the RSS guard's style. The response is read into a variable first and the exec's exit status kept; the `\r` strip runs as a second step on the variable rather than in the same pipeline, because the status of a command substitution is its last command's without `pipefail`, which not every operator shell sets. An awk then requires, in order, that the exec succeeded, that the body carries exactly the line `x-prism-metrics-state: fresh`, and that the RSS gauge family `qbit_prism_process_resident_memory_bytes`, the series the reading order starts from, is present; the other families in the filter are not required, since a histogram bucket or a `runtime_task_stalled{task}` series can legitimately be absent from a scrape. On any failure the loop prints the time, the condition that failed (the exit status, the state header actually seen, or the missing family) and the header lines to stderr, then breaks. The `grep -E` filter and the `sed` timestamp prefix are unchanged; the restart guard and the RSS-sample guard are untouched. The prose holds the metrics sample to the missing-sample rule and says the step-2 gate proves freshness only once, and step 6 now lists a failed or non-fresh scrape among the reasons a run is not judged. Verified with `sh -n`, `bash -n` and `zsh -n` on the extracted loop, and with a stub `docker` and a stub `sleep` in sh, bash and zsh that serve a normal `/proc/1/status` body to the `cat` call and, for the `curl` call, a fresh body (two samples, eight rows, no message), then from the second sample a `curl` exit 7, a `stale` header, or a fresh header without the RSS gauge: each of the nine failure runs wrote the first sample's four rows and stopped with its message. `python3 -m unittest tests.test_docs_lab_prism_references -v` passes (56 tests). Co-Authored-By: Claude Fable 5.1 * 3.x.x: normalise ANSI-C quoted words in the doc-command guard (#318 review) bash treats $'…' (ANSI-C quoting) and $"…" (locale quoting) as quoted strings and strips them like plain quotes: on bash 3.2, `python3 -m $'json.tool'`, `python3 -m $'json.'tool`, `python3 -m $"json.tool"` and `$'python3' -m json.tool` all run json.tool. The word grammar read the `$` as an unquoted character, so `unquote` produced `$lab.prism.deleted`, which matched no target, and the guard reported nothing for a command that runs a deleted module. A quoted piece may now carry the `$` prefix, in both the target word and the interpreter word, and `unquote` strips it with the quotes, so `$'lab.prism.'deleted` reads as `lab.prism.deleted` and `$'python3'` as `python3`. A `$` directly before a quote belongs to the quoted piece and is never an unquoted character, which keeps the split of a word unique and the alternation free of ambiguous nesting; a `$` before anything else is an ordinary character, so `$VAR` words behave as before. Backslash escapes stay out of scope as the docstring says: `$'lab\x2eprism.deleted'` keeps its backslash, names no target and is not reported. Tests cover the evading module, script and interpreter forms, the same forms with existing targets, the escape and unterminated-quote cases, and a wrapped form at its line. Co-Authored-By: Claude Fable 5.1 * 3.x.x: invalidate the soak on a failed process collection (#318 review) The step-3 capture loop accepted a `/metrics` scrape when the exec exited 0, the body carried exactly `x-prism-metrics-state: fresh` and the family `qbit_prism_process_resident_memory_bytes` was merely present. The header proves less than that check assumed. `metrics_snapshot.rs` describes only the age of the last snapshot publication and says a fresh failed probe is still fresh, and the collector families are overlaid at scrape time by `Metrics::overlay_collections`: when the process collector's last success is absent or older than `COLLECTOR_STALE_AFTER` (30 s), `snapshots::invalidate` renders `qbit_prism_collector_available{collector="process"} 0` and `qbit_prism_process_resident_memory_bytes -1`, the rendering the registry test `process_collector_reads_real_input_and_distinguishes_failure_from_zero` asserts for a failed read. Such a body passed the loop, so it recorded a correlated sample with no RSS value, the series the reading order starts from, instead of invalidating the soak. The step-2 gate already requires the gauge at exactly 1, but proves it once, at the start. The awk now requires, in order, that the exec exited 0, the exact line `x-prism-metrics-state: fresh`, the exact line `qbit_prism_collector_available{collector="process"} 1`, and an RSS sample whose value is a nonnegative integer, so `-1` is rejected. On failure it prints the first condition that failed in the existing style: the exit status, the state header seen or its absence, the collector gauge line seen or its absence, or the RSS line seen or its absence. The surrounding shell is unchanged. The prose states the new conditions and why the header alone is not enough, and step 6 lists an unavailable process collector and an unusable RSS value among the reasons a run is not judged. Step 2, the RSS `/proc/1/status` guard and the restart guard are untouched. Verified with `sh -n`, `bash -n` and `zsh -n` on the extracted loop, and with a stub `docker` and a stub `sleep` in sh (bash 3.2), bash 3.2 and zsh 5.9. The stub serves a normal `/proc/1/status` body to the `cat` call and, for the `curl` call, a fresh body with the gauge at 1 and RSS at 4194304 first, then the case under test: the gauge at 0 with RSS `-1`, the gauge at 1 with RSS `-1`, no gauge line, and a `stale` header. Each of the twelve failure runs wrote the first sample's five rows to `soak-metrics.log` and stopped with its own message; the second sample's RSS row still lands in `soak-rss.csv`, since that read precedes the scrape. The three control runs, with the second body also good and the stub `sleep` ending the loop, wrote ten rows and no message. The awk program gives the same answer for every case under macOS awk 20200816, busybox 1.37.0, gawk 5.3.2 and mawk 1.3.4. `python3 -m unittest tests.test_docs_lab_prism_references` passes (60 tests). Co-Authored-By: Claude Fable 5.1 * 3.x.x: require exactly two fields in the soak judge (#318 review) The `awk` bound check refused a row with fewer than two fields but read any row with more, so `82800,300,000` judged as 300 bytes and `0,100,garbage` set the baseline, and corrupted input could suppress a breach while the prose promised an exact `seconds,rss_bytes` format. The deleted Python evaluator had the same looseness (`read_rss_samples` at `1398bbc^` took the first two comma-separated fields and ignored the rest), so the port was faithful to it and wrong with it. The predicate is now `NF != 2`: a row with a third field, including a trailing comma, is unusable input (exit `2`) like a truncated one. The capture loop writes exactly two fields, so a row with more was corrupted somewhere between the loop and the judge and is refused rather than read. The prose says the field count is exact and names the tightening over the Python tool. Verified with the judge fence extracted from the doc and run under `sh` with BSD awk: a flat 24 h series passes (exit 0), `0,100,garbage`, `82800,300,000`, `100,200,` and a one-field row each exit 2 with the row named, `-1` samples, comments and blank lines are still skipped, a post-warm-up breach still exits 1, and whitespace around a field still counts as two fields; the `NF < 2` form exited 0 on both three-field rows. `python3 -m unittest tests.test_docs_lab_prism_references` passes (60 tests). Co-Authored-By: Claude Fable 5.1 * 3.x.x: write each soak run into its own directory (#318 review) The step-3 capture loop appended every sample to `soak-process.log`, `soak-rss.csv` and `soak-metrics.log` in the working directory with `>>`, and nothing truncated them or moved them aside. The restart rule says an invalid run is started over from step 2, so a second run appended to the first run's files, and the judge, which sorts by timestamp before it reads, then took its first hour, and so its baseline, from the earlier run, judged the later run against that baseline, and satisfied the span floor with the earlier run's first sample and the later run's last. Truncating instead would have destroyed the evidence the restart rule says to keep. Every run now writes into its own directory, `soak-`, and the loop is chained to the `mkdir` that creates it with `&&`: `mkdir` without `-p` refuses a name that already exists, and on that refusal the loop does not run at all, so a run can never append to another run's files and never writes anything the operator did not name. The share-ack histogram and the `/metrics` snapshots in steps 3 and 4 go into the same directory, step 6 judges the run's own `soak-rss.csv`, step 7 records the run directory, and the restart rule keeps the directory rather than "the log". The bound section says the sort applies to partial logs of one run and points at the per-run directory as the reason two runs never share a file. Verified with `sh -n`, `bash -n` and `zsh -n` on the loop extracted from the doc, and with a stub `docker`, `sleep` and `date` in sh, bash and zsh: a fresh run created its directory and wrote two RSS rows, three process readings and eight metrics lines before the stub flipped the process state and the restart guard stopped it; a second run under the same start time was refused by `mkdir`, the loop did not run, the first run's files were byte-identical afterwards and nothing was written at top level; a third run under a later start time created a second directory and left the first untouched. `python3 -m unittest tests.test_docs_lab_prism_references` passes (60 tests). Co-Authored-By: Claude Fable 5.1 * 3.x.x: leave -h and -V out of the doc-command guard's prefix options (#318 review) `PYTHON_FLAG` listed `h` and `V` among the option letters that may sit between the interpreter and its target, so the guard read `python3 -V lab/removed.py` and `python3 -h -m lab.removed` as commands that run the missing target and would have failed CI on a diagnostic that never reaches it. CPython prints the help text for `-h` and its alias `-?`, and the version for `-V`, and exits before it reads the target, so such a command runs nothing; the comment above the class already excluded `-c` for the same reason. The class is now every single-letter option other than `-c`, `-m`, `-h` (with `-?`) and `-V`. Because `PYTHON_FLAG` also builds the `-W`/`-X` cluster form and the `-m` cluster (`MODULE_OPTION`), `-hW error`, `-VX dev`, `-hm lab.x` and `-Vm lab.x` are no longer read as options either, which is what CPython does with them. The comment names the letters, the behaviour and the spellings checked, and a new `ScannerTests` case covers `-h`, `-?`, `-V`, `-VV`, `-OO -h`, `-Oh`, `-hOO`, `-V -X dev`, `-VX dev` and `-hW error` before a missing module (bare and quoted) and a missing script, `-V --` before a script, and the clusters `-hm`, `-Vm`, `-Ohm`, `-hIm` and `-VVm` with the module attached or following; each is not a command, and the prose contract still sees the reference. Verified on CPython 3.14.6 with `lab/gone.py` and `gone.py` that raise `SystemExit("RAN")`: every spelling above printed the help text or the version and exited 0 with neither file run, while `python3 -m lab.gone` and `python3 gone.py` printed `RAN`. Mutation check: with the old class restored in a copy of the module, the new test fails in ten of its subtests (every one but `-?`, which was never in the class). `python3 -m unittest tests.test_docs_lab_prism_references` passes (61 tests). Co-Authored-By: Claude Fable 5.1 * 3.x.x: invalidate the soak on a gap between samples (#318 review) The step-3 capture loop never compared one sample's time with the previous one's, and the `awk` judge's span floor reads only the first and last timestamps. A host suspended for an hour, or a `docker inspect` or `docker exec ... cat` that stalled for one (neither call is bounded; only the `curl` carries `--max-time`), therefore left a hole in the series that nothing reported: the iteration after the stall wrote a valid row, the loop went on, and the judge passed a file in which an excursion could have risen and drained unseen. The deleted Python evaluator (`evaluate_rss_bound` in `lab/prism/process_telemetry.py` at `1398bbc^`) had no gap rule either, so the port was faithful to it and wrong with it. The judge now takes `max_gap=360` and refuses a series in which two consecutive usable samples are more than 360 s apart as unusable input (exit 2), naming the interval and both timestamps. The rule sits after the `-1` skip, so a hole bridged by `-1` rows is still a hole, and a single `-1` row between two cadenced samples opens a 600 s gap; the loop never writes one, and the prose says so. The loop keeps the previous iteration's `now` and, before anything is written, stops the run as invalid when more than 360 s have passed since it, printing the interval, its own timestamp and the last sample's. The tolerance is the 300 s of sleep plus a minute for the three reads. The prose explains the rule in the bound section as a tightening over the Python tool of the same kind as the exact field count, adds a step-3 paragraph on the interval check beside the missing-sample one, and lists a gap among the reasons step 6 does not judge a run. Verified with the judge fence extracted from the doc under macOS awk 20200816, and under busybox 1.37.0, gawk 5.3.2 and mawk 1.3.4 in an alpine container, all agreeing: a flat 300 s series passes (exit 0); a 3900 s hole after hour 2, the same hole bridged by `-1` rows, a 2.5x excursion that drained inside the hole, the hole in shuffled input and a hole inside the warm-up each exit 2 with the gap named; a 360 s interval passes and a 361 s one exits 2; a breach still exits 1, and a malformed row, a short span, comments and blank lines behave as before. The unchanged judge at the previous HEAD passed every hole case. The loop, extracted from the doc, passes `sh -n`, `bash -n` and `zsh -n`, and with a stub `docker`, `sleep` and `date` under sh (bash 3.2 in POSIX mode), bash 3.2 and zsh 5.9: a control run in cadence wrote three RSS rows before the stub's process flip ended it; a run whose second `date +%s` is 3900 s after the first stopped with the gap message after exactly one row, one process line and one metrics sample; a 360 s interval continued and a 361 s one stopped. `python3 -m unittest tests.test_docs_lab_prism_references` passes (61 tests). Co-Authored-By: Claude Fable 5.1 * 3.x.x: stop the soak when an append to the run's files fails (#318 review) The step-3 capture loop appended its process reading, RSS row and metrics lines with `>>` and never looked at the status. The operator's shell is not under `set -e`, so when the run directory filled or its filesystem returned an I/O error the loop went on reading and failing to write, and once the rows already on disk spanned the 82,800 s the judge's floor asks for they would have passed it, with the hours after the disk filled, and the process readings and metrics lines that should explain them, missing from the record. The gap rule cannot see this: the series simply ends. Each of the three appends is now checked and the first that fails stops the run as invalid, naming the file and the sample time, in the style of the other stop paths. The metrics lines are read into a variable before they are appended, as the body and the response already are: the status of the pipeline would be `sed`'s, and what a failed write does to that is each implementation's own affair, whereas the builtin `printf` reports one as `1` in `sh`, `bash` and `zsh` alike. The grep pattern is unchanged. The prose adds the rule beside the missing-sample and gap paragraphs, and step 6 lists a failed append among the reasons a run is not judged. Verified with the loop extracted from the doc: `sh -n`, `bash -n` and `zsh -n` pass, and with the stub `docker`, `sleep` and `date` under sh (bash 3.2 in POSIX mode), bash 3.2, zsh 5.9, busybox 1.37.0 sh and dash 0.5.13, the control and gap runs behave as before; with the stub `sleep` replacing `soak-rss.csv`, `soak-process.log` or `soak-metrics.log` by a directory after the first iteration, the second iteration stops with the append message naming that file and nothing further is written. In an alpine container, `echo x >> /dev/full` and `printf '%s\n' x >> /dev/full` return 1 in busybox sh, dash, bash 5.3.9 and zsh 5.9, and `if !` and `||` take the failure branch while the shell keeps running. `python3 -m unittest tests.test_docs_lab_prism_references` passes (61 tests). Co-Authored-By: Claude Fable 5.1 * 3.x.x: record why a soak is invalid and return nonzero (#318 review) Every stop path of the step-3 capture loop printed its message to stderr and left the loop with `break`, whose status is 0, and wrote nothing to the run directory to say the run was cut short. The iteration that finds a bad metrics sample has already written its RSS row, so a stop after 23 hours left a directory whose CSV satisfies the span floor and nothing, in the directory or in the status, to say it was invalid: a wrapper that ran the fence as a script, or an operator who kept the directory and lost the shell's scrollback, would have judged and passed it. The fence now defines `invalid`, which copies what it is piped to `$run/soak-invalid` and to stderr, and `capture`, a function that holds the loop and is called at the end of the fence. Every stop path writes its message through `invalid` and returns 1 from `capture`; `mkdir "$run" || return` keeps the per-run-directory guard and returns `mkdir`'s status before the loop runs. The loop body is otherwise unchanged. The fence is pasted into the operator's interactive shell, which steps 4 to 7 read `$run` from, so the stop paths `return` rather than `exit`, and `run=` and `c=` stay outside the function. The prose describes `capture`, says the directory now records why it is invalid and that the only way `capture` returns by itself is an invalidation, notes that `tee` still delivers the message to stderr when the disk itself is the fault, and step 6 refuses to judge a directory that holds `soak-invalid` whatever its CSV would say; the restart rule points at the marker for the message. Verified with the fence extracted from the doc: `sh -n`, `bash -n` and `zsh -n` pass, and with the stub `docker`, `sleep` and `date` under sh (bash 3.2 in POSIX mode), bash 3.2, zsh 5.9, busybox 1.37.0 sh and dash 0.5.13: the control, gap and three append-failure runs write the same rows as before, `capture` returns 1 on each, `soak-invalid` holds exactly the stop message (`cmp`-identical to the stderr text for the control and gap runs), and `$run` is still set afterwards; a refused `mkdir` returns its status, runs nothing and writes no marker. In an alpine container, `tee -a /dev/full >&2` still delivers its input to stderr and exits 1 with busybox and coreutils tee, through `invalid` in busybox sh, dash, bash 5.3.9 and zsh 5.9. `python3 -m unittest tests.test_docs_lab_prism_references` passes (61 tests). Co-Authored-By: Claude Fable 5.1 * 3.x.x: read quoted option words in the doc-command guard (#318 review) The guard read the interpreter and the target as whole shell words and unquoted them, but matched the option words between them with a regex on the raw line, so the hyphen and the letters had to be bare. bash strips the quotes of an option word before CPython sees it, so `python3 "-O" -m lab.prism.deleted`, `python3 '-m' lab.prism.deleted`, `python3 "-m"lab.prism.deleted`, `python3 $'-OO' -m …`, `python3 "-"O -m …`, `python3 "-X" dev -m …` and `python3 "--" lab/prism/deleted.py` all run the deleted target and were reported as nothing, and in a ratcheted document such a spelling could replace a dangling mention without moving the count. The option regexes and the `quoted` helper are replaced by a tokenizer. `INTERPRETER_CANDIDATE` finds each candidate interpreter word as before; `command_target` then reads the words after it one shell word at a time with `NEXT_WORD`, unquotes each, and applies the option grammar to the unquoted word in order: a flag cluster is skipped; a `-W`/`-X` cluster takes its argument attached or as the next word; `--check-hash-based-pycs` takes the next word, which must be one of its three modes; a `-m` cluster names the module attached or as the next word; `--` makes the next word the script; any other word is the script, so `-c…`, `-h…`, `-?`, `-V…` and long options still end the command at a word that names no lab target. A word with a mismatched quote is no shell word, so the command ends there, as bash rejects the line. `dead_commands` reports the same `(first line, command, missing path)` rows, with the command sliced from the interpreter word to the target word; the only observable change is that a module and a script command on one line now report in source order. `PYTHON_FLAG` and the target grammar are unchanged; the comments describe the tokenizer and the spellings checked. Verified on bash 3.2 and CPython 3.14.6 that every spelling above runs `json.tool` or a `lab/gone.py` that raises exactly as its bare form, that `'-h'`, `"-hm"`, `"-V"`, `"-Wm"`, `"-Xm"`, `"--check-hash-based-pycs" sometimes` and `'--' -m` still run nothing, and that `"-O'` and `-X "dev'` are rejected by bash before Python starts. Five new `ScannerTests` cover the quoted, concatenated and ANSI-C option words before missing and existing module and script targets, the wrapped form at the right line, the quoted non-command options, and mismatched option-word quotes. Mutation check: with the scanner at the previous HEAD, the new tests fail in 16 places. Over the 59 docs in scope the old and new scanners read the same 17 Python commands with identical kind and target. `python3 -m unittest tests.test_docs_lab_prism_references` passes (66 tests). Co-Authored-By: Claude Fable 5.1 * 3.x.x: cover quoted --check-hash-based-pycs modes in the doc-command guard (#318 review) A review of the regex scanner noted that `python3 --check-hash-based-pycs $'always' -m lab.prism.deleted` and `python3 --check-hash-based-pycs alwa"ys" -m lab.prism.deleted` were read as no command, while bash hands CPython the bare mode and the missing target runs. The tokenizer that replaced the regexes in a1c542a unquotes every word after the interpreter before the grammar sees it, the mode included, so both spellings are reported there; this adds the test that says so. `test_quoted_hash_mode_arguments_are_read_as_shell_words` covers `$'always'`, `alwa"ys"`, `'never'`, `$"default"`, `de'fault'` and `"al"ways` before a missing module and a missing script, and the same before an existing module, and checks that `"sometimes"`, `$'some'times` and two mismatched-quote spellings are no command while the prose contract still sees the reference. Verified on bash 3.2 and CPython 3.14.6 that each of the six spellings runs `json.tool`, and that `$'some'times` is rejected with "--check-hash-based-pycs must be one of 'default', 'always', or 'never'" before anything runs. `python3 -m unittest tests.test_docs_lab_prism_references` passes (67 tests). Co-Authored-By: Claude Fable 5.1 * 3.x.x: re-read the coordinator identity after each soak sample (#318 review) The step-3 `capture` function checked the coordinator's identity before each sample's reads and nowhere else. A restart between that check and the `/proc/1/status` and `/metrics` reads recorded the replacement's RSS and metrics under the original identity in `soak-process.log`; every later iteration's check would catch it, but the last iteration of a valid run has no later iteration, since the operator ends the run after 24 h, so a restart in that window on the last sample left a directory with no `soak-invalid` that read as a single-process run. The identity is now read again after the two reads, after the metrics append and before the sleep, and a change ends the run as invalid through `invalid` and `return 1` like every other stop path, printing the sample time and both readings. The pre-sample check and the one `soak-process.log` line per sample are unchanged. That sample's rows are already in the files, which is fine because the marker keeps the directory from being judged, as step 6 says. The step-3 intro names the second check and the restart paragraph explains the window. Verified with the fence extracted from the doc, `sh -n`, `bash -n` and `zsh -n` passing, and with the stub `docker`, `sleep` and `date` under sh (bash 3.2 in POSIX mode), bash 3.2 and zsh 5.9, where the stub's inspect answer flips `RestartCount` at a chosen call: flipping at the post-read check of the second sample stops the run with the new message after two RSS rows, two process lines at the original identity and eight metrics lines; flipping at the first sample's post-read check stops it after one row; flipping at the second sample's pre-check still stops with the existing message; with no flip, five samples run with no identity or gap message and the run ends on a gap sentinel. In every case `capture` returns 1 and `soak-invalid` is byte-identical to stderr. `python3 -m unittest tests.test_docs_lab_prism_references` passes (67 tests). Co-Authored-By: Claude Fable 5.1 * 3.x.x: fetch the pinned qbit through a cached, two-route, digest-checked installer (#318 CI) Run 34857720110 lost the native PRISM job before any Rust test started: the release download route on github.com answered 504 for the whole of curl's seven-second --retry 3 window (an eleven-second first attempt, then three instant 504s carrying github.com's own error page), so the install step exited 22 and the gate-manifest proof failed for want of a manifest. The run four minutes earlier fetched the same asset in seven seconds. Replace the inline curl with .github/scripts/install-prism-qbit.sh, which keeps the version, asset id and digest pinned in one place and takes the tarball from the first source that verifies: - a cached copy (actions/cache keyed on the script's hash, so a pin bump rotates the key); a cached file with the wrong digest is deleted, and a cache-service error is only a warning; - the release download route, with five retries under a bounded ninety-second retry timer plus connect and stall timeouts; - the release-asset API route on api.github.com, which fronts the same storage through a different tier and needs no token for a public repo. Nothing is extracted before its sha256 matches the pin, whichever source delivered it, and QBITD_BIN is still written to the job environment. tests/test_install_prism_qbit.py drives the real script against a loopback server scripted per route: fallback after a gateway error, rejection of wrong-digest bytes from a route or the cache, failure when every source is unusable, a verified cache hit making no request, and the workflow calling the script with only the cache directory configured. Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_01H5oqbzF1nZ43sn7454ezH5 * 3.x.x: end the soak from the capture loop and gate the judge on its marker (#318 review) The step-3 `capture` loop returned only through its `invalid` paths; a valid run was "ended by the operator after 24 h". So a SIGHUP from a closed terminal, a SIGTERM from a host going down or an interrupt after the CSV spanned the judge's 82,800 s floor but before 24 h ran no stop path, wrote no `soak-invalid`, and left a directory that step 6 judged and the floor passed, indistinguishable from a valid run. The loop now keeps the time of its first sample and ends the run itself: after the first sample taken 86,400 s or more after it, once that sample's reads, appends and both identity checks have passed, it writes one line to `$run/soak-complete.tmp`, renames it to `$run/soak-complete` and returns 0. The rename publishes the marker whole or not at all: a redirect straight to the marker's name would create the file before the write, and a write that then failed on a full disk could keep `invalid` from writing `soak-invalid` on the same disk, leaving an empty marker and no invalid marker. A failed write or rename ends the run as invalid through `invalid` like a failed append; an interruption between the two leaves only the temporary file, which nothing reads. The `mkdir` rule already keeps `capture` out of an existing directory, so a marker is only ever written by the loop that took every sample in it. Step 6 gains an executable gate, `completed`, that refuses a directory holding `soak-invalid` or lacking a regular-file `soak-complete`, and prints the completion line otherwise; the bound check runs only after it. The awk judge is unchanged. The prose that said the operator ends a valid run is reworded, the judge's hour of slack is tied to the marker, and the marker joins step 7's record. `tests/test_prism_soak_capture.py` extracts the capture, gate and judge fences from the doc and runs them under sh (dash) and bash with shell functions for `date`, `sleep` and `docker`, where `sleep` advances a file-backed clock: a full run completes with 289 samples, the marker, no stderr and a passing gate and judge; a SIGTERM at 23 h leaves a CSV the judge alone accepts and a directory the gate refuses; an identity change on the final sample's post-read check writes `soak-invalid` and no marker; a marker write onto /dev/full with an unwritable `soak-invalid` publishes nothing and the gate still refuses; a SIGTERM between the write and the rename leaves only the temporary file; an existing directory is not written into; and the gate refuses a directory with both markers, a directory as the marker, or only the temporary file. Mutating the gate to ignore the marker, or the fence back to a direct redirect, fails the corresponding tests. `python3 -m unittest tests.test_prism_soak_capture tests.test_docs_lab_prism_references` passes (75 tests). Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_01B3vaLLrCSvBv4qzDEiX6Dj * 3.x.x: hold each soak sample's end to the previous one's before completing the run (#318 review) `capture` stamped every row with the `now` it read before the sample's reads, and its completion check used that same `now`. A `docker inspect` or `docker exec ... cat` that stalled inside the final sample's reads was therefore invisible: on any other sample the next iteration's start-time gap check would have seen it, but the sample that completes the 24 h has no next iteration, so a stall of an hour passed both identity checks, published `soak-complete` claiming the samples span 0..86400 s for a process observed at 90000, and returned 0. Read the clock again after the post-read identity check and hold each sample's end to the previous sample's end with the same 360 s tolerance the start-time check and the judge's `max_gap` use, before the completion block can run. The 300 s sleep lies inside that interval, so the reads get the same minute on the final sample that the start-time check leaves them on every other one; a read that overruns it ends the run through `invalid` with the sample's time, when its reads ended and how far behind the previous sample's that was, publishes no marker and returns 1. The start-time check stays for a hole in the sleep, and the marker is still written to `soak-complete.tmp` and renamed. Cover it on the fake clock: the final pre-read inspect advancing the clock from 86400 to 90000 with the identity and both bodies healthy now returns 1, records that message, leaves no marker, and is refused by the gate although the bound check alone would pass the CSV; a 60 s overrun on the final sample still completes, and a 61 s one is rejected on the final sample as it already was on an ordinary one. Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_01Kfm1tpNt7zHUJGa9sQKv22 * fix: reject failed soak process identity probes (#318) * fix: recognize non-identifier Python module targets * fix: preserve quoted literal module targets (#318) * fix: require fresh share-ack soak snapshots (#318) * fix: preserve literal script targets in documentation guard (#318) * fix: validate and atomically publish full soak snapshots (#318) * fix: invalidate soak captures on backward clock readings (#318) * fix: skip shell comments in documentation command guard (#318) * fix: schedule soak snapshots during capture (#318) * fix: resolve complete prose references in documentation guard (#318) * fix: normalize lab script paths in documentation guard (#318) * fix: reject failed RSS reads during soak capture (#318) * docs: mark retired soak settings for the native guidance guard (#318) * test: avoid redundant full-soak identity fault simulations (#318) * fix: distinguish shell data from executable documentation commands (#318) * test: publish supervisor fixture readiness atomically (#318) * fix: require accepted-share progress throughout the soak (#318) * fix: recognize compound-shell documentation commands (#318) * fix: consume shell option arguments before command strings (#318) * fix: consume sudo chdir arguments in doc commands (#318) * fix: exclude sudo edit mode f… * v3.0.0 ci: test the pinned-tool installer's outage and integrity guards (#392) #380 hardened install-pinned-tool.sh against release-host outages, but its validation harness lived outside the repository, so nothing in CI exercised the retry window, the hard deadline, the verified cache or the fail-closed handling of bad cache entries. Commit that harness as .github/scripts/test-install-pinned-tool.sh and run it in lint-compile-compose. The test is hermetic: a Python http.server on 127.0.0.1 stands in for the release host, and a routing-only curl shim on PATH sends the pinned https://github.com/ URL there while passing every other argument through to the real curl. Scripts under test are derived from the committed installer by replacing exactly four whole lines (the two sha256= pins with the fixture digests, the timeout --signal=KILL line and the --retry line); each must match once and a diff must show nothing else, so every guard runs as committed and sha256sum is never shimmed. The pure-failure scenarios also run against the unmodified installer through the shim alone. 38 scenarios and 453 assertions cover both tools in about 22 seconds: a 504 outage longer than the old seven-second window, a warm cache with the host refusing connections (zero requests), a warm cache under a path sha256sum cannot name (the private copy is what is checked), substituted download bytes with and without a cache, poisoned cache entries with wrong bytes, a FIFO, a symlink to a FIFO and a symlink to a digest-correct file, a 404 failing fast, a stalled transfer killed by the deadline, and the pins themselves. Bad assets carry an execution marker that must never appear. Closes #389 Co-authored-by: Claude Fable 5.1 * v3.0.0 fix: independently verify durable payout window boundaries (#379) * fix: independently verify durable payout window boundaries Re-derive anchored eligibility and the oldest crossing share from immutable ledger history before taking the settlement lock. Preserve canonical audit bytes, transactional count checks, and existing claim and payout fences. Refs #356 * refactor: isolate durable window boundary arithmetic Fold authenticated shares on a blocking thread after page equality and the newest probe, preserving saturating semantics and refusal order. Share anchored eligibility, document the delayed-anchor query budget, and isolate boundary contracts with database-free coverage. * v3.0.0 fix: bound prepared template and balance blob cleanup (#378) * fix: bound prepared template and balance blob cleanup Sweep indexed blob pages with the existing settlement then ordering locks, retain every remaining reference, and carry one deadline through PostgreSQL statements and commit. Exercise repair, candidate retention, bounded progress, and cancellation against disposable PostgreSQL. Refs #273 * fix: honor the ledger write fence during blob cleanup Reuse the established fatal-state and legacy-writer fence before deleting jobs or blobs. Verify that a halted cluster preserves every row and cursor position and that the same page becomes collectible after recovery. * fix: bound expiry selection to indexed job keys Use a stable statement cutoff for the expiry index and a bounded array initplan for deletion by primary key. Preserve the outer expiry recheck and verify that a renewal committed during the DELETE row-lock wait retains the prepared row and its blobs. * fix: keep job expiry outside blob cleanup locks * v3.0.0: Enforce audit import and rehearse one-way recovery (#366) * v3.0.0 fix: enforce audit import and rehearse one-way recovery * fix: include durable recovery obligations in evidence * fix: fingerprint immutable CTV recovery payloads * fix: fingerprint complete CTV manifest-set payload * fix: include fatal state in recovery evidence * fix: fingerprint native CTV recovery checkpoints * fix: fingerprint native audit reconstruction payloads * fix: fingerprint pending candidate recovery payloads * fix: fingerprint share hash replay protection * fix: fingerprint stored canonical audit bytes * fix: reject missing native recovery tables * fix: pass recovery test database URLs explicitly to PostgreSQL tools * fix: require native metadata before exporting recovery evidence * fix: fingerprint imported audit lookup metadata * fix: adapt recovery rehearsal to referenced candidates * fix: fingerprint the share sequence allocator during recovery * fix: validate the complete native migration history for recovery * fix: fingerprint candidate window reference columns * fix: fingerprint balance snapshots retained by pending candidates * fix: require the native cluster singleton in recovery exports * fix: validate native recovery migration history table structure * fix: reject recovery relations resolved outside the ledger schema * fix: fingerprint the remaining durable recovery allocators * fix: fingerprint the cluster chainwork recovery checkpoint * fix: fingerprint durable CTV broadcast retry progress * fix: fingerprint durable block lifecycle timestamps * fix: fingerprint the pinned cluster configuration A restore that loses or changes qbit_prism_cluster.config_fingerprint while preserving every exported accounting row still compared equal. Ledger::configure treats NULL as an uninitialized cluster and pins whatever consensus, payout and signing configuration the restarted frontend supplies, whereas the intact value rejects a mismatch, so such a restore could silently start under different payout rules or keys. Export the non-NULL fingerprint as cluster_config. The fresh-migration NULL default is omitted, so migrating an unchanged legacy backup still produces identical evidence. The regression pins, rotates and resets the fingerprint with everything else fixed, then carries a pinned value through the native backup/restore roundtrip. Co-Authored-By: Claude Fable 5.1 * fix: preserve the payout revision fence A restore that rewinds qbit_prism_cluster.payout_revision while retaining the exported block, carry, candidate and chain-checkpoint rows still compared equal. Pending candidates store their issuance revision, and candidate_revision_valid plus the landing and reconciliation guards require exact equality with the cluster value, so rewinding it to an older candidate's revision can revive payout work the current database deliberately fenced out. Export the nonzero revision as payout_revision. The migration default of zero is omitted, so migrating an unchanged legacy backup still produces identical evidence; updated_at alone remains invisible. The regression advances, re-advances and rewinds the revision with everything else fixed, then carries an advanced value through the native backup/restore roundtrip. Co-Authored-By: Claude Fable 5.1 * fix: fingerprint the durable ledger clock A restore that rewinds qbit_prism_cluster.ledger_clock_ms while keeping every exported share and snapshot anchor still compared equal. append and snapshot advance this value as the monotonic timestamp barrier behind share acceptance times and window anchors, precisely so accounting never depends on host clock sync. If the database clock is behind the prior high-water mark after recovery, new shares receive timestamps below existing snapshot anchors and later window reads can exclude otherwise eligible shares as being from the future. Export the nonzero clock as ledger_clock. The migration default of zero is omitted, so migrating an unchanged legacy backup still produces identical evidence, while the first acknowledged append is required to make the clock visible. The regression rewinds, zeroes and restores the clock with everything else fixed and requires exact baseline equality once the original value is back. Co-Authored-By: Claude Fable 5.1 * fix: fingerprint durable candidate retry state A restore that rewinds qbit_block_candidate_outbox.attempt_count or last_error while keeping a pending candidate's payload, window and state still compared equal. claim_candidate_lane serves attempt_count = 0 first and retry_candidate backs off by the count, so the approved restore can move retried work back into the fresh lane and retry it at once, and the parked sentinel that keeps an undecodable row off the due lane can be lost or reimposed unseen. Export attempt_count and last_error, legacy columns that migration carries verbatim, and next_attempt_at only as its parked sentinel. The finite schedule stays out: migration 002 backfills next_attempt_at from clock_timestamp(), so two migrations of one legacy backup would differ, and retry_candidate bounds it to sixty seconds past a clock that any restore outlives. Claim ownership and updated_at remain invisible. The claim regression now expects the attempt to be visible and the lease not to be; the retry-state regressions mutate and restore each field on the native pending candidate and on the terminal row in both the native and frozen 2.x exports. Co-Authored-By: Claude Fable 5.1 * fix: fingerprint durable candidate creation order A restore that rewinds or swaps qbit_block_candidate_outbox.created_at across pending candidates still compared equal. claim_candidate_lane serves the newest due candidate first and breaks retry-lane ties on created_at, so the approved restore could submit a different competing block first and change which candidate is accepted and accounted. Export created_at rendered AT TIME ZONE 'UTC', as the block lifecycle timestamps are, so the digest does not depend on the session TimeZone. The column is legacy, set once by its enqueue default and rewritten by no claim, retry, park, terminal or migration statement, and the outbox carries no immutability trigger over it, so a fresh migration of an unchanged backup keeps the value verbatim. Claim ownership, updated_at and the finite next_attempt_at schedule remain invisible. The retry regressions now also shift created_at on the native pending candidate and on the terminal row in both the native and frozen 2.x exports, and a new regression enqueues a sibling candidate and swaps the two creation times in place, requiring only the candidates digest to change and the swap back to restore the paired baseline. Co-Authored-By: Claude Fable 5.1 * fix: fingerprint durable audit creation order A restore that rewinds or swaps qbit_pool_audit_bundles.created_at across same-height audits carrying the same commitment leaf still compared equal. The by-commitment lookup and the latest-evidence read both break the block-height tie on created_at, and backfill_ctv repairs fanouts in creation order, so the approved restore could serve a different public artifact for the same leaf and a different latest evidence body. Export created_at rendered AT TIME ZONE 'UTC' in both the native and frozen 2.x branches, as the block lifecycle timestamps are, so the digest does not depend on the session TimeZone. The column is legacy, present in the frozen 2.x schema with the same clock_timestamp() default, set once by the landing INSERT and rewritten by no statement: the bits backfills and the legacy import UPDATE only their own columns, no migration from 002 to 010 touches it, and the table carries no trigger, so a fresh migration of an unchanged backup keeps the value verbatim. The regression now copies the newest audited block and its audit under a sibling hash at the same height in both the native and frozen 2.x exports, checks that the latest-evidence read flips with the creation order, swaps the two creation times in place, requires only the audits digest to change, and restores the paired baseline by swapping back and removing the sibling. Co-Authored-By: Claude Fable 5.1 --------- Co-authored-by: Claude Fable 5.1 * v3.0.0 fix: observe audit and migration pool checkouts (#390) * fix: observe audit and migration pool checkouts Route seven direct checkout sites through the shared observer, preserving per-page release and SQL error outcomes. Keep migration source reads on one acquired connection and retain Send futures. Refs #352. * fix: preserve pool interface for audit proof instrumentation Keep the shared acquisition timer on the original pool argument so the independent boundary-proof change composes without changing either algorithm. Refs #352. * test: start checkout cancellation deadline after first poll Synchronize each caller at the held pool slot before starting its timeout so caller preparation cannot consume the measured checkout interval. Preserve the duration lower bound and cancellation/recovery assertions. Refs #352. * v3.0.0 feat: attribute Stratum admission refusals and stale-job causes (#393) Operators could not tell server admission saturation from transport failure, and one `stale-job` reason hid four different decisions (#384, #386). Observe the limits and decisions that already exist, without changing admission, credit or any wire response. Add three native families with closed, identity-free labels: - qbit_prism_stratum_connection_refusals_total{reason=global_limit| username_limit}, recorded once at the listener's global refusal and at the per-username authorization refusal. Same-username reauthorization, retained permits and successful requests never count. - qbit_prism_stratum_connection_limit, the configured global capacity. StratumConfig now keeps that capacity beside its semaphore in ConnectionLimit, so the gauge never reads free permits; the default 384 and PRISM_STRATUM_MAX_CONNECTIONS build it exactly as before. It is -1 until a listener starts. - qbit_prism_stale_job_rejections_total{cause=resume_expired|fee_floor| parent_grace|payout_revision}, recorded once at each existing refusal branch in execution order: Stratum's post-resume expiry, then the coordinator's fee floor, parent or failed grace parent check, and payout revision. Splitting the combined parent/revision branch keeps its short-circuit order; no later predicate runs to choose a cause. Every finite series starts at zero. The coarse reason_id counter and the share observation are unchanged, so causes are not double counted. Valid immediate-parent grace with revision drift still credits at the durable revision and records no cause. Regenerate the inventory (42 coordinator families), add an admission saturation runbook, and point the rejection guide at registered series with a per-cause diagnosis table instead of unregistered histograms. Tests cover a real limit-one socket, username-cap refusal and reauthorization, configured capacity while permits are held, all four causes with expiry+fee, fee+revision and parent+revision overlaps, accepted grace with revision drift, complete wire responses, closed labels, allocation-free events, both-direction inventory scrapes and real HTTP scrapes, plus an ignored test that serves /metrics for manual inspection. Co-authored-by: Claude Opus 5 * 3.x.x: Record the #271 share-append throughput measurements (#396) * Record the #271 throughput matrix measured on 5d0042f6 The harness fixes since the first matrix changed what its latency, rate and divergence columns mean, so the earlier numbers could not be cited. This records the matrix re-run on the fixed harness, one host and one release build, with the host, method and reproduction commands beside every table. It covers the replication premise probes, the 20k D1 matrix at three repeats, the calibration probes, 100k at one frontend (two repeats) and the blocked 200k, 400k and 500k attempts. It also records what could not be measured and why: 100k at four frontends did not fit in memory, and the D1 dense run aborts before its dense phase because slow_database submits outlast the settle limit. The comparison with the earlier bases shows which changes the harness fixes explain and which they do not. The metric gaps section lists what the server's own series could not resolve, for #278 and #279. Co-Authored-By: Claude Opus 5 Part of #271 * Lead the #271 measurements with what the matrix established The first version described the matrix's shape before its results, left the 400k refusal inside a table, and recorded the dense block and the 4-frontend 100k runs as open questions. They have since been decided. This revision states the results first: no configuration reaches 500 shares/s, sync costs 12-18 %, every size from 200k is refused by the JSONB ceiling, and 100k at four frontends does not fit on this host. The 400k result now has its own section, because #271 stays open for it. It explains why #265 cannot be observed while #273 blocks startup. The dense block is recorded as dropped. Its one run is kept as a finding: the harness correctly refuses to start the dense phase while slow_database submits outlast the settle limit. The single 100k 4-frontend sync attempt is added. The comparison across harness bases gains the slow_database ACK p50, which moved from 0.4 ms to 12-15 s once refusals stopped being counted as acknowledgements. Co-Authored-By: Claude Opus 5 Part of #271 * Name the sender of the SIGTERM that ended the 100k sync attempt The document said it could not confirm who terminated `w11-100k-fe4-sync-r1`, because the side report was never written. That was the right call from the measurement host, which cannot see it. The coordinator's memory guard runs on a different machine and its log records both the decision and the reading it acted on, so the sender is now named and the log lines quoted. Two things follow that the document was not yet saying. The two 100k four-frontend results are not equal evidence. The async attempt stopped itself at the harness's own floor, which demonstrates the harness declining to measure what it cannot measure. This one was stopped from outside, which demonstrates only that free memory crossed an external threshold. The conclusion rests on the async run, and the document now says so in both the summary and the detail rather than presenting the pair as interchangeable. And the setup itself is at fault, which is worth recording where someone repeating these measurements will see it. The harness floor is 6,144 MiB and the guard's is about 5,866 MiB, a gap of roughly 280 MiB. Memory fell from 11,658 to 5,686 MiB in under a minute, so at that gradient the gap is a few seconds and the external guard was always going to win. The harness should have been given more room to stop itself first. That is a fact about how this measurement was configured, not about PRISM. Part of #271 --------- Co-authored-by: Claude Opus 5 * fix: fence compact publication and replacement lease authority (#377) * fix: fence compact publication and replacement lease authority Capture cold build authority before construction and revalidate node, revision, balances, epoch and expiry before atomic publication. Bind replacement leases to the exact issued publication and coherent current balance state across issue, resume and share admission. Refs #273. * fix: reprove compact authority after publication lock waits Refresh external payout authority after local publication lock backpressure without holding state locks through I/O or renewing the original expiry clock. Preserve backend unavailability for an unready current publication while returning deliberate misses for retired requested work. * fix: retain ordinary authority when a leased tip returns A proof started during replacement may finish after the node returns to the published parent. Accept only with the original current revision and fresh polling, and keep the digest override restricted to active replacement leases. * fix: enforce lease authority at commit and issued delivery * fix: retain original issuance authority across handoffs * test: preserve merged inputs in cold authority fixture * fix: preserve issuance authority across unchanged refreshes Track actual prepared publications separately from tip observation ordering. Cached refreshes retain the original proof while new prepared installations still revoke it, even at the same parent and revision. Exercise real issue, persist, resume and compact publication across both transitions. * fix: report definite commit-gate refusals precisely Preserve the established no-credit response reason while identifying a refused local gate separately from database failure. Authority loss and lock contention both prove COMMIT was not sent; neither changes unknown-outcome reconciliation. * v3.0.0 feat: offer found blocks before audit landing (#391) Reserve one offer per candidate and reconcile post-offer work without resubmitting after a crash. Preserve issued audit and payout evidence when current balances change, with additive active-block accounting. Add migration 011, lifecycle and upgrade regressions, offer latency checks, and operational guidance for recovery and the serial submission queue. Refs #266, #358. * v3.0.0 fix: observe ledger job pool checkouts (#394) * fix: observe ledger job pool checkouts Route job reads and expiry through the shared acquisition boundary without changing SQL or connection lifetimes. Add PostgreSQL timing and release-scope coverage and document the remaining checkout sites. Refs #352. * test: share checkout fixtures and clean up after failures * v3.0.0 fix: park invalid claimed candidates with durable reasons Park still-owned claimed candidates after deterministic validation failures, preserving their evidence and recording bounded diagnostic reasons. Require explicit frontend shutdown for the offer cutover and add migration 012 to reject old startup registrations queued behind migration. Cover both registration orders, upgrade paths and recovery metadata. Docker lint passes after rebasing onto current 3.x.x. All 19 CI jobs passed before this rebase; merge without waiting for another CI run was explicitly requested. Closes #387 * 3.x.x: Trim the share ledger indexes online (migration 011) (#369) * 3.x.x: Trim the share ledger indexes online (migration 011) Map the six secondary indexes of qbit_share_ledger to the native readers and trim them to what those readers use (#153): replace the seven-column share_seq covering index, which no native read ever used index-only, with a three-column one that makes the frozen qbit_prism_window page walk and the landing durable-range count index-only; drop payout_order_key from the miner index, which nothing reads; drop the job_issued_at and template_height indexes, which have no native consumer. WHERE accepted stays: the readers exclude rejected rows by contract (the window and rollup tests write them), and the predicate keeps the frozen 001 walk index-only without carrying the column. CREATE INDEX CONCURRENTLY cannot run in the migrator's transaction, so 011 is the first ONLINE_MIGRATIONS entry: its file is replayed in the scratch schema to derive the index definitions, applied inside the transaction on an empty ledger, and otherwise applied after the commit on a dedicated connection with CONCURRENTLY, under a schema-keyed session lock, recorded last. The runner inspects every reserved name before any DDL, rebuilds an invalid leftover, keeps an identical earlier build and refuses anything else by name. Four gated tests cover the empty-ledger path, a frozen 2.x.x source with rows, a build with a writer transaction held open, and resume/keep/refuse. The ops doc gains the index inventory and the production measurement runbook the issue's gate asks for; the migration guide the online apply. Co-Authored-By: Claude Fable 5.1 * fix: run native ledger index migrations concurrently Always defer online migrations for existing native schemas, including ledgers whose first share is still uncommitted. Keep fresh and empty legacy cutovers transactional under their writer locks. Poll the online advisory lock without holding a waiting statement snapshot, so simultaneous frontend upgrades cannot deadlock concurrent index builds. Scope the first-share regression observer to its own ledger. * fix: refuse foreign invalid online migration indexes * test: update migration fixtures for schema version 11 * fix: verify old index definitions before online removal * ci: replace stale test artifacts on job retries * fix: recheck online index drop targets after concurrent builds The online runner decides its plan before any DDL: every index the migration creates and drops is looked up once, and each Rebuild and Drop step then runs DROP INDEX CONCURRENTLY by name. Between that look-up and the drops sit the CREATE INDEX CONCURRENTLY builds, which take hours on a large ledger, and nothing in that window stops an operator's DDL under a reserved name: the advisory lock orders only runners, and ALTER INDEX RENAME locks the index alone, so a release index can be moved aside and another created under its name (on any table) while a build holds the ledger. The stale plan then dropped an index the migration did not own, after the preflight had promised it would not. Each destructive step now looks its target up again as it is reached and drops only if the name still holds what the plan saw: the same table and definition, and for a rebuild still the invalid index an interrupted build left. Step::Drop carries the IndexDefinition for that instead of the table name. Anything else refuses the run, naming the index, what it reads as now and what the plan held, and, unlike the preflight refusals, what the run had already built and dropped by then, since that is no longer nothing; the next start plans afresh and keeps those. The recheck and the drop remain two statements, because CONCURRENTLY refuses a transaction block and PostgreSQL drops by name, so a change that lands in the round trip between them is still not caught. That window is milliseconds rather than the hours of a build and cannot be closed in SQL; the code says so. The new gated test holds the first build behind an open writer, swaps a drop target (and, in a second phase, the invalid index a rebuild would replace) for an operator's index on another table while it waits, and asserts the refusal, its account of what ran, the untouched operator index, and that the next start finishes the migration keeping the built indexes. Against the previous runner the same test fails because the operator's index is dropped. It is listed in test/prism-gated-tests.txt, which the proof job checks every executed gated test against. * fix: require migration 011 in the recovery evidence export Adding 011 to REQUIRED_SCHEMA_VERSIONS left the PL/pgSQL gate in scripts/prism-recovery-evidence.sql at 2 through 10, so a native database that startup refuses for lacking 011 still exported evidence, against the script's stated requirement to mirror startup. The gated regression that removes each declared version in turn caught it in CI ("missing required migration 11 was accepted"). Add 11 to the script's required_versions, and add a migration unit test that reads the declaration back from the script and compares it with REQUIRED_SCHEMA_VERSIONS, so the next drift fails in the fast unit job rather than only under the PostgreSQL gate. Point the constant's doc at the script. Co-Authored-By: Claude Fable 5.1 * fix: verify the whole online index set before recording migration 11 The online runner rechecks only the names it is about to drop. A replacement that was already valid at preflight is planned as Keep and never looked at again, and a build checks its own index only as it finishes; the other replacement's CREATE INDEX CONCURRENTLY then runs for hours on a large ledger, in which an operator can rename or replace either just as they can a drop target: the advisory lock orders only runners, and ALTER INDEX RENAME locks the index alone. The run still recorded 11 afterwards, so every later start trusted the record and accepted a database with a required replacement missing or altered. Every name the migration declares is now looked up once more inside the transaction that records the version, after MIGRATION_LOCK and before the INSERT: each created index must be valid on its table with exactly the declared definition, and each dropped name absent. Anything else refuses the run before the commit, naming the index, what it reads as now, what the migration declares, and what the run had already built and dropped; the version stays unrecorded and the next start plans afresh. The preflight, the drop rechecks and the post-build check stay; the phrasing of what a name holds now is shared with drop_planned. What remains open is the round trip between the look-up and the INSERT, since nothing here can lock an index against another session's DDL, not the hours of a build. The new gated test pre-builds one replacement so the run keeps it, holds the other build behind an open writer, renames the kept index and puts an operator's index under its name while the build waits, and asserts the refusal, that 11 is not recorded, that the operator's index and the renamed one are untouched, and that once the name is restored the next start records 11 keeping both indexes. Without the check the same test fails because the run records 11 over the swapped index. It is listed in test/prism-gated-tests.txt. Co-Authored-By: Claude Fable 5.1 * fix: finish renumbering the index trim to 012 in the gate and tests PR #391 (offer found blocks before audit landing) landed on 3.x.x first and took migration 011 for the offer lifecycle: it added 11 to REQUIRED_SCHEMA_VERSIONS, to the required_versions array of scripts/prism-recovery-evidence.sql, and to the tests that spell that constant out. Rebasing this branch onto it renumbered the share ledger index trim to 012 on the Rust side (the migration file, NATIVE_MIGRATIONS, ONLINE_MIGRATIONS, REQUIRED_SCHEMA_VERSIONS, and the gated fixtures the branch had already updated), but three places kept the old number. scripts/prism-recovery-evidence.sql still required 2 through 11. The commit meant to add the trim to it ("require migration 011 in the recovery evidence export") was written while the trim was 011; after the rebase 3.x.x had already added 11 for the lifecycle, so that commit's change to the array disappeared in the conflict resolution and only its comment survived. Startup refuses a native database missing 12 (012 is recorded only once its concurrent index builds have finished), so the export accepted a database startup refuses. The unit test recovery_evidence_requires_the_migrations_startup_requires reports exactly that, and the gated regression assert_native_metadata_required would report "missing required migration 12 was accepted". Add 12. the_pre_ddl_declaration_fence_runs_whenever_6_or_11_is_recorded, from REQUIRED_SCHEMA_VERSIONS in the refusal it expects. The fence itself is right: a record carrying 11 without the candidate_offer_lifecycle row is refused before any DDL whether or not 12 is recorded, because 012 declares no capability; only the recorded set the refusal names grew. Derive the expected prefix from schema_version_list(REQUIRED_SCHEMA_VERSIONS) so the next migration does not stale it again. tests/offer_lifecycle.rs, also from #391 and PostgreSQL-gated, compared the record after Ledger::connect(initialize=true) with 2 through 11 and expected refusals naming that set. connect applies 012 online and records it before returning, so the record there is now 2 through 12, as the branch already made candidate_window_migration.rs, ledger_2x.rs, session_sequence.rs and window_reference.rs expect. Update ALL_VERSIONS and the three refusal prefixes; this file could not be run here without PostgreSQL. The six [2..11] assertions in tests/support/index_trim.rs are deliberate: undo_012 puts a database back to 11, and the others assert that a refused online run left 12 unrecorded. The [2, 3, 4, 5, 7, 8, 9, 10, 11] inputs in migration.rs model a selectively restored record with 11 and not 6; the fence looks only at 6 and 11, so they stay as they are. cargo fmt --all also reflows index_trim.rs and ledger_2x.rs, where this branch's own assertions exceeded rustfmt's default width. Co-Authored-By: Claude Fable 5.1 * fix: name migration 012 in the index trim's expected refusals and docs Shard 3 of the PostgreSQL job fails in migration_012_refuses_a_drop_target_swapped_while_it_built and migration_012_refuses_to_record_when_a_kept_index_moved_while_it_built, both at the `error.contains(refusal)` assertion of swap_during_build. It is the same incomplete 011 -> 012 renumber as the previous commit, this time in string literals. The online runner (ledger/migration/online.rs) formats "refusing to continue migration {version}" and "refusing to record migration {version}" from the version ONLINE_MIGRATIONS carries, which is 12, and the text CI prints says 12, while the three expected refusals handed to swap_during_build still said 11. Both tests failed the same way on 4d57301, before the previous commit. The literals stay literals rather than being derived. This file is about migration 012 by name: its module doc, every test name, undo_012 and its DELETE ... WHERE version=12. ONLINE_MIGRATIONS is not public, and the one public number, the last of REQUIRED_SCHEMA_VERSIONS, would follow a future transactional 013 while the migration under test stayed 12, so deriving from it would break in exactly the case it is meant to survive. The same sweep found two comments in the file saying the run "records 11", and three lines in the "Migration 012" section of docs/prism-rust-migration.md, written on this branch, that say the runner records 11 and quote its refusal as "migration 11". Renumbered too. Everything else in the crate, scripts and docs that still says 11 or 011 belongs to the offer lifecycle, or is a deliberate pre-012 state: "back to 11", "a database at 11", and the [2..11] record assertions after a refused run. The two tests need PostgreSQL and did not run here. cargo fmt --all --check and cargo clippy -p qbit-prism-server --all-targets -- -D warnings pass. Co-Authored-By: Claude Fable 5.1 * fix: move online share-ledger index trim to migration 013 Preserve upstream migration 012's startup fence and use 013 for the online index trim. Align the migration include, recovery gate, fixtures, expected refusals, integration gate names, and operator docs. --------- Co-authored-by: Claude Fable 5.1 * v3.0.0 fix: resolve share ACK deadlines and unknown rejection labels (#398) * fix: resolve share ACK deadlines and unknown rejection labels * fix: guard histogram capacity and clarify rejection taxonomy * v3.0.0 feat: add offline fee-policy transitions (#401) * v3.0.0 feat: add offline fee-policy transitions * fix: keep rejected same-ID startups non-quiescent * 3.x.x: Add the qbit-prism-load Stratum-to-PostgreSQL load harness and dense-cadence scenario (#342) * 3.x.x: add the qbit-prism-load Stratum-to-PostgreSQL load harness Adds a new workspace crate that drives real Stratum sockets against real `qbit-prism-server run` child processes and produces the `qbit-prism-capacity-evidence/v2` artifact nothing produced before (#303), plus a side report carrying everything the artifact's schema cannot say. The crate owns: - a fake qbit node with a real height-to-hash map, a hash-to-parent map, strictly increasing chainwork, `submitblock` and `waitfornewblock`, and an in-process way to mint an external tip. It serves `1e7fffff` template bits: the stock `207fffff` makes every valid share a block, because the codec caps the share target at the network target; - window arithmetic that solves for the diff-1 share difficulty whose scaled weight makes the payout window exactly the requested length, and asserts the result rather than assuming it; - a production-shaped window seed adapted from the #264 gate fixture, parameterised by the network difficulty and the live share difficulty; - managed PostgreSQL: a primary plus one streaming standby, with the physical slot, an explicit `application_name`, the D3 synchronous flip and a teardown that runs on every exit path; - an in-harness TCP delay proxy between the frontends and PostgreSQL; - Stratum client sessions whose header recipe reproduces the server's `codec::Job::assemble_submission` byte for byte, so a share identifier computed before the submit is the identifier the ledger stores; - measurement: client ACK percentiles, server histogram bucket deltas, ORDER_LOCK wait sampling from `pg_locks.waitstart`, per-frontend CPU and RSS, reconnect statistics, time to usable work and a rejection census; - exact reconciliation with a documented canonical share-identifier digest. Unknown, stale and failed measurements stay distinct from zero, and the three configuration keys the native runtime does not read (#288) are recorded as unread rather than presented as tuning controls. Part of #271 Co-Authored-By: Claude Opus 5 * 3.x.x: document the load harness and quiesce it before the sockets close Adds the crate README: prerequisites, every flag, the D1 plan, the outputs and their schemas, how to validate the artifact, the honest-value notes, measurement hygiene, and the sizes that are blocked until #273 and #265. Two fixes a first end-to-end run found: - the run now quiesces every session and waits for outstanding submits to settle before it closes the sockets. Stopping with submits in flight manufactured indeterminate shares, two of which had already committed, and the run then reported a durability finding it had created itself. Offers a paused session will never send release their outstanding slot, so the drain cannot wait for work that is not coming; - the side report now carries the ACK p99 the artifact phases actually reached and the limit that would admit it, beside the forecast that would admit the slowest phase's rate. The ORDER_LOCK sampling predicate is derived from the hex key rather than typed in decimal, and the report prints the derivation: the high half of 0x505249534d000002 is 0x50524953, which is 1347570003. Part of #271 Co-Authored-By: Claude Opus 5 * 3.x.x: count backend refusals as offered shares the artifact cannot hide The first two-frontend run at a 20,000-share window showed what a 10 ms one-way database delay does: the refresh loop blocks, each frontend's readiness poll goes stale behind it, and 1,873 submits come back with `current chain state is unavailable`, with another 167 hitting the share-commit timeout. Those refusals were being excluded from `offered_valid_shares` alongside the races the server is entitled to lose, which let the slow_database phase report offered == acknowledged == 399 and look clean while 2,040 submits had failed. Only the transient classes are excluded now — a stale job after a tip change or a payout-revision bump, an unknown or retired job, a closed pool — and each of those is a share PostgreSQL never saw, so reconciliation is untouched. A backend refusal is a capacity result, not a harness defect, so it still does not make the run exit non-zero; it simply makes the artifact honestly invalid for that phase. The side report now carries the rejected-valid and harness-bug counts per phase, and the README says which gates bind under delay and what each would need. Part of #271 Co-Authored-By: Claude Opus 5 * 3.x.x: account for offers a paused session never sent A drained frontend restart pauses that frontend's sessions, and any offer the scheduler had already placed on a paused session is discarded rather than sent. The two-frontend run showed the gap as one dispatched share that appears in no phase's offered set. The sessions now report each discarded offer, and the side report carries the count, so `dispatched` reconciles against `offered` without a reader having to guess. Part of #271 Co-Authored-By: Claude Opus 5 * 3.x.x: separate an ACK/commit divergence from a durability loss Two back-to-back two-frontend runs at a 20,000-share window found the same thing under a 10 ms one-way database delay: two shares per run that PostgreSQL holds and that the server had already refused with `ledger-confirmation-failed`. Nothing acknowledged was ever lost - `missing_acknowledged_share_ids` was 0 in every phase of both runs - so the gap runs the other way. The mechanism is in the coordinator: the append is wrapped in `tokio::time::timeout(share_commit_timeout, save)`, and when that deadline fires the sqlx future is dropped mid-COMMIT, which PostgreSQL may still complete. The share is then credited in the payout window while its miner is told it was not confirmed. That is not the same failure as credited work disappearing, so the harness no longer reports them as one thing: - an acknowledged share PostgreSQL does not hold, or a committed share nothing explains, stays a durability loss and exits 4; - a share PostgreSQL holds that the server refused with `ledger-confirmation-failed` is an ACK/commit divergence and exits 5, listed under `ack_commit_divergence` with its phase, frontend, session, send-to-response time and the commit deadline it crossed; - harness-bug rejections move to exit 7. Divergent shares stay counted in `unexpected_committed_share_ids` and in `rejected_valid_shares`, so the artifact is honestly invalid for the phase and cannot hide them. Also from the coordinator's review of the ORDER_LOCK sampler: the lock is database-wide, so each frontend now connects with `application_name=load-fe-` and the sampler attributes waiters by it, verifying against `pg_stat_activity` that the driver really carried the name rather than assuming it. Waiters that are not this run's frontends are reported separately as foreign, and the sampling intervals are bounded above as well as below. Part of #271 Co-Authored-By: Claude Opus 5 * 3.x.x: make the printed capacity-evidence command shell-safe `SHOW server_version` returns `16.15 (Ubuntu 16.15-0ubuntu0.24.04.1)` on Ubuntu, and the harness printed it unquoted, so pasting the command it told the reader to run produced a bash syntax error rather than a verdict. Every value in the printed command is now quoted when it needs to be, and a test parses the rendered command with `bash -n` to prove a shell accepts it. Part of #271 Co-Authored-By: Claude Opus 5 * 3.x.x: keep the outstanding-submit counter exact for every submit kind A scheduled block and a re-offer both insert a pending submit, and both responses go through the path that releases an outstanding slot, but neither took a slot first. The counter is unsigned, so the first such response would have wrapped it and left that session looking permanently full to the scheduler. Both now take a slot and give it back on a write failure, and a re-offer whose write fails no longer leaves a pending entry behind that a later response id could match. Found by review, before either code path had run under load. Part of #271 Co-Authored-By: Claude Opus 5 * 3.x.x: say what a zero warm-up does to the tip measurements `--external-tips` mints during warm-up, so `--warmup-seconds 0` leaves the time-to-usable-work section empty rather than zero, and a scheduled block is followed by a burst of payout-revision rejections on every frontend. Both are now in the flag table. Part of #271 Co-Authored-By: Claude Opus 5 * 3.x.x: unit-test the two reconciliation classifications directly The decision that separates an ACK/commit divergence from a durability loss, and the decision about what belongs in a phase's offered set, were only exercised end to end by a live run. Both are now pure functions with their own tests: only `ledger-confirmation-failed` makes a committed share a divergence, a committed row the harness never offered is a loss, and the offered set keeps every backend refusal, harness bug and lost response while excluding only the races the server was entitled to lose. Part of #271 Co-Authored-By: Claude Opus 5 * 3.x.x: adopt the #322 gate rules and cite #324 for the divergence The shared test-gate variables belong to the gate crate, and `scripts/check_gate_env_reads.py` fails on a second reader of one, so the harness takes its own `QBIT_PRISM_LOAD_PG_BIN_DIR` before falling back to `pg_config --bindir`. A test pins the name and asserts it is none of the shared ones. The crate manifest now matches the others: `rust-version.workspace = true` alongside `version.workspace = true`, with edition 2021, MIT and `publish = false` unchanged. The base predates #322, which is what adds `rust-version` to `[workspace.package]`, so this branch adds it there too at the pinned 1.98.1; #322 already carries that line on 3.x.x and the integration should keep its own. Toolchain 1.98.1 found one new lint in this crate, `chunks_exact_to_as_chunks` on the word swap, now written as `as_chunks_mut::<4>().0`. The operator's decision is recorded where a reader will meet it: the README's ACK/commit divergence section, its exit-code table and the side report's own section all cite #324, and so does the evidence note. Part of #271 Co-Authored-By: Claude Opus 5 * 3.x.x: make the mid-flight kill land while submits are outstanding The first mid-flight-kill run reported zero indeterminate shares, because with no database delay an acknowledgement takes about four milliseconds: at twenty shares a second across fifty sessions, almost nothing is in flight at any instant and the SIGKILL tore down an idle socket. The phase now runs under the same one-way proxy delay as `slow_database`, which holds submits open long enough for the kill to mean something, and the kill waits for the target frontend to actually be holding work before it fires. The report carries `submits_outstanding_at_kill` so a reader can see whether the scenario exercised at all rather than reading an empty list as a clean result. Part of #271 Co-Authored-By: Claude Opus 5 * 3.x.x: measure the dense-cadence scenario against the payout revision #271's sixth acceptance criterion asks what a payout-revision bump costs every frontend when own blocks land on #224's shape: about 9 s apart, and in 18-20 s pairs. `--cadence dense` adds a side phase that lands them on that pattern and measures the cost three ways, per landing and per frontend. The authoritative list of bumps comes from the database, not from a rejection: the harness samples `payout_revision` and `clock_timestamp()` out of the singleton `qbit_prism_cluster` row every 25 ms on its side pool, outside the frontends' path and outside the delay proxy, and records every change on both clocks. One landing can cause more than one bump -- the chainwork increase at the new tip in `ledger/window.rs`, then candidate confirmation in `ledger/blocks.rs` -- so the report counts them rather than assuming. `coordinator.rs` checks the observed tip before the payout revision, which is why W2's one scheduled block produced `new tip work is pending` and not `new payout work is pending`. The two windows are therefore reported apart, with a combined window beside them, and it is the combined one that #291 should budget against. Time to new-tip work is the run's existing definition restricted to one frontend's sessions. Time to new-revision work cannot be exact: the wire carries no payout revision, so the first `clean_jobs=true` notify at or after the bump stands in for the first job built at the new revision, and every field that uses it says so. Attribution is total. A landing's span runs from its own pool tip change to the next landing's, so every rebuild-pending rejection and every bump belongs to exactly one landing or to `unattributed`, and both counts are reconciled in the report. A landing the session could not produce, one the frontend rejected and one the node rejected each get their own entry. A run with no landing reports `landings: 0` and `bumps: 0` with the reason and no windows, and the budget proposal is null rather than zero. The blocks are matched to the node by hash, not by time: the part of a share identifier after the colon is the header's display hash, which is exactly what the fake node records for `submitblock`. The gap pattern and the phase length are checked against each other at the entry boundary, because a pattern that cannot hold ten landings measures nothing. With `--cadence dense`, `--scheduled-blocks` becomes the dense phase's landing budget and `steady_state` schedules none, so the budget is not spent before the phase that measures it; without it, nothing about a run changes, and the capacity-evidence artifact is untouched either way. Part of #271 Co-Authored-By: Claude Opus 5 * 3.x.x: unit-test the cadence schedule and its attribution The report builder is a pure function over already-collected records, so the parts that decide what a number means are testable without a database. Three tests cover them: the gap generator repeats cyclically, starts at the lead-in and reserves the measurement tail, and its entry validation refuses a gap below the floor, a non-finite one, an empty field and a phase too short for ten landings; synthetic rejections and bumps land on the right landing, with the ones before the first landing counted as unattributed rather than dropped, and a landing the session never produced keeping its error text; and a run with no landing reports zeros, a reason and no percentile. A fourth asserts the wiring: the dense phase is a side phase with no proxy delay after `slow_database`, a run without `--cadence dense` does not grow one, and the artifact's phases are the same either way. Part of #271 Co-Authored-By: Claude Opus 5 * 3.x.x: document the dense-cadence scenario The flag table, the phase list and the outputs section gain the new phase and its four flags, and a section of its own says where the bumps come from, why the tip and payout windows are reported apart, which of the two times is an approximation and why, what "lost valid work" counts, and what the report says when nothing landed. Part of #271 Co-Authored-By: Claude Opus 5 * 3.x.x: ship the profile bytes the artifact's digest names (F1) `subject.database_profile_sha256` was taken over the canonical JSON string while the file was written with a trailing newline appended, so the two never agreed and `sha256sum database-profile.json` disagreed with the artifact in every bundle the harness has produced. A mismatched digest in an evidence bundle honestly reads as corruption or tampering, and nothing inside the harness could catch it: the self-check and `--expect-database-profile-sha256` both compare the in-memory digest with itself. The document is now written through `profile::write_document`, which writes exactly the bytes that were digested and says why nothing may be appended. Digesting the file's bytes instead would have made the digest a function of the serializer, which that module exists to avoid. The regression test writes the profile, reads the file back and asserts that the SHA-256 of its bytes is the digest the artifact names, that the last byte is not a newline, and that the file still parses. The README's claim now says the digest is over the file's exact bytes and that `sha256sum` is the value the artifact names. Found by the adversarial review of W2 (F1). Part of #271 Co-Authored-By: Claude Opus 5 * 3.x.x: say 7, not 5, for a harness-bug rejection (F4) The honest-values section still said the harness-bug classes exit 5, which was true before the ACK/commit divergence was split into its own class and took that code. The exit-code table and `run.rs` both say 7. Found by the adversarial review of W2 (F4). Part of #271 Co-Authored-By: Claude Opus 5 * 3.x.x: check the PostgreSQL bin directory at the entry boundary (F5) Every other input was range-checked in `Args::validate`; `--pg-bin-dir` was not, so a typo surfaced as `running initdb: No such file or directory` long after the output directory and the temporary cluster root had been created, and named neither the flag that was wrong nor what was missing. `cluster::verify_bin_dir` checks that the resolved directory exists and holds `initdb`, `pg_ctl` and `pg_basebackup`, and `run::execute` resolves and checks it immediately after `Args::validate`, before it creates anything. A `--database-url` run needs none of those binaries and is not asked for them, which is also why the check is not in `Args::validate` itself: it would make the crate's own unit tests depend on a PostgreSQL installation, and they stay database-free. The test covers the four answers: a directory that does not exist, one with no binaries, one with all three, and one missing a single binary. Found by the adversarial review of W2 (F5). Part of #271 Co-Authored-By: Claude Opus 5 * 3.x.x: let the temporary cluster root own its own cleanup (F3) `create_dir_all` ran before the `ManagedPostgres` whose `Drop` removes the root existed, so every `?` in between -- reading the current user, `initdb`, allocating a port, starting the primary -- returned with `/tmp/prism-load-` still on disk. An operator iterating on flags accumulated empty directories, and a failure after `initdb` succeeded left a whole data directory behind. `TempRoot` is created the instant the directory is, honours `--keep-artifacts` and is moved into `ManagedPostgres`, which no longer removes the root itself: the guard drops after `stop` returns, so the data directory is never removed under a live server. The whole window is closed without a second copy of the cleanup. The test writes a file into the root and asserts nothing survives the guard, and that `--keep-artifacts` keeps it. Found by the adversarial review of W2 (F3). Part of #271 Co-Authored-By: Claude Opus 5 * 3.x.x: see the foreign ORDER_LOCK holder, not just the waiter (F2) The sampler selected only ungranted rows, so a process that *held* ORDER_LOCK was invisible to every number in the section. That is exactly what a foreign stall looks like: the holder blocks every frontend, the frontends queue up as waiters and are correctly recognised as this run's own, and the report then billed the whole stall to them while asserting `foreign_waiter_samples: 0`. The adversarial review held the lock for 3.02 s from a separate `psql` and watched `waiter_seconds_estimate` inflate 19x with nothing foreign reported anywhere. The query now selects `l.granted` and keeps granted rows, and `split_lock_rows` splits one sample four ways: this run's waiters, this run's holders, foreign waiters and foreign holders. Every existing waiter number -- `max_waiters`, `mean_waiters`, `waiter_seconds_estimate`, the episode count and the longest observed wait -- still comes from this run's ungranted rows alone, so nothing changes meaning; the granted rows are new information beside them. A frontend holding the lock is the normal case and is in no foreign counter. Beside the waiter fields there are now `foreign_holder_samples`, `foreign_holder_application_names` and `foreign_holder_seconds_estimate`, and one `foreign_contention_observed` so a reader does not have to notice a zero in the right field. It is `null`, not `false`, when the sampler could not attribute rows at all: unknown is not the absence of a problem. The split is a pure function and has its own test, including the no-attribution case where nothing can be called foreign. The README now describes holders and waiters as the different events they are. Found by the adversarial review of W2 (F2). Part of #271 Co-Authored-By: Claude Opus 5 * 3.x.x: give the README the length the validator actually asks for The default gap pattern needs a 150 s phase to place ten landings, not the 170 s the README said. The number the code computes is the one in the refusal message, which a reader is more likely to meet than this sentence, so the sentence now points at it. Part of #271 Co-Authored-By: Claude Opus 5 * 3.x.x: left-join the activity row and stamp every lock poll (F2 addendum) Two ways the ORDER_LOCK sampler could still be wrong about what it saw. The inner join to `pg_stat_activity` dropped any lock row whose backend the sampler's role cannot read, so such a row was invisible rather than foreign. It is now a left join, and a row with no readable activity counts as foreign: nothing about it can be shown to belong to this run, and counting it as one of this run's frontends is the direction that hides a stall. That holds even with no attribution at all, where every other row counts as ours. In the foreign name lists it appears as `(activity row not visible to the sampler's role)` rather than as an empty `application_name`. The server timestamp was read out of the first returned row, so a poll that found no lock had no server clock and fell back to the harness's own `Utc::now()` -- one sample, two clocks. `clock_timestamp()` now comes from a one-row outer source left-joined to the locks, so every poll carries it, including a poll that finds nothing. If it is ever missing the sampler records a failure instead of substituting its own clock. Rows that are this run's own and ungranted are unchanged, so every waiter figure keeps its meaning. The split's test covers the unreadable row in both the attributed and the unattributed case. Part of #271 Co-Authored-By: Claude Opus 5 * 3.x.x: sample the lock the rebuild queues on, not just the one the append takes (F1) The dense-cadence phase exists to measure the rebuild after a landing, and the rebuild path takes SETTLEMENT_LOCK (0x505249534d000003) first and ORDER_LOCK second -- observe_chain_view, the job build and candidate confirmation or abandonment all do. The sampler watched ORDER_LOCK alone, so in an adversarial run 37.3% of this run's ungranted PRISM advisory-lock samples in the dense phase were invisible to every reported number, and a reader sizing #291's soak budget off waiter_seconds_estimate was reading part of the queueing with no way to tell which part. The sampler now selects both locks in one poll, tags each row with its objid, and summarizes them apart: each phase carries a settlement_lock block beside order_lock with the same shape and the same own/foreign split. Because one poll carries both, the two blocks cover exactly the same instants and share a samples count and a sampler cost, which the definitions say. Every existing order_lock key keeps its name and its meaning; the blocks gain taken_by, which names the server path that takes the lock, so a block says what it measures rather than leaving it to be inferred from the name. The dense section's definitions gain advisory_locks_sampled: which locks are sampled, which path takes each, which are not sampled and why, and where the blocks live. MIGRATION_LOCK and CPFP_FUNDING_LOCK stay unsampled -- neither is on the append or rebuild path this phase measures. Part of #271 Co-Authored-By: Claude Opus 5 * 3.x.x: count and verify every lost share, not only the attributed ones (F2) lost_total and lost_in_postgres were accumulated inside the per-landing, per-frontend loop, so they covered exactly the rejections a landing's span owns. A rebuild-pending rejection that fell inside no span was reported in rejection_attribution.unattributed and then silently left the lost-work census -- and was never looked up in this run's committed share identifiers. That is the one case the cross-check exists for. A landing whose pool tip change is missing -- accepted_without_node_submission, a tip record lost to a frontend restart, a block the node recorded late -- leaves its rejections unattributed, so a lost share that really is in qbit_share_ledger would have been reported as shares_found_in_postgres: 0. The check read as a clean pass in exactly the situation that would make it fail. lost_valid_work now counts every rebuild-pending rejection in the phase and publishes the split -- shares, shares_attributed, shares_unattributed -- with shares_found_in_postgres checked over all of them. The per-landing, per-frontend lost_valid_shares tables are untouched and still sum to shares_attributed. The attribution test is extended to arm the case: its unattributed rejection, and only it, is placed in the committed set, so the test asserts shares == 5, shares_unattributed == 1 and shares_found_in_postgres == 1 naming the share. Before this change that last number was 0. Part of #271 Co-Authored-By: Claude Opus 5 * 3.x.x: let the revision sampler report partial blindness (F3) blind() was `baseline.is_none()`, so it could only fire for a sampler that never read anything. A sampler whose first read succeeded and whose next thousand failed -- the side pool loses its connection, the primary restarts, the pool is exhausted -- reported blind: false, blind_reason: null and changes_observed: 0 beside an error count in the thousands. The one boolean a reader is invited to trust to tell "observed zero" from "observed nothing" said "observed zero", which is the misreading the flag exists to prevent. blind() is now also true when no change was observed and errors occurred, and blind_reason names which of the two cases it is rather than repeating one sentence for both. Errors beside observed changes stay non-blind: the sampler did see the revision move. Beside samples and errors the section now reports coverage -- samples over samples plus errors, null rather than 0 when the sampler never ticked -- so a sampler that watched 3% of the phase is visible as such whatever blind says (EP-OBSERVABILITY). Part of #271 Co-Authored-By: Claude Opus 5 * 3.x.x: keep the approximation label on the summaries (F4) Time to new-revision work is an approximation -- mining.notify carries no payout revision, so the first clean_jobs notify at or after the bump stands in for it -- and it was labelled in two of the three places it appears. The per-landing, per-frontend tables carry new_revision_work_approximation beside the number, and definitions carries it under time_to_new_revision_work. The four summaries.overall and summaries.per_frontend[] fields carried nothing. Those are the numbers people quote: the headline tables in the run reports carry a "time to new-revision work ms" column and it comes from the summaries. Quoted from there, an approximation built on a clean_jobs proxy reads as a measured time to the first job at the new payout revision, which is the one thing the Stratum wire cannot tell you. The label is carried into the summarised objects rather than adding definitions entries keyed on the summary names. Both fixes were acceptable; this one puts the label beside the number, so it survives a reader copying the object out, while a definitions entry only helps a reader who goes looking -- and the finding was that readers quote the summary and do not go looking. Part of #271 Co-Authored-By: Claude Opus 5 * 3.x.x: give summarize a unit, because counts are not milliseconds (F5) measure::summarize hard-coded unit: "milliseconds", and five of the dense section's distributions are counts of shares. They shipped as "combined_rebuild_pending_rejections_per_landing": { "clock": "counts, not milliseconds; one sample per landing", "unit": "milliseconds", "p50": 61.0, "p99": 85.0, "max": 85.0 } with the clock string apologising for the unit rather than the unit being right. A consumer generic over the summary shape -- a dashboard, a diff of two runs -- reads `unit` and renders 85 discarded shares as "85 ms"; it does not look at a clock field. EP-OBSERVABILITY asks that every number carry its unit, and here one was carried and was wrong, which is worse than absent. summarize now takes the unit. measure::MILLISECONDS and measure::COUNT name the two in use; the five count call sites pass COUNT through a count_summary helper and everything else passes MILLISECONDS. COUNT_CLOCK drops the apology and names the clock again: one sample per landing and frontend, counting rejections stamped on the harness monotonic clock. Part of #271 Co-Authored-By: Claude Opus 5 * 3.x.x: make bumps mean what definitions.bump says (F6) definitions.bump is "an observed change of qbit_prism_cluster.payout_revision", and the top-level bumps key published attributed_bumps -- the subset a landing's span owns. The crate's own test made the divergence concrete: bumps 3 while bump_records held 4 and bump_attribution.unattributed was 1. The failure is a no-landing run in which the revision moves anyway -- an operator touching the cluster, a migration, a second harness. It reported landings: 0 and bumps: 0, which the README presents as the honest no-landing output, while changes_observed and bump_attribution.unattributed both said otherwise two keys away. Nothing was hidden, but the headline number meant something other than the word next to it. bumps is now revisions.changes.len(), equal to revision_sampler.changes_observed, and bump_attribution carries the split -- it gains observed so the three reconcile in one place. definitions.bump says which key is which, and unknown_is_not_zero no longer claims a no-landing run reports bumps 0. The test that pinned the old meaning now asserts bumps == 4 == changes_observed with attributed 3 / unattributed 1, and the no-landing test gains the case the finding named: nothing landed, the revision moved twice, bumps 2, both unattributed. Part of #271 Co-Authored-By: Claude Opus 5 * 3.x.x: count the landings that have a window, not the ones that landed (F7) landings and windows_available were derived from the `landed` tally, while resolve grants a landing its attribution span whenever its own pool tip change is present -- regardless of outcome. A landing whose tip moved but whose submitblock record is missing falls to AcceptedWithoutNodeSubmission and still receives a span, and therefore a full per-frontend window table and a contribution to summaries. If that were the only such landing the section would have reported landings: 0, windows_available: false and a reason saying every attempt failed, while landing_records[].frontends and summaries.overall carried real windows and proposed_budget_for_issue_291 carried a real p99. Both now count the spans that were actually created, and the no-landing reason keys on the same number, so "a window exists" and "a window is reported as existing" cannot disagree. landing_outcomes.landed is unchanged and still reported one key away, so the two can be read against each other, and definitions.landings says which is which. Part of #271 Co-Authored-By: Claude Opus 5 * 3.x.x: name the new crate in the release-identity assertion `check_version_skew.py` prints every workspace crate it matched, and `test_checked_in_tree_has_one_release_identity` asserts that line literally. Adding `qbit-prism-load` to the workspace changes the line, so the assertion has to name it too. The script itself already passes; only the expected text was stale. Part of #271 * 3.x.x: keep an unknown commit outcome apart from a lost share PR #333 adds a third answer to the share-commit path: when the COMMIT still has no reply after the deadline and its grace window, the server answers `ledger-outcome-unknown` instead of claiming the append failed. The harness did not know the reason, so it fell through to `RejectionClass::Unknown`, and a share PostgreSQL then held was filed as "committed share that was never acknowledged" -- a durability loss, exit 4. That is a false data-loss alarm for the one case where the server was being honest about not knowing. Give it its own gap kind and its own report block. The three outcomes are three different claims: the server said no and was wrong (`ack_commit_divergence`), the server said it did not know (`unknown_outcome_commits`), and nothing explains the row at all (`durability_findings`). Only the last means credited work disappeared. The run still fails, with exit 5, because the miner was refused a share the database holds either way. Found by running this harness against PR #333 for #324's confirmation. That run produced no `ledger-outcome-unknown` at all -- with an asynchronous standby the COMMIT reply takes milliseconds -- so the gap was never hit, but the next run on a slower database would hit it. Part of #271 * 3.x.x: redact the password inside any URL the reports print (F4) `--database-url` in the usual `postgresql://user:password@host/db` form is carried to the frontends as `PRISM_DATABASE_URL`, and the predicate that decided what to redact only looked at the key: the secret list, or a name containing SEED or PASSWORD. So `database-profile.json` and `load-harness-report.json`, both meant to be attached to issues and pull requests, printed the database password in clear. Redact by value as well as by key: every URL-valued variable loses the password half of its userinfo and the value of any `password` query parameter (the form libpq and sqlx also accept), whatever the variable is called, so a URL added later is covered without special-casing its name. The user, host, port, database and the non-secret parameters stay legible, because they are what a reader of the profile needs. Review feedback on the load-harness pull request. Part of #271 Co-Authored-By: Claude Fable 5.1 * 3.x.x: start every invocation's frontend logs empty (F9) Frontend stdout and stderr were opened in append mode on every spawn, so a reused `--out` directory kept the lines of earlier invocations. The blocked-run classifier reads and classifies the whole stderr file, so one JSONB-ceiling refusal from a previous, failed run made a currently healthy run exit 3 and report itself blocked. Truncate on the first launch of an invocation and append only for a restart within it: the previous process's lines in a mid-flight-kill or drained-restart scenario are this run's evidence and must stay. The test launches a stand-in shell script as the frontend, seeds a stale refusal line into its log, and checks that the launch discards it while a restart keeps this invocation's own lines. Review feedback on the load-harness pull request. Part of #271 Co-Authored-By: Claude Fable 5.1 * 3.x.x: make an unknown server build profile need the override too (F3) The guard was `profile != Debug || --allow-debug-server`, so `BuildProfile::Unknown` passed with no override. The profile is inferred from the binary's parent directory, so a copied binary, a symlink or an installed path is `Unknown`, and a debug build at such a path could produce a qualification artifact the harness otherwise treats as invalid. Resolve symlinks before reading the directory, so a link into `target/release` is shown to be a release build, and require `--allow-debug-server` for `Unknown` as for `Debug`: a profile that cannot be determined is not evidence of a release build. The refusal says why and what to do. The check moves into `frontend::check_server_profile`, where a test can reach it. Review feedback on the load-harness pull request, confirmed by the coordinator. Part of #271 Co-Authored-By: Claude Fable 5.1 * 3.x.x: let the delay proxy front a database named by hostname (F7) The proxy's upstream was built with `"host:port".parse::()`, which only accepts a numeric address. `postgresql://user@postgres.example/db` is an ordinary PostgreSQL URL, and the SQLx connections before that point succeed on it, so external-database mode got as far as the proxy and then failed before a single frontend started. Resolve the host once, at the entry boundary, through the same system resolver SQLx used, refuse a host that does not resolve with its name, and try every address the name resolved to when a frontend connects, as `TcpStream::connect` does for a name, so a host that resolves to both address families works whichever one PostgreSQL listens on. The written upstream and the resolved addresses are recorded in the side report's `delay_proxy` block. Review feedback on the load-harness pull request, confirmed by the coordinator. Part of #271 Co-Authored-By: Claude Fable 5.1 * 3.x.x: gate startup on sessions holding work, not connection events (F12) The startup gate compared a cumulative count of `Connected` events with the session count. Every successful connection increments it, so a session that dropped and reconnected while another had not finished its initial handshake satisfied the total on the other's behalf. The phases then began although the stated precondition -- every session holds work -- was false, and the churn showed up as phase shortfall. Fold the events in one place, `Collected::apply`, and keep the set of sessions that are connected right now: added on `Connected`, removed on `Disconnected`. The gate reads that set's size. The event count stays in the report as what it is, a count of events. Review feedback on the load-harness pull request. Part of #271 Co-Authored-By: Claude Fable 5.1 * 3.x.x: record a submit under the phase that offered it (F8) A session stamped each submit with `SessionShared::phase` at the moment it sent it. An offer the scheduler placed while the session could not send -- mid-reconnect, or still in its handshake -- sat in the work channel across the phase boundary and was then stamped with the next phase. It had been counted as `dispatched` in the old phase, but landed in the new phase's reconciliation and latency, possibly during its boundary scrapes, before its measured duration even started, so a phase's rate and its artifact described different offers. Carry the phase in the work item: `Work::Submit { phase }` is stamped by the scheduler, and the session records the submit under it however late it sends it. Scheduled blocks and re-offers are sent the instant their control message arrives and keep reading the live phase. The test drives a real session against a minimal in-process Stratum server that holds the authorize reply, queues two offers in `steady_state`, moves the run to `reconnect`, releases the handshake and checks both records say `steady_state`. Review feedback on the load-harness pull request. Part of #271 Co-Authored-By: Claude Fable 5.1 * 3.x.x: tie the server binary to the revision the artifact names (F1) `coordinator_revision` came from `git rev-parse HEAD` in the harness's checkout, while the server under test was whatever `--server-bin` pointed at. A `target/release/qbit-prism-server` left over from an earlier checkout kept the tree clean and the recorded revision current while the bytes that ran were older, and the artifact then self-validated and attributed measurements to a commit that did not produce them. The binary digest identifies the bytes, not the source, so it did not help. The server embeds no build revision, and this crate cannot add one. What it can do is apply Cargo's own freshness test: the dep-info file Cargo writes beside the binary lists every workspace source it was compiled from. If that file exists, names this checkout's server crate root, and no listed source -- nor `Cargo.lock` or the workspace `Cargo.toml`, which the list omits -- is newer than the binary, then the binary is what this tree builds, and with a clean tree that is HEAD. Otherwise the revision cannot be established and the run is refused with the reason: no dep-info (a copied or installed binary), a newer source, or a different checkout. `--allow-unverified-server-revision` runs anyway and forces `artifact_kind: example`, as `--allow-dirty-tree` does, and the side report records the evidence under `versions.server_revision_evidence`. Review feedback on the load-harness pull request, confirmed by the coordinator. Part of #271 Co-Authored-By: Claude Fable 5.1 * 3.x.x: withhold the artifact when the run aborts (F6) Crossing the memory floor or losing a frontend only broke out of the phase loop; execution continued into artifact generation. An abort before the three required phases made `artifact::build` error out before the side report was written or `EXIT_ABORTED` returned, so the abort report never existed. An abort during a long `slow_database` phase emitted the partial phase as `completed: true`, so an aborted run could produce a self-validating qualification artifact. Decide about the artifact before building it. An aborted run withholds `capacity-evidence.json`, removes one an earlier run left at the same path so nothing self-validating survives the abort, writes the side report with `aborted` set, the cut-short phase marked `completed: false` and `validator.artifact_written: false` with the reason, and exits 6. A completed run is unchanged. Review feedback on the load-harness pull request. Part of #271 Co-Authored-By: Claude Fable 5.1 * 3.x.x: drive the drained restart without stalling the scheduler (F2, F5) The reconnect phase's restart awaited `drained_restart` from inside the 1 ms scheduling loop, so the drain, kill, relaunch and readiness wait stopped every other frontend's scheduled traffic for the whole outage, and afterwards the elapsed-time token calculation turned the missed interval into one burst plus shortfall. The phase exists to measure the offered rate *while one frontend is unavailable*, which is exactly what that stalled scheduler could not measure (F2). The same function waited a fixed 10 s for the frontend's submits to settle and then restarted it anyway. That is reachable with the default 15 s share-commit timeout and likely with a longer one or a contended database, and killing the process then converts the harness's own in-flight requests into `NoResponse` records inside the artifact's reconnect phase, misclassifying the experiment (F5). Replace the awaited function with `restart::RestartDriver`, a state machine the loop polls: every poll returns after bounded synchronous work (a counter read, a channel `try_recv`, a kill-and-spawn) and the readiness probe and metrics scrapes run in spawned tasks. The drain waits the configured share-commit timeout plus a 5 s margin, one deadline carried through the dependent work, and a frontend whose sessions still have submits outstanding after that is not restarted: the driver reports the failure and the run aborts with exit 6 and its report, rather than proceeding and pretending it drained. A restart still in flight at the phase boundary is seen through so its sessions are retargeted before the next phase offers to them. Each side of the restart is scraped, which the next commit uses. The tests launch stand-in frontends (a shell script and a one-line HTTP server for `/healthz` and `/metrics`) and check that no single poll exceeds 100 ms while the whole restart takes hundreds, that only the restarted frontend's sessions are paused and retargeted, and that a frontend whose submits never settle is left running and the failure named. Review feedback on the load-harness pull request. Part of #271 Co-Authored-By: Claude Fable 5.1 * 3.x.x: assemble the server ACK histogram across a restart (F10) The reconnect phase restarts a frontend between its two boundary scrapes, so that frontend's in-process Prometheus counters go back to zero, and subtracting the old process's pre-phase values from the new process's post-phase values gave negative counts, sums and buckets. The coordinator confirmed it against real data: in two runs of the #324 reproduction the `reconnect` phase's `server_share_ack_seconds` for `load-fe-1` reported `accepted` counts of -1273 and -1238 with every bucket negative. The report had been shipping negative share counts. Assemble the delta from one segment per process, each bracketed by its own scrapes: the phase-start scrape to the pre-kill scrape, then the post-restart scrape to the phase-end scrape, summed. The entry says what happened -- `counter_resets`, `segments`, and a note -- so a summed number is never mistaken for a plain difference. A reset no scrape bracketed (a mid-flight kill) leaves `counts` empty with `unavailable_reason` set: unknown, which is kept apart from zero (EP-OBSERVABILITY). A counter that went backwards between two scrapes with no recorded restart is reported the same way rather than as a negative number, so the failure mode cannot come back by another route. The test replays the coordinator's case: 1300 before, 27 after, no restart recorded gives unknown with the reason, not -1273; a restart without scrapes gives unknown with the reset count; a restart with both sides scraped gives (1500 - 1300) + (27 - 0) = 227 over two segments; a failed scrape on one side fails the whole delta. Review feedback on the load-harness pull request, confirmed by the coordinator against measurement data. Part of #271 Co-Authored-By: Claude Fable 5.1 * 3.x.x: resolve dep-info paths before matching them, and record why try_offer is safe Two things found reviewing the review fixes locally. The revision check compared a canonicalised `repo_root` against the paths Cargo wrote into the dep-info file, which are the paths it was invoked with and are not canonical. On macOS a checkout under /tmp or /var really lives at /private/tmp or /private/var, and a symlinked home or work directory does the same anywhere. The exact `PathBuf` comparison then refused a binary that had in fact just been built from this checkout, so the run either stopped or fell back to an `example` artifact -- the opposite of what the check is for. A plain Linux CI checkout is unaffected, which is why this passed on the worker's host and failed here. Resolve each listed source before matching, keeping an unresolvable path as written so the error still names what the file said. `try_offer`'s safety was established during review and then lived only in a report that is not committed. It rests on the function being synchronous and the counter being read from one task, neither of which is visible at the call site: the wrapping `fetch_sub` through `usize::MAX` between the `try_send` and the `fetch_add` is unobservable only under those conditions. Written down, because the obvious tidy-up -- making it async, or reading the counter from a spawned task -- reintroduces the bug that was argued not to exist. Part of #271 * 3.x.x: make a paused session ineligible for offers, not a queue for them (G1) The drained restart pauses the affected frontend's sessions and lets the scheduler keep running, but the scheduler still offered to every session and `SessionHandle::try_offer` had no notion of paused, so each paused session kept accepting offers into its bounded queue. That was wrong in three ways. Every offer a paused session absorbed was a token taken from the frontends that were up. The queued offers counted as outstanding, so the drain that waits for outstanding to reach zero could not complete while the scheduler ran and the restart aborted the run at its deadline instead of happening. And whatever had queued went out as one burst on resume, which is the failover behaviour the reconnect phase exists to measure rather than manufacture. The paused state was private to the session task, so the scheduler could not see it. It is now a shared flag on `SessionHandle`, read by both the decisions it governs (EP-STATE): the session task reads it to decide whether it may send, and `try_offer` refuses while it is set so the token goes to the next session. The restart driver sets it before it sends the pause, so nothing queues between the decision and the task reading its control channel, and clears it with the retarget that ends the restart; the teardown pause sets it the same way. A client-initiated reconnect that lands on a paused session no longer lifts the pause, since the pause belongs to the frontend being restarted and lifting it would resume offers to a session whose frontend is down. Test: a_paused_session_is_not_offered_work_until_it_is_retargeted keeps offering through a stand-in restart and shows the paused session refuses every offer, the healthy one takes the token, the drain completes on the in-flight submits alone, and the retarget makes the session eligible again. It fails at the previous head on the first refusal. Part of #271 Co-Authored-By: Claude Fable 5.1 * 3.x.x: count the package manifests among the revision check's inputs (G2) The check that ties the server binary to the checkout's HEAD compared the binary's timestamp with Cargo's dep-info sources plus the lock file and the workspace manifest. It left out the server package's own Cargo.toml and the manifests of its path dependencies. A package manifest can change what is built without touching a source or the lock file -- enabling a feature of a dependency already in the lock is the plain case -- so a binary built at an older checkout, with only such a change between it and the current clean HEAD, passed every timestamp comparison, was reported Established, and the artifact attributed the run to a HEAD that did not build it. That is the stale-binary case the check exists to close (EP-OBSERVABILITY). The server manifest is now always an input, and the manifests of the path dependencies are derived rather than listed: Cargo's dep-info covers the server's path dependencies transitively, so each listed source is walked up to the nearest Cargo.toml inside the checkout, and that set of manifests is checked with the rest. The list comes from the same record as the sources, so it cannot drift from the manifests as a hand-written one could. `sources_checked` counts the distinct inputs, manifests included; the README and the flag's help say so. Test: a_package_manifest_newer_than_the_binary_leaves_the_revision_unestablished builds a fake checkout with three crates, checks the derived manifest set, and shows that a newer server manifest, a newer path-dependency manifest, or a missing server manifest each leave the revision unestablished naming the file. It fails at the previous head, where the newer manifests go unnoticed. The existing revision test's fixture now carries the crate manifests it was missing. Part of #271 Co-Authored-By: Claude Fable 5.1 * 3.x.x: clear an earlier run's outputs when an invocation takes --out (G3) The abort path withheld the artifact and removed a stale capacity-evidence.json, but the blocked path did not: it wrote its side report and left whatever else was in the directory. Reusing a default --out after a successful run and then blocking during frontend startup left a fresh blocked report beside the previous run's self-validating artifact and database profile, which read as evidence for a run that did not produce them (EP-OBSERVABILITY). The same held for a run that failed before it measured anything, which writes no report at all. Rather than teach the blocked path what the abort path knows and leave the error path still wrong, the obligation is met once, at entry: when an invocation takes the directory it removes any capacity-evidence.json, database-profile.json and load-harness-report.json an earlier run left there, and both the completed and the blocked side report list what was removed under stale_outputs_removed. Every exit path then has the same guarantee -- the directory holds only this invocation's outputs -- without each one having to repeat the removal, and a path added later gets it for free. The abort path keeps its own removal as defence in depth; the frontend logs are untouched here because Frontend::launch already starts each one empty. Test: an_invocation_removes_the_previous_run_s_outputs_when_it_takes_the_directory seeds an earlier run's three outputs and a frontend log, takes the directory, and shows the outputs gone and named, the log kept, and a partial leftover removed by name. The entry point is new, so at the previous head the test does not build rather than fail. Part of #271 Co-Authored-By: Claude Fable 5.1 * 3.x.x: read the proxy's upstream with the parser that accepted the URL (G4) `host_port` decided whether a database URL already carried a port by looking for a colon in its authority. A bracketed IPv6 literal such as `postgresql://user@[::1]/db` has colons inside the brackets, so the authority was returned as `[::1]` with no port, `DelayProxy::open` handed that to a lookup that needs one, and the run failed after SQLx had already connected with the same URL. The URL is now read with SQLx's own connect-options parser, the one that accepted it, so the host and port the proxy fronts are the ones SQLx dialled: a hostname, an IPv4 or a bracketed IPv6 literal, with or without an explicit port, the libpq-style `host` and `port` query parameters, and the default port when none is given (EP-VALIDATION). An IPv6 literal is bracketed on the way out, which is what a `host:port` lookup needs. A URL naming a Unix socket is refused with the reason, since the proxy cannot front one, and one that does not parse is refused with the URL redacted. Test: a_bracketed_ipv6_authority_keeps_or_gains_its_port covers both bracketed forms, keeps the hostname and IPv4 cases, adds the query parameter, socket and unparsable cases, and checks the result composes with the authority rewrite that follows. At the previous head it fails on the portless bracketed form, which came back as `[::1]`. Part of #271 Co-Authored-By: Claude Fable 5.1 * 3.x.x: refuse a socket that arrives as a path-shaped host, and stop pinning a platform default The G4 fix reads the upstream through SQLx's own parser, which is right, but it inherited a default this code had not had to think about before. A URL with no host does not fail: SQLx supplies one, and what it supplies depends on the platform. On macOS it is a Unix socket directory, which arrives as a *host* that happens to be a path, so `get_socket()` is empty and the socket guard does not fire. `lookup_host` is then handed `/private/tmp:5432` and reports a path as a DNS failure, which is the confusing error G4 set out to remove, one case further along. Refuse a path-shaped host with the same reason as an explicit socket, and name the offending host in the message. The test pinned that default to `localhost:5432`, so it passed on the Linux host it was written on and failed here. A platform-dependent value is not something to assert; what must hold everywhere is that a socket is refused however it arrives and a frontable default is a TCP host. The case now asserts that instead, and holds either way. Part of #271 * 3.x.x: keep the endpoint parameters out of the proxied URL, and prove the delay before believing it (H1) `rewrite_host` replaced the authority and kept the query string. SQLx reads the libpq-style `host`, `hostaddr` and `port` parameters after the authority and lets them win, so with `postgresql:///db?host=db.internal&port=5433` every frontend dialled the database directly, the proxy carried nothing, and the `slow_database` phase reported a 10 ms delay that was never applied. The rewrite now drops those three parameters, compared percent-decoded as SQLx decodes them, and keeps every other option in its order; user info is looked for inside the authority only, so an `@` in the query is not taken for it. The run no longer assumes the proxy is on the frontends' path. Before the run, with the slow phase's delay set, and again before every phase with its own delay set, a SELECT 1 round trip is timed through the proxied URL exactly as the frontends received it. Each direction is held once and the proxy's sleep never returns early, so a proxied trip costs at least twice the delay; one that comes back sooner did not go through the proxy. The entry check refuses to start, and a delayed phase that fails it aborts the run with the artifact withheld. Every phase's observation and the floor it was checked against are in the side report. Tests: endpoint_parameters_never_survive_the_rewrite (fails at the base with the bypass URL rewritten to itself), a_round_trip_that_did_not_pay_the_delay_is_refused. Part of #271 Co-Authored-By: Claude Fable 5.1 * 3.x.x: change the proxy delay only once the previous phase has settled (H2) At a phase boundary the proxy's delay was set the moment the previous phase's scheduler returned, with that phase's last submits still in flight. The proxy reads its delay per chunk, so reconnect-phase submits could pay the slow-database delay and slow-database submits outstanding when it was cleared could finish without it, each keeping the phase stamp it was offered with. A phase's rate and p99 then no longer described the delay it reported. Teardown had the same shape: the delay was cleared before the drain. The boundary now applies a phase's delay through `apply_phase_delay`, which leaves a delay that is not changing alone, otherwise waits for every session's outstanding count to reach zero -- nothing is offered meanwhile, the scheduler has returned -- and only then sets the new delay. If the submits have not settled within the share-commit timeout plus its margin, the delay is left where it was and the run aborts with the count, as the reconnect phase's restart already does, rather than leak records across the change. Teardown clears the delay after its drain. The wait is recorded in the next phase's entry, null when the delay did not change. Test: a_phase_delay_is_applied_only_once_the_previous_phase_has_settled, against a real DelayProxy and stand-in sessions: unchanged delays wait for nothing, a change waits for the last outstanding submit, and a change whose submits never settle is refused with the delay untouched. Part of #271 Co-Authored-By: Claude Fable 5.1 * 3.x.x: summarise ACK latency over acknowledgements only (H3) `phase_latency` filtered on the phase and `!reoffer`, so a refused share's response time was summarised with the acknowledgements, and the artifact's overall ACK figures were built the same way. A refusal is answered in microseconds because the server never reached PostgreSQL for it, so a phase dominated by refusals reported a p50 of 0.70 ms while its accepted shares took seconds, and an artifact could pass the ACK p99 limit its accepted shares were over. Refusals are already kept out of offered_valid_shares; their timing had no business in the ACK percentiles either. `ack_latency` now keeps only `Outcome::Accepted` records that are not re-offers, and is used for every phase's ack_p50/p99 and for the overall figures. The refusals' timing is not dropped: each phase's side-report entry carries `client_rejection_latency` beside `client_ack_latency`, with a definition saying which is which. Every other summary that walks SubmitRecords was checked: rejection_report counts refusals by reason, classify_gaps reads a refused share's own latency to say whether the answer came after the commit deadline, the mid-flight report reads re-offers, the dense-cadence windows read refusals by the instant they were read, and time_to_usable_work reads tip sightings. Each of those means to read what it reads; none feeds the ACK figures. Test: ack_latency_is_the_latency_of_acknowledgements_only -- two accepted shares in the seconds and ninety-eight sub-millisecond refusals give a p50 of 2,400 ms, not 0.7; the same holds for the overall aggregation, and the refusals are summarised apart. At the base the same records gave a sub-millisecond p50 through the old filter. Part of #271 Co-Authored-By: Claude Fable 5.1 * 3.x.x: stop the new-revision search at the landing's span (H4) The search for a frontend's first clean_jobs job after a bump ran to the end of the phase. When the frontend had not served the new revision before the next landing, that landing's own clean_jobs notify was selected and reported as the earlier landing's new-revision work: time_to_new_revision_work was understated by however long the frontend had really gone without, and rejected_before_new_revision_work stopped at the wrong event. The search now ends at the landing's span, which is where the next landing's tip change begins. A frontend with no such job inside the span is its own outcome: sessions_with_new_revision_work is 0, the time is unknown rather than a number, rejected_before_new_revision_work is null, and both carry new_revision_work_unavailable_reason saying no job at the new revision was seen inside the span. The approximation label and the definitions say where the search stops. Test: new_revision_work_is_never_borrowed_from_the_next_landing. At the base the first landing's frontend reported one session with new-revision work, 9.3 s of it, and two rejections before it; all three were the next landing's. Part of #271 Co-Authored-By: Claude Fable 5.1 * 3.x.x: attribute a reconnect to the phase that asked for it (H5) A reconnect record was stamped with the run's phase at the moment the new connection completed. A reconnect started near the end of the `reconnect` phase that finished its handshake after `slow_database` began was therefore labelled slow_database: it went missing from phases.reconnect.reconnect_events, which can fail the reconnect gate although the reconnect the run configured did complete, and turned up under a phase that configured none. `Control::Reconnect` now carries the phase that sent it, as `Work::Submit` has since the same mistake was fixed for submits, and the session keeps the initiating phase for the reconnect in progress: the sender's phase for a requested reconnect, and the run's phase at the instant the socket closed or a retarget asked for a fresh connection. Every attempt's record, failed or completed, carries that phase however many boundaries pass before the reconnect completes. Test: a_reconnect_is_attributed_to_the_phase_that_asked_for_it, against the fake Stratum server with the new connection held inside its handshake while the shared phase moves on. At the base the record read "slow_database". Part of #271 Co-Authored-By: Claude Fable 5.1 * 3.x.x: withhold the artifact when a hard block is logged after startup (I1) The startup check re-read every frontend's stderr once all of them were ready and returned blocked on a JSONB-ceiling refusal. The same scan ran again after the load stopped, but its result only went into the side report's log_matches under a hard-coded `blocked: false`: a ceiling refusal logged during the run -- a scheduled-block rebuild persisting a job over the container ceiling, say -- left the artifact built, written, self-validated, and the run able to exit 0. A candidate refusal leaves ordinary shares flowing, so the capacity validator's own gates would not necessarily catch it (EP-OBSERVABILITY). The late scan is now checked with `is_hard_block` like the early one. A hit withholds the artifact with the log line as its reason, removes a stale artifact at the path as the abort path does, sets `blocked.blocked` true with the line under `blocked.error`, and exits 3 ahead of every other outcome; the side report still carries every number the run produced. `artifact::write_or_withhold` takes a `Withhold` naming which claim applies -- aborted or blocked -- instead of an abort reason alone, and the exit code order is one function, `RunOutcome::exit_code`, so a test can pin it. Test: a_hard_block_logged_after_startup_withholds_the_artifact_and_exits_blocked. At the base a ceiling line found by the final scan changed nothing: the artifact was written and the exit code came from the reconciliation findings alone. Part of #271 Co-Authored… * v3.0.0 fix: isolate public-reader PostgreSQL credentials (#399) * fix: isolate public-reader PostgreSQL credentials * test: qualify literal public-reader credentials * 3.x.x: Keep one-shot operator commands out of frontend heartbeat registration (#412) * Keep one-shot operator commands out of frontend heartbeat registration (3.x.x) self-check, import-audits, backfill-ctv and broadcast-ctv connected through the frontend constructors, so every invocation registered a starting heartbeat that nothing ever attested stopped: a generated instance ID left a row fatal-state clear refuses, and a configured one overwrote the live frontend's status and session-owner token. Add Ledger::connect_tool and Coordinator::new_tool, a third registration mode beside the frontend and the recovery operator: the same startup gates (schema version, declared capabilities, migration source, halt guard and cluster fingerprint) with no heartbeat write. Claims a tool takes remain fenced by their own tokens. Part of #260, follows #360 (fatal-state recovery) and #362 (heartbeat writers). Refs #381. * Assert self-check preserves the live frontend row and document the non-registering tools The self-check CLI test asserted the leftover starting heartbeat the old constructor wrote over the seeded legacy health payload. It now proves the opposite: the frontend's row, session-owner token included, is byte-identical after the diagnostic ran under its ID, and a generated ID leaves no row at all. The slow-cadence sample stops the fixture frontend first, since the diagnostic no longer re-registers over its row. Runbooks: the fatal-state recovery and CTV sections, the HA reference's self-check description, the migration guide's import/backfill notes and the 3.0.0 release notes state that the one-shot commands keep every startup gate but write no heartbeat, and that rows left by earlier builds still need the documented investigation. Refs #381. * Add a regression for one-shot tools registering no heartbeat (#381) Cover the four one-shot commands (self-check, import-audits, backfill-ctv, broadcast-ctv) against a real PostgreSQL cluster: a generated instance ID leaves no qbit_prism_instances row on success or on failure, a live frontend's row (status with its session-owner token, heartbeat_at and started_at) is byte-identical after each tool runs under its instance ID, the halt guard still refuses every tool with "cluster halted", and fatal-state clear succeeds once the real frontend stops because no tool row is left to refuse. With the registering constructors restored in tools.rs the test fails at the first row-count assertion, so it guards the fix rather than the setup. Refs #381. Co-Authored-By: Claude Fable 5.1 --------- Co-authored-by: Claude Fable 5.1 * 3.x.x: time online migration pool checkout before detach (#403) * 3.x.x: time online migration pool checkout before detach * test: tighten online migration checkout assertions * 3.x.x: Preserve bounded live-regtest diagnostics when fixture startup fails (#409) * v3.0.0 test: preserve live-regtest startup diagnostics * v3.0.0 test: redact escaped live-regtest credentials * v3.0.0 test: redact secrets that carry control characters sanitize_line normalized control characters to the replacement character before replacing known secrets, so a password that carried one after percent-decoding no longer matched any stored form. A child that wrote the decoded value raw emitted nearly the whole credential. Store each secret as normalization leaves it alongside the written, Debug and JSON forms. A form that normalizes to replacement characters alone is skipped, since it would match every normalized control byte. The planted-credential table gains the raw-control case, which shows the leak without this change. * v3.0.0 test: rescan a URL that follows another without whitespace redact_urls ended a URL only at whitespace or a quote, so a second URL joined to a safe one by punctuation fell inside the first span. The authority check stopped at the second URL first slash before reaching its userinfo, and the whole run was emitted verbatim, credentials included. Hand back everything from an embedded scheme so the second URL is scanned on its own. A span already being redacted is still consumed whole. The planted-credential table gains the joined-URL case, which leaks without this change. * v3.0.0 test: match secrets against the tail before it is split A decoded password can hold a line break. The child writes it raw, the tail reader splits it across two lines, and neither half matches the stored value, so both were emitted. The same shape produced the control-character leak fixed in 1989756c, where normalization rewrote the character before the match ran. Replace known secrets over the whole retained tail, before it is split into lines again and before anything rewrites it, so the written value covers both. A secret spanning a break now joins the lines it was split across. Only a secret containing a line break can match across the join, since every other one is separated by the newline itself. This supersedes the normalized secret forms added in 1989756c, which can no longer match anything now that secrets meet the raw text; its test case stays and still fails without this change. The planted-credential table gains the line-break case. * v3.0.0 test: register CRLF secrets as the tail reader leaves them read_tail strips each line trailing carriage return, so a decoded password holding a CRLF is already a plain line break by the time sanitize matches against the tail, and the stored value no longer matches either half. Register each form with CRLF collapsed alongside the written, Debug and JSON forms. The planted-credential table gains the CRLF case, which emits both halves without this change. * v3.0.0 test: register the long pieces of a multiline secret The tail window can start inside the first line of a password that spans two. That line goes with the partial line the window cut, and the continuation survives as an ordinary complete line the whole value no longer matches, so it was emitted verbatim. Register each piece of a secret that spans lines, down to eight characters. Shorter pieces are ordinary words, and redacting those would empty the diagnostics to protect a few characters, so at most seven characters of a clipped secret can survive. The boundary test gains the clipped case. * 3.x.x: Give each integration fixture its own database (#410) * v3.0.0 test: isolate ledger fixture databases * v3.0.0 test: isolate audit fixture databases * v3.0.0 test: isolate restore session settings * 3.x.x: Run the seven ignored dependency contracts and fix the heartbeat qualification step (#408) * v3.0.0 test: require ignored issued-job dependency contracts * v3.0.0 docs: require an executed heartbeat qualification test * v3.0.0 fix: redact public readiness database errors (#416) * fix: redact public readiness database errors * fix: publish readiness before writing operator warnings * fix: categorize readiness query cancellation explicitly * fix: explain connection retries in readiness timeout diagnostics * 3.x.x: time durable audit boundary pool checkouts (#417) * 3.x.x: time durable audit boundary pool checkouts Refs #352. Observe the bootstrap, newest, and partial-history oldest boundary acquisitions without changing SQL or proof ordering, and qualify their exact checkout counts. * test: clean audit fixtures after metric check failures Return the new checkout-count and bootstrap SQL checks through the existing Result cleanup path instead of panicking before db.close. The missing-oldest-wrapper negative control still fails by one observation and now leaves no test schema behind. * v3.0.0 feat: activate compact prepared-work runtime (#397) * fix: fence compact publication and replacement lease authority Capture cold build authority before construction and revalidate node, revision, balances, epoch and expiry before atomic publication. Bind replacement leases to the exact issued publication and coherent current balance state across issue, resume and share admission. Refs #273. * fix: reprove compact authority after publication lock waits Refresh external payout authority after local publication lock backpressure without holding state locks through I/O or renewing the original expiry clock. Preserve backend unavailability for an unready current publication while returning deliberate misses for retired requested work. * fix: retain ordinary authority when a leased tip returns A proof started during replacement may finish after the node returns to the published parent. Accept only with the original current revision and fresh polling, and keep the digest override restricted to active replacement leases. * fix: enforce lease authority at commit and issued delivery * fix: retain original issuance authority across handoffs * test: preserve merged inputs in cold authority fixture * fix: preserve issuance authority across unchanged refreshes Track actual prepared publications separately from tip observation ordering. Cached refreshes retain the original proof while new prepared installations still revoke it, even at the same parent and revision. Exercise real issue, persist, resume and compact publication across both transitions. * fix: report definite commit-gate refusals precisely Preserve the established no-credit response reason while identifying a refused local gate separately from database failure. Authority loss and lock contention both prove COMMIT was not sent; neither changes unknown-outcome reconciliation. * test: observe prepared work writes and reconstruction rows * feat: wire compact prepared runtime and bounded recovery Implementation checkpoint: the runtime integration subset passes on PostgreSQL 16, while existing unit fixtures still require adaptation to metadata views before full qualification. * fix: retain initial publication and expiry across recovery waits * test: observe original audits through compact references * test: require real compact refresh and resume at 400k and 500k * test: retain original authority controls through compact runtime fixtures * test: pause completed PostgreSQL commits before reply delivery * test: exercise public compact runtime resume and Stratum reconnect * test: observe reconstructed shares and delayed refresh supersession * test: compare resume expiry with the wire clock type * test: reconstruct original compact audits through public builders * test: preserve original expiry through real SQL waits * test: bound real runtime cleanup and every recovery admission wait * fix: retain only CTV presence in issued runtime metadata * test: qualify first large template insertion with a real accepted block * test: revalidate issuance after completed commit response waits * test: require completed commit publication races in the gate manifest * docs: record end-to-end validation at the integration head * test: compare only the CTV presence carried by runtime metadata * fix: supply original compact balances to candidate preparation * test: cover optimized compact scale shard ownership * test: await completed repair cleanup ownership * test: retire compact refresh JSONB ceiling exception * fix: retain resume capacity through metadata decoding cleanup * fix: retain snapshot admission through decoding and cleanup * test: isolate compact refresh WAL measurements on private primaries * fix: preserve published payout compatibility on compact resume * test: identify compact reconstruction timeout phases * refactor: separate compact runtime fixtures and shared builders * test: buffer observer socket reads during compact reconstruction * test: analyze bulk-loaded shares before runtime qualification * 3.x.x: Fence the mid-flight kill census and recognize quorum replication (#421) * 3.x.x: Fence the mid-flight kill census by record identity (#406) The mid-flight kill's census -- the no-responses it re-offers, and the only shares it exempts from `classify_gaps` -- was the suffix of the submit log after the length it had when the kill began. A no-response a victim session emitted before the kill could still be queued behind a busy collector when the SIGKILL landed; the kill's own post-kill barriers then flushed it, and it arrived in that suffix. `indeterminate_after_kill` read it as kill-induced, and if the share had committed, `classify_gaps` exempted an unrelated mid-run disconnect from the ordinary no-response check and the run stopped exiting 5. That is a false negative in harness attribution, on the one question the harness exists to answer. Pausing the sessions does not close it: a pause stops a session taking new work, but the event is already in flight when the pause is sent, so no snapshot point makes an index-based boundary correct. Fixed with explicit event identity rather than a pre-kill collector barrier. `SessionShared` carries a run-wide `kill_fence`; every `SubmitRecord` is stamped with the fence its session reads in the thread that builds the record; `KillDriver` bumps the fence once, immediately before `kill()`, and the census is the killed frontend's own non-re-offered no-responses whose fence is at or above that value. A barrier before the kill would have made the census depend on the collector being idle at exactly the wrong moment -- it would have to drain what is in flight, which is a wait of unbounded length inside the scheduling loop, and it would still say nothing about a record emitted a microsecond later. Identity is decided at the point the record is created, so no delivery lag can move a record across the boundary and no wait is added to the kill (EP-STATE). `submits_before` is gone, along with the slice it indexed; there is no index-based boundary left in the census path. The post-kill `CensusBarrier` stays: the fence gives the census its membership, the barrier gives it completeness, and both are needed. The fence is published before the SIGKILL rather than after it, so a no-response the kill itself caused cannot fall below the boundary: over-including a nanosecond of pre-kill time costs a re-offer, while under-including would report the kill's own destroyed answer as a transport loss and manufacture a spurious exit 5 (EP-OBSERVABILITY). `outstanding_at_kill` is unchanged -- still the count read immediately before `kill()`, still the report's kill boundary. The scenario is unchanged: no drain before the kill, healthy frontends still schedulable, waits still bounded (EP-ERRORS). Tests: a pre-kill no-response held behind the collector and released after the kill stays out of the census and still reaches exit 5 on its committed share; a kill-induced no-response is still re-offered and an acknowledged re-offer PostgreSQL lacks still exits 4; the kill still interrupts three pending submits, keeps the healthy frontend schedulable throughout and reports the count at the kill boundary. The existing census unit test now places the pre-kill record last in the log, where a length-based boundary would have claimed it. Co-Authored-By: Claude Opus 5 * 3.x.x: Document the mid-flight kill census fence (#406) The README described the census as the no-responses recorded after the kill, which was both the old implementation and the defect. Say what decides membership now: the fence each record carries from the moment it was built, against the fence the kill publishes in the instant before the SIGKILL. Say why the barriers remain -- membership and completeness are two facts -- and why the fence is published before the signal rather than after it. Only the mid-flight-kill sections are touched. Co-Authored-By: Claude Opus 5 * 3.x.x: Pin where the mid-flight kill census fence is published (#406) The fence closed the collector-lag hole, but nothing pinned *where* it is published. Moving the bump from immediately before the SIGKILL into KillDriver::start passed the whole suite, and that is a defect, not a refactor: start runs when the phase decides to kill, and poll then waits up to WORK_WAIT for the target to hold work. A fence published at construction sweeps every record built during that wait into the census -- up to twenty seconds of ordinary mid-run no-responses exempted from the ordinary check, which can suppress exit 5 on a share that committed. That is the widening this issue exists to prevent, approached from the other side. The new test asserts the run fence is unchanged by construction and by the polls that find nothing outstanding, stamps a no-response during that wait, then asserts the killing poll advances the fence exactly once and that the census holds only the post-kill record. * 3.x.x: Recognize quorum replication in the load harness (#407) `detect_replication` read only `sync_state = 'sync'` as synchronous. An external cluster with `synchronous_standby_names = 'ANY n (...)'` reports its candidates as `quorum`, which fell through to `async`: a correct `--replication sync` run was refused as a contradicted premise (exit 8), and a `--replication async` declaration agreed with the wrong observation. Split the read from the classification. `detect_replication` keeps the I/O and the read-error arm; `classify_replication` is a pure function over the `sync_state` column with the arms in their existing order: no rows is `none`, any null state is `unknown`, any `sync` or `quorum` row is `sync`, and everything else is `async`. `potential` is deliberately not synchronous: under `FIRST n` it is a standby that would be promoted if a member left, and no commit waits on it. The null arm stays above the synchronous one so a hidden state beside a visible one is still unknown. The managed cluster gains a `SynchronousMethod` so a test can bring up the `ANY 1 (prism_standby_1)` topology with the same code that runs `FIRST 1`; `FIRST 1` remains the default and the only method the harness itself uses, and each method waits for the one `sync_state` it produces. Tests: unit coverage of every PostgreSQL 16 `sync_state` value, null, mixed rows and no rows on the pure classifier; a premise test proving a quorum observation agrees with a declared `sync` and contradicts a declared `async` at entry and after the load, through the premise block and the exit-8 path; and a gated test against a real PostgreSQL 16 primary and standby under `ANY 1`, alongside the managed `FIRST 1` path, registered in the gated-test list so the native job must execute it. The README's replication-premise bullet now names `quorum` as recognized beside `sync` and `potential` as not synchronous; its existing sentences on the null, unreadable and no-row cases are unchanged. Co-Authored-By: Claude Fable 5.1 * 3.x.x: Stamp the mid-flight kill fence when the closure is observed (#406) The census fence closed the gap between a session emitting a record and the collector applying it. It did not close the gap between a session observing the cause and building the records, and on the socket-closure path those are two different moments. The reader task queues `Incoming::Closed` the instant the socket ends. `outstanding` is still non-zero, because nothing has failed the pending submits yet, so a `KillDriver` polling just then still sees work in flight, pauses the session and bumps the fence. The session's `select!` is biased on control, so it takes that pause first and only reaches the queued closure on the next pass, where `fail_pending` read the fence -- by then the kill's value. A disconnect that happened before the kill, for its own unrelated reason, was therefore stamped as kill-induced: the census re-offered it and `classify_gaps` exempted it from the ordinary mid-run no-response check, so a share the database holds could evade the acknowledgement-loss check and the run exit 0. `quiesce` had the same shape at its close arm. The identity is now captured when the event happens rather than when the record is convenient to build (EP-STATE, EP-OBSERVABILITY). `Incoming::Closed` carries the fence the reader read at the instant it saw the socket end, in both the end-of-stream and the read-error arm, and `fail_pending` takes the moment as a parameter so every caller states which one its records belong to: the carried fence for an observed closure, and a fence read on the spot for the causes that are happening right there -- a reader that simply stopped, the quiesce deadline expiring, the run ending. `a_socket_closed_before_the_kill_is_not_stamped_as_kill_induced` drives the real ordering through a real session on a real socket: the closure is observed and queued, the pause and the bump land while it is still queued, and `fail_pending` runs only afterwards. The no-response that comes out is outside the census, is not re-offered, and still reaches the ordinary acknowledgement-loss path and exit 5. With the fix reverted the test reports the kill's fence on that record and fails. Co-Authored-By: Claude Opus 5 * 3.x.x: Capture the kill fence before handling a write failure (#406) The last no-response construction site still read the fence where the record is assembled rather than where its cause was observed. A submit whose write fails stays counted as outstanding until offer's caller decrements it, so a KillDriver polling in between still sees work in flight and can pause, bump the fence and SIGKILL while that branch is removing the pending entry and building the record. The later read then stamped a write failure that preceded the kill as the kill's, and the census would re-offer it and report that the kill interrupted work it never touched. The fence is now read as the first operation after the write error is observed, the same rule Incoming::Closed already follows. consume's site is left alone: it builds answered records, which the census never considers. This window is closed by construction rather than by a test. Provoking it needs a write failure racing a kill inside a few synchronous instructions, which no deterministic test can pin without contorting the offer path. * v3.0.0 test: measure job persistence through public Stratum (#402) * fix: fence compact publication and replacement lease authority Capture cold build authority before construction and revalidate node, revision, balances, epoch and expiry before atomic publication. Bind replacement leases to the exact issued publication and coherent current balance state across issue, resume and share admission. Refs #273. * fix: reprove compact authority after publication lock waits Refresh external payout authority after local publication lock backpressure without holding state locks through I/O or renewing the original expiry clock. Preserve backend unavailability for an unready current publication while returning deliberate misses for retired requested work. * fix: retain ordinary authority when a leased tip returns A proof started during replacement may finish after the node returns to the published parent. Accept only with the original current revision and fresh polling, and keep the digest override restricted to active replacement leases. * fix: enforce lease authority at commit and issued delivery * fix: retain original issuance authority across handoffs * test: preserve merged inputs in cold authority fixture * fix: preserve issuance authority across unchanged refreshes Track actual prepared publications separately from tip observation ordering. Cached refreshes retain the original proof while new prepared installations still revoke it, even at the same parent and revision. Exercise real issue, persist, resume and compact publication across both transitions. * fix: report definite commit-gate refusals precisely Preserve the established no-credit response reason while identifying a refused local gate separately from database failure. Authority loss and lock contention both prove COMMIT was not sent; neither changes unknown-outcome reconciliation. * test: observe prepared work writes and reconstruction rows * feat: wire compact prepared runtime and bounded recovery Implementation checkpoint: the runtime integration subset passes on PostgreSQL 16, while existing unit fixtures still require adaptation to metadata views before full qualification. * fix: retain initial publication and expiry across recovery waits * test: observe original audits through compact references * test: require real compact refresh and resume at 400k and 500k * test: retain original authority controls through compact runtime fixtures * test: pause completed PostgreSQL commits before reply delivery * test: exercise public compact runtime resume and Stratum reconnect * test: observe reconstructed shares and delayed refresh supersession * test: compare resume expiry with the wire clock type * test: reconstruct original compact audits through public builders * test: preserve original expiry through real SQL waits * test: bound real runtime cleanup and every recovery admission wait * fix: retain only CTV presence in issued runtime metadata * test: qualify first large template insertion with a real accepted block * test: revalidate issuance after completed commit response waits * test: require completed commit publication races in the gate manifest * docs: record end-to-end validation at the integration head * test: compare only the CTV presence carried by runtime metadata * fix: supply original compact balances to candidate preparation * test: cover optimized compact scale shard ownership * test: await completed repair cleanup ownership * test: retire compact refresh JSONB ceiling exception * fix: retain resume capacity through metadata decoding cleanup * test: measure job persistence through public Stratum Add isolated one- and two-frontend measurement fixtures, durable revision fencing checks, and a controlled settlement-lock baseline. Keep the 2000-session case explicit and performance acceptance separate from harness success. Refs #275. * fix: retain snapshot admission through decoding and cleanup * test: isolate compact refresh WAL measurements on private primaries * test: make persistence measurements expose retries and exact fences * docs: publish frozen-head 2000-session persistence measurements * test: anchor candidate ACK hold after append entry and prune reports * fix: preserve published payout compatibility on compact resume * test: identify compact reconstruction timeout phases * refactor: separate compact runtime fixtures and shared builders * test: buffer observer socket reads during compact reconstruction * test: analyze bulk-loaded shares before runtime qualification * 3.x.x: Keep the pg_restore session out of the fixture pool (#422) * v3.0.0 test: keep the pg_restore session off the fixture pool support::recovery::restore ran the SQL that pg_restore emits on a connection borrowed from the fixture pool. That SQL opens with session SET statements that survive COMMIT: PostgreSQL 16 emits statement_timeout = 0, lock_timeout = 0 and idle_in_transaction_session_timeout = 0 besides the search_path reset, and sqlx 0.8.6 returns a pooled connection through a spawned task with no session reset, so a later pooled query inherited whichever of them landed on the same connection. Run the restore on a dedicated PgConnection instead: one transaction, commit, then close. The restore result is evaluated before the close result so a failed close cannot hide the restore error. The session is built from the fixture URL, which pins search_path to the target schema, and the helper checks current_schema() against the target before restoring into it. Drop #391's in-transaction search_path reset. It returned the pooled session to the fixture schema before reuse; the dedicated session is closed and never reused, so the mechanism that produced the intermittent "relation does not exist" failure is gone rather than patched over. The new test pins search_path along with the timeouts. The new gated test opens its target with the ledger's operator timeouts on the URL, because the fixture's plain pool defaults every setting pg_restore touches to 0 and would not observe the leak. It waits for sqlx to return every connection and then reads all of them, so the check does not depend on release scheduling. Before this change it fails on every run with "pooled lock_timeout is "0", expected "5s"". Closes #411. Co-Authored-By: Claude Fable 5.1 * v3.0.0 test: pin the restore-over-close error precedence The pg_restore session helper evaluated the restore result before the close result so a failed close could not hide why the restore failed, but nothing tested that order: swapping the two lines left every migration_rollback test green, because no test injects a failing close against a live connection. Move the precedence into a pure helper, restore_outcome, and unit-test all four combinations directly. When both fail, the restore error is returned with the close failure appended to its message, so the operator reads the restore error first and the close failure is not lost either. The session is still closed on both paths; only which error is reported is pinned. Swapping the precedence in the helper now fails restore_outcome_prefers_the_restore_error_when_both_fail. Follow-up to #411. Co-Authored-By: Claude Fable 5.1 * 3.x.x: Align public-reader authentication validation (#426) * 3.x.x: Align public-reader credential validation * 3.x.x: Qualify reader preflight against prepared deployment images * 3.x.x: Keep reader image preparation scoped to the qbit lane * 3.x.x: Build the disposable reader fixture with its local image store * docs: record release-profile public Stratum measurements (#431) * 3.x.x: pin the lost tip race and settle proven orphans terminally (#415) (#424) * 3.x.x: pin the lost tip race and settle proven orphans terminal (#415) Part of #413, the 2026-09-16 mainnet orphan stall: the 2.x.x coordinator held all job delivery for 307 s after one of its own blocks lost a tip race. 3.x.x cannot stall that way, because the block becomes a `reconciliation` row and issuance never reads candidate state. Two gaps remained, and this closes both. The pin (tests/lost_race_pin.rs, Postgres-gated). A real Coordinator against a scripted node reproduces the race exactly: submitblock returns null, our hash is the best tip, the node reorgs to a same-height competitor inside 100 ms, and the tip advances two further heights. It asserts the row lands in reconciliation with the not-active reason and its accepted offer, that submitblock was called exactly once, that the landed audit survives, that the block-only proof is neither credited nor failed while the block can still become active, and that new work for each new tip is issued within 2 s of the tip becoming visible while the orphan row is claimed by a live lease and held under FOR UPDATE (measured at ~60 ms; the incident's number was 307 s). It passes on the tree that introduces it; it fails only if issuance is ever coupled to candidate state. The disposition. Reconciliation rows were retried forever and every unfinished row counted toward qbit_prism_block_candidates_pending and qbit_prism_block_candidate_oldest_pending_seconds, so a handful of orphans kept the migrated pending-candidate alerts firing permanently. After the post-offer landing, one coherent tip observation that shows a DIFFERENT block active at the candidate's height with at least PRISM_CANDIDATE_ORPHAN_CONFIRMATIONS confirmations (new setting, default 6, 1..=1000; confirmations are tip_height - height + 1 from the observed tip) now settles the row terminal as `orphaned`: * Ledger::orphan_candidate_at_revision reuses the existing fences and adds no lock: settlement and order locks, the payout revision the observation was taken at (a reorg reconciler that confirmed the block meanwhile bumped it, so a stale "not active" verdict cannot overwrite a newer "active" one), the strictly-live claim token, and the block's own qbit_pool_blocks row, which must not be confirmed. State, reason, completed_at and the block's `inactive` chain state are one transaction. Never from pending. * The row keeps its document, block bytes, window reference and offer record (an operator surface for #268 can show what was offered and why); it is never claimed, offered or retried again. The pending gauges use CandidateState::UNFINISHED_SQL and stop counting it; the new counter qbit_prism_block_candidates_orphaned_total is incremented once the settlement committed, never on a failed observation, which still settles the row back into reconciliation (unknown stays distinct from zero). * Reactivation: the block keeps its landed audit and its pool block, so the ordinary reorg reconciler (Ledger::reconcile_blocks_in) confirms it and credits its deferred share from that preserved evidence if the chain ever reactivates it. The terminal row never reopens; tested end to end. * The block-only acknowledgement path (miner_submit.rs) answers a share on an orphaned block `ledger-confirmation-failed`, like an abandoned one. Migration 015 replaces 011's three lifecycle CHECKs under their own names, only after deparsing 011's definitions and finding them unchanged, and declares candidate_orphan_disposition = 1 so a pre-015 binary, which has no name for the state, refuses the database at connect (mixed-version rollout is rejected explicitly). Old rows are not rewritten. REQUIRED_SCHEMA_VERSIONS, NATIVE_MIGRATIONS, NATIVE_CAPABILITIES and the declared-capability gate learn 15, and so do the test supports (undo helpers, version sets, the capability restores, the 013 online-migration cases) and the recovery evidence scripts, which treat `orphaned` as terminal while still exporting its balance snapshot. The in-flight-registration fixture in offer_lifecycle.rs polls for the migrator's lock wait for 15 s instead of 3 s: the scratch apply of every native migration is ~12% longer with 015 and the old budget was no longer met with four tests queued on the migration lock; the assertion is unchanged. Tests: the four lost_race_pin contracts (the pin, the disposition with its reactivation, the ordering mirror of stale_candidate_active_proof_cannot_overwrite_a_newer_reorg, and the failed observation), the 015 lifecycle shapes and a terminal-orphan ledger contract in offer_lifecycle, and the migration_rollback offer-recovery fingerprints extended with the orphaned state (its evidence stays visible to a restore and it no longer counts as unfinished). All are listed in test/prism-gated-tests.txt. Docs: prism-native-metrics.md (regenerated inventory plus the gauge exclusion and counter contract), prism-alert-migration.md (pending-candidate rule guidance), prism-rejections.md, prism-ledger-ops.md (the state table) and the crate README setting row. Co-Authored-By: Claude Fable 5.1 * Forward and validate the candidate orphan confirmation threshold * Harden orphan settlement, migration safety, and reactivation accounting --------- Co-authored-by: Claude Fable 5.1 Co-authored-by: Alex * 3.x.x: Reuse payout windows during template refresh (#428) * Reuse anchored payout windows across template refreshes * Keep refresh cleanup order without redundant binding * Register refresh split PostgreSQL regression gates * Fence unpublished window cleanup and balance generations * Keep cached inputs bounded without blocking lease-holder resume * Clarify unmeasured memory qualification criterion * Preserve published work cadence while shares invalidate refresh inputs * Revalidate refresh inputs after build admission * Drain cancelled checkout before measuring refresh SQL * Cover native refresh liveness and yield CTV to newer tips (#433) * Cover native refresh liveness and yield CTV to newer tips * Bound the CTV pending-tip yield by the replacement-build budget A refresh that observes a newer tip and then keeps failing before publication (template fetch, fee validation, prepared-work persistence) left the observed tip permanently different from the published one, so every broadcaster pass exited at its first fanout boundary and settlement was stranded for the whole outage even though its own chain checks could succeed. The yield for an unpublished tip now lasts at most the replacement-build budget (PRISM_TEMPLATE_REFRESH_FAILURE_EXIT_SECONDS, default 120) from the first departure, the same bound the published-tip lease already uses; failed refreshes and newer tips do not renew it, and a zero budget disables the yield like the lease. An unpublished startup tip anchors the same budget so the first publication cannot strand settlement either. The pass-tip mismatch yield is unchanged: it ends with the pass. The new gated test breaks every template fetch after a tip change: a frontend inside its budget yields, and one whose budget elapsed settles every remaining row on its next pass. Without the bound the second half fails with "settlement stayed stranded". Co-Authored-By: Claude Fable 5.1 * Record CTV chunk metrics when completion persistence fails When a claimed fanout attempt succeeded but persisting its completion failed, the early return skipped the chunk-row and chunk-duration observations, so the new telemetry undercounted attempts exactly during persistence incidents while the failed-attempt branch still counted its own completion failures. The pass now observes the chunk after either completion path and then propagates a completion failure, so every claimed attempt is counted once. The gated test severs the completion write after it executes: the pass ends with the error, the attempt is counted, its claim stays fenced and a later pass finishes the other rows. Without the fix the test fails with "chunk rows missed the attempt". Co-Authored-By: Claude Fable 5.1 * Yield CTV only to tips observed after the pass began The pass-tip mismatch yield compared the last tip observation with the tip the pass had just read from the node, regardless of when that observation was recorded. When the node advanced before any refresh or block notification observed it, every pass read the newer tip while the observation still named the published one, so each pass yielded at its first fanout boundary. Unlike the unpublished-tip yield, this mismatch had no budget: settlement stayed stranded until a blocked or absent refresh caught up with the node. The pass now records when it began and yields only to an observation recorded after that instant that differs from the pass tip, including a replacement that already published. An observation older than the pass is stale evidence, not a newer tip, so the mismatch yield is bounded by the pass as its comment already claimed. The unpublished-tip yield keeps its replacement-build budget. The new gated test advances the node without any observer and expects the pass to settle every row; on the previous commit it fails with "0 of 5 rows". A unit test pins the observation-order semantics. Co-Authored-By: Claude Fable 5.1 * Document the CTV chunk-row histogram ladder The native chunk-row histogram keeps the 2.x.x name and type but counts rows with a single finite bound of 1, while the metric inventory and the histogram consumer guide still said every event histogram measures elapsed seconds on the default time ladder and that only the share ACK histogram differs. A consumer following those documents would query `le` series that the native exposition never emits, or read the row count as seconds. Both documents also still counted the pre-branch coordinator family total after this branch added three. The compatibility annotations now describe the row unit, the `le="1"` and `+Inf` bounds, the `_sum` equals `_count` identity and the retired 2.x.x ladder, so the generated inventory row carries them; the consumer guide gains a section on reading the family and the family count is 46. A comment beside the bucket arm points at the guide. No exposition changes; the generator drift check and the four inventory tests pass. Co-Authored-By: Claude Fable 5.1 --------- Co-authored-by: Claude Fable 5.1 * 3.x.x: Follow revision-fenced equal-work tip replacements (#429) * 3.x.x: follow revision-fenced equal-work tip replacements * 3.x.x: preserve concurrent same-tip observation idempotence * 3.x.x: consume equal-work node transition witnesses before observation * 3.x.x: retain chain predecessor after lower-work refusal * 3.x.x: bind equal-work observations to durable chain epochs * 3.x.x: preserve migration fixtures with chain epoch metadata Pin the current schema and capability inventories including migration 018. Remove its column, capability and version together when reconstructing historical pre-006 and pre-011 schemas; preserve the frozen 2.x release SQL, digests, and exact historical-state assertions. * 3.x.x: bound immediate chain observation retries per trigger * v3.0.0 test: cover native candidate storms, recovery, and offer provenance (#438) * Scaffold the native candidate-storm suite and its one cardinality reader (#270) The storm scenarios split across four targets by the seam each needs, because `candidates::faults` and `Coordinator::offer_probe` are `#[cfg(test)]` and so are invisible to an integration test, while `CARGO_BIN_EXE_*` exists only in an integration target. Two new `#[cfg(test)]` module declarations are the only production lines this issue adds; the four test files are empty frames their writers fill. `PRISM_TEST_STORM_CANDIDATES` sizes every scenario from one reader, so the reduced cardinality the native PostgreSQL job runs and the incident's 3,120 cannot drift apart. It is not a gate variable: the gate decides whether a test runs, this decides how large it is, so every storm test executes in both paths and none is `#[ignore]`d. Values below the CI default are refused rather than quietly measuring less than CI already proves. Refs #270 * Re-pin the storm scaffolding and require one database per fixture (#270) The base moved seven commits. Two of them change this suite's design rather than its line numbers. #410 gave each integration fixture its own PostgreSQL database, because advisory locks are database-scoped and fixtures sharing a database queued on each other's migration, order and settlement keys. For this suite that is load-bearing: a drain of thousands of siblings takes one claim each, every claim touches the ordering and settlement keys, and a per-row cost measured in a shared database measures other fixtures' lock waits instead of the drain. It also sharpens the counting invariant over a wall-clock one, since with the database isolated the claim and statement counts are clean while wall clock still carries the runner's noise. #412 added a registration mode that runs every startup gate but writes no `starting` heartbeat, so the restart frame now says to be deliberate about which constructor the relaunched child uses and what the instance table should hold afterwards. The offer-evidence module records why it is empty: #415 may add a terminal disposition for a proven orphan, which would change what a reconciliation row can become, and assertions written before that decision would be written against a rule that is about to change. Refs #270 * 3.x.x: pin the lost tip race and settle proven orphans terminal (#415) Part of #413, the 2026-09-16 mainnet orphan stall: the 2.x.x coordinator held all job delivery for 307 s after one of its own blocks lost a tip race. 3.x.x cannot stall that way, because the block becomes a `reconciliation` row and issuance never reads candidate state. Two gaps remained, and this closes both. The pin (tests/lost_race_pin.rs, Postgres-gated). A real Coordinator against a scripted node reproduces the race exactly: submitblock returns null, our hash is the best tip, the node reorgs to a same-height competitor inside 100 ms, and the tip advances two further heights. It asserts the row lands in reconciliation with the not-active reason and its accepted offer, that submitblock was called exactly once, that the landed audit survives, that the block-only proof is neither credited nor failed while the block can still become active, and that new work for each new tip is issued within 2 s of the tip becoming visible while the orphan row is claimed by a live lease and held under FOR UPDATE (measured at ~60 ms; the incident's number was 307 s). It passes on the tree that introduces it; it fails only if issuance is ever coupled to candidate state. The disposition. Reconciliation rows were retried forever and every unfinished row counted toward qbit_prism_block_candidates_pending and qbit_prism_block_candidate_oldest_pending_seconds, so a handful of orphans kept the migrated pending-candidate alerts firing permanently. After the post-offer landing, one coherent tip observation that shows a DIFFERENT block active at the candidate's height with at least PRISM_CANDIDATE_ORPHAN_CONFIRMATIONS confirmations (new setting, default 6, 1..=1000; confirmations are tip_height - height + 1 from the observed tip) now settles the row terminal as `orphaned`: * Ledger::orphan_candidate_at_revision reuses the existing fences and adds no lock: settlement and order locks, the payout revision the observation was taken at (a reorg reconciler that confirmed the block meanwhile bumped it, so a stale "not active" verdict cannot overwrite a newer "active" one), the strictly-live claim token, and the block's own qbit_pool_blocks row, which must not be confirmed. State, reason, completed_at and the block's `inactive` chain state are one transaction. Never from pending. * The row keeps its document, block bytes, window reference and offer record (an operator surface for #268 can show what was offered and why); it is never claimed, offered or retried again. The pending gauges use CandidateState::UNFINISHED_SQL and stop counting it; the new counter qbit_prism_block_candidates_orphaned_total is incremented once the settlement committed, never on a failed observation, which still settles the row back into reconciliation (unknown stays distinct from zero). * Reactivation: the block keeps its landed audit and its pool block, so the ordinary reorg reconciler (Ledger::reconcile_blocks_in) confirms it and credits its deferred share from that preserved evidence if the chain ever reactivates it. The terminal row never reopens; tested end to end. * The block-only acknowledgement path (miner_submit.rs) answers a share on an orphaned block `ledger-confirmation-failed`, like an abandoned one. Migration 015 replaces 011's three lifecycle CHECKs under their own names, only after deparsing 011's definitions and finding them unchanged, and declares candidate_orphan_disposition = 1 so a pre-015 binary, which has no name for the state, refuses the database at connect (mixed-version rollout is rejected explicitly). Old rows are not rewritten. REQUIRED_SCHEMA_VERSIONS, NATIVE_MIGRATIONS, NATIVE_CAPABILITIES and the declared-capability gate learn 15, and so do the test supports (undo helpers, version sets, the capability restores, the 013 online-migration cases) and the recovery evidence scripts, which treat `orphaned` as terminal while still exporting its balance snapshot. The in-flight-registration fixture in offer_lifecycle.rs polls for the migrator's lock wait for 15 s instead of 3 s: the scratch apply of every native migration is ~12% longer with 015 and the old budget was no longer met with four tests queued on the migration lock; the assertion is unchanged. Tests: the four lost_race_pin contracts (the pin, the disposition with its reactivation, the ordering mirror of stale_candidate_active_proof_cannot_overwrite_a_newer_reorg, and the failed observation), the 015 lifecycle shapes and a terminal-orphan ledger contract in offer_lifecycle, and the migration_rollback offer-recovery fingerprints extended with the orphaned state (its evidence stays visible to a restore and it no longer counts as unfinished). All are listed in test/prism-gated-tests.txt. Docs: prism-native-metrics.md (regenerated inventory plus the gauge exclusion and counter contract), prism-alert-migration.md (pending-candidate rule guidance), prism-rejections.md, prism-ledger-ops.md (the state table) and the crate README setting row. Co-Authored-By: Claude Fable 5.1 * 3.x.x: Prove drain cost and admission at storm cardinality (#270) The 2026-08-20 testnet4 incident left 3,120 durable block candidates behind one decided height. 2.x.x measured that sibling set with a Python instrument #244 deleted; this fills the native target with the properties, not the harness. Four gated scenarios, every asserted quantity an integer from a counter the test owns: - the clean baseline drain: N siblings at one decided height, one of them the block the node reports active, through the real claim path. Chain calls are exactly six per drained row (`CHAIN_PROBES_PER_SIBLING`, derived from the native pre-offer path and pinned with what each call is), `submitblock` is exactly once for each hash offered and zero for every hash that was not, the dispatch-sequence delta and `SUM(attempt_count)` both equal the claims made, and the abandoned set *is* the sibling set while the decided height's own block stays pending without offer evidence. Brand-new work is admitted, offered and landed three times while the backlog is still owed; - statements per drained row on the wire, through `support/ledger_execution_proxy.rs`, asserted equal at the baseline cardinality and at the run's own in one process against one schema, then against a pinned 26 whose comment names all four transactions. That equality is what makes the reduced-N CI path prove what a local run at 3,120 proves; - the two claim lanes and the dispatch probe keep their partial indexes over N *unfinished due* rows beside 50,000 retained ones, with no `Seq Scan` and no `Sort`, EXPLAINing the statements `Ledger` returns rather than a copy; and one claim under `ANALYZE, BUFFERS` reads one row and discards none, so per-claim work does not walk the siblings; - admission against a backlog of N: the fresh lane returns the block just found, by hash, for thirty-two rounds, and the oldest-due lane still serves both an old never-attempted row and a due retry, so neither lane starves the other. The fixture takes its own PostgreSQL database through `support/ledger_database.rs` (#410). That is load-bearing rather than hygienic: a drain of N siblings is N claims, each touching the ordering and settlement keys, so a per-row cost measured in a shared database would be measuring other fixtures' lock waits. Nothing asserts a duration. Wall clock, microseconds per row, the plan's shared buffer counts and both cardinalities are recorded under `storm_scale::record` and never asserted. Cardinality comes from the one shared reader; its first consumer also proves its floor, ceiling and malformed refusals in a plain test that needs no database and is therefore not gated. No production code, schema or metric family changes. Co-Authored-By: Claude Opus 5 * Record the dispatch probe's plan instead of asserting it at storm size (#270) The incident-cardinality run found what the reduced-N run cannot: at 3,120 unfinished rows PostgreSQL stops using 011's partial index for the dispatch probe and takes a sequential scan, and it does so even when nothing is due. The probe compares `next_attempt_at` against `clock_timestamp()`, which is volatile, so no histogram applies and the planner falls back to roughly a third of the indexed rows: about 1,035 estimated matches at 3,120 unfinished rows. It concludes an `EXISTS` will hit one almost at once and charges the scan's startup cost as near zero. At 24 or 100 rows the same estimate is small enough that the index wins, which is why the original assertion passed in the required job and failed only here. When rows really are due the gamble pays. The damaging case is a pool left idle behind a storm, thousands of unfinished rows all backing off and none due, where that plan reads the whole outbox including retained terminal history on every poll to prove nothing is due -- a per-poll cost that grows with retained history, in exactly the state this suite exists to study, and one 011's index cannot prevent because the planner declines to use it. That is a finding about the server, not about the test, so the probe's plan is recorded at the run's cardinality rather than asserted away. The guarantee is asserted where it holds: trimming the unfinished set to the baseline restores the index scan over the same table, the same index and freshly analysed statistics, so the recorded pair is direct evidence that the flip follows unfinished cardinality and not a missing index. Both claim lanes keep their index scans at 3,120 and are still asserted at every cardinality. Per-row drain cost is unaffected and remains independent of N: six chain calls and twenty-six wire statements per drained row, equal at the baseline and at 3,120. Refs #270 * Add native candidate restart coverage and map remaining storm gaps (#270) * Run local database storm scenarios at incident cardinality (#270) * Forward and validate the candidate orphan confirmation threshold * Harden orphan settlement, migration safety, and reactivation accounting * 3.x.x: pin the lost tip race and settle proven orphans terminal (#415) Part of #413, the 2026-09-16 mainnet orphan stall: the 2.x.x coordinator held all job delivery for 307 s after one of its own blocks lost a tip race. 3.x.x cannot stall that way, because the block becomes a `reconciliation` row and issuance never reads candidate state. Two gaps remained, and this closes both. The pin (tests/lost_race_pin.rs, Postgres-gated). A real Coordinator against a scripted node reproduces the race exactly: submitblock returns null, our hash is the best tip, the node reorgs to a same-height competitor inside 100 ms, and the tip advances two further heights. It asserts the row lands in reconciliation with the not-active reason and its accepted offer, that submitblock was called exactly once, that the landed audit survives, that the block-only proof is neither credited nor failed while the block can still become active, and that new work for each new tip is issued within 2 s of the tip becoming visible while the orphan row is claimed by a live lease and held under FOR UPDATE (measured at ~60 ms; the incident's number was 307 s). It passes on the tree that introduces it; it fails only if issuance is ever coupled to candidate state. The disposition. Reconciliation rows were retried forever and every unfinished row counted toward qbit_prism_block_candidates_pending and qbit_prism_block_candidate_oldest_pending_seconds, so a handful of orphans kept the migrated pending-candidate alerts firing permanently. After the post-offer landing, one coherent tip observation that shows a DIFFERENT block active at the candidate's height with at least PRISM_CANDIDATE_ORPHAN_CONFIRMATIONS confirmations (new setting, default 6, 1..=1000; confirmations are tip_height - height + 1 from the observed tip) now settles the row terminal as `orphaned`: * Ledger::orphan_candidate_at_revision reuses the existing fences and adds no lock: settlement and order locks, the payout revision the observation was taken at (a reorg reconciler that confirmed the block meanwhile bumped it, so a stale "not active" verdict cannot overwrite a newer "active" one), the strictly-live claim token, and the block's own qbit_pool_blocks row, which must not be confirmed. State, reason, completed_at and the block's `inactive` chain state are one transaction. Never from pending. * The row keeps its document, block bytes, window reference and offer record (an operator surface for #268 can show what was offered and why); it is never claimed, offered or retried again. The pending gauges use CandidateState::UNFINISHED_SQL and stop counting it; the new counter qbit_prism_block_candidates_orphaned_total is incremented once the settlement committed, never on a failed observation, which still settles the row back into reconciliation (unknown stays distinct from zero). * Reactivation: the block keeps its landed audit and its pool block, so the ordinary reorg reconciler (Ledger::reconcile_blocks_in) confirms it and credits its deferred share from that preserved evidence if the chain ever reactivates it. The terminal row never reopens; tested end to end. * The block-only acknowledgement path (miner_submit.rs) answers a share on an orphaned block `ledger-confirmation-failed`, like an abandoned one. Migration 015 replaces 011's three lifecycle CHECKs under their own names, only after deparsing 011's definitions and finding them unchanged, and declares candidate_orphan_disposition = 1 so a pre-015 binary, which has no name for the state, refuses the database at connect (mixed-version rollout is rejected explicitly). Old rows are not rewritten. REQUIRED_SCHEMA_VERSIONS, NATIVE_MIGRATIONS, NATIVE_CAPABILITIES and the declared-capability gate learn 15, and so do the test supports (undo helpers, version sets, the capability restores, the 013 online-migration cases) and the recovery evidence scripts, which treat `orphaned` as terminal while still exporting its balance snapshot. The in-flight-registration fixture in offer_lifecycle.rs polls for the migrator's lock wait for 15 s instead of 3 s: the scratch apply of every native migration is ~12% longer with 015 and the old budget was no longer met with four tests queued on the migration lock; the assertion is unchanged. Tests: the four lost_race_pin contracts (the pin, the disposition with its reactivation, the ordering mirror of stale_candidate_active_proof_cannot_overwrite_a_newer_reorg, and the failed observation), the 015 lifecycle shapes and a terminal-orphan ledger contract in offer_lifecycle, and the migration_rollback offer-recovery fingerprints extended with the orphaned state (its evidence stays visible to a restore and it no longer counts as unfinished). All are listed in test/prism-gated-tests.txt. Docs: prism-native-metrics.md (regenerated inventory plus the gauge exclusion and counter contract), prism-alert-migration.md (pending-candidate rule guidance), prism-rejections.md, prism-ledger-ops.md (the state table) and the crate README setting row. Co-Authored-By: Claude Fable 5.1 * Forward and validate the candidate orphan confirmation threshold * Harden orphan settlement, migration safety, and reactivation accounting * Test native candidate cleanup faults and offer provenance * Map native storm recovery and provenance coverage * Enforce reader password authentication in native test fixtures --------- Co-authored-by: Dan Hepworth Co-authored-by: Claude Fable 5.1 * 3.x.x: Coalesce refresh economics and accepted-cutoff probes (#437) * Add coherent refresh economics probe and PostgreSQL regressions * Pin refresh behavior for same-revision drift during fee validation * Measure baseline idle refresh probe executions on PostgreSQL * Cover economics drift while refresh waits for build admission * Coalesce refresh cutoff and payout observations after build admission * Preserve refresh guard failure ordering before share access * Distinguish balance metadata from share payload in refresh evidence * 3.x.x: Measure session reservation cleanup pool checkout waits (#436) * fix: time session reservation cleanup pool checkouts Carry the ledger metrics handle into SessionId and record only the explicit and spawned cleanup checkout boundaries. Preserve token fencing, release scope and cancellation fallback. Add five disposable PostgreSQL tests and update the partial acquisition coverage inventory. Refs #352. * docs: clarify session cleanup timing and runtime boundaries * 3.x.x: Batch compact issued-job persistence (#435) * feat: add atomic compact issued-job batches * feat: coalesce compact issued persistence per coordinator * fix: roll back canceled batch begins before reuse * test: separate batch cleanup fixture from runtime SQL * Preserve live issued-batch peers when callers cancel * Assert retained metadata after canceled batch commits * Document extranonce retention for canceled issued jobs * 3.x.x: Qualify ledger proxy client-close ordering (#441) * v3.0.0 test: qualify ledger proxy client-close ordering * v3.0.0 test: reject invalid proxy close evidence * v3.0.0 test: publish proxy acknowledgements atomically * 3.x.x: Native candidate recover --apply for already-accepted pending blocks (#439) * 3.x.x: Add the candidates list and abandon operator commands `qbit-prism-server` had no way to see or finish a block candidate: a native-era row that keeps failing was retried forever, and the only way to inspect or stop it was editing rows by hand. `candidates list` is an inventory. It opens the read-only pool shape `fatal-state show` uses, so "takes no claim" is enforced by PostgreSQL rather than by review, and it keeps working while the cluster is halted. Neither payload column is projected: the height is extracted to a scalar server-side, guarded by a digit pattern so a row an unknown writer left behind reports an unknown height instead of failing the inventory. Rows print in the oldest-due claim lane's own ordering, so the row a server works next comes first and parked rows sort last. An empty inventory is a success, because that is what a cutover runbook waits for. `candidates abandon` applies to `pending` only, which is the one unfinished state from which no `submitblock` can yet have been made. The whole safety property is the statement's `WHERE`: `pending`, no live claim, and no landed block. It is a predicate and never a preceding `SELECT`, because a row can move between a check and a write; the diagnosis that produces the operator's message runs read-only after it, and gates nothing. The column set is the epoch-supersession path's, verbatim, with the operator's reason taking `last_error`'s place, and `next_attempt_at` deliberately untouched so a parked row keeps that evidence. Each refusal has its own exit status, so a runbook can tell a row that was never there from one already abandoned, and both from the two refusals that protect the invariant: a block that may already have been offered, and a block whose accounting has landed. Both commands read PRISM_DATABASE_URL and nothing else. Neither builds a node client, reads a signing key, loads `Config` or starts a listener, so "never calls submitblock" holds structurally. Co-Authored-By: Claude Opus 5 * 3.x.x: Cover the candidate operator commands in the PostgreSQL gate Six gated tests drive the real binary against a recorder in front of the fake node and assert it made no RPC call at all, `submitblock` included, and registered no instance. The child is given a database URL and a node URL and nothing else, so a command that loaded `Config` or started a listener could not have succeeded. `list` is asserted not to claim three ways: it runs to completion while the cluster is halted, which every write path refuses; every row is byte-identical afterwards, with `claim_token`, `claim_instance_id`, `claim_expires_at`, `attempt_count` and the dispatch sequence compared explicitly; and the statement it issues is described by PostgreSQL, so the assertion that neither payload column is projected reads the result columns rather than the SQL text. `abandon` is asserted to leave an `offer_reserved`, `offered` or `reconciliation` row byte-identical, to refuse a live foreign claim and then succeed once that claim has expired, to refuse an already-terminal row and a pending row whose block has landed, and to refuse while the cluster is halted or a legacy writer lease is live. On the one row it does take, the resulting row is compared field by field with a row the epoch-supersession statement abandoned. Co-Authored-By: Claude Opus 5 * 3.x.x: Document the candidate commands and the two cutover drains `prism-ledger-ops.md` gains a `Candidate commands` subsection three lines below the state table it explains: the two command lines, a real `list` output block, what each column means, the parked-versus-retrying distinction, the refusal table with its exit codes, and the two fences `abandon` inherits from the ordinary ledger write path — a halted cluster and a live legacy Python writer lease, which is what stops an operator abandon racing the 2.x.x writer during a cutover. `prism-rust-migration.md` gains the native counterpart to the legacy drain, and says plainly that they are not the same drain. The legacy instruction is unchanged: every 2.x.x candidate is still drained with the pinned legacy image before migrating, and the migrator still refuses while any remains. The native commands act on native-era rows only, and running them neither satisfies nor bypasses that check. `prism-deleted-test-map.md` keeps the pending-block-recovery row partial and its open-gap link intact, adds the six new tests where they genuinely cover a case, and fills in the per-case mapping the row promised. The recovery command's plan ordering, fail-closed selection across a plan and apply idempotence stay open; the cooperative-JSON codec cases stay retired. Co-Authored-By: Claude Opus 5 * 3.x.x: Assert a second candidate abandon cannot overwrite the first The state predicate is the abandon statement's version guard, so two operator abandons completing out of order are safe without any lock on top. Prove it end to end: after the successful abandon, a second one with a different reason exits 4, says the row is already abandoned, and leaves it byte-identical, so the reason the first operator recorded survives. Co-Authored-By: Claude Opus 5 * 3.x.x: Connect candidates abandon as a one-shot tool `abandon` writes an ordinary ledger row, so it belongs in the same bucket as `import-audits`, `backfill-ctv`, `broadcast-ctv` and `self-check`, not with recovery and migration: `Registration::Tool` refuses a halted cluster at connect, exactly as a frontend would, instead of getting as far as the write guard. The write guard call stays, because it is what refuses a live legacy Python writer lease, which the connection does not cover. `list` keeps the read-only pool. It opens no ledger and writes nothing, `default_transaction_read_only=on` is a stronger guarantee than a tool connection would give, and a tool connection would refuse a halted cluster — which is exactly when an operator needs to inspect candidates. A new gated test gives `abandon` the four properties a one-shot tool owes: a generated instance ID leaves no row on success or on a refusal, a live frontend's row and session-owner token are untouched when the command runs under that frontend's ID, the halt guard holds and the halted run changes nothing, and once the real frontend stops no row is left that the recovery command's instance gate would refuse. Substituting the registering connection for the tool connection fails all five abandon tests. Co-Authored-By: Claude Opus 5 * 3.x.x: Point the deleted-test map's open recover work at #418 home of the unfinished recover work would become dead ends. Row 84's "not covered" clause and every plan-level case in the per-case table now link #418 instead. Row 84 stays partial, and the #285 and #270 links are untouched. The per-case table also says which revision it enumerates. The file has seventeen cases at 504846c, the `origin/2.x.x` commit this document pins throughout; three of them arrived with #258, so a reader who checks v2.0.1 instead sees fourteen and none of those three. Saying so stops the next reader concluding the extra rows are invented. Co-Authored-By: Claude Opus 5 * 3.x.x: Name #418 in the deleted-test map's issue-link list The per-case retarget moved every recover link from #268 to #418, but the header still named #268 as one of the issues those links point at. No link points there now, so the sentence described the document's previous state. * 3.x.x: Refuse abandoning unsupported candidate storage versions * 3.x.x: Serialize candidate abandon with in-flight landing * Preserve legacy candidate evidence during operator abandon * Report truncated candidate inventories in text and JSON output * Reserve live regtest ports until each child starts * Use one claim timestamp for abandon refusal diagnosis * 3.x.x: Add the candidates recover operator command Land already-accepted blocks for an explicit allowlist of at most 32 candidates (#418), plan-only by default and never calling submitblock. The plan reads the listed rows on the inventory's read-only pool, loads no payload column and proves each block active on the node; --apply takes a token-fenced claim by hash through the existing claim mechanism, adopts a pending row before anything lands, and drives the coordinator's own post-offer landing and finish under one operator deadline, releasing the claim with the reason on any stop. The refusal taxonomy extends the abandon exit codes (9 not active, 10 selection refused, 11 deadline, 12 landing refused). Co-Authored-By: Claude Fable 5.1 * 3.x.x: Cover candidates recover in the PostgreSQL gate Five gated tests drive the real binary against a scripted fake node that records every RPC method and has no submitblock arm: plan ordering without a claim or a payload, fail-closed selection for every refusal, a real apply that lands a pending and a reconciliation candidate and verifies them idempotently on rerun, a live foreign claim refused and an unauthenticated row released, and a landing paused inside the database past the deadline. The seeding helpers of the list and abandon tests are shared with the new module. Co-Authored-By: Claude Fable 5.1 * 3.x.x: Document candidates recover and update the deleted-test map The ledger-ops guide gains the third candidate command beside list and abandon, with the configuration asymmetry, the plan, the apply steps, the deadline, the exit table and a native runbook; the migration guide names it in the two-drains section; and the deleted-test map's pending-block-recovery row drops its #418 clause, staying partial only for #285. Co-Authored-By: Claude Fable 5.1 * fix: adapt recovery to typed chain observations * fix: distinguish recovery RPC failures from missing blocks * fix: release recovery claims after claim-phase timeouts * fix: classify recovery blocks above the active tip as inactive * fix: treat orphaned candidates as terminal during recovery * fix: report orphaned candidates as terminal on abandon * fix: preserve allowlist order for recovery parent refusals * fix: report unconfirmed recovery claim cleanup * fix: exit recovery after cleanup failure without waiting for rebuilds * fix: revalidate completed candidates before skipping recovery --------- Co-authored-by: Alex Co-authored-by: Claude Opus 5 * v3.0.0 fix: time ledger audit reconstruction checkouts (#442) * fix: time ledger audit reconstruction pool checkouts * test: qualify audit reconstruction checkout failures * 3.x.x: Bound candidate dispatch probes to unfinished rows (#443) * 3.x.x: Bound candidate dispatch probes to unfinished rows Use the existing ordered unfinished index before allocating a dispatch slot, preserving advancing clock predicates and downstream claim ownership. Add canonical PostgreSQL plan, boundary, churn, and failure-path coverage with measured limitations. Refs #432 * test: Keep dispatch probe evidence and failure claims precise Scope the empty-outbox fault control to error propagation and pool recovery, and document unknown allocation after cancellation. Bind qualification evidence to its tree and query, assert one EXPLAIN scan loop, and parse row counters as numbers. Refs #432 * docs: Align storm evidence with the ordered dispatch probe * test: let acknowledgement barrier hand off its Tokio worker (#445) * 3.x.x: Record post-batch large-window delivery misses (#444) * test: qualify large-window issued-job delivery * test: preserve incomplete delivery attribution and run scope * docs: record post-batch 400k and 500k delivery observations * docs: bind delivery evidence to artifact manifest and cleanup * ci: retain share-append throughput diagnostics on failure * docs: clarify frozen qualification scope and timing provenance * test: consume piped Compose input in production-gate fixture * docs: correct retained-window inference for parent transitions * test: select recovery cleanup timeout without database timer race * 3.x.x: Partition qbit_share_ledger and add the detach-and-archive retention path (migrations 015 and 016) (#419) * Partition the share ledger: migrations 015/016, online conversion runner, design record Migration 015 drops the two inbound foreign keys onto share_id, adds the partition catalog and maintenance functions, and moves solver attribution onto qbit_pool_blocks. Migration 016 converts qbit_share_ledger into a RANGE (share_seq) partitioned table by attaching the release table as its first partition after a validated bound, applied inside the transaction on empty ledgers and by a resumable online runner otherwise. The append path consults qbit_prism_share_hashes first and bounds its ledger probe. Part of #144. * Test the partition conversion and teach the upgrade fixtures migrations 015 and 016 Six gated tests cover the empty-ledger transactional path, the online runner resuming from every stage, refusal of a reserved partition name before any DDL, the lock-timeout retry behind an open writer, re-bounding when headroom ran out, and per-leaf share_id uniqueness with the append path keeping it global. undo_016 and undo_015 let the older upgrade fixtures simulate builds without those migrations; the index-trim tests assert the trimmed set on the parent and on the first partition, where the release table's indexes keep their OIDs under their _p0 names. Container measurements are recorded in the design record. Part of #144. * Document share ledger partitioning and retention (#144) Rewrite the retention contract wherever the docs said no archive or prune path exists. Shares stay immutable and are never deleted; retention is detach-and-archive of a whole partition after its audits are sealed, and the immutability trigger sits on the parent and on every leaf. docs/prism-ledger-ops.md: rewrite "Audit storage and retention"; update "Share ledger indexes" for the per-partition set, the per-leaf _share_id_key, qbit_prism_share_hashes as the global share_id authority, and the probe floor rule with the append path's one unbounded fallback; add "Share ledger partitions and retention" with the grid and catalog columns, lead partitions and the ensure interval, the five online horizon conditions, sealing per D6 with the two documented reader changes, the share-archive command table and an example session, the archive root layout and hand verification, the restore procedure, and the before/after measurement runbook for acceptance criterion 4 with a container evidence skeleton for the coordinator. docs/prism-rust-migration.md: startup gate at migrations 2 through 16 with 016 recorded after the swap; native-collision row through 016 plus the reserved qbit_share_ledger_p name family; new subsections for 015 and for 016's three stages, locks, headroom and re-bounding, resumability, every refusal with its remedy, what to plan for on a large ledger, and the one-way statement from D5. docs/prism-storage-sizing.md, docs/README.md, PRISM.md, docs/prism-public-operator-endpoint-split.md and the 3.x.x development-line notes follow the same contract. Part of #144. * Re-check the partition bound's headroom from the validated stage too A run interrupted after validating the bound could find the sequence past it on resume and the swap refusing forever; prepare now runs from that stage as well, moving the bound out and validating again. The design record describes the append path as implemented and the verify rule for a detached partition; the settings checker's retired-setting marker is added. Part of #144. * Record the container evidence and the validated-stage re-bound in the docs Part of #144. * Make every remaining share ledger reader partition-aware (3.x.x, #144) The ledger is RANGE partitioned on share_seq with per-leaf share_id uniqueness and no global share_id index, and a partition can leave. Four kinds of reader had to change. Bounded probes. The two block-only reconciliation probes in miner_submit.rs carry share_seq >= qbit_prism_share_probe_floor(): both look for a share the connection they are acknowledging submitted seconds ago, so PostgreSQL prunes to at most three leaves instead of descending one per-leaf index per attached partition. share_accepted_at_ms probes with the bound first and, only on a miss, once without it: vardiff evidence is normally seconds old, but a miss is not an answer, so an older row that is still online is found exactly. Solver attribution at landing. land_candidate writes solver_miner_id, solver_share_id, solver_share_difficulty and solver_network_difficulty from the accepted share whose share_id suffix is the block hash, the expression migration 015 backfilled with, in a subselect so a block with no matching share still lands with the columns NULL. Nothing else inserts or rewrites qbit_pool_blocks: the release functions only update chain and maturity state. Dashboard queries. The blocks page, both leaderboards and the pool snapshot read those columns instead of suffix-matching the ledger per request, and keep the ledger lookup as the fallback for a row whose columns are unset. The guard sits inside the LATERAL rather than on the join because a LEFT JOIN's ON clause is evaluated per inner row and would still run the lookup for every block; inside, it is a One-Time Filter and the index scan reads "never executed" (2 buffers against 202 over 50 blocks). latest_evidence lifetime counts. accepted_share_count and distinct_miner_count come from the permanent daily rollups plus the raw rows above the rollup watermark, in one statement. The watermark is one of the conditions a partition must clear before it may be detached, so the sum is exact and stays exact afterwards; a database whose rollups have never run keeps the raw lifetime counts. The two api_database fixtures hand-build a 001/002/003 schema and then serve the public read models from it. They now record what the migrator records and bring the schema forward on the native path before the router is built, because the read models require migration 015's columns. The chain-state fixture inserts its blocks afterwards and around the landing path, so its rows exercise the ledger fallback. * Keep the share ledger partition lead attached, and retry an append past it (3.x.x, #144) qbit_share_ledger has no DEFAULT partition, so an append whose share_seq has run past the last attached bound is refused with SQLSTATE 23514. Two things keep that from reaching a miner. The maintenance task. src/partitions.rs runs qbit_prism_share_partition_ensure() under a 30 s statement timeout, once at startup and then every PRISM_SHARE_PARTITION_ENSURE_INTERVAL_SECONDS (default 60, 1 to 86400). The startup call is a precondition of serving: an instance that cannot maintain its partitions refuses to start rather than accept shares until its lead runs out. The task logs an intact lead at debug, creations at info, and a failure at warning without ending, because the next tick retries and the lead is hours of appends wide. check-config validates the setting with the rest. The append retry. append_checked runs one complete attempt through a new append_prepared, and on a 23514 whose message names the missing partition, and only then, attaches the lead on a fresh pool connection and runs the whole attempt again exactly once, prepared candidate persistence and pre-commit gate included. That failure is definite and total: the ledger INSERT raised it inside the attempt's own transaction, which is rolled back with the share, the clock update, the hash row and the candidate bytes undone together. Any other error, and a second failure of any kind, is returned unchanged. pre_commit is still called at most once; a refused attempt never reaches it. qbit_prism_share_ledger_partition_lead_rows reports the remaining headroom, read by the database collector in the same snapshot as the pending-candidate counts, so a failed read leaves all three unknown (-1) rather than reporting exhausted headroom. Five gated tests in share_partitions_postgres cover the landing columns and all four readers across a detach and across nulled columns, the evidence counts before, during and after the rollup sweep, the bounded probes and the unbounded vardiff retry below the floor, the append retry, and ensure's idempotence; unit tests cover the setting. * Require the solver columns in the public reader's schema readiness check The dashboard queries read qbit_pool_blocks.solver_* since 015, so a public tier answering before that migration must report the schema as not ready rather than fail each request. Part of #144. * Add the share ledger retention tool and serve sealed audits from stored bytes Retention half of #144, decision D6 of docs/prism-share-ledger-partitioning.md. ledger/archive.rs is the operator path a partition leaves the online ledger by, and the archive format v1 it leaves through: - `seal` stores the canonical bytes of every audit whose share snapshot intersects the partition, rebuilt from the still-online shares and proved against the digest the block committed to, so the block keeps serving its artifact after its shares are gone. The write is guarded on the digest and on the column still being NULL, so a repeated or concurrent seal stores nothing twice; `sealed_at` is recorded only once none is left. - `plan` evaluates the five conditions of D6 per partition (payout window floor at 8*D*M, retention age, rollup watermark, sealed audits, unfinished outbox and deferred references) and prints them as JSON with each blocker named. A missing input is `unknown`, never `clear`: no rollup watermark row, a catalog row whose relation is gone, an attached partition with no catalog row, an interrupted detach. - `archive` streams the rows in `share_seq` order, 4096 to a page, into `/qbit_share_ledger//rows.ndjson.gz` with a canonical `manifest.json` carrying the bounds, counts, both digests and the hash chain to the previously archived partition. Both files are written to temp names and renamed, and a failed run leaves none. - `verify` re-reads the archive, recomputes both digests, checks the manifest against the catalog row and the chain, and, while the partition is attached, streams the live rows again and compares. A mismatch names what differed. - `detach` requires every plan condition plus sealed, archived and verified, with no override, runs `DETACH PARTITION ... CONCURRENTLY` outside a transaction, and finalizes an interrupted attempt instead of restarting it. `pg_inherits` is the truth: a detach whose catalog update was lost is reconciled by the next run. - `drop` removes a detached, verified partition; `restore` rebuilds one from its archive in a single transaction, proves it by re-streaming the digest, and can attach it back under its recorded bounds without ever moving the share sequence. Readers now prefer stored canonical bytes over reconstruction whatever the row's shape (`Ledger::audit_bundle`, the public bundle hydration and the two bundle queries), which is what keeps an archived block's advertised `audit_bundle_sha256` serveable once its window is no longer in the ledger. tests/share_archive.rs covers the round trip and the chain, tamper detection, every named blocker, the interrupted detach, the restore with and without `--attach`, seal idempotence under concurrency, and acceptance criterion 5: a landed block still served by the ledger API, the operator tool and the public artifact route after its partition was dropped. Co-Authored-By: Claude Fable 5.1 * Record an archive verification only when the live rows were compared A verify after the detach checks the archive against itself and the catalog and reports that without recording it, so it never stands in for the comparison the detach and drop require. The command tables in the docs and the design record carry the tool's flags, and the design record carries the tool's documented behaviours (not_applicable for a detached partition, duplicate refusal under --check-duplicates, one-transaction restore, canonical manifest encoding). The offer-lifecycle tests watch for the migration's lock wait for longer, since every migrate's scratch replay now covers two more migrations. Part of #144. * Refuse to archive or verify an open partition, and recount before detach and drop An archive and a verification of a partition whose upper bound the share sequence had not reached recorded a copy of a range still being written: later appends changed the live rows without clearing archive_verified_at, detach then trusted the timestamp, and drop discarded rows the archive never held. `archive` and `verify` now refuse a partition while qbit_prism_share_next_seq() is below its upper_seq. Ledger rows are immutable, so an append that committed after the comparison is the only way the live rows can diverge from a verified archive: `detach` compares the live row count plan just took with archive_rows, and `drop` counts the standalone relation before DROP TABLE, each refusing on a mismatch with both numbers named. Review finding on #419. Co-Authored-By: Claude Fable 5.1 * Refuse a non-adjacent manifest chain when archiving and when verifying An operator who archived p0 and then p2 got a manifest for p2 that linked to p0, since both archive and verify pick the nearest archived predecessor, and verify only reported the gap as chain_adjacent=false while still recording archive_verified_at. p2 could then be detached and dropped with the promised contiguous chain incomplete, and p1's later archive left p2's link stale. `archive` now refuses a partition whose nearest archived predecessor does not end where it starts, so partitions are archived in upper_seq order and links never go stale, and `verify` treats a gap as a failure instead of a field; chain_adjacent leaves the verify output since it can no longer be false. Both share one predecessor lookup, taken before the rows are streamed. Review finding on #419. Co-Authored-By: Claude Fable 5.1 * Clear every later archive's verification when a manifest is rewritten, and refuse once one has left the ledger `archive --force` gives a partition a new manifest and so a new digest, which every later manifest links to through previous_manifest_sha256. Only the rewritten partition's verification was cleared, so an already-verified successor kept archive_verified_at and could be detached and dropped without the chain check ever running against the new link. The archive now clears archive_verified_at on every archived partition above the one being written, names them in its output, and refuses when any of them is no longer attached: a detached or dropped successor cannot be archived again to follow a new manifest, so the recorded one has to stay. The successors are read before the rows are streamed and their count is checked again inside the catalog transaction. Review finding on #419. Co-Authored-By: Claude Fable 5.1 * Plan each share range page with its bounds so the partitioned ledger keeps the ordered scan The paged window read, the audit verification pages and the audit range read all select a share_seq range through a cached prepared statement. After five executions PostgreSQL may switch such a statement to a generic plan, and on the partitioned ledger the generic plan for a range with unknown bounds estimates a few hundred rows, so it appends the leaves and sorts them instead of walking the primary key in order. Each page then reads and sorts the whole remaining range: 180 ms a page against a 400k row window, up from 4 ms, which is what pushed the 500k cross-frontend resume past its 25-second budget in CI. Run each page as an unnamed statement so it is planned with its bounds; planning a page costs a fraction of a millisecond. The single-bound snapshot walk keeps its ordered scan under the generic plan and is left alone. Co-Authored-By: Claude Fable 5.1 * Name new share ledger partitions from a catalog counter so a width change never reuses a name qbit_prism_share_partition_ensure() named the next partition after the highest attached bound divided by partition_rows, which is documented as changeable after the conversion. Once the width changes, that division maps the bound back onto a name already in the catalog: doubling the width at a bound of five widths asks for qbit_share_ledger_p2, and qbit_prism_share_partition_create() refuses the existing relation, so neither the maintenance task nor the append's own retry could extend the ledger again. Take the number from qbit_prism_share_partition_next_number(), one above the highest number the catalog has ever recorded in any state, so names are unique and in bound order whatever the width, and a dropped partition keeps its name reserved for its archive. Bounds stay contiguous: each new partition starts at the highest attached bound and is one current width wide. The docs describe the naming the same way. Co-Authored-By: Claude Fable 5.1 * Wait for in-flight appends before streaming a partition for archive or verify qbit_prism_share_next_seq() passing the partition's bound does not prove every row below it is visible: sequences are nontransactional, and an append draws its share_seq inside a transaction that holds ORDER_LOCK until it ends, so the last value of the partition can be handed out while its row is still uncommitted. An archive or a verification streamed at that moment misses the row; DETACH CONCURRENTLY then waits for the append and takes the committed row with the partition, out of the online ledger and into a standalone relation the drop refuses because its count no longer matches the archive. After the bound check, take and release the ordering lock outside any transaction: it returns once every append that drew a value before the bound was passed has committed or rolled back, and any later append draws a value at or above the bound. The regression holds such an append open, shows the archive waits, and checks the archive and its verification hold the late row. Co-Authored-By: Claude Fable 5.1 * Pin partitions read by unfinished candidate payout windows * Fence archive verification against concurrent manifest rewrites * Require the recorded archive digest when restoring a partition * Record archive drain waits through the timed advisory lock helper * Align migration fixtures with catalog-based partition numbering * Register the existing partition and archive regression tests in CI * Sync archive directory entries before recording the copy of record * Keep empty-ledger conversion bounds above the next share sequence * Keep latest evidence available without optional rollup tables * Evaluate retention blockers in one consistent snapshot * Allow deliberate retention scans to exceed request timeouts * Capture solver attribution from older frontends throughout conversion * Serialize archive lifecycle changes through detach reconciliation * Retain prior archive versions until replacement recording succeeds * Document the archive version directory in the runbook and CLI help The archive command writes both files into /qbit_share_ledger/// since prior versions started being retained, but the operator runbook's command table, layout block and hand-verification example still described the flat partition directory, so the prescribed off-host check failed with missing files. Name the version directory in all three, explain how it is selected from the catalog's archive_manifest_sha256, and correct the archive --force help text, which still said it overwrote the existing archive. Co-Authored-By: Claude Fable 5.1 * Seal audit bytes on a maintenance connection without a statement timeout The seal rebuilt each artifact and stored its canonical bytes through the operator pool, whose connections carry the 15 s statement_timeout. A production window is over a hundred megabytes, so either the unpaged range read of the rebuild or the TOASTed write could time out on every attempt, leaving sealed_at unset and the partition impossible to detach. audit_canonical_bytes and materialize_audit_row now read through an AuditReader: the pool, one checkout per statement so the public read pool keeps applying each request's remaining deadline, or one connection whose session settings the caller chose. The seal runs its cursor, rebuild and write on the same close-on-drop maintenance connection the detach and drop already use, so nothing escapes into pooled reads even when cancelled. The regression test lowers a tool pool's statement_timeout, delays the artifact write past it with a trigger, and checks the seal still stores the bytes while the pool keeps its own timeout. Co-Authored-By: Claude Fable 5.1 * Read the archive back from disk before dropping a detached partition `share-archive drop` relied on `archive_verified_at` and a row count before `DROP TABLE`, so an archive deleted, truncated or altered between verify and drop would have been made the copy of record while no restorable copy existed. The drop now takes `--dir ` and, right before the one irreversible statement, re-reads the recorded manifest and rows through the same checks verify applies: canonical manifest, digest against the catalog, row count and rows digest, gzip and stream digests of the rows file. Any failure refuses the drop with nothing changed, and the response names the manifest that was read back as `archive_checked`. The manifest and rows checks are factored out of verify into `read_recorded_manifest` and `check_recorded_rows`, so both commands read the archive the same way. The runbook, the design record and the CLI help carry the new flag, and a gated test covers a missing, truncated, altered and reformatted archive between detach and drop, then the drop once it is whole. Co-Authored-By: Claude Fable 5.1 * Register the slow-artifact seal test in the gated manifest 0790e494 added share_archive::seal_stores_a_slow_artifact_without_changing_pool_timeouts without listing it in test/prism-gated-tests.txt, so the prism-integration-proof job failed with "executed but is not in the expected list". Co-Authored-By: Claude Fable 5.1 * Read the probe floor from the attached partition bounds, not the width qbit_prism_share_probe_floor() subtracted two current partition_rows from the next share_seq. partition_rows is documented as changeable, and once it is raised after narrower partitions were attached, that floor spans every one of them still attached below the sequence, so each bounded share_id probe on the acknowledgement path descends one per-leaf index per retained narrow partition instead of three. The floor is now the lower bound of the attached partition two below the one the next share_seq lands in, read from qbit_prism_share_partitions: at most three leaves holding rows, plus the empty lead, whatever width each partition was created with, and never above the old floor when the width has not changed. Before the conversion the catalog is empty and the floor is 0, as before. The probe test moves the sequence into p3 rather than p2, since the floor now clears p0 only once two full partitions sit below the sequence's, and asserts the floor's value. A new test quadruples the width, moves the sequence into the first wide cell and checks the floor is p3's lower bound and that EXPLAIN removes exactly the three leaves below it; against the two-widths formula it fails with a floor of 0. Co-Authored-By: Claude Fable 5.1 * Refuse to archive or certify over an unverified predecessor A manifest's link proves one hop. After a rewrite of p0 cleared the verifications of p1 and p2, p2 could be written again over p1's unchanged digest, certified on that link alone, and detached, leaving p1 linked to the replaced p0 digest and no longer rewritable behind a departed successor. Both archive and verify now require the nearest archived predecessor's archive_verified_at, and verify's fence re-checks it on the locked row, so a verification stands for the whole chain below it and a repair runs from the rewritten partition up. Co-Authored-By: Claude Fable 5.1 * Gate the rollup condition on the partition's newest committed row The rollup sweep advances qbit_hashrate_rollup_progress to the newest row it folded, never to a share_seq the sequence handed to an append that rolled back. The plan compared the watermark with upper_seq - 1, so when a rolled-back append drew a partition's last value the sequence had passed the bound while no sweep of that partition could ever report it, and the blocker named a share_seq that will never exist. Compare with max(share_seq) of the partition's committed rows instead, read by the same scan that counts them, and report it as newest_share_seq in the plan. Appends commit in share_seq order under ORDER_LOCK, so a watermark at or past the newest row covers every committed row below it. A partition the sequence has not passed stays blocked, with the sequence position named: its rows are not all known yet, and an empty ledger whose sweep has run must not list every partition as eligible. Co-Authored-By: Claude Fable 5.1 * Hold a partition detached by hand to the seal before recording or dropping it The reconcile branch of share-archive detach, which brings the catalog in line with pg_inherits when the DDL of an earlier run succeeded and its catalog update did not, required only archive_verified_at. It cannot tell that run from a DETACH PARTITION an operator ran by hand after archive and verify but before seal, and drop then accepted the recorded state without looking at sealed_at either. A canonical artifact is rebuilt from shares the parent still holds, so recording and dropping such a relation would leave every intersecting audit without the bytes it advertises, with no way to rebuild them. Take the seal directly for a relation that is off the parent, since plan marks every condition not applicable there: sealed_at must be recorded, and no audit row whose snapshot intersects the partition may lack its canonical_audit_bytes, the same audits_sealed condition a detach requires. Both the reconcile and the drop refuse otherwise and print the ATTACH PARTITION statement under the recorded bounds, since the seal itself needs the rows online. The regression test detaches by hand without sealing and checks the reconcile, a forged detached state at the drop, and a recorded sealed_at that an unsealed audit row overrides; the round-trip chain test now seals the partition it detaches by hand. Co-Authored-By: Claude Fable 5.1 * Hold an interrupted detach to every condition before finalizing it The detach-pending mark PostgreSQL leaves when a DETACH PARTITION ... CONCURRENTLY is interrupted proves only that a detach was started, by share-archive detach or by an operator's own statement, not that any condition held. detach took the mark as that proof and issued FINALIZE past every blocker plan reports, so a partition the payout window still reaches or an unfinished candidate still references could be finalized, recorded detached and then dropped. Require every condition again for a pending partition. The mark keeps the relation attached and present, so plan's blockers are the whole of that gate, and the mark itself stays the one unknown FINALIZE is not refused for. A partition that fails prints the FINALIZE and ATTACH PARTITION statements that bring its rows back, since the mark can only go forward and the parent hides the rows meanwhile. That hiding reached the payout window condition, which is read from the parent. For a pending partition holding the oldest accepted rows the window ran out of visible rows above it, reported the lowest row it saw as the floor, and the condition read clear; with every accepted row hidden the ledger looked empty and read clear for that reason. plan now reads the window's counted weight with its floor, and a pending partition with accepted rows is blocked when the window fell short of its weight, because it would have gone on into those rows; a floor reached at full weight above the partition stays exact. The rollup sweep reads the parent too and cannot fold rows it does not see, and nothing can show that afterwards, so the docs now say never to detach an unfolded partition by hand. The regression test interrupts a hand-run concurrent detach on a partition the window still reaches, checks that the parent hides its rows, that plan blocks the window condition for them, that detach refuses with the two statements and leaves the mark and the catalog alone, and that the same mark is finalized once the window has moved on. Co-Authored-By: Claude Fable 5.1 * fix: enforce retention gates when reconciling detached partitions * fix: preserve exact replays of legacy duplicate headers * fix: rebuild global share hashes when importing archives * fix: reject archive imports that leave routing gaps * fix: reject archive imports ahead of the share sequence * fix: reject duplicate share IDs in archive imports * fix: reject archive imports behind the rollup watermark * fix: require a recorded manifest for archive reattachment * fix: bound imported partition numbers to bigint range * fix: preserve uniqueness of legacy rejected share IDs * fix: reject archive imports of previously credited share IDs * fix: reject archive imports overlapping cataloged ranges * fix: bound legacy share ID fallback to conversion range * fix: retain rejected share IDs across partition retirement --------- Co-authored-by: Claude Fable 5.1 * docs: reconcile B281 HA architecture with approved async D3 (#449) * fix: time coordinator-owned database checkouts (#446) * WIP: preserve unqualified coordinator checkout timing tests * fix: time coordinator-owned database checkouts * test: clarify coordinator checkout polling cases * test: qualify metrics cardinality and identifier privacy (#450) * test: bound metrics cardinality and guard identifier privacy Exercise closed enum variants and seeded identifiers through real HTTP, Stratum and PostgreSQL collector paths. Document native health acceptance separately from intentional legacy shape differences. Refs #278. * docs: clarify metrics privacy evidence and focused checks * Bound public readiness warnings and report recovery (#451) * docs: define bounded public readiness event policy * fix: bound public readiness warnings and emit recovery * fix: time rollup database checkouts (#453) * 3.x.x: prepare bounded HA functional qualification (#452) * test: prepare bounded disposable HA qualification for #281 * test: use shared integration gate names in HA runner * test: require complete HA qualification suites * test: await owned HA endpoint shutdown and retain cleanup evidence * fix(prism): time landed audit pool checkouts (#454) * Finish the #270 candidate-storm remainder: pre-offer kill, relaunched-process recovery cost, chain-RPC fault table (#459) Refs #270. Test and documentation only. Adds the pre-offer kill and relaunched-process statement-cost scenarios to tests/candidate_storm_restart.rs, and a table-driven chain-RPC fault suite to src/ledger/candidates/storm_fault_tests.rs; updates the gate manifest and four rows of docs/prism-deleted-test-map.md. * fix(prism): isolate compact persistence from settlement locking (#456) Retain cluster SHARE before all compact persistence authority checks and join blob collection to the same protocol with an exclusive cluster fence after SETTLEMENT and ORDER, before fresh reference scans. Prove runtime and direct persistence under a held settlement stub, preserve authority/expiry/cancellation assertions at the true row fence, and cover surviving orphan repair races in both collector orders. Require coordinated stop-all upgrades because old collectors cannot fence the new writers. Part of #275; the full refresh performance target remains open. * 3.x.x: complete pool checkout timing and guard its coverage (#457) * fix(prism): complete run-role pool acquisition coverage Time partition maintenance and fallback checkouts plus candidate heartbeat reads at the existing checkout boundary. Add a reviewed source census, CI guard, negative controls and focused PostgreSQL coverage while preserving wrappers and deadlines. Refs #352 * fix(tests): isolate API fixture migration locks by database Use the existing FixtureDatabase helper so concurrent API fixtures do not contend on database-scoped migration keys. Preserve API assertions and timeout budgets, keep cleanup on failure or cancellation, and abort the mock RPC task when its fixture is dropped. Add a regression that holds one fixture migration lock while opening another, and register its required integration execution. * docs(prism): align pool timing coverage descriptions Include rollup and share-partition acquisitions in metric and alert coverage, replace stale partial-coverage claims with the source census and explicit exclusions, and regenerate the inventory, migration table and alert patch. Alert expressions, thresholds and deployment gates are unchanged. * 3.x.x: bound refresh snapshot payload reads (#466) * 3.x.x: bound refresh snapshot payload reads Move an exclusively retained suffix into fresh ledger acquisition, validate single-leaf incarnation and complete eligible membership, and read only a bounded appended page before trimming. Preserve hashes, authority and cancellation admission, with conservative full-read fallback. Add same-anchor PostgreSQL differential and cancellation coverage plus the 400k paired qualification record. Median last-client latency improves 30.11% in the measured one-frontend workload; the one-second goal remains unmet and partition-crossing fallback overhead remains explicit. Refs #275 Co-Authored-By: OpenAI Codex * 3.x.x: classify delta acquisition as a borrowed transaction The checkout census enrolls calls named advance because the rollup helper accepts a pool. Snapshot delta advance instead receives the existing transaction, so classify this exact call under the borrowed policy without changing any checkout, timer or scanner rule. The real-source gate and all 14 guard tests, including negative controls, pass after reproducing the unclassified-call failure. Refs #275 Co-Authored-By: OpenAI Codex * fix(prism): invalidate retained shares across writer timelines * test(prism): qualify timeline acquisition at the default pool size --------- Co-authored-by: OpenAI Codex * 3.x.x: overlap independent refresh computation (#476) * perf: overlap native reference and audit preparation * fix: refresh CPU preparation checkout census * 3.x.x: Reconcile the deleted-test map with closed issues and add a check that keeps it honest (#455) * ci: check the deleted-test map against the tree and against issue state docs/prism-deleted-test-map.md is parity evidence for the cutover go/no-go (#291), and nothing read it: the issues it linked closed one by one while the map kept naming them as the owners of uncovered behaviour (#448). scripts/check_deleted_test_map.py reads it. Offline, in the required lint-compile-compose job, it fails when a `path::name` reference does not resolve to a test function, when a table line does not parse (a line it cannot read is an error, never a skipped row), when a status breaks the legend, or when the summary, the section counts or the needs triage index differ from the rows. With --check-issues it asks GitHub for the state of every issue the map depends on and fails when one is closed. That runs as an advisory job kept out of `checks`, so an issue closing cannot block an unrelated pull request, and it can still go red. It lives in ci.yml because schedule and workflow_dispatch only fire from the default branch and this map lives on 3.x.x. An unknown state (network, rate limit, unexpected reply) exits 2 and is never read as open. Co-Authored-By: Claude Fable 5.1 * docs: reconcile the deleted-test map with the issues that closed 29 of the map's 34 issue links pointed at closed issues, and every open gap row named one, so by the map's own legend each gap was either filled or orphaned and the map did not say which (#448). The pull requests that closed #266, #271, #272, #280, #285, #287 and #288 never touched it. Every row that linked a closed issue now names the native tests that landed, retires what has no native subject with the reason, and writes what is still uncovered as "not covered, no open issue". No row links a closed issue. The five open gap rows and the open gap case are partial. tests/test_prism_landing_interleavings.py is decided: partial with nothing left uncovered, and no new native test. The issue's premise that no native test has the two-distinct-block shape does not hold. #355 added three tests for exactly that #272 criterion which the map never cited, a checks each block's audit row, and the #133 mechanism (a shared publication floor gating a per-block evidence file written after confirmation) has no native counterpart: the audit row is inserted, keyed by block hash, in the landing transaction. The seven needs triage rows were re-read at a1937054. Two are now partial (the CTV yield contract from #433, the changed-amount balance fence from it from a needs triage row, and the new closed-issue check failed on the next run. That row was re-read at cb2aa8bf: #450 covers its one owned case (no hash or height in /metrics), and it stays needs triage because landing-phase attribution still has no native family and no owner. Also corrected, because they made the same claim in plain text or were stale for the same pull requests: the job-builder, #224 cadence, #224 ledger, representative-independence and coordinator-metrics rows. The legend now says a linked issue must be open and a plain #N is history, and a new section documents the checks. Summary: 5 full, 76 partial, 0 open gap, 5 needs triage, 56 retired. Co-Authored-By: Claude Fable 5.1 * docs: correct the map's new citations after an adversarial review Five reviewers who had not written the reconciled rows read the body of every newly cited test, 152 citations in all, and tried to break each claim. 142 held, 10 were overstated, none was unsupported, and five "retired" clauses named something that still exists natively. The corrections all move the map toward claiming less: - test_prism_hot_path.py: the 600-client submit burst is not retired. Native submits read-lock what publication write-locks and no test bounds the wait, so it is now "not covered, no open issue". - prism_postgres_a1_migration_gate.py: the pre-ordinal backfill and the non-positive and duplicate ordinal refusals are not retired. 001 still runs them under the native migrator, which does not refuse a pre-v2.0.0 source. - test_prism_settlement_balance_comparison.py: a re-spelled payout address is a digest divergence natively, the opposite of what the row said, and is untested. - test_prism_share_ledger.py: "no automatic SQL retry" had one exception, the share append re-run after a no-partition refusal, which has a test. - The representative-independence and CTV rows called live things retired (per-solver bootstrap identity, PRISM_CTV_BROADCASTER_LIMIT); the validation, delivery, retained-jobs, landing, reward-window, compose and perf rows each had a clause worded past what its test asserts. Two under-claims are fixed too: the TipState::observe sequence guard is tested, and the reward-window row now cites the test that pins 8. The landing-interleavings decision was attacked directly and held. Statuses and summary counts are unchanged. Co-Authored-By: Claude Fable 5.1 * fix: validate deleted-test names against pinned manifest * fix: pin deleted-test case inventory * fix: require test citations in partial map rows * fix: include unowned partial gaps in map summary * fix: require Python map citations in unittest discovery * fix: exclude empty native remainders from map summary * fix: include open case rows in deleted-test summary * fix: ignore Rust fixture text in test citations * fix: reject Rust map citations to tests skipped by CI * fix: reject skipped Python parity citations * fix: require reachable Cargo tests for parity citations * fix: recognize uncovered markers without semicolon separators * docs: reconcile deleted-test owners after issue 270 closed * fix: require successful runs for Python parity citations * fix: reject malformed deleted-test owner links * fix: include case rows in the triage index check --------- Co-authored-by: Claude Fable 5.1 * 3.x.x: measure accepted-block revision work delivery (#475) * feat: observe accepted-block revision work through Stratum delivery * test: exercise payout maturity and document idle-listener boundary * fix: keep saturated landing observations closed until restart * fix: preserve unknown original revisions after peer confirmation * fix: sort landing integration gate entries * fix: bind landing observations at the ledger commit boundary * fix: retire proven-orphan landing waits without delivery samples * fix: stop landing completion after ordering evidence is lost * fix: expose unknown landing tracking without masking known stalls * fix: reconcile terminal orphan observations across frontends * test: keep orphan cancellation controls lint clean * 3.x.x: Classify proven stale commit-gate refusals (#488) * fix: classify proven stale commit-gate refusals Retain the first gate closure cause without changing admission predicates, deadlines or commit authority. Emit the existing stale-job response only for a completed typed refusal of stale lease authority, preserving backend, uncertain and duplicate-credit outcomes. * fix: preserve backend classification for unavailable tip proof * 3.x.x: pipeline canonical audit hashing during refresh (#490) Compute the accepted-share audit prefix before native digest work on the existing lane, then finish via an async body continuation under the original admission. Stream reward commitment JSON into its existing hash. Preserve canonical bytes and public owned APIs, with frozen legacy byte/hash comparisons and cancellation, saturation and authority tests. Refs #275 * 3.x.x: report unknown harness measurements as null, never zero (#483) (#492) * 3.x.x: report unknown harness measurements as null, never zero Four places in qbit-prism-load's report wrote a plain zero where the measurement was unknown (#483). Each now reports null with a reason in the report's existing unavailable_reason style, and the measured zero each was indistinguishable from keeps its 0: - A lock summary over an empty sampling window carries max_waiters, samples_with_waiters, episodes_at_least, foreign_waiter_samples and foreign_holder_samples as null beside its unavailable_reason; a window the sampler polled and found quiet carries 0. The summariser is a pure function so both cases are testable without a database. - PhaseEvidence and ArtifactInputs carry Option percentiles, and artifact::build refuses a phase or run without them instead of writing 0.000. The side report's validator.worst_artifact_phase_ack_p99 is null with a reason naming the unmeasured phase rather than folding 0.0. - time_to_usable_work's all_sessions_milliseconds is null with all_sessions_unavailable_reason unless every session got usable work while the tip was the tip, as its definition already said; the slowest served session remains latency_milliseconds.max. - A phase without a reconciliation reports achieved_rate_shares_per_second null with achieved_rate_unavailable_reason; a reconciliation that acknowledged nothing reports 0. The printed summary prints each as None, like the percentiles beside them. The artifact's schema and its server-side reader are unchanged; the side report gains nullable values and reason fields only. One regression test per case with a real-zero or partial control, in tests/harness.rs. Co-Authored-By: Claude Fable 5.1 * 3.x.x: print an unknown acknowledged count as None on the summary line Review follow-up on #483. The printed phase line still read the acknowledged count with unwrap_or_default, so a phase whose reconciliation is null printed acked=0 two columns from rate=None: the one place in the diff's blast radius where unknown was still zero. It now prints the count the same way as the rate and the waiter maxima, Some(value) when measured and None when the report has null, and the summary-line test asserts the unknown, measured-zero and measured cases side by side. The missing-reconciliation arm of achieved_rate is documented as the guard it is: run_inner reconciles every phase it reports, so neither the old 0 nor the new null is reached by a real run. Its reason text no longer claims the acknowledged count is unknown to the harness, only that there is no reconciled count to rate. The README now says what the summary line prints (Some(value) or None) and that the all-sessions figure and the validator's worst p99 are not on it. Co-Authored-By: Claude Fable 5.1 --------- Co-authored-by: Claude Fable 5.1 * 3.x.x: retire the #458 cross-restart remainder in the deleted-test map (#495) Refs #458. Rows 236 and 239 still named #458 as the owner of an uncovered behaviour after the acceptance reconciliation of 2026-09-22 accepted the bounded process-local landing contract, so the closed-issue guard reopened it. Both rows stay partial and keep their named landing tests; the unconditional exactly-once-across-restart remainder is now written as "not covered, no open issue" with the reason and the #458 decision as history. No runtime, test, count or workflow change. Co-authored-by: Claude Fable 5.1 * 3.x.x: bound share-append admission before pool checkout (#491) * 3.x.x: bound share-append admission before pool checkout Admit at most N-1 share appends per frontend pool before connection checkout, so appends waiting on ORDER_LOCK cannot occupy every connection that refresh needs after an external tip. Handles sharing a pool share the cap. Admission follows the transaction through SQLx connection cleanup, including cancellation and an uncertain COMMIT reply. Revision fences, the commit gate, the original submission deadline, the missing-partition retry and duplicate handling are unchanged. Refs #481 Co-Authored-By: Claude Fable 5.1 * 3.x.x: classify the admission checkout sites in append_prepared The pool checkout census requires an exact classification for every checkout-shaped call. Admission::acquire is a semaphore wait, not a SQLx checkout; Ledger::acquire is the existing delegated timed checkout; the explicit Transaction::begin borrows that connection and adds no checkout. The replaced self.begin() entry is removed. Refs #481 Co-Authored-By: Claude Fable 5.1 * 3.x.x: spawn append admission cleanup on the captured runtime handle Cancellation can drop the append connection guard from a thread that never entered the runtime context. An ambient tokio::spawn panics there, and for a caller-supplied pool with min_connections > 0 the panic during SQLx's own off-context drop became a double panic that aborted the process. Capture the runtime handle when admission is granted and spawn the cleanup on it, as the blocking-drop helper already does. Permit ordering through rollback and connection return, pool-close wakeups, the original deadline and every SQL fence are unchanged. Add two gated regressions: an append polled to ORDER_LOCK and dropped on a plain thread must not panic and must hold its permit until the queued rollback and return finish; a min_connections pool must see exactly SQLx's single baseline panic and still clean up. Bound the two leaked-admission assertions with a timeout, pin the SQLx return_to_pool assumption and the registry ceiling in comments, and document abort queue position, the acquire_timeout failure-mode change and the scope of off-runtime drops. Refs #481 Co-Authored-By: Claude Fable 5.1 * 3.x.x: qualify off-runtime cleanup to the ledger's default pool The admission doc said an append dropped outside the runtime context cleans up without panicking, unconditionally. That holds for the ledger's own pool, which sets no min_connections. A caller-supplied pool with min_connections > 0 still reaches SQLx's own off-context spawn after the cleanup is queued and panics there exactly once, as it did before admission existed; the guard removes only the double panic. Say so, and that off-runtime drops of SQLx guards are not supported in general. Documentation only. Refs #481 Co-Authored-By: Claude Fable 5.1 --------- Co-authored-by: Claude Fable 5.1 * 3.x.x: signing-transition command and migration 019 for signing-key rotation (#461 S-B) (#464) * Add migration 019, the signing-key rotation journal Signing-key rotation resets the pinned cluster fingerprint, and today the rehearsal does that by hand with nothing recording it. Migration 019 adds qbit_prism_signing_transitions: an immutable journal (trigger like 014's) that keeps the retired fingerprint, the retired policy document with both old public keys, every instance row with its measured heartbeat age and the freshness window applied, and the unchanged payout revision. It is additive and declares no capability: a binary that does not know the table never reads or writes it, so a 2.x.x or 3.x.x frontend that predates it keeps working. The rest is the mechanical registration a new migration needs: REQUIRED_SCHEMA_VERSIONS, NATIVE_MIGRATIONS and the migrate_schema apply block in migration.rs (mirroring 014: plain, no shutdown proof); the verbatim version lists and messages the tests assert in offer_lifecycle, ledger_2x, index_trim and session_sequence; the pre-006 historical fixture in ledger_2x, which undoes 019 together with 014; the required_versions array in the recovery evidence script; and the migration note in the migration guide. No behaviour beyond knowing 019. Co-Authored-By: Claude Fable 5.1 * Add the signing-transition command D4 accepted signing keys on every frontend on condition that a rotation is rehearsed before cutover. Ledger::configure already treats a NULL cluster fingerprint as a rotation and refuses to pin over unfinished candidates signed with other keys, but nothing sets the fingerprint to NULL except an operator's hand-written UPDATE, with no guard and no record. `qbit-prism-server signing-transition --confirm` is that reset. Everything is decided and written in one transaction that holds the settlement and order advisory locks, the instance registration lock and the cluster row FOR UPDATE, the same row lock configure takes and every candidate writer reads FOR SHARE, so a concurrent startup or write serialises behind it and then sees the reset. It refuses a halted cluster (fatal-state clear compares the pinned fingerprint, so the halt must be cleared first), a pinned fingerprint that is not the invoking configuration's (the journal must record the keys being retired), any frontend whose row is not `stopped` unless its heartbeat is older, by the database clock, than the freshness window self-check derives from PRISM_HEALTH_REFRESH_SECONDS (a killed frontend never writes `stopped`; an unreadable status or a future-dated heartbeat is unknown and counts as live), and any unfinished block candidate whatever keys it carries, which is stricter than configure's other-keys test. A fingerprint that is already NULL is reported with the last journal row and nothing is written, so a lost commit response is resolved by running again. One 120-second deadline covers the node calls, the lock waits and the transaction; the reset statement runs before the journal insert so any failure rolls both back. Without --confirm the command prints what it would check and do, reads neither its configuration nor the database, and exits non-zero. The invoking configuration is resolved exactly as startup and policy-transition resolve it, by calling policy_transition's resolved_policy; that function's visibility changes from private to pub(super) and nothing else in policy_transition.rs changes, so policy-transition keeps refusing key changes. Multi-epoch verification stays deferred: the journal's previous_policy is its anchor. Co-Authored-By: Claude Fable 5.1 * Test the signing-key rotation reset Gated tests for signing-transition against a disposable PostgreSQL, each fixture in its own database because the command holds the database-scoped settlement and order advisory locks. The killed frontend is a real health heartbeat never followed by `stopped`, aged by one UPDATE of heartbeat_at rather than a sleep. Refusals: a fresh `starting` or `running` heartbeat, an unreadable status, a future-dated heartbeat, both frontends named when both are live; a killed frontend refuses at ages 0 and 14 seconds and, with PRISM_HEALTH_REFRESH_SECONDS=20, still at 16, then proceeds at 16 under the default window, through the CLI and its real environment reader; a `pending` candidate, and `offer_reserved`, `offered` and `reconciliation` rows whose keys equal the invoking configuration's, which configure itself accepts; a halted cluster; the new-key environment. Effects: the journal row with the old fingerprint, both old public keys, the instance rows with their measured ages and the unchanged payout revision; the first new-key configure pins; old-key configure and old-environment reruns are then refused; the journal is immutable; an old-key start after the reset re-pins and the reset can be repeated. Concurrency: the command waits on a holder of the cluster row and judges what that holder committed; held at its journal write after every check has passed, neither a heartbeat nor a configure can land before its commit; an injected journal failure after the reset statement rolls the fingerprint back and releases the locks. Without --confirm nothing is read or changed, even with an empty environment. Under the three review mutants each fails as intended: a commit between the checks and the reset fails the held-at-journal test; a status-only liveness rule fails the killed-frontend test; ignoring offered rows fails the same-keys test. The successful CLI run prints its wall clock under --nocapture for the rotation rehearsal to compare against. Co-Authored-By: Claude Fable 5.1 * Document signing-key rotation The ledger operations guide said signing-key rotation was deferred. It now has a command, so the guide gets a "Signing-key rotation" section: what signing-transition checks and writes and under which locks, the step-by-step procedure with the old-key environment, the freshness rule a killed frontend is judged by, that the first configure after the reset pins whatever keys it brings (an old-key start undoes the rotation safely and the command can be rerun), that pre-rotation bundles verify only with the old public keys the journal keeps, and that backfill-ctv and import-audits must not run with the new key against them. Multi-epoch verification remains deferred and is said so. It also records the one exposure the command shares with the by-hand reset: the fatal-state writers are not fenced on the fingerprint, so a partitioned-but-alive old frontend, indistinguishable from a killed one once its heartbeat is stale, could halt the cluster while the fingerprint is unset, when neither fatal-state clear nor any startup can proceed. The recovery (restore previous_fingerprint from the journal row, clear with the old keys, rerun) is written down, and the command's halted refusal points at it. Co-Authored-By: Claude Fable 5.1 * Say what bounds a signing-transition lock wait The deadline comment and the operations guide said one 120-second deadline covers every lock wait. It does not: the operator connection sets lock_timeout from PRISM_DATABASE_LOCK_TIMEOUT_MS (5 seconds by default) and a 15-second statement timeout, so a contended cluster row, instance table or advisory lock ends the command after about five seconds with "canceling statement due to lock timeout". That is safe, it is before COMMIT and rolls back, but the guidance was wrong. Both now say that 120 seconds is the ceiling for the whole command, what bounds a single wait and a single statement, and that a lock timeout means something still holds the row or the locks, which is what a running frontend or tool does: confirm everything is stopped and rerun rather than raising the timeout blindly. The command attaches the same advice to a lock_not_available failure; nothing else about its behaviour changes. A test holds the cluster row, runs the CLI with a 200 ms lock timeout, and checks the message, that nothing changed, and that the same command then succeeds. The guide's rerun sentence also states its assumption: reporting the journal row after an ambiguous commit relies on nothing having pinned a fingerprint in between, which is why restarts stay disabled until the new-key frontends start; if an old-key process did start, the rerun refuses while it is live and otherwise resets a second time. Co-Authored-By: Claude Fable 5.1 * Classify the signing-transition checkout in the pool-acquire inventory The census guard added by #457 requires every fresh pool checkout in the production module graph to carry an exact classification. The signing-transition transaction begins through Ledger::begin like policy-transition does, so it takes the same delegated ledger policy. Co-Authored-By: Claude Fable 5.1 --------- Co-authored-by: Claude Fable 5.1 * 3.x.x: check unread PRISM settings on the serve path (#461 S-A) (#462) * Inventory the final 2.x.x orphan-confirmation setting The retired inventory was taken from the Python runtime at 3e233d22, but the 2.x.x line continued to e76234aa and added one more runtime-read name, PRISM_CANDIDATE_ORPHAN_TERMINAL_CONFIRMATIONS, the predecessor of native PRISM_CANDIDATE_ORPHAN_CONFIRMATIONS. Without it the name could be presented as live guidance again, and the cutover inventory would claim a completeness it does not have. Record both inventory commits in the header so the next audit knows where the sweep stopped. Co-Authored-By: Claude Fable 5.1 * Check unread PRISM settings on the serve path Only check-config called config::check_environment(); Command::Run went straight to Config::from_env(). A deploy that skips the preflight could therefore start a native frontend silently on a 2.x.x environment, where every leftover PRISM_* name is inert and the operator has no signal that the settings they set are doing nothing. Run the same check first, in the same order check-config uses, so production refuses to start and lab mode warns before any listener binds. The check reads the process environment the runtime itself reads and reuses config::production_mode(), so there is no second reader and no separate notion of production. The diagnostic names each offending variable and never its value. Co-Authored-By: Claude Fable 5.1 * Pin the serve-path check ahead of parsing and startup The serve-path tests proved the diagnostic appeared but not that it came first or that it stopped the process. Moving check_environment() after Config::from_env() left every test green, and turning the production bail into a warning also left them green, because the fixture's unreachable node made `run` exit non-zero either way. Give the production test a known setting the parser rejects and assert its error is absent, so the parse can only be reached once the unread names are gone; assert the node's transport failure is absent too, so the refusal is the whole failure rather than one line before a later exit. The lab test now asserts that same transport failure is present, which is the positive proof that a warning let startup continue. The dead listener, port and elapsed-time scaffolding is gone: server::run contacts the node before binding stratum, so none of it observed anything. Correct the guidance while it is being rewritten. check-config does not validate every setting group; the public reader's database settings belong to check-public-database-config. Scope the paragraph to the coordinator serve path, which is the only command this change touches. Co-Authored-By: Claude Fable 5.1 --------- Co-authored-by: Claude Fable 5.1 * 3.x.x: alert on order and settlement advisory-lock waits (#461 S-C) (#463) * Alert on order and settlement advisory-lock waits qbit_prism_database_advisory_lock_wait_seconds has been recorded since #328 with no rule attached, so a coordinator can sit behind the order or settlement advisory lock indefinitely without anyone being paged. Add two native rules. PrismDatabaseAdvisoryLockWaitHigh watches the per-lock p99 over five minutes, and PrismDatabaseAdvisoryLockFailuresCritical pages on the per-lock failure ratio, where failure also covers a wait abandoned by cancellation. Both exclude lock="migration": that lock is taken only at schema initialization, where waiting out an online migration window is expected rather than a fault. The family is rendered from the cached snapshot body rather than read live at scrape time, so both rules carry the snapshot-fresh gate the other cached-body rules use. A stale or unpublished body suppresses them instead of repeating an old percentile, and the failure ratio stays absent rather than zero until a first failure is observed, which is why no-data remains OK. Fixture scenarios cover per-lock discrimination, the per-lock sample minimum, migration-only waits, stale and missing snapshot series, and both sides of the five-percent failure boundary. The inventory prohibition on this family is narrowed to these two rule UIDs so no other rule can reach for it, and the first-offer prohibition is untouched. Co-Authored-By: Claude Fable 5.1 * Correct the stale first-offer and lock-wait rule prose The migration narrative still said proof-to-first-offer waits were declared with a rule deferred, and that advisory-lock waits had no rule written. Both are now wrong: the first-offer histogram is instrumented as qbit_prism_block_submit_seconds under #391, and the advisory-lock rules land in this branch. Say what is actually true. First-offer carries no firing rule because it is observed at most once per found block, which is too sparse for a rate or a percentile, not because a rule is pending. Name the two advisory-lock rules and record that lock="migration" is deliberately outside them. The same correction applies to the retired-name reasons, so the generated migration tables and the review-only deployment patch are regenerated. Co-Authored-By: Claude Fable 5.1 * Exercise the critical rule's per-lock sample minimum Every sample-minimum scenario so far carried only result="success" series, so PrismDatabaseAdvisoryLockFailuresCritical's ratio was absent and its >=10 gate was never evaluated. Collapsing that gate to a shared minimum across both locks therefore passed the suite untouched. Add a scenario where a busy clean lock records 100 waits with no failures while the other lock records two waits, one of them a failure. The ratio is 0.5 but the sample is two, so the per-lock gate drops it and the contract value stays zero; a shared minimum would lend the clean lock's hundred waits to the failing one and page. Add a second scenario where both locks breach on their own samples, pinning the value at two for the percentile and the ratio alike so neither rule may silently collapse two locks into one series. Co-Authored-By: Claude Fable 5.1 --------- Co-authored-by: Claude Fable 5.1 * 3.x.x: Bound the public artifact route: dedicated rebuild limit, in-flight cap, no post-permit hashing (#469) (#472) * feat(prism): admit public artifact reads under their own limit An artifact read of a native block is window-sized work that outlives its read connection: an unsealed block is rebuilt from its share range, a sealed one has its stored canonical bytes digested and parsed. The route took no permit at all, so N concurrent reads took N read connections and N windows of memory, and the sealed path ran its digest and parse inline on a runtime worker thread. Both now run under a dedicated limit, PRISM_PUBLIC_AUDIT_REBUILD_CONCURRENCY (default 1), built beside the imported-audit decode limit and resized with it, but sized on its own: a larger PRISM_POSTGRES_READ_CONCURRENCY serves more dashboard reads and admits no more windows. The permit moves into the blocking job, as the bundle route's already does, so a dropped request cannot free it before that job ends, and waiting for it spends the request's own deadline. audit_canonical_bytes keeps its signature and delegates to audit_canonical_bytes_admitted, which takes the permit; ledger-internal callers keep passing nothing. Its rebuild now returns the canonical bytes the materialization's own blocking job already produced and proved against the row's digest, instead of re-serializing the rebuilt body on a runtime thread afterwards, which is what lets one permit span the whole reconstruction. Ahead of that, PRISM_PUBLIC_AUDIT_ARTIFACT_MAX_IN_FLIGHT (default 32) caps the audit artifact requests running or queued. Past the cap a request is refused after the route's two indexed point lookups and before any audit read, with 503 audit_artifact_busy, Retry-After and no-store, in microseconds rather than a spent deadline. CTV manifests on the same route are served from stored JSON and are never counted or refused, identical requests still collapse into one computation, and a refusal is never cached. The public service counts them in qbit_prism_public_audit_artifact_refusals_total, with HELP and TYPE. Tests: tests/artifact_admission.rs drives the route at 50 req/s while sampling pool-summary, and covers permit discipline under cancellation and failure, the refusal's zero audit reads through the execution proxy, isolation from the read concurrency, byte identity of both artifact shapes, and the settings' validated ranges. Co-Authored-By: Claude Opus 5 * docs(prism): settings and sizing for artifact-read admission PRISM_PUBLIC_AUDIT_REBUILD_CONCURRENCY and PRISM_PUBLIC_AUDIT_ARTIFACT_MAX_IN_FLIGHT reach a Compose deployment only if Compose passes them, so both services forward them with the code defaults, and the mainnet contract fixture pins non-default values through that path. prism-storage-sizing.md gains the memory bound the route now has: rebuild concurrency times the peak bytes per read, with the per-share canonical length measured at 1,114 B and the peak-to-artifact multiples measured at a 5,000-share window, and the 400,000-share figures marked as derived from them rather than measured. It also records that raising the read concurrency no longer admits more window-sized rebuilds. prism-ledger-ops.md describes the 503/Retry-After refusal and both settings for operators. Co-Authored-By: Claude Opus 5 * docs(prism): count the artifact refusal family in the inventory census The public role gains one family, so the inventory's family count moves from 14 to 15 and the note on untyped public exposition records its one exception. Co-Authored-By: Claude Opus 5 * docs(prism): state that request collapsing needs the response cache The in-flight cap counts computations, and only the response cache makes two identical artifact requests one computation; with it off each request is its own. Say so where the cap is described. Co-Authored-By: Claude Opus 5 * docs(prism): document the artifact route's busy refusal for API consumers The public contract already states the read deadline's 503 read_timeout and the no-store rule; the artifact route's admission refusal is the second 503 a client can see there, and the only response in this API that carries Retry-After. Co-Authored-By: Claude Opus 5 * test(prism): guard the unsealed rebuild's permit and the refusal's cacheability Two paths the first round left unguarded. The cancellation test sealed its row before gating the blocking thread, so only the sealed decode's permit was proven to outlive its request; the unsealed rebuild holds its permit one level down, inside the materialization, and a permit kept by the request frame instead would have passed. The gate-and-time-out sequence is now a helper and runs on both shapes, unsealed first. The refusal was only ever exercised with the response cache off and with a distinct-hash load that never reached the cap, so nothing noticed if a refusal were stored as a cache entry or handed to a follower as a success. The new test runs with the cache on and one admission slot held: every request that shares the refused computation gets the same 503 and no-store, a later request is still refused by the cap rather than from a stored body, and the first read after the slot frees up computes the artifact instead of hitting the cache -- with a second read proving that success is cacheable, so that assertion is a real one. Verified by mutation: passing None into the materialization while the request frame holds the permit, storing an Entry for a refusal, and returning the refusal as Ok(payload) each make exactly these tests fail. Co-Authored-By: Claude Opus 5 * docs(prism): scope the artifact memory bound to the route it applies to Three corrections to what the admission half claims. The sizing and operations notes said a larger read pool admits no more window-sized reads. That is true of /public/v1/artifacts, not of the public service: /public/v1/blocks//settlement-artifacts falls back to the audit-bundle reader for a block with no CTV fanout set, and that reader still rebuilds or decodes the window under the shared imported-audit decode limit, which is sized from PRISM_POSTGRES_READ_CONCURRENCY. Both bounds are now stated, with the note that moving that route under the artifact limit would change another route's behaviour and is a follow-up. The bound also covers only what the permit admits. The route's own re-hash of the returned bytes, the ETag digest and the response body all outlive the permit, so a read's footprint exceeds it by about one more copy of the artifact; the two SHA-256 passes are pre-existing and unchanged here. The refusal counter is a public-service metric: the mining process serves the same route on its operator listener and applies the same cap, but has no public-service metrics owner, so those refusals are uncounted. The inventory meaning says so. Co-Authored-By: Claude Opus 5 * perf(prism): stop hashing an audit artifact twice on the runtime threads At a 100,000-share window the route's own 50 req/s criterion failed: 1 of 84 pool-summary samples took 20.137 s and came back as a failed node RPC, while the read pool was idle (p99 0.107 s). The cause was not the pool. Every 200 response hashed the whole 111 MB body twice more after its permit was released -- once to check the returned bytes against the requested address, once for the ETag -- and one SHA-256 pass over that body measures 1.6-1.8 s in a debug build, so 73 responses of CPU-bound work saturated the four worker threads. The load harness now measures both halves of that: one pass over the artifact, and the overshoot of a task that only sleeps, which reached 23.534 s against a median of 0.001 s. Neither pass was buying a guarantee the request did not already have. The bytes are digested against `audit_bundle_sha256` inside the blocking job, under the permit, and the row is the one that column matched, so comparing the row's digest to the requested address as a string proves the same thing; the ETag is that address, which is the hash of these bytes by construction, so a payload served by its content address now carries it instead of being hashed again. The manifest branch still hashes its own small body and serves it under the address that proved it. No digest verification is weakened and responses, ETag included, are byte-identical. The same load now leaves pool-summary at 0.118 s maximum with no failed sample and the sleeping probe's overshoot at 0.435 s, and the load window drains in 28.9 s instead of 57.3 s. Co-Authored-By: Claude Opus 5 * docs(prism): record the CPU cost the artifact route no longer pays The sizing note listed the route's re-hash and its ETag digest as work outside the memory bound. Both are gone, and the reason belongs in the same section: one SHA-256 pass over a 111 MB artifact measures 1.6-1.8 s in a debug build, so at 50 requests per second those passes saturated the public process's worker threads and failed pool-summary on a node RPC while the read pool was idle. The section now states what the bound covers, what the removal changed, the before and after numbers, and that the legacy fallback body is still hashed for its ETag because it has no proven content address to carry. Co-Authored-By: Claude Opus 5 * chore(prism): carry the artifact-route checkout classifications to their admitted signatures --------- Co-authored-by: Claude Opus 5 * 3.x.x: Stratum per-IP connection cap and per-session request and malformed-frame budgets (#468) (#471) * feat(stratum): per-source connection cap and per-session rate budgets Adds the deferred P2 admission half of #276. Every new limit defaults to zero, which is disabled, so a default deployment keeps today's behaviour. PRISM_STRATUM_MAX_CONNECTIONS_PER_IP is decided in the accept loop from the address the accepted socket reports, before the global permit is taken and before a session task is spawned, so a refused source costs one hash lookup and never reaches the database. The registry mirrors the per-username limiter: weak references pruned on every decision, non-blocking acquisition, release by drop. Both listeners share one registry, so an address spends one budget across the ordinary and high-difficulty ports. Three per-session budgets are plain fields on the session, measured over one shared window. Each is a rate with a full window of burst headroom, never a session lifetime total, and valid mining.submit traffic is never charged: - malformed frames, counting frames rather than bytes, and including the malformed non-submit frames that were previously answered and forgotten; - distinct unknown job IDs. The cost is bounded first by a small per-session negative cache with a short TTL, so repeats of one unknown ID cost no further ledger query; the budget then bounds the rate of distinct IDs; - mining.authorize attempts, charged before the address validation RPC, which bounds the validateaddress calls username cycling can drive from one connection. Re-authorising within the budget behaves as before. Each refusal or budget disconnect records a closed-enum reason on stratum_connection_refusals_total and emits a structured tracing event. The peer address appears only in the event, never in a label. Co-Authored-By: Claude Opus 5 * test(observability): admit the new refusal reasons into the census Four new ConnectionRefusalReason label values add four series to the startup and saturated censuses and widen the independently pinned contract and the fixed-size refusal helper. Kept apart from the behaviour change so it rebases cleanly against other census edits. Co-Authored-By: Claude Opus 5 * test(stratum): cover #276's admission criteria and their controls Socket tests drive the production listener and request loop. C1: a malformed-frame flood disconnects after the budget, counting frames rather than bytes, and 1,000 unknown-job submits from one session produce a bounded number of qbit_prism_jobs lookups and a disconnect. The gated PostgreSQL case counts real server executions through the wire-protocol proxy, which attributes them per pool connection rather than per Stratum session, so it runs its one session in isolation and asserts the global count: 33 lookups at a budget of 32. C2: username cycling from one session is bounded to the budget's address validations, counted at the fixture backend. C3: a connection beyond the per-source cap is refused with no execution at all through the proxy, no session allocation and no response byte; the refusal counter carries the new closed reason; zero disables the cap; and a slot frees when a connection closes. Controls: 1,000 valid submits inside one window at the documented recommended values are neither disconnected nor charged; an honest staleness burst after a payout revision change is not disconnected; re-authorizing within the budget still works; a budget disconnect releases the global, per-source and per-username permits and the database session reservation; and each new setting is exercised through check-config with valid, empty, boundary, malformed and out-of-range values. Co-Authored-By: Claude Opus 5 * fix(stratum): drop the unknown-job negative cache and share the per-IP cap test A resume miss is not proof that a job ID is bogus. Besides honest staleness, a resume misses when its publication stamp changes mid-resume, when issuance authority loses a lease or tip race during revalidation, and while this frontend is momentarily behind on the parent or payout revision. Each clears on the miner's next share, but the per-session negative cache answered that share "stale job" for up to ten seconds without re-querying, and it was active at the default configuration. Remove it entirely. The unknown-job budget alone now bounds the cost: every resume miss is charged, repeated IDs included, so a flood produces exactly budget + 1 ledger lookups before the disconnect (33 at a budget of 32 through the execution proxy, with repeated IDs in the flood). The default stays 0 and the recommended value is documented as well above an honest miner's in-flight submits per interval. The per-IP registry now prunes dead keys only when a new key is inserted, so an accept from a known address is O(1). A new test proves one address spends one cap across the default and high-difficulty listeners, and another proves a transient resume miss recovers on the next submit. docs/prism-metrics-cardinality-privacy.md now matches this branch's census (47 families, 268 series, 170 at startup, six refusal reasons). Co-Authored-By: Claude Opus 5 * test(stratum): wait for the budget disconnect's reservation release instead of reading it once --------- Co-authored-by: Claude Opus 5 * 3.x.x: launch load-harness frontends at the production initial-job admission (#275) (#497) * 3.x.x: launch load-harness frontends at the production initial-job admission (#275) The harness sized PRISM_STRATUM_MAX_PENDING_INITIAL_JOBS to sessions_per_frontend + 16 (128 at least), so a 2,000-session frontend ran with 2,016 permits while production runs the server default of 128. That let every session build its first job at once and, on the real process, inflated the post-publish delivery tail by 50-190 ms against the default, so #275's real-process figures were taken at an admission production never runs. Add --stratum-max-pending-initial-jobs, defaulting to 128 and validated at entry against the connection cap the harness derives (the server refuses more permits than connections). The flag and its default are the only sources: Args::stratum_limits is the one derivation, validate checks it, shared_environment exports it and the child's environment is cleared, so the value the process reads is the validated one. The side report records, per frontend, the launched value read back from its environment, its source (default or the flag), the production default and the pre-flag value for the run's shape, with a null rather than a zero when the launch environment cannot state it. The capacity-evidence artifact is unchanged: the validator's 16 keys do not include the admission. Evidence taken before the flag keeps its meaning: pass the old value to reproduce it, and the README, the honest-value notes and the throughput measurements doc say which admission such runs used. The endpoint, the 400k population, 2,000 sessions, deadlines, receipt reconciliation and the load gates are untouched. Tests: the flag omitted, nondefault (2016, 1) and invalid (0, above the cap, non-numeric) at entry; the value reaching a really launched process, through the run's own launch path against a stand-in that echoes what it read, with a value in the harness's own environment shown not to leak; and the provenance block's null-not-zero. Refs #275 Co-Authored-By: Claude Fable 5.1 * 3.x.x: prove the cleared child environment on an inherited variable, and state the permit queue honestly (#275) /code-review on #497: `std::env::set_var` inside a threaded test binary races libc's `getenv` in other tests (the reason it became unsafe in edition 2024), and the property it served -- no second reader between the flag and the frontend -- is `Command::env_clear` in `Frontend::spawn`. The launch test now shows that on a variable the test process really inherits from Cargo, `CARGO_MANIFEST_DIR`, echoed by the stand-in as unset. The README said each queued session waits behind at most 128 builds; the permit queue is first-in, first-out, so the last of 2,000 waits behind every earlier waiter, inside the server's 30 s initial-job timeout. It now says so, and what a slow database or a much larger session count does to the tail. Refs #275 Co-Authored-By: Claude Fable 5.1 * 3.x.x: keep the pre-flag admission formula on StratumLimits and record the admission once (#275) Thermo-nuclear pass on #497, run inline: the pre-flag sizing (sessions_per_frontend + 16, at least 128) was inlined in the side report's block while every other shape-derived limit lives on StratumLimits, and the topology block repeated the per-frontend admission record, so one report carried the same fact from two sources (the arguments and the launch environment). The formula is now a StratumLimits method, and the per-frontend block -- read back from the environment the process was launched with -- is the one record. Refs #275 Co-Authored-By: Claude Fable 5.1 * 3.x.x: state the pre-flag admission floor, the key count and the restart-phase caveat (#275) Independent adversarial review of #497 (Fable, on adeef89a): - The throughput measurements doc gave the pre-flag admission as sessions_per_frontend + 16 without the 128 floor, so the 200-session premise probes, which ran the floor, would have been "reproduced" at 116. The row now states the floor and says which rows need the flag. - The README's test summary said 16 configuration keys; the server's CONFIGURATION_KEYS has 17 and the test asserts against its length. - The README's honest-values note covered the setup queue but not the reconnect phase's drained restart, where a frontend's whole session set re-enters the same permit queue: reconnect p50/max are not comparable across the admission-flag boundary either, and it says so. Docs only; no source change, so the 2,000-session proof on 7d242ad4 stands. Refs #275 Co-Authored-By: Claude Fable 5.1 --------- Co-authored-by: Claude Fable 5.1 * 3.x.x: gate the accepted-revision-work paging rule like the other criticals (#496) * 3.x.x: gate the accepted-revision-work paging rule like the other criticals Refs #493. PrismAcceptedRevisionWorkPendingCritical was the only paging rule with for 0s, no_data_state Alerting and no conjunction with the snapshot gauges, the scrape or connected miners. A frontend with no connected miners has no mining.notify write that can close its wait, so it paged 307 s after every pool block and never cleared; a stale or unavailable snapshot, a failed scrape and a missing series paged too. The rule now requires a known age (>= 0), an available and fresh metrics snapshot, a scraped target and at least one authorized miner on the reporting frontend, with a one-minute dwell and no-data and evaluation errors treated as OK. The dwell only suppresses single-scrape gate flaps on a gauge that is monotonic during a stall and stays well under the five-minute hold that delayed #413; a prolonged known stall with miners connected still pages. The warning keeps zero dwell and no-data Alerting, so missing or unknown measurements stay visible. HELP for the histogram and pending gauge now states that every frontend samples every block from its own observation; metric names and types are unchanged, and no landing, coordinator or ledger state changes. Scenarios: thirteen existing landing cases gain the gate series, three now expect no critical sample where the guard drops -1 or absent data, and eleven new cases cover each gate, the dwell, a miner flap under the dwell and a stall outlasting it. The deployment test asserts the rendered paging rule's for, noDataState and execErrState. Co-Authored-By: Claude Fable 5.1 * 3.x.x: page through a stale snapshot and state the miner-gate consequence plainly Refs #493. Review follow-up on #496. The paging rule no longer conjoins the metrics snapshot gauges: the pending gauge is a live overlay rendered on every scrape, and a database-pool stall that also stalls the health publisher must still page. The known-age, authorized-miner and scraped-target terms and the one-minute dwell stay. The miner count is a snapshot family, so while the snapshot is stale the gate uses a count a few refresh intervals old; that rare false page is accepted over a missed stall. The two snapshot fixtures now expect the page. The rule basis, the regenerated migration table and deployment patch, and the metrics document now say what happens when every miner leaves a frontend during a stall: this page resolves and nothing else pages for that frontend, because the coverage rules cannot fire with zero authorized clients and the connected-clients rule is a cluster-wide non-paging warning; only the pending warning keeps reporting the age. The miner-flap fixture is now discriminating: a one-sample flap at 1m15s expects no alert while a new no-flap control at the same evaluation expects firing. The deployment test asserts, for every native page=true rule, a non-zero dwell, OK no-data and error states, and an availability conjunction, with the one-minute check separate. Co-Authored-By: Claude Fable 5.1 --------- Co-authored-by: Claude Fable 5.1 * 3.x.x: make the candidate_storm drain-cost and offer_lifecycle claim tests deterministic (#498) Both tests raced sqlx's asynchronous connection return. When a PoolConnection is dropped, sqlx 0.8 spawns a task that pings the server and only then puts the connection back on the idle queue; a Transaction dropped on an error path only queues its ROLLBACK, which that same task flushes. Neither test synchronized with that task. candidate_storm::drained_rows_cost_the_same_statements_at_the_baseline_and_storm_cardinalities failed with "a pooled connection opened inside the measured window". The ledger pool's maximum is 6 but the drain only ever opened 2 connections; when both were in return flight at a transaction boundary, the next acquire found the idle queue empty and opened a third, whose after_connect statements were charged to the row in flight (28 statements instead of 26). Draining warmup rows cannot settle this, so saturate_ledger_pool now opens every slot before the first measured window and drain_set asserts the pool is still at its maximum when a measured window opens. At the maximum, the same acquire waits for the return instead. offer_lifecycle::every_unfinished_state_is_claimable_retained_and_reserved_only_once failed with "offered not claimable". The refused finish_candidate just before had locked the row FOR KEY SHARE in a transaction whose ROLLBACK was still queued; the expiry UPDATE (FOR NO KEY UPDATE) does not conflict with that lock, while the claim lane's FOR UPDATE SKIP LOCKED does, so the lane skipped the row. expire_lease now takes the row FOR UPDATE before the expiry UPDATE, so it commits only after every earlier transaction on the row has rolled back. The same helper replaces the identical closure in the signer-rotation test. No assertion changed. The claim path's SKIP LOCKED gap is the documented approximate service ratio and the next poll retries it, so this is not a product race. Co-authored-by: Claude Fable 5.1 * 3.x.x: cut the per-session checkouts on the new-tip fan-out (#275) (#499) * 3.x.x: gate the pool checkout probe on the idle gap (#275) Every pool checkout pinged the connection before use, on top of the unconditional ping SQLx runs when the connection is returned. Under a work fan-out the pool never idles for more than a few milliseconds, so the acquire ping was one of three PostgreSQL round trips per checkout that proved nothing the release ping had not proved moments earlier. The checkout probe now runs only when the connection has been idle for at least PRISM_DATABASE_ACQUIRE_PROBE_IDLE_MS (default 100 ms; 0 restores a probe on every checkout). A connection that dies inside that window fails its first statement with the same I/O error any mid-statement disconnect already produces and is discarded by the release ping; no payout, authority, deadline, digest, fence or admission path changes. Gated tests cover both sides of the threshold against a terminated backend and the setting's bounds. Co-Authored-By: Claude Fable 5.1 (cherry picked from commit 6770d670b6104510d3e35afd573d856bebb13e0e) * 3.x.x: prove a build survives a connection killed inside the checkout-probe window (#275) End-to-end against a real coordinator and PostgreSQL: with the probe window wide open, terminating the pool's idle backends makes the next job build fail once with a Stratum error, and the following attempt serves work for the same tip. The pool discards the dead connection on release, so recovery is one retry on the session timer, never a wait for the next refresh. Co-Authored-By: Claude Fable 5.1 (cherry picked from commit 812a7bdd94129bbccb0725dbdf409513330e14d3) * 3.x.x: read the clock and payout revision in one statement per issuance boundary (#275) Every issuance revalidation with an absolute expiry read the database clock and the payout revision as two statements on two pool checkouts, six PostgreSQL round trips for two values of one snapshot, at up to two boundaries per delivered job. The work ledger now reads both in one statement on one checkout, with the same row guards as payout_revision. The expired-clock branch still returns before the revision is used, a database failure still precedes every local check, and no deadline, authority, fence, payout or digest semantics change. The in-memory test ledger keeps its clock gate in the same order for the cancellation tests. Co-Authored-By: Claude Fable 5.1 (cherry picked from commit cb15a7ac55a519db6ecc1ffa541a4f2c6de83b76) * 3.x.x: share one in-flight clock+revision read per frontend across issuance boundaries (#275) Concurrent issuance boundaries on a frontend now share a single in-flight clocked_payout_revision read (coordinator/clocked_flight.rs) under a join-after-request rule: a caller joins only a read that started after its own request instant, so a deadline derived from the shared value can only be shorter than a private read would give, never stale; a failed read is shared and the next request reads again, and a driver task runs every flight to completion so a leader cancelled with no joiner cannot park the read on its pool checkout. work_authority_in_epoch takes both values from the flight at every boundary, including each revalidation, where the clock is still evaluated against the recorded expiry and can only shorten the deadline fixed by the first read; the issued record's deadline clock comes from the same flight. The post-save revalidation and the PAYOUT_REVISION_SQL row guards are unchanged. WorkLedger::payout_revision became unused and is removed from that trait; the submit path keeps its own. Also registers PRISM_DATABASE_ACQUIRE_PROBE_IDLE_MS (from the checkout-probe commit) in the native-settings inventory, without which production start-up rejected the Compose-provided setting as unread, and fixes the probe-delivery test's backend reference (&*coordinator). New: five unit tests on the flight rule (join-after-request, shared failure, later request does not join an in-flight read, a joiner completes a read whose leader was cancelled, an abandoned read still completes) and the PostgreSQL-gated tests/fanout_revision_bump.rs (a payout revision bump between build_job and persist_issued_job is refused and writes no row), registered in the gate manifest. Measured with two real frontends, 2,000 sessions, 400k window, admission 128: node tip to last usable notify 1150-1165 -> 996-1005 ms median against a 28 ms A/A delta; publish-to-last-notify 374 -> 245 ms; steady share-ack p99 29-89 -> 6-12 ms; -285 ms at 200 shares/s. Fault injection (backend kills, revision bumps and a primary restart inside the fan-out) served every session on the next attempt with the revision guards firing as before. Co-Authored-By: Claude Fable 5.1 --------- Co-authored-by: Claude Fable 5.1 * 3.x.x: keep the refresh delta path across retargets and multi-page deltas (#275) (#501) * 3.x.x: keep the refresh delta path across retargets and multi-page deltas (#275) RetainedShares::eligible refused any change in scaled network difficulty and any delta over 4,096 sequence slots, so with a per-block retarget every tip's refresh paid the serial full scan (about 100 pages at 400k). The harness hid it: constant bits and a timed tip with no new shares. snapshot_delta::advance now replays the full reader's newest-first saturating fold for the fresh target over the delta above the retained cutoff (ascending pages, up to 262,144 slots), the retained rows, and, for a heavier target, a bounded margin below the retained first row (up to 16 pages; a per-block retarget moves the crossing row by a fraction of a percent of the window). The merged window must be strictly ascending and cross exactly at its first row (the fold reaches zero and the fold without the first row does not); a walk that runs out of history is refused as partial, because the count proof cannot vouch for rows older than the first row. WindowRef is the digest of exactly the merged window, and the leaf, partition and WAL-timeline witness is unchanged: re-read over the merged range without the count, then with it. The reads are READ COMMITTED like the full scan's; consistency comes from immutable rows, the anchored predicate and the anchor's update of the singleton every append updates, and the final count catches what can still appear between statements. Memory at both bounds is stated next to the constants (about 640 MB at 400k, 720 MB at 500k). Measure it: qbit_prism_refresh_window_acquisitions_total{outcome} counts every snapshot's acquisition decision, qbit_prism_refresh_seconds {trigger,acquisition} times every published rebuild, and one "refresh published" line per rebuild carries the row and page counts. The load harness gains opt-in --retarget-bits (different bits on every tip, and a tenth more seeded history below the window) and --background-shares-per-second; per-tip client latency gains min, p90 and a per-frontend breakdown. Defaults are unchanged. Differential tests against the full reader cover retargets in both directions, multi-page deltas and margins, both size bounds, seeded random walks over target and inserts, and landing re-derivation of a delta-assembled window with its superset and subset refused; the independent review's adversarial differential over a legacy-style fixture is adopted, and a coordinator-level test lands a found block, through the coordinator's own submit loop, on a window the delta path built after a retarget. Two real frontends, 2,000 sessions, 400k window, admission 128, every tip retargeting with 133 shares/s, base b61747ec (with #499's fan-out cut) against this branch rebased onto it: the candidate advances by the delta path on every tip (refresh 0.80 s) where the base full-scans (about 1.7 s); the last session holds fresh work 1.03-1.04 s after the tip instead of 1.86-1.92 s (829 and 886 ms in the AB and BA pairs, the two base runs 55 ms apart), and hashrate loss at 60 s drops from 3.0% to 1.6%. On 7b78ea6f the same pair saved 887 and 830 ms against an A/A noise of 3 ms. Backend kills mid-fan-out, revision bumps between build and persist, and primary restarts during a tip all keep every session's work correct with exact receipt reconciliation and landed blocks. Refs #275 Co-Authored-By: Claude Fable 5.1 * 3.x.x: rekey the snapshot delta advance call in the pool-acquire census (#275) The delta path's advance call dropped its network_difficulty argument in 3f75d273, and scripts/prism_pool_acquires.json still keyed the old call text, so the pool-acquire guard reported the new call unclassified and the old key stale. Rekey the entry to the new text with the same classification (one transaction-scoped checkout); no code changes. Refs #275 Co-Authored-By: Claude Fable 5.1 --------- Co-authored-by: Claude Fable 5.1 * 3.x.x: attribute build deadlines to known revision-work waits only (#493) (#500) * 3.x.x: attribute build deadlines to known revision-work waits only (#493) After a lost or cancelled settlement COMMIT reply, the accepting frontend's landing tracking is unknown for that block (`unknown_revision`), yet the block still counted toward `pending_count`. `revision_work_build` derived `Build.pending` from that raw count (or saturation), so every later initial-job deadline, including admission and database waits unrelated to the block, incremented `revision_work_build_timeouts_total` and stamped `degraded` on the unknown wait (#493 point 5). `Build.pending` now comes from the known waits only: open identities with neither a lost settlement reply nor an unsettled orphan verdict, and no lost ordering history, which is exactly the set the pending gauge reports. While only unknown tracking remains, a deadline hit is an ordinary `job_delivery_failures_total` event, the tracking-unknown gauge keeps reporting the unknown state and the pending gauge stays -1; nothing is counted as a revision-work failure or as zero. A known wait beside unknown tracking still attributes its deadline as before. Mining, delivery, fences and the #493/#496 paging rules are unchanged; HELP, inventory and the rule basis state the known-wait rule. Tests: three landing unit tests (lost settlement reply with a recovering first confirmation, known wait beside unknown tracking, unsettled orphan verdict, untrackable identity and lost ordering); one PostgreSQL-gated test that loses a real COMMIT reply through the wire proxy and then holds a real Stratum initial-job build at admission until its deadline; two promtool scenarios for the corrected series (unknown-only with no timeout, unknown beside a known 308 s stall with a real timeout). Co-Authored-By: Claude Fable 5.1 * 3.x.x: attribute a build deadline to a wait that recovered mid-build (#493) Review finding F1 on #500: `Build.pending` is a snapshot at the build's start, so a wait that was unknown then (lost settlement reply) and recovered through a proven first confirmation while the build was in flight was not attributed when the deadline fired, and its later delivery read `published`; the base counted that deadline. `deadline_hit` now also attributes when a known wait accepted before the build is open at the deadline instant; the degraded stamp for open waits accepted before the build is unchanged. Unknown-only tracking at both instants still counts nothing. The reviewer's discriminating test is adopted as a regression test; the inventory prose and metadata state the two attribution instants. Co-Authored-By: Claude Fable 5.1 --------- Co-authored-by: Claude Fable 5.1 * 3.x.x: Close the four needs-triage rows of the deleted-test map with native tests (#460) (#465) * test(prism): pin every window-multiplier literal to the constant The payout window's difficulty multiplier has one value, qbit_prism::PRISM_WINDOW_MULTIPLIER, but several spellings: the dashboard query files and the shipped schema bake it in as a SQL literal, the archive horizon spells it in inline SQL, and two read models render it into JSON. Nothing tied those copies together, so raising the multiplier could leave a dashboard or the audit window view scaling by the old value. The new test enumerates the literals across the server's Rust and SQL sources and the shipped schema, asserts each equals the constant — listing every disagreeing site in one failure rather than stopping at the first — and asserts the discovered site set is exactly the expected one, so a new spelling, or a site the scanner stops recognising, fails with a message telling the author to derive it from the constant or extend the list. A scanner that matches nothing fails too. The frozen 2.x schema fixture and the rollback scripts are outside the scanned roots. Closes one needs-triage row of docs/prism-deleted-test-map.md (#460). Co-Authored-By: Claude Opus 5 * test(prism): assert the divergent landing's alert and its provenance Landing an offered block whose as-issued prior balances are no longer the canonical ones is allowed — the node has or may have the block — but it is reported. The refusal, the marker and the additive balances were covered; the warning itself was not, so it could have been dropped silently. The new test moves the canonical balances the way production moves them: a second block of the same window lands, is confirmed, and its carry rows accrue a below-floor account, so `SUM(gross - onchain)` shifts. Landing the held claim on those moved balances must raise exactly one WARN naming the block, its lifecycle state and both digests, and the same test holds the landing to the rest of its contract — the provenance marker records the audit that was issued, the carry rows keep the as-issued priors rather than the moved ones, and the integrity validator stays clean — so the test cannot pass on the warning alone. A control with no mover asserts the marker is written anyway and no alert is raised. The subscriber is attached to the landing future alone, never installed globally, because this binary runs many tests. Closes one needs-triage row of docs/prism-deleted-test-map.md (#460). Co-Authored-By: Claude Opus 5 * test: qualify the audit reconstruction permit contract `materialize_audit_row` hands its permit to the blocking job rather than holding it in the caller's future, because Tokio keeps running a queued job after the future that awaited it is dropped. Nothing proved that. Three gated tests drive the function directly, each on a runtime with a single blocking thread the test parks, so ordering is a fact and no assertion reads a clock: - a caller dropped after its job is queued does not release the permit; it comes back only once the released thread has run the job; - a second caller waiting on the same one-permit semaphore reaches no statement of its own until that job ends, and both rows then hydrate to the signed body and its declared canonical digest; - EP-ERRORS: a digest mismatch inside the job returns the permit when the job ends and not before, while a missing snapshot row and a cancellation on the first statement return it with the future. Row 216 of the deleted-test map moves from `needs triage` to `partial`: the permit bound is the property now covered, and the bounded audit-body read the retired Python streaming view asserted is not, since `materialize_audit_row` still hydrates the whole share array. Co-Authored-By: Claude Opus 5 * docs(prism): reconcile the deleted-test map for the three closed rows Integration edits on top of the #460 R1, R2 and R3 tests: the multiplier scanner also walks crates/qbit-prism-server/migrations/, so a migration that redefines the window view cannot add a copy unnoticed; row 216 states what the permit tests do not observe (the job holding its permit for its whole duration); the summary counts and the "Needs triage" section follow the rows that left it. Co-Authored-By: Claude Fable 5.1 * Bind Stratum listeners before coordinator startup Move the primary Stratum bind, its "PRISM listening" line and the high difficulty bind above Coordinator::new. Coordinator startup registers the instance with a `starting` row and nothing retracts it when run returns an error, so a restart that lost the bind race to its predecessor left a row behind that `fatal-state clear` refuses on. Every fallible step before Coordinator::new now touches only configuration and sockets. The log line's instance and worker fields come from the Config before it moves into the coordinator; their values are unchanged. The audit HTTP listener still binds after Coordinator::new because it serves the coordinator's ledger through ApiState. Co-Authored-By: Claude Opus 5 * test: qualify the Stratum bind order on restart Add a gated target that runs the real server binary against a fake node and a database of its own, and proves what the bind order buys. A restart whose primary or high difficulty port is still held by its predecessor exits with the bind error and code 1, and qbit_prism_instances stays empty, so nothing is left for `fatal-state clear` to refuse; the next start on a free port serves and stops cleanly. While the node holds the chain check inside Coordinator::new, the announced primary address already accepts a connection and the instance is still unregistered. The high difficulty listener is bound when, and only when, its port is configured. Not covered, and named in the map row: bind retry against a draining predecessor, retry abort on shutdown and zero-retry fail-fast, which native `run` has no subject for, and port release on a shutdown during the node readiness wait, which needs a signal handler that does not exist yet. Co-Authored-By: Claude Opus 5 * docs(prism): reconcile the four #460 rows onto the reconciled map and drop the duplicated partition attach * docs(prism): cite #472's artifact admission tests on the audit-view row instead of the closed #267 * ci: re-run after an unrelated latency-bound qualification flake (incident_2_ack_latency) --------- Co-authored-by: Claude Opus 5 * 3.x.x: capture own blocks found on same-parent superseded payout work (#478) (#504) Each own landing bumps the payout revision twice on the same parent: once when a frontend observes the new tip, and again when the block confirms. Work issued between the two carries the earlier revision, and 3.x.x refused a block found on it `stale-job`, dropping a valid coinbase that 2.x.x would have submitted. At the #275 bar the exposure window W (settlement bump to new-revision work) measured 1.04 s p50 and 1.35 s max, so with a 60 s target spacing a pool with share p loses f = p*W/T of its own blocks: 0.52% to 0.68% at p = 30%, about 2 to 3 coinbases a day. A block-pass proof on the current tip is now captured whatever its payout revision. The block is enqueued and offered, lands with the prior balances its work was issued with, and any divergence from the canonical balances becomes additive carry-forward debt; on-chain amounts and the integrity report stay exact. The share is deferred and credited at confirmation. - Block-only work. A same-parent replacement retires a session's job to JobKind::BlockOnly. submit_share fences it before any grace branch: it never earns share credit, its captured block defers no share, and its share is answered `stale-job` once the enqueue settles. The graveyard drops block-only work at a parent change and caps it per kind, and the reservation lookup walks burial order. - Overpay ceiling. A captured block can pay a balance that an earlier landing already paid down. In every order of confirmations, the debt it creates is at most F, the sum of the positive balances in its own as-issued snapshot. The offer reservation reads the revision FOR SHARE: a block still current is reserved at once; a superseded one is reserved only if F is within PRISM_CAPTURE_OVERPAY_CEILING_BPS of its coinbase value (default 100, range 0 to 10000, 0 turns capture off), and is otherwise abandoned unoffered. An abandoned capture answers `stale-job` (payout_revision, or parent_grace for a lost race), never `ledger-confirmation-failed`. A bound that cannot be computed reserves nothing and abandons nothing. - Records. Migration 020 adds qbit_prism_payout_divergences (the offer decision, F and the ceiling; at confirmation, the debt the block created), qbit_prism_payout_divergence_accounts and qbit_prism_payout_divergence_report(). Confirmation records are written in the transaction that makes the block's rows count, in the settlement and in the chain reconciler. The carry-forward integrity report and self-check gain a payout_divergence line that never fails a check. - Metrics: qbit_prism_capture_offer_decisions_total{decision}, qbit_prism_divergent_landings_total, qbit_prism_divergent_landing_overpay_sats_total and the qbit_prism_carry_forward_debt_sats gauge. - Docs: a ledger-ops section on capture, the bound, sizing the ceiling against the live positive carry float, the records, the accepted cost, known record limitations and rolling upgrade; rejection-cause notes; the metric inventory and migration guide. Tests: new b478_capture_overpay (10 gated), b478_payout_integrity (the payout-integrity reviewer's 3 tests) and b478_stale_revision_block binaries; lib tests for the block-only fence on every credit path, the retained-job cap and lookup, and the bound; the readiness_rpc capture pin; stratum protocol tests for job kinds; the candidate_storm_restart statement pin re-derived at 64. The full gated suite executed all 707 expected gated tests. Real-process runs at the #275 bar with 5 s landing gaps: base lands 8 of 15 own blocks and refuses 7; this change lands 15 of 15, 14 as captures, with no timing regression at equal load. Review: two rounds by a general reviewer and a payout-integrity reviewer, then a fix-delta verification by both, ending with no blockers. The per-finding ledger is in the pull request. Codex was not run. Co-authored-by: Claude Opus 5.5 * 3.x.x: single-pass tee, chunk-parallel counted-share JSON and lean tip probes for the refresh (#275) (#502) * 3.x.x: single-pass tee, chunk-parallel counted-share JSON and lean tip probes for the refresh (#275) The refresh's CPU path serialized the accepted-share array twice (audit prefix and WindowRef digest), the counted-share array twice (commitment leaf and audit suffix), folded it with two million string clones on one thread and hashed the slice digest with millions of tiny SHA updates; the two pipeline lanes were balanced, so neither could shrink alone. Builder-only, byte-identical cuts (verifiers stay serial and untouched): - qbit_prism::parallel: an ordered chunk pipeline on one process-wide worker pool (PRISM_REFRESH_BUILD_THREADS; default three quarters of the cores, at most four, the measured knee; 0 runs the builder serially), write_array/write_array_frames (exactly serde_json::to_writer of the slice, every element still written by serde_json, chunks concatenated in order) and StructSplice, a serializer that keeps a struct's derived field order and skip rules while one field's value bytes come from a chunk-parallel writer. PrismRewardManifest's derive and AuditBundleRef::serialize_suffix remain the only definitions of the audit bytes; u128 fields reach serde_json's own serialize_u128 as before. - CanonicalAuditHashPrefix::new_with_share_digest serializes the accepted shares once and feeds both SHA states, the WindowRef digest on its own thread over the same frames; finish_with writes the suffix with the reward manifest's shares in ordered chunks. - The commitment leaf and, with CTV, the fan-out precommitments digest the reward manifest once through the same chunked writer. - compute_prism_window_parallel decides the counted difficulties serially, builds the CountedShare chunks on the pool with their slice-digest bytes and entitlement partial sums, and streams the digest on its own thread. Eligibility, ordering and the per-share digest encoding are shared with the serial fold. - The compact-record capture no longer waits for the counted-share drop; it runs on the blocking pool under the same build admission. Lean tip probes: the ledger probe before the reuse check is skipped when the template fingerprint already differs, the probe under the build permit when the cached window's revision is below the one the chain-observation write returned (every revision write is an increment), and the landing metric takes the anchor transaction's revision. The same-template path, the in-transaction save fence and the lock-loop re-prove are unchanged. AUDIT_BUILDER_VERSION is unchanged. Property tests build randomized windows through both paths at six thread/chunk geometries and compare every byte and digest; golden vectors for a fixed window were recorded from the serial builders and re-derived on the base tree. At the #275 bar (two frontends, 2,000 sessions, 400k window, background shares, eight tips per run, balanced A/B on the merged 3.x.x with the fan-out work) the refresh drops from 736-747 ms to 397-403 ms and tip-to-last-notify from 955-973 ms to 588-624 ms, with the fan-out tail and share-ack latency unchanged; RSS per frontend after the tips rises by 100-240 MiB (the pool's allocator arenas; it does not grow with further tips) and peak RSS by about 70 MiB. Refs #275 Co-Authored-By: Claude Fable 5.1 * 3.x.x: size the pool-fixed test by the running pool and pin the probe gating (#275) The builder-threads test asserted that the detected parallelism is never serial, which only holds on hosts whose pool has at least two threads: builder_threads() is three quarters of the cores clamped to 1..=4, so a two-core runner starts one thread and Parallelism::new(1, ..) is the serial path. The test now asserts the relationship: the detected worker count is the running pool's (at most eight) and it is serial exactly when that count is one. Reproduced with the lib tests pinned to one and two cores; the fixed test passes there and at the full core set. No other test in this change reads the host's core count. The coordinator's refresh probes were only covered indirectly. A new test counts them alone (the memory ledger keeps that counter next to the state reads that lease proofs and reuse fences also make): a same-template poll probes once, a new tip whose transition bumped the revision probes not at all and reads one snapshot, and a tip a peer already recorded probes once under admission and reuses the cached window without a snapshot read. Refs #275 Co-Authored-By: Claude Fable 5.1 --------- Co-authored-by: Claude Fable 5.1 * 3.x.x: make the probe-window dead-connection test deterministic (#507) ledger::connect::acquire::probe_tests::a_dead_idle_connection_inside_the_probe_window_fails_one_statement_then_recovers failed in CI with "a statement on the terminated backend N returned pid M". The test terminates the pool's one backend right after a statement, but sqlx returns a dropped PoolConnection from a spawned task that pings the server before making the connection idle. When the backend is terminated while that ping is in flight, the ping fails, sqlx discards the connection on the spot, and the checkout under test opens a fresh connection instead of receiving the dead socket, so the statement succeeds on a new pid. On a two-core host with the runtime and the server competing for CPU this happened in 21 of 100 runs. Both probe tests now wait, bounded, until the pool reports the connection idle before terminating its backend, and assert it is still idle when the checkout starts, so the dead socket is deterministically the one handed out. The sibling threshold-zero test could pass for the wrong reason in the same race (a connection discarded at release, not by the probe); it gets the same synchronization. No assertion was weakened; the acquire code is untouched. Refs #499 Co-authored-by: Claude Fable 5.1 * 3.x.x: keep a lost race out of the revision-work paging age and dwell the unknown warning (#493) (#503) * 3.x.x: keep a lost race out of the revision-work paging age and dwell the unknown warning (#493) Points 1 and 7 of #493. Telemetry and alert semantics only: no mining, submit, landing, payout or deployment change. Point 1. A null `submitblock` reply opened a known delivery wait whether or not the node held the block on its active chain, so a lost tip race grew `accepted_block_revision_work_pending_seconds` until its orphan proof (`PRISM_CANDIDATE_ORPHAN_CONFIRMATIONS` competitor confirmations plus the reconciliation backoff) and paged the #496-gated critical whenever that took longer than 307 s. The offering frontend now records such an acceptance as unlanded: outside the known pending age and build-deadline attribution, on its own gauge `accepted_block_unlanded_seconds` (age of the oldest one; zero when none; -1 under bounded-tracking saturation, never a silent zero). The post-offer landing observation, a reconcile proof or a settlement observation lands it on its original acceptance clock, so the histogram still measures from the definitive acceptance; a committed proven orphan closes it without a sample. A tombstone never reopens and a landed block is never downgraded. `PrismAcceptedBlockUnlanded` warns (never pages) at 30 minutes for 3 m. The same lost race also paged `PrismBlockCandidateOldestPendingCritical` (60 s, 3 m) from the database collector on every coordinator, because a `reconciliation` row is unfinished. The collector now reads, in the same snapshot, `block_candidate_oldest_unacknowledged_seconds`: the oldest unfinished row without a definitive node reply (`pending`, `offer_reserved`, and offered rows whose one `submitblock` outcome is `unknown`). The paging rule reads that gauge, so a found block not reaching the node pages exactly as before and an unknown outcome keeps paging because receipt is unproven, while a node-accepted lost race, a rejected block or a failing landing age only the all-unfinished gauge: the 15 s warning still reports them and the new non-paging `PrismBlockCandidateStuck` warns at 30 minutes for 3 m. All three candidate gauges read -1 together when the collector fails. Point 7. `PrismAcceptedRevisionWorkPending` had `for: 0s` and included the tracking-unknown gauge and the -1 age, so any failed template refresh (one 2 s poll interval of unknown) warned. The unknown terms move to `PrismAcceptedRevisionWorkTrackingUnknown` with a two-minute dwell and no-data Alerting; the pending warning keeps its documented no-dwell budget, missing-series and scrape-down terms unchanged. Review fixes (independent review: one blocker, two major, three minor). A definitive node rejection keeps paging through the unacknowledged age; only a side-chain reply (a lost race to a first-arriving block) is excluded. A row adopted on the node's own active-chain evidence is not unacknowledged. A won block whose audit landing keeps failing is reported by `block_candidate_oldest_landing_failed_seconds`, warned at 60 s for 3 m and paged at 307 s for 1 m, the floor and dwell the revision-work critical applies when the landing fails after the frontend's observation, so the class is never silent whichever step refused. The candidate paging rule falls back to the all-unfinished age on a target that does not export the new series, so a mixed-version rollout never loses the page. The -1 HELP wording and the backoff prose are corrected. The reason and reply vocabulary is shared as constants between the coordinator and the collector. Tests: three landing unit tests (lost race never a known wait, landing on the acceptance clock through every active-chain path, saturation as -1); four PostgreSQL-gated tests in `landing_metrics.rs` (a real lost race through the fake node, retried at five and six competitor confirmations, with an active block as the positive control; an offer without a definitive reply beside a node-accepted lost race in the collector's census; a `bad-cb-payee` rejection beside an `inconclusive` reply; a landing transaction aborted through the wire proxy and landed on the retry); the census, contract and inventory updates (56 families); 26 promtool scenarios (one failed refresh under and after the dwell, persistent unknown under and after the dwell, saturation beside a known age, the unchanged budget warning, a lost race that proves its orphan inside the threshold, a stuck unlanded row under and after the dwell, saturated unlanded tracking, a lost race that never pages either critical, a pre-offer or unknown-outcome row that still pages, a rejected block that pages, a stuck reconciliation row that warns, a failing landing under and past both dwells and one landed on its first retry, the mixed-version fallback, collector failure); the deployment patch test asserts the four new warnings dwell and never page, that the landing-failed critical is a gated one-minute paging rule, and that the candidate paging rule reads the unacknowledged age with the fallback. Co-Authored-By: Claude Fable 5.1 * 3.x.x: keep the gated-test list sorted (#493) The expected-list checker requires byte order; the new `candidate_collector_keeps_an_acknowledged_lost_race_out_of_the_paging_age` entry was inserted before the `cancelled_*` lines. Moved after them. Co-Authored-By: Claude Fable 5.1 * 3.x.x: read the landing-failed age from the missing pool-block row, not the error text (#493) Deep-review findings on #503, all verified against the code. The landing-failed gauge was discriminated only by the `last_error` prefix that `settle_offered_inner` writes for a refused landing. Every other writer of the same reconciliation row (a node or database error inside the landing, which surfaces as "post-offer processing failed", a refused operator recovery, a failed lease renewal) overwrote it, so the age read zero and both landing-failed alerts resolved while nothing had landed, and a landing that kept failing on a hard error paged nowhere. The collector now also counts every reconciliation row with no `qbit_pool_blocks` row: the landing transaction is that row's only writer, so its absence is the durable fact, and the prefix only adds a landed row that no longer authenticates. The age is measured from the offer reservation, the start of the post-offer lifecycle and the instant the frontend's acceptance clock starts, so pre-offer retry backoff never counts toward the 307 s critical. Structure: `Acceptance::known()` is the one predicate behind `known_waits` and `resolve`; `accept_block` takes an `Observation` (unlanded, active, confirmed) instead of two bools; `adoption_evidence` is the single writer of the adoption reply; the side-chain reply list is bound as `text[]`; the census query is one legible statement with named predicates; the candidate paging fallback is `unacknowledged or on(...) oldest_pending` (the `unless` was redundant); the promtool script checks every rule as an alert rule; the metadata file keeps its hand-authored formatting. Tests: the landing-failure gated test now also fails a retry transiently (a tip change under the observation overwrites `last_error`) and asserts the age holds and never predates the offer. Co-Authored-By: Claude Fable 5.1 * 3.x.x: prove the errored post-offer path is reported and keep deadlines off unlanded blocks (#493) Workstream A review of #503 (three findings). M1: an accepted block whose post-offer processing keeps erroring before the landing (a database error in the landed-audit check or the rebuild, or a node or readiness error in the candidate observation) carries the "post-offer processing failed" reason. Every such error returns before the landing transaction, which is the only writer of the pool-block row, so the previous commit's no-pool-block predicate already reports it. A gated test now proves it on real rows: the observation fails on every attempt, the landing-failed age is positive, keeps growing across a failing retry, never predates the offer reservation, nothing is unacknowledged and the frontend identity stays unlanded; restoring the node lands the audit and the age returns to zero. L1: a build deadline attributed through another known wait no longer marks an unlanded block degraded: it has no revision work yet, so a deadline that fired before it landed cannot have delayed its delivery. Unit test added. L2: the HELP, metadata and contract prose now state the one deliberate exception to "an unlanded acceptance is a known state": while the terminal probe that carries peer orphan evidence fails, an unlanded-only frontend reads tracking unknown and a -1 pending age, and the unlanded gauge keeps the age. Co-Authored-By: Claude Fable 5.1 --------- Co-authored-by: Claude Fable 5.1 * 3.x.x: Export node peers, initial block download, observation age and hashrate-rollup lag (#467) (#470) * 3.x.x: export node and hashrate-rollup gauges as unknown, never stale The remaining P2 of #278 adds four label-less gauges to the native registry. Node observation gets one site: readiness::chain_info_with_metrics records only what the attempt already fetched, so it adds no RPC call. A check that short-circuits before getnetworkinfo - regtest, initial block download, or blocks disagreeing with headers - leaves the peer count unknown at -1 rather than keeping an earlier reading that would advertise an unconfirmed peer floor; an answered zero stays a real zero. The sync flag is recorded whenever getblockchaininfo answers with the boolean, including the 1 that then makes readiness fail. The observation age is stamped on that answered call rather than on a fully successful check, so an unready-but-reachable node keeps a small age while an unreachable node's age grows. Concurrent refresh, block-wait, candidate and broadcaster observations are ordered by the instant each attempt began, and the publish happens under the same guard that authorised it, so a slower older attempt cannot roll back a newer one. The rollup watermark is a share sequence, not a time, so the lag is defined as the monotonic time since this frontend's last pass that advanced the watermark within its batch bound. A lost race or a full batch leaves the stamp alone, so alerts must take the minimum across instances. With the rollup disabled the family is declared without any sample, because a frontend that never runs it has no lag to report. All four are live, so every scrape recomputes them from one registry read. Unknown is -1, never NaN and never a stale value. The setters reuse reserved closed keys and allocate nothing. Co-Authored-By: Claude Opus 5 * 3.x.x: guard the answered peer count and keep the latest node answer Moving the peer recording below the min_peers check passed every named suite, so the invariant that a count below the floor is still an answered count was unguarded. The new test drives the real entry point with a node answering 0 and 1 peers against a floor of 2: readiness refuses it, yet the gauge reads the answered count rather than -1, so the exposition cannot hide the very shortfall that closed admission, and an answered zero stays a real zero. The observation age also discarded a later answer whenever its attempt had started earlier than the one already published. The age asks when the node last answered, not which attempt owns the published peer and sync pair, so it now keeps the latest answer any attempt received; the ordering guard still governs that pair alone. This adds no RPC call and leaves the setters allocation-free. Co-Authored-By: Claude Opus 5 * fix(metrics): take the registry before the node observation state so a scrape cannot deadlock an observation --------- Co-authored-by: Claude Opus 5 * 3.x.x: keep a ledger future cancelled inside BEGIN from leaving its transaction on the pooled connection (#482) (#515) Every D1 run of #447 logged PostgreSQL's "there is already a transaction in progress" on the share-append path. Reproduced deterministically: SQLx 0.8.6 counts a transaction only once the reply to BEGIN is read, and its cancellation guard rolls back nothing before that, so a persist task aborted at the share-commit deadline (or dropped by a miner disconnect), the metrics collector's deadline, or the subscribe timeout around session allocation, dropped between the BEGIN write and its reply, returned the connection to the pool with the server still in a transaction. The release ping consumed the reply, the next checkout's BEGIN landed inside the leftover, and the same backend pid served it. No advisory lock can be held across it (every later await is at depth 1 and queues ROLLBACK, flushed by the pool return), but a statement run on a directly acquired connection inside the leftover would be reported durable without being so. Guard, one per form of BEGIN: - `ledger::shielded_begin` runs the BEGIN round trip in a task of its own and joins it. `Ledger::begin`, the startup transaction and the metrics collector use it; a cancelled caller detaches, the task reads the reply and the transaction it drops queues the ROLLBACK the pool return flushes. - `AppendConnection::begin` records completion; when BEGIN did not complete, the guard's drop detaches the connection from the pool synchronously and closes it on the captured runtime, so the pool opens a replacement and can never offer that connection again, even at shutdown. The successful path is unchanged: an append is the same ten statements on one connection, verified on the wire. Tests: `ledger_cancelled_begin` holds one BEGIN reply at the execution proxy (new `pause_statement`), cancels the future there by abort and by drop, and asserts from `pg_stat_activity`, `pg_locks`, the proxy's execution log and a warning-capturing subscriber that no backend is left idle in transaction, no ORDER_LOCK or SETTLEMENT_LOCK is held, and the next transaction starts without the warning. Against the base both tests fail on the idle-in- transaction backend and the BEGIN, BEGIN wire. Docs: `docs/prism-ledger-ops.md` records the trigger paths, the argument, the guard and the rejected `after_release` alternative with its cost. Co-authored-by: Claude Fable 5.1 * 3.x.x: refuse an unmeasurable memory floor, find pg_stat_statements as .dylib and surface why a managed cluster did not start (#485) (#514) qbit-prism-load on macOS (#447) ran with three silent gaps: - --min-mem-available-mib read MemAvailable from /proc/meminfo and did nothing when it could not, so the run proceeded unguarded. A floor the host cannot measure is now refused at entry, naming the flag and --min-mem-available-mib 0 for a run without one (EP-CONFIG). A phase's side report counts the checks at which MemAvailable could not be read mid-run (mem_available_unread_checks), so a floor that lapsed does not read as having held (EP-OBSERVABILITY). - pg_stat_statements was preloaded only when pkglibdir held pg_stat_statements.so; Homebrew ships pg_stat_statements.dylib. Either name is accepted. - The cluster root under macOS's default TMPDIR put PostgreSQL's socket at 107 bytes, over the 103 it accepts, and the harness exited with "pg_ctl failed" and no reason, because the log went with the root. The root's random part is 12 hex digits instead of 32, which fits that TMPDIR; a base too deep for the socket is refused before anything is created, naming TMPDIR and the platform limit; and a server that does not start reports the last lines of its own log beside pg_ctl's output. The README records macOS as supported for smoke runs, with the memory floor, peak-RSS and TMPDIR caveats. A new gated test starts a cluster PostgreSQL refuses and checks the error carries the server's reason; its id is added to test/prism-gated-tests.txt. Co-authored-by: Claude Opus 5.5 * 3.x.x: attribute the current server's stale-job and unknown-job rejections to dense-cadence landings (#480) (#513) * 3.x.x: attribute the current server's stale-job and unknown-job rejections to dense-cadence landings (#480) The dense-cadence section attributed a rejection to a landing only for `new tip work is pending` or `new payout work is pending`, which the server no longer sends on the submit path. Every #447 dense run at D1/400k therefore attributed none of its 16,461-21,504 stale-job and 1,512-2,997 unknown-job rejections, all `stale job`, and proposed a p99 of 0.0 rejections per landing. A landing now owns, inside its span, stale-job and unknown-job with `stale job` as well as the two older messages. Backend refusals (backend-rpc-unavailable, ledger-confirmation-failed, ledger-outcome-unknown, internal-error), a fee-floor stale-job, pool-closed and harness-bug classes are tallied by reason_id beside the cost, never inside it. A stale-job or unknown-job message the harness does not know, or an unknown reason, is unrecognised: the span's combined window, lost shares and rejected-before-new-revision-work become null with a reason, and the #291 budget is null with its reason rather than a percentile over the rest. Every budget number now carries its own unavailable reason, and a same_edge_as_issue_458 block reports the acceptance -> new-revision-work count and time, the edge qbit_prism_accepted_block_to_revision_work_seconds measures. The README's dense-cadence section names the messages and classes that count. Co-Authored-By: Claude Opus 5.5 * 3.x.x: classify the unknown-job budget refusal as expected, not unrecognised (#480) stratum.rs refuses a submit with code 20, `too many unknown job submissions` and no reason_id once a session has spent its unknown-job budget. The classifier sent every reason-less message other than `too many connections` to Unknown, so dense-cadence attribution made it Unrecognised, and one such refusal inside a landing's span nulled that span's combined window, lost shares and rejected-before-new-revision-work, and the #291 budget. D1/400k dense runs answer 1,500-3,000 unknown-job submits per run, so the refusal can erase exactly the numbers #480 restores. It is now Expected, and so NotOwned for a landing: a rate limit on job lookups, not retired work. It is matched only on that exact shape (code 20, no reason_id, that message) and tallied beside the cost under the existing `(no reason_id)` marker. The classifier-table and span tests cover it, and the README table and the section's counted_classes text name it. Co-Authored-By: Claude Opus 5.5 * 3.x.x: publish the #458-comparable acceptance time and keep unknown durability checks null (#480 review) The same_edge_as_issue_458 block published the p99 of each landing and frontend's latest session's acceptance -> new-revision-work time. The server's accepted_block_to_revision_work_seconds ends at the earliest delivery on the frontend (Landing::resolve, min_by_key), so the comparable figure is the earliest session's. acceptance_to_new_revision_work_p99_millis now takes the per-table min, and the per-table max is published beside it as acceptance_to_new_revision_work_with_fanout_p99_millis. Both are notify timings, so an unrecognised rejection no longer nulls them; only a run with no landing does. The clock_note and the README name where the server's start really is: the end of observe_candidate on the submitting frontend, and reconcile's tip discovery on the others. lost_valid_work.shares_found_in_postgres is null with a reason when shares is null, and a table's lost_valid_shares_found_in_postgres is null when its lost_valid_shares is. The checks that were made are published apart, including a new lookup of the unrecognised rejections' shares. A test covers #447's exact shape: landings whose spans held no owned rejection report a measured 0 count and a non-null budget, and a null window with its reason. The run-wide rejection table's rebuild_pending flag now uses the dense section's ownership test. The README states that attribution is by time, not lineage, and lists all seven count distributions. Co-Authored-By: Claude Opus 5.5 --------- Co-authored-by: Claude Opus 5.5 * 3.x.x: #502 follow-ups: pool drain state in an Arc, prefix-only rebuild, trigger precedence, retained memory (#509) (#512) * 3.x.x: #502 follow-ups: pool drain state in an Arc, prefix-only rebuild, trigger precedence, retained memory (#509) F1, soundness. ordered_chunks kept its drain state (the ready map, the error, the outstanding count and their condvar) on the caller's frame and lent it to every job through a lifetime-erasing transmute. A job's last actions are decrementing outstanding and unlocking the mutex, and the drain waiter can see zero and return between the unlock's swap and its futex wake, so that wake could run on a freed frame. The state now lives in an Arc cloned into each job; only produce stays borrowed, and a job's last use of it is inside catch_unwind, before the decrement. The twelve parallel unit tests pass under Miri. F3, hot path. The reused-window rebuild (prepare_refresh_body_with) called new_with_share_digest and discarded the digest, spawning a hasher thread and hashing the whole share array a second time on every same-tip template churn. CanonicalAuditHashPrefix::new_with hashes the prefix alone; a thread-local probe counts the digests computed, and tests in both crates pin that the rebuild computes none and that its hashes are the pipelined refresh's. F4, observability. A new template on the same tip runs no ledger probe at entry since #502, so the refresh trigger label was always template there even when the revision, balances, reanchor, shares or fee changed. The label is now named on that path from the window the rebuild used, by the same precedence (classify_refresh) as the probed path; a coordinator test drives a same-tip template change with a revision bump and reads revision, then template alone, from the metric. F2, resources. docs/prism-storage-sizing.md records the per-frontend retained-memory change with #502's numbers and the knobs that bound it (PRISM_REFRESH_BUILD_THREADS, MALLOC_ARENA_MAX) and the build slot no longer bounding live counted-share bodies exactly. No code change. No artifact byte changes: the byte-identity suites, golden pins, frozen corpus and audit_cli goldens are untouched and pass. Closes #509 Co-Authored-By: Claude Fable 5.1 * 3.x.x: drop the jobs' untaken chunks on the calling thread (#509 review) With the drain state in an Arc, a chunk the consumer never took (a producer or consumer error, or a consumer panic unwinding through the drain) stayed in the shared state and was dropped by whichever thread released the last Arc clone, which can be a pool thread after ordered_chunks has returned. C is generic and may borrow the caller's frame: the counted-share fold's chunks are keyed by &str into a caller local. No current chunk type dereferences its borrow on drop, but the SAFETY invariant that justifies the transmute did not hold for them. Drain::settle now waits for the jobs and takes the first panic, the first error and the untaken chunks under the lock, on the calling thread, on the return path (Drain::finish) and on the unwind path (Drop) alike, so a clone of the shared state that outlives the call owns only empty containers. Found independently by the adversarial and code-review lanes of the review. Co-Authored-By: Claude Fable 5.1 --------- Co-authored-by: Claude Fable 5.1 * 3.x.x: keep the share append's ORDER_LOCK hold free of partition re-planning and job-cohort row waits (#479) (#517) * 3.x.x: keep the share append's ORDER_LOCK hold free of partition re-planning and job-cohort row waits (#479) The append's share_id probe on the partitioned ledger now reads its share_seq bounds in the statement the append already runs first (the inlined bodies of migration 016's qbit_prism_share_probe_floor() and qbit_prism_share_next_seq(), under the same ORDER_LOCK) and binds them, and the append transaction's BEGIN carries SET LOCAL plan_cache_mode = force_generic_plan: the probe is planned once per connection and pruned at executor startup to the leaves between the bounds, instead of a per-leaf filter over every attached partition (or, with bound values under the default plan_cache_mode, a custom plan re-planned on every share inside the lock). The candidate set is unchanged: every row that can exist is below the next sequence value, and the credited-header, rejected-sequence and legacy-range fallbacks are untouched, so duplicates below the floor and after a detach are answered exactly as before. Job-persistence fences on the cluster row (batched and single compact saves, prepared blobs) take FOR KEY SHARE instead of FOR SHARE, so the append's non-key ledger_clock_ms UPDATE no longer waits, while holding ORDER_LOCK, for a cohort of up to 64 job saves to commit. Every authority writer of payout_revision, config_fingerprint or fatal_error takes the row FOR UPDATE first (lock_cluster_authority, added to the block-landing revision bump, the disconnected-block fatal write and the fanout halt; the others already did), which KEY SHARE conflicts with, as does blob GC's exclusive fence: a bump, a configuration change, a halt or a blob sweep still waits for an in-flight cohort to commit. Tests: the probe's generic plan removes the leaves above the sequence at startup and the inlined bounds equal the functions in both catalog states; a duplicate below the floor is still idempotent across a partition boundary and refused after its partition leaves the ledger; a KEY SHARE fence lets the share clock through and blocks an authority writer until commit. Gate manifest: three ids added. Measured with #271's 20k argument list on a quiet host, three interleaved repeats against 5d0042f6: steady_state 500 shares/s sustained in every repeat with no shortfall (head before: 421-475 with a shortfall in all three), ACK p50 2.4-2.5 ms (before: 4.0-4.8 s; 5d0042f6: 2.8 ms-1.3 s), burst 523-565 against 462-586, and the lock holder no longer CPU-active in the probe nor blocked on the cluster row. Co-Authored-By: Claude Opus 5.5 * 3.x.x: pin the cluster-row authority contract behind the KEY SHARE job fences and bring the fence tests to it (#479) Job persistence fences authority with FOR KEY SHARE on the cluster row so that the share append's ledger_clock_ms UPDATE no longer waits for a job cohort. That fence holds only because every authority writer takes the row FOR UPDATE before it writes: a bare UPDATE of payout_revision, config_fingerprint or fatal_error takes FOR NO KEY UPDATE and passes it. Every production writer already did; nothing enforced it, and seven gated fence tests still modelled the writer as a bare UPDATE. - Contract tests: with a cohort's fence held, each production authority writer, driven through its API, must block on it and then complete: configure, tip observation, the settlement, reconcile and orphan revision bumps, both halts, fatal-state clear, the policy and the signing transitions. A source scan fails on any bare authority UPDATE. Removing lock_cluster_authority fails both. - The seven fence tests take the row FOR UPDATE before their UPDATE and keep every wait assertion; the b275 hold, which was the clock UPDATE itself, is now a real authority write. Converses: a held clock UPDATE does not delay persistence, and a held 64-child cohort does not hold up Ledger::append. - The fanout halt locks the row before it validates the revision. The settlement paths keep the lock at the bump: they validate under SETTLEMENT_LOCK, which every revision writer holds, and locking at the check would make every cohort wait for the whole landing. - The execution proxy records every completion tag of a batch and gains begins_transaction(); append_admission's no-BEGIN guards and ledger_cancelled_begin use it, and candidate_storm_restart opens a transaction run on the first tag, not the last. - The probe tests run the ledger sessions' own prepared statements and require a generic plan on every append; candidate_storm_restart's statement pin is 65 (the bump's authority lock). - Archive attach refuses a partition holding a row at or above the next share sequence, the premise of the probe's ceiling, after every existing attach refusal so that each keeps its meaning. - Docs: the cluster-row lock matrix, the bare authority UPDATE ban, the drain-before-upgrade rule for builds before this change, MultiXact use with a D1 measurement to-do, the append's eleven statements, the plan cache note, the runbook's fingerprint restore under the row lock, and stale FOR SHARE comments. Co-Authored-By: Claude Opus 5.5 * 3.x.x: #517 review follow-ups: the attach ceiling check reads the re-streamed summary, and the cohort-fence helper aborts its writer (#479) - Archive attach: the ceiling check reuses `summary.last_share_seq`, which was re-streamed from the restored rows, instead of a second max(share_seq) scan, and its comment now says so. A fresh import already refused the same condition, so the check matters for a re-attach. It still runs after every existing refusal. - `waits_for_the_cohort_fence` aborts the writer task if the helper leaves early (timeout or failed probe), so no writer outlives the test's database. - The append probe's comment notes that the settlement's deferred-share credit runs it inside the settlement transaction under the default plan mode, once per landed block. Co-Authored-By: Claude Opus 5.5 --------- Co-authored-by: Claude Opus 5.5 * 3.x.x: show the live positive float on the payout_divergence report line (#506 item 3) (#520) * 3.x.x: show the live positive float on the payout_divergence report line (#506 item 3) The payout_divergence line showed the capture debt but not the live positive carry float, so an operator could not check the capture ceiling against the float from the report alone. - The line gains positive_float_sats: the sum of max(0, balance) over the current carry-forward balances, one per account. That is the F that work issued against those balances would be held to if captured. It is appended to qbit_prism_payout_divergence_report() in the same statement, so it shares the debt fields' snapshot. No migration, no new lock, no change to capture decisions. - One helper, ledger::payout_divergence_line, serves the line to /audit/carry-forward-integrity, self-check and the #478 tests. - The line shows no ceiling headroom: the ceiling depends on the next block's coinbase value and each frontend's setting, and the database holds neither. The ops doc gives the comparison and the smallest setting that admits the float. - prism-ledger-ops.md: how to read the line, and sizing the ceiling against the live float, with margin because a capture is held to the float when its work was issued. - Tests: a new gated test pins the float on known balances (per-account net, an indebted account adds nothing, equal to overpay_bound of the same balances, "0" on an empty pool); api_database pins the served field. The pool-acquire inventory classifies the helper and its callers. Refs #506 Co-Authored-By: Claude Opus 5.5 * 3.x.x: #506 float line review fixes: capture-off floor, partition-crossing and end-to-end pins (#520) - Docs: the smallest ceiling setting that admits the float is max(1, ceil(float x 10000 / coinbase)) bps, since 0 turns capture off; above 10000 no setting admits it. - The float reads qbit_current_owed_balances(), so it equals /owed-balances by construction (same value as before). - The float test puts each account's second row under another payout order key, so its net crosses carry partitions, and two_blocks pins the float read before A lands to B's recorded overpay bound. - payout_divergence_line takes the pool; the pool-acquire reason says so. - seed_block documents the columns it does not keep consistent. Refs #506 Co-Authored-By: Claude Opus 5.5 --------- Co-authored-by: Claude Opus 5.5 * 3.x.x: report a won block whose landing hangs in offered through the landing-failed age (#510) (#519) A block the node accepted stays in state 'offered' until its post-offer landing runs, and that landing waits for a build permit with no deadline while the heartbeat keeps the claim. Such a row was unacknowledged age 0, landing-failed age 0 and revision-work-pending age 0, so a landing held by partial build contention paged nowhere; only the 30-minute warnings would fire. The landing-failed filter now reads state IN ('offered','reconciliation') with the same conditions (no pool-block row, or a landing-refused last error), still measured from the offer reservation. A healthy offered row lands within seconds: the offer never waits for build capacity, the permit queue is FIFO behind short job builds, and each rebuild is bounded by the 60 s deadline, a timed-out one moving the row to reconciliation where it already counted. The warning needs the age at 60 s for 3 m (240 s unlanded), so no threshold changes. A lost race and a rejection land their audit before the chain verdict and keep reading 0 once landed. Tests: a gated landing_metrics test holds every build permit after an accepted offer and asserts the age is above 0 and growing while the row is offered, and 0 once it lands (it fails on the old filter); the lost race test now also asserts a landed lost race reads 0. Two alert scenarios pin the hung offered row: pending under the paging dwell at 366 s and paging at 307 s + 1 m, with unacknowledged 0 and no revision work pending. The gauge HELP, metric metadata, native metrics doc and both landing-failed rule descriptions are regenerated. Refs #510 Co-authored-by: Claude Opus 5.5 * 3.x.x: live regtest: a qbitd stop or crash around submitblock offers each block once and lands or reconciles it (#521 scenario 4) (#523) * 3.x.x: live regtest: a qbitd stop or crash around submitblock offers each block once and lands or reconciles it (#521 scenario 4) Four gated live_regtest cases put a test-owned JSON-RPC proxy between both PRISM servers and the real regtest qbitd. The proxy forwards every call unchanged, records every submitblock, and holds one: before the request is forwarded, or after the node's accepting reply was read. With the call held and all proxy traffic paused, the test SIGSTOPs the node (resumed with SIGCONT) or SIGKILLs it (restarted on the same data directory and RPC port), then releases the call. A Stratum client in the test mines one block at a time, so each fault has exactly one block in flight. Every case asserts that /healthz and /metrics (health_state, node_observation_age_seconds) on both servers show the node unavailable and then recovered; that each block is offered exactly once; that the held block's offer outcome is recorded (unknown or accepted); that no row is left pending, offer_reserved or offered; that the accepted-unlanded alarm is raised only for an accepted block and clears; that a new block mined through the other server after recovery lands; and that the carry integrity report is clean. - stopped before submitblock: the node processes the buffered request when resumed and the block lands. - stopped after accepting: the post-offer observation fails, the row waits in reconciliation, and the block lands after resume. - killed before submitblock: the node never had the block; the row stays in reconciliation, is never offered again, and settles as a proven orphan once a competing block has six confirmations. This pins the known defect #522 (the restarted node would accept the block on the same tip) rather than desired behaviour; the #522 fix flips the case. - killed after accepting: an unclean qbitd exit loses the accepted block it had not flushed (the test flushes the chain before the fault, so only the held block is lost), and the row reconciles to a proven orphan as above. A fifth case, killed after accepting and restarted with -reindex (which recovers the block, so it lands), is #[ignore] for a nightly run and is not in the gated list. Mutants each fail at least one case: a blind submitblock retry on a transport error (both before-submitblock cases, "offered exactly once"), readiness that ignores the node poll age (all four), reconciliation rows that never observe the chain again (stopped after accepting), and no proven-orphan settlement (both killed cases). Refs #521 Co-Authored-By: Claude Opus 5.5 * 3.x.x: live node-outage proxy: close idle server connections with the node's, and release held replies inside the submitblock deadline (#521) Review fixes to the scenario-4 test fixture: - The proxy only read from the node after forwarding a request, so a node connection that closed while idle (qbitd's idle timeout, or the kill) left the server's side open; the server could reuse it and have a submitblock recorded as unknown. The relay now closes the server's connection as soon as the node's closes, as a direct connection would. - The Stratum client no longer polls the outbox after submitting, so a held accepting reply is released well inside the server's 1-second submitblock deadline on a slow runner. - The -reindex variant is described as opt-in (run with --ignored, for the nightly job #521 adds) rather than as already nightly, and the proxy's Content-Length-only framing is documented. Refs #521 Co-Authored-By: Claude Opus 5.5 --------- Co-authored-by: Claude Opus 5.5 * 3.x.x: follow weighted multi-address shares to spent outputs on a live regtest node (#521 scenario 7) (#524) Add a gated live_regtest case in which 40 payout addresses, with a seeded whale, Zipf tail, near-floor and near-zero weights, mine through four found blocks. They submit real Stratum shares from 59 sessions on both servers, at worker difficulties spanning about three orders of magnitude. The node is restarted with -legacyretarget and ramped 1024x under a mock clock, because on regtest's limit a share can be at most twice as easy as a block. The case recomputes each block's payout from the ledger's accepted shares and the chain alone: - the PPLNS window, including the partly counted oldest share; - the gross split, floor, carry-forward and pool-fee dust sweep; - the direct/fanout partition and the fanout fee. It checks that recomputation against the coinbase outputs, every carry-forward row and the final balances. After maturity, it checks each fanout's net outputs, every payee address's receipts in the payees' wallet, a confirmed wallet spend of direct and fanout outputs, and the carry-forward integrity report. A 130-recipient nightly variant is #[ignore] and opt-in. The fixture gains an additive server_env hook, applied after every other server setting. Refs #521 Refs #474 Co-authored-by: Claude Opus 5.5 * 3.x.x: offer again a won block whose submitblock never connected, instead of losing it as unknown (#522) (#527) * 3.x.x: offer again a won block whose submitblock never connected, instead of losing it as unknown (#522) A submitblock that fails before its connection is established (the node refuses the connection, a name or route failure, the connect timeout) provably never reached the node. Rpc::call_timeout now returns a typed RpcNotSentError for exactly reqwest's is_connect() errors. hyper raises that kind only while obtaining a connection, before any request is handed to one. The offer path returns such an attempt's own reservation to pending in one claim-fenced transaction: the offer columns are cleared, the attempt is recorded in last_error, and the row is offered again after the ordinary pending backoff. Every failure after the connection existed (a reset or close after the write, a lost reply, the request timeout, any HTTP or JSON-RPC error) stays unknown and is never offered again. A crash between the refused connect and the row update leaves offer_reserved, which recovery still treats as delivery unknown. Tests: rpc_not_sent (classification against real sockets), three gated PostgreSQL cases (refused lands once, reset after write stays unknown, crash window recovers unknown), and #521 scenario 4's kill-before-submit live case flipped to land the block exactly once after qbitd restarts. Closes #522 Refs #521 Co-Authored-By: Claude Opus 5.5 * 3.x.x: #522 review follow-ups: classify the release checkout, leased-candidate wording, outage fault helpers - Classify release_unsent_offer's ledger transaction in the pool-acquire census like its sibling offer transitions (the guard failed CI). - Say what the next claim's pre-offer phase does for a leased candidate: it is offered without the supersession screen, as every leased candidate is (#350), in the ledger doc comment and the operations doc. - Move the two held-block fault paths of the scenario-4 live test into held_offer and unsent_offer, so outage_steps no longer branches on the hold in four places. Co-Authored-By: Claude Opus 5.5 --------- Co-authored-by: Claude Opus 5.5 * 3.x.x: live two-node regtest scenarios for a lost same-height race and a deep reorg with own blocks on both branches (#521) (#530) The live_regtest fixture ran one qbitd, so no end-to-end test had a competing miner or a reorg deeper than one block. This adds a second regtest node and two gated scenarios from #521. Fixture support (additive): - support/live_two_node.rs: a second qbitd that listens on loopback and that the fixture's node connects out to; partition with setnetworkactive false, heal with setnetworkactive true plus addnode onetry, and mine on a chosen node with generatetoaddress. - Fixture::start_server_with applies environment overrides last, so a frontend can be started against the second node. - HighdiffClient is visible to sibling modules and gains open_as, so a scenario can submit block-only proofs as another payout identity. Scenario 1, an own block loses a real same-height race: the pool's block is solved on the shared parent, its offer is held at the claim renewal just before submitblock while the partitioned node's competitor reaches the pool's node, and the node then answers inconclusive. The test asserts new-tip work reaches a connected miner within 15 s of every tip change, the row stays in reconciliation without counting toward the unacknowledged or landing-failed ages through a retry at 5 confirmations, it settles orphaned at exactly the default 6, no candidate or landing gauge reaches a paging threshold, the deferred share is not credited while orphaned, and a reorg back (invalidateblock, a longer branch, reconsiderblock) credits it exactly once without reopening the row. Scenario 2, a reorg deeper than the orphan depth with own blocks on both branches: the pool mines two own blocks on node A, moves its frontend to node B once B's branch has more work, mines two own blocks there, and the partition heals onto A's longer branch, reorganizing B nine blocks deep. The test asserts the integrity report stays clean, each miner's public owed, lifetime and pending balances return exactly to the branch-A values (and differed on branch B) with no payout divergence, every own block is credited exactly once, branch A's CTV fanouts recover from reorged and confirm at maturity, and branch B's are never broadcast. Refs #521 Co-authored-by: Claude Opus 5.5 * 3.x.x: live PostgreSQL primary failover with a found block mid-landing (#521 scenario 3) (#531) Two gated live_regtest drills run the documented promotion procedure (fence the writers, capture evidence and WAL positions, stop the old primary, pg_promote, move the stable writer endpoint) against a disposable primary with one asynchronous standby, under real regtest mining, while a gate in front of qbitd holds the found block's submitblock after its reservation committed. - Async loss: replication is cut while miners keep receiving ACKs, and the node's answer is lost with the old primary. Every share committed before the cut survives; the gap's shares are lost, counted and printed. The replicated offer_reserved row is recovered as an unknown outcome, usually after its 120 s candidate lease expires, and lands on the new primary with no second submitblock. - Fenced switch: the standby replays through the old primary's flush LSN, no share or candidate is lost, and the held attempt records the accepted answer on the new primary. Both then check that job delivery resumes on both frontends, that no row is left in offer_reserved, that share ids and sequence numbers stay unique, and that the carry-forward integrity report is clean. The fixture gains start_server_with(index, fee, overrides), so a scenario can override server settings (here the RPC endpoint and the submit deadline). Refs #521, #529 Co-authored-by: Claude Opus 5.5 * 3.x.x: make the landing_metrics lost-commit deadline and api_database migration tests deterministic (#528, #532) (#537) landing_metrics (#528): lost_commit_reply_timeouts_are_not_revision_work_failures and its sibling actual_build_deadline_counts_once_and_later_notify_is_degraded hold a session's initial-job build at admission until its 0.05 s deadline, then release the admission permit and wait for that session's notify. The listener closes a session that still has no job on its first one-second timer tick past initial_job_timeout_seconds, so the release, the retried build and its persist (each bounded by the same 0.05 s) had to finish within about a second of connecting. On a loaded 2-vCPU runner they did not, and the test read "Stratum closed before a response". The later delivery now comes from deliver(), a listener with the default deadlines, on the same frontend; every metric assertion is unchanged. api_database (#532): both tests migrate their own schema in one shared database. Ledger::connect(.., true) takes the migration advisory lock, whose key (0x505249534d000001) is database-wide, under the 5 s lock timeout, so the second migration waited for the whole of the first and a loaded runner pushed that past 5 s. The tests now run their Ledger::connect calls one at a time behind a static tokio mutex. The production lock timeout is unchanged. Test-only. Closes #528. Closes #532. Co-authored-by: Claude Opus 5.5 * 3.x.x: load-harness realism, checked-in presets, a nightly workflow and a per-PR smoke run (#521 item 6) (#536) * 3.x.x: load-harness realism, checked-in presets, a nightly workflow and a per-PR smoke run (#521 item 6) qbit-prism-load gains a realistic population, all of it off by default so every existing run takes its old code paths byte for byte: - --recipients N and --recipient-weights uniform|zipf:s|pareto:a| whale:f+: distinct payout addresses whose sessions and seeded window shares follow the weights. The window's recipients are the live sessions' addresses, with the programs the server derives for them. - --session-hashrate-sigma and --session-difficulty vardiff:: each session asks for its own difficulty with d= in its Stratum password. The configured difficulty is the window difficulty over the mean offered multiplier, so the window keeps its length. - --template-bits: a harder fake network, so a three-order spread over a 400k window stays above 2^-32, the least a share target can express. The entry check refuses a spread below that floor. - --arrival bursty:cv1,cv60,max: per-second and per-minute rate bursts. - --pool-fee-bps: the pool fee on, as mainnet runs it. - --seed: every draw is reproducible. - --plan tips: warm-up only, with no artifact. The side report records the generated and driven population per address, and each phase's realised arrival. Presets in crates/qbit-prism-load/presets pin every result flag. The harness and a test both refuse a preset that omits one or leaves it to a default. There are 16: the #271 D1 20k fixture, #275's tip-delivery benchmark, a 500-address variant of each, #473's eight D1 cells exactly as run on a1937054 (pre-#497 admission), mainnet-floor, growth-5x, growth-20x and the smoke preset. qbit-prism-load-gate holds a run to its preset's gates: exit 0, reconciliation, per-phase shortfall, #473's rule where asked, and the tip-to-last-notify p99. It prints #473's verdict-table columns. .github/workflows/prism-load-nightly.yml runs the nightly presets on 8 vCPU runners, and any selection on dispatch. It records pg_test_fsync, uploads the reports and writes a job summary. Its live-nightly job runs the opt-in live_regtest variants listed in test/prism-nightly-gated-tests.txt and proves them with the gate manifest. The per-PR smoke run is a gated test that drives the smoke preset against a debug server and adds about a minute to its shard. Refs #521, #473 Co-Authored-By: Claude Opus 5.5 * 3.x.x: refuse a pinned preset flag on the command line, and gate reconciliation as the harness defines it (#521) A preset flag pinned false or null is left off the expanded command line, so clap never saw it twice. The command line could then set it and run a different workload under the preset's name and SHA-256: --recipients, a burst, --mid-flight-kill or --database-url on tip-275, for example. expand_command_line now refuses any flag the preset pins, whatever its value. The gate failed a run whose reconciliation had a committed share without an acknowledgement, although the harness explains each one and exits 0 for a submit whose answer the drain window cut off. It now gates acknowledged shares missing from PostgreSQL. It reports those commits beside their explanations. An unexplained commit is still a durability finding, and a divergence inside the run still exits 5, and both remain gated. Refs #521 Co-Authored-By: Claude Opus 5.5 --------- Co-authored-by: Claude Opus 5.5 * 3.x.x: refuse CTV settlement without a pool fee and page when no work reaches the observed tip (#525) (#534) * 3.x.x: refuse CTV settlement without a pool fee and page when no work reaches the observed tip (#525) With CTV settlement on and the pool fee off, the first positive balance below the payout floor fails every work build on the next tip with PayoutExceedsCandidateBalance: only the pool fee can take sub-floor dust (add_swept_dust_to_pool_fee). No job is published on the new parent, and mining stops until the dust reaches the floor or leaves the window. The only symptom was a WARN line per refresh. Fail fast. Config::ensure_pool_fee_settles_dust refuses PRISM_CTV_SETTLEMENT_ENABLED=1 without PRISM_POOL_FEE_ENABLED=1, names both settings and the reason, and accepts any fee, 0 bps included. Four entry points check it: server::run (before any listener or the database), check-config, self-check, and a policy-transition target. The current side of a transition, and the operator tools, still read a fee-less configuration, so policy-transition can repair a cluster that ran without a fee. make doctor mirrors the rule, including a process-environment override of the sourced .env. apply_payout_policy is unchanged. Make the stall page. Semantic coverage cannot see it: sessions are compared with the latest published generation, which stays on the old parent while every refresh fails, so the ratio reads 1. The new gauge qbit_prism_current_parent_work_missing_seconds reads the coordinator's tip state at each health publication. It is the time since the observed tip first departed from the published tip, clocked like the replacement-build lease: retries and newer tips never renew it, and it is 0 once work is published on the tip and -1 before any tip observation. PrismCurrentParentWorkMissingCritical pages at 120 s for 1 m, gated on a fresh snapshot, on either settlement path. That makes it the safety net for direct settlement, which applies the same policy and is not refused here. Fixtures: the live regtest CTV servers run a 0-bps fee unless a case sets its own PRISM_POOL_FEE_* in server_env, as the weighted-recipients case does with an address. The config_cli CTV success case and the CTV policy-transition support also run a 0-bps fee. The mainnet Compose fixture enables a fee. .env.example and compose.yaml explain the exposure; their defaults (CTV off) were already accepted. Tests: - config unit tests: refuse, accept at 200 and at 0 bps, direct not refused. - pool_fee_guard (CLI): check-config refuses and accepts; run and self-check refuse; a policy-transition target is refused, but not a fee-less current configuration. - TipState unit tests. - Gated parent_work_metric: the real server against the fake node. One miner holds work on tip A; tip B fails every refresh after its observation. The gauge grows while coverage stays 1 and health 0, and it returns to 0 when B publishes. - Six promtool scenarios and doctor tests. Mutants caught: gauge never published, gauge blind to the departure, guard disabled, doctor check disabled, and rule threshold, dwell or unknown gate broken. repro_prism_stall still pins the policy. Closes #525 Co-Authored-By: Claude Opus 5.5 * 3.x.x: page on a template refresh that keeps failing, on any parent (#525 review) Review found that the first gauge counted only after the observed tip departed from the published one. A #525 stall can start on the same parent: in direct mode, dust from gross alone fails the 60 s re-anchor rebuild on the current tip. Readiness then lapses and shares are refused, but the gauge read 0, and it paged only after the next network block, never if the pool finds most blocks. A refresh that failed before its own tip observation (e.g. a database error) or hung was also invisible. The gauge is now qbit_prism_work_refresh_stalled_seconds: the time since Coordinator::refresh_once last succeeded (published or revalidated work), counted from start before the first success. Only that success renews readiness, so the age is exactly how long the frontend has been unable to serve current work, whatever the cause. The rule is renamed PrismWorkRefreshStalledCritical, with the same 120 s / 1 m / fresh-snapshot shape. The TipState departure clock and its tests are removed, so the lease predicate is no longer duplicated. work_refresh_metric replaces parent_work_metric. The real server runs against the fake node through two stalls: a same-parent one (tip unchanged, template for another parent) and a new-tip one. The gauge keeps growing through both while semantic coverage reads 1, and it drops back below 3 s once a refresh succeeds. Mutants caught: the clock renewed on failed refreshes, never renewed, or never published. Also from review: - self-check runs its local checks before returning the pool-fee refusal; - the mainnet-deployment pool-fee block no longer splits the walletless broadcaster lead-in from its example; - the non-CTV exposure cites #535; - the policy-transition fixture derives its fee policy from one list; - a doctor test proves a process-environment disable beats an env-file enable (it fails with the override removed). Refs #525 Co-Authored-By: Claude Opus 5.5 --------- Co-authored-by: Claude Opus 5.5 * 3.x.x: model rental churn in the load harness: bursts, abrupt departures, reconnect storms, and a rental-churn nightly preset (#521) (#539) Production does not hold a fixed set of sessions. Hash rental arrives as a burst of sessions within seconds, mines for a heavy-tailed lifetime and leaves by dropping its sockets. A rental failing over between pools drops and reconnects many sessions at once. Mainnet records no connection velocity, so the model is a seeded parameter sweep, off by default and pinned by presets. --churn-seconds adds a churn side phase after slow_database (after warm-up under --plan tips), with no proxy delay and outside the artifact. In it: - Rental bursts (--rental-bursts: a list or pareto/lognormal draws) connect within a burst window. The rentals mine for the heaviest address at --rental-hashrate times a mean session. - Each rental leaves after a seeded heavy-tailed lifetime by closing its socket with no quiesce. Its abandoned submits are recorded as churn closes. A share the server committed anyway is reported as such, never as an ACK/commit divergence. - Reconnect storms drop a seeded fraction of the connected sessions, which come back within a seeded delay. - --churn-tips mints tips through the phase. The connection cap and file-descriptor limit are sized for the peak rentals. The report's churn block records: - the plan and the realised churn: spawns, departures, abrupt closes, storms, reconnects, and connects and concurrent sessions per second; - new-session time to first job; - per-tip delivery over the sessions connected at the tip: served, left first, or unserved. Two new gates hold the connected-at-tip p99 and the first-job p99. The run-wide time_to_usable_work stays over the run's own sessions. The rental-churn preset (nightly, 8 vCPU) runs the mainnet-floor population through bursts of 100, 500 and 2,000 sessions within 10 s, Pareto(1.2) lifetimes from 30 s, and storms of 10%, 25% and 50%, with 8 tips. The smoke preset gains a 30 s churn phase. Refs #521 Co-authored-by: Claude Opus 5.5 * 3.x.x: live regtest: a dense-cadence soak lands every found block, rebuilds in budget and holds RSS flat (#521 scenario 5) (#540) Two throttled CPU miners, one on each server, mine the regtest node's easy target, so the pool's own blocks land a few seconds apart. Every landing bumps the payout revision that the other server's work was built on (the #478 path). They run for PRISM_DENSE_SOAK_SECONDS (default 600). The test is #[ignore], nightly only: it is added to test/prism-nightly-gated-tests.txt, which the load-harness nightly's live-nightly job runs and proves with the gate manifest. It asserts four things: - Every block the pool found on live work is a candidate. Live work means an accepted share outside stale grace whose hash meets the network target. The exception is a race already lost when the block was found: the block that won its height was captured before the share was accepted. - Every candidate settles. It ends submitted on the node's active chain, orphaned, or abandoned before its offer only because its parent was superseded by a landed pool block at its height. Every block on the chain since the soak began is a confirmed pool block. - Each server's accepted-block-to-revision-work p99 (#458) stays within PRISM_DENSE_SOAK_REBUILD_P99_SECONDS (default 5), with no degraded landing. - Each server's RSS median over the last fifth of the soak is within a quarter, plus 32 MiB, of its median over the second fifth. After the miners stop, seven blocks that are not the pool's bury the last landings past the orphan depth. A race lost at the very end is therefore proven before the candidates are read. Refs #521 Co-authored-by: Claude Opus 5.5 * 3.x.x: legible load-harness preset names, a build-once nightly matrix, and a storm-window entry check (#521) (#559) Presets are renamed for what they measure: - throughput-20k-window-1fe[-500-addresses], from d1-20k[-realistic]; - throughput-{200,400,500}k-window-{1,2,4}fe-{async,sync}, from #473's d1-473 cells; - tip-delivery-2000-miners-400k-2fe-retarget[-500-addresses], from tip-275[-realistic]. The 500-address variant is now manual; - mainnet-shape-{130,650,2600}-addresses, from mainnet-floor, growth-5x and growth-20x; - rental-churn-bursts-and-storms; - pr-smoke. Their flags are unchanged, and each description keeps its issue references. The old names still select their presets for one release: presets/aliases.txt is read by the matrix planner (the dispatch preset input) and by prism-load-run.sh, each with a deprecation warning. The nightly matrix builds once. The plan job pins the ref to one commit. A single job builds the release server, harness and gate there, and ships them with their Cargo dep-info. Each preset job checks out that commit and unpacks the binaries into target/release, so the harness still ties the server to the checkout. It no longer installs Rust or builds. With more than one reconnect storm, --storm-reconnect-seconds must now be shorter than --storm-interval-seconds. Otherwise a storm could pick sessions the previous one left offline, whose closes the client ignores, and overstate its drops (#539 review). Refs #521 Co-authored-by: Claude Opus 5.5 * 3.x.x: offer again a won block whose submitblock qbitd answered from its warmup, instead of losing it as unknown (#526) (#566) * 3.x.x: offer again a won block whose submitblock qbitd answered from its warmup, instead of losing it as unknown (#526) A submitblock retry that reached qbitd while it was still warming up after a restart was answered with JSON-RPC error -28 (RPC_IN_WARMUP) and recorded as an unknown outcome. Under the never-offer-twice rule the block was never offered again and the reward was lost, although the warm node would have accepted it on the same tip. qbitd, like the Bitcoin Core it derives from, checks its warmup flag as the first step of CRPCTable::execute, before it looks the method up; nothing else raises -28, and the flag is cleared once, at the end of startup, and never set again. Checked in the pinned v1.0.0 source and against the v1.0.0 binary held in its warmup: a submitblock sent then is answered -28 (HTTP 500), the node has no record of the block once warm, and the same block is then accepted. A -28 reply to submitblock is therefore classified like #522's refused connection: the call provably never ran, so the reservation returns to pending through Ledger::release_unsent_offer, with the ordinary backoff and the pre-offer screen on the next claim. Only a submitblock reply counts; every other error code, a warmup answer to any other call, and every answered call keep today's behaviour. Tests: unit tests for the classification; gated PostgreSQL tests for the state transition and for a warmup answer after the call ran (never offered again); and a live regtest sibling of #522's outage case that restarts qbitd held in its warmup, so the retry is answered -28 by the real node and the block lands exactly once. Closes #526 Co-Authored-By: Claude Opus 5.5 * 3.x.x: #526 review: one relay warmup mode, a non-hanging FIFO release, and the one-node RPC endpoint assumption - The test relay keeps its warmup state in one Submit mode (Warming) instead of a mode plus a flag. - The live case releases qbitd's fee-estimates FIFO with a non-blocking open polled until the node waits in its open, so a node that never gets there fails the wait instead of hanging the test. - The live case uses RPC_IN_WARMUP instead of a literal -28; the coordinator names the classification not_run and logs that the node never ran the offer (the stored "offer not sent" prefix is unchanged). - RpcReplyError::in_warmup and the ledger operations doc state the assumption both not-run rules make: the RPC URL reaches one node; an endpoint that retries a failed POST on another backend is unsupported. Refs #526 Co-Authored-By: Claude Opus 5.5 --------- Co-authored-by: Claude Opus 5.5 * 3.x.x: make the live_regtest port-reservation, failover share-count and weighted-recipients RPC waits deterministic under load (#533) (#563) fixture_ports_stay_reserved_until_their_child_starts proved a release by binding the released port again. A released port goes back to the host, and a concurrent test's free_port() child or reservation can bind it first, so the bind failed with "Address already in use". The test now checks the reservation's own sockets: exactly the released ports leave the held set, their listening sockets are closed (by socket inode, on Linux), the others stay open and still refuse a bind, and dropping the reservations closes every socket. real_two_server_mining_failover_audit_and_reorg read the accepted share count, then the two API counts, right after stopping the miners. A share a server read before its miner was killed can still commit in between. The count and the credited-header count now come from one snapshot, and each API count is compared with a count that is the same before and after the API reads, retried for up to 30 s. A wrong API count still fails. real_weighted_recipients_pay_exact_pplns_outputs_through_fanout_spend's difficulty ramp mines 250 blocks per generatetoaddress; at a host load average of 50 one batch outran the fixture client's 45 s timeout. That call and the file's wallet calls (the payees' many-input sendall is the other heavy one) now carry a 600 s per-request deadline. Neither is idempotent, so neither is retried. Test-only. Closes #533. Refs #562. Co-authored-by: Claude Opus 5.5 * 3.x.x: require a pool fee in every settlement mode and default a fresh setup to a 0-bps fee (#535) (#568) * 3.x.x: require a pool fee in every settlement mode and default a fresh setup to a 0-bps fee (#535) Without a pool fee, apply_payout_policy cannot settle sub-floor dust, so the first positive balance below the payout floor fails every work build and mining stops. #534 refused this only for CTV settlement; direct settlement applies the same policy and was the default in .env.example, compose.yaml and the load harness. Config::ensure_pool_fee_settles_dust now refuses a pool without a fee in both modes, naming the mode, the setting and the reason. run, check-config, self-check and a policy-transition target refuse it; the current side of a transition still reads a fee-less configuration, so policy-transition remains the way to enable a fee on a cluster that ran without one. make doctor mirrors the rule. PRISM_POOL_FEE_BPS=0 is still allowed and pays nothing until dust must be swept. apply_payout_policy is unchanged. Defaults: compose.yaml enables the fee (rate unset, so 0 bps) but sets no recipient, because the deploy role runs compose.yaml alone and renders an address without a program; a recipient default there would put both beside each other and every server would refuse to start. .env.example, which a fresh setup copies to .env, enables a 0-bps fee to a synthetic lab program (recipient dev-pool-fee) that nobody holds a key for, and production refuses that program. The "exactly one address or program" and "requires PRISM_POOL_FEE_ENABLED=1" messages now say what to set. The load harness launches every frontend with the fee on; legacy presets run it at 0 bps. A 0-bps fee adds no output until dust must be swept, where a fee-off frontend would have stalled, so a run that completed fee-off reproduces its results at 0 bps. Tests: config unit tests and the pool_fee_guard CLI tests cover refusal and acceptance in both modes (0 and 200 bps), the production refusal of the lab program, and .env.example as a valid configuration in both modes; doctor tests cover both modes and the .env.example defaults; the mainnet compose contract renders compose.yaml as the deploy role does and checks it names exactly one recipient. Every test fixture that launched a fee-less server now runs a 0-bps fee. Closes #535 Co-Authored-By: Claude Opus 5.5 * 3.x.x: keep compose.yaml's pool fee off by default, and check the fee recipient in make doctor (#535 review) Review of the first commit found that compose.yaml's new PRISM_POOL_FEE_ENABLED default of 1, with no recipient default, broke a deployment that never set the flag. Such a deployment got the fee enabled with an empty address and an empty program, so Config::from_env failed. Every tool that reads the configuration failed with it, including the current side of policy-transition, which is the documented way to enable a fee. Compose cannot supply a recipient, so the default goes back to 0: that deployment now gets the clear startup refusal, and policy-transition still reads it. .env.example stays the valid fresh setup. make doctor now also refuses an enabled fee with no recipient. In production it refuses .env.example's lab program where no address replaces it; an address beside the sourced lab program is left to the server, which sees only what Compose passes. As with the flag, process-environment overrides of the address and program apply. Tests: policy-transition from compose's fee-off defaults (every fee setting empty, the flag unset or 0, both modes) reads the current configuration and accepts a 0-bps target; doctor tests for the missing recipient and the production lab program. The 0-bps fixture now lives only in tests/support/pool_fee.rs, which live_regtest and fatal_state reuse. Co-Authored-By: Claude Opus 5.5 --------- Co-authored-by: Claude Opus 5.5 * 3.x.x: release a CTV fanout claim whose completion is refused after a successful attempt, instead of holding it for 120 s (#569) (#571) When a block landed between a successful fanout attempt and its completion, the payout revision bump made finish_fanout fail with "payout revision changed while observing chain state". The error left run_once with the claim still held, so the fanout stayed claimed, with no attempt recorded, until its 120 s lease expired, and no server could check it again. The attempt-failure branch already released its claim through finish(..., 'failed'); only success followed by a refused completion leaked. run_once now hands such a claim back through the new Ledger::release_fanout_claim, then ends the pass with the same error as before. The release is an early expiry: it clears the claim fenced by the holder's token, records no attempt and leaves the recheck schedule alone. A completion that did commit, or a claim another frontend has taken over, is not touched. The next claim re-verifies the chain from scratch, as after an expiry, so broadcasts stay exactly as idempotent as before and every revision check is unchanged. A completion that loses its connection is also a refused completion, so refresh_liveness's lost-completion case now expects the claim back and every row finished by the next pass. Tests: a gated PostgreSQL case holds a real attempt at its tip re-check, bumps the payout revision, and asserts the refused completion leaves the fanout unclaimed with no attempt recorded, that the next pass settles every row, and that a stale token cannot release another frontend's claim. Co-authored-by: Claude Opus 5.5 * 3.x.x: bound the failover test's share and subscribe waits by the server's own answer bounds (#562) (#572) real_two_server_mining_failover_audit_and_reorg waited a fixed 20 s for three newly credited shares on the surviving server after the other is killed, and again on the restarted server. Every regtest share is also a block. A proof on work whose payout revision a landing superseded is captured (#478), and its answer waits for that server's own landing of the block, up to block_only_ack_timeout (max(share_commit_timeout, 60 s), 60 s here). Under host load a landing takes seconds, so three credited shares outlasted 20 s. Both waits now allow 20 s plus three full answer bounds. The shares required and the counts compared are unchanged. The fixture's subscribe read deadline was a fixed 5 s, but the server bounds session allocation by initial_job_timeout_seconds (30 s by default); it now waits 35 s. A killed peer's own claims do not stall the survivor: with server 0 killed while its block was offered (its row left claimed for 120 s), the survivor credited three shares within 1.2 s. Its captured proofs answered in about 200 ms. Test-only. Closes #562. Co-authored-by: Claude Opus 5.5 * 3.x.x: wait, within a bound, for the failover standby to flush a found block's reservation before submitblock (#529) (#570) * 3.x.x: wait, within a bound, for the failover standby to flush a found block's reservation before submitblock (#529) Under asynchronous D3 the replication gap can hold a found block's candidate row and offer reservation as well as shares. A block offered to the node whose rows are then lost with the primary is on chain with no candidate, audit or pool-blocks record on the promoted primary, and no code path lands an on-chain block without its candidate row (#529 item c). After the offer reservation commits, the offering frontend now reads the primary's pg_current_wal_flush_lsn(), which covers the candidate row and the reservation, and polls pg_catalog.pg_stat_replication until the named standby's flush_lsn (state streaming) reaches it, then calls submitblock. It runs before the strictly-live renewal, so the token fence stays immediately before the send. flush_lsn is what synchronous_commit=on would wait for; sent_lsn or write_lsn would not survive a promotion. - PRISM_OFFER_STANDBY_APPLICATION_NAME (unset = off) and PRISM_OFFER_STANDBY_FLUSH_WAIT_MS (default 250, 0 = off, at most 10000), parsed into Config, printed by check-config, wired through Compose and .env.example, and listed in the settings inventory. - The wait never holds the block: a standby that is not streaming is not waited for (absent); at the bound the block is offered anyway (lagging); a database error or a writer role without pg_read_all_stats is reported at once (failed). Every outcome but confirmed logs the block hash at WARN for post-failover reconciliation. Each poll takes and returns a pooled connection; one timeout covers every checkout and read. - synchronous_standby_names stays empty: no commit waits, shares and their ACKs are unchanged. D3 is unchanged. - qbit_prism_block_offer_standby_wait_total{outcome} (all four series start at zero) and a PrismBlockOfferStandbyUnconfirmed warning (any non-confirmed outcome in 30 minutes, no dwell, never pages), generated into the migration table and the qbit-tools patch, with seven promtool scenarios. - self-check refuses a configured wait when the writer role cannot read replication positions (GRANT pg_monitor) or the name matches no single streaming standby, and reports what it read. - Docs: the HA reference gains a found-block section (behaviour, grant, measurements, the amended accepted-loss statement) and promotion step 6, reconciling the logged hashes after a failover; the ledger operations guide points to it. Next to #291's sync experiment it records that PostgreSQL disables statement_timeout before the COMMIT's sync-rep wait: with the standby down a SET LOCAL synchronous_commit COMMIT blocked for 18.7 minutes despite statement_timeout=2s, so a statement timeout cannot be the duration guard. Tests (all gated, listed in test/prism-gated-tests.txt): - live_regtest offer_standby drill on the #531 fixture: a 400 ms lagging replication link, cut the moment the found block reaches the node, then fence, stop, promote and move the endpoint. The promoted primary holds the reservation and the block lands once. - postgres_failover offer_standby_flush on a disposable primary/standby pair with a non-superuser writer: fails fast without pg_monitor (and self-check refuses), confirms after the grant, returns at the bound with the lag while the standby's WAL receiver is suspended, confirms after it resumes, does not wait for a stopped standby (self-check refuses), and ignores another application name. - coordinator offer_standby_tests: an absent standby lets the block go at once (counted absent, landed once); with the wait off nothing is counted. - Config parsing unit test. Mutants caught: no wait (drill, absent test); sent_lsn instead of flush_lsn (drill, pair test); an absent standby waited for until the bound (pair test, absent test); no privilege check (pair test); counter not recorded (absent test); alert without lagging, with a 5 m window, counting confirmed, or with a 3 m dwell (promtool scenarios). Closes #529 Refs #291 Co-Authored-By: Claude Opus 5.5 * 3.x.x: confirm the standby wait only when every same-named standby flushed, and document the reserved-but-unsent window (#529 review) From the #570 review lanes: - A second standby sharing the application_name, or a lingering walsender from a dropped connection, could confirm the wait for the standby that had not flushed (ORDER BY flush_lsn DESC LIMIT 1). The wait now confirms only when every connected standby under the name has flushed through the target, so such a row can delay it to the bound but never confirm it falsely. Runtime and self-check read the same rows (one STANDBY_ROWS predicate), and self-check refuses a name matching more than one standby or one that has not reported a flush position yet. - A standby in state catchup after a reconnect counted as absent and got no wait; connected standbys in streaming or catchup are now waited for. - The wait widens the window in which a crash or a lost claim leaves a block reserved but never sent, from one round trip to at most the bound. That is the price of keeping the reservation on the standby; it is now stated at the call site, in the module and in the HA doc's accepted-loss list. - StandbyWait { waited, durability } replaces the per-variant waited field; durability.outcome() returns the metric label, so the second four-case label mapping and the nested result go away, and the coordinator logs one INFO or one WARN (fields and message prefix unchanged). - The poll's read ceiling (bound / 1 ms) is documented, and the HA doc no longer claims write_lsn is tested. The postgres_failover pair test adds a second standby under the same name whose WAL receiver is suspended: the wait lags to the bound although the first standby flushed, and self-check refuses the name. A mutant that confirms when any same-named row flushed (bool_or) fails it; the earlier mutants (no wait, sent_lsn, absent waited to the bound, no privilege check, no counter) still fail their tests. Refs #529 Co-Authored-By: Claude Opus 5.5 --------- Co-authored-by: Claude Opus 5.5 * 3.x.x: hand fanout and candidate claims back at shutdown, keep a lost failure's backoff, count refused sends and requeue refused fanouts behind due rows (#573) (#576) * 3.x.x: hand fanout and candidate claims back at shutdown, keep a lost failure's backoff, count refused sends and requeue refused fanouts behind due rows (#573) Four claim and attempt bookkeeping edge cases the #569 audit found in the CTV broadcaster and the candidate submit loop. None loses funds or duplicates a broadcast; each delayed a payout or lost diagnostic state. Shutdown mid-attempt (3). The candidate submit loop dropped its in-flight work at shutdown and kept the claim, so on every rolling restart another frontend waited out the 120 s lease before it could land a won block. It now hands the claim back through the existing token- and state-fenced release_recovery_claim, which keeps the state and schedule exactly as an expiry would. The CTV broadcaster ignored shutdown for a whole pass and was aborted after the 30 s drain, again holding its claim. run() now passes shutdown into the new run_pass: it stops claiming between fanouts, and inside an attempt it abandons the attempt, as the 90 s deadline already does, and releases the claim. Both releases are bounded at 5 s and fall back to the expiry. Lost failure record (2). When process() failed and finish(.., 'failed') failed too, the claim was held until expiry and the row was then retried at once. The new Ledger::release_failed_fanout records the failed attempt and hands the claim back with the backoff finish would have set. It takes neither the settlement lock nor the active-parent check, and leaves the settlement status alone. Refused sends (4). A real send whose completion was refused left no attempt row, since the next claim finds the fanout in the mempool and records only an observation. The refused completion's hand-back now records it as a submitted attempt with the refusal as its error. A refused check-only observation still records nothing. Head-of-line (1). A refused completion now hands its row back due at once but behind every row already due (next_broadcast_attempt_at = clock_timestamp()), so a row refused on every attempt cannot stop the next pass from reaching the others. Every new write is fenced by the holder's token: a completion that did commit, or a claim another frontend took over, is not touched. The next claim re-verifies the chain from scratch as after an expiry, so broadcasts are exactly as idempotent as before, and require_revision and require_fanout are unchanged. No migration, no metric change. Tests: five gated PostgreSQL cases, one per item and one per loop for the shutdown item, each failing against its mutant; a token-fence step fails when the fence is removed. #569's refused-completion case now expects the row due at once rather than NULL. Co-Authored-By: Claude Opus 5.5 * 3.x.x: share fanout attempt accounting with finish_fanout and pin the requeue order against earlier-due rows (#573 review) Review follow-ups on the #573 hand-backs, with no change to what they do: - finish_fanout and the hand-backs now build the attempt counters, the backoff and the history insert-and-prune from one set of helpers in ledger/fanout.rs (attempt_columns, ATTEMPT_BACKOFF, record_attempt_history), so the two paths cannot drift. The check-only rule is one is_check_only used by both finish_fanout and requeue_refused_fanout, which now takes the attempt's result and decides for itself whether it was a send. The hand-back takes a typed LostAttempt and a static schedule. - A requeued send's recorded error reads "completion not persisted", since the requeue also covers a completion lost to a database error, not only a refusal. - The shutdown releases log whether they failed or timed out. - The candidate shutdown comment no longer claims more than the release does: it changes when a successor may take the row, not what the successor then does. - The requeue-order test now makes every competing row due a minute ago, so a requeue that keeps an earlier schedule (a COALESCE mutant) fails it; before, NULL schedules let such a mutant pass. Co-Authored-By: Claude Opus 5.5 --------- Co-authored-by: Claude Opus 5.5 * 3.x.x: cap a block proof's acknowledgement wait at the share commit timeout instead of 60 s (#574) (#577) * 3.x.x: cap a block proof's acknowledgement wait at the share commit timeout instead of 60 s (#574) A captured (#478) block proof's share is deferred to its block's landing, and its Stratum answer waited for this frontend's own landing for up to block_only_ack_timeout = max(share_commit_timeout, 60 s). The miner's session can submit nothing else meanwhile, and some firmware or rental marketplaces treat a long silence as a dead pool (#562 measured 16.6 s with 8 s of landing delay). - block_only_ack_timeout is now share_commit_timeout itself (15 s by default), derived by config::block_only_ack_timeout, which from_env and the new test share. It still bounds every block-bearing submission: a block-only, deferred or captured proof's wait for its enqueue and landing, and a share-pass append that carries the block. - At the cap the proof gets the reply every share still pending at its deadline gets: Stratum error 20, "share outcome is not yet known", reason_id ledger-outcome-unknown. Only the wait ends there: the append or landing runs on, the block is offered and landed, and the share is credited once when that commits. - The existing unknown-outcome WARN, which already names the block hash, share id and phase, now carries elapsed_ms. - New counter qbit_prism_block_proof_ack_capped_total{path=share|block_only}, both series zero at start. No new alert: the reply already counts in qbit_prism_rejections_total{reason_id="ledger-outcome-unknown"}, which PrismShareAppendFailures warns on; the counter says why. - A candidate probe that runs out the bound before the enqueue is still a definite ledger-confirmation-failed and is not counted; the pool's own 15 s acquire timeout already ended it at about the same time before. - live_regtest's failover waits from #562 are unchanged: they count credits, which still follow the landing; only their comment changes. Tests (gated, listed in test/prism-gated-tests.txt), in b574_ack_cap: a one-shot trigger holds the block's write until the test releases it (a captured proof at its offered record; a block-carrying share-pass append at its candidate insert). Each proof is answered at the 3 s cap with the pending-outcome reply while the write is held, uncredited, and counted once on its path; after the release the share is credited exactly once and the block lands. Mutants: restoring the 60 s floor in config::block_only_ack_timeout fails both at the cap; dropping the counter fails both. Closes #574 Co-Authored-By: Claude Opus 5.5 * 3.x.x: never let the ack bound cancel a found block's probe or enqueue, and give a block-carrying append the share grace (#574 review) From the #577 review lanes: - Block loss (adversarial lane). persist_block_only ran its duplicate probe under the acknowledgement bound, so once #574 shortened the bound to share_commit_timeout, a probe queued on a saturated pool or a slow leaf could run out the bound. The proof was then answered ledger-confirmation-failed with nothing enqueued, and the found block was never offered. The probe and the enqueue now run in one spawned task that neither the bound nor a cancelled submission stops. Only the wait for it is bounded: past the bound the proof is answered ledger-outcome-unknown, counted as capped, and the task is followed until the block reaches the outbox. A probe error is still a definite failure with its database error intact (EnqueueStep::ProbeFailed). - Grace (all three lanes, coordinator decision A). A share-pass append that carries a found block now follows a plain share's ladder: the share deadline, plus share_commit_grace once COMMIT is in flight. It is never refused, so it runs on past an unknown answer. Before, #574 gave it 15 s with no grace, less than a plain share. block_only_ack_timeout now bounds only block-only proofs (captured, deferred, below-target), which stay capped at share_commit_timeout exactly. - Audit of the block path after the acknowledgement clock starts: - the block-carrying append already ran in a task the answer does not abort; - the probe and enqueue now do too; - the disposition poll only reads; - the landing runs in the candidate loop. Nothing on the path is cancelled by the bound any more. - Docs: the metrics guide no longer describes late block-carrying confirmations past a block-only bound; the counter's help and status describe both bounds. live_regtest's allowance is renamed landing_credit_seconds. Tests: - b574_ack_cap gains two gated tests: - a captured proof whose pre-enqueue probe is held past the cap is answered unknown at the cap and counted, and its block is enqueued after the release, lands, and credits the share once; - a block-carrying append held inside COMMIT past the deadline is accepted within the grace and counted as a late confirmation, not as capped. - The answer-time slack is now 2 s, below the 5 s grace. - Updated for the new contract: - commit_reconcile_candidate_bearing_append_is_never_refused: accepted within the grace while COMMIT is in flight; answered at the share deadline, not the block-only bound, while held before COMMIT; - the block-only acquire tests: a cancelled or bounded probe keeps waiting instead of failing, and the block still reaches the outbox. Mutants, each failing at least one test: - the probe bounded again: ledger-confirmation-failed, block never enqueued; - no grace for a committing block-carrying append; - grace for every block-carrying append; - the 60 s floor; - the counter dropped. Refs #574 Co-Authored-By: Claude Opus 5.5 * 3.x.x: classify the block-only probe's checkout in its spawned task (#574 review) The block-only proof's duplicate probe now takes its pool checkout inside the spawned probe-and-enqueue task (ledger.acquire()) instead of on the coordinator (self.ledger.acquire()). Split the persist_block_only census entry accordingly; both stay delegated to Ledger::acquire, which meters the checkout. Refs #574 Co-Authored-By: Claude Opus 5.5 --------- Co-authored-by: Claude Opus 5.5 * 3.x.x: checked-in e2e scenario manifest and its CI check (#543) (#560) test/e2e-scenarios.toml lists every end-to-end and load scenario from issue, pass criteria and whether it runs. scripts/check_e2e_scenarios.py derives each running scenario's lanes from its evidence (the PR and nightly gated lists, the load presets, ungated tests) and fails when a lane runs a live_regtest/load_smoke test, nightly variant or preset the manifest does not name, when the manifest claims a lane its evidence does not run in, or when an unexercised entry lacks a reason. CI runs it through tests/test_check_e2e_scenarios.py in the required python-tests shards. The functional entries map to existing tests; vardiff convergence had none and is tracked in #558. Co-authored-by: Claude Opus 5.5 * 3.x.x: run the 2.x.x migration lifecycle weekly on a real node: migrate, mine, reconcile, isolated-restore rollback (#545) (#565) * 3.x.x: run the 2.x.x migration lifecycle weekly on a real node: migrate, mine, reconcile, isolated-restore rollback (#545) #487 S20. A 2.x.x ledger built from the frozen schema_2x release SQL and scaled up (60,000 ledger rows with rejects, 250 Zipf-weighted recipients, 36 found blocks with signed audit bundles in all three historical body layouts, chained carry-forward rows, drained candidates) goes through docs/prism-rust-migration.md#recovery-and-rollback, the one-way migration with isolated-restore reconciliation #287 settled: - the drained source's evidence and a pg_dump backup; - an isolated restore that reconciles exactly before any native ACK; - migrate and import-audits, with evidence unchanged and every historical artifact served byte for byte from PostgreSQL; - both native servers mining their own blocks on the real regtest node over the migrated history: 2.x.x rows untouched, share sequence and publication order continued, the carry chain replayed across the cutover, native audits verified against the node's coinbase, self-check complete; - the post-ACK boundary: the runbook's tail query is exactly the native shares, the isolated restore still equals the source and holds none of them, and the legacy revert refuses the native schema. The legacy pool blocks are real blocks on the regtest chain, mature under the tip, because the native observer checks the latest mature block against the node and a missing one is a fatal state. The scenario is an #[ignore] live_regtest test listed in the new test/prism-weekly-gated-tests.txt. prism-load-nightly.yml gains a Sunday schedule and a live-weekly job that runs that list and proves it executed with check_gate_manifest.py; the weekly schedule runs only that job, and dispatch can add it with `weekly`. Fixture gains open_with_inputs and server_command so the scenario can take PostgreSQL's tools through the gate and run operator subcommands with a frontend's settings. Refs #545, #487 Co-Authored-By: Claude Opus 5.5 * 3.x.x: seed every configured 2.x.x ledger row in the lifecycle scenario (#545) 60,000 rows over 36 blocks left 24 unwritten: the first LEGACY_SHARES % LEGACY_BLOCKS blocks now take one more row, and the seed refuses a count other than LEGACY_SHARES. Co-Authored-By: Claude Opus 5.5 * 3.x.x: rehearse migrate and import-audits on an isolated restore in the lifecycle scenario (#545) Runbook step 4 runs the forward migration and the audit import against the isolated restore. A second restore of the same backup now takes them and must reconcile exactly with the source and serve every artifact, before the cutover runs the same commands on the source. The first restore stays the pre-migration database the rollback checks read. Fixture::tool takes the database it runs against. Co-Authored-By: Claude Opus 5.5 --------- Co-authored-by: Claude Opus 5.5 * 3.x.x: a dispatch-only Blacksmith runner probe for the load harness (#541) (#561) * 3.x.x: a dispatch-only Blacksmith runner probe for the load harness (#541) S1 of #487. prism-load-runner-probe.yml builds the server and harness and runs short-plan-20k-window-1fe (new, manual) and throughput-20k-window-1fe in one job and checkout on each chosen runner class (8, 16, 32 vCPU by default), with the target directory on a sticky disk in one job and the Actions cache in another. Each job records pg_test_fsync on the cluster filesystem and the kernel's view of its block device, build, seeding and run wall times, peak host memory, shortfall per phase, the tip-to-last-notify p99 and whether every provenance check passed without an override. pg_test_fsync-only jobs give the VM-to-VM spread, an optional job probes the maximum job duration, and a collate job writes the per-class table. No schedule, so no standing spend. Co-Authored-By: Claude Opus 5.5 * 3.x.x: pin the runner probe's ref to one commit in the plan job (#541) Every probe, fsync-spread and collate job checks out the SHA the plan job resolved, so a push to the ref while runners queue cannot make classes build and measure different commits (Codex review). Co-Authored-By: Claude Opus 5.5 * 3.x.x: read the runner probe's duration as decimal before its bound check (#541) A leading zero made bash compare the value as octal, so 05000 passed the 4,200-minute cap and then ran as 5,000 (Codex review). Co-Authored-By: Claude Opus 5.5 * 3.x.x: count a failed pg_test_fsync VM in the runner probe's spread (#541) A VM whose pg_test_fsync failed was dropped from its class's range, so three VMs with one failure read as a complete two-VM spread. The table now names how many VMs are unknown (Codex review). Co-Authored-By: Claude Opus 5.5 * 3.x.x: treat pg_test_fsync output marked failed as unknown in the runner probe (#541) A run that printed the one-write section and then exited nonzero was parsed as a good sample although its file carries the failure line; the parser now checks that line first (Codex review). Co-Authored-By: Claude Opus 5.5 * 3.x.x: count every planned runner-probe job in the table, and leave a partial tip p99 unknown (#541) The collate step reads the plan job's matrices, so a probe or fsync job that failed before writing its row is named as missing and counted among a class's unknown VMs instead of silently narrowing the range. A run with no row at all still writes the table. A tip some session never got work on now leaves the tip-to-last-notify p99 unknown, as the gate has it, with the missing count kept (Codex review). Co-Authored-By: Claude Opus 5.5 --------- Co-authored-by: Claude Opus 5.5 * 3.x.x: name #570's standby drill and #566's warmup drill in the e2e scenario manifest (#560) (#584) The manifest check added by #560 fails on 3.x.x (tests/test_check_e2e_scenarios.py::test_the_checked_in_manifest_passes): two live_regtest tests in the PR gated list are cited by no running scenario. Both belong to existing #521 scenarios: - pg_failover_tests::offer_standby::a_found_block_offered_after_the_standby_flush_survives_a_replication_cut_at_submitblock (#570, #529 item c) runs on the #521 scenario 3 failover fixture; its criteria gain the standby flush wait's claim. - node_outage_tests::node_restarted_into_warmup_before_submitblock_lands_the_block_once (#566) is a #521 scenario 4 node-outage drill. scripts/check_e2e_scenarios.py and its tests pass. Refs #560, #570, #566. Co-authored-by: Claude Opus 5.5 * 3.x.x: name #541's runner-probe preset in the e2e scenario manifest (#560) (#592) #561 added the dispatch-only short-plan-20k-window-1fe preset while #560's manifest check landed, so check_e2e_scenarios fails on 3.x.x with "dispatch runs preset short-plan-20k-window-1fe, which no running scenario names". This adds one dispatch scenario for it; the check and its unit tests pass. Co-authored-by: Claude Opus 5.5 (1M context) * 3.x.x: retry a failed candidate attempt's claim release until the database answers, finish an offer in flight at shutdown, and record a recovered reservation's outcome as unknown (#529, #578) (#585) * 3.x.x: retry a failed candidate attempt's claim release until the database answers, finish an offer in flight at shutdown, and record a recovered reservation's outcome as unknown (#529, #578) A PostgreSQL failover while a found block is mid-landing delayed the landing by up to the 120 s candidate lease (#529 item b). The offering frontend's attempt failed on the old primary's dead connections, its claim release failed on the same connections, and no frontend could take the row until the lease expired. The submit loop now keeps a failed attempt whose release did not reach the database, and retries the same token- and lease-fenced release (the one retry_candidate runs, split out as release_candidate_claim) about once a second until the database answers or the lease's local end passes. At shutdown it tries each once more through release_recovery_claim. - Structural, not by convention: claim_release::Attempt owns the claim, the future of Attempt::run borrows it, and the only ways to release it (Attempt::failed, and at shutdown Attempt::hand_back_at_shutdown, which is #573's hand-back moved there unchanged: same release, reason, bound and logs) consume it, so the borrow checker drops the work first. Nothing this process started under the claim, submitblock included, can still run once the row is released. - The release is fenced on the token and the live lease, so it changes nothing once another frontend has taken the row or the lease expired. Offer-state rows are never offered again by the recovery, as before. - Retries keep their schedule while the loop processes other rows; before each claim the loop waits for at most one bounded retry (the lease's 5 s bound, never past the lease's local end). - The lease's local end comes from the heartbeat itself (the start of the last successful renewal plus the lease), never later than the database's own expiry. - WARN logs when a release is deferred and when it succeeds or expires. - A holder that died with the old primary still leaves its claim to the lease. A takeover keyed on the promotion (a timeline-stamped claim) would cover it, but needs a migration and preempts every healthy holder after a promotion; it stays a documented option in #529. A graceful shutdown between the offer reservation's commit and submitblock dropped the attempt, leaving an offer_reserved row that was never sent and that recovery never offers again, so a restart could lose a found block; #570's standby wait sits inside that window (#578). The offer path now holds an offer section from just before the reservation until the node's answer is recorded or the unsent reservation is returned to pending. A shutdown that finds one open keeps driving the attempt until it closes, within the send's deadline, the standby wait's bound and the lease's bound for the reservation and the recording, logs that it waits, and only then stops; the landing after it is left to the next claim, which lands the offered row without another offer. Pre-dispatch failures still return the reservation through release_unsent_offer. A crash in the window is unchanged: recovery reconciles the reservation as delivery unknown. A recovered offer_reserved row whose call's answer was never recorded landed submitted with offer_outcome NULL (#529 item a). finish_candidate now records unknown for an offer-state row without an outcome, as the reconciliation and orphan dispositions already do; pending rows keep none. Tests (gated, listed in test/prism-gated-tests.txt): - claim_release_tests, a third frontend reaching PostgreSQL through a TCP relay the test cuts and heals, with a 30 s lease: the claim is released 0.3 s after the database answers and only after the attempt's work was dropped, then another frontend recovers the reservation and lands it with outcome unknown and no submitblock; a deferred release leaves a claim another frontend took after expiry untouched; shutdown releases a deferred claim. - offer_shutdown_tests: a shutdown with the attempt parked right after the reservation (before the standby wait) or right before the send waits for the offer; the block is sent once, recorded accepted, and landed once. - The live async failover drill now asserts landing within 30 s of the promotion (2.3 s here; 119.8 s with the retry removed) and the unknown outcome. Docs: the ledger operations guide describes the deferred release, the shutdown mid-offer and the recorded outcome, and the HA reference's promotion step 5 says when a mid-landing block lands. Closes #529. Closes #578. Co-Authored-By: Claude Opus 5.5 * 3.x.x: bound the submit loop's shutdown by one 20 s budget, give the offer section the renewal's allowance, and classify the moved pool checkouts (#529, #578) Review of #585 found two ways the shutdown path could run out of time. - The offer-section bound allowed the lease's bound twice, for the reservation and the recording, but a section has three database steps: the strictly-live renewal before submitblock is the third. A slow but live database could end the wait during the send and drop a reserved block, the loss #578 prevents. It now allows the lease's bound three times. - After the shutdown signal the loop could spend the section wait, the claim hand-back and one bounded attempt per deferred release in series, past the server's 30 s join of its tasks, which then aborts the loop and leaves its claims to the lease. Everything after the signal now shares one 20 s budget (SHUTDOWN_BUDGET), leaving room for one bounded retry already running when the signal came, and a signal that arrives during the pre-claim retry claims no new row. scripts/prism_pool_acquires.json follows the moves: the heartbeat's checkout is now in with_tracked_heartbeat, retry_candidate's BEGIN in release_candidate_claim, and Attempt::run is classified non-database (a name collision, like the server's runtime run), which the Python shard's pool-acquire check requires. Co-Authored-By: Claude Opus 5.5 --------- Co-authored-by: Claude Opus 5.5 * 3.x.x: nightly e2e for PostgreSQL disk exhaustion and wall-clock jumps, and a per-PR Stratum transcript replay (#575) (#586) * 3.x.x: nightly e2e for PostgreSQL disk exhaustion and wall-clock jumps, and a per-PR Stratum transcript replay (#575) Three live_regtest scenarios for #575's item 5 and the automatable half of item 3. Disk exhaustion (nightly): the test's own PostgreSQL 16 keeps its whole data directory on a small ext4 image mounted with fuse2fs through the setuid fusermount helper, so no root is needed. Clients mine shares that are not blocks on both servers while a ballast fills the filesystem to ENOSPC, mine through the full state past the paging rules' `for`, then the space is freed. It asserts that every acknowledged share is in the ledger, that refusals are explicit (some with the reasons PrismShareAppendFailures counts) and no ledger-confirmation-failed share is credited, that a paging rule's condition (mirrored from docs/prism-native-alert-rules.json) holds for its `for` on every server, and that both original server processes accept shares again, land a block and keep carry integrity once space is freed. The fill/release mechanism is a reusable DiskFullInjector for #554. Clock jumps (nightly): both servers and a private PostgreSQL run under libfaketime (realtime only). Each server jumps minutes and hours behind and ahead; the database jumps two hours ahead and back while a found block's submitblock is held mid-landing. It asserts that every acknowledged share is credited once with a monotonic ledger clock, that every block is offered exactly once, lands once and matches its audit bundle, and carry integrity. A held block's answer may be ledger-outcome-unknown at the share commit timeout (#577); it must still land and be credited once. A server more than PRISM_TEMPLATE_MAX_AGE_SECONDS ahead of the node serves no work (fail-safe) and PrismWorkRefreshStalledCritical fires; corrected, it serves again. One outbox row is left in reconciliation after the database clock steps back, which #581 tracks and the test pins. Transcript replay (per PR, about 4 s): a JSON-lines Stratum transcript format and harness that replays recorded client sessions against a live server and checks each answer for compatibility, with a synthetic corpus for cgminer, bmminer version rolling, NiceHash extranonce.subscribe, Braiins mining.configure extensions and esp-miner suggest_difficulty. The nightly job installs fuse2fs, fuse3, faketime and libfaketime, and the three tests are named in test/e2e-scenarios.toml (#543). Refs #575, #554 Co-Authored-By: Claude Opus 5.5 * 3.x.x: #575 e2e review fixes: every clock-jump share must be accepted, bounded disk-full phases, one Stratum session Review follow-ups on the #575 scenarios; tests only. - Clock jumps: every ordinary share a serving server mines must be accepted. Before, only the block was checked, so a server refusing every share after a clock step still passed. The one exception is the phase right after the database clock steps back, where the stale-job and ledger-confirmation-failed refusals #581 records are allowed. - Clock jumps: each block's outbox row is given its reconciliation retry (10 s per attempt) to finish before it is checked. The nightly dispatch on this branch caught a row released into reconciliation by a post-offer step that lost a payout-revision race, checked mid-backoff. - Disk exhaustion: every phase is bounded (a server that never accepts fails the test instead of hanging the job), and the full state ends when a paging rule has fired on each server, bounded, instead of a fixed 200 s after the first refusal of either server. - One StratumSession serves both the share client and the transcript replay; answers are typed (accepted, rejected, timed out, lost) instead of matched on error text. - Rule mirrors, the sliding [5m] window and the `for` verdicts live in the alert-rules module; the submitblock proxy and the host-tool helpers get their own support files; impossible states are gone from the types. Refs #575, #554 Co-Authored-By: Claude Opus 5.5 --------- Co-authored-by: Claude Opus 5.5 * 3.x.x: rehearse the 2.x.x cutover on a mainnet-shaped ledger and on a pg_dump, with each step's time and longest lock (#575) (#587) * 3.x.x: rehearse the 2.x.x cutover on a mainnet-shaped ledger and on a pg_dump, with each step's time and longest lock (#575) #575 item 1. Migrations were tested only on the synthetic schema_2x fixture. This adds a cutover rehearsal on #545's lifecycle scenario: - A seeded generator for a mainnet-shaped 2.x.x database, from the frozen release SQL. The shape is #521's aggregates: 257 identities over 76 days in three eras, a whale at 82-98% over a Zipf tail, bursty 5-minute rates (peak 394/s), and 13,566 blocks with signed audits in the three body layouts, PPLNS carry chains, payout entries, immature, inactive, reversed and rejected blocks, CTV rows, v1/v2 outbox rows, rollups and worker difficulty. Scale is 1x or 3x; a share-density knob thins rows only. #545's seed moves into the same module unchanged (byte-identical rows and files on equal inputs). - A rehearsal driver: check-config, migrate and import-audits exactly as the migration guide runs them, timed and with the longest lock per step sampled from pg_locks. It then checks the recovery evidence, every 2.x.x table's counts and sums, every address's balances (SQL and the running API), the payout window against the native snapshot, a first Stratum job and self-check. - Per PR, migration_rollback runs it on a pg_dump of a PR-sized mainnet-shaped ledger. The weekly live_regtest scenario cuts over 1x at 1/16 of the rows on a real regtest chain, then mines and reconciles as #545 does. - `make prism-cutover-rehearsal DUMP=... LEDGER_KEY=...` rehearses an operator's snapshot in a private PostgreSQL cluster and prints a pass/fail report. `make prism-cutover-rehearsal-dump` writes a mainnet-shaped dump, and `make prism-cutover-rehearsal-generated` does both in one command. All three are documented in the migration guide. - The weekly job builds in release and allows 120 minutes for the 1x/16 scenario. The rehearsal found #582: 002's share-hash backfill cannot finish within the migrate transaction's statement timeout on a production-sized ledger. The weekly scenario runs migrate at the 600 s maximum until it is fixed. It first asserts that migrate at the default timeout still refuses the ledger and changes nothing, so the check fails once #582 is fixed. Refs #575 Co-Authored-By: Claude Opus 5.5 * 3.x.x: cutover rehearsal review fixes: continuous lock holds, stricter invariants, operator environment, dump formats (#575) Review findings on the cutover rehearsal: - The lock sampler now reports continuous holds: a lock released and taken again is two holds, keyed by relation OID and full advisory key. Online steps are recognised only by the statement that is the call, so 016's whole-file text no longer reads as 017's swap. The per-PR test no longer requires the millisecond 013/017 steps to have been sampled. - The native window is read as an operator tool, which registers no frontend heartbeat. It is compared in its own order with each share's credit policy, and must be non-empty. The per-PR ledger has 40 blocks, so the window spans more than one 4,096-row page of both readers. - The table facts now cover the pool rollups row by row, the writer lease, worker difficulty's update time, and every balance recomputed from the carry rows. They are read again after the frontend has run. The per-PR test checks that the restore reproduces the seeded source. - Operator mode: ENV_FILE runs the rehearsal with the operator's reviewed production environment. NODE_RPC could not pass: a mainnet node forces production mode, which refuses test keys. The private cluster requires a random password and is stopped even after a partial start. Directory, tar and plain dumps restore. Relative paths resolve from where make runs, and SCHEMA must be an identifier. The generator's pg_dump keeps the URL's other connection parameters. - Weekly: #545's carry-continuation check now also runs on the mainnet ledger. The #582 check runs from 2M rows (the 1x/16 ledger), says when it is skipped, and requires that no scratch schema or native table survived. The generator bounds its immature blocks. Refs #575 Co-Authored-By: Claude Opus 5.5 --------- Co-authored-by: Claude Opus 5.5 * 3.x.x: run the weekly cutover rehearsal at 1/32 of mainnet's share rows until #582 lands (#575) (#593) At 4.13M rows (1/16), migration 002's share-hash backfill alone took 321 s of the 600 s statement-timeout cap the weekly scenario runs migrate with, on a 22-core host. A slower runner could reach the cap and fail the weekly run on #582 rather than on anything new. 1/32 (about 2.05M rows) keeps the same shape and still exercises the #582 expected-failure check, whose row threshold drops from 2M to 1.5M (002 already took 26-38 s at 1.03M). Return to 1/16 once #582 lets migrate run on default settings. Co-authored-by: Claude Opus 5.5 (1M context) * 3.x.x: fuzz the Stratum line codec, session state machine and codec parsers, with per-PR property tests (#575) (#591) * 3.x.x: fuzz the Stratum line codec, session state machine and codec parsers, with per-PR property tests (#575) Issue #575 item 4: nothing fuzzed or property-tested the Stratum front door. This adds three cargo-fuzz targets, a nightly script that runs them for a fixed time and gates on no crash, and a small proptest suite that runs in every PR. - Seam: the per-connection loop is now `stratum::serve_connection`, generic over the read and write halves. `run_listener` still accepts, applies admission, calls `set_nodelay` and passes the TCP halves, so timeouts, buffer bounds, budgets and shutdown are unchanged; the helpers that wrote to `OwnedWriteHalf` take any `AsyncWrite`. It is `#[doc(hidden)]` and exists so the targets and tests can drive the production loop over an in-memory pipe, with no socket or database. - `crates/qbit-prism-server/fuzz`, outside the workspace (own lock file, nightly toolchain): `stratum_lines` (arbitrary bytes in arbitrary fragments), `stratum_session` (scripts of subscribe, authorize, configure, suggest_difficulty, submit and extranonce requests, with live placeholders, tip and payout-revision refreshes, reconnects, pipelining and injected backend faults, over a configurable listener) and `codec_parsers` (transaction, block, coinbase-split, compact, difficulty, hex and template parsers). A `MiningBackend` with no database builds real jobs and credits by the coordinator's gate. - Invariants checked on every iteration: no panic; heap growth bounded per iteration by a counting allocator, plus libFuzzer's malloc and RSS limits; every frame answered once, in order, with its own id, and an oversize frame refused with one null-id answer and a close; only documented codes and reason IDs, with the determinable submit refusals predicted exactly; notifications only for work built for that connection; and no credit for a share that fails validation, re-verified from the built job and the raw wire fields by code independent of `assemble_submission`, including #478's no credit for superseded same-parent work. - Seeds from the stratum_protocol and stratum_codec vectors, generated and replayed by `cargo test` in `fuzz/`, which also checks that each session seed reaches the state it is named for; a libFuzzer dictionary. - `tests/stratum_properties.rs` (proptest, 8 properties, 24 to 256 cases): frame id round trip under any fragmentation, the exact message bound, submission field round trip, and difficulty/target conversions (exact clamped floor quotient, monotonicity, round trip, compact encoding, scaled difficulty). Ungated: they read no environment input. - `.github/scripts/prism-stratum-fuzz.sh` pins nightly-2026-09-24 and cargo-fuzz 0.13.2 (sha256), runs the three targets at once for a given time, merges each corpus down, writes a summary and exits 1 on any finding. The nightly job that calls it is a separate workflow change. - docs/prism-stratum-fuzzing.md: targets, seam, invariants, local runs, nightly job. Refs #575 Co-Authored-By: Claude Opus 5.5 * 3.x.x: keep the Stratum fuzz corpus when its nightly merge fails (#575) A libFuzzer -merge=1 that failed still deleted the grown corpus and moved the partial merge output into its place, and the summary never named the failure. The corpus is now replaced only when the merge exits 0; a failed merge keeps the whole corpus, fails the run and prints an error naming the merge log and its merge-* artifacts. Refs #575 Co-Authored-By: Claude Opus 5.5 --------- Co-authored-by: Claude Opus 5.5 * 3.x.x: run the Stratum fuzz targets nightly for 20 minutes (#575) (#594) Adds the stratum-fuzz job to the PRISM load-harness nightly workflow, as prepared in #591: the three cargo-fuzz targets from #591 run at once for 20 minutes on the 17 2 * * * schedule, from the checked-in seeds plus the corpus the previous night kept, gated on no crash, leak, timeout or OOM input; the corpus is saved either way and any finding is uploaded. A `fuzz` dispatch input runs it by hand; it defaults to off so preset dispatches (for example a soak calibration) don't also pay for fuzzing. Co-authored-by: Claude Opus 5.5 (1M context) * 3.x.x: drive a real regtest qbitd from qbit-prism-load, ramped to the fake node's difficulty and reconciled against the chain (#547) (#590) `--node qbitd` runs the harness's existing phases, share reconciliation and exit codes against a managed pair of regtest qbitd nodes instead of the in-process fake. The frontends reach pool node A through a recording JSON-RPC relay. Peer B, which A connects to as #530's PeerNode does, mints every external tip with generatetoaddress. The chain is ramped first, as #524 does. -legacyretarget epochs on qbit regtest are 1,440 blocks, each clamped to 4x, so eight retargets reach bits 1e7fffc0 at height 12,960. That is the fake node's 1e7fffff to 0.0008%, so the window, share difficulty and hashes per share are the fake run's. Two node rules shape the clock: qbit refuses a block about ten minutes ahead of its clock, and it serves a template more than 150 s after the tip at the regtest limit. So the ramp's shared mock clock ends at wall time, the clocks are then cleared, and B mints a keepalive once the tip's own timestamp is 120 s old. The timestamp matters because a pool block carries its job's template time. The 1,439 blocks before the next retarget are headroom, checked at entry. A share is a block with probability 8 / window whatever the ramp, so a phase at R shares/s implies an own block every W / (8R) seconds. Real-node mode refuses a phase outside 9-600 s and reports the band, each phase's interval and the stepped-over block solutions under node.cadence_band. --plan short at its defaults implies one every 50 s. A watcher on A stamps every tip change: a waitfornewblock long-poll given the last tip, then a height walk. The relay records each submitblock verdict verbatim in the fake node's submission shape. After the load, every block above the ramp must be exactly one peer mint or one accepted submission, on A's active chain, at the ramped bits, with every template at those bits. Any disagreement is a contradicted premise (exit 8); a node that exits is an abort (6). Provenance checks are unchanged; artifact_kind is always example (#487 decision 1). Fake-node mode is unchanged. Its code path is not edited, real-node report keys appear only in real-node runs, and presets gain "--node": "fake". fake_node_mode_is_unchanged checks the fake node's answers to a fixed RPC corpus and a pr-smoke frontend's environment against goldens taken before this change; load_smoke checks the side report's key sets against a third. The tips plan now schedules --scheduled-blocks in its warm-up, its only phase; before, it silently scheduled none. The gated real_node_smoke_reconciles_against_postgres_and_the_chain runs the new real-node-smoke preset on every PR: 50 sessions, 3 peer tips and 1 own block. short-plan-real-node and short-plan-fake-node, the bridging pair for #552, run nightly: the load harness's preset jobs now install the pinned qbit, as the live-nightly job does. Review fixes. The peer link is kept against qbit's peer timeouts with a very large -peertimeout (the ramp's message stamps are taken under a mock clock that starts about 36 minutes in the past), and a lost link is named: peer-to-peer logging goes on after the ramp, a disagreement adds both nodes' peers and chain tips to node.chain_reconciliation, and the smoke test prints that and the nodes' post-ramp logs when it fails. A mint first waits (10 s at most) for B to hold A's tip, so it never forks a pool block still on its way. The relay forwards and records in its own task, so a frontend that gives up on a request cannot lose the record of a block A took. A rejected submitblock verdict (anything but acceptance, duplicate or inconclusive), an unreadable submit and a tip asked of B but never minted are problems. A node that exits or can no longer be reached aborts the run (exit 6) instead of becoming a contradiction (8). Tips are stamped when B minted them or A answered the submit, if earlier than the watcher's wake-up. A failed start and an early drop remove the data only after both nodes are dead. QBITD_BIN is read through the test gate's name for it, and the runner-probe preset pins "--node": "fake". Closes #547 Co-authored-by: Claude Opus 5.5 * 3.x.x: run the load nightly on a run-load label, skip unchanged nights, report scheduled failures, and fingerprint each preset's host (#549) (#596) * 3.x.x: run the load nightly on a run-load label, skip unchanged nights, report scheduled failures, and fingerprint each preset's host (#549) - pull_request (labeled) with run-load runs the nightly preset set on the PR's merge result, in a per-PR concurrency group. - A guard job cancels a nightly whose 3.x.x head is the commit the last scheduled nightly tested (prism-load-tested-commit artifact), so the skip shows as cancelled, not passed. Weekly and dispatch runs are not skipped. - A failed scheduled run opens or comments on the one open prism-load-nightly-failure issue. - prism-load-run.sh writes host.json (runner label, mount options, write cache, FUA) via prism_load_probe.py host --runner; preset artifacts are kept 90 days. Co-Authored-By: Claude Opus 5.5 * 3.x.x: find the last tested commit through the artifacts API and record it only when every preset reached its gate (#549) Co-Authored-By: Claude Opus 5.5 * 3.x.x: mark a preset as gated only on the gate's 0 or 1 exit, not on gate.md existing (#549) Co-Authored-By: Claude Opus 5.5 --------- Co-authored-by: Claude Opus 5.5 * 3.x.x: long soak automation: a one-lifetime soak plan for the load harness, a weekly 5.5 h soak and a deployment soak report (#575) (#588) * 3.x.x: long soak automation: a one-lifetime soak plan for the load harness, a weekly 5.5 h soak and a deployment soak report (#575) Issue #575 item 2. The longest automated run was about twelve minutes. - qbit-prism-load --plan soak: a soak preset's `soak` block names checked-in presets whose workload flags are looped over the soak preset's own cluster and population for one server lifetime (no frontend restart or kill, no degraded-database phase; phases named c..). Every sample_seconds it samples each frontend's RSS, descriptors and threads, the database's connections per frontend, idle-in-transaction count, pg_wal size, share sequence, partition and relation sizes, and the ACK latency since the last sample. At rollover_minutes it advances the share sequence to just below its partition's bound so the live load crosses into the next partition, and every archive_every_minutes it runs the operator's share-archive seal, archive, verify, detach and drop under the load. At the end every partition that left the ledger is read back (dropped ones through share-archive restore, un-attached) and the run's exact reconciliation includes them. - soak gates (soak.rs), shared by the harness gate and the deployment report: the slope of RSS's windowed peaks (a 400k-window block landing steps RSS by about 800 MiB, so per-sample fits fail healthy servers) and of descriptors after the warm-up, RSS within 2x the warm-up peak, per-client connection peak and drift, no connection idle in a transaction over 60 s, bounded pg_wal, minimum rollovers and archived partitions, one lifetime, no new payout divergence. Unknown readings fail, never pass. - presets: soak-weekly (13 cycles of mainnet-shape-130-addresses, rental-churn-bursts-and-storms and tip-delivery = 5.5 h, 4 rollovers, retention every 10 min, schedule weekly, timeout 355 min), soak-short (19 min of tip-delivery, manual; the leak gate) and soak-smoke (the nightly opt-in gated test soak_smoke). Per PR, the gated soak_sampling test samples a fresh PRISM schema and its own process, forces a rollover and holds the samples to the gates in under a second. - qbit-prism-soak-report: the same gates hourly over a live deployment from a Prometheus URL and a read-only database URL (flags or environment), writing a pass/fail Markdown report; acknowledged shares are checked against committed ledger rows. docs/prism-soak.md covers the gates, the weekly job and how to start a multi-day testnet4 soak. - qbit-prism-server: a qbit_prism_process_open_fds gauge beside the RSS gauge (the server is non-dumpable, so no other process of its user can count its descriptors), and a test-only soak-leak-mutant feature that keeps 16 KiB per accepted share, which fails soak-short's RSS gate. - prism-load-nightly.yml: a Saturday schedule whose plan selects `weekly`, its own concurrency group, and a live-nightly step that runs each listed package and binary (soak_smoke joins the live_regtest variants). prism_load_matrix.py gains `weekly`; check_e2e_scenarios.py the weekly lane; test/e2e-scenarios.toml the three soak scenarios; prism-load-run.sh appends the soak samples to the job summary. Refs #575, #556 Co-Authored-By: Claude Opus 5.5 * 3.x.x: soak review fixes: per-lifetime warm-up, restart-safe deployment counters, every churn phase gated, soak module split (#575) Review of #588 (Refs #575, #556): - A deployment soak's warm-up is now each lifetime's own: a server restarted after the warm-up warms up again, so the 2x warm-up-peak bound no longer reads unknown for the rest of the soak and the new lifetime's cold start is not fitted as a trend. - qbit-prism-soak-report keeps a process's lifetime and its last counter reading from whichever earlier sample had them: a sample that could not read the counter neither starts a lifetime nor drops the stretch from the acknowledged count. - Committed shares are counted interval by interval while each interval's rows are live, so a partition the operator's retention later removes cannot make committed < acknowledged. - Every churn phase of a soak gets its own report (soak.churn_phases) and is held to the churn budgets; the churn section was only cycle 1's. - soak.rs is split into soak/{mod,sample,evaluate,tests}.rs and evaluate() into one helper per gate group; soak_sampling samples only its own connections on the shard's shared database. Co-Authored-By: Claude Opus 5.5 * 3.x.x: soak adversarial-review fixes: scrape-aligned deployment reconciliation, dark processes and unread divergences fail, resets() restarts (#575) An independent adversarial review of #588 (Refs #575, #556): - qbit-prism-soak-report counted committed rows from 120 s before the counter baseline, a surplus that could hide a smaller loss. Both sides now cover the same span: the first reading is the baseline, and each sample adds the counters' growth and the accepted rows between the two readings' scrape times (timestamp(), the earliest among the processes). The check is committed plus the rows of the processes' scrape spread >= acknowledged; --scrape-slack-seconds is gone. A sample that cannot read the counters or the ledger advances neither side. - A counter that reset since its last reading (resets()) is a restart even when the new process's count has already passed the old reading. - A process whose series vanishes stays in the samples with every figure unknown, and a trend whose latest sample cannot read the process fails. - The payout-divergence gate fails when any sample could not read the table, instead of passing or disappearing. - The registry test asserts qbit_prism_process_open_fds reads -1 for an unreadable descriptor directory and after a failed collection (a mutant that drops it from the collector's invalidation now fails). Co-Authored-By: Claude Opus 5.5 --------- Co-authored-by: Claude Opus 5.5 * 3.x.x: run the shipped images under the Compose prism profile with the HA overlay, weekly and on Dockerfile or Compose changes (#544) (#597) * 3.x.x: run the shipped images under the Compose prism profile with the HA overlay, weekly and on Dockerfile or Compose changes (#544) `docker compose build`, brings up compose.yaml + compose.prism-ha.yaml with the prism profile on a real regtest node, mines with cpuminer-opt on one frontend and qbit-prism-miner on the other, and checks: both frontends ready and observed by each one's self-check; #281 criterion 1 across them; a found block per client on the active chain whose audit bundle both frontends serve and which verifies against the node's coinbase; exact client-to-ledger share reconciliation; and the public API listing the verified blocks. The `shipped-images` job in prism-load-nightly.yml runs it on the weekly schedule, on dispatch, and on pull requests and 3.x.x pushes that touch a Dockerfile, a Compose file or what they build from; those events run no other job. The e2e scenario manifest marks L6 running, and its checker now holds `lane_checks` to the driver's CHECKS. Co-Authored-By: Claude Opus 5.5 * 3.x.x: pace L6's cpuminer with a local difficulty multiplier, and bound client logs (#544) The first L6 run passed everything but cpuminer: at the lab's 1e-9 share floor it submitted 41M shares in four minutes without waiting for answers, never read a job after the first block, and left a 5 GB log. Hash locally to a harder target (--diff-multiplier, 1e7 by default), parse client output as it streams, cap each client's log and echo, and keep each self-check's stderr in the report. Co-Authored-By: Claude Opus 5.5 * 3.x.x: run L6 when a PR's or push's changed-file list reaches the API's cap (#544) The compare API lists at most 300 files and the PR files API 3,000, so a watched change past the cap would have skipped L6. A list at its cap now runs the lane, and renamed-from paths count as changed too. Co-Authored-By: Claude Opus 5.5 * 3.x.x: build L6 from config/upstream.env as the Makefile does, and watch it (#544) prepare-qbit-source.sh and the Makefile prefer the tracked config/upstream.env over its example, but the lane's Compose commands read the example, so the two could build from different pins, and a change to config/upstream.env alone would not run L6. Pick the file the same way, watch both, and key the qbit source cache on both. Co-Authored-By: Claude Opus 5.5 * 3.x.x: have L6's other worker submit before the resume, and require a non-duplicate refusal (#544) Once the legitimate resume had credited the header, the other worker's refusal could have been a mere duplicate, which would not show that another worker cannot resume the job. It now submits first, while the proof is unspent, and its refusal must not be duplicate-share. Co-Authored-By: Claude Opus 5.5 * 3.x.x: count-reconcile L6's clients only when every submission was answered (#544) Neither shipped client logs a share's identity, so the lane compares counts. With a submission unanswered at shutdown, a lost acknowledged share and a committed unanswered one would cancel out, so the check no longer passes on bounds: it needs ledger rows equal to acknowledged shares and nothing unanswered. Both runs so far ended with nothing unanswered. Co-Authored-By: Claude Opus 5.5 --------- Co-authored-by: Claude Opus 5.5 * 3.x.x: Record the D1 production-window load matrix (200k/400k/500k) that #271 could not run (#473) * 3.x.x: Read macOS process CPU time as Mach ticks, not nanoseconds `proc_pid_rusage` reports `ri_user_time` and `ri_system_time` in Mach absolute-time units. The harness divided them by 1e9 as if they were nanoseconds. On Intel the timebase is 1/1 and the two agree; on Apple silicon it is 125/3, so every frontend's `cpu_seconds` and `cpu_cores_mean` was under-reported by about 42 times. #447's runs on an M5 Max exposed it: 0.2500 s of CPU read as 0.0060 s. Scale by `mach_timebase_info`. A timebase the kernel will not give, or a zero denominator, makes the figure unknown rather than wrong. The arithmetic is a plain function so it is tested on every platform, and a second test burns a known amount on the thread's own CPU clock and requires the process figure to cover it. That one fails against the old arithmetic (0.0048 s reported for 0.2 s burned); nothing compared the reading with CPU really spent before, which is how this went unseen. Runs recorded before this change carry the under-reported figure. Refs #447 Co-Authored-By: Claude Fable 5.1 * 3.x.x: Record the D1 production-window load matrix #271 could not run #447. The 200k, 400k and 500k windows all exited 3 at startup on #271's base, refused by the JSONB ceiling. #273 has since merged, so they can run, and this records them as a second part of docs/prism-throughput-measurements.md with its own host, method and tables. No table mixes it with #271's host or build; statements in the #271 part that are no longer true of 3.x.x carry a pointer instead of being rewritten. Base a1937054, 39 runs, on a laptop: no #291 rehearsal host had been named, and the document says what that changes. The part that decides the verdict is that PostgreSQL on macOS does not force the drive cache on commit, and the append commits inside ORDER_LOCK. A control that changes only wal_sync_method takes 500 shares/s to about 117. - 200k, 400k and 500k run; all 22 rate-matrix runs reconcile exactly. - 500 shares/s: rate sustained in 22 of 22 repeats. By the strict rule 4 of 8 configurations are met; the other 4 each refused 1-3 valid shares of about 150,000 at the commit gate when a tip's work arrived. - 2,000 shares/s: not met by any configuration (1,468-1,623 median). More frontends add ORDER_LOCK waiters, not throughput. - Found blocks land at 400k under load, and `candidates list` shows nothing unfinished. Dense cadence ran at D1 at 400k, three of three. - Build control: #271's build sustains 500 shares/s on this host too, so its 238-304 was its host; and its burst ceiling here is about a fifth higher than the current build's. Both cross runs were refused, so whether the server or the harness moved is not known. Every table is generated from the side reports; none is typed by hand. Counts stated in prose are computed from the reports when the part is built, with guards that refuse to build when the prose is wrong. Refs #447 Co-Authored-By: Claude Fable 5.1 --------- Co-authored-by: Claude Fable 5.1 * 3.x.x: report the weekly soak's resident-memory failure as expected on #600, and fail once it looks fixed (#600) (#611) * 3.x.x: report the weekly soak's resident-memory failure as expected on #600, and fail once it looks fixed (#600) The first full soak-weekly run passed every gate but resident memory: a 400k-window frontend's resident set ratchets up with block landings (#600). Until that is resolved the weekly job would fail, and file a scheduled failure, every Saturday for a known cause. A soak gate key, rss_expected_failure, names the issue ("#600") or is null. While it is set, the resident-memory rows (envelope slope and warm-up-peak ratio) of every process are an expected failure as a group: - a measured failure is reported as "expected failure (#600): ..." and does not fail the soak; - when every resident-memory row passes, a "#600 looks fixed" row fails the soak, so the key goes back to null and the gate is real again; - an unknown reading still fails, and is no evidence either way; - the descriptor slope and every other gate stay hard. It is the group, not each row, because in #600's run fe-1's warm-up ratio passed (1.16x) while its slope and both of fe-0's rows failed; per row, that passing row would have read as "looks fixed". Only soak-weekly names #600. soak-short, soak-smoke and the testnet4 deployment gates keep a real resident-memory gate (null). The bounds (16 MiB/h, 2x the warm-up peak) are unchanged. Held to #600's own run: the gate over that run's report and samples now passes, with the three failing resident-memory rows reported as expected. Refs #600, #575 Co-Authored-By: Claude Opus 5.5 * 3.x.x: a soak with no resident-memory row is not "#600 looks fixed" (#600) Review of #611: with no sample naming a server process there is no resident-memory row, and expect_rss_failure read that empty group as "every resident-memory row passed". The run already fails on "server processes: unknown"; the extra "looks fixed" row could send an operator to drop rss_expected_failure on no evidence. An empty group is now left alone, as an unknown row is. Co-Authored-By: Claude Opus 5.5 --------- Co-authored-by: Claude Opus 5.5 * 3.x.x: deliver a job build that a same-tip republication overtakes, so publications faster than one fan-out no longer starve job delivery (#598) (#616) Each Stratum session builds and saves its own job. The build's admission proof compared the published tip's publication stamp, (tip hash, publication generation), by equality after the build and before and after the save, and every publication bumps that generation, including a reanchor, new shares or a template change on the same tip with the same payout revision. With 2,000 sessions on one frontend and a 1 s reanchor, almost every build was overtaken before it was saved and was thrown away ("payout snapshot stale"), so a new tip, and some first jobs, reached no session. Fresh work now keeps its admission across a later publication that kept the same published tip, parent, payout revision, prior balances and fee, and is delivered exactly as the jobs other sessions already hold from its publication; the session's next pass brings it up to date. The published tip now carries a tip generation that changes only when the published hash does, so a tip change, including A -> B -> A, still revokes the build, as do a payout revision change (still re-proved against the database at every step and inside the save transaction), a prior-balances change and a fee change. Resumed work and replacement leases keep the exact publication stamp. #377's authority tests now expect a reanchor republication to admit issue and persist, and a balances republication to refuse them. A gated test holds a real coordinator's build between build and revalidation while it republishes: a reanchor, new shares or a template change delivers and saves the build, byte-identical in its payout outputs to the live job of its publication; a new tip, A -> B -> A, a revision change with or without a republication, and a fee change discard it. Closes #598 Co-authored-by: Claude Opus 5.5 * 3.x.x: remove Config::block_only_ack_timeout, share the mature-CTV-fanout fixture, and stop the load tests rebuilding the server's dependencies (#579, #580) (#617) * 3.x.x: remove Config::block_only_ack_timeout, share the mature-CTV-fanout fixture, and stop the load tests rebuilding the server's dependencies (#579, #580) #580: after #574 the block-only acknowledgement bound always equalled share_commit_timeout, so the field and config::block_only_ack_timeout are gone and persist_block_only reads share_commit_timeout. Production behaviour is unchanged. The 13 fixtures that set the field (60 s next to a 15 s share deadline) drop it. Tests that shortened the block-only bound now shorten share_commit_timeout instead. Tests that relied on the longer bound: - commit_reconcile_block_only_confirmed_after_the_share_deadline_is_accepted only covered a bound longer than the share deadline. It is now ..._confirmed_before_its_bound_is_accepted, with a 10 s share deadline and without the late-confirmation assertion. The gated list is renamed to match. - The held-before-COMMIT case in commit_reconcile_candidate_bearing_append_is_never_refused keeps its < 1 s ceiling, which now bounds the 5 s grace. - block_only_work_block_never_takes_the_credited_share_path runs on an offline ledger whose probe fails at the pool's 30 s acquire timeout. It now sets a 60 s share deadline so that failure is still its answer. The docs that named the field now name the share deadline. #579: seed_candidate and one parameterised mature_fanouts(f, observed) move into tests/support/compact_runtime_e2e/ctv_fanout.rs. refresh_liveness and ctv_fanout_bookkeeping both use it. load_smoke, real_node and soak_smoke build the server as #554's faults test does: qbit-prism-load is selected beside it, and the CARGO_PKG_* and CARGO_MANIFEST_* variables that cargo test sets are removed, because ring's build script tracks them. The nested build no longer recompiles ring, rustls, sqlx and the server library on every run, and the next cargo invocation no longer recompiles them back. Closes #579, closes #580. Co-Authored-By: Claude Opus 5.5 * 3.x.x: #580 review: describe the ORDER_LOCK checkpoint without the share deadline The share deadline in commit_reconcile_block_only_enqueue_behind_the_order_lock_is_not_cut_off is now 10 s, so the 1.5 s checkpoint no longer falls past it. The comment and failure message now say what the checkpoint still proves: the proof is not answered while its enqueue waits behind ORDER_LOCK. Co-Authored-By: Claude Opus 5.5 --------- Co-authored-by: Claude Opus 5.5 * 3.x.x: measure ORDER_LOCK holds by holder and landing-window share acks, guard settlement's hold over appends, and alert on slow landings (#602) (#615) * 3.x.x: measure ORDER_LOCK holds by holder and landing-window share acks, guard settlement's hold over appends, and alert on slow landings (#602) Step 1 of #602: measurement and guard, no behaviour change. Lock scope and settlement logic are unchanged. R3. qbit_prism_database_order_lock_hold_seconds{holder} records every ORDER_LOCK hold, from the grant of the lock to the end of its transaction, beside the existing wait histogram. Every ledger acquisition goes through Ledger::lock_order, which returns a guard the holder keeps until COMMIT or ROLLBACK. The guard starts from the instant the wait ended, so the append path reads no extra clock. Ten holders: append, candidate_insert, first_confirmation (the settlement that first confirmed its block), settlement (any other, including an abandon), reconcile, orphan, prepared (the refresh's window snapshot), cleanup, fatal_state and operator (policy and signing transitions, archive verify and restore). The schema cutover, whose transaction its caller commits, and archive's statement-scoped drain are not observed. The ladder runs from 1 ms to 10 s. A source scan keeps every other acquisition on the helper and every guard bound. R4. qbit_prism_share_ack_landing_window_seconds{result} holds the share_ack_seconds observations whose submission arrived within 30 s after a pool block acceptance this frontend observed: its own submitblock acceptance, or a peer's block on the active chain, seen before it is confirmed or already confirmed at the tip. A buried block first seen confirmed, as after a restart, opens none. It uses the same ladder, so steady state is a subtraction. The window's end sits in an atomic, so outside a window the Stratum path pays one atomic load, and the arrival instant it compares is the one the ack's elapsed time already uses. R7. Each window gets a verdict 30 s after it stops admitting submissions: its nearest-rank p99 exceeds a bound exactly when more than 1% of its acks did, counted exactly. qbit_prism_share_ack_slow_landing_windows {p99_above_seconds="2"|"10"} publishes the consecutive slow windows, -1 before the first verdict. PrismLandingAckP99Slow tickets at three windows in a row over 2 s; PrismLandingAckP99High warns at one over 10 s; PrismPoolResubmissionRejectRatioHigh warns when duplicate-share, unknown-job and stale-job rejections exceed 5% of the pool's decisions over five minutes. Thresholds are provisional. The deployment patch is regenerated. R6. A gated test holds a settlement open with a trigger that sleeps in its first-confirmation UPDATE and appends from the other frontend meanwhile. It is an expected failure: it passes while the appends wait for the settlement's COMMIT and fails with "#602 looks fixed" once they commit within #602's budget. Refs #602 Co-Authored-By: Claude Opus 5.5 * 3.x.x: open #602 landing windows independently of the revision-work tracker, let a window settle past the next acceptance, and simplify the hold guard (#602) Review follow-ups on the first commit: - A landing window no longer depends on the revision-work tracker's insert. That tracker's saturation, retired-height and capacity limits would have stopped every later window and frozen the slow-window gauge. LandingAcks now de-duplicates by block height itself, and the three acceptance hooks call it directly. - A later acceptance leaves the previous window settling, so its late (slow) acknowledgements count toward its own verdict instead of the next window's. - lock_order reads the clock after lock() returns. The shared lock helpers keep their original signatures. - Stratum and test acknowledgements share one recording path. - The settlement-hold probe matches only this database's sleeping qbit_pool_blocks UPDATE. - The source scan requires every explicit end of a hold to follow its COMMIT. - The R4 test no longer refreshes before the offering frontend's ack, since the refresh's reconciler would open the window on its own. - Docs: reconcile runs on every template build, and first_confirmation and settlement are counted per attempt. Refs #602 Co-Authored-By: Claude Opus 5.5 * 3.x.x: tell #602 landing windows apart by block hash, not height, so a reorg's replacement block opens its own (#602) A reorg's replacement pool block at the same or a lower height is a new settlement under ORDER_LOCK, but height de-duplication gave it no landing window. LandingAcks now remembers the last 64 blocks that opened a window by hash (case-insensitive), so the reconciler seeing a block this frontend offered still opens nothing, and any other block opens its own. Refs #602 Co-Authored-By: Claude Opus 5.5 --------- Co-authored-by: Claude Opus 5.5 * 3.x.x: document the end-to-end and load CI lanes, what each proves and how to run it (#546) (#612) * 3.x.x: document the end-to-end and load CI lanes, what each proves and how to run it (#546) docs/prism-e2e-load-ci.md: for each of #487's lanes L0-L6, what runs today, what it proves and what it does not; how to dispatch a run; how to read the job summary and gate.md; where the evidence lives; and how to reproduce a run locally with .github/scripts/prism-load-run.sh. Lanes and parts not landed yet (L3 #550, L4 under load #553, L5 #556, the bridging lane #552, the trend and regression rule #551, runner classes #542) are marked "not yet running" with their owner, for those PRs to fill in. Linked from docs/README.md and the harness README's "Presets and the nightly run". Co-Authored-By: Claude Opus 5.5 * 3.x.x: lanes doc: split a gated test id into package, binary and filter, and say which pushes run ci.yml (#546) A gated id is ::::, and some live in qbit-prism-load, so the reproduction command now takes the package and the binary from the id and passes only the test path to --exact, and says to check for "1 passed" because an empty filter also exits 0. ci.yml runs on every PR and only on pushes to main, 1.x.x, 2.x.x and 3.x.x. Co-Authored-By: Claude Opus 5.5 --------- Co-authored-by: Claude Opus 5.5 * 3.x.x: #474 A: direct-settlement recipients, signed audit bundle checks, owning-wallet spends and a nightly 1,001-recipient chunk-boundary case (#607) * 3.x.x: pay weighted recipients through direct settlement, check each block's signed audit bundle, spend with the owning wallets, and run 1,001 recipients across the fanout chunk boundary nightly (#474) #474 A. The weighted-recipients fixture now: - runs a per-PR direct-settlement (CTV off) case: six addresses from six wallets, unequal weights, carry below the floor and the pool fee's dust sweep, through three blocks; - verifies every found block's signed audit bundle against the chain's coinbase with the pinned test keys and compares its window weights, accounts, pool fee, coinbase outputs and CTV partition with the independent model, refusing missing, extra, doubled or moved credit; - spreads payee addresses over several wallets and spends the smallest direct output and the smallest output of each fanout with the wallet that owns it; - adds a nightly case with 1,001 payable recipients: 999 seeded through Ledger::append, 4 submitting over Stratum (2 both), paid across fanout chunks of 1,000 and 1, reporting seeded and submitted counts separately. Co-Authored-By: Claude Opus 5.5 * 3.x.x: report the weighted-recipients cases' wall clock by phase, so a slow run says where its time went (#474) The per-PR CTV case took 346.8 s on its first CI run against 50-103 s on 3.x.x and 55 s locally; the summary could not say which phase grew. Each case now prints the monotonic wall clock of every phase (ramp, wallets and servers, rounds, model and audit, maturity, fanouts, receipts, spends) before its scenario checks, so a failing run reports them too. Co-Authored-By: Claude Opus 5.5 --------- Co-authored-by: Claude Opus 5.5 * 3.x.x: repeat runner-probe runs per class and summarise their variance for the regression rule (#542) (#605) * 3.x.x: repeat runner-probe runs per class and summarise their variance for the regression rule (#542) The runner probe could run each class only once per cache backend, so the harness's noise could not be measured. The probe workflow now takes `presets`, `repeats` (separate jobs per class, for the VM-to-VM spread) and `in_job_repeats` (runs per preset in one job, for the run-to-run spread), validated with the checked-in presets before any probe runner starts, with per-run timeouts that end the run's process group and a job timeout that covers setup, build, every run and the reports. `scripts/prism_load_probe.py variance` writes a versioned JSON document (qbit.prism.runner-probe-variance.v1): per class, preset and fsync-cost band, median, MAD and CV over every run, over per-VM medians and within each VM, with n and unknown counts (missing or failed runs are unknown, never zero), and each class's runner-size evidence for the per-lane pick. Rows move to v2 (repeat, commit, a list of runs per preset); v1 rows are still read. The run plan and the pick rule are in the workflow header. Refs #542 Co-Authored-By: Claude Opus 5.5 * 3.x.x: keep the probe's run loop going after a failed run, and read v1 runs as unknown when they failed (#542) The run step's shell is bash -e, so a gate exit 1 or a timeout ended the loop before its status was read: every later preset and repeat went unrun and the timeout cleanup never ran. The status is now caught. v1 rows are upgraded with each run's unmeasured reason, so a failed v1 run is unknown in the variance and not a sample. A job's own summary table shows the presets its row ran, not the defaults. With actions-cache, only repeat 1 saves the target, so repeats do not evict other workflows' caches. Refs #542 Co-Authored-By: Claude Opus 5.5 * 3.x.x: hold each probe run to its own VM's memory headroom, and do not wait on zombies when ending a run (#542) size_evidence compared every run's headroom with a quarter of the smallest MemTotal in the class, so a larger VM short of its own quarter could pass. Each run is now held to max(fraction x its own MemTotal, the preset's floor). end_group counted a zombie left by a PID 1 that reaps slowly as a live member of the group, so it waited the whole grace and the timeout test failed in such containers. Liveness now reads process states from /proc. Refs #542 Co-Authored-By: Claude Opus 5.5 * 3.x.x: count a probe run without a gate verdict as unknown, and list every headline metric in a fully unknown group (#542) A run whose harness exited 0 but whose gate exited 2, panicked or never started was counted as measured; only gate exits 0 and 1 are verdicts (prism-load-run.sh), so any other is now an unknown run. A group whose every run was unknown emitted no metrics at all. The run metrics and each gated phase's metrics are now always listed, with n 0 and the unknown count. Refs #542 Co-Authored-By: Claude Opus 5.5 * 3.x.x: list the phase metrics of an all-phase preset's fully unknown group (#542) A preset whose gates name no phases (pr-smoke, the soak presets) gates every phase it drives, so its unknown group now lists the phases any of its runs reported, with n 0 and the unknown count. Refs #542 Co-Authored-By: Claude Opus 5.5 * 3.x.x: give the probe's run step a cleanup allowance for every run, not one (#542) Each run that times out can spend about 40 s ending its process group, its postmasters and its clusters, and the run step had one fixed five minutes of slack for all of them, so a plan of many short runs could time the step out before the last run wrote its measurement. The slack is now five minutes plus one per planned run; the run plan's ceilings are recomputed. Refs #542 Co-Authored-By: Claude Opus 5.5 --------- Co-authored-by: Claude Opus 5.5 * 3.x.x: run the short plan on the fake and the real node on one runner and report what the real node costs (#552) (#613) * 3.x.x: run the short plan on the fake and the real node on one runner and report what the real node costs (#552) A `bridging` job in prism-load-nightly.yml runs short-plan-fake-node and short-plan-real-node one after the other on one 8 vCPU runner, alternating the order with the run number, on the nightly schedule, the run-load label and on dispatch (`bridging: true`). It runs in addition to the matrix's own jobs for the two presets. scripts/prism_load_bridge.py reads both run directories and writes bridge-row.json (qbit.prism.load-bridge.v1) and a job summary table of real minus fake: client_ack_latency p50/p99 per phase, the slowest session's time to usable work over the external tips, per-frontend and summed peak RSS, and the pool node's submitblock and getblocktemplate relay latency and peak RSS. What only the real node measures stays unknown on the fake side, never 0, and a side that failed its harness or gate marks the row untrusted. The row states it is not a D1 verdict. Both halves always run and the row is always written; the job then fails if either half failed its gate, and the report job files a scheduled failure. The manifest's bridging-fake-and-real-node scenario now runs (nightly), citing the pair; tests/test_prism_load_bridge.py holds the job's wiring, its timeout against the presets' and the pair differing only in --node. The lanes doc and the harness README describe the lane. Co-Authored-By: Claude Opus 5.5 * 3.x.x: record a nightly's commit as tested only once the bridge reached its gates (#552) A bridge that failed during setup or timed out still let the report job record the commit as tested when the matrix presets reached theirs, so the guard skipped the next unchanged night and the same-runner comparison was never retried. The bridging job now leaves a prism-load-gated-bridging marker when both halves' gates returned 0 or 1, and the report job requires it beside the matrix presets' markers. Co-Authored-By: Claude Opus 5.5 --------- Co-authored-by: Claude Opus 5.5 * 3.x.x: inject faults under load in a load-harness fault phase, per PR and nightly on the real node (#554) (#623) * 3.x.x: inject faults under load in a load-harness fault phase, per PR and nightly on the real node (#554) --faults adds a `faults` side phase to qbit-prism-load in which each named fault is injected while the sessions keep mining, then held to its #554 pass criteria on top of the run's own reconciliation. A fault that misses one exits 9, and the gate shows one row per fault. No server code changes. The faults: a slow database through the delay proxy, PostgreSQL at max_connections with the frontends' idle backends terminated, a long SETTLEMENT_LOCK holder with a tip minted during the hold, SIGKILL of the frontend holding a block the node accepted, SIGTERM of a frontend with a found block's offer in flight, a rolling restart of both frontends, and a reconnect storm. A public-api process on the standby and every frontend's /metrics are scraped throughout. A per-frontend JSON-RPC relay, used only when a fault phase is asked for, holds or withholds a scheduled block's submitblock so the offer is in flight at a known moment. faults-pr-smoke runs the SIGTERM drain and the settlement-lock holder per PR on the fake node (tests/faults.rs, gated). faults-short-real-node runs all seven nightly on the real node with 500 sessions. The plan can be randomised and seeded for #556's soak. Every preset pins --faults, the manifest names each fault with its criteria, and check_e2e_scenarios.py knows the faults test binary. Refs #554 Co-Authored-By: Claude Opus 5.5 * 3.x.x: fix the fault phase's review findings: run it last, and close checks that could pass without testing anything (#554) Review of the fault phase found checks that could pass without testing what they name, plus cleanup and timing gaps. No server code changes. Checks that could pass without testing anything: - settlement-lock on the real node took the first tip change after the mint request as the fault's tip, so a keepalive or found block could stand in for it. It now uses the fault's own mint as the pool node saw it. A tip that reached the pool node only after the release fails "the tip's jobs waited for the release" instead of passing on zero sessions. - "no share waited on the lock holder" now covers the whole hold, not only the time up to the mint. A refresh that queued for the holder while holding ORDER_LOCK would stall shares only after the mint. - frontend-sigkill waited up to 20 s for outstanding shares before the kill. By then the frontend's 1 s submitblock deadline could have recorded the call as unknown, so its answer was no longer what the kill lost. The kill now comes at once. The check requires it inside that deadline and requires the node to have accepted the block, since a rejection or an RPC error is also an answer. The kill's census of re-offers is in the evidence; possible losses are reported, not gated, as for the mid-flight kill. - With the public API on the standby, the pool-exhaustion window is no longer excluded from its figures; that fault takes only the primary's slots. - --faults is refused under --plan soak and in a soak's looped presets until #556, where the verdict would otherwise go missing. Ordering, cleanup and timing: - The faults phase now runs after every other phase, as documented. The phase used to come before mid_flight_kill, so a fault cut short by the phase bound could leave a frontend down for that phase to report as a crash. - The phase bound gives rolling-restart one allowance per frontend. - The settlement-lock /metrics sampler stops when its recovery window closes. - The read tier starts before the phase's clock, so the per-second buckets line up with the fault windows. - The lock's release is stamped as ROLLBACK is sent. - A finished task's handle is polled in place, so a Pending cannot drop its result. - A lock holder that fails before taking the lock reports its own error. - The relay forwards to the node's origin plus the path the call came in on, instead of doubling a node URL path. The new unit tests pin the phase order (they fail on the old order), the rolling-restart bound, the soak refusal and what counts as an accepted block. faults-pr-smoke passes in 33-34 s, as before (no added per-PR cost). Refs #554 Co-Authored-By: Claude Opus 5.5 --------- Co-authored-by: Claude Opus 5.5 * 3.x.x: run #521's chain-event scenarios under 2,000 Stratum sessions, and add tip-disagreement, peer-loss and external-tip scenarios (#553) (#618) * 3.x.x: run #521's chain-event scenarios under 2,000 Stratum sessions, and add tip-disagreement, peer-loss and external-tip scenarios (#553) #487 S15 (L4). #521's fixtures run unchanged under load: each case opens the fixture as its own test does, starts a session load against its frontends, and calls the scenario's own function (decision 12). - live_session_load.rs: 2,000 Stratum sessions that fail over between frontends and submit share-only proofs at a fixed rate, so the load never finds a block. It stamps every node tip with a waitfornewblock long-poll and checks three things: every acknowledged share is a durable row and no refused share is; no refusal a correct miner cannot earn; every session gets work on every lasting tip within 15 s while its node answers and its frontend is ready. It reports #481's time to usable work per frontend and per tip. - live_chain_events_load.rs, nightly: scenarios 1 and 2, scenario 4's crash after acceptance, and three new scenarios with their criteria. Two frontends on two nodes disagree about the tip: the ledger takes one side, and the other frontend stands down with no work on its node's tip. The pool's node loses its only peer: regtest exempts the peer floor, so the isolated own block is deactivated and credited once. External tips come at the 60 s target spacing. Weekly: the other scenario-4 cases, the soak, 45 minutes of external tips at 400 shares/s, and two expected failures: 2,000 sessions on one frontend at a 1 s reanchor (#598) and scenario 7 at 2,000 wallets (#604). - Load frontends use the production 60 s reanchor. At the fixture's 1 s, 2,000 sessions on one frontend get no work (#598). - check_e2e_scenarios.py: ids in test/prism-weekly-gated-tests.txt run in the weekly lane, which now also needs the live-weekly job's schedule and gate-manifest step, and every weekly id must be named by a running scenario, as for the nightly list. The manifest cites every weekly id, #545's and #575's included, and each expected failure has its own scenario owned by its issue. - live-weekly's timeout goes from 120 to 180 min for the long set, about 85 minutes. - Shared support changes: pub(super) on scenario functions, outage_case_on, solve_share, the 2,000-wallet connection limit, and assert_no_duplicate_headers reading its two counts in one statement (under load, shares landed between them). Refs #553 Co-Authored-By: Claude Opus 5.5 * 3.x.x: hold #553's load to its claims, and guard #598 nightly instead of expecting its failure weekly (#553) Review of the first commit found checks weaker than the manifest claims, and #616 has since fixed #598. - Scenarios 1, 2 and 4 now reach their first event under the whole load. #521's server_ready waits on a ready gate that the load installs (99% of sessions holding work), and scenario 4's case takes a `ready` hook before its first block. Before this, scenario 1's race came with 42 of 2,000 sessions connected and scenario 4's fault with about 1,350. - Sessions spread their first connections from the first ready frontend, not from the load's start. - The load records every notify, so work on a parent a session held before still serves that tip after a reorg back. - Refusals are an allow-list (the harness's expected and backend classes). An impossible or unclassified refusal fails, and ledger-outcome-unknown counts as unanswered, since its share may land. - External tips and the weekly soak hold at least one tip per frontend to the bound. The peer-loss case holds the isolated block itself to it on every session. - The 1 s-reanchor case becomes #598's nightly guard at 5 shares/s, #598's reproducer rate: each of three tips must reach all 2,000 sessions on the one frontend within 15 s. It fails before #616 and passes after it. It runs nightly because live-nightly builds debug; in release the fan-out outruns a 1 s reanchor even without the fix. - The 2,000-wallet case leaves the weekly list, since in release (the weekly job's build) it fails on #622 rather than #604. It stays an ignored case for running by hand, counting as #604 only once at least 100 sessions opened; the wallet fixture's error now says how many had. wallets-2000-cap stays unexercised, naming #604 and #622, and the expected-failure scenarios go. - The load soak validates its duration and budget as #521's does, and the load settings reuse the soak's env reader. - live-nightly's and live-weekly's comments name the jobs they run. Refs #553 Co-Authored-By: Claude Opus 5.5 * 3.x.x: count work a session held at the stamp, hold lasting tips to the bound, and match the untyped refusal exactly (#553) Review of the second commit found two checks that could pass without holding what they claim. - Since the load records every notify, a republication on a parent after the next tip's stamp could count as work on that parent when the node returned to it, although the session had moved to the next tip's work in between. Work before a stamp now counts only while the session still held it at the stamp: its last run of notifies on the tip, from after the node's previous tip. A new unit test fails on the old lookup. - "At least one tip held to the bound" counted gated tips, and a gated tip replaced within 15 s can only count sessions as replaced. The report adds held_tips (gated and lasting the bound), which the external-tips, soak, scenario 1, 2 and 4 checks and #598's guard now use; the soak ends on a closing tip, since its own blocks come seconds apart. - The one refusal without a reason_id a correct load can earn is the unknown-job budget, now matched on its code and exact message as the harness does. "too many connections" is an authorize-time error and never reaches the submit counts. - The manifest's criteria say what the load checks: the refusal allow-list, and every session holding work in #598's guard. Stale comments are fixed, and until_connected reuses holding. Refs #553 Co-Authored-By: Claude Opus 5.5 * 3.x.x: document #553's chain events under session load in the CI lane guide (#553) #612's lane guide still listed #553's scenarios as not yet running. L4 now names the nightly and weekly sets, the load's own checks and #598's guard, and keeps only the 2,000-wallet case, on #604 and #622, as not proven. Refs #553 Co-Authored-By: Claude Opus 5.5 --------- Co-authored-by: Claude Opus 5.5 * 3.x.x: seed the 1,001-recipient case at the template's parent height, as the submit path records it, and check every share records it (#474) (#624) #607 review (discussion_r4148698096): the seeded shares recorded the candidate height from getblocktemplate, while the production submit path and bootstrap share record template_parent_height(candidate), one lower, as live_ctv_cpfp.rs does. They now record the parent height, and check_sources requires every accepted share, seeded or submitted over Stratum, to carry the found block's parent height. Co-authored-by: Claude Opus 5.5 * 3.x.x: test that vardiff settles each session into its share-interval band over a real Stratum socket (#558) (#609) * 3.x.x: test that vardiff settles each session into its share-interval band over a real Stratum socket (#558) crates/qbit-prism-server/tests/vardiff_convergence.rs starts the native listener over a pared-down fake backend and connects two miners at the same startup difficulty, one 16x below its target difficulty and one 16x above. Each solves real share proofs at its job's difficulty and paces submits from a fixed hashrate, anchored to the previous share's due time and restarted when work at a new difficulty arrives. With target_seconds 0.1 and retarget_seconds 0.5, each session must settle within 4 retargets (2 expected at the 4x step limit) into 1.5x of the target share interval and then change at most once over the next 8 retarget periods. The client's pace is checked as a premise, so a starved host fails as such rather than as a vardiff finding. It takes 8 s of wall time and little CPU, so it runs per PR through cargo test (10 of 10 runs passed locally, and 10 of 10 pinned to 2 CPUs beside three busy loops). test/e2e-scenarios.toml's vardiff-convergence now runs and cites it. Co-Authored-By: Claude Opus 5.5 * 3.x.x: vardiff convergence test: bound settling in retarget periods and reject any reversal (#558) The listener only advertises a changed difficulty, so counting advertised changes let a session that spent many unchanged evaluation periods outside the band still pass as "settled within 4 retargets". Settling is now also bounded in elapsed time: within 6 retarget periods (3.0 s) of the first job, beside the 4-adjustment bound. The fast session settles in 2.0 periods and the slow one in 3.6 (it cannot be evaluated before the listener's first one-second tick). Counting changes after settling also let 16x -> 4x -> 1x -> 1.4x pass. Every adjustment, before and after settling, must now keep the direction of the first one, and a unit test covers that rule. 20 of 20 local runs passed, 10 of them pinned to 2 CPUs beside three busy loops. Co-Authored-By: Claude Opus 5.5 * 3.x.x: vardiff convergence test: check the client's pace at every difficulty, not only the last (#558) A client starved during an early window but on pace at its final difficulty passed the pacing premise, so a trajectory its stall bent (an idle step, then a correction) would have read as a vardiff finding. The premise now also counts, for each span between advertised difficulties, the shares sent against the whole intervals the span holds at the client's hashrate, and fails a span short by more than one in-flight share or a fifth. That catches a stalled two-share window. Unit tests cover the count and the rule. 20 of 20 local runs passed, 10 of them pinned to 2 CPUs beside three busy loops; under load the worst span was 61 of 62 in these runs (77 of 80 in an earlier batch), inside the tolerance. Co-Authored-By: Claude Opus 5.5 * 3.x.x: vardiff convergence test: catch a stall at any difficulty and let the pace premise decide how a failure reads (#558) A span expecting one share could be missed entirely (a 2.2 s stall right after the first job) and still pass the per-span count, so the late settling it caused would have read as a vardiff regression. The client now keeps each share's scheduled time even when it resumes after a stall, and records a share still unsent when a new difficulty replaces it; any share more than 0.25 s past due, sent or replaced, breaks the pace premise. The premise now decides how a failure is reported, not whether a converged session passes: a session that converged passes whatever its client's pace, and one that did not reads "vardiff did not converge" only when the client kept its pace, otherwise "premise failed ... so these are not vardiff findings". Staying settled now also needs shares to show it: at least half those the hold implies at the band's longest interval, so a host that froze after settling cannot pass on no evidence. Unit tests cover the verdict, the missed one-share span and the settled-without-shares case. 20 of 20 local runs passed on a host at load average 26 from other work, 10 of them pinned to 2 CPUs beside three more busy loops; one run saw a 0.29 s stall with the session still converging, and passed. With the settled-shares rule added, 20 of 20 more passed, 10 of them pinned beside the busy loops. Co-Authored-By: Claude Opus 5.5 * 3.x.x: vardiff convergence test: count a share as past due even while the last one awaits its answer (#558) A share held back because the previous one was still unanswered was not recorded as past due when a new difficulty replaced it, so a listener starved between reading a share and answering it could advertise an idle retarget while the client's premise stayed clean. It is now recorded whenever it is past due. 20 of 20 local runs passed on a host at load average 19 from other work, 10 of them pinned to 2 CPUs beside three more busy loops; five runs saw stalls of 0.26-0.35 s and still converged, so they passed. Co-Authored-By: Claude Opus 5.5 * 3.x.x: vardiff convergence test: every adjustment must leave the difficulty closer to its target (#558) Checking direction alone passed 0.0625x -> 1.0x -> 1.4x: no reversal, and the overshoot stays in the band. Each adjustment must now also leave the difficulty closer to its target as a ratio, so an overshoot farther from it fails, while crossing the target to a closer point does not. A unit test covers both. 20 of 20 local runs passed, 10 of them pinned to 2 CPUs beside three busy loops. Co-Authored-By: Claude Opus 5.5 --------- Co-authored-by: Claude Opus 5.5 * 3.x.x: record the load lanes' trend rows on ci-evidence, promote flagged and soak bundles, and report regressions against the trailing same-class median (#551) (#606) - scripts/prism_load_trend.py: one versioned JSONL row per preset run (commit, lane, preset and its sha256, runner class, fsync cost and band, headline numbers, artifact and asset URLs; unmeasured numbers are null), rows for the non-preset jobs and for planned presets that left none, and an append to trend//YYYY-MM.jsonl on the orphan ci-evidence branch that bootstraps it, never forces, dedupes by run id, attempt and preset, and retries a rejected or unconfirmed push from a fresh fetch under one deadline. Promotion tars a run's artifact onto the rolling ci-evidence/YYYY-MM pre-release or a tag's release; `cite` promotes a cited run by hand. - scripts/prism_load_regress.py with test/prism-load-regression.toml: report-only rule, trailing median per lane, preset, runner class and fsync band, spread from #542's variance file when given, else the baseline's scaled MAD (provisional); names the commit range from the last good run to the first bad one, and keeps unknown distinct from a pass. - .github/actions/prism-load-evidence: the reusable read/regress/promote/ append/issue sequence for the nightly and #550's tag job. - prism-load-nightly.yml: each preset job builds its row; a scheduled-only `evidence` job (the only contents: write job) records and files; a read-only `evidence-preview` reports for run-load PRs and dispatches. - #542's runner-probe variance document (qbit.prism.runner-probe-variance.v1) supplies the spread, per run and fsync band; docs/prism-e2e-load-ci.md documents the branch, rows, promotion and rule; #552's bridging job and the other lane jobs get job rows at their own attempt. Refs #551 Co-authored-by: Claude Opus 5.5 * 3.x.x: lost node replies: kill the owning frontend after qbitd accepted a block, fanout or CPFP package, and lose a credited share's ACK (#474) (#610) * 3.x.x: lost node replies: kill the owning frontend after qbitd accepted a block, fanout or CPFP package, and lose a credited share's ACK (#474) - A proxy lets submitblock, sendrawtransaction (fee-bearing CTV fanout) or submitpackage (legacy zero-fee fanout and CPFP child) run on the node, confirms the node holds the side effect, and withholds the reply. A database trigger gate holds the owner's journal update, the owner is SIGKILLed and its waiting backend terminated, and the survivor takes the dead owner's lease: one offer or identical broadcasts, the original signed bytes, no fabricated rejection, one credit, one payout, the CPFP funding reservation kept until the child confirms, then released. - A Stratum relay withholds the server's acceptance of a durably credited share and closes the connection: the miner's outcome stays unknown, and its identical retry is a duplicate on both frontends. The reply hold, journal gate, frontend kill and ACK relay live in support/live_fault_seams.rs for slice B to reuse. All four run per PR (about 45 s together). Co-Authored-By: Claude Opus 5.5 * 3.x.x: close the fanout journal gate before the broadcaster can send, so a slow test cannot let the owner journal the held call first (#474) The lost sendrawtransaction and submitpackage cases closed the gate only after the proxy reported the node's acceptance, leaving the owner's 15 s RPC timeout to race a delayed test task. The gate now closes before the coinbase matures (fanout) or before the broadcasters start (CPFP), and the wait for the held reply lets through, and records, any attempt that ends without sending. The block case already closed its gate before mining. Co-Authored-By: Claude Opus 5.5 --------- Co-authored-by: Claude Opus 5.5 * 3.x.x: return a landing's freed heap to the kernel with malloc_trim after each block landing (#600) (#627) #600: every 400k-window block landing rebuilds the payout window and audit, about 1.4 GB of short-lived allocations, and glibc keeps what they free in its arenas. The landing frontend's resident set climbed with each landing to a 3.2-3.3 GB plateau while its live heap returned to about 250 MiB. - The frontend that landed calls malloc_trim(0) once the landing has released its rebuilt window, on the blocking thread that released it (OffRuntime gains a follow-up that runs after the drop): never on a runtime worker, never under a lock or a database transaction. glibc builds only; elsewhere it is a no-op that records nothing. - PRISM_LANDING_MALLOC_TRIM_ENABLED (default 1) turns it off; run, check-config and `candidates recover` read it, compose and .env.example pass it. - Metrics: qbit_prism_landing_malloc_trim_seconds (histogram), qbit_prism_landing_malloc_trim_released_bytes_total (resident set before minus after each trim) and qbit_prism_landing_malloc_trim_resident_bytes (the resident set after the latest trim). The resident set is read with the process collector's VmRSS parser, now shared. - Gated test landing_malloc_trim: three landings over a 20k-share production-shaped window through the real offer path; the resident floor after each landing's trim stays within 0.65 of the landings' resident burst. Trim off, the malloc_trim call removed and the follow-up removed each fail it; OffRuntime's test now fails a trim run before the drop. - Measured on the 400k two-frontend soak shape, two 30-minute runs each: landing frontends' cycle floors 0.55-0.69 GB with the trim against 1.35-2.54 GB without, peaks 0.75-1.64 GB against 1.44-3.17 GB; each trim 62-100 ms; no change in landing latency or share ACK tails. - Docs: landing memory per frontend and the memory limit to set (3 GiB at 400k with the trim, 5 GiB without), the capacity-readiness malloc_trim section, PRISM.md. Refs #600 Co-authored-by: Claude Opus 5.5 * 3.x.x: fail the database over, fill its WAL volume and restart over a candidate backlog under load, nightly and weekly on the real node (#554) (#635) * 3.x.x: fail the database over, fill its WAL volume and restart over a candidate backlog under load, nightly and weekly on the real node (#554) The second half of #554's faults, in qbit-prism-load's `faults` phase. Each is injected while the sessions keep mining and held to its criteria on top of the run's own reconciliation. No server code changes. - primary-kill: a barrier shows the standby flushed the primary's position, replication is cut, the primary keeps acknowledging for `cut` s and is stopped immediately, and the async standby is promoted. Every acknowledged share the new primary lacks is listed; none may predate the barrier, and each must be absent from what the standby had received. - primary-switch: writers fenced, the standby replays through the old primary's flush position, then the switch. Nothing may be lost. - block-failover: the relay holds a found block's submitblock while replication is cut and the primary stopped. With #529's standby wait on, the reservation must be on the promoted primary, and the block must reach the node once and land within 30 s of the promotion (#585). - wal-disk-full: #575's DiskFullInjector fills the primary's pg_wal volume until PostgreSQL stops, then frees it. Nothing acknowledged may be lost or acknowledged while it is down. The frontends must serve again without a restart, and every /metrics must show #575's paging condition. - candidate-backlog: the relay answers submitblock with qbitd's warmup error (#526) while found blocks pile up, every frontend is restarted, the relay heals. Every backlog row must reach a terminal state, and none may reach the node twice. For these, the managed cluster streams its standby through a replication link the harness can cut. Every writer (the frontends and the harness's side pool) reaches the primary through a writer endpoint, and the public reader reaches the standby through a read endpoint. A promotion moves the endpoints and rebuilds the standby; nothing is restarted. The run excuses, from its durability finding, only the acknowledged shares a failover's verdict proves lie in the replication gap. faults-short-real-node (nightly) gains every new fault but the full WAL volume. faults-long-real-node (weekly, #588's Saturday selection, 16 vCPU) runs all twelve with 2,000 sessions. faults-failover-fake-node runs the five on dispatch. No per-PR test is added. Refs #554 Co-Authored-By: Claude Opus 5.5 * 3.x.x: install fuse2fs in the load-run job for a preset that fills the WAL volume (#554) faults-long-real-node, on the Saturday `weekly` selection beside the soak, and faults-failover-fake-node, on dispatch, run #554's wal-disk-full fault. That fault mounts the primary's pg_wal on a small ext4 image with fuse2fs, without root, as #575's disk-full scenario does. The run job now installs fuse2fs and fuse3 when the preset's --faults lists wal-disk-full, and the header and the dispatch input name the new weekly preset and its cost. Refs #554 Co-Authored-By: Claude Opus 5.5 * 3.x.x: close the #554 fault checks review found could pass or flake, and finish an abandoned failover (#554) Review of #635 (/code-review medium, a quality review and an adversarial review) found checks that could pass without proving their claim, a flake, and a cleanup gap. No server code changes. - candidate-backlog passed even if every backlog row was abandoned at the relaunch. It now requires one backlog block to land, offered once. - block-failover took its barrier before waiting up to 30 s for the found block, so that whole window counted as the replication gap. The barrier is now taken after the held call arrives, just before the cut. The held call goes to the node within 800 ms of arriving, inside the frontend's 1 s submitblock deadline, however long the stop takes. - wal-disk-full now starts its down intervals and its outcome-unknown window at the fill's start, so a PANIC during the fill is counted. An acknowledgement inside a down interval fails the check only when its share is absent from PostgreSQL, since a late answer to an earlier commit is not a lie. - An abandoned failover waits for its in-flight stop or promotion, promotes once, heals a cut link and moves the writer endpoint, so the run's end still finds a primary. - The gap key's constant is renamed to match its value; the frozen set's lookback is named and leaves re-offers out; a unit test pins the backlog's unfinished states to the server's CandidateState::UNFINISHED_SQL. Refs #554 Co-Authored-By: Claude Opus 5.5 --------- Co-authored-by: Claude Opus 5.5 * 3.x.x: give first jobs their own admission, so a session opening during a same-tip rebuild storm no longer waits for the whole fan-out (#604) (#625) * 3.x.x: give first jobs their own admission, so a session opening during a same-tip rebuild storm no longer waits for the whole fan-out (#604) Every publication makes every connected session rebuild its job, and those rebuilds shared the one first come, first served initial-job admission (PRISM_STRATUM_MAX_PENDING_INITIAL_JOBS, 128) with sessions that had no work yet. Persistence, after that permit is released, is a second queue: the issued-job batcher admits 128 saves in order. A session opening during a fan-out therefore waited for the whole fan-out, and opening sessions one at a time grew with the square of the session count at a fast reanchor. A session that holds current work (its newest job, for its current worker, is on the published parent and payout revision) now rebuilds through a rebuild lane: a quarter of the initial-job permits, at least one, with no setting of its own. It takes the lane before the shared admission and keeps it until its job is persisted, so such rebuilds occupy at most the lane between admission and a durable job, and a session without current work (a first job, a re-authorized worker, a tip change, a payout revision landing) takes only the shared admission, as before, and waits for the lane's rebuilds rather than every session's. The published parent and revision come from a new MiningBackend::published_work_hint, an admission hint only. What is built, persisted, credited or offered is unchanged. A new gauge, qbit_prism_stratum_rebuild_lane_waiters, counts rebuilds waiting for the lane. The lane also bounds how many rebuilds share one issued-job batch, so while commits are slow, same-tip rebuilds complete at about the lane per commit; a larger lane would make first jobs wait longer. Tests: a gated coordinator test keeps 1,024 sessions rebuilding back to back against PostgreSQL and opens 8 sessions one at a time; each first job has about 15-40 storm deliveries ahead of it, where the previous admission put about 1,025 ahead, and a lane released before persistence about 125-210. Mock-backend tests pin that first jobs, re-authorized workers and sessions whose work a tip change or revision landing superseded bypass a full lane, that a lane wait times out and is retried like any delivery, and that a rebuild keeps its lane permit through persistence and only there. #553's by-hand 2,000-wallet case expected this starvation. It now opens every session and fails later, on submits that wait behind their session's queued rebuild (#621), so its expectation, name and catalogue entry now point at #621. Closes #604 Co-Authored-By: Claude Opus 5.5 * 3.x.x: let a rebuild waiting in the lane leave it once a publication supersedes its work, and address the review of #625 (#604) A session's current-work check ran once, before its wait for the rebuild lane, so sessions still queued there when a new tip or payout revision landed received that new work only through the lane. The wait now also watches the session's publications: on each one it checks again and, once the work is superseded, takes only the shared admission, as a session without current work does. The lane acquire stays pinned, so a same-tip publication keeps the rebuild's place in the lane. Also from review: - Jobs carry the template's parent hash lowercased; the published-work hint is compared without regard to case. - A job resumed after a reconnect is retired work, not current work. - A backend without a published-work hint admits every delivery through the shared admission, as before #604, instead of treating all held work as current. - The help text describes the lane's sessions as those whose work is still on the published parent and payout revision; the lane size is a const fn with a note on when to revisit it, and the gated test's bound is three lanes (96). - The pool-acquire gate classifies the lane's two semaphore acquires as non-database, like the initial-job admission. Tests: mock tests pin that a rebuild waiting in the lane stays there through a same-tip publication and leaves it on a tip change or revision landing, that a resumed job does not take the lane, and (through an uppercase hint) the case-insensitive comparison; each fails against its mutant. Co-Authored-By: Claude Opus 5.5 * 3.x.x: keep the first-job lane test's storm running until it has made a full pass, so a fast first job on a slow runner does not void the test (#604) On CI the eight first jobs opened before the 1,024-session storm had delivered 1,024 rebuilds (1,001), and the test's check that the storm really rebuilt every session failed. The storm now keeps running, for at most 60 s, until it has, so the check depends on the storm, not on how fast the host is. The first-job counts it asserts are unchanged. Co-Authored-By: Claude Opus 5.5 --------- Co-authored-by: Claude Opus 5.5 * 3.x.x: answer a session's submits while its own job rebuild is built and persisted, and measure what still waits for it (#621) (#642) * 3.x.x: answer a session's submits while its own job rebuild is built and persisted, and measure what still waits for it (#621) A Stratum session ran `deliver_job` after its `select!` and read no request until it returned, so a `mining.submit` arriving during a rebuild was answered only after the job was admitted, built and persisted: up to `PRISM_STRATUM_INITIAL_JOB_TIMEOUT_SECONDS` when rebuilds queue, as in branches raced, so a submit could also wait for a second rebuild. The delivery is split in two. `prepare_job` admits, builds and persists the job from `DeliveryInputs` read from the session when it starts (worker, extranonce1, difficulty, miner version mask) and borrows nothing else from it; the session loop keeps it as an in-flight future and polls it from the same `select!`. `announce_job` runs when it completes and does what the second half of `deliver_job` did: version mask, difficulty, then notify, then retiring and burying the work it replaces. So: - A submit (or health probe) arriving meanwhile is answered at once, against the jobs the session already holds, which are the ones the miner holds: the new job is not announced yet. The delivery keeps being polled while that request is handled, so it never stalls behind a slow submit holding its admission permit; one that finishes meanwhile is announced after the answer. A share on work the rebuild will supersede is decided as before, by the coordinator against the durable payout revision and published tip; the session's grace anchor is the last announcement, as for a share that arrived just before the rebuild. - A job is announced only after it is durable, and its difficulty and notify are written together, after any answer already given. - A request that changes what the job is built from (subscribe, authorize, configure, suggest_difficulty, or a method the server does not know) waits for the delivery, and the session reads nothing after it until it is handled, so responses keep request order and those requests see the session exactly as before. - A retarget does not run under a delivery in flight (that job keeps the difficulty it was built at); a due retarget rides the next delivery as it starts, so back-to-back rebuilds cannot starve vardiff. A share accepted while a retarget's delivery is in flight also counts in the vardiff state a failed delivery restores. - `retry_job` is consumed when a delivery starts, so a publication seen while one is in flight is delivered after it instead of being cleared by it; a failed delivery is still retried after the next event, not at once. - The initial-job deadline does not cut a delivery in flight, as the timer could not fire during one before. - The `select!` is biased with reads last: every other branch is ready at most once per tick, publication or delivery and returns promptly, so none starves a pending request, and a request backlog cannot hold off a finished delivery, a publication or the unauthenticated-peer deadline. What still makes a request wait for the session's own delivery is measured by the new histogram `qbit_prism_stratum_request_delivery_wait_seconds`: one observation per held request, from its complete frame to the end of the delivery. A submit queued behind it is read only afterwards, and `share_ack_seconds` starts at that read, so this family carries that wait. It is registered at startup, documented in the generated inventory and counted in the observability contract and census (+1 family, +14 series). The in-crate `deliver_job` remains as a test-only wrapper over the two halves. #625's rebuild lane composes: its lane wait moves into `prepare_job`'s build, its `prior` hint into `DeliveryInputs`, and the lane permit still drops once the job is persisted, before any write. Tests (no sleeps; a gate holds the build): a submit sent during a held rebuild is answered before the release (fails on 3.x.x: no reply in 5 s); a health probe is answered too; the rebuild is persisted while a submit is held in the backend and announced after its answer; a publication seen during the hold is delivered after it; a retarget earned during the hold rides the next delivery; a suggest_difficulty during the hold waits, holds the submit sent after it, and is observed once in the new histogram; a failed retarget delivery keeps the share answered while it was in flight. Each fails against a mutant that drops the guarantee it names. Co-Authored-By: Claude Opus 5.5 * 3.x.x: count a delivery its session abandons as a cancellation, keep a delivery moving while an invalid frame is answered, and list the held requests exactly (#621) The adversarial review of the previous commit found three things. A delivery its session abandons is no failure. Since the previous commit a session notices EOF, a protocol error or shutdown while its delivery is in flight and drops the delivery. Dropping it counted in `job_delivery_failures_total`, so in #621's own regime every churned miner added a "failed delivery" beside the build, persistence and deadline failures that counter is for. Before the previous commit the loop could not end mid-delivery: the build finished and its announcement was written to whatever was left of the socket. A delivery now settles in exactly one of three counters: a success, a failure (an admission, build or persistence error or deadline, or a failed announcement write, as before), or the new `job_delivery_cancellations_total`. That counter covers a delivery still being admitted, built or persisted, or built but not yet announced, when its session ends. It is registered at startup at 0 and also exposed as `stratum.job_delivery_cancellations` in /health, an additive field. The metadata of the failure counter now names the boundary. The answer to a frame that is not a JSON object was written outside `alongside`, so a peer that stopped reading could hold the in-flight delivery, and an admission permit taken inside its build, without progress for up to the 20 s write timeout. It is now written alongside the delivery, like every other answer. The oversized-frame answer is left as is, because the session ends right after it. The held-request lists in the new histogram's help, its metadata and the `answered_during_delivery` doc comment named only subscribe, authorize, configure, suggest_difficulty and unknown methods. The code also holds `mining.extranonce.subscribe` and object frames whose method is missing or not a string. Every JSON object frame other than `mining.submit` and `mining.get_health` is held. The metadata said "Zero series from start"; it now says the buckets, `_sum` and `_count` are registered at startup and read 0 until the first wait. Composing with #625. The review approved the rebase on four conditions; the fourth was the alongside write above, now done. The other three, for whoever rebases: - Keep the rebuild-lane permit a local of `prepare_job`, dropped when persistence returns. Do not move it into `PreparedJob`, or it stays held while a concurrent submit finishes (`backend.submit` can take seconds, #602) before the announcement. - Capture `prior`, the current-work hint, in `DeliveryInputs` as an owned (parent, payout revision) pair. Do not clone the `Job`, which carries the ~2,000-output coinbase. - Give `prepare_job` its own clone of the refresh receiver for the lane wait's re-check. Do not share the session loop's receiver. Tests: - `a_delivery_abandoned_by_its_session_is_a_cancellation_not_a_failure`: one session disconnects and another is shut down, each mid-rebuild. On the previous commit this read 2 failures. - `a_rebuild_keeps_moving_while_an_invalid_frame_answer_is_held`: a gated writer holds the invalid-frame answer, and the rebuild is still persisted meanwhile. On the previous commit it stalled until the write was released. - `a_failed_announcement_write_is_still_a_failed_delivery`: the boundary of the first test. Each fails against a mutant that undoes its fix and passes with it. None sleeps. Co-Authored-By: Claude Opus 5.5 * 3.x.x: classify prepare_job's initial-job admission in the checkout census (#621) The delivery split moved the shared initial-job admission, `config.initial_job_limit.acquire()`, from `deliver_job` into `prepare_job`. `scripts/check_prism_pool_acquires.py` keys its census by function, so CI's Python shard 2 saw an unclassified acquire and a stale classification. The entry keeps its "non-database" class (a Tokio semaphore, not a database checkout) under its new name. Co-Authored-By: Claude Opus 5.5 * 3.x.x: close a budget-breached session before announcing a delivery that finished during its answer (#621) Codex's review of #642: when a request that breaches a per-session budget (a malformed frame, an unknown-job submit, an authorize) was answered while the session's delivery was in flight, `alongside` could finish that delivery meanwhile. The loop then announced the job before its budget check closed the session. Before #621 a breach closed the session right after its answer and never announced anything. A peer that stopped reading after its rejection could also hold its connection for up to the write timeout while those writes waited. The budget check now runs first in the post-select pass, before a finished delivery is announced. It runs again after a held request is handled, since that request can breach a budget too. A delivery dropped this way counts as a cancellation: its session ended before the announcement. `a_budget_breach_closes_the_session_before_a_finished_delivery_is_announced` holds the breaching frame's answer while the delivery finishes. Without the fix it reads the job's set_difficulty after the breach. Co-Authored-By: Claude Opus 5.5 * 3.x.x: test that a rebuild's lane permit is released at persistence while a submit delays its announcement (#621) #625's rebuild lane is now on 3.x.x, and this branch composes with it on the three conditions its review set: - the lane permit stays a local of `prepare_job`, dropped once persistence returns, never carried in `PreparedJob`; - `prior` is captured in `DeliveryInputs` as an owned (parent, payout revision) pair; - `prepare_job` gets its own clone of the refresh receiver for the lane wait's re-check. The first condition matters only since #621: a session may now hold a persisted job unannounced while it finishes a submit (`alongside`), and `backend.submit` can take seconds (#602). `a_rebuild_releases_its_lane_permit_while_a_submit_delays_its_announcement` holds a submit in the backend while the rebuild persists, and requires the lane permit back before that submit is answered. It fails against a mutant that carries the permit in `PreparedJob` until the announcement. The lane test backend gains a submit gate for it. Co-Authored-By: Claude Opus 5.5 * 3.x.x: abandon a delivery in flight before a closing session's last writes (#621) Codex's review of #642: a frame over the size limit closes the session, but its rejection was written while the session's delivery was still in flight and no longer polled. A peer that stopped reading could hold that write for up to the 20 s write timeout. Meanwhile the paused delivery kept the initial-job admission permit, or #604's rebuild-lane permit, that it took inside its build. Polling the delivery during that write would only build a job no one announces, so the session now abandons it instead. On leaving its loop for any reason, the session drops a delivery still in flight, and any finished but unannounced one, before its last writes: the oversized-frame rejection, which moves after the loop, and the writer's shutdown. Either counts as a cancellation. `an_oversized_frame_abandons_the_delivery_before_its_rejection_is_written` holds the rejection's write and requires the admission permit back and the cancellation counted meanwhile. Without the fix the permit stays held (127 of 128). Co-Authored-By: Claude Opus 5.5 --------- Co-authored-by: Claude Opus 5.5 * 3.x.x: settle load runs through CTV fanout with --ctv-settlement, and count each landed block's direct, fanout and carried recipients (#548) (#644) The harness launched every frontend with PRISM_CTV_SETTLEMENT_ENABLED=0, so no load run ever built a CTV fanout and the direct-to-fanout split mainnet runs was never exercised under load. #555's 10,000-recipient point (ten fanout chunks per block) waits on it. --ctv-settlement sets the three CTV keys mainnet-compose.env pins (settlement on, an explicit 1,000 bits/1,000 weight market rate, the 1.2x premium) through the one launch composition, leaves the broadcaster off and the settlement shape at the server's defaults. Off, the environment is byte-for-byte what it was (the pr-smoke golden is unchanged). The explicit rate means no frontend needs estimatesmartfee, which a regtest node cannot answer; the fake node's relay floor is the same rate. Every checked-in preset pins the flag false, so each preset's sha256 changes once. The side report gains a settlement block, flag on or off: for every block the node accepted, the miner recipients the server's landing paid directly, through fanout, and carried, read from qbit_payout_carry_forward and the fanout artifacts' manifest outputs, which the landing writes in one transaction. A landed block with no fanout is a measured 0; an accepted block with no landing, or with fanout chunks whose outputs cannot be read, is listed as unmeasured; a failed read leaves the counts null. A block with any fanout chunk whose manifest lists no readable output is unmeasured, not only one whose chunks list none between them, and the totals break the measured blocks down by chain_state, since a block landed and later reorged still settled. #554's two new fault presets pin the flag false too. A new gated test runs the fake node with 400 addresses, no pool fee and the flag on, and checks 12 direct recipients (with a fee, the fee holds a direct slot) and the rest in one chunk. Co-authored-by: Claude Opus 5.5 * 3.x.x: pause the session load's shares around scenario 2's own blocks, and start each replayed transcript after earlier blocks land (#631, #632) (#640) * 3.x.x: pause the session load's shares around scenario 2's own blocks, and start each replayed transcript after earlier blocks land (#631, #632) #631: on regtest a load share weighs about half an own block. The share-only settings' 2^-32 is the easiest share target there is, and an own block's block-only proof is credited at regtest's ~2^-31 network difficulty. So ~15 load shares fill the 8x window, and which earlier own share survives in each own block's window is a timing race. The nightly deep reorg under 2,000 sessions hit the case where branch B paid both miners exactly what branch A did. The load can now pause its share offers (sessions stay connected and keep taking work), and deep_reorg pauses them from its seed blocks until branch B's own blocks are paid, then resumes them for the heal and the reorg. Without a load the hook does nothing. #632: esp-miner-suggest-difficulty finds a block, which lands after its submit is answered. The next transcript's first share could pass its revision check before the landing and commit after it, and was refused as ledger-confirmation-failed. current() now also waits until no candidate is unfinished and /healthz payout_state_generation is the cluster's payout_revision. Closes #631, closes #632. Refs #553, #586. Co-Authored-By: Claude Opus 5.5 * 3.x.x: close the session load's pause barrier on shares already being sent, and on shares that commit after their answer (#631) Review follow-ups: - A session now counts itself as unanswered before it reads the pause flag. Before, a share could be mid-write when the pauser read zero. - After the answers drain, the pause also waits until the share ledger holds still for 2 s. A lost or ledger-outcome-unknown answer may still commit. - The drain allows twice the answer bound. A stuck share is cleared on the first tick past the bound, and ticks can be up to the bound apart. Refs #631, #553. Co-Authored-By: Claude Opus 5.5 --------- Co-authored-by: Claude Opus 5.5 * 3.x.x: give the PRISM PostgreSQL CI shards 45 minutes and balance them by measured duration (#647) (#648) Round-robin over the sorted target list put the qbit-prism-server lib (~5.5 min), compact_runtime_scale (~4 min, debug plus the optimized ignored run) and live_regtest (~11.5 min) on the same shard on every run since they are four apart in the list. That shard ran 25-30 min of tests and was cancelled at the job's 30-minute cap on slower runners, while the other three ran 7-16 min. The shard runner now assigns targets longest first, each to the shard with the least estimated time so far. The estimates are the measured durations of the five heavy targets from the 2026-10-01 shard logs and 20 s for every other target (their mean). Replaying those logs, the four shards come to 14.8, 16.8, 16.6 and 17.7 min of tests instead of 14.9, 29.2, 13.7 and 8.5. Each target still runs on exactly one shard, with its ignored selection on the same shard, so the gate-manifest proof is unchanged. Each shard's job summary now lists every command's duration, which is the data for the next re-measure. The job's cap goes to 45 minutes as an interim margin until the job summaries confirm the balance. Co-authored-by: Claude Opus 5.5 * 3.x.x: refuse a block on work from a lost replication gap before its offer, and stop issuing that work after an async promotion (#619) (#645) * 3.x.x: refuse a block on work from a lost replication gap before its offer, and stop issuing that work after an async promotion (#619) Rebased onto 3.x.x 3f1fba6b (#642, #644) as one commit. It squashes the series that was PR #645's head 60efdddd: - 263e4cd4 reproduce blocks on gap-issued work after an async promotion without qbitd, and weigh the options - 37b1af7d require the mode 2 reproduction's new session to be issued readable work, not just any refusal - 2fa8c80f refuse a block on work from a lost replication gap before its offer, and stop issuing that work after an async promotion - 850547eb document the refusal: the accepted D3 loss, the runbook for an already-offered block, and the decided design - 7d229acf assert each test's outcome before its metric checks - e7468f4d record the before/after runs in the design note - c0a8018e tag work with the scan's timeline, name window_not_held in deferrals, count late refusals once, and pin the probe-error, out-of-order and clock cases - 60efdddd state the clock precondition and correct the runbook Before a candidate is enqueued, the enqueue transaction checks that the writer still holds the candidate's PPLNS window: the window's last share row must exist and must be the window's own row. If it does not, the block is refused as stale-job (window_not_held) and is never offered. Prepared work is tagged with the writer timeline (#466's expression, read in the scan transaction), so a promotion retires it, and issuance authority refuses a window the promoted writer no longer holds. The repair path and save_compact_prepared now require the endpoint to be the window's own row, not just any row with that share_seq. Every b619 reproduction now passes and is gated. The only conflict was the cardinality pins: they become 77 families, 305 startup series and 956 populated series. These are 3.x.x's new families and series plus this series' own. Co-Authored-By: Claude Opus 5.5 * 3.x.x: classify #619's two pool checkouts, and pin the refusal at enqueue of a window that reaches past its anchor (#619) CI on 384eb2a6 failed two checks for this PR's own reasons. - The pool-acquire guard needs every checkout classified. Two new sites are classified "ledger" (delegated), like the existing Ledger::acquire sites: the landing's window probe in coordinator.rs::rebuild_claim_parts, and WorkLedger::window_held. - audit_body_normalization's durable-range test built a window that ends with a share accepted after the anchor. It expected the landing's proof to refuse that window. The enqueue's #619 check now refuses it first, because the last row is not held at the anchor. The case now asserts that refusal: it is typed WindowNotHeld for the block and its last share_seq, nothing is pending, and no block or audit row is written. The other three cases still reach the landing's proof. Share history is immutable, so a candidate enqueued before this check existed cannot be built to reach the proof's final count. Co-Authored-By: Claude Opus 5.5 * 3.x.x: list #619's two gated lib tests in the expected gate list (#619) The gate proof on 5c261ad2 found two gated tests that ran but are not in test/prism-gated-tests.txt. Both are lib tests from the #619 review round: - coordinator::miner_submit::acquire_tests:: a_failed_window_probe_is_unknown_never_stale_and_enqueues_nothing - ledger::window::payout_state::tests:: a_snapshot_reads_its_writer_timeline_in_the_transaction_that_reads_its_rows The list now holds 804 tests, the same number the proof saw run. Co-Authored-By: Claude Opus 5.5 --------- Co-authored-by: Claude Opus 5.5 * 3.x.x: renew readiness from a guarded tip poll so a slow rebuild no longer refuses current work as 'tip polling stale' (#622) (#643) Readiness (last_poll), which job preparation, share and block admission and /healthz require within PRISM_HEALTH_TIP_POLL_MAX_AGE_SECONDS, was renewed only when a refresh completed. Every refresh starts with a refresh-grade node poll, but at a fast reanchor each poll is a full rebuild with no overall deadline, so a rebuild that outlasted the health timeout after its poll, or a run of rebuilds failing after polling, refused the current, valid work as 'tip polling stale' and reported unready while the node answered on the published tip. A refresh-grade poll now renews readiness, stamped when it was sent and never backwards, only when: no newer observation found another tip and it found the published tip; no readiness revocation raced it; the published template is within PRISM_TEMPLATE_MAX_AGE_SECONDS; readiness was not already revoked; no refresh has since found the cluster on a heavier chain; and, under CTV settlement, a relay floor read no older than the poll is recorded and the published fee meets it (a refresh renews after its own fee_policy, or without CTV right after its agreeing reconciliation; the block wait re-reads getmempoolinfo, and a failed read renews nothing). The block wait polls on an unchanged notification once readiness is one interval old, at most once per interval: a third of the health timeout, at most 5 s, which is also its long-poll wait. Revision, fee and lease checks are unchanged; work_refresh_stalled_seconds still pages on a stalled rebuild. New metrics: qbit_prism_tip_poll_age_seconds (-1 while unavailable) and qbit_prism_job_preparation_deferrals_total{reason} (tip_polling_stale, tip_polling_unavailable, new_tip_pending, work_retired, fee_floor, other); the tip refusals are typed with unchanged log text. Docs, the generated inventory, the HA readiness section, the server README and the PrismWorkRefreshStalledCritical rationale follow. This fixes the mechanism behind the refusal; that it was #622's trigger on qbitd is unproven until a #553 2,000-wallet release re-run reads the deferral reasons beside tip_poll_age and work_refresh_stalled. Under CTV settlement a refresh's fee policy must also meet the floor kept, publication checks the work's fee against the current floor, floor reads are ordered by when their own getmempoolinfo was sent, and a block-wait floor read that leaves the kept floor above the published fee revokes readiness, so admissions and submissions in flight fence. A retired authority keeps #645's refusal context (payout snapshot stale, or window_not_held after a writer timeline change) and counts as work_retired. Squashed onto 3.x.x 90327e8f from the reviewed series a98ca185, 86c827c7, c5b43f4d, 04ec7f24 (Codex approved 04ec7f24), itself squashed from 4a43b839, 9a13168c, 8ed5f6c6, 43882f5b, 81c4b070, f1fd3128, 05377597 (originally 1ba4776e, 9cccdb24, d671cbe9, 5d56b114, 3abba7cb on 4f4b218d). Co-authored-by: Claude Opus 5.5 * 3.x.x: give each rollups test fixture its own database so sibling migrations cannot time out its setup (#603) (#641) The rollups::tests fixture created a schema in the shared test database and ran Ledger::connect(.., true) there. Migration takes MIGRATION_LOCK (0x505249534d000001) with pg_advisory_xact_lock, and advisory locks are scoped to the database, not the schema, so every fixture in the lib binary that migrates into that database queues on one key under the ledger pool's 5 s lock_timeout. On a loaded host the queue passed 5 s and setup failed with a bare "canceling statement due to lock timeout", which is what #603 recorded. The server log of a reproduction showed the timed-out statement was the rollup fixture's SELECT pg_advisory_xact_lock($1), sixth in a queue of other fixtures' migrations. The fixture now opens a per-case database with the shared FixtureDatabase helper (#410's fix for the same class of contention), keeps its schema name as the application_name the cases look up in pg_stat_activity, and attaches setup context to the connect/migrate step. The test error now keeps any cleanup failure as context instead of dropping it. No assertion, timeout or production lock_timeout changes. Holding MIGRATION_LOCK in the gate database from a second session, the base fails both #603 cases 3/3 with that error; with this change they pass 20/20 and all 11 rollups::tests pass 5/5. Co-authored-by: Claude Opus 5.5 * 3.x.x: wait for L6's wallet when createwallet outlasts its RPC timeout (#637) (#650) L6 created its wallet with one createwallet call under the 30 s client timeout every qbitd RPC has. qbitd takes 24.5-30 s there, nearly all of it deriving the create-time P2MR keys, so about one job in three failed with a bare "TimeoutError: timed out" and a summary that blamed the frontends. The node finishes the wallet after the client gives up, and qbitd adds a wallet to listwallets only once it is created and loaded. So a client timeout is now an unknown outcome: the lane polls listwallets until the wallet is there, under one 300 s deadline shared with the call, and never issues createwallet again. Other createwallet errors still fail at once. Rpc.call names the method when it times out, the report and summary name the phase an error stopped and how long it ran, and the report records how the wallet arrived. A fake qbitd that holds createwallet on an event reproduces the CI failure on the old driver and passes on this one. Co-authored-by: Claude Opus 5.5 * 3.x.x: hold the cutover rehearsal's frontend ports until the frontend spawns (#639) (#651) The rehearsal picked the frontend's Stratum, high-diff and audit ports by binding 127.0.0.1:0 and dropping the listener at once, then ran check-config, migrate and import-audits before spawning the frontend. Nothing held the ports through that window: two picks could return the same port, or another socket could take one, and the frontend's last bind (the audit listener) failed with "Address already in use". Ports::reserve now keeps the three listeners until frontend() releases them immediately before spawn, as live_regtest's PortReservations does. The readiness failure names all three addresses. The private PostgreSQL cluster holds its port through initdb until pg_ctl start, and work_refresh_metric holds its two ports until its server spawns. The frontend still binds the audit listener after coordinator startup, so a short window between release and that bind remains; closing it needs a server-side bind-port-0-and-publish seam, which this change does not add. Co-authored-by: Claude Opus 5.5 * 3.x.x: time candidate claim leases on the taker's monotonic clock, label database refusals truthfully, and page a PostgreSQL outage on its own rule (#581) (#646) * 3.x.x: time candidate claim leases on the taker's monotonic clock, label database refusals truthfully, and page a PostgreSQL outage on its own rule (#581) #586's nightly variants showed two gaps. Every outbox deadline was the database clock, so a clock step moved it by the step. Forward, a second frontend took a live claim whose block was still being offered. Backward, the reconciliation row that takeover left waited two hours, and a dead holder's claim would have waited lease + step. Separately, a PostgreSQL outage held only PrismBlockCandidateMetricsUnavailable: the share-refusal rules were gated on a snapshot whose freshness flaps while the publisher waits on the database, and their counters were frozen in it. Database stalls were refused backend-rpc-unavailable, blaming a healthy node. Leases (option B, operator-chosen): migration 021 adds claim_renewals and claim_lease_seconds, a partial index over claimed rows, and the candidate_claim_observed_lease capability (offline, as 018). A frontend times each claim version (token, renewals) on its own monotonic clock from the reply that first shows it, and takes the row over only after the whole lease, by compare and set on that version, whatever the row's schedule unless it is parked. Holders fence on their token; their heartbeat deadline is never extended from the database clock. A row last written later than the clock proves a backward step and is due at once. candidates recover waits out an observed lease (--unsafe-database-clock-expiry keeps the old rule) and its release keeps a schedule within the row's own backoff. candidates abandon times a claim the same way and abandons by compare and set on the version it watched, so a forward step no longer lets it take a pending row from a live holder (Codex on #646); the same flag keeps the old rule. The pool-acquire inventory classifies the revocation helper. Attribution and alerting: backend-database-unavailable for session allocation, the payout reads, job persistence and resume when a SQLx error is in the chain, and stale readiness when the latest refresh failed on the database. rejections_total and accepted_shares_total render at scrape time, the two share-refusal rules drop the snapshot gate, and PrismDatabaseUnavailable (database collector unavailable for 2 m, paging) is added, with the qbit-tools patch regenerated. #586's nightly tests now assert the new behaviour. Three adversarial verifications: leases and labels requested changes (a renewal after a backward step stranding a dead holder's row; stale readiness during an outage still blamed the node), both fixed with regression tests; the re-verification approved. Rebased onto 3.x.x eedf331b as one commit (PR #646's series eb5f65ea, 1604a754, 5ce263dc), past #642, #645 and #643: the cardinality pins are 79 families, 313 startup and 964 populated series; the stratum normalization pin stays 15. ReadinessState keeps #643's poll fields and refresh_failed_on_database, whose comment now says that a guarded tip poll (#622) decides whether readiness ages during a database outage and this flag decides the label once it does. Co-Authored-By: Claude Opus 5.5 * 3.x.x: record a refresh's database failure before it releases the refresh lock, so an older refresh cannot overwrite a newer one's (#581) Codex on #646: refresh_once wrote refresh_failed_on_database after refresh_once_inner had returned and dropped refresh_lock. Two overlapping refreshes run serialized under that lock, but the older one could record its outcome after the newer one had: a success overwriting the database failure that followed it. If readiness then aged out, the refusal would blame the node although the latest refresh failed on PostgreSQL. The work under the lock moves to refresh_locked, and refresh_once_inner records the outcome before the lock guard drops, so outcomes land in the order the refreshes ran. The lock order (refresh lock, then readiness) is the one the refresh already takes. Test: a refresh paused in its revision read fails on the database while the test holds readiness, so it cannot record yet. The refresh lock must still be held, and the newer, successful refresh's outcome is the one left. On the old code the lock was already free (the headline assertion fails). The stale-readiness test's comment now names #643's guarded tip poll. Co-Authored-By: Claude Opus 5.5 * 3.x.x: label an issued persistence that outlived its deadline as the database's, whichever of the deadline and the statement failed first (#581) CI shard 0 on 0b215397 failed compact_runtime_e2e's issued_expiry_during_sql_wait test: a persistence queued on a database lock past its 1 s deadline was refused backend-database-unavailable when the statement its shortened statement_timeout cancelled failed first, and backend-rpc-unavailable when the deadline fired first (a plain string error). The label depended on which future completed first. Persistence waits on the ledger alone (its clock and revision reads, the batch transaction and its row locks, the compact repair it writes), so an expiry is the database not answering in time. save_issued_record, the one place that knows the persistence's own deadline, now marks a failure that outlived it with a typed IssuedPersistenceExpired, and persist_issued_job labels that backend-database-unavailable before the SQLx-chain rule. A failure inside the deadline keeps that rule (a statement error is the database's; an authority refusal stays backend-rpc-unavailable). The full error chain is logged. Tests: a unit test forces both orderings on the fake ledger (a statement failing inside the deadline; the deadline elapsing while the save still waits, its statement failing afterwards), both backend-database-unavailable; without the classification the second is labelled the node's, as in CI. The e2e test now runs both orderings on PostgreSQL (the waiting statement cancelled inside the deadline; the lock held past it) and expects the database label for each. A sweep of other backend-rpc-unavailable assertions found none on a database-wait path. prism-rejections.md names the case. Co-Authored-By: Claude Opus 5.5 --------- Co-authored-by: Claude Opus 5.5 * 3.x.x: stop SharedLog losing events whose callsite another test reached first, and order the block-only poll faults after the enqueue (#649) (#653) commit_reconcile_block_only_poll_errors_are_retried_then_unknown failed under the full parallel --lib run because its log lost the "block-only disposition poll failed; retrying" warning, not because of the wall clock. Every recorded failure (three from 2026-10-01, and one reproduced here on eedf331b) shows the first poll failing at exactly the 5 s lock_timeout and the final warning carrying phase="poll-error", so the polls did fail and retry in time; only that one event was missing. tracing caches one interest per callsite for the process. While a single dispatcher is registered, a callsite's first hit asks only the hitting thread's default. miner_submit::acquire_tests run block-only polls against a minimal schema, so their polls fail and reach the same callsite on threads without a subscriber. When that happened while this test's SharedLog was the only registered dispatcher, the callsite was cached `never` and the SharedLog dropped the event. A diagnostic run showed the acquire test's first hit on a thread with the global (none) default, between this test's submit and its first failed poll. SharedLog::dispatch now first installs, once, a global default that records nothing and answers every callsite `sometimes`. There are then always two registered dispatchers, and it is the default of any thread without its own, so no callsite can be cached `never` and each event asks the current dispatcher. A unit test reaches a fresh callsite on a thread without a subscriber while one SharedLog is live: it fails without the default and passes with it. The test also no longer takes its table lock after watching the candidate become pending, which left a 3 s wall-clock window (bound minus lock_timeout) for the first poll to fail. A wrapper on the probe floor that every poll calls fails each poll with SQLSTATE 55P03 (lock_not_available) once the candidate is committed, and counts the failures in a sequence; the duplicate probe before the enqueue passes through. The test asserts at least two failed polls from the database as well as the logged retry, and still that the answer is ledger-outcome-unknown at the bound, never a fabricated outcome, and that the later confirmation credits the proof once. Co-authored-by: Claude Opus 5.5 * 3.x.x: order the tips plan's scheduled block against its warm-up tips instead of racing them (#638) (#652) * 3.x.x: order the tips plan's scheduled block against its warm-up tips instead of racing them (#638) The real-node smoke landed no own block in 3-8% of CI runs. The scheduled block went out at 15 s and the third external tip was minted at 18 s on the wall clock alone. When the block's search and the server's asynchronous offer outlasted the gap, the tip took the block's height. The block could also be mined on a job from before the previous tip, when that tip's notify was slow. In a phase that holds both scheduled blocks and external tips (the tips plan's warm-up), the block is now sent once every mint asked of the node is on its tip and session 0 holds work on that tip. No tip is minted from the moment a block is due until it has settled. Settled means one of three things: the node answered a submitblock for that block's hash; the server refused it stale-job or low-difficulty, which prove nothing entered the ledger; or the session reported that it could not send it. A ledger-outcome-unknown answer does not settle it, because a block candidate's append can still be offered. ExternalMint gains settled_tip and block_answered for both nodes. The phase report records each ordered block and any tips left unminted. real_node.rs prints the relay and node submissions, the mints, the client failures and the node logs when pool_blocks is not 1. Four harness tests drive the real scheduler with injected stand-in delays. All four fail with the ordering switched off, three of them with prev-blk-not-found, and pass with it. Co-Authored-By: Claude Opus 5.5 * 3.x.x: settle a scheduled block its session's task never received, so the next tip is not held to the phase's end (#638) A session whose task has already stopped has a closed control channel. The ScheduledBlock send then fails and no ClientFailure is ever reported for it, so the landing stayed outstanding and every later warm-up tip was held until the deadline. The refused send is now the landing's verdict: the block never left the harness. This was raised in the Codex review of #652. Co-Authored-By: Claude Opus 5.5 * 3.x.x: hold keepalives with the tips, and let a failed write wait for the node's verdict (#638) Two holes in the ordering, raised in the second Codex review of #652. A scheduled block whose socket write failed is reported both as a no-response submit and as a recorded ClientFailure. The write can fail after the line reached the server, so only an unrecorded failure proves the block was never sent. A recorded one now waits for the node's verdict on the block's hash, like any other unanswered submit. qbitd's keepalive task asked for its own mints regardless of the ordering. A keepalive requested between the settled-tip read and the block's send, or while the block was outstanding, could take the block's height. The scheduler now holds keepalives over the same span as the tips. The flag is set under the keepalive's stop_minting lock before the settled tip is read, so a keepalive is either already counted as pending or waits for the verdict. The hold is released at the phase's end. Co-Authored-By: Claude Opus 5.5 * 3.x.x: see a landing outstanding at the warm-up deadline through before releasing the keepalives (#638) The keepalive hold was dropped at the phase's end, even when the scheduled block was still searching or waiting for the node. A keepalive due during teardown, or the next phase's tips under --plan tips, could then take the block's height. This was raised in the third Codex review of #652. The phase now sees an outstanding landing through at the boundary, within the drain limit, with the keepalives still held. The wait is boundary time, like the drained restart's wait, and nothing is offered meanwhile. The hold is released only after the landing settles or the limit runs out. An aborted phase does not wait. Co-Authored-By: Claude Opus 5.5 * 3.x.x: let only the node settle a scheduled block the session sent, since a stale-job block can still be offered (#638) A block on block-only work, retired by a same-parent payout replacement, is answered stale-job and still captured and offered (#478). Scheduled blocks bump the payout revision, so this is reachable here. Treating stale-job (or low-difficulty) as the block's verdict released the next tip before the node had answered. This was raised in the fourth Codex review of #652. A block the session sent is now settled only by the node's answer. A refused block the node never sees holds the tips to the boundary and is reported unsettled, with the held tips as unminted; that run has already failed the smoke. The harness tests pin the stale-job capture and the unsettled report. The stand-in run's commit timeout is 1 s, so the boundary's wait is short. Co-Authored-By: Claude Opus 5.5 --------- Co-authored-by: Claude Opus 5.5 * 3.x.x: take an abandoned soak candidate's height from the parent its miner found it on, so a #478 capture without a share row is checked too (#658) (#660) The 10-05 nightly failed the dense-cadence soak's final checks with "abandoned candidate ... has no share row". The check read the height an abandoned candidate competed for from its credited share in qbit_share_ledger. A #478 capture has none. A block found on payout work already superseded at its submit check goes through persist_block_only: its share is deferred until the block confirms, and block-only work defers none. When the other server's block wins the height first, the capture is abandoned "parent superseded" like any lost race, and its deferred share is never credited. The soak drives exactly this revision churn, so the check missed a capture losing its race. qbit-prism-miner now adds block_hash and previousblockhash to each block-target submit event, in display order (codec::hash_display, as the server names blocks). The soak maps each block found to its parent and takes an abandoned candidate's height from that parent's header. It still requires that a landed pool block holds that height, and now fails if an abandoned candidate is no block either miner found. This covers credited candidates and both kinds of capture. Consumers that count block submits through block_target_met (the shipped-image lane) see the same events. Co-authored-by: Claude Opus 5.5 * 3.x.x: mine each clock-jump phase on settled work, so a held block's landing cannot race the phase's first share (#656) (#659) * 3.x.x: mine each clock-jump phase on settled work, so a held block's landing cannot race the phase's first share (#656) The 10-04 nightly failed phase 3 of the wall-clock jump test. Server 0 refused the phase's first share as ledger-confirmation-failed and logged "payout revision changed before share commit" in the same second the node took the held block. held_landing returned as soon as the held block was the node's tip, and the phase mined at once, on work from before two payout-revision moves: the landing's and the servers' observation of the new tip. The share passed its submit check and met the moved revision at the append's fence (#632). Since #581 the holder lands its held block itself as soon as it is released, so phase 3 now has phase 4's shape. Phase 4's refusals were the same race (10-02 and 10-05 nightlies). The test allowed them as REFUSED_AFTER_STEP_BACK and blamed the clock stepping back. Fixture::settled is #640's wait from the transcript replay's current(), now shared. It waits until no candidate is unfinished and the server's /healthz shows the node's tip with payout_state_generation at qbit_prism_cluster.payout_revision. The replay calls it with the same conditions as before. The clock-jump test waits on it before every serving mine and before each held block, and REFUSED_AFTER_STEP_BACK is removed: phases 3 and 4 must accept every share, like every other phase. Co-Authored-By: Claude Opus 5.5 * 3.x.x: count a clock-jump phase's settling against its WORK_SECONDS, so waiting for settled work does not double a server's serving budget (#656) The settle wait added before each mine had its own 30 s, ahead of the 30 s work_on still gave a miner. A corrected server therefore had up to 60 s to serve again, where the test's WORK_SECONDS documents 30 s. Run::mine and held_landing now set one WORK_SECONDS deadline, settle within it and give work_on what remains. The ahead phase's no-work check keeps its own WORK_SECONDS. Fixture::settled takes its time limit, and the transcript replay passes its 30 s as before. Co-Authored-By: Claude Opus 5.5 * 3.x.x: retrigger CI after an unrelated authorization_rpc statement-timeout failure (#656) Shard 2 failed authorization_rpc's authorization_fallback_requires_a_definitive_address_result with "canceling statement due to statement timeout". That test is outside this change, and the same shard passed on 26a866fb. The token cannot rerun a job, so this empty commit runs CI again. Co-Authored-By: Claude Opus 5.5 --------- Co-authored-by: Claude Opus 5.5 * 3.x.x: map 002's legacy share hashes after the migration transaction, in bounded batches with a durable cursor, and record 2 last (#582) (#663) * 3.x.x: map 002's legacy share hashes after the migration transaction, in bounded batches with a durable cursor, and record 2 last (#582) Migration 002 backfilled qbit_prism_share_hashes with one INSERT ... SELECT DISTINCT ON over every accepted share, inside migrate's transaction and under its statement timeout. At the default 15 s it failed from about a million shares. At 4.13M it took 321 s of the 600 s cap, with a per-share cost that kept rising, so the production ledger (65.5M shares) could not be migrated with any allowed setting. 002's file now ends at the table. A fresh or empty source records 2 in the transaction as before. A populated 2.x.x source gets a one-row cursor, qbit_prism_share_hash_backfill (next_seq, end_seq), in the transaction, and 2 is not recorded there. After the commit, before 013 and 017, the runner in ledger/migration/share_hashes.rs maps the legacy shares in batches of consecutive share_seq, under the online runner lock. Each batch is 002's statement restricted to its range: one statement, in a transaction that also advances the cursor, sized toward 500 ms under the server's own statement timeout. The last transaction checks, under the migration lock, that the cursor has passed every row, then drops the cursor and records 2. The mapping is the single statement's, row for row: batches run in ascending order with DISTINCT ON inside each, and ON CONFLICT DO NOTHING applies across them. Until 2 is recorded, every start refuses the database, this build's and every earlier one's. The 2.x.x writer is fenced by 002's lease trigger. So nothing is appended while legacy headers are unmapped, and the append path is unchanged. require_schema_version names the cursor in its refusal, even if 2 was recorded by hand. An interrupted migrate resumes at the cursor: the native path accepts 3 without 2 when the cursor table exists, and still refuses it otherwise. A 2.x.x source that already holds the cursor's name is refused before any DDL. Tests: - The batched mapping equals the single statement's on a ledger that repeats headers across batch boundaries in upper case, with gaps, rejected rows, non-hex IDs and bare 64-digit IDs. - A run killed mid-batch leaves the last committed cursor; starts refuse the database; migrate then resumes to the exact mapping. - A held cursor name is refused before any DDL. - The weekly mainnet-shaped scenario drops its 600 s override and its #582 expected-failure check, returns to 1/16 density, and requires the backfill step. - The cutover rehearsal reports the backfill as its own step. Measured on a mainnet-shaped ledger of 8.25M shares at the default 15 s timeout, migrate completes in 462 s. The backfill takes 348-413 s at PostgreSQL's default memory settings, and 251 s with shared_buffers 4GB and max_wal_size 32GB. docs/prism-rust-migration.md has the section, the readiness rule and the sizing. Co-Authored-By: Claude Opus 5.5 * 3.x.x: test that 002's backfill maps a row past its planned end, and update the weekly job's #582 note (#582) The runner re-reads the ledger's end under the migration lock before it records 2. A row past the end the cursor was planned to is mapped by another pass, not left unmapped. Nothing appends before 2 is recorded, so only a writer that went around every gate could put one there; the new test inserts one while the backfill waits on the runner lock, and checks that it is mapped and the cursor is dropped. The weekly live-regtest job's comment now describes the scenario as it runs after #582: 1/16 of mainnet's rows, about 4.1M, cut over at the server's default statement timeout. No job, step or setting changes. The first push of this PR left the comment out: the token then lacked the workflow scope (#665). Co-Authored-By: Claude Opus 5.5 * 3.x.x: cap and retry 002's backfill batches, let its record wait for its locks, and refuse a pending cursor in the evidence export (#582) From /code-review high on the backfill: - Batch size counts share_seq values, not rows. After a gap or a run of rejected shares, batches are cheap and grew to 200,000, and the next dense range could outlast the statement timeout. BATCH_MAX is now 50,000, under 4 s at the slowest measured rate. A batch that times out all the same is rolled back and retried at half the size; an operator's cancel request (same SQLSTATE, another message) still stops the run. - The recording transaction ran under the pool's 5 s lock timeout after all the mapping was done. Like 013's and 017's records, it now waits for the migration lock and for readers of the cursor table. - The recovery evidence export refuses while qbit_prism_share_hash_backfill exists, as startup does, so a hand-recorded 2 cannot export a partial mapping. The metadata regression checks it. - A start that takes no lock could find the cursor table, then lose it to the recording transaction's DROP. progress() reads that as complete rather than failing with 42P01. A non-table under the cursor's name is refused by name. One catalog probe serves both checks. - The cursor records start_seq. The refusal and the documented percent query state the mapped and unmapped share_seq ranges instead of reading end_seq as a count. - Each batch checks the cursor with a conditional UPDATE after the insert, instead of a separate SELECT ... FOR UPDATE. - The interrupt test terminates the waiting batch first and asserts the durable state afterwards, so the batch's own lock timeout cannot race it on a slow runner. - The docs warn against running an earlier 3.x.x build while the backfill is pending: its remedy for 3 without 2 is to record 2 by hand. Co-Authored-By: Claude Opus 5.5 --------- Co-authored-by: Claude Opus 5.5 * 3.x.x: add a block submission and broadcast kill switch for rehearsals against a real node (#291) (#661) * 3.x.x: add a block submission and broadcast kill switch for rehearsals against a real node (#291) PRISM_BLOCK_SUBMIT_ENABLED, which defaults to 1, lets the production pair run #291's rehearsals against a restored mainnet ledger and a real mainnet qbitd without any block submission or transaction broadcast reaching the network. With the switch off (0, false, no or off): - The submit loop claims no candidate. A found block is still validated, credited and enqueued, but it stays pending and submitblock is never called. process_candidate_inner refuses a claim in any state, so no other caller can offer or settle a held row. - The CTV fanout broadcaster does not start, even with PRISM_CTV_BROADCASTER_ENABLED=1. It is PRISM's only user of sendrawtransaction, submitpackage and the CPFP wallet. broadcast-ctv refuses before it connects, and run_pass refuses before it reads the node. - The frontend's node client and its wallet clients refuse every relaying call before it is written: submitblock, sendrawtransaction, submitpackage and the wallet's own sends. The offer path treats that refusal as provably not sent. - Reporting: check-config leads with a WARNING, and self-check reports a block_submission section and warns on stderr. The health payload and the heartbeat carry block_submission_enabled. qbit_prism_block_submission_enabled reads 0, or -1 until the setting is published, and the hold logs a warning at startup. The setting is per frontend, read at startup, and outside the cluster fingerprint. Compose passes it to both HA frontends. The ops runbook, the configuration reference, the server README table, .env.example and the go-live checks document it. Follow-ups: #664, a database-level hold that fails closed, and #666, an alert on the gauge. Tests: - unit tests for the config, the gauge and the offer classification; - CLI tests for check-config, broadcast-ctv and self-check; - a real-socket test that a held client sends no relaying call; - in-process PostgreSQL tests of the held submit loop and the refused claim; - a real-binary test: a held frontend serves a due outbox for a whole health cadence and its counting fake node sees no submitblock, then the same frontend, relaunched with submission on, offers each block exactly once; - a broadcaster test: a held frontend claims and sends no due fanout, and its enabled peer then sends them all; - a Compose contract test. Co-Authored-By: Claude Opus 5.5 * 3.x.x: time CTV fanout claim leases on the taker's monotonic clock, and label session timeouts and hung refreshes by the dependency they wait on (#654, #655) (#667) * 3.x.x: time CTV fanout claim leases on the taker's monotonic clock, so a database clock step neither double-claims nor strands a fanout (#654) #646 moved candidate claims onto an observed lease (#581), but CTV fanout claims still compared `claim_expires_at` and `next_broadcast_attempt_at` with `clock_timestamp()`. A forward database clock step, which a failover can cause, made a live claim look expired: a second frontend took the fanout over and broadcast it again, and the holder's settlement write was refused. A backward step left a dead holder's claim waiting lease + step, and an attempt or confirmation check scheduled before it waiting its delay + step. Migration 022 (offline, as 021) adds `claim_renewals` and `claim_lease_seconds` to `qbit_ctv_fanout_artifacts`, a partial index over claimed fanouts and the `fanout_claim_observed_lease` capability. Fanout claims now follow #581's contract: - Every fanout claim poll surveys the claimed fanouts and times each version (token, renewals) on the frontend's own monotonic clock, through the same ClaimObserver, now shared in ledger/claim_observer.rs with one instance per kind of claim. It takes a fanout over only after watching the same version for the whole lease, by compare and set on that version, before the lane, which takes only unclaimed due fanouts. A claimed fanout is taken over whatever its schedule unless held at infinity. - The holder is fenced on its token alone. Its lease is its own monotonic deadline: the broadcaster's 90 s attempt deadline is below the 120 s lease every renewal takes, now named constants with a compile-time check. - The survey makes due, at once, every unclaimed fanout whose last write is later than the database clock (a backward step), and a shutdown release keeps its schedule but never later than the clock. - Every statement that clears a fanout claim clears its lease and version. - `ledger::revoke_fanout_claims` replaces the tests' ad hoc `claim_expires_at` rewrites. The lane's claimable predicate is unchanged in meaning, written once and shared by the lane, the takeover and the survey. EXPLAIN on 100k settled fanouts: the survey is two index scans (0.3 ms); the lane's plan and cost are the same as before. Tests: five ledger clock-step tests (forward on a live claim and its holder's writes; backward on a dead holder, on a retry, after one renewal, and on a shutdown release) and a broadcaster test with a forward step in the middle of an attempt. All six fail on b8aeb291 with their headline assertion and pass here. The fanout tests that expired a claim by rewriting `claim_expires_at` now revoke it, and assert that only a takeover fences the old holder. Migration pins, the recovery-evidence script, the pool-acquire inventory and the gated-test list include 022. * 3.x.x: label a session timeout and a hung refresh by the dependency they were waiting on, so a database stall no longer reads as the node's (#655) #646 labelled database refusals backend-database-unavailable but left two paths blaming the node during a database stall. The Stratum session's own timeouts (session allocation, job persistence, job resume, initial job delivery) could not tell what the backend was waiting on, and a template refresh that hung on the database, rather than failing, recorded no database failure, so the stale-readiness refusal that followed named the node. A new crate::waiting module lets work name the steps it waits on the ledger database, into a tracker its timer reads when the deadline passes, while the work is still pending (dropping the work ends its marks). The tracker reaches the work through a task-local, as READ_DEADLINE does, so the MiningBackend trait and its implementations are unchanged. The coordinator marks the ledger allocation, the whole issued persistence (consistent with IssuedPersistenceExpired, so the label never depends on which deadline passes first), each ledger step of resume and of the issuance-authority reads, and every ledger step of a refresh. Node calls, CPU work and admission stay unattributed and keep backend-rpc-unavailable. Marks are never placed inside work shared between tasks: a waiter marks its own await of a shared read, and the resume flight's shared reconstruction keeps its own tracker, which each waiting session follows. The session runs each backend call under waiting::timeout and labels an expiry with StratumError::timed_out. The coordinator registers the refresh in flight with its start and tracker; a submit refused for aged-out readiness is the database's when the latest refresh failed on it (#646), or when the refresh in flight has outlived its deadline, the health timeout, and is still waiting on the database. Tests: allocation, persistence and resume stalled on the database through the real coordinator; a refresh held in its first ledger read past its deadline; controls for a refresh held on the node or inside its deadline, and for a backend stall it does not name; a third-party backend naming its step; the tracker's own semantics. The four regressions fail on 3.x.x b8aeb291 with backend-rpc-unavailable. Docs: the rejection registry, the stratum admission note, the metric inventory and the load harness's counted classes. * 3.x.x: revoke a dead fanout owner's claim in the live lost-reply tests, mark the replacement-lease payout read, and tighten the revocation hook (#654, #655) CI shard 0 on aca451e9: live_regtest's two fanout lost-reply tests waited 60 s for a survivor that, since #654, takes a dead owner's fanout over only after watching its claim for the whole 120 s lease. Their shared expire_dead_lease helper revoked candidate claims but only backdated claim_expires_at for fanouts, which no longer ends a fanout claim. It now revokes both kinds. Both tests pass locally against the pinned qbitd 1.0.0. /code-review high on #667: - prove_published_lease's payout_state read was unmarked, so an issuance under a replacement lease that stalled there still read as the node's. It is now an on_database step. - revoke_fanout_claims touches only claimed fanouts and no schedule. A takeover does not wait for a schedule, and its due_now option could only make held (infinity) fanouts due. - The survey reply decoding and the takeable-claim filter are shared in claim_observer.rs by the candidate and fanout claim polls. - refresh_once_inner's failure flag no longer shadows on_database. - Docs: a claimed fanout is held only at infinity (its own schedule is no hold); push a dead holder's fanout with broadcast-ctv by revoking its claim first; issued-job persistence counts as one database step as a whole, as IssuedPersistenceExpired does. * 3.x.x: mark a persistence's ledger writes rather than the whole persistence, so a session timeout in its repair's encoding or admission is not the database's (#655) Codex on #667 (P2), and /code-review high: persist_issued_job marked the whole issued persistence as a database step. On the compact repair path that includes the repair lock, build-slot admission and the blocking encode, so a session deadline passing during CPU work or admission read as the database's although no statement was pending. The two writes in save_with_dependency are now the marked steps: the issued batcher's save, whose batcher does nothing but write batches, and the compact repair's save. The clock, revision, window and payout reads were already marked where they run. A persistence that outlives the issued job's own deadline stays the database's, as #581 decided (IssuedPersistenceExpired), and docs/prism-rejections.md now says which persistence steps are marked. Test: a persistence whose repair is held in its encode at a session deadline reads Unattributed. With the whole persistence marked it read Database (the test fails); the batch-write stall test still reads the database's. --------- Co-authored-by: Claude Opus 5.5 * 3.x.x: page when a frontend holds block submission, not only once it holds a found block (#666) (#676) #661's kill switch (PRISM_BLOCK_SUBMIT_ENABLED=0) holds every block a frontend finds. A production frontend left that way paged only once a found block had been held for a while: PrismBlockCandidateOldestPendingCritical needs an age of 60 s and then a 3-minute dwell, which may be too late for the block. PrismBlockSubmissionHeld (critical, page=true) pages a minute after a frontend reports qbit_prism_block_submission_enabled != 1, whether or not it has found anything: 0 while the switch holds submission, and -1 for a frontend that has not published its setting. The gauge is served from the cached metrics snapshot, so the rule is gated on a successful scrape rather than a fresh snapshot, and a held frontend keeps paging through a stale one. It reads the gauge's last sample over 2 minutes and requires one successful scrape in that window, so a failed scrape's staleness markers do not restart the dwell. No data and evaluation errors are OK, as for the other paging rules, and an image older than #661 exports no series. Nine promtool scenarios cover the page and its pending dwell, a failed scrape every 50 s, a submitting frontend, the unpublished setting, a failed scrape, a missing series, one held frontend of a pair, and a stale snapshot. The deployment test expects 79 rules and checks the rule's dwell, gauge and gate; the migration table and the review-only qbit-tools patch are regenerated. The rehearsal runbook says to scrape a rehearsal's frontends under another job or network, or to silence this rule and the pending-candidate rules by the rehearsal frontends' instance labels; the configuration reference and the go-live checks name the rule. Co-Authored-By: Claude Opus 5.5 * 3.x.x: capture a found block whose payout revision moves before its share's commit, instead of losing it to the append's fence (#657) (#674) * 3.x.x: capture a found block whose payout revision moves before its share's commit, instead of losing it to the append's fence (#657) A block-bearing share was admitted on the credit path at payout revision R, and its candidate rode in the share's append. append_prepared re-read the revision under ORDER_LOCK. If a same-parent settlement had committed in between, it bailed with "payout revision changed before share commit" before append_in and the candidate insert. The transaction rolled back with the candidate in it, the miner got ledger-confirmation-failed, and nothing enqueued the block: the found block was never offered. #504 (#478) captures a block found on same-parent superseded work only when the supersession is already visible at the submit check. The fence now captures the block instead, in the same transaction. A fenced append's candidate is prepared in two forms before the transaction opens: the credited one, and its capture. The capture is the same document with the solver share as deferred_share, its digest, and the deferred share row. The block and the as-issued balances are skipped fields, so the capture costs one more O(1) serialization, and the transaction still inserts only prepared bytes. When the revision moved, append_prepared writes the capture and commits it. It does so under the same ORDER_LOCK and FOR SHARE read, through the same writer fence, as-issued snapshot and #619 window probe as any enqueue, and never also appends the share. A share without a block keeps the refusal, now the typed PayoutRevisionChanged, still answered ledger-confirmation-failed. AppendResult.captured reports the capture, and SubmitLedger returns Appended::{Inserted, Recorded, Captured}. The share path answers a capture the way it answers a block-only proof. It waits up to share_commit_timeout for the block's disposition, using deferred_share_disposition, extracted unchanged from persist_block_only. The share is accepted once confirmation credits the deferred share, answered stale-job if the capture is abandoned, and answered unknown at the bound. The capture is the append's own work, so an append still running past its acknowledgement deadline captures its block in the background. The capture is byte for byte the row capture_stale enqueues, and a test pins that. So its offer is held to PRISM_CAPTURE_OVERPAY_CEILING_BPS (offered, abandoned or disabled), it lands as issued, and its share is credited once on confirmation. Tests: - coordinator::commit_reconcile_tests, PostgreSQL. The test holds ORDER_LOCK, bumps the revision behind a queued append and releases the lock. Four cases: - a block is captured, offered once, landed with its audit bundle and credited once; integrity is clean, accepted shares equal credited headers, and a resubmission adds nothing; - an append queued past its deadline is answered unknown and still captures; - with capture off the block is abandoned with the disabled decision and never offered; - a plain share writes nothing. Against the base product code the three block tests fail with no candidate ever enqueued. - ledger::candidates::capture_form_tests: the capture form equals capture_stale's preparation. - coordinator::miner_tests::fence_capture: #657's MemoryLedger reproduction, now captured; MemoryLedger models the fence as production does. Co-Authored-By: Claude Opus 5.5 * 3.x.x: #657 review: a resubmitted credited block is a duplicate at the fence, and capture off refuses as before - The fence's capture probes for the share first, with the probe persist_block_only uses. A proof already credited with its candidate, resubmitted as the revision moves, is answered duplicate-share and writes nothing, instead of failing on "candidate identity conflict". - With PRISM_CAPTURE_OVERPAY_CEILING_BPS=0 the fence refuses a block-bearing share as before (MovedRevision::Refuse), as the submit check refuses one whose revision already moved. No capture form is prepared when capture is off. - The capture and the credited append share one commit-gate tail. - persist_share_pass takes the block hash from submit_share, which removes the unreachable no-block arm. The sync-rep guard no longer checks the gate state first. - The disposition poll's retry log names its path. - scripts/prism_pool_acquires.json: the acquire that moved into deferred_share_disposition is classified there, which fixes CI Python shard 2. - Tests: the capture-off refusal (unit and PostgreSQL), and a credited block resubmitted into a moving revision is a duplicate (PostgreSQL, gated). The capture-off docs are updated. Co-Authored-By: Claude Opus 5.5 --------- Co-authored-by: Claude Opus 5.5 * 3.x.x: attribute 016's block solvers in statements of at most 128 blocks, so the migration transaction stays inside the default statement timeout on a cold cache (#672) (#680) * 3.x.x: attribute 016's block solvers in statements of at most 128 blocks, so the migration transaction stays inside the default statement timeout on a cold cache (#672) Migration 016 fills qbit_pool_blocks.solver_* for the blocks a 2.x.x pool found before it, with one probe of qbit_share_ledger_accepted_block_suffix_idx per block (13,566 on mainnet). Its file ran that as one UPDATE ... CROSS JOIN LATERAL statement inside the migration transaction, under the statement timeout. Each probe is two random reads, so on a cold cache the statement grows with the ledger. On mainnet-shaped ledgers, from a quiet disk, it took 6.7 s at 8.25M shares and 11.2 s at 16.5M; under a concurrent writer it took 14.0 s at 8.25M; and #582's rehearsal on a saturated host saw it refused at the 15 s default. 016's file keeps its columns, landing trigger, catalog and functions, and loses the UPDATE. Right after the file, in the same transaction, the migrator makes the same probes in statements of at most 128 blocks, keyset-ordered by block_hash over the blocks without a solver (ledger/migration/block_solvers.rs). The attribution, locks and atomicity are unchanged, and a block whose solving share is not found keeps NULL columns. The longest statement measured 71-592 ms across those runs, and 0.47 s under the concurrent writer. Test: on a frozen 2.x.x source with 300 blocks (three statements), the attribution equals the old single statement's lookup, row for row. That covers the latest accepted share winning, the share_seq tie-break, case folding, rejected shares, missing shares and bare 64-digit IDs. A unit test pins the probe to 016's. Co-Authored-By: Claude Opus 5.5 * 3.x.x: pin 016's batched solver probe to its landing trigger, test the native path, and fix the statement count (#672) From /code-review high on #680: - The unit test reads the solver lookup from 016's landing trigger and requires the batch statement to make exactly that probe, rather than comparing the batch with a hand copy of itself. The trigger attributes every block landed after 016 and the batch the ones before it, so the two must not drift. - A native ledger without 16, with blocks and shares, applies 016 on the native path with the same attribution as the old single statement. - Every statement is counted, the final empty one too, so the logged count agrees with the slowest-statement figure: 107 for 13,566 blocks. - 016's header comment points to the note after the landing trigger, and the docs no longer say the file itself backfills the columns. Co-Authored-By: Claude Opus 5.5 --------- Co-authored-by: Claude Opus 5.5 * 3.x.x: hold block submission for the whole cluster from the ledger, whatever each frontend's switch says (#664) (#682) #661's PRISM_BLOCK_SUBMIT_ENABLED=0 holds only a frontend started with it. On a rehearsal against a restored mainnet ledger, a frontend started without it would claim and offer every pending row. Migration 023 (offline, like 021 and 022) adds a permanent one-row table, qbit_prism_submission_hold, and an append-only journal of each set and clear that changes it, qbit_prism_submission_hold_events, and declares block_submission_hold = 1 so a pre-023 binary refuses the database at connect. While the hold is set: - claim_candidate and claim_fanout claim nothing, in the same single writer-fence statement as before; - reserve_offer_within reads the hold FOR SHARE on the hold's own row and refuses with SubmissionHeld, leaving the row pending. Only submission-hold set and clear lock that row, so a hold commits either before a reservation or after one, and the read takes no lock on the cluster row, so the hold adds no wait behind a landing or blob GC; - the CTV broadcaster refuses its pass and rechecks before every send; a hold set during an attempt hands its claim back unattempted, bounded at 5 s like the shutdown hand-back. qbit-prism-server submission-hold show|set|clear reads only PRISM_DATABASE_URL, so a frontend-only setting left invalid cannot stop an operator. show is read-only, gives up after 15 s and reads pre-023 ledgers; set and clear require --reason and are journaled, and clear is refused while candidates are pending unless --offer-pending-candidates. fatal-state clear, candidates abandon and submission-hold share one operator-reason rule, and writable shares its columns and checks with the hold-aware fence. Health and the heartbeat keep block_submission_enabled as the frontend's own switch and add block_submission_hold as the frontend last read it, in one statement with the payout revision: held is null until the first read succeeds, and a later failed read keeps the last one, so a database outage neither reports nor pages a frontend as held. A running frontend logs when the hold appears and clears. qbit_prism_block_submission_enabled is now live: 0 when the switch or the hold holds, 1 when neither does, -1 when the switch is on and the hold has not been read yet. check-config stays offline and says where the hold is reported; self-check reports submission_hold and warns. PrismBlockSubmissionHeld (#666) now covers the hold and its remedy. The ops runbook documents the hold, its journal and the set and clear procedures for rehearsal and cutover; the migration notes, the configuration reference, the server README and the go-live checks name it. Fixtures, capability rollbacks, the recovery-evidence script and the pool-acquire gate cover 023. Tests: a real-binary test (a frontend started without the switch on a held ledger sends no submitblock until clear --offer-pending-candidates, then offers each block once), PostgreSQL tests of the held claim and reservation, the hold fence (a reservation waits for a hold being set, never for the cluster row), the journal's immutability, the held broadcaster and a hold set mid-attempt, and CLI, self-check, gauge and hold-view tests. Co-Authored-By: Claude Opus 5.5 * 3.x.x: keep the PostgreSQL CI shards' database on a tmpfs, give their ledgers a 60 s statement timeout, and log slow statements (#677) The PostgreSQL shards kept failing pull requests in tests those PRs did not touch, with "canceling statement due to statement timeout": four flakes on 2026-10-05, in authorization_rpc (the migration's catalog fingerprint), candidate_window_switch (two 300,000-recipient balance snapshots) and b275_persistence_measure. A probe on this branch logged every statement over 1 s. On a quiet 2 vCPU runner the heaviest statements take 1 to 4 s, so the ledger's 15 s timeout only breaks in a slow spell. The two failing shard-2 jobs had the slowest end-of-job disk syncs of 129 shard jobs, because Cargo's writeback shares the runner's disk with PostgreSQL's WAL. The PostgreSQL service now keeps its data directory on a 4 GB tmpfs. fsync, full_page_writes and synchronous_commit stay on, as the ledger and the fixtures require, and the container peaked at 0.8 GB of memory. The job sets PRISM_DATABASE_STATEMENT_TIMEOUT_MS=60000 for the ledgers the tests open in process. Production keeps 15 s, and so do the servers and operator tools the tests spawn, which drop inherited PRISM_ settings. authorization_rpc forwards the variable to its env-cleared child, and candidate_dispatch_probe restores the configured timeout after its 100 ms check instead of a literal 15s. A continue-on-error step makes the service log every statement and lock wait over 1 s, and the runner prints that log when the job ends. CI_MIN_SHARES_PER_SEC keeps 250. On the tmpfs service the slowest level was 1,084, 1,056, 1,103 and 670 shares/s in four runs, and 702 in the probe, so 250 is about 37% of the slowest. A fifth run fell to 132 while the runner stalled, as the disk service had five times; #683 tracks that flake. The floor is not loosened, because a floor low enough to pass such a stall would miss the collapse it exists for. docs/prism-integration-test-gate.md describes the CI database. Co-Authored-By: Claude Opus 5.5 * 3.x.x: read only the due fanouts in the CTV claim lane, through a partial index, so a claim never scans settled history (#668) (#679) Settled fanouts stay in qbit_ctv_fanout_artifacts after their 1,000th confirmation, and mainnet adds about 65,000 to 130,000 a year at #521's block rate. The claim lane matched them on its status and schedule predicates and dropped them only after reading them: a sequential scan of the whole table on every claim, which a broadcaster pass repeats once per fanout it attempts. EXPLAIN (ANALYZE, BUFFERS) on PostgreSQL 16, warm cache, prepared, per claim, 7,500 watched fanouts: 63 ms at 100,000 settled fanouts and 357 to 412 ms at 1,000,000 (1 GB heap). Migration 024 adds qbit_ctv_fanout_artifacts_lane_idx, a partial index over the fanouts the lane may still claim (to broadcast or check, or confirmed under 1,000 deep), keyed by next_broadcast_attempt_at. The lane (Ledger::fanout_lane_sql) gathers its candidates into one array first: the due ones as two ranges of that index, never attempted and scheduled at or before statement_timestamp() (stable, so the index serves it as a bound), and the deep checkpoint (FANOUT_CHECKPOINT_SQL) from qbit_prism_fanout_checkpoint_idx. It looks them up by primary key and applies its full predicate and order as before. The only change in behaviour: a fanout falling due during the claim statement itself waits for the next poll. After, same data: 1.6 ms (100,000 settled, 377 due) and 1.8 to 2.0 ms (1,000,000 settled, 377 due), the generic plan included. A claim now costs what the due fanouts cost; the settled history only deepens the primary key. When every watched fanout is due, as when the broadcaster cannot recheck each one every 5 s, it reads them all: 28 ms at 100,000 settled and 38 ms at 1,000,000, against 67 and 412 ms before. 024 is additive like 019 and 020 (no capability, no shutdown proof, not offline) and applied online like 013 (ONLINE_MIGRATIONS). On an existing ledger the runner builds the index with CREATE INDEX CONCURRENTLY after the migration transaction commits and records 24 once it is valid, so no write to the fanout table waits for the build, a found block's landing (persist_fanouts) included: about 0.6 s at 1,000,000 settled fanouts. A fresh or empty source applies the file inside the transaction. It is 024 because #682 took 023, the offline block submission hold: an upgrade from 22 applies 023 in the migration transaction and then builds 024 online. Tests: - ledger_postgres::fanout_lane_plan::a_fanout_claim_poll_never_reads_the_settled_history plans and executes both statements of a claim poll, the survey and the lane, prepared, over 20,000 settled and 500 watched fanouts, through five custom plans and the generic one: no sequential scan of the fanout table, the lane index read, at most 16 fanout rows visited by the lane (lossy-bitmap rechecks counted), the survey within the watched fanouts. The pre-024 lane fails it ("scanned the fanout table"), and so does a single IS NULL OR range filter ("visited 508 fanout rows"). - ledger_postgres::fanout_lane_plan::migration_024_builds_the_lane_index_without_blocking_fanout_writes upgrades a ledger at 23 while a fanout write is open: the build waits for it, another fanout write lands meanwhile, then the index is valid, every required version is recorded and the lane claims through it. An in-transaction build fails it with a lock timeout. - ledger::fanout::tests pin the lane's copied SQL without a database: the candidate predicate against 024's index predicate, and the checkpoint against the one FANOUT_CLAIMABLE_SQL admits. Migration pins, the share-hash backfill test's online set, the cutover rehearsal's step label, the recovery-evidence script and the gated-test list include 024; prism-ledger-ops.md and prism-rust-migration.md describe the lane, its costs and the upgrade. Co-Authored-By: Claude Opus 5.5 * 3.x.x: drive frontends the harness did not launch, through their load balancer, with mergeable per-machine stats (#291) (#662) #291's failover drill, load-balancer exercise and soak must load the deployed frontends behind the operator's TCP load balancer, and qbit-prism-load could only drive frontends it launched itself. qbit-prism-load external drives the harness's own Stratum sessions at any frontend or balancer. They subscribe, authorize as
.-s and mine real proof of work on the jobs the target sends, each at the difficulty the target advertised for it (client::DifficultySource::Advertised). The open-loop token bucket offers --rate shares a second only to sessions that hold work, so an outage is shortfall, never a burst on recovery. Each session is offered at most once a 1 ms tick, so a late tick (a suspended process) also owes shortfall, not a burst. Nothing is resolved or connected without --i-understand-external-target. A job advertised above --max-difficulty (2^-16 by default, 2^-14 at most) is never mined, so a mainnet frontend gets nothing. Each process writes a qbit.prism.external-load.v1 document: - shares/s, accepts and rejects by reason; - ACK latency percentiles from log-linear histograms; - reconnects with their outage, and time to first job; - tip fan-out and the offer accounting; - a per-wall-clock-second timeline. qbit-prism-load external-merge adds several machines' documents exactly. It refuses a document given twice, or one no run could have written. --share-log writes each submit's ledger share_id, for reconciliation against qbit_share_ledger. An external session's search runs off the runtime. It stops as soon as its connection has something waiting or the run stops. A share is therefore not sent on work a job already waiting has retired, and an outage is timed when it happened. The harness's own sessions (DifficultySource::Configured) behave as before. The README lists the frontend settings a target needs: the share difficulty, the vardiff floor, vardiff off or capped at the clients' ceiling, and the connection and initial-job admission limits. Tests: - external_target: 20 tests, database-free, against a Stratum target that checks every share's proof of work. - A unit test that a tick offers each session once, however fast sessions free up. - The gated external_frontend: two in-repo frontends behind a round-robin balancer, one SIGKILLed and relaunched mid-run. Two external processes run, their documents are merged, and their share logs are reconciled against qbit_share_ledger. Follow-ups: #673, #678, #685. Co-Authored-By: Claude Opus 5.5 * 3.x.x: fence earlier builds off a pending share-hash backfill with the share_hash_backfill_pending capability, so recording 2 by hand cannot let one serve (#669) (#681) * 3.x.x: fence earlier builds off a pending share-hash backfill with the share_hash_backfill_pending capability, so recording 2 by hand cannot let one serve (#669) #582 records migration 2 only after 002's share-hash backfill. Every start refuses the database until then, because every build requires 2. But an earlier build that meets 3 without 2 tells the operator to record 2 by hand once 002's objects are present, and then it would serve with legacy headers unmapped. The docs said never to do that; nothing enforced it. While the backfill is pending, the database now declares the capability share_hash_backfill_pending = 1, using the standard capability mechanism: - The migration transaction that creates the cursor declares it, once 006 has created the capability table. - Every resume declares it again (ON CONFLICT DO NOTHING). So a backfill a #663-era build started, or one whose row was deleted, is fenced by the next migrate before its batches start. - The transaction that drops the cursor and records 2 removes it. It is the one capability native code removes, because it describes a state, not a format. Every build before #669 refuses it at connect and at migrate, whatever the migration record says. This release refuses the cursor itself, and so does the evidence export, so recording 2 by hand unlocks nothing. A declaration whose cursor is gone can only be left by dropping the cursor by hand. Every migrate refuses it before any DDL, with that cause instead of the "3 without 2, record it by hand" remedy. The connect gate and the recovery evidence export refuse it too. Tests: - An earlier build's capability list refuses the fence; this build's accepts it. - The evidence export's list of understood capabilities is pinned to NATIVE_CAPABILITIES. - Fence lifecycle: declared with the cursor; kept when 2 is recorded by hand, with starts still refused; declared again by a resume after a hand deletion; removed with the cursor when 2 is recorded. - A hand-dropped cursor leaves the fence, which migrate refuses; with 2 also recorded, migrate still refuses and the start finds 13 and 17 missing. - A fence on a fully migrated database is refused by every start and migrate. - The evidence export refuses a fence without its cursor. Co-Authored-By: Claude Opus 5.5 * 3.x.x: declare a resumed backfill's fence from its runner, under the runners' lock, so an earlier runner cannot leave one behind (#669) Codex (P1) and /code-review high on #681: a resume declared the fence in its migration transaction, which does not take the runners' lock. A runner of a #663-era build that was still mapping could then finish, drop the cursor and record 2 without knowing the fence. This release's runner, queued behind it, found 2 recorded and returned, and the orphaned fence would have locked out a fully mapped ledger. - The migration transaction now declares the fence only for a cursor it creates itself. - A runner that resumes the backfill declares it again before its first batch, after acquiring the runners' lock. An earlier runner holding that lock has finished by then, and no fence is declared that it would leave. - The native path reads the fence from the validated capability inventory (inspect_source_schema). This replaces a second, weaker lookup, and its own cursor probe gives way to a flag set where the cursor is created. - The orphaned-fence refusal now offers a verified way out besides the full restore: remove the declaration once qbit_prism_share_hashes is shown to map every legacy header. The docs give the query. - The docs and release notes say precisely how earlier builds react: a pre-#582 build's first migrate still suggests recording 2, which changes nothing. Tests: - A new test plays the earlier runner: it holds the runners' lock while a resume's transaction commits, then finishes the backfill without touching capabilities. The resume declares no fence meanwhile, and the database starts afterwards. - The lifecycle test watches the resume's runner declare the fence again while a batch waits on an open insert, before removing it with the cursor. Co-Authored-By: Claude Opus 5.5 * 3.x.x: declare the backfill fence only with its cursor, never on a resume, so it cannot meet an earlier build's runner (#669) Codex (P1) on 4f0f004a: moving a resume's declaration into the runner, after the runners' lock, still left a race. An earlier build's migrate that passed its capability check before the fence existed can queue for the lock. If this release's runner declares the fence and then stops on an error, a cancel or a restart, the queued earlier runner finishes the backfill, drops the cursor and records 2 without removing a fence it does not know. Every later start and migrate would then refuse the database. The fence is now declared only by the migration transaction that creates the cursor, never on a resume. Migrations serialize on the migration lock, and no earlier build's migrate passes its capability check once the fence has committed. So a fence never coexists with an earlier build's runner: any runner an earlier build started belongs to a cursor an earlier build created, which carries no fence. The price is that a backfill a #582 build started stays unfenced to its end; the docs say so. Tests: - New: a resume of an unfenced backfill crashes mid-batch without declaring a fence. The test then plays the earlier runner finishing the backfill, and migrate and starts succeed afterwards. - The lifecycle test sees the creation-time fence held through a resume's batches and removed with the cursor. Co-Authored-By: Claude Opus 5.5 * 3.x.x: drop an unneeded mut from the fence lifecycle test (#669) 7671768c's rewrite left the blocker transaction mutable, but it only rolls back, and clippy -D warnings refuses that. Co-Authored-By: Claude Opus 5.5 * 3.x.x: say the backfill fence covers only backfills started from #669 on (#669) Codex (P2) on 7671768c: the release note read as if every pending backfill were fenced. A backfill that a #582 build started before #669 carries no declaration, and a resume does not add one, so recording 2 by hand could still let a pre-#582 build start with legacy headers unmapped. The note now says so, and the guide's list of how earlier builds react says it describes a fenced backfill. Co-Authored-By: Claude Opus 5.5 * 3.x.x: bring the fence's comments in line with its creation-only declaration (#669) One test's doc still said a resume declares the fence from its runner, which 7671768c removed, and the migrator's comments said a resumed backfill is never fenced "again", which is wrong for a cursor an earlier build created. Co-Authored-By: Claude Opus 5.5 * 3.x.x: refuse a newer release's share_hash_backfill_pending as newer, never as an orphaned fence to delete (#669) Codex (P2) on c20fdf29: on migrate's native path, the orphaned-fence check matched the capability by name and ran before refuse_newer_native_database. A newer release that declares share_hash_backfill_pending at another value, with no cursor, was told its declaration was an orphaned fence = 1. The remedy offered was to delete it after checking only the old share mapping, which could remove that release's own safety declaration instead of saying "upgrade the server". The native path now validates capability rows before it reads the record, as connect does. Then the orphan check, and the 3-without-2 check after it, only ever see values this release understands. The orphan check also matches value 1 exactly, here and at connect, where refuse_unknown_capabilities already ran first. The evidence export already raised on any other value inside its loop, before its own orphan check. Tests: both cursorless-fence tests now also raise the fence to 2. The migrated database is refused at start and at migrate as a newer release's. In the 3-without-2 state, migrate refuses it the same way, naming neither the orphan nor the record. Both fail on c20fdf29. Co-Authored-By: Claude Opus 5.5 * 3.x.x: qualify the guide's capability narrative for backfills a #582 build started (#669) The narrative said every earlier build is refused for as long as legacy headers are unmapped. That holds only for a backfill this release started: one a #582 build started carries no declaration, as the backfill section and the release note (37b56cca) already say. Co-Authored-By: Claude Opus 5.5 * 3.x.x: keep migrate's capability checks in their order, matching only value 1 as an orphaned fence (#669) e05038c8 moved refuse_newer_native_database ahead of refuse_undeclared_native_database and the record check, so the fence's name at a newer release's value would be refused as newer. That broke offer_lifecycle's migrated_database_without_its_offer_lifecycle_declaration_is_refused_at_connect_and_at_migrate (CI shard 2). An edited candidate_offer_lifecycle = 0 must be refused as an edited declaration, before the newer check, as connect refuses it, and it was refused as "a newer PRISM release wrote this database". The order is back as it was. The orphaned-fence check still matches value 1 only, which is Codex's other remedy: at another value the name is a newer release's declaration, refused as any newer capability is and never offered for deletion. A migrated database gets "upgrade the server". With 3 recorded and not 2, the record's refusal comes first, as it does for any newer capability, and the next migrate after 2 is recorded refuses the value as newer. The cursorless-fence test now probes value 2 in both record states; set_fence replaces the inline updates. Co-Authored-By: Claude Opus 5.5 * 3.x.x: leave a newer release's backfill fence, cursor and record to it, reading the fence again before recording 2 (#669) Codex (P2) on 1d3843b7: the runner maps between holds of the migration lock, so a newer release's migrate can commit meanwhile. If that release declares share_hash_backfill_pending at its own value, the final transaction deleted the declaration by name, dropped the cursor and recorded 2, defeating that release's fence. The final transaction now reads the fence again under the migration lock. Absent (a cursor an earlier build created) or 1, it records 2 as before. At any other value it refuses, naming the value, and leaves the cursor, the declaration and the record to the newer release; this release's next migrate refuses the value as newer. Test: a_runner_leaves_a_newer_releases_fence_with_its_cursor_and_record holds a resume's batch on an open insert, declares the fence at 2 as a newer release would, and then requires the runner to refuse with the cursor, the value and the record untouched. It fails on 1d3843b7. The guide's fence section and its refusals table describe the check. Co-Authored-By: Claude Opus 5.5 * 3.x.x: rewrap a fence test comment after 024 (#669) Co-Authored-By: Claude Opus 5.5 --------- Co-authored-by: Claude Opus 5.5 * 3.x.x: settle the SIGTERM drain's block before the next fault, so a settlement-lock hold cannot start ahead of the landing's revision bump (#686) (#691) faults_under_load_drain_an_offer_in_flight_and_outlast_a_settlement_lock_holder failed three times, on two PRs and one 3.x.x push. Each time every retained-work share in the first 3 s of the settlement-lock hold was refused stale-job. The sigterm-drain fault leaves its found block for the relaunched frontend to land. The landing confirms the block and bumps the payout revision, and until work at the new revision is published every share on the old work is refused stale-job. The harness started the next fault as soon as the frontend answered /healthz. When the bump came just before settlement-lock took SETTLEMENT_LOCK, the holder also blocked the republication. SigtermDrain now settles before the next fault, like the SIGKILL's landing wait and #640's gate. Once the frontend serves again, it polls the block's row until it is submitted. Then it reads qbit_prism_cluster.payout_revision and every frontend's /healthz until each one is ok, publishes payout_state_generation equal to that revision, and has delivered that work to every authorized session. The recovery window waits for the settle, bounded at 60 s, and the phase's time bound includes that wait. A new check fails the drain when it did not settle. A row that goes terminal without landing ends the wait at once, and a block the shutdown itself landed counts as landed. The evidence reports the landing and settle times and the last reading. Locally, pinned to two CPUs, the per-PR test passed 20 of 20 before and after. With the hold started 3 s earlier (baseline=0), 12 of 20 passed before, every failure with CI's message, and 20 of 20 after, with no refusal in any hold. The block landed 0.5 to 3.3 s, and every frontend's work was current 2 to 6 s, after the frontend served again. #692 tracks the same guard for the other faults that land a block. Co-Authored-By: Claude Opus 5.5 * 3.x.x: document the post-offer payout-revision race's ALERT lines as benign, and when to act on them (#630) (#696) Every post-offer write of a claimed block candidate is fenced on the payout revision its claim observed: the landing, the finish as submitted and the settlement as orphaned. Any frontend moves that revision when it records a new tip or confirms, deactivates or matures a block. That happens right after an own block is accepted, when a refresh records the block as the new tip and confirms it once its audit has landed. A write that loses that race is refused. The frontend logs "ALERT: offered candidate could not land" or "... could not be processed", with a reason ending "payout revision changed while observing chain state", and the row settles in reconciliation. Its next claim observes again and finishes it, and the confirmation and the deferred-share credit commit once, whichever side commits them first. The ledger-ops runbook now explains what those lines mean and why they are benign, how they count in the candidate census and alerts, and when to act on them. The alert assurance holds for a freshly found block: the gauges measure a row's age from when it was created or reserved, so an old row, or one with many attempts behind it, can fire an alert even when the race clears on its next retry. The rule is to act only if the block is still listed after a few retries or its reason changes, and never by abandoning it. The product fix, retrying the write in place, stays open in #630. Co-Authored-By: Claude Opus 5.5 * 3.x.x: async promotion with live miners: account for every offered, ACKed, rejected and unknown share, keep sessions and work, and land a block on the promoted ledger (#474) (#620) Refs #474 (slice B). Test-only. A gated live_regtest drill, pg_failover_tests::async_live_miners, runs an asynchronous PostgreSQL promotion under four scripted Stratum miners on two native frontends and a real qbitd. It reuses live_pg_failover.rs's D3 pair, relays and promotion procedure. Every submission is recorded with the answer its miner got. A deferred trigger holds a share's COMMIT across the writer fence. The drill then reconciles exactly: - every acknowledged share is on the old primary; - every acknowledged share the promoted ledger lacks lies in the replication gap and is named; - no refusal was committed, and nothing sent after the fence was acknowledged. After the promotion, the same sessions submit on their current jobs and on work they kept from before the cut and from inside the gap. A surviving share replayed while its parent is the tip is a duplicate. The block solved on a job issued in the gap must be refused as stale with its share uncredited, or land paying only promoted rows. Since #645 fixed #619, which this drill found, it is refused before its offer, so the drill submits it; the opt-in reproducer is folded in. A real block then lands. Its audit must equal an independent read of the promoted window, which holds sequence numbers lost shares had. Every later answer must match the promoted ledger's credits. ShareClient gains solve and submit_solved, so a miner can keep solved work and submit it later. Taken over per #626 and rebased onto 3.x.x. Codex asked that the refused gap block be held to zero credit, and it is. /code-review high found the duplicate replay depended on the 3 s stale grace once the final block moved the tip, so it now runs earlier. Its other findings are in #697. The drill passed 27 local runs against PostgreSQL 16 and the pinned qbit 1.0.0. Two runs under heavy host load failed in its setup; those races are in #697 too. Co-Authored-By: Claude Opus 5.5 * 3.x.x: measure a throughput level that falls below the floor once more, and fail only if both passes miss (#683) (#693) share_append_throughput_floor failed whenever the 2 vCPU runner stalled during one level's few seconds of appends: once at 132 shares/s on the tmpfs service and five times on disk, each time with the other levels normal (#683). A floor low enough to pass such a stall would no longer catch the 3-4x collapse it exists for, so CI_MIN_SHARES_PER_SEC stays at 250. The CI-sized test now measures a level that falls below the floor a second time, with fresh writers whose ids carry a -retry suffix, and only that pass decides. A stall misses once; a collapse of the append path misses on both passes and still fails. A helper, measure_level, runs the passes, and a database-free test drives it with scripted passes. The first pass stays visible: the level's line names its rate. The report's results[].first_pass keeps the whole first pass in the shape of a level entry, lock evidence and writers included, and the top-level share_count counts both passes. The summary line lists first_passes_below_floor, and the failure message gives both rates when both passes miss. A foreign backend during a first pass still marks the run contaminated. If the second pass fails, the error report keeps the first pass's entry. The explicit full-size characterisation stays single-pass. The floor's doc says to count first passes when re-deriving it. Locally, on a private PostgreSQL 16 cluster with the test and the cluster pinned to the same two CPUs, an 8 s stall was injected into level 1 by SIGSTOP of every cluster process. Before, the test failed at 162.6 shares/s. After, the first pass read 162.2 and the second 456.9, and the test passed. With a floor no run reaches, it still fails, with both passes' rates in the message and the report. A run with no injected stall, on the loaded build host, hit a real slow spell: the 4-appender level read 241.6 and then 260.6 shares/s, and passed. Co-Authored-By: Claude Opus 5.5 * 3.x.x: reusable release benchmark: A/B driver, D1 comparison summarizer and reference host (#511) (#564) Part of #511 (S4 of #487): the A/B release benchmark that #479 STEP 1 ran by hand, as one command and a summarizer. qbit-prism-load-compare holds two builds' repeats of one preset to #473's D1 rule: exit 0, no shortfall, no refused valid share, no unanswered submit, and ACK p99 within the limit, in every gated phase. It renders one table per D1 phase and each run's own gates. Every counted run must have driven what the preset pins, as this harness's parser and planner derive it: topology, window, server settings, realism, churn, node, cadence, tips, blocks, reconnects, the memory floor and each planned phase's rate, length and completion. A figure a report does not carry fails rather than counting as zero, and a report naming another build's revision is refused. A pinned count is a count of distinct entries, and the session count is the distinct sessions holding work when the load began, which the harness now reports. scripts/prism_load_ab.py builds each ref in release mode in its own worktree and runs the pair under a per-series and a host benchmark lock. It adds a cooldown and a load gate before each run, alternates which build goes first, and records the host. Every process left below it is stopped on every exit path. A build whose harness predates a pinned flag runs without it only where legacy-flags.json shows it ran that value anyway; otherwise the pair is refused. That includes --ctv-settlement, which builds before #548 ran off. The release preset pair resolves against 5d0042f6. docs/prism-release-benchmark.md names the reference host and the preset pair, and docs/mainnet-deployment.md gets the checklist line. Taken over per #626. 3.x.x was merged in, and the last three Codex findings were fixed: distinct tip and block hashes, distinct sessions instead of connection events, and incomplete phases. /code-review high then found two places where the new checks read differently from the counts beside them, and those were fixed too. Codex's later findings, seven hardening checks against a hypothetically regressed candidate harness, are deferred to #700. Three of them cannot occur on the release preset pair. All of qbit-prism-load's tests passed locally, with its gated binaries against PostgreSQL 16 and the pinned qbitd, as did the Python driver tests. #511 stays open: one recorded A/B series on the reference host is still to run. Co-Authored-By: Claude Opus 5.5 * 3.x.x: answer a share refused by the payout-revision fence stale-job, not ledger-confirmation-failed (#675) (#698) A share whose payout revision moves between its submit check and the append's read under ORDER_LOCK is refused by the append's fence before any write. Since #674 the refusal is a typed PayoutRevisionChanged, but it still reached the generic Failed arm and was answered ledger-confirmation-failed ("share was not confirmed by the database"), with a "share persistence failed" WARN. Its work was superseded, which the submit check itself answers stale-job a moment later; the ledger is healthy. persist_share_pass now maps that refusal to Superseded(PayoutRevision), beside the existing stale-authority gate mapping. The miner is answered stale-job (code 21, "stale job"), still never credited, and the payout_revision stale-job cause is counted once. A plain share's refusal is counted, not logged, like the submit check's. With capture off, a block-bearing share is refused the same way, and its found block is never offered, so a WARN names the lost block. The refusal no longer feeds PrismShareAppendFailures, which fired around every revision bump at production share rates. The load harness classes these refusals as expected landing races instead of backend refusals. The tests that pinned the old answer now pin stale-job, the cause count, and the absence of a persistence WARN. The rejection, metric and PayoutRevisionChanged docs say where the refusal is counted. Co-Authored-By: Claude Opus 5.5 * 3.x.x: production-window matrix (L3) on the version bump, release-candidate, v* tags and weekly (#550) (#608) Refs #550 (S5 of #487). prism-load-l3.yml runs #447's production-window matrix as checked-in suites (crates/qbit-prism-load/presets/suites.toml): - l3-full: 11 cells x 3 repeats on the 32 vCPU class. It runs on a same-repository PR that changes VERSION, on the release-candidate label, on a v* tag and on dispatch. - l3-reduced: 400k at 1, 2 and 4 frontends, weekly on 3.x.x. #473's found-block and dense-cadence cells become presets. qbit-prism-load-compare --collate renders one build's suite into #473's tables with #511's summarizer. It writes a complete/passed verdict, and lists any missing, foreign or edited run as failing the suite rather than dropping it. A tag promotes the newest finished same-repository PR run of l3-full whose verdict is complete for the tagged tree (scripts/prism_l3_promote.py), and carries that verdict; otherwise it runs the suite. check_e2e_scenarios.py gains a suites evidence key beside 3.x.x's lane_checks. Taken over per #626 and restacked onto 3.x.x after #564. The new presets pin --ctv-settlement false and --faults null, as every preset must since #644 and #623. collate fills #564's predates_report_fields. docs/prism-e2e-load-ci.md gets #626's L3 section, which notes that dispatch and the weekly run need the file on main. Spend: once this is on 3.x.x, a v* tag or a VERSION-changing PR starts l3-full (about $42); the coordinator accepted that for the release-candidate bump. Co-Authored-By: Claude Opus 5.5 * 3.x.x: take the settlement-lock fault's lock only on current work and a quiet cluster, whichever fault landed a block before it (#692) (#699) #691 made the SIGTERM drain wait until its block had landed and every frontend served work at the payout revision the landing committed. Other faults land a block too, frontend-sigkill, block-failover and candidate-backlog, and none waits for the republication. A settlement-lock straight after one took SETTLEMENT_LOCK while the landing's work was unpublished, blocked that publication too, and every retained-work share in the first half of its hold was refused stale-job, the same failure as #686. The settlement-lock fault now waits, from the start of its injection, until every frontend is ready, publishes payout_state_generation equal to the cluster's payout_revision and has delivered that work to every authorized session, with no backend holding or waiting for SETTLEMENT_LOCK, so the holder is not queued behind a settlement still under way. The wait is bounded at 60 s, included in the phase's time bound; a cluster that never goes quiet gets the lock at the bound. A found block still to land does not hold the wait: every bump of the payout revision takes SETTLEMENT_LOCK, so none can happen under the lock. The holder reads the payout revision in its own transaction once it has the lock, and the fault reads every frontend's /healthz once more before the hold starts. A new check passes only if, under the lock, every frontend was ready and publishing work at that revision, so a tip that reached the frontends after the last reading fails the hold instead of passing it. The evidence records when the cluster was quiet, the last reading before the lock, the revision under the lock and the reading under it; the hold's ACK summary is now taken from the lock's acquisition. The wait is the drain's work half, moved into a WorkCurrentWait both faults use, with unit tests for its states. Locally, pinned to two CPUs, a preset copy running frontend-sigkill,settlement-lock with baseline=0 failed the settlement-lock fault in 8 of 8 runs before, each with "0 of 60 accepted (stale-job 60)". With the quiet wait it passed in all 36 runs, 10 of 10 at the final head, where the lock queued at most 0.02 s behind a quiet reading and the hold's ACK p99 was 9-19 ms. On the real node, with two frontends and 500 sessions, pool-exhaustion then settlement-lock passed with the cluster quiet at once and every frontend serving the locked revision. Co-Authored-By: Claude Opus 5.5 * 3.x.x: version 3.0.0-rc.1 and its release notes (#557) (#703) Set VERSION and [workspace.package].version to 3.0.0-rc.1, the first release candidate of 3.0.0. Cargo.lock and the fuzz crate's own lock move only their workspace-member entries; no third-party dependency changes. doc/release-notes-3.0.0-rc.1.md describes the candidate: - what it is: the tree #291's go/no-go evaluates, not a production release, run with both submission holds for a rehearsal; - the highlights since 2.0.x; - the upgrade contract: D5, confirmed-only block counts, the retired settings, migrations through 024, the qbit 1.0.0 pin; - how a candidate is verified: CI and L3 on the version-bump PR, the tag promoting L3 on a tree match (#608), the nightly set dispatched by hand, and the evidence bundle on the tag's pre-release (#557); - the issues and #291 criteria still open when it was cut. The development-line notes name 3.0.0-rc.1 as the tree's identity, and they and the 3.0.0 notes link to the candidate's notes. The text was reviewed twice and checked against #284 (U3). L3 run 37417900875 (l3-full, 33 runs) measured this tree and passed every run. Co-Authored-By: Claude Opus 5.5 * 3.x.x: let the read tier's scrapes in flight finish before it stops public-api, and record why a request failed (#701) (#702) faults-long-real-node failed twice, at c95febb3 and at the rc.1 candidate, each time on exactly one public-API request in its last fault, block-failover (#701). The cause was the harness, not the public API. At the fault phase's end run.rs stopped the read tier, and the stop killed public-api at once. The scrape still in flight failed, and its sample, timed from its start, landed in the last fault's window, which runs to the phase's end. The request counts fit in both runs. The read tier now stops scraping at the phase boundary (stop_scraping()). It finishes after that phase's own end measurements and before any later phase or the teardown stops a process its scrapes read (finish()). finish() awaits the scrape loop and waits for every scrape in flight to record its sample, bounded by twice the client's 5 s total timeout. Only then does it kill public-api, reaped on a blocking thread. Nothing is dropped. A wait that gives up is counted in read_tier_process.scrapes_cut_off and written to stderr. A failed request is also recorded with its cause chain, redacted and capped at 400 characters, not only reqwest's URL line. A body that could not be read keeps the status it followed. "public-api was not healthy within 60 s" names the last attempt's status or cause. Tests: - finish_lets_the_scrapes_in_flight_finish_before_it_stops_public_api uses a stand-in public process and a slow endpoint. It requires every in-flight scrape answered while the process still ran, and fails if finish() kills first. - the_wait_for_scrapes_in_flight_is_bounded covers the bound. - a_failed_request_records_why_it_failed covers the cause chain. A local run of the fault smoke with the read tier on reported scrapes_cut_off 0. Co-Authored-By: Claude Opus 5.5 * 3.x.x: #555's session points and #521's mainnet shapes in the weekly reduced L3 (#555) (#614) Refs #555 (S17 of #487), part a. The weekly l3-reduced suite now also runs #555's session points and #521's mainnet shapes, nine jobs a week on the 32 vCPU class: - session-point-5000- and session-point-10000-sessions-400k-2fe-async: throughput-400k-window-2fe-async at 5,000 and 10,000 sessions, with admission at sessions per frontend + 16. A test holds them to differ in exactly those two flags. They are capacity-envelope points, not D1 verdicts: they gate reconciliation and steady_state shortfall. Refused and unanswered shares are reported, not gated. - mainnet-shape-{130,650,2600}-addresses and rental-churn-bursts-and-storms. The manifest marks session-points and mainnet-shapes-reduced-l3 as running. The 50,000-recipient trigger is written down in the README: the 10,000-recipient point costing more than 25% more per recipient than the 1,000-recipient run. That point still has no preset; since #548 (#644) it can be written with --ctv-settlement on. Taken over per #626 and restacked onto 3.x.x after #608. Both presets pin --ctv-settlement false and --faults null. docs/prism-e2e-load-ci.md gets #626's l3-reduced row, cost and capacity-envelope paragraph. Co-Authored-By: Claude Opus 5.5 * 3.x.x: stream the recovery-evidence export with FETCH_COUNT in the script, the runbooks and the rehearsal harness (#705) (#706) Without FETCH_COUNT, psql buffers a whole result set before printing a row. On the restored production copy (65.9M shares), the documented recovery-evidence export reached 63 GB RSS and was OOM-killed twice on a 62 GB host. With FETCH_COUNT=10000 the same export ran at about 4 MB. scripts/prism-recovery-evidence.sql now sets FETCH_COUNT to 10000 when the caller left it unset, so every psql caller streams, and a caller's -v FETCH_COUNT= is kept. FETCH_COUNT counts rows, not bytes, so the audit_bodies and audit_snapshots exports, whose rows can be megabytes, fetch 16 rows at a time and then restore the caller's value. SET LOCAL cursor_tuple_fraction = 1.0 keeps psql 16's cursor batches on whole-result plans, and the comments warn that those psql versions batch only statements starting with SELECT or VALUES. The five export commands in docs/prism-ledger-ops.md and docs/prism-rust-migration.md pass -v FETCH_COUNT=10000 and say why, which also covers copies of the script from before this change. The cutover rehearsal harness ran the same psql without FETCH_COUNT and collected all of its stdout in memory before writing it to a file. It now streams psql's stdout straight into the file, spawning and waiting because tokio's output() always pipes stdout, and runs the script's own default. The frozen 2.x backup and restore test also exports one row per fetch and requires the same evidence, so every result crosses batch boundaries. No server code changes; batching changes how rows are delivered, not what they are. Co-Authored-By: Claude Opus 5.5 * 3.x.x: check legacy carry-forward chains per payout program, as 2.x keeps them, not per case-sensitive label (migration 025) (#708) (#710) 2.x keeps carry-forward balances per payout script: qbit_current_carry_forward_balances() groups by p2mr_program only, and 3.x's native writer reads its prior balances from that function. The legacy (unmarked-row) rule of qbit_carry_forward_integrity_mismatches() instead summed each running balance within (miner_id, payout_order_key, p2mr_program). Stratum keeps an address as typed, so a miner who re-authorized in uppercase split one script into two case labels. On union's ledger that gave 5,506 false mismatches, and 3.x self-check and fatal-state clear refuse any mismatch. Migration 025 replaces the function, changing only the legacy rule's window to PARTITION BY p2mr_program. The marked rule, the current-balance drift check and the summary table are unchanged, and migrate re-runs 025 whenever it re-runs 011. Run read-only over all 2,753,849 active carry rows of the restored union copy, the program rule finds 0 mismatches. The recovery-evidence summarizer clears a database finding only when the 2.x per-label rule also makes it, so source (2.x) and migrated (025) summaries of the same ledger are equal, and any other finding still fails. Gated tests cover a two-label interleaved fixture, real chain breaks in either label, self-check, fatal-state clear and evidence equality; each fails without the fix. A legacy recovery fixture that paid three miners from one program, which 2.x cannot write, now gives each miner its own program. Co-Authored-By: Claude Opus 5.5 * 3.x.x: gate the weekly soak's resident memory for real, now that #627 fixed #600 (#714) The weekly 5.5 h soak at c95febb3 (run 37403129136), which includes #627's post-landing malloc_trim, passed every resident-memory row. load-fe-0 peaked at 1,026.9 MiB against a warm-up peak of 880.3 MiB (1.17x, window-peak slope +8.65 MiB/h), and load-fe-1 at 1,292.7 against 1,141.3 MiB (1.13x, -15.75 MiB/h). Every other gate passed too. The job failed only on the expected-failure marker, which reported that #600 looks fixed. So soak-weekly's rss_expected_failure goes back to null. The test in tests/soak_plan.rs now checks every preset with a soak block and every soak-gates file, at least four configurations, so an expected resident-memory failure can't come back in any of them unnoticed. docs/prism-soak.md now describes the marker generically and records #600's history with the first weekly run's margins. This changes only the harness preset, the test and the docs, with no server code. Refs #600. Co-Authored-By: Claude Opus 5.5 * 3.x.x: add a sidecar tool that proves legacy audit bodies whose segment range digests use Python's escaped JSON (#709) (#715) 1.x and 2.x hashed each segment_range and segment_prefix share part as the sha256 of Python's json.dumps, whose default ensure_ascii=True writes non-ASCII text as \uXXXX escapes. 3.x serializes the same shares with serde_json, which writes raw UTF-8. So a range whose shares carry a non-ASCII worker name fails in 3.x, and import-audits stops on a body whose content is intact. N1 hit this on 339 union-era bodies in the cutover rehearsal. scripts/prism_legacy_range_sidecars.py lets the cutover import those bodies through the canonical sidecars, which import-audits already reads first. It takes block_hash,body_uri,audit_bundle_sha256 rows. For each row it first runs the 3.x canonicalizer on the body as it is. If that fails, it classifies each range part by the serialization that reproduces its digest, and rewrites the failing parts as inline parts that hold the slot's shares. It then proves the result with the canonicalizer, which requires sha256(canonical bundle) to equal audit_bundle_sha256. With --write it publishes the sidecar atomically, by hard-linking a complete, fsynced file. It never replaces an existing sidecar, and it exits 3 on a sidecar that import would refuse. The tests cover: - body-ref and v2 bodies written the 1.x way; - ASCII bodies, which need nothing; - path resolution, done as import does it; - every existing-sidecar case; - a publication that can't complete; - bad rows. A test against the real binaries runs in CI's rust-tests job and in make test-prism-legacy-range-sidecars. docs/prism-rust-migration.md documents the procedure, and the restore step in docs/prism-ledger-ops.md links to it. This is tooling only: a script, its tests, a Makefile target, one CI step and docs. The product fix is for 3.x to accept the escaped form itself; that is #709, after GA. Refs #709. Co-Authored-By: Claude Opus 5.5 * 3.x.x: answer a share whose commit gate closed for unavailable authority as backend-rpc-unavailable, not ledger-confirmation-failed (#716) (#718) A share admitted under a tip-change lease checks its chain authority at the pre-COMMIT hook with non-blocking try_reads. While the tip poll, a fee or readiness write, or the publication holds or is queued for one of those locks, the read fails and the gate closes as AuthorityUnavailable. Nothing is written. rc.1 answered such a share ledger-confirmation-failed, which is what PrismShareAppendFailures counts, so the alert fired at tip changes under load. N2's D1 run saw 4 such refusals in 500/s for 300 s across 7 tip changes. A failed try_read proves neither stale work nor a ledger failure, so the share is now answered the way admission answers readiness it can't prove. The reason is backend-rpc-unavailable, or backend-database-unavailable when the latest refresh failed on the ledger database, with "current chain state is unavailable". The share is still never credited. StaleAuthority stays stale-job. A deadline closure, or a refusal that records no closure, stays ledger-confirmation-failed. When a lease fence closed the gate and its rollback is still in flight at the 15 s deadline, the fence's closure now decides the answer; before, the deadline overrode it. The INFO line "share commit gate refused before COMMIT" now carries the closure. PrismShareAppendFailures matches only ledger-confirmation-failed and ledger-outcome-unknown, so this contention no longer fires it. The 5%-per-reason ratio alert still counts both reasons, and the load harness already classifies them as backend refusals. A new test covers every closure kind and the database case, and a deadline-race test fails on the old code. The authority_lease expectations and the candidate_window_qualification allowance are updated, and prism-rejections.md, prism-native-metrics.md and the metric metadata describe the new answer. The credit-preserving fix, retrying outside ORDER_LOCK instead of refusing, stays open as #716. Refs #716. Co-Authored-By: Claude Opus 5.5 * 3.x.x: version 3.0.0-rc.2 and its release notes (#557) (#717) Set VERSION and [workspace.package].version to 3.0.0-rc.2: rc.1 plus migration 025 (#708, #710), #718's commit-gate answer for #716, and the legacy-audit-body sidecar tool (#715). Both Cargo.lock files move only their workspace-member entries. doc/release-notes-3.0.0-rc.2.md covers: - the changes since rc.1 (U3's lines, and #716 fix A in the coordinator's words); - upgrading from 2.0.x (the sidecar tool and FETCH_COUNT) and from rc.1 (migrate before any rc.2 frontend starts); - the lean verification, since the runtime changes are 025 and #718; - the known issues, including #711, #712, #713 and #704. The development-line notes name 3.0.0-rc.2 as the tree's identity, they and the 3.0.0 notes link both candidates' notes, and a doc example's tag reads v3.0.0-rc.1. L3 run 37444397320 (l3-full) measured this tree. 32 of 33 runs passed. throughput-400k-window-2fe-async-3-blocks-r2 refused one valid share at the commit gate (#716's open part, answered backend-rpc-unavailable), and the coordinator carried that verdict as a named exception. Co-Authored-By: Claude Opus 5.5 * 3.x.x: cut the share append's ORDER_LOCK hold from nine client round trips to four (#711) (#719) A share append held ORDER_LOCK, the advisory lock every share serializes on, across nine client round trips after the lock was granted. On the pair, the frontend reaches PostgreSQL through the docker bridge, haproxy and verify-full TLS, so each round trip is expensive. Mostly client wait, the hold set the ceiling that missed D1's peak gate through the real frontends (window p99 1,116 ms against 1,000 ms). A new share now sends three statements and COMMIT under the lock, with no migration: - The first is one SELECT. It reads the write guard's columns and the cluster row's payout_revision (FOR SHARE OF c when fenced), and the probe's bounds and header lookup as a materialized CTE. - The second is the bounded share_id probe, a materialized CTE, UNION ALL the fallback the first read chose (a retained rejected sequence or the legacy range). The fallback is gated by a one-time NOT EXISTS filter, so it runs only on a bounded miss, and both branches keep their bounds as parameters. - The third is one statement of data-modifying CTEs: the ledger clock UPDATE, the share INSERT and the header-mapping INSERT. The "job from the future" check moved into the INSERT's WHERE. A duplicate in the recent leaves takes two statements and COMMIT. A credited header's unbounded full-parent probe stays a statement of its own. The lock stays a statement of its own: under READ COMMITTED, reads in the lock's own statement would use a snapshot from before the lock was granted. check_writable still refuses a halted cluster or a live legacy writer before the revision is compared. The fenced read's FOR SHARE still compares the latest revision. When a writer committed to the cluster row while that read waited (probe_current is false), the append reads the probe again in a statement of its own, so the probe comes from a snapshot taken after the wait, as when the fence was separate. No trigger fires on an INSERT into either table, and the header mappings have no foreign key, so the merged writes write the same rows. Behaviour is unchanged for: - 23514's missing-partition retry and 23505; - #482's cancel handling; - the capture path; - the commit gate; - the OrderHold metrics; - the settlement's deferred-share credit. New tests: - the wire-level statement count under ORDER_LOCK, fenced, unfenced and duplicate; - a future-job refusal that leaves the clock, the sequence and the mappings untouched; - guard refusals ordered before the fence; - a fenced append waiting out a committed and a rolled-back revision writer; - the probe reread after that wait. The pruning test is rewritten for the new statements; the bounded probe still removes 4 of 4 leaves at executor startup. Locally, throughput_floor ran 18-37% faster direct and 51-75% faster through one extra TCP hop. Refs #711. Co-Authored-By: Claude Opus 5.5 * 3.x.x: export the recovery evidence in parallel from one exported snapshot, byte-identical to the serial export (#712) (#721) The recovery-evidence export took 85 minutes on the 65.9M-share union copy, mostly in the shares and share_hashes kinds, and psql's FETCH_COUNT cursor never gets a parallel plan. scripts/prism-recovery-evidence-parallel.py prints exactly the bytes that the serial psql -f scripts/prism-recovery-evidence.sql prints. It uses sessions that share one exported snapshot, so the summarizer and every runbook cmp stay unchanged. A coordinator session runs the script's preamble and calls pg_export_snapshot(). Each worker imports the snapshot, runs the preamble again and exports one part. A part is a share_seq range of the shares, a header-hash range of the share hashes (COLLATE "C", as the script orders them), or the kinds around them. The driver cuts the script itself at those two exports and adds only a WHERE, so no query is copied. It refuses to run if those statements change. Sampled range boundaries balance the work. The first and last ranges are open, so every row lands in exactly one part, and each range sorts under SET LOCAL work_mem. The export fails closed. With FETCH_COUNT set, psql 16 exits 0 when it can't write its output, so each part's rows go through a pipe that the driver writes, and a full disk fails the export. The completion marker is printed only after every session, the coordinator's commit included, has succeeded. Sessions never prompt for a password, and SIGTERM, SIGHUP or ^C stops every session and removes the parts. The only change to the SQL moves the has_* \gset before the shares export. It prints nothing. A gated test compares the parallel export with the serial one byte for byte, on a frozen 2.x source and on its migrated three-partition ledger, at 1, 3 and 8 jobs and at FETCH_COUNT 1. It checks that the serial output equals the script before this change, and that a broken integrity report leaves no parts and no marker. Python tests cover the cut, the predicates and the failure paths. Locally, 8 jobs took 24 s against 115 s serially on a 6M-share 2.x copy, and about 12 s against 57 s once migrated, every run byte-identical. Refs #712. Co-Authored-By: Claude Opus 5.5 * 3.x.x: version 3.0.0-rc.3 and its release notes (#557) (#722) Set VERSION and [workspace.package].version to 3.0.0-rc.3: rc.2 plus #719 (#711 fix B: the share append holds ORDER_LOCK for 3 statements plus COMMIT, down from 9 client round trips) and #721 (#712: the parallel recovery-evidence export). Both Cargo.lock files move only their workspace-member entries. doc/release-notes-3.0.0-rc.3.md covers: - the changes since rc.2, with N2's pair measurements on a build whose runtime is #719's head, each figure saying what it measured; - upgrading, with no migration from rc.2 and the parallel export for a 2.0.x cutover; - the same evidence set as rc.2; - the known issues: #716's open part with its three measured rates, #711 still open for fix A, and #712's remaining parts. The development-line notes name 3.0.0-rc.3, and they and the 3.0.0 notes link all three candidates' notes. Co-Authored-By: Claude Opus 5.5 * 3.x.x: listen with a 4096 backlog (PRISM_STRATUM_LISTEN_BACKLOG) instead of mio's 128, so a reconnect burst doesn't overflow the accept queue (#723) tokio's TcpListener::bind listens with mio's fixed backlog of 128, the standard library's value, and every frontend listener used it. On the pair, 400 simultaneous Stratum connects through HAProxy overflowed the accept queue 175 times (TcpExtListenOverflows). HAProxy's observe layer4 then marked the frontend down, and 158 sessions moved to the remote backup frontend and stayed there. Miners that reconnect together, at a cutover, a frontend restart or an LB failover, can outrun the accept loop by more than 128. listen.rs's bind_listener binds with an explicit backlog. It keeps TcpListener::bind's semantics: resolved addresses are tried in order, the first that listens wins, SO_REUSEADDR is set and IPV6_V6ONLY keeps the system default. Both Stratum listeners take PRISM_STRATUM_LISTEN_BACKLOG, which is revived from the retired 2.x setting. It defaults to 4096, and the kernel caps it at net.core.somaxconn. The audit/operator and public HTTP listeners use 1024. The startup log reports the backlog and the kernel's cap, and warns when the cap is lower. A test sends 300 connects to a listener that never accepts. With mio's 128, 133 complete; with the helper and a backlog of 512, all 300 complete. config_cli pins the setting's range. Co-Authored-By: Claude Opus 5.5 * 3.x.x: version 3.0.0-rc.4 and its release notes (#557) (#725) Set VERSION and [workspace.package].version to 3.0.0-rc.4: rc.3 plus #723 (the Stratum listeners listen with PRISM_STRATUM_LISTEN_BACKLOG, default 4096, instead of mio's fixed 128). Both Cargo.lock files move only their workspace-member entries. doc/release-notes-3.0.0-rc.4.md covers: - the change and the pair's C5 finding (175 accept-queue overflows in a 400-connect burst, frontend A marked down, 158 sessions spilled to the backup); - upgrading: the setting is read again (rc.1 to rc.3 did not read it), no migration, somaxconn at least as high as the backlog, and removing the always-passed setting before rolling a frontend back to rc.3 or earlier; - the same evidence set as rc.3, with the pair evidence rc.3's notes cite carried; - the known issues carried from rc.3 and the #724 flake. The development-line notes name 3.0.0-rc.4, and they and the 3.0.0 notes link all four candidates' notes. Co-Authored-By: Claude Opus 5.5 * 3.x.x: summarize the recovery evidence in parallel, byte-identical to the serial summarizer (cutover window) (#728) The recovery-evidence summarizer parsed and canonicalized every row on one thread, at about 29 MB/s. #721 parallelised the export, but the summary of the 65.99M-share copy still took about 45 minutes, and on the migrated side that is on the cutover's critical path. prism-recovery-evidence.py --jobs N parses newline-aligned byte ranges of the export in N worker processes. Each worker returns the canonical bytes of each run of rows, plus the few fields the checks need. The parent replays the results in file order through the same summary object the serial path uses. Every digest, count and check is therefore the serial summarizer's: the completion marker, the integrity-report order, share sequence order and the carry-integrity chains. A range a worker cannot read exactly as a text-mode read would (invalid UTF-8, an unexpected exception, or a failed worker) is read serially from there on. A refusal near an undecodable byte falls back to a full serial pass, so the exit code, stdout and stderr always match --jobs 1, which is the serial summarizer. Memory is bounded by jobs + 2 chunks in flight. New tests compare exit code, stdout and stderr between --jobs 1 and --jobs 2-4. Range boundaries go down to every line, across every kind, refusal, encoding edge and fallback. The gated recovery harness summarizes its fixture export serially and in parallel and requires identical bytes. On a 3.25 GB synthetic export, 8 jobs ran 6.5x faster than serial, with identical output. Co-Authored-By: Claude Opus 5.5 * 3.x.x: list the legacy audit rows import-audits refuses for 2.x's window-proof parts digest, and write their sidecars in shards (#731) (#732) 2.x wrote each audit-bundle.v2 body's window-proof share_parts_digest_hex over its share parts in insertion order. 3.x recomputes it with sorted keys and refuses every such body (#731). import-audits reads a row's canonical sidecar before its body, so a v2 body without one stops the import with "window proof share_parts_digest_hex mismatch". On union, the bodies from before 2026-08-24, about 9,500 of them, have no sidecar. scripts/prism_legacy_parts_digest_check.py lists those rows without reading any slot. It follows import's precedence, so a sidecar in the import root wins, and it recomputes a v2 body's digest both ways. It writes the refused rows in the sidecar tool's --rows format and can run in parallel. It exits 0 only when no row is refused for this digest and no body is missing, unreadable or for another digest. scripts/prism_legacy_range_sidecars_sharded.sh runs scripts/prism_legacy_range_sidecars.py --write over those rows in disjoint shards, each with its own CSV, work directory and report. It exits 0 only when every shard succeeds and no temporary file is left in the import root. The sidecar tool publishes atomically and never replaces a sidecar, so shards can't collide, and a rerun verifies instead of rewriting. The tests use a v2 body and its two slots, written by 2.x.x's own writer. The checker must call the body 2x-order-only. The real canonicalizer must refuse it, and accept it once its digest is recomputed with sorted keys. The runner must turn copies of it into sidecars that decompress to the committed bytes. CI runs these tests beside the sidecar tool's, and the migration guide documents the steps. Co-Authored-By: Claude Opus 5.5 * 3.x.x: a go-live gate that checks every self-check fact but legacy audit completeness while import-audits runs (#734) (#736) In production, self-check stops at legacy audit completeness and skips its local checks until import-audits has stored every legacy body (#734). A cutover that admits traffic while the import still runs therefore has no self-check to gate on for hours. scripts/prism_legacy_import_admission_gate.sh checks every fact self-check checks, decided as self_check() and self_check_local() decide them, except legacy completeness. It reports that as pending, along with whether an import-audits session can be seen. Native audit rows that self-check would count as incomplete still fail. The checks: - check-config; - the live-instance census, classified as live_instances() does; - the submission hold; - carry-forward integrity; - durability; - the offer standby, when its flush wait is on; - healthcheck; - the highdiff floor probe, authorized with self-check's own username fallbacks. Every query runs in a read-only transaction, and the database password reaches psql through PGPASSWORD, never a command line. The tests run the gate with stand-ins for qbit-prism-server and psql: every pass and fail path, the census, the standby and highdiff switches, and a local Stratum listener. A gated test runs it against a freshly migrated database. The migration guide describes when to use it. Co-Authored-By: Claude Opus 5.5 * 3.x.x: let the import admission gate run its integrity report on a hot standby, or skip it, under statement and lock timeouts (#738) (#740) The gate's check 5 ran qbit_carry_forward_integrity_report() on the writer with no statement timeout. At production size the report takes about a minute, and a snapshot held that long on a primary with live miners piles up dead versions of the cluster row every share updates, which slows the share append's lock (#738). PRISM_INTEGRITY_REPORT_DATABASE_URL now runs the report on a hot standby instead. The gate refuses that URL when it is not in recovery, is not a replica of this writer's database (system identifier and database name), is on another timeline than the writer, replicates a writer that waits for apply, has hot_standby_feedback on, has replayed no transaction, or is stale: its last replayed transaction is older than PRISM_INTEGRITY_REPORT_MAX_AGE_SECONDS (default 300). The live frontends' heartbeats keep the writer committing, so the age always applies. WAL positions are recorded but never used to call a standby fresh, since after a failover they aren't comparable across timelines. The gate warns unless replay is paused or both conflict delays are -1. PRISM_GATE_INTEGRITY=skip skips the check with a WARN. Every query now runs with a 15 s statement timeout and a 5 s lock timeout. The report gets 120 s on the writer and 600 s on a standby. A failed check names a statement timeout, lock timeout or recovery conflict without echoing psql's stderr, which can name the host. The standby's credential reaches psql through a subshell's environment, and a blank URL counts as unset. The Python shards already ran the gate's hermetic tests. Its real-schema tests now run in PostgreSQL shard 3, where a missing input fails them as it fails the gated Rust tests. Co-Authored-By: Claude Opus 5.5 * 3.x.x: compute the carry-forward integrity report's mismatches and drift once each, as migration 026 (#737) (#744) 001's qbit_carry_forward_integrity_report() counts qbit_carry_forward_integrity_mismatches() and qbit_carry_forward_current_drift() in two scalar subqueries and lists them in two more, so each runs twice. Both are CPU-bound. On a production-sized ledger, 2,753,849 active carry rows on a standby with replay paused, EXPLAIN ANALYZE put the validator at 20.29 s and the drift check at 9.14 s, and the report took 57.42 s. self-check, fatal-state clear and the import admission gate all wait for it, and a clear holds its transaction open that long while the chain tip may move. Migration 026 replaces the report in place. It keeps 001's subqueries word for word, the active row count and the two listings, and evaluates each once in a MATERIALIZED CTE, in 001's order: the count, the validator, then the drift check, so a run that fails stops where 001's does. Each set's count is its listing's length. Running each check once is estimated at about 29 s at production size, the sum of the two parts, not a timed run; on a fixture of 400,000 clean carry rows the report took 1.2 s instead of 2.3 s. The report feeds the recovery evidence's source and migrated summaries and the admission gate, so it must be evaluated as 001's is. Listing a materialized copy of each set, the first plan, is not. 001's mismatch order, block height then carry_forward_seq, ties for every finding without a carry row, whose seq is NULL, and a manifest that names an account twice repeats a seq. 001 lists tied findings in the order the validator's own sort leaves them, since its aggregate reads that sort as presorted. Rows read back from a CTE are sorted again, and once that sort spills it can reorder ties: with 2,000 findings at one height and work_mem at 64kB, that design and 001 differed at 1,405 of 2,000 positions. 001's own order of ties changes with work_mem, so no tiebreak is added. 026 is applied in the migration transaction with no capability and no shutdown proof, as 025 is, and an earlier binary accepts the unknown migration with a warning and reads the same report. migrate runs 026 again whenever it runs 001, as it runs 025 again after 011. 001 runs only without 3, and a recorded history without 3 is refused before any DDL, so for now that is only a fresh or 2.x.x source, where 26 is missing anyway. Two gated tests cover it. One checks the report against 001's, installed beside it, on empty, clean, legacy-break, as-issued and drift fixtures and on the 2,000 tied findings at the default and the smallest work_mem. It counts one run of each set per report, checks that a report whose sets both fail fails in the validator, as 001's does, and fails against the per-set CTE design. The other checks that 26 is recorded once, survives a migrate that runs 011 and 025 again and one with nothing to do, and is applied once to the record an earlier build left, and that a record with 26 but not 3 is refused with 026's report kept. ledger_2x's undo_006 restores 001's report with its version, and every version pin gains 26. Co-Authored-By: Claude Opus 5.5 * 3.x.x: let migrate --defer-share-hashes serve with 002's share-hash backfill pending once it has mapped the recent range (#743) On a production-sized ledger the share-hash backfill of migration 002 is most of what migrate does, and in the W1 cutover it must not sit inside the outage. migrate --defer-share-hashes now maps only the recent range before anything serves: every accepted legacy share whose template height is within 1,000, the coinbase maturity, of the highest one, from the first such share to the cursor's end, in 002's batches restricted to that height. It then raises the backfill's fence, share_hash_backfill_pending, from 1 to 2 in one transaction under the migration lock, after checking that nothing appended, and records the range's bounds beside the cursor. The step runs in the slot the backfill takes otherwise, before 013 drops the index that serves its bounds; 013, 017 and 024 follow with the cursor present. Plain migrate keeps rc.4's behaviour: the whole backfill, then 2. Serving on the recent range alone is safe because a native share is credited only on a job this cluster issued, whose parent is the tip or, within stale grace, the tip's parent. A header commits to its parent, so every copy of a legacy header has the same template height, and a credited native share can repeat only a header in the range unless the chain reorganized deeper than coinbase maturity survives. Every such header is mapped before anything serves, so the append's existing header check stays exact, and the range holds every copy of each header it maps, so the earliest copy still wins. At 2 every start of this release accepts the database with 2 unrecorded and logs a warning naming the cursor; the one start gate serves frontends, tools and operator commands alike. rc.4 understands the fence only at 1, so it refuses the database at connect and at migrate. A fence without its cursor is refused at 1 or 2, naming the value, and a value past 2 is a newer release's. A pending backfill at 2 is the operator's to finish: frontends started with PRISM_POSTGRES_INIT_SCHEMA=1, tools and operator connects leave it alone, and only plain migrate, through the new Ledger::connect_migrate, maps the rest while frontends serve. It runs after any other pending online migration, and its end stays where the cursor was planned: every row past it is a native share, which mapped its own header as it was appended. Before it records 2 it checks that the end lies at or below the conversion bound, and refuses, naming the share, if any native share repeats the header of an accepted legacy share, leaving the cursor and the fence in place. That check probes 2.x's header suffix index and the share ID index once per native row. It reads the native shares from the end up to the highest share_seq at its start, in chunks sized like the backfill's batches, each one short statement in a transaction of its own under the pool's statement timeout, and halves a chunk that times out: one statement over days of native shares would outlast that timeout, or hold a snapshot on the serving primary (#738). A share appended after the check starts cannot repeat a legacy header, since every one is mapped by then. A deferred run refuses a backfill a build before #669 started, which carries no fence and is never upgraded, and share-archive restore, and detach and drop of a partition below the cursor's end, refuse while any backfill is pending. The deferred run claims the backfill before it maps anything, with a deferred_at column the code adds beside the cursor, in the migration transaction that creates the cursor or finds it at fence 1. A deferred run that stops before serving is permitted leaves the claim, and then every other connect that migrates, a compose frontend's included, refuses the database as every start does. It runs neither the whole backfill, hours while the operator's retry waits for the runners' lock, nor 013, which drops the index the retry's range reads. A connect that planned the backfill before the claim reads it again once it holds that lock. Only the operator's migrate takes it on: --defer-share-hashes again, or plain migrate, chosen explicitly, which maps everything before anything serves. Without a claim nothing changes at fence 1. The transaction that raises the fence also keeps every native share at or above the cursor's end. It finds the ledger's share_seq sequence by its owning column, and when its next value would be below the end, as explicit inserts or a restore that left it uncalled can leave it, moves it to hand out the end next and logs the old and new values. A sequence that does not lag is not written to, so the cutover's evidence, which compares its last_value, sees the source's. And a plain migrate planned at fence 1, ahead of 017, that finds the fence at 2 once it holds the runners' lock, because a deferred run raised it meanwhile and stopped before 017, refuses before it maps a batch rather than at its record for want of 017's conversion bound. The recent range's first share is found through a materialized CTE: a bare min(share_seq) was planned as a primary-key walk that read the whole legacy table before reaching the range. Fourteen gated tests cover the deferral, the frontends' restraint at fence 2 and after a deferred run that stopped early, a connect queued behind a claim, a finish under concurrent appends with a frozen end, the double-credit refusal for both legacy ID forms and in the last of many chunks, the archive refusals, 013, 017 and 024 with the cursor present, a lagging share_seq sequence and an untouched healthy one, a finish that finds the fence raised while it waited, a resume of a part-way backfill, and the refusal of an unfenced one. The tests that used 2 as a newer release's value now use 3. The migration guide documents serving with the backfill pending. While migration 2's share-hash backfill was pending, share-archive detach and drop refused only a partition whose lower bound lay below the cursor's end, which kept the release table attached for the legacy shares the backfill still had to map. A native partition above the end could still leave: had the deferred backfill stayed pending long enough for a post-cutover partition to become archive-eligible, it could have been detached or dropped. The double-credit check that plain migrate runs before it records 2 reads the native shares through the attached parent, so the shares in that partition would have escaped it, and 2 could have been recorded over a header credited twice there. Detach and drop now refuse every share-ledger partition while the cursor exists, at fence 1 or 2, whatever its bounds. The refusal names the pending backfill and the remedy: finish it with qbit-prism-server migrate, then archive. Restore was already refused at any fence. Plan, seal, archive and verify are unchanged, as is every other archive eligibility condition. The doc comment, the start gate's warning and the migration guide say so. A gated test makes a native partition above the backfill's end ready to leave in every other respect: archived after the release table, sealed, verified and below the online horizon. While the backfill is pending its detach is refused, and so is its drop once it has been taken off the parent by hand. After plain migrate records 2, the same partition detaches and drops. The test that let a lead partition through now expects it refused. With the lower-bound predicate restored, both tests fail. Co-Authored-By: Claude Opus 5.5 * 3.x.x: let fatal-state clear accept a tip that only extended during its run, and bound its integrity report by its own deadline (#737) (#742) fatal-state clear captures the node's tip once and judges every observation against it. It then refused, before the integrity report and again before the commit, whenever getbestblockhash no longer named that tip. At production size a clear takes about a minute, nearly all of it the report, and tips arrive about once a minute, so a drill cleared about one attempt in three. Each failure rolled back for a retry, although a chain that only grew had invalidated nothing. Both checks now accept a pure extension. When the best block differs, the clear walks its headers back to the captured height, where the block must be the captured tip. A block commits to its ancestry, so every block at or below the captured height is then the one the capture saw: every pool-block and deep-fanout observation and the reconciliation built on them still hold, and maturity, computed at the captured height, can only come late. A best block at or below the captured height, another block there and a header the node does not know refuse as a reorganization, and so does a tip more than 1,000 blocks up; any other RPC failure propagates. The one observation the node was never asked for is that a pool block above the captured height is inactive, which the new blocks could falsify, so a moved tip still refuses while such a row exists. The event's reconciliation adds final_tip_hash, which descends from the captured tip, and tip_extended. The observations hold only if the node answered them from a chain through the captured tip. The chain can go over to a fork during the observation loop and come back onto a longer chain through that tip by the first check, which the walk alone would pass with stale observations. So the first check, once the tip has moved, asks the node every observation again and refuses on any changed answer. The check before the commit only walks, since nothing is observed after the first. A chain back on exactly the captured tip by the first check, which takes the fork being invalidated, and a second flap during the re-read itself still pass; the first passed before #737 too. The clear's hard-coded 120 s bound becomes fatal-state clear --timeout-seconds, 10 to 3600 and 120 by default, which clap checks before anything is read or connected. Ledger::clear_fatal_state_within takes the bound, and clear_fatal_state keeps 120 s for its callers. The report no longer runs under the session's statement timeout, 15 s by default. integrity_report_bounded reads the session's statement_timeout and sets a transaction-local one for the report in one statement, and puts the session's value back after it, never RESET, which would leave the server default. The clear gives it the time left in its bound less 5 s and one RPC timeout, for the commit and the closing tip check's node calls, and refuses with under a second left, so PostgreSQL cancels a report that runs out of time before the client gives up and nothing outlives a failed clear. self-check runs it in a read-only transaction with 300 s. Neither needs PRISM_DATABASE_STATEMENT_TIMEOUT_MS raised any more. The fatal-state tests' RPC proxy can now, before a chosen tip check, extend the chain by two blocks, move it onto a fork tip whose own ancestry alone shows the fork, shorten it, run it 1,001 blocks ahead, answer an unknown header, or flap it to a fork during the observation loop; the fake node's header for its tip carries its height. New gated tests cover an extension accepted at either check and when the node grows its own chain, each reorganization refused at either check with the reconciliation rolled back, a flap refused over a deep fanout and over a pool block, an extension refused over a pool block above the captured height, the report's own timeouts in self-check and the clear through the CLI under a 1 s session timeout, and a report that outlasts the bound cancelled by the server before the deadline. A config_cli test checks the flag's range before connecting. The operator guide describes the extension rule, the refusals, the retry guidance, the flag, and self-check's MVCC caveat on a busy primary. The closing tip check runs after the integrity report, and when the tip has moved it walks the new best block's headers back to the captured height: one getbestblockhash plus up to 1,000 sequential getblockheader calls. Before the report the clear kept back 5 s plus one node RPC timeout for it and the commit. A node that answers each header slowly, but within its RPC timeout, could therefore walk into those 5 s and let the bound's deadline fall inside the COMMIT, whose outcome would then be unknown. The closing check now runs under a deadline of its own, the bound less the commit's 5 s. A walk that cannot finish by then refuses with nothing committed and the halt kept, so the UPDATE, the event INSERT and the COMMIT always start with their 5 s. A test proxy mode answers a tip 500 blocks up with a 50 ms delay per header: the clear refuses about 5 s before its 10 s bound. Without the new deadline the same test runs into the bound itself and fails. The clear bounded everything, its COMMIT included, with one outer timeout. The closing tip check now ends 5 s before the deadline, but that left a fixed 5 s for the UPDATE, the event INSERT and the COMMIT. Under synchronous_commit=remote_apply, which the ledger's connections keep when the server sets it, a healthy COMMIT waits on a standby's replay and can take longer. The outer timeout would then cancel it in flight and report a failure whose outcome is unknown. The bounded part now ends with the event inserted and returns the open transaction, with the ORDER_LOCK hold guard. The caller commits it outside the bound and waits for the server's answer. An expired bound is therefore always before the COMMIT: nothing was committed and the halt stays, which its message now says. Only a connection lost during the COMMIT still leaves the outcome to be read from the event table. A deferred constraint trigger that sleeps 12 s inside the COMMIT stands in for the standby: the clear outlasts its 10 s bound and returns the committed event. With the COMMIT back under the bound, that test fails. Co-Authored-By: Claude Opus 5.5 * 3.x.x: let migrate --offline-indexes build 013's and 024's indexes with a plain parallel CREATE INDEX in one transaction when no instance is live (#745) The W1 cutover's migrate step runs with every frontend stopped, yet 013 and 024 build their indexes the online way: CREATE INDEX CONCURRENTLY reads the share ledger twice and waits out every older snapshot, and DROP INDEX CONCURRENTLY runs statement by statement. The plan puts the gain of building them plainly at about 3.7 down to 1.5 to 2 minutes of outage, to be measured in the pair rehearsal. `qbit-prism-server migrate --offline-indexes [--index-build-workers N] [--index-build-memory SIZE]` runs each index migration's plan in one transaction instead. N defaults to 4, at most PostgreSQL's 1024. SIZE is a maintenance_work_mem as PostgreSQL writes it, checked before connecting; without it the run takes 2GB, or the server's setting when that is higher. A build's sort takes about that much in all, divided among its leader and workers. The transaction takes the migration lock and sets a 5 s lock timeout, the workers and the memory for itself. Before any DDL it refuses an instance that has not reported drained or stopped and a live legacy writer lease, naming plain migrate, which builds CONCURRENTLY and records without stopping frontends, as the way out. It then locks the migration's tables ACCESS EXCLUSIVE in name order, runs each rendered definition verbatim as a plain build, drops with a plain DROP INDEX, verifies the declared set and records the version as it commits. Only a lock timeout reads as a transaction still holding a table; any other failure, in the guards or the table lock, surfaces as itself. An interrupted migration rolls back whole, so no invalid index and no partial drop remain while the migrations recorded before it stay recorded, and a rerun without the flag builds concurrently. The switch is the operator CLI's alone. MigrateOptions carries it beside #743's share-hash backfill mode, so migrate takes --defer-share-hashes and --offline-indexes together, as the W1 cutover's migrate step runs them: the recent range is mapped first, before 013 drops the index that serves it, and 013 and 024 then build offline with the backfill pending. Ledger::connect_migrate takes the options, or a ShareHashBackfill alone as before, through #743's shared operator connect; every other connect passes the default, so frontends with PRISM_POSTGRES_INIT_SCHEMA=1 run the same loop and stay on CONCURRENTLY. 013's SQL and registration are unchanged and no migration number is taken. Both modes share the preflight plan, the step loop and the record, and the legacy lease refusal is factored out of migrate_schema. Six gated index_trim tests watch offline runs through event triggers: the rendered definitions run verbatim in the transaction that records the version while an append waits; a live instance or legacy lease is refused before any DDL; an open writer or reader ends the run with 55P03 and nothing changed; a terminated run rolls back whole, after which a start without the flag builds concurrently; the operator's build memory is taken even below the server's, which is otherwise never lowered; and a lock timeout inside the guards reads as itself. A gated share_hash_backfill test runs the cutover's migrate step, both flags in one run on a populated 2.x.x source: the recent range is mapped at fence 2, 013 and 024 build offline with nothing concurrent, and a frontend then starts and serves. Unit tests pin the statements, the lock failure classification and the memory parser, and config_cli pins the flags. The docs describe the mode, its worker and memory needs and starting frontends only after migrate returns, and the isolated-restore runbooks pass the flag. Co-Authored-By: Claude Opus 5.5 * 3.x.x: open the async-promotion drill's gap session on published work, not on the persisted prepared row (#724) (#747) The async-promotion drill opens a Stratum session inside the replication gap and requires its job's prepared window to reach past the last share the standby replicated. It waited for that by polling qbit_prism_jobs every 200 ms for a saved prepared:live-0 row past the replicated share, then opened the session. A new session is issued the work the frontend has published in memory, though, and a refresh saves the prepared row before it publishes the work: reserve_fresh_compact commits the row, the work is checked against the node and the ledger twice more, and only then does publish() install it. A session opened in between is issued the work published before. At the drill's 1 s reanchor a frontend keeps its published work while shares arrive, rebuilding at once only for the first share, so when the gap is reached within about a second of the first share that work is still the first share's window. Required CI failed this way three times, each with a window ending at share 1 or 2 against 22 to 24 replicated (jobs 112476793983, 113062547171 and 113070225231), and #697 lists the same failure from local runs. The drill keeps its wait for the saved row, which still fails first for a frontend that never prepares work over the gap. It then opens the gap session in a loop bounded at 5 s, a few reanchor intervals, while the miners keep submitting in the gap. Each pass is bounded by what is left of the 5 s. It connects, takes the id of the session's first job from its first notify through a new ShareClient::job_id accessor, and reads that job's prepared window. A session whose window does not reach past the replicated share is dropped, and another opens 200 ms later. A first job whose issued row or prepared record is absent fails the drill at once, since issuance saves both before it announces a job. The first session whose window does reach past it solves the gap share and the gap block, which must both be on that job. That check replaces the old re-query of the block's window, which the shared notify made redundant. A replaced session solves and submits nothing, and the book records only the four scripted miners' submissions and kept work, so it and every count built on it are unchanged. The replaced sessions' windows are reported in the drill's summary line, and the timeline marks when the gap session opened. To prove it, a scratch 300 ms sleep between the reservation and the publication, never committed, made the old drill fail 8 of 8 runs with CI's message (last share Some(1) against 22). With the same sleep the drill passed 8 of 8 runs in its first form and 4 of 4 as amended, each run replacing one or two sessions issued the first share's window. Without the sleep, the first form passed 10 of 10 runs unpinned and 10 of 10 pinned to two CPUs, and the amended drill 3 of 3 unpinned and 4 of 6 pinned. Both pinned failures were the other setup race #697 lists, a COMMIT held across the fence answered as acknowledged, which happens after the gap session has opened. Interleaved and pinned, the base, the first form and the amended drill then passed 8 of 8 runs each. This addresses only the gap-window failure; #724's stale-job and backend-rpc-unavailable answers and #697's other items are separate. Co-Authored-By: Claude Opus 5.5 * 3.x.x: finish a deferred share-hash backfill while frontends serve, in throttled batches, and let the evidence export and self-check read it pending (#746) migrate --defer-share-hashes leaves migration 2's share-hash backfill pending at share_hash_backfill_pending = 2 while frontends serve. The new qbit-prism-server backfill-share-hashes maps the rest and records 2. Plain migrate on such a database now takes the same path, at the default throttle, instead of rc.4's unthrottled batches, with or without #745's --offline-indexes, which changes only how 013 and 024 build. A Throttle bounds every batch: at most --max-batch share_seq values (5,000), one statement in a transaction of its own, cancelled at --statement-timeout-ms (2 s, at most 5 s) and retried at half its size, and a rest of took * (1 - d) / d after each, where d is --duty-cycle (0.5). Every native append updates the cluster row under ORDER_LOCK, and a snapshot held on the primary keeps that row's dead versions (#738), so no statement holds one longer than its timeout, and none is held between batches. Each bound is checked in one place, which the flags' parsers call before anything connects. The chunked double-credit check that runs before 2 is recorded keeps to the same throttle. A batch or chunk that times out at its smallest size says to run backfill-share-hashes with a smaller --max-batch, or else a larger --statement-timeout-ms up to the cap; plain migrate's error names that command, as migrate takes no throttle flags. While frontends serve, the transaction that records 2 waits at most 2 s for the migration lock or the cursor table's lock, and its statements keep to the throttle's timeout. An attempt that waits longer is rolled back, holding nothing, logged, and tried again after a backoff from 2 s doubling to 30 s, 20 times, about ten minutes in all. The run then stops, naming what held it and to run backfill-share-hashes again, which goes straight to the double-credit check and the record. The record of a backfill before anything serves is unchanged. The command refuses, before it maps anything, a backfill that does not permit serving, and it shares A1's checks under the runners' lock: the fence and the claim, 017's conversion bound, the frozen end and the double-credit refusal. On a database whose backfill has finished it maps nothing and succeeds, so a retry after a lost reply is safe. It prints what it did as JSON, already_complete included. The recovery evidence export now reads a database at fence 2, as every start does. It exports the cursor as one share_hashes_deferred record in place of the partial mapping, and the summarizer marks the share_hashes kind deferred with it and says so in a note. The preamble, which every parallel session runs, reads only the catalog, and the cursor is read last, after the integrity report, so an export holds the cursor table's lock only until it commits, right after. The refusal of a cursor at fence 1 names migrate --defer-share-hashes to serve and plain migrate to finish; one that a build before #669 started, with no fence, names plain migrate alone. A complete export's summary is unchanged, and the parallel export and summarizer stay byte-identical to the serial ones. The export checks each capability's value against the highest that startup accepts, from one declaration that a unit test compares with NATIVE_CAPABILITIES. self-check samples the backfill beside the heartbeats, in one read of the cursor and its fence under one 5 s deadline, and reports it in a share_hash_backfill field with a warning line, never as a failure: "pending" with the fence and the cursor, or "unknown" with why it could not be read, a connection failure's text withheld. The field is absent only when the read succeeded and nothing is pending. The start gate's warning, migrate --defer-share-hashes's output and the share-archive refusals while the backfill is pending name backfill-share-hashes as the way to finish it. The migration guide and the ledger runbook describe when and how to run it after go-live, its throttle, how to watch it, the record's retries, and how to compare evidence while it is pending. The recovery test helpers now refuse an export script edited after the test binary was built. The serial export runs the copy compiled in, while the parallel exporter and the summarizer read the file, so a full suite whose script changed under it compared two scripts' exports: the deferred export test once failed so, at the line the edit had moved. The record of 2 while frontends serve retried a failed attempt without asking whether it had committed. An attempt can commit and still fail: its COMMIT's reply lost with the connection, or cut short after the commit was written. A retry then met the cursor that commit had dropped, failed on the missing table, and reported a finished backfill as failed. Now every failed attempt is followed, before anything else, by a read of what it left, in one statement of a fresh transaction under the record's own timeouts: whether 2 is recorded, and whether the cursor table exists. With 2 recorded and the cursor gone the backfill is done: the run logs that the attempt's reply was lost or cancelled but 2 is recorded, and succeeds as any finished run does. With 2 unrecorded and the cursor in place nothing committed, and a wait past a lock or statement timeout is retried as before. Either other state was made by hand, since the one transaction that records 2 drops the cursor in the same commit, and is refused loudly rather than retried. A read that fails too stops the run, naming backfill-share-hashes, which finds 2 recorded or records it. The statement timeout still covers the COMMIT. Under synchronous replication a COMMIT waits for the standby with its xid running and its locks held, which #738 rules out for long, and a cancelled wait leaves the commit local, which PostgreSQL reports as done with a warning. The re-check covers every way an attempt can fail after its commit, and a comment by the record's transaction says why. A gated lib test injects the failure through a test-only fault hook, in the pattern of the candidate claims' faults: keyed by schema, it lets the first record attempt commit and then return PostgreSQL's 57014. The run reports the backfill done, with 2 recorded and the cursor and the fence gone. A unit test pins the four states the read can find. The migration guide and the ledger runbook say what the run does after a lost reply. Every read of the share-hash cursor took any 42P01 for the cursor dropped since its look-up, by the transaction that records 2, and read it as nothing pending. But the read can miss another relation it needs: self-check's reads the fence from qbit_prism_schema_capabilities, so a database with its cursor beside a missing capability table, a damaged one self-check should diagnose, read as one whose backfill had finished, and self-check omitted both the field and the warning. Now a 42P01 reads as nothing pending only if the cursor is gone when it is looked up again, in one short statement. Otherwise the read's own error goes on, and self-check reports the backfill as unknown with it. Both readers of the cursor, self-check's and the one every start and migrate makes, go through that one place. The record of 2 has no such shortcut: its re-check after a failed attempt reads the catalog and the migration history, and a failure there stops the run. self-check's backfill test in ledger_postgres renames the capability table beside a pending cursor and requires state unknown with the missing relation named, and the warning. A gated lib test drops the cursor between its look-up and its read, through a test-only fault the record test's hook now shares, and requires both readers to read nothing pending. The lib tests move to share_hashes/postgres_tests.rs. Co-Authored-By: Claude Opus 5.5 * 3.x.x: version 3.0.0-rc.5 and its release notes (#557) (#749) Set VERSION and [workspace.package].version to 3.0.0-rc.5: rc.4 plus #737's fixes in #742 (fatal-state clear accepts a tip that only grew, takes --timeout-seconds, and runs the integrity report under its own statement timeout) and #744 (migration 026: the report computes each set once); the deferred share-hash backfill in #743 and #746 (migrate --defer-share-hashes, then the throttled backfill-share-hashes after go-live); #745 (migrate --offline-indexes, which composes with --defer-share-hashes); the cutover tooling merged after rc.4's tag (#728, #732, #736, #740); and #747, a test-only fix for #724. Both Cargo.lock files move only their workspace-member entries. doc/release-notes-3.0.0-rc.5.md, dated 2026-10-08, covers: - the changes, with the report's estimated 57 to 29 s since 026, the migrate --defer-share-hashes command measured on the pair at 5 min 06 s (539d6c91, before #745), backfill-share-hashes' throttle, record of 2 and safe rerun, the evidence scripts' changes, and the offline index builds' estimated gain; - upgrading: migrate is needed again (a binary-only upgrade refuses "missing migration(s) 26", and this export refuses a database without 026), the cutover's migrate step with both flags, and finishing the backfill after go-live under #738's snapshot rule, with self-check only at low share rates. Rolling back to rc.4 at fence 2 goes only by finishing the backfill first or from the pre-migrate backup or physical copy, and each rc.4 refusal is quoted where it applies: a start meets the cursor first and says to run migrate, which rc.4's migrate refuses. The rules still in force: #737's statement timeout for rc.4's tools, 026 or not, #738's snapshot rule, and fatal-state clear --timeout-seconds, whose bound ends before the commit; - the same evidence set as rc.4's, with each PR's merge SHA, the pair's serving evidence carried, and the rehearsal's full measurements left to the cutover's go/no-go record; - the known issues: rc.4's, minus #737's F1, F2, F4 and F5, plus #737's open part, #738 (its fix planned for rc.6), #739, #741, #731 and #734, with #582 closed and #712 and #724 updated. docs/prism-rust-migration.md now says how earlier candidates refuse a database at fence 2: a start with the backfill's own refusal, whose advice to run migrate holds only for this release, and their migrate, or a start with PRISM_POSTGRES_INIT_SCHEMA=1, with the fence. The development-line notes name 3.0.0-rc.5, and they and the 3.0.0 notes link all five candidates' notes. Co-Authored-By: Claude Opus 5.5 * 3.x.x: re-check a failed record-2 attempt on a fresh connection when its own one dropped (#748) (#750) After a failed attempt to record 2 while frontends serve, the run reads whether the attempt recorded 2 all the same, before it retries. It read on the attempt's own connection. When that connection had dropped, its COMMIT's reply lost with it, the read failed too, so a run whose COMMIT had recorded 2 stopped with "run backfill-share-hashes again". That was safe, the rerun finding 2 recorded, but it reported a finished backfill as failed. Now the read runs on a fresh connection when the attempt's own one is lost: when the attempt or the read on it failed with a transport, TLS or protocol error, or the server ended the session (SQLSTATE class 08, 57P01 to 57P05, 25P03), or the attempt got no reply within its deadline. The fresh connection comes from the run's own connect options, the operator pool's or the migrating connect's: the same URL, search path and TLS. Its read keeps the record's 2 s lock timeout and the throttle's statement timeout. A lost session is most often a restart, a crash or a failover, so a connect or read that meets one is tried again after 250 ms, doubling to 2 s, within the throttle's statement timeout plus 2 s plus a 10 s connect allowance. Each fresh connection is closed again on every path, within a second, or dropped. With 2 recorded and the cursor gone the run succeeds. Only if the fresh read fails too does the run stop, with both failures named. Each attempt is also bounded on the client's side, by its ten statements' statement and lock timeouts plus 5 s. The server's own timeouts end a healthy attempt well within that, so an attempt still without a reply is on a half-open connection, a partition say, whose COMMIT would otherwise wait out TCP's retransmissions for some 15 minutes. It is abandoned and its connection taken for lost. The caller's connection is never replaced. The run reports whether it lost that connection, and the caller then drops it quietly rather than close it, so a lost connection's failing close cannot turn a finished run into an error, and the connection cannot become a session without the runners' lock or the pool's settings. A lost connection that had recorded nothing stops the run instead of a retry. The runners' lock, a session lock on that connection, went with it, and another runner may have taken it since. Retrying would mean taking it again and redoing the checks made under it, the fence, the frozen end and the double-credit check, which is what a rerun of backfill-share-hashes does. The stop says whether the attempt or the read after it lost the connection, keeping the attempt's own error and the read's, and that a COMMIT still in flight there can land moments later. Only an attempt whose own connection answered the read is retried, which one match makes plain. Seven gated lib tests drive the record against a real PostgreSQL, five through the crate's execution proxy. A statement trigger marks the record's DELETE of the fence, since the start gate refuses any trigger on the migration history. - A COMMIT that completes before its connection drops ends in success, with 2 recorded and the cursor and the fence gone. - One whose fresh connects are refused, then told 57P03, reads 2 recorded on the third. - One whose reply is held past the attempt's deadline is read afresh then. - A drop before the COMMIT stops the run with 2 unrecorded, and the rerun records 2. - A test-only fault fails an attempt with 55P03 and terminates the connection before the read: the stop keeps both errors. Unit tests pin which SQLSTATEs and errors read as a lost connection and the client-side deadlines. The migration guide's record paragraph and refusal table and the ledger runbook say what the run does after a lost connection, and that 2 was not recorded when checked. The record of 2 reported a lost connection only on success: the run returned RunConnection::Lost, and the caller dropped the connection rather than close it. When a record attempt lost its connection, or got no reply on it within its client-side deadline, and the fresh read then found 2 unrecorded, the run returned the rerun instruction as an error, and both callers, backfill_share_hashes and apply_online_migration, closed the connection the run had given up on. A close writes Terminate and shuts the socket down. On a half-open connection whose send buffer is full that write could wait on TCP, defeating the attempt's new bound instead of returning the instruction at once. Now the lost state travels beside the result. The run marks a RunConnection out-parameter lost the moment an attempt, or the read after it, loses the connection, before the fresh read and whatever the run then comes to. Both callers close the connection through close_unless_lost, which drops a lost one without a word to the server on success and failure alike. Nothing else closes a lost connection: the fresh read closes only its own connection, within a second, and the operator's pool never held the detached one. A gated lib test holds the reply to the record's DELETE, before any COMMIT, through the execution proxy. The attempt is abandoned at its client-side deadline, shortened to 3 s for its schema, as the held transaction holds the database's migration lock that the binary's other tests need. The run stops with the rerun instruction within a few seconds of that deadline, with 2 unrecorded and the cursor and the fence in place. The proxy, which now records the connections whose client sent Terminate and the relays that ended, sees the lost connection dropped, never closed. On loopback a close of that connection returns at once, so the Terminate is what tells a close from a drop. The record test of a connection lost while checking now also requires it marked lost. Co-Authored-By: Claude Opus 5.5 * 3.x.x: let the async-promotion drill's new-tip shares wait for work at the payout revision the final block's confirmation set (#724) (#753) The async-promotion drill's last check sends a share from each surviving session on work for the tip the final block moved to, and requires it accepted and credited once. It took the first job built on the new tip as that work. The final block moves the payout revision twice on the same parent: once when a frontend first records the new tip (window.rs), and again when the landing confirms the block (blocks.rs). Work published on the new tip between the two carries the first revision. Once the confirmation commits, a frontend answers shares on that work stale-job for the payout revision until it republishes: the landing frontend at once, the others at their next refresh (#752). A session whose first new-tip job came from that work was answered stale-job. Required CI failed this way twice (jobs 112241378522 and 113174599773). Each session now waits until its latest job on the new tip comes from work at the ledger's payout revision. The drill reads the job's prepared record and the cluster's revision on the promoted primary, and reads past a job on superseded work to the session's next one through a new ShareClient::newer_work. A job without its issued row or prepared record is passed over the same way, as the final block's loop does (#619). If the revision moves past a share's work after that check, the refused share is waited out once its ledger row is checked absent. Any other refusal fails, naming the job's and the ledger's revisions. A session may meet work on one superseded revision, possibly on more than one job; a second would be a revision move this drill never makes, and fails. The original symptom stays as a check. The first job on superseded work the drill meets gets one share, which must be refused and leave no ledger row. It is answered stale-job, or unknown-job if the job was retired past PRISM_STRATUM_SAME_TIP_JOB_RETENTION_SECONDS (30 s) and pruned before the drill reached its session: a pruned job is resumed from its durable rows only while its frontend still publishes that payout revision. Every step of the phase (the tip read, the revision and ledger reads, both submissions, the retries and the wait for newer work) runs within what is left of one 30 s deadline, through a small PhaseDeadline helper. An expiry names the phase, the session, the job and how long the phase had run, instead of waiting out the client's 70 s answer timeout. With scratch server delays that reproduce CI's message on every run, the old drill failed 10 of 10 runs and the amended drill passed 6 of 6. Unpinned and pinned to two CPUs it passed 6 of 6 each, and after each Codex fix 3 of 3. One further scratch held the kept share 35 s past retention; it was answered unknown-job and accepted with no ledger row. Only the drill changes; #752 tracks the server's stale-job burst after each of the pool's own blocks. Co-Authored-By: Claude Opus 5.5 * 3.x.x: bound the pool hashrate series' ledger reads at both ends, so no plan walks the ledger (#754) The pool hashrate series serves fully rolled buckets from the rollup tables. It re-reads two kinds of rows from the ledger: the boundary (the current bucket and the leading partial bucket) and the tail (the shares past the rollup watermark). Neither read gave the planner an index range it could use. The boundary bounded accepted_at with an OR and a NULL guard on the range start. The tail bounded share_seq only from below, with the same guard on time and an OR on the miner. On a production-sized ledger (66M shares, rollups current), rc.5's default pool request walked all of p0's seq_walk index for the boundary, removing 66,090,385 rows to return 2,390, in 19-40 s. With the boundary fixed, the generic plan's tail still walked p0's accepted_recent_idx for no row, in 17-27 s. Pool hashrate-series requests therefore ran past the public read deadline: PRISM answered 503 read_timeout, and the proxy in front of it 504. A custom plan also walked a big miner's whole history. 2.x builds the same read with bounded predicates, so this is a 3.x regression. The boundary is now two disjoint slices, each a range on accepted_at bounded at both ends: - [GREATEST(current_bucket, started_at), ended_at]; - [started_at, LEAST(first_full_bucket, current_bucket)), which is empty without a range start. The tail takes its shares from a subquery bounded by share_seq on both sides, by the watermark and by max(share_seq), which is one backward probe per partition. OFFSET 0 fences the subquery, so the time and miner tests only filter its stream. The tail therefore reads only the shares past the watermark: none while the rollups are current, and the lag while they catch up. With no progress row it reads every share, as rc.5 did. On a production-sized ledger the fixed query took 9-23 ms for the pool and 2-15 ms for miners. That held in 20 of 20 runs, across generic plans, custom plans with the anchor pinned or NULL, and auto plans after six executions. Its output equalled rc.5's in 7 of 7 anchored requests, for which rc.5 took 15.8-25.2 s. On a 6M-share copy with a 600k-share miner, every request ran in 1-17 ms at random_page_cost 4 and 1.1. There the output matched rc.5's in 104 cases per watermark state, 312 in all. The only exceptions were unanchored requests whose range start slid while the old query ran, and they differed only in the leading bucket. Two tests in api_2x pin this. - chart_rollups_match_raw_for_every_range_bucket_and_watermark requires the rollup series to equal the raw-ledger series in 78 cases (every range and bucket pair, context, anchor and subject). It checks them before the rollups run, at a current watermark, and with shares past it. - chart_rollups_read_only_the_partial_buckets_under_any_plan runs the chart's plans over 20,000 rolled-up shares, half of them one miner's. It covers the pool at three ranges, a small miner and the big miner, in custom and generic modes, at random_page_cost 4 and 1.1, with the watermark current and again with 300 shares past it. Each plan may read at most 1,000 ledger rows plus the lag. Every ledger index scan outside a Limit probe must bound one column from both sides, and no sequential scan may read ledger rows. It fails against rc.5's query and against the boundary-only version. The change is read-only SQL in the public read path: no migration, no schema change, no write path. Co-Authored-By: Claude Opus 5.5 * 3.x.x: version 3.0.0-rc.6 and its release notes (#557) (#756) Set VERSION and [workspace.package].version to 3.0.0-rc.6: rc.5 plus #754 (the pool hashrate series bounds both of its ledger reads, the boundary buckets and the tail past the rollup watermark, at both ends, fixing a 3.x regression that timed it out on large ledgers), #750 (#748: a failed record of 2 is re-checked on a fresh connection when its own one was lost, and each attempt is bounded on the client's side), and #753, a test-only fix for #724. Both Cargo.lock files move only their workspace-member entries. doc/release-notes-3.0.0-rc.6.md covers: - the changes, with #754's figures from its pull request (19 to 40 s and a 504 on a 66M-share ledger before, and 17 to 27 s with only the boundary fixed; 1 to 17 ms on a 6M-share copy after, under every plan mode and random_page_cost tried, with its output matching rc.5's; and on a production-sized ledger, 9 to 23 ms for the pool and 2 to 15 ms for miners in 20 of 20 runs, against rc.5's 15.8 to 25.2 s, with identical output), #750's lost connections, fresh read, client-side deadlines and two new stops, and #753's wait for current work; - upgrading: rc.6 adds no migration, capability, setting or script, so an upgrade from rc.5 and a rollback to it replace the binary only, at fence 2 too, and a database that rc.5's migrate migrated is served by rc.6 unchanged, which a cutover that pinned rc.5 can rely on. An upgrade from rc.4 or earlier is as in rc.5's notes, which the section links: migrate for 026, fence 2 and the deferred backfill, #738's snapshot rule and fatal-state clear --timeout-seconds; - the same evidence set as rc.5's, with the pair's serving evidence carried; - the known issues: rc.5's, with #748 fixed and #724's drill race and new-tip shares fixed on the test side (#747, #753), plus #752 after the cutover, #738's fix now planned after the cutover, and #739 not in this candidate. The development-line notes name 3.0.0-rc.6, and they and the 3.0.0 notes link all six candidates' notes. docs/prism-rust-migration.md names rc.5 where it said "this release" of serving at fence 2 and finishing the backfill. That is in three places: that every build before rc.5 refuses fence 2, that the starts of rc.5 and later accept it, and that only rc.5 and later finish a pending backfill. In this candidate's tree "this release" would have excluded rc.5, which does both. Co-Authored-By: Claude Opus 5.5 * DO NOT MERGE without coordinator GO: 3.x.x: drop an idle node connection after 20 s, below qbitd's 30 s idle close (#759) (#760) * 3.x.x: drop an idle node connection after 20 s, below qbitd's 30 s idle close, and send each relay call on a new connection (#759) The frontends' qbitd RPC client kept an idle keep-alive connection for 60 seconds, while qbitd closes one after its -rpcservertimeout, 30 seconds by default. A request written into a connection just as the node closes it is never read, and fails after the connection existed. That is the plain "transport failed" error, which nothing retries. On submitblock it is an unknown outcome, and the block is never offered again. A production deployment logs that error from the CTV broadcaster on about one pass in eight to fifteen, which fits this race: the broadcaster's 30 second interval runs in step with the 2 second refresh interval. The client now keeps an idle connection for at most 20 seconds (POOL_IDLE_TIMEOUT), so it drops one before the node can close it. A compile-time assert keeps the bound below QBITD_DEFAULT_RPCSERVERTIMEOUT. node_client_builder() carries the bound, and the public API's node client and the load harness's node clients are built from it. Rpc::with_pool_idle_timeout sets another bound, for a node with a shorter idle close and for tests. Every RELAY_METHODS call now goes through a second client that keeps no idle connection, so each relay opens a new one. A relay can then fail only to connect, which is provably unsent, or on the connection it just opened, which is a genuine unknown rather than this race. A node with a shorter idle close than the bound, or a proxy that drops idle connections, can no longer race a relay. A relay is rare, and the new connection costs one connect. tests/rpc_idle_close.rs runs the real client against a loopback node that drops a request arriving after its idle limit, as the race does, and counts the connections it accepts. With a shorter bound, the next call opens a new connection and is answered. With the old long bound, it reuses the stale one and fails with the production error, which is not RpcNotSentError. A relay call after the pause is answered on a new connection even with the long bound. The migration guide notes the node-side requirement. Co-Authored-By: Claude Opus 5.5 * 3.x.x: keep relay calls on the bounded pool, and build Rpc's client from node_client_builder (review on #760) A new connection for each relay put its connect inside submitblock's 1 second deadline (PRISM_BLOCK_SUBMIT_RPC_TIMEOUT_SECONDS). A deadline that passes while connecting is a timeout, not a connect error, so the block would count as an unknown offer and never be offered again: a lost SYN or a slow accept would cost a block that never reached the node. A warm pooled connection within the 20 second bound is the safer path, so relays share it again, as they did before this PR. Rpc's client now starts from node_client_builder(), like the public API's and the load harness's. The pool_idle_timeout accessor and its test only restated the constant, so they are gone. The docs name a proxy's idle timeout beside qbitd's. Co-Authored-By: Claude Opus 5.5 --------- Co-authored-by: Claude Opus 5.5 * 3.x.x: write each legacy audit before SETTLEMENT_LOCK and bound import-audits (#765) (#770) import-audits no longer holds SETTLEMENT_LOCK while it sends a legacy audit's UPDATE. Each row's parameters are encoded off the runtime before any transaction opens. The UPDATE then runs on a connection of its own, under transaction-local write timeouts (PRISM_IMPORT_AUDITS_WRITE_TIMEOUT_MS), holding only that row's lock. After that the importer polls SETTLEMENT_LOCK with pg_try_advisory_xact_lock for up to PRISM_IMPORT_AUDITS_LOCK_TIMEOUT_MS, and only the write guard and the COMMIT run under it. A SETTLEMENT_LOCK holder waiting on the row preempts it, and a row that exceeds a bound is rolled back and named in the error, so a re-run resumes at it. SIGTERM and SIGINT stop the command between rows, or roll back the row in progress. Each row logs its parameter sizes and phase times on stderr. Fixes #765. Co-Authored-By: Claude Opus 5.5 * 3.0.x: version 3.0.0 and its release notes (#557) (#776) * 3.0.x: version 3.0.0 and its release notes (#557) 3.0.0 is 3.0.0-rc.6 plus #770 (#765) and #760 (#759), the two fixes #772 prepared as 3.0.0-rc.7, which was never tagged. The 3.0.x branch starts at #770's merge, 4c134913, before 3.x.x took the 3.1 dual writer. - VERSION, [workspace.package].version and the workspace-member entries of both Cargo.lock files move to 3.0.0. - doc/release-notes-3.0.0.md becomes the 3.0.0 notes: #772's rc.7 notes as the changes since rc.6, then the D5 recovery contract it held. - The development-line note and the migration guide name 3.0.0. Co-Authored-By: Claude Opus 5.5 * 3.0.x: update rustls, rustls-webpki and webpki-roots for six RustSec advisories cargo deny fails on rc.6's lockfile with RUSTSEC-2026-0346, -0347, -0348 and -0349 (rustls < 0.23.46), RUSTSEC-2026-0382 (rustls-webpki < 0.103.16) and RUSTSEC-2026-0381 (webpki-roots < 1.0.10). Move each to its first fixed patch release with cargo update --precise, in the root and fuzz lockfiles, as #354 did for RUSTSEC-2026-0285. No other dependency changes. Co-Authored-By: Claude Opus 5.5 * 3.0.x: describe 3.0.0 as this line's production release The development-line note's status, not-done and upgrade sections, and the README banner, said no production release of the Rust server existed. With the coordinator's go-ahead for 3.0.0 they now point to 3.0.0 on 3.0.x and its notes, and say 3.x.x continues as 3.1. The 3.0.0 notes list the TLS dependency update. Co-Authored-By: Claude Opus 5.5 --------- Co-authored-by: Claude Opus 5.5 --------- Co-authored-by: Claude Opus 5 Co-authored-by: Dan Hepworth Co-authored-by: Anatolie Co-authored-by: OpenAI Codex Co-authored-by: Robert Clarke --- .env.example | 504 +- .github/actionlint.yaml | 12 + .github/actions/apt-install/action.yml | 56 + .../actions/prism-load-evidence/action.yml | 237 + .github/actions/rust-toolchain/action.yml | 22 + .github/apt/ubuntu-mirrors.txt | 15 + .github/scripts/apt-install.sh | 206 + .github/scripts/install-pinned-tool.sh | 197 + .github/scripts/install-prism-qbit.sh | 112 + .github/scripts/prism-load-run.sh | 131 + .github/scripts/prism-stratum-fuzz.sh | 134 + .github/scripts/test-install-pinned-tool.sh | 834 + .github/scripts/test-prism-image-runtime.sh | 138 + .github/scripts/test-qbit-image-runtime.sh | 51 +- .github/workflows/ci.yml | 491 +- .github/workflows/prism-load-l3.yml | 536 + .github/workflows/prism-load-runner-probe.yml | 233 +- .hadolint.yaml | 5 + Cargo.lock | 3096 +++- Cargo.toml | 13 + Makefile | 115 +- PRISM.md | 1105 +- README.md | 65 +- VERSION | 2 +- compose.prism-external-db.yaml | 19 + compose.prism-ha.yaml | 32 + compose.production.yaml | 41 +- compose.yaml | 305 +- crates/qbit-pool-builder/Cargo.toml | 3 +- crates/qbit-pool-builder/src/lib.rs | 37 +- crates/qbit-prism-load/Cargo.toml | 54 + crates/qbit-prism-load/README.md | 2004 +++ .../examples/ha_qualification.rs | 652 + crates/qbit-prism-load/legacy-flags.json | 137 + crates/qbit-prism-load/presets/aliases.txt | 19 + .../dense-cadence-400k-window-1fe-async.json | 87 + .../dense-cadence-400k-window-2fe-async.json | 87 + .../presets/faults-failover-fake-node.json | 85 + .../presets/faults-long-real-node.json | 86 + .../presets/faults-pr-smoke.json | 85 + .../presets/faults-short-real-node.json | 86 + .../presets/mainnet-shape-130-addresses.json | 87 + .../presets/mainnet-shape-2600-addresses.json | 87 + .../presets/mainnet-shape-650-addresses.json | 87 + crates/qbit-prism-load/presets/pr-smoke.json | 83 + .../presets/real-node-smoke.json | 83 + .../rental-churn-bursts-and-storms.json | 86 + ...n-point-10000-sessions-400k-2fe-async.json | 86 + ...on-point-5000-sessions-400k-2fe-async.json | 86 + .../presets/short-plan-20k-window-1fe.json | 88 + .../presets/short-plan-fake-node.json | 88 + .../presets/short-plan-real-node.json | 88 + .../qbit-prism-load/presets/soak-short.json | 114 + .../qbit-prism-load/presets/soak-smoke.json | 115 + .../qbit-prism-load/presets/soak-weekly.json | 119 + crates/qbit-prism-load/presets/suites.toml | 74 + .../throughput-200k-window-1fe-async.json | 87 + ...roughput-20k-window-1fe-500-addresses.json | 86 + .../presets/throughput-20k-window-1fe.json | 86 + .../throughput-400k-window-1fe-async.json | 87 + ...ughput-400k-window-2fe-async-3-blocks.json | 87 + .../throughput-400k-window-2fe-async.json | 87 + .../throughput-400k-window-2fe-sync.json | 87 + .../throughput-400k-window-4fe-async.json | 87 + .../throughput-500k-window-1fe-async.json | 87 + .../throughput-500k-window-2fe-async.json | 87 + .../throughput-500k-window-4fe-async.json | 87 + ...iners-400k-2fe-retarget-500-addresses.json | 88 + ...elivery-2000-miners-400k-2fe-retarget.json | 88 + .../qbit-prism-load/soak-gates/testnet4.json | 16 + crates/qbit-prism-load/src/artifact.rs | 426 + crates/qbit-prism-load/src/cadence.rs | 1749 +++ crates/qbit-prism-load/src/churn.rs | 985 ++ crates/qbit-prism-load/src/classify.rs | 292 + crates/qbit-prism-load/src/cli.rs | 1114 ++ crates/qbit-prism-load/src/client.rs | 2034 +++ crates/qbit-prism-load/src/cluster.rs | 1142 ++ crates/qbit-prism-load/src/collate.rs | 534 + crates/qbit-prism-load/src/compare.rs | 2718 ++++ crates/qbit-prism-load/src/compare_main.rs | 146 + crates/qbit-prism-load/src/digest.rs | 126 + .../qbit-prism-load/src/external/histogram.rs | 288 + crates/qbit-prism-load/src/external/mod.rs | 1191 ++ crates/qbit-prism-load/src/external/stats.rs | 1701 ++ crates/qbit-prism-load/src/fault/backlog.rs | 508 + crates/qbit-prism-load/src/fault/database.rs | 410 + crates/qbit-prism-load/src/fault/disk.rs | 627 + crates/qbit-prism-load/src/fault/endpoint.rs | 226 + crates/qbit-prism-load/src/fault/failover.rs | 1115 ++ crates/qbit-prism-load/src/fault/frontend.rs | 1522 ++ crates/qbit-prism-load/src/fault/mod.rs | 1149 ++ crates/qbit-prism-load/src/fault/plan.rs | 673 + crates/qbit-prism-load/src/fault/read_tier.rs | 714 + crates/qbit-prism-load/src/fault/rpc_relay.rs | 489 + crates/qbit-prism-load/src/fault/verdict.rs | 1485 ++ crates/qbit-prism-load/src/frontend.rs | 660 + crates/qbit-prism-load/src/gate.rs | 813 + crates/qbit-prism-load/src/gate_main.rs | 99 + crates/qbit-prism-load/src/kill.rs | 322 + crates/qbit-prism-load/src/lib.rs | 45 + crates/qbit-prism-load/src/main.rs | 95 + crates/qbit-prism-load/src/measure.rs | 1349 ++ crates/qbit-prism-load/src/node.rs | 509 + crates/qbit-prism-load/src/preset.rs | 409 + crates/qbit-prism-load/src/profile.rs | 119 + crates/qbit-prism-load/src/provenance.rs | 219 + crates/qbit-prism-load/src/proxy.rs | 194 + crates/qbit-prism-load/src/qbitd.rs | 1869 +++ crates/qbit-prism-load/src/realism.rs | 669 + crates/qbit-prism-load/src/report.rs | 200 + crates/qbit-prism-load/src/restart.rs | 328 + crates/qbit-prism-load/src/run.rs | 5620 +++++++ crates/qbit-prism-load/src/settlement.rs | 255 + crates/qbit-prism-load/src/soak/evaluate.rs | 805 + crates/qbit-prism-load/src/soak/mod.rs | 275 + crates/qbit-prism-load/src/soak/sample.rs | 335 + crates/qbit-prism-load/src/soak/tests.rs | 461 + crates/qbit-prism-load/src/soak_driver.rs | 731 + .../qbit-prism-load/src/soak_report_main.rs | 485 + crates/qbit-prism-load/src/window.rs | 621 + crates/qbit-prism-load/tests/churn.rs | 369 + crates/qbit-prism-load/tests/collate.rs | 306 + crates/qbit-prism-load/tests/compare.rs | 2217 +++ .../qbit-prism-load/tests/ctv_settlement.rs | 185 + .../tests/external_frontend.rs | 422 + .../qbit-prism-load/tests/external_target.rs | 1752 +++ crates/qbit-prism-load/tests/faults.rs | 196 + .../golden/fake_frontend_environment.json | 61 + .../tests/golden/fake_node_transcript.json | 544 + .../tests/golden/fake_side_report_keys.json | 95 + crates/qbit-prism-load/tests/harness.rs | 11538 ++++++++++++++ crates/qbit-prism-load/tests/legacy_flags.rs | 107 + crates/qbit-prism-load/tests/load_smoke.rs | 649 + .../tests/quorum_replication.rs | 101 + crates/qbit-prism-load/tests/real_node.rs | 496 + crates/qbit-prism-load/tests/realism.rs | 1035 ++ crates/qbit-prism-load/tests/soak_plan.rs | 231 + crates/qbit-prism-load/tests/soak_sampling.rs | 164 + crates/qbit-prism-load/tests/soak_smoke.rs | 218 + crates/qbit-prism-server/Cargo.toml | 57 + crates/qbit-prism-server/README.md | 179 + crates/qbit-prism-server/fuzz/.gitignore | 4 + crates/qbit-prism-server/fuzz/Cargo.lock | 2971 ++++ crates/qbit-prism-server/fuzz/Cargo.toml | 51 + .../fuzz/fuzz_targets/codec_parsers.rs | 8 + .../fuzz/fuzz_targets/stratum_lines.rs | 8 + .../fuzz/fuzz_targets/stratum_session.rs | 10 + .../fuzz/seeds/codec_parsers/assembled_block | Bin 0 -> 277 bytes .../codec_parsers/coinbase_split_placeholder | Bin 0 -> 197 bytes .../compact_mainnet_difficulty_1 | Bin 0 -> 24 bytes .../fuzz/seeds/codec_parsers/compact_negative | Bin 0 -> 24 bytes .../fuzz/seeds/codec_parsers/compact_overflow | Bin 0 -> 24 bytes .../fuzz/seeds/codec_parsers/compact_regtest | Bin 0 -> 24 bytes .../fuzz/seeds/codec_parsers/hex_field | 1 + .../codec_parsers/noncanonical_compact_size | Bin 0 -> 198 bytes .../fuzz/seeds/codec_parsers/segwit_coinbase | Bin 0 -> 196 bytes .../seeds/codec_parsers/stripped_coinbase | Bin 0 -> 160 bytes .../fuzz/seeds/codec_parsers/template_json | 1 + .../template_mask_and_transactions | 1 + .../fuzz/seeds/stratum_lines/deep_nesting | Bin 0 -> 404 bytes .../fuzz/seeds/stratum_lines/exactly_at_bound | Bin 0 -> 516 bytes .../fragmented_json_across_writes | Bin 0 -> 53 bytes .../stratum_lines/invalid_utf8_and_controls | Bin 0 -> 77 bytes .../fuzz/seeds/stratum_lines/login_fragmented | Bin 0 -> 133 bytes .../fuzz/seeds/stratum_lines/login_one_write | Bin 0 -> 133 bytes .../fuzz/seeds/stratum_lines/malformed_budget | 5 + .../fuzz/seeds/stratum_lines/odd_ids | Bin 0 -> 325 bytes .../seeds/stratum_lines/oversize_frame_closes | Bin 0 -> 309 bytes .../stratum_lines/trailing_partial_frame | Bin 0 -> 68 bytes .../seeds/stratum_lines/unknown_job_submit | Bin 0 -> 240 bytes .../seeds/stratum_session/backend_failures | 7 + .../configure_after_work_advertises_mask | 6 + .../stratum_session/configure_malformed_masks | 4 + .../seeds/stratum_session/duplicate_share | 4 + .../fuzz/seeds/stratum_session/highdiff_floor | 4 + .../seeds/stratum_session/login_and_share | 3 + .../mainnet_difficulty_low_shares | 4 + .../stratum_session/oversize_then_reconnect | 7 + .../password_difficulty_options | 4 + .../payout_revision_retires_same_parent_work | 5 + .../seeds/stratum_session/pipelined_requests | 11 + .../seeds/stratum_session/protocol_errors | 20 + .../reauthorize_keeps_original_worker | 5 + .../reconnect_resumes_difficulty | 9 + .../reconnect_resumes_issued_job | 6 + .../seeds/stratum_session/session_budgets | 11 + .../stratum_session/short_retention_eviction | 9 + .../stratum_session/suggest_high_difficulty | 7 + .../stratum_session/tip_change_stale_grace | 6 + .../stratum_session/tip_change_without_grace | 5 + .../fuzz/seeds/stratum_session/username_limit | 8 + .../vardiff_retarget_keeps_old_targets | 10 + .../seeds/stratum_session/version_rolling | 5 + .../fuzz/seeds/stratum_session/zero_mask | 5 + crates/qbit-prism-server/fuzz/src/alloc.rs | 60 + crates/qbit-prism-server/fuzz/src/backend.rs | 381 + crates/qbit-prism-server/fuzz/src/checks.rs | 567 + crates/qbit-prism-server/fuzz/src/driver.rs | 202 + crates/qbit-prism-server/fuzz/src/lib.rs | 44 + crates/qbit-prism-server/fuzz/src/lines.rs | 69 + crates/qbit-prism-server/fuzz/src/parsers.rs | 171 + crates/qbit-prism-server/fuzz/src/pow.rs | 84 + crates/qbit-prism-server/fuzz/src/script.rs | 354 + crates/qbit-prism-server/fuzz/stratum.dict | 69 + crates/qbit-prism-server/fuzz/tests/seeds.rs | 552 + .../migrations/002_multi_instance.sql | 138 + .../migrations/003_2x_compatibility.sql | 12 + .../migrations/004_cpfp_retired_funding.sql | 19 + .../migrations/005_candidate_dispatch.sql | 7 + .../migrations/006_source_schema.sql | 138 + .../007_candidate_window_reference.sql | 52 + .../008_prepared_window_reference.sql | 76 + .../migrations/009_wrap_safe_sessions.sql | 27 + .../migrations/010_fatal_state_recovery.sql | 39 + .../migrations/011_offer_before_landing.sql | 802 + .../migrations/012_offer_startup_fence.sql | 12 + .../013_share_ledger_index_trim.sql | 64 + .../migrations/014_policy_transition.sql | 28 + .../015_candidate_orphan_disposition.sql | 291 + .../016_share_ledger_partition_catalog.sql | 544 + .../017_share_ledger_partitions.sql | 31 + .../018_chain_observation_epoch.sql | 15 + .../migrations/019_signing_transitions.sql | 32 + .../migrations/020_payout_divergences.sql | 106 + .../021_candidate_claim_observed_lease.sql | 56 + .../022_fanout_claim_observed_lease.sql | 58 + .../migrations/023_block_submission_hold.sql | 75 + .../migrations/024_fanout_lane_index.sql | 32 + ...025_carry_forward_integrity_by_program.sql | 459 + ...26_carry_forward_integrity_report_once.sql | 114 + crates/qbit-prism-server/src/api.rs | 1144 ++ crates/qbit-prism-server/src/api/charts.rs | 257 + .../src/api/metrics_snapshot.rs | 130 + .../src/api/metrics_snapshot/tests.rs | 599 + .../src/api/operator_auth.rs | 72 + crates/qbit-prism-server/src/api/public.rs | 435 + .../src/api/public_service.rs | 707 + .../src/api/public_service/metrics_tests.rs | 98 + .../api/public_service/readiness_events.rs | 159 + .../src/api/public_service/readiness_tests.rs | 601 + .../src/api/queries/audit_block_payouts.sql | 17 + .../src/api/queries/audit_bundle.sql | 31 + .../queries/audit_bundle_by_commitment.sql | 36 + .../src/api/queries/audit_share_window.sql | 20 + .../api/queries/dashboard_block_markers.sql | 16 + .../src/api/queries/dashboard_blocks.sql | 94 + .../queries/dashboard_hashrate_rollups.sql | 83 + .../api/queries/dashboard_hashrate_series.sql | 23 + .../src/api/queries/dashboard_leaderboard.sql | 103 + .../queries/dashboard_miner_earning_rows.sql | 78 + .../queries/dashboard_miner_payout_rows.sql | 89 + .../queries/dashboard_miner_share_summary.sql | 44 + .../queries/dashboard_miner_worker_rows.sql | 63 + .../api/queries/dashboard_pool_snapshot.sql | 108 + .../queries/dashboard_reward_leaderboard.sql | 115 + .../src/api/queries/read_replica_status.sql | 6 + .../src/api/queries/read_schema_ready.sql | 23 + .../api/queries/recipient_payout_history.sql | 33 + .../qbit-prism-server/src/api/read_models.rs | 629 + .../src/api/response_cache.rs | 340 + .../src/bin/qbit-prism-miner.rs | 366 + crates/qbit-prism-server/src/broadcaster.rs | 1149 ++ crates/qbit-prism-server/src/capacity.rs | 907 ++ crates/qbit-prism-server/src/codec.rs | 765 + crates/qbit-prism-server/src/config.rs | 1073 ++ .../qbit-prism-server/src/config/database.rs | 109 + .../src/config/environment.rs | 26 + .../src/config/native-settings.txt | 163 + .../src/config/policy_transition.rs | 50 + .../src/config/retired-settings.txt | 103 + crates/qbit-prism-server/src/coordinator.rs | 4069 +++++ .../src/coordinator/bundle_build.rs | 130 + .../coordinator/candidate_checkout_tests.rs | 277 + .../src/coordinator/candidate_lease_tests.rs | 1216 ++ .../coordinator/candidate_shutdown_tests.rs | 66 + .../src/coordinator/chain_observation.rs | 326 + .../src/coordinator/claim_release.rs | 333 + .../src/coordinator/claim_release_tests.rs | 363 + .../src/coordinator/clocked_flight.rs | 307 + .../src/coordinator/clocked_revision_tests.rs | 33 + .../src/coordinator/commit_reconcile_tests.rs | 1544 ++ .../src/coordinator/compact_decode_tests.rs | 302 + .../src/coordinator/compact_resume.rs | 310 + .../src/coordinator/compact_runtime.rs | 161 + .../src/coordinator/d2_below_target_tests.rs | 1008 ++ .../src/coordinator/d2_bootstrap_tests.rs | 907 ++ .../src/coordinator/d2_test_support.rs | 295 + .../src/coordinator/fee_policy_tests.rs | 176 + .../src/coordinator/first_job_lane_tests.rs | 359 + .../src/coordinator/issued_batcher.rs | 318 + .../coordinator/issued_batcher_pg_tests.rs | 152 + .../src/coordinator/issued_batcher_tests.rs | 548 + .../src/coordinator/miner_submit.rs | 1095 ++ .../coordinator/miner_submit_acquire_tests.rs | 887 ++ .../miner_tests/admission_races.rs | 209 + .../miner_tests/authority_lease.rs | 1600 ++ .../src/coordinator/miner_tests/block_only.rs | 159 + .../src/coordinator/miner_tests/blockwait.rs | 437 + .../miner_tests/candidate_preparation.rs | 77 + .../miner_tests/commit_reconcile.rs | 642 + .../miner_tests/compact_authority.rs | 656 + .../miner_tests/compact_prepared.rs | 1371 ++ .../src/coordinator/miner_tests/config.rs | 53 + .../src/coordinator/miner_tests/credit.rs | 434 + .../coordinator/miner_tests/fence_capture.rs | 199 + .../coordinator/miner_tests/interleavings.rs | 134 + .../src/coordinator/miner_tests/mod.rs | 866 + .../coordinator/miner_tests/observations.rs | 155 + .../miner_tests/prepared_expiry.rs | 402 + .../miner_tests/published_lease.rs | 196 + .../src/coordinator/miner_tests/refresh.rs | 401 + .../coordinator/miner_tests/refresh_window.rs | 510 + .../coordinator/miner_tests/resume_inputs.rs | 333 + .../miner_tests/runtime_recovery.rs | 388 + .../coordinator/miner_tests/stale_causes.rs | 160 + .../src/coordinator/miner_tests/tip_poll.rs | 402 + .../src/coordinator/miner_tests/work_store.rs | 815 + .../coordinator/offer_not_executed_tests.rs | 112 + .../src/coordinator/offer_not_sent_tests.rs | 471 + .../src/coordinator/offer_shutdown.rs | 97 + .../src/coordinator/offer_shutdown_tests.rs | 119 + .../src/coordinator/offer_standby_tests.rs | 169 + .../src/coordinator/offer_warmup_tests.rs | 173 + .../src/coordinator/prepared_storage.rs | 235 + .../coordinator/prepared_storage/compact.rs | 691 + .../prepared_storage/compact/refresh_tests.rs | 150 + .../prepared_storage/compact/test_support.rs | 411 + .../src/coordinator/publication_authority.rs | 176 + .../src/coordinator/refresh_window.rs | 167 + .../src/coordinator/refresh_window/compute.rs | 482 + .../coordinator/same_tip_republish_tests.rs | 483 + .../src/coordinator/storm_evidence_tests.rs | 1292 ++ .../src/coordinator/submit_disabled_tests.rs | 386 + .../src/coordinator/submit_ledger.rs | 249 + .../src/coordinator/test_serial.rs | 17 + .../src/coordinator/tip_observation.rs | 1030 ++ .../src/coordinator/window_incident_tests.rs | 611 + .../src/coordinator/window_ref_tests.rs | 463 + .../src/coordinator/window_switch_tests.rs | 1981 +++ .../src/coordinator/work_ledger.rs | 317 + crates/qbit-prism-server/src/ledger.rs | 148 + .../src/ledger/append_admission.rs | 174 + .../qbit-prism-server/src/ledger/archive.rs | 3065 ++++ crates/qbit-prism-server/src/ledger/audit.rs | 658 + .../src/ledger/audit/tests.rs | 142 + .../src/ledger/authority_contract_tests.rs | 140 + crates/qbit-prism-server/src/ledger/blocks.rs | 1011 ++ .../src/ledger/candidates.rs | 2690 ++++ .../ledger/candidates/capture_form_tests.rs | 133 + .../src/ledger/candidates/park_tests.rs | 533 + .../ledger/candidates/storm_fault_tests.rs | 2455 +++ .../src/ledger/claim_observer.rs | 257 + .../qbit-prism-server/src/ledger/connect.rs | 903 ++ .../src/ledger/connect/acquire.rs | 77 + .../src/ledger/connect/acquire/job_tests.rs | 296 + .../src/ledger/connect/acquire/probe_tests.rs | 168 + .../ledger/connect/acquire/test_support.rs | 131 + .../src/ledger/connect/acquire/tests.rs | 189 + .../src/ledger/difficulty.rs | 113 + .../src/ledger/divergence.rs | 588 + crates/qbit-prism-server/src/ledger/fanout.rs | 639 + .../src/ledger/fatal_state.rs | 624 + .../qbit-prism-server/src/ledger/instances.rs | 569 + crates/qbit-prism-server/src/ledger/jobs.rs | 185 + .../src/ledger/jobs/cleanup.rs | 143 + .../src/ledger/jobs/compact_issued.rs | 380 + .../src/ledger/jobs/compact_issued_batch.rs | 266 + .../ledger/jobs/compact_issued_batch_tests.rs | 78 + .../src/ledger/jobs/prepared.rs | 490 + .../qbit-prism-server/src/ledger/migration.rs | 6628 ++++++++ .../src/ledger/migration/block_solvers.rs | 97 + .../src/ledger/migration/online.rs | 1113 ++ .../src/ledger/migration/partition.rs | 340 + .../src/ledger/migration/share_hashes.rs | 2615 +++ .../migration/share_hashes/postgres_tests.rs | 834 + .../src/ledger/policy_transition.rs | 173 + .../src/ledger/signing_transition.rs | 282 + .../src/ledger/standby_durability.rs | 243 + .../src/ledger/submission_hold.rs | 253 + crates/qbit-prism-server/src/ledger/window.rs | 1972 +++ .../src/ledger/window/blocking_drop.rs | 146 + .../src/ledger/window/blocking_drop/tests.rs | 304 + .../src/ledger/window/payout_state.rs | 170 + .../src/ledger/window/payout_state_tests.rs | 511 + .../src/ledger/window/reference_tests.rs | 233 + .../src/ledger/window/snapshot_delta.rs | 522 + .../src/ledger/window/snapshot_delta/tests.rs | 696 + .../snapshot_delta/tests/adversarial.rs | 327 + .../snapshot_delta/tests/physical_failover.rs | 372 + crates/qbit-prism-server/src/lib.rs | 27 + crates/qbit-prism-server/src/listen.rs | 116 + crates/qbit-prism-server/src/main.rs | 25 + crates/qbit-prism-server/src/memory.rs | 164 + crates/qbit-prism-server/src/metrics.rs | 378 + .../src/metrics/collectors.rs | 149 + .../qbit-prism-server/src/metrics/events.rs | 380 + .../src/metrics/events/tests.rs | 187 + .../qbit-prism-server/src/metrics/labels.rs | 175 + .../qbit-prism-server/src/metrics/landing.rs | 662 + .../src/metrics/landing/tests.rs | 770 + .../src/metrics/landing_acks.rs | 379 + .../qbit-prism-server/src/metrics/registry.rs | 552 + .../qbit-prism-server/src/metrics/runtime.rs | 408 + .../src/metrics/snapshots.rs | 640 + crates/qbit-prism-server/src/partitions.rs | 170 + .../src/partitions/acquire_tests.rs | 299 + .../qbit-prism-server/src/process_security.rs | 43 + crates/qbit-prism-server/src/readiness.rs | 113 + crates/qbit-prism-server/src/rollups.rs | 124 + crates/qbit-prism-server/src/rollups.sql | 89 + crates/qbit-prism-server/src/rollups/tests.rs | 476 + crates/qbit-prism-server/src/rpc.rs | 279 + crates/qbit-prism-server/src/server.rs | 738 + crates/qbit-prism-server/src/stratum.rs | 2282 +++ .../src/stratum/retained_jobs.rs | 186 + .../src/stratum/retained_tests.rs | 657 + .../src/stratum/session_timer_tests.rs | 379 + .../src/stratum/share_observation.rs | 28 + .../src/stratum/stale_grace.rs | 112 + .../src/stratum/stale_grace_tests.rs | 609 + crates/qbit-prism-server/src/tools.rs | 2808 ++++ crates/qbit-prism-server/src/vardiff.rs | 282 + crates/qbit-prism-server/src/waiting.rs | 319 + crates/qbit-prism-server/tests/api_2x.rs | 525 + crates/qbit-prism-server/tests/api_config.rs | 485 + .../qbit-prism-server/tests/api_contract.rs | 302 + .../qbit-prism-server/tests/api_database.rs | 646 + .../tests/append_admission.rs | 1010 ++ .../tests/artifact_admission.rs | 1301 ++ .../tests/audit_body_normalization.rs | 826 + .../tests/audit_completeness.rs | 339 + .../tests/audit_reconstruction_permit.rs | 611 + .../tests/authorization_rpc.rs | 326 + .../tests/b275_persistence_measure.rs | 135 + .../tests/b478_capture_overpay.rs | 1033 ++ .../tests/b478_payout_integrity.rs | 475 + .../tests/b478_stale_revision_block.rs | 272 + .../qbit-prism-server/tests/b574_ack_cap.rs | 497 + .../tests/b602_order_lock_hold.rs | 434 + .../tests/b619_gap_work_after_promotion.rs | 1371 ++ .../tests/candidate_dispatch_probe.rs | 494 + .../tests/candidate_storm.rs | 1650 ++ .../tests/candidate_storm_restart.rs | 2731 ++++ .../tests/candidate_window_migration.rs | 1090 ++ .../tests/candidate_window_qualification.rs | 1513 ++ .../tests/candidate_window_switch.rs | 1813 +++ crates/qbit-prism-server/tests/capacity.rs | 483 + .../tests/chain_epoch_migration.rs | 278 + .../tests/checkout_probe_delivery.rs | 95 + .../tests/cluster_row_fences.rs | 113 + .../tests/compact_issued_batch.rs | 734 + .../tests/compact_issued_repair.rs | 1164 ++ .../tests/compact_prepared_storage.rs | 966 ++ .../tests/compact_runtime_e2e.rs | 1092 ++ .../tests/compact_runtime_observer.rs | 390 + .../tests/compact_runtime_scale.rs | 251 + crates/qbit-prism-server/tests/config_cli.rs | 1295 ++ .../tests/ctv_fanout_bookkeeping.rs | 565 + .../tests/equal_work_scheduler.rs | 441 + .../qbit-prism-server/tests/equal_work_tip.rs | 790 + .../tests/fanout_revision_bump.rs | 85 + .../fixtures/health_2x_known_fields.json | 16 + .../tests/fixtures/metric_events.prom | 195 + .../fixtures/schema_2x/001_share_ledger.sql | 2622 ++++ .../schema_2x/002_candidate_bodies.sql | 511 + .../tests/fixtures/schema_2x/README.md | 29 + .../fixtures/stratum_transcripts/README.md | 69 + .../bmminer-version-rolling.jsonl | 13 + .../braiins-configure-extensions.jsonl | 11 + .../stratum_transcripts/cgminer-plain.jsonl | 11 + .../esp-miner-suggest-difficulty.jsonl | 15 + .../nicehash-extranonce-subscribe.jsonl | 13 + .../tests/healthcheck_cli.rs | 111 + .../tests/import_audits_settlement_lock.rs | 1532 ++ .../tests/issued_job_dependency.rs | 384 + .../tests/jsonb_ceiling_gate.rs | 3184 ++++ .../tests/landing_malloc_trim.rs | 205 + .../tests/landing_metrics.rs | 1548 ++ .../tests/ledger_cancelled_begin.rs | 636 + .../tests/ledger_deadline_atomicity.rs | 1123 ++ .../tests/ledger_postgres.rs | 1825 +++ .../tests/ledger_single_execution.rs | 1728 ++ .../tests/ledger_wait_metrics.rs | 453 + .../qbit-prism-server/tests/live_regtest.rs | 2486 +++ .../qbit-prism-server/tests/lost_race_pin.rs | 1233 ++ .../tests/metrics_allocation.rs | 310 + .../tests/migration_rollback.rs | 2788 ++++ .../qbit-prism-server/tests/observability.rs | 7 + .../tests/observability/cardinality.rs | 239 + .../tests/observability/contract.rs | 424 + .../tests/observability/database_privacy.rs | 91 + .../tests/observability/inventory.rs | 343 + .../tests/observability/privacy.rs | 96 + .../tests/observability/registry.rs | 538 + .../tests/observability/runtime.rs | 162 + .../tests/observability/stratum.rs | 774 + .../tests/observability_database.rs | 387 + .../qbit-prism-server/tests/offer_latency.rs | 457 + .../tests/offer_lifecycle.rs | 3385 ++++ .../tests/orphan_disposition_migration.rs | 571 + .../qbit-prism-server/tests/pool_fee_guard.rs | 339 + .../tests/postgres_failover.rs | 249 + .../tests/prepared_qualification_contract.rs | 251 + .../tests/public_database_config.rs | 326 + .../qbit-prism-server/tests/readiness_rpc.rs | 1206 ++ .../tests/refresh_liveness.rs | 753 + .../tests/refresh_window_split.rs | 537 + .../tests/rollups_postgres.rs | 145 + .../qbit-prism-server/tests/rpc_deadlines.rs | 348 + .../qbit-prism-server/tests/rpc_idle_close.rs | 146 + .../qbit-prism-server/tests/rpc_not_sent.rs | 261 + .../qbit-prism-server/tests/self_check_cli.rs | 454 + .../tests/session_cleanup_metrics.rs | 374 + .../qbit-prism-server/tests/share_archive.rs | 3975 +++++ .../tests/share_partitions_postgres.rs | 1423 ++ .../tests/signing_transition.rs | 1055 ++ .../tests/stratum_admission.rs | 203 + .../tests/stratum_admission_postgres.rs | 362 + .../qbit-prism-server/tests/stratum_codec.rs | 246 + .../tests/stratum_properties.rs | 491 + .../tests/stratum_protocol.rs | 2378 +++ .../tests/stratum_rebuild_lane.rs | 547 + .../tests/stratum_restart_bind.rs | 683 + .../tests/support/audit_acquire_metrics.rs | 506 + .../tests/support/audit_durable_range.rs | 493 + .../support/audit_reconstruction_metrics.rs | 407 + .../tests/support/authority_writers.rs | 242 + .../support/b275_persistence_measure/mod.rs | 868 + .../b275_persistence_measure/socket.rs | 201 + .../tests/support/blob_cleanup.rs | 627 + .../tests/support/block_solvers.rs | 145 + .../tests/support/candidates_cli.rs | 1606 ++ .../tests/support/candidates_recover.rs | 1796 +++ .../tests/support/case_labels.rs | 129 + .../tests/support/clock_steps.rs | 682 + .../tests/support/cohort_fence.rs | 76 + .../support/compact_runtime_e2e/assertions.rs | 152 + .../support/compact_runtime_e2e/audit.rs | 159 + .../support/compact_runtime_e2e/ctv_fanout.rs | 154 + .../tests/support/compact_runtime_e2e/mod.rs | 355 + .../support/compact_runtime_e2e/socket.rs | 176 + .../tests/support/compact_runtime_observer.rs | 114 + .../tests/support/cutover_rehearsal.rs | 1607 ++ .../tests/support/cutover_rehearsal_cases.rs | 416 + .../tests/support/disk_full_injector.rs | 185 + .../tests/support/fake_qbitd.rs | 339 + .../tests/support/fanout_lane_plan.rs | 284 + .../tests/support/fatal_state.rs | 1587 ++ .../tests/support/index_trim.rs | 1572 ++ .../tests/support/jsonb_inventory.rs | 331 + .../tests/support/ledger_2x.rs | 5622 +++++++ .../tests/support/ledger_2x_seed.rs | 2074 +++ .../tests/support/ledger_database.rs | 364 + .../tests/support/ledger_execution_proxy.rs | 1314 ++ .../support/ledger_execution_proxy_close.rs | 615 + .../tests/support/live_alert_rules.rs | 280 + .../tests/support/live_chain_events_load.rs | 914 ++ .../tests/support/live_clock_jump.rs | 793 + .../tests/support/live_compact_runtime.rs | 201 + .../tests/support/live_ctv_cpfp.rs | 460 + .../tests/support/live_dense_soak.rs | 553 + .../tests/support/live_disk_exhaustion.rs | 582 + .../tests/support/live_fault_seams.rs | 589 + .../tests/support/live_highdiff.rs | 570 + .../tests/support/live_host_tools.rs | 30 + .../tests/support/live_lost_reply.rs | 877 ++ .../tests/support/live_migration_lifecycle.rs | 1009 ++ .../tests/support/live_node_outage.rs | 1148 ++ .../support/live_pg_async_live_miners.rs | 1476 ++ .../tests/support/live_pg_failover.rs | 1068 ++ .../tests/support/live_pg_offer_standby.rs | 372 + .../tests/support/live_private_postgres.rs | 181 + .../tests/support/live_recipient_boundary.rs | 379 + .../tests/support/live_session_load.rs | 1723 ++ .../tests/support/live_share_client.rs | 594 + .../tests/support/live_submit_proxy.rs | 163 + .../tests/support/live_transcript_replay.rs | 482 + .../tests/support/live_two_node.rs | 1001 ++ .../tests/support/live_weighted_recipients.rs | 2683 ++++ .../tests/support/offer_standby_flush.rs | 341 + .../tests/support/policy_transition.rs | 671 + .../tests/support/pool_fee.rs | 40 + .../tests/support/prepared_work_assertions.rs | 114 + .../tests/support/public_replica.rs | 362 + .../tests/support/recovery.rs | 595 + .../tests/support/rehearsal_node.rs | 213 + .../tests/support/scripted_node.rs | 212 + .../tests/support/session_sequence.rs | 416 + .../tests/support/share_hash_backfill.rs | 3087 ++++ .../tests/support/share_partitions.rs | 836 + .../tests/support/storm_scale.rs | 91 + .../tests/support/stratum_admission.rs | 248 + .../tests/support/wal_primary.rs | 112 + .../tests/support/window_fixture.rs | 378 + .../tests/throughput_floor.rs | 2349 +++ .../tests/vardiff_convergence.rs | 769 + .../tests/window_balance_order.rs | 161 + .../tests/window_multiplier_invariant.rs | 268 + .../tests/window_read_oracle.rs | 1355 ++ .../tests/window_reference.rs | 786 + .../tests/window_reference/cancellation.rs | 115 + .../tests/window_reference/payout_state.rs | 227 + .../tests/work_refresh_metric.rs | 327 + crates/qbit-prism-test-gate/Cargo.toml | 10 + crates/qbit-prism-test-gate/src/lib.rs | 722 + crates/qbit-prism-test-gate/tests/process.rs | 231 + crates/qbit-prism/Cargo.toml | 9 +- crates/qbit-prism/README.md | 41 +- .../fixtures/vectors/below_target_credit.json | 2074 +++ .../vectors/bootstrap_transition.json | 1030 ++ .../fixtures/vectors/carry_only.json | 649 + .../fixtures/vectors/exporter/README.md | 23 + .../exporter/export_money_path_vectors.rs | 1086 ++ .../vectors/exporter/export_rule_decisions.py | 435 + .../fixtures/vectors/fanout_fee_and_ties.json | 468 + .../qbit-prism/fixtures/vectors/pool_fee.json | 975 ++ .../fixtures/vectors/settlement_chunks.json | 748 + .../qbit-prism/fixtures/vectors/window.json | 802 + crates/qbit-prism/sql/001_share_ledger.sql | 8 +- ...dger_revert_audit_publication_sequence.sql | 18 +- crates/qbit-prism/src/audit_body_ref.rs | 92 +- crates/qbit-prism/src/audit_hash.rs | 407 + .../src/bin/qbit-prism-build-audit-bundle.rs | 18 +- crates/qbit-prism/src/ctv.rs | 2 +- crates/qbit-prism/src/lib.rs | 1248 +- crates/qbit-prism/src/parallel.rs | 1148 ++ crates/qbit-prism/src/settlement.rs | 6 +- crates/qbit-prism/src/window.rs | 44 +- .../tests/adversarial_invariants.rs | 2 +- .../tests/adversarial_settlement_cases.rs | 6 +- crates/qbit-prism/tests/audit_cli.rs | 89 +- crates/qbit-prism/tests/audit_parts.rs | 940 ++ .../qbit-prism/tests/audit_trust_boundary.rs | 6 +- crates/qbit-prism/tests/money_path_vectors.rs | 375 + crates/qbit-prism/tests/parallel_builder.rs | 630 + .../tests/reward_manifest_header.rs | 273 + .../qbit-prism/tests/support/window_corpus.rs | 945 ++ crates/qbit-prism/tests/window_daemon_gate.rs | 510 + .../qbit-prism/tests/window_frozen_vectors.rs | 159 + deny.toml | 59 + doc/mining.md | 2 +- doc/release-notes-1.1.0.md | 4 +- doc/release-notes-2.0.0.md | 125 +- doc/release-notes-2.0.2.md | 44 + doc/release-notes-3.0.0-rc.1.md | 279 + doc/release-notes-3.0.0-rc.2.md | 119 + doc/release-notes-3.0.0-rc.3.md | 90 + doc/release-notes-3.0.0-rc.4.md | 102 + doc/release-notes-3.0.0-rc.5.md | 447 + doc/release-notes-3.0.0-rc.6.md | 293 + doc/release-notes-3.0.0.md | 309 + doc/release-notes-3.x.x-development-line.md | 157 + docker/bitcoin/Dockerfile | 8 + docker/ckpool/Dockerfile | 8 + docker/qbit/Dockerfile | 15 + docker/real-miner/Dockerfile | 8 + docs/README.md | 35 +- docs/mainnet-deployment.md | 447 +- docs/prism-alert-migration.json | 1016 ++ docs/prism-alert-migration.md | 420 + docs/prism-alert-pr-draft.md | 46 + docs/prism-alert-review-ledger.json | 343 + docs/prism-alert-rules-qbit-tools.patch | 1890 +++ docs/prism-alert-verification.md | 140 + docs/prism-async-promotion-gap-blocks.md | 697 + docs/prism-b4-stratum-admission.md | 189 + docs/prism-b8-miner-parity.md | 167 + docs/prism-b8-prepared-dependency.md | 84 + docs/prism-candidate-storage.md | 391 - docs/prism-capacity-readiness.md | 1366 +- docs/prism-compact-runtime-e2e-history.md | 160 + docs/prism-configuration.md | 232 + docs/prism-coordinator-refactor/README.md | 7 + docs/prism-coordinator-refactor/validation.md | 10 +- docs/prism-coordinator-refactor/window-ref.md | 1260 ++ docs/prism-deleted-test-cases.txt | 23 + docs/prism-deleted-test-files.txt | 147 + docs/prism-deleted-test-map.md | 417 + docs/prism-dispatch-probe-evidence.md | 73 + docs/prism-e2e-load-ci.md | 597 + docs/prism-ha-functional-qualification.md | 178 + docs/prism-ha-readiness-probe-harness.md | 22 + docs/prism-ha-reference-architecture.md | 740 + docs/prism-integration-test-gate.md | 245 + docs/prism-ledger-ops.md | 4726 +++++- docs/prism-metric-metadata.json | 432 + docs/prism-metrics-cardinality-privacy.md | 117 + docs/prism-metrics-histogram-consumers.md | 162 + docs/prism-native-alert-rules.json | 1140 ++ docs/prism-native-metrics.md | 714 + docs/prism-overload-alerts.md | 24 +- docs/prism-payout-artifact-measurement.md | 910 +- docs/prism-pool-acquire-metrics.md | 255 + docs/prism-postgres-alert-rules.json | 40 + docs/prism-postgres-exporter-queries.yaml | 57 + docs/prism-postgres-replica.md | 200 +- docs/prism-prepared-qualification.md | 146 + docs/prism-prepared-window-reference-plan.md | 386 + docs/prism-public-operator-endpoint-split.md | 426 +- docs/prism-refresh-probe.md | 166 + docs/prism-rejections.md | 167 +- docs/prism-release-benchmark.md | 254 + docs/prism-rust-migration.md | 2497 +++ docs/prism-session-sequence.md | 105 + docs/prism-share-append-admission.md | 66 + docs/prism-share-ledger-partitioning.md | 401 + docs/prism-soak.md | 353 + docs/prism-storage-sizing.md | 689 +- docs/prism-stratum-fuzzing.md | 128 + docs/prism-throughput-measurements.md | 1932 +++ docs/prism-window-reference-foundation.md | 402 + docs/public-dashboard-api-v1.openapi.yaml | 14 +- docs/public-dashboard-api/README.md | 182 +- docs/refresh-window-split.md | 224 + docs/releasing.md | 60 +- docs/window-reader-lifetime.md | 119 + lab/auxpow/Dockerfile | 2 + lab/prism/Dockerfile | 103 +- lab/prism/__init__.py | 1 - lab/prism/accepted_preview_telemetry.py | 723 - lab/prism/audit_artifacts.py | 6154 -------- lab/prism/audit_bundle_view.py | 2366 --- lab/prism/audit_http.py | 706 - lab/prism/backfill_audit_bundle_canonical.py | 1117 -- lab/prism/backfill_ctv_fanouts.py | 387 - lab/prism/background_services.py | 2285 --- lab/prism/block_candidates.py | 8967 ----------- lab/prism/block_finalization.py | 2365 --- lab/prism/bounded_executor.py | 166 - lab/prism/bundle_compiler.py | 2639 ---- lab/prism/candidate_codec.py | 2238 --- lab/prism/candidate_spool_view.py | 1029 -- lab/prism/candidate_store.py | 1687 -- lab/prism/candidate_window.py | 266 - lab/prism/capacity_evidence.py | 563 - lab/prism/coordinator_config.py | 2252 --- lab/prism/coordinator_shutdown.py | 300 - lab/prism/ctv_broadcaster.py | 525 - lab/prism/ctv_broadcaster_daemon.py | 301 - lab/prism/ctv_runtime.py | 411 - lab/prism/direct_stratum.py | 532 - lab/prism/endpoint_registry.py | 805 - lab/prism/future_callbacks.py | 31 - lab/prism/job_build_benchmark.py | 546 - lab/prism/job_bundle.py | 3628 ----- lab/prism/job_delivery.py | 3042 ---- lab/prism/metrics.py | 2031 --- lab/prism/observability.py | 780 - lab/prism/payout_state.py | 5944 ------- lab/prism/prism_coordinator.py | 10216 ------------ lab/prism/prism_tools.py | 32 - lab/prism/process_telemetry.py | 1622 -- lab/prism/progress_health.py | 698 - lab/prism/public_api.py | 2262 --- lab/prism/public_read_service.py | 1535 -- lab/prism/recover_pending_blocks.py | 476 - lab/prism/recovery_json.py | 38 - lab/prism/reorg_reconciler.py | 1366 -- lab/prism/rpc.py | 241 - lab/prism/run_ctv_broadcaster_daemon.py | 179 - lab/prism/share_json_stream.py | 363 - lab/prism/share_ledger.py | 13106 ---------------- lab/prism/share_submission.py | 799 - lab/prism/share_writer.py | 831 - lab/prism/statement_spool.py | 105 - lab/prism/stratum_session.py | 1583 -- lab/prism/template_artifacts.py | 605 - lab/prism/tip_refresh.py | 4092 ----- lab/prism/vardiff_service.py | 2538 --- lab/prism/worker_difficulty_store.py | 664 - lab/prism/writer_lease_timing.py | 859 - rust-toolchain.toml | 9 + scripts/check-env.sh | 350 +- scripts/check_deleted_test_map.py | 817 + scripts/check_e2e_scenarios.py | 459 + scripts/check_gate_env_reads.py | 108 + scripts/check_gate_manifest.py | 223 + scripts/check_prism_pool_acquires.py | 329 + scripts/check_prism_settings.py | 93 + scripts/check_toolchain_pin.py | 207 + scripts/check_version_skew.py | 134 + scripts/generate_prism_alerts.py | 226 + scripts/generate_prism_metrics.py | 76 + scripts/measure_postgres_wal.py | 55 + scripts/prism-recovery-evidence-parallel.py | 374 + scripts/prism-recovery-evidence.py | 624 + scripts/prism-recovery-evidence.sql | 698 + scripts/prism-self-check.py | 1145 -- scripts/prism_capacity_evidence.py | 108 - scripts/prism_ha_qualification.py | 233 + scripts/prism_ha_readiness_probe.py | 113 + scripts/prism_l3_promote.py | 178 + scripts/prism_legacy_import_admission_gate.sh | 417 + scripts/prism_legacy_parts_digest_check.py | 238 + scripts/prism_legacy_range_sidecars.py | 397 + .../prism_legacy_range_sidecars_sharded.sh | 54 + scripts/prism_load_ab.py | 1114 ++ scripts/prism_load_bridge.py | 403 + scripts/prism_load_matrix.py | 230 + scripts/prism_load_probe.py | 1440 ++ scripts/prism_load_regress.py | 780 + scripts/prism_load_trend.py | 982 ++ scripts/prism_pool_acquires.json | 345 + scripts/prism_pool_wait_scenarios.py | 169 + scripts/prism_shipped_image_lane.py | 1099 ++ scripts/run_python_test_shard.py | 54 +- scripts/run_rust_test_shard.py | 128 + scripts/rust_test_reachability.py | 259 + scripts/test_prism_alert_deployment.py | 195 + scripts/test_prism_alert_promql.py | 108 + scripts/test_prism_postgres_alerts.py | 203 + test/e2e-scenarios.toml | 924 ++ test/prism-cutover-rehearsal.sh | 131 + test/prism-gated-tests.txt | 932 ++ test/prism-load-regression.toml | 113 + test/prism-native-tests.sh | 178 + test/prism-nightly-gated-tests.txt | 22 + test/prism-postgres-lib.sh | 43 - test/prism-weekly-gated-tests.txt | 14 + test/test-prism-combined-regtest.sh | 577 - test/test-prism-postgres-candidate-storage.sh | 65 - test/test-prism-postgres-ledger.sh | 2342 --- test/test-prism-postgres-native-ledger.sh | 605 - test/test-prism-postgres-scale.sh | 500 - test/test-prism-postgres-seed-guard.sh | 606 - test/test-prism-postgres-throughput.sh | 302 +- test/test-prism-public-read-replica.sh | 89 - test/test-prism-regtest.sh | 628 - test/test-prism-stratum-regtest-live.sh | 811 - tests/fixtures/fake_serve_builder.py | 405 - tests/fixtures/mainnet-compose.env | 20 + tests/fixtures/prism-alert-scenarios.json | 3768 +++++ tests/fixtures/prism-capacity-evidence.json | 11 +- tests/fixtures/prism-deployed-alerts.json | 453 + .../prism-legacy-v2-audit-body/README.md | 49 + ...555eaba5eab2a4903e17179ba792d58780a4b.json | 1 + .../prism-audit-share-segment-slot-1-4.json | 1 + .../prism-audit-share-segment-slot-5-8.json | 1 + tests/fixtures/prism-load-trend/README.md | 14 + .../current/throughput-20k-window-1fe.json | 66 + .../regression/history/trend/L2/2026-09.jsonl | 12 + .../prism-load-trend/variance/generate.py | 68 + .../variance/runner-probe-variance.json | 599 + .../fixtures/window_pipeline_parity/README.md | 130 + .../inputs-unpinned.json | 1 + .../window_pipeline_parity/supplementary.json | 610 + tests/perf/b275_delta_read.md | 210 + tests/perf/b275_persistence_measure.md | 162 + tests/perf/b275_post_batch_qualification.md | 256 + tests/perf/b275_release_profile.md | 164 + tests/perf/candidate_storage.py | 162 - tests/perf/candidate_window_membership.py | 223 - tests/perf/canonical_audit_retirement.py | 172 - tests/perf/serialize_block_share_cost.py | 1338 -- .../perf/stratum_session_concurrency_load.py | 2028 --- tests/perf/window_builder_pipe_latency.py | 951 -- tests/perf/window_ledger_decode_latency.py | 593 - tests/perf/window_paging_allocation.py | 1236 -- tests/perf/window_pipeline_gil_scaling.py | 2390 --- tests/prism_candidate_storm.py | 2508 --- tests/prism_concurrency_harness.py | 4175 ----- tests/prism_coordinator_test_support.py | 500 - tests/prism_landing_harness.py | 951 -- tests/prism_postgres_a1_gate.py | 2306 --- tests/prism_postgres_a1_migration_gate.py | 1782 --- tests/prism_postgres_a1_process_gate.py | 2624 ---- tests/prism_postgres_a1_revert_gate.py | 548 - tests/prism_postgres_candidate_gate.py | 281 - tests/prism_public_read_replica_gate.py | 288 - tests/prism_regtest_crash_coordinator.py | 73 - tests/prism_regtest_share_miner.py | 72 - tests/prism_vardiff_test_support.py | 1053 -- tests/prism_window_rows_reference.py | 97 - tests/prism_worker_difficulty_store_gate.py | 427 - tests/test_check_deleted_test_map.py | 1264 ++ tests/test_check_e2e_scenarios.py | 455 + tests/test_check_env_production_gate.py | 456 +- tests/test_check_env_public_credentials.py | 212 + tests/test_check_gate_env_reads.py | 111 + tests/test_check_gate_manifest.py | 209 + tests/test_check_prism_pool_acquires.py | 183 + tests/test_check_prism_settings.py | 97 + tests/test_check_toolchain_pin.py | 189 + tests/test_check_version_skew.py | 177 + tests/test_ctv_broadcaster.py | 436 - tests/test_ctv_broadcaster_daemon.py | 1101 -- tests/test_docs_lab_prism_references.py | 3783 +++++ tests/test_install_prism_qbit.py | 299 + tests/test_mainnet_compose_contract.py | 250 +- tests/test_makefile_lifecycle.py | 116 +- tests/test_measure_postgres_wal.py | 49 + tests/test_mining_compose_profiles.py | 262 +- tests/test_prism_accepted_block_accounting.py | 656 - ...sm_accepted_preview_publication_latency.py | 880 -- .../test_prism_accepted_preview_telemetry.py | 654 - tests/test_prism_admission_lock.py | 651 - tests/test_prism_allocator_experiment.py | 256 - tests/test_prism_ancestor_redrive.py | 809 - tests/test_prism_audit_api.py | 478 - tests/test_prism_audit_artifacts.py | 4564 ------ tests/test_prism_audit_bundle_view.py | 925 -- tests/test_prism_audit_http.py | 555 - tests/test_prism_audit_root_scan.py | 144 - ...t_prism_backfill_audit_bundle_canonical.py | 1215 -- tests/test_prism_background_services.py | 1022 -- tests/test_prism_block_candidate_collapse.py | 4851 ------ tests/test_prism_block_candidates.py | 12464 --------------- tests/test_prism_block_finalization.py | 271 - ...st_prism_bounded_canonical_finalization.py | 1304 -- tests/test_prism_bounded_executor.py | 181 - tests/test_prism_builder_pipe_transport.py | 1771 --- tests/test_prism_candidate_codec.py | 457 - tests/test_prism_candidate_helper.py | 75 - tests/test_prism_candidate_replay_bounds.py | 119 - tests/test_prism_candidate_schema_refusal.py | 82 - .../test_prism_candidate_spool_integration.py | 80 - tests/test_prism_candidate_spool_view.py | 845 - tests/test_prism_candidate_storage_races.py | 252 - tests/test_prism_candidate_storm.py | 1150 -- tests/test_prism_candidate_window.py | 411 - tests/test_prism_capacity_evidence.py | 335 - tests/test_prism_completion_callbacks.py | 238 - tests/test_prism_compose_profile.py | 564 - tests/test_prism_concurrency_harness.py | 992 -- tests/test_prism_container_hardening.py | 145 + tests/test_prism_coordinator_config.py | 2616 --- .../test_prism_coordinator_config_loading.py | 352 - tests/test_prism_coordinator_job_cache.py | 384 - tests/test_prism_coordinator_metrics.py | 1198 -- tests/test_prism_coordinator_shutdown.py | 2130 --- tests/test_prism_ctv_refresh_priority.py | 315 - tests/test_prism_ctv_runtime.py | 547 - tests/test_prism_dequeue_stale_skip.py | 928 -- tests/test_prism_direct_stratum.py | 727 - tests/test_prism_endpoint_registry.py | 672 - tests/test_prism_first_job_latency.py | 916 -- tests/test_prism_ha_qualification.py | 272 + tests/test_prism_ha_readiness_probe.py | 78 + tests/test_prism_heap_census.py | 1155 -- tests/test_prism_hot_path.py | 978 -- .../test_prism_immutable_refresh_artifacts.py | 303 - tests/test_prism_incremental_payout_window.py | 822 - tests/test_prism_initial_job_delivery.py | 868 - tests/test_prism_issue_224_cadence.py | 596 - tests/test_prism_issue_224_core.py | 1040 -- tests/test_prism_issue_224_landing.py | 1094 -- tests/test_prism_issue_224_ledger.py | 759 - ...est_prism_job_build_exception_retention.py | 604 - tests/test_prism_job_build_promises.py | 388 - tests/test_prism_job_build_retention.py | 144 - tests/test_prism_job_builder.py | 4810 ------ tests/test_prism_job_delivery.py | 207 - tests/test_prism_l3_promote.py | 182 + tests/test_prism_landing_append_epoch.py | 515 - tests/test_prism_landing_budget_watchdog.py | 710 - tests/test_prism_landing_harness_model.py | 1388 -- tests/test_prism_landing_interleavings.py | 206 - ...test_prism_lease_heartbeat_tail_latency.py | 978 -- ...rism_lease_heartbeat_under_fenced_write.py | 631 - tests/test_prism_lease_monitor_stalls.py | 1019 -- tests/test_prism_lease_orphaned_lock.py | 384 - ...est_prism_lease_renewal_vs_expiry_claim.py | 564 - tests/test_prism_lease_takeover.py | 746 - ...test_prism_legacy_import_admission_gate.py | 677 + tests/test_prism_legacy_parts_digest_check.py | 572 + tests/test_prism_legacy_range_sidecars.py | 430 + tests/test_prism_load_ab.py | 781 + tests/test_prism_load_bridge.py | 281 + tests/test_prism_load_matrix.py | 217 + tests/test_prism_load_probe.py | 739 + tests/test_prism_load_regress.py | 458 + tests/test_prism_load_trend.py | 561 + tests/test_prism_metrics.py | 3480 ---- tests/test_prism_observability.py | 1119 -- tests/test_prism_payout_state.py | 6043 ------- ...est_prism_payout_window_daemon_recenter.py | 501 - tests/test_prism_pending_block_recovery.py | 520 - tests/test_prism_process_telemetry.py | 309 - tests/test_prism_progress_health.py | 954 -- tests/test_prism_public_credentials.py | 561 + tests/test_prism_public_dashboard_api.py | 3554 ----- tests/test_prism_public_http_limits.py | 151 - tests/test_prism_public_read_service.py | 2571 --- tests/test_prism_public_refresh_limits.py | 76 - tests/test_prism_reconnect_backpressure.py | 1132 -- tests/test_prism_recovery_evidence.py | 317 + tests/test_prism_recovery_evidence_jobs.py | 389 + .../test_prism_recovery_evidence_parallel.py | 247 + tests/test_prism_refresh_retry_pacing.py | 281 - tests/test_prism_reorg_reconciler.py | 633 - tests/test_prism_replay_enumeration_gate.py | 526 - .../test_prism_representative_independence.py | 445 - tests/test_prism_retained_jobs.py | 1248 -- tests/test_prism_reward_window_invariant.py | 87 - tests/test_prism_self_check.py | 1202 -- ...est_prism_settlement_balance_comparison.py | 139 - tests/test_prism_share_ledger.py | 9669 ------------ tests/test_prism_share_submission.py | 374 - tests/test_prism_share_writer.py | 839 - tests/test_prism_share_writer_service.py | 654 - tests/test_prism_shipped_image_lane.py | 455 + ..._prism_singleflight_exception_retention.py | 571 - tests/test_prism_soak_capture.py | 992 ++ tests/test_prism_statement_spool.py | 106 - tests/test_prism_stratum_restart_bind.py | 223 - tests/test_prism_stratum_session.py | 507 - ...test_prism_streamed_block_hex_consumers.py | 120 - tests/test_prism_streamed_submitblock.py | 85 - tests/test_prism_tip_publication_boundary.py | 966 -- tests/test_prism_tip_refresh.py | 185 - tests/test_prism_tip_refresh_delivery.py | 3180 ---- tests/test_prism_tip_refresh_epoch.py | 1915 --- tests/test_prism_tip_refresh_validation.py | 3405 ---- tests/test_prism_vardiff.py | 1416 -- tests/test_prism_vardiff_durable_resume.py | 1045 -- tests/test_prism_vardiff_fast_arrival.py | 451 - tests/test_prism_vardiff_resume.py | 889 -- tests/test_prism_vardiff_service.py | 108 - tests/test_prism_window_pipeline_rust.py | 1472 -- tests/test_prism_worker_difficulty_store.py | 700 - tests/test_prism_writer_lease_policy.py | 660 - tests/test_public_dashboard_api_contract.py | 761 - tests/test_qbit_ckpool_preflight.py | 5 +- tests/test_qbit_image_runtime_smoke.py | 41 +- tests/test_run_python_test_shard.py | 110 + tests/test_run_rust_test_shard.py | 173 + .../test_window_paging_allocation_harness.py | 337 - tests/test_window_pipeline_parity.py | 1155 -- tests/test_window_pipeline_parity_gate.py | 84 - tests/window_pipeline_parity.py | 2147 --- tests/window_pipeline_parity_gate.py | 79 - 1031 files changed, 396691 insertions(+), 271461 deletions(-) create mode 100644 .github/actionlint.yaml create mode 100644 .github/actions/apt-install/action.yml create mode 100644 .github/actions/prism-load-evidence/action.yml create mode 100644 .github/actions/rust-toolchain/action.yml create mode 100644 .github/apt/ubuntu-mirrors.txt create mode 100644 .github/scripts/apt-install.sh create mode 100755 .github/scripts/install-pinned-tool.sh create mode 100755 .github/scripts/install-prism-qbit.sh create mode 100755 .github/scripts/prism-load-run.sh create mode 100755 .github/scripts/prism-stratum-fuzz.sh create mode 100644 .github/scripts/test-install-pinned-tool.sh create mode 100644 .github/scripts/test-prism-image-runtime.sh create mode 100644 .github/workflows/prism-load-l3.yml create mode 100644 .hadolint.yaml create mode 100644 compose.prism-external-db.yaml create mode 100644 compose.prism-ha.yaml create mode 100644 crates/qbit-prism-load/Cargo.toml create mode 100644 crates/qbit-prism-load/README.md create mode 100644 crates/qbit-prism-load/examples/ha_qualification.rs create mode 100644 crates/qbit-prism-load/legacy-flags.json create mode 100644 crates/qbit-prism-load/presets/aliases.txt create mode 100644 crates/qbit-prism-load/presets/dense-cadence-400k-window-1fe-async.json create mode 100644 crates/qbit-prism-load/presets/dense-cadence-400k-window-2fe-async.json create mode 100644 crates/qbit-prism-load/presets/faults-failover-fake-node.json create mode 100644 crates/qbit-prism-load/presets/faults-long-real-node.json create mode 100644 crates/qbit-prism-load/presets/faults-pr-smoke.json create mode 100644 crates/qbit-prism-load/presets/faults-short-real-node.json create mode 100644 crates/qbit-prism-load/presets/mainnet-shape-130-addresses.json create mode 100644 crates/qbit-prism-load/presets/mainnet-shape-2600-addresses.json create mode 100644 crates/qbit-prism-load/presets/mainnet-shape-650-addresses.json create mode 100644 crates/qbit-prism-load/presets/pr-smoke.json create mode 100644 crates/qbit-prism-load/presets/real-node-smoke.json create mode 100644 crates/qbit-prism-load/presets/rental-churn-bursts-and-storms.json create mode 100644 crates/qbit-prism-load/presets/session-point-10000-sessions-400k-2fe-async.json create mode 100644 crates/qbit-prism-load/presets/session-point-5000-sessions-400k-2fe-async.json create mode 100644 crates/qbit-prism-load/presets/short-plan-20k-window-1fe.json create mode 100644 crates/qbit-prism-load/presets/short-plan-fake-node.json create mode 100644 crates/qbit-prism-load/presets/short-plan-real-node.json create mode 100644 crates/qbit-prism-load/presets/soak-short.json create mode 100644 crates/qbit-prism-load/presets/soak-smoke.json create mode 100644 crates/qbit-prism-load/presets/soak-weekly.json create mode 100644 crates/qbit-prism-load/presets/suites.toml create mode 100644 crates/qbit-prism-load/presets/throughput-200k-window-1fe-async.json create mode 100644 crates/qbit-prism-load/presets/throughput-20k-window-1fe-500-addresses.json create mode 100644 crates/qbit-prism-load/presets/throughput-20k-window-1fe.json create mode 100644 crates/qbit-prism-load/presets/throughput-400k-window-1fe-async.json create mode 100644 crates/qbit-prism-load/presets/throughput-400k-window-2fe-async-3-blocks.json create mode 100644 crates/qbit-prism-load/presets/throughput-400k-window-2fe-async.json create mode 100644 crates/qbit-prism-load/presets/throughput-400k-window-2fe-sync.json create mode 100644 crates/qbit-prism-load/presets/throughput-400k-window-4fe-async.json create mode 100644 crates/qbit-prism-load/presets/throughput-500k-window-1fe-async.json create mode 100644 crates/qbit-prism-load/presets/throughput-500k-window-2fe-async.json create mode 100644 crates/qbit-prism-load/presets/throughput-500k-window-4fe-async.json create mode 100644 crates/qbit-prism-load/presets/tip-delivery-2000-miners-400k-2fe-retarget-500-addresses.json create mode 100644 crates/qbit-prism-load/presets/tip-delivery-2000-miners-400k-2fe-retarget.json create mode 100644 crates/qbit-prism-load/soak-gates/testnet4.json create mode 100644 crates/qbit-prism-load/src/artifact.rs create mode 100644 crates/qbit-prism-load/src/cadence.rs create mode 100644 crates/qbit-prism-load/src/churn.rs create mode 100644 crates/qbit-prism-load/src/classify.rs create mode 100644 crates/qbit-prism-load/src/cli.rs create mode 100644 crates/qbit-prism-load/src/client.rs create mode 100644 crates/qbit-prism-load/src/cluster.rs create mode 100644 crates/qbit-prism-load/src/collate.rs create mode 100644 crates/qbit-prism-load/src/compare.rs create mode 100644 crates/qbit-prism-load/src/compare_main.rs create mode 100644 crates/qbit-prism-load/src/digest.rs create mode 100644 crates/qbit-prism-load/src/external/histogram.rs create mode 100644 crates/qbit-prism-load/src/external/mod.rs create mode 100644 crates/qbit-prism-load/src/external/stats.rs create mode 100644 crates/qbit-prism-load/src/fault/backlog.rs create mode 100644 crates/qbit-prism-load/src/fault/database.rs create mode 100644 crates/qbit-prism-load/src/fault/disk.rs create mode 100644 crates/qbit-prism-load/src/fault/endpoint.rs create mode 100644 crates/qbit-prism-load/src/fault/failover.rs create mode 100644 crates/qbit-prism-load/src/fault/frontend.rs create mode 100644 crates/qbit-prism-load/src/fault/mod.rs create mode 100644 crates/qbit-prism-load/src/fault/plan.rs create mode 100644 crates/qbit-prism-load/src/fault/read_tier.rs create mode 100644 crates/qbit-prism-load/src/fault/rpc_relay.rs create mode 100644 crates/qbit-prism-load/src/fault/verdict.rs create mode 100644 crates/qbit-prism-load/src/frontend.rs create mode 100644 crates/qbit-prism-load/src/gate.rs create mode 100644 crates/qbit-prism-load/src/gate_main.rs create mode 100644 crates/qbit-prism-load/src/kill.rs create mode 100644 crates/qbit-prism-load/src/lib.rs create mode 100644 crates/qbit-prism-load/src/main.rs create mode 100644 crates/qbit-prism-load/src/measure.rs create mode 100644 crates/qbit-prism-load/src/node.rs create mode 100644 crates/qbit-prism-load/src/preset.rs create mode 100644 crates/qbit-prism-load/src/profile.rs create mode 100644 crates/qbit-prism-load/src/provenance.rs create mode 100644 crates/qbit-prism-load/src/proxy.rs create mode 100644 crates/qbit-prism-load/src/qbitd.rs create mode 100644 crates/qbit-prism-load/src/realism.rs create mode 100644 crates/qbit-prism-load/src/report.rs create mode 100644 crates/qbit-prism-load/src/restart.rs create mode 100644 crates/qbit-prism-load/src/run.rs create mode 100644 crates/qbit-prism-load/src/settlement.rs create mode 100644 crates/qbit-prism-load/src/soak/evaluate.rs create mode 100644 crates/qbit-prism-load/src/soak/mod.rs create mode 100644 crates/qbit-prism-load/src/soak/sample.rs create mode 100644 crates/qbit-prism-load/src/soak/tests.rs create mode 100644 crates/qbit-prism-load/src/soak_driver.rs create mode 100644 crates/qbit-prism-load/src/soak_report_main.rs create mode 100644 crates/qbit-prism-load/src/window.rs create mode 100644 crates/qbit-prism-load/tests/churn.rs create mode 100644 crates/qbit-prism-load/tests/collate.rs create mode 100644 crates/qbit-prism-load/tests/compare.rs create mode 100644 crates/qbit-prism-load/tests/ctv_settlement.rs create mode 100644 crates/qbit-prism-load/tests/external_frontend.rs create mode 100644 crates/qbit-prism-load/tests/external_target.rs create mode 100644 crates/qbit-prism-load/tests/faults.rs create mode 100644 crates/qbit-prism-load/tests/golden/fake_frontend_environment.json create mode 100644 crates/qbit-prism-load/tests/golden/fake_node_transcript.json create mode 100644 crates/qbit-prism-load/tests/golden/fake_side_report_keys.json create mode 100644 crates/qbit-prism-load/tests/harness.rs create mode 100644 crates/qbit-prism-load/tests/legacy_flags.rs create mode 100644 crates/qbit-prism-load/tests/load_smoke.rs create mode 100644 crates/qbit-prism-load/tests/quorum_replication.rs create mode 100644 crates/qbit-prism-load/tests/real_node.rs create mode 100644 crates/qbit-prism-load/tests/realism.rs create mode 100644 crates/qbit-prism-load/tests/soak_plan.rs create mode 100644 crates/qbit-prism-load/tests/soak_sampling.rs create mode 100644 crates/qbit-prism-load/tests/soak_smoke.rs create mode 100644 crates/qbit-prism-server/Cargo.toml create mode 100644 crates/qbit-prism-server/README.md create mode 100644 crates/qbit-prism-server/fuzz/.gitignore create mode 100644 crates/qbit-prism-server/fuzz/Cargo.lock create mode 100644 crates/qbit-prism-server/fuzz/Cargo.toml create mode 100644 crates/qbit-prism-server/fuzz/fuzz_targets/codec_parsers.rs create mode 100644 crates/qbit-prism-server/fuzz/fuzz_targets/stratum_lines.rs create mode 100644 crates/qbit-prism-server/fuzz/fuzz_targets/stratum_session.rs create mode 100644 crates/qbit-prism-server/fuzz/seeds/codec_parsers/assembled_block create mode 100644 crates/qbit-prism-server/fuzz/seeds/codec_parsers/coinbase_split_placeholder create mode 100644 crates/qbit-prism-server/fuzz/seeds/codec_parsers/compact_mainnet_difficulty_1 create mode 100644 crates/qbit-prism-server/fuzz/seeds/codec_parsers/compact_negative create mode 100644 crates/qbit-prism-server/fuzz/seeds/codec_parsers/compact_overflow create mode 100644 crates/qbit-prism-server/fuzz/seeds/codec_parsers/compact_regtest create mode 100644 crates/qbit-prism-server/fuzz/seeds/codec_parsers/hex_field create mode 100644 crates/qbit-prism-server/fuzz/seeds/codec_parsers/noncanonical_compact_size create mode 100644 crates/qbit-prism-server/fuzz/seeds/codec_parsers/segwit_coinbase create mode 100644 crates/qbit-prism-server/fuzz/seeds/codec_parsers/stripped_coinbase create mode 100644 crates/qbit-prism-server/fuzz/seeds/codec_parsers/template_json create mode 100644 crates/qbit-prism-server/fuzz/seeds/codec_parsers/template_mask_and_transactions create mode 100644 crates/qbit-prism-server/fuzz/seeds/stratum_lines/deep_nesting create mode 100644 crates/qbit-prism-server/fuzz/seeds/stratum_lines/exactly_at_bound create mode 100644 crates/qbit-prism-server/fuzz/seeds/stratum_lines/fragmented_json_across_writes create mode 100644 crates/qbit-prism-server/fuzz/seeds/stratum_lines/invalid_utf8_and_controls create mode 100644 crates/qbit-prism-server/fuzz/seeds/stratum_lines/login_fragmented create mode 100644 crates/qbit-prism-server/fuzz/seeds/stratum_lines/login_one_write create mode 100644 crates/qbit-prism-server/fuzz/seeds/stratum_lines/malformed_budget create mode 100644 crates/qbit-prism-server/fuzz/seeds/stratum_lines/odd_ids create mode 100644 crates/qbit-prism-server/fuzz/seeds/stratum_lines/oversize_frame_closes create mode 100644 crates/qbit-prism-server/fuzz/seeds/stratum_lines/trailing_partial_frame create mode 100644 crates/qbit-prism-server/fuzz/seeds/stratum_lines/unknown_job_submit create mode 100644 crates/qbit-prism-server/fuzz/seeds/stratum_session/backend_failures create mode 100644 crates/qbit-prism-server/fuzz/seeds/stratum_session/configure_after_work_advertises_mask create mode 100644 crates/qbit-prism-server/fuzz/seeds/stratum_session/configure_malformed_masks create mode 100644 crates/qbit-prism-server/fuzz/seeds/stratum_session/duplicate_share create mode 100644 crates/qbit-prism-server/fuzz/seeds/stratum_session/highdiff_floor create mode 100644 crates/qbit-prism-server/fuzz/seeds/stratum_session/login_and_share create mode 100644 crates/qbit-prism-server/fuzz/seeds/stratum_session/mainnet_difficulty_low_shares create mode 100644 crates/qbit-prism-server/fuzz/seeds/stratum_session/oversize_then_reconnect create mode 100644 crates/qbit-prism-server/fuzz/seeds/stratum_session/password_difficulty_options create mode 100644 crates/qbit-prism-server/fuzz/seeds/stratum_session/payout_revision_retires_same_parent_work create mode 100644 crates/qbit-prism-server/fuzz/seeds/stratum_session/pipelined_requests create mode 100644 crates/qbit-prism-server/fuzz/seeds/stratum_session/protocol_errors create mode 100644 crates/qbit-prism-server/fuzz/seeds/stratum_session/reauthorize_keeps_original_worker create mode 100644 crates/qbit-prism-server/fuzz/seeds/stratum_session/reconnect_resumes_difficulty create mode 100644 crates/qbit-prism-server/fuzz/seeds/stratum_session/reconnect_resumes_issued_job create mode 100644 crates/qbit-prism-server/fuzz/seeds/stratum_session/session_budgets create mode 100644 crates/qbit-prism-server/fuzz/seeds/stratum_session/short_retention_eviction create mode 100644 crates/qbit-prism-server/fuzz/seeds/stratum_session/suggest_high_difficulty create mode 100644 crates/qbit-prism-server/fuzz/seeds/stratum_session/tip_change_stale_grace create mode 100644 crates/qbit-prism-server/fuzz/seeds/stratum_session/tip_change_without_grace create mode 100644 crates/qbit-prism-server/fuzz/seeds/stratum_session/username_limit create mode 100644 crates/qbit-prism-server/fuzz/seeds/stratum_session/vardiff_retarget_keeps_old_targets create mode 100644 crates/qbit-prism-server/fuzz/seeds/stratum_session/version_rolling create mode 100644 crates/qbit-prism-server/fuzz/seeds/stratum_session/zero_mask create mode 100644 crates/qbit-prism-server/fuzz/src/alloc.rs create mode 100644 crates/qbit-prism-server/fuzz/src/backend.rs create mode 100644 crates/qbit-prism-server/fuzz/src/checks.rs create mode 100644 crates/qbit-prism-server/fuzz/src/driver.rs create mode 100644 crates/qbit-prism-server/fuzz/src/lib.rs create mode 100644 crates/qbit-prism-server/fuzz/src/lines.rs create mode 100644 crates/qbit-prism-server/fuzz/src/parsers.rs create mode 100644 crates/qbit-prism-server/fuzz/src/pow.rs create mode 100644 crates/qbit-prism-server/fuzz/src/script.rs create mode 100644 crates/qbit-prism-server/fuzz/stratum.dict create mode 100644 crates/qbit-prism-server/fuzz/tests/seeds.rs create mode 100644 crates/qbit-prism-server/migrations/002_multi_instance.sql create mode 100644 crates/qbit-prism-server/migrations/003_2x_compatibility.sql create mode 100644 crates/qbit-prism-server/migrations/004_cpfp_retired_funding.sql create mode 100644 crates/qbit-prism-server/migrations/005_candidate_dispatch.sql create mode 100644 crates/qbit-prism-server/migrations/006_source_schema.sql create mode 100644 crates/qbit-prism-server/migrations/007_candidate_window_reference.sql create mode 100644 crates/qbit-prism-server/migrations/008_prepared_window_reference.sql create mode 100644 crates/qbit-prism-server/migrations/009_wrap_safe_sessions.sql create mode 100644 crates/qbit-prism-server/migrations/010_fatal_state_recovery.sql create mode 100644 crates/qbit-prism-server/migrations/011_offer_before_landing.sql create mode 100644 crates/qbit-prism-server/migrations/012_offer_startup_fence.sql create mode 100644 crates/qbit-prism-server/migrations/013_share_ledger_index_trim.sql create mode 100644 crates/qbit-prism-server/migrations/014_policy_transition.sql create mode 100644 crates/qbit-prism-server/migrations/015_candidate_orphan_disposition.sql create mode 100644 crates/qbit-prism-server/migrations/016_share_ledger_partition_catalog.sql create mode 100644 crates/qbit-prism-server/migrations/017_share_ledger_partitions.sql create mode 100644 crates/qbit-prism-server/migrations/018_chain_observation_epoch.sql create mode 100644 crates/qbit-prism-server/migrations/019_signing_transitions.sql create mode 100644 crates/qbit-prism-server/migrations/020_payout_divergences.sql create mode 100644 crates/qbit-prism-server/migrations/021_candidate_claim_observed_lease.sql create mode 100644 crates/qbit-prism-server/migrations/022_fanout_claim_observed_lease.sql create mode 100644 crates/qbit-prism-server/migrations/023_block_submission_hold.sql create mode 100644 crates/qbit-prism-server/migrations/024_fanout_lane_index.sql create mode 100644 crates/qbit-prism-server/migrations/025_carry_forward_integrity_by_program.sql create mode 100644 crates/qbit-prism-server/migrations/026_carry_forward_integrity_report_once.sql create mode 100644 crates/qbit-prism-server/src/api.rs create mode 100644 crates/qbit-prism-server/src/api/charts.rs create mode 100644 crates/qbit-prism-server/src/api/metrics_snapshot.rs create mode 100644 crates/qbit-prism-server/src/api/metrics_snapshot/tests.rs create mode 100644 crates/qbit-prism-server/src/api/operator_auth.rs create mode 100644 crates/qbit-prism-server/src/api/public.rs create mode 100644 crates/qbit-prism-server/src/api/public_service.rs create mode 100644 crates/qbit-prism-server/src/api/public_service/metrics_tests.rs create mode 100644 crates/qbit-prism-server/src/api/public_service/readiness_events.rs create mode 100644 crates/qbit-prism-server/src/api/public_service/readiness_tests.rs create mode 100644 crates/qbit-prism-server/src/api/queries/audit_block_payouts.sql create mode 100644 crates/qbit-prism-server/src/api/queries/audit_bundle.sql create mode 100644 crates/qbit-prism-server/src/api/queries/audit_bundle_by_commitment.sql create mode 100644 crates/qbit-prism-server/src/api/queries/audit_share_window.sql create mode 100644 crates/qbit-prism-server/src/api/queries/dashboard_block_markers.sql create mode 100644 crates/qbit-prism-server/src/api/queries/dashboard_blocks.sql create mode 100644 crates/qbit-prism-server/src/api/queries/dashboard_hashrate_rollups.sql create mode 100644 crates/qbit-prism-server/src/api/queries/dashboard_hashrate_series.sql create mode 100644 crates/qbit-prism-server/src/api/queries/dashboard_leaderboard.sql create mode 100644 crates/qbit-prism-server/src/api/queries/dashboard_miner_earning_rows.sql create mode 100644 crates/qbit-prism-server/src/api/queries/dashboard_miner_payout_rows.sql create mode 100644 crates/qbit-prism-server/src/api/queries/dashboard_miner_share_summary.sql create mode 100644 crates/qbit-prism-server/src/api/queries/dashboard_miner_worker_rows.sql create mode 100644 crates/qbit-prism-server/src/api/queries/dashboard_pool_snapshot.sql create mode 100644 crates/qbit-prism-server/src/api/queries/dashboard_reward_leaderboard.sql create mode 100644 crates/qbit-prism-server/src/api/queries/read_replica_status.sql create mode 100644 crates/qbit-prism-server/src/api/queries/read_schema_ready.sql create mode 100644 crates/qbit-prism-server/src/api/queries/recipient_payout_history.sql create mode 100644 crates/qbit-prism-server/src/api/read_models.rs create mode 100644 crates/qbit-prism-server/src/api/response_cache.rs create mode 100644 crates/qbit-prism-server/src/bin/qbit-prism-miner.rs create mode 100644 crates/qbit-prism-server/src/broadcaster.rs create mode 100644 crates/qbit-prism-server/src/capacity.rs create mode 100644 crates/qbit-prism-server/src/codec.rs create mode 100644 crates/qbit-prism-server/src/config.rs create mode 100644 crates/qbit-prism-server/src/config/database.rs create mode 100644 crates/qbit-prism-server/src/config/environment.rs create mode 100644 crates/qbit-prism-server/src/config/native-settings.txt create mode 100644 crates/qbit-prism-server/src/config/policy_transition.rs create mode 100644 crates/qbit-prism-server/src/config/retired-settings.txt create mode 100644 crates/qbit-prism-server/src/coordinator.rs create mode 100644 crates/qbit-prism-server/src/coordinator/bundle_build.rs create mode 100644 crates/qbit-prism-server/src/coordinator/candidate_checkout_tests.rs create mode 100644 crates/qbit-prism-server/src/coordinator/candidate_lease_tests.rs create mode 100644 crates/qbit-prism-server/src/coordinator/candidate_shutdown_tests.rs create mode 100644 crates/qbit-prism-server/src/coordinator/chain_observation.rs create mode 100644 crates/qbit-prism-server/src/coordinator/claim_release.rs create mode 100644 crates/qbit-prism-server/src/coordinator/claim_release_tests.rs create mode 100644 crates/qbit-prism-server/src/coordinator/clocked_flight.rs create mode 100644 crates/qbit-prism-server/src/coordinator/clocked_revision_tests.rs create mode 100644 crates/qbit-prism-server/src/coordinator/commit_reconcile_tests.rs create mode 100644 crates/qbit-prism-server/src/coordinator/compact_decode_tests.rs create mode 100644 crates/qbit-prism-server/src/coordinator/compact_resume.rs create mode 100644 crates/qbit-prism-server/src/coordinator/compact_runtime.rs create mode 100644 crates/qbit-prism-server/src/coordinator/d2_below_target_tests.rs create mode 100644 crates/qbit-prism-server/src/coordinator/d2_bootstrap_tests.rs create mode 100644 crates/qbit-prism-server/src/coordinator/d2_test_support.rs create mode 100644 crates/qbit-prism-server/src/coordinator/fee_policy_tests.rs create mode 100644 crates/qbit-prism-server/src/coordinator/first_job_lane_tests.rs create mode 100644 crates/qbit-prism-server/src/coordinator/issued_batcher.rs create mode 100644 crates/qbit-prism-server/src/coordinator/issued_batcher_pg_tests.rs create mode 100644 crates/qbit-prism-server/src/coordinator/issued_batcher_tests.rs create mode 100644 crates/qbit-prism-server/src/coordinator/miner_submit.rs create mode 100644 crates/qbit-prism-server/src/coordinator/miner_submit_acquire_tests.rs create mode 100644 crates/qbit-prism-server/src/coordinator/miner_tests/admission_races.rs create mode 100644 crates/qbit-prism-server/src/coordinator/miner_tests/authority_lease.rs create mode 100644 crates/qbit-prism-server/src/coordinator/miner_tests/block_only.rs create mode 100644 crates/qbit-prism-server/src/coordinator/miner_tests/blockwait.rs create mode 100644 crates/qbit-prism-server/src/coordinator/miner_tests/candidate_preparation.rs create mode 100644 crates/qbit-prism-server/src/coordinator/miner_tests/commit_reconcile.rs create mode 100644 crates/qbit-prism-server/src/coordinator/miner_tests/compact_authority.rs create mode 100644 crates/qbit-prism-server/src/coordinator/miner_tests/compact_prepared.rs create mode 100644 crates/qbit-prism-server/src/coordinator/miner_tests/config.rs create mode 100644 crates/qbit-prism-server/src/coordinator/miner_tests/credit.rs create mode 100644 crates/qbit-prism-server/src/coordinator/miner_tests/fence_capture.rs create mode 100644 crates/qbit-prism-server/src/coordinator/miner_tests/interleavings.rs create mode 100644 crates/qbit-prism-server/src/coordinator/miner_tests/mod.rs create mode 100644 crates/qbit-prism-server/src/coordinator/miner_tests/observations.rs create mode 100644 crates/qbit-prism-server/src/coordinator/miner_tests/prepared_expiry.rs create mode 100644 crates/qbit-prism-server/src/coordinator/miner_tests/published_lease.rs create mode 100644 crates/qbit-prism-server/src/coordinator/miner_tests/refresh.rs create mode 100644 crates/qbit-prism-server/src/coordinator/miner_tests/refresh_window.rs create mode 100644 crates/qbit-prism-server/src/coordinator/miner_tests/resume_inputs.rs create mode 100644 crates/qbit-prism-server/src/coordinator/miner_tests/runtime_recovery.rs create mode 100644 crates/qbit-prism-server/src/coordinator/miner_tests/stale_causes.rs create mode 100644 crates/qbit-prism-server/src/coordinator/miner_tests/tip_poll.rs create mode 100644 crates/qbit-prism-server/src/coordinator/miner_tests/work_store.rs create mode 100644 crates/qbit-prism-server/src/coordinator/offer_not_executed_tests.rs create mode 100644 crates/qbit-prism-server/src/coordinator/offer_not_sent_tests.rs create mode 100644 crates/qbit-prism-server/src/coordinator/offer_shutdown.rs create mode 100644 crates/qbit-prism-server/src/coordinator/offer_shutdown_tests.rs create mode 100644 crates/qbit-prism-server/src/coordinator/offer_standby_tests.rs create mode 100644 crates/qbit-prism-server/src/coordinator/offer_warmup_tests.rs create mode 100644 crates/qbit-prism-server/src/coordinator/prepared_storage.rs create mode 100644 crates/qbit-prism-server/src/coordinator/prepared_storage/compact.rs create mode 100644 crates/qbit-prism-server/src/coordinator/prepared_storage/compact/refresh_tests.rs create mode 100644 crates/qbit-prism-server/src/coordinator/prepared_storage/compact/test_support.rs create mode 100644 crates/qbit-prism-server/src/coordinator/publication_authority.rs create mode 100644 crates/qbit-prism-server/src/coordinator/refresh_window.rs create mode 100644 crates/qbit-prism-server/src/coordinator/refresh_window/compute.rs create mode 100644 crates/qbit-prism-server/src/coordinator/same_tip_republish_tests.rs create mode 100644 crates/qbit-prism-server/src/coordinator/storm_evidence_tests.rs create mode 100644 crates/qbit-prism-server/src/coordinator/submit_disabled_tests.rs create mode 100644 crates/qbit-prism-server/src/coordinator/submit_ledger.rs create mode 100644 crates/qbit-prism-server/src/coordinator/test_serial.rs create mode 100644 crates/qbit-prism-server/src/coordinator/tip_observation.rs create mode 100644 crates/qbit-prism-server/src/coordinator/window_incident_tests.rs create mode 100644 crates/qbit-prism-server/src/coordinator/window_ref_tests.rs create mode 100644 crates/qbit-prism-server/src/coordinator/window_switch_tests.rs create mode 100644 crates/qbit-prism-server/src/coordinator/work_ledger.rs create mode 100644 crates/qbit-prism-server/src/ledger.rs create mode 100644 crates/qbit-prism-server/src/ledger/append_admission.rs create mode 100644 crates/qbit-prism-server/src/ledger/archive.rs create mode 100644 crates/qbit-prism-server/src/ledger/audit.rs create mode 100644 crates/qbit-prism-server/src/ledger/audit/tests.rs create mode 100644 crates/qbit-prism-server/src/ledger/authority_contract_tests.rs create mode 100644 crates/qbit-prism-server/src/ledger/blocks.rs create mode 100644 crates/qbit-prism-server/src/ledger/candidates.rs create mode 100644 crates/qbit-prism-server/src/ledger/candidates/capture_form_tests.rs create mode 100644 crates/qbit-prism-server/src/ledger/candidates/park_tests.rs create mode 100644 crates/qbit-prism-server/src/ledger/candidates/storm_fault_tests.rs create mode 100644 crates/qbit-prism-server/src/ledger/claim_observer.rs create mode 100644 crates/qbit-prism-server/src/ledger/connect.rs create mode 100644 crates/qbit-prism-server/src/ledger/connect/acquire.rs create mode 100644 crates/qbit-prism-server/src/ledger/connect/acquire/job_tests.rs create mode 100644 crates/qbit-prism-server/src/ledger/connect/acquire/probe_tests.rs create mode 100644 crates/qbit-prism-server/src/ledger/connect/acquire/test_support.rs create mode 100644 crates/qbit-prism-server/src/ledger/connect/acquire/tests.rs create mode 100644 crates/qbit-prism-server/src/ledger/difficulty.rs create mode 100644 crates/qbit-prism-server/src/ledger/divergence.rs create mode 100644 crates/qbit-prism-server/src/ledger/fanout.rs create mode 100644 crates/qbit-prism-server/src/ledger/fatal_state.rs create mode 100644 crates/qbit-prism-server/src/ledger/instances.rs create mode 100644 crates/qbit-prism-server/src/ledger/jobs.rs create mode 100644 crates/qbit-prism-server/src/ledger/jobs/cleanup.rs create mode 100644 crates/qbit-prism-server/src/ledger/jobs/compact_issued.rs create mode 100644 crates/qbit-prism-server/src/ledger/jobs/compact_issued_batch.rs create mode 100644 crates/qbit-prism-server/src/ledger/jobs/compact_issued_batch_tests.rs create mode 100644 crates/qbit-prism-server/src/ledger/jobs/prepared.rs create mode 100644 crates/qbit-prism-server/src/ledger/migration.rs create mode 100644 crates/qbit-prism-server/src/ledger/migration/block_solvers.rs create mode 100644 crates/qbit-prism-server/src/ledger/migration/online.rs create mode 100644 crates/qbit-prism-server/src/ledger/migration/partition.rs create mode 100644 crates/qbit-prism-server/src/ledger/migration/share_hashes.rs create mode 100644 crates/qbit-prism-server/src/ledger/migration/share_hashes/postgres_tests.rs create mode 100644 crates/qbit-prism-server/src/ledger/policy_transition.rs create mode 100644 crates/qbit-prism-server/src/ledger/signing_transition.rs create mode 100644 crates/qbit-prism-server/src/ledger/standby_durability.rs create mode 100644 crates/qbit-prism-server/src/ledger/submission_hold.rs create mode 100644 crates/qbit-prism-server/src/ledger/window.rs create mode 100644 crates/qbit-prism-server/src/ledger/window/blocking_drop.rs create mode 100644 crates/qbit-prism-server/src/ledger/window/blocking_drop/tests.rs create mode 100644 crates/qbit-prism-server/src/ledger/window/payout_state.rs create mode 100644 crates/qbit-prism-server/src/ledger/window/payout_state_tests.rs create mode 100644 crates/qbit-prism-server/src/ledger/window/reference_tests.rs create mode 100644 crates/qbit-prism-server/src/ledger/window/snapshot_delta.rs create mode 100644 crates/qbit-prism-server/src/ledger/window/snapshot_delta/tests.rs create mode 100644 crates/qbit-prism-server/src/ledger/window/snapshot_delta/tests/adversarial.rs create mode 100644 crates/qbit-prism-server/src/ledger/window/snapshot_delta/tests/physical_failover.rs create mode 100644 crates/qbit-prism-server/src/lib.rs create mode 100644 crates/qbit-prism-server/src/listen.rs create mode 100644 crates/qbit-prism-server/src/main.rs create mode 100644 crates/qbit-prism-server/src/memory.rs create mode 100644 crates/qbit-prism-server/src/metrics.rs create mode 100644 crates/qbit-prism-server/src/metrics/collectors.rs create mode 100644 crates/qbit-prism-server/src/metrics/events.rs create mode 100644 crates/qbit-prism-server/src/metrics/events/tests.rs create mode 100644 crates/qbit-prism-server/src/metrics/labels.rs create mode 100644 crates/qbit-prism-server/src/metrics/landing.rs create mode 100644 crates/qbit-prism-server/src/metrics/landing/tests.rs create mode 100644 crates/qbit-prism-server/src/metrics/landing_acks.rs create mode 100644 crates/qbit-prism-server/src/metrics/registry.rs create mode 100644 crates/qbit-prism-server/src/metrics/runtime.rs create mode 100644 crates/qbit-prism-server/src/metrics/snapshots.rs create mode 100644 crates/qbit-prism-server/src/partitions.rs create mode 100644 crates/qbit-prism-server/src/partitions/acquire_tests.rs create mode 100644 crates/qbit-prism-server/src/process_security.rs create mode 100644 crates/qbit-prism-server/src/readiness.rs create mode 100644 crates/qbit-prism-server/src/rollups.rs create mode 100644 crates/qbit-prism-server/src/rollups.sql create mode 100644 crates/qbit-prism-server/src/rollups/tests.rs create mode 100644 crates/qbit-prism-server/src/rpc.rs create mode 100644 crates/qbit-prism-server/src/server.rs create mode 100644 crates/qbit-prism-server/src/stratum.rs create mode 100644 crates/qbit-prism-server/src/stratum/retained_jobs.rs create mode 100644 crates/qbit-prism-server/src/stratum/retained_tests.rs create mode 100644 crates/qbit-prism-server/src/stratum/session_timer_tests.rs create mode 100644 crates/qbit-prism-server/src/stratum/share_observation.rs create mode 100644 crates/qbit-prism-server/src/stratum/stale_grace.rs create mode 100644 crates/qbit-prism-server/src/stratum/stale_grace_tests.rs create mode 100644 crates/qbit-prism-server/src/tools.rs create mode 100644 crates/qbit-prism-server/src/vardiff.rs create mode 100644 crates/qbit-prism-server/src/waiting.rs create mode 100644 crates/qbit-prism-server/tests/api_2x.rs create mode 100644 crates/qbit-prism-server/tests/api_config.rs create mode 100644 crates/qbit-prism-server/tests/api_contract.rs create mode 100644 crates/qbit-prism-server/tests/api_database.rs create mode 100644 crates/qbit-prism-server/tests/append_admission.rs create mode 100644 crates/qbit-prism-server/tests/artifact_admission.rs create mode 100644 crates/qbit-prism-server/tests/audit_body_normalization.rs create mode 100644 crates/qbit-prism-server/tests/audit_completeness.rs create mode 100644 crates/qbit-prism-server/tests/audit_reconstruction_permit.rs create mode 100644 crates/qbit-prism-server/tests/authorization_rpc.rs create mode 100644 crates/qbit-prism-server/tests/b275_persistence_measure.rs create mode 100644 crates/qbit-prism-server/tests/b478_capture_overpay.rs create mode 100644 crates/qbit-prism-server/tests/b478_payout_integrity.rs create mode 100644 crates/qbit-prism-server/tests/b478_stale_revision_block.rs create mode 100644 crates/qbit-prism-server/tests/b574_ack_cap.rs create mode 100644 crates/qbit-prism-server/tests/b602_order_lock_hold.rs create mode 100644 crates/qbit-prism-server/tests/b619_gap_work_after_promotion.rs create mode 100644 crates/qbit-prism-server/tests/candidate_dispatch_probe.rs create mode 100644 crates/qbit-prism-server/tests/candidate_storm.rs create mode 100644 crates/qbit-prism-server/tests/candidate_storm_restart.rs create mode 100644 crates/qbit-prism-server/tests/candidate_window_migration.rs create mode 100644 crates/qbit-prism-server/tests/candidate_window_qualification.rs create mode 100644 crates/qbit-prism-server/tests/candidate_window_switch.rs create mode 100644 crates/qbit-prism-server/tests/capacity.rs create mode 100644 crates/qbit-prism-server/tests/chain_epoch_migration.rs create mode 100644 crates/qbit-prism-server/tests/checkout_probe_delivery.rs create mode 100644 crates/qbit-prism-server/tests/cluster_row_fences.rs create mode 100644 crates/qbit-prism-server/tests/compact_issued_batch.rs create mode 100644 crates/qbit-prism-server/tests/compact_issued_repair.rs create mode 100644 crates/qbit-prism-server/tests/compact_prepared_storage.rs create mode 100644 crates/qbit-prism-server/tests/compact_runtime_e2e.rs create mode 100644 crates/qbit-prism-server/tests/compact_runtime_observer.rs create mode 100644 crates/qbit-prism-server/tests/compact_runtime_scale.rs create mode 100644 crates/qbit-prism-server/tests/config_cli.rs create mode 100644 crates/qbit-prism-server/tests/ctv_fanout_bookkeeping.rs create mode 100644 crates/qbit-prism-server/tests/equal_work_scheduler.rs create mode 100644 crates/qbit-prism-server/tests/equal_work_tip.rs create mode 100644 crates/qbit-prism-server/tests/fanout_revision_bump.rs create mode 100644 crates/qbit-prism-server/tests/fixtures/health_2x_known_fields.json create mode 100644 crates/qbit-prism-server/tests/fixtures/metric_events.prom create mode 100644 crates/qbit-prism-server/tests/fixtures/schema_2x/001_share_ledger.sql create mode 100644 crates/qbit-prism-server/tests/fixtures/schema_2x/002_candidate_bodies.sql create mode 100644 crates/qbit-prism-server/tests/fixtures/schema_2x/README.md create mode 100644 crates/qbit-prism-server/tests/fixtures/stratum_transcripts/README.md create mode 100644 crates/qbit-prism-server/tests/fixtures/stratum_transcripts/bmminer-version-rolling.jsonl create mode 100644 crates/qbit-prism-server/tests/fixtures/stratum_transcripts/braiins-configure-extensions.jsonl create mode 100644 crates/qbit-prism-server/tests/fixtures/stratum_transcripts/cgminer-plain.jsonl create mode 100644 crates/qbit-prism-server/tests/fixtures/stratum_transcripts/esp-miner-suggest-difficulty.jsonl create mode 100644 crates/qbit-prism-server/tests/fixtures/stratum_transcripts/nicehash-extranonce-subscribe.jsonl create mode 100644 crates/qbit-prism-server/tests/healthcheck_cli.rs create mode 100644 crates/qbit-prism-server/tests/import_audits_settlement_lock.rs create mode 100644 crates/qbit-prism-server/tests/issued_job_dependency.rs create mode 100644 crates/qbit-prism-server/tests/jsonb_ceiling_gate.rs create mode 100644 crates/qbit-prism-server/tests/landing_malloc_trim.rs create mode 100644 crates/qbit-prism-server/tests/landing_metrics.rs create mode 100644 crates/qbit-prism-server/tests/ledger_cancelled_begin.rs create mode 100644 crates/qbit-prism-server/tests/ledger_deadline_atomicity.rs create mode 100644 crates/qbit-prism-server/tests/ledger_postgres.rs create mode 100644 crates/qbit-prism-server/tests/ledger_single_execution.rs create mode 100644 crates/qbit-prism-server/tests/ledger_wait_metrics.rs create mode 100644 crates/qbit-prism-server/tests/live_regtest.rs create mode 100644 crates/qbit-prism-server/tests/lost_race_pin.rs create mode 100644 crates/qbit-prism-server/tests/metrics_allocation.rs create mode 100644 crates/qbit-prism-server/tests/migration_rollback.rs create mode 100644 crates/qbit-prism-server/tests/observability.rs create mode 100644 crates/qbit-prism-server/tests/observability/cardinality.rs create mode 100644 crates/qbit-prism-server/tests/observability/contract.rs create mode 100644 crates/qbit-prism-server/tests/observability/database_privacy.rs create mode 100644 crates/qbit-prism-server/tests/observability/inventory.rs create mode 100644 crates/qbit-prism-server/tests/observability/privacy.rs create mode 100644 crates/qbit-prism-server/tests/observability/registry.rs create mode 100644 crates/qbit-prism-server/tests/observability/runtime.rs create mode 100644 crates/qbit-prism-server/tests/observability/stratum.rs create mode 100644 crates/qbit-prism-server/tests/observability_database.rs create mode 100644 crates/qbit-prism-server/tests/offer_latency.rs create mode 100644 crates/qbit-prism-server/tests/offer_lifecycle.rs create mode 100644 crates/qbit-prism-server/tests/orphan_disposition_migration.rs create mode 100644 crates/qbit-prism-server/tests/pool_fee_guard.rs create mode 100644 crates/qbit-prism-server/tests/postgres_failover.rs create mode 100644 crates/qbit-prism-server/tests/prepared_qualification_contract.rs create mode 100644 crates/qbit-prism-server/tests/public_database_config.rs create mode 100644 crates/qbit-prism-server/tests/readiness_rpc.rs create mode 100644 crates/qbit-prism-server/tests/refresh_liveness.rs create mode 100644 crates/qbit-prism-server/tests/refresh_window_split.rs create mode 100644 crates/qbit-prism-server/tests/rollups_postgres.rs create mode 100644 crates/qbit-prism-server/tests/rpc_deadlines.rs create mode 100644 crates/qbit-prism-server/tests/rpc_idle_close.rs create mode 100644 crates/qbit-prism-server/tests/rpc_not_sent.rs create mode 100644 crates/qbit-prism-server/tests/self_check_cli.rs create mode 100644 crates/qbit-prism-server/tests/session_cleanup_metrics.rs create mode 100644 crates/qbit-prism-server/tests/share_archive.rs create mode 100644 crates/qbit-prism-server/tests/share_partitions_postgres.rs create mode 100644 crates/qbit-prism-server/tests/signing_transition.rs create mode 100644 crates/qbit-prism-server/tests/stratum_admission.rs create mode 100644 crates/qbit-prism-server/tests/stratum_admission_postgres.rs create mode 100644 crates/qbit-prism-server/tests/stratum_codec.rs create mode 100644 crates/qbit-prism-server/tests/stratum_properties.rs create mode 100644 crates/qbit-prism-server/tests/stratum_protocol.rs create mode 100644 crates/qbit-prism-server/tests/stratum_rebuild_lane.rs create mode 100644 crates/qbit-prism-server/tests/stratum_restart_bind.rs create mode 100644 crates/qbit-prism-server/tests/support/audit_acquire_metrics.rs create mode 100644 crates/qbit-prism-server/tests/support/audit_durable_range.rs create mode 100644 crates/qbit-prism-server/tests/support/audit_reconstruction_metrics.rs create mode 100644 crates/qbit-prism-server/tests/support/authority_writers.rs create mode 100644 crates/qbit-prism-server/tests/support/b275_persistence_measure/mod.rs create mode 100644 crates/qbit-prism-server/tests/support/b275_persistence_measure/socket.rs create mode 100644 crates/qbit-prism-server/tests/support/blob_cleanup.rs create mode 100644 crates/qbit-prism-server/tests/support/block_solvers.rs create mode 100644 crates/qbit-prism-server/tests/support/candidates_cli.rs create mode 100644 crates/qbit-prism-server/tests/support/candidates_recover.rs create mode 100644 crates/qbit-prism-server/tests/support/case_labels.rs create mode 100644 crates/qbit-prism-server/tests/support/clock_steps.rs create mode 100644 crates/qbit-prism-server/tests/support/cohort_fence.rs create mode 100644 crates/qbit-prism-server/tests/support/compact_runtime_e2e/assertions.rs create mode 100644 crates/qbit-prism-server/tests/support/compact_runtime_e2e/audit.rs create mode 100644 crates/qbit-prism-server/tests/support/compact_runtime_e2e/ctv_fanout.rs create mode 100644 crates/qbit-prism-server/tests/support/compact_runtime_e2e/mod.rs create mode 100644 crates/qbit-prism-server/tests/support/compact_runtime_e2e/socket.rs create mode 100644 crates/qbit-prism-server/tests/support/compact_runtime_observer.rs create mode 100644 crates/qbit-prism-server/tests/support/cutover_rehearsal.rs create mode 100644 crates/qbit-prism-server/tests/support/cutover_rehearsal_cases.rs create mode 100644 crates/qbit-prism-server/tests/support/disk_full_injector.rs create mode 100644 crates/qbit-prism-server/tests/support/fake_qbitd.rs create mode 100644 crates/qbit-prism-server/tests/support/fanout_lane_plan.rs create mode 100644 crates/qbit-prism-server/tests/support/fatal_state.rs create mode 100644 crates/qbit-prism-server/tests/support/index_trim.rs create mode 100644 crates/qbit-prism-server/tests/support/jsonb_inventory.rs create mode 100644 crates/qbit-prism-server/tests/support/ledger_2x.rs create mode 100644 crates/qbit-prism-server/tests/support/ledger_2x_seed.rs create mode 100644 crates/qbit-prism-server/tests/support/ledger_database.rs create mode 100644 crates/qbit-prism-server/tests/support/ledger_execution_proxy.rs create mode 100644 crates/qbit-prism-server/tests/support/ledger_execution_proxy_close.rs create mode 100644 crates/qbit-prism-server/tests/support/live_alert_rules.rs create mode 100644 crates/qbit-prism-server/tests/support/live_chain_events_load.rs create mode 100644 crates/qbit-prism-server/tests/support/live_clock_jump.rs create mode 100644 crates/qbit-prism-server/tests/support/live_compact_runtime.rs create mode 100644 crates/qbit-prism-server/tests/support/live_ctv_cpfp.rs create mode 100644 crates/qbit-prism-server/tests/support/live_dense_soak.rs create mode 100644 crates/qbit-prism-server/tests/support/live_disk_exhaustion.rs create mode 100644 crates/qbit-prism-server/tests/support/live_fault_seams.rs create mode 100644 crates/qbit-prism-server/tests/support/live_highdiff.rs create mode 100644 crates/qbit-prism-server/tests/support/live_host_tools.rs create mode 100644 crates/qbit-prism-server/tests/support/live_lost_reply.rs create mode 100644 crates/qbit-prism-server/tests/support/live_migration_lifecycle.rs create mode 100644 crates/qbit-prism-server/tests/support/live_node_outage.rs create mode 100644 crates/qbit-prism-server/tests/support/live_pg_async_live_miners.rs create mode 100644 crates/qbit-prism-server/tests/support/live_pg_failover.rs create mode 100644 crates/qbit-prism-server/tests/support/live_pg_offer_standby.rs create mode 100644 crates/qbit-prism-server/tests/support/live_private_postgres.rs create mode 100644 crates/qbit-prism-server/tests/support/live_recipient_boundary.rs create mode 100644 crates/qbit-prism-server/tests/support/live_session_load.rs create mode 100644 crates/qbit-prism-server/tests/support/live_share_client.rs create mode 100644 crates/qbit-prism-server/tests/support/live_submit_proxy.rs create mode 100644 crates/qbit-prism-server/tests/support/live_transcript_replay.rs create mode 100644 crates/qbit-prism-server/tests/support/live_two_node.rs create mode 100644 crates/qbit-prism-server/tests/support/live_weighted_recipients.rs create mode 100644 crates/qbit-prism-server/tests/support/offer_standby_flush.rs create mode 100644 crates/qbit-prism-server/tests/support/policy_transition.rs create mode 100644 crates/qbit-prism-server/tests/support/pool_fee.rs create mode 100644 crates/qbit-prism-server/tests/support/prepared_work_assertions.rs create mode 100644 crates/qbit-prism-server/tests/support/public_replica.rs create mode 100644 crates/qbit-prism-server/tests/support/recovery.rs create mode 100644 crates/qbit-prism-server/tests/support/rehearsal_node.rs create mode 100644 crates/qbit-prism-server/tests/support/scripted_node.rs create mode 100644 crates/qbit-prism-server/tests/support/session_sequence.rs create mode 100644 crates/qbit-prism-server/tests/support/share_hash_backfill.rs create mode 100644 crates/qbit-prism-server/tests/support/share_partitions.rs create mode 100644 crates/qbit-prism-server/tests/support/storm_scale.rs create mode 100644 crates/qbit-prism-server/tests/support/stratum_admission.rs create mode 100644 crates/qbit-prism-server/tests/support/wal_primary.rs create mode 100644 crates/qbit-prism-server/tests/support/window_fixture.rs create mode 100644 crates/qbit-prism-server/tests/throughput_floor.rs create mode 100644 crates/qbit-prism-server/tests/vardiff_convergence.rs create mode 100644 crates/qbit-prism-server/tests/window_balance_order.rs create mode 100644 crates/qbit-prism-server/tests/window_multiplier_invariant.rs create mode 100644 crates/qbit-prism-server/tests/window_read_oracle.rs create mode 100644 crates/qbit-prism-server/tests/window_reference.rs create mode 100644 crates/qbit-prism-server/tests/window_reference/cancellation.rs create mode 100644 crates/qbit-prism-server/tests/window_reference/payout_state.rs create mode 100644 crates/qbit-prism-server/tests/work_refresh_metric.rs create mode 100644 crates/qbit-prism-test-gate/Cargo.toml create mode 100644 crates/qbit-prism-test-gate/src/lib.rs create mode 100644 crates/qbit-prism-test-gate/tests/process.rs create mode 100644 crates/qbit-prism/fixtures/vectors/below_target_credit.json create mode 100644 crates/qbit-prism/fixtures/vectors/bootstrap_transition.json create mode 100644 crates/qbit-prism/fixtures/vectors/carry_only.json create mode 100644 crates/qbit-prism/fixtures/vectors/exporter/README.md create mode 100644 crates/qbit-prism/fixtures/vectors/exporter/export_money_path_vectors.rs create mode 100644 crates/qbit-prism/fixtures/vectors/exporter/export_rule_decisions.py create mode 100644 crates/qbit-prism/fixtures/vectors/fanout_fee_and_ties.json create mode 100644 crates/qbit-prism/fixtures/vectors/pool_fee.json create mode 100644 crates/qbit-prism/fixtures/vectors/settlement_chunks.json create mode 100644 crates/qbit-prism/fixtures/vectors/window.json create mode 100644 crates/qbit-prism/src/audit_hash.rs create mode 100644 crates/qbit-prism/src/parallel.rs create mode 100644 crates/qbit-prism/tests/audit_parts.rs create mode 100644 crates/qbit-prism/tests/money_path_vectors.rs create mode 100644 crates/qbit-prism/tests/parallel_builder.rs create mode 100644 crates/qbit-prism/tests/reward_manifest_header.rs create mode 100644 crates/qbit-prism/tests/support/window_corpus.rs create mode 100644 crates/qbit-prism/tests/window_daemon_gate.rs create mode 100644 crates/qbit-prism/tests/window_frozen_vectors.rs create mode 100644 deny.toml create mode 100644 doc/release-notes-2.0.2.md create mode 100644 doc/release-notes-3.0.0-rc.1.md create mode 100644 doc/release-notes-3.0.0-rc.2.md create mode 100644 doc/release-notes-3.0.0-rc.3.md create mode 100644 doc/release-notes-3.0.0-rc.4.md create mode 100644 doc/release-notes-3.0.0-rc.5.md create mode 100644 doc/release-notes-3.0.0-rc.6.md create mode 100644 doc/release-notes-3.0.0.md create mode 100644 doc/release-notes-3.x.x-development-line.md create mode 100644 docs/prism-alert-migration.json create mode 100644 docs/prism-alert-migration.md create mode 100644 docs/prism-alert-pr-draft.md create mode 100644 docs/prism-alert-review-ledger.json create mode 100644 docs/prism-alert-rules-qbit-tools.patch create mode 100644 docs/prism-alert-verification.md create mode 100644 docs/prism-async-promotion-gap-blocks.md create mode 100644 docs/prism-b4-stratum-admission.md create mode 100644 docs/prism-b8-miner-parity.md create mode 100644 docs/prism-b8-prepared-dependency.md delete mode 100644 docs/prism-candidate-storage.md create mode 100644 docs/prism-compact-runtime-e2e-history.md create mode 100644 docs/prism-configuration.md create mode 100644 docs/prism-coordinator-refactor/window-ref.md create mode 100644 docs/prism-deleted-test-cases.txt create mode 100644 docs/prism-deleted-test-files.txt create mode 100644 docs/prism-deleted-test-map.md create mode 100644 docs/prism-dispatch-probe-evidence.md create mode 100644 docs/prism-e2e-load-ci.md create mode 100644 docs/prism-ha-functional-qualification.md create mode 100644 docs/prism-ha-readiness-probe-harness.md create mode 100644 docs/prism-ha-reference-architecture.md create mode 100644 docs/prism-integration-test-gate.md create mode 100644 docs/prism-metric-metadata.json create mode 100644 docs/prism-metrics-cardinality-privacy.md create mode 100644 docs/prism-metrics-histogram-consumers.md create mode 100644 docs/prism-native-alert-rules.json create mode 100644 docs/prism-native-metrics.md create mode 100644 docs/prism-pool-acquire-metrics.md create mode 100644 docs/prism-postgres-alert-rules.json create mode 100644 docs/prism-postgres-exporter-queries.yaml create mode 100644 docs/prism-prepared-qualification.md create mode 100644 docs/prism-prepared-window-reference-plan.md create mode 100644 docs/prism-refresh-probe.md create mode 100644 docs/prism-release-benchmark.md create mode 100644 docs/prism-rust-migration.md create mode 100644 docs/prism-session-sequence.md create mode 100644 docs/prism-share-append-admission.md create mode 100644 docs/prism-share-ledger-partitioning.md create mode 100644 docs/prism-soak.md create mode 100644 docs/prism-stratum-fuzzing.md create mode 100644 docs/prism-throughput-measurements.md create mode 100644 docs/prism-window-reference-foundation.md create mode 100644 docs/refresh-window-split.md create mode 100644 docs/window-reader-lifetime.md delete mode 100644 lab/prism/__init__.py delete mode 100644 lab/prism/accepted_preview_telemetry.py delete mode 100644 lab/prism/audit_artifacts.py delete mode 100644 lab/prism/audit_bundle_view.py delete mode 100644 lab/prism/audit_http.py delete mode 100644 lab/prism/backfill_audit_bundle_canonical.py delete mode 100644 lab/prism/backfill_ctv_fanouts.py delete mode 100644 lab/prism/background_services.py delete mode 100644 lab/prism/block_candidates.py delete mode 100644 lab/prism/block_finalization.py delete mode 100644 lab/prism/bounded_executor.py delete mode 100644 lab/prism/bundle_compiler.py delete mode 100644 lab/prism/candidate_codec.py delete mode 100644 lab/prism/candidate_spool_view.py delete mode 100644 lab/prism/candidate_store.py delete mode 100644 lab/prism/candidate_window.py delete mode 100644 lab/prism/capacity_evidence.py delete mode 100644 lab/prism/coordinator_config.py delete mode 100644 lab/prism/coordinator_shutdown.py delete mode 100644 lab/prism/ctv_broadcaster.py delete mode 100644 lab/prism/ctv_broadcaster_daemon.py delete mode 100644 lab/prism/ctv_runtime.py delete mode 100644 lab/prism/direct_stratum.py delete mode 100644 lab/prism/endpoint_registry.py delete mode 100644 lab/prism/future_callbacks.py delete mode 100644 lab/prism/job_build_benchmark.py delete mode 100644 lab/prism/job_bundle.py delete mode 100644 lab/prism/job_delivery.py delete mode 100644 lab/prism/metrics.py delete mode 100644 lab/prism/observability.py delete mode 100644 lab/prism/payout_state.py delete mode 100644 lab/prism/prism_coordinator.py delete mode 100644 lab/prism/prism_tools.py delete mode 100644 lab/prism/process_telemetry.py delete mode 100644 lab/prism/progress_health.py delete mode 100644 lab/prism/public_api.py delete mode 100644 lab/prism/public_read_service.py delete mode 100644 lab/prism/recover_pending_blocks.py delete mode 100644 lab/prism/recovery_json.py delete mode 100644 lab/prism/reorg_reconciler.py delete mode 100644 lab/prism/rpc.py delete mode 100644 lab/prism/run_ctv_broadcaster_daemon.py delete mode 100644 lab/prism/share_json_stream.py delete mode 100644 lab/prism/share_ledger.py delete mode 100644 lab/prism/share_submission.py delete mode 100644 lab/prism/share_writer.py delete mode 100644 lab/prism/statement_spool.py delete mode 100644 lab/prism/stratum_session.py delete mode 100644 lab/prism/template_artifacts.py delete mode 100644 lab/prism/tip_refresh.py delete mode 100644 lab/prism/vardiff_service.py delete mode 100644 lab/prism/worker_difficulty_store.py delete mode 100644 lab/prism/writer_lease_timing.py create mode 100644 rust-toolchain.toml create mode 100644 scripts/check_deleted_test_map.py create mode 100644 scripts/check_e2e_scenarios.py create mode 100644 scripts/check_gate_env_reads.py create mode 100644 scripts/check_gate_manifest.py create mode 100644 scripts/check_prism_pool_acquires.py create mode 100644 scripts/check_prism_settings.py create mode 100644 scripts/check_toolchain_pin.py create mode 100644 scripts/check_version_skew.py create mode 100644 scripts/generate_prism_alerts.py create mode 100644 scripts/generate_prism_metrics.py create mode 100755 scripts/measure_postgres_wal.py create mode 100644 scripts/prism-recovery-evidence-parallel.py create mode 100644 scripts/prism-recovery-evidence.py create mode 100644 scripts/prism-recovery-evidence.sql delete mode 100755 scripts/prism-self-check.py delete mode 100644 scripts/prism_capacity_evidence.py create mode 100644 scripts/prism_ha_qualification.py create mode 100644 scripts/prism_ha_readiness_probe.py create mode 100644 scripts/prism_l3_promote.py create mode 100755 scripts/prism_legacy_import_admission_gate.sh create mode 100644 scripts/prism_legacy_parts_digest_check.py create mode 100644 scripts/prism_legacy_range_sidecars.py create mode 100755 scripts/prism_legacy_range_sidecars_sharded.sh create mode 100644 scripts/prism_load_ab.py create mode 100644 scripts/prism_load_bridge.py create mode 100644 scripts/prism_load_matrix.py create mode 100644 scripts/prism_load_probe.py create mode 100644 scripts/prism_load_regress.py create mode 100644 scripts/prism_load_trend.py create mode 100644 scripts/prism_pool_acquires.json create mode 100644 scripts/prism_pool_wait_scenarios.py create mode 100644 scripts/prism_shipped_image_lane.py create mode 100644 scripts/run_rust_test_shard.py create mode 100644 scripts/rust_test_reachability.py create mode 100644 scripts/test_prism_alert_deployment.py create mode 100644 scripts/test_prism_alert_promql.py create mode 100644 scripts/test_prism_postgres_alerts.py create mode 100644 test/e2e-scenarios.toml create mode 100755 test/prism-cutover-rehearsal.sh create mode 100644 test/prism-gated-tests.txt create mode 100644 test/prism-load-regression.toml create mode 100755 test/prism-native-tests.sh create mode 100644 test/prism-nightly-gated-tests.txt delete mode 100644 test/prism-postgres-lib.sh create mode 100644 test/prism-weekly-gated-tests.txt delete mode 100644 test/test-prism-combined-regtest.sh delete mode 100755 test/test-prism-postgres-candidate-storage.sh delete mode 100644 test/test-prism-postgres-ledger.sh delete mode 100755 test/test-prism-postgres-native-ledger.sh delete mode 100755 test/test-prism-postgres-scale.sh delete mode 100755 test/test-prism-postgres-seed-guard.sh delete mode 100755 test/test-prism-public-read-replica.sh delete mode 100755 test/test-prism-regtest.sh delete mode 100755 test/test-prism-stratum-regtest-live.sh delete mode 100644 tests/fixtures/fake_serve_builder.py create mode 100644 tests/fixtures/prism-alert-scenarios.json create mode 100644 tests/fixtures/prism-deployed-alerts.json create mode 100644 tests/fixtures/prism-legacy-v2-audit-body/README.md create mode 100644 tests/fixtures/prism-legacy-v2-audit-body/prism-audit-bundle-body-0000000000000000000000000000000000000000000000000000c8c8c8c8c8c8-65b11e1b7e2025472fad2e4cd6b555eaba5eab2a4903e17179ba792d58780a4b.json create mode 100644 tests/fixtures/prism-legacy-v2-audit-body/prism-audit-share-segment-slot-1-4.json create mode 100644 tests/fixtures/prism-legacy-v2-audit-body/prism-audit-share-segment-slot-5-8.json create mode 100644 tests/fixtures/prism-load-trend/README.md create mode 100644 tests/fixtures/prism-load-trend/regression/current/throughput-20k-window-1fe.json create mode 100644 tests/fixtures/prism-load-trend/regression/history/trend/L2/2026-09.jsonl create mode 100644 tests/fixtures/prism-load-trend/variance/generate.py create mode 100644 tests/fixtures/prism-load-trend/variance/runner-probe-variance.json create mode 100644 tests/fixtures/window_pipeline_parity/README.md create mode 100644 tests/fixtures/window_pipeline_parity/inputs-unpinned.json create mode 100644 tests/fixtures/window_pipeline_parity/supplementary.json create mode 100644 tests/perf/b275_delta_read.md create mode 100644 tests/perf/b275_persistence_measure.md create mode 100644 tests/perf/b275_post_batch_qualification.md create mode 100644 tests/perf/b275_release_profile.md delete mode 100644 tests/perf/candidate_storage.py delete mode 100644 tests/perf/candidate_window_membership.py delete mode 100644 tests/perf/canonical_audit_retirement.py delete mode 100644 tests/perf/serialize_block_share_cost.py delete mode 100644 tests/perf/stratum_session_concurrency_load.py delete mode 100644 tests/perf/window_builder_pipe_latency.py delete mode 100644 tests/perf/window_ledger_decode_latency.py delete mode 100644 tests/perf/window_paging_allocation.py delete mode 100644 tests/perf/window_pipeline_gil_scaling.py delete mode 100644 tests/prism_candidate_storm.py delete mode 100644 tests/prism_concurrency_harness.py delete mode 100644 tests/prism_coordinator_test_support.py delete mode 100644 tests/prism_landing_harness.py delete mode 100644 tests/prism_postgres_a1_gate.py delete mode 100644 tests/prism_postgres_a1_migration_gate.py delete mode 100644 tests/prism_postgres_a1_process_gate.py delete mode 100644 tests/prism_postgres_a1_revert_gate.py delete mode 100644 tests/prism_postgres_candidate_gate.py delete mode 100644 tests/prism_public_read_replica_gate.py delete mode 100644 tests/prism_regtest_crash_coordinator.py delete mode 100644 tests/prism_regtest_share_miner.py delete mode 100644 tests/prism_vardiff_test_support.py delete mode 100644 tests/prism_window_rows_reference.py delete mode 100644 tests/prism_worker_difficulty_store_gate.py create mode 100644 tests/test_check_deleted_test_map.py create mode 100644 tests/test_check_e2e_scenarios.py create mode 100644 tests/test_check_env_public_credentials.py create mode 100644 tests/test_check_gate_env_reads.py create mode 100644 tests/test_check_gate_manifest.py create mode 100644 tests/test_check_prism_pool_acquires.py create mode 100644 tests/test_check_prism_settings.py create mode 100644 tests/test_check_toolchain_pin.py create mode 100644 tests/test_check_version_skew.py delete mode 100644 tests/test_ctv_broadcaster.py delete mode 100644 tests/test_ctv_broadcaster_daemon.py create mode 100644 tests/test_docs_lab_prism_references.py create mode 100644 tests/test_install_prism_qbit.py create mode 100644 tests/test_measure_postgres_wal.py delete mode 100644 tests/test_prism_accepted_block_accounting.py delete mode 100644 tests/test_prism_accepted_preview_publication_latency.py delete mode 100644 tests/test_prism_accepted_preview_telemetry.py delete mode 100644 tests/test_prism_admission_lock.py delete mode 100644 tests/test_prism_allocator_experiment.py delete mode 100644 tests/test_prism_ancestor_redrive.py delete mode 100644 tests/test_prism_audit_api.py delete mode 100644 tests/test_prism_audit_artifacts.py delete mode 100644 tests/test_prism_audit_bundle_view.py delete mode 100644 tests/test_prism_audit_http.py delete mode 100644 tests/test_prism_audit_root_scan.py delete mode 100644 tests/test_prism_backfill_audit_bundle_canonical.py delete mode 100644 tests/test_prism_background_services.py delete mode 100644 tests/test_prism_block_candidate_collapse.py delete mode 100644 tests/test_prism_block_candidates.py delete mode 100644 tests/test_prism_block_finalization.py delete mode 100644 tests/test_prism_bounded_canonical_finalization.py delete mode 100644 tests/test_prism_bounded_executor.py delete mode 100644 tests/test_prism_builder_pipe_transport.py delete mode 100644 tests/test_prism_candidate_codec.py delete mode 100644 tests/test_prism_candidate_helper.py delete mode 100644 tests/test_prism_candidate_replay_bounds.py delete mode 100644 tests/test_prism_candidate_schema_refusal.py delete mode 100644 tests/test_prism_candidate_spool_integration.py delete mode 100644 tests/test_prism_candidate_spool_view.py delete mode 100644 tests/test_prism_candidate_storage_races.py delete mode 100644 tests/test_prism_candidate_storm.py delete mode 100644 tests/test_prism_candidate_window.py delete mode 100644 tests/test_prism_capacity_evidence.py delete mode 100644 tests/test_prism_completion_callbacks.py delete mode 100644 tests/test_prism_compose_profile.py delete mode 100644 tests/test_prism_concurrency_harness.py create mode 100644 tests/test_prism_container_hardening.py delete mode 100644 tests/test_prism_coordinator_config.py delete mode 100644 tests/test_prism_coordinator_config_loading.py delete mode 100644 tests/test_prism_coordinator_job_cache.py delete mode 100644 tests/test_prism_coordinator_metrics.py delete mode 100644 tests/test_prism_coordinator_shutdown.py delete mode 100644 tests/test_prism_ctv_refresh_priority.py delete mode 100644 tests/test_prism_ctv_runtime.py delete mode 100644 tests/test_prism_dequeue_stale_skip.py delete mode 100644 tests/test_prism_direct_stratum.py delete mode 100644 tests/test_prism_endpoint_registry.py delete mode 100644 tests/test_prism_first_job_latency.py create mode 100644 tests/test_prism_ha_qualification.py create mode 100644 tests/test_prism_ha_readiness_probe.py delete mode 100644 tests/test_prism_heap_census.py delete mode 100644 tests/test_prism_hot_path.py delete mode 100644 tests/test_prism_immutable_refresh_artifacts.py delete mode 100644 tests/test_prism_incremental_payout_window.py delete mode 100644 tests/test_prism_initial_job_delivery.py delete mode 100644 tests/test_prism_issue_224_cadence.py delete mode 100644 tests/test_prism_issue_224_core.py delete mode 100644 tests/test_prism_issue_224_landing.py delete mode 100644 tests/test_prism_issue_224_ledger.py delete mode 100644 tests/test_prism_job_build_exception_retention.py delete mode 100644 tests/test_prism_job_build_promises.py delete mode 100644 tests/test_prism_job_build_retention.py delete mode 100644 tests/test_prism_job_builder.py delete mode 100644 tests/test_prism_job_delivery.py create mode 100644 tests/test_prism_l3_promote.py delete mode 100644 tests/test_prism_landing_append_epoch.py delete mode 100644 tests/test_prism_landing_budget_watchdog.py delete mode 100644 tests/test_prism_landing_harness_model.py delete mode 100644 tests/test_prism_landing_interleavings.py delete mode 100644 tests/test_prism_lease_heartbeat_tail_latency.py delete mode 100644 tests/test_prism_lease_heartbeat_under_fenced_write.py delete mode 100644 tests/test_prism_lease_monitor_stalls.py delete mode 100644 tests/test_prism_lease_orphaned_lock.py delete mode 100644 tests/test_prism_lease_renewal_vs_expiry_claim.py delete mode 100644 tests/test_prism_lease_takeover.py create mode 100644 tests/test_prism_legacy_import_admission_gate.py create mode 100644 tests/test_prism_legacy_parts_digest_check.py create mode 100644 tests/test_prism_legacy_range_sidecars.py create mode 100644 tests/test_prism_load_ab.py create mode 100644 tests/test_prism_load_bridge.py create mode 100644 tests/test_prism_load_matrix.py create mode 100644 tests/test_prism_load_probe.py create mode 100644 tests/test_prism_load_regress.py create mode 100644 tests/test_prism_load_trend.py delete mode 100644 tests/test_prism_metrics.py delete mode 100644 tests/test_prism_observability.py delete mode 100644 tests/test_prism_payout_state.py delete mode 100644 tests/test_prism_payout_window_daemon_recenter.py delete mode 100644 tests/test_prism_pending_block_recovery.py delete mode 100644 tests/test_prism_process_telemetry.py delete mode 100644 tests/test_prism_progress_health.py create mode 100644 tests/test_prism_public_credentials.py delete mode 100644 tests/test_prism_public_dashboard_api.py delete mode 100644 tests/test_prism_public_http_limits.py delete mode 100644 tests/test_prism_public_read_service.py delete mode 100644 tests/test_prism_public_refresh_limits.py delete mode 100644 tests/test_prism_reconnect_backpressure.py create mode 100644 tests/test_prism_recovery_evidence.py create mode 100644 tests/test_prism_recovery_evidence_jobs.py create mode 100644 tests/test_prism_recovery_evidence_parallel.py delete mode 100644 tests/test_prism_refresh_retry_pacing.py delete mode 100644 tests/test_prism_reorg_reconciler.py delete mode 100644 tests/test_prism_replay_enumeration_gate.py delete mode 100644 tests/test_prism_representative_independence.py delete mode 100644 tests/test_prism_retained_jobs.py delete mode 100644 tests/test_prism_reward_window_invariant.py delete mode 100644 tests/test_prism_self_check.py delete mode 100644 tests/test_prism_settlement_balance_comparison.py delete mode 100644 tests/test_prism_share_ledger.py delete mode 100644 tests/test_prism_share_submission.py delete mode 100644 tests/test_prism_share_writer.py delete mode 100644 tests/test_prism_share_writer_service.py create mode 100644 tests/test_prism_shipped_image_lane.py delete mode 100644 tests/test_prism_singleflight_exception_retention.py create mode 100644 tests/test_prism_soak_capture.py delete mode 100644 tests/test_prism_statement_spool.py delete mode 100644 tests/test_prism_stratum_restart_bind.py delete mode 100644 tests/test_prism_stratum_session.py delete mode 100644 tests/test_prism_streamed_block_hex_consumers.py delete mode 100644 tests/test_prism_streamed_submitblock.py delete mode 100644 tests/test_prism_tip_publication_boundary.py delete mode 100644 tests/test_prism_tip_refresh.py delete mode 100644 tests/test_prism_tip_refresh_delivery.py delete mode 100644 tests/test_prism_tip_refresh_epoch.py delete mode 100644 tests/test_prism_tip_refresh_validation.py delete mode 100644 tests/test_prism_vardiff.py delete mode 100644 tests/test_prism_vardiff_durable_resume.py delete mode 100644 tests/test_prism_vardiff_fast_arrival.py delete mode 100644 tests/test_prism_vardiff_resume.py delete mode 100644 tests/test_prism_vardiff_service.py delete mode 100644 tests/test_prism_window_pipeline_rust.py delete mode 100644 tests/test_prism_worker_difficulty_store.py delete mode 100644 tests/test_prism_writer_lease_policy.py delete mode 100644 tests/test_public_dashboard_api_contract.py create mode 100644 tests/test_run_rust_test_shard.py delete mode 100644 tests/test_window_paging_allocation_harness.py delete mode 100644 tests/test_window_pipeline_parity.py delete mode 100644 tests/test_window_pipeline_parity_gate.py delete mode 100644 tests/window_pipeline_parity.py delete mode 100644 tests/window_pipeline_parity_gate.py diff --git a/.env.example b/.env.example index a5cc28273..86a2cd645 100644 --- a/.env.example +++ b/.env.example @@ -35,8 +35,10 @@ QBIT_DATA_SOURCE=qbit-data BITCOIN_DATA_SOURCE=bitcoin-data PRISM_POSTGRES_DATA_SOURCE=prism-postgres-data PRISM_POSTGRES_WAL_SOURCE=prism-postgres-wal -PRISM_POSTGRES_REPLICA_DATA_SOURCE=prism-postgres-replica-data PRISM_AUDIT_DATA_SOURCE=prism-audit-data +# Production only: host directory holding the PRISM signing seed files. It is +# mounted read-only into each mining frontend; see docs/mainnet-deployment.md. +PRISM_SECRETS_SOURCE= QBIT_CHAIN=regtest # Compose defaults this argument to regtest; mainnet deployments must set @@ -47,6 +49,9 @@ QBIT_LISTEN=0 QBIT_RPC_USER=qbitrpc QBIT_RPC_PASSWORD=change-this QBIT_RPC_PORT=18452 +# Optional complete RPC endpoint for both native Prism services, including +# HTTPS or a proxy path. Empty uses QBIT_RPC_HOST/QBIT_RPC_PORT. +QBIT_RPC_URL= QBIT_P2P_PORT=18460 QBIT_RPC_PORT_HOST=127.0.0.1:18452 QBIT_P2P_PORT_HOST=127.0.0.1:18460 @@ -164,107 +169,98 @@ AUXPOW_MINER_PASSWORD=x AUXPOW_REAL_MINER_THREADS=1 AUXPOW_REAL_MINER_TIMEOUT_SECONDS=120 -# Direct PRISM Stratum. Production should use Postgres, real signing seeds, and -# a trusted ledger public key distributed out of band from audit bundles. +# Native Rust PRISM. Every instance connects to the same PostgreSQL writer endpoint. +# Stop Python coordinators before enabling the migration; existing ledger rows remain valid. +# For an external HA database use compose.prism-external-db.yaml and set PRISM_DATABASE_URL. +# Each process receives a UUID unless an explicit, unique PRISM_INSTANCE_ID is configured. +PRISM_INSTANCE_ID= +# Empty chooses available CPU parallelism; builder threads perform CPU work concurrently. +PRISM_RUNTIME_WORKERS= +PRISM_DATABASE_MAX_CONNECTIONS=16 +PRISM_DATABASE_STATEMENT_TIMEOUT_MS=15000 +PRISM_DATABASE_LOCK_TIMEOUT_MS=5000 +PRISM_DATABASE_ACQUIRE_PROBE_IDLE_MS=100 +PRISM_SHARE_COMMIT_TIMEOUT_SECONDS=15 +PRISM_STRATUM_MAX_MESSAGE_BYTES=16384 +# CTV workers coordinate claims in PostgreSQL across all instances. +PRISM_CTV_BROADCASTER_POLL_SECONDS=30 +# Difficulty values below are for regtest. Production requires explicit non-lab values. +# Legacy *_SATS payout options remain aliases for *_BITS options. +# Python worker leases, psql subprocess options, share commit queues, and scheduler tuning +# are obsolete. Accounts/audit snapshots now live in PostgreSQL; the audit mount reads legacy bodies. PRISM_STRATUM_BIND=0.0.0.0 PRISM_STRATUM_PORT=3340 PRISM_STRATUM_PORT_HOST=3340 PRISM_AUDIT_BIND=127.0.0.1 PRISM_AUDIT_PORT=3341 PRISM_AUDIT_DIR=/var/lib/qbit-prism/audit -PRISM_AUDIT_SHARE_SEGMENT_SIZE=10000 -PRISM_AUDIT_LIVE_BUNDLE_RETENTION=5 -PRISM_AUDIT_CANDIDATE_RETENTION_SECONDS=86400 -PRISM_CTV_BROADCAST_ATTEMPT_DETAIL_LIMIT=20 -PRISM_CTV_BROADCAST_RETRY_BACKOFF_SECONDS=300 -PRISM_EVIDENCE_PATH=/var/lib/qbit-prism/audit/prism-live-evidence.json -# Allocator telemetry on /metrics (issue #226). 1 reads glibc mallinfo2 once -# per scrape; 0 renders the allocator gauges unavailable and skips the call. -# The interpreter and component gauges are unconditional either way. -PRISM_MALLOC_TELEMETRY=1 -# glibc malloc arena cap for the coordinator process (issue #226). The image -# defaults to 2 and compose passes this value through to override it; read -# docs/mainnet-deployment.md "PRISM Allocator Settings" before changing it, -# and never leave it empty (glibc reads "" as 0). -MALLOC_ARENA_MAX=2 -# Operator-triggered heap census (SIGUSR1) and malloc_trim (SIGRTMIN+1), -# issue #226 part 2. All off by default; read docs/prism-capacity-readiness.md -# "Heap Census, Allocator Control, and the Resident-Set Bound" before enabling -# any of them. Reports land under PRISM_AUDIT_DIR/heap-census unless -# PRISM_HEAP_CENSUS_DIR is set. -PRISM_HEAP_CENSUS=0 -PRISM_HEAP_CENSUS_TOP_N=40 -PRISM_HEAP_CENSUS_MAX_BYTES=262144 -PRISM_HEAP_CENSUS_MAX_SECONDS=10 -PRISM_HEAP_CENSUS_MIN_INTERVAL_SECONDS=60 -PRISM_HEAP_CENSUS_TRACEMALLOC=0 -PRISM_MALLOC_TRIM_SIGNAL=0 -PRISM_MALLOC_TRIM_INTERVAL_SECONDS=0 -PRISM_STOP_AFTER_BLOCK=0 -PRISM_MAX_BLOCKS=2147483647 PRISM_BLOCKPOLL_SECONDS=2 +PRISM_HASHRATE_ROLLUP_ENABLED=1 +PRISM_HASHRATE_ROLLUP_INTERVAL_SECONDS=15 +PRISM_HASHRATE_ROLLUP_BATCH_SHARES=50000 +# Empty share ledger partitions kept attached ahead of the sequence are +# maintained on this cadence by every frontend; whole seconds, 1 to 86400. +PRISM_SHARE_PARTITION_ENSURE_INTERVAL_SECONDS=60 +PRISM_MIN_PEERS=1 +PRISM_TEMPLATE_MAX_AGE_SECONDS=120 +# Published submit authority; zero restores a live tip RPC for each share. +PRISM_SUBMIT_TIP_MAX_AGE_SECONDS=10 +# Maximum extension while a detected replacement is being prepared. +PRISM_TEMPLATE_REFRESH_FAILURE_EXIT_SECONDS=120 PRISM_BLOCKWAIT_ENABLED=1 -PRISM_BLOCKWAIT_TIMEOUT_SECONDS=5 -# Found-block fast lane: submitblock runs before writer admission/accounting. +# General node reads and wallet RPCs use their own deadline. The short submit +# deadline applies only to submitblock; ambiguous results recover from the outbox. +PRISM_RPC_TIMEOUT_SECONDS=15 PRISM_BLOCK_SUBMIT_RPC_TIMEOUT_SECONDS=1 -# Fresh deadline for each submitter Postgres statement and local DB gate -# (cheap outbox polls and fast-lane-adjacent calls only). -PRISM_BLOCK_SUBMIT_DB_TIMEOUT_SECONDS=1 -# Startup and ancestor re-drive payloads per cursor page (1..1024). -# Set to 1 as a temporary mitigation for large candidate JSON; see #255. -# Smaller pages add DB/RPC round trips and do not bound one candidate's bytes. -PRISM_BLOCK_REPLAY_PAGE_SIZE=1024 -# Landing-class deadline for the post-acceptance accounting tail (persist, -# prior balances, confirm, prepared-state rejection). Must stay well above -# the poll budget; see issue #188. -PRISM_BLOCK_LANDING_DB_TIMEOUT_SECONDS=30 -# Cap for the escalated landing deadline on timed-out landing retries. -PRISM_BLOCK_LANDING_DB_TIMEOUT_MAX_SECONDS=120 -# Fail-closed bound on landed-but-unresolved accepted-parent transitions. -PRISM_ACCEPTED_PARENT_UNRESOLVED_DEPTH_MAX=8 -# Bound candidate cleanup retry records and their retained terminal fences. -# Must be positive and no greater than 8192; lower to constrain overload memory. -PRISM_BLOCK_CANDIDATE_CLEANUP_RETRY_BACKLOG_MAX=4096 -# Emit the contended lock and current submitter phase at this cadence. -PRISM_BLOCK_SUBMIT_LOCK_WAIT_LOG_SECONDS=5 -# Exit nonzero when a timeout-ignoring RPC/DB worker pool stays exhausted this long. -PRISM_BLOCK_SUBMIT_STUCK_CALL_EXIT_SECONDS=30 -PRISM_REORG_RECONCILER_ENABLED=1 +# Rehearsal kill switch (#291): 0 keeps every found block pending and never +# calls submitblock, and keeps the CTV broadcaster from starting. Leave it at +# 1 in production. +PRISM_BLOCK_SUBMIT_ENABLED=1 +# Confirmations on a competing block before an offered candidate is settled orphaned (1..1000). +PRISM_CANDIDATE_ORPHAN_CONFIRMATIONS=6 +PRISM_CAPTURE_OVERPAY_CEILING_BPS=100 +# #529, HA only: the dedicated failover standby's application_name. A found +# block's offer then waits up to PRISM_OFFER_STANDBY_FLUSH_WAIT_MS (0 = off, +# at most 10000) for that standby to flush the block's rows; it is offered +# anyway at the bound. The writer role needs pg_monitor. Empty = no wait. +PRISM_OFFER_STANDBY_APPLICATION_NAME= +PRISM_OFFER_STANDBY_FLUSH_WAIT_MS=250 PRISM_VERSION_ROLLING_MASK=1fffe000 PRISM_COINBASE_TAG=/PRISM/ -# The /public/v1 read surface runs in its own process (the prism-public-api -# compose service, `python3 -m lab.prism.public_read_service`). Every -# PRISM_PUBLIC_* knob below is read by that service, not by prism-coordinator. -# PRISM_PUBLIC_REWARD_WINDOW_CACHE_SECONDS (further down) is the one exception: -# it is read by the ledger, and both processes construct one. +# The public read tier runs as `qbit-prism-server public-api` with its own +# PostgreSQL pool. Shared PRISM_PUBLIC_* metadata/cache settings also apply to +# the combined coordinator listener. PRISM_PUBLIC_API_BIND=0.0.0.0 PRISM_PUBLIC_API_PORT=3342 +PRISM_PUBLIC_RUNTIME_WORKERS=2 +PRISM_PUBLIC_READ_STATEMENT_TIMEOUT_SECONDS=20 +PRISM_POSTGRES_READ_CONCURRENCY=4 +# Content-addressed audit artifacts (/public/v1/artifacts/ of a block's +# audit) are rebuilt from the share window or decoded from sealed bytes, each +# holding the whole window in memory. At most this many run at once, whatever +# PRISM_POSTGRES_READ_CONCURRENCY is (1..64). +PRISM_PUBLIC_AUDIT_REBUILD_CONCURRENCY=1 +# Audit artifact requests running or waiting for a rebuild; the next distinct +# one gets 503 audit_artifact_busy with Retry-After at once (1..4096). +PRISM_PUBLIC_AUDIT_ARTIFACT_MAX_IN_FLIGHT=32 PRISM_PUBLIC_API_PORT_HOST=3342 -# Bound accepted public HTTP connections; excess connections are closed. -PRISM_PUBLIC_HTTP_MAX_CONNECTIONS=64 -# Total request-header deadline and socket I/O idle timeout. One response per connection. -PRISM_PUBLIC_HTTP_TIMEOUT_SECONDS=10 # How often the public read service re-checks that it can reach Postgres for # its own /healthz. Its answer goes stale after max(3 * this, 15) seconds. # The same probe carries the replication check below, so it is also how often # the replica contract is re-evaluated. PRISM_PUBLIC_READINESS_PROBE_INTERVAL_SECONDS=5 -# Per-request database deadline across all origin reads. Set 0 to disable, -# or choose a shorter deadline to finish before the fronting proxy times out. -PRISM_PUBLIC_READ_STATEMENT_TIMEOUT_SECONDS=20 -# Read-only DSN for the standby the public tier serves from, using the -# PRISM_POSTGRES_* credentials the base backup cloned from the primary. This is -# deliberately not PRISM_DATABASE_URL: that one names the primary, and the -# coordinator needs it to. -# Leave empty for Compose to derive the replica URL from PRISM_POSTGRES_USER, -# PRISM_POSTGRES_PASSWORD, and PRISM_POSTGRES_DB. Set an explicit URL for a -# separately provisioned standby or different read credentials. -PRISM_PUBLIC_DATABASE_URL= -# Optional public-read psql command. Empty derives it from the public DSN -# above. This is separate from PRISM_POSTGRES_PSQL_COMMAND for the primary; -# use a replica command here when PRISM_POSTGRES_NATIVE_CLIENT=psql needs -# a custom wrapper or connection arguments. -PRISM_PUBLIC_PSQL_COMMAND= +# DSN for the standby the public tier serves from. This example uses the lab +# bootstrap role; production should use a dedicated reader with its own password. +# This is deliberately not PRISM_DATABASE_URL, which names the writer endpoint. +# See docs/prism-postgres-replica.md#public-reader-credentials for grants and +# authentication. URL-encode special characters in a password inside the DSN. +PRISM_PUBLIC_DATABASE_URL=postgresql://qbit:change-this@prism-postgres-replica:5432/qbit +# Optional password for a public-reader DSN that omits its password. Compose +# passes only this value as the public container's PGPASSWORD; it never falls +# back to PRISM_POSTGRES_PASSWORD. A password in the DSN takes precedence. +# Set the passwordless reader DSN too: this carrier alone does not select a role. +# Single-quote raw .env passwords to keep $ and # literal (do not URL-encode). +PRISM_PUBLIC_POSTGRES_PASSWORD='' # require = refuse replica-backed routes unless the backing server is in # recovery and its replication stream is live within the bound below; off = # serve whatever the DSN points at, which is the pre-replica behaviour. Leave @@ -277,31 +273,22 @@ PRISM_PUBLIC_REPLICA_MODE=require PRISM_PUBLIC_REPLICA_MAX_LAG_SECONDS=60 PRISM_PUBLIC_POOL_NAME=PRISM PRISM_PUBLIC_STRATUM_HOST= -# REQUIRED for make up-prism-pool and prism-public-api. The API runs no -# Stratum listener, so it cannot infer the pool's endpoint. Left empty it -# refuses to start rather than +# REQUIRED for prism-public-api: that process runs no Stratum listener, so it +# cannot infer the pool's endpoint. Left empty it refuses to start rather than # advertise 127.0.0.1 to miners as the pool's Stratum URL. PRISM_PUBLIC_STRATUM_URL= -# Optional external URL for the high-diff endpoint in /public/v1/mining-configuration. -# Used only when PRISM_STRATUM_HIGHDIFF_PORT is set; empty derives from -# PRISM_PUBLIC_STRATUM_URL first, then PRISM_PUBLIC_STRATUM_HOST, plus -# PRISM_STRATUM_HIGHDIFF_PORT. PRISM_PUBLIC_STRATUM_HIGHDIFF_URL= PRISM_PUBLIC_POOL_FEE_BPS=0 PRISM_PUBLIC_CONFIGURATION_LABEL="PRISM default" PRISM_PUBLIC_CONFIGURATION_DESCRIPTION= PRISM_PUBLIC_BLOCK_TEMPLATE_POLICY="pool-selected qbit block template with PRISM payout settlement" PRISM_PUBLIC_MINIMUM_PAYOUT_BITS= -# Trailing window (seconds) used to steady /public/v1/hashrate-series points on -# sub-window buckets (5m); 0 serves raw per-bucket rates. Clamped to [0, 86400]. PRISM_PUBLIC_HASHRATE_SMOOTHING_SECONDS=1800 PRISM_PUBLIC_EXPLORER_TX_URL_PREFIX= PRISM_PUBLIC_EXPLORER_BLOCK_URL_PREFIX= PRISM_PUBLIC_CACHE_ENABLED=1 PRISM_PUBLIC_CACHE_TTL_SECONDS=5 PRISM_PUBLIC_CACHE_STALE_WHILE_REVALIDATE_SECONDS=30 -# Pool-wide aggregate endpoints (pool-summary, hashrate-series, miner workers) -# are expensive to recompute and tolerate more staleness than row reads. PRISM_PUBLIC_AGGREGATE_CACHE_TTL_SECONDS=30 PRISM_PUBLIC_AGGREGATE_CACHE_STALE_WHILE_REVALIDATE_SECONDS=30 PRISM_PUBLIC_CONFIG_CACHE_TTL_SECONDS=300 @@ -313,17 +300,7 @@ PRISM_PUBLIC_CACHE_MAX_RESPONSE_BYTES=1048576 PRISM_PUBLIC_CACHE_DEBUG_HEADERS=0 PRISM_STRATUM_EXTRANONCE2_SIZE=8 PRISM_STRATUM_SHARE_DIFF=0.000000001 -PRISM_STRATUM_SHARE_WEIGHT=1 -PRISM_STRATUM_SHARE_WEIGHTS_JSON= PRISM_STRATUM_VARDIFF=1 -# The 1e-9 values below are for local labs. Production requires reviewed, -# explicit non-lab difficulty values. -# Release provenance (digest-qualified *_IMAGE references and absolute -# *_DATA_SOURCE host paths) is always enforced when QBIT_CHAIN=mainnet; no -# variable can disable it there. Set this to 1 to hold another chain to the same -# contract. Leave it 0 for production pools that build images from the pinned -# source (for example a public testnet4 deployment), which keeps every -# behavioral production check active. QBIT_REQUIRE_RELEASE_PROVENANCE=0 PRISM_STRATUM_VARDIFF_TARGET_SECONDS=15 PRISM_STRATUM_VARDIFF_MIN_DIFF=0.000000001 @@ -333,204 +310,73 @@ PRISM_STRATUM_VARDIFF_MAX_STEP_UP=4 PRISM_STRATUM_VARDIFF_MAX_STEP_DOWN=4 PRISM_STRATUM_VARDIFF_START_DIFF=0.000000001 PRISM_STRATUM_VARDIFF_EWMA_ALPHA=0.4 -PRISM_STRATUM_VARDIFF_RETARGET_TOLERANCE=0.25 -# Fast-arrival initial convergence. Credit is target-based: a session is paid -# for the difficulty its job was stamped at, so every retarget interval spent -# climbing toward the difficulty it is really working at is credit the miner -# never receives. A rental router that filters submissions at its own, much -# higher difficulty makes this acute -- the pool sees only the survivors, so -# the ordinary MAX_STEP_UP ladder needs several full RETARGET_SECONDS windows -# to catch up. These let ONE early move per connection, before the ordinary -# interval and with a wider step bound, close that gap. Set to 0 to restore -# the previous behavior exactly. +# Persist accepted-work evidence across reconnects, listeners, and frontends. +PRISM_STRATUM_VARDIFF_RESUME=1 +PRISM_STRATUM_VARDIFF_RESUME_TTL_SECONDS=900 +PRISM_STRATUM_VARDIFF_RESUME_MAX_ENTRIES=8192 +PRISM_STRATUM_VARDIFF_RESUME_MAX_START_FACTOR=1024 PRISM_STRATUM_VARDIFF_INITIAL_CONVERGENCE=1 -# Step bound for that one initial move; must be >= PRISM_STRATUM_VARDIFF_MAX_STEP_UP, -# which every later retarget goes back to. 64 is what lets a 16384 start cross -# the 500000 a NiceHash-style router filters at in a single move. PRISM_STRATUM_VARDIFF_INITIAL_MAX_STEP_UP=64 -# Accepted shares of evidence required before the early move is allowed. PRISM_STRATUM_VARDIFF_INITIAL_MIN_SHARES=8 -# Minimum observed/current difficulty gap required. This is the noise guard: -# at 8 shares an exactly-on-target miner still has a ~35% spread on its -# observed rate, so demanding a 4x gap keeps an ordinary miner from being -# pushed up merely because this path exists, and makes the early move -# upward-only. Lower it only with evidence. PRISM_STRATUM_VARDIFF_INITIAL_MIN_STEP_UP=4 -# Minimum window length before an early observation counts, so a burst -# delivered in a near-zero interval cannot divide real work by an almost-zero -# elapsed time and explode the observed rate. PRISM_STRATUM_VARDIFF_INITIAL_MIN_SECONDS=1 -# Observation only: the difficulty at which qbit_prism_vardiff_high_diff_arrival_* -# records how long, and how many accepted shares, a session needed to arrive. -# Never gates or clamps a retarget. Defaults to the high-diff listener floor. -# Set it inside the listener's MAX_DIFF or the arrival histograms stay empty. -PRISM_STRATUM_VARDIFF_HIGH_DIFF_ARRIVAL_DIFF=500000 -# Set to 0 to disable idle zero-share vardiff step-down sweeps. -PRISM_STRATUM_VARDIFF_IDLE_SWEEP_SECONDS=15 -# Reconnect difficulty resume: a reconnecting worker (same listener + exact -# username) starts at its last converged difficulty instead of the lane -# start, so a mass reconnect does not re-flood the share path while every -# session re-climbs. 0 restores lane-start behavior exactly. -# LIMITATION: retention is keyed by (listener, exact Stratum username) and is -# last-writer-wins, so heterogeneous rigs that deliberately share one username -# share one retained value: a small rig reconnecting after a big one can adopt -# the big rig's converged difficulty (and vice versa) until its own shares -# retarget it. The adopted value is still bounded by the lane clamp below. -# Give each rig its own worker name (address.rig) to keep the values separate; -# PRISM_STRATUM_MAX_CONNECTIONS_PER_USERNAME bounds how many connections can -# share one identity in the first place. -# With the PostgreSQL ledger, accepted-share evidence is also retained in -# qbit_worker_difficulty and preloaded before listeners accept, so recent -# converged values survive coordinator restarts. Non-PostgreSQL backends keep -# only the process-local cache and therefore start empty after a restart. -# The TTL is measured from accepted-share evidence recorded at a committed -# retarget or orderly disconnect. A long-running session that no longer -# retargets can therefore age out before an abrupt process exit even while it -# remains connected; this is bounded reconnect assistance, not an active- -# session lease or a guarantee that every restart resumes every worker. -PRISM_STRATUM_VARDIFF_RESUME=1 -# How long a retained difficulty stays adoptable. WARNING: when -# PRISM_STRATUM_VARDIFF_IDLE_SWEEP_SECONDS=0 (as deployed) there is no idle -# step-down sweep, so a session resumed above its true hashrate can only be -# corrected by an accepted share; keep this TTL short enough that retained -# values still reflect recent hashrate. The TTL always measures the age of the -# converged value itself: reconnecting and disconnecting without submitting a -# share does not extend it. -PRISM_STRATUM_VARDIFF_RESUME_TTL_SECONDS=900 -# Bound on retained per-worker entries (2x the deployed 4096 connection cap). -PRISM_STRATUM_VARDIFF_RESUME_MAX_ENTRIES=8192 -# Plausibility ceiling for a resumed value, as a multiple of the lane's own -# start difficulty (1024 = five 4x retarget steps). A retained value above -# min(lane max difficulty, start * factor) is clamped before adoption. -# WARNING: with PRISM_STRATUM_VARDIFF_IDLE_SWEEP_SECONDS=0 (as deployed), a -# rig that converged high and reconnects within the TTL on weaker hardware -# can resume too high, emit no shares, and give the retargeter no signal to -# come down — stuck for that connection's life (bounded by the TTL above; -# only that miner is harmed). On hardware where converged-high rigs stall -# after resume, lower this factor. -PRISM_STRATUM_VARDIFF_RESUME_MAX_START_FACTOR=1024 -# Seconds a share computed against the just-replaced tip is still credited -# (never block-submitted). Keep a small window that covers miner job-switch -# latency: at 0 every in-flight prior-tip share is rejected at each block, -# which miners read as pool failure. +PRISM_STRATUM_VARDIFF_RETARGET_TOLERANCE=0.25 PRISM_STRATUM_STALE_GRACE_SECONDS=3 -# Clean same-tip jobs remain valid against their original context for this -# bounded window. Set the TTL to 0 to disable; positive TTLs require positive -# per-connection caps. Production additionally requires a positive pool-wide -# connection limit, bounding retained contexts by the product of both limits. PRISM_STRATUM_SAME_TIP_JOB_RETENTION_SECONDS=30 PRISM_STRATUM_SAME_TIP_JOB_RETENTION_PER_CONNECTION=64 -# Staged latest-wins tip-refresh epochs. Leave disabled until rollout. -PRISM_TIP_REFRESH_EPOCH_FANOUT=0 -# Same-tip contexts retained past their connection's disconnect so a -# same-username reconnect (proxy flap) can submit in-flight work instead of -# eating unknown-job rejects. Count cap across all dead connections; entries -# keep the same-tip retention TTL. 0 restores purge-on-disconnect. -PRISM_STRATUM_DISCONNECTED_JOB_RETENTION=16384 -# Persistent ready-pool fanout workers. Values above 16 are rejected. -PRISM_TIP_REFRESH_MAX_WORKERS=16 -PRISM_BUNDLE_BUILD_TIMEOUT_SECONDS=60 -# Reuse one exact payout window across routine generations. Normal delta -# refreshes are limited to the re-anchor cadence; found-block settlement -# bypasses the interval, and the full oracle periodically checks parity. -PRISM_PAYOUT_ARTIFACT_REUSE=1 -PRISM_PAYOUT_ARTIFACT_REARM_MIN_SECONDS=5 PRISM_PAYOUT_ARTIFACT_REANCHOR_SECONDS=60 -PRISM_PAYOUT_ARTIFACT_MIN_BUILD_INTERVAL_SECONDS=60 -PRISM_PAYOUT_ARTIFACT_FULL_RESCAN_SECONDS=3600 -PRISM_PAYOUT_ARTIFACT_MAX_ANCHOR_AGE_SECONDS=300 -# Route the payout-window fold, canonical digest, and incremental advance -# through the persistent --serve audit builder instead of in-process Python. -# Strictly boolean (anything else refuses startup); inert while the daemon -# transport is disabled, and any daemon anomaly still degrades to the -# in-process pipeline for that materialization without flipping this. The -# daemon carries integers in fixed widths (share_difficulty/network_difficulty -# u128, share_seq/template_height u64, ntime u32, millisecond stamps i64); a -# window holding a value beyond them is declined as out_of_range and folded -# in-process, with the daemon kept alive. -PRISM_WINDOW_PIPELINE_RUST=0 -PRISM_WORKER_METRICS_LIMIT=100 -# Coordinator maintenance of public hashrate rollups. Set 0 for raw share scans. -PRISM_HASHRATE_ROLLUP_ENABLED=1 -# Pause between caught-up passes; initial backfill runs batches back to back. -PRISM_HASHRATE_ROLLUP_INTERVAL_SECONDS=15 -PRISM_HASHRATE_ROLLUP_BATCH_SHARES=50000 -# Global thread-per-connection capacity across every PRISM listener. The -# production-facing default is above the normal 200-250 connection population; -# set it explicitly to 0 only for focused local/regtest unlimited operation. +# Connections the kernel queues on each Stratum listener before accept; a +# reconnect burst beyond it has its SYNs dropped. The kernel caps it at +# net.core.somaxconn. +PRISM_STRATUM_LISTEN_BACKLOG=4096 PRISM_STRATUM_MAX_CONNECTIONS=384 +# Zero disables the limit; retained work keeps its original username's slot. PRISM_STRATUM_MAX_CONNECTIONS_PER_USERNAME=0 -# Bound authorized clients awaiting first current-tip work and expire them if -# delivery cannot make progress. A zero timeout disables expiry for focused -# tests; the pending-client bound remains positive. +# Concurrent connections allowed from one observed peer address, across both +# Stratum listeners. Zero disables the cap. Leave it at zero unless the +# deployment preserves miner source addresses: behind a terminating TCP proxy, +# NAT egress or a Docker userland proxy many miners share one address. +# See docs/prism-b4-stratum-admission.md before raising it. +PRISM_STRATUM_MAX_CONNECTIONS_PER_IP=0 +# Window the three per-session budgets below are measured over, in seconds. +PRISM_STRATUM_SESSION_BUDGET_INTERVAL_SECONDS=60 +# Malformed frames one session may send per interval before it is +# disconnected. Frames, not bytes; zero disables the budget. +PRISM_STRATUM_MAX_MALFORMED_FRAMES_PER_INTERVAL=0 +# Unknown-job submits (each one ledger lookup) one session may make per +# interval before it is disconnected. Honest staleness after a tip change and +# transient resume races produce bursts here, so size it well above a miner's +# in-flight submits; zero disables the budget. +PRISM_STRATUM_MAX_UNKNOWN_JOBS_PER_INTERVAL=0 +# mining.authorize attempts one session may make per interval, bounding the +# address validation RPCs a single connection can drive. Zero disables it. +PRISM_STRATUM_MAX_AUTHORIZE_ATTEMPTS_PER_INTERVAL=0 +# Job builds admitted at once per frontend. A quarter of them, at least one, +# is also the rebuild lane (no setting of its own): sessions whose work is +# still on the published parent and payout revision rebuild through it and +# keep it until the job is persisted, so the rest stay free for first jobs and +# for sessions whose work a new tip or payout revision superseded (#604). PRISM_STRATUM_MAX_PENDING_INITIAL_JOBS=128 PRISM_STRATUM_INITIAL_JOB_TIMEOUT_SECONDS=30 -# Dedicated first-job delivery workers. Raise under reconnect churn so first -# notifies do not queue behind each other; the pending-client bound above is -# the ceiling, since extra workers past it can never receive a task. -PRISM_INITIAL_JOB_MAX_WORKERS=4 -# Shared job-build executor threads. The scheduler admits at most two -# concurrent build flights, so values above 2 are rejected; lower to 1 to -# serialize builds on GIL-constrained single-core hosts. PRISM_JOB_BUILD_EXECUTOR_WORKERS=2 -# Interpreter thread switch interval, in seconds. Unset leaves CPython's own -# default (5ms) in force, which is what every deployment has run to date; -# setting it calls sys.setswitchinterval() before any thread starts. -# This trades share-ack tail latency against interpreter overhead, and the -# balance is host-specific -- #143's measurements moved throughput 15% and -# CPU-per-share 13% across a sweep, with the 5ms default showing the worst -# ack tail of the three settings tried. Re-measure per host with -# tests/perf/stratum_session_concurrency_load.py rather than copying a value -# between machines; a host with slower cores or a different core count will -# not share an optimum. Values above 1 are rejected at startup as a misplaced -# decimal point, and values below 1e-06 because CPython stores the interval in -# whole microseconds and would truncate them to zero -- a switch check at every -# opportunity. For the same reason a tuned value may read back rounded down to -# the nearest microsecond. -# PRISM_PYTHON_SWITCH_INTERVAL_SECONDS=0.002 -# Suppress mining-delivery health failures during bounded startup preparation. -PRISM_MINING_HEALTH_STARTUP_GRACE_SECONDS=30 -# Keep listener threads alive during transient process/system descriptor -# exhaustion instead of waiting for the watchdog to restart the coordinator. -PRISM_STRATUM_ACCEPT_RESOURCE_EXHAUSTION_BACKOFF_SECONDS=1 -# Successful payout-address validations use a bounded LRU/TTL cache. Set either -# value to 0 to disable caching without disabling concurrent request deduping. -PRISM_PAYOUT_ADDRESS_CACHE_MAX_ENTRIES=4096 -PRISM_PAYOUT_ADDRESS_CACHE_TTL_SECONDS=3600 -# Raise the coordinator's soft descriptor allowance without lowering the -# existing deployment's hard ceiling. +# Refresh builder pool threads per frontend process; empty = three quarters of +# the cores (at most 4), 0 = build serially, at most 64. +PRISM_REFRESH_BUILD_THREADS= +# After each block landing, return the heap the landing freed to the kernel +# (malloc_trim, glibc builds; #600). 0 turns it off, which lets a 400k-window +# frontend's resident set climb to about 3.3 GB instead of about 0.8 GB. +PRISM_LANDING_MALLOC_TRIM_ENABLED=1 PRISM_COORDINATOR_NOFILE_SOFT=65536 PRISM_COORDINATOR_NOFILE_HARD=524288 -# Optional second Stratum listener for rental-scale miners (e.g. NiceHash -# orders, which require a >=500000 share difficulty from the first -# mining.set_difficulty they see). Disabled unless PRISM_STRATUM_HIGHDIFF_PORT -# is set. Difficulty bounds are independent of the default listener; all other -# vardiff knobs (target/retarget/steps) inherit PRISM_STRATUM_VARDIFF_*. -# Clients may raise (never lower) their difficulty within these bounds via the -# password convention d=N,md=N or mining.suggest_difficulty. PRISM_STRATUM_HIGHDIFF_PORT= -# Host publish mapping for the high-diff listener. Leave empty while the -# listener is disabled (compose then maps an ephemeral loopback port so host -# 4334 is never occupied); set to 4334 together with -# PRISM_STRATUM_HIGHDIFF_PORT when enabling it. PRISM_STRATUM_HIGHDIFF_PORT_HOST= PRISM_STRATUM_HIGHDIFF_BIND= PRISM_STRATUM_HIGHDIFF_START_DIFF=500000 -# MIN_DIFF is a wire guarantee: stamped jobs advertise at least this value -# even while qbit network difficulty is below it (young chain, test networks), -# because marketplace verification judges the first mining.set_difficulty. PRISM_STRATUM_HIGHDIFF_MIN_DIFF=500000 PRISM_STRATUM_HIGHDIFF_MAX_DIFF=4294967296 -# Fixed difficulty when vardiff is disabled; empty tracks -# PRISM_STRATUM_HIGHDIFF_START_DIFF and must stay within [MIN_DIFF, MAX_DIFF]. PRISM_STRATUM_HIGHDIFF_SHARE_DIFF= -# Below this many distinct accepted miners, jobs run in collection mode and a -# solved block settles solver-pays-all through its signed bootstrap manifest. -PRISM_MIN_READY_MINERS=3 -# Empty means testnet/signet chains use the built-in tq fallback address, while -# regtest and mainnet keep invalid usernames fail-closed unless explicitly set. +# Used for explicitly invalid or unsupported addresses. RPC failures reject authorization. PRISM_USERNAME_FALLBACK_ADDRESS= -# Fixed absolute miner payout floor. Empty uses the formula inputs below. PRISM_PAYOUT_MIN_OUTPUT_SATS= PRISM_PAYOUT_P2MR_SPEND_INPUT_BYTES=3680 PRISM_PAYOUT_TARGET_FEERATE_SATS_PER_BYTE=1 @@ -538,98 +384,50 @@ PRISM_PAYOUT_SAFETY_MULTIPLIER=4 PRISM_CTV_SETTLEMENT_ENABLED=0 PRISM_CTV_FANOUT_FEE_MARKET_RATE_BITS_PER_1000_WEIGHT= PRISM_CTV_FANOUT_FEE_PREMIUM_BPS=12000 -PRISM_CTV_FANOUT_FEE_ESTIMATE_TARGET_BLOCKS=2 PRISM_CTV_BROADCASTER_ENABLED= +PRISM_CTV_SPEND_SCAN_BLOCKS=32 PRISM_CTV_BROADCASTER_WALLET= PRISM_CTV_BROADCASTER_FEE_BITS=0 PRISM_CTV_BROADCASTER_LIMIT=100 -PRISM_CTV_BROADCASTER_CHUNK_SIZE=5 -PRISM_CTV_BROADCASTER_INTERVAL_SECONDS=30 -PRISM_POOL_FEE_ENABLED=0 -PRISM_POOL_FEE_BPS= -# Configure either a qbit P2MR fee address or a raw 32-byte P2MR program. +# Without a pool fee, sub-floor dust cannot be settled: once any balance is +# positive but below the payout floor, every work build fails and mining stops +# (#525). Every settlement mode, direct and CTV, therefore requires +# PRISM_POOL_FEE_ENABLED=1 with a recipient (#535); PRISM_POOL_FEE_BPS=0 is +# allowed and pays nothing until dust must be swept. +# The recipient below is a synthetic lab program nobody holds a key for, so a +# fresh lab setup is valid; production refuses it. For a real pool, set your +# own recipient: either PRISM_POOL_FEE_ADDRESS= with +# PRISM_POOL_FEE_P2MR_PROGRAM_HEX and PRISM_POOL_FEE_RECIPIENT_ID left empty, +# or your own 32-byte PRISM_POOL_FEE_P2MR_PROGRAM_HEX and a +# PRISM_POOL_FEE_RECIPIENT_ID. Exactly one of the address and the program. +PRISM_POOL_FEE_ENABLED=1 +PRISM_POOL_FEE_BPS=0 PRISM_POOL_FEE_ADDRESS= -PRISM_POOL_FEE_P2MR_PROGRAM_HEX= -PRISM_POOL_FEE_RECIPIENT_ID= +PRISM_POOL_FEE_P2MR_PROGRAM_HEX=dfeedfeedfeedfeedfeedfeedfeedfeedfeedfeedfeedfeedfeedfeedfeedfee +PRISM_POOL_FEE_RECIPIENT_ID=dev-pool-fee PRISM_POOL_FEE_ORDER_KEY= -# Coinbase output ordering: canonical (compatibility default) or pool-fee-first. -# pool-fee-first requires an enabled pool fee, keeps the fee out of CTV fanout, -# and emits a positive fee output at coinbase vout 0. PRISM_COINBASE_OUTPUT_POLICY=canonical PRISM_POSTGRES_IMAGE=postgres:16-alpine PRISM_POSTGRES_DB=qbit PRISM_POSTGRES_USER=qbit PRISM_POSTGRES_PASSWORD=change-this -# Replica bootstrap retry budget and delay, forwarded to the standby container. -PRISM_POSTGRES_REPLICA_BASEBACKUP_ATTEMPTS=60 -PRISM_POSTGRES_REPLICA_BASEBACKUP_RETRY_SECONDS=5 PRISM_DATABASE_URL=postgresql://qbit:change-this@prism-postgres:5432/qbit -PRISM_POSTGRES_PSQL_COMMAND= -# Initialization applies 001_share_ledger.sql and 002_candidate_bodies.sql, -# including durable vardiff state and candidate-body storage. If disabled, -# apply both files from crates/qbit-prism/sql in that order before startup. -# The coordinator and offline recovery command refuse an unmigrated database. PRISM_POSTGRES_INIT_SCHEMA=1 -PRISM_POSTGRES_READ_CONCURRENCY=4 -# Issue #255: block-candidate bodies are written as bounded 256 KiB chunks -# (storage version 2). Set to 1 only for the compatibility/rollback release -# that must keep writing the legacy whole-jsonb candidate; the reader handles -# both versions. Once any version-2 row exists, a pre-#255 coordinator must -# not be started against the database. -PRISM_CANDIDATE_STORAGE_VERSION=2 -# Local spool directory for hydrated replay bodies (empty: the system temp -# directory) and the byte reservation that bounds concurrent spool usage. -PRISM_CANDIDATE_SPOOL_DIR= -PRISM_CANDIDATE_SPOOL_RESERVATION_BYTES=4294967296 -# How long the server lets a coordinator session sit idle inside an open -# transaction before aborting it and releasing its locks. Bounds how long a -# vanished client (partition, SIGSTOP, VM pause) can strand the writer-lease -# row lock and block a successor's startup. -PRISM_POSTGRES_IDLE_IN_TRANSACTION_TIMEOUT_SECONDS=15 -# Server-side TCP keepalives toward coordinator sessions, the backstop for -# backends the idle-in-transaction timer cannot cover. On a TCP connection -# whose platform implements them, the defaults bound dead-socket detection at -# idle + count x interval = 30 + 3x10 = 60s instead of the OS default of -# roughly two hours; Unix-socket DSNs ignore them. -PRISM_POSTGRES_TCP_KEEPALIVES_IDLE_SECONDS=30 -PRISM_POSTGRES_TCP_KEEPALIVES_INTERVAL_SECONDS=10 -PRISM_POSTGRES_TCP_KEEPALIVES_COUNT=3 -# How long the coordinator reuses one pool-wide PRISM reward-window aggregate -# across public miner-page requests before re-running the recursive window -# scan; 0 recomputes on every request. -# Read by the ledger, so prism-coordinator and prism-public-api both set it. -# It is also the one cache stacked under another on the public surface, which -# is why /public/v1/miners/{recipient_id} carries a larger staleness budget -# (105s) than the other plain row reads (15s). -PRISM_PUBLIC_REWARD_WINDOW_CACHE_SECONDS=30 -PRISM_SHARE_COMMIT_BATCH_SIZE=64 -PRISM_SHARE_COMMIT_LINGER_MILLISECONDS=5 -PRISM_SHARE_COMMIT_TIMEOUT_SECONDS=15 -PRISM_TEMPLATE_REFRESH_FAILURE_EXIT_SECONDS=120 -PRISM_COORDINATION_BLOCKED_EXIT_SECONDS=900 -# How long an own-hash tip observation keeps protecting a submitted block -# candidate from terminal abandonment while instantaneous chain probes -# disagree (lost submitblock acks, transient fork views). -PRISM_OBSERVED_TIP_ACCEPT_WINDOW_SECONDS=300 -PRISM_HEALTH_PENDING_REFRESH_MAX_AGE_SECONDS=15 PRISM_HEALTH_TIP_POLL_MAX_AGE_SECONDS=15 -PRISM_LEDGER_WRITER_ID=prism-coordinator -PRISM_LEDGER_WRITER_EPOCH=1 -PRISM_LEDGER_WRITER_SESSION_TOKEN= PRISM_LEDGER_WRITER_PUBLIC_KEY_HEX= -PRISM_LEDGER_LEASE_TTL_SECONDS=60 -# Per-attempt lock/statement deadline and attempt budget for the startup -# writer-lease acquisition. Attempts x deadline (5x5s = 25s) is sized to -# outlast PRISM_POSTGRES_IDLE_IN_TRANSACTION_TIMEOUT_SECONDS, so an orphaned -# lock is normally reaped mid-budget and startup self-heals; an acquisition -# that never completes within the budget exits the coordinator visibly -# instead of blocking for hours. -PRISM_LEDGER_LEASE_ACQUIRE_LOCK_TIMEOUT_SECONDS=5 -PRISM_LEDGER_LEASE_ACQUIRE_ATTEMPTS=5 -# Bounds durable writer flushing before shutdown leaves the lease to its TTL. -PRISM_WRITER_QUIESCENCE_TIMEOUT_SECONDS=15 PRISM_MANIFEST_SIGNING_SEED_HEX= PRISM_LEDGER_ATTESTATION_SIGNING_SEED_HEX= +# Mounted-file forms of the two seeds, as paths inside the container. +# compose.production.yaml points them at PRISM_SECRETS_SOURCE and clears the +# direct values above, which production rejects. A lab may use either form, +# but not both for one seed. +PRISM_MANIFEST_SIGNING_SEED_HEX_FILE= +PRISM_LEDGER_ATTESTATION_SIGNING_SEED_HEX_FILE= +# Optional bearer token for the operator listener (PRISM_AUDIT_PORT). Set at +# most one form; the built-in healthcheck sends it, and the public API never +# receives it. +PRISM_OPERATOR_BEARER_TOKEN= +PRISM_OPERATOR_BEARER_TOKEN_FILE= PRISM_ALLOW_MEMORY_LEDGER=0 PRISM_ALLOW_TEST_SIGNING_SEEDS=0 PRISM_ALLOW_BUNDLE_EMBEDDED_LEDGER_KEY=0 diff --git a/.github/actionlint.yaml b/.github/actionlint.yaml new file mode 100644 index 000000000..e8a0cb81d --- /dev/null +++ b/.github/actionlint.yaml @@ -0,0 +1,12 @@ +# actionlint configuration: the workflow runs on Blacksmith runners, whose +# labels actionlint does not know. +self-hosted-runner: + labels: + - blacksmith-2vcpu-ubuntu-2404 + # The nightly load harness (prism-load-nightly.yml) runs its presets on + # the runner each preset names. + - blacksmith-8vcpu-ubuntu-2404 + - blacksmith-16vcpu-ubuntu-2404 + # The runner probe (prism-load-runner-probe.yml, #541) also measures the + # 32 vCPU class. + - blacksmith-32vcpu-ubuntu-2404 diff --git a/.github/actions/apt-install/action.yml b/.github/actions/apt-install/action.yml new file mode 100644 index 000000000..d1da95da2 --- /dev/null +++ b/.github/actions/apt-install/action.yml @@ -0,0 +1,56 @@ +name: Install Ubuntu packages +description: >- + Install Ubuntu packages with .github/scripts/apt-install.sh, which retries, + falls back to the mirrors in .github/apt/ubuntu-mirrors.txt, and keeps the + downloaded packages in a cache (saved with actions/cache) so an install can + still succeed while the Ubuntu mirrors are unreachable. + +inputs: + packages: + description: Space-separated Ubuntu package names. + required: true + +runs: + using: composite + steps: + - name: Compute the package cache key + id: key + shell: bash + env: + PACKAGES: ${{ inputs.packages }} + run: | + set -euo pipefail + read -r -d '' -a packages <<< "${PACKAGES}" || true + digest="$(printf '%s\n' "${packages[@]}" | sort -u | sha256sum | cut -c1-16)" + # One cache per package set and month: a new month saves a fresh set, + # and restores from the previous month until then. + prefix="apt-install-v1-${RUNNER_OS}-${RUNNER_ARCH}-${digest}-" + { + printf 'prefix=%s\n' "${prefix}" + printf 'key=%s%s\n' "${prefix}" "$(date -u +%Y-%m)" + } >> "${GITHUB_OUTPUT}" + + - name: Restore the package cache + id: restore + uses: actions/cache/restore@v6 + with: + path: ~/.cache/apt-install + key: ${{ steps.key.outputs.key }} + restore-keys: ${{ steps.key.outputs.prefix }} + + - name: Install the packages + shell: bash + env: + PACKAGES: ${{ inputs.packages }} + run: | + set -euo pipefail + read -r -d '' -a packages <<< "${PACKAGES}" || true + APT_INSTALL_CACHE_DIR="${HOME}/.cache/apt-install" \ + bash "${GITHUB_ACTION_PATH}/../../scripts/apt-install.sh" "${packages[@]}" + + - name: Save the package cache + if: steps.restore.outputs.cache-hit != 'true' + uses: actions/cache/save@v6 + with: + path: ~/.cache/apt-install + key: ${{ steps.key.outputs.key }} diff --git a/.github/actions/prism-load-evidence/action.yml b/.github/actions/prism-load-evidence/action.yml new file mode 100644 index 000000000..6fea4282e --- /dev/null +++ b/.github/actions/prism-load-evidence/action.yml @@ -0,0 +1,237 @@ +name: Record PRISM load evidence +description: >- + #551 (S6 of #487): read the ci-evidence trend, apply the report-only + regression rule to this run's trend rows, and, when `write` is true, + promote the selected bundles to a release, append the rows to ci-evidence + and open or update the one regression tracking issue. With `write` false + (pull requests and dispatches) it only reads and reports: nothing is + recorded, promoted or filed. Expects this repository checked out at the + workspace root (the scripts and the regression config are read from it) + and the rows as scripts/prism_load_trend.py `row` wrote them, one *.json + per preset run. A writing job needs `contents: write` (rows, releases), + `issues: write` and `actions: read` (to download the bundles it promotes); + it must be a scheduled run or a `v*` tag push, and the first step refuses + anything else, as `append` refuses any push row whose `ref` is not + refs/tags/v*. For #550's tag job: rows built with `--lane L3 --ref + "$GITHUB_REF"`, `rows-dir` holding them, + `promote: all`, `release: ` (the release must already exist), + and `regression-config: ''` unless L3 joins the rule's lanes. + +inputs: + rows-dir: + description: Directory of this run's trend rows (*.json). + required: true + write: + description: >- + `true` to promote, append and file the issue; only on a schedule or a + `v*` tag push. + required: false + default: "false" + promote: + description: >- + Which rows' bundles to promote: `policy` (L3, L5 and the weekly soak always, L2 when + flagged), `all`, `flagged` or `none`. + required: false + default: policy + release: + description: >- + `monthly` for the rolling ci-evidence/YYYY-MM pre-release (created when + missing), or an existing release's tag (the full suite's). + required: false + default: monthly + regression-config: + description: The rule's thresholds; empty skips the rule. + required: false + default: test/prism-load-regression.toml + variance: + description: >- + #542's variance document (scripts/prism_load_probe.py variance --out). + Empty uses test/prism-load-variance.json when it is checked in, and the + provisional spread otherwise. + required: false + default: "" + issue-label: + description: The label of the one regression tracking issue. + required: false + default: prism-load-regression + artifact-name: + description: Name of the artifact the verdict is uploaded as. + required: false + default: prism-load-regression-verdict + github-token: + description: The job's token. + required: true + +outputs: + regressions: + description: Numbers the rule flagged; `unknown` when it could not run, empty when skipped. + value: ${{ steps.regress.outputs.regressions }} + +runs: + using: composite + steps: + - name: Refuse a write from an event that may not write + if: ${{ inputs.write == 'true' }} + shell: bash + env: + EVENT: ${{ github.event_name }} + REF: ${{ github.ref }} + run: | + set -euo pipefail + if [[ "${EVENT}" == schedule || ( "${EVENT}" == push && "${REF}" == refs/tags/v* ) ]]; then + exit 0 + fi + echo "::error::${EVENT} on ${REF} may not write evidence; pull request and dispatch runs produce artifacts only (#551)" + exit 1 + + - name: Read the trend from ci-evidence + id: history + shell: bash + run: | + set -uo pipefail + dir="${RUNNER_TEMP}/prism-load-history" + rm -rf "${dir}" + if python3 scripts/prism_load_trend.py fetch --out "${dir}"; then + printf 'ok=true\ndir=%s\n' "${dir}" >> "${GITHUB_OUTPUT}" + else + echo "::warning::could not read ci-evidence; the regression rule has no history this run" + printf 'ok=false\n' >> "${GITHUB_OUTPUT}" + fi + + - name: Apply the regression rule (report-only) + id: regress + if: ${{ inputs.regression-config != '' }} + shell: bash + env: + ROWS: ${{ inputs.rows-dir }} + CONFIG: ${{ inputs.regression-config }} + VARIANCE: ${{ inputs.variance }} + HISTORY_OK: ${{ steps.history.outputs.ok }} + HISTORY: ${{ steps.history.outputs.dir }} + run: | + set -uo pipefail + verdict="${RUNNER_TEMP}/prism-load-verdict" + mkdir -p "${verdict}" + if [[ "${HISTORY_OK}" != true ]]; then + # Unknown, not "nothing regressed" (EP-OBSERVABILITY). + printf 'regressions=unknown\n' >> "${GITHUB_OUTPUT}" + printf '### Load regression rule (#551)\n\nNo verdict: ci-evidence could not be read.\n' >> "${GITHUB_STEP_SUMMARY}" + exit 0 + fi + args=(--history "${HISTORY}" --rows "${ROWS}" --config "${CONFIG}" + --json "${verdict}/verdict.json" --markdown "${verdict}/verdict.md" + --github-output "${GITHUB_OUTPUT}" + --repository "${GITHUB_REPOSITORY}" --server-url "${GITHUB_SERVER_URL}") + if [[ -z "${VARIANCE}" && -f test/prism-load-variance.json ]]; then + VARIANCE=test/prism-load-variance.json + fi + if [[ -n "${VARIANCE}" ]]; then + args+=(--variance "${VARIANCE}") + fi + if python3 scripts/prism_load_regress.py "${args[@]}" > /dev/null; then + cat "${verdict}/verdict.md" >> "${GITHUB_STEP_SUMMARY}" + printf 'verdict=%s\n' "${verdict}/verdict.json" >> "${GITHUB_OUTPUT}" + else + printf 'regressions=unknown\n' >> "${GITHUB_OUTPUT}" + printf '### Load regression rule (#551)\n\nNo verdict: the rule could not read its inputs (see the log).\n' >> "${GITHUB_STEP_SUMMARY}" + fi + + - name: Upload the verdict + if: ${{ steps.regress.outputs.verdict != '' }} + uses: actions/upload-artifact@v7 + with: + name: ${{ inputs.artifact-name }} + overwrite: true + retention-days: 90 + path: ${{ runner.temp }}/prism-load-verdict + if-no-files-found: error + + - name: Promote the selected bundles + id: promote + if: ${{ inputs.write == 'true' && inputs.promote != 'none' }} + shell: bash + env: + GH_TOKEN: ${{ inputs.github-token }} + ROWS: ${{ inputs.rows-dir }} + SELECT: ${{ inputs.promote }} + RELEASE: ${{ inputs.release }} + VERDICT: ${{ steps.regress.outputs.verdict }} + run: | + set -uo pipefail + args=(--rows "${ROWS}" --select "${SELECT}" --release "${RELEASE}" --repository "${GITHUB_REPOSITORY}") + if [[ -n "${VERDICT}" ]]; then + args+=(--verdict "${VERDICT}") + fi + # A failed promotion is written into its row as failed and the rows + # are still recorded; the last step fails the job. + if python3 scripts/prism_load_trend.py promote "${args[@]}"; then + printf 'ok=true\n' >> "${GITHUB_OUTPUT}" + else + printf 'ok=false\n' >> "${GITHUB_OUTPUT}" + fi + + - name: Append the rows to ci-evidence + id: append + if: ${{ inputs.write == 'true' }} + shell: bash + env: + ROWS: ${{ inputs.rows-dir }} + run: | + set -uo pipefail + if python3 scripts/prism_load_trend.py append --rows "${ROWS}"; then + printf 'ok=true\n' >> "${GITHUB_OUTPUT}" + else + printf 'ok=false\n' >> "${GITHUB_OUTPUT}" + fi + + - name: Open or update the regression tracking issue + if: >- + ${{ inputs.write == 'true' && steps.regress.outputs.regressions != '' + && steps.regress.outputs.regressions != '0' && steps.regress.outputs.regressions != 'unknown' }} + shell: bash + env: + GH_TOKEN: ${{ inputs.github-token }} + LABEL: ${{ inputs.issue-label }} + COUNT: ${{ steps.regress.outputs.regressions }} + RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} + run: | + set -euo pipefail + repo="${GITHUB_REPOSITORY}" + gh label create "${LABEL}" --repo "${repo}" --color D93F0B \ + --description "The load regression rule flagged a number (#551); report-only" 2>/dev/null || true + # An issue body holds 65,536 characters. + body="$(head -c 60000 "${RUNNER_TEMP}/prism-load-verdict/verdict.md") + + Kept by the prism-load-evidence action (#551) from ${RUN_URL}. Report-only: it does not page and does not block a merge. The body always shows the latest flagged run; each flagged run also comments. Close it once the cause is fixed or the change is accepted." + issue="$(gh issue list --repo "${repo}" --label "${LABEL}" --state open --limit 1 --json number --jq '.[0].number // empty')" + if [[ -n "${issue}" ]]; then + gh issue edit "${issue}" --repo "${repo}" --body "${body}" + gh issue comment "${issue}" --repo "${repo}" \ + --body "${RUN_URL} flagged ${COUNT} number(s); the body now shows that run." + else + gh issue create --repo "${repo}" --label "${LABEL}" \ + --title "PRISM load: the regression rule flagged a number (report-only)" --body "${body}" + fi + + - name: Fail if evidence was not recorded + if: ${{ always() }} + shell: bash + env: + WRITE: ${{ inputs.write }} + HISTORY: ${{ steps.history.outputs.ok }} + REGRESSIONS: ${{ steps.regress.outputs.regressions }} + PROMOTED: ${{ steps.promote.outputs.ok }} + APPENDED: ${{ steps.append.outputs.ok }} + run: | + set -euo pipefail + failed=() + [[ "${HISTORY}" == false ]] && failed+=("reading ci-evidence") + [[ "${REGRESSIONS}" == unknown ]] && failed+=("the regression rule") + [[ "${PROMOTED}" == false ]] && failed+=("a promotion (see the rows' promotion field)") + if [[ "${WRITE}" == true && "${APPENDED}" != true ]]; then + failed+=("appending the rows to ci-evidence") + fi + if (( ${#failed[@]} )); then + printf '::error::not done: %s\n' "${failed[@]}" + exit 1 + fi diff --git a/.github/actions/rust-toolchain/action.yml b/.github/actions/rust-toolchain/action.yml new file mode 100644 index 000000000..e85a67584 --- /dev/null +++ b/.github/actions/rust-toolchain/action.yml @@ -0,0 +1,22 @@ +name: Set up the pinned Rust toolchain +description: >- + Install the toolchain that rust-toolchain.toml pins and assert that the + active rustc and cargo report exactly that version. The file is the one + source of the pin; scripts/check_toolchain_pin.py holds Cargo.toml's + rust-version and the lab/prism/Dockerfile builder image to it. + +runs: + using: composite + steps: + - name: Install the toolchain named in rust-toolchain.toml + shell: bash + run: | + set -euo pipefail + # rustup 1.28 and later read rust-toolchain.toml when no toolchain is + # named, installing the pinned channel with its profile and components. + rustup toolchain install --no-self-update + rustup show active-toolchain + + - name: Assert the active toolchain is the pin + shell: bash + run: python3 scripts/check_toolchain_pin.py --installed diff --git a/.github/apt/ubuntu-mirrors.txt b/.github/apt/ubuntu-mirrors.txt new file mode 100644 index 000000000..3b7f6cfcc --- /dev/null +++ b/.github/apt/ubuntu-mirrors.txt @@ -0,0 +1,15 @@ +# Fallback Ubuntu mirrors for CI package installs (.github/scripts/apt-install.sh). +# +# The runner's own Ubuntu sources are tried first. If an install fails there, +# the helper points apt at this list instead, and apt tries these mirrors in +# priority order (without a priority, apt picks mirrors at random). Metadata +# must be separated from the URI by a tab. Azure's cloud mirror, the kernel.org mirror and Princeton's mirror +# run on infrastructure separate from Canonical's archive, which comes last. +# +# Every mirror here must carry the release, -updates, -backports and +# -security suites for amd64. apt checks everything against the Ubuntu +# archive signing key, so a mirror cannot change what gets installed. +http://azure.archive.ubuntu.com/ubuntu/ priority:1 +https://mirrors.edge.kernel.org/ubuntu/ priority:2 +https://mirror.math.princeton.edu/pub/ubuntu/ priority:3 +http://archive.ubuntu.com/ubuntu/ priority:4 diff --git a/.github/scripts/apt-install.sh b/.github/scripts/apt-install.sh new file mode 100644 index 000000000..3d24f5eea --- /dev/null +++ b/.github/scripts/apt-install.sh @@ -0,0 +1,206 @@ +#!/usr/bin/env bash +set -euo pipefail + +# Install Ubuntu packages on a CI runner and ride out Ubuntu mirror outages. +# +# - apt retries each download itself (Acquire::Retries). +# - If the whole update-and-install still fails, for example because the +# archive refused connections or a mirror was mid-sync ("Hash Sum mismatch"), +# this clears the downloaded package lists, waits, and tries again. After the +# first failure it also replaces the runner's Ubuntu sources with the pinned +# fallback mirrors in .github/apt/ubuntu-mirrors.txt, which apt tries in order. +# - Every attempt but the last treats a partially failed `apt-get update` as a +# failure, instead of silently installing from stale indexes. The last attempt +# keeps apt's default leniency, so this is never stricter than a plain +# update-and-install. +# - With APT_INSTALL_CACHE_DIR set (the apt-install composite action sets it and +# keeps it with actions/cache), the packages a successful install downloaded +# are kept there. They seed the next install, and if every online attempt +# fails they are installed directly with dpkg instead. Nothing the runner +# already has at the same or a newer version is reinstalled or downgraded. +# +# Usage: bash .github/scripts/apt-install.sh package [package...] +# +# Settings: +# APT_INSTALL_ATTEMPTS default 4 +# APT_INSTALL_PAUSE_SECONDS default 30, multiplied by the attempt number +# APT_INSTALL_DEADLINE_SECONDS default 480; after this, the next attempt is +# the last +# APT_INSTALL_FALLBACK_MIRRORS default .github/apt/ubuntu-mirrors.txt +# APT_INSTALL_CACHE_DIR unset: no package cache + +ATTEMPTS="${APT_INSTALL_ATTEMPTS:-4}" +PAUSE_SECONDS="${APT_INSTALL_PAUSE_SECONDS:-30}" +DEADLINE_SECONDS="${APT_INSTALL_DEADLINE_SECONDS:-480}" +SCRIPT_DIR="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)" +FALLBACK_MIRRORS="${APT_INSTALL_FALLBACK_MIRRORS:-${SCRIPT_DIR}/../apt/ubuntu-mirrors.txt}" +CACHE_DIR="${APT_INSTALL_CACHE_DIR:-}" +# apt downloads into a directory of its own, so the packages outlive the +# image's post-install clean-up and can be copied into the cache. +ARCHIVES=/var/cache/apt-install/archives + +if [[ $# -eq 0 ]]; then + printf 'apt-install: no packages given\n' >&2 + exit 2 +fi + +SUDO=() +if [[ "$(id -u)" -ne 0 ]]; then + SUDO=(sudo) +fi + +APT_OPTIONS=( + -o Acquire::Retries=5 + -o Acquire::http::Timeout=30 + -o Acquire::https::Timeout=30 + -o "Dir::Cache::archives=${ARCHIVES}/" + -o APT::Keep-Downloaded-Packages=true +) + +shopt -s nullglob + +# Print the installed version of a package, or nothing if it is not installed. +installed_version() { + local status="" version="" + # shellcheck disable=SC2016 # dpkg-query's own ${field} syntax, not a shell expansion + read -r status version < <(dpkg-query --show \ + --showformat='${db:Status-Status} ${Version}\n' "$1" 2>/dev/null) || true + if [[ "${status}" == installed ]]; then + printf '%s' "${version}" + fi +} + +fallback_active=0 + +# Replace the runner's Ubuntu sources with the pinned fallback mirror list. +use_fallback_mirrors() { + local arch codename source + if (( fallback_active == 1 )); then + return 0 + fi + fallback_active=1 + if [[ ! -r "${FALLBACK_MIRRORS}" ]]; then + printf '::warning::apt-install: no fallback mirror list at %s\n' "${FALLBACK_MIRRORS}" + return 0 + fi + arch="$(dpkg --print-architecture)" + if [[ "${arch}" != amd64 && "${arch}" != i386 ]]; then + printf '::warning::apt-install: the fallback mirrors carry amd64 and i386 only, not %s\n' "${arch}" + return 0 + fi + # shellcheck source=/dev/null + codename="$(. /etc/os-release && printf '%s' "${VERSION_CODENAME:-}")" + if [[ -z "${codename}" ]]; then + printf '::warning::apt-install: cannot tell the Ubuntu release, so the fallback mirrors are not used\n' + return 0 + fi + for source in /etc/apt/sources.list /etc/apt/sources.list.d/*.list /etc/apt/sources.list.d/*.sources; do + if [[ -f "${source}" ]] && + grep -q -E '(archive|security)\.ubuntu\.com|mirror\+file:[^[:space:]]*ubuntu' "${source}"; then + "${SUDO[@]}" mv -- "${source}" "${source}.apt-install-disabled" + fi + done + "${SUDO[@]}" install -m 0644 -- "${FALLBACK_MIRRORS}" /etc/apt/apt-install-ubuntu-mirrors.txt + printf 'Types: deb\nURIs: mirror+file:/etc/apt/apt-install-ubuntu-mirrors.txt\nSuites: %s %s-updates %s-backports %s-security\nComponents: main restricted universe multiverse\nSigned-By: /usr/share/keyrings/ubuntu-archive-keyring.gpg\n' \ + "${codename}" "${codename}" "${codename}" "${codename}" | + "${SUDO[@]}" tee /etc/apt/sources.list.d/apt-install-fallback.sources >/dev/null + # apt retries a failed file on the same mirror before moving to the next one, + # so its growing retry delay would keep an unreachable mirror blocking every + # download for minutes. Retry immediately instead. + APT_OPTIONS+=(-o Acquire::Retries::Delay=false) + printf '::warning::apt-install: switched the Ubuntu sources to the fallback mirrors in .github/apt/ubuntu-mirrors.txt\n' +} + +# Replace the cache with exactly the downloaded packages this install used. +save_cache() { + local fresh="${CACHE_DIR}.new" deb package kept=0 + rm -rf -- "${fresh}" + mkdir -p -- "${fresh}/archives" + for deb in "${ARCHIVES}"/*.deb; do + package="$(dpkg-deb --field "${deb}" Package)" + if [[ "$(installed_version "${package}")" == "$(dpkg-deb --field "${deb}" Version)" ]]; then + cp -- "${deb}" "${fresh}/archives/" + kept=$(( kept + 1 )) + fi + done + printf 'saved %s: %d packages for %s\n' "$(date -u +%Y-%m-%dT%H:%MZ)" "${kept}" "$*" \ + > "${fresh}/manifest.txt" + rm -rf -- "${CACHE_DIR}" + mv -- "${fresh}" "${CACHE_DIR}" +} + +# Install the cached packages with dpkg, skipping any the runner already has +# at the same or a newer version, then check every requested package is there. +install_from_cache() { + local debs=() deb package version current + for deb in "${CACHE_DIR}"/archives/*.deb; do + package="$(dpkg-deb --field "${deb}" Package)" + version="$(dpkg-deb --field "${deb}" Version)" + current="$(installed_version "${package}")" + if [[ -n "${current}" ]] && dpkg --compare-versions "${current}" ge "${version}"; then + continue + fi + debs+=("${deb}") + done + printf '::warning::apt-install: the Ubuntu mirrors are unreachable; installing %d cached packages (%s)\n' \ + "${#debs[@]}" "$(head -n 1 "${CACHE_DIR}/manifest.txt" 2>/dev/null || printf 'no manifest')" + if (( ${#debs[@]} > 0 )); then + "${SUDO[@]}" env DEBIAN_FRONTEND=noninteractive dpkg --install "${debs[@]}" || return 1 + fi + for package in "$@"; do + if [[ -z "$(installed_version "${package%%=*}")" ]]; then + return 1 + fi + done +} + +"${SUDO[@]}" mkdir -p "${ARCHIVES}/partial" +if [[ -n "${CACHE_DIR}" ]]; then + cached=("${CACHE_DIR}"/archives/*.deb) + if (( ${#cached[@]} > 0 )); then + # apt reuses a cached package whose checksum still matches the index. + "${SUDO[@]}" cp -- "${cached[@]}" "${ARCHIVES}/" + fi +fi + +attempt=1 +while true; do + last=0 + if (( attempt >= ATTEMPTS || SECONDS >= DEADLINE_SECONDS )); then + last=1 + fi + update_options=() + if (( last == 0 )); then + update_options=(--error-on=any) + fi + + if "${SUDO[@]}" apt-get "${APT_OPTIONS[@]}" "${update_options[@]}" update && + "${SUDO[@]}" env DEBIAN_FRONTEND=noninteractive \ + apt-get "${APT_OPTIONS[@]}" install -y --no-install-recommends "$@"; then + if [[ -n "${CACHE_DIR}" ]]; then + save_cache "$@" || + printf '::warning::apt-install: could not update the package cache in %s\n' "${CACHE_DIR}" + fi + exit 0 + fi + + if (( last == 1 )); then + break + fi + + pause=$(( PAUSE_SECONDS * attempt )) + printf '::warning::apt-install: attempt %d of %d failed; clearing package lists and retrying in %ds\n' \ + "${attempt}" "${ATTEMPTS}" "${pause}" + use_fallback_mirrors + "${SUDO[@]}" find /var/lib/apt/lists -mindepth 1 -maxdepth 1 ! -name lock -exec rm -rf {} + + sleep "${pause}" + attempt=$(( attempt + 1 )) +done + +if [[ -n "${CACHE_DIR}" && -d "${CACHE_DIR}/archives" ]] && install_from_cache "$@"; then + exit 0 +fi + +printf '::error::apt-install: installing %s failed after %d attempts%s\n' \ + "$*" "${attempt}" "${CACHE_DIR:+, and the package cache could not supply it}" +exit 1 diff --git a/.github/scripts/install-pinned-tool.sh b/.github/scripts/install-pinned-tool.sh new file mode 100755 index 000000000..785034e07 --- /dev/null +++ b/.github/scripts/install-pinned-tool.sh @@ -0,0 +1,197 @@ +#!/usr/bin/env bash +# Install one pinned CI tool from its GitHub release after checking its sha256. +# +# Usage: install-pinned-tool.sh +# install-pinned-tool.sh --cache-metadata +# +# The versions and digests below are the pins; bump them together. Each digest +# is the sha256 of the release asset as published (cargo-deny also publishes a +# .sha256 file next to its tarball, which matched when the pin was set). +# +# Downloads ride out GitHub release outages: curl retries transient failures +# (timeouts, 408, 429, 5xx) every 15 seconds for up to 180 seconds, and the +# whole download is killed (SIGKILL) at 210 seconds. Other failures, such as a +# 404, fail at once. +# +# With PINNED_TOOL_CACHE_DIR set, the release asset as published is kept under +# //// and installed from there on the next +# run without any download. A cached asset must be a regular file; it is copied +# to private scratch space and checked against the pinned sha256 before anything +# is extracted, installed or run. A bad entry fails without refetching, so +# remove it by hand. +# Unset means no cache. +# +# --cache-metadata writes the cache key and paths for actions/cache to +# GITHUB_OUTPUT (cache-dir, path and key), so the pins stay in this file. It +# needs PINNED_TOOL_CACHE_DIR, RUNNER_OS and RUNNER_ARCH. +set -euo pipefail + +usage='usage: install-pinned-tool.sh + install-pinned-tool.sh --cache-metadata ' + +metadata_only=0 +if [[ "${1:-}" == --cache-metadata ]]; then + metadata_only=1 + shift + if [[ $# -ne 1 ]]; then + printf '%s\n' "${usage}" >&2 + exit 2 + fi +fi + +tool="${1:?${usage}}" +if (( metadata_only == 0 )); then + destination="${2:?${usage}}" +fi + +case "${tool}" in + cargo-deny) + version=0.20.2 + asset="cargo-deny-${version}-x86_64-unknown-linux-musl" + url="https://github.com/EmbarkStudios/cargo-deny/releases/download/${version}/${asset}.tar.gz" + sha256=9f12ed4c49936e09b48bf862b595cde2fe64fcbd9d74dfacac6131ca824c8d5f + ;; + hadolint) + version=2.15.1 + url="https://github.com/hadolint/hadolint/releases/download/v${version}/hadolint-linux-x86_64" + sha256=c7187db94eeeeca956519a6af171adc31453941a1e777961f6e680f697c8c507 + ;; + *) + echo "install-pinned-tool: unknown tool ${tool}; expected cargo-deny or hadolint" >&2 + exit 2 + ;; +esac + +if [[ -n "${PINNED_TOOL_CACHE_DIR+set}" && -z "${PINNED_TOOL_CACHE_DIR}" ]]; then + echo "install-pinned-tool: PINNED_TOOL_CACHE_DIR is set but empty; unset it to disable the cache" >&2 + exit 2 +fi +cache_dir="${PINNED_TOOL_CACHE_DIR:-}" +cache_entry="" +cached_asset="" +if [[ -n "${cache_dir}" ]]; then + cache_entry="${cache_dir}/${tool}/${version}/${sha256}" + cached_asset="${cache_entry}/${url##*/}" +fi + +if (( metadata_only == 1 )); then + for name in PINNED_TOOL_CACHE_DIR RUNNER_OS RUNNER_ARCH GITHUB_OUTPUT; do + if [[ -z "${!name:-}" ]]; then + echo "install-pinned-tool: --cache-metadata needs ${name}" >&2 + exit 2 + fi + done + { + printf 'cache-dir=%s\n' "${cache_dir}" + printf 'path=%s\n' "${cache_entry}" + printf 'key=pinned-tool-v1-%s-%s-%s-%s-%s\n' \ + "${RUNNER_OS}" "${RUNNER_ARCH}" "${tool}" "${version}" "${sha256}" + } >> "${GITHUB_OUTPUT}" + exit 0 +fi + +scratch="$(mktemp -d)" +publishing="" +download_pid="" +cleanup() { + rm -rf "${scratch}" + if [[ -n "${publishing}" ]]; then + rm -f "${publishing}" + fi +} +trap cleanup EXIT +trap 'exit 130' INT +trap 'exit 143' TERM + +cancel_download() { + # timeout forwards TERM to its curl process and reaps it before exiting. + # Ignore further cancellation while waiting for that cleanup to finish. + trap '' INT TERM + kill -TERM "${download_pid}" 2>/dev/null || true + wait "${download_pid}" 2>/dev/null || true + exit "$1" +} + +verify() { + printf '%s %s\n' "${sha256}" "$1" | sha256sum --check --quiet +} + +if [[ -n "${cached_asset}" && ( -e "${cached_asset}" || -L "${cached_asset}" ) ]]; then + # Only a plain file is read: a link to a FIFO or device could block or fill + # the disk before the check. + if [[ -L "${cached_asset}" || ! -f "${cached_asset}" ]]; then + echo "install-pinned-tool: cached ${cached_asset} is not a regular file; not using or refetching it, remove it and retry" >&2 + exit 1 + fi + # Check the private copy, not the shared entry, so nothing can change what + # was checked before it is used. + cp -- "${cached_asset}" "${scratch}/download" + if ! verify "${scratch}/download"; then + echo "install-pinned-tool: cached ${cached_asset} does not match the pinned sha256; not using or refetching it, remove it and retry" >&2 + exit 1 + fi + printf 'install-pinned-tool: using cached %s\n' "${cached_asset}" +else + status=0 + # Record signals during launch so cancellation cannot orphan the child + # between starting it and recording its PID. + cancelled=0 + trap 'cancelled=130' INT + trap 'cancelled=143' TERM + timeout --signal=KILL 210 \ + curl --fail --location --silent --show-error \ + --retry 12 --retry-delay 15 --retry-max-time 180 \ + --connect-timeout 10 --max-time 30 \ + --output "${scratch}/download" "${url}" & + download_pid=$! + trap 'cancel_download 130' INT + trap 'cancel_download 143' TERM + if (( cancelled != 0 )); then + cancel_download "${cancelled}" + fi + # Unlike a foreground external command, Bash's wait services traps promptly. + wait "${download_pid}" || status=$? + trap 'exit 130' INT + trap 'exit 143' TERM + download_pid="" + if (( status == 137 )); then + echo "install-pinned-tool: download of ${url} did not finish within 210 seconds" >&2 + exit 1 + elif (( status != 0 )); then + echo "install-pinned-tool: download of ${url} failed (curl exit ${status}); transient errors were retried for up to 180 seconds" >&2 + exit 1 + fi + if ! verify "${scratch}/download"; then + echo "install-pinned-tool: ${url} does not match the pinned sha256" >&2 + exit 1 + fi + if [[ -n "${cache_entry}" ]]; then + # Publish under a temporary name in the same directory, check that copy, + # then rename it into place, so readers and concurrent writers only ever + # see a whole, checked asset. + mkdir -p "${cache_entry}" + publishing="$(mktemp "${cache_entry}/.publish.XXXXXX")" + cp -- "${scratch}/download" "${publishing}" + if ! verify "${publishing}"; then + echo "install-pinned-tool: copy of ${url} into ${cache_entry} does not match the pinned sha256" >&2 + exit 1 + fi + chmod 0644 "${publishing}" + mv -fT -- "${publishing}" "${cached_asset}" + publishing="" + printf 'install-pinned-tool: cached %s\n' "${cached_asset}" + fi +fi + +mkdir -p "${destination}" +case "${tool}" in + cargo-deny) + tar -xzf "${scratch}/download" -C "${scratch}" + install -m 0755 "${scratch}/${asset}/cargo-deny" "${destination}/cargo-deny" + ;; + hadolint) + install -m 0755 "${scratch}/download" "${destination}/hadolint" + ;; +esac +printf 'install-pinned-tool: %s %s -> %s\n' "${tool}" "${version}" "${destination}/${tool}" +"${destination}/${tool}" --version diff --git a/.github/scripts/install-prism-qbit.sh b/.github/scripts/install-prism-qbit.sh new file mode 100755 index 000000000..fa1e44f9d --- /dev/null +++ b/.github/scripts/install-prism-qbit.sh @@ -0,0 +1,112 @@ +#!/usr/bin/env bash +# Install the pinned qbit release the PRISM native integration tests run +# against, checking its sha256 before anything is extracted. +# +# Usage: install-prism-qbit.sh +# +# The tarball is taken from the first of these that yields the pinned digest: +# 1. PRISM_QBIT_CACHE_DIR, when set (a cached copy whose digest mismatches is +# deleted, not used); +# 2. the release's browser download route on github.com; +# 3. the release-asset API route on api.github.com, which fronts the same +# storage through a different tier, so a github.com gateway error alone +# does not fail the install (no token: the repository is public). +# Each route is fetched with a bounded retry budget. A copy that verified is +# left in the cache directory so the next run needs no network at all. The +# script fails only when every source is unusable. +# +# The version, asset id and digest below are the pin; bump them together. The +# digest is the sha256 GitHub publishes for the asset +# (gh api repos/Qbit-Org/qbit/releases/assets/ --jq .digest). +set -euo pipefail + +destination="${1:?usage: install-prism-qbit.sh }" + +version=1.0.0 +asset_id=478569140 +asset="qbit-${version}-x86_64-linux-gnu.tar.gz" +sha256=ae121af03263b55d530e3f3e8719a71362d0950cba82f54a2bc2d6c437a029b5 + +release_url="https://github.com/Qbit-Org/qbit/releases/download/v${version}/${asset}" +asset_api_url="https://api.github.com/repos/Qbit-Org/qbit/releases/assets/${asset_id}" +retries=5 +retry_max_time=90 +cache_dir="${PRISM_QBIT_CACHE_DIR:-}" + +scratch="$(mktemp -d)" +trap 'rm -rf "${scratch}"' EXIT + +log() { + printf 'install-prism-qbit: %s\n' "$*" >&2 +} + +# verify : succeed when the file's sha256 is the pin; log both on mismatch. +verify() { + local actual + actual="$(sha256sum "$1" | cut -d ' ' -f 1)" + if [[ "${actual}" != "${sha256}" ]]; then + log "digest mismatch for $1: expected ${sha256}, got ${actual}" + return 1 + fi +} + +# fetch [curl args...]: download with a bounded +# retry budget and verify; on any failure remove the output and fail. +fetch() { + local route="$1" url="$2" output="$3" + shift 3 + log "fetching ${asset} from the ${route} (${url})" + if ! curl --fail --location --silent --show-error \ + --connect-timeout 30 --speed-limit 1024 --speed-time 60 \ + --retry "${retries}" --retry-max-time "${retry_max_time}" --retry-all-errors \ + "$@" --output "${output}" "${url}"; then + log "the ${route} did not deliver ${asset}" + rm -f "${output}" + return 1 + fi + if ! verify "${output}"; then + rm -f "${output}" + return 1 + fi +} + +tarball="" +if [[ -n "${cache_dir}" ]] && [[ -f "${cache_dir}/${asset}" ]]; then + if verify "${cache_dir}/${asset}"; then + log "using the cached ${asset} from ${cache_dir}" + tarball="${cache_dir}/${asset}" + else + log "discarding the cached ${asset} from ${cache_dir}" + rm -f "${cache_dir}/${asset}" + fi +fi + +if [[ -z "${tarball}" ]]; then + if fetch 'release download route' "${release_url}" "${scratch}/${asset}"; then + tarball="${scratch}/${asset}" + elif fetch 'release-asset API route' "${asset_api_url}" "${scratch}/${asset}" \ + --header 'Accept: application/octet-stream'; then + tarball="${scratch}/${asset}" + else + log "no source delivered ${asset} with digest ${sha256}: the release download route and the release-asset API route both failed" + exit 1 + fi + if [[ -n "${cache_dir}" ]]; then + mkdir -p "${cache_dir}" + cp "${tarball}" "${cache_dir}/${asset}.partial" + mv "${cache_dir}/${asset}.partial" "${cache_dir}/${asset}" + tarball="${cache_dir}/${asset}" + fi +fi + +mkdir -p "${destination}" +tar -xzf "${tarball}" -C "${destination}" +qbitd="${destination}/qbit-${version}/bin/qbitd" +if [[ ! -x "${qbitd}" ]]; then + log "${asset} did not contain an executable qbit-${version}/bin/qbitd" + exit 1 +fi +if [[ -n "${GITHUB_ENV:-}" ]]; then + printf 'QBITD_BIN=%s\n' "${qbitd}" >> "${GITHUB_ENV}" +fi +log "qbit ${version} -> ${qbitd}" diff --git a/.github/scripts/prism-load-run.sh b/.github/scripts/prism-load-run.sh new file mode 100755 index 000000000..193303457 --- /dev/null +++ b/.github/scripts/prism-load-run.sh @@ -0,0 +1,131 @@ +#!/usr/bin/env bash +# Run one checked-in load-harness preset and gate it (#521). The nightly +# workflow runs exactly this, and so can anyone reproducing a nightly result. +# +# Usage: prism-load-run.sh [operational flags] +# +# Any further arguments go to the harness after the preset's, so only flags +# the preset does not set are accepted: `--allow-dirty-tree` to reproduce a +# run from a modified checkout, say. A preset flag given again is refused. +# +# Expects release builds of qbit-prism-server, qbit-prism-load and +# qbit-prism-load-gate in ${TARGET_DIR:-target}/release, and PostgreSQL 16 +# server binaries in PG_BIN_DIR (default /usr/lib/postgresql/16/bin). +# +# Writes into the output directory: +# host.json the host fingerprint from scripts/prism_load_probe.py +# host: the runner label (RUNNER_LABEL, when set), +# CPU, memory, kernel, and the filesystem and block +# device under the cluster (mount options, write +# cache, FUA) +# pg_test_fsync.txt the WAL volume's commit cost, on the filesystem +# the harness builds its cluster on +# harness-exit-code the harness's exit code +# gate.md the gate's verdict table (a soak preset's soak +# gates included) +# gate-exit-code the gate's exit code: 0 or 1 is a verdict, anything +# else (2, a panic, a binary that would not start) is not +# everything the harness writes (load-harness-report.json, logs/, and for +# a soak preset soak-samples.jsonl, soak-events.jsonl, soak-report.md) +# +# Optional overrides of the preset's gates: PRISM_LOAD_MAX_SHORTFALL and +# PRISM_LOAD_TIP_LAST_NOTIFY_P99_BUDGET_MS (empty keeps the preset's). The +# gate's table is also appended to GITHUB_STEP_SUMMARY when that is set. +# Exits with the gate's status: 0 pass, 1 fail, 2 unreadable inputs. +set -euo pipefail + +preset="${1:?usage: prism-load-run.sh [flags]}" +out="${2:?usage: prism-load-run.sh [flags]}" +shift 2 +root="$(git rev-parse --show-toplevel)" +target="${TARGET_DIR:-${root}/target}/release" +pg_bin="${PG_BIN_DIR:-/usr/lib/postgresql/16/bin}" +presets="${root}/crates/qbit-prism-load/presets" +# A deprecated name (presets/aliases.txt) runs the preset it was renamed to. +if [[ ! -f "${presets}/${preset}.json" && -f "${presets}/aliases.txt" ]]; then + renamed="$(awk -v old="${preset}" '$1 == old && $1 !~ /^#/ { print $2 }' "${presets}/aliases.txt")" + if [[ -n "${renamed}" ]]; then + echo "prism-load-run: ${preset} is deprecated; running ${renamed}" >&2 + preset="${renamed}" + fi +fi +preset_file="${presets}/${preset}.json" +test -f "${preset_file}" || { echo "no preset ${preset_file}" >&2; exit 2; } + +mkdir -p "${out}" +# The harness builds its cluster under TMPDIR, and PostgreSQL refuses a +# socket path over 107 bytes, so the directory is kept short. +export TMPDIR="${PRISM_LOAD_TMPDIR:-${RUNNER_TEMP:-/tmp}/pload}" +mkdir -p "${TMPDIR}" + +# #549: the fingerprint is evidence beside the result, so failing to take it +# is noted in the file and does not stop the run. +python3 "${root}/scripts/prism_load_probe.py" host --out "${out}/host.json" --dir "${TMPDIR}" \ + ${RUNNER_LABEL:+--runner "${RUNNER_LABEL}"} \ + || echo '{"error": "prism_load_probe.py host failed"}' > "${out}/host.json" + +{ + echo "# pg_test_fsync on the harness's cluster filesystem (${TMPDIR})" + df -h "${TMPDIR}" || true + "${pg_bin}/pg_test_fsync" -s 2 -f "${TMPDIR}/pg_test_fsync.out" +} > "${out}/pg_test_fsync.txt" 2>&1 || echo "pg_test_fsync failed; see above" >> "${out}/pg_test_fsync.txt" +rm -f "${TMPDIR}/pg_test_fsync.out" + +set +e +"${target}/qbit-prism-load" \ + --preset "${preset_file}" \ + --server-bin "${target}/qbit-prism-server" \ + --pg-bin-dir "${pg_bin}" \ + --out "${out}" \ + "$@" +code=$? +set -e +echo "${code}" > "${out}/harness-exit-code" + +gate=("${target}/qbit-prism-load-gate" + --report "${out}/load-harness-report.json" + --preset "${preset_file}" + --exit-code "${code}") +if [[ -n "${PRISM_LOAD_MAX_SHORTFALL:-}" ]]; then + gate+=(--max-shortfall "${PRISM_LOAD_MAX_SHORTFALL}") +fi +if [[ -n "${PRISM_LOAD_TIP_LAST_NOTIFY_P99_BUDGET_MS:-}" ]]; then + gate+=(--tip-last-notify-p99-budget-ms "${PRISM_LOAD_TIP_LAST_NOTIFY_P99_BUDGET_MS}") +fi +set +e +"${gate[@]}" > "${out}/gate.md" +status=$? +set -e +echo "${status}" > "${out}/gate-exit-code" +cat "${out}/gate.md" +if [[ -n "${GITHUB_STEP_SUMMARY:-}" ]]; then + { + cat "${out}/gate.md" + echo + if [[ -f "${out}/soak-report.md" ]]; then + echo "
soak samples" + echo + sed -n '/^#### Samples/,$p' "${out}/soak-report.md" + echo + echo "
" + echo + fi + echo "
Host" + echo + echo '```json' + cat "${out}/host.json" + echo '```' + echo + echo "
" + echo + echo "
pg_test_fsync" + echo + echo '```' + cat "${out}/pg_test_fsync.txt" + echo '```' + echo + echo "
" + echo + } >> "${GITHUB_STEP_SUMMARY}" +fi +exit "${status}" diff --git a/.github/scripts/prism-stratum-fuzz.sh b/.github/scripts/prism-stratum-fuzz.sh new file mode 100755 index 000000000..7f6accf86 --- /dev/null +++ b/.github/scripts/prism-stratum-fuzz.sh @@ -0,0 +1,134 @@ +#!/usr/bin/env bash +# Run the Stratum cargo-fuzz targets for a fixed time and gate on no crash +# (#575). The nightly workflow runs exactly this, and so can anyone +# reproducing a nightly result. +# +# Usage: prism-stratum-fuzz.sh [seconds] +# +# Installs the pinned nightly toolchain and the pinned cargo-fuzz release +# (checked against its sha256), builds the targets in +# crates/qbit-prism-server/fuzz, and runs all three at once for the given +# seconds each (default 1200): stratum_session, stratum_lines and +# codec_parsers. Each starts from the checked-in seeds and +# /, which it grows; afterwards each target's corpus +# is merged down to the inputs that add coverage, so a cached corpus stays +# small from night to night. +# +# Writes into the output directory: +# .log libFuzzer's log, final stats included +# artifacts// crash-*, leak-*, timeout-* and oom-* inputs +# summary.md one row per target, also appended to +# GITHUB_STEP_SUMMARY when that is set +# Exits 1 when any target found a crash, leak, timeout or out-of-memory +# input or exited abnormally, after every target has run; 0 otherwise. +# Reproduce an artifact with: +# cargo + fuzz run --fuzz-dir crates/qbit-prism-server/fuzz +# and PRISM_FUZZ_TRACE=1 to print the conversation. +set -euo pipefail + +corpus="${1:?usage: prism-stratum-fuzz.sh [seconds]}" +out="${2:?usage: prism-stratum-fuzz.sh [seconds]}" +seconds="${3:-1200}" +[[ "${seconds}" =~ ^[1-9][0-9]*$ ]] || { echo "prism-stratum-fuzz: seconds must be a positive integer" >&2; exit 2; } + +# The toolchain the targets were last run and measured on. cargo-fuzz needs +# nightly for its sanitizer flags; the workspace itself stays on the stable +# pin in rust-toolchain.toml. +NIGHTLY=nightly-2026-09-24 +CARGO_FUZZ_VERSION=0.13.2 +CARGO_FUZZ_SHA256=b5b704018b63e0f151c17a057ac53b5111e1db545d1b9f72fee79f08a545931c +TARGETS=(stratum_session stratum_lines codec_parsers) +declare -A MAX_LEN=([stratum_session]=16384 [stratum_lines]=20000 [codec_parsers]=8192) + +root="$(git rev-parse --show-toplevel)" +fuzz_dir="${root}/crates/qbit-prism-server/fuzz" +mkdir -p "${corpus}" "${out}/artifacts" +corpus="$(cd -- "${corpus}" && pwd)" +out="$(cd -- "${out}" && pwd)" +tools="$(mktemp -d)" +trap 'rm -rf "${tools}"' EXIT + +rustup toolchain install "${NIGHTLY}" --profile minimal --no-self-update + +asset="cargo-fuzz-${CARGO_FUZZ_VERSION}-x86_64-unknown-linux-musl.tar.gz" +curl --fail --location --silent --show-error --retry 12 --retry-delay 15 \ + --retry-max-time 180 --connect-timeout 10 --max-time 60 \ + --output "${tools}/${asset}" \ + "https://github.com/rust-fuzz/cargo-fuzz/releases/download/${CARGO_FUZZ_VERSION}/${asset}" +printf '%s %s\n' "${CARGO_FUZZ_SHA256}" "${tools}/${asset}" | sha256sum --check --quiet +tar -xzf "${tools}/${asset}" -C "${tools}" cargo-fuzz +export PATH="${tools}:${PATH}" +cargo fuzz --version + +cd "${fuzz_dir}" +# The release binary is a musl build, whose default target would be musl too. +cargo "+${NIGHTLY}" fuzz build -O --target x86_64-unknown-linux-gnu +bin_dir="${CARGO_TARGET_DIR:-${fuzz_dir}/target}/x86_64-unknown-linux-gnu/release" + +pids=() +for target in "${TARGETS[@]}"; do + mkdir -p "${corpus}/${target}" "${out}/artifacts/${target}" + "${bin_dir}/${target}" "${corpus}/${target}" "${fuzz_dir}/seeds/${target}" \ + -dict="${fuzz_dir}/stratum.dict" -max_len="${MAX_LEN[${target}]}" \ + -max_total_time="${seconds}" -rss_limit_mb=2048 -malloc_limit_mb=512 \ + -timeout=30 -print_final_stats=1 \ + -artifact_prefix="${out}/artifacts/${target}/" \ + > "${out}/${target}.log" 2>&1 & + pids+=("$!") +done + +status=0 +declare -A EXIT MERGE +for i in "${!TARGETS[@]}"; do + code=0 + wait "${pids[${i}]}" || code=$? + EXIT[${TARGETS[${i}]}]="${code}" + (( code == 0 )) || status=1 +done + +{ + echo "### Stratum fuzzing, ${seconds} s per target on ${NIGHTLY}" + echo + echo "| target | exit | runs | exec/s | coverage (edges) | features | corpus | findings |" + echo "| --- | --- | --- | --- | --- | --- | --- | --- |" +} > "${out}/summary.md" +for target in "${TARGETS[@]}"; do + log="${out}/${target}.log" + final_stat() { sed -n "s/^stat::$1: *//p" "${log}" | tail -n 1; } + last="$(grep -E '^#[0-9]+' "${log}" | tail -n 1 || true)" + cov="$(sed -nE 's/.* cov: ([0-9]+).*/\1/p' <<< "${last}")" + ft="$(sed -nE 's/.* ft: ([0-9]+).*/\1/p' <<< "${last}")" + findings="$(find "${out}/artifacts/${target}" -type f | wc -l)" + # Keep only inputs that add coverage, so the cache does not grow nightly. + # A merge that fails keeps the whole grown corpus instead of a partial one. + merged="$(mktemp -d)" + MERGE[${target}]=0 + "${bin_dir}/${target}" -merge=1 -max_len="${MAX_LEN[${target}]}" -rss_limit_mb=2048 \ + -malloc_limit_mb=512 -timeout=30 -artifact_prefix="${out}/artifacts/${target}/merge-" \ + "${merged}" "${corpus}/${target}" > "${out}/${target}.merge.log" 2>&1 || MERGE[${target}]=$? + if [[ "${MERGE[${target}]}" == 0 ]]; then + rm -rf "${corpus:?}/${target}" + mv "${merged}" "${corpus}/${target}" + else + status=1 + rm -rf "${merged}" + fi + size="$(find "${corpus}/${target}" -type f | wc -l)" + echo "| ${target} | ${EXIT[${target}]} | $(final_stat number_of_executed_units) | $(final_stat average_exec_per_sec) | ${cov:-?} | ${ft:-?} | ${size} | ${findings} |" \ + >> "${out}/summary.md" +done +if [[ -n "${GITHUB_STEP_SUMMARY:-}" ]]; then + cat "${out}/summary.md" >> "${GITHUB_STEP_SUMMARY}" +fi +cat "${out}/summary.md" +for target in "${TARGETS[@]}"; do + if [[ "${EXIT[${target}]}" != 0 ]]; then + echo "::error::${target} failed; its input is under artifacts/${target} and its log is ${target}.log" + grep -m 5 -E 'invariant violated|panicked at|SUMMARY|ERROR' "${out}/${target}.log" || true + fi + if [[ "${MERGE[${target}]}" != 0 ]]; then + echo "::error::${target}'s corpus merge failed (exit ${MERGE[${target}]}); its corpus was kept unmerged, see ${target}.merge.log and artifacts/${target}/merge-*" + grep -m 5 -E 'invariant violated|panicked at|SUMMARY|ERROR' "${out}/${target}.merge.log" || true + fi +done +exit "${status}" diff --git a/.github/scripts/test-install-pinned-tool.sh b/.github/scripts/test-install-pinned-tool.sh new file mode 100644 index 000000000..10d09df0e --- /dev/null +++ b/.github/scripts/test-install-pinned-tool.sh @@ -0,0 +1,834 @@ +#!/usr/bin/env bash +# Prove that install-pinned-tool.sh rides out a release-host outage, installs +# from its verified cache with the network gone, and refuses anything that does +# not match the pinned sha256 on the download path and the cache path alike, +# without reaching the network, Docker or GitHub. +# +# A Python http.server on 127.0.0.1 stands in for the release host. Each +# scenario sets its behaviour (serve, 504 until a deadline, 404, stall, refuse) +# through a control file, and the server logs every connection so the test can +# assert request counts, not only exit statuses. +# +# The installer hardcodes its pins and its 180/210-second timing, so a hermetic +# test can neither serve bytes with the published digests nor wait out the real +# windows. Two narrow bridges close that gap and leave every guard as the +# committed code: +# * A routing-only curl shim on PATH rewrites the pinned https://github.com/ +# URL to the loopback server and execs the real curl with every other +# argument untouched, so the installer's own retry, deadline and error +# handling run as written. +# * Scripts under test are derived from the committed installer by replacing +# exactly four whole lines: the two sha256= pins (with the digests of the +# fixture assets, computed here with sha256sum) and the two timing lines +# (timeout --signal=KILL and --retry ...). Each line must match exactly +# once, and a diff against the committed file must show only those lines, +# so an edit to any guard flows straight into the test and an edit to a +# substituted line fails loudly. sha256sum is never shimmed: verify() runs +# as committed, only the expected value differs. Two variants are derived: +# a 15-second deadline for the outage scenarios and a 2-second deadline so +# the stalled-transfer scenario finishes fast. +# The pure-failure scenarios (404, substituted bytes, poisoned cache, non-regular +# cache) also run against the unmodified committed installer, with its real pins +# and timing, through the shim alone. +# +# Bad assets and poisoned cache entries carry an execution marker: running them +# touches a sentinel file, which must never appear. One warm-cache run uses a +# cache directory whose path contains a newline: sha256sum --check cannot name +# such a file, so it passes only if the private copy, not the shared entry, is +# what is checked. +set -euo pipefail + +SCRIPT_DIR="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)" +INSTALLER="${SCRIPT_DIR}/install-pinned-tool.sh" +TOOLS=(cargo-deny hadolint) + +# The pins as they must stay; a quiet bump trips this test. +EXPECTED_CARGO_DENY_VERSION=0.20.2 +EXPECTED_CARGO_DENY_SHA256=9f12ed4c49936e09b48bf862b595cde2fe64fcbd9d74dfacac6131ca824c8d5f +EXPECTED_HADOLINT_VERSION=2.15.1 +EXPECTED_HADOLINT_SHA256=c7187db94eeeeca956519a6af171adc31453941a1e777961f6e680f697c8c507 + +# Derived-variant timing. Retries run every second; the outage lasts longer +# than the seven seconds the pre-#380 installer retried for. +OUTAGE_SECONDS=8 +OUTAGE_DEADLINE=15 +STALL_DEADLINE=2 +RETRY_MAX_TIME=12 +# Bound on one installer run, so a regression that hangs (a cp blocking on a +# FIFO, a 404 that retries) fails this test instead of the job. +RUN_LIMIT=20 + +if [[ -z "${EPOCHREALTIME:-}" ]]; then + echo 'pinned-tool test: FAIL: bash 5 or newer is required (EPOCHREALTIME)' >&2 + exit 1 +fi +START_US="${EPOCHREALTIME/./}" +scenarios=0 +assertions=0 +SCENARIO="" +WORK="" +SERVER_PID="" +RUN_OUTPUT="" +REQUEST_LOG="" + +fail() { + printf 'pinned-tool test: FAIL: %s\n' "$*" >&2 + if [[ -n "${SCENARIO}" ]]; then + printf 'pinned-tool test: while running %s\n' "${SCENARIO}" >&2 + fi + if [[ -n "${RUN_OUTPUT}" && -s "${RUN_OUTPUT}" ]]; then + printf -- '--- installer output ---\n' >&2 + cat "${RUN_OUTPUT}" >&2 + fi + if [[ -n "${REQUEST_LOG}" && -s "${REQUEST_LOG}" ]]; then + printf -- '--- release-host requests ---\n' >&2 + cat "${REQUEST_LOG}" >&2 + fi + exit 1 +} + +cleanup() { + if [[ -n "${SERVER_PID}" ]]; then + kill -TERM "${SERVER_PID}" 2>/dev/null || true + wait "${SERVER_PID}" 2>/dev/null || true + fi + if [[ -n "${WORK}" ]]; then + rm -rf -- "${WORK}" + fi +} +trap cleanup EXIT + +assert_eq() { # + assertions=$((assertions + 1)) + [[ "$1" == "$2" ]] || fail "$3: expected '$2', got '$1'" +} + +assert_ne() { # + assertions=$((assertions + 1)) + [[ "$1" != "$2" ]] || fail "$3: got '$1'" +} + +assert_ge() { # + assertions=$((assertions + 1)) + (( $1 >= $2 )) || fail "$3: expected at least $2, got $1" +} + +assert_lt() { # + assertions=$((assertions + 1)) + (( $1 < $2 )) || fail "$3: expected under $2, got $1" +} + +assert_prefix() { # + assertions=$((assertions + 1)) + [[ "$1" == "$2"* ]] || fail "$3: '$1' does not start with '$2'" +} + +assert_exists() { # + assertions=$((assertions + 1)) + [[ -e "$1" || -L "$1" ]] || fail "$2: $1 is missing" +} + +assert_absent() { # + assertions=$((assertions + 1)) + [[ ! -e "$1" && ! -L "$1" ]] || fail "$2: $1 exists" +} + +assert_output_contains() { # + assertions=$((assertions + 1)) + grep -qF -- "$1" "${RUN_OUTPUT}" || fail "installer output lacks '$1'" +} + +assert_output_lacks() { # + assertions=$((assertions + 1)) + ! grep -qF -- "$1" "${RUN_OUTPUT}" || fail "installer output contains '$1': $2" +} + +assert_same_bytes() { # + assertions=$((assertions + 1)) + cmp -s -- "$1" "$2" || fail "$3: $1 differs from $2" +} + +scenario() { #
`. +pub const MARKER_PREFIX: &str = "prism-execution-marker"; +/// AFTER ROW notices installed by the compact-runtime measurement fixture. +pub const JSONB_WRITE_PREFIX: &str = "prism-jsonb-write "; + +/// Observed row mutations, including unchanged JSONB values carried by UPDATE. +/// These are server executions, not proof of a subsequent transaction commit. +#[derive(Clone, Debug, Default)] +pub struct JsonbWrites { + pub rows: u64, + pub values: u64, + pub max_uncompressed_bytes: u64, +} + +#[derive(serde::Deserialize)] +#[serde(deny_unknown_fields)] +struct JsonbWriteNotice { + table: String, + operation: String, + values: BTreeMap, +} + +const SSL_REQUEST: u32 = 80_877_103; +const GSSENC_REQUEST: u32 = 80_877_104; +const CANCEL_REQUEST: u32 = 80_877_102; + +/// Which client frame asked the server to run the statement. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum Protocol { + /// `Execute` of a bound portal. + Extended, + /// `Query` text, possibly several statements. + Simple, +} + +/// What the server answered, as seen on the wire. +#[derive(Clone, Debug, PartialEq, Eq)] +pub enum Outcome { + /// No answer observed yet, or the connection closed before one. + Pending, + /// `CommandComplete` (or `EmptyQueryResponse`); `delivered` becomes true + /// only after the transport accepts the entire frame. This does not prove + /// that the peer application read the bytes. + Completed { tag: String, delivered: bool }, + /// Server-observed `ErrorResponse` with its SQLSTATE and message, whether + /// or not the client received it. This is never a complete success response. + Failed { code: String, message: String }, +} + +/// A marker NOTICE attached to an execution by a fixture trigger. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct Marker { + pub table: String, + pub op: String, +} + +/// One `Execute` or `Query` frame and the server's answer to it. +#[derive(Clone, Debug)] +pub struct Execution { + /// Global order across every connection the proxy served. + pub seq: u64, + pub connection: u64, + pub protocol: Protocol, + /// Statement text learned from `Parse` (through the portal) or `Query`. + pub sql: String, + pub markers: Vec, + /// The latest answer; for a simple-query batch, its last completion. + pub outcome: Outcome, + /// Every completion tag the server sent for this execution, in order: a + /// simple-query batch such as `BEGIN; SET LOCAL …` has one per statement. + pub tags: Vec, + /// Actual DataRow frames, counted without retaining their payloads. + pub rows_received: u64, + pub jsonb_writes: BTreeMap<(String, String), JsonbWrites>, + select_rows: Option, + ready: bool, +} + +impl Execution { + pub fn marked(&self, table: &str, op: &str) -> bool { + self.markers + .iter() + .any(|marker| marker.table == table && marker.op == op) + } + + /// Whether this execution opens a transaction: its first statement's + /// completion is `BEGIN`, or, before the server has answered, its text + /// starts with `BEGIN`. A batch such as `BEGIN; SET LOCAL …` counts; its + /// last completion (`completion`) is the `SET`. + pub fn begins_transaction(&self) -> bool { + match self.tags.first() { + Some(tag) => tag == "BEGIN", + None => self + .sql + .trim_start() + .get(..5) + .is_some_and(|head| head.eq_ignore_ascii_case("BEGIN")), + } + } + + /// The completion tag, whether or not it reached the client; for a + /// simple-query batch, its last statement's. + pub fn completion(&self) -> Option<&str> { + match &self.outcome { + Outcome::Completed { tag, .. } => Some(tag), + _ => None, + } + } + + pub fn delivered(&self) -> bool { + matches!( + self.outcome, + Outcome::Completed { + delivered: true, + .. + } + ) + } + + /// The server-observed SQLSTATE, not proof the client received that error. + pub fn sqlstate(&self) -> Option<&str> { + match &self.outcome { + Outcome::Failed { code, .. } => Some(code), + _ => None, + } + } + + pub fn is_commit(&self) -> bool { + self.completion() == Some("COMMIT") + } + + /// Simple-query batches can produce several completions; only their final + /// ReadyForQuery proves the remainder of the batch is no longer pending. + pub fn complete_response(&self) -> bool { + self.delivered() && (self.protocol == Protocol::Extended || self.ready) + } + + /// A successfully completed SELECT's actual rows, including measured zero. + /// Partial, failed, lost-response and mixed-result observations are errors. + pub fn returned_rows(&self) -> Result { + ensure!(self.complete_response(), "SELECT response is incomplete"); + let reported = self + .select_rows + .context("execution has no SELECT completion")?; + ensure!( + reported == self.rows_received, + "SELECT completion differs from observed DataRow count" + ); + Ok(self.rows_received) + } +} + +/// One `ErrorResponse` the server sent, kept apart from executions: a +/// statement can be rejected without any frame having asked the server to +/// run it. A statement the client prepares first is rejected at `Parse`, for +/// example, when it is sent into an aborted transaction, and no `Execute` +/// frame for it ever exists. +/// This records the server's rejection, not delivery of that error to the client. +#[derive(Clone, Debug)] +pub struct Rejection { + /// Same order as [`Execution::seq`], so [`ExecutionProxy::mark`] covers + /// both. + pub seq: u64, + pub connection: u64, + pub code: String, + pub message: String, + pub frame: RejectedFrame, +} + +/// What the rejected client frame was. +#[derive(Clone, Debug, PartialEq, Eq)] +pub enum RejectedFrame { + /// A `Parse` of this statement text; the server never ran it. + Parse(String), + /// The recorded execution with this sequence number, which failed. + Execution(u64), + /// Any other frame, such as a `Bind`. + Other, +} + +/// When a planned fault closes the connection. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum FaultPhase { + /// After the server completed the marked statement, before the client + /// hears about it; the transaction is still open and will abort. + AfterExecution, + /// After the server completed the `COMMIT` that follows the marked + /// statement on the same connection, before the client hears about it. + AfterCommit, +} + +/// Withhold one acknowledgement related to the next execution that carries +/// the marker for `table`/`op`. +#[derive(Clone, Debug)] +pub struct Fault { + pub table: String, + pub op: String, + pub phase: FaultPhase, +} + +/// Holds a completed COMMIT reply after PostgreSQL has released transaction +/// locks. Dropping the handle releases delivery, including on test cancellation. +pub struct CommitPause(Arc); + +#[derive(Default)] +struct CommitPauseState { + entered: Notify, + release: Notify, + seq: AtomicU64, + released: AtomicBool, +} + +impl CommitPause { + pub async fn entered(&self) -> u64 { + loop { + let entered = self.0.entered.notified(); + tokio::pin!(entered); + entered.as_mut().enable(); + let seq = self.0.seq.load(Ordering::SeqCst); + if seq != 0 { + return seq; + } + entered.await; + } + } + pub fn release(&self) { + self.0.released.store(true, Ordering::SeqCst); + self.0.release.notify_waiters(); + } +} + +impl Drop for CommitPause { + fn drop(&mut self) { + self.release(); + } +} + +struct PausePlan { + table: String, + op: String, + control: Arc, +} + +/// Holds the reply of the next execution whose statement text is exactly +/// `sql`, on whichever connection runs it, from the frame `at` names on. +struct StatementPausePlan { + sql: String, + at: PauseAt, + control: Arc, +} + +/// Where in a statement's reply a [`StatementPausePlan`] holds delivery. +#[derive(Clone, Copy, PartialEq, Eq)] +enum PauseAt { + /// The first `CommandComplete`: the client has read none of the reply. + Completion, + /// The `ReadyForQuery` that closes a simple query: every completion of + /// the batch has been delivered, only the end of the round trip is held. + Ready, +} + +#[derive(Default)] +struct State { + executions: Vec, + rejections: Vec, + plan: Option, + /// Connection whose next `COMMIT` completion is withheld. + armed: Option, + /// Sequence number of the execution whose acknowledgement was withheld. + fired: Option, + pause_plan: Option, + pause_armed: Option, + statement_pause: Option, + /// Connections whose client sent `Terminate`, a graceful close. + terminated: HashSet, + /// Connections whose relay has ended, both sockets closed. + ended: HashSet, +} + +struct Shared { + seq: AtomicU64, + connections: AtomicU64, + state: Mutex, +} + +#[derive(Default)] +struct Connection { + statements: HashMap, String>, + portals: HashMap, String>, + /// Statement text of every `Parse` the server has not yet answered with + /// `ParseComplete`, oldest first. + parses: VecDeque, + /// Indices into `State::executions`, oldest first. + inflight: VecDeque, +} + +/// Statement text of an `Execute` whose `Bind` or `Parse` the proxy did not +/// see. A client whose whole session passes through the proxy never produces +/// one, so [`ExecutionProxy::executions_since`] refuses to report it. +const UNPARSED: &str = ""; +const UNBOUND: &str = ""; + +pub struct ExecutionProxy { + addr: SocketAddr, + shared: Arc, + /// Taken by [`Self::finish`], so a shared proxy is finished once. + accept: Mutex>>>, + tasks: Arc>>>, + failure: Mutex>, +} + +impl ExecutionProxy { + /// Listen on a loopback port and forward every client to `upstream`. + pub async fn start(upstream: SocketAddr) -> Result { + let listener = TcpListener::bind("127.0.0.1:0").await?; + let addr = listener.local_addr()?; + let shared = Arc::new(Shared { + seq: AtomicU64::new(0), + connections: AtomicU64::new(0), + state: Mutex::default(), + }); + let tasks: Arc>>> = Arc::default(); + let accept = { + let shared = shared.clone(); + let tasks = tasks.clone(); + tokio::spawn(async move { + loop { + let (client, _) = listener.accept().await.context("proxy accept")?; + let id = shared.connections.fetch_add(1, Ordering::SeqCst) + 1; + tasks.lock().expect("proxy task registry").spawn(serve( + client, + upstream, + id, + shared.clone(), + )); + } + }) + }; + Ok(Self { + addr, + shared, + accept: Mutex::new(Some(accept)), + tasks, + failure: Mutex::default(), + }) + } + + /// `database_url` with its host and port replaced by this proxy. + pub fn rewrite_url(&self, database_url: &str) -> Result { + let mut url = url::Url::parse(database_url)?; + url.set_host(Some(&self.addr.ip().to_string()))?; + url.set_port(Some(self.addr.port())) + .map_err(|()| anyhow::anyhow!("database URL cannot carry a port"))?; + Ok(url.to_string()) + } + + /// Client sockets accepted so far, including ones closed since. + pub fn connections(&self) -> u64 { + self.shared.connections.load(Ordering::SeqCst) + } + + /// A point in the execution order; see [`Self::executions_since`]. + pub fn mark(&self) -> u64 { + self.shared.seq.load(Ordering::SeqCst) + } + + /// Every execution recorded after `mark`, on every connection, oldest + /// first. Fails if a proxy task has failed, so a broken observer cannot + /// pass as an empty observation, and fails if an execution's statement + /// is unknown, so it cannot escape identification by text. + pub fn executions_since(&self, mark: u64) -> Result> { + self.check()?; + let state = self.shared.state.lock().expect("proxy state"); + let executions: Vec = state + .executions + .iter() + .filter(|execution| execution.seq > mark) + .cloned() + .collect(); + if let Some(unknown) = executions + .iter() + .find(|execution| [UNPARSED, UNBOUND].contains(&execution.sql.as_str())) + { + bail!("the proxy could not identify an execution's statement: {unknown:?}"); + } + Ok(executions) + } + + /// Every `ErrorResponse` recorded after `mark`, on every connection, + /// oldest first, including rejections no recorded execution asked for. + pub fn rejections_since(&self, mark: u64) -> Result> { + self.check()?; + let state = self.shared.state.lock().expect("proxy state"); + Ok(state + .rejections + .iter() + .filter(|rejection| rejection.seq > mark) + .cloned() + .collect()) + } + + /// Arm one fault; it disarms itself when it fires. + pub fn plan(&self, fault: Fault) { + let mut state = self.shared.state.lock().expect("proxy state"); + state.plan = Some(fault); + state.armed = None; + state.fired = None; + } + + /// Pause only the completed COMMIT following the next marked mutation. + /// The observer reports completed-but-undelivered until forwarding succeeds. + pub fn pause_after_commit(&self, table: &str, op: &str) -> Result { + let mut state = self.shared.state.lock().expect("proxy state"); + ensure!( + state.plan.is_none() && state.pause_plan.is_none(), + "proxy already has a delivery plan" + ); + let control = Arc::new(CommitPauseState::default()); + state.pause_plan = Some(PausePlan { + table: table.into(), + op: op.into(), + control: control.clone(), + }); + state.pause_armed = None; + Ok(CommitPause(control)) + } + + /// Hold the reply of the next execution whose statement text is exactly + /// `sql` (a simple-protocol `Query`, such as `BEGIN`), from its + /// `CommandComplete` on, so the client stays blocked waiting for that + /// statement's `ReadyForQuery` until the handle releases it or drops. + /// Nothing is severed: the server has already run the statement, and the + /// connection stays usable once the reply is delivered. + pub fn pause_statement(&self, sql: &str) -> Result { + self.plan_statement_pause(sql, PauseAt::Completion) + } + + /// Like [`Self::pause_statement`], but delivers every `CommandComplete` + /// of the simple query `sql` and holds only its closing `ReadyForQuery`, + /// so a multi-statement query such as `BEGIN; SET LOCAL …` can be held + /// after the transport has accepted all of its completions but before + /// its round trip ends. + pub fn pause_statement_ready(&self, sql: &str) -> Result { + self.plan_statement_pause(sql, PauseAt::Ready) + } + + fn plan_statement_pause(&self, sql: &str, at: PauseAt) -> Result { + let mut state = self.shared.state.lock().expect("proxy state"); + ensure!( + state.plan.is_none() && state.pause_plan.is_none() && state.statement_pause.is_none(), + "proxy already has a delivery plan" + ); + let control = Arc::new(CommitPauseState::default()); + state.statement_pause = Some(StatementPausePlan { + sql: sql.into(), + at, + control: control.clone(), + }); + Ok(CommitPause(control)) + } + + /// Whether the client of connection `id` (an [`Execution::connection`]) + /// sent `Terminate`, closing it gracefully, rather than only dropping + /// the socket. Final once [`Self::ended`] says so. + pub fn terminated(&self, id: u64) -> bool { + self.shared + .state + .lock() + .expect("proxy state") + .terminated + .contains(&id) + } + + /// Whether connection `id`'s relay has ended, both its sockets closed. + pub fn ended(&self, id: u64) -> bool { + self.shared + .state + .lock() + .expect("proxy state") + .ended + .contains(&id) + } + + /// Sequence number of the execution whose acknowledgement the last + /// planned fault withheld, once it has fired. + pub fn fired(&self) -> Option { + self.shared.state.lock().expect("proxy state").fired + } + + /// Surface any connection task that ended with an error or a panic. + pub fn check(&self) -> Result<()> { + let mut failure = self.failure.lock().expect("proxy failure"); + if let Some(error) = failure.as_ref() { + bail!("proxy observation previously failed: {error}"); + } + let mut tasks = self.tasks.lock().expect("proxy task registry"); + while let Some(joined) = tasks.try_join_next() { + if let Err(error) = settle(joined) { + *failure = Some(format!("{error:#}")); + return Err(error); + } + } + Ok(()) + } + + /// Stop accepting, close every connection and join every task. Every + /// task is joined even when one of them failed; the first failure is + /// returned. A second call finds nothing left to join. + pub async fn finish(&self) -> Result<()> { + let accept = self.accept.lock().expect("proxy accept handle").take(); + let mut outcome = Ok(()); + if let Some(accept) = accept { + accept.abort(); + match accept.await { + Ok(result) => outcome = result, + Err(error) if error.is_cancelled() => {} + Err(error) => outcome = Err(anyhow::anyhow!("proxy accept task failed: {error}")), + } + } + let mut tasks = std::mem::take(&mut *self.tasks.lock().expect("proxy task registry")); + tasks.abort_all(); + while let Some(joined) = tasks.join_next().await { + let settled = settle(joined); + if outcome.is_ok() { + outcome = settled; + } + } + outcome + } +} + +impl Drop for ExecutionProxy { + fn drop(&mut self) { + // A proxy dropped without `finish` must not leave forwarding tasks + // holding sockets to the disposable database. + if let Ok(mut accept) = self.accept.lock() { + if let Some(accept) = accept.take() { + accept.abort(); + } + } + if let Ok(mut tasks) = self.tasks.lock() { + tasks.abort_all(); + } + } +} + +fn settle(joined: std::result::Result, tokio::task::JoinError>) -> Result<()> { + match joined { + Ok(result) => result.context("proxy connection task failed"), + Err(error) if error.is_cancelled() => Ok(()), + Err(error) => bail!("proxy connection task panicked: {error}"), + } +} + +/// Executions since a mark that are the operation marked `table`/`op`: +/// the marked executions themselves plus every other frame that ran the +/// same statement text, marked or not. A statement replayed inside an +/// aborted transaction fails before its triggers run and so carries no +/// marker, but it is the same statement and is counted. +pub fn target_executions<'a>( + executions: &'a [Execution], + table: &str, + op: &str, +) -> Vec<&'a Execution> { + let identities: HashSet<&str> = executions + .iter() + .filter(|execution| execution.marked(table, op)) + .map(|execution| execution.sql.as_str()) + .collect(); + executions + .iter() + .filter(|execution| { + execution.marked(table, op) || identities.contains(execution.sql.as_str()) + }) + .collect() +} + +/// How many statements of the operation marked `table`/`op` the executions +/// carried: one per marker, since the fixture trigger is statement level, so +/// a simple `Query` frame that runs the statement twice counts twice, plus +/// one for each unmarked frame of the same text (a replay the server failed +/// before its triggers ran). This counts statements sent in execution frames; +/// a statement rejected at `Parse` is a [`Rejection`], not a count. A frame +/// whose connection a fault closed stops being observed there, so later +/// statements of that same frame are not counted. +pub fn target_statement_count(executions: &[Execution], table: &str, op: &str) -> usize { + target_executions(executions, table, op) + .iter() + .map(|execution| { + execution + .markers + .iter() + .filter(|marker| marker.table == table && marker.op == op) + .count() + .max(1) + }) + .sum() +} + +/// Executions since a mark that carry a marker for any table other than +/// `table`, that is, writes the fixture observes elsewhere. +pub fn other_marked_executions<'a>(executions: &'a [Execution], table: &str) -> Vec<&'a Execution> { + executions + .iter() + .filter(|execution| execution.markers.iter().any(|marker| marker.table != table)) + .collect() +} + +async fn serve( + mut client: TcpStream, + upstream: SocketAddr, + id: u64, + shared: Arc, +) -> Result<()> { + let mut server = TcpStream::connect(upstream) + .await + .context("proxy upstream connect")?; + client.set_nodelay(true)?; + server.set_nodelay(true)?; + // Startup messages carry no type byte: a length, then a version code. + loop { + let mut length = [0u8; 4]; + match client.read_exact(&mut length).await { + Ok(_) => {} + Err(error) if error.kind() == std::io::ErrorKind::UnexpectedEof => return Ok(()), + Err(error) => return Err(error.into()), + } + let length = usize::try_from(u32::from_be_bytes(length))?; + ensure!((8..=10_000).contains(&length), "invalid startup length"); + let mut body = vec![0u8; length - 4]; + client.read_exact(&mut body).await?; + let code = u32::from_be_bytes(body[..4].try_into().expect("four bytes")); + let mut frame = Vec::with_capacity(length); + frame.extend_from_slice(&u32::try_from(length)?.to_be_bytes()); + frame.extend_from_slice(&body); + match code { + SSL_REQUEST | GSSENC_REQUEST => client.write_all(b"N").await?, + CANCEL_REQUEST => { + server.write_all(&frame).await?; + return Ok(()); + } + _ => { + server.write_all(&frame).await?; + break; + } + } + } + let (client_read, client_write) = client.into_split(); + let (server_read, server_write) = server.into_split(); + // A frame's kind, length and body are decoded separately. Buffer socket + // reads so a large result does not pay three reads per DataRow; keep the + // existing frame-by-frame observation, forwarding and fault boundaries. + let client_read = BufReader::new(client_read); + let server_read = BufReader::new(server_read); + let connection = Arc::new(Mutex::new(Connection::default())); + // Whichever direction ends first (client EOF, server EOF, or a fault) + // drops both sockets; the server then aborts any open transaction. + let relayed = relay( + pump_client( + client_read, + server_write, + connection.clone(), + shared.clone(), + id, + ), + pump_server(server_read, client_write, connection, shared.clone(), id), + ) + .await; + shared.state.lock().expect("proxy state").ended.insert(id); + relayed +} + +async fn relay( + client: impl Future>, + server: impl Future>, +) -> Result<()> { + tokio::pin!(client); + tokio::select! { + result = &mut client => result, + result = server => match result? { + ServerEnd::Closed => Ok(()), + ServerEnd::ClientWrite(error) => { + // A client can close while a reply is being forwarded, before + // select polls its readable EOF. Only a frame-boundary EOF + // observed NOW justifies the same outcome as the client arm. + // Do not wait for it, accept resets, or hide malformed/partial + // frames. Polling the original future preserves any buffered + // COMMIT/ROLLBACK requests and their still-unknown outcomes. + // Reactor readiness or cooperative yielding can still leave + // EOF pending: conservatively retain the error in that case. + if error.kind() == std::io::ErrorKind::BrokenPipe { + if let Some(result) = client.as_mut().now_or_never() { + return result.context("client boundary after failed reply delivery"); + } + } + Err(error).context("forwarding server frame to client") + } + }, + } +} + +async fn read_frame(reader: &mut R) -> Result)>> { + let mut kind = [0u8; 1]; + match reader.read_exact(&mut kind).await { + Ok(_) => {} + Err(error) if error.kind() == std::io::ErrorKind::UnexpectedEof => return Ok(None), + Err(error) => return Err(error.into()), + } + let mut length = [0u8; 4]; + reader.read_exact(&mut length).await?; + let length = usize::try_from(u32::from_be_bytes(length))?; + ensure!(length >= 4, "invalid frame length"); + let mut body = vec![0u8; length - 4]; + reader.read_exact(&mut body).await?; + Ok(Some((kind[0], body))) +} + +fn frame(kind: u8, body: &[u8]) -> Result> { + let mut bytes = Vec::with_capacity(body.len() + 5); + bytes.push(kind); + bytes.extend_from_slice(&u32::try_from(body.len() + 4)?.to_be_bytes()); + bytes.extend_from_slice(body); + Ok(bytes) +} + +fn cstring(body: &[u8], at: &mut usize) -> Result> { + let end = body[*at..] + .iter() + .position(|&byte| byte == 0) + .map(|offset| *at + offset) + .context("unterminated string in frame")?; + let value = body[*at..end].to_vec(); + *at = end + 1; + Ok(value) +} + +fn text(bytes: Vec) -> String { + String::from_utf8_lossy(&bytes).into_owned() +} + +fn record( + shared: &Shared, + connection: &Mutex, + id: u64, + protocol: Protocol, + sql: String, +) { + let seq = shared.seq.fetch_add(1, Ordering::SeqCst) + 1; + let index = { + let mut state = shared.state.lock().expect("proxy state"); + state.executions.push(Execution { + seq, + connection: id, + protocol, + sql, + markers: Vec::new(), + outcome: Outcome::Pending, + tags: Vec::new(), + rows_received: 0, + jsonb_writes: BTreeMap::new(), + select_rows: None, + ready: false, + }); + state.executions.len() - 1 + }; + connection + .lock() + .expect("proxy connection") + .inflight + .push_back(index); +} + +async fn pump_client( + mut from: impl AsyncRead + Unpin, + mut to: impl AsyncWrite + Unpin, + connection: Arc>, + shared: Arc, + id: u64, +) -> Result<()> { + while let Some((kind, body)) = read_frame(&mut from).await? { + let mut at = 0; + match kind { + b'P' => { + let name = cstring(&body, &mut at)?; + let sql = text(cstring(&body, &mut at)?); + let mut state = connection.lock().expect("proxy connection"); + state.parses.push_back(sql.clone()); + state.statements.insert(name, sql); + } + b'B' => { + let portal = cstring(&body, &mut at)?; + let statement = cstring(&body, &mut at)?; + let mut state = connection.lock().expect("proxy connection"); + let sql = state + .statements + .get(&statement) + .cloned() + .unwrap_or_else(|| UNPARSED.to_owned()); + state.portals.insert(portal, sql); + } + b'E' => { + let portal = cstring(&body, &mut at)?; + let sql = connection + .lock() + .expect("proxy connection") + .portals + .get(&portal) + .cloned() + .unwrap_or_else(|| UNBOUND.to_owned()); + record(&shared, &connection, id, Protocol::Extended, sql); + } + b'Q' => { + let sql = text(cstring(&body, &mut at)?); + record(&shared, &connection, id, Protocol::Simple, sql); + } + b'C' => { + ensure!(!body.is_empty(), "empty Close frame"); + at = 1; + let name = cstring(&body, &mut at)?; + let mut state = connection.lock().expect("proxy connection"); + match body[0] { + b'S' => { + state.statements.remove(&name); + } + b'P' => { + state.portals.remove(&name); + } + _ => {} + } + } + b'X' => { + ensure!(body.is_empty(), "invalid Terminate frame"); + shared + .state + .lock() + .expect("proxy state") + .terminated + .insert(id); + } + // Other messages used by authentication, extended queries and + // COPY do not start a separately counted execution. Unknown kinds + // (including unobserved FunctionCall execution) must not become + // evidence of a clean stream just because EOF follows them. + b'D' | b'H' | b'S' | b'p' | b'd' | b'c' | b'f' => {} + _ => bail!("unsupported client frame kind {kind:#x}"), + } + to.write_all(&frame(kind, &body)?).await?; + } + Ok(()) +} + +enum Action { + Forward, + Deliver(Delivery), + Sever, + Pause { + delivery: Delivery, + control: Arc, + }, +} + +#[derive(Clone, Copy)] +enum Delivery { + Complete(usize), + Ready(u8), +} + +enum ServerEnd { + Closed, + ClientWrite(std::io::Error), +} + +async fn pump_server( + mut from: impl AsyncRead + Unpin, + mut to: impl AsyncWrite + Unpin, + connection: Arc>, + shared: Arc, + id: u64, +) -> Result { + while let Some((kind, body)) = read_frame(&mut from).await? { + let action = match kind { + b'C' => { + let mut at = 0; + let tag = text(cstring(&body, &mut at)?); + complete(&shared, &connection, id, tag)? + } + b'I' => complete(&shared, &connection, id, String::new())?, + b'D' => { + let index = front(&connection).context("DataRow without an observed execution")?; + let mut state = shared.state.lock().expect("proxy state"); + let count = &mut state.executions[index].rows_received; + *count = count.checked_add(1).context("DataRow counter overflow")?; + Action::Forward + } + b'1' => { + connection + .lock() + .expect("proxy connection") + .parses + .pop_front(); + Action::Forward + } + b'E' => { + let (code, message) = fields(&body)?; + fail(&shared, &connection, id, code, message); + Action::Forward + } + b'N' => { + let (_, message) = fields(&body)?; + notice(&shared, &connection, &message)?; + Action::Forward + } + b'Z' => { + ensure!( + matches!(body.as_slice(), [b'I' | b'T' | b'E']), + "invalid ReadyForQuery frame" + ); + ready_action(&shared, &connection, body[0]) + } + _ => Action::Forward, + }; + let delivery = match action { + Action::Forward => None, + Action::Deliver(delivery) => Some(delivery), + Action::Sever => return Ok(ServerEnd::Closed), + Action::Pause { delivery, control } => { + // No observer lock or PostgreSQL transaction lock is held here. + loop { + let release = control.release.notified(); + tokio::pin!(release); + release.as_mut().enable(); + if control.released.load(Ordering::SeqCst) { + break; + } + release.await; + } + Some(delivery) + } + }; + let bytes = frame(kind, &body)?; + let forwarded = match delivery { + Some(delivery) => { + forward_observed(&mut to, &bytes, &shared, &connection, id, delivery).await + } + None => to.write_all(&bytes).await, + }; + if let Err(error) = forwarded { + return Ok(ServerEnd::ClientWrite(error)); + } + } + Ok(ServerEnd::Closed) +} + +/// Serialize acknowledgement publication with its final nonblocking write. +/// A peer can read the bytes on another thread before poll_write returns; +/// observer readers must not see the old incomplete state in that interval. +/// Guards exist only inside one poll, never across an await. A failed/partial +/// write publishes nothing, so this does not premark or roll back success. +async fn forward_observed( + to: &mut (impl AsyncWrite + Unpin), + mut bytes: &[u8], + shared: &Shared, + connection: &Mutex, + id: u64, + delivery: Delivery, +) -> io::Result<()> { + while !bytes.is_empty() { + let written = poll_fn(|cx| { + let mut connection = connection.lock().expect("proxy connection"); + let mut state = shared.state.lock().expect("proxy state"); + let result = Pin::new(&mut *to).poll_write(cx, bytes); + if matches!(result, Poll::Ready(Ok(written)) if written == bytes.len()) { + match delivery { + Delivery::Complete(index) => { + if let Outcome::Completed { delivered, .. } = + &mut state.executions[index].outcome + { + *delivered = true; + } + } + Delivery::Ready(status) => ready(&mut state, &mut connection, id, status), + } + } + result + }) + .await?; + if written == 0 { + return Err(io::ErrorKind::WriteZero.into()); + } + bytes = &bytes[written..]; + } + Ok(()) +} + +/// SQLSTATE and message of an ErrorResponse or NoticeResponse. +fn fields(body: &[u8]) -> Result<(String, String)> { + let mut code = String::new(); + let mut message = String::new(); + let mut at = 0; + while at < body.len() && body[at] != 0 { + let kind = body[at]; + at += 1; + let value = text(cstring(body, &mut at)?); + match kind { + b'C' => code = value, + b'M' => message = value, + _ => {} + } + } + Ok((code, message)) +} + +fn front(connection: &Mutex) -> Option { + connection + .lock() + .expect("proxy connection") + .inflight + .front() + .copied() +} + +fn pop(connection: &Mutex) { + connection + .lock() + .expect("proxy connection") + .inflight + .pop_front(); +} + +fn complete( + shared: &Shared, + connection: &Mutex, + id: u64, + tag: String, +) -> Result { + let Some(index) = front(connection) else { + return Ok(Action::Forward); + }; + let mut state = shared.state.lock().expect("proxy state"); + let State { + executions, + rejections: _, + plan, + armed, + fired, + pause_plan, + pause_armed, + statement_pause, + terminated: _, + ended: _, + } = &mut *state; + let execution = &mut executions[index]; + if let Some(count) = tag.strip_prefix("SELECT ") { + let count: u64 = count.parse().context("invalid SELECT completion count")?; + execution.select_rows = Some( + execution + .select_rows + .unwrap_or(0) + .checked_add(count) + .context("SELECT completion counter overflow")?, + ); + } + let protocol = execution.protocol; + let mut sever = false; + if let Some(fault) = plan.as_ref() { + let marked = execution.marked(&fault.table, &fault.op); + match fault.phase { + FaultPhase::AfterExecution if marked => sever = true, + FaultPhase::AfterCommit if marked => *armed = Some(id), + FaultPhase::AfterCommit if *armed == Some(id) && tag == "COMMIT" => sever = true, + _ => {} + } + } + let mut pause = None; + if !sever { + if let Some(plan) = pause_plan.as_ref() { + if execution.marked(&plan.table, &plan.op) { + *pause_armed = Some(id); + } else if *pause_armed == Some(id) && tag == "COMMIT" { + pause = Some(plan.control.clone()); + *pause_plan = None; + *pause_armed = None; + } + } + if pause.is_none() { + if let Some(plan) = statement_pause.as_ref() { + if plan.at == PauseAt::Completion && execution.sql == plan.sql { + pause = Some(plan.control.clone()); + *statement_pause = None; + } + } + } + } + execution.tags.push(tag.clone()); + execution.outcome = Outcome::Completed { + tag, + delivered: false, + }; + if let Some(control) = &pause { + control.seq.store(execution.seq, Ordering::SeqCst); + control.entered.notify_waiters(); + } + if sever { + *fired = Some(execution.seq); + *plan = None; + *armed = None; + } + drop(state); + if protocol == Protocol::Extended { + pop(connection); + } + if sever { + Ok(Action::Sever) + } else if let Some(control) = pause { + Ok(Action::Pause { + delivery: Delivery::Complete(index), + control, + }) + } else { + Ok(Action::Deliver(Delivery::Complete(index))) + } +} + +/// A `ReadyForQuery` is delivered as observed unless it closes the simple +/// query a [`PauseAt::Ready`] plan is waiting for. +fn ready_action(shared: &Shared, connection: &Mutex, status: u8) -> Action { + let delivery = Delivery::Ready(status); + let Some(index) = front(connection) else { + return Action::Deliver(delivery); + }; + let mut state = shared.state.lock().expect("proxy state"); + let execution = &state.executions[index]; + let seq = execution.seq; + let matched = state.statement_pause.as_ref().is_some_and(|plan| { + plan.at == PauseAt::Ready + && execution.protocol == Protocol::Simple + && execution.sql == plan.sql + }); + if !matched { + return Action::Deliver(delivery); + } + let control = state.statement_pause.take().expect("matched plan").control; + control.seq.store(seq, Ordering::SeqCst); + control.entered.notify_waiters(); + Action::Pause { delivery, control } +} + +/// An `ErrorResponse` answers the oldest unanswered client frame. A client +/// sends `Parse` ahead of the `Bind` and `Execute` that use it, so an +/// unanswered `Parse` is the rejected frame; otherwise it is the oldest +/// in-flight execution, if any. +fn fail(shared: &Shared, connection: &Mutex, id: u64, code: String, message: String) { + let (parse, index) = { + let mut state = connection.lock().expect("proxy connection"); + match state.parses.pop_front() { + Some(sql) => (Some(sql), None), + None => (None, state.inflight.pop_front()), + } + }; + let seq = shared.seq.fetch_add(1, Ordering::SeqCst) + 1; + let mut state = shared.state.lock().expect("proxy state"); + let frame = match (parse, index) { + (Some(sql), _) => RejectedFrame::Parse(sql), + (None, Some(index)) => { + let execution = &mut state.executions[index]; + execution.outcome = Outcome::Failed { + code: code.clone(), + message: message.clone(), + }; + RejectedFrame::Execution(execution.seq) + } + (None, None) => RejectedFrame::Other, + }; + state.rejections.push(Rejection { + seq, + connection: id, + code, + message, + frame, + }); +} + +fn notice(shared: &Shared, connection: &Mutex, message: &str) -> Result<()> { + if let Some(encoded) = message.strip_prefix(JSONB_WRITE_PREFIX) { + let notice: JsonbWriteNotice = + serde_json::from_str(encoded).context("invalid JSONB write observation")?; + ensure!( + !notice.table.is_empty() + && matches!(notice.operation.as_str(), "INSERT" | "UPDATE") + && notice.values.values().all(|bytes| *bytes > 0), + "invalid JSONB write measurement" + ); + let index = front(connection).context("JSONB write without an observed execution")?; + let mut state = shared.state.lock().expect("proxy state"); + let summary = state.executions[index] + .jsonb_writes + .entry((notice.table, notice.operation)) + .or_default(); + summary.rows = summary + .rows + .checked_add(1) + .context("JSONB row counter overflow")?; + summary.values = summary + .values + .checked_add(notice.values.len().try_into()?) + .context("JSONB value counter overflow")?; + summary.max_uncompressed_bytes = summary + .max_uncompressed_bytes + .max(notice.values.values().copied().max().unwrap_or(0)); + return Ok(()); + } + let Some(rest) = message.strip_prefix(MARKER_PREFIX) else { + return Ok(()); + }; + let mut words = rest.split_whitespace(); + let (Some(table), Some(op)) = (words.next(), words.next()) else { + return Ok(()); + }; + let Some(index) = front(connection) else { + return Ok(()); + }; + shared.state.lock().expect("proxy state").executions[index] + .markers + .push(Marker { + table: table.to_owned(), + op: op.to_owned(), + }); + Ok(()) +} + +fn ready(state: &mut State, connection: &mut Connection, id: u64, status: u8) { + // Every frame before this `ReadyForQuery` has been answered, or was + // skipped by the server after an earlier error in the same batch; a + // skipped execution stays `Pending` and nothing later settles it. + for index in &connection.inflight { + state.executions[*index].ready = true; + } + connection.parses.clear(); + connection.inflight.clear(); + if status == b'I' { + // The transaction ended without the COMMIT the fault was waiting for + // (a rollback): the arming no longer applies to this connection. + if state.armed == Some(id) && state.fired.is_none() { + state.armed = None; + } + if state.pause_armed == Some(id) { + state.pause_armed = None; + } + } +} + +#[cfg(test)] +#[path = "ledger_execution_proxy_close.rs"] +mod close_tests; diff --git a/crates/qbit-prism-server/tests/support/ledger_execution_proxy_close.rs b/crates/qbit-prism-server/tests/support/ledger_execution_proxy_close.rs new file mode 100644 index 000000000..9e3752748 --- /dev/null +++ b/crates/qbit-prism-server/tests/support/ledger_execution_proxy_close.rs @@ -0,0 +1,615 @@ +//! Deterministic transport scheduling at the proxy's actual pump boundary. +use super::*; +use std::io; +use std::pin::Pin; +use std::task::{Context as TaskContext, Poll, Waker}; +use tokio::io::ReadBuf; + +#[derive(Default)] +struct CloseGate { + closed: AtomicBool, + reader: Mutex>, +} + +impl CloseGate { + fn close(&self) { + self.closed.store(true, Ordering::SeqCst); + if let Some(waker) = self.reader.lock().unwrap().take() { + waker.wake(); + } + } +} + +enum ClientEnd { + Eof, + Reset, + Pending, +} + +struct ClosingClient { + frames: std::io::Cursor>, + after_close: std::io::Cursor>, + gate: Arc, + end: ClientEnd, +} + +impl AsyncRead for ClosingClient { + fn poll_read( + mut self: Pin<&mut Self>, + cx: &mut TaskContext<'_>, + buf: &mut ReadBuf<'_>, + ) -> Poll> { + if self.frames.position() < self.frames.get_ref().len() as u64 { + return Pin::new(&mut self.frames).poll_read(cx, buf); + } + *self.gate.reader.lock().unwrap() = Some(cx.waker().clone()); + if !self.gate.closed.load(Ordering::SeqCst) { + return Poll::Pending; + } + if self.after_close.position() < self.after_close.get_ref().len() as u64 { + return Pin::new(&mut self.after_close).poll_read(cx, buf); + } + match self.end { + ClientEnd::Eof => Poll::Ready(Ok(())), + ClientEnd::Reset => Poll::Ready(Err(io::ErrorKind::ConnectionReset.into())), + ClientEnd::Pending => Poll::Pending, + } + } +} + +struct BrokenClientWriter { + gate: Arc, + error: io::ErrorKind, + /// Accept only this many bytes before failing, including partial frames. + remaining: usize, +} + +struct PauseAtReadyDelivery { + socket: TcpStream, + remaining: Option, + release: std::sync::mpsc::Receiver<()>, +} + +impl AsyncWrite for PauseAtReadyDelivery { + fn poll_write( + mut self: Pin<&mut Self>, + cx: &mut TaskContext<'_>, + bytes: &[u8], + ) -> Poll> { + if self.remaining.is_none() && bytes.first() == Some(&b'Z') { + self.remaining = Some(bytes.len()); + } + let result = Pin::new(&mut self.socket).poll_write(cx, bytes); + if let Poll::Ready(Ok(written)) = result { + if let Some(remaining) = &mut self.remaining { + *remaining -= written; + if *remaining == 0 { + self.remaining = None; + // Test-only scheduler barrier: the real TCP write has + // completed, but its proxy worker cannot publish yet. + // Hand off the Tokio worker while blocking so the TCP + // reader and I/O driver can progress even with one worker. + if let Err(error) = tokio::task::block_in_place(|| { + self.release.recv_timeout(std::time::Duration::from_secs(5)) + }) { + return Poll::Ready(Err(io::Error::other(error))); + } + } + } + } + result + } + + fn poll_flush(mut self: Pin<&mut Self>, cx: &mut TaskContext<'_>) -> Poll> { + Pin::new(&mut self.socket).poll_flush(cx) + } + + fn poll_shutdown(mut self: Pin<&mut Self>, cx: &mut TaskContext<'_>) -> Poll> { + Pin::new(&mut self.socket).poll_shutdown(cx) + } +} + +impl AsyncWrite for BrokenClientWriter { + fn poll_write( + mut self: Pin<&mut Self>, + _: &mut TaskContext<'_>, + bytes: &[u8], + ) -> Poll> { + if self.remaining > 0 { + let written = self.remaining.min(bytes.len()); + self.remaining -= written; + return Poll::Ready(Ok(written)); + } + // Client EOF becomes readable in the same poll that its write fails. + // The server pump therefore wins select before the client is repolled. + self.gate.close(); + Poll::Ready(Err(self.error.into())) + } + + fn poll_flush(self: Pin<&mut Self>, _: &mut TaskContext<'_>) -> Poll> { + Poll::Ready(Ok(())) + } + + fn poll_shutdown(self: Pin<&mut Self>, _: &mut TaskContext<'_>) -> Poll> { + Poll::Ready(Ok(())) + } +} + +fn observations() -> Arc { + Arc::new(Shared { + seq: AtomicU64::new(0), + connections: AtomicU64::new(0), + state: Mutex::default(), + }) +} + +async fn ordered_close( + requests: Vec, + after_close: Vec, + responses: Vec, + end: ClientEnd, + write_error: io::ErrorKind, + accepted_bytes: usize, +) -> (Result<()>, Arc) { + let shared = observations(); + let connection = Arc::new(Mutex::new(Connection::default())); + let gate = Arc::new(CloseGate::default()); + let client = pump_client( + ClosingClient { + frames: io::Cursor::new(requests), + after_close: io::Cursor::new(after_close), + gate: gate.clone(), + end, + }, + tokio::io::sink(), + connection.clone(), + shared.clone(), + 1, + ); + tokio::pin!(client); + // Requests are observed and forwarded before the reply; EOF is not yet + // readable. This fixes the order without sleeps or scheduler assumptions. + assert!(futures_util::poll!(&mut client).is_pending()); + let server = pump_server( + responses.as_slice(), + BrokenClientWriter { + gate, + error: write_error, + remaining: accepted_bytes, + }, + connection, + shared.clone(), + 1, + ); + (relay(client, server).await, shared) +} + +#[tokio::test] +async fn client_eof_after_commit_write_failure_keeps_rollback_pending() -> Result<()> { + let (result, shared) = ordered_close( + [frame(b'Q', b"COMMIT\0")?, frame(b'Q', b"ROLLBACK\0")?].concat(), + frame(b'X', b"")?, + frame(b'C', b"COMMIT\0")?, + ClientEnd::Eof, + io::ErrorKind::BrokenPipe, + 0, + ) + .await; + result?; + let state = shared.state.lock().unwrap(); + assert_eq!(state.executions.len(), 2); + assert_eq!(state.executions[0].sql, "COMMIT"); + assert_eq!( + state.executions[0].outcome, + Outcome::Completed { + tag: "COMMIT".into(), + delivered: false + } + ); + assert!(!state.executions[0].complete_response()); + assert_eq!(state.executions[1].sql, "ROLLBACK"); + assert_eq!(state.executions[1].outcome, Outcome::Pending); + Ok(()) +} + +#[tokio::test] +async fn buffered_rollback_before_eof_is_counted_without_claiming_it_completed() -> Result<()> { + let (result, shared) = ordered_close( + frame(b'Q', b"COMMIT\0")?, + [frame(b'Q', b"ROLLBACK\0")?, frame(b'X', b"")?].concat(), + frame(b'C', b"COMMIT\0")?, + ClientEnd::Eof, + io::ErrorKind::BrokenPipe, + 0, + ) + .await; + result?; + let state = shared.state.lock().unwrap(); + assert_eq!(state.executions.len(), 2); + assert_eq!(state.executions[1].sql, "ROLLBACK"); + assert_eq!(state.executions[1].outcome, Outcome::Pending); + assert!(!state.executions[0].delivered()); + Ok(()) +} + +#[tokio::test] +async fn eof_without_a_commit_acknowledgement_keeps_both_controls_pending() -> Result<()> { + let (result, shared) = ordered_close( + [frame(b'Q', b"COMMIT\0")?, frame(b'Q', b"ROLLBACK\0")?].concat(), + vec![], + frame(b'N', b"Mclosing\0\0")?, + ClientEnd::Eof, + io::ErrorKind::BrokenPipe, + 0, + ) + .await; + result?; + let state = shared.state.lock().unwrap(); + assert_eq!(state.executions.len(), 2); + for execution in &state.executions { + assert_eq!(execution.outcome, Outcome::Pending); + assert!(!execution.complete_response()); + assert!(!execution.is_commit()); + } + Ok(()) +} + +#[tokio::test] +async fn client_eof_winning_first_keeps_unobserved_commit_and_rollback_pending() -> Result<()> { + let shared = observations(); + let connection = Arc::new(Mutex::new(Connection::default())); + let gate = Arc::new(CloseGate::default()); + let client = pump_client( + ClosingClient { + frames: io::Cursor::new( + [frame(b'Q', b"COMMIT\0")?, frame(b'Q', b"ROLLBACK\0")?].concat(), + ), + after_close: io::Cursor::new(vec![]), + gate: gate.clone(), + end: ClientEnd::Eof, + }, + tokio::io::sink(), + connection, + shared.clone(), + 1, + ); + tokio::pin!(client); + assert!(futures_util::poll!(&mut client).is_pending()); + gate.close(); + relay(client, std::future::pending()).await?; + let state = shared.state.lock().unwrap(); + assert_eq!(state.executions.len(), 2); + for execution in &state.executions { + assert_eq!(execution.outcome, Outcome::Pending); + assert!(!execution.complete_response()); + } + Ok(()) +} + +#[tokio::test] +async fn broken_pipe_without_observed_eof_remains_an_error() -> Result<()> { + let (result, _) = ordered_close( + frame(b'Q', b"COMMIT\0")?, + vec![], + frame(b'C', b"COMMIT\0")?, + ClientEnd::Pending, + io::ErrorKind::BrokenPipe, + 0, + ) + .await; + assert_eq!( + result + .unwrap_err() + .downcast_ref::() + .unwrap() + .kind(), + io::ErrorKind::BrokenPipe + ); + Ok(()) +} + +#[tokio::test] +async fn unrelated_client_read_and_write_resets_remain_errors() -> Result<()> { + for (end, write_error) in [ + (ClientEnd::Reset, io::ErrorKind::BrokenPipe), + (ClientEnd::Eof, io::ErrorKind::ConnectionReset), + ] { + let (result, _) = ordered_close( + frame(b'Q', b"COMMIT\0")?, + vec![], + frame(b'C', b"COMMIT\0")?, + end, + write_error, + 0, + ) + .await; + assert_eq!( + result + .unwrap_err() + .downcast_ref::() + .unwrap() + .kind(), + io::ErrorKind::ConnectionReset + ); + } + Ok(()) +} + +#[tokio::test] +async fn truncated_and_malformed_client_frames_cannot_prove_clean_close() -> Result<()> { + for tail in [ + vec![b'Q', 0, 0], + vec![b'Q', 0, 0, 0, 7, b'x'], + frame(b'Q', b"unterminated")?, + frame(b'X', b"invalid")?, + ] { + let (result, _) = ordered_close( + frame(b'Q', b"COMMIT\0")?, + tail, + frame(b'C', b"COMMIT\0")?, + ClientEnd::Eof, + io::ErrorKind::BrokenPipe, + 0, + ) + .await; + assert!(result.is_err(), "malformed client frame was accepted"); + } + Ok(()) +} + +#[tokio::test] +async fn malformed_server_frames_remain_errors() -> Result<()> { + for response in [ + vec![b'C', 0, 0, 0, 3], + vec![b'C', 0, 0, 0, 7, b'x'], + frame(b'C', b"COMMIT")?, + frame(b'N', b"Munterminated")?, + ] { + let (result, _) = ordered_close( + frame(b'Q', b"COMMIT\0")?, + vec![], + response, + ClientEnd::Eof, + io::ErrorKind::BrokenPipe, + 0, + ) + .await; + assert!(result.is_err(), "malformed server frame was accepted"); + } + Ok(()) +} + +#[tokio::test] +async fn malformed_ready_for_query_cannot_complete_a_select() -> Result<()> { + for body in [b"".as_slice(), b"?", b"II"] { + let (result, shared) = ordered_close( + frame(b'Q', b"SELECT 1 WHERE false\0")?, + vec![], + [frame(b'C', b"SELECT 0\0")?, frame(b'Z', body)?].concat(), + ClientEnd::Pending, + io::ErrorKind::BrokenPipe, + usize::MAX, + ) + .await; + assert!(result.is_err(), "malformed ReadyForQuery was accepted"); + assert!(shared.state.lock().unwrap().executions[0] + .returned_rows() + .is_err()); + } + Ok(()) +} + +#[tokio::test] +async fn unknown_frontend_frame_cannot_qualify_broken_pipe_recovery() -> Result<()> { + let (result, _) = ordered_close( + frame(b'Q', b"COMMIT\0")?, + frame(b'?', b"")?, + frame(b'C', b"COMMIT\0")?, + ClientEnd::Eof, + io::ErrorKind::BrokenPipe, + 0, + ) + .await; + assert!(result.is_err(), "unknown frontend frame was accepted"); + Ok(()) +} + +#[tokio::test] +async fn lost_or_partial_completion_and_ready_frames_keep_select_incomplete() -> Result<()> { + let completion = frame(b'C', b"SELECT 0\0")?; + let response = [completion.clone(), frame(b'Z', b"I")?].concat(); + for accepted in [0, 1, completion.len(), completion.len() + 1] { + let (result, shared) = ordered_close( + frame(b'Q', b"SELECT 1 WHERE false\0")?, + vec![], + response.clone(), + ClientEnd::Eof, + io::ErrorKind::BrokenPipe, + accepted, + ) + .await; + result?; + let state = shared.state.lock().unwrap(); + assert_eq!(state.executions.len(), 1); + let execution = &state.executions[0]; + assert_eq!(execution.delivered(), accepted >= completion.len()); + assert!(!execution.complete_response()); + assert!(execution.returned_rows().is_err()); + } + Ok(()) +} + +#[tokio::test] +async fn successful_zero_and_incorrect_row_counts_remain_distinct() -> Result<()> { + for count in [0, 1] { + let (result, shared) = ordered_close( + frame(b'Q', b"SELECT 1 WHERE false\0")?, + vec![], + [ + frame(b'C', format!("SELECT {count}\0").as_bytes())?, + frame(b'Z', b"I")?, + ] + .concat(), + ClientEnd::Pending, + io::ErrorKind::BrokenPipe, + usize::MAX, + ) + .await; + result?; + let state = shared.state.lock().unwrap(); + assert_eq!(state.executions.len(), 1); + let execution = &state.executions[0]; + assert!(execution.complete_response()); + if count == 0 { + assert_eq!(execution.returned_rows()?, 0); + } else { + assert!(execution.returned_rows().is_err()); + } + } + Ok(()) +} + +// One worker makes an unannounced blocking wait in the writer starve the TCP +// reader deterministically, rather than depending on multi-worker scheduling. +#[tokio::test(flavor = "multi_thread", worker_threads = 1)] +async fn acknowledgement_delivery_cannot_expose_an_incomplete_observer_snapshot() -> Result<()> { + let shared = observations(); + let connection = Arc::new(Mutex::new(Connection::default())); + record( + &shared, + &connection, + 1, + Protocol::Simple, + "SELECT 1 WHERE false".into(), + ); + let proxy = observer(shared.clone()); + let listener = TcpListener::bind("127.0.0.1:0").await?; + let (client, server) = tokio::join!( + TcpStream::connect(listener.local_addr()?), + listener.accept() + ); + let mut client = BufReader::new(client?); + let (release, pause) = std::sync::mpsc::channel(); + let replies = [frame(b'C', b"SELECT 0\0")?, frame(b'Z', b"I")?].concat(); + let mut tasks = JoinSet::new(); + tasks.spawn(pump_server( + io::Cursor::new(replies), + PauseAtReadyDelivery { + socket: server?.0, + remaining: None, + release: pause, + }, + connection, + shared.clone(), + 1, + )); + let received = tokio::time::timeout(std::time::Duration::from_secs(5), async { + ensure!(read_frame(&mut client).await? == Some((b'C', b"SELECT 0\0".to_vec()))); + ensure!(read_frame(&mut client).await? == Some((b'Z', b"I".to_vec()))); + Ok::<_, anyhow::Error>(()) + }) + .await; + // Inspect through the public observer while the forwarding worker is + // paused. If publication holds its lock, readers must wait for it; if it + // does not, an actual snapshot here must already report completion. + let readable = shared.state.try_lock().is_ok(); + let during_delivery = readable.then(|| proxy.executions_since(0)); + let released = release.send(()); + let pumped = tasks.join_next().await.context("missing server pump")?; + received??; + released?; + pumped??; + if let Some(snapshot) = during_delivery { + assert!( + snapshot?[0].complete_response(), + "client-visible acknowledgement raced observer publication" + ); + } + assert_eq!( + shared.state.lock().unwrap().executions[0].returned_rows()?, + 0 + ); + Ok(()) +} + +#[tokio::test] +async fn undelivered_error_remains_a_server_observation_not_a_complete_response() -> Result<()> { + let (result, shared) = ordered_close( + frame(b'Q', b"COMMIT\0")?, + frame(b'X', b"")?, + frame(b'E', b"C40001\0Mserialization failure\0\0")?, + ClientEnd::Eof, + io::ErrorKind::BrokenPipe, + 0, + ) + .await; + result?; + let state = shared.state.lock().unwrap(); + assert_eq!(state.executions.len(), 1); + let execution = &state.executions[0]; + assert_eq!(execution.sqlstate(), Some("40001")); + assert!(!execution.delivered()); + assert!(!execution.complete_response()); + assert!(execution.returned_rows().is_err()); + assert_eq!(state.rejections.len(), 1); + assert_eq!( + state.rejections[0].frame, + RejectedFrame::Execution(execution.seq) + ); + Ok(()) +} + +fn observer(shared: Arc) -> ExecutionProxy { + ExecutionProxy { + addr: "127.0.0.1:0".parse().unwrap(), + shared, + accept: Mutex::default(), + tasks: Arc::default(), + failure: Mutex::default(), + } +} + +#[tokio::test] +async fn executions_reject_unknown_frames_and_keep_reported_task_failures() -> Result<()> { + let (result, shared) = ordered_close( + frame(b'E', b"unbound\0\0\0\0\0")?, + vec![], + frame(b'C', b"SELECT 0\0")?, + ClientEnd::Eof, + io::ErrorKind::BrokenPipe, + 0, + ) + .await; + result?; + let proxy = observer(shared); + assert!(proxy + .executions_since(0) + .unwrap_err() + .to_string() + .contains("could not identify")); + proxy.finish().await?; + + let proxy = observer(observations()); + let task = proxy + .tasks + .lock() + .unwrap() + .spawn(async { bail!("unrelated connection failure") }); + // Await this exact task's completion without draining the registry that + // check() must inspect. No timing delay stands in for observing it. + while !task.is_finished() { + tokio::task::yield_now().await; + } + for _ in 0..2 { + assert!(proxy + .executions_since(0) + .unwrap_err() + .to_string() + .contains("failed")); + } + // Cleanup joins resources; check() retains the already-reported failure. + proxy.finish().await?; + assert!(proxy.tasks.lock().unwrap().is_empty()); + assert!(proxy.executions_since(0).is_err()); + Ok(()) +} diff --git a/crates/qbit-prism-server/tests/support/live_alert_rules.rs b/crates/qbit-prism-server/tests/support/live_alert_rules.rs new file mode 100644 index 000000000..a15d46fa0 --- /dev/null +++ b/crates/qbit-prism-server/tests/support/live_alert_rules.rs @@ -0,0 +1,280 @@ +//! The rendered PRISM alert rules, evaluated on live `/metrics` scrapes +//! (#575). A scenario mirrors a rule's condition in Rust, pins the parts of +//! the rule's expression it mirrors (so a changed rule fails the scenario +//! instead of silently diverging), and asks whether the condition held on +//! every scrape for at least the rule's `for` duration: whether Prometheus, +//! scraping as often, would have fired it. +use super::*; +use std::collections::{BTreeMap, BTreeSet}; + +/// The rules the deployment renders. +const ALERT_RULES: &str = include_str!("../../../../docs/prism-native-alert-rules.json"); + +/// One scrape of a server's `/metrics`: every sample, keyed by its series as +/// rendered (`name` or `name{labels}`). +#[derive(Clone, Debug)] +pub(crate) struct Scrape { + pub server: usize, + pub at: Instant, + series: BTreeMap, +} + +impl Scrape { + pub(crate) fn parse(server: usize, body: &str) -> Self { + let series = body + .lines() + .filter(|line| !line.starts_with('#')) + .filter_map(|line| { + let (series, value) = line.rsplit_once(' ')?; + Some((series.to_owned(), value.parse().ok()?)) + }) + .collect(); + Self { + server, + at: Instant::now(), + series, + } + } + + /// Every series of family `name`, with its value. + fn family<'a>(&'a self, name: &'a str) -> impl Iterator + 'a { + self.series.iter().filter_map(move |(series, value)| { + let rest = series.strip_prefix(name)?; + (rest.is_empty() || rest.starts_with('{')).then_some((rest, *value)) + }) + } + + /// The first sample of family `name`: an unlabelled gauge's value. + pub(crate) fn value(&self, name: &str) -> Option { + self.family(name).next().map(|(_, value)| value) + } + + /// The sample of family `name` whose `label` is `value`. + pub(crate) fn labelled(&self, name: &str, label: &str, value: &str) -> Option { + let needle = format!("{label}=\"{value}\""); + self.family(name) + .find(|(labels, _)| labels.contains(&needle)) + .map(|(_, sample)| sample) + } + + /// Family `name` summed by `label`. + pub(crate) fn by_label(&self, name: &str, label: &str) -> BTreeMap { + let key = format!("{label}=\""); + let mut sums = BTreeMap::new(); + for (labels, sample) in self.family(name) { + let Some(value) = labels + .split_once(&key) + .and_then(|(_, rest)| rest.split_once('"')) + .map(|(value, _)| value.to_owned()) + else { + continue; + }; + *sums.entry(value).or_default() += sample; + } + sums + } + + /// Family `name` summed over every series. + pub(crate) fn sum(&self, name: &str) -> f64 { + self.family(name).map(|(_, value)| value).sum() + } + + /// The snapshot gate every PRISM rule here is `and`-ed with: a complete + /// metrics snapshot that is not stale. + pub(crate) fn gate_open(&self) -> bool { + self.value("qbit_prism_metrics_snapshot_available") == Some(1.0) + && self.value("qbit_prism_metrics_snapshot_stale") == Some(0.0) + } +} + +pub(crate) async fn scrape(client: &reqwest::Client, server: usize, port: u16) -> Result { + let body = client + .get(format!("http://127.0.0.1:{port}/metrics")) + .send() + .await? + .error_for_status()? + .text() + .await?; + Ok(Scrape::parse(server, &body)) +} + +/// A rendered rule: its expression and `for` duration. +pub(crate) struct Rule { + pub title: &'static str, + pub expr: String, + pub hold: Duration, +} + +/// The rule titled `title`, whose expression must still contain every one +/// of `fragments`: the parts a scenario's mirror of it depends on. +pub(crate) fn rule(title: &'static str, fragments: &[&str]) -> Result { + let rules: Value = serde_json::from_str(ALERT_RULES)?; + let found = rules["rules"] + .as_array() + .context("alert rules missing")? + .iter() + .find(|rule| rule["title"] == title) + .with_context(|| format!("{title} rule missing"))?; + let expr = found["expr"] + .as_str() + .context("rule has no expr")? + .to_owned(); + for fragment in fragments { + ensure!( + expr.contains(fragment), + "{title} no longer contains {fragment:?}; update the scenario's mirror of it: {expr}" + ); + } + let text = found["for"].as_str().context("rule has no for")?; + let (number, unit) = text.split_at(text.len() - 1); + let number: u64 = number.parse()?; + let hold = Duration::from_secs(match unit { + "s" => number, + "m" => number * 60, + "h" => number * 3600, + _ => bail!("{title}: unsupported for {text:?}"), + }); + Ok(Rule { title, expr, hold }) +} + +impl Rule { + /// The alternatives of the expression's `label=~"a|b"` matcher. + pub(crate) fn label_alternatives(&self, label: &str) -> Result> { + let matcher = format!("{label}=~\""); + let (_, rest) = self + .expr + .split_once(&matcher) + .with_context(|| format!("{} has no {label} matcher", self.title))?; + let (alternatives, _) = rest.split_once('"').context("unterminated matcher")?; + Ok(alternatives.split('|').map(str::to_owned).collect()) + } +} + +/// The `[5m]` range of the rules' `increase`. +const INCREASE_WINDOW: Duration = Duration::from_secs(300); + +/// A mirrored condition: `holds(base, scrape)`, where `base` is the scrape +/// an `increase(...[5m])` at `scrape` starts from. +type Condition = Box bool + Send + Sync>; + +/// A rule and its condition mirrored in Rust. +pub(crate) struct Mirror { + pub rule: Rule, + holds: Condition, +} + +impl Mirror { + /// The rule titled `title`, pinned by `fragments` (see [`rule`]). + pub(crate) fn new( + title: &'static str, + fragments: &[&str], + holds: impl Fn(&Scrape, &Scrape) -> bool + Send + Sync + 'static, + ) -> Result { + Ok(Self { + rule: rule(title, fragments)?, + holds: Box::new(holds), + }) + } + + /// The longest time the condition held on consecutive scrapes of one + /// server's `series`, oldest first, from its first scrape to its last. + /// An increase is measured from the last scrape at least five minutes + /// older, or from the series' first while there is none. + pub(crate) fn held(&self, series: &[&Scrape]) -> Duration { + let Some(first) = series.first() else { + return Duration::ZERO; + }; + let base = |scrape: &Scrape| { + series + .iter() + .take_while(|older| older.at + INCREASE_WINDOW <= scrape.at) + .last() + .unwrap_or(first) + }; + let mut longest = Duration::ZERO; + let mut since = None; + for scrape in series { + if (self.holds)(base(scrape), scrape) { + let start = *since.get_or_insert(scrape.at); + longest = longest.max(scrape.at - start); + } else { + since = None; + } + } + longest + } +} + +/// `PrismMetricsSnapshotStale`, which every scenario here watches. +pub(crate) fn snapshot_stale() -> Result { + Mirror::new( + "PrismMetricsSnapshotStale", + &["qbit_prism_metrics_snapshot_stale", "> bool 0"], + |_, s| { + s.value("qbit_prism_metrics_snapshot_stale") + .is_some_and(|stale| stale > 0.0) + }, + ) +} + +/// Which `mirrors` fire on each of `servers`, and a report of how long each +/// held: a rule fires when its condition held for at least its `for`. +pub(crate) struct Verdict { + pub fired: BTreeMap>, + pub report: String, +} + +impl Verdict { + pub(crate) fn of(scrapes: &[Scrape], servers: &[usize], mirrors: &[Mirror]) -> Self { + let mut fired: BTreeMap> = servers + .iter() + .map(|server| (*server, BTreeSet::new())) + .collect(); + let mut lines = Vec::new(); + for mirror in mirrors { + let mut spans = Vec::new(); + for server in servers { + let series: Vec<_> = scrapes.iter().filter(|s| s.server == *server).collect(); + let held = mirror.held(&series); + let fires = held >= mirror.rule.hold; + if fires { + fired.entry(*server).or_default().insert(mirror.rule.title); + } + spans.push(format!( + "server-{server} {:.0}s{}", + held.as_secs_f64(), + if fires { " FIRES" } else { "" } + )); + } + lines.push(format!( + "{} (for {:?}): {}", + mirror.rule.title, + mirror.rule.hold, + spans.join(", ") + )); + } + Self { + fired, + report: lines.join("\n"), + } + } +} + +/// Scrape every `(server, port)` every quarter second for `duration`. +pub(crate) async fn sample( + client: &reqwest::Client, + ports: &[(usize, u16)], + duration: Duration, +) -> Vec { + let started = Instant::now(); + let mut scrapes = Vec::new(); + while started.elapsed() < duration { + for (server, port) in ports { + if let Ok(scrape) = scrape(client, *server, *port).await { + scrapes.push(scrape); + } + } + tokio::time::sleep(Duration::from_millis(250)).await; + } + scrapes +} diff --git a/crates/qbit-prism-server/tests/support/live_chain_events_load.rs b/crates/qbit-prism-server/tests/support/live_chain_events_load.rs new file mode 100644 index 000000000..d0e644b15 --- /dev/null +++ b/crates/qbit-prism-server/tests/support/live_chain_events_load.rs @@ -0,0 +1,914 @@ +//! #553 (#487 S15): chain events on the real node under session load, L4. +//! +//! #521's fixtures run here with their assertions unchanged (#487 decision +//! 12): each case opens the fixture as its own test does, starts +//! [`SessionLoad`] against the fixture's frontends, and calls the scenario's +//! own function. The servers run with [`load_server_env`], so the load's +//! shares can miss the regtest block target. The load adds its own checks +//! (see `live_session_load.rs`) and reports #481's time to usable work per +//! frontend. Scenario 2 pauses the load's shares, not its sessions, from its +//! seed blocks until branch B's own blocks are paid, since its balance +//! comparison needs every own share in the window (`deep_reorg`); its reorg +//! runs under the whole load. Scenarios 1, 2 and 4 run at 2,000 sessions, +//! D1's session count, as does scenario 5's soak; scenario 7 at the real-node +//! wallet cap of 2,000 payees (#487 decision 7), whose workers are its load, +//! waits on #621 and #622 and runs in no lane yet. +//! +//! #553 adds three scenarios, with the pass criteria set here: +//! - two frontends on two nodes that briefly disagree about the tip; +//! - the pool's node losing its only peer while miners are connected, which +//! regtest, exempt from the peer floor, shows as an isolated node; +//! - external tips at qbit's 60 s target spacing, with the time to usable +//! work per frontend. +//! +//! None of these runs per PR. The nightly set (test/prism-nightly-gated-tests.txt) +//! is scenarios 1 and 2, the crash case of scenario 4, and the three new +//! scenarios with 5 minutes of external tips, all at 100 shares/s, and #598's +//! guard, 2,000 sessions on one frontend at the fixture's 1 s reanchor. The +//! weekly set (test/prism-weekly-gated-tests.txt) is the other scenario-4 +//! cases, the soak and 45 minutes of external tips at 400 shares/s, +//! mainnet-floor's peak. +//! +//! For a local run, `PRISM_L4_SESSIONS` and `PRISM_L4_SHARE_RATE` override a +//! case's load, `PRISM_L4_REANCHOR_SECONDS` the frontends' reanchor, and +//! `PRISM_L4_SERVER_RUST_LOG` their log filter; `PRISM_L4_LOG_DIR` keeps the +//! processes' logs there. +use super::dense_soak_tests::{setting, soak}; +use super::node_outage_tests::{outage_case_on, Fault, Hold}; +use super::session_load::{ + load_server_env, load_settings, under_load, LoadPlan, SessionLoad, DELIVERY_BOUND, +}; +use super::two_node_tests::{ + assert_no_duplicate_headers, chain_state, credits, deep_reorg, finish, lost_race, node_a_best, + node_a_mine, own_block, server_ready, start_frontend, LoadShares, PeerNode, LOAD_SHARES, + READY_GATE, +}; +use super::weighted_recipients_tests::{weighted_case, Scenario}; +use super::*; +use rand::{rngs::StdRng, Rng, SeedableRng}; + +/// D1's session count, which L4 runs every scenario at. +const SESSIONS: usize = 2_000; +/// The nightly set's offered rate: 2,000 sessions each sharing every 20 s. +const NIGHTLY_RATE: f64 = 100.0; +/// The weekly set's: mainnet-floor's peak (#521's shapes; mainnet's worst +/// five-minute rate was 394 shares/s). +const WEEKLY_RATE: f64 = 400.0; +/// The #598 guard's offered rate, #598's reproducer. Before the fix, at 5 +/// shares/s the 2nd and 3rd tips reached almost no session in every debug +/// run; at 100 some runs left only ten sessions without work, and one none. +const STARVATION_RATE: f64 = 5.0; +/// qbit's target spacing. +const TARGET_SPACING_SECONDS: f64 = 60.0; +/// How long a frontend may take to report itself ready again after a +/// disagreement. The servers poll their node every 0.2 s +/// (`PRISM_BLOCKPOLL_SECONDS`). +const READINESS_BOUND: u64 = 30; +/// How long a disagreement or a partition is held: past the delivery bound, +/// so a tip on either side is one the load's check holds to it. +const HOLD: Duration = Duration::from_secs(20); + +/// #521 scenario 7 at the real-node wallet cap: 2,000 payees through six +/// blocks. The shape is the Zipf tail without a whale: a whale near 85%, as +/// mainnet-floor has, leaves all but about 160 of 2,000 below the payout +/// floor (measured), which the scenario's own check refuses as degenerate; +/// without one about 600 are paid, and carried dust brings more across. +const WALLET_CAP: Scenario = Scenario { + name: "wallet-cap-2000", + seed: 0x0553_2000, + payees: 2_000, + blocks: 6, + whales: &[], + near_zero: 40, + ctv: true, + wallets: 8, +}; + +/// Open the fixture without servers, with the load's server settings. +async fn open(ctv: bool, sessions: usize) -> Result { + gate::required_inputs( + gate::site!(), + &[gate::Input::QbitdBin, gate::Input::DatabaseUrl], + )?; + let Some(mut fixture) = Fixture::open_with_servers(ctv, false).await? else { + bail!("the live fixture's inputs were required above"); + }; + fixture.server_env = load_server_env(sessions); + if let Ok(seconds) = std::env::var("PRISM_L4_REANCHOR_SECONDS") { + fixture + .server_env + .push(("PRISM_PAYOUT_ARTIFACT_REANCHOR_SECONDS".into(), seconds)); + } + if let Ok(filter) = std::env::var("PRISM_L4_SERVER_RUST_LOG") { + fixture.server_env.push(("RUST_LOG".into(), filter)); + } + Ok(fixture) +} + +/// Copy the fixture's process logs to `PRISM_L4_LOG_DIR`, when it is set, +/// before cleanup removes them: a local diagnosis aid. +fn keep_logs(fixture: &Fixture) { + let Ok(target) = std::env::var("PRISM_L4_LOG_DIR") else { + return; + }; + let target = std::path::Path::new(&target).join(fixture.schema.as_str()); + if std::fs::create_dir_all(&target).is_err() { + return; + } + if let Ok(entries) = std::fs::read_dir(fixture.directory.path()) { + for entry in entries.flatten() { + if entry + .path() + .extension() + .is_some_and(|extension| extension == "log") + { + let _ = std::fs::copy(entry.path(), target.join(entry.file_name())); + } + } + } + eprintln!("live-load: logs kept in {}", target.display()); +} + +/// The load's own payout identity: a wallet address no scenario pays or +/// reads, so no scenario's balances include it. +async fn load_address(fixture: &Fixture) -> Result { + Ok(fixture + .rpc("getnewaddress", json!(["load", "p2mr"])) + .await? + .as_str() + .context("load address missing")? + .to_owned()) +} + +/// While alive, #521's `server_ready` waits after each server turns ready +/// until 99% of the load's sessions hold work: scenarios 1 and 2 start their +/// own servers, and their first events follow within seconds. +struct ReadyGateGuard; + +impl ReadyGateGuard { + fn install(load: &SessionLoad) -> Result { + *READY_GATE + .lock() + .map_err(|_| anyhow::anyhow!("ready gate poisoned"))? = Some(load.holding(0.99)); + Ok(Self) + } +} + +impl Drop for ReadyGateGuard { + fn drop(&mut self) { + if let Ok(mut gate) = READY_GATE.lock() { + *gate = None; + } + } +} + +/// While alive, #521 scenario 2 can pause the load's shares around its own +/// blocks (see `deep_reorg`): a load share weighs about half an own block on +/// regtest. +struct LoadSharesGuard; + +impl LoadSharesGuard { + fn install(load: &SessionLoad) -> Result { + *LOAD_SHARES + .lock() + .map_err(|_| anyhow::anyhow!("load shares poisoned"))? = Some(LoadShares { + pause: load.share_pause(), + answered: load.answered(), + }); + Ok(Self) + } +} + +impl Drop for LoadSharesGuard { + fn drop(&mut self) { + if let Ok(mut shares) = LOAD_SHARES.lock() { + *shares = None; + } + } +} + +/// After a #521 scenario, one more tip on the fixture's node and the +/// delivery bound, so the load's check holds every frontend still running to +/// at least one tip: a scenario often ends seconds after its last one. +async fn closing_tip(fixture: &Fixture) -> Result<()> { + let address = load_address(fixture).await?; + fixture + .rpc("generatetoaddress", json!([1, address])) + .await?; + tokio::time::sleep(DELIVERY_BOUND + Duration::from_secs(1)).await; + Ok(()) +} + +/// The load report's delivery of `tip` on `frontend`. +fn tip_delivery<'a>(report: &'a Value, frontend: usize, tip: &str) -> Result<&'a Value> { + report["tips"] + .as_array() + .context("no tips in the report")? + .iter() + .find(|delivery| delivery["frontend"] == frontend && delivery["tip"] == tip) + .with_context(|| format!("no delivery of {tip} on frontend {frontend}: {report}")) +} + +/// Every frontend in `frontends` had at least one tip held to the bound: +/// gated, and lasting the bound, so no session could count as replaced. +fn held_to_the_bound(report: &Value, frontends: &[usize]) -> Result<()> { + for frontend in frontends { + ensure!( + report["time_to_usable_work"][frontend]["held_tips"].as_u64() >= Some(1), + "the load's check held no tip of frontend {frontend} to the bound: {report}" + ); + } + Ok(()) +} + +/// A load on the two-node fixture: node A (the fixture's) is watched node 0 +/// and node B watched node 1. +async fn two_node_plan( + fixture: &Fixture, + peer: &PeerNode, + name: &str, + sessions: usize, + rate: f64, + node_of: [Option; 2], +) -> Result { + let mut plan = + LoadPlan::on_fixture(fixture, name, sessions, rate, load_address(fixture).await?); + plan.nodes = vec![fixture.rpc_port, peer.rpc_port]; + plan.node_of = node_of.to_vec(); + Ok(plan) +} + +/// Run `case` against a fixture with a second node, then report both nodes +/// on failure and clean up. +async fn two_node_case(ctv: bool, sessions: usize, case: F) -> Result<()> +where + F: AsyncFnOnce(&mut Fixture, &PeerNode) -> Result<()>, +{ + let mut fixture = open(ctv, sessions).await?; + let peer = PeerNode::start(&fixture).await; + let result = match &peer { + Ok(peer) => case(&mut fixture, peer).await, + Err(error) => Err(anyhow::anyhow!("{error:#}")), + }; + keep_logs(&fixture); + finish(fixture, peer, result).await +} + +/// Run `case` against a one-node fixture and clean up. +async fn one_node_case(ctv: bool, sessions: usize, case: F) -> Result<()> +where + F: AsyncFnOnce(&mut Fixture) -> Result<()>, +{ + let mut fixture = open(ctv, sessions).await?; + let result = case(&mut fixture).await; + keep_logs(&fixture); + if result.is_err() { + eprintln!("{}", fixture.diagnostics()); + } + let cleanup = fixture.cleanup().await; + result.and(cleanup) +} + +// --------------------------------------------------------------------------- +// #521's scenarios, unchanged, under load. + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +#[ignore = "#553 L4 nightly: #521 scenario 1 under 2,000 sessions"] +async fn nightly_lost_race_under_2000_sessions_orphans_once_and_serves_every_session() -> Result<()> +{ + let (sessions, rate) = load_settings(SESSIONS, NIGHTLY_RATE)?; + two_node_case(false, sessions, async |fixture, peer| { + let plan = two_node_plan( + fixture, + peer, + "scenario-1-lost-race", + sessions, + rate, + [Some(0), Some(0)], + ) + .await?; + let report = under_load(fixture, plan, async |fixture, load| { + let _gate = ReadyGateGuard::install(load)?; + lost_race(fixture, peer).await?; + closing_tip(fixture).await + }) + .await?; + held_to_the_bound(&report, &[0]) + }) + .await +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +#[ignore = "#553 L4 nightly: #521 scenario 2 under 2,000 sessions"] +async fn nightly_deep_reorg_under_2000_sessions_keeps_balances_exact_and_serves_every_session( +) -> Result<()> { + let (sessions, rate) = load_settings(SESSIONS, NIGHTLY_RATE)?; + two_node_case(true, sessions, async |fixture, peer| { + // Frontend 1 is started against node B by the scenario. + let plan = two_node_plan( + fixture, + peer, + "scenario-2-deep-reorg", + sessions, + rate, + [Some(0), Some(1)], + ) + .await?; + let report = under_load(fixture, plan, async |fixture, load| { + let _gate = ReadyGateGuard::install(load)?; + let _shares = LoadSharesGuard::install(load)?; + deep_reorg(fixture, peer).await?; + closing_tip(fixture).await + }) + .await?; + // The scenario stops frontend 0 midway; its sessions fail over to + // frontend 1 and stay there, so frontend 0 serves none at the end. + held_to_the_bound(&report, &[1]) + }) + .await +} + +async fn outage_under_load(name: &str, fault: Fault, hold: Hold, rate: f64) -> Result<()> { + let (sessions, rate) = load_settings(SESSIONS, rate)?; + one_node_case(false, sessions, async |fixture| { + let plan = + LoadPlan::on_fixture(fixture, name, sessions, rate, load_address(fixture).await?); + let report = under_load(fixture, plan, async |fixture, load| { + outage_case_on( + fixture, + fault, + hold, + false, + // The case starts the servers; the fault waits for the load. + async |_| load.until_connected(0.99, 120).await, + async |fixture| closing_tip(fixture).await, + ) + .await + }) + .await?; + held_to_the_bound(&report, &[0, 1]) + }) + .await +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +#[ignore = "#553 L4 nightly: #521 scenario 4's crash after acceptance under 2,000 sessions"] +async fn nightly_node_killed_after_accepting_under_2000_sessions_reconciles_the_lost_block_once( +) -> Result<()> { + outage_under_load( + "scenario-4-killed-after-accepting", + Fault::Kill, + Hold::Reply, + NIGHTLY_RATE, + ) + .await +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +#[ignore = "#553 L4 weekly: #521 scenario 4 under 2,000 sessions"] +async fn weekly_node_stopped_before_submitblock_under_2000_sessions_lands_the_block_once( +) -> Result<()> { + outage_under_load( + "scenario-4-stopped-before-submitblock", + Fault::Stop, + Hold::Request, + WEEKLY_RATE, + ) + .await +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +#[ignore = "#553 L4 weekly: #521 scenario 4 under 2,000 sessions"] +async fn weekly_node_stopped_after_accepting_under_2000_sessions_lands_the_block_once() -> Result<()> +{ + outage_under_load( + "scenario-4-stopped-after-accepting", + Fault::Stop, + Hold::Reply, + WEEKLY_RATE, + ) + .await +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +#[ignore = "#553 L4 weekly: #521 scenario 4 under 2,000 sessions"] +async fn weekly_node_killed_before_submitblock_under_2000_sessions_lands_the_block_once( +) -> Result<()> { + outage_under_load( + "scenario-4-killed-before-submitblock", + Fault::Kill, + Hold::Reservation, + WEEKLY_RATE, + ) + .await +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +#[ignore = "#553 L4 weekly: #521 scenario 4 under 2,000 sessions"] +async fn weekly_node_restarted_into_warmup_under_2000_sessions_lands_the_block_once() -> Result<()> +{ + outage_under_load( + "scenario-4-restarted-into-warmup", + Fault::Kill, + Hold::Warmup, + WEEKLY_RATE, + ) + .await +} + +/// #521 scenario 5's soak, for `PRISM_DENSE_SOAK_SECONDS` (default 1,200 +/// here, the top of #521's 10-20 minutes), under 2,000 sessions. Under the +/// share-only settings a soak miner's share is a block about half the time +/// rather than always; the soak's own check that the cadence stayed dense +/// still holds it. +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +#[ignore = "#553 L4 weekly: #521 scenario 5 under 2,000 sessions"] +async fn weekly_dense_soak_under_2000_sessions_lands_every_block_and_holds_rss_flat() -> Result<()> +{ + let (sessions, rate) = load_settings(SESSIONS, WEEKLY_RATE)?; + let seconds: u64 = setting("PRISM_DENSE_SOAK_SECONDS", 1_200)?; + ensure!( + seconds >= 60, + "PRISM_DENSE_SOAK_SECONDS must be at least 60, not {seconds}" + ); + let budget: f64 = setting("PRISM_DENSE_SOAK_REBUILD_P99_SECONDS", 5.0)?; + ensure!( + budget.is_finite() && budget > 0.0, + "PRISM_DENSE_SOAK_REBUILD_P99_SECONDS must be finite and positive" + ); + one_node_case(false, sessions, async |fixture| { + for index in 0..2 { + let process = fixture.start_server(index)?; + fixture.servers.push(process); + } + for index in 0..2 { + server_ready(fixture, index).await?; + } + let plan = LoadPlan::on_fixture( + fixture, + "scenario-5-dense-soak", + sessions, + rate, + load_address(fixture).await?, + ); + let report = under_load(fixture, plan, async |fixture, load| { + load.until_connected(0.99, 120).await?; + soak(fixture, seconds, budget).await?; + // The soak's own blocks come seconds apart, under the bound. + closing_tip(fixture).await + }) + .await?; + held_to_the_bound(&report, &[0, 1]) + }) + .await +} + +/// #521 scenario 7 at the real-node wallet cap (#487 decision 7), expected +/// to fail on #621 until it is fixed. Its ~3,000 worker sessions, about +/// 1,500 per frontend, are the load, and the fixture opens them one at a +/// time. Since #604's rebuild lane every session opens, but at the fixture's +/// 1 s reanchor each debug frontend builds far fewer jobs per second than it +/// has sessions, so every session always has a rebuild queued, and a session +/// answers a submit only after its queued rebuild completes, which outlasts +/// the fixture's 30 s wait for a submit's reply (#621). At the production +/// 60 s every economic assertion passes, but the fixture's rounds rely on a +/// fast reanchor to put each round into its block's window, so the +/// scenario's own degeneracy check fails there. The case passes while +/// submits wait behind rebuilds; once #621 is fixed it runs the scenario +/// unchanged and says so, and then the expectation goes. A worker that never +/// gets its first job is #604 again, and fails the case. On a slower host +/// the rounds can instead outlast the fixture's mock-clock horizon (qbitd's +/// mock time is set about 24 minutes ahead; past it the template ages beyond +/// 120 s and a share is refused because the current chain state is +/// unavailable), which also fails the case: that is the fixture running out +/// of time, not a new finding. +/// +/// In no lane: that holds in debug, but in release (the weekly job's build) +/// a later round's new tip reaches some worker late, with `tip polling +/// stale` deferrals (#622). Run it by hand until both are fixed. +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +#[ignore = "#553 L4, by hand: #521 scenario 7 at 2,000 wallets, #621's expected failure (#622 in release)"] +async fn wallets_2000_share_to_spend_waits_on_submit_acks_until_621_is_fixed() -> Result<()> { + gate::required_inputs( + gate::site!(), + &[gate::Input::QbitdBin, gate::Input::DatabaseUrl], + )?; + match weighted_case(&WALLET_CAP).await { + Err(error) if format!("{error:#}").contains("got no reply to a submit") => { + eprintln!("#621 reproduced at 2,000 wallets: {error:#}"); + Ok(()) + } + Err(error) => { + Err(error.context("the 2,000-wallet case failed for a reason other than #621")) + } + Ok(()) => bail!( + "#621 looks fixed: #521 scenario 7 passed at 2,000 wallets; drop this expectation and, \ + once #622 is fixed too, list the case weekly" + ), + } +} + +/// #598's regression guard: 2,000 sessions on one frontend at the live +/// fixture's 1 s reanchor. Before #598's fix each session's job build was +/// superseded by the next publication before it completed, so a new tip +/// reached almost no session; now every one of three external tips, 20 s +/// apart, must reach every session within the delivery bound, with the +/// load's other checks. Nightly, in debug: in release the fan-out outruns +/// the reanchor and even the unfixed server passes. +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +#[ignore = "#553 L4 nightly: #598's guard, 2,000 sessions on one frontend at a 1 s reanchor"] +async fn nightly_2000_sessions_on_one_frontend_at_a_1s_reanchor_get_work_on_every_tip() -> Result<()> +{ + let (sessions, rate) = load_settings(SESSIONS, STARVATION_RATE)?; + two_node_case(false, sessions, async |fixture, peer| { + fixture + .server_env + .push(("PRISM_PAYOUT_ARTIFACT_REANCHOR_SECONDS".into(), "1".into())); + peer.heal(fixture).await?; + fixture.servers.push(fixture.start_server(0)?); + server_ready(fixture, 0).await?; + let mut plan = two_node_plan( + fixture, + peer, + "one-frontend-1s-reanchor", + sessions, + rate, + [None, None], + ) + .await?; + plan.stratum.truncate(1); + plan.api.truncate(1); + plan.node_of = vec![Some(0)]; + let report = under_load(fixture, plan, async |fixture, load| { + load.until_connected(0.99, 120).await?; + for _ in 0..3 { + external_tip(fixture, peer).await?; + tokio::time::sleep(DELIVERY_BOUND + Duration::from_secs(5)).await; + } + Ok(()) + }) + .await?; + ensure!( + report["time_to_usable_work"][0]["held_tips"].as_u64() >= Some(3), + "the load's check held too few of the three tips to the bound: {report}" + ); + Ok(()) + }) + .await +} + +// --------------------------------------------------------------------------- +// #553's scenarios. + +async fn health(fixture: &Fixture, index: usize) -> Result<(bool, Value)> { + let response = fixture + .client + .get(format!("http://127.0.0.1:{}/healthz", fixture.api[index])) + .timeout(Duration::from_secs(5)) + .send() + .await?; + let ok = response.status().is_success(); + Ok((ok, response.json().await.unwrap_or(Value::Null))) +} + +async fn until_health( + fixture: &Fixture, + index: usize, + ready: bool, + label: &str, +) -> Result { + let started = Instant::now(); + until( + &format!("frontend {index} {label}"), + READINESS_BOUND, + || async { + Ok(health(fixture, index) + .await + .is_ok_and(|(ok, _)| ok == ready)) + }, + ) + .await?; + Ok(started.elapsed()) +} + +/// Mine one block on node B and wait until node A holds it. +async fn external_tip(fixture: &Fixture, peer: &PeerNode) -> Result { + let tip = peer.mine(1, &fixture.address).await?.remove(0); + until("node A on node B's tip", 30, || async { + Ok(node_a_best(fixture).await? == tip) + }) + .await?; + Ok(tip) +} + +async fn cluster_tip(fixture: &Fixture) -> Result> { + Ok( + sqlx::query_scalar("SELECT best_tip_hash FROM qbit_prism_cluster WHERE singleton") + .fetch_one(&fixture.pool) + .await?, + ) +} + +async fn payout_divergences(fixture: &Fixture) -> Result { + Ok( + sqlx::query_scalar("SELECT count(*) FROM qbit_prism_payout_divergences") + .fetch_one(&fixture.pool) + .await?, + ) +} + +/// Both frontends on node A, node B its only peer, and B then partitioned +/// away while 2,000 sessions mine. On regtest the peer floor +/// (`PRISM_MIN_PEERS`) does not apply (`readiness.rs`; pinned by +/// `readiness_rpc.rs`'s `regtest_needs_no_peers_but_rejects_explicit_header_lag`, +/// and the floor itself by its public-chain tests), so what a regtest node +/// can show is the rest: the pool keeps mining on its isolated node while +/// the network moves on without it. B mines two blocks it cannot relay; the +/// pool finds an own block on A's stale tip, which becomes A's tip. When the +/// peer comes back, A reorganizes onto B's longer chain. The own block must +/// end inactive and credited exactly once, with no payout divergence, +/// duplicate header or integrity drift. Every session must have work on the +/// isolated own block and on B's chain after the heal within the delivery +/// bound, and the load's own checks hold throughout. +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +#[ignore = "#553 L4 nightly: the pool's node loses its peer under 2,000 sessions"] +async fn nightly_node_loses_its_peer_under_2000_sessions_orphans_the_isolated_block_and_recovers( +) -> Result<()> { + let (sessions, rate) = load_settings(SESSIONS, NIGHTLY_RATE)?; + two_node_case(false, sessions, async |fixture, peer| { + peer.heal(fixture).await?; + for index in 0..2 { + let process = fixture.start_server(index)?; + fixture.servers.push(process); + } + for index in 0..2 { + server_ready(fixture, index).await?; + } + let plan = + two_node_plan(fixture, peer, "node-loses-peer", sessions, rate, [Some(0), Some(0)]) + .await?; + let mut isolated_tip: Option = None; + let report = under_load(fixture, plan, async |fixture, load| { + load.until_connected(0.99, 120).await?; + let stale = external_tip(fixture, peer).await?; + tokio::time::sleep(DELIVERY_BOUND + Duration::from_secs(1)).await; + + peer.partition(fixture).await?; + let unseen = peer.mine(2, &fixture.address).await?; + let miner = format!("{}.highdiff", fixture.address); + let isolated = own_block(fixture, 0, &miner, &stale, 1).await?; + ensure!( + node_a_best(fixture).await? == isolated, + "the isolated own block is not node A's tip" + ); + isolated_tip = Some(isolated.clone()); + tokio::time::sleep(HOLD).await; + let ready_while_isolated = [ + health(fixture, 0).await?.0, + health(fixture, 1).await?.0, + ]; + + let healed = peer.heal(fixture).await?; + ensure!( + Some(&healed) == unseen.last(), + "node A did not reorganize onto node B's chain: {healed}" + ); + until("the isolated own block deactivated", 60, || async { + Ok(chain_state(fixture, &isolated).await?.as_deref() == Some("inactive")) + }) + .await?; + ensure!( + credits(fixture, &miner, &isolated).await? == 1, + "the isolated own block is not credited exactly once" + ); + tokio::time::sleep(DELIVERY_BOUND + Duration::from_secs(1)).await; + load.until_connected(0.99, 60).await?; + fixture.integrity().await?; + ensure!( + payout_divergences(fixture).await? == 0, + "the peer loss recorded a payout divergence" + ); + assert_no_duplicate_headers(fixture).await?; + eprintln!( + "node-loses-peer: own block {isolated} on the isolated node, deactivated when node A \ + reorganized onto {healed}; frontends ready while isolated: {ready_while_isolated:?}" + ); + Ok(()) + }) + .await?; + let isolated = isolated_tip.context("the isolated block was not recorded")?; + for frontend in 0..2 { + ensure!( + report["time_to_usable_work"][frontend]["gated_tips"].as_u64() >= Some(2), + "the load's check held too few of frontend {frontend}'s tips to the bound: {report}" + ); + // The isolated block is the case: beyond the count above, it + // must be gated and reach every eligible session in the bound. + let delivery = tip_delivery(&report, frontend, &isolated)?; + ensure!( + delivery["gated"] == true + && delivery["eligible"].as_u64() > Some(0) + && delivery["served"] == delivery["eligible"] + && delivery["all_sessions_s"] + .as_f64() + .is_some_and(|seconds| seconds <= DELIVERY_BOUND.as_secs_f64()), + "frontend {frontend} did not give every session work on the isolated block within \ + {DELIVERY_BOUND:?}: {delivery}" + ); + } + Ok(()) + }) + .await +} + +/// Frontend 0 on node A and frontend 1 on node B, 1,000 sessions each. Every +/// tip of B's reaches A only after a propagation delay, so the frontends +/// briefly disagree three times; then a partition makes them disagree for +/// [`HOLD`]: A and B each mine a block at the same height, two tips of equal +/// work. The ledger's chain view never rolls back and refuses a conflicting +/// equal-work view (`Ledger::observe_chain_view`), so exactly one of the two +/// tips becomes the cluster's; the frontend whose node holds the other must +/// report itself unready and give its sessions no work on that tip, and no +/// payout divergence or duplicate header may follow. B then outgrows the +/// split and the partition heals: both nodes and both frontends converge on +/// B's branch, the cluster takes its tip, and every session on both +/// frontends gets work on it within the delivery bound. +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +#[ignore = "#553 L4 nightly: two frontends on two nodes disagree about the tip under 2,000 sessions"] +async fn nightly_two_frontends_on_two_nodes_disagree_briefly_and_converge_under_2000_sessions( +) -> Result<()> { + let (sessions, rate) = load_settings(SESSIONS, NIGHTLY_RATE)?; + two_node_case(false, sessions, async |fixture, peer| { + peer.heal(fixture).await?; + fixture.servers.push(fixture.start_server(0)?); + fixture.servers.push(start_frontend(fixture, 1, peer.rpc_port)?); + for index in 0..2 { + server_ready(fixture, index).await?; + } + let plan = two_node_plan( + fixture, + peer, + "two-frontends-disagree", + sessions, + rate, + [Some(0), Some(1)], + ) + .await?; + // (the split's losing tip, its frontend, the converged tip) + let mut split: Option<(String, usize, String)> = None; + let report = under_load(fixture, plan, async |fixture, load| { + load.until_connected(0.99, 120).await?; + for _ in 0..3 { + external_tip(fixture, peer).await?; + tokio::time::sleep(DELIVERY_BOUND + Duration::from_secs(1)).await; + } + + peer.partition(fixture).await?; + // Another payee than node A's, or the two blocks of one second + // on one parent are the same block. + let other = load_address(fixture).await?; + let tip_a = node_a_mine(fixture, 1).await?.remove(0); + let tip_b = peer.mine(1, &other).await?.remove(0); + ensure!(tip_a != tip_b, "the split did not fork"); + tokio::time::sleep(HOLD).await; + // The first observer's tip is the cluster's; the other frontend's + // node follows a conflicting equal-work tip, which the ledger + // refuses: that frontend reports itself unready and gives no work + // on its node's tip, while the first stays ready. + let cluster = cluster_tip(fixture).await?; + let (accepted, refused, loser) = if cluster.as_deref() == Some(tip_a.as_str()) { + (0, tip_b.clone(), 1) + } else if cluster.as_deref() == Some(tip_b.as_str()) { + (1, tip_a.clone(), 0) + } else { + bail!("the cluster's tip {cluster:?} is neither side of the split ({tip_a}, {tip_b})"); + }; + let during = [health(fixture, 0).await?, health(fixture, 1).await?]; + ensure!( + during[accepted].0 && !during[loser].0, + "during the split frontend {accepted} (the cluster's tip) must be ready and frontend \ + {loser} (the refused tip) unready: {during:?}" + ); + + let winner = peer.mine(1, &other).await?.remove(0); + let healed = peer.heal(fixture).await?; + ensure!( + healed == winner, + "the nodes converged on {healed}, not node B's longer branch {winner}" + ); + for index in 0..2 { + until_health(fixture, index, true, "ready after convergence").await?; + } + until("the cluster on the converged tip", 30, || async { + Ok(cluster_tip(fixture).await?.as_deref() == Some(winner.as_str())) + }) + .await?; + tokio::time::sleep(DELIVERY_BOUND + Duration::from_secs(1)).await; + load.until_connected(0.99, 60).await?; + fixture.integrity().await?; + ensure!( + payout_divergences(fixture).await? == 0, + "the disagreement recorded a payout divergence" + ); + assert_no_duplicate_headers(fixture).await?; + eprintln!( + "two-frontends-disagree: split {tip_a} (A) / {tip_b} (B), the cluster took \ + {cluster:?} and frontend {loser} stood down; converged on {winner}" + ); + split = Some((refused, loser, winner)); + Ok(()) + }) + .await?; + let (refused, loser, winner) = split.context("the split was not recorded")?; + let delivery = |frontend: usize, tip: &str| tip_delivery(&report, frontend, tip); + let refused = delivery(loser, &refused)?; + ensure!( + refused["served"] == 0 && refused["eligible"].as_u64() > Some(0), + "frontend {loser} gave work on the tip the ledger refused: {refused}" + ); + // A frontend that was unready when the converged tip arrived is not + // held to the bound by the load's check; hold it here. + for frontend in 0..2 { + let converged = delivery(frontend, &winner)?; + ensure!( + converged["served"] == converged["eligible"] + && converged["all_sessions_s"] + .as_f64() + .is_some_and(|seconds| seconds <= DELIVERY_BOUND.as_secs_f64()), + "frontend {frontend} did not give every session work on the converged tip within \ + {DELIVERY_BOUND:?}: {converged}" + ); + ensure!( + report["time_to_usable_work"][frontend]["gated_tips"].as_u64() >= Some(3), + "the load's check held too few of frontend {frontend}'s tips to the bound: {report}" + ); + } + Ok(()) + }) + .await +} + +/// External tips from node B at qbit's 60 s target spacing, exponential +/// gaps from a fixed seed, for `seconds`, to 2,000 sessions on two +/// frontends on node A. About one gap in ten is under 6 s, #481's +/// replaced-tip case. Every tip that lasts the delivery bound must reach +/// every session on both frontends within it, and the load's report gives +/// the time to usable work per frontend and per tip (#481's figure; #275 +/// owns a tighter objective, so it is reported, not gated). +async fn external_tips(name: &str, seconds: u64, rate: f64) -> Result<()> { + let (sessions, rate) = load_settings(SESSIONS, rate)?; + let seconds: u64 = setting("PRISM_L4_TIPS_SECONDS", seconds)?; + two_node_case(false, sessions, async |fixture, peer| { + peer.heal(fixture).await?; + for index in 0..2 { + let process = fixture.start_server(index)?; + fixture.servers.push(process); + } + for index in 0..2 { + server_ready(fixture, index).await?; + } + let plan = two_node_plan(fixture, peer, name, sessions, rate, [Some(0), Some(0)]).await?; + let mut minted = 0usize; + let report = under_load(fixture, plan, async |fixture, load| { + load.until_connected(0.99, 120).await?; + let mut rng = StdRng::seed_from_u64(0x0553_0060); + let started = Instant::now(); + loop { + let gap = -TARGET_SPACING_SECONDS * (1.0 - rng.gen::()).ln(); + let at = started.elapsed().as_secs_f64() + gap; + if at >= seconds as f64 { + break; + } + tokio::time::sleep(Duration::from_secs_f64(gap)).await; + external_tip(fixture, peer).await?; + minted += 1; + } + tokio::time::sleep( + Duration::from_secs(seconds).saturating_sub(started.elapsed()) + + DELIVERY_BOUND + + Duration::from_secs(1), + ) + .await; + fixture.integrity().await + }) + .await?; + ensure!(minted > 0, "no external tip in {seconds} s"); + eprintln!("{name}: {minted} external tips in {seconds} s"); + held_to_the_bound(&report, &[0, 1]) + }) + .await +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +#[ignore = "#553 L4 nightly: 5 minutes of external tips under 2,000 sessions"] +async fn nightly_external_tips_at_target_spacing_reach_every_session_on_both_frontends( +) -> Result<()> { + external_tips("external-tips-5m", 300, NIGHTLY_RATE).await +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +#[ignore = "#553 L4 weekly: 45 minutes of external tips under 2,000 sessions"] +async fn weekly_external_tips_at_target_spacing_reach_every_session_on_both_frontends() -> Result<()> +{ + external_tips("external-tips-45m", 2_700, WEEKLY_RATE).await +} diff --git a/crates/qbit-prism-server/tests/support/live_clock_jump.rs b/crates/qbit-prism-server/tests/support/live_clock_jump.rs new file mode 100644 index 000000000..463415ff8 --- /dev/null +++ b/crates/qbit-prism-server/tests/support/live_clock_jump.rs @@ -0,0 +1,793 @@ +//! #575 item 5: wall-clock jumps while the pool is mining. +//! +//! Both servers run under libfaketime, each with its own offset file, and so +//! does the test's own PostgreSQL 16 cluster, whose clock drives every lease +//! and claim expiry (`clock_timestamp()`) and the ledger clock. Only the +//! realtime clock moves (`FAKETIME_DONT_FAKE_MONOTONIC=1`), as an NTP step +//! does: every timeout and deadline, which the servers take from the +//! monotonic clock, is unaffected. The node keeps the real clock. +//! +//! Both servers reach the node through a proxy that records every +//! `submitblock` and can hold one, so a found block can be mid-landing, its +//! claim live, while the database clock jumps. +//! +//! Phases, each mining shares and a block on every server that must serve: +//! 1. baseline; +//! 2. server 0 ten minutes behind, then ten minutes ahead; server 1 three +//! hours behind, then three hours ahead (minutes and hours, back and +//! forward). In each, the other server lands a block first, so the moved +//! one must build work on a template it has not seen. Behind, it serves. +//! Ahead of the node by more than `PRISM_TEMPLATE_MAX_AGE_SECONDS` (120 s) +//! it calls every template stale and serves no current work (fail-safe); +//! at three hours ahead the test watches it for 200 s, with a miner +//! retrying, and `PrismWorkRefreshStalledCritical` must fire. Corrected, +//! it serves again; +//! 3. the database two hours ahead while server 0's block is held +//! mid-landing, so the database clock calls every live claim expired; +//! since #581 no frontend takes a claim over by the database clock, so +//! the holder keeps it; +//! 4. the database back to the real clock (two hours back) while server 1's +//! block is held mid-landing; +//! 5. every clock real again. +//! +//! A server mines in a phase only once its work is settled +//! ([`Fixture::settled`]): on the node's tip, at the cluster's payout +//! revision, with no block still landing. A landing moves the payout revision +//! after the node takes the block, and a share on work from before the move +//! would meet the append's revision fence (#632), whatever the clocks. Since +//! #581 each held block is landed by its holder as soon as it is released, so +//! phases 3 and 4 start mining while that landing is in flight. +//! +//! It asserts that each server's HTTP `Date` and the database's +//! `clock_timestamp()` really moved; every acknowledged share is in the +//! ledger, credited once, with `accepted_at` non-decreasing in ledger order; +//! every accepted block was offered exactly once and landed once, as one +//! confirmed pool block on the node's chain whose coinbase matches its audit +//! bundle; and the carry-forward integrity report is clean. Every outbox row +//! is `submitted` (#581): neither held block is taken over across its +//! database step, so each records its own one offer as accepted, rather +//! than phase 3's block being recovered as an unknown offer and stranded in +//! reconciliation with a retry two hours out, as it was before #581. +use super::alert_rules::{sample, scrape, snapshot_stale, Mirror, Verdict}; +use super::host_tools::program; +use super::private_postgres::{ClusterOptions, PrivateCluster}; +use super::share_client::{start_share_only_servers, Proof, ShareClient, Submitted}; +use super::submit_proxy::SubmitProxy; +use super::*; +use std::{collections::BTreeMap, path::Path}; + +/// Shares each serving server mines per phase. +const PHASE_SHARES: usize = 5; +/// How long a server has to serve work on the current tip in a phase, +/// settling included. +const WORK_SECONDS: u64 = 30; +/// How long a held landing stays held after its clock jump. +const HOLD_AFTER_JUMP: Duration = Duration::from_secs(8); +/// Phases 3 and 4, by the label their records carry. +const HELD_FORWARD: &str = "database +2 h mid-landing"; +const HELD_BACK: &str = "database back 2 h mid-landing"; +/// A generous `submitblock` deadline, so a held call outlives the jump. +const SUBMIT_TIMEOUT_SECONDS: &str = "60"; + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +#[ignore = "nightly #575: server and database wall-clock jumps under mining; needs libfaketime"] +async fn wall_clock_jumps_keep_windows_claims_and_payouts_and_land_every_block_once() -> Result<()> +{ + // Selected explicitly, so a missing input fails rather than skipping. + let inputs = gate::required_inputs( + gate::site!(), + &[ + gate::Input::QbitdBin, + gate::Input::DatabaseUrl, + gate::Input::PgBinDir, + ], + )?; + let library = faketime_library()?; + drop(SERIAL.lock().await); + let home = tempfile::Builder::new().prefix("b575c-").tempdir()?; + let clocks = Clocks::new(home.path())?; + let cluster = PrivateCluster::start(ClusterOptions { + bin: PathBuf::from(&inputs[2]), + data: home.path().join("data"), + home: home.path().to_path_buf(), + initdb: &[], + settings: "", + env: clocks + .environment(&library, &clocks.database) + .into_iter() + .map(|(name, value)| (name.to_owned(), value)) + .collect(), + })?; + let Some(mut fixture) = Fixture::open_on_database(false, false, Some(&cluster.url())).await? + else { + bail!("the live fixture's inputs were required above"); + }; + let mut proxy = None; + let result = async { + let started = SubmitProxy::start(fixture.rpc_port).await?; + let proxy = proxy.insert(started); + jumps(&mut fixture, &clocks, &library, proxy).await + } + .await; + if result.is_err() { + eprintln!("{}", fixture.diagnostics()); + eprintln!("{}", cluster.diagnostics()); + } + // Real clocks for cleanup. + let _ = clocks.set_all(0); + if let Some(proxy) = &proxy { + proxy.release_all(); + } + let cleanup = fixture.cleanup().await; + drop(proxy); + drop(cluster); + drop(home); + result.and(cleanup) +} + +/// libfaketime, found next to the `faketime` wrapper on `PATH`, as the +/// Debian/Ubuntu `faketime` and `libfaketime` packages install it. +fn faketime_library() -> Result { + let wrapper = program("faketime").context("install the faketime and libfaketime packages")?; + let prefix = wrapper + .parent() + .and_then(Path::parent) + .context("faketime has no prefix")?; + let mut candidates = vec![prefix.join("lib/faketime/libfaketime.so.1")]; + for entry in std::fs::read_dir(prefix.join("lib"))? { + candidates.push(entry?.path().join("faketime/libfaketime.so.1")); + } + candidates + .into_iter() + .find(|candidate| candidate.is_file()) + .with_context(|| format!("libfaketime.so.1 not found under {}", prefix.display())) +} + +/// The offset files: one per server and one for the database. libfaketime +/// re-reads a file at most once a second. +struct Clocks { + servers: [PathBuf; 2], + database: PathBuf, +} + +impl Clocks { + fn new(home: &Path) -> Result { + let clocks = Self { + servers: [home.join("clock-server-0"), home.join("clock-server-1")], + database: home.join("clock-database"), + }; + clocks.set_all(0)?; + Ok(clocks) + } + + fn environment(&self, library: &Path, file: &Path) -> Vec<(&'static str, String)> { + vec![ + ("LD_PRELOAD", library.display().to_string()), + ("FAKETIME_TIMESTAMP_FILE", file.display().to_string()), + ("FAKETIME_CACHE_DURATION", "1".into()), + ("FAKETIME_DONT_FAKE_MONOTONIC", "1".into()), + ] + } + + /// Replace an offset atomically, so a reader never sees half a write. + fn set(file: &Path, seconds: i64) -> Result<()> { + let staging = file.with_extension("new"); + std::fs::write(&staging, format!("{seconds:+}\n"))?; + std::fs::rename(staging, file)?; + Ok(()) + } + + fn set_all(&self, seconds: i64) -> Result<()> { + for file in self.servers.iter().chain([&self.database]) { + Self::set(file, seconds)?; + } + Ok(()) + } +} + +fn unix_now() -> i64 { + std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .expect("clock after the epoch") + .as_secs() as i64 +} + +/// Move server `index`'s clock to `offset` seconds from real time and wait +/// until it has. The server's HTTP `Date` shows its clock, but hyper caches +/// that header until its clock passes the cached second, so it moves only +/// when the clock passes its highest value so far (`highest`). A move that +/// does not is given twice libfaketime's one-second re-read; the first move +/// of each server, forward, has proven the library reads its file. (The +/// server makes itself non-dumpable, so its `/proc` maps are unreadable.) +async fn move_server( + f: &Fixture, + clocks: &Clocks, + highest: &mut [i64; 2], + index: usize, + offset: i64, +) -> Result<()> { + Clocks::set(&clocks.servers[index], offset)?; + if offset <= highest[index] { + tokio::time::sleep(Duration::from_millis(2500)).await; + return Ok(()); + } + highest[index] = offset; + until( + &format!("server {index} clock at {offset:+}s"), + 15, + || async { + let response = f + .client + .get(format!("http://127.0.0.1:{}/healthz", f.api[index])) + .send() + .await?; + let date = response + .headers() + .get("date") + .context("no Date header")? + .to_str()?; + let served = chrono::DateTime::parse_from_rfc2822(date)?.timestamp(); + Ok((served - unix_now() - offset).abs() <= 3) + }, + ) + .await +} + +/// Wait until the database's `clock_timestamp()` is `offset` seconds from +/// real time. +async fn database_offset(f: &Fixture, offset: i64) -> Result<()> { + until(&format!("database clock at {offset:+}s"), 15, || async { + let served: f64 = + sqlx::query_scalar("SELECT extract(epoch FROM clock_timestamp())::float8") + .fetch_one(&f.pool) + .await?; + Ok((served as i64 - unix_now() - offset).abs() <= 3) + }) + .await +} + +/// What one phase did on one server. +enum Served { + /// Its shares and the block, which landed. + Mined { + submitted: Vec, + block: String, + }, + /// No current work reached a miner: why. + NoWork(String), +} + +/// Mine `PHASE_SHARES` shares and one block on `server`, on work on the +/// current tip by `deadline`, and wait for the block on the node. Every share +/// and the block must be accepted. +async fn mine_on(f: &Fixture, server: usize, label: &str, deadline: Instant) -> Result { + let tip = f.rpc("getbestblockhash", json!([])).await?; + let tip = tip.as_str().context("tip missing")?.to_owned(); + let username = format!("{}.clock-{server}", f.address); + let current = async { + let mut client = ShareClient::connect(f.stratum[server], &username).await?; + client + .work_on(&tip, deadline.saturating_duration_since(Instant::now())) + .await?; + Ok::<_, anyhow::Error>(client) + }; + let mut client = match current.await { + Ok(client) => client, + Err(error) => return Ok(Served::NoWork(format!("{error:#}"))), + }; + let mut submitted = Vec::new(); + for _ in 0..PHASE_SHARES { + let share = client.submit(Proof::Share).await?; + ensure!( + share.answer.accepted(), + "{label}: server {server} answered a share {}", + share.answer + ); + submitted.push(share); + } + let block = client.submit(Proof::Block).await?; + ensure!( + block.answer.accepted(), + "{label}: server {server} refused its block: {}", + block.answer + ); + let hash = block.hash.clone(); + submitted.push(block); + until( + &format!("{label}: server {server}'s block on the node"), + 60, + || async { Ok(f.rpc("getbestblockhash", json!([])).await? == json!(hash)) }, + ) + .await?; + Ok(Served::Mined { + submitted, + block: hash, + }) +} + +/// Record of every phase, for the final checks and the report. +#[derive(Default)] +struct Run { + submitted: Vec<(String, Submitted)>, + /// Held blocks answered ledger-outcome-unknown, whose shares must still + /// be credited once their landing commits. + pending: Vec, + blocks: Vec<(String, String)>, + notes: Vec, +} + +impl Run { + /// Mine on `server` in a phase where it must serve work, once its work + /// is settled, and keep what it did. + async fn mine(&mut self, f: &Fixture, server: usize, label: &str) -> Result<()> { + let deadline = Instant::now() + Duration::from_secs(WORK_SECONDS); + f.settled(server, WORK_SECONDS) + .await + .with_context(|| format!("{label}: server {server} before mining"))?; + match mine_on(f, server, label, deadline).await? { + Served::NoWork(why) => bail!("{label}: server {server} served no current work: {why}"), + Served::Mined { submitted, block } => { + self.submitted.extend( + submitted + .into_iter() + .map(|submitted| (label.to_owned(), submitted)), + ); + self.blocks.push((label.to_owned(), block)); + Ok(()) + } + } + } +} + +/// `-10 min`, `+3 h`: an offset as the phase names show it. +fn span(seconds: i64) -> String { + if seconds.abs() >= 3600 { + format!("{:+} h", seconds / 3600) + } else { + format!("{:+} min", seconds / 60) + } +} + +/// How long a server that serves no work is watched for an alert: longer +/// than the stall threshold plus the `for` of every rule below. +const UNSERVED_WATCH: Duration = Duration::from_secs(200); + +/// While `server` serves no work, keep a miner connecting and authorizing +/// to it (as a real one would) and report which paging rules fire on it. +/// The refresh stall is what this state is, so its rule must. +async fn unserved_alerts(f: &Fixture, server: usize) -> Result { + let mirrors = [ + Mirror::new( + "PrismWorkRefreshStalledCritical", + &["qbit_prism_work_refresh_stalled_seconds", ">= bool 120"], + |_, s| { + s.gate_open() + && s.value("qbit_prism_work_refresh_stalled_seconds") + .is_some_and(|stalled| stalled >= 120.0) + }, + )?, + Mirror::new( + "PrismCurrentWorkGapHigh", + &[ + "qbit_prism_stratum_current_tip_coverage_gap_seconds", + "> bool 15", + ], + |_, s| { + s.gate_open() + && s.value("qbit_prism_stratum_current_tip_coverage_gap_seconds") + .is_some_and(|gap| gap > 15.0) + && s.value("qbit_prism_authorized_clients") + .is_some_and(|n| n > 0.0) + }, + )?, + Mirror::new( + "PrismSemanticWorkCoverageLoss", + &[ + "qbit_prism_stratum_semantic_current_work_ratio", + "< bool 0.95", + ], + |_, s| { + s.gate_open() + && s.value("qbit_prism_stratum_semantic_current_work_ratio") + .is_some_and(|ratio| (0.0..0.95).contains(&ratio)) + && s.value("qbit_prism_authorized_clients") + .is_some_and(|n| n > 0.0) + }, + )?, + Mirror::new( + "PrismTimeToUsableWorkHigh", + &[ + "qbit_prism_stratum_oldest_pending_initial_job_seconds", + "> bool 15", + ], + |_, s| { + s.gate_open() + && s.value("qbit_prism_stratum_oldest_pending_initial_job_seconds") + .is_some_and(|age| age > 15.0) + && s.value("qbit_prism_stratum_pending_initial_jobs") + .is_some_and(|n| n > 0.0) + }, + )?, + snapshot_stale()?, + ]; + let port = f.stratum[server]; + let username = format!("{}.clock-waiting-{server}", f.address); + let waiting = tokio::spawn(async move { + loop { + // Waits for work that never comes, until the server closes it. + let _ = ShareClient::connect(port, &username).await; + tokio::time::sleep(Duration::from_millis(200)).await; + } + }); + let scrapes = sample(&f.client, &[(server, f.api[server])], UNSERVED_WATCH).await; + waiting.abort(); + let verdict = Verdict::of(&scrapes, &[server], &mirrors); + let report = format!( + "server {server} unserved for {UNSERVED_WATCH:?}: {}", + verdict.report.replace('\n', ", ") + ); + ensure!( + verdict.fired[&server].contains("PrismWorkRefreshStalledCritical"), + "no work-refresh page while a server served no work: {report}" + ); + Ok(report) +} + +/// Phases 3 and 4: hold `holder`'s next found block at the proxy, move the +/// database clock to `offset` while it is held, then release it and wait +/// for it on the node. +async fn held_landing( + f: &Fixture, + clocks: &Clocks, + proxy: &SubmitProxy, + run: &mut Run, + holder: usize, + offset: i64, + label: &str, +) -> Result<()> { + let deadline = Instant::now() + Duration::from_secs(WORK_SECONDS); + f.settled(holder, WORK_SECONDS) + .await + .with_context(|| format!("{label}: server {holder} before its held block"))?; + let held = proxy.arm(); + let tip = f.rpc("getbestblockhash", json!([])).await?; + let tip = tip.as_str().context("tip missing")?.to_owned(); + let username = format!("{}.clock-held-{holder}", f.address); + let mut client = ShareClient::connect(f.stratum[holder], &username).await?; + client + .work_on(&tip, deadline.saturating_duration_since(Instant::now())) + .await?; + // The block's answer waits for its landing, which the proxy holds: + // submit it concurrently. + let submit = tokio::spawn(async move { client.submit(Proof::Block).await }); + let hit = tokio::time::timeout(Duration::from_secs(30), held) + .await + .context("no submitblock reached the proxy")??; + let reserved: String = + sqlx::query_scalar("SELECT state FROM qbit_block_candidate_outbox WHERE block_hash=$1") + .bind(&hit.block) + .fetch_one(&f.pool) + .await?; + Clocks::set(&clocks.database, offset)?; + database_offset(f, offset).await?; + tokio::time::sleep(HOLD_AFTER_JUMP).await; + let offers_while_held = proxy.offers(&hit.block); + let _ = hit.release.send(()); + let block = submit.await??; + ensure!( + block.hash == hit.block, + "{label}: the held block {} is not the one submitted, {}", + hit.block, + block.hash + ); + run.notes.push(format!( + "{label}: held block was {reserved} at the jump; {offers_while_held} submitblock call(s) while held; answer {}", + block.answer + )); + // The answer waits for the landing up to the share commit timeout + // (#577); a hold that outlasts it is answered ledger-outcome-unknown, + // and the block still lands and its share is credited once. + ensure!( + block.answer.accepted() || block.answer.reason_id() == Some("ledger-outcome-unknown"), + "{label}: the held block was refused: {}", + block.answer + ); + if !block.answer.accepted() { + run.pending.push(block.share_id.clone()); + } + let hash = block.hash.clone(); + run.submitted.push((label.to_owned(), block)); + run.blocks.push((label.to_owned(), hash.clone())); + until( + &format!("{label}: the held block on the node"), + 60, + || async { Ok(f.rpc("getbestblockhash", json!([])).await? == json!(hash)) }, + ) + .await +} + +async fn jumps( + f: &mut Fixture, + clocks: &Clocks, + library: &Path, + proxy: &SubmitProxy, +) -> Result<()> { + let mut servers = Vec::new(); + for index in 0..2 { + let mut overrides: Vec<(&str, String)> = vec![ + ("QBIT_RPC_PORT", proxy.port.to_string()), + ( + "PRISM_BLOCK_SUBMIT_RPC_TIMEOUT_SECONDS", + SUBMIT_TIMEOUT_SECONDS.into(), + ), + ]; + overrides.extend(clocks.environment(library, &clocks.servers[index])); + servers.push((index, overrides)); + } + start_share_only_servers(f, &servers).await?; + let mut run = Run::default(); + + // 1. Baseline, and proof that the clocks are the injected ones. + let mut highest = [-1i64; 2]; + for index in 0..2 { + move_server(f, clocks, &mut highest, index, 0).await?; + } + database_offset(f, 0).await?; + for server in 0..2 { + run.mine(f, server, "baseline").await?; + } + + // 2. Each server alone, minutes then hours: behind, then ahead. + // In each phase the other server lands a block first, so the moved one + // must build work on a template it has not seen. + for (server, seconds, sample_alerts) in [(0usize, 600i64, false), (1, 3 * 3600, true)] { + let other = 1 - server; + let behind = format!("server {server} {}", span(-seconds)); + move_server(f, clocks, &mut highest, server, -seconds).await?; + run.mine(f, other, &behind).await?; + run.mine(f, server, &behind).await?; + + // Ahead of the node by more than PRISM_TEMPLATE_MAX_AGE_SECONDS + // (120 s), every template the server reads looks stale: it builds no + // work on the new tip (fail-safe), until its clock is corrected. + let ahead = format!("server {server} {}", span(seconds)); + move_server(f, clocks, &mut highest, server, seconds).await?; + run.mine(f, other, &ahead).await?; + let deadline = Instant::now() + Duration::from_secs(WORK_SECONDS); + let Served::NoWork(why) = mine_on(f, server, &ahead, deadline).await? else { + bail!("{ahead}: server {server} served work on a template its clock calls stale"); + }; + run.notes.push(format!( + "{ahead}: server {server} served no current work, as expected: {why}" + )); + if sample_alerts { + run.notes.push(unserved_alerts(f, server).await?); + } + move_server(f, clocks, &mut highest, server, 0).await?; + run.mine(f, server, &format!("server {server} corrected")) + .await?; + } + + // 3 and 4. The database clock jumps while a block is mid-landing. + held_landing(f, clocks, proxy, &mut run, 0, 2 * 3600, HELD_FORWARD).await?; + for server in 0..2 { + run.mine(f, server, HELD_FORWARD).await?; + } + held_landing(f, clocks, proxy, &mut run, 1, 0, HELD_BACK).await?; + for server in 0..2 { + run.mine(f, server, HELD_BACK).await?; + } + + // 5. Every clock real again: every share is accepted again. + for index in 0..2 { + move_server(f, clocks, &mut highest, index, 0).await?; + } + Clocks::set(&clocks.database, 0)?; + database_offset(f, 0).await?; + for server in 0..2 { + run.mine(f, server, "real again").await?; + } + f.quiesce().await?; + verify(f, &mut run, proxy).await +} + +async fn verify(f: &Fixture, run: &mut Run, proxy: &SubmitProxy) -> Result<()> { + // Shares: every acknowledgement is durable and credited once, and the + // ledger clock never ran backwards. + let ledger: BTreeMap = + sqlx::query_as::<_, (String, bool)>("SELECT share_id, accepted FROM qbit_share_ledger") + .fetch_all(&f.pool) + .await? + .into_iter() + .collect(); + let lost: Vec<_> = run + .submitted + .iter() + .filter(|(_, submitted)| submitted.answer.accepted()) + .filter(|(_, submitted)| ledger.get(&submitted.share_id) != Some(&true)) + .map(|(label, submitted)| format!("{label}: {}", submitted.share_id)) + .collect(); + ensure!(lost.is_empty(), "acknowledged shares missing: {lost:?}"); + let uncredited: Vec<_> = run + .pending + .iter() + .filter(|share_id| ledger.get(*share_id) != Some(&true)) + .collect(); + ensure!( + uncredited.is_empty(), + "held blocks answered ledger-outcome-unknown were never credited: {uncredited:?}" + ); + let (accepted, credited): (i64, i64) = sqlx::query_as( + "SELECT (SELECT count(*) FROM qbit_share_ledger WHERE accepted),(SELECT count(*) FROM qbit_prism_share_hashes)", + ) + .fetch_one(&f.pool) + .await?; + ensure!( + accepted == credited, + "duplicate headers credited: {accepted} accepted shares, {credited} credited headers" + ); + let regressions: Vec<(i64, String, String)> = sqlx::query_as( + "SELECT share_seq, accepted_at::text, previous::text FROM (SELECT share_seq, accepted_at, lag(accepted_at) OVER (ORDER BY share_seq) AS previous FROM qbit_share_ledger WHERE accepted) s WHERE accepted_at < previous", + ) + .fetch_all(&f.pool) + .await?; + ensure!( + regressions.is_empty(), + "the ledger clock ran backwards: {regressions:?}" + ); + + // Blocks: each landed exactly once, on the node's chain, with a coinbase + // that matches its audit bundle. + let key = ManifestSigningKey::from_seed_hex(&"22".repeat(32))?.public_key_hex(); + let mut offers = BTreeMap::new(); + for (label, hash) in &run.blocks { + // A row the server released into reconciliation (a post-offer step + // that lost a race, say) finishes on its retry, 10 s per attempt + // later; quiesce does not wait for that. Since #581 a retry the + // database clock's step back left two hours out is due at once, so + // every row finishes. + let rows = || async { + Ok::<_, anyhow::Error>( + sqlx::query_as::<_, (String, i32, Option)>( + "SELECT state, attempt_count, offer_outcome FROM qbit_block_candidate_outbox WHERE block_hash=$1", + ) + .bind(hash) + .fetch_all(&f.pool) + .await?, + ) + }; + until( + &format!("{label}: block {hash}'s outbox row settled"), + 120, + || async { + let rows = rows().await?; + Ok(rows.len() == 1 && rows[0].0 == "submitted") + }, + ) + .await?; + let rows: Vec<(String, i32)> = rows() + .await? + .into_iter() + .map(|(state, attempts, _)| (state, attempts)) + .collect(); + let landed: Vec = + sqlx::query_scalar("SELECT chain_state FROM qbit_pool_blocks WHERE block_hash=$1") + .bind(hash) + .fetch_all(&f.pool) + .await?; + ensure!( + landed == ["confirmed"], + "{label}: block {hash} pool-block rows {landed:?}" + ); + ensure!( + rows.len() == 1 && rows[0].0 == "submitted", + "{label}: block {hash} outbox rows {rows:?}" + ); + if label == HELD_FORWARD || label == HELD_BACK { + // #581: the held block's holder kept its claim across the + // database step and recorded its one offer itself. Before #581 + // the +2 h step handed phase 3's live claim to a second frontend, + // which recovered the reservation as an unknown offer. + let (outcome,): (Option,) = sqlx::query_as( + "SELECT offer_outcome FROM qbit_block_candidate_outbox WHERE block_hash=$1", + ) + .bind(hash) + .fetch_one(&f.pool) + .await?; + ensure!( + outcome.as_deref() == Some("accepted") && proxy.offers(hash) == 1, + "{label}: block {hash} offer outcome {outcome:?}, {} submitblock call(s): its claim was taken over across the database step", + proxy.offers(hash) + ); + } + let block = f.rpc("getblock", json!([hash, 2])).await?; + ensure!( + block["confirmations"].as_i64().is_some_and(|c| c >= 1), + "{label}: block {hash} is not on the node's active chain" + ); + let body: Value = f + .client + .get(format!( + "http://127.0.0.1:{}/audit/blocks/{hash}/bundle", + f.api[0] + )) + .send() + .await? + .error_for_status()? + .json() + .await?; + let bundle: AuditBundle = serde_json::from_value(body["audit_bundle"].clone())?; + let coinbase = f + .rpc( + "getrawtransaction", + json!([block["tx"][0]["txid"], false, hash]), + ) + .await?; + verify_audit_bundle_against_coinbase_tx_hex( + &bundle, + coinbase.as_str().context("node coinbase missing")?, + &key, + ) + .with_context(|| format!("{label}: block {hash} coinbase differs from its audit bundle"))?; + offers.insert(hash.clone(), (label.clone(), rows[0].1, proxy.offers(hash))); + } + // Only the pool mines, so every block on the chain above the first is + // one of the pool's. + let pool_blocks: i64 = + sqlx::query_scalar("SELECT count(*) FROM qbit_pool_blocks WHERE chain_state='confirmed'") + .fetch_one(&f.pool) + .await?; + ensure!( + pool_blocks == run.blocks.len() as i64, + "{pool_blocks} confirmed pool blocks for {} accepted blocks", + run.blocks.len() + ); + let double: Vec<_> = offers + .iter() + .filter(|(_, (_, _, submitted))| *submitted != 1) + .collect(); + f.integrity().await?; + let acknowledged = run + .submitted + .iter() + .filter(|(_, submitted)| submitted.answer.accepted()) + .count(); + let refused: BTreeMap = run + .submitted + .iter() + .filter(|(_, submitted)| !submitted.answer.accepted()) + .fold(BTreeMap::new(), |mut counts, (label, submitted)| { + *counts + .entry(format!("{label}: {}", submitted.answer)) + .or_default() += 1; + counts + }); + eprintln!( + "clock jumps: {acknowledged} acknowledged, refused {refused:?}; {} blocks landed once each; offers per block {:?}", + run.blocks.len(), + offers + ); + // Why shares were refused, from each server's own account. + for (server, port) in f.api.into_iter().enumerate() { + let causes = scrape(&f.client, server, port) + .await? + .by_label("qbit_prism_stale_job_rejections_total", "cause"); + let causes: BTreeMap<_, _> = causes.into_iter().filter(|(_, n)| *n > 0.0).collect(); + let log = std::fs::read_to_string(f.directory.path().join(format!("server-{server}.log"))) + .unwrap_or_default(); + let failures: Vec<_> = log + .lines() + .filter(|line| line.contains("share persistence failed")) + .take(3) + .collect(); + eprintln!("clock jumps: server {server} stale-job causes {causes:?}; persistence failures {failures:?}"); + } + for note in &run.notes { + eprintln!("clock jumps: {note}"); + } + ensure!( + double.is_empty(), + "blocks offered other than exactly once: {double:?}" + ); + Ok(()) +} diff --git a/crates/qbit-prism-server/tests/support/live_compact_runtime.rs b/crates/qbit-prism-server/tests/support/live_compact_runtime.rs new file mode 100644 index 000000000..7ebdd6873 --- /dev/null +++ b/crates/qbit-prism-server/tests/support/live_compact_runtime.rs @@ -0,0 +1,201 @@ +//! A real wallet/mempool template, first blob insertion, compact A/B recovery, +//! and node-accepted block. Fixture loading and validation stay outside WAL. +use super::*; +use qbit_prism_server::{coordinator::Coordinator, metrics::Metrics, stratum::MiningBackend}; +use std::sync::Arc; + +#[path = "prepared_work_assertions.rs"] +#[allow(dead_code)] +mod assertions; +#[path = "fake_qbitd.rs"] +#[allow(dead_code)] +mod configuration; +#[path = "jsonb_inventory.rs"] +#[allow(dead_code)] +mod jsonb_inventory; +#[path = "compact_runtime_observer.rs"] +mod observer; +#[path = "ledger_execution_proxy.rs"] +mod proxy; +#[path = "wal_primary.rs"] +mod wal_primary; +#[path = "window_fixture.rs"] +#[allow(dead_code)] +mod window_fixture; + +async fn large_wallet_template(f: &Fixture) -> Result { + // Coinbase maturity is 1000 on this chain. These 200 independent inputs + // create four standard transactions, comfortably below block weight limits. + f.rpc("generatetoaddress", json!([1200, f.address])).await?; + let unspent = f + .rpc("listunspent", json!([1000, 9_999_999, [], true])) + .await?; + let funding: Vec<_> = unspent + .as_array() + .context("wallet UTXOs missing")? + .iter() + .filter(|row| row["spendable"] == true) + .take(200) + .collect(); + ensure!( + funding.len() == 200, + "insufficient mature independent wallet inputs" + ); + for inputs in funding.chunks(50) { + let amount = inputs + .iter() + .try_fold(0u64, |sum, row| -> Result { + sum.checked_add(qbit_prism_server::broadcaster::amount_bits(&row["amount"])?) + .context("funding amount overflow") + })? + .checked_sub(1_000_000) + .context("insufficient fee funding")?; + let outputs = serde_json::Map::from_iter([( + f.address.clone(), + serde_json::from_str::(&format!( + "{}.{:08}", + amount / 100_000_000, + amount % 100_000_000 + ))?, + )]); + let outpoints: Vec<_> = inputs + .iter() + .map(|row| json!({"txid":row["txid"], "vout":row["vout"]})) + .collect(); + let unsigned = f + .rpc("createrawtransaction", json!([outpoints, outputs])) + .await?; + let signed = f + .rpc("signrawtransactionwithwallet", json!([unsigned])) + .await?; + ensure!( + signed["complete"] == true, + "wallet did not sign every input" + ); + f.rpc("sendrawtransaction", json!([signed["hex"]])).await?; + } + let template = f + .rpc("getblocktemplate", json!([{"rules":["segwit"]}])) + .await?; + let transactions = template["transactions"] + .as_array() + .context("template transactions missing")?; + ensure!( + transactions.len() == 4, + "real template did not include every signed transaction" + ); + let bytes = serde_json::to_vec(&template)?.len(); + ensure!( + bytes > 1_000_000, + "real template must exceed the former JSONB bound: {bytes}" + ); + eprintln!( + "large real-node template: transactions={}, serialized_bytes={bytes}", + transactions.len() + ); + Ok(template) +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn real_large_template_first_insertion_resumes_and_mines_with_bounded_jsonb_and_wal( +) -> Result<()> { + let Some(bin) = gate::pg_bin_dir(gate::site!())? else { + return Ok(()); + }; + let primary = wal_primary::Primary::start(bin).await?; + let Some(fixture) = Fixture::open_on_database(false, false, Some(&primary.url)).await? else { + return Ok(()); + }; + let mut frontends = Vec::new(); + let mut observed = None; + let result = async { + let (version, fsync): (String, String) = sqlx::query_as( + "SELECT current_setting('server_version_num'),current_setting('fsync')") + .fetch_one(&fixture.pool).await?; + ensure!(version.parse::()? / 10_000 == 16 && fsync == "on", "qualification requires durable PostgreSQL 16"); + large_wallet_template(&fixture).await?; + let url = url::Url::parse(&fixture.database_url)?; + let upstream = tokio::net::lookup_host((url.host_str().context("database host")?, url.port().unwrap_or(5432))) + .await?.next().context("database address missing")?; + let proxy = proxy::ExecutionProxy::start(upstream).await?; + let database_url = proxy.rewrite_url(&fixture.database_url)?; + observed = Some(proxy); + let proxy = observed.as_ref().unwrap(); + for instance in ["large-template-a", "large-template-b"] { + let mut config = configuration::coordinator_config_at(database_url.clone(), + format!("http://127.0.0.1:{}/", fixture.rpc_port), instance)?; + config.chain = "regtest".into(); + config.rpc_user = "prismtest".into(); + config.rpc_password = "prismtest".into(); + config.min_peers = 0; + config.submit_tip_max_age = Duration::from_secs(120); + config.health_timeout = Duration::from_secs(120); + frontends.push(Coordinator::new(config, Arc::new(Metrics::default())).await?); + } + let (a, b) = (&frontends[0], &frontends[1]); + let plan = window_fixture::WindowPlan::new(5_000)?; + plan.load(&fixture.pool, "real-template-window").await?; + plan.verify_round_trip(&fixture.pool, &[1, 2500, 5000]).await?; + let existing: i64 = sqlx::query_scalar("SELECT count(*) FROM qbit_prism_templates") + .fetch_one(&fixture.pool).await?; + ensure!(existing == 0, "first insertion proof must start without any template blob"); + let probe = observer::JsonbProbe::install(&fixture.pool, &fixture.schema).await?; + sqlx::query("CHECKPOINT").execute(&fixture.pool).await?; + let mark = proxy.mark(); + let before = observer::insert_lsn(&fixture.pool).await?; + let started = Instant::now(); + a.refresh_once().await?; + let elapsed = started.elapsed(); + let after = observer::insert_lsn(&fixture.pool).await?; + let wal = observer::wal_bytes(&fixture.pool, &before, &after).await?; + let measured = probe.measure(proxy, mark)?; + eprintln!("real-node refresh observation: max_jsonb={}, insert_wal_bytes={wal}, refresh_seconds={:.3}", measured.max_uncompressed_bytes, elapsed.as_secs_f64()); + let prepared = a.prepared.read().await.clone().context("refresh did not publish")?; + assertions::assert_refresh_measurements(5000, prepared.window.shares.context("window missing")?.share_count, + Some(measured.max_uncompressed_bytes), Some(wal))?; + let stored = a.ledger.compact_prepared(&prepared.storage_key).await?.context("typed original missing")?; + let template_bytes = serde_json::to_vec(&stored.template)?.len(); + ensure!(template_bytes > 1_000_000, "refresh did not capture the real large template"); + let blob_bytes: i64 = sqlx::query_scalar("SELECT octet_length(template_bytes)::bigint FROM qbit_prism_templates WHERE template_sha256=$1") + .bind(&stored.record.template_sha256).fetch_one(&fixture.pool).await?; + ensure!(blob_bytes == template_bytes as i64, "first template bytes changed in storage"); + let worker = a.authorize(&format!("{}.large", fixture.address)).await?; + let issued = a.build_job(&worker, "12345678", 1e-12, 0.0).await?; + a.persist_issued_job(&worker, &issued, 0, Duration::from_secs(60)).await?; + b.refresh_once().await?; + let resumed = tokio::time::timeout(Duration::from_secs(25), b.resume_job(&worker, &issued.wire.job_id)) + .await??.context("cross-frontend reconstruction missed")?; + ensure!(resumed.wire.coinb1 == issued.wire.coinb1 && resumed.wire.coinb2 == issued.wire.coinb2 + && resumed.wire.transactions == issued.wire.transactions + && resumed.wire.merkle_branch == issued.wire.merkle_branch, "reconstructed large template work changed"); + let mut solved = None; + for nonce in 0..10_000u32 { + let proof = resumed.wire.assemble_submission(&"00".repeat(resumed.wire.extranonce2_size), + &format!("{:08x}", resumed.wire.ntime), &format!("{nonce:08x}"), None, 0)?; + if proof.block_pass { solved = Some(proof); break; } + } + let solved = solved.context("bounded regtest proof search exhausted")?; + ensure!(fixture.rpc("submitblock", json!([solved.block_hex])).await?.is_null(), + "node rejected the reconstructed real template block"); + ensure!(fixture.rpc("getbestblockhash", json!([])).await? == solved.block_hash_hex, + "node did not activate the reconstructed block"); + let hashes = stored.record.audit_hashes.context("original audit hashes missing")?; + eprintln!("real large-template compact refresh: template_bytes={template_bytes}, shares=5000, max_jsonb={}, insert_wal_bytes={wal}, refresh_seconds={:.3}, audit_sha256={}, manifest_sha256={}, accepted_block={}", + measured.max_uncompressed_bytes, elapsed.as_secs_f64(), hashes.audit_bundle_sha256, + hashes.coinbase_manifest_sha256, solved.block_hash_hex); + Ok::<_, anyhow::Error>(()) + }.await; + for frontend in frontends { + frontend.ledger.pool.close().await; + } + let proxy_result = match observed { + Some(proxy) => proxy.finish().await, + None => Ok(()), + }; + if result.is_err() { + eprintln!("{}", fixture.diagnostics()); + } + let cleanup = fixture.cleanup().await; + let primary = primary.close().await; + result.and(proxy_result).and(cleanup).and(primary) +} diff --git a/crates/qbit-prism-server/tests/support/live_ctv_cpfp.rs b/crates/qbit-prism-server/tests/support/live_ctv_cpfp.rs new file mode 100644 index 000000000..b58d19201 --- /dev/null +++ b/crates/qbit-prism-server/tests/support/live_ctv_cpfp.rs @@ -0,0 +1,460 @@ +//! Legacy zero-fee fanouts still need a wallet-funded P2A package. Exercise +//! recovery after the original broadcaster died with a reserved wallet UTXO. +use super::*; +use qbit_prism::{AcceptedShare, FoundBlock, PayoutPolicy, SettlementModeConfig}; +use qbit_prism_server::{ + codec, + ledger::{BlockObservation, Candidate, CandidateCtv, Ledger, SignerKeys, WindowRef}, +}; + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn real_cpfp_recovers_reserved_funding_after_owner_crash() -> Result<()> { + cpfp_recovery_case(RecoveryCase::Locked).await +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn real_cpfp_abandoned_child_repair_retains_unremovable_spent_lock() -> Result<()> { + cpfp_recovery_case(RecoveryCase::Abandoned).await +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn real_cpfp_replaces_unsigned_funding_spent_after_reservation_before_wallet_lock( +) -> Result<()> { + cpfp_recovery_case(RecoveryCase::SpentBeforeLock).await +} + +#[derive(Clone, Copy)] +enum RecoveryCase { + Locked, + Abandoned, + SpentBeforeLock, +} + +async fn cpfp_recovery_case(case: RecoveryCase) -> Result<()> { + let repair_abandoned = matches!(case, RecoveryCase::Abandoned); + let replace_unsigned = matches!(case, RecoveryCase::SpentBeforeLock); + let Some(mut fixture) = Fixture::open(true).await? else { + return Ok(()); + }; + let result=async { + for server in &mut fixture.servers {server.stop();} + let ledger=Ledger::connect(&fixture.database_url,"crashed-broadcaster".into(),4,false).await?; + let (fanout_txid,height)=land_legacy_zero_fee_fanout(&fixture,&ledger).await?; + let abandoned=ledger.claim_fanout(60).await?.context("mature fanout claim missing")?; + let unspent=fixture.rpc("listunspent",json!([1,9_999_999,[],true])).await?; + let funding=unspent.as_array().context("wallet UTXO list missing")?.iter().find(|row|row["spendable"]==true).context("mature sponsorship funding missing")?; + let funding_txid=funding["txid"].as_str().context("funding txid missing")?; + let funding_vout:u32=funding["vout"].as_u64().context("funding vout missing")?.try_into()?; + let amount=qbit_prism_server::broadcaster::amount_bits(&funding["amount"])?; + ensure!(ledger.reserve_cpfp_funding(&abandoned,"prism",funding_txid,funding_vout,amount).await?,"funding reservation failed"); + let outpoint=json!({"txid":funding_txid,"vout":funding_vout}); + if replace_unsigned { + // The process dies after committing the reservation, before its + // external wallet lock. A wallet user can still spend this input. + let reserved=ledger.cpfp_package(&fanout_txid).await?.context("reservation missing")?; + ensure!(reserved["signed_child_hex"].is_null(),"fixture already signed a child"); + spend_reserved_funding(&fixture,&outpoint,amount).await?; + ensure!(fixture.rpc("gettxout",json!([funding_txid,funding_vout,false])).await?.is_null(),"reserved input was not spent on the active chain"); + } else { + ensure!(fixture.rpc("lockunspent",json!([false,[outpoint],true])).await?==true,"simulated owner's wallet lock failed"); + } + // The owner dies before signing. Two successors must recover either + // the still-protected reservation or replacement spendable funding. + // #654: a successor takes the claim over only once it has watched it + // go unrenewed for its lease; revoking it stands in for that wait. + qbit_prism_server::ledger::revoke_fanout_claims(&fixture.pool,Some(&fanout_txid)).await?; + for index in 0..2 {fixture.servers[index]=fixture.start_server_with_sponsorship(index,Some(100_000))?;} + until("recovered CPFP package in mempool",40,||async { + let package=ledger.cpfp_package(&fanout_txid).await?; + let Some(package)=package else{return Ok(false)}; + let Some(child)=package["child_txid"].as_str() else{return Ok(false)}; + let mempool=fixture.rpc("getrawmempool",json!([])).await?; + Ok(mempool.as_array().is_some_and(|rows|rows.contains(&json!(fanout_txid))&&rows.contains(&json!(child)))) + }).await?; + let package=ledger.cpfp_package(&fanout_txid).await?.context("signed package not durable")?; + if replace_unsigned { + ensure!(package["funding_txid"]!=funding_txid || package["funding_vout"]!=funding_vout,"takeover kept the confirmed-spent unsigned input"); + let archived:bool=sqlx::query_scalar("SELECT EXISTS(SELECT 1 FROM qbit_prism_cpfp_retired_funding WHERE fanout_txid=$1 AND funding_txid=$2 AND funding_vout=$3)") + .bind(&fanout_txid).bind(funding_txid).bind(i64::from(funding_vout)).fetch_one(&fixture.pool).await?; + ensure!(archived,"invalid reservation lost cleanup history"); + ensure!(ledger.reserve_cpfp_funding(&abandoned,"prism",funding_txid,funding_vout,amount).await.is_err(),"taken-over owner replaced successor funding"); + ensure!(ledger.save_cpfp_package(&abandoned,"00",&"ff".repeat(32)).await.is_err(),"taken-over owner overwrote signed package"); + let replacement_txid=package["funding_txid"].as_str().context("replacement txid missing")?; + let replacement_vout:u32=package["funding_vout"].as_u64().context("replacement vout missing")?.try_into()?; + ensure_funding_excluded(&fixture,replacement_txid,replacement_vout).await?; + ensure!(sqlx::query_scalar::<_,i64>("SELECT count(*) FROM qbit_prism_cpfp_packages").fetch_one(&fixture.pool).await?==1,"takeover duplicated the active signed package"); + fixture.rpc("generatetoaddress",json!([1,fixture.address])).await?; + until("replacement-funded CPFP fanout confirmed",40,||async {Ok(sqlx::query_scalar::<_,String>("SELECT settlement_status FROM qbit_ctv_fanout_artifacts WHERE fanout_txid=$1").bind(&fanout_txid).fetch_one(&fixture.pool).await?=="confirmed")}).await?; + ensure!(fixture.rpc("gettransaction",json!([package["child_txid"]])).await?["confirmations"].as_u64().unwrap_or(0)>0,"replacement child did not confirm"); + until("retired spent reservation cleanup acknowledged",15,||async { + Ok(sqlx::query_scalar::<_,bool>("SELECT wallet_lock_released FROM qbit_prism_cpfp_retired_funding WHERE fanout_txid=$1 AND funding_txid=$2 AND funding_vout=$3") + .bind(&fanout_txid).bind(funding_txid).bind(i64::from(funding_vout)).fetch_one(&fixture.pool).await?) + }).await?; + ensure!(ledger.cpfp_package(&fanout_txid).await?.context("signed package disappeared")?["signed_child_hex"]==package["signed_child_hex"],"confirmation rewrote signed replacement bytes"); + fixture.integrity().await?; + eprintln!("live CPFP regtest: database-only unsigned reservation became confirmed-spent; two broadcasters archived it, fenced the taken-over owner, funded one immutable replacement package and confirmed the payout"); + ledger.pool.close().await; + return Ok(()); + } + ensure!(package["funding_txid"]==funding_txid && package["funding_vout"]==funding_vout,"recovery changed protected funding outpoint"); + ensure!(package["wallet_lock_released"]==false,"mempool acceptance released unconfirmed funding reservation"); + ensure_funding_excluded(&fixture,funding_txid,funding_vout).await?; + ensure!(sqlx::query_scalar::<_,i64>("SELECT count(*) FROM qbit_prism_cpfp_packages").fetch_one(&fixture.pool).await?==1,"concurrent broadcasters duplicated the package"); + // Evict the entire package through a node restart. The pending wallet + // spend must keep funding unavailable independently of the mempool. + stop_broadcasters(&mut fixture,&fanout_txid).await?; + restart_node(&mut fixture,true,height+1000).await?; + ensure_funding_excluded(&fixture,funding_txid,funding_vout).await?; + // Upgrade recovery also repairs a reservation prematurely released by + // an older broadcaster. The stored signed transaction stays immutable. + if repair_abandoned { + fixture.rpc("abandontransaction",json!([package["child_txid"]])).await?; + let available=fixture.rpc("listunspent",json!([1,9_999_999,[],true])).await?; + ensure!(available.as_array().is_some_and(|rows|rows.iter().any(|coin|coin["txid"]==funding_txid&&coin["vout"]==funding_vout)),"abandoned child did not expose funding for persistent-lock repair"); + } + sqlx::query("UPDATE qbit_prism_cpfp_packages SET wallet_lock_released=true WHERE fanout_txid=$1").bind(&fanout_txid).execute(&fixture.pool).await?; + restart_broadcasters(&mut fixture,&fanout_txid).await?; + wait_for_package(&fixture,&fanout_txid,&package["child_txid"]).await?; + until("unconfirmed wallet reservation restored",15,||async { + let pending=ledger.cpfp_package(&fanout_txid).await?.is_some_and(|p|p["wallet_lock_released"]==false); + let available=fixture.rpc("listunspent",json!([1,9_999_999,[],true])).await?; + Ok(pending&&available.as_array().is_some_and(|rows|rows.iter().all(|coin|coin["txid"]!=funding_txid||coin["vout"]!=funding_vout))) + }).await?; + ensure!(ledger.cpfp_package(&fanout_txid).await?.context("package disappeared")?["signed_child_hex"]==package["signed_child_hex"],"eviction recovery rewrote the signed package"); + // A different node can relay the exact package without its original + // wallet; cleanup remains pending until that wallet is available. + stop_broadcasters(&mut fixture,&fanout_txid).await?; + restart_node(&mut fixture,false,height+1000).await?; + restart_broadcasters(&mut fixture,&fanout_txid).await?; + wait_for_package(&fixture,&fanout_txid,&package["child_txid"]).await?; + ensure!(ledger.cpfp_package(&fanout_txid).await?.context("package disappeared")?["wallet_lock_released"]==false,"walletless replay discarded cleanup responsibility"); + fixture.rpc("generatetoaddress",json!([1,fixture.address])).await?; + until("recovered CPFP fanout confirmed",40,||async {Ok(sqlx::query_scalar::<_,String>("SELECT settlement_status FROM qbit_ctv_fanout_artifacts WHERE fanout_txid=$1").bind(&fanout_txid).fetch_one(&fixture.pool).await?=="confirmed")}).await?; + stop_broadcasters(&mut fixture,&fanout_txid).await?; + restart_node(&mut fixture,true,height+1001).await?; + let child=fixture.rpc("gettransaction",json!([package["child_txid"]])).await?; + ensure!(child["confirmations"].as_u64().unwrap_or(0)>0,"sponsorship child was not actually confirmed"); + let before:chrono::DateTime=sqlx::query_scalar("SELECT updated_at FROM qbit_ctv_fanout_artifacts WHERE fanout_txid=$1").bind(&fanout_txid).fetch_one(&fixture.pool).await?; + restart_broadcasters(&mut fixture,&fanout_txid).await?; + if repair_abandoned { + until("confirmed repaired child cleanup attempted",15,||async { + Ok(sqlx::query_scalar::<_,bool>("SELECT updated_at>$2 AND claim_token IS NULL FROM qbit_ctv_fanout_artifacts WHERE fanout_txid=$1").bind(&fanout_txid).bind(before).fetch_one(&fixture.pool).await?) + }).await?; + ensure!(ledger.cpfp_package(&fanout_txid).await?.context("package disappeared")?["wallet_lock_released"]==false,"failed per-output cleanup was reported as released"); + let locks=fixture.rpc("listlockunspent",json!([])).await?; + ensure!(locks.as_array().is_some_and(|rows|rows.contains(&outpoint)),"expected Qbit's retained repaired lock on a spent output"); + ensure!(fixture.rpc("gettxout",json!([funding_txid,funding_vout,false])).await?.is_null(),"retained lock was not a harmless confirmed-spent outpoint"); + ensure!(ledger.cpfp_package(&fanout_txid).await?.context("package disappeared")?["signed_child_hex"]==package["signed_child_hex"],"abandoned-child repair changed signed bytes"); + } else { + until("confirmed child releases funding wallet reservation",15,||async { + let released=ledger.cpfp_package(&fanout_txid).await?.is_some_and(|p|p["wallet_lock_released"]==true); + let locks=fixture.rpc("listlockunspent",json!([])).await?; + Ok(released&&!locks.as_array().is_some_and(|rows|rows.contains(&outpoint))) + }).await?; + } + fixture.integrity().await?; + eprintln!("live CPFP regtest: two broadcasters recovered funding through eviction and wallet restart, repaired released reservation, replayed exact signed child without its wallet and confirmed it; abandoned_repair={repair_abandoned}, cleanup_deferred={repair_abandoned}"); + ledger.pool.close().await; + Ok::<_,anyhow::Error>(()) + }.await; + if result.is_err() { + eprintln!("{}", fixture.diagnostics()); + } + let cleanup = fixture.cleanup().await; + result.and(cleanup) +} + +/// Land a block whose only payout is a legacy anchored zero-fee CTV fanout, +/// built in process by `ledger` with the servers stopped, and mine it to +/// maturity. Returns the fanout txid and the block height. Shared with the +/// #474 lost `submitpackage` reply case. +pub(crate) async fn land_legacy_zero_fee_fanout( + fixture: &Fixture, + ledger: &Ledger, +) -> Result<(String, u64)> { + let template = fixture + .rpc("getblocktemplate", json!([{"rules":["segwit"]}])) + .await?; + let height = template["height"] + .as_u64() + .context("template height missing")?; + let ntime = template["curtime"] + .as_u64() + .context("template time missing")?; + let target = codec::target_from_compact(codec::parse_u32_hex( + template["bits"].as_str().context("template bits missing")?, + )?)?; + let network = codec::scaled_target_difficulty(&target)?; + let validation = fixture + .rpc("validateaddress", json!([fixture.address])) + .await?; + let script = validation["scriptPubKey"] + .as_str() + .context("payout script missing")?; + ledger + .append( + AcceptedShare { + share_seq: 0, + share_id: format!("cpfp:{}", "12".repeat(32)), + miner_id: fixture.address.clone(), + order_key: fixture.address.clone(), + p2mr_program_hex: script[4..].into(), + share_difficulty: network, + network_difficulty: network, + template_height: height + .checked_sub(1) + .context("template parent height missing")?, + job_id: "legacy-cpfp".into(), + job_issued_at_ms: 1, + accepted_at_ms: 0, + ntime: ntime.try_into()?, + credit_policy: None, + }, + None, + ) + .await?; + let snapshot = ledger.snapshot(network).await?; + let manifest_key = ManifestSigningKey::from_seed_hex(&"11".repeat(32))?; + let ledger_key = ManifestSigningKey::from_seed_hex(&"22".repeat(32))?; + let bundle = qbit_prism::build_audit_bundle_with_ctv_settlement_options( + snapshot.shares.clone(), + FoundBlock { + block_height: height, + coinbase_value_sats: template["coinbasevalue"] + .as_u64() + .context("subsidy missing")?, + network_difficulty: network, + anchor_job_issued_at_ms: snapshot.anchor_ms, + }, + snapshot.prior_balances.clone(), + PayoutPolicy::day_one_default(), + u64::MAX, + SettlementModeConfig::default(), + None, + Some("00".repeat(12)), + codec::witness_merkle_leaves_hex(&codec::transactions_from_template(&template)?), + &manifest_key, + &ledger_key, + )?; + let fanout = bundle + .ctv_fanout_manifest_set + .as_ref() + .context("missing zero-fee fanout")? + .manifests + .first() + .context("missing fanout chunk")?; + ensure!( + fanout.precommitment.fanout_fee_sats == 0 && fanout.precommitment.anchor_vout.is_some(), + "fixture did not build a legacy anchored zero-fee fanout" + ); + let fanout_txid = fanout.fanout_txid.clone(); + let job = codec::Job::from_manifest( + "legacy-cpfp".into(), + &template, + &bundle.signed_coinbase_manifest.manifest, + "00000000", + 8, + 1e-9, + 0.0, + true, + )?; + let mut solved = None; + for nonce in 0u32..10_000 { + let submission = job.assemble_submission( + &"00".repeat(8), + &format!("{ntime:08x}"), + &format!("{nonce:08x}"), + None, + 0, + )?; + if submission.block_pass { + solved = Some(submission); + break; + } + } + let solved = solved.context("regtest proof search exhausted")?; + let block_hash = solved.block_hash_hex.clone(); + let block_bytes = hex::decode(&solved.block_hex)?; + ledger + .enqueue_candidate(Candidate { + block_hash: block_hash.clone(), + block_sha256: Candidate::block_digest_hex(&block_bytes), + job_id: "legacy-cpfp".into(), + payout_revision: snapshot.payout_revision, + window: WindowRef::from_snapshot(&snapshot)?, + bootstrap_share: None, + found_block: bundle.found_block.clone(), + payout_policy: PayoutPolicy::day_one_default(), + ctv: Some(CandidateCtv { + direct_floor_sats: u64::MAX, + settlement_config: SettlementModeConfig::default(), + fanout_fee_policy: None, + }), + audit_builder_version: qbit_prism::AUDIT_BUILDER_VERSION, + signer_keys: SignerKeys::of(&manifest_key, &ledger_key), + leased: false, + coinbase_suffix_hex: "00".repeat(12), + deferred_share: None, + block_bytes, + as_issued_balances: Vec::new(), + }) + .await?; + let candidate = ledger + .claim_candidate(60) + .await? + .context("candidate claim missing")? + .with_bundle(bundle); + ledger + .land_candidate(&candidate, &ledger_key.public_key_hex()) + .await?; + ensure!( + fixture + .rpc("submitblock", json!([solved.block_hex])) + .await? + .is_null(), + "legacy CTV coinbase rejected" + ); + ledger + .finish_candidate_at_revision(&candidate, true, None, snapshot.payout_revision) + .await?; + fixture + .rpc("generatetoaddress", json!([1000, fixture.address])) + .await?; + ledger + .reconcile_blocks_at_revision( + &[BlockObservation { + block_hash, + active: true, + }], + height + 1000, + ledger.payout_revision().await?, + ) + .await?; + Ok((fanout_txid, height)) +} + +async fn spend_reserved_funding(fixture: &Fixture, outpoint: &Value, amount: u64) -> Result<()> { + let output = amount + .checked_sub(100_000) + .context("funding cannot cover fixture spend fee")?; + let mut outputs = serde_json::Map::new(); + outputs.insert( + fixture.address.clone(), + serde_json::from_str(&format!( + "{}.{:08}", + output / 100_000_000, + output % 100_000_000 + ))?, + ); + let unsigned = fixture + .rpc("createrawtransaction", json!([[outpoint], outputs])) + .await?; + let signed = fixture + .rpc("signrawtransactionwithwallet", json!([unsigned])) + .await?; + ensure!( + signed["complete"] == true, + "wallet did not sign the competing funding spend" + ); + fixture + .rpc("sendrawtransaction", json!([signed["hex"]])) + .await?; + fixture + .rpc("generatetoaddress", json!([1, fixture.address])) + .await?; + Ok(()) +} + +async fn stop_broadcasters(fixture: &mut Fixture, fanout: &str) -> Result<()> { + // Reserve a quiet interval before killing processes, so this test does not + // depend on waiting out a 120-second claim lease at every node restart. + until("broadcaster attempt quiescent",20,||async { + Ok(sqlx::query("UPDATE qbit_ctv_fanout_artifacts SET next_broadcast_attempt_at=clock_timestamp()+interval '1 hour' WHERE fanout_txid=$1 AND claim_token IS NULL") + .bind(fanout).execute(&fixture.pool).await?.rows_affected()==1) + }).await?; + for server in &mut fixture.servers { + server.stop(); + } + Ok(()) +} +async fn restart_broadcasters(fixture: &mut Fixture, fanout: &str) -> Result<()> { + sqlx::query("UPDATE qbit_ctv_fanout_artifacts SET next_broadcast_attempt_at=clock_timestamp() WHERE fanout_txid=$1").bind(fanout).execute(&fixture.pool).await?; + for index in 0..2 { + fixture.servers[index] = fixture.start_server_with_sponsorship(index, Some(100_000))?; + } + Ok(()) +} +async fn wait_for_package(fixture: &Fixture, fanout: &str, child: &Value) -> Result<()> { + until("durable package replayed after eviction", 35, || async { + let mempool = fixture.rpc("getrawmempool", json!([])).await?; + Ok(mempool + .as_array() + .is_some_and(|rows| rows.contains(&json!(fanout)) && rows.contains(child))) + }) + .await +} +async fn restart_node(fixture: &mut Fixture, wallet: bool, height: u64) -> Result<()> { + fixture.rpc("stop", json!([])).await?; + until("qbit chain state flushed on shutdown", 20, || { + std::future::ready( + fixture + .node + .child + .try_wait() + .map(|status| status.is_some()) + .map_err(Into::into), + ) + }) + .await?; + let mut node = Command::new(&fixture.qbitd); + node.args([ + "-regtest", + "-server=1", + "-listen=0", + "-dnsseed=0", + "-discover=0", + "-fallbackfee=0.00001", + "-rpcuser=prismtest", + "-rpcpassword=prismtest", + "-txindex=0", + "-persistmempool=0", + ]) + .arg(if wallet { + "-wallet=prism" + } else { + "-disablewallet=1" + }) + .arg(format!("-datadir={}", fixture.directory.path().display())) + .arg(format!("-rpcport={}", fixture.rpc_port)) + .arg(format!("-port={}", free_port()?)); + fixture.node = Process::spawn( + &mut node, + fixture.directory.path().join("qbit-restarted.log"), + )?; + until("qbit restarted after package eviction", 30, || async { + Ok(fixture.rpc("getblockcount", json!([])).await? == json!(height)) + }) + .await?; + ensure!( + fixture + .rpc("getrawmempool", json!([])) + .await? + .as_array() + .is_some_and(Vec::is_empty), + "node restart retained the package mempool" + ); + Ok(()) +} + +async fn ensure_funding_excluded(fixture: &Fixture, txid: &str, vout: u32) -> Result<()> { + let available = fixture + .rpc("listunspent", json!([0, 9_999_999, [], true])) + .await?; + ensure!( + available.as_array().is_some_and(|rows| rows + .iter() + .all(|coin| coin["txid"] != txid || coin["vout"] != vout)), + "wallet coin selection could reuse unconfirmed CPFP funding" + ); + Ok(()) +} diff --git a/crates/qbit-prism-server/tests/support/live_dense_soak.rs b/crates/qbit-prism-server/tests/support/live_dense_soak.rs new file mode 100644 index 000000000..176762521 --- /dev/null +++ b/crates/qbit-prism-server/tests/support/live_dense_soak.rs @@ -0,0 +1,553 @@ +//! #521 scenario 5: dense-cadence landings and a soak, with a real node. +//! +//! Two CPU miners, one on each server, mine the regtest node's easy target at +//! a throttled rate, so the pool's own blocks land seconds apart and every +//! landing bumps the payout revision the other server's work was built on: +//! the #478 path, for as long as `PRISM_DENSE_SOAK_SECONDS` (default 600). +//! Nightly only (`--ignored`); the nightly load-harness workflow runs it. +//! +//! It asserts that: +//! - every block the pool found on live work (an accepted share outside +//! stale grace whose hash meets the network target) is a candidate, unless +//! the block that won its height was captured before that share was +//! accepted: a race already lost when it was found; +//! - every candidate settles: `submitted` on the node's active chain, +//! `orphaned` off it, or abandoned before its offer only because its parent +//! was superseded by a landed pool block at its height; and every block on +//! the node's chain since the soak began is a confirmed pool block; +//! - each server's accepted-block-to-revision-work time (#458) stays within +//! `PRISM_DENSE_SOAK_REBUILD_P99_SECONDS` (default 5) at p99, with no +//! degraded landing; +//! - each server's RSS is flat: the median of the last fifth of the soak is +//! within a quarter (plus 32 MiB) of the median of its second fifth. +//! +//! After the miners stop, seven blocks that are not the pool's bury the last +//! landings past the orphan depth, so a race lost at the very end is proven +//! before the candidates are read. +use super::*; +use num_bigint::BigUint; +use qbit_prism_server::codec::{parse_u32_hex, target_from_compact}; +use std::collections::{BTreeMap, HashMap}; + +const DEFAULT_SOAK_SECONDS: u64 = 600; +const DEFAULT_REBUILD_P99_SECONDS: f64 = 5.0; +const SAMPLE_SECONDS: u64 = 5; +/// Blocks mined after the soak, past `PRISM_CANDIDATE_ORPHAN_CONFIRMATIONS` +/// (6), so a lost race at the end is proven. +const ORPHAN_BURIAL_BLOCKS: u64 = 7; +/// Regtest's constant target. +const REGTEST_BITS: &str = "207fffff"; + +pub(super) fn setting(name: &str, default: T) -> Result +where + T::Err: std::fmt::Display, +{ + match std::env::var(name) { + Ok(value) => value + .trim() + .parse() + .map_err(|error| anyhow::anyhow!("{name}={value:?}: {error}")), + Err(std::env::VarError::NotPresent) => Ok(default), + Err(error) => bail!("{name}: {error}"), + } +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +#[ignore = "nightly #521 scenario 5: a dense-cadence soak of PRISM_DENSE_SOAK_SECONDS (default 600)"] +async fn dense_cadence_soak_lands_every_block_rebuilds_in_budget_and_holds_rss_flat() -> Result<()> +{ + // Selected explicitly, so a missing input fails rather than skipping. + gate::required_inputs( + gate::site!(), + &[gate::Input::QbitdBin, gate::Input::DatabaseUrl], + )?; + let seconds: u64 = setting("PRISM_DENSE_SOAK_SECONDS", DEFAULT_SOAK_SECONDS)?; + ensure!( + seconds >= 60, + "PRISM_DENSE_SOAK_SECONDS must be at least 60, not {seconds}" + ); + let budget: f64 = setting( + "PRISM_DENSE_SOAK_REBUILD_P99_SECONDS", + DEFAULT_REBUILD_P99_SECONDS, + )?; + ensure!( + budget.is_finite() && budget > 0.0, + "PRISM_DENSE_SOAK_REBUILD_P99_SECONDS must be finite and positive" + ); + let Some(mut fixture) = Fixture::open(false).await? else { + bail!("the live fixture's inputs were required above"); + }; + let result = soak(&mut fixture, seconds, budget).await; + if result.is_err() { + eprintln!("{}", fixture.diagnostics()); + } + let cleanup = fixture.cleanup().await; + result.and(cleanup) +} + +/// A throttled miner on server `index` for `seconds`: one hash a second and a +/// pause after each accepted share, so each miner finds a block every few +/// seconds on regtest's easy target and the two together land blocks seconds +/// apart. +fn start_soak_miner(fixture: &mut Fixture, index: usize, seconds: u64) -> Result<()> { + let mut command = Command::new(env!("CARGO_BIN_EXE_qbit-prism-miner")); + command.args([ + "--address", + &format!("127.0.0.1:{}", fixture.stratum[index]), + "--username", + &format!("{}.soak-{index}", fixture.address), + "--threads", + "1", + "--hashes-per-second", + "1", + "--duration-seconds", + &seconds.to_string(), + "--pause-after-share-ms", + "4000", + ]); + fixture.miners.push(Process::spawn( + &mut command, + fixture + .directory + .path() + .join(format!("soak-miner-{index}.json")), + )?); + Ok(()) +} + +/// Resident set size of a process, in KiB, from `/proc`. +fn rss_kib(pid: u32) -> Option { + let status = std::fs::read_to_string(format!("/proc/{pid}/status")).ok()?; + status.lines().find_map(|line| { + line.strip_prefix("VmRSS:")? + .trim() + .strip_suffix("kB")? + .trim() + .parse() + .ok() + }) +} + +fn median(values: &mut [u64]) -> Option { + if values.is_empty() { + return None; + } + values.sort_unstable(); + Some(values[values.len() / 2]) +} + +/// The block solutions a miner found and what the pool answered. +struct MinerBlocks { + found: usize, + accepted: usize, + /// The pool's refusal messages. + refused: Vec, + /// The parent each block was found on, by block hash (display order). + parents: HashMap, +} + +/// [`MinerBlocks`] from a miner's JSON-lines log. +fn miner_blocks(log: &std::path::Path) -> Result { + let text = std::fs::read_to_string(log)?; + let mut block_requests = std::collections::HashSet::new(); + let mut accepted = 0; + let mut refused = Vec::new(); + let mut parents = HashMap::new(); + for line in text.lines() { + let Ok(event) = serde_json::from_str::(line) else { + continue; + }; + match event["event"].as_str() { + Some("submit") if event["block_target_met"] == true => { + if let Some(id) = event["request_id"].as_u64() { + block_requests.insert(id); + } + if let (Some(hash), Some(parent)) = ( + event["block_hash"].as_str(), + event["previousblockhash"].as_str(), + ) { + parents.insert(hash.to_owned(), parent.to_owned()); + } + } + Some("share") + if event["response"]["id"] + .as_u64() + .is_some_and(|id| block_requests.contains(&id)) => + { + if event["accepted"] == true { + accepted += 1; + } else { + refused.push(event["response"]["error"].to_string()); + } + } + _ => {} + } + } + Ok(MinerBlocks { + found: block_requests.len(), + accepted, + refused, + parents, + }) +} + +/// `(le, cumulative count)` buckets of one labelled histogram series, summed +/// over every value of its other labels. +fn histogram(body: &str, name: &str) -> BTreeMap { + let mut buckets = BTreeMap::new(); + let prefix = format!("{name}_bucket{{"); + for line in body.lines().filter(|line| line.starts_with(&prefix)) { + let Some((labels, value)) = line.rsplit_once(' ') else { + continue; + }; + let Some(le) = labels + .split(['{', ',', '}']) + .find_map(|label| label.strip_prefix("le=\"")) + .map(|le| le.trim_end_matches('"').to_owned()) + else { + continue; + }; + *buckets.entry(le).or_insert(0.0) += value.parse::().unwrap_or(0.0); + } + buckets +} + +/// The smallest bucket bound that holds at least `quantile` of the samples: +/// a p99 bound, never an interpolation. `None` without samples. +fn quantile_bound(buckets: &BTreeMap, quantile: f64) -> Option { + let total = *buckets.get("+Inf")?; + if total == 0.0 { + return None; + } + let mut bounds: Vec<(f64, f64)> = buckets + .iter() + .map(|(le, count)| { + ( + if le == "+Inf" { + f64::INFINITY + } else { + le.parse().unwrap_or(f64::INFINITY) + }, + *count, + ) + }) + .collect(); + bounds.sort_by(|a, b| a.0.total_cmp(&b.0)); + bounds + .into_iter() + .find(|(_, count)| *count >= quantile * total) + .map(|(bound, _)| bound) +} + +fn labelled_count(body: &str, name: &str, label: &str) -> f64 { + let prefix = format!("{name}_count{{"); + body.lines() + .filter(|line| line.starts_with(&prefix) && line.contains(label)) + .filter_map(|line| line.rsplit_once(' ')?.1.parse::().ok()) + .sum() +} + +async fn metrics_body(fixture: &Fixture, index: usize) -> Result { + Ok(fixture + .client + .get(format!("http://127.0.0.1:{}/metrics", fixture.api[index])) + .send() + .await? + .error_for_status()? + .text() + .await?) +} + +pub(super) async fn soak(fixture: &mut Fixture, seconds: u64, budget: f64) -> Result<()> { + let start_height = fixture + .rpc("getblockcount", json!([])) + .await? + .as_u64() + .context("height")?; + let pids: Vec = fixture + .servers + .iter() + .map(|server| server.child.lock().id()) + .collect(); + for index in 0..2 { + start_soak_miner(fixture, index, seconds + 30)?; + } + let started = std::time::Instant::now(); + let mut rss: Vec> = vec![Vec::new(); pids.len()]; + let mut heights = Vec::new(); + while started.elapsed() < Duration::from_secs(seconds) { + tokio::time::sleep(Duration::from_secs(SAMPLE_SECONDS)).await; + let at = started.elapsed().as_secs(); + for (index, pid) in pids.iter().enumerate() { + if let Some(kib) = rss_kib(*pid) { + rss[index].push((at, kib)); + } + } + if let Ok(height) = fixture.rpc("getblockcount", json!([])).await { + heights.push((at, height.as_u64().unwrap_or_default())); + } + for (index, server) in fixture.servers.iter().enumerate() { + ensure!( + server.child.try_wait()?.is_none(), + "server {index} exited during the soak" + ); + } + } + fixture.quiesce().await?; + let end_height = fixture + .rpc("getblockcount", json!([])) + .await? + .as_u64() + .context("height")?; + let landed = end_height - start_height; + ensure!( + landed as f64 >= seconds as f64 / 30.0, + "only {landed} blocks landed in a {seconds} s soak: the cadence was not dense" + ); + let cadence = seconds as f64 / landed as f64; + // A lost same-height race is proven orphaned only once the winning branch + // is buried past the orphan depth, and the chain stops with the miners: + // bury the last landings under blocks that are not the pool's, then let + // every candidate settle. + let burier = fixture.rpc("getnewaddress", json!(["", "p2mr"])).await?; + fixture + .rpc("generatetoaddress", json!([ORPHAN_BURIAL_BLOCKS, burier])) + .await?; + until("every candidate settled", 120, || async { + Ok(sqlx::query_scalar::<_, i64>( + "SELECT count(*) FROM qbit_block_candidate_outbox \ + WHERE state NOT IN ('submitted','orphaned','abandoned')", + ) + .fetch_one(&fixture.pool) + .await? + == 0) + }) + .await?; + + // Every found block the pool accepted is a candidate. + let mut found = 0; + let mut accepted = 0; + let mut refused: BTreeMap = BTreeMap::new(); + let mut parents = HashMap::new(); + for miner in &fixture.miners { + let blocks = miner_blocks(&miner.log)?; + found += blocks.found; + accepted += blocks.accepted; + for reason in blocks.refused { + *refused.entry(reason).or_default() += 1; + } + parents.extend(blocks.parents); + } + let rows: Vec<(String, String, Option)> = sqlx::query_as( + "SELECT block_hash,state,offer_outcome FROM qbit_block_candidate_outbox ORDER BY created_at", + ) + .fetch_all(&fixture.pool) + .await?; + // Every block the pool found on live work is a candidate: each accepted + // share not credited under stale grace whose hash meets the network + // target. A solution on a tip already replaced is accepted under stale + // grace and is no block to land; one on superseded payout work is + // captured although its share is refused (#478), so candidates may + // outnumber these, but never the solutions the miners found. + let network_target = target_from_compact(parse_u32_hex(REGTEST_BITS)?)?; + let live: Vec = sqlx::query_scalar::<_, String>( + "SELECT share_id FROM qbit_share_ledger WHERE accepted AND credit_policy IS NULL \ + AND writer_id LIKE 'live-%'", + ) + .fetch_all(&fixture.pool) + .await? + .into_iter() + .filter_map(|share_id| share_id.rsplit_once(':').map(|(_, hash)| hash.to_owned())) + .filter(|hash| { + BigUint::parse_bytes(hash.as_bytes(), 16).is_some_and(|value| value <= network_target) + }) + .collect(); + let candidates: std::collections::HashSet<&str> = + rows.iter().map(|(hash, _, _)| hash.as_str()).collect(); + let missing: Vec<&String> = live + .iter() + .filter(|hash| !candidates.contains(hash.as_str())) + .collect(); + // A block not captured is a race already lost only when the block that + // won its height was captured before this share was accepted: the other + // server's landing had replaced the parent, and this server had not yet + // observed it. Any other uncaptured block is a lost block. + let mut lost_races = 0usize; + let mut uncaptured = Vec::new(); + for hash in &missing { + let row: Option<(i64, chrono::DateTime)> = sqlx::query_as( + "SELECT template_height::bigint, accepted_at FROM qbit_share_ledger \ + WHERE share_id LIKE '%:' || $1", + ) + .bind(hash.as_str()) + .fetch_optional(&fixture.pool) + .await?; + let Some((height, accepted_at)) = row else { + uncaptured.push(format!("{hash}: no share row")); + continue; + }; + let winner: String = + serde_json::from_value(fixture.rpc("getblockhash", json!([height + 1])).await?)?; + let captured: Option> = sqlx::query_scalar( + "SELECT created_at FROM qbit_block_candidate_outbox WHERE block_hash=$1", + ) + .bind(&winner) + .fetch_optional(&fixture.pool) + .await?; + match captured { + Some(at) if winner != **hash && at <= accepted_at => lost_races += 1, + _ => uncaptured.push(format!( + "{hash} at height {}: accepted {accepted_at}, the height's winner {winner} \ + captured {captured:?}", + height + 1 + )), + } + } + ensure!( + uncaptured.is_empty(), + "{} block(s) found on live work were never captured:\n{}", + uncaptured.len(), + uncaptured.join("\n") + ); + ensure!( + rows.len() <= found, + "{} candidates but the miners found only {found} block solutions", + rows.len() + ); + let mut states: HashMap<&str, usize> = HashMap::new(); + for (_, state, _) in &rows { + *states.entry(state.as_str()).or_default() += 1; + } + // A candidate ends submitted (landed), orphaned (proven lost after its + // offer), or abandoned before its offer because its parent was no + // longer the tip: a same-height race the other server's block already + // won, which must then be what the chain holds at that height. Abandoned + // for any other reason -- a superseded payout revision with capture off + // (#478) -- is a found block the pool threw away. + let abandoned: Vec<(String, Option)> = sqlx::query_as( + "SELECT block_hash,last_error FROM qbit_block_candidate_outbox WHERE state='abandoned'", + ) + .fetch_all(&fixture.pool) + .await?; + for (hash, reason) in &abandoned { + ensure!( + reason.as_deref() == Some("parent superseded"), + "candidate {hash} was abandoned for {reason:?}, not a superseded parent" + ); + // Its height from the parent its miner built it on, not from its + // share: a block captured on superseded payout work (#478) has no + // credited share, its share deferred until the block confirms (never, + // once abandoned) or, from block-only work, never credited. + let parent = parents + .get(hash) + .with_context(|| format!("abandoned candidate {hash} is no block a miner found"))?; + let height = fixture.rpc("getblockheader", json!([parent])).await?["height"] + .as_i64() + .with_context(|| { + format!("abandoned candidate {hash}: parent {parent} has no height") + })? + + 1; + let winner: String = + serde_json::from_value(fixture.rpc("getblockhash", json!([height])).await?)?; + ensure!( + winner != *hash + && rows + .iter() + .any(|(row, state, _)| *row == winner && state == "submitted"), + "abandoned candidate {hash}: height {height} is held by {winner}, not a landed pool block" + ); + } + ensure!( + rows.iter().all(|(_, state, _)| { + state == "submitted" || state == "orphaned" || state == "abandoned" + }), + "a candidate did not settle: {states:?}" + ); + // Every pool block on the active chain is credited, and every block on + // it since the soak began is the pool's. + let chain: HashMap = sqlx::query_as::<_, (String, String)>( + "SELECT block_hash,chain_state FROM qbit_pool_blocks", + ) + .fetch_all(&fixture.pool) + .await? + .into_iter() + .collect(); + let mut active = std::collections::HashSet::new(); + for height in start_height + 1..=end_height { + let hash: String = + serde_json::from_value(fixture.rpc("getblockhash", json!([height])).await?)?; + ensure!( + chain.get(&hash).map(String::as_str) == Some("confirmed"), + "block {hash} at height {height} is on the node's chain but not a confirmed pool block" + ); + active.insert(hash); + } + for (hash, state, _) in &rows { + match state.as_str() { + "submitted" => ensure!( + active.contains(hash), + "submitted candidate {hash} is not on the node's active chain" + ), + // A lost same-height race: the pool's own sibling won the height. + _ => ensure!( + !active.contains(hash), + "orphaned candidate {hash} is on the node's active chain" + ), + } + } + + // Rebuilds within budget, on each server. + let family = "qbit_prism_accepted_block_to_revision_work_seconds"; + let mut rebuild_p99 = Vec::new(); + for index in 0..fixture.servers.len() { + let body = metrics_body(fixture, index).await?; + let buckets = histogram(&body, family); + let bound = quantile_bound(&buckets, 0.99) + .with_context(|| format!("server {index} recorded no {family} sample"))?; + let samples = buckets.get("+Inf").copied().unwrap_or(0.0); + ensure!( + bound <= budget, + "server {index}: {family} p99 is above {bound} s over {samples} landings, over the \ + {budget} s budget" + ); + let degraded = labelled_count(&body, family, "result=\"degraded\""); + ensure!( + degraded == 0.0, + "server {index}: {degraded} degraded landing(s)" + ); + rebuild_p99.push((bound, samples)); + } + + // RSS flat, per server. + let mut rss_summary = Vec::new(); + for (index, series) in rss.iter().enumerate() { + let fifth = seconds / 5; + let window = |from: u64, to: u64| -> Vec { + series + .iter() + .filter(|(at, _)| *at >= from && *at < to) + .map(|(_, kib)| *kib) + .collect() + }; + let early = median(&mut window(fifth, 2 * fifth)) + .with_context(|| format!("server {index}: no RSS sample in the second fifth"))?; + let late = median(&mut window(4 * fifth, seconds + 1)) + .with_context(|| format!("server {index}: no RSS sample in the last fifth"))?; + let ceiling = early + early / 4 + 32 * 1024; + ensure!( + late <= ceiling, + "server {index}: RSS grew from a median {early} KiB to {late} KiB, over {ceiling} KiB" + ); + rss_summary.push((early, late)); + } + eprintln!( + "live dense soak: {seconds} s, {landed} blocks landed (one every {cadence:.1} s), {found} \ + block solutions found, {accepted} accepted ({} on live work, {lost_races} of them races \ + already lost when found), refusals {refused:?}, candidates {states:?}; \ + revision work p99 \ + bound (s, landings) per server {rebuild_p99:?}; RSS median KiB (second fifth, last \ + fifth) per server {rss_summary:?}; heights {heights:?}", + live.len() + ); + Ok(()) +} diff --git a/crates/qbit-prism-server/tests/support/live_disk_exhaustion.rs b/crates/qbit-prism-server/tests/support/live_disk_exhaustion.rs new file mode 100644 index 000000000..761cd10e8 --- /dev/null +++ b/crates/qbit-prism-server/tests/support/live_disk_exhaustion.rs @@ -0,0 +1,582 @@ +//! #575 item 5: PostgreSQL runs out of disk while the pool is mining. +//! +//! The test's own PostgreSQL 16 cluster keeps its whole data directory +//! (tables, indexes and `pg_wal`) on the disk-full injector's small +//! filesystem, created without root (see `disk_full_injector.rs`); its log +//! and socket live off it, as an operator's would. Both servers of the live +//! fixture use that cluster. +//! +//! Test clients mine shares that are not blocks on both servers while the +//! injector fills the filesystem to `ENOSPC`, keep mining through the full +//! state until a paging rule has fired on each server (bounded), then the +//! injector frees the space. If PostgreSQL itself stopped (here it does: a +//! `PANIC` on a WAL write, then a crash recovery that cannot write either), +//! it is started again as its supervisor would; the PRISM servers are never +//! restarted or repaired. Every phase is bounded. +//! +//! It asserts that: +//! - every share a client saw acknowledged, before, during and after the full +//! state, is in the ledger as accepted: no acknowledgement outran its +//! durability; +//! - the full state is refused with explicit rejections, some with the +//! reasons `PrismShareAppendFailures` counts, and that rule's input rises; +//! no share refused as `ledger-confirmation-failed` (the ledger did not +//! record it) is credited; no submission waited past the answer bound; +//! - the outage pages on every server through its own rule and at least one +//! more: each condition, mirrored from `docs/prism-native-alert-rules.json`, +//! holds on every scrape for its `for`. Since #581 that is +//! `PrismDatabaseUnavailable` on the live database collector, beside the +//! share-refusal rules, whose counters are rendered at scrape time and no +//! longer gated on the snapshot's freshness, and +//! `PrismBlockCandidateMetricsUnavailable`; only `PrismMetricsSnapshotStale` +//! still flaps with the snapshot's freshness; +//! - once space is freed, both original server processes accept shares again +//! within a bound, a block found afterwards lands on the node, and the +//! carry-forward integrity report is clean. +use super::alert_rules::{rule, scrape, snapshot_stale, Mirror, Scrape, Verdict}; +use super::disk_full_injector::DiskFullInjector; +use super::private_postgres::{ClusterOptions, PrivateCluster}; +use super::share_client::{start_share_only_servers, Answer, Proof, ShareClient, Submitted}; +use super::*; +use std::{ + collections::{BTreeMap, BTreeSet}, + sync::{ + atomic::{AtomicBool, Ordering}, + Arc, + }, +}; + +/// The filesystem's size. `initdb` with 1 MiB WAL segments leaves most of it +/// free for the fixture's schema, and the ballast takes the rest. +const VOLUME_MIB: u64 = 160; +/// Test clients per server. +const CLIENTS_PER_SERVER: usize = 2; +/// Shares each client must have acknowledged before the fill and after the +/// recovery, each within [`STEADY_BOUND`]. +const STEADY_SHARES: usize = 10; +const STEADY_BOUND: Duration = Duration::from_secs(120); +/// How long the fill may take to be refused on both servers. +const FILL_BOUND: Duration = Duration::from_secs(180); +/// How long the full state may last once both servers refuse, waiting for +/// the outage's rules to fire on each (see [`outage_paged`]): their +/// 3-minute `for` at most, plus time for PostgreSQL to go down after the +/// first refusal. +const FULL_BOUND: Duration = Duration::from_secs(360); + +const REJECTIONS: &str = "qbit_prism_rejections_total"; +const ACCEPTED: &str = "qbit_prism_accepted_shares_total"; + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +#[ignore = "nightly #575: PostgreSQL's data volume fills under mining; needs fuse2fs and /dev/fuse"] +async fn postgres_volume_full_refuses_shares_cleanly_and_recovers_once_space_is_freed() -> Result<()> +{ + // Selected explicitly, so a missing input fails rather than skipping. + let inputs = gate::required_inputs( + gate::site!(), + &[ + gate::Input::QbitdBin, + gate::Input::DatabaseUrl, + gate::Input::PgBinDir, + ], + )?; + let bin = PathBuf::from(&inputs[2]); + let reasons = rule("PrismShareAppendFailures", &[REJECTIONS, "reason_id=~"])? + .label_alternatives("reason_id")?; + let mirrors = paging_mirrors(&reasons)?; + // As in the failover drills: wait for the fixture guard once before the + // cluster's loopback connections start. + drop(SERIAL.lock().await); + let disk = DiskFullInjector::mount(VOLUME_MIB)?; + let data = disk.volume().join("data"); + std::fs::create_dir(&data)?; + // 1 MiB WAL segments, so WAL grows and recycles in small steps; the log + // and socket stay off the volume. + let mut cluster = PrivateCluster::start(ClusterOptions { + bin, + data, + home: disk.home().to_path_buf(), + initdb: &["--wal-segsize=1"], + settings: "-c min_wal_size=4MB -c max_wal_size=16MB", + env: Vec::new(), + })?; + let Some(mut fixture) = Fixture::open_on_database(false, false, Some(&cluster.url())).await? + else { + bail!("the live fixture's inputs were required above"); + }; + let result = exhaust(&mut fixture, &disk, &mut cluster, &reasons, &mirrors).await; + if result.is_err() { + eprintln!("{}", fixture.diagnostics()); + eprintln!("{}", cluster.diagnostics()); + } + // The ballast must be gone, and the cluster up, for cleanup's DROP SCHEMA. + let _ = disk.stop(); + let _ = cluster.ensure_running(); + let cleanup = fixture.cleanup().await; + drop(cluster); + drop(disk); + result.and(cleanup) +} + +/// One submission, the server it went to and the phase it was made in. +struct Record { + server: usize, + phase: Phase, + submitted: Submitted, +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord)] +enum Phase { + Before, + Full, + After, +} + +/// A test client: its server, its worker, its live session if it has one, +/// and the shares acknowledged in the current phase. +struct Miner { + server: usize, + username: String, + session: Option, + accepted: usize, +} + +async fn exhaust( + f: &mut Fixture, + disk: &DiskFullInjector, + cluster: &mut PrivateCluster, + reasons: &BTreeSet, + mirrors: &[Mirror], +) -> Result<()> { + start_share_only_servers(f, &[(0, Vec::new()), (1, Vec::new())]).await?; + let mut records = Vec::new(); + let mut miners = Vec::new(); + for server in 0..2 { + for slot in 0..CLIENTS_PER_SERVER { + miners.push(Miner { + server, + username: format!("{}.disk-{server}-{slot}", f.address), + session: None, + accepted: 0, + }); + } + } + + // Before: every client mines on a healthy database. + let miners = steady(f, miners, Phase::Before, &mut records).await?; + let refused_before = alert_input(f, reasons).await?; + ensure!( + records + .iter() + .all(|record| record.submitted.answer.accepted()), + "a share was refused before the volume filled: {}", + summary(&records) + ); + + // Full: fill the volume and keep mining until a paging rule fires on + // both servers. + let mut baseline = Vec::new(); + for (server, port) in f.api.into_iter().enumerate() { + baseline.push(scrape(&f.client, server, port).await?); + } + let free_before = disk.free_bytes()?; + let ballast = disk.start()?; + eprintln!( + "disk exhaustion: {free_before} bytes were free; ballast {ballast} bytes; {} left", + disk.free_bytes()? + ); + let (miners, scrapes) = full(f, miners, mirrors, baseline, &mut records).await?; + let refused_full = alert_input(f, reasons).await?; + let postgres_alive_when_full = cluster.running()?; + + // After: free the space. PostgreSQL is started again only if it stopped. + disk.stop()?; + let restarted = cluster.ensure_running()?; + for (index, server) in f.servers.iter().enumerate() { + ensure!( + server.child.try_wait()?.is_none(), + "server {index} exited during the full state" + ); + } + let recovery_started = Instant::now(); + let miners = steady(f, miners, Phase::After, &mut records).await?; + let recovered_in = recovery_started.elapsed(); + drop(miners); + + // A block found after the recovery lands. + let mut miner = + ShareClient::connect(f.stratum[0], &format!("{}.disk-block", f.address)).await?; + let block = miner.submit(Proof::Block).await?; + ensure!( + block.answer.accepted(), + "post-recovery block: {}", + block.answer + ); + until("the post-recovery block on the node", 60, || async { + Ok(f.rpc("getbestblockhash", json!([])).await? == json!(block.hash)) + }) + .await?; + f.quiesce().await?; + + // Every acknowledged share is durable and accepted; no definite failure + // is credited. + let ledger: BTreeMap = + sqlx::query_as::<_, (String, bool)>("SELECT share_id, accepted FROM qbit_share_ledger") + .fetch_all(&f.pool) + .await? + .into_iter() + .collect(); + let lost: Vec<_> = records + .iter() + .filter(|record| record.submitted.answer.accepted()) + .filter(|record| ledger.get(&record.submitted.share_id) != Some(&true)) + .map(|record| format!("{:?} {}", record.phase, record.submitted.share_id)) + .collect(); + ensure!( + lost.is_empty(), + "acknowledged shares missing from the ledger: {lost:?}" + ); + let credited_failures: Vec<_> = records + .iter() + .filter(|record| record.submitted.answer.reason_id() == Some("ledger-confirmation-failed")) + .filter(|record| ledger.get(&record.submitted.share_id) == Some(&true)) + .map(|record| record.submitted.share_id.clone()) + .collect(); + ensure!( + credited_failures.is_empty(), + "shares refused as ledger-confirmation-failed were credited: {credited_failures:?}" + ); + + // The full state was refused cleanly, the alert's input rose, and a + // paging rule fired on every server. + let full_records: Vec<_> = records + .iter() + .filter(|record| record.phase == Phase::Full) + .collect(); + ensure!( + full_records.iter().any(|record| record + .submitted + .answer + .reason_id() + .is_some_and(|reason| reasons.contains(reason))), + "no share was refused with a reason {reasons:?} alerts on: {}", + summary(&records) + ); + let timed_out: Vec<_> = full_records + .iter() + .filter(|record| matches!(record.submitted.answer, Answer::TimedOut)) + .map(|record| record.submitted.share_id.clone()) + .collect(); + ensure!( + timed_out.is_empty(), + "submissions waited past the answer bound: {timed_out:?}" + ); + ensure!( + refused_full > refused_before, + "the PrismShareAppendFailures input did not rise: {refused_before} before, {refused_full} when full" + ); + let verdict = Verdict::of(&scrapes, &[0, 1], mirrors); + ensure!( + outage_paged(&verdict), + "the outage did not page on every server through PrismDatabaseUnavailable and at least one more rule (#581):\n{}\n{}", + verdict.report, + gates(&scrapes) + ); + f.integrity().await?; + eprintln!("disk exhaustion alert conditions:\n{}", verdict.report); + eprintln!( + "disk exhaustion: {}; alert input {refused_before} -> {refused_full}; PostgreSQL {} when full{}; recovered in {:.1}s; {} ledger rows", + summary(&records), + if postgres_alive_when_full { "up" } else { "down" }, + if restarted { ", restarted after freeing space" } else { "" }, + recovered_in.as_secs_f64(), + ledger.len(), + ); + eprintln!("disk exhaustion PostgreSQL log:\n{}", cluster.errors()); + Ok(()) +} + +/// Every miner mines on its server until it has `target` acknowledgements +/// in this phase (or, with `None`, until `stop`), reconnecting after a lost +/// session, and stops at `deadline` whatever it has. +async fn mine( + f: &Fixture, + miners: Vec, + phase: Phase, + target: Option, + deadline: Instant, + stop: &Arc, + records: &mut Vec, +) -> Result> { + let mut tasks = Vec::new(); + for mut miner in miners { + let port = f.stratum[miner.server]; + let stop = stop.clone(); + tasks.push(tokio::spawn(async move { + let mut submitted = Vec::new(); + miner.accepted = 0; + while !stop.load(Ordering::SeqCst) + && Instant::now() < deadline + && target.is_none_or(|target| miner.accepted < target) + { + let mut session = match miner.session.take() { + Some(session) => session, + None => match ShareClient::connect(port, &miner.username).await { + Ok(session) => session, + Err(_) => { + tokio::time::sleep(Duration::from_millis(500)).await; + continue; + } + }, + }; + let outcome = session.submit(Proof::Share).await?; + match outcome.answer { + Answer::Accepted => miner.accepted += 1, + // A refused miner keeps mining; this only paces the log. + _ => tokio::time::sleep(Duration::from_millis(20)).await, + } + // A session that answered is kept; a lost one is replaced. + if matches!(outcome.answer, Answer::Accepted | Answer::Rejected(_)) { + miner.session = Some(session); + } + submitted.push(outcome); + } + Ok::<_, anyhow::Error>((miner, submitted)) + })); + } + let mut next = Vec::new(); + for task in tasks { + let (miner, submitted) = task.await??; + records.extend(submitted.into_iter().map(|submitted| Record { + server: miner.server, + phase, + submitted, + })); + next.push(miner); + } + Ok(next) +} + +/// A phase on a healthy database: every miner gets [`STEADY_SHARES`] +/// acknowledged within [`STEADY_BOUND`]. +async fn steady( + f: &Fixture, + miners: Vec, + phase: Phase, + records: &mut Vec, +) -> Result> { + let never = Arc::new(AtomicBool::new(false)); + let deadline = Instant::now() + STEADY_BOUND; + let miners = mine( + f, + miners, + phase, + Some(STEADY_SHARES), + deadline, + &never, + records, + ) + .await?; + ensure!( + miners.iter().all(|miner| miner.accepted >= STEADY_SHARES), + "{phase:?}: not every miner had {STEADY_SHARES} shares acknowledged within {STEADY_BOUND:?}: {}", + summary(records) + ); + Ok(miners) +} + +/// The full state: mine while scraping both servers until a paging rule +/// fires on each, bounded by [`FILL_BOUND`] for both to refuse and +/// [`FULL_BOUND`] after that. Returns the miners and every scrape, after +/// the pre-fill `baseline`. +async fn full( + f: &Fixture, + miners: Vec, + mirrors: &[Mirror], + baseline: Vec, + records: &mut Vec, +) -> Result<(Vec, Vec)> { + let started = Instant::now(); + let stop = Arc::new(AtomicBool::new(false)); + // Refusals are observed on the servers' own counters, independent of + // the clients' answers. + let before: Vec = (0..2) + .map(|server| { + baseline + .iter() + .filter(|s| s.server == server) + .map(|s| s.sum(REJECTIONS)) + .fold(0.0, f64::max) + }) + .collect(); + let sampler = async { + let mut scrapes = baseline; + let mut refused_at = [None; 2]; + for round in 0u64.. { + for (server, port) in f.api.into_iter().enumerate() { + if let Ok(scrape) = scrape(&f.client, server, port).await { + if scrape.sum(REJECTIONS) > before[server] && refused_at[server].is_none() { + refused_at[server] = Some(Instant::now()); + } + scrapes.push(scrape); + } + } + let both = refused_at[0].zip(refused_at[1]).map(|(a, b)| a.max(b)); + let done = match both { + None => started.elapsed() >= FILL_BOUND, + // Evaluated every five seconds: the rules' `for` is minutes. + Some(at) => { + at.elapsed() >= FULL_BOUND + || (round % 20 == 0 + && outage_paged(&Verdict::of(&scrapes, &[0, 1], mirrors))) + } + }; + if done { + break; + } + tokio::time::sleep(Duration::from_millis(250)).await; + } + stop.store(true, Ordering::SeqCst); + (refused_at.iter().all(Option::is_some), scrapes) + }; + let deadline = started + FILL_BOUND + FULL_BOUND + Duration::from_secs(60); + let (miners, (refused, scrapes)) = tokio::join!( + mine(f, miners, Phase::Full, None, deadline, &stop, records), + sampler + ); + let miners = miners?; + ensure!( + refused, + "both servers did not refuse within {FILL_BOUND:?} of filling the volume: {}", + summary(records) + ); + Ok((miners, scrapes)) +} + +/// Whether the outage paged on every server through its own rule and at +/// least one more. Before #581 only `PrismBlockCandidateMetricsUnavailable` +/// held: the metrics snapshot is published by the health probe, which still +/// publishes when it fails, so its freshness flapped and every rule gated on +/// a fresh snapshot flapped with it. +fn outage_paged(verdict: &Verdict) -> bool { + verdict + .fired + .values() + .all(|rules| rules.contains("PrismDatabaseUnavailable") && rules.len() >= 2) +} + +/// The rules that can page for this state, mirrored. With PostgreSQL down, +/// `PrismDatabaseUnavailable` holds on the database collector, which is live +/// at scrape time, as does `PrismBlockCandidateMetricsUnavailable`; the +/// share-refusal rules read the live share counters and are no longer gated +/// on the snapshot (#581); `PrismMetricsSnapshotStale` still flaps with the +/// snapshot's freshness. +fn paging_mirrors(reasons: &BTreeSet) -> Result> { + fn increase(base: &Scrape, s: &Scrape, reason: &str) -> f64 { + let count = |scrape: &Scrape| { + scrape + .by_label(REJECTIONS, "reason_id") + .get(reason) + .copied() + .unwrap_or(0.0) + }; + count(s) - count(base) + } + let reasons = reasons.clone(); + Ok(vec![ + Mirror::new( + "PrismShareAppendFailures", + &[REJECTIONS, "> bool 0"], + move |base, s| reasons.iter().any(|reason| increase(base, s, reason) > 0.0), + )?, + Mirror::new( + "PrismRejectRatioByReasonHigh", + &["> bool 0.05", ">= 100"], + |base, s| { + let total = (s.sum(ACCEPTED) - base.sum(ACCEPTED)) + + (s.sum(REJECTIONS) - base.sum(REJECTIONS)); + total >= 100.0 + && s.by_label(REJECTIONS, "reason_id") + .keys() + .any(|reason| increase(base, s, reason) / total > 0.05) + }, + )?, + snapshot_stale()?, + Mirror::new( + "PrismDatabaseUnavailable", + &["collector=\"database\"} == bool 0", "up{"], + // A scrape that answered is the rule's `up == 1`. + |_, s| { + s.labelled("qbit_prism_collector_available", "collector", "database") == Some(0.0) + }, + )?, + Mirror::new( + "PrismBlockCandidateMetricsUnavailable", + &["collector=\"database\"} == bool 0"], + |_, s| { + s.labelled("qbit_prism_collector_available", "collector", "database") == Some(0.0) + }, + )?, + ]) +} + +/// How each server's snapshot gate and database collector changed. +fn gates(scrapes: &[Scrape]) -> String { + (0..2) + .map(|server| { + let mut changes = Vec::new(); + let mut previous = None; + for scrape in scrapes.iter().filter(|s| s.server == server) { + let state = ( + scrape.value("qbit_prism_metrics_snapshot_available"), + scrape.value("qbit_prism_metrics_snapshot_stale"), + scrape.labelled("qbit_prism_collector_available", "collector", "database"), + ); + if previous != Some(state) { + changes.push(format!("{state:?}")); + previous = Some(state); + } + } + format!( + "server-{server} (snapshot available, stale, database collector): {}", + changes.join(" -> ") + ) + }) + .collect::>() + .join("\n") +} + +/// The `PrismShareAppendFailures` input across both servers. +async fn alert_input(f: &Fixture, reasons: &BTreeSet) -> Result { + let mut total = 0.0; + for (server, port) in f.api.into_iter().enumerate() { + let refused = scrape(&f.client, server, port) + .await? + .by_label(REJECTIONS, "reason_id"); + total += reasons + .iter() + .filter_map(|reason| refused.get(reason)) + .sum::(); + } + Ok(total) +} + +/// Counts per phase, server and answer, for reports and failures. +fn summary(records: &[Record]) -> String { + let mut counts: BTreeMap<(Phase, usize, String), usize> = BTreeMap::new(); + for record in records { + *counts + .entry(( + record.phase, + record.server, + record.submitted.answer.to_string(), + )) + .or_default() += 1; + } + counts + .iter() + .map(|((phase, server, answer), count)| { + format!("{phase:?}/server-{server} {answer}={count}") + }) + .collect::>() + .join(", ") +} diff --git a/crates/qbit-prism-server/tests/support/live_fault_seams.rs b/crates/qbit-prism-server/tests/support/live_fault_seams.rs new file mode 100644 index 000000000..9da76c457 --- /dev/null +++ b/crates/qbit-prism-server/tests/support/live_fault_seams.rs @@ -0,0 +1,589 @@ +//! Fault seams for the #474 live cases, shared by its lost-reply cases (C) +//! and its async-promotion case (B). +//! +//! - [`RpcReplyHold`]: a JSON-RPC proxy between the servers and the node that +//! records every call and its answer and, when armed for a method, lets +//! the next such call run on the node and then withholds the node's reply +//! until the test releases it. The node has performed the side effect; +//! the calling server has not heard so. +//! - [`JournalGate`]: a database trigger that holds a chosen row update on a +//! lock the test takes, so a frontend cannot journal an outcome while the +//! test kills it. Crash ordering comes from this barrier, not from sleeps. +//! - [`kill_frontend`], [`end_frontend_backends`] and [`start_frontend`]: +//! `SIGKILL` one server, terminate its database backends, and restart it +//! pointed at a proxy. +//! - [`StratumRelay`]: a Stratum proxy between a miner and a server that, +//! when armed for a request id, withholds the server's answer and closes +//! both sides, so the miner never receives an acknowledgement the server +//! sent only after its ledger outcome was durable. +use super::*; +use std::sync::Arc; +use tokio::{ + io::AsyncReadExt, + net::{tcp::OwnedWriteHalf, TcpStream}, + sync::{oneshot, watch}, +}; + +/// One JSON-RPC call a server made through [`RpcReplyHold`] and the node's +/// answer, `None` until it arrived (or when it never did). +#[derive(Clone, Debug)] +pub(crate) struct RpcCall { + pub method: String, + pub params: Value, + pub reply: Option, +} + +/// A reply the node sent and the proxy withholds: the call's parameters, +/// the node's full answer, and the release. +pub(crate) struct HeldReply { + pub params: Value, + pub reply: Value, + pub release: oneshot::Sender<()>, +} + +struct RpcState { + upstream: u16, + calls: Mutex>, + trap: Mutex)>>, + /// Flipped at drop to release every held reply. + released: watch::Sender, +} + +/// A transparent HTTP/1.1 JSON-RPC proxy in front of the node that can +/// withhold the reply to one call after the node answered it. Only the held +/// connection waits; every other call passes. +pub(crate) struct RpcReplyHold { + pub port: u16, + state: Arc, + task: tokio::task::JoinHandle<()>, +} + +impl RpcReplyHold { + pub(crate) async fn start(upstream: u16) -> Result { + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await?; + let port = listener.local_addr()?.port(); + let state = Arc::new(RpcState { + upstream, + calls: Mutex::new(Vec::new()), + trap: Mutex::new(None), + released: watch::Sender::new(false), + }); + let shared = state.clone(); + let task = tokio::spawn(async move { + while let Ok((downstream, _)) = listener.accept().await { + let state = shared.clone(); + tokio::spawn(async move { + // A failed connection is what the server sees as a + // transport failure; nothing else to report. + let _ = relay_rpc(state, downstream).await; + }); + } + }); + Ok(Self { port, state, task }) + } + + /// Withhold the reply to the next call of `method`, once the node has + /// answered it. + pub(crate) fn arm(&self, method: &str) -> oneshot::Receiver { + let (sender, receiver) = oneshot::channel(); + *self.state.trap.lock().unwrap() = Some((method.into(), sender)); + receiver + } + + /// Every call of `method` so far, in the order the proxy read them. + pub(crate) fn calls(&self, method: &str) -> Vec { + self.state + .calls + .lock() + .unwrap() + .iter() + .filter(|call| call.method == method) + .cloned() + .collect() + } +} + +impl Drop for RpcReplyHold { + fn drop(&mut self) { + self.state.released.send_replace(true); + self.task.abort(); + } +} + +/// Relays one server connection, one request and reply at a time. Each +/// request goes to the node on a new connection: a held reply can outlast +/// qbitd's idle timeout, and the next request must not be written into a +/// connection the node closed meanwhile. +async fn relay_rpc(state: Arc, downstream: TcpStream) -> Result<()> { + let (down_read, mut down_write) = downstream.into_split(); + let mut down_read = BufReader::new(down_read); + loop { + let Some((request, body)) = read_http(&mut down_read).await? else { + return Ok(()); + }; + let call: Value = serde_json::from_slice(&body).unwrap_or_default(); + let method = call["method"].as_str().unwrap_or_default().to_owned(); + let index = { + let mut calls = state.calls.lock().unwrap(); + calls.push(RpcCall { + method: method.clone(), + params: call["params"].clone(), + reply: None, + }); + calls.len() - 1 + }; + let upstream = TcpStream::connect(("127.0.0.1", state.upstream)).await?; + let (up_read, mut up_write) = upstream.into_split(); + up_write.write_all(&request).await?; + let Some((reply, reply_body)) = read_http(&mut BufReader::new(up_read)).await? else { + return Ok(()); + }; + let answer: Value = serde_json::from_slice(&reply_body).unwrap_or_default(); + state.calls.lock().unwrap()[index].reply = Some(answer.clone()); + let trap = { + let mut trap = state.trap.lock().unwrap(); + match trap.take() { + Some((armed, sender)) if armed == method => Some(sender), + other => { + *trap = other; + None + } + } + }; + if let Some(sender) = trap { + let (release, released) = oneshot::channel(); + let mut cleanup = state.released.subscribe(); + let held = HeldReply { + params: call["params"].clone(), + reply: answer, + release, + }; + if sender.send(held).is_ok() { + tokio::select! { + _ = released => {} + _ = cleanup.wait_for(|released| *released) => {} + } + } + } + down_write.write_all(&reply).await?; + } +} + +/// One HTTP/1.1 message framed by `Content-Length`, as qbitd and the +/// servers' client frame every message: the raw bytes and the body. `None` +/// at a clean end of stream. +async fn read_http(reader: &mut BufReader) -> Result, Vec)>> +where + R: tokio::io::AsyncRead + Unpin, +{ + let mut raw = Vec::new(); + let mut length = None; + loop { + let start = raw.len(); + if reader.read_until(b'\n', &mut raw).await? == 0 { + ensure!(raw.is_empty(), "connection closed inside an HTTP header"); + return Ok(None); + } + let line = std::str::from_utf8(&raw[start..])?.trim_end(); + if line.is_empty() { + break; + } + if let Some((name, value)) = line.split_once(':') { + if name.eq_ignore_ascii_case("content-length") { + length = Some(value.trim().parse::()?); + } + } + } + let mut body = vec![0; length.context("HTTP message without Content-Length")?]; + reader.read_exact(&mut body).await?; + raw.extend_from_slice(&body); + Ok(Some((raw, body))) +} + +/// A `BEFORE UPDATE` trigger on one table of the fixture's schema that, +/// for rows matching its condition, waits on an advisory lock. The trigger +/// takes the lock shared, so an open gate never serializes the servers; the +/// test takes it exclusively on its own connection to close the gate. +pub(crate) struct JournalGate { + name: String, + table: String, + key: String, + holder: Option<(sqlx::pool::PoolConnection, i32)>, +} + +impl JournalGate { + /// Install an open gate named `name` on `table` for updates where `when` + /// (a trigger `WHEN` condition over `OLD` and `NEW`) holds. + pub(crate) async fn install( + fixture: &Fixture, + name: &str, + table: &str, + when: &str, + ) -> Result { + let key = format!("{}.{name}", fixture.schema); + sqlx::raw_sql(&format!( + "CREATE FUNCTION {schema}.{name}() RETURNS trigger LANGUAGE plpgsql AS $$ BEGIN PERFORM pg_advisory_xact_lock_shared(hashtext('{key}')::bigint); RETURN NEW; END $$; CREATE TRIGGER {name} BEFORE UPDATE ON {schema}.{table} FOR EACH ROW WHEN ({when}) EXECUTE FUNCTION {schema}.{name}();", + schema = fixture.schema + )) + .execute(&fixture.pool) + .await?; + Ok(Self { + name: name.into(), + table: table.into(), + key, + holder: None, + }) + } + + /// Close the gate: every matching update from now on waits. + pub(crate) async fn close(&mut self, fixture: &Fixture) -> Result<()> { + ensure!( + self.holder.is_none(), + "gate {} is already closed", + self.name + ); + let mut connection = fixture.pool.acquire().await?; + sqlx::query("SELECT pg_advisory_lock(hashtext($1)::bigint)") + .bind(&self.key) + .execute(&mut *connection) + .await?; + let pid = sqlx::query_scalar("SELECT pg_backend_pid()") + .fetch_one(&mut *connection) + .await?; + self.holder = Some((connection, pid)); + Ok(()) + } + + /// Open the gate, keeping its trigger: every waiting update proceeds. + pub(crate) async fn open(&mut self) -> Result<()> { + let (connection, _) = self.holder.take().context("the gate is open")?; + self.unlock(connection).await + } + + /// Release the gate's lock on its holder. If the unlock fails, the + /// connection is closed rather than returned to the pool still holding + /// the session lock; closing it releases the lock. + async fn unlock( + &self, + mut connection: sqlx::pool::PoolConnection, + ) -> Result<()> { + let unlocked = sqlx::query("SELECT pg_advisory_unlock(hashtext($1)::bigint)") + .bind(&self.key) + .execute(&mut *connection) + .await; + if unlocked.is_err() { + drop(connection.detach()); + } + unlocked?; + Ok(()) + } + + /// The backends whose update waits on the closed gate. + pub(crate) async fn waiting(&self, fixture: &Fixture) -> Result> { + let (_, pid) = self.holder.as_ref().context("the gate is open")?; + Ok(sqlx::query_scalar( + "SELECT pid FROM pg_stat_activity WHERE $1=ANY(pg_blocking_pids(pid)) ORDER BY pid", + ) + .bind(pid) + .fetch_all(&fixture.admin) + .await?) + } + + /// Open the gate and remove its trigger. + pub(crate) async fn remove(mut self, fixture: &Fixture) -> Result<()> { + if let Some((connection, _)) = self.holder.take() { + self.unlock(connection).await?; + } + sqlx::raw_sql(&format!( + "DROP TRIGGER {name} ON {schema}.{table}; DROP FUNCTION {schema}.{name}();", + name = self.name, + table = self.table, + schema = fixture.schema + )) + .execute(&fixture.pool) + .await?; + Ok(()) + } +} + +/// A gate dropped while closed (a failed case) closes its holder +/// connection instead of returning it to the pool with the session lock +/// held, which would hold every gated update, and the teardown, forever. +impl Drop for JournalGate { + fn drop(&mut self) { + if let Some((connection, _)) = self.holder.take() { + drop(connection.detach()); + } + } +} + +/// Start server `index` with its node calls going to `rpc_port` (a proxy), +/// sponsoring CPFP packages at `fee` when given, and wait until it is ready. +pub(crate) async fn start_frontend( + fixture: &Fixture, + index: usize, + rpc_port: u16, + fee: Option, +) -> Result { + let mut database = url::Url::parse(&fixture.database_url)?; + database + .query_pairs_mut() + .append_pair("application_name", &backend_name(fixture, index)); + let process = fixture.start_server_with( + index, + fee, + &[ + ("QBIT_RPC_PORT", rpc_port.to_string()), + ("PRISM_DATABASE_URL", database.to_string()), + ], + )?; + until(&format!("server {index} readiness"), 60, || async { + Ok(fixture + .client + .get(format!("http://127.0.0.1:{}/healthz", fixture.api[index])) + .send() + .await? + .status() + .is_success()) + }) + .await?; + // The name is how its backends are found after a kill. + let named: i64 = + sqlx::query_scalar("SELECT count(*) FROM pg_stat_activity WHERE application_name=$1") + .bind(backend_name(fixture, index)) + .fetch_one(&fixture.admin) + .await?; + ensure!( + named > 0, + "server {index}'s database connections do not carry their application_name" + ); + Ok(process) +} + +/// `SIGKILL` server `index` and reap it: no graceful shutdown, no claim +/// handed back, no journal written after this returns. +pub(crate) fn kill_frontend(fixture: &mut Fixture, index: usize) -> Result<()> { + let server = fixture + .servers + .get_mut(index) + .with_context(|| format!("server {index} was never started"))?; + server.stop(); + ensure!( + server.child.try_wait()?.is_some(), + "server {index} is still running after SIGKILL" + ); + Ok(()) +} + +/// The `application_name` of server `index`'s database connections when +/// [`start_frontend`] started it: unique to the fixture, so no other test's +/// backends on a shared cluster match it. +fn backend_name(fixture: &Fixture, index: usize) -> String { + format!("{}-live-{index}", fixture.schema) +} + +/// Terminate every database backend of server `index`, which the test has +/// already killed, and wait until they are gone. A statement the dead +/// process wrote before it died can still be sitting unread in a backend's +/// socket; run later, an implicit single-statement transaction would commit +/// for a client that no longer exists. Terminated, it rolls back. Returns +/// how many backends were terminated. +pub(crate) async fn end_frontend_backends(fixture: &Fixture, index: usize) -> Result { + ensure!( + fixture.servers[index].child.try_wait()?.is_some(), + "server {index} is still running" + ); + let name = backend_name(fixture, index); + let backends: Vec = + sqlx::query_scalar("SELECT pid FROM pg_stat_activity WHERE application_name=$1") + .bind(&name) + .fetch_all(&fixture.admin) + .await?; + for pid in &backends { + sqlx::query("SELECT pg_terminate_backend($1)") + .bind(pid) + .execute(&fixture.admin) + .await?; + } + until(&format!("server {index}'s backends gone"), 10, || async { + Ok(sqlx::query_scalar::<_, i64>( + "SELECT count(*) FROM pg_stat_activity WHERE application_name=$1", + ) + .bind(&name) + .fetch_one(&fixture.admin) + .await? + == 0) + }) + .await?; + Ok(backends.len()) +} + +/// The server index of instance id `live-`. +pub(crate) fn frontend_index(instance: &str) -> Result { + let index = instance + .strip_prefix("live-") + .with_context(|| format!("unexpected instance id {instance}"))? + .parse()?; + ensure!(index < 2, "unexpected instance id {instance}"); + Ok(index) +} + +/// Mark the dead owner's lease on `table`'s row `key_column = key` as +/// expired, which is what the lease would be after its 120 s: the database +/// clock's estimate moves into the past, and the claim, a candidate's or a +/// fanout's, is revoked, so a survivor takes it over at its next claim poll +/// (#581, #654). Refused unless `owner` still holds it, so a live claim is +/// never cut short. +pub(crate) async fn expire_dead_lease( + fixture: &Fixture, + table: &str, + key_column: &str, + key: &str, + owner: &str, +) -> Result<()> { + let expired = sqlx::query(&format!( + "UPDATE {table} SET claim_expires_at=clock_timestamp()-interval '1 second' WHERE {key_column}=$1 AND claim_instance_id=$2 AND claim_token IS NOT NULL" + )) + .bind(key) + .bind(owner) + .execute(&fixture.pool) + .await? + .rows_affected(); + ensure!( + expired == 1, + "the dead owner {owner} does not hold the claim on {key}" + ); + // #581, #654: a candidate or CTV fanout claim is taken over once a + // survivor has watched it go unrenewed for its lease, never by the + // database clock; revoking it is what lets the survivor take it at once. + match table { + "qbit_block_candidate_outbox" => { + qbit_prism_server::ledger::revoke_candidate_claims(&fixture.pool, Some(key), false) + .await?; + } + "qbit_ctv_fanout_artifacts" => { + qbit_prism_server::ledger::revoke_fanout_claims(&fixture.pool, Some(key)).await?; + } + _ => {} + } + Ok(()) +} + +/// The server's withheld answer to an armed request, and the switch that +/// closes both sides of the relayed connection without delivering it. +pub(crate) struct HeldAnswer { + pub answer: Value, + pub sever: oneshot::Sender<()>, +} + +struct StratumState { + upstream: u16, + trap: Mutex)>>, + released: watch::Sender, +} + +/// A line-oriented Stratum proxy in front of one server's listener. +pub(crate) struct StratumRelay { + pub port: u16, + state: Arc, + task: tokio::task::JoinHandle<()>, +} + +impl StratumRelay { + pub(crate) async fn start(upstream: u16) -> Result { + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await?; + let port = listener.local_addr()?.port(); + let state = Arc::new(StratumState { + upstream, + trap: Mutex::new(None), + released: watch::Sender::new(false), + }); + let shared = state.clone(); + let task = tokio::spawn(async move { + while let Ok((downstream, _)) = listener.accept().await { + let state = shared.clone(); + tokio::spawn(async move { + let _ = relay_stratum(state, downstream).await; + }); + } + }); + Ok(Self { port, state, task }) + } + + /// Withhold the server's answer to request `id` on any relayed + /// connection, then close that connection when the test severs it. + pub(crate) fn arm(&self, id: Value) -> oneshot::Receiver { + let (sender, receiver) = oneshot::channel(); + *self.state.trap.lock().unwrap() = Some((id, sender)); + receiver + } +} + +impl Drop for StratumRelay { + fn drop(&mut self) { + self.state.released.send_replace(true); + self.task.abort(); + } +} + +/// Copies the miner's bytes to the server unchanged, and the server's lines +/// to the miner one at a time, until the armed answer arrives: that line is +/// never written, and once severed both sides are closed. +async fn relay_stratum(state: Arc, downstream: TcpStream) -> Result<()> { + let upstream = TcpStream::connect(("127.0.0.1", state.upstream)).await?; + let (mut down_read, down_write) = downstream.into_split(); + let (up_read, mut up_write) = upstream.into_split(); + let requests = tokio::spawn(async move { + let _ = tokio::io::copy(&mut down_read, &mut up_write).await; + let _ = up_write.shutdown().await; + }); + let result = relay_answers(&state, BufReader::new(up_read), down_write).await; + // Dropping the copy closes the miner's read half and the server's + // write half; returning drops the rest. + requests.abort(); + result +} + +async fn relay_answers( + state: &StratumState, + mut lines: BufReader, + mut down_write: OwnedWriteHalf, +) -> Result<()> { + loop { + let mut line = String::new(); + if lines.read_line(&mut line).await? == 0 { + return Ok(()); + } + let message: Value = serde_json::from_str(&line).unwrap_or_default(); + let trap = { + let mut trap = state.trap.lock().unwrap(); + match trap.take() { + Some((id, sender)) if message.get("method").is_none() && message["id"] == id => { + Some(sender) + } + other => { + *trap = other; + None + } + } + }; + if let Some(sender) = trap { + let (sever, severed) = oneshot::channel(); + let mut cleanup = state.released.subscribe(); + if sender + .send(HeldAnswer { + answer: message, + sever, + }) + .is_ok() + { + tokio::select! { + _ = severed => {} + _ = cleanup.wait_for(|released| *released) => {} + } + } + return Ok(()); + } + down_write.write_all(line.as_bytes()).await?; + } +} diff --git a/crates/qbit-prism-server/tests/support/live_highdiff.rs b/crates/qbit-prism-server/tests/support/live_highdiff.rs new file mode 100644 index 000000000..76f88b390 --- /dev/null +++ b/crates/qbit-prism-server/tests/support/live_highdiff.rs @@ -0,0 +1,570 @@ +use super::*; +use num_bigint::BigUint; +use qbit_prism_server::codec::{ + difficulty_target, double_sha256, hash_display, parse_u32_hex, scaled_target_difficulty, + target_from_compact, +}; +use tokio::net::tcp::{OwnedReadHalf, OwnedWriteHalf}; + +pub(super) struct HighdiffClient { + reader: BufReader, + writer: OwnedWriteHalf, + buffer: Vec, + pub(super) username: String, + extranonce1: String, + extranonce2_size: usize, + difficulty: f64, + notify: Value, +} + +impl HighdiffClient { + pub(super) async fn open(fixture: &Fixture, index: usize) -> Result { + Self::open_as(fixture, index, format!("{}.highdiff", fixture.address)).await + } + + /// `open` as the worker `username`, whose address is its payout identity. + pub(super) async fn open_as(fixture: &Fixture, index: usize, username: String) -> Result { + let stream = tokio::net::TcpStream::connect(("127.0.0.1", fixture.highdiff[index])).await?; + let (read, write) = stream.into_split(); + let mut client = Self { + reader: BufReader::new(read), + writer: write, + buffer: Vec::new(), + username, + extranonce1: String::new(), + extranonce2_size: 0, + difficulty: 0.0, + notify: Value::Null, + }; + client + .send(json!({"id":1,"method":"mining.subscribe","params":["highdiff-regtest"]})) + .await?; + client + .send(json!({"id":2,"method":"mining.authorize","params":[client.username,"x"]})) + .await?; + tokio::time::timeout(Duration::from_secs(15), async { + loop { + let message = client.read().await?; + ensure!( + message.get("error").is_none_or(Value::is_null), + "highdiff handshake rejected: {message}" + ); + if message["id"] == 1 { + client.extranonce1 = message["result"][1] + .as_str() + .context("extranonce1 missing")? + .into(); + client.extranonce2_size = message["result"][2] + .as_u64() + .context("extranonce2 size missing")? + .try_into()?; + } + if message["method"] == "mining.notify" { + break; + } + } + ensure!( + client.difficulty >= 500_000.0, + "first highdiff advertisement below floor: {}", + client.difficulty + ); + ensure!( + !client.extranonce1.is_empty() && client.extranonce2_size > 0, + "subscription entropy missing" + ); + Ok::<_, anyhow::Error>(()) + }) + .await??; + Ok(client) + } + + pub(super) async fn send(&mut self, payload: Value) -> Result<()> { + self.writer + .write_all(format!("{payload}\n").as_bytes()) + .await?; + Ok(()) + } + + async fn read(&mut self) -> Result { + ensure!( + self.reader.read_until(b'\n', &mut self.buffer).await? > 0, + "highdiff connection closed" + ); + ensure!( + self.buffer.len() < 2 * 1024 * 1024, + "oversized highdiff message" + ); + let message: Value = serde_json::from_slice(&self.buffer)?; + self.buffer.clear(); + if message["method"] == "mining.set_difficulty" { + self.difficulty = message["params"][0] + .as_f64() + .context("invalid difficulty")?; + } + if message["method"] == "mining.notify" { + self.notify = message.clone(); + } + Ok(message) + } + + pub(super) async fn response(&mut self, id: u64) -> Result { + loop { + let value = self.read().await?; + if value["id"] == id { + return Ok(value); + } + } + } + + pub(super) async fn wait_for_parent(&mut self, parent: &str) -> Result<()> { + let mut wire_parent = hex::decode(parent)?; + wire_parent.reverse(); + let (words, _) = wire_parent.as_chunks_mut::<4>(); + for word in words { + word.reverse(); + } + let expected = hex::encode(wire_parent); + tokio::time::timeout(Duration::from_secs(15), async { + while self.notify["params"][1].as_str() != Some(expected.as_str()) { + self.read().await?; + } + Ok::<_, anyhow::Error>(()) + }) + .await + .context("replacement parent was not delivered")??; + Ok(()) + } + + pub(super) fn solve(&self, id: u64, future_time: bool) -> Result<(Value, String, u128)> { + let params = self.notify["params"].as_array().context("notify missing")?; + let field = |index: usize| params[index].as_str().context("invalid notify field"); + let extranonce2 = "00".repeat(self.extranonce2_size); + let coinbase = hex::decode(format!( + "{}{}{extranonce2}{}", + field(2)?, + self.extranonce1, + field(3)? + ))?; + let mut merkle = double_sha256(&coinbase); + for sibling in params[4].as_array().context("merkle branch missing")? { + let sibling = hex::decode(sibling.as_str().context("invalid sibling")?)?; + ensure!(sibling.len() == 32, "invalid sibling length"); + merkle = double_sha256(&[merkle.as_slice(), sibling.as_slice()].concat()); + } + let mut previous = hex::decode(field(1)?)?; + for word in previous.as_chunks_mut::<4>().0 { + word.reverse(); + } + let bits = parse_u32_hex(field(6)?)?; + let network_target = target_from_compact(bits)?; + let share_target = difficulty_target(self.difficulty)?; + ensure!( + share_target < network_target, + "test requires highdiff floor harder than network" + ); + let ntime = if future_time { + std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH)? + .as_secs() as u32 + + 4 * 60 * 60 + } else { + parse_u32_hex(field(7)?)? + }; + let version = parse_u32_hex(field(5)?)?; + for nonce in 0..10_000u32 { + let header = [ + version.to_le_bytes().as_slice(), + previous.as_slice(), + merkle.as_slice(), + ntime.to_le_bytes().as_slice(), + bits.to_le_bytes().as_slice(), + nonce.to_le_bytes().as_slice(), + ] + .concat(); + let hash = double_sha256(&header); + let value = BigUint::from_bytes_le(&hash); + if value <= network_target && value > share_target { + let request = json!({"id":id,"method":"mining.submit","params":[self.username,field(0)?,extranonce2,format!("{ntime:08x}"),format!("{nonce:08x}")]}); + return Ok(( + request, + hash_display(&hash), + scaled_target_difficulty(&network_target)?, + )); + } + } + bail!("no regtest block-only proof in constrained nonce budget") + } +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn real_highdiff_block_only_proof_waits_for_active_chain_credit() -> Result<()> { + let Some(fixture) = Fixture::open(false).await? else { + return Ok(()); + }; + let result=async { + let mut miner=HighdiffClient::open(&fixture,0).await?; + let (request,block_hash,network_work)=miner.solve(10,false)?; + let share_id=format!("{}:{block_hash}",miner.username); + // Pause candidate claims independently of job/share persistence. + // This proves an outbox write alone cannot produce a block-only ACK + // without blocking a concurrent replacement job on this connection. + let pause_key=Uuid::new_v4().as_u128() as i64; + sqlx::raw_sql(&format!("CREATE FUNCTION test_pause_candidate_claim() RETURNS trigger LANGUAGE plpgsql AS $$ BEGIN IF NEW.claim_token IS NOT NULL AND NEW.claim_token IS DISTINCT FROM OLD.claim_token THEN PERFORM pg_advisory_xact_lock({pause_key}); END IF; RETURN NEW; END $$; CREATE TRIGGER test_pause_candidate_claim BEFORE UPDATE OF claim_token ON qbit_block_candidate_outbox FOR EACH ROW EXECUTE FUNCTION test_pause_candidate_claim();")).execute(&fixture.pool).await?; + let mut settlement=fixture.pool.begin().await?; + sqlx::query("SELECT pg_advisory_xact_lock($1)").bind(pause_key).execute(&mut *settlement).await?; + miner.send(request).await?; + until("durable highdiff candidate intent",5,||async { + Ok(sqlx::query_scalar::<_,bool>("SELECT EXISTS(SELECT 1 FROM qbit_block_candidate_outbox WHERE block_hash=$1 AND state='pending')") + .bind(&block_hash).fetch_one(&fixture.pool).await?) + }).await?; + let before:i64=sqlx::query_scalar("SELECT count(*) FROM qbit_share_ledger WHERE share_id=$1").bind(&share_id).fetch_one(&fixture.pool).await?; + ensure!(before==0,"block-only proof was credited before active-chain finalization"); + ensure!(tokio::time::timeout(Duration::from_millis(250),miner.response(10)).await.is_err(),"block-only proof ACKed before finalization"); + settlement.commit().await?; + let response=tokio::time::timeout(Duration::from_secs(15),miner.response(10)).await??; + ensure!(response["result"]==true && response["error"].is_null(),"valid highdiff block rejected: {response}"); + let row=sqlx::query("SELECT s.share_difficulty::text AS work,s.network_difficulty::text AS network,b.chain_state,s.accepted FROM qbit_share_ledger s JOIN qbit_pool_blocks b ON b.block_hash=$2 WHERE s.share_id=$1") + .bind(&share_id).bind(&block_hash).fetch_one(&fixture.pool).await?; + ensure!(row.try_get::("accepted")? && row.try_get::("chain_state")?=="confirmed","ACK preceded durable active-chain credit"); + ensure!(row.try_get::("work")?==network_work.to_string() && row.try_get::("network")?==network_work.to_string(),"block-only proof credited assigned highdiff instead of proven network work"); + let block=fixture.rpc("getblockheader",json!([block_hash])).await?; + ensure!(block["confirmations"].as_i64().unwrap_or(0)>0,"ACKed block is not on the active chain"); + assert_share_height_boundary(&fixture,&share_id,block["height"].as_u64().context("confirmed block height missing")?).await?; + drop(miner); + + // A valid PoW with an invalid future timestamp reaches submitblock, + // whose terminal rejection must produce no accepted share or ACK. + let mut miner=HighdiffClient::open(&fixture,1).await?; + // A new connection may legitimately receive still-published work + // during the replacement lease. To exercise node timestamp rejection, + // first receive work that actually builds on the just-confirmed block. + miner.wait_for_parent(&block_hash).await?; + let (request,rejected_hash,_)=miner.solve(11,true)?; + miner.send(request).await?; + let response=tokio::time::timeout(Duration::from_secs(15),miner.response(11)).await??; + ensure!(response["result"]!=true && !response["error"].is_null(),"node-rejected block-only proof received an accepted ACK"); + let rejected_share_id=format!("{}:{rejected_hash}",miner.username); + let credits:i64=sqlx::query_scalar("SELECT count(*) FROM qbit_share_ledger WHERE share_id=$1").bind(&rejected_share_id).fetch_one(&fixture.pool).await?; + ensure!(credits==0,"rejected block-only proof inflated share accounting"); + let disposition=sqlx::query("SELECT state,last_error FROM qbit_block_candidate_outbox WHERE block_hash=$1").bind(&rejected_hash).fetch_one(&fixture.pool).await?; + ensure!(disposition.try_get::("state")?=="reconciliation","a candidate the node rejected after its offer stays in reconciliation with its evidence, never abandoned"); + ensure!(disposition.try_get::,_>("last_error")?.is_some_and(|e|e.contains("time-too-new")),"test did not reach node timestamp rejection"); + fixture.integrity().await?; + eprintln!("live highdiff: withheld ACK until durable active-chain credit; credited exactly {network_work} network work; rejected future block received no ACK/credit"); + Ok::<_,anyhow::Error>(()) + }.await; + if result.is_err() { + eprintln!("{}", fixture.diagnostics()); + } + let cleanup = fixture.cleanup().await; + result.and(cleanup) +} + +async fn assert_share_height_boundary( + fixture: &Fixture, + share_id: &str, + candidate_height: u64, +) -> Result<()> { + let candidate_height = i64::try_from(candidate_height)?; + let parent_height = candidate_height + .checked_sub(1) + .context("candidate height has no parent")?; + let persisted: i64 = + sqlx::query_scalar("SELECT template_height FROM qbit_share_ledger WHERE share_id=$1") + .bind(share_id) + .fetch_one(&fixture.pool) + .await?; + ensure!( + persisted == parent_height, + "share recorded candidate height instead of parent height" + ); + for (minimum, expected) in [(parent_height, true), (candidate_height, false)] { + let included: bool = sqlx::query_scalar( + "SELECT EXISTS(SELECT 1 FROM qbit_shares_since_template_height($1) WHERE share_id=$2)", + ) + .bind(minimum) + .bind(share_id) + .fetch_one(&fixture.pool) + .await?; + ensure!( + included == expected, + "share crossed incorrect inclusive template-height boundary {minimum}" + ); + } + Ok(()) +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn real_share_height_queries_use_template_parent_height() -> Result<()> { + use qbit_prism_server::{ + coordinator::{Coordinator, JobContext}, + stratum::MiningBackend, + }; + let Some(fixture) = Fixture::open(false).await? else { + return Ok(()); + }; + let result = async { + let coordinator = Coordinator::new( + direct_coordinator_config(&fixture)?, + std::sync::Arc::new(qbit_prism_server::metrics::Metrics::default()), + ) + .await?; + coordinator.refresh_once().await?; + let worker = coordinator + .authorize(&format!("{}.height", fixture.address)) + .await?; + let session = coordinator.new_session_id().await?; + let job = coordinator + .build_job(&worker, &format!("{session:08x}"), 1e-12, 0.0) + .await?; + let parent = fixture + .rpc("getblockheader", json!([job.wire.previousblockhash])) + .await?; + let parent_height = parent["height"] + .as_u64() + .context("template parent height missing")?; + let candidate_height = parent_height + .checked_add(1) + .context("candidate height overflow")?; + ensure!( + job.context.bundle.found_block.block_height == candidate_height, + "job candidate height differs from real node parent" + ); + let submission = (0..10_000u32) + .find_map(|nonce| { + let proof = job + .wire + .assemble_submission( + &"00".repeat(job.wire.extranonce2_size), + &format!("{:08x}", job.wire.ntime), + &format!("{nonce:08x}"), + None, + 0, + ) + .ok()?; + (proof.share_pass && !proof.block_pass).then_some(proof) + }) + .context("no constrained non-block regtest share")?; + // Invalid internal/restored context must fail before any durable ACK; + // subtraction must never wrap height zero to a huge ledger height. + let mut malformed = job.clone(); + let mut bundle = (*job.context.bundle).clone(); + bundle.found_block.block_height = 0; + malformed.context = std::sync::Arc::new(JobContext { + prepared: job.context.prepared.clone(), + worker: job.context.worker.clone(), + bundle: std::sync::Arc::new(bundle), + bootstrap_share: job.context.bootstrap_share.clone(), + issuance_authority: job.context.issuance_authority.clone(), + }); + let rejected = coordinator + .submit(&worker, &malformed, submission.clone(), false.into()) + .await + .err() + .context("zero candidate height accepted")?; + ensure!( + rejected.reason_id.as_deref() == Some("internal-error"), + "wrong invalid-height error: {rejected}" + ); + let share_id = format!("{}:{}", worker.username, submission.block_hash_hex); + coordinator + .submit(&worker, &job, submission, false.into()) + .await?; + assert_share_height_boundary(&fixture, &share_id, candidate_height).await?; + coordinator.ledger.pool.close().await; + Ok::<_, anyhow::Error>(()) + } + .await; + if result.is_err() { + eprintln!("{}", fixture.diagnostics()); + } + let cleanup = fixture.cleanup().await; + result.and(cleanup) +} + +fn direct_coordinator_config(fixture: &Fixture) -> Result { + use qbit_prism_server::config::Config; + Ok(Config { + database_url: fixture.database_url.clone(), + instance_id: "live-observed-tip".into(), + database_connections: 4, + initialize_schema: false, + chain: "regtest".into(), + expected_genesis_hash: None, + min_peers: 1, + template_max_age: Duration::from_secs(120), + submit_tip_max_age: Duration::from_secs(10), + template_refresh_failure_exit: Duration::from_secs(120), + rpc_url: format!("http://127.0.0.1:{}/", fixture.rpc_port), + rpc_user: "prismtest".into(), + rpc_password: "prismtest".into(), + rpc_timeout: Duration::from_secs(10), + block_submit_timeout: Duration::from_secs(1), + block_submit_enabled: true, + poll_interval: Duration::from_secs(1), + blockwait: false, + build_workers: 2, + refresh_build_threads: None, + runtime_workers: 2, + snapshot_interval: Duration::from_secs(30), + health_timeout: Duration::from_secs(15), + share_commit_timeout: Duration::from_secs(15), + share_commit_grace: Duration::from_secs(5), + candidate_orphan_confirmations: 6, + capture_overpay_ceiling_bps: 100, + offer_standby: None, + extranonce2_size: 8, + coinbase_tag: "/PRISM/".into(), + manifest_seed: "11".repeat(32), + ledger_seed: "22".repeat(32), + ledger_public_key: ManifestSigningKey::from_seed_hex(&"22".repeat(32))?.public_key_hex(), + username_fallback: None, + payout_policy: super::live_payout_policy(), + fee_address: None, + ctv_enabled: false, + ctv_config: qbit_prism::SettlementModeConfig::default(), + ctv_direct_floor: 10_485_760, + ctv_fee: None, + ctv_fee_premium_bps: 12000, + ctv_broadcast: false, + ctv_broadcast_interval: Duration::from_secs(10), + version_mask: qbit_prism_server::codec::VERSION_ROLLING_MASK, + audit_bind: "127.0.0.1".into(), + audit_port: free_port()?, + }) +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn observed_tip_advance_fences_old_prepared_jobs_and_health() -> Result<()> { + use qbit_prism_server::{coordinator::Coordinator, stratum::MiningBackend}; + let Some(fixture) = Fixture::open(false).await? else { + return Ok(()); + }; + let result=async { + let coordinator=Coordinator::new(direct_coordinator_config(&fixture)?, std::sync::Arc::new(qbit_prism_server::metrics::Metrics::default())).await?; + coordinator.refresh_once().await?; + ensure!(coordinator.health().await["ready"]==true,"initial published template is not healthy"); + let worker=coordinator.authorize(&format!("{}.fence",fixture.address)).await?; + let session=coordinator.new_session_id().await?; + let job=coordinator.build_job(&worker,&format!("{session:08x}"),1e-9,0.0).await?; + coordinator.persist_issued_job(&worker,&job,0,Duration::from_secs(30)).await?; + let submission=(0..10_000u32).find_map(|nonce| { + let proof=job.wire.assemble_submission(&"00".repeat(8),&format!("{:08x}",job.wire.ntime),&format!("{nonce:08x}"),None,0).ok()?; + proof.share_pass.then_some(proof) + }).context("no constrained regtest proof")?; + let original=job.wire.previousblockhash.clone(); + // This is the state between observing a validated new tip and + // successfully building/publishing its payout work. A failed builder + // must not leave the older immutable prepared object authoritative. + *coordinator.observed_tip.write().await=qbit_prism_server::coordinator::TipState::baseline("fe".repeat(32)); + ensure!(coordinator.health().await["ready"]==false,"stale prepared work remained healthy after tip observation"); + ensure!(coordinator.build_job(&worker,&format!("{session:08x}"),1e-9,0.0).await.is_err(),"new miner received work for superseded observed tip"); + ensure!(coordinator.resume_job(&worker,&job.wire.job_id).await.is_err(),"reconnect restored work while new observed tip was pending"); + let rejected=coordinator.submit(&worker,&job,submission, false.into()).await.err().context("old share was accepted while new tip publication was pending")?; + ensure!(rejected.reason_id.as_deref()==Some("stale-job"),"wrong stale tip rejection: {rejected}"); + let credits:i64=sqlx::query_scalar("SELECT count(*) FROM qbit_share_ledger WHERE accepted").fetch_one(&fixture.pool).await?; + ensure!(credits==0,"untrusted cached job inflated share accounting"); + *coordinator.observed_tip.write().await=qbit_prism_server::coordinator::TipState::baseline(original); + ensure!(coordinator.health().await["ready"]==true,"restoring tip authority did not restore health"); + coordinator.ledger.pool.close().await; + eprintln!("observed-tip regression: health, new jobs, resume and submitted shares all fenced during unpublished tip transition"); + Ok::<_,anyhow::Error>(()) + }.await; + if result.is_err() { + eprintln!("{}", fixture.diagnostics()); + } + let cleanup = fixture.cleanup().await; + result.and(cleanup) +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn observability_grace_counter_moves_only_after_real_durable_credit() -> Result<()> { + use qbit_prism_server::{coordinator::Coordinator, metrics::Metrics, stratum::MiningBackend}; + let Some(fixture) = Fixture::open(false).await? else { + return Ok(()); + }; + let result = async { + let metrics = std::sync::Arc::new(Metrics::default()); + let coordinator = + Coordinator::new(direct_coordinator_config(&fixture)?, metrics.clone()).await?; + coordinator.refresh_once().await?; + let worker = coordinator + .authorize(&format!("{}.metrics", fixture.address)) + .await?; + let session = coordinator.new_session_id().await?; + let job = coordinator + .build_job(&worker, &format!("{session:08x}"), 1e-12, 0.0) + .await?; + let mut proofs = (0..10_000u32).filter_map(|nonce| { + let proof = job + .wire + .assemble_submission( + &"00".repeat(job.wire.extranonce2_size), + &format!("{:08x}", job.wire.ntime), + &format!("{nonce:08x}"), + None, + 0, + ) + .ok()?; + (proof.share_pass && !proof.block_pass).then_some(proof) + }); + let normal = proofs.next().context("no normal regtest proof")?; + let grace = proofs.next().context("no grace regtest proof")?; + coordinator + .submit(&worker, &job, normal, false.into()) + .await?; + ensure!( + metrics + .render() + .lines() + .any(|l| l == "qbit_prism_grace_credited_shares_total 0"), + "normal acceptance counted as grace" + ); + fixture + .rpc("generatetoaddress", json!([1, fixture.address])) + .await?; + coordinator.refresh_once().await?; + let share_id = format!("{}:{}", worker.username, grace.block_hash_hex); + coordinator + .submit(&worker, &job, grace.clone(), true.into()) + .await?; + let policy: Option = + sqlx::query_scalar("SELECT credit_policy FROM qbit_share_ledger WHERE share_id=$1") + .bind(&share_id) + .fetch_one(&coordinator.ledger.pool) + .await?; + ensure!( + policy.as_deref() == Some("stale-grace"), + "expected real durable grace policy" + ); + ensure!( + metrics + .render() + .lines() + .any(|l| l == "qbit_prism_grace_credited_shares_total 1"), + "durable grace hook did not move" + ); + let duplicate = coordinator + .submit(&worker, &job, grace, true.into()) + .await + .unwrap_err(); + ensure!( + duplicate.reason_id.as_deref() == Some("duplicate-share"), + "unexpected duplicate outcome" + ); + ensure!( + metrics + .render() + .lines() + .any(|l| l == "qbit_prism_grace_credited_shares_total 1"), + "duplicate double-counted grace" + ); + coordinator.ledger.pool.close().await; + Ok::<_, anyhow::Error>(()) + } + .await; + if result.is_err() { + eprintln!("{}", fixture.diagnostics()); + } + result.and(fixture.cleanup().await) +} diff --git a/crates/qbit-prism-server/tests/support/live_host_tools.rs b/crates/qbit-prism-server/tests/support/live_host_tools.rs new file mode 100644 index 000000000..bd05579a8 --- /dev/null +++ b/crates/qbit-prism-server/tests/support/live_host_tools.rs @@ -0,0 +1,30 @@ +//! Host programs the live scenarios drive: finding one and running it to +//! completion (#575). Self-contained (std and `anyhow`), so any test binary +//! can include it with `#[path]`. +use anyhow::{ensure, Context, Result}; +use std::{ + path::{Path, PathBuf}, + process::Command, +}; + +/// A program on `PATH`, or in the system directories `mkfs` lives in. +pub(crate) fn program(name: &str) -> Result { + let path = std::env::var_os("PATH").unwrap_or_default(); + std::env::split_paths(&path) + .chain(["/usr/sbin", "/sbin"].map(PathBuf::from)) + .map(|directory| directory.join(name)) + .find(|candidate| candidate.is_file()) + .with_context(|| format!("{name} not found on PATH")) +} + +pub(crate) fn run(program: &Path, args: &[&str]) -> Result<()> { + let output = Command::new(program).args(args).output()?; + ensure!( + output.status.success(), + "{} {args:?} failed: {} {}", + program.display(), + String::from_utf8_lossy(&output.stdout), + String::from_utf8_lossy(&output.stderr) + ); + Ok(()) +} diff --git a/crates/qbit-prism-server/tests/support/live_lost_reply.rs b/crates/qbit-prism-server/tests/support/live_lost_reply.rs new file mode 100644 index 000000000..fd2f8d900 --- /dev/null +++ b/crates/qbit-prism-server/tests/support/live_lost_reply.rs @@ -0,0 +1,877 @@ +//! #474 slice C: the node accepted the operation, but its reply was lost. +//! +//! Both servers reach a real qbitd through [`RpcReplyHold`]. In each node +//! case the proxy lets one call run on the node (`submitblock`, +//! `sendrawtransaction` for a fee-bearing CTV fanout, or `submitpackage` for a +//! legacy zero-fee fanout and its CPFP child), confirms the node holds the +//! side effect, and withholds the reply. A [`JournalGate`] holds the owning +//! frontend's next journal update for that row, so however long the kill +//! takes, the frontend cannot record the outcome. The test `SIGKILL`s the +//! owner and, with the gate still closed, terminates every database backend +//! it had (a statement it wrote before dying must not commit afterwards), +//! confirms the row carries no outcome, and hands the dead owner's +//! lease to the survivor. The survivor must reconcile against the chain and +//! finish: the original block or transaction, with its original bytes, and +//! no fabricated rejection, duplicate credit or second payout. The killed +//! frontend is then restarted and must not redo anything. +//! +//! The miner case is the Stratum analogue: a [`StratumRelay`] withholds the +//! server's acknowledgement of a share, which the server sends only after +//! the share's ledger outcome is durable (`coordinator/miner_submit.rs`), +//! checks the credit is committed, and closes the miner's connection. The +//! miner's outcome for that submission stays unknown; its retry of the +//! identical share on a new connection, to either frontend, is refused as a +//! duplicate, and the ledger holds exactly one credit. +//! +//! A dead owner's claim lease is expired by [`expire_dead_lease`] rather than +//! waited out (120 s for candidates and fanouts): only after the owner was +//! reaped and its backends terminated, and only while that owner still holds +//! it, so the survivor takes over through the production claim predicate. +use super::fault_seams::{ + end_frontend_backends, expire_dead_lease, frontend_index, kill_frontend, start_frontend, + HeldReply, JournalGate, RpcReplyHold, StratumRelay, +}; +use super::share_client::{ + reason_id, start_share_only_servers, Answer, Proof, ShareClient, StratumSession, ANSWER, +}; +use super::*; +use qbit_prism_server::{ + codec::{double_sha256, hash_display, strip_witness_transaction}, + ledger::Ledger, +}; + +/// A found block whose `submitblock` reply is lost and whose owning +/// frontend dies before it records the offer outcome. The survivor settles +/// the unrecorded reservation as delivery unknown, finds the block on the +/// chain and lands it once: one `submitblock`, the node's bytes, one credit. +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn lost_submitblock_reply_owner_killed_survivor_lands_the_block_once() -> Result<()> { + let Some(mut fixture) = Fixture::open_with_servers(false, false).await? else { + return Ok(()); + }; + let proxy = RpcReplyHold::start(fixture.rpc_port).await?; + let result = lost_block_reply(&mut fixture, &proxy).await; + finish(fixture, result).await +} + +/// A mature fee-bearing CTV fanout whose `sendrawtransaction` the node +/// accepted, with the reply lost and the owning broadcaster killed before +/// it journals the attempt. The survivor finds it in the mempool and +/// confirms the one fanout with its signed bytes: no second payout. +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn lost_sendrawtransaction_reply_owner_killed_survivor_confirms_the_fanout_once() -> Result<()> +{ + let Some(mut fixture) = Fixture::open_with_servers(true, false).await? else { + return Ok(()); + }; + let proxy = RpcReplyHold::start(fixture.rpc_port).await?; + let result = lost_fanout_reply(&mut fixture, &proxy).await; + finish(fixture, result).await +} + +/// A legacy zero-fee fanout whose CPFP package the node accepted through +/// `submitpackage`, with the reply lost and the owning broadcaster killed +/// after it reserved, locked and signed its wallet funding. The survivor +/// keeps the one immutable package and its funding reservation pending +/// until the child confirms, then releases the wallet lock. +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn lost_submitpackage_reply_owner_killed_survivor_confirms_the_cpfp_package_once( +) -> Result<()> { + // Started directly on the node, so the schema exists for the in-process + // builder, and stopped before any work: the proxied servers replace them. + let Some(mut fixture) = Fixture::open(true).await? else { + return Ok(()); + }; + for server in &mut fixture.servers { + server.stop(); + } + let proxy = RpcReplyHold::start(fixture.rpc_port).await?; + let result = lost_package_reply(&mut fixture, &proxy).await; + finish(fixture, result).await +} + +/// A share credited durably whose acknowledgement never reaches the miner: +/// the connection closes between the server's answer and its delivery. The +/// retry of the identical share, on either frontend, is a duplicate, and the +/// ledger holds exactly one credit. +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn share_credited_but_ack_lost_retry_is_deduplicated_exactly() -> Result<()> { + let Some(mut fixture) = Fixture::open_with_servers(false, false).await? else { + return Ok(()); + }; + let result = lost_share_ack(&mut fixture).await; + finish(fixture, result).await +} + +async fn finish(fixture: Fixture, result: Result<()>) -> Result<()> { + if result.is_err() { + eprintln!("{}", fixture.diagnostics()); + } + let cleanup = fixture.cleanup().await; + result.and(cleanup) +} + +async fn start_frontends( + fixture: &mut Fixture, + proxy: &RpcReplyHold, + fee: Option, +) -> Result<()> { + for index in 0..2 { + let process = start_frontend(fixture, index, proxy.port, fee).await?; + match fixture.servers.get_mut(index) { + Some(stopped) => *stopped = process, + None => fixture.servers.push(process), + } + } + Ok(()) +} + +/// Kill `owner` and every database backend it had, with `gate` closed, and +/// open the gate. Returns how many of its updates the gate held and how +/// many backends were terminated. +async fn kill_owner_behind_gate( + fixture: &mut Fixture, + gate: JournalGate, + owner: usize, +) -> Result<(usize, usize)> { + kill_frontend(fixture, owner)?; + // Only then open the gate: an update held there, or still unread in a + // backend's socket, rolls back with its backend. + let held = gate.waiting(fixture).await?.len(); + let terminated = end_frontend_backends(fixture, owner).await?; + gate.remove(fixture).await?; + Ok((held, terminated)) +} + +/// The block hash of a `submitblock` call's parameters. +fn submitted_block(params: &Value) -> Result { + let block = hex::decode(params[0].as_str().context("submitblock without a block")?)?; + Ok(hash_display(&double_sha256( + block.get(..80).context("block shorter than its header")?, + ))) +} + +/// The txid of a raw transaction. +fn txid(raw: &str) -> Result { + Ok(hash_display(&double_sha256(&strip_witness_transaction( + &hex::decode(raw)?, + )?))) +} + +async fn wait_candidate(fixture: &Fixture, block: &str, state: &str, seconds: u64) -> Result<()> { + until(&format!("candidate {block} {state}"), seconds, || async { + let current: Option = + sqlx::query_scalar("SELECT state FROM qbit_block_candidate_outbox WHERE block_hash=$1") + .bind(block) + .fetch_optional(&fixture.pool) + .await?; + Ok(current.as_deref() == Some(state)) + }) + .await +} + +/// Mine one block on server `index`'s current work and wait for it to land. +async fn mine_block(fixture: &Fixture, index: usize, label: &str) -> Result { + let tip = fixture.rpc("getbestblockhash", json!([])).await?; + let tip = tip.as_str().context("tip missing")?; + let mut client = ShareClient::connect( + fixture.stratum[index], + &format!("{}.{label}", fixture.address), + ) + .await?; + client.work_on(tip, Duration::from_secs(30)).await?; + let submitted = client.submit(Proof::Block).await?; + ensure!( + submitted.answer.accepted(), + "block {} on server {index}: {}", + submitted.hash, + submitted.answer + ); + wait_candidate(fixture, &submitted.hash, "submitted", 60).await?; + Ok(submitted.hash) +} + +/// The accepted credits and credited headers for one header hash. +async fn credits(fixture: &Fixture, hash: &str) -> Result<(i64, i64)> { + Ok(sqlx::query_as( + "SELECT (SELECT count(*) FROM qbit_share_ledger WHERE accepted AND share_id LIKE '%:'||$1),(SELECT count(*) FROM qbit_prism_share_hashes WHERE header_hash=$1)", + ) + .bind(hash) + .fetch_one(&fixture.pool) + .await?) +} + +async fn lost_block_reply(fixture: &mut Fixture, proxy: &RpcReplyHold) -> Result<()> { + start_frontends(fixture, proxy, None).await?; + // Holds the recording of the offer's outcome from the start: under the + // 1 s `submitblock` deadline an owner would otherwise record `unknown` + // on its own before the kill. + let mut gate = JournalGate::install( + fixture, + "hold_offer_outcome", + "qbit_block_candidate_outbox", + "OLD.state='offer_reserved' AND NEW.state IS DISTINCT FROM 'offer_reserved'", + ) + .await?; + gate.close(fixture).await?; + let held = proxy.arm("submitblock"); + let tip = fixture.rpc("getbestblockhash", json!([])).await?; + let mut client = ShareClient::connect( + fixture.stratum[0], + &format!("{}.lost-reply", fixture.address), + ) + .await?; + client + .work_on( + tip.as_str().context("tip missing")?, + Duration::from_secs(30), + ) + .await?; + // The miner's answer is collected at the end: it may be lost with the + // owner, or wait on the landing, but it must never be a rejection. + let miner = tokio::spawn(async move { client.submit(Proof::Block).await }); + let held = tokio::time::timeout(Duration::from_secs(60), held) + .await + .context("no submitblock reached the proxy")??; + let block = submitted_block(&held.params)?; + ensure!( + held.reply["error"].is_null() && held.reply["result"].is_null(), + "the node did not accept {block}: {}", + held.reply + ); + let header = fixture.rpc("getblockheader", json!([block])).await?; + ensure!( + header["confirmations"].as_i64().unwrap_or(0) > 0, + "the accepted block {block} is not on the node's active chain" + ); + // The row carries its block bytes until it finishes. + let (owner, stored): (String, Vec) = sqlx::query_as( + "SELECT claim_instance_id,block_bytes FROM qbit_block_candidate_outbox WHERE block_hash=$1 AND state='offer_reserved' AND offer_outcome IS NULL", + ) + .bind(&block) + .fetch_one(&fixture.pool) + .await + .context("the held block's row is not an unrecorded reservation")?; + let owner_index = frontend_index(&owner)?; + let terminated = kill_owner_behind_gate(fixture, gate, owner_index).await?; + let _ = held.release.send(()); + let (state, outcome, instance): (String, Option, Option) = sqlx::query_as( + "SELECT state,offer_outcome,claim_instance_id FROM qbit_block_candidate_outbox WHERE block_hash=$1", + ) + .bind(&block) + .fetch_one(&fixture.pool) + .await?; + ensure!( + state == "offer_reserved" && outcome.is_none() && instance.as_deref() == Some(&owner), + "the killed owner journaled its offer: {state}, {outcome:?}, {instance:?}" + ); + expire_dead_lease( + fixture, + "qbit_block_candidate_outbox", + "block_hash", + &block, + &owner, + ) + .await?; + wait_candidate(fixture, &block, "submitted", 90).await?; + let submitted = miner.await??; + ensure!(submitted.hash == block, "the miner submitted another block"); + let answer_ok = match &submitted.answer { + Answer::Accepted | Answer::Lost(_) => true, + answer => answer.reason_id() == Some("ledger-outcome-unknown"), + }; + ensure!( + answer_ok, + "the miner was answered {} for a block the node accepted", + submitted.answer + ); + + // The killed frontend comes back and mines a block of its own. + fixture.servers[owner_index] = start_frontend(fixture, owner_index, proxy.port, None).await?; + let after = mine_block(fixture, owner_index, "after-restart").await?; + fixture.quiesce().await?; + + let offers: Vec = proxy + .calls("submitblock") + .iter() + .map(|call| submitted_block(&call.params)) + .collect::>()?; + ensure!( + offers == [block.clone(), after.clone()], + "each block must be offered exactly once: {offers:?}" + ); + let rows: Vec<(String, String, Option, bool)> = sqlx::query_as( + "SELECT block_hash,state,offer_outcome,claim_token IS NULL FROM qbit_block_candidate_outbox ORDER BY created_at", + ) + .fetch_all(&fixture.pool) + .await?; + ensure!( + rows == vec![ + ( + block.clone(), + "submitted".into(), + Some("unknown".into()), + true + ), + ( + after.clone(), + "submitted".into(), + Some("accepted".into()), + true + ), + ], + "unexpected candidate outcomes: {rows:?}" + ); + let chain: Vec<(String, String)> = + sqlx::query_as("SELECT block_hash,chain_state FROM qbit_pool_blocks ORDER BY block_height") + .fetch_all(&fixture.pool) + .await?; + ensure!( + chain + == vec![ + (block.clone(), "confirmed".into()), + (after.clone(), "confirmed".into()) + ], + "pool blocks disagree with the chain: {chain:?}" + ); + // The bytes the frontend journaled before the offer are the node's. + let raw = fixture.rpc("getblock", json!([block, 0])).await?; + ensure!( + hex::decode(raw.as_str().context("raw block missing")?)? == stored + && held.params[0].as_str().map(hex::decode).transpose()? == Some(stored.clone()), + "the journaled, offered and chain block bytes differ" + ); + // The survivor's audit verifies against the coinbase the node holds. + let survivor = 1 - owner_index; + let body: Value = fixture + .client + .get(format!( + "http://127.0.0.1:{}/audit/blocks/{block}/bundle", + fixture.api[survivor] + )) + .send() + .await? + .error_for_status()? + .json() + .await?; + let bundle: AuditBundle = serde_json::from_value(body["audit_bundle"].clone())?; + let verbose = fixture.rpc("getblock", json!([block, 2])).await?; + let coinbase = fixture + .rpc( + "getrawtransaction", + json!([verbose["tx"][0]["txid"], false, block]), + ) + .await?; + let key = ManifestSigningKey::from_seed_hex(&"22".repeat(32))?.public_key_hex(); + verify_audit_bundle_against_coinbase_tx_hex( + &bundle, + coinbase.as_str().context("node coinbase missing")?, + &key, + )?; + for hash in [&block, &after] { + let (credited, headers) = credits(fixture, hash).await?; + ensure!( + (credited, headers) == (1, 1), + "block {hash} credited {credited} time(s) under {headers} header(s)" + ); + } + fixture.integrity().await?; + eprintln!( + "lost submitblock reply: {block} accepted by the node, owner {owner} killed before journaling ({} update(s) held by the gate, {} backend(s) terminated); survivor live-{survivor} landed it as delivery unknown; one offer, node bytes, one credit, audit verified; miner answer {}; restarted {owner} then landed {after}", + terminated.0, + terminated.1, + submitted.answer + ); + Ok(()) +} + +/// The first CTV fanout of a block mined on server 0, once it is recorded: +/// its txid, signed bytes and block height. +async fn mined_fanout(fixture: &Fixture) -> Result<(String, String, i64)> { + let block = mine_block(fixture, 0, "fanout").await?; + let query = "SELECT a.fanout_txid,a.manifest->>'fanout_tx_hex',b.block_height FROM qbit_ctv_fanout_artifacts a JOIN qbit_pool_blocks b USING(block_hash) WHERE a.block_hash=$1 AND b.chain_state='confirmed'"; + until("the block's CTV fanout recorded", 60, || async { + Ok(sqlx::query(query) + .bind(&block) + .fetch_optional(&fixture.pool) + .await? + .is_some()) + }) + .await?; + Ok(sqlx::query_as(query) + .bind(&block) + .fetch_one(&fixture.pool) + .await?) +} + +/// The fanout row's journal: status, attempts, and claim owner. +async fn fanout_journal(fixture: &Fixture, fanout: &str) -> Result<(String, i64, Option)> { + Ok(sqlx::query_as( + "SELECT settlement_status,broadcast_attempt_count,claim_instance_id FROM qbit_ctv_fanout_artifacts WHERE fanout_txid=$1", + ) + .bind(fanout) + .fetch_one(&fixture.pool) + .await?) +} + +/// Install the gate that holds a fanout's completion: any update that ends +/// or replaces a claim (a renewal keeps the token and passes). +async fn fanout_gate(fixture: &Fixture) -> Result { + JournalGate::install( + fixture, + "hold_fanout_completion", + "qbit_ctv_fanout_artifacts", + "OLD.claim_token IS NOT NULL AND NEW.claim_token IS DISTINCT FROM OLD.claim_token", + ) + .await +} + +/// With a broadcaster's call held after the node accepted it, behind the +/// closed `gate`: kill its owner, check the attempt stayed unjournaled, and +/// hand its lease to the survivor. Returns the owner's instance id and index. +async fn kill_fanout_owner( + fixture: &mut Fixture, + gate: JournalGate, + fanout: &str, +) -> Result<(String, usize)> { + // The gate was closed before the call: the attempt cannot be journaled, + // so this is the row as the owner left it before sending. + let (status, attempts, owner) = fanout_journal(fixture, fanout).await?; + let owner = owner.context("the held fanout has no claim owner")?; + ensure!( + !matches!(status.as_str(), "broadcast_submitted" | "confirmed"), + "the owner journaled its held send before the kill: {status}" + ); + let owner_index = frontend_index(&owner)?; + let terminated = kill_owner_behind_gate(fixture, gate, owner_index).await?; + let journal = fanout_journal(fixture, fanout).await?; + ensure!( + journal == (status, attempts, Some(owner.clone())), + "the killed owner journaled its attempt: {journal:?}" + ); + expire_dead_lease( + fixture, + "qbit_ctv_fanout_artifacts", + "fanout_txid", + fanout, + &owner, + ) + .await?; + eprintln!( + "killed fanout owner {owner}; {} update(s) held by the gate, {} backend(s) terminated", + terminated.0, terminated.1 + ); + Ok((owner, owner_index)) +} + +/// Wait for the armed call's withheld reply while `gate` stays closed, so +/// the owner can never journal the call first, however late this task runs. +/// An attempt that ends without sending would wait on the gate and hold its +/// claim: it is let through, and the gate closes again at once. Returns the +/// held reply and how many such attempts passed. +async fn held_behind_gate( + fixture: &Fixture, + gate: &mut JournalGate, + mut held: tokio::sync::oneshot::Receiver, + call: &str, +) -> Result<(HeldReply, usize)> { + let deadline = Instant::now() + Duration::from_secs(60); + let mut passed = 0; + loop { + tokio::select! { + biased; + reply = &mut held => { + return Ok((reply.with_context(|| format!("the proxy dropped the held {call}"))?, passed)); + } + _ = tokio::time::sleep(Duration::from_millis(200)) => {} + } + ensure!( + Instant::now() < deadline, + "no {call} reached the proxy ({passed} unsent attempt(s) let through the gate)" + ); + if !gate.waiting(fixture).await?.is_empty() { + // The proxy hands over the held reply before it withholds the + // node's answer, so an update that follows a send always finds + // the reply already here: take it instead of opening the gate. + // Only an attempt that never sent gets through. + match held.try_recv() { + Ok(reply) => return Ok((reply, passed)), + Err(tokio::sync::oneshot::error::TryRecvError::Closed) => { + bail!("the proxy dropped the held {call}") + } + Err(tokio::sync::oneshot::error::TryRecvError::Empty) => {} + } + // Closing again waits for the released update's transaction. + gate.open().await?; + gate.close(fixture).await?; + passed += 1; + } + } +} + +async fn wait_fanout(fixture: &Fixture, fanout: &str, status: &str) -> Result<()> { + until(&format!("fanout {fanout} {status} and unclaimed"), 60, || async { + Ok(sqlx::query_scalar::<_, bool>( + "SELECT settlement_status=$2 AND claim_token IS NULL FROM qbit_ctv_fanout_artifacts WHERE fanout_txid=$1", + ) + .bind(fanout) + .bind(status) + .fetch_one(&fixture.pool) + .await?) + }) + .await +} + +async fn in_mempool(fixture: &Fixture, txid: &str) -> Result { + Ok(fixture + .rpc("getrawmempool", json!([])) + .await? + .as_array() + .is_some_and(|rows| rows.contains(&json!(txid)))) +} + +/// The fanout confirmed on chain with exactly `raw` as its bytes, spending +/// its covenant output, and nothing left in the mempool to pay again. +async fn confirmed_once(fixture: &Fixture, fanout: &str, raw: &str) -> Result<()> { + let (block, height): (String, i64) = sqlx::query_as( + "SELECT confirmed_block_hash,confirmed_block_height FROM qbit_ctv_fanout_artifacts WHERE fanout_txid=$1", + ) + .bind(fanout) + .fetch_one(&fixture.pool) + .await?; + ensure!( + fixture.rpc("getblockhash", json!([height])).await? == json!(block), + "the recorded confirmation is not on the active chain" + ); + let onchain = fixture + .rpc("getrawtransaction", json!([fanout, false, block])) + .await?; + ensure!( + onchain.as_str() == Some(raw), + "the confirmed fanout's bytes differ from the signed manifest" + ); + let manifest: Value = + sqlx::query_scalar("SELECT manifest FROM qbit_ctv_fanout_artifacts WHERE fanout_txid=$1") + .bind(fanout) + .fetch_one(&fixture.pool) + .await?; + ensure!( + fixture + .rpc( + "gettxout", + json!([ + manifest["parent_coinbase_txid"], + manifest["parent_coinbase_vout"], + true + ]) + ) + .await? + .is_null(), + "the covenant output is still unspent" + ); + ensure!( + fixture + .rpc("getrawmempool", json!([])) + .await? + .as_array() + .is_some_and(Vec::is_empty), + "the mempool still holds a payout transaction" + ); + Ok(()) +} + +async fn lost_fanout_reply(fixture: &mut Fixture, proxy: &RpcReplyHold) -> Result<()> { + start_frontends(fixture, proxy, None).await?; + // Installed open once the servers made the schema; nothing is claimable + // before the coinbase matures. + let mut gate = fanout_gate(fixture).await?; + let (fanout, raw, height) = mined_fanout(fixture).await?; + ensure!( + txid(&raw)? == fanout, + "the manifest's bytes are not its txid" + ); + // Closed before the coinbase matures: nothing is claimable until then. + gate.close(fixture).await?; + let held = proxy.arm("sendrawtransaction"); + let tip = fixture + .rpc("getblockcount", json!([])) + .await? + .as_i64() + .context("tip missing")?; + fixture + .rpc( + "generatetoaddress", + json!([height + 1000 - tip, fixture.address]), + ) + .await?; + let (held, passed) = held_behind_gate(fixture, &mut gate, held, "sendrawtransaction").await?; + ensure!( + held.params[0].as_str() == Some(raw.as_str()) && held.reply["result"] == json!(fanout), + "the node did not accept the fanout's signed bytes: {}", + held.reply + ); + ensure!( + in_mempool(fixture, &fanout).await?, + "the accepted fanout is not in the mempool" + ); + let (owner, owner_index) = kill_fanout_owner(fixture, gate, &fanout).await?; + let _ = held.release.send(()); + // The survivor finds the fanout in the mempool and records it. + wait_fanout(fixture, &fanout, "broadcast_submitted").await?; + fixture.servers[owner_index] = start_frontend(fixture, owner_index, proxy.port, None).await?; + fixture + .rpc("generatetoaddress", json!([1, fixture.address])) + .await?; + wait_fanout(fixture, &fanout, "confirmed").await?; + fixture.quiesce().await?; + + confirmed_once(fixture, &fanout, &raw).await?; + let sends = proxy.calls("sendrawtransaction"); + ensure!( + !sends.is_empty() + && sends + .iter() + .all(|call| call.params[0].as_str() == Some(raw.as_str())), + "a broadcast carried other bytes than the signed fanout" + ); + let fanouts: i64 = sqlx::query_scalar("SELECT count(*) FROM qbit_ctv_fanout_artifacts") + .fetch_one(&fixture.pool) + .await?; + ensure!(fanouts == 1, "{fanouts} fanouts recorded for one block"); + fixture.integrity().await?; + let (_, attempts, _) = fanout_journal(fixture, &fanout).await?; + eprintln!( + "lost sendrawtransaction reply: fanout {fanout} accepted by the node, owner {owner} killed before journaling; survivor recorded it from the mempool and it confirmed with its signed bytes; {} identical broadcast(s), {attempts} journaled attempt(s), {passed} unsent attempt(s) let through the gate first", + sends.len() + ); + Ok(()) +} + +async fn lost_package_reply(fixture: &mut Fixture, proxy: &RpcReplyHold) -> Result<()> { + let ledger = Ledger::connect(&fixture.database_url, "legacy-builder".into(), 4, false).await?; + let result = async { + let (fanout, _) = cpfp_tests::land_legacy_zero_fee_fanout(fixture, &ledger).await?; + let raw: String = sqlx::query_scalar( + "SELECT manifest->>'fanout_tx_hex' FROM qbit_ctv_fanout_artifacts WHERE fanout_txid=$1", + ) + .bind(&fanout) + .fetch_one(&fixture.pool) + .await?; + // Closed before any broadcaster runs. + let mut gate = fanout_gate(fixture).await?; + gate.close(fixture).await?; + let held = proxy.arm("submitpackage"); + start_frontends(fixture, proxy, Some(100_000)).await?; + let (held, passed) = held_behind_gate(fixture, &mut gate, held, "submitpackage").await?; + let child_raw = held.params[0][1] + .as_str() + .context("package without a child")? + .to_owned(); + let child = txid(&child_raw)?; + ensure!( + held.params[0][0].as_str() == Some(raw.as_str()) + && held.reply["result"]["package_msg"] == "success", + "the node did not accept the package: {}", + held.reply + ); + for tx in [&fanout, &child] { + ensure!(in_mempool(fixture, tx).await?, "the accepted package's {tx} is not in the mempool"); + } + // The owner saved its signed child before the call. + let package: (Option, String, i32, bool) = sqlx::query_as( + "SELECT signed_child_hex,funding_txid,funding_vout,wallet_lock_released FROM qbit_prism_cpfp_packages WHERE fanout_txid=$1", + ) + .bind(&fanout) + .fetch_one(&fixture.pool) + .await?; + ensure!( + package.0.as_deref() == Some(child_raw.as_str()) && !package.3, + "the durable package is not the one submitted: {package:?}" + ); + let (funding_txid, funding_vout) = (package.1.clone(), package.2); + let (owner, owner_index) = kill_fanout_owner(fixture, gate, &fanout).await?; + let _ = held.release.send(()); + wait_fanout(fixture, &fanout, "broadcast_submitted").await?; + // Unconfirmed: the funding stays reserved and locked, its cleanup + // explicitly pending, whoever owns the row now. + let released: bool = sqlx::query_scalar( + "SELECT wallet_lock_released FROM qbit_prism_cpfp_packages WHERE fanout_txid=$1", + ) + .bind(&fanout) + .fetch_one(&fixture.pool) + .await?; + ensure!(!released, "the takeover released an unconfirmed funding reservation"); + // The reservation is consumed by the durable child alone: qbitd's + // wallet drops its lock once a mempool transaction spends the coin, + // so the database flag above is the pending cleanup record, and the + // funding outpoint must be spent by exactly that child. + let outpoint = json!({"txid":funding_txid,"vout":funding_vout}); + ensure!( + fixture + .rpc("gettxout", json!([funding_txid, funding_vout, true])) + .await? + .is_null(), + "the reserved funding is unspent in the mempool" + ); + let decoded = fixture + .rpc("decoderawtransaction", json!([child_raw])) + .await?; + ensure!( + decoded["vin"].as_array().is_some_and(|inputs| inputs + .iter() + .any(|input| input["txid"] == funding_txid && input["vout"] == funding_vout)), + "the durable child does not spend the reserved funding" + ); + fixture.servers[owner_index] = + start_frontend(fixture, owner_index, proxy.port, Some(100_000)).await?; + fixture + .rpc("generatetoaddress", json!([1, fixture.address])) + .await?; + wait_fanout(fixture, &fanout, "confirmed").await?; + until("the confirmed child releases its funding lock", 30, || async { + let released: bool = sqlx::query_scalar( + "SELECT wallet_lock_released FROM qbit_prism_cpfp_packages WHERE fanout_txid=$1", + ) + .bind(&fanout) + .fetch_one(&fixture.pool) + .await?; + let locks = fixture.rpc("listlockunspent", json!([])).await?; + Ok(released && !locks.as_array().is_some_and(|rows| rows.contains(&outpoint))) + }) + .await?; + fixture.quiesce().await?; + + confirmed_once(fixture, &fanout, &raw).await?; + let confirmed = fixture.rpc("gettransaction", json!([child])).await?; + ensure!( + confirmed["confirmations"].as_u64().unwrap_or(0) > 0, + "the CPFP child did not confirm" + ); + let packages: Vec<(String, Option)> = sqlx::query_as( + "SELECT fanout_txid,signed_child_hex FROM qbit_prism_cpfp_packages", + ) + .fetch_all(&fixture.pool) + .await?; + ensure!( + packages == vec![(fanout.clone(), Some(child_raw.clone()))], + "the package was duplicated or rewritten: {packages:?}" + ); + let submits = proxy.calls("submitpackage"); + ensure!( + submits + .iter() + .all(|call| call.params[0] == json!([raw, child_raw])), + "a package submission carried other bytes" + ); + let retired: i64 = sqlx::query_scalar("SELECT count(*) FROM qbit_prism_cpfp_retired_funding") + .fetch_one(&fixture.pool) + .await?; + ensure!(retired == 0, "the takeover retired the signed package's funding"); + fixture.integrity().await?; + eprintln!( + "lost submitpackage reply: package {fanout}+{child} accepted by the node, owner {owner} killed before journaling; survivor recorded it from the mempool, kept funding {funding_txid}:{funding_vout} reserved until the child confirmed, then released it; {} identical submission(s), {passed} unsent attempt(s) let through the gate first", + submits.len() + ); + Ok::<_, anyhow::Error>(()) + } + .await; + ledger.pool.close().await; + result +} + +async fn lost_share_ack(fixture: &mut Fixture) -> Result<()> { + start_share_only_servers(fixture, &[(0, Vec::new()), (1, Vec::new())]).await?; + let relay = StratumRelay::start(fixture.stratum[0]).await?; + let username = format!("{}.lost-ack", fixture.address); + let mut session = StratumSession::open(relay.port).await?; + session + .request(&json!({"id":1,"method":"mining.subscribe","params":["prism-lost-ack"]})) + .await?; + let authorized = session + .request(&json!({"id":2,"method":"mining.authorize","params":[username,"x"]})) + .await?; + ensure!(authorized["result"] == true, "authorize: {authorized}"); + let (params, submitted) = session + .solve_latest(&username, Some(Proof::Share), false) + .await?; + let held = relay.arm(json!(10)); + session + .send(&json!({"id":10,"method":"mining.submit","params":params})) + .await?; + let held = tokio::time::timeout(ANSWER, held) + .await + .context("the server never answered the share")??; + ensure!( + held.answer["result"] == true && held.answer["error"].is_null(), + "the server did not accept the share: {}", + held.answer + ); + // The answer follows the ledger outcome: the credit is already durable. + ensure!( + credits(fixture, &submitted.hash).await? == (1, 1), + "the acknowledged share is not durably credited" + ); + let _ = held.sever.send(()); + let first = match session.answer(&json!(10), Duration::from_secs(10)).await { + Ok(message) => bail!("the miner received the withheld answer: {message}"), + Err(error) => Answer::Lost(format!("{error:#}")), + }; + + // The miner retries the identical submission on a new connection, to + // the same frontend and to the other one. Resuming the job right after + // the credit can race the frontend's republication of its work, which + // answers `unknown-job` and must be re-queried (`stratum.rs`, "a miss is + // not proof the ID is bogus"): a miner that retries is answered again, + // up to a few times, and nothing but that miss may precede the answer. + let mut retries = Vec::new(); + for index in [0, 1] { + let mut retry = StratumSession::open(fixture.stratum[index]).await?; + retry + .request(&json!({"id":1,"method":"mining.subscribe","params":["prism-lost-ack"]})) + .await?; + let authorized = retry + .request(&json!({"id":2,"method":"mining.authorize","params":[username,"x"]})) + .await?; + ensure!(authorized["result"] == true, "authorize: {authorized}"); + let mut misses = 0; + let answer = loop { + let answer = retry + .request(&json!({"id":11+misses,"method":"mining.submit","params":params})) + .await?; + if reason_id(&answer) != Some("unknown-job") || misses == 5 { + break answer; + } + misses += 1; + tokio::time::sleep(Duration::from_millis(200)).await; + }; + ensure!( + answer["result"] != true && reason_id(&answer) == Some("duplicate-share"), + "the retry on server {index} was not refused as a duplicate after {misses} transient miss(es): {answer}" + ); + retries.push((index, misses)); + } + ensure!( + credits(fixture, &submitted.hash).await? == (1, 1), + "the retried share was credited again" + ); + let rows: Vec<(String, bool)> = sqlx::query_as( + "SELECT share_id,accepted FROM qbit_share_ledger WHERE share_id LIKE $1||':%'", + ) + .bind(&username) + .fetch_all(&fixture.pool) + .await?; + ensure!( + rows == vec![(submitted.share_id.clone(), true)], + "the miner's ledger rows are not exactly the one credit: {rows:?}" + ); + fixture.integrity().await?; + eprintln!( + "lost share ACK: {} credited durably, the server's acceptance withheld and the connection closed; the miner's own outcome stays {first}; its identical retry was refused as a duplicate on both frontends (transient unknown-job misses first: {:?}); one ledger row, one credited header", + submitted.share_id, + retries + .iter() + .map(|(index, misses)| format!("server {index}: {misses}")) + .collect::>() + ); + Ok(()) +} diff --git a/crates/qbit-prism-server/tests/support/live_migration_lifecycle.rs b/crates/qbit-prism-server/tests/support/live_migration_lifecycle.rs new file mode 100644 index 000000000..ba827c094 --- /dev/null +++ b/crates/qbit-prism-server/tests/support/live_migration_lifecycle.rs @@ -0,0 +1,1009 @@ +//! #487 S20 (#545): the 2.x.x-to-native migration lifecycle on a real node. +//! +//! A 2.x.x ledger, built from the frozen `schema_2x` release SQL and scaled +//! up from the three rows `migration_rollback` seeds, goes through the whole +//! of `docs/prism-rust-migration.md#recovery-and-rollback`, the one-way +//! migration with isolated-restore reconciliation that #287 settled: +//! +//! 1. the drained source's evidence and a `pg_dump` backup (step 2); +//! 2. the backup restored into an isolated database, reconciling exactly +//! with the source before any native acknowledgement (step 3); +//! 3. `migrate` and `import-audits` rehearsed on a second isolated restore +//! (step 4), then run on the source for the cutover, each reconciling +//! exactly with the source and serving every historical artifact byte for +//! byte (step 5); +//! 4. both native servers started on the migrated ledger, mining on the real +//! node until their own blocks land, with the history untouched and +//! continued: sequences, publication order and carry integrity (step 7's +//! `self-check` included); +//! 5. the post-ACK boundary (step 6): the runbook's tail query names exactly +//! the native shares, the isolated restore still equals the source and +//! holds none of them, and the legacy revert refuses the native schema. +//! +//! The 2.x.x history lives on the regtest chain the servers mine: every +//! legacy pool block is a real block at its height, mature under the tip, so +//! the native observer checks the latest of them against the node as it +//! would in production, where a mature block missing from the chain is a +//! fatal state. #291's rehearsal stays the authority on production-sized +//! data; this is the repeatable version, run weekly. +use super::*; +use std::path::Path; + +#[path = "recovery.rs"] +#[allow(dead_code)] +mod recovery; + +// #575: the mainnet-shaped seed and the measured cutover rehearsal. +#[path = "cutover_rehearsal.rs"] +#[allow(dead_code)] +mod rehearsal; +use rehearsal::seed; + +/// #545's legacy history (`seed::seed_s20`): distinct payout recipients, +/// ledger rows and found blocks. +const LEGACY_RECIPIENTS: usize = seed::S20_RECIPIENTS; +const LEGACY_SHARES: usize = seed::S20_SHARES; +const LEGACY_BLOCKS: usize = seed::S20_BLOCKS; +/// Legacy pool blocks sit this many heights apart on the regtest chain. +const LEGACY_SPACING: u64 = 5; +/// The regtest coinbase maturity: the tip is mined this far past the last +/// legacy block so all of them are mature, as a drained 2.x.x pool's are. +const MATURITY: u64 = 100; +/// Native pool blocks each server must land before the post-ACK checks. +const NATIVE_BLOCKS: i64 = 3; +/// The live servers' signing keys (`PRISM_ALLOW_TEST_SIGNING_SEEDS`). The +/// legacy history is signed with the same keys, as a cutover keeps them. +const COINBASE_SEED: &str = "11"; +const LEDGER_SEED: &str = "22"; + +/// The 2.x.x columns of `qbit_share_ledger`, which native migrations extend: +/// the legacy prefix is compared on exactly these, before and after. +const LEGACY_SHARE_COLUMNS: &str = "share_seq,share_id,miner_id,payout_order_key,p2mr_program,share_difficulty,network_difficulty,template_height,job_id,job_issued_at,ntime,accepted_at,accepted,reject_reason,writer_id,writer_epoch,credit_policy"; + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +#[ignore = "weekly: a scaled 2.x.x ledger through migration, mining and rollback; run with --ignored"] +async fn weekly_2x_ledger_migrates_mines_reconciles_and_restores_in_isolation() -> Result<()> { + let Some(mut inputs) = gate::inputs( + gate::site!(), + &[ + gate::Input::QbitdBin, + gate::Input::DatabaseUrl, + gate::Input::PgBinDir, + ], + )? + else { + return Ok(()); + }; + let pg_bin = PathBuf::from(inputs.remove(2)); + let database = inputs.remove(1); + let qbitd = inputs.remove(0); + let mut fixture = Fixture::open_with_inputs(qbitd, &database, false, false).await?; + // Two isolated restores of the backup: one stays the pre-migration + // database for the rollback, the other rehearses the migration. + let restored = recovery::Database::open(&database).await; + let rehearsal = recovery::Database::open(&database).await; + let result = match (&restored, &rehearsal) { + (Ok(restored), Ok(rehearsal)) => run(&mut fixture, restored, rehearsal, &pg_bin).await, + (Err(error), _) | (_, Err(error)) => { + Err(anyhow::anyhow!("isolated restore database: {error:#}")) + } + }; + if result.is_err() { + eprintln!("{}", fixture.diagnostics()); + } + let cleanup = fixture.cleanup().await; + let mut closed = Ok(()); + for database in [restored, rehearsal].into_iter().flatten() { + closed = closed.and(database.close().await); + } + result.and(cleanup).and(closed) +} + +/// #575: the share density of the weekly mainnet-shaped ledger. At 1/16 it +/// holds about 4.1M of mainnet's 65.5M shares, each carrying 16 times a +/// share's difficulty; every block, address and payout row is mainnet's. +/// `migrate` runs at the server's default statement timeout: since #582, +/// 002's share-hash backfill runs after the migration transaction in +/// bounded batches, and the cutover fails if it does not. +const WEEKLY_DENSITY: f64 = 1.0 / 16.0; + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +#[ignore = "weekly: #575's mainnet-shaped 2.x.x ledger through the measured cutover, mining and rollback; run with --ignored"] +async fn weekly_mainnet_shaped_2x_ledger_cuts_over_measured_mines_and_restores_in_isolation( +) -> Result<()> { + let Some(mut inputs) = gate::inputs( + gate::site!(), + &[ + gate::Input::QbitdBin, + gate::Input::DatabaseUrl, + gate::Input::PgBinDir, + ], + )? + else { + return Ok(()); + }; + let pg_bin = PathBuf::from(inputs.remove(2)); + let database = inputs.remove(1); + let qbitd = inputs.remove(0); + // A smaller density for a host without the weekly runner's disk. + let density = match std::env::var("PRISM_REHEARSAL_WEEKLY_DENSITY") { + Ok(value) => value + .parse::() + .context("PRISM_REHEARSAL_WEEKLY_DENSITY must be a number")?, + Err(_) => WEEKLY_DENSITY, + }; + let shape = seed::MainnetShape::mainnet(1, density)?; + let mut fixture = Fixture::open_with_inputs(qbitd, &database, false, false).await?; + let result = run_mainnet(&mut fixture, shape, &database, &pg_bin).await; + if result.is_err() { + eprintln!("{}", fixture.diagnostics()); + } + let cleanup = fixture.cleanup().await; + result.and(cleanup) +} + +/// The phases' wall times, printed as the rehearsal records them. +#[derive(Default)] +struct Timings(Vec<(&'static str, Duration)>); +impl Timings { + async fn time(&mut self, phase: &'static str, work: impl Future) -> T { + let started = Instant::now(); + let value = work.await; + self.0.push((phase, started.elapsed())); + value + } +} + +async fn run( + fixture: &mut Fixture, + restored: &recovery::Database, + rehearsal: &recovery::Database, + pg_bin: &Path, +) -> Result<()> { + let mut timings = Timings::default(); + let artifacts_dir = tempfile::tempdir()?; + let legacy = timings + .time( + "seed 2.x.x", + seed_legacy_history(fixture, artifacts_dir.path()), + ) + .await?; + let source = recovery::Database { + admin: fixture.admin.clone(), + pool: fixture.pool.clone(), + schema: fixture.schema.clone(), + url: fixture.database_url.clone(), + }; + + // Step 2: the drained source baseline and its backup. + let source_evidence = timings + .time("source evidence", recovery::evidence(&source, pg_bin)) + .await?; + ensure!(source_evidence["unfinished_candidates"] == 0); + ensure!(source_evidence["carry_forward_integrity"]["mismatch_count"] == 0); + ensure!(source_evidence["accepted_shares"] == legacy.accepted_shares); + ensure!(source_evidence["last_share_seq"] == legacy.last_share_seq); + ensure!(source_evidence["records"]["audits"]["count"] == LEGACY_BLOCKS); + let legacy_rows = legacy_snapshot(&fixture.pool, &legacy).await?; + let archive = timings + .time("pg_dump", recovery::backup(&source, pg_bin)) + .await?; + + // Step 3: the pre-ACK recovery path, an isolated restore of the backup. + timings + .time( + "isolated restore", + recovery::restore(&archive, &source, restored, pg_bin), + ) + .await?; + ensure!( + recovery::evidence(restored, pg_bin).await? == source_evidence, + "the isolated restore does not reconcile with the drained source" + ); + ensure!( + native_tables_absent(&restored.pool).await?, + "the isolated restore must be the pre-migration database" + ); + + // Step 4: the forward migration and the audit import rehearsed on a + // second isolated restore of the same backup, reconciled exactly, with + // canonical reads of every artifact (step 5). + timings + .time( + "rehearsal restore", + recovery::restore(&archive, &source, rehearsal, pg_bin), + ) + .await?; + let artifacts_root = artifacts_dir.path().to_string_lossy().into_owned(); + timings + .time( + "rehearsal migrate and import", + migrate_and_import(fixture, &rehearsal.url, &artifacts_root), + ) + .await?; + ensure!( + recovery::evidence(rehearsal, pg_bin).await? == source_evidence, + "the rehearsed migration and import changed the reconciled history" + ); + recovery::assert_artifacts(&rehearsal.pool, &legacy.artifacts).await?; + + // The cutover: the same commands on the source, reconciled again before + // any native traffic. + timings + .time( + "migrate and import", + migrate_and_import(fixture, &fixture.database_url, &artifacts_root), + ) + .await?; + ensure!( + recovery::evidence(&source, pg_bin).await? == source_evidence, + "migration and import changed the reconciled history" + ); + // Every original external file is now unavailable: public reads must + // come from PostgreSQL. + artifacts_dir.close()?; + recovery::assert_artifacts(&fixture.pool, &legacy.artifacts).await?; + ensure!(legacy_snapshot(&fixture.pool, &legacy).await? == legacy_rows); + + let (tail, native) = mine_and_reconcile( + fixture, + restored, + pg_bin, + &legacy, + &legacy_rows, + &source_evidence, + &mut timings, + true, + None, + ) + .await?; + + let phases: Vec = timings + .0 + .iter() + .map(|(phase, took)| format!("{phase} {:.1}s", took.as_secs_f64())) + .collect(); + eprintln!( + "live migration lifecycle: {} legacy shares ({} accepted) and {LEGACY_BLOCKS} blocks for {LEGACY_RECIPIENTS} recipients reconciled through migrate and import; {} native shares and {} native blocks after cutover are exactly the records an isolated restore discards; {}", + LEGACY_SHARES, + legacy.accepted_shares, + tail, + native, + phases.join(", ") + ); + Ok(()) +} + +/// The native servers on the migrated ledger, mining on the real node until +/// each lands its own blocks, then the history checks and the post-ACK +/// boundary against the isolated restore. Returns the native tail's share +/// and block counts. +#[allow(clippy::too_many_arguments)] +async fn mine_and_reconcile( + fixture: &mut Fixture, + restored: &recovery::Database, + pg_bin: &Path, + legacy: &Legacy, + legacy_rows: &Value, + source_evidence: &Value, + timings: &mut Timings, + expect_continued_carry: bool, + api_balances: Option<&rehearsal::Balances>, +) -> Result<(usize, usize)> { + let source = recovery::Database { + admin: fixture.admin.clone(), + pool: fixture.pool.clone(), + schema: fixture.schema.clone(), + url: fixture.database_url.clone(), + }; + // The native servers on the migrated ledger, mining on the real node. + let mining = Instant::now(); + for index in 0..2 { + let process = fixture.start_server(index)?; + fixture.servers.push(process); + } + for index in 0..2 { + until( + &format!("PRISM HTTP readiness of server {index}"), + 60, + || async { + Ok(fixture + .client + .get(format!("http://127.0.0.1:{}/healthz", fixture.api[index])) + .send() + .await? + .status() + .is_success()) + }, + ) + .await?; + } + // Before any native share: every 2.x.x address's balances, as each + // frontend's public API serves them, equal what 2.x.x computed. + if let Some(before) = api_balances { + for port in fixture.api { + let served = rehearsal::api_balances(&fixture.client, port, before).await?; + ensure!( + &served == before, + "the migrated API serves other balances than 2.x.x computed ({} addresses)", + before.len() + ); + } + } + fixture.start_miner(0)?; + fixture.start_miner(1)?; + until("native pool blocks from both servers", 90, || async { + let landed: Vec = sqlx::query_scalar( + "SELECT count(DISTINCT b.block_hash) FROM qbit_pool_blocks b JOIN qbit_block_candidate_outbox o USING(block_hash) JOIN qbit_share_ledger s USING(share_id) WHERE b.chain_state='confirmed' AND s.writer_id=ANY($1) GROUP BY s.writer_id", + ) + .bind(["live-0", "live-1"]) + .fetch_all(&fixture.pool) + .await?; + Ok(landed.len() == 2 && landed.iter().all(|count| *count >= NATIVE_BLOCKS)) + }) + .await?; + fixture.quiesce().await?; + timings.0.push(("native mining", mining.elapsed())); + fixture.integrity().await?; + // The carry chain crosses the cutover: native blocks pay 2.x.x + // recipients on top of the balances 2.x.x left them, and the integrity + // report above replays that chain across both eras. + if expect_continued_carry { + let continued: i64 = sqlx::query_scalar( + "SELECT count(*) FROM qbit_payout_carry_forward WHERE carry_forward_seq>$1 AND prior_balance_sats<>0 AND miner_id IN (SELECT miner_id FROM qbit_payout_carry_forward WHERE carry_forward_seq<=$1)", + ) + .bind(legacy.last_carry_seq) + .fetch_one(&fixture.pool) + .await?; + ensure!( + continued > 0, + "no native carry row continued a 2.x.x balance" + ); + } + + // The history is untouched and continued. + ensure!( + &legacy_snapshot(&fixture.pool, legacy).await? == legacy_rows, + "native mining changed a 2.x.x row" + ); + let native = native_blocks(fixture, legacy).await?; + ensure!(!native.is_empty(), "no native pool block confirmed"); + let legacy_last_ordinal = legacy.last_publication_ordinal; + for (hash, ordinal) in &native { + ensure!( + ordinal.is_some_and(|ordinal| ordinal > legacy_last_ordinal), + "native block {hash} did not continue the 2.x.x publication order: {ordinal:?} after {legacy_last_ordinal}" + ); + verify_native_audit(fixture, hash).await?; + } + let self_check = timings + .time( + "self-check", + fixture.tool(&fixture.database_url, &["self-check"]), + ) + .await?; + let self_check: Value = serde_json::from_str(&self_check)?; + ensure!( + self_check["audit_completeness"] + == json!({"missing_stored_bodies": 0, "missing_canonical_bytes": 0}), + "self-check reports incomplete history: {self_check}" + ); + + // Step 6: the post-ACK boundary, from the runbook's own tail query. + let tail: Vec<(i64, String, String)> = sqlx::query_as( + "SELECT share_seq, share_id, writer_id FROM qbit_share_ledger WHERE accepted AND share_seq > $1 ORDER BY share_seq", + ) + .bind(legacy.last_share_seq) + .fetch_all(&fixture.pool) + .await?; + let native_accepted: i64 = sqlx::query_scalar( + "SELECT count(*) FROM qbit_share_ledger WHERE accepted AND writer_id=ANY($1)", + ) + .bind(["live-0", "live-1"]) + .fetch_one(&fixture.pool) + .await?; + ensure!( + !tail.is_empty() && tail.len() as i64 == native_accepted, + "the post-ACK tail is not exactly the native shares: {} rows, {native_accepted} native", + tail.len() + ); + ensure!( + tail.iter().all( + |(seq, _, writer)| *seq > legacy.last_share_seq + legacy.sequence_headroom + && (writer == "live-0" || writer == "live-1") + ), + "a native share reused the 2.x.x sequence range or has another writer" + ); + let current_evidence = recovery::evidence(&source, pg_bin).await?; + ensure!( + current_evidence["accepted_shares"] == legacy.accepted_shares + native_accepted, + "evidence does not count the native tail" + ); + ensure!(current_evidence["carry_forward_integrity"]["mismatch_count"] == 0); + ensure!(current_evidence["records"]["shares"] != source_evidence["records"]["shares"]); + + // The rollback after the ACK: the isolated restore is still exactly the + // source, and every native record is what restoring it would discard. + ensure!( + recovery::evidence(restored, pg_bin).await? == *source_evidence, + "the isolated restore drifted while the native servers ran" + ); + let tail_ids: Vec<&str> = tail.iter().map(|(_, id, _)| id.as_str()).collect(); + let survived: i64 = + sqlx::query_scalar("SELECT count(*) FROM qbit_share_ledger WHERE share_id=ANY($1)") + .bind(&tail_ids) + .fetch_one(&restored.pool) + .await?; + ensure!(survived == 0, "the older restore holds post-cutover shares"); + let native_hashes: Vec<&str> = native.iter().map(|(hash, _)| hash.as_str()).collect(); + let restored_blocks: i64 = + sqlx::query_scalar("SELECT count(*) FROM qbit_pool_blocks WHERE block_hash=ANY($1)") + .bind(&native_hashes) + .fetch_one(&restored.pool) + .await?; + ensure!( + restored_blocks == 0, + "the older restore holds native blocks" + ); + assert_legacy_revert_refuses(fixture).await?; + + Ok((tail.len(), native.len())) +} + +/// #575's weekly rehearsal: a mainnet-shaped 2.x.x ledger over a real +/// regtest chain, backed up and restored in isolation, cut over with the +/// measured `check-config`, `migrate` and `import-audits` and every +/// invariant of `rehearsal::rehearse`, then mined natively and reconciled +/// exactly as #545's scenario is. +async fn run_mainnet( + fixture: &mut Fixture, + shape: seed::MainnetShape, + database: &str, + pg_bin: &Path, +) -> Result<()> { + let mut timings = Timings::default(); + let artifacts_dir = tempfile::tempdir()?; + let work = tempfile::tempdir()?; + let plan = seed::MainnetPlan::new(shape, chrono::Utc::now().timestamp_millis())?; + let chain = timings + .time("mine the 2.x.x chain", mine_plan_chain(fixture, &plan)) + .await?; + let coinbase_key = ManifestSigningKey::from_seed_hex(&COINBASE_SEED.repeat(32))?; + let ledger_key = ManifestSigningKey::from_seed_hex(&LEDGER_SEED.repeat(32))?; + let (seeded, summary) = timings + .time( + "seed 2.x.x", + plan.write( + &fixture.pool, + artifacts_dir.path(), + &chain, + &coinbase_key, + &ledger_key, + ), + ) + .await?; + let mut legacy = Legacy::from(seeded); + legacy.digest_rows = true; + let source = recovery::Database { + admin: fixture.admin.clone(), + pool: fixture.pool.clone(), + schema: fixture.schema.clone(), + url: fixture.database_url.clone(), + }; + + // Step 2: the drained source baseline and its backup. + let source_evidence = timings + .time("source evidence", recovery::evidence(&source, pg_bin)) + .await?; + ensure!(source_evidence["unfinished_candidates"] == 0); + ensure!(source_evidence["accepted_shares"] == legacy.accepted_shares); + ensure!(source_evidence["last_share_seq"] == legacy.last_share_seq); + let legacy_rows = legacy_snapshot(&fixture.pool, &legacy).await?; + let before_balances = rehearsal::balances(&fixture.pool).await?; + let dump = work.path().join("source.dump"); + timings + .time( + "pg_dump", + pg_dump_schema(database, &fixture.schema, pg_bin, &dump), + ) + .await?; + + // Step 3: the pre-ACK recovery path, an isolated restore of the backup + // into a database of its own. + let (restored, restored_name) = timings + .time( + "isolated restore", + restore_database(fixture, database, &dump, pg_bin), + ) + .await?; + let result = async { + ensure!( + recovery::evidence(&restored, pg_bin).await? == source_evidence, + "the isolated restore does not reconcile with the drained source" + ); + ensure!( + native_tables_absent(&restored.pool).await?, + "the isolated restore must be the pre-migration database" + ); + + // Steps 4-5 on the source itself, measured: check-config, migrate, + // import-audits, then evidence, sums, balances and the window. + let mut report = rehearsal::Report::new(summary); + let target = rehearsal::Target { + url: fixture.database_url.clone(), + schema: fixture.schema.clone(), + admin: fixture.admin.clone(), + pool: fixture.pool.clone(), + }; + let started = Instant::now(); + rehearsal::rehearse( + &target, + database, + &rehearsal::Options { + pg_bin: pg_bin.to_owned(), + audit_root: artifacts_dir.path().to_owned(), + ledger_public_key: ledger_key.public_key_hex(), + node: rehearsal::NodeChoice::External { + url: format!("http://127.0.0.1:{}/", fixture.rpc_port), + user: "prismtest".into(), + password: "prismtest".into(), + chain: "regtest".into(), + }, + start_frontend: false, + // The server's own statement timeout (#582). + extra_env: Vec::new(), + // The cutover runs on the source itself; the isolated + // restore is checked against it above. + source_evidence: None, + }, + &mut report, + ) + .await?; + timings.0.push(("measured cutover", started.elapsed())); + eprintln!("{}", report.render()); + eprintln!("{}", serde_json::to_string(&report)?); + ensure!(report.pass, "the measured cutover failed"); + // The ledger has rows, so 002's share-hash backfill ran after the + // migration transaction, as its own measured step (#582). + ensure!( + report + .steps + .iter() + .any(|step| step.name == "migrate: 002 share-hash backfill"), + "the cutover did not run 002's share-hash backfill online" + ); + // A sample of the historical artifacts, served from PostgreSQL + // once the original files are gone. + artifacts_dir.close()?; + let sample: Vec = legacy + .artifacts + .iter() + .step_by((legacy.artifacts.len() / 40).max(1)) + .chain(legacy.artifacts.last()) + .map(|artifact| recovery::Artifact { + block_hash: artifact.block_hash.clone(), + digest: artifact.digest.clone(), + canonical: artifact.canonical.clone(), + }) + .collect(); + recovery::assert_artifacts(&fixture.pool, &sample).await?; + ensure!(legacy_snapshot(&fixture.pool, &legacy).await? == legacy_rows); + + let (tail, native) = mine_and_reconcile( + fixture, + &restored, + pg_bin, + &legacy, + &legacy_rows, + &source_evidence, + &mut timings, + true, + Some(&before_balances), + ) + .await?; + let phases: Vec = timings + .0 + .iter() + .map(|(phase, took)| format!("{phase} {:.1}s", took.as_secs_f64())) + .collect(); + eprintln!( + "live mainnet-shaped cutover: {} legacy rows ({} accepted) and {} blocks for {} addresses reconciled through the measured cutover; {tail} native shares and {native} native blocks after cutover are exactly the records an isolated restore discards; {}", + legacy.last_share_seq, + legacy.accepted_shares, + legacy.hashes.len(), + before_balances.len(), + phases.join(", ") + ); + anyhow::Ok(()) + } + .await; + restored.pool.close().await; + restored.admin.close().await; + let dropped = sqlx::query(&format!("DROP DATABASE {restored_name} WITH (FORCE)")) + .execute(&fixture.admin) + .await; + result.and(dropped.map(|_| ()).map_err(Into::into)) +} + +/// Mines the regtest chain the plan's blocks sit on, from genesis: every +/// height at the time the plan gives it (`setmocktime`), so each legacy block +/// carries its found time and the tip ends where history does. qbitd refuses +/// a block more than ten minutes ahead of its clock, and a burst of 14,000 +/// blocks at the current time would run past that; the fixture's first block +/// is invalidated so history can start in the past. +async fn mine_plan_chain( + fixture: &Fixture, + plan: &seed::MainnetPlan, +) -> Result> { + let height = fixture + .rpc("getblockcount", json!([])) + .await? + .as_u64() + .context("block count missing")?; + if height > 0 { + let first = fixture.rpc("getblockhash", json!([1])).await?; + fixture.rpc("invalidateblock", json!([first])).await?; + } + let times = plan.chain_times(); + let mined = async { + for time in × { + fixture.rpc("setmocktime", json!([time])).await?; + fixture + .rpc("generatetoaddress", json!([1, fixture.address])) + .await?; + } + anyhow::Ok(()) + } + .await; + // The node's own clock again, whatever happened. + fixture.rpc("setmocktime", json!([0])).await?; + mined?; + // Every height from genesis, so each block's parent is at hand. + let mut hashes = Vec::with_capacity(times.len() + 1); + for height in 0..=times.len() as u64 { + hashes.push( + fixture + .rpc("getblockhash", json!([height])) + .await? + .as_str() + .context("block hash missing")? + .to_owned(), + ); + } + let (offsets, _) = plan.chain_offsets(); + Ok(offsets + .into_iter() + .map(|offset| { + let height = 1 + offset; + seed::ChainBlock { + height, + hash: hashes[height as usize].clone(), + parent: hashes[height as usize - 1].clone(), + } + }) + .collect()) +} + +/// `pg_dump` of one schema in the custom format, as an operator backs up. +async fn pg_dump_schema(database: &str, schema: &str, pg_bin: &Path, file: &Path) -> Result<()> { + let output = tokio::process::Command::new(pg_bin.join("pg_dump")) + .args([ + "--format=custom", + "--no-owner", + "--no-privileges", + "--schema", + ]) + .arg(schema) + .arg("--file") + .arg(file) + .arg("--dbname") + .arg(database) + .output() + .await?; + ensure!( + output.status.success(), + "pg_dump failed: {}", + String::from_utf8_lossy(&output.stderr) + ); + Ok(()) +} + +/// Restores `dump` into a new database on the same server, schema name and +/// all, with `pg_restore`: the scale #545's statement-by-statement restore +/// does not reach. +async fn restore_database( + fixture: &Fixture, + database: &str, + dump: &Path, + pg_bin: &Path, +) -> Result<(recovery::Database, String)> { + let name = format!("prism_rollback_{}", Uuid::new_v4().simple()); + sqlx::query(&format!("CREATE DATABASE {name}")) + .execute(&fixture.admin) + .await?; + let mut url = url::Url::parse(database)?; + url.set_path(&format!("/{name}")); + let output = tokio::process::Command::new(pg_bin.join("pg_restore")) + .args([ + "--no-owner", + "--no-privileges", + "--exit-on-error", + "--dbname", + ]) + .arg(url.as_str()) + .arg(dump) + .output() + .await?; + ensure!( + output.status.success(), + "pg_restore failed: {}", + String::from_utf8_lossy(&output.stderr) + ); + let admin = PgPool::connect(url.as_str()).await?; + url.query_pairs_mut() + .append_pair("options", &format!("-csearch_path={}", fixture.schema)); + Ok(( + recovery::Database { + admin, + pool: PgPool::connect(url.as_str()).await?, + schema: fixture.schema.clone(), + url: url.into(), + }, + name, + )) +} + +/// The legacy rows #545's snapshot compares, as per-chunk digests, so a +/// ledger of millions of rows compares without holding it in memory. +async fn legacy_digest(pool: &PgPool, legacy: &Legacy) -> Result { + Ok(sqlx::query_scalar(&format!( + "SELECT jsonb_build_object(\ + 'shares',(SELECT jsonb_agg(h ORDER BY c) FROM (SELECT share_seq/100000 AS c,md5(string_agg(md5(ROW({LEGACY_SHARE_COLUMNS})::text),'' ORDER BY share_seq)) AS h FROM qbit_share_ledger WHERE share_seq<=$1 GROUP BY 1) s),\ + 'blocks',(SELECT md5(string_agg(md5(ROW(b.block_hash,b.block_height,b.parent_hash,b.coinbase_txid,b.payout_manifest_sha256,b.chain_state,b.maturity_state,b.audit_publication_sequence,a.audit_bundle_sha256)::text),'' ORDER BY b.block_hash)) FROM qbit_pool_blocks b JOIN qbit_pool_audit_bundles a USING(block_hash) WHERE b.block_hash=ANY($2)),\ + 'carry',(SELECT jsonb_agg(h ORDER BY c) FROM (SELECT carry_forward_seq/100000 AS c,md5(string_agg(md5(ROW(carry_forward_seq,block_height,block_hash,miner_id,p2mr_program,gross_amount_sats,prior_balance_sats,candidate_balance_sats,onchain_amount_sats,carry_forward_balance_sats,action,maturity_state)::text),'' ORDER BY carry_forward_seq)) AS h FROM qbit_payout_carry_forward WHERE carry_forward_seq<=$3 GROUP BY 1) q))" + )) + .bind(legacy.last_share_seq) + .bind(&legacy.hashes) + .bind(legacy.last_carry_seq) + .fetch_one(pool) + .await?) +} + +/// `migrate` then `import-audits` against `database_url`, as the runbook +/// runs them, requiring the recorded 2.x.x source and complete history. +async fn migrate_and_import(fixture: &Fixture, database_url: &str, root: &str) -> Result<()> { + let migrated = fixture.tool(database_url, &["migrate"]).await?; + ensure!( + migrated.contains("database source:") && !migrated.contains("unrecorded"), + "migrate did not record the 2.x.x source: {migrated}" + ); + let imported = fixture + .tool(database_url, &["import-audits", "--root", root]) + .await?; + ensure!( + imported.contains(&format!("Imported {LEGACY_BLOCKS} audit bodies")) + && imported.contains("\"missing_stored_bodies\":0") + && imported.contains("\"missing_canonical_bytes\":0"), + "import-audits left history incomplete: {imported}" + ); + Ok(()) +} + +/// What the seed wrote that later phases check against. +struct Legacy { + accepted_shares: i64, + last_share_seq: i64, + sequence_headroom: i64, + last_publication_ordinal: i64, + last_carry_seq: i64, + hashes: Vec, + artifacts: Vec, + /// Compare the legacy rows by digest rather than as one JSON document, + /// for a ledger too large to hold in memory. + digest_rows: bool, +} + +impl From for Legacy { + fn from(seeded: seed::Seeded) -> Self { + Self { + accepted_shares: seeded.accepted_shares, + last_share_seq: seeded.last_share_seq, + sequence_headroom: seeded.sequence_headroom, + last_publication_ordinal: seeded.last_publication_ordinal, + last_carry_seq: seeded.last_carry_seq, + hashes: seeded.hashes, + artifacts: seeded + .artifacts + .into_iter() + .map(|artifact| recovery::Artifact { + block_hash: artifact.block_hash, + digest: artifact.digest, + canonical: artifact.canonical, + }) + .collect(), + digest_rows: false, + } + } +} + +/// Mines the 2.x.x-era chain, then writes #545's drained 2.x.x ledger over +/// it (`seed::seed_s20`): the frozen release DDL, ordered shares from a +/// skewed recipient set with rejects among them, a signed audit bundle per +/// found block in the three historical body layouts, chained carry-forward +/// rows, and terminal candidate rows. +async fn seed_legacy_history(fixture: &Fixture, root: &Path) -> Result { + let tip = fixture + .rpc("getblockcount", json!([])) + .await? + .as_u64() + .context("block count missing")?; + let first = tip + 1; + let last = first + (LEGACY_BLOCKS as u64 - 1) * LEGACY_SPACING; + let mined = last - tip + MATURITY; + fixture + .rpc("generatetoaddress", json!([mined, fixture.address])) + .await?; + let mut blocks = Vec::with_capacity(LEGACY_BLOCKS); + for index in 0..LEGACY_BLOCKS as u64 { + let height = first + index * LEGACY_SPACING; + blocks.push(chain_block(fixture, height).await?); + } + let now_ms = chrono::Utc::now().timestamp_millis(); + let coinbase_key = ManifestSigningKey::from_seed_hex(&COINBASE_SEED.repeat(32))?; + let ledger_key = ManifestSigningKey::from_seed_hex(&LEDGER_SEED.repeat(32))?; + Ok(seed::seed_s20( + &fixture.pool, + root, + &blocks, + now_ms, + &coinbase_key, + &ledger_key, + ) + .await? + .into()) +} + +/// The block the node has at `height`, with its parent. +async fn chain_block(fixture: &Fixture, height: u64) -> Result { + let hash = fixture + .rpc("getblockhash", json!([height])) + .await? + .as_str() + .context("block hash missing")? + .to_owned(); + let header = fixture.rpc("getblockheader", json!([hash])).await?; + let parent = header["previousblockhash"] + .as_str() + .context("parent hash missing")? + .to_owned(); + Ok(seed::ChainBlock { + height, + hash, + parent, + }) +} + +/// Every 2.x.x row a migration or native writer could touch, on the 2.x.x +/// columns only: the share prefix, the blocks with their publication order +/// and audit identity, and the carry chain. +async fn legacy_snapshot(pool: &PgPool, legacy: &Legacy) -> Result { + if legacy.digest_rows { + return legacy_digest(pool, legacy).await; + } + Ok(sqlx::query_scalar(&format!( + "SELECT jsonb_build_object(\ + 'shares',(SELECT jsonb_agg(to_jsonb(s) ORDER BY share_seq) FROM (SELECT {LEGACY_SHARE_COLUMNS} FROM qbit_share_ledger WHERE share_seq<=$1) s),\ + 'blocks',(SELECT jsonb_agg(jsonb_build_array(b.block_hash,b.block_height,b.parent_hash,b.coinbase_txid,b.payout_manifest_sha256,b.chain_state,b.maturity_state,b.audit_publication_sequence,a.audit_bundle_sha256) ORDER BY b.block_height) FROM qbit_pool_blocks b JOIN qbit_pool_audit_bundles a USING(block_hash) WHERE b.block_hash=ANY($2)),\ + 'carry',(SELECT jsonb_agg(jsonb_build_array(carry_forward_seq,block_height,block_hash,miner_id,p2mr_program,gross_amount_sats,prior_balance_sats,candidate_balance_sats,onchain_amount_sats,carry_forward_balance_sats,action,maturity_state) ORDER BY carry_forward_seq) FROM qbit_payout_carry_forward WHERE carry_forward_seq<=$3))" + )) + .bind(legacy.last_share_seq) + .bind(&legacy.hashes) + .bind(legacy.last_carry_seq) + .fetch_one(pool) + .await?) +} + +/// The confirmed pool blocks the native servers found, with their +/// publication ordinals. +async fn native_blocks(fixture: &Fixture, legacy: &Legacy) -> Result)>> { + Ok(sqlx::query_as( + "SELECT block_hash, audit_publication_sequence FROM qbit_pool_blocks WHERE chain_state='confirmed' AND NOT block_hash=ANY($1) ORDER BY block_height", + ) + .bind(&legacy.hashes) + .fetch_all(&fixture.pool) + .await?) +} + +/// A native block's audit bundle, served by the public API, verifies against +/// the coinbase the node actually has. +async fn verify_native_audit(fixture: &Fixture, hash: &str) -> Result<()> { + let body: Value = fixture + .client + .get(format!( + "http://127.0.0.1:{}/audit/blocks/{hash}/bundle", + fixture.api[1] + )) + .send() + .await? + .error_for_status()? + .json() + .await?; + let bundle: AuditBundle = serde_json::from_value(body["audit_bundle"].clone())?; + let block = fixture.rpc("getblock", json!([hash, 2])).await?; + let coinbase = fixture + .rpc( + "getrawtransaction", + json!([block["tx"][0]["txid"], false, hash]), + ) + .await?; + let key = ManifestSigningKey::from_seed_hex(&LEDGER_SEED.repeat(32))?.public_key_hex(); + verify_audit_bundle_against_coinbase_tx_hex( + &bundle, + coinbase.as_str().context("node coinbase missing")?, + &key, + )?; + Ok(()) +} + +async fn native_tables_absent(pool: &PgPool) -> Result { + Ok( + sqlx::query_scalar("SELECT to_regclass('qbit_prism_schema_migrations') IS NULL") + .fetch_one(pool) + .await?, + ) +} + +/// 2.x.x's destructive publication-ordinal revert refuses the native schema +/// and names the one-way procedure, leaving the schema as it was. +async fn assert_legacy_revert_refuses(fixture: &Fixture) -> Result<()> { + let mut connection = fixture.pool.acquire().await?; + let error = sqlx::raw_sql(include_str!( + "../../../qbit-prism/sql/001_share_ledger_revert_audit_publication_sequence.sql" + )) + .execute(&mut *connection) + .await + .err() + .context("the legacy revert ran against the native schema")?; + sqlx::query("ROLLBACK").execute(&mut *connection).await?; + let message = error.to_string(); + ensure!( + message.contains("one-way migration") + && message.contains("docs/prism-rust-migration.md#recovery-and-rollback"), + "the legacy revert refused without naming the procedure: {message}" + ); + Ok(()) +} + +impl Fixture { + /// Runs an operator subcommand of the server binary with server 0's + /// settings against `database_url`, as the runbook runs it beside the + /// frontends, and returns its standard output. The migrated ledger's + /// writer key is the one the history was signed with. + async fn tool(&self, database_url: &str, args: &[&str]) -> Result { + let ledger_public_key = + ManifestSigningKey::from_seed_hex(&LEDGER_SEED.repeat(32))?.public_key_hex(); + let mut command = self.server_command( + 0, + None, + &[ + ("PRISM_DATABASE_URL", database_url.to_owned()), + ("PRISM_LEDGER_WRITER_PUBLIC_KEY_HEX", ledger_public_key), + ], + ); + command.args(args).stdin(Stdio::null()); + let output = tokio::time::timeout( + Duration::from_secs(300), + tokio::process::Command::from(command).output(), + ) + .await + .with_context(|| format!("qbit-prism-server {args:?} did not finish in 300 s"))??; + let stdout = String::from_utf8(output.stdout)?; + ensure!( + output.status.success(), + "qbit-prism-server {args:?} failed with {}: {stdout}{}", + output.status, + String::from_utf8_lossy(&output.stderr) + ); + Ok(stdout) + } +} diff --git a/crates/qbit-prism-server/tests/support/live_node_outage.rs b/crates/qbit-prism-server/tests/support/live_node_outage.rs new file mode 100644 index 000000000..2dcdb2f77 --- /dev/null +++ b/crates/qbit-prism-server/tests/support/live_node_outage.rs @@ -0,0 +1,1148 @@ +//! #521 scenario 4: a qbitd outage around `submitblock`, with a real node. +//! +//! Both servers reach the node through an HTTP JSON-RPC proxy owned by the +//! test. The proxy forwards every call unchanged, records every `submitblock` +//! it sees, and can hold one: just before the request is forwarded (the node +//! has not seen the block), or just after the node's reply was read (the node +//! accepted the block, the offering server has not yet heard so). While the +//! call is held, all proxy traffic is paused, the test stops (`SIGSTOP`) or +//! kills (`SIGKILL`) the node, and only then is the call released. Blocks are +//! mined one at a time by a Stratum client in the test, so each fault has +//! exactly one block in flight. +//! +//! A request the proxy has read was written by the server, so a kill before +//! the proxy forwards it is, for the server, a call that may have reached the +//! node. The kill-before-submit case therefore faults earlier (#522): a +//! database trigger holds the block's offer reservation while the node is +//! killed and the proxy closes its port, so the one `submitblock` finds the +//! connection refused before any byte is written. The port stays closed until +//! the restarted node has finished its warmup, so the servers see only refused +//! connections, never a warmup reply. +//! +//! The warmup case (#526) faults the same way, then restarts the node held in +//! its warmup and reopens the port: the retry reaches the real node, which +//! answers `RPC_IN_WARMUP` (-28) without running the call. The row must go +//! back to `pending` again, and the block lands once the node is warm. The +//! hold is deterministic: the node's fee-estimates file is a FIFO, which qbitd +//! opens during startup after its RPC server is up and before its warmup ends, +//! and that open blocks until the test opens the other end. +//! +//! Each case asserts: both servers report the node unavailable on /healthz +//! and /metrics and recover; every block is offered exactly once; the held block lands +//! (the node resumed with it, reindexed it, or, when the offer was never +//! sent, received it once it was back) or reconciles to a proven orphan (the +//! node lost the accepted block in the crash); no row is left in +//! `offer_reserved` or `offered`; and a new block found after recovery lands. +use super::*; +use num_bigint::BigUint; +use qbit_prism_server::codec::{ + difficulty_target, double_sha256, hash_display, parse_u32_hex, target_from_compact, +}; +use qbit_prism_server::rpc::RPC_IN_WARMUP; +use std::{collections::HashMap, sync::Arc}; +use tokio::{ + io::AsyncReadExt, + net::{ + tcp::{OwnedReadHalf, OwnedWriteHalf}, + TcpStream, + }, + sync::{oneshot, watch}, +}; + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn node_stopped_before_submitblock_lands_the_block_once_after_resume() -> Result<()> { + outage_case(Fault::Stop, Hold::Request, false).await +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn node_stopped_after_accepting_lands_the_block_once_after_resume() -> Result<()> { + outage_case(Fault::Stop, Hold::Reply, false).await +} + +/// #522: the node dies after the block passed Stratum admission and before +/// its `submitblock`, which finds the port refusing connections. Nothing +/// reached the node, so the row returns to `pending` and retries while the +/// port refuses, and the restarted node, on the same tip, receives the block +/// exactly once and it lands. +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn node_killed_before_submitblock_lands_the_block_once_after_restart() -> Result<()> { + outage_case(Fault::Kill, Hold::Reservation, false).await +} + +/// #526: as above, but the retry reaches the restarted node while it is +/// still warming up. The node answers -28 without running the call, the row +/// returns to `pending` again, and once the node is warm the block lands: +/// exactly one `submitblock` the node ran. +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn node_restarted_into_warmup_before_submitblock_lands_the_block_once() -> Result<()> { + outage_case(Fault::Kill, Hold::Warmup, false).await +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn node_killed_after_accepting_reconciles_the_lost_block_once() -> Result<()> { + outage_case(Fault::Kill, Hold::Reply, false).await +} + +/// The crash variant whose restart runs `-reindex`, which recovers the +/// accepted block from the block files. Opt-in (`--ignored`): no PR job runs +/// it; it is meant for the nightly job #521 adds. +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +#[ignore = "opt-in #521 variant for the nightly job: run with --ignored"] +async fn node_killed_after_accepting_and_reindexed_lands_the_block_once() -> Result<()> { + outage_case(Fault::Kill, Hold::Reply, true).await +} + +#[derive(Clone, Copy, Debug, PartialEq)] +pub(super) enum Fault { + /// `SIGSTOP`, later `SIGCONT`: the node keeps its sockets and state. + Stop, + /// `SIGKILL`, later a restart on the same data directory and RPC port. + Kill, +} + +#[derive(Clone, Copy, Debug, PartialEq)] +pub(super) enum Hold { + /// Before the `submitblock` request is forwarded to the node. + Request, + /// After the node's reply to `submitblock` was read, before it is + /// relayed to the offering server. + Reply, + /// Before the `submitblock` is sent: the held block's offer reservation + /// waits on a database trigger, and the proxy refuses connections from + /// the fault until the node is back. + Reservation, + /// As `Reservation`, but the port reopens while the restarted node is + /// held in its warmup, so a retry is answered -28 (#526). + Warmup, +} + +async fn outage_case(fault: Fault, hold: Hold, reindex: bool) -> Result<()> { + let Some(mut fixture) = Fixture::open_with_servers(false, false).await? else { + return Ok(()); + }; + let result = outage_case_on( + &mut fixture, + fault, + hold, + reindex, + async |_| Ok(()), + async |_| Ok(()), + ) + .await; + let cleanup = fixture.cleanup().await; + result.and(cleanup) +} + +/// One case on a fixture opened without servers, which the caller cleans +/// up: `ready` once both servers report healthy, before the first block, +/// and `after` while the servers still reach the node through the proxy. +/// #553 runs the cases under session load. +pub(super) async fn outage_case_on( + fixture: &mut Fixture, + fault: Fault, + hold: Hold, + reindex: bool, + ready: R, + after: F, +) -> Result<()> +where + R: AsyncFnOnce(&mut Fixture) -> Result<()>, + F: AsyncFnOnce(&mut Fixture) -> Result<()>, +{ + let proxy = RpcProxy::start(fixture.rpc_port).await?; + let mut result = outage_steps(fixture, &proxy, fault, hold, reindex, ready).await; + if result.is_ok() { + result = after(fixture).await; + } + if result.is_err() { + eprintln!("proxy saw submitblock for {:?}", proxy.submitted()); + eprintln!("{}", fixture.diagnostics()); + } + proxy.stop(); + result +} + +async fn outage_steps( + fixture: &mut Fixture, + proxy: &RpcProxy, + fault: Fault, + hold: Hold, + reindex: bool, + ready: R, +) -> Result<()> +where + R: AsyncFnOnce(&mut Fixture) -> Result<()>, +{ + // The servers are the only clients that go through the proxy; the test's + // own calls go to the node directly. + let node_port = fixture.rpc_port; + fixture.rpc_port = proxy.port; + let started = (0..2).try_for_each(|index| { + let process = fixture.start_server(index)?; + fixture.servers.push(process); + Ok::<_, anyhow::Error>(()) + }); + fixture.rpc_port = node_port; + started?; + wait_health(fixture, true, 30).await?; + ready(fixture).await?; + + // A block before the fault: the proxy is transparent. + let genesis = fixture.rpc("getbestblockhash", json!([])).await?; + let mut client = BlockClient::current(fixture, 0, &genesis).await?; + let before = client.mine().await?; + wait_state(fixture, &before, "submitted", 30).await?; + if fault == Fault::Kill { + // A crash loses whatever the node has not flushed. Flush now so the + // kill loses at most the held block, never the chain before it. + fixture.rpc("gettxoutsetinfo", json!([])).await?; + } + + // The held block, on current work that builds on the first one. It + // ends with one outcome: a request the node read and answered after its + // resume, the node's acceptance, or (never sent while the port refused) + // the acceptance of its one offer after the restart. + let pid = libc::pid_t::try_from(fixture.node.child.lock().id())?; + let mut client = BlockClient::current(fixture, 0, &json!(before)).await?; + let outcome = match hold { + Hold::Request => "unknown", + Hold::Reply | Hold::Reservation | Hold::Warmup => "accepted", + }; + let held = match hold { + Hold::Reservation | Hold::Warmup => { + unsent_offer(fixture, proxy, &mut client, fault, pid, &before).await? + } + Hold::Request | Hold::Reply => { + held_offer(fixture, proxy, &mut client, fault, pid, hold, outcome).await? + } + }; + wait_health(fixture, false, 60).await?; + for index in 0..2 { + let label = format!("server {index} metrics showing the node unavailable"); + until(&label, 30, || async { + let during = metrics(fixture, index).await?; + Ok(sample(&during, "qbit_prism_health_state") == Some(0.0) + && sample(&during, "qbit_prism_node_observation_age_seconds") + .is_some_and(|age| age >= 5.0)) + }) + .await?; + } + // The accepting frontend's alarm for an accepted block that has not + // landed; nothing was accepted while the request was held. + ensure!( + (unlanded(fixture).await? > 0.0) == (hold == Hold::Reply), + "accepted-unlanded signal disagrees with the node's answer" + ); + + match fault { + Fault::Stop => signal(pid, libc::SIGCONT)?, + Fault::Kill if hold == Hold::Warmup => { + restart_node_in_warmup(fixture, proxy, &held).await? + } + Fault::Kill => restart_node(fixture, reindex).await?, + } + if hold == Hold::Reservation { + // The restarted node is out of its warmup: its port accepts again. + proxy.reopen().await?; + } + wait_health(fixture, true, 60).await?; + for index in 0..2 { + let label = format!("server {index} metrics showing the node recovered"); + until(&label, 30, || async { + let recovered = metrics(fixture, index).await?; + Ok(sample(&recovered, "qbit_prism_health_state") == Some(1.0) + && sample(&recovered, "qbit_prism_node_observation_age_seconds") + .is_some_and(|age| (0.0..5.0).contains(&age))) + }) + .await?; + } + + // The node resumed with the held request in its socket, accepted the + // block and kept it (only a reindex recovers it after a crash), or was + // never sent it and receives its one offer now: the block lands. + // Otherwise the node lost the accepted block in the crash, and the row + // reconciles to a proven orphan once a competing block has the orphan + // confirmations; an offered row is never offered again. + let lands = fault == Fault::Stop || reindex || matches!(hold, Hold::Reservation | Hold::Warmup); + if lands { + wait_state(fixture, &held, "submitted", 90).await?; + let header = fixture.rpc("getblockheader", json!([held])).await?; + ensure!( + header["confirmations"].as_i64().unwrap_or(0) > 0, + "the landed block is not on the node's active chain" + ); + } else { + ensure!( + fixture.rpc("getblockheader", json!([held])).await.is_err(), + "the node holds a block the case expects it never received or lost" + ); + until("the held row released into reconciliation", 60, || async { + Ok(sqlx::query_scalar::<_, bool>( + "SELECT state='reconciliation' AND claim_token IS NULL FROM qbit_block_candidate_outbox WHERE block_hash=$1", + ) + .bind(&held) + .fetch_one(&fixture.pool) + .await?) + }) + .await?; + fixture + .rpc("generatetoaddress", json!([6, fixture.address])) + .await?; + wait_state(fixture, &held, "orphaned", 90).await?; + } + + // Job delivery resumes: a new connection to the other server receives + // work on the node's current tip, and its block lands. + let tip = fixture.rpc("getbestblockhash", json!([])).await?; + let mut client = BlockClient::current(fixture, 1, &tip).await?; + let after = client.mine().await?; + wait_state(fixture, &after, "submitted", 60).await?; + + fixture.quiesce().await?; + let stuck: i64 = sqlx::query_scalar( + "SELECT count(*) FROM qbit_block_candidate_outbox WHERE state IN ('pending','offer_reserved','offered')", + ) + .fetch_one(&fixture.pool) + .await?; + ensure!(stuck == 0, "{stuck} candidate row(s) stuck before landing"); + // A call the node answered from its warmup never ran (#526): only the + // warmup case sees one, and only for the held block. + let submitted = proxy.submitted(); + let warmup = proxy.warmup_answered(); + ensure!( + if hold == Hold::Warmup { + !warmup.is_empty() && warmup.iter().all(|block| *block == held) + } else { + warmup.is_empty() + }, + "unexpected submitblock calls answered in warmup: {warmup:?}" + ); + let mut offers = HashMap::<&str, usize>::new(); + for block in &submitted { + *offers.entry(block.as_str()).or_default() += 1; + } + for block in &warmup { + *offers + .get_mut(block.as_str()) + .context("a warmup answer without its call")? -= 1; + } + let expected = [before.as_str(), held.as_str(), after.as_str()]; + ensure!( + offers.len() == expected.len() + && expected.iter().all(|block| offers.get(block) == Some(&1)), + "each block must be run by the node exactly once; submitblock calls: {submitted:?}, answered in warmup: {warmup:?}" + ); + let rows: Vec<(String, String, Option)> = sqlx::query_as( + "SELECT block_hash,state,offer_outcome FROM qbit_block_candidate_outbox ORDER BY created_at", + ) + .fetch_all(&fixture.pool) + .await?; + let held_state = if lands { "submitted" } else { "orphaned" }; + ensure!( + rows == vec![ + (before.clone(), "submitted".into(), Some("accepted".into())), + (held.clone(), held_state.into(), Some(outcome.into())), + (after.clone(), "submitted".into(), Some("accepted".into())), + ], + "unexpected candidate outcomes: {rows:?}" + ); + let chain: Vec<(String, String)> = + sqlx::query_as("SELECT block_hash,chain_state FROM qbit_pool_blocks ORDER BY block_height") + .fetch_all(&fixture.pool) + .await?; + for block in [&before, &after] { + ensure!( + chain.contains(&(block.clone(), "confirmed".into())), + "block {block} is not confirmed in the pool: {chain:?}" + ); + } + ensure!( + lands == chain.contains(&(held.clone(), "confirmed".into())), + "held block credit disagrees with the node: {chain:?}" + ); + // A peer's orphan settlement reaches the accepting frontend through its + // 10-second database collector. + until("accepted-unlanded alarm cleared", 30, || async { + Ok(unlanded(fixture).await? == 0.0) + }) + .await?; + fixture.integrity().await?; + eprintln!( + "live node outage ({fault:?}, held {hold:?}, reindex {reindex}): held block {held} ended {held_state} after {} submitblock call(s) in all, {} answered in warmup", + submitted.len(), + warmup.len() + ); + Ok(()) +} + +/// The fault itself: `SIGSTOP` the node, or kill it. +fn inject(fixture: &mut Fixture, fault: Fault, pid: libc::pid_t) -> Result<()> { + match fault { + Fault::Stop => signal(pid, libc::SIGSTOP), + Fault::Kill => { + fixture.node.stop(); + Ok(()) + } + } +} + +/// Mine the held block with its `submitblock` held in the proxy (`hold`), +/// inject the fault, release the call, and wait for its recorded outcome. +async fn held_offer( + fixture: &mut Fixture, + proxy: &RpcProxy, + client: &mut BlockClient, + fault: Fault, + pid: libc::pid_t, + hold: Hold, + outcome: &str, +) -> Result { + let hit = proxy.arm(hold); + let held = client.mine().await?; + let hit = tokio::time::timeout(Duration::from_secs(30), hit) + .await + .context("no submitblock reached the proxy")??; + ensure!( + hit.block == held, + "held submitblock was for {}, not the mined block {held}", + hit.block + ); + inject(fixture, fault, pid)?; + let _ = hit.release.send(()); + until("the held offer's outcome recorded", 30, || async { + Ok(sqlx::query_scalar::<_, Option>( + "SELECT offer_outcome FROM qbit_block_candidate_outbox WHERE block_hash=$1", + ) + .bind(&held) + .fetch_one(&fixture.pool) + .await? + .as_deref() + == Some(outcome)) + }) + .await?; + Ok(held) +} + +/// Mine the held block with its offer reservation held in the database, +/// inject the fault, close the proxy's port, and release the reservation: +/// the one `submitblock` finds the connection refused (#522). The row must +/// return to `pending` unsent and keep retrying, with nothing reaching the +/// node, while the port refuses. +async fn unsent_offer( + fixture: &mut Fixture, + proxy: &RpcProxy, + client: &mut BlockClient, + fault: Fault, + pid: libc::pid_t, + before: &str, +) -> Result { + let gate = ReservationGate::close(fixture).await?; + let held = client.mine().await?; + gate.wait_for_reservation(fixture).await?; + inject(fixture, fault, pid)?; + proxy.refuse().await?; + gate.open(fixture).await?; + wait_unsent(fixture, &held, "connection refused").await?; + until("a second unsent offer while the port refuses", 30, || async { + Ok(sqlx::query_scalar::<_, bool>( + "SELECT attempt_count>=2 AND state='pending' AND offer_outcome IS NULL FROM qbit_block_candidate_outbox WHERE block_hash=$1", + ) + .bind(&held) + .fetch_one(&fixture.pool) + .await?) + }) + .await?; + ensure!( + proxy.submitted() == [before], + "a submitblock reached the node while its port refused: {:?}", + proxy.submitted() + ); + Ok(held) +} + +fn signal(pid: libc::pid_t, signal: libc::c_int) -> Result<()> { + // SAFETY: kill(2) takes no pointers; `pid` is the fixture's own child. + ensure!( + unsafe { libc::kill(pid, signal) } == 0, + "signal {signal} to qbitd failed: {}", + std::io::Error::last_os_error() + ); + Ok(()) +} + +/// Restart the killed node on its data directory and RPC port, as the CPFP +/// cases restart theirs. +async fn restart_node(fixture: &mut Fixture, reindex: bool) -> Result<()> { + spawn_node(fixture, reindex)?; + until("qbit restarted after the crash", 60, || async { + Ok(fixture.rpc("getblockcount", json!([])).await?.is_u64()) + }) + .await +} + +fn spawn_node(fixture: &mut Fixture, reindex: bool) -> Result<()> { + let mut node = Command::new(&fixture.qbitd); + node.args([ + "-regtest", + "-server=1", + "-listen=0", + "-dnsseed=0", + "-discover=0", + "-fallbackfee=0.00001", + "-rpcuser=prismtest", + "-rpcpassword=prismtest", + "-txindex=0", + "-wallet=prism", + ]) + .arg(format!("-datadir={}", fixture.directory.path().display())) + .arg(format!("-rpcport={}", fixture.rpc_port)) + .arg(format!("-port={}", free_port()?)); + if reindex { + node.arg("-reindex"); + } + fixture.node = Process::spawn( + &mut node, + fixture.directory.path().join("qbit-restarted.log"), + )?; + Ok(()) +} + +/// Restart the killed node held in its warmup and reopen the proxy's port +/// (#526). qbitd opens its fee-estimates file in startup step 6, after its +/// RPC server is up (step 4a) and before its warmup ends (step 13); as a +/// FIFO, that open blocks until the test opens the other end, and until then +/// the node answers every call -28. The held block's retry reaches the node +/// and is answered from the warmup; the row must be back in `pending` with +/// that answer as its reason, and only then is the node released. +async fn restart_node_in_warmup(fixture: &mut Fixture, proxy: &RpcProxy, held: &str) -> Result<()> { + let fifo = fixture + .directory + .path() + .join("regtest") + .join("fee_estimates.dat"); + match std::fs::remove_file(&fifo) { + Err(error) if error.kind() != std::io::ErrorKind::NotFound => return Err(error.into()), + _ => {} + } + let path = std::ffi::CString::new(fifo.as_os_str().as_encoded_bytes())?; + // SAFETY: `path` is a valid NUL-terminated string for the call's duration. + ensure!( + unsafe { libc::mkfifo(path.as_ptr(), 0o600) } == 0, + "mkfifo {}: {}", + fifo.display(), + std::io::Error::last_os_error() + ); + spawn_node(fixture, false)?; + until( + "the restarted qbit answering from its warmup", + 60, + || async { Ok(warmup_code(fixture).await == Some(RPC_IN_WARMUP)) }, + ) + .await?; + proxy.reopen().await?; + until("the held block's retry answered in warmup", 60, || async { + Ok(proxy.warmup_answered().iter().any(|block| block == held)) + }) + .await?; + wait_unsent(fixture, held, "\"code\":-28").await?; + ensure!( + warmup_code(fixture).await == Some(RPC_IN_WARMUP), + "the node left its warmup while it was held" + ); + // Release the startup: an empty fee-estimates file is read, rejected and + // ignored. A non-blocking open for writing fails (ENXIO) until the node + // is waiting in its open for reading, so a node that never gets there + // fails the wait instead of hanging the test. The node still reads the + // file's time after its open returns, so the FIFO goes only once the + // warmup has ended; the node's later flush then writes a plain file. + until( + "the held node opening its fee-estimates file", + 30, + || async { + use std::os::unix::fs::OpenOptionsExt; + Ok(std::fs::OpenOptions::new() + .write(true) + .custom_flags(libc::O_NONBLOCK) + .open(&fifo) + .is_ok()) + }, + ) + .await?; + until("qbit out of its warmup", 60, || async { + Ok(fixture.rpc("getblockcount", json!([])).await?.is_u64()) + }) + .await?; + std::fs::remove_file(&fifo)?; + Ok(()) +} + +/// The JSON-RPC error code of a direct `getblockcount`, `None` when it +/// succeeded or the node did not answer. +async fn warmup_code(fixture: &Fixture) -> Option { + let reply: Value = fixture + .client + .post(format!("http://127.0.0.1:{}/", fixture.rpc_port)) + .basic_auth("prismtest", Some("prismtest")) + .json(&json!({"jsonrpc":"1.0","id":"live-test","method":"getblockcount","params":[]})) + .send() + .await + .ok()? + .json() + .await + .ok()?; + reply["error"]["code"].as_i64() +} + +/// Holds every offer reservation on the fixture's outbox: a trigger waits on +/// an advisory lock the test holds on its own connection (#522). +struct ReservationGate { + connection: sqlx::pool::PoolConnection, + pid: i32, +} + +impl ReservationGate { + async fn close(fixture: &Fixture) -> Result { + sqlx::raw_sql(&format!( + "CREATE FUNCTION {schema}.hold_offer_reservation() RETURNS trigger LANGUAGE plpgsql AS $$ BEGIN PERFORM pg_advisory_xact_lock(hashtext(TG_TABLE_SCHEMA)::bigint); RETURN NEW; END $$; CREATE TRIGGER hold_offer_reservation BEFORE UPDATE OF state ON {schema}.qbit_block_candidate_outbox FOR EACH ROW WHEN (OLD.state='pending' AND NEW.state='offer_reserved') EXECUTE FUNCTION {schema}.hold_offer_reservation();", + schema = fixture.schema + )) + .execute(&fixture.pool) + .await?; + let mut connection = fixture.pool.acquire().await?; + sqlx::query("SELECT pg_advisory_lock(hashtext($1)::bigint)") + .bind(&fixture.schema) + .execute(&mut *connection) + .await?; + let pid = sqlx::query_scalar("SELECT pg_backend_pid()") + .fetch_one(&mut *connection) + .await?; + Ok(Self { connection, pid }) + } + + /// Until an offer reservation waits on the gate. + async fn wait_for_reservation(&self, fixture: &Fixture) -> Result<()> { + until("an offer reservation waiting on the gate", 30, || async { + Ok(sqlx::query_scalar::<_, bool>( + "SELECT EXISTS(SELECT 1 FROM pg_stat_activity WHERE $1=ANY(pg_blocking_pids(pid)))", + ) + .bind(self.pid) + .fetch_one(&fixture.admin) + .await?) + }) + .await + } + + /// Release the waiting reservation and remove the trigger. + async fn open(mut self, fixture: &Fixture) -> Result<()> { + sqlx::query("SELECT pg_advisory_unlock(hashtext($1)::bigint)") + .bind(&fixture.schema) + .execute(&mut *self.connection) + .await?; + drop(self.connection); + sqlx::raw_sql(&format!( + "DROP TRIGGER hold_offer_reservation ON {schema}.qbit_block_candidate_outbox; DROP FUNCTION {schema}.hold_offer_reservation();", + schema = fixture.schema + )) + .execute(&fixture.pool) + .await?; + Ok(()) + } +} + +/// Until the held block's offer is recorded as not sent: back in `pending` +/// with no outcome and `answer` (the refusal, or the warmup reply) in its +/// reason. A recorded outcome fails at once: the offer would never be +/// offered again. +async fn wait_unsent(fixture: &Fixture, block: &str, answer: &str) -> Result<()> { + let started = Instant::now(); + loop { + let (state, outcome, error): (String, Option, Option) = sqlx::query_as( + "SELECT state,offer_outcome,last_error FROM qbit_block_candidate_outbox WHERE block_hash=$1", + ) + .bind(block) + .fetch_one(&fixture.pool) + .await?; + ensure!( + outcome.is_none(), + "the offer not sent ({answer}) was recorded with outcome {outcome:?} ({state}): {error:?}" + ); + if state == "pending" + && error + .as_deref() + .is_some_and(|error| error.starts_with("offer not sent") && error.contains(answer)) + { + return Ok(()); + } + ensure!( + started.elapsed() < Duration::from_secs(30), + "the offer not sent ({answer}) was not returned to pending: {state}, {error:?}" + ); + tokio::time::sleep(Duration::from_millis(100)).await; + } +} + +async fn wait_health(fixture: &Fixture, ready: bool, seconds: u64) -> Result<()> { + for index in 0..2 { + let label = format!( + "server {index} health {}", + if ready { "ready" } else { "unavailable" } + ); + until(&label, seconds, || async { + let response = fixture + .client + .get(format!("http://127.0.0.1:{}/healthz", fixture.api[index])) + .send() + .await?; + Ok(response.status().is_success() == ready) + }) + .await?; + } + Ok(()) +} + +async fn metrics(fixture: &Fixture, index: usize) -> Result { + Ok(fixture + .client + .get(format!("http://127.0.0.1:{}/metrics", fixture.api[index])) + .send() + .await? + .error_for_status()? + .text() + .await?) +} + +/// A sample of an unlabelled gauge. +fn sample(body: &str, name: &str) -> Option { + body.lines() + .find_map(|line| line.strip_prefix(name)?.strip_prefix(' ')?.parse().ok()) +} + +/// The oldest accepted-but-unlanded block's age across both servers. +async fn unlanded(fixture: &Fixture) -> Result { + let mut oldest = 0.0f64; + for index in 0..2 { + let body = metrics(fixture, index).await?; + oldest = oldest.max( + sample(&body, "qbit_prism_accepted_block_unlanded_seconds") + .context("accepted-unlanded gauge missing")?, + ); + } + Ok(oldest) +} + +async fn wait_state(fixture: &Fixture, block: &str, state: &str, seconds: u64) -> Result<()> { + until(&format!("candidate {block} {state}"), seconds, || async { + let current: Option = + sqlx::query_scalar("SELECT state FROM qbit_block_candidate_outbox WHERE block_hash=$1") + .bind(block) + .fetch_optional(&fixture.pool) + .await?; + Ok(current.as_deref() == Some(state)) + }) + .await +} + +/// A Stratum client that mines one block per call on the current job. +struct BlockClient { + reader: BufReader, + writer: OwnedWriteHalf, + username: String, + extranonce1: String, + extranonce2_size: usize, + difficulty: Option, + notify: Value, + next_id: u64, +} + +impl BlockClient { + /// A new connection to `index` once that server is ready on `tip`, so + /// its first job carries the current tip and payout revision. + async fn current(fixture: &Fixture, index: usize, tip: &Value) -> Result { + until(&format!("server {index} ready on {tip}"), 30, || async { + let health: Value = fixture + .client + .get(format!("http://127.0.0.1:{}/healthz", fixture.api[index])) + .send() + .await? + .json() + .await?; + Ok(health["ok"] == true && &health["observed_tip"] == tip) + }) + .await?; + let mut client = Self::open(fixture, index).await?; + client + .wait_for_parent(tip.as_str().context("tip hash missing")?) + .await?; + Ok(client) + } + + async fn open(fixture: &Fixture, index: usize) -> Result { + let stream = TcpStream::connect(("127.0.0.1", fixture.stratum[index])).await?; + let (read, writer) = stream.into_split(); + let mut client = Self { + reader: BufReader::new(read), + writer, + username: format!("{}.outage-{index}", fixture.address), + extranonce1: String::new(), + extranonce2_size: 0, + difficulty: None, + notify: Value::Null, + next_id: 10, + }; + client + .send(json!({"id":1,"method":"mining.subscribe","params":["outage-regtest"]})) + .await?; + let subscribed = client.response(1).await?; + client.extranonce1 = subscribed["result"][1] + .as_str() + .context("extranonce1 missing")? + .into(); + client.extranonce2_size = subscribed["result"][2] + .as_u64() + .context("extranonce2 size missing")? + .try_into()?; + client + .send(json!({"id":2,"method":"mining.authorize","params":[client.username,"x"]})) + .await?; + let authorized = client.response(2).await?; + ensure!(authorized["result"] == true, "authorize: {authorized}"); + client.work().await?; + Ok(client) + } + + async fn send(&mut self, payload: Value) -> Result<()> { + self.writer + .write_all(format!("{payload}\n").as_bytes()) + .await?; + Ok(()) + } + + async fn read(&mut self) -> Result { + let mut line = String::new(); + let read = tokio::time::timeout(Duration::from_secs(20), self.reader.read_line(&mut line)) + .await + .context("no Stratum message within 20 s")??; + ensure!(read > 0, "Stratum connection closed"); + let message: Value = serde_json::from_str(&line)?; + if message["method"] == "mining.set_difficulty" { + self.difficulty = message["params"][0].as_f64(); + } + if message["method"] == "mining.notify" { + self.notify = message.clone(); + } + Ok(message) + } + + async fn response(&mut self, id: u64) -> Result { + loop { + let message = self.read().await?; + if message["id"] == id { + return Ok(message); + } + } + } + + /// Until both a job and a share difficulty have arrived. + async fn work(&mut self) -> Result<()> { + while self.notify.is_null() || self.difficulty.is_none() { + self.read().await?; + } + Ok(()) + } + + /// Until the current job builds on `parent` (display order). + async fn wait_for_parent(&mut self, parent: &str) -> Result<()> { + let mut wire = hex::decode(parent)?; + wire.reverse(); + for word in wire.as_chunks_mut::<4>().0 { + word.reverse(); + } + let expected = hex::encode(wire); + while self.notify["params"][1].as_str() != Some(expected.as_str()) { + self.read().await?; + } + Ok(()) + } + + /// Solve the current job for a block, submit it, and return its hash. + /// Nothing is awaited after the submission: a held reply must be + /// released inside the server's 1-second `submitblock` deadline. + async fn mine(&mut self) -> Result { + self.work().await?; + let params = self.notify["params"] + .as_array() + .context("notify missing")? + .clone(); + let field = |index: usize| params[index].as_str().context("invalid notify field"); + self.next_id += 1; + let extranonce2 = format!( + "{:0width$x}", + self.next_id, + width = self.extranonce2_size * 2 + ); + let coinbase = hex::decode(format!( + "{}{}{extranonce2}{}", + field(2)?, + self.extranonce1, + field(3)? + ))?; + let mut merkle = double_sha256(&coinbase); + for sibling in params[4].as_array().context("merkle branch missing")? { + let sibling = hex::decode(sibling.as_str().context("invalid sibling")?)?; + merkle = double_sha256(&[merkle.as_slice(), sibling.as_slice()].concat()); + } + let mut previous = hex::decode(field(1)?)?; + for word in previous.as_chunks_mut::<4>().0 { + word.reverse(); + } + let bits = parse_u32_hex(field(6)?)?; + let target = target_from_compact(bits)?.min(difficulty_target( + self.difficulty.context("share difficulty missing")?, + )?); + let ntime = parse_u32_hex(field(7)?)?; + let version = parse_u32_hex(field(5)?)?; + for nonce in 0..1_000_000u32 { + let header = [ + version.to_le_bytes().as_slice(), + previous.as_slice(), + merkle.as_slice(), + ntime.to_le_bytes().as_slice(), + bits.to_le_bytes().as_slice(), + nonce.to_le_bytes().as_slice(), + ] + .concat(); + let hash = double_sha256(&header); + if BigUint::from_bytes_le(&hash) <= target { + let request = json!({"id":self.next_id,"method":"mining.submit","params":[self.username,field(0)?,extranonce2,format!("{ntime:08x}"),format!("{nonce:08x}")]}); + self.send(request).await?; + return Ok(hash_display(&hash)); + } + } + bail!("no regtest block in the nonce budget") + } +} + +/// One held `submitblock`: its block hash, and the release the test sends +/// once the fault is in place. +struct Hit { + block: String, + release: oneshot::Sender<()>, +} + +struct ProxyState { + upstream: u16, + submitted: Mutex>, + /// Blocks whose `submitblock` the node answered -28 from its warmup: + /// calls it never ran (#526). + warmup_answered: Mutex>, + trap: Mutex)>>, + /// Closed while a held call waits for its release: every connection + /// pauses before its next forward, so nothing reaches the node or the + /// servers between the hold and the fault. + open: watch::Sender, + /// Every relayed connection, so a refusal can close them all. + connections: Mutex>>, +} + +/// A transparent HTTP/1.1 JSON-RPC proxy in front of the node. +struct RpcProxy { + port: u16, + state: Arc, + /// The accept loop, `None` while the proxy refuses connections. + task: Mutex>>, +} + +impl RpcProxy { + async fn start(upstream: u16) -> Result { + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await?; + let port = listener.local_addr()?.port(); + let state = Arc::new(ProxyState { + upstream, + submitted: Mutex::new(Vec::new()), + warmup_answered: Mutex::new(Vec::new()), + trap: Mutex::new(None), + open: watch::Sender::new(true), + connections: Mutex::new(Vec::new()), + }); + let proxy = Self { + port, + state, + task: Mutex::new(None), + }; + proxy.serve(listener); + Ok(proxy) + } + + fn serve(&self, listener: tokio::net::TcpListener) { + let shared = self.state.clone(); + *self.task.lock().unwrap() = Some(tokio::spawn(async move { + while let Ok((downstream, _)) = listener.accept().await { + let state = shared.clone(); + let connection = tokio::spawn(async move { + // A failed connection is what the server sees as a + // transport failure; nothing else to report. + let _ = relay(state, downstream).await; + }); + shared.connections.lock().unwrap().push(connection); + } + })); + } + + /// Close the listener and every relayed connection, as a dead node's + /// port is closed, and prove that a new connection is refused. + async fn refuse(&self) -> Result<()> { + let task = self.task.lock().unwrap().take(); + if let Some(task) = task { + task.abort(); + let _ = task.await; + } + let connections = std::mem::take(&mut *self.state.connections.lock().unwrap()); + for connection in connections { + connection.abort(); + let _ = connection.await; + } + let refused = TcpStream::connect(("127.0.0.1", self.port)).await; + ensure!( + refused + .as_ref() + .is_err_and(|error| error.kind() == std::io::ErrorKind::ConnectionRefused), + "the proxy port still accepts connections: {refused:?}" + ); + Ok(()) + } + + /// Accept connections on the same port again. + async fn reopen(&self) -> Result<()> { + let listener = tokio::net::TcpListener::bind(("127.0.0.1", self.port)).await?; + self.serve(listener); + Ok(()) + } + + fn stop(&self) { + if let Some(task) = self.task.lock().unwrap().take() { + task.abort(); + } + for connection in self.state.connections.lock().unwrap().drain(..) { + connection.abort(); + } + } + + fn arm(&self, hold: Hold) -> oneshot::Receiver { + let (sender, receiver) = oneshot::channel(); + *self.state.trap.lock().unwrap() = Some((hold, sender)); + receiver + } + + fn submitted(&self) -> Vec { + self.state.submitted.lock().unwrap().clone() + } + + fn warmup_answered(&self) -> Vec { + self.state.warmup_answered.lock().unwrap().clone() + } +} + +/// Relays one client connection, one request and reply at a time, as +/// HTTP/1.1 without pipelining does. +async fn relay(state: Arc, downstream: TcpStream) -> Result<()> { + let upstream = TcpStream::connect(("127.0.0.1", state.upstream)).await?; + let (down_read, mut down_write) = downstream.into_split(); + let (up_read, mut up_write) = upstream.into_split(); + let mut down_read = BufReader::new(down_read); + let mut up_read = BufReader::new(up_read); + let mut open = state.open.subscribe(); + loop { + // A node connection that closes while idle (qbitd's idle timeout, + // or a kill) closes the server's connection too, as a direct + // connection would, so the server never reuses a dead one. + let next = tokio::select! { + next = read_http(&mut down_read) => next?, + _ = up_read.fill_buf() => return Ok(()), + }; + let Some((request, body)) = next else { + return Ok(()); + }; + let submitted = submitted_block(&body); + let trap = submitted.as_ref().and_then(|block| { + state.submitted.lock().unwrap().push(block.clone()); + let trap = state.trap.lock().unwrap().take(); + trap.map(|(hold, sender)| (block, hold, sender)) + }); + let (before, after) = match trap { + Some((block, Hold::Request, sender)) => (Some((block.clone(), sender)), None), + Some((block, Hold::Reply, sender)) => (None, Some((block.clone(), sender))), + // Never armed: a reservation is held in the database. + Some((_, Hold::Reservation | Hold::Warmup, _)) | None => (None, None), + }; + if let Some((block, sender)) = before { + hold_here(&state, block, sender).await; + } + open.wait_for(|open| *open).await?; + up_write.write_all(&request).await?; + let Some((reply, reply_body)) = read_http(&mut up_read).await? else { + return Ok(()); + }; + if let Some(block) = &submitted { + let answer: Value = serde_json::from_slice(&reply_body).unwrap_or_default(); + if answer["error"]["code"] == RPC_IN_WARMUP { + state.warmup_answered.lock().unwrap().push(block.clone()); + } + } + if let Some((block, sender)) = after { + hold_here(&state, block, sender).await; + } + open.wait_for(|open| *open).await?; + down_write.write_all(&reply).await?; + } +} + +async fn hold_here(state: &ProxyState, block: String, sender: oneshot::Sender) { + state.open.send_replace(false); + let (release, released) = oneshot::channel(); + if sender.send(Hit { block, release }).is_ok() { + let _ = released.await; + } + state.open.send_replace(true); +} + +/// The block hash of a `submitblock` request body, if it is one. +fn submitted_block(body: &[u8]) -> Option { + let request: Value = serde_json::from_slice(body).ok()?; + if request["method"] != "submitblock" { + return None; + } + let block = hex::decode(request["params"][0].as_str()?).ok()?; + Some(hash_display(&double_sha256(block.get(..80)?))) +} + +/// One HTTP/1.1 message with a `Content-Length` body: the raw bytes and the +/// body. `None` at a clean end of stream. qbitd's JSON-RPC server and the +/// servers' client both frame every message by `Content-Length`; anything +/// else fails the connection loudly rather than being misread. +async fn read_http(reader: &mut BufReader) -> Result, Vec)>> +where + R: tokio::io::AsyncRead + Unpin, +{ + let mut raw = Vec::new(); + let mut length = None; + loop { + let start = raw.len(); + if reader.read_until(b'\n', &mut raw).await? == 0 { + ensure!(raw.is_empty(), "connection closed inside an HTTP header"); + return Ok(None); + } + let line = std::str::from_utf8(&raw[start..])?.trim_end(); + if line.is_empty() { + break; + } + if let Some((name, value)) = line.split_once(':') { + if name.eq_ignore_ascii_case("content-length") { + length = Some(value.trim().parse::()?); + } + } + } + let mut body = vec![0; length.context("HTTP message without Content-Length")?]; + reader.read_exact(&mut body).await?; + raw.extend_from_slice(&body); + Ok(Some((raw, body))) +} diff --git a/crates/qbit-prism-server/tests/support/live_pg_async_live_miners.rs b/crates/qbit-prism-server/tests/support/live_pg_async_live_miners.rs new file mode 100644 index 000000000..603b1da37 --- /dev/null +++ b/crates/qbit-prism-server/tests/support/live_pg_async_live_miners.rs @@ -0,0 +1,1476 @@ +//! #474 slice B: an asynchronous PostgreSQL promotion while live miners, +//! their Stratum sessions and both frontends' caches survive. +//! +//! The pair, relays and procedure are the parent module's (#521 scenario +//! 3): a D3 primary with one dedicated asynchronous standby, both native +//! frontends on the stable writer endpoint, a real qbitd. Instead of CPU +//! miners stopped at the fence, four scripted Stratum miners +//! ([`ShareClient`]), each with its own payout address, submit shares on +//! their own sessions from before the replication cut, through the +//! unreplicated interval and across the fence, and keep those sessions +//! through the promotion. Every submission is recorded by identity with +//! the answer the miner received: acknowledged, rejected, or unknown (no +//! answer, a lost connection, or `ledger-outcome-unknown`). +//! +//! Submissions are in flight at the fence by construction: a deferred +//! trigger on the header map holds every share's COMMIT, already sent, on +//! an advisory lock the test takes just before it fences the writers. The +//! held COMMITs complete on the old primary after the fence, for clients +//! that are gone; nothing else is admitted there. +//! +//! D3 is asynchronous: acknowledged shares in the replication gap are lost +//! by design, and this case asserts no zero-loss property. It asserts that +//! every loss is accounted for exactly: each acknowledged share is on the +//! old primary; each acknowledged share the promoted ledger lacks was +//! committed inside the gap, the suffix of the commit order the standby +//! never received, and is named; no definite rejection was +//! committed anywhere; the old writer committed nothing after the fence +//! but the COMMITs it had already received. +//! +//! After the endpoint moves, the same sessions submit shares on their +//! current jobs and the work each miner kept from before the cut and from +//! inside the gap +//! (including a session opened inside the gap, whose job was issued there): +//! kept work is either resumed under its original authority or refused +//! truthfully as stale. A surviving share replayed, while its parent is still +//! the tip, is a duplicate. The block the gap session solved commits to the +//! lost window, so it must be refused as stale or land paying only rows the +//! promoted ledger holds (#619; since #645 it is refused before its offer). +//! A real block on work whose window the promoted +//! ledger holds then lands, and its audit, verified against the node's +//! coinbase, pays exactly a window read independently from the promoted +//! ledger, which holds sequence numbers lost shares had: no share the +//! promotion lost and no history a frontend cached before it. Then the lost +//! acknowledged shares are replayed. Each answer must match the promoted +//! ledger: accepted means credited exactly once, and a truthful stale +//! refusal means not credited. +//! Once the final block has moved the tip, every surviving session receives +//! work for it and a share on that work is credited exactly once. +//! #466's replaced-history regression +//! (`ledger/window/snapshot_delta/tests/physical_failover.rs:: +//! physical_async_failover_replaced_history_matches_full_read`) covers the +//! delta read itself and is not repeated here. +use super::*; +use crate::share_client::{ + reason_id, start_share_only_servers, Answer, Proof, ShareClient, Submitted, +}; +use std::collections::{BTreeMap, BTreeSet}; + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn async_promotion_with_live_miners_accounts_for_every_share_and_lands_a_block() -> Result<()> +{ + let Some(bin) = gate::pg_bin_dir(gate::site!())? else { + return Ok(()); + }; + let _drill = DRILLS.lock().await; + drop(SERIAL.lock().await); + let mut pair = Pair::start(bin.into()).await?; + let Some(mut fixture) = + Fixture::open_on_database(false, false, Some(&pair.url(pair.writer.port))).await? + else { + return Ok(()); + }; + let mut timeline = Timeline { + started: Instant::now(), + events: Vec::new(), + }; + let result = promotion(&mut fixture, &mut pair, &mut timeline).await; + if result.is_err() { + eprintln!("{}", fixture.diagnostics()); + eprintln!("{}", pair.diagnostics()); + eprintln!("promotion timeline: {}", timeline.render()); + pair.writer.route_to(pair.writable_port()); + } + let cleanup = fixture.cleanup().await; + drop(pair); + result.and(cleanup) +} + +/// A prepared record's window (first and last share sequence number and +/// share count) and how many accepted shares the ledger holds in that range +/// under the landing's predicate (accepted and issued by the window anchor). +type PreparedWindow = (Option, Option, Option, i64); + +/// A candidate's window (first and last share sequence number, share count, +/// anchor), the accepted rows in its range, and those the landing reads. +type CandidateWindow = (Option, Option, Option, Option, i64, i64); + +/// The scripted miners submitting through the promotion. +const MINERS: usize = 4; + +/// Where the procedure was when a submission was sent or answered. +#[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord)] +enum Phase { + BeforeCut, + Gap, + Fenced, +} + +/// What the miner learned about one submission. +#[derive(Clone, Debug, PartialEq, Eq)] +enum Outcome { + Acknowledged, + Rejected(String), + Unknown(String), +} + +impl Outcome { + fn of(answer: &Answer) -> Self { + match answer { + Answer::Accepted => Self::Acknowledged, + Answer::Rejected(response) => match reason_id(response) { + Some("ledger-outcome-unknown") => Self::Unknown("ledger-outcome-unknown".into()), + Some(reason) => Self::Rejected(reason.into()), + None => Self::Rejected(response["error"].to_string()), + }, + other => Self::Unknown(other.to_string()), + } + } +} + +/// One submission by identity, the phase it was sent and answered in, and +/// what the miner learned. +#[derive(Clone, Debug)] +struct Record { + miner: usize, + sent: Phase, + answered: Phase, + params: Value, + share_id: String, + outcome: Outcome, +} + +impl Record { + /// The submission again, for a replay on the same session. + fn replay(&self) -> (Value, Submitted) { + ( + self.params.clone(), + Submitted { + hash: self.share_id.rsplit(':').next().unwrap_or_default().into(), + share_id: self.share_id.clone(), + answer: Answer::TimedOut, + }, + ) + } +} + +/// Work a miner solved and kept without submitting it, and the phase it was +/// kept in. +struct Kept { + miner: usize, + kept: Phase, + params: Value, + submitted: Submitted, +} + +#[derive(Default)] +struct Book { + records: Vec, + kept: Vec, +} + +/// One miner's loop: keep one share of work from before the cut and one +/// from inside the gap, and submit shares until one it sent after the fence +/// has been answered. Returns the session, which must still be connected. +async fn mine( + miner: usize, + mut client: ShareClient, + phase: watch::Receiver, + book: Arc>, +) -> Result { + let mut kept = BTreeSet::new(); + loop { + let sent = *phase.borrow(); + if matches!(sent, Phase::BeforeCut | Phase::Gap) && kept.insert(sent) { + let (params, submitted) = client.solve(Proof::Share).await?; + book.lock().unwrap().kept.push(Kept { + miner, + kept: sent, + params, + submitted, + }); + } + let (params, submitted) = client.solve(Proof::Share).await?; + let submitted = client.submit_solved(params.clone(), submitted).await; + let answered = *phase.borrow(); + let lost = matches!(submitted.answer, Answer::Lost(_)); + book.lock().unwrap().records.push(Record { + miner, + sent, + answered, + params, + share_id: submitted.share_id, + outcome: Outcome::of(&submitted.answer), + }); + ensure!( + !lost, + "miner {miner}'s session did not survive: {}", + submitted.answer + ); + if sent == Phase::Fenced { + return Ok(client); + } + // Paced so the gap holds tens of shares, not thousands. + tokio::time::sleep(Duration::from_millis(50)).await; + } +} + +/// The miners that have kept work from `phase`. +fn kept_by(book: &Mutex, phase: Phase) -> BTreeSet { + book.lock() + .unwrap() + .kept + .iter() + .filter(|work| work.kept == phase) + .map(|work| work.miner) + .collect() +} + +/// Positive acknowledgements recorded so far that were sent in `phase` and +/// are not in `except`. +fn acknowledged(book: &Mutex, phase: Phase, except: &BTreeSet) -> usize { + book.lock() + .unwrap() + .records + .iter() + .filter(|record| { + record.sent == phase + && record.outcome == Outcome::Acknowledged + && !except.contains(&record.share_id) + }) + .count() +} + +/// Client backends on `pool`'s cluster other than the observer's own. +async fn writer_backends(pool: &PgPool) -> Result { + Ok(sqlx::query_scalar( + "SELECT count(*) FROM pg_stat_activity WHERE backend_type='client backend' AND application_name<>'b474-observer'", + ) + .fetch_one(pool) + .await?) +} + +/// A deferred trigger on the header map that holds every share's COMMIT, +/// once the client has sent it, while the test holds an advisory lock on +/// the old primary. Open, it takes the lock shared and holds nothing. +struct CommitHold { + key: String, + holder: Option<(sqlx::pool::PoolConnection, i32)>, +} + +impl CommitHold { + async fn install(primary: &PgPool, schema: &str) -> Result { + let key = format!("{schema}.b474_commit_hold"); + sqlx::raw_sql(&format!( + "CREATE FUNCTION {schema}.b474_commit_hold() RETURNS trigger LANGUAGE plpgsql AS $$ BEGIN PERFORM pg_advisory_xact_lock_shared(hashtext('{key}')::bigint); RETURN NULL; END $$; CREATE CONSTRAINT TRIGGER b474_commit_hold AFTER INSERT ON {schema}.qbit_prism_share_hashes DEFERRABLE INITIALLY DEFERRED FOR EACH ROW EXECUTE FUNCTION {schema}.b474_commit_hold();" + )) + .execute(primary) + .await?; + Ok(Self { key, holder: None }) + } + + async fn close(&mut self, primary: &PgPool) -> Result<()> { + let mut connection = primary.acquire().await?; + sqlx::query("SELECT pg_advisory_lock(hashtext($1)::bigint)") + .bind(&self.key) + .execute(&mut *connection) + .await?; + let pid = sqlx::query_scalar("SELECT pg_backend_pid()") + .fetch_one(&mut *connection) + .await?; + self.holder = Some((connection, pid)); + Ok(()) + } + + /// COMMITs waiting on the closed hold. + async fn held(&self, primary: &PgPool) -> Result { + let (_, pid) = self.holder.as_ref().context("the hold is open")?; + Ok(sqlx::query_scalar( + "SELECT count(*) FROM pg_stat_activity WHERE $1=ANY(pg_blocking_pids(pid))", + ) + .bind(pid) + .fetch_one(primary) + .await?) + } + + async fn open(&mut self) -> Result<()> { + let (mut connection, _) = self.holder.take().context("the hold is open")?; + sqlx::query("SELECT pg_advisory_unlock(hashtext($1)::bigint)") + .bind(&self.key) + .execute(&mut *connection) + .await?; + Ok(()) + } + + /// Remove the trigger, which replicated before the cut, from `pool`. + async fn remove(pool: &PgPool, schema: &str) -> Result<()> { + sqlx::raw_sql(&format!( + "DROP TRIGGER b474_commit_hold ON {schema}.qbit_prism_share_hashes; DROP FUNCTION {schema}.b474_commit_hold();" + )) + .execute(pool) + .await?; + Ok(()) + } +} + +/// Every share in the ledger of `schema` with its sequence number. +async fn sequences(pool: &PgPool, schema: &str) -> Result> { + Ok(sqlx::query_as::<_, (String, i64)>(&format!( + "SELECT share_id,share_seq FROM {schema}.qbit_share_ledger" + )) + .fetch_all(pool) + .await? + .into_iter() + .collect()) +} + +/// A fresh payout address from the fixture's wallet. +async fn address(f: &Fixture) -> Result { + Ok(f.rpc("getnewaddress", json!(["", "p2mr"])) + .await? + .as_str() + .context("address missing")? + .into()) +} + +/// Verify `block`'s audit against the node's coinbase and return the shares +/// it pays, with their sequence numbers. +async fn paid_shares(f: &Fixture, block: &str) -> Result> { + let body: Value = f + .client + .get(format!( + "http://127.0.0.1:{}/audit/blocks/{block}/bundle", + f.api[1] + )) + .send() + .await? + .error_for_status()? + .json() + .await?; + let bundle: AuditBundle = serde_json::from_value(body["audit_bundle"].clone())?; + let verbose = f.rpc("getblock", json!([block, 2])).await?; + let coinbase = f + .rpc( + "getrawtransaction", + json!([verbose["tx"][0]["txid"], false, block]), + ) + .await?; + let key = ManifestSigningKey::from_seed_hex(&"22".repeat(32))?.public_key_hex(); + verify_audit_bundle_against_coinbase_tx_hex( + &bundle, + coinbase.as_str().context("node coinbase missing")?, + &key, + )?; + bundle + .shares + .iter() + .map(|share| Ok((share.share_id.clone(), i64::try_from(share.share_seq)?))) + .collect() +} + +/// The window a block found on the job of `share_id` (the block's own +/// credited share) pays, read from the promoted ledger independently of the +/// block's audit: every share issued and accepted by that job's issuance, +/// newest first, until their weight reaches `PRISM_WINDOW_MULTIPLIER` +/// network difficulties. +async fn expected_window(f: &Fixture, share_id: &str) -> Result> { + let (anchor, network): (i64, String) = sqlx::query_as( + "SELECT floor(extract(epoch FROM job_issued_at)*1000)::bigint,network_difficulty::text FROM qbit_share_ledger WHERE accepted AND share_id=$1", + ) + .bind(share_id) + .fetch_one(&f.pool) + .await + .with_context(|| format!("the block's share {share_id} is not credited"))?; + let eligible: Vec<(String, i64, String)> = sqlx::query_as( + "SELECT share_id,share_seq,share_difficulty::text FROM qbit_share_ledger WHERE accepted AND floor(extract(epoch FROM job_issued_at)*1000)<=$1 AND floor(extract(epoch FROM accepted_at)*1000)<=$1 ORDER BY share_seq DESC", + ) + .bind(anchor) + .fetch_all(&f.pool) + .await?; + let weight = |text: &str| -> Result { + let (whole, fraction) = text.split_once('.').unwrap_or((text, "")); + ensure!( + fraction.bytes().all(|digit| digit == b'0'), + "fractional scaled difficulty {text}" + ); + Ok(whole.parse()?) + }; + let requested = weight(&network)? + .checked_mul(qbit_prism::PRISM_WINDOW_MULTIPLIER) + .context("window weight overflow")?; + let mut total = 0u128; + let mut window = BTreeSet::new(); + for (share, seq, difficulty) in eligible { + if total >= requested { + break; + } + total += weight(&difficulty)?; + window.insert((share, seq)); + } + Ok(window) +} + +/// The prepared window behind issued job `job_id` on the promoted ledger, +/// with how many accepted shares that ledger holds in its range under the +/// landing's predicate; `None` when the job or its prepared record is absent. +async fn job_window(f: &Fixture, job_id: &str) -> Result> { + Ok(sqlx::query_as( + "SELECT p.window_first_share_seq,p.window_last_share_seq,p.window_share_count,(SELECT count(*) FROM qbit_share_ledger l WHERE l.accepted AND l.share_seq BETWEEN p.window_first_share_seq AND p.window_last_share_seq AND l.accepted_at<=to_timestamp(p.window_anchor_ms::double precision/1000) AND l.job_issued_at<=to_timestamp(p.window_anchor_ms::double precision/1000)) FROM qbit_prism_jobs j JOIN qbit_prism_jobs p ON p.job_id=j.payload->>'prepared_key' WHERE j.job_id=$1", + ) + .bind(job_id) + .fetch_optional(&f.pool) + .await?) +} + +/// The last share of the prepared window behind issued job `job_id`, read in +/// `schema` through `pool`: `None` when the job's issued row or its prepared +/// record is absent there, `Some(None)` for a window without shares. +async fn prepared_last_share( + pool: &PgPool, + schema: &str, + job_id: &str, +) -> Result>> { + Ok(sqlx::query_scalar::<_, Option>(&format!( + "SELECT p.window_last_share_seq FROM {schema}.qbit_prism_jobs j JOIN {schema}.qbit_prism_jobs p ON p.job_id=j.payload->>'prepared_key' WHERE j.job_id=$1" + )) + .bind(job_id) + .fetch_optional(pool) + .await?) +} + +/// The payout revision of the work behind issued job `job_id`, and the +/// ledger's own, read on the promoted primary; `None` when the job or its +/// prepared record is absent there. +async fn work_revision(f: &Fixture, job_id: &str) -> Result> { + Ok(sqlx::query_as( + "SELECT p.payout_revision,c.payout_revision FROM qbit_prism_jobs j JOIN qbit_prism_jobs p ON p.job_id=j.payload->>'prepared_key' CROSS JOIN qbit_prism_cluster c WHERE j.job_id=$1 AND c.singleton", + ) + .bind(job_id) + .fetch_optional(&f.pool) + .await?) +} + +/// That `share`, refused on the new tip's job `job`, left no row in the +/// promoted ledger. +async fn ensure_no_ledger_row( + f: &Fixture, + miner: usize, + job: &str, + share: &Submitted, +) -> Result<()> { + let rows: i64 = sqlx::query_scalar("SELECT count(*) FROM qbit_share_ledger WHERE share_id=$1") + .bind(&share.share_id) + .fetch_one(&f.pool) + .await?; + ensure!( + rows == 0, + "miner {miner}'s share {} on job {job} was answered {} but has {rows} ledger row(s)", + share.share_id, + share.answer + ); + Ok(()) +} + +/// Why miner `miner`'s surviving session has no current work on `tip`: +/// none at all, or only the jobs it passed over. +fn passed_over( + miner: usize, + tip: &str, + superseded: &[(usize, String, i64, i64)], + unrecorded: &[(usize, String)], + retired: &[(usize, String)], +) -> String { + let stale: Vec<(&str, i64, i64)> = superseded + .iter() + .filter(|entry| entry.0 == miner) + .map(|(_, job, work, ledger)| (job.as_str(), *work, *ledger)) + .collect(); + let jobs = |entries: &[(usize, String)]| -> Vec { + entries + .iter() + .filter(|entry| entry.0 == miner) + .map(|(_, job)| job.clone()) + .collect() + }; + let (missing, pruned) = (jobs(unrecorded), jobs(retired)); + if stale.is_empty() && missing.is_empty() && pruned.is_empty() { + format!("miner {miner}'s surviving session got no work on the new tip {tip}") + } else { + format!("miner {miner}'s surviving session got work on the new tip {tip} only at a superseded payout revision, without its issued row or prepared record, or retired past retention, before the phase's deadline: it passed over (job, work revision, ledger revision) {stale:?}, jobs without a record {missing:?} and jobs answered unknown-job {pruned:?}") + } +} + +/// The one deadline of the drill's new-tip phase. Every step in the phase +/// is bounded by what is left of it, share submissions included (Codex on +/// #753), and an expiry names the phase, what was waiting and how long the +/// phase had run. +struct PhaseDeadline { + started: Instant, + deadline: Instant, +} + +impl PhaseDeadline { + fn start(limit: Duration) -> Self { + let started = Instant::now(); + Self { + started, + deadline: started + limit, + } + } + + fn left(&self) -> Duration { + self.deadline.saturating_duration_since(Instant::now()) + } + + /// `step` within what is left of the phase; `waiting` says what was + /// waiting if the deadline passes first. + async fn bound( + &self, + waiting: impl FnOnce() -> String, + step: impl std::future::Future>, + ) -> Result { + match tokio::time::timeout(self.left(), step).await { + Ok(result) => result, + Err(_) => bail!( + "the new-tip phase's deadline passed {:.1?} into the phase while {}", + self.started.elapsed(), + waiting() + ), + } + } +} + +/// Until `block` is confirmed on the promoted primary; a timeout names the +/// candidate's state and last error. +async fn landed(f: &Fixture, block: &str) -> Result<()> { + let result = until( + &format!("block {block} landed and confirmed on the promoted primary"), + 90, + || async { + Ok(sqlx::query_scalar::<_, String>( + "SELECT chain_state FROM qbit_pool_blocks WHERE block_hash=$1", + ) + .bind(block) + .fetch_optional(&f.pool) + .await? + .as_deref() + == Some("confirmed")) + }, + ) + .await; + if result.is_err() { + let row: Option<(String, Option, Option)> = sqlx::query_as( + "SELECT state,offer_outcome,last_error FROM qbit_block_candidate_outbox WHERE block_hash=$1", + ) + .bind(block) + .fetch_optional(&f.pool) + .await?; + let header = f.rpc("getblockheader", json!([block])).await; + // The window the candidate carries, and what the ledger holds of it. + let window: Option = sqlx::query_as( + "SELECT o.window_first_share_seq,o.window_last_share_seq,o.window_share_count,o.window_anchor_ms,(SELECT count(*) FROM qbit_share_ledger l WHERE l.accepted AND l.share_seq BETWEEN o.window_first_share_seq AND o.window_last_share_seq),(SELECT count(*) FROM qbit_share_ledger l WHERE l.accepted AND l.share_seq BETWEEN o.window_first_share_seq AND o.window_last_share_seq AND l.accepted_at<=to_timestamp(o.window_anchor_ms::double precision/1000) AND l.job_issued_at<=to_timestamp(o.window_anchor_ms::double precision/1000)) FROM qbit_block_candidate_outbox o WHERE o.block_hash=$1", + ) + .bind(block) + .fetch_optional(&f.pool) + .await?; + return result.with_context(|| { + format!( + "candidate (state, offer outcome, last error): {row:?}; window (first, last, count, anchor ms, rows in range, rows the landing predicate reads): {window:?}; node confirmations: {:?}", + header.map(|header| header["confirmations"].clone()) + ) + }); + } + Ok(()) +} + +async fn promotion(f: &mut Fixture, pair: &mut Pair, timeline: &mut Timeline) -> Result<()> { + let observer = |port: u16| format!("{}?application_name=b474-observer", pair.url(port)); + let primary = PgPool::connect(&observer(pair.primary.port)).await?; + let standby = PgPool::connect(&observer(pair.standby.port)).await?; + let schema = f.schema.clone(); + let names: String = sqlx::query_scalar("SHOW synchronous_standby_names") + .fetch_one(&primary) + .await?; + ensure!( + names.is_empty(), + "D3 requires an asynchronous standby, found {names:?}" + ); + + start_share_only_servers(f, &[(0, Vec::new()), (1, Vec::new())]).await?; + let mut hold = CommitHold::install(&primary, &schema).await?; + let (phase, watcher) = watch::channel(Phase::BeforeCut); + let book = Arc::new(Mutex::new(Book::default())); + let mut miners = Vec::new(); + for miner in 0..MINERS { + let username = format!("{}.b474-{miner}", address(f).await?); + let client = ShareClient::connect(f.stratum[miner % 2], &username).await?; + miners.push(tokio::spawn(mine( + miner, + client, + watcher.clone(), + book.clone(), + ))); + } + + // Before the cut: shares the standby replays. + // Every miner keeps work from each phase before the procedure leaves it. + let every_miner: BTreeSet = (0..MINERS).collect(); + until( + "shares on the primary, and work kept by every miner, before the cut", + 30, + || async { + Ok(shares(&primary, &schema).await?.len() >= 12 + && kept_by(&book, Phase::BeforeCut) == every_miner) + }, + ) + .await?; + let pre_cut = shares(&primary, &schema).await?; + until("standby replay of the pre-cut shares", 15, || async { + Ok(shares(&standby, &schema).await?.is_superset(&pre_cut)) + }) + .await?; + + // The unreplicated interval. + phase.send_replace(Phase::Gap); + pair.replication.fence(); + timeline.mark("replication cut"); + until("standby replay settled after the cut", 15, || async { + Ok(sqlx::query_scalar::<_, bool>( + "SELECT pg_last_wal_replay_lsn()=pg_last_wal_receive_lsn()", + ) + .fetch_one(&standby) + .await?) + }) + .await?; + let frozen = shares(&standby, &schema).await?; + ensure!( + frozen.is_superset(&pre_cut), + "the standby lost pre-cut shares" + ); + // The promoted primary resumes the share sequence after the value the + // standby last received. Grow the gap well past it, so the first shares + // credited after the promotion reuse sequence numbers lost shares had. + let resumes_after: i64 = sqlx::query_scalar(&format!( + "SELECT last_value FROM {schema}.qbit_share_ledger_share_seq_seq" + )) + .fetch_one(&standby) + .await?; + until( + "acknowledged shares inside the replication gap, past the standby's sequence", + 30, + || async { + let committed: Option = sqlx::query_scalar(&format!( + "SELECT max(share_seq) FROM {schema}.qbit_share_ledger" + )) + .fetch_one(&primary) + .await?; + Ok(acknowledged(&book, Phase::Gap, &frozen) >= 8 + && committed.is_some_and(|seq| seq >= resumes_after + 20) + && kept_by(&book, Phase::Gap) == every_miner) + }, + ) + .await?; + // A session opened inside the gap, on work server 0 has published over + // it (it re-anchors every second): its job, and the payout its coinbase + // commits to, come from a window the promotion will lose. + let replicated: i64 = sqlx::query_scalar(&format!( + "SELECT max(share_seq) FROM {schema}.qbit_share_ledger" + )) + .fetch_one(&standby) + .await?; + until("server 0's work prepared over the gap", 15, || async { + Ok(sqlx::query_scalar::<_, bool>(&format!( + "SELECT EXISTS(SELECT 1 FROM {schema}.qbit_prism_jobs WHERE job_id LIKE 'prepared:live-0:%' AND window_last_share_seq>$1)" + )) + .bind(replicated) + .fetch_one(&primary) + .await?) + }) + .await?; + // The saved row is not yet the work a new session is issued: server 0 + // commits it, checks the node and the ledger again, and only then + // publishes the work (#724). A session opened in between is issued the + // work published before, whose window can end before the cut. So the + // gap session is opened again, for up to a few reanchor intervals while + // the miners keep submitting in the gap, until its first job's window + // reaches past the replicated share. A replaced session solves and + // submits nothing, so the book, which holds the four miners' + // submissions and kept work, never sees it; the replaced sessions' + // windows are reported with the result. Issuance saves a job and its + // prepared record before it announces the job, so a first job without + // them is a fault, not older work. + let gap_username = format!("{}.b474-gap", address(f).await?); + let bound = Duration::from_secs(5); + let opening = Instant::now(); + let mut older = Vec::new(); + let (mut gap_miner, gap_job, gap_window) = loop { + let left = bound.saturating_sub(opening.elapsed()); + ensure!( + !left.is_zero(), + "the gap-issued job's window does not reach past the replicated share {replicated}: in the {:.1?} after server 0 saved work over the gap, it issued {} new session(s) only older work (their windows' last shares: {older:?})", + opening.elapsed(), + older.len() + ); + let pass = tokio::time::timeout(left, async { + let session = ShareClient::connect(f.stratum[0], &gap_username).await?; + let job = session + .job_id() + .context("the gap session's first job has no id")? + .to_owned(); + let window = prepared_last_share(&primary, &schema, &job).await?; + Ok::<_, anyhow::Error>((session, job, window)) + }) + .await; + let Ok(pass) = pass else { + bail!( + "no gap session had a first job and its window within {bound:?} after server 0 saved work over the gap ({:.1?} elapsed); {} earlier session(s) were issued only older work (their windows' last shares: {older:?})", + opening.elapsed(), + older.len() + ); + }; + let (session, job, window) = pass?; + match window { + Some(Some(last)) if last > replicated => break (session, job, last), + Some(last) => older.push(last), + None => bail!( + "server 0 issued gap session job {job}, but the old primary holds no issued row or prepared record for it, which issuance saves before it announces a job" + ), + } + tokio::time::sleep(Duration::from_millis(200)).await; + }; + timeline.mark("gap session opened on published work"); + let gap_share = gap_miner.solve(Proof::Share).await?; + let gap_block = gap_miner.solve(Proof::Block).await?; + ensure!( + gap_share.0[1].as_str() == Some(gap_job.as_str()) + && gap_block.0[1].as_str() == Some(gap_job.as_str()), + "the gap share and block were solved on jobs {} and {}, not on the gap session's first job {gap_job}, whose window reaches past the replicated share {replicated}", + gap_share.0[1], + gap_block.0[1] + ); + + // Fence the old writer with COMMITs in flight. + hold.close(&primary).await?; + // Appends serialize on the ledger's own lock, so one COMMIT is held and + // the other miners' appends queue behind it, before their COMMITs. + until("a share COMMIT held in flight", 15, || async { + Ok(hold.held(&primary).await? >= 1) + }) + .await?; + // Closing the hold waited for every COMMIT already past it, and every + // later one waits on it: this is the old ledger at the fence, less the + // held COMMITs. + let held = hold.held(&primary).await?; + let before_fence = shares(&primary, &schema).await?; + phase.send_replace(Phase::Fenced); + pair.writer.fence(); + timeline.mark("writers fenced with COMMITs in flight"); + // The held COMMITs were sent before the fence; they complete on the old + // primary for clients that are gone. + hold.open().await?; + until( + "no writer connection left on the old primary", + 15, + || async { Ok(writer_backends(&primary).await? == 0) }, + ) + .await?; + let refused = tokio::time::timeout( + Duration::from_secs(10), + PgPool::connect(&pair.url(pair.writer.port)), + ) + .await; + ensure!( + !matches!(refused, Ok(Ok(_))), + "the fenced writer endpoint still admits a connection" + ); + let at_fence = shares(&primary, &schema).await?; + let completed: BTreeSet = at_fence.difference(&before_fence).cloned().collect(); + ensure!( + before_fence.is_subset(&at_fence) && (1..=held).contains(&i64::try_from(completed.len())?), + "after the fence the old primary committed {} share(s), but only {held} COMMIT(s) were held", + completed.len() + ); + // Every miner sends one more share after the fence and is answered. + let mut sessions = Vec::new(); + for miner in miners { + sessions.push(miner.await??); + } + timeline.mark("post-fence submissions answered"); + let old_sequences = sequences(&primary, &schema).await?; + let old_shares: BTreeSet = old_sequences.keys().cloned().collect(); + ensure!( + old_shares == at_fence, + "the fenced old primary committed {} share(s) after its writers were gone", + old_shares.difference(&at_fence).count() + ); + let flush: String = sqlx::query_scalar("SELECT pg_current_wal_flush_lsn()::text") + .fetch_one(&primary) + .await?; + let gap_bytes: i64 = + sqlx::query_scalar("SELECT pg_wal_lsn_diff($1::pg_lsn,pg_last_wal_receive_lsn())::bigint") + .bind(&flush) + .fetch_one(&standby) + .await?; + primary.close().await; + pair.primary.stop()?; + timeline.mark("old primary stopped"); + let promoted: bool = sqlx::query_scalar("SELECT pg_promote(true,60)") + .fetch_one(&standby) + .await?; + ensure!(promoted, "pg_promote did not complete within 60 seconds"); + let recovering: bool = sqlx::query_scalar("SELECT pg_is_in_recovery()") + .fetch_one(&standby) + .await?; + ensure!(!recovering, "the promoted standby is still in recovery"); + pair.promoted = true; + timeline.mark("promoted"); + CommitHold::remove(&standby, &schema).await?; + + // Reconcile every recorded identity with both ledgers. + let new_shares = shares(&standby, &schema).await?; + ensure!( + new_shares == frozen, + "promotion changed what the standby had received" + ); + ensure!( + new_shares.is_subset(&old_shares), + "the promoted ledger holds shares the old primary never committed" + ); + let lost: BTreeSet = old_shares.difference(&new_shares).cloned().collect(); + ensure!( + !lost.is_empty() && gap_bytes > 0, + "the drill did not produce an unreplicated interval" + ); + ensure!(lost.is_disjoint(&pre_cut), "a lost share predates the cut"); + let records = book.lock().unwrap().records.clone(); + let offered: BTreeSet = records + .iter() + .map(|record| record.share_id.clone()) + .collect(); + ensure!( + offered.len() == records.len(), + "a share identity was offered twice" + ); + ensure!( + old_shares.is_subset(&offered), + "the ledger holds a share no miner offered" + ); + // Appends serialize on the ledger's lock, so sequence order is commit + // order and the standby received an exact prefix of it: every lost share + // comes after every replicated one. The gap is that suffix, the WAL the + // standby never received, not the moment the test flipped its phase; a + // share acknowledged just before the cut whose commit had not streamed + // yet is lost with it, as D3 allows, and is named separately below. + let replicated_through = new_shares + .iter() + .filter_map(|share| old_sequences.get(share)) + .max() + .copied() + .context("nothing was replicated")?; + let early: Vec<(&String, i64)> = lost + .iter() + .filter_map(|share| old_sequences.get(share).map(|seq| (share, *seq))) + .filter(|(_, seq)| *seq <= replicated_through) + .collect(); + ensure!( + early.is_empty(), + "lost shares precede the replicated prefix (through share {replicated_through}): {early:?}" + ); + let mut acked_missing = BTreeSet::new(); + let mut acked_before_cut = Vec::new(); + let mut unknown = BTreeMap::<&str, usize>::new(); + let mut rejected = BTreeMap::::new(); + for record in &records { + let (on_old, on_new) = ( + old_shares.contains(&record.share_id), + new_shares.contains(&record.share_id), + ); + match &record.outcome { + Outcome::Acknowledged => { + ensure!( + on_old, + "{} was acknowledged but never committed: {record:?}", + record.share_id + ); + if !on_new { + acked_missing.insert(record.share_id.clone()); + if record.answered == Phase::BeforeCut { + acked_before_cut.push(record.share_id.clone()); + } + } + } + Outcome::Rejected(reason) => { + ensure!( + !on_old, + "{} was rejected ({reason}) but committed on the old primary", + record.share_id + ); + *rejected.entry(reason.clone()).or_default() += 1; + } + Outcome::Unknown(_) => { + let class = match (on_old, on_new) { + (_, true) => "on the promoted ledger", + (true, false) => "lost in the gap", + (false, false) => "never committed", + }; + *unknown.entry(class).or_default() += 1; + } + } + } + ensure!( + !acked_missing.is_empty(), + "no acknowledged share fell in the gap" + ); + // The held COMMITs completed after the fence for clients that were gone: + // their miners were never told they were credited. + let held_records: Vec<&Record> = records + .iter() + .filter(|record| completed.contains(&record.share_id)) + .collect(); + ensure!( + held_records.len() == completed.len() + && held_records + .iter() + .all(|record| matches!(record.outcome, Outcome::Unknown(_))), + "a COMMIT held across the fence was answered as known: {held_records:?}" + ); + // Submissions sent before the fence and answered after it. + let in_flight: Vec<&Record> = records + .iter() + .filter(|record| record.sent < Phase::Fenced && record.answered == Phase::Fenced) + .collect(); + ensure!( + records + .iter() + .filter(|record| record.sent == Phase::Fenced) + .all(|record| record.outcome != Outcome::Acknowledged), + "a share sent after the fence was acknowledged" + ); + + // Move the stable endpoint to the promoted primary. + pair.writer.route_to(pair.standby.port); + timeline.mark("endpoint moved"); + for index in 0..2 { + until( + &format!("server {index} readiness after the move"), + 60, + || healthy(f, index), + ) + .await?; + } + + // What a brand-new session is issued right after the move: observed + // only. A window prepared inside the gap is #619's second mode, which + // #645 retires and its own gated tests hold. + let mut first_jobs = Vec::new(); + for index in 0..2 { + let mut probe = ShareClient::connect( + f.stratum[index], + &format!("{}.b474-probe", address(f).await?), + ) + .await?; + let (params, _) = probe.solve(Proof::Share).await?; + let window = job_window(f, params[1].as_str().context("job id missing")?).await?; + first_jobs.push(match window { + Some((Some(_), Some(_), Some(count), held)) if count == held => { + format!("server {index}: whole") + } + other => format!("server {index}: window not held {other:?}"), + }); + } + + // The same sessions: their current jobs, kept work, and replays. No new + // work is pushed to them until the tip moves; that is checked at the end. + let mut after = Vec::new(); + for (miner, session) in sessions.iter_mut().enumerate() { + let current = session.submit(Proof::Share).await?; + ensure!( + current.answer.accepted(), + "miner {miner}'s surviving session was answered {} on its current job", + current.answer + ); + after.push(("on its current job", current)); + } + let kept = std::mem::take(&mut book.lock().unwrap().kept); + let mut each: Vec<(usize, Phase)> = kept.iter().map(|work| (work.miner, work.kept)).collect(); + each.sort(); + let expected: Vec<(usize, Phase)> = (0..MINERS) + .flat_map(|miner| [(miner, Phase::BeforeCut), (miner, Phase::Gap)]) + .collect(); + ensure!( + each == expected, + "kept work is not exactly one share per miner per phase: {each:?}" + ); + for work in kept { + let label = match work.kept { + Phase::BeforeCut => "kept before the cut", + _ => "kept in the gap", + }; + let answer = sessions[work.miner] + .submit_solved(work.params, work.submitted) + .await; + after.push((label, answer)); + } + let answer = gap_miner.submit_solved(gap_share.0, gap_share.1).await; + after.push(("kept on a job issued in the gap", answer)); + // The newest surviving acknowledgement, on the freshest job, replayed + // while its parent is still the tip: the server settles a stale parent + // before it looks for a duplicate, so once a block moved the tip and the + // stale grace ran out the same replay would read `stale-job`. + let survivor = records + .iter() + .rev() + .find(|record| { + record.outcome == Outcome::Acknowledged && new_shares.contains(&record.share_id) + }) + .context("no acknowledged share survived")?; + let (params, submitted) = survivor.replay(); + let duplicate = sessions[survivor.miner] + .submit_solved(params, submitted) + .await; + ensure!( + duplicate.answer.reason_id() == Some("duplicate-share"), + "a surviving acknowledged share replayed after the promotion was answered {}", + duplicate.answer + ); + + // The block on the gap-issued job, submitted while the tip is still the + // one its job builds on: its coinbase was built from the lost window, so + // it must be refused as stale authority, or land paying only shares the + // promoted ledger holds (#619; since #645 it is refused before its offer). + let stale_block = gap_miner.submit_solved(gap_block.0, gap_block.1).await; + let stale_paid = match &stale_block.answer { + Answer::Accepted => { + landed(f, &stale_block.hash).await.with_context(|| { + format!("the block on the gap-issued job (window through share {gap_window:?}, standby through {replicated}) was accepted") + })?; + Some(paid_shares(f, &stale_block.hash).await?) + } + answer => { + ensure!( + matches!(answer.reason_id(), Some("stale-job" | "unknown-job")), + "the block on the gap-issued job was answered {answer}" + ); + None + } + }; + // Its answer must match the promoted ledger like every other answer + // after the promotion: landed, it is credited once; refused, not at all. + after.push(("the block on the gap-issued job", stale_block.clone())); + + // A real block on work whose window the promoted ledger holds, by a new + // session. A job on a window prepared inside the gap (#619's second + // mode, retired since #645) would cover sequence numbers that now belong + // to other shares, accepted after that window's anchor, so the landing's + // window predicate would read none of them. Any such window is reported, + // and the session replaced until its job's window reads whole under that + // same predicate. + // Its window must cover the shares just credited on the promoted + // primary, which reuse sequence numbers lost shares had: server 1 + // re-anchors its prepared work every second. + let credited: i64 = sqlx::query_scalar("SELECT max(share_seq) FROM qbit_share_ledger") + .fetch_one(&f.pool) + .await?; + until( + "server 1's work prepared over the post-promotion credits", + 15, + || async { + Ok(sqlx::query_scalar::<_, bool>( + "SELECT EXISTS(SELECT 1 FROM qbit_prism_jobs WHERE job_id LIKE 'prepared:live-1:%' AND window_last_share_seq>=$1)", + ) + .bind(credited) + .fetch_one(&f.pool) + .await?) + }, + ) + .await?; + let tip = f.rpc("getbestblockhash", json!([])).await?; + let tip = tip.as_str().context("tip missing")?; + let username = format!("{}.b474-final", address(f).await?); + let started = Instant::now(); + let mut stale_windows = Vec::new(); + let (mut finisher, params, submitted) = loop { + let mut finisher = ShareClient::connect(f.stratum[1], &username).await?; + finisher.work_on(tip, Duration::from_secs(30)).await?; + let (params, submitted) = finisher.solve(Proof::Block).await?; + let window = job_window(f, params[1].as_str().context("job id missing")?).await?; + match window { + // Older work, prepared before the credits: not stale, just early. + Some((Some(_), Some(last), Some(count), held)) if count == held && last < credited => {} + Some((Some(first), Some(last), Some(count), held)) if count == held => { + eprintln!( + "final block job {} prepared window {first}..={last}, {count} shares, all held", + params[1] + ); + break (finisher, params, submitted); + } + other => stale_windows.push(format!("{other:?}")), + } + ensure!( + started.elapsed() < Duration::from_secs(30), + "#619: server 1 kept issuing jobs whose window the promoted ledger lacks (prepared first, last, count, held): {stale_windows:?}" + ); + tokio::time::sleep(Duration::from_millis(500)).await; + }; + let block = finisher.submit_solved(params, submitted).await; + ensure!( + block.answer.accepted(), + "the block after the promotion was answered {}", + block.answer + ); + landed(f, &block.hash).await?; + let paid = paid_shares(f, &block.hash).await?; + // Sequence numbers the lost shares held that the promoted ledger has + // since given to other shares: history a frontend must not reuse. + let lost_sequences: BTreeSet = lost + .iter() + .filter_map(|share| old_sequences.get(share).copied()) + .collect(); + let replaced: Vec = sqlx::query_scalar( + "SELECT share_seq FROM qbit_share_ledger WHERE share_seq=ANY($1) ORDER BY share_seq", + ) + .bind(lost_sequences.iter().copied().collect::>()) + .fetch_all(&f.pool) + .await?; + // The final block pays exactly the promoted ledger's window, read + // independently of its audit, and that window holds sequence numbers the + // lost shares had: the replaced history a frontend's cached window must + // not stand in for (#466). + let window = expected_window(f, &block.share_id).await?; + ensure!( + paid == window, + "the final block's audit is not a full read of the promoted ledger's window: {} paid, {} expected; paid only {:?}, expected only {:?}", + paid.len(), + window.len(), + paid.difference(&window).collect::>(), + window.difference(&paid).collect::>() + ); + let paid_reused: Vec = paid + .iter() + .map(|(_, seq)| *seq) + .filter(|seq| lost_sequences.contains(seq)) + .collect(); + ensure!( + !paid_reused.is_empty(), + "the final block's window holds no sequence number a lost share had: {paid:?}" + ); + + for record in records + .iter() + .filter(|record| acked_missing.contains(&record.share_id)) + { + let (params, submitted) = record.replay(); + let replay = sessions[record.miner] + .submit_solved(params, submitted) + .await; + after.push(("lost acknowledged share replayed", replay)); + } + for (label, submitted) in &after { + let truthful = match &submitted.answer { + Answer::Accepted => true, + answer => matches!(answer.reason_id(), Some("stale-job" | "unknown-job")), + }; + ensure!( + truthful, + "{label} share {} was answered {} after the promotion", + submitted.share_id, + submitted.answer + ); + } + // Every answer after the promotion matches the promoted ledger: an + // accepted submission is credited exactly once, a refused one not at all. + let ids: Vec<&str> = after + .iter() + .map(|(_, submitted)| submitted.share_id.as_str()) + .collect(); + let credited_after: BTreeMap = sqlx::query_as( + "SELECT share_id,count(*) FROM qbit_share_ledger WHERE accepted AND share_id=ANY($1) GROUP BY share_id", + ) + .bind(&ids) + .fetch_all(&f.pool) + .await? + .into_iter() + .collect(); + for (label, submitted) in &after { + let credits = credited_after + .get(&submitted.share_id) + .copied() + .unwrap_or(0); + ensure!( + credits == i64::from(submitted.answer.accepted()), + "{label} share {} was answered {} but is credited {credits} time(s) on the promoted ledger", + submitted.share_id, + submitted.answer + ); + } + + if let Some(stale_paid) = &stale_paid { + let window = expected_window(f, &stale_block.share_id).await?; + ensure!( + *stale_paid == window, + "the gap-issued block's audit is not a full read of the promoted ledger's window: {} paid, {} expected", + stale_paid.len(), + window.len() + ); + } + + // Work issued after the promotion reaches the surviving sessions: the + // final block moved the tip, each session is notified of work on it, + // and a share on that work is accepted and credited exactly once. + // The final block moves the payout revision twice on the same parent: + // when a frontend first records the new tip, and when the block's + // landing confirms it. Work published on the new tip in between carries + // the first revision. Once the confirmation commits, its frontend + // refuses a share on that work, or on anything else it still publishes, + // as stale-job until it republishes on the confirmed revision (#752). + // So a job on the new tip is not yet current work (#724). Each session + // waits past jobs on superseded work for work at the ledger's revision, + // and submits on that. A share refused because the revision moved after + // that check is waited out the same way. A job without its issued row + // or prepared record is passed over too, as the final block's loop + // passes over a window the promoted ledger lacks (#619), and so is a job + // whose share is answered unknown-job, the server having pruned it past + // its retention. The first job on superseded work the drill meets also + // gets one share, which must be refused, stale-job or, pruned, + // unknown-job, and leave no ledger row: the original symptom, kept as a + // check. One deadline bounds the whole phase: every step in it, share + // submissions and ledger reads included, so a slow answer to one miner + // cannot run the phase past it (Codex on #753). + let phase = PhaseDeadline::start(Duration::from_secs(30)); + let tip = phase + .bound( + || "the node was asked for its best block".into(), + f.rpc("getbestblockhash", json!([])), + ) + .await?; + let tip = tip.as_str().context("tip missing")?.to_owned(); + let mut on_new_tip = 0; + // Every job passed over on superseded work, or refused after a later + // revision move: (miner, job, work revision, ledger revision). + let mut superseded: Vec<(usize, String, i64, i64)> = Vec::new(); + // Every job passed over without its issued row or prepared record. + let mut unrecorded: Vec<(usize, String)> = Vec::new(); + // Every job whose share was answered unknown-job, the server having + // pruned it past its retention: (miner, job). + let mut retired: Vec<(usize, String)> = Vec::new(); + // The share kept as the original symptom's check: (miner, job, share, + // answer). + let mut refused_on_superseded = None; + for (miner, session) in sessions.iter_mut().enumerate() { + let share = loop { + session.work_on(&tip, phase.left()).await.with_context(|| { + format!( + "{}, {:.1?} into the new-tip phase", + passed_over(miner, &tip, &superseded, &unrecorded, &retired), + phase.started.elapsed() + ) + })?; + let job = session + .job_id() + .context("a surviving session's job on the new tip has no id")? + .to_owned(); + let revisions = phase + .bound( + || format!("miner {miner}'s job {job} had its payout revisions read"), + work_revision(f, &job), + ) + .await?; + match revisions { + Some((work, ledger)) if work == ledger => { + let share = phase + .bound( + || format!("miner {miner}'s surviving session waited for the answer to its share on job {job}"), + session.submit(Proof::Share), + ) + .await?; + if share.answer.accepted() { + break share; + } + // Refused. Two answers are waited out: stale-job for a + // revision that moved past this job's work after the + // check, and unknown-job for a job the server pruned, + // retired past its retention before this session read + // the job that replaced it. + let now = phase + .bound( + || format!("miner {miner}'s job {job} had its payout revisions read after its share was answered {}", share.answer), + work_revision(f, &job), + ) + .await?; + match (share.answer.reason_id(), now) { + (Some("stale-job"), Some((work, ledger))) if work != ledger => { + phase + .bound( + || format!("miner {miner}'s refused share on job {job} was looked up in the ledger"), + ensure_no_ledger_row(f, miner, &job, &share), + ) + .await?; + superseded.push((miner, job.clone(), work, ledger)); + after.push(("on new-tip work a later revision move superseded", share)); + } + (Some("unknown-job"), _) => { + phase + .bound( + || format!("miner {miner}'s refused share on job {job} was looked up in the ledger"), + ensure_no_ledger_row(f, miner, &job, &share), + ) + .await?; + retired.push((miner, job.clone())); + after.push(("on new-tip work retired past retention", share)); + } + (_, now) => bail!( + "miner {miner}'s share on work for the new tip was answered {} (job {job}; its work and the ledger's payout revisions {revisions:?} before the share, {now:?} after it)", + share.answer + ), + } + } + Some((work, ledger)) => { + if refused_on_superseded.is_none() { + // Refused stale-job for its revision, or unknown-job: + // the server retires a job when it delivers the next + // one, read or not, and prunes it once retired past + // its retention (30 s by default), which a slow run + // can outlast before it reads this session. + let share = phase + .bound( + || format!("miner {miner}'s surviving session waited for the answer to its share on job {job}, superseded work at payout revision {work}"), + session.submit(Proof::Share), + ) + .await?; + ensure!( + matches!(share.answer.reason_id(), Some("stale-job" | "unknown-job")), + "miner {miner}'s share on job {job}, new-tip work at payout revision {work} that the ledger's {ledger} superseded, was answered {}, not stale-job or, for a job retired past retention, unknown-job", + share.answer + ); + phase + .bound( + || format!("miner {miner}'s refused share on job {job} was looked up in the ledger"), + ensure_no_ledger_row(f, miner, &job, &share), + ) + .await?; + refused_on_superseded = Some(( + miner, + job.clone(), + share.share_id.clone(), + share.answer.to_string(), + )); + after.push(("on new-tip work the confirmation superseded", share)); + } + superseded.push((miner, job.clone(), work, ledger)); + } + None => unrecorded.push((miner, job.clone())), + } + // The final block moves the revision past new-tip work once, + // with its confirmation (#752). So a session meets work on at + // most one superseded revision: the first revision, whether its + // job was passed over or its share refused when the confirmation + // landed after the check. It can pass over more than one job on + // that revision, such as a republication for new shares before + // the confirmation, so only distinct revisions are bounded. A + // second one would be a revision move this drill never makes. + let stale: BTreeSet = superseded + .iter() + .filter(|entry| entry.0 == miner) + .map(|entry| entry.2) + .collect(); + ensure!( + stale.len() <= 1, + "miner {miner}'s surviving session met new-tip work on {} superseded payout revisions {stale:?}, but the final block supersedes its new-tip work once (#752): {}", + stale.len(), + passed_over(miner, &tip, &superseded, &unrecorded, &retired) + ); + // The next job, unless an answer above already brought it. + if session.job_id() == Some(job.as_str()) { + phase + .bound( + || { + format!( + "miner {miner}'s surviving session waited for a job after {job}: {}", + passed_over(miner, &tip, &superseded, &unrecorded, &retired) + ) + }, + session.newer_work(), + ) + .await + .with_context(|| { + format!("miner {miner}'s surviving session, waiting for a job after {job} (its work and the ledger's payout revisions: {revisions:?})") + })?; + } + }; + let credits: i64 = phase + .bound( + || { + format!( + "miner {miner}'s accepted share {} was counted in the ledger", + share.share_id + ) + }, + async { + Ok(sqlx::query_scalar( + "SELECT count(*) FROM qbit_share_ledger WHERE accepted AND share_id=$1", + ) + .bind(&share.share_id) + .fetch_one(&f.pool) + .await?) + }, + ) + .await?; + ensure!( + credits == 1, + "miner {miner}'s share on the new tip is credited {credits} time(s)" + ); + on_new_tip += 1; + } + + f.quiesce().await?; + let (rows, ids, seqs, headers): (i64, i64, i64, i64) = sqlx::query_as( + "SELECT count(*),count(DISTINCT share_id),count(DISTINCT share_seq),(SELECT count(*) FROM qbit_prism_share_hashes h JOIN qbit_share_ledger l USING(share_id) WHERE l.accepted AND h.header_hash=right(l.share_id,64)) FROM qbit_share_ledger WHERE accepted", + ) + .fetch_one(&f.pool) + .await?; + ensure!( + rows == ids && rows == seqs && rows == headers, + "share identities, sequence numbers or header map disagree after the promotion: {rows} rows, {ids} ids, {seqs} seqs, {headers} headers" + ); + f.integrity().await?; + let mut answered_after = BTreeMap::::new(); + for (label, submitted) in &after { + *answered_after + .entry(format!("{label}: {}", submitted.answer)) + .or_default() += 1; + } + let count = |outcome: fn(&Outcome) -> bool| { + records + .iter() + .filter(|record| outcome(&record.outcome)) + .count() + }; + eprintln!( + "live async promotion with live miners: {} shares offered by {MINERS} miners on 2 frontends: {} acknowledged, {} rejected {rejected:?}, {} unknown {unknown:?}; \ + {} in flight at the fence, answered {:?}; {} committed before the cut, {} lost in the gap ({gap_bytes} WAL bytes), of which {} acknowledged: {acked_missing:?} ({} of them answered before the cut flag: {acked_before_cut:?}); \ + the old writer committed only its {} held COMMIT(s) after the fence; after the promotion: {answered_after:?}, {on_new_tip} surviving session(s) credited on work for the new tip (jobs passed over on superseded work, as (miner, job, work revision, ledger revision): {superseded:?}; without an issued row or prepared record, as (miner, job): {unrecorded:?}; jobs answered unknown-job, retired past retention, as (miner, job): {retired:?}; a share on superseded work refused without a ledger row, as (miner, job, share, answer): {refused_on_superseded:?}), a surviving share replayed: {}; \ + the gap-issued block: {} (gap sessions replaced first for older work, their windows' last shares: {older:?}); the final block {} paid {} shares, a full read of the promoted window (sessions replaced first for a stale window: {stale_windows:?}; new sessions right after the move: {first_jobs:?}) (of which sequence numbers a lost share had: {paid_reused:?}); {} lost sequence number(s) since reused: {replaced:?}; timeline: {}", + records.len(), + count(|outcome| *outcome == Outcome::Acknowledged), + count(|outcome| matches!(outcome, Outcome::Rejected(_))), + count(|outcome| matches!(outcome, Outcome::Unknown(_))), + in_flight.len(), + in_flight.iter().map(|record| &record.outcome).collect::>(), + pre_cut.len(), + lost.len(), + acked_missing.len(), + acked_before_cut.len(), + completed.len(), + duplicate.answer, + match &stale_paid { + Some(paid) => format!("{} landed paying {} shares", stale_block.hash, paid.len()), + None => format!("{} refused: {}", stale_block.hash, stale_block.answer), + }, + block.hash, + paid.len(), + replaced.len(), + timeline.render(), + ); + standby.close().await; + Ok(()) +} diff --git a/crates/qbit-prism-server/tests/support/live_pg_failover.rs b/crates/qbit-prism-server/tests/support/live_pg_failover.rs new file mode 100644 index 000000000..d05d8096a --- /dev/null +++ b/crates/qbit-prism-server/tests/support/live_pg_failover.rs @@ -0,0 +1,1068 @@ +//! #521 scenario 3: a PostgreSQL primary failover while a found block is +//! mid-landing, on the live fixture (real qbitd, two servers, CPU miners). +//! +//! The database is a managed primary with one dedicated asynchronous standby, +//! the approved D3 policy: `synchronous_standby_names=''` on the primary and +//! `synchronous_commit=on` in writer sessions, so a positive share ACK waits +//! for local WAL durability only. Both servers reach the primary through a +//! stable writer endpoint and the standby streams through a replication link; +//! each is a TCP relay the test can fence. The failover is the operator +//! procedure in `docs/prism-ha-reference-architecture.md` ("Promotion, fencing +//! and the stable writer endpoint"): fence every writer connection, capture the +//! acknowledged-share evidence and WAL positions, stop the old primary, promote +//! the standby with `pg_promote(wait => true)`, confirm it left recovery, and +//! only then move the writer endpoint to it. +//! +//! A gate in front of qbitd holds every `submitblock` call from the first one +//! on. The first call's reservation (`offer_reserved`) is durable before the +//! call is made, so while the gate holds it the found block is mid-landing: +//! offered or about to be, with its outcome not yet recorded. Held calls reach +//! the node one at a time in arrival order once released. +//! +//! Two drills share the procedure: +//! +//! - [`Drill::AsyncLoss`] cuts replication while the miners keep receiving +//! ACKs, so the promoted standby lacks a real unreplicated interval, and +//! releases the held call only after the writers are fenced, so the offering +//! frontend learns the node's answer but cannot record it. The documented +//! consequence is asserted exactly: every share committed before the cut +//! survives, the shares committed in the gap are lost and are counted and +//! printed, and the reservation, which was replicated, is recovered as an +//! unknown outcome and landed on the new primary without a second call. +//! The offering frontend's claim release fails with the old primary's +//! connections; it retries the release once the new primary answers +//! (#529), so the landing follows within [`LANDING_BOUND`] of the +//! promotion instead of after the 120 s candidate lease. +//! - [`Drill::Fenced`] is the planned switch: after the fence, the standby +//! replays through the old primary's flush LSN before promotion, so no +//! acknowledged share may be lost, and the held call returns only after the +//! endpoint moved, so the same attempt records the offer on the new primary. +use super::*; +use axum::{ + body::Bytes, + extract::State, + http::{header, HeaderMap, StatusCode}, + response::{IntoResponse, Response}, +}; +use qbit_prism_server::codec::{double_sha256, hash_display}; +use std::{ + collections::BTreeSet, + path::Path, + sync::{ + atomic::{AtomicBool, AtomicU16, AtomicU64, Ordering}, + Arc, + }, +}; +use tokio::{sync::watch, task::JoinHandle}; + +/// #529: the found-block offer's standby flush wait under a replication cut; +/// it reuses this module's pair, relays and node gate. +#[path = "live_pg_offer_standby.rs"] +mod offer_standby; + +/// #474 B: the async promotion with live Stratum miners whose sessions and +/// frontends survive it; it reuses this module's pair and relays. +#[path = "live_pg_async_live_miners.rs"] +mod async_live_miners; + +/// The dedicated standby's `application_name` and physical slot. +const STANDBY: &str = "prism_standby_1"; + +/// How soon after the promotion the found block must land on the new +/// primary, in both drills. Before #529's deferred claim release the async +/// drill waited out the 120 s candidate lease; now it waits for the +/// offering frontend's pool to reach the moved endpoint and for one retried +/// release. +const LANDING_BOUND: Duration = Duration::from_secs(30); + +/// The two drills start their clusters before `Fixture::open_on_database` +/// takes the fixture's `SERIAL` guard; this keeps a second pair from idling +/// beside the first. +static DRILLS: tokio::sync::Mutex<()> = tokio::sync::Mutex::const_new(()); + +/// When each step of the procedure happened, for the report and a failure. +struct Timeline { + started: Instant, + events: Vec<(Duration, &'static str)>, +} + +impl Timeline { + fn mark(&mut self, event: &'static str) { + self.events.push((self.started.elapsed(), event)); + } + + fn render(&self) -> String { + self.events + .iter() + .map(|(at, event)| format!("{:.2}s {event}", at.as_secs_f64())) + .collect::>() + .join(", ") + } +} + +/// Which failover the scenario runs; see the module documentation. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +enum Drill { + AsyncLoss, + Fenced, +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn real_async_failover_mid_landing_loses_only_the_replication_gap() -> Result<()> { + run(Drill::AsyncLoss).await +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn real_fenced_failover_mid_landing_keeps_every_acknowledged_share() -> Result<()> { + run(Drill::Fenced).await +} + +async fn run(drill: Drill) -> Result<()> { + let Some(bin) = gate::pg_bin_dir(gate::site!())? else { + return Ok(()); + }; + let _drill = DRILLS.lock().await; + // Wait for the fixture guard once before starting the pair: its loopback + // connections, whose source ports the kernel picks, would otherwise race + // the binary's start-up burst, where + // `fixture_ports_stay_reserved_until_their_child_starts` rebinds ports it + // has just released. + drop(SERIAL.lock().await); + let mut pair = Pair::start(bin.into()).await?; + let Some(mut fixture) = + Fixture::open_on_database(false, false, Some(&pair.url(pair.writer.port))).await? + else { + return Ok(()); + }; + let mut timeline = Timeline { + started: Instant::now(), + events: Vec::new(), + }; + let result = async { + let mut node = NodeGate::open(fixture.rpc_port).await?; + let drilled = failover(&mut fixture, &mut pair, &mut node, drill, &mut timeline).await; + node.stop(); + drilled + } + .await; + if result.is_err() { + eprintln!("{}", fixture.diagnostics()); + eprintln!("{}", pair.diagnostics()); + eprintln!("failover timeline: {}", timeline.render()); + // Let cleanup reach whichever cluster still accepts writes. + pair.writer.route_to(pair.writable_port()); + } + let cleanup = fixture.cleanup().await; + drop(pair); + result.and(cleanup) +} + +async fn failover( + f: &mut Fixture, + pair: &mut Pair, + node: &mut NodeGate, + drill: Drill, + timeline: &mut Timeline, +) -> Result<()> { + let primary = PgPool::connect(&pair.url(pair.primary.port)).await?; + let standby = PgPool::connect(&pair.url(pair.standby.port)).await?; + let schema = f.schema.clone(); + let names: String = sqlx::query_scalar("SHOW synchronous_standby_names") + .fetch_one(&primary) + .await?; + ensure!( + names.is_empty(), + "D3 requires an asynchronous standby, found {names:?}" + ); + + // The found block's `submitblock` must outlive the failover, so the call + // deadline is raised from its one-second default. Every other setting is + // the fixture's. + for index in 0..2 { + let process = f.start_server_with( + index, + None, + &[ + ("QBIT_RPC_PORT", node.port.to_string()), + ("PRISM_BLOCK_SUBMIT_RPC_TIMEOUT_SECONDS", "120".into()), + ], + )?; + f.servers.push(process); + } + for index in 0..2 { + until(&format!("server {index} readiness"), 30, || { + healthy(f, index) + }) + .await?; + } + node.hold()?; + f.start_miner(0)?; + f.start_miner(1)?; + + // The found block, reserved and held at the node. + let block = node.first_arrival(Duration::from_secs(60)).await?; + timeline.mark("held"); + let row_state = + format!("SELECT state FROM {schema}.qbit_block_candidate_outbox WHERE block_hash=$1"); + let state_on = |pool: &PgPool| { + let (pool, query, block) = (pool.clone(), row_state.clone(), block.clone()); + async move { + Ok::<_, anyhow::Error>( + sqlx::query_scalar::<_, String>(&query) + .bind(&block) + .fetch_optional(&pool) + .await?, + ) + } + }; + ensure!( + state_on(&primary).await?.as_deref() == Some("offer_reserved"), + "the held submitblock for {block} was not preceded by its durable reservation" + ); + // The reservation must be on the standby before anything is cut: this + // scenario is about a replicated reservation whose landing is in flight. + until("reservation replicated to the standby", 15, || async { + Ok(state_on(&standby).await?.as_deref() == Some("offer_reserved")) + }) + .await?; + + let mut pre_cut = BTreeSet::new(); + let mut frozen = BTreeSet::new(); + let mut acknowledged_at_cut = None; + if drill == Drill::AsyncLoss { + until("pre-cut shares on the primary", 30, || async { + Ok(shares(&primary, &schema).await?.len() >= 3) + }) + .await?; + pre_cut = shares(&primary, &schema).await?; + until("standby replay of the pre-cut shares", 15, || async { + Ok(shares(&standby, &schema).await?.is_superset(&pre_cut)) + }) + .await?; + pair.replication.fence(); + timeline.mark("replication cut"); + let at_cut = acknowledged(f)?; + acknowledged_at_cut = Some(at_cut); + until("standby replay settled after the cut", 15, || async { + Ok(sqlx::query_scalar::<_, bool>( + "SELECT pg_last_wal_replay_lsn()=pg_last_wal_receive_lsn()", + ) + .fetch_one(&standby) + .await?) + }) + .await?; + frozen = shares(&standby, &schema).await?; + ensure!( + frozen.is_superset(&pre_cut), + "the standby lost pre-cut shares" + ); + // The unreplicated interval: shares committed and acknowledged on the + // primary that the standby can no longer receive. + until( + "acknowledged shares inside the replication gap", + 30, + || async { + let committed = shares(&primary, &schema).await?; + Ok(!committed.is_subset(&frozen) && acknowledged(f)? > at_cut) + }, + ) + .await?; + } + + if drill == Drill::Fenced { + // The held attempt must keep its claim through the switch: fence only + // on a fresh lease, well before the heartbeat's next renewal (every + // 30 s), so no renewal can fall inside the outage. + let renewed_since = format!( + "SELECT 120-extract(epoch FROM claim_expires_at-clock_timestamp())::float8 \ + FROM {schema}.qbit_block_candidate_outbox WHERE block_hash=$1" + ); + until("a fresh lease on the held block's claim", 40, || async { + let since: f64 = sqlx::query_scalar(&renewed_since) + .bind(&block) + .fetch_one(&primary) + .await?; + Ok(since < 15.0) + }) + .await?; + } + // Fence the old primary first: no writer can reach it from here on. + pair.writer.fence(); + timeline.mark("writers fenced"); + for miner in &mut f.miners { + miner.stop(); + } + // ACKs the miners received after the cut: none can follow the fence. + let acknowledged_in_gap = match acknowledged_at_cut { + Some(at_cut) => acknowledged(f)? - at_cut, + None => 0, + }; + if drill == Drill::AsyncLoss { + // The node accepts the block, but the offering frontend cannot record + // the answer: the outcome is lost with the primary. + node.release(); + node.forwarded(&block, Duration::from_secs(15)).await?; + timeline.mark("node answered"); + } + + // The acknowledged-share evidence and WAL positions, read out of band + // from the fenced primary, so they are final. + let old_shares = shares(&primary, &schema).await?; + let old_candidates = candidates(&primary, &schema).await?; + let flush: String = sqlx::query_scalar("SELECT pg_current_wal_flush_lsn()::text") + .fetch_one(&primary) + .await?; + if drill == Drill::Fenced { + // The planned switch: replay through the captured flush LSN. + until( + "standby replay through the fenced primary's flush LSN", + 15, + || async { + Ok( + sqlx::query_scalar::<_, bool>("SELECT pg_last_wal_replay_lsn()>=$1::pg_lsn") + .bind(&flush) + .fetch_one(&standby) + .await?, + ) + }, + ) + .await?; + } + let gap_bytes: i64 = + sqlx::query_scalar("SELECT pg_wal_lsn_diff($1::pg_lsn,pg_last_wal_receive_lsn())::bigint") + .bind(&flush) + .fetch_one(&standby) + .await?; + primary.close().await; + pair.primary.stop()?; + timeline.mark("old primary stopped"); + + // Promote the one eligible standby and confirm it left recovery. + let promoted_at = Instant::now(); + let promoted: bool = sqlx::query_scalar("SELECT pg_promote(true,60)") + .fetch_one(&standby) + .await?; + ensure!(promoted, "pg_promote did not complete within 60 seconds"); + let recovering: bool = sqlx::query_scalar("SELECT pg_is_in_recovery()") + .fetch_one(&standby) + .await?; + ensure!(!recovering, "the promoted standby is still in recovery"); + pair.promoted = true; + timeline.mark("promoted"); + let new_shares = shares(&standby, &schema).await?; + let new_candidates = candidates(&standby, &schema).await?; + ensure!( + state_on(&standby).await?.as_deref() == Some("offer_reserved"), + "the promoted primary must hold the found block's reservation" + ); + ensure!( + new_shares.is_subset(&old_shares), + "the promoted primary holds shares the old primary never committed" + ); + let lost: BTreeSet<_> = old_shares.difference(&new_shares).cloned().collect(); + let lost_candidates: BTreeSet<_> = old_candidates + .difference(&new_candidates) + .cloned() + .collect(); + match drill { + Drill::AsyncLoss => { + ensure!( + new_shares.is_superset(&pre_cut), + "an acknowledged share committed before the replication cut was lost" + ); + ensure!( + new_shares == frozen, + "promotion changed what the standby had received" + ); + ensure!( + !lost.is_empty() && gap_bytes > 0, + "the drill did not produce an unreplicated interval" + ); + ensure!(lost.is_disjoint(&pre_cut), "a lost share predates the cut"); + } + Drill::Fenced => { + ensure!( + lost.is_empty() && lost_candidates.is_empty() && gap_bytes <= 0, + "a fenced switch with replay through the flush LSN lost {} shares and {} candidates ({gap_bytes} WAL bytes)", + lost.len(), + lost_candidates.len() + ); + } + } + + if drill == Drill::AsyncLoss { + // Move the endpoint only once the offering attempt has failed, so it + // cannot record the answer on the new primary. Its claim release + // fails on the same dead connections and is retried until the moved + // endpoint answers (#529); the row is then recovered from + // offer_reserved. + until( + "the offering attempt to fail on the fenced endpoint", + 60, + || async { server_logged(f, &["candidate remains recoverable", &block]) }, + ) + .await?; + timeline.mark("offering attempt failed"); + } + // Move the stable writer endpoint to the sole primary. + pair.writer.route_to(pair.standby.port); + let moved_after = promoted_at.elapsed(); + timeline.mark("endpoint moved"); + if drill == Drill::Fenced { + node.release(); + node.forwarded(&block, Duration::from_secs(15)).await?; + timeline.mark("node answered"); + } + + // The landing completes on the new primary, from its offer_reserved row. + until( + "the found block's landing on the new primary", + 180, + || async { Ok(state_on(&f.pool).await?.as_deref() == Some("submitted")) }, + ) + .await?; + let landed_after = promoted_at.elapsed(); + timeline.mark("landed"); + ensure!( + landed_after <= LANDING_BOUND, + "the found block landed {:.1} s after the promotion, beyond {} s: the recovery waited for the candidate lease", + landed_after.as_secs_f64(), + LANDING_BOUND.as_secs() + ); + let (outcome, offered_at, reserved_by): (Option, Option, Option) = + sqlx::query_as( + "SELECT offer_outcome,offered_at_ms,offer_reserved_by FROM qbit_block_candidate_outbox WHERE block_hash=$1", + ) + .bind(&block) + .fetch_one(&f.pool) + .await?; + ensure!( + reserved_by.is_some(), + "the landed row lost its reservation record" + ); + match drill { + // The answer was lost with the old primary: the recovered reservation + // lands with an unknown outcome and no call time (#529). + Drill::AsyncLoss => ensure!( + outcome.as_deref() == Some("unknown") && offered_at.is_none(), + "the lost answer was recorded as {outcome:?} at {offered_at:?}, expected an unknown outcome" + ), + // The same attempt recorded the node's answer on the new primary. + Drill::Fenced => ensure!( + outcome.as_deref() == Some("accepted") && offered_at.is_some(), + "the offering attempt recorded {outcome:?} at {offered_at:?}, expected its accepted answer" + ), + } + let audits: i64 = + sqlx::query_scalar("SELECT count(*) FROM qbit_pool_audit_bundles WHERE block_hash=$1") + .bind(&block) + .fetch_one(&f.pool) + .await?; + ensure!(audits == 1, "the landed block has {audits} audit bundles"); + until( + "the landed block confirmed on the new primary", + 30, + || async { + Ok(sqlx::query_scalar::<_, String>( + "SELECT chain_state FROM qbit_pool_blocks WHERE block_hash=$1", + ) + .bind(&block) + .fetch_optional(&f.pool) + .await? + .as_deref() + == Some("confirmed")) + }, + ) + .await?; + let header = f.rpc("getblockheader", json!([block])).await?; + ensure!( + header["confirmations"].as_i64().unwrap_or(0) >= 1, + "the landed block is not on the node's active chain: {header}" + ); + + // Job delivery resumes on both frontends against the new primary. + for index in 0..2 { + until( + &format!("server {index} readiness after the move"), + 60, + || healthy(f, index), + ) + .await?; + } + let before = [f.count(0).await?, f.count(1).await?]; + f.start_miner(0)?; + f.start_miner(1)?; + until( + "accepted shares on both servers after the move", + 60, + || async { Ok(f.count(0).await? > before[0] && f.count(1).await? > before[1]) }, + ) + .await?; + + f.quiesce().await?; + let reserved: i64 = sqlx::query_scalar( + "SELECT count(*) FROM qbit_block_candidate_outbox WHERE state='offer_reserved'", + ) + .fetch_one(&f.pool) + .await?; + ensure!(reserved == 0, "{reserved} rows are stuck in offer_reserved"); + let (rows, ids, seqs): (i64, i64, i64) = sqlx::query_as( + "SELECT count(*),count(DISTINCT share_id),count(DISTINCT share_seq) FROM qbit_share_ledger", + ) + .fetch_one(&f.pool) + .await?; + ensure!( + rows == ids && rows == seqs, + "share identities or sequence numbers repeat after the failover: {rows} rows, {ids} ids, {seqs} seqs" + ); + f.integrity().await?; + let arrivals = node.arrivals(); + let offered: BTreeSet<_> = arrivals.iter().collect(); + ensure!( + offered.len() == arrivals.len(), + "a block was offered to the node more than once: {arrivals:?}" + ); + ensure!( + node.forward_order().first() == Some(&block), + "the found block was not the first held call to reach the node" + ); + // Blocks whose candidate rows the unreplicated interval took with it, + // although the node received them. + let lost_offered: Vec<_> = arrivals + .iter() + .filter(|hash| lost_candidates.contains(*hash)) + .collect(); + + eprintln!( + "live failover ({drill:?}): block {block} held at submitblock, landed on the promoted \ + primary {:.1} s after promotion with outcome {outcome:?}; {} submitblock calls, each once; replication gap \ + {gap_bytes} WAL bytes, {} committed shares lost ({} ACKs received by miners after the \ + cut), {} candidate rows lost ({} of them offered to the node); {} shares survived; \ + endpoint moved {:.1} s after promotion; lost shares: {lost:?}; lost candidates: \ + {lost_candidates:?}; timeline: {}", + landed_after.as_secs_f64(), + arrivals.len(), + lost.len(), + acknowledged_in_gap, + lost_candidates.len(), + lost_offered.len(), + new_shares.len(), + moved_after.as_secs_f64(), + timeline.render(), + ); + standby.close().await; + Ok(()) +} + +/// Every share in the ledger of `schema`, by identity. +async fn shares(pool: &PgPool, schema: &str) -> Result> { + Ok( + sqlx::query_scalar(&format!("SELECT share_id FROM {schema}.qbit_share_ledger")) + .fetch_all(pool) + .await? + .into_iter() + .collect(), + ) +} + +/// Every block candidate in the outbox of `schema`, by block hash. +async fn candidates(pool: &PgPool, schema: &str) -> Result> { + Ok(sqlx::query_scalar(&format!( + "SELECT block_hash FROM {schema}.qbit_block_candidate_outbox" + )) + .fetch_all(pool) + .await? + .into_iter() + .collect()) +} + +async fn healthy(f: &Fixture, index: usize) -> Result { + let response = f + .client + .get(format!("http://127.0.0.1:{}/healthz", f.api[index])) + .send() + .await?; + if !response.status().is_success() { + return Ok(false); + } + let body: Value = response.json().await?; + Ok(body["ok"] == true) +} + +/// The positive share ACKs the miners have printed so far. +fn acknowledged(f: &Fixture) -> Result { + let mut count = 0; + for entry in std::fs::read_dir(f.directory.path())? { + let path = entry?.path(); + let name = path + .file_name() + .and_then(|name| name.to_str()) + .unwrap_or(""); + if !(name.starts_with("miner-") && name.ends_with(".json")) { + continue; + } + // A line still being written does not parse and is not counted yet. + count += std::fs::read_to_string(&path)? + .lines() + .filter_map(|line| serde_json::from_str::(line).ok()) + .filter(|event| event["event"] == "share" && event["accepted"] == true) + .count(); + } + Ok(count) +} + +/// Whether one line of either server's log contains every fragment. +fn server_logged(f: &Fixture, fragments: &[&str]) -> Result { + for index in 0..f.servers.len() { + let log = std::fs::read_to_string(f.directory.path().join(format!("server-{index}.log")))?; + if log + .lines() + .any(|line| fragments.iter().all(|fragment| line.contains(fragment))) + { + return Ok(true); + } + } + Ok(false) +} + +/// One disposable PostgreSQL 16 cluster. +struct Cluster { + bin: PathBuf, + data: PathBuf, + port: u16, + /// Holds `port` until the server binds it, so no concurrent test takes it. + reservation: Option, + running: bool, +} + +impl Cluster { + fn new(bin: &Path, data: PathBuf) -> Result { + let reservation = std::net::TcpListener::bind("127.0.0.1:0")?; + Ok(Self { + bin: bin.to_path_buf(), + data, + port: reservation.local_addr()?.port(), + reservation: Some(reservation), + running: false, + }) + } + + fn command(bin: &Path, binary: &str, args: &[&str]) -> Result<()> { + let output = Command::new(bin.join(binary)).args(args).output()?; + ensure!( + output.status.success(), + "{binary} failed: {} {}", + String::from_utf8_lossy(&output.stdout), + String::from_utf8_lossy(&output.stderr) + ); + Ok(()) + } + + fn control(&self, args: &[&str]) -> Result<()> { + let mut all = vec![ + "-D", + self.data.to_str().context("database path")?, + "-t", + "30", + ]; + all.extend_from_slice(args); + Self::command(&self.bin, "pg_ctl", &all) + } + + fn start(&mut self, socket: &Path, settings: &str) -> Result<()> { + // Marked first, so a partially successful start is still stopped. + self.running = true; + self.reservation = None; + self.control(&[ + "-l", + self.data.with_extension("log").to_str().context("log path")?, + "-o", + &format!( + "-h 127.0.0.1 -p {} -k {} -c fsync=on -c full_page_writes=on -c max_connections=200 {settings}", + self.port, + socket.display() + ), + "-w", + "start", + ]) + } + + /// An immediate stop: for the primary, the loss of the server. + fn stop(&mut self) -> Result<()> { + if self.running { + self.control(&["-m", "immediate", "-w", "stop"])?; + self.running = false; + } + Ok(()) + } +} + +impl Drop for Cluster { + fn drop(&mut self) { + let _ = self.stop(); + } +} + +/// The D3 primary and its dedicated asynchronous standby, the stable writer +/// endpoint in front of the primary, and the standby's replication link. +struct Pair { + primary: Cluster, + standby: Cluster, + writer: Relay, + replication: Relay, + promoted: bool, + user: String, + /// Declared last, so both clusters stop before their files go. + directory: tempfile::TempDir, +} + +impl Pair { + async fn start(bin: PathBuf) -> Result { + // Keep the Unix socket path short and pg_ctl's option path space-free. + let directory = tempfile::Builder::new() + .prefix("b521-") + .tempdir_in("/tmp")?; + let user = String::from_utf8(Command::new("id").arg("-un").output()?.stdout)? + .trim() + .to_owned(); + let mut primary = Cluster::new(&bin, directory.path().join("primary"))?; + Cluster::command( + &bin, + "initdb", + &[ + "-D", + primary.data.to_str().context("database path")?, + "-A", + "trust", + "--no-locale", + "-E", + "UTF8", + ], + )?; + // synchronous_standby_names stays at its empty default: asynchronous. + primary.start( + directory.path(), + "-c synchronous_commit=on -c wal_level=replica -c max_wal_senders=10 -c max_replication_slots=10", + )?; + let replication = Relay::open(primary.port).await?; + let writer = Relay::open(primary.port).await?; + let mut pair = Self { + standby: Cluster::new(&bin, directory.path().join("standby"))?, + primary, + writer, + replication, + promoted: false, + user, + directory, + }; + let admin = PgPool::connect(&pair.url(pair.primary.port)).await?; + sqlx::query("SELECT pg_create_physical_replication_slot($1)") + .bind(STANDBY) + .execute(&admin) + .await?; + // The base backup and the streaming connection both go through the + // replication link, which `-R` records as the standby's conninfo. + let conninfo = format!( + "host=127.0.0.1 port={} user={} application_name={STANDBY}", + pair.replication.port, pair.user + ); + Cluster::command( + &bin, + "pg_basebackup", + &[ + "-D", + pair.standby.data.to_str().context("database path")?, + "-d", + &conninfo, + "-X", + "stream", + "-R", + "-S", + STANDBY, + "-c", + "fast", + ], + )?; + let auto = pair.standby.data.join("postgresql.auto.conf"); + let mut settings = std::fs::read_to_string(&auto)?; + settings.push_str(&format!( + "\nprimary_conninfo = '{conninfo}'\nprimary_slot_name = '{STANDBY}'\n" + )); + std::fs::write(&auto, settings)?; + let socket = pair.directory.path().to_path_buf(); + pair.standby.start(&socket, "-c hot_standby=on")?; + until("asynchronous standby streaming", 60, || async { + Ok(sqlx::query_scalar::<_, bool>( + "SELECT EXISTS(SELECT 1 FROM pg_stat_replication WHERE application_name=$1 AND state='streaming' AND sync_state='async')", + ) + .bind(STANDBY) + .fetch_one(&admin) + .await?) + }) + .await?; + admin.close().await; + Ok(pair) + } + + fn url(&self, port: u16) -> String { + format!("postgresql://{}@127.0.0.1:{port}/postgres", self.user) + } + + fn writable_port(&self) -> u16 { + if self.promoted { + self.standby.port + } else { + self.primary.port + } + } + + fn diagnostics(&self) -> String { + let mut report = String::new(); + for (role, cluster) in [("primary", &self.primary), ("standby", &self.standby)] { + let log = std::fs::read_to_string(cluster.data.with_extension("log")) + .unwrap_or_else(|error| format!("unreadable: {error}")); + let tail: Vec<_> = log.lines().rev().take(20).collect(); + report.push_str(&format!( + "--- {role} PostgreSQL log (last 20 lines, newest first)\n" + )); + for line in tail { + report.push_str(line); + report.push('\n'); + } + } + report + } +} + +/// A TCP relay the test can fence: every new connection is refused and every +/// existing one closed, the network isolation the procedure's fence requires. +/// Routing it elsewhere admits connections again, to the new target. +struct Relay { + port: u16, + target: Arc, + fenced: Arc, + /// Milliseconds each target-to-client chunk is held, for connections + /// accepted while it is non-zero: replication lag, for the #529 drills. + delay_ms: Arc, + severed: watch::Sender, + task: JoinHandle<()>, +} + +impl Relay { + async fn open(target: u16) -> Result { + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await?; + let port = listener.local_addr()?.port(); + let target = Arc::new(AtomicU16::new(target)); + let fenced = Arc::new(AtomicBool::new(false)); + let delay_ms = Arc::new(AtomicU64::new(0)); + let (severed, _) = watch::channel(0u64); + let task = tokio::spawn({ + let (target, fenced, severed, delay_ms) = ( + target.clone(), + fenced.clone(), + severed.clone(), + delay_ms.clone(), + ); + async move { + while let Ok((mut client, _)) = listener.accept().await { + // Subscribe before reading the fence: a fence set after + // this point also bumps the generation and closes it. + let mut generation = severed.subscribe(); + if fenced.load(Ordering::SeqCst) { + continue; + } + let target = target.load(Ordering::SeqCst); + let delay = Duration::from_millis(delay_ms.load(Ordering::SeqCst)); + tokio::spawn(async move { + tokio::select! { + _ = generation.changed() => {} + _ = async { + if let Ok(mut upstream) = + tokio::net::TcpStream::connect(("127.0.0.1", target)).await + { + // PostgreSQL disables Nagle on its own sockets; the + // relay does too, or its small replies wait ~40 ms. + let _ = (client.set_nodelay(true), upstream.set_nodelay(true)); + if delay.is_zero() { + let _ = tokio::io::copy_bidirectional(&mut client, &mut upstream).await; + } else { + offer_standby::delayed_pump(client, upstream, delay).await; + } + } + } => {} + } + }); + } + } + }); + Ok(Self { + port, + target, + fenced, + delay_ms, + severed, + task, + }) + } + + fn fence(&self) { + self.fenced.store(true, Ordering::SeqCst); + self.severed.send_modify(|generation| *generation += 1); + } + + fn route_to(&self, target: u16) { + self.target.store(target, Ordering::SeqCst); + self.fenced.store(false, Ordering::SeqCst); + } +} + +impl Drop for Relay { + fn drop(&mut self) { + self.fence(); + self.task.abort(); + } +} + +/// A JSON-RPC relay in front of qbitd that counts every `submitblock` and, +/// while held, queues them; released, they reach the node one at a time in +/// arrival order. +struct NodeGate { + port: u16, + state: Arc, + hold: Option>, + task: JoinHandle<()>, +} + +struct GateState { + node: String, + client: reqwest::Client, + queue: Arc>, + /// The hash of every `submitblock` in arrival order. + arrivals: watch::Sender>, + /// The hash of every `submitblock` the node answered, in answer order. + forwarded: watch::Sender>, +} + +impl NodeGate { + async fn open(node_port: u16) -> Result { + let state = Arc::new(GateState { + node: format!("http://127.0.0.1:{node_port}/"), + client: reqwest::Client::builder() + .timeout(Duration::from_secs(60)) + .build()?, + queue: Arc::new(tokio::sync::Mutex::new(())), + arrivals: watch::channel(Vec::new()).0, + forwarded: watch::channel(Vec::new()).0, + }); + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await?; + let port = listener.local_addr()?.port(); + let app = axum::Router::new() + .fallback(relay_rpc) + .with_state(state.clone()); + let task = tokio::spawn(async move { + let _ = axum::serve(listener, app).await; + }); + Ok(Self { + port, + state, + hold: None, + task, + }) + } + + fn hold(&mut self) -> Result<()> { + let queue = self.state.queue.clone().try_lock_owned(); + self.hold = Some(queue.context("a submitblock was in flight before the hold")?); + Ok(()) + } + + fn release(&mut self) { + self.hold = None; + } + + async fn first_arrival(&self, limit: Duration) -> Result { + let mut arrivals = self.state.arrivals.subscribe(); + let first = tokio::time::timeout(limit, arrivals.wait_for(|hashes| !hashes.is_empty())) + .await + .context("no block was found and offered")??; + Ok(first[0].clone()) + } + + async fn forwarded(&self, block: &str, limit: Duration) -> Result<()> { + let mut forwarded = self.state.forwarded.subscribe(); + tokio::time::timeout( + limit, + forwarded.wait_for(|hashes| hashes.iter().any(|hash| hash == block)), + ) + .await + .with_context(|| format!("the node never answered the held submitblock for {block}"))??; + Ok(()) + } + + fn arrivals(&self) -> Vec { + self.state.arrivals.borrow().clone() + } + + fn forward_order(&self) -> Vec { + self.state.forwarded.borrow().clone() + } + + fn stop(&mut self) { + self.release(); + self.task.abort(); + } +} + +async fn relay_rpc( + State(gate): State>, + headers: HeaderMap, + body: Bytes, +) -> Response { + let request: Value = serde_json::from_slice(&body).unwrap_or(Value::Null); + let submitted = (request["method"] == "submitblock").then(|| { + request["params"][0] + .as_str() + .and_then(|block| hex::decode(block).ok()) + .filter(|block| block.len() >= 80) + .map_or_else( + || "".to_owned(), + |block| hash_display(&double_sha256(&block[..80])), + ) + }); + let _turn = match &submitted { + Some(hash) => { + gate.arrivals + .send_modify(|hashes| hashes.push(hash.clone())); + Some(gate.queue.lock().await) + } + None => None, + }; + let mut upstream = gate + .client + .post(&gate.node) + .header(header::CONTENT_TYPE, "application/json") + .body(body); + if let Some(authorization) = headers.get(header::AUTHORIZATION) { + upstream = upstream.header(header::AUTHORIZATION, authorization.clone()); + } + let answer = async { + let response = upstream.send().await?; + let status = response.status(); + Ok::<_, reqwest::Error>((status, response.bytes().await?)) + } + .await; + match answer { + Ok((status, bytes)) => { + if let Some(hash) = submitted { + gate.forwarded.send_modify(|hashes| hashes.push(hash)); + } + ( + StatusCode::from_u16(status.as_u16()).unwrap_or(StatusCode::BAD_GATEWAY), + [(header::CONTENT_TYPE, "application/json")], + bytes, + ) + .into_response() + } + Err(_) => StatusCode::BAD_GATEWAY.into_response(), + } +} diff --git a/crates/qbit-prism-server/tests/support/live_pg_offer_standby.rs b/crates/qbit-prism-server/tests/support/live_pg_offer_standby.rs new file mode 100644 index 000000000..9ba5d7a99 --- /dev/null +++ b/crates/qbit-prism-server/tests/support/live_pg_offer_standby.rs @@ -0,0 +1,372 @@ +//! #529: a found block's offer waits, within a bound, for the dedicated +//! asynchronous standby to flush its candidate row and reservation +//! (`PRISM_OFFER_STANDBY_APPLICATION_NAME`, `PRISM_OFFER_STANDBY_FLUSH_WAIT_MS`), +//! while every share append stays asynchronous under D3. +//! +//! The drill cuts replication the moment the found block's `submitblock` +//! reaches the node, over a replication link that lags by a fixed delay, so +//! without the wait the standby cannot hold the block's row at the cut. The +//! old primary is then fenced and lost, the standby promoted, and the held +//! attempt records the node's answer on the new primary: the block survives +//! the failover and lands once. Without the wait, or with one that polled +//! `sent_lsn` instead of `flush_lsn`, the standby has no row for the offered +//! block at the cut. +use super::*; +use tokio::io::{AsyncReadExt, AsyncWriteExt}; + +/// The drill's wait bound: far above the link delay, so only a broken wait +/// lets the offer out before the standby has the row. +const DRILL_BOUND_MS: u64 = 5_000; +/// The replication link's delay in the drill: far above the test's reaction +/// from the node's first arrival to the cut. +const DRILL_LAG: Duration = Duration::from_millis(400); + +/// Relay `client` and `upstream`, holding every upstream-to-client chunk for +/// `delay` before it is written, in order; client-to-upstream is immediate. +pub(super) async fn delayed_pump( + client: tokio::net::TcpStream, + upstream: tokio::net::TcpStream, + delay: Duration, +) { + let (mut client_read, mut client_write) = client.into_split(); + let (mut upstream_read, mut upstream_write) = upstream.into_split(); + let (chunks, mut due) = + tokio::sync::mpsc::unbounded_channel::<(tokio::time::Instant, Vec)>(); + let inbound = async move { + let mut buffer = vec![0u8; 64 * 1024]; + while let Ok(read) = upstream_read.read(&mut buffer).await { + if read == 0 + || chunks + .send((tokio::time::Instant::now() + delay, buffer[..read].to_vec())) + .is_err() + { + break; + } + } + }; + let delivered = async move { + while let Some((at, bytes)) = due.recv().await { + tokio::time::sleep_until(at).await; + if client_write.write_all(&bytes).await.is_err() { + break; + } + } + }; + let outbound = async move { + let _ = tokio::io::copy(&mut client_read, &mut upstream_write).await; + }; + tokio::select! { + _ = outbound => {} + _ = async { tokio::join!(inbound, delivered); } => {} + } +} + +impl Relay { + /// Delay the target-to-client direction of connections accepted from + /// here on. + fn set_delay(&self, delay: Duration) { + self.delay_ms + .store(delay.as_millis() as u64, Ordering::SeqCst); + } +} + +impl Pair { + /// Reconnect the standby through a replication link that lags by `delay` + /// and wait until it streams again. + async fn lag_replication(&self, delay: Duration) -> Result<()> { + self.replication.set_delay(delay); + self.replication.fence(); + self.replication.route_to(self.primary.port); + let admin = PgPool::connect(&self.url(self.primary.port)).await?; + until("standby streaming through the lagging link", 60, || async { + Ok(sqlx::query_scalar::<_, bool>( + "SELECT EXISTS(SELECT 1 FROM pg_stat_replication WHERE application_name=$1 AND state='streaming')", + ) + .bind(STANDBY) + .fetch_one(&admin) + .await?) + }) + .await?; + admin.close().await; + Ok(()) + } +} + +fn wait_settings(bound_ms: u64) -> Vec<(&'static str, String)> { + vec![ + ( + "RUST_LOG", + "warn,qbit_prism_server::coordinator=info".to_owned(), + ), + ("PRISM_OFFER_STANDBY_APPLICATION_NAME", STANDBY.to_owned()), + ("PRISM_OFFER_STANDBY_FLUSH_WAIT_MS", bound_ms.to_string()), + ] +} + +async fn start_servers( + f: &mut Fixture, + rpc_port: u16, + extra: &[(&'static str, String)], +) -> Result<()> { + for index in 0..2 { + let mut overrides = vec![ + ("QBIT_RPC_PORT", rpc_port.to_string()), + ("PRISM_BLOCK_SUBMIT_RPC_TIMEOUT_SECONDS", "120".to_owned()), + ]; + overrides.extend(extra.iter().cloned()); + let process = f.start_server_with(index, None, &overrides)?; + f.servers.push(process); + } + for index in 0..2 { + until(&format!("server {index} readiness"), 30, || { + healthy(f, index) + }) + .await?; + } + Ok(()) +} + +/// `line` without its terminal colour sequences. +fn strip_ansi(line: &str) -> String { + let mut plain = String::with_capacity(line.len()); + let mut chars = line.chars(); + while let Some(c) = chars.next() { + if c == '\u{1b}' { + for c in chars.by_ref() { + if c.is_ascii_alphabetic() { + break; + } + } + } else { + plain.push(c); + } + } + plain +} + +/// The wait each server logged per block: `(block, outcome, waited_us)`. +fn logged_waits(f: &Fixture) -> Result> { + let field = |line: &str, name: &str| { + line.split_whitespace() + .find_map(|word| word.strip_prefix(&format!("{name}="))) + .map(|value| value.trim_matches('"').to_owned()) + }; + let mut waits = Vec::new(); + for index in 0..f.servers.len() { + let log = std::fs::read_to_string(f.directory.path().join(format!("server-{index}.log")))?; + for line in log + .lines() + .map(strip_ansi) + .filter(|line| line.contains("block offer standby wait")) + { + let line = line.as_str(); + if let (Some(block), Some(outcome), Some(waited)) = ( + field(line, "block"), + field(line, "outcome"), + field(line, "waited_us").and_then(|value| value.parse().ok()), + ) { + waits.push((block, outcome, waited)); + } + } + } + Ok(waits) +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn a_found_block_offered_after_the_standby_flush_survives_a_replication_cut_at_submitblock( +) -> Result<()> { + let Some(bin) = gate::pg_bin_dir(gate::site!())? else { + return Ok(()); + }; + let _drill = DRILLS.lock().await; + drop(SERIAL.lock().await); + let mut pair = Pair::start(bin.into()).await?; + pair.lag_replication(DRILL_LAG).await?; + let Some(mut fixture) = + Fixture::open_on_database(false, false, Some(&pair.url(pair.writer.port))).await? + else { + return Ok(()); + }; + let mut timeline = Timeline { + started: Instant::now(), + events: Vec::new(), + }; + let result = async { + let mut node = NodeGate::open(fixture.rpc_port).await?; + let drilled = cut_after_found(&mut fixture, &mut pair, &mut node, &mut timeline).await; + node.stop(); + drilled + } + .await; + if result.is_err() { + eprintln!("{}", fixture.diagnostics()); + eprintln!("{}", pair.diagnostics()); + eprintln!("failover timeline: {}", timeline.render()); + pair.writer.route_to(pair.writable_port()); + } + let cleanup = fixture.cleanup().await; + drop(pair); + result.and(cleanup) +} + +async fn cut_after_found( + f: &mut Fixture, + pair: &mut Pair, + node: &mut NodeGate, + timeline: &mut Timeline, +) -> Result<()> { + let primary = PgPool::connect(&pair.url(pair.primary.port)).await?; + let standby = PgPool::connect(&pair.url(pair.standby.port)).await?; + let schema = f.schema.clone(); + let names: String = sqlx::query_scalar("SHOW synchronous_standby_names") + .fetch_one(&primary) + .await?; + ensure!( + names.is_empty(), + "D3 requires an asynchronous standby, found {names:?}" + ); + start_servers(f, node.port, &wait_settings(DRILL_BOUND_MS)).await?; + node.hold()?; + f.start_miner(0)?; + f.start_miner(1)?; + + // The found block reaches the node: cut replication at once, inside the + // link's delay, as a primary lost right after its block was offered. + let block = node.first_arrival(Duration::from_secs(60)).await?; + pair.replication.fence(); + timeline.mark("held and replication cut"); + let row_state = + format!("SELECT state FROM {schema}.qbit_block_candidate_outbox WHERE block_hash=$1"); + let state_on = |pool: &PgPool| { + let (pool, query, block) = (pool.clone(), row_state.clone(), block.clone()); + async move { + Ok::<_, anyhow::Error>( + sqlx::query_scalar::<_, String>(&query) + .bind(&block) + .fetch_optional(&pool) + .await?, + ) + } + }; + ensure!( + state_on(&primary).await?.as_deref() == Some("offer_reserved"), + "the held submitblock for {block} was not preceded by its durable reservation" + ); + until("standby replay settled after the cut", 15, || async { + Ok(sqlx::query_scalar::<_, bool>( + "SELECT pg_last_wal_replay_lsn()=pg_last_wal_receive_lsn()", + ) + .fetch_one(&standby) + .await?) + }) + .await?; + let on_standby = state_on(&standby).await?; + ensure!( + on_standby.as_deref() == Some("offer_reserved"), + "the found block {block} was offered before the standby held its reservation (standby row: {on_standby:?}); a failover now loses it" + ); + let waits = logged_waits(f)?; + let confirmed = waits + .iter() + .find(|(hash, _, _)| *hash == block) + .with_context(|| format!("no standby wait was logged for {block}"))?; + ensure!( + confirmed.1 == "confirmed" && confirmed.2 >= DRILL_LAG.as_micros() as u64 / 2, + "the offer of {block} did not wait for the lagging standby: {confirmed:?}" + ); + + // Lose the old primary: fence every writer, stop it, promote the standby + // and move the endpoint, then let the held attempt's answer through. + pair.writer.fence(); + timeline.mark("writers fenced"); + for miner in &mut f.miners { + miner.stop(); + } + let old_candidates = candidates(&primary, &schema).await?; + primary.close().await; + pair.primary.stop()?; + timeline.mark("old primary stopped"); + let promoted_at = Instant::now(); + let promoted: bool = sqlx::query_scalar("SELECT pg_promote(true,60)") + .fetch_one(&standby) + .await?; + ensure!(promoted, "pg_promote did not complete within 60 seconds"); + pair.promoted = true; + timeline.mark("promoted"); + ensure!( + state_on(&standby).await?.as_deref() == Some("offer_reserved"), + "the promoted primary must hold the found block's reservation" + ); + let new_candidates = candidates(&standby, &schema).await?; + let lost_candidates: BTreeSet<_> = old_candidates + .difference(&new_candidates) + .cloned() + .collect(); + pair.writer.route_to(pair.standby.port); + timeline.mark("endpoint moved"); + node.release(); + node.forwarded(&block, Duration::from_secs(15)).await?; + timeline.mark("node answered"); + + until( + "the found block's landing on the new primary", + 180, + || async { Ok(state_on(&f.pool).await?.as_deref() == Some("submitted")) }, + ) + .await?; + let landed_after = promoted_at.elapsed(); + timeline.mark("landed"); + let audits: i64 = + sqlx::query_scalar("SELECT count(*) FROM qbit_pool_audit_bundles WHERE block_hash=$1") + .bind(&block) + .fetch_one(&f.pool) + .await?; + ensure!(audits == 1, "the landed block has {audits} audit bundles"); + until( + "the landed block confirmed on the new primary", + 30, + || async { + Ok(sqlx::query_scalar::<_, String>( + "SELECT chain_state FROM qbit_pool_blocks WHERE block_hash=$1", + ) + .bind(&block) + .fetch_optional(&f.pool) + .await? + .as_deref() + == Some("confirmed")) + }, + ) + .await?; + f.quiesce().await?; + f.integrity().await?; + let arrivals = node.arrivals(); + let offers = arrivals.iter().filter(|hash| **hash == block).count(); + ensure!( + offers == 1, + "the found block was offered {offers} times: {arrivals:?}" + ); + let offered: BTreeSet<_> = arrivals.iter().collect(); + ensure!( + offered.len() == arrivals.len(), + "a block was offered to the node more than once: {arrivals:?}" + ); + let unconfirmed = waits + .iter() + .filter(|(_, outcome, _)| outcome != "confirmed") + .count(); + eprintln!( + "b529 drill: block {block} offered after a {} us standby wait over a {} ms lagging link, \ + replication cut at its arrival, landed once {:.1} s after promotion; {} submitblock calls; \ + {} candidate rows lost with the gap ({} of them offered unconfirmed); logged waits: {waits:?}; \ + timeline: {}", + confirmed.2, + DRILL_LAG.as_millis(), + landed_after.as_secs_f64(), + arrivals.len(), + lost_candidates.len(), + unconfirmed, + timeline.render(), + ); + standby.close().await; + Ok(()) +} diff --git a/crates/qbit-prism-server/tests/support/live_private_postgres.rs b/crates/qbit-prism-server/tests/support/live_private_postgres.rs new file mode 100644 index 000000000..00db4e41b --- /dev/null +++ b/crates/qbit-prism-server/tests/support/live_private_postgres.rs @@ -0,0 +1,181 @@ +//! A disposable PostgreSQL 16 cluster a live scenario owns and can break: +//! its data directory on a filesystem the scenario controls (#575 disk +//! exhaustion) or its clock under an injected offset (#575 clock jumps). Its +//! log and Unix socket live in `home`, outside the data directory. +//! +//! Any test binary can include it with `#[path]`, next to +//! `live_host_tools.rs` as module `host_tools`; it needs `anyhow`. +use super::host_tools::run; +use anyhow::{ensure, Context, Result}; +use std::{collections::BTreeMap, os::unix::fs::PermissionsExt, path::PathBuf, process::Command}; + +pub(crate) struct ClusterOptions<'a> { + /// `PRISM_TEST_PG_BIN_DIR`, from the gate. + pub bin: PathBuf, + /// An empty or absent directory for the cluster's data. + pub data: PathBuf, + /// A short directory for the log and the Unix socket. + pub home: PathBuf, + /// Extra `initdb` arguments. + pub initdb: &'a [&'a str], + /// Extra `postgres` options, after the durable defaults. + pub settings: &'a str, + /// Environment for the postmaster and every backend it forks. + pub env: Vec<(String, String)>, +} + +pub(crate) struct PrivateCluster { + bin: PathBuf, + data: PathBuf, + log: PathBuf, + socket: PathBuf, + settings: String, + env: Vec<(String, String)>, + port: u16, + /// Holds `port` until the first start binds it, so no concurrent socket + /// takes it meanwhile (#533). A restart reuses the port unreserved. + reservation: Option, + user: String, +} + +impl PrivateCluster { + /// `initdb` and start, with trust authentication on loopback only. + pub(crate) fn start(options: ClusterOptions<'_>) -> Result { + if !options.data.exists() { + std::fs::create_dir(&options.data)?; + } + std::fs::set_permissions(&options.data, std::fs::Permissions::from_mode(0o700))?; + let user = String::from_utf8(Command::new("id").arg("-un").output()?.stdout)? + .trim() + .to_owned(); + let mut initdb = vec![ + "-D", + options.data.to_str().context("data path")?, + "-A", + "trust", + "--no-locale", + "-E", + "UTF8", + ]; + initdb.extend_from_slice(options.initdb); + run(&options.bin.join("initdb"), &initdb)?; + let reservation = std::net::TcpListener::bind("127.0.0.1:0")?; + let mut cluster = Self { + log: options.home.join("postgresql.log"), + socket: options.home, + settings: options.settings.to_owned(), + env: options.env, + bin: options.bin, + data: options.data, + port: reservation.local_addr()?.port(), + reservation: Some(reservation), + user, + }; + cluster.pg_start()?; + Ok(cluster) + } + + fn pg_ctl(&self, args: &[&str]) -> Result { + let mut all = vec!["-D", self.data.to_str().context("data path")?, "-t", "60"]; + all.extend_from_slice(args); + Ok(Command::new(self.bin.join("pg_ctl")) + .args(&all) + .envs(self.env.iter().map(|(name, value)| (name, value))) + .output()?) + } + + fn pg_start(&mut self) -> Result<()> { + self.reservation = None; + let options = format!( + "-h 127.0.0.1 -p {} -k {} -c fsync=on -c full_page_writes=on -c synchronous_commit=on -c max_connections=200 {}", + self.port, + self.socket.display(), + self.settings + ); + let output = self.pg_ctl(&[ + "-l", + self.log.to_str().context("log path")?, + "-o", + &options, + "-w", + "start", + ])?; + ensure!( + output.status.success(), + "pg_ctl start failed: {} {}\n{}", + String::from_utf8_lossy(&output.stdout), + String::from_utf8_lossy(&output.stderr), + self.diagnostics() + ); + Ok(()) + } + + pub(crate) fn url(&self) -> String { + format!( + "postgresql://{}@127.0.0.1:{}/postgres", + self.user, self.port + ) + } + + pub(crate) fn running(&self) -> Result { + Ok(self.pg_ctl(&["status"])?.status.success()) + } + + /// Start the cluster if its postmaster is gone; whether it had to be. + pub(crate) fn ensure_running(&mut self) -> Result { + if self.running()? { + return Ok(false); + } + self.pg_start()?; + Ok(true) + } + + /// The log's PANIC, FATAL and ERROR lines, grouped by message with + /// quoted names and numbers removed. + pub(crate) fn errors(&self) -> String { + let log = std::fs::read_to_string(&self.log).unwrap_or_default(); + let mut kinds: BTreeMap = BTreeMap::new(); + for line in log.lines() { + let Some(level) = ["PANIC", "FATAL", "ERROR"] + .into_iter() + .find(|level| line.contains(&format!("{level}:"))) + else { + continue; + }; + let message = line + .split_once(&format!("{level}:")) + .map_or(line, |(_, message)| message) + .trim(); + let shape: String = message + .split('"') + .enumerate() + .map(|(index, part)| if index % 2 == 1 { "\"…\"" } else { part }) + .collect(); + let shape: String = shape.chars().filter(|c| !c.is_ascii_digit()).collect(); + *kinds.entry(format!("{level}: {shape}")).or_default() += 1; + } + kinds + .iter() + .map(|(kind, count)| format!("{count:>6} {kind}")) + .collect::>() + .join("\n") + } + + pub(crate) fn diagnostics(&self) -> String { + let log = std::fs::read_to_string(&self.log) + .unwrap_or_else(|error| format!("unreadable: {error}")); + let tail: Vec<_> = log.lines().rev().take(30).collect(); + let mut report = String::from("--- private PostgreSQL log (last 30 lines, newest first)\n"); + for line in tail { + report.push_str(line); + report.push('\n'); + } + report + } +} + +impl Drop for PrivateCluster { + fn drop(&mut self) { + let _ = self.pg_ctl(&["-m", "immediate", "-w", "stop"]); + } +} diff --git a/crates/qbit-prism-server/tests/support/live_recipient_boundary.rs b/crates/qbit-prism-server/tests/support/live_recipient_boundary.rs new file mode 100644 index 000000000..cd2b37052 --- /dev/null +++ b/crates/qbit-prism-server/tests/support/live_recipient_boundary.rs @@ -0,0 +1,379 @@ +//! #474 A: 1,001 payable recipients across the 1,000-recipient fanout chunk +//! boundary, from accepted work through the spend of an output of each +//! chunk. +//! +//! How the accepted work arrives is part of the claim, so it is counted by +//! source. Most of it is seeded: before any server starts, one share per +//! seeded payee is appended through the production ledger append +//! (`Ledger::append`, writer `SEED_WRITER`), at a weight chosen here. The +//! rest is submitted: `SUBMITTING` payees open real Stratum sessions, on +//! both servers, and submit shares at the seeded unit weight, and one of +//! them finds the block. Two submitting payees are also seeded, so one +//! account combines both sources. This is not 1,001 live mining sessions, +//! and the summary does not say it is. +//! +//! The payout is then recomputed from the ledger and compared with the +//! chain, the ledger's accounts and the signed audit bundle exactly as in the +//! weighted scenarios, and the owning wallets spend the smallest output of +//! each fanout chunk. +use super::*; +use qbit_prism::AcceptedShare; +use qbit_prism_server::ledger::Ledger; + +/// Payable payout addresses: one more than a fanout chunk holds. +const RECIPIENTS: usize = MAX_FANOUT_RECIPIENTS + 1; +/// Payees `0..SUBMITTING` submit over Stratum; the rest are seeded only. +const SUBMITTING: usize = 4; +/// Payees `SUBMITTING - BOTH..SUBMITTING` also have a seeded share. +const BOTH: usize = 2; +/// Wallets the 1,001 addresses are generated by, round robin. +const WALLETS: usize = 16; +/// The ledger writer of the seeded shares. +const SEED_WRITER: &str = "seed-474"; +/// Payees weigh 1 to `MAX_UNITS` units, so amounts differ up to fourfold. +const MAX_UNITS: u64 = 4; +/// The payout floor for this case: at ~2,500 units of weight over a regtest +/// coinbase every payee's amount stays several times above it. +const BOUNDARY_FLOOR_BITS: u64 = 100_000; + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +#[ignore = "nightly: 1,001 recipients across two fanout chunks; run with --ignored"] +async fn nightly_1001_recipients_cross_the_fanout_chunk_boundary_to_spent_outputs() -> Result<()> { + let Some(mut fixture) = Fixture::open_with_servers(true, false).await? else { + return Ok(()); + }; + let started = Instant::now(); + let result = run(&mut fixture).await; + if result.is_err() { + eprintln!("{}", fixture.diagnostics()); + } + let cleanup = fixture.cleanup().await; + eprintln!( + "live recipient boundary: {:.1}s wall clock including cleanup", + started.elapsed().as_secs_f64() + ); + result.and(cleanup) +} + +async fn run(fixture: &mut Fixture) -> Result<()> { + let started = Instant::now(); + let ramp_address = harden_chain(fixture).await?; + let template = fixture + .rpc("getblocktemplate", json!([{"rules": ["segwit"]}])) + .await?; + let network = block_weight(template["bits"].as_str().context("template bits missing")?)?; + // A share records the height of the tip its job was built on, the + // template's parent, as the production submit path does. + let parent_height = template["height"] + .as_u64() + .context("template height missing")? + .checked_sub(1) + .context("template at height zero")?; + let ntime = u32::try_from(template["curtime"].as_u64().context("curtime missing")?)?; + let policy = Policy { + ctv: true, + floor: BOUNDARY_FLOOR_BITS, + pool: pool_identity(fixture).await?, + }; + let payees = Payees::create(fixture, RECIPIENTS, WALLETS).await?; + let chain_ready = started.elapsed(); + + // Unequal weights in whole units, the unit being one share at the + // difficulty the submitting sessions use. All of it fits in half the + // window, so every payee is in the found block's window. + let mut rng = StdRng::seed_from_u64(0x0474_1001); + let units: Vec = (0..RECIPIENTS) + .map(|_| rng.gen_range(1..=MAX_UNITS)) + .collect(); + let total_units = units.iter().sum::() + BOTH as u64 + 1; + let window = network as f64 * qbit_prism::PRISM_WINDOW_MULTIPLIER as f64; + let difficulty = (window / 2.0 / total_units as f64 / share_weight(1.0)? as f64).clamp( + MIN_DIFFICULTY, + network as f64 / 2.0 / share_weight(1.0)? as f64, + ); + let unit = share_weight(difficulty)?; + ensure!( + u128::from(total_units) * unit < network * qbit_prism::PRISM_WINDOW_MULTIPLIER, + "the recipients' work does not fit one window" + ); + + // Seeded work, through the production append, before the servers start + // and load the window. + let seeding = Instant::now(); + let seeded = seed( + fixture, + &payees.ids, + &units, + unit, + network, + parent_height, + ntime, + ) + .await?; + let seeding = seeding.elapsed(); + + start_servers(fixture, &policy).await?; + let mut sessions = Vec::with_capacity(SUBMITTING); + for payee in 0..SUBMITTING { + let username = format!("{}.boundary", payees.ids[payee].recipient); + sessions.push(Session::open(fixture.stratum[payee % 2], username, difficulty).await?); + } + let mut submitted: HashMap = HashMap::new(); + let mut stale_retries = 0; + let mut tasks = tokio::task::JoinSet::new(); + for (payee, mut session) in sessions.into_iter().enumerate() { + let count = units[payee]; + tasks.spawn(async move { + let mut shares = Vec::new(); + for _ in 0..count { + shares.push(session.submit(false).await?); + } + Ok::<_, anyhow::Error>((payee, session, shares)) + }); + } + let mut sessions: Vec> = (0..SUBMITTING).map(|_| None).collect(); + while let Some(joined) = tasks.join_next().await { + let (payee, session, shares) = joined??; + sessions[payee] = Some(session); + for share in shares { + stale_retries += share.stale_retries; + ensure!( + submitted.insert(share.share_id, payee).is_none(), + "a share id was acknowledged twice" + ); + } + } + // Let the reanchor publish work over the submitted shares first. + tokio::time::sleep(Duration::from_millis(1_600)).await; + let finder = sessions[0].as_mut().context("finder missing")?; + let block = find_block(fixture, finder, "boundary").await?; + stale_retries += block.stale_retries; + ensure!( + submitted.insert(block.share_id.clone(), 0).is_none(), + "the block share id was acknowledged twice" + ); + drop(sessions); + fixture.quiesce().await?; + let found = [Found { + hash: block.hash, + share_id: block.share_id, + }]; + + let shares = ledger_shares(fixture).await?; + let sources = check_sources( + &shares, + &seeded, + &submitted, + &payees.ids, + unit, + parent_height, + )?; + let verifying = Instant::now(); + let verified = + verify_payouts(fixture, &ramp_address, &found, &shares, &policy, &payees).await?; + let verifying = verifying.elapsed(); + + let (window, settled) = verified.expected.first().context("no block verified")?; + let payable = settled + .accounts + .values() + .filter(|account| account.onchain > 0) + .count(); + let chunks: Vec = settled.chunks.iter().map(Vec::len).collect(); + ensure!( + window.weights.len() == RECIPIENTS && !window.cut, + "the window holds {} of the {RECIPIENTS} recipients (cut: {})", + window.weights.len(), + window.cut + ); + ensure!( + payable == RECIPIENTS && settled.accounts.values().all(|account| account.carry == 0), + "{payable} of {RECIPIENTS} recipients were payable" + ); + ensure!( + chunks.len() == 2 && chunks[0] == MAX_FANOUT_RECIPIENTS, + "fanout chunks {chunks:?} do not cross the {MAX_FANOUT_RECIPIENTS}-recipient boundary" + ); + // One output of each chunk, and one direct payee output if any. + let direct_payees = settled + .direct + .keys() + .filter(|script| payees.owners.contains_key(*script)) + .count(); + ensure!( + verified.spent == chunks.len() + usize::from(direct_payees > 0), + "{} outputs spent for {} chunks and {direct_payees} direct payees", + verified.spent, + chunks.len() + ); + let amounts: Vec = settled + .accounts + .values() + .map(|account| account.onchain) + .collect(); + eprintln!( + "live recipient boundary: {RECIPIENTS} payable recipients in {WALLETS} wallets; accepted work: \ + {} seeded shares for {} recipients through Ledger::append (writer {SEED_WRITER}), {} shares \ + ({stale_retries} stale retries) submitted over Stratum by {} recipients in {SUBMITTING} live \ + sessions on both servers, {} recipients with both; one block, window of {} shares, not cut; \ + {payable} recipients paid {}..{} bits, direct outputs {}, fanout chunks {chunks:?}, {} bit pool \ + fee; the signed audit bundle matched the model; {} payee outputs (one per chunk) spent by {} \ + owning wallets; wall clock: chain ramp and wallets {:.1}s, seeding {:.1}s, verification, \ + maturity and spend {:.1}s ({})", + sources.seeded_shares, + sources.seeded_recipients, + sources.submitted_shares, + sources.submitted_recipients, + sources.both, + window.shares, + amounts.iter().min().context("no amounts")?, + amounts.iter().max().context("no amounts")?, + settled.direct.len(), + settled.pool_fee, + verified.spent, + verified.spending_wallets, + chain_ready.as_secs_f64(), + seeding.as_secs_f64(), + verifying.as_secs_f64(), + verified.phases, + ); + Ok(()) +} + +/// Appends one seeded share for each payee from `SUBMITTING - BOTH` on, +/// weighing its units (one unit for the payees that also submit), and +/// returns the seeded share ids with the payee each credits. +async fn seed( + fixture: &Fixture, + payees: &[Identity], + units: &[u64], + unit: u128, + network: u128, + parent_height: u64, + ntime: u32, +) -> Result> { + let ledger = + Ledger::connect_tool(&fixture.database_url, SEED_WRITER.into(), 2, true, None).await?; + let result = async { + // Old enough that every seeded job precedes the found block's. + let issued = i64::try_from( + std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH)? + .as_millis(), + )? - 600_000; + let mut seeded = HashMap::new(); + for payee in SUBMITTING - BOTH..payees.len() { + let identity = &payees[payee]; + let weight = if payee < SUBMITTING { + unit + } else { + unit * u128::from(units[payee]) + }; + let offset = i64::try_from(payee)?; + let share = AcceptedShare { + share_seq: 0, + share_id: format!("{SEED_WRITER}:{payee}"), + miner_id: identity.recipient.clone(), + order_key: identity.order_key.clone(), + p2mr_program_hex: identity.program.clone(), + share_difficulty: weight, + network_difficulty: network, + template_height: parent_height, + job_id: SEED_WRITER.into(), + job_issued_at_ms: issued + offset, + accepted_at_ms: issued + offset + 1, + ntime, + credit_policy: None, + }; + seeded.insert(share.share_id.clone(), (payee, weight)); + ledger.append(share, None).await?; + } + Ok::<_, anyhow::Error>(seeded) + } + .await; + ledger.pool.close().await; + result +} + +/// How the ledger's accepted work arrived. +struct Sources { + seeded_shares: usize, + seeded_recipients: usize, + submitted_shares: usize, + submitted_recipients: usize, + both: usize, +} + +/// The ledger holds exactly the seeded shares, under the seed writer, and +/// the shares the servers acknowledged, under the server each session used, +/// each credited to its payee at the unit or seeded weight, and every one +/// built on the same tip, `parent_height`: the seeded shares record the +/// height the servers record for the submitted ones. +fn check_sources( + shares: &[LedgerShare], + seeded: &HashMap, + submitted: &HashMap, + payees: &[Identity], + unit: u128, + parent_height: u64, +) -> Result { + ensure!( + shares.len() == seeded.len() + submitted.len(), + "ledger holds {} accepted shares; {} were seeded and {} acknowledged", + shares.len(), + seeded.len(), + submitted.len() + ); + let mut from_seed = BTreeSet::new(); + let mut from_stratum = BTreeSet::new(); + for share in shares { + let (payee, weight, writer) = if let Some((payee, weight)) = seeded.get(&share.share_id) { + from_seed.insert(*payee); + (*payee, *weight, SEED_WRITER.to_owned()) + } else { + let payee = *submitted.get(&share.share_id).with_context(|| { + format!("ledger share {} was never acknowledged", share.share_id) + })?; + from_stratum.insert(payee); + (payee, unit, format!("live-{}", payee % 2)) + }; + let identity = &payees[payee]; + ensure!( + share.miner == identity.recipient + && share.order_key == identity.order_key + && share.program == identity.program + && share.weight == weight + && share.writer == writer + && share.credit_policy.is_none(), + "share {} credited to {} at {} by {}, not {} at {weight} by {writer}", + share.share_id, + share.miner, + share.weight, + share.writer, + identity.recipient + ); + ensure!( + u64::try_from(share.template_height).ok() == Some(parent_height), + "share {} records template height {}, the found block's parent is {parent_height}", + share.share_id, + share.template_height + ); + } + let seeded_recipients = from_seed.len(); + let submitted_recipients = from_stratum.len(); + let both = from_seed.intersection(&from_stratum).count(); + ensure!( + submitted_recipients == SUBMITTING + && both == BOTH + && seeded_recipients + submitted_recipients - both == RECIPIENTS, + "work came from {seeded_recipients} seeded and {submitted_recipients} submitting \ + recipients, {both} both" + ); + Ok(Sources { + seeded_shares: seeded.len(), + seeded_recipients, + submitted_shares: submitted.len(), + submitted_recipients, + both, + }) +} diff --git a/crates/qbit-prism-server/tests/support/live_session_load.rs b/crates/qbit-prism-server/tests/support/live_session_load.rs new file mode 100644 index 000000000..ce6315918 --- /dev/null +++ b/crates/qbit-prism-server/tests/support/live_session_load.rs @@ -0,0 +1,1723 @@ +//! Session load for the live suite, at #487 L4's session counts (#553). +//! +//! [`SessionLoad`] keeps `sessions` Stratum connections open against a +//! fixture's frontends, as a pool's miners would. Each session has a home +//! frontend and fails over to the next ready one when its connection closes, +//! and together they offer a fixed share rate. Every proof is a share that +//! misses its job's block target, so the load never finds a block and never +//! changes which own blocks a scenario has. The servers run with +//! [`load_server_env`]: #575's share-only settings, which make such a share +//! possible on regtest, room for every session, and the production reanchor. +//! A load share is credited at about 2^-32, the easiest share target there +//! is, but regtest's block difficulty is only about 2^-31, which is what an +//! own block's block-only proof is credited at. So a load share weighs about +//! half an own block, ~15 of them fill the payout window (8x the network +//! difficulty), and the load takes most of every own block's reward. A +//! scenario whose assertions depend on which own shares are in the window +//! pauses the load's shares around its own blocks +//! ([`SessionLoad::share_pause`]); the sessions stay connected and keep +//! taking work. What the load does exercise is the share append under +//! `ORDER_LOCK`, the database pool, and the notify fan-out to every session on +//! every tip. +//! +//! The load records what each session was told and what it was answered. It +//! stamps every tip of every watched node with a `waitfornewblock` long-poll +//! given the last tip, and it samples each frontend's `/healthz`. +//! [`LoadRecord::check`] then holds the run to three things: +//! - Every acknowledged share is a durable ledger row, no refused share is one, +//! and every durable row was acknowledged or has an unknown answer. +//! - No session earns a refusal a correct share-only miner never earns: a +//! duplicate, low-difficulty or malformed submit. +//! - A session on a frontend has work on each tip of the frontend's node +//! within [`DELIVERY_BOUND`], provided the tip lasted that long, the node +//! answered throughout, and the frontend reported itself ready when the +//! bound expired. That is #413's condition, and the two-node scenarios' +//! notify bound. +//! +//! It reports #481's time to usable work per frontend and per tip: from the +//! node's stamp to each session's first notify on that parent. A tip replaced +//! before a session was served is counted as replaced, never as zero. +use super::dense_soak_tests::setting; +use super::share_client::{solve_share, Answer, SHARE_ONLY_SETTINGS}; +use super::*; +use rand::{rngs::StdRng, Rng, SeedableRng}; +use std::{ + collections::{BTreeMap, HashMap, HashSet}, + sync::{ + atomic::{AtomicBool, AtomicUsize, Ordering}, + Arc, + }, +}; +use tokio::{ + io::Lines, + net::{ + tcp::{OwnedReadHalf, OwnedWriteHalf}, + TcpStream, + }, + sync::watch, + task::JoinHandle, + time::MissedTickBehavior, +}; + +/// How long a session on a ready frontend may wait for work on a new tip: +/// the native current-work-gap warning's 15 s, as the two-node scenarios' +/// `NOTIFY_BOUND` (#413 held delivery for 307 s). +pub(super) const DELIVERY_BOUND: Duration = Duration::from_secs(15); + +/// Connections a frontend keeps free beyond the load for a scenario's own +/// clients (its highdiff watcher and miners, a block client). +const SCENARIO_CONNECTIONS: usize = 100; + +/// The server's default `PRISM_STRATUM_MAX_CONNECTIONS`, which a smaller +/// load keeps: the pending initial-job limit (128) may not exceed it. +const DEFAULT_MAX_CONNECTIONS: usize = 384; + +/// A request's answer bound: the server's share commit timeout plus grace +/// (#577), as the share client's. +const ANSWER_BOUND: Duration = super::share_client::ANSWER; + +/// Subscribe, authorize and first work: the server bounds session allocation +/// by its 30 s initial-job timeout (#562). +const HANDSHAKE_BOUND: Duration = Duration::from_secs(35); + +/// How long [`SessionLoad::stop`] waits for answers still outstanding. +const DRAIN_BOUND: Duration = Duration::from_secs(30); + +/// Refusals a session that submits only valid, distinct shares on work it +/// was given can earn: a race with the tip or the payout revision, a job +/// the session no longer holds, or a backend refusal +/// (`qbit_prism_load::classify`'s expected and backend classes). Any other, +/// such as `duplicate-share`, `low-difficulty`, `unauthorized-worker` or an +/// `invalid-*`, fails the load, as does one nobody classified yet. +const EXPECTED_REASONS: [&str; 7] = [ + "stale-job", + "unknown-job", + "pool-closed", + "backend-rpc-unavailable", + "backend-database-unavailable", + "ledger-confirmation-failed", + "internal-error", +]; +/// A refusal whose COMMIT outcome the server does not know: the share may +/// still land, so it counts as unanswered (#324). +const OUTCOME_UNKNOWN: &str = "ledger-outcome-unknown"; +/// Refusals without a `reason_id` that a correct load can earn, by code +/// and exact message: the unknown-job budget, as +/// `qbit_prism_load::classify::is_unknown_job_budget` matches it. +const EXPECTED_UNTYPED: [&str; 1] = ["20 too many unknown job submissions"]; +const UNTYPED: &str = "(no reason_id)"; + +/// The payout-artifact reanchor a load run's frontends use: the server's, +/// the harness's and Compose's default, where the live fixture's is 1 s. +/// Before #598's fix, 2,000 sessions on one frontend at 1 s got no work on a +/// new tip; one nightly case keeps the fixture's 1 s as #598's guard. +const REANCHOR_SECONDS: u64 = 60; + +/// The frontend settings a load run needs: #575's share-only settings, a +/// connection limit that holds every session on one frontend, since a +/// scenario that stops one frontend moves every session to the other, and +/// the production reanchor ([`REANCHOR_SECONDS`]). +pub(super) fn load_server_env(sessions: usize) -> Vec<(String, String)> { + SHARE_ONLY_SETTINGS + .iter() + .map(|(name, value)| (name.to_string(), value.to_string())) + .chain([ + ( + "PRISM_STRATUM_MAX_CONNECTIONS".to_owned(), + (sessions + SCENARIO_CONNECTIONS) + .max(DEFAULT_MAX_CONNECTIONS) + .to_string(), + ), + ( + "PRISM_PAYOUT_ARTIFACT_REANCHOR_SECONDS".to_owned(), + REANCHOR_SECONDS.to_string(), + ), + ]) + .collect() +} + +/// What a load offers and where. +#[derive(Clone, Debug)] +pub(super) struct LoadPlan { + pub name: String, + pub sessions: usize, + /// Shares a second, over every session together. + pub rate: f64, + /// The payout address of every load session; each session's worker is + /// `load`. + pub address: String, + /// Frontend `i`'s Stratum and HTTP ports. + pub stratum: Vec, + pub api: Vec, + /// RPC ports of the watched nodes (`prismtest` credentials). + pub nodes: Vec, + /// Which watched node frontend `i` follows; `None` for none, whose tips + /// are then not measured. + pub node_of: Vec>, + pub seed: u64, +} + +impl LoadPlan { + /// `sessions` sessions offering `rate` shares a second through both of + /// the fixture's frontends, paid to `address`, both frontends on the + /// fixture's node, which is the only node watched. + pub(super) fn on_fixture( + fixture: &Fixture, + name: &str, + sessions: usize, + rate: f64, + address: String, + ) -> Self { + Self { + name: name.to_owned(), + sessions, + rate, + address, + stratum: fixture.stratum.to_vec(), + api: fixture.api.to_vec(), + nodes: vec![fixture.rpc_port], + node_of: vec![Some(0), Some(0)], + seed: 0x0553_0000 ^ sessions as u64, + } + } + + fn username(&self, index: usize) -> String { + format!("{}.load{index}", self.address) + } +} + +/// One tip a watched node took, when it took it. +#[derive(Clone, Debug)] +struct TipStamp { + node: usize, + hash: String, + at: Instant, +} + +/// One connection of one session. +#[derive(Clone, Debug)] +struct Connection { + frontend: usize, + /// Subscribed, authorized and holding work. + ready_at: Instant, + closed_at: Option, + /// Every notify's parent, in arrival order: a republication on one + /// parent is an entry of its own. + parents: Vec<(Instant, String)>, +} + +#[derive(Clone, Debug)] +struct SubmitRecord { + share_id: String, + frontend: usize, + answer: Answer, +} + +#[derive(Debug, Default)] +struct SessionLog { + connections: Vec, + submits: Vec, + connect_failures: usize, + last_failure: Option, +} + +/// State every session and poller shares. +struct Shared { + plan: LoadPlan, + /// Each frontend's latest `/healthz` answer. + ready: Vec, + /// `(when, frontend, ready)` at each change, the first sample included. + health: Mutex>, + /// Per frontend: the slowest `/healthz` answer and the timed-out samples. + health_stats: Mutex>, + tips: Mutex>, + /// `(node, from, to)`: spans in which a watched node did not answer. + unreachable: Mutex>, + /// Sessions holding work right now. + connected: AtomicUsize, + /// While set, sessions offer no shares ([`SessionLoad::share_pause`]). + paused: AtomicBool, + /// The open pause's start and the closed pauses' total. + pauses: Mutex<(Option, Duration)>, + /// Sessions waiting for a share's answer right now. + unanswered: AtomicUsize, +} + +impl Shared { + /// The first ready frontend from `home` on, in failover order. + fn pick(&self, home: usize) -> Option { + let count = self.ready.len(); + (0..count) + .map(|offset| (home + offset) % count) + .find(|frontend| self.ready[*frontend].load(Ordering::SeqCst)) + } +} + +/// A running load. [`SessionLoad::stop`] ends it and returns the record. +pub(super) struct SessionLoad { + shared: Arc, + stop: watch::Sender, + sessions: Vec>, + pollers: Vec>, + started: Instant, +} + +impl SessionLoad { + pub(super) fn start(plan: LoadPlan) -> Result { + ensure!( + plan.sessions > 0 && plan.rate.is_finite() && plan.rate > 0.0, + "a load needs sessions and a positive rate: {plan:?}" + ); + ensure!( + plan.stratum.len() == plan.api.len() && plan.stratum.len() == plan.node_of.len(), + "every frontend needs a Stratum port, an HTTP port and a node: {plan:?}" + ); + let started = Instant::now(); + let shared = Arc::new(Shared { + ready: plan.api.iter().map(|_| AtomicBool::new(false)).collect(), + health: Mutex::new(Vec::new()), + health_stats: Mutex::new(vec![(Duration::ZERO, 0); plan.api.len()]), + tips: Mutex::new(Vec::new()), + unreachable: Mutex::new(Vec::new()), + connected: AtomicUsize::new(0), + paused: AtomicBool::new(false), + pauses: Mutex::new((None, Duration::ZERO)), + unanswered: AtomicUsize::new(0), + plan, + }); + let (stop, stopped) = watch::channel(false); + let client = reqwest::Client::builder().build()?; + let mut pollers: Vec> = (0..shared.plan.api.len()) + .map(|frontend| { + tokio::spawn(poll_health( + shared.clone(), + frontend, + client.clone(), + stopped.clone(), + )) + }) + .collect(); + for node in 0..shared.plan.nodes.len() { + pollers.push(tokio::spawn(watch_node( + shared.clone(), + node, + stopped.clone(), + ))); + } + let sessions = (0..shared.plan.sessions) + .map(|index| tokio::spawn(run_session(shared.clone(), index, stopped.clone()))) + .collect(); + Ok(Self { + shared, + stop, + sessions, + pollers, + started, + }) + } + + /// Sessions holding work right now. + pub(super) fn connected(&self) -> usize { + self.shared.connected.load(Ordering::SeqCst) + } + + /// Whether at least `fraction` of the sessions hold work, as a check + /// that outlives this borrow. + pub(super) fn holding(&self, fraction: f64) -> Arc bool + Send + Sync> { + let shared = self.shared.clone(); + let wanted = self.wanted(fraction); + Arc::new(move || shared.connected.load(Ordering::SeqCst) >= wanted) + } + + fn wanted(&self, fraction: f64) -> usize { + (self.shared.plan.sessions as f64 * fraction).ceil() as usize + } + + /// Wait until at least `fraction` of the sessions hold work. + pub(super) async fn until_connected(&self, fraction: f64, seconds: u64) -> Result<()> { + let holding = self.holding(fraction); + until( + &format!("{} load sessions holding work", self.wanted(fraction)), + seconds, + || async { Ok(holding()) }, + ) + .await + .with_context(|| format!("{} of them hold work", self.connected())) + } + + /// A switch that pauses (`true`) or resumes (`false`) every session's + /// share offers, which outlives this borrow. A paused session stays + /// connected and keeps taking work, so the delivery check still holds + /// it to every tip; only its shares stop. + pub(super) fn share_pause(&self) -> Arc { + let shared = self.shared.clone(); + Arc::new(move |paused| { + shared.paused.store(paused, Ordering::SeqCst); + let mut pauses = shared.pauses.lock().expect("pauses"); + match (paused, pauses.0) { + (true, None) => pauses.0 = Some(Instant::now()), + (false, Some(since)) => *pauses = (None, pauses.1 + since.elapsed()), + _ => {} + } + }) + } + + /// Whether no session waits for a share's answer, as a check that + /// outlives this borrow: after a pause, every share offered before it has + /// its answer, has run out its answer bound ([`ANSWER_BOUND`]), or lost + /// its connection. A share answered `ledger-outcome-unknown` or lost may + /// still commit, so a caller that needs the ledger still waits for it. + pub(super) fn answered(&self) -> Arc bool + Send + Sync> { + let shared = self.shared.clone(); + Arc::new(move || shared.unanswered.load(Ordering::SeqCst) == 0) + } + + /// Stop offering shares, wait for outstanding answers, close every + /// session and return what the load saw. + pub(super) async fn stop(self) -> LoadRecord { + let paused = { + let pauses = self.shared.pauses.lock().expect("pauses"); + pauses.1 + pauses.0.map_or(Duration::ZERO, |since| since.elapsed()) + }; + let offered_seconds = (self.started.elapsed() - paused).as_secs_f64(); + let _ = self.stop.send(true); + let mut sessions = Vec::with_capacity(self.sessions.len()); + for handle in self.sessions { + // A session task that panicked leaves an empty log, which the + // check then reports as a session that never held work. + sessions.push(handle.await.unwrap_or_default()); + } + for poller in self.pollers { + poller.abort(); + let _ = poller.await; + } + let ended = Instant::now(); + let shared = &self.shared; + let health = shared.health.lock().expect("health").clone(); + let health_stats = shared.health_stats.lock().expect("health stats").clone(); + let tips = shared.tips.lock().expect("tips").clone(); + let unreachable = shared.unreachable.lock().expect("unreachable").clone(); + LoadRecord { + plan: shared.plan.clone(), + started: self.started, + ended, + offered_seconds, + paused_seconds: paused.as_secs_f64(), + health, + health_stats, + tips, + unreachable, + sessions, + } + } +} + +/// How long a `/healthz` answer may take before the sample counts as not +/// ready. A slow answer is reported (`healthz_max_ms`, `healthz_timeouts`). +const HEALTH_TIMEOUT: Duration = Duration::from_secs(5); + +/// Sample frontend `frontend`'s `/healthz` every 250 ms, on its own task so +/// one slow frontend does not delay the other's samples. +async fn poll_health( + shared: Arc, + frontend: usize, + client: reqwest::Client, + mut stop: watch::Receiver, +) { + let mut ticker = tokio::time::interval(Duration::from_millis(250)); + ticker.set_missed_tick_behavior(MissedTickBehavior::Delay); + let url = format!("http://127.0.0.1:{}/healthz", shared.plan.api[frontend]); + let mut last: Option = None; + loop { + tokio::select! { + _ = stop.changed() => return, + _ = ticker.tick() => {} + } + let asked = Instant::now(); + let answer = client.get(&url).timeout(HEALTH_TIMEOUT).send().await; + let elapsed = asked.elapsed(); + let ready = match &answer { + Ok(response) => { + let mut stats = shared.health_stats.lock().expect("health stats"); + stats[frontend].0 = stats[frontend].0.max(elapsed); + response.status().is_success() + } + Err(error) => { + if error.is_timeout() { + shared.health_stats.lock().expect("health stats")[frontend].1 += 1; + } + false + } + }; + shared.ready[frontend].store(ready, Ordering::SeqCst); + if last != Some(ready) { + last = Some(ready); + shared + .health + .lock() + .expect("health") + .push((asked, frontend, ready)); + } + } +} + +/// A watcher call slower than this, or failed, marks its node unreachable +/// for the call's span. +const NODE_ANSWER_BOUND: Duration = Duration::from_secs(3); + +/// Stamp every tip change on watched node `node`. `waitfornewblock` is +/// given the tip last seen, so a change between two calls wakes the next +/// one at once. A stopped or restarting node is retried, not an error: the +/// scenario owns the node's life. +async fn watch_node(shared: Arc, node: usize, mut stop: watch::Receiver) { + let client = reqwest::Client::builder() + .timeout(Duration::from_secs(6)) + .build() + .expect("watcher client"); + let url = format!("http://127.0.0.1:{}/", shared.plan.nodes[node]); + let mut last: Option = None; + loop { + let (method, params) = match &last { + Some(tip) => ("waitfornewblock", json!([1000, tip])), + None => ("getbestblockhash", json!([])), + }; + let asked = Instant::now(); + let call = client + .post(&url) + .basic_auth("prismtest", Some("prismtest")) + .json(&json!({"jsonrpc":"1.0","id":"load-watch","method":method,"params":params})) + .send(); + let answer = tokio::select! { + _ = stop.changed() => return, + answer = call => answer, + }; + let at = Instant::now(); + let payload: Option = match answer { + Ok(response) => response.json().await.ok(), + Err(_) => None, + }; + let hash = payload.and_then(|payload| { + let result = &payload["result"]; + result["hash"] + .as_str() + .or_else(|| result.as_str()) + .map(str::to_owned) + }); + // A long-poll answers within its one second unless the node is + // stopped, killed or restarting: then the node could not have given + // the frontends work either. + if hash.is_none() || at - asked > NODE_ANSWER_BOUND { + shared + .unreachable + .lock() + .expect("unreachable") + .push((node, asked, at)); + } + match hash { + Some(hash) if last.as_deref() != Some(hash.as_str()) => { + shared.tips.lock().expect("tips").push(TipStamp { + node, + hash: hash.clone(), + at, + }); + last = Some(hash); + } + Some(_) => {} + None => { + tokio::select! { + _ = stop.changed() => return, + _ = tokio::time::sleep(Duration::from_millis(200)) => {} + } + } + } + } +} + +/// A subscribed, authorized connection holding work. +struct Live { + lines: Lines>, + writer: OwnedWriteHalf, + frontend: usize, + extranonce1: String, + extranonce2_size: usize, + difficulty: f64, + notify: Value, + counter: u64, + next_id: u64, + /// The request id and submit index of the share awaiting its answer. + outstanding: Option<(u64, usize, Instant)>, +} + +/// The display-order hash of a notify's previous-block field, which Stratum +/// sends with each 32-bit word byte-swapped. +fn notify_parent(notify: &Value) -> Option { + let mut wire = hex::decode(notify["params"][1].as_str()?).ok()?; + if wire.len() != 32 { + return None; + } + for word in wire.as_chunks_mut::<4>().0 { + word.reverse(); + } + wire.reverse(); + Some(hex::encode(wire)) +} + +async fn handshake(port: u16, frontend: usize, username: &str) -> Result<(Live, Connection)> { + let stream = tokio::time::timeout( + Duration::from_secs(10), + TcpStream::connect(("127.0.0.1", port)), + ) + .await + .context("Stratum connect timed out")??; + stream.set_nodelay(true)?; + let (read, mut writer) = stream.into_split(); + let mut lines = BufReader::new(read).lines(); + writer + .write_all( + format!( + "{}\n{}\n", + json!({"id":1,"method":"mining.subscribe","params":["prism-live-load"]}), + json!({"id":2,"method":"mining.authorize","params":[username,"x"]}) + ) + .as_bytes(), + ) + .await?; + let deadline = Instant::now() + HANDSHAKE_BOUND; + let (mut extranonce1, mut extranonce2_size) = (None, None); + let (mut authorized, mut difficulty, mut notify) = (false, None, None); + while !(authorized && difficulty.is_some() && notify.is_some() && extranonce1.is_some()) { + let left = deadline.saturating_duration_since(Instant::now()); + ensure!(!left.is_zero(), "no work within {HANDSHAKE_BOUND:?}"); + let line = tokio::time::timeout(left, lines.next_line()) + .await + .context("handshake timed out")?? + .context("closed during the handshake")?; + let message: Value = serde_json::from_str(&line)?; + match (message["id"].as_u64(), message["method"].as_str()) { + (Some(1), None) => { + ensure!(message["error"].is_null(), "subscribe: {message}"); + extranonce1 = message["result"][1].as_str().map(str::to_owned); + extranonce2_size = message["result"][2].as_u64(); + } + (Some(2), None) => { + ensure!(message["result"] == true, "authorize: {message}"); + authorized = true; + } + (_, Some("mining.set_difficulty")) => difficulty = message["params"][0].as_f64(), + (_, Some("mining.notify")) => notify = Some(message), + _ => {} + } + } + let ready_at = Instant::now(); + let notify = notify.expect("checked above"); + let parent = notify_parent(¬ify).context("notify parent malformed")?; + Ok(( + Live { + lines, + writer, + frontend, + extranonce1: extranonce1.expect("checked above"), + extranonce2_size: usize::try_from(extranonce2_size.context("extranonce2 size")?)?, + difficulty: difficulty.expect("checked above"), + notify, + counter: 0, + next_id: 10, + outstanding: None, + }, + Connection { + frontend, + ready_at, + closed_at: None, + parents: vec![(ready_at, parent)], + }, + )) +} + +/// One session: connect to the first ready frontend from its home, offer a +/// share each interval while it holds work, answer nothing twice, and +/// reconnect when the server closes it. +async fn run_session( + shared: Arc, + index: usize, + mut stop: watch::Receiver, +) -> SessionLog { + let plan = &shared.plan; + let mut log = SessionLog::default(); + let username = plan.username(index); + let home = index % plan.stratum.len(); + let interval = Duration::from_secs_f64(plan.sessions as f64 / plan.rate); + let mut rng = StdRng::seed_from_u64(plan.seed ^ index as u64); + let mut ticker = tokio::time::interval_at( + tokio::time::Instant::now() + interval.mul_f64(rng.gen::()), + interval, + ); + ticker.set_missed_tick_behavior(MissedTickBehavior::Skip); + // Spread the first connections from the first ready frontend, so the + // frontends' initial-job admission (128 pending by default) is not the + // thing measured: a case may start its servers after the load. + while shared.pick(home).is_none() { + tokio::select! { + _ = stop.changed() => return log, + _ = tokio::time::sleep(Duration::from_millis(250)) => {} + } + } + let spread = Duration::from_millis(u64::try_from(index).unwrap_or(0) * 5 % 10_000); + tokio::select! { + _ = stop.changed() => return log, + _ = tokio::time::sleep(spread) => {} + } + let mut live: Option = None; + let mut stopping: Option = None; + loop { + // A dropped sender (the test failed before stopping the load) stops + // the session too. + if stopping.is_none() && (*stop.borrow() || stop.has_changed().is_err()) { + stopping = Some(Instant::now()); + } + if let Some(since) = stopping { + let drained = live.as_ref().is_none_or(|live| live.outstanding.is_none()); + if drained || since.elapsed() > DRAIN_BOUND { + break; + } + } + let Some(current) = live.as_mut() else { + if stopping.is_some() { + break; + } + match shared.pick(home) { + None => { + tokio::select! { + _ = stop.changed() => {} + _ = tokio::time::sleep(Duration::from_millis(250)) => {} + } + } + Some(frontend) => { + let port = plan.stratum[frontend]; + let attempt = tokio::select! { + _ = stop.changed() => None, + attempt = handshake(port, frontend, &username) => Some(attempt), + }; + match attempt { + None => {} + Some(Ok((connection, record))) => { + log.connections.push(record); + shared.connected.fetch_add(1, Ordering::SeqCst); + live = Some(connection); + } + Some(Err(error)) => { + log.connect_failures += 1; + log.last_failure = Some(format!("frontend {frontend}: {error:#}")); + tokio::select! { + _ = stop.changed() => {} + _ = tokio::time::sleep(Duration::from_millis(500)) => {} + } + } + } + } + } + continue; + }; + // Whether `unanswered` counts this session: while it waits for an + // answer, and from before it checks the pause to the end of a submit, + // so a pause never misses a share already on its way. + let mut counted = current.outstanding.is_some(); + let wait = if stopping.is_some() { + DRAIN_BOUND + } else { + Duration::from_secs(3600) + }; + let event = tokio::select! { + _ = stop.changed(), if stopping.is_none() => Event::Stop, + _ = ticker.tick(), if stopping.is_none() => Event::Tick, + line = tokio::time::timeout(wait, current.lines.next_line()) => match line { + Ok(Ok(Some(line))) => Event::Line(line), + Ok(Ok(None)) => Event::Closed("closed by the server".into()), + Ok(Err(error)) => Event::Closed(format!("read: {error}")), + Err(_) => Event::Stop, + }, + }; + let failure = match event { + Event::Stop => None, + Event::Tick => { + if let Some((_, submit, sent)) = current.outstanding { + if sent.elapsed() > ANSWER_BOUND { + log.submits[submit].answer = Answer::TimedOut; + current.outstanding = None; + } + } + if current.outstanding.is_none() && !counted { + shared.unanswered.fetch_add(1, Ordering::SeqCst); + counted = true; + } + if current.outstanding.is_none() && !shared.paused.load(Ordering::SeqCst) { + submit(current, &username, &mut log) + .await + .err() + .map(|error| format!("submit: {error:#}")) + } else { + None + } + } + Event::Line(line) => receive(current, &line, &mut log) + .err() + .map(|error| format!("{error:#}")), + Event::Closed(why) => Some(why), + }; + if let Some(why) = failure { + close(&shared, &mut live, &mut log, &why); + } + match ( + counted, + live.as_ref().is_some_and(|live| live.outstanding.is_some()), + ) { + (false, true) => { + shared.unanswered.fetch_add(1, Ordering::SeqCst); + } + (true, false) => { + shared.unanswered.fetch_sub(1, Ordering::SeqCst); + } + _ => {} + } + } + if live.is_some() { + close(&shared, &mut live, &mut log, "load stopped"); + } + log +} + +enum Event { + Stop, + Tick, + Line(String), + Closed(String), +} + +/// Solve the latest job for a share that is not a block and send it. +async fn submit(live: &mut Live, username: &str, log: &mut SessionLog) -> Result<()> { + live.counter += 1; + let solution = solve_share( + &live.notify, + &live.extranonce1, + live.extranonce2_size, + live.difficulty, + live.counter, + )?; + live.next_id += 1; + let id = live.next_id; + let request = json!({"id":id,"method":"mining.submit","params":[ + username, live.notify["params"][0], solution.extranonce2, solution.ntime, solution.nonce, + ]}); + let sent = Instant::now(); + log.submits.push(SubmitRecord { + share_id: format!("{username}:{}", solution.hash), + frontend: live.frontend, + answer: Answer::Lost("never written".into()), + }); + let submit = log.submits.len() - 1; + live.writer + .write_all(format!("{request}\n").as_bytes()) + .await?; + log.submits[submit].answer = Answer::TimedOut; + live.outstanding = Some((id, submit, sent)); + Ok(()) +} + +fn receive(live: &mut Live, line: &str, log: &mut SessionLog) -> Result<()> { + let message: Value = serde_json::from_str(line)?; + match message["method"].as_str() { + Some("mining.notify") => { + let parent = notify_parent(&message).context("notify parent malformed")?; + // Every notify, not only a changed parent: after a tip the + // session never got, work on a parent it held before (a reorg + // back) is new work for that tip. + let connection = log.connections.last_mut().context("no connection")?; + connection.parents.push((Instant::now(), parent)); + live.notify = message; + } + Some("mining.set_difficulty") => { + live.difficulty = message["params"][0] + .as_f64() + .context("difficulty missing")?; + } + Some(_) => {} + None => { + if let Some((id, submit, _)) = live.outstanding { + if message["id"].as_u64() == Some(id) { + log.submits[submit].answer = + if message["result"] == true && message["error"].is_null() { + Answer::Accepted + } else { + Answer::Rejected(message) + }; + live.outstanding = None; + } + } + } + } + Ok(()) +} + +fn close(shared: &Shared, live: &mut Option, log: &mut SessionLog, why: &str) { + let Some(connection) = live.take() else { + return; + }; + if let Some((_, submit, _)) = connection.outstanding { + log.submits[submit].answer = Answer::Lost(why.to_owned()); + } + if let Some(record) = log.connections.last_mut() { + record.closed_at = Some(Instant::now()); + } + shared.connected.fetch_sub(1, Ordering::SeqCst); +} + +/// Everything a stopped load saw. +pub(super) struct LoadRecord { + plan: LoadPlan, + started: Instant, + ended: Instant, + /// Seconds the load offered shares: its run, less its pauses. + offered_seconds: f64, + paused_seconds: f64, + health: Vec<(Instant, usize, bool)>, + health_stats: Vec<(Duration, usize)>, + tips: Vec, + unreachable: Vec<(usize, Instant, Instant)>, + sessions: Vec, +} + +/// One tip as one frontend's sessions received it. +#[derive(Clone, Debug, Default, PartialEq)] +pub(super) struct Delivery { + pub frontend: usize, + pub node: usize, + pub hash: String, + /// Seconds from the load's start to the node's stamp. + pub stamped: f64, + /// Seconds until the node's next tip; `None` if it lasted to the end. + pub lifetime: Option, + /// The frontend's node answered from the stamp through the bound after + /// it, and the frontend reported itself ready when the bound expired. + pub gated: bool, + /// Sessions holding work on this frontend at the stamp. + pub eligible: usize, + pub served: usize, + /// Served before the watcher's stamp, counted as zero seconds. + pub ahead_of_stamp: usize, + /// Not served before the node's next tip replaced this one. + pub replaced: usize, + /// Closed before being served, or the load ended first. + pub gone: usize, + /// Not served within the bound although the tip lasted: a violation + /// when `gated`. + pub late: usize, + /// Time to usable work of each served session, seconds, sorted. + pub seconds: Vec, +} + +impl Delivery { + fn quantile(&self, quantile: f64) -> Option { + let last = self.seconds.len().checked_sub(1)?; + Some(self.seconds[((last as f64) * quantile).round() as usize]) + } + + /// When every eligible session had work, the time the last one got it. + fn all_sessions(&self) -> Option { + (self.eligible > 0 && self.served == self.eligible) + .then(|| self.seconds.last().copied()) + .flatten() + } + + /// Gated and lasting the bound: a tip that every session had to get in + /// time, where a shorter one can only count sessions as replaced. + fn held(&self) -> bool { + self.gated + && self + .lifetime + .is_none_or(|lifetime| lifetime >= DELIVERY_BOUND.as_secs_f64()) + } + + fn json(&self) -> Value { + json!({ + "frontend": self.frontend, + "node": self.node, + "tip": self.hash, + "stamped_s": round(self.stamped), + "lifetime_s": self.lifetime.map(round), + "gated": self.gated, + "eligible": self.eligible, + "served": self.served, + "ahead_of_stamp": self.ahead_of_stamp, + "replaced": self.replaced, + "gone": self.gone, + "late": self.late, + "p50_s": self.quantile(0.5).map(round), + "p99_s": self.quantile(0.99).map(round), + "max_s": self.seconds.last().copied().map(round), + "all_sessions_s": self.all_sessions().map(round), + }) + } +} + +fn round(value: f64) -> f64 { + (value * 1000.0).round() / 1000.0 +} + +/// When `connection` got work on `stamp`'s tip, which the node took after +/// its previous tip at `previous`. Ahead of the stamp only while the +/// connection still held that work at the stamp: of its last run of +/// notifies on the tip before the stamp, the first after `previous` (the +/// frontend saw the tip before the watcher). A notify on the tip that one +/// on another parent followed was for an earlier time this block was the +/// tip, or a republication before the frontend saw the next one. Otherwise +/// the first notify on the tip after the stamp. +fn served_at( + connection: &Connection, + stamp: &TipStamp, + previous: Option, +) -> Option { + let before = connection.parents.partition_point(|(at, _)| *at < stamp.at); + let held = connection.parents[..before] + .iter() + .rev() + .take_while(|(_, parent)| parent == &stamp.hash) + .map(|(at, _)| *at) + .filter(|at| previous.is_none_or(|previous| *at > previous)) + .last(); + held.or_else(|| { + connection.parents[before..] + .iter() + .find(|(_, parent)| parent == &stamp.hash) + .map(|(at, _)| *at) + }) +} + +/// How the load's submissions ended and what the ledger holds. +#[derive(Debug, Default, PartialEq)] +pub(super) struct ShareCounts { + pub submitted: usize, + pub accepted: usize, + pub rejected: BTreeMap, + /// Timed out or lost with the connection: the server may hold either. + pub unknown: usize, + pub unknown_durable: usize, +} + +/// Whether a correct load can earn a refusal counted under `reason`: an +/// allow-list, so an impossible refusal and one this check does not know +/// both fail. +fn expected_refusal(reason: &str) -> bool { + if let Some(refusal) = reason.strip_prefix(UNTYPED) { + return EXPECTED_UNTYPED.contains(&refusal.trim_start()); + } + EXPECTED_REASONS.contains(&reason) +} + +/// Hold `submits` to the durable ledger rows `durable`: every accepted +/// share is durable, no refused share is, and a durable share was accepted +/// or has an unknown answer. +fn reconcile(submits: &[&SubmitRecord], durable: &HashSet) -> Result { + let mut counts = ShareCounts { + submitted: submits.len(), + ..ShareCounts::default() + }; + let mut answered: HashMap<&str, &Answer> = HashMap::new(); + for record in submits { + ensure!( + answered + .insert(record.share_id.as_str(), &record.answer) + .is_none(), + "share {} was submitted twice", + record.share_id + ); + match &record.answer { + Answer::Accepted => { + counts.accepted += 1; + ensure!( + durable.contains(&record.share_id), + "acknowledged share {} is not in the ledger", + record.share_id + ); + } + Answer::Rejected(_) if record.answer.reason_id() == Some(OUTCOME_UNKNOWN) => { + counts.unknown += 1; + counts.unknown_durable += usize::from(durable.contains(&record.share_id)); + } + Answer::Rejected(response) => { + let reason = match record.answer.reason_id() { + Some(reason) => reason.to_owned(), + None => format!( + "{UNTYPED} {} {}", + response["error"][0], + response["error"][1].as_str().unwrap_or("") + ), + }; + *counts.rejected.entry(reason.clone()).or_default() += 1; + ensure!( + !durable.contains(&record.share_id), + "refused share {} ({reason}) is in the ledger", + record.share_id + ); + } + Answer::TimedOut | Answer::Lost(_) => { + counts.unknown += 1; + counts.unknown_durable += usize::from(durable.contains(&record.share_id)); + } + } + } + for share in durable { + ensure!( + answered.contains_key(share.as_str()), + "ledger share {share} was never submitted by the load" + ); + } + Ok(counts) +} + +impl LoadRecord { + fn seconds(&self, at: Instant) -> f64 { + at.saturating_duration_since(self.started).as_secs_f64() + } + + fn node_of(&self, frontend: usize) -> Option { + self.plan.node_of.get(frontend).copied().flatten() + } + + /// Frontend `frontend`'s last `/healthz` sample at or before `at` was + /// ready. A frontend fences its health for the moments between a new tip + /// and work on it (`observed_tip_advance_fences_old_prepared_jobs_and_health`), + /// so readiness is read when the bound expires, not throughout it. + fn ready_at(&self, frontend: usize, at: Instant) -> bool { + self.health + .iter() + .rev() + .find(|(sampled, index, _)| *index == frontend && *sampled <= at) + .is_some_and(|(_, _, ready)| *ready) + } + + /// The intervals, in seconds from the start, in which `frontend` did not + /// answer `/healthz` with success; an open one ends with the load. + fn unready(&self, frontend: usize) -> Vec<[f64; 2]> { + let mut intervals = Vec::new(); + let mut since: Option = None; + for (at, index, ready) in &self.health { + if *index != frontend { + continue; + } + match (ready, since) { + (false, None) => since = Some(*at), + (true, Some(from)) => { + intervals.push([round(self.seconds(from)), round(self.seconds(*at))]); + since = None; + } + _ => {} + } + } + if let Some(from) = since { + intervals.push([round(self.seconds(from)), round(self.seconds(self.ended))]); + } + intervals + } + + /// Watched node `node` answered its watcher throughout `from`..`to`. + fn reachable_through(&self, node: usize, from: Instant, to: Instant) -> bool { + !self + .unreachable + .iter() + .any(|(index, start, end)| *index == node && *start < to && *end > from) + } + + /// Every tip each frontend's node took, as the frontend's sessions got it. + pub(super) fn deliveries(&self) -> Vec { + let mut deliveries = Vec::new(); + for frontend in 0..self.plan.stratum.len() { + let connections: Vec<&Connection> = self + .sessions + .iter() + .flat_map(|session| &session.connections) + .filter(|connection| connection.frontend == frontend) + .collect(); + for (position, stamp) in self.tips.iter().enumerate() { + if self.node_of(frontend) != Some(stamp.node) { + continue; + } + let previous = self.tips[..position] + .iter() + .rev() + .find(|earlier| earlier.node == stamp.node) + .map(|earlier| earlier.at); + let next = self.tips[position + 1..] + .iter() + .find(|later| later.node == stamp.node) + .map(|later| later.at); + let bound_end = stamp.at + DELIVERY_BOUND; + let mut delivery = Delivery { + frontend, + node: stamp.node, + hash: stamp.hash.clone(), + stamped: self.seconds(stamp.at), + lifetime: next.map(|next| (next - stamp.at).as_secs_f64()), + gated: bound_end <= self.ended + && self.ready_at(frontend, bound_end) + && self.reachable_through(stamp.node, stamp.at, bound_end), + ..Delivery::default() + }; + for connection in &connections { + if connection.ready_at > stamp.at + || connection + .closed_at + .is_some_and(|closed| closed <= stamp.at) + { + continue; + } + delivery.eligible += 1; + match served_at(connection, stamp, previous) { + Some(at) if next.is_none_or(|next| at <= next) => { + if at < stamp.at { + delivery.ahead_of_stamp += 1; + } + let seconds = at.saturating_duration_since(stamp.at).as_secs_f64(); + delivery.served += 1; + delivery.seconds.push(seconds); + if seconds > DELIVERY_BOUND.as_secs_f64() { + delivery.late += 1; + } + } + _ => { + let deadline = next.map_or(bound_end, |next| next.min(bound_end)); + if next.is_some_and(|next| next < bound_end) { + delivery.replaced += 1; + } else if connection.closed_at.is_some_and(|closed| closed < deadline) + || self.ended < deadline + { + delivery.gone += 1; + } else { + delivery.late += 1; + } + } + } + } + delivery.seconds.sort_by(f64::total_cmp); + deliveries.push(delivery); + } + } + deliveries + } + + /// Hold the load to its checks and return its report, which is also + /// printed as one `live-load : {json}` line. + pub(super) async fn check(&self, fixture: &Fixture) -> Result { + let prefix = format!("{}.load", self.plan.address); + let durable: HashSet = sqlx::query_scalar( + "SELECT share_id FROM qbit_share_ledger WHERE accepted AND starts_with(share_id,$1)", + ) + .bind(&prefix) + .fetch_all(&fixture.pool) + .await? + .into_iter() + .collect(); + let report = self.report(&durable); + eprintln!("live-load {}: {report}", self.plan.name); + self.verdict(&durable)?; + Ok(report) + } + + fn submits(&self) -> Vec<&SubmitRecord> { + self.sessions + .iter() + .flat_map(|session| &session.submits) + .collect() + } + + /// The checks, on what the ledger holds. + fn verdict(&self, durable: &HashSet) -> Result<()> { + let submits = self.submits(); + let counts = reconcile(&submits, durable)?; + let unexpected: Vec = counts + .rejected + .iter() + .filter(|(reason, _)| !expected_refusal(reason)) + .map(|(reason, count)| format!("{count} {reason}")) + .collect(); + ensure!( + unexpected.is_empty(), + "the load earned refusals a correct miner never earns, or unrecognised ones: {}", + unexpected.join(", ") + ); + ensure!( + counts.accepted > 0, + "no load share was acknowledged: {counts:?}" + ); + let never = self + .sessions + .iter() + .filter(|session| session.connections.is_empty()) + .count(); + ensure!( + never == 0, + "{never} of {} load sessions never held work; last failure: {:?}", + self.sessions.len(), + self.sessions + .iter() + .find_map(|session| session.last_failure.clone()) + ); + let late: Vec = self + .deliveries() + .iter() + .filter(|delivery| delivery.gated && delivery.late > 0) + .map(|delivery| { + format!( + "frontend {} tip {} at {:.1}s: {} of {} sessions without work within {DELIVERY_BOUND:?} (max served {:?}s)", + delivery.frontend, + delivery.hash, + delivery.stamped, + delivery.late, + delivery.eligible, + delivery.seconds.last() + ) + }) + .collect(); + ensure!( + late.is_empty(), + "new-tip work did not reach every session on a ready frontend: {}", + late.join("; ") + ); + Ok(()) + } + + fn report(&self, durable: &HashSet) -> Value { + let submits = self.submits(); + let counts = reconcile(&submits, durable); + let deliveries = self.deliveries(); + let mut frontends = Vec::new(); + for frontend in 0..self.plan.stratum.len() { + let own: Vec<&Delivery> = deliveries + .iter() + .filter(|delivery| delivery.frontend == frontend && delivery.eligible > 0) + .collect(); + let mut all: Vec = own + .iter() + .flat_map(|delivery| delivery.seconds.iter().copied()) + .collect(); + all.sort_by(f64::total_cmp); + let pick = |quantile: f64| { + all.len() + .checked_sub(1) + .map(|last| round(all[((last as f64) * quantile).round() as usize])) + }; + frontends.push(json!({ + "frontend": frontend, + "unready_s": self.unready(frontend), + "healthz_max_ms": (self.health_stats[frontend].0.as_secs_f64() * 1000.0).round(), + "healthz_timeouts": self.health_stats[frontend].1, + "tips": own.len(), + "gated_tips": own.iter().filter(|delivery| delivery.gated).count(), + "held_tips": own.iter().filter(|delivery| delivery.held()).count(), + "session_deliveries": all.len(), + "p50_s": pick(0.5), + "p99_s": pick(0.99), + "max_s": all.last().copied().map(round), + "worst_all_sessions_s": own + .iter() + .filter_map(|delivery| delivery.all_sessions()) + .max_by(f64::total_cmp) + .map(round), + "replaced": own.iter().map(|delivery| delivery.replaced).sum::(), + "late": own.iter().map(|delivery| delivery.late).sum::(), + "submitted": submits.iter().filter(|record| record.frontend == frontend).count(), + })); + } + let connections: usize = self + .sessions + .iter() + .map(|session| session.connections.len()) + .sum(); + json!({ + "sessions": self.plan.sessions, + "offered_rate": self.plan.rate, + "seconds": round(self.offered_seconds), + "paused_seconds": round(self.paused_seconds), + "offered": (self.plan.rate * self.offered_seconds).floor(), + "shares": match &counts { + Ok(counts) => json!({ + "submitted": counts.submitted, + "accepted": counts.accepted, + "rejected": counts.rejected, + "unknown": counts.unknown, + "unknown_durable": counts.unknown_durable, + "durable": durable.len(), + }), + Err(error) => json!({"error": format!("{error:#}")}), + }, + "node_unreachable_s": (0..self.plan.nodes.len()) + .map(|node| { + self.unreachable + .iter() + .filter(|(index, _, _)| *index == node) + .map(|(_, from, to)| [round(self.seconds(*from)), round(self.seconds(*to))]) + .collect::>() + }) + .collect::>(), + "connections": connections, + "reconnects": connections.saturating_sub(self.plan.sessions), + "connect_failures": self.sessions.iter().map(|session| session.connect_failures).sum::(), + "delivery_bound_s": DELIVERY_BOUND.as_secs_f64(), + "time_to_usable_work": frontends, + "tips": deliveries + .iter() + .filter(|delivery| delivery.eligible > 0) + .map(Delivery::json) + .collect::>(), + }) + } +} + +/// The scenario-level load settings: `PRISM_L4_SESSIONS` and +/// `PRISM_L4_SHARE_RATE`, or each case's defaults. +pub(super) fn load_settings(sessions: usize, rate: f64) -> Result<(usize, f64)> { + let sessions: usize = setting("PRISM_L4_SESSIONS", sessions)?; + let rate: f64 = setting("PRISM_L4_SHARE_RATE", rate)?; + ensure!( + sessions > 0 && rate.is_finite() && rate > 0.0, + "PRISM_L4_SESSIONS and PRISM_L4_SHARE_RATE must be positive" + ); + Ok((sessions, rate)) +} + +/// Run `scenario` on `fixture` under `plan`'s load, then stop the load and +/// check it. The scenario's own error comes first; the load's check runs +/// only when the scenario passed, since a failed scenario can leave its +/// frontends in any state. +pub(super) async fn under_load( + fixture: &mut Fixture, + plan: LoadPlan, + scenario: F, +) -> Result +where + F: AsyncFnOnce(&mut Fixture, &SessionLoad) -> Result<()>, +{ + let load = SessionLoad::start(plan)?; + let result = scenario(fixture, &load).await; + let record = load.stop().await; + result?; + record.check(fixture).await +} + +#[cfg(test)] +mod tests { + use super::*; + + fn at(start: Instant, seconds: f64) -> Instant { + start + Duration::from_secs_f64(seconds) + } + + fn record( + start: Instant, + connections: Vec, + tips: &[(usize, &str, f64)], + ) -> LoadRecord { + LoadRecord { + plan: LoadPlan { + name: "unit".into(), + sessions: connections.len(), + rate: 1.0, + address: "qbrt1load".into(), + stratum: vec![1, 2], + api: vec![3, 4], + nodes: vec![5, 6], + node_of: vec![Some(0), Some(1)], + seed: 0, + }, + started: start, + ended: at(start, 100.0), + offered_seconds: 100.0, + paused_seconds: 0.0, + health: vec![(start, 0, true), (start, 1, true)], + health_stats: vec![(Duration::ZERO, 0); 2], + unreachable: Vec::new(), + tips: tips + .iter() + .map(|(node, hash, seconds)| TipStamp { + node: *node, + hash: (*hash).into(), + at: at(start, *seconds), + }) + .collect(), + sessions: connections + .into_iter() + .map(|connection| SessionLog { + connections: vec![connection], + ..SessionLog::default() + }) + .collect(), + } + } + + fn connection(start: Instant, frontend: usize, parents: &[(f64, &str)]) -> Connection { + Connection { + frontend, + ready_at: at(start, 1.0), + closed_at: None, + parents: parents + .iter() + .map(|(seconds, parent)| (at(start, *seconds), (*parent).into())) + .collect(), + } + } + + #[test] + fn delivery_follows_each_frontends_node_and_gates_only_lasting_tips() { + let start = Instant::now(); + let record = record( + start, + vec![ + connection(start, 0, &[(1.0, "g"), (10.5, "a"), (20.2, "c")]), + connection(start, 0, &[(1.0, "g"), (12.0, "a")]), + connection(start, 1, &[(1.0, "g"), (10.1, "a")]), + ], + &[ + (0, "g", 0.0), + (1, "g", 0.0), + (0, "a", 10.0), + (1, "a", 9.9), + (0, "b", 20.0), + (0, "c", 20.1), + ], + ); + let deliveries = record.deliveries(); + let find = |frontend: usize, hash: &str| { + deliveries + .iter() + .find(|delivery| delivery.frontend == frontend && delivery.hash == hash) + .unwrap_or_else(|| panic!("no delivery of {hash} on {frontend}")) + }; + let a0 = find(0, "a"); + assert_eq!((a0.eligible, a0.served, a0.late), (2, 2, 0)); + assert!((a0.seconds[1] - 2.0).abs() < 1e-6, "{a0:?}"); + assert!(a0.gated && a0.lifetime.is_some_and(|life| (life - 10.0).abs() < 1e-6)); + let a1 = find(1, "a"); + assert_eq!((a1.eligible, a1.served), (1, 1)); + assert!((a1.seconds[0] - 0.2).abs() < 1e-6, "{a1:?}"); + // Replaced 0.1 s later: never served, never late. + let b0 = find(0, "b"); + assert_eq!((b0.served, b0.replaced, b0.late), (0, 2, 0)); + // The last tip lasts: the second session never got it. + let c0 = find(0, "c"); + assert_eq!((c0.served, c0.late), (1, 1)); + assert!(c0.gated); + assert!(!deliveries + .iter() + .any(|delivery| delivery.frontend == 1 && delivery.hash == "b")); + let error = record.verdict(&HashSet::new()).unwrap_err().to_string(); + assert!(error.contains("no load share was acknowledged"), "{error}"); + } + + #[test] + fn work_on_a_parent_held_before_serves_its_tip_again_after_a_reorg_back() { + let start = Instant::now(); + // `receive` records every notify, so the second notify on "a" after + // the missed tip "b" is there to find. + let record = record( + start, + vec![connection( + start, + 0, + &[(1.0, "g"), (10.5, "a"), (20.6, "a")], + )], + &[ + (0, "g", 0.0), + (0, "a", 10.0), + (0, "b", 20.0), + (0, "a", 20.1), + ], + ); + let deliveries = record.deliveries(); + let back = deliveries + .iter() + .rfind(|delivery| delivery.hash == "a") + .expect("the reorg back's delivery"); + assert_eq!((back.served, back.late), (1, 0), "{back:?}"); + assert!((back.seconds[0] - 0.5).abs() < 1e-6, "{back:?}"); + } + + #[test] + fn work_the_session_moved_off_before_a_reorg_back_does_not_serve_it() { + let start = Instant::now(); + // A republication on "x" after "y"'s stamp, then "y" itself: when + // the node returns to "x" the session holds work on "y", so the + // stale notify is not work on the returned tip. The second session + // still holds that republished work, which counts ahead of the stamp. + let record = record( + start, + vec![ + connection(start, 0, &[(1.0, "x"), (20.1, "x"), (20.5, "y")]), + connection(start, 0, &[(1.0, "x"), (20.1, "x")]), + ], + &[(0, "x", 0.0), (0, "y", 20.0), (0, "x", 30.0)], + ); + let deliveries = record.deliveries(); + let back = deliveries + .iter() + .rfind(|delivery| delivery.hash == "x") + .expect("the reorg back's delivery"); + assert_eq!( + (back.eligible, back.served, back.ahead_of_stamp, back.late), + (2, 1, 1, 1), + "{back:?}" + ); + assert!(back.held()); + // "y" lasted 10 s, under the bound: gated, but not held. + let y = deliveries + .iter() + .find(|delivery| delivery.hash == "y") + .expect("y's delivery"); + assert!(y.gated && !y.held(), "{y:?}"); + } + + #[test] + fn a_frontend_still_unready_when_the_bound_expires_is_not_gated() { + let start = Instant::now(); + let mut record = record( + start, + vec![connection(start, 0, &[(1.0, "g")])], + &[(0, "g", 0.0), (0, "a", 10.0)], + ); + let gated = |record: &LoadRecord| { + record + .deliveries() + .into_iter() + .find(|delivery| delivery.hash == "a") + .map(|delivery| (delivery.late, delivery.gated)) + }; + // The post-tip health fence: unready briefly, ready again. + record.health.push((at(start, 10.2), 0, false)); + record.health.push((at(start, 12.0), 0, true)); + assert_eq!(gated(&record), Some((1, true))); + record.health.push((at(start, 20.0), 0, false)); + assert_eq!(gated(&record), Some((1, false))); + } + + #[test] + fn a_tip_whose_node_stops_answering_within_the_bound_is_not_gated() { + let start = Instant::now(); + let mut record = record( + start, + vec![connection(start, 0, &[(1.0, "g")])], + &[(0, "g", 0.0), (0, "a", 10.0)], + ); + record + .unreachable + .push((0, at(start, 11.0), at(start, 17.0))); + let a0 = record + .deliveries() + .into_iter() + .find(|delivery| delivery.hash == "a") + .expect("delivery"); + assert_eq!((a0.late, a0.gated), (1, false)); + record.unreachable = vec![(1, at(start, 11.0), at(start, 17.0))]; + let a0 = record + .deliveries() + .into_iter() + .find(|delivery| delivery.hash == "a") + .expect("delivery"); + assert!(a0.gated, "another node's outage exempted it"); + } + + fn submit(share: &str, answer: Answer) -> SubmitRecord { + SubmitRecord { + share_id: share.into(), + frontend: 0, + answer, + } + } + + #[test] + fn reconciliation_needs_every_ack_durable_and_no_refusal_durable() { + let refused = Answer::Rejected( + json!({"id":3,"result":null,"error":[21,"stale job",{"reason_id":"stale-job"}]}), + ); + let submits = [ + submit("s1", Answer::Accepted), + submit("s2", refused.clone()), + submit("s3", Answer::TimedOut), + submit("s4", Answer::Lost("closed".into())), + ]; + let records: Vec<&SubmitRecord> = submits.iter().collect(); + let durable = |shares: &[&str]| shares.iter().map(|share| (*share).to_owned()).collect(); + let counts = reconcile(&records, &durable(&["s1", "s3"])).expect("reconciles"); + assert_eq!( + (counts.accepted, counts.unknown, counts.unknown_durable), + (1, 2, 1) + ); + assert_eq!(counts.rejected.get("stale-job"), Some(&1)); + assert!(reconcile(&records, &durable(&[])) + .unwrap_err() + .to_string() + .contains("acknowledged share s1")); + assert!(reconcile(&records, &durable(&["s1", "s2"])) + .unwrap_err() + .to_string() + .contains("refused share s2")); + assert!(reconcile(&records, &durable(&["s1", "s9"])) + .unwrap_err() + .to_string() + .contains("s9 was never submitted")); + + // An unknown COMMIT outcome may land: it counts as unanswered. + let unknown = Answer::Rejected(json!({"id":4,"result":null, + "error":[20,"ledger outcome unknown",{"reason_id":"ledger-outcome-unknown"}]})); + let submits = [submit("s5", unknown)]; + let records: Vec<&SubmitRecord> = submits.iter().collect(); + let counts = reconcile(&records, &durable(&["s5"])).expect("an unknown outcome may land"); + assert_eq!((counts.unknown, counts.unknown_durable), (1, 1)); + assert!(counts.rejected.is_empty()); + } + + #[test] + fn only_refusals_a_correct_miner_can_earn_are_expected() { + for reason in [ + "stale-job", + "unknown-job", + "backend-rpc-unavailable", + "backend-database-unavailable", + ] { + assert!(expected_refusal(reason), "{reason}"); + } + for reason in [ + "duplicate-share", + "low-difficulty", + "unauthorized-worker", + "invalid-extranonce", + "invalid-ntime-or-nonce", + "something-new", + ] { + assert!(!expected_refusal(reason), "{reason}"); + } + assert!(expected_refusal(&format!( + "{UNTYPED} 20 too many unknown job submissions" + ))); + // Only the unknown-job budget, at its code and exact message. + for untyped in [ + "21 too many unknown job submissions", + "20 too many unknown job submissions, and more", + "20 too many connections for username", + "", + ] { + assert!( + !expected_refusal(&format!("{UNTYPED} {untyped}")), + "{untyped}" + ); + } + } + + #[test] + fn notify_parent_is_the_display_hash() { + let display = "00112233445566778899aabbccddeeff00112233445566778899aabbccddeeff"; + let mut wire = hex::decode(display).expect("hex"); + wire.reverse(); + for word in wire.as_chunks_mut::<4>().0 { + word.reverse(); + } + let notify = json!({"params": ["job", hex::encode(wire)]}); + assert_eq!(notify_parent(¬ify).as_deref(), Some(display)); + } +} diff --git a/crates/qbit-prism-server/tests/support/live_share_client.rs b/crates/qbit-prism-server/tests/support/live_share_client.rs new file mode 100644 index 000000000..c5e201520 --- /dev/null +++ b/crates/qbit-prism-server/tests/support/live_share_client.rs @@ -0,0 +1,594 @@ +//! Stratum v1 clients for the live suite that solve work in the test and +//! record exactly what the server answered (#575). +//! +//! [`StratumSession`] is one connection and what the server has told it: +//! extranonce, difficulty, latest job, version mask, and every notification +//! in order, each checked for the shape miners parse. [`ShareClient`] is a +//! scripted miner on top of it that submits shares or blocks and records the +//! answer to each, so a scenario can compare every acknowledged share with +//! the ledger. The transcript replay drives a session directly. +//! +//! Regtest's block target (`207fffff`) is easier than a difficulty-1 share, so +//! a server left at its default share difficulty turns every share into a +//! block. A scenario that needs shares that are not blocks runs its servers +//! with [`SHARE_ONLY_SETTINGS`]: vardiff off and a share target above the +//! block target, so about half of all hashes are shares that miss the block +//! target and the client can choose which kind it submits. +use super::*; +use num_bigint::BigUint; +use qbit_prism_server::codec::{ + difficulty_target, double_sha256, hash_display, parse_u32_hex, target_from_compact, +}; +use tokio::{ + net::{ + tcp::{OwnedReadHalf, OwnedWriteHalf}, + TcpStream, + }, + time::error::Elapsed, +}; + +/// Server settings under which a share can miss the regtest block target: +/// a fixed share difficulty of about 2^-32, whose target is above regtest's. +pub(crate) const SHARE_ONLY_SETTINGS: [(&str, &str); 2] = [ + ("PRISM_STRATUM_VARDIFF", "0"), + ("PRISM_STRATUM_SHARE_DIFF", "0.00000000025"), +]; + +/// How long a request waits for its answer: well past the server's own +/// bound, the share commit timeout plus grace (15 s by default), which also +/// bounds a block whose answer waits for its landing (#577). +pub(crate) const ANSWER: Duration = Duration::from_secs(70); + +/// Start `servers` with [`SHARE_ONLY_SETTINGS`] and each one's `overrides`, +/// and wait until every one is ready. +pub(crate) async fn start_share_only_servers( + f: &mut Fixture, + servers: &[(usize, Vec<(&str, String)>)], +) -> Result<()> { + f.server_env = SHARE_ONLY_SETTINGS + .iter() + .map(|(name, value)| (name.to_string(), value.to_string())) + .collect(); + for (index, overrides) in servers { + let process = f.start_server_with(*index, None, overrides)?; + f.servers.push(process); + } + for (index, _) in servers { + until(&format!("server {index} readiness"), 60, || async { + Ok(f.client + .get(format!("http://127.0.0.1:{}/healthz", f.api[*index])) + .send() + .await? + .status() + .is_success()) + }) + .await?; + } + Ok(()) +} + +/// Which target a submitted header meets. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub(crate) enum Proof { + /// The share target but not the block target: a share and nothing else. + Share, + /// The block target (and so the share target too). + Block, +} + +/// What the server did with one submission. +#[derive(Clone, Debug)] +pub(crate) enum Answer { + Accepted, + /// A JSON-RPC error or a `false` result: the full response. + Rejected(Value), + /// No answer within [`ANSWER`]. + TimedOut, + /// The connection failed before the answer. + Lost(String), +} + +impl Answer { + pub(crate) fn accepted(&self) -> bool { + matches!(self, Answer::Accepted) + } + + /// The `reason_id` of a rejection's `[code, message, {"reason_id": ...}]`. + pub(crate) fn reason_id(&self) -> Option<&str> { + match self { + Answer::Rejected(response) => reason_id(response), + _ => None, + } + } +} + +impl std::fmt::Display for Answer { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + Answer::Accepted => f.write_str("accepted"), + Answer::Rejected(response) => match reason_id(response) { + Some(reason) => f.write_str(reason), + None => write!(f, "rejected {}", response["error"]), + }, + Answer::TimedOut => f.write_str("unanswered: timed out"), + Answer::Lost(why) => write!(f, "unanswered: connection lost ({why})"), + } + } +} + +/// The `reason_id` of a response's `[code, message, {"reason_id": ...}]` error. +pub(crate) fn reason_id(response: &Value) -> Option<&str> { + response["error"][2]["reason_id"].as_str() +} + +/// One submission and its answer. `share_id` is the ledger's identity for +/// it: the worker name and the header hash in display order. +#[derive(Clone, Debug)] +pub(crate) struct Submitted { + pub share_id: String, + pub hash: String, + pub answer: Answer, +} + +/// A solved header for one job. +pub(crate) struct Solution { + pub extranonce2: String, + pub ntime: String, + pub nonce: String, + /// The rolled version bits, as `mining.submit`'s sixth field, when the + /// job was solved with version rolling. + pub version_bits: Option, + /// Display-order header hash. + pub hash: String, +} + +/// A share that is not a block when the job's share target allows one; a +/// block otherwise (a share difficulty at or above the network's, as a +/// miner's `mining.suggest_difficulty` can ask for on regtest). +fn easiest_proof(notify: &Value, difficulty: f64) -> Result { + let bits = notify["params"][6] + .as_str() + .context("notify bits missing")?; + let block = target_from_compact(parse_u32_hex(bits)?)?; + Ok(if difficulty_target(difficulty)? > block { + Proof::Share + } else { + Proof::Block + }) +} + +/// Solve `notify` for `proof`. `counter` picks the extranonce2 (and so the +/// merkle root); `version_mask`, when nonzero, rolls the lowest bit of the +/// mask into the version, as a version-rolling miner does. +fn solve( + notify: &Value, + extranonce1: &str, + extranonce2_size: usize, + difficulty: f64, + version_mask: u32, + counter: u64, + proof: Proof, +) -> Result { + let params = notify["params"].as_array().context("notify missing")?; + let field = |index: usize| { + params + .get(index) + .and_then(Value::as_str) + .with_context(|| format!("notify field {index} missing")) + }; + let extranonce2 = format!("{:0width$x}", counter, width = extranonce2_size * 2); + ensure!( + extranonce2.len() == extranonce2_size * 2, + "extranonce2 counter {counter} exceeds {extranonce2_size} bytes" + ); + let coinbase = hex::decode(format!( + "{}{extranonce1}{extranonce2}{}", + field(2)?, + field(3)? + ))?; + let mut merkle = double_sha256(&coinbase); + for sibling in params + .get(4) + .and_then(Value::as_array) + .context("merkle branch missing")? + { + let sibling = hex::decode(sibling.as_str().context("invalid sibling")?)?; + merkle = double_sha256(&[merkle.as_slice(), sibling.as_slice()].concat()); + } + let mut previous = hex::decode(field(1)?)?; + for word in previous.as_chunks_mut::<4>().0 { + word.reverse(); + } + let bits = parse_u32_hex(field(6)?)?; + let block_target = target_from_compact(bits)?; + let share_target = difficulty_target(difficulty)?; + let ntime = parse_u32_hex(field(7)?)?; + let base_version = parse_u32_hex(field(5)?)?; + let (version, version_bits) = if version_mask == 0 { + (base_version, None) + } else { + let rolled = version_mask.isolate_lowest_one(); + let version = (base_version & !version_mask) | ((base_version ^ rolled) & version_mask); + (version, Some(format!("{:08x}", version & version_mask))) + }; + for nonce in 0..1_000_000u32 { + let header = [ + version.to_le_bytes().as_slice(), + previous.as_slice(), + merkle.as_slice(), + ntime.to_le_bytes().as_slice(), + bits.to_le_bytes().as_slice(), + nonce.to_le_bytes().as_slice(), + ] + .concat(); + let hash = double_sha256(&header); + let value = BigUint::from_bytes_le(&hash); + let found = match proof { + Proof::Share => value <= share_target && value > block_target, + Proof::Block => value <= block_target && value <= share_target, + }; + if found { + return Ok(Solution { + extranonce2, + ntime: format!("{ntime:08x}"), + nonce: format!("{nonce:08x}"), + version_bits, + hash: hash_display(&hash), + }); + } + } + bail!("no {proof:?} solution in the nonce budget (share difficulty {difficulty})") +} + +/// Solve `notify` for a share that misses its block target, without +/// version rolling: the session load's proof (#553). +pub(crate) fn solve_share( + notify: &Value, + extranonce1: &str, + extranonce2_size: usize, + difficulty: f64, + counter: u64, +) -> Result { + solve( + notify, + extranonce1, + extranonce2_size, + difficulty, + 0, + counter, + Proof::Share, + ) +} + +/// A hex string of even length, of `length` digits when given. +pub(crate) fn is_hex(value: &Value, length: Option) -> bool { + value.as_str().is_some_and(|text| { + text.len() % 2 == 0 + && length.is_none_or(|length| text.len() == length) + && text.bytes().all(|byte| byte.is_ascii_hexdigit()) + }) +} + +/// A notification's parameters have the shape miners parse. +fn check_notification(message: &Value) -> Result<()> { + let params = message["params"].as_array(); + let method = message["method"].as_str().unwrap_or_default(); + let valid = match method { + "mining.notify" => params.is_some_and(|p| { + p.len() == 9 + && p[0].as_str().is_some_and(|job| !job.is_empty()) + && is_hex(&p[1], Some(64)) + && is_hex(&p[2], None) + && is_hex(&p[3], None) + && p[4] + .as_array() + .is_some_and(|branch| branch.iter().all(|node| is_hex(node, Some(64)))) + && (5..=7).all(|index| is_hex(&p[index], Some(8))) + && p[8].is_boolean() + }), + "mining.set_difficulty" => { + params.is_some_and(|p| p.len() == 1 && p[0].as_f64().is_some_and(|d| d > 0.0)) + } + "mining.set_version_mask" => params.is_some_and(|p| p.len() == 1 && is_hex(&p[0], Some(8))), + _ => true, + }; + ensure!(valid, "malformed {method}: {message}"); + Ok(()) +} + +/// One Stratum connection and what the server has told it so far. +pub(crate) struct StratumSession { + reader: BufReader, + writer: OwnedWriteHalf, + pub extranonce1: Option, + pub extranonce2_size: Option, + pub difficulty: Option, + /// The latest `mining.notify`. + pub notify: Option, + /// The version-rolling mask granted by `mining.configure` or the latest + /// `mining.set_version_mask`; zero when none. + pub version_mask: u32, + /// Every notification method, in arrival order. + pub notifications: Vec, + counter: u64, +} + +impl StratumSession { + pub(crate) async fn open(port: u16) -> Result { + let stream = tokio::time::timeout( + Duration::from_secs(10), + TcpStream::connect(("127.0.0.1", port)), + ) + .await + .context("Stratum connect timed out")??; + let (read, writer) = stream.into_split(); + Ok(Self { + reader: BufReader::new(read), + writer, + extranonce1: None, + extranonce2_size: None, + difficulty: None, + notify: None, + version_mask: 0, + notifications: Vec::new(), + counter: 0, + }) + } + + pub(crate) async fn send(&mut self, message: &Value) -> Result<()> { + self.writer + .write_all(format!("{message}\n").as_bytes()) + .await?; + Ok(()) + } + + /// The next message within `limit`, keeping and checking every + /// notification. A timeout is an error that downcasts to [`Elapsed`]. + async fn next(&mut self, limit: Duration) -> Result { + let mut line = String::new(); + let read = tokio::time::timeout(limit, self.reader.read_line(&mut line)) + .await + .map_err(anyhow::Error::new) + .with_context(|| format!("no Stratum message within {limit:?}"))??; + ensure!(read > 0, "Stratum connection closed"); + let message: Value = serde_json::from_str(&line)?; + if let Some(method) = message["method"].as_str() { + check_notification(&message)?; + self.notifications.push(method.to_owned()); + match method { + "mining.set_difficulty" => self.difficulty = message["params"][0].as_f64(), + "mining.notify" => self.notify = Some(message.clone()), + "mining.set_version_mask" => { + self.version_mask = + parse_u32_hex(message["params"][0].as_str().context("mask missing")?)?; + } + _ => {} + } + } + Ok(message) + } + + /// The response to request `id`, within `limit`. + pub(crate) async fn answer(&mut self, id: &Value, limit: Duration) -> Result { + let deadline = Instant::now() + limit; + loop { + let message = self + .next(deadline.saturating_duration_since(Instant::now())) + .await?; + if message.get("method").is_none() && &message["id"] == id { + return Ok(message); + } + } + } + + /// Send `request` and wait for its response; a subscribe or configure + /// response also updates the session. + pub(crate) async fn request(&mut self, request: &Value) -> Result { + self.send(request).await?; + let answer = self.answer(&request["id"], ANSWER).await?; + self.learn(request["method"].as_str().unwrap_or_default(), &answer)?; + Ok(answer) + } + + /// Keep what a response to `method` tells the session. + pub(crate) fn learn(&mut self, method: &str, answer: &Value) -> Result<()> { + match method { + "mining.subscribe" => { + self.extranonce1 = answer["result"][1].as_str().map(str::to_owned); + self.extranonce2_size = answer["result"][2].as_u64().map(|size| size as usize); + } + "mining.configure" => { + if let Some(mask) = answer["result"]["version-rolling.mask"].as_str() { + self.version_mask = parse_u32_hex(mask)?; + } + } + _ => {} + } + Ok(()) + } + + /// Until a job and a share difficulty have arrived. + pub(crate) async fn work(&mut self) -> Result<()> { + while self.notify.is_none() || self.difficulty.is_none() { + self.next(ANSWER).await?; + } + Ok(()) + } + + /// Until a job other than the latest one arrives. + pub(crate) async fn newer_work(&mut self) -> Result<()> { + let job = |session: &Self| { + session + .notify + .as_ref() + .map(|notify| notify["params"][0].clone()) + }; + let latest = job(self); + while job(self) == latest { + self.next(ANSWER).await?; + } + Ok(()) + } + + /// Until the latest job builds on `parent` (display order). + pub(crate) async fn work_on(&mut self, parent: &str, limit: Duration) -> Result<()> { + let mut wire = hex::decode(parent)?; + wire.reverse(); + for word in wire.as_chunks_mut::<4>().0 { + word.reverse(); + } + let expected = hex::encode(wire); + let deadline = Instant::now() + limit; + while self + .notify + .as_ref() + .and_then(|notify| notify["params"][1].as_str()) + != Some(expected.as_str()) + { + let left = deadline.saturating_duration_since(Instant::now()); + ensure!(!left.is_zero(), "no work on {parent} within {limit:?}"); + self.next(left).await?; + } + Ok(()) + } + + /// Solve the latest job as `username`: for `proof`, or the easiest proof + /// the job allows when `None`; with version rolling when `rolls`. Returns + /// the `mining.submit` params and the ledger's share id. + pub(crate) async fn solve_latest( + &mut self, + username: &str, + proof: Option, + rolls: bool, + ) -> Result<(Value, Submitted)> { + self.work().await?; + ensure!( + !rolls || self.version_mask != 0, + "version rolling needs a granted mask" + ); + self.counter += 1; + let notify = self.notify.as_ref().context("no job")?; + let difficulty = self.difficulty.context("no difficulty")?; + let proof = match proof { + Some(proof) => proof, + None => easiest_proof(notify, difficulty)?, + }; + let solution = solve( + notify, + self.extranonce1.as_deref().context("no extranonce1")?, + self.extranonce2_size.context("no extranonce2 size")?, + difficulty, + if rolls { self.version_mask } else { 0 }, + self.counter, + proof, + )?; + let mut params = vec![ + json!(username), + notify["params"][0].clone(), + json!(solution.extranonce2), + json!(solution.ntime), + json!(solution.nonce), + ]; + if rolls { + params.push(json!(solution.version_bits.context("no version bits")?)); + } + let submitted = Submitted { + share_id: format!("{username}:{}", solution.hash), + hash: solution.hash, + answer: Answer::TimedOut, + }; + Ok((Value::Array(params), submitted)) + } +} + +/// A subscribed and authorized miner that submits one proof at a time. +pub(crate) struct ShareClient { + session: StratumSession, + pub username: String, + next_id: u64, +} + +impl ShareClient { + /// Subscribe and authorize `username` on `port`, and wait for work. + pub(crate) async fn connect(port: u16, username: &str) -> Result { + let mut session = StratumSession::open(port).await?; + let subscribed = session + .request( + &json!({"id":1,"method":"mining.subscribe","params":["prism-live-share-client"]}), + ) + .await?; + ensure!( + session.extranonce1.is_some() && session.extranonce2_size.is_some(), + "subscribe: {subscribed}" + ); + let authorized = session + .request(&json!({"id":2,"method":"mining.authorize","params":[username,"x"]})) + .await?; + ensure!(authorized["result"] == true, "authorize: {authorized}"); + session.work().await?; + Ok(Self { + session, + username: username.into(), + next_id: 10, + }) + } + + /// Until the latest job builds on `parent` (display order). + pub(crate) async fn work_on(&mut self, parent: &str, limit: Duration) -> Result<()> { + self.session.work_on(parent, limit).await + } + + /// Until a job other than the latest one read so far arrives. + pub(crate) async fn newer_work(&mut self) -> Result<()> { + self.session.newer_work().await + } + + /// The id of the latest job read so far, the one [`ShareClient::solve`] + /// solves next; reads nothing more from the server. + pub(crate) fn job_id(&self) -> Option<&str> { + self.session + .notify + .as_ref() + .and_then(|notify| notify["params"][0].as_str()) + } + + /// Solve the latest job for `proof`, submit it, and wait for the answer. + /// An error means nothing was submitted; a failure after the submission + /// is [`Answer::TimedOut`] or [`Answer::Lost`]. + pub(crate) async fn submit(&mut self, proof: Proof) -> Result { + let (params, submitted) = self.solve(proof).await?; + Ok(self.submit_solved(params, submitted).await) + } + + /// Solve the latest job for `proof` without submitting it: work a miner + /// keeps, to submit later with [`ShareClient::submit_solved`] (#474). + pub(crate) async fn solve(&mut self, proof: Proof) -> Result<(Value, Submitted)> { + self.session + .solve_latest(&self.username, Some(proof), false) + .await + } + + /// Submit `params`, solved earlier, and wait for the answer, recorded in + /// `submitted`. + pub(crate) async fn submit_solved( + &mut self, + params: Value, + mut submitted: Submitted, + ) -> Submitted { + self.next_id += 1; + let id = json!(self.next_id); + let request = json!({"id":id,"method":"mining.submit","params":params}); + submitted.answer = match self.session.send(&request).await { + Err(error) => Answer::Lost(format!("write: {error}")), + Ok(()) => match self.session.answer(&id, ANSWER).await { + Ok(message) if message["result"] == true && message["error"].is_null() => { + Answer::Accepted + } + Ok(message) => Answer::Rejected(message), + Err(error) if error.downcast_ref::().is_some() => Answer::TimedOut, + Err(error) => Answer::Lost(format!("{error:#}")), + }, + }; + submitted + } +} diff --git a/crates/qbit-prism-server/tests/support/live_submit_proxy.rs b/crates/qbit-prism-server/tests/support/live_submit_proxy.rs new file mode 100644 index 000000000..40e5a1245 --- /dev/null +++ b/crates/qbit-prism-server/tests/support/live_submit_proxy.rs @@ -0,0 +1,163 @@ +//! An HTTP/1.1 JSON-RPC proxy in front of a live fixture's node that +//! records every `submitblock` and, when armed, holds the next one until the +//! test releases it, while every other call passes (#575). A held call keeps +//! a found block mid-landing, its claim live, for as long as a scenario +//! needs. +use super::*; +use qbit_prism_server::codec::{double_sha256, hash_display}; +use std::sync::Arc; +use tokio::{ + io::AsyncReadExt, + net::TcpStream, + sync::{oneshot, watch}, +}; + +/// A held `submitblock`: its block and the release. +pub(crate) struct Hit { + pub block: String, + pub release: oneshot::Sender<()>, +} + +struct ProxyState { + upstream: u16, + submitted: Mutex>, + trap: Mutex>>, + /// Flipped to release every held call at cleanup. + released: watch::Sender, +} + +/// An HTTP/1.1 JSON-RPC proxy in front of the node that records every +/// `submitblock` and, when armed, holds the next one until its release while +/// every other call passes. +pub(crate) struct SubmitProxy { + pub port: u16, + state: Arc, + task: tokio::task::JoinHandle<()>, +} + +impl SubmitProxy { + pub(crate) async fn start(upstream: u16) -> Result { + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await?; + let port = listener.local_addr()?.port(); + let state = Arc::new(ProxyState { + upstream, + submitted: Mutex::new(Vec::new()), + trap: Mutex::new(None), + released: watch::Sender::new(false), + }); + let shared = state.clone(); + let task = tokio::spawn(async move { + while let Ok((downstream, _)) = listener.accept().await { + let state = shared.clone(); + tokio::spawn(async move { + let _ = relay(state, downstream).await; + }); + } + }); + Ok(Self { port, state, task }) + } + + /// Hold the next `submitblock`. + pub(crate) fn arm(&self) -> oneshot::Receiver { + let (sender, receiver) = oneshot::channel(); + *self.state.trap.lock().unwrap() = Some(sender); + receiver + } + + /// How many `submitblock` calls for `block` reached the proxy. + pub(crate) fn offers(&self, block: &str) -> usize { + self.state + .submitted + .lock() + .unwrap() + .iter() + .filter(|submitted| *submitted == block) + .count() + } + + pub(crate) fn release_all(&self) { + self.state.released.send_replace(true); + } +} + +impl Drop for SubmitProxy { + fn drop(&mut self) { + self.release_all(); + self.task.abort(); + } +} + +/// Relays one server connection, one request and reply at a time. Each +/// request goes to the node on a new connection, opened after any hold: a +/// held call can outlast qbitd's idle timeout, and must not be written into +/// a connection the node has closed meanwhile. +async fn relay(state: Arc, downstream: TcpStream) -> Result<()> { + let (down_read, mut down_write) = downstream.into_split(); + let mut down_read = BufReader::new(down_read); + loop { + let Some((request, body)) = read_http(&mut down_read).await? else { + return Ok(()); + }; + if let Some(block) = submitted_block(&body) { + state.submitted.lock().unwrap().push(block.clone()); + let trap = state.trap.lock().unwrap().take(); + if let Some(sender) = trap { + let (release, released) = oneshot::channel(); + let mut cleanup = state.released.subscribe(); + if sender.send(Hit { block, release }).is_ok() { + tokio::select! { + _ = released => {} + _ = cleanup.wait_for(|released| *released) => {} + } + } + } + } + let upstream = TcpStream::connect(("127.0.0.1", state.upstream)).await?; + let (up_read, mut up_write) = upstream.into_split(); + up_write.write_all(&request).await?; + let Some((reply, _)) = read_http(&mut BufReader::new(up_read)).await? else { + return Ok(()); + }; + down_write.write_all(&reply).await?; + } +} + +/// The block hash of a `submitblock` request body, if it is one. +fn submitted_block(body: &[u8]) -> Option { + let request: Value = serde_json::from_slice(body).ok()?; + if request["method"] != "submitblock" { + return None; + } + let block = hex::decode(request["params"][0].as_str()?).ok()?; + Some(hash_display(&double_sha256(block.get(..80)?))) +} + +/// One HTTP/1.1 message framed by `Content-Length`, as qbitd and the +/// servers' client frame every message: the raw bytes and the body. +async fn read_http(reader: &mut BufReader) -> Result, Vec)>> +where + R: tokio::io::AsyncRead + Unpin, +{ + let mut raw = Vec::new(); + let mut length = None; + loop { + let start = raw.len(); + if reader.read_until(b'\n', &mut raw).await? == 0 { + ensure!(raw.is_empty(), "connection closed inside an HTTP header"); + return Ok(None); + } + let line = std::str::from_utf8(&raw[start..])?.trim_end(); + if line.is_empty() { + break; + } + if let Some((name, value)) = line.split_once(':') { + if name.eq_ignore_ascii_case("content-length") { + length = Some(value.trim().parse::()?); + } + } + } + let mut body = vec![0; length.context("HTTP message without Content-Length")?]; + reader.read_exact(&mut body).await?; + raw.extend_from_slice(&body); + Ok(Some((raw, body))) +} diff --git a/crates/qbit-prism-server/tests/support/live_transcript_replay.rs b/crates/qbit-prism-server/tests/support/live_transcript_replay.rs new file mode 100644 index 000000000..dcf12842b --- /dev/null +++ b/crates/qbit-prism-server/tests/support/live_transcript_replay.rs @@ -0,0 +1,482 @@ +//! #575 item 3: replay recorded Stratum v1 client transcripts against a live +//! regtest server and check that every answer is one the client accepts. +//! +//! A transcript is a JSON-lines file in `tests/fixtures/stratum_transcripts` +//! (see the README there): a header line, then one line per message with its +//! `direction` (`c2s` from the miner, `s2c` from the pool it was recorded +//! against), a `timestamp` in seconds from the start, and the `message`. +//! The client's lines are replayed in order on one connection; the pool's +//! lines are the expectations. +//! +//! Replay rewrites only what the recorded pool chose and the live one must +//! choose again: the payout address in worker names becomes the fixture's +//! regtest address (the worker suffix is kept), and a submit the recorded +//! pool accepted is solved again on the live server's latest job, with its +//! extranonce2 size, share difficulty and negotiated version-rolling mask, +//! keeping the recorded shape (five fields, or six with version bits). It is +//! a share that is not a block unless the session's share difficulty is at +//! the network's, as a large `mining.suggest_difficulty` makes it on regtest. +//! Each transcript starts once every block found earlier has settled and the +//! server serves the node's tip at the cluster's payout revision; a re-solved +//! submit refused as `stale-job` or `unknown-job` (its job superseded in +//! flight) is solved again on newer work, up to three times. A +//! submit the recorded pool refused is sent as recorded. Timestamps order the +//! lines; replay does not wait out recorded gaps. +//! +//! Compatibility, per request method: `mining.subscribe` answers +//! `[subscriptions, extranonce1, extranonce2_size]`; `mining.configure` +//! answers every requested extension with a boolean, and grants version +//! rolling, within the requested mask, when the recorded pool did; +//! `mining.authorize`, `mining.extranonce.subscribe`, +//! `mining.suggest_difficulty` and `mining.submit` succeed when the recorded +//! pool's did and fail when it failed; any other method fails or succeeds as +//! it did. Every response carries `id`, `result` and `error`. Every +//! `mining.notify`, `mining.set_difficulty` and `mining.set_version_mask` +//! the live server sends has the shape miners parse, the first difficulty +//! precedes the first job, and each kind of work notification the recording +//! holds is sent live. Every accepted replayed share is in the ledger. +use super::share_client::{is_hex, reason_id, start_share_only_servers, StratumSession, ANSWER}; +use super::*; +use qbit_prism_server::codec::parse_u32_hex; +use std::{collections::BTreeSet, path::Path}; + +/// The corpus, relative to this package. +const CORPUS: &str = concat!( + env!("CARGO_MANIFEST_DIR"), + "/tests/fixtures/stratum_transcripts" +); + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +enum Direction { + ClientToServer, + ServerToClient, +} + +struct Line { + direction: Direction, + timestamp: f64, + message: Value, +} + +struct Transcript { + name: String, + source: String, + lines: Vec, +} + +impl Transcript { + /// Parse and check one transcript file. + fn load(path: &Path) -> Result { + let text = std::fs::read_to_string(path)?; + let mut rows = text + .lines() + .enumerate() + .filter(|(_, row)| !row.trim().is_empty()); + let (_, header) = rows.next().context("empty transcript")?; + let header: Value = serde_json::from_str(header)?; + let header = &header["transcript"]; + let name = header["name"] + .as_str() + .context("header needs transcript.name")? + .to_owned(); + let stem = path.file_stem().and_then(|stem| stem.to_str()); + ensure!( + stem == Some(name.as_str()), + "transcript.name must match the file name" + ); + let source = header["source"] + .as_str() + .context("header needs transcript.source")? + .to_owned(); + ensure!( + source == "synthetic" || source == "captured", + "transcript.source must be \"synthetic\" or \"captured\", not {source:?}" + ); + for field in ["dialect", "notes"] { + ensure!( + header[field] + .as_str() + .is_some_and(|value| !value.is_empty()), + "header needs transcript.{field}" + ); + } + let mut lines = Vec::new(); + let mut previous = 0.0f64; + for (number, row) in rows { + let parsed: Value = serde_json::from_str(row) + .with_context(|| format!("line {}: not JSON", number + 1))?; + let direction = match parsed["direction"].as_str() { + Some("c2s") => Direction::ClientToServer, + Some("s2c") => Direction::ServerToClient, + other => bail!("line {}: direction {other:?} is not c2s or s2c", number + 1), + }; + let timestamp = parsed["timestamp"] + .as_f64() + .with_context(|| format!("line {}: timestamp must be seconds", number + 1))?; + ensure!( + timestamp >= previous, + "line {}: timestamp {timestamp} precedes {previous}", + number + 1 + ); + previous = timestamp; + let message = parsed["message"].clone(); + ensure!( + message.is_object(), + "line {}: message must be an object", + number + 1 + ); + lines.push(Line { + direction, + timestamp, + message, + }); + } + let transcript = Self { + name, + source, + lines, + }; + // Every client request with an id has the recorded answer that is + // its expectation. + for line in transcript.client_lines() { + if !line.message["id"].is_null() && line.message.get("method").is_some() { + transcript + .recorded_answer(&line.message["id"]) + .with_context(|| format!("request {} has no recorded answer", line.message))?; + } + } + Ok(transcript) + } + + fn client_lines(&self) -> impl Iterator { + self.lines + .iter() + .filter(|line| line.direction == Direction::ClientToServer) + } + + fn recorded_answer(&self, id: &Value) -> Option<&Value> { + self.lines + .iter() + .filter(|line| line.direction == Direction::ServerToClient) + .map(|line| &line.message) + .find(|message| message.get("method").is_none() && &message["id"] == id) + } + + /// The notification methods the recorded pool sent. + fn recorded_notifications(&self) -> BTreeSet { + self.lines + .iter() + .filter(|line| line.direction == Direction::ServerToClient) + .filter_map(|line| line.message["method"].as_str().map(str::to_owned)) + .collect() + } +} + +fn corpus() -> Result> { + let mut paths: Vec<_> = std::fs::read_dir(CORPUS)? + .map(|entry| entry.map(|entry| entry.path())) + .collect::>()?; + paths.retain(|path| { + path.extension() + .is_some_and(|extension| extension == "jsonl") + }); + paths.sort(); + ensure!(!paths.is_empty(), "no transcripts in {CORPUS}"); + paths + .iter() + .map(|path| Transcript::load(path).with_context(|| format!("{}", path.display()))) + .collect() +} + +/// The corpus parses and every file is labelled; runs without any input. +#[test] +fn stratum_transcript_corpus_is_well_formed_and_labelled() -> Result<()> { + let transcripts = corpus()?; + for transcript in &transcripts { + ensure!( + transcript.client_lines().count() > 0, + "{}: no client lines", + transcript.name + ); + } + Ok(()) +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn recorded_stratum_transcripts_replay_compatibly_against_a_live_server() -> Result<()> { + let transcripts = corpus()?; + let Some(mut fixture) = Fixture::open_with_servers(false, false).await? else { + return Ok(()); + }; + let result = async { + start_share_only_servers(&mut fixture, &[(0, Vec::new())]).await?; + let mut accepted = Vec::new(); + let mut report = Vec::new(); + for transcript in &transcripts { + let replayed = replay(&fixture, transcript).await.with_context(|| { + format!("transcript {} ({})", transcript.name, transcript.source) + })?; + report.push(format!( + "{} ({}, {:.1}s recorded): {} requests, {} shares accepted{}", + transcript.name, + transcript.source, + transcript.lines.last().map_or(0.0, |line| line.timestamp), + replayed.requests, + replayed.accepted.len(), + if replayed.not_sent.is_empty() { + String::new() + } else { + format!("; recorded but not sent live: {:?}", replayed.not_sent) + } + )); + accepted.extend(replayed.accepted); + } + let ledger: BTreeSet = + sqlx::query_scalar("SELECT share_id FROM qbit_share_ledger WHERE accepted") + .fetch_all(&fixture.pool) + .await? + .into_iter() + .collect(); + let missing: Vec<_> = accepted.iter().filter(|id| !ledger.contains(*id)).collect(); + ensure!( + missing.is_empty(), + "accepted replayed shares missing: {missing:?}" + ); + eprintln!("transcript replay:\n {}", report.join("\n ")); + Ok::<_, anyhow::Error>(()) + } + .await; + if result.is_err() { + eprintln!("{}", fixture.diagnostics()); + } + let cleanup = fixture.cleanup().await; + result.and(cleanup) +} + +/// What one transcript's replay did. +struct Replayed { + requests: usize, + /// Share ids of accepted replayed submits. + accepted: Vec, + /// Notification methods the recorded pool sent that the live one did not. + not_sent: BTreeSet, +} + +/// The fixture's regtest address with the recorded worker suffix, which +/// keeps only characters a worker name needs. +fn live_username(fixture: &Fixture, recorded: &Value) -> String { + let suffix = recorded + .as_str() + .and_then(|name| name.rsplit_once('.')) + .map(|(_, worker)| worker) + .filter(|worker| { + !worker.is_empty() + && worker + .chars() + .all(|c| c.is_ascii_alphanumeric() || c == '-' || c == '_') + }) + .unwrap_or("replay"); + format!("{}.{suffix}", fixture.address) +} + +/// How often a re-solved submit refused as superseded is solved again. +const STALE_RETRIES: usize = 3; + +/// Whether a refusal says the job was superseded, not that the share or the +/// dialect was wrong. +fn superseded(answer: &Value) -> bool { + matches!(reason_id(answer), Some("stale-job" | "unknown-job")) +} + +/// Until the server is ready on the node's current tip, so a transcript +/// starts on current work: a block a previous transcript found is built on, +/// and has landed, so a recorded accepted submit cannot meet the revision +/// fence ([`Fixture::settled`]). +async fn current(fixture: &Fixture) -> Result<()> { + fixture.settled(0, 30).await +} + +async fn replay(fixture: &Fixture, transcript: &Transcript) -> Result { + current(fixture).await?; + let mut live = StratumSession::open(fixture.stratum[0]).await?; + let mut replayed = Replayed { + requests: 0, + accepted: Vec::new(), + not_sent: BTreeSet::new(), + }; + let mut username = None; + for line in transcript.client_lines() { + let recorded = &line.message; + let method = recorded["method"].as_str().unwrap_or_default(); + let id = recorded["id"].clone(); + let expected = transcript.recorded_answer(&id).cloned(); + let mut message = recorded.clone(); + // Set for a submit the recorded pool accepted: the worker name and + // whether it rolls versions. It is solved again on live work. + let mut resolve = None; + match method { + "mining.authorize" => { + let name = live_username(fixture, &recorded["params"][0]); + message["params"][0] = json!(name); + username = Some(name); + } + "mining.submit" => { + let name = username + .clone() + .unwrap_or_else(|| live_username(fixture, &recorded["params"][0])); + message["params"][0] = json!(name); + if expected + .as_ref() + .is_some_and(|answer| answer["result"] == true) + { + let rolls = recorded["params"].as_array().map_or(0, Vec::len) >= 6; + resolve = Some((name, rolls)); + } + } + _ => {} + } + let mut share_id = None; + if let Some((name, rolls)) = &resolve { + let (params, submitted) = live.solve_latest(name, None, *rolls).await?; + message["params"] = params; + share_id = Some(submitted.share_id); + } + live.send(&message).await?; + if id.is_null() { + continue; + } + replayed.requests += 1; + let mut answer = live.answer(&id, ANSWER).await?; + // A re-solved submit whose job a new tip superseded in flight is + // solved again on the newer work, as a miner's next share would be. + if let Some((name, rolls)) = &resolve { + for _ in 0..STALE_RETRIES { + if !superseded(&answer) { + break; + } + live.newer_work().await?; + let (params, submitted) = live.solve_latest(name, None, *rolls).await?; + message["params"] = params; + share_id = Some(submitted.share_id); + live.send(&message).await?; + answer = live.answer(&id, ANSWER).await?; + } + } + let expected = expected.context("no recorded answer")?; + compatible(method, recorded, &expected, &answer) + .with_context(|| format!("{method} answered {answer}, recorded {expected}"))?; + live.learn(method, &answer)?; + if let Some(share_id) = share_id { + replayed.accepted.push(share_id); + } + } + // Work the recording received must be sent live too, difficulty first. + let recorded = transcript.recorded_notifications(); + if recorded.contains("mining.notify") { + live.work().await?; + } + let position = |method: &str| live.notifications.iter().position(|sent| sent == method); + if let (Some(difficulty), Some(job)) = + (position("mining.set_difficulty"), position("mining.notify")) + { + ensure!( + difficulty < job, + "the first mining.notify preceded the first mining.set_difficulty" + ); + } + for method in &recorded { + if !live.notifications.contains(method) { + ensure!( + !matches!(method.as_str(), "mining.notify" | "mining.set_difficulty"), + "the live server never sent {method}" + ); + replayed.not_sent.insert(method.clone()); + } + } + Ok(replayed) +} + +/// Whether the live `answer` to `request` is one the recorded client, which +/// got `expected`, accepts. See the module documentation. +fn compatible(method: &str, request: &Value, expected: &Value, answer: &Value) -> Result<()> { + let object = answer.as_object().context("answer is not an object")?; + ensure!( + object.contains_key("result") && object.contains_key("error"), + "answer lacks result or error" + ); + let succeeded = |message: &Value| message["error"].is_null() && message["result"] != false; + let result = &answer["result"]; + match method { + "mining.subscribe" => { + let parts = result + .as_array() + .context("subscribe result is not an array")?; + ensure!(parts.len() >= 3, "subscribe result has fewer than 3 parts"); + ensure!(parts[0].is_array(), "subscriptions are not an array"); + ensure!(is_hex(&parts[1], None), "extranonce1 is not hex"); + ensure!( + parts[2] + .as_u64() + .is_some_and(|size| (1..=16).contains(&size)), + "extranonce2_size is not 1..=16" + ); + } + "mining.configure" => { + ensure!(answer["error"].is_null(), "configure failed"); + let granted = result + .as_object() + .context("configure result is not an object")?; + for extension in request["params"][0].as_array().into_iter().flatten() { + let extension = extension.as_str().context("extension is not a string")?; + ensure!( + granted.get(extension).is_some_and(Value::is_boolean), + "{extension} not answered with a boolean" + ); + } + if expected["result"]["version-rolling"] == true { + ensure!( + result["version-rolling"] == true, + "version rolling refused where the recorded pool granted it" + ); + ensure!( + is_hex(&result["version-rolling.mask"], Some(8)), + "mask is not 8 hex digits" + ); + let mask = + parse_u32_hex(result["version-rolling.mask"].as_str().unwrap_or_default())?; + let requested = request["params"][1]["version-rolling.mask"] + .as_str() + .map_or(Ok(u32::MAX), parse_u32_hex)?; + ensure!(mask != 0, "an empty version-rolling mask"); + ensure!( + mask & !requested == 0, + "mask {mask:08x} exceeds the requested {requested:08x}" + ); + } + } + _ => { + ensure!( + succeeded(answer) == succeeded(expected), + "{} where the recorded pool {}", + if succeeded(answer) { + "succeeded" + } else { + "failed" + }, + if succeeded(expected) { + "succeeded" + } else { + "failed" + } + ); + if matches!( + method, + "mining.authorize" | "mining.submit" | "mining.extranonce.subscribe" + ) && expected["result"] == true + { + ensure!(*result == true, "result is not true"); + } + } + } + Ok(()) +} diff --git a/crates/qbit-prism-server/tests/support/live_two_node.rs b/crates/qbit-prism-server/tests/support/live_two_node.rs new file mode 100644 index 000000000..1aa5d49fb --- /dev/null +++ b/crates/qbit-prism-server/tests/support/live_two_node.rs @@ -0,0 +1,1001 @@ +//! Own blocks through real two-node partitions (#521 scenarios 1 and 2). +//! +//! The fixture's node (A) serves the pool. A second regtest node (B) is +//! peered with it: B listens on loopback and A, started `-listen=0`, connects +//! out to it with `addnode … onetry`. A partition is B's +//! `setnetworkactive false`, which drops the connection and refuses a new +//! one; healing turns B's network back on and reconnects A. Blocks are mined +//! on a chosen node with `generatetoaddress`, and the pool's own blocks are +//! block-only proofs submitted through a frontend's highdiff port, so each +//! scenario controls exactly which own blocks exist and on which branch. +use super::highdiff_tests::HighdiffClient; +use super::*; + +/// How long a connected miner may wait for work on a new tip. The native +/// current-work-gap warning fires at 15 s; the #413 stall held delivery for +/// 307 s. Each wait is measured from just before the call that moves the +/// pool's node to the new tip. +const NOTIFY_BOUND: Duration = Duration::from_secs(15); + +/// `PRISM_CANDIDATE_ORPHAN_CONFIRMATIONS` as the fixture runs it: the +/// servers start without the variable, so this is the production default. +const ORPHAN_CONFIRMATIONS: u64 = 6; + +/// The second regtest node. +pub(super) struct PeerNode { + process: Process, + pub(super) rpc_port: u16, + p2p_port: u16, + client: reqwest::Client, +} + +impl PeerNode { + pub(super) async fn start(fixture: &Fixture) -> Result { + let directory = fixture.directory.path().join("node-b"); + std::fs::create_dir_all(&directory)?; + let rpc_port = fixture.ports.reserve()?; + let p2p_port = fixture.ports.reserve()?; + let mut command = Command::new(&fixture.qbitd); + command + .args([ + "-regtest", + "-server=1", + "-listen=1", + "-bind=127.0.0.1", + "-dnsseed=0", + "-discover=0", + "-rpcuser=prismtest", + "-rpcpassword=prismtest", + "-txindex=0", + ]) + .arg(format!("-datadir={}", directory.display())) + .arg(format!("-rpcport={rpc_port}")) + .arg(format!("-port={p2p_port}")); + fixture.ports.release(&[rpc_port, p2p_port]); + let node = Self { + process: Process::spawn(&mut command, fixture.directory.path().join("qbit-b.log"))?, + rpc_port, + p2p_port, + client: fixture.client.clone(), + }; + let ready = until("second qbit RPC", 30, || async { + Ok(node.rpc("getblockchaininfo", json!([])).await?["chain"] == "regtest") + }) + .await; + match ready { + Ok(()) => Ok(node), + Err(error) => Err(error.context(node.diagnostics())), + } + } + + pub(super) async fn rpc(&self, method: &str, params: Value) -> Result { + let payload: Value = self + .client + .post(format!("http://127.0.0.1:{}/", self.rpc_port)) + .basic_auth("prismtest", Some("prismtest")) + .json(&json!({"jsonrpc":"1.0","id":"live-test-b","method":method,"params":params})) + .send() + .await? + .json() + .await?; + ensure!( + payload["error"].is_null(), + "node B RPC {method}: {}", + payload["error"] + ); + Ok(payload["result"].clone()) + } + + pub(super) async fn best(&self) -> Result { + best_of(self.rpc("getbestblockhash", json!([])).await?) + } + + pub(super) async fn mine(&self, blocks: u64, address: &str) -> Result> { + mined( + self.rpc("generatetoaddress", json!([blocks, address])) + .await?, + ) + } + + /// Connect A to B and wait until both hold the same active tip. + pub(super) async fn heal(&self, fixture: &Fixture) -> Result { + self.rpc("setnetworkactive", json!([true])).await?; + let address = format!("127.0.0.1:{}", self.p2p_port); + until("node A connected to node B", 30, || async { + if fixture.rpc("getconnectioncount", json!([])).await? == 0 { + // `onetry` makes one attempt; repeat it until one holds. + fixture + .rpc("addnode", json!([address.as_str(), "onetry"])) + .await?; + return Ok(false); + } + Ok(self.rpc("getconnectioncount", json!([])).await? != 0) + }) + .await?; + until("nodes A and B on one active tip", 30, || async { + Ok(node_a_best(fixture).await? == self.best().await?) + }) + .await?; + node_a_best(fixture).await + } + + pub(super) async fn partition(&self, fixture: &Fixture) -> Result<()> { + self.rpc("setnetworkactive", json!([false])).await?; + until("nodes A and B partitioned", 30, || async { + Ok(fixture.rpc("getconnectioncount", json!([])).await? == 0 + && self.rpc("getconnectioncount", json!([])).await? == 0) + }) + .await + } + + fn diagnostics(&self) -> String { + diagnostics::report(&[("qbit node b".to_owned(), Some(&self.process))], None) + } +} + +fn best_of(value: Value) -> Result { + Ok(value + .as_str() + .context("best block hash missing")? + .to_owned()) +} + +fn mined(value: Value) -> Result> { + value + .as_array() + .context("generated block hashes missing")? + .iter() + .map(|hash| Ok(hash.as_str().context("invalid block hash")?.to_owned())) + .collect() +} + +fn count(value: Value) -> Result { + value.as_i64().context("block count missing") +} + +async fn height(fixture: &Fixture, hash: &str) -> Result { + count(fixture.rpc("getblockheader", json!([hash])).await?["height"].clone()) +} + +async fn new_address(fixture: &Fixture) -> Result { + Ok(fixture + .rpc("getnewaddress", json!(["", "p2mr"])) + .await? + .as_str() + .context("wallet address missing")? + .to_owned()) +} + +pub(super) async fn node_a_best(fixture: &Fixture) -> Result { + best_of(fixture.rpc("getbestblockhash", json!([])).await?) +} + +pub(super) async fn node_a_mine(fixture: &Fixture, blocks: u64) -> Result> { + mined( + fixture + .rpc("generatetoaddress", json!([blocks, fixture.address])) + .await?, + ) +} + +/// Start frontend `index` against the node whose RPC listens on `rpc_port`. +pub(super) fn start_frontend(fixture: &Fixture, index: usize, rpc_port: u16) -> Result { + fixture.start_server_with(index, None, &[("QBIT_RPC_PORT", rpc_port.to_string())]) +} + +/// A check [`server_ready`] also waits on once the server is ready: #553 +/// installs one while a session load runs, so a scenario's next step comes +/// under the whole load. `None` otherwise; live cases run one at a time. +pub(super) type ReadyGate = std::sync::Arc bool + Send + Sync>; +pub(super) static READY_GATE: std::sync::Mutex> = std::sync::Mutex::new(None); + +/// A session load's share offers, which #553 installs while a load runs and +/// [`deep_reorg`] pauses around its own blocks. `None` otherwise. +#[derive(Clone)] +pub(super) struct LoadShares { + /// Pause (`true`) or resume (`false`) the load's share offers. + pub pause: std::sync::Arc, + /// Whether no load share waits for its answer. + pub answered: std::sync::Arc bool + Send + Sync>, +} +pub(super) static LOAD_SHARES: std::sync::Mutex> = std::sync::Mutex::new(None); + +/// Pause a running load's share offers, until every offered share has its +/// answer and the share ledger has stopped growing, or resume them. Nothing +/// without a load. +async fn pause_load_shares(fixture: &Fixture, paused: bool) -> Result<()> { + let shares = LOAD_SHARES + .lock() + .map_err(|_| anyhow::anyhow!("load shares poisoned"))? + .clone(); + let Some(shares) = shares else { + return Ok(()); + }; + (shares.pause)(paused); + if !paused { + return Ok(()); + } + // A stuck share is cleared on the session's first tick past its answer + // bound, so allow the bound twice: ticks up to the bound apart. + let bound = 2 * super::share_client::ANSWER.as_secs(); + until("the load's offered shares answered", bound, || async { + Ok((shares.answered)()) + }) + .await?; + // A share whose answer was lost, or `ledger-outcome-unknown`, may still + // commit: wait until the ledger holds still for two seconds. + let deadline = Instant::now() + Duration::from_secs(60); + let mut last = None; + loop { + let rows: i64 = sqlx::query_scalar("SELECT count(*) FROM qbit_share_ledger") + .fetch_one(&fixture.pool) + .await?; + if last == Some(rows) { + return Ok(()); + } + ensure!( + Instant::now() < deadline, + "the share ledger kept growing after the load's shares paused" + ); + last = Some(rows); + tokio::time::sleep(Duration::from_secs(2)).await; + } +} + +pub(super) async fn server_ready(fixture: &Fixture, index: usize) -> Result<()> { + until( + &format!("PRISM readiness of server {index}"), + 60, + || async { + Ok(fixture + .client + .get(format!("http://127.0.0.1:{}/healthz", fixture.api[index])) + .send() + .await? + .status() + .is_success()) + }, + ) + .await?; + let gate = READY_GATE + .lock() + .map_err(|_| anyhow::anyhow!("ready gate poisoned"))? + .clone(); + match gate { + Some(gate) => until("the ready gate", 120, || async { Ok(gate()) }).await, + None => Ok(()), + } +} + +/// Wait until `client` holds work on `parent`, and require that it arrived +/// within `NOTIFY_BOUND` of `changed`, taken just before the call that moved +/// the pool's node to that tip, so the bound is never measured short. +async fn delivered(client: &mut HighdiffClient, parent: &str, changed: Instant) -> Result<()> { + client + .wait_for_parent(parent) + .await + .with_context(|| format!("no mining.notify on new tip {parent}"))?; + let latency = changed.elapsed(); + ensure!( + latency <= NOTIFY_BOUND, + "new-tip work for {parent} reached the miner {latency:?} after the tip changed, over the {NOTIFY_BOUND:?} bound" + ); + eprintln!("two-node: new-tip work for {parent} delivered in {latency:?}"); + Ok(()) +} + +/// One own block: a block-only proof by worker `username` on frontend +/// `index`'s work for `parent`, acknowledged only once the block is durably +/// credited on the active chain. +pub(super) async fn own_block( + fixture: &Fixture, + index: usize, + username: &str, + parent: &str, + id: u64, +) -> Result { + let mut miner = HighdiffClient::open_as(fixture, index, username.to_owned()).await?; + miner.wait_for_parent(parent).await?; + let (request, hash, _) = miner.solve(id, false)?; + miner.send(request).await?; + let response = tokio::time::timeout(Duration::from_secs(60), miner.response(id)) + .await + .with_context(|| format!("own block {hash} was not acknowledged"))??; + ensure!( + response["result"] == true && response["error"].is_null(), + "own block {hash} rejected: {response}" + ); + Ok(hash) +} + +pub(super) async fn chain_state(fixture: &Fixture, hash: &str) -> Result> { + Ok( + sqlx::query_scalar("SELECT chain_state FROM qbit_pool_blocks WHERE block_hash=$1") + .bind(hash) + .fetch_optional(&fixture.pool) + .await?, + ) +} + +async fn chain_states_are(fixture: &Fixture, hashes: &[&str], state: &str) -> Result { + for hash in hashes { + if chain_state(fixture, hash).await?.as_deref() != Some(state) { + return Ok(false); + } + } + Ok(true) +} + +/// Accepted ledger rows for the share a block-only proof of `hash` defers. +pub(super) async fn credits(fixture: &Fixture, username: &str, hash: &str) -> Result { + Ok( + sqlx::query_scalar("SELECT count(*) FROM qbit_share_ledger WHERE accepted AND share_id=$1") + .bind(format!("{username}:{hash}")) + .fetch_one(&fixture.pool) + .await?, + ) +} + +pub(super) async fn assert_no_duplicate_headers(fixture: &Fixture) -> Result<()> { + // One statement, so one snapshot: under session load (#553) shares land + // between two separate counts. An append writes both rows in one + // transaction. + let (accepted, unique): (i64, i64) = sqlx::query_as( + "SELECT (SELECT count(*) FROM qbit_share_ledger WHERE accepted),\ + (SELECT count(*) FROM qbit_prism_share_hashes)", + ) + .fetch_one(&fixture.pool) + .await?; + ensure!( + accepted == unique, + "{accepted} accepted rows for {unique} headers: a header was credited twice" + ); + Ok(()) +} + +/// What each of `miners` is owed as the public miner dashboard reports it +/// (carry-forward, lifetime earnings of confirmed blocks, and payouts +/// pending maturity), which every running frontend must agree on, and the +/// cluster's carry-forward balances by program. +async fn balances( + fixture: &Fixture, + miners: &[String], +) -> Result<(Vec, Vec<(String, String)>)> { + let mut reported = Vec::new(); + for miner in miners { + let mut views = Vec::new(); + for (index, server) in fixture.servers.iter().enumerate() { + if server.child.try_wait()?.is_some() { + continue; + } + let view: Value = fixture + .client + .get(format!( + "http://127.0.0.1:{}/public/v1/miners/{miner}", + fixture.api[index] + )) + .send() + .await? + .error_for_status()? + .json() + .await?; + views.push(json!({ + "miner": miner, + "owed": view["owed_balance_bits"], + "lifetime": view["lifetime_earnings_bits"], + "pending": view["pending_maturity_bits"], + })); + } + let first = views.first().context("no running frontend")?.clone(); + ensure!( + views.iter().all(|view| *view == first), + "frontends disagree on {miner}'s balances: {views:?}" + ); + reported.push(first); + } + let carry = sqlx::query_as( + "SELECT encode(p2mr_program,'hex'),balance_sats::text FROM qbit_current_carry_forward_balances() ORDER BY 1", + ) + .fetch_all(&fixture.pool) + .await?; + Ok((reported, carry)) +} + +#[derive(Debug)] +struct CandidateRow { + state: String, + outcome: Option, + reply: Option, + last_error: Option, + attempts: i32, + claimed: bool, +} + +async fn candidate(fixture: &Fixture, hash: &str) -> Result { + let row = sqlx::query("SELECT state,offer_outcome,offer_reply,last_error,attempt_count,claim_token IS NOT NULL AS claimed FROM qbit_block_candidate_outbox WHERE block_hash=$1") + .bind(hash) + .fetch_one(&fixture.pool) + .await?; + Ok(CandidateRow { + state: row.try_get("state")?, + outcome: row.try_get("offer_outcome")?, + reply: row.try_get("offer_reply")?, + last_error: row.try_get("last_error")?, + attempts: row.try_get("attempt_count")?, + claimed: row.try_get("claimed")?, + }) +} + +/// The candidate and landing gauges of one frontend's `/metrics`, the +/// inputs of the native candidate and landing pages. +#[derive(Debug)] +struct CandidateGauges { + pending: f64, + unacknowledged: f64, + landing_failed: f64, + revision_work_pending: f64, + orphaned_total: f64, +} + +async fn candidate_gauges(fixture: &Fixture, index: usize) -> Result { + let body = fixture + .client + .get(format!("http://127.0.0.1:{}/metrics", fixture.api[index])) + .send() + .await? + .error_for_status()? + .text() + .await?; + let sample = |name: &str| -> Result { + let name = format!("qbit_prism_{name}"); + body.lines() + .find_map(|line| { + let (metric, value) = line.split_once(' ')?; + (metric == name).then(|| value.trim().parse::()) + }) + .with_context(|| format!("{name} missing from /metrics"))? + .with_context(|| format!("{name} is not a number")) + }; + Ok(CandidateGauges { + pending: sample("block_candidates_pending")?, + unacknowledged: sample("block_candidate_oldest_unacknowledged_seconds")?, + landing_failed: sample("block_candidate_oldest_landing_failed_seconds")?, + revision_work_pending: sample("accepted_block_revision_work_pending_seconds")?, + orphaned_total: sample("block_candidates_orphaned_total")?, + }) +} + +/// Fail on a sample at a paging threshold of the native rules: the +/// candidate critical page at an unacknowledged age of 60 s, and the +/// landing-failed and revision-work critical pages at 307 s. With +/// `acknowledged`, the lost race is known to be settled and acknowledged, +/// so its unacknowledged and landing-failed ages must be exactly zero; an +/// unknown (-1) reading fails too. +fn unpaged(gauges: &CandidateGauges, acknowledged: bool) -> Result<()> { + ensure!( + gauges.unacknowledged < 60.0 + && gauges.landing_failed < 307.0 + && gauges.revision_work_pending < 307.0, + "a candidate or landing gauge reached its paging threshold: {gauges:?}" + ); + ensure!( + !acknowledged || (gauges.unacknowledged == 0.0 && gauges.landing_failed == 0.0), + "the acknowledged lost race counts toward a paging age: {gauges:?}" + ); + Ok(()) +} + +/// `until`, sampling frontend 0's candidate gauges on every poll. Unlike an +/// error inside an `until` condition, which is only retried, a paging +/// sample (see `unpaged`) fails at once. +async fn until_unpaged( + fixture: &Fixture, + label: &str, + seconds: u64, + acknowledged: bool, + mut condition: F, +) -> Result<()> +where + F: FnMut(CandidateGauges) -> Fut, + Fut: Future>, +{ + let started = Instant::now(); + let mut last = None; + loop { + match candidate_gauges(fixture, 0).await { + Ok(gauges) => { + unpaged(&gauges, acknowledged)?; + match condition(gauges).await { + Ok(true) => return Ok(()), + Ok(false) => {} + Err(error) => last = Some(error), + } + } + Err(error) => last = Some(error), + } + if started.elapsed() > Duration::from_secs(seconds) { + bail!( + "timed out waiting for {label}: {}", + last.map_or_else(|| "condition not met".into(), |e| e.to_string()) + ); + } + tokio::time::sleep(Duration::from_millis(200)).await; + } +} + +/// Pause the offer of any reserved candidate at its last database step +/// before `submitblock`, the claim renewal at the external boundary, until +/// the returned transaction ends. The renewal times out after 5 s, so the +/// hold must stay short. No settlement or cluster lock is held meanwhile. +async fn hold_offers( + fixture: &Fixture, +) -> Result<(sqlx::Transaction<'static, sqlx::Postgres>, i64)> { + let key = i64::from(rand_key()); + sqlx::raw_sql(&format!( + "CREATE FUNCTION test_b521_hold_offer() RETURNS trigger LANGUAGE plpgsql AS $$ BEGIN IF OLD.state='offer_reserved' AND NEW.state='offer_reserved' AND NEW.claim_token IS NOT DISTINCT FROM OLD.claim_token THEN PERFORM pg_advisory_xact_lock({key}); END IF; RETURN NEW; END $$; CREATE TRIGGER test_b521_hold_offer BEFORE UPDATE OF claim_expires_at ON qbit_block_candidate_outbox FOR EACH ROW EXECUTE FUNCTION test_b521_hold_offer();" + )) + .execute(&fixture.pool) + .await?; + let mut hold = fixture.pool.begin().await?; + sqlx::query("SELECT pg_advisory_xact_lock($1)") + .bind(key) + .execute(&mut *hold) + .await?; + Ok((hold, key)) +} + +async fn release_offers( + fixture: &Fixture, + hold: sqlx::Transaction<'static, sqlx::Postgres>, +) -> Result<()> { + hold.rollback().await?; + sqlx::raw_sql( + "DROP TRIGGER test_b521_hold_offer ON qbit_block_candidate_outbox; DROP FUNCTION test_b521_hold_offer();", + ) + .execute(&fixture.pool) + .await?; + Ok(()) +} + +async fn offer_held(fixture: &Fixture, key: i64) -> Result { + Ok(sqlx::query_scalar( + "SELECT EXISTS(SELECT 1 FROM pg_locks WHERE locktype='advisory' AND NOT granted AND classid=0 AND objid::bigint=$1 AND objsubid=1)", + ) + .bind(key) + .fetch_one(&fixture.pool) + .await?) +} + +/// A positive advisory key below 2^31, so it is the lock's `objid`. +fn rand_key() -> u32 { + (Uuid::new_v4().as_u128() as u32) & 0x7fff_ffff | 1 +} + +/// Scenario 1. The pool's own block reaches the pool's node after a +/// same-height competitor mined on the other side of a partition: the node +/// answers `inconclusive`, the block is a side-chain block, and the +/// competitor's chain then buries it. The offer is held at its +/// `submitblock` fence while the competitor reaches node A (by +/// `submitblock`, as a relayed block would, so the arrival order is exact), +/// which is the pool's real window between its supersession check and its +/// offer; nothing else is simulated. The candidate must stay acknowledged and unpaged while it +/// waits for its orphan proof, settle `orphaned` exactly at the default +/// confirmation depth, never stall job delivery, and be credited once when +/// a reorg back activates it. +pub(super) async fn lost_race(fixture: &mut Fixture, peer: &PeerNode) -> Result<()> { + let base = peer.heal(fixture).await?; + fixture.servers.push(fixture.start_server(0)?); + server_ready(fixture, 0).await?; + let mut watcher = HighdiffClient::open(fixture, 0).await?; + let mut miner = HighdiffClient::open(fixture, 0).await?; + miner.wait_for_parent(&base).await?; + watcher.wait_for_parent(&base).await?; + peer.partition(fixture).await?; + let competitor = peer.mine(1, &fixture.address).await?.remove(0); + + // Our block on the same parent, offered only after the competitor is + // node A's tip. + let (request, own, _) = miner.solve(10, false)?; + let (hold, key) = hold_offers(fixture).await?; + miner.send(request).await?; + until( + "own block offer held at its submitblock fence", + 30, + || async { offer_held(fixture, key).await }, + ) + .await?; + ensure!( + candidate(fixture, &own).await?.state == "offer_reserved", + "held candidate is not reserved" + ); + let competitor_bytes = peer.rpc("getblock", json!([competitor, 0])).await?; + let changed = Instant::now(); + let reply = fixture + .rpc("submitblock", json!([competitor_bytes])) + .await?; + ensure!(reply.is_null(), "node A refused the competitor: {reply}"); + ensure!( + node_a_best(fixture).await? == competitor, + "competitor is not node A's tip" + ); + release_offers(fixture, hold).await?; + delivered(&mut watcher, &competitor, changed).await?; + + until("lost race settled in reconciliation", 30, || async { + Ok(candidate(fixture, &own).await?.state == "reconciliation") + }) + .await?; + let settled = candidate(fixture, &own).await?; + ensure!( + settled.outcome.as_deref() == Some("rejected") + && settled.reply.as_deref() == Some("inconclusive"), + "the node did not answer the lost race as a side-chain block: {settled:?}" + ); + ensure!( + chain_state(fixture, &own) + .await? + .is_some_and(|state| state != "confirmed"), + "the lost race's audit did not land, or its block is credited" + ); + ensure!( + credits(fixture, &miner.username, &own).await? == 0, + "a side-chain block's deferred share was credited" + ); + // The row is unfinished but acknowledged: the node holds the block. It + // counts as pending and never toward the unacknowledged or + // landing-failed ages that page. + // Samples taken before the settlement may still show the reservation's + // age; the first one showing the row acknowledged is after it, and from + // then on every sample must show zero for both paging ages. + until_unpaged( + fixture, + "census of the acknowledged lost race", + 20, + false, + |gauges| async move { Ok(gauges.pending == 1.0 && gauges.unacknowledged == 0.0) }, + ) + .await?; + unpaged(&candidate_gauges(fixture, 0).await?, true)?; + + // The competitor's chain reaches one confirmation short of the orphan + // proof: the row must survive a retry at that depth. + peer.heal(fixture).await?; + let changed = Instant::now(); + let hashes = peer + .mine(ORPHAN_CONFIRMATIONS - 2, &fixture.address) + .await?; + let short = hashes.last().context("no block mined")?.clone(); + until("node A on the deeper competitor chain", 30, || async { + Ok(node_a_best(fixture).await? == short) + }) + .await?; + delivered(&mut watcher, &short, changed).await?; + let before = candidate(fixture, &own).await?.attempts; + until_unpaged( + fixture, + "a reconciliation retry one confirmation short", + 60, + true, + |_| async { + let row = candidate(fixture, &own).await?; + Ok(row.attempts > before && !row.claimed) + }, + ) + .await?; + let retried = candidate(fixture, &own).await?; + ensure!( + retried.state == "reconciliation", + "a competitor with {} confirmations settled the row: {retried:?}", + ORPHAN_CONFIRMATIONS - 1 + ); + + // Exactly the default depth proves the orphan. + let changed = Instant::now(); + let proof = peer.mine(1, &fixture.address).await?.remove(0); + until("node A at the orphan-proof depth", 30, || async { + Ok(node_a_best(fixture).await? == proof) + }) + .await?; + delivered(&mut watcher, &proof, changed).await?; + until("lost race settled orphaned", 90, || async { + Ok(candidate(fixture, &own).await?.state == "orphaned") + }) + .await?; + let orphaned = candidate(fixture, &own).await?; + ensure!( + orphaned.last_error.as_deref().is_some_and( + |reason| reason.contains(&format!("with {ORPHAN_CONFIRMATIONS} confirmations")) + ), + "orphan proof at an unexpected depth: {orphaned:?}" + ); + until_unpaged( + fixture, + "census after the orphan proof", + 20, + false, + |gauges| async move { Ok(gauges.pending == 0.0 && gauges.orphaned_total == 1.0) }, + ) + .await?; + ensure!( + credits(fixture, &miner.username, &own).await? == 0, + "an orphaned block's deferred share was credited" + ); + fixture.integrity().await?; + + // A reorg back: node A invalidates the competitor, extends our block + // past the competitor's chain, and node B follows. + fixture.rpc("invalidateblock", json!([competitor])).await?; + ensure!( + node_a_best(fixture).await? == own, + "node A did not fall back to the pool's block" + ); + let changed = Instant::now(); + let reorg = node_a_mine(fixture, ORPHAN_CONFIRMATIONS).await?; + let back = reorg.last().context("no block mined")?.clone(); + until("node B reorged back onto the pool's block", 30, || async { + Ok(peer.best().await? == back) + }) + .await?; + delivered(&mut watcher, &back, changed).await?; + until("reactivated own block credited", 30, || async { + Ok( + chain_state(fixture, &own).await?.as_deref() == Some("confirmed") + && credits(fixture, &miner.username, &own).await? == 1, + ) + }) + .await?; + // `reconsiderblock` makes the competitor's chain valid again; it has + // less work, so nothing moves, and a further tip changes nothing either. + fixture.rpc("reconsiderblock", json!([competitor])).await?; + let changed = Instant::now(); + let next = peer.mine(1, &fixture.address).await?.remove(0); + until("node A on the next tip", 30, || async { + Ok(node_a_best(fixture).await? == next) + }) + .await?; + delivered(&mut watcher, &next, changed).await?; + ensure!( + credits(fixture, &miner.username, &own).await? == 1, + "the reactivated block was credited more than once" + ); + ensure!( + candidate(fixture, &own).await?.state == "orphaned", + "the terminal orphan disposition reopened" + ); + assert_no_duplicate_headers(fixture).await?; + fixture.integrity().await?; + let gauges = candidate_gauges(fixture, 0).await?; + unpaged(&gauges, false)?; + ensure!( + gauges.pending == 0.0, + "a candidate is unfinished after the reorg back: {gauges:?}" + ); + eprintln!("two-node lost race: {own} answered inconclusive, reconciled without paging at 1 and {} confirmations, orphaned at {ORPHAN_CONFIRMATIONS}, credited once after the reorg back", ORPHAN_CONFIRMATIONS - 1); + Ok(()) +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn real_own_block_loses_same_height_race_is_orphaned_and_credited_once_on_reorg_back( +) -> Result<()> { + let Some(mut fixture) = Fixture::open_with_servers(false, false).await? else { + return Ok(()); + }; + let peer = PeerNode::start(&fixture).await; + let result = match &peer { + Ok(peer) => lost_race(&mut fixture, peer).await, + Err(error) => Err(anyhow::anyhow!("{error:#}")), + }; + finish(fixture, peer, result).await +} + +/// Scenario 2. The pool mines own blocks on both sides of a partition, the +/// frontend moving from node A to node B between them, and the partition +/// then heals onto A's branch, which reorganizes B nine blocks deep, past +/// the orphan-confirmation depth. Branch A's own blocks are reactivated and +/// credited once, branch B's are deactivated, every miner's reported +/// balances return exactly to the branch-A values with no payout +/// divergence, integrity stays clean, and the +/// reactivated blocks' CTV fanouts recover from `reorged` and confirm at +/// maturity while branch B's are never broadcast. +pub(super) async fn deep_reorg(fixture: &mut Fixture, peer: &PeerNode) -> Result<()> { + // Two payout identities, so each own block splits its reward and the + // miners' balances depend on which branch is active. + let main = format!("{}.highdiff", fixture.address); + let second = new_address(fixture).await?; + let small = format!("{second}.small"); + let miners = [fixture.address.clone(), second]; + let mut tip = peer.heal(fixture).await?; + fixture.servers.push(fixture.start_server(0)?); + server_ready(fixture, 0).await?; + // Under a session load (#553), its shares stop until branch B's own + // blocks are paid. On regtest a load share weighs about half an own + // block (credited at the network difficulty), so ~15 of them fill the + // window: each own block would pay at most one earlier own share, by + // timing, and branch B could pay both miners what branch A did. + pause_load_shares(fixture, true).await?; + // Shared own blocks before the partition seed the payout window, so + // every later own block pays both miners through a CTV fanout. + let mut seeds = Vec::new(); + for (id, username) in [(1, &main), (2, &main), (3, &small)] { + tip = own_block(fixture, 0, username, &tip, id).await?; + seeds.push((tip.clone(), username.clone())); + } + let fork = peer.heal(fixture).await?; + ensure!(fork == tip, "node B did not take the shared own blocks"); + peer.partition(fixture).await?; + + // Branch A: two own blocks through frontend 0 on node A. + let a1 = own_block(fixture, 0, &main, &fork, 4).await?; + let a2 = own_block(fixture, 0, &main, &a1, 5).await?; + let branch_a = [a1.as_str(), a2.as_str()]; + ensure!( + chain_states_are(fixture, &branch_a, "confirmed").await?, + "branch A's own blocks are not confirmed" + ); + fixture.integrity().await?; + let balances_a = balances(fixture, &miners).await?; + + // Node B outgrows branch A and the pool moves there: frontend 0 stops + // and frontend 1 starts against node B. + let b_tip = peer.mine(3, &fixture.address).await?.remove(2); + fixture.servers[0].stop(); + let server = start_frontend(fixture, 1, peer.rpc_port)?; + fixture.servers.push(server); + server_ready(fixture, 1).await?; + until("branch A's own blocks deactivated", 30, || async { + chain_states_are(fixture, &branch_a, "inactive").await + }) + .await?; + fixture.integrity().await?; + let b1 = own_block(fixture, 1, &main, &b_tip, 6).await?; + let b2 = own_block(fixture, 1, &main, &b1, 7).await?; + let branch_b = [b1.as_str(), b2.as_str()]; + ensure!( + chain_states_are(fixture, &branch_b, "confirmed").await?, + "branch B's own blocks are not confirmed" + ); + fixture.integrity().await?; + let balances_b = balances(fixture, &miners).await?; + ensure!( + balances_b != balances_a, + "branch B's own blocks left the balances at branch A's, {balances_a:?}; the comparison below would prove nothing" + ); + // Every own block is paid as found; the reorg runs under the whole load. + pause_load_shares(fixture, false).await?; + // Branch B: 3 + 2 own + 4 = 9 blocks past the fork, so B1 has + // ORPHAN_CONFIRMATIONS confirmations; branch A grows to 10 unobserved. + peer.mine(4, &fixture.address).await?; + node_a_mine(fixture, 8).await?; + let fork_height = height(fixture, &fork).await?; + let b_height = count(peer.rpc("getblockcount", json!([])).await?)?; + ensure!( + b_height - fork_height > i64::try_from(ORPHAN_CONFIRMATIONS)?, + "branch B is only {} blocks deep", + b_height - fork_height + ); + + // Heal onto branch A: node B reorganizes past the orphan depth under + // frontend 1. + let winner = peer.heal(fixture).await?; + ensure!( + winner == node_a_best(fixture).await?, + "the healed nodes did not take branch A" + ); + until("deep reorg reconciled", 60, || async { + Ok(chain_states_are(fixture, &branch_a, "confirmed").await? + && chain_states_are(fixture, &branch_b, "inactive").await?) + }) + .await?; + fixture.integrity().await?; + // Both frontends run again on the converged nodes, and agree. + fixture.servers[0] = fixture.start_server(0)?; + server_ready(fixture, 0).await?; + let balances_healed = balances(fixture, &miners).await?; + ensure!( + balances_healed == balances_a, + "balances after the deep reorg differ from branch A's: {balances_healed:?}, expected {balances_a:?}" + ); + let divergences: i64 = sqlx::query_scalar("SELECT count(*) FROM qbit_prism_payout_divergences") + .fetch_one(&fixture.pool) + .await?; + ensure!( + divergences == 0, + "the deep reorg recorded {divergences} payout divergences" + ); + let own = seeds + .iter() + .cloned() + .chain([&a1, &a2, &b1, &b2].map(|hash| (hash.clone(), main.clone()))); + for (hash, username) in own { + ensure!( + credits(fixture, &username, &hash).await? == 1, + "own block {hash} is not credited exactly once" + ); + } + assert_no_duplicate_headers(fixture).await?; + + // CTV fanouts: branch A's recover from `reorged`, branch B's stay. + let fanouts = |hashes: [&str; 2]| { + let pool = fixture.pool.clone(); + let hashes: Vec = hashes.iter().map(|hash| (*hash).to_owned()).collect(); + async move { + let rows: Vec<(String, String, i64)> = sqlx::query_as("SELECT fanout_txid,settlement_status,broadcast_attempt_count FROM qbit_ctv_fanout_artifacts WHERE block_hash=ANY($1) ORDER BY fanout_txid") + .bind(&hashes) + .fetch_all(&pool) + .await?; + anyhow::Ok(rows) + } + }; + let recovered = fanouts([&a1, &a2]).await?; + ensure!( + !recovered.is_empty() + && recovered + .iter() + .all(|(_, status, _)| status == "awaiting_maturity"), + "branch A's fanouts did not recover from the reorg: {recovered:?}" + ); + let reorged = fanouts([&b1, &b2]).await?; + ensure!( + !reorged.is_empty() && reorged.iter().all(|(_, status, _)| status == "reorged"), + "branch B's fanouts are not reorged: {reorged:?}" + ); + let a1_height = height(fixture, &a1).await?; + let tip = count(fixture.rpc("getblockcount", json!([])).await?)?; + node_a_mine(fixture, u64::try_from(a1_height + 1001 - tip)?).await?; + let txids: Vec = recovered.iter().map(|(txid, _, _)| txid.clone()).collect(); + until("recovered fanouts broadcast at maturity", 90, || async { + let mempool = fixture.rpc("getrawmempool", json!([])).await?; + let mempool = mempool.as_array().context("mempool missing")?; + Ok(txids.iter().all(|txid| mempool.contains(&json!(txid)))) + }) + .await?; + node_a_mine(fixture, 1).await?; + until("recovered fanouts confirmed", 60, || async { + Ok(fanouts([&a1, &a2]) + .await? + .iter() + .all(|(_, status, _)| status == "confirmed")) + }) + .await?; + let reorged = fanouts([&b1, &b2]).await?; + ensure!( + reorged + .iter() + .all(|(_, status, attempts)| status == "reorged" && *attempts == 0), + "a fanout of the losing branch was broadcast: {reorged:?}" + ); + fixture.integrity().await?; + eprintln!("two-node deep reorg: own blocks {a1} {a2} on branch A and {b1} {b2} on branch B; B reorganized {} deep onto A; balances {balances_a:?} exact (branch B had {balances_b:?}), reactivated blocks credited once, {} fanouts recovered and confirmed", b_height - fork_height, txids.len()); + Ok(()) +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn real_deep_reorg_with_own_blocks_on_both_branches_keeps_credit_balances_and_fanouts_exact( +) -> Result<()> { + let Some(mut fixture) = Fixture::open_with_servers(true, false).await? else { + return Ok(()); + }; + let peer = PeerNode::start(&fixture).await; + let result = match &peer { + Ok(peer) => deep_reorg(&mut fixture, peer).await, + Err(error) => Err(anyhow::anyhow!("{error:#}")), + }; + finish(fixture, peer, result).await +} + +/// Report both nodes on failure, stop node B, then clean the fixture up, +/// which releases the serial guard only after every process is stopped. +pub(super) async fn finish( + fixture: Fixture, + peer: Result, + result: Result<()>, +) -> Result<()> { + if result.is_err() { + eprintln!("{}", fixture.diagnostics()); + if let Ok(peer) = &peer { + eprintln!("{}", peer.diagnostics()); + } + } + drop(peer); + result.and(fixture.cleanup().await) +} diff --git a/crates/qbit-prism-server/tests/support/live_weighted_recipients.rs b/crates/qbit-prism-server/tests/support/live_weighted_recipients.rs new file mode 100644 index 000000000..5d48ad66a --- /dev/null +++ b/crates/qbit-prism-server/tests/support/live_weighted_recipients.rs @@ -0,0 +1,2683 @@ +//! #521 scenario 7, the many-recipient share-to-spend item of #474: many +//! payout addresses mine with seeded, skewed weights and a spread of +//! per-worker difficulty through several found blocks, and every sat is +//! followed from the accepted share to the spent output. +//! +//! Regtest's proof-of-work limit leaves no room below the block target: a +//! share can be at most twice as easy as a block, so a spread of worker +//! difficulty means nothing there. The node is therefore restarted with +//! `-legacyretarget` and mined quickly until each epoch's 4x clamp has raised +//! the block target at least `RAMP_FACTOR` times, under a mock clock pinned +//! to the tip so the node's future-time rule never paces the ramp. Worker +//! difficulties then span about three orders of magnitude below the block, +//! as on mainnet. +//! +//! Every share is a real Stratum submission from a session this test drives, +//! on both servers, with the difficulty each worker asks for in its password. +//! The test chooses share-only proofs for ordinary shares and a block-bearing +//! proof once per round, so it decides when a block is found and which worker +//! finds it. Everything asserted about payouts is recomputed here from the +//! ledger's accepted shares and the chain, never read from the server's +//! bundles or summaries: the PPLNS window, the gross split, the payout floor +//! and carry-forward, the pool fee's dust sweep, the direct/fanout partition +//! and the fanout fee. The signed audit bundle each block publishes is then +//! verified against the chain's coinbase and compared, account by account, +//! with that model (#474 A). The node pays, and each payee address's own +//! wallet spends what it was paid. +//! +//! Direct settlement (CTV off) runs the same scenario with every payable +//! recipient in the coinbase. `live_recipient_boundary.rs` reuses this model +//! for 1,001 recipients across the fanout chunk boundary. +use super::*; +use num_bigint::BigUint; +use qbit_prism::{PayoutPolicyAccountType, PayoutPolicyAction, SettlementMode}; +use qbit_prism_server::{ + broadcaster::amount_bits, + codec::{ + difficulty_target, double_sha256, hash_display, parse_u32_hex, scaled_target_difficulty, + target_from_compact, + }, +}; +use rand::{rngs::StdRng, Rng, SeedableRng}; +use std::collections::{BTreeMap, BTreeSet, HashMap}; +use tokio::net::tcp::{OwnedReadHalf, OwnedWriteHalf}; + +#[path = "live_recipient_boundary.rs"] +mod recipient_boundary; + +/// The per-PR case: 40 payout addresses (one large and one smaller whale, a +/// Zipf tail, one payee held just under the floor and four that submit a +/// single minimum-difficulty share) through four found blocks. +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn real_weighted_recipients_pay_exact_pplns_outputs_through_fanout_spend() -> Result<()> { + weighted_case(&PER_PR).await +} + +/// The nightly case: the mainnet floor from #521's production shape, about +/// 130 recipients with one whale near 85% and a Zipf tail, through six +/// found blocks. Opt-in: the per-PR job does not select it. +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +#[ignore = "nightly: 130 recipients through six blocks; run with --ignored"] +async fn nightly_mainnet_floor_weighted_recipients_share_to_spend() -> Result<()> { + weighted_case(&MAINNET_FLOOR).await +} + +/// #474 A, direct settlement: CTV off, so every payable recipient is a +/// coinbase output. Six addresses, each generated by its own wallet, with +/// unequal weights, a payee carried below the floor until it crosses it, a +/// near-zero payee and the pool fee's dust sweep, through three blocks. +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn real_weighted_recipients_pay_exact_direct_outputs_without_ctv() -> Result<()> { + weighted_case(&DIRECT).await +} + +pub(super) struct Scenario { + pub(super) name: &'static str, + pub(super) seed: u64, + pub(super) payees: usize, + pub(super) blocks: usize, + /// Each whale's share of every round's work, in address order. + pub(super) whales: &'static [f64], + /// Addresses that submit one minimum-difficulty share in one round. + pub(super) near_zero: usize, + /// CTV settlement; direct coinbase settlement when false. + pub(super) ctv: bool, + /// Wallets the payee addresses are spread over, round robin. Each + /// output is spent by the wallet that generated its address. + pub(super) wallets: usize, +} + +const PER_PR: Scenario = Scenario { + name: "per-PR", + seed: 0x0521_0007, + payees: 40, + blocks: 4, + whales: &[0.80, 0.07], + near_zero: 4, + ctv: true, + wallets: 8, +}; + +const MAINNET_FLOOR: Scenario = Scenario { + name: "mainnet-floor", + seed: 0x0521_0130, + payees: 130, + blocks: 6, + whales: &[0.85], + near_zero: 10, + ctv: true, + wallets: 8, +}; + +const DIRECT: Scenario = Scenario { + name: "direct", + seed: 0x0474_000a, + payees: 6, + blocks: 3, + whales: &[0.55, 0.20], + near_zero: 1, + ctv: false, + wallets: 6, +}; + +/// The block target is raised at least this many times over regtest's limit. +const RAMP_FACTOR: f64 = 1024.0; +/// Each round submits this fraction of the PPLNS window's weight, so a later +/// block's window ends inside an earlier round and cuts one share. +const ROUND_WINDOW_FRACTION: f64 = 0.6; +/// The lowest worker difficulty: a share weight about 5.4e5 in the ledger's +/// scaled units, where an unraised regtest block weighs 1e6. +const MIN_DIFFICULTY: f64 = 2.5e-10; +/// The payout floor. Mainnet's is lower, but its coinbase is larger and its +/// shares far smaller against the block, so dust needs a higher floor here. +const FLOOR_BITS: u64 = 1_000_000; +/// The server's default direct-coinbase floor, as on mainnet. +const DIRECT_FLOOR_BITS: u64 = 10_485_760; +/// Fewer direct slots than direct-eligible recipients, so overflow reaches +/// the fanout by amount priority. +const MAX_DIRECT_OUTPUTS: usize = 4; +/// The server defaults for the coinbase budget and the fanout chunk size. +const MAX_SETTLEMENT_OUTPUTS: usize = 16; +const MAX_FANOUT_RECIPIENTS: usize = 1_000; +/// Mainnet runs with a pool fee; without one, any sub-floor balance makes the +/// selected recipients' balances fall short of the coinbase and no work can +/// be built (`repro_prism_stall`). The fee output fronts the swept dust. +const POOL_FEE_BPS: u64 = 200; +/// The fixture's CTV market rate and the server's default premium. +const FANOUT_RATE_PER_1000_WEIGHT: u64 = 1_000; +const FANOUT_PREMIUM_BPS: u64 = 12_000; +/// The fanout weight estimate: fixed bytes plus one P2MR output each. +const FANOUT_FIXED_WEIGHT: u64 = 90; +const FANOUT_OUTPUT_WEIGHT: u64 = 43; +/// The server's default `PRISM_STRATUM_MAX_CONNECTIONS`. +const DEFAULT_MAX_CONNECTIONS: usize = 384; + +pub(super) async fn weighted_case(scenario: &Scenario) -> Result<()> { + let Some(mut fixture) = Fixture::open_with_servers(scenario.ctv, false).await? else { + return Ok(()); + }; + let started = Instant::now(); + let result = run(&mut fixture, scenario).await; + if result.is_err() { + eprintln!("{}", fixture.diagnostics()); + } + let cleanup = fixture.cleanup().await; + eprintln!( + "live weighted recipients ({}): {:.1}s including cleanup", + scenario.name, + started.elapsed().as_secs_f64() + ); + result.and(cleanup) +} + +async fn run(fixture: &mut Fixture, scenario: &Scenario) -> Result<()> { + let mut phases = Phases::default(); + let mut since = Instant::now(); + let ramp_address = harden_chain(fixture).await?; + phases.lap("chain ramp", &mut since); + let template = fixture + .rpc("getblocktemplate", json!([{"rules": ["segwit"]}])) + .await?; + let network = block_weight(template["bits"].as_str().context("template bits missing")?)?; + let coinbase = template["coinbasevalue"] + .as_u64() + .context("coinbase value missing")?; + let policy = Policy { + ctv: scenario.ctv, + floor: FLOOR_BITS, + pool: pool_identity(fixture).await?, + }; + let payees = Payees::create(fixture, scenario.payees, scenario.wallets).await?; + let plan = Plan::new(scenario, network, coinbase)?; + start_servers_for(fixture, &policy, plan.workers.len()).await?; + phases.lap("wallets and servers", &mut since); + + let mut sessions = Vec::with_capacity(plan.workers.len()); + for worker in &plan.workers { + let username = format!("{}.{}", payees.ids[worker.payee].recipient, worker.name); + let opened = sessions.len(); + sessions.push(Some( + Session::open(fixture.stratum[worker.server], username, worker.difficulty) + .await + .with_context(|| format!("{opened} of {} sessions opened", plan.workers.len()))?, + )); + } + let mut accepted: HashMap = HashMap::new(); + let mut stale_retries = 0; + let mut found = Vec::with_capacity(plan.rounds.len()); + for (index, round) in plan.rounds.iter().enumerate() { + let mut tasks = tokio::task::JoinSet::new(); + for &(worker, count) in &round.shares { + let mut session = sessions[worker].take().context("session reused")?; + tasks.spawn(async move { + let mut shares = Vec::with_capacity(count); + for _ in 0..count { + shares.push(session.submit(false).await?); + } + Ok::<_, anyhow::Error>((worker, session, shares)) + }); + } + while let Some(joined) = tasks.join_next().await { + let (worker, session, shares) = joined??; + sessions[worker] = Some(session); + for share in shares { + stale_retries += share.stale_retries; + ensure!( + accepted.insert(share.share_id, worker).is_none(), + "a share id was acknowledged twice" + ); + } + } + // Let the one-second reanchor publish work over this round's shares + // before the block proof, so the window covers most of the round. + tokio::time::sleep(Duration::from_millis(1_600)).await; + let finder = sessions[round.finder].as_mut().context("finder missing")?; + let block = find_block(fixture, finder, &format!("round {index}")).await?; + stale_retries += block.stale_retries; + ensure!( + accepted + .insert(block.share_id.clone(), round.finder) + .is_none(), + "a block share id was acknowledged twice" + ); + for session in sessions.iter_mut().flatten() { + session.wait_for_parent(&block.hash).await?; + } + found.push(Found { + hash: block.hash, + share_id: block.share_id, + }); + } + drop(sessions); + fixture.quiesce().await?; + phases.lap("sessions and mining rounds", &mut since); + + let shares = ledger_shares(fixture).await?; + let attribution = check_attribution(&shares, &accepted, &plan, &payees.ids)?; + let verified = + verify_payouts(fixture, &ramp_address, &found, &shares, &policy, &payees).await?; + + let summary = Summary::new(&plan, &attribution, &verified, stale_retries); + phases.0.extend(verified.phases.0.iter().copied()); + eprintln!( + "live weighted recipients ({}): wall clock by phase: {phases}", + scenario.name + ); + summary.check(scenario)?; + eprintln!("live weighted recipients ({}): {summary}", scenario.name); + Ok(()) +} + +/// Submits a block-bearing proof from `finder` and waits until the ledger +/// has it confirmed as the node's tip. +async fn find_block(fixture: &Fixture, finder: &mut Session, label: &str) -> Result { + finder.drain(Duration::from_millis(300)).await?; + let block = finder.submit(true).await?; + let hash = &block.hash; + until(&format!("{label} block {hash} confirmed"), 90, || async { + let state: Option = + sqlx::query_scalar("SELECT chain_state FROM qbit_pool_blocks WHERE block_hash=$1") + .bind(hash) + .fetch_optional(&fixture.pool) + .await?; + Ok(state.as_deref() == Some("confirmed") + && fixture.rpc("getbestblockhash", json!([])).await? == json!(hash)) + }) + .await?; + Ok(block) +} + +/// What `verify_payouts` established, for a scenario's summary. +struct Verified { + expected: Vec<(Window, Settled)>, + /// Payee outputs spent, and how many wallets spent them. + spent: usize, + spending_wallets: usize, + phases: Phases, +} + +/// Wall-clock seconds (the test process's monotonic clock) of each named +/// phase, in order, so a slow run says where its time went. +#[derive(Default)] +struct Phases(Vec<(&'static str, f64)>); + +impl Phases { + /// Records the phase that ended now, begun at `*since`, and restarts it. + fn lap(&mut self, name: &'static str, since: &mut Instant) { + self.0.push((name, since.elapsed().as_secs_f64())); + *since = Instant::now(); + } +} + +impl std::fmt::Display for Phases { + fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + for (index, (name, seconds)) in self.0.iter().enumerate() { + let separator = if index == 0 { "" } else { ", " }; + write!(formatter, "{separator}{name} {seconds:.1}s")?; + } + Ok(()) + } +} + +/// Recomputes every found block's payout from the ledger's accepted shares +/// and compares the chain, the ledger's accounts and the signed audit bundle +/// with it; then mines through maturity, follows each fanout to its outputs +/// and has the owning wallets spend a sample of them. +async fn verify_payouts( + fixture: &Fixture, + ramp_address: &str, + found: &[Found], + shares: &[LedgerShare], + policy: &Policy, + payees: &Payees, +) -> Result { + let mut phases = Phases::default(); + let mut since = Instant::now(); + let blocks = chain_blocks(fixture, found).await?; + let mut prior = Carried::new(); + let mut expected = Vec::with_capacity(blocks.len()); + let mut bundle_fanouts = Vec::with_capacity(blocks.len()); + for (index, (block, found)) in blocks.iter().zip(found).enumerate() { + let anchor = shares + .iter() + .find(|share| share.share_id == found.share_id) + .context("block share missing from the ledger")? + .issued_ms; + let window = pplns_window(shares, anchor, block.network)?; + let settled = settle(block.value, &window, &prior, policy) + .with_context(|| format!("model of block {} at {}", block.hash, block.height))?; + check_coinbase(block, &settled, &policy.pool, &payees.ids)?; + check_carry_rows(fixture, block, &settled).await?; + bundle_fanouts.push( + check_audit_bundle(fixture, index % 2, block, &window, &settled, policy) + .await + .with_context(|| format!("audit bundle of block {}", block.hash))?, + ); + prior = settled.next_prior(&prior); + expected.push((window, settled)); + } + check_final_balances(fixture, &prior).await?; + phases.lap("model, ledger and audit bundles", &mut since); + + let last = blocks.last().context("no block found")?.height; + mine( + fixture, + ramp_address, + last + qbit_prism::QBIT_COINBASE_MATURITY_BLOCKS - tip_height(fixture).await?, + ) + .await?; + phases.lap("maturity mining", &mut since); + // The server's fanout ids only say what to wait for; the outputs checked + // below are found on the chain by the covenant outpoints they spend. + let fanouts: Vec = + sqlx::query_scalar("SELECT fanout_txid FROM qbit_ctv_fanout_artifacts") + .fetch_all(&fixture.pool) + .await?; + ensure!( + fanouts.len() + == expected + .iter() + .map(|(_, settled)| settled.chunks.len()) + .sum::(), + "fanout artifact count differs from the model's chunks" + ); + // A fanout that matured before the last batch may already be mined. + until("every mature fanout broadcast", 60, || async { + let mempool = fixture.rpc("getrawmempool", json!([])).await?; + let mempool = mempool.as_array().context("mempool missing")?; + let confirmed: Vec = sqlx::query_scalar( + "SELECT fanout_txid FROM qbit_ctv_fanout_artifacts WHERE settlement_status='confirmed'", + ) + .fetch_all(&fixture.pool) + .await?; + Ok(fanouts + .iter() + .all(|txid| confirmed.contains(txid) || mempool.contains(&json!(txid)))) + }) + .await?; + mine(fixture, ramp_address, 1).await?; + until("every fanout confirmed in the read model", 60, || async { + let confirmed: i64 = sqlx::query_scalar( + "SELECT count(*) FROM qbit_ctv_fanout_artifacts WHERE settlement_status='confirmed'", + ) + .fetch_one(&fixture.pool) + .await?; + Ok(usize::try_from(confirmed)? == fanouts.len()) + }) + .await?; + phases.lap("fanout broadcast and confirmation", &mut since); + let paid = check_fanouts(fixture, &blocks, &expected, &bundle_fanouts, &payees.ids).await?; + check_wallet_receipts(fixture, &expected, &paid, payees).await?; + phases.lap("chain and wallet receipts", &mut since); + let (spent, spending_wallets) = + spend_sample(fixture, ramp_address, blocks.len(), &paid, payees).await?; + fixture.integrity().await?; + phases.lap("owner spends and integrity", &mut since); + Ok(Verified { + expected, + spent, + spending_wallets, + phases, + }) +} + +// --------------------------------------------------------------------------- +// The chain: a harder regtest, the servers, and wallet access. + +/// Restarts the node with `-legacyretarget`, raises the block target at +/// least `RAMP_FACTOR` times and returns the address the ramp mined to. The +/// node's clock is left mocked at the tip, which later mining moves forward. +async fn harden_chain(fixture: &mut Fixture) -> Result { + fixture.rpc("stop", json!([])).await?; + until("qbit node exit before restart", 60, || async { + Ok(fixture.node.child.try_wait()?.is_some()) + }) + .await?; + // The fixture's node arguments, plus the retargeting rule and the wallet. + let mut command = Command::new(&fixture.qbitd); + command + .args([ + "-regtest", + "-server=1", + "-listen=0", + "-dnsseed=0", + "-discover=0", + "-fallbackfee=0.00001", + "-rpcuser=prismtest", + "-rpcpassword=prismtest", + "-txindex=0", + "-legacyretarget", + "-wallet=prism", + ]) + .arg(format!("-datadir={}", fixture.directory.path().display())) + .arg(format!("-rpcport={}", fixture.rpc_port)); + fixture.node = Process::spawn( + &mut command, + fixture.directory.path().join("qbit-retarget.log"), + )?; + until("retargeting qbit RPC", 30, || async { + Ok(fixture.rpc("getblockchaininfo", json!([])).await?["chain"] == "regtest") + }) + .await?; + fixture.rpc("createwallet", json!(["ramp"])).await?; + let address = wallet_rpc(fixture, "ramp", "getnewaddress", json!(["", "p2mr"])) + .await? + .as_str() + .context("ramp address missing")? + .to_owned(); + fixture.rpc("unloadwallet", json!(["ramp"])).await?; + let base = fixture.rpc("getblockchaininfo", json!([])).await?["difficulty"] + .as_f64() + .context("difficulty missing")?; + loop { + let info = fixture.rpc("getblockchaininfo", json!([])).await?; + let difficulty = info["difficulty"].as_f64().context("difficulty missing")?; + if difficulty >= base * RAMP_FACTOR * 0.999 { + break; + } + ensure!( + info["blocks"].as_u64().context("height missing")? < 20_000, + "legacy retargeting did not raise the target" + ); + mine(fixture, &address, 500).await?; + } + Ok(address) +} + +/// Mines `count` blocks to `address`, moving the mock clock to the tip first +/// so the node's future-time rule never waits on the wall clock. +async fn mine(fixture: &Fixture, address: &str, count: u64) -> Result<()> { + let mut left = count; + while left > 0 { + let batch = left.min(250); + let tip_time = fixture.rpc("getblockchaininfo", json!([])).await?["time"] + .as_u64() + .context("tip time missing")?; + let now = std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH)? + .as_secs(); + fixture + .rpc("setmocktime", json!([tip_time.max(now)])) + .await?; + generate(fixture, batch, address).await?; + left -= batch; + } + Ok(()) +} + +/// The deadline of this file's node-heavy RPCs: ramp mining and the wallet's +/// many-input `sendall`. Both finish in seconds on an idle host, but a +/// 250-block batch outran the fixture client's 45 s timeout at a host load +/// average of 50 (#533). Neither call is idempotent, so they are never +/// retried, only given time to finish. +const HEAVY_RPC_TIMEOUT: Duration = Duration::from_secs(600); + +/// `generatetoaddress`, under `HEAVY_RPC_TIMEOUT`. +async fn generate(fixture: &Fixture, batch: u64, address: &str) -> Result<()> { + let payload: Value = fixture + .client + .post(format!("http://127.0.0.1:{}/", fixture.rpc_port)) + .basic_auth("prismtest", Some("prismtest")) + .timeout(HEAVY_RPC_TIMEOUT) + .json(&json!({"jsonrpc":"1.0","id":"live-test","method":"generatetoaddress","params":[batch, address]})) + .send() + .await? + .json() + .await?; + ensure!( + payload["error"].is_null(), + "RPC generatetoaddress: {}", + payload["error"] + ); + Ok(()) +} + +async fn tip_height(fixture: &Fixture) -> Result { + fixture + .rpc("getblockcount", json!([])) + .await? + .as_u64() + .context("tip height missing") +} + +/// The ledger's scaled block weight for compact `bits`: regtest's limit +/// weighs one million, and a target `n` times harder weighs `n` million. +fn block_weight(bits: &str) -> Result { + let limit = BigUint::from(0x7f_ffff_u32) << (8 * (0x20 - 3)); + let target = target_from_compact(parse_u32_hex(bits)?)?; + u128::try_from(limit * 1_000_000_u32 / target).context("block weight exceeds u128") +} + +/// A wallet RPC under `HEAVY_RPC_TIMEOUT`: the payees' `sendall` signs every +/// input it spends. +async fn wallet_rpc(fixture: &Fixture, wallet: &str, method: &str, params: Value) -> Result { + let payload: Value = fixture + .client + .post(format!( + "http://127.0.0.1:{}/wallet/{wallet}", + fixture.rpc_port + )) + .basic_auth("prismtest", Some("prismtest")) + .timeout(HEAVY_RPC_TIMEOUT) + .json(&json!({"jsonrpc":"1.0","id":"live-test","method":method,"params":params})) + .send() + .await? + .json() + .await?; + ensure!( + payload["error"].is_null(), + "RPC {wallet}/{method}: {}", + payload["error"] + ); + Ok(payload["result"].clone()) +} + +/// A payout identity as the ledger keys it: the address is both the miner id +/// and the order key, and the program is its P2MR witness program. +#[derive(Clone, Debug, PartialEq, Eq, PartialOrd, Ord)] +struct Identity { + order_key: String, + recipient: String, + program: String, +} + +impl Identity { + fn script(&self) -> String { + format!("5220{}", self.program) + } +} + +async fn identity_of(fixture: &Fixture, address: &str) -> Result { + let validation = fixture.rpc("validateaddress", json!([address])).await?; + let script = validation["scriptPubKey"] + .as_str() + .context("script missing")?; + ensure!( + script.len() == 68 && script.starts_with("5220"), + "{address} is not P2MR: {script}" + ); + Ok(Identity { + order_key: address.into(), + recipient: address.into(), + program: script[4..].into(), + }) +} + +/// The pool fee address, from the fixture's own wallet. +async fn pool_identity(fixture: &Fixture) -> Result { + let address = wallet_rpc(fixture, "prism", "getnewaddress", json!(["", "p2mr"])).await?; + identity_of( + fixture, + address.as_str().context("pool fee address missing")?, + ) + .await +} + +/// The payee addresses, each generated by one of `wallets` wallets created +/// for them, round robin, and the wallet that owns each. +struct Payees { + ids: Vec, + /// The owning wallet of each payee's script. + owners: BTreeMap, + wallets: Vec, +} + +impl Payees { + async fn create(fixture: &Fixture, count: usize, wallets: usize) -> Result { + ensure!( + (1..=count).contains(&wallets), + "{wallets} wallets for {count} payees" + ); + let wallets: Vec = (0..wallets).map(|index| format!("payee-{index}")).collect(); + for wallet in &wallets { + fixture.rpc("createwallet", json!([wallet])).await?; + } + let mut ids = Vec::with_capacity(count); + let mut owners = BTreeMap::new(); + for index in 0..count { + let wallet = &wallets[index % wallets.len()]; + let address = wallet_rpc(fixture, wallet, "getnewaddress", json!(["", "p2mr"])).await?; + let identity = + identity_of(fixture, address.as_str().context("payee address missing")?).await?; + ensure!( + owners.insert(identity.script(), wallet.clone()).is_none(), + "payee programs are not distinct" + ); + ids.push(identity); + } + Ok(Self { + ids, + owners, + wallets, + }) + } + + fn scripts(&self) -> BTreeSet { + self.owners.keys().cloned().collect() + } +} + +/// The payout settings the servers run with, which the model applies. +struct Policy { + /// CTV settlement: the direct floor and slots below, the rest through + /// fanouts. Direct coinbase settlement of every recipient when false. + ctv: bool, + floor: u64, + pool: Identity, +} + +async fn start_servers(fixture: &mut Fixture, policy: &Policy) -> Result<()> { + start_servers_for(fixture, policy, 0).await +} + +/// [`start_servers`] with room for `workers` sessions. +async fn start_servers_for(fixture: &mut Fixture, policy: &Policy, workers: usize) -> Result<()> { + let mut env = vec![ + // Fixed per-worker difficulty: each worker's password sets it. + ("PRISM_STRATUM_VARDIFF", "0".to_owned()), + ("PRISM_STRATUM_VARDIFF_MIN_DIFF", "1e-12".to_owned()), + ("PRISM_PAYOUT_MIN_OUTPUT_BITS", policy.floor.to_string()), + ("PRISM_POOL_FEE_ENABLED", "1".to_owned()), + ("PRISM_POOL_FEE_BPS", POOL_FEE_BPS.to_string()), + ("PRISM_POOL_FEE_ADDRESS", policy.pool.recipient.clone()), + ]; + if policy.ctv { + env.extend([ + ( + "PRISM_DIRECT_COINBASE_PAYOUT_FLOOR_BITS", + DIRECT_FLOOR_BITS.to_string(), + ), + ( + "PRISM_MAX_DIRECT_COINBASE_OUTPUTS", + MAX_DIRECT_OUTPUTS.to_string(), + ), + ]); + } + // #553's 2,000-wallet case has one session per worker, more than the + // server's default of 384 connections. + if workers > DEFAULT_MAX_CONNECTIONS { + env.push(("PRISM_STRATUM_MAX_CONNECTIONS", (workers + 64).to_string())); + } + fixture.server_env = env + .into_iter() + .map(|(name, value)| (name.to_owned(), value)) + .collect(); + for index in 0..2 { + let process = fixture.start_server(index)?; + fixture.servers.push(process); + } + for index in 0..2 { + until( + &format!("PRISM HTTP readiness of server {index}"), + 30, + || async { + Ok(fixture + .client + .get(format!("http://127.0.0.1:{}/healthz", fixture.api[index])) + .send() + .await? + .status() + .is_success()) + }, + ) + .await?; + } + Ok(()) +} + +// --------------------------------------------------------------------------- +// The seeded plan: addresses, workers, and the shares of each round. + +struct WorkerPlan { + payee: usize, + name: String, + server: usize, + difficulty: f64, + /// The ledger weight of one share at this difficulty. + weight: u128, +} + +struct Round { + /// `(worker, share count)`; the finder's block proof is extra. + shares: Vec<(usize, usize)>, + finder: usize, +} + +struct Plan { + workers: Vec, + rounds: Vec, +} + +fn share_weight(difficulty: f64) -> Result { + scaled_target_difficulty(&difficulty_target(difficulty)?) +} + +impl Plan { + /// Payees in address order: the whales, a Zipf tail, one payee paced + /// just under the floor each round (so its carried balance is paid once + /// it crosses), then the near-zero payees. + fn new(scenario: &Scenario, network: u128, coinbase: u64) -> Result { + let mut rng = StdRng::seed_from_u64(scenario.seed); + let whales = scenario.whales.len(); + let tail = scenario + .payees + .checked_sub(whales + 1 + scenario.near_zero) + .context("scenario has too few payees")?; + let threshold = 0.6 * FLOOR_BITS as f64 / coinbase as f64; + let tail_total = 1.0 - scenario.whales.iter().sum::() - threshold; + let zipf: Vec = (0..tail) + .map(|rank| 1.0 / ((rank + 1) as f64).powf(1.1)) + .collect(); + let zipf_sum: f64 = zipf.iter().sum(); + let fractions: Vec = scenario + .whales + .iter() + .copied() + .chain(zipf.iter().map(|weight| tail_total * weight / zipf_sum)) + .chain([threshold]) + .collect(); + let round_work = + network as f64 * qbit_prism::PRISM_WINDOW_MULTIPLIER as f64 * ROUND_WINDOW_FRACTION; + // Share-only proofs need a share target easier than the block. + let max_difficulty = network as f64 / 2.0 / share_weight(1.0)? as f64; + ensure!( + max_difficulty / MIN_DIFFICULTY >= 500.0, + "block target too easy for a wide spread" + ); + let clamp = |difficulty: f64| difficulty.clamp(MIN_DIFFICULTY, max_difficulty); + + let mut workers = Vec::new(); + let mut owned: Vec> = Vec::with_capacity(scenario.payees); + for (payee, fraction) in fractions.iter().enumerate() { + let count = if payee < whales { + 3 + } else if payee == fractions.len() - 1 { + 1 + } else { + match rng.gen_range(0..10) { + 0..=5 => 1, + 6..=8 => 2, + _ => 3, + } + }; + let per_worker = round_work * fraction / count as f64; + let mut ids = Vec::with_capacity(count); + for index in 0..count { + // Pace each worker at 2-30 shares a round, as vardiff would. + let shares = (2.0f64).powf(rng.gen_range(1.0..4.9)); + let difficulty = if payee == 0 && index == 0 { + max_difficulty + } else if payee == fractions.len() - 1 { + MIN_DIFFICULTY + } else { + clamp(per_worker / shares / share_weight(1.0)? as f64) + }; + ids.push(workers.len()); + workers.push(WorkerPlan { + payee, + name: format!("w{index}"), + server: (payee + index) % 2, + difficulty, + weight: share_weight(difficulty)?, + }); + } + owned.push(ids); + } + for payee in fractions.len()..scenario.payees { + owned.push(vec![workers.len()]); + workers.push(WorkerPlan { + payee, + name: "w0".into(), + server: payee % 2, + difficulty: MIN_DIFFICULTY, + weight: share_weight(MIN_DIFFICULTY)?, + }); + } + + let mut rounds: Vec = (0..scenario.blocks) + .map(|_| Round { + shares: Vec::new(), + finder: 0, + }) + .collect(); + for (payee, fraction) in fractions.iter().enumerate() { + let always = payee < whales || payee == fractions.len() - 1; + let mut active: Vec = (0..scenario.blocks) + .map(|_| always || rng.gen_bool(0.8)) + .collect(); + if !active.contains(&true) { + active[rng.gen_range(0..scenario.blocks)] = true; + } + for (round, active) in active.into_iter().enumerate() { + if !active { + continue; + } + let work = round_work * fraction * rng.gen_range(0.7..1.3); + let ids = &owned[payee]; + let mut any = false; + for &worker in ids { + let count = + (work / ids.len() as f64 / workers[worker].weight as f64).round() as usize; + if count > 0 { + rounds[round].shares.push((worker, count)); + any = true; + } + } + if !any { + let easiest = *ids + .iter() + .min_by_key(|worker| workers[**worker].weight) + .expect("every payee has a worker"); + rounds[round].shares.push((easiest, 1)); + } + } + } + for (index, payee) in (fractions.len()..scenario.payees).enumerate() { + // Spread over every round but the last, so every block carries + // some dust and each near-zero balance is carried at least once. + rounds[index % (scenario.blocks - 1)] + .shares + .push((owned[payee][0], 1)); + } + for (index, round) in rounds.iter_mut().enumerate() { + round.finder = if index == 0 { + owned[0][0] + } else { + round.shares[rng.gen_range(0..round.shares.len())].0 + }; + } + Ok(Self { workers, rounds }) + } +} + +// --------------------------------------------------------------------------- +// A Stratum session this test drives. + +struct Session { + reader: BufReader, + writer: OwnedWriteHalf, + buffer: Vec, + username: String, + extranonce1: String, + extranonce2_size: usize, + extranonce2: u64, + next_id: u64, + /// The difficulty this worker asked for. The server derives the share + /// target from it exactly and advertises that target's difficulty, which + /// can differ from it in the last bits. + requested: f64, + difficulty: f64, + /// The latest notify and the difficulty advertised with it. + notify: Value, + job_difficulty: f64, +} + +struct Submitted { + share_id: String, + hash: String, + stale_retries: usize, +} + +impl Session { + async fn open(port: u16, username: String, difficulty: f64) -> Result { + let stream = tokio::net::TcpStream::connect(("127.0.0.1", port)).await?; + let (read, write) = stream.into_split(); + let mut session = Self { + reader: BufReader::new(read), + writer: write, + buffer: Vec::new(), + username, + extranonce1: String::new(), + extranonce2_size: 0, + extranonce2: 0, + next_id: 3, + requested: difficulty, + difficulty: 0.0, + notify: Value::Null, + job_difficulty: 0.0, + }; + session + .send(json!({"id":1,"method":"mining.subscribe","params":["weighted-regtest"]})) + .await?; + let password = format!("d={difficulty},md={difficulty}"); + session + .send(json!({"id":2,"method":"mining.authorize","params":[session.username,password]})) + .await?; + tokio::time::timeout(Duration::from_secs(20), async { + let mut authorized = false; + loop { + let message = session.read().await?; + ensure!( + message.get("error").is_none_or(Value::is_null), + "handshake rejected: {message}" + ); + if message["id"] == 1 { + session.extranonce1 = message["result"][1] + .as_str() + .context("extranonce1 missing")? + .into(); + session.extranonce2_size = message["result"][2] + .as_u64() + .context("extranonce2 size missing")? + .try_into()?; + } + if message["id"] == 2 { + ensure!(message["result"] == true, "authorize refused: {message}"); + authorized = true; + } + if authorized + && message["method"] == "mining.notify" + && (session.job_difficulty / difficulty - 1.0).abs() < 1e-9 + { + return Ok::<_, anyhow::Error>(()); + } + } + }) + .await + .with_context(|| { + format!( + "{} never received work at difficulty {difficulty}", + session.username + ) + })??; + Ok(session) + } + + async fn send(&mut self, payload: Value) -> Result<()> { + self.writer + .write_all(format!("{payload}\n").as_bytes()) + .await?; + Ok(()) + } + + async fn read(&mut self) -> Result { + ensure!( + self.reader.read_until(b'\n', &mut self.buffer).await? > 0, + "{} connection closed", + self.username + ); + let message: Value = serde_json::from_slice(&self.buffer)?; + self.buffer.clear(); + if message["method"] == "mining.set_difficulty" { + self.difficulty = message["params"][0] + .as_f64() + .context("invalid difficulty")?; + } + if message["method"] == "mining.notify" { + self.notify = message.clone(); + self.job_difficulty = self.difficulty; + } + Ok(message) + } + + /// Reads whatever the server sent until it is quiet for `quiet`. + async fn drain(&mut self, quiet: Duration) -> Result<()> { + while let Ok(message) = tokio::time::timeout(quiet, self.read()).await { + message?; + } + Ok(()) + } + + async fn wait_for_parent(&mut self, parent: &str) -> Result<()> { + let mut wire = hex::decode(parent)?; + wire.reverse(); + for word in wire.as_chunks_mut::<4>().0 { + word.reverse(); + } + let expected = hex::encode(wire); + tokio::time::timeout(Duration::from_secs(30), async { + while self.notify["params"][1].as_str() != Some(expected.as_str()) { + self.read().await?; + } + Ok::<_, anyhow::Error>(()) + }) + .await + .with_context(|| format!("{} got no work on parent {parent}", self.username))??; + Ok(()) + } + + /// Submits one proof: a share-only proof, or with `block` one that also + /// meets the block target. A `stale-job` refusal is retried on newer + /// work and counted; any other refusal fails the test. + async fn submit(&mut self, block: bool) -> Result { + for stale_retries in 0..8 { + let id = self.next_id; + self.next_id += 1; + let (request, hash) = self.solve(id, block)?; + let job = request["params"][1].clone(); + self.send(request).await?; + let response = tokio::time::timeout(Duration::from_secs(30), async { + loop { + let message = self.read().await?; + if message["id"] == id { + return Ok::<_, anyhow::Error>(message); + } + } + }) + .await + .with_context(|| format!("{} got no reply to a submit", self.username))??; + if response["result"] == true { + return Ok(Submitted { + share_id: format!("{}:{hash}", self.username), + hash, + stale_retries, + }); + } + ensure!( + response["error"].to_string().contains("stale"), + "{} share refused: {response}", + self.username + ); + // Work newer than the refused job may already have arrived. + tokio::time::timeout(Duration::from_secs(20), async { + while self.notify["params"][0] == job { + self.read().await?; + } + Ok::<_, anyhow::Error>(()) + }) + .await + .context("no new work after a stale refusal")??; + } + bail!("{} kept receiving stale refusals", self.username) + } + + fn solve(&mut self, id: u64, block: bool) -> Result<(Value, String)> { + let params = self.notify["params"] + .as_array() + .context("no work yet")? + .clone(); + let field = |index: usize| params[index].as_str().context("invalid notify field"); + let mut previous = hex::decode(field(1)?)?; + for word in previous.as_chunks_mut::<4>().0 { + word.reverse(); + } + let bits = parse_u32_hex(field(6)?)?; + let ntime = parse_u32_hex(field(7)?)?; + let version = parse_u32_hex(field(5)?)?; + let network_target = target_bytes(&target_from_compact(bits)?); + ensure!( + (self.job_difficulty / self.requested - 1.0).abs() < 1e-9, + "{} was sent work at difficulty {}", + self.username, + self.job_difficulty + ); + let share_target = target_bytes(&difficulty_target(self.requested)?); + for _ in 0..64 { + let extranonce2 = format!( + "{:0width$x}", + self.extranonce2, + width = self.extranonce2_size * 2 + ); + self.extranonce2 += 1; + let coinbase = hex::decode(format!( + "{}{}{extranonce2}{}", + field(2)?, + self.extranonce1, + field(3)? + ))?; + let mut merkle = double_sha256(&coinbase); + for sibling in params[4].as_array().context("merkle branch missing")? { + let sibling = hex::decode(sibling.as_str().context("invalid sibling")?)?; + merkle = double_sha256(&[merkle.as_slice(), sibling.as_slice()].concat()); + } + let mut header = [ + version.to_le_bytes().as_slice(), + previous.as_slice(), + merkle.as_slice(), + ntime.to_le_bytes().as_slice(), + bits.to_le_bytes().as_slice(), + &[0; 4], + ] + .concat(); + for nonce in 0..=u16::MAX as u32 { + header[76..].copy_from_slice(&nonce.to_le_bytes()); + let hash = double_sha256(&header); + if at_most(&hash, &share_target) && at_most(&hash, &network_target) == block { + let request = json!({"id":id,"method":"mining.submit","params":[ + self.username, field(0)?, extranonce2, format!("{ntime:08x}"), format!("{nonce:08x}") + ]}); + return Ok((request, hash_display(&hash))); + } + } + } + bail!("no proof within the search budget") + } +} + +/// A target as 32 little-endian bytes, the order a header hash compares in. +fn target_bytes(target: &BigUint) -> [u8; 32] { + let mut bytes = [0; 32]; + let le = target.to_bytes_le(); + bytes[..le.len()].copy_from_slice(&le); + bytes +} + +fn at_most(hash: &[u8; 32], target: &[u8; 32]) -> bool { + hash.iter().rev().cmp(target.iter().rev()) != std::cmp::Ordering::Greater +} + +struct Found { + hash: String, + share_id: String, +} + +// --------------------------------------------------------------------------- +// What the ledger and the chain recorded. + +struct LedgerShare { + seq: i64, + share_id: String, + miner: String, + order_key: String, + program: String, + weight: u128, + /// The height of the tip the share's job was built on. + template_height: i64, + issued_ms: i64, + accepted_ms: i64, + writer: String, + credit_policy: Option, +} + +async fn ledger_shares(fixture: &Fixture) -> Result> { + let rows = sqlx::query( + "SELECT share_seq,share_id,miner_id,payout_order_key,encode(p2mr_program,'hex') AS program,share_difficulty::text AS weight,template_height, + round(extract(epoch FROM job_issued_at)*1000)::bigint AS issued_ms,round(extract(epoch FROM accepted_at)*1000)::bigint AS accepted_ms, + writer_id,credit_policy + FROM qbit_share_ledger WHERE accepted ORDER BY share_seq", + ) + .fetch_all(&fixture.pool) + .await?; + rows.iter() + .map(|row| { + Ok(LedgerShare { + seq: row.try_get("share_seq")?, + share_id: row.try_get("share_id")?, + miner: row.try_get("miner_id")?, + order_key: row.try_get("payout_order_key")?, + program: row.try_get("program")?, + weight: row.try_get::("weight")?.parse()?, + template_height: row.try_get("template_height")?, + issued_ms: row.try_get("issued_ms")?, + accepted_ms: row.try_get("accepted_ms")?, + writer: row.try_get("writer_id")?, + credit_policy: row.try_get("credit_policy")?, + }) + }) + .collect() +} + +struct Attribution { + shares: usize, + /// Payees whose accepted shares came through both servers. + both_servers: usize, + /// Payees with accepted shares from more than one worker difficulty. + multi_difficulty: usize, + /// Largest over smallest share weight among accepted shares. + spread: f64, +} + +/// The ledger holds exactly the shares the servers acknowledged, each under +/// the payout identity, weight and server of the worker that submitted it. +fn check_attribution( + shares: &[LedgerShare], + accepted: &HashMap, + plan: &Plan, + payees: &[Identity], +) -> Result { + ensure!( + shares.len() == accepted.len(), + "ledger holds {} accepted shares, the servers acknowledged {}", + shares.len(), + accepted.len() + ); + let mut servers: BTreeMap> = BTreeMap::new(); + let mut weights: BTreeMap> = BTreeMap::new(); + for share in shares { + let worker = &plan.workers[*accepted + .get(&share.share_id) + .with_context(|| format!("ledger share {} was never acknowledged", share.share_id))?]; + let payee = &payees[worker.payee]; + ensure!( + share.miner == payee.recipient + && share.order_key == payee.order_key + && share.program == payee.program, + "share {} credited to {}/{}, not {}", + share.share_id, + share.miner, + share.program, + payee.recipient + ); + ensure!( + share.weight == worker.weight, + "share {} weighs {}, its worker's difficulty {} weighs {}", + share.share_id, + share.weight, + worker.difficulty, + worker.weight + ); + ensure!( + share.writer == format!("live-{}", worker.server), + "share {} written by {}, submitted to server {}", + share.share_id, + share.writer, + worker.server + ); + ensure!( + share.credit_policy.is_none(), + "share {} has a credit policy", + share.share_id + ); + ensure!( + share.issued_ms <= share.accepted_ms, + "share {} precedes its job", + share.share_id + ); + servers + .entry(worker.payee) + .or_default() + .insert(share.writer.clone()); + weights + .entry(worker.payee) + .or_default() + .insert(share.weight); + } + let all: Vec = shares.iter().map(|share| share.weight).collect(); + Ok(Attribution { + shares: shares.len(), + both_servers: servers + .values() + .filter(|servers| servers.len() == 2) + .count(), + multi_difficulty: weights.values().filter(|weights| weights.len() > 1).count(), + spread: *all.iter().max().context("no shares")? as f64 + / *all.iter().min().context("no shares")? as f64, + }) +} + +struct ChainBlock { + hash: String, + height: u64, + network: u128, + value: u64, + coinbase_txid: String, + /// `(vout, script hex, amount)` of every coinbase output. + outputs: Vec<(u32, String, u64)>, +} + +fn outputs_of(tx: &Value) -> Result> { + tx["vout"] + .as_array() + .context("outputs missing")? + .iter() + .map(|output| { + Ok(( + output["n"].as_u64().context("vout missing")?.try_into()?, + output["scriptPubKey"]["hex"] + .as_str() + .context("script missing")? + .into(), + amount_bits(&output["value"])?, + )) + }) + .collect() +} + +/// The found blocks as the active chain has them, in order. +async fn chain_blocks(fixture: &Fixture, found: &[Found]) -> Result> { + let mut blocks: Vec = Vec::with_capacity(found.len()); + for found in found { + let block = fixture.rpc("getblock", json!([found.hash, 2])).await?; + let height = block["height"].as_u64().context("height missing")?; + ensure!( + fixture.rpc("getblockhash", json!([height])).await? == json!(found.hash), + "found block {} is not on the active chain", + found.hash + ); + if let Some(previous) = blocks.last() { + ensure!(height > previous.height, "found blocks out of order"); + } + let coinbase = &block["tx"][0]; + let outputs = outputs_of(coinbase)?; + blocks.push(ChainBlock { + hash: found.hash.clone(), + height, + network: block_weight(block["bits"].as_str().context("bits missing")?)?, + value: outputs.iter().map(|(_, _, amount)| amount).sum(), + coinbase_txid: coinbase["txid"] + .as_str() + .context("coinbase txid missing")? + .into(), + outputs, + }); + } + Ok(blocks) +} + +// --------------------------------------------------------------------------- +// The independent model: PPLNS window, payout policy and settlement. + +struct Window { + /// Counted weight per payout program. + weights: BTreeMap, + shares: usize, + /// The oldest counted share was counted in part. + cut: bool, +} + +/// Newest first from the block's snapshot, eight blocks of weight, the last +/// share counted only up to what is left. +fn pplns_window(shares: &[LedgerShare], anchor_ms: i64, network: u128) -> Result { + let mut remaining = network + .checked_mul(qbit_prism::PRISM_WINDOW_MULTIPLIER) + .context("window weight overflow")?; + let mut eligible: Vec<&LedgerShare> = shares + .iter() + .filter(|share| share.issued_ms <= anchor_ms && share.accepted_ms <= anchor_ms) + .collect(); + eligible.sort_by_key(|share| std::cmp::Reverse(share.seq)); + let mut window = Window { + weights: BTreeMap::new(), + shares: 0, + cut: false, + }; + for share in eligible { + if remaining == 0 { + break; + } + let counted = share.weight.min(remaining); + remaining -= counted; + window.cut = counted < share.weight; + window.shares += 1; + let identity = Identity { + order_key: share.order_key.clone(), + recipient: share.miner.clone(), + program: share.program.clone(), + }; + window + .weights + .entry(share.program.clone()) + .or_insert((identity, 0)) + .1 += counted; + } + ensure!(window.shares > 0, "empty window at anchor {anchor_ms}"); + Ok(window) +} + +/// Splits `total` in proportion to the weights: the floor of each exact +/// share, then one sat each to the largest remainders, ties by identity. +fn allocate(total: u64, weights: &[(Identity, u128)]) -> Result> { + let mut ordered = weights.to_vec(); + ordered.sort_by(|left, right| left.0.cmp(&right.0)); + let sum: u128 = ordered.iter().map(|(_, weight)| weight).sum(); + ensure!(sum > 0, "nothing to allocate over"); + let mut shares: Vec<(Identity, u64, u128)> = ordered + .into_iter() + .map(|(identity, weight)| { + let product = u128::from(total) * weight; + Ok((identity, u64::try_from(product / sum)?, product % sum)) + }) + .collect::>()?; + let left = total - shares.iter().map(|(_, amount, _)| amount).sum::(); + let mut order: Vec = (0..shares.len()).collect(); + order.sort_by(|&left, &right| { + shares[right] + .2 + .cmp(&shares[left].2) + .then_with(|| shares[left].0.cmp(&shares[right].0)) + }); + for index in order.into_iter().take(usize::try_from(left)?) { + shares[index].1 += 1; + } + Ok(shares + .into_iter() + .map(|(identity, amount, _)| (identity, amount)) + .collect()) +} + +#[derive(Clone, Debug, PartialEq, Eq)] +struct Account { + identity: Identity, + gross: u64, + prior: i128, + candidate: i128, + onchain: u64, + fee: u64, + carry: i128, +} + +struct Settled { + /// Miner accounts by payout program. + accounts: BTreeMap, + pool_fee: u64, + /// The pool fee's share of its fanout's fee, when a fanout pays it. + pool_settlement_fee: u64, + earned_fee: u64, + swept_dust: u64, + /// Direct coinbase outputs by script. + direct: BTreeMap, + /// Fanout chunks: `(identity, gross, net)` in canonical order. + chunks: Vec>, +} + +/// Carried balances by payout program, with the identity they belong to. +type Carried = BTreeMap; + +impl Settled { + /// Each block moves a balance by what it earned less what it was paid. + fn next_prior(&self, prior: &Carried) -> Carried { + let mut next = prior.clone(); + for (program, account) in &self.accounts { + next.entry(program.clone()) + .or_insert_with(|| (account.identity.clone(), 0)) + .1 += i128::from(account.gross) - i128::from(account.onchain); + } + next.retain(|_, (_, balance)| *balance != 0); + next + } +} + +fn settle(value: u64, window: &Window, prior: &Carried, policy: &Policy) -> Result { + let earned_fee = u64::try_from(u128::from(value) * u128::from(POOL_FEE_BPS) / 10_000)?; + let miner_value = value - earned_fee; + ensure!(miner_value >= policy.floor, "coinbase below the floor"); + let weights: Vec<(Identity, u128)> = window.weights.values().cloned().collect(); + let mut accounts: BTreeMap = BTreeMap::new(); + for (identity, gross) in allocate(miner_value, &weights)? { + let prior = prior + .get(&identity.program) + .map_or(0, |(_, balance)| *balance); + accounts.insert( + identity.program.clone(), + Account { + identity, + gross, + prior, + candidate: prior + i128::from(gross), + onchain: 0, + fee: 0, + carry: 0, + }, + ); + } + for (program, (identity, balance)) in prior { + if !accounts.contains_key(program) { + accounts.insert( + program.clone(), + Account { + identity: identity.clone(), + gross: 0, + prior: *balance, + candidate: *balance, + onchain: 0, + fee: 0, + carry: 0, + }, + ); + } + } + let floor = i128::from(policy.floor); + let mut selected: Vec = accounts + .iter() + .filter(|(_, account)| account.candidate >= floor) + .map(|(program, _)| program.clone()) + .collect(); + let eligible_sum: i128 = selected + .iter() + .map(|program| accounts[program].candidate) + .sum(); + let mut swept_dust = 0; + if selected.is_empty() { + swept_dust = miner_value; + } else if eligible_sum < i128::from(miner_value) { + // The pool fee fronts the dust: every eligible balance is paid whole. + swept_dust = miner_value - u64::try_from(eligible_sum)?; + for program in &selected { + let account = accounts.get_mut(program).expect("selected"); + account.onchain = u64::try_from(account.candidate)?; + } + } else { + loop { + ensure!(!selected.is_empty(), "no recipient reaches the floor"); + let weights: Vec<(Identity, u128)> = selected + .iter() + .map(|program| { + let account = &accounts[program]; + Ok((account.identity.clone(), u128::try_from(account.candidate)?)) + }) + .collect::>()?; + let paid = allocate(miner_value, &weights)?; + let under: BTreeSet = paid + .iter() + .filter(|(_, amount)| *amount < policy.floor) + .map(|(identity, _)| identity.program.clone()) + .collect(); + if under.is_empty() { + for (identity, amount) in paid { + let account = accounts.get_mut(&identity.program).expect("selected"); + ensure!( + i128::from(amount) <= account.candidate, + "payout exceeds balance" + ); + account.onchain = amount; + } + break; + } + selected.retain(|program| !under.contains(program)); + } + } + for account in accounts.values_mut() { + account.carry = account.candidate - i128::from(account.onchain); + } + let pool_fee = earned_fee + swept_dust; + + // CTV settlement: the largest amounts at or above the direct floor take + // the direct slots; everyone else is paid through fanout chunks. Direct + // settlement pays every recipient in the coinbase. + let mut recipients: Vec<(Identity, u64)> = accounts + .values() + .filter(|account| account.onchain > 0) + .map(|account| (account.identity.clone(), account.onchain)) + .collect(); + if pool_fee > 0 { + recipients.push((policy.pool.clone(), pool_fee)); + } + recipients.sort_by(|left, right| left.0.cmp(&right.0)); + let mut candidates: Vec<&(Identity, u64)> = recipients + .iter() + .filter(|(_, amount)| !policy.ctv || *amount >= DIRECT_FLOOR_BITS) + .collect(); + candidates.sort_by(|left, right| right.1.cmp(&left.1).then_with(|| left.0.cmp(&right.0))); + let mut direct_count = candidates.len(); + if policy.ctv { + direct_count = direct_count.min(MAX_DIRECT_OUTPUTS); + while direct_count + (recipients.len() - direct_count).div_ceil(MAX_FANOUT_RECIPIENTS) + > MAX_SETTLEMENT_OUTPUTS + { + direct_count -= 1; + } + } + let direct_keys: BTreeSet<&Identity> = candidates[..direct_count] + .iter() + .map(|(identity, _)| identity) + .collect(); + let direct: BTreeMap = recipients + .iter() + .filter(|(identity, _)| direct_keys.contains(identity)) + .map(|(identity, amount)| (identity.script(), *amount)) + .collect(); + let fanout: Vec<(Identity, u64)> = recipients + .iter() + .filter(|(identity, _)| !direct_keys.contains(identity)) + .cloned() + .collect(); + let mut chunks = Vec::new(); + let mut pool_settlement_fee = 0; + for chunk in fanout.chunks(MAX_FANOUT_RECIPIENTS) { + let fee = fanout_fee(chunk.len()); + let gross: Vec<(Identity, u128)> = chunk + .iter() + .map(|(identity, amount)| (identity.clone(), u128::from(*amount))) + .collect(); + let total: u64 = chunk.iter().map(|(_, amount)| amount).sum(); + ensure!(fee < total, "fanout fee exceeds its chunk"); + // The fee's proportional split is the same largest-remainder rule. + let fees = allocate(fee, &gross)?; + let mut paid = Vec::with_capacity(chunk.len()); + for ((identity, amount), (fee_identity, fee)) in chunk.iter().zip(fees) { + ensure!(identity == &fee_identity, "fee split out of order"); + let net = amount - fee; + ensure!( + net >= policy.floor, + "fanout fee pushes {} below the floor", + identity.recipient + ); + if let Some(account) = accounts.get_mut(&identity.program) { + account.fee = fee; + } else { + pool_settlement_fee = fee; + } + paid.push((identity.clone(), *amount, net)); + } + chunks.push(paid); + } + Ok(Settled { + accounts, + pool_fee, + pool_settlement_fee, + earned_fee, + swept_dust, + direct, + chunks, + }) +} + +fn fanout_fee(recipients: usize) -> u64 { + let compact = if recipients < 253 { 1 } else { 3 }; + let weight = FANOUT_FIXED_WEIGHT + compact + FANOUT_OUTPUT_WEIGHT * recipients as u64; + (weight * FANOUT_RATE_PER_1000_WEIGHT * FANOUT_PREMIUM_BPS).div_ceil(1_000 * 10_000) +} + +// --------------------------------------------------------------------------- +// Comparing the model with the chain, the ledger and the wallets. + +/// The coinbase pays each direct recipient its exact amount, one covenant +/// output per fanout chunk carrying that chunk's gross, and nothing else. +fn check_coinbase( + block: &ChainBlock, + settled: &Settled, + pool: &Identity, + payees: &[Identity], +) -> Result<()> { + let known: BTreeSet = payees + .iter() + .chain(std::iter::once(pool)) + .map(Identity::script) + .collect(); + let mut direct = BTreeMap::new(); + let mut covenants = Vec::new(); + for (_, script, amount) in &block.outputs { + if *amount == 0 { + continue; + } + if known.contains(script) { + ensure!( + direct.insert(script.clone(), *amount).is_none(), + "block {} pays {script} twice", + block.hash + ); + } else { + covenants.push(*amount); + } + } + ensure!( + direct == settled.direct, + "block {} direct outputs differ from the recomputed PPLNS split:\n chain {direct:?}\n model {:?}", + block.hash, + settled.direct + ); + let mut expected: Vec = settled + .chunks + .iter() + .map(|chunk| chunk.iter().map(|(_, gross, _)| gross).sum()) + .collect(); + covenants.sort_unstable(); + expected.sort_unstable(); + ensure!( + covenants == expected, + "block {} covenant outputs {covenants:?}, recomputed chunks {expected:?}", + block.hash + ); + Ok(()) +} + +/// Every payee account the ledger recorded for the block matches the model; +/// accounts the model lacks carry nothing at all. +async fn check_carry_rows(fixture: &Fixture, block: &ChainBlock, settled: &Settled) -> Result<()> { + let rows = sqlx::query( + "SELECT miner_id,payout_order_key,encode(p2mr_program,'hex') AS program,gross_amount_sats,prior_balance_sats::text AS prior, + candidate_balance_sats::text AS candidate,onchain_amount_sats,settlement_fee_sats,carry_forward_balance_sats::text AS carry,action + FROM qbit_payout_carry_forward WHERE block_hash=$1 AND maturity_state<>'reversed'", + ) + .bind(&block.hash) + .fetch_all(&fixture.pool) + .await?; + let mut seen = BTreeSet::new(); + for row in &rows { + let program: String = row.try_get("program")?; + let recorded = Account { + identity: Identity { + order_key: row.try_get("payout_order_key")?, + recipient: row.try_get("miner_id")?, + program: program.clone(), + }, + gross: u64::try_from(row.try_get::("gross_amount_sats")?)?, + prior: row.try_get::("prior")?.parse()?, + candidate: row.try_get::("candidate")?.parse()?, + onchain: u64::try_from(row.try_get::("onchain_amount_sats")?)?, + fee: u64::try_from(row.try_get::("settlement_fee_sats")?)?, + carry: row.try_get::("carry")?.parse()?, + }; + let action: String = row.try_get("action")?; + match settled.accounts.get(&program) { + Some(expected) => { + ensure!( + &recorded == expected, + "block {} account differs from the model:\n ledger {recorded:?}\n model {expected:?}", + block.hash + ); + let paid = if expected.onchain > 0 { + "onchain" + } else { + "accrued" + }; + ensure!( + action == paid, + "block {} account {program} action {action}", + block.hash + ); + } + None => ensure!( + recorded.gross == 0 + && recorded.prior == 0 + && recorded.onchain == 0 + && recorded.carry == 0, + "block {} records an account the model lacks: {recorded:?}", + block.hash + ), + } + seen.insert(program); + } + for (program, account) in &settled.accounts { + ensure!( + seen.contains(program) || (account.prior == 0 && account.gross == 0), + "block {} ledger lacks account {program}", + block.hash + ); + } + Ok(()) +} + +/// The ledger's current carry-forward per program equals the model's. +async fn check_final_balances(fixture: &Fixture, carried: &Carried) -> Result<()> { + let model: BTreeMap = carried + .iter() + .map(|(program, (_, balance))| (program.clone(), *balance)) + .collect(); + let rows = sqlx::query( + "SELECT encode(p2mr_program,'hex') AS program,sum(gross_amount_sats-onchain_amount_sats)::text AS balance + FROM qbit_payout_carry_forward WHERE maturity_state<>'reversed' GROUP BY 1", + ) + .fetch_all(&fixture.pool) + .await?; + let mut recorded = BTreeMap::new(); + for row in rows { + let balance: i128 = row.try_get::("balance")?.parse()?; + if balance != 0 { + recorded.insert(row.try_get::("program")?, balance); + } + } + ensure!( + recorded == model, + "carried balances differ from the model:\n ledger {recorded:?}\n model {model:?}" + ); + Ok(()) +} + +/// A payee output on the chain: of which found block, direct in its coinbase +/// or through one of its fanouts. +struct PaidOutput { + block: usize, + /// The fanout transaction that paid it; `None` for a coinbase output. + fanout: Option, + txid: String, + vout: u32, + script: String, + amount: u64, +} + +type Paid = Vec; + +/// Each block's fanouts spend its coinbase covenant outputs, are the ones its +/// signed audit bundle committed to, and pay each fanout recipient its exact +/// net amount. Returns every payee output: the direct coinbase ones and the +/// fanout ones. +async fn check_fanouts( + fixture: &Fixture, + blocks: &[ChainBlock], + expected: &[(Window, Settled)], + bundle_fanouts: &[BTreeSet], + payees: &[Identity], +) -> Result { + let mut covenants: HashMap<(String, u32), usize> = HashMap::new(); + let payee_scripts: BTreeSet = payees.iter().map(Identity::script).collect(); + let mut paid = Paid::new(); + for (index, (block, (_, settled))) in blocks.iter().zip(expected).enumerate() { + for (vout, script, amount) in &block.outputs { + if payee_scripts.contains(script) { + paid.push(PaidOutput { + block: index, + fanout: None, + txid: block.coinbase_txid.clone(), + vout: *vout, + script: script.clone(), + amount: *amount, + }); + } else if *amount > 0 && !settled.direct.contains_key(script) { + covenants.insert((block.coinbase_txid.clone(), *vout), index); + } + } + } + let first = + blocks.first().context("no block")?.height + qbit_prism::QBIT_COINBASE_MATURITY_BLOCKS; + let tip = tip_height(fixture).await?; + let mut spent: BTreeMap>> = BTreeMap::new(); + let mut spenders: BTreeMap> = BTreeMap::new(); + for height in first..=tip { + let hash = fixture.rpc("getblockhash", json!([height])).await?; + let block = fixture.rpc("getblock", json!([hash, 2])).await?; + for tx in block["tx"] + .as_array() + .context("block txs missing")? + .iter() + .skip(1) + { + let spends: Vec = tx["vin"] + .as_array() + .context("inputs missing")? + .iter() + .filter_map(|input| { + let outpoint = ( + input["txid"].as_str()?.to_owned(), + u32::try_from(input["vout"].as_u64()?).ok()?, + ); + covenants.get(&outpoint).copied() + }) + .collect(); + let [index] = spends[..] else { + ensure!(spends.is_empty(), "a transaction spends several covenants"); + continue; + }; + let txid = tx["txid"].as_str().context("txid missing")?; + ensure!( + spenders.entry(index).or_default().insert(txid.into()), + "fanout {txid} mined twice" + ); + let mut outputs = BTreeMap::new(); + for (vout, script, amount) in outputs_of(tx)? { + if amount == 0 { + continue; + } + ensure!( + outputs.insert(script.clone(), amount).is_none(), + "fanout pays {script} twice" + ); + if payee_scripts.contains(&script) { + paid.push(PaidOutput { + block: index, + fanout: Some(txid.into()), + txid: txid.into(), + vout, + script, + amount, + }); + } + } + spent.entry(index).or_default().push(outputs); + } + } + for (index, (block, (_, settled))) in blocks.iter().zip(expected).enumerate() { + let committed = &bundle_fanouts[index]; + let mined = spenders.remove(&index).unwrap_or_default(); + ensure!( + &mined == committed, + "block {} fanouts on the chain {mined:?}, its audit bundle committed to {committed:?}", + block.hash + ); + let mut actual = spent.remove(&index).unwrap_or_default(); + let mut model: Vec> = settled + .chunks + .iter() + .map(|chunk| { + chunk + .iter() + .map(|(identity, _, net)| (identity.script(), *net)) + .collect() + }) + .collect(); + actual.sort(); + model.sort(); + ensure!( + actual == model, + "block {} fanout outputs differ from the model:\n chain {actual:?}\n model {model:?}", + block.hash + ); + } + Ok(paid) +} + +/// Each payee address received on the chain exactly what the model paid it +/// across all blocks, and the wallet that generated the address, and no +/// other payee wallet, holds all of it spendable. +async fn check_wallet_receipts( + fixture: &Fixture, + expected: &[(Window, Settled)], + paid: &Paid, + payees: &Payees, +) -> Result<()> { + let payee_scripts = payees.scripts(); + let mut model: BTreeMap = BTreeMap::new(); + for (_, settled) in expected { + let fanout = settled + .chunks + .iter() + .flatten() + .map(|(identity, _, net)| (identity.script(), *net)); + let direct = settled + .direct + .iter() + .map(|(script, amount)| (script.clone(), *amount)); + for (script, amount) in direct.chain(fanout) { + if payee_scripts.contains(&script) { + *model.entry(script).or_insert(0) += amount; + } + } + } + let mut chain: BTreeMap = BTreeMap::new(); + for output in paid { + *chain.entry(output.script.clone()).or_insert(0) += output.amount; + } + ensure!( + chain == model, + "payee receipts on the chain differ from the model:\n chain {chain:?}\n model {model:?}" + ); + for wallet in &payees.wallets { + let unspent = wallet_rpc(fixture, wallet, "listunspent", json!([1, 9_999_999])).await?; + let mut held: BTreeMap = BTreeMap::new(); + for coin in unspent.as_array().context("wallet coins missing")? { + ensure!(coin["spendable"] == true, "{wallet} cannot spend {coin}"); + *held + .entry( + coin["scriptPubKey"] + .as_str() + .context("coin script missing")? + .into(), + ) + .or_insert(0) += amount_bits(&coin["amount"])?; + } + let owned: BTreeMap = model + .iter() + .filter(|(script, _)| payees.owners.get(*script) == Some(wallet)) + .map(|(script, amount)| (script.clone(), *amount)) + .collect(); + ensure!( + held == owned, + "{wallet} holds other than what its addresses were paid:\n wallet {held:?}\n model {owned:?}" + ); + } + Ok(()) +} + +/// For every block, the smallest direct payee output and the smallest payee +/// output of each of its fanouts are spent, each by the wallet that owns its +/// address, and every spend confirms. Returns the outputs spent and the +/// number of wallets that spent them. +async fn spend_sample( + fixture: &Fixture, + ramp_address: &str, + blocks: usize, + paid: &Paid, + payees: &Payees, +) -> Result<(usize, usize)> { + let mut sources: BTreeSet<(usize, Option<&String>)> = BTreeSet::new(); + for output in paid { + sources.insert((output.block, output.fanout.as_ref())); + } + let mut inputs: BTreeMap<&String, Vec> = BTreeMap::new(); + for (block, fanout) in &sources { + let smallest = paid + .iter() + .filter(|output| output.block == *block && output.fanout.as_ref() == *fanout) + .min_by_key(|output| output.amount) + .context("a paying source lost its outputs")?; + let owner = payees + .owners + .get(&smallest.script) + .context("paid output has no owning wallet")?; + inputs + .entry(owner) + .or_default() + .push(json!({"txid": smallest.txid, "vout": smallest.vout})); + } + ensure!( + sources + .iter() + .map(|(block, _)| block) + .collect::>() + .len() + == blocks, + "a block paid no payee output to spend" + ); + let sink = wallet_rpc(fixture, "prism", "getnewaddress", json!(["", "p2mr"])).await?; + let mut spends = Vec::with_capacity(inputs.len()); + for (wallet, inputs) in &inputs { + let sent = wallet_rpc( + fixture, + wallet, + "sendall", + json!([[sink], null, "unset", null, {"inputs": inputs}]), + ) + .await?; + spends.push(( + *wallet, + sent["txid"] + .as_str() + .context("spend txid missing")? + .to_owned(), + )); + } + mine(fixture, ramp_address, 1).await?; + for (wallet, txid) in &spends { + let spend = wallet_rpc(fixture, wallet, "gettransaction", json!([txid])).await?; + ensure!( + spend["confirmations"].as_u64().unwrap_or(0) >= 1, + "{wallet}'s spend did not confirm: {spend}" + ); + } + for input in inputs.values().flatten() { + ensure!( + fixture + .rpc("gettxout", json!([input["txid"], input["vout"], false])) + .await? + .is_null(), + "spent payee output {input} is still unspent" + ); + } + Ok((sources.len(), inputs.len())) +} + +// --------------------------------------------------------------------------- +// The signed audit proof. + +/// Compares credit keyed by account: every expected account is credited +/// once with exactly its expected value, and no other account is credited. +/// Missing, extra and doubly credited accounts, and value moved from one +/// account to another, are all refused and named. +fn compare_credit( + what: &str, + expected: impl IntoIterator, + actual: impl IntoIterator, +) -> Result<()> +where + K: Ord + std::fmt::Debug, + V: PartialEq + std::fmt::Debug, +{ + let mut model = BTreeMap::new(); + for (key, value) in expected { + ensure!( + model.insert(key, value).is_none(), + "{what}: the model credits an account twice" + ); + } + let mut problems = Vec::new(); + let mut seen = BTreeMap::new(); + for (key, value) in actual { + match model.get(&key) { + None => problems.push(format!("extra credit {key:?} = {value:?}")), + Some(expected) if *expected != value => problems.push(format!( + "{key:?} credited {value:?}, the model {expected:?}" + )), + Some(_) => {} + } + if seen.insert(key, ()).is_some() { + problems.push("an account credited twice".into()); + } + } + for (key, value) in &model { + if !seen.contains_key(key) { + problems.push(format!("missing credit {key:?} = {value:?}")); + } + } + ensure!( + problems.is_empty(), + "{what} differs from the model: {}", + problems.join("; ") + ); + Ok(()) +} + +/// An account as the audit bundle and the model both name it. +type AccountKey = (String, String, String); + +fn key_of(identity: &Identity) -> AccountKey { + ( + identity.recipient.clone(), + identity.order_key.clone(), + identity.program.clone(), + ) +} + +fn key(recipient: &str, order_key: &str, program: &str) -> AccountKey { + ( + recipient.into(), + order_key.into(), + program.to_ascii_lowercase(), + ) +} + +/// Fetches block `block`'s audit bundle from server `server`, verifies its +/// signatures, keys and coinbase against the chain, and compares every +/// credit it signs with the model: the window's weights, each account's +/// gross, balances, payout, fee and carry, the pool fee, each coinbase +/// output and, under CTV, the settlement partition and chunks. Returns the +/// fanout transactions it commits to. +async fn check_audit_bundle( + fixture: &Fixture, + server: usize, + block: &ChainBlock, + window: &Window, + settled: &Settled, + policy: &Policy, +) -> Result> { + let body: Value = fixture + .client + .get(format!( + "http://127.0.0.1:{}/audit/blocks/{}/bundle", + fixture.api[server], block.hash + )) + .send() + .await? + .error_for_status()? + .json() + .await?; + let bundle: AuditBundle = serde_json::from_value(body["audit_bundle"].clone())?; + let coinbase = fixture + .rpc( + "getrawtransaction", + json!([block.coinbase_txid, false, block.hash]), + ) + .await?; + // The fixture's servers sign with the test seeds: 11 for the coinbase + // manifest, 22 for the ledger window. A bundle signed by any other key + // would verify against its own embedded key, so both are pinned. + let ledger_key = ManifestSigningKey::from_seed_hex(&"22".repeat(32))?.public_key_hex(); + let manifest_key = ManifestSigningKey::from_seed_hex(&"11".repeat(32))?.public_key_hex(); + let report = verify_audit_bundle_against_coinbase_tx_hex( + &bundle, + coinbase.as_str().context("node coinbase missing")?, + &ledger_key, + )?; + ensure!( + bundle.signed_coinbase_manifest.signature.public_key_hex == manifest_key, + "coinbase manifest signed by {}", + bundle.signed_coinbase_manifest.signature.public_key_hex + ); + ensure!( + report.coinbase_txid == block.coinbase_txid && report.block_height == block.height, + "bundle proves coinbase {} at {}", + report.coinbase_txid, + report.block_height + ); + ensure!( + report.coinbase_value_sats == block.value && report.min_output_sats == policy.floor, + "bundle value {} and floor {}", + report.coinbase_value_sats, + report.min_output_sats + ); + + let reward = &bundle.reward_manifest; + ensure!( + reward.included_share_count == window.shares, + "bundle counts {} shares, the model's window {}", + reward.included_share_count, + window.shares + ); + compare_credit( + "window weights", + window + .weights + .values() + .map(|(identity, weight)| (key_of(identity), *weight)), + reward.entitlements.iter().map(|entitlement| { + ( + key( + &entitlement.recipient_id, + &entitlement.order_key, + &entitlement.p2mr_program_hex, + ), + entitlement.weight, + ) + }), + )?; + + let manifest = &bundle.payout_policy_manifest; + let action = |onchain: u64| { + if onchain > 0 { + PayoutPolicyAction::Onchain + } else { + PayoutPolicyAction::Accrued + } + }; + let mut pool_accounts = Vec::new(); + let mut miner_accounts = Vec::new(); + for account in &manifest.accounts { + let entry = ( + key( + &account.recipient_id, + &account.order_key, + &account.p2mr_program_hex, + ), + ( + account.gross_amount_sats, + account.prior_balance_sats, + account.candidate_balance_sats, + account.onchain_amount_sats, + account.settlement_fee_sats, + account.carry_forward_balance_sats, + account.action.clone(), + ), + ); + match account.account_type { + PayoutPolicyAccountType::Miner => miner_accounts.push(entry), + PayoutPolicyAccountType::PoolFee => pool_accounts.push(entry), + } + } + compare_credit( + "miner accounts", + settled.accounts.values().map(|account| { + ( + key_of(&account.identity), + ( + account.gross, + account.prior, + account.candidate, + account.onchain, + account.fee, + account.carry, + action(account.onchain), + ), + ) + }), + miner_accounts, + )?; + let pool_fee = i128::from(settled.pool_fee); + compare_credit( + "pool fee account", + (settled.pool_fee > 0).then(|| { + ( + key_of(&policy.pool), + ( + settled.pool_fee, + 0, + pool_fee, + settled.pool_fee, + settled.pool_settlement_fee, + 0, + PayoutPolicyAction::Onchain, + ), + ) + }), + pool_accounts, + )?; + let fee = manifest.pool_fee.as_ref().context("no pool fee manifest")?; + ensure!( + u64::from(fee.fee_bps) == POOL_FEE_BPS + && fee.earned_pool_fee_sats == settled.earned_fee + && fee.swept_dust_liability_sats == settled.swept_dust + && fee.amount_sats == settled.pool_fee + && key(&fee.recipient_id, &fee.order_key, &fee.p2mr_program_hex) + == key_of(&policy.pool), + "pool fee manifest {fee:?}, model earned {} swept {} to {}", + settled.earned_fee, + settled.swept_dust, + policy.pool.recipient + ); + + // The signed coinbase: each direct output is credited to the account + // whose program its script pays; the other outputs are the covenants. + let identities: BTreeMap = settled + .accounts + .values() + .map(|account| &account.identity) + .chain(std::iter::once(&policy.pool)) + .map(|identity| (identity.script(), identity)) + .collect(); + let mut direct = Vec::new(); + let mut covenants = Vec::new(); + for output in &bundle.signed_coinbase_manifest.manifest.outputs { + if output.amount_sats == 0 { + continue; + } + if identities.contains_key(&output.script_pubkey_hex) { + ensure!( + output.script_pubkey_hex == format!("5220{}", output.p2mr_program_hex), + "output {} credits {} but pays {}", + output.vout, + output.p2mr_program_hex, + output.script_pubkey_hex + ); + direct.push(( + key( + &output.recipient_id, + &output.order_key, + &output.p2mr_program_hex, + ), + output.amount_sats, + )); + } else { + covenants.push(output.amount_sats); + } + } + compare_credit( + "direct coinbase credit", + settled.direct.iter().map(|(script, amount)| { + let identity = identities[script]; + (key_of(identity), *amount) + }), + direct, + )?; + let mut chunk_totals: Vec = settled + .chunks + .iter() + .map(|chunk| chunk.iter().map(|(_, gross, _)| gross).sum()) + .collect(); + chunk_totals.sort_unstable(); + covenants.sort_unstable(); + ensure!( + covenants == chunk_totals, + "bundle covenant outputs {covenants:?}, the model's chunks {chunk_totals:?}" + ); + + let mut fanouts = BTreeSet::new(); + if policy.ctv { + let decision = bundle + .settlement_mode_decision + .as_ref() + .context("CTV bundle without a settlement decision")?; + let mode = match (settled.direct.is_empty(), settled.chunks.is_empty()) { + (false, true) => SettlementMode::DirectCoinbase, + (true, false) => SettlementMode::CtvFanout, + _ => SettlementMode::HybridCoinbaseCtvFanout, + }; + ensure!( + decision.mode == mode && decision.fanout_chunk_count == settled.chunks.len(), + "settlement {:?} in {} chunks, the model {mode:?} in {}", + decision.mode, + decision.fanout_chunk_count, + settled.chunks.len() + ); + compare_credit( + "direct settlement recipients", + settled + .direct + .iter() + .map(|(script, amount)| (key_of(identities[script]), *amount)), + decision.direct_recipients.iter().map(|recipient| { + ( + key( + &recipient.recipient_id, + &recipient.order_key, + &recipient.p2mr_program_hex, + ), + recipient.amount_sats, + ) + }), + )?; + compare_credit( + "fanout settlement recipients", + settled + .chunks + .iter() + .enumerate() + .flat_map(|(index, chunk)| { + chunk + .iter() + .map(move |(identity, gross, _)| ((index, key_of(identity)), *gross)) + }), + decision.fanout_chunks.iter().flat_map(|chunk| { + chunk.recipients.iter().map(|recipient| { + ( + ( + chunk.chunk_index, + key( + &recipient.recipient_id, + &recipient.order_key, + &recipient.p2mr_program_hex, + ), + ), + recipient.amount_sats, + ) + }) + }), + )?; + if let Some(set) = &bundle.ctv_fanout_manifest_set { + for manifest in &set.manifests { + ensure!( + manifest.parent_coinbase_txid == block.coinbase_txid, + "fanout {} spends coinbase {}", + manifest.fanout_txid, + manifest.parent_coinbase_txid + ); + ensure!( + fanouts.insert(manifest.fanout_txid.clone()), + "bundle commits to fanout {} twice", + manifest.fanout_txid + ); + } + } + ensure!( + fanouts.len() == settled.chunks.len(), + "bundle commits to {} fanouts, the model has {} chunks", + fanouts.len(), + settled.chunks.len() + ); + } else { + ensure!( + bundle + .settlement_mode_decision + .as_ref() + .is_none_or(|decision| decision.mode == SettlementMode::DirectCoinbase + && decision.fanout_chunks.is_empty()) + && bundle.ctv_fanout_manifest_set.is_none(), + "direct settlement bundle carries CTV settlement" + ); + } + Ok(fanouts) +} + +#[test] +fn credit_comparison_refuses_missing_extra_double_and_moved_credit() { + let expected = || [("alice", 5u64), ("bob", 3), ("carol", 1)]; + assert!(compare_credit("exact", expected(), expected()).is_ok()); + let failures = [ + ("missing", vec![("alice", 5), ("bob", 3)], "missing credit"), + ( + "extra", + vec![("alice", 5), ("bob", 3), ("carol", 1), ("mallory", 1)], + "extra credit", + ), + ( + "double", + vec![("alice", 5), ("bob", 3), ("carol", 1), ("carol", 1)], + "credited twice", + ), + ( + "moved", + vec![("alice", 3), ("bob", 5), ("carol", 1)], + "credited 3, the model 5", + ), + ( + "renamed", + vec![("alice", 5), ("bob", 3), ("dave", 1)], + "missing credit \"carol\"", + ), + ]; + for (case, actual, reason) in failures { + let error = compare_credit(case, expected(), actual) + .expect_err(case) + .to_string(); + assert!(error.contains(reason), "{case}: {error}"); + } +} + +// --------------------------------------------------------------------------- +// The scenario was what it claims to be. + +struct Summary { + payees: usize, + workers: usize, + shares: usize, + stale_retries: usize, + both_servers: usize, + multi_difficulty: usize, + spread: f64, + blocks: usize, + cut_windows: usize, + window_shares: Vec, + direct: Vec, + fanout: Vec, + paid_payees: usize, + dust_accounts: Vec, + dust_later_paid: usize, + swept: u64, + pool_fee: u64, + audited: usize, + spent: usize, + spending_wallets: usize, +} + +impl Summary { + fn new( + plan: &Plan, + attribution: &Attribution, + verified: &Verified, + stale_retries: usize, + ) -> Self { + let expected = &verified.expected; + let mut paid: BTreeSet<&String> = BTreeSet::new(); + let mut accrued: BTreeSet<&String> = BTreeSet::new(); + let mut dust_later_paid = BTreeSet::new(); + for (_, settled) in expected { + for (program, account) in &settled.accounts { + if account.onchain > 0 { + paid.insert(program); + if accrued.contains(program) && account.prior > 0 { + dust_later_paid.insert(program); + } + } else if account.carry > 0 { + accrued.insert(program); + } + } + } + Self { + payees: plan + .workers + .iter() + .map(|worker| worker.payee) + .collect::>() + .len(), + workers: plan.workers.len(), + shares: attribution.shares, + stale_retries, + both_servers: attribution.both_servers, + multi_difficulty: attribution.multi_difficulty, + spread: attribution.spread, + blocks: expected.len(), + cut_windows: expected.iter().filter(|(window, _)| window.cut).count(), + window_shares: expected.iter().map(|(window, _)| window.shares).collect(), + direct: expected + .iter() + .map(|(_, settled)| settled.direct.len()) + .collect(), + fanout: expected + .iter() + .map(|(_, settled)| settled.chunks.iter().map(Vec::len).sum()) + .collect(), + paid_payees: paid.len(), + dust_accounts: expected + .iter() + .map(|(_, settled)| { + settled + .accounts + .values() + .filter(|account| account.onchain == 0 && account.carry > 0) + .count() + }) + .collect(), + dust_later_paid: dust_later_paid.len(), + swept: expected.iter().map(|(_, settled)| settled.swept_dust).sum(), + pool_fee: expected.iter().map(|(_, settled)| settled.pool_fee).sum(), + audited: expected.len(), + spent: verified.spent, + spending_wallets: verified.spending_wallets, + } + } + + /// The seeded scenario exercised what it is for; a change that made it + /// degenerate (one recipient, no dust, no cut) fails here. + fn check(&self, scenario: &Scenario) -> Result<()> { + ensure!( + self.payees == scenario.payees && self.blocks == scenario.blocks, + "scenario size changed: {self}" + ); + ensure!( + self.spread >= 500.0, + "share weights span only {:.0}x: {self}", + self.spread + ); + ensure!( + self.both_servers > 0 && self.multi_difficulty > 0, + "no address mined across servers and difficulties: {self}" + ); + ensure!( + self.cut_windows > 0, + "no window ended inside a share: {self}" + ); + if scenario.ctv { + ensure!( + self.direct.iter().all(|count| *count > 0) + && self + .fanout + .iter() + .all(|count| *count >= scenario.payees / 8), + "blocks did not pay both direct and many fanout recipients: {self}" + ); + } else { + ensure!( + self.direct.iter().all(|count| *count >= 3) + && self.fanout.iter().all(|count| *count == 0), + "direct settlement did not pay several recipients in every coinbase: {self}" + ); + } + ensure!( + self.dust_accounts.iter().all(|count| *count > 0) && self.swept > 0, + "no balance was carried below the floor: {self}" + ); + ensure!( + self.dust_later_paid > 0, + "no carried balance was later paid: {self}" + ); + ensure!( + self.paid_payees * 3 >= scenario.payees, + "too few payees were paid: {self}" + ); + ensure!( + self.audited == scenario.blocks && self.spent >= scenario.blocks, + "not every block was audited and spent from: {self}" + ); + Ok(()) + } +} + +impl std::fmt::Display for Summary { + fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!( + formatter, + "{} payees on {} workers submitted {} accepted shares ({} stale retries), {} payees across both servers, \ + {} at several difficulties, share weights spanning {:.0}x; {} blocks with window shares {:?} ({} cut), \ + direct outputs {:?}, fanout recipients {:?}, {} payees paid, sub-floor accounts {:?} ({} later paid, {} bits swept \ + into the {} bit pool fee), {} signed audit bundles matched the model, {} payee outputs spent by \ + {} owning wallets", + self.payees, + self.workers, + self.shares, + self.stale_retries, + self.both_servers, + self.multi_difficulty, + self.spread, + self.blocks, + self.window_shares, + self.cut_windows, + self.direct, + self.fanout, + self.paid_payees, + self.dust_accounts, + self.dust_later_paid, + self.swept, + self.pool_fee, + self.audited, + self.spent, + self.spending_wallets + ) + } +} diff --git a/crates/qbit-prism-server/tests/support/offer_standby_flush.rs b/crates/qbit-prism-server/tests/support/offer_standby_flush.rs new file mode 100644 index 000000000..731e7a622 --- /dev/null +++ b/crates/qbit-prism-server/tests/support/offer_standby_flush.rs @@ -0,0 +1,341 @@ +//! #529 on a disposable primary and its dedicated asynchronous standby +//! (`synchronous_standby_names=''`): the found-block offer's standby wait +//! confirms against the standby's `flush_lsn`, returns at its bound while the +//! standby is behind, does not wait for a standby that stopped, and cannot +//! confirm for a writer role without `pg_monitor` or for a second standby that +//! shares the name and lags. `self-check`'s report refuses each of those. The writer is an ordinary role +//! that owns the database, as in production. +//! +//! A standby falls behind for real: its WAL receiver is suspended, so it +//! neither flushes nor reports, while its connection stays `streaming` on the +//! primary. The primary's WAL sender still sends into the socket, so +//! `sent_lsn` passes the target while `flush_lsn` does not. +use super::*; +use qbit_prism_server::ledger::{OfferStandbyWait, StandbyDurability, StandbyWait}; + +const STANDBY: &str = "prism_standby_1"; +const WRITER: &str = "prism_writer"; + +/// A disposable standby of the pair's primary. +struct Standby { + _cluster: Cluster, + port: u16, +} + +/// Dropped in field order: the standbys stop before their primary, and the +/// directory holding them all goes last. +struct Pair { + standby: Option, + _primary: Cluster, + admin: PgPool, + writer: Ledger, + bin: PathBuf, + primary_port: u16, + username: String, + next_share: u64, + dir: tempfile::TempDir, +} + +impl Pair { + async fn start(bin: PathBuf) -> Result { + let dir = tempfile::tempdir()?; + let primary = Cluster { + bin: bin.clone(), + data: dir.path().join("primary"), + }; + primary.command( + "initdb", + &[ + "-D", + primary.data.to_str().context("invalid test path")?, + "-A", + "trust", + "--no-locale", + "-E", + "UTF8", + ], + )?; + let primary_port = port()?; + primary.start(primary_port, dir.path())?; + let username = String::from_utf8(Command::new("id").arg("-un").output()?.stdout)? + .trim() + .to_owned(); + let admin = PgPool::connect(&format!( + "postgresql://{username}@127.0.0.1:{primary_port}/postgres" + )) + .await?; + let names: String = sqlx::query_scalar("SHOW synchronous_standby_names") + .fetch_one(&admin) + .await?; + ensure!(names.is_empty(), "D3 is asynchronous, found {names:?}"); + sqlx::raw_sql(&format!( + "CREATE ROLE {WRITER} LOGIN NOSUPERUSER; ALTER DATABASE postgres OWNER TO {WRITER}; \ + ALTER SCHEMA public OWNER TO {WRITER}" + )) + .execute(&admin) + .await?; + let writer = Ledger::connect( + &format!("postgresql://{WRITER}@127.0.0.1:{primary_port}/postgres"), + "offer-standby".into(), + 4, + true, + ) + .await?; + let mut pair = Self { + standby: None, + _primary: primary, + admin, + writer, + bin, + primary_port, + username, + next_share: 1, + dir, + }; + pair.standby = Some(pair.add_standby("standby", STANDBY, "").await?); + pair.until_connected(STANDBY, 1).await?; + Ok(pair) + } + + /// Base-backup and start a standby under `name`, with `settings` appended + /// to its configuration. + async fn add_standby(&self, directory: &str, name: &str, settings: &str) -> Result { + let cluster = Cluster { + bin: self.bin.clone(), + data: self.dir.path().join(directory), + }; + cluster.command( + "pg_basebackup", + &[ + "-D", + cluster.data.to_str().context("invalid test path")?, + "-d", + &format!( + "host=127.0.0.1 port={} user={} application_name={name}", + self.primary_port, self.username + ), + "-R", + "-X", + "stream", + "-c", + "fast", + ], + )?; + std::fs::write( + cluster.data.join("postgresql.auto.conf"), + format!( + "{}\n{settings}\n", + std::fs::read_to_string(cluster.data.join("postgresql.auto.conf"))? + ), + )?; + let port = port()?; + cluster.start(port, self.dir.path())?; + Ok(Standby { + _cluster: cluster, + port, + }) + } + + async fn until_connected(&self, name: &str, rows: i64) -> Result<()> { + timeout(Duration::from_secs(30), async { + loop { + let streaming: i64 = sqlx::query_scalar( + "SELECT count(*) FROM pg_stat_replication WHERE application_name=$1 AND state='streaming'", + ) + .bind(name) + .fetch_one(&self.admin) + .await?; + if streaming == rows { + return Ok::<_, anyhow::Error>(()); + } + sleep(Duration::from_millis(20)).await; + } + }) + .await + .with_context(|| format!("{name} never showed {rows} streaming rows"))? + } + + /// Commit a share on the primary: new WAL the standby has to flush. + async fn commit(&mut self) -> Result<()> { + self.writer.append(proof(self.next_share), None).await?; + self.next_share += 1; + Ok(()) + } + + async fn wait(&self, name: &str, bound: Duration) -> StandbyWait { + self.writer + .await_standby_flush(&OfferStandbyWait { + application_name: name.into(), + bound, + }) + .await + } + + async fn report(&self, name: &str) -> Result { + self.writer + .offer_standby_report(&OfferStandbyWait { + application_name: name.into(), + bound: Duration::from_secs(10), + }) + .await + } + + /// Suspend a standby's WAL receiver until the guard drops. + async fn stall(&self, standby: &Standby) -> Result { + let pool = PgPool::connect(&format!( + "postgresql://{}@127.0.0.1:{}/postgres", + self.username, standby.port + )) + .await?; + let pid: i32 = sqlx::query_scalar("SELECT pid FROM pg_stat_wal_receiver") + .fetch_one(&pool) + .await?; + pool.close().await; + Suspended::pid(pid) + } +} + +struct Suspended(i32); +impl Suspended { + fn pid(pid: i32) -> Result { + let status = Command::new("kill") + .args(["-STOP", &pid.to_string()]) + .status()?; + ensure!( + status.success(), + "could not suspend the standby's WAL receiver" + ); + Ok(Self(pid)) + } +} +impl Drop for Suspended { + fn drop(&mut self) { + let _ = Command::new("kill") + .args(["-CONT", &self.0.to_string()]) + .status(); + } +} + +fn refused(report: &qbit_prism_server::ledger::OfferStandbyReport) -> String { + report + .ensure_usable() + .err() + .map(|error| error.to_string()) + .unwrap_or_default() +} + +/// One pair, in the order an operator meets these states: a writer role +/// without `pg_monitor`, the grant, a healthy standby, a lagging one, a second +/// standby sharing the name, and a stopped one. +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn offer_standby_wait_confirms_the_flush_returns_at_the_bound_and_never_waits_for_a_stopped_standby( +) -> Result<()> { + let Some(bin) = gate::pg_bin_dir(gate::site!())? else { + return Ok(()); + }; + let mut pair = Pair::start(bin.into()).await?; + let long = Duration::from_secs(10); + let bound = Duration::from_millis(400); + + // Without pg_read_all_stats the positions read NULL: the wait reports + // `failed` at once, and self-check refuses it, naming the grant. + pair.commit().await?; + let wait = pair.wait(STANDBY, long).await; + ensure!( + matches!(&wait.durability, StandbyDurability::Failed { error } if error.contains("pg_monitor")) + && wait.waited < Duration::from_secs(2), + "a writer that cannot read replication positions must fail fast: {wait:?}" + ); + let report = pair.report(STANDBY).await?; + ensure!( + !report.role_can_read_positions && refused(&report).contains("GRANT pg_monitor"), + "self-check accepted a writer without pg_monitor: {report:?}" + ); + + sqlx::query(&format!("GRANT pg_monitor TO {WRITER}")) + .execute(&pair.admin) + .await?; + + // Healthy: confirmed well inside the bound, and self-check accepts. + pair.commit().await?; + let wait = pair.wait(STANDBY, long).await; + ensure!( + matches!(wait.durability, StandbyDurability::Confirmed) + && wait.waited < Duration::from_secs(5), + "a healthy standby did not confirm: {wait:?}" + ); + let report = pair.report(STANDBY).await?; + ensure!( + report.role_can_read_positions && report.standbys == 1 && report.reporting_flush == 1, + "{report:?}" + ); + report.ensure_usable()?; + // Another application name is not the failover standby. + let other = pair.wait("prism_public_replica", long).await; + ensure!( + matches!(other.durability, StandbyDurability::Absent), + "a standby under another application_name counted: {other:?}" + ); + + // Lagging: the standby flushes nothing; the wait ends at its bound, never + // before and never much after, with the lag, and the caller offers. + { + let standby = pair.standby.as_ref().context("the standby")?; + let _stalled = pair.stall(standby).await?; + pair.commit().await?; + let wait = pair.wait(STANDBY, bound).await; + ensure!( + matches!(wait.durability, StandbyDurability::Lagging { lag_bytes } if lag_bytes > 0) + && wait.waited >= bound + && wait.waited < bound + Duration::from_secs(2), + "a standby that has not flushed the reservation must hold the offer for the bound, then let it go: {wait:?}" + ); + } + // Resumed, it catches up and confirms. + let wait = pair.wait(STANDBY, long).await; + ensure!( + matches!(wait.durability, StandbyDurability::Confirmed), + "the resumed standby did not confirm: {wait:?}" + ); + + // A second standby under the same name that falls behind: the healthy one + // has flushed, but which of the two would be promoted is unknown, so the + // wait must not confirm, and self-check refuses the ambiguous name. + { + let twin = pair.add_standby("twin", STANDBY, "").await?; + pair.until_connected(STANDBY, 2).await?; + let _stalled = pair.stall(&twin).await?; + pair.commit().await?; + let wait = pair.wait(STANDBY, bound).await; + ensure!( + matches!(wait.durability, StandbyDurability::Lagging { .. }) && wait.waited >= bound, + "a lagging second standby under the name was confirmed by the healthy one: {wait:?}" + ); + let report = pair.report(STANDBY).await?; + ensure!( + report.standbys == 2 && refused(&report).contains("matches 2 connected standbys"), + "{report:?}" + ); + } + pair.until_connected(STANDBY, 1).await?; + + // Stopped: nothing is connected, so the offer does not wait at all, and + // self-check refuses a wait that protects nothing. + drop(pair.standby.take()); + pair.until_connected(STANDBY, 0).await?; + pair.commit().await?; + let wait = pair.wait(STANDBY, long).await; + ensure!( + matches!(wait.durability, StandbyDurability::Absent) + && wait.waited < Duration::from_secs(2), + "a stopped standby held the offer: {wait:?}" + ); + let report = pair.report(STANDBY).await?; + ensure!( + report.standbys == 0 && refused(&report).contains("matches 0 connected standbys"), + "{report:?}" + ); + pair.writer.pool.close().await; + pair.admin.close().await; + Ok(()) +} diff --git a/crates/qbit-prism-server/tests/support/policy_transition.rs b/crates/qbit-prism-server/tests/support/policy_transition.rs new file mode 100644 index 000000000..a3c63a0c5 --- /dev/null +++ b/crates/qbit-prism-server/tests/support/policy_transition.rs @@ -0,0 +1,671 @@ +use super::*; +use qbit_prism_server::{ + config::Config, + ledger::{HeartbeatHealth, HeartbeatStatus, OfferOutcome}, +}; +use serde_json::Value; +use std::time::Duration; + +use super::fake_qbitd as fake; + +async fn setup(db: &Database) -> Result<(Ledger, Ledger, fake::FakeNode, Config)> { + setup_ctv(db, false).await +} + +async fn setup_ctv(db: &Database, ctv: bool) -> Result<(Ledger, Ledger, fake::FakeNode, Config)> { + let node = fake::FakeNode::open().await?; + let mut config = fake::coordinator_config(db.url.clone(), &node, "operator")?; + config.manifest_seed = "42".repeat(32); + config.ledger_seed = "43".repeat(32); + config.ledger_public_key = keys().1.public_key_hex(); + config.username_fallback = Some("policy-test-fallback".into()); + config.ctv_enabled = ctv; + if ctv { + config.ctv_fee = Some(qbit_prism::FanoutFeeRatePolicy::new(1, 12000)); + // #525: CTV settlement runs only with a pool fee; `CTV_POOL_FEE_ENV` + // configures this same 0 bps fee in the CLI's environment. + let [_, _, (_, recipient), (_, program)] = CTV_POOL_FEE_ENV; + config.payout_policy.pool_fee_policy = Some(qbit_prism::PoolFeePolicy { + fee_bps: 0, + recipient_id: recipient.into(), + order_key: recipient.into(), + p2mr_program_hex: program.into(), + }); + } + let a = db.ledger("frontend-a").await?; + let b = db.ledger("frontend-b").await?; + for ledger in [&a, &b] { + ledger + .configure( + &config.fingerprint(&"00".repeat(32))?, + &SignerKeys::of(&keys().0, &keys().1), + ) + .await?; + } + Ok((a, b, node, config)) +} + +const CTV_POOL_FEE_ENV: [(&str, &str); 4] = [ + ("PRISM_POOL_FEE_ENABLED", "1"), + ("PRISM_POOL_FEE_BPS", "0"), + ("PRISM_POOL_FEE_RECIPIENT_ID", "ctv-pool-fee"), + ( + "PRISM_POOL_FEE_P2MR_PROGRAM_HEX", + "fefefefefefefefefefefefefefefefefefefefefefefefefefefefefefefefe", + ), +]; + +fn changed_fee(config: &Config) -> Config { + let mut next = config.clone(); + next.payout_policy.pool_fee_policy = Some(qbit_prism::PoolFeePolicy { + fee_bps: 200, + recipient_id: "fee".into(), + order_key: "fee".into(), + p2mr_program_hex: "11".repeat(32), + }); + next +} + +#[tokio::test] +async fn ctv_fee_transition_can_repair_a_rate_below_the_live_floor() -> Result<()> { + let Some(db) = Database::open().await? else { + return Ok(()); + }; + let (a, b, node, config) = setup_ctv(&db, true).await?; + a.heartbeat(HeartbeatStatus::Stopped).await?; + b.heartbeat(HeartbeatStatus::Stopped).await?; + let before = state(&a).await?; + let dir = tempfile::tempdir()?; + let path = dir.path().join("ctv.env"); + std::fs::write( + &path, + "PRISM_CTV_FANOUT_FEE_MARKET_RATE_BITS_PER_1000_WEIGHT=2\n", + )?; + let run = || { + let mut command = cli(&db, &node, &path); + command + .env("PRISM_CTV_SETTLEMENT_ENABLED", "1") + .env("PRISM_CTV_FANOUT_FEE_MARKET_RATE_BITS_PER_1000_WEIGHT", "1") + .envs(CTV_POOL_FEE_ENV); + command + }; + let output = run().output().await?; + assert!(!output.status.success()); + assert!(String::from_utf8_lossy(&output.stderr).contains("relay floor")); + assert_eq!(state(&a).await?, before); + std::fs::write(&path, "PRISM_CTV_FANOUT_FEE_MARKET_RATE_BITS_PER_1000_WEIGHT=2000\nPRISM_CTV_FANOUT_FEE_PREMIUM_BPS=15000\n")?; + let output = run().output().await?; + assert!( + output.status.success(), + "{}", + String::from_utf8_lossy(&output.stderr) + ); + let event: Value = serde_json::from_slice(&output.stdout)?; + let mut next = config; + next.ctv_fee = Some(qbit_prism::FanoutFeeRatePolicy::new(2000, 15000)); + assert_eq!( + event["config_fingerprint"], + next.fingerprint(&"00".repeat(32))? + ); + // Switching to the automatic estimator binds its premium; it uses the + // same floor validation and does not replace that input with today's estimate. + let mut automatic = next.clone(); + automatic.ctv_fee = None; + automatic.ctv_fee_premium_bps = 16000; + a.transition_policy(&next, &automatic).await?; + let mut premium = automatic.clone(); + premium.ctv_fee_premium_bps = 17000; + a.transition_policy(&automatic, &premium).await?; + assert_eq!( + a.payout_revision().await?, + before["cluster"]["payout_revision"].as_i64().unwrap() + 3 + ); + db.close(vec![a, b]).await +} + +async fn state(ledger: &Ledger) -> Result { + Ok(sqlx::query_scalar("SELECT jsonb_build_object('cluster',(SELECT to_jsonb(c) FROM qbit_prism_cluster c),'outbox',(SELECT jsonb_agg(to_jsonb(o) ORDER BY block_hash) FROM qbit_block_candidate_outbox o),'events',(SELECT jsonb_agg(to_jsonb(e)) FROM qbit_prism_policy_transitions e))") + .fetch_one(&ledger.pool).await?) +} + +#[tokio::test] +async fn two_frontends_must_stop_and_old_work_and_startup_are_rejected() -> Result<()> { + let Some(db) = Database::open().await? else { + return Ok(()); + }; + let (a, b, _node, config) = setup(&db).await?; + let next = changed_fee(&config); + let before = state(&a).await?; + for status in [ + json!({"state":"starting","candidate_offer_lifecycle":1}), + json!({"state":"draining"}), + json!({"state":"drained"}), + json!({}), + json!({"schema":"qbit.prism.audit-health.v1","ready":true,"state":"stopped"}), + ] { + sqlx::query("UPDATE qbit_prism_instances SET status=$1,heartbeat_at=clock_timestamp()-interval '1 day' WHERE instance_id='frontend-b'") + .bind(status).execute(&a.pool).await?; + let error = a + .transition_policy(&config, &next) + .await + .unwrap_err() + .to_string(); + assert!( + error.contains("frontend-b") && error.contains("stopped"), + "{error}" + ); + assert_eq!(state(&a).await?, before); + } + a.heartbeat(HeartbeatStatus::Stopped).await?; + b.heartbeat(HeartbeatStatus::Stopped).await?; + let revision = a.payout_revision().await?; + let event = a.transition_policy(&config, &next).await?; + assert_eq!(event["payout_revision"], revision + 1); + assert_eq!(event["previous_revision"], revision); + assert_eq!( + event["config_fingerprint"], + next.fingerprint(&"00".repeat(32))? + ); + assert!( + b.new_session_id().await.is_err(), + "stopped frontend admitted a session" + ); + assert!(b + .save_job("old-job", &json!({}), revision, "parent", 60) + .await + .is_err()); + assert!(b + .append_at_revision(share(1), None, revision) + .await + .is_err()); + let mut rejected_config = config.clone(); + rejected_config.instance_id = "frontend-b".into(); + let rejected = qbit_prism_server::coordinator::Coordinator::new( + rejected_config, + std::sync::Arc::new(qbit_prism_server::metrics::Metrics::default()), + ) + .await + .err() + .context("old-policy coordinator started")?; + assert!(rejected.to_string().contains("fingerprint mismatch")); + assert_eq!( + sqlx::query_scalar::<_, String>( + "SELECT status->>'state' FROM qbit_prism_instances WHERE instance_id='frontend-b'" + ) + .fetch_one(&a.pool) + .await?, + "starting" + ); + let restart = db.ledger("frontend-b").await?; + let error = restart + .configure( + &config.fingerprint(&"00".repeat(32))?, + &SignerKeys::of(&keys().0, &keys().1), + ) + .await + .unwrap_err() + .to_string(); + assert!( + error.contains(&format!("payout revision {}", revision + 1)), + "{error}" + ); + restart + .configure( + &next.fingerprint(&"00".repeat(32))?, + &SignerKeys::of(&keys().0, &keys().1), + ) + .await?; + restart + .save_job("new-job", &json!({}), revision + 1, "parent", 60) + .await?; + assert!( + restart + .append_at_revision(share(2), None, revision + 1) + .await? + .inserted + ); + db.close(vec![a, b, restart]).await +} + +#[tokio::test] +async fn rejected_same_id_startup_cannot_authorize_a_policy_transition() -> Result<()> { + let Some(db) = Database::open().await? else { + return Ok(()); + }; + let (a, b, _node, config) = setup(&db).await?; + let next = changed_fee(&config); + let held = a.new_session_id().await?; + a.heartbeat(HeartbeatStatus::Health(HeartbeatHealth::new( + true, + Default::default(), + ))) + .await?; + b.heartbeat(HeartbeatStatus::Stopped).await?; + a.append(share(1), None).await?; + let pending = candidate(&a.snapshot(100).await?, 2895)?; + a.enqueue_candidate(pending.candidate).await?; + let before = state(&a).await?; + + // The rejected process shares a's ID but owns none of its live sessions. + let mut rejected_config = next.clone(); + rejected_config.instance_id = "frontend-a".into(); + let rejected = qbit_prism_server::coordinator::Coordinator::new( + rejected_config, + std::sync::Arc::new(qbit_prism_server::metrics::Metrics::default()), + ) + .await + .err() + .context("mismatched-policy coordinator started")?; + assert!(rejected.to_string().contains("fingerprint mismatch")); + + // Try before a's next heartbeat: the shared row must not prove quiescence. + let error = a + .transition_policy(&config, &next) + .await + .unwrap_err() + .to_string(); + assert!( + error.contains("frontend-a") && error.contains("stopped"), + "{error}" + ); + assert_eq!(state(&a).await?, before); + assert_eq!( + sqlx::query_scalar::<_, String>( + "SELECT status->>'state' FROM qbit_prism_instances WHERE instance_id='frontend-a'" + ) + .fetch_one(&a.pool) + .await?, + "starting" + ); + a.new_session_id().await?.release().await?; + assert!(a.heartbeat(HeartbeatStatus::Stopped).await.is_err()); + held.release().await?; + a.heartbeat(HeartbeatStatus::Stopped).await?; + let event = a.transition_policy(&config, &next).await?; + assert_eq!( + event["payout_revision"], + before["cluster"]["payout_revision"].as_i64().unwrap() + 1 + ); + assert_eq!(event["abandoned_candidates"], 1); + db.close(vec![a, b]).await +} + +#[tokio::test] +async fn offered_candidates_keep_original_evidence_and_unoffered_are_abandoned() -> Result<()> { + let Some(db) = Database::open().await? else { + return Ok(()); + }; + let (a, b, _node, config) = setup(&db).await?; + a.append(share(1), None).await?; + let snapshot = a.snapshot(100).await?; + let mut offered = Vec::new(); + for index in 0..3 { + let block = candidate(&snapshot, 2890 + index)?; + a.enqueue_candidate(block.candidate.clone()).await?; + let claim = a.claim_candidate(60).await?.unwrap(); + a.reserve_offer(&claim).await?; + if index > 0 { + a.record_offer(&claim, 1_700_000_000_123, OfferOutcome::Accepted, None) + .await?; + } + if index > 1 { + a.reconcile_candidate(&claim, "landing deferred").await?; + } + // Make this row unavailable so the next loop claims its new block. + sqlx::query("UPDATE qbit_block_candidate_outbox SET next_attempt_at=clock_timestamp()+interval '1 day' WHERE block_hash=$1") + .bind(&block.block_hash).execute(&a.pool).await?; + offered.push((block, claim)); + } + let parked = candidate(&snapshot, 2893)?; + a.enqueue_candidate(parked.candidate.clone()).await?; + let parked_claim = a.claim_candidate(60).await?.unwrap(); + a.reserve_offer(&parked_claim).await?; + a.reconcile_candidate(&parked_claim, "requires operator recovery") + .await?; + sqlx::query( + "UPDATE qbit_block_candidate_outbox SET next_attempt_at='infinity' WHERE block_hash=$1", + ) + .bind(&parked.block_hash) + .execute(&a.pool) + .await?; + let pending = candidate(&snapshot, 2894)?; + a.enqueue_candidate(pending.candidate.clone()).await?; + let pending_claim = a.claim_candidate(60).await?.unwrap(); + let evidence: Vec = sqlx::query_scalar("SELECT jsonb_build_object('state',state,'candidate',candidate,'block_bytes',encode(block_bytes,'hex'),'offer_reserved_at',offer_reserved_at,'offer_reply',offer_reply,'offer_outcome',offer_outcome,'parked',next_attempt_at='infinity'::timestamptz,'last_error',last_error) FROM qbit_block_candidate_outbox WHERE state<>'pending' ORDER BY block_hash") + .fetch_all(&a.pool).await?; + a.heartbeat(HeartbeatStatus::Stopped).await?; + b.heartbeat(HeartbeatStatus::Stopped).await?; + let event = a.transition_policy(&config, &changed_fee(&config)).await?; + assert_eq!(event["abandoned_candidates"], 1); + assert_eq!(event["retained_candidates"], 4); + let after: Vec = sqlx::query_scalar("SELECT jsonb_build_object('state',state,'candidate',candidate,'block_bytes',encode(block_bytes,'hex'),'offer_reserved_at',offer_reserved_at,'offer_reply',offer_reply,'offer_outcome',offer_outcome,'parked',next_attempt_at='infinity'::timestamptz,'last_error',last_error) FROM qbit_block_candidate_outbox WHERE state<>'abandoned' ORDER BY block_hash") + .fetch_all(&a.pool).await?; + assert_eq!(evidence, after); + let terminal: (String, String, bool) = sqlx::query_as("SELECT state,last_error,candidate IS NULL AND block_bytes IS NULL AND window_anchor_ms IS NULL FROM qbit_block_candidate_outbox WHERE block_hash=$1") + .bind(&pending.block_hash).fetch_one(&a.pool).await?; + assert_eq!( + terminal, + ("abandoned".into(), "epoch-superseded".into(), true) + ); + assert!(a.reserve_offer(&pending_claim).await.is_err()); + for (_, old_claim) in &offered { + assert!(a.renew_candidate_claim(old_claim, 60).await.is_err()); + } + let recovery = db.ledger("recovery").await?; + for _ in 0..3 { + let claim = recovery + .claim_candidate(60) + .await? + .context("retained candidate missing")?; + let (block, _) = offered + .iter() + .find(|(block, _)| block.block_hash == claim.candidate.block_hash) + .unwrap(); + assert!( + recovery.reserve_offer(&claim).await.is_err(), + "offered block could be offered twice" + ); + let claim = block.claim(claim); + let revision = recovery.payout_revision().await?; + recovery + .land_candidate_at_revision(&claim, &keys().1.public_key_hex(), revision) + .await?; + let bytes = qbit_prism::canonical_audit_bundle_bytes(&block.bundle)?; + let stored = recovery.audit_bundle(&block.block_hash).await?.unwrap(); + let stored: AuditBundle = serde_json::from_value(stored)?; + assert_eq!(qbit_prism::canonical_audit_bundle_bytes(&stored)?, bytes); + verify_audit_bundle_with_ledger_public_key(&stored, &config.ledger_public_key)?; + recovery + .finish_candidate_at_revision(&claim, true, None, revision) + .await?; + } + assert!(recovery.claim_candidate(60).await?.is_none()); + db.close(vec![a, b, recovery]).await +} + +#[tokio::test] +async fn only_fee_fields_can_change_and_retry_does_not_bump_again() -> Result<()> { + let Some(db) = Database::open().await? else { + return Ok(()); + }; + let (a, b, _node, config) = setup(&db).await?; + a.heartbeat(HeartbeatStatus::Stopped).await?; + b.heartbeat(HeartbeatStatus::Stopped).await?; + let before = state(&a).await?; + let mut variants = vec![config.clone()]; + let mut keys_changed = changed_fee(&config); + keys_changed.manifest_seed = "44".repeat(32); + variants.push(keys_changed); + let mut ledger_key = changed_fee(&config); + ledger_key.ledger_public_key = + ManifestSigningKey::from_seed_hex(&"45".repeat(32))?.public_key_hex(); + ledger_key.ledger_seed = "45".repeat(32); + variants.push(ledger_key); + let mut payout = changed_fee(&config); + payout.payout_policy.safety_multiplier += 1; + variants.push(payout); + let mut ctv = changed_fee(&config); + ctv.ctv_enabled = true; + variants.push(ctv); + let mut fallback = changed_fee(&config); + fallback.username_fallback = Some("other".into()); + variants.push(fallback); + for target in variants { + assert!(a.transition_policy(&config, &target).await.is_err()); + assert_eq!(state(&a).await?, before); + } + a.transition_policy(&config, &changed_fee(&config)).await?; + let once = state(&a).await?; + assert!(a + .transition_policy(&config, &changed_fee(&config)) + .await + .is_err()); + assert_eq!(state(&a).await?, once); + for statement in [ + "UPDATE qbit_prism_policy_transitions SET abandoned_candidates=42", + "DELETE FROM qbit_prism_policy_transitions", + "TRUNCATE qbit_prism_policy_transitions", + ] { + assert!(sqlx::query(statement).execute(&a.pool).await.is_err()); + } + db.close(vec![a, b]).await +} + +#[tokio::test] +async fn event_failure_rolls_back_fingerprint_revision_and_candidate_disposition() -> Result<()> { + let Some(db) = Database::open().await? else { + return Ok(()); + }; + let (a, b, _node, config) = setup(&db).await?; + a.append(share(1), None).await?; + a.enqueue_candidate(candidate(&a.snapshot(100).await?, 2895)?.candidate) + .await?; + a.heartbeat(HeartbeatStatus::Stopped).await?; + b.heartbeat(HeartbeatStatus::Stopped).await?; + sqlx::raw_sql("CREATE FUNCTION reject_transition() RETURNS trigger LANGUAGE plpgsql AS $$ BEGIN RAISE EXCEPTION 'injected event failure'; END $$; CREATE TRIGGER reject_transition BEFORE INSERT ON qbit_prism_policy_transitions FOR EACH ROW EXECUTE FUNCTION reject_transition();") + .execute(&a.pool).await?; + let before = state(&a).await?; + assert!(a + .transition_policy(&config, &changed_fee(&config)) + .await + .unwrap_err() + .to_string() + .contains("injected event failure")); + assert_eq!(state(&a).await?, before); + db.close(vec![a, b]).await +} + +#[tokio::test] +async fn historical_audits_and_ctv_artifacts_remain_byte_identical() -> Result<()> { + let Some(db) = Database::open().await? else { + return Ok(()); + }; + let (a, b, _node, config) = setup(&db).await?; + prepare_mature_cpfp_fanouts(&a, 2).await?; + let hash: String = sqlx::query_scalar("SELECT block_hash FROM qbit_pool_audit_bundles") + .fetch_one(&a.pool) + .await?; + let bundle: AuditBundle = serde_json::from_value(a.audit_bundle(&hash).await?.unwrap())?; + let bytes = qbit_prism::canonical_audit_bundle_bytes(&bundle)?; + let artifacts = |pool: PgPool| async move { + sqlx::query_scalar::<_, Value>("SELECT jsonb_build_object('sets',(SELECT jsonb_agg(to_jsonb(s) ORDER BY block_hash) FROM qbit_ctv_fanout_sets s),'artifacts',(SELECT jsonb_agg(to_jsonb(a) ORDER BY fanout_txid) FROM qbit_ctv_fanout_artifacts a))") + .fetch_one(&pool).await + }; + let before = artifacts(a.pool.clone()).await?; + assert_eq!(before["artifacts"].as_array().unwrap().len(), 2); + a.heartbeat(HeartbeatStatus::Stopped).await?; + b.heartbeat(HeartbeatStatus::Stopped).await?; + a.transition_policy(&config, &changed_fee(&config)).await?; + assert_eq!(artifacts(a.pool.clone()).await?, before); + let after: AuditBundle = serde_json::from_value(a.audit_bundle(&hash).await?.unwrap())?; + assert_eq!(qbit_prism::canonical_audit_bundle_bytes(&after)?, bytes); + verify_audit_bundle_with_ledger_public_key(&after, &config.ledger_public_key)?; + db.close(vec![a, b]).await +} + +#[tokio::test] +async fn policy_transition_cannot_clear_a_fatal_state() -> Result<()> { + let Some(db) = Database::open().await? else { + return Ok(()); + }; + let (a, b, _node, config) = setup(&db).await?; + a.heartbeat(HeartbeatStatus::Stopped).await?; + b.heartbeat(HeartbeatStatus::Stopped).await?; + sqlx::query("UPDATE qbit_prism_cluster SET fatal_error='unresolved reorg' WHERE singleton") + .execute(&a.pool) + .await?; + let before = state(&a).await?; + assert!(a + .transition_policy(&config, &changed_fee(&config)) + .await + .unwrap_err() + .to_string() + .contains("cluster halted")); + assert_eq!(state(&a).await?, before); + db.close(vec![a, b]).await +} + +fn cli(db: &Database, node: &fake::FakeNode, path: &std::path::Path) -> tokio::process::Command { + let mut command = tokio::process::Command::new(env!("CARGO_BIN_EXE_qbit-prism-server")); + command + .env_clear() + .kill_on_drop(true) + .args(["policy-transition", "--to"]) + .arg(path) + .env("PRISM_DATABASE_URL", &db.url) + .env("QBIT_RPC_URL", &node.url) + .env("QBIT_CHAIN", "testnet") + .env("PRISM_ALLOW_TEST_SIGNING_SEEDS", "1") + .env("PRISM_MANIFEST_SIGNING_SEED_HEX", "42".repeat(32)) + .env("PRISM_LEDGER_ATTESTATION_SIGNING_SEED_HEX", "43".repeat(32)) + .env( + "PRISM_LEDGER_WRITER_PUBLIC_KEY_HEX", + keys().1.public_key_hex(), + ) + .env("PRISM_RUNTIME_WORKERS", "2"); + command.env("PRISM_USERNAME_FALLBACK_ADDRESS", "policy-test-fallback"); + command +} + +#[tokio::test] +async fn transition_waits_for_both_accounting_locks() -> Result<()> { + let Some(db) = Database::open().await? else { + return Ok(()); + }; + let (a, b, _node, mut config) = setup(&db).await?; + a.heartbeat(HeartbeatStatus::Stopped).await?; + b.heartbeat(HeartbeatStatus::Stopped).await?; + for key in [0x505249534d000003i64, 0x505249534d000002i64] { + let mut blocked = a.pool.begin().await?; + sqlx::query("SELECT pg_advisory_xact_lock($1)") + .bind(key) + .execute(&mut *blocked) + .await?; + let mut next = changed_fee(&config); + next.payout_policy.pool_fee_policy.as_mut().unwrap().fee_bps = config + .payout_policy + .pool_fee_policy + .as_ref() + .map_or(200, |fee| fee.fee_bps + 100); + let before = state(&a).await?; + let task = tokio::spawn({ + let ledger = a.clone(); + let current = config.clone(); + let target = next.clone(); + async move { ledger.transition_policy(¤t, &target).await } + }); + tokio::time::timeout(Duration::from_secs(3), async { + loop { + let waiting: bool = sqlx::query_scalar("SELECT EXISTS(SELECT 1 FROM pg_locks WHERE locktype='advisory' AND NOT granted AND classid=$1::bigint::oid AND objid=$2::bigint::oid)") + .bind(key >> 32).bind(key & 0xffff_ffff).fetch_one(&a.pool).await?; + if waiting { return Ok::<_, anyhow::Error>(()); } + tokio::time::sleep(Duration::from_millis(10)).await; + } + }).await??; + assert!(!task.is_finished()); + assert_eq!(state(&a).await?, before); + blocked.commit().await?; + task.await??; + config = next; + } + db.close(vec![a, b]).await +} + +#[tokio::test] +async fn registration_racing_the_stopped_scan_is_not_missed() -> Result<()> { + let Some(db) = Database::open().await? else { + return Ok(()); + }; + let (a, b, _node, config) = setup(&db).await?; + a.heartbeat(HeartbeatStatus::Stopped).await?; + b.heartbeat(HeartbeatStatus::Stopped).await?; + let before = state(&a).await?; + let mut registering = a.pool.begin().await?; + sqlx::query("INSERT INTO qbit_prism_instances(instance_id,status) VALUES('racing-frontend','{\"state\":\"starting\",\"candidate_offer_lifecycle\":1}')") + .execute(&mut *registering).await?; + let task = tokio::spawn({ + let ledger = a.clone(); + let next = changed_fee(&config); + async move { ledger.transition_policy(&config, &next).await } + }); + tokio::time::timeout(Duration::from_secs(3), async { + loop { + let waiting: bool = sqlx::query_scalar("SELECT EXISTS(SELECT 1 FROM pg_locks WHERE relation='qbit_prism_instances'::regclass AND NOT granted)") + .fetch_one(&a.pool).await?; + if waiting { return Ok::<_, anyhow::Error>(()); } + tokio::time::sleep(Duration::from_millis(10)).await; + } + }).await??; + registering.commit().await?; + assert!(task + .await? + .unwrap_err() + .to_string() + .contains("racing-frontend")); + assert_eq!(state(&a).await?, before); + db.close(vec![a, b]).await +} + +#[tokio::test] +async fn cli_loads_env_overrides_and_resolves_fee_addresses_without_registering() -> Result<()> { + let Some(db) = Database::open().await? else { + return Ok(()); + }; + let (a, b, node, config) = setup(&db).await?; + a.heartbeat(HeartbeatStatus::Stopped).await?; + b.heartbeat(HeartbeatStatus::Stopped).await?; + let dir = tempfile::tempdir()?; + let path = dir.path().join("next.env"); + std::fs::write( + &path, + "export PRISM_POOL_FEE_ENABLED=1\nPRISM_POOL_FEE_BPS='200'\nPRISM_POOL_FEE_ADDRESS=fee\n", + )?; + let output = + tokio::time::timeout(Duration::from_secs(30), cli(&db, &node, &path).output()).await??; + assert!( + output.status.success(), + "{}", + String::from_utf8_lossy(&output.stderr) + ); + let event: Value = serde_json::from_slice(&output.stdout)?; + assert_eq!( + event["config_fingerprint"], + changed_fee(&config).fingerprint(&"00".repeat(32))? + ); + assert_eq!( + event["policy"]["payout_policy"]["pool_fee_policy"]["fee_bps"], + 200 + ); + assert_eq!( + sqlx::query_scalar::<_, i64>("SELECT count(*) FROM qbit_prism_instances") + .fetch_one(&a.pool) + .await?, + 2 + ); + assert!(!String::from_utf8_lossy(&output.stdout).contains(&config.manifest_seed)); + db.close(vec![a, b]).await +} + +#[tokio::test] +async fn cli_rejects_invalid_target_without_exposing_env_source() -> Result<()> { + let Some(db) = Database::open().await? else { + return Ok(()); + }; + let (a, b, node, _config) = setup(&db).await?; + let dir = tempfile::tempdir()?; + let path = dir.path().join("bad.env"); + let before = state(&a).await?; + for input in [ + "PRISM_POOL_FEE_BPS='secret-unclosed", + "PRISM_POOL_FEE_ENABLED=1\nPRISM_POOL_FEE_ADDRESS=fee\nPRISM_POOL_FEE_BPS=10001", + "PRISM_DATABASE_URL=postgresql://elsewhere/other", + "PRISM_POOL_FEE_ENABLED=maybe", + ] { + std::fs::write(&path, input)?; + let output = cli(&db, &node, &path).output().await?; + assert!(!output.status.success()); + assert!(!String::from_utf8_lossy(&output.stderr).contains("secret-unclosed")); + assert_eq!(state(&a).await?, before); + } + db.close(vec![a, b]).await +} diff --git a/crates/qbit-prism-server/tests/support/pool_fee.rs b/crates/qbit-prism-server/tests/support/pool_fee.rs new file mode 100644 index 000000000..41a239a4c --- /dev/null +++ b/crates/qbit-prism-server/tests/support/pool_fee.rs @@ -0,0 +1,40 @@ +//! #535: every server refuses to run, check or self-check its configuration +//! without a pool fee, in either settlement mode. Tests that launch one run +//! this 0-bps fee, which pays nothing and adds no output until dust below +//! the payout floor must be swept. + +pub const ZERO_BPS_POOL_FEE: [(&str, &str); 4] = [ + ("PRISM_POOL_FEE_ENABLED", "1"), + ("PRISM_POOL_FEE_BPS", "0"), + ("PRISM_POOL_FEE_RECIPIENT_ID", "pool-fee"), + ( + "PRISM_POOL_FEE_P2MR_PROGRAM_HEX", + "fefefefefefefefefefefefefefefefefefefefefefefefefefefefefefefefe", + ), +]; + +/// `ZERO_BPS_POOL_FEE` as the payout policy the server reads from it, for a +/// fixture that pins the cluster's configuration in process. +#[allow(dead_code)] +pub fn zero_bps_policy() -> qbit_prism::PoolFeePolicy { + qbit_prism::PoolFeePolicy { + fee_bps: 0, + recipient_id: ZERO_BPS_POOL_FEE[2].1.into(), + order_key: ZERO_BPS_POOL_FEE[2].1.into(), + p2mr_program_hex: ZERO_BPS_POOL_FEE[3].1.into(), + } +} + +/// The 0-bps fee, unless `names` configure a pool fee of their own: an +/// address beside the default program, or a disabled fee beside its rate, +/// would be refused as a configuration error rather than tested. +#[allow(dead_code)] +pub fn default_pool_fee<'a>( + mut names: impl Iterator, +) -> &'static [(&'static str, &'static str)] { + if names.any(|name| name.starts_with("PRISM_POOL_FEE_")) { + &[] + } else { + &ZERO_BPS_POOL_FEE + } +} diff --git a/crates/qbit-prism-server/tests/support/prepared_work_assertions.rs b/crates/qbit-prism-server/tests/support/prepared_work_assertions.rs new file mode 100644 index 000000000..5157183b3 --- /dev/null +++ b/crates/qbit-prism-server/tests/support/prepared_work_assertions.rs @@ -0,0 +1,114 @@ +//! Assertions for real refresh/resume measurements, not a measurement source. +//! Callers must isolate the refresh and pass observed values; missing values +//! deliberately fail qualification instead of becoming zero. + +use anyhow::{ensure, Context, Result}; +use qbit_prism_server::ledger::WindowRef; +use serde::Deserialize; +use serde_json::Value; + +pub const REGRESSION_SHARES: u64 = 400_000; +pub const HEADROOM_SHARES: u64 = 500_000; +pub const JSONB_LIMIT_BYTES: u64 = 1_000_000; +pub const WAL_LIMIT_BYTES: u64 = 5_000_000; + +/// Inspect the entire prepared payload, including nested objects and arrays. +/// Only root `window.shares` may name shares: it must be the existing WindowRef +/// metadata shape (null or ShareRange), never materialized share rows. This is +/// explicitly narrower than #273's literal no-`shares`-key acceptance wording. +pub fn assert_no_materialized_shares(payload: &Value) -> Result<()> { + inspect_prepared_payload(payload, true) +} + +fn inspect_prepared_payload(payload: &Value, at_root: bool) -> Result<()> { + match payload { + Value::Object(fields) => { + ensure!( + !fields.contains_key("shares"), + "stored payload contains shares key" + ); + for (key, value) in fields { + if at_root && key == "window" { + assert_window_metadata(value)?; + } else { + inspect_prepared_payload(value, false)?; + } + } + } + Value::Array(values) => { + for value in values { + inspect_prepared_payload(value, false)?; + } + } + _ => {} + } + Ok(()) +} + +fn assert_window_metadata(value: &Value) -> Result<()> { + // Reuse the actual serializer's integer and lowercase digest rules. Its + // deserializer ignores unknown fields and defaults missing Option fields; + // exact round-trip equality rejects both, including hidden share arrays. + let window = WindowRef::deserialize(value).context("invalid window metadata")?; + ensure!( + serde_json::to_value(window)? == *value, + "window metadata has missing, extra or noncanonical fields" + ); + if let Some(range) = window.shares { + // Match the private ShareRange::bounds / CompactPrepared::validate + // constraints without changing their persisted representation. + ensure!( + range.first_share_seq >= 1 + && range.last_share_seq >= range.first_share_seq + && i64::try_from(range.last_share_seq).is_ok() + && range.share_count >= 1 + && range.share_count <= range.last_share_seq - range.first_share_seq + 1, + "invalid window range metadata" + ); + } + Ok(()) +} + +/// `max_jsonb_bytes` is the maximum observed uncompressed refresh write, +/// not pg_column_size of a possibly compressed/TOASTed stored row. +/// `wal_bytes` is the server-wide insert-LSN delta bracketing just refresh. +/// This qualifies a non-cached refresh that writes logged prepared work. +/// Cached no-write reuse must be measured and checked separately. +pub fn assert_refresh_measurements( + expected_shares: u64, + published_shares: u64, + max_jsonb_bytes: Option, + wal_bytes: Option, +) -> Result<()> { + ensure!(expected_shares > 0, "qualification window must be nonempty"); + ensure!( + published_shares == expected_shares, + "published window count differs" + ); + let jsonb = max_jsonb_bytes.context("refresh JSONB measurement unavailable")?; + // A non-cached refresh must persist a prepared record. + ensure!(jsonb > 0, "no refresh JSONB write was observed"); + ensure!( + jsonb < JSONB_LIMIT_BYTES, + "refresh JSONB value is not under 1 MB: observed={jsonb} limit={JSONB_LIMIT_BYTES}" + ); + let wal = wal_bytes.context("refresh WAL measurement unavailable")?; + ensure!(wal > 0, "no WAL was observed for the logged refresh write"); + ensure!( + wal < WAL_LIMIT_BYTES, + "refresh WAL is not under 5 MB: observed={wal} limit={WAL_LIMIT_BYTES}" + ); + Ok(()) +} + +/// Compare canonical bytes produced by the real issuing and resuming jobs. +/// Exact equality preserves both their hashes and signed economics; a caller +/// must not substitute a digest of WindowRef or the legacy window encoding. +pub fn assert_canonical_equality(issued: &[u8], resumed: &[u8]) -> Result<()> { + ensure!(!issued.is_empty(), "issued canonical audit is absent"); + ensure!( + issued == resumed, + "resumed canonical audit differs from issued audit" + ); + Ok(()) +} diff --git a/crates/qbit-prism-server/tests/support/public_replica.rs b/crates/qbit-prism-server/tests/support/public_replica.rs new file mode 100644 index 000000000..a00709201 --- /dev/null +++ b/crates/qbit-prism-server/tests/support/public_replica.rs @@ -0,0 +1,362 @@ +use super::*; +use serde_json::Value; +use std::{ + fs::File, + io::Write, + process::{Child, Stdio}, +}; + +struct PublicProcess { + child: Child, + log: PathBuf, + url: String, +} +impl PublicProcess { + fn start(database: &str, port: u16, log: PathBuf) -> Result { + let output = File::create(&log)?; + let child = Command::new(env!("CARGO_BIN_EXE_qbit-prism-server")) + .arg("public-api") + // The public role must start independently of all signing keys, + // mining configuration, node credentials and inherited writer DSNs. + .env_clear() + .env("PRISM_DATABASE_URL", database) + .env("PRISM_RUNTIME_WORKERS", "2") + .env("PRISM_PUBLIC_API_BIND", "127.0.0.1") + .env("PRISM_PUBLIC_API_PORT", port.to_string()) + .env("PRISM_PUBLIC_STRATUM_URL", "stratum+tcp://127.0.0.1:3340") + .env("PRISM_PUBLIC_REPLICA_MODE", "require") + .env("PRISM_PUBLIC_REPLICA_MAX_LAG_SECONDS", "2") + .env("PRISM_PUBLIC_READINESS_PROBE_INTERVAL_SECONDS", "0.1") + .env("PRISM_POSTGRES_READ_CONCURRENCY", "2") + .env("PRISM_PUBLIC_READ_STATEMENT_TIMEOUT_SECONDS", "2") + .env("PRISM_PUBLIC_CACHE_ENABLED", "0") + .env("RUST_LOG", "info") + .stdout(Stdio::from(output.try_clone()?)) + .stderr(Stdio::from(output)) + .spawn()?; + Ok(Self { + child, + log, + url: format!("http://127.0.0.1:{port}"), + }) + } + + async fn health_until( + &mut self, + client: &reqwest::Client, + expected: u16, + predicate: impl Fn(&Value) -> bool, + ) -> Result { + let mut last = String::new(); + let result = timeout(Duration::from_secs(20), async { + loop { + if let Some(status) = self.child.try_wait()? { + anyhow::bail!("public process exited: {status}") + } + if let Ok(response) = client.get(format!("{}/healthz", self.url)).send().await { + let status = response.status().as_u16(); + let body: Value = response.json().await?; + if status == expected && predicate(&body) { + return Ok::<_, anyhow::Error>(body); + } + last = format!("HTTP {status}: {body}"); + } + sleep(Duration::from_millis(100)).await; + } + }) + .await; + result.with_context(|| { + format!( + "public readiness timeout; last={last}; log={}", + std::fs::read_to_string(&self.log).unwrap_or_default() + ) + })? + } +} +impl Drop for PublicProcess { + fn drop(&mut self) { + let _ = self.child.kill(); + let _ = self.child.wait(); + } +} + +struct PausedWalSender(i32); +impl PausedWalSender { + fn pause(pid: i32) -> Result { + let status = Command::new("kill") + .args(["-STOP", &pid.to_string()]) + .status()?; + ensure!(status.success(), "could not suspend disposable WAL sender"); + Ok(Self(pid)) + } +} +impl Drop for PausedWalSender { + fn drop(&mut self) { + let _ = Command::new("kill") + .args(["-CONT", &self.0.to_string()]) + .status(); + } +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn read_only_public_service_tracks_real_standby_stream_and_refuses_promotion() -> Result<()> { + let Some(bin) = gate::pg_bin_dir(gate::site!())? else { + return Ok(()); + }; + let dir = tempfile::tempdir()?; + let primary = Cluster { + bin: bin.clone().into(), + data: dir.path().join("primary"), + }; + let standby = Cluster { + bin: bin.into(), + data: dir.path().join("standby"), + }; + primary.command( + "initdb", + &[ + "-D", + primary.data.to_str().unwrap(), + "-A", + "trust", + "--no-locale", + "-E", + "UTF8", + ], + )?; + // Short real WAL keepalives make a two-second heartbeat budget meaningful + // even while the pool is idle. These settings are copied by basebackup. + writeln!(std::fs::OpenOptions::new().append(true).open(primary.data.join("postgresql.conf"))?,"\nwal_sender_timeout = '1s'\nwal_receiver_status_interval = '1s'\nwal_retrieve_retry_interval = '100ms'")?; + let primary_port = port()?; + let standby_port = port()?; + primary.start(primary_port, dir.path())?; + let username = String::from_utf8(Command::new("id").arg("-un").output()?.stdout)? + .trim() + .to_owned(); + let primary_url = format!("postgresql://{username}@127.0.0.1:{primary_port}/postgres"); + let writer = Ledger::connect(&primary_url, "only-writer".into(), 4, true).await?; + writer.append(proof(1), None).await?; + sqlx::raw_sql("CREATE ROLE public_reader LOGIN; GRANT CONNECT ON DATABASE postgres TO public_reader; GRANT USAGE ON SCHEMA public TO public_reader; GRANT SELECT ON ALL TABLES IN SCHEMA public TO public_reader; GRANT pg_monitor TO public_reader; ALTER ROLE public_reader SET default_transaction_read_only=on;").execute(&writer.pool).await?; + standby.command("pg_basebackup",&["-D",standby.data.to_str().unwrap(),"-d",&format!("host=127.0.0.1 port={primary_port} user={username} application_name=public_test_standby"),"-R","-X","stream","-c","fast"])?; + standby.start(standby_port, dir.path())?; + let standby_url = format!("postgresql://{username}@127.0.0.1:{standby_port}/postgres"); + let standby_admin = PgPool::connect(&standby_url).await?; + timeout(Duration::from_secs(15), async { + loop { + if sqlx::query_scalar::<_, bool>( + "SELECT EXISTS(SELECT 1 FROM pg_stat_wal_receiver WHERE status='streaming')", + ) + .fetch_one(&standby_admin) + .await? + { + return Ok::<_, anyhow::Error>(()); + } + sleep(Duration::from_millis(100)).await; + } + }) + .await??; + let client = reqwest::Client::builder() + .timeout(Duration::from_secs(2)) + .build()?; + let reader_url = format!("postgresql://public_reader@127.0.0.1:{standby_port}/postgres"); + let mut public = PublicProcess::start(&reader_url, port()?, dir.path().join("public.log"))?; + public + .health_until(&client, 200, |v| { + v["ok"] == true && v["replica"]["in_recovery"] == true + }) + .await?; + // Committed after basebackup: this row must reach the independent public + // process through actual streaming replication, never a primary fallback. + let hash = "ad".repeat(32); + sqlx::query("INSERT INTO qbit_pool_blocks(block_hash,block_height,parent_hash,coinbase_txid,payout_manifest_sha256,chain_state) VALUES($1,101,repeat('00',32),repeat('01',32),repeat('02',32),'confirmed')").bind(&hash).execute(&writer.pool).await?; + timeout(Duration::from_secs(10), async { + loop { + let response = client + .get(format!("{}/public/v1/blocks", public.url)) + .send() + .await?; + if response.status().is_success() { + let body: Value = response.json().await?; + if body["rows"] + .as_array() + .is_some_and(|rows| rows.iter().any(|row| row["hash"] == hash)) + { + return Ok::<_, anyhow::Error>(()); + } + } + sleep(Duration::from_millis(100)).await; + } + }) + .await??; + assert_eq!( + client + .get(format!("{}/status", public.url)) + .send() + .await? + .status() + .as_u16(), + 404, + "public role exposed the operator API" + ); + assert_eq!( + client + .post(format!("{}/public/v1/blocks", public.url)) + .send() + .await? + .status() + .as_u16(), + 405 + ); + let reader = PgPool::connect(&reader_url).await?; + assert!( + !sqlx::query_scalar::<_, bool>( + "SELECT has_table_privilege(current_user,'qbit_share_ledger','INSERT')" + ) + .fetch_one(&reader) + .await? + ); + assert!(sqlx::query( + "INSERT INTO qbit_prism_instances(instance_id) VALUES('public-must-not-write')" + ) + .execute(&reader) + .await + .is_err()); + assert_eq!( + sqlx::query_scalar::<_, i64>("SELECT count(*) FROM qbit_prism_instances") + .fetch_one(&writer.pool) + .await?, + 1 + ); + + // Even valid readable schema/data on a primary cannot make require mode + // ready. The public role needs no writer credentials to prove this. + let mut on_primary = PublicProcess::start( + &format!("postgresql://public_reader@127.0.0.1:{primary_port}/postgres"), + port()?, + dir.path().join("primary-public.log"), + )?; + on_primary + .health_until(&client, 503, |v| v["replica"]["in_recovery"] == false) + .await?; + assert_eq!( + client + .get(format!("{}/public/v1/blocks", on_primary.url)) + .send() + .await? + .status() + .as_u16(), + 503 + ); + drop(on_primary); + + // A connected receiver alone is insufficient: freeze its disposable + // sender without closing the socket and let the real heartbeat age out. + let sender: i32 = sqlx::query_scalar( + "SELECT pid FROM pg_stat_replication WHERE application_name='public_test_standby'", + ) + .fetch_one(&writer.pool) + .await?; + let paused = PausedWalSender::pause(sender)?; + let stale = public + .health_until(&client, 503, |v| { + v["replica"]["receiver_heartbeat_age_seconds"] + .as_f64() + .is_some_and(|age| age > 2.0) + }) + .await?; + assert_eq!(stale["replica"]["in_recovery"], true); + assert_eq!( + sqlx::query_scalar::<_, i64>("SELECT count(*) FROM pg_stat_wal_receiver") + .fetch_one(&standby_admin) + .await?, + 1, + "test must exercise a connected but stale stream" + ); + assert_eq!( + client + .get(format!("{}/public/v1/blocks", public.url)) + .send() + .await? + .status() + .as_u16(), + 503 + ); + drop(paused); + public + .health_until(&client, 200, |v| v["ok"] == true) + .await?; + + // Stop the WAL source while leaving standby SQL queries available. The + // service must fence stale reads, then recover after streaming reconnects. + writer.pool.close().await; + primary.control(&["-m", "immediate", "-w", "stop"])?; + let unready = public + .health_until(&client, 503, |v| { + v["replica"]["in_recovery"] == true && v["ok"] == false + }) + .await?; + assert!( + unready["replica"]["receiver_heartbeat_age_seconds"].is_null() + || unready["replica"]["receiver_heartbeat_age_seconds"] + .as_f64() + .unwrap_or_default() + > 2.0 + ); + assert_eq!( + client + .get(format!("{}/public/v1/blocks", public.url)) + .send() + .await? + .status() + .as_u16(), + 503 + ); + assert_eq!( + sqlx::query_scalar::<_, i64>("SELECT count(*) FROM qbit_pool_blocks") + .fetch_one(&standby_admin) + .await?, + 1, + "standby itself must still be queryable during stream failure" + ); + primary.start(primary_port, dir.path())?; + public + .health_until(&client, 200, |v| { + v["ok"] == true && v["replica"]["in_recovery"] == true + }) + .await?; + assert_eq!( + client + .get(format!("{}/public/v1/blocks", public.url)) + .send() + .await? + .status() + .as_u16(), + 200 + ); + + primary.control(&["-m", "immediate", "-w", "stop"])?; + standby.control(&["-w", "promote"])?; + public + .health_until(&client, 503, |v| v["replica"]["in_recovery"] == false) + .await?; + assert_eq!( + client + .get(format!("{}/public/v1/blocks", public.url)) + .send() + .await? + .status() + .as_u16(), + 503, + "promotion must fence existing pooled public sessions" + ); + assert_eq!( + sqlx::query_scalar::<_, i64>("SELECT count(*) FROM qbit_prism_instances") + .fetch_one(&standby_admin) + .await?, + 1 + ); + reader.close().await; + standby_admin.close().await; + drop(public); + Ok(()) +} diff --git a/crates/qbit-prism-server/tests/support/recovery.rs b/crates/qbit-prism-server/tests/support/recovery.rs new file mode 100644 index 000000000..fdfd2d7b7 --- /dev/null +++ b/crates/qbit-prism-server/tests/support/recovery.rs @@ -0,0 +1,595 @@ +use anyhow::{ensure, Context, Result}; +use qbit_pool_builder::ManifestSigningKey; +use qbit_prism::{AcceptedShare, FoundBlock, PayoutPolicy}; +use qbit_prism_server::api::{self, ApiConfig, ApiState}; +use serde_json::Value; +use sqlx::{Connection, PgConnection, PgPool}; +use std::{io::Write, path::Path, process::Stdio, sync::Arc}; +use tokio::process::Command; +use tower::ServiceExt; + +pub struct Database { + pub admin: PgPool, + pub pool: PgPool, + pub schema: String, + pub url: String, +} + +impl Database { + pub async fn open(raw: &str) -> Result { + let admin = PgPool::connect(raw).await?; + let schema = format!("prism_recovery_{}", uuid::Uuid::new_v4().simple()); + sqlx::query(&format!("CREATE SCHEMA {schema}")) + .execute(&admin) + .await?; + let mut url = url::Url::parse(raw)?; + url.query_pairs_mut() + .append_pair("options", &format!("-csearch_path={schema}")); + let pool = PgPool::connect(url.as_str()).await?; + Ok(Self { + admin, + pool, + schema, + url: url.into(), + }) + } + + pub async fn close(self) -> Result<()> { + self.pool.close().await; + sqlx::query(&format!("DROP SCHEMA {} CASCADE", self.schema)) + .execute(&self.admin) + .await?; + self.admin.close().await; + Ok(()) + } +} + +pub fn ledger_key() -> ManifestSigningKey { + ManifestSigningKey::from_seed_hex(&"43".repeat(32)).unwrap() +} + +pub fn share(seq: u64) -> AcceptedShare { + AcceptedShare { + share_seq: seq, + share_id: format!("recovery:{seq:064x}"), + miner_id: "miner".into(), + order_key: "miner".into(), + p2mr_program_hex: "11".repeat(32), + share_difficulty: 1, + network_difficulty: 100, + template_height: 100, + job_id: "legacy-job".into(), + job_issued_at_ms: 1_800_000_000_000, + accepted_at_ms: 1_800_000_001_000 + seq as i64, + ntime: 1_800_000_000, + credit_policy: None, + } +} + +pub struct Artifact { + pub block_hash: String, + pub digest: String, + pub canonical: Vec, +} + +/// Genuine frozen 2.x DDL, with ordered shares and all three historical body +/// layouts: canonical-sidecar-only, external body, and inline body. The latter +/// two deliberately have no canonical sidecar for the importer to rely on. +pub async fn seed_legacy(pool: &PgPool, root: &Path) -> Result> { + sqlx::raw_sql(include_str!("../fixtures/schema_2x/001_share_ledger.sql")) + .execute(pool) + .await?; + sqlx::raw_sql(include_str!( + "../fixtures/schema_2x/002_candidate_bodies.sql" + )) + .execute(pool) + .await?; + let shares = vec![share(1), share(5), share(8)]; + for share in &shares { + sqlx::query("INSERT INTO qbit_share_ledger(share_seq,share_id,miner_id,payout_order_key,p2mr_program,share_difficulty,network_difficulty,template_height,job_id,job_issued_at,accepted_at,ntime,writer_id,writer_epoch) VALUES($1,$2,'miner','miner',decode(repeat('11',32),'hex'),1,100,100,'legacy-job',to_timestamp($3::double precision/1000),to_timestamp($4::double precision/1000),1800000000,'python',7)") + .bind(share.share_seq as i64).bind(&share.share_id) + .bind(share.job_issued_at_ms).bind(share.accepted_at_ms).execute(pool).await?; + } + sqlx::query("SELECT setval('qbit_share_ledger_share_seq_seq',40)") + .execute(pool) + .await?; + let coinbase_key = ManifestSigningKey::from_seed_hex(&"42".repeat(32))?; + let mut artifacts = Vec::new(); + for index in 0..3 { + let bundle = qbit_prism::build_audit_bundle( + shares.clone(), + FoundBlock { + block_height: 101 + index, + coinbase_value_sats: 500_000_000, + network_difficulty: 100, + anchor_job_issued_at_ms: 1_800_000_002_000, + }, + vec![], + PayoutPolicy::day_one_default(), + &coinbase_key, + &ledger_key(), + )?; + let report = qbit_prism::verify_audit_bundle_with_ledger_public_key( + &bundle, + &ledger_key().public_key_hex(), + )?; + let canonical = qbit_prism::canonical_audit_bundle_bytes(&bundle)?; + let hash = format!("{:02x}", 80 + index).repeat(32); + let body_path = root.join(format!("legacy-audit-{hash}.json")); + let inline = if index == 2 { + Some(serde_json::to_value(&bundle)?) + } else { + None + }; + if index == 0 { + let path = root.join(format!( + "prism-audit-bundle-canonical-{hash}-{}.json.gz", + report.audit_bundle_sha256_hex + )); + let mut encoder = flate2::GzBuilder::new() + .mtime(0) + .write(std::fs::File::create(path)?, flate2::Compression::best()); + encoder.write_all(&canonical)?; + encoder.finish()?; + ensure!(!body_path.exists(), "sidecar-only row must have no body"); + } else if index == 1 { + std::fs::write(&body_path, serde_json::to_vec(&bundle)?)?; + } + sqlx::query("INSERT INTO qbit_pool_blocks(block_hash,block_height,parent_hash,coinbase_txid,payout_manifest_sha256,chain_state) VALUES($1,$2,repeat('00',32),$3,$4,'confirmed')") + .bind(&hash).bind((101 + index) as i64).bind(&report.coinbase_txid) + .bind(&report.coinbase_manifest_sha256_hex).execute(pool).await?; + sqlx::query("INSERT INTO qbit_pool_audit_bundles(block_hash,audit_bundle,audit_bundle_sha256,coinbase_tx_hex,body_uri) VALUES($1,$2,$3,$4,$5)") + .bind(&hash).bind(inline).bind(&report.audit_bundle_sha256_hex).bind(&report.coinbase_tx_hex) + .bind((index != 2).then(|| body_path.to_string_lossy().into_owned())).execute(pool).await?; + // Each miner its own program, so each row starts its chain: 2.x keeps + // one chain per program, whatever label a block pays it under (#708). + sqlx::query("INSERT INTO qbit_payout_carry_forward(block_height,block_hash,miner_id,payout_order_key,p2mr_program,gross_amount_sats,prior_balance_sats,candidate_balance_sats,onchain_amount_sats,carry_forward_balance_sats,action) VALUES($1,$2,$3,$3,decode(repeat($4,32),'hex'),1000,0,1000,0,1000,'accrued')") + .bind((101 + index) as i64).bind(&hash).bind(format!("miner-{index}")) + .bind(format!("{:02x}", 0x11 + index)) + .execute(pool).await?; + artifacts.push(Artifact { + block_hash: hash, + digest: report.audit_bundle_sha256_hex, + canonical, + }); + } + Ok(artifacts) +} + +/// Stable accounting facts only: native migration metadata and the summary's +/// repair timestamp are intentionally outside the recovery comparison. +pub async fn accounting_state(pool: &PgPool) -> Result { + Ok(sqlx::query_scalar( + "SELECT jsonb_build_object(\ + 'shares',(SELECT jsonb_agg(to_jsonb(s) ORDER BY share_seq) FROM qbit_share_ledger s),\ + 'audits',(SELECT jsonb_agg(jsonb_build_array(block_hash,audit_bundle_sha256) ORDER BY block_hash) FROM qbit_pool_audit_bundles),\ + 'ordinals',(SELECT jsonb_agg(jsonb_build_array(block_hash,audit_publication_sequence) ORDER BY audit_publication_sequence) FROM qbit_pool_blocks),\ + 'carry',(SELECT jsonb_agg(to_jsonb(c) ORDER BY carry_forward_seq) FROM qbit_payout_carry_forward c),\ + 'current',(SELECT jsonb_agg(to_jsonb(c)-'updated_at' ORDER BY miner_id,payout_order_key,p2mr_program) FROM qbit_payout_carry_forward_current c),\ + 'drift',(SELECT count(*) FROM qbit_carry_forward_current_drift()))", + ) + .fetch_one(pool) + .await?) +} + +pub async fn import_cli(db: &Database, root: &Path) -> Result { + let output = Command::new(env!("CARGO_BIN_EXE_qbit-prism-server")) + .args(["import-audits", "--root"]) + .arg(root) + .env_clear() + .env("PATH", std::env::var_os("PATH").unwrap_or_default()) + .env("PRISM_DATABASE_URL", &db.url) + .env("PRISM_DATABASE_MAX_CONNECTIONS", "4") + .env("PRISM_RUNTIME_WORKERS", "2") + .env( + "PRISM_LEDGER_WRITER_PUBLIC_KEY_HEX", + ledger_key().public_key_hex(), + ) + .output() + .await?; + ensure!( + output.status.success(), + "import-audits failed: {}", + String::from_utf8_lossy(&output.stderr) + ); + Ok(String::from_utf8(output.stdout)?) +} + +pub async fn assert_artifacts(pool: &PgPool, artifacts: &[Artifact]) -> Result<()> { + let app = api::router(ApiState::new( + pool.clone(), + ApiConfig { + cache_enabled: false, + ..Default::default() + }, + Arc::new(qbit_prism_server::metrics::Metrics::default()), + )); + for artifact in artifacts { + let uri = format!("/public/v1/artifacts/{}", artifact.digest); + let response = app + .clone() + .oneshot( + axum::http::Request::builder() + .uri(&uri) + .body(axum::body::Body::empty())?, + ) + .await?; + ensure!(response.status() == axum::http::StatusCode::OK); + ensure!(response + .headers() + .get("x-prism-artifact-canonical-state") + .is_none()); + let etag = format!("\"{}\"", artifact.digest); + ensure!(response.headers().get("etag").context("missing ETag")? == etag.as_str()); + ensure!( + axum::body::to_bytes(response.into_body(), usize::MAX) + .await? + .as_ref() + == artifact.canonical + ); + let stored: Vec = sqlx::query_scalar( + "SELECT canonical_audit_bytes FROM qbit_pool_audit_bundles WHERE block_hash=$1", + ) + .bind(&artifact.block_hash) + .fetch_one(pool) + .await?; + ensure!(stored == artifact.canonical); + let conditional = app + .clone() + .oneshot( + axum::http::Request::builder() + .uri(&uri) + .header("if-none-match", &etag) + .body(axum::body::Body::empty())?, + ) + .await?; + ensure!(conditional.status() == axum::http::StatusCode::NOT_MODIFIED); + ensure!(axum::body::to_bytes(conditional.into_body(), usize::MAX) + .await? + .is_empty()); + } + Ok(()) +} + +/// Exercise PostgreSQL's real archive writer/reader. INSERT output permits +/// executing the restored SQL through SQLx; only a random test schema name is +/// remapped, so the original database never receives restore statements. +pub async fn backup(db: &Database, pg_bin: &Path) -> Result> { + let output = postgres_command(db, pg_bin, "pg_dump")? + .args([ + "--format=custom", + "--inserts", + "--no-owner", + "--no-privileges", + "--schema", + ]) + .arg(&db.schema) + .output() + .await?; + ensure!( + output.status.success(), + "pg_dump failed: {}", + String::from_utf8_lossy(&output.stderr) + ); + ensure!( + output.stdout.starts_with(b"PGDMP"), + "not a PostgreSQL archive" + ); + Ok(output.stdout) +} + +fn postgres_command(db: &Database, pg_bin: &Path, program: &str) -> Result { + let mut command = Command::new(pg_bin.join(program)); + connect(&mut command, db)?; + Ok(command) +} + +// libpq expands a connection URI supplied as dbname, but not one inherited +// through PGDATABASE. Keep passwords out of the process argument list. +fn connect(command: &mut Command, db: &Database) -> Result<()> { + let mut url = url::Url::parse(&db.url)?; + let mut password = url + .password() + .map(|value| { + percent_encoding::percent_decode_str(value) + .decode_utf8() + .map(String::from) + }) + .transpose()?; + url.set_password(None) + .map_err(|_| anyhow::anyhow!("invalid PostgreSQL connection URL"))?; + let options: Vec<(String, String)> = url + .query_pairs() + .filter_map(|(key, value)| { + if key == "password" { + password = Some(value.into_owned()); + None + } else { + Some((key.into_owned(), value.into_owned())) + } + }) + .collect(); + url.query_pairs_mut().clear().extend_pairs(options); + command.arg("--dbname").arg(url.as_str()); + if let Some(password) = password { + command.env("PGPASSWORD", password); + } + Ok(()) +} + +pub async fn restore( + archive: &[u8], + source: &Database, + target: &Database, + pg_bin: &Path, +) -> Result<()> { + let mut file = tempfile::tempfile()?; + file.write_all(archive)?; + use std::io::{Seek, SeekFrom}; + file.seek(SeekFrom::Start(0))?; + let output = Command::new(pg_bin.join("pg_restore")) + .args([ + "--format=custom", + "--no-owner", + "--no-privileges", + "--file=-", + ]) + .stdin(Stdio::from(file)) + .output() + .await?; + ensure!( + output.status.success(), + "pg_restore failed: {}", + String::from_utf8_lossy(&output.stderr) + ); + // Current PostgreSQL patch releases add psql-only safety metacommands. + // No psql input is evaluated here; retain every actual SQL statement. + let sql = String::from_utf8(output.stdout)? + .lines() + .filter(|line| !line.starts_with("\\restrict ") && !line.starts_with("\\unrestrict ")) + .collect::>() + .join("\n") + .replace(&format!("CREATE SCHEMA {};", source.schema), "") + .replace(&source.schema, &target.schema); + // The SQL opens with session SET statements (search_path, lock_timeout, + // statement_timeout, ...) that survive COMMIT, so it runs on a session of + // its own that is closed afterwards, never on one that would carry those + // settings back into `target.pool`. Only the URL, which pins search_path + // to the fixture schema, is shared with the pool. + let mut connection = PgConnection::connect(&target.url).await?; + let restored = async { + let schema: String = sqlx::query_scalar("SELECT current_schema()") + .fetch_one(&mut connection) + .await?; + ensure!( + schema == target.schema, + "restore session resolves {schema:?}, not the target schema {:?}", + target.schema + ); + let mut transaction = connection.begin().await?; + sqlx::raw_sql(&sql).execute(&mut *transaction).await?; + transaction.commit().await?; + anyhow::Ok(()) + } + .await; + // The session is closed on both paths; `restore_outcome` pins which + // error is reported when the close fails as well. + let closed = connection.close().await.map_err(anyhow::Error::from); + restore_outcome(restored, closed) +} + +/// Combines the restore result with the result of closing its session. +/// +/// The restore error wins: a failed close must not hide why the restore itself +/// failed. A close failure is reported on its own only when the restore +/// succeeded, and is appended to the restore error's message when both fail +/// so it is not silently lost. +fn restore_outcome(restored: Result<()>, closed: Result<()>) -> Result<()> { + match (restored, closed) { + (Ok(()), closed) => closed, + (Err(restore), Ok(())) => Err(restore), + (Err(restore), Err(close)) => { + let message = + format!("{restore:#}; closing the restore session also failed: {close:#}"); + Err(restore.context(message)) + } + } +} + +#[test] +fn restore_outcome_is_ok_when_restore_and_close_succeed() { + assert!(restore_outcome(Ok(()), Ok(())).is_ok()); +} + +#[test] +fn restore_outcome_reports_the_restore_error_when_close_succeeds() { + let error = restore_outcome(Err(anyhow::anyhow!("restore boom")), Ok(())) + .expect_err("a failed restore must fail the outcome"); + assert_eq!(error.to_string(), "restore boom"); +} + +#[test] +fn restore_outcome_reports_the_close_error_when_restore_succeeds() { + let error = restore_outcome(Ok(()), Err(anyhow::anyhow!("close boom"))) + .expect_err("a failed close must fail the outcome"); + assert_eq!(error.to_string(), "close boom"); +} + +#[test] +fn restore_outcome_prefers_the_restore_error_when_both_fail() { + let error = restore_outcome( + Err(anyhow::anyhow!("restore boom")), + Err(anyhow::anyhow!("close boom")), + ) + .expect_err("two failures must fail the outcome"); + let message = error.to_string(); + assert!( + message.starts_with("restore boom"), + "the restore error must lead the message: {message:?}" + ); + assert!( + message.contains("close boom"), + "the close failure must not be lost from the message: {message:?}" + ); + assert_eq!( + error.root_cause().to_string(), + "restore boom", + "the root cause must be the restore error, not the close error" + ); +} + +pub async fn evidence(db: &Database, pg_bin: &Path) -> Result { + evidence_with_bytea(db, pg_bin, "hex").await +} + +pub async fn evidence_with_bytea(db: &Database, pg_bin: &Path, format: &str) -> Result { + evidence_with_options(db, pg_bin, format, None).await +} + +/// The evidence with the caller's FETCH_COUNT instead of the script's default +/// (#705). One row per fetch makes every result cross batch boundaries. +pub async fn evidence_with_fetch_count(db: &Database, pg_bin: &Path, rows: u32) -> Result { + evidence_with_options(db, pg_bin, "hex", Some(rows)).await +} + +async fn evidence_with_options( + db: &Database, + pg_bin: &Path, + format: &str, + fetch_count: Option, +) -> Result { + let records = records_with_options(db, pg_bin, SCRIPT, format, fetch_count).await?; + summarize(records.path()).await +} + +/// The checked-in export script. +pub const SCRIPT: &str = include_str!("../../../../scripts/prism-recovery-evidence.sql"); + +/// Where the parallel exporter and the summarizer read the export script. +const SCRIPT_PATH: &str = concat!( + env!("CARGO_MANIFEST_DIR"), + "/../../scripts/prism-recovery-evidence.sql" +); + +/// Fails unless the export script on disk is still [`SCRIPT`]. The serial +/// export runs the copy built into this binary, while the parallel exporter +/// and the summarizer read the scripts from disk as they run. A script edited +/// after the build would make their outputs differ by the edit, which no +/// export could cause: a deferred export once differed from its serial +/// export at the share-hash cursor's line, because an edit moved that line +/// while a suite built before it ran. +fn ensure_script_as_built() -> Result<()> { + let on_disk = + std::fs::read_to_string(SCRIPT_PATH).with_context(|| format!("reading {SCRIPT_PATH}"))?; + ensure!( + on_disk == SCRIPT, + "scripts/prism-recovery-evidence.sql changed after this test binary was built: the serial export runs the copy built in, the parallel exporter and the summarizer the file. Rebuild and rerun" + ); + Ok(()) +} + +/// The rows the serial export of `script` prints, as the runbooks save them. +pub async fn records( + db: &Database, + pg_bin: &Path, + script: &str, +) -> Result { + records_with_options(db, pg_bin, script, "hex", None).await +} + +async fn records_with_options( + db: &Database, + pg_bin: &Path, + script: &str, + format: &str, + fetch_count: Option, +) -> Result { + use std::io::{Seek, SeekFrom}; + ensure!(matches!(format, "hex" | "escape")); + let mut input = tempfile::tempfile()?; + writeln!(input, "SET search_path TO {};", db.schema)?; + writeln!(input, "SET bytea_output TO '{format}';")?; + input.write_all(script.as_bytes())?; + input.seek(SeekFrom::Start(0))?; + // The export goes straight to a file: at production size it is tens of + // gigabytes, and the script's FETCH_COUNT keeps psql from buffering it + // too (#705). tokio's `output()` would pipe stdout whatever it was set + // to, so spawn. + let records = tempfile::NamedTempFile::new()?; + let mut command = postgres_command(db, pg_bin, "psql")?; + command.args(["-XqAt", "-v", "ON_ERROR_STOP=1"]); + if let Some(rows) = fetch_count { + command.args(["-v", &format!("FETCH_COUNT={rows}")]); + } + let output = command + .stdin(Stdio::from(input)) + .stdout(Stdio::from(records.reopen()?)) + .stderr(Stdio::piped()) + .spawn()? + .wait_with_output() + .await?; + ensure!( + output.status.success(), + "recovery evidence SQL failed: {}", + String::from_utf8_lossy(&output.stderr) + ); + Ok(records) +} + +/// The summary of exported rows, as `scripts/prism-recovery-evidence.py` +/// prints it. +pub async fn summarize(records: &Path) -> Result { + let output = summarize_with(records, &[]).await?; + ensure!( + output.status.success(), + "recovery evidence summary failed: {}", + String::from_utf8_lossy(&output.stderr) + ); + Ok(serde_json::from_slice(&output.stdout)?) +} + +/// `scripts/prism-recovery-evidence.py` with `args` over `records`: what it +/// printed, and how it exited. +pub async fn summarize_with(records: &Path, args: &[&str]) -> Result { + ensure_script_as_built()?; + Ok(Command::new("python3") + .arg(concat!( + env!("CARGO_MANIFEST_DIR"), + "/../../scripts/prism-recovery-evidence.py" + )) + .args(args) + .arg(records) + .output() + .await?) +} + +/// Run the parallel export (#712) with `args`, its parts under `work_dir`: +/// what it printed, and how it exited. +pub async fn parallel_export( + db: &Database, + pg_bin: &Path, + work_dir: &Path, + args: &[&str], +) -> Result<(tempfile::NamedTempFile, std::process::Output)> { + ensure_script_as_built()?; + let records = tempfile::NamedTempFile::new()?; + let mut command = Command::new("python3"); + command + .arg(concat!( + env!("CARGO_MANIFEST_DIR"), + "/../../scripts/prism-recovery-evidence-parallel.py" + )) + .arg("--psql") + .arg(pg_bin.join("psql")) + .arg("--work-dir") + .arg(work_dir) + .args(args); + connect(&mut command, db)?; + let output = command + .stdout(Stdio::from(records.reopen()?)) + .stderr(Stdio::piped()) + .spawn()? + .wait_with_output() + .await?; + Ok((records, output)) +} diff --git a/crates/qbit-prism-server/tests/support/rehearsal_node.rs b/crates/qbit-prism-server/tests/support/rehearsal_node.rs new file mode 100644 index 000000000..c57a69972 --- /dev/null +++ b/crates/qbit-prism-server/tests/support/rehearsal_node.rs @@ -0,0 +1,213 @@ +//! The cutover rehearsal's qbit node: the chain a restored 2.x.x ledger was +//! mined on, served from the ledger itself (#575). +//! +//! A mainnet snapshot's blocks are not on regtest, and a started frontend +//! checks its pool blocks against the node: `getblockhash` at a confirmed +//! block's height must name that block, and the tip decides which blocks +//! the reconciler matures. This node answers from the rows of +//! `qbit_pool_blocks`: each confirmed block at its height, a synthetic hash +//! at every other height, and a tip that leaves every immature block +//! immature. Everything else follows `crates/qbit-prism-load/src/node.rs`, +//! reduced to what one frontend's startup, first job and self-check call. +use anyhow::{Context, Result}; +use axum::{extract::State, routing::post, Json, Router}; +use serde_json::{json, Value}; +use sha2::{Digest, Sha256}; +use sqlx::{PgPool, Row}; +use std::{ + collections::HashMap, + sync::{Arc, Mutex}, +}; +use tokio::task::JoinHandle; + +/// Every address with this prefix is a valid P2MR address here, paying to +/// `5220 || sha256(address)`, the program the seeds record. +pub const ADDRESS_PREFIX: &str = "qb1"; +/// `207fffff`: the stock template bits the load harness serves. +const TEMPLATE_BITS: &str = "207fffff"; + +struct Chain { + tag: String, + tip: u64, + /// Heights whose block is one of the ledger's, and the reverse map. + pool: HashMap, + heights: HashMap, + parents: HashMap, +} + +impl Chain { + fn hash_at(&self, height: u64) -> Option { + if height > self.tip { + return None; + } + if height == 0 { + return Some("00".repeat(32)); + } + Some(self.pool.get(&height).cloned().unwrap_or_else(|| { + hex::encode(Sha256::digest(format!( + "prism-rehearsal-chain:{}:{height}", + self.tag + ))) + })) + } + + fn parent_of(&self, hash: &str) -> Option { + if let Some(parent) = self.parents.get(hash) { + return Some(parent.clone()); + } + let height = *self.heights.get(hash)?; + self.hash_at(height.checked_sub(1)?) + } +} + +/// A running rehearsal node. +pub struct RehearsalNode { + pub url: String, + task: JoinHandle<()>, +} + +impl Drop for RehearsalNode { + fn drop(&mut self) { + self.task.abort(); + } +} + +impl RehearsalNode { + /// Serves the chain `pool`'s ledger records. `tag` names the synthetic + /// hashes of the heights without a pool block, which the seeds use for + /// the same heights, so a seeded chain is served exactly. + pub async fn from_ledger(pool: &PgPool, tag: &str) -> Result { + let rows = sqlx::query("SELECT block_hash,block_height,parent_hash FROM qbit_pool_blocks WHERE chain_state='confirmed' ORDER BY block_height") + .fetch_all(pool) + .await?; + let mut chain = Chain { + tag: tag.to_owned(), + tip: 0, + pool: HashMap::new(), + heights: HashMap::new(), + parents: HashMap::new(), + }; + let mut highest = 0u64; + for row in rows { + let hash: String = row.try_get("block_hash")?; + let height = u64::try_from(row.try_get::("block_height")?)?; + chain + .parents + .insert(hash.clone(), row.try_get("parent_hash")?); + chain.heights.insert(hash.clone(), height); + chain.pool.insert(height, hash); + highest = highest.max(height); + } + // The source's tip was at or above its highest confirmed block, and + // every block it left immature was less than the maturity depth + // under that tip, so it is under this one too: serving the highest + // confirmed block as the tip matures nothing 2.x.x had not. + chain.tip = highest.max(1); + Self::serve(chain).await + } + + async fn serve(chain: Chain) -> Result { + let chain = Arc::new(Mutex::new(chain)); + let listener = tokio::net::TcpListener::bind("127.0.0.1:0") + .await + .context("bind rehearsal node")?; + let url = format!("http://{}/", listener.local_addr()?); + let app = Router::new().route("/", post(answer)).with_state(chain); + let task = tokio::spawn(async move { + let _ = axum::serve(listener, app).await; + }); + Ok(Self { url, task }) + } +} + +fn payout_script_hex(address: &str) -> String { + format!("5220{}", hex::encode(Sha256::digest(address.as_bytes()))) +} + +fn double_sha256(bytes: &[u8]) -> [u8; 32] { + Sha256::digest(Sha256::digest(bytes)).into() +} + +async fn answer(State(chain): State>>, Json(request): Json) -> Json { + let method = request["method"].as_str().unwrap_or_default(); + if method == "waitfornewblock" { + // Nothing but the frontend itself mines here: wait out a bounded + // share of the long poll, then report the tip. + let wait = request["params"][0].as_u64().unwrap_or(1_000).min(1_000); + tokio::time::sleep(std::time::Duration::from_millis(wait)).await; + } + let params = &request["params"]; + let now = chrono::Utc::now().timestamp(); + let error = |code: i64, message: &str| { + Json( + json!({"id": request["id"], "result": Value::Null, "error": {"code": code, "message": message}}), + ) + }; + let mut chain = chain.lock().expect("chain lock"); + let tip_hash = chain.hash_at(chain.tip).unwrap_or_default(); + let result = match method { + "getblockchaininfo" => json!({ + "chain": "test", + "initialblockdownload": false, + "blocks": chain.tip, + "headers": chain.tip, + "bestblockhash": tip_hash, + "chainwork": format!("{:x}", u128::from(chain.tip) + 1), + }), + "getnetworkinfo" => json!({"connections": 2}), + "getbestblockhash" => json!(tip_hash), + "getblocktemplate" => json!({ + "height": chain.tip + 1, + "coinbasevalue": 5_000_000_000u64, + "previousblockhash": tip_hash, + "version": 0x2000_0000u32, + "bits": TEMPLATE_BITS, + "curtime": now, + "mintime": now - 1, + "transactions": [], + }), + "getblockhash" => match params[0].as_u64().and_then(|height| chain.hash_at(height)) { + Some(hash) => json!(hash), + None => return error(-8, "Block height out of range"), + }, + "getblockheader" => match chain.parent_of(params[0].as_str().unwrap_or_default()) { + Some(parent) => json!({"previousblockhash": parent}), + None => json!({}), + }, + "validateaddress" => { + let address = params[0].as_str().unwrap_or_default(); + if address.starts_with(ADDRESS_PREFIX) { + json!({"isvalid": true, "scriptPubKey": payout_script_hex(address)}) + } else { + json!({"isvalid": false}) + } + } + "submitblock" => { + let block = hex::decode(params[0].as_str().unwrap_or_default()).unwrap_or_default(); + if block.len() < 80 { + json!("bad-block-length") + } else { + let mut header = double_sha256(&block[..80]); + header.reverse(); + let hash = hex::encode(header); + let mut parent = block[4..36].to_vec(); + parent.reverse(); + if hex::encode(parent) != tip_hash { + json!("prev-blk-not-found") + } else { + let height = chain.tip + 1; + chain.tip = height; + chain.parents.insert(hash.clone(), tip_hash.clone()); + chain.heights.insert(hash.clone(), height); + chain.pool.insert(height, hash); + Value::Null + } + } + } + "waitfornewblock" => json!({"hash": tip_hash, "height": chain.tip}), + "estimatesmartfee" => json!({"feerate": "0.00001"}), + "getmempoolinfo" => json!({"minrelaytxfee": "0.00001", "mempoolminfee": "0.00001"}), + _ => return error(-32601, "unexpected RPC"), + }; + Json(json!({"id": request["id"], "result": result, "error": Value::Null})) +} diff --git a/crates/qbit-prism-server/tests/support/scripted_node.rs b/crates/qbit-prism-server/tests/support/scripted_node.rs new file mode 100644 index 000000000..da836b619 --- /dev/null +++ b/crates/qbit-prism-server/tests/support/scripted_node.rs @@ -0,0 +1,212 @@ +//! A scripted, mutable fake qbit node for coordinator tests that offer blocks. +//! +//! `tests/offer_latency.rs` and `tests/lost_race_pin.rs` both drive a real +//! `Coordinator` through `submitblock` against a chain the test moves. This is +//! that one node: an active chain by height, each block's parent, every +//! `submitblock` arrival, and the tip history. Unlike `support/fake_qbitd.rs`, +//! whose static tip serves suites that never submit, this node accepts blocks +//! and reorganizes. +//! +//! It answers the way qbitd does where the coordinator can tell the +//! difference: `getblockhash` above the tip, or at a height the node holds no +//! block for, is RPC error -8 (never the tip, never null), and +//! `getblockheader` for a hash it does not know is error -5. A block hash is +//! derived from its parent and its height, so a chain rebuilt after a reorg +//! never re-issues a hash under a different parent. +//! +//! Every tip change adds one unit of chainwork, so a replacement always has +//! strictly more cumulative work than the tip it replaces. +use qbit_prism_server::codec; +use serde_json::{json, Value}; +use std::{ + collections::HashMap, + sync::Arc, + time::{Duration, Instant}, +}; +use tokio::{sync::Mutex, task::JoinHandle}; + +/// The genesis hash both fixtures answer for height 0. +pub const GENESIS: &str = "0000000000000000000000000000000000000000000000000000000000000000"; + +pub struct ChainState { + pub tip: String, + pub height: u64, + pub chainwork: u64, + /// The active chain by height, for `getblockhash`. + blocks: HashMap, + /// Every block's parent the node has seen, for `getblockheader`. + parents: HashMap, + /// When each `submitblock` arrived, by block hash, taken before the + /// request is decoded: the node-entry boundary. + pub submissions: HashMap>, + /// Every best tip in order, with when it became the tip. + pub tips: Vec<(String, Instant)>, + /// When set, the next accepted `submitblock` loses a tip race to this + /// same-height competitor inside the handler, before the reply is sent. + lose_next_race_to: Option, +} + +impl ChainState { + /// A chain whose tip is `tip` at `height`, on the genesis parent. + pub fn new(tip: &str, height: u64) -> Self { + Self { + tip: tip.to_owned(), + height, + chainwork: 1, + blocks: HashMap::from([(0, GENESIS.to_owned()), (height, tip.to_owned())]), + parents: HashMap::from([(tip.to_owned(), GENESIS.to_owned())]), + submissions: HashMap::new(), + tips: vec![(tip.to_owned(), Instant::now())], + lose_next_race_to: None, + } + } + + /// The active block at `height`, if the node holds one. + pub fn block_at(&self, height: u64) -> Option<&str> { + self.blocks.get(&height).map(String::as_str) + } + + fn set_tip(&mut self, height: u64, hash: &str) { + let parent = self + .blocks + .get(&(height - 1)) + .cloned() + .expect("a new tip needs an active parent"); + self.blocks.retain(|at, _| *at < height); + self.blocks.insert(height, hash.to_owned()); + self.parents.insert(hash.to_owned(), parent); + self.tip = hash.to_owned(); + self.height = height; + self.chainwork += 1; + self.tips.push((hash.to_owned(), Instant::now())); + } + + /// Replace the active block at `height` with `hash`, on the active block + /// below it, and make it the tip, discarding everything above: the + /// node's own reorg. + pub fn reorg_to(&mut self, height: u64, hash: &str) { + self.set_tip(height, hash); + } + + /// Extend the active chain by one block of no interest to the pool. Its + /// hash commits to its parent and its height. + pub fn advance(&mut self) { + let height = self.height + 1; + let hash = codec::hash_display(&codec::double_sha256( + format!("{}:{height}", self.tip).as_bytes(), + )); + self.set_tip(height, &hash); + } + + /// The next block `submitblock` accepts is the best tip only inside the + /// handler: before the reply, `competitor` replaces it at the same height. + pub fn lose_next_race_to(&mut self, competitor: &str) { + self.lose_next_race_to = Some(competitor.to_owned()); + } + + /// How long `hash` was the best tip before the next tip replaced it, if + /// it was the tip and has been replaced. + pub fn tip_lifetime(&self, hash: &str) -> Option { + let at = self.tips.iter().position(|(tip, _)| tip == hash)?; + let (_, next) = self.tips.get(at + 1)?; + Some(next.duration_since(self.tips[at].1)) + } +} + +pub struct ScriptedNode { + pub url: String, + pub state: Arc>, + task: JoinHandle<()>, +} + +impl ScriptedNode { + pub async fn open(state: ChainState) -> anyhow::Result { + let state = Arc::new(Mutex::new(state)); + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await?; + let url = format!("http://{}/", listener.local_addr()?); + let app = axum::Router::new() + .route("/", axum::routing::post(reply)) + .with_state(state.clone()); + let task = tokio::spawn(async move { + let _ = axum::serve(listener, app).await; + }); + Ok(Self { url, state, task }) + } + + /// Stop answering: every later RPC fails to connect. + pub fn stop(&self) { + self.task.abort(); + } +} + +impl Drop for ScriptedNode { + fn drop(&mut self) { + self.task.abort(); + } +} + +fn rpc_error(code: i64, message: String) -> Result { + Err(json!({"code": code, "message": message})) +} + +async fn reply( + axum::extract::State(node): axum::extract::State>>, + axum::Json(request): axum::Json, +) -> axum::Json { + let arrived = Instant::now(); + let now = chrono::Utc::now().timestamp(); + let mut node = node.lock().await; + let result = match request["method"].as_str().unwrap_or("") { + "getblockhash" => match request["params"][0].as_u64() { + Some(height) if height <= node.height => match node.block_at(height) { + Some(hash) => Ok(json!(hash)), + None => rpc_error(-8, format!("no active block held at height {height}")), + }, + _ => rpc_error(-8, "Block height out of range".into()), + }, + "getblockheader" => { + let hash = request["params"][0].as_str().unwrap_or(""); + match node.parents.get(hash) { + Some(parent) => Ok(json!({"previousblockhash": parent})), + None => rpc_error(-5, "Block not found".into()), + } + } + "getblockchaininfo" => Ok(json!({"chain":"test","initialblockdownload":false, + "blocks":node.height,"headers":node.height,"bestblockhash":node.tip, + "chainwork":format!("{:x}",node.chainwork)})), + "getbestblockhash" => Ok(json!(node.tip)), + "getnetworkinfo" => Ok(json!({"connections":2})), + "getblocktemplate" => Ok( + json!({"height":node.height+1,"coinbasevalue":500_000_000u64, + "previousblockhash":node.tip,"version":0x20000000u32,"bits":"207fffff", + "curtime":now,"mintime":now-1,"transactions":[]}), + ), + "estimatesmartfee" => Ok(json!({"feerate":"0.00001"})), + "getmempoolinfo" => Ok(json!({"minrelaytxfee":"0.00001","mempoolminfee":"0.00001"})), + "validateaddress" => { + Ok(json!({"isvalid":true,"scriptPubKey":format!("5220{}","11".repeat(32))})) + } + "submitblock" => { + let block = hex::decode(request["params"][0].as_str().unwrap_or("")).unwrap(); + let hash = codec::hash_display(&codec::double_sha256(&block[..80])); + node.submissions + .entry(hash.clone()) + .or_default() + .push(arrived); + // Accepted: null reply, and the block is the new best tip. + let height = node.height + 1; + node.set_tip(height, &hash); + // The scripted lost race runs here, under the same lock and + // before the reply, so no observation can see the block active. + if let Some(competitor) = node.lose_next_race_to.take() { + node.reorg_to(height, &competitor); + } + Ok(Value::Null) + } + method => rpc_error(-32601, format!("unexpected scripted-node RPC {method}")), + }; + axum::Json(match result { + Ok(result) => json!({"id":request["id"],"result":result,"error":null}), + Err(error) => json!({"id":request["id"],"result":null,"error":error}), + }) +} diff --git a/crates/qbit-prism-server/tests/support/session_sequence.rs b/crates/qbit-prism-server/tests/support/session_sequence.rs new file mode 100644 index 000000000..0c30169ec --- /dev/null +++ b/crates/qbit-prism-server/tests/support/session_sequence.rs @@ -0,0 +1,416 @@ +use super::*; +use qbit_prism_server::ledger::{HeartbeatStatus, SessionAllocationExhausted}; + +async fn wrap(pool: &PgPool) -> Result<()> { + sqlx::query("SELECT setval('qbit_prism_session_sequence',4294967295,true)") + .execute(pool) + .await?; + Ok(()) +} + +#[tokio::test] +async fn stopped_guard_blocks_cleanup_and_owner_filter_preserves_other_reservations() -> Result<()> +{ + let Some(db) = Database::open().await? else { + return Ok(()); + }; + let ledger = db.ledger("shutdown-guard").await?; + let other = db.ledger("other-owner").await?; + let held = ledger.new_session_id().await?; + let owner_token: String = sqlx::query_scalar( + "SELECT status->>'session_owner_token' FROM qbit_prism_instances WHERE instance_id='shutdown-guard'", + ) + .fetch_one(&ledger.pool) + .await?; + sqlx::query("INSERT INTO qbit_prism_session_reservations(extranonce1,instance_id,owner_token,reservation_token) VALUES(99,'shutdown-guard',$1,'leftover')") + .bind(owner_token) + .execute(&ledger.pool) + .await?; + let other_token: String = sqlx::query_scalar("SELECT status->>'session_owner_token' FROM qbit_prism_instances WHERE instance_id='other-owner'") + .fetch_one(&other.pool).await?; + sqlx::query("INSERT INTO qbit_prism_session_reservations(extranonce1,instance_id,owner_token,reservation_token) VALUES(100,'other-owner',$1,'other')") + .bind(other_token).execute(&other.pool).await?; + let stopped = ledger.heartbeat(HeartbeatStatus::Stopped).await; + assert!( + stopped.is_err(), + "stopped must fail while a guard is active" + ); + let mixed_health = HeartbeatStatus::Health(serde_json::from_value(json!({ + "schema":"qbit.prism.audit-health.v1", "ready":false, "state":"stopped" + }))?); + assert!( + ledger.heartbeat(mixed_health).await.is_err(), + "health payloads must not publish a reclaimable stopped marker with active sessions" + ); + let state: String = sqlx::query_scalar( + "SELECT status->>'state' FROM qbit_prism_instances WHERE instance_id='shutdown-guard'", + ) + .fetch_one(&ledger.pool) + .await?; + assert_eq!( + state, "starting", + "failed shutdown must not change the stored marker" + ); + let count: i64 = sqlx::query_scalar("SELECT count(*) FROM qbit_prism_session_reservations") + .fetch_one(&ledger.pool) + .await?; + assert_eq!(count, 3, "failed shutdown retains all reservations"); + held.release().await?; + ledger.heartbeat(HeartbeatStatus::Stopped).await?; + ledger.release_session_owner_reservations().await?; + let count: i64 = sqlx::query_scalar("SELECT count(*) FROM qbit_prism_session_reservations") + .fetch_one(&ledger.pool) + .await?; + assert_eq!(count, 1, "successful stopped path cleans only this owner"); + db.close(vec![ledger, other]).await +} + +async fn seed_job(pool: &PgPool, id: &str, extra: &str, live: bool) -> Result<()> { + sqlx::query("INSERT INTO qbit_prism_jobs(job_id,instance_id,parent_hash,payout_revision,payload,expires_at) VALUES($1,'pre-009','parent',0,$2,clock_timestamp()+$3*interval '1 hour')") + .bind(id).bind(json!({"extranonce1":extra,"old-field":"retained"})) + .bind(if live {1_i64} else {-1_i64}).execute(pool).await?; + Ok(()) +} + +#[tokio::test] +async fn wrap_reuses_expired_jobs_and_skips_live_jobs_and_sessions() -> Result<()> { + let Some(db) = Database::open().await? else { + return Ok(()); + }; + let a = db.ledger("a").await?; + let b = db.ledger("b").await?; + seed_job(&a.pool, "expired", "00000001", false).await?; + seed_job(&a.pool, "live", "00000002", true).await?; + wrap(&a.pool).await?; + let first = a.new_session_id().await?; + assert_eq!(first.value(), 1, "an expired job must not prevent wrap"); + wrap(&a.pool).await?; + let next = b.new_session_id().await?; + assert_eq!( + next.value(), + 3, + "the live reservation and live job both block reuse" + ); + first.release().await?; + next.release().await?; + seed_job(&a.pool, "live-next", "00000001", true).await?; + wrap(&a.pool).await?; + let next = b.new_session_id().await?; + assert_eq!( + next.value(), + 3, + "a live job at the wrap boundary must be skipped" + ); + next.release().await?; + db.close(vec![a, b]).await +} + +#[tokio::test] +async fn migration_009_preserves_preexisting_jobs_and_runs_once_for_two_frontends() -> Result<()> { + let Some(db) = Database::open().await? else { + return Ok(()); + }; + let pool = PgPool::connect(&db.url).await?; + // Build the actual pre-006, pre-009 schema, not an already-migrated + // approximation; the runner applies both, each in its place. + sqlx::raw_sql(include_str!("../../../qbit-prism/sql/001_share_ledger.sql")) + .execute(&pool) + .await?; + for sql in [ + include_str!("../../migrations/002_multi_instance.sql"), + include_str!("../../migrations/003_2x_compatibility.sql"), + include_str!("../../migrations/004_cpfp_retired_funding.sql"), + include_str!("../../migrations/005_candidate_dispatch.sql"), + ] { + sqlx::raw_sql(sql).execute(&pool).await?; + } + sqlx::raw_sql("CREATE TABLE qbit_prism_schema_migrations(version integer PRIMARY KEY,applied_at timestamptz NOT NULL DEFAULT clock_timestamp()); INSERT INTO qbit_prism_schema_migrations(version) VALUES(2),(3),(4),(5)") + .execute(&pool).await?; + seed_job(&pool, "expired", "00000001", false).await?; + seed_job(&pool, "old-arbitrary", "0000000A", true).await?; + let mut before: Vec = + sqlx::query_scalar("SELECT to_jsonb(j) FROM qbit_prism_jobs j ORDER BY job_id") + .fetch_all(&pool) + .await?; + // The combined runner adds 008's nullable references. Require exactly + // those null additions while preserving every original field and value. + for row in &mut before { + for column in [ + "template_sha256", + "window_anchor_ms", + "window_first_share_seq", + "window_last_share_seq", + "window_prior_balances_sha256", + "window_share_count", + "window_snapshot_sha256", + ] { + assert!(row + .as_object_mut() + .unwrap() + .insert(column.into(), serde_json::Value::Null) + .is_none()); + } + } + let (a, b) = tokio::try_join!(db.ledger("a"), db.ledger("b"))?; + let after: Vec = + sqlx::query_scalar("SELECT to_jsonb(j) FROM qbit_prism_jobs j ORDER BY job_id") + .fetch_all(&pool) + .await?; + assert_eq!(before, after, "migration must not rewrite pre-009 rows"); + let versions: Vec = + sqlx::query_scalar("SELECT version FROM qbit_prism_schema_migrations ORDER BY version") + .fetch_all(&pool) + .await?; + assert_eq!( + versions, + vec![ + 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, + 26 + ] + ); + let cycled: bool = sqlx::query_scalar( + "SELECT seqcycle FROM pg_sequence WHERE seqrelid='qbit_prism_session_sequence'::regclass", + ) + .fetch_one(&pool) + .await?; + assert!(cycled); + wrap(&pool).await?; + let first = a.new_session_id().await?; + assert_eq!(first.value(), 1); + sqlx::query("SELECT setval('qbit_prism_session_sequence',9,true)") + .execute(&pool) + .await?; + let next = b.new_session_id().await?; + assert_eq!( + next.value(), + 11, + "pre-009 uppercase hexadecimal is protected" + ); + first.release().await?; + next.release().await?; + pool.close().await; + db.close(vec![a, b]).await +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn two_frontends_race_across_wrap_without_duplicate_allocations() -> Result<()> { + let Some(db) = Database::open().await? else { + return Ok(()); + }; + let a = db.ledger("a").await?; + let b = db.ledger("b").await?; + let a_pid: i32 = sqlx::query_scalar("SELECT pg_backend_pid()") + .fetch_one(&a.pool) + .await?; + let b_pid: i32 = sqlx::query_scalar("SELECT pg_backend_pid()") + .fetch_one(&b.pool) + .await?; + assert_ne!(a_pid, b_pid, "must use two real PostgreSQL connections"); + sqlx::raw_sql("ALTER SEQUENCE qbit_prism_session_sequence MAXVALUE 32; SELECT setval('qbit_prism_session_sequence',32,true)") + .execute(&a.pool).await?; + let ids = futures_util::future::try_join_all((0..32).map(|n| { + let ledger = if n % 2 == 0 { &a } else { &b }; + ledger.new_session_id() + })) + .await?; + let unique: std::collections::HashSet<_> = ids.iter().map(|id| id.value()).collect(); + assert_eq!(unique.len(), 32); + for id in ids { + id.release().await?; + } + + // Only one free value forces both frontends to attempt the SAME candidate, + // including after nextval cycles, rather than merely testing nextval itself. + seed_job(&a.pool, "occupied-second-value", "00000002", true).await?; + sqlx::raw_sql("ALTER SEQUENCE qbit_prism_session_sequence RESTART WITH 1 MAXVALUE 2") + .execute(&a.pool) + .await?; + let (left, right) = tokio::join!(a.new_session_id(), b.new_session_id()); + let (winner, error) = match (left, right) { + (Ok(id), Err(error)) | (Err(error), Ok(id)) => (id, error), + other => panic!("exactly one concurrent allocation must win: {other:?}"), + }; + assert_eq!(winner.value(), 1); + assert!(error.is::(), "{error:#}"); + assert!(!error.to_string().contains("database unavailable")); + let count: i64 = sqlx::query_scalar("SELECT count(*) FROM qbit_prism_session_reservations") + .fetch_one(&a.pool) + .await?; + assert_eq!(count, 1, "failed contender must not release the winner"); + winner.release().await?; + db.close(vec![a, b]).await +} + +#[tokio::test] +async fn only_stopped_owners_are_reclaimed_and_old_cleanup_cannot_release_a_replacement( +) -> Result<()> { + let Some(db) = Database::open().await? else { + return Ok(()); + }; + let a = db.ledger("a").await?; + let b = db.ledger("b").await?; + let held = a.new_session_id().await?; + assert_eq!(held.value(), 1); + sqlx::query("UPDATE qbit_prism_instances SET heartbeat_at=clock_timestamp()-interval '2 days' WHERE instance_id='a'") + .execute(&a.pool).await?; + wrap(&a.pool).await?; + let skipped = b.new_session_id().await?; + assert_eq!( + skipped.value(), + 2, + "stale heartbeats do not prove session exit" + ); + skipped.release().await?; + sqlx::query("DELETE FROM qbit_prism_instances WHERE instance_id='a'") + .execute(&a.pool) + .await?; + wrap(&a.pool).await?; + let skipped = b.new_session_id().await?; + assert_eq!( + skipped.value(), + 2, + "absent owners do not prove session exit" + ); + skipped.release().await?; + let (owner_token, old_token): (String, String) = sqlx::query_as("SELECT owner_token,reservation_token FROM qbit_prism_session_reservations WHERE extranonce1=1") + .fetch_one(&a.pool).await?; + assert!(a.heartbeat(HeartbeatStatus::Stopped).await.is_err()); + held.release().await?; + // Recreate a missed cleanup after the actual guard has ended. A stopped + // heartbeat must never be published while that live guard still exists. + sqlx::query("INSERT INTO qbit_prism_session_reservations(extranonce1,instance_id,owner_token,reservation_token) VALUES(1,'a',$1,$2)") + .bind(owner_token).bind(&old_token).execute(&a.pool).await?; + a.heartbeat(HeartbeatStatus::Stopped).await?; + seed_job(&a.pool, "stopped-live-job", "00000001", true).await?; + wrap(&a.pool).await?; + let skipped = b.new_session_id().await?; + assert_eq!( + skipped.value(), + 2, + "stopped-owner reclaim must still check jobs" + ); + skipped.release().await?; + sqlx::query("UPDATE qbit_prism_jobs SET expires_at=clock_timestamp()-interval '1 second'") + .execute(&a.pool) + .await?; + wrap(&a.pool).await?; + let replacement = b.new_session_id().await?; + assert_eq!(replacement.value(), 1); + let late_cleanup = sqlx::query( + "DELETE FROM qbit_prism_session_reservations WHERE extranonce1=1 AND reservation_token=$1", + ) + .bind(old_token) + .execute(&a.pool) + .await? + .rows_affected(); + assert_eq!(late_cleanup, 0); + let owner: String = sqlx::query_scalar( + "SELECT instance_id FROM qbit_prism_session_reservations WHERE extranonce1=1", + ) + .fetch_one(&b.pool) + .await?; + assert_eq!( + owner, "b", + "late cleanup cannot delete a replacement's token" + ); + replacement.release().await?; + db.close(vec![a, b]).await +} + +#[tokio::test] +async fn stopped_requires_no_pending_or_live_sessions_and_cannot_reclaim_another_incarnation( +) -> Result<()> { + let Some(db) = Database::open().await? else { + return Ok(()); + }; + let a = db.ledger("reused-instance-id").await?; + let held = a.new_session_id().await?; + let other_incarnation = db.ledger("reused-instance-id").await?; + let allocator = db.ledger("allocator").await?; + assert!( + a.heartbeat(HeartbeatStatus::Stopped).await.is_err(), + "a live session prevents stopped" + ); + assert!( + a.clone().heartbeat(HeartbeatStatus::Stopped).await.is_err(), + "Ledger clones share admission state" + ); + other_incarnation + .heartbeat(HeartbeatStatus::Stopped) + .await?; + wrap(&a.pool).await?; + let next = allocator.new_session_id().await?; + assert_eq!(next.value(),2,"stopped from a second process with the same instance ID cannot reclaim the first process's session"); + next.release().await?; + held.release().await?; + + // Poll allocation until it waits on a saturated real PostgreSQL pool, + // retaining the pending future across timeout instead of cancelling it. + let mut connections = Vec::new(); + for _ in 0..8 { + connections.push(a.pool.acquire().await?); + } + let mut pending = Box::pin(a.new_session_id()); + assert!( + tokio::time::timeout(std::time::Duration::from_millis(20), pending.as_mut()) + .await + .is_err() + ); + assert!( + a.heartbeat(HeartbeatStatus::Stopped).await.is_err(), + "an allocation awaiting a connection also prevents stopped" + ); + drop(pending); + drop(connections); + a.heartbeat(HeartbeatStatus::Stopped).await?; + let error = a.new_session_id().await.unwrap_err(); + assert!( + error.to_string().contains("allocator is stopped"), + "stopped closes future admission" + ); + db.close(vec![a, other_incarnation, allocator]).await +} + +#[tokio::test] +async fn cancelling_a_session_releases_only_its_reservation() -> Result<()> { + let Some(db) = Database::open().await? else { + return Ok(()); + }; + let a = db.ledger("a").await?; + let b = db.ledger("b").await?; + let held = b.new_session_id().await?; + let (ready_tx, ready_rx) = tokio::sync::oneshot::channel(); + let worker = a.clone(); + let session = tokio::spawn(async move { + let id = worker.new_session_id().await.unwrap(); + ready_tx.send(id.value()).unwrap(); + std::future::pending::<()>().await; + drop(id); + }); + let cancelled = ready_rx.await?; + session.abort(); + assert!(session.await.unwrap_err().is_cancelled()); + tokio::time::timeout(std::time::Duration::from_secs(5), async { + loop { + let exists: bool = sqlx::query_scalar( + "SELECT EXISTS(SELECT 1 FROM qbit_prism_session_reservations WHERE extranonce1=$1)", + ) + .bind(i64::from(cancelled)) + .fetch_one(&a.pool) + .await?; + if !exists { + break; + } + tokio::time::sleep(std::time::Duration::from_millis(10)).await; + } + Ok::<_, anyhow::Error>(()) + }) + .await??; + let remaining: Vec = + sqlx::query_scalar("SELECT extranonce1 FROM qbit_prism_session_reservations") + .fetch_all(&a.pool) + .await?; + assert_eq!(remaining, vec![i64::from(held.value())]); + held.release().await?; + db.close(vec![a, b]).await +} diff --git a/crates/qbit-prism-server/tests/support/share_hash_backfill.rs b/crates/qbit-prism-server/tests/support/share_hash_backfill.rs new file mode 100644 index 000000000..12473ed07 --- /dev/null +++ b/crates/qbit-prism-server/tests/support/share_hash_backfill.rs @@ -0,0 +1,3087 @@ +//! Migration 002's share-hash backfill on a populated 2.x.x source (#582). +//! +//! 002 maps the header of every accepted legacy share whose ID ends in 64 hex +//! digits, the earliest `share_seq` winning. That used to be one statement +//! inside the migration transaction, which outlasted the statement timeout +//! on a production-sized ledger. It now runs after the commit, in batches of +//! consecutive `share_seq` with a durable cursor, and records 2 last. These +//! tests hold the batched result to the single statement's on a ledger that +//! repeats headers across batch boundaries, interrupt a run mid-batch and +//! resume it, and check that nothing serves the database in between. With +//! `migrate --defer-share-hashes` only the recent range is mapped before +//! the database serves, and plain `migrate` maps the rest while frontends +//! append; the last tests here hold that to the same rule. +use super::*; +use qbit_prism_server::ledger::{ + archive, IndexBuildMode, MigrateOptions, ShareHashBackfill, ShareHashThrottle, + REQUIRED_SCHEMA_VERSIONS, +}; +use std::collections::HashSet; +use std::time::Duration; +use tokio::time::{sleep, timeout}; + +/// The class of the online runners' session lock (`ONLINE_DDL_LOCK_CLASS`), +/// keyed by the ledger's schema. +const RUNNER_LOCK_CLASS: i32 = 0x5052_4953; +/// Ledger rows the seed writes, before its gaps: several batches at the +/// runner's first batch of 10,000 `share_seq`. +const SEED_ROWS: i64 = 60_000; +/// A share whose header first appears in a late batch; `share(LATE)` repeats +/// that header under another identity. +const LATE: u64 = 50_001; + +/// A 2.x.x ledger that exercises 002's rule: one header per share, with +/// gaps in `share_seq`; every 97th row repeats, in upper case, the header of +/// the share 9,001 below or above it, so the earlier copy wins whichever +/// side it is on and the pairs straddle batch boundaries; non-hex legacy +/// IDs, bare 64-digit IDs and rejected rows among them. +async fn seed(pool: &PgPool) -> Result<()> { + sqlx::query( + "INSERT INTO qbit_share_ledger(share_seq,share_id,miner_id,payout_order_key,p2mr_program,share_difficulty,network_difficulty,template_height,job_id,job_issued_at,ntime,accepted_at,accepted,reject_reason,writer_id,writer_epoch) \ + SELECT g,i.id,'miner-'||g%5,'miner-'||g%5,decode(repeat('11',32),'hex'),1,100,100,'job',to_timestamp(1),1,to_timestamp(2),g%1000<>13,CASE WHEN g%1000=13 THEN 'duplicate-share' END,'backfill-test',0 \ + FROM generate_series(1,$1::bigint) g, LATERAL (SELECT CASE \ + WHEN g%97=0 THEN 'repeat-'||g||':'||upper(lpad(to_hex(g+CASE WHEN g%2=0 THEN -9001 ELSE 9001 END),64,'0')) \ + WHEN g%1000=7 THEN 'legacy:'||g \ + WHEN g%1000=11 THEN lpad(to_hex(g),64,'0') \ + ELSE 'worker-'||g%5||':'||lpad(to_hex(g),64,'0') END AS id) i \ + WHERE g%89<>0", + ) + .bind(SEED_ROWS) + .execute(pool) + .await?; + sqlx::query("SELECT setval('qbit_share_ledger_share_seq_seq',$1)") + .bind(SEED_ROWS) + .execute(pool) + .await?; + Ok(()) +} + +/// What 002's single statement mapped, for the legacy shares below `below` +/// (all of them with `i64::MAX`): the header of each accepted share whose +/// ID ends in 64 hex digits, to its earliest share. +async fn expected_mapping(pool: &PgPool, below: i64) -> Result> { + Ok(sqlx::query_as( + "SELECT DISTINCT ON (lower(right(share_id,64))) lower(right(share_id,64)),share_id FROM qbit_share_ledger WHERE accepted AND share_id ~ '[0-9a-fA-F]{64}$' AND share_seq<$1 ORDER BY lower(right(share_id,64)),share_seq", + ) + .bind(below) + .fetch_all(pool) + .await?) +} + +async fn mapping(pool: &PgPool) -> Result> { + Ok(sqlx::query_as( + "SELECT header_hash,share_id FROM qbit_prism_share_hashes ORDER BY header_hash", + ) + .fetch_all(pool) + .await?) +} + +async fn schema_versions(pool: &PgPool) -> Result> { + Ok( + sqlx::query_scalar("SELECT version FROM qbit_prism_schema_migrations ORDER BY version") + .fetch_all(pool) + .await?, + ) +} + +/// The cursor of a pending backfill, `None` once its table is gone. +async fn cursor(pool: &PgPool) -> Result> { + if !sqlx::query_scalar::<_, bool>( + "SELECT to_regclass('qbit_prism_share_hash_backfill') IS NOT NULL", + ) + .fetch_one(pool) + .await? + { + return Ok(None); + } + Ok(Some( + sqlx::query_as("SELECT next_seq,end_seq FROM qbit_prism_share_hash_backfill") + .fetch_one(pool) + .await?, + )) +} + +/// 2 is recorded after every migration of the transaction and before the +/// online 013, 017 and 024, which each check that they come after every +/// lower version. +async fn assert_recorded_in_order(pool: &PgPool) -> Result<()> { + let (after_transaction, before_online): (bool, bool) = sqlx::query_as( + "SELECT (SELECT applied_at FROM qbit_prism_schema_migrations WHERE version=2) >= (SELECT max(applied_at) FROM qbit_prism_schema_migrations WHERE version NOT IN (2,13,17,24)), \ + (SELECT applied_at FROM qbit_prism_schema_migrations WHERE version=2) <= (SELECT min(applied_at) FROM qbit_prism_schema_migrations WHERE version IN (13,17,24))", + ) + .fetch_one(pool) + .await?; + ensure!( + after_transaction && before_online, + "2 is not recorded between the transaction's migrations and 013, 017 and 024" + ); + Ok(()) +} + +/// Every start refuses a database whose backfill has not finished, naming +/// its cursor, whether or not 2 is recorded. +async fn assert_refused_at(url: &str, next_seq: i64, end_seq: i64) -> Result<()> { + let error = Ledger::connect(url, "cold".into(), 8, false) + .await + .err() + .context("a start accepted a database whose share-hash backfill has not finished")? + .to_string(); + ensure!( + error.contains("migration 2's share-hash backfill has not finished") + && error.contains(&format!("those from {next_seq} up to {end_seq} are not")), + "{error}" + ); + Ok(()) +} + +#[tokio::test] +async fn migration_002_maps_a_populated_2x_ledger_in_batches_as_its_single_statement_did( +) -> Result<()> { + let Some(db) = Database::open().await? else { + return Ok(()); + }; + let pool = PgPool::connect(&db.url).await?; + two_x::apply_frozen_2x_schema(&pool, SourceState::Pre258).await?; + seed(&pool).await?; + let expected = expected_mapping(&pool, i64::MAX).await?; + // The seed exercises every branch of the rule. + ensure!(expected.iter().any(|(_, id)| id.starts_with("repeat-"))); + ensure!(expected.iter().any(|(_, id)| id.len() == 64)); + ensure!(!expected.iter().any(|(_, id)| id.starts_with("legacy:"))); + let ledger = db.ledger("cutover").await?; + assert_eq!(mapping(&pool).await?, expected); + assert_eq!(cursor(&pool).await?, None, "the progress table outlived 2"); + assert_eq!(schema_versions(&pool).await?, REQUIRED_SCHEMA_VERSIONS); + assert_recorded_in_order(&pool).await?; + // A legacy header stays credited to its legacy share. + let error = ledger + .append(share(LATE), None) + .await + .err() + .context("a native share repeating a legacy header was credited")?; + assert!( + format!("{error:#}").contains("header already credited globally"), + "{error:#}" + ); + // A second start has nothing left to map. + let again = db.ledger("again").await?; + assert_eq!(mapping(&pool).await?, expected); + db.close(vec![ledger, again]).await +} + +#[tokio::test] +async fn migration_002_backfill_resumes_from_its_last_committed_batch_after_an_interrupt( +) -> Result<()> { + let Some(db) = Database::open().await? else { + return Ok(()); + }; + let pool = PgPool::connect(&db.url).await?; + two_x::apply_frozen_2x_schema(&pool, SourceState::Pre258).await?; + seed(&pool).await?; + let expected = expected_mapping(&pool, i64::MAX).await?; + let (first, end): (i64, i64) = + sqlx::query_as("SELECT min(share_seq),max(share_seq)+1 FROM qbit_share_ledger") + .fetch_one(&pool) + .await?; + let late = format!("{LATE:064x}"); + let late_seq: i64 = sqlx::query_scalar( + "SELECT min(share_seq) FROM qbit_share_ledger WHERE accepted AND lower(right(share_id,64))=$1", + ) + .bind(&late) + .fetch_one(&pool) + .await?; + assert_eq!(late_seq, LATE as i64, "the seed moved the late header"); + + // Hold the runners' lock so the migration transaction commits and the + // backfill waits before its first batch. + let mut runners = PgConnection::connect(&db.url).await?; + sqlx::query("SELECT pg_advisory_lock($1,hashtext(current_schema()))") + .bind(RUNNER_LOCK_CLASS) + .execute(&mut runners) + .await?; + let mut migrate = Box::pin(Ledger::connect(&db.url, "interrupted".into(), 8, true)); + let committed = timeout(Duration::from_secs(60), async { + while cursor(&pool).await?.is_none() { + sleep(Duration::from_millis(10)).await; + } + Ok::<_, anyhow::Error>(()) + }); + tokio::select! { + result = &mut migrate => bail!("migrate ended before the backfill ran: {:?}", result.err()), + result = committed => result.context("the migration transaction never committed")??, + } + // Committed: every migration but 2 and the online 013, 017 and 024 recorded, + // nothing mapped yet, and no start serves the database. + let pending: Vec = REQUIRED_SCHEMA_VERSIONS + .iter() + .copied() + .filter(|version| ![2, 13, 17, 24].contains(version)) + .collect(); + assert_eq!(schema_versions(&pool).await?, pending); + assert_eq!(cursor(&pool).await?, Some((first, end))); + assert!(mapping(&pool).await?.is_empty()); + assert_refused_at(&db.url, first, end).await?; + + // An open insert of the late header stops the batch that reaches it. + let mut blocker = pool.begin().await?; + sqlx::query( + "INSERT INTO qbit_prism_share_hashes(header_hash,share_id) VALUES($1,'blocker:'||$1)", + ) + .bind(&late) + .execute(&mut *blocker) + .await?; + sqlx::query("SELECT pg_advisory_unlock($1,hashtext(current_schema()))") + .bind(RUNNER_LOCK_CLASS) + .execute(&mut runners) + .await?; + let waiting = timeout(Duration::from_secs(60), async { + loop { + let pid: Option = sqlx::query_scalar("SELECT pid FROM pg_stat_activity WHERE datname=current_database() AND pid<>pg_backend_pid() AND query LIKE 'INSERT INTO qbit_prism_share_hashes%' AND wait_event_type='Lock'") + .fetch_optional(&pool) + .await?; + if let Some(pid) = pid { + return Ok::<_, anyhow::Error>(pid); + } + sleep(Duration::from_millis(10)).await; + } + }); + let pid: i32 = tokio::select! { + result = &mut migrate => bail!("migrate ended before its batch reached the late header: {:?}", result.err()), + result = waiting => result.context("no batch waited on the late header")??, + }; + // Interrupt the run mid-batch, as a crash or a killed migrate would, at + // once: the waiting batch also runs under the pool's lock timeout. + sqlx::query("SELECT pg_terminate_backend($1)") + .bind(pid) + .execute(&pool) + .await?; + let failed = timeout(Duration::from_secs(30), migrate).await?; + ensure!( + failed.is_err(), + "migrate succeeded although its batch was killed" + ); + blocker.rollback().await?; + // Earlier batches committed and the cursor says where they ended. The + // killed batch left nothing: the mapping is the single statement's over + // exactly that prefix, 2 is not recorded, and starts still refuse. + let (next, planned_end) = cursor(&pool).await?.context("the progress table is gone")?; + assert_eq!(planned_end, end); + assert!( + next > first && next <= late_seq, + "cursor {next} is not past the first batch and before {late_seq}" + ); + assert_eq!(mapping(&pool).await?, expected_mapping(&pool, next).await?); + assert_eq!(schema_versions(&pool).await?, pending); + assert_refused_at(&db.url, next, end).await?; + // A hand-recorded 2 does not let a start serve while the table says + // legacy shares are unmapped. + sqlx::query("INSERT INTO qbit_prism_schema_migrations(version) VALUES(2)") + .execute(&pool) + .await?; + assert_refused_at(&db.url, next, end).await?; + sqlx::query("DELETE FROM qbit_prism_schema_migrations WHERE version=2") + .execute(&pool) + .await?; + + // migrate resumes at the cursor and finishes: the single statement's + // mapping, 2 recorded before 013, 017 and 024, the progress table gone. + let resumed = timeout(Duration::from_secs(120), db.ledger("resumed")).await??; + assert_eq!(mapping(&pool).await?, expected); + assert_eq!(cursor(&pool).await?, None); + assert_eq!(schema_versions(&pool).await?, REQUIRED_SCHEMA_VERSIONS); + assert_recorded_in_order(&pool).await?; + let error = resumed + .append(share(LATE), None) + .await + .err() + .context("a native share repeating a legacy header was credited")?; + assert!( + format!("{error:#}").contains("header already credited globally"), + "{error:#}" + ); + runners.close().await?; + db.close(vec![resumed]).await +} + +/// Nothing appends before 2 is recorded, so the end the cursor was planned +/// to is the ledger's end. A row past it, from a writer that went around +/// every gate, is still mapped before 2 is recorded, not left unmapped. +#[tokio::test] +async fn migration_002_maps_a_row_past_its_planned_end_before_recording_2() -> Result<()> { + let Some(db) = Database::open().await? else { + return Ok(()); + }; + let pool = PgPool::connect(&db.url).await?; + two_x::apply_frozen_2x_schema(&pool, SourceState::Pre258).await?; + seed(&pool).await?; + let mut runners = PgConnection::connect(&db.url).await?; + sqlx::query("SELECT pg_advisory_lock($1,hashtext(current_schema()))") + .bind(RUNNER_LOCK_CLASS) + .execute(&mut runners) + .await?; + let mut migrate = Box::pin(Ledger::connect(&db.url, "cutover".into(), 8, true)); + let committed = timeout(Duration::from_secs(60), async { + while cursor(&pool).await?.is_none() { + sleep(Duration::from_millis(10)).await; + } + Ok::<_, anyhow::Error>(()) + }); + tokio::select! { + result = &mut migrate => bail!("migrate ended before the backfill ran: {:?}", result.err()), + result = committed => result.context("the migration transaction never committed")??, + } + let (_, end) = cursor(&pool).await?.context("the progress table is gone")?; + let straggler = "ee".repeat(32); + sqlx::query( + "INSERT INTO qbit_share_ledger(share_seq,share_id,miner_id,payout_order_key,p2mr_program,share_difficulty,network_difficulty,template_height,job_id,job_issued_at,ntime,accepted_at,accepted,writer_id,writer_epoch) \ + VALUES($1,'straggler:'||$2,'miner-0','miner-0',decode(repeat('11',32),'hex'),1,100,100,'job',to_timestamp(1),1,to_timestamp(2),true,'backfill-test',0)", + ) + .bind(end + 5) + .bind(&straggler) + .execute(&pool) + .await?; + sqlx::query("SELECT pg_advisory_unlock($1,hashtext(current_schema()))") + .bind(RUNNER_LOCK_CLASS) + .execute(&mut runners) + .await?; + let ledger = timeout(Duration::from_secs(120), migrate).await??; + assert_eq!( + mapping(&pool).await?, + expected_mapping(&pool, i64::MAX).await? + ); + let mapped: Option = + sqlx::query_scalar("SELECT share_id FROM qbit_prism_share_hashes WHERE header_hash=$1") + .bind(&straggler) + .fetch_optional(&pool) + .await?; + assert_eq!(mapped, Some(format!("straggler:{straggler}"))); + assert_eq!(cursor(&pool).await?, None); + assert_eq!(schema_versions(&pool).await?, REQUIRED_SCHEMA_VERSIONS); + runners.close().await?; + db.close(vec![ledger]).await +} + +#[tokio::test] +async fn migration_002_refuses_a_2x_source_holding_the_backfill_progress_name_before_any_ddl( +) -> Result<()> { + let Some(db) = Database::open().await? else { + return Ok(()); + }; + let pool = PgPool::connect(&db.url).await?; + two_x::apply_frozen_2x_schema(&pool, SourceState::Pre258).await?; + seed(&pool).await?; + sqlx::raw_sql("CREATE TABLE qbit_prism_share_hash_backfill(operator_notes text)") + .execute(&pool) + .await?; + let error = db + .ledger("cutover") + .await + .err() + .context("migrate adopted a foreign qbit_prism_share_hash_backfill")? + .to_string(); + assert!( + error.contains("a table named qbit_prism_share_hash_backfill already exists") + && error.contains("Nothing was changed"), + "{error}" + ); + let untouched: bool = sqlx::query_scalar( + "SELECT to_regclass('qbit_prism_schema_migrations') IS NULL AND to_regclass('qbit_prism_share_hashes') IS NULL", + ) + .fetch_one(&pool) + .await?; + assert!(untouched, "the refused migrate changed the source"); + db.close(Vec::new()).await +} + +/// The value the database declares the pending backfill's fence at, if +/// any (#669). +async fn fence_value(pool: &PgPool) -> Result> { + Ok(sqlx::query_scalar( + "SELECT capability_value FROM qbit_prism_schema_capabilities WHERE capability='share_hash_backfill_pending'", + ) + .fetch_optional(pool) + .await?) +} + +/// Whether the database declares the pending backfill's fence (#669). +async fn fence_declared(pool: &PgPool) -> Result { + Ok(sqlx::query_scalar( + "SELECT EXISTS(SELECT 1 FROM qbit_prism_schema_capabilities WHERE capability='share_hash_backfill_pending' AND capability_value=1)", + ) + .fetch_one(pool) + .await?) +} + +/// Commit the migration transaction of a populated 2.x.x source and leave +/// its backfill pending: the runners' lock is held while it commits, and the +/// waiting `migrate` is then dropped. Returns the connection holding the +/// lock. +async fn pending_backfill(db: &Database, pool: &PgPool) -> Result { + stop_after_the_transaction( + db, + pool, + Ledger::connect(&db.url, "pending".into(), 8, true), + ) + .await +} + +/// Run `migrate` on a populated 2.x.x source until its migration +/// transaction has committed the cursor, then drop it before its online +/// part runs, as a crash or a killed `migrate` would: the runners' lock is +/// held meanwhile. Returns the connection holding the lock. +async fn stop_after_the_transaction( + db: &Database, + pool: &PgPool, + migrate: impl std::future::Future>, +) -> Result { + let mut runners = PgConnection::connect(&db.url).await?; + sqlx::query("SELECT pg_advisory_lock($1,hashtext(current_schema()))") + .bind(RUNNER_LOCK_CLASS) + .execute(&mut runners) + .await?; + let mut migrate = Box::pin(migrate); + let committed = timeout(Duration::from_secs(60), async { + while cursor(pool).await?.is_none() { + sleep(Duration::from_millis(10)).await; + } + Ok::<_, anyhow::Error>(()) + }); + tokio::select! { + result = &mut migrate => bail!("migrate ended before the backfill ran: {:?}", result.err()), + result = committed => result.context("the migration transaction never committed")??, + } + drop(migrate); + Ok(runners) +} + +/// The fence keeps every build before #669 off the database until 2 is +/// recorded, whatever the record says: declared with the cursor, still +/// declared when 2 is recorded by hand and while a resume maps, and removed +/// only with the cursor. That an earlier build refuses the declaration is +/// `a_pending_share_hash_backfill_fences_every_earlier_build`. +#[tokio::test] +async fn migration_002_fences_earlier_builds_until_its_backfill_is_recorded() -> Result<()> { + let Some(db) = Database::open().await? else { + return Ok(()); + }; + let pool = PgPool::connect(&db.url).await?; + two_x::apply_frozen_2x_schema(&pool, SourceState::Pre258).await?; + seed(&pool).await?; + let expected = expected_mapping(&pool, i64::MAX).await?; + let mut runners = pending_backfill(&db, &pool).await?; + assert!( + fence_declared(&pool).await?, + "a pending backfill is not fenced" + ); + // Recorded by hand, 2 neither lifts the fence nor lets this build start. + sqlx::query("INSERT INTO qbit_prism_schema_migrations(version) VALUES(2)") + .execute(&pool) + .await?; + assert!(fence_declared(&pool).await?); + let (next, end) = cursor(&pool).await?.context("the progress table is gone")?; + assert_refused_at(&db.url, next, end).await?; + sqlx::query("DELETE FROM qbit_prism_schema_migrations WHERE version=2") + .execute(&pool) + .await?; + // A resume maps under the fence: seen while a batch waits on an open + // insert. + let blocker = block_late_header(&pool).await?; + sqlx::query("SELECT pg_advisory_unlock($1,hashtext(current_schema()))") + .bind(RUNNER_LOCK_CLASS) + .execute(&mut runners) + .await?; + let mut resume = Box::pin(Ledger::connect(&db.url, "resumed".into(), 8, true)); + tokio::select! { + result = &mut resume => bail!("the resume ended before its batch reached the late header: {:?}", result.err()), + result = batch_waiting(&pool) => { result?; } + } + assert!(fence_declared(&pool).await?, "a resume mapped unfenced"); + blocker.rollback().await?; + let resumed = timeout(Duration::from_secs(120), resume).await??; + // Recorded: the fence went with the cursor. + assert!( + !fence_declared(&pool).await?, + "the fence outlived the backfill" + ); + assert_eq!(cursor(&pool).await?, None); + assert_eq!(mapping(&pool).await?, expected); + assert_eq!(schema_versions(&pool).await?, REQUIRED_SCHEMA_VERSIONS); + runners.close().await?; + db.close(vec![resumed]).await +} + +/// An open insert of the late header, which stops the batch that reaches +/// it until the transaction ends. +async fn block_late_header(pool: &PgPool) -> Result> { + let mut blocker = pool.begin().await?; + sqlx::query( + "INSERT INTO qbit_prism_share_hashes(header_hash,share_id) VALUES($1,'blocker:'||$1)", + ) + .bind(format!("{LATE:064x}")) + .execute(&mut *blocker) + .await?; + Ok(blocker) +} + +/// The backend of a batch waiting on `block_late_header`. +async fn batch_waiting(pool: &PgPool) -> Result { + timeout(Duration::from_secs(60), async { + loop { + let pid: Option = sqlx::query_scalar("SELECT pid FROM pg_stat_activity WHERE datname=current_database() AND pid<>pg_backend_pid() AND query LIKE 'INSERT INTO qbit_prism_share_hashes%' AND wait_event_type='Lock'") + .fetch_optional(pool) + .await?; + if let Some(pid) = pid { + return Ok::<_, anyhow::Error>(pid); + } + sleep(Duration::from_millis(10)).await; + } + }) + .await + .context("no batch waited on the late header")? +} + +/// A backfill an earlier build started carries no fence, and this release +/// does not declare one when it resumes it: an earlier build's runner that +/// passed its capability check before then could finish the backfill after +/// this release's runner stopped, and would leave the fence behind (#669). +/// The test crashes this release's runner mid-batch, then plays the +/// earlier runner, which finishes the backfill knowing nothing of the +/// fence. Nothing is left that refuses the database. +#[tokio::test] +async fn a_crashed_resume_leaves_no_fence_for_an_earlier_runner_to_strand() -> Result<()> { + let Some(db) = Database::open().await? else { + return Ok(()); + }; + let pool = PgPool::connect(&db.url).await?; + two_x::apply_frozen_2x_schema(&pool, SourceState::Pre258).await?; + seed(&pool).await?; + let expected = expected_mapping(&pool, i64::MAX).await?; + let mut runners = pending_backfill(&db, &pool).await?; + // A backfill an earlier build started declares no fence. + sqlx::query( + "DELETE FROM qbit_prism_schema_capabilities WHERE capability='share_hash_backfill_pending'", + ) + .execute(&pool) + .await?; + let blocker = block_late_header(&pool).await?; + sqlx::query("SELECT pg_advisory_unlock($1,hashtext(current_schema()))") + .bind(RUNNER_LOCK_CLASS) + .execute(&mut runners) + .await?; + let mut resume = Box::pin(Ledger::connect(&db.url, "this-build".into(), 8, true)); + let pid = tokio::select! { + result = &mut resume => bail!("the resume ended before its batch reached the late header: {:?}", result.err()), + result = batch_waiting(&pool) => result?, + }; + assert!( + !fence_declared(&pool).await?, + "a resume declared a fence an earlier runner could leave behind" + ); + // This release's runner crashes mid-batch. + sqlx::query("SELECT pg_terminate_backend($1)") + .bind(pid) + .execute(&pool) + .await?; + ensure!( + timeout(Duration::from_secs(30), resume).await?.is_err(), + "the resume succeeded although its batch was killed" + ); + blocker.rollback().await?; + // The earlier runner, queued for the lock, finishes the backfill the way + // #663's runner does: maps the rest, drops the cursor and records 2. + sqlx::query("SELECT pg_advisory_lock($1,hashtext(current_schema()))") + .bind(RUNNER_LOCK_CLASS) + .execute(&mut runners) + .await?; + let (next, end) = cursor(&pool).await?.context("the progress table is gone")?; + let mut tx = runners.begin().await?; + sqlx::query("INSERT INTO qbit_prism_share_hashes(header_hash,share_id) SELECT DISTINCT ON (lower(right(share_id,64))) lower(right(share_id,64)),share_id FROM qbit_share_ledger WHERE accepted AND share_id ~ '[0-9a-fA-F]{64}$' AND share_seq>=$1 AND share_seq<$2 ORDER BY lower(right(share_id,64)),share_seq ON CONFLICT DO NOTHING") + .bind(next) + .bind(end) + .execute(&mut *tx) + .await?; + sqlx::raw_sql("DROP TABLE qbit_prism_share_hash_backfill; INSERT INTO qbit_prism_schema_migrations(version) VALUES(2)") + .execute(&mut *tx) + .await?; + tx.commit().await?; + sqlx::query("SELECT pg_advisory_unlock($1,hashtext(current_schema()))") + .bind(RUNNER_LOCK_CLASS) + .execute(&mut runners) + .await?; + // No fence was left: migrate finishes 013, 017 and 024, and the database + // starts. + assert!( + !fence_declared(&pool).await?, + "a fence outlived the backfill" + ); + let migrated = db.ledger("after").await?; + assert_eq!(mapping(&pool).await?, expected); + assert_eq!(schema_versions(&pool).await?, REQUIRED_SCHEMA_VERSIONS); + let started = Ledger::connect(&db.url, "cold".into(), 8, false).await?; + runners.close().await?; + db.close(vec![migrated, started]).await +} + +/// A newer release may declare the fence at its own value while this +/// release's runner maps between holds of the migration lock (#669). That +/// declaration is the newer release's to remove, with the cursor and the +/// record: the runner reads the fence again under the migration lock +/// before it records 2, and leaves all three. This release's next migrate +/// refuses the newer value, and every start still refuses the pending +/// backfill. +#[tokio::test] +async fn a_runner_leaves_a_newer_releases_fence_with_its_cursor_and_record() -> Result<()> { + let Some(db) = Database::open().await? else { + return Ok(()); + }; + let pool = PgPool::connect(&db.url).await?; + two_x::apply_frozen_2x_schema(&pool, SourceState::Pre258).await?; + seed(&pool).await?; + let mut runners = pending_backfill(&db, &pool).await?; + let blocker = block_late_header(&pool).await?; + sqlx::query("SELECT pg_advisory_unlock($1,hashtext(current_schema()))") + .bind(RUNNER_LOCK_CLASS) + .execute(&mut runners) + .await?; + let mut resume = Box::pin(Ledger::connect(&db.url, "this-build".into(), 8, true)); + tokio::select! { + result = &mut resume => bail!("the resume ended before its batch reached the late header: {:?}", result.err()), + result = batch_waiting(&pool) => { result?; } + } + // A newer release declares the fence at its own value meanwhile: past + // 2, which is this release's. + set_fence(&pool, 3).await?; + blocker.rollback().await?; + let error = timeout(Duration::from_secs(120), resume) + .await? + .err() + .context("the runner recorded 2 over a newer release's fence")?; + let text = format!("{error:#}"); + ensure!( + text.contains("refusing to record migration 2") + && text.contains("share_hash_backfill_pending = 3") + && text.contains("share_hash_backfill_pending 1 to 2 only") + && text.contains("upgrade the server"), + "{text}" + ); + assert_eq!( + fence_value(&pool).await?, + Some(3), + "the runner removed a newer release's fence" + ); + assert!( + cursor(&pool).await?.is_some(), + "the runner dropped the cursor" + ); + assert!(!schema_versions(&pool).await?.contains(&2)); + let error = Ledger::connect(&db.url, "cold".into(), 8, false) + .await + .err() + .context("a start accepted a pending backfill")? + .to_string(); + ensure!( + error.contains("share-hash backfill has not finished"), + "{error}" + ); + let error = db + .ledger("migrate") + .await + .err() + .context("migrate accepted share_hash_backfill_pending = 3")? + .to_string(); + ensure!( + error.contains("share_hash_backfill_pending = 3, but this server understands share_hash_backfill_pending 1 to 2 only") + && error.contains("upgrade the server"), + "{error}" + ); + runners.close().await?; + db.close(Vec::new()).await +} + +/// A runner of an earlier build that is still mapping when this release +/// resumes the backfill holds the runners' lock, finishes, drops the cursor +/// and records 2 without knowing the fence. This release must not have +/// declared one meanwhile, which nothing would remove (#669), and a resume +/// declares none. The test plays the earlier runner, holding the lock while +/// the resume's transaction commits, then finishing the backfill the way +/// #663's runner does. +#[tokio::test] +async fn a_resume_leaves_no_fence_when_an_earlier_runner_finishes_the_backfill() -> Result<()> { + let Some(db) = Database::open().await? else { + return Ok(()); + }; + let pool = PgPool::connect(&db.url).await?; + two_x::apply_frozen_2x_schema(&pool, SourceState::Pre258).await?; + seed(&pool).await?; + let expected = expected_mapping(&pool, i64::MAX).await?; + let mut runners = pending_backfill(&db, &pool).await?; + // A backfill an earlier build started declares no fence. + sqlx::query( + "DELETE FROM qbit_prism_schema_capabilities WHERE capability='share_hash_backfill_pending'", + ) + .execute(&pool) + .await?; + let mut resume = Box::pin(Ledger::connect(&db.url, "this-build".into(), 8, true)); + // The resume's transaction has committed once its runner polls for the + // lock the earlier runner holds. + let queued = timeout(Duration::from_secs(60), async { + loop { + let polling: bool = sqlx::query_scalar("SELECT EXISTS(SELECT 1 FROM pg_stat_activity WHERE datname=current_database() AND pid<>pg_backend_pid() AND query LIKE 'SELECT pg_try_advisory_lock%')") + .fetch_one(&pool) + .await?; + if polling { + return Ok::<_, anyhow::Error>(()); + } + sleep(Duration::from_millis(10)).await; + } + }); + tokio::select! { + result = &mut resume => bail!("the resume ended while the runners' lock was held: {:?}", result.err()), + result = queued => result.context("the resume's runner never queued for the runners' lock")??, + } + assert!( + !fence_declared(&pool).await?, + "the resume's transaction declared a fence an earlier runner would leave behind" + ); + // The earlier runner finishes: maps the rest, drops the cursor and + // records 2, in its last transaction, knowing nothing of the fence. + let (next, end) = cursor(&pool).await?.context("the progress table is gone")?; + let mut tx = runners.begin().await?; + sqlx::query("INSERT INTO qbit_prism_share_hashes(header_hash,share_id) SELECT DISTINCT ON (lower(right(share_id,64))) lower(right(share_id,64)),share_id FROM qbit_share_ledger WHERE accepted AND share_id ~ '[0-9a-fA-F]{64}$' AND share_seq>=$1 AND share_seq<$2 ORDER BY lower(right(share_id,64)),share_seq ON CONFLICT DO NOTHING") + .bind(next) + .bind(end) + .execute(&mut *tx) + .await?; + sqlx::raw_sql("DROP TABLE qbit_prism_share_hash_backfill; INSERT INTO qbit_prism_schema_migrations(version) VALUES(2)") + .execute(&mut *tx) + .await?; + tx.commit().await?; + sqlx::query("SELECT pg_advisory_unlock($1,hashtext(current_schema()))") + .bind(RUNNER_LOCK_CLASS) + .execute(&mut runners) + .await?; + // This release's runner finds the backfill complete, and the database + // starts: no fence was left behind. + let resumed = timeout(Duration::from_secs(120), resume).await??; + assert!( + !fence_declared(&pool).await?, + "a fence outlived the backfill" + ); + assert_eq!(mapping(&pool).await?, expected); + assert_eq!(schema_versions(&pool).await?, REQUIRED_SCHEMA_VERSIONS); + let started = Ledger::connect(&db.url, "cold".into(), 8, false).await?; + runners.close().await?; + db.close(vec![resumed, started]).await +} + +/// A fence whose cursor is gone, which only a hand-dropped cursor leaves, +/// is refused by migrate before any DDL and, once 2 is recorded by hand +/// too, by every start: recording 2 by hand unlocks nothing (#669). That +/// holds at 2, the value that permits serving, as at 1. Past 2 the name is +/// a newer release's declaration, refused as any newer capability is and +/// never offered for deletion as an orphan. +#[tokio::test] +async fn migration_002_refuses_a_backfill_fence_left_without_its_cursor() -> Result<()> { + let Some(db) = Database::open().await? else { + return Ok(()); + }; + let pool = PgPool::connect(&db.url).await?; + two_x::apply_frozen_2x_schema(&pool, SourceState::Pre258).await?; + seed(&pool).await?; + let runners = pending_backfill(&db, &pool).await?; + runners.close().await?; + sqlx::raw_sql("DROP TABLE qbit_prism_share_hash_backfill") + .execute(&pool) + .await?; + let pending: Vec = schema_versions(&pool).await?; + let refused = |error: String| -> Result<()> { + ensure!( + error.contains("declares share_hash_backfill_pending = 1, but migration 2's share-hash backfill cursor qbit_prism_share_hash_backfill is gone"), + "{error}" + ); + Ok(()) + }; + refused( + db.ledger("migrate") + .await + .err() + .context("migrate accepted a fence without its cursor")? + .to_string(), + )?; + assert_eq!(schema_versions(&pool).await?, pending); + assert!(fence_declared(&pool).await?); + // At 2, serving's value, the fence is this release's too, and an orphan. + set_fence(&pool, 2).await?; + let error = db + .ledger("migrate-serving") + .await + .err() + .context("migrate accepted a serving fence without its cursor")? + .to_string(); + ensure!( + error.contains("declares share_hash_backfill_pending = 2, but migration 2's share-hash backfill cursor qbit_prism_share_hash_backfill is gone") + && !error.contains("Restore the full pre-migration backup"), + "{error}" + ); + assert_eq!(schema_versions(&pool).await?, pending); + // Past 2 the name is a newer release's: never an orphan to delete, and + // refused in the order any newer capability is, after the record. + set_fence(&pool, 3).await?; + let error = db + .ledger("migrate-newer") + .await + .err() + .context("migrate accepted a newer release's share_hash_backfill_pending = 3")? + .to_string(); + ensure!( + error.contains("migration 3 is recorded and 2 is not") + && !error.contains("is gone") + && !error.contains("DELETE FROM qbit_prism_schema_capabilities"), + "{error}" + ); + assert_eq!(schema_versions(&pool).await?, pending); + set_fence(&pool, 1).await?; + // Recording 2 by hand as well lets nothing start: 013, 017 and 024 + // never ran behind the pending backfill, and migrate still names the + // fence. + sqlx::query("INSERT INTO qbit_prism_schema_migrations(version) VALUES(2)") + .execute(&pool) + .await?; + let error = Ledger::connect(&db.url, "cold".into(), 8, false) + .await + .err() + .context("a start accepted a fence without its cursor")? + .to_string(); + ensure!(error.contains("missing migration(s) 13, 17, 24"), "{error}"); + refused( + db.ledger("migrate-again") + .await + .err() + .context("migrate accepted a fence without its cursor and a hand-recorded 2")? + .to_string(), + )?; + assert!(fence_declared(&pool).await?); + // With the record past 3 without 2, serving's value is no different: a + // start still refuses the missing 13, 17 and 24, and migrate names the + // orphan. A newer release's value is refused as newer. + set_fence(&pool, 2).await?; + let error = Ledger::connect(&db.url, "cold".into(), 8, false) + .await + .err() + .context("a start accepted a serving fence without its cursor")? + .to_string(); + ensure!(error.contains("missing migration(s) 13, 17, 24"), "{error}"); + let error = db + .ledger("migrate-serving-again") + .await + .err() + .context("migrate accepted a serving fence without its cursor and a hand-recorded 2")? + .to_string(); + ensure!( + error.contains("declares share_hash_backfill_pending = 2, but migration 2's share-hash backfill cursor qbit_prism_share_hash_backfill is gone"), + "{error}" + ); + set_fence(&pool, 3).await?; + let error = db + .ledger("migrate-newer-again") + .await + .err() + .context("migrate accepted a newer release's share_hash_backfill_pending = 3 and a hand-recorded 2")? + .to_string(); + ensure!( + error.contains("share_hash_backfill_pending = 3, but this server understands share_hash_backfill_pending 1 to 2 only") + && error.contains("upgrade the server") + && !error.contains("is gone"), + "{error}" + ); + db.close(Vec::new()).await +} + +/// Declare the fence at `value`, as a newer release might. +async fn set_fence(pool: &PgPool, value: i32) -> Result<()> { + sqlx::query("UPDATE qbit_prism_schema_capabilities SET capability_value=$1 WHERE capability='share_hash_backfill_pending'") + .bind(value) + .execute(pool) + .await?; + Ok(()) +} + +/// A fence declared on a database with every migration recorded and no +/// cursor is refused at every start and every migrate: only the +/// transaction that records 2 removes it, with the cursor, so it can only +/// be left by hand, and legacy shares may be unmapped (#669). So it is at +/// 2, which permits serving only beside its cursor. Past 2 the name is a +/// newer release's declaration: refused as that release's, without the +/// remedy that would delete it. +#[tokio::test] +async fn every_start_refuses_a_backfill_fence_on_a_migrated_database() -> Result<()> { + let Some(db) = Database::open().await? else { + return Ok(()); + }; + let pool = PgPool::connect(&db.url).await?; + let first = db.ledger("first").await?; + sqlx::query("INSERT INTO qbit_prism_schema_capabilities(capability,capability_value) VALUES('share_hash_backfill_pending',1)") + .execute(&pool) + .await?; + for (what, error) in [ + ( + "a start", + Ledger::connect(&db.url, "cold".into(), 8, false) + .await + .err(), + ), + ("migrate", db.ledger("migrate").await.err()), + ] { + let error = error + .with_context(|| format!("{what} accepted a fence without its cursor"))? + .to_string(); + ensure!( + error.contains("declares share_hash_backfill_pending = 1, but migration 2's share-hash backfill cursor qbit_prism_share_hash_backfill is gone"), + "{what}: {error}" + ); + } + set_fence(&pool, 2).await?; + for (what, error) in [ + ( + "a start", + Ledger::connect(&db.url, "cold".into(), 8, false) + .await + .err(), + ), + ("migrate", db.ledger("migrate").await.err()), + ] { + let error = error + .with_context(|| format!("{what} accepted a serving fence without its cursor"))? + .to_string(); + ensure!( + error.contains("declares share_hash_backfill_pending = 2, but migration 2's share-hash backfill cursor qbit_prism_share_hash_backfill is gone") + && !error.contains("Restore the full pre-migration backup"), + "{what}: {error}" + ); + } + set_fence(&pool, 3).await?; + for (what, error) in [ + ( + "a start", + Ledger::connect(&db.url, "cold".into(), 8, false) + .await + .err(), + ), + ("migrate", db.ledger("migrate").await.err()), + ] { + let error = error + .with_context(|| format!("{what} accepted share_hash_backfill_pending = 3"))? + .to_string(); + ensure!( + error.contains("share_hash_backfill_pending = 3, but this server understands share_hash_backfill_pending 1 to 2 only") + && error.contains("upgrade the server") + && !error.contains("is gone"), + "{what}: {error}" + ); + } + assert_eq!(schema_versions(&pool).await?, REQUIRED_SCHEMA_VERSIONS); + db.close(vec![first]).await +} + +/// Rows the recent range's seed writes: twenty headers to a template +/// height, so heights 0 to 3,000. +const HEIGHT_ROWS: i64 = 60_000; +/// The recent range's lowest template height on that seed: the highest an +/// accepted share has, 3,000, less the 1,000 the range reaches. +const RECENT_MIN_HEIGHT: i64 = 2_000; +/// The first share whose own header is at `RECENT_MIN_HEIGHT`: the range's +/// first share would be here without the copies far below it. +const RECENT_FIRST_OWN: i64 = 20 * RECENT_MIN_HEIGHT; + +/// A 2.x.x ledger for the recent range: `seed`'s rule, with template +/// heights. A header commits to its parent, so every copy of a header has +/// its height: twenty headers to a height, whichever row carries a copy. +/// Every 97th row repeats, in upper case, the header of the row 9,001 +/// below it or, where it can, 9,001 above it; so headers just above the +/// range's lowest height have their earliest copies far below the first +/// share whose own header is there. +async fn seed_heights(pool: &PgPool) -> Result<()> { + sqlx::query( + "INSERT INTO qbit_share_ledger(share_seq,share_id,miner_id,payout_order_key,p2mr_program,share_difficulty,network_difficulty,template_height,job_id,job_issued_at,ntime,accepted_at,accepted,reject_reason,writer_id,writer_epoch) \ + SELECT g,i.id,'miner-'||g%5,'miner-'||g%5,decode(repeat('11',32),'hex'),1,100,h.header/20,'job',to_timestamp(1),1,to_timestamp(2),g%1000<>13,CASE WHEN g%1000=13 THEN 'duplicate-share' END,'backfill-test',0 \ + FROM generate_series(1,$1::bigint) g, \ + LATERAL (SELECT CASE WHEN g%97<>0 THEN g WHEN g%2=0 AND g>9001 THEN g-9001 WHEN g+9001<=$1 THEN g+9001 ELSE g-9001 END AS header) h, \ + LATERAL (SELECT CASE \ + WHEN g%97=0 THEN 'repeat-'||g||':'||upper(lpad(to_hex(h.header),64,'0')) \ + WHEN g%1000=7 THEN 'legacy:'||g \ + WHEN g%1000=11 THEN lpad(to_hex(g),64,'0') \ + ELSE 'worker-'||g%5||':'||lpad(to_hex(g),64,'0') END AS id) i \ + WHERE g%89<>0", + ) + .bind(HEIGHT_ROWS) + .execute(pool) + .await?; + sqlx::query("SELECT setval('qbit_share_ledger_share_seq_seq',$1)") + .bind(HEIGHT_ROWS) + .execute(pool) + .await?; + Ok(()) +} + +/// What the recent range maps: 002's rule over the accepted legacy shares +/// at `min_height` and above. +async fn expected_recent_mapping(pool: &PgPool, min_height: i64) -> Result> { + Ok(sqlx::query_as( + "SELECT DISTINCT ON (lower(right(share_id,64))) lower(right(share_id,64)),share_id FROM qbit_share_ledger WHERE accepted AND share_id ~ '[0-9a-fA-F]{64}$' AND template_height>=$1 ORDER BY lower(right(share_id,64)),share_seq", + ) + .bind(min_height) + .fetch_all(pool) + .await?) +} + +/// The mapping of the legacy shares alone, those below `end`. +async fn legacy_mapping(pool: &PgPool, end: i64) -> Result> { + Ok(sqlx::query_as( + "SELECT h.header_hash,h.share_id FROM qbit_prism_share_hashes h WHERE h.share_id IN (SELECT share_id FROM qbit_share_ledger WHERE share_seq<$1) ORDER BY h.header_hash", + ) + .bind(end) + .fetch_all(pool) + .await?) +} + +/// The share ledger's end: one past its highest `share_seq`. +async fn ledger_end(pool: &PgPool) -> Result { + Ok( + sqlx::query_scalar("SELECT COALESCE(max(share_seq),-1)+1 FROM qbit_share_ledger") + .fetch_one(pool) + .await?, + ) +} + +/// What the recent range recorded beside the cursor. +async fn recent_range(pool: &PgPool) -> Result<(Option, Option)> { + Ok(sqlx::query_as( + "SELECT recent_min_height,recent_start_seq FROM qbit_prism_share_hash_backfill WHERE singleton", + ) + .fetch_one(pool) + .await?) +} + +/// Every required migration but 2: what a deferred backfill leaves. +fn all_but_2() -> Vec { + REQUIRED_SCHEMA_VERSIONS + .iter() + .copied() + .filter(|version| *version != 2) + .collect() +} + +/// A 2.x.x source seeded for the recent range and migrated with +/// `--defer-share-hashes`. Returns the source's planned end. +async fn deferred(db: &Database, pool: &PgPool) -> Result<(Ledger, i64)> { + two_x::apply_frozen_2x_schema(pool, SourceState::Pre258).await?; + seed_heights(pool).await?; + let end = ledger_end(pool).await?; + let ledger = Ledger::connect_migrate(&db.url, ShareHashBackfill::Defer).await?; + Ok((ledger, end)) +} + +/// `migrate --defer-share-hashes` maps the recent range and permits +/// serving. Every accepted legacy share within 1,000 template heights of +/// the highest maps its header to the earliest copy, copies far below the +/// first share whose own header is in the range included, and nothing +/// below the range is mapped. The fence is 2, the cursor stays where it was +/// planned, 2 is not recorded, and every start serves the database: a +/// native share repeating a header in the range is refused, and a new one +/// is credited. +#[tokio::test] +async fn migrate_defer_share_hashes_maps_the_recent_range_and_permits_serving() -> Result<()> { + let Some(db) = Database::open().await? else { + return Ok(()); + }; + let pool = PgPool::connect(&db.url).await?; + two_x::apply_frozen_2x_schema(&pool, SourceState::Pre258).await?; + seed_heights(&pool).await?; + let (first, end): (i64, i64) = + sqlx::query_as("SELECT min(share_seq),max(share_seq)+1 FROM qbit_share_ledger") + .fetch_one(&pool) + .await?; + let top: i64 = + sqlx::query_scalar("SELECT max(template_height) FROM qbit_share_ledger WHERE accepted") + .fetch_one(&pool) + .await?; + assert_eq!(top - 1_000, RECENT_MIN_HEIGHT, "the seed moved its heights"); + let expected = expected_recent_mapping(&pool, RECENT_MIN_HEIGHT).await?; + let start_seq: i64 = sqlx::query_scalar( + "SELECT min(share_seq) FROM qbit_share_ledger WHERE accepted AND template_height>=$1", + ) + .bind(RECENT_MIN_HEIGHT) + .fetch_one(&pool) + .await?; + // The range's first share is an early copy of a header in it, far below + // the first share whose own header is there, and wins its header. + let (first_id,): (String,) = + sqlx::query_as("SELECT share_id FROM qbit_share_ledger WHERE share_seq=$1") + .bind(start_seq) + .fetch_one(&pool) + .await?; + ensure!( + start_seq < RECENT_FIRST_OWN && first_id.starts_with("repeat-"), + "the seed's copies no longer straddle the range's first share: {start_seq} {first_id}" + ); + assert!(expected.iter().any(|(_, id)| *id == first_id)); + ensure!(expected.iter().any(|(_, id)| id.len() == 64)); + + let migrated = Ledger::connect_migrate(&db.url, ShareHashBackfill::Defer).await?; + // The range, row for row, and each header at its earliest share, as the + // whole backfill maps it. + let mapped = mapping(&pool).await?; + assert_eq!(mapped, expected); + let whole: HashSet<(String, String)> = expected_mapping(&pool, i64::MAX) + .await? + .into_iter() + .collect(); + assert!(mapped.iter().all(|entry| whole.contains(entry))); + let below: i64 = sqlx::query_scalar("SELECT count(*) FROM qbit_prism_share_hashes h JOIN qbit_share_ledger l ON l.share_id=h.share_id WHERE l.template_height<$1") + .bind(RECENT_MIN_HEIGHT) + .fetch_one(&pool) + .await?; + assert_eq!(below, 0, "a header below the recent range was mapped"); + // Serving is permitted; the rest is pending where it was planned. + assert_eq!(fence_value(&pool).await?, Some(2)); + assert_eq!(cursor(&pool).await?, Some((first, end))); + assert_eq!( + recent_range(&pool).await?, + (Some(RECENT_MIN_HEIGHT), Some(start_seq)) + ); + assert_eq!(schema_versions(&pool).await?, all_but_2()); + assert_eq!( + migrated.pending_share_hash_backfill().await?, + Some((first, end)) + ); + let cold = Ledger::connect(&db.url, "cold".into(), 8, false).await?; + let error = cold + .append(share(55_000), None) + .await + .err() + .context("a native share repeating a header in the recent range was credited")?; + assert!( + format!("{error:#}").contains("header already credited globally"), + "{error:#}" + ); + assert!(cold.append(share(1_000_000), None).await?.inserted); + db.close(vec![migrated, cold]).await +} + +/// A frontend that compose starts with PRISM_POSTGRES_INIT_SCHEMA=1 +/// migrates at its start, as tools and operator connects may. None of them +/// maps a backfill that permits serving: the cursor, the fence and the +/// mapping stay as `--defer-share-hashes` left them, and so does a second +/// deferred `migrate`. +#[tokio::test] +async fn a_frontend_with_init_schema_leaves_a_deferred_backfill_to_migrate() -> Result<()> { + let Some(db) = Database::open().await? else { + return Ok(()); + }; + let pool = PgPool::connect(&db.url).await?; + let (migrated, _) = deferred(&db, &pool).await?; + let pending = cursor(&pool) + .await? + .context("the deferred backfill has no cursor")?; + let mapped = mapping(&pool).await?; + let frontend = db.ledger("frontend").await?; + let tool = Ledger::connect_tool(&db.url, "tool".into(), 2, true, None).await?; + let operator = Ledger::connect_operator(&db.url, true).await?; + let again = Ledger::connect_migrate(&db.url, ShareHashBackfill::Defer).await?; + assert_eq!( + cursor(&pool).await?, + Some(pending), + "a connect moved the cursor" + ); + assert_eq!(fence_value(&pool).await?, Some(2)); + assert_eq!( + mapping(&pool).await?, + mapped, + "a connect mapped legacy shares" + ); + assert_eq!(schema_versions(&pool).await?, all_but_2()); + db.close(vec![migrated, frontend, tool, operator, again]) + .await +} + +/// Plain `migrate` finishes a deferred backfill while frontends append, +/// up to the end its cursor was planned to: the legacy mapping becomes the +/// whole backfill's, 2 is recorded, and the cursor and the fence go. The +/// native shares, appended before and during the run, keep the headers +/// they mapped themselves, and a row past the planned end that no append +/// wrote stays unmapped: the end is never extended over native rows. +#[tokio::test] +async fn migrate_finishes_a_deferred_backfill_up_to_its_planned_end_while_frontends_append( +) -> Result<()> { + let Some(db) = Database::open().await? else { + return Ok(()); + }; + let pool = PgPool::connect(&db.url).await?; + let (migrated, end) = deferred(&db, &pool).await?; + let (first, _) = cursor(&pool) + .await? + .context("the deferred backfill has no cursor")?; + let frontend = db.ledger("frontend").await?; + let mut native = Vec::new(); + for id in 1_000_000..1_000_003 { + native.push(frontend.append(share(id), None).await?.share); + } + // A row past the planned end that no append wrote, so nothing mapped it. + let straggler = "ee".repeat(32); + sqlx::query( + "INSERT INTO qbit_share_ledger(share_id,miner_id,payout_order_key,p2mr_program,share_difficulty,network_difficulty,template_height,job_id,job_issued_at,ntime,accepted_at,accepted,writer_id,writer_epoch) \ + VALUES('straggler:'||$1,'miner-0','miner-0',decode(repeat('11',32),'hex'),1,100,100,'job',to_timestamp(1),1,to_timestamp(2),true,'backfill-test',0)", + ) + .bind(&straggler) + .execute(&pool) + .await?; + // A legacy header below the recent range, held open so the run stops in + // the batch that reaches it. + let held = 30_001_u64; + let mut blocker = pool.begin().await?; + sqlx::query( + "INSERT INTO qbit_prism_share_hashes(header_hash,share_id) VALUES($1,'blocker:'||$1)", + ) + .bind(format!("{held:064x}")) + .execute(&mut *blocker) + .await?; + let mut finish = Box::pin(Ledger::connect_migrate(&db.url, ShareHashBackfill::Finish)); + tokio::select! { + result = &mut finish => bail!("migrate ended before its batch reached the held header: {:?}", result.err()), + result = batch_waiting(&pool) => { result?; } + } + // Frontends keep appending while it maps. + for id in 2_000_000..2_000_003 { + native.push(frontend.append(share(id), None).await?.share); + } + let (next, planned) = cursor(&pool).await?.context("the cursor is gone mid-run")?; + assert_eq!(planned, end, "the run moved its planned end"); + assert!(next > first && next <= held as i64, "cursor {next}"); + blocker.rollback().await?; + let finished = timeout(Duration::from_secs(120), finish).await??; + + assert_eq!(cursor(&pool).await?, None); + assert_eq!(fence_value(&pool).await?, None); + assert_eq!(schema_versions(&pool).await?, REQUIRED_SCHEMA_VERSIONS); + let legacy = expected_mapping(&pool, end).await?; + assert_eq!(legacy_mapping(&pool, end).await?, legacy); + for share in &native { + assert!(share.share_seq as i64 >= end); + let header = share.share_id[share.share_id.len() - 64..].to_owned(); + let holder: Option = + sqlx::query_scalar("SELECT share_id FROM qbit_prism_share_hashes WHERE header_hash=$1") + .bind(&header) + .fetch_optional(&pool) + .await?; + assert_eq!(holder.as_deref(), Some(share.share_id.as_str())); + } + let unmapped: Option = + sqlx::query_scalar("SELECT share_id FROM qbit_prism_share_hashes WHERE header_hash=$1") + .bind(&straggler) + .fetch_optional(&pool) + .await?; + assert_eq!(unmapped, None, "the backfill read past its planned end"); + let total: i64 = sqlx::query_scalar("SELECT count(*) FROM qbit_prism_share_hashes") + .fetch_one(&pool) + .await?; + assert_eq!(total, (legacy.len() + native.len()) as i64); + db.close(vec![migrated, frontend, finished]).await +} + +/// A native share that repeats a legacy header below the recent range was +/// credited twice: nothing mapped the header when it was appended, and the +/// batch that reached the legacy copy met the native share's mapping and +/// said nothing. Plain `migrate` maps every legacy share, then refuses to +/// record 2, naming the native share, and leaves the cursor at its end and +/// the fence at 2: frontends keep serving, and the double credit is +/// reported, not recorded over. Both forms of a legacy ID are found: a +/// worker-scoped one through 2.x's header-suffix index, and the bare +/// 64-digit header through the share ID index. +#[tokio::test] +async fn migrate_refuses_to_record_2_over_a_native_share_repeating_a_legacy_header() -> Result<()> { + for (legacy_seq, bare) in [(1_005_u64, false), (1_011_u64, true)] { + let Some(db) = Database::open().await? else { + return Ok(()); + }; + let pool = PgPool::connect(&db.url).await?; + let (migrated, end) = deferred(&db, &pool).await?; + let (legacy_id, height): (String, i64) = sqlx::query_as( + "SELECT share_id,template_height FROM qbit_share_ledger WHERE share_seq=$1 AND accepted", + ) + .bind(legacy_seq as i64) + .fetch_one(&pool) + .await?; + assert_eq!(legacy_id.len() == 64, bare, "{legacy_id}"); + assert!(height < RECENT_MIN_HEIGHT); + let frontend = db.ledger("frontend").await?; + let repeated = frontend.append(share(legacy_seq), None).await?; + assert!( + repeated.inserted, + "the header below the recent range was refused before the backfill reached it" + ); + frontend.append(share(3_000_000), None).await?; + + let error = Ledger::connect_migrate(&db.url, ShareHashBackfill::Finish) + .await + .err() + .context("migrate recorded 2 over a header credited twice")?; + let text = format!("{error:#}"); + ensure!( + text.contains(&format!( + "refusing to record migration 2: native share {}, at share_seq {}, repeats header {legacy_seq:064x}", + repeated.share.share_id, repeated.share.share_seq + )) && text.contains("so that header was credited twice"), + "{legacy_id}: {text}" + ); + // Every legacy share is mapped; the record, the cursor and the fence + // wait for the reconciliation. + assert_eq!(cursor(&pool).await?, Some((end, end))); + assert_eq!(fence_value(&pool).await?, Some(2)); + assert_eq!(schema_versions(&pool).await?, all_but_2()); + let holder: Option = + sqlx::query_scalar("SELECT share_id FROM qbit_prism_share_hashes WHERE header_hash=$1") + .bind(format!("{legacy_seq:064x}")) + .fetch_optional(&pool) + .await?; + assert_eq!(holder.as_deref(), Some(repeated.share.share_id.as_str())); + let cold = Ledger::connect(&db.url, "cold".into(), 8, false).await?; + let again = Ledger::connect_migrate(&db.url, ShareHashBackfill::Finish) + .await + .err() + .context("a second migrate recorded 2 over a header credited twice")?; + ensure!( + format!("{again:#}").contains("so that header was credited twice"), + "{again:#}" + ); + db.close(vec![migrated, frontend, cold]).await?; + } + Ok(()) +} + +/// Share-archive restore, detach and drop refuse while a backfill is +/// pending, here with serving permitted: a restore maps its rows' headers, +/// the release table holds legacy shares the backfill has still to map, +/// and a lead partition, above the backfill's end, holds native shares the +/// double-credit check reads through the parent, so it is refused too. +/// Once 2 is recorded nothing is refused for the backfill. +#[tokio::test] +async fn share_archive_restore_detach_and_drop_refuse_while_a_backfill_is_pending() -> Result<()> { + let Some(db) = Database::open().await? else { + return Ok(()); + }; + let pool = PgPool::connect(&db.url).await?; + let (migrated, end) = deferred(&db, &pool).await?; + let root = tempfile::tempdir()?; + let manifest = root.path().join("absent").join("manifest.json"); + let options = || archive::PlanOptions { + network_difficulty: "100".into(), + retention_days: 0, + window_multiple: 4, + check_duplicates: false, + }; + let lead: String = sqlx::query_scalar("SELECT partition_name FROM qbit_prism_share_partitions WHERE lower_seq IS NOT NULL ORDER BY lower_seq LIMIT 1") + .fetch_one(&pool) + .await?; + let lead_lower: i64 = sqlx::query_scalar( + "SELECT lower_seq FROM qbit_prism_share_partitions WHERE partition_name=$1", + ) + .bind(&lead) + .fetch_one(&pool) + .await?; + assert!(lead_lower >= end); + let operator = Ledger::connect_operator(&db.url, false).await?; + const PENDING: &str = "while migration 2's share-hash backfill is pending"; + for attach in [false, true] { + let error = archive::restore(&operator, &manifest, root.path(), attach) + .await + .err() + .context("a restore ran while the backfill was pending")? + .to_string(); + ensure!( + error.contains(&format!("refusing to restore a share archive {PENDING}")), + "{error}" + ); + } + let error = archive::detach(&operator, "qbit_share_ledger_p0", &options()) + .await + .err() + .context("the release table was detached while the backfill was pending")? + .to_string(); + ensure!( + error.contains(&format!( + "refusing to detach qbit_share_ledger_p0 {PENDING}" + )) && error.contains("so every partition stays attached until 2 is recorded"), + "{error}" + ); + let error = archive::drop_partition(&operator, "qbit_share_ledger_p0", root.path()) + .await + .err() + .context("the release table was dropped while the backfill was pending")? + .to_string(); + ensure!( + error.contains(&format!("refusing to drop qbit_share_ledger_p0 {PENDING}")), + "{error}" + ); + // A lead partition holds no legacy share, but the double-credit check + // reads its native shares through the parent: refused too. + let error = archive::detach(&operator, &lead, &options()) + .await + .err() + .context("a lead partition was detached while the backfill was pending")? + .to_string(); + ensure!( + error.contains(&format!("refusing to detach {lead} {PENDING}")), + "{error}" + ); + // Recorded, nothing refuses for the backfill. + let finished = Ledger::connect_migrate(&db.url, ShareHashBackfill::Finish).await?; + assert_eq!(cursor(&pool).await?, None); + let error = archive::restore(&operator, &manifest, root.path(), true) + .await + .err() + .context("a restore read an absent manifest")?; + ensure!(!format!("{error:#}").contains(PENDING), "{error:#}"); + let error = archive::detach(&operator, "qbit_share_ledger_p0", &options()) + .await + .err() + .context("an unsealed release table was detached")?; + ensure!(!format!("{error:#}").contains(PENDING), "{error:#}"); + if let Err(error) = archive::detach(&operator, &lead, &options()).await { + ensure!(!format!("{error:#}").contains(PENDING), "{lead}: {error:#}"); + } + db.close(vec![migrated, operator, finished]).await +} + +/// Deferred, 013, 017 and 024 run with the backfill pending, which they +/// never did before: the recent range runs first, while 2.x's +/// template-height index it reads is there, and the cursor table is no +/// object 013's, 017's or 024's checks look at. A later `migrate` of either +/// kind applies nothing over them; plain `migrate` maps the rest after +/// them and records 2 last. +#[tokio::test] +async fn migrations_13_17_and_24_run_with_a_deferred_backfill_pending() -> Result<()> { + let Some(db) = Database::open().await? else { + return Ok(()); + }; + let pool = PgPool::connect(&db.url).await?; + let (migrated, end) = deferred(&db, &pool).await?; + assert_eq!(schema_versions(&pool).await?, all_but_2()); + let pending = cursor(&pool) + .await? + .context("the deferred backfill has no cursor")?; + assert_eq!(pending.1, end); + let (partitioned, trimmed, lane): (bool, bool, bool) = sqlx::query_as( + "SELECT (SELECT relkind='p' FROM pg_class WHERE oid='qbit_share_ledger'::regclass), \ + to_regclass('qbit_share_ledger_template_height_idx') IS NULL \ + AND to_regclass('qbit_share_ledger_accepted_seq_walk_idx') IS NOT NULL \ + AND to_regclass('qbit_share_ledger_accepted_miner_history_idx') IS NOT NULL, \ + to_regclass('qbit_ctv_fanout_artifacts_lane_idx') IS NOT NULL", + ) + .fetch_one(&pool) + .await?; + assert!(partitioned, "017 did not convert the share ledger"); + assert!(trimmed, "013 did not trim the share ledger's indexes"); + assert!(lane, "024 did not build the fanout lane index"); + // The recent range ran: it records its bounds beside the cursor. + assert_eq!(recent_range(&pool).await?.0, Some(RECENT_MIN_HEIGHT)); + // Nothing is applied twice, and a start that migrates applies nothing. + let again = Ledger::connect_migrate(&db.url, ShareHashBackfill::Defer).await?; + let frontend = db.ledger("frontend").await?; + assert_eq!(schema_versions(&pool).await?, all_but_2()); + assert_eq!(cursor(&pool).await?, Some(pending)); + let finished = Ledger::connect_migrate(&db.url, ShareHashBackfill::Finish).await?; + assert_eq!(schema_versions(&pool).await?, REQUIRED_SCHEMA_VERSIONS); + assert_eq!(cursor(&pool).await?, None); + assert_eq!(fence_value(&pool).await?, None); + assert_eq!( + mapping(&pool).await?, + expected_mapping(&pool, i64::MAX).await? + ); + let last: bool = sqlx::query_scalar("SELECT (SELECT applied_at FROM qbit_prism_schema_migrations WHERE version=2) >= (SELECT max(applied_at) FROM qbit_prism_schema_migrations)") + .fetch_one(&pool) + .await?; + assert!(last, "2 is not recorded last"); + let recorded = Ledger::connect_migrate(&db.url, ShareHashBackfill::Finish).await?; + db.close(vec![migrated, again, frontend, finished, recorded]) + .await +} + +/// The W1 cutover's migrate step, `migrate --defer-share-hashes +/// --offline-indexes`, in one run on a populated 2.x.x source. The recent +/// range is mapped and serving permitted, with the rest pending where it was +/// planned; 013 and 024 then build offline with the cursor present, each in +/// the transaction that records it and nothing concurrent; and the start +/// gate admits a frontend that migrates at its start, which leaves the +/// backfill alone and serves. +#[tokio::test] +async fn migrate_defer_share_hashes_with_offline_indexes_permits_serving_in_one_run() -> Result<()> +{ + let Some(db) = Database::open().await? else { + return Ok(()); + }; + let pool = PgPool::connect(&db.url).await?; + two_x::apply_frozen_2x_schema(&pool, SourceState::Pre258).await?; + seed_heights(&pool).await?; + let (first, end): (i64, i64) = + sqlx::query_as("SELECT min(share_seq),max(share_seq)+1 FROM qbit_share_ledger") + .fetch_one(&pool) + .await?; + let expected = expected_recent_mapping(&pool, RECENT_MIN_HEIGHT).await?; + super::index_trim::install_probe(&pool, &db.schema).await?; + let options = MigrateOptions { + share_hashes: ShareHashBackfill::Defer, + index_build: IndexBuildMode::Offline { + workers: 2, + memory_kb: None, + }, + }; + let migrated = Ledger::connect_migrate(&db.url, options).await?; + assert_eq!(mapping(&pool).await?, expected); + assert_eq!(fence_value(&pool).await?, Some(2)); + assert_eq!(cursor(&pool).await?, Some((first, end))); + assert_eq!(recent_range(&pool).await?.0, Some(RECENT_MIN_HEIGHT)); + assert_eq!(schema_versions(&pool).await?, all_but_2()); + super::index_trim::assert_built_offline(&pool).await?; + let frontend = db.ledger("frontend").await?; + assert!(frontend.append(share(1_000_000), None).await?.inserted); + assert_eq!(cursor(&pool).await?, Some((first, end))); + assert_eq!(schema_versions(&pool).await?, all_but_2()); + db.close(vec![migrated, frontend]).await +} + +/// A backfill a build before #669 started declares no fence, and this +/// release never declares one on a resume (#669), so serving cannot be +/// permitted on it: `migrate --defer-share-hashes` refuses it before any +/// DDL and changes nothing, and plain `migrate` finishes it as before. +#[tokio::test] +async fn migrate_defer_share_hashes_refuses_a_backfill_an_earlier_build_started() -> Result<()> { + let Some(db) = Database::open().await? else { + return Ok(()); + }; + let pool = PgPool::connect(&db.url).await?; + two_x::apply_frozen_2x_schema(&pool, SourceState::Pre258).await?; + seed(&pool).await?; + let expected = expected_mapping(&pool, i64::MAX).await?; + let runners = pending_backfill(&db, &pool).await?; + runners.close().await?; + sqlx::query( + "DELETE FROM qbit_prism_schema_capabilities WHERE capability='share_hash_backfill_pending'", + ) + .execute(&pool) + .await?; + let pending = cursor(&pool).await?.context("the progress table is gone")?; + let versions = schema_versions(&pool).await?; + let error = Ledger::connect_migrate(&db.url, ShareHashBackfill::Defer) + .await + .err() + .context("migrate --defer-share-hashes deferred an unfenced backfill")? + .to_string(); + ensure!( + error.contains( + "refusing to defer migration 2's share-hash backfill: a build before #669 started it" + ) && error.contains("Nothing was changed") + && error.contains("without --defer-share-hashes"), + "{error}" + ); + assert_eq!(cursor(&pool).await?, Some(pending)); + assert_eq!(fence_value(&pool).await?, None, "a resume declared a fence"); + assert!(mapping(&pool).await?.is_empty()); + assert_eq!(schema_versions(&pool).await?, versions); + assert_refused_at(&db.url, pending.0, pending.1).await?; + let finished = Ledger::connect_migrate(&db.url, ShareHashBackfill::Finish).await?; + assert_eq!(mapping(&pool).await?, expected); + assert_eq!(cursor(&pool).await?, None); + assert_eq!(fence_value(&pool).await?, None); + assert_eq!(schema_versions(&pool).await?, REQUIRED_SCHEMA_VERSIONS); + db.close(vec![finished]).await +} + +/// A backfill that plain `migrate` started and left part-way, fenced at 1, +/// is deferred on its resume as on a fresh source: the native path runs +/// the recent range in the backfill's slot. The headers the interrupted +/// run mapped stay as they were, including early copies of headers in the +/// range, which the range meets again and keeps; serving is permitted with +/// the cursor where that run left it. +#[tokio::test] +async fn migrate_defer_share_hashes_resumes_a_backfill_plain_migrate_left_part_way() -> Result<()> { + let Some(db) = Database::open().await? else { + return Ok(()); + }; + let pool = PgPool::connect(&db.url).await?; + two_x::apply_frozen_2x_schema(&pool, SourceState::Pre258).await?; + seed_heights(&pool).await?; + let mut runners = pending_backfill(&db, &pool).await?; + let (first, end) = cursor(&pool).await?.context("the progress table is gone")?; + assert_eq!(fence_value(&pool).await?, Some(1)); + // The interrupted run's batches: below `left`, which lies past the + // recent range's first share, so the two overlap. + let left = 35_000_i64; + let mut tx = runners.begin().await?; + sqlx::query("INSERT INTO qbit_prism_share_hashes(header_hash,share_id) SELECT DISTINCT ON (lower(right(share_id,64))) lower(right(share_id,64)),share_id FROM qbit_share_ledger WHERE accepted AND share_id ~ '[0-9a-fA-F]{64}$' AND share_seq>=$1 AND share_seq<$2 ORDER BY lower(right(share_id,64)),share_seq ON CONFLICT DO NOTHING") + .bind(first) + .bind(left) + .execute(&mut *tx) + .await?; + sqlx::query("UPDATE qbit_prism_share_hash_backfill SET next_seq=$1 WHERE singleton") + .bind(left) + .execute(&mut *tx) + .await?; + tx.commit().await?; + runners.close().await?; + assert_refused_at(&db.url, left, end).await?; + let prefix: HashSet<(String, String)> = + expected_mapping(&pool, left).await?.into_iter().collect(); + let recent: HashSet<(String, String)> = expected_recent_mapping(&pool, RECENT_MIN_HEIGHT) + .await? + .into_iter() + .collect(); + ensure!( + prefix.intersection(&recent).next().is_some(), + "the interrupted run mapped no header of the recent range" + ); + + let resumed = Ledger::connect_migrate(&db.url, ShareHashBackfill::Defer).await?; + let mapped = mapping(&pool).await?; + let expected: HashSet<(String, String)> = prefix.union(&recent).cloned().collect(); + assert_eq!(mapped.len(), expected.len(), "a header is mapped twice"); + assert_eq!(mapped.into_iter().collect::>(), expected); + assert_eq!(fence_value(&pool).await?, Some(2)); + assert_eq!(cursor(&pool).await?, Some((left, end))); + assert_eq!(schema_versions(&pool).await?, all_but_2()); + let cold = Ledger::connect(&db.url, "cold".into(), 8, false).await?; + db.close(vec![resumed, cold]).await +} + +/// The ledger's share_seq sequence, as `(last_value, is_called)`. +async fn share_seq_state(pool: &PgPool) -> Result<(i64, bool)> { + Ok( + sqlx::query_as("SELECT last_value,is_called FROM qbit_share_ledger_share_seq_seq") + .fetch_one(pool) + .await?, + ) +} + +/// The share_seq sequence's `last_value` as `pg_sequences` lists it, the +/// value the cutover's evidence compares: NULL while it is uncalled. +async fn listed_last_value(pool: &PgPool) -> Result> { + Ok(sqlx::query_scalar("SELECT last_value FROM pg_sequences WHERE schemaname=current_schema() AND sequencename='qbit_share_ledger_share_seq_seq'") + .fetch_one(pool) + .await?) +} + +/// Wait until a runner polls for the runners' lock the test holds: a +/// `migrate` whose transaction has committed, its online part queued. +async fn runner_queued(pool: &PgPool) -> Result<()> { + timeout(Duration::from_secs(60), async { + loop { + let polling: bool = sqlx::query_scalar("SELECT EXISTS(SELECT 1 FROM pg_stat_activity WHERE datname=current_database() AND pid<>pg_backend_pid() AND query LIKE 'SELECT pg_try_advisory_lock%')") + .fetch_one(pool) + .await?; + if polling { + return Ok::<_, anyhow::Error>(()); + } + sleep(Duration::from_millis(10)).await; + } + }) + .await + .context("no runner queued for the runners' lock")? +} + +/// Ensure a connect was refused for a deferred backfill whose recent range +/// is not mapped, naming the deferred run's retry. +fn refused_for_the_deferred_range(what: &str, error: Option) -> Result<()> { + let text = format!( + "{:#}", + error.with_context(|| format!( + "{what} accepted a deferred backfill whose recent range is not mapped" + ))? + ); + ensure!( + text.contains("`qbit-prism-server migrate --defer-share-hashes` claimed it and stopped before its recent range was mapped") + && text.contains("Run `qbit-prism-server migrate --defer-share-hashes` again"), + "{what}: {text}" + ); + Ok(()) +} + +/// A `migrate --defer-share-hashes` that stopped after its migration +/// transaction, before its recent range permitted serving, leaves its claim +/// on the cursor at fence 1. A frontend that compose starts with +/// PRISM_POSTGRES_INIT_SCHEMA=1 then runs neither the whole backfill nor +/// 013, which drops the index the range reads: it refuses the database, +/// naming the deferred run's retry, as every start does, and so do a +/// tool's and an operator's connect that migrate. The retry then maps the +/// range and permits serving as a first run does. +#[tokio::test] +async fn a_deferred_migrate_stopped_before_its_recent_range_is_left_to_the_operator() -> Result<()> +{ + let Some(db) = Database::open().await? else { + return Ok(()); + }; + let pool = PgPool::connect(&db.url).await?; + two_x::apply_frozen_2x_schema(&pool, SourceState::Pre258).await?; + seed_heights(&pool).await?; + let expected = expected_recent_mapping(&pool, RECENT_MIN_HEIGHT).await?; + let runners = stop_after_the_transaction( + &db, + &pool, + Ledger::connect_migrate(&db.url, ShareHashBackfill::Defer), + ) + .await?; + runners.close().await?; + let pending = cursor(&pool).await?.context("the progress table is gone")?; + assert_eq!(fence_value(&pool).await?, Some(1)); + let claimed: bool = + sqlx::query_scalar("SELECT deferred_at IS NOT NULL FROM qbit_prism_share_hash_backfill") + .fetch_one(&pool) + .await?; + assert!( + claimed, + "the deferred run committed no claim with its cursor" + ); + let versions = schema_versions(&pool).await?; + + refused_for_the_deferred_range( + "a frontend that migrates", + timeout(Duration::from_secs(120), db.ledger("frontend")) + .await? + .err(), + )?; + refused_for_the_deferred_range( + "a tool that migrates", + Ledger::connect_tool(&db.url, "tool".into(), 2, true, None) + .await + .err(), + )?; + refused_for_the_deferred_range( + "an operator connect that migrates", + Ledger::connect_operator(&db.url, true).await.err(), + )?; + refused_for_the_deferred_range( + "a start", + Ledger::connect(&db.url, "cold".into(), 8, false) + .await + .err(), + )?; + // None of them mapped a share or applied a migration, so 2.x's + // template-height index, which serves the range, is still there. + assert_eq!( + cursor(&pool).await?, + Some(pending), + "a connect moved the cursor" + ); + assert!( + mapping(&pool).await?.is_empty(), + "a connect mapped legacy shares" + ); + assert_eq!( + schema_versions(&pool).await?, + versions, + "a connect applied a migration" + ); + let indexed: bool = sqlx::query_scalar( + "SELECT to_regclass('qbit_share_ledger_template_height_idx') IS NOT NULL", + ) + .fetch_one(&pool) + .await?; + assert!(indexed, "013 ran ahead of the recent range"); + + // The retry completes as a first run does. + let retried = Ledger::connect_migrate(&db.url, ShareHashBackfill::Defer).await?; + assert_eq!(mapping(&pool).await?, expected); + assert_eq!(fence_value(&pool).await?, Some(2)); + assert_eq!(cursor(&pool).await?, Some(pending)); + assert_eq!(schema_versions(&pool).await?, all_but_2()); + let frontend = db.ledger("frontend").await?; + db.close(vec![retried, frontend]).await +} + +/// A connect that planned the whole backfill at fence 1 before a deferred +/// `migrate` claimed it, and then waited for the runners' lock, reads the +/// claim again once it holds the lock: it refuses before it maps anything, +/// and before 013, as one that found the claim at its migration does. +#[tokio::test] +async fn a_connect_queued_behind_a_deferred_claim_maps_nothing() -> Result<()> { + let Some(db) = Database::open().await? else { + return Ok(()); + }; + let pool = PgPool::connect(&db.url).await?; + two_x::apply_frozen_2x_schema(&pool, SourceState::Pre258).await?; + seed_heights(&pool).await?; + let mut runners = PgConnection::connect(&db.url).await?; + sqlx::query("SELECT pg_advisory_lock($1,hashtext(current_schema()))") + .bind(RUNNER_LOCK_CLASS) + .execute(&mut runners) + .await?; + // A frontend migrates the source, without a claim: its backfill is + // planned, and waits for the lock. + let mut frontend = Box::pin(db.ledger("frontend")); + tokio::select! { + result = &mut frontend => bail!("the frontend's connect ended while the runners' lock was held: {:?}", result.err()), + result = runner_queued(&pool) => result?, + } + let pending = cursor(&pool).await?.context("the progress table is gone")?; + assert_eq!(fence_value(&pool).await?, Some(1)); + let versions = schema_versions(&pool).await?; + // A deferred `migrate` claims it meanwhile, as its migration + // transaction does. + sqlx::raw_sql("ALTER TABLE qbit_prism_share_hash_backfill ADD COLUMN deferred_at timestamptz; UPDATE qbit_prism_share_hash_backfill SET deferred_at=clock_timestamp()") + .execute(&pool) + .await?; + sqlx::query("SELECT pg_advisory_unlock($1,hashtext(current_schema()))") + .bind(RUNNER_LOCK_CLASS) + .execute(&mut runners) + .await?; + refused_for_the_deferred_range( + "a frontend queued behind the claim", + timeout(Duration::from_secs(120), frontend).await?.err(), + )?; + assert_eq!(cursor(&pool).await?, Some(pending), "the frontend mapped"); + assert!(mapping(&pool).await?.is_empty(), "the frontend mapped"); + assert_eq!( + schema_versions(&pool).await?, + versions, + "the frontend applied an online migration" + ); + runners.close().await?; + db.close(Vec::new()).await +} + +/// Plain `migrate` planned at fence 1 runs the backfill ahead of 017. A +/// deferred `migrate` can raise the fence to 2 while that run waits for the +/// runners' lock, and stop before 017. Recording 2 under fence 2 needs +/// 017's conversion bound, so the run would refuse at its record after +/// every batch, hours on a production ledger: it refuses before it maps +/// anything instead. A rerun plans from fence 2, applies 013, 017 and 024 +/// first, then maps the rest and records 2. +#[tokio::test] +async fn plain_migrate_refuses_before_mapping_when_the_fence_reached_2_while_it_waited( +) -> Result<()> { + let Some(db) = Database::open().await? else { + return Ok(()); + }; + let pool = PgPool::connect(&db.url).await?; + two_x::apply_frozen_2x_schema(&pool, SourceState::Pre258).await?; + seed(&pool).await?; + let expected = expected_mapping(&pool, i64::MAX).await?; + let mut runners = PgConnection::connect(&db.url).await?; + sqlx::query("SELECT pg_advisory_lock($1,hashtext(current_schema()))") + .bind(RUNNER_LOCK_CLASS) + .execute(&mut runners) + .await?; + let mut finish = Box::pin(Ledger::connect_migrate(&db.url, ShareHashBackfill::Finish)); + tokio::select! { + result = &mut finish => bail!("migrate ended while the runners' lock was held: {:?}", result.err()), + result = runner_queued(&pool) => result?, + } + let pending = cursor(&pool).await?.context("the progress table is gone")?; + assert_eq!(fence_value(&pool).await?, Some(1)); + // A deferred `migrate` raised the fence meanwhile and stopped before + // 017. + set_fence(&pool, 2).await?; + sqlx::query("SELECT pg_advisory_unlock($1,hashtext(current_schema()))") + .bind(RUNNER_LOCK_CLASS) + .execute(&mut runners) + .await?; + let error = timeout(Duration::from_secs(120), finish) + .await? + .err() + .context("plain migrate recorded 2 without 017's conversion bound")?; + let text = format!("{error:#}"); + ensure!( + text.contains( + "refusing to finish migration 2's share-hash backfill before mapping anything" + ) && text + .contains("The share-hash fence changed while this run waited for the runner lock") + && text.contains("rerun `qbit-prism-server migrate`"), + "{text}" + ); + assert_eq!( + cursor(&pool).await?, + Some(pending), + "the run moved the cursor" + ); + assert!( + mapping(&pool).await?.is_empty(), + "the run mapped legacy shares" + ); + let finished = Ledger::connect_migrate(&db.url, ShareHashBackfill::Finish).await?; + assert_eq!(mapping(&pool).await?, expected); + assert_eq!(cursor(&pool).await?, None); + assert_eq!(fence_value(&pool).await?, None); + assert_eq!(schema_versions(&pool).await?, REQUIRED_SCHEMA_VERSIONS); + runners.close().await?; + db.close(vec![finished]).await +} + +/// The double-credit check reads the native shares in chunks of +/// `share_seq`, from the backfill's end up to the last share appended +/// before it starts, each chunk a short statement of its own. Here the +/// native shares span a million `share_seq`, two hundred times the largest +/// chunk at the default throttle, and only the last one repeats a legacy +/// header: plain +/// `migrate` still refuses to record 2, naming it, and leaves the cursor +/// and the fence as they are. +#[tokio::test] +async fn migrate_refuses_a_double_credit_in_the_last_chunk_of_its_check() -> Result<()> { + let Some(db) = Database::open().await? else { + return Ok(()); + }; + let pool = PgPool::connect(&db.url).await?; + let (migrated, end) = deferred(&db, &pool).await?; + let frontend = db.ledger("frontend").await?; + // A legacy header below the recent range, so nothing refuses its + // repeat at the append. + let legacy_seq = 1_005_u64; + // A clean share at the end, one half-way up, and the double credit + // last; the sequence skips the share_seq between them. + let mut appended = Vec::new(); + for (id, after) in [ + (1_000_000, None), + (1_000_001, Some(end + 500_000)), + (legacy_seq, Some(end + 1_000_000)), + ] { + if let Some(after) = after { + sqlx::query( + "SELECT setval(pg_get_serial_sequence('qbit_share_ledger','share_seq'),$1)", + ) + .bind(after) + .execute(&pool) + .await?; + } + let result = frontend.append(share(id), None).await?; + assert!(result.inserted, "share {id} was refused"); + appended.push(result.share); + } + assert!(appended[0].share_seq as i64 >= end); + let repeated = appended.pop().context("nothing was appended")?; + assert!(repeated.share_seq as i64 > end + 1_000_000); + + let error = Ledger::connect_migrate(&db.url, ShareHashBackfill::Finish) + .await + .err() + .context("migrate recorded 2 over a header credited twice in its check's last chunk")?; + let text = format!("{error:#}"); + ensure!( + text.contains(&format!( + "refusing to record migration 2: native share {}, at share_seq {}, repeats header {legacy_seq:064x}", + repeated.share_id, repeated.share_seq + )), + "{text}" + ); + assert_eq!(cursor(&pool).await?, Some((end, end))); + assert_eq!(fence_value(&pool).await?, Some(2)); + assert_eq!(schema_versions(&pool).await?, all_but_2()); + db.close(vec![migrated, frontend]).await +} + +/// Native shares draw their share_seq from the ledger's sequence, and plain +/// `migrate` takes every row below the frozen end for a legacy share. A +/// source whose sequence lags behind its rows, called or not, as explicit +/// inserts or a restore leave it, has the sequence moved up to the end by +/// the transaction that permits serving: the next native share lands at +/// the end, not in a gap below it. +#[tokio::test] +async fn migrate_defer_share_hashes_moves_a_lagging_share_seq_sequence_up_to_the_frozen_end( +) -> Result<()> { + // A share_seq the seed leaves free, every 89th, far below its end. + const GAP: i64 = 89 * 300; + for is_called in [true, false] { + let Some(db) = Database::open().await? else { + return Ok(()); + }; + let pool = PgPool::connect(&db.url).await?; + two_x::apply_frozen_2x_schema(&pool, SourceState::Pre258).await?; + seed_heights(&pool).await?; + let end = ledger_end(&pool).await?; + // Behind the ledger's rows: GAP is the next value either way. + let last_value = if is_called { GAP - 1 } else { GAP }; + sqlx::query("SELECT setval('qbit_share_ledger_share_seq_seq',$1,$2)") + .bind(last_value) + .bind(is_called) + .execute(&pool) + .await?; + assert_eq!(share_seq_state(&pool).await?, (last_value, is_called)); + + let migrated = Ledger::connect_migrate(&db.url, ShareHashBackfill::Defer).await?; + assert_eq!(fence_value(&pool).await?, Some(2)); + assert_eq!( + share_seq_state(&pool).await?, + (end - 1, true), + "is_called {is_called}: the sequence was not moved up to the end" + ); + let frontend = db.ledger("frontend").await?; + let native = frontend.append(share(1_000_000), None).await?; + assert!(native.inserted); + assert_eq!( + native.share.share_seq as i64, end, + "is_called {is_called}: the native share did not land at the frozen end" + ); + db.close(vec![migrated, frontend]).await?; + } + Ok(()) +} + +/// The converse: a sequence that hands out the end next, or a value past +/// it, as a promoted physical copy's does, is left exactly as the source +/// had it, `last_value` and `is_called` alike. The cutover's evidence +/// compares `pg_sequences.last_value` before and after `migrate`. +#[tokio::test] +async fn migrate_defer_share_hashes_leaves_a_share_seq_sequence_at_or_past_the_end_untouched( +) -> Result<()> { + for ahead in [None, Some(32)] { + let Some(db) = Database::open().await? else { + return Ok(()); + }; + let pool = PgPool::connect(&db.url).await?; + two_x::apply_frozen_2x_schema(&pool, SourceState::Pre258).await?; + seed_heights(&pool).await?; + let end = ledger_end(&pool).await?; + if let Some(ahead) = ahead { + // Past the end, and not called since. + sqlx::query("SELECT setval('qbit_share_ledger_share_seq_seq',$1,false)") + .bind(end + ahead) + .execute(&pool) + .await?; + } + let before = share_seq_state(&pool).await?; + // The seed leaves it called at the ledger's last row, which hands + // out the end next. + assert_eq!( + before, + ahead.map_or((end - 1, true), |ahead| (end + ahead, false)) + ); + let listed = listed_last_value(&pool).await?; + + let migrated = Ledger::connect_migrate(&db.url, ShareHashBackfill::Defer).await?; + assert_eq!(fence_value(&pool).await?, Some(2)); + assert_eq!( + share_seq_state(&pool).await?, + before, + "the deferred run wrote to a sequence that did not lag" + ); + assert_eq!(listed_last_value(&pool).await?, listed); + db.close(vec![migrated]).await?; + } + Ok(()) +} + +/// A partition's bounds as the catalog records them, `(lower_seq, +/// upper_seq)`; the release table's lower bound is MINVALUE, NULL. +async fn partition_bounds(pool: &PgPool, partition: &str) -> Result<(Option, i64)> { + Ok(sqlx::query_as( + "SELECT lower_seq,upper_seq FROM qbit_prism_share_partitions WHERE partition_name=$1", + ) + .bind(partition) + .fetch_one(pool) + .await?) +} + +/// Native-shaped shares at `[first, last]`, written straight into the +/// ledger as the archive tests place them: accepted, recent, at +/// `difficulty`, with IDs that end in no legacy share's header. +async fn insert_native_rows(pool: &PgPool, first: i64, last: i64, difficulty: i64) -> Result<()> { + sqlx::query( + "INSERT INTO qbit_share_ledger(share_seq,share_id,miner_id,payout_order_key,p2mr_program,share_difficulty,network_difficulty,template_height,job_id,job_issued_at,ntime,accepted_at,accepted,writer_id,writer_epoch) \ + SELECT i,'native:'||i,'miner-0','miner-0',decode(repeat('11',32),'hex'),$3::text::numeric,100,3001,'job',statement_timestamp(),1,statement_timestamp(),true,'native-test',0 \ + FROM generate_series($1::bigint,$2::bigint) g(i)", + ) + .bind(first) + .bind(last) + .bind(difficulty.to_string()) + .execute(pool) + .await?; + Ok(()) +} + +/// A native partition above the backfill's end that is ready to leave the +/// ledger in every other respect (archived after the release table, +/// sealed, verified, below the online horizon) stays attached while the +/// backfill is pending: the double-credit check reads native shares +/// through the parent, and a partition that left would hide its shares +/// from it. Detach is refused, and so is drop, the step that cannot be +/// undone, of the partition taken off the parent by hand all the same. +/// Once plain `migrate` has recorded 2, the same partition is detached and +/// dropped. +#[tokio::test] +async fn an_archive_ready_native_partition_stays_attached_until_2_is_recorded() -> Result<()> { + const P0: &str = "qbit_share_ledger_p0"; + const P1: &str = "qbit_share_ledger_p1"; + const P2: &str = "qbit_share_ledger_p2"; + let Some(db) = Database::open().await? else { + return Ok(()); + }; + let pool = PgPool::connect(&db.url).await?; + let (migrated, end) = deferred(&db, &pool).await?; + let (p1_lower, p1_upper) = partition_bounds(&pool, P1).await?; + let p1_lower = p1_lower.context("p1 has no lower bound")?; + let p2_lower = partition_bounds(&pool, P2) + .await? + .0 + .context("p2 has no lower bound")?; + assert!(p1_lower >= end, "p1 starts below the backfill's end"); + // Native shares in p1, and later ones in p2 that hold the payout window, + // the sequence past p1 and every share folded into the rollups: p1 lies + // below the online horizon. + insert_native_rows(&pool, p1_lower, p1_lower + 4, 5).await?; + insert_native_rows(&pool, p2_lower, p2_lower + 4, 1_000_000).await?; + sqlx::query("SELECT setval(pg_get_serial_sequence('qbit_share_ledger','share_seq'),$1)") + .bind(p2_lower + 4) + .execute(&pool) + .await?; + while qbit_prism_server::rollups::advance(&pool, 50_000) + .await? + .scanned + > 0 + {} + let root = tempfile::tempdir()?; + let operator = Ledger::connect_operator(&db.url, false).await?; + // The archive chain starts at the release table. + archive::archive(&operator, P0, root.path(), false, "operator").await?; + archive::verify(&operator, P0, root.path()).await?; + archive::archive(&operator, P1, root.path(), false, "operator").await?; + archive::seal(&operator, P1).await?; + archive::verify(&operator, P1, root.path()).await?; + let options = archive::PlanOptions { + network_difficulty: "100".into(), + retention_days: 0, + window_multiple: 4, + check_duplicates: false, + }; + let report = archive::plan(&operator, &options).await?; + let blockers = &report + .partitions + .iter() + .find(|entry| entry.record.partition_name == P1) + .context("p1 is not in the plan")? + .blockers; + ensure!( + blockers.is_empty(), + "{P1} is not ready to leave in its own right: {blockers:?}" + ); + let refused = |step: &str, error: Option| -> Result<()> { + let text = format!( + "{:#}", + error.with_context(|| format!( + "{step} took {P1} off the ledger while the backfill was pending" + ))? + ); + ensure!( + text.contains(&format!( + "refusing to {step} {P1} while migration 2's share-hash backfill is pending" + )) && text.contains("so every partition stays attached until 2 is recorded") + && text.contains("Run `qbit-prism-server backfill-share-hashes`, or plain `qbit-prism-server migrate`, to finish the backfill"), + "{text}" + ); + Ok(()) + }; + refused( + "detach", + archive::detach(&operator, P1, &options).await.err(), + )?; + // Taken off the parent by hand all the same, it is not dropped either, + // and is put back. + sqlx::raw_sql(&format!("ALTER TABLE qbit_share_ledger DETACH PARTITION {P1}; UPDATE qbit_prism_share_partitions SET state='detached',detached_at=clock_timestamp() WHERE partition_name='{P1}'")) + .execute(&pool) + .await?; + refused( + "drop", + archive::drop_partition(&operator, P1, root.path()) + .await + .err(), + )?; + let kept: bool = sqlx::query_scalar("SELECT to_regclass($1) IS NOT NULL") + .bind(P1) + .fetch_one(&pool) + .await?; + assert!(kept, "the refused drop removed {P1}"); + sqlx::raw_sql(&format!("ALTER TABLE qbit_share_ledger ATTACH PARTITION {P1} FOR VALUES FROM ({p1_lower}) TO ({p1_upper}); UPDATE qbit_prism_share_partitions SET state='attached',detached_at=NULL WHERE partition_name='{P1}'")) + .execute(&pool) + .await?; + + // Recorded, the same partition leaves. + let finished = Ledger::connect_migrate(&db.url, ShareHashBackfill::Finish).await?; + assert_eq!(cursor(&pool).await?, None); + assert_eq!(schema_versions(&pool).await?, REQUIRED_SCHEMA_VERSIONS); + let detached = archive::detach(&operator, P1, &options).await?; + ensure!(detached["action"] == "detached", "{detached}"); + let dropped = archive::drop_partition(&operator, P1, root.path()).await?; + ensure!(dropped["relation_dropped"] == true, "{dropped}"); + db.close(vec![migrated, operator, finished]).await +} + +/// `qbit-prism-server backfill-share-hashes` with `args`, against `db` +/// alone: no setting of the test runner's reaches it, but `env`. +async fn backfill_cli( + db: &Database, + args: &[&str], + env: &[(&str, &str)], +) -> Result { + let mut command = tokio::process::Command::new(env!("CARGO_BIN_EXE_qbit-prism-server")); + for (key, _) in std::env::vars().filter(|(key, _)| { + key.starts_with("PRISM_") || key.starts_with("QBIT_") || key == "RUST_LOG" + }) { + command.env_remove(key); + } + command + .arg("backfill-share-hashes") + .args(args) + .kill_on_drop(true) + .env("PRISM_DATABASE_URL", &db.url) + .env("PRISM_RUNTIME_WORKERS", "2") + .env("RUST_LOG", "info") + .env("NO_COLOR", "1") + .envs(env.iter().copied()); + Ok(timeout(Duration::from_secs(300), command.output()).await??) +} + +/// Log every batch the backfill commits: the cursor's move, the statement +/// timeout the batch ran under, and when it moved. A test's own trigger on +/// the cursor table, which goes with the table when 2 is recorded. +async fn log_batches(pool: &PgPool) -> Result<()> { + sqlx::raw_sql( + "CREATE TABLE test_share_hash_batches(from_seq bigint NOT NULL, to_seq bigint NOT NULL, statement_timeout text NOT NULL, at timestamptz NOT NULL); \ + CREATE FUNCTION test_log_share_hash_batch() RETURNS trigger LANGUAGE plpgsql AS $$ BEGIN \ + INSERT INTO test_share_hash_batches VALUES (OLD.next_seq, NEW.next_seq, current_setting('statement_timeout'), clock_timestamp()); RETURN NEW; END $$; \ + CREATE TRIGGER test_log_share_hash_batch AFTER UPDATE ON qbit_prism_share_hash_backfill FOR EACH ROW EXECUTE FUNCTION test_log_share_hash_batch();", + ) + .execute(pool) + .await?; + Ok(()) +} + +/// The batches `log_batches` logged, in order: `[from, to)`, the +/// statement timeout, and the seconds since the first. +async fn logged_batches(pool: &PgPool) -> Result> { + Ok(sqlx::query_as( + "SELECT from_seq,to_seq,statement_timeout,extract(epoch FROM at - min(at) OVER ())::float8 FROM test_share_hash_batches ORDER BY at", + ) + .fetch_all(pool) + .await?) +} + +/// Make every statement that maps headers, a batch's or an append's, run +/// `body` first: a statement trigger of the test's own on the mapping. +async fn slow_mapping(pool: &PgPool, body: &str) -> Result<()> { + sqlx::raw_sql(&format!( + "CREATE OR REPLACE FUNCTION test_slow_share_hashes() RETURNS trigger LANGUAGE plpgsql AS $$ BEGIN {body} RETURN NULL; END $$; \ + CREATE OR REPLACE TRIGGER test_slow_share_hashes BEFORE INSERT ON qbit_prism_share_hashes FOR EACH STATEMENT EXECUTE FUNCTION test_slow_share_hashes();" + )) + .execute(pool) + .await?; + Ok(()) +} + +async fn drop_slow_mapping(pool: &PgPool) -> Result<()> { + sqlx::raw_sql("DROP TRIGGER test_slow_share_hashes ON qbit_prism_share_hashes; DROP FUNCTION test_slow_share_hashes();") + .execute(pool) + .await?; + Ok(()) +} + +/// `backfill-share-hashes` maps the rest of a deferred backfill while a +/// frontend appends, up to the end the cursor was planned to, and records +/// 2: the legacy mapping is the whole backfill's, the native shares, before +/// and during the run, keep the headers they mapped themselves, and the +/// cursor and the fence are gone. It prints what it did. A second run has +/// nothing left to map, and says so as a success, changing nothing. +#[tokio::test] +async fn backfill_share_hashes_maps_the_rest_while_frontends_append_and_records_2() -> Result<()> { + let Some(db) = Database::open().await? else { + return Ok(()); + }; + let pool = PgPool::connect(&db.url).await?; + let (migrated, end) = deferred(&db, &pool).await?; + let (first, _) = cursor(&pool) + .await? + .context("the deferred backfill has no cursor")?; + let pending = Ledger::inspect_share_hash_backfill(&db.url) + .await? + .context("self-check's read found no pending backfill")?; + assert_eq!( + ( + pending.fence, + pending.start_seq, + pending.next_seq, + pending.end_seq, + pending.remaining_seqs, + pending.recent_min_height + ), + ( + Some(2), + first, + first, + end, + end - first, + Some(RECENT_MIN_HEIGHT) + ) + ); + let frontend = db.ledger("frontend").await?; + let mut native = Vec::new(); + for id in 1_000_000..1_000_003 { + native.push(frontend.append(share(id), None).await?.share); + } + // A legacy header below the recent range, held open so the run waits in + // the batch that reaches it while the frontend appends. + let mut blocker = pool.begin().await?; + sqlx::query( + "INSERT INTO qbit_prism_share_hashes(header_hash,share_id) VALUES($1,'blocker:'||$1)", + ) + .bind(format!("{:064x}", 30_001_u64)) + .execute(&mut *blocker) + .await?; + // The batch that reaches it waits under the throttle's cap, 5 s, which + // outlasts the appends below; a batch cut there would be tried again at + // half its size. The pool's lock timeout, also 5 s, must not cut it + // first: a lock timeout stops a batch rather than halving it. + let mut run = Box::pin(backfill_cli( + &db, + &[ + "--max-batch", + "2000", + "--statement-timeout-ms", + "5000", + "--duty-cycle", + "0.5", + ], + &[("PRISM_DATABASE_LOCK_TIMEOUT_MS", "60000")], + )); + tokio::select! { + output = &mut run => bail!("backfill-share-hashes ended before its batch reached the held header: {:?}", output?), + waiting = batch_waiting(&pool) => { waiting?; } + } + for id in 2_000_000..2_000_003 { + native.push(frontend.append(share(id), None).await?.share); + } + let (next, planned) = cursor(&pool).await?.context("the cursor is gone mid-run")?; + assert_eq!(planned, end, "the run moved its planned end"); + assert!(next > first && next <= 30_001, "cursor {next}"); + blocker.rollback().await?; + let output = run.await?; + let stderr = String::from_utf8_lossy(&output.stderr); + ensure!( + output.status.success(), + "backfill-share-hashes failed: {stderr}" + ); + ensure!( + stderr.contains("share-hash backfill batches throttled") + && stderr.contains("duty_cycle=0.5"), + "{stderr}" + ); + let printed: serde_json::Value = serde_json::from_slice(&output.stdout)?; + assert_eq!(printed["recorded"], true); + assert_eq!(printed["already_complete"], false); + assert_eq!(printed["next_seq"], first); + assert_eq!(printed["end_seq"], end); + assert_eq!(printed["seqs"], end - first); + assert!(printed["mapped"].as_u64() > Some(0), "{printed}"); + + assert_eq!(cursor(&pool).await?, None); + assert_eq!(fence_value(&pool).await?, None); + assert_eq!(schema_versions(&pool).await?, REQUIRED_SCHEMA_VERSIONS); + assert_eq!(Ledger::inspect_share_hash_backfill(&db.url).await?, None); + let legacy = expected_mapping(&pool, end).await?; + assert_eq!(legacy_mapping(&pool, end).await?, legacy); + for share in &native { + assert!(share.share_seq as i64 >= end); + let holder: Option = + sqlx::query_scalar("SELECT share_id FROM qbit_prism_share_hashes WHERE header_hash=$1") + .bind(&share.share_id[share.share_id.len() - 64..]) + .fetch_optional(&pool) + .await?; + assert_eq!(holder.as_deref(), Some(share.share_id.as_str())); + } + let total: i64 = sqlx::query_scalar("SELECT count(*) FROM qbit_prism_share_hashes") + .fetch_one(&pool) + .await?; + assert_eq!(total, (legacy.len() + native.len()) as i64); + // A legacy header below the recent range is credited to its legacy share. + let error = frontend + .append(share(1_005), None) + .await + .err() + .context("a native share repeating a legacy header was credited")?; + assert!( + format!("{error:#}").contains("header already credited globally"), + "{error:#}" + ); + + // A retry, say after a lost reply, finds 2 recorded: nothing to map, and + // a success. + let again = backfill_cli(&db, &[], &[]).await?; + ensure!( + again.status.success(), + "{}", + String::from_utf8_lossy(&again.stderr) + ); + let printed: serde_json::Value = serde_json::from_slice(&again.stdout)?; + assert_eq!( + ( + &printed["recorded"], + &printed["already_complete"], + &printed["mapped"], + &printed["seqs"] + ), + ( + &serde_json::json!(true), + &serde_json::json!(true), + &serde_json::json!(0), + &serde_json::json!(0) + ), + "{printed}" + ); + assert_eq!(schema_versions(&pool).await?, REQUIRED_SCHEMA_VERSIONS); + db.close(vec![migrated, frontend]).await +} + +/// A native share that repeats a legacy header below the recent range was +/// credited twice, which `backfill-share-hashes` refuses to record 2 over, +/// as plain `migrate` does: every legacy share is mapped, the cursor stands +/// at its end, the fence stays at 2 and 2 unrecorded. +#[tokio::test] +async fn backfill_share_hashes_refuses_to_record_2_over_a_native_share_repeating_a_legacy_header( +) -> Result<()> { + let Some(db) = Database::open().await? else { + return Ok(()); + }; + let pool = PgPool::connect(&db.url).await?; + let (migrated, end) = deferred(&db, &pool).await?; + let frontend = db.ledger("frontend").await?; + let repeated = frontend.append(share(1_005), None).await?; + assert!( + repeated.inserted, + "the header below the recent range was refused" + ); + let output = backfill_cli(&db, &[], &[]).await?; + let stderr = String::from_utf8_lossy(&output.stderr); + ensure!( + !output.status.success() + && stderr.contains(&format!( + "refusing to record migration 2: native share {}, at share_seq {}, repeats header {:064x}", + repeated.share.share_id, repeated.share.share_seq, 1_005 + )) + && stderr.contains("so that header was credited twice"), + "{stderr}" + ); + assert_eq!(cursor(&pool).await?, Some((end, end))); + assert_eq!(fence_value(&pool).await?, Some(2)); + assert_eq!(schema_versions(&pool).await?, all_but_2()); + db.close(vec![migrated, frontend]).await +} + +/// `backfill-share-hashes` maps only a backfill that permits serving. At +/// fence 1 every start refuses the database, the command's included, and +/// says what to run instead; nothing is mapped. Under the runners' lock it +/// checks again, so a fence that changed after the start gate is refused +/// too, naming what to run: at 1, without a fence (a backfill a build +/// before #669 started), and at a newer release's value. So is a backfill +/// whose record could not be written, 017's conversion bound missing, as +/// plain `migrate` refuses it: before any batch. A database whose backfill +/// has finished has nothing to map, which is a success. +#[tokio::test] +async fn backfill_share_hashes_refuses_a_backfill_that_does_not_permit_serving() -> Result<()> { + let Some(db) = Database::open().await? else { + return Ok(()); + }; + let pool = PgPool::connect(&db.url).await?; + two_x::apply_frozen_2x_schema(&pool, SourceState::Pre258).await?; + seed_heights(&pool).await?; + assert_eq!(Ledger::inspect_share_hash_backfill(&db.url).await?, None); + let runners = pending_backfill(&db, &pool).await?; + runners.close().await?; + let pending = cursor(&pool).await?.context("the progress table is gone")?; + assert_eq!(fence_value(&pool).await?, Some(1)); + let output = backfill_cli(&db, &[], &[]).await?; + let stderr = String::from_utf8_lossy(&output.stderr); + ensure!( + !output.status.success() + && stderr.contains( + "database is not ready: migration 2's share-hash backfill has not finished" + ) + && stderr.contains("Run `qbit-prism-server migrate`") + && stderr.contains("`qbit-prism-server migrate --defer-share-hashes`"), + "{stderr}" + ); + assert_eq!(cursor(&pool).await?, Some(pending)); + assert!(mapping(&pool).await?.is_empty(), "a refused run mapped"); + + let deferred = Ledger::connect_migrate(&db.url, ShareHashBackfill::Defer).await?; + let pending = cursor(&pool).await?.context("the progress table is gone")?; + let mapped = mapping(&pool).await?; + let operator = Ledger::connect_operator(&db.url, false).await?; + for (value, refusal) in [ + (Some(1), "refusing to backfill share hashes: migration 2's share-hash backfill does not permit serving (share_hash_backfill_pending = 1)"), + (None, "refusing to backfill share hashes: a build before #669 started migration 2's share-hash backfill"), + (Some(3), "refusing to backfill share hashes: a newer PRISM release declared share_hash_backfill_pending = 3, but this server understands share_hash_backfill_pending 1 to 2 only"), + ] { + match value { + Some(value) => set_fence(&pool, value).await?, + None => { + sqlx::query("DELETE FROM qbit_prism_schema_capabilities WHERE capability='share_hash_backfill_pending'") + .execute(&pool) + .await?; + } + } + let error = operator + .backfill_share_hashes(&ShareHashThrottle::default()) + .await + .err() + .with_context(|| format!("backfill-share-hashes mapped at fence {value:?}"))? + .to_string(); + ensure!(error.contains(refusal), "{value:?}: {error}"); + assert_eq!(cursor(&pool).await?, Some(pending), "{value:?}"); + assert_eq!(mapping(&pool).await?, mapped, "{value:?}"); + assert_eq!(schema_versions(&pool).await?, all_but_2(), "{value:?}"); + if value.is_none() { + sqlx::query("INSERT INTO qbit_prism_schema_capabilities(capability,capability_value) VALUES('share_hash_backfill_pending',2)") + .execute(&pool) + .await?; + } else { + set_fence(&pool, 2).await?; + } + } + let bound: i64 = sqlx::query_scalar( + "SELECT conversion_bound FROM qbit_prism_share_partitioning WHERE singleton", + ) + .fetch_one(&pool) + .await?; + sqlx::query("UPDATE qbit_prism_share_partitioning SET conversion_bound=NULL WHERE singleton") + .execute(&pool) + .await?; + let error = operator + .backfill_share_hashes(&ShareHashThrottle::default()) + .await + .err() + .context("backfill-share-hashes mapped without 017's conversion bound")? + .to_string(); + ensure!( + error.contains( + "refusing to finish migration 2's share-hash backfill before mapping anything" + ), + "{error}" + ); + assert_eq!(cursor(&pool).await?, Some(pending)); + assert_eq!(mapping(&pool).await?, mapped); + sqlx::query("UPDATE qbit_prism_share_partitioning SET conversion_bound=$1 WHERE singleton") + .bind(bound) + .execute(&pool) + .await?; + let finished = operator + .backfill_share_hashes(&ShareHashThrottle::default()) + .await?; + assert_eq!(finished.range, Some(pending)); + assert!(!finished.already_complete()); + assert_eq!(schema_versions(&pool).await?, REQUIRED_SCHEMA_VERSIONS); + // Finished, there is nothing to map: a success, not a refusal. + let again = operator + .backfill_share_hashes(&ShareHashThrottle::default()) + .await?; + assert!(again.already_complete(), "{again:?}"); + assert_eq!((again.mapped, again.range), (0, None)); + db.close(vec![deferred, operator]).await +} + +/// While frontends serve, every batch is one statement in its own +/// transaction under the throttle's statement timeout, and the run rests +/// between batches so that they take at most the duty cycle's share of the +/// time: at a duty cycle of 0.25, three times as long as each batch took. +/// A test trigger makes each batch's statement take 150 ms, so batches are +/// at least 600 ms apart. Without the rest they would be about 150 ms apart, +/// and without the per-batch timeout each would run under the pool's. +#[tokio::test] +async fn backfill_share_hashes_runs_each_batch_under_its_statement_timeout_and_rests_between_batches( +) -> Result<()> { + let Some(db) = Database::open().await? else { + return Ok(()); + }; + let pool = PgPool::connect(&db.url).await?; + let (migrated, end) = deferred(&db, &pool).await?; + let (first, _) = cursor(&pool).await?.context("no cursor")?; + log_batches(&pool).await?; + slow_mapping(&pool, "PERFORM pg_sleep(0.15);").await?; + let operator = Ledger::connect_operator(&db.url, false).await?; + let throttle = ShareHashThrottle::new(5_000, Duration::from_millis(1_500), 0.25)?; + let finished = operator.backfill_share_hashes(&throttle).await?; + drop_slow_mapping(&pool).await?; + assert_eq!(finished.range, Some((first, end))); + let batches = logged_batches(&pool).await?; + ensure!(batches.len() >= 10, "{batches:?}"); + assert_eq!(batches.first().map(|batch| batch.0), Some(first)); + assert_eq!(batches.last().map(|batch| batch.1), Some(end)); + for (index, (from, to, statement_timeout, _)) in batches.iter().enumerate() { + ensure!( + statement_timeout == "1500ms", + "batch {index} ran under statement_timeout {statement_timeout}" + ); + ensure!( + to - from <= 5_000, + "batch {index} covered {} share_seq", + to - from + ); + if index > 0 { + ensure!(*from == batches[index - 1].1, "{batches:?}"); + } + } + for pair in batches.windows(2) { + let apart = pair[1].3 - pair[0].3; + ensure!( + apart >= 0.5, + "batches {apart:.3} s apart: no rest of three times a 150 ms batch: {batches:?}" + ); + } + assert_eq!(schema_versions(&pool).await?, REQUIRED_SCHEMA_VERSIONS); + assert_eq!( + mapping(&pool).await?, + expected_mapping(&pool, i64::MAX).await? + ); + db.close(vec![migrated, operator]).await +} + +/// A batch that outlasts the throttle's statement timeout is rolled back and +/// tried again at half its size, down to the smallest batch, which fails the +/// run instead, every earlier batch kept and the remedy named. A test +/// trigger holds the mapping statement for a second: always, then for the +/// first two attempts only, so the run's first batch commits at 1,000 +/// `share_seq`, a quarter of 4,000, and no batch exceeds 4,000. +#[tokio::test] +async fn a_batch_past_the_throttles_statement_timeout_is_retried_at_half_its_size() -> Result<()> { + let Some(db) = Database::open().await? else { + return Ok(()); + }; + let pool = PgPool::connect(&db.url).await?; + let (migrated, end) = deferred(&db, &pool).await?; + let (first, _) = cursor(&pool).await?.context("no cursor")?; + log_batches(&pool).await?; + let operator = Ledger::connect_operator(&db.url, false).await?; + let throttle = ShareHashThrottle::new(4_000, Duration::from_millis(300), 1.0)?; + slow_mapping(&pool, "PERFORM pg_sleep(1);").await?; + let error = operator + .backfill_share_hashes(&throttle) + .await + .err() + .context("a run whose every batch outlasted its timeout succeeded")?; + let text = format!("{error:#}"); + ensure!( + text.contains(&format!("Resume from share_seq {first} with `qbit-prism-server backfill-share-hashes`")) + && text.contains("even a batch of 1000 share_seq, the smallest, outlasted its 300 ms statement timeout, so run it with a smaller --max-batch, or else a larger --statement-timeout-ms, at most 5000") + && text.contains("canceling statement due to statement timeout"), + "{text}" + ); + assert_eq!(cursor(&pool).await?, Some((first, end))); + assert!(logged_batches(&pool).await?.is_empty()); + sqlx::raw_sql("CREATE SEQUENCE test_slow_attempts") + .execute(&pool) + .await?; + slow_mapping( + &pool, + "IF nextval('test_slow_attempts') <= 2 THEN PERFORM pg_sleep(1); END IF;", + ) + .await?; + let finished = operator.backfill_share_hashes(&throttle).await?; + drop_slow_mapping(&pool).await?; + assert_eq!(finished.range, Some((first, end))); + let sizes: Vec = logged_batches(&pool) + .await? + .iter() + .map(|(from, to, _, _)| to - from) + .collect(); + // Halved twice, from 4,000 to the smallest batch; then it grows as + // batches run fast, within the throttle's size. + ensure!(sizes.first() == Some(&1_000), "{sizes:?}"); + ensure!(sizes.iter().all(|size| *size <= 4_000), "{sizes:?}"); + assert_eq!(sizes.iter().sum::(), end - first); + assert_eq!(schema_versions(&pool).await?, REQUIRED_SCHEMA_VERSIONS); + db.close(vec![migrated, operator]).await +} + +/// The double-credit check reads the native shares on the serving primary +/// too, so it keeps to the backfill's throttle as the batches do: each of +/// its statements in a transaction of its own under the throttle's +/// timeout. Here every batch is done and a table lock holds the check's +/// first read: it is cancelled at the throttle's 300 ms, as a statement +/// timeout, not after the pool's 5 s lock timeout, and the run stops before +/// recording 2, naming what to run. +#[tokio::test] +async fn the_double_credit_check_runs_under_the_throttles_statement_timeout() -> Result<()> { + let Some(db) = Database::open().await? else { + return Ok(()); + }; + let pool = PgPool::connect(&db.url).await?; + let (migrated, end) = deferred(&db, &pool).await?; + let frontend = db.ledger("frontend").await?; + assert!(frontend.append(share(1_000_000), None).await?.inserted); + // Every batch done: the run goes straight to the check. + sqlx::query("UPDATE qbit_prism_share_hash_backfill SET next_seq=end_seq WHERE singleton") + .execute(&pool) + .await?; + let operator = Ledger::connect_operator(&db.url, false).await?; + let mut holder = pool.begin().await?; + sqlx::query("LOCK TABLE qbit_share_ledger IN ACCESS EXCLUSIVE MODE") + .execute(&mut *holder) + .await?; + let throttle = ShareHashThrottle::new(5_000, Duration::from_millis(300), 0.5)?; + let started = std::time::Instant::now(); + let error = operator.backfill_share_hashes(&throttle).await.err(); + let took = started.elapsed(); + holder.rollback().await?; + let text = format!( + "{:#}", + error.context("the check read the ledger through a table lock")? + ); + ensure!( + text.contains("finding the last native share") + && text.contains("run `qbit-prism-server backfill-share-hashes` again") + && text.contains("canceling statement due to statement timeout"), + "{text}" + ); + ensure!( + took < Duration::from_secs(4), + "the check waited {took:?} for the lock: not the throttle's 300 ms" + ); + assert_eq!(cursor(&pool).await?, Some((end, end))); + assert_eq!(fence_value(&pool).await?, Some(2)); + assert_eq!(schema_versions(&pool).await?, all_but_2()); + db.close(vec![migrated, frontend, operator]).await +} + +/// The migration lock's key (`ledger::MIGRATION_LOCK`), a transaction-scoped +/// advisory lock in the ledger's database. +const MIGRATION_LOCK: i64 = 0x5052_4953_4d00_0001; + +/// The default throttle but for its statement timeout, the cap, 5 s, which +/// outlasts the record's 2 s lock timeout: each lock wait is then cut by the +/// lock timeout alone. At the defaults both are 2 s, and either may cut it. +fn record_throttle(attempts: u32, backoff: Duration) -> Result { + ShareHashThrottle::new( + ShareHashThrottle::DEFAULT_MAX_BATCH, + Duration::from_millis(ShareHashThrottle::MAX_STATEMENT_TIMEOUT_MS), + ShareHashThrottle::DEFAULT_DUTY_CYCLE, + )? + .with_record_attempts(attempts, backoff) +} + +/// Every attempt of the transaction that records 2 that is waiting on a +/// lock, as `(backend, its statement's start)`, polled until `until` +/// resolves: what tells one attempt from the next. +async fn record_attempts_waiting(pool: &PgPool, statement: &str) -> Result> { + Ok(sqlx::query_as( + "SELECT pid,extract(epoch FROM query_start)::float8 FROM pg_stat_activity WHERE datname=current_database() AND pid<>pg_backend_pid() AND wait_event_type='Lock' AND query LIKE $1", + ) + .bind(format!("{statement}%")) + .fetch_all(pool) + .await?) +} + +/// Poll `statement`'s lock waits until `attempts` distinct ones were seen, +/// and return the longest any of them had waited when seen: an attempt +/// that waits at most its lock timeout is seen again and again, each time +/// with a new start. +async fn watch_record_attempts( + pool: &PgPool, + statement: &str, + attempts: usize, +) -> Result { + let mut starts = std::collections::BTreeSet::new(); + let mut longest = 0.0_f64; + timeout(Duration::from_secs(60), async { + while starts.len() < attempts { + let now: f64 = sqlx::query_scalar("SELECT extract(epoch FROM clock_timestamp())::float8") + .fetch_one(pool) + .await?; + for (_, started) in record_attempts_waiting(pool, statement).await? { + starts.insert(started.to_bits()); + longest = longest.max(now - started); + } + sleep(Duration::from_millis(50)).await; + } + Ok::<_, anyhow::Error>(()) + }) + .await + .with_context(|| { + format!( + "saw {} attempts of `{statement}` waiting, not {attempts}: one that waits without a lock timeout is never seen again", + starts.len() + ) + })??; + Ok(Duration::from_secs_f64(longest)) +} + +/// The transaction that records 2 while frontends serve never waits long +/// for a lock (#738). A recovery evidence export reads the cursor table in +/// its snapshot and holds that lock until it commits; here an open +/// transaction that read the cursor stands in for one. The record's `DROP +/// TABLE` waits at most its two-second lock timeout, rolls back, holding +/// nothing, and is tried again after a backoff; once the reader commits, +/// the next attempt records 2. Nothing hangs. The statement timeout is the +/// cap, so that only the lock timeout can cut a wait under 4 s. +#[tokio::test] +async fn the_record_of_2_waits_out_a_reader_of_the_cursor_in_short_attempts() -> Result<()> { + let Some(db) = Database::open().await? else { + return Ok(()); + }; + let pool = PgPool::connect(&db.url).await?; + let (migrated, end) = deferred(&db, &pool).await?; + let mut reader = pool.begin().await?; + sqlx::query("SELECT count(*) FROM qbit_prism_share_hash_backfill") + .execute(&mut *reader) + .await?; + let operator = Ledger::connect_operator(&db.url, false).await?; + let throttle = record_throttle(20, Duration::from_millis(200))?; + let mut run = Box::pin(operator.backfill_share_hashes(&throttle)); + let longest = tokio::select! { + finished = &mut run => bail!("recorded 2 through a reader of the cursor: {finished:?}"), + watched = watch_record_attempts(&pool, "DROP TABLE qbit_prism_share_hash_backfill", 2) => watched?, + }; + ensure!( + longest < Duration::from_secs(4), + "an attempt waited {longest:?} for the cursor's lock" + ); + assert_eq!(cursor(&pool).await?, Some((end, end))); + assert_eq!(schema_versions(&pool).await?, all_but_2()); + reader.commit().await?; + let finished = timeout(Duration::from_secs(60), run).await??; + assert_eq!(finished.range.map(|(_, to)| to), Some(end)); + assert_eq!(cursor(&pool).await?, None); + assert_eq!(schema_versions(&pool).await?, REQUIRED_SCHEMA_VERSIONS); + db.close(vec![migrated, operator]).await +} + +/// The same for the migration lock, which every migrating start takes: the +/// record waits at most its lock timeout an attempt for it too, the +/// advisory lock's wait included. Held through every attempt, the run stops +/// cleanly, naming what held it and what to run, with every batch done, the +/// cursor at its end, the fence at 2 and 2 unrecorded. A rerun waits again, +/// and records 2 once the lock is released. Under the cap's statement +/// timeout, as above. +#[tokio::test] +async fn the_record_of_2_stops_cleanly_behind_a_held_migration_lock_and_records_once_released( +) -> Result<()> { + let Some(db) = Database::open().await? else { + return Ok(()); + }; + let pool = PgPool::connect(&db.url).await?; + let (migrated, end) = deferred(&db, &pool).await?; + let mut holder = pool.begin().await?; + sqlx::query("SELECT pg_advisory_xact_lock($1)") + .bind(MIGRATION_LOCK) + .execute(&mut *holder) + .await?; + let operator = Ledger::connect_operator(&db.url, false).await?; + let impatient = record_throttle(3, Duration::from_millis(100))?; + let started = std::time::Instant::now(); + let error = timeout( + Duration::from_secs(60), + operator.backfill_share_hashes(&impatient), + ) + .await + .context("the record waited for the migration lock without a bound")? + .err() + .context("recorded 2 under a held migration lock")?; + let took = started.elapsed(); + let text = format!("{error:#}"); + ensure!( + text.contains("refusing to wait any longer to record migration 2: something held MIGRATION_LOCK or the share-hash cursor through 3 attempts") + && text.contains("Run `qbit-prism-server backfill-share-hashes` again once it is released") + && text.contains("canceling statement due to lock timeout"), + "{text}" + ); + // Three attempts of two seconds each, and their backoffs. + ensure!(took >= Duration::from_secs(6), "stopped after {took:?}"); + assert_eq!(cursor(&pool).await?, Some((end, end))); + assert_eq!(fence_value(&pool).await?, Some(2)); + assert_eq!(schema_versions(&pool).await?, all_but_2()); + + let patient = record_throttle(20, Duration::from_millis(200))?; + let mut run = Box::pin(operator.backfill_share_hashes(&patient)); + let longest = tokio::select! { + finished = &mut run => bail!("recorded 2 under a held migration lock: {finished:?}"), + watched = watch_record_attempts(&pool, "SELECT pg_advisory_xact_lock", 2) => watched?, + }; + ensure!( + longest < Duration::from_secs(4), + "an attempt waited {longest:?} for the migration lock" + ); + holder.rollback().await?; + let finished = timeout(Duration::from_secs(60), run).await??; + assert!(!finished.already_complete()); + assert_eq!(cursor(&pool).await?, None); + assert_eq!(fence_value(&pool).await?, None); + assert_eq!(schema_versions(&pool).await?, REQUIRED_SCHEMA_VERSIONS); + db.close(vec![migrated, operator]).await +} + +/// Plain `migrate` maps the rest of a backfill that permits serving at the +/// default throttle, by `backfill-share-hashes`'s path: batches of at most +/// 5,000 `share_seq`, each under a 2 s statement timeout, resting as long as +/// each took. Never rc.4's batches, which double to 50,000 under the pool's +/// timeout, back to back (#738). The same with `--offline-indexes`, which +/// changes only how 013 and 024 build (#745). +#[tokio::test] +async fn plain_migrate_maps_a_deferred_backfill_at_the_default_throttle() -> Result<()> { + for index_build in [ + IndexBuildMode::Concurrent, + IndexBuildMode::Offline { + workers: 2, + memory_kb: None, + }, + ] { + plain_migrate_at_the_default_throttle(MigrateOptions { + share_hashes: ShareHashBackfill::Finish, + index_build, + }) + .await + .with_context(|| format!("plain migrate with {index_build:?} index builds"))?; + } + Ok(()) +} + +async fn plain_migrate_at_the_default_throttle(options: MigrateOptions) -> Result<()> { + let Some(db) = Database::open().await? else { + return Ok(()); + }; + let pool = PgPool::connect(&db.url).await?; + let (migrated, end) = deferred(&db, &pool).await?; + log_batches(&pool).await?; + slow_mapping(&pool, "PERFORM pg_sleep(0.15);").await?; + let finished = Ledger::connect_migrate(&db.url, options).await?; + drop_slow_mapping(&pool).await?; + let batches = logged_batches(&pool).await?; + ensure!(batches.len() >= 10, "{batches:?}"); + assert_eq!(batches.last().map(|batch| batch.1), Some(end)); + for (index, (from, to, statement_timeout, _)) in batches.iter().enumerate() { + ensure!( + statement_timeout == "2s" && to - from <= 5_000, + "batch {index}: {} share_seq under statement_timeout {statement_timeout}", + to - from + ); + } + for pair in batches.windows(2) { + let apart = pair[1].3 - pair[0].3; + ensure!( + apart >= 0.25, + "batches {apart:.3} s apart: no rest as long as a 150 ms batch: {batches:?}" + ); + } + assert_eq!(schema_versions(&pool).await?, REQUIRED_SCHEMA_VERSIONS); + assert_eq!(cursor(&pool).await?, None); + db.close(vec![migrated, finished]).await +} + +/// `self-check` reports a deferred backfill, its fence and its cursor, and +/// warns that `backfill-share-hashes` has still to run, without failing on +/// it: here it fails on the node it cannot reach, and only on that. Once 2 +/// is recorded, the field and the warning are gone. One it cannot read is +/// `unknown`, with why, never taken for done: a cursor beside a missing +/// capability table, which holds the fence, or a view under the cursor's +/// name. +#[tokio::test] +async fn self_check_reports_a_deferred_backfill_without_failing_on_it() -> Result<()> { + let Some(db) = Database::open().await? else { + return Ok(()); + }; + let pool = PgPool::connect(&db.url).await?; + let (migrated, end) = deferred(&db, &pool).await?; + let (first, _) = cursor(&pool).await?.context("no cursor")?; + let self_check = || async { + let mut command = tokio::process::Command::new(env!("CARGO_BIN_EXE_qbit-prism-server")); + for (key, _) in std::env::vars().filter(|(key, _)| { + key.starts_with("PRISM_") || key.starts_with("QBIT_") || key == "RUST_LOG" + }) { + command.env_remove(key); + } + command + .arg("self-check") + .kill_on_drop(true) + .env("PRISM_RUNTIME_WORKERS", "2") + .env("PRISM_INSTANCE_ID", "self-check-backfill") + .env("PRISM_DATABASE_URL", &db.url) + .env("QBIT_RPC_URL", "http://127.0.0.1:0/") + .env("PRISM_RPC_TIMEOUT_SECONDS", "1") + .env("QBIT_CHAIN", "regtest") + .env("PRISM_ALLOW_TEST_SIGNING_SEEDS", "1"); + for (name, value) in pool_fee::ZERO_BPS_POOL_FEE { + command.env(name, value); + } + let output = timeout(Duration::from_secs(60), command.output()).await??; + let report: serde_json::Value = serde_json::from_slice(&output.stdout) + .with_context(|| String::from_utf8_lossy(&output.stderr).into_owned())?; + Ok::<_, anyhow::Error>(( + output.status.success(), + report, + String::from_utf8_lossy(&output.stderr).into_owned(), + )) + }; + let (ok, report, stderr) = self_check().await?; + let backfill = &report["share_hash_backfill"]; + ensure!( + !ok && report["ok"] == false + && backfill["state"] == "pending" + && backfill["fence"] == 2 + && backfill["start_seq"] == first + && backfill["next_seq"] == first + && backfill["end_seq"] == end + && backfill["remaining_seqs"] == end - first + && backfill["recent_min_height"] == RECENT_MIN_HEIGHT + && backfill["updated_at"].is_string(), + "{report}" + ); + ensure!( + stderr.contains(&format!("WARNING: migration 2's share-hash backfill is pending (share_hash_backfill_pending = 2): the legacy shares from share_seq {first} up to {end}")) + && stderr.contains("Run `qbit-prism-server backfill-share-hashes` while frontends serve") + && stderr.contains("qbit RPC getblockhash transport failed"), + "{stderr}" + ); + // The read misses the capability table, not the cursor: that is no + // finished backfill, and the error is reported (Codex on #746). + sqlx::raw_sql("ALTER TABLE qbit_prism_schema_capabilities RENAME TO test_capabilities_aside") + .execute(&pool) + .await?; + let missing = self_check().await; + sqlx::raw_sql("ALTER TABLE test_capabilities_aside RENAME TO qbit_prism_schema_capabilities") + .execute(&pool) + .await?; + let (ok, report, stderr) = missing?; + let backfill = &report["share_hash_backfill"]; + ensure!( + !ok && backfill["state"] == "unknown" + && backfill["error"].as_str().is_some_and(|error| error + .contains("relation \"qbit_prism_schema_capabilities\" does not exist")), + "{report}" + ); + ensure!( + stderr.contains("WARNING: migration 2's share-hash backfill could not be read"), + "{stderr}" + ); + let operator = Ledger::connect_operator(&db.url, false).await?; + operator + .backfill_share_hashes(&ShareHashThrottle::default()) + .await?; + let (_, report, stderr) = self_check().await?; + ensure!(report.get("share_hash_backfill").is_none(), "{report}"); + ensure!( + !stderr.contains("share-hash backfill is pending"), + "{stderr}" + ); + // A backfill self-check cannot read is never reported as done: here a + // view holds the cursor's name, which every start refuses too. + sqlx::raw_sql("CREATE VIEW qbit_prism_share_hash_backfill AS SELECT 1 AS singleton") + .execute(&pool) + .await?; + let (ok, report, stderr) = self_check().await?; + sqlx::raw_sql("DROP VIEW qbit_prism_share_hash_backfill") + .execute(&pool) + .await?; + let backfill = &report["share_hash_backfill"]; + ensure!( + !ok && backfill["state"] == "unknown" + && backfill["error"].as_str().is_some_and(|error| error + .contains("a view named qbit_prism_share_hash_backfill holds the name")), + "{report}" + ); + ensure!( + stderr.contains("WARNING: migration 2's share-hash backfill could not be read") + && stderr.contains("self-check cannot tell whether it is pending"), + "{stderr}" + ); + db.close(vec![migrated, operator]).await +} diff --git a/crates/qbit-prism-server/tests/support/share_partitions.rs b/crates/qbit-prism-server/tests/support/share_partitions.rs new file mode 100644 index 000000000..a78ce01ac --- /dev/null +++ b/crates/qbit-prism-server/tests/support/share_partitions.rs @@ -0,0 +1,836 @@ +//! Migration 017 (#144): the share ledger partition conversion. +//! +//! The conversion attaches the release table as the first partition of a +//! new partitioned parent after a validated bound, inside the migration +//! transaction where the ledger is empty and through the resumable online +//! runner otherwise. These tests cover the shape both paths leave, that +//! rows and the sequence survive the online path from every stage it can +//! resume at, that a reserved name is refused before any DDL, that the +//! swap waits for an open writer with a lock timeout rather than queuing +//! every append behind it, that a bound whose headroom ran out is moved, +//! and what per-leaf share_id uniqueness means for the append path. +//! +//! `undo_017` and `undo_016` put a migrated database back to what a build +//! without those migrations left, for the older upgrade fixtures that +//! simulate such builds. +use super::*; +use anyhow::ensure; +use qbit_prism_server::ledger::REQUIRED_SCHEMA_VERSIONS; +use sqlx::Row; +use std::sync::atomic::{AtomicBool, Ordering}; +use std::time::Duration; +use tokio::time::{sleep, timeout}; + +/// The default partition width, `qbit_prism_share_partitioning.partition_rows`. +pub(super) const WIDTH: i64 = 16_777_216; + +const PARENT_INDEXES: [&str; 5] = [ + "qbit_share_ledger_accepted_block_suffix_idx", + "qbit_share_ledger_accepted_miner_history_idx", + "qbit_share_ledger_accepted_recent_idx", + "qbit_share_ledger_accepted_seq_walk_idx", + "qbit_share_ledger_pkey", +]; +const LEAF_SUFFIXES: [&str; 6] = [ + "_accepted_block_suffix_idx", + "_accepted_miner_history_idx", + "_accepted_recent_idx", + "_accepted_seq_walk_idx", + "_pkey", + "_share_id_key", +]; + +pub(super) async fn schema_versions(pool: &PgPool) -> Result> { + Ok( + sqlx::query_scalar("SELECT version FROM qbit_prism_schema_migrations ORDER BY version") + .fetch_all(pool) + .await?, + ) +} + +async fn relkind(pool: &PgPool, name: &str) -> Result> { + Ok( + sqlx::query_scalar("SELECT c.relkind::text FROM pg_class c WHERE c.oid=to_regclass($1)") + .bind(name) + .fetch_optional(pool) + .await?, + ) +} + +/// The index names on one relation, sorted. +async fn index_names(pool: &PgPool, table: &str) -> Result> { + Ok(sqlx::query_scalar( + "SELECT i.relname::text FROM pg_index x JOIN pg_class i ON i.oid=x.indexrelid WHERE x.indrelid=to_regclass($1) ORDER BY 1", + ) + .bind(table) + .fetch_all(pool) + .await?) +} + +/// The partitions PostgreSQL has attached, by name with the bound it holds. +async fn attached(pool: &PgPool) -> Result> { + let rows = sqlx::query("SELECT c.relname::text AS name,pg_get_expr(c.relpartbound,c.oid) AS bound FROM pg_inherits i JOIN pg_class c ON c.oid=i.inhrelid WHERE i.inhparent='qbit_share_ledger'::regclass ORDER BY 1") + .fetch_all(pool) + .await?; + rows.iter() + .map(|row| Ok((row.try_get("name")?, row.try_get("bound")?))) + .collect() +} + +/// The catalog rows: name, lower (None is MINVALUE), upper, state. +pub(super) async fn catalog(pool: &PgPool) -> Result, i64, String)>> { + let rows = sqlx::query("SELECT partition_name,lower_seq,upper_seq,state FROM qbit_prism_share_partitions ORDER BY upper_seq") + .fetch_all(pool) + .await?; + rows.iter() + .map(|row| { + Ok(( + row.try_get("partition_name")?, + row.try_get("lower_seq")?, + row.try_get("upper_seq")?, + row.try_get("state")?, + )) + }) + .collect() +} + +async fn conversion_bound(pool: &PgPool) -> Result> { + Ok(sqlx::query_scalar( + "SELECT conversion_bound FROM qbit_prism_share_partitioning WHERE singleton", + ) + .fetch_one(pool) + .await?) +} + +async fn share_count(pool: &PgPool) -> Result { + Ok(sqlx::query_scalar("SELECT count(*) FROM qbit_share_ledger") + .fetch_one(pool) + .await?) +} + +/// A share row as a writer leaves it, at an explicit `share_seq` when one +/// is given, without the native ordering lock. +async fn insert_share<'e>( + executor: impl sqlx::Executor<'e, Database = sqlx::Postgres>, + seq: Option, + id: u64, + miner: &str, +) -> Result<()> { + sqlx::query("INSERT INTO qbit_share_ledger(share_seq,share_id,miner_id,payout_order_key,p2mr_program,share_difficulty,network_difficulty,template_height,job_id,job_issued_at,ntime,accepted_at,accepted,writer_id,writer_epoch) VALUES(COALESCE($3,nextval('qbit_share_ledger_share_seq_seq')),$1,$2,$2,decode(repeat('11',32),'hex'),1,100,100,'job',to_timestamp(1),1,clock_timestamp(),true,'share-partitions',0)") + .bind(format!("{miner}:{id:064x}")) + .bind(miner) + .bind(seq) + .execute(executor) + .await?; + Ok(()) +} + +/// Everything the conversion leaves, on either path: a partitioned parent +/// with the release index names, the release table as `_p0` with the leaf +/// set (its own share_id uniqueness included), every attached partition in +/// the catalog and the catalog's attached rows attached, the immutability +/// trigger on the parent and every leaf, the release function returning +/// the parent's row type, and 17 recorded after every other version. +pub(super) async fn assert_converted(pool: &PgPool) -> Result<()> { + assert_eq!( + relkind(pool, "qbit_share_ledger").await?.as_deref(), + Some("p") + ); + assert_eq!( + index_names(pool, "qbit_share_ledger").await?, + PARENT_INDEXES + ); + let partitions = attached(pool).await?; + assert!( + partitions + .iter() + .any(|(name, bound)| name == "qbit_share_ledger_p0" && bound.contains("MINVALUE")), + "{partitions:?}" + ); + let rows = catalog(pool).await?; + let attached_rows: Vec<&str> = rows + .iter() + .filter(|(_, _, _, state)| state == "attached") + .map(|(name, ..)| name.as_str()) + .collect(); + let mut attached_names: Vec<&str> = partitions.iter().map(|(name, _)| name.as_str()).collect(); + attached_names.sort_unstable(); + let mut expected = attached_rows.clone(); + expected.sort_unstable(); + assert_eq!(attached_names, expected, "catalog and pg_inherits disagree"); + for (name, _) in &partitions { + let expected: Vec = LEAF_SUFFIXES + .iter() + .map(|suffix| format!("{name}{suffix}")) + .collect(); + assert_eq!(index_names(pool, name).await?, expected); + let trigger: bool = sqlx::query_scalar("SELECT EXISTS(SELECT 1 FROM pg_trigger WHERE tgrelid=to_regclass($1) AND tgname='qbit_prism_immutable_share_history')") + .bind(name).fetch_one(pool).await?; + assert!(trigger, "{name} has no immutability trigger"); + } + let parent_trigger: bool = sqlx::query_scalar("SELECT EXISTS(SELECT 1 FROM pg_trigger WHERE tgrelid='qbit_share_ledger'::regclass AND tgname='qbit_prism_immutable_share_history')") + .fetch_one(pool).await?; + assert!(parent_trigger); + let returns_parent: bool = sqlx::query_scalar("SELECT (SELECT prorettype FROM pg_proc WHERE oid='qbit_shares_since_template_height(bigint)'::regprocedure)=(SELECT reltype FROM pg_class WHERE oid='qbit_share_ledger'::regclass)") + .fetch_one(pool).await?; + assert!( + returns_parent, + "qbit_shares_since_template_height returns the old row type" + ); + let converted: bool = sqlx::query_scalar( + "SELECT converted_at IS NOT NULL FROM qbit_prism_share_partitioning WHERE singleton", + ) + .fetch_one(pool) + .await?; + assert!(converted); + assert_eq!(schema_versions(pool).await?, REQUIRED_SCHEMA_VERSIONS); + let recorded_last: bool = sqlx::query_scalar("SELECT (SELECT applied_at FROM qbit_prism_schema_migrations WHERE version=17) >= (SELECT max(applied_at) FROM qbit_prism_schema_migrations WHERE version<17)") + .fetch_one(pool) + .await?; + assert!(recorded_last); + Ok(()) +} + +/// Undo 017 on a converted ledger: detach and drop the empty lead +/// partitions, detach the release table, give it the sequence back before +/// the parent (which owns it) is dropped, drop the parent and the release +/// function bound to its row type, rename the release table and its six +/// indexes back, drop the bound, recreate the function on the plain table, +/// clear the catalog and remove the version row. The 016 objects stay. +pub(super) async fn undo_017(pool: &PgPool) -> Result<()> { + ensure!( + relkind(pool, "qbit_share_ledger").await?.as_deref() == Some("p"), + "undo_017 needs a converted ledger" + ); + let versions = schema_versions(pool).await?; + ensure!(versions.contains(&17)); + sqlx::raw_sql( + "DO $$ + DECLARE part text; + BEGIN + FOR part IN SELECT inhrelid::regclass::text FROM pg_inherits WHERE inhparent='qbit_share_ledger'::regclass AND inhrelid<>'qbit_share_ledger_p0'::regclass LOOP + EXECUTE format('ALTER TABLE qbit_share_ledger DETACH PARTITION %I', part); + EXECUTE format('DROP TABLE %I', part); + END LOOP; + END $$; + ALTER TABLE qbit_share_ledger DETACH PARTITION qbit_share_ledger_p0; + ALTER SEQUENCE qbit_share_ledger_share_seq_seq OWNED BY qbit_share_ledger_p0.share_seq; + DROP FUNCTION qbit_shares_since_template_height(bigint); + DROP TABLE qbit_share_ledger; + ALTER TABLE qbit_share_ledger_p0 RENAME TO qbit_share_ledger; + ALTER INDEX qbit_share_ledger_p0_pkey RENAME TO qbit_share_ledger_pkey; + ALTER INDEX qbit_share_ledger_p0_share_id_key RENAME TO qbit_share_ledger_share_id_key; + ALTER INDEX qbit_share_ledger_p0_accepted_recent_idx RENAME TO qbit_share_ledger_accepted_recent_idx; + ALTER INDEX qbit_share_ledger_p0_accepted_block_suffix_idx RENAME TO qbit_share_ledger_accepted_block_suffix_idx; + ALTER INDEX qbit_share_ledger_p0_accepted_seq_walk_idx RENAME TO qbit_share_ledger_accepted_seq_walk_idx; + ALTER INDEX qbit_share_ledger_p0_accepted_miner_history_idx RENAME TO qbit_share_ledger_accepted_miner_history_idx; + ALTER TABLE qbit_share_ledger DROP CONSTRAINT qbit_share_ledger_p0_bound; + CREATE FUNCTION qbit_shares_since_template_height(min_template_height bigint) + RETURNS SETOF qbit_share_ledger LANGUAGE sql STABLE AS $$ + SELECT * FROM qbit_share_ledger WHERE accepted AND template_height >= min_template_height ORDER BY share_seq ASC; + $$; + DELETE FROM qbit_prism_share_partitions; + UPDATE qbit_prism_share_partitioning SET converted_at=NULL, conversion_bound=NULL, updated_at=clock_timestamp(); + DELETE FROM qbit_prism_schema_migrations WHERE version=17", + ) + .execute(pool) + .await?; + assert_eq!( + relkind(pool, "qbit_share_ledger").await?.as_deref(), + Some("r") + ); + assert_eq!( + schema_versions(pool).await?, + versions + .into_iter() + .filter(|version| *version != 17) + .collect::>() + ); + Ok(()) +} + +/// Undo 016 on a ledger 017 has been undone on: the catalog, the +/// functions, the solver columns and the version row go, and the two +/// release foreign keys onto share_id come back under their release names. +pub(super) async fn undo_016(pool: &PgPool) -> Result<()> { + ensure!( + relkind(pool, "qbit_share_ledger").await?.as_deref() == Some("r"), + "undo 017 before 016" + ); + let versions = schema_versions(pool).await?; + ensure!(versions.contains(&16) && !versions.contains(&17)); + sqlx::raw_sql( + "DELETE FROM qbit_prism_schema_migrations WHERE version=16; + DROP FUNCTION qbit_prism_share_ledger_convert_swap(); + DROP FUNCTION qbit_prism_share_ledger_convert_validate(); + DROP FUNCTION qbit_prism_share_ledger_convert_prepare(bigint); + DROP FUNCTION qbit_prism_share_partition_ensure(); + DROP FUNCTION qbit_prism_share_partition_next_number(); + DROP FUNCTION qbit_prism_share_partition_create(text,bigint,bigint); + DROP FUNCTION qbit_prism_share_probe_floor(); + DROP FUNCTION qbit_prism_share_next_seq(); + DROP TABLE qbit_prism_share_partitions; + DROP TABLE qbit_prism_share_partitioning; + DROP TABLE qbit_prism_rejected_share_ids; + DROP TRIGGER qbit_pool_blocks_capture_solver ON qbit_pool_blocks; + DROP FUNCTION qbit_prism_capture_block_solver(); + ALTER TABLE qbit_pool_blocks DROP COLUMN solver_miner_id, DROP COLUMN solver_share_id, DROP COLUMN solver_share_difficulty, DROP COLUMN solver_network_difficulty; + ALTER TABLE qbit_block_candidate_outbox ADD CONSTRAINT qbit_block_candidate_outbox_share_id_fkey FOREIGN KEY (share_id) REFERENCES qbit_share_ledger(share_id); + ALTER TABLE qbit_prism_share_hashes ADD CONSTRAINT qbit_prism_share_hashes_share_id_fkey FOREIGN KEY (share_id) REFERENCES qbit_share_ledger(share_id)", + ) + .execute(pool) + .await?; + assert_eq!( + schema_versions(pool).await?, + versions + .into_iter() + .filter(|version| *version != 16) + .collect::>() + ); + Ok(()) +} + +/// The tableoid a share landed in. +async fn partition_of(pool: &PgPool, share_seq: i64) -> Result { + Ok(sqlx::query_scalar( + "SELECT tableoid::regclass::text FROM qbit_share_ledger WHERE share_seq=$1", + ) + .bind(share_seq) + .fetch_one(pool) + .await?) +} + +/// A fresh ledger is converted inside the migration transaction, the +/// cutover locks excluding writers: its first partition is one cell, +/// the lead is attached above it, appends route into the first cell, and +/// a second start applies nothing. +#[tokio::test] +async fn migration_017_converts_an_empty_ledger_in_the_transaction_and_appends_route_to_the_first_cell( +) -> Result<()> { + let Some(db) = Database::open().await? else { + return Ok(()); + }; + let pool = PgPool::connect(&db.url).await?; + let a = db.ledger("a").await?; + assert_converted(&pool).await?; + let rows = catalog(&pool).await?; + assert_eq!( + rows.iter() + .map(|(name, lower, upper, _)| (name.as_str(), *lower, *upper)) + .collect::>(), + [ + ("qbit_share_ledger_p0", None, WIDTH), + ("qbit_share_ledger_p1", Some(WIDTH), 2 * WIDTH), + ("qbit_share_ledger_p2", Some(2 * WIDTH), 3 * WIDTH), + ("qbit_share_ledger_p3", Some(3 * WIDTH), 4 * WIDTH), + ("qbit_share_ledger_p4", Some(4 * WIDTH), 5 * WIDTH), + ] + ); + assert_eq!(conversion_bound(&pool).await?, Some(WIDTH)); + let first = a.append(share(1), None).await?; + let second = a.append(share(2), None).await?; + assert_eq!( + (first.share.share_seq, second.share.share_seq), + (1, 2), + "the sequence starts at 1 and is never restarted" + ); + assert_eq!(partition_of(&pool, 1).await?, "qbit_share_ledger_p0"); + assert_eq!(a.snapshot(100).await?.shares.len(), 2); + let counted: i64 = + sqlx::query_scalar("SELECT count(*) FROM qbit_prism_window(clock_timestamp(), 1600)") + .fetch_one(&pool) + .await?; + assert_eq!(counted, 2); + let b = db.ledger("b").await?; + assert_eq!(catalog(&pool).await?, rows); + assert_converted(&pool).await?; + db.close(vec![a, b]).await +} + +#[tokio::test] +async fn migration_017_converts_empty_2x_ledgers_at_sequence_boundaries() -> Result<()> { + for state in [SourceState::Pre258, SourceState::Applied258] { + for next_seq in [WIDTH - 1, WIDTH, WIDTH + 1] { + let Some(db) = Database::open().await? else { + return Ok(()); + }; + let pool = PgPool::connect(&db.url).await?; + let result = async { + super::two_x::apply_frozen_2x_schema(&pool, state).await?; + // An empty ledger may have consumed sequence values through + // rolled-back writes, independently of its row count. + sqlx::query("SELECT setval('qbit_share_ledger_share_seq_seq',$1,false)") + .bind(next_seq) + .execute(&pool) + .await?; + assert_eq!(share_count(&pool).await?, 0); + let ledger = db.ledger("empty-boundary").await?; + assert_converted(&pool).await?; + let expected_bound = if next_seq < WIDTH { WIDTH } else { 2 * WIDTH }; + assert_eq!(conversion_bound(&pool).await?, Some(expected_bound)); + let appended = ledger.append(share(1), None).await?; + assert_eq!(appended.share.share_seq, next_seq as u64); + assert_eq!(partition_of(&pool, next_seq).await?, "qbit_share_ledger_p0"); + Ok(ledger) + } + .await; + pool.close().await; + match result { + Ok(ledger) => db.close(vec![ledger]).await?, + Err(error) => { + db.close(Vec::new()).await?; + return Err(error); + } + } + } + } + Ok(()) +} + +/// Older frontends omit the new solver columns while conversion is pending. +#[tokio::test] +async fn migration_016_captures_old_frontend_solvers_during_online_conversion() -> Result<()> { + let Some(db) = Database::open().await? else { + return Ok(()); + }; + let ledger = db.ledger("old-landing").await?; + let result = async { + let hash = "ab".repeat(32); + let share_id = format!("legacy.rig:{hash}"); + let mut solving = share(1); + solving.share_id = share_id.clone(); + ledger.append(solving, None).await?; + // 016 and its backfill have committed. Keep 017 pending, as during + // the online validation phase while old frontends remain active. + undo_017(&ledger.pool).await?; + sqlx::query("INSERT INTO qbit_pool_blocks(block_hash,block_height,parent_hash,coinbase_txid,payout_manifest_sha256) VALUES($1,100,'parent','coinbase','manifest')") + .bind(&hash) + .execute(&ledger.pool) + .await?; + let solver: (Option, Option, Option, Option) = + sqlx::query_as("SELECT solver_miner_id,solver_share_id,solver_share_difficulty::text,solver_network_difficulty::text FROM qbit_pool_blocks WHERE block_hash=$1") + .bind(&hash) + .fetch_one(&ledger.pool) + .await?; + ensure!( + solver.0.is_some() && solver.1.as_deref() == Some(share_id.as_str()) + && solver.2.is_some() && solver.3.is_some(), + "an old frontend landing after the backfill lost its solver: {solver:?}" + ); + let converted = db.ledger("converted").await?; + assert_converted(&ledger.pool).await?; + sqlx::raw_sql("ALTER TABLE qbit_share_ledger DETACH PARTITION qbit_share_ledger_p0") + .execute(&ledger.pool) + .await?; + let stored: Option = sqlx::query_scalar("SELECT solver_share_id FROM qbit_pool_blocks WHERE block_hash=$1") + .bind(&hash) + .fetch_one(&ledger.pool) + .await?; + ensure!(stored.as_deref() == Some(share_id.as_str()), "detach lost solver attribution"); + Ok(converted) + } + .await; + match result { + Ok(converted) => db.close(vec![ledger, converted]).await, + Err(error) => { + db.close(vec![ledger]).await?; + Err(error) + } + } +} + +/// A ledger with rows is converted after the commit by the online runner, +/// resumable from every stage: nothing prepared, the bound pending, the +/// bound validated. The rows and the sequence survive each time, the first +/// partition's bound is two widths above the sequence on the grid, and the +/// lead partitions follow it. +#[tokio::test] +async fn migration_017_converts_a_populated_ledger_online_and_resumes_from_every_stage( +) -> Result<()> { + let Some(db) = Database::open().await? else { + return Ok(()); + }; + let pool = PgPool::connect(&db.url).await?; + let first = db.ledger("first").await?; + for id in 1..=5 { + first.append(share(id), None).await?; + } + let mut ledgers = vec![first]; + for prepared in ["nothing", "prepare", "prepare+validate"] { + // Each round below appends one more share after its conversion. + let rows = 4 + i64::try_from(ledgers.len())?; + undo_017(&pool).await?; + assert_eq!(share_count(&pool).await?, rows); + if prepared != "nothing" { + let bound: i64 = sqlx::query_scalar("SELECT qbit_prism_share_ledger_convert_prepare()") + .fetch_one(&pool) + .await?; + assert_eq!(bound, 3 * WIDTH, "next_seq 6 plus two widths, on the grid"); + } + if prepared == "prepare+validate" { + let validated: Vec = + sqlx::query_scalar("SELECT qbit_prism_share_ledger_convert_validate()") + .fetch_one(&pool) + .await?; + assert_eq!(validated, ["qbit_share_ledger_p0_bound"]); + } + // Without initialize, a start refuses the database, naming the gap. + let error = Ledger::connect(&db.url, "cold".into(), 8, false) + .await + .err() + .context("a non-initializing start accepted a database missing 017")? + .to_string(); + assert!(error.contains("missing migration(s) 17"), "{error}"); + let resumed = db.ledger(&format!("resumed-{prepared}")).await?; + assert_converted(&pool).await?; + assert_eq!(share_count(&pool).await?, rows); + let rows = catalog(&pool).await?; + assert_eq!( + rows[0], + ( + "qbit_share_ledger_p0".to_owned(), + None, + 3 * WIDTH, + "attached".to_owned() + ) + ); + assert_eq!(rows[1].0, "qbit_share_ledger_p1"); + assert_eq!((rows[1].1, rows[1].2), (Some(3 * WIDTH), 4 * WIDTH)); + assert_eq!(conversion_bound(&pool).await?, Some(3 * WIDTH)); + let next = resumed + .append(share(100 + ledgers.len() as u64), None) + .await?; + assert_eq!(next.share.share_seq, 5 + ledgers.len() as u64); + assert_eq!( + partition_of(&pool, i64::try_from(next.share.share_seq)?).await?, + "qbit_share_ledger_p0", + "rows keep landing in the release table until the sequence reaches the bound" + ); + ledgers.push(resumed); + } + db.close(ledgers).await +} + +/// A relation under a name the partitions take is refused before the +/// bound is prepared: the table stays plain, no constraint is added and +/// nothing is recorded; once the name is free the conversion goes through. +#[tokio::test] +async fn migration_017_refuses_a_reserved_partition_name_before_any_ddl() -> Result<()> { + let Some(db) = Database::open().await? else { + return Ok(()); + }; + let pool = PgPool::connect(&db.url).await?; + let first = db.ledger("first").await?; + first.append(share(1), None).await?; + undo_017(&pool).await?; + let versions = schema_versions(&pool).await?; + sqlx::raw_sql("CREATE VIEW qbit_share_ledger_p7 AS SELECT 1 AS one") + .execute(&pool) + .await?; + let error = db + .ledger("collision") + .await + .err() + .context("the conversion ran with a reserved name taken")? + .to_string(); + assert!(error.contains("view qbit_share_ledger_p7"), "{error}"); + assert!(error.contains("nothing was changed"), "{error}"); + assert_eq!( + relkind(&pool, "qbit_share_ledger").await?.as_deref(), + Some("r") + ); + let bound_present: bool = sqlx::query_scalar("SELECT EXISTS(SELECT 1 FROM pg_constraint WHERE conrelid='qbit_share_ledger'::regclass AND conname='qbit_share_ledger_p0_bound')") + .fetch_one(&pool).await?; + assert!(!bound_present, "the bound was prepared before the refusal"); + assert_eq!(schema_versions(&pool).await?, versions); + assert_eq!(conversion_bound(&pool).await?, None); + sqlx::raw_sql("DROP VIEW qbit_share_ledger_p7") + .execute(&pool) + .await?; + let converted = db.ledger("converted").await?; + assert_converted(&pool).await?; + assert_eq!(share_count(&pool).await?, 1); + db.close(vec![first, converted]).await +} + +/// The prepare and swap steps take the table lock with a short timeout +/// and retry, so an open writer delays the conversion without holding the +/// appends queued behind an ACCESS EXCLUSIVE request; once the writer +/// commits, the conversion completes with its row. +#[tokio::test] +async fn migration_017_waits_for_an_open_writer_without_blocking_appends() -> Result<()> { + let Some(db) = Database::open().await? else { + return Ok(()); + }; + let pool = PgPool::connect(&db.url).await?; + let first = db.ledger("first").await?; + first.append(share(1), None).await?; + undo_017(&pool).await?; + let mut writer = pool.begin().await?; + insert_share(&mut *writer, None, 2, "alice").await?; + let finished = AtomicBool::new(false); + let migrate = async { + let ledger = Ledger::connect(&db.url, "online".into(), 8, true).await; + finished.store(true, Ordering::SeqCst); + ledger + }; + let observe = async { + // The prepare step's ACCESS EXCLUSIVE request waits behind the + // writer's row lock on the table, and is retried with a timeout. + timeout(Duration::from_secs(60), async { + loop { + let waiting: bool = sqlx::query_scalar("SELECT EXISTS(SELECT 1 FROM pg_stat_activity a JOIN pg_locks l ON l.pid=a.pid WHERE a.query LIKE 'SELECT qbit_prism_share_ledger_convert_prepare%' AND a.wait_event_type='Lock' AND l.locktype='relation' AND l.relation='qbit_share_ledger'::regclass AND NOT l.granted)") + .fetch_one(&pool) + .await?; + if waiting { + return Ok::<_, anyhow::Error>(()); + } + ensure!( + !finished.load(Ordering::SeqCst), + "the conversion finished while a writer transaction was open" + ); + sleep(Duration::from_millis(50)).await; + } + }) + .await + .context("the conversion never waited for the open writer")??; + // Appends keep landing while it retries: the request is dropped at + // each lock timeout rather than queued. + timeout(Duration::from_secs(10), async { + let mut tx = pool.begin().await?; + insert_share(&mut *tx, None, 3, "bob").await?; + tx.commit().await?; + Ok::<_, anyhow::Error>(()) + }) + .await + .context("an append blocked behind the conversion's lock request")??; + ensure!(!finished.load(Ordering::SeqCst)); + writer.commit().await?; + Ok::<_, anyhow::Error>(()) + }; + let (online, observed) = tokio::join!(migrate, observe); + observed?; + let online = online?; + assert_converted(&pool).await?; + assert_eq!(share_count(&pool).await?, 3); + db.close(vec![first, online]).await +} + +/// A bound prepared by an earlier, interrupted run, pending or already +/// validated, is moved further out when the sequence has come within one +/// width of it, then validated and swapped; the first partition's bound is +/// the new one. +#[tokio::test] +async fn migration_017_moves_a_bound_whose_headroom_ran_out() -> Result<()> { + let Some(db) = Database::open().await? else { + return Ok(()); + }; + let pool = PgPool::connect(&db.url).await?; + let first = db.ledger("first").await?; + first.append(share(1), None).await?; + let mut ledgers = vec![first]; + for (round, validate) in [false, true].into_iter().enumerate() { + undo_017(&pool).await?; + // The bound is reprepared on the grid from the current sequence. + let expected_bound = (round as i64 + 1) * WIDTH; + let bound: i64 = sqlx::query_scalar("SELECT qbit_prism_share_ledger_convert_prepare(10)") + .fetch_one(&pool) + .await?; + assert_eq!(bound, expected_bound); + if validate { + sqlx::query_scalar::<_, Vec>( + "SELECT qbit_prism_share_ledger_convert_validate()", + ) + .fetch_one(&pool) + .await?; + } + // The writers ran on: the sequence is now within one width of the bound. + sqlx::query("SELECT setval('qbit_share_ledger_share_seq_seq',$1)") + .bind(expected_bound - 5) + .execute(&pool) + .await?; + let resumed = db.ledger(&format!("resumed-{round}")).await?; + assert_converted(&pool).await?; + assert_eq!( + conversion_bound(&pool).await?, + Some(expected_bound + 2 * WIDTH) + ); + assert_eq!(catalog(&pool).await?[0].2, expected_bound + 2 * WIDTH); + let validated: bool = sqlx::query_scalar("SELECT convalidated FROM pg_constraint WHERE conrelid='qbit_share_ledger_p0'::regclass AND conname='qbit_share_ledger_p0_bound'") + .fetch_one(&pool).await?; + assert!(validated); + let next = resumed.append(share(2 + round as u64), None).await?; + assert_eq!(next.share.share_seq, u64::try_from(expected_bound - 4)?); + ledgers.push(resumed); + } + db.close(ledgers).await +} + +#[tokio::test] +async fn retained_legacy_header_duplicates_remain_exactly_replayable() -> Result<()> { + let Some(db) = Database::open().await? else { + return Ok(()); + }; + let pool = PgPool::connect(&db.url).await?; + super::two_x::apply_frozen_2x_schema(&pool, SourceState::Pre258).await?; + // The legacy writer accepted the same header under two worker identities. + // Migration 002 preserves both rows but maps the header to the first only. + insert_share(&pool, None, 1, "first").await?; + insert_share(&pool, None, 1, "later").await?; + let ledger = db.ledger("legacy-replay").await?; + let result = async { + assert_converted(&pool).await?; + let shares = ledger.snapshot(100).await?.shares; + ensure!(shares.len() == 2, "migration lost a legacy duplicate"); + let hash = format!("{:064x}", 1); + let credited: String = sqlx::query_scalar( + "SELECT share_id FROM qbit_prism_share_hashes WHERE header_hash=$1", + ) + .bind(&hash) + .fetch_one(&pool) + .await?; + ensure!(credited == format!("first:{hash}")); + let later = shares + .iter() + .find(|row| row.share_id == format!("later:{hash}")) + .context("later legacy share is missing")?; + let bound = conversion_bound(&pool).await?.context("conversion bound is missing")?; + for older_partition in [false, true] { + if older_partition { + sqlx::query("SELECT setval('qbit_share_ledger_share_seq_seq',$1)") + .bind(bound + 2 * WIDTH + 10) + .execute(&pool) + .await?; + sqlx::query("SELECT qbit_prism_share_partition_ensure()") + .execute(&pool) + .await?; + } + let floor: i64 = sqlx::query_scalar("SELECT qbit_prism_share_probe_floor()") + .fetch_one(&pool) + .await?; + ensure!((floor > later.share_seq as i64) == older_partition); + let before: (i64, i64) = sqlx::query_as( + "SELECT ledger_clock_ms,(SELECT last_value FROM qbit_share_ledger_share_seq_seq) FROM qbit_prism_cluster WHERE singleton", + ) + .fetch_one(&pool) + .await?; + for original in &shares { + let replay = ledger.append(original.clone(), None).await?; + ensure!(!replay.inserted && replay.share == *original); + } + let mut altered = later.clone(); + altered.share_difficulty += 1; + let error = ledger.append(altered, None).await.unwrap_err().to_string(); + ensure!(error.contains("duplicate share_id payload mismatch"), "{error}"); + let mut other = later.clone(); + other.share_id = format!("new-worker:{hash}"); + let error = ledger.append(other, None).await.unwrap_err().to_string(); + ensure!(error.contains("header already credited globally"), "{error}"); + ensure!(share_count(&pool).await? == 2); + let after: (i64, i64) = sqlx::query_as( + "SELECT ledger_clock_ms,(SELECT last_value FROM qbit_share_ledger_share_seq_seq) FROM qbit_prism_cluster WHERE singleton", + ) + .fetch_one(&pool) + .await?; + ensure!(before == after, "replays changed the ledger clock or sequence"); + } + sqlx::raw_sql("ALTER TABLE qbit_share_ledger DETACH PARTITION qbit_share_ledger_p0") + .execute(&pool) + .await?; + for original in shares { + let error = ledger.append(original, None).await.unwrap_err().to_string(); + ensure!(error.contains("duplicate-share"), "{error}"); + } + ensure!(share_count(&pool).await? == 0, "a detached share was credited again"); + let mappings: Vec = sqlx::query_scalar("SELECT share_id FROM qbit_prism_share_hashes") + .fetch_all(&pool) + .await?; + ensure!(mappings == [credited], "replays changed the global header mapping"); + Ok(()) + } + .await; + pool.close().await; + db.close(vec![ledger]).await?; + result +} + +/// share_id uniqueness is per leaf after the conversion. The append path +/// is what keeps it global: it consults qbit_prism_share_hashes first, so +/// an exact replay of a share in an older partition is matched (bounded +/// probe first, unbounded on a miss), a payload mismatch is refused, and +/// another share_id with the same header hash is refused. A row inserted +/// around the append path with a share_id another leaf holds is the one +/// duplicate PostgreSQL no longer refuses, which the design record states. +#[tokio::test] +async fn share_id_uniqueness_is_per_leaf_and_the_append_path_keeps_it_global() -> Result<()> { + let Some(db) = Database::open().await? else { + return Ok(()); + }; + let pool = PgPool::connect(&db.url).await?; + let ledger = db.ledger("a").await?; + let original = ledger.append(share(1), None).await?; + assert_eq!(original.share.share_seq, 1); + // Move the sequence into the fourth cell, p3: the probe floor becomes + // the lower bound of p1, two cells below, past the first row. + sqlx::query("SELECT setval('qbit_share_ledger_share_seq_seq',$1)") + .bind(3 * WIDTH + 10) + .execute(&pool) + .await?; + let created: i32 = sqlx::query_scalar("SELECT qbit_prism_share_partition_ensure()") + .fetch_one(&pool) + .await?; + assert!(created >= 1); + let floor: i64 = sqlx::query_scalar("SELECT qbit_prism_share_probe_floor()") + .fetch_one(&pool) + .await?; + assert_eq!( + floor, WIDTH, + "the probe floor is not the lower bound of p1, past the first row" + ); + let later = ledger.append(share(2), None).await?; + assert_eq!(later.share.share_seq, u64::try_from(3 * WIDTH + 11)?); + assert_eq!( + partition_of(&pool, 3 * WIDTH + 11).await?, + "qbit_share_ledger_p3" + ); + // An exact replay of the first share is found past the bounded probe. + let replay = ledger.append(share(1), None).await?; + assert!(!replay.inserted); + assert_eq!(replay.share, original.share); + // The same share_id with another payload is refused. + let mut altered = share(1); + altered.share_difficulty = 2; + let error = ledger.append(altered, None).await.unwrap_err().to_string(); + assert!( + error.contains("duplicate share_id payload mismatch"), + "{error}" + ); + // Another identity submitting the same header is refused globally. + let mut other = share(1); + other.share_id = format!("other:{:064x}", 1); + other.miner_id = "other".into(); + let error = ledger.append(other, None).await.unwrap_err().to_string(); + assert!( + error.contains("header already credited globally"), + "{error}" + ); + assert_eq!(share_count(&pool).await?, 2); + // Around the append path, another leaf accepts the first share_id: the + // per-leaf index cannot see the other partition. + insert_share(&pool, Some(3 * WIDTH + 12), 1, "worker").await?; + let copies: i64 = + sqlx::query_scalar("SELECT count(*) FROM qbit_share_ledger WHERE share_id=$1") + .bind(format!("worker:{:064x}", 1)) + .fetch_one(&pool) + .await?; + assert_eq!(copies, 2); + let same_leaf = insert_share(&pool, Some(3 * WIDTH + 13), 1, "worker").await; + assert!( + same_leaf + .unwrap_err() + .to_string() + .contains("qbit_share_ledger_p3_share_id_key"), + "the leaf's own uniqueness holds" + ); + db.close(vec![ledger]).await +} diff --git a/crates/qbit-prism-server/tests/support/storm_scale.rs b/crates/qbit-prism-server/tests/support/storm_scale.rs new file mode 100644 index 000000000..976da0b06 --- /dev/null +++ b/crates/qbit-prism-server/tests/support/storm_scale.rs @@ -0,0 +1,91 @@ +//! The candidate-storm suite's cardinality, read once from the environment (#270). +//! +//! `PRISM_TEST_STORM_CANDIDATES` sizes every storm scenario. The default, 100, +//! is the reduced cardinality the `prism-native-postgres` job runs; +//! `test/prism-native-tests.sh` raises it to 3,120, the durable candidate +//! count the 2026-08-20 testnet4 incident left behind one decided height, for +//! a local run. One reader serves every storm test in every target, so the CI +//! path and the local path cannot drift apart. +//! +//! This is not a gate variable. The shared gate +//! (`crates/qbit-prism-test-gate`) decides whether a test runs at all; this +//! decides how large it is once it does. Every storm test therefore executes +//! in both paths and is listed in `test/prism-gated-tests.txt`; none is +//! `#[ignore]`d, because `scripts/run_rust_test_shard.py` runs ignored tests +//! only from its own allowlist and `scripts/check_gate_manifest.py` fails both +//! on a listed test that did not execute and on a test that executed without +//! being listed. +//! +//! Every per-row cost invariant is measured at two cardinalities inside one +//! process, [`BASELINE_CANDIDATES`] and the run's own, and asserted equal. +//! Comparing two points in the same run is what makes the reduced-N CI path +//! prove the same property as a local run at the incident's cardinality, +//! rather than a weaker version of it. + +use anyhow::{ensure, Context, Result}; + +/// The small cardinality every per-row cost is compared against. Large enough +/// that a per-row constant is distinguishable from a fixed setup cost, small +/// enough to add nothing measurable to the job. +pub const BASELINE_CANDIDATES: usize = 24; + +/// The reduced cardinality a run uses when the environment says nothing, and +/// the floor any explicit value must meet: a lower setting would let a run +/// record `executed` for the whole suite while proving less than CI does. +pub const DEFAULT_CANDIDATES: usize = 100; + +/// The durable block candidates the 2026-08-20 testnet4 incident left behind +/// one decided height. What a local run measures. +pub const OBSERVED_STORM_CANDIDATES: usize = 3_120; + +/// An explicit cardinality above this is refused rather than attempted: past +/// it a run is a capacity experiment, not this suite, and #270 fences the +/// set-oriented collapse selector out of scope. +pub const MAX_CANDIDATES: usize = 100_000; + +/// The variable that sizes the suite. +pub const CANDIDATES_VAR: &str = "PRISM_TEST_STORM_CANDIDATES"; + +/// Fixed, greppable prefix of every recorded measurement line. Durations and +/// buffer counts are reported under it and never asserted: a timing assertion +/// on a shared two-vCPU runner is flake, and a flaky gate is worse than none. +pub const REPORT_PREFIX: &str = "[prism-storm]"; + +/// The storm cardinality for this run. +/// +/// Absent means [`DEFAULT_CANDIDATES`]. A value that is present but not a +/// count, or outside `DEFAULT_CANDIDATES..=MAX_CANDIDATES`, is an error naming +/// the variable and what it held: a misconfigured run must fail loudly rather +/// than quietly measure something smaller than CI already proves. +pub fn storm_candidates() -> Result { + let count = match std::env::var(CANDIDATES_VAR) { + Ok(raw) => raw + .trim() + .parse::() + .with_context(|| format!("{CANDIDATES_VAR}={raw:?} is not a candidate count"))?, + Err(std::env::VarError::NotPresent) => DEFAULT_CANDIDATES, + Err(error) => return Err(error).context(format!("{CANDIDATES_VAR} is not readable")), + }; + ensure!( + (DEFAULT_CANDIDATES..=MAX_CANDIDATES).contains(&count), + "{CANDIDATES_VAR}={count} is outside {DEFAULT_CANDIDATES}..={MAX_CANDIDATES}; \ + the suite proves its per-row invariant against {BASELINE_CANDIDATES} baseline rows \ + and a lower setting would prove less than the reduced-N CI path" + ); + ensure!( + count > BASELINE_CANDIDATES, + "{CANDIDATES_VAR}={count} is not above the {BASELINE_CANDIDATES}-row baseline" + ); + Ok(count) +} + +/// Print one recorded measurement: never asserted, always reported, so a PR +/// body quotes the run rather than a memory of it. `name` identifies the +/// scenario, `pairs` are `key=value` facts in the units they were observed in. +pub fn record(name: &str, pairs: &[(&str, String)]) { + let rendered: Vec = pairs + .iter() + .map(|(key, value)| format!("{key}={value}")) + .collect(); + println!("{REPORT_PREFIX} {name} {}", rendered.join(" ")); +} diff --git a/crates/qbit-prism-server/tests/support/stratum_admission.rs b/crates/qbit-prism-server/tests/support/stratum_admission.rs new file mode 100644 index 000000000..da13a2459 --- /dev/null +++ b/crates/qbit-prism-server/tests/support/stratum_admission.rs @@ -0,0 +1,248 @@ +//! Actual listener/socket fixture; only backend I/O is controlled. +#![allow(dead_code)] +use qbit_prism_server::{ + codec::Submission, + ledger::{Ledger, SessionAllocationExhausted, SessionId}, + stratum::{ + run_listener, MiningBackend, MiningJob, StaleGrace, StratumConfig, StratumError, + StratumStats, Worker, + }, +}; +use serde_json::Value; +use std::sync::{ + atomic::{AtomicBool, AtomicU32, Ordering}, + Arc, +}; +use tokio::{ + io::{AsyncBufReadExt, AsyncWriteExt, BufReader}, + net::{TcpListener, TcpStream}, + sync::{watch, Notify}, + task::JoinHandle, + time::{timeout, Duration}, +}; + +#[derive(Default)] +pub struct Backend { + pub ledger: Option, + pub allocation_calls: AtomicU32, + pub ids: AtomicU32, + pub authorize_calls: AtomicU32, + pub resume_calls: AtomicU32, + pub build_calls: AtomicU32, + pub fail_once: AtomicBool, + pub stall_once: AtomicBool, + /// Stalls the next allocation in a step it names as the ledger + /// database's (#655). + pub stall_on_database_once: AtomicBool, + pub allocation_started: Notify, + pub allocation_release: Notify, +} +impl MiningBackend for Backend { + type Context = (); + async fn new_session_id(&self) -> Result { + self.allocation_calls.fetch_add(1, Ordering::SeqCst); + if self.fail_once.swap(false, Ordering::SeqCst) { + return Err(StratumError::backend("controlled allocator unavailable")); + } + let id = match &self.ledger { + Some(ledger) => ledger.new_session_id().await.map_err(|error| { + if error.is::() { + StratumError::new(20, error.to_string(), "session-allocation-exhausted") + } else { + StratumError::backend("database unavailable") + } + })?, + None => (self.ids.fetch_add(1, Ordering::SeqCst) + 1).into(), + }; + // Models a consumed sequence value whose RPC reply misses the deadline. + if self.stall_once.swap(false, Ordering::SeqCst) { + self.allocation_started.notify_one(); + self.allocation_release.notified().await; + } + // The same, held by the database: the backend names the step. + if self.stall_on_database_once.swap(false, Ordering::SeqCst) { + self.allocation_started.notify_one(); + qbit_prism_server::waiting::on_database(self.allocation_release.notified()).await; + } + Ok(id) + } + async fn authorize(&self, username: &str) -> Result { + self.authorize_calls.fetch_add(1, Ordering::SeqCst); + Ok(Worker { + username: username.into(), + payout_address: "controlled-payout".into(), + worker_name: None, + p2mr_program_hex: "ab".repeat(32), + }) + } + /// This fixture issues no work, so every lookup is a real miss. With a + /// ledger attached the miss is the actual `qbit_prism_jobs` query, which + /// is what a query-count proof has to observe. + async fn resume_job( + &self, + _: &Worker, + job_id: &str, + ) -> Result>, StratumError> { + self.resume_calls.fetch_add(1, Ordering::SeqCst); + if let Some(ledger) = &self.ledger { + ledger + .job(job_id) + .await + .map_err(|_| StratumError::backend("job lookup unavailable"))?; + } + Ok(None) + } + async fn build_job( + &self, + _: &Worker, + _: &str, + _: f64, + _: f64, + ) -> Result, StratumError> { + self.build_calls.fetch_add(1, Ordering::SeqCst); + Err(StratumError::backend("controlled no prepared work")) + } + async fn submit( + &self, + _: &Worker, + _: &MiningJob<()>, + _: Submission, + _: StaleGrace, + ) -> Result<(), StratumError> { + unreachable!("this fixture issues no work") + } +} + +pub struct Server { + pub address: std::net::SocketAddr, + pub backend: Arc, + pub stats: Arc, + pub metrics: Arc, + shutdown: watch::Sender, + _refresh: watch::Sender, + task: JoinHandle>, +} +impl Server { + pub async fn start(config: StratumConfig, backend: Arc) -> Self { + Self::start_with_metrics( + config, + backend, + Arc::new(qbit_prism_server::metrics::Metrics::default()), + ) + .await + } + pub async fn start_with_metrics( + mut config: StratumConfig, + backend: Arc, + metrics: Arc, + ) -> Self { + config.vardiff.enabled = false; + let stats = config.stats.clone(); + let listener = TcpListener::bind("127.0.0.1:0").await.unwrap(); + let address = listener.local_addr().unwrap(); + let (shutdown, shutdown_rx) = watch::channel(false); + let (refresh, refresh_rx) = watch::channel(0); + let task = tokio::spawn(run_listener( + listener, + config, + backend.clone(), + refresh_rx, + shutdown_rx, + metrics.clone(), + )); + Self { + address, + backend, + stats, + metrics, + shutdown, + _refresh: refresh, + task, + } + } + pub async fn connections(&self, count: usize) { + timeout(Duration::from_secs(5), async { + while self.stats.snapshot(0).connections != count { + tokio::task::yield_now().await; + } + }) + .await + .expect("connection did not enter/leave the actual session loop"); + } + pub async fn stop(self) { + self.shutdown.send(true).unwrap(); + timeout(Duration::from_secs(12), self.task) + .await + .unwrap() + .unwrap() + .unwrap(); + } +} + +pub struct Client(pub BufReader); +impl Client { + pub async fn connect(server: &Server) -> Self { + Self(BufReader::new( + TcpStream::connect(server.address).await.unwrap(), + )) + } + pub async fn send(&mut self, request: Value) { + let mut bytes = serde_json::to_vec(&request).unwrap(); + bytes.push(b'\n'); + self.0.get_mut().write_all(&bytes).await.unwrap(); + } + pub async fn read(&mut self) -> Value { + let mut line = String::new(); + let count = timeout(Duration::from_secs(5), self.0.read_line(&mut line)) + .await + .unwrap() + .unwrap(); + assert!(count > 0, "session closed before returning a response"); + serde_json::from_str(&line).unwrap() + } + pub async fn request(&mut self, request: Value) -> Value { + self.send(request).await; + self.read().await + } + /// Read every remaining response and require the listener to close. + pub async fn expect_closed(&mut self) -> Vec { + let mut seen = Vec::new(); + loop { + let mut line = String::new(); + let count = timeout(Duration::from_secs(5), self.0.read_line(&mut line)) + .await + .expect("the listener never closed the connection") + .unwrap(); + if count == 0 { + return seen; + } + seen.push(serde_json::from_str(&line).unwrap()); + } + } +} + +/// A refused socket is closed at accept without a byte written, exactly as the +/// existing global limit closes one. +pub async fn assert_socket_refused(address: std::net::SocketAddr) { + use tokio::io::AsyncReadExt; + let mut refused = TcpStream::connect(address).await.unwrap(); + let mut bytes = Vec::new(); + match timeout(Duration::from_secs(5), refused.read_to_end(&mut bytes)) + .await + .expect("the refused socket stayed open") + { + Ok(_) => assert!(bytes.is_empty(), "refused socket received {bytes:?}"), + Err(error) => assert_eq!(error.kind(), std::io::ErrorKind::ConnectionReset), + } +} + +/// One closed-enum refusal series from the live registry. +pub fn refusal_total(metrics: &qbit_prism_server::metrics::Metrics, reason: &str) -> f64 { + let key = format!("qbit_prism_stratum_connection_refusals_total{{reason=\"{reason}\"}} "); + metrics + .render() + .lines() + .find_map(|line| line.strip_prefix(&key)) + .map(|value| value.parse().unwrap()) + .unwrap_or(f64::NAN) +} diff --git a/crates/qbit-prism-server/tests/support/wal_primary.rs b/crates/qbit-prism-server/tests/support/wal_primary.rs new file mode 100644 index 000000000..d7c7de329 --- /dev/null +++ b/crates/qbit-prism-server/tests/support/wal_primary.rs @@ -0,0 +1,112 @@ +//! A private durable primary for server-wide insert-LSN measurements. +//! Separate schemas do not isolate WAL from other tests or their maintenance. +use anyhow::{ensure, Context, Result}; +use std::{path::PathBuf, process::Command}; + +pub struct Primary { + pub url: String, + bin: PathBuf, + data: PathBuf, + running: bool, + _directory: tempfile::TempDir, +} + +impl Primary { + pub async fn start(bin: String) -> Result { + tokio::task::spawn_blocking(move || Self::start_blocking(bin)).await? + } + + fn start_blocking(bin: String) -> Result { + // Keep the Unix socket path short and pg_ctl's option path space-free. + let directory = tempfile::Builder::new() + .prefix("compact-wal-") + .tempdir_in("/tmp")?; + let data = directory.path().join("data"); + let port = std::net::TcpListener::bind("127.0.0.1:0")? + .local_addr()? + .port(); + let mut primary = Self { + url: String::new(), + bin: bin.into(), + data, + running: false, + _directory: directory, + }; + primary.command( + "initdb", + &[ + "-D", + primary.data.to_str().context("database path")?, + "-A", + "trust", + "--no-locale", + "-E", + "UTF8", + ], + )?; + // Mark before starting so any partially successful startup is stopped. + primary.running = true; + primary.control(&[ + "-l", + primary.data.with_extension("log").to_str().context("log path")?, + "-o", + &format!( + "-h 127.0.0.1 -p {port} -k {} -c fsync=on -c full_page_writes=on -c synchronous_commit=on", + primary._directory.path().display() + ), + "-w", + "start", + ])?; + let username = Command::new("id").arg("-un").output()?; + ensure!( + username.status.success(), + "could not identify database owner" + ); + let username = String::from_utf8(username.stdout)?; + let mut url = url::Url::parse(&format!("postgresql://127.0.0.1:{port}/postgres"))?; + url.set_username(username.trim()) + .map_err(|_| anyhow::anyhow!("invalid database owner"))?; + primary.url = url.to_string(); + Ok(primary) + } + + fn command(&self, binary: &str, args: &[&str]) -> Result<()> { + let output = Command::new(self.bin.join(binary)).args(args).output()?; + ensure!( + output.status.success(), + "{binary} failed: {} {}", + String::from_utf8_lossy(&output.stdout), + String::from_utf8_lossy(&output.stderr) + ); + Ok(()) + } + + fn control(&self, args: &[&str]) -> Result<()> { + let mut all = vec![ + "-D", + self.data.to_str().context("database path")?, + "-t", + "10", + ]; + all.extend_from_slice(args); + self.command("pg_ctl", &all) + } + + fn stop(&mut self) -> Result<()> { + if self.running { + self.control(&["-m", "immediate", "-w", "stop"])?; + self.running = false; + } + Ok(()) + } + + pub async fn close(mut self) -> Result<()> { + tokio::task::spawn_blocking(move || self.stop()).await? + } +} + +impl Drop for Primary { + fn drop(&mut self) { + let _ = self.stop(); + } +} diff --git a/crates/qbit-prism-server/tests/support/window_fixture.rs b/crates/qbit-prism-server/tests/support/window_fixture.rs new file mode 100644 index 000000000..a5a3ce67b --- /dev/null +++ b/crates/qbit-prism-server/tests/support/window_fixture.rs @@ -0,0 +1,378 @@ +//! Deterministic, production-shaped payout-window fixture for the JSONB +//! ceiling gate (issue #264). +//! +//! The row shape follows the #258 measurement fixture (`share()` in +//! `tests/prism_postgres_candidate_gate.py` and `tests/perf/candidate_storage.py` +//! at commit `f4aff1b40893f7886096c09989c50aebc0905e60`) mapped onto the native +//! [`AcceptedShare`] and the `qbit_share_ledger` row. +//! +//! Two deliberate deviations from that Python fixture are documented inline +//! below: the share difficulty is scaled down so that exactly `n` shares fill +//! the payout window, and `share_id` carries a fixed-width index. Both are +//! compensated by `miner_id` padding so that the serialized share stays at +//! [`TARGET_SHARE_BYTES`] regardless of `n`. +//! +//! Everything here is a pure function of the share index: no wall clock and no +//! OS randomness, so the same parameters always produce byte-identical shares. + +use anyhow::{ensure, Context, Result}; +use qbit_prism::AcceptedShare; +use sqlx::{PgPool, Row}; +use std::time::Instant; + +/// `Ledger::snapshot` walks the ledger backwards until the summed +/// `share_difficulty` reaches `WINDOW_MULTIPLIER x network_difficulty` +/// (`crates/qbit-prism-server/src/ledger.rs`, `snapshot`). +pub const WINDOW_MULTIPLIER: u128 = 8; + +/// The in-process fake node serves the stock `207fffff` template bits, which +/// `codec::scaled_target_difficulty` maps to this network difficulty. The +/// coordinator feeds it straight into `Ledger::snapshot`. +pub const FAKE_NODE_NETWORK_DIFFICULTY: u128 = 1_000_000; + +/// Total window weight the snapshot walk has to accumulate. +pub const WINDOW_WEIGHT: u128 = FAKE_NODE_NETWORK_DIFFICULTY * WINDOW_MULTIPLIER; + +/// Serialized size of one native `AcceptedShare`, in bytes, at share index 1. +/// The measured production figure behind the #264 readiness review is ~581 B +/// of JSON text per share (372,257 shares, ~232 MB canonical). +pub const TARGET_SHARE_BYTES: usize = 581; + +/// Acceptance band for the measured average, asserted by the fixture's own +/// test. The average drifts a few tenths of a byte above `TARGET_SHARE_BYTES` +/// because `share_seq` is a bare JSON number and widens from 1 to 6 digits. +pub const SHARE_BYTES_BAND: (f64, f64) = (560.0, 620.0); + +/// 2023-11-14T22:13:20Z. Far enough in the past that every fixture share +/// satisfies the snapshot anchor (`max(ledger clock, now)`) at any run date. +const JOB_ISSUED_AT_BASE_MS: i64 = 1_700_000_000_000; +/// #258 puts `accepted_at_ms` exactly one millisecond after `job_issued_at_ms`. +const ACCEPTED_AT_OFFSET_MS: i64 = 1; +/// Distinct payout recipients, as in #258 (`miner_id = f"m{index % 5}"`). +const RECIPIENTS: u64 = 5; +/// #258 records a small `network_difficulty` on the row itself; it is display +/// metadata and is unrelated to the window weight above. +const ROW_NETWORK_DIFFICULTY: u128 = 1000; +/// The fake node's tip height. #258 used a placeholder `9`; the byte width +/// difference is absorbed by the `miner_id` padding. +const TEMPLATE_HEIGHT: u64 = 100; +const JOB_ID: &str = "job-a"; +/// #258 gives every share the same `order_key`, so the payout universe stays +/// at `RECIPIENTS` accounts and the bundle size is dominated by the window. +const ORDER_KEY: &str = "k"; +/// Zero-padded width of the share index inside `share_id`. #258 uses the bare +/// index; a fixed width keeps the serialized share size independent of `n`, +/// which is what makes the gate's two-point linear fit trustworthy. +const SHARE_ID_INDEX_WIDTH: usize = 12; +/// Rows per server-side `INSERT ... SELECT`. One statement per batch keeps the +/// load to a handful of round trips even at 400,000 shares. +const LOAD_BATCH_ROWS: u64 = 50_000; + +/// A window of exactly `share_count` production-shaped shares. +#[derive(Clone, Debug)] +pub struct WindowPlan { + share_count: u64, + share_difficulty: u128, + miner_pad: usize, +} + +/// Throughput of one [`WindowPlan::load`]. +#[derive(Clone, Debug)] +pub struct LoadStats { + pub rows: u64, + pub seconds: f64, + pub rows_per_second: f64, + pub serialized_bytes: u64, +} + +impl WindowPlan { + /// `share_count` must divide [`WINDOW_WEIGHT`] exactly so that the snapshot + /// walk stops on exactly the requested number of shares. + /// + /// The alternative was to keep #258's 21-digit `share_difficulty` and scale + /// the fake node's `bits` until `n` such shares fill the window. That needs + /// a network difficulty of ~5e24 at 400,000 shares, which the 24-bit + /// compact-bits mantissa cannot express exactly, so the window length would + /// no longer be an exact function of `n`. Padding `miner_id` instead keeps + /// both the window length and the bytes per share exact. + pub fn new(share_count: u64) -> Result { + ensure!(share_count > 0, "share count must be positive"); + let weight = u128::from(share_count); + ensure!( + weight <= WINDOW_WEIGHT, + "share count {share_count} exceeds the window weight {WINDOW_WEIGHT}; \ + each share needs a difficulty of at least 1" + ); + ensure!( + WINDOW_WEIGHT.is_multiple_of(weight), + "share count {share_count} must divide the window weight {WINDOW_WEIGHT} exactly, \ + otherwise the snapshot window would not contain exactly {share_count} shares" + ); + let share_difficulty = WINDOW_WEIGHT / weight; + let mut plan = Self { + share_count, + share_difficulty, + miner_pad: 0, + }; + // Pad `miner_id` (as #258 does with `row["miner_id"] += "x" * 240`) + // until the share at index 1 serializes to exactly TARGET_SHARE_BYTES. + // The probe uses a fixed index so the padding, and therefore the bytes + // per share, are identical at every share count. + let bare = serde_json::to_vec(&plan.share(1))?.len(); + ensure!( + bare < TARGET_SHARE_BYTES, + "unpadded share is already {bare} bytes, over the {TARGET_SHARE_BYTES}-byte target" + ); + plan.miner_pad = TARGET_SHARE_BYTES - bare; + Ok(plan) + } + + pub fn share_count(&self) -> u64 { + self.share_count + } + + pub fn share_difficulty(&self) -> u128 { + self.share_difficulty + } + + /// The `network_difficulty` argument for `Ledger::snapshot`, chosen so the + /// window is exactly `share_count` shares wide. Equal to the fake node's + /// difficulty by construction, so the coordinator's own refresh agrees. + pub fn window_network_difficulty(&self) -> u128 { + FAKE_NODE_NETWORK_DIFFICULTY + } + + fn miner_id(&self, index: u64) -> String { + format!("m{}{}", index % RECIPIENTS, "x".repeat(self.miner_pad)) + } + + fn p2mr_program_hex(index: u64) -> String { + // Five distinct 32-byte programs, one per recipient, so the payout + // builder sees five real accounts. #258 reuses "ab" * 32 everywhere; + // the byte width is identical. + format!("{:02x}", 0xaa + (index % RECIPIENTS)).repeat(32) + } + + fn share_id(index: u64) -> String { + // The non-ASCII character is deliberate: #258 uses it to keep the + // UTF-8 byte length above the character length, as real worker names do. + format!( + "miner-{}:é{:0width$}", + index % RECIPIENTS, + index, + width = SHARE_ID_INDEX_WIDTH + ) + } + + /// The share the fixture writes at `share_seq == index` (1-based). + pub fn share(&self, index: u64) -> AcceptedShare { + AcceptedShare { + share_seq: index, + share_id: Self::share_id(index), + miner_id: self.miner_id(index), + order_key: ORDER_KEY.to_owned(), + p2mr_program_hex: Self::p2mr_program_hex(index), + share_difficulty: self.share_difficulty, + network_difficulty: ROW_NETWORK_DIFFICULTY, + template_height: TEMPLATE_HEIGHT, + job_id: JOB_ID.to_owned(), + job_issued_at_ms: JOB_ISSUED_AT_BASE_MS + i64::try_from(index).unwrap_or(i64::MAX), + accepted_at_ms: JOB_ISSUED_AT_BASE_MS + + ACCEPTED_AT_OFFSET_MS + + i64::try_from(index).unwrap_or(i64::MAX), + ntime: 1_700_000_000, + credit_policy: None, + } + } + + /// Exact average serialized size of the whole window, in bytes per share. + pub fn average_share_bytes(&self) -> Result { + let mut total = 0u64; + for index in 1..=self.share_count { + total += u64::try_from(serde_json::to_vec(&self.share(index))?.len())?; + } + Ok(total as f64 / self.share_count as f64) + } + + /// Load the whole window with server-side batched inserts. `Ledger::append` + /// costs milliseconds per share and would need hours at 400,000; this runs + /// `LOAD_BATCH_ROWS` rows per statement with no per-row round trip. + pub async fn load(&self, pool: &PgPool, writer_id: &str) -> Result { + let existing: i64 = sqlx::query_scalar("SELECT count(*) FROM qbit_share_ledger") + .fetch_one(pool) + .await?; + ensure!( + existing == 0, + "window fixture needs an empty qbit_share_ledger, found {existing} rows" + ); + let started = Instant::now(); + let mut first = 1u64; + while first <= self.share_count { + let last = (first + LOAD_BATCH_ROWS - 1).min(self.share_count); + // No INSERT trigger guards qbit_share_ledger; the trigger added by + // migration 002 only rejects UPDATE, DELETE and TRUNCATE. + sqlx::query( + "INSERT INTO qbit_share_ledger(share_seq,share_id,miner_id,payout_order_key,\ + p2mr_program,share_difficulty,network_difficulty,template_height,job_id,\ + job_issued_at,ntime,accepted_at,credit_policy,accepted,writer_id,writer_epoch) \ + SELECT i,\ + 'miner-'||(i%$3)::text||':é'||lpad(i::text,$4,'0'),\ + 'm'||(i%$3)::text||$5,\ + $6,\ + decode(repeat(lpad(to_hex(170+(i%$3)::int),2,'0'),32),'hex'),\ + $7::text::numeric,$8::text::numeric,$9,$10,\ + to_timestamp(($11+i)::double precision/1000),$12,\ + to_timestamp(($13+i)::double precision/1000),\ + NULL,true,$14,0 \ + FROM generate_series($1::bigint,$2::bigint) AS g(i)", + ) + .bind(i64::try_from(first)?) + .bind(i64::try_from(last)?) + .bind(i64::try_from(RECIPIENTS)?) + .bind(i32::try_from(SHARE_ID_INDEX_WIDTH)?) + .bind("x".repeat(self.miner_pad)) + .bind(ORDER_KEY) + .bind(self.share_difficulty.to_string()) + .bind(ROW_NETWORK_DIFFICULTY.to_string()) + .bind(i64::try_from(TEMPLATE_HEIGHT)?) + .bind(JOB_ID) + .bind(JOB_ISSUED_AT_BASE_MS) + .bind(1_700_000_000i64) + .bind(JOB_ISSUED_AT_BASE_MS + ACCEPTED_AT_OFFSET_MS) + .bind(writer_id) + .execute(pool) + .await + .with_context(|| format!("loading shares {first}..={last}"))?; + // The same backfill migration 002 runs for pre-existing history. + sqlx::query( + "INSERT INTO qbit_prism_share_hashes(header_hash,share_id) \ + SELECT encode(sha256(convert_to(share_id,'UTF8')),'hex'),share_id \ + FROM qbit_share_ledger WHERE share_seq BETWEEN $1 AND $2 \ + ON CONFLICT DO NOTHING", + ) + .bind(i64::try_from(first)?) + .bind(i64::try_from(last)?) + .execute(pool) + .await?; + first = last + 1; + } + sqlx::query("SELECT setval(pg_get_serial_sequence('qbit_share_ledger','share_seq'),$1)") + .bind(i64::try_from(self.share_count)?) + .execute(pool) + .await?; + let seconds = started.elapsed().as_secs_f64(); + let serialized_bytes = + (self.average_share_bytes()? * self.share_count as f64).round() as u64; + Ok(LoadStats { + rows: self.share_count, + seconds, + rows_per_second: self.share_count as f64 / seconds.max(f64::MIN_POSITIVE), + serialized_bytes, + }) + } + + /// Round-trip check: every loaded row must decode back to [`Self::share`]. + /// `land_candidate` re-reads the window from SQL and demands exact equality + /// with the bundle, so a mismatch here would fail the landing phase later. + pub async fn verify_round_trip(&self, pool: &PgPool, indexes: &[u64]) -> Result<()> { + for &index in indexes { + let row = sqlx::query( + "SELECT share_seq,share_id,miner_id,payout_order_key,\ + encode(p2mr_program,'hex') AS program,share_difficulty::text AS difficulty,\ + network_difficulty::text AS network_difficulty,template_height,job_id,\ + (extract(epoch FROM job_issued_at)*1000)::bigint AS job_issued_at_ms,\ + (extract(epoch FROM accepted_at)*1000)::bigint AS accepted_at_ms,\ + ntime,credit_policy FROM qbit_share_ledger WHERE share_seq=$1", + ) + .bind(i64::try_from(index)?) + .fetch_one(pool) + .await + .with_context(|| format!("share {index} is missing"))?; + let actual = AcceptedShare { + share_seq: u64::try_from(row.try_get::("share_seq")?)?, + share_id: row.try_get("share_id")?, + miner_id: row.try_get("miner_id")?, + order_key: row.try_get("payout_order_key")?, + p2mr_program_hex: row.try_get("program")?, + share_difficulty: row.try_get::("difficulty")?.parse()?, + network_difficulty: row.try_get::("network_difficulty")?.parse()?, + template_height: u64::try_from(row.try_get::("template_height")?)?, + job_id: row.try_get("job_id")?, + job_issued_at_ms: row.try_get("job_issued_at_ms")?, + accepted_at_ms: row.try_get("accepted_at_ms")?, + ntime: u32::try_from(row.try_get::("ntime")?)?, + credit_policy: row.try_get("credit_policy")?, + }; + ensure!( + actual == self.share(index), + "share {index} did not round-trip: stored {actual:?} expected {:?}", + self.share(index) + ); + } + Ok(()) + } +} + +#[test] +fn fixture_shares_are_production_sized_and_deterministic() -> Result<()> { + // A few thousand shares, as required by #264: assert the average bytes per + // share of the native serialized form and print the exact figure. + let plan = WindowPlan::new(4_000)?; + let average = plan.average_share_bytes()?; + println!( + "window_fixture: n=4000 share_difficulty={} miner_pad={} average_share_bytes={average:.3}", + plan.share_difficulty(), + plan.miner_pad + ); + assert!( + average >= SHARE_BYTES_BAND.0 && average <= SHARE_BYTES_BAND.1, + "average {average:.3} B/share is outside the {:?} band", + SHARE_BYTES_BAND + ); + assert_eq!( + serde_json::to_vec(&plan.share(1))?.len(), + TARGET_SHARE_BYTES, + "share 1 must land exactly on the byte target" + ); + assert_eq!(plan.share(7), plan.share(7), "shares must be deterministic"); + assert_eq!( + serde_json::to_vec(&WindowPlan::new(4_000)?.share(9))?, + serde_json::to_vec(&plan.share(9))?, + "two plans with the same parameters must produce byte-identical shares" + ); + Ok(()) +} + +#[test] +fn fixture_share_size_is_independent_of_the_window_size() -> Result<()> { + // The gate fits size(n) = a + b*n through two reduced sizes and projects to + // 400,000. That is only sound if bytes per share does not move with n. + let mut sizes = Vec::new(); + for count in [5_000u64, 20_000, 50_000, 400_000] { + let plan = WindowPlan::new(count)?; + let one = serde_json::to_vec(&plan.share(1))?.len(); + sizes.push((count, one, plan.share_difficulty(), plan.miner_pad)); + } + println!("window_fixture: (n, bytes(share 1), share_difficulty, miner_pad) = {sizes:?}"); + assert!( + sizes.iter().all(|entry| entry.1 == TARGET_SHARE_BYTES), + "share 1 must be {TARGET_SHARE_BYTES} B at every window size: {sizes:?}" + ); + Ok(()) +} + +#[test] +fn fixture_rejects_window_sizes_it_cannot_fill_exactly() { + for bad in [0u64, 3, 7, 300_000, 8_000_001] { + assert!( + WindowPlan::new(bad).is_err(), + "share count {bad} cannot fill the window exactly and must be rejected" + ); + } + for good in [1u64, 5_000, 20_000, 50_000, 100_000, 200_000, 400_000] { + assert!( + WindowPlan::new(good).is_ok(), + "share count {good} divides the window weight and must be accepted" + ); + } +} diff --git a/crates/qbit-prism-server/tests/throughput_floor.rs b/crates/qbit-prism-server/tests/throughput_floor.rs new file mode 100644 index 000000000..25f55e7b0 --- /dev/null +++ b/crates/qbit-prism-server/tests/throughput_floor.rs @@ -0,0 +1,2349 @@ +//! End-to-end share-append throughput floor for the native ledger (issue #271). +//! +//! What this measures, and what it deliberately does not: +//! +//! * It measures `Ledger::append`, the real production share path. Every +//! append is one transaction that takes the cluster-wide `ORDER_LOCK` +//! advisory lock, checks writability, bumps the single-row ledger clock and +//! writes two rows (`src/ledger/window.rs`). Share appends are therefore +//! serialized cluster-wide, and the number this file produces is the rate at +//! which that serialized transaction retires. +//! * It is **not** comparable with the 2.x.x `qbit.prism.postgres-throughput.v1` +//! report, which timed one bulk `INSERT ... SELECT ... FROM generate_series`: +//! one statement, one commit, no advisory lock and no concurrency. That is +//! why this report carries a new schema name, `…v2`. +//! * It is **not** a Stratum-level number either. Nothing here speaks the +//! Stratum protocol; the frontend's parsing, hashing and RPC work is absent, +//! so the rate here is an upper bound on what a full pool can commit. +//! +//! The run seeds a payout window with the #264 fixture generator, then appends +//! shares at several concurrency levels (1, 2 and 4 appenders by default), each +//! appender holding its own `Ledger` with its own pool and its own +//! `instance_id`, which is how separate production frontends behave. While a +//! level runs, a connection outside every appender pool samples `pg_locks` for +//! backends waiting on `ORDER_LOCK`, so the report can say how much of the wall +//! clock was spent queueing on that one lock. +//! +//! Every figure is host-specific. The report records the CPU model, core count, +//! memory, OS, PostgreSQL version and build profile alongside the rates, and no +//! rate from one machine should be carried to another. +//! +//! ```text +//! PRISM_TEST_DATABASE_URL=postgres://prism_test:prism_test@127.0.0.1:5432/prism_test \ +//! cargo test -p qbit-prism-server --test throughput_floor -- --nocapture +//! ``` +//! +//! # Run this against a database nothing else is appending to +//! +//! The per-run schema isolates the *tables*, but `ORDER_LOCK` is a PostgreSQL +//! advisory lock, and an advisory lock is scoped to the **database**, not to a +//! schema. Any other backend appending shares into the same database — another +//! copy of this test, a developer's PRISM, a staging frontend — serializes +//! against this run's appenders and roughly halves the rate this file reports. +//! So: point `PRISM_TEST_DATABASE_URL` at a database that is doing nothing +//! else, ideally a disposable one. +//! +//! Because "ideally" is not "always", the run does not rely on the operator +//! getting that right. Every connection it opens carries a per-run +//! `application_name` (`throughput-`), and the lock sampler splits what +//! it sees into this run's waiters and everyone else's. A level during which +//! any foreign backend held or waited on `ORDER_LOCK` is marked +//! `contaminated: true`, in the level, at the top of the report and in the +//! failure message. Such a level is still measured and still gated against the +//! floor — the rate it recorded is the rate the appends actually achieved — but +//! the number describes two workloads sharing one lock rather than this one, +//! and nothing downstream should treat it as a property of the append path. +//! +//! `contaminated` is `null`, not `false`, on a level whose sampler could not +//! look (an error, or a level that ended before the first poll returned). +//! "Nobody checked" is not "nothing was there". +//! +//! # A cancelled run leaves its schema behind +//! +//! Cleanup is an ordinary `await` on every exit path, so a passing run, a +//! failing run and a panicking run all drop their schema. A **cancelled** run +//! does not: `SIGINT` (Ctrl-C) kills the process before the drop runs, and the +//! `prism_throughput_` schema survives with its seeded window in it. +//! Installing a signal handler would mean a new dependency, which this test is +//! not allowed to add, so the leak is documented rather than fixed. It is the +//! same behaviour as the sibling database tests in this crate. +//! +//! Find and drop the leftovers by hand: +//! +//! ```text +//! SELECT nspname FROM pg_namespace WHERE nspname LIKE 'prism\_throughput\_%'; +//! DROP SCHEMA prism_throughput_0ee059ccc0314cbb8a814c0319f846d5 CASCADE; +//! ``` +//! +//! Only do that when no run is in flight: a live run's schema matches the same +//! pattern. + +use anyhow::{bail, ensure, Context, Result}; +use chrono::{DateTime, SecondsFormat, Utc}; +use qbit_prism_server::ledger::Ledger; +use qbit_prism_test_gate as gate; +use serde_json::{Map, Value}; +use sqlx::{postgres::PgPoolOptions, PgPool, Row}; +use std::{ + collections::BTreeSet, + io::Write, + path::{Path, PathBuf}, + sync::{ + atomic::{AtomicBool, Ordering}, + Arc, + }, + time::{Duration, Instant}, +}; +use tokio::sync::{Barrier, Notify}; +use uuid::Uuid; + +/// The #264 window generator, shared read-only with the JSONB ceiling gate. +/// `dead_code` is allowed because each test binary that includes the module +/// gets its own copy and uses a different part of it; this file seeds and +/// builds shares but never calls the round-trip verifier, and warning about +/// that would only push the fixture towards whichever consumer compiled last. +#[path = "support/window_fixture.rs"] +#[allow(dead_code)] +mod window_fixture; +use window_fixture::{WindowPlan, WINDOW_WEIGHT}; + +// --------------------------------------------------------------------------- +// Constants +// --------------------------------------------------------------------------- + +/// Report schema. Deliberately not `…v1`: see the module docs for why the two +/// generations of this report cannot be compared. +const REPORT_SCHEMA: &str = "qbit.prism.postgres-throughput.v2"; + +/// `ORDER_LOCK` is `0x505249534d000002` (`src/ledger.rs`). PostgreSQL splits a +/// 64-bit advisory key across `pg_locks.classid` (the high half) and +/// `pg_locks.objid` (the low half), with `objsubid = 1` marking the two-int +/// form. These are those two halves, spelled out so a change to the key in +/// production shows up here as a sampler that suddenly sees no waiters rather +/// than as a silently wrong number. +const ORDER_LOCK_CLASSID: i64 = 0x5052_4953; +const ORDER_LOCK_OBJID: i64 = 0x4d00_0002; +const ORDER_LOCK_OBJSUBID: i32 = 1; + +/// The CI floor, calibrated on the runner that enforces it. +/// +/// It comes from this test's own summary line in four runs of the +/// `prism-native-postgres` job on `blacksmith-2vcpu-ubuntu-2404`: a debug +/// build, the job's PostgreSQL 16 service, a 20k-share window and 2,000 appends +/// per level. The slowest level in each run was 1,290, 847, 1,242 and 1,200 +/// shares/s, so runs differ by about a third from one to the next. 250 is about +/// 30% of the slowest level seen. That leaves enough headroom for a noisy +/// runner, while a 3-4x collapse of the append path still fails the build. +/// +/// Those runs kept the service's data directory on disk; it has been a tmpfs +/// since #677. On the tmpfs the slowest level was 1,084, 1,056, 1,103 and 670 +/// shares/s in four runs of the job, and 702 in a probe that also loaded +/// `auto_explain`, so 250 is about 37% of the slowest. A fifth run fell to 132 +/// while the runner itself stalled, as disk runs had before (#683). 250 stays: +/// a floor low enough to pass such a stall would miss the collapse it is for. +/// Instead the CI-sized test measures a level below the floor a second time, +/// and only that pass decides: a stall of a few seconds misses once, a +/// collapse twice. +/// +/// Re-derive it from the same summary lines, counting their +/// `first_passes_below_floor` too, if the runner, the PostgreSQL service or +/// the append path changes. On any other host, +/// `QBIT_PRISM_MIN_SHARES_PER_SEC` overrides it. +const CI_MIN_SHARES_PER_SEC: f64 = 250.0; + +/// Each appender appends sequentially, so one connection would do; `connect` +/// floors the pool at 2 anyway (`src/ledger/connect.rs`). Keeping it at the +/// floor means the pool can never be what limits a level. +const APPENDER_POOL_CONNECTIONS: u32 = 2; + +/// Pool for the seeding ledger. The fixture loads in batched statements over a +/// single connection; the floor of 2 is plenty. +const SEED_POOL_CONNECTIONS: u32 = 2; + +const WRITER_SEED: &str = "throughput-seed"; + +const MIN_SHARES_VAR: &str = "QBIT_PRISM_MIN_SHARES_PER_SEC"; +const APPENDERS_VAR: &str = "QBIT_PRISM_THROUGHPUT_APPENDERS"; +const SHARES_VAR: &str = "QBIT_PRISM_THROUGHPUT_SHARES"; +const WINDOW_SHARES_VAR: &str = "QBIT_PRISM_THROUGHPUT_WINDOW_SHARES"; +const LOCK_SAMPLE_VAR: &str = "QBIT_PRISM_THROUGHPUT_LOCK_SAMPLE_MS"; +const REPORT_VAR: &str = "QBIT_PRISM_THROUGHPUT_REPORT"; + +const DEFAULT_APPENDERS: &str = "1,2,4"; +const DEFAULT_LOCK_SAMPLE_MS: u64 = 10; +/// The sampler's interval is bounded at both ends, at entry. +/// +/// Below 1 ms there is no interval at all, and the "sample" would be a busy +/// loop competing with the appenders for the server. Above 1000 ms the poll can +/// no longer resolve what it exists to measure: an `ORDER_LOCK` wait lasts +/// milliseconds, so a sampler that looks once a second mostly looks between the +/// waits and reports a handful of points as if they were a series. A coarse +/// interval also used to be dead time at the end of every level; the sampler +/// now wakes on demand, but the ceiling stays because the *measurement* is what +/// a second-scale interval ruins. +const MIN_LOCK_SAMPLE_MS: u64 = 1; +const MAX_LOCK_SAMPLE_MS: u64 = 1_000; +/// An appender count above this is refused: the levels are meant to model a +/// handful of production frontends, and a four-figure count would exhaust the +/// server's connection slots long before it said anything about the lock. +const MAX_APPENDERS: u64 = 64; + +const NOTES: &str = "v2 measures per-share transactional Ledger::append: one BEGIN, one \ + ORDER_LOCK advisory lock, one writability check, one ledger-clock update, \ + two INSERTs and one COMMIT per share. It is NOT comparable with v1, which \ + timed a single bulk INSERT ... SELECT ... FROM generate_series with no \ + advisory lock, no concurrency and one commit for the whole batch. Rates are \ + host-specific; read them together with the machine, postgres_version, \ + durability and build_profile fields."; + +// --------------------------------------------------------------------------- +// Integration gate +// --------------------------------------------------------------------------- +// +// Whether the database tests here run, fail or skip is decided by the shared +// integration gate (`qbit_prism_test_gate`, #286), the one decision table every +// gated test in the workspace uses. With no database URL, a test fails under +// the integration switch or in the `prism-native-postgres` job, and skips +// otherwise; CI's database-free `rust-tests` job therefore skips it, while the +// native job runs it against PostgreSQL 16 with no workflow change. +// `share_append_throughput_floor` is listed in `test/prism-gated-tests.txt`, so +// that job also proves it executed. The full-size test is not selected in CI +// and is not listed. See `docs/prism-integration-test-gate.md`. + +// --------------------------------------------------------------------------- +// Environment parsing (EP-VALIDATION) +// --------------------------------------------------------------------------- + +/// A variable that is set but unreadable is an error, never an absent one: +/// treating a non-Unicode `QBIT_PRISM_MIN_SHARES_PER_SEC` as unset would +/// quietly substitute the compiled-in floor for a deliberately configured one. +fn env_raw(name: &str) -> Result> { + match std::env::var(name) { + Ok(value) => Ok(Some(value)), + Err(std::env::VarError::NotPresent) => Ok(None), + Err(error) => bail!("{name} is set but not readable: {error}"), + } +} + +/// The floor, in shares per second. Must be a finite decimal at or above zero. +/// +/// `f64::from_str` accepts `nan`, `inf` and `-inf`, and every one of them would +/// make the comparison against the measured rate meaningless (`NaN` compares +/// false against everything, so a `nan` floor would pass every run). They are +/// rejected here, at entry, before any database work. +fn parse_minimum(raw: Option<&str>, fallback: f64) -> Result { + let Some(raw) = raw else { + return Ok(fallback); + }; + let trimmed = raw.trim(); + ensure!( + !trimmed.is_empty(), + "{MIN_SHARES_VAR} is set to an empty value; it must be a finite decimal >= 0, \ + or unset to use the compiled-in floor" + ); + let value: f64 = trimmed.parse().with_context(|| { + format!("{MIN_SHARES_VAR}={trimmed:?} is not a decimal number of shares per second") + })?; + ensure!( + value.is_finite(), + "{MIN_SHARES_VAR}={trimmed:?} is not finite; a non-finite floor can never be compared \ + against a measured rate" + ); + ensure!( + value >= 0.0, + "{MIN_SHARES_VAR}={trimmed:?} is negative; the floor must be >= 0" + ); + Ok(value) +} + +/// A positive count. Zero and negative values are refused rather than clamped, +/// because "append zero shares" would report an infinite rate. +fn parse_positive_u64(name: &str, raw: Option<&str>, fallback: u64) -> Result { + let Some(raw) = raw else { + return Ok(fallback); + }; + let trimmed = raw.trim(); + ensure!( + !trimmed.is_empty(), + "{name} is set to an empty value; it must be a positive integer" + ); + let value: u64 = trimmed + .parse() + .with_context(|| format!("{name}={trimmed:?} is not a positive integer"))?; + ensure!(value > 0, "{name}={trimmed:?} must be greater than zero"); + Ok(value) +} + +/// The lock sampling interval, in milliseconds, bounded to +/// `MIN_LOCK_SAMPLE_MS..=MAX_LOCK_SAMPLE_MS`. +/// +/// Checked here, at entry, alongside every other variable and before any +/// connection is opened (EP-VALIDATION), so a run configured to produce +/// meaningless wait figures costs nothing rather than costing a full seeding +/// pass first. +fn parse_lock_sample_ms(raw: Option<&str>, fallback: u64) -> Result { + let value = parse_positive_u64(LOCK_SAMPLE_VAR, raw, fallback)?; + ensure!( + (MIN_LOCK_SAMPLE_MS..=MAX_LOCK_SAMPLE_MS).contains(&value), + "{LOCK_SAMPLE_VAR}={value} is outside {MIN_LOCK_SAMPLE_MS}..={MAX_LOCK_SAMPLE_MS} \ + milliseconds; an interval coarser than a second cannot resolve millisecond lock \ + waits, and would report a few stray points as if they were a series" + ); + Ok(value) +} + +/// The window size must divide the fixture's window weight exactly, otherwise +/// the payout window the seeded shares form would not be the requested number +/// of shares wide (`tests/support/window_fixture.rs`). `WindowPlan::new` says +/// the same thing, but it says it after the schema exists; saying it here keeps +/// the promise that a misconfigured run costs nothing. +fn parse_window_shares(raw: Option<&str>, fallback: u64) -> Result { + let value = parse_positive_u64(WINDOW_SHARES_VAR, raw, fallback)?; + let weight = u128::from(value); + ensure!( + weight <= WINDOW_WEIGHT, + "{WINDOW_SHARES_VAR}={value} exceeds the fixture window weight {WINDOW_WEIGHT}; \ + each seeded share needs a difficulty of at least 1" + ); + ensure!( + WINDOW_WEIGHT.is_multiple_of(weight), + "{WINDOW_SHARES_VAR}={value} must divide the fixture window weight {WINDOW_WEIGHT} \ + exactly, otherwise the seeded payout window would not be {value} shares wide" + ); + Ok(value) +} + +/// The concurrency levels, as a comma-separated list of appender counts. +/// +/// Duplicates are refused: two levels with the same count would hand their +/// appenders the same `writer_id`s, and the per-appender committed counts in +/// the report would then mix two levels together. +fn parse_appenders(raw: Option<&str>) -> Result> { + let raw = raw.unwrap_or(DEFAULT_APPENDERS); + let trimmed = raw.trim(); + ensure!( + !trimmed.is_empty(), + "{APPENDERS_VAR} is set to an empty value; it must be a comma-separated list of \ + appender counts, for example {DEFAULT_APPENDERS:?}" + ); + let mut levels = Vec::new(); + for field in trimmed.split(',') { + let field = field.trim(); + ensure!( + !field.is_empty(), + "{APPENDERS_VAR}={trimmed:?} has an empty entry; every entry must be an appender \ + count between 1 and {MAX_APPENDERS}" + ); + let count: u64 = field.parse().with_context(|| { + format!("{APPENDERS_VAR} entry {field:?} is not a positive integer") + })?; + ensure!( + (1..=MAX_APPENDERS).contains(&count), + "{APPENDERS_VAR} entry {field:?} must be between 1 and {MAX_APPENDERS}" + ); + let count = u32::try_from(count)?; + ensure!( + !levels.contains(&count), + "{APPENDERS_VAR}={trimmed:?} repeats the appender count {count}; every level needs \ + a distinct count so its appenders get distinct writer IDs" + ); + levels.push(count); + } + Ok(levels) +} + +/// Splits a level's share budget across its appenders. +/// +/// When the budget does not divide evenly the remainder goes to the lowest +/// appender indexes, one share each, so the split is a pure function of +/// `(total, appenders)` and two runs of the same configuration append exactly +/// the same shares. The level's aggregate rate is over the whole budget either +/// way, so an uneven split does not bias it. +fn distribute(total: u64, appenders: u32) -> Vec { + let appenders = u64::from(appenders); + let base = total / appenders; + let remainder = total % appenders; + (0..appenders) + .map(|index| base + u64::from(index < remainder)) + .collect() +} + +/// `none` when nothing is streaming, `sync` when a commit can be made to wait +/// for a standby, `async` when standbys exist but none of them holds a commit. +/// +/// Both inputs matter. `pg_stat_replication.sync_state` names what each +/// connected standby is right now, and `synchronous_standby_names` names what +/// the server is configured to wait for. A non-empty `synchronous_standby_names` +/// with no connected standby is still `sync`: commits will block as soon as one +/// is expected, which is exactly the durability fact a throughput number has to +/// be read against. +fn derive_replication_mode(sync_states: &[String], standby_names: &str) -> &'static str { + if sync_states + .iter() + .any(|state| matches!(state.trim(), "sync" | "quorum")) + { + return "sync"; + } + if !standby_names.trim().is_empty() { + return "sync"; + } + if sync_states.is_empty() { + "none" + } else { + "async" + } +} + +/// Everything the run needs, validated before it touches a database. +#[derive(Clone, Debug)] +struct Config { + test_name: String, + /// The `application_name` every connection this run opens carries, so the + /// lock sampler can tell this run's backends from everyone else's + /// (EP-STATE). Unique per run, never per test name: two copies of the same + /// test against one database must not look like one run. + application_name: String, + window_shares: u64, + shares_per_level: u64, + appenders: Vec, + lock_sample: Duration, + minimum: f64, + minimum_source: String, + report_path: PathBuf, + /// Measure a level below the floor once more (#683): the CI-sized test + /// only, so the explicit full-size characterisation stays single-pass. + retry_below_floor: bool, +} + +/// How the test was selected, which decides what a missing database means. +/// +/// `Ci` is the test CI runs unasked: with no database it skips, and the gate +/// still fails it in the native job. `Explicit` is the `#[ignore]`d run an +/// operator selects by name, which must fail rather than skip. +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +enum Selection { + Ci, + Explicit, +} + +/// Per-test defaults, overridden by the `QBIT_PRISM_THROUGHPUT_*` variables. +#[derive(Clone, Copy, Debug)] +struct Defaults { + window_shares: u64, + shares_per_level: u64, +} + +impl Config { + fn from_env(test_name: &str, defaults: Defaults, selection: Selection) -> Result { + let window_shares = parse_window_shares( + env_raw(WINDOW_SHARES_VAR)?.as_deref(), + defaults.window_shares, + )?; + let shares_per_level = parse_positive_u64( + SHARES_VAR, + env_raw(SHARES_VAR)?.as_deref(), + defaults.shares_per_level, + )?; + let appenders = parse_appenders(env_raw(APPENDERS_VAR)?.as_deref())?; + let widest = appenders.iter().copied().max().unwrap_or(1); + ensure!( + shares_per_level >= u64::from(widest), + "{SHARES_VAR}={shares_per_level} is below the widest level's appender count \ + {widest}; every appender must get at least one share" + ); + let lock_sample_ms = + parse_lock_sample_ms(env_raw(LOCK_SAMPLE_VAR)?.as_deref(), DEFAULT_LOCK_SAMPLE_MS)?; + let raw_minimum = env_raw(MIN_SHARES_VAR)?; + let minimum = parse_minimum(raw_minimum.as_deref(), CI_MIN_SHARES_PER_SEC)?; + let minimum_source = match raw_minimum { + Some(_) => format!("{MIN_SHARES_VAR} environment variable"), + None => "compiled-in CI_MIN_SHARES_PER_SEC (calibrated on the CI runner)".to_owned(), + }; + let report_path = match env_raw(REPORT_VAR)? { + Some(raw) => { + let trimmed = raw.trim(); + ensure!( + !trimmed.is_empty(), + "{REPORT_VAR} is set to an empty value; it must be a writable file path" + ); + PathBuf::from(trimmed) + } + None => default_report_path(test_name), + }; + Ok(Self { + test_name: test_name.to_owned(), + application_name: format!("throughput-{}", Uuid::new_v4().simple()), + window_shares, + shares_per_level, + appenders, + lock_sample: Duration::from_millis(lock_sample_ms), + minimum, + minimum_source, + report_path, + retry_below_floor: selection == Selection::Ci, + }) + } +} + +/// `/target/prism-postgres-throughput-.json`. +/// +/// The path carries the test name because the two floor tests can run in +/// parallel threads of one binary (`--include-ignored`), and a single default +/// path meant whichever finished last silently overwrote the other. With a path +/// per test neither can lose, and the `test` field in the report says which one +/// wrote it even when an operator has redirected both with +/// `QBIT_PRISM_THROUGHPUT_REPORT`, which remains the explicit override. +fn default_report_path(test_name: &str) -> PathBuf { + let manifest = Path::new(env!("CARGO_MANIFEST_DIR")); + let root = manifest + .parent() + .and_then(Path::parent) + .unwrap_or(manifest) + .to_path_buf(); + root.join("target") + .join(format!("prism-postgres-throughput-{test_name}.json")) +} + +// --------------------------------------------------------------------------- +// Per-run schema +// --------------------------------------------------------------------------- + +/// A throwaway schema on the configured server, dropped on every exit path. +/// +/// The run creates tables, loads six-figure row counts and takes a cluster-wide +/// advisory lock; none of that may land in `public` or in another test's +/// schema. `search_path` is pushed into the URL rather than set per session, so +/// every pool `Ledger::connect` opens from that URL lands in the same place. +/// +/// The same trick carries the run's `application_name`: it goes into the URL, +/// which is the only thing `Ledger::connect` is handed, so the seeding ledger, +/// every appender ledger and the sampler all announce themselves with it +/// without any production code having to know this test exists. +struct Database { + admin: PgPool, + schema: String, + url: String, +} + +impl Database { + async fn open(raw: &str, application_name: &str) -> Result { + let mut url = url::Url::parse(raw)?; + url.query_pairs_mut() + .append_pair("application_name", application_name); + let admin = PgPool::connect(url.as_str()).await?; + // EP-STATE: everything downstream — which waiter is ours, which level + // is contaminated, whether the floor describes one workload — rests on + // the server having actually recorded this name. sqlx 0.8 parses + // `application_name` out of the URL and sends it in the startup packet, + // but "the library is supposed to" is not evidence, and a silently + // ignored parameter would make every foreign backend look like ours and + // every contaminated run look clean. So the run asks the server, once, + // and refuses to measure anything if the answer is wrong. + let announced: String = sqlx::query_scalar("SELECT current_setting('application_name')") + .fetch_one(&admin) + .await + .context("reading back the run's application_name")?; + if announced != application_name { + admin.close().await; + bail!( + "the server recorded application_name {announced:?} for a connection opened with \ + application_name={application_name:?}; without it the lock sampler cannot tell \ + this run's backends from any other, so no measurement here would be trustworthy" + ); + } + let schema = format!("prism_throughput_{}", Uuid::new_v4().simple()); + sqlx::query(&format!("CREATE SCHEMA {schema}")) + .execute(&admin) + .await?; + url.query_pairs_mut() + .append_pair("options", &format!("-csearch_path={schema}")); + Ok(Self { + admin, + schema, + url: url.to_string(), + }) + } + + /// EP-ERRORS: the schema goes away whether the run passed or failed, so a + /// red run leaves no `prism_throughput_%` schema on a server that outlives + /// it. + async fn close(self) -> Result<()> { + sqlx::query(&format!("DROP SCHEMA {} CASCADE", self.schema)) + .execute(&self.admin) + .await?; + self.admin.close().await; + Ok(()) + } +} + +// --------------------------------------------------------------------------- +// ORDER_LOCK wait sampling +// --------------------------------------------------------------------------- + +/// The sampler's stop signal: a flag it can test, plus a wake-up it can wait +/// on. +/// +/// The flag alone was not enough. The sampler used to test it only at the top +/// of the loop and then block in `sleep(interval)`, so every level paid up to +/// one whole interval of dead time after its appends had finished, and +/// `run_level` sat in `sampler.await` for all of it. Pairing the flag with a +/// `Notify` makes the sleep interruptible: `stop()` sets the flag *and* hands +/// the sampler a permit, `notify_one` stores that permit even if the sampler is +/// not waiting yet, so there is no window in which the signal can be lost +/// (EP-ERRORS: shutdown is prompt and guaranteed, not eventual). +#[derive(Default)] +struct SamplerStop { + flag: AtomicBool, + wake: Notify, +} + +impl SamplerStop { + fn stop(&self) { + self.flag.store(true, Ordering::Relaxed); + self.wake.notify_one(); + } + + fn stopped(&self) -> bool { + self.flag.load(Ordering::Relaxed) + } + + /// Sleeps for `interval`, or returns as soon as `stop` is called. + async fn sleep_until_stopped(&self, interval: Duration) { + tokio::select! { + _ = tokio::time::sleep(interval) => {} + _ = self.wake.notified() => {} + } + } +} + +/// What a level's sampler saw. Every figure is an estimate from a 100 Hz poll, +/// never an exact accounting, and the report labels it as such. +#[derive(Debug, Default)] +struct LockWaitSummary { + samples: u64, + total_waiters: u64, + max_waiters: u64, + /// Distinct `(pid, waitstart)` pairs. A wait shorter than the sampling + /// interval can be missed entirely, so this is a lower bound and is + /// reported as one. + episodes: BTreeSet<(i32, String)>, + /// Sum over samples of `waiters x (time since the previous sample)`, a + /// Riemann estimate of waiter-seconds. + waiter_seconds: f64, + /// The largest number of backends *not* from this run seen holding or + /// waiting on `ORDER_LOCK` in a single sample. + foreign_waiters_max: u64, + /// Distinct pids of backends not from this run seen holding or waiting on + /// `ORDER_LOCK` at any point in the level. + foreign_backends: BTreeSet, + /// Set when the sampler could not do its job. Then every figure above is + /// reported as `null`: an unsampled level has an unknown wait, not a zero + /// one (EP-OBSERVABILITY). + error: Option, +} + +impl LockWaitSummary { + /// The error to report, which is the sampler's own error if it had one and + /// otherwise the fact that it never completed a poll. + /// + /// A level that ends before the first query returns used to emit + /// `mean_waiters: 0.0`, `max_waiters: 0` and `estimated_waiter_seconds: + /// 0.0` beside `samples: 0`. That reads as "nobody ever waited on the lock" + /// when the truth is "nobody looked", which is precisely the zero-for- + /// unknown this file forbids elsewhere (EP-OBSERVABILITY). + fn effective_error(&self) -> Option { + self.error.clone().or_else(|| { + (self.samples == 0) + .then(|| "the level ended before the sampler completed a poll".to_owned()) + }) + } + + /// `Some(true)` when a foreign backend was seen, `Some(false)` when the + /// sampler looked and saw none, `None` when it could not look. `None` is + /// deliberate: "the sampler never ran" is not evidence that the database + /// was quiet, and rendering it as `false` would be the same lie as + /// rendering an unmeasured wait as zero. + fn contaminated(&self) -> Option { + match self.effective_error() { + Some(_) => None, + None => Some(!self.foreign_backends.is_empty()), + } + } + + fn to_json(&self, interval: Duration) -> Value { + let mut object = Map::new(); + object.insert( + "sample_interval_milliseconds".to_owned(), + Value::from(interval.as_millis() as u64), + ); + object.insert( + "method".to_owned(), + Value::from( + "pg_locks left-joined to pg_stat_activity, filtered to the ORDER_LOCK advisory \ + lock in this database, polled on a connection outside every appender pool; a \ + row is this run's when pg_stat_activity.application_name matches the run's \ + application_name, and foreign otherwise", + ), + ); + object.insert( + "estimate".to_owned(), + Value::from( + "sampled: waiter counts are point estimates, episodes are a lower bound because \ + a wait shorter than the interval can be missed, and waiter_seconds is a Riemann \ + sum over the samples", + ), + ); + // The foreign figures are about a *different* population from the four + // below them: this run's waiters are ungranted holders-in-waiting of + // ORDER_LOCK, while a foreign backend counts whether it is waiting on + // the lock or holding it, because either way it is serialising against + // this run's appends. + object.insert( + "foreign_scope".to_owned(), + Value::from( + "a backend is foreign when it is not part of this run; it counts whether it was \ + waiting on ORDER_LOCK or holding it, because either serialises against this \ + run's appends. A backend whose pg_stat_activity row this role may not read \ + counts as foreign, which is the safe direction", + ), + ); + let error = self.effective_error(); + match &error { + Some(error) => { + object.insert("error".to_owned(), Value::from(error.clone())); + for key in [ + "mean_waiters", + "max_waiters", + "episodes_lower_bound", + "estimated_waiter_seconds", + "foreign_waiters_max", + "foreign_backends_seen", + ] { + object.insert(key.to_owned(), Value::Null); + } + // A real sampler error leaves the sample count unknown too; a + // level that simply ended too early knows exactly how many + // polls completed, and it was none. + object.insert( + "samples".to_owned(), + match self.error { + Some(_) => Value::Null, + None => Value::from(self.samples), + }, + ); + } + None => { + object.insert("error".to_owned(), Value::Null); + object.insert("samples".to_owned(), Value::from(self.samples)); + let mean = self.total_waiters as f64 / self.samples as f64; + object.insert("mean_waiters".to_owned(), json_f64(mean)); + object.insert("max_waiters".to_owned(), Value::from(self.max_waiters)); + object.insert( + "episodes_lower_bound".to_owned(), + Value::from(self.episodes.len() as u64), + ); + object.insert( + "estimated_waiter_seconds".to_owned(), + json_f64(self.waiter_seconds), + ); + object.insert( + "foreign_waiters_max".to_owned(), + Value::from(self.foreign_waiters_max), + ); + object.insert( + "foreign_backends_seen".to_owned(), + Value::from(self.foreign_backends.len() as u64), + ); + } + } + object.insert( + "contaminated".to_owned(), + self.contaminated().map_or(Value::Null, Value::from), + ); + Value::Object(object) + } +} + +/// Polls for backends queued behind `ORDER_LOCK` until `stop` is signalled. +/// +/// The connection is the caller's, opened outside every appender pool, so the +/// sampler can never be the reason an appender waits for a connection. The +/// first query failure ends the sampling and is recorded; carrying on would +/// produce a series with an unknown hole in it. +/// +/// The query deliberately does not filter on `granted`. This run's *waiters* +/// are the ungranted rows, as before, but a foreign backend counts either way: +/// one that holds `ORDER_LOCK` is making this run's appenders wait just as +/// surely as one that is queued behind them. +async fn sample_order_lock( + pool: PgPool, + interval: Duration, + application_name: String, + stop: Arc, +) -> LockWaitSummary { + const SQL: &str = "SELECT l.pid, l.granted, l.waitstart, \ + (a.application_name IS NOT DISTINCT FROM $4) AS own \ + FROM pg_locks l \ + LEFT JOIN pg_stat_activity a ON a.pid = l.pid \ + WHERE l.locktype = 'advisory' \ + AND l.classid = $1::bigint::oid \ + AND l.objid = $2::bigint::oid \ + AND l.objsubid = $3 \ + AND l.database = (SELECT oid FROM pg_database WHERE datname = current_database())"; + let mut summary = LockWaitSummary::default(); + let mut previous = Instant::now(); + while !stop.stopped() { + let rows = sqlx::query(SQL) + .bind(ORDER_LOCK_CLASSID) + .bind(ORDER_LOCK_OBJID) + .bind(ORDER_LOCK_OBJSUBID) + .bind(&application_name) + .fetch_all(&pool) + .await; + let now = Instant::now(); + let elapsed = now.duration_since(previous).as_secs_f64(); + previous = now; + match rows { + Ok(rows) => { + let mut waiters = 0u64; + let mut foreign = 0u64; + for row in rows { + let pid: i32 = match row.try_get("pid") { + Ok(pid) => pid, + Err(error) => { + summary.error = Some(format!("pg_locks.pid is unreadable: {error}")); + return summary; + } + }; + let granted: bool = match row.try_get("granted") { + Ok(granted) => granted, + Err(error) => { + summary.error = + Some(format!("pg_locks.granted is unreadable: {error}")); + return summary; + } + }; + let own: bool = match row.try_get("own") { + Ok(own) => own, + Err(error) => { + summary.error = Some(format!( + "pg_stat_activity.application_name is unreadable: {error}" + )); + return summary; + } + }; + if !own { + foreign += 1; + summary.foreign_backends.insert(pid); + continue; + } + if granted { + continue; + } + waiters += 1; + // `waitstart` arrived in PostgreSQL 14 and can still be + // NULL for a lock whose wait had not been recorded when the + // snapshot was taken. + let waitstart = match row.try_get::>, _>("waitstart") { + Ok(Some(at)) => at.to_rfc3339_opts(SecondsFormat::Micros, true), + Ok(None) => "unknown".to_owned(), + Err(error) => { + summary.error = + Some(format!("pg_locks.waitstart is unreadable: {error}")); + return summary; + } + }; + summary.episodes.insert((pid, waitstart)); + } + summary.samples += 1; + summary.total_waiters += waiters; + summary.max_waiters = summary.max_waiters.max(waiters); + summary.waiter_seconds += waiters as f64 * elapsed; + summary.foreign_waiters_max = summary.foreign_waiters_max.max(foreign); + } + Err(error) => { + summary.error = Some(format!("ORDER_LOCK sampling failed: {error}")); + return summary; + } + } + stop.sleep_until_stopped(interval).await; + } + summary +} + +/// `pg_stat_statements` aggregates for the advisory-lock statement, when the +/// extension is installed. It is a low-overhead cross-check on the sampler: +/// `SELECT pg_advisory_xact_lock($1)` is almost entirely lock wait. +/// +/// The caveat, recorded in the report: `MIGRATION_LOCK` and `SETTLEMENT_LOCK` +/// share that statement text, so the aggregate covers all three keys. This run +/// takes the other two only while creating the schema, before any level starts +/// and before the per-level reset. +async fn read_lock_statements(pool: &PgPool) -> Value { + const QUERY_MATCH: &str = "%pg_advisory_xact_lock%"; + let mut object = Map::new(); + object.insert( + "statement".to_owned(), + Value::from("SELECT pg_advisory_xact_lock($1)"), + ); + object.insert( + "caveat".to_owned(), + Value::from( + "pg_stat_statements normalizes MIGRATION_LOCK, ORDER_LOCK and SETTLEMENT_LOCK to one \ + statement; a level's figures are the difference of two reads taken around it, so \ + they cover whatever that level's appends took, and the other two keys contribute \ + only if something else took them meanwhile. The rows are filtered to this database, \ + so advisory locks taken elsewhere on the same server are excluded; an advisory lock \ + is scoped to its database and cannot contend with this run", + ), + ); + let unavailable = |object: &mut Map, reason: String| { + object.insert("status".to_owned(), Value::from("unavailable")); + object.insert("reason".to_owned(), Value::from(reason)); + object.insert("calls".to_owned(), Value::Null); + object.insert("total_exec_time_milliseconds".to_owned(), Value::Null); + }; + let namespace = match lock_statements_namespace(pool).await { + Ok(Some(namespace)) => namespace, + Ok(None) => { + unavailable( + &mut object, + "the pg_stat_statements extension is not installed on this server".to_owned(), + ); + return Value::Object(object); + } + Err(error) => { + unavailable( + &mut object, + format!("could not look up the pg_stat_statements extension: {error}"), + ); + return Value::Object(object); + } + }; + // `pg_stat_statements` is server-wide. Without the `dbid` filter an advisory + // lock taken in any other database on the same server lands in this sum, and + // so in the difference the level reports, even though a lock in another + // database cannot contend with this one. The level would then be marked + // uncontaminated, correctly, while its lock figures described someone else's + // work. + let row = sqlx::query(&format!( + "SELECT coalesce(sum(calls),0)::bigint AS calls, \ + coalesce(sum(total_exec_time),0)::double precision AS total_exec_time \ + FROM {namespace}.pg_stat_statements \ + WHERE query LIKE $1 \ + AND dbid = (SELECT oid FROM pg_database WHERE datname = current_database())" + )) + .bind(QUERY_MATCH) + .fetch_one(pool) + .await; + match row { + Ok(row) => { + let calls: Result = row.try_get("calls"); + let total: Result = row.try_get("total_exec_time"); + match (calls, total) { + (Ok(calls), Ok(total)) => { + object.insert("status".to_owned(), Value::from("available")); + object.insert("reason".to_owned(), Value::Null); + object.insert("calls".to_owned(), Value::from(calls)); + object.insert("total_exec_time_milliseconds".to_owned(), json_f64(total)); + } + (Err(error), _) | (_, Err(error)) => unavailable( + &mut object, + format!("pg_stat_statements columns are unreadable: {error}"), + ), + } + } + Err(error) => unavailable( + &mut object, + format!("pg_stat_statements could not be read: {error}"), + ), + } + Value::Object(object) +} + +/// The schema `pg_stat_statements` was installed into, already quoted by the +/// server, or `None` when the extension is absent. +/// +/// The name has to be resolved rather than written unqualified. Every +/// connection in this run carries `options=-csearch_path=`, which +/// *replaces* the search path, so `public` is not on it; an unqualified +/// `pg_stat_statements` would then fail to resolve on a server that has the +/// extension installed, and the report would say "unavailable" about a view +/// that was sitting right there. +async fn lock_statements_namespace(pool: &PgPool) -> Result, sqlx::Error> { + sqlx::query_scalar( + "SELECT quote_ident(n.nspname) FROM pg_extension e \ + JOIN pg_namespace n ON n.oid = e.extnamespace \ + WHERE e.extname = 'pg_stat_statements'", + ) + .fetch_optional(pool) + .await +} + +/// The counters a level added, as the difference of two reads. +/// +/// Resetting `pg_stat_statements` would clear the whole server's statistics, +/// including other databases and applications; an operator who points this run +/// at a shared server has not agreed to that. A difference costs nothing and +/// measures the same thing, so long as the report says so. +fn lock_statement_delta(before: &Value, after: &Value) -> Value { + let mut object = match after.as_object() { + Some(object) => object.clone(), + None => return after.clone(), + }; + object.insert( + "method".to_owned(), + Value::from( + "the difference of pg_stat_statements counters read before and after the level; the \ + server's statistics are never reset, so an unrelated workload's history cannot be \ + destroyed by a measurement", + ), + ); + let number = |value: &Value, key: &str| value.get(key).and_then(Value::as_f64); + match ( + number(before, "calls"), + number(after, "calls"), + number(before, "total_exec_time_milliseconds"), + number(after, "total_exec_time_milliseconds"), + ) { + (Some(calls_before), Some(calls_after), Some(time_before), Some(time_after)) => { + object.insert( + "calls".to_owned(), + Value::from((calls_after - calls_before).max(0.0) as i64), + ); + object.insert( + "total_exec_time_milliseconds".to_owned(), + json_f64((time_after - time_before).max(0.0)), + ); + } + _ => { + // One of the two reads could not produce a counter, so the + // difference is unknown rather than zero (EP-OBSERVABILITY). + object.insert("calls".to_owned(), Value::Null); + object.insert("total_exec_time_milliseconds".to_owned(), Value::Null); + } + } + Value::Object(object) +} + +// --------------------------------------------------------------------------- +// Measurement +// --------------------------------------------------------------------------- + +struct LevelResult { + appenders: u32, + share_count: i64, + append_seconds: f64, + shares_per_second: f64, + passed_minimum: bool, + /// The whole first pass, when it fell below the floor and the level was + /// measured a second time; the fields around it are that second pass's + /// (#683). + first_pass: Option>, + /// `Some(true)` when a backend outside this run held or waited on + /// `ORDER_LOCK` while this level was being timed, `Some(false)` when the + /// sampler looked and found none, `None` when it could not look. + contaminated: Option, + per_appender: Vec<(String, i64)>, + order_lock: Value, +} + +struct Measurement { + postgres_version: String, + postgres_server_version: String, + replication_mode: String, + durability: Durability, + levels: Vec, + share_count: i64, + slowest_shares_per_second: f64, + passed_minimum: bool, + /// The run's verdict, folded from the levels: `Some(true)` if any level saw + /// a foreign backend, `Some(false)` only if every level looked and none + /// did, `None` when no level saw contamination but at least one could not + /// tell. + contaminated: Option, +} + +/// Folds the levels' verdicts into the run's. `true` wins over everything, and +/// an unknown wins over `false`: one level that could not be checked is enough +/// to stop the run claiming it had the database to itself. +fn fold_contamination(levels: impl IntoIterator>) -> Option { + let mut verdict = Some(false); + for level in levels { + match level { + Some(true) => return Some(true), + Some(false) => {} + None => verdict = None, + } + } + verdict +} + +/// The sentence the report and the failure message both carry when a level ran +/// against a database that something else was using. +const CONTAMINATION_NOTE: &str = + "the database was shared: a backend outside this run held or waited on ORDER_LOCK while a \ + level was being timed. ORDER_LOCK is database-wide, not schema-wide, so those appends \ + serialised against this run's. The rates below are what this run's appends actually \ + achieved and they are still gated against the floor, but they describe two workloads \ + sharing one lock rather than the append path on its own, and must not be carried forward \ + as a property of the append path. Re-run against a database nothing else is appending to."; + +/// The same warning for a run in which some level could not be checked at all. +const CONTAMINATION_UNKNOWN_NOTE: &str = + "whether the database was shared is unknown: at least one level's ORDER_LOCK sampler could \ + not complete a poll, so nothing observed that level's foreign backends. Read the per-level \ + order_lock.error before treating these rates as describing one workload."; + +struct Durability { + fsync: String, + full_page_writes: String, + synchronous_commit: String, +} + +/// Read through an appender's own pool, so the recorded `synchronous_commit` is +/// the session value PRISM actually commits under: `Ledger::connect`'s +/// `after_connect` forces it to `on` (or leaves `remote_apply` alone) on every +/// connection it opens, and a server-level default read on some other +/// connection could disagree. +async fn read_durability(pool: &PgPool) -> Result { + let row = sqlx::query( + "SELECT current_setting('fsync') AS fsync, \ + current_setting('full_page_writes') AS full_page_writes, \ + current_setting('synchronous_commit') AS synchronous_commit", + ) + .fetch_one(pool) + .await?; + Ok(Durability { + fsync: row.try_get("fsync")?, + full_page_writes: row.try_get("full_page_writes")?, + synchronous_commit: row.try_get("synchronous_commit")?, + }) +} + +/// Replication is read, never assumed. A server that refuses the catalogue +/// (an insufficiently privileged role, for instance) leaves the field +/// `unknown` rather than the more flattering `none`. +async fn read_replication_mode(pool: &PgPool) -> String { + let states: Result, _> = + sqlx::query_scalar("SELECT coalesce(sync_state,'') FROM pg_stat_replication") + .fetch_all(pool) + .await; + let names: Result = sqlx::query_scalar("SHOW synchronous_standby_names") + .fetch_one(pool) + .await; + match (states, names) { + (Ok(states), Ok(names)) => derive_replication_mode(&states, &names).to_owned(), + _ => "unknown".to_owned(), + } +} + +async fn committed_count(pool: &PgPool, writer_id: &str) -> Result { + sqlx::query_scalar("SELECT count(*) FROM qbit_share_ledger WHERE accepted AND writer_id = $1") + .bind(writer_id) + .fetch_one(pool) + .await + .with_context(|| format!("counting committed shares for writer {writer_id}")) +} + +async fn measure(db: &Database, config: &Config) -> Result { + // The seeding ledger is the only one that initialises: it creates the + // schema, the migrations and the `qbit_prism_cluster` singleton the + // appenders' `writable()` check reads. + let seed = Ledger::connect(&db.url, WRITER_SEED.to_owned(), SEED_POOL_CONNECTIONS, true) + .await + .context("connecting the seeding ledger")?; + let outcome = seed_and_run(db, config, &seed).await; + seed.pool.close().await; + outcome +} + +async fn seed_and_run(db: &Database, config: &Config, seed: &Ledger) -> Result { + let plan = WindowPlan::new(config.window_shares)?; + let load = plan + .load(&seed.pool, WRITER_SEED) + .await + .context("seeding the payout window")?; + println!( + "throughput_floor: seeded {} shares in {:.3}s ({:.0} rows/s, ~{} serialized bytes)", + load.rows, load.seconds, load.rows_per_second, load.serialized_bytes + ); + + let postgres_version: String = sqlx::query_scalar("SELECT version()") + .fetch_one(&seed.pool) + .await?; + let postgres_server_version: String = sqlx::query_scalar("SHOW server_version") + .fetch_one(&seed.pool) + .await?; + let replication_mode = read_replication_mode(&seed.pool).await; + + // Share indexes never repeat, across levels or against the seeded window, + // so every `share_id` and every derived header hash is globally unique and + // no append can be turned into an idempotent no-op by a collision. + let mut next_index = config.window_shares + 1; + let mut durability = None; + let mut levels = Vec::new(); + let mut total_shares = 0i64; + let mut slowest = f64::INFINITY; + let mut passed_all = true; + + for &appenders in &config.appenders { + let result = measure_level(config.retry_below_floor, async |suffix| { + if !suffix.is_empty() { + println!( + "throughput_floor: {appenders} appender(s): the first pass fell below the \ + floor; measuring the level once more" + ); + } + let mut ledgers = Vec::new(); + for index in 0..appenders { + let ledger = Ledger::connect( + &db.url, + format!("throughput-appender-{appenders}-{index}{suffix}"), + APPENDER_POOL_CONNECTIONS, + false, + ) + .await + .with_context(|| format!("connecting appender {index} of level {appenders}"))?; + if durability.is_none() { + durability = Some(read_durability(&ledger.pool).await?); + } + ledgers.push(ledger); + } + let result = run_level(db, config, &plan, &ledgers, appenders, &mut next_index).await; + for ledger in &ledgers { + ledger.pool.close().await; + } + result + }) + .await?; + total_shares += result.share_count + + result + .first_pass + .as_ref() + .map_or(0, |first| first.share_count); + slowest = slowest.min(result.shares_per_second); + passed_all &= result.passed_minimum; + println!( + "throughput_floor: {} appender(s): {} shares in {:.3}s = {:.1} shares/s (floor {:.1}, {}){}", + result.appenders, + result.share_count, + result.append_seconds, + result.shares_per_second, + config.minimum, + if result.passed_minimum { "pass" } else { "FAIL" }, + result + .first_pass + .as_ref() + .map(|first| format!( + ", after a first pass at {:.1} shares/s", + first.shares_per_second + )) + .unwrap_or_default(), + ); + match result.contaminated { + Some(true) => println!( + "throughput_floor: WARNING, level of {} appender(s) is CONTAMINATED: {}", + result.appenders, CONTAMINATION_NOTE + ), + None => println!( + "throughput_floor: WARNING, level of {} appender(s): {}", + result.appenders, CONTAMINATION_UNKNOWN_NOTE + ), + Some(false) => {} + } + match result + .first_pass + .as_ref() + .and_then(|first| first.contaminated) + { + Some(true) => println!( + "throughput_floor: WARNING, the first pass of the level of {} appender(s) is \ + CONTAMINATED: {}", + result.appenders, CONTAMINATION_NOTE + ), + Some(false) => {} + None if result.first_pass.is_some() => println!( + "throughput_floor: WARNING, the first pass of the level of {} appender(s): {}", + result.appenders, CONTAMINATION_UNKNOWN_NOTE + ), + None => {} + } + levels.push(result); + } + + let durability = match durability { + Some(durability) => durability, + None => bail!("no appender level ran, so no durability settings were read"), + }; + let contaminated = levels_contamination(&levels); + Ok(Measurement { + postgres_version, + postgres_server_version, + replication_mode, + durability, + levels, + share_count: total_shares, + slowest_shares_per_second: slowest, + passed_minimum: passed_all, + contaminated, + }) +} + +/// One level, measured once more when its first pass falls below the floor +/// and `retry` allows it (#683). Only the second pass decides: a runner that +/// stalls for a few seconds drops one pass below the floor, while a collapse +/// of the append path misses on both. The first pass is kept on the result. +/// `pass` runs one pass, with a suffix for its writer ids, so the second +/// pass's committed shares are counted apart from the first's. +async fn measure_level( + retry: bool, + mut pass: impl std::ops::AsyncFnMut(&'static str) -> Result, +) -> Result { + let first = pass("").await?; + if first.passed_minimum || !retry { + return Ok(first); + } + let mut second = pass("-retry").await.map_err(|error| { + error.context(SecondPassFailed { + appenders: first.appenders, + first_pass_shares_per_second: first.shares_per_second, + first_pass: level_entry(&first), + }) + })?; + second.first_pass = Some(Box::new(first)); + Ok(second) +} + +/// The context on a second pass's error (#683). The error below it says what +/// went wrong; the first pass it followed rides along, so the error report +/// still carries that pass's entry. +#[derive(Debug)] +struct SecondPassFailed { + appenders: u32, + first_pass_shares_per_second: f64, + first_pass: Value, +} + +impl std::fmt::Display for SecondPassFailed { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!( + f, + "measuring the level of {} appender(s) a second time, after a first pass at {:.1} \ + shares/s below the floor", + self.appenders, self.first_pass_shares_per_second + ) + } +} + +/// The run's contamination verdict, over every pass it timed: a foreign +/// backend during a first pass that was then measured again still counts. +fn levels_contamination(levels: &[LevelResult]) -> Option { + fold_contamination(levels.iter().flat_map(|level| { + std::iter::once(level.contaminated) + .chain(level.first_pass.as_ref().map(|first| first.contaminated)) + })) +} + +async fn run_level( + db: &Database, + config: &Config, + plan: &WindowPlan, + ledgers: &[Ledger], + appenders: u32, + next_index: &mut u64, +) -> Result { + let split = distribute(config.shares_per_level, appenders); + let sampler_pool = PgPoolOptions::new() + .max_connections(1) + .connect(&db.url) + .await + .context("opening the ORDER_LOCK sampling connection")?; + let statements_before = read_lock_statements(&sampler_pool).await; + let stop = Arc::new(SamplerStop::default()); + let sampler = tokio::spawn(sample_order_lock( + sampler_pool.clone(), + config.lock_sample, + config.application_name.clone(), + stop.clone(), + )); + + // The main task is the extra participant, so every appender is released at + // the same instant and the clock starts there rather than at the first + // `spawn`, which would have charged task startup to the append rate. + let barrier = Arc::new(Barrier::new(ledgers.len() + 1)); + let mut handles = Vec::new(); + for (position, ledger) in ledgers.iter().enumerate() { + let count = split[position]; + let first = *next_index; + *next_index += count; + let ledger = ledger.clone(); + let plan = plan.clone(); + let barrier = Arc::clone(&barrier); + handles.push(tokio::spawn(async move { + barrier.wait().await; + for index in first..first + count { + let result = ledger + .append(plan.share(index), None) + .await + .with_context(|| { + format!("appending share index {index} as {}", ledger.instance_id) + })?; + // A share that was already there is not throughput. It would + // also mean the index allocation above had collided, which is + // worth failing the level over rather than counting. + ensure!( + result.inserted, + "share index {index} was already present, so appender {} measured a \ + no-op instead of an insert", + ledger.instance_id + ); + } + Ok::<(), anyhow::Error>(()) + })); + } + // Before the barrier, not after: the appenders are released the moment the + // main participant arrives, and a task the runtime resumes first would + // otherwise do work outside the measured window and inflate the rate. + let started = Instant::now(); + barrier.wait().await; + let mut failure: Option = None; + for handle in handles { + // Every appender is awaited even after one has failed, so the level + // ends with no task still holding a connection or the advisory lock. + let outcome = match handle.await { + Ok(outcome) => outcome, + Err(error) => Err(anyhow::anyhow!("an appender task panicked: {error}")), + }; + if let Err(error) = outcome { + if failure.is_none() { + failure = Some(error); + } + } + } + let append_seconds = started.elapsed().as_secs_f64(); + // Signalled, not just flagged: the sampler is woken out of its sleep, so + // the level ends when the appends end rather than one interval later. + stop.stop(); + let lock_summary = match sampler.await { + Ok(summary) => summary, + Err(error) => LockWaitSummary { + error: Some(format!("the ORDER_LOCK sampler panicked: {error}")), + ..LockWaitSummary::default() + }, + }; + let statements = lock_statement_delta( + &statements_before, + &read_lock_statements(&sampler_pool).await, + ); + sampler_pool.close().await; + if let Some(error) = failure { + return Err(error.context(format!("level of {appenders} appender(s) failed"))); + } + + let mut per_appender = Vec::new(); + let mut share_count = 0i64; + for ledger in ledgers { + let count = committed_count(&ledger.pool, &ledger.instance_id).await?; + share_count += count; + per_appender.push((ledger.instance_id.clone(), count)); + } + let expected = i64::try_from(config.shares_per_level)?; + ensure!( + share_count == expected, + "level of {appenders} appender(s) committed {share_count} shares, expected {expected}" + ); + let shares_per_second = share_count as f64 / append_seconds.max(f64::MIN_POSITIVE); + let contaminated = lock_summary.contaminated(); + let mut order_lock = lock_summary.to_json(config.lock_sample); + if let Some(object) = order_lock.as_object_mut() { + object.insert("pg_stat_statements".to_owned(), statements); + } + Ok(LevelResult { + appenders, + share_count, + append_seconds, + shares_per_second, + passed_minimum: shares_per_second >= config.minimum, + first_pass: None, + contaminated, + per_appender, + order_lock, + }) +} + +// --------------------------------------------------------------------------- +// Report +// --------------------------------------------------------------------------- + +/// `serde_json` cannot hold a non-finite number, and silently turning one into +/// `null` would hide it. Every rate here is finite by construction, but a +/// division that surprises us should surface as `null` in the report rather +/// than as a panic in the middle of a measured run. +fn json_f64(value: f64) -> Value { + serde_json::Number::from_f64(value).map_or(Value::Null, Value::Number) +} + +/// `GITHUB_SHA` when CI set it, otherwise the checked-out commit, otherwise +/// `null`. A missing commit is recorded as missing: a report that cannot say +/// what it measured should say so. +fn commit_id() -> Value { + if let Ok(Some(sha)) = env_raw("GITHUB_SHA") { + let sha = sha.trim(); + if !sha.is_empty() { + return Value::from(sha); + } + } + let output = std::process::Command::new("git") + .args(["rev-parse", "HEAD"]) + .current_dir(env!("CARGO_MANIFEST_DIR")) + .output(); + match output { + Ok(output) if output.status.success() => { + let sha = String::from_utf8_lossy(&output.stdout).trim().to_owned(); + if sha.is_empty() { + Value::Null + } else { + Value::from(sha) + } + } + _ => Value::Null, + } +} + +/// Host facts, from procfs on Linux and `sysctl` on macOS. Anything this host +/// does not expose is `null`, never a plausible-looking default: a throughput +/// number whose machine is unknown must read as unknown. +fn machine() -> Value { + let mut object = Map::new(); + object.insert("os".to_owned(), Value::from(std::env::consts::OS)); + let (cpu_model, cpu_count, memory_bytes) = match std::env::consts::OS { + "linux" => ( + proc_cpuinfo_field("model name"), + proc_cpu_count(), + proc_meminfo_total_bytes(), + ), + "macos" => ( + sysctl("machdep.cpu.brand_string"), + sysctl("hw.ncpu").and_then(|value| value.parse::().ok()), + sysctl("hw.memsize").and_then(|value| value.parse::().ok()), + ), + _ => (None, None, None), + }; + object.insert( + "cpu_model".to_owned(), + cpu_model.map_or(Value::Null, Value::from), + ); + object.insert( + "cpu_count".to_owned(), + cpu_count.map_or(Value::Null, Value::from), + ); + object.insert( + "memory_bytes".to_owned(), + memory_bytes.map_or(Value::Null, Value::from), + ); + Value::Object(object) +} + +fn proc_cpuinfo_field(field: &str) -> Option { + let text = std::fs::read_to_string("/proc/cpuinfo").ok()?; + text.lines() + .filter_map(|line| line.split_once(':')) + .find(|(key, _)| key.trim() == field) + .map(|(_, value)| value.trim().to_owned()) + .filter(|value| !value.is_empty()) +} + +fn proc_cpu_count() -> Option { + let text = std::fs::read_to_string("/proc/cpuinfo").ok()?; + let count = text + .lines() + .filter(|line| { + line.split_once(':') + .is_some_and(|(k, _)| k.trim() == "processor") + }) + .count() as u64; + (count > 0).then_some(count) +} + +fn proc_meminfo_total_bytes() -> Option { + let text = std::fs::read_to_string("/proc/meminfo").ok()?; + let line = text.lines().find(|line| line.starts_with("MemTotal:"))?; + let kib: u64 = line.split_whitespace().nth(1)?.parse().ok()?; + kib.checked_mul(1024) +} + +fn sysctl(name: &str) -> Option { + let output = std::process::Command::new("sysctl") + .args(["-n", name]) + .output() + .ok()?; + if !output.status.success() { + return None; + } + let value = String::from_utf8_lossy(&output.stdout).trim().to_owned(); + (!value.is_empty()).then_some(value) +} + +/// The fields that identify a run rather than describe its outcome: who ran, +/// how it was configured, on what, from which commit. +/// +/// Both reports start here — the measured one and the one an errored run leaves +/// behind — so a failed run is identifiable by exactly the same fields as a +/// successful one, and a reader never has to guess which test or which +/// configuration produced the file in front of them. +fn report_identity(config: &Config) -> Map { + let mut root = Map::new(); + root.insert("schema".to_owned(), Value::from(REPORT_SCHEMA)); + root.insert( + "generated_at".to_owned(), + Value::from(Utc::now().to_rfc3339_opts(SecondsFormat::Millis, true)), + ); + root.insert("test".to_owned(), Value::from(config.test_name.clone())); + root.insert( + "application_name".to_owned(), + Value::from(config.application_name.clone()), + ); + root.insert( + "window_shares".to_owned(), + Value::from(config.window_shares), + ); + root.insert( + "appended_shares_per_level".to_owned(), + Value::from(config.shares_per_level), + ); + root.insert("min_shares_per_second".to_owned(), json_f64(config.minimum)); + root.insert( + "min_shares_per_second_source".to_owned(), + Value::from(config.minimum_source.clone()), + ); + root.insert("commit".to_owned(), commit_id()); + root.insert( + "build_profile".to_owned(), + Value::from(if cfg!(debug_assertions) { + "debug" + } else { + "release" + }), + ); + root.insert("machine".to_owned(), machine()); + root +} + +/// The report an errored run leaves behind, so a run that never reached a +/// measurement still says who it was, how it was configured and what went +/// wrong, instead of leaving a stale report from an earlier run as the only +/// file on disk. +/// +/// `results` is an empty array, not a missing key: a consumer that iterates the +/// levels finds none rather than tripping over the shape of the file, and +/// `passed_minimum` is `false` because a run that could not measure certainly +/// did not clear the floor. +fn build_error_report(config: &Config, error: &anyhow::Error) -> Value { + let mut root = report_identity(config); + // `{:#}` renders the whole anyhow chain, outermost first, so the report + // keeps the context each layer added rather than only the final cause. + root.insert("error".to_owned(), Value::from(format!("{error:#}"))); + root.insert("passed_minimum".to_owned(), Value::from(false)); + root.insert("results".to_owned(), Value::Array(Vec::new())); + // A second pass that failed leaves the first pass's entry behind (#683). + if let Some(failed) = error.downcast_ref::() { + root.insert( + "first_pass_before_the_error".to_owned(), + failed.first_pass.clone(), + ); + } + Value::Object(root) +} + +fn build_report(config: &Config, measurement: &Measurement) -> Value { + let mut root = report_identity(config); + root.insert("error".to_owned(), Value::Null); + root.insert( + "share_count".to_owned(), + Value::from(measurement.share_count), + ); + root.insert( + "shares_per_second".to_owned(), + json_f64(measurement.slowest_shares_per_second), + ); + root.insert( + "passed_minimum".to_owned(), + Value::from(measurement.passed_minimum), + ); + root.insert( + "contaminated".to_owned(), + measurement.contaminated.map_or(Value::Null, Value::from), + ); + root.insert( + "contamination_note".to_owned(), + match measurement.contaminated { + Some(true) => Value::from(CONTAMINATION_NOTE), + None => Value::from(CONTAMINATION_UNKNOWN_NOTE), + Some(false) => Value::Null, + }, + ); + root.insert( + "postgres_version".to_owned(), + Value::from(measurement.postgres_version.clone()), + ); + root.insert( + "postgres_server_version".to_owned(), + Value::from(measurement.postgres_server_version.clone()), + ); + root.insert( + "replication_mode".to_owned(), + Value::from(measurement.replication_mode.clone()), + ); + let mut durability = Map::new(); + durability.insert( + "fsync".to_owned(), + Value::from(measurement.durability.fsync.clone()), + ); + durability.insert( + "full_page_writes".to_owned(), + Value::from(measurement.durability.full_page_writes.clone()), + ); + durability.insert( + "synchronous_commit".to_owned(), + Value::from(measurement.durability.synchronous_commit.clone()), + ); + root.insert("durability".to_owned(), Value::Object(durability)); + let results = measurement + .levels + .iter() + .map(level_entry) + .collect::>(); + root.insert("results".to_owned(), Value::Array(results)); + root.insert("notes".to_owned(), Value::from(NOTES)); + Value::Object(root) +} + +/// One level's report entry. A first pass below the floor sits under +/// `first_pass` in the same shape, its lock evidence included (#683). +fn level_entry(level: &LevelResult) -> Value { + let mut entry = Map::new(); + entry.insert("appenders".to_owned(), Value::from(level.appenders)); + entry.insert("share_count".to_owned(), Value::from(level.share_count)); + entry.insert("append_seconds".to_owned(), json_f64(level.append_seconds)); + entry.insert( + "shares_per_second".to_owned(), + json_f64(level.shares_per_second), + ); + entry.insert( + "passed_minimum".to_owned(), + Value::from(level.passed_minimum), + ); + entry.insert( + "first_pass".to_owned(), + level.first_pass.as_deref().map_or(Value::Null, level_entry), + ); + entry.insert( + "contaminated".to_owned(), + level.contaminated.map_or(Value::Null, Value::from), + ); + let per_appender = level + .per_appender + .iter() + .map(|(writer_id, count)| { + let mut one = Map::new(); + one.insert("writer_id".to_owned(), Value::from(writer_id.clone())); + one.insert("share_count".to_owned(), Value::from(*count)); + Value::Object(one) + }) + .collect::>(); + entry.insert("per_appender".to_owned(), Value::Array(per_appender)); + entry.insert("order_lock".to_owned(), level.order_lock.clone()); + Value::Object(entry) +} + +/// Writes the report and returns its path. Called before the floor assertion so +/// that a run which fails the floor still leaves the evidence behind. +fn write_report(config: &Config, measurement: &Measurement) -> Result<()> { + write_json(config, &build_report(config, measurement)) +} + +fn write_json(config: &Config, report: &Value) -> Result<()> { + if let Some(parent) = config.report_path.parent() { + std::fs::create_dir_all(parent) + .with_context(|| format!("creating {}", parent.display()))?; + } + let mut text = serde_json::to_string_pretty(report)?; + text.push('\n'); + std::fs::write(&config.report_path, text) + .with_context(|| format!("writing {}", config.report_path.display()))?; + println!( + "throughput_floor: report written to {}", + config.report_path.display() + ); + Ok(()) +} + +/// One line per measured run, written straight to the process's stderr. +/// +/// libtest captures `println!` and `eprintln!` from a passing test by default. +/// A write through `std::io::stderr()` keeps the per-level rates visible even +/// when capture is enabled. CI also retains the detailed report artifact; +/// this summary puts the rates directly in the log for calibrating +/// `CI_MIN_SHARES_PER_SEC` without downloading that artifact. +fn emit_summary(config: &Config, measurement: &Measurement) { + let levels = measurement + .levels + .iter() + .map(|level| format!("{}:{:.1}", level.appenders, level.shares_per_second)) + .collect::>() + .join(","); + let first_passes = measurement + .levels + .iter() + .filter_map(|level| { + level + .first_pass + .as_ref() + .map(|first| format!("{}:{:.1}", level.appenders, first.shares_per_second)) + }) + .collect::>(); + let first_passes = if first_passes.is_empty() { + "none".to_owned() + } else { + first_passes.join(",") + }; + let contaminated = match measurement.contaminated { + Some(true) => "true", + Some(false) => "false", + None => "unknown", + }; + let line = format!( + "throughput_floor summary: test={} build={} window={} shares_per_level={} \ + shares_per_second_by_appenders={} first_passes_below_floor={} floor={:.1} passed={} \ + contaminated={}\n", + config.test_name, + if cfg!(debug_assertions) { + "debug" + } else { + "release" + }, + config.window_shares, + config.shares_per_level, + levels, + first_passes, + config.minimum, + measurement.passed_minimum, + contaminated, + ); + // The report file already holds everything this line summarises, so a + // failed write to stderr must not fail the measurement. + let _ = std::io::stderr().lock().write_all(line.as_bytes()); +} + +// --------------------------------------------------------------------------- +// Test bodies +// --------------------------------------------------------------------------- + +async fn run_floor(test_name: &str, defaults: Defaults, selection: Selection) -> Result<()> { + // Configuration is validated first, so a malformed variable costs nothing: + // no connection, no schema, no seeding. Nothing below this line can run + // with an out-of-range sampling interval or an uncomparable floor + // (EP-VALIDATION). + let config = Config::from_env(test_name, defaults, selection)?; + // An explicitly selected `#[ignore]`d run must never skip: the gate crate + // provides `required_database_url` for exactly that, and a silent success + // here would look like a measurement that simply found nothing to say. + let raw = match selection { + Selection::Ci => match gate::database_url(gate::site!())? { + Some(url) => url, + None => return Ok(()), + }, + Selection::Explicit => gate::required_database_url(gate::site!())?, + }; + println!( + "throughput_floor: window={} shares/level={} appenders={:?} floor={:.1} ({}) \ + application_name={}", + config.window_shares, + config.shares_per_level, + config.appenders, + config.minimum, + config.minimum_source, + config.application_name, + ); + let measurement = match measure_run(&config, &raw).await { + Ok(measurement) => measurement, + Err(error) => { + // A run that dies connecting, seeding or appending still owes the + // operator a file saying what it was and what went wrong; without + // one, the newest report on disk belongs to some earlier run and + // reads as though this one never happened. The write can only add + // context to the original error, never replace it (EP-ERRORS). + return Err( + match write_json(&config, &build_error_report(&config, &error)) { + Ok(()) => error, + Err(write_error) => error.context(format!( + "the error report could not be written either: {write_error:#}" + )), + }, + ); + } + }; + write_report(&config, &measurement)?; + emit_summary(&config, &measurement); + for level in &measurement.levels { + ensure!( + level.passed_minimum, + "share-append throughput floor: the level of {} appender(s) sustained {:.2} \ + shares/s{}, below the floor of {:.2} shares/s ({}).{} The report is at {}.", + level.appenders, + level.shares_per_second, + level + .first_pass + .as_ref() + .map(|first| format!( + " on a second pass, after {:.2} on the first", + first.shares_per_second + )) + .unwrap_or_default(), + config.minimum, + config.minimum_source, + match level.contaminated { + Some(true) => format!(" NOTE: {CONTAMINATION_NOTE}"), + None => format!(" NOTE: {CONTAMINATION_UNKNOWN_NOTE}"), + Some(false) => String::new(), + }, + config.report_path.display() + ); + } + Ok(()) +} + +/// Opens the run's schema, measures it, and drops the schema on the way out. +/// +/// Split out of `run_floor` so that every error from the connection onwards +/// arrives at one place, where the error report is written. +async fn measure_run(config: &Config, raw: &str) -> Result { + let db = Database::open(raw, &config.application_name).await?; + let measured = measure(&db, config).await; + // The schema goes away either way. A cleanup failure is attached to the + // measurement error rather than replacing it, so a leaked schema can never + // hide the failure that actually mattered. + let cleanup = db.close().await; + match measured { + Ok(measurement) => { + cleanup?; + Ok(measurement) + } + Err(error) => Err(match cleanup { + Ok(()) => error, + Err(cleanup) => error.context(format!("schema cleanup also failed: {cleanup}")), + }), + } +} + +/// The CI-sized floor: small enough to finish well inside the +/// `prism-native-postgres` job's 20-minute budget on a 2-vCPU runner, large +/// enough that a real regression in the append path moves it. +/// +/// This test is deliberately not `#[ignore]`d. `prism-native-postgres` runs +/// `cargo test --locked -p qbit-prism-server --all-targets` against a +/// PostgreSQL 16 service, so the floor enters CI with no workflow change; the +/// database-free `rust-tests` job runs the same target and skips through the +/// guard above. +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn share_append_throughput_floor() -> Result<()> { + run_floor( + "share_append_throughput_floor", + Defaults { + window_shares: 20_000, + shares_per_level: 2_000, + }, + Selection::Ci, + ) + .await +} + +/// The full-size floor, for a host with time to spare. `test/test-prism-postgres-throughput.sh` +/// runs this one, in release, against a disposable server. +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +#[ignore = "full-size run: 100k seeded shares and 20k appends per level; run it with \ + test/test-prism-postgres-throughput.sh or --ignored"] +async fn share_append_throughput_floor_full_size() -> Result<()> { + run_floor( + "share_append_throughput_floor_full_size", + Defaults { + window_shares: 100_000, + shares_per_level: 20_000, + }, + Selection::Explicit, + ) + .await +} + +// --------------------------------------------------------------------------- +// Database-free unit tests +// --------------------------------------------------------------------------- + +#[test] +fn environment_parsers_reject_what_cannot_be_a_floor() { + // A `nan` floor would compare false against every rate and pass every run; + // an infinite one would fail every run. Both are configuration errors, not + // extreme floors. + for bad in [ + "", " ", "nan", "NaN", "inf", "-inf", "abc", "-1", "-0.5", "1e", + ] { + assert!( + parse_minimum(Some(bad), 1.0).is_err(), + "{MIN_SHARES_VAR}={bad:?} must be refused" + ); + } + for (good, expected) in [("0", 0.0), ("25", 25.0), (" 12.5 ", 12.5), ("1e3", 1000.0)] { + assert_eq!( + parse_minimum(Some(good), 1.0).expect("valid floor"), + expected + ); + } + assert_eq!(parse_minimum(None, 7.5).expect("fallback"), 7.5); +} + +#[test] +fn environment_parsers_reject_impossible_counts() { + for bad in ["", "0", "-1", "abc", "1.5"] { + assert!( + parse_positive_u64(SHARES_VAR, Some(bad), 10).is_err(), + "{SHARES_VAR}={bad:?} must be refused" + ); + } + assert_eq!( + parse_positive_u64(SHARES_VAR, Some(" 2000 "), 10).expect("valid count"), + 2_000 + ); + assert_eq!( + parse_positive_u64(SHARES_VAR, None, 10).expect("fallback"), + 10 + ); + + // 30000 does not divide 8_000_000, so the seeded window would not be + // 30000 shares wide. + for bad in ["30000", "0", "3", "8000001", "abc"] { + assert!( + parse_window_shares(Some(bad), 20_000).is_err(), + "{WINDOW_SHARES_VAR}={bad:?} must be refused" + ); + } + for good in ["20000", "100000", "400000", "1"] { + assert!( + parse_window_shares(Some(good), 20_000).is_ok(), + "{WINDOW_SHARES_VAR}={good:?} divides {WINDOW_WEIGHT} and must be accepted" + ); + } +} + +#[test] +fn lock_sample_interval_is_bounded_at_both_ends() { + // Zero is not an interval, and neither is a negative or fractional one. + for bad in ["", "0", "-1", "1.5", "abc"] { + assert!( + parse_lock_sample_ms(Some(bad), DEFAULT_LOCK_SAMPLE_MS).is_err(), + "{LOCK_SAMPLE_VAR}={bad:?} must be refused" + ); + } + // Above a second the poll can no longer resolve a millisecond lock wait, + // which is the only thing it exists to measure. + for bad in ["1001", "20000", "18446744073709551615"] { + let error = parse_lock_sample_ms(Some(bad), DEFAULT_LOCK_SAMPLE_MS) + .expect_err("an interval above the ceiling must be refused") + .to_string(); + assert!( + error.contains("coarser than a second"), + "the rejection must say why a coarse interval is useless, got {error:?}" + ); + } + for (good, expected) in [("1", 1), (" 10 ", 10), ("1000", 1_000)] { + assert_eq!( + parse_lock_sample_ms(Some(good), DEFAULT_LOCK_SAMPLE_MS).expect("in range"), + expected + ); + } + assert_eq!( + parse_lock_sample_ms(None, DEFAULT_LOCK_SAMPLE_MS).expect("fallback"), + DEFAULT_LOCK_SAMPLE_MS + ); + assert!( + (MIN_LOCK_SAMPLE_MS..=MAX_LOCK_SAMPLE_MS).contains(&DEFAULT_LOCK_SAMPLE_MS), + "the compiled-in default must itself be inside the range it enforces" + ); +} + +#[test] +fn a_level_that_never_sampled_reports_unknown_rather_than_zero() { + let interval = Duration::from_millis(10); + + let never_polled = LockWaitSummary::default(); + let json = never_polled.to_json(interval); + assert_eq!( + json["samples"], + Value::from(0u64), + "the count is known: none" + ); + for unknown in [ + "mean_waiters", + "max_waiters", + "episodes_lower_bound", + "estimated_waiter_seconds", + "foreign_waiters_max", + "foreign_backends_seen", + ] { + assert_eq!( + json[unknown], + Value::Null, + "{unknown} must be null when nobody looked, never 0" + ); + } + assert_eq!( + json["error"], + Value::from("the level ended before the sampler completed a poll") + ); + assert_eq!( + json["contaminated"], + Value::Null, + "an unsampled level cannot claim the database was quiet" + ); + + // A sampler that failed outright does not even know how many polls it got + // through, so its sample count is null as well. + let failed = LockWaitSummary { + error: Some("ORDER_LOCK sampling failed: connection closed".to_owned()), + samples: 4, + ..LockWaitSummary::default() + }; + let json = failed.to_json(interval); + assert_eq!(json["samples"], Value::Null); + assert_eq!(json["contaminated"], Value::Null); + + // A sampler that did its job reports real numbers, including honest zeros: + // here a level that was genuinely alone on the lock. + let quiet = LockWaitSummary { + samples: 6, + total_waiters: 3, + max_waiters: 1, + waiter_seconds: 0.03, + ..LockWaitSummary::default() + }; + let json = quiet.to_json(interval); + assert_eq!(json["samples"], Value::from(6u64)); + assert_eq!(json["mean_waiters"], json_f64(0.5)); + assert_eq!(json["max_waiters"], Value::from(1u64)); + assert_eq!(json["foreign_backends_seen"], Value::from(0u64)); + assert_eq!(json["error"], Value::Null); + assert_eq!(json["contaminated"], Value::from(false)); + + let shared = LockWaitSummary { + samples: 6, + foreign_waiters_max: 2, + foreign_backends: [4242, 4243].into_iter().collect(), + ..LockWaitSummary::default() + }; + let json = shared.to_json(interval); + assert_eq!(json["foreign_waiters_max"], Value::from(2u64)); + assert_eq!(json["foreign_backends_seen"], Value::from(2u64)); + assert_eq!(json["contaminated"], Value::from(true)); +} + +#[test] +fn contamination_folds_true_over_unknown_over_false() { + assert_eq!(fold_contamination([]), Some(false)); + assert_eq!(fold_contamination([Some(false), Some(false)]), Some(false)); + assert_eq!(fold_contamination([Some(false), None]), None); + assert_eq!(fold_contamination([None, Some(true)]), Some(true)); + assert_eq!( + fold_contamination([Some(true), Some(false)]), + Some(true), + "one shared level makes the whole run's numbers shared" + ); +} + +/// A pass as `run_level` returns one, for the database-free retry tests. +fn timed_pass(shares_per_second: f64, contaminated: Option) -> LevelResult { + LevelResult { + appenders: 1, + share_count: 2_000, + append_seconds: 2_000.0 / shares_per_second, + shares_per_second, + passed_minimum: shares_per_second >= CI_MIN_SHARES_PER_SEC, + first_pass: None, + contaminated, + per_appender: Vec::new(), + order_lock: Value::Null, + } +} + +/// Runs `measure_level` over scripted passes and returns the level with the +/// writer-id suffixes it asked for, in order. +async fn scripted_level(retry: bool, passes: Vec) -> (LevelResult, Vec<&'static str>) { + let mut passes = passes.into_iter(); + let mut suffixes = Vec::new(); + let level = measure_level(retry, async |suffix| { + suffixes.push(suffix); + Ok(passes.next().expect("no further pass was scripted")) + }) + .await + .expect("the scripted passes succeed"); + (level, suffixes) +} + +#[tokio::test] +async fn a_level_below_the_floor_is_measured_once_more_and_only_that_pass_decides() { + // Met the floor: one pass, nothing kept beside it. + let (level, suffixes) = scripted_level(true, vec![timed_pass(500.0, Some(false))]).await; + assert_eq!(suffixes, [""]); + assert!(level.passed_minimum && level.first_pass.is_none()); + + // A stall: the first pass misses, the second decides and keeps the first + // pass's rate and contamination beside its own. + let (level, suffixes) = scripted_level( + true, + vec![ + timed_pass(130.0, Some(true)), + timed_pass(480.0, Some(false)), + ], + ) + .await; + assert_eq!(suffixes, ["", "-retry"]); + assert!(level.passed_minimum); + assert_eq!(level.shares_per_second, 480.0); + let first = level.first_pass.as_ref().expect("the first pass is kept"); + assert_eq!((first.shares_per_second, first.share_count), (130.0, 2_000)); + // Its foreign backend still marks the run, though the second pass was clean. + assert_eq!(levels_contamination(&[level]), Some(true)); + + // A collapse misses on both passes and still fails, with both rates. + let (level, _) = scripted_level( + true, + vec![ + timed_pass(130.0, Some(false)), + timed_pass(120.0, Some(false)), + ], + ) + .await; + assert!(!level.passed_minimum); + assert_eq!(level.shares_per_second, 120.0); + assert_eq!( + level + .first_pass + .as_ref() + .map(|first| first.shares_per_second), + Some(130.0) + ); + + // Without the retry (the explicit full-size run) one pass decides. + let (level, suffixes) = scripted_level(false, vec![timed_pass(130.0, Some(false))]).await; + assert_eq!(suffixes, [""]); + assert!(!level.passed_minimum && level.first_pass.is_none()); + + // A second pass that errors keeps the first pass for the error report, + // under whatever context the run adds on its way out. + let mut passes = vec![ + Ok(timed_pass(130.0, Some(false))), + Err(anyhow::anyhow!("pool timed out")), + ] + .into_iter(); + let error = measure_level(true, async |_| { + passes.next().expect("no further pass was scripted") + }) + .await + .err() + .expect("the second pass errors") + .context("measuring the run"); + let failed = error + .downcast_ref::() + .expect("the first pass rides on the error"); + assert_eq!(failed.first_pass["shares_per_second"], 130.0); + assert!(format!("{error:#}").contains("pool timed out")); +} + +#[test] +fn the_sampler_stop_signal_is_not_lost_before_the_sleep() { + // `notify_one` stores a permit, so a stop raised while the sampler is + // querying rather than sleeping still ends the very next sleep. If this + // ever regressed to `notify_waiters`, the signal would be dropped and the + // level would stall for a whole interval, which is the bug this replaced. + let runtime = tokio::runtime::Builder::new_current_thread() + .enable_time() + .build() + .expect("a current-thread runtime"); + runtime.block_on(async { + let stop = SamplerStop::default(); + stop.stop(); + assert!(stop.stopped()); + let started = Instant::now(); + tokio::time::timeout( + Duration::from_secs(5), + stop.sleep_until_stopped(Duration::from_secs(3_600)), + ) + .await + .expect("a signalled stop must not wait out the interval"); + assert!( + started.elapsed() < Duration::from_secs(1), + "the sleep returned after {:?}, so the stop permit was lost", + started.elapsed() + ); + }); +} + +#[test] +fn appender_levels_are_distinct_and_bounded() { + assert_eq!(parse_appenders(None).expect("default"), vec![1, 2, 4]); + assert_eq!( + parse_appenders(Some(" 1, 8 ")).expect("spaced"), + vec![1u32, 8] + ); + for bad in ["", "0", "65", "-1", "1,,2", "1,1", "four", "1,2,1"] { + assert!( + parse_appenders(Some(bad)).is_err(), + "{APPENDERS_VAR}={bad:?} must be refused" + ); + } +} + +#[test] +fn share_budgets_split_deterministically() { + assert_eq!(distribute(2_000, 4), vec![500, 500, 500, 500]); + assert_eq!(distribute(10, 4), vec![3, 3, 2, 2]); + assert_eq!(distribute(7, 1), vec![7]); + for (total, appenders) in [(2_000u64, 4u32), (10, 4), (20_001, 3), (7, 7)] { + let split = distribute(total, appenders); + assert_eq!(split.len(), appenders as usize); + assert_eq!( + split.iter().sum::(), + total, + "the whole budget is spent" + ); + assert!(split.iter().all(|&count| count > 0), "no idle appender"); + } +} + +#[test] +fn replication_mode_is_derived_from_what_the_server_reports() { + assert_eq!(derive_replication_mode(&[], ""), "none"); + assert_eq!( + derive_replication_mode(&["async".to_owned()], ""), + "async", + "a streaming standby that holds no commit is async" + ); + assert_eq!( + derive_replication_mode(&["async".to_owned(), "sync".to_owned()], "standby1"), + "sync" + ); + assert_eq!( + derive_replication_mode(&["quorum".to_owned()], "ANY 1 (a, b)"), + "sync", + "quorum is a synchronous state" + ); + assert_eq!( + derive_replication_mode(&[], "standby1"), + "sync", + "a configured synchronous standby that has not connected still blocks commits" + ); + assert_eq!( + derive_replication_mode(&["potential".to_owned()], " "), + "async", + "whitespace is not a standby name" + ); +} diff --git a/crates/qbit-prism-server/tests/vardiff_convergence.rs b/crates/qbit-prism-server/tests/vardiff_convergence.rs new file mode 100644 index 000000000..c0a564872 --- /dev/null +++ b/crates/qbit-prism-server/tests/vardiff_convergence.rs @@ -0,0 +1,769 @@ +//! #558 (#487 S14 follow-up): vardiff converges per session over a real +//! Stratum socket. +//! +//! Two miners of different hashrates connect to the native listener at the +//! same startup difficulty: one well below its target difficulty (16x) and +//! one well above it (16x). Each solves real share proofs at its current +//! job's difficulty and paces its submits from a fixed hashrate: a share +//! every `difficulty / hashrate` seconds, on a schedule anchored to the +//! previous share's due time so jitter does not accumulate, and restarted +//! when a job at a new difficulty arrives, as a miner starts hashing the new +//! job then. Vardiff should +//! bring each session to `hashrate * target_seconds`, the difficulty at +//! which it submits one share per `target_seconds`. +//! +//! What each session is held to, from the difficulties the listener sent it +//! and when: +//! - it settles within `MAX_RETARGET_PERIODS_TO_SETTLE` retarget periods of +//! its first job, and within `MAX_ADJUSTMENTS_TO_SETTLE` adjustments: from +//! then on every difficulty keeps the share interval inside `BAND` of the +//! target interval. The listener sends nothing for an evaluation that +//! leaves the difficulty alone, so the period bound is on elapsed time; +//! - every adjustment, before and after settling, moves toward the target: +//! the difficulty never reverses direction, and never ends farther from +//! the target (as a ratio) than it started; +//! - it stays settled for at least `SETTLED_RETARGET_PERIODS` retarget +//! periods, with at most one further adjustment. +//! +//! The listener's clock is the real one (vardiff reads `std::time::Instant`), +//! so the premise that the client kept its pace is checked instead of +//! assumed. A session that converged passes either way. One that did not +//! is reported as a vardiff finding only if the premise held, and +//! otherwise as the premise failing (see `verdict`), so a stalled host is +//! not read as a vardiff regression. The fake backend is a pared-down copy of +//! `tests/stratum_protocol.rs`'s: no persistence, no resume, one tip. + +use qbit_pool_builder::{build_manifest, CoinbaseBuildRequest, WeightedEntitlement}; +use qbit_prism_server::{ + codec::{difficulty_target, double_sha256, Job, Submission}, + ledger::SessionId, + stratum::*, +}; +use serde_json::{json, Value}; +use std::{ + collections::HashSet, + sync::{ + atomic::{AtomicU32, AtomicU64, Ordering}, + Arc, Mutex, + }, + time::Duration, +}; +use tokio::{ + io::{AsyncBufReadExt, AsyncWriteExt, BufReader}, + net::{TcpListener, TcpStream}, + sync::{mpsc, watch}, + time::{sleep_until, timeout, Instant}, +}; + +/// The share interval vardiff aims for, and how often it may retarget. +const TARGET_SECONDS: f64 = 0.1; +const RETARGET_SECONDS: f64 = 0.5; +/// Both sessions start here; each target is 16x away from it. +const START_DIFFICULTY: f64 = 1.6e-8; +const START_ERROR: f64 = 16.0; +/// Settled: the share interval is within this factor of TARGET_SECONDS. +const BAND: f64 = 1.5; +/// A 16x error takes two adjustments at vardiff's 4x step limit; two more +/// allow for a window cut short by the listener's once-a-second timer. +const MAX_ADJUSTMENTS_TO_SETTLE: usize = 4; +/// Vardiff evaluates a session on each accepted share and on the listener's +/// once-a-second timer, at least RETARGET_SECONDS apart. The fast session +/// settles after two share-driven periods (1.0 s). The slow one sends no +/// share before the listener's first timer tick past RETARGET_SECONDS (0.5 +/// to 1.5 s in, about 1.0 s in practice: an idle step down), then needs one +/// more window of two shares (0.8 s): about 3.6 periods, 4.6 at worst. Six +/// periods (3.0 s) cover the worst case. +const MAX_RETARGET_PERIODS_TO_SETTLE: f64 = 6.0; +/// A share waiting longer than this past its due time means the client +/// stalled, half a retarget period. It only decides how a failure is +/// reported (see `verdict`), never whether a converged session passes. +const STALL_SECONDS: f64 = 0.25; +/// How long a session must then be seen staying settled. +const SETTLED_RETARGET_PERIODS: f64 = 8.0; +const RUN: Duration = Duration::from_secs(8); +/// Regtest bits would make every share a block; mainnet-scale bits keep the +/// shares proving exactly their job's share difficulty. +const NETWORK_BITS: &str = "1d00ffff"; + +#[derive(Default)] +struct Backend { + sessions: AtomicU32, + jobs: AtomicU64, + shares: Mutex>, +} + +impl MiningBackend for Backend { + type Context = (); + async fn new_session_id(&self) -> Result { + Ok((self.sessions.fetch_add(1, Ordering::Relaxed) + 1).into()) + } + async fn authorize(&self, username: &str) -> Result { + Ok(Worker { + username: username.into(), + payout_address: username.split('.').next().unwrap().into(), + worker_name: username.split_once('.').map(|(_, w)| w.into()), + p2mr_program_hex: "ab".repeat(32), + }) + } + async fn build_job( + &self, + worker: &Worker, + extranonce1: &str, + difficulty: f64, + minimum: f64, + ) -> Result, StratumError> { + let template = json!({"version":0x20000000u32,"bits":NETWORK_BITS,"curtime":1_700_000_000u32, + "previousblockhash":format!("{:064x}", 0),"transactions":[]}); + let manifest = build_manifest(CoinbaseBuildRequest { + block_height: 1, + coinbase_value_sats: 5_000_000_000, + entitlements: vec![WeightedEntitlement { + recipient_id: worker.payout_address.clone(), + order_key: worker.payout_address.clone(), + p2mr_program_hex: worker.p2mr_program_hex.clone(), + weight: 1, + }], + witness_nonce_hex: None, + witness_merkle_leaves_hex: vec![], + coinbase_script_sig_suffix_hex: Some(format!("{extranonce1}{}", "00".repeat(8))), + pinned_first_output: None, + }) + .unwrap(); + let job = Job::from_manifest( + format!("job-{}", self.jobs.fetch_add(1, Ordering::Relaxed)), + &template, + &manifest, + extranonce1, + 8, + difficulty, + minimum, + true, + ) + .unwrap(); + Ok(MiningJob { + wire: job, + context: Arc::new(()), + }) + } + async fn submit( + &self, + _worker: &Worker, + _job: &MiningJob<()>, + submission: Submission, + _grace: StaleGrace, + ) -> Result<(), StratumError> { + if !submission.share_pass { + return Err(StratumError::new( + 23, + "low difficulty share", + "low-difficulty", + )); + } + if !self + .shares + .lock() + .unwrap() + .insert(submission.block_hash_hex) + { + return Err(StratumError::new(22, "duplicate share", "duplicate-share")); + } + Ok(()) + } +} + +/// A share as submitted: seconds since the first job when it was due and +/// when it was actually sent, and its job's difficulty. +#[derive(Debug)] +struct Share { + due: f64, + sent: f64, + difficulty: f64, +} + +/// What one session saw: every difficulty it was sent, at seconds since its +/// first job, and every share it submitted. +#[derive(Debug, Default)] +struct Trace { + difficulties: Vec<(f64, f64)>, + shares: Vec, + /// Shares still unsent when a new difficulty replaced them, including + /// one held back while the last share awaited its answer: seconds when + /// it arrived, and how long past due the share was by then. + overdue_at_change: Vec<(f64, f64)>, + refusals: Vec, + ran_seconds: f64, +} + +/// A share proof for the current job at its share difficulty. +fn solve(notify: &Value, extranonce1: &str, difficulty: f64, nonce: &mut u32) -> Value { + let p = notify["params"].as_array().unwrap(); + let text = |index: usize| p[index].as_str().unwrap(); + let coinbase = hex::decode(format!( + "{}{extranonce1}0000000000000000{}", + text(2), + text(3) + )) + .unwrap(); + let merkle = double_sha256(&coinbase); + let mut previous = hex::decode(text(1)).unwrap(); + for word in previous.as_chunks_mut::<4>().0 { + word.reverse(); + } + let word = |index: usize| u32::from_str_radix(text(index), 16).unwrap().to_le_bytes(); + let target = difficulty_target(difficulty).unwrap(); + loop { + *nonce = nonce.checked_add(1).expect("nonce space exhausted"); + let header = [ + word(5).as_slice(), + previous.as_slice(), + merkle.as_slice(), + word(7).as_slice(), + word(6).as_slice(), + nonce.to_le_bytes().as_slice(), + ] + .concat(); + if num_bigint::BigUint::from_bytes_le(&double_sha256(&header)) <= target { + return json!({"id":null,"method":"mining.submit", + "params":[p[0].clone(),p[0],"0000000000000000",p[7],format!("{nonce:08x}")]}); + } + } +} + +/// Mine as `username` at `hashrate` (difficulty per second) for `RUN`. +async fn mine(address: std::net::SocketAddr, username: &str, hashrate: f64) -> Trace { + let (read, mut writer) = TcpStream::connect(address).await.unwrap().into_split(); + // One reader task, so waiting on a line never races the pacing timer. + let (lines, mut incoming) = mpsc::unbounded_channel::(); + tokio::spawn(async move { + let mut reader = BufReader::new(read).lines(); + while let Ok(Some(line)) = reader.next_line().await { + if lines.send(serde_json::from_str(&line).unwrap()).is_err() { + break; + } + } + }); + let mut send = async |value: Value| { + writer + .write_all(format!("{value}\n").as_bytes()) + .await + .unwrap() + }; + let next = async |incoming: &mut mpsc::UnboundedReceiver| { + timeout(Duration::from_secs(5), incoming.recv()) + .await + .expect("the listener sent nothing for 5 s") + .expect("the listener closed the connection") + }; + + send(json!({"id":1,"method":"mining.subscribe","params":[]})).await; + send(json!({"id":2,"method":"mining.authorize","params":[username,"x"]})).await; + let mut extranonce1 = String::new(); + let mut announced = None; + let mut job: Option<(Value, f64)> = None; + while job.is_none() { + let message = next(&mut incoming).await; + match (message["id"].as_u64(), message["method"].as_str()) { + (Some(1), _) => extranonce1 = message["result"][1].as_str().unwrap().into(), + (Some(2), _) => assert_eq!(message["result"], true, "{username} was not authorized"), + (_, Some("mining.set_difficulty")) => announced = message["params"][0].as_f64(), + (_, Some("mining.notify")) => job = Some((message, announced.unwrap())), + _ => {} + } + } + let (mut notify, mut difficulty) = job.unwrap(); + let started = Instant::now(); + let seconds = |at: Instant| at.duration_since(started).as_secs_f64(); + let deadline = started + RUN; + let mut trace = Trace { + difficulties: vec![(0.0, difficulty)], + ..Default::default() + }; + let (mut last_due, mut outstanding, mut id, mut nonce) = (started, None, 10u64, 0u32); + loop { + let interval = Duration::from_secs_f64(difficulty / hashrate); + let scheduled = last_due + interval; + let mut due = scheduled; + // A client that fell more than a share behind (a stalled host) + // resumes its pace instead of submitting the backlog in a burst. The + // share keeps its scheduled time, so the stall is still measured. + if Instant::now().saturating_duration_since(due) > interval { + due = Instant::now(); + } + tokio::select! { + _ = sleep_until(deadline) => break, + message = incoming.recv() => { + let message = message.expect("the listener closed the connection"); + match message["method"].as_str() { + Some("mining.set_difficulty") => announced = message["params"][0].as_f64(), + Some("mining.notify") => { + notify = message; + let sent = announced.expect("a job before any difficulty"); + if sent != difficulty { + let now = Instant::now(); + // Past due whether or not the last share is still + // unanswered: a slow answer delays the next share too. + if now > scheduled { + trace.overdue_at_change.push(( + seconds(now), + now.duration_since(scheduled).as_secs_f64(), + )); + } + trace.difficulties.push((seconds(now), sent)); + last_due = now; + } + difficulty = sent; + } + _ if message["id"].as_u64() == outstanding => { + if message["result"] != true { + trace.refusals.push(message); + } + outstanding = None; + } + _ => {} + } + } + _ = sleep_until(due), if outstanding.is_none() => { + let mut submit = solve(¬ify, &extranonce1, difficulty, &mut nonce); + submit["id"] = json!(id); + submit["params"][0] = json!(username); + send(submit).await; + trace.shares.push(Share { due: seconds(scheduled), sent: seconds(Instant::now()), difficulty }); + outstanding = Some(id); + id += 1; + last_due = due; + } + } + } + trace.ran_seconds = seconds(Instant::now()); + trace +} + +/// The pace the client kept at each difficulty it was sent: for each span +/// from one advertised difficulty to the next (or the end of the run), the +/// shares it sent there against the whole intervals the span holds at its +/// hashrate. The schedule restarts at each new difficulty, so a client that +/// kept its pace sent exactly that many, give or take the one in flight. +/// Returns `(seconds at the span's start, sent, expected)` per span. +fn pace_by_difficulty(trace: &Trace, hashrate: f64) -> Vec<(f64, usize, usize)> { + trace + .difficulties + .iter() + .enumerate() + .map(|(index, &(start, difficulty))| { + let end = trace + .difficulties + .get(index + 1) + .map_or(trace.ran_seconds, |(at, _)| *at); + let sent = trace + .shares + .iter() + .filter(|share| { + share.difficulty == difficulty && share.sent >= start && share.sent < end + }) + .count(); + // The epsilon keeps a whole number of intervals whole in floating point. + let expected = ((end - start) / (difficulty / hashrate) + 1e-9).floor() as usize; + (start, sent, expected) + }) + .collect() +} + +/// The longest any share waited past its due time, sent or not: a share +/// sent late, or one still unsent when a new difficulty replaced it, as +/// `(seconds when it was sent or replaced, seconds past due)`. +fn worst_stall(trace: &Trace) -> (f64, f64) { + trace + .shares + .iter() + .map(|share| (share.sent, share.sent - share.due)) + .chain(trace.overdue_at_change.iter().copied()) + .fold( + (0.0, 0.0), + |worst, next| if next.1 > worst.1 { next } else { worst }, + ) +} + +/// Whether a span's shares fell short of its pace: up to a fifth of the +/// expected shares may be missing, and never fewer than one (the share in +/// flight at the span's end). +fn short_of_pace(sent: usize, expected: usize) -> bool { + sent + (expected / 5).max(1) < expected +} + +/// The index of the first adjustment that moves away from where the first +/// one went (a reversal), or None when every adjustment keeps its direction. +fn reversal(difficulties: &[(f64, f64)]) -> Option { + let mut direction = None; + for (index, pair) in difficulties.windows(2).enumerate() { + let up = pair[1].1 > pair[0].1; + if *direction.get_or_insert(up) != up { + return Some(index + 1); + } + } + None +} + +/// The index of the first adjustment that leaves the difficulty farther +/// from `target`, as a ratio either way, than it was, or None when every +/// adjustment brings it closer. This holds after settling too, even inside +/// the band: vardiff only moves by 25% or more (its tolerance), and a window +/// cut short by the listener's timer misjudges this client's paced rate by at +/// most one share in six, which the EWMA shrinks further. So a move of that +/// size away from the target is a wrong estimate, not noise; and if the +/// client stalled, `verdict` says so. +fn away_from_target(difficulties: &[(f64, f64)], target: f64) -> Option { + let error = |difficulty: f64| (difficulty / target).ln().abs(); + difficulties + .windows(2) + .position(|pair| error(pair[1].1) > error(pair[0].1)) + .map(|index| index + 1) +} + +/// The index of the first difficulty from which every later one is within +/// `BAND` of `target`, or None when the session never settled. +fn settled_from(difficulties: &[(f64, f64)], target: f64) -> Option { + let within = |difficulty: f64| (1.0 / BAND..=BAND).contains(&(difficulty / target)); + let outside = difficulties.iter().rposition(|(_, d)| !within(*d)); + match outside { + None => Some(0), + Some(last) if last + 1 < difficulties.len() => Some(last + 1), + Some(_) => None, + } +} + +/// Why the client may not have kept its pace, so that the listener measured +/// something other than the hashrate the test states; empty when it kept +/// it. Judged at every difficulty the session was sent, as the shares were +/// actually sent: a share that waited more than STALL_SECONDS past due +/// (sent late, or still unsent when a new difficulty replaced it), a span +/// short of its shares, or a final mean interval off the hashrate's. The +/// client sends a share only once the last is answered, so a listener that +/// acknowledges slowly shows up here as well as a stalled host. +fn premise_problems(trace: &Trace, hashrate: f64) -> Vec { + let mut problems = Vec::new(); + let (at, past_due) = worst_stall(trace); + if past_due > STALL_SECONDS { + problems.push(format!( + "a share was {past_due:.3}s past due at {at:.2}s, a stall over {STALL_SECONDS}s" + )); + } + let short: Vec = pace_by_difficulty(trace, hashrate) + .iter() + .filter(|(_, sent, expected)| short_of_pace(*sent, *expected)) + .map(|(at, sent, expected)| format!("from {at:.2}s: {sent} of {expected}")) + .collect(); + if !short.is_empty() { + problems.push(format!("too few shares at some difficulty {short:?}")); + } + let (final_at, final_difficulty) = *trace.difficulties.last().unwrap(); + let expected = final_difficulty / hashrate; + let paced: Vec = trace + .shares + .iter() + .filter(|share| share.difficulty == final_difficulty && share.sent >= final_at) + .map(|share| share.sent) + .collect(); + if paced.len() < 10 { + problems.push(format!( + "only {} shares at the final difficulty", + paced.len() + )); + } else { + let measured = (paced.last().unwrap() - paced[0]) / (paced.len() - 1) as f64; + if !(0.8..=1.25).contains(&(measured / expected)) { + problems.push(format!( + "a {measured:.3}s mean interval at the final difficulty against the \ + {expected:.3}s the hashrate implies" + )); + } + } + problems +} + +/// Every way the session failed to converge as stated; empty when it did. +/// The second value is when it settled, if it did. +fn convergence_failures(trace: &Trace, target: f64) -> (Vec, Option) { + let mut failures = Vec::new(); + let start = trace.difficulties[0].1 / target; + if (start - START_ERROR).abs() >= 1e-9 && (start - 1.0 / START_ERROR).abs() >= 1e-9 { + failures.push(format!("it did not start {START_ERROR}x from its target")); + } + if let Some(index) = reversal(&trace.difficulties) { + failures.push(format!( + "adjustment {index} reversed the direction of the ones before it" + )); + } + if let Some(index) = away_from_target(&trace.difficulties, target) { + failures.push(format!( + "adjustment {index} left the difficulty farther from its target" + )); + } + let Some(settled) = settled_from(&trace.difficulties, target) else { + failures.push(format!("it never settled within {BAND}x of its target")); + return (failures, None); + }; + if settled > MAX_ADJUSTMENTS_TO_SETTLE { + failures.push(format!( + "it took {settled} adjustments to settle, more than {MAX_ADJUSTMENTS_TO_SETTLE}" + )); + } + let settled_at = trace.difficulties[settled].0; + if settled_at > MAX_RETARGET_PERIODS_TO_SETTLE * RETARGET_SECONDS { + failures.push(format!( + "it settled at {settled_at:.2}s, later than {MAX_RETARGET_PERIODS_TO_SETTLE} \ + retarget periods ({:.2}s)", + MAX_RETARGET_PERIODS_TO_SETTLE * RETARGET_SECONDS + )); + } + let held = trace.ran_seconds - settled_at; + if held < SETTLED_RETARGET_PERIODS * RETARGET_SECONDS { + failures.push(format!( + "it settled at {settled_at:.2}s, leaving {held:.2}s, less than \ + {SETTLED_RETARGET_PERIODS} retarget periods to show it stays settled" + )); + } + let changes = trace.difficulties.len() - 1 - settled; + if changes > 1 { + failures.push(format!("it changed {changes} times after settling")); + } + // Staying settled is shown by shares the listener saw after settling: + // at least half those the hold implies at the band's longest interval. + let shown = trace + .shares + .iter() + .filter(|share| share.sent >= settled_at) + .count(); + let needed = (held / (BAND * TARGET_SECONDS) / 2.0).floor() as usize; + if shown < needed { + failures.push(format!( + "only {shown} shares after settling, fewer than the {needed} needed to show it \ + stays settled over {held:.2}s" + )); + } + (failures, Some(settled_at)) +} + +/// The verdict. A session that converged passes, whatever its client's +/// pace: convergence despite a noisy client is still convergence. One that +/// did not is a vardiff finding only if the client kept its pace; otherwise +/// the failure names the broken premise first, so a stalled host is not +/// read as a vardiff regression. +fn verdict(premise: &[String], failures: &[String]) -> Result<(), String> { + match (failures.is_empty(), premise.is_empty()) { + (true, _) => Ok(()), + (false, true) => Err(format!("vardiff did not converge: {}", failures.join("; "))), + (false, false) => Err(format!( + "premise failed, the client did not keep its pace ({}), so these are not \ + vardiff findings: {}", + premise.join("; "), + failures.join("; ") + )), + } +} + +fn check(name: &str, trace: &Trace, hashrate: f64) { + let target = hashrate * TARGET_SECONDS; + assert!( + trace.refusals.is_empty(), + "{name}: refused shares {:?}", + trace.refusals + ); + let steps: Vec = trace + .difficulties + .iter() + .map(|(at, d)| format!("{at:.2}s:{:.3}x", d / target)) + .collect(); + let spans: Vec = pace_by_difficulty(trace, hashrate) + .iter() + .map(|(at, sent, expected)| format!("{at:.2}s:{sent}/{expected}")) + .collect(); + let (past_due_at, past_due) = worst_stall(trace); + let context = format!( + "{name}: difficulty / target over time {steps:?}; shares sent / expected per difficulty \ + {spans:?}; longest past due {past_due:.3}s at {past_due_at:.2}s" + ); + let premise = premise_problems(trace, hashrate); + let (failures, settled_at) = convergence_failures(trace, target); + if let Err(reason) = verdict(&premise, &failures) { + panic!("{context}: {reason}"); + } + eprintln!( + "{context}; settled in {:.1} retarget periods; client pace problems: {premise:?}", + settled_at.unwrap_or(f64::NAN) / RETARGET_SECONDS + ); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn sessions_below_and_above_their_target_settle_into_the_share_interval_band() { + let mut config = StratumConfig { + startup_difficulty: START_DIFFICULTY, + ..Default::default() + }; + config.vardiff.target_seconds = TARGET_SECONDS; + config.vardiff.retarget_seconds = RETARGET_SECONDS; + config.vardiff.minimum = 1e-9; + config.vardiff.maximum = 1e-3; + config.vardiff.validate().unwrap(); + let listener = TcpListener::bind("127.0.0.1:0").await.unwrap(); + let address = listener.local_addr().unwrap(); + let (_refresh, refresh_rx) = watch::channel(0); + let (shutdown, shutdown_rx) = watch::channel(false); + let task = tokio::spawn(run_listener( + listener, + config, + Arc::new(Backend::default()), + refresh_rx, + shutdown_rx, + Arc::new(qbit_prism_server::metrics::Metrics::default()), + )); + // Difficulty per second: the fast miner's target is 16x its start, the + // slow one's 1/16 of it. + let fast = START_DIFFICULTY * START_ERROR / TARGET_SECONDS; + let slow = START_DIFFICULTY / START_ERROR / TARGET_SECONDS; + let (fast_trace, slow_trace) = tokio::join!( + mine(address, "miner.fast", fast), + mine(address, "miner.slow", slow) + ); + shutdown.send(true).unwrap(); + task.await.unwrap().unwrap(); + check("below its target (fast)", &fast_trace, fast); + check("above its target (slow)", &slow_trace, slow); +} + +#[test] +fn settling_is_the_first_difficulty_after_the_last_one_outside_the_band() { + let target = 1.0; + assert_eq!( + settled_from(&[(0.0, 16.0), (1.0, 4.0), (2.0, 1.1)], target), + Some(2) + ); + assert_eq!(settled_from(&[(0.0, 1.2)], target), Some(0)); + assert_eq!( + settled_from(&[(0.0, 16.0), (1.0, 1.0), (2.0, 0.5)], target), + None + ); + assert_eq!( + settled_from(&[(0.0, 16.0), (1.0, 1.0), (2.0, 0.5), (3.0, 0.9)], target), + Some(3) + ); +} + +#[test] +fn a_reversal_is_any_adjustment_against_the_first_ones_direction() { + assert_eq!(reversal(&[(0.0, 16.0), (1.0, 4.0), (2.0, 1.0)]), None); + assert_eq!(reversal(&[(0.0, 0.0625), (1.0, 0.25), (2.0, 0.99)]), None); + assert_eq!(reversal(&[(0.0, 16.0)]), None); + // Settled at 1x, then back up to 1.4x: inside the band, but a reversal. + assert_eq!( + reversal(&[(0.0, 16.0), (1.0, 4.0), (2.0, 1.0), (3.0, 1.4)]), + Some(3) + ); + assert_eq!(reversal(&[(0.0, 0.0625), (1.0, 1.2), (2.0, 0.9)]), Some(2)); +} + +#[test] +fn a_span_short_of_its_pace_is_found_even_when_it_expects_two_shares() { + let share = |sent: f64, difficulty: f64| Share { + due: sent, + sent, + difficulty, + }; + // Hashrate 1: a difficulty-0.4 span of 0.8 s expects 2 shares. + let trace = |shares| Trace { + difficulties: vec![(0.0, 1.6), (1.0, 0.4), (1.8, 0.1)], + shares, + ran_seconds: 2.8, + ..Default::default() + }; + let paced = trace( + (0..10) + .map(|i| share(1.8 + 0.1 * f64::from(i), 0.1)) + .collect(), + ); + assert_eq!(pace_by_difficulty(&paced, 1.0)[1], (1.0, 0, 2)); + let full = trace( + [share(1.4, 0.4), share(1.79, 0.4)] + .into_iter() + .chain((1..10).map(|i| share(1.8 + 0.1 * f64::from(i), 0.1))) + .collect(), + ); + assert_eq!(pace_by_difficulty(&full, 1.0)[1], (1.0, 2, 2)); + assert_eq!(pace_by_difficulty(&full, 1.0)[2], (1.8, 9, 10)); + assert!(short_of_pace(0, 2), "a stalled two-share window is found"); + assert!(!short_of_pace(1, 2) && !short_of_pace(0, 0) && !short_of_pace(9, 10)); + assert!( + !short_of_pace(77, 80), + "the worst span seen under load passes" + ); + assert!(short_of_pace(63, 80)); +} + +#[test] +fn a_failure_with_a_broken_premise_is_not_reported_as_a_vardiff_finding() { + let failures = vec!["it settled at 3.40s".to_string()]; + let stalled = vec!["a share was 0.600s past due at 2.20s".to_string()]; + assert_eq!( + verdict(&stalled, &[]), + Ok(()), + "convergence despite a stall passes" + ); + assert!(verdict(&[], &failures) + .unwrap_err() + .starts_with("vardiff did not converge")); + assert!(verdict(&stalled, &failures) + .unwrap_err() + .starts_with("premise failed")); + // Codex's case on #609: a one-share span missed by a 2.2 s stall. + let trace = Trace { + difficulties: vec![(0.0, 1.6), (2.2, 0.4)], + shares: vec![], + overdue_at_change: vec![(2.2, 0.6)], + ran_seconds: 2.2, + ..Default::default() + }; + assert!(premise_problems(&trace, 1.0) + .iter() + .any(|problem| problem.starts_with("a share was 0.600s past due at 2.20s"))); +} + +#[test] +fn settling_without_shares_to_show_it_held_is_a_failure() { + let trace = |shares: usize| Trace { + difficulties: vec![(0.0, 16.0), (0.5, 4.0), (1.0, 1.0)], + shares: (0..shares) + .map(|i| Share { + due: 1.1 + 0.1 * i as f64, + sent: 1.1 + 0.1 * i as f64, + difficulty: 1.0, + }) + .collect(), + ran_seconds: 8.0, + ..Default::default() + }; + // Target 1 at TARGET_SECONDS: 7 s held needs 7 / 0.15 / 2 = 23 shares. + let (failures, _) = convergence_failures(&trace(2), 1.0); + assert!(failures + .iter() + .any(|failure| failure.starts_with("only 2 shares after settling"))); + assert!(convergence_failures(&trace(60), 1.0).0.is_empty()); +} + +#[test] +fn an_adjustment_that_overshoots_farther_from_the_target_is_found() { + assert_eq!( + away_from_target(&[(0.0, 0.0625), (1.0, 0.25), (2.0, 0.99)], 1.0), + None + ); + assert_eq!( + away_from_target(&[(0.0, 16.0), (1.0, 4.0), (2.0, 1.0)], 1.0), + None + ); + // Same direction all the way, but past the target and away from it. + assert_eq!( + away_from_target(&[(0.0, 0.0625), (1.0, 1.0), (2.0, 1.4)], 1.0), + Some(2) + ); + assert_eq!(reversal(&[(0.0, 0.0625), (1.0, 1.0), (2.0, 1.4)]), None); + // Crossing the target to a closer point is still closer. + assert_eq!(away_from_target(&[(0.0, 1.3), (1.0, 0.9)], 1.0), None); +} diff --git a/crates/qbit-prism-server/tests/window_balance_order.rs b/crates/qbit-prism-server/tests/window_balance_order.rs new file mode 100644 index 000000000..a24fc8a62 --- /dev/null +++ b/crates/qbit-prism-server/tests/window_balance_order.rs @@ -0,0 +1,161 @@ +//! The current-balance read keeps its SQL order (djh58's ruling on #327). +//! +//! `Ledger::snapshot` and `Ledger::read_window(BalanceSource::Current)` both +//! return the carried balances in the order +//! `qbit_current_carry_forward_balances()` yields them; only a clone is sorted, +//! for the digest. That order is the canonical `(order_key, recipient_id, +//! p2mr_program_hex)` key, compared under the database's collation, so on a C +//! collation it already equals the byte-wise canonical sort and a sort added in +//! the decoder would change nothing observable. Here the carry table's text +//! columns use an ICU collation, as a database created with a natural-language +//! default would, so the SQL order and the byte-wise sort disagree, and the +//! test sees which one the readers return. +//! +//! `window_reference::as_issued_snapshot_preserves_persisted_balance_order` +//! covers the as-issued path. + +use anyhow::{bail, ensure, Context, Result}; +use qbit_prism::{AcceptedShare, CarryForwardBalance}; +use qbit_prism_server::ledger::{BalanceSource, Ledger, WindowRef}; +use qbit_prism_test_gate as gate; +use sqlx::PgPool; + +/// ICU collations PostgreSQL 16 ships when built with ICU, which orders +/// `a-order` before `B-order`, where a byte-wise sort puts `B` first. +const COLLATIONS: &[&str] = &["und-x-icu", "en-x-icu", "en_US.utf8", "en_US"]; + +fn canonical(mut balances: Vec) -> Vec { + balances.sort_by(|a, b| { + a.order_key + .cmp(&b.order_key) + .then_with(|| a.recipient_id.cmp(&b.recipient_id)) + .then_with(|| a.p2mr_program_hex.cmp(&b.p2mr_program_hex)) + }); + balances +} + +fn order_keys(balances: &[CarryForwardBalance]) -> Vec<&str> { + balances + .iter() + .map(|balance| balance.order_key.as_str()) + .collect() +} + +async fn body(pool: &PgPool, ledger: &Ledger) -> Result<()> { + let mut collation = None; + for name in COLLATIONS { + let orders: Option = sqlx::query_scalar(&format!( + "SELECT 'a-order' < ('B-order' COLLATE \"{name}\") FROM pg_collation WHERE collname=$1 LIMIT 1" + )) + .bind(name) + .fetch_optional(pool) + .await + .ok() + .flatten(); + if orders == Some(true) { + collation = Some(*name); + break; + } + } + let Some(collation) = collation else { + bail!( + "none of the collations {COLLATIONS:?} exists on this server with a non-byte-wise order; \ + the test needs one to make the SQL order differ from the canonical sort" + ); + }; + sqlx::raw_sql(&format!( + "ALTER TABLE qbit_payout_carry_forward_current \ + ALTER COLUMN miner_id TYPE text COLLATE \"{collation}\", \ + ALTER COLUMN payout_order_key TYPE text COLLATE \"{collation}\"; \ + INSERT INTO qbit_pool_blocks(block_hash,block_height,parent_hash,coinbase_txid,payout_manifest_sha256,chain_state) \ + VALUES(repeat('aa',32),100,repeat('00',32),repeat('ab',32),repeat('ac',32),'confirmed'); \ + INSERT INTO qbit_payout_carry_forward(block_height,block_hash,miner_id,payout_order_key,p2mr_program,gross_amount_sats,prior_balance_sats,candidate_balance_sats,onchain_amount_sats,carry_forward_balance_sats,action) \ + VALUES(100,repeat('aa',32),'miner-b','B-order',decode(repeat('22',32),'hex'),500,0,500,0,500,'accrued'), \ + (100,repeat('aa',32),'miner-a','a-order',decode(repeat('11',32),'hex'),1000,0,1000,0,1000,'accrued');" + )) + .execute(pool) + .await?; + ledger + .append( + AcceptedShare { + share_seq: 0, + share_id: "balance-order:share".into(), + miner_id: "miner-a".into(), + order_key: "a-order".into(), + p2mr_program_hex: "11".repeat(32), + share_difficulty: 800, + network_difficulty: 100, + template_height: 100, + job_id: "job".into(), + job_issued_at_ms: 1, + accepted_at_ms: 0, + ntime: 1_800_000_000, + credit_policy: None, + }, + None, + ) + .await?; + + let sql: Vec = sqlx::query_scalar( + "SELECT payout_order_key::text FROM qbit_current_carry_forward_balances()", + ) + .fetch_all(pool) + .await?; + ensure!( + sql == ["a-order", "B-order"], + "under {collation} the SQL order is {sql:?}, not [a-order, B-order]; the precondition does not hold" + ); + let snapshot = ledger.snapshot(100).await?; + ensure!( + order_keys(&canonical(snapshot.prior_balances.clone())) == ["B-order", "a-order"], + "the byte-wise canonical sort agrees with the SQL order, so this case proves nothing" + ); + ensure!( + order_keys(&snapshot.prior_balances) == sql, + "Ledger::snapshot returned the balances as {:?}, not in their SQL order {sql:?}", + order_keys(&snapshot.prior_balances) + ); + let window = WindowRef::from_snapshot(&snapshot)?; + let read = ledger + .read_window(&window, BalanceSource::Current) + .await + .map_err(|error| anyhow::anyhow!("the current-balance window read failed: {error}"))?; + ensure!( + read.prior_balances == snapshot.prior_balances, + "read_window(Current) returned the balances as {:?}, not in their SQL order {sql:?}", + order_keys(&read.prior_balances) + ); + Ok(()) +} + +#[tokio::test] +async fn current_balance_reads_keep_the_sql_order_under_a_non_bytewise_collation() -> Result<()> { + let Some(raw) = gate::database_url(gate::site!())? else { + return Ok(()); + }; + let admin = PgPool::connect(&raw).await?; + let schema = format!("prism_balance_order_{}", uuid::Uuid::new_v4().simple()); + sqlx::query(&format!("CREATE SCHEMA {schema}")) + .execute(&admin) + .await?; + let outcome = async { + let mut url = url::Url::parse(&raw)?; + url.query_pairs_mut() + .append_pair("options", &format!("-csearch_path={schema}")); + let ledger = Ledger::connect(url.as_str(), "balance-order".into(), 4, true) + .await + .context("the ledger did not connect")?; + let result = body(&ledger.pool.clone(), &ledger).await; + ledger.pool.close().await; + result + } + .await; + let dropped = sqlx::query(&format!("DROP SCHEMA {schema} CASCADE")) + .execute(&admin) + .await; + admin.close().await; + match (outcome, dropped) { + (Ok(()), dropped) => dropped.map(|_| ()).map_err(Into::into), + (Err(error), _) => Err(error), + } +} diff --git a/crates/qbit-prism-server/tests/window_multiplier_invariant.rs b/crates/qbit-prism-server/tests/window_multiplier_invariant.rs new file mode 100644 index 000000000..368be9985 --- /dev/null +++ b/crates/qbit-prism-server/tests/window_multiplier_invariant.rs @@ -0,0 +1,268 @@ +//! The payout window's difficulty multiplier has one value, +//! `qbit_prism::PRISM_WINDOW_MULTIPLIER`, but several spellings. SQL bakes it +//! in as a literal in the dashboard query files and in the shipped schema, and +//! a handful of Rust sites embed it in inline SQL or render it into a read +//! model. This target enumerates those literal spellings and pins every one of +//! them to the constant, so changing the multiplier cannot leave a dashboard, +//! the audit window view or the archive horizon scaling by the old value. +//! +//! A multiplier site is the expression that scales a network difficulty into a +//! requested window weight, or the reported multiplier itself. The scanner +//! matches five idioms, each kept narrow so ordinary `* 8` arithmetic is not +//! swept in: +//! +//! * `difficulty-text-bind` — `$N::text::numeric * M`, the shape that passes +//! the scaled difficulty into `qbit_prism_window(...)` from a query file or +//! from inline SQL in Rust. +//! * `sql-difficulty-scale` — `[audit_]network_difficulty[::numeric] * +//! M::numeric`, the requested window weight computed from a column. The +//! trailing `::numeric` is required, so the bind form above cannot match +//! twice. +//! * `sql-window-multiplier-column` — `M::numeric AS window_multiplier`, the +//! value the audit window view reports. +//! * `json-window-multiplier` — `"window_multiplier": M` in a rendered read +//! model. A site that derives the field from the constant spells no literal +//! and is correctly invisible here. +//! * `rust-difficulty-scale` — `* Mu8` on a line that mentions a difficulty, +//! the `BigUint` scaling behind a read model's `requested_window_weight`. +//! +//! `*_revert_*.sql` rollback scripts are excluded: they undo a migration +//! rather than describe the deployed schema. So is +//! `tests/fixtures/schema_2x/001_share_ledger.sql`, a frozen copy of the 2.x +//! schema kept for the migration tests — it must keep whatever multiplier 2.x +//! shipped, and it lives outside the scanned roots for that reason. + +use std::path::{Path, PathBuf}; + +/// One multiplier literal: where it is, which idiom matched, and what it says. +struct Site { + path: String, + line: usize, + idiom: &'static str, + value: u128, +} + +/// Every file and the number of multiplier literals it is expected to spell. +/// A new spelling, or a site the scanner stops recognising, has to show up +/// here — either derive it from `PRISM_WINDOW_MULTIPLIER` instead, or add it. +const EXPECTED_SITES: &[(&str, usize)] = &[ + ("crates/qbit-prism-server/src/api/public.rs", 2), + ( + "crates/qbit-prism-server/src/api/queries/dashboard_blocks.sql", + 1, + ), + ( + "crates/qbit-prism-server/src/api/queries/dashboard_pool_snapshot.sql", + 1, + ), + ( + "crates/qbit-prism-server/src/api/queries/dashboard_reward_leaderboard.sql", + 1, + ), + ("crates/qbit-prism-server/src/api/read_models.rs", 2), + ("crates/qbit-prism-server/src/ledger/archive.rs", 2), + ("crates/qbit-prism/sql/001_share_ledger.sql", 3), +]; + +/// Reads the decimal integer at the front of `text`, with what follows it. +fn leading_integer(text: &str) -> Option<(u128, &str)> { + let end = text + .find(|c: char| !c.is_ascii_digit()) + .unwrap_or(text.len()); + Some((text[..end].parse().ok()?, &text[end..])) +} + +/// Reads the decimal integer at the end of `text`. +fn trailing_integer(text: &str) -> Option { + let start = text + .char_indices() + .rev() + .take_while(|(_, c)| c.is_ascii_digit()) + .last() + .map(|(index, _)| index)?; + text[start..].parse().ok() +} + +/// Steps over horizontal space, a single `*`, and the space after it. +fn after_times(text: &str) -> Option<&str> { + let rest = text.trim_start_matches([' ', '\t']).strip_prefix('*')?; + Some(rest.trim_start_matches([' ', '\t'])) +} + +fn scan_line(path: &str, number: usize, line: &str, rust: bool, sites: &mut Vec) { + let mut push = |idiom, value| { + sites.push(Site { + path: path.to_owned(), + line: number, + idiom, + value, + }) + }; + + for (offset, marker) in line.match_indices("::text::numeric") { + let bind = line[..offset].trim_end_matches(|c: char| c.is_ascii_digit()); + if bind.len() == offset || !bind.ends_with('$') { + continue; + } + if let Some((value, _)) = + after_times(&line[offset + marker.len()..]).and_then(leading_integer) + { + push("difficulty-text-bind", value); + } + } + + for (offset, marker) in line.match_indices("network_difficulty") { + let rest = &line[offset + marker.len()..]; + let rest = rest.strip_prefix("::numeric").unwrap_or(rest); + let Some((value, tail)) = after_times(rest).and_then(leading_integer) else { + continue; + }; + if tail.starts_with("::numeric") { + push("sql-difficulty-scale", value); + } + } + + for (offset, _) in line.match_indices("window_multiplier") { + let head = line[..offset].trim_end(); + let Some(head) = head + .strip_suffix("AS") + .or_else(|| head.strip_suffix("as")) + .map(str::trim_end) + .and_then(|head| head.strip_suffix("::numeric")) + else { + continue; + }; + if let Some(value) = trailing_integer(head) { + push("sql-window-multiplier-column", value); + } + } + + for (offset, marker) in line.match_indices("\"window_multiplier\"") { + let rest = line[offset + marker.len()..].trim_start_matches([' ', '\t']); + let Some(rest) = rest.strip_prefix(':') else { + continue; + }; + if let Some((value, _)) = leading_integer(rest.trim_start_matches([' ', '\t'])) { + push("json-window-multiplier", value); + } + } + + // Rust arithmetic only counts on a line that names a difficulty, which + // keeps unrelated `* 8` byte and second arithmetic out of the scan. + if rust && line.contains("difficulty") { + for (offset, _) in line.match_indices('*') { + let rest = line[offset + 1..].trim_start_matches([' ', '\t']); + let Some((value, tail)) = leading_integer(rest) else { + continue; + }; + let Some(tail) = tail.strip_prefix("u8") else { + continue; + }; + if !tail.starts_with(|c: char| c.is_alphanumeric() || c == '_') { + push("rust-difficulty-scale", value); + } + } + } +} + +/// The server's Rust and SQL sources, its migrations and the shipped schema, +/// relative to the workspace root so a failure names a path a reader can open. +/// No migration spells the multiplier today; the directory is scanned so one +/// that redefines the window view cannot introduce a copy unnoticed. +fn scanned_sources() -> Vec<(String, PathBuf)> { + let workspace = Path::new(env!("CARGO_MANIFEST_DIR")) + .join("../..") + .canonicalize() + .expect("the workspace root is two levels above this crate"); + let mut sources = Vec::new(); + let mut pending = vec![ + workspace.join("crates/qbit-prism-server/src"), + workspace.join("crates/qbit-prism-server/migrations"), + workspace.join("crates/qbit-prism/sql"), + ]; + while let Some(directory) = pending.pop() { + let entries = std::fs::read_dir(&directory) + .unwrap_or_else(|error| panic!("reading {}: {error}", directory.display())); + for entry in entries { + let path = entry.expect("a readable directory entry").path(); + if path.is_dir() { + pending.push(path); + continue; + } + let name = path.file_name().unwrap_or_default().to_string_lossy(); + if !(name.ends_with(".rs") || (name.ends_with(".sql") && !name.contains("_revert_"))) { + continue; + } + let relative = path + .strip_prefix(&workspace) + .unwrap_or(&path) + .to_string_lossy() + .into_owned(); + sources.push((relative, path.clone())); + } + } + sources.sort(); + sources +} + +#[test] +fn every_window_multiplier_literal_matches_the_constant() { + let mut sites = Vec::new(); + for (relative, path) in scanned_sources() { + let rust = relative.ends_with(".rs"); + let text = std::fs::read_to_string(&path) + .unwrap_or_else(|error| panic!("reading {relative}: {error}")); + for (index, line) in text.lines().enumerate() { + scan_line(&relative, index + 1, line, rust, &mut sites); + } + } + assert!( + !sites.is_empty(), + "the multiplier scanner matched nothing; its idioms have gone stale, \ + see this file's header comment" + ); + + // Report every disagreeing site at once: a multiplier change that missed + // several spellings should not be fixed one failure at a time. + let wrong: Vec = sites + .iter() + .filter(|site| site.value != qbit_prism::PRISM_WINDOW_MULTIPLIER) + .map(|site| format!("{}:{}: found {}", site.path, site.line, site.value)) + .collect(); + assert!( + wrong.is_empty(), + "every window multiplier literal must equal \ + qbit_prism::PRISM_WINDOW_MULTIPLIER ({}), but {} {}:\n{}", + qbit_prism::PRISM_WINDOW_MULTIPLIER, + wrong.len(), + if wrong.len() == 1 { + "site disagrees" + } else { + "sites disagree" + }, + wrong.join("\n") + ); + + let mut found: Vec<(&str, usize)> = Vec::new(); + for site in &sites { + match found.iter_mut().find(|(path, _)| *path == site.path) { + Some((_, count)) => *count += 1, + None => found.push((&site.path, 1)), + } + } + found.sort(); + assert_eq!( + found, + EXPECTED_SITES.to_vec(), + "the set of multiplier spellings changed. A new site must either derive \ + its value from qbit_prism::PRISM_WINDOW_MULTIPLIER — the bound-parameter \ + or constant form the rest of the server already uses — or be added to \ + EXPECTED_SITES in this file. A site that vanished means the scanner no \ + longer recognises its idiom; see this file's header comment. \ + Idioms matched: {:?}", + sites + .iter() + .map(|site| format!("{}:{}: {}", site.path, site.line, site.idiom)) + .collect::>() + ); +} diff --git a/crates/qbit-prism-server/tests/window_read_oracle.rs b/crates/qbit-prism-server/tests/window_read_oracle.rs new file mode 100644 index 000000000..edfbf7773 --- /dev/null +++ b/crates/qbit-prism-server/tests/window_read_oracle.rs @@ -0,0 +1,1355 @@ +//! An independent SQL oracle for the payout-window read (issue #269, item 3). +//! +//! `Ledger::snapshot` builds the payout window by paging backwards through +//! `qbit_share_ledger` 4096 rows at a time, subtracting each row's difficulty +//! from a remaining weight until the weight is exhausted. This file recomputes +//! the same window with a single unpaged statement -- one eligibility filter, +//! one window function for the weight cut, one `json_agg` -- and compares the +//! two answers row by row and field by field. +//! +//! The oracle is deliberately written from scratch. Its eligibility filter, +//! its weight cut and its anchor barrier share no text, no constant and no +//! private item with `ledger.rs`, and it reaches the ledger only through the +//! public `Ledger` API, so it keeps working across the planned split of +//! `ledger.rs` into submodules. +//! +//! Two helpers do repeat SQL that `ledger.rs` also contains, and both are +//! deliberate: +//! +//! * `oracle_cutoff` is the same one-line aggregate as the read's cutoff +//! query. There is only one way to write "the newest accepted sequence", so +//! an independent phrasing would be a rename rather than a second opinion. +//! It still earns its place, because it is a *separate copy*: a change to +//! the read's cutoff moves one copy and not the other, and the two then +//! disagree. +//! * `compare_conversion` embeds the read's own +//! `to_timestamp($N::double precision/1000)`. That is the point -- it asks +//! what the read's conversion does to a given millisecond. It classifies +//! anchors and read-backs and never decides membership, so it cannot make a +//! wrong window look right; and if the read's conversion changed, this copy +//! would go stale and the mismatch would surface as a failure rather than a +//! silent pass. +//! +//! PRISM_TEST_DATABASE_URL=postgres://postgres:prism@127.0.0.1:5432/postgres \ +//! cargo test -p qbit-prism-server --test window_read_oracle -- --nocapture + +use anyhow::{anyhow, bail, ensure, Context, Result}; +use qbit_prism::AcceptedShare; +use qbit_prism_server::ledger::{Ledger, Snapshot}; +use qbit_prism_test_gate as gate; +use serde::Deserialize; +use sqlx::PgPool; + +#[path = "support/ledger_database.rs"] +#[allow(dead_code)] +mod ledger_database; +use ledger_database::FixtureDatabase; + +// --------------------------------------------------------------------------- +// Integration guard +// --------------------------------------------------------------------------- +// +// Every test here takes `PRISM_TEST_DATABASE_URL` through the workspace's +// shared integration gate (`qbit_prism_test_gate`), whose decision table this +// file's original guard was the model for: present, run; missing while +// `PRISM_TEST_REQUIRE_INTEGRATION=1` or `GITHUB_JOB=prism-native-postgres`, +// fail naming the variable and the test; missing otherwise, print one +// prefixed skip line and return. The `prism-native-postgres` job sets the +// switch, and these eight tests are listed in `test/prism-gated-tests.txt`, +// so the job's execution manifest proves they ran. See +// `docs/prism-integration-test-gate.md`. + +// --------------------------------------------------------------------------- +// Schema harness +// --------------------------------------------------------------------------- + +/// Runs `body` against a ledger on a schema of its own, in a database of its +/// own, and drops that database afterwards whether the body passed or failed. +/// PostgreSQL scopes advisory locks to a database, so a shared database would +/// queue this ledger's migration and order keys behind other tests' ledgers. +/// +/// The body's own error is what a failing test reports; a cleanup failure is +/// attached to it as context rather than replacing it, so a leaked database can +/// never hide the assertion that actually failed. When the body passed, a +/// cleanup failure is the result, because a database this file could not drop +/// is worth knowing about. Every path reaches the cleanup, a failing one +/// included, so a red run leaves no fixture database behind on a server that +/// outlives it. +async fn in_fresh_schema(raw: &str, body: F) -> Result<()> +where + F: FnOnce(Ledger) -> Fut, + Fut: std::future::Future>, +{ + let fixture = FixtureDatabase::open(raw, "prism_test_").await?; + // A failure to connect still has to drop the database `open` just made. + let (outcome, ledger) = + match Ledger::connect(&fixture.url, "window-read-oracle".to_owned(), 8, true).await { + Ok(ledger) => (body(ledger.clone()).await, Some(ledger)), + Err(error) => (Err(error), None), + }; + // The ledger's pool closes before its database is dropped. + if let Some(ledger) = ledger { + ledger.pool.close().await; + } + fixture.close(outcome).await +} + +/// The integration guard followed by `in_fresh_schema`, for a test that wants +/// exactly one schema. Returns without running the body when the guard skips. +async fn with_schema(body: F) -> Result<()> +where + F: FnOnce(Ledger) -> Fut, + Fut: std::future::Future>, +{ + let Some(raw) = gate::database_url(gate::site!())? else { + return Ok(()); + }; + in_fresh_schema(&raw, body).await +} + +// --------------------------------------------------------------------------- +// Exact timestamp arithmetic +// --------------------------------------------------------------------------- + +/// Builds a `timestamptz` from an integer count of microseconds since the +/// epoch, using integer arithmetic only. +/// +/// Splitting the count into whole seconds plus a sub-second remainder keeps +/// every intermediate factor small, which matters for the far-future anchors +/// this file exercises. This is the fixture's own convention; it is not the +/// conversion `ledger.rs` uses. +fn timestamp_from_micros(expr: &str) -> String { + format!( + "(timestamptz 'epoch' + (({expr}) / 1000000) * interval '1 second' \ + + (({expr}) % 1000000) * interval '1 microsecond')" + ) +} + +/// The same idea for a millisecond count, used for the oracle's anchor +/// barrier. +fn timestamp_from_millis(expr: &str) -> String { + format!( + "(timestamptz 'epoch' + (({expr}) / 1000) * interval '1 second' \ + + (({expr}) % 1000) * interval '1 millisecond')" + ) +} + +// --------------------------------------------------------------------------- +// Fixture rows +// --------------------------------------------------------------------------- + +/// One ledger row, written with direct SQL. +/// +/// Direct inserts are what let a scenario express what the public append path +/// cannot: `accepted = false` rows, timestamps placed exactly on a boundary, +/// sub-millisecond timestamps, and a `job_issued_at` that isolates its own +/// predicate by sitting after `accepted_at`. Migration 002 makes share rows +/// immutable, so every fixture is built by insert alone -- nothing here +/// updates or deletes a share. +#[derive(Clone, Debug)] +struct ShareSpec { + share_id: String, + miner_id: String, + order_key: String, + program_hex: String, + share_difficulty: u128, + network_difficulty: u128, + template_height: i64, + job_id: String, + job_issued_at_us: i64, + accepted_at_us: i64, + ntime: i64, + credit_policy: Option, + accepted: bool, +} + +impl ShareSpec { + /// A distinct, valid accepted row landing at `accepted_at_us`. + fn new(index: u64, accepted_at_us: i64) -> Self { + let index_i64 = i64::try_from(index).unwrap_or(i64::MAX); + Self { + share_id: format!("oracle:{index:064x}"), + miner_id: format!("miner-{}", index % 5), + order_key: format!("order-{}", index % 3), + program_hex: format!("{:02x}", index % 251).repeat(32), + share_difficulty: 8, + network_difficulty: 1_000_000, + template_height: 900_000 + index_i64, + job_id: format!("job-{index}"), + job_issued_at_us: accepted_at_us - 1_000, + accepted_at_us, + ntime: 1_800_000_000 + index_i64 % 97, + credit_policy: None, + accepted: true, + } + } + + fn difficulty(mut self, difficulty: u128) -> Self { + self.share_difficulty = difficulty; + self + } + + fn rejected(mut self) -> Self { + self.accepted = false; + self + } + + fn issued_at_us(mut self, micros: i64) -> Self { + self.job_issued_at_us = micros; + self + } +} + +/// A share for the public append path. `accepted_at_ms` is left at zero +/// because `Ledger::append` stamps it from the ledger clock. +fn writer_share(index: u64, job_issued_at_ms: i64) -> AcceptedShare { + AcceptedShare { + share_seq: 0, + share_id: format!("writer:{index:064x}"), + miner_id: format!("miner-{}", index % 5), + order_key: format!("order-{}", index % 3), + p2mr_program_hex: format!("{:02x}", index % 251).repeat(32), + share_difficulty: 8, + network_difficulty: 1_000_000, + template_height: 900_000 + index, + job_id: format!("job-{index}"), + job_issued_at_ms, + accepted_at_ms: 0, + ntime: 1_800_000_000 + u32::try_from(index % 97).unwrap_or(0), + credit_policy: None, + } +} + +/// Places two shares on the anchor through the public append path, and +/// returns their sequences as `(on the anchor, one millisecond after it)`. +/// +/// `Ledger::append` stamps `accepted_at` with +/// `GREATEST(ledger_clock_ms, floor(now))` -- no `+1`, unlike the read's +/// anchor bump -- and stores both timestamps through the writer's own +/// conversion, requiring only that `job_issued_at_ms <= accepted_at_ms`. So +/// pinning the ledger clock chooses the acceptance millisecond exactly, and +/// the public API alone can put a real share on either side of an anchor: one +/// accepted at the anchor on both timestamps, and one accepted a millisecond +/// past it. The credited window must contain the first and not the second. +async fn append_anchor_pair(ledger: &Ledger, anchor_ms: i64, index: u64) -> Result<(u64, u64)> { + set_ledger_clock(&ledger.pool, anchor_ms + 1).await?; + let after = ledger + .append(writer_share(index, anchor_ms), None) + .await + .context("appending the share accepted after the anchor")? + .share; + set_ledger_clock(&ledger.pool, anchor_ms).await?; + let on = ledger + .append(writer_share(index + 1, anchor_ms), None) + .await + .context("appending the share accepted on the anchor")? + .share; + ensure!( + on.accepted_at_ms == anchor_ms && on.job_issued_at_ms == anchor_ms, + "the append path was expected to accept a share exactly on anchor {anchor_ms}, got \ + accepted_at_ms={} job_issued_at_ms={}", + on.accepted_at_ms, + on.job_issued_at_ms + ); + ensure!( + after.accepted_at_ms == anchor_ms + 1, + "the append path was expected to accept a share one millisecond after anchor \ + {anchor_ms}, got accepted_at_ms={}", + after.accepted_at_ms + ); + Ok((on.share_seq, after.share_seq)) +} + +/// Inserts one fixture row and returns the sequence PostgreSQL assigned it. +async fn insert_share(pool: &PgPool, spec: &ShareSpec) -> Result { + let sql = format!( + "INSERT INTO qbit_share_ledger(share_id,miner_id,payout_order_key,p2mr_program,\ + share_difficulty,network_difficulty,template_height,job_id,job_issued_at,ntime,\ + accepted_at,credit_policy,accepted,reject_reason,writer_id,writer_epoch) \ + VALUES($1,$2,$3,decode($4,'hex'),$5::text::numeric,$6::text::numeric,$7,$8,{job},$10,\ + {accepted},$12,$13,$14,'window-read-oracle',0) RETURNING share_seq", + job = timestamp_from_micros("$9"), + accepted = timestamp_from_micros("$11"), + ); + let reject_reason = if spec.accepted { + None + } else { + Some("stale-job") + }; + let seq: i64 = sqlx::query_scalar(&sql) + .bind(&spec.share_id) + .bind(&spec.miner_id) + .bind(&spec.order_key) + .bind(&spec.program_hex) + .bind(spec.share_difficulty.to_string()) + .bind(spec.network_difficulty.to_string()) + .bind(spec.template_height) + .bind(&spec.job_id) + .bind(spec.job_issued_at_us) + .bind(spec.ntime) + .bind(spec.accepted_at_us) + .bind(&spec.credit_policy) + .bind(spec.accepted) + .bind(reject_reason) + .fetch_one(pool) + .await?; + Ok(seq) +} + +/// Bulk-inserts `count` accepted rows of difficulty 8, one millisecond apart, +/// the newest landing at `last_accepted_us`. One statement keeps the +/// multi-page scenarios cheap. +async fn insert_run(pool: &PgPool, count: i64, last_accepted_us: i64) -> Result<()> { + let accepted = format!("({last_accepted_us}::bigint - ($2 - i) * 1000)"); + let issued = format!("({accepted} - 1000)"); + let sql = format!( + "INSERT INTO qbit_share_ledger(share_id,miner_id,payout_order_key,p2mr_program,\ + share_difficulty,network_difficulty,template_height,job_id,job_issued_at,ntime,\ + accepted_at,credit_policy,accepted,writer_id,writer_epoch) \ + SELECT 'oracle-run:'||lpad(i::text,32,'0'),'miner-'||(i%5),'order-'||(i%3),\ + decode(repeat(lpad(to_hex(i%251),2,'0'),32),'hex'),8::numeric,1000000::numeric,\ + 900000+i,'job-'||i,{job},1800000000+(i%97),{acc},NULL,true,'window-read-oracle',0 \ + FROM generate_series($1,$2) i", + job = timestamp_from_micros(&issued), + acc = timestamp_from_micros(&accepted), + ); + sqlx::query(&sql) + .bind(1_i64) + .bind(count) + .execute(pool) + .await?; + Ok(()) +} + +/// Pins the anchor `Ledger::snapshot` will capture, and the millisecond +/// `Ledger::append` will stamp on the next share it accepts. +/// +/// Both take `GREATEST(ledger_clock_ms, floor(now))`, so any value above the +/// wall clock becomes that millisecond exactly. +async fn set_ledger_clock(pool: &PgPool, anchor_ms: i64) -> Result<()> { + let now_ms = wall_clock_ms(pool).await?; + ensure!( + anchor_ms > now_ms, + "fixture anchor {anchor_ms} must sit above the wall clock {now_ms} to pin the snapshot" + ); + sqlx::query("UPDATE qbit_prism_cluster SET ledger_clock_ms=$1 WHERE singleton") + .bind(anchor_ms) + .execute(pool) + .await?; + Ok(()) +} + +async fn wall_clock_ms(pool: &PgPool) -> Result { + Ok( + sqlx::query_scalar("SELECT floor(extract(epoch FROM clock_timestamp())*1000)::bigint") + .fetch_one(pool) + .await?, + ) +} + +/// Margin between the wall clock and a derived anchor. A day is far more than +/// a test run needs, and it keeps a derived anchor legible in a failure +/// message as "tomorrow" rather than "a few seconds from now". +const ANCHOR_MARGIN_MS: i64 = 86_400_000; + +/// Derived anchors land on a round millisecond boundary so a scenario can add +/// its own suffix -- `999`, `1_000`, `1_001` -- without disturbing the +/// magnitude. +const ANCHOR_GRANULARITY_MS: i64 = 1_000_000; + +/// An anchor at an ordinary magnitude, derived from the database clock. +/// +/// A pinned anchor has to sit above the wall clock, so a hard-coded ordinary +/// anchor is a dated fuse: the suite would start failing on the day the clock +/// passed it, for no reason in the code. Deriving it at run time removes the +/// date from the fixture while keeping the millisecond suffix, which is the +/// part the boundary scenarios actually exercise. +async fn derived_anchor(pool: &PgPool, suffix_ms: i64) -> Result { + let now_ms = wall_clock_ms(pool).await?; + let floor = now_ms + .checked_add(ANCHOR_MARGIN_MS) + .context("derived anchor margin overflow")?; + // Round strictly up, so the anchor clears the margin even when the wall + // clock already sits on a granularity boundary. + let base = floor + .div_euclid(ANCHOR_GRANULARITY_MS) + .checked_add(1) + .and_then(|units| units.checked_mul(ANCHOR_GRANULARITY_MS)) + .context("derived anchor rounding overflow")?; + base.checked_add(suffix_ms) + .context("derived anchor suffix overflow") +} + +/// How a scenario chooses its anchor. +/// +/// No fixed anchor may ever fall below the wall clock, so fixed anchors are +/// used only for the far-future magnitude cases; everything at an ordinary +/// magnitude is derived from the database clock at run time. +#[derive(Clone, Copy, Debug)] +enum Anchor { + Derived(i64), + Fixed(i64), +} + +impl Anchor { + async fn resolve(self, pool: &PgPool) -> Result { + match self { + Anchor::Derived(suffix_ms) => derived_anchor(pool, suffix_ms).await, + Anchor::Fixed(anchor_ms) => Ok(anchor_ms), + } + } +} + +// --------------------------------------------------------------------------- +// The oracle +// --------------------------------------------------------------------------- + +/// One share as the oracle sees it. Difficulties travel as decimal text +/// because a JSON number cannot carry a `u128`. +#[derive(Debug, Deserialize)] +struct OracleShare { + share_seq: i64, + share_id: String, + miner_id: String, + order_key: String, + p2mr_program_hex: String, + share_difficulty: String, + network_difficulty: String, + template_height: i64, + job_id: String, + job_issued_at_ms: i64, + accepted_at_ms: i64, + /// Whether `job_issued_at` is stored on a whole millisecond. + job_issued_at_on_grid: bool, + /// Whether `accepted_at` is stored on a whole millisecond. + accepted_at_on_grid: bool, + ntime: i64, + credit_policy: Option, +} + +#[derive(Debug, Deserialize)] +struct OracleResult { + eligible: i64, + window: Vec, +} + +/// Recomputes the payout window in one unpaged statement. +/// +/// Eligibility is `accepted`, `share_seq <= cutoff`, and both timestamps at or +/// before the anchor, where the anchor barrier is built by exact integer +/// millisecond arithmetic rather than by the read's floating-point conversion. +/// +/// The weight cut is a window function: walking newest-first, a row belongs to +/// the window exactly when the total difficulty of everything newer than it is +/// still below the requested weight. That is the closed form of the read's +/// subtract-until-zero loop, saturation included -- once the running sum +/// reaches the weight, no older row can qualify -- and it keeps the row that +/// straddles the boundary. +/// +/// Each row also reports, per timestamp, whether the stored value lies on a +/// whole millisecond. `assert_window_matches` uses those two flags to decide +/// how strictly to hold the read's millisecond fields; see +/// `compare_millisecond_field`. +async fn oracle_window(pool: &PgPool, snapshot: &Snapshot, weight: u128) -> Result { + let barrier = timestamp_from_millis("$2"); + let sql = format!( + "WITH eligible AS ( + SELECT share_seq,share_id,miner_id,payout_order_key, + encode(p2mr_program,'hex') AS p2mr_program_hex, + share_difficulty,network_difficulty,template_height,job_id, + floor(extract(epoch FROM job_issued_at)*1000)::bigint AS job_issued_at_ms, + floor(extract(epoch FROM accepted_at)*1000)::bigint AS accepted_at_ms, + extract(microseconds FROM job_issued_at)::bigint%1000=0 + AS job_issued_at_on_grid, + extract(microseconds FROM accepted_at)::bigint%1000=0 + AS accepted_at_on_grid, + ntime,credit_policy + FROM qbit_share_ledger + WHERE accepted + AND share_seq <= $1 + AND accepted_at <= {barrier} + AND job_issued_at <= {barrier} + ), weighted AS ( + SELECT eligible.*, + COALESCE(sum(share_difficulty) OVER ( + ORDER BY share_seq DESC + ROWS BETWEEN UNBOUNDED PRECEDING AND 1 PRECEDING),0) AS newer_weight + FROM eligible + ) + SELECT json_build_object( + 'eligible',(SELECT count(*) FROM eligible), + 'window',COALESCE(( + SELECT json_agg(json_build_object( + 'share_seq',share_seq, + 'share_id',share_id, + 'miner_id',miner_id, + 'order_key',payout_order_key, + 'p2mr_program_hex',p2mr_program_hex, + 'share_difficulty',share_difficulty::text, + 'network_difficulty',network_difficulty::text, + 'template_height',template_height, + 'job_id',job_id, + 'job_issued_at_ms',job_issued_at_ms, + 'accepted_at_ms',accepted_at_ms, + 'job_issued_at_on_grid',job_issued_at_on_grid, + 'accepted_at_on_grid',accepted_at_on_grid, + 'ntime',ntime, + 'credit_policy',credit_policy) ORDER BY share_seq) + FROM weighted WHERE newer_weight < $3::numeric),'[]'::json))" + ); + let value: serde_json::Value = sqlx::query_scalar(&sql) + .bind(i64::try_from(snapshot.share_seq)?) + .bind(snapshot.anchor_ms) + .bind(weight.to_string()) + .fetch_one(pool) + .await?; + serde_json::from_value(value).context("decoding the oracle window") +} + +/// The cutoff the read should have captured, recomputed independently. +async fn oracle_cutoff(pool: &PgPool) -> Result { + Ok(sqlx::query_scalar( + "SELECT COALESCE(max(share_seq),0) FROM qbit_share_ledger WHERE accepted", + ) + .fetch_one(pool) + .await?) +} + +/// Total rows in the fixture, for the per-scenario row-count report. +async fn total_rows(pool: &PgPool) -> Result { + Ok(sqlx::query_scalar("SELECT count(*) FROM qbit_share_ledger") + .fetch_one(pool) + .await?) +} + +// --------------------------------------------------------------------------- +// Comparison +// --------------------------------------------------------------------------- + +fn compare_field( + scenario: &str, + position: usize, + field: &str, + read: &T, + oracle: &T, +) -> Result<()> { + ensure!( + read == oracle, + "{scenario}: share {position} disagrees on {field}: \ + Ledger::snapshot={read:?} oracle={oracle:?}" + ); + Ok(()) +} + +/// Compares one millisecond field of one share against the oracle. +/// +/// The oracle reports the floor of the stored timestamp, and whether that +/// timestamp lies on a whole millisecond. +/// +/// On the grid, the read-back must be exact. That is the case that matters: +/// every timestamp `Ledger::append` can write lands on a whole millisecond, +/// because `AcceptedShare` carries integer milliseconds and nothing finer, and +/// so does every direct insert this file places at a whole-millisecond offset. +/// +/// Off the grid, the stored instant falls between two milliseconds and +/// `AcceptedShare` has no way to say so. Only a direct insert can produce such +/// a row -- scenarios c and e write them deliberately, 400 microseconds either +/// side of the anchor, to prove the barrier compares instants rather than +/// milliseconds -- and rounding one is as defensible as flooring it. So the +/// read may return the floor or the floor plus one, and nothing else: a read +/// that lost a whole millisecond, or gained two, still fails. +/// +/// The two timestamps are judged independently, each on its own flag, so a +/// read that treated `accepted_at` and `job_issued_at` differently would still +/// be caught. Every other field, membership and order stay exact. +fn compare_millisecond_field( + scenario: &str, + position: usize, + field: &str, + read: i64, + floor: i64, + on_grid: bool, +) -> Result<()> { + if on_grid { + ensure!( + read == floor, + "{scenario}: share {position} disagrees on {field}: the stored timestamp lies on a \ + whole millisecond, so Ledger::snapshot had to return {floor}, but it returned {read}" + ); + return Ok(()); + } + ensure!( + read == floor || read == floor + 1, + "{scenario}: share {position} disagrees on {field}: the stored timestamp sits between \ + milliseconds, so Ledger::snapshot had to return {floor} or {}, whichever its rounding \ + convention gives, but it returned {read}", + floor + 1 + ); + Ok(()) +} + +/// Asserts membership, order and every field of every share. +fn assert_window_matches( + scenario: &str, + read: &[AcceptedShare], + oracle: &[OracleShare], +) -> Result<()> { + if read.len() != oracle.len() { + let read_head = read.first().map(|share| (share.share_seq, &share.share_id)); + let oracle_head = oracle + .first() + .map(|share| (share.share_seq, &share.share_id)); + bail!( + "{scenario}: window size disagrees: Ledger::snapshot returned {} shares \ + (first {read_head:?}), oracle returned {} shares (first {oracle_head:?})", + read.len(), + oracle.len() + ); + } + for (position, (read, oracle)) in read.iter().zip(oracle.iter()).enumerate() { + compare_field( + scenario, + position, + "share_seq", + &read.share_seq, + &u64::try_from(oracle.share_seq)?, + )?; + compare_field( + scenario, + position, + "share_id", + &read.share_id, + &oracle.share_id, + )?; + compare_field( + scenario, + position, + "miner_id", + &read.miner_id, + &oracle.miner_id, + )?; + compare_field( + scenario, + position, + "order_key", + &read.order_key, + &oracle.order_key, + )?; + compare_field( + scenario, + position, + "p2mr_program_hex", + &read.p2mr_program_hex, + &oracle.p2mr_program_hex, + )?; + compare_field( + scenario, + position, + "share_difficulty", + &read.share_difficulty, + &oracle.share_difficulty.parse::()?, + )?; + compare_field( + scenario, + position, + "network_difficulty", + &read.network_difficulty, + &oracle.network_difficulty.parse::()?, + )?; + compare_field( + scenario, + position, + "template_height", + &read.template_height, + &u64::try_from(oracle.template_height)?, + )?; + compare_field(scenario, position, "job_id", &read.job_id, &oracle.job_id)?; + compare_millisecond_field( + scenario, + position, + "job_issued_at_ms", + read.job_issued_at_ms, + oracle.job_issued_at_ms, + oracle.job_issued_at_on_grid, + )?; + compare_millisecond_field( + scenario, + position, + "accepted_at_ms", + read.accepted_at_ms, + oracle.accepted_at_ms, + oracle.accepted_at_on_grid, + )?; + compare_field( + scenario, + position, + "ntime", + &read.ntime, + &u32::try_from(oracle.ntime)?, + )?; + compare_field( + scenario, + position, + "credit_policy", + &read.credit_policy, + &oracle.credit_policy, + )?; + } + for pair in read.windows(2) { + ensure!( + pair[0].share_seq < pair[1].share_seq, + "{scenario}: Ledger::snapshot returned shares out of ascending order at {} then {}", + pair[0].share_seq, + pair[1].share_seq + ); + } + Ok(()) +} + +/// Ceiling on a single payout-window read. +/// +/// The read pages backwards until its weight is exhausted or a page comes back +/// empty, so a cursor that stops advancing turns it into an unbounded loop +/// rather than a wrong answer. The largest fixture here -- 12000 rows over +/// three pages -- returns in well under a second, so this ceiling is pure +/// headroom for a slow or contended server while still converting +/// non-termination into a named failure instead of a hung test run. +const SNAPSHOT_CEILING: std::time::Duration = std::time::Duration::from_secs(30); + +/// Pins the anchor, runs the read under the ceiling, and confirms the read +/// captured the anchor that was pinned. +async fn pinned_snapshot( + scenario: &str, + ledger: &Ledger, + anchor_ms: i64, + network_difficulty: u128, +) -> Result { + set_ledger_clock(&ledger.pool, anchor_ms).await?; + let snapshot = tokio::time::timeout(SNAPSHOT_CEILING, ledger.snapshot(network_difficulty)) + .await + .map_err(|_| { + anyhow!( + "{scenario}: Ledger::snapshot did not return within {SNAPSHOT_CEILING:?}; \ + the paged read is not making progress" + ) + })??; + ensure!( + snapshot.anchor_ms == anchor_ms, + "{scenario}: expected the pinned anchor {anchor_ms}, got {}", + snapshot.anchor_ms + ); + Ok(snapshot) +} + +/// Runs the read, runs the oracle, and asserts they agree on the anchor, the +/// cutoff and the window. Prints the row counts the acceptance run reports. +async fn check_scenario( + scenario: &str, + ledger: &Ledger, + anchor_ms: i64, + network_difficulty: u128, +) -> Result { + let snapshot = pinned_snapshot(scenario, ledger, anchor_ms, network_difficulty).await?; + let weight = network_difficulty + .checked_mul(8) + .context("window weight overflow")?; + let expected = oracle_window(&ledger.pool, &snapshot, weight).await?; + let cutoff = oracle_cutoff(&ledger.pool).await?; + ensure!( + snapshot.share_seq == u64::try_from(cutoff)?, + "{scenario}: cutoff disagrees: Ledger::snapshot={} oracle={cutoff}", + snapshot.share_seq + ); + assert_window_matches(scenario, &snapshot.shares, &expected.window)?; + println!( + "scenario {scenario}: rows={} eligible={} window={} anchor_ms={anchor_ms} \ + cutoff={cutoff} weight={weight}", + total_rows(&ledger.pool).await?, + expected.eligible, + snapshot.shares.len() + ); + Ok(snapshot) +} + +fn credited_sequences(snapshot: &Snapshot) -> Result> { + snapshot + .shares + .iter() + .map(|share| Ok(i64::try_from(share.share_seq)?)) + .collect() +} + +// --------------------------------------------------------------------------- +// (a) page boundaries +// --------------------------------------------------------------------------- + +/// Every fixture row carries difficulty 8 and the read's weight is +/// `network_difficulty * 8`, so the window is exactly `network_difficulty` +/// rows long. That puts the cut wherever a sub-case asks for it: one row +/// before the read's 4096-row page, exactly on it, one row after it, deep +/// enough to need a third page, and past the end of history. +#[tokio::test] +async fn window_matches_the_oracle_across_page_boundaries() -> Result<()> { + with_schema(|ledger| async move { + let anchor_ms = derived_anchor(&ledger.pool, 0).await?; + // 12000 rows one millisecond apart, the newest a minute before the + // anchor. + insert_run(&ledger.pool, 12_000, anchor_ms * 1_000 - 60_000_000).await?; + + for (label, network_difficulty, expected_len) in [ + ("page-boundary-minus-one", 4_095_u128, 4_095_usize), + ("page-boundary-exact", 4_096, 4_096), + ("page-boundary-plus-one", 4_097, 4_097), + ("three-pages", 9_000, 9_000), + ("weight-exceeds-history", 20_000, 12_000), + ] { + let snapshot = check_scenario( + &format!("a:{label}"), + &ledger, + anchor_ms, + network_difficulty, + ) + .await?; + ensure!( + snapshot.shares.len() == expected_len, + "a:{label}: expected a {expected_len}-row window, got {}", + snapshot.shares.len() + ); + } + Ok(()) + }) + .await +} + +// --------------------------------------------------------------------------- +// (b) the crossing row +// --------------------------------------------------------------------------- + +/// The row whose difficulty overshoots the remaining weight is still credited; +/// the next older row is not. +#[tokio::test] +async fn the_row_that_crosses_the_weight_boundary_is_credited() -> Result<()> { + with_schema(|ledger| async move { + let anchor_ms = derived_anchor(&ledger.pool, 0).await?; + let base_us = anchor_ms * 1_000 - 60_000_000; + // Newest first the difficulties are 3, 7, 5, 1, 1. A weight of 8 + // leaves 5 after the newest row, and the difficulty-7 row + // overshoots that. + let mut seqs = Vec::new(); + for (index, difficulty) in [1_u128, 1, 5, 7, 3].into_iter().enumerate() { + let index = u64::try_from(index)?; + let spec = ShareSpec::new(index + 1, base_us + i64::try_from(index)? * 1_000) + .difficulty(difficulty); + seqs.push(insert_share(&ledger.pool, &spec).await?); + } + + let snapshot = check_scenario("b:crossing-row", &ledger, anchor_ms, 1).await?; + let credited = credited_sequences(&snapshot)?; + ensure!( + credited == vec![seqs[3], seqs[4]], + "b:crossing-row: expected the difficulty-7 crossing row and the newest row, got \ + {credited:?}" + ); + Ok(()) + }) + .await +} + +/// A difficulty at the top of the `u128` range survives the `numeric` round +/// trip and closes the window on the row that carries it. +#[tokio::test] +async fn a_u128_maximum_difficulty_is_decoded_and_closes_the_window() -> Result<()> { + with_schema(|ledger| async move { + let anchor_ms = derived_anchor(&ledger.pool, 0).await?; + let base_us = anchor_ms * 1_000 - 60_000_000; + insert_share(&ledger.pool, &ShareSpec::new(1, base_us)).await?; + let crossing = insert_share( + &ledger.pool, + &ShareSpec::new(2, base_us + 1_000).difficulty(u128::MAX), + ) + .await?; + let mut newest = ShareSpec::new(3, base_us + 2_000); + newest.network_difficulty = u128::MAX; + let newest = insert_share(&ledger.pool, &newest).await?; + + let snapshot = + check_scenario("b:u128-maximum-difficulty", &ledger, anchor_ms, 1_000).await?; + let credited = credited_sequences(&snapshot)?; + ensure!( + credited == vec![crossing, newest], + "b:u128-maximum-difficulty: expected the saturating row and the newest row, got \ + {credited:?}" + ); + Ok(()) + }) + .await +} + +// --------------------------------------------------------------------------- +// (c) and (e) the anchor barrier +// --------------------------------------------------------------------------- + +/// Microsecond offsets, relative to the anchor, at which a boundary scenario +/// places rows. `0` means exactly on the anchor. +const BOUNDARY_OFFSETS_US: &[i64] = &[-1_000, -400, 0, 400, 1_000]; + +/// Asserts the anchor's millisecond survives the read's own +/// `to_timestamp(double precision/1000)` conversion unchanged. +/// +/// This is a precondition on the fixture, not part of the oracle. The oracle +/// compares stored timestamps against an exact integer-millisecond barrier +/// while the read compares them against the converted one. Where the +/// conversion is exact those are the same instant and every row kind agrees. +/// Where it is not, the two barriers sit a microsecond apart, and a row placed +/// on the anchor falls on one side or the other purely according to how it was +/// written: a fixture storing the exact millisecond and a share written through +/// `Ledger::append` land on opposite sides. Past roughly 2^43 epoch- +/// milliseconds -- the year 2248 -- a `double precision` no longer resolves a +/// microsecond, and that is where the inexact anchors live. Measured on +/// PostgreSQL 16 in that region, every millisecond ending in `999` converts one +/// microsecond high, every one ending in `001` converts one microsecond low, +/// and `000` is exact; at 2^42 and at every realistic epoch millisecond all +/// three suffixes are exact. These scenarios therefore stay below 2^43, and +/// this guard stops the fixture silently picking an anchor that does not. +/// Millisecond behaviour past 2^43 is asserted separately, in +/// `writer_rows_follow_integer_milliseconds_past_2_43`, in the integer +/// milliseconds `AcceptedShare` actually carries. +async fn ensure_anchor_converts_exactly( + pool: &PgPool, + scenario: &str, + anchor_ms: i64, +) -> Result<()> { + ensure!( + converts_exactly(pool, anchor_ms).await?, + "{scenario}: anchor {anchor_ms} does not survive the read's double-precision \ + conversion, so the exact-time oracle and the read would disagree on any row placed \ + exactly on it; pick an anchor below 2^43 epoch-milliseconds" + ); + Ok(()) +} + +/// Whether `ms` is representable exactly by the writer's and the read's shared +/// `to_timestamp(double precision/1000)` conversion. +/// +/// Classification only: it decides which assertion a millisecond is held to, +/// never what the payout window should contain. +async fn converts_exactly(pool: &PgPool, ms: i64) -> Result { + compare_conversion(pool, ms, "=").await +} + +/// Whether the conversion of `ms` lands *below* the exact millisecond. +/// +/// This is the direction that matters for the read-back. `share_from_row` +/// floors, so a conversion one microsecond high still floors to the same +/// millisecond and round-trips cleanly; only a conversion below the +/// millisecond loses one. +async fn converts_below(pool: &PgPool, ms: i64) -> Result { + compare_conversion(pool, ms, "<").await +} + +async fn compare_conversion(pool: &PgPool, ms: i64, operator: &str) -> Result { + let sql = format!( + "SELECT to_timestamp($1::double precision/1000) {operator} {}", + timestamp_from_millis("$1") + ); + Ok(sqlx::query_scalar(&sql).bind(ms).fetch_one(pool).await?) +} + +/// Builds a fixture that brackets `anchor_ms` on both timestamp predicates and +/// checks the window against the oracle. +/// +/// Each offset appears twice: once moving `accepted_at` while `job_issued_at` +/// stays far in the past, and once moving `job_issued_at` while `accepted_at` +/// stays far in the past. The second family deliberately issues a job after +/// the share was accepted, which no real writer does, because that is the only +/// way to isolate the `job_issued_at` predicate from the `accepted_at` one. +/// +/// On top of those, one pair goes in through the public append path, so the +/// barrier is asserted against a share the writer could really have produced +/// and not only against fixtures. +/// +/// The weight far exceeds the fixture's total difficulty, so the anchor +/// barrier -- not the weight cut -- decides membership. +async fn check_anchor_boundary( + scenario: &str, + raw_url: &str, + anchor: Anchor, + offsets: &[i64], +) -> Result<()> { + in_fresh_schema(raw_url, |ledger| async move { + let anchor_ms = anchor.resolve(&ledger.pool).await?; + ensure_anchor_converts_exactly(&ledger.pool, scenario, anchor_ms).await?; + let anchor_us = anchor_ms + .checked_mul(1_000) + .context("anchor microseconds overflow")?; + let far_past_us = anchor_us - 3_600_000_000; + let mut index = 0_u64; + for offset in offsets { + index += 1; + insert_share( + &ledger.pool, + &ShareSpec::new(index, anchor_us + offset).issued_at_us(far_past_us), + ) + .await?; + index += 1; + insert_share( + &ledger.pool, + &ShareSpec::new(index, far_past_us).issued_at_us(anchor_us + offset), + ) + .await?; + } + let (writer_on_anchor, writer_after_anchor) = + append_anchor_pair(&ledger, anchor_ms, WRITER_INDEX_BASE).await?; + + let snapshot = check_scenario(scenario, &ledger, anchor_ms, 1_000_000).await?; + // Two rows per non-positive offset, one from each predicate family, + // plus the appended share accepted on the anchor. + let expected = offsets.iter().filter(|offset| **offset <= 0).count() * 2 + 1; + ensure!( + snapshot.shares.len() == expected, + "{scenario}: expected {expected} rows at or before the anchor, got {}", + snapshot.shares.len() + ); + let credited = credited_sequences(&snapshot)?; + ensure!( + credited.contains(&i64::try_from(writer_on_anchor)?), + "{scenario}: writer-on-anchor share {writer_on_anchor}, accepted by the public append \ + path at exactly the anchor, was not credited; window {credited:?}" + ); + ensure!( + !credited.contains(&i64::try_from(writer_after_anchor)?), + "{scenario}: writer-after-anchor share {writer_after_anchor}, accepted one \ + millisecond past the anchor, was credited; window {credited:?}" + ); + Ok(()) + }) + .await +} + +/// Where the appended share indices start, clear of the direct-insert rows. +const WRITER_INDEX_BASE: u64 = 1_000; + +/// Rows sitting exactly on the anchor are credited on either timestamp; rows a +/// millisecond past it are not. +#[tokio::test] +async fn rows_on_the_anchor_are_credited_and_rows_after_it_are_not() -> Result<()> { + let Some(url) = gate::database_url(gate::site!())? else { + return Ok(()); + }; + check_anchor_boundary( + "c:anchor-barrier", + &url, + Anchor::Derived(0), + BOUNDARY_OFFSETS_US, + ) + .await +} + +/// The same barrier at millisecond values that stress the read's +/// floating-point conversion: anchors ending in `999`, `000` and `001`, at an +/// ordinary magnitude and near 2^42 milliseconds (the year 2109). +/// +/// The large magnitude is fixed because that is the whole point of the case, +/// and it sits centuries ahead of any wall clock this suite will meet. The +/// ordinary magnitude is derived, so it can never become a dated fuse. Every +/// anchor here converts exactly, which `ensure_anchor_converts_exactly` +/// enforces, so all three row kinds -- direct inserts on the anchor and 400 +/// microseconds either side of it, and the appended pair -- must agree with the +/// oracle. Past 2^43 the conversion stops being exact, and that region is +/// covered by `writer_rows_follow_integer_milliseconds_past_2_43` instead. +#[tokio::test] +async fn the_anchor_barrier_holds_at_large_millisecond_values() -> Result<()> { + let Some(url) = gate::database_url(gate::site!())? else { + return Ok(()); + }; + for (label, anchor) in [ + ("ordinary-999", Anchor::Derived(999)), + ("ordinary-000", Anchor::Derived(1_000)), + ("ordinary-001", Anchor::Derived(1_001)), + ("pow42-999", Anchor::Fixed(4_398_046_510_999)), + ("pow42-000", Anchor::Fixed(4_398_046_511_000)), + ("pow42-001", Anchor::Fixed(4_398_046_511_001)), + ] { + check_anchor_boundary(&format!("e:{label}"), &url, anchor, BOUNDARY_OFFSETS_US).await?; + } + Ok(()) +} + +// --------------------------------------------------------------------------- +// (g) writer-only milliseconds past 2^43 +// --------------------------------------------------------------------------- + +/// Past roughly 2^43 epoch-milliseconds the read's +/// `to_timestamp(double precision/1000)` no longer resolves a microsecond, so +/// a stored timestamp can sit a microsecond either side of its millisecond and +/// the exact-time oracle stops being the right yardstick. The contract +/// `AcceptedShare` carries is integer milliseconds, so that is what this test +/// asserts, and it does so without the oracle and without a single direct +/// insert: every row goes in through `Ledger::append`, exactly as the +/// production writer writes it. +/// +/// A row is credited if and only if the `accepted_at_ms` that `append` +/// returned is at or before the anchor. That holds at all three anchors: the +/// barrier and the stored value are converted the same way, so they shift +/// together and the credited set stays right. +/// +/// The read-back of that millisecond does not hold. `share_from_row` converts +/// `accepted_at` with chrono's `timestamp_millis()`, which floors, so a value +/// stored a microsecond low reads back a whole millisecond early, and the +/// millisecond the audit bundle carries is off by one. This is a real defect, +/// reported separately; it is confined to dates past about the year 2248 and +/// is unreachable at a realistic epoch millisecond. +/// +/// The defect follows the individual millisecond, not the anchor's suffix: the +/// rows a millisecond either side of an anchor have their own suffixes and +/// their own conversions. Measured on PostgreSQL 16 across the 2000001 +/// consecutive milliseconds -- about 33 minutes -- centred on 2^43, 480000 of +/// them (24%) read back one millisecond early and the rest are exact; across a +/// span of the same length at 2^42, none are. Of the nine milliseconds this +/// test writes, 8796093022998 and 8796093023001 read back early and the other +/// seven are exact. +/// +/// Only a conversion that lands *below* its millisecond loses one: flooring a +/// value a microsecond high returns the same millisecond, so those round-trip +/// cleanly. Membership is therefore asserted exactly, at every anchor, while +/// the read-back is *bounded* rather than skipped or pinned: a millisecond +/// whose conversion is at or above it must round-trip unchanged, and one whose +/// conversion is below it may lose the millisecond to the floor but no more +/// than one, and may never read back late. +/// +/// Bounding it rather than requiring the drift keeps the defect visible +/// without freezing it into the suite. The printed `read_back_early` list +/// names the milliseconds that actually drifted, so a silent change of +/// behaviour is still obvious; and the day `share_from_row` rounds instead of +/// flooring, those milliseconds land on a drift of zero, the list comes back +/// empty, and this test keeps passing rather than failing the fix. +/// +/// The whole file tolerates that fix, not just this test. The one other place +/// the flooring convention is visible is the sub-millisecond fixture rows in +/// scenarios c and e, and `compare_millisecond_field` accepts either +/// convention for exactly those rows while holding every whole-millisecond row +/// -- which is every row a writer can produce -- to an exact read-back. +#[tokio::test] +async fn writer_rows_follow_integer_milliseconds_past_2_43() -> Result<()> { + let Some(url) = gate::database_url(gate::site!())? else { + return Ok(()); + }; + for (label, anchor_ms) in [ + ("pow43-999", 8_796_093_022_999_i64), + ("pow43-000", 8_796_093_023_000), + ("pow43-001", 8_796_093_023_001), + ] { + let scenario = format!("g:{label}"); + in_fresh_schema(&url, |ledger| async move { + // One share a millisecond before the anchor, one on it, one after. + let mut appended = Vec::new(); + for (position, offset) in [("before", -1_i64), ("on", 0), ("after", 1)] { + let accepted_ms = anchor_ms + .checked_add(offset) + .context("writer anchor offset overflow")?; + set_ledger_clock(&ledger.pool, accepted_ms).await?; + let index = WRITER_INDEX_BASE + u64::try_from(appended.len())?; + let share = ledger + .append(writer_share(index, accepted_ms), None) + .await + .with_context(|| format!("{scenario}: appending the {position} share"))? + .share; + ensure!( + share.accepted_at_ms == accepted_ms && share.job_issued_at_ms == accepted_ms, + "{scenario}: expected the {position} share at {accepted_ms}, got \ + accepted_at_ms={} job_issued_at_ms={}", + share.accepted_at_ms, + share.job_issued_at_ms + ); + appended.push((position, share)); + } + + let snapshot = pinned_snapshot(&scenario, &ledger, anchor_ms, 1_000_000).await?; + let mut read_back_early = Vec::new(); + for (position, share) in &appended { + // The contract: credited exactly when the accepted millisecond is + // at or before the anchor millisecond. + let expected = share.accepted_at_ms <= anchor_ms; + let credited = snapshot + .shares + .iter() + .find(|candidate| candidate.share_id == share.share_id); + ensure!( + credited.is_some() == expected, + "{scenario}: the {position} share, accepted at {} against anchor {anchor_ms}, \ + should {} been credited", + share.accepted_at_ms, + if expected { "have" } else { "not have" } + ); + let Some(credited) = credited else { + continue; + }; + let drift = credited.accepted_at_ms - share.accepted_at_ms; + ensure!( + credited.job_issued_at_ms - share.job_issued_at_ms == drift, + "{scenario}: the {position} share's two timestamps drifted apart on read-back: \ + appended accepted_at_ms={} job_issued_at_ms={}, read back accepted_at_ms={} \ + job_issued_at_ms={}", + share.accepted_at_ms, + share.job_issued_at_ms, + credited.accepted_at_ms, + credited.job_issued_at_ms + ); + if converts_below(&ledger.pool, share.accepted_at_ms).await? { + // A bound, not a pin: the read-back may lose the millisecond + // to the floor, but never more than one and never late. A fix + // that rounds instead of flooring lands on `drift == 0` and + // keeps this passing. + ensure!( + drift == -1 || drift == 0, + "{scenario}: millisecond {} converts below its own millisecond, so the \ + {position} share had to read back either unchanged or exactly one \ + millisecond early; it read back {} ({drift} ms)", + share.accepted_at_ms, + credited.accepted_at_ms + ); + if drift == -1 { + read_back_early.push(share.accepted_at_ms); + } + } else { + ensure!( + drift == 0, + "{scenario}: millisecond {} converts at or above its own millisecond, so \ + the {position} share had to round-trip unchanged, but it read back {} \ + ({drift} ms)", + share.accepted_at_ms, + credited.accepted_at_ms + ); + } + } + println!( + "scenario {scenario}: rows={} window={} anchor_ms={anchor_ms} cutoff={} \ + read_back_early={read_back_early:?}", + total_rows(&ledger.pool).await?, + snapshot.shares.len(), + snapshot.share_seq + ); + Ok(()) + }) + .await?; + } + Ok(()) +} + +// --------------------------------------------------------------------------- +// (d) rejected rows +// --------------------------------------------------------------------------- + +/// Rejected rows interleave with accepted ones. They stay out of the window, +/// they consume none of the weight despite carrying enormous difficulties, and +/// the newest of them does not move the cutoff. +#[tokio::test] +async fn rejected_rows_are_excluded_and_consume_no_weight() -> Result<()> { + with_schema(|ledger| async move { + let anchor_ms = derived_anchor(&ledger.pool, 0).await?; + let base_us = anchor_ms * 1_000 - 60_000_000; + let huge = 1_000_000_000_000_000_000_000_000_000_000_u128; + let mut accepted_seqs = Vec::new(); + let mut rejected_seqs = Vec::new(); + for index in 0..8_u64 { + let accepted_at_us = base_us + i64::try_from(index)? * 1_000; + let spec = ShareSpec::new(index + 1, accepted_at_us); + if index % 2 == 0 { + accepted_seqs.push(insert_share(&ledger.pool, &spec).await?); + } else { + let spec = spec.difficulty(huge).rejected(); + rejected_seqs.push(insert_share(&ledger.pool, &spec).await?); + } + } + // The highest sequence in the table belongs to a rejected row, so the + // cutoff has to fall back to the newest accepted row. + rejected_seqs.push( + insert_share( + &ledger.pool, + &ShareSpec::new(9, base_us + 9_000) + .difficulty(huge) + .rejected(), + ) + .await?, + ); + + // Weight 24 credits exactly three difficulty-8 accepted rows. Had the + // rejected rows consumed weight, the window would be a single row. + let snapshot = check_scenario("d:rejected-interleaved", &ledger, anchor_ms, 3).await?; + let credited = credited_sequences(&snapshot)?; + for seq in &rejected_seqs { + ensure!( + !credited.contains(seq), + "d:rejected-interleaved: rejected row {seq} appeared in the window" + ); + } + ensure!( + credited.as_slice() == &accepted_seqs[accepted_seqs.len() - 3..], + "d:rejected-interleaved: expected the three newest accepted rows, got {credited:?}" + ); + ensure!( + i64::try_from(snapshot.share_seq)? == *accepted_seqs.last().context("accepted rows")?, + "d:rejected-interleaved: the cutoff followed a rejected row" + ); + Ok(()) + }) + .await +} + +// --------------------------------------------------------------------------- +// (f) empty windows +// --------------------------------------------------------------------------- + +/// An empty window arises three ways: no rows at all, only rejected rows, and +/// accepted rows that all sit after the anchor. +#[tokio::test] +async fn empty_windows_return_no_shares() -> Result<()> { + with_schema(|ledger| async move { + let anchor_ms = derived_anchor(&ledger.pool, 0).await?; + + let snapshot = check_scenario("f:no-rows", &ledger, anchor_ms, 1_000).await?; + ensure!( + snapshot.shares.is_empty() && snapshot.share_seq == 0, + "f:no-rows: expected an empty window and a zero cutoff, got {} shares and cutoff {}", + snapshot.shares.len(), + snapshot.share_seq + ); + + for index in 0..4_u64 { + let accepted_at_us = anchor_ms * 1_000 - 60_000_000 + i64::try_from(index)? * 1_000; + let spec = ShareSpec::new(index + 1, accepted_at_us).rejected(); + insert_share(&ledger.pool, &spec).await?; + } + let snapshot = + check_scenario("f:only-rejected-rows", &ledger, anchor_ms + 1_000, 1_000).await?; + ensure!( + snapshot.shares.is_empty() && snapshot.share_seq == 0, + "f:only-rejected-rows: expected an empty window and a zero cutoff, got {} shares and \ + cutoff {}", + snapshot.shares.len(), + snapshot.share_seq + ); + + let future_anchor_ms = anchor_ms + 100_000; + for index in 0..4_u64 { + let accepted_at_us = future_anchor_ms * 1_000 + 1_000 + i64::try_from(index)? * 1_000; + insert_share(&ledger.pool, &ShareSpec::new(index + 100, accepted_at_us)).await?; + } + let snapshot = check_scenario( + "f:all-rows-after-the-anchor", + &ledger, + future_anchor_ms, + 1_000, + ) + .await?; + ensure!( + snapshot.shares.is_empty() && snapshot.share_seq > 0, + "f:all-rows-after-the-anchor: expected an empty window behind a non-zero cutoff, got {} \ + shares and cutoff {}", + snapshot.shares.len(), + snapshot.share_seq + ); + Ok(()) + }) + .await +} diff --git a/crates/qbit-prism-server/tests/window_reference.rs b/crates/qbit-prism-server/tests/window_reference.rs new file mode 100644 index 000000000..0bd01820f --- /dev/null +++ b/crates/qbit-prism-server/tests/window_reference.rs @@ -0,0 +1,786 @@ +//! Isolated B273 schema/reader tests; these do not enable compact runtime jobs. +//! Run through test/prism-native-tests.sh cargo-args --locked -p qbit-prism-server +//! --test window_reference. Only the explicit scale measurements are ignored. +use anyhow::{ensure, Context, Result}; +use futures_util::future::LocalBoxFuture; +use qbit_pool_builder::ManifestSigningKey; +use qbit_prism::{AcceptedShare, CarryForwardBalance, FoundBlock, PayoutPolicy}; +use qbit_prism_server::ledger::{BalanceSource, Ledger, ShareRange, WindowError, WindowRef}; +use qbit_prism_test_gate as gate; +use serde_json::{json, Value}; +use sha2::{Digest, Sha256}; +use sqlx::{postgres::PgPoolOptions, PgPool}; +use std::{sync::Mutex, time::Duration}; +use tokio::time::{sleep, timeout}; + +#[allow(dead_code)] +#[path = "support/window_fixture.rs"] +mod window_fixture; + +#[path = "window_reference/cancellation.rs"] +mod cancellation; +#[path = "window_reference/payout_state.rs"] +mod payout_state; + +const ANCHOR: i64 = 1_700_000_000_000; +const MIGRATION_008: &str = include_str!("../migrations/008_prepared_window_reference.sql"); +const MIGRATION_009: &str = include_str!("../migrations/009_wrap_safe_sessions.sql"); + +async fn preinstall_migration(db: &Database, version: i32, migration: &str) -> Result<()> { + let mut tx = db.pool.begin().await?; + sqlx::query("SELECT pg_advisory_xact_lock($1)") + .bind(0x505249534d000001i64) + .execute(&mut *tx) + .await?; + sqlx::raw_sql(migration).execute(&mut *tx).await?; + sqlx::query("INSERT INTO qbit_prism_schema_migrations(version) VALUES($1)") + .bind(version) + .execute(&mut *tx) + .await?; + tx.commit().await?; + Ok(()) +} + +#[path = "support/ledger_database.rs"] +#[allow(dead_code)] +mod ledger_database; +use ledger_database::FixtureDatabase; + +struct Database { + fixture: FixtureDatabase, + /// A default-size pool on the fixture database, separate from `pool`: + /// tests hold locks and advisory gates on its connections and poll + /// `pg_stat_activity` and `pg_locks` through it while `pool` is busy. + admin: PgPool, + pool: PgPool, + url: String, + schema: String, + ledgers: Mutex>, +} + +impl Database { + /// A pre-008 schema in its own database. + async fn open() -> Result> { + let Some(raw) = gate::database_url(gate::site!())? else { + return Ok(None); + }; + let fixture = FixtureDatabase::open(&raw, "prism_window_ref_").await?; + let admin = match PgPool::connect(&fixture.url).await { + Ok(admin) => admin, + Err(error) => return Err(fixture.abandon(error.into()).await), + }; + let pool = match PgPoolOptions::new() + .max_connections(1) + .connect(&fixture.url) + .await + { + Ok(pool) => pool, + Err(error) => { + admin.close().await; + return Err(fixture.abandon(error.into()).await); + } + }; + let installed = async { + // Build the actual pre-008 schema, without temporarily installing 008 + // and then trying to reverse-engineer its predecessor by dropping fields. + sqlx::raw_sql(include_str!("../../qbit-prism/sql/001_share_ledger.sql")) + .execute(&pool) + .await?; + let mut tx = pool.begin().await?; + for migration in [ + include_str!("../migrations/002_multi_instance.sql"), + include_str!("../migrations/003_2x_compatibility.sql"), + include_str!("../migrations/004_cpfp_retired_funding.sql"), + include_str!("../migrations/005_candidate_dispatch.sql"), + ] { + sqlx::raw_sql(migration).execute(&mut *tx).await?; + } + sqlx::raw_sql("CREATE TABLE qbit_prism_schema_migrations(version integer PRIMARY KEY, applied_at timestamptz NOT NULL DEFAULT clock_timestamp()); INSERT INTO qbit_prism_schema_migrations(version) VALUES(2),(3),(4),(5)") + .execute(&mut *tx).await?; + tx.commit().await?; + anyhow::Ok(()) + } + .await; + if let Err(error) = installed { + pool.close().await; + admin.close().await; + return Err(fixture.abandon(error).await); + } + Ok(Some(Self { + admin, + pool, + url: fixture.url.clone(), + schema: fixture.schema.clone(), + fixture, + ledgers: Mutex::new(Vec::new()), + })) + } + + async fn ledger(&self) -> Result { + let ledger = Ledger::connect(&self.url, "window-reference".into(), 4, true).await?; + self.ledgers.lock().unwrap().push(ledger.pool.clone()); + Ok(ledger) + } + + /// Closes every pool this fixture owns, then drops its database. A test + /// error wins; a cleanup error is attached to it as context. + async fn close(self, result: Result<()>) -> Result<()> { + let ledgers = self.ledgers.into_inner().unwrap(); + for pool in ledgers { + pool.close().await; + } + self.pool.close().await; + self.admin.close().await; + self.fixture.close(result).await + } +} + +async fn run( + body: impl for<'a> FnOnce(&'a Database) -> LocalBoxFuture<'a, Result<()>>, +) -> Result<()> { + let Some(db) = Database::open().await? else { + return Ok(()); + }; + let result = body(&db).await; + db.close(result).await +} + +fn share(seq: u64, filtered: bool) -> AcceptedShare { + AcceptedShare { + share_seq: seq, + share_id: format!("share-{seq}"), + miner_id: "miner".into(), + order_key: "miner".into(), + p2mr_program_hex: "11".repeat(32), + share_difficulty: 1, + network_difficulty: 1000, + template_height: 100, + job_id: "job".into(), + job_issued_at_ms: ANCHOR + + if filtered && seq.is_multiple_of(13) { + 1 + } else { + -1 + }, + accepted_at_ms: ANCHOR + i64::from(filtered && seq.is_multiple_of(11)), + ntime: 100, + credit_policy: (seq.is_multiple_of(2)).then(|| "stale-grace".into()), + } +} + +async fn seed_shares(pool: &PgPool, first: i64, last: i64, filtered: bool) -> Result<()> { + sqlx::query("INSERT INTO qbit_share_ledger(share_seq,share_id,miner_id,payout_order_key,p2mr_program,share_difficulty,network_difficulty,template_height,job_id,job_issued_at,ntime,accepted_at,credit_policy,accepted,reject_reason,writer_id,writer_epoch) + SELECT i,'share-'||i,'miner','miner',decode(repeat('11',32),'hex'),1,1000,100,'job', + to_timestamp(($4+CASE WHEN $3 AND i%13=0 THEN 1 ELSE -1 END)::double precision/1000),100, + to_timestamp(($4+CASE WHEN $3 AND i%11=0 THEN 1 ELSE 0 END)::double precision/1000), + CASE WHEN i%2=0 THEN 'stale-grace' END,NOT $3 OR i%7<>0, + CASE WHEN $3 AND i%7=0 THEN 'stale-job' END,'window-ref-test',0 + FROM generate_series($1::bigint,$2::bigint) AS g(i)") + .bind(first).bind(last).bind(filtered).bind(ANCHOR).execute(pool).await?; + Ok(()) +} + +struct HashWriter(Sha256); +impl std::io::Write for HashWriter { + fn write(&mut self, bytes: &[u8]) -> std::io::Result { + self.0.update(bytes); + Ok(bytes.len()) + } + fn flush(&mut self) -> std::io::Result<()> { + Ok(()) + } +} +fn reference(shares: &[AcceptedShare], balances: &[CarryForwardBalance]) -> WindowRef { + let mut hash = HashWriter(Sha256::new()); + serde_json::to_writer(&mut hash, shares).unwrap(); + WindowRef { + anchor_ms: ANCHOR, + prior_balances_digest: qbit_prism::prior_balances_digest(balances), + shares: shares.first().map(|first| ShareRange { + first_share_seq: first.share_seq, + last_share_seq: shares.last().unwrap().share_seq, + share_count: shares.len() as u64, + snapshot_sha256: hash.0.finalize().into(), + }), + } +} + +async fn stored_balances(pool: &PgPool, balances: &[CarryForwardBalance]) -> Result<()> { + sqlx::query( + "INSERT INTO qbit_prism_balance_snapshots(prior_balances_digest,balances) VALUES($1,$2)", + ) + .bind(hex::encode(qbit_prism::prior_balances_digest(balances))) + .bind(serde_json::to_vec(balances)?) + .execute(pool) + .await?; + Ok(()) +} + +#[tokio::test] +async fn ascending_pages_match_native_rows_and_filter_gaps() -> Result<()> { + run(|db| { + Box::pin(async move { + let ledger = db.ledger().await?; + seed_shares(&db.pool, 1, 6000, true).await?; + let expected: Vec<_> = (1..=6000) + .filter(|i| i % 7 != 0 && i % 11 != 0 && i % 13 != 0) + .map(|i| share(i, true)) + .collect(); + // Also cover >4096 accepted rows, an exact page boundary, and a singleton. + for shares in [&expected[..], &expected[..4096], &expected[..1]] { + let result = ledger + .read_window(&reference(shares, &[]), BalanceSource::Current) + .await?; + ensure!(result.shares == shares, "native window rows differ"); + } + Ok(()) + }) + }) + .await +} + +#[tokio::test] +async fn as_issued_balances_and_signed_outputs_survive_current_revision_change() -> Result<()> { + run(|db| { + Box::pin(async move { + let ledger = db.ledger().await?; + seed_shares(&db.pool, 1, 3, false).await?; + let shares: Vec<_> = (1..=3).map(|i| share(i, false)).collect(); + let original = vec![CarryForwardBalance { + recipient_id: "prior".into(), + order_key: "prior".into(), + p2mr_program_hex: "22".repeat(32), + balance_sats: 12345, + }]; + stored_balances(&db.pool, &original).await?; + let window = reference(&shares, &original); + sqlx::query("INSERT INTO qbit_payout_carry_forward_current(miner_id,payout_order_key,p2mr_program,balance_sats,active_row_count) VALUES('prior','prior',decode(repeat('22',32),'hex'),12345,1)") + .execute(&db.pool).await?; + ensure!(ledger.read_window(&window, BalanceSource::Current).await?.prior_balances == original, + "original balances did not match the current view before replacement"); + let manifest = ManifestSigningKey::from_seed_hex(&"01".repeat(32))?; + let ledger_key = ManifestSigningKey::from_seed_hex(&"02".repeat(32))?; + let found = FoundBlock { + block_height: 101, + coinbase_value_sats: 5_000_000_000, + network_difficulty: 1000, + anchor_job_issued_at_ms: ANCHOR, + }; + let before = qbit_prism::build_audit_bundle_body( + &shares, + found.clone(), + original.clone(), + PayoutPolicy::day_one_default(), + &manifest, + &ledger_key, + )?; + let mut replacement = db.pool.begin().await?; + sqlx::raw_sql("UPDATE qbit_prism_cluster SET payout_revision=7; UPDATE qbit_payout_carry_forward_current SET balance_sats=67890") + .execute(&mut *replacement).await?; + replacement.commit().await?; + let after = ledger.read_window(&window, BalanceSource::AsIssued).await?; + ensure!( + after.payout_revision == 7 && after.prior_balances == original, + "issued/current fences mixed" + ); + let body = qbit_prism::build_audit_bundle_body( + &after.shares, + found, + after.prior_balances, + PayoutPolicy::day_one_default(), + &manifest, + &ledger_key, + )?; + ensure!( + qbit_prism::canonical_audit_bundle_bytes_from_parts(&before, &shares)? + == qbit_prism::canonical_audit_bundle_bytes_from_parts(&body, &after.shares)?, + "original signed audit bytes changed" + ); + ensure!( + matches!( + ledger.read_window(&window, BalanceSource::Current).await, + Err(WindowError::PriorBalancesChanged { .. }) + ), + "current source hid changed balances" + ); + Ok(()) + }) + }) + .await +} + +#[tokio::test] +async fn as_issued_snapshot_preserves_persisted_balance_order() -> Result<()> { + run(|db| { + Box::pin(async move { + let ledger = db.ledger().await?; + seed_shares(&db.pool, 1, 1, false).await?; + let original = vec![ + CarryForwardBalance { + recipient_id: "a-recipient".into(), + order_key: "z-order".into(), + p2mr_program_hex: "33".repeat(32), + balance_sats: 11, + }, + CarryForwardBalance { + recipient_id: "z-recipient".into(), + order_key: "a-order".into(), + p2mr_program_hex: "22".repeat(32), + balance_sats: 7, + }, + ]; + let window = reference(&[share(1, false)], &original); + stored_balances(&db.pool, &original).await?; + let result = ledger.read_window(&window, BalanceSource::AsIssued).await?; + ensure!( + result.prior_balances == original, + "as-issued reader reordered the persisted legacy bundle" + ); + Ok(()) + }) + }) + .await +} + +#[tokio::test] +async fn empty_window_never_queries_share_history_and_errors_keep_their_types() -> Result<()> { + run(|db| { + Box::pin(async move { + let ledger = db.ledger().await?; + sqlx::query("ALTER TABLE qbit_share_ledger RENAME TO hidden_history") + .execute(&db.pool) + .await?; + let empty = reference(&[], &[]); + ensure!( + ledger + .read_window(&empty, BalanceSource::Current) + .await? + .shares + .is_empty(), + "empty range read shares" + ); + ensure!( + matches!( + ledger.read_window(&empty, BalanceSource::AsIssued).await, + Err(WindowError::BalanceSnapshotMissing { .. }) + ), + "missing balance blob was not an error" + ); + stored_balances(&db.pool, &[]).await?; + ensure!( + ledger + .read_window(&empty, BalanceSource::AsIssued) + .await? + .shares + .is_empty(), + "as-issued empty read" + ); + let mut missing = reference(&[share(1, false)], &[]); + ensure!( + matches!( + ledger.read_window(&missing, BalanceSource::Current).await, + Err(WindowError::Database(_)) + ), + "missing table lost database error" + ); + missing.prior_balances_digest = [1; 32]; + ensure!( + matches!( + ledger.read_window(&missing, BalanceSource::Current).await, + Err(WindowError::PriorBalancesChanged { .. }) + ), + "range queried before balances" + ); + for (key, bytes) in [ + ("01".repeat(32), b"not-json".to_vec()), + ("02".repeat(32), b"[]".to_vec()), + ] { + sqlx::query("INSERT INTO qbit_prism_balance_snapshots VALUES($1,$2)") + .bind(&key) + .bind(bytes) + .execute(&db.pool) + .await?; + let bad = WindowRef { + prior_balances_digest: hex::decode(key)?.try_into().unwrap(), + ..missing + }; + ensure!( + matches!( + ledger.read_window(&bad, BalanceSource::AsIssued).await, + Err(WindowError::Decode(_)) + ), + "corruption did not precede range read" + ); + } + sqlx::query("INSERT INTO qbit_payout_carry_forward_current(miner_id,payout_order_key,p2mr_program,balance_sats,active_row_count) VALUES('overflow','overflow',decode(repeat('22',32),'hex'),power(10::numeric,50),1)") + .execute(&db.pool).await?; + ensure!(matches!(ledger.read_window(&empty, BalanceSource::Current).await, + Err(WindowError::Decode(_))), "unrepresentable balance lost decode error"); + Ok(()) + }) + }) + .await +} + +#[tokio::test] +async fn fatal_cluster_refuses_window_revision_reads() -> Result<()> { + run(|db| { + Box::pin(async move { + let ledger = db.ledger().await?; + sqlx::query("UPDATE qbit_prism_cluster SET fatal_error='halted for review'") + .execute(&db.pool) + .await?; + let result = ledger + .read_window(&reference(&[], &[]), BalanceSource::Current) + .await; + ensure!( + matches!(result, Err(WindowError::Database(sqlx::Error::RowNotFound))), + "fatal cluster must not expose a payout revision: {result:?}" + ); + Ok(()) + }) + }) + .await +} + +#[tokio::test] +async fn missing_range_count_and_digest_fail_without_partial_windows() -> Result<()> { + run(|db| { + Box::pin(async move { + let ledger = db.ledger().await?; + seed_shares(&db.pool, 1, 1, false).await?; + seed_shares(&db.pool, 3, 5, false).await?; + let expected: Vec<_> = (1..=5).map(|i| share(i, false)).collect(); + let window = reference(&expected, &[]); + ensure!( + matches!( + ledger.read_window(&window, BalanceSource::Current).await, + Err(WindowError::Incomplete { + expected: 5, + got: 4 + }) + ), + "interior gap not rejected" + ); + let mut missing = window; + missing.shares.as_mut().unwrap().last_share_seq = 6; + ensure!( + matches!( + ledger.read_window(&missing, BalanceSource::Current).await, + Err(WindowError::Incomplete { got: 0, .. }) + ), + "endpoint probe did not fail first" + ); + let mut too_many = window; + too_many.shares.as_mut().unwrap().share_count = 3; + ensure!( + matches!( + ledger.read_window(&too_many, BalanceSource::Current).await, + Err(WindowError::Incomplete { + expected: 3, + got: 4 + }) + ), + "extra rows not rejected" + ); + let mut mismatch = window; + mismatch.shares.as_mut().unwrap().share_count = 4; + ensure!( + matches!( + ledger.read_window(&mismatch, BalanceSource::Current).await, + Err(WindowError::SnapshotDigestMismatch { .. }) + ), + "wrong native digest not rejected" + ); + sqlx::query("INSERT INTO qbit_share_ledger(share_seq,share_id,miner_id,payout_order_key,p2mr_program,share_difficulty,network_difficulty,template_height,job_id,job_issued_at,ntime,accepted_at,credit_policy,writer_id,writer_epoch) SELECT 6,'overflow',miner_id,payout_order_key,p2mr_program,power(10::numeric,50),network_difficulty,template_height,job_id,job_issued_at,ntime,accepted_at,credit_policy,writer_id,writer_epoch FROM qbit_share_ledger WHERE share_seq=1") + .execute(&db.pool).await?; + ensure!(matches!(ledger.read_window(&reference(&[share(6, false)], &[]), BalanceSource::Current).await, + Err(WindowError::Decode(_))), "unrepresentable share lost decode error"); + Ok(()) + }) + }) + .await +} + +async fn wait_for_endpoint(pool: &PgPool, schema: &str) -> Result<()> { + timeout(Duration::from_secs(5), async { + loop { + let blocked: bool = sqlx::query_scalar("SELECT EXISTS(SELECT 1 FROM pg_stat_activity WHERE wait_event_type='Lock' AND query LIKE 'SELECT EXISTS(SELECT 1 FROM qbit_share_ledger%' AND pid IN (SELECT pid FROM pg_locks WHERE relation=$1::regclass))") + .bind(format!("{schema}.qbit_share_ledger")).fetch_one(pool).await?; + if blocked { return Ok::<_, anyhow::Error>(()); } + sleep(Duration::from_millis(10)).await; + } + }).await.context("window reader never reached endpoint probe")? +} + +#[tokio::test] +async fn revision_balances_and_pages_use_one_repeatable_read_snapshot() -> Result<()> { + run(|db| Box::pin(async move { + let ledger = db.ledger().await?; + seed_shares(&db.pool, 1, 1, false).await?; + seed_shares(&db.pool, 3, 5000, false).await?; + let expected: Vec<_> = (1..=5000).filter(|i| *i != 2).map(|i| share(i, false)).collect(); + let window = reference(&expected, &[]); + let original_revision = ledger.payout_revision().await?; + let mut lock = db.pool.begin().await?; + sqlx::query("LOCK TABLE qbit_share_ledger IN ACCESS EXCLUSIVE MODE").execute(&mut *lock).await?; + let reader = tokio::spawn(async move { ledger.read_window(&window, BalanceSource::Current).await }); + wait_for_endpoint(&db.admin, &db.schema).await?; + // The locking transaction inserts a previously absent interior row and + // changes the fence/economics before letting the paged reader proceed. + let row = share(2, false); + sqlx::query("INSERT INTO qbit_share_ledger(share_seq,share_id,miner_id,payout_order_key,p2mr_program,share_difficulty,network_difficulty,template_height,job_id,job_issued_at,ntime,accepted_at,credit_policy,writer_id,writer_epoch) VALUES(2,'share-2','miner','miner',decode(repeat('11',32),'hex'),1,1000,100,'job',to_timestamp($1::double precision/1000),100,to_timestamp($2::double precision/1000),'stale-grace','window-ref-test',0)") + .bind(row.job_issued_at_ms).bind(row.accepted_at_ms).execute(&mut *lock).await?; + sqlx::raw_sql("UPDATE qbit_prism_cluster SET payout_revision=payout_revision+1; INSERT INTO qbit_payout_carry_forward_current(miner_id,payout_order_key,p2mr_program,balance_sats,active_row_count) VALUES('prior','prior',decode(repeat('22',32),'hex'),12345,1)") + .execute(&mut *lock).await?; + lock.commit().await?; + let result = timeout(Duration::from_secs(5), reader).await???; + ensure!(result.payout_revision == original_revision && result.prior_balances.is_empty() + && result.shares == expected, "reader mixed database snapshots"); + Ok(()) + })).await +} + +#[tokio::test] +async fn database_timeout_and_cancellation_return_the_single_connection() -> Result<()> { + run(|db| { + Box::pin(async move { + let mut ledger = db.ledger().await?; + // Keep normal construction, but constrain the reader to this test's + // single-connection pool to make a leaked transaction observable. + ledger.pool = db.pool.clone(); + seed_shares(&db.pool, 1, 1, false).await?; + let window = reference(&[share(1, false)], &[]); + for cancel in [false, true] { + sqlx::query(if cancel { + "SET statement_timeout='5s'" + } else { + "SET statement_timeout='80ms'" + }) + .execute(&db.pool) + .await?; + let mut lock = db.admin.begin().await?; + sqlx::query(&format!( + "LOCK TABLE {}.qbit_share_ledger IN ACCESS EXCLUSIVE MODE", + db.schema + )) + .execute(&mut *lock) + .await?; + if cancel { + let source = ledger.clone(); + let read = tokio::spawn(async move { + source.read_window(&window, BalanceSource::Current).await + }); + wait_for_endpoint(&db.admin, &db.schema).await?; + read.abort(); + ensure!( + read.await.unwrap_err().is_cancelled(), + "reader was not cancelled" + ); + } else { + let error = ledger + .read_window(&window, BalanceSource::Current) + .await + .unwrap_err(); + ensure!( + matches!(error, WindowError::Database(sqlx::Error::Database(ref error)) + if error.code().as_deref() == Some("57014")), + "statement timeout lost its SQLSTATE: {error}" + ); + } + lock.rollback().await?; + let mut connection = timeout(Duration::from_secs(2), db.pool.acquire()).await??; + let read_only: String = sqlx::query_scalar("SHOW transaction_read_only") + .fetch_one(&mut *connection) + .await?; + ensure!( + read_only == "off", + "cancelled read-only transaction leaked into pool" + ); + drop(connection); + ensure!( + ledger + .read_window(&window, BalanceSource::Current) + .await? + .shares + .len() + == 1, + "reader did not recover" + ); + } + Ok(()) + }) + }) + .await +} + +#[tokio::test] +async fn migration_preserves_legacy_payloads_and_applies_missing_008_below_009() -> Result<()> { + run(|db| Box::pin(async move { + let original = json!({"extranonce1":"ABCDEF01", "snapshot":{"shares":[{"legacy":true}]}}); + sqlx::query("INSERT INTO qbit_prism_jobs(job_id,instance_id,parent_hash,payout_revision,payload,expires_at) VALUES('legacy','test','parent',0,$1,clock_timestamp()+interval '1 hour')") + .bind(&original).execute(&db.pool).await?; + // Model an actual post-009 database before the 008 runner starts. + preinstall_migration(db, 9, MIGRATION_009).await?; + let _ledger = db.ledger().await?; + let row: Value = sqlx::query_scalar("SELECT to_jsonb(j) FROM qbit_prism_jobs j WHERE job_id='legacy'").fetch_one(&db.pool).await?; + ensure!(row["payload"] == original && row["window_anchor_ms"].is_null() + && row["window_first_share_seq"].is_null() && row["template_sha256"].is_null(), "legacy row was rewritten"); + let _restarted = db.ledger().await?; + sqlx::raw_sql(MIGRATION_008).execute(&db.pool).await?; + let versions: Vec = sqlx::query_scalar("SELECT version FROM qbit_prism_schema_migrations ORDER BY version") + .fetch_all(&db.pool).await?; + ensure!(versions == qbit_prism_server::ledger::REQUIRED_SCHEMA_VERSIONS, "migration membership/order wrong: {versions:?}"); + Ok(()) + })).await +} + +#[tokio::test] +async fn window_sql_check_accepts_exactly_the_three_null_states() -> Result<()> { + run(|db| Box::pin(async move { + let _ledger = db.ledger().await?; + // Every partial-null combination matters: PostgreSQL CHECK accepts NULL. + for mask in 0..64 { + let set = |bit| mask & (1 << bit) != 0; + let result = sqlx::query("INSERT INTO qbit_prism_jobs(job_id,instance_id,parent_hash,payout_revision,payload,expires_at,window_anchor_ms,window_prior_balances_sha256,window_first_share_seq,window_last_share_seq,window_share_count,window_snapshot_sha256) VALUES($1,'test','parent',0,'{}',clock_timestamp(),$2,$3,$4,$5,$6,$7)") + .bind(format!("null-state-{mask}")).bind(set(0).then_some(ANCHOR)).bind(set(1).then(|| "00".repeat(32))) + .bind(set(2).then_some(1i64)).bind(set(3).then_some(3i64)).bind(set(4).then_some(2i64)) + .bind(set(5).then(|| "00".repeat(32))).execute(&db.pool).await; + ensure!(result.is_ok() == matches!(mask, 0 | 3 | 63), "incorrect null-state {mask}: {result:?}"); + } + for (column, value) in [("window_first_share_seq","0"), ("window_last_share_seq","0"), + ("window_share_count","0"), ("window_share_count","4"), + ("window_snapshot_sha256","repeat('A',64)"), ("window_prior_balances_sha256","'00'"), + ("template_sha256","repeat('g',64)")] { + ensure!(sqlx::query(&format!("UPDATE qbit_prism_jobs SET {column}={value} WHERE job_id='null-state-63'")) + .execute(&db.pool).await.is_err(), "invalid {column} accepted"); + } + Ok(()) + })).await +} + +#[tokio::test] +async fn external_blobs_are_immutable_and_remain_deletable_for_gc() -> Result<()> { + run(|db| { + Box::pin(async move { + let _ledger = db.ledger().await?; + for (table, key, payload) in [ + ("qbit_prism_templates", "template_sha256", "template_bytes"), + ( + "qbit_prism_balance_snapshots", + "prior_balances_digest", + "balances", + ), + ] { + let insert = format!( + "INSERT INTO {table}({key},{payload}) VALUES($1,$2) ON CONFLICT DO NOTHING" + ); + sqlx::query(&insert) + .bind("ab".repeat(32)) + .bind(b"original".to_vec()) + .execute(&db.pool) + .await?; + let retry = sqlx::query(&insert) + .bind("ab".repeat(32)) + .bind(b"conflict".to_vec()) + .execute(&db.pool) + .await?; + ensure!( + retry.rows_affected() == 0, + "immutable retry overwrote original" + ); + for assignment in [ + format!("{key}=repeat('cd',32)"), + format!("{payload}=decode('00','hex')"), + ] { + let error = sqlx::query(&format!("UPDATE {table} SET {assignment}")) + .execute(&db.pool) + .await + .unwrap_err(); + ensure!( + error.as_database_error().and_then(|e| e.code()).as_deref() + == Some("55000"), + "immutable update accepted" + ); + } + let bytes: Vec = sqlx::query_scalar(&format!("SELECT {payload} FROM {table}")) + .fetch_one(&db.pool) + .await?; + ensure!(bytes == b"original", "blob content changed"); + ensure!( + sqlx::query(&format!("DELETE FROM {table}")) + .execute(&db.pool) + .await? + .rows_affected() + == 1, + "GC cannot remove blobs" + ); + } + Ok(()) + }) + }) + .await +} + +#[tokio::test] +async fn landed_009_coexists_with_008_in_both_orders_under_runner() -> Result<()> { + for nine_first in [true, false] { + run(move |db| Box::pin(async move { + let (version, migration) = if nine_first { (9, MIGRATION_009) } else { (8, MIGRATION_008) }; + preinstall_migration(db, version, migration).await?; + // Each branch has exactly one predecessor before the actual + // runner installs the missing migration and then restarts. + db.ledger().await?; + db.ledger().await?; + let index: String = sqlx::query_scalar("SELECT indexdef FROM pg_indexes WHERE schemaname=current_schema() AND indexname='qbit_prism_jobs_extranonce1_expiry_idx'") + .fetch_one(&db.pool).await?; + ensure!(index.contains("lower(") && index.contains("extranonce1") && index.contains("expires_at"), "009 index changed"); + let versions: Vec = sqlx::query_scalar("SELECT version FROM qbit_prism_schema_migrations ORDER BY version").fetch_all(&db.pool).await?; + ensure!(versions == qbit_prism_server::ledger::REQUIRED_SCHEMA_VERSIONS, "008/009 order failed"); + let cycle: bool = sqlx::query_scalar("SELECT seqcycle FROM pg_sequence WHERE seqrelid='qbit_prism_session_sequence'::regclass") + .fetch_one(&db.pool).await?; + ensure!(cycle, "009 session sequence did not retain CYCLE"); + Ok(()) + })).await?; + } + Ok(()) +} + +fn require_database() -> Result<()> { + gate::required_database_url(gate::site!())?; + Ok(()) +} + +#[tokio::test] +#[ignore = "reader-only 400k/500k PostgreSQL qualification; not a refresh/resume or WAL gate"] +async fn production_shaped_reader_at_400k_and_500k() -> Result<()> { + require_database()?; + for count in [400_000, 500_000] { + run(move |db| { + Box::pin(async move { + let ledger = db.ledger().await?; + let plan = window_fixture::WindowPlan::new(count)?; + plan.load(&db.pool, "window-reference-scale").await?; + let expected: Vec<_> = (1..=count).map(|i| plan.share(i)).collect(); + let mut window = reference(&expected, &[]); + window.anchor_ms = ANCHOR + count as i64 + 1; + let started = std::time::Instant::now(); + let rebuilt = ledger.read_window(&window, BalanceSource::Current).await?; + let elapsed = started.elapsed(); + ensure!( + rebuilt.shares == expected, + "large reader changed native rows" + ); + println!( + "reader_only shares={count} returned_rows={} elapsed_ms={} native_digest={}", + rebuilt.shares.len(), + elapsed.as_millis(), + hex::encode(window.shares.unwrap().snapshot_sha256) + ); + Ok(()) + }) + }) + .await?; + } + Ok(()) +} diff --git a/crates/qbit-prism-server/tests/window_reference/cancellation.rs b/crates/qbit-prism-server/tests/window_reference/cancellation.rs new file mode 100644 index 000000000..964ed6992 --- /dev/null +++ b/crates/qbit-prism-server/tests/window_reference/cancellation.rs @@ -0,0 +1,115 @@ +//! Explicit scale evidence for cancellation of a partially accumulated window. +use super::*; +use std::sync::{ + atomic::{AtomicU64, Ordering}, + Arc, +}; + +#[tokio::test(flavor = "current_thread")] +#[ignore = "400k partial-read cancellation measurement; requires disposable PostgreSQL"] +async fn cancelling_400k_read_after_96_pages_measures_runtime_stall() -> Result<()> { + require_database()?; + run(|db| Box::pin(async move { + const COUNT: u64 = 400_000; + const ACCUMULATED: u64 = 96 * 4096; + let mut ledger = db.ledger().await?; + ledger.pool = db.pool.clone(); // cancellation must return this sole connection + window_fixture::WindowPlan::new(COUNT)?.load(&db.pool, "window-cancel-scale").await?; + let gate_key: i64 = sqlx::query_scalar("SELECT oid::bigint FROM pg_namespace WHERE nspname=$1") + .bind(&db.schema).fetch_one(&db.pool).await?; + sqlx::raw_sql(&format!( + "ALTER TABLE qbit_share_ledger RENAME TO window_cancel_source; + CREATE FUNCTION cancel_after_page(seq bigint, value text) RETURNS text + LANGUAGE plpgsql VOLATILE AS 'BEGIN IF seq > {ACCUMULATED} THEN + PERFORM pg_advisory_xact_lock({gate_key}); END IF; RETURN value; END;'" + )).execute(&db.pool).await?; + // Keep every real column and its type. Only the projected share ID is + // gated, so endpoint probes and the first 96 keyset pages can finish. + let columns: String = sqlx::query_scalar("SELECT string_agg(CASE WHEN attname='share_id' THEN 'cancel_after_page(share_seq,share_id) AS share_id' ELSE quote_ident(attname) END, ',' ORDER BY attnum) FROM pg_attribute WHERE attrelid='window_cancel_source'::regclass AND attnum>0 AND NOT attisdropped") + .fetch_one(&db.pool).await?; + // Keep the view ordered so PostgreSQL cannot evaluate the delayed + // projection for all 400k rows in a sort before the first page's LIMIT. + sqlx::raw_sql(&format!("CREATE VIEW qbit_share_ledger AS SELECT {columns} FROM window_cancel_source ORDER BY share_seq")) + .execute(&db.pool).await?; + // This is a cancellation benchmark, not a query-plan/throughput one. + // The test-only connection must stream to the gate, never evaluate the + // delayed projection while materializing an entire range for a sort. + sqlx::raw_sql("ANALYZE window_cancel_source; SET enable_bitmapscan=off; SET enable_seqscan=off; SET enable_sort=off") + .execute(&db.pool).await?; + let plan: Value = sqlx::query_scalar(&format!( + "EXPLAIN (FORMAT JSON) SELECT * FROM qbit_share_ledger WHERE accepted AND share_seq>0 AND share_seq<={COUNT} AND accepted_at<=to_timestamp({}::double precision/1000) AND job_issued_at<=to_timestamp({}::double precision/1000) ORDER BY share_seq LIMIT 4096", + ANCHOR + COUNT as i64 + 1, ANCHOR + COUNT as i64 + 1, + )).fetch_one(&db.pool).await?; + require_streaming_plan(&plan[0]["Plan"])?; + let mut gate = db.admin.begin().await?; + sqlx::query("SELECT pg_advisory_xact_lock($1)").bind(gate_key).execute(&mut *gate).await?; + let window = WindowRef { + anchor_ms: ANCHOR + COUNT as i64 + 1, + prior_balances_digest: qbit_prism::prior_balances_digest(&[]), + shares: Some(ShareRange { + first_share_seq: 1, last_share_seq: COUNT, + share_count: COUNT, + // Completion is impossible while the final page is gated. + snapshot_sha256: [0; 32], + }), + }; + let source = ledger.clone(); + let reader = tokio::spawn(async move { source.read_window(&window, BalanceSource::Current).await }); + timeout(Duration::from_secs(120), async { + loop { + let blocked: bool = sqlx::query_scalar("SELECT EXISTS(SELECT 1 FROM pg_locks WHERE locktype='advisory' AND NOT granted AND classid=0 AND objid::bigint=$1)") + .bind(gate_key).fetch_one(&db.admin).await?; + if blocked { return Ok::<_, anyhow::Error>(()); } + ensure!(!reader.is_finished(), "reader finished before the final-page gate"); + sleep(Duration::from_millis(10)).await; + } + }).await.context("400k reader never reached its final page")??; + + let max_tick_ns = Arc::new(AtomicU64::new(0)); + let ticks = max_tick_ns.clone(); + let (ready, started) = tokio::sync::oneshot::channel(); + let ticker = tokio::spawn(async move { + let mut previous = std::time::Instant::now(); + ready.send(()).unwrap(); + loop { + sleep(Duration::from_millis(1)).await; + let now = std::time::Instant::now(); + ticks.fetch_max(now.duration_since(previous).as_nanos() as u64, Ordering::Relaxed); + previous = now; + } + }); + started.await?; + let cancel_at = std::time::Instant::now(); + reader.abort(); + ensure!(reader.await.unwrap_err().is_cancelled(), "400k reader was not cancelled"); + let abort_join = cancel_at.elapsed(); + gate.rollback().await?; + let recovered: i32 = timeout(Duration::from_secs(5), sqlx::query_scalar("SELECT 1").fetch_one(&ledger.pool)).await??; + ensure!(recovered == 1, "cancelled reader retained the sole connection"); + sleep(Duration::from_millis(250)).await; + ticker.abort(); + ensure!(ticker.await.unwrap_err().is_cancelled(), "ticker did not stop"); + println!("window_cancel_only fixture_rows={COUNT} completed_pages=96 accumulated_shares={ACCUMULATED} abort_join_us={} max_1ms_tick_gap_us={} connection_recovered=true", + abort_join.as_micros(), max_tick_ns.load(Ordering::Relaxed) / 1000); + // Report both measurements; no unapproved timing threshold or claim + // about the full refresh/resume deadline follows from this sample. + Ok(()) + })).await +} + +fn require_streaming_plan(plan: &Value) -> Result<()> { + let kind = plan["Node Type"].as_str().context("missing EXPLAIN node")?; + ensure!( + matches!( + kind, + "Limit" | "Subquery Scan" | "Index Scan" | "Index Only Scan" | "Result" + ), + "cancellation gate could run before page accumulation: {plan}" + ); + if let Some(children) = plan["Plans"].as_array() { + for child in children { + require_streaming_plan(child)?; + } + } + Ok(()) +} diff --git a/crates/qbit-prism-server/tests/window_reference/payout_state.rs b/crates/qbit-prism-server/tests/window_reference/payout_state.rs new file mode 100644 index 000000000..b43f45d73 --- /dev/null +++ b/crates/qbit-prism-server/tests/window_reference/payout_state.rs @@ -0,0 +1,227 @@ +//! Real database evidence for the B-owned admission observation, not a runtime +//! lease or A265 candidate test. Existing as-issued reconstruction stays intact. +use super::*; + +fn balance(sats: i128) -> CarryForwardBalance { + CarryForwardBalance { + recipient_id: "prior".into(), + order_key: "prior".into(), + p2mr_program_hex: "22".repeat(32), + balance_sats: sats, + } +} + +async fn seed(pool: &PgPool) -> Result<()> { + sqlx::query("INSERT INTO qbit_payout_carry_forward_current(miner_id,payout_order_key,p2mr_program,balance_sats,active_row_count) VALUES('prior','prior',decode(repeat('22',32),'hex'),12345,1)") + .execute(pool).await?; + Ok(()) +} + +async fn wait_for_balances(db: &Database) -> Result<()> { + timeout(Duration::from_secs(5), async { + loop { + let blocked: bool = sqlx::query_scalar("SELECT EXISTS(SELECT 1 FROM pg_stat_activity WHERE wait_event_type='Lock' AND query LIKE 'SELECT miner_id,payout_order_key,%' AND pid IN (SELECT pid FROM pg_locks WHERE relation=$1::regclass AND NOT granted))") + .bind(format!("{}.qbit_payout_carry_forward_current", db.schema)) + .fetch_one(&db.admin).await?; + if blocked { + return Ok::<_, anyhow::Error>(()); + } + sleep(Duration::from_millis(10)).await; + } + }) + .await + .context("payout reader never reached the balance read")? +} + +#[tokio::test] +async fn revision_only_change_and_balance_change_are_distinct_without_reading_a_window( +) -> Result<()> { + run(|db| Box::pin(async move { + let ledger = db.ledger().await?; + // Admission must not require share history, even for a nonempty job, + // and must not confuse today's digest with the as-issued blob's digest. + sqlx::raw_sql("ALTER TABLE qbit_share_ledger RENAME TO hidden_history; ALTER TABLE qbit_prism_balance_snapshots RENAME TO hidden_snapshots") + .execute(&db.pool).await?; + seed(&db.pool).await?; + let issued = ledger.payout_state().await?; + ensure!(issued.prior_balances_digest == qbit_prism::prior_balances_digest(&[balance(12345)]), + "current balance digest differs from the reference format"); + sqlx::query("UPDATE qbit_prism_cluster SET payout_revision=payout_revision+1") + .execute(&db.pool).await?; + let same_balances = ledger.payout_state().await?; + ensure!(same_balances.payout_revision == issued.payout_revision + 1 + && same_balances.prior_balances_digest == issued.prior_balances_digest, + "a revision change alone invalidated the balance identity"); + let mut change = db.pool.begin().await?; + sqlx::raw_sql("UPDATE qbit_prism_cluster SET payout_revision=payout_revision+1; UPDATE qbit_payout_carry_forward_current SET balance_sats=67890") + .execute(&mut *change).await?; + change.commit().await?; + let changed = ledger.payout_state().await?; + ensure!(changed.payout_revision == same_balances.payout_revision + 1 + && changed.prior_balances_digest == qbit_prism::prior_balances_digest(&[balance(67890)]) + && changed.prior_balances_digest != issued.prior_balances_digest, + "changed balances retained the old eligibility digest"); + Ok(()) + })).await +} + +#[tokio::test] +async fn revision_and_balance_digest_cannot_mix_across_a_blocked_read() -> Result<()> { + run(|db| Box::pin(async move { + let ledger = db.ledger().await?; + seed(&db.pool).await?; + let original = ledger.payout_state().await?; + let gate_key: i64 = sqlx::query_scalar("SELECT oid::bigint FROM pg_namespace WHERE nspname=$1") + .bind(&db.schema).fetch_one(&db.pool).await?; + // Delay the *first* SELECT after it captures the revision. Blocking + // the balance SELECT instead could preserve that statement's snapshot + // under READ COMMITTED too, and would not prove transaction coherence. + sqlx::raw_sql(&format!( + "ALTER TABLE qbit_prism_cluster RENAME TO payout_state_source; + CREATE FUNCTION delayed_payout_revision(revision bigint) RETURNS bigint + LANGUAGE plpgsql VOLATILE AS 'BEGIN PERFORM pg_advisory_xact_lock({gate_key}); RETURN revision; END;'; + CREATE VIEW qbit_prism_cluster AS SELECT singleton,fatal_error, + delayed_payout_revision(payout_revision) AS payout_revision FROM payout_state_source" + )).execute(&db.pool).await?; + let mut change = db.pool.begin().await?; + sqlx::query("SELECT pg_advisory_xact_lock($1)") + .bind(gate_key).execute(&mut *change).await?; + let source = ledger.clone(); + let reader = tokio::spawn(async move { source.payout_state().await }); + timeout(Duration::from_secs(5), async { + loop { + let blocked: bool = sqlx::query_scalar("SELECT EXISTS(SELECT 1 FROM pg_locks WHERE locktype='advisory' AND NOT granted AND classid=0 AND objid::bigint=$1)") + .bind(gate_key).fetch_one(&db.admin).await?; + if blocked { return Ok::<_, anyhow::Error>(()); } + sleep(Duration::from_millis(10)).await; + } + }).await.context("revision query did not reach the advisory gate")??; + // The revision argument has been evaluated from the old row; the + // balance statement has not started. READ COMMITTED would mix them. + sqlx::raw_sql("UPDATE payout_state_source SET payout_revision=payout_revision+1; UPDATE qbit_payout_carry_forward_current SET balance_sats=67890") + .execute(&mut *change).await?; + change.commit().await?; + let during = timeout(Duration::from_secs(5), reader).await???; + ensure!(during == original, "payout eligibility mixed two MVCC snapshots"); + let after = ledger.payout_state().await?; + ensure!(after.payout_revision == original.payout_revision + 1 + && after.prior_balances_digest == qbit_prism::prior_balances_digest(&[balance(67890)]), + "a new read did not see the committed pair"); + Ok(()) + })).await +} + +#[tokio::test] +async fn unavailable_read_only_and_corrupt_balances_remain_errors() -> Result<()> { + run(|db| { + Box::pin(async move { + let mut ledger = db.ledger().await?; + // One connection makes the session setting and its later cleanup exact. + ledger.pool = db.pool.clone(); + let empty = ledger.payout_state().await?; + ensure!( + empty.prior_balances_digest == qbit_prism::prior_balances_digest(&[]), + "valid zero state is not the empty balance digest" + ); + sqlx::query("SET default_transaction_read_only=on") + .execute(&db.pool) + .await?; + let read_only = ledger.payout_state().await; + sqlx::query("SET default_transaction_read_only=off") + .execute(&db.pool) + .await?; + ensure!( + matches!( + read_only, + Err(WindowError::Database(sqlx::Error::RowNotFound)) + ), + "read-only configuration was overridden or treated as a valid state" + ); + sqlx::query("UPDATE qbit_prism_cluster SET fatal_error='test failure'") + .execute(&db.pool) + .await?; + ensure!( + matches!( + ledger.payout_state().await, + Err(WindowError::Database(sqlx::Error::RowNotFound)) + ), + "fatal state was treated as a valid admission observation" + ); + sqlx::query("UPDATE qbit_prism_cluster SET fatal_error=NULL") + .execute(&db.pool) + .await?; + seed(&db.pool).await?; + sqlx::query( + "UPDATE qbit_payout_carry_forward_current SET balance_sats=power(10::numeric,50)", + ) + .execute(&db.pool) + .await?; + ensure!( + matches!(ledger.payout_state().await, Err(WindowError::Decode(_))), + "balance overflow was hidden as ineligible work" + ); + sqlx::query("ALTER TABLE qbit_payout_carry_forward_current RENAME TO hidden_current") + .execute(&db.pool) + .await?; + ensure!( + matches!(ledger.payout_state().await, Err(WindowError::Database(_))), + "balance read failure was hidden as ineligible work" + ); + Ok(()) + }) + }) + .await +} + +#[tokio::test] +async fn timeout_and_cancelled_balance_read_release_the_connection() -> Result<()> { + run(|db| { + Box::pin(async move { + let mut ledger = db.ledger().await?; + ledger.pool = db.pool.clone(); + seed(&db.pool).await?; + for cancel in [false, true] { + sqlx::query(if cancel { + "SET statement_timeout='5s'" + } else { + "SET statement_timeout='80ms'" + }) + .execute(&db.pool) + .await?; + let mut lock = db.admin.begin().await?; + sqlx::query(&format!( + "LOCK TABLE {}.qbit_payout_carry_forward_current IN ACCESS EXCLUSIVE MODE", + db.schema + )) + .execute(&mut *lock) + .await?; + if cancel { + let source = ledger.clone(); + let reader = tokio::spawn(async move { source.payout_state().await }); + wait_for_balances(db).await?; + reader.abort(); + ensure!( + reader.await.unwrap_err().is_cancelled(), + "read was not cancelled" + ); + } else { + let error = ledger.payout_state().await.unwrap_err(); + ensure!( + matches!(error, WindowError::Database(sqlx::Error::Database(ref error)) + if error.code().as_deref() == Some("57014")), + "statement timeout lost its SQLSTATE: {error}" + ); + } + lock.rollback().await?; + let recovered = timeout(Duration::from_secs(2), ledger.payout_state()).await??; + ensure!( + recovered.prior_balances_digest + == qbit_prism::prior_balances_digest(&[balance(12345)]), + "payout reader did not recover its connection" + ); + } + Ok(()) + }) + }) + .await +} diff --git a/crates/qbit-prism-server/tests/work_refresh_metric.rs b/crates/qbit-prism-server/tests/work_refresh_metric.rs new file mode 100644 index 000000000..09cb2761c --- /dev/null +++ b/crates/qbit-prism-server/tests/work_refresh_metric.rs @@ -0,0 +1,327 @@ +//! #525: a template refresh that keeps failing must be visible to the paging +//! rule, not only as a `template refresh deferred` WARN line. +//! +//! The child is the real server binary (`run`) against the in-process fake +//! node (`support/fake_qbitd.rs`) and a PostgreSQL database of its own. One +//! miner holds the published job on tip A. Two stalls follow, each ended by a +//! good template: +//! +//! - on the same parent: the node stays on A and answers a template for +//! another parent, so every refresh fails with the tip unchanged, as a +//! `PayoutExceedsCandidateBalance` rebuild on the same parent does; +//! - on a new tip: the node moves to B and keeps answering a template for A, +//! so every refresh observes B and fails after the observation. +//! +//! Neither stall changes the published generation, so the miner still holds +//! "current" work and semantic coverage reads 1 throughout; +//! `qbit_prism_work_refresh_stalled_seconds` is the signal that grows, and it +//! returns to about zero once a refresh succeeds. +//! +//! ```text +//! PRISM_TEST_DATABASE_URL=postgresql://user@127.0.0.1:5432/postgres \ +//! cargo test --locked -p qbit-prism-server --test work_refresh_metric +//! ``` +use anyhow::{ensure, Context, Result}; +use qbit_pool_builder::ManifestSigningKey; +use qbit_prism_server::ledger::Ledger; +use qbit_prism_test_gate as gate; +use serde_json::json; +use std::{ + net::TcpListener, + process::Stdio, + time::{Duration, Instant}, +}; +use tokio::{ + io::{AsyncBufReadExt, AsyncWriteExt, BufReader}, + net::TcpStream, + process::{Child, Command}, +}; + +#[allow(dead_code)] +#[path = "support/fake_qbitd.rs"] +mod fake_qbitd; +use fake_qbitd::{FakeNode, TEMPLATE_BITS}; + +#[allow(dead_code)] +#[path = "support/ledger_database.rs"] +mod ledger_database; +use ledger_database::FixtureDatabase; +#[path = "support/pool_fee.rs"] +mod pool_fee; + +const STALLED: &str = "qbit_prism_work_refresh_stalled_seconds"; +const COVERAGE: &str = "qbit_prism_stratum_semantic_current_work_ratio"; +const AUTHORIZED: &str = "qbit_prism_authorized_clients"; +const HEALTH: &str = "qbit_prism_health_state"; +/// A healthy frontend refreshes every 0.2 s here and republishes the gauge +/// every second, so it reads well below this. +const HEALTHY: f64 = 3.; +/// The hang guard for one wait; no property is decided by comparing to it. +const DEADLINE: Duration = Duration::from_secs(90); +const POLL: Duration = Duration::from_millis(200); + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn failing_refreshes_grow_the_refresh_stall_gauge_while_coverage_reads_one() -> Result<()> { + let Some(raw) = gate::database_url(gate::site!())? else { + return Ok(()); + }; + let database = FixtureDatabase::open(&raw, "prism_work_refresh_").await?; + // The parent initializes the schema and registers no instance. + let ledger = + match Ledger::connect_tool(&database.url, "work-refresh-fixture".into(), 4, true, None) + .await + { + Ok(ledger) => ledger, + Err(error) => return Err(database.abandon(error).await), + }; + let outcome = stalls(&database.url).await; + ledger.pool.close().await; + database.close(outcome).await +} + +async fn stalls(database_url: &str) -> Result<()> { + let node = FakeNode::open().await?; + let (stratum, stratum_held) = reserve_port()?; + let (api, api_held) = reserve_port()?; + let log = tempfile::NamedTempFile::new()?; + let mut server = spawn( + database_url, + &node.url, + (stratum, api), + [stratum_held, api_held], + &log, + )?; + let result = async { + let client = reqwest::Client::new(); + let metrics = format!("http://127.0.0.1:{api}/metrics"); + let _miner = Miner::connect(stratum, &mut server).await?; + let serving = |body: &str| { + sample(body, STALLED).is_some_and(|age| (0. ..HEALTHY).contains(&age)) + && sample(body, HEALTH) == Some(1.) + && sample(body, AUTHORIZED) == Some(1.) + && sample(body, COVERAGE) == Some(1.) + }; + let (tip_a, tip_b) = ("ab".repeat(32), "bc".repeat(32)); + + wait(&client, &metrics, &mut server, "work on tip A", serving).await?; + // Healthy refreshes keep renewing it. + tokio::time::sleep(Duration::from_secs(3)).await; + ensure!( + serving(&scrape(&client, &metrics).await?), + "healthy frontend drifted" + ); + + // Same parent: a template for another parent fails every refresh + // while the node's tip stays the published one. + node.set_template(Some(template(&"cd".repeat(32), 101))); + stalled(&client, &metrics, &mut server, "the same-parent stall").await?; + node.set_template(None); + wait(&client, &metrics, &mut server, "recovery on tip A", serving).await?; + + // New tip: B, whose every refresh fails after the tip observation. + node.set_template(Some(template(&tip_a, 101))); + node.set_tip(&tip_b, &tip_a, 101, "02"); + stalled(&client, &metrics, &mut server, "the new-tip stall").await?; + node.set_template(None); + wait(&client, &metrics, &mut server, "work on tip B", serving).await?; + Ok(()) + } + .await; + let _ = server.start_kill(); + let _ = server.wait().await; + result.map_err(|error| { + let bytes = std::fs::read(log.path()).unwrap_or_default(); + let tail = String::from_utf8_lossy(&bytes[bytes.len().saturating_sub(4096)..]).into_owned(); + error.context(format!( + "server (Stratum 127.0.0.1:{stratum}, audit 127.0.0.1:{api}) stderr tail:\n{tail}" + )) + }) +} + +/// The gauge grows through a stall while the published generation, so +/// semantic coverage, stays put. `health_state` is not asserted: on the same +/// parent it stays ready until the health timeout. +async fn stalled( + client: &reqwest::Client, + url: &str, + server: &mut Child, + what: &str, +) -> Result<()> { + let first = wait(client, url, server, what, |body| { + sample(body, STALLED).is_some_and(|age| age >= 5.) + }) + .await?; + tokio::time::sleep(Duration::from_secs(3)).await; + let later = scrape(client, url).await?; + let (before, after) = ( + sample(&first, STALLED).unwrap(), + sample(&later, STALLED).context("gauge vanished")?, + ); + ensure!( + after >= before + 2., + "{what}: the gauge must keep growing: {before} then {after}" + ); + for body in [&first, &later] { + ensure!(sample(body, COVERAGE) == Some(1.), "{what}: coverage moved"); + ensure!(sample(body, AUTHORIZED) == Some(1.), "{what}: miner left"); + } + Ok(()) +} + +fn template(parent: &str, height: u64) -> serde_json::Value { + let now = chrono::Utc::now().timestamp(); + json!({ + "height":height,"coinbasevalue":5_000_000_000u64,"previousblockhash":parent, + "version":0x20000000u32,"bits":TEMPLATE_BITS,"curtime":now,"mintime":now-1, + "transactions":[] + }) +} + +/// A port held by its bound listener until the server is about to bind it +/// (#639): a dropped ephemeral bind can be picked again or handed to another +/// socket first. +fn reserve_port() -> Result<(u16, TcpListener)> { + let listener = TcpListener::bind("127.0.0.1:0")?; + Ok((listener.local_addr()?.port(), listener)) +} + +fn spawn( + database_url: &str, + node_url: &str, + (stratum, api): (u16, u16), + held: [TcpListener; 2], + log: &tempfile::NamedTempFile, +) -> Result { + let mut command = Command::new(env!("CARGO_BIN_EXE_qbit-prism-server")); + for (key, _) in + std::env::vars().filter(|(key, _)| key.starts_with("PRISM_") || key.starts_with("QBIT_")) + { + command.env_remove(key); + } + let ledger_seed = "22".repeat(32); + command + .arg("run") + .kill_on_drop(true) + .stdin(Stdio::null()) + .stdout(Stdio::null()) + .stderr(Stdio::from(log.reopen()?)) + .env("RUST_LOG", "warn") + .env("PRISM_DATABASE_URL", database_url) + .env("PRISM_INSTANCE_ID", "work-refresh-frontend") + .env("PRISM_DATABASE_MAX_CONNECTIONS", "8") + .env("PRISM_RUNTIME_WORKERS", "2") + .env("PRISM_JOB_BUILD_EXECUTOR_WORKERS", "2") + .env("QBIT_CHAIN", "testnet") + .env("QBIT_RPC_URL", node_url) + .env("PRISM_ALLOW_TEST_SIGNING_SEEDS", "1") + .envs(pool_fee::ZERO_BPS_POOL_FEE) + .env("PRISM_MANIFEST_SIGNING_SEED_HEX", "11".repeat(32)) + .env("PRISM_LEDGER_ATTESTATION_SIGNING_SEED_HEX", &ledger_seed) + .env( + "PRISM_LEDGER_WRITER_PUBLIC_KEY_HEX", + ManifestSigningKey::from_seed_hex(&ledger_seed)?.public_key_hex(), + ) + .env("PRISM_STRATUM_BIND", "127.0.0.1") + .env("PRISM_STRATUM_PORT", stratum.to_string()) + .env("PRISM_AUDIT_PORT", api.to_string()) + // The node has no long poll; a short poll sees tip B promptly, and a + // one-second health tick republishes the metrics snapshot. + .env("PRISM_BLOCKWAIT_ENABLED", "0") + .env("PRISM_BLOCKPOLL_SECONDS", "0.2") + .env("PRISM_HEALTH_REFRESH_SECONDS", "1") + .env("PRISM_HASHRATE_ROLLUP_ENABLED", "0"); + drop(held); + command.spawn().context("spawning the server binary") +} + +/// One authorized Stratum session that keeps whatever work it is sent. +struct Miner { + _reader: tokio::task::JoinHandle<()>, +} + +impl Miner { + async fn connect(port: u16, server: &mut Child) -> Result { + let started = Instant::now(); + let stream = loop { + match TcpStream::connect(("127.0.0.1", port)).await { + Ok(stream) => break stream, + Err(error) => { + alive(server)?; + ensure!( + started.elapsed() < DEADLINE, + "Stratum never listened: {error}" + ); + tokio::time::sleep(POLL).await; + } + } + }; + let (read, mut write) = stream.into_split(); + for request in [ + json!({"id":1,"method":"mining.subscribe","params":["work-refresh/1"]}), + json!({"id":2,"method":"mining.authorize","params":["work-refresh.rig","x"]}), + ] { + write.write_all(format!("{request}\n").as_bytes()).await?; + } + let reader = tokio::spawn(async move { + let _write = write; + let mut lines = BufReader::new(read).lines(); + while let Ok(Some(_)) = lines.next_line().await {} + }); + Ok(Self { _reader: reader }) + } +} + +fn alive(server: &mut Child) -> Result<()> { + if let Some(status) = server.try_wait()? { + anyhow::bail!("the server exited with {status}"); + } + Ok(()) +} + +async fn scrape(client: &reqwest::Client, url: &str) -> Result { + Ok(client + .get(url) + .timeout(Duration::from_secs(5)) + .send() + .await? + .error_for_status()? + .text() + .await?) +} + +async fn wait( + client: &reqwest::Client, + url: &str, + server: &mut Child, + what: &str, + done: impl Fn(&str) -> bool, +) -> Result { + let started = Instant::now(); + let mut last = String::new(); + loop { + alive(server)?; + if let Ok(body) = scrape(client, url).await { + if done(&body) { + return Ok(body); + } + last = body; + } + ensure!( + started.elapsed() < DEADLINE, + "timed out waiting for {what}; last scrape:\n{}", + last.lines() + .filter(|line| [STALLED, COVERAGE, AUTHORIZED, HEALTH] + .iter() + .any(|name| line.starts_with(name))) + .collect::>() + .join("\n") + ); + tokio::time::sleep(POLL).await; + } +} + +fn sample(body: &str, name: &str) -> Option { + body.lines() + .find_map(|line| line.strip_prefix(name)?.strip_prefix(' ')) + .and_then(|value| value.trim().parse().ok()) +} diff --git a/crates/qbit-prism-test-gate/Cargo.toml b/crates/qbit-prism-test-gate/Cargo.toml new file mode 100644 index 000000000..2add42051 --- /dev/null +++ b/crates/qbit-prism-test-gate/Cargo.toml @@ -0,0 +1,10 @@ +[package] +name = "qbit-prism-test-gate" +version.workspace = true +rust-version.workspace = true +edition = "2021" +license = "MIT" +description = "One integration gate for every environment-gated test in the workspace" +publish = false + +[dependencies] diff --git a/crates/qbit-prism-test-gate/src/lib.rs b/crates/qbit-prism-test-gate/src/lib.rs new file mode 100644 index 000000000..5e178288c --- /dev/null +++ b/crates/qbit-prism-test-gate/src/lib.rs @@ -0,0 +1,722 @@ +//! One integration gate for every test whose execution depends on an +//! environment input (#286). +//! +//! A gated test asks the gate for the inputs it needs at the top of its body, +//! from the thread libtest runs the test on. The gate reads them, applies one +//! decision table, records the decision in the execution manifest, and then +//! hands the values back, fails the test, or prints one skip line and tells +//! the test to return. +//! +//! | input set and non-empty | switch | result | +//! | --- | --- | --- | +//! | yes | -- | run | +//! | no | `PRISM_TEST_REQUIRE_INTEGRATION=1` | fail, naming the missing variable and the test | +//! | no | `GITHUB_JOB=prism-native-postgres` | fail, same | +//! | no | -- | print one skip line with a fixed, greppable prefix, and return | +//! +//! The three variables play different roles. This repository sets the inputs +//! itself, in the `prism-native-postgres` job of `.github/workflows/ci.yml` +//! and in `test/prism-native-tests.sh`. GitHub sets `GITHUB_JOB` to the +//! running job's id, so matching it means a database outage in that job +//! surfaces as a failure instead of a silent pass even if the job's own +//! environment were edited. `PRISM_TEST_REQUIRE_INTEGRATION=1` is the opt-in +//! switch for any run that wants every integration test to be mandatory. +//! +//! Keying on `CI` would be wrong: GitHub sets `CI=true` in every job, +//! including `rust-tests`, which runs the whole workspace with no database. +//! +//! A variable whose value is not valid Unicode is an error, never "absent": +//! treating it as unset would let a deliberately configured run skip a test +//! without a word. An empty or whitespace-only value counts as unset. A +//! non-empty but malformed value is not second-guessed here; it reaches the +//! test, which fails with the connection or spawn error an operator needs. +//! +//! When `PRISM_TEST_GATE_MANIFEST` names a file, every decision appends one +//! line to it, `executed `, `skipped ` or `failed `, where the id +//! is `::::`. Each line is one `O_APPEND` write, +//! so parallel test threads and several test binaries can share the file. +//! `scripts/check_gate_manifest.py` compares it with the checked-in list of +//! gated tests in `test/prism-gated-tests.txt`. +//! +//! `scripts/check_gate_env_reads.py` fails when any Rust file outside this +//! crate names one of the gate variables in a string literal, so a new gated +//! test cannot read them directly and bypass the table. + +use std::env::VarError; +use std::fmt; +use std::fs::OpenOptions; +use std::io::Write; +use std::path::PathBuf; + +/// The opt-in switch: `1` makes every missing input a failure. +pub const SWITCH_VAR: &str = "PRISM_TEST_REQUIRE_INTEGRATION"; +/// Set by GitHub to the id of the running job. +pub const JOB_VAR: &str = "GITHUB_JOB"; +/// The job whose id demands the integration suite. +pub const REQUIRED_JOB: &str = "prism-native-postgres"; +/// Path of the execution manifest; unset means no manifest. +pub const MANIFEST_VAR: &str = "PRISM_TEST_GATE_MANIFEST"; +/// Fixed prefix of every skip line the gate prints. +pub const SKIP_PREFIX: &str = "[prism-test-gate] skipped"; + +/// An environment input a gated test can depend on. +#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)] +pub enum Input { + /// `PRISM_TEST_DATABASE_URL`: a disposable PostgreSQL the test may write to. + DatabaseUrl, + /// `PRISM_TEST_PG_BIN_DIR`: a directory with `initdb`, `pg_ctl` and + /// `pg_basebackup`, for tests that run their own clusters. + PgBinDir, + /// `QBITD_BIN`: a `qbitd` executable for regtest. + QbitdBin, +} + +impl Input { + /// Every input, in the order the manifest and messages list them. + pub const ALL: [Input; 3] = [Input::DatabaseUrl, Input::PgBinDir, Input::QbitdBin]; + + /// The environment variable that carries the input. + pub const fn name(self) -> &'static str { + match self { + Input::DatabaseUrl => "PRISM_TEST_DATABASE_URL", + Input::PgBinDir => "PRISM_TEST_PG_BIN_DIR", + Input::QbitdBin => "QBITD_BIN", + } + } + + fn read(self) -> Raw { + std::env::var(self.name()) + } +} + +impl fmt::Display for Input { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.write_str(self.name()) + } +} + +/// A raw environment read, exactly as `std::env::var` reports it. +pub type Raw = Result; + +/// What made a missing input a failure rather than a skip. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum Demand { + /// `PRISM_TEST_REQUIRE_INTEGRATION=1`. + Switch, + /// `GITHUB_JOB=prism-native-postgres`. + Job, + /// The test was selected explicitly (`#[ignore]`, run with `--ignored`), + /// so a vacuous pass would be exactly what the selection tried to avoid. + Explicit, +} + +impl fmt::Display for Demand { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + Demand::Switch => write!(f, "{SWITCH_VAR}=1"), + Demand::Job => write!(f, "{JOB_VAR}={REQUIRED_JOB}"), + Demand::Explicit => f.write_str("the explicit selection of this test"), + } + } +} + +/// The outcome of the decision table for one test. +#[derive(Clone, Debug, PartialEq, Eq)] +pub enum Decision { + /// Every input is present; the values, trimmed, in the order requested. + Run(Vec), + /// At least one input is missing and something demands the suite. + Fail { missing: Vec, demand: Demand }, + /// At least one input is missing and nothing demands the suite. + Skip { missing: Vec }, +} + +/// Why the gate could not give a test its inputs. +#[derive(Debug)] +pub enum Error { + /// A variable is set to a value that is not valid Unicode. + Unreadable { + variable: &'static str, + detail: String, + }, + /// The switch holds something other than `1`, `0` or nothing. + BadSwitch { value: String }, + /// An input is missing and something demands it. + Required { + test: String, + missing: Vec, + demand: Demand, + }, + /// The gate was called from a thread libtest did not name after a test. + UnnamedThread { thread: String }, + /// The manifest could not be appended to. + Manifest { + path: PathBuf, + source: std::io::Error, + }, +} + +impl fmt::Display for Error { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + Error::Unreadable { variable, detail } => { + write!(f, "{variable} is set but not readable: {detail}") + } + Error::BadSwitch { value } => write!( + f, + "{SWITCH_VAR}={value:?} is not recognised; set it to 1 to require every \ + integration test, or leave it unset" + ), + Error::Required { + test, + missing, + demand, + } => write!( + f, + "{test} requires {}: {} while {demand} demands the integration suite", + join(missing, ", "), + unset(missing), + ), + Error::UnnamedThread { thread } => write!( + f, + "the integration gate must be called from the test's own thread (libtest names \ + it after the test), not from {thread:?}; call it at the top of the test body" + ), + Error::Manifest { path, source } => { + write!( + f, + "cannot append to {MANIFEST_VAR}={}: {source}", + path.display() + ) + } + } + } +} + +impl std::error::Error for Error { + fn source(&self) -> Option<&(dyn std::error::Error + 'static)> { + match self { + Error::Manifest { source, .. } => Some(source), + _ => None, + } + } +} + +fn join(inputs: &[Input], separator: &str) -> String { + inputs + .iter() + .map(|input| input.name()) + .collect::>() + .join(separator) +} + +fn unset(inputs: &[Input]) -> String { + match inputs { + [one] => format!("{one} is unset or empty"), + many => format!("{} are unset or empty", join(many, " and ")), + } +} + +/// Reads one variable, treating a non-Unicode value as an error. +fn readable(variable: &'static str, raw: &Raw) -> Result, Error> { + match raw { + Ok(value) => Ok(Some(value.clone())), + Err(VarError::NotPresent) => Ok(None), + Err(error) => Err(Error::Unreadable { + variable, + detail: error.to_string(), + }), + } +} + +/// The decision table over injected values, with no environment access. +/// +/// `values[i]` is the raw read of `inputs[i]`. The switch is consulted before +/// the job id, so a run that sets both is reported as demanded by the switch. +pub fn decide( + inputs: &[Input], + values: &[Raw], + switch: &Raw, + job: &Raw, +) -> Result { + assert_eq!( + inputs.len(), + values.len(), + "one raw value per requested input" + ); + let mut present = Vec::with_capacity(inputs.len()); + let mut missing = Vec::new(); + for (input, raw) in inputs.iter().zip(values) { + match readable(input.name(), raw)? { + Some(value) if !value.trim().is_empty() => present.push(value.trim().to_owned()), + _ => missing.push(*input), + } + } + let switch_on = match readable(SWITCH_VAR, switch)?.as_deref().map(str::trim) { + None | Some("") | Some("0") => false, + Some("1") => true, + Some(other) => { + return Err(Error::BadSwitch { + value: other.to_owned(), + }) + } + }; + let job_demands = readable(JOB_VAR, job)?.as_deref() == Some(REQUIRED_JOB); + if missing.is_empty() { + Ok(Decision::Run(present)) + } else if switch_on { + Ok(Decision::Fail { + missing, + demand: Demand::Switch, + }) + } else if job_demands { + Ok(Decision::Fail { + missing, + demand: Demand::Job, + }) + } else { + Ok(Decision::Skip { missing }) + } +} + +/// Where a gated test lives: the Cargo package and the test binary. Build it +/// with [`site!`] so both names come from the calling crate. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct Site { + /// `CARGO_PKG_NAME` of the crate holding the test. + pub package: &'static str, + /// `CARGO_CRATE_NAME` of the test binary: the integration test's file + /// stem, or the library's crate name for `#[cfg(test)]` modules. + pub binary: &'static str, +} + +/// The [`Site`] of the calling crate and test binary. +#[macro_export] +macro_rules! site { + () => { + $crate::Site { + package: env!("CARGO_PKG_NAME"), + binary: env!("CARGO_CRATE_NAME"), + } + }; +} + +impl Site { + /// `::::`, where the test path is the name + /// libtest gave the thread it runs the test on (`module::test_fn`). + pub fn test_id(self) -> Result { + let thread = std::thread::current(); + let name = match thread.name() { + Some(name) if !name.is_empty() && name != "main" && !name.starts_with("tokio-") => name, + other => { + return Err(Error::UnnamedThread { + thread: other.unwrap_or("").to_owned(), + }) + } + }; + Ok(format!("{}::{}::{name}", self.package, self.binary)) + } +} + +/// Appends ` ` to the manifest at `path` as one `O_APPEND` write. +/// +/// The whole line is built first and handed to a single `write`, never +/// `write_all`, which could split it across two writes after a short write and +/// let another thread's line land in between. A short write is an error. +pub fn record_to(path: &std::path::Path, kind: &str, id: &str) -> Result<(), Error> { + let line = format!("{kind} {id}\n"); + let manifest = |source| Error::Manifest { + path: path.to_path_buf(), + source, + }; + let mut file = OpenOptions::new() + .create(true) + .append(true) + .open(path) + .map_err(manifest)?; + let written = file.write(line.as_bytes()).map_err(manifest)?; + if written != line.len() { + return Err(manifest(std::io::Error::new( + std::io::ErrorKind::WriteZero, + format!( + "short write: {written} of {} bytes; the manifest line may be torn", + line.len() + ), + ))); + } + Ok(()) +} + +/// Records the decision in the manifest named by `PRISM_TEST_GATE_MANIFEST`, +/// if any. +fn record(kind: &str, id: &str) -> Result<(), Error> { + match readable(MANIFEST_VAR, &std::env::var(MANIFEST_VAR))? { + Some(path) if !path.trim().is_empty() => record_to(&PathBuf::from(path.trim()), kind, id), + _ => Ok(()), + } +} + +/// Prints one line on the process's real stderr, past libtest's capture, so +/// a skip is visible without `--nocapture`. The line goes out in a single +/// `write`, so parallel test threads cannot interleave their lines; a short +/// write is not retried, since a retry would be the second write. +fn announce(line: &str) { + let line = format!("{line}\n"); + #[cfg(unix)] + { + use std::os::fd::AsFd; + if let Ok(fd) = std::io::stderr().as_fd().try_clone_to_owned() { + let mut stderr = std::fs::File::from(fd); + let _ = stderr.write(line.as_bytes()); + return; + } + } + eprint!("{line}"); +} + +/// Applies the table to the process environment for `inputs`. +/// +/// Returns the values in the order requested, or `None` after printing the +/// skip line. A demanded but missing input is an error, as is an unreadable +/// variable or an unwritable manifest. +pub fn inputs(site: Site, inputs: &[Input]) -> Result>, Error> { + settle(site, inputs, false) +} + +/// Like [`inputs`], for a test that is selected explicitly (`#[ignore]`, run +/// with `--ignored`): a missing input always fails, since nothing else stops +/// the explicit selection from passing without running. Every `#[ignore]` +/// gated test in the workspace uses this entry point. +pub fn required_inputs(site: Site, inputs: &[Input]) -> Result, Error> { + settle(site, inputs, true).map(|values| values.expect("explicit tests never skip")) +} + +fn settle(site: Site, inputs: &[Input], explicit: bool) -> Result>, Error> { + let id = site.test_id()?; + let values: Vec = inputs.iter().map(|input| input.read()).collect(); + let decision = decide( + inputs, + &values, + &std::env::var(SWITCH_VAR), + &std::env::var(JOB_VAR), + )?; + let (decision, demand) = match decision { + Decision::Skip { missing } if explicit => ( + Decision::Fail { + missing, + demand: Demand::Explicit, + }, + Some(Demand::Explicit), + ), + Decision::Fail { missing, demand } => (Decision::Fail { missing, demand }, Some(demand)), + other => (other, None), + }; + match decision { + Decision::Run(values) => { + record("executed", &id)?; + Ok(Some(values)) + } + Decision::Fail { missing, .. } => { + record("failed", &id)?; + Err(Error::Required { + test: id, + missing, + demand: demand.expect("a failing decision names its demand"), + }) + } + Decision::Skip { missing } => { + record("skipped", &id)?; + announce(&format!( + "{SKIP_PREFIX} {id}: {}; set {} to run it, or {SWITCH_VAR}=1 to fail instead", + unset(&missing), + join(&missing, " and "), + )); + Ok(None) + } + } +} + +/// `PRISM_TEST_DATABASE_URL`, or `None` after the skip line. +pub fn database_url(site: Site) -> Result, Error> { + Ok(inputs(site, &[Input::DatabaseUrl])?.map(|mut values| values.remove(0))) +} + +/// `PRISM_TEST_DATABASE_URL` for an explicitly selected test; never skips. +pub fn required_database_url(site: Site) -> Result { + Ok(required_inputs(site, &[Input::DatabaseUrl])?.remove(0)) +} + +/// `PRISM_TEST_PG_BIN_DIR`, or `None` after the skip line. +pub fn pg_bin_dir(site: Site) -> Result, Error> { + Ok(inputs(site, &[Input::PgBinDir])?.map(|mut values| values.remove(0))) +} + +/// `(QBITD_BIN, PRISM_TEST_DATABASE_URL)`, or `None` after the skip line. +pub fn qbitd_and_database_url(site: Site) -> Result, Error> { + Ok( + inputs(site, &[Input::QbitdBin, Input::DatabaseUrl])?.map(|mut values| { + let database = values.remove(1); + (values.remove(0), database) + }), + ) +} + +#[cfg(test)] +mod tests { + use super::*; + use std::ffi::OsString; + + fn set(value: &str) -> Raw { + Ok(value.to_owned()) + } + + fn unset() -> Raw { + Err(VarError::NotPresent) + } + + #[cfg(unix)] + fn unreadable() -> Raw { + use std::os::unix::ffi::OsStringExt; + Err(VarError::NotUnicode(OsString::from_vec(vec![0xff, 0xfe]))) + } + + #[cfg(not(unix))] + fn unreadable() -> Raw { + Err(VarError::NotUnicode(OsString::from("\u{FFFD}"))) + } + + const DB: &[Input] = &[Input::DatabaseUrl]; + const LIVE: &[Input] = &[Input::QbitdBin, Input::DatabaseUrl]; + + #[test] + fn present_inputs_run_whatever_the_switch_and_job_say() { + for switch in [unset(), set("0"), set("1"), set("")] { + for job in [unset(), set("rust-tests"), set(REQUIRED_JOB)] { + let decision = decide(DB, &[set(" postgres://x/y ")], &switch, &job).unwrap(); + assert_eq!(decision, Decision::Run(vec!["postgres://x/y".into()])); + } + } + let decision = decide( + LIVE, + &[set("/bin/qbitd"), set("postgres://x/y")], + &unset(), + &unset(), + ); + assert_eq!( + decision.unwrap(), + Decision::Run(vec!["/bin/qbitd".into(), "postgres://x/y".into()]) + ); + } + + #[test] + fn a_missing_input_fails_when_the_switch_is_on() { + for job in [unset(), set("rust-tests"), set(REQUIRED_JOB)] { + let decision = decide(DB, &[unset()], &set("1"), &job).unwrap(); + assert_eq!( + decision, + Decision::Fail { + missing: vec![Input::DatabaseUrl], + demand: Demand::Switch, + } + ); + } + } + + #[test] + fn a_missing_input_fails_in_the_native_job() { + for switch in [unset(), set(""), set("0")] { + let decision = decide(DB, &[unset()], &switch, &set(REQUIRED_JOB)).unwrap(); + assert_eq!( + decision, + Decision::Fail { + missing: vec![Input::DatabaseUrl], + demand: Demand::Job, + } + ); + } + } + + #[test] + fn a_missing_input_skips_when_nothing_demands_it() { + for switch in [unset(), set(""), set("0")] { + for job in [unset(), set("rust-tests"), set("")] { + let decision = decide(DB, &[unset()], &switch, &job).unwrap(); + assert_eq!( + decision, + Decision::Skip { + missing: vec![Input::DatabaseUrl] + } + ); + } + } + } + + #[test] + fn empty_and_whitespace_values_count_as_unset() { + for value in ["", " ", "\t\n"] { + let decision = decide(DB, &[set(value)], &unset(), &unset()).unwrap(); + assert_eq!( + decision, + Decision::Skip { + missing: vec![Input::DatabaseUrl] + } + ); + } + } + + #[test] + fn every_missing_input_is_named_in_request_order() { + let decision = decide(LIVE, &[unset(), set(" ")], &set("1"), &unset()).unwrap(); + assert_eq!( + decision, + Decision::Fail { + missing: vec![Input::QbitdBin, Input::DatabaseUrl], + demand: Demand::Switch, + } + ); + let decision = decide(LIVE, &[set("/bin/qbitd"), unset()], &unset(), &unset()).unwrap(); + assert_eq!( + decision, + Decision::Skip { + missing: vec![Input::DatabaseUrl] + } + ); + } + + #[test] + fn an_unreadable_variable_is_an_error_never_absent() { + let error = decide(DB, &[unreadable()], &unset(), &unset()).unwrap_err(); + assert!( + error + .to_string() + .starts_with("PRISM_TEST_DATABASE_URL is set but not readable"), + "{error}" + ); + let error = decide(DB, &[set("postgres://x/y")], &unreadable(), &unset()).unwrap_err(); + assert!( + error + .to_string() + .starts_with("PRISM_TEST_REQUIRE_INTEGRATION is set but not readable"), + "{error}" + ); + let error = decide(DB, &[unset()], &unset(), &unreadable()).unwrap_err(); + assert!( + error + .to_string() + .starts_with("GITHUB_JOB is set but not readable"), + "{error}" + ); + // Even a present input does not excuse an unreadable sibling. + let error = decide(LIVE, &[set("/bin/qbitd"), unreadable()], &unset(), &unset()); + assert!(error.is_err()); + } + + #[test] + fn an_unrecognised_switch_value_is_an_error() { + for value in ["yes", "true", "2", "on"] { + let error = decide(DB, &[unset()], &set(value), &unset()).unwrap_err(); + assert!( + matches!(&error, Error::BadSwitch { value: got } if got == value), + "{error}" + ); + // Also when the inputs are present: a typo must not pass silently. + assert!(decide(DB, &[set("postgres://x/y")], &set(value), &unset()).is_err()); + } + assert!(decide(DB, &[unset()], &set(" 1 "), &unset()).is_ok()); + } + + #[test] + fn the_switch_is_named_before_the_job_when_both_demand() { + let decision = decide(DB, &[unset()], &set("1"), &set(REQUIRED_JOB)).unwrap(); + assert!(matches!( + decision, + Decision::Fail { + demand: Demand::Switch, + .. + } + )); + } + + #[test] + fn failure_messages_name_the_test_the_variables_and_the_demand() { + let error = Error::Required { + test: "qbit-prism-server::live_regtest::real_two_server_mining".into(), + missing: vec![Input::QbitdBin, Input::DatabaseUrl], + demand: Demand::Job, + }; + assert_eq!( + error.to_string(), + "qbit-prism-server::live_regtest::real_two_server_mining requires QBITD_BIN, \ + PRISM_TEST_DATABASE_URL: QBITD_BIN and PRISM_TEST_DATABASE_URL are unset or empty \ + while GITHUB_JOB=prism-native-postgres demands the integration suite" + ); + let error = Error::Required { + test: "t".into(), + missing: vec![Input::PgBinDir], + demand: Demand::Explicit, + }; + assert_eq!( + error.to_string(), + "t requires PRISM_TEST_PG_BIN_DIR: PRISM_TEST_PG_BIN_DIR is unset or empty while the \ + explicit selection of this test demands the integration suite" + ); + } + + #[test] + fn the_test_id_comes_from_the_test_thread() { + let site = Site { + package: "pkg", + binary: "bin", + }; + assert_eq!( + site.test_id().unwrap(), + "pkg::bin::tests::the_test_id_comes_from_the_test_thread" + ); + let off_thread = std::thread::spawn(move || site.test_id()).join().unwrap(); + assert!( + matches!(off_thread, Err(Error::UnnamedThread { .. })), + "{off_thread:?}" + ); + let worker = std::thread::Builder::new() + .name("tokio-runtime-worker".into()) + .spawn(move || site.test_id()) + .unwrap() + .join() + .unwrap(); + assert!(matches!(worker, Err(Error::UnnamedThread { .. }))); + } + + #[test] + fn manifest_lines_append_one_per_decision() { + let dir = std::env::temp_dir().join(format!( + "prism-test-gate-{}-{}", + std::process::id(), + line!() + )); + std::fs::create_dir_all(&dir).unwrap(); + let path = dir.join("manifest.txt"); + record_to(&path, "executed", "a::b::c").unwrap(); + record_to(&path, "skipped", "a::b::d").unwrap(); + record_to(&path, "failed", "a::b::e").unwrap(); + assert_eq!( + std::fs::read_to_string(&path).unwrap(), + "executed a::b::c\nskipped a::b::d\nfailed a::b::e\n" + ); + let error = + record_to(&dir.join("missing").join("manifest.txt"), "executed", "x").unwrap_err(); + assert!( + error + .to_string() + .starts_with("cannot append to PRISM_TEST_GATE_MANIFEST="), + "{error}" + ); + std::fs::remove_dir_all(&dir).unwrap(); + } + + #[test] + fn the_skip_prefix_is_stable() { + // CI greps the test log for this exact text; change it together with + // scripts/check_gate_manifest.py and docs/prism-integration-test-gate.md. + assert_eq!(SKIP_PREFIX, "[prism-test-gate] skipped"); + } +} diff --git a/crates/qbit-prism-test-gate/tests/process.rs b/crates/qbit-prism-test-gate/tests/process.rs new file mode 100644 index 000000000..65065bb1c --- /dev/null +++ b/crates/qbit-prism-test-gate/tests/process.rs @@ -0,0 +1,231 @@ +//! The runtime path end to end: a child copy of this binary runs one gated +//! probe under a controlled environment, and the parent checks what the gate +//! did to the exit status, the manifest and stderr. + +use qbit_prism_test_gate::{self as gate, Input, SKIP_PREFIX}; +use std::path::PathBuf; +use std::process::Command; + +const PROBE: &str = "child_probe"; +const REQUIRED_PROBE: &str = "child_required_probe"; + +/// Runs in the child only. Explicitly selected so the parent's run does not +/// count it; the parent runs it with `--ignored --exact`. +#[test] +#[ignore = "driven by the parent tests below through a child process"] +fn child_probe() { + match gate::database_url(gate::site!()) { + Ok(Some(url)) => println!("probe ran with {url}"), + Ok(None) => println!("probe skipped"), + Err(error) => panic!("{error}"), + } +} + +#[test] +#[ignore = "driven by the parent tests below through a child process"] +fn child_required_probe() { + match gate::required_database_url(gate::site!()) { + Ok(url) => println!("probe ran with {url}"), + Err(error) => panic!("{error}"), + } +} + +struct Run { + success: bool, + stdout: String, + stderr: String, + manifest: String, +} + +fn run_probe(probe: &str, env: &[(&str, &str)]) -> Run { + let dir = std::env::temp_dir().join(format!( + "prism-test-gate-process-{}-{}", + std::process::id(), + std::thread::current().name().unwrap_or("t") + )); + std::fs::create_dir_all(&dir).unwrap(); + let manifest: PathBuf = dir.join("manifest.txt"); + let _ = std::fs::remove_file(&manifest); + let mut command = Command::new(std::env::current_exe().unwrap()); + command + .args(["--ignored", "--exact", probe, "--nocapture"]) + .env_clear() + .env(gate::MANIFEST_VAR, &manifest); + for (name, value) in env { + command.env(name, value); + } + let output = command.output().unwrap(); + let run = Run { + success: output.status.success(), + stdout: String::from_utf8_lossy(&output.stdout).into_owned(), + stderr: String::from_utf8_lossy(&output.stderr).into_owned(), + manifest: std::fs::read_to_string(&manifest).unwrap_or_default(), + }; + std::fs::remove_dir_all(&dir).unwrap(); + run +} + +fn id(probe: &str) -> String { + format!("qbit-prism-test-gate::process::{probe}") +} + +#[test] +fn a_present_input_runs_and_is_recorded_as_executed() { + let run = run_probe(PROBE, &[(Input::DatabaseUrl.name(), " postgres://x/y ")]); + assert!(run.success, "{}{}", run.stdout, run.stderr); + assert!( + run.stdout.contains("probe ran with postgres://x/y"), + "{}", + run.stdout + ); + assert_eq!(run.manifest, format!("executed {}\n", id(PROBE))); + assert!(!run.stderr.contains(SKIP_PREFIX), "{}", run.stderr); +} + +#[test] +fn a_missing_input_skips_with_one_prefixed_line() { + let run = run_probe(PROBE, &[]); + assert!(run.success, "{}{}", run.stdout, run.stderr); + assert!(run.stdout.contains("probe skipped"), "{}", run.stdout); + assert_eq!(run.manifest, format!("skipped {}\n", id(PROBE))); + let skip_lines: Vec<&str> = run + .stderr + .lines() + .filter(|line| line.starts_with(SKIP_PREFIX)) + .collect(); + assert_eq!( + skip_lines, + vec![format!( + "{SKIP_PREFIX} {}: PRISM_TEST_DATABASE_URL is unset or empty; set \ + PRISM_TEST_DATABASE_URL to run it, or PRISM_TEST_REQUIRE_INTEGRATION=1 to fail instead", + id(PROBE) + )] + ); +} + +#[test] +fn the_skip_line_is_visible_without_nocapture() { + // Same as above, but with libtest capturing: the line must still reach stderr. + let dir = std::env::temp_dir().join(format!("prism-test-gate-capture-{}", std::process::id())); + std::fs::create_dir_all(&dir).unwrap(); + let output = Command::new(std::env::current_exe().unwrap()) + .args(["--ignored", "--exact", PROBE]) + .env_clear() + .output() + .unwrap(); + std::fs::remove_dir_all(&dir).unwrap(); + assert!(output.status.success()); + let stderr = String::from_utf8_lossy(&output.stderr); + assert_eq!(stderr.matches(SKIP_PREFIX).count(), 1, "{stderr}"); +} + +#[test] +fn a_missing_input_fails_when_the_switch_is_on() { + let run = run_probe(PROBE, &[(gate::SWITCH_VAR, "1")]); + assert!(!run.success, "{}{}", run.stdout, run.stderr); + assert_eq!(run.manifest, format!("failed {}\n", id(PROBE))); + let expected = format!( + "{} requires PRISM_TEST_DATABASE_URL: PRISM_TEST_DATABASE_URL is unset or empty while \ + PRISM_TEST_REQUIRE_INTEGRATION=1 demands the integration suite", + id(PROBE) + ); + assert!( + run.stdout.contains(&expected) || run.stderr.contains(&expected), + "{}{}", + run.stdout, + run.stderr + ); + assert!(!run.stderr.contains(SKIP_PREFIX), "{}", run.stderr); +} + +#[test] +fn a_missing_input_fails_in_the_native_job() { + let run = run_probe(PROBE, &[(gate::JOB_VAR, gate::REQUIRED_JOB)]); + assert!(!run.success, "{}{}", run.stdout, run.stderr); + assert_eq!(run.manifest, format!("failed {}\n", id(PROBE))); + let expected = format!( + "GITHUB_JOB={} demands the integration suite", + gate::REQUIRED_JOB + ); + assert!( + run.stdout.contains(&expected) || run.stderr.contains(&expected), + "{}{}", + run.stdout, + run.stderr + ); +} + +#[test] +fn another_job_id_still_skips() { + let run = run_probe(PROBE, &[(gate::JOB_VAR, "rust-tests"), ("CI", "true")]); + assert!(run.success, "{}{}", run.stdout, run.stderr); + assert_eq!(run.manifest, format!("skipped {}\n", id(PROBE))); +} + +#[test] +fn an_explicitly_selected_test_never_skips() { + let run = run_probe(REQUIRED_PROBE, &[]); + assert!(!run.success, "{}{}", run.stdout, run.stderr); + assert_eq!(run.manifest, format!("failed {}\n", id(REQUIRED_PROBE))); + let expected = "while the explicit selection of this test demands the integration suite"; + assert!( + run.stdout.contains(expected) || run.stderr.contains(expected), + "{}{}", + run.stdout, + run.stderr + ); + let run = run_probe( + REQUIRED_PROBE, + &[(Input::DatabaseUrl.name(), "postgres://x/y")], + ); + assert!(run.success, "{}{}", run.stdout, run.stderr); + assert_eq!(run.manifest, format!("executed {}\n", id(REQUIRED_PROBE))); +} + +#[test] +fn an_unwritable_manifest_fails_the_test() { + let output = Command::new(std::env::current_exe().unwrap()) + .args(["--ignored", "--exact", PROBE, "--nocapture"]) + .env_clear() + .env(Input::DatabaseUrl.name(), "postgres://x/y") + .env( + gate::MANIFEST_VAR, + "/nonexistent-prism-gate-dir/manifest.txt", + ) + .output() + .unwrap(); + assert!(!output.status.success()); + let text = format!( + "{}{}", + String::from_utf8_lossy(&output.stdout), + String::from_utf8_lossy(&output.stderr) + ); + assert!( + text.contains("cannot append to PRISM_TEST_GATE_MANIFEST="), + "{text}" + ); +} + +#[cfg(unix)] +#[test] +fn an_unreadable_input_fails_instead_of_skipping() { + use std::ffi::OsStr; + use std::os::unix::ffi::OsStrExt; + let output = Command::new(std::env::current_exe().unwrap()) + .args(["--ignored", "--exact", PROBE, "--nocapture"]) + .env_clear() + .env(Input::DatabaseUrl.name(), OsStr::from_bytes(&[0xff, 0xfe])) + .output() + .unwrap(); + assert!(!output.status.success()); + let text = format!( + "{}{}", + String::from_utf8_lossy(&output.stdout), + String::from_utf8_lossy(&output.stderr) + ); + assert!( + text.contains("PRISM_TEST_DATABASE_URL is set but not readable"), + "{text}" + ); + assert!(!text.contains(SKIP_PREFIX), "{text}"); +} diff --git a/crates/qbit-prism/Cargo.toml b/crates/qbit-prism/Cargo.toml index 4055f94fe..b2f35ddf1 100644 --- a/crates/qbit-prism/Cargo.toml +++ b/crates/qbit-prism/Cargo.toml @@ -1,8 +1,12 @@ [package] name = "qbit-prism" -version = "2.0.0" +version.workspace = true +rust-version.workspace = true edition = "2021" license = "MIT" +# Built and shipped from this repository; its path dependencies carry no +# version, which cargo-deny permits only for crates that are never published. +publish = false description = "Payouts, Rewards, and Integrity Settlement Manifest pool accounting for qbit" [dependencies] @@ -15,3 +19,6 @@ thiserror = "2.0" [dev-dependencies] hex = "0.4" +# Every environment-gated test in this crate must take its inputs through the +# shared gate; see docs/prism-integration-test-gate.md. +qbit-prism-test-gate = { path = "../qbit-prism-test-gate" } diff --git a/crates/qbit-prism/README.md b/crates/qbit-prism/README.md index be0b8a302..36603dd0b 100644 --- a/crates/qbit-prism/README.md +++ b/crates/qbit-prism/README.md @@ -17,27 +17,25 @@ The Postgres schema in [`sql/001_share_ledger.sql`](sql/001_share_ledger.sql) defines: - `qbit_share_ledger`: append-only canonical share log ordered by `share_seq` -- `qbit_ledger_writer_lease`: one-row coordination table for a single logical - writer/failover epoch -- `qbit_pool_blocks.audit_publication_sequence`: durable confirmation and - reactivation order for current audit evidence, independent of block height +- `qbit_ledger_writer_lease`: retained legacy migration metadata; native servers + do not acquire this application writer lease - `qbit_prism_window(...)`: deterministic newest-backward window query with partial oldest-share weighting -The schema file is cumulative and idempotent. Existing deployments must rerun -it before upgrading when automatic schema initialization is disabled; see the -[ledger operations contract](../../docs/prism-ledger-ops.md#audit-publication-ordering-migration). - -Stratum frontends should enqueue share submissions outside this table. Only the -active ledger writer inserts rows into `qbit_share_ledger`; that is what keeps -all miners in the same reward universe. +The native [`qbit-prism-server`](../qbit-prism-server/README.md) applies the +additive [multi-instance migration](../qbit-prism-server/migrations/002_multi_instance.sql) +and [2.x compatibility migration](../qbit-prism-server/migrations/003_2x_compatibility.sql). +All instances append through short PostgreSQL transactions sharing an ordering +lock, and successful ordinary share ACKs follow commit. A single global share +order gives all miners the same reward universe without electing one frontend +as writer. The migration prevents a legacy Python writer from restarting +against the upgraded database. [`sql/001_share_ledger_revert_audit_publication_sequence.sql`](sql/001_share_ledger_revert_audit_publication_sequence.sql) -returns `qbit_pool_blocks` to its pre-publication-ordinal shape for operators -rolling back past that migration; it discards assigned ordinals, so read -"Publication Ordinal Rollback And Schema Revert" in -[`docs/prism-ledger-ops.md`](../../docs/prism-ledger-ops.md) before applying -it. +is retained only for legacy Python databases and refuses native databases. +It discards assigned ordinals. Native rollback requires stopping all writers +and restoring the pre-migration backup, as described in the +[cutover and rollback guide](../../docs/prism-rust-migration.md). ## Reward Rule @@ -139,12 +137,19 @@ accepted share slice, found-block anchor, prior carry-forward balances, payout floor parameters, PRISM reward manifest, payout-policy/accrual manifest, and the signed deterministic coinbase manifest. -The verifier and canonicalizer also accept storage-oriented compact artifacts: +Native database audits store non-share bundle fields and an immutable share +snapshot reference. The server reconstructs the canonical slice from the shared +ledger and verifies its digest and full bundle SHA before serving the logical +v1/v1.1 body. Accepted ledger rows cannot be changed or deleted, so overlapping +block windows reuse existing share storage without changing published hashes. + +The verifier and canonicalizer also accept historical storage-oriented artifacts: legacy `qbit.prism.audit-body-ref.v1` files and `qbit.prism.audit-bundle.v2` proof bodies. Those formats keep share payloads in referenced segment files, verify each segment or range hash, and reconstruct the same canonical v1 bundle before checking payout math. Public `/public/v1` -responses remain logical v1 bundles. +responses remain logical v1/v1.1 bundles. Import legacy filesystem bodies into +the shared database during [native cutover](../../docs/prism-rust-migration.md). `verify_audit_bundle` recomputes the full path from exported data and a verifier-supplied trusted ledger writer public key. The ledger attestation binds diff --git a/crates/qbit-prism/fixtures/vectors/below_target_credit.json b/crates/qbit-prism/fixtures/vectors/below_target_credit.json new file mode 100644 index 000000000..fe7ef64b3 --- /dev/null +++ b/crates/qbit-prism/fixtures/vectors/below_target_credit.json @@ -0,0 +1,2074 @@ +{ + "cases": [ + { + "d2_entry": "d2b-below-target-credit", + "entry_point": "bundle_payout", + "expected_2xx": { + "ok": { + "counted_shares": [ + { + "counted_difficulty": 4000000, + "miner_id": "miner-a", + "share_seq": 4 + }, + { + "counted_difficulty": 1000000, + "miner_id": "miner-c", + "share_seq": 3 + }, + { + "counted_difficulty": 2000000, + "miner_id": "miner-b", + "share_seq": 2 + }, + { + "counted_difficulty": 1000000, + "miner_id": "miner-b", + "share_seq": 1 + } + ], + "counted_window_weight": 8000000, + "entitlements": [ + { + "order_key": "miner-a", + "p2mr_program_hex": "0101010101010101010101010101010101010101010101010101010101010101", + "recipient_id": "miner-a", + "weight": 4000000 + }, + { + "order_key": "miner-b", + "p2mr_program_hex": "0202020202020202020202020202020202020202020202020202020202020202", + "recipient_id": "miner-b", + "weight": 3000000 + }, + { + "order_key": "miner-c", + "p2mr_program_hex": "0303030303030303030303030303030303030303030303030303030303030303", + "recipient_id": "miner-c", + "weight": 1000000 + } + ], + "payout_policy_manifest": { + "accounts": [ + { + "action": "onchain", + "candidate_balance_sats": 250000000, + "carry_forward_balance_sats": 0, + "gross_amount_sats": 250000000, + "onchain_amount_sats": 250000000, + "order_key": "miner-a", + "p2mr_program_hex": "0101010101010101010101010101010101010101010101010101010101010101", + "prior_balance_sats": 0, + "recipient_id": "miner-a" + }, + { + "action": "onchain", + "candidate_balance_sats": 187500000, + "carry_forward_balance_sats": 0, + "gross_amount_sats": 187500000, + "onchain_amount_sats": 187500000, + "order_key": "miner-b", + "p2mr_program_hex": "0202020202020202020202020202020202020202020202020202020202020202", + "prior_balance_sats": 0, + "recipient_id": "miner-b" + }, + { + "action": "onchain", + "candidate_balance_sats": 62500000, + "carry_forward_balance_sats": 0, + "gross_amount_sats": 62500000, + "onchain_amount_sats": 62500000, + "order_key": "miner-c", + "p2mr_program_hex": "0303030303030303030303030303030303030303030303030303030303030303", + "prior_balance_sats": 0, + "recipient_id": "miner-c" + } + ], + "block_height": 202, + "coinbase_value_sats": 500000000, + "floor_formula": "3680 bytes/input * 1 sat/byte * 4x safety", + "min_output_sats": 14720, + "onchain_entitlements": [ + { + "order_key": "miner-a", + "p2mr_program_hex": "0101010101010101010101010101010101010101010101010101010101010101", + "recipient_id": "miner-a", + "weight": 250000000 + }, + { + "order_key": "miner-b", + "p2mr_program_hex": "0202020202020202020202020202020202020202020202020202020202020202", + "recipient_id": "miner-b", + "weight": 187500000 + }, + { + "order_key": "miner-c", + "p2mr_program_hex": "0303030303030303030303030303030303030303030303030303030303030303", + "recipient_id": "miner-c", + "weight": 62500000 + } + ], + "schema": "qbit.prism.payout-policy.v1" + } + } + }, + "expected_3xx": { + "ok": { + "counted_shares": [ + { + "counted_difficulty": 1000000, + "miner_id": "miner-a", + "share_seq": 4 + }, + { + "counted_difficulty": 1000000, + "miner_id": "miner-c", + "share_seq": 3 + }, + { + "counted_difficulty": 2000000, + "miner_id": "miner-b", + "share_seq": 2 + }, + { + "counted_difficulty": 2000000, + "miner_id": "miner-b", + "share_seq": 1 + } + ], + "counted_window_weight": 6000000, + "entitlements": [ + { + "order_key": "miner-a", + "p2mr_program_hex": "0101010101010101010101010101010101010101010101010101010101010101", + "recipient_id": "miner-a", + "weight": 1000000 + }, + { + "order_key": "miner-b", + "p2mr_program_hex": "0202020202020202020202020202020202020202020202020202020202020202", + "recipient_id": "miner-b", + "weight": 4000000 + }, + { + "order_key": "miner-c", + "p2mr_program_hex": "0303030303030303030303030303030303030303030303030303030303030303", + "recipient_id": "miner-c", + "weight": 1000000 + } + ], + "payout_policy_manifest": { + "accounts": [ + { + "action": "onchain", + "candidate_balance_sats": 83333334, + "carry_forward_balance_sats": 0, + "gross_amount_sats": 83333334, + "onchain_amount_sats": 83333334, + "order_key": "miner-a", + "p2mr_program_hex": "0101010101010101010101010101010101010101010101010101010101010101", + "prior_balance_sats": 0, + "recipient_id": "miner-a" + }, + { + "action": "onchain", + "candidate_balance_sats": 333333333, + "carry_forward_balance_sats": 0, + "gross_amount_sats": 333333333, + "onchain_amount_sats": 333333333, + "order_key": "miner-b", + "p2mr_program_hex": "0202020202020202020202020202020202020202020202020202020202020202", + "prior_balance_sats": 0, + "recipient_id": "miner-b" + }, + { + "action": "onchain", + "candidate_balance_sats": 83333333, + "carry_forward_balance_sats": 0, + "gross_amount_sats": 83333333, + "onchain_amount_sats": 83333333, + "order_key": "miner-c", + "p2mr_program_hex": "0303030303030303030303030303030303030303030303030303030303030303", + "prior_balance_sats": 0, + "recipient_id": "miner-c" + } + ], + "block_height": 202, + "coinbase_value_sats": 500000000, + "floor_formula": "3680 bytes/input * 1 sat/byte * 4x safety", + "min_output_sats": 14720, + "onchain_entitlements": [ + { + "order_key": "miner-a", + "p2mr_program_hex": "0101010101010101010101010101010101010101010101010101010101010101", + "recipient_id": "miner-a", + "weight": 83333334 + }, + { + "order_key": "miner-b", + "p2mr_program_hex": "0202020202020202020202020202020202020202020202020202020202020202", + "recipient_id": "miner-b", + "weight": 333333333 + }, + { + "order_key": "miner-c", + "p2mr_program_hex": "0303030303030303030303030303030303030303030303030303030303030303", + "recipient_id": "miner-c", + "weight": 83333333 + } + ], + "schema": "qbit.prism.payout-policy.v1" + } + } + }, + "input": { + "bundle_2xx": { + "found_block": { + "anchor_job_issued_at_ms": 9010, + "block_height": 202, + "coinbase_value_sats": 500000000, + "network_difficulty": 1000000 + }, + "policy": { + "p2mr_spend_input_bytes": 3680, + "safety_multiplier": 4, + "target_feerate_sats_per_byte": 1 + }, + "prior_balances": [], + "shares": [ + { + "accepted_at_ms": 9000, + "job_id": "job-9000", + "job_issued_at_ms": 9000, + "miner_id": "miner-b", + "network_difficulty": 1000000, + "ntime": 1800000000, + "order_key": "miner-b", + "p2mr_program_hex": "0202020202020202020202020202020202020202020202020202020202020202", + "share_difficulty": 2000000, + "share_id": "share-1", + "share_seq": 1, + "template_height": 100 + }, + { + "accepted_at_ms": 9001, + "job_id": "job-9001", + "job_issued_at_ms": 9001, + "miner_id": "miner-b", + "network_difficulty": 1000000, + "ntime": 1800000000, + "order_key": "miner-b", + "p2mr_program_hex": "0202020202020202020202020202020202020202020202020202020202020202", + "share_difficulty": 2000000, + "share_id": "share-2", + "share_seq": 2, + "template_height": 100 + }, + { + "accepted_at_ms": 9002, + "job_id": "job-9002", + "job_issued_at_ms": 9002, + "miner_id": "miner-c", + "network_difficulty": 1000000, + "ntime": 1800000000, + "order_key": "miner-c", + "p2mr_program_hex": "0303030303030303030303030303030303030303030303030303030303030303", + "share_difficulty": 1000000, + "share_id": "share-3", + "share_seq": 3, + "template_height": 100 + }, + { + "accepted_at_ms": 9003, + "job_id": "job-below-target", + "job_issued_at_ms": 9003, + "miner_id": "miner-a", + "network_difficulty": 1000000, + "ntime": 1800000000, + "order_key": "miner-a", + "p2mr_program_hex": "0101010101010101010101010101010101010101010101010101010101010101", + "share_difficulty": 4000000, + "share_id": "miner-a.rig:abababababababababababababababababababababababababababababababab", + "share_seq": 4, + "template_height": 200 + } + ] + }, + "bundle_3xx": { + "found_block": { + "anchor_job_issued_at_ms": 9010, + "block_height": 202, + "coinbase_value_sats": 500000000, + "network_difficulty": 1000000 + }, + "policy": { + "p2mr_spend_input_bytes": 3680, + "safety_multiplier": 4, + "target_feerate_sats_per_byte": 1 + }, + "prior_balances": [], + "shares": [ + { + "accepted_at_ms": 9000, + "job_id": "job-9000", + "job_issued_at_ms": 9000, + "miner_id": "miner-b", + "network_difficulty": 1000000, + "ntime": 1800000000, + "order_key": "miner-b", + "p2mr_program_hex": "0202020202020202020202020202020202020202020202020202020202020202", + "share_difficulty": 2000000, + "share_id": "share-1", + "share_seq": 1, + "template_height": 100 + }, + { + "accepted_at_ms": 9001, + "job_id": "job-9001", + "job_issued_at_ms": 9001, + "miner_id": "miner-b", + "network_difficulty": 1000000, + "ntime": 1800000000, + "order_key": "miner-b", + "p2mr_program_hex": "0202020202020202020202020202020202020202020202020202020202020202", + "share_difficulty": 2000000, + "share_id": "share-2", + "share_seq": 2, + "template_height": 100 + }, + { + "accepted_at_ms": 9002, + "job_id": "job-9002", + "job_issued_at_ms": 9002, + "miner_id": "miner-c", + "network_difficulty": 1000000, + "ntime": 1800000000, + "order_key": "miner-c", + "p2mr_program_hex": "0303030303030303030303030303030303030303030303030303030303030303", + "share_difficulty": 1000000, + "share_id": "share-3", + "share_seq": 3, + "template_height": 100 + }, + { + "accepted_at_ms": 9003, + "job_id": "job-below-target", + "job_issued_at_ms": 9003, + "miner_id": "miner-a", + "network_difficulty": 1000000, + "ntime": 1800000000, + "order_key": "miner-a", + "p2mr_program_hex": "0101010101010101010101010101010101010101010101010101010101010101", + "share_difficulty": 1000000, + "share_id": "miner-a.rig:abababababababababababababababababababababababababababababababab", + "share_seq": 4, + "template_height": 200 + } + ] + }, + "rule": { + "decision_2xx": { + "assigned_share_difficulty": 4000000, + "block_worthy": true, + "credit_share_on_accept": true, + "credited": true, + "credited_at": "node acceptance of the block (synchronous submit)", + "credited_difficulty": 4000000, + "implemented_at": [ + "lab/prism/share_submission.py:360 classify_submit_work", + "lab/prism/share_writer.py:281 runtime.accepted_share_difficulty(context)", + "lab/prism/prism_coordinator.py:8410 accepted_share_difficulty", + "lab/prism/prism_coordinator.py:8155 _submit_synchronous_credit_candidate", + "lab/prism/block_finalization.py:2027 append_accepted_share when credit_share_on_accept" + ], + "network_difficulty": 1000000, + "rejection": null, + "route": "synchronous_block" + }, + "decision_3xx": { + "credited": true, + "credited_at": "confirmation transaction after node acceptance; the submit acknowledgement waits for it", + "credited_difficulty": 1000000, + "implemented_at": [ + "crates/qbit-prism-server/src/coordinator.rs:1648-1654 network difficulty for a block-only proof", + "crates/qbit-prism-server/src/coordinator.rs:1698 deferred_share held with the block candidate", + "crates/qbit-prism-server/src/coordinator.rs:1726-1747 submit waits for the credit row or fails", + "crates/qbit-prism-server/src/ledger/blocks.rs:200 credit_deferred_share on confirmation" + ] + }, + "scenario": { + "block_pass": true, + "credited_share": { + "accepted_at_ms": 9003, + "job_issued_at_ms": 9003, + "share_id": "miner-a.rig:abababababababababababababababababababababababababababababababab", + "share_seq": 4 + }, + "ledger_shares": [ + { + "accepted_at_ms": 9000, + "job_id": "job-9000", + "job_issued_at_ms": 9000, + "miner_id": "miner-b", + "network_difficulty": 1000000, + "ntime": 1800000000, + "order_key": "miner-b", + "p2mr_program_hex": "0202020202020202020202020202020202020202020202020202020202020202", + "share_difficulty": 2000000, + "share_id": "share-1", + "share_seq": 1, + "template_height": 100 + }, + { + "accepted_at_ms": 9001, + "job_id": "job-9001", + "job_issued_at_ms": 9001, + "miner_id": "miner-b", + "network_difficulty": 1000000, + "ntime": 1800000000, + "order_key": "miner-b", + "p2mr_program_hex": "0202020202020202020202020202020202020202020202020202020202020202", + "share_difficulty": 2000000, + "share_id": "share-2", + "share_seq": 2, + "template_height": 100 + }, + { + "accepted_at_ms": 9002, + "job_id": "job-9002", + "job_issued_at_ms": 9002, + "miner_id": "miner-c", + "network_difficulty": 1000000, + "ntime": 1800000000, + "order_key": "miner-c", + "p2mr_program_hex": "0303030303030303030303030303030303030303030303030303030303030303", + "share_difficulty": 1000000, + "share_id": "share-3", + "share_seq": 3, + "template_height": 100 + } + ], + "network_target": 57896037716911750921221705069588091649609539881711309849342236841432341020672, + "next_block": { + "anchor_job_issued_at_ms": 9010, + "block_height": 202, + "coinbase_value_sats": 500000000, + "network_difficulty": 1000000 + }, + "node_outcome": "accepted", + "share_pass": false, + "share_target": 14474009429227937730305426267397022912402384970427827462335559210358085255168, + "solver": { + "p2mr_program_hex": "0101010101010101010101010101010101010101010101010101010101010101", + "payout_address": "miner-a" + } + } + } + }, + "name": "block-only-proof-accepted", + "topic": 9, + "why": "The hash meets the network target but not the floor-raised share target, and the node accepts the block." + }, + { + "d2_entry": "d2b-below-target-credit", + "entry_point": "bundle_payout", + "expected_2xx": { + "ok": { + "counted_shares": [ + { + "counted_difficulty": 4000000, + "miner_id": "miner-a", + "share_seq": 4 + }, + { + "counted_difficulty": 1000000, + "miner_id": "miner-c", + "share_seq": 3 + }, + { + "counted_difficulty": 2000000, + "miner_id": "miner-b", + "share_seq": 2 + }, + { + "counted_difficulty": 1000000, + "miner_id": "miner-b", + "share_seq": 1 + } + ], + "counted_window_weight": 8000000, + "entitlements": [ + { + "order_key": "miner-a", + "p2mr_program_hex": "0101010101010101010101010101010101010101010101010101010101010101", + "recipient_id": "miner-a", + "weight": 4000000 + }, + { + "order_key": "miner-b", + "p2mr_program_hex": "0202020202020202020202020202020202020202020202020202020202020202", + "recipient_id": "miner-b", + "weight": 3000000 + }, + { + "order_key": "miner-c", + "p2mr_program_hex": "0303030303030303030303030303030303030303030303030303030303030303", + "recipient_id": "miner-c", + "weight": 1000000 + } + ], + "payout_policy_manifest": { + "accounts": [ + { + "action": "onchain", + "candidate_balance_sats": 250000000, + "carry_forward_balance_sats": 0, + "gross_amount_sats": 250000000, + "onchain_amount_sats": 250000000, + "order_key": "miner-a", + "p2mr_program_hex": "0101010101010101010101010101010101010101010101010101010101010101", + "prior_balance_sats": 0, + "recipient_id": "miner-a" + }, + { + "action": "onchain", + "candidate_balance_sats": 187500000, + "carry_forward_balance_sats": 0, + "gross_amount_sats": 187500000, + "onchain_amount_sats": 187500000, + "order_key": "miner-b", + "p2mr_program_hex": "0202020202020202020202020202020202020202020202020202020202020202", + "prior_balance_sats": 0, + "recipient_id": "miner-b" + }, + { + "action": "onchain", + "candidate_balance_sats": 62500000, + "carry_forward_balance_sats": 0, + "gross_amount_sats": 62500000, + "onchain_amount_sats": 62500000, + "order_key": "miner-c", + "p2mr_program_hex": "0303030303030303030303030303030303030303030303030303030303030303", + "prior_balance_sats": 0, + "recipient_id": "miner-c" + } + ], + "block_height": 202, + "coinbase_value_sats": 500000000, + "floor_formula": "3680 bytes/input * 1 sat/byte * 4x safety", + "min_output_sats": 14720, + "onchain_entitlements": [ + { + "order_key": "miner-a", + "p2mr_program_hex": "0101010101010101010101010101010101010101010101010101010101010101", + "recipient_id": "miner-a", + "weight": 250000000 + }, + { + "order_key": "miner-b", + "p2mr_program_hex": "0202020202020202020202020202020202020202020202020202020202020202", + "recipient_id": "miner-b", + "weight": 187500000 + }, + { + "order_key": "miner-c", + "p2mr_program_hex": "0303030303030303030303030303030303030303030303030303030303030303", + "recipient_id": "miner-c", + "weight": 62500000 + } + ], + "schema": "qbit.prism.payout-policy.v1" + } + } + }, + "expected_3xx": { + "ok": { + "counted_shares": [ + { + "counted_difficulty": 1000000, + "miner_id": "miner-a", + "share_seq": 4 + }, + { + "counted_difficulty": 1000000, + "miner_id": "miner-c", + "share_seq": 3 + }, + { + "counted_difficulty": 2000000, + "miner_id": "miner-b", + "share_seq": 2 + }, + { + "counted_difficulty": 2000000, + "miner_id": "miner-b", + "share_seq": 1 + } + ], + "counted_window_weight": 6000000, + "entitlements": [ + { + "order_key": "miner-a", + "p2mr_program_hex": "0101010101010101010101010101010101010101010101010101010101010101", + "recipient_id": "miner-a", + "weight": 1000000 + }, + { + "order_key": "miner-b", + "p2mr_program_hex": "0202020202020202020202020202020202020202020202020202020202020202", + "recipient_id": "miner-b", + "weight": 4000000 + }, + { + "order_key": "miner-c", + "p2mr_program_hex": "0303030303030303030303030303030303030303030303030303030303030303", + "recipient_id": "miner-c", + "weight": 1000000 + } + ], + "payout_policy_manifest": { + "accounts": [ + { + "action": "onchain", + "candidate_balance_sats": 83333334, + "carry_forward_balance_sats": 0, + "gross_amount_sats": 83333334, + "onchain_amount_sats": 83333334, + "order_key": "miner-a", + "p2mr_program_hex": "0101010101010101010101010101010101010101010101010101010101010101", + "prior_balance_sats": 0, + "recipient_id": "miner-a" + }, + { + "action": "onchain", + "candidate_balance_sats": 333333333, + "carry_forward_balance_sats": 0, + "gross_amount_sats": 333333333, + "onchain_amount_sats": 333333333, + "order_key": "miner-b", + "p2mr_program_hex": "0202020202020202020202020202020202020202020202020202020202020202", + "prior_balance_sats": 0, + "recipient_id": "miner-b" + }, + { + "action": "onchain", + "candidate_balance_sats": 83333333, + "carry_forward_balance_sats": 0, + "gross_amount_sats": 83333333, + "onchain_amount_sats": 83333333, + "order_key": "miner-c", + "p2mr_program_hex": "0303030303030303030303030303030303030303030303030303030303030303", + "prior_balance_sats": 0, + "recipient_id": "miner-c" + } + ], + "block_height": 202, + "coinbase_value_sats": 500000000, + "floor_formula": "3680 bytes/input * 1 sat/byte * 4x safety", + "min_output_sats": 14720, + "onchain_entitlements": [ + { + "order_key": "miner-a", + "p2mr_program_hex": "0101010101010101010101010101010101010101010101010101010101010101", + "recipient_id": "miner-a", + "weight": 83333334 + }, + { + "order_key": "miner-b", + "p2mr_program_hex": "0202020202020202020202020202020202020202020202020202020202020202", + "recipient_id": "miner-b", + "weight": 333333333 + }, + { + "order_key": "miner-c", + "p2mr_program_hex": "0303030303030303030303030303030303030303030303030303030303030303", + "recipient_id": "miner-c", + "weight": 83333333 + } + ], + "schema": "qbit.prism.payout-policy.v1" + } + } + }, + "input": { + "bundle_2xx": { + "found_block": { + "anchor_job_issued_at_ms": 9010, + "block_height": 202, + "coinbase_value_sats": 500000000, + "network_difficulty": 1000000 + }, + "policy": { + "p2mr_spend_input_bytes": 3680, + "safety_multiplier": 4, + "target_feerate_sats_per_byte": 1 + }, + "prior_balances": [], + "shares": [ + { + "accepted_at_ms": 9000, + "job_id": "job-9000", + "job_issued_at_ms": 9000, + "miner_id": "miner-b", + "network_difficulty": 1000000, + "ntime": 1800000000, + "order_key": "miner-b", + "p2mr_program_hex": "0202020202020202020202020202020202020202020202020202020202020202", + "share_difficulty": 2000000, + "share_id": "share-1", + "share_seq": 1, + "template_height": 100 + }, + { + "accepted_at_ms": 9001, + "job_id": "job-9001", + "job_issued_at_ms": 9001, + "miner_id": "miner-b", + "network_difficulty": 1000000, + "ntime": 1800000000, + "order_key": "miner-b", + "p2mr_program_hex": "0202020202020202020202020202020202020202020202020202020202020202", + "share_difficulty": 2000000, + "share_id": "share-2", + "share_seq": 2, + "template_height": 100 + }, + { + "accepted_at_ms": 9002, + "job_id": "job-9002", + "job_issued_at_ms": 9002, + "miner_id": "miner-c", + "network_difficulty": 1000000, + "ntime": 1800000000, + "order_key": "miner-c", + "p2mr_program_hex": "0303030303030303030303030303030303030303030303030303030303030303", + "share_difficulty": 1000000, + "share_id": "share-3", + "share_seq": 3, + "template_height": 100 + }, + { + "accepted_at_ms": 9003, + "job_id": "job-below-target", + "job_issued_at_ms": 9003, + "miner_id": "miner-a", + "network_difficulty": 1000000, + "ntime": 1800000000, + "order_key": "miner-a", + "p2mr_program_hex": "0101010101010101010101010101010101010101010101010101010101010101", + "share_difficulty": 4000000, + "share_id": "miner-a.rig:abababababababababababababababababababababababababababababababab", + "share_seq": 4, + "template_height": 200 + } + ] + }, + "bundle_3xx": { + "found_block": { + "anchor_job_issued_at_ms": 9010, + "block_height": 202, + "coinbase_value_sats": 500000000, + "network_difficulty": 1000000 + }, + "policy": { + "p2mr_spend_input_bytes": 3680, + "safety_multiplier": 4, + "target_feerate_sats_per_byte": 1 + }, + "prior_balances": [], + "shares": [ + { + "accepted_at_ms": 9000, + "job_id": "job-9000", + "job_issued_at_ms": 9000, + "miner_id": "miner-b", + "network_difficulty": 1000000, + "ntime": 1800000000, + "order_key": "miner-b", + "p2mr_program_hex": "0202020202020202020202020202020202020202020202020202020202020202", + "share_difficulty": 2000000, + "share_id": "share-1", + "share_seq": 1, + "template_height": 100 + }, + { + "accepted_at_ms": 9001, + "job_id": "job-9001", + "job_issued_at_ms": 9001, + "miner_id": "miner-b", + "network_difficulty": 1000000, + "ntime": 1800000000, + "order_key": "miner-b", + "p2mr_program_hex": "0202020202020202020202020202020202020202020202020202020202020202", + "share_difficulty": 2000000, + "share_id": "share-2", + "share_seq": 2, + "template_height": 100 + }, + { + "accepted_at_ms": 9002, + "job_id": "job-9002", + "job_issued_at_ms": 9002, + "miner_id": "miner-c", + "network_difficulty": 1000000, + "ntime": 1800000000, + "order_key": "miner-c", + "p2mr_program_hex": "0303030303030303030303030303030303030303030303030303030303030303", + "share_difficulty": 1000000, + "share_id": "share-3", + "share_seq": 3, + "template_height": 100 + }, + { + "accepted_at_ms": 9003, + "job_id": "job-below-target", + "job_issued_at_ms": 9003, + "miner_id": "miner-a", + "network_difficulty": 1000000, + "ntime": 1800000000, + "order_key": "miner-a", + "p2mr_program_hex": "0101010101010101010101010101010101010101010101010101010101010101", + "share_difficulty": 1000000, + "share_id": "miner-a.rig:abababababababababababababababababababababababababababababababab", + "share_seq": 4, + "template_height": 200 + } + ] + }, + "rule": { + "decision_2xx": { + "assigned_share_difficulty": 4000000, + "block_worthy": true, + "credit_share_on_accept": true, + "credited": true, + "credited_at": "node acceptance of the block; later confirmation adds nothing", + "credited_difficulty": 4000000, + "implemented_at": [ + "lab/prism/share_submission.py:360 classify_submit_work", + "lab/prism/share_writer.py:281 runtime.accepted_share_difficulty(context)", + "lab/prism/prism_coordinator.py:8410 accepted_share_difficulty", + "lab/prism/block_finalization.py:2027 append_accepted_share when credit_share_on_accept" + ], + "network_difficulty": 1000000, + "rejection": null, + "route": "synchronous_block" + }, + "decision_3xx": { + "credited": true, + "credited_at": "reconciliation's confirmation transaction", + "credited_difficulty": 1000000, + "implemented_at": [ + "crates/qbit-prism-server/src/coordinator.rs:1648-1654 network difficulty for a block-only proof", + "crates/qbit-prism-server/src/coordinator.rs:1698 deferred_share held with the block candidate", + "crates/qbit-prism-server/src/coordinator.rs:1726-1747 submit waits for the credit row or fails", + "crates/qbit-prism-server/src/ledger/blocks.rs:295 credit_deferred_share on reconciled confirmation" + ] + }, + "scenario": { + "block_pass": true, + "credited_share": { + "accepted_at_ms": 9003, + "job_issued_at_ms": 9003, + "share_id": "miner-a.rig:abababababababababababababababababababababababababababababababab", + "share_seq": 4 + }, + "ledger_shares": [ + { + "accepted_at_ms": 9000, + "job_id": "job-9000", + "job_issued_at_ms": 9000, + "miner_id": "miner-b", + "network_difficulty": 1000000, + "ntime": 1800000000, + "order_key": "miner-b", + "p2mr_program_hex": "0202020202020202020202020202020202020202020202020202020202020202", + "share_difficulty": 2000000, + "share_id": "share-1", + "share_seq": 1, + "template_height": 100 + }, + { + "accepted_at_ms": 9001, + "job_id": "job-9001", + "job_issued_at_ms": 9001, + "miner_id": "miner-b", + "network_difficulty": 1000000, + "ntime": 1800000000, + "order_key": "miner-b", + "p2mr_program_hex": "0202020202020202020202020202020202020202020202020202020202020202", + "share_difficulty": 2000000, + "share_id": "share-2", + "share_seq": 2, + "template_height": 100 + }, + { + "accepted_at_ms": 9002, + "job_id": "job-9002", + "job_issued_at_ms": 9002, + "miner_id": "miner-c", + "network_difficulty": 1000000, + "ntime": 1800000000, + "order_key": "miner-c", + "p2mr_program_hex": "0303030303030303030303030303030303030303030303030303030303030303", + "share_difficulty": 1000000, + "share_id": "share-3", + "share_seq": 3, + "template_height": 100 + } + ], + "network_target": 57896037716911750921221705069588091649609539881711309849342236841432341020672, + "next_block": { + "anchor_job_issued_at_ms": 9010, + "block_height": 202, + "coinbase_value_sats": 500000000, + "network_difficulty": 1000000 + }, + "node_outcome": "confirmed-after-reconciliation", + "share_pass": false, + "share_target": 14474009429227937730305426267397022912402384970427827462335559210358085255168, + "solver": { + "p2mr_program_hex": "0101010101010101010101010101010101010101010101010101010101010101", + "payout_address": "miner-a" + } + } + } + }, + "name": "block-only-proof-confirmed-after-reconciliation", + "topic": 9, + "why": "The node accepted the block but the durable acknowledgement was lost; reconciliation later finds it on the active chain." + }, + { + "d2_entry": "d2b-below-target-credit", + "entry_point": "bundle_payout", + "expected_2xx": { + "ok": { + "counted_shares": [ + { + "counted_difficulty": 4000000, + "miner_id": "miner-a", + "share_seq": 4 + }, + { + "counted_difficulty": 1000000, + "miner_id": "miner-c", + "share_seq": 3 + }, + { + "counted_difficulty": 2000000, + "miner_id": "miner-b", + "share_seq": 2 + }, + { + "counted_difficulty": 1000000, + "miner_id": "miner-b", + "share_seq": 1 + } + ], + "counted_window_weight": 8000000, + "entitlements": [ + { + "order_key": "miner-a", + "p2mr_program_hex": "0101010101010101010101010101010101010101010101010101010101010101", + "recipient_id": "miner-a", + "weight": 4000000 + }, + { + "order_key": "miner-b", + "p2mr_program_hex": "0202020202020202020202020202020202020202020202020202020202020202", + "recipient_id": "miner-b", + "weight": 3000000 + }, + { + "order_key": "miner-c", + "p2mr_program_hex": "0303030303030303030303030303030303030303030303030303030303030303", + "recipient_id": "miner-c", + "weight": 1000000 + } + ], + "payout_policy_manifest": { + "accounts": [ + { + "action": "onchain", + "candidate_balance_sats": 250000000, + "carry_forward_balance_sats": 0, + "gross_amount_sats": 250000000, + "onchain_amount_sats": 250000000, + "order_key": "miner-a", + "p2mr_program_hex": "0101010101010101010101010101010101010101010101010101010101010101", + "prior_balance_sats": 0, + "recipient_id": "miner-a" + }, + { + "action": "onchain", + "candidate_balance_sats": 187500000, + "carry_forward_balance_sats": 0, + "gross_amount_sats": 187500000, + "onchain_amount_sats": 187500000, + "order_key": "miner-b", + "p2mr_program_hex": "0202020202020202020202020202020202020202020202020202020202020202", + "prior_balance_sats": 0, + "recipient_id": "miner-b" + }, + { + "action": "onchain", + "candidate_balance_sats": 62500000, + "carry_forward_balance_sats": 0, + "gross_amount_sats": 62500000, + "onchain_amount_sats": 62500000, + "order_key": "miner-c", + "p2mr_program_hex": "0303030303030303030303030303030303030303030303030303030303030303", + "prior_balance_sats": 0, + "recipient_id": "miner-c" + } + ], + "block_height": 202, + "coinbase_value_sats": 500000000, + "floor_formula": "3680 bytes/input * 1 sat/byte * 4x safety", + "min_output_sats": 14720, + "onchain_entitlements": [ + { + "order_key": "miner-a", + "p2mr_program_hex": "0101010101010101010101010101010101010101010101010101010101010101", + "recipient_id": "miner-a", + "weight": 250000000 + }, + { + "order_key": "miner-b", + "p2mr_program_hex": "0202020202020202020202020202020202020202020202020202020202020202", + "recipient_id": "miner-b", + "weight": 187500000 + }, + { + "order_key": "miner-c", + "p2mr_program_hex": "0303030303030303030303030303030303030303030303030303030303030303", + "recipient_id": "miner-c", + "weight": 62500000 + } + ], + "schema": "qbit.prism.payout-policy.v1" + } + } + }, + "expected_3xx": { + "ok": { + "counted_shares": [ + { + "counted_difficulty": 1000000, + "miner_id": "miner-a", + "share_seq": 4 + }, + { + "counted_difficulty": 1000000, + "miner_id": "miner-c", + "share_seq": 3 + }, + { + "counted_difficulty": 2000000, + "miner_id": "miner-b", + "share_seq": 2 + }, + { + "counted_difficulty": 2000000, + "miner_id": "miner-b", + "share_seq": 1 + } + ], + "counted_window_weight": 6000000, + "entitlements": [ + { + "order_key": "miner-a", + "p2mr_program_hex": "0101010101010101010101010101010101010101010101010101010101010101", + "recipient_id": "miner-a", + "weight": 1000000 + }, + { + "order_key": "miner-b", + "p2mr_program_hex": "0202020202020202020202020202020202020202020202020202020202020202", + "recipient_id": "miner-b", + "weight": 4000000 + }, + { + "order_key": "miner-c", + "p2mr_program_hex": "0303030303030303030303030303030303030303030303030303030303030303", + "recipient_id": "miner-c", + "weight": 1000000 + } + ], + "payout_policy_manifest": { + "accounts": [ + { + "action": "onchain", + "candidate_balance_sats": 83333334, + "carry_forward_balance_sats": 0, + "gross_amount_sats": 83333334, + "onchain_amount_sats": 83333334, + "order_key": "miner-a", + "p2mr_program_hex": "0101010101010101010101010101010101010101010101010101010101010101", + "prior_balance_sats": 0, + "recipient_id": "miner-a" + }, + { + "action": "onchain", + "candidate_balance_sats": 333333333, + "carry_forward_balance_sats": 0, + "gross_amount_sats": 333333333, + "onchain_amount_sats": 333333333, + "order_key": "miner-b", + "p2mr_program_hex": "0202020202020202020202020202020202020202020202020202020202020202", + "prior_balance_sats": 0, + "recipient_id": "miner-b" + }, + { + "action": "onchain", + "candidate_balance_sats": 83333333, + "carry_forward_balance_sats": 0, + "gross_amount_sats": 83333333, + "onchain_amount_sats": 83333333, + "order_key": "miner-c", + "p2mr_program_hex": "0303030303030303030303030303030303030303030303030303030303030303", + "prior_balance_sats": 0, + "recipient_id": "miner-c" + } + ], + "block_height": 202, + "coinbase_value_sats": 500000000, + "floor_formula": "3680 bytes/input * 1 sat/byte * 4x safety", + "min_output_sats": 14720, + "onchain_entitlements": [ + { + "order_key": "miner-a", + "p2mr_program_hex": "0101010101010101010101010101010101010101010101010101010101010101", + "recipient_id": "miner-a", + "weight": 83333334 + }, + { + "order_key": "miner-b", + "p2mr_program_hex": "0202020202020202020202020202020202020202020202020202020202020202", + "recipient_id": "miner-b", + "weight": 333333333 + }, + { + "order_key": "miner-c", + "p2mr_program_hex": "0303030303030303030303030303030303030303030303030303030303030303", + "recipient_id": "miner-c", + "weight": 83333333 + } + ], + "schema": "qbit.prism.payout-policy.v1" + } + } + }, + "input": { + "bundle_2xx": { + "found_block": { + "anchor_job_issued_at_ms": 9010, + "block_height": 202, + "coinbase_value_sats": 500000000, + "network_difficulty": 1000000 + }, + "policy": { + "p2mr_spend_input_bytes": 3680, + "safety_multiplier": 4, + "target_feerate_sats_per_byte": 1 + }, + "prior_balances": [], + "shares": [ + { + "accepted_at_ms": 9000, + "job_id": "job-9000", + "job_issued_at_ms": 9000, + "miner_id": "miner-b", + "network_difficulty": 1000000, + "ntime": 1800000000, + "order_key": "miner-b", + "p2mr_program_hex": "0202020202020202020202020202020202020202020202020202020202020202", + "share_difficulty": 2000000, + "share_id": "share-1", + "share_seq": 1, + "template_height": 100 + }, + { + "accepted_at_ms": 9001, + "job_id": "job-9001", + "job_issued_at_ms": 9001, + "miner_id": "miner-b", + "network_difficulty": 1000000, + "ntime": 1800000000, + "order_key": "miner-b", + "p2mr_program_hex": "0202020202020202020202020202020202020202020202020202020202020202", + "share_difficulty": 2000000, + "share_id": "share-2", + "share_seq": 2, + "template_height": 100 + }, + { + "accepted_at_ms": 9002, + "job_id": "job-9002", + "job_issued_at_ms": 9002, + "miner_id": "miner-c", + "network_difficulty": 1000000, + "ntime": 1800000000, + "order_key": "miner-c", + "p2mr_program_hex": "0303030303030303030303030303030303030303030303030303030303030303", + "share_difficulty": 1000000, + "share_id": "share-3", + "share_seq": 3, + "template_height": 100 + }, + { + "accepted_at_ms": 9003, + "job_id": "job-below-target", + "job_issued_at_ms": 9003, + "miner_id": "miner-a", + "network_difficulty": 1000000, + "ntime": 1800000000, + "order_key": "miner-a", + "p2mr_program_hex": "0101010101010101010101010101010101010101010101010101010101010101", + "share_difficulty": 4000000, + "share_id": "miner-a.rig:abababababababababababababababababababababababababababababababab", + "share_seq": 4, + "template_height": 200 + } + ] + }, + "bundle_3xx": { + "found_block": { + "anchor_job_issued_at_ms": 9010, + "block_height": 202, + "coinbase_value_sats": 500000000, + "network_difficulty": 1000000 + }, + "policy": { + "p2mr_spend_input_bytes": 3680, + "safety_multiplier": 4, + "target_feerate_sats_per_byte": 1 + }, + "prior_balances": [], + "shares": [ + { + "accepted_at_ms": 9000, + "job_id": "job-9000", + "job_issued_at_ms": 9000, + "miner_id": "miner-b", + "network_difficulty": 1000000, + "ntime": 1800000000, + "order_key": "miner-b", + "p2mr_program_hex": "0202020202020202020202020202020202020202020202020202020202020202", + "share_difficulty": 2000000, + "share_id": "share-1", + "share_seq": 1, + "template_height": 100 + }, + { + "accepted_at_ms": 9001, + "job_id": "job-9001", + "job_issued_at_ms": 9001, + "miner_id": "miner-b", + "network_difficulty": 1000000, + "ntime": 1800000000, + "order_key": "miner-b", + "p2mr_program_hex": "0202020202020202020202020202020202020202020202020202020202020202", + "share_difficulty": 2000000, + "share_id": "share-2", + "share_seq": 2, + "template_height": 100 + }, + { + "accepted_at_ms": 9002, + "job_id": "job-9002", + "job_issued_at_ms": 9002, + "miner_id": "miner-c", + "network_difficulty": 1000000, + "ntime": 1800000000, + "order_key": "miner-c", + "p2mr_program_hex": "0303030303030303030303030303030303030303030303030303030303030303", + "share_difficulty": 1000000, + "share_id": "share-3", + "share_seq": 3, + "template_height": 100 + }, + { + "accepted_at_ms": 9003, + "job_id": "job-below-target", + "job_issued_at_ms": 9003, + "miner_id": "miner-a", + "network_difficulty": 1000000, + "ntime": 1800000000, + "order_key": "miner-a", + "p2mr_program_hex": "0101010101010101010101010101010101010101010101010101010101010101", + "share_difficulty": 1000000, + "share_id": "miner-a.rig:abababababababababababababababababababababababababababababababab", + "share_seq": 4, + "template_height": 200 + } + ] + }, + "rule": { + "decision_2xx": { + "assigned_share_difficulty": 4000000, + "block_worthy": true, + "credit_share_on_accept": true, + "credited": true, + "credited_at": "node acceptance; the share row survives the disconnect", + "credited_difficulty": 4000000, + "implemented_at": [ + "lab/prism/share_submission.py:360 classify_submit_work", + "lab/prism/share_writer.py:281 runtime.accepted_share_difficulty(context)", + "lab/prism/prism_coordinator.py:8410 accepted_share_difficulty", + "lab/prism/block_finalization.py:2027 append_accepted_share when credit_share_on_accept", + "lab/prism/accepted_preview_telemetry.py:144 no reorg path updates or deletes qbit_share_ledger" + ], + "network_difficulty": 1000000, + "rejection": null, + "route": "synchronous_block" + }, + "decision_3xx": { + "credited": true, + "credited_at": "confirmation transaction; the share row survives the later disconnect", + "credited_difficulty": 1000000, + "implemented_at": [ + "crates/qbit-prism-server/src/coordinator.rs:1648-1654 network difficulty for a block-only proof", + "crates/qbit-prism-server/src/coordinator.rs:1698 deferred_share held with the block candidate", + "crates/qbit-prism-server/src/coordinator.rs:1726-1747 submit waits for the credit row or fails", + "crates/qbit-prism-server/src/ledger/blocks.rs:200 credit_deferred_share on confirmation; no server statement updates or deletes qbit_share_ledger" + ] + }, + "scenario": { + "block_pass": true, + "credited_share": { + "accepted_at_ms": 9003, + "job_issued_at_ms": 9003, + "share_id": "miner-a.rig:abababababababababababababababababababababababababababababababab", + "share_seq": 4 + }, + "ledger_shares": [ + { + "accepted_at_ms": 9000, + "job_id": "job-9000", + "job_issued_at_ms": 9000, + "miner_id": "miner-b", + "network_difficulty": 1000000, + "ntime": 1800000000, + "order_key": "miner-b", + "p2mr_program_hex": "0202020202020202020202020202020202020202020202020202020202020202", + "share_difficulty": 2000000, + "share_id": "share-1", + "share_seq": 1, + "template_height": 100 + }, + { + "accepted_at_ms": 9001, + "job_id": "job-9001", + "job_issued_at_ms": 9001, + "miner_id": "miner-b", + "network_difficulty": 1000000, + "ntime": 1800000000, + "order_key": "miner-b", + "p2mr_program_hex": "0202020202020202020202020202020202020202020202020202020202020202", + "share_difficulty": 2000000, + "share_id": "share-2", + "share_seq": 2, + "template_height": 100 + }, + { + "accepted_at_ms": 9002, + "job_id": "job-9002", + "job_issued_at_ms": 9002, + "miner_id": "miner-c", + "network_difficulty": 1000000, + "ntime": 1800000000, + "order_key": "miner-c", + "p2mr_program_hex": "0303030303030303030303030303030303030303030303030303030303030303", + "share_difficulty": 1000000, + "share_id": "share-3", + "share_seq": 3, + "template_height": 100 + } + ], + "network_target": 57896037716911750921221705069588091649609539881711309849342236841432341020672, + "next_block": { + "anchor_job_issued_at_ms": 9010, + "block_height": 202, + "coinbase_value_sats": 500000000, + "network_difficulty": 1000000 + }, + "node_outcome": "reorged-after-acceptance", + "share_pass": false, + "share_target": 14474009429227937730305426267397022912402384970427827462335559210358085255168, + "solver": { + "p2mr_program_hex": "0101010101010101010101010101010101010101010101010101010101010101", + "payout_address": "miner-a" + } + } + } + }, + "name": "block-only-proof-reorged-after-acceptance", + "topic": 9, + "why": "The block was accepted and credited, then disconnected by a reorg; the next block's window is built afterwards." + }, + { + "entry_point": "bundle_payout", + "expected": { + "ok": { + "counted_shares": [ + { + "counted_difficulty": 1000000, + "miner_id": "miner-c", + "share_seq": 3 + }, + { + "counted_difficulty": 2000000, + "miner_id": "miner-b", + "share_seq": 2 + }, + { + "counted_difficulty": 2000000, + "miner_id": "miner-b", + "share_seq": 1 + } + ], + "counted_window_weight": 5000000, + "entitlements": [ + { + "order_key": "miner-b", + "p2mr_program_hex": "0202020202020202020202020202020202020202020202020202020202020202", + "recipient_id": "miner-b", + "weight": 4000000 + }, + { + "order_key": "miner-c", + "p2mr_program_hex": "0303030303030303030303030303030303030303030303030303030303030303", + "recipient_id": "miner-c", + "weight": 1000000 + } + ], + "payout_policy_manifest": { + "accounts": [ + { + "action": "onchain", + "candidate_balance_sats": 400000000, + "carry_forward_balance_sats": 0, + "gross_amount_sats": 400000000, + "onchain_amount_sats": 400000000, + "order_key": "miner-b", + "p2mr_program_hex": "0202020202020202020202020202020202020202020202020202020202020202", + "prior_balance_sats": 0, + "recipient_id": "miner-b" + }, + { + "action": "onchain", + "candidate_balance_sats": 100000000, + "carry_forward_balance_sats": 0, + "gross_amount_sats": 100000000, + "onchain_amount_sats": 100000000, + "order_key": "miner-c", + "p2mr_program_hex": "0303030303030303030303030303030303030303030303030303030303030303", + "prior_balance_sats": 0, + "recipient_id": "miner-c" + } + ], + "block_height": 202, + "coinbase_value_sats": 500000000, + "floor_formula": "3680 bytes/input * 1 sat/byte * 4x safety", + "min_output_sats": 14720, + "onchain_entitlements": [ + { + "order_key": "miner-b", + "p2mr_program_hex": "0202020202020202020202020202020202020202020202020202020202020202", + "recipient_id": "miner-b", + "weight": 400000000 + }, + { + "order_key": "miner-c", + "p2mr_program_hex": "0303030303030303030303030303030303030303030303030303030303030303", + "recipient_id": "miner-c", + "weight": 100000000 + } + ], + "schema": "qbit.prism.payout-policy.v1" + } + } + }, + "input": { + "bundle_2xx": { + "found_block": { + "anchor_job_issued_at_ms": 9010, + "block_height": 202, + "coinbase_value_sats": 500000000, + "network_difficulty": 1000000 + }, + "policy": { + "p2mr_spend_input_bytes": 3680, + "safety_multiplier": 4, + "target_feerate_sats_per_byte": 1 + }, + "prior_balances": [], + "shares": [ + { + "accepted_at_ms": 9000, + "job_id": "job-9000", + "job_issued_at_ms": 9000, + "miner_id": "miner-b", + "network_difficulty": 1000000, + "ntime": 1800000000, + "order_key": "miner-b", + "p2mr_program_hex": "0202020202020202020202020202020202020202020202020202020202020202", + "share_difficulty": 2000000, + "share_id": "share-1", + "share_seq": 1, + "template_height": 100 + }, + { + "accepted_at_ms": 9001, + "job_id": "job-9001", + "job_issued_at_ms": 9001, + "miner_id": "miner-b", + "network_difficulty": 1000000, + "ntime": 1800000000, + "order_key": "miner-b", + "p2mr_program_hex": "0202020202020202020202020202020202020202020202020202020202020202", + "share_difficulty": 2000000, + "share_id": "share-2", + "share_seq": 2, + "template_height": 100 + }, + { + "accepted_at_ms": 9002, + "job_id": "job-9002", + "job_issued_at_ms": 9002, + "miner_id": "miner-c", + "network_difficulty": 1000000, + "ntime": 1800000000, + "order_key": "miner-c", + "p2mr_program_hex": "0303030303030303030303030303030303030303030303030303030303030303", + "share_difficulty": 1000000, + "share_id": "share-3", + "share_seq": 3, + "template_height": 100 + } + ] + }, + "bundle_3xx": { + "found_block": { + "anchor_job_issued_at_ms": 9010, + "block_height": 202, + "coinbase_value_sats": 500000000, + "network_difficulty": 1000000 + }, + "policy": { + "p2mr_spend_input_bytes": 3680, + "safety_multiplier": 4, + "target_feerate_sats_per_byte": 1 + }, + "prior_balances": [], + "shares": [ + { + "accepted_at_ms": 9000, + "job_id": "job-9000", + "job_issued_at_ms": 9000, + "miner_id": "miner-b", + "network_difficulty": 1000000, + "ntime": 1800000000, + "order_key": "miner-b", + "p2mr_program_hex": "0202020202020202020202020202020202020202020202020202020202020202", + "share_difficulty": 2000000, + "share_id": "share-1", + "share_seq": 1, + "template_height": 100 + }, + { + "accepted_at_ms": 9001, + "job_id": "job-9001", + "job_issued_at_ms": 9001, + "miner_id": "miner-b", + "network_difficulty": 1000000, + "ntime": 1800000000, + "order_key": "miner-b", + "p2mr_program_hex": "0202020202020202020202020202020202020202020202020202020202020202", + "share_difficulty": 2000000, + "share_id": "share-2", + "share_seq": 2, + "template_height": 100 + }, + { + "accepted_at_ms": 9002, + "job_id": "job-9002", + "job_issued_at_ms": 9002, + "miner_id": "miner-c", + "network_difficulty": 1000000, + "ntime": 1800000000, + "order_key": "miner-c", + "p2mr_program_hex": "0303030303030303030303030303030303030303030303030303030303030303", + "share_difficulty": 1000000, + "share_id": "share-3", + "share_seq": 3, + "template_height": 100 + } + ] + }, + "rule": { + "decision_2xx": { + "assigned_share_difficulty": 4000000, + "block_worthy": true, + "credit_share_on_accept": true, + "credited": false, + "credited_at": null, + "credited_difficulty": null, + "implemented_at": [ + "lab/prism/share_submission.py:360 classify_submit_work", + "lab/prism/share_writer.py:281 runtime.accepted_share_difficulty(context)", + "lab/prism/prism_coordinator.py:8410 accepted_share_difficulty", + "lab/prism/prism_coordinator.py:8221 block_landed false rejects low difficulty share" + ], + "network_difficulty": 1000000, + "rejection": null, + "route": "synchronous_block" + }, + "decision_3xx": { + "credited": false, + "credited_at": "never: the submission fails with block-only proof was not accepted on the active chain", + "credited_difficulty": null, + "implemented_at": [ + "crates/qbit-prism-server/src/coordinator.rs:1648-1654 network difficulty for a block-only proof", + "crates/qbit-prism-server/src/coordinator.rs:1698 deferred_share held with the block candidate", + "crates/qbit-prism-server/src/coordinator.rs:1726-1747 submit waits for the credit row or fails" + ] + }, + "scenario": { + "block_pass": true, + "credited_share": { + "accepted_at_ms": 9003, + "job_issued_at_ms": 9003, + "share_id": "miner-a.rig:abababababababababababababababababababababababababababababababab", + "share_seq": 4 + }, + "ledger_shares": [ + { + "accepted_at_ms": 9000, + "job_id": "job-9000", + "job_issued_at_ms": 9000, + "miner_id": "miner-b", + "network_difficulty": 1000000, + "ntime": 1800000000, + "order_key": "miner-b", + "p2mr_program_hex": "0202020202020202020202020202020202020202020202020202020202020202", + "share_difficulty": 2000000, + "share_id": "share-1", + "share_seq": 1, + "template_height": 100 + }, + { + "accepted_at_ms": 9001, + "job_id": "job-9001", + "job_issued_at_ms": 9001, + "miner_id": "miner-b", + "network_difficulty": 1000000, + "ntime": 1800000000, + "order_key": "miner-b", + "p2mr_program_hex": "0202020202020202020202020202020202020202020202020202020202020202", + "share_difficulty": 2000000, + "share_id": "share-2", + "share_seq": 2, + "template_height": 100 + }, + { + "accepted_at_ms": 9002, + "job_id": "job-9002", + "job_issued_at_ms": 9002, + "miner_id": "miner-c", + "network_difficulty": 1000000, + "ntime": 1800000000, + "order_key": "miner-c", + "p2mr_program_hex": "0303030303030303030303030303030303030303030303030303030303030303", + "share_difficulty": 1000000, + "share_id": "share-3", + "share_seq": 3, + "template_height": 100 + } + ], + "network_target": 57896037716911750921221705069588091649609539881711309849342236841432341020672, + "next_block": { + "anchor_job_issued_at_ms": 9010, + "block_height": 202, + "coinbase_value_sats": 500000000, + "network_difficulty": 1000000 + }, + "node_outcome": "rejected", + "share_pass": false, + "share_target": 14474009429227937730305426267397022912402384970427827462335559210358085255168, + "solver": { + "p2mr_program_hex": "0101010101010101010101010101010101010101010101010101010101010101", + "payout_address": "miner-a" + } + } + } + }, + "name": "block-only-proof-rejected", + "topic": 9, + "why": "The node rejects the block, or it is never on the active chain: no credit on either version." + }, + { + "entry_point": "bundle_payout", + "expected": { + "ok": { + "counted_shares": [ + { + "counted_difficulty": 4000000, + "miner_id": "miner-a", + "share_seq": 4 + }, + { + "counted_difficulty": 1000000, + "miner_id": "miner-c", + "share_seq": 3 + }, + { + "counted_difficulty": 2000000, + "miner_id": "miner-b", + "share_seq": 2 + }, + { + "counted_difficulty": 1000000, + "miner_id": "miner-b", + "share_seq": 1 + } + ], + "counted_window_weight": 8000000, + "entitlements": [ + { + "order_key": "miner-a", + "p2mr_program_hex": "0101010101010101010101010101010101010101010101010101010101010101", + "recipient_id": "miner-a", + "weight": 4000000 + }, + { + "order_key": "miner-b", + "p2mr_program_hex": "0202020202020202020202020202020202020202020202020202020202020202", + "recipient_id": "miner-b", + "weight": 3000000 + }, + { + "order_key": "miner-c", + "p2mr_program_hex": "0303030303030303030303030303030303030303030303030303030303030303", + "recipient_id": "miner-c", + "weight": 1000000 + } + ], + "payout_policy_manifest": { + "accounts": [ + { + "action": "onchain", + "candidate_balance_sats": 250000000, + "carry_forward_balance_sats": 0, + "gross_amount_sats": 250000000, + "onchain_amount_sats": 250000000, + "order_key": "miner-a", + "p2mr_program_hex": "0101010101010101010101010101010101010101010101010101010101010101", + "prior_balance_sats": 0, + "recipient_id": "miner-a" + }, + { + "action": "onchain", + "candidate_balance_sats": 187500000, + "carry_forward_balance_sats": 0, + "gross_amount_sats": 187500000, + "onchain_amount_sats": 187500000, + "order_key": "miner-b", + "p2mr_program_hex": "0202020202020202020202020202020202020202020202020202020202020202", + "prior_balance_sats": 0, + "recipient_id": "miner-b" + }, + { + "action": "onchain", + "candidate_balance_sats": 62500000, + "carry_forward_balance_sats": 0, + "gross_amount_sats": 62500000, + "onchain_amount_sats": 62500000, + "order_key": "miner-c", + "p2mr_program_hex": "0303030303030303030303030303030303030303030303030303030303030303", + "prior_balance_sats": 0, + "recipient_id": "miner-c" + } + ], + "block_height": 202, + "coinbase_value_sats": 500000000, + "floor_formula": "3680 bytes/input * 1 sat/byte * 4x safety", + "min_output_sats": 14720, + "onchain_entitlements": [ + { + "order_key": "miner-a", + "p2mr_program_hex": "0101010101010101010101010101010101010101010101010101010101010101", + "recipient_id": "miner-a", + "weight": 250000000 + }, + { + "order_key": "miner-b", + "p2mr_program_hex": "0202020202020202020202020202020202020202020202020202020202020202", + "recipient_id": "miner-b", + "weight": 187500000 + }, + { + "order_key": "miner-c", + "p2mr_program_hex": "0303030303030303030303030303030303030303030303030303030303030303", + "recipient_id": "miner-c", + "weight": 62500000 + } + ], + "schema": "qbit.prism.payout-policy.v1" + } + } + }, + "input": { + "bundle_2xx": { + "found_block": { + "anchor_job_issued_at_ms": 9010, + "block_height": 202, + "coinbase_value_sats": 500000000, + "network_difficulty": 1000000 + }, + "policy": { + "p2mr_spend_input_bytes": 3680, + "safety_multiplier": 4, + "target_feerate_sats_per_byte": 1 + }, + "prior_balances": [], + "shares": [ + { + "accepted_at_ms": 9000, + "job_id": "job-9000", + "job_issued_at_ms": 9000, + "miner_id": "miner-b", + "network_difficulty": 1000000, + "ntime": 1800000000, + "order_key": "miner-b", + "p2mr_program_hex": "0202020202020202020202020202020202020202020202020202020202020202", + "share_difficulty": 2000000, + "share_id": "share-1", + "share_seq": 1, + "template_height": 100 + }, + { + "accepted_at_ms": 9001, + "job_id": "job-9001", + "job_issued_at_ms": 9001, + "miner_id": "miner-b", + "network_difficulty": 1000000, + "ntime": 1800000000, + "order_key": "miner-b", + "p2mr_program_hex": "0202020202020202020202020202020202020202020202020202020202020202", + "share_difficulty": 2000000, + "share_id": "share-2", + "share_seq": 2, + "template_height": 100 + }, + { + "accepted_at_ms": 9002, + "job_id": "job-9002", + "job_issued_at_ms": 9002, + "miner_id": "miner-c", + "network_difficulty": 1000000, + "ntime": 1800000000, + "order_key": "miner-c", + "p2mr_program_hex": "0303030303030303030303030303030303030303030303030303030303030303", + "share_difficulty": 1000000, + "share_id": "share-3", + "share_seq": 3, + "template_height": 100 + }, + { + "accepted_at_ms": 9003, + "job_id": "job-below-target", + "job_issued_at_ms": 9003, + "miner_id": "miner-a", + "network_difficulty": 1000000, + "ntime": 1800000000, + "order_key": "miner-a", + "p2mr_program_hex": "0101010101010101010101010101010101010101010101010101010101010101", + "share_difficulty": 4000000, + "share_id": "miner-a.rig:abababababababababababababababababababababababababababababababab", + "share_seq": 4, + "template_height": 200 + } + ] + }, + "bundle_3xx": { + "found_block": { + "anchor_job_issued_at_ms": 9010, + "block_height": 202, + "coinbase_value_sats": 500000000, + "network_difficulty": 1000000 + }, + "policy": { + "p2mr_spend_input_bytes": 3680, + "safety_multiplier": 4, + "target_feerate_sats_per_byte": 1 + }, + "prior_balances": [], + "shares": [ + { + "accepted_at_ms": 9000, + "job_id": "job-9000", + "job_issued_at_ms": 9000, + "miner_id": "miner-b", + "network_difficulty": 1000000, + "ntime": 1800000000, + "order_key": "miner-b", + "p2mr_program_hex": "0202020202020202020202020202020202020202020202020202020202020202", + "share_difficulty": 2000000, + "share_id": "share-1", + "share_seq": 1, + "template_height": 100 + }, + { + "accepted_at_ms": 9001, + "job_id": "job-9001", + "job_issued_at_ms": 9001, + "miner_id": "miner-b", + "network_difficulty": 1000000, + "ntime": 1800000000, + "order_key": "miner-b", + "p2mr_program_hex": "0202020202020202020202020202020202020202020202020202020202020202", + "share_difficulty": 2000000, + "share_id": "share-2", + "share_seq": 2, + "template_height": 100 + }, + { + "accepted_at_ms": 9002, + "job_id": "job-9002", + "job_issued_at_ms": 9002, + "miner_id": "miner-c", + "network_difficulty": 1000000, + "ntime": 1800000000, + "order_key": "miner-c", + "p2mr_program_hex": "0303030303030303030303030303030303030303030303030303030303030303", + "share_difficulty": 1000000, + "share_id": "share-3", + "share_seq": 3, + "template_height": 100 + }, + { + "accepted_at_ms": 9003, + "job_id": "job-below-target", + "job_issued_at_ms": 9003, + "miner_id": "miner-a", + "network_difficulty": 1000000, + "ntime": 1800000000, + "order_key": "miner-a", + "p2mr_program_hex": "0101010101010101010101010101010101010101010101010101010101010101", + "share_difficulty": 4000000, + "share_id": "miner-a.rig:abababababababababababababababababababababababababababababababab", + "share_seq": 4, + "template_height": 200 + } + ] + }, + "rule": { + "decision_2xx": { + "assigned_share_difficulty": 4000000, + "block_worthy": true, + "credit_share_on_accept": false, + "credited": true, + "credited_at": "share acceptance (share passed its target)", + "credited_difficulty": 4000000, + "implemented_at": [ + "lab/prism/share_submission.py:360 classify_submit_work", + "lab/prism/share_writer.py:281 runtime.accepted_share_difficulty(context)", + "lab/prism/prism_coordinator.py:8410 accepted_share_difficulty" + ], + "network_difficulty": 1000000, + "rejection": null, + "route": "async_block" + }, + "decision_3xx": { + "credited": true, + "credited_at": "share acceptance (share passed its target)", + "credited_difficulty": 4000000, + "implemented_at": [ + "crates/qbit-prism-server/src/coordinator.rs:1649-1651 share target difficulty when the share passes" + ] + }, + "scenario": { + "block_pass": true, + "credited_share": { + "accepted_at_ms": 9003, + "job_issued_at_ms": 9003, + "share_id": "miner-a.rig:abababababababababababababababababababababababababababababababab", + "share_seq": 4 + }, + "ledger_shares": [ + { + "accepted_at_ms": 9000, + "job_id": "job-9000", + "job_issued_at_ms": 9000, + "miner_id": "miner-b", + "network_difficulty": 1000000, + "ntime": 1800000000, + "order_key": "miner-b", + "p2mr_program_hex": "0202020202020202020202020202020202020202020202020202020202020202", + "share_difficulty": 2000000, + "share_id": "share-1", + "share_seq": 1, + "template_height": 100 + }, + { + "accepted_at_ms": 9001, + "job_id": "job-9001", + "job_issued_at_ms": 9001, + "miner_id": "miner-b", + "network_difficulty": 1000000, + "ntime": 1800000000, + "order_key": "miner-b", + "p2mr_program_hex": "0202020202020202020202020202020202020202020202020202020202020202", + "share_difficulty": 2000000, + "share_id": "share-2", + "share_seq": 2, + "template_height": 100 + }, + { + "accepted_at_ms": 9002, + "job_id": "job-9002", + "job_issued_at_ms": 9002, + "miner_id": "miner-c", + "network_difficulty": 1000000, + "ntime": 1800000000, + "order_key": "miner-c", + "p2mr_program_hex": "0303030303030303030303030303030303030303030303030303030303030303", + "share_difficulty": 1000000, + "share_id": "share-3", + "share_seq": 3, + "template_height": 100 + } + ], + "network_target": 57896037716911750921221705069588091649609539881711309849342236841432341020672, + "next_block": { + "anchor_job_issued_at_ms": 9010, + "block_height": 202, + "coinbase_value_sats": 500000000, + "network_difficulty": 1000000 + }, + "node_outcome": "accepted", + "share_pass": true, + "share_target": 14474009429227937730305426267397022912402384970427827462335559210358085255168, + "solver": { + "p2mr_program_hex": "0101010101010101010101010101010101010101010101010101010101010101", + "payout_address": "miner-a" + } + } + } + }, + "name": "share-and-block-proof-control", + "topic": 9, + "why": "Control: the hash also meets the share target, so both versions credit the assigned share difficulty." + } + ], + "produced_by": "PYTHONDONTWRITEBYTECODE=1 PYTHONPATH=. python3 crates/qbit-prism/examples/export_rule_decisions.py | cargo run --locked -q -p qbit-prism --example export_money_path_vectors -- vectors-out", + "schema": "qbit.prism.money-path-vectors.v1", + "source_commit": "504846cc0b72e8f86ed17f896d4ccbbe196a31dc", + "topics": [ + { + "id": 9, + "name": "below-target block credit" + } + ] +} diff --git a/crates/qbit-prism/fixtures/vectors/bootstrap_transition.json b/crates/qbit-prism/fixtures/vectors/bootstrap_transition.json new file mode 100644 index 000000000..d660d0a3c --- /dev/null +++ b/crates/qbit-prism/fixtures/vectors/bootstrap_transition.json @@ -0,0 +1,1030 @@ +{ + "cases": [ + { + "d2_entry": "d2a-bootstrap-pooling", + "entry_point": "bundle_payout", + "expected_2xx": { + "ok": { + "counted_shares": [ + { + "counted_difficulty": 100, + "miner_id": "miner-b", + "share_seq": 1 + } + ], + "counted_window_weight": 100, + "entitlements": [ + { + "order_key": "miner-b", + "p2mr_program_hex": "0202020202020202020202020202020202020202020202020202020202020202", + "recipient_id": "miner-b", + "weight": 100 + } + ], + "payout_policy_manifest": { + "accounts": [ + { + "action": "onchain", + "candidate_balance_sats": 500000000, + "carry_forward_balance_sats": 0, + "gross_amount_sats": 500000000, + "onchain_amount_sats": 500000000, + "order_key": "miner-b", + "p2mr_program_hex": "0202020202020202020202020202020202020202020202020202020202020202", + "prior_balance_sats": 0, + "recipient_id": "miner-b" + } + ], + "block_height": 101, + "coinbase_value_sats": 500000000, + "floor_formula": "3680 bytes/input * 1 sat/byte * 4x safety", + "min_output_sats": 14720, + "onchain_entitlements": [ + { + "order_key": "miner-b", + "p2mr_program_hex": "0202020202020202020202020202020202020202020202020202020202020202", + "recipient_id": "miner-b", + "weight": 500000000 + } + ], + "schema": "qbit.prism.payout-policy.v1" + } + } + }, + "expected_3xx": { + "ok": { + "counted_shares": [ + { + "counted_difficulty": 20, + "miner_id": "miner-b", + "share_seq": 2 + }, + { + "counted_difficulty": 30, + "miner_id": "miner-a", + "share_seq": 1 + } + ], + "counted_window_weight": 50, + "entitlements": [ + { + "order_key": "miner-a", + "p2mr_program_hex": "0101010101010101010101010101010101010101010101010101010101010101", + "recipient_id": "miner-a", + "weight": 30 + }, + { + "order_key": "miner-b", + "p2mr_program_hex": "0202020202020202020202020202020202020202020202020202020202020202", + "recipient_id": "miner-b", + "weight": 20 + } + ], + "payout_policy_manifest": { + "accounts": [ + { + "action": "onchain", + "candidate_balance_sats": 300000000, + "carry_forward_balance_sats": 0, + "gross_amount_sats": 300000000, + "onchain_amount_sats": 300000000, + "order_key": "miner-a", + "p2mr_program_hex": "0101010101010101010101010101010101010101010101010101010101010101", + "prior_balance_sats": 0, + "recipient_id": "miner-a" + }, + { + "action": "onchain", + "candidate_balance_sats": 200000000, + "carry_forward_balance_sats": 0, + "gross_amount_sats": 200000000, + "onchain_amount_sats": 200000000, + "order_key": "miner-b", + "p2mr_program_hex": "0202020202020202020202020202020202020202020202020202020202020202", + "prior_balance_sats": 0, + "recipient_id": "miner-b" + } + ], + "block_height": 101, + "coinbase_value_sats": 500000000, + "floor_formula": "3680 bytes/input * 1 sat/byte * 4x safety", + "min_output_sats": 14720, + "onchain_entitlements": [ + { + "order_key": "miner-a", + "p2mr_program_hex": "0101010101010101010101010101010101010101010101010101010101010101", + "recipient_id": "miner-a", + "weight": 300000000 + }, + { + "order_key": "miner-b", + "p2mr_program_hex": "0202020202020202020202020202020202020202020202020202020202020202", + "recipient_id": "miner-b", + "weight": 200000000 + } + ], + "schema": "qbit.prism.payout-policy.v1" + } + } + }, + "input": { + "bundle_2xx": { + "found_block": { + "anchor_job_issued_at_ms": 5000, + "block_height": 101, + "coinbase_value_sats": 500000000, + "network_difficulty": 100 + }, + "policy": { + "p2mr_spend_input_bytes": 3680, + "safety_multiplier": 4, + "target_feerate_sats_per_byte": 1 + }, + "prior_balances": [], + "shares": [ + { + "accepted_at_ms": 5000, + "job_id": "bootstrap-job", + "job_issued_at_ms": 5000, + "miner_id": "miner-b", + "network_difficulty": 100, + "ntime": 1800000000, + "order_key": "miner-b", + "p2mr_program_hex": "0202020202020202020202020202020202020202020202020202020202020202", + "share_difficulty": 100, + "share_id": "bootstrap-share", + "share_seq": 1, + "template_height": 100 + } + ] + }, + "bundle_3xx": { + "found_block": { + "anchor_job_issued_at_ms": 5000, + "block_height": 101, + "coinbase_value_sats": 500000000, + "network_difficulty": 100 + }, + "policy": { + "p2mr_spend_input_bytes": 3680, + "safety_multiplier": 4, + "target_feerate_sats_per_byte": 1 + }, + "prior_balances": [], + "shares": [ + { + "accepted_at_ms": 4000, + "job_id": "job-4000", + "job_issued_at_ms": 4000, + "miner_id": "miner-a", + "network_difficulty": 100, + "ntime": 1800000000, + "order_key": "miner-a", + "p2mr_program_hex": "0101010101010101010101010101010101010101010101010101010101010101", + "share_difficulty": 30, + "share_id": "share-1", + "share_seq": 1, + "template_height": 100 + }, + { + "accepted_at_ms": 4001, + "job_id": "job-4001", + "job_issued_at_ms": 4001, + "miner_id": "miner-b", + "network_difficulty": 100, + "ntime": 1800000000, + "order_key": "miner-b", + "p2mr_program_hex": "0202020202020202020202020202020202020202020202020202020202020202", + "share_difficulty": 20, + "share_id": "share-2", + "share_seq": 2, + "template_height": 100 + } + ] + }, + "rule": { + "decision_2xx": { + "accepted_share_count": 2, + "bundle_built_by": "lab/prism/job_bundle.py:3550 build_collection_bundle (executed)", + "distinct_miner_count": 2, + "implemented_at": [ + "lab/prism/prism_coordinator.py:7778 pool_readiness_latched", + "lab/prism/job_bundle.py:2132 _job_bundle_mode", + "lab/prism/job_bundle.py:3589 prior_balances=[] in build_collection_bundle" + ], + "job_bundle_mode": "collection", + "pool_readiness_latched": false + }, + "decision_3xx": { + "bootstrap": false, + "bundle_built_by": "ledger snapshot window (crates/qbit-prism-server/src/coordinator.rs:747)", + "implemented_at": [ + "crates/qbit-prism-server/src/coordinator.rs:556 solver-only bundle only when snapshot.shares is empty", + "crates/qbit-prism-server/src/coordinator.rs:759,773 snapshot.prior_balances passed in bootstrap" + ], + "prior_balances_kept": true + }, + "scenario": { + "anchor_ms": 5000, + "ledger_shares": [ + { + "accepted_at_ms": 4000, + "job_id": "job-4000", + "job_issued_at_ms": 4000, + "miner_id": "miner-a", + "network_difficulty": 100, + "ntime": 1800000000, + "order_key": "miner-a", + "p2mr_program_hex": "0101010101010101010101010101010101010101010101010101010101010101", + "share_difficulty": 30, + "share_id": "share-1", + "share_seq": 1, + "template_height": 100 + }, + { + "accepted_at_ms": 4001, + "job_id": "job-4001", + "job_issued_at_ms": 4001, + "miner_id": "miner-b", + "network_difficulty": 100, + "ntime": 1800000000, + "order_key": "miner-b", + "p2mr_program_hex": "0202020202020202020202020202020202020202020202020202020202020202", + "share_difficulty": 20, + "share_id": "share-2", + "share_seq": 2, + "template_height": 100 + } + ], + "min_ready_miners": 3, + "network_difficulty": 100, + "prior_balances": [], + "solver": { + "p2mr_program_hex": "0202020202020202020202020202020202020202020202020202020202020202", + "payout_address": "miner-b" + }, + "template": { + "coinbasevalue": 500000000, + "curtime": 1800000000, + "height": 101, + "previousblockhash": "0000000000000000000000000000000000000000000000000000000000000000" + } + } + } + }, + "name": "below-gate-with-other-miners-shares", + "topic": 8, + "why": "Two distinct miners have accepted shares, below the default readiness gate of three. 2.x.x issues a collection job that pays the solver the whole coinbase; 3.x.x uses the normal window as soon as any share exists." + }, + { + "entry_point": "bundle_payout", + "expected": { + "ok": { + "counted_shares": [ + { + "counted_difficulty": 50, + "miner_id": "miner-c", + "share_seq": 3 + }, + { + "counted_difficulty": 20, + "miner_id": "miner-b", + "share_seq": 2 + }, + { + "counted_difficulty": 30, + "miner_id": "miner-a", + "share_seq": 1 + } + ], + "counted_window_weight": 100, + "entitlements": [ + { + "order_key": "miner-a", + "p2mr_program_hex": "0101010101010101010101010101010101010101010101010101010101010101", + "recipient_id": "miner-a", + "weight": 30 + }, + { + "order_key": "miner-b", + "p2mr_program_hex": "0202020202020202020202020202020202020202020202020202020202020202", + "recipient_id": "miner-b", + "weight": 20 + }, + { + "order_key": "miner-c", + "p2mr_program_hex": "0303030303030303030303030303030303030303030303030303030303030303", + "recipient_id": "miner-c", + "weight": 50 + } + ], + "payout_policy_manifest": { + "accounts": [ + { + "action": "onchain", + "candidate_balance_sats": 150000000, + "carry_forward_balance_sats": 6000, + "gross_amount_sats": 150000000, + "onchain_amount_sats": 149994000, + "order_key": "miner-a", + "p2mr_program_hex": "0101010101010101010101010101010101010101010101010101010101010101", + "prior_balance_sats": 0, + "recipient_id": "miner-a" + }, + { + "action": "onchain", + "candidate_balance_sats": 100000000, + "carry_forward_balance_sats": 4000, + "gross_amount_sats": 100000000, + "onchain_amount_sats": 99996000, + "order_key": "miner-b", + "p2mr_program_hex": "0202020202020202020202020202020202020202020202020202020202020202", + "prior_balance_sats": 0, + "recipient_id": "miner-b" + }, + { + "action": "onchain", + "candidate_balance_sats": 250000000, + "carry_forward_balance_sats": 9999, + "gross_amount_sats": 250000000, + "onchain_amount_sats": 249990001, + "order_key": "miner-c", + "p2mr_program_hex": "0303030303030303030303030303030303030303030303030303030303030303", + "prior_balance_sats": 0, + "recipient_id": "miner-c" + }, + { + "action": "onchain", + "candidate_balance_sats": 20000, + "carry_forward_balance_sats": 1, + "gross_amount_sats": 0, + "onchain_amount_sats": 19999, + "order_key": "miner-old", + "p2mr_program_hex": "0909090909090909090909090909090909090909090909090909090909090909", + "prior_balance_sats": 20000, + "recipient_id": "miner-old" + } + ], + "block_height": 101, + "coinbase_value_sats": 500000000, + "floor_formula": "3680 bytes/input * 1 sat/byte * 4x safety", + "min_output_sats": 14720, + "onchain_entitlements": [ + { + "order_key": "miner-a", + "p2mr_program_hex": "0101010101010101010101010101010101010101010101010101010101010101", + "recipient_id": "miner-a", + "weight": 149994000 + }, + { + "order_key": "miner-b", + "p2mr_program_hex": "0202020202020202020202020202020202020202020202020202020202020202", + "recipient_id": "miner-b", + "weight": 99996000 + }, + { + "order_key": "miner-c", + "p2mr_program_hex": "0303030303030303030303030303030303030303030303030303030303030303", + "recipient_id": "miner-c", + "weight": 249990001 + }, + { + "order_key": "miner-old", + "p2mr_program_hex": "0909090909090909090909090909090909090909090909090909090909090909", + "recipient_id": "miner-old", + "weight": 19999 + } + ], + "schema": "qbit.prism.payout-policy.v1" + } + } + }, + "input": { + "bundle_2xx": { + "found_block": { + "anchor_job_issued_at_ms": 5000, + "block_height": 101, + "coinbase_value_sats": 500000000, + "network_difficulty": 100 + }, + "policy": { + "p2mr_spend_input_bytes": 3680, + "safety_multiplier": 4, + "target_feerate_sats_per_byte": 1 + }, + "prior_balances": [ + { + "balance_sats": 20000, + "order_key": "miner-old", + "p2mr_program_hex": "0909090909090909090909090909090909090909090909090909090909090909", + "recipient_id": "miner-old" + } + ], + "shares": [ + { + "accepted_at_ms": 4000, + "job_id": "job-4000", + "job_issued_at_ms": 4000, + "miner_id": "miner-a", + "network_difficulty": 100, + "ntime": 1800000000, + "order_key": "miner-a", + "p2mr_program_hex": "0101010101010101010101010101010101010101010101010101010101010101", + "share_difficulty": 30, + "share_id": "share-1", + "share_seq": 1, + "template_height": 100 + }, + { + "accepted_at_ms": 4001, + "job_id": "job-4001", + "job_issued_at_ms": 4001, + "miner_id": "miner-b", + "network_difficulty": 100, + "ntime": 1800000000, + "order_key": "miner-b", + "p2mr_program_hex": "0202020202020202020202020202020202020202020202020202020202020202", + "share_difficulty": 20, + "share_id": "share-2", + "share_seq": 2, + "template_height": 100 + }, + { + "accepted_at_ms": 4002, + "job_id": "job-4002", + "job_issued_at_ms": 4002, + "miner_id": "miner-c", + "network_difficulty": 100, + "ntime": 1800000000, + "order_key": "miner-c", + "p2mr_program_hex": "0303030303030303030303030303030303030303030303030303030303030303", + "share_difficulty": 50, + "share_id": "share-3", + "share_seq": 3, + "template_height": 100 + } + ] + }, + "bundle_3xx": { + "found_block": { + "anchor_job_issued_at_ms": 5000, + "block_height": 101, + "coinbase_value_sats": 500000000, + "network_difficulty": 100 + }, + "policy": { + "p2mr_spend_input_bytes": 3680, + "safety_multiplier": 4, + "target_feerate_sats_per_byte": 1 + }, + "prior_balances": [ + { + "balance_sats": 20000, + "order_key": "miner-old", + "p2mr_program_hex": "0909090909090909090909090909090909090909090909090909090909090909", + "recipient_id": "miner-old" + } + ], + "shares": [ + { + "accepted_at_ms": 4000, + "job_id": "job-4000", + "job_issued_at_ms": 4000, + "miner_id": "miner-a", + "network_difficulty": 100, + "ntime": 1800000000, + "order_key": "miner-a", + "p2mr_program_hex": "0101010101010101010101010101010101010101010101010101010101010101", + "share_difficulty": 30, + "share_id": "share-1", + "share_seq": 1, + "template_height": 100 + }, + { + "accepted_at_ms": 4001, + "job_id": "job-4001", + "job_issued_at_ms": 4001, + "miner_id": "miner-b", + "network_difficulty": 100, + "ntime": 1800000000, + "order_key": "miner-b", + "p2mr_program_hex": "0202020202020202020202020202020202020202020202020202020202020202", + "share_difficulty": 20, + "share_id": "share-2", + "share_seq": 2, + "template_height": 100 + }, + { + "accepted_at_ms": 4002, + "job_id": "job-4002", + "job_issued_at_ms": 4002, + "miner_id": "miner-c", + "network_difficulty": 100, + "ntime": 1800000000, + "order_key": "miner-c", + "p2mr_program_hex": "0303030303030303030303030303030303030303030303030303030303030303", + "share_difficulty": 50, + "share_id": "share-3", + "share_seq": 3, + "template_height": 100 + } + ] + }, + "rule": { + "decision_2xx": { + "accepted_share_count": 3, + "bundle_built_by": "lab/prism/job_bundle.py:3400 ready build (ledger window and prior balances)", + "distinct_miner_count": 3, + "implemented_at": [ + "lab/prism/prism_coordinator.py:7778 pool_readiness_latched", + "lab/prism/job_bundle.py:2132 _job_bundle_mode", + "lab/prism/job_bundle.py:3589 prior_balances=[] in build_collection_bundle" + ], + "job_bundle_mode": "ready", + "pool_readiness_latched": true + }, + "decision_3xx": { + "bootstrap": false, + "bundle_built_by": "ledger snapshot window (crates/qbit-prism-server/src/coordinator.rs:747)", + "implemented_at": [ + "crates/qbit-prism-server/src/coordinator.rs:556 solver-only bundle only when snapshot.shares is empty", + "crates/qbit-prism-server/src/coordinator.rs:759,773 snapshot.prior_balances passed in bootstrap" + ], + "prior_balances_kept": true + }, + "scenario": { + "anchor_ms": 5000, + "ledger_shares": [ + { + "accepted_at_ms": 4000, + "job_id": "job-4000", + "job_issued_at_ms": 4000, + "miner_id": "miner-a", + "network_difficulty": 100, + "ntime": 1800000000, + "order_key": "miner-a", + "p2mr_program_hex": "0101010101010101010101010101010101010101010101010101010101010101", + "share_difficulty": 30, + "share_id": "share-1", + "share_seq": 1, + "template_height": 100 + }, + { + "accepted_at_ms": 4001, + "job_id": "job-4001", + "job_issued_at_ms": 4001, + "miner_id": "miner-b", + "network_difficulty": 100, + "ntime": 1800000000, + "order_key": "miner-b", + "p2mr_program_hex": "0202020202020202020202020202020202020202020202020202020202020202", + "share_difficulty": 20, + "share_id": "share-2", + "share_seq": 2, + "template_height": 100 + }, + { + "accepted_at_ms": 4002, + "job_id": "job-4002", + "job_issued_at_ms": 4002, + "miner_id": "miner-c", + "network_difficulty": 100, + "ntime": 1800000000, + "order_key": "miner-c", + "p2mr_program_hex": "0303030303030303030303030303030303030303030303030303030303030303", + "share_difficulty": 50, + "share_id": "share-3", + "share_seq": 3, + "template_height": 100 + } + ], + "min_ready_miners": 3, + "network_difficulty": 100, + "prior_balances": [ + { + "balance_sats": 20000, + "order_key": "miner-old", + "p2mr_program_hex": "0909090909090909090909090909090909090909090909090909090909090909", + "recipient_id": "miner-old" + } + ], + "solver": { + "p2mr_program_hex": "0202020202020202020202020202020202020202020202020202020202020202", + "payout_address": "miner-b" + }, + "template": { + "coinbasevalue": 500000000, + "curtime": 1800000000, + "height": 101, + "previousblockhash": "0000000000000000000000000000000000000000000000000000000000000000" + } + } + } + }, + "name": "at-readiness-gate", + "topic": 8, + "why": "Three distinct miners reach the gate, so 2.x.x latches ready and both versions build the normal window with prior balances." + }, + { + "entry_point": "bundle_payout", + "expected": { + "ok": { + "counted_shares": [ + { + "counted_difficulty": 100, + "miner_id": "miner-b", + "share_seq": 1 + } + ], + "counted_window_weight": 100, + "entitlements": [ + { + "order_key": "miner-b", + "p2mr_program_hex": "0202020202020202020202020202020202020202020202020202020202020202", + "recipient_id": "miner-b", + "weight": 100 + } + ], + "payout_policy_manifest": { + "accounts": [ + { + "action": "onchain", + "candidate_balance_sats": 500000000, + "carry_forward_balance_sats": 0, + "gross_amount_sats": 500000000, + "onchain_amount_sats": 500000000, + "order_key": "miner-b", + "p2mr_program_hex": "0202020202020202020202020202020202020202020202020202020202020202", + "prior_balance_sats": 0, + "recipient_id": "miner-b" + } + ], + "block_height": 101, + "coinbase_value_sats": 500000000, + "floor_formula": "3680 bytes/input * 1 sat/byte * 4x safety", + "min_output_sats": 14720, + "onchain_entitlements": [ + { + "order_key": "miner-b", + "p2mr_program_hex": "0202020202020202020202020202020202020202020202020202020202020202", + "recipient_id": "miner-b", + "weight": 500000000 + } + ], + "schema": "qbit.prism.payout-policy.v1" + } + } + }, + "input": { + "bundle_2xx": { + "found_block": { + "anchor_job_issued_at_ms": 5000, + "block_height": 101, + "coinbase_value_sats": 500000000, + "network_difficulty": 100 + }, + "policy": { + "p2mr_spend_input_bytes": 3680, + "safety_multiplier": 4, + "target_feerate_sats_per_byte": 1 + }, + "prior_balances": [], + "shares": [ + { + "accepted_at_ms": 5000, + "job_id": "bootstrap-job", + "job_issued_at_ms": 5000, + "miner_id": "miner-b", + "network_difficulty": 100, + "ntime": 1800000000, + "order_key": "miner-b", + "p2mr_program_hex": "0202020202020202020202020202020202020202020202020202020202020202", + "share_difficulty": 100, + "share_id": "bootstrap-share", + "share_seq": 1, + "template_height": 100 + } + ] + }, + "bundle_3xx": { + "found_block": { + "anchor_job_issued_at_ms": 5000, + "block_height": 101, + "coinbase_value_sats": 500000000, + "network_difficulty": 100 + }, + "policy": { + "p2mr_spend_input_bytes": 3680, + "safety_multiplier": 4, + "target_feerate_sats_per_byte": 1 + }, + "prior_balances": [], + "shares": [ + { + "accepted_at_ms": 5000, + "job_id": "bootstrap-job", + "job_issued_at_ms": 5000, + "miner_id": "miner-b", + "network_difficulty": 100, + "ntime": 1800000000, + "order_key": "miner-b", + "p2mr_program_hex": "0202020202020202020202020202020202020202020202020202020202020202", + "share_difficulty": 100, + "share_id": "bootstrap-share", + "share_seq": 1, + "template_height": 100 + } + ] + }, + "rule": { + "decision_2xx": { + "accepted_share_count": 0, + "bundle_built_by": "lab/prism/job_bundle.py:3550 build_collection_bundle (executed)", + "distinct_miner_count": 0, + "implemented_at": [ + "lab/prism/prism_coordinator.py:7778 pool_readiness_latched", + "lab/prism/job_bundle.py:2132 _job_bundle_mode", + "lab/prism/job_bundle.py:3589 prior_balances=[] in build_collection_bundle" + ], + "job_bundle_mode": "collection", + "pool_readiness_latched": false + }, + "decision_3xx": { + "bootstrap": true, + "bundle_built_by": "synthetic solver share (crates/qbit-prism-server/src/coordinator.rs:727)", + "implemented_at": [ + "crates/qbit-prism-server/src/coordinator.rs:556 solver-only bundle only when snapshot.shares is empty", + "crates/qbit-prism-server/src/coordinator.rs:759,773 snapshot.prior_balances passed in bootstrap" + ], + "prior_balances_kept": true + }, + "scenario": { + "anchor_ms": 5000, + "ledger_shares": [], + "min_ready_miners": 3, + "network_difficulty": 100, + "prior_balances": [], + "solver": { + "p2mr_program_hex": "0202020202020202020202020202020202020202020202020202020202020202", + "payout_address": "miner-b" + }, + "template": { + "coinbasevalue": 500000000, + "curtime": 1800000000, + "height": 101, + "previousblockhash": "0000000000000000000000000000000000000000000000000000000000000000" + } + } + } + }, + "name": "empty-ledger", + "topic": 8, + "why": "No shares and no balances: both versions pay the solver the whole coinbase through the synthetic bootstrap share." + }, + { + "d2_entry": "d2c-prior-balances-during-bootstrap", + "entry_point": "bundle_payout", + "expected_2xx": { + "ok": { + "counted_shares": [ + { + "counted_difficulty": 100, + "miner_id": "miner-b", + "share_seq": 1 + } + ], + "counted_window_weight": 100, + "entitlements": [ + { + "order_key": "miner-b", + "p2mr_program_hex": "0202020202020202020202020202020202020202020202020202020202020202", + "recipient_id": "miner-b", + "weight": 100 + } + ], + "payout_policy_manifest": { + "accounts": [ + { + "action": "onchain", + "candidate_balance_sats": 500000000, + "carry_forward_balance_sats": 0, + "gross_amount_sats": 500000000, + "onchain_amount_sats": 500000000, + "order_key": "miner-b", + "p2mr_program_hex": "0202020202020202020202020202020202020202020202020202020202020202", + "prior_balance_sats": 0, + "recipient_id": "miner-b" + } + ], + "block_height": 101, + "coinbase_value_sats": 500000000, + "floor_formula": "3680 bytes/input * 1 sat/byte * 4x safety", + "min_output_sats": 14720, + "onchain_entitlements": [ + { + "order_key": "miner-b", + "p2mr_program_hex": "0202020202020202020202020202020202020202020202020202020202020202", + "recipient_id": "miner-b", + "weight": 500000000 + } + ], + "schema": "qbit.prism.payout-policy.v1" + } + } + }, + "expected_3xx": { + "ok": { + "counted_shares": [ + { + "counted_difficulty": 100, + "miner_id": "miner-b", + "share_seq": 1 + } + ], + "counted_window_weight": 100, + "entitlements": [ + { + "order_key": "miner-b", + "p2mr_program_hex": "0202020202020202020202020202020202020202020202020202020202020202", + "recipient_id": "miner-b", + "weight": 100 + } + ], + "payout_policy_manifest": { + "accounts": [ + { + "action": "onchain", + "candidate_balance_sats": 500000000, + "carry_forward_balance_sats": 19999, + "gross_amount_sats": 500000000, + "onchain_amount_sats": 499980001, + "order_key": "miner-b", + "p2mr_program_hex": "0202020202020202020202020202020202020202020202020202020202020202", + "prior_balance_sats": 0, + "recipient_id": "miner-b" + }, + { + "action": "onchain", + "candidate_balance_sats": 20000, + "carry_forward_balance_sats": 1, + "gross_amount_sats": 0, + "onchain_amount_sats": 19999, + "order_key": "miner-old", + "p2mr_program_hex": "0909090909090909090909090909090909090909090909090909090909090909", + "prior_balance_sats": 20000, + "recipient_id": "miner-old" + } + ], + "block_height": 101, + "coinbase_value_sats": 500000000, + "floor_formula": "3680 bytes/input * 1 sat/byte * 4x safety", + "min_output_sats": 14720, + "onchain_entitlements": [ + { + "order_key": "miner-b", + "p2mr_program_hex": "0202020202020202020202020202020202020202020202020202020202020202", + "recipient_id": "miner-b", + "weight": 499980001 + }, + { + "order_key": "miner-old", + "p2mr_program_hex": "0909090909090909090909090909090909090909090909090909090909090909", + "recipient_id": "miner-old", + "weight": 19999 + } + ], + "schema": "qbit.prism.payout-policy.v1" + } + } + }, + "input": { + "bundle_2xx": { + "found_block": { + "anchor_job_issued_at_ms": 5000, + "block_height": 101, + "coinbase_value_sats": 500000000, + "network_difficulty": 100 + }, + "policy": { + "p2mr_spend_input_bytes": 3680, + "safety_multiplier": 4, + "target_feerate_sats_per_byte": 1 + }, + "prior_balances": [], + "shares": [ + { + "accepted_at_ms": 5000, + "job_id": "bootstrap-job", + "job_issued_at_ms": 5000, + "miner_id": "miner-b", + "network_difficulty": 100, + "ntime": 1800000000, + "order_key": "miner-b", + "p2mr_program_hex": "0202020202020202020202020202020202020202020202020202020202020202", + "share_difficulty": 100, + "share_id": "bootstrap-share", + "share_seq": 1, + "template_height": 100 + } + ] + }, + "bundle_3xx": { + "found_block": { + "anchor_job_issued_at_ms": 5000, + "block_height": 101, + "coinbase_value_sats": 500000000, + "network_difficulty": 100 + }, + "policy": { + "p2mr_spend_input_bytes": 3680, + "safety_multiplier": 4, + "target_feerate_sats_per_byte": 1 + }, + "prior_balances": [ + { + "balance_sats": 20000, + "order_key": "miner-old", + "p2mr_program_hex": "0909090909090909090909090909090909090909090909090909090909090909", + "recipient_id": "miner-old" + } + ], + "shares": [ + { + "accepted_at_ms": 5000, + "job_id": "bootstrap-job", + "job_issued_at_ms": 5000, + "miner_id": "miner-b", + "network_difficulty": 100, + "ntime": 1800000000, + "order_key": "miner-b", + "p2mr_program_hex": "0202020202020202020202020202020202020202020202020202020202020202", + "share_difficulty": 100, + "share_id": "bootstrap-share", + "share_seq": 1, + "template_height": 100 + } + ] + }, + "rule": { + "decision_2xx": { + "accepted_share_count": 0, + "bundle_built_by": "lab/prism/job_bundle.py:3550 build_collection_bundle (executed)", + "distinct_miner_count": 0, + "implemented_at": [ + "lab/prism/prism_coordinator.py:7778 pool_readiness_latched", + "lab/prism/job_bundle.py:2132 _job_bundle_mode", + "lab/prism/job_bundle.py:3589 prior_balances=[] in build_collection_bundle" + ], + "job_bundle_mode": "collection", + "pool_readiness_latched": false + }, + "decision_3xx": { + "bootstrap": true, + "bundle_built_by": "synthetic solver share (crates/qbit-prism-server/src/coordinator.rs:727)", + "implemented_at": [ + "crates/qbit-prism-server/src/coordinator.rs:556 solver-only bundle only when snapshot.shares is empty", + "crates/qbit-prism-server/src/coordinator.rs:759,773 snapshot.prior_balances passed in bootstrap" + ], + "prior_balances_kept": true + }, + "scenario": { + "anchor_ms": 5000, + "ledger_shares": [], + "min_ready_miners": 3, + "network_difficulty": 100, + "prior_balances": [ + { + "balance_sats": 20000, + "order_key": "miner-old", + "p2mr_program_hex": "0909090909090909090909090909090909090909090909090909090909090909", + "recipient_id": "miner-old" + } + ], + "solver": { + "p2mr_program_hex": "0202020202020202020202020202020202020202020202020202020202020202", + "payout_address": "miner-b" + }, + "template": { + "coinbasevalue": 500000000, + "curtime": 1800000000, + "height": 101, + "previousblockhash": "0000000000000000000000000000000000000000000000000000000000000000" + } + } + } + }, + "name": "bootstrap-carry-only-account-at-or-above-floor", + "topic": 8, + "why": "An empty share window with a carry-only account whose carry (20000 sats) is above the 14720-sat floor. 2.x.x drops prior balances from the collection bundle, so the account is not paid in this block; 3.x.x keeps the snapshot's prior balances in bootstrap and pays it." + } + ], + "produced_by": "PYTHONDONTWRITEBYTECODE=1 PYTHONPATH=. python3 crates/qbit-prism/examples/export_rule_decisions.py | cargo run --locked -q -p qbit-prism --example export_money_path_vectors -- vectors-out", + "schema": "qbit.prism.money-path-vectors.v1", + "source_commit": "504846cc0b72e8f86ed17f896d4ccbbe196a31dc", + "topics": [ + { + "id": 8, + "name": "bootstrap transition" + } + ] +} diff --git a/crates/qbit-prism/fixtures/vectors/carry_only.json b/crates/qbit-prism/fixtures/vectors/carry_only.json new file mode 100644 index 000000000..82c085e59 --- /dev/null +++ b/crates/qbit-prism/fixtures/vectors/carry_only.json @@ -0,0 +1,649 @@ +{ + "cases": [ + { + "entry_point": "apply_payout_policy", + "expected": { + "ok": { + "accounts": [ + { + "action": "onchain", + "candidate_balance_sats": 500000000, + "carry_forward_balance_sats": 15000, + "gross_amount_sats": 500000000, + "onchain_amount_sats": 499985000, + "order_key": "01", + "p2mr_program_hex": "0101010101010101010101010101010101010101010101010101010101010101", + "prior_balance_sats": 0, + "recipient_id": "miner-a" + }, + { + "action": "onchain", + "candidate_balance_sats": 15000, + "carry_forward_balance_sats": 0, + "gross_amount_sats": 0, + "onchain_amount_sats": 15000, + "order_key": "02", + "p2mr_program_hex": "0202020202020202020202020202020202020202020202020202020202020202", + "prior_balance_sats": 15000, + "recipient_id": "carry-only" + } + ], + "block_height": 10, + "coinbase_value_sats": 500000000, + "floor_formula": "3680 bytes/input * 1 sat/byte * 4x safety", + "min_output_sats": 14720, + "onchain_entitlements": [ + { + "order_key": "01", + "p2mr_program_hex": "0101010101010101010101010101010101010101010101010101010101010101", + "recipient_id": "miner-a", + "weight": 499985000 + }, + { + "order_key": "02", + "p2mr_program_hex": "0202020202020202020202020202020202020202020202020202020202020202", + "recipient_id": "carry-only", + "weight": 15000 + } + ], + "schema": "qbit.prism.payout-policy.v1" + } + }, + "input": { + "policy": { + "p2mr_spend_input_bytes": 3680, + "safety_multiplier": 4, + "target_feerate_sats_per_byte": 1 + }, + "prior_balances": [ + { + "balance_sats": 15000, + "order_key": "02", + "p2mr_program_hex": "0202020202020202020202020202020202020202020202020202020202020202", + "recipient_id": "carry-only" + } + ], + "reward_manifest": { + "anchor_job_issued_at_ms": 1, + "anchor_share_seq": 1, + "block_height": 10, + "coinbase_value_sats": 500000000, + "counted_window_weight": 1, + "entitlements": [ + { + "order_key": "01", + "p2mr_program_hex": "0101010101010101010101010101010101010101010101010101010101010101", + "recipient_id": "miner-a", + "weight": 1 + } + ], + "included_share_count": 1, + "network_difficulty": 1, + "newest_share_seq": 1, + "oldest_share_seq": 1, + "requested_window_weight": 8, + "schema": "qbit.prism.reward-manifest.v1", + "share_slice_digest_hex": "0000000000000000000000000000000000000000000000000000000000000000", + "shares": [], + "window_multiplier": 8 + } + }, + "name": "prior-only-account-crosses-floor", + "topic": 3, + "why": "An account with no share in the window but a 15000-sat carry (floor 14720) is a payout candidate on its own." + }, + { + "entry_point": "apply_payout_policy", + "expected": { + "ok": { + "accounts": [ + { + "action": "onchain", + "candidate_balance_sats": 500000000, + "carry_forward_balance_sats": 14720, + "gross_amount_sats": 500000000, + "onchain_amount_sats": 499985280, + "order_key": "01", + "p2mr_program_hex": "0101010101010101010101010101010101010101010101010101010101010101", + "prior_balance_sats": 0, + "recipient_id": "miner-a" + }, + { + "action": "onchain", + "candidate_balance_sats": 14720, + "carry_forward_balance_sats": 0, + "gross_amount_sats": 0, + "onchain_amount_sats": 14720, + "order_key": "02", + "p2mr_program_hex": "0202020202020202020202020202020202020202020202020202020202020202", + "prior_balance_sats": 14720, + "recipient_id": "carry-only" + } + ], + "block_height": 10, + "coinbase_value_sats": 500000000, + "floor_formula": "3680 bytes/input * 1 sat/byte * 4x safety", + "min_output_sats": 14720, + "onchain_entitlements": [ + { + "order_key": "01", + "p2mr_program_hex": "0101010101010101010101010101010101010101010101010101010101010101", + "recipient_id": "miner-a", + "weight": 499985280 + }, + { + "order_key": "02", + "p2mr_program_hex": "0202020202020202020202020202020202020202020202020202020202020202", + "recipient_id": "carry-only", + "weight": 14720 + } + ], + "schema": "qbit.prism.payout-policy.v1" + } + }, + "input": { + "policy": { + "p2mr_spend_input_bytes": 3680, + "safety_multiplier": 4, + "target_feerate_sats_per_byte": 1 + }, + "prior_balances": [ + { + "balance_sats": 14720, + "order_key": "02", + "p2mr_program_hex": "0202020202020202020202020202020202020202020202020202020202020202", + "recipient_id": "carry-only" + } + ], + "reward_manifest": { + "anchor_job_issued_at_ms": 1, + "anchor_share_seq": 1, + "block_height": 10, + "coinbase_value_sats": 500000000, + "counted_window_weight": 1, + "entitlements": [ + { + "order_key": "01", + "p2mr_program_hex": "0101010101010101010101010101010101010101010101010101010101010101", + "recipient_id": "miner-a", + "weight": 1 + } + ], + "included_share_count": 1, + "network_difficulty": 1, + "newest_share_seq": 1, + "oldest_share_seq": 1, + "requested_window_weight": 8, + "schema": "qbit.prism.reward-manifest.v1", + "share_slice_digest_hex": "0000000000000000000000000000000000000000000000000000000000000000", + "shares": [], + "window_multiplier": 8 + } + }, + "name": "prior-only-account-exactly-at-floor", + "topic": 3, + "why": "A carry of exactly 14720 is eligible, but its proportional share of the coinbase can land a sat under the floor." + }, + { + "entry_point": "apply_payout_policy", + "expected": { + "ok": { + "accounts": [ + { + "action": "onchain", + "candidate_balance_sats": 500000000, + "carry_forward_balance_sats": 0, + "gross_amount_sats": 500000000, + "onchain_amount_sats": 500000000, + "order_key": "01", + "p2mr_program_hex": "0101010101010101010101010101010101010101010101010101010101010101", + "prior_balance_sats": 0, + "recipient_id": "miner-a" + }, + { + "action": "accrued", + "candidate_balance_sats": 14719, + "carry_forward_balance_sats": 14719, + "gross_amount_sats": 0, + "onchain_amount_sats": 0, + "order_key": "02", + "p2mr_program_hex": "0202020202020202020202020202020202020202020202020202020202020202", + "prior_balance_sats": 14719, + "recipient_id": "carry-only" + } + ], + "block_height": 10, + "coinbase_value_sats": 500000000, + "floor_formula": "3680 bytes/input * 1 sat/byte * 4x safety", + "min_output_sats": 14720, + "onchain_entitlements": [ + { + "order_key": "01", + "p2mr_program_hex": "0101010101010101010101010101010101010101010101010101010101010101", + "recipient_id": "miner-a", + "weight": 500000000 + } + ], + "schema": "qbit.prism.payout-policy.v1" + } + }, + "input": { + "policy": { + "p2mr_spend_input_bytes": 3680, + "safety_multiplier": 4, + "target_feerate_sats_per_byte": 1 + }, + "prior_balances": [ + { + "balance_sats": 14719, + "order_key": "02", + "p2mr_program_hex": "0202020202020202020202020202020202020202020202020202020202020202", + "recipient_id": "carry-only" + } + ], + "reward_manifest": { + "anchor_job_issued_at_ms": 1, + "anchor_share_seq": 1, + "block_height": 10, + "coinbase_value_sats": 500000000, + "counted_window_weight": 1, + "entitlements": [ + { + "order_key": "01", + "p2mr_program_hex": "0101010101010101010101010101010101010101010101010101010101010101", + "recipient_id": "miner-a", + "weight": 1 + } + ], + "included_share_count": 1, + "network_difficulty": 1, + "newest_share_seq": 1, + "oldest_share_seq": 1, + "requested_window_weight": 8, + "schema": "qbit.prism.reward-manifest.v1", + "share_slice_digest_hex": "0000000000000000000000000000000000000000000000000000000000000000", + "shares": [], + "window_multiplier": 8 + } + }, + "name": "prior-only-account-one-sat-under-floor", + "topic": 3, + "why": "A carry of 14719 is not eligible and keeps accruing; the miner is paid the whole coinbase." + }, + { + "entry_point": "apply_payout_policy", + "expected": { + "ok": { + "accounts": [ + { + "action": "onchain", + "candidate_balance_sats": 95000, + "carry_forward_balance_sats": 10179, + "gross_amount_sats": 95000, + "onchain_amount_sats": 84821, + "order_key": "01", + "p2mr_program_hex": "0101010101010101010101010101010101010101010101010101010101010101", + "prior_balance_sats": 0, + "recipient_id": "large" + }, + { + "action": "onchain", + "candidate_balance_sats": 17000, + "carry_forward_balance_sats": 1821, + "gross_amount_sats": 5000, + "onchain_amount_sats": 15179, + "order_key": "02", + "p2mr_program_hex": "0202020202020202020202020202020202020202020202020202020202020202", + "prior_balance_sats": 12000, + "recipient_id": "small" + } + ], + "block_height": 10, + "coinbase_value_sats": 100000, + "floor_formula": "3680 bytes/input * 1 sat/byte * 4x safety", + "min_output_sats": 14720, + "onchain_entitlements": [ + { + "order_key": "01", + "p2mr_program_hex": "0101010101010101010101010101010101010101010101010101010101010101", + "recipient_id": "large", + "weight": 84821 + }, + { + "order_key": "02", + "p2mr_program_hex": "0202020202020202020202020202020202020202020202020202020202020202", + "recipient_id": "small", + "weight": 15179 + } + ], + "schema": "qbit.prism.payout-policy.v1" + } + }, + "input": { + "policy": { + "p2mr_spend_input_bytes": 3680, + "safety_multiplier": 4, + "target_feerate_sats_per_byte": 1 + }, + "prior_balances": [ + { + "balance_sats": 12000, + "order_key": "02", + "p2mr_program_hex": "0202020202020202020202020202020202020202020202020202020202020202", + "recipient_id": "small" + } + ], + "reward_manifest": { + "anchor_job_issued_at_ms": 1, + "anchor_share_seq": 1, + "block_height": 10, + "coinbase_value_sats": 100000, + "counted_window_weight": 100, + "entitlements": [ + { + "order_key": "01", + "p2mr_program_hex": "0101010101010101010101010101010101010101010101010101010101010101", + "recipient_id": "large", + "weight": 95 + }, + { + "order_key": "02", + "p2mr_program_hex": "0202020202020202020202020202020202020202020202020202020202020202", + "recipient_id": "small", + "weight": 5 + } + ], + "included_share_count": 2, + "network_difficulty": 1, + "newest_share_seq": 1, + "oldest_share_seq": 1, + "requested_window_weight": 8, + "schema": "qbit.prism.reward-manifest.v1", + "share_slice_digest_hex": "0000000000000000000000000000000000000000000000000000000000000000", + "shares": [], + "window_multiplier": 8 + } + }, + "name": "prior-plus-gross-crosses-floor", + "topic": 3, + "why": "A 5000-sat gross plus a 12000-sat carry crosses the floor and is paid on-chain." + }, + { + "entry_point": "apply_payout_policy", + "expected": { + "ok": { + "accounts": [ + { + "action": "onchain", + "candidate_balance_sats": 90000, + "carry_forward_balance_sats": 5888, + "gross_amount_sats": 90000, + "onchain_amount_sats": 84112, + "order_key": "01", + "p2mr_program_hex": "0101010101010101010101010101010101010101010101010101010101010101", + "prior_balance_sats": 0, + "recipient_id": "miner-a" + }, + { + "action": "onchain", + "candidate_balance_sats": 17000, + "carry_forward_balance_sats": 1112, + "gross_amount_sats": 10000, + "onchain_amount_sats": 15888, + "order_key": "03", + "p2mr_program_hex": "abababababababababababababababababababababababababababababababab", + "prior_balance_sats": 7000, + "recipient_id": "alias-y" + } + ], + "block_height": 10, + "coinbase_value_sats": 100000, + "floor_formula": "3680 bytes/input * 1 sat/byte * 4x safety", + "min_output_sats": 14720, + "onchain_entitlements": [ + { + "order_key": "01", + "p2mr_program_hex": "0101010101010101010101010101010101010101010101010101010101010101", + "recipient_id": "miner-a", + "weight": 84112 + }, + { + "order_key": "03", + "p2mr_program_hex": "abababababababababababababababababababababababababababababababab", + "recipient_id": "alias-y", + "weight": 15888 + } + ], + "schema": "qbit.prism.payout-policy.v1" + } + }, + "input": { + "policy": { + "p2mr_spend_input_bytes": 3680, + "safety_multiplier": 4, + "target_feerate_sats_per_byte": 1 + }, + "prior_balances": [ + { + "balance_sats": 4000, + "order_key": "03", + "p2mr_program_hex": "ABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABAB", + "recipient_id": "alias-y" + }, + { + "balance_sats": 3000, + "order_key": "07", + "p2mr_program_hex": "abababababababababababababababababababababababababababababababab", + "recipient_id": "alias-z" + } + ], + "reward_manifest": { + "anchor_job_issued_at_ms": 1, + "anchor_share_seq": 1, + "block_height": 10, + "coinbase_value_sats": 100000, + "counted_window_weight": 100, + "entitlements": [ + { + "order_key": "01", + "p2mr_program_hex": "0101010101010101010101010101010101010101010101010101010101010101", + "recipient_id": "miner-a", + "weight": 90 + }, + { + "order_key": "05", + "p2mr_program_hex": "abababababababababababababababababababababababababababababababab", + "recipient_id": "alias-x", + "weight": 5 + }, + { + "order_key": "03", + "p2mr_program_hex": "ABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABAB", + "recipient_id": "alias-y", + "weight": 5 + } + ], + "included_share_count": 3, + "network_difficulty": 1, + "newest_share_seq": 1, + "oldest_share_seq": 1, + "requested_window_weight": 8, + "schema": "qbit.prism.reward-manifest.v1", + "share_slice_digest_hex": "0000000000000000000000000000000000000000000000000000000000000000", + "shares": [], + "window_multiplier": 8 + } + }, + "name": "aliases-of-one-program-aggregate", + "topic": 3, + "why": "Two recipient ids, and carry rows in upper- and lower-case hex, for one payout program aggregate into one account before floor selection." + }, + { + "entry_point": "apply_payout_policy", + "expected": { + "ok": { + "accounts": [ + { + "action": "onchain", + "candidate_balance_sats": 60000, + "carry_forward_balance_sats": 2857, + "gross_amount_sats": 50000, + "onchain_amount_sats": 57143, + "order_key": "01", + "p2mr_program_hex": "0101010101010101010101010101010101010101010101010101010101010101", + "prior_balance_sats": 10000, + "recipient_id": "miner-a" + }, + { + "action": "onchain", + "candidate_balance_sats": 45000, + "carry_forward_balance_sats": 2143, + "gross_amount_sats": 50000, + "onchain_amount_sats": 42857, + "order_key": "02", + "p2mr_program_hex": "0202020202020202020202020202020202020202020202020202020202020202", + "prior_balance_sats": -5000, + "recipient_id": "miner-b" + } + ], + "block_height": 10, + "coinbase_value_sats": 100000, + "floor_formula": "3680 bytes/input * 1 sat/byte * 4x safety", + "min_output_sats": 14720, + "onchain_entitlements": [ + { + "order_key": "01", + "p2mr_program_hex": "0101010101010101010101010101010101010101010101010101010101010101", + "recipient_id": "miner-a", + "weight": 57143 + }, + { + "order_key": "02", + "p2mr_program_hex": "0202020202020202020202020202020202020202020202020202020202020202", + "recipient_id": "miner-b", + "weight": 42857 + } + ], + "schema": "qbit.prism.payout-policy.v1" + } + }, + "input": { + "policy": { + "p2mr_spend_input_bytes": 3680, + "safety_multiplier": 4, + "target_feerate_sats_per_byte": 1 + }, + "prior_balances": [ + { + "balance_sats": 10000, + "order_key": "01", + "p2mr_program_hex": "0101010101010101010101010101010101010101010101010101010101010101", + "recipient_id": "miner-a" + }, + { + "balance_sats": -5000, + "order_key": "02", + "p2mr_program_hex": "0202020202020202020202020202020202020202020202020202020202020202", + "recipient_id": "miner-b" + } + ], + "reward_manifest": { + "anchor_job_issued_at_ms": 1, + "anchor_share_seq": 1, + "block_height": 10, + "coinbase_value_sats": 100000, + "counted_window_weight": 100, + "entitlements": [ + { + "order_key": "01", + "p2mr_program_hex": "0101010101010101010101010101010101010101010101010101010101010101", + "recipient_id": "miner-a", + "weight": 50 + }, + { + "order_key": "02", + "p2mr_program_hex": "0202020202020202020202020202020202020202020202020202020202020202", + "recipient_id": "miner-b", + "weight": 50 + } + ], + "included_share_count": 2, + "network_difficulty": 1, + "newest_share_seq": 1, + "oldest_share_seq": 1, + "requested_window_weight": 8, + "schema": "qbit.prism.reward-manifest.v1", + "share_slice_digest_hex": "0000000000000000000000000000000000000000000000000000000000000000", + "shares": [], + "window_multiplier": 8 + } + }, + "name": "overpayment-debt-reduces-candidate", + "topic": 3, + "why": "A negative carry (overpayment debt after a reorg) reduces that account's candidate balance and its payout." + }, + { + "entry_point": "apply_payout_policy", + "expected": { + "error": "on-chain payouts would exceed selected candidate balances: coinbase value 100000 > selected candidate balance 95000" + }, + "input": { + "policy": { + "p2mr_spend_input_bytes": 3680, + "safety_multiplier": 4, + "target_feerate_sats_per_byte": 1 + }, + "prior_balances": [ + { + "balance_sats": -5000, + "order_key": "02", + "p2mr_program_hex": "0202020202020202020202020202020202020202020202020202020202020202", + "recipient_id": "miner-b" + } + ], + "reward_manifest": { + "anchor_job_issued_at_ms": 1, + "anchor_share_seq": 1, + "block_height": 10, + "coinbase_value_sats": 100000, + "counted_window_weight": 100, + "entitlements": [ + { + "order_key": "01", + "p2mr_program_hex": "0101010101010101010101010101010101010101010101010101010101010101", + "recipient_id": "miner-a", + "weight": 50 + }, + { + "order_key": "02", + "p2mr_program_hex": "0202020202020202020202020202020202020202020202020202020202020202", + "recipient_id": "miner-b", + "weight": 50 + } + ], + "included_share_count": 2, + "network_difficulty": 1, + "newest_share_seq": 1, + "oldest_share_seq": 1, + "requested_window_weight": 8, + "schema": "qbit.prism.reward-manifest.v1", + "share_slice_digest_hex": "0000000000000000000000000000000000000000000000000000000000000000", + "shares": [], + "window_multiplier": 8 + } + }, + "name": "overpayment-debt-without-fee-is-an-error", + "topic": 3, + "why": "With debt and no other carry the eligible candidates sum below the coinbase, and without a pool fee to absorb it the policy fails." + } + ], + "produced_by": "PYTHONDONTWRITEBYTECODE=1 PYTHONPATH=. python3 crates/qbit-prism/examples/export_rule_decisions.py | cargo run --locked -q -p qbit-prism --example export_money_path_vectors -- vectors-out", + "schema": "qbit.prism.money-path-vectors.v1", + "source_commit": "504846cc0b72e8f86ed17f896d4ccbbe196a31dc", + "topics": [ + { + "id": 3, + "name": "carry-only recipients" + } + ] +} diff --git a/crates/qbit-prism/fixtures/vectors/exporter/README.md b/crates/qbit-prism/fixtures/vectors/exporter/README.md new file mode 100644 index 000000000..605d2059b --- /dev/null +++ b/crates/qbit-prism/fixtures/vectors/exporter/README.md @@ -0,0 +1,23 @@ +# Money-path vector exporter + +These two files produced the frozen vectors one directory up +(`crates/qbit-prism/fixtures/vectors/*.json`). They are kept outside any cargo target, so 3.x.x +never compiles them. `export_rule_decisions.py` runs the 2.x.x Python rule code for the bootstrap +transition and below-target block credit. `export_money_path_vectors.rs` calls the 2.x.x +`qbit-prism` engine for every topic and computes each rule's payout consequence. The 3.x.x test +`crates/qbit-prism/tests/money_path_vectors.rs` replays every case. To regenerate, extract the 2.x.x +source commit `504846cc0b72e8f86ed17f896d4ccbbe196a31dc` into an empty directory: +`git archive 504846cc0b72e8f86ed17f896d4ccbbe196a31dc | tar -x -C "$D"`. Copy both files into +`crates/qbit-prism/examples/` there. From the root of that tree, run +`PYTHONDONTWRITEBYTECODE=1 PYTHONPATH=. python3 crates/qbit-prism/examples/export_rule_decisions.py | cargo run --locked -q -p qbit-prism --example export_money_path_vectors -- vectors-out`. +Copy `vectors-out/*.json` here. The export is deterministic: two runs must be byte-identical. The +exporter refuses to write a bootstrap or credit case where the two rules pay differently unless +that case is mapped to a D2 entry in `docs/prism-rust-migration.md`. + +The test pins the sha256 of every vector file (`VECTOR_SHA256`). A re-export that changes any file +must update those pins in the same reviewed commit; never edit a vector by hand. + +Known gap: the TRUC per-transaction cap (1160 fanout recipients) is frozen only as a configuration +bound (`fanout-size-at-the-truc-cap` and `fanout-size-one-over-the-truc-cap-is-an-error`, each with +three recipients). A real 1160-recipient chunk, and a 1161-recipient one, would add about 1.6 MB to +`settlement_chunks.json`, so they are not exported. diff --git a/crates/qbit-prism/fixtures/vectors/exporter/export_money_path_vectors.rs b/crates/qbit-prism/fixtures/vectors/exporter/export_money_path_vectors.rs new file mode 100644 index 000000000..cc7b58482 --- /dev/null +++ b/crates/qbit-prism/fixtures/vectors/exporter/export_money_path_vectors.rs @@ -0,0 +1,1086 @@ +//! Export the money-path vectors frozen from 2.x.x. +//! +//! Built as a `qbit-prism` example inside a 2.x.x source tree; see README.md +//! in this directory for the exact regeneration command. Topics 1 to 7 call +//! the 2.x.x engine entry points directly. Topics 8 and 9 read the 2.x.x rule +//! decisions printed by `export_rule_decisions.py` on stdin, apply the 3.x.x +//! rule as implemented by the cited `qbit-prism-server` lines, and compute the +//! payout consequence of both rules with the same engine. + +use std::fs; +use std::io::Read; +use std::path::Path; + +use qbit_pool_builder::WeightedEntitlement; +use qbit_prism::{ + apply_payout_policy, apply_proportional_fanout_fee, build_prism_reward_manifest, + compute_prism_window, select_settlement_mode, select_settlement_mode_with_pinned_direct, + AcceptedShare, CarryForwardBalance, CoinbaseOutputPolicy, FoundBlock, PayoutPolicy, + PoolFeePolicy, PrismError, PrismRewardManifest, SettlementModeConfig, SettlementRecipient, + PRISM_WINDOW_MULTIPLIER, +}; +use serde::de::DeserializeOwned; +use serde::Serialize; +use serde_json::{json, Value}; + +const SCHEMA: &str = "qbit.prism.money-path-vectors.v1"; +const SOURCE_COMMIT: &str = "504846cc0b72e8f86ed17f896d4ccbbe196a31dc"; +const PRODUCED_BY: &str = "PYTHONDONTWRITEBYTECODE=1 PYTHONPATH=. python3 crates/qbit-prism/examples/export_rule_decisions.py | cargo run --locked -q -p qbit-prism --example export_money_path_vectors -- vectors-out"; + +const D2A: &str = "d2a-bootstrap-pooling"; +const D2B: &str = "d2b-below-target-credit"; +const D2C: &str = "d2c-prior-balances-during-bootstrap"; + +fn to_value(value: &T) -> Value { + serde_json::to_value(value).expect("vector values serialize") +} + +fn field(input: &Value, name: &str) -> T { + serde_json::from_value(input[name].clone()).unwrap_or_else(|err| panic!("{name}: {err}")) +} + +fn outcome(result: Result) -> Value { + match result { + Ok(value) => json!({ "ok": to_value(&value) }), + Err(err) => json!({ "error": err.to_string() }), + } +} + +/// Evaluate one case input through its entry point. The 3.x.x consumer +/// (`crates/qbit-prism/tests/money_path_vectors.rs`) mirrors this dispatcher. +fn run(entry_point: &str, input: &Value) -> Value { + match entry_point { + "compute_prism_window" => outcome(compute_prism_window( + &field::>(input, "shares"), + &field(input, "found_block"), + )), + "apply_payout_policy" => outcome(apply_payout_policy( + &field(input, "reward_manifest"), + &field::>(input, "prior_balances"), + &field(input, "policy"), + )), + "apply_proportional_fanout_fee" => outcome(apply_proportional_fanout_fee( + &field::>(input, "recipients"), + field(input, "fee_sats"), + field(input, "min_output_sats"), + )), + "select_settlement_mode" => { + let recipients = field::>(input, "recipients"); + let floor = field(input, "direct_floor_sats"); + let config = field::(input, "config"); + match field::>(input, "pinned_direct") { + Some(pinned) => outcome(select_settlement_mode_with_pinned_direct( + &recipients, + floor, + &config, + Some(&pinned), + )), + None => outcome(select_settlement_mode(&recipients, floor, &config)), + } + } + "bundle_payout" => outcome(bundle_payout(input)), + other => panic!("unknown entry point {other}"), + } +} + +/// The payout consequence of one bundle's economic inputs: the reward window +/// and the payout policy manifest the coinbase is built from. +fn bundle_payout(input: &Value) -> Result { + let reward = build_prism_reward_manifest( + &field::>(input, "shares"), + &field(input, "found_block"), + )?; + let payout = apply_payout_policy( + &reward, + &field::>(input, "prior_balances"), + &field(input, "policy"), + )?; + Ok(json!({ + "counted_window_weight": to_value(&reward.counted_window_weight), + "counted_shares": reward + .shares + .iter() + .map(|share| json!({ + "share_seq": share.share_seq, + "miner_id": share.miner_id, + "counted_difficulty": to_value(&share.counted_difficulty), + })) + .collect::>(), + "entitlements": to_value(&reward.entitlements), + "payout_policy_manifest": to_value(&payout), + })) +} + +fn case(topic: u8, name: &str, why: &str, entry_point: &str, input: Value) -> Value { + let expected = run(entry_point, &input); + json!({ + "name": name, + "topic": topic, + "why": why, + "entry_point": entry_point, + "input": input, + "expected": expected, + }) +} + +/// A rule case computes the payout under both rules. Equal payouts carry one +/// `expected`; a difference must be a listed D2 entry, or export stops. +fn rule_case( + topic: u8, + name: &str, + why: &str, + rule: Value, + bundle_2xx: Value, + bundle_3xx: Value, + d2_entry: Option<&str>, +) -> Value { + let expected_2xx = run("bundle_payout", &bundle_2xx); + let expected_3xx = run("bundle_payout", &bundle_3xx); + let input = json!({ "rule": rule, "bundle_2xx": bundle_2xx, "bundle_3xx": bundle_3xx }); + let mut case = json!({ + "name": name, + "topic": topic, + "why": why, + "entry_point": "bundle_payout", + "input": input, + }); + match (expected_2xx == expected_3xx, d2_entry) { + (true, None) => case["expected"] = expected_3xx, + (false, Some(anchor)) => { + case["expected_2xx"] = expected_2xx; + case["expected_3xx"] = expected_3xx; + case["d2_entry"] = json!(anchor); + } + (true, Some(anchor)) => panic!("{name}: listed as {anchor} but both rules pay the same"), + (false, None) => panic!("{name}: undocumented 2.x.x/3.x.x payout difference; stop and ask"), + } + case +} + +fn program(byte: u8) -> String { + hex::encode([byte; 32]) +} + +// --------------------------------------------------------------------------- +// Topics 1 and 2: compute_prism_window + +fn share( + share_seq: u64, + miner_id: &str, + order_key: &str, + byte: u8, + share_difficulty: u128, + job_issued_at_ms: i64, +) -> AcceptedShare { + AcceptedShare { + share_seq, + share_id: format!("share-{share_seq}"), + miner_id: miner_id.to_string(), + order_key: order_key.to_string(), + p2mr_program_hex: program(byte), + share_difficulty, + network_difficulty: 10, + template_height: 100, + job_id: format!("job-{job_issued_at_ms}"), + job_issued_at_ms, + accepted_at_ms: job_issued_at_ms, + ntime: 1_800_000_000, + credit_policy: None, + } +} + +fn found(network_difficulty: u128, anchor_job_issued_at_ms: i64) -> FoundBlock { + FoundBlock { + block_height: 101, + coinbase_value_sats: 500_000_000, + network_difficulty, + anchor_job_issued_at_ms, + } +} + +fn window(shares: Vec, found_block: FoundBlock) -> Value { + json!({ "shares": to_value(&shares), "found_block": to_value(&found_block) }) +} + +fn boundary_shares() -> Vec { + vec![ + share(1, "miner-old", "01", 1, 999, 1000), + share(2, "miner-b", "02", 2, 40, 1001), + share(3, "miner-a", "03", 3, 30, 1002), + share(4, "miner-c", "04", 4, 50, 1003), + ] +} + +fn window_cases() -> Vec { + let w = "compute_prism_window"; + let mut after_anchor = share(3, "miner-late", "03", 3, 1000, 2000); + after_anchor.accepted_at_ms = 1500; + let mut accepted_after_anchor = share(4, "miner-slow", "04", 4, 1000, 1000); + accepted_after_anchor.accepted_at_ms = 1002; + let mut unsorted = boundary_shares(); + unsorted.swap(0, 2); + unsorted.swap(1, 3); + vec![ + case( + 1, + "exact-fit-boundary", + "8 x 10 = 80 is filled exactly by the two newest shares (50 + 30); the older shares get nothing.", + w, + window(boundary_shares(), found(10, 1003)), + ), + case( + 2, + "fractional-oldest-share", + "The same shares with network difficulty 9: 72 = 50 + 22, so the oldest counted share counts 22 of its 30.", + w, + window(boundary_shares(), found(9, 1003)), + ), + case( + 2, + "oldest-share-counts-one-unit", + "Weight 8 after a 7-difficulty share leaves exactly one unit for a 1000-difficulty share.", + w, + window( + vec![ + share(1, "miner-a", "01", 1, 1000, 1000), + share(2, "miner-b", "02", 2, 7, 1001), + ], + found(1, 1001), + ), + ), + case( + 1, + "small-log-below-weight", + "The whole eligible log (60) is below the requested weight (800), so every share counts in full.", + w, + window( + vec![ + share(1, "miner-a", "01", 1, 10, 1000), + share(2, "miner-b", "02", 2, 20, 1001), + share(3, "miner-a", "01", 1, 30, 1002), + ], + found(100, 1002), + ), + ), + case( + 1, + "shares-after-anchor-excluded", + "A share whose job was issued after the anchor, and one issued before it but accepted after it, are both excluded.", + w, + window( + vec![ + share(1, "miner-a", "01", 1, 10, 1000), + share(2, "miner-b", "02", 2, 20, 1001), + after_anchor, + accepted_after_anchor, + ], + found(10, 1001), + ), + ), + case( + 1, + "unsorted-input", + "The boundary shares in scrambled sequence order give the same newest-first window as sorted input.", + w, + window(unsorted, found(9, 1003)), + ), + case( + 1, + "zero-network-difficulty-is-an-error", + "A found block with network difficulty 0 cannot define a window.", + w, + window(boundary_shares(), found(0, 1003)), + ), + case( + 1, + "no-eligible-share-is-an-error", + "Every share is after the anchor, so the window is empty.", + w, + window(vec![share(1, "miner-a", "01", 1, 10, 2000)], found(10, 1000)), + ), + ] +} + +// --------------------------------------------------------------------------- +// Topics 3 and 4: apply_payout_policy + +fn reward(coinbase_value_sats: u64, entitlements: &[(&str, &str, String, u128)]) -> PrismRewardManifest { + PrismRewardManifest { + schema: "qbit.prism.reward-manifest.v1".to_string(), + block_height: 10, + coinbase_value_sats, + network_difficulty: 1, + window_multiplier: PRISM_WINDOW_MULTIPLIER, + requested_window_weight: 8, + counted_window_weight: entitlements.iter().map(|entry| entry.3).sum(), + anchor_job_issued_at_ms: 1, + anchor_share_seq: 1, + newest_share_seq: 1, + oldest_share_seq: 1, + included_share_count: entitlements.len(), + share_slice_digest_hex: "00".repeat(32), + shares: Vec::new(), + entitlements: entitlements + .iter() + .map(|(recipient_id, order_key, p2mr_program_hex, weight)| WeightedEntitlement { + recipient_id: recipient_id.to_string(), + order_key: order_key.to_string(), + p2mr_program_hex: p2mr_program_hex.clone(), + weight: *weight, + }) + .collect(), + } +} + +fn prior(recipient_id: &str, order_key: &str, p2mr_program_hex: String, balance_sats: i128) -> CarryForwardBalance { + CarryForwardBalance { + recipient_id: recipient_id.to_string(), + order_key: order_key.to_string(), + p2mr_program_hex, + balance_sats, + } +} + +fn fee_policy(fee_bps: u16, output_policy: CoinbaseOutputPolicy) -> PayoutPolicy { + let mut policy = PayoutPolicy::day_one_default(); + policy.pool_fee_policy = Some(PoolFeePolicy { + fee_bps, + recipient_id: "pool-fee".to_string(), + order_key: "99".to_string(), + p2mr_program_hex: program(0x99), + }); + policy.coinbase_output_policy = output_policy; + policy +} + +fn policy_input( + reward_manifest: PrismRewardManifest, + prior_balances: Vec, + policy: PayoutPolicy, +) -> Value { + json!({ + "reward_manifest": to_value(&reward_manifest), + "prior_balances": to_value(&prior_balances), + "policy": to_value(&policy), + }) +} + +fn carry_only_cases() -> Vec { + let p = "apply_payout_policy"; + let day_one = PayoutPolicy::day_one_default; + let one_miner = || reward(500_000_000, &[("miner-a", "01", program(1), 1)]); + let alias_upper = program(0xab).to_uppercase(); + vec![ + case( + 3, + "prior-only-account-crosses-floor", + "An account with no share in the window but a 15000-sat carry (floor 14720) is a payout candidate on its own.", + p, + policy_input(one_miner(), vec![prior("carry-only", "02", program(2), 15_000)], day_one()), + ), + case( + 3, + "prior-only-account-exactly-at-floor", + "A carry of exactly 14720 is eligible, but its proportional share of the coinbase can land a sat under the floor.", + p, + policy_input(one_miner(), vec![prior("carry-only", "02", program(2), 14_720)], day_one()), + ), + case( + 3, + "prior-only-account-one-sat-under-floor", + "A carry of 14719 is not eligible and keeps accruing; the miner is paid the whole coinbase.", + p, + policy_input(one_miner(), vec![prior("carry-only", "02", program(2), 14_719)], day_one()), + ), + case( + 3, + "prior-plus-gross-crosses-floor", + "A 5000-sat gross plus a 12000-sat carry crosses the floor and is paid on-chain.", + p, + policy_input( + reward(100_000, &[("large", "01", program(1), 95), ("small", "02", program(2), 5)]), + vec![prior("small", "02", program(2), 12_000)], + day_one(), + ), + ), + case( + 3, + "aliases-of-one-program-aggregate", + "Two recipient ids, and carry rows in upper- and lower-case hex, for one payout program aggregate into one account before floor selection.", + p, + policy_input( + reward( + 100_000, + &[ + ("miner-a", "01", program(1), 90), + ("alias-x", "05", program(0xab), 5), + ("alias-y", "03", alias_upper.clone(), 5), + ], + ), + vec![ + prior("alias-y", "03", alias_upper, 4_000), + prior("alias-z", "07", program(0xab), 3_000), + ], + day_one(), + ), + ), + case( + 3, + "overpayment-debt-reduces-candidate", + "A negative carry (overpayment debt after a reorg) reduces that account's candidate balance and its payout.", + p, + policy_input( + reward(100_000, &[("miner-a", "01", program(1), 50), ("miner-b", "02", program(2), 50)]), + vec![ + prior("miner-a", "01", program(1), 10_000), + prior("miner-b", "02", program(2), -5_000), + ], + day_one(), + ), + ), + case( + 3, + "overpayment-debt-without-fee-is-an-error", + "With debt and no other carry the eligible candidates sum below the coinbase, and without a pool fee to absorb it the policy fails.", + p, + policy_input( + reward(100_000, &[("miner-a", "01", program(1), 50), ("miner-b", "02", program(2), 50)]), + vec![prior("miner-b", "02", program(2), -5_000)], + day_one(), + ), + ), + ] +} + +fn pool_fee_cases() -> Vec { + let p = "apply_payout_policy"; + let two_miners = |coinbase| { + reward(coinbase, &[("miner-a", "01", program(1), 1), ("miner-b", "02", program(2), 2)]) + }; + vec![ + case( + 4, + "fee-125-bps-non-divisible-coinbase", + "100003 x 125 / 10000 = 1250.0375 rounds down to 1250; the miners split the rest 1:2 with the remainder sat by largest remainder.", + p, + policy_input(two_miners(100_003), vec![], fee_policy(125, CoinbaseOutputPolicy::Canonical)), + ), + case( + 4, + "fee-125-bps-three-equal-miners", + "500000003 x 125 / 10000 rounds down to 6250000; 493750003 split three ways leaves one remainder sat for the first account in canonical order.", + p, + policy_input( + reward( + 500_000_003, + &[ + ("miner-a", "01", program(1), 1), + ("miner-b", "02", program(2), 1), + ("miner-c", "03", program(3), 1), + ], + ), + vec![], + fee_policy(125, CoinbaseOutputPolicy::Canonical), + ), + ), + case( + 4, + "fee-0-bps-sweeps-sub-floor-dust", + "A zero-bps fee earns nothing, but the sub-floor miner's allocation is swept into the pool-fee output.", + p, + policy_input( + reward(100_000, &[("miner-a", "01", program(1), 99_999), ("dust", "02", program(2), 1)]), + vec![], + fee_policy(0, CoinbaseOutputPolicy::Canonical), + ), + ), + case( + 4, + "fee-first-with-sub-floor-fee", + "Pool-fee-first with a 10-sat fee (1 bps of 100000): the fee stays an on-chain account even though it is below the 14720 floor.", + p, + policy_input(two_miners(100_000), vec![], fee_policy(1, CoinbaseOutputPolicy::PoolFeeFirst)), + ), + case( + 4, + "fee-first-without-pool-fee-is-an-error", + "Pool-fee-first with no pool-fee policy cannot pin a fee output.", + p, + policy_input( + two_miners(100_000), + vec![], + PayoutPolicy { + coinbase_output_policy: CoinbaseOutputPolicy::PoolFeeFirst, + ..PayoutPolicy::day_one_default() + }, + ), + ), + case( + 4, + "fee-10000-bps-leaves-miners-below-floor", + "At the 10000-bps cap the fee is the whole coinbase and the miner reward (0) is below the floor.", + p, + policy_input(two_miners(100_000), vec![], fee_policy(10_000, CoinbaseOutputPolicy::Canonical)), + ), + case( + 4, + "fee-10001-bps-is-an-error", + "One basis point over 100% is rejected.", + p, + policy_input(two_miners(100_000), vec![], fee_policy(10_001, CoinbaseOutputPolicy::Canonical)), + ), + case( + 4, + "fee-1-bps-floors-to-zero", + "1 bps of a 9999-sat coinbase is 0.9999 sats and floors to a zero pool fee; with a 1000-sat fixed floor the miners take the whole coinbase, and the fee account is recorded with 0 sats but gets no coinbase output.", + p, + policy_input(two_miners(9_999), vec![], fixed_floor(fee_policy(1, CoinbaseOutputPolicy::Canonical))), + ), + case( + 4, + "fee-first-1-bps-floors-to-zero", + "The same zero pool fee under pool-fee-first: the fee account is recorded with 0 sats, and no zero-amount fee output is pinned.", + p, + policy_input(two_miners(9_999), vec![], fixed_floor(fee_policy(1, CoinbaseOutputPolicy::PoolFeeFirst))), + ), + case( + 4, + "fee-1-bps-floors-to-zero-under-day-one-floor", + "The same zero pool fee with the day-one 14720-sat floor: the 9999-sat coinbase is below the floor, so the policy fails.", + p, + policy_input(two_miners(9_999), vec![], fee_policy(1, CoinbaseOutputPolicy::Canonical)), + ), + ] +} + +/// A 1000-sat fixed floor, so a coinbase small enough for a 1-bps fee to +/// floor to zero still leaves the miners payable. +fn fixed_floor(mut policy: PayoutPolicy) -> PayoutPolicy { + policy.min_output_sats = Some(1_000); + policy +} + +// --------------------------------------------------------------------------- +// Topics 5 and 6: apply_proportional_fanout_fee and largest-remainder ties + +fn recipient(recipient_id: &str, order_key: &str, byte: u8, amount_sats: u64) -> SettlementRecipient { + SettlementRecipient { + recipient_id: recipient_id.to_string(), + order_key: order_key.to_string(), + p2mr_program_hex: program(byte), + amount_sats, + } +} + +fn fee_input(recipients: Vec, fee_sats: u64, min_output_sats: u64) -> Value { + json!({ + "recipients": to_value(&recipients), + "fee_sats": fee_sats, + "min_output_sats": min_output_sats, + }) +} + +fn fanout_fee_cases() -> Vec { + let f = "apply_proportional_fanout_fee"; + let at_floor = || vec![recipient("large", "01", 1, 100_000), recipient("small", "02", 2, 10_000)]; + vec![ + case( + 5, + "prune-below-floor-and-recompute", + "Both small recipients fall under 9950 after their fee share; they carry forward fee-free and the whole fee is recomputed over the large one.", + f, + fee_input( + vec![ + recipient("small-a", "01", 1, 10_000), + recipient("small-b", "02", 2, 10_000), + recipient("large", "03", 3, 100_000), + ], + 1_000, + 9_950, + ), + ), + case( + 5, + "net-exactly-at-floor-is-payable", + "The small recipient's fee share is exactly 100, leaving 9900: at the floor, so it is paid.", + f, + fee_input(at_floor(), 1_100, 9_900), + ), + case( + 5, + "net-one-sat-under-floor-is-carried", + "The same split with floor 9901 leaves the small recipient one sat under, so it carries and the large one pays the whole fee.", + f, + fee_input(at_floor(), 1_100, 9_901), + ), + case( + 5, + "all-carry-when-no-recipient-survives", + "The only recipient falls under the floor, so every recipient carries and no fee is applied.", + f, + fee_input(vec![recipient("tiny", "01", 1, 10_000)], 1_000, 9_500), + ), + case( + 5, + "zero-floor-is-an-error", + "min_output_sats must be positive.", + f, + fee_input(vec![recipient("tiny", "01", 1, 10_000)], 1_000, 0), + ), + case( + 6, + "equal-remainders-broken-by-canonical-order", + "Three equal recipients share a 10-sat fee: 3 each plus one remainder sat, which goes to the first account in canonical order regardless of input order.", + f, + fee_input( + vec![ + recipient("c", "03", 3, 100), + recipient("a", "01", 1, 100), + recipient("b", "02", 2, 100), + ], + 10, + 1, + ), + ), + case( + 6, + "equal-remainders-order-key-before-recipient-id", + "The canonical tie-break compares order_key before recipient_id, so the remainder sat goes to order key 01 even though its id sorts last.", + f, + fee_input(vec![recipient("a-miner", "02", 2, 100), recipient("z-miner", "01", 1, 100)], 1, 1), + ), + case( + 6, + "three-way-split-of-500000000", + "500000000 over three equal weights is 166666666 each plus two remainder sats for the first two accounts in canonical order.", + "apply_payout_policy", + policy_input( + reward( + 500_000_000, + &[ + ("miner-c", "03", program(3), 1), + ("miner-a", "01", program(1), 1), + ("miner-b", "02", program(2), 1), + ], + ), + vec![], + PayoutPolicy::day_one_default(), + ), + ), + ] +} + +// --------------------------------------------------------------------------- +// Topic 7: select_settlement_mode + +fn settlement_config( + max_coinbase_settlement_outputs: usize, + max_direct_coinbase_outputs: usize, + max_fanout_recipients_per_transaction: usize, +) -> SettlementModeConfig { + SettlementModeConfig { + max_coinbase_settlement_outputs, + max_direct_coinbase_outputs, + max_fanout_recipients_per_transaction, + reserved_coinbase_outputs: 0, + } +} + +fn settlement_input( + recipients: Vec, + direct_floor_sats: u64, + config: SettlementModeConfig, + pinned_direct: Option, +) -> Value { + json!({ + "recipients": to_value(&recipients), + "direct_floor_sats": direct_floor_sats, + "config": to_value(&config), + "pinned_direct": to_value(&pinned_direct), + }) +} + +fn dust(count: u8) -> Vec { + (1..=count) + .map(|index| recipient(&format!("dust-{index}"), &format!("{index:02}"), index, 1_000)) + .collect() +} + +fn settlement_cases() -> Vec { + let s = "select_settlement_mode"; + let floor = 20_000; + let pinned = recipient("pool-fee", "ff", 0xff, 25); + vec![ + case( + 7, + "all-sub-floor-into-bounded-chunks", + "Five sub-floor recipients with at most two per fanout become three chunks, filled in canonical order.", + s, + settlement_input(dust(5), floor, settlement_config(16, 12, 2), None), + ), + case( + 7, + "hybrid-largest-direct-and-chunks", + "Two direct slots go to the largest floor-crossing liabilities; the overflow above the floor and the dust are chunked.", + s, + settlement_input( + vec![ + recipient("dust-b", "05", 5, 4_000), + recipient("direct-b", "02", 2, 40_000), + recipient("overflow", "03", 3, 30_000), + recipient("direct-a", "01", 1, 50_000), + recipient("dust-a", "04", 4, 5_000), + ], + floor, + settlement_config(500, 2, 2), + None, + ), + ), + case( + 7, + "direct-floor-boundary", + "One sat under the direct floor routes to fanout; exactly at and above it are direct.", + s, + settlement_input( + vec![ + recipient("below", "01", 1, floor - 1), + recipient("at", "02", 2, floor), + recipient("above", "03", 3, floor + 1), + ], + floor, + settlement_config(500, 10, 10), + None, + ), + ), + case( + 7, + "chunks-at-the-output-cap", + "Six sub-floor recipients at two per chunk need exactly the three-output coinbase budget.", + s, + settlement_input(dust(6), floor, settlement_config(3, 0, 2), None), + ), + case( + 7, + "chunks-one-over-the-output-cap-is-an-error", + "A seventh recipient needs a fourth chunk, one more than the three-output budget.", + s, + settlement_input(dust(7), floor, settlement_config(3, 0, 2), None), + ), + case( + 7, + "fanout-size-at-the-truc-cap", + "1160 recipients per fanout (49973 weight bytes) is the largest chunk size under the 50000-byte TRUC limit.", + s, + settlement_input(dust(3), floor, settlement_config(16, 12, 1_160), None), + ), + case( + 7, + "fanout-size-one-over-the-truc-cap-is-an-error", + "1161 recipients per fanout exceeds the per-transaction maximum.", + s, + settlement_input(dust(3), floor, settlement_config(16, 12, 1_161), None), + ), + case( + 7, + "pinned-direct-below-floor", + "A 25-sat pinned pool fee is settled directly, takes one of the two direct slots, and never enters a fanout chunk.", + s, + settlement_input( + vec![ + recipient("miner-a", "01", 1, 30_000), + recipient("miner-b", "02", 2, 1_000), + pinned.clone(), + ], + floor, + settlement_config(500, 2, 1_000), + Some(pinned.clone()), + ), + ), + case( + 7, + "pinned-direct-without-a-direct-slot-is-an-error", + "A pinned recipient needs max_direct_coinbase_outputs of at least one.", + s, + settlement_input( + vec![recipient("miner-a", "01", 1, 30_000), pinned.clone()], + floor, + settlement_config(500, 0, 1_000), + Some(pinned), + ), + ), + ] +} + +// --------------------------------------------------------------------------- +// Topic 8: bootstrap transition + +fn bootstrap_cases(decisions: &Value) -> Vec { + decisions["bootstrap"] + .as_array() + .expect("bootstrap decisions") + .iter() + .map(|exported| { + let name = exported["name"].as_str().expect("name"); + let scenario = &exported["scenario"]; + let mut decision_2xx = exported["decision_2xx"].clone(); + let mut bundle_2xx = decision_2xx + .as_object_mut() + .expect("decision object") + .remove("bundle") + .expect("2.x.x bundle inputs"); + bundle_2xx["policy"] = to_value(&PayoutPolicy::day_one_default()); + + // 3.x.x rule: the solver-only bundle is used only when the ledger + // snapshot has no shares, and prior balances are always kept. + let ledger = field::>(scenario, "ledger_shares"); + let template = &scenario["template"]; + let solver = &scenario["solver"]; + let network: u128 = field(scenario, "network_difficulty"); + let anchor: i64 = field(scenario, "anchor_ms"); + let found_block = FoundBlock { + block_height: field(template, "height"), + coinbase_value_sats: field(template, "coinbasevalue"), + network_difficulty: network, + anchor_job_issued_at_ms: anchor, + }; + let bootstrap = ledger.is_empty(); + let shares = if bootstrap { + let payout_address: String = field(solver, "payout_address"); + vec![AcceptedShare { + share_seq: 1, + share_id: "bootstrap-share".to_string(), + miner_id: payout_address.clone(), + order_key: payout_address, + p2mr_program_hex: field(solver, "p2mr_program_hex"), + share_difficulty: network, + network_difficulty: network, + template_height: found_block.block_height - 1, + job_id: "bootstrap-job".to_string(), + job_issued_at_ms: anchor, + accepted_at_ms: anchor, + ntime: field(template, "curtime"), + credit_policy: None, + }] + } else { + ledger + }; + let bundle_3xx = json!({ + "shares": to_value(&shares), + "found_block": to_value(&found_block), + "prior_balances": scenario["prior_balances"].clone(), + "policy": to_value(&PayoutPolicy::day_one_default()), + }); + let decision_3xx = json!({ + "bootstrap": bootstrap, + "bundle_built_by": if bootstrap { + "synthetic solver share (crates/qbit-prism-server/src/coordinator.rs:727)" + } else { + "ledger snapshot window (crates/qbit-prism-server/src/coordinator.rs:747)" + }, + "prior_balances_kept": true, + "implemented_at": [ + "crates/qbit-prism-server/src/coordinator.rs:556 solver-only bundle only when snapshot.shares is empty", + "crates/qbit-prism-server/src/coordinator.rs:759,773 snapshot.prior_balances passed in bootstrap", + ], + }); + let d2_entry = match name { + "below-gate-with-other-miners-shares" => Some(D2A), + "bootstrap-carry-only-account-at-or-above-floor" => Some(D2C), + _ => None, + }; + rule_case( + 8, + name, + exported["why"].as_str().expect("why"), + json!({ "scenario": scenario, "decision_2xx": decision_2xx, "decision_3xx": decision_3xx }), + bundle_2xx, + bundle_3xx, + d2_entry, + ) + }) + .collect() +} + +// --------------------------------------------------------------------------- +// Topic 9: below-target block credit + +fn credit_cases(decisions: &Value) -> Vec { + decisions["below_target_credit"] + .as_array() + .expect("below-target decisions") + .iter() + .map(|exported| { + let name = exported["name"].as_str().expect("name"); + let scenario = &exported["scenario"]; + let decision_2xx = exported["decision_2xx"].clone(); + let ledger = field::>(scenario, "ledger_shares"); + let next_block = field::(scenario, "next_block"); + let solver = &scenario["solver"]; + let credited = &scenario["credited_share"]; + let assigned: u128 = field(&decision_2xx, "assigned_share_difficulty"); + let network: u128 = field(&decision_2xx, "network_difficulty"); + let share_pass: bool = field(scenario, "share_pass"); + let block_pass: bool = field(scenario, "block_pass"); + let node_outcome: String = field(scenario, "node_outcome"); + let credited_share = |share_difficulty: u128| { + let payout_address: String = field(solver, "payout_address"); + AcceptedShare { + share_seq: field(credited, "share_seq"), + share_id: field(credited, "share_id"), + miner_id: payout_address.clone(), + order_key: payout_address, + p2mr_program_hex: field(solver, "p2mr_program_hex"), + share_difficulty, + network_difficulty: network, + template_height: next_block.block_height - 2, + job_id: "job-below-target".to_string(), + job_issued_at_ms: field(credited, "job_issued_at_ms"), + accepted_at_ms: field(credited, "accepted_at_ms"), + ntime: 1_800_000_000, + credit_policy: None, + } + }; + let bundle = |credit: Option| { + let mut shares = ledger.clone(); + shares.extend(credit.map(credited_share)); + json!({ + "shares": to_value(&shares), + "found_block": to_value(&next_block), + "prior_balances": [], + "policy": to_value(&PayoutPolicy::day_one_default()), + }) + }; + let credit_2xx = if field::(&decision_2xx, "credited") { + Some(field::(&decision_2xx, "credited_difficulty")) + } else { + None + }; + + // 3.x.x rule: a share-passing proof is credited at its share + // target; a block-only proof earns network difficulty, and only + // once its block is confirmed on the active chain. + let (credit_3xx, credited_at, implemented_at): (Option, &str, Vec<&str>) = + if share_pass { + ( + Some(assigned), + "share acceptance (share passed its target)", + vec!["crates/qbit-prism-server/src/coordinator.rs:1649-1651 share target difficulty when the share passes"], + ) + } else if block_pass { + let rule = vec![ + "crates/qbit-prism-server/src/coordinator.rs:1648-1654 network difficulty for a block-only proof", + "crates/qbit-prism-server/src/coordinator.rs:1698 deferred_share held with the block candidate", + "crates/qbit-prism-server/src/coordinator.rs:1726-1747 submit waits for the credit row or fails", + ]; + match node_outcome.as_str() { + "accepted" => ( + Some(network), + "confirmation transaction after node acceptance; the submit acknowledgement waits for it", + [rule, vec!["crates/qbit-prism-server/src/ledger/blocks.rs:200 credit_deferred_share on confirmation"]].concat(), + ), + "confirmed-after-reconciliation" => ( + Some(network), + "reconciliation's confirmation transaction", + [rule, vec!["crates/qbit-prism-server/src/ledger/blocks.rs:295 credit_deferred_share on reconciled confirmation"]].concat(), + ), + "reorged-after-acceptance" => ( + Some(network), + "confirmation transaction; the share row survives the later disconnect", + [rule, vec!["crates/qbit-prism-server/src/ledger/blocks.rs:200 credit_deferred_share on confirmation; no server statement updates or deletes qbit_share_ledger"]].concat(), + ), + "rejected" => ( + None, + "never: the submission fails with block-only proof was not accepted on the active chain", + rule, + ), + other => panic!("unknown node outcome {other}"), + } + } else { + (None, "never: low difficulty share", vec![]) + }; + let decision_3xx = json!({ + "credited": credit_3xx.is_some(), + "credited_difficulty": credit_3xx.map(|value| to_value(&value)), + "credited_at": credited_at, + "implemented_at": implemented_at, + }); + let d2_entry = (credit_2xx != credit_3xx).then_some(D2B); + rule_case( + 9, + name, + exported["why"].as_str().expect("why"), + json!({ "scenario": scenario, "decision_2xx": decision_2xx, "decision_3xx": decision_3xx }), + bundle(credit_2xx), + bundle(credit_3xx), + d2_entry, + ) + }) + .collect() +} + +fn write(dir: &Path, file: &str, topics: &[(u8, &str)], cases: Vec) { + let document = json!({ + "schema": SCHEMA, + "source_commit": SOURCE_COMMIT, + "produced_by": PRODUCED_BY, + "topics": topics + .iter() + .map(|(id, name)| json!({ "id": id, "name": name })) + .collect::>(), + "cases": cases, + }); + let mut text = serde_json::to_string_pretty(&document).expect("vector document"); + text.push('\n'); + fs::write(dir.join(file), text).expect("write vector file"); +} + +fn main() { + let dir = std::env::args() + .nth(1) + .expect("usage: export_money_path_vectors < rule_decisions.json"); + let dir = Path::new(&dir); + let mut raw = String::new(); + std::io::stdin() + .read_to_string(&mut raw) + .expect("read rule decisions"); + let decisions: Value = serde_json::from_str(&raw).expect("rule decisions JSON"); + fs::create_dir_all(dir).expect("create output directory"); + write( + dir, + "window.json", + &[ + (1, "window clipping and the oldest-share boundary"), + (2, "fractional oldest share"), + ], + window_cases(), + ); + write(dir, "carry_only.json", &[(3, "carry-only recipients")], carry_only_cases()); + write( + dir, + "pool_fee.json", + &[(4, "pool fee: bps rounding down, remainder split, fee-first pinning")], + pool_fee_cases(), + ); + write( + dir, + "fanout_fee_and_ties.json", + &[ + (5, "dust floor exclusion and recompute"), + (6, "largest-remainder ties"), + ], + fanout_fee_cases(), + ); + write(dir, "settlement_chunks.json", &[(7, "CTV chunk splits")], settlement_cases()); + write( + dir, + "bootstrap_transition.json", + &[(8, "bootstrap transition")], + bootstrap_cases(&decisions), + ); + write( + dir, + "below_target_credit.json", + &[(9, "below-target block credit")], + credit_cases(&decisions), + ); +} diff --git a/crates/qbit-prism/fixtures/vectors/exporter/export_rule_decisions.py b/crates/qbit-prism/fixtures/vectors/exporter/export_rule_decisions.py new file mode 100644 index 000000000..e9e6df9a6 --- /dev/null +++ b/crates/qbit-prism/fixtures/vectors/exporter/export_rule_decisions.py @@ -0,0 +1,435 @@ +"""Export the 2.x.x bootstrap and below-target credit rule decisions. + +Run from the root of a 2.x.x source tree (see README.md in this directory). +Each scenario is fed through the real 2.x.x Python decision code with +in-memory stand-ins for the ledger and bundle builder, so no database or +psycopg import is needed. The JSON printed on stdout is consumed by +export_money_path_vectors.rs, which computes the payout consequence of every +decision with the qbit-prism engine. + +Facts that the 2.x.x code only establishes against a live node or database +(credit timing, what happens after a reorg) are recorded as cited rule-table +entries rather than executed. +""" + +from __future__ import annotations + +import json +import sys +import threading +import types +from typing import Any + +from lab.prism.coordinator_config import env_int +from lab.prism.job_bundle import JobBundleService +from lab.prism.prism_coordinator import PrismCoordinator +from lab.prism.share_submission import SubmitRejected, classify_submit_work +from lab.prism.template_artifacts import scaled_target_difficulty, target_from_compact + +POW_LIMIT_TARGET = target_from_compact("207fffff") + + +def program(byte: int) -> str: + return f"{byte:02x}" * 32 + + +def ledger_share( + share_seq: int, miner: str, byte: int, difficulty: int, at_ms: int, network: int +) -> dict[str, Any]: + return { + "share_seq": share_seq, + "share_id": f"share-{share_seq}", + "miner_id": miner, + "order_key": miner, + "p2mr_program_hex": program(byte), + "share_difficulty": difficulty, + "network_difficulty": network, + "template_height": 100, + "job_id": f"job-{at_ms}", + "job_issued_at_ms": at_ms, + "accepted_at_ms": at_ms, + "ntime": 1_800_000_000, + } + + +# --------------------------------------------------------------------------- +# Topic 8: bootstrap transition + + +class _Ledger: + """Share ledger without an aggregate query: accepted_share_stats falls back + to distinct miner_id over all_shares, as the in-memory 2.x.x ledger does.""" + + def __init__(self, shares: list[dict[str, Any]]) -> None: + self._shares = [types.SimpleNamespace(**share) for share in shares] + + def all_shares(self) -> list[Any]: + return list(self._shares) + + +class _CaptureBundleRuntime: + def build_audit_bundle(self, **kwargs: Any) -> dict[str, Any]: + return kwargs + + +def bootstrap_decision(scenario: dict[str, Any]) -> dict[str, Any]: + coordinator = types.SimpleNamespace( + ledger=_Ledger(scenario["ledger_shares"]), + min_ready_miners=scenario["min_ready_miners"], + lock=threading.Lock(), + _pool_ready_latched=False, + _retained_collection_refresh=None, + _progress_note_refresh_pending=lambda: None, + ) + coordinator.accepted_share_stats = types.MethodType( + PrismCoordinator.accepted_share_stats, coordinator + ) + coordinator.pool_readiness_latched = types.MethodType( + PrismCoordinator.pool_readiness_latched, coordinator + ) + stats = coordinator.accepted_share_stats() + service = object.__new__(JobBundleService) + service._runtime = coordinator + mode = JobBundleService._job_bundle_mode(service, None) + + template = scenario["template"] + solver = scenario["solver"] + if mode == "collection": + capture = object.__new__(JobBundleService) + capture._runtime = _CaptureBundleRuntime() + captured = JobBundleService.build_collection_bundle( + capture, + template=template, + transaction_hexes=(), + worker=types.SimpleNamespace( + payout_address=solver["payout_address"], + p2mr_program_hex=solver["p2mr_program_hex"], + ), + network_difficulty=scenario["network_difficulty"], + issued_at_ms=scenario["anchor_ms"], + suffix_hex="", + ) + bundle = { + "shares": captured["shares"], + "found_block": captured["found_block"], + "prior_balances": captured["prior_balances"], + } + built_by = "lab/prism/job_bundle.py:3550 build_collection_bundle (executed)" + else: + # The ready path builds from the ledger window and the published + # prior balances (lab/prism/job_bundle.py:3400-3408); it needs the + # ledger service, so its economic inputs are restated here. + bundle = { + "shares": scenario["ledger_shares"], + "found_block": { + "block_height": int(template["height"]), + "coinbase_value_sats": int(template["coinbasevalue"]), + "network_difficulty": scenario["network_difficulty"], + "anchor_job_issued_at_ms": scenario["anchor_ms"], + }, + "prior_balances": scenario["prior_balances"], + } + built_by = "lab/prism/job_bundle.py:3400 ready build (ledger window and prior balances)" + return { + "accepted_share_count": stats[0], + "distinct_miner_count": stats[1], + "pool_readiness_latched": coordinator._pool_ready_latched, + "job_bundle_mode": mode, + "bundle_built_by": built_by, + "implemented_at": [ + "lab/prism/prism_coordinator.py:7778 pool_readiness_latched", + "lab/prism/job_bundle.py:2132 _job_bundle_mode", + "lab/prism/job_bundle.py:3589 prior_balances=[] in build_collection_bundle", + ], + "bundle": bundle, + } + + +def bootstrap_cases() -> list[dict[str, Any]]: + network = 100 + anchor = 5_000 + min_ready = env_int("PRISM_MIN_READY_MINERS", 3, environ={}) + template = { + "height": 101, + "coinbasevalue": 500_000_000, + "curtime": 1_800_000_000, + "previousblockhash": "00" * 32, + } + solver_b = {"payout_address": "miner-b", "p2mr_program_hex": program(2)} + two_miners = [ + ledger_share(1, "miner-a", 1, 30, 4_000, network), + ledger_share(2, "miner-b", 2, 20, 4_001, network), + ] + three_miners = two_miners + [ledger_share(3, "miner-c", 3, 50, 4_002, network)] + carry_only = [ + { + "recipient_id": "miner-old", + "order_key": "miner-old", + "p2mr_program_hex": program(9), + "balance_sats": 20_000, + } + ] + scenarios = [ + { + "name": "below-gate-with-other-miners-shares", + "why": ( + "Two distinct miners have accepted shares, below the default " + "readiness gate of three. 2.x.x issues a collection job that " + "pays the solver the whole coinbase; 3.x.x uses the normal " + "window as soon as any share exists." + ), + "ledger_shares": two_miners, + "prior_balances": [], + }, + { + "name": "at-readiness-gate", + "why": ( + "Three distinct miners reach the gate, so 2.x.x latches ready " + "and both versions build the normal window with prior balances." + ), + "ledger_shares": three_miners, + "prior_balances": carry_only, + }, + { + "name": "empty-ledger", + "why": ( + "No shares and no balances: both versions pay the solver the " + "whole coinbase through the synthetic bootstrap share." + ), + "ledger_shares": [], + "prior_balances": [], + }, + { + "name": "bootstrap-carry-only-account-at-or-above-floor", + "why": ( + "An empty share window with a carry-only account whose carry " + "(20000 sats) is above the 14720-sat floor. 2.x.x drops prior " + "balances from the collection bundle, so the account is not paid " + "in this block; 3.x.x keeps the snapshot's prior balances in " + "bootstrap and pays it." + ), + "ledger_shares": [], + "prior_balances": carry_only, + }, + ] + cases = [] + for scenario in scenarios: + full = { + "min_ready_miners": min_ready, + "network_difficulty": network, + "anchor_ms": anchor, + "template": template, + "solver": solver_b, + "ledger_shares": scenario["ledger_shares"], + "prior_balances": scenario["prior_balances"], + } + cases.append( + { + "name": scenario["name"], + "why": scenario["why"], + "scenario": full, + "decision_2xx": bootstrap_decision(full), + } + ) + return cases + + +# --------------------------------------------------------------------------- +# Topic 9: below-target block credit + + +def accepted_share_difficulty(share_target: int) -> int: + coordinator = types.SimpleNamespace(share_weights_by_username={}) + context = types.SimpleNamespace( + worker=types.SimpleNamespace(username="miner-a.rig", payout_address="miner-a"), + job=types.SimpleNamespace(share_target=share_target), + ) + return PrismCoordinator.accepted_share_difficulty(coordinator, context) + + +# Node and chain outcomes, with what 2.x.x does after the submit route. These +# paths run against the node and the database, so they are cited, not run. +OUTCOMES_2XX = { + "accepted": { + "credited": True, + "credited_at": "node acceptance of the block (synchronous submit)", + "implemented_at": [ + "lab/prism/prism_coordinator.py:8155 _submit_synchronous_credit_candidate", + "lab/prism/block_finalization.py:2027 append_accepted_share when credit_share_on_accept", + ], + }, + "confirmed-after-reconciliation": { + "credited": True, + "credited_at": "node acceptance of the block; later confirmation adds nothing", + "implemented_at": [ + "lab/prism/block_finalization.py:2027 append_accepted_share when credit_share_on_accept", + ], + }, + "reorged-after-acceptance": { + "credited": True, + "credited_at": "node acceptance; the share row survives the disconnect", + "implemented_at": [ + "lab/prism/block_finalization.py:2027 append_accepted_share when credit_share_on_accept", + "lab/prism/accepted_preview_telemetry.py:144 no reorg path updates or deletes qbit_share_ledger", + ], + }, + "rejected": { + "credited": False, + "credited_at": None, + "implemented_at": [ + "lab/prism/prism_coordinator.py:8221 block_landed false rejects low difficulty share", + ], + }, +} + + +def credit_decision(scenario: dict[str, Any]) -> dict[str, Any]: + submission = types.SimpleNamespace( + share_pass=scenario["share_pass"], + block_pass=scenario["block_pass"], + header_hex="00" * 80, + ) + context = types.SimpleNamespace(worker=types.SimpleNamespace(username="miner-a.rig")) + assigned = accepted_share_difficulty(scenario["share_target"]) + network = scaled_target_difficulty(scenario["network_target"]) + try: + decision = classify_submit_work(context, submission, credit_policy=None) + except SubmitRejected as rejected: + return { + "route": None, + "rejection": {"code": rejected.code, "reason": rejected.reason}, + "assigned_share_difficulty": assigned, + "network_difficulty": network, + "credited": False, + "credited_difficulty": None, + "credited_at": None, + "implemented_at": ["lab/prism/share_submission.py:360 classify_submit_work"], + } + outcome = OUTCOMES_2XX[scenario["node_outcome"]] + credited = decision.route != "synchronous_block" or outcome["credited"] + return { + "route": decision.route, + "block_worthy": decision.block_worthy, + "credit_share_on_accept": decision.credit_share_on_accept, + "rejection": None, + "assigned_share_difficulty": assigned, + "network_difficulty": network, + "credited": credited, + # 2.x.x stamps every accepted share, synchronous or not, with the + # assigned share difficulty (lab/prism/share_writer.py:281). + "credited_difficulty": assigned if credited else None, + "credited_at": ( + outcome["credited_at"] + if decision.route == "synchronous_block" + else "share acceptance (share passed its target)" + ), + "implemented_at": [ + "lab/prism/share_submission.py:360 classify_submit_work", + "lab/prism/share_writer.py:281 runtime.accepted_share_difficulty(context)", + "lab/prism/prism_coordinator.py:8410 accepted_share_difficulty", + ] + + (outcome["implemented_at"] if decision.route == "synchronous_block" else []), + } + + +def credit_cases() -> list[dict[str, Any]]: + # Network target at the pow limit (difficulty 1_000_000); the listener + # floor holds the share target four times harder (difficulty 4_000_000). + network_target = POW_LIMIT_TARGET + share_target = POW_LIMIT_TARGET * 1_000_000 // 4_000_000 + next_network = scaled_target_difficulty(network_target) + ledger = [ + ledger_share(1, "miner-b", 2, 2_000_000, 9_000, next_network), + ledger_share(2, "miner-b", 2, 2_000_000, 9_001, next_network), + ledger_share(3, "miner-c", 3, 1_000_000, 9_002, next_network), + ] + common = { + "share_target": share_target, + "network_target": network_target, + "solver": {"payout_address": "miner-a", "p2mr_program_hex": program(1)}, + "ledger_shares": ledger, + "credited_share": { + "share_seq": 4, + "share_id": "miner-a.rig:" + "ab" * 32, + "job_issued_at_ms": 9_003, + "accepted_at_ms": 9_003, + }, + "next_block": { + "block_height": 202, + "coinbase_value_sats": 500_000_000, + "network_difficulty": next_network, + "anchor_job_issued_at_ms": 9_010, + }, + } + scenarios = [ + ( + "block-only-proof-accepted", + "The hash meets the network target but not the floor-raised share " + "target, and the node accepts the block.", + False, + True, + "accepted", + ), + ( + "block-only-proof-confirmed-after-reconciliation", + "The node accepted the block but the durable acknowledgement was " + "lost; reconciliation later finds it on the active chain.", + False, + True, + "confirmed-after-reconciliation", + ), + ( + "block-only-proof-reorged-after-acceptance", + "The block was accepted and credited, then disconnected by a reorg; " + "the next block's window is built afterwards.", + False, + True, + "reorged-after-acceptance", + ), + ( + "block-only-proof-rejected", + "The node rejects the block, or it is never on the active chain: " + "no credit on either version.", + False, + True, + "rejected", + ), + ( + "share-and-block-proof-control", + "Control: the hash also meets the share target, so both versions " + "credit the assigned share difficulty.", + True, + True, + "accepted", + ), + ] + cases = [] + for name, why, share_pass, block_pass, node_outcome in scenarios: + scenario = dict(common) + scenario.update( + {"share_pass": share_pass, "block_pass": block_pass, "node_outcome": node_outcome} + ) + cases.append( + { + "name": name, + "why": why, + "scenario": scenario, + "decision_2xx": credit_decision(scenario), + } + ) + return cases + + +def main() -> int: + json.dump( + {"bootstrap": bootstrap_cases(), "below_target_credit": credit_cases()}, + sys.stdout, + sort_keys=True, + separators=(",", ":"), + ) + sys.stdout.write("\n") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/crates/qbit-prism/fixtures/vectors/fanout_fee_and_ties.json b/crates/qbit-prism/fixtures/vectors/fanout_fee_and_ties.json new file mode 100644 index 000000000..f37553e09 --- /dev/null +++ b/crates/qbit-prism/fixtures/vectors/fanout_fee_and_ties.json @@ -0,0 +1,468 @@ +{ + "cases": [ + { + "entry_point": "apply_proportional_fanout_fee", + "expected": { + "ok": { + "applied_fee_sats": 1000, + "carry_forward_recipients": [ + { + "fee_sats": 0, + "gross_amount_sats": 10000, + "net_amount_sats": 10000, + "order_key": "01", + "p2mr_program_hex": "0101010101010101010101010101010101010101010101010101010101010101", + "recipient_id": "small-a" + }, + { + "fee_sats": 0, + "gross_amount_sats": 10000, + "net_amount_sats": 10000, + "order_key": "02", + "p2mr_program_hex": "0202020202020202020202020202020202020202020202020202020202020202", + "recipient_id": "small-b" + } + ], + "payable_recipients": [ + { + "fee_sats": 1000, + "gross_amount_sats": 100000, + "net_amount_sats": 99000, + "order_key": "03", + "p2mr_program_hex": "0303030303030303030303030303030303030303030303030303030303030303", + "recipient_id": "large" + } + ], + "requested_fee_sats": 1000, + "total_gross_sats": 120000 + } + }, + "input": { + "fee_sats": 1000, + "min_output_sats": 9950, + "recipients": [ + { + "amount_sats": 10000, + "order_key": "01", + "p2mr_program_hex": "0101010101010101010101010101010101010101010101010101010101010101", + "recipient_id": "small-a" + }, + { + "amount_sats": 10000, + "order_key": "02", + "p2mr_program_hex": "0202020202020202020202020202020202020202020202020202020202020202", + "recipient_id": "small-b" + }, + { + "amount_sats": 100000, + "order_key": "03", + "p2mr_program_hex": "0303030303030303030303030303030303030303030303030303030303030303", + "recipient_id": "large" + } + ] + }, + "name": "prune-below-floor-and-recompute", + "topic": 5, + "why": "Both small recipients fall under 9950 after their fee share; they carry forward fee-free and the whole fee is recomputed over the large one." + }, + { + "entry_point": "apply_proportional_fanout_fee", + "expected": { + "ok": { + "applied_fee_sats": 1100, + "carry_forward_recipients": [], + "payable_recipients": [ + { + "fee_sats": 1000, + "gross_amount_sats": 100000, + "net_amount_sats": 99000, + "order_key": "01", + "p2mr_program_hex": "0101010101010101010101010101010101010101010101010101010101010101", + "recipient_id": "large" + }, + { + "fee_sats": 100, + "gross_amount_sats": 10000, + "net_amount_sats": 9900, + "order_key": "02", + "p2mr_program_hex": "0202020202020202020202020202020202020202020202020202020202020202", + "recipient_id": "small" + } + ], + "requested_fee_sats": 1100, + "total_gross_sats": 110000 + } + }, + "input": { + "fee_sats": 1100, + "min_output_sats": 9900, + "recipients": [ + { + "amount_sats": 100000, + "order_key": "01", + "p2mr_program_hex": "0101010101010101010101010101010101010101010101010101010101010101", + "recipient_id": "large" + }, + { + "amount_sats": 10000, + "order_key": "02", + "p2mr_program_hex": "0202020202020202020202020202020202020202020202020202020202020202", + "recipient_id": "small" + } + ] + }, + "name": "net-exactly-at-floor-is-payable", + "topic": 5, + "why": "The small recipient's fee share is exactly 100, leaving 9900: at the floor, so it is paid." + }, + { + "entry_point": "apply_proportional_fanout_fee", + "expected": { + "ok": { + "applied_fee_sats": 1100, + "carry_forward_recipients": [ + { + "fee_sats": 0, + "gross_amount_sats": 10000, + "net_amount_sats": 10000, + "order_key": "02", + "p2mr_program_hex": "0202020202020202020202020202020202020202020202020202020202020202", + "recipient_id": "small" + } + ], + "payable_recipients": [ + { + "fee_sats": 1100, + "gross_amount_sats": 100000, + "net_amount_sats": 98900, + "order_key": "01", + "p2mr_program_hex": "0101010101010101010101010101010101010101010101010101010101010101", + "recipient_id": "large" + } + ], + "requested_fee_sats": 1100, + "total_gross_sats": 110000 + } + }, + "input": { + "fee_sats": 1100, + "min_output_sats": 9901, + "recipients": [ + { + "amount_sats": 100000, + "order_key": "01", + "p2mr_program_hex": "0101010101010101010101010101010101010101010101010101010101010101", + "recipient_id": "large" + }, + { + "amount_sats": 10000, + "order_key": "02", + "p2mr_program_hex": "0202020202020202020202020202020202020202020202020202020202020202", + "recipient_id": "small" + } + ] + }, + "name": "net-one-sat-under-floor-is-carried", + "topic": 5, + "why": "The same split with floor 9901 leaves the small recipient one sat under, so it carries and the large one pays the whole fee." + }, + { + "entry_point": "apply_proportional_fanout_fee", + "expected": { + "ok": { + "applied_fee_sats": 0, + "carry_forward_recipients": [ + { + "fee_sats": 0, + "gross_amount_sats": 10000, + "net_amount_sats": 10000, + "order_key": "01", + "p2mr_program_hex": "0101010101010101010101010101010101010101010101010101010101010101", + "recipient_id": "tiny" + } + ], + "payable_recipients": [], + "requested_fee_sats": 1000, + "total_gross_sats": 10000 + } + }, + "input": { + "fee_sats": 1000, + "min_output_sats": 9500, + "recipients": [ + { + "amount_sats": 10000, + "order_key": "01", + "p2mr_program_hex": "0101010101010101010101010101010101010101010101010101010101010101", + "recipient_id": "tiny" + } + ] + }, + "name": "all-carry-when-no-recipient-survives", + "topic": 5, + "why": "The only recipient falls under the floor, so every recipient carries and no fee is applied." + }, + { + "entry_point": "apply_proportional_fanout_fee", + "expected": { + "error": "cannot select a settlement mode: min_output_sats must be positive" + }, + "input": { + "fee_sats": 1000, + "min_output_sats": 0, + "recipients": [ + { + "amount_sats": 10000, + "order_key": "01", + "p2mr_program_hex": "0101010101010101010101010101010101010101010101010101010101010101", + "recipient_id": "tiny" + } + ] + }, + "name": "zero-floor-is-an-error", + "topic": 5, + "why": "min_output_sats must be positive." + }, + { + "entry_point": "apply_proportional_fanout_fee", + "expected": { + "ok": { + "applied_fee_sats": 10, + "carry_forward_recipients": [], + "payable_recipients": [ + { + "fee_sats": 4, + "gross_amount_sats": 100, + "net_amount_sats": 96, + "order_key": "01", + "p2mr_program_hex": "0101010101010101010101010101010101010101010101010101010101010101", + "recipient_id": "a" + }, + { + "fee_sats": 3, + "gross_amount_sats": 100, + "net_amount_sats": 97, + "order_key": "02", + "p2mr_program_hex": "0202020202020202020202020202020202020202020202020202020202020202", + "recipient_id": "b" + }, + { + "fee_sats": 3, + "gross_amount_sats": 100, + "net_amount_sats": 97, + "order_key": "03", + "p2mr_program_hex": "0303030303030303030303030303030303030303030303030303030303030303", + "recipient_id": "c" + } + ], + "requested_fee_sats": 10, + "total_gross_sats": 300 + } + }, + "input": { + "fee_sats": 10, + "min_output_sats": 1, + "recipients": [ + { + "amount_sats": 100, + "order_key": "03", + "p2mr_program_hex": "0303030303030303030303030303030303030303030303030303030303030303", + "recipient_id": "c" + }, + { + "amount_sats": 100, + "order_key": "01", + "p2mr_program_hex": "0101010101010101010101010101010101010101010101010101010101010101", + "recipient_id": "a" + }, + { + "amount_sats": 100, + "order_key": "02", + "p2mr_program_hex": "0202020202020202020202020202020202020202020202020202020202020202", + "recipient_id": "b" + } + ] + }, + "name": "equal-remainders-broken-by-canonical-order", + "topic": 6, + "why": "Three equal recipients share a 10-sat fee: 3 each plus one remainder sat, which goes to the first account in canonical order regardless of input order." + }, + { + "entry_point": "apply_proportional_fanout_fee", + "expected": { + "ok": { + "applied_fee_sats": 1, + "carry_forward_recipients": [], + "payable_recipients": [ + { + "fee_sats": 1, + "gross_amount_sats": 100, + "net_amount_sats": 99, + "order_key": "01", + "p2mr_program_hex": "0101010101010101010101010101010101010101010101010101010101010101", + "recipient_id": "z-miner" + }, + { + "fee_sats": 0, + "gross_amount_sats": 100, + "net_amount_sats": 100, + "order_key": "02", + "p2mr_program_hex": "0202020202020202020202020202020202020202020202020202020202020202", + "recipient_id": "a-miner" + } + ], + "requested_fee_sats": 1, + "total_gross_sats": 200 + } + }, + "input": { + "fee_sats": 1, + "min_output_sats": 1, + "recipients": [ + { + "amount_sats": 100, + "order_key": "02", + "p2mr_program_hex": "0202020202020202020202020202020202020202020202020202020202020202", + "recipient_id": "a-miner" + }, + { + "amount_sats": 100, + "order_key": "01", + "p2mr_program_hex": "0101010101010101010101010101010101010101010101010101010101010101", + "recipient_id": "z-miner" + } + ] + }, + "name": "equal-remainders-order-key-before-recipient-id", + "topic": 6, + "why": "The canonical tie-break compares order_key before recipient_id, so the remainder sat goes to order key 01 even though its id sorts last." + }, + { + "entry_point": "apply_payout_policy", + "expected": { + "ok": { + "accounts": [ + { + "action": "onchain", + "candidate_balance_sats": 166666667, + "carry_forward_balance_sats": 0, + "gross_amount_sats": 166666667, + "onchain_amount_sats": 166666667, + "order_key": "01", + "p2mr_program_hex": "0101010101010101010101010101010101010101010101010101010101010101", + "prior_balance_sats": 0, + "recipient_id": "miner-a" + }, + { + "action": "onchain", + "candidate_balance_sats": 166666667, + "carry_forward_balance_sats": 0, + "gross_amount_sats": 166666667, + "onchain_amount_sats": 166666667, + "order_key": "02", + "p2mr_program_hex": "0202020202020202020202020202020202020202020202020202020202020202", + "prior_balance_sats": 0, + "recipient_id": "miner-b" + }, + { + "action": "onchain", + "candidate_balance_sats": 166666666, + "carry_forward_balance_sats": 0, + "gross_amount_sats": 166666666, + "onchain_amount_sats": 166666666, + "order_key": "03", + "p2mr_program_hex": "0303030303030303030303030303030303030303030303030303030303030303", + "prior_balance_sats": 0, + "recipient_id": "miner-c" + } + ], + "block_height": 10, + "coinbase_value_sats": 500000000, + "floor_formula": "3680 bytes/input * 1 sat/byte * 4x safety", + "min_output_sats": 14720, + "onchain_entitlements": [ + { + "order_key": "01", + "p2mr_program_hex": "0101010101010101010101010101010101010101010101010101010101010101", + "recipient_id": "miner-a", + "weight": 166666667 + }, + { + "order_key": "02", + "p2mr_program_hex": "0202020202020202020202020202020202020202020202020202020202020202", + "recipient_id": "miner-b", + "weight": 166666667 + }, + { + "order_key": "03", + "p2mr_program_hex": "0303030303030303030303030303030303030303030303030303030303030303", + "recipient_id": "miner-c", + "weight": 166666666 + } + ], + "schema": "qbit.prism.payout-policy.v1" + } + }, + "input": { + "policy": { + "p2mr_spend_input_bytes": 3680, + "safety_multiplier": 4, + "target_feerate_sats_per_byte": 1 + }, + "prior_balances": [], + "reward_manifest": { + "anchor_job_issued_at_ms": 1, + "anchor_share_seq": 1, + "block_height": 10, + "coinbase_value_sats": 500000000, + "counted_window_weight": 3, + "entitlements": [ + { + "order_key": "03", + "p2mr_program_hex": "0303030303030303030303030303030303030303030303030303030303030303", + "recipient_id": "miner-c", + "weight": 1 + }, + { + "order_key": "01", + "p2mr_program_hex": "0101010101010101010101010101010101010101010101010101010101010101", + "recipient_id": "miner-a", + "weight": 1 + }, + { + "order_key": "02", + "p2mr_program_hex": "0202020202020202020202020202020202020202020202020202020202020202", + "recipient_id": "miner-b", + "weight": 1 + } + ], + "included_share_count": 3, + "network_difficulty": 1, + "newest_share_seq": 1, + "oldest_share_seq": 1, + "requested_window_weight": 8, + "schema": "qbit.prism.reward-manifest.v1", + "share_slice_digest_hex": "0000000000000000000000000000000000000000000000000000000000000000", + "shares": [], + "window_multiplier": 8 + } + }, + "name": "three-way-split-of-500000000", + "topic": 6, + "why": "500000000 over three equal weights is 166666666 each plus two remainder sats for the first two accounts in canonical order." + } + ], + "produced_by": "PYTHONDONTWRITEBYTECODE=1 PYTHONPATH=. python3 crates/qbit-prism/examples/export_rule_decisions.py | cargo run --locked -q -p qbit-prism --example export_money_path_vectors -- vectors-out", + "schema": "qbit.prism.money-path-vectors.v1", + "source_commit": "504846cc0b72e8f86ed17f896d4ccbbe196a31dc", + "topics": [ + { + "id": 5, + "name": "dust floor exclusion and recompute" + }, + { + "id": 6, + "name": "largest-remainder ties" + } + ] +} diff --git a/crates/qbit-prism/fixtures/vectors/pool_fee.json b/crates/qbit-prism/fixtures/vectors/pool_fee.json new file mode 100644 index 000000000..573fd2562 --- /dev/null +++ b/crates/qbit-prism/fixtures/vectors/pool_fee.json @@ -0,0 +1,975 @@ +{ + "cases": [ + { + "entry_point": "apply_payout_policy", + "expected": { + "ok": { + "accounts": [ + { + "action": "onchain", + "candidate_balance_sats": 32918, + "carry_forward_balance_sats": 0, + "gross_amount_sats": 32918, + "onchain_amount_sats": 32918, + "order_key": "01", + "p2mr_program_hex": "0101010101010101010101010101010101010101010101010101010101010101", + "prior_balance_sats": 0, + "recipient_id": "miner-a" + }, + { + "action": "onchain", + "candidate_balance_sats": 65835, + "carry_forward_balance_sats": 0, + "gross_amount_sats": 65835, + "onchain_amount_sats": 65835, + "order_key": "02", + "p2mr_program_hex": "0202020202020202020202020202020202020202020202020202020202020202", + "prior_balance_sats": 0, + "recipient_id": "miner-b" + }, + { + "account_type": "pool_fee", + "action": "onchain", + "candidate_balance_sats": 1250, + "carry_forward_balance_sats": 0, + "gross_amount_sats": 1250, + "onchain_amount_sats": 1250, + "order_key": "99", + "p2mr_program_hex": "9999999999999999999999999999999999999999999999999999999999999999", + "prior_balance_sats": 0, + "recipient_id": "pool-fee" + } + ], + "block_height": 10, + "coinbase_value_sats": 100003, + "floor_formula": "3680 bytes/input * 1 sat/byte * 4x safety", + "min_output_sats": 14720, + "onchain_entitlements": [ + { + "order_key": "01", + "p2mr_program_hex": "0101010101010101010101010101010101010101010101010101010101010101", + "recipient_id": "miner-a", + "weight": 32918 + }, + { + "order_key": "02", + "p2mr_program_hex": "0202020202020202020202020202020202020202020202020202020202020202", + "recipient_id": "miner-b", + "weight": 65835 + }, + { + "order_key": "99", + "p2mr_program_hex": "9999999999999999999999999999999999999999999999999999999999999999", + "recipient_id": "pool-fee", + "weight": 1250 + } + ], + "pool_fee": { + "amount_sats": 1250, + "earned_pool_fee_sats": 1250, + "fee_bps": 125, + "order_key": "99", + "p2mr_program_hex": "9999999999999999999999999999999999999999999999999999999999999999", + "recipient_id": "pool-fee", + "swept_dust_liability_sats": 0 + }, + "schema": "qbit.prism.payout-policy.v1" + } + }, + "input": { + "policy": { + "p2mr_spend_input_bytes": 3680, + "pool_fee_policy": { + "fee_bps": 125, + "order_key": "99", + "p2mr_program_hex": "9999999999999999999999999999999999999999999999999999999999999999", + "recipient_id": "pool-fee" + }, + "safety_multiplier": 4, + "target_feerate_sats_per_byte": 1 + }, + "prior_balances": [], + "reward_manifest": { + "anchor_job_issued_at_ms": 1, + "anchor_share_seq": 1, + "block_height": 10, + "coinbase_value_sats": 100003, + "counted_window_weight": 3, + "entitlements": [ + { + "order_key": "01", + "p2mr_program_hex": "0101010101010101010101010101010101010101010101010101010101010101", + "recipient_id": "miner-a", + "weight": 1 + }, + { + "order_key": "02", + "p2mr_program_hex": "0202020202020202020202020202020202020202020202020202020202020202", + "recipient_id": "miner-b", + "weight": 2 + } + ], + "included_share_count": 2, + "network_difficulty": 1, + "newest_share_seq": 1, + "oldest_share_seq": 1, + "requested_window_weight": 8, + "schema": "qbit.prism.reward-manifest.v1", + "share_slice_digest_hex": "0000000000000000000000000000000000000000000000000000000000000000", + "shares": [], + "window_multiplier": 8 + } + }, + "name": "fee-125-bps-non-divisible-coinbase", + "topic": 4, + "why": "100003 x 125 / 10000 = 1250.0375 rounds down to 1250; the miners split the rest 1:2 with the remainder sat by largest remainder." + }, + { + "entry_point": "apply_payout_policy", + "expected": { + "ok": { + "accounts": [ + { + "action": "onchain", + "candidate_balance_sats": 164583335, + "carry_forward_balance_sats": 0, + "gross_amount_sats": 164583335, + "onchain_amount_sats": 164583335, + "order_key": "01", + "p2mr_program_hex": "0101010101010101010101010101010101010101010101010101010101010101", + "prior_balance_sats": 0, + "recipient_id": "miner-a" + }, + { + "action": "onchain", + "candidate_balance_sats": 164583334, + "carry_forward_balance_sats": 0, + "gross_amount_sats": 164583334, + "onchain_amount_sats": 164583334, + "order_key": "02", + "p2mr_program_hex": "0202020202020202020202020202020202020202020202020202020202020202", + "prior_balance_sats": 0, + "recipient_id": "miner-b" + }, + { + "action": "onchain", + "candidate_balance_sats": 164583334, + "carry_forward_balance_sats": 0, + "gross_amount_sats": 164583334, + "onchain_amount_sats": 164583334, + "order_key": "03", + "p2mr_program_hex": "0303030303030303030303030303030303030303030303030303030303030303", + "prior_balance_sats": 0, + "recipient_id": "miner-c" + }, + { + "account_type": "pool_fee", + "action": "onchain", + "candidate_balance_sats": 6250000, + "carry_forward_balance_sats": 0, + "gross_amount_sats": 6250000, + "onchain_amount_sats": 6250000, + "order_key": "99", + "p2mr_program_hex": "9999999999999999999999999999999999999999999999999999999999999999", + "prior_balance_sats": 0, + "recipient_id": "pool-fee" + } + ], + "block_height": 10, + "coinbase_value_sats": 500000003, + "floor_formula": "3680 bytes/input * 1 sat/byte * 4x safety", + "min_output_sats": 14720, + "onchain_entitlements": [ + { + "order_key": "01", + "p2mr_program_hex": "0101010101010101010101010101010101010101010101010101010101010101", + "recipient_id": "miner-a", + "weight": 164583335 + }, + { + "order_key": "02", + "p2mr_program_hex": "0202020202020202020202020202020202020202020202020202020202020202", + "recipient_id": "miner-b", + "weight": 164583334 + }, + { + "order_key": "03", + "p2mr_program_hex": "0303030303030303030303030303030303030303030303030303030303030303", + "recipient_id": "miner-c", + "weight": 164583334 + }, + { + "order_key": "99", + "p2mr_program_hex": "9999999999999999999999999999999999999999999999999999999999999999", + "recipient_id": "pool-fee", + "weight": 6250000 + } + ], + "pool_fee": { + "amount_sats": 6250000, + "earned_pool_fee_sats": 6250000, + "fee_bps": 125, + "order_key": "99", + "p2mr_program_hex": "9999999999999999999999999999999999999999999999999999999999999999", + "recipient_id": "pool-fee", + "swept_dust_liability_sats": 0 + }, + "schema": "qbit.prism.payout-policy.v1" + } + }, + "input": { + "policy": { + "p2mr_spend_input_bytes": 3680, + "pool_fee_policy": { + "fee_bps": 125, + "order_key": "99", + "p2mr_program_hex": "9999999999999999999999999999999999999999999999999999999999999999", + "recipient_id": "pool-fee" + }, + "safety_multiplier": 4, + "target_feerate_sats_per_byte": 1 + }, + "prior_balances": [], + "reward_manifest": { + "anchor_job_issued_at_ms": 1, + "anchor_share_seq": 1, + "block_height": 10, + "coinbase_value_sats": 500000003, + "counted_window_weight": 3, + "entitlements": [ + { + "order_key": "01", + "p2mr_program_hex": "0101010101010101010101010101010101010101010101010101010101010101", + "recipient_id": "miner-a", + "weight": 1 + }, + { + "order_key": "02", + "p2mr_program_hex": "0202020202020202020202020202020202020202020202020202020202020202", + "recipient_id": "miner-b", + "weight": 1 + }, + { + "order_key": "03", + "p2mr_program_hex": "0303030303030303030303030303030303030303030303030303030303030303", + "recipient_id": "miner-c", + "weight": 1 + } + ], + "included_share_count": 3, + "network_difficulty": 1, + "newest_share_seq": 1, + "oldest_share_seq": 1, + "requested_window_weight": 8, + "schema": "qbit.prism.reward-manifest.v1", + "share_slice_digest_hex": "0000000000000000000000000000000000000000000000000000000000000000", + "shares": [], + "window_multiplier": 8 + } + }, + "name": "fee-125-bps-three-equal-miners", + "topic": 4, + "why": "500000003 x 125 / 10000 rounds down to 6250000; 493750003 split three ways leaves one remainder sat for the first account in canonical order." + }, + { + "entry_point": "apply_payout_policy", + "expected": { + "ok": { + "accounts": [ + { + "action": "onchain", + "candidate_balance_sats": 99999, + "carry_forward_balance_sats": 0, + "gross_amount_sats": 99999, + "onchain_amount_sats": 99999, + "order_key": "01", + "p2mr_program_hex": "0101010101010101010101010101010101010101010101010101010101010101", + "prior_balance_sats": 0, + "recipient_id": "miner-a" + }, + { + "action": "accrued", + "candidate_balance_sats": 1, + "carry_forward_balance_sats": 1, + "gross_amount_sats": 1, + "onchain_amount_sats": 0, + "order_key": "02", + "p2mr_program_hex": "0202020202020202020202020202020202020202020202020202020202020202", + "prior_balance_sats": 0, + "recipient_id": "dust" + }, + { + "account_type": "pool_fee", + "action": "onchain", + "candidate_balance_sats": 1, + "carry_forward_balance_sats": 0, + "gross_amount_sats": 1, + "onchain_amount_sats": 1, + "order_key": "99", + "p2mr_program_hex": "9999999999999999999999999999999999999999999999999999999999999999", + "prior_balance_sats": 0, + "recipient_id": "pool-fee" + } + ], + "block_height": 10, + "coinbase_value_sats": 100000, + "floor_formula": "3680 bytes/input * 1 sat/byte * 4x safety", + "min_output_sats": 14720, + "onchain_entitlements": [ + { + "order_key": "01", + "p2mr_program_hex": "0101010101010101010101010101010101010101010101010101010101010101", + "recipient_id": "miner-a", + "weight": 99999 + }, + { + "order_key": "99", + "p2mr_program_hex": "9999999999999999999999999999999999999999999999999999999999999999", + "recipient_id": "pool-fee", + "weight": 1 + } + ], + "pool_fee": { + "amount_sats": 1, + "earned_pool_fee_sats": 0, + "fee_bps": 0, + "order_key": "99", + "p2mr_program_hex": "9999999999999999999999999999999999999999999999999999999999999999", + "recipient_id": "pool-fee", + "swept_dust_liability_sats": 1 + }, + "schema": "qbit.prism.payout-policy.v1" + } + }, + "input": { + "policy": { + "p2mr_spend_input_bytes": 3680, + "pool_fee_policy": { + "fee_bps": 0, + "order_key": "99", + "p2mr_program_hex": "9999999999999999999999999999999999999999999999999999999999999999", + "recipient_id": "pool-fee" + }, + "safety_multiplier": 4, + "target_feerate_sats_per_byte": 1 + }, + "prior_balances": [], + "reward_manifest": { + "anchor_job_issued_at_ms": 1, + "anchor_share_seq": 1, + "block_height": 10, + "coinbase_value_sats": 100000, + "counted_window_weight": 100000, + "entitlements": [ + { + "order_key": "01", + "p2mr_program_hex": "0101010101010101010101010101010101010101010101010101010101010101", + "recipient_id": "miner-a", + "weight": 99999 + }, + { + "order_key": "02", + "p2mr_program_hex": "0202020202020202020202020202020202020202020202020202020202020202", + "recipient_id": "dust", + "weight": 1 + } + ], + "included_share_count": 2, + "network_difficulty": 1, + "newest_share_seq": 1, + "oldest_share_seq": 1, + "requested_window_weight": 8, + "schema": "qbit.prism.reward-manifest.v1", + "share_slice_digest_hex": "0000000000000000000000000000000000000000000000000000000000000000", + "shares": [], + "window_multiplier": 8 + } + }, + "name": "fee-0-bps-sweeps-sub-floor-dust", + "topic": 4, + "why": "A zero-bps fee earns nothing, but the sub-floor miner's allocation is swept into the pool-fee output." + }, + { + "entry_point": "apply_payout_policy", + "expected": { + "ok": { + "accounts": [ + { + "action": "onchain", + "candidate_balance_sats": 33330, + "carry_forward_balance_sats": 0, + "gross_amount_sats": 33330, + "onchain_amount_sats": 33330, + "order_key": "01", + "p2mr_program_hex": "0101010101010101010101010101010101010101010101010101010101010101", + "prior_balance_sats": 0, + "recipient_id": "miner-a" + }, + { + "action": "onchain", + "candidate_balance_sats": 66660, + "carry_forward_balance_sats": 0, + "gross_amount_sats": 66660, + "onchain_amount_sats": 66660, + "order_key": "02", + "p2mr_program_hex": "0202020202020202020202020202020202020202020202020202020202020202", + "prior_balance_sats": 0, + "recipient_id": "miner-b" + }, + { + "account_type": "pool_fee", + "action": "onchain", + "candidate_balance_sats": 10, + "carry_forward_balance_sats": 0, + "gross_amount_sats": 10, + "onchain_amount_sats": 10, + "order_key": "99", + "p2mr_program_hex": "9999999999999999999999999999999999999999999999999999999999999999", + "prior_balance_sats": 0, + "recipient_id": "pool-fee" + } + ], + "block_height": 10, + "coinbase_output_policy": "pool-fee-first", + "coinbase_value_sats": 100000, + "floor_formula": "3680 bytes/input * 1 sat/byte * 4x safety", + "min_output_sats": 14720, + "onchain_entitlements": [ + { + "order_key": "01", + "p2mr_program_hex": "0101010101010101010101010101010101010101010101010101010101010101", + "recipient_id": "miner-a", + "weight": 33330 + }, + { + "order_key": "02", + "p2mr_program_hex": "0202020202020202020202020202020202020202020202020202020202020202", + "recipient_id": "miner-b", + "weight": 66660 + }, + { + "order_key": "99", + "p2mr_program_hex": "9999999999999999999999999999999999999999999999999999999999999999", + "recipient_id": "pool-fee", + "weight": 10 + } + ], + "pool_fee": { + "amount_sats": 10, + "earned_pool_fee_sats": 10, + "fee_bps": 1, + "order_key": "99", + "p2mr_program_hex": "9999999999999999999999999999999999999999999999999999999999999999", + "recipient_id": "pool-fee", + "swept_dust_liability_sats": 0 + }, + "schema": "qbit.prism.payout-policy.v1" + } + }, + "input": { + "policy": { + "coinbase_output_policy": "pool-fee-first", + "p2mr_spend_input_bytes": 3680, + "pool_fee_policy": { + "fee_bps": 1, + "order_key": "99", + "p2mr_program_hex": "9999999999999999999999999999999999999999999999999999999999999999", + "recipient_id": "pool-fee" + }, + "safety_multiplier": 4, + "target_feerate_sats_per_byte": 1 + }, + "prior_balances": [], + "reward_manifest": { + "anchor_job_issued_at_ms": 1, + "anchor_share_seq": 1, + "block_height": 10, + "coinbase_value_sats": 100000, + "counted_window_weight": 3, + "entitlements": [ + { + "order_key": "01", + "p2mr_program_hex": "0101010101010101010101010101010101010101010101010101010101010101", + "recipient_id": "miner-a", + "weight": 1 + }, + { + "order_key": "02", + "p2mr_program_hex": "0202020202020202020202020202020202020202020202020202020202020202", + "recipient_id": "miner-b", + "weight": 2 + } + ], + "included_share_count": 2, + "network_difficulty": 1, + "newest_share_seq": 1, + "oldest_share_seq": 1, + "requested_window_weight": 8, + "schema": "qbit.prism.reward-manifest.v1", + "share_slice_digest_hex": "0000000000000000000000000000000000000000000000000000000000000000", + "shares": [], + "window_multiplier": 8 + } + }, + "name": "fee-first-with-sub-floor-fee", + "topic": 4, + "why": "Pool-fee-first with a 10-sat fee (1 bps of 100000): the fee stays an on-chain account even though it is below the 14720 floor." + }, + { + "entry_point": "apply_payout_policy", + "expected": { + "error": "pool-fee-first coinbase output policy requires a configured pool fee policy" + }, + "input": { + "policy": { + "coinbase_output_policy": "pool-fee-first", + "p2mr_spend_input_bytes": 3680, + "safety_multiplier": 4, + "target_feerate_sats_per_byte": 1 + }, + "prior_balances": [], + "reward_manifest": { + "anchor_job_issued_at_ms": 1, + "anchor_share_seq": 1, + "block_height": 10, + "coinbase_value_sats": 100000, + "counted_window_weight": 3, + "entitlements": [ + { + "order_key": "01", + "p2mr_program_hex": "0101010101010101010101010101010101010101010101010101010101010101", + "recipient_id": "miner-a", + "weight": 1 + }, + { + "order_key": "02", + "p2mr_program_hex": "0202020202020202020202020202020202020202020202020202020202020202", + "recipient_id": "miner-b", + "weight": 2 + } + ], + "included_share_count": 2, + "network_difficulty": 1, + "newest_share_seq": 1, + "oldest_share_seq": 1, + "requested_window_weight": 8, + "schema": "qbit.prism.reward-manifest.v1", + "share_slice_digest_hex": "0000000000000000000000000000000000000000000000000000000000000000", + "shares": [], + "window_multiplier": 8 + } + }, + "name": "fee-first-without-pool-fee-is-an-error", + "topic": 4, + "why": "Pool-fee-first with no pool-fee policy cannot pin a fee output." + }, + { + "entry_point": "apply_payout_policy", + "expected": { + "error": "coinbase value 0 is below minimum output floor 14720" + }, + "input": { + "policy": { + "p2mr_spend_input_bytes": 3680, + "pool_fee_policy": { + "fee_bps": 10000, + "order_key": "99", + "p2mr_program_hex": "9999999999999999999999999999999999999999999999999999999999999999", + "recipient_id": "pool-fee" + }, + "safety_multiplier": 4, + "target_feerate_sats_per_byte": 1 + }, + "prior_balances": [], + "reward_manifest": { + "anchor_job_issued_at_ms": 1, + "anchor_share_seq": 1, + "block_height": 10, + "coinbase_value_sats": 100000, + "counted_window_weight": 3, + "entitlements": [ + { + "order_key": "01", + "p2mr_program_hex": "0101010101010101010101010101010101010101010101010101010101010101", + "recipient_id": "miner-a", + "weight": 1 + }, + { + "order_key": "02", + "p2mr_program_hex": "0202020202020202020202020202020202020202020202020202020202020202", + "recipient_id": "miner-b", + "weight": 2 + } + ], + "included_share_count": 2, + "network_difficulty": 1, + "newest_share_seq": 1, + "oldest_share_seq": 1, + "requested_window_weight": 8, + "schema": "qbit.prism.reward-manifest.v1", + "share_slice_digest_hex": "0000000000000000000000000000000000000000000000000000000000000000", + "shares": [], + "window_multiplier": 8 + } + }, + "name": "fee-10000-bps-leaves-miners-below-floor", + "topic": 4, + "why": "At the 10000-bps cap the fee is the whole coinbase and the miner reward (0) is below the floor." + }, + { + "entry_point": "apply_payout_policy", + "expected": { + "error": "pool fee bps 10001 exceeds 10000" + }, + "input": { + "policy": { + "p2mr_spend_input_bytes": 3680, + "pool_fee_policy": { + "fee_bps": 10001, + "order_key": "99", + "p2mr_program_hex": "9999999999999999999999999999999999999999999999999999999999999999", + "recipient_id": "pool-fee" + }, + "safety_multiplier": 4, + "target_feerate_sats_per_byte": 1 + }, + "prior_balances": [], + "reward_manifest": { + "anchor_job_issued_at_ms": 1, + "anchor_share_seq": 1, + "block_height": 10, + "coinbase_value_sats": 100000, + "counted_window_weight": 3, + "entitlements": [ + { + "order_key": "01", + "p2mr_program_hex": "0101010101010101010101010101010101010101010101010101010101010101", + "recipient_id": "miner-a", + "weight": 1 + }, + { + "order_key": "02", + "p2mr_program_hex": "0202020202020202020202020202020202020202020202020202020202020202", + "recipient_id": "miner-b", + "weight": 2 + } + ], + "included_share_count": 2, + "network_difficulty": 1, + "newest_share_seq": 1, + "oldest_share_seq": 1, + "requested_window_weight": 8, + "schema": "qbit.prism.reward-manifest.v1", + "share_slice_digest_hex": "0000000000000000000000000000000000000000000000000000000000000000", + "shares": [], + "window_multiplier": 8 + } + }, + "name": "fee-10001-bps-is-an-error", + "topic": 4, + "why": "One basis point over 100% is rejected." + }, + { + "entry_point": "apply_payout_policy", + "expected": { + "ok": { + "accounts": [ + { + "action": "onchain", + "candidate_balance_sats": 3333, + "carry_forward_balance_sats": 0, + "gross_amount_sats": 3333, + "onchain_amount_sats": 3333, + "order_key": "01", + "p2mr_program_hex": "0101010101010101010101010101010101010101010101010101010101010101", + "prior_balance_sats": 0, + "recipient_id": "miner-a" + }, + { + "action": "onchain", + "candidate_balance_sats": 6666, + "carry_forward_balance_sats": 0, + "gross_amount_sats": 6666, + "onchain_amount_sats": 6666, + "order_key": "02", + "p2mr_program_hex": "0202020202020202020202020202020202020202020202020202020202020202", + "prior_balance_sats": 0, + "recipient_id": "miner-b" + }, + { + "account_type": "pool_fee", + "action": "onchain", + "candidate_balance_sats": 0, + "carry_forward_balance_sats": 0, + "gross_amount_sats": 0, + "onchain_amount_sats": 0, + "order_key": "99", + "p2mr_program_hex": "9999999999999999999999999999999999999999999999999999999999999999", + "prior_balance_sats": 0, + "recipient_id": "pool-fee" + } + ], + "block_height": 10, + "coinbase_value_sats": 9999, + "floor_formula": "configured fixed floor: 1000 sats", + "min_output_sats": 1000, + "onchain_entitlements": [ + { + "order_key": "01", + "p2mr_program_hex": "0101010101010101010101010101010101010101010101010101010101010101", + "recipient_id": "miner-a", + "weight": 3333 + }, + { + "order_key": "02", + "p2mr_program_hex": "0202020202020202020202020202020202020202020202020202020202020202", + "recipient_id": "miner-b", + "weight": 6666 + } + ], + "pool_fee": { + "amount_sats": 0, + "earned_pool_fee_sats": 0, + "fee_bps": 1, + "order_key": "99", + "p2mr_program_hex": "9999999999999999999999999999999999999999999999999999999999999999", + "recipient_id": "pool-fee", + "swept_dust_liability_sats": 0 + }, + "schema": "qbit.prism.payout-policy.v1" + } + }, + "input": { + "policy": { + "min_output_sats": 1000, + "p2mr_spend_input_bytes": 3680, + "pool_fee_policy": { + "fee_bps": 1, + "order_key": "99", + "p2mr_program_hex": "9999999999999999999999999999999999999999999999999999999999999999", + "recipient_id": "pool-fee" + }, + "safety_multiplier": 4, + "target_feerate_sats_per_byte": 1 + }, + "prior_balances": [], + "reward_manifest": { + "anchor_job_issued_at_ms": 1, + "anchor_share_seq": 1, + "block_height": 10, + "coinbase_value_sats": 9999, + "counted_window_weight": 3, + "entitlements": [ + { + "order_key": "01", + "p2mr_program_hex": "0101010101010101010101010101010101010101010101010101010101010101", + "recipient_id": "miner-a", + "weight": 1 + }, + { + "order_key": "02", + "p2mr_program_hex": "0202020202020202020202020202020202020202020202020202020202020202", + "recipient_id": "miner-b", + "weight": 2 + } + ], + "included_share_count": 2, + "network_difficulty": 1, + "newest_share_seq": 1, + "oldest_share_seq": 1, + "requested_window_weight": 8, + "schema": "qbit.prism.reward-manifest.v1", + "share_slice_digest_hex": "0000000000000000000000000000000000000000000000000000000000000000", + "shares": [], + "window_multiplier": 8 + } + }, + "name": "fee-1-bps-floors-to-zero", + "topic": 4, + "why": "1 bps of a 9999-sat coinbase is 0.9999 sats and floors to a zero pool fee; with a 1000-sat fixed floor the miners take the whole coinbase, and the fee account is recorded with 0 sats but gets no coinbase output." + }, + { + "entry_point": "apply_payout_policy", + "expected": { + "ok": { + "accounts": [ + { + "action": "onchain", + "candidate_balance_sats": 3333, + "carry_forward_balance_sats": 0, + "gross_amount_sats": 3333, + "onchain_amount_sats": 3333, + "order_key": "01", + "p2mr_program_hex": "0101010101010101010101010101010101010101010101010101010101010101", + "prior_balance_sats": 0, + "recipient_id": "miner-a" + }, + { + "action": "onchain", + "candidate_balance_sats": 6666, + "carry_forward_balance_sats": 0, + "gross_amount_sats": 6666, + "onchain_amount_sats": 6666, + "order_key": "02", + "p2mr_program_hex": "0202020202020202020202020202020202020202020202020202020202020202", + "prior_balance_sats": 0, + "recipient_id": "miner-b" + }, + { + "account_type": "pool_fee", + "action": "onchain", + "candidate_balance_sats": 0, + "carry_forward_balance_sats": 0, + "gross_amount_sats": 0, + "onchain_amount_sats": 0, + "order_key": "99", + "p2mr_program_hex": "9999999999999999999999999999999999999999999999999999999999999999", + "prior_balance_sats": 0, + "recipient_id": "pool-fee" + } + ], + "block_height": 10, + "coinbase_output_policy": "pool-fee-first", + "coinbase_value_sats": 9999, + "floor_formula": "configured fixed floor: 1000 sats", + "min_output_sats": 1000, + "onchain_entitlements": [ + { + "order_key": "01", + "p2mr_program_hex": "0101010101010101010101010101010101010101010101010101010101010101", + "recipient_id": "miner-a", + "weight": 3333 + }, + { + "order_key": "02", + "p2mr_program_hex": "0202020202020202020202020202020202020202020202020202020202020202", + "recipient_id": "miner-b", + "weight": 6666 + } + ], + "pool_fee": { + "amount_sats": 0, + "earned_pool_fee_sats": 0, + "fee_bps": 1, + "order_key": "99", + "p2mr_program_hex": "9999999999999999999999999999999999999999999999999999999999999999", + "recipient_id": "pool-fee", + "swept_dust_liability_sats": 0 + }, + "schema": "qbit.prism.payout-policy.v1" + } + }, + "input": { + "policy": { + "coinbase_output_policy": "pool-fee-first", + "min_output_sats": 1000, + "p2mr_spend_input_bytes": 3680, + "pool_fee_policy": { + "fee_bps": 1, + "order_key": "99", + "p2mr_program_hex": "9999999999999999999999999999999999999999999999999999999999999999", + "recipient_id": "pool-fee" + }, + "safety_multiplier": 4, + "target_feerate_sats_per_byte": 1 + }, + "prior_balances": [], + "reward_manifest": { + "anchor_job_issued_at_ms": 1, + "anchor_share_seq": 1, + "block_height": 10, + "coinbase_value_sats": 9999, + "counted_window_weight": 3, + "entitlements": [ + { + "order_key": "01", + "p2mr_program_hex": "0101010101010101010101010101010101010101010101010101010101010101", + "recipient_id": "miner-a", + "weight": 1 + }, + { + "order_key": "02", + "p2mr_program_hex": "0202020202020202020202020202020202020202020202020202020202020202", + "recipient_id": "miner-b", + "weight": 2 + } + ], + "included_share_count": 2, + "network_difficulty": 1, + "newest_share_seq": 1, + "oldest_share_seq": 1, + "requested_window_weight": 8, + "schema": "qbit.prism.reward-manifest.v1", + "share_slice_digest_hex": "0000000000000000000000000000000000000000000000000000000000000000", + "shares": [], + "window_multiplier": 8 + } + }, + "name": "fee-first-1-bps-floors-to-zero", + "topic": 4, + "why": "The same zero pool fee under pool-fee-first: the fee account is recorded with 0 sats, and no zero-amount fee output is pinned." + }, + { + "entry_point": "apply_payout_policy", + "expected": { + "error": "coinbase value 9999 is below minimum output floor 14720" + }, + "input": { + "policy": { + "p2mr_spend_input_bytes": 3680, + "pool_fee_policy": { + "fee_bps": 1, + "order_key": "99", + "p2mr_program_hex": "9999999999999999999999999999999999999999999999999999999999999999", + "recipient_id": "pool-fee" + }, + "safety_multiplier": 4, + "target_feerate_sats_per_byte": 1 + }, + "prior_balances": [], + "reward_manifest": { + "anchor_job_issued_at_ms": 1, + "anchor_share_seq": 1, + "block_height": 10, + "coinbase_value_sats": 9999, + "counted_window_weight": 3, + "entitlements": [ + { + "order_key": "01", + "p2mr_program_hex": "0101010101010101010101010101010101010101010101010101010101010101", + "recipient_id": "miner-a", + "weight": 1 + }, + { + "order_key": "02", + "p2mr_program_hex": "0202020202020202020202020202020202020202020202020202020202020202", + "recipient_id": "miner-b", + "weight": 2 + } + ], + "included_share_count": 2, + "network_difficulty": 1, + "newest_share_seq": 1, + "oldest_share_seq": 1, + "requested_window_weight": 8, + "schema": "qbit.prism.reward-manifest.v1", + "share_slice_digest_hex": "0000000000000000000000000000000000000000000000000000000000000000", + "shares": [], + "window_multiplier": 8 + } + }, + "name": "fee-1-bps-floors-to-zero-under-day-one-floor", + "topic": 4, + "why": "The same zero pool fee with the day-one 14720-sat floor: the 9999-sat coinbase is below the floor, so the policy fails." + } + ], + "produced_by": "PYTHONDONTWRITEBYTECODE=1 PYTHONPATH=. python3 crates/qbit-prism/examples/export_rule_decisions.py | cargo run --locked -q -p qbit-prism --example export_money_path_vectors -- vectors-out", + "schema": "qbit.prism.money-path-vectors.v1", + "source_commit": "504846cc0b72e8f86ed17f896d4ccbbe196a31dc", + "topics": [ + { + "id": 4, + "name": "pool fee: bps rounding down, remainder split, fee-first pinning" + } + ] +} diff --git a/crates/qbit-prism/fixtures/vectors/settlement_chunks.json b/crates/qbit-prism/fixtures/vectors/settlement_chunks.json new file mode 100644 index 000000000..dfaa50665 --- /dev/null +++ b/crates/qbit-prism/fixtures/vectors/settlement_chunks.json @@ -0,0 +1,748 @@ +{ + "cases": [ + { + "entry_point": "select_settlement_mode", + "expected": { + "ok": { + "coinbase_settlement_output_count": 3, + "direct_amount_sats": 0, + "direct_recipient_count": 0, + "direct_recipients": [], + "fanout_amount_sats": 5000, + "fanout_chunk_count": 3, + "fanout_chunks": [ + { + "amount_sats": 2000, + "chunk_index": 0, + "recipient_count": 2, + "recipients": [ + { + "amount_sats": 1000, + "order_key": "01", + "p2mr_program_hex": "0101010101010101010101010101010101010101010101010101010101010101", + "recipient_id": "dust-1" + }, + { + "amount_sats": 1000, + "order_key": "02", + "p2mr_program_hex": "0202020202020202020202020202020202020202020202020202020202020202", + "recipient_id": "dust-2" + } + ] + }, + { + "amount_sats": 2000, + "chunk_index": 1, + "recipient_count": 2, + "recipients": [ + { + "amount_sats": 1000, + "order_key": "03", + "p2mr_program_hex": "0303030303030303030303030303030303030303030303030303030303030303", + "recipient_id": "dust-3" + }, + { + "amount_sats": 1000, + "order_key": "04", + "p2mr_program_hex": "0404040404040404040404040404040404040404040404040404040404040404", + "recipient_id": "dust-4" + } + ] + }, + { + "amount_sats": 1000, + "chunk_index": 2, + "recipient_count": 1, + "recipients": [ + { + "amount_sats": 1000, + "order_key": "05", + "p2mr_program_hex": "0505050505050505050505050505050505050505050505050505050505050505", + "recipient_id": "dust-5" + } + ] + } + ], + "fanout_recipient_count": 5, + "mode": "ctv_fanout", + "reason": "5 recipients routed to 3 CTV fanout chunk(s) capped at 2 recipients each; 3 coinbase settlement outputs used", + "reserved_coinbase_output_count": 0, + "total_amount_sats": 5000 + } + }, + "input": { + "config": { + "max_coinbase_settlement_outputs": 16, + "max_direct_coinbase_outputs": 12, + "max_fanout_recipients_per_transaction": 2, + "reserved_coinbase_outputs": 0 + }, + "direct_floor_sats": 20000, + "pinned_direct": null, + "recipients": [ + { + "amount_sats": 1000, + "order_key": "01", + "p2mr_program_hex": "0101010101010101010101010101010101010101010101010101010101010101", + "recipient_id": "dust-1" + }, + { + "amount_sats": 1000, + "order_key": "02", + "p2mr_program_hex": "0202020202020202020202020202020202020202020202020202020202020202", + "recipient_id": "dust-2" + }, + { + "amount_sats": 1000, + "order_key": "03", + "p2mr_program_hex": "0303030303030303030303030303030303030303030303030303030303030303", + "recipient_id": "dust-3" + }, + { + "amount_sats": 1000, + "order_key": "04", + "p2mr_program_hex": "0404040404040404040404040404040404040404040404040404040404040404", + "recipient_id": "dust-4" + }, + { + "amount_sats": 1000, + "order_key": "05", + "p2mr_program_hex": "0505050505050505050505050505050505050505050505050505050505050505", + "recipient_id": "dust-5" + } + ] + }, + "name": "all-sub-floor-into-bounded-chunks", + "topic": 7, + "why": "Five sub-floor recipients with at most two per fanout become three chunks, filled in canonical order." + }, + { + "entry_point": "select_settlement_mode", + "expected": { + "ok": { + "coinbase_settlement_output_count": 4, + "direct_amount_sats": 90000, + "direct_recipient_count": 2, + "direct_recipients": [ + { + "amount_sats": 50000, + "order_key": "01", + "p2mr_program_hex": "0101010101010101010101010101010101010101010101010101010101010101", + "recipient_id": "direct-a" + }, + { + "amount_sats": 40000, + "order_key": "02", + "p2mr_program_hex": "0202020202020202020202020202020202020202020202020202020202020202", + "recipient_id": "direct-b" + } + ], + "fanout_amount_sats": 39000, + "fanout_chunk_count": 2, + "fanout_chunks": [ + { + "amount_sats": 35000, + "chunk_index": 0, + "recipient_count": 2, + "recipients": [ + { + "amount_sats": 30000, + "order_key": "03", + "p2mr_program_hex": "0303030303030303030303030303030303030303030303030303030303030303", + "recipient_id": "overflow" + }, + { + "amount_sats": 5000, + "order_key": "04", + "p2mr_program_hex": "0404040404040404040404040404040404040404040404040404040404040404", + "recipient_id": "dust-a" + } + ] + }, + { + "amount_sats": 4000, + "chunk_index": 1, + "recipient_count": 1, + "recipients": [ + { + "amount_sats": 4000, + "order_key": "05", + "p2mr_program_hex": "0505050505050505050505050505050505050505050505050505050505050505", + "recipient_id": "dust-b" + } + ] + } + ], + "fanout_recipient_count": 3, + "mode": "hybrid_coinbase_ctv_fanout", + "reason": "2 recipients paid directly by largest-liability priority; 3 recipients routed to 2 CTV fanout chunk(s); 4 of 500 coinbase settlement outputs used", + "reserved_coinbase_output_count": 0, + "total_amount_sats": 129000 + } + }, + "input": { + "config": { + "max_coinbase_settlement_outputs": 500, + "max_direct_coinbase_outputs": 2, + "max_fanout_recipients_per_transaction": 2, + "reserved_coinbase_outputs": 0 + }, + "direct_floor_sats": 20000, + "pinned_direct": null, + "recipients": [ + { + "amount_sats": 4000, + "order_key": "05", + "p2mr_program_hex": "0505050505050505050505050505050505050505050505050505050505050505", + "recipient_id": "dust-b" + }, + { + "amount_sats": 40000, + "order_key": "02", + "p2mr_program_hex": "0202020202020202020202020202020202020202020202020202020202020202", + "recipient_id": "direct-b" + }, + { + "amount_sats": 30000, + "order_key": "03", + "p2mr_program_hex": "0303030303030303030303030303030303030303030303030303030303030303", + "recipient_id": "overflow" + }, + { + "amount_sats": 50000, + "order_key": "01", + "p2mr_program_hex": "0101010101010101010101010101010101010101010101010101010101010101", + "recipient_id": "direct-a" + }, + { + "amount_sats": 5000, + "order_key": "04", + "p2mr_program_hex": "0404040404040404040404040404040404040404040404040404040404040404", + "recipient_id": "dust-a" + } + ] + }, + "name": "hybrid-largest-direct-and-chunks", + "topic": 7, + "why": "Two direct slots go to the largest floor-crossing liabilities; the overflow above the floor and the dust are chunked." + }, + { + "entry_point": "select_settlement_mode", + "expected": { + "ok": { + "coinbase_settlement_output_count": 3, + "direct_amount_sats": 40001, + "direct_recipient_count": 2, + "direct_recipients": [ + { + "amount_sats": 20000, + "order_key": "02", + "p2mr_program_hex": "0202020202020202020202020202020202020202020202020202020202020202", + "recipient_id": "at" + }, + { + "amount_sats": 20001, + "order_key": "03", + "p2mr_program_hex": "0303030303030303030303030303030303030303030303030303030303030303", + "recipient_id": "above" + } + ], + "fanout_amount_sats": 19999, + "fanout_chunk_count": 1, + "fanout_chunks": [ + { + "amount_sats": 19999, + "chunk_index": 0, + "recipient_count": 1, + "recipients": [ + { + "amount_sats": 19999, + "order_key": "01", + "p2mr_program_hex": "0101010101010101010101010101010101010101010101010101010101010101", + "recipient_id": "below" + } + ] + } + ], + "fanout_recipient_count": 1, + "mode": "hybrid_coinbase_ctv_fanout", + "reason": "2 recipients paid directly by largest-liability priority; 1 recipients routed to 1 CTV fanout chunk(s); 3 of 500 coinbase settlement outputs used", + "reserved_coinbase_output_count": 0, + "total_amount_sats": 60000 + } + }, + "input": { + "config": { + "max_coinbase_settlement_outputs": 500, + "max_direct_coinbase_outputs": 10, + "max_fanout_recipients_per_transaction": 10, + "reserved_coinbase_outputs": 0 + }, + "direct_floor_sats": 20000, + "pinned_direct": null, + "recipients": [ + { + "amount_sats": 19999, + "order_key": "01", + "p2mr_program_hex": "0101010101010101010101010101010101010101010101010101010101010101", + "recipient_id": "below" + }, + { + "amount_sats": 20000, + "order_key": "02", + "p2mr_program_hex": "0202020202020202020202020202020202020202020202020202020202020202", + "recipient_id": "at" + }, + { + "amount_sats": 20001, + "order_key": "03", + "p2mr_program_hex": "0303030303030303030303030303030303030303030303030303030303030303", + "recipient_id": "above" + } + ] + }, + "name": "direct-floor-boundary", + "topic": 7, + "why": "One sat under the direct floor routes to fanout; exactly at and above it are direct." + }, + { + "entry_point": "select_settlement_mode", + "expected": { + "ok": { + "coinbase_settlement_output_count": 3, + "direct_amount_sats": 0, + "direct_recipient_count": 0, + "direct_recipients": [], + "fanout_amount_sats": 6000, + "fanout_chunk_count": 3, + "fanout_chunks": [ + { + "amount_sats": 2000, + "chunk_index": 0, + "recipient_count": 2, + "recipients": [ + { + "amount_sats": 1000, + "order_key": "01", + "p2mr_program_hex": "0101010101010101010101010101010101010101010101010101010101010101", + "recipient_id": "dust-1" + }, + { + "amount_sats": 1000, + "order_key": "02", + "p2mr_program_hex": "0202020202020202020202020202020202020202020202020202020202020202", + "recipient_id": "dust-2" + } + ] + }, + { + "amount_sats": 2000, + "chunk_index": 1, + "recipient_count": 2, + "recipients": [ + { + "amount_sats": 1000, + "order_key": "03", + "p2mr_program_hex": "0303030303030303030303030303030303030303030303030303030303030303", + "recipient_id": "dust-3" + }, + { + "amount_sats": 1000, + "order_key": "04", + "p2mr_program_hex": "0404040404040404040404040404040404040404040404040404040404040404", + "recipient_id": "dust-4" + } + ] + }, + { + "amount_sats": 2000, + "chunk_index": 2, + "recipient_count": 2, + "recipients": [ + { + "amount_sats": 1000, + "order_key": "05", + "p2mr_program_hex": "0505050505050505050505050505050505050505050505050505050505050505", + "recipient_id": "dust-5" + }, + { + "amount_sats": 1000, + "order_key": "06", + "p2mr_program_hex": "0606060606060606060606060606060606060606060606060606060606060606", + "recipient_id": "dust-6" + } + ] + } + ], + "fanout_recipient_count": 6, + "mode": "ctv_fanout", + "reason": "6 recipients routed to 3 CTV fanout chunk(s) capped at 2 recipients each; 3 coinbase settlement outputs used", + "reserved_coinbase_output_count": 0, + "total_amount_sats": 6000 + } + }, + "input": { + "config": { + "max_coinbase_settlement_outputs": 3, + "max_direct_coinbase_outputs": 0, + "max_fanout_recipients_per_transaction": 2, + "reserved_coinbase_outputs": 0 + }, + "direct_floor_sats": 20000, + "pinned_direct": null, + "recipients": [ + { + "amount_sats": 1000, + "order_key": "01", + "p2mr_program_hex": "0101010101010101010101010101010101010101010101010101010101010101", + "recipient_id": "dust-1" + }, + { + "amount_sats": 1000, + "order_key": "02", + "p2mr_program_hex": "0202020202020202020202020202020202020202020202020202020202020202", + "recipient_id": "dust-2" + }, + { + "amount_sats": 1000, + "order_key": "03", + "p2mr_program_hex": "0303030303030303030303030303030303030303030303030303030303030303", + "recipient_id": "dust-3" + }, + { + "amount_sats": 1000, + "order_key": "04", + "p2mr_program_hex": "0404040404040404040404040404040404040404040404040404040404040404", + "recipient_id": "dust-4" + }, + { + "amount_sats": 1000, + "order_key": "05", + "p2mr_program_hex": "0505050505050505050505050505050505050505050505050505050505050505", + "recipient_id": "dust-5" + }, + { + "amount_sats": 1000, + "order_key": "06", + "p2mr_program_hex": "0606060606060606060606060606060606060606060606060606060606060606", + "recipient_id": "dust-6" + } + ] + }, + "name": "chunks-at-the-output-cap", + "topic": 7, + "why": "Six sub-floor recipients at two per chunk need exactly the three-output coinbase budget." + }, + { + "entry_point": "select_settlement_mode", + "expected": { + "error": "cannot select a settlement mode: recipient partition requires more than 3 coinbase settlement outputs" + }, + "input": { + "config": { + "max_coinbase_settlement_outputs": 3, + "max_direct_coinbase_outputs": 0, + "max_fanout_recipients_per_transaction": 2, + "reserved_coinbase_outputs": 0 + }, + "direct_floor_sats": 20000, + "pinned_direct": null, + "recipients": [ + { + "amount_sats": 1000, + "order_key": "01", + "p2mr_program_hex": "0101010101010101010101010101010101010101010101010101010101010101", + "recipient_id": "dust-1" + }, + { + "amount_sats": 1000, + "order_key": "02", + "p2mr_program_hex": "0202020202020202020202020202020202020202020202020202020202020202", + "recipient_id": "dust-2" + }, + { + "amount_sats": 1000, + "order_key": "03", + "p2mr_program_hex": "0303030303030303030303030303030303030303030303030303030303030303", + "recipient_id": "dust-3" + }, + { + "amount_sats": 1000, + "order_key": "04", + "p2mr_program_hex": "0404040404040404040404040404040404040404040404040404040404040404", + "recipient_id": "dust-4" + }, + { + "amount_sats": 1000, + "order_key": "05", + "p2mr_program_hex": "0505050505050505050505050505050505050505050505050505050505050505", + "recipient_id": "dust-5" + }, + { + "amount_sats": 1000, + "order_key": "06", + "p2mr_program_hex": "0606060606060606060606060606060606060606060606060606060606060606", + "recipient_id": "dust-6" + }, + { + "amount_sats": 1000, + "order_key": "07", + "p2mr_program_hex": "0707070707070707070707070707070707070707070707070707070707070707", + "recipient_id": "dust-7" + } + ] + }, + "name": "chunks-one-over-the-output-cap-is-an-error", + "topic": 7, + "why": "A seventh recipient needs a fourth chunk, one more than the three-output budget." + }, + { + "entry_point": "select_settlement_mode", + "expected": { + "ok": { + "coinbase_settlement_output_count": 1, + "direct_amount_sats": 0, + "direct_recipient_count": 0, + "direct_recipients": [], + "fanout_amount_sats": 3000, + "fanout_chunk_count": 1, + "fanout_chunks": [ + { + "amount_sats": 3000, + "chunk_index": 0, + "recipient_count": 3, + "recipients": [ + { + "amount_sats": 1000, + "order_key": "01", + "p2mr_program_hex": "0101010101010101010101010101010101010101010101010101010101010101", + "recipient_id": "dust-1" + }, + { + "amount_sats": 1000, + "order_key": "02", + "p2mr_program_hex": "0202020202020202020202020202020202020202020202020202020202020202", + "recipient_id": "dust-2" + }, + { + "amount_sats": 1000, + "order_key": "03", + "p2mr_program_hex": "0303030303030303030303030303030303030303030303030303030303030303", + "recipient_id": "dust-3" + } + ] + } + ], + "fanout_recipient_count": 3, + "mode": "ctv_fanout", + "reason": "3 recipients routed to 1 CTV fanout chunk(s) capped at 1160 recipients each; 1 coinbase settlement outputs used", + "reserved_coinbase_output_count": 0, + "total_amount_sats": 3000 + } + }, + "input": { + "config": { + "max_coinbase_settlement_outputs": 16, + "max_direct_coinbase_outputs": 12, + "max_fanout_recipients_per_transaction": 1160, + "reserved_coinbase_outputs": 0 + }, + "direct_floor_sats": 20000, + "pinned_direct": null, + "recipients": [ + { + "amount_sats": 1000, + "order_key": "01", + "p2mr_program_hex": "0101010101010101010101010101010101010101010101010101010101010101", + "recipient_id": "dust-1" + }, + { + "amount_sats": 1000, + "order_key": "02", + "p2mr_program_hex": "0202020202020202020202020202020202020202020202020202020202020202", + "recipient_id": "dust-2" + }, + { + "amount_sats": 1000, + "order_key": "03", + "p2mr_program_hex": "0303030303030303030303030303030303030303030303030303030303030303", + "recipient_id": "dust-3" + } + ] + }, + "name": "fanout-size-at-the-truc-cap", + "topic": 7, + "why": "1160 recipients per fanout (49973 weight bytes) is the largest chunk size under the 50000-byte TRUC limit." + }, + { + "entry_point": "select_settlement_mode", + "expected": { + "error": "cannot select a settlement mode: max_fanout_recipients_per_transaction must be <= 1160" + }, + "input": { + "config": { + "max_coinbase_settlement_outputs": 16, + "max_direct_coinbase_outputs": 12, + "max_fanout_recipients_per_transaction": 1161, + "reserved_coinbase_outputs": 0 + }, + "direct_floor_sats": 20000, + "pinned_direct": null, + "recipients": [ + { + "amount_sats": 1000, + "order_key": "01", + "p2mr_program_hex": "0101010101010101010101010101010101010101010101010101010101010101", + "recipient_id": "dust-1" + }, + { + "amount_sats": 1000, + "order_key": "02", + "p2mr_program_hex": "0202020202020202020202020202020202020202020202020202020202020202", + "recipient_id": "dust-2" + }, + { + "amount_sats": 1000, + "order_key": "03", + "p2mr_program_hex": "0303030303030303030303030303030303030303030303030303030303030303", + "recipient_id": "dust-3" + } + ] + }, + "name": "fanout-size-one-over-the-truc-cap-is-an-error", + "topic": 7, + "why": "1161 recipients per fanout exceeds the per-transaction maximum." + }, + { + "entry_point": "select_settlement_mode", + "expected": { + "ok": { + "coinbase_settlement_output_count": 3, + "direct_amount_sats": 30025, + "direct_recipient_count": 2, + "direct_recipients": [ + { + "amount_sats": 30000, + "order_key": "01", + "p2mr_program_hex": "0101010101010101010101010101010101010101010101010101010101010101", + "recipient_id": "miner-a" + }, + { + "amount_sats": 25, + "order_key": "ff", + "p2mr_program_hex": "ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff", + "recipient_id": "pool-fee" + } + ], + "fanout_amount_sats": 1000, + "fanout_chunk_count": 1, + "fanout_chunks": [ + { + "amount_sats": 1000, + "chunk_index": 0, + "recipient_count": 1, + "recipients": [ + { + "amount_sats": 1000, + "order_key": "02", + "p2mr_program_hex": "0202020202020202020202020202020202020202020202020202020202020202", + "recipient_id": "miner-b" + } + ] + } + ], + "fanout_recipient_count": 1, + "mode": "hybrid_coinbase_ctv_fanout", + "reason": "2 recipients paid directly by largest-liability priority; 1 recipients routed to 1 CTV fanout chunk(s); 3 of 500 coinbase settlement outputs used; 1 direct settlement slot(s) pinned by the coinbase output policy", + "reserved_coinbase_output_count": 0, + "total_amount_sats": 31025 + } + }, + "input": { + "config": { + "max_coinbase_settlement_outputs": 500, + "max_direct_coinbase_outputs": 2, + "max_fanout_recipients_per_transaction": 1000, + "reserved_coinbase_outputs": 0 + }, + "direct_floor_sats": 20000, + "pinned_direct": { + "amount_sats": 25, + "order_key": "ff", + "p2mr_program_hex": "ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff", + "recipient_id": "pool-fee" + }, + "recipients": [ + { + "amount_sats": 30000, + "order_key": "01", + "p2mr_program_hex": "0101010101010101010101010101010101010101010101010101010101010101", + "recipient_id": "miner-a" + }, + { + "amount_sats": 1000, + "order_key": "02", + "p2mr_program_hex": "0202020202020202020202020202020202020202020202020202020202020202", + "recipient_id": "miner-b" + }, + { + "amount_sats": 25, + "order_key": "ff", + "p2mr_program_hex": "ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff", + "recipient_id": "pool-fee" + } + ] + }, + "name": "pinned-direct-below-floor", + "topic": 7, + "why": "A 25-sat pinned pool fee is settled directly, takes one of the two direct slots, and never enters a fanout chunk." + }, + { + "entry_point": "select_settlement_mode", + "expected": { + "error": "cannot select a settlement mode: pinned direct recipient requires max_direct_coinbase_outputs >= 1" + }, + "input": { + "config": { + "max_coinbase_settlement_outputs": 500, + "max_direct_coinbase_outputs": 0, + "max_fanout_recipients_per_transaction": 1000, + "reserved_coinbase_outputs": 0 + }, + "direct_floor_sats": 20000, + "pinned_direct": { + "amount_sats": 25, + "order_key": "ff", + "p2mr_program_hex": "ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff", + "recipient_id": "pool-fee" + }, + "recipients": [ + { + "amount_sats": 30000, + "order_key": "01", + "p2mr_program_hex": "0101010101010101010101010101010101010101010101010101010101010101", + "recipient_id": "miner-a" + }, + { + "amount_sats": 25, + "order_key": "ff", + "p2mr_program_hex": "ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff", + "recipient_id": "pool-fee" + } + ] + }, + "name": "pinned-direct-without-a-direct-slot-is-an-error", + "topic": 7, + "why": "A pinned recipient needs max_direct_coinbase_outputs of at least one." + } + ], + "produced_by": "PYTHONDONTWRITEBYTECODE=1 PYTHONPATH=. python3 crates/qbit-prism/examples/export_rule_decisions.py | cargo run --locked -q -p qbit-prism --example export_money_path_vectors -- vectors-out", + "schema": "qbit.prism.money-path-vectors.v1", + "source_commit": "504846cc0b72e8f86ed17f896d4ccbbe196a31dc", + "topics": [ + { + "id": 7, + "name": "CTV chunk splits" + } + ] +} diff --git a/crates/qbit-prism/fixtures/vectors/window.json b/crates/qbit-prism/fixtures/vectors/window.json new file mode 100644 index 000000000..19a0a6a87 --- /dev/null +++ b/crates/qbit-prism/fixtures/vectors/window.json @@ -0,0 +1,802 @@ +{ + "cases": [ + { + "entry_point": "compute_prism_window", + "expected": { + "ok": { + "anchor_job_issued_at_ms": 1003, + "anchor_share_seq": 4, + "counted_window_weight": 80, + "entitlements": [ + { + "order_key": "03", + "p2mr_program_hex": "0303030303030303030303030303030303030303030303030303030303030303", + "recipient_id": "miner-a", + "weight": 30 + }, + { + "order_key": "04", + "p2mr_program_hex": "0404040404040404040404040404040404040404040404040404040404040404", + "recipient_id": "miner-c", + "weight": 50 + } + ], + "requested_window_weight": 80, + "shares": [ + { + "accepted_at_ms": 1003, + "counted_difficulty": 50, + "job_issued_at_ms": 1003, + "miner_id": "miner-c", + "order_key": "04", + "p2mr_program_hex": "0404040404040404040404040404040404040404040404040404040404040404", + "share_difficulty": 50, + "share_id": "share-4", + "share_seq": 4 + }, + { + "accepted_at_ms": 1002, + "counted_difficulty": 30, + "job_issued_at_ms": 1002, + "miner_id": "miner-a", + "order_key": "03", + "p2mr_program_hex": "0303030303030303030303030303030303030303030303030303030303030303", + "share_difficulty": 30, + "share_id": "share-3", + "share_seq": 3 + } + ] + } + }, + "input": { + "found_block": { + "anchor_job_issued_at_ms": 1003, + "block_height": 101, + "coinbase_value_sats": 500000000, + "network_difficulty": 10 + }, + "shares": [ + { + "accepted_at_ms": 1000, + "job_id": "job-1000", + "job_issued_at_ms": 1000, + "miner_id": "miner-old", + "network_difficulty": 10, + "ntime": 1800000000, + "order_key": "01", + "p2mr_program_hex": "0101010101010101010101010101010101010101010101010101010101010101", + "share_difficulty": 999, + "share_id": "share-1", + "share_seq": 1, + "template_height": 100 + }, + { + "accepted_at_ms": 1001, + "job_id": "job-1001", + "job_issued_at_ms": 1001, + "miner_id": "miner-b", + "network_difficulty": 10, + "ntime": 1800000000, + "order_key": "02", + "p2mr_program_hex": "0202020202020202020202020202020202020202020202020202020202020202", + "share_difficulty": 40, + "share_id": "share-2", + "share_seq": 2, + "template_height": 100 + }, + { + "accepted_at_ms": 1002, + "job_id": "job-1002", + "job_issued_at_ms": 1002, + "miner_id": "miner-a", + "network_difficulty": 10, + "ntime": 1800000000, + "order_key": "03", + "p2mr_program_hex": "0303030303030303030303030303030303030303030303030303030303030303", + "share_difficulty": 30, + "share_id": "share-3", + "share_seq": 3, + "template_height": 100 + }, + { + "accepted_at_ms": 1003, + "job_id": "job-1003", + "job_issued_at_ms": 1003, + "miner_id": "miner-c", + "network_difficulty": 10, + "ntime": 1800000000, + "order_key": "04", + "p2mr_program_hex": "0404040404040404040404040404040404040404040404040404040404040404", + "share_difficulty": 50, + "share_id": "share-4", + "share_seq": 4, + "template_height": 100 + } + ] + }, + "name": "exact-fit-boundary", + "topic": 1, + "why": "8 x 10 = 80 is filled exactly by the two newest shares (50 + 30); the older shares get nothing." + }, + { + "entry_point": "compute_prism_window", + "expected": { + "ok": { + "anchor_job_issued_at_ms": 1003, + "anchor_share_seq": 4, + "counted_window_weight": 72, + "entitlements": [ + { + "order_key": "03", + "p2mr_program_hex": "0303030303030303030303030303030303030303030303030303030303030303", + "recipient_id": "miner-a", + "weight": 22 + }, + { + "order_key": "04", + "p2mr_program_hex": "0404040404040404040404040404040404040404040404040404040404040404", + "recipient_id": "miner-c", + "weight": 50 + } + ], + "requested_window_weight": 72, + "shares": [ + { + "accepted_at_ms": 1003, + "counted_difficulty": 50, + "job_issued_at_ms": 1003, + "miner_id": "miner-c", + "order_key": "04", + "p2mr_program_hex": "0404040404040404040404040404040404040404040404040404040404040404", + "share_difficulty": 50, + "share_id": "share-4", + "share_seq": 4 + }, + { + "accepted_at_ms": 1002, + "counted_difficulty": 22, + "job_issued_at_ms": 1002, + "miner_id": "miner-a", + "order_key": "03", + "p2mr_program_hex": "0303030303030303030303030303030303030303030303030303030303030303", + "share_difficulty": 30, + "share_id": "share-3", + "share_seq": 3 + } + ] + } + }, + "input": { + "found_block": { + "anchor_job_issued_at_ms": 1003, + "block_height": 101, + "coinbase_value_sats": 500000000, + "network_difficulty": 9 + }, + "shares": [ + { + "accepted_at_ms": 1000, + "job_id": "job-1000", + "job_issued_at_ms": 1000, + "miner_id": "miner-old", + "network_difficulty": 10, + "ntime": 1800000000, + "order_key": "01", + "p2mr_program_hex": "0101010101010101010101010101010101010101010101010101010101010101", + "share_difficulty": 999, + "share_id": "share-1", + "share_seq": 1, + "template_height": 100 + }, + { + "accepted_at_ms": 1001, + "job_id": "job-1001", + "job_issued_at_ms": 1001, + "miner_id": "miner-b", + "network_difficulty": 10, + "ntime": 1800000000, + "order_key": "02", + "p2mr_program_hex": "0202020202020202020202020202020202020202020202020202020202020202", + "share_difficulty": 40, + "share_id": "share-2", + "share_seq": 2, + "template_height": 100 + }, + { + "accepted_at_ms": 1002, + "job_id": "job-1002", + "job_issued_at_ms": 1002, + "miner_id": "miner-a", + "network_difficulty": 10, + "ntime": 1800000000, + "order_key": "03", + "p2mr_program_hex": "0303030303030303030303030303030303030303030303030303030303030303", + "share_difficulty": 30, + "share_id": "share-3", + "share_seq": 3, + "template_height": 100 + }, + { + "accepted_at_ms": 1003, + "job_id": "job-1003", + "job_issued_at_ms": 1003, + "miner_id": "miner-c", + "network_difficulty": 10, + "ntime": 1800000000, + "order_key": "04", + "p2mr_program_hex": "0404040404040404040404040404040404040404040404040404040404040404", + "share_difficulty": 50, + "share_id": "share-4", + "share_seq": 4, + "template_height": 100 + } + ] + }, + "name": "fractional-oldest-share", + "topic": 2, + "why": "The same shares with network difficulty 9: 72 = 50 + 22, so the oldest counted share counts 22 of its 30." + }, + { + "entry_point": "compute_prism_window", + "expected": { + "ok": { + "anchor_job_issued_at_ms": 1001, + "anchor_share_seq": 2, + "counted_window_weight": 8, + "entitlements": [ + { + "order_key": "01", + "p2mr_program_hex": "0101010101010101010101010101010101010101010101010101010101010101", + "recipient_id": "miner-a", + "weight": 1 + }, + { + "order_key": "02", + "p2mr_program_hex": "0202020202020202020202020202020202020202020202020202020202020202", + "recipient_id": "miner-b", + "weight": 7 + } + ], + "requested_window_weight": 8, + "shares": [ + { + "accepted_at_ms": 1001, + "counted_difficulty": 7, + "job_issued_at_ms": 1001, + "miner_id": "miner-b", + "order_key": "02", + "p2mr_program_hex": "0202020202020202020202020202020202020202020202020202020202020202", + "share_difficulty": 7, + "share_id": "share-2", + "share_seq": 2 + }, + { + "accepted_at_ms": 1000, + "counted_difficulty": 1, + "job_issued_at_ms": 1000, + "miner_id": "miner-a", + "order_key": "01", + "p2mr_program_hex": "0101010101010101010101010101010101010101010101010101010101010101", + "share_difficulty": 1000, + "share_id": "share-1", + "share_seq": 1 + } + ] + } + }, + "input": { + "found_block": { + "anchor_job_issued_at_ms": 1001, + "block_height": 101, + "coinbase_value_sats": 500000000, + "network_difficulty": 1 + }, + "shares": [ + { + "accepted_at_ms": 1000, + "job_id": "job-1000", + "job_issued_at_ms": 1000, + "miner_id": "miner-a", + "network_difficulty": 10, + "ntime": 1800000000, + "order_key": "01", + "p2mr_program_hex": "0101010101010101010101010101010101010101010101010101010101010101", + "share_difficulty": 1000, + "share_id": "share-1", + "share_seq": 1, + "template_height": 100 + }, + { + "accepted_at_ms": 1001, + "job_id": "job-1001", + "job_issued_at_ms": 1001, + "miner_id": "miner-b", + "network_difficulty": 10, + "ntime": 1800000000, + "order_key": "02", + "p2mr_program_hex": "0202020202020202020202020202020202020202020202020202020202020202", + "share_difficulty": 7, + "share_id": "share-2", + "share_seq": 2, + "template_height": 100 + } + ] + }, + "name": "oldest-share-counts-one-unit", + "topic": 2, + "why": "Weight 8 after a 7-difficulty share leaves exactly one unit for a 1000-difficulty share." + }, + { + "entry_point": "compute_prism_window", + "expected": { + "ok": { + "anchor_job_issued_at_ms": 1002, + "anchor_share_seq": 3, + "counted_window_weight": 60, + "entitlements": [ + { + "order_key": "01", + "p2mr_program_hex": "0101010101010101010101010101010101010101010101010101010101010101", + "recipient_id": "miner-a", + "weight": 40 + }, + { + "order_key": "02", + "p2mr_program_hex": "0202020202020202020202020202020202020202020202020202020202020202", + "recipient_id": "miner-b", + "weight": 20 + } + ], + "requested_window_weight": 800, + "shares": [ + { + "accepted_at_ms": 1002, + "counted_difficulty": 30, + "job_issued_at_ms": 1002, + "miner_id": "miner-a", + "order_key": "01", + "p2mr_program_hex": "0101010101010101010101010101010101010101010101010101010101010101", + "share_difficulty": 30, + "share_id": "share-3", + "share_seq": 3 + }, + { + "accepted_at_ms": 1001, + "counted_difficulty": 20, + "job_issued_at_ms": 1001, + "miner_id": "miner-b", + "order_key": "02", + "p2mr_program_hex": "0202020202020202020202020202020202020202020202020202020202020202", + "share_difficulty": 20, + "share_id": "share-2", + "share_seq": 2 + }, + { + "accepted_at_ms": 1000, + "counted_difficulty": 10, + "job_issued_at_ms": 1000, + "miner_id": "miner-a", + "order_key": "01", + "p2mr_program_hex": "0101010101010101010101010101010101010101010101010101010101010101", + "share_difficulty": 10, + "share_id": "share-1", + "share_seq": 1 + } + ] + } + }, + "input": { + "found_block": { + "anchor_job_issued_at_ms": 1002, + "block_height": 101, + "coinbase_value_sats": 500000000, + "network_difficulty": 100 + }, + "shares": [ + { + "accepted_at_ms": 1000, + "job_id": "job-1000", + "job_issued_at_ms": 1000, + "miner_id": "miner-a", + "network_difficulty": 10, + "ntime": 1800000000, + "order_key": "01", + "p2mr_program_hex": "0101010101010101010101010101010101010101010101010101010101010101", + "share_difficulty": 10, + "share_id": "share-1", + "share_seq": 1, + "template_height": 100 + }, + { + "accepted_at_ms": 1001, + "job_id": "job-1001", + "job_issued_at_ms": 1001, + "miner_id": "miner-b", + "network_difficulty": 10, + "ntime": 1800000000, + "order_key": "02", + "p2mr_program_hex": "0202020202020202020202020202020202020202020202020202020202020202", + "share_difficulty": 20, + "share_id": "share-2", + "share_seq": 2, + "template_height": 100 + }, + { + "accepted_at_ms": 1002, + "job_id": "job-1002", + "job_issued_at_ms": 1002, + "miner_id": "miner-a", + "network_difficulty": 10, + "ntime": 1800000000, + "order_key": "01", + "p2mr_program_hex": "0101010101010101010101010101010101010101010101010101010101010101", + "share_difficulty": 30, + "share_id": "share-3", + "share_seq": 3, + "template_height": 100 + } + ] + }, + "name": "small-log-below-weight", + "topic": 1, + "why": "The whole eligible log (60) is below the requested weight (800), so every share counts in full." + }, + { + "entry_point": "compute_prism_window", + "expected": { + "ok": { + "anchor_job_issued_at_ms": 1001, + "anchor_share_seq": 2, + "counted_window_weight": 30, + "entitlements": [ + { + "order_key": "01", + "p2mr_program_hex": "0101010101010101010101010101010101010101010101010101010101010101", + "recipient_id": "miner-a", + "weight": 10 + }, + { + "order_key": "02", + "p2mr_program_hex": "0202020202020202020202020202020202020202020202020202020202020202", + "recipient_id": "miner-b", + "weight": 20 + } + ], + "requested_window_weight": 80, + "shares": [ + { + "accepted_at_ms": 1001, + "counted_difficulty": 20, + "job_issued_at_ms": 1001, + "miner_id": "miner-b", + "order_key": "02", + "p2mr_program_hex": "0202020202020202020202020202020202020202020202020202020202020202", + "share_difficulty": 20, + "share_id": "share-2", + "share_seq": 2 + }, + { + "accepted_at_ms": 1000, + "counted_difficulty": 10, + "job_issued_at_ms": 1000, + "miner_id": "miner-a", + "order_key": "01", + "p2mr_program_hex": "0101010101010101010101010101010101010101010101010101010101010101", + "share_difficulty": 10, + "share_id": "share-1", + "share_seq": 1 + } + ] + } + }, + "input": { + "found_block": { + "anchor_job_issued_at_ms": 1001, + "block_height": 101, + "coinbase_value_sats": 500000000, + "network_difficulty": 10 + }, + "shares": [ + { + "accepted_at_ms": 1000, + "job_id": "job-1000", + "job_issued_at_ms": 1000, + "miner_id": "miner-a", + "network_difficulty": 10, + "ntime": 1800000000, + "order_key": "01", + "p2mr_program_hex": "0101010101010101010101010101010101010101010101010101010101010101", + "share_difficulty": 10, + "share_id": "share-1", + "share_seq": 1, + "template_height": 100 + }, + { + "accepted_at_ms": 1001, + "job_id": "job-1001", + "job_issued_at_ms": 1001, + "miner_id": "miner-b", + "network_difficulty": 10, + "ntime": 1800000000, + "order_key": "02", + "p2mr_program_hex": "0202020202020202020202020202020202020202020202020202020202020202", + "share_difficulty": 20, + "share_id": "share-2", + "share_seq": 2, + "template_height": 100 + }, + { + "accepted_at_ms": 1500, + "job_id": "job-2000", + "job_issued_at_ms": 2000, + "miner_id": "miner-late", + "network_difficulty": 10, + "ntime": 1800000000, + "order_key": "03", + "p2mr_program_hex": "0303030303030303030303030303030303030303030303030303030303030303", + "share_difficulty": 1000, + "share_id": "share-3", + "share_seq": 3, + "template_height": 100 + }, + { + "accepted_at_ms": 1002, + "job_id": "job-1000", + "job_issued_at_ms": 1000, + "miner_id": "miner-slow", + "network_difficulty": 10, + "ntime": 1800000000, + "order_key": "04", + "p2mr_program_hex": "0404040404040404040404040404040404040404040404040404040404040404", + "share_difficulty": 1000, + "share_id": "share-4", + "share_seq": 4, + "template_height": 100 + } + ] + }, + "name": "shares-after-anchor-excluded", + "topic": 1, + "why": "A share whose job was issued after the anchor, and one issued before it but accepted after it, are both excluded." + }, + { + "entry_point": "compute_prism_window", + "expected": { + "ok": { + "anchor_job_issued_at_ms": 1003, + "anchor_share_seq": 4, + "counted_window_weight": 72, + "entitlements": [ + { + "order_key": "03", + "p2mr_program_hex": "0303030303030303030303030303030303030303030303030303030303030303", + "recipient_id": "miner-a", + "weight": 22 + }, + { + "order_key": "04", + "p2mr_program_hex": "0404040404040404040404040404040404040404040404040404040404040404", + "recipient_id": "miner-c", + "weight": 50 + } + ], + "requested_window_weight": 72, + "shares": [ + { + "accepted_at_ms": 1003, + "counted_difficulty": 50, + "job_issued_at_ms": 1003, + "miner_id": "miner-c", + "order_key": "04", + "p2mr_program_hex": "0404040404040404040404040404040404040404040404040404040404040404", + "share_difficulty": 50, + "share_id": "share-4", + "share_seq": 4 + }, + { + "accepted_at_ms": 1002, + "counted_difficulty": 22, + "job_issued_at_ms": 1002, + "miner_id": "miner-a", + "order_key": "03", + "p2mr_program_hex": "0303030303030303030303030303030303030303030303030303030303030303", + "share_difficulty": 30, + "share_id": "share-3", + "share_seq": 3 + } + ] + } + }, + "input": { + "found_block": { + "anchor_job_issued_at_ms": 1003, + "block_height": 101, + "coinbase_value_sats": 500000000, + "network_difficulty": 9 + }, + "shares": [ + { + "accepted_at_ms": 1002, + "job_id": "job-1002", + "job_issued_at_ms": 1002, + "miner_id": "miner-a", + "network_difficulty": 10, + "ntime": 1800000000, + "order_key": "03", + "p2mr_program_hex": "0303030303030303030303030303030303030303030303030303030303030303", + "share_difficulty": 30, + "share_id": "share-3", + "share_seq": 3, + "template_height": 100 + }, + { + "accepted_at_ms": 1003, + "job_id": "job-1003", + "job_issued_at_ms": 1003, + "miner_id": "miner-c", + "network_difficulty": 10, + "ntime": 1800000000, + "order_key": "04", + "p2mr_program_hex": "0404040404040404040404040404040404040404040404040404040404040404", + "share_difficulty": 50, + "share_id": "share-4", + "share_seq": 4, + "template_height": 100 + }, + { + "accepted_at_ms": 1000, + "job_id": "job-1000", + "job_issued_at_ms": 1000, + "miner_id": "miner-old", + "network_difficulty": 10, + "ntime": 1800000000, + "order_key": "01", + "p2mr_program_hex": "0101010101010101010101010101010101010101010101010101010101010101", + "share_difficulty": 999, + "share_id": "share-1", + "share_seq": 1, + "template_height": 100 + }, + { + "accepted_at_ms": 1001, + "job_id": "job-1001", + "job_issued_at_ms": 1001, + "miner_id": "miner-b", + "network_difficulty": 10, + "ntime": 1800000000, + "order_key": "02", + "p2mr_program_hex": "0202020202020202020202020202020202020202020202020202020202020202", + "share_difficulty": 40, + "share_id": "share-2", + "share_seq": 2, + "template_height": 100 + } + ] + }, + "name": "unsorted-input", + "topic": 1, + "why": "The boundary shares in scrambled sequence order give the same newest-first window as sorted input." + }, + { + "entry_point": "compute_prism_window", + "expected": { + "error": "network difficulty must be positive" + }, + "input": { + "found_block": { + "anchor_job_issued_at_ms": 1003, + "block_height": 101, + "coinbase_value_sats": 500000000, + "network_difficulty": 0 + }, + "shares": [ + { + "accepted_at_ms": 1000, + "job_id": "job-1000", + "job_issued_at_ms": 1000, + "miner_id": "miner-old", + "network_difficulty": 10, + "ntime": 1800000000, + "order_key": "01", + "p2mr_program_hex": "0101010101010101010101010101010101010101010101010101010101010101", + "share_difficulty": 999, + "share_id": "share-1", + "share_seq": 1, + "template_height": 100 + }, + { + "accepted_at_ms": 1001, + "job_id": "job-1001", + "job_issued_at_ms": 1001, + "miner_id": "miner-b", + "network_difficulty": 10, + "ntime": 1800000000, + "order_key": "02", + "p2mr_program_hex": "0202020202020202020202020202020202020202020202020202020202020202", + "share_difficulty": 40, + "share_id": "share-2", + "share_seq": 2, + "template_height": 100 + }, + { + "accepted_at_ms": 1002, + "job_id": "job-1002", + "job_issued_at_ms": 1002, + "miner_id": "miner-a", + "network_difficulty": 10, + "ntime": 1800000000, + "order_key": "03", + "p2mr_program_hex": "0303030303030303030303030303030303030303030303030303030303030303", + "share_difficulty": 30, + "share_id": "share-3", + "share_seq": 3, + "template_height": 100 + }, + { + "accepted_at_ms": 1003, + "job_id": "job-1003", + "job_issued_at_ms": 1003, + "miner_id": "miner-c", + "network_difficulty": 10, + "ntime": 1800000000, + "order_key": "04", + "p2mr_program_hex": "0404040404040404040404040404040404040404040404040404040404040404", + "share_difficulty": 50, + "share_id": "share-4", + "share_seq": 4, + "template_height": 100 + } + ] + }, + "name": "zero-network-difficulty-is-an-error", + "topic": 1, + "why": "A found block with network difficulty 0 cannot define a window." + }, + { + "entry_point": "compute_prism_window", + "expected": { + "error": "no eligible shares at or before job issue time 1000" + }, + "input": { + "found_block": { + "anchor_job_issued_at_ms": 1000, + "block_height": 101, + "coinbase_value_sats": 500000000, + "network_difficulty": 10 + }, + "shares": [ + { + "accepted_at_ms": 2000, + "job_id": "job-2000", + "job_issued_at_ms": 2000, + "miner_id": "miner-a", + "network_difficulty": 10, + "ntime": 1800000000, + "order_key": "01", + "p2mr_program_hex": "0101010101010101010101010101010101010101010101010101010101010101", + "share_difficulty": 10, + "share_id": "share-1", + "share_seq": 1, + "template_height": 100 + } + ] + }, + "name": "no-eligible-share-is-an-error", + "topic": 1, + "why": "Every share is after the anchor, so the window is empty." + } + ], + "produced_by": "PYTHONDONTWRITEBYTECODE=1 PYTHONPATH=. python3 crates/qbit-prism/examples/export_rule_decisions.py | cargo run --locked -q -p qbit-prism --example export_money_path_vectors -- vectors-out", + "schema": "qbit.prism.money-path-vectors.v1", + "source_commit": "504846cc0b72e8f86ed17f896d4ccbbe196a31dc", + "topics": [ + { + "id": 1, + "name": "window clipping and the oldest-share boundary" + }, + { + "id": 2, + "name": "fractional oldest share" + } + ] +} diff --git a/crates/qbit-prism/sql/001_share_ledger.sql b/crates/qbit-prism/sql/001_share_ledger.sql index 02fb55488..e084f5eaf 100644 --- a/crates/qbit-prism/sql/001_share_ledger.sql +++ b/crates/qbit-prism/sql/001_share_ledger.sql @@ -1,8 +1,8 @@ -- Canonical ordered share ledger for qbit PRISM mining. -- --- Invariant: only one logical writer inserts into qbit_share_ledger. Stratum --- frontends may scale horizontally, but they must feed that writer through a --- queue instead of inserting shares independently. +-- Base schema retained for migration of existing ledgers. The native server's +-- 002_multi_instance.sql migration replaces the process writer lease with +-- short transaction locks that order commits from every active frontend. -- -- The whole file applies inside exactly one transaction, enforced by the -- BEGIN/COMMIT wrapper below rather than by any caller's flags. This is what @@ -1136,7 +1136,7 @@ AS $$ -- strictly positive (schema CHECK): positivity keeps the cumulative -- weight strictly increasing, which guarantees the crossing row lies -- inside the last fetched page. Relaxing that constraint requires - -- revisiting this walk and the matching one in lab/prism/share_ledger.py. + -- revisiting this walk and the native server's ledger window scan. WITH RECURSIVE pages AS ( SELECT page.min_share_seq, page.page_weight, diff --git a/crates/qbit-prism/sql/001_share_ledger_revert_audit_publication_sequence.sql b/crates/qbit-prism/sql/001_share_ledger_revert_audit_publication_sequence.sql index 1d07deebd..5138ae04f 100644 --- a/crates/qbit-prism/sql/001_share_ledger_revert_audit_publication_sequence.sql +++ b/crates/qbit-prism/sql/001_share_ledger_revert_audit_publication_sequence.sql @@ -14,9 +14,11 @@ -- base backup. -- -- DATA LOSS: dropping the column discards every assigned publication --- ordinal. Read "Publication Ordinal Rollback And Schema Revert" in --- docs/prism-ledger-ops.md first; it states what is lost and the required --- procedure (stop the coordinator, take a backup). +-- ordinal. This is a legacy Python-only tool: stop every writer and take a +-- backup first. Native deployments use one-way migration and forward repair +-- or isolated-restore reconciliation; see docs/prism-rust-migration.md#recovery-and-rollback. +-- An older restore after a native ACK loses acknowledged history. The guard +-- below refuses native schemas before any destructive statement. -- -- The revert is one transaction serialized behind the same advisory lock as -- the forward migration. Any failing step aborts the whole file, so the @@ -30,6 +32,16 @@ SELECT pg_advisory_xact_lock( hashtext('qbit_audit_publication_sequence_migration') ); +DO $$ +BEGIN + IF to_regclass(format('%I.qbit_prism_schema_migrations', current_schema())) + IS NOT NULL THEN + RAISE EXCEPTION + 'legacy ordinal revert cannot run against native Prism; one-way migration; use forward repair or isolated-restore reconciliation in docs/prism-rust-migration.md#recovery-and-rollback; never discard acknowledged shares without accounting reconciliation'; + END IF; +END; +$$; + -- Fail before touching anything if the pool-block table is missing, then -- take the table lock every later step relies on. ACCESS EXCLUSIVE is what -- DROP COLUMN acquires anyway; taking it first excludes every reader and diff --git a/crates/qbit-prism/src/audit_body_ref.rs b/crates/qbit-prism/src/audit_body_ref.rs index 398b45982..caa65fdac 100644 --- a/crates/qbit-prism/src/audit_body_ref.rs +++ b/crates/qbit-prism/src/audit_body_ref.rs @@ -67,7 +67,7 @@ struct AuditWindowCompletenessProof { share_parts: Vec, } -#[derive(Debug, Deserialize, Serialize)] +#[derive(Debug, Serialize)] #[serde(tag = "kind")] enum SharePart { #[serde(rename = "segment")] @@ -107,6 +107,96 @@ enum SharePart { }, } +// Serde's internally tagged enum buffer cannot represent u128, including +// difficulties larger than u64::MAX. Dispatch on the JSON tag explicitly so +// inline shares retain serde_json's arbitrary-precision integer decoding. +impl<'de> Deserialize<'de> for SharePart { + fn deserialize>(deserializer: D) -> Result { + #[derive(Deserialize)] + struct Fields { + kind: String, + first_share_seq: u64, + last_share_seq: u64, + share_count: usize, + segment_first_share_seq: Option, + segment_last_share_seq: Option, + sha256: Option, + body_uri: Option, + range_sha256: Option, + prefix_sha256: Option, + shares: Option>, + } + let value = Value::deserialize(deserializer)?; + let fields: Fields = serde_json::from_value(value).map_err(serde::de::Error::custom)?; + let missing = |field: &'static str| ::missing_field(field); + let first_share_seq = fields.first_share_seq; + let last_share_seq = fields.last_share_seq; + let share_count = fields.share_count; + match fields.kind.as_str() { + "segment" => Ok(Self::Segment { + first_share_seq, + last_share_seq, + share_count, + sha256: fields.sha256.ok_or_else(|| missing("sha256"))?, + body_uri: fields.body_uri.ok_or_else(|| missing("body_uri"))?, + }), + "segment_range" => Ok(Self::SegmentRange { + first_share_seq, + last_share_seq, + share_count, + segment_first_share_seq: fields.segment_first_share_seq, + segment_last_share_seq: fields.segment_last_share_seq, + range_sha256: fields.range_sha256.ok_or_else(|| missing("range_sha256"))?, + body_uri: fields.body_uri.ok_or_else(|| missing("body_uri"))?, + }), + "segment_prefix" => Ok(Self::SegmentPrefix { + first_share_seq, + last_share_seq, + share_count, + prefix_sha256: fields + .prefix_sha256 + .ok_or_else(|| missing("prefix_sha256"))?, + body_uri: fields.body_uri.ok_or_else(|| missing("body_uri"))?, + }), + "inline" => Ok(Self::Inline { + first_share_seq, + last_share_seq, + share_count, + shares: fields.shares.ok_or_else(|| missing("shares"))?, + }), + other => Err(serde::de::Error::unknown_variant( + other, + &["segment", "segment_range", "segment_prefix", "inline"], + )), + } + } +} + +#[cfg(test)] +mod inline_share_tests { + use super::*; + + #[test] + fn internally_tagged_inline_shares_preserve_u128_difficulty() { + let difficulty = u128::from(u64::MAX) + 100; + let part = serde_json::json!({ + "kind":"inline", "first_share_seq":1, "last_share_seq":1, + "share_count":1, "shares":[{ + "share_seq":1,"share_id":"test","miner_id":"miner", + "order_key":"miner","p2mr_program_hex":"11".repeat(32), + "share_difficulty":difficulty,"network_difficulty":difficulty, + "template_height":1,"job_id":"job","job_issued_at_ms":1, + "accepted_at_ms":1,"ntime":1 + }] + }); + let SharePart::Inline { shares, .. } = serde_json::from_value(part).unwrap() else { + panic!("expected inline share part") + }; + assert_eq!(shares[0].share_difficulty, difficulty); + assert_eq!(shares[0].network_difficulty, difficulty); + } +} + #[derive(Clone, Debug, Deserialize, Serialize)] struct AuditShareSegment { schema: String, diff --git a/crates/qbit-prism/src/audit_hash.rs b/crates/qbit-prism/src/audit_hash.rs new file mode 100644 index 000000000..04f9dd59c --- /dev/null +++ b/crates/qbit-prism/src/audit_hash.rs @@ -0,0 +1,407 @@ +//! One canonical field definition for whole and split audit serialization. +use super::*; +use serde::ser::SerializeStruct; +use std::io::Write; + +impl Serialize for AuditBundleRef<'_> { + fn serialize(&self, serializer: S) -> Result { + let mut state = serializer.serialize_struct("AuditBundle", self.field_count() + 2)?; + state.serialize_field("schema", self.schema)?; + state.serialize_field("shares", self.shares)?; + self.serialize_suffix(state) + } +} + +impl AuditBundleRef<'_> { + fn field_count(&self) -> usize { + 7 + usize::from(self.coinbase_script_sig_suffix_hex.is_some()) + + usize::from(!self.witness_merkle_leaves_hex.is_empty()) + + usize::from(!self.audit_commitment_leaves_hex.is_empty()) + + usize::from(self.audit_commitment_root_hex.is_some()) + + usize::from(self.settlement_mode_decision.is_some()) + + usize::from(self.ctv_fanout_fee_policy.is_some()) + + usize::from(self.ctv_fanout_manifest_set.is_some()) + } + + fn serialize_suffix(&self, mut state: S) -> Result { + state.serialize_field("found_block", self.found_block)?; + state.serialize_field("prior_balances", self.prior_balances)?; + state.serialize_field("payout_policy", self.payout_policy)?; + if self.coinbase_script_sig_suffix_hex.is_some() { + state.serialize_field( + "coinbase_script_sig_suffix_hex", + self.coinbase_script_sig_suffix_hex, + )?; + } else { + state.skip_field("coinbase_script_sig_suffix_hex")?; + } + if !self.witness_merkle_leaves_hex.is_empty() { + state.serialize_field("witness_merkle_leaves_hex", self.witness_merkle_leaves_hex)?; + } else { + state.skip_field("witness_merkle_leaves_hex")?; + } + if !self.audit_commitment_leaves_hex.is_empty() { + state.serialize_field( + "audit_commitment_leaves_hex", + self.audit_commitment_leaves_hex, + )?; + } else { + state.skip_field("audit_commitment_leaves_hex")?; + } + if self.audit_commitment_root_hex.is_some() { + state.serialize_field("audit_commitment_root_hex", self.audit_commitment_root_hex)?; + } else { + state.skip_field("audit_commitment_root_hex")?; + } + state.serialize_field("ledger_window_attestation", self.ledger_window_attestation)?; + state.serialize_field("reward_manifest", self.reward_manifest)?; + state.serialize_field("payout_policy_manifest", self.payout_policy_manifest)?; + if self.settlement_mode_decision.is_some() { + state.serialize_field("settlement_mode_decision", self.settlement_mode_decision)?; + } else { + state.skip_field("settlement_mode_decision")?; + } + if self.ctv_fanout_fee_policy.is_some() { + state.serialize_field("ctv_fanout_fee_policy", self.ctv_fanout_fee_policy)?; + } else { + state.skip_field("ctv_fanout_fee_policy")?; + } + if self.ctv_fanout_manifest_set.is_some() { + state.serialize_field("ctv_fanout_manifest_set", self.ctv_fanout_manifest_set)?; + } else { + state.skip_field("ctv_fanout_manifest_set")?; + } + state.serialize_field("signed_coinbase_manifest", self.signed_coinbase_manifest)?; + state.end() + } +} + +struct Suffix<'a>(AuditBundleRef<'a>); + +impl Serialize for Suffix<'_> { + fn serialize(&self, serializer: S) -> Result { + self.0 + .serialize_suffix(serializer.serialize_struct("AuditBundle", self.0.field_count())?) + } +} + +pub(super) struct DigestWriter(pub Sha256); + +impl Write for DigestWriter { + fn write(&mut self, bytes: &[u8]) -> std::io::Result { + self.0.update(bytes); + Ok(bytes.len()) + } + + fn flush(&mut self) -> std::io::Result<()> { + Ok(()) + } +} + +/// Replace only the serializer's opening object brace; all fields and escaping +/// still go through the same serializer as the complete canonical bundle. +struct ContinueObject { + writer: W, + started: bool, +} + +impl Write for ContinueObject { + fn write(&mut self, bytes: &[u8]) -> std::io::Result { + if !self.started && !bytes.is_empty() { + if bytes[0] != b'{' { + return Err(std::io::Error::other("audit suffix must be an object")); + } + self.writer.write_all(b",")?; + self.started = true; + self.writer.write_all(&bytes[1..])?; + Ok(bytes.len()) + } else { + self.writer.write(bytes) + } + } + + fn flush(&mut self) -> std::io::Result<()> { + self.writer.flush() + } +} + +fn write_prefix( + mut writer: impl Write, + schema: &str, + shares: &[AcceptedShare], +) -> Result<(), serde_json::Error> { + write_prefix_header(&mut writer, schema)?; + serde_json::to_writer(writer, shares) +} + +/// The bytes before the share array: `{"schema":...,"shares":`. +fn write_prefix_header(mut writer: impl Write, schema: &str) -> Result<(), serde_json::Error> { + writer + .write_all(b"{\"schema\":") + .map_err(serde_json::Error::io)?; + serde_json::to_writer(&mut writer, schema)?; + writer + .write_all(b",\"shares\":") + .map_err(serde_json::Error::io) +} + +fn write_suffix(writer: impl Write, body: &AuditBundleBody) -> Result<(), serde_json::Error> { + serde_json::to_writer( + ContinueObject { + writer, + started: false, + }, + &Suffix(AuditBundleRef::from_parts(body, &[])), + ) +} + +/// [`write_suffix`] for the builder: the same `serialize_suffix` field list, +/// with the `reward_manifest` value's share array produced in ordered chunks. +fn write_suffix_with( + writer: &mut W, + body: &AuditBundleBody, + parallelism: Parallelism, +) -> Result<(), serde_json::Error> { + if parallelism.is_serial() { + return write_suffix(writer, body); + } + Suffix(AuditBundleRef::from_parts(body, &[])).serialize(parallel::StructSplice::continuing( + writer, + "reward_manifest", + |writer| write_reward_manifest(writer, &body.reward_manifest, parallelism), + )) +} + +/// Every byte reaches both digests: the prefix state after its header, and +/// the bare share-array digest. +struct Tee<'a> { + prefix: &'a mut Sha256, + array: &'a mut Sha256, +} + +impl Write for Tee<'_> { + fn write(&mut self, bytes: &[u8]) -> std::io::Result { + self.prefix.update(bytes); + self.array.update(bytes); + Ok(bytes.len()) + } + + fn flush(&mut self) -> std::io::Result<()> { + Ok(()) + } +} + +/// A single build's canonical SHA256 state after `schema` and accepted `shares`. +/// It owns no share array and is deliberately consumed on completion. This is +/// not a native window digest or a proof authorizing reuse across refreshes. +pub struct CanonicalAuditHashPrefix { + writer: DigestWriter, + schema: &'static str, +} + +thread_local! { + /// Share-array digests computed on this thread by + /// [`CanonicalAuditHashPrefix::new_with_share_digest`]: a probe for the + /// builder's tests, which pin that a rebuild computes none. + static SHARE_DIGESTS: std::cell::Cell = const { std::cell::Cell::new(0) }; +} + +impl CanonicalAuditHashPrefix { + /// How many share-array digests + /// [`CanonicalAuditHashPrefix::new_with_share_digest`] has computed on the + /// calling thread. A test probe: a rebuild that takes + /// [`CanonicalAuditHashPrefix::new_with`] leaves it unchanged. + #[doc(hidden)] + pub fn share_digests_computed_on_this_thread() -> usize { + SHARE_DIGESTS.with(std::cell::Cell::get) + } + + /// Hash the original ordered window while its audit body is being built. + pub fn new(shares: &[AcceptedShare]) -> Result { + let schema = audit_bundle_schema_for_shares(shares); + let mut writer = DigestWriter(Sha256::new()); + write_prefix(&mut writer, schema, shares)?; + Ok(Self { writer, schema }) + } + + /// [`CanonicalAuditHashPrefix::new`] for the builder: the share array is + /// serialized in ordered chunks on `parallelism`, and nothing but the + /// prefix state is computed. The rebuild of a window whose `WindowRef` + /// already carries the share-array digest takes this; a first build of + /// a window takes [`CanonicalAuditHashPrefix::new_with_share_digest`]. + /// The state is that of the serial computation. + pub fn new_with( + shares: &[AcceptedShare], + parallelism: Parallelism, + ) -> Result { + let schema = audit_bundle_schema_for_shares(shares); + let mut writer = DigestWriter(Sha256::new()); + write_prefix_header(&mut writer, schema)?; + parallel::write_array(&mut writer, shares, parallelism)?; + Ok(Self { writer, schema }) + } + + /// [`CanonicalAuditHashPrefix::new`] for the builder, which also returns + /// `sha256(serde_json::to_vec(shares))`, the bare share-array digest a + /// ledger `WindowRef` carries, from the same single serialization of the + /// window: every array byte updates both states. Both results are those + /// of the two separate serial computations. + pub fn new_with_share_digest( + shares: &[AcceptedShare], + parallelism: Parallelism, + ) -> Result<(Self, [u8; 32]), serde_json::Error> { + SHARE_DIGESTS.with(|count| count.set(count.get() + 1)); + let schema = audit_bundle_schema_for_shares(shares); + let mut writer = DigestWriter(Sha256::new()); + write_prefix_header(&mut writer, schema)?; + let array = if parallelism.is_serial() { + let mut array = Sha256::new(); + parallel::write_array( + &mut Tee { + prefix: &mut writer.0, + array: &mut array, + }, + shares, + parallelism, + )?; + array + } else { + // Each ordered frame updates the prefix here, then moves to the + // bare digest's thread, so the two SHA states run concurrently + // over the same buffers; the pool hands them back to the workers. + let pool = parallel::BufferPool::new(); + std::thread::scope(|scope| { + let (send, hasher) = parallel::spawn_hasher(scope, &pool, 4); + let written = parallel::write_array_frames(shares, parallelism, &pool, |frame| { + writer.0.update(&frame); + send.send(frame) + .map_err(|_| parallel::stopped("share-array digest")) + }); + drop(send); + let array = hasher.join().expect("share-array digest thread"); + written.map(|()| array) + })? + }; + Ok((Self { writer, schema }, array.finalize().into())) + } + + /// Finish with the body built from the exact same immutable share window. + /// As with `AuditBundleBody::into_bundle`, the caller owns that association; + /// this performs no membership verification and grants no publication right. + pub fn finish(self, body: &AuditBundleBody) -> Result { + self.finish_with(body, Parallelism::serial()) + } + + /// [`CanonicalAuditHashPrefix::finish`] for the builder: the suffix's + /// counted-share array is serialized in ordered chunks on `parallelism` + /// worker threads; the digest is the serial one. + pub fn finish_with( + mut self, + body: &AuditBundleBody, + parallelism: Parallelism, + ) -> Result { + if body.schema != self.schema { + return Err(PrismError::AuditMismatch { artifact: "schema" }); + } + write_suffix_with(&mut self.writer, body, parallelism)?; + Ok(hex::encode(self.writer.0.finalize())) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + fn fixture() -> (AuditBundleBody, Vec) { + let input: serde_json::Value = serde_json::from_str(include_str!( + "../fixtures/bootstrap-small-log.prism-fixture.json" + )) + .unwrap(); + let shares: Vec = serde_json::from_value(input["shares"].clone()).unwrap(); + let key = ManifestSigningKey::from_seed_hex(&"42".repeat(32)).unwrap(); + let ledger_key = ManifestSigningKey::from_seed_hex(&"43".repeat(32)).unwrap(); + let body = build_audit_bundle_body( + &shares, + serde_json::from_value(input["found_block"].clone()).unwrap(), + vec![], + PayoutPolicy::day_one_default(), + &key, + &ledger_key, + ) + .unwrap(); + (body, shares) + } + + #[test] + fn prefix_only_constructor_matches_the_serial_prefix_and_computes_no_share_digest() { + let (body, shares) = fixture(); + let serial = CanonicalAuditHashPrefix::new(&shares) + .unwrap() + .finish(&body) + .unwrap(); + for parallelism in [ + Parallelism::serial(), + Parallelism::new(2, 2), + Parallelism::new(3, 5), + Parallelism::new(8, 1), + ] { + let digests = CanonicalAuditHashPrefix::share_digests_computed_on_this_thread(); + let prefix = CanonicalAuditHashPrefix::new_with(&shares, parallelism).unwrap(); + assert_eq!( + CanonicalAuditHashPrefix::share_digests_computed_on_this_thread(), + digests, + "new_with computed a share digest at {parallelism:?}" + ); + assert_eq!(prefix.finish_with(&body, parallelism).unwrap(), serial); + let (with_digest, array) = + CanonicalAuditHashPrefix::new_with_share_digest(&shares, parallelism).unwrap(); + assert_eq!( + CanonicalAuditHashPrefix::share_digests_computed_on_this_thread(), + digests + 1 + ); + assert_eq!(with_digest.finish_with(&body, parallelism).unwrap(), serial); + assert_eq!( + array, + <[u8; 32]>::from(Sha256::digest(serde_json::to_vec(&shares).unwrap())) + ); + } + } + + #[test] + fn split_framing_emits_exact_bytes_and_rejects_a_different_schema() { + let (mut body, shares) = fixture(); + for shares in [&shares[..], &[]] { + let mut bytes = Vec::new(); + write_prefix(&mut bytes, &body.schema, shares).unwrap(); + write_suffix(&mut bytes, &body).unwrap(); + assert_eq!( + bytes, + canonical_audit_bundle_bytes_from_parts(&body, shares).unwrap() + ); + } + let prefix = CanonicalAuditHashPrefix::new(&shares).unwrap(); + body.schema.push_str("-different"); + assert!(matches!( + prefix.finish(&body), + Err(PrismError::AuditMismatch { artifact: "schema" }) + )); + } + + #[test] + fn prefix_and_suffix_propagate_writer_failures() { + struct Failed; + impl Write for Failed { + fn write(&mut self, _: &[u8]) -> std::io::Result { + Err(std::io::Error::other("injected writer failure")) + } + fn flush(&mut self) -> std::io::Result<()> { + Ok(()) + } + } + let (body, shares) = fixture(); + assert!(write_prefix(Failed, &body.schema, &shares) + .unwrap_err() + .is_io()); + assert!(write_suffix(Failed, &body).unwrap_err().is_io()); + } +} diff --git a/crates/qbit-prism/src/bin/qbit-prism-build-audit-bundle.rs b/crates/qbit-prism/src/bin/qbit-prism-build-audit-bundle.rs index 078db22b0..fc09d733d 100644 --- a/crates/qbit-prism/src/bin/qbit-prism-build-audit-bundle.rs +++ b/crates/qbit-prism/src/bin/qbit-prism-build-audit-bundle.rs @@ -576,7 +576,11 @@ fn serve_requests( let bundle = match bundle_result { Ok(bundle) => bundle, Err(error) => { - respond_error(&stdout, &format!("audit bundle build failed: {error}"), false)?; + respond_error( + &stdout, + &format!("audit bundle build failed: {error}"), + false, + )?; continue; } }; @@ -682,7 +686,11 @@ fn serve_prepare_window( match request.mode.as_deref() { Some("full") => { let Some(window_weight) = request.window_weight else { - respond_error(stdout, "prepare_window full carries no window_weight", false)?; + respond_error( + stdout, + "prepare_window full carries no window_weight", + false, + )?; return Ok(()); }; let page_size = request.page_size.unwrap_or(DEFAULT_WINDOW_PAGE_SIZE); @@ -773,7 +781,11 @@ fn serve_prepare_window( } Some("advance") => { let Some(base_digest) = request.base_digest else { - respond_error(stdout, "prepare_window advance carries no base_digest", false)?; + respond_error( + stdout, + "prepare_window advance carries no base_digest", + false, + )?; return Ok(()); }; let position = window_cache.iter().position(|(key, state)| { diff --git a/crates/qbit-prism/src/ctv.rs b/crates/qbit-prism/src/ctv.rs index 67bb2c842..59c946daa 100644 --- a/crates/qbit-prism/src/ctv.rs +++ b/crates/qbit-prism/src/ctv.rs @@ -2491,7 +2491,7 @@ mod tests { verify_ctv_fanout_manifest_structure(&manifest).unwrap(); verify_ctv_fanout_manifest_commitment_leaf( &manifest, - &[manifest.commitment_witness_leaf_hex.clone()], + std::slice::from_ref(&manifest.commitment_witness_leaf_hex), ) .unwrap(); } diff --git a/crates/qbit-prism/src/lib.rs b/crates/qbit-prism/src/lib.rs index 595c9674c..4782f2ad4 100644 --- a/crates/qbit-prism/src/lib.rs +++ b/crates/qbit-prism/src/lib.rs @@ -77,6 +77,13 @@ pub use ctv::*; mod broadcast; pub use broadcast::*; +mod audit_hash; +pub use audit_hash::CanonicalAuditHashPrefix; + +/// Builder-only ordered parallel serialization; verifiers stay serial. +pub mod parallel; +pub use parallel::Parallelism; + mod audit_body_ref; pub use audit_body_ref::*; @@ -95,6 +102,25 @@ pub const QBIT_COINBASE_MATURITY_BLOCKS: u64 = 1_000; pub const PRISM_AUDIT_COMMITMENT_LEAF_TAG: &str = "qbit.prism.audit.commitment.v1"; pub const AUDIT_BUNDLE_SCHEMA_V1: &str = "qbit.prism.audit-bundle.v1"; pub const AUDIT_BUNDLE_SCHEMA_V1_1: &str = "qbit.prism.audit-bundle.v1.1"; +/// The version of the audit-bundle builders themselves, as distinct from the +/// `schema` strings above, which the share shape chooses rather than the +/// builder logic. +/// +/// A stored window reference reproduces its bundle only under the builder that +/// first built it, so a candidate and a prepared job store this value beside +/// their other inputs and a rebuild refuses a reference whose stored version +/// differs from this binary's, rather than rebuilding it with the current +/// builder. +/// +/// **Bump rule: any change that alters `canonical_audit_bundle_bytes` for the +/// same inputs bumps this constant, in the same commit.** A new reward rule, a +/// new manifest field or a canonical-schema change all qualify. The golden +/// digests in `tests/audit_cli.rs` are keyed by this constant, so a new output +/// is recorded as a new entry beside the old one, never by overwriting one; +/// review then sees both. A release that bumps it ships a migration step whose +/// refusal predicate counts `state='pending'` outbox rows at the old value, and +/// the operator drains them with the old frontends first. +pub const AUDIT_BUILDER_VERSION: u16 = 1; #[derive(Debug, thiserror::Error)] pub enum PrismError { @@ -258,20 +284,15 @@ pub struct PayoutPolicy { /// `PoolFeeFirst` reserves one direct settlement slot for a positive pool fee, /// keeps the fee out of CTV fanout chunks, and emits the fee output at /// coinbase vout 0 while every other output keeps canonical ordering after it. -#[derive(Clone, Copy, Debug, Deserialize, Serialize, PartialEq, Eq)] +#[derive(Clone, Copy, Debug, Default, Deserialize, Serialize, PartialEq, Eq)] pub enum CoinbaseOutputPolicy { + #[default] #[serde(rename = "canonical")] Canonical, #[serde(rename = "pool-fee-first")] PoolFeeFirst, } -impl Default for CoinbaseOutputPolicy { - fn default() -> Self { - Self::Canonical - } -} - #[derive(Clone, Debug, Deserialize, Serialize, PartialEq, Eq)] pub struct PoolFeePolicy { pub fee_bps: u16, @@ -337,19 +358,14 @@ pub struct PoolFeeManifest { pub p2mr_program_hex: String, } -#[derive(Clone, Debug, Deserialize, Serialize, PartialEq, Eq)] +#[derive(Clone, Debug, Default, Deserialize, Serialize, PartialEq, Eq)] #[serde(rename_all = "snake_case")] pub enum PayoutPolicyAccountType { + #[default] Miner, PoolFee, } -impl Default for PayoutPolicyAccountType { - fn default() -> Self { - Self::Miner - } -} - #[derive(Clone, Debug, Deserialize, Serialize, PartialEq, Eq)] #[serde(rename_all = "snake_case")] pub enum PayoutPolicyAction { @@ -385,13 +401,120 @@ pub enum PayoutMaturityState { Reversed, } -#[derive(Clone, Debug, Deserialize, Serialize, PartialEq, Eq)] +/// A complete audit bundle: the payout window (`shares`) and everything +/// derived from it. +/// +/// `Serialize` is hand-written and delegates to the private `AuditBundleRef` +/// view, the single definition of the canonical audit bytes that the borrowed +/// parts API ([`AuditBundleBody`] plus `&[AcceptedShare]`) shares. That view alone +/// decides field order and which absent optional fields are omitted, so the +/// fields below carry no serialization attributes; `#[serde(default)]` is for +/// the derived `Deserialize`, which accepts a bundle that omits them. +#[derive(Clone, Debug, Deserialize, PartialEq, Eq)] pub struct AuditBundle { pub schema: String, pub shares: Vec, pub found_block: FoundBlock, pub prior_balances: Vec, pub payout_policy: PayoutPolicy, + #[serde(default)] + pub coinbase_script_sig_suffix_hex: Option, + #[serde(default)] + pub witness_merkle_leaves_hex: Vec, + #[serde(default)] + pub audit_commitment_leaves_hex: Vec, + #[serde(default)] + pub audit_commitment_root_hex: Option, + pub ledger_window_attestation: LedgerWindowAttestation, + pub reward_manifest: PrismRewardManifest, + pub payout_policy_manifest: PayoutPolicyManifest, + #[serde(default)] + pub settlement_mode_decision: Option, + #[serde(default)] + pub ctv_fanout_fee_policy: Option, + #[serde(default)] + pub ctv_fanout_manifest_set: Option, + pub signed_coinbase_manifest: SignedPayoutManifest, +} + +impl Serialize for AuditBundle { + fn serialize(&self, serializer: S) -> Result { + AuditBundleRef::from_bundle(self).serialize(serializer) + } +} + +impl AuditBundle { + /// Split the bundle into its body and its payout window. Both halves are + /// moved out; no share is copied. + pub fn into_parts(self) -> (AuditBundleBody, Vec) { + let AuditBundle { + schema, + shares, + found_block, + prior_balances, + payout_policy, + coinbase_script_sig_suffix_hex, + witness_merkle_leaves_hex, + audit_commitment_leaves_hex, + audit_commitment_root_hex, + ledger_window_attestation, + reward_manifest, + payout_policy_manifest, + settlement_mode_decision, + ctv_fanout_fee_policy, + ctv_fanout_manifest_set, + signed_coinbase_manifest, + } = self; + let body = AuditBundleBody { + schema, + found_block, + prior_balances, + payout_policy, + coinbase_script_sig_suffix_hex, + witness_merkle_leaves_hex, + audit_commitment_leaves_hex, + audit_commitment_root_hex, + ledger_window_attestation, + reward_manifest, + payout_policy_manifest, + settlement_mode_decision, + ctv_fanout_fee_policy, + ctv_fanout_manifest_set, + signed_coinbase_manifest, + }; + (body, shares) + } +} + +/// Every [`AuditBundle`] field except the top-level `shares` payout window. +/// It is the in-memory body of a native bundle, not the Python-era +/// `AuditBodyRef` envelope. +/// +/// The parts API ([`build_audit_bundle_body`] and its siblings, +/// [`verify_audit_parts`], [`canonical_audit_bundle_bytes_from_parts`]) lets +/// a caller that owns the window keep it and lend it as `&[AcceptedShare]` +/// instead of handing a `Vec`, usually a clone, to the builder. +/// [`AuditBundleBody::into_bundle`] and [`AuditBundle::into_parts`] convert +/// between the two forms by moving the window, never copying it. +/// +/// `reward_manifest` still carries its own `Vec`, the counted +/// records derived from the window. A stored native audit row drops that copy +/// too and keeps only a [`PrismRewardManifestHeader`]; see +/// [`restore_reward_manifest`]. +/// +/// The body's own serde form is not an audit artifact. It omits `shares`, so +/// it has no canonical bytes and no published sha256. Produce those with +/// [`canonical_audit_bundle_bytes_from_parts`]. Decoding rejects unknown +/// fields, so a full bundle (which has `shares`) cannot be read as a body and +/// silently lose its window; decode it as [`AuditBundle`] and call +/// [`AuditBundle::into_parts`]. +#[derive(Clone, Debug, Deserialize, Serialize, PartialEq, Eq)] +#[serde(deny_unknown_fields)] +pub struct AuditBundleBody { + pub schema: String, + pub found_block: FoundBlock, + pub prior_balances: Vec, + pub payout_policy: PayoutPolicy, #[serde(default, skip_serializing_if = "Option::is_none")] pub coinbase_script_sig_suffix_hex: Option, #[serde(default, skip_serializing_if = "Vec::is_empty")] @@ -412,6 +535,167 @@ pub struct AuditBundle { pub signed_coinbase_manifest: SignedPayoutManifest, } +impl AuditBundleBody { + /// Assemble a bundle that owns `shares`. The window is moved in, not + /// copied. Pass the window the body was built from; this does not check + /// it, and [`verify_audit_bundle`] rejects a mismatched pair. + pub fn into_bundle(self, shares: Vec) -> AuditBundle { + let AuditBundleBody { + schema, + found_block, + prior_balances, + payout_policy, + coinbase_script_sig_suffix_hex, + witness_merkle_leaves_hex, + audit_commitment_leaves_hex, + audit_commitment_root_hex, + ledger_window_attestation, + reward_manifest, + payout_policy_manifest, + settlement_mode_decision, + ctv_fanout_fee_policy, + ctv_fanout_manifest_set, + signed_coinbase_manifest, + } = self; + AuditBundle { + schema, + shares, + found_block, + prior_balances, + payout_policy, + coinbase_script_sig_suffix_hex, + witness_merkle_leaves_hex, + audit_commitment_leaves_hex, + audit_commitment_root_hex, + ledger_window_attestation, + reward_manifest, + payout_policy_manifest, + settlement_mode_decision, + ctv_fanout_fee_policy, + ctv_fanout_manifest_set, + signed_coinbase_manifest, + } + } +} + +/// Borrowed view of an audit bundle and the single definition of its +/// canonical JSON. Field order and `skip_serializing_if` rules match +/// [`AuditBundle`]'s declaration exactly. The owned bundle and +/// `(&AuditBundleBody, &[AcceptedShare])` both serialize through this view, so the +/// two forms produce identical bytes. +/// +/// Never `#[serde(flatten)]` a body into this struct: flatten would move +/// `shares` out of second position and buffer through serde's `Content`, +/// which cannot carry `arbitrary_precision` `u128` values. +#[derive(Clone, Copy)] +struct AuditBundleRef<'a> { + schema: &'a String, + shares: &'a [AcceptedShare], + found_block: &'a FoundBlock, + prior_balances: &'a Vec, + payout_policy: &'a PayoutPolicy, + coinbase_script_sig_suffix_hex: &'a Option, + witness_merkle_leaves_hex: &'a Vec, + audit_commitment_leaves_hex: &'a Vec, + audit_commitment_root_hex: &'a Option, + ledger_window_attestation: &'a LedgerWindowAttestation, + reward_manifest: &'a PrismRewardManifest, + payout_policy_manifest: &'a PayoutPolicyManifest, + settlement_mode_decision: &'a Option, + ctv_fanout_fee_policy: &'a Option, + ctv_fanout_manifest_set: &'a Option, + signed_coinbase_manifest: &'a SignedPayoutManifest, +} + +impl<'a> AuditBundleRef<'a> { + fn from_bundle(bundle: &'a AuditBundle) -> Self { + let AuditBundle { + schema, + shares, + found_block, + prior_balances, + payout_policy, + coinbase_script_sig_suffix_hex, + witness_merkle_leaves_hex, + audit_commitment_leaves_hex, + audit_commitment_root_hex, + ledger_window_attestation, + reward_manifest, + payout_policy_manifest, + settlement_mode_decision, + ctv_fanout_fee_policy, + ctv_fanout_manifest_set, + signed_coinbase_manifest, + } = bundle; + Self { + schema, + shares, + found_block, + prior_balances, + payout_policy, + coinbase_script_sig_suffix_hex, + witness_merkle_leaves_hex, + audit_commitment_leaves_hex, + audit_commitment_root_hex, + ledger_window_attestation, + reward_manifest, + payout_policy_manifest, + settlement_mode_decision, + ctv_fanout_fee_policy, + ctv_fanout_manifest_set, + signed_coinbase_manifest, + } + } + + fn from_parts(body: &'a AuditBundleBody, shares: &'a [AcceptedShare]) -> Self { + let AuditBundleBody { + schema, + found_block, + prior_balances, + payout_policy, + coinbase_script_sig_suffix_hex, + witness_merkle_leaves_hex, + audit_commitment_leaves_hex, + audit_commitment_root_hex, + ledger_window_attestation, + reward_manifest, + payout_policy_manifest, + settlement_mode_decision, + ctv_fanout_fee_policy, + ctv_fanout_manifest_set, + signed_coinbase_manifest, + } = body; + Self { + schema, + shares, + found_block, + prior_balances, + payout_policy, + coinbase_script_sig_suffix_hex, + witness_merkle_leaves_hex, + audit_commitment_leaves_hex, + audit_commitment_root_hex, + ledger_window_attestation, + reward_manifest, + payout_policy_manifest, + settlement_mode_decision, + ctv_fanout_fee_policy, + ctv_fanout_manifest_set, + signed_coinbase_manifest, + } + } + + fn write_canonical(&self, writer: W) -> Result<(), serde_json::Error> { + serde_json::to_writer(writer, self) + } + + fn canonical_bytes(&self) -> Result, serde_json::Error> { + let mut bytes = Vec::new(); + self.write_canonical(&mut bytes)?; + Ok(bytes) + } +} + #[derive(Clone, Debug, Deserialize, Serialize, PartialEq, Eq)] pub struct LedgerWindowAttestation { pub schema: String, @@ -516,6 +800,62 @@ pub fn compute_prism_window( shares: &[AcceptedShare], found_block: &FoundBlock, ) -> Result { + let (eligible, requested_window_weight) = eligible_window(shares, found_block)?; + + let mut remaining = requested_window_weight; + let mut counted_shares = Vec::new(); + for share in eligible { + if remaining == 0 { + break; + } + let counted = share.share_difficulty.min(remaining); + counted_shares.push(CountedShare { + share_seq: share.share_seq, + share_id: share.share_id.clone(), + miner_id: share.miner_id.clone(), + order_key: share.order_key.clone(), + p2mr_program_hex: share.p2mr_program_hex.clone(), + share_difficulty: share.share_difficulty, + counted_difficulty: counted, + job_issued_at_ms: share.job_issued_at_ms, + accepted_at_ms: share.accepted_at_ms, + credit_policy: share.credit_policy.clone(), + }); + remaining -= counted; + } + + if counted_shares.is_empty() { + return Err(PrismError::EmptyWindow { + anchor_job_issued_at_ms: found_block.anchor_job_issued_at_ms, + }); + } + let anchor_share_seq = counted_shares[0].share_seq; + + let counted_window_weight = counted_shares + .iter() + .try_fold(0_u128, |sum, share| { + sum.checked_add(share.counted_difficulty) + }) + .ok_or(PrismError::WindowOverflow)?; + let entitlements = aggregate_entitlements(&counted_shares)?; + + Ok(PrismWindow { + anchor_job_issued_at_ms: found_block.anchor_job_issued_at_ms, + anchor_share_seq, + requested_window_weight, + counted_window_weight, + shares: counted_shares, + entitlements, + }) +} + +/// Validation, eligibility and newest-first ordering, shared by the serial +/// fold and the builder's parallel fold so both take the same shares in the +/// same order. +fn eligible_window<'a>( + shares: &'a [AcceptedShare], + found_block: &FoundBlock, +) -> Result<(Vec<&'a AcceptedShare>, u128), PrismError> { if found_block.network_difficulty == 0 { return Err(PrismError::ZeroNetworkDifficulty); } @@ -555,56 +895,195 @@ pub fn compute_prism_window( .windows(2) .all(|pair| pair[0].share_seq > pair[1].share_seq) { - eligible.sort_by(|left, right| right.share_seq.cmp(&left.share_seq)); + eligible.sort_by_key(|share| std::cmp::Reverse(share.share_seq)); } + Ok((eligible, requested_window_weight)) +} +/// The builder's fold: the `PrismWindow` of [`compute_prism_window`] and the +/// [`share_slice_digest_hex`] of its shares, with the counted shares +/// constructed on worker threads in chunk order while this thread streams +/// the slice digest over each finished chunk and merges its entitlement +/// weights. The counted difficulties are decided serially first, so every +/// chunk is a pure function of the ordered eligible shares. +fn compute_prism_window_parallel( + shares: &[AcceptedShare], + found_block: &FoundBlock, + parallelism: Parallelism, +) -> Result<(PrismWindow, String), PrismError> { + let (eligible, requested_window_weight) = eligible_window(shares, found_block)?; let mut remaining = requested_window_weight; - let mut counted_shares = Vec::new(); - for share in eligible { + let mut counted = Vec::with_capacity(eligible.len()); + for share in &eligible { if remaining == 0 { break; } - let counted = share.share_difficulty.min(remaining); - counted_shares.push(CountedShare { - share_seq: share.share_seq, - share_id: share.share_id.clone(), - miner_id: share.miner_id.clone(), - order_key: share.order_key.clone(), - p2mr_program_hex: share.p2mr_program_hex.clone(), - share_difficulty: share.share_difficulty, - counted_difficulty: counted, - job_issued_at_ms: share.job_issued_at_ms, - accepted_at_ms: share.accepted_at_ms, - credit_policy: share.credit_policy.clone(), - }); - remaining -= counted; + let taken = share.share_difficulty.min(remaining); + counted.push(taken); + remaining -= taken; } - - if counted_shares.is_empty() { + if counted.is_empty() { return Err(PrismError::EmptyWindow { anchor_job_issued_at_ms: found_block.anchor_job_issued_at_ms, }); } - let anchor_share_seq = counted_shares[0].share_seq; - - let counted_window_weight = counted_shares + let counted_window_weight = counted .iter() - .try_fold(0_u128, |sum, share| { - sum.checked_add(share.counted_difficulty) - }) + .try_fold(0_u128, |sum, taken| sum.checked_add(*taken)) .ok_or(PrismError::WindowOverflow)?; - let entitlements = aggregate_entitlements(&counted_shares)?; + let taken = &eligible[..counted.len()]; + let anchor_share_seq = taken[0].share_seq; + let chunk_len = parallelism.chunk_len().min(taken.len()); + let chunks = taken.len().div_ceil(chunk_len); + let mut counted_shares = Vec::with_capacity(taken.len()); + let mut weights: BTreeMap<(&str, &str, &str), u128> = BTreeMap::new(); + // Workers build the counted shares, their slice-digest bytes and their + // entitlement partial sums per chunk; this thread appends and merges in + // order and hands each chunk's digest bytes to the digest thread. + let pool = parallel::BufferPool::new(); + let digest = std::thread::scope(|scope| { + let (send, hasher) = parallel::spawn_hasher(scope, &pool, 4); + let folded = parallel::ordered_chunks( + chunks, + parallelism.workers(), + |index| { + let range = index * chunk_len..((index + 1) * chunk_len).min(taken.len()); + let mut chunk = Vec::with_capacity(range.len()); + let mut digest_bytes = pool.take(); + let mut chunk_weights: BTreeMap<(&str, &str, &str), u128> = BTreeMap::new(); + for (share, credited) in taken[range.clone()].iter().zip(&counted[range]) { + chunk.push(CountedShare { + share_seq: share.share_seq, + share_id: share.share_id.clone(), + miner_id: share.miner_id.clone(), + order_key: share.order_key.clone(), + p2mr_program_hex: share.p2mr_program_hex.clone(), + share_difficulty: share.share_difficulty, + counted_difficulty: *credited, + job_issued_at_ms: share.job_issued_at_ms, + accepted_at_ms: share.accepted_at_ms, + credit_policy: share.credit_policy.clone(), + }); + update_slice_digest(&mut digest_bytes, chunk.last().expect("just pushed")); + let weight = chunk_weights + .entry(( + share.miner_id.as_str(), + share.order_key.as_str(), + share.p2mr_program_hex.as_str(), + )) + .or_insert(0); + *weight = weight + .checked_add(*credited) + .ok_or(PrismError::WindowOverflow)?; + } + Ok((chunk, digest_bytes, chunk_weights)) + }, + |_, (chunk, digest_bytes, chunk_weights)| { + send.send(digest_bytes) + .map_err(|_| PrismError::Json(parallel::stopped("share slice digest")))?; + counted_shares.extend(chunk); + for (key, weight) in chunk_weights { + let total = weights.entry(key).or_insert(0); + *total = total + .checked_add(weight) + .ok_or(PrismError::WindowOverflow)?; + } + Ok::<(), PrismError>(()) + }, + ); + drop(send); + let hasher = hasher.join().expect("share slice digest thread"); + folded.map(|()| hasher) + })?; + let entitlements = weights + .into_iter() + .map( + |((miner_id, order_key, p2mr_program_hex), weight)| WeightedEntitlement { + recipient_id: miner_id.to_owned(), + order_key: order_key.to_owned(), + p2mr_program_hex: p2mr_program_hex.to_owned(), + weight, + }, + ) + .collect(); + Ok(( + PrismWindow { + anchor_job_issued_at_ms: found_block.anchor_job_issued_at_ms, + anchor_share_seq, + requested_window_weight, + counted_window_weight, + shares: counted_shares, + entitlements, + }, + hex::encode(digest.finalize()), + )) +} - Ok(PrismWindow { - anchor_job_issued_at_ms: found_block.anchor_job_issued_at_ms, - anchor_share_seq, - requested_window_weight, - counted_window_weight, - shares: counted_shares, - entitlements, +/// [`build_prism_reward_manifest`] for the builder: serial parallelism is +/// that function; anything wider folds through [`compute_prism_window_parallel`] +/// and produces the identical manifest. +fn build_prism_reward_manifest_parallel( + shares: &[AcceptedShare], + found_block: &FoundBlock, + parallelism: Parallelism, +) -> Result { + if parallelism.is_serial() { + return build_prism_reward_manifest(shares, found_block); + } + let (window, share_slice_digest_hex) = + compute_prism_window_parallel(shares, found_block, parallelism)?; + let newest_share_seq = window.shares[0].share_seq; + let oldest_share_seq = window + .shares + .last() + .expect("non-empty PRISM windows have a last share") + .share_seq; + Ok(PrismRewardManifest { + schema: "qbit.prism.reward-manifest.v1".to_string(), + block_height: found_block.block_height, + coinbase_value_sats: found_block.coinbase_value_sats, + network_difficulty: found_block.network_difficulty, + window_multiplier: PRISM_WINDOW_MULTIPLIER, + requested_window_weight: window.requested_window_weight, + counted_window_weight: window.counted_window_weight, + anchor_job_issued_at_ms: window.anchor_job_issued_at_ms, + anchor_share_seq: window.anchor_share_seq, + newest_share_seq, + oldest_share_seq, + included_share_count: window.shares.len(), + share_slice_digest_hex, + shares: window.shares, + entitlements: window.entitlements, }) } +/// Write exactly `serde_json::to_writer(writer, manifest)`: the derived +/// field order, with the `shares` array's bytes produced in ordered chunks. +pub(crate) fn write_reward_manifest( + writer: &mut W, + manifest: &PrismRewardManifest, + parallelism: Parallelism, +) -> Result<(), serde_json::Error> { + if parallelism.is_serial() { + return serde_json::to_writer(writer, manifest); + } + manifest.serialize(parallel::StructSplice::object(writer, "shares", |writer| { + parallel::write_array(writer, &manifest.shares, parallelism) + })) +} + +/// `sha256(canonical_reward_manifest_bytes(manifest))` without materializing +/// the bytes: the builder's input to the commitment leaf and, with CTV +/// settlement, to every fan-out precommitment. +fn reward_manifest_sha256( + manifest: &PrismRewardManifest, + parallelism: Parallelism, +) -> Result<[u8; 32], serde_json::Error> { + let mut writer = audit_hash::DigestWriter(Sha256::new()); + write_reward_manifest(&mut writer, manifest, parallelism)?; + Ok(writer.0.finalize().into()) +} + pub fn build_prism_reward_manifest( shares: &[AcceptedShare], found_block: &FoundBlock, @@ -648,22 +1127,196 @@ pub fn canonical_payout_policy_manifest_bytes( } pub fn canonical_audit_bundle_bytes(bundle: &AuditBundle) -> Result, serde_json::Error> { - serde_json::to_vec(bundle) + AuditBundleRef::from_bundle(bundle).canonical_bytes() +} + +/// Canonical audit bundle bytes for `body` with `shares` as its window. The +/// bytes are identical to [`canonical_audit_bundle_bytes`] of +/// `body.into_bundle(shares)`, and so is their sha256, but no bundle is +/// assembled: the caller keeps ownership of the window and no copy of it is +/// made outside the returned bytes. +pub fn canonical_audit_bundle_bytes_from_parts( + body: &AuditBundleBody, + shares: &[AcceptedShare], +) -> Result, serde_json::Error> { + AuditBundleRef::from_parts(body, shares).canonical_bytes() +} + +/// Streaming form of [`canonical_audit_bundle_bytes_from_parts`]: writes the +/// same bytes to `writer` without buffering them. The caller keeps ownership +/// of the window and no copy of it is made. +pub fn write_canonical_audit_bundle_from_parts( + writer: W, + body: &AuditBundleBody, + shares: &[AcceptedShare], +) -> Result<(), serde_json::Error> { + AuditBundleRef::from_parts(body, shares).write_canonical(writer) +} + +/// Every [`PrismRewardManifest`] field except `shares`, in the same +/// declaration order and with the same serde attributes. This is what a +/// normalized native audit row stores under `reward_manifest`: the counted +/// window is a pure fold over `(shares, found_block)`, so the row keeps only +/// the header and [`restore_reward_manifest`] rebuilds the window on read. +/// +/// The serde form is not an audit artifact and has no canonical bytes. +/// Decoding rejects unknown fields, so a full manifest (which has `shares`) +/// cannot be read as a header and silently lose its window; a stored body +/// that still carries `reward_manifest.shares` is decoded as +/// [`PrismRewardManifest`] instead. +#[derive(Clone, Debug, Deserialize, Serialize, PartialEq, Eq)] +#[serde(deny_unknown_fields)] +pub struct PrismRewardManifestHeader { + pub schema: String, + pub block_height: u64, + pub coinbase_value_sats: u64, + pub network_difficulty: u128, + pub window_multiplier: u128, + pub requested_window_weight: u128, + pub counted_window_weight: u128, + pub anchor_job_issued_at_ms: i64, + pub anchor_share_seq: u64, + pub newest_share_seq: u64, + pub oldest_share_seq: u64, + pub included_share_count: usize, + pub share_slice_digest_hex: String, + pub entitlements: Vec, +} + +impl PrismRewardManifest { + /// Split the manifest into its header and its counted window. The window + /// is moved out, never copied. + pub fn into_parts(self) -> (PrismRewardManifestHeader, Vec) { + let PrismRewardManifest { + schema, + block_height, + coinbase_value_sats, + network_difficulty, + window_multiplier, + requested_window_weight, + counted_window_weight, + anchor_job_issued_at_ms, + anchor_share_seq, + newest_share_seq, + oldest_share_seq, + included_share_count, + share_slice_digest_hex, + shares, + entitlements, + } = self; + ( + PrismRewardManifestHeader { + schema, + block_height, + coinbase_value_sats, + network_difficulty, + window_multiplier, + requested_window_weight, + counted_window_weight, + anchor_job_issued_at_ms, + anchor_share_seq, + newest_share_seq, + oldest_share_seq, + included_share_count, + share_slice_digest_hex, + entitlements, + }, + shares, + ) + } +} + +impl PrismRewardManifestHeader { + /// Reassemble a manifest around `shares`. The window is moved in, not + /// copied, and not checked: pair a header only with the window + /// [`restore_reward_manifest`] rebuilt for it, or with the window + /// [`PrismRewardManifest::into_parts`] split off it. + pub fn into_manifest(self, shares: Vec) -> PrismRewardManifest { + let PrismRewardManifestHeader { + schema, + block_height, + coinbase_value_sats, + network_difficulty, + window_multiplier, + requested_window_weight, + counted_window_weight, + anchor_job_issued_at_ms, + anchor_share_seq, + newest_share_seq, + oldest_share_seq, + included_share_count, + share_slice_digest_hex, + entitlements, + } = self; + PrismRewardManifest { + schema, + block_height, + coinbase_value_sats, + network_difficulty, + window_multiplier, + requested_window_weight, + counted_window_weight, + anchor_job_issued_at_ms, + anchor_share_seq, + newest_share_seq, + oldest_share_seq, + included_share_count, + share_slice_digest_hex, + shares, + entitlements, + } + } +} + +/// Rebuild the counted-share window [`build_prism_reward_manifest`] folded +/// out of `shares`, and prove the rebuilt header is identical to `header`. +/// +/// The fold is deterministic, so a header that matches field for field, +/// `share_slice_digest_hex` and `counted_window_weight` included, can only +/// have come from this window. A header that differs is refused as +/// [`PrismError::AuditMismatch`] in `reward_manifest`, the same error +/// [`verify_audit_bundle`] raises for a bundle whose manifest does not match +/// its shares. There is no way to obtain the rebuilt window without this +/// check: the fold's own output is returned only once the header has matched. +pub fn restore_reward_manifest( + header: PrismRewardManifestHeader, + shares: &[AcceptedShare], + found_block: &FoundBlock, +) -> Result { + let (rebuilt, window) = build_prism_reward_manifest(shares, found_block)?.into_parts(); + if rebuilt != header { + return Err(PrismError::AuditMismatch { + artifact: "reward_manifest", + }); + } + Ok(rebuilt.into_manifest(window)) } pub fn prism_audit_commitment_leaf_hex( reward_manifest: &PrismRewardManifest, payout_policy_manifest: &PayoutPolicyManifest, ) -> Result { - let reward_manifest_hash = Sha256::digest(canonical_reward_manifest_bytes(reward_manifest)?); + let mut reward_writer = audit_hash::DigestWriter(Sha256::new()); + serde_json::to_writer(&mut reward_writer, reward_manifest)?; + let reward_manifest_hash = reward_writer.0.finalize(); let payout_policy_manifest_hash = Sha256::digest(canonical_payout_policy_manifest_bytes( payout_policy_manifest, )?); + Ok(commitment_leaf_hex( + &reward_manifest_hash, + &payout_policy_manifest_hash, + )) +} + +/// The leaf over both manifests' canonical sha256 digests: one recipe for +/// the serial verifier above and the builder, which supplies the reward +/// digest from [`reward_manifest_sha256`]. +fn commitment_leaf_hex(reward_manifest_hash: &[u8], payout_policy_manifest_hash: &[u8]) -> String { let mut payload = Vec::with_capacity(PRISM_AUDIT_COMMITMENT_LEAF_TAG.len() + 64); payload.extend_from_slice(PRISM_AUDIT_COMMITMENT_LEAF_TAG.as_bytes()); - payload.extend_from_slice(&reward_manifest_hash); - payload.extend_from_slice(&payout_policy_manifest_hash); - Ok(sha256_hex(&payload)) + payload.extend_from_slice(reward_manifest_hash); + payload.extend_from_slice(payout_policy_manifest_hash); + sha256_hex(&payload) } pub fn audit_commitment_root_hex(commitment_leaves_hex: &[String]) -> Result { @@ -706,7 +1359,7 @@ fn parse_audit_commitment_leaf(raw: &str, index: usize) -> Result<[u8; 32], Pris fn merkle_root(mut hashes: Vec<[u8; 32]>) -> [u8; 32] { debug_assert!(!hashes.is_empty()); while hashes.len() > 1 { - let mut next = Vec::with_capacity((hashes.len() + 1) / 2); + let mut next = Vec::with_capacity(hashes.len().div_ceil(2)); for pair in hashes.chunks(2) { let left = pair[0]; let right = if pair.len() == 2 { pair[1] } else { pair[0] }; @@ -1255,6 +1908,9 @@ fn audit_bundle_schema_for_shares(shares: &[AcceptedShare]) -> &'static str { } } +/// Build an audit bundle that owns `shares`. The window moves into +/// [`AuditBundle::shares`] unchanged and in input order; it is not copied. +/// A caller that must keep the window uses [`build_audit_bundle_body`] instead. pub fn build_audit_bundle( shares: Vec, found_block: FoundBlock, @@ -1263,7 +1919,30 @@ pub fn build_audit_bundle( coinbase_signing_key: &ManifestSigningKey, ledger_signing_key: &ManifestSigningKey, ) -> Result { - build_audit_bundle_with_coinbase_script_sig_suffix( + Ok(build_audit_bundle_body( + &shares, + found_block, + prior_balances, + payout_policy, + coinbase_signing_key, + ledger_signing_key, + )? + .into_bundle(shares)) +} + +/// Borrowing form of [`build_audit_bundle`]. The caller keeps ownership of +/// the payout window: the builder only reads `shares` and makes no copy of +/// it. Pair the body with the same window through [`AuditBundleBody::into_bundle`], +/// [`canonical_audit_bundle_bytes_from_parts`] or [`verify_audit_parts`]. +pub fn build_audit_bundle_body( + shares: &[AcceptedShare], + found_block: FoundBlock, + prior_balances: Vec, + payout_policy: PayoutPolicy, + coinbase_signing_key: &ManifestSigningKey, + ledger_signing_key: &ManifestSigningKey, +) -> Result { + build_audit_bundle_body_with_coinbase_script_sig_suffix( shares, found_block, prior_balances, @@ -1274,6 +1953,8 @@ pub fn build_audit_bundle( ) } +/// Owning form of [`build_audit_bundle_body_with_coinbase_script_sig_suffix`]. The +/// window moves into the bundle unchanged; it is not copied. pub fn build_audit_bundle_with_coinbase_script_sig_suffix( shares: Vec, found_block: FoundBlock, @@ -1283,7 +1964,31 @@ pub fn build_audit_bundle_with_coinbase_script_sig_suffix( coinbase_signing_key: &ManifestSigningKey, ledger_signing_key: &ManifestSigningKey, ) -> Result { - build_audit_bundle_with_coinbase_options( + Ok(build_audit_bundle_body_with_coinbase_script_sig_suffix( + &shares, + found_block, + prior_balances, + payout_policy, + coinbase_script_sig_suffix_hex, + coinbase_signing_key, + ledger_signing_key, + )? + .into_bundle(shares)) +} + +/// Borrowing form of [`build_audit_bundle_with_coinbase_script_sig_suffix`]. +/// The caller keeps ownership of the payout window: the builder only reads +/// `shares` and makes no copy of it. +pub fn build_audit_bundle_body_with_coinbase_script_sig_suffix( + shares: &[AcceptedShare], + found_block: FoundBlock, + prior_balances: Vec, + payout_policy: PayoutPolicy, + coinbase_script_sig_suffix_hex: Option, + coinbase_signing_key: &ManifestSigningKey, + ledger_signing_key: &ManifestSigningKey, +) -> Result { + build_audit_bundle_body_with_coinbase_options( shares, found_block, prior_balances, @@ -1295,6 +2000,10 @@ pub fn build_audit_bundle_with_coinbase_script_sig_suffix( ) } +/// Owning form of [`build_audit_bundle_body_with_coinbase_options`]. The window +/// moves into the bundle unchanged; it is not copied. +// The parameter list mirrors the borrowed form it forwards to, so callers see one signature. +#[allow(clippy::too_many_arguments)] pub fn build_audit_bundle_with_coinbase_options( shares: Vec, found_block: FoundBlock, @@ -1305,6 +2014,62 @@ pub fn build_audit_bundle_with_coinbase_options( coinbase_signing_key: &ManifestSigningKey, ledger_signing_key: &ManifestSigningKey, ) -> Result { + Ok(build_audit_bundle_body_with_coinbase_options( + &shares, + found_block, + prior_balances, + payout_policy, + coinbase_script_sig_suffix_hex, + witness_merkle_leaves_hex, + coinbase_signing_key, + ledger_signing_key, + )? + .into_bundle(shares)) +} + +/// Borrowing form of [`build_audit_bundle_with_coinbase_options`]. The +/// caller keeps ownership of the payout window: the builder only reads +/// `shares` and makes no copy of it. +#[allow(clippy::too_many_arguments)] +pub fn build_audit_bundle_body_with_coinbase_options( + shares: &[AcceptedShare], + found_block: FoundBlock, + prior_balances: Vec, + payout_policy: PayoutPolicy, + coinbase_script_sig_suffix_hex: Option, + witness_merkle_leaves_hex: Vec, + coinbase_signing_key: &ManifestSigningKey, + ledger_signing_key: &ManifestSigningKey, +) -> Result { + build_audit_bundle_body_with_coinbase_options_parallel( + shares, + found_block, + prior_balances, + payout_policy, + coinbase_script_sig_suffix_hex, + witness_merkle_leaves_hex, + coinbase_signing_key, + ledger_signing_key, + Parallelism::serial(), + ) +} + +/// [`build_audit_bundle_body_with_coinbase_options`] with the counted-share +/// fold and the reward manifest's leaf digest spread over `parallelism` +/// worker threads. The body is identical for every parallelism; verifiers +/// never take this path. +#[allow(clippy::too_many_arguments)] +pub fn build_audit_bundle_body_with_coinbase_options_parallel( + shares: &[AcceptedShare], + found_block: FoundBlock, + prior_balances: Vec, + payout_policy: PayoutPolicy, + coinbase_script_sig_suffix_hex: Option, + witness_merkle_leaves_hex: Vec, + coinbase_signing_key: &ManifestSigningKey, + ledger_signing_key: &ManifestSigningKey, + parallelism: Parallelism, +) -> Result { if coinbase_signing_key .public_key_hex() .eq_ignore_ascii_case(&ledger_signing_key.public_key_hex()) @@ -1312,7 +2077,7 @@ pub fn build_audit_bundle_with_coinbase_options( return Err(PrismError::LedgerAttestationKeyReuse); } let reward_manifest = profile_audit_build_phase(AUDIT_BUILD_PAYOUT_DERIVATION_PHASE, || { - build_prism_reward_manifest(&shares, &found_block) + build_prism_reward_manifest_parallel(shares, &found_block, parallelism) })?; let ledger_window_attestation = profile_audit_build_phase(AUDIT_BUILD_SIGNING_PHASE, || { build_ledger_window_attestation(&reward_manifest, &prior_balances, ledger_signing_key) @@ -1323,10 +2088,14 @@ pub fn build_audit_bundle_with_coinbase_options( })?; let (audit_commitment_leaves_hex, audit_commitment_root_hex) = profile_audit_build_phase(AUDIT_BUILD_COINBASE_PHASE, || { - let leaves = vec![prism_audit_commitment_leaf_hex( - &reward_manifest, - &payout_policy_manifest, - )?]; + let reward_manifest_hash = reward_manifest_sha256(&reward_manifest, parallelism)?; + let payout_policy_manifest_hash = Sha256::digest( + canonical_payout_policy_manifest_bytes(&payout_policy_manifest)?, + ); + let leaves = vec![commitment_leaf_hex( + &reward_manifest_hash, + &payout_policy_manifest_hash, + )]; let root = audit_commitment_root_hex(&leaves)?; Ok::<_, PrismError>((leaves, root)) })?; @@ -1342,9 +2111,8 @@ pub fn build_audit_bundle_with_coinbase_options( let signed_coinbase_manifest = build_profiled_signed_manifest(coinbase_request, coinbase_signing_key)?; - Ok(AuditBundle { - schema: audit_bundle_schema_for_shares(&shares).to_string(), - shares, + Ok(AuditBundleBody { + schema: audit_bundle_schema_for_shares(shares).to_string(), found_block, prior_balances, payout_policy, @@ -1362,6 +2130,10 @@ pub fn build_audit_bundle_with_coinbase_options( }) } +/// Owning form of [`build_audit_bundle_body_with_ctv_settlement_options`]. The +/// window moves into the bundle unchanged; it is not copied. +// The parameter list mirrors the borrowed form it forwards to, so callers see one signature. +#[allow(clippy::too_many_arguments)] pub fn build_audit_bundle_with_ctv_settlement_options( shares: Vec, found_block: FoundBlock, @@ -1375,6 +2147,75 @@ pub fn build_audit_bundle_with_ctv_settlement_options( coinbase_signing_key: &ManifestSigningKey, ledger_signing_key: &ManifestSigningKey, ) -> Result { + Ok(build_audit_bundle_body_with_ctv_settlement_options( + &shares, + found_block, + prior_balances, + payout_policy, + direct_floor_sats, + settlement_config, + ctv_fanout_fee_policy, + coinbase_script_sig_suffix_hex, + witness_merkle_leaves_hex, + coinbase_signing_key, + ledger_signing_key, + )? + .into_bundle(shares)) +} + +/// Borrowing form of [`build_audit_bundle_with_ctv_settlement_options`]. The +/// caller keeps ownership of the payout window: the builder only reads +/// `shares` and makes no copy of it. +#[allow(clippy::too_many_arguments)] +pub fn build_audit_bundle_body_with_ctv_settlement_options( + shares: &[AcceptedShare], + found_block: FoundBlock, + prior_balances: Vec, + payout_policy: PayoutPolicy, + direct_floor_sats: u64, + settlement_config: SettlementModeConfig, + ctv_fanout_fee_policy: Option, + coinbase_script_sig_suffix_hex: Option, + witness_merkle_leaves_hex: Vec, + coinbase_signing_key: &ManifestSigningKey, + ledger_signing_key: &ManifestSigningKey, +) -> Result { + build_audit_bundle_body_with_ctv_settlement_options_parallel( + shares, + found_block, + prior_balances, + payout_policy, + direct_floor_sats, + settlement_config, + ctv_fanout_fee_policy, + coinbase_script_sig_suffix_hex, + witness_merkle_leaves_hex, + coinbase_signing_key, + ledger_signing_key, + Parallelism::serial(), + ) +} + +/// [`build_audit_bundle_body_with_ctv_settlement_options`] with the +/// counted-share fold and the reward manifest's digest spread over +/// `parallelism` worker threads. The reward manifest is digested once for +/// the commitment leaf and every fan-out precommitment; the body is +/// identical for every parallelism, and verifiers never take this path. +#[allow(clippy::too_many_arguments)] +pub fn build_audit_bundle_body_with_ctv_settlement_options_parallel( + shares: &[AcceptedShare], + found_block: FoundBlock, + prior_balances: Vec, + payout_policy: PayoutPolicy, + direct_floor_sats: u64, + settlement_config: SettlementModeConfig, + ctv_fanout_fee_policy: Option, + coinbase_script_sig_suffix_hex: Option, + witness_merkle_leaves_hex: Vec, + coinbase_signing_key: &ManifestSigningKey, + ledger_signing_key: &ManifestSigningKey, + parallelism: Parallelism, +) -> Result { if coinbase_signing_key .public_key_hex() .eq_ignore_ascii_case(&ledger_signing_key.public_key_hex()) @@ -1382,7 +2223,7 @@ pub fn build_audit_bundle_with_ctv_settlement_options( return Err(PrismError::LedgerAttestationKeyReuse); } let reward_manifest = profile_audit_build_phase(AUDIT_BUILD_PAYOUT_DERIVATION_PHASE, || { - build_prism_reward_manifest(&shares, &found_block) + build_prism_reward_manifest_parallel(shares, &found_block, parallelism) })?; let ledger_window_attestation = profile_audit_build_phase(AUDIT_BUILD_SIGNING_PHASE, || { build_ledger_window_attestation(&reward_manifest, &prior_balances, ledger_signing_key) @@ -1412,15 +2253,17 @@ pub fn build_audit_bundle_with_ctv_settlement_options( }; Ok::<_, PrismError>((payout_manifest, settlement_decision, fee_recipients)) })?; - let (reward_manifest_sha256_hex, payout_policy_manifest_sha256_hex) = + let (reward_manifest_hash, payout_policy_manifest_hash) = profile_audit_build_phase(AUDIT_BUILD_CTV_MANIFEST_PHASE, || { Ok::<_, PrismError>(( - sha256_hex(&canonical_reward_manifest_bytes(&reward_manifest)?), - sha256_hex(&canonical_payout_policy_manifest_bytes( + reward_manifest_sha256(&reward_manifest, parallelism)?, + Sha256::digest(canonical_payout_policy_manifest_bytes( &payout_policy_manifest, )?), )) })?; + let reward_manifest_sha256_hex = hex::encode(reward_manifest_hash); + let payout_policy_manifest_sha256_hex = hex::encode(payout_policy_manifest_hash); let prepared_fanouts = profile_audit_build_phase(AUDIT_BUILD_CTV_MANIFEST_PHASE, || { let mut prepared = Vec::new(); @@ -1471,10 +2314,10 @@ pub fn build_audit_bundle_with_ctv_settlement_options( let (audit_commitment_leaves_hex, audit_commitment_root_hex) = profile_audit_build_phase(AUDIT_BUILD_COINBASE_PHASE, || { - let mut leaves = vec![prism_audit_commitment_leaf_hex( - &reward_manifest, - &payout_policy_manifest, - )?]; + let mut leaves = vec![commitment_leaf_hex( + &reward_manifest_hash, + &payout_policy_manifest_hash, + )]; leaves.extend( prepared_fanouts .iter() @@ -1526,9 +2369,8 @@ pub fn build_audit_bundle_with_ctv_settlement_options( )?) }; - Ok(AuditBundle { - schema: audit_bundle_schema_for_shares(&shares).to_string(), - shares, + Ok(AuditBundleBody { + schema: audit_bundle_schema_for_shares(shares).to_string(), found_block, prior_balances, payout_policy, @@ -1836,15 +2678,39 @@ pub fn verify_audit_bundle( bundle: &AuditBundle, ledger_writer_public_key_hex: &str, ) -> Result { - if bundle.schema != audit_bundle_schema_for_shares(&bundle.shares) { + verify_audit_view( + AuditBundleRef::from_bundle(bundle), + ledger_writer_public_key_hex, + ) +} + +/// Verify `body` against the payout window `shares` it was built from. The +/// caller keeps ownership of the window and no copy of it is made. The +/// report equals [`verify_audit_bundle`]'s for the assembled bundle, +/// including `audit_bundle_sha256_hex`, which covers the bytes with shares. +pub fn verify_audit_parts( + body: &AuditBundleBody, + shares: &[AcceptedShare], + ledger_writer_public_key_hex: &str, +) -> Result { + verify_audit_view( + AuditBundleRef::from_parts(body, shares), + ledger_writer_public_key_hex, + ) +} + +fn verify_audit_view( + bundle: AuditBundleRef<'_>, + ledger_writer_public_key_hex: &str, +) -> Result { + if *bundle.schema != audit_bundle_schema_for_shares(bundle.shares) { return Err(PrismError::AuditMismatch { artifact: "schema" }); } verify_ledger_window_attestation(bundle, ledger_writer_public_key_hex)?; - verify_signed_manifest(&bundle.signed_coinbase_manifest)?; + verify_signed_manifest(bundle.signed_coinbase_manifest)?; - let expected_reward_manifest = - build_prism_reward_manifest(&bundle.shares, &bundle.found_block)?; - if expected_reward_manifest != bundle.reward_manifest { + let expected_reward_manifest = build_prism_reward_manifest(bundle.shares, bundle.found_block)?; + if expected_reward_manifest != *bundle.reward_manifest { return Err(PrismError::AuditMismatch { artifact: "reward_manifest", }); @@ -1852,8 +2718,8 @@ pub fn verify_audit_bundle( let mut expected_policy_manifest = apply_payout_policy( &expected_reward_manifest, - &bundle.prior_balances, - &bundle.payout_policy, + bundle.prior_balances, + bundle.payout_policy, )?; if let (Some(decision), Some(fee_policy)) = ( bundle.settlement_mode_decision.as_ref(), @@ -1861,7 +2727,7 @@ pub fn verify_audit_bundle( ) { apply_ctv_fanout_fee_accounting(&mut expected_policy_manifest, decision, fee_policy)?; } - if expected_policy_manifest != bundle.payout_policy_manifest { + if expected_policy_manifest != *bundle.payout_policy_manifest { return Err(PrismError::AuditMismatch { artifact: "payout_policy_manifest", }); @@ -1881,7 +2747,7 @@ pub fn verify_audit_bundle( bundle.ctv_fanout_manifest_set.as_ref(), &expected_policy_manifest, &bundle.signed_coinbase_manifest.manifest, - &bundle.audit_commitment_leaves_hex, + bundle.audit_commitment_leaves_hex, )?; if let Some(fanout_set) = &bundle.ctv_fanout_manifest_set { expected_audit_commitment_leaves.extend( @@ -1896,7 +2762,7 @@ pub fn verify_audit_bundle( expected_policy_manifest.onchain_entitlements.clone() }; if !hex_vec_eq_ignore_ascii_case( - &bundle.audit_commitment_leaves_hex, + bundle.audit_commitment_leaves_hex, &expected_audit_commitment_leaves, ) { return Err(PrismError::AuditMismatch { @@ -1946,7 +2812,34 @@ pub fn verify_audit_bundle_against_coinbase_tx_hex( onchain_coinbase_tx_hex: &str, ledger_writer_public_key_hex: &str, ) -> Result { - let report = verify_audit_bundle(bundle, ledger_writer_public_key_hex)?; + verify_audit_view_against_coinbase_tx_hex( + AuditBundleRef::from_bundle(bundle), + onchain_coinbase_tx_hex, + ledger_writer_public_key_hex, + ) +} + +/// Parts form of [`verify_audit_bundle_against_coinbase_tx_hex`]. The caller +/// keeps ownership of the window and no copy of it is made. +pub fn verify_audit_parts_against_coinbase_tx_hex( + body: &AuditBundleBody, + shares: &[AcceptedShare], + onchain_coinbase_tx_hex: &str, + ledger_writer_public_key_hex: &str, +) -> Result { + verify_audit_view_against_coinbase_tx_hex( + AuditBundleRef::from_parts(body, shares), + onchain_coinbase_tx_hex, + ledger_writer_public_key_hex, + ) +} + +fn verify_audit_view_against_coinbase_tx_hex( + bundle: AuditBundleRef<'_>, + onchain_coinbase_tx_hex: &str, + ledger_writer_public_key_hex: &str, +) -> Result { + let report = verify_audit_view(bundle, ledger_writer_public_key_hex)?; if !report .coinbase_tx_hex .eq_ignore_ascii_case(onchain_coinbase_tx_hex.trim()) @@ -1962,7 +2855,39 @@ pub fn verify_audit_bundle_against_coinbase_tx_hex_and_expected_coinbase_value( ledger_writer_public_key_hex: &str, expected_coinbase_value_sats: u64, ) -> Result { - let report = verify_audit_bundle_against_coinbase_tx_hex( + verify_audit_view_against_coinbase_tx_hex_and_expected_coinbase_value( + AuditBundleRef::from_bundle(bundle), + onchain_coinbase_tx_hex, + ledger_writer_public_key_hex, + expected_coinbase_value_sats, + ) +} + +/// Parts form of +/// [`verify_audit_bundle_against_coinbase_tx_hex_and_expected_coinbase_value`]. +/// The caller keeps ownership of the window and no copy of it is made. +pub fn verify_audit_parts_against_coinbase_tx_hex_and_expected_coinbase_value( + body: &AuditBundleBody, + shares: &[AcceptedShare], + onchain_coinbase_tx_hex: &str, + ledger_writer_public_key_hex: &str, + expected_coinbase_value_sats: u64, +) -> Result { + verify_audit_view_against_coinbase_tx_hex_and_expected_coinbase_value( + AuditBundleRef::from_parts(body, shares), + onchain_coinbase_tx_hex, + ledger_writer_public_key_hex, + expected_coinbase_value_sats, + ) +} + +fn verify_audit_view_against_coinbase_tx_hex_and_expected_coinbase_value( + bundle: AuditBundleRef<'_>, + onchain_coinbase_tx_hex: &str, + ledger_writer_public_key_hex: &str, + expected_coinbase_value_sats: u64, +) -> Result { + let report = verify_audit_view_against_coinbase_tx_hex( bundle, onchain_coinbase_tx_hex, ledger_writer_public_key_hex, @@ -1977,17 +2902,17 @@ pub fn verify_audit_bundle_against_coinbase_tx_hex_and_expected_coinbase_value( } fn audit_verification_report( - bundle: &AuditBundle, + bundle: AuditBundleRef<'_>, coinbase_manifest: &PayoutManifest, ) -> Result { let reward_manifest_sha256_hex = - sha256_hex(&canonical_reward_manifest_bytes(&bundle.reward_manifest)?); + sha256_hex(&canonical_reward_manifest_bytes(bundle.reward_manifest)?); let payout_policy_manifest_sha256_hex = sha256_hex(&canonical_payout_policy_manifest_bytes( - &bundle.payout_policy_manifest, + bundle.payout_policy_manifest, )?); let prism_audit_commitment_leaf_hex = - prism_audit_commitment_leaf_hex(&bundle.reward_manifest, &bundle.payout_policy_manifest)?; - let audit_commitment_root_hex = audit_commitment_root_hex(&bundle.audit_commitment_leaves_hex)?; + prism_audit_commitment_leaf_hex(bundle.reward_manifest, bundle.payout_policy_manifest)?; + let audit_commitment_root_hex = audit_commitment_root_hex(bundle.audit_commitment_leaves_hex)?; Ok(AuditVerificationReport { schema: "qbit.prism.audit-verification-report.v1".to_string(), block_height: bundle.found_block.block_height, @@ -1997,7 +2922,7 @@ fn audit_verification_report( prism_audit_commitment_leaf_hex, audit_commitment_root_hex, coinbase_manifest_sha256_hex: sha256_hex(&serde_json::to_vec(coinbase_manifest)?), - audit_bundle_sha256_hex: sha256_hex(&canonical_audit_bundle_bytes(bundle)?), + audit_bundle_sha256_hex: sha256_hex(&bundle.canonical_bytes()?), coinbase_txid: coinbase_manifest.coinbase_txid.clone(), coinbase_wtxid: coinbase_manifest.coinbase_wtxid.clone(), coinbase_tx_hex: coinbase_manifest.coinbase_tx_hex.clone(), @@ -2056,10 +2981,10 @@ fn is_zero_u64(value: &u64) -> bool { } fn verify_ledger_window_attestation( - bundle: &AuditBundle, + bundle: AuditBundleRef<'_>, writer_public_key_hex: &str, ) -> Result<(), PrismError> { - let attestation = &bundle.ledger_window_attestation; + let attestation = bundle.ledger_window_attestation; if attestation.schema != "qbit.prism.ledger-window-attestation.v1" { return Err(PrismError::AuditMismatch { artifact: "ledger_attestation_schema", @@ -2079,7 +3004,7 @@ fn verify_ledger_window_attestation( artifact: "ledger_attestation_share_digest", }); } - if attestation.prior_balances_digest_hex != prior_balances_digest_hex(&bundle.prior_balances) { + if attestation.prior_balances_digest_hex != prior_balances_digest_hex(bundle.prior_balances) { return Err(PrismError::AuditMismatch { artifact: "ledger_attestation_prior_digest", }); @@ -2113,25 +3038,45 @@ fn sha256_hex(bytes: &[u8]) -> String { fn share_slice_digest_hex(shares: &[CountedShare]) -> String { let mut hasher = Sha256::new(); for share in shares { - update_u64(&mut hasher, share.share_seq); - update_string(&mut hasher, &share.share_id); - update_string(&mut hasher, &share.miner_id); - update_string(&mut hasher, &share.order_key); - update_string(&mut hasher, &share.p2mr_program_hex); - update_u128(&mut hasher, share.share_difficulty); - update_u128(&mut hasher, share.counted_difficulty); - update_i64(&mut hasher, share.job_issued_at_ms); - update_i64(&mut hasher, share.accepted_at_ms); - if let Some(credit_policy) = &share.credit_policy { - update_string(&mut hasher, "credit_policy"); - update_string(&mut hasher, credit_policy); - } + update_slice_digest(&mut hasher, share); } hex::encode(hasher.finalize()) } -fn prior_balances_digest_hex(balances: &[CarryForwardBalance]) -> String { - let mut ordered = balances.to_vec(); +/// One counted share's contribution to the slice digest, in window order. +fn update_slice_digest(hasher: &mut impl Absorb, share: &CountedShare) { + update_u64(hasher, share.share_seq); + update_string(hasher, &share.share_id); + update_string(hasher, &share.miner_id); + update_string(hasher, &share.order_key); + update_string(hasher, &share.p2mr_program_hex); + update_u128(hasher, share.share_difficulty); + update_u128(hasher, share.counted_difficulty); + update_i64(hasher, share.job_issued_at_ms); + update_i64(hasher, share.accepted_at_ms); + if let Some(credit_policy) = &share.credit_policy { + update_string(hasher, "credit_policy"); + update_string(hasher, credit_policy); + } +} + +/// The sha256 digest a ledger window attestation commits to for the prior +/// carry-forward balances. It equals +/// `ledger_window_attestation.prior_balances_digest_hex` decoded from hex. +/// +/// The balances are sorted by `(order_key, recipient_id, p2mr_program_hex)`, +/// so the digest does not depend on input order. Each balance then feeds the +/// hash, in order, with: +/// +/// - `recipient_id`, `order_key` and `p2mr_program_hex`, each as its UTF-8 +/// byte length (`u64`, big-endian) followed by its bytes; +/// - `balance_sats` as a 16-byte big-endian two's-complement `i128`. +/// +/// No count or separator is added, so an empty slice hashes to sha256 of the +/// empty string. The digest is part of a signed, persisted attestation and +/// must not change for any input. +pub fn prior_balances_digest(balances: &[CarryForwardBalance]) -> [u8; 32] { + let mut ordered = balances.iter().collect::>(); ordered.sort_by(|left, right| { left.order_key .cmp(&right.order_key) @@ -2146,28 +3091,51 @@ fn prior_balances_digest_hex(balances: &[CarryForwardBalance]) -> String { update_string(&mut hasher, &balance.p2mr_program_hex); update_i128(&mut hasher, balance.balance_sats); } - hex::encode(hasher.finalize()) + hasher.finalize().into() +} + +fn prior_balances_digest_hex(balances: &[CarryForwardBalance]) -> String { + hex::encode(prior_balances_digest(balances)) +} + +/// Where the hand-rolled digests put their bytes: the hasher itself, or a +/// buffer the builder hands to the hasher later in one update. The digest is +/// the same either way; only the number of hasher calls differs. +trait Absorb { + fn absorb(&mut self, bytes: &[u8]); } -fn update_string(hasher: &mut Sha256, value: &str) { +impl Absorb for Sha256 { + fn absorb(&mut self, bytes: &[u8]) { + self.update(bytes); + } +} + +impl Absorb for Vec { + fn absorb(&mut self, bytes: &[u8]) { + self.extend_from_slice(bytes); + } +} + +fn update_string(hasher: &mut impl Absorb, value: &str) { update_u64(hasher, value.len() as u64); - hasher.update(value.as_bytes()); + hasher.absorb(value.as_bytes()); } -fn update_u64(hasher: &mut Sha256, value: u64) { - hasher.update(value.to_be_bytes()); +fn update_u64(hasher: &mut impl Absorb, value: u64) { + hasher.absorb(&value.to_be_bytes()); } -fn update_i64(hasher: &mut Sha256, value: i64) { - hasher.update(value.to_be_bytes()); +fn update_i64(hasher: &mut impl Absorb, value: i64) { + hasher.absorb(&value.to_be_bytes()); } -fn update_i128(hasher: &mut Sha256, value: i128) { - hasher.update(value.to_be_bytes()); +fn update_i128(hasher: &mut impl Absorb, value: i128) { + hasher.absorb(&value.to_be_bytes()); } -fn update_u128(hasher: &mut Sha256, value: u128) { - hasher.update(value.to_be_bytes()); +fn update_u128(hasher: &mut impl Absorb, value: u128) { + hasher.absorb(&value.to_be_bytes()); } fn aggregate_entitlements( @@ -2796,6 +3764,35 @@ mod tests { assert_fixture(include_str!( "../fixtures/difficulty-change.prism-fixture.json" )); + + let fixture = load_policy_fixture(include_str!( + "../fixtures/power-law-accrual.prism-fixture.json" + )); + let reward_manifest = + build_prism_reward_manifest(&fixture.shares, &fixture.found_block).unwrap(); + let policy_manifest = apply_payout_policy( + &reward_manifest, + &power_law_prior_balances(), + &PayoutPolicy::day_one_default(), + ) + .unwrap(); + assert_eq!( + policy_manifest.min_output_sats, + fixture.expected_min_output_sats + ); + assert_eq!( + policy_manifest.onchain_entitlements.len(), + fixture.expected_onchain_count + ); + assert_eq!( + policy_manifest + .accounts + .iter() + .filter(|account| account.action == PayoutPolicyAction::Accrued) + .map(|account| account.recipient_id.clone()) + .collect::>(), + fixture.expected_accrued_recipients + ); } #[test] @@ -4254,12 +5251,7 @@ mod tests { vec!["pool-fee", "ctv-fanout-0"] ); assert_eq!( - bundle - .ctv_fanout_manifest_set - .as_ref() - .unwrap() - .manifests[0] - .parent_coinbase_vout, + bundle.ctv_fanout_manifest_set.as_ref().unwrap().manifests[0].parent_coinbase_vout, 1 ); diff --git a/crates/qbit-prism/src/parallel.rs b/crates/qbit-prism/src/parallel.rs new file mode 100644 index 000000000..30b2455ee --- /dev/null +++ b/crates/qbit-prism/src/parallel.rs @@ -0,0 +1,1148 @@ +//! Builder-only ordered parallel serialization: the same bytes, produced by +//! several threads in index order. +//! +//! Nothing here defines a canonical byte layout. Each array element and each +//! struct field value is written by `serde_json`'s own serializer (the same +//! escaping and the same `arbitrary_precision` integer digits as the whole +//! value would get), and the only framing added by hand is the `[`, `,`, `]` +//! and `{`, `,`, `"name":`, `}` that `serde_json` writes around them. Verifier +//! paths never call into this module: they serialize whole values serially. +use serde::ser::{Impossible, SerializeStruct, Serializer}; +use serde::Serialize; +use sha2::{Digest, Sha256}; +use std::cell::Cell; +use std::collections::{BTreeMap, VecDeque}; +use std::io::Write; +use std::sync::atomic::{AtomicUsize, Ordering}; +use std::sync::mpsc::{sync_channel, SyncSender}; +use std::sync::{Arc, Condvar, Mutex, OnceLock}; +use std::thread::{Scope, ScopedJoinHandle}; + +/// How much of a build may run on worker threads. +/// +/// [`Parallelism::serial`] keeps every byte on the calling thread through the +/// unmodified `serde_json::to_writer` path; anything wider splits share arrays +/// into `chunk_len` element chunks that `workers` threads serialize while the +/// caller consumes them in index order. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct Parallelism { + workers: usize, + chunk_len: usize, +} + +impl Parallelism { + const MAX_WORKERS: usize = 8; + const DEFAULT_CHUNK_LEN: usize = 4096; + + /// One thread, whole-value `serde_json` serialization. + pub const fn serial() -> Self { + Self { + workers: 1, + chunk_len: usize::MAX, + } + } + + /// `workers` threads over `chunk_len` element chunks; both are raised to + /// at least one. + pub fn new(workers: usize, chunk_len: usize) -> Self { + Self { + workers: workers.max(1), + chunk_len: chunk_len.max(1), + } + } + + /// This process's builder parallelism: [`Parallelism::serial`] when the + /// builder pool is disabled ([`configure_builder_threads`] with `0`), + /// otherwise a per-stage window of up to eight chunks in flight (at most + /// the pool's thread count) over 4096-element chunks. The window bounds + /// look-ahead memory per stage; the threads are the pool's (see + /// [`builder_threads`]), shared by every stage in the process. + pub fn detect() -> Self { + let threads = builder_threads(); + if threads == 0 { + Self::serial() + } else { + Self::new(threads.min(Self::MAX_WORKERS), Self::DEFAULT_CHUNK_LEN) + } + } + + pub fn workers(&self) -> usize { + self.workers + } + + pub fn chunk_len(&self) -> usize { + self.chunk_len + } + + pub fn is_serial(&self) -> bool { + self.workers <= 1 + } +} + +impl Default for Parallelism { + fn default() -> Self { + Self::serial() + } +} + +/// One job on the builder pool: a closure borrowing the caller's `produce`, +/// its lifetime erased by [`ordered_chunks`], which never returns (or +/// unwinds) before every job's last use of that borrow. +type Job = Box; + +/// The builder pool's thread count for this process, fixed by the operator +/// before the pool starts. `0` disables the pool: every builder runs serially. +static CONFIGURED_THREADS: OnceLock = OnceLock::new(); + +/// The pool itself, started on first use. +static POOL: OnceLock<&'static Pool> = OnceLock::new(); + +thread_local! { + /// Set on every pool thread: a builder that runs *on* the pool must not + /// wait for the pool, so [`ordered_chunks`] runs inline there. + static ON_POOL_THREAD: Cell = const { Cell::new(false) }; +} + +/// [`configure_builder_threads`] was called after the count was fixed, by an +/// earlier call or by the pool starting, with a different value. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct BuilderThreadsFixed { + /// The count in force. + pub threads: usize, +} + +impl std::fmt::Display for BuilderThreadsFixed { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!( + f, + "builder threads already fixed at {} for this process", + self.threads + ) + } +} + +impl std::error::Error for BuilderThreadsFixed {} + +/// Fix the builder pool's thread count for this process before the pool +/// starts: `0` disables the pool and every builder runs serially +/// ([`Parallelism::detect`] returns [`Parallelism::serial`]). Idempotent for +/// the same value; a different value after the count is fixed is an error. +pub fn configure_builder_threads(threads: usize) -> Result<(), BuilderThreadsFixed> { + if let Some(pool) = POOL.get() { + let started = pool.started(); + return if started == threads { + Ok(()) + } else { + Err(BuilderThreadsFixed { threads: started }) + }; + } + match CONFIGURED_THREADS.set(threads) { + Ok(()) => Ok(()), + Err(_) => { + let fixed = *CONFIGURED_THREADS.get().expect("set"); + if fixed == threads { + Ok(()) + } else { + Err(BuilderThreadsFixed { threads: fixed }) + } + } + } +} + +/// The builder pool's thread count: the configured value, else three +/// quarters of the host's cores clamped to `1..=4`. +/// +/// Four is where the measured refresh stops getting faster: every stage's +/// consumer (the SHA-256 states and the fold's merge) is the bound, and +/// four producers keep it fed; more threads only spread the counted-share +/// strings and chunk buffers over more allocator arenas (at 400k shares, +/// 16 threads retained ~130 MiB more per frontend than four for the same +/// refresh time; two threads left the tee's producers as the bottleneck). +/// +/// What that costs on a host: the pool's threads plus, per refresh, its two +/// lane threads (Tokio's blocking pool) and its two digest threads, so up to +/// 8 CPU-bound threads per frontend process at the default; every frontend +/// on the host has its own pool, so two frontends can run 16. The pool only +/// works during the refresh's build, which ends before the fan-out. +/// Operators size it with `PRISM_REFRESH_BUILD_THREADS` (up to 64). +pub fn builder_threads() -> usize { + if let Some(pool) = POOL.get() { + return pool.started(); + } + CONFIGURED_THREADS.get().copied().unwrap_or_else(|| { + (std::thread::available_parallelism() + .map(|count| count.get()) + .unwrap_or(1) + * 3 + / 4) + .clamp(1, 4) + }) +} + +/// The builder's worker threads: one process-wide pool, started on first +/// use and never stopped, so every refresh's chunk work lands on the same +/// threads. That keeps their allocator arenas stable across refreshes; +/// threads spawned per stage would each take a fresh arena and leave the +/// counted-share strings and chunk buffers fragmented across dozens of them. +struct Pool { + queue: Mutex>, + signal: Condvar, + /// Threads that actually started; a spawn failure leaves fewer, and + /// none at all makes every caller run serially. + started: AtomicUsize, +} + +impl Pool { + /// The pool, started with [`builder_threads`] threads on first use; + /// `None` when the operator disabled it. + fn global() -> Option<&'static Pool> { + if builder_threads() == 0 { + return None; + } + Some(POOL.get_or_init(|| { + let pool: &'static Pool = Box::leak(Box::new(Pool { + queue: Mutex::new(VecDeque::new()), + signal: Condvar::new(), + started: AtomicUsize::new(0), + })); + for index in 0..builder_threads() { + // A failed spawn is an ordinary resource error: keep the + // threads that did start; with none, callers stay serial. + if std::thread::Builder::new() + .name(format!("prism-build-{index}")) + .spawn(move || pool.run()) + .is_ok() + { + pool.started.fetch_add(1, Ordering::SeqCst); + } + } + pool + })) + } + + fn started(&self) -> usize { + self.started.load(Ordering::SeqCst) + } + + fn run(&self) { + ON_POOL_THREAD.with(|flag| flag.set(true)); + loop { + let job = { + let mut queue = self.queue.lock().unwrap_or_else(|e| e.into_inner()); + loop { + if let Some(job) = queue.pop_front() { + break job; + } + queue = self.signal.wait(queue).unwrap_or_else(|e| e.into_inner()); + } + }; + // Jobs catch their own panics; one that unwound here would take + // a pool thread with it. + job(); + } + } + + fn submit(&self, job: Job) { + self.queue + .lock() + .unwrap_or_else(|e| e.into_inner()) + .push_back(job); + self.signal.notify_one(); + } +} + +struct State { + ready: BTreeMap, + failed: Option, + panicked: Option>, + outstanding: usize, +} + +impl State { + fn stopping(&self) -> bool { + self.failed.is_some() || self.panicked.is_some() + } +} + +/// One [`ordered_chunks`] call's state, shared with its jobs through an +/// `Arc` rather than borrowed. A job's last actions, decrementing +/// `outstanding` and unlocking the mutex, touch only this, and a finished +/// job may still be inside that unlock (its futex wake) after the caller +/// has seen zero and returned; nothing on the caller's frame is involved. +/// Only `produce` stays borrowed, and its last use is inside a job's +/// `catch_unwind`, before the decrement. [`Drain`] empties the state on the +/// calling thread once the jobs are done, so a clone that outlives the call +/// owns only empty containers. +struct Shared { + state: Mutex>, + signal: Condvar, +} + +/// Waits for every submitted job to finish, also while unwinding, and takes +/// everything the jobs left behind: no job may outlive the borrowed +/// `produce` it was given, and the chunks and error they produced, which may +/// borrow from the caller too, are dropped or returned on the calling +/// thread. The last clone of the shared state, which a finished job may +/// still hold, then owns nothing the caller lent. +struct Drain<'a, C, E>(&'a Shared); + +/// What the jobs left once the last of them finished. +type Settled = ( + Option>, + Option, + BTreeMap, +); + +impl Drain<'_, C, E> { + fn settle(shared: &Shared) -> Settled { + let mut state = shared.state.lock().unwrap_or_else(|e| e.into_inner()); + while state.outstanding > 0 { + state = shared.signal.wait(state).unwrap_or_else(|e| e.into_inner()); + } + ( + state.panicked.take(), + state.failed.take(), + std::mem::take(&mut state.ready), + ) + } + + /// The first panic and the first error the jobs or the consumer + /// reported, once every job has finished. Chunks the consumer never + /// took are dropped here. + fn finish(self) -> (Option>, Option) { + let shared = self.0; + std::mem::forget(self); + let (panicked, failed, _untaken) = Self::settle(shared); + (panicked, failed) + } +} + +impl Drop for Drain<'_, C, E> { + fn drop(&mut self) { + drop(Self::settle(self.0)); + } +} + +/// Produce chunks `0..count` on the builder pool and consume each one, in +/// index order, on the calling thread. At most `2 * workers` chunks are +/// submitted ahead of the consumer, so memory stays bounded by that many +/// chunks. The first error, from either side, stops further submission and +/// is returned once the jobs already running have finished; a panic in +/// `produce` is re-raised here, likewise only after every job has finished, +/// and a panic in `consume` unwinds only after every job has finished. +/// +/// Contract for `produce`: it must never block on anything the consumer +/// feeds (the consumer runs on the calling thread and waits for chunks in +/// order) and it must not depend on the pool, because it may itself be run +/// on a pool thread. A caller that is already on a pool thread, a disabled +/// pool, or a pool whose threads failed to start all run this function +/// serially on the calling thread, so nested use cannot deadlock the pool. +pub(crate) fn ordered_chunks( + count: usize, + workers: usize, + produce: impl Fn(usize) -> Result + Sync, + mut consume: impl FnMut(usize, C) -> Result<(), E>, +) -> Result<(), E> +where + C: Send, + E: Send, +{ + if count == 0 { + return Ok(()); + } + let pool = match Pool::global() { + Some(pool) if pool.started() > 0 && !ON_POOL_THREAD.with(Cell::get) => pool, + _ => { + #[cfg(test)] + SERIAL_FALLBACKS.fetch_add(1, Ordering::SeqCst); + for index in 0..count { + consume(index, produce(index)?)?; + } + return Ok(()); + } + }; + let max_inflight = workers.clamp(1, count) * 2; + let shared = Arc::new(Shared { + state: Mutex::new(State { + ready: BTreeMap::new(), + failed: None, + panicked: None, + outstanding: 0, + }), + signal: Condvar::new(), + }); + let drain = Drain(&shared); + let produce: &(dyn Fn(usize) -> Result + Sync) = &produce; + let mut next = 0; + for index in 0..count { + { + let mut guard = shared.state.lock().unwrap_or_else(|e| e.into_inner()); + while next < count && next - index < max_inflight && !guard.stopping() { + guard.outstanding += 1; + let submitted = next; + next += 1; + let shared = Arc::clone(&shared); + let job: Box = Box::new(move || { + let outcome = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| { + produce(submitted) + })); + let mut guard = shared.state.lock().unwrap_or_else(|e| e.into_inner()); + match outcome { + Ok(Ok(chunk)) => { + guard.ready.insert(submitted, chunk); + } + Ok(Err(error)) => { + guard.failed.get_or_insert(error); + } + Err(panic) => { + guard.panicked.get_or_insert(panic); + } + } + guard.outstanding -= 1; + shared.signal.notify_all(); + }); + // SAFETY: the only borrow the job carries is `produce`, a + // parameter of this function that outlives every exit path: + // `drain`, declared after it, waits for `outstanding` to + // reach zero before it is dropped, on return and on unwind + // alike, jobs never block, so that wait ends, and a job's + // last use of `produce` is inside `catch_unwind`, before it + // decrements `outstanding`. Everything a job touches after + // that decrement is owned by its clone of `shared`. + let job: Job = unsafe { std::mem::transmute(job) }; + pool.submit(job); + } + } + let chunk = { + let mut guard = shared.state.lock().unwrap_or_else(|e| e.into_inner()); + loop { + if let Some(chunk) = guard.ready.remove(&index) { + break Some(chunk); + } + if guard.stopping() { + break None; + } + guard = shared.signal.wait(guard).unwrap_or_else(|e| e.into_inner()); + } + }; + let Some(chunk) = chunk else { + break; + }; + if let Err(error) = consume(index, chunk) { + shared + .state + .lock() + .unwrap_or_else(|e| e.into_inner()) + .failed + .get_or_insert(error); + break; + } + } + let (panicked, failed) = drain.finish(); + if let Some(panic) = panicked { + std::panic::resume_unwind(panic); + } + match failed { + Some(error) => Err(error), + None => Ok(()), + } +} + +/// Times [`ordered_chunks`] ran serially because the pool was unavailable +/// or the caller was already on a pool thread. +#[cfg(test)] +static SERIAL_FALLBACKS: AtomicUsize = AtomicUsize::new(0); + +/// Reusable chunk buffers. A pipeline allocates at most its in-flight count +/// of them, each growing once to its largest chunk, instead of one fresh +/// allocation per chunk. That bound holds as long as consumers hand every +/// buffer back with [`BufferPool::give`] (the hasher thread of +/// [`spawn_hasher`] does); a buffer that is dropped instead is simply +/// allocated anew by the next producer. +pub(crate) struct BufferPool(Mutex>>); + +impl BufferPool { + pub(crate) fn new() -> Self { + Self(Mutex::new(Vec::new())) + } + + pub(crate) fn take(&self) -> Vec { + self.0 + .lock() + .unwrap_or_else(|e| e.into_inner()) + .pop() + .unwrap_or_default() + } + + pub(crate) fn give(&self, mut buffer: Vec) { + buffer.clear(); + self.0 + .lock() + .unwrap_or_else(|e| e.into_inner()) + .push(buffer); + } +} + +/// A SHA-256 state advanced on its own thread over the owned buffers sent +/// to it, in send order; each buffer goes back to `pool` once hashed. Drop +/// the sender, then join for the state. +pub(crate) fn spawn_hasher<'scope, 'env: 'scope>( + scope: &'scope Scope<'scope, 'env>, + pool: &'env BufferPool, + depth: usize, +) -> (SyncSender>, ScopedJoinHandle<'scope, Sha256>) { + let (send, receive) = sync_channel::>(depth); + let handle = scope.spawn(move || { + let mut hasher = Sha256::new(); + for buffer in receive { + hasher.update(&buffer); + pool.give(buffer); + } + hasher + }); + (send, handle) +} + +/// The error a pipeline reports when a helper thread it feeds has gone away. +pub(crate) fn stopped(what: &str) -> serde_json::Error { + serde_json::Error::io(std::io::Error::other(format!("{what} thread stopped"))) +} + +/// The bytes of `serde_json::to_writer(items)` as owned frames handed to +/// `consume` in order: `[`, each chunk (`,`-prefixed after the first), `]`. +/// Every element is still written by `serde_json::to_writer`. `consume` owns +/// each frame and returns it to `pool` when done, so the producers reuse it. +/// A slice no longer than one chunk arrives as a single frame. +pub(crate) fn write_array_frames( + items: &[T], + parallelism: Parallelism, + pool: &BufferPool, + mut consume: impl FnMut(Vec) -> Result<(), serde_json::Error>, +) -> Result<(), serde_json::Error> +where + T: Serialize + Sync, +{ + let chunk_len = parallelism.chunk_len.min(items.len()).max(1); + if parallelism.is_serial() || items.len() <= chunk_len { + let mut frame = pool.take(); + serde_json::to_writer(&mut frame, items)?; + return consume(frame); + } + let chunks = items.len().div_ceil(chunk_len); + let mut open = pool.take(); + open.push(b'['); + consume(open)?; + ordered_chunks( + chunks, + parallelism.workers, + |index| { + let chunk = &items[index * chunk_len..((index + 1) * chunk_len).min(items.len())]; + let mut buffer = pool.take(); + for (position, item) in chunk.iter().enumerate() { + if position > 0 || index > 0 { + buffer.push(b','); + } + serde_json::to_writer(&mut buffer, item)?; + } + Ok::<_, serde_json::Error>(buffer) + }, + |_, buffer| consume(buffer), + )?; + let mut close = pool.take(); + close.push(b']'); + consume(close) +} + +/// Write exactly the bytes of `serde_json::to_writer(writer, items)`. +/// +/// Serial parallelism, or a slice no longer than one chunk, is that call. +/// Otherwise the frames of [`write_array_frames`] reach `writer` in order. +pub(crate) fn write_array( + writer: &mut W, + items: &[T], + parallelism: Parallelism, +) -> Result<(), serde_json::Error> +where + T: Serialize + Sync, + W: Write, +{ + if parallelism.is_serial() || items.len() <= parallelism.chunk_len { + return serde_json::to_writer(writer, items); + } + let pool = BufferPool::new(); + write_array_frames(items, parallelism, &pool, |frame| { + writer.write_all(&frame).map_err(serde_json::Error::io)?; + pool.give(frame); + Ok(()) + }) +} + +/// A serializer for one struct value whose bytes are exactly `serde_json`'s +/// compact struct output, with a single named field's value bytes supplied +/// by `splice` instead of by the field's own `Serialize`. +/// +/// The struct's field order, names and `skip_serializing_if` decisions stay +/// with its derived `Serialize` implementation: this only writes the framing +/// (`{`, `,`, `"name":`, `}`) and hands every other field value, and every +/// field name, to `serde_json::to_writer` on the same writer. A value that +/// is not a struct is refused; nested values never reach this serializer. +pub(crate) struct StructSplice<'w, W, F> { + writer: &'w mut W, + field: &'static str, + splice: Option, + opening: &'static [u8], +} + +impl<'w, W, F> StructSplice<'w, W, F> +where + W: Write, + F: FnOnce(&mut W) -> Result<(), serde_json::Error>, +{ + /// A complete `{...}` object. + pub(crate) fn object(writer: &'w mut W, field: &'static str, splice: F) -> Self { + Self { + writer, + field, + splice: Some(splice), + opening: b"{", + } + } + + /// The remaining fields of an object whose opening brace and earlier + /// fields were already written: `,"name":...}` instead of `{"name":...}`, + /// exactly what `audit_hash::ContinueObject` produces. + pub(crate) fn continuing(writer: &'w mut W, field: &'static str, splice: F) -> Self { + Self { + writer, + field, + splice: Some(splice), + opening: b",", + } + } +} + +fn unsupported(what: &str) -> serde_json::Error { + serde::ser::Error::custom(format!( + "StructSplice serializes one struct; {what} is not a struct" + )) +} + +macro_rules! refuse { + ($($method:ident($($argument:ident: $type:ty),*) -> $what:literal;)*) => { + $(fn $method(self, $($argument: $type),*) -> Result<(), serde_json::Error> { + $(let _ = $argument;)* + Err(unsupported($what)) + })* + }; +} + +impl<'w, W, F> Serializer for StructSplice<'w, W, F> +where + W: Write, + F: FnOnce(&mut W) -> Result<(), serde_json::Error>, +{ + type Ok = (); + type Error = serde_json::Error; + type SerializeSeq = Impossible<(), serde_json::Error>; + type SerializeTuple = Impossible<(), serde_json::Error>; + type SerializeTupleStruct = Impossible<(), serde_json::Error>; + type SerializeTupleVariant = Impossible<(), serde_json::Error>; + type SerializeMap = Impossible<(), serde_json::Error>; + type SerializeStruct = SpliceFields<'w, W, F>; + type SerializeStructVariant = Impossible<(), serde_json::Error>; + + fn serialize_struct( + self, + _name: &'static str, + _len: usize, + ) -> Result { + self.writer + .write_all(self.opening) + .map_err(serde_json::Error::io)?; + Ok(SpliceFields { + writer: self.writer, + field: self.field, + splice: self.splice, + first: true, + }) + } + + refuse! { + serialize_bool(value: bool) -> "bool"; + serialize_i8(value: i8) -> "i8"; + serialize_i16(value: i16) -> "i16"; + serialize_i32(value: i32) -> "i32"; + serialize_i64(value: i64) -> "i64"; + serialize_u8(value: u8) -> "u8"; + serialize_u16(value: u16) -> "u16"; + serialize_u32(value: u32) -> "u32"; + serialize_u64(value: u64) -> "u64"; + serialize_f32(value: f32) -> "f32"; + serialize_f64(value: f64) -> "f64"; + serialize_char(value: char) -> "char"; + serialize_str(value: &str) -> "str"; + serialize_bytes(value: &[u8]) -> "bytes"; + serialize_none() -> "none"; + serialize_unit() -> "unit"; + serialize_unit_struct(name: &'static str) -> "unit struct"; + serialize_unit_variant(name: &'static str, index: u32, variant: &'static str) -> "unit variant"; + } + + fn serialize_some(self, _value: &T) -> Result<(), Self::Error> { + Err(unsupported("some")) + } + + fn serialize_newtype_struct( + self, + _name: &'static str, + _value: &T, + ) -> Result<(), Self::Error> { + Err(unsupported("newtype struct")) + } + + fn serialize_newtype_variant( + self, + _name: &'static str, + _index: u32, + _variant: &'static str, + _value: &T, + ) -> Result<(), Self::Error> { + Err(unsupported("newtype variant")) + } + + fn serialize_seq(self, _len: Option) -> Result { + Err(unsupported("seq")) + } + + fn serialize_tuple(self, _len: usize) -> Result { + Err(unsupported("tuple")) + } + + fn serialize_tuple_struct( + self, + _name: &'static str, + _len: usize, + ) -> Result { + Err(unsupported("tuple struct")) + } + + fn serialize_tuple_variant( + self, + _name: &'static str, + _index: u32, + _variant: &'static str, + _len: usize, + ) -> Result { + Err(unsupported("tuple variant")) + } + + fn serialize_map(self, _len: Option) -> Result { + Err(unsupported("map")) + } + + fn serialize_struct_variant( + self, + _name: &'static str, + _index: u32, + _variant: &'static str, + _len: usize, + ) -> Result { + Err(unsupported("struct variant")) + } +} + +pub(crate) struct SpliceFields<'w, W, F> { + writer: &'w mut W, + field: &'static str, + splice: Option, + first: bool, +} + +impl SerializeStruct for SpliceFields<'_, W, F> +where + W: Write, + F: FnOnce(&mut W) -> Result<(), serde_json::Error>, +{ + type Ok = (); + type Error = serde_json::Error; + + fn serialize_field( + &mut self, + key: &'static str, + value: &T, + ) -> Result<(), Self::Error> { + if !self.first { + self.writer.write_all(b",").map_err(serde_json::Error::io)?; + } + self.first = false; + serde_json::to_writer(&mut *self.writer, key)?; + self.writer.write_all(b":").map_err(serde_json::Error::io)?; + if key == self.field { + let splice = self + .splice + .take() + .ok_or_else(|| serde::ser::Error::custom("spliced field serialized twice"))?; + splice(&mut *self.writer) + } else { + serde_json::to_writer(&mut *self.writer, value) + } + } + + fn skip_field(&mut self, _key: &'static str) -> Result<(), Self::Error> { + Ok(()) + } + + fn end(self) -> Result { + self.writer.write_all(b"}").map_err(serde_json::Error::io) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use std::sync::atomic::{AtomicUsize, Ordering}; + + #[derive(Serialize)] + struct Row { + seq: u64, + text: String, + wide: u128, + #[serde(skip_serializing_if = "Option::is_none")] + note: Option, + } + + fn rows(count: usize) -> Vec { + (0..count) + .map(|index| Row { + seq: index as u64, + text: format!("r\"{index}\\\u{1F600}\n\u{0}"), + wide: if index % 3 == 0 { + u128::MAX - index as u128 + } else { + index as u128 + }, + note: (index % 2 == 0).then(|| "credit".into()), + }) + .collect() + } + + #[test] + fn write_array_matches_serde_json_for_every_chunking() { + for count in [0usize, 1, 2, 3, 7, 8, 9, 64, 65, 200] { + let items = rows(count); + let expected = serde_json::to_vec(&items).unwrap(); + for (workers, chunk_len) in [(1, 1), (2, 1), (3, 2), (4, 7), (8, 8), (5, 9), (2, 1000)] + { + let mut actual = Vec::new(); + write_array(&mut actual, &items, Parallelism::new(workers, chunk_len)).unwrap(); + assert_eq!( + actual, expected, + "count {count} workers {workers} chunk {chunk_len}" + ); + } + let mut serial = Vec::new(); + write_array(&mut serial, &items, Parallelism::serial()).unwrap(); + assert_eq!(serial, expected); + } + } + + #[test] + fn ordered_chunks_consumes_in_order_with_bounded_lookahead() { + let peak = AtomicUsize::new(0); + let inflight = AtomicUsize::new(0); + let mut seen = Vec::new(); + ordered_chunks( + 50, + 3, + |index| { + let now = inflight.fetch_add(1, Ordering::SeqCst) + 1; + peak.fetch_max(now, Ordering::SeqCst); + std::thread::sleep(std::time::Duration::from_micros((index % 5) as u64 * 50)); + Ok::<_, ()>(index * 10) + }, + |index, value| { + inflight.fetch_sub(1, Ordering::SeqCst); + seen.push((index, value)); + std::thread::sleep(std::time::Duration::from_micros(100)); + Ok(()) + }, + ) + .unwrap(); + assert_eq!(seen, (0..50).map(|i| (i, i * 10)).collect::>()); + assert!( + peak.load(Ordering::SeqCst) <= 6, + "peak {}", + peak.load(Ordering::SeqCst) + ); + } + + #[test] + fn ordered_chunks_reports_the_first_producer_error_and_stops() { + let produced = AtomicUsize::new(0); + let mut consumed = 0; + let error = ordered_chunks( + 1000, + 4, + |index| { + produced.fetch_add(1, Ordering::SeqCst); + if index == 5 { + Err(format!("chunk {index} failed")) + } else { + Ok(index) + } + }, + |_, _| { + consumed += 1; + Ok(()) + }, + ) + .unwrap_err(); + assert_eq!(error, "chunk 5 failed"); + assert!(consumed <= 5); + assert!(produced.load(Ordering::SeqCst) < 1000); + } + + #[test] + fn nested_ordered_chunks_runs_serially_on_a_pool_thread_and_completes() { + let before = SERIAL_FALLBACKS.load(Ordering::SeqCst); + let mut outer = Vec::new(); + ordered_chunks( + 40, + 4, + |index| { + // A producer that itself uses the ordered pipeline: it must + // not park the pool thread waiting for the pool. + let mut inner = Vec::new(); + ordered_chunks( + 5, + 4, + |j| Ok::<_, ()>(j * 10), + |_, v| { + inner.push(v); + Ok(()) + }, + )?; + Ok::<_, ()>((index, inner)) + }, + |_, value| { + outer.push(value); + Ok(()) + }, + ) + .unwrap(); + assert_eq!(outer.len(), 40); + for (index, (seen, inner)) in outer.iter().enumerate() { + assert_eq!(*seen, index); + assert_eq!(*inner, vec![0, 10, 20, 30, 40]); + } + assert!(SERIAL_FALLBACKS.load(Ordering::SeqCst) >= before + 40); + } + + #[test] + fn consumer_panic_unwinds_only_after_every_submitted_job_finished() { + for round in 0..20 { + let started = AtomicUsize::new(0); + let finished = AtomicUsize::new(0); + // A borrowed canary the jobs read; it must be intact whenever a + // job runs, i.e. no job may run after this frame is gone. + let canary = vec![0xA5u8; 4096]; + let result = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| { + ordered_chunks( + 64, + 8, + |index| { + started.fetch_add(1, Ordering::SeqCst); + std::thread::sleep(std::time::Duration::from_micros(150)); + assert!(canary.iter().all(|byte| *byte == 0xA5)); + finished.fetch_add(1, Ordering::SeqCst); + Ok::<_, ()>(index) + }, + |index, _| { + if index == round % 3 { + panic!("injected consumer panic"); + } + Ok(()) + }, + ) + })); + assert_eq!( + result.unwrap_err().downcast_ref::<&str>().copied(), + Some("injected consumer panic") + ); + assert_eq!( + started.load(Ordering::SeqCst), + finished.load(Ordering::SeqCst), + "round {round}: a job was still running after the unwind" + ); + let settled = finished.load(Ordering::SeqCst); + std::thread::sleep(std::time::Duration::from_millis(5)); + assert_eq!(finished.load(Ordering::SeqCst), settled); + } + let mut seen = Vec::new(); + ordered_chunks( + 16, + 4, + |i| Ok::<_, ()>(i * 3), + |i, v| { + seen.push((i, v)); + Ok(()) + }, + ) + .unwrap(); + assert_eq!(seen, (0..16).map(|i| (i, i * 3)).collect::>()); + } + + #[test] + fn builder_threads_are_fixed_once_the_pool_runs() { + ordered_chunks(3, 2, Ok::<_, ()>, |_, _| Ok(())).unwrap(); + let running = builder_threads(); + assert!(running >= 1); + assert_eq!(configure_builder_threads(running), Ok(())); + assert_eq!( + configure_builder_threads(running + 1), + Err(BuilderThreadsFixed { threads: running }) + ); + // The detected parallelism follows the running pool: one worker per + // pool thread (at most eight in flight), which is the serial path on + // a host whose pool has a single thread. + let detected = Parallelism::detect(); + assert_eq!(detected.workers(), running.min(Parallelism::MAX_WORKERS)); + assert_eq!(detected.is_serial(), running == 1); + } + + #[test] + fn ordered_chunks_reraises_a_producer_panic_after_draining() { + let finished = AtomicUsize::new(0); + let result = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| { + ordered_chunks( + 40, + 4, + |index| { + if index == 2 { + panic!("injected producer panic"); + } + std::thread::sleep(std::time::Duration::from_micros(200)); + finished.fetch_add(1, Ordering::SeqCst); + Ok::<_, ()>(index) + }, + |_, _| Ok(()), + ) + })); + let payload = result.unwrap_err(); + assert_eq!( + payload.downcast_ref::<&str>().copied(), + Some("injected producer panic") + ); + // Every job submitted before the panic ran to completion before the + // panic reached this thread; none is still running now. + let settled = finished.load(Ordering::SeqCst); + std::thread::sleep(std::time::Duration::from_millis(20)); + assert_eq!(finished.load(Ordering::SeqCst), settled); + assert!(settled < 40); + } + + #[test] + fn ordered_chunks_reports_a_consumer_error_and_stops() { + let produced = AtomicUsize::new(0); + let error = ordered_chunks( + 1000, + 4, + |index| { + produced.fetch_add(1, Ordering::SeqCst); + Ok::<_, String>(index) + }, + |index, _| { + if index == 3 { + Err("sink closed".to_string()) + } else { + Ok(()) + } + }, + ) + .unwrap_err(); + assert_eq!(error, "sink closed"); + assert!(produced.load(Ordering::SeqCst) < 1000); + } + + #[test] + fn frames_through_a_hasher_thread_give_the_serial_digest_with_a_bounded_pool() { + let items = rows(300); + let expected = Sha256::digest(serde_json::to_vec(&items).unwrap()); + let pool = BufferPool::new(); + let actual = std::thread::scope(|scope| { + let (send, hasher) = spawn_hasher(scope, &pool, 2); + write_array_frames(&items, Parallelism::new(3, 7), &pool, |frame| { + send.send(frame).map_err(|_| stopped("hasher")) + }) + .unwrap(); + drop(send); + hasher.join().unwrap().finalize() + }); + assert_eq!(actual, expected); + let pooled = pool.0.lock().unwrap().len(); + assert!(pooled <= 3 * 2 + 2 + 2, "pooled buffers {pooled}"); + } + + #[test] + fn write_array_propagates_writer_failures() { + struct Failed; + impl Write for Failed { + fn write(&mut self, _: &[u8]) -> std::io::Result { + Err(std::io::Error::other("injected writer failure")) + } + fn flush(&mut self) -> std::io::Result<()> { + Ok(()) + } + } + let error = write_array(&mut Failed, &rows(20), Parallelism::new(2, 3)).unwrap_err(); + assert!(error.is_io()); + } + + #[derive(Serialize)] + struct Outer { + first: u128, + #[serde(skip_serializing_if = "Option::is_none")] + missing: Option, + rows: Vec, + last: String, + } + + #[test] + fn struct_splice_matches_serde_json_and_continues_objects() { + let outer = Outer { + first: u128::MAX, + missing: None, + rows: rows(30), + last: "tail \"quoted\"".into(), + }; + let expected = serde_json::to_vec(&outer).unwrap(); + let mut actual = Vec::new(); + outer + .serialize(StructSplice::object(&mut actual, "rows", |writer| { + write_array(writer, &outer.rows, Parallelism::new(3, 4)) + })) + .unwrap(); + assert_eq!(actual, expected); + let mut continued = Vec::new(); + outer + .serialize(StructSplice::continuing(&mut continued, "rows", |writer| { + write_array(writer, &outer.rows, Parallelism::new(3, 4)) + })) + .unwrap(); + assert_eq!(continued[0], b','); + assert_eq!(&continued[1..], &expected[1..]); + } + + #[test] + fn struct_splice_refuses_non_struct_values() { + let mut out = Vec::new(); + let error = 7u8 + .serialize(StructSplice::object(&mut out, "rows", |_| Ok(()))) + .unwrap_err(); + assert!(error.to_string().contains("not a struct")); + let error = vec![1u8] + .serialize(StructSplice::object(&mut out, "rows", |_| Ok(()))) + .unwrap_err(); + assert!(error.to_string().contains("not a struct")); + } +} diff --git a/crates/qbit-prism/src/settlement.rs b/crates/qbit-prism/src/settlement.rs index 56c38d521..b7889482a 100644 --- a/crates/qbit-prism/src/settlement.rs +++ b/crates/qbit-prism/src/settlement.rs @@ -1144,10 +1144,10 @@ mod tests { assert_eq!(decision.fanout_recipient_count, 2_000); assert_eq!(decision.fanout_chunk_count, 2); assert_eq!(decision.coinbase_settlement_output_count, 3); - assert!(decision - .fanout_chunks + assert!(decision.fanout_chunks.iter().all(|chunk| chunk + .recipients .iter() - .all(|chunk| chunk.recipients.iter().all(|r| r.recipient_id != "pool-fee"))); + .all(|r| r.recipient_id != "pool-fee"))); } #[test] diff --git a/crates/qbit-prism/src/window.rs b/crates/qbit-prism/src/window.rs index 1f65aed11..17984a982 100644 --- a/crates/qbit-prism/src/window.rs +++ b/crates/qbit-prism/src/window.rs @@ -2,8 +2,8 @@ //! //! This is the Rust side of the #131 window-pipeline migration: a //! byte-identical reimplementation of the coordinator's -//! `IncrementalShareWindow` (`lab/prism/share_ledger.py`) proven against the -//! differential parity oracle in `tests/window_pipeline_parity.py`. Every +//! legacy `IncrementalShareWindow`, previously checked against a Python +//! differential parity oracle and retained here with native regression coverage. Every //! rule here mirrors a named invariant of the Python fold: eligibility on //! both timestamps, ascending `share_seq` with duplicate detection, duplicate //! `share_id` detection, non-positive difficulty rejection, the exact @@ -50,7 +50,7 @@ //! request -- and the coordinator folds that window in-process, where //! Python's unbounded integers produce the right answer; the daemon keeps //! its state and is never retired for it. [`prepare_window_out_of_range`] -//! is the classifier, and the parity oracle (`tests/window_pipeline_parity.py`) +//! is the classifier, and the historical differential parity oracle //! mirrors the same table as the `rust-daemon` adapter's declared domain. use crate::AcceptedShare; @@ -102,9 +102,7 @@ impl fmt::Display for WindowFoldError { let message = match self { WindowFoldError::NonPositiveWindowWeight => "window_weight must be positive", WindowFoldError::NonPositivePageSize => "page_size must be positive", - WindowFoldError::DuplicateShareSeq => { - "full payout window contains duplicate share_seq" - } + WindowFoldError::DuplicateShareSeq => "full payout window contains duplicate share_seq", WindowFoldError::DuplicateShareId => "full payout window contains duplicate share_id", WindowFoldError::NonPositiveDifficulty => { "full payout window contains non-positive difficulty" @@ -755,8 +753,7 @@ impl PayoutWindow { let mut expired_rows: usize = 0; let mut first_retained_page: usize = 0; while first_retained_page < pages.len() - && total_difficulty - pages[first_retained_page].total_difficulty - >= self.window_weight + && total_difficulty - pages[first_retained_page].total_difficulty >= self.window_weight { let page = &pages[first_retained_page]; total_difficulty -= page.total_difficulty; @@ -882,7 +879,7 @@ mod tests { job_issued_at_ms: job_ms, accepted_at_ms: accepted_ms, ntime: 1_700_000_000, - credit_policy: if seq % 4 == 0 { + credit_policy: if seq.is_multiple_of(4) { Some("stale-grace".to_string()) } else { None @@ -909,11 +906,7 @@ mod tests { fn crossing_row_is_retained() { // Difficulties (2, 3, 8) at weight 10: the final whole share crossing // window_weight (seq 2) is retained, total 11 > 10. - let records = vec![ - share(1, 2, 6, 7), - share(2, 3, 8, 9), - share(3, 8, 10, 10), - ]; + let records = vec![share(1, 2, 6, 7), share(2, 3, 8, 9), share(3, 8, 10, 10)]; let window = PayoutWindow::from_full_snapshot(records, 10, 10, 512).expect("fold"); assert_eq!(window.record_count(), 2); assert_eq!(window.total_difficulty, 11); @@ -963,14 +956,19 @@ mod tests { #[test] fn advance_matches_full_rebuild_and_reports_byte_surgery() { let snapshot: Vec = (1..=7) - .map(|seq| share(seq, [5, 1, 1, 2, 1, 3, 1][usize::try_from(seq).unwrap() - 1], 100 - i64::try_from(seq).unwrap(), 100 - i64::try_from(seq).unwrap())) + .map(|seq| { + share( + seq, + [5, 1, 1, 2, 1, 3, 1][usize::try_from(seq).unwrap() - 1], + 100 - i64::try_from(seq).unwrap(), + 100 - i64::try_from(seq).unwrap(), + ) + }) .collect(); - let window = - PayoutWindow::from_full_snapshot(snapshot.clone(), 100, 9, 3).expect("fold"); + let window = PayoutWindow::from_full_snapshot(snapshot.clone(), 100, 9, 3).expect("fold"); let old_items = window.canonical_items_bytes(); let delta = vec![share(9, 2, 101, 99), share(10, 4, 90, 101)]; - let (advanced, stats, byte_delta) = - window.advance(delta.clone(), 101).expect("advance"); + let (advanced, stats, byte_delta) = window.advance(delta.clone(), 101).expect("advance"); let mut union = snapshot; union.extend(delta); @@ -1505,8 +1503,12 @@ mod tests { let mut union = uniform_shares(4, 1); union.extend(delta); - let rebuilt = PayoutWindow::from_full_snapshot(union, 11, 1_000, page_size).expect("rebuild"); - assert_eq!(advanced.canonical_digest_hex(), rebuilt.canonical_digest_hex()); + let rebuilt = + PayoutWindow::from_full_snapshot(union, 11, 1_000, page_size).expect("rebuild"); + assert_eq!( + advanced.canonical_digest_hex(), + rebuilt.canonical_digest_hex() + ); let mut mirrored = old_items[byte_delta.retained_drop_bytes..].to_vec(); mirrored.extend_from_slice(&byte_delta.appended_items); assert_eq!(mirrored, advanced.canonical_items_bytes()); diff --git a/crates/qbit-prism/tests/adversarial_invariants.rs b/crates/qbit-prism/tests/adversarial_invariants.rs index ab1ce902c..e981a90fc 100644 --- a/crates/qbit-prism/tests/adversarial_invariants.rs +++ b/crates/qbit-prism/tests/adversarial_invariants.rs @@ -734,7 +734,7 @@ fn t7d_500_output_coinbase_under_max_block_weight() { #[test] fn t7e_coinbase_weight_bytes_are_reported_separately_from_entitlement_weight() { // The coordinator computes weight headroom as 2_000_000 - len(hex)//2 - // (prism_coordinator.py:921), assuming WSF=1 so weight == serialized + // used by the coordinator, assuming WSF=1 so weight == serialized // bytes. The Rust builder exposes total_weight as the sum of entitlement // WEIGHTS, not block weight, and exposes byte weight separately. let req = CoinbaseBuildRequest { diff --git a/crates/qbit-prism/tests/adversarial_settlement_cases.rs b/crates/qbit-prism/tests/adversarial_settlement_cases.rs index eeb74eb01..c19eba7c0 100644 --- a/crates/qbit-prism/tests/adversarial_settlement_cases.rs +++ b/crates/qbit-prism/tests/adversarial_settlement_cases.rs @@ -167,7 +167,10 @@ fn signed_carry_total_equals_prior_total_across_blocks() { let net_diff = 2_000_001u128; let prior_balances: Vec = Vec::new(); - for block_index in 0..4_u64 { + // The loop this case grew from broke after its first block, so it only ever + // checked block 0; the single case is kept as written. + let block_index = 0_u64; + { let base_ms = 10_000 + (block_index as i64 * 100); let shares = vec![ share( @@ -226,7 +229,6 @@ fn signed_carry_total_equals_prior_total_across_blocks() { prior_total, 0, "no failed block should mutate signed prior balances" ); - break; } } diff --git a/crates/qbit-prism/tests/audit_cli.rs b/crates/qbit-prism/tests/audit_cli.rs index 92238ee98..26c6741c4 100644 --- a/crates/qbit-prism/tests/audit_cli.rs +++ b/crates/qbit-prism/tests/audit_cli.rs @@ -2,6 +2,7 @@ use qbit_pool_builder::ManifestSigningKey; use qbit_prism::{ build_audit_bundle, canonical_audit_bundle_bytes, verify_audit_bundle, AcceptedShare, AuditBundle, CarryForwardBalance, CoinbaseOutputPolicy, FoundBlock, PayoutPolicy, + AUDIT_BUILDER_VERSION, }; use serde::Deserialize; use sha2::{Digest, Sha256}; @@ -207,7 +208,9 @@ fn canonicalizer_restores_typed_bytes_for_reordered_unicode_and_optional_input() let unicode_canonical = canonical_audit_bundle_bytes(&unicode_bundle).unwrap(); let unicode_output = canonicalize_cli_value(&reordered, "unicode"); assert_eq!(unicode_output, unicode_canonical); - assert!(unicode_output.windows("miner-é".len()).any(|window| window == "miner-é".as_bytes())); + assert!(unicode_output + .windows("miner-é".len()) + .any(|window| window == "miner-é".as_bytes())); } #[test] @@ -347,6 +350,51 @@ fn canonicalize_cli_emits_verifier_hash_bytes() { assert_eq!(reparsed, bundle); } +/// The CC5 fixture's canonical output under one builder version. +struct Cc5Golden { + builder_version: u16, + canonical_bytes_len: usize, + canonical_sha256: &'static str, + reward_manifest_sha256: &'static str, + payout_policy_manifest_sha256: &'static str, + coinbase_manifest_sha256: &'static str, + share_slice_digest: &'static str, + audit_commitment_root: &'static str, +} + +/// One entry per `AUDIT_BUILDER_VERSION` that has ever produced these bytes. +/// +/// A change that alters `canonical_audit_bundle_bytes` for the same inputs +/// bumps `AUDIT_BUILDER_VERSION` and **adds** the new digests here, next to +/// the old ones; a stored candidate or prepared job at an older version is +/// refused rather than rebuilt, so both outputs stay on the record and review +/// sees the change. Never edit an existing entry: an expected value that moved +/// under an unchanged version is a canonical-hash regression, not a golden to +/// refresh. +const CC5_GOLDENS: &[Cc5Golden] = &[Cc5Golden { + builder_version: 1, + canonical_bytes_len: 10_999, + canonical_sha256: "65b11e1b7e2025472fad2e4cd6b555eaba5eab2a4903e17179ba792d58780a4b", + reward_manifest_sha256: "14feb3360ba2d97faadf178151ca7c09bbb6a6e59e6c39a079e7d97986357ae1", + payout_policy_manifest_sha256: + "2db25eb6db270fb0e5ef9100158b2bfcca95ae6228717fb9779de47fbe11a668", + coinbase_manifest_sha256: "635e6133c760cbed7965b0475a273a82495141b4e3939a9908441baa532a0c39", + share_slice_digest: "fc39e87eaedeb6cb6442afbdf060ddc81a93846acc3ffdb47cf202c408c6a9d3", + audit_commitment_root: "492c8e5f83049d3f6b04a175a531f130a8c52c2fa944b06741f442587f365d3a", +}]; + +fn cc5_golden() -> &'static Cc5Golden { + CC5_GOLDENS + .iter() + .find(|golden| golden.builder_version == AUDIT_BUILDER_VERSION) + .unwrap_or_else(|| { + panic!( + "no CC5 golden entry for AUDIT_BUILDER_VERSION {AUDIT_BUILDER_VERSION}; \ + add the new builder's digests to CC5_GOLDENS, keeping every older entry" + ) + }) +} + #[test] fn build_audit_bundle_cli_canonical_output_matches_cc5_golden_bytes() { let fixture: Fixture = serde_json::from_str(include_str!( @@ -492,30 +540,31 @@ fn build_audit_bundle_cli_canonical_output_matches_cc5_golden_bytes() { .contains("qbit-prism-build-phase-metrics")); let report = verify_audit_bundle(&emitted_bundle, &ledger_public_key_hex()).unwrap(); - assert_eq!(output.stdout.len(), 10_999); + let golden = cc5_golden(); + assert_eq!(output.stdout.len(), golden.canonical_bytes_len); assert_eq!( hex::encode(Sha256::digest(&output.stdout)), - "65b11e1b7e2025472fad2e4cd6b555eaba5eab2a4903e17179ba792d58780a4b" + golden.canonical_sha256 ); assert_eq!( report.reward_manifest_sha256_hex, - "14feb3360ba2d97faadf178151ca7c09bbb6a6e59e6c39a079e7d97986357ae1" + golden.reward_manifest_sha256 ); assert_eq!( report.payout_policy_manifest_sha256_hex, - "2db25eb6db270fb0e5ef9100158b2bfcca95ae6228717fb9779de47fbe11a668" + golden.payout_policy_manifest_sha256 ); assert_eq!( report.coinbase_manifest_sha256_hex, - "635e6133c760cbed7965b0475a273a82495141b4e3939a9908441baa532a0c39" + golden.coinbase_manifest_sha256 ); assert_eq!( emitted_bundle.reward_manifest.share_slice_digest_hex, - "fc39e87eaedeb6cb6442afbdf060ddc81a93846acc3ffdb47cf202c408c6a9d3" + golden.share_slice_digest ); assert_eq!( report.audit_commitment_root_hex, - "492c8e5f83049d3f6b04a175a531f130a8c52c2fa944b06741f442587f365d3a" + golden.audit_commitment_root ); assert_eq!(report.onchain_output_count, 3); assert_eq!(report.accrued_account_count, 3); @@ -1208,8 +1257,7 @@ fn build_audit_bundle_serve_mode_caches_parsed_windows() { let one_shot_input = { let mut input = build_fields.clone(); input["shares"] = serde_json::json!([]); - input["compact_share_identities"] = - serde_json::Value::Array(compact_identities.clone()); + input["compact_share_identities"] = serde_json::Value::Array(compact_identities.clone()); input["compact_shares"] = serde_json::Value::Array(compact_shares.clone()); input }; @@ -1234,8 +1282,7 @@ fn build_audit_bundle_serve_mode_caches_parsed_windows() { "stderr: {}", String::from_utf8_lossy(&one_shot.stderr) ); - let expected_summary: serde_json::Value = - serde_json::from_slice(&one_shot.stdout).unwrap(); + let expected_summary: serde_json::Value = serde_json::from_slice(&one_shot.stdout).unwrap(); let mut daemon = Command::new(env!("CARGO_BIN_EXE_qbit-prism-build-audit-bundle")) .arg("--serve") @@ -1260,8 +1307,7 @@ fn build_audit_bundle_serve_mode_caches_parsed_windows() { assert_eq!(handshake["tool"], "qbit-prism-build-audit-bundle"); let mut request_with_window = build_fields.clone(); - request_with_window["window_key"] = - serde_json::json!({"share_snapshot_sha256": "window-a"}); + request_with_window["window_key"] = serde_json::json!({"share_snapshot_sha256": "window-a"}); request_with_window["compact_share_identities"] = serde_json::Value::Array(compact_identities.clone()); request_with_window["compact_shares"] = serde_json::Value::Array(compact_shares.clone()); @@ -1299,9 +1345,13 @@ fn build_audit_bundle_serve_mode_caches_parsed_windows() { assert_eq!(hit["window_cache"]["hits"], 1); let mut request_unknown = build_fields.clone(); - request_unknown["window_key"] = - serde_json::json!({"share_snapshot_sha256": "window-unknown"}); - writeln!(stdin, "{}", serde_json::to_string(&request_unknown).unwrap()).unwrap(); + request_unknown["window_key"] = serde_json::json!({"share_snapshot_sha256": "window-unknown"}); + writeln!( + stdin, + "{}", + serde_json::to_string(&request_unknown).unwrap() + ) + .unwrap(); line.clear(); stdout.read_line(&mut line).unwrap(); let missing: serde_json::Value = serde_json::from_str(&line).unwrap(); @@ -1661,7 +1711,10 @@ fn prepare_window_advance_ignores_the_append_invalidation_epoch_tag() { assert_eq!(full["ok"], true, "full response: {line}"); assert_eq!(full["record_count"], 3); let base_digest = full["share_snapshot_sha256"].as_str().unwrap().to_string(); - drain_section(&mut stdout, full["window_items_len"].as_u64().unwrap() as usize); + drain_section( + &mut stdout, + full["window_items_len"].as_u64().unwrap() as usize, + ); // A build at a newer epoch must not evict the prepared window either. let build = serde_json::json!({ diff --git a/crates/qbit-prism/tests/audit_parts.rs b/crates/qbit-prism/tests/audit_parts.rs new file mode 100644 index 000000000..d7240cf50 --- /dev/null +++ b/crates/qbit-prism/tests/audit_parts.rs @@ -0,0 +1,940 @@ +//! Equivalence between the owned `build_audit_bundle*` entry points and the +//! borrowed parts API (`build_audit_bundle_body*`, `verify_audit_parts*`, +//! `canonical_audit_bundle_bytes_from_parts`). + +use std::sync::Arc; + +use qbit_pool_builder::ManifestSigningKey; +use qbit_prism::{ + build_audit_bundle, build_audit_bundle_body, build_audit_bundle_body_with_coinbase_options, + build_audit_bundle_body_with_coinbase_script_sig_suffix, + build_audit_bundle_body_with_ctv_settlement_options, build_audit_bundle_with_coinbase_options, + build_audit_bundle_with_coinbase_script_sig_suffix, + build_audit_bundle_with_ctv_settlement_options, canonical_audit_bundle_bytes, + canonical_audit_bundle_bytes_from_parts, prior_balances_digest, verify_audit_bundle, + verify_audit_bundle_against_coinbase_tx_hex, + verify_audit_bundle_against_coinbase_tx_hex_and_expected_coinbase_value, verify_audit_parts, + verify_audit_parts_against_coinbase_tx_hex, + verify_audit_parts_against_coinbase_tx_hex_and_expected_coinbase_value, + write_canonical_audit_bundle_from_parts, AcceptedShare, AuditBundle, AuditBundleBody, + CarryForwardBalance, FanoutFeeRatePolicy, FoundBlock, PayoutPolicy, PoolFeePolicy, PrismError, + SettlementModeConfig, AUDIT_BUNDLE_SCHEMA_V1, AUDIT_BUNDLE_SCHEMA_V1_1, +}; +use sha2::{Digest, Sha256}; + +// Frozen derived serializer from base 2e81ca3c, independent of the production +// manual serializer and split framing. Do not update it with the implementation. +mod legacy { + use super::*; + use qbit_pool_builder::SignedPayoutManifest; + use qbit_prism::{ + CtvFanoutManifestSet, LedgerWindowAttestation, PayoutPolicyManifest, PrismRewardManifest, + SettlementModeDecision, + }; + use serde::Serialize; + #[derive(Clone, Copy, Serialize)] + #[serde(rename = "AuditBundle")] + pub(super) struct AuditBundleRef<'a> { + schema: &'a String, + shares: &'a [AcceptedShare], + found_block: &'a FoundBlock, + prior_balances: &'a Vec, + payout_policy: &'a PayoutPolicy, + #[serde(skip_serializing_if = "ref_option_is_none")] + coinbase_script_sig_suffix_hex: &'a Option, + #[serde(skip_serializing_if = "ref_vec_is_empty")] + witness_merkle_leaves_hex: &'a Vec, + #[serde(skip_serializing_if = "ref_vec_is_empty")] + audit_commitment_leaves_hex: &'a Vec, + #[serde(skip_serializing_if = "ref_option_is_none")] + audit_commitment_root_hex: &'a Option, + ledger_window_attestation: &'a LedgerWindowAttestation, + reward_manifest: &'a PrismRewardManifest, + payout_policy_manifest: &'a PayoutPolicyManifest, + #[serde(skip_serializing_if = "ref_option_is_none")] + settlement_mode_decision: &'a Option, + #[serde(skip_serializing_if = "ref_option_is_none")] + ctv_fanout_fee_policy: &'a Option, + #[serde(skip_serializing_if = "ref_option_is_none")] + ctv_fanout_manifest_set: &'a Option, + signed_coinbase_manifest: &'a SignedPayoutManifest, + } + + impl<'a> AuditBundleRef<'a> { + pub(super) fn from_parts(body: &'a AuditBundleBody, shares: &'a [AcceptedShare]) -> Self { + let AuditBundleBody { + schema, + found_block, + prior_balances, + payout_policy, + coinbase_script_sig_suffix_hex, + witness_merkle_leaves_hex, + audit_commitment_leaves_hex, + audit_commitment_root_hex, + ledger_window_attestation, + reward_manifest, + payout_policy_manifest, + settlement_mode_decision, + ctv_fanout_fee_policy, + ctv_fanout_manifest_set, + signed_coinbase_manifest, + } = body; + Self { + schema, + shares, + found_block, + prior_balances, + payout_policy, + coinbase_script_sig_suffix_hex, + witness_merkle_leaves_hex, + audit_commitment_leaves_hex, + audit_commitment_root_hex, + ledger_window_attestation, + reward_manifest, + payout_policy_manifest, + settlement_mode_decision, + ctv_fanout_fee_policy, + ctv_fanout_manifest_set, + signed_coinbase_manifest, + } + } + + fn write_canonical(&self, writer: W) -> Result<(), serde_json::Error> { + serde_json::to_writer(writer, self) + } + + pub(super) fn canonical_bytes(&self) -> Result, serde_json::Error> { + let mut bytes = Vec::new(); + self.write_canonical(&mut bytes)?; + Ok(bytes) + } + } + + fn ref_option_is_none(value: &&Option) -> bool { + value.is_none() + } + + fn ref_vec_is_empty(value: &&Vec) -> bool { + value.is_empty() + } +} + +const WINDOW_SHARES: u64 = 400; +const MINERS: u64 = 7; + +/// Compare the emitted bytes without keeping a second large canonical Vec. +struct CompareBytes<'a>(&'a [u8]); +impl std::io::Write for CompareBytes<'_> { + fn write(&mut self, bytes: &[u8]) -> std::io::Result { + assert_eq!(&self.0[..bytes.len()], bytes); + self.0 = &self.0[bytes.len()..]; + Ok(bytes.len()) + } + fn flush(&mut self) -> std::io::Result<()> { + Ok(()) + } +} + +fn check_large_canonical_window(count: usize, credit: bool) { + let seed = window_shares(None).remove(1); + let shares: Vec<_> = (0..count) + .map(|index| { + let mut share = seed.clone(); + share.share_seq = index as u64 + 1; + share.share_id = format!("share-\"\\\n\té-{index}"); + share.job_id = "job-\"\\\r\té".into(); + share.share_difficulty = if index == 0 { + u128::from(u64::MAX) + 12_345 + } else { + 20 + }; + share.network_difficulty = u128::MAX; + share.credit_policy = credit.then(|| "stale-grace".into()); + share + }) + .collect(); + let mut block = found_block(); + block.network_difficulty = (count.max(1) as u128 * 20) / 8; + // Empty-window serialization is defined even though the builder correctly + // rejects an empty payout. Runtime refresh represents that case with None. + let one = [seed]; + let body = build_audit_bundle_body( + if shares.is_empty() { &one } else { &shares }, + block, + prior_balances(), + PayoutPolicy::day_one_default(), + &manifest_signing_key(), + &ledger_signing_key(), + ) + .unwrap(); + if count > 0 { + assert_eq!(body.reward_manifest.shares.len(), count); + assert!( + body.reward_manifest + .shares + .last() + .unwrap() + .counted_difficulty + < shares[0].share_difficulty + ); + } + let expected = legacy::AuditBundleRef::from_parts(&body, &shares) + .canonical_bytes() + .unwrap(); + let mut compare = CompareBytes(&expected); + write_canonical_audit_bundle_from_parts(&mut compare, &body, &shares).unwrap(); + assert!(compare.0.is_empty()); + assert_eq!( + qbit_prism::CanonicalAuditHashPrefix::new(&shares) + .unwrap() + .finish(&body) + .unwrap(), + sha256_hex(&expected) + ); + drop(expected); + + // Independent original leaf recipe: tagged concatenation of the two + // SHA256 digests over the legacy owned canonical byte APIs. + let mut leaf = Sha256::new(); + leaf.update(qbit_prism::PRISM_AUDIT_COMMITMENT_LEAF_TAG.as_bytes()); + leaf.update(Sha256::digest( + qbit_prism::canonical_reward_manifest_bytes(&body.reward_manifest).unwrap(), + )); + leaf.update(Sha256::digest( + qbit_prism::canonical_payout_policy_manifest_bytes(&body.payout_policy_manifest).unwrap(), + )); + assert_eq!( + qbit_prism::prism_audit_commitment_leaf_hex( + &body.reward_manifest, + &body.payout_policy_manifest + ) + .unwrap(), + hex::encode(leaf.finalize()) + ); +} + +#[test] +fn pipeline_matches_frozen_serializer_empty_one_and_multiple_pages() { + check_large_canonical_window(0, false); + for count in [1, 8193] { + for credit in [false, true] { + check_large_canonical_window(count, credit); + } + } +} + +#[test] +#[ignore = "400k canonical byte and commitment proof; run under the heavy test lock"] +fn pipeline_matches_frozen_serializer_400k() { + for credit in [false, true] { + check_large_canonical_window(400_000, credit); + } +} + +/// sha256 and length of each case's canonical bytes, recorded from the owned +/// builders at 3.x.x 1398bbc1 (before the parts API existed). Both paths must +/// still produce exactly these bytes. +/// +/// These constants are the real guard against byte drift: the owned wrappers +/// now build through the parts API, so owned-equals-parts assertions are +/// tautological by construction. They were built by these fixtures against +/// 1398bbc1's owned builders; never re-record from a build that contains the +/// parts API. +const BASE_CANONICAL: [(&str, &str, usize); 6] = [ + ( + "build_audit_bundle/v1", + "937cb8ac3c00c94540e10e17fb3697ed6efb08d25bae3fa73a414c07c46aeed1", + 240_707, + ), + ( + "build_audit_bundle/v1.1+pool-fee", + "bb4cedab4b4923472ca3ab348a47d3b80a80f0f373537ffc47b4331319b3873f", + 243_744, + ), + ( + "script_sig_suffix/v1", + "974a3743e418f4085a323403b4e738b45370ab856b9f54a4ae00810db5e467b2", + 240_775, + ), + ( + "coinbase_options/v1.1+suffix+witness", + "3b65dee47f30a3795660fc1d1e08f70709fae3cb6598af78c36996e1f66fa560", + 243_101, + ), + ( + "ctv_settlement/v1+fee+suffix+witness", + "2bbec9d9dea5fdb98509417a8a57228ea71b682c257e7589cdbb4960bed2e554", + 249_975, + ), + ( + "ctv_settlement/v1.1", + "d4398423dce8429e58508a05773e361e62bb41c08bbaf00ad51e106addac2613", + 250_890, + ), +]; + +fn manifest_signing_key() -> ManifestSigningKey { + ManifestSigningKey::from_seed_hex(&"42".repeat(32)).unwrap() +} + +fn ledger_signing_key() -> ManifestSigningKey { + ManifestSigningKey::from_seed_hex(&"43".repeat(32)).unwrap() +} + +fn ledger_public_key_hex() -> String { + ledger_signing_key().public_key_hex() +} + +fn sha256_hex(bytes: &[u8]) -> String { + hex::encode(Sha256::digest(bytes)) +} + +/// A few hundred shares from several miners. The oldest share's difficulty +/// exceeds `u64::MAX`, so it is the partially counted window boundary and its +/// `u128` values exercise arbitrary-precision JSON. `credit_policy_every` +/// marks every n-th share `stale-grace`, which selects the v1.1 schema. +fn window_shares(credit_policy_every: Option) -> Vec { + (1..=WINDOW_SHARES) + .map(|share_seq| { + let miner = share_seq % MINERS; + let share_difficulty = if share_seq == 1 { + u128::from(u64::MAX) + 12_345 + } else { + u128::from((miner + 1) * (miner + 1) * (share_seq % 5 + 1)) + }; + let job_issued_at_ms = 1_000 + i64::try_from(share_seq).unwrap(); + AcceptedShare { + share_seq, + share_id: format!("share-{share_seq}"), + miner_id: format!("miner-{miner}"), + order_key: format!("{miner:02}"), + p2mr_program_hex: hex::encode([u8::try_from(miner + 1).unwrap(); 32]), + share_difficulty, + network_difficulty: 5_000, + template_height: 100, + job_id: format!("job-{job_issued_at_ms}"), + job_issued_at_ms, + accepted_at_ms: job_issued_at_ms, + ntime: 1_800_000_000, + credit_policy: credit_policy_every + .filter(|every| share_seq % every == 0) + .map(|_| "stale-grace".to_string()), + } + }) + .collect() +} + +fn found_block() -> FoundBlock { + FoundBlock { + block_height: 101, + coinbase_value_sats: 500_000_000, + network_difficulty: 5_000, + anchor_job_issued_at_ms: 1_000 + i64::try_from(WINDOW_SHARES).unwrap(), + } +} + +fn prior_balances() -> Vec { + vec![ + CarryForwardBalance { + recipient_id: "miner-2".to_string(), + order_key: "02".to_string(), + p2mr_program_hex: hex::encode([3_u8; 32]), + balance_sats: 4_800, + }, + CarryForwardBalance { + recipient_id: "miner-departed".to_string(), + order_key: "99".to_string(), + p2mr_program_hex: hex::encode([0x99_u8; 32]), + balance_sats: 12_000, + }, + ] +} + +fn pool_fee_payout_policy() -> PayoutPolicy { + PayoutPolicy { + pool_fee_policy: Some(PoolFeePolicy { + fee_bps: 200, + recipient_id: "pool-fee".to_string(), + order_key: "ff".to_string(), + p2mr_program_hex: hex::encode([0xee_u8; 32]), + }), + ..PayoutPolicy::day_one_default() + } +} + +fn witness_leaves() -> Vec { + vec!["11".repeat(32), "22".repeat(32)] +} + +fn ctv_config() -> SettlementModeConfig { + SettlementModeConfig { + max_coinbase_settlement_outputs: 4, + max_direct_coinbase_outputs: 2, + max_fanout_recipients_per_transaction: 3, + reserved_coinbase_outputs: 0, + } +} + +#[derive(Clone)] +enum Shape { + Plain, + ScriptSigSuffix(Option), + CoinbaseOptions { + suffix: Option, + witness: Vec, + }, + CtvSettlement { + direct_floor_sats: u64, + config: SettlementModeConfig, + fee_policy: Option, + suffix: Option, + witness: Vec, + }, +} + +#[derive(Clone)] +struct Case { + name: &'static str, + shares: Vec, + prior_balances: Vec, + payout_policy: PayoutPolicy, + shape: Shape, +} + +fn cases() -> Vec { + vec![ + Case { + name: "build_audit_bundle/v1", + shares: window_shares(None), + prior_balances: prior_balances(), + payout_policy: PayoutPolicy::day_one_default(), + shape: Shape::Plain, + }, + Case { + name: "build_audit_bundle/v1.1+pool-fee", + shares: window_shares(Some(13)), + prior_balances: prior_balances(), + payout_policy: pool_fee_payout_policy(), + shape: Shape::Plain, + }, + Case { + name: "script_sig_suffix/v1", + shares: window_shares(None), + prior_balances: prior_balances(), + payout_policy: PayoutPolicy::day_one_default(), + shape: Shape::ScriptSigSuffix(Some("aaaaaaaa".to_string())), + }, + Case { + name: "coinbase_options/v1.1+suffix+witness", + shares: window_shares(Some(11)), + prior_balances: prior_balances(), + payout_policy: PayoutPolicy::day_one_default(), + shape: Shape::CoinbaseOptions { + suffix: Some("0badc0de".to_string()), + witness: witness_leaves(), + }, + }, + Case { + name: "ctv_settlement/v1+fee+suffix+witness", + shares: window_shares(None), + prior_balances: prior_balances(), + payout_policy: PayoutPolicy::day_one_default(), + shape: Shape::CtvSettlement { + direct_floor_sats: 60_000_000, + config: ctv_config(), + fee_policy: Some(FanoutFeeRatePolicy::new(1_000, 12_000)), + suffix: Some("aaaaaaaa".to_string()), + witness: witness_leaves(), + }, + }, + Case { + name: "ctv_settlement/v1.1", + shares: window_shares(Some(17)), + prior_balances: prior_balances(), + payout_policy: PayoutPolicy::day_one_default(), + shape: Shape::CtvSettlement { + direct_floor_sats: 60_000_000, + config: ctv_config(), + fee_policy: None, + suffix: None, + witness: Vec::new(), + }, + }, + ] +} + +/// The pre-existing owned entry point for the case's builder shape. +fn build_owned(case: Case) -> AuditBundle { + let Case { + shares, + prior_balances, + payout_policy, + shape, + .. + } = case; + let coinbase_key = manifest_signing_key(); + let ledger_key = ledger_signing_key(); + match shape { + Shape::Plain => build_audit_bundle( + shares, + found_block(), + prior_balances, + payout_policy, + &coinbase_key, + &ledger_key, + ), + Shape::ScriptSigSuffix(suffix) => build_audit_bundle_with_coinbase_script_sig_suffix( + shares, + found_block(), + prior_balances, + payout_policy, + suffix, + &coinbase_key, + &ledger_key, + ), + Shape::CoinbaseOptions { suffix, witness } => build_audit_bundle_with_coinbase_options( + shares, + found_block(), + prior_balances, + payout_policy, + suffix, + witness, + &coinbase_key, + &ledger_key, + ), + Shape::CtvSettlement { + direct_floor_sats, + config, + fee_policy, + suffix, + witness, + } => build_audit_bundle_with_ctv_settlement_options( + shares, + found_block(), + prior_balances, + payout_policy, + direct_floor_sats, + config, + fee_policy, + suffix, + witness, + &coinbase_key, + &ledger_key, + ), + } + .unwrap() +} + +/// The borrowing entry point for the same shape, lending `shares`. +fn build_borrowed(case: &Case, shares: &[AcceptedShare]) -> AuditBundleBody { + let coinbase_key = manifest_signing_key(); + let ledger_key = ledger_signing_key(); + let prior_balances = case.prior_balances.clone(); + let payout_policy = case.payout_policy.clone(); + match case.shape.clone() { + Shape::Plain => build_audit_bundle_body( + shares, + found_block(), + prior_balances, + payout_policy, + &coinbase_key, + &ledger_key, + ), + Shape::ScriptSigSuffix(suffix) => build_audit_bundle_body_with_coinbase_script_sig_suffix( + shares, + found_block(), + prior_balances, + payout_policy, + suffix, + &coinbase_key, + &ledger_key, + ), + Shape::CoinbaseOptions { suffix, witness } => { + build_audit_bundle_body_with_coinbase_options( + shares, + found_block(), + prior_balances, + payout_policy, + suffix, + witness, + &coinbase_key, + &ledger_key, + ) + } + Shape::CtvSettlement { + direct_floor_sats, + config, + fee_policy, + suffix, + witness, + } => build_audit_bundle_body_with_ctv_settlement_options( + shares, + found_block(), + prior_balances, + payout_policy, + direct_floor_sats, + config, + fee_policy, + suffix, + witness, + &coinbase_key, + &ledger_key, + ), + } + .unwrap() +} + +#[test] +fn fixture_covers_the_branches_the_builders_take() { + let cases = cases(); + assert_eq!(cases.len(), BASE_CANONICAL.len()); + let bundles = cases.into_iter().map(build_owned).collect::>(); + let schemas = bundles + .iter() + .map(|bundle| bundle.schema.as_str()) + .collect::>(); + assert!(schemas.contains(&AUDIT_BUNDLE_SCHEMA_V1)); + assert!(schemas.contains(&AUDIT_BUNDLE_SCHEMA_V1_1)); + for bundle in &bundles { + let miners = bundle + .reward_manifest + .shares + .iter() + .map(|share| share.miner_id.as_str()) + .collect::>(); + assert_eq!(miners.len(), usize::try_from(MINERS).unwrap()); + assert!(bundle.shares.len() >= 300); + assert!(bundle + .shares + .iter() + .any(|share| share.share_difficulty > u128::from(u64::MAX))); + } + assert!(bundles + .iter() + .any(|bundle| bundle.payout_policy.pool_fee_policy.is_some())); + assert!(bundles + .iter() + .any(|bundle| bundle.coinbase_script_sig_suffix_hex.is_some())); + assert!(bundles + .iter() + .any(|bundle| !bundle.witness_merkle_leaves_hex.is_empty())); + assert!(bundles.iter().any(|bundle| bundle + .ctv_fanout_manifest_set + .as_ref() + .is_some_and(|set| set.manifests.len() > 1) + && bundle.ctv_fanout_fee_policy.is_some())); + assert!(bundles + .iter() + .any(|bundle| bundle.ctv_fanout_manifest_set.is_some() + && bundle.ctv_fanout_fee_policy.is_none())); +} + +/// For every builder shape: owned-path bytes, borrowed-parts bytes and +/// `into_parts` -> `into_bundle` bytes are identical and equal the bytes the +/// owned builders produced before the parts API existed; the sha256 values +/// and verification reports agree across both verifiers. +#[test] +fn parts_path_matches_owned_path_byte_for_byte() { + let ledger_key = ledger_public_key_hex(); + for (case, (name, base_sha256, base_len)) in cases().into_iter().zip(BASE_CANONICAL) { + assert_eq!(case.name, name); + + let owned = build_owned(case.clone()); + let owned_bytes = canonical_audit_bundle_bytes(&owned).unwrap(); + assert_eq!(owned_bytes.len(), base_len, "{name}: length drifted"); + assert_eq!( + sha256_hex(&owned_bytes), + base_sha256, + "{name}: bytes drifted" + ); + assert_eq!(serde_json::to_vec(&owned).unwrap(), owned_bytes, "{name}"); + + let body = build_borrowed(&case, &case.shares); + let parts_bytes = canonical_audit_bundle_bytes_from_parts(&body, &case.shares).unwrap(); + let legacy_bytes = legacy::AuditBundleRef::from_parts(&body, &case.shares) + .canonical_bytes() + .unwrap(); + assert_eq!( + parts_bytes, legacy_bytes, + "{name}: frozen base serializer differs" + ); + assert_eq!(parts_bytes, owned_bytes, "{name}: parts bytes differ"); + let mut streamed = Vec::new(); + write_canonical_audit_bundle_from_parts(&mut streamed, &body, &case.shares).unwrap(); + assert_eq!(streamed, owned_bytes, "{name}: streamed bytes differ"); + assert_eq!( + qbit_prism::CanonicalAuditHashPrefix::new(&case.shares) + .unwrap() + .finish(&body) + .unwrap(), + base_sha256, + "{name}: split audit hash differs from pre-parts legacy digest" + ); + + let (split_body, split_shares) = owned.clone().into_parts(); + assert_eq!(split_body, body, "{name}"); + assert_eq!(split_shares, case.shares, "{name}"); + let reassembled = split_body.into_bundle(split_shares); + assert_eq!(reassembled, owned, "{name}"); + assert_eq!( + canonical_audit_bundle_bytes(&reassembled).unwrap(), + owned_bytes, + "{name}: reassembled bytes differ" + ); + assert_eq!( + body.clone().into_bundle(case.shares.clone()), + owned, + "{name}" + ); + + let decoded: AuditBundle = serde_json::from_slice(&parts_bytes).unwrap(); + assert_eq!(decoded, owned, "{name}: canonical bytes do not decode"); + + let bundle_report = verify_audit_bundle(&owned, &ledger_key).unwrap(); + let parts_report = verify_audit_parts(&body, &case.shares, &ledger_key).unwrap(); + assert_eq!(parts_report, bundle_report, "{name}: reports differ"); + assert_eq!(bundle_report.audit_bundle_sha256_hex, base_sha256, "{name}"); + + let coinbase_tx_hex = bundle_report.coinbase_tx_hex.to_ascii_uppercase(); + assert_eq!( + verify_audit_parts_against_coinbase_tx_hex( + &body, + &case.shares, + &coinbase_tx_hex, + &ledger_key + ) + .unwrap(), + verify_audit_bundle_against_coinbase_tx_hex(&owned, &coinbase_tx_hex, &ledger_key) + .unwrap(), + "{name}" + ); + let coinbase_value_sats = found_block().coinbase_value_sats; + assert_eq!( + verify_audit_parts_against_coinbase_tx_hex_and_expected_coinbase_value( + &body, + &case.shares, + &coinbase_tx_hex, + &ledger_key, + coinbase_value_sats, + ) + .unwrap(), + verify_audit_bundle_against_coinbase_tx_hex_and_expected_coinbase_value( + &owned, + &coinbase_tx_hex, + &ledger_key, + coinbase_value_sats, + ) + .unwrap(), + "{name}" + ); + assert!(matches!( + verify_audit_parts_against_coinbase_tx_hex(&body, &case.shares, "00", &ledger_key), + Err(PrismError::AuditCoinbaseTxMismatch) + )); + assert!(matches!( + verify_audit_parts_against_coinbase_tx_hex_and_expected_coinbase_value( + &body, + &case.shares, + &coinbase_tx_hex, + &ledger_key, + coinbase_value_sats + 1, + ), + Err(PrismError::ExpectedCoinbaseValueMismatch { .. }) + )); + } +} + +#[test] +fn parts_verifier_reads_the_lent_window() { + let ledger_key = ledger_public_key_hex(); + let cases = cases(); + + let v1 = &cases[0]; + let body = build_borrowed(v1, &v1.shares); + let mut reassigned = v1.shares.clone(); + reassigned[200].miner_id = "miner-interloper".to_string(); + assert!(matches!( + verify_audit_parts(&body, &reassigned, &ledger_key), + Err(PrismError::AuditMismatch { + artifact: "reward_manifest" + }) + )); + + let v1_1 = &cases[1]; + let body = build_borrowed(v1_1, &v1_1.shares); + assert_eq!(body.schema, AUDIT_BUNDLE_SCHEMA_V1_1); + let mut stripped = v1_1.shares.clone(); + for share in &mut stripped { + share.credit_policy = None; + } + assert!(matches!( + verify_audit_parts(&body, &stripped, &ledger_key), + Err(PrismError::AuditMismatch { artifact: "schema" }) + )); +} + +/// sha256 and length of the canonical bytes of the `build_audit_bundle/v1` +/// case with every optional top-level field absent: after building, +/// `audit_commitment_leaves_hex` is cleared and `audit_commitment_root_hex` +/// set to `None` (the builder already leaves the suffix, witness leaves, +/// settlement decision and CTV fields absent). Recorded by running this exact +/// fixture and mutation as a test in a `git archive` copy of 3.x.x 1398bbc1, +/// where `AuditBundle` still derived `Serialize` with its own +/// `skip_serializing_if` rules. Never re-record from a build that contains the +/// parts API. +const BASE_ALL_OPTIONAL_ABSENT: (&str, usize) = ( + "c3cf29fd48b638ce5fc26b6aacd90bf1251995f0a443e5a216ff00cc99361196", + 240_513, +); + +/// `ledger_window_attestation.prior_balances_digest_hex` of every case (they +/// share `prior_balances()`), recorded from 1398bbc1 alongside +/// [`BASE_ALL_OPTIONAL_ABSENT`]. +const BASE_PRIOR_BALANCES_DIGEST_HEX: &str = + "25d51842a8bb7447861eae9fdd5e4b7730cf4fea67fe945cb99dc824f4ec0b81"; + +const OPTIONAL_BUNDLE_KEYS: [&str; 7] = [ + "coinbase_script_sig_suffix_hex", + "witness_merkle_leaves_hex", + "audit_commitment_leaves_hex", + "audit_commitment_root_hex", + "settlement_mode_decision", + "ctv_fanout_fee_policy", + "ctv_fanout_manifest_set", +]; + +/// The `AuditBundleRef` view alone decides which absent fields are omitted; +/// with all of them absent, owned and parts bytes still match the base. +#[test] +fn all_optional_fields_absent_matches_base() { + let case = cases().remove(0); + assert_eq!(case.name, "build_audit_bundle/v1"); + let mut bundle = build_owned(case); + bundle.audit_commitment_leaves_hex.clear(); + bundle.audit_commitment_root_hex = None; + assert_eq!(bundle.schema, AUDIT_BUNDLE_SCHEMA_V1); + assert!(bundle.coinbase_script_sig_suffix_hex.is_none()); + assert!(bundle.witness_merkle_leaves_hex.is_empty()); + assert!(bundle.settlement_mode_decision.is_none()); + assert!(bundle.ctv_fanout_fee_policy.is_none()); + assert!(bundle.ctv_fanout_manifest_set.is_none()); + + let (base_sha256, base_len) = BASE_ALL_OPTIONAL_ABSENT; + let bytes = canonical_audit_bundle_bytes(&bundle).unwrap(); + assert_eq!(bytes.len(), base_len); + assert_eq!(sha256_hex(&bytes), base_sha256); + + let json: serde_json::Value = serde_json::from_slice(&bytes).unwrap(); + let keys = json.as_object().unwrap(); + for key in OPTIONAL_BUNDLE_KEYS { + assert!(!keys.contains_key(key), "{key} serialized while absent"); + } + + let (body, shares) = bundle.clone().into_parts(); + assert_eq!( + qbit_prism::CanonicalAuditHashPrefix::new(&shares) + .unwrap() + .finish(&body) + .unwrap(), + base_sha256, + ); + assert_eq!( + legacy::AuditBundleRef::from_parts(&body, &shares) + .canonical_bytes() + .unwrap(), + bytes + ); + assert_eq!( + canonical_audit_bundle_bytes_from_parts(&body, &shares).unwrap(), + bytes + ); + let decoded: AuditBundle = serde_json::from_slice(&bytes).unwrap(); + assert_eq!(decoded, bundle); +} + +/// The exported digest is the one the ledger window attestation signs, for +/// bundles built by the existing owned builders, and ignores input order. +#[test] +fn prior_balances_digest_matches_the_attestation() { + for case in cases() { + assert_eq!(case.prior_balances, prior_balances(), "{}", case.name); + let name = case.name; + let bundle = build_owned(case); + let digest = prior_balances_digest(&bundle.prior_balances); + assert_eq!( + hex::encode(digest), + bundle.ledger_window_attestation.prior_balances_digest_hex, + "{name}" + ); + assert_eq!( + hex::encode(digest), + BASE_PRIOR_BALANCES_DIGEST_HEX, + "{name}" + ); + let mut reversed = bundle.prior_balances.clone(); + reversed.reverse(); + assert_ne!(reversed, bundle.prior_balances, "{name}: fixture order"); + assert_eq!(prior_balances_digest(&reversed), digest, "{name}"); + } +} + +/// A full bundle carries `shares`, which `AuditBundleBody` has no field for; +/// decoding one as a body must fail rather than silently drop the window. +#[test] +fn audit_bundle_body_rejects_a_full_bundle() { + let case = cases().remove(0); + let body = build_borrowed(&case, &case.shares); + let bundle_bytes = canonical_audit_bundle_bytes_from_parts(&body, &case.shares).unwrap(); + + let error = serde_json::from_slice::(&bundle_bytes).unwrap_err(); + assert!( + error.to_string().contains("unknown field `shares`"), + "{error}" + ); + + let body_json = serde_json::to_vec(&body).unwrap(); + assert_eq!( + serde_json::from_slice::(&body_json).unwrap(), + body + ); +} + +/// A clone of the window would need a second allocation while the first is +/// still alive, so an unchanged buffer address proves the window was moved +/// or lent, never copied. +#[test] +fn builders_never_copy_the_window() { + let ledger_key = ledger_public_key_hex(); + for case in cases() { + let name = case.name; + + // Owned wrappers move the caller's Vec into the bundle. + let owned_case = case.clone(); + let buffer = owned_case.shares.as_ptr(); + let capacity = owned_case.shares.capacity(); + let bundle = build_owned(owned_case); + assert_eq!(bundle.shares.as_ptr(), buffer, "{name}: owned path copied"); + assert_eq!(bundle.shares.capacity(), capacity, "{name}"); + + // into_parts / into_bundle move the window through unchanged. + let (body, shares) = bundle.into_parts(); + assert_eq!(shares.as_ptr(), buffer, "{name}: into_parts copied"); + let bundle = body.into_bundle(shares); + assert_eq!(bundle.shares.as_ptr(), buffer, "{name}: into_bundle copied"); + + // A shared window (as a server snapshot would hold it) is only lent: + // no extra owner, no new buffer. + let window: Arc<[AcceptedShare]> = Arc::from(case.shares.clone()); + let window_buffer = window.as_ptr(); + let body = build_borrowed(&case, &window); + let bytes = canonical_audit_bundle_bytes_from_parts(&body, &window).unwrap(); + verify_audit_parts(&body, &window, &ledger_key).unwrap(); + assert_eq!(Arc::strong_count(&window), 1, "{name}"); + assert_eq!(window.as_ptr(), window_buffer, "{name}"); + assert_eq!( + bytes, + canonical_audit_bundle_bytes(&bundle).unwrap(), + "{name}" + ); + } +} diff --git a/crates/qbit-prism/tests/audit_trust_boundary.rs b/crates/qbit-prism/tests/audit_trust_boundary.rs index 7c1666e6e..3f3984013 100644 --- a/crates/qbit-prism/tests/audit_trust_boundary.rs +++ b/crates/qbit-prism/tests/audit_trust_boundary.rs @@ -217,9 +217,11 @@ fn insolvent_dust_block_is_rejected() { )); } -// Rust window selection matches the Python/SQL job+accept contract. +// Shares issued or accepted after the anchor are excluded from the window. The Python +// differential for the window fold is the frozen corpus replay (`window_frozen_vectors`, +// `window_daemon_gate`); no test on this branch runs the ledger's SQL window read against it. #[test] -fn window_converges_with_python_sql() { +fn window_excludes_shares_after_the_anchor() { let anchor = 1001i64; let nd = 10u128; let shares = vec![ diff --git a/crates/qbit-prism/tests/money_path_vectors.rs b/crates/qbit-prism/tests/money_path_vectors.rs new file mode 100644 index 000000000..793af2540 --- /dev/null +++ b/crates/qbit-prism/tests/money_path_vectors.rs @@ -0,0 +1,375 @@ +//! Replay the money-path vectors frozen from 2.x.x (issue #269) through the +//! 3.x.x engine. +//! +//! The vectors and their exporter live in `fixtures/vectors/`. Every case +//! must reproduce its frozen value exactly. A case may differ from 2.x.x only +//! through a D2 entry (`expected_2xx`, `expected_3xx` and `d2_entry`) in the +//! bootstrap or below-target credit topics. The set of entries the cases use +//! must equal the set documented in `docs/prism-rust-migration.md`, and every +//! vector file is pinned by its sha256. + +use std::collections::{BTreeMap, BTreeSet}; + +use qbit_prism::{ + apply_payout_policy, apply_proportional_fanout_fee, build_prism_reward_manifest, + compute_prism_window, select_settlement_mode, select_settlement_mode_with_pinned_direct, + AcceptedShare, CarryForwardBalance, PrismError, SettlementModeConfig, SettlementRecipient, +}; +use serde::de::DeserializeOwned; +use serde::Serialize; +use serde_json::{json, Value}; +use sha2::{Digest, Sha256}; + +const SCHEMA: &str = "qbit.prism.money-path-vectors.v1"; +const SOURCE_COMMIT: &str = "504846cc0b72e8f86ed17f896d4ccbbe196a31dc"; +/// The sha256 of each vector file as exported from `SOURCE_COMMIT`. The +/// vectors change only through a re-export in a reviewed commit that updates +/// these pins. +const VECTOR_SHA256: [(&str, &str); 7] = [ + ( + "window.json", + "aef6c5bb2304c345d9cac198f2654c2fd987e09d8677d99b6ae847798020d90d", + ), + ( + "carry_only.json", + "ed10a4361a0362e835f50c392bd5d0cf54f85b35a2c1ae48cc63bebede9c35e7", + ), + ( + "pool_fee.json", + "ee9e64ed43f20dee3da6543fc2f5bde893526a09ad54bffbee5c8611e7d4d146", + ), + ( + "fanout_fee_and_ties.json", + "f9e8c92377ba1d1b98e430d28a0ed6c94e7b60c4b59e9557fe28dce700d5be30", + ), + ( + "settlement_chunks.json", + "4dcddf8d71dd5ff9c51122b5075587f987dba36959830d2a798a8a589675bf0f", + ), + ( + "bootstrap_transition.json", + "6db2c6b8c00a1ce8402c6c97c7dd542b037a005afb538987c0cd75786c1b3fab", + ), + ( + "below_target_credit.json", + "66a419f9a61b9ca4678895f6774ade52837eb6e52f2e530dde66f06154a8754e", + ), +]; +const MIGRATION_DOC: &str = include_str!("../../../docs/prism-rust-migration.md"); +const D2_SECTION_HEADING: &str = "## Payout differences from 2.x.x (decision D2)"; +const D2_ENTRIES: [&str; 3] = [ + "d2a-bootstrap-pooling", + "d2b-below-target-credit", + "d2c-prior-balances-during-bootstrap", +]; +const D2_TOPICS: [u64; 2] = [8, 9]; +const VECTOR_FILES: [(&str, &str); 7] = [ + ( + "window.json", + include_str!("../fixtures/vectors/window.json"), + ), + ( + "carry_only.json", + include_str!("../fixtures/vectors/carry_only.json"), + ), + ( + "pool_fee.json", + include_str!("../fixtures/vectors/pool_fee.json"), + ), + ( + "fanout_fee_and_ties.json", + include_str!("../fixtures/vectors/fanout_fee_and_ties.json"), + ), + ( + "settlement_chunks.json", + include_str!("../fixtures/vectors/settlement_chunks.json"), + ), + ( + "bootstrap_transition.json", + include_str!("../fixtures/vectors/bootstrap_transition.json"), + ), + ( + "below_target_credit.json", + include_str!("../fixtures/vectors/below_target_credit.json"), + ), +]; + +fn to_value(value: &T) -> Value { + serde_json::to_value(value).expect("engine output serializes") +} + +fn field(input: &Value, name: &str) -> Result { + serde_json::from_value(input[name].clone()).map_err(|err| format!("input {name}: {err}")) +} + +fn outcome(result: Result) -> Value { + match result { + Ok(value) => json!({ "ok": to_value(&value) }), + Err(err) => json!({ "error": err.to_string() }), + } +} + +/// Mirrors the exporter's dispatcher over the same public entry points. +fn run(entry_point: &str, input: &Value) -> Result { + Ok(match entry_point { + "compute_prism_window" => outcome(compute_prism_window( + &field::>(input, "shares")?, + &field(input, "found_block")?, + )), + "apply_payout_policy" => outcome(apply_payout_policy( + &field(input, "reward_manifest")?, + &field::>(input, "prior_balances")?, + &field(input, "policy")?, + )), + "apply_proportional_fanout_fee" => outcome(apply_proportional_fanout_fee( + &field::>(input, "recipients")?, + field(input, "fee_sats")?, + field(input, "min_output_sats")?, + )), + "select_settlement_mode" => { + let recipients = field::>(input, "recipients")?; + let floor = field(input, "direct_floor_sats")?; + let config = field::(input, "config")?; + match field::>(input, "pinned_direct")? { + Some(pinned) => outcome(select_settlement_mode_with_pinned_direct( + &recipients, + floor, + &config, + Some(&pinned), + )), + None => outcome(select_settlement_mode(&recipients, floor, &config)), + } + } + "bundle_payout" => bundle_payout(input)?, + other => return Err(format!("unknown entry point {other}")), + }) +} + +/// The payout consequence of one bundle: the reward window and the payout +/// policy manifest, computed exactly as the exporter did. +fn bundle_payout(input: &Value) -> Result { + let shares = field::>(input, "shares")?; + let found_block = field(input, "found_block")?; + let prior_balances = field::>(input, "prior_balances")?; + let policy = field(input, "policy")?; + Ok(outcome( + build_prism_reward_manifest(&shares, &found_block).and_then(|reward| { + let payout = apply_payout_policy(&reward, &prior_balances, &policy)?; + Ok(json!({ + "counted_window_weight": to_value(&reward.counted_window_weight), + "counted_shares": reward + .shares + .iter() + .map(|share| json!({ + "share_seq": share.share_seq, + "miner_id": share.miner_id, + "counted_difficulty": to_value(&share.counted_difficulty), + })) + .collect::>(), + "entitlements": to_value(&reward.entitlements), + "payout_policy_manifest": to_value(&payout), + })) + }), + )) +} + +/// Record every leaf where `actual` departs from `expected`. +fn diff(path: &str, expected: &Value, actual: &Value, mismatches: &mut Vec) { + match (expected, actual) { + (Value::Object(want), Value::Object(got)) => { + for key in want + .keys() + .chain(got.keys().filter(|key| !want.contains_key(*key))) + { + let child = format!("{path}.{key}"); + match (want.get(key), got.get(key)) { + (Some(want), Some(got)) => diff(&child, want, got, mismatches), + (Some(want), None) => { + mismatches.push(format!("{child}: expected {want}, got nothing")) + } + (None, Some(got)) => { + mismatches.push(format!("{child}: expected nothing, got {got}")) + } + (None, None) => unreachable!(), + } + } + } + (Value::Array(want), Value::Array(got)) if want.len() == got.len() => { + for (index, (want, got)) in want.iter().zip(got).enumerate() { + diff(&format!("{path}[{index}]"), want, got, mismatches); + } + } + _ if expected == actual => {} + _ => mismatches.push(format!("{path}: expected {expected}, got {actual}")), + } +} + +fn check( + path: &str, + expected: &Value, + entry_point: &str, + input: &Value, + mismatches: &mut Vec, +) { + match run(entry_point, input) { + Ok(actual) => diff(path, expected, &actual, mismatches), + Err(err) => mismatches.push(format!("{path}: {err}")), + } +} + +/// Every anchor in the doc's D2 section, up to the next level-two heading. +fn documented_d2_entries() -> Result, String> { + let start = MIGRATION_DOC + .find(D2_SECTION_HEADING) + .ok_or_else(|| format!("docs/prism-rust-migration.md has no {D2_SECTION_HEADING:?}"))?; + let section = &MIGRATION_DOC[start + D2_SECTION_HEADING.len()..]; + let section = §ion[..section.find("\n## ").unwrap_or(section.len())]; + Ok(section + .split("::new(); + let mut d2_cases = BTreeMap::::new(); + + for (file, raw) in VECTOR_FILES { + let actual_sha256 = hex::encode(Sha256::digest(raw.as_bytes())); + match VECTOR_SHA256.iter().find(|(name, _)| *name == file) { + Some((_, pinned)) if *pinned == actual_sha256 => {} + pinned => mismatches.push(format!( + "{file} / sha256: pinned {}, got {actual_sha256}; the vectors change only through a \ + re-export from {SOURCE_COMMIT} in a reviewed commit that updates the pin", + pinned.map_or("nothing", |(_, pin)| pin) + )), + } + let document: Value = + serde_json::from_str(raw).unwrap_or_else(|err| panic!("{file} is not JSON: {err}")); + if document["schema"] != SCHEMA { + mismatches.push(format!( + "{file} / schema: expected {SCHEMA}, got {}", + document["schema"] + )); + } + if document["source_commit"] != SOURCE_COMMIT { + mismatches.push(format!( + "{file} / source_commit: expected {SOURCE_COMMIT}, got {}", + document["source_commit"] + )); + } + if document["produced_by"].as_str().is_none_or(str::is_empty) { + mismatches.push(format!( + "{file} / produced_by: missing regeneration command" + )); + } + for case in document["cases"].as_array().into_iter().flatten() { + let name = case["name"].as_str().unwrap_or(""); + let at = |field: &str| format!("{file} / {name} / {field}"); + let Some(topic) = case["topic"].as_u64() else { + mismatches.push(at("topic: missing")); + continue; + }; + *cases_per_topic.entry(topic).or_default() += 1; + let entry_point = case["entry_point"].as_str().unwrap_or_default(); + let input = &case["input"]; + + if let Some(anchor) = case.get("d2_entry") { + let anchor = anchor.as_str().unwrap_or_default(); + *d2_cases.entry(anchor.to_string()).or_default() += 1; + if !D2_TOPICS.contains(&topic) { + mismatches.push(at(&format!( + "d2_entry: topic {topic} may not differ from 2.x.x" + ))); + } + if !MIGRATION_DOC.contains(&format!("id=\"{anchor}\"")) { + mismatches.push(at(&format!( + "d2_entry: anchor {anchor} is missing from docs/prism-rust-migration.md" + ))); + } + if case["expected_2xx"] == case["expected_3xx"] { + mismatches.push(at("d2_entry: expected_2xx equals expected_3xx")); + } + // The 3.x.x rule's payout, and the frozen 2.x.x rule's payout + // recomputed by this engine: only the rule may differ. + check( + &at("expected_3xx"), + &case["expected_3xx"], + entry_point, + &input["bundle_3xx"], + &mut mismatches, + ); + check( + &at("expected_2xx"), + &case["expected_2xx"], + entry_point, + &input["bundle_2xx"], + &mut mismatches, + ); + } else if case.get("expected_2xx").is_some() || case.get("expected_3xx").is_some() { + mismatches.push(at("expected_2xx/expected_3xx without a d2_entry")); + } else if entry_point == "bundle_payout" { + for bundle in ["bundle_3xx", "bundle_2xx"] { + check( + &at(&format!("expected ({bundle})")), + &case["expected"], + entry_point, + &input[bundle], + &mut mismatches, + ); + } + } else { + check( + &at("expected"), + &case["expected"], + entry_point, + input, + &mut mismatches, + ); + } + } + } + + // A documented entry with no vector, or a vector with no entry, fails. + let used: BTreeSet = d2_cases.keys().cloned().collect(); + let listed: BTreeSet = D2_ENTRIES.iter().map(|entry| entry.to_string()).collect(); + match documented_d2_entries() { + Ok(documented) => { + if documented != listed { + mismatches.push(format!( + "docs/prism-rust-migration.md D2 entries: expected {listed:?}, got {documented:?}" + )); + } + if used != documented { + mismatches.push(format!( + "D2 entries used by vectors {used:?} differ from the documented entries {documented:?}" + )); + } + } + Err(err) => mismatches.push(err), + } + + for topic in 1..=9 { + let count = cases_per_topic.get(&topic).copied().unwrap_or(0); + println!("money-path vectors: topic {topic}: {count} case(s)"); + if count < 2 { + mismatches.push(format!( + "topic {topic}: expected at least 2 cases, found {count}" + )); + } + } + println!( + "money-path vectors: {} D2-tagged case(s): {d2_cases:?}", + d2_cases.values().sum::() + ); + assert!( + mismatches.is_empty(), + "{} money-path vector mismatch(es):\n{}", + mismatches.len(), + mismatches.join("\n") + ); +} diff --git a/crates/qbit-prism/tests/parallel_builder.rs b/crates/qbit-prism/tests/parallel_builder.rs new file mode 100644 index 000000000..9b71e0be3 --- /dev/null +++ b/crates/qbit-prism/tests/parallel_builder.rs @@ -0,0 +1,630 @@ +//! The builder's parallel fast path (#275) produces the serial path's bytes. +//! +//! Every randomized window is built twice, once through the unchanged serial +//! entry points and once through the `_parallel` variants at several worker +//! and chunk geometries, and the canonical audit bytes, the pipelined audit +//! hash, the share-array digest and the commitment leaf must be identical. +//! The verifier, which is serial, must accept the parallel body. Golden +//! vectors pin one fixed window's digests so a change to either path is +//! caught even when both drift together. + +use qbit_pool_builder::ManifestSigningKey; +use qbit_prism::{ + build_audit_bundle_body_with_coinbase_options, + build_audit_bundle_body_with_coinbase_options_parallel, + build_audit_bundle_body_with_ctv_settlement_options, + build_audit_bundle_body_with_ctv_settlement_options_parallel, + canonical_audit_bundle_bytes_from_parts, canonical_reward_manifest_bytes, + prism_audit_commitment_leaf_hex, restore_reward_manifest, verify_audit_parts, AcceptedShare, + AuditBundleBody, CanonicalAuditHashPrefix, CarryForwardBalance, FanoutFeeRatePolicy, + FoundBlock, Parallelism, PayoutPolicy, PrismError, SettlementModeConfig, +}; +use sha2::{Digest, Sha256}; + +struct Rng(u64); + +impl Rng { + fn next(&mut self) -> u64 { + // xorshift64*: deterministic, dependency-free. + let mut x = self.0; + x ^= x >> 12; + x ^= x << 25; + x ^= x >> 27; + self.0 = x; + x.wrapping_mul(0x2545_F491_4F6C_DD1D) + } + + fn below(&mut self, bound: u64) -> u64 { + self.next() % bound.max(1) + } + + fn pick<'a, T>(&mut self, items: &'a [T]) -> &'a T { + &items[self.below(items.len() as u64) as usize] + } +} + +const STRING_EDGES: &[&str] = &[ + "plain", + "quote\"inside", + "back\\slash", + "new\nline\ttab\r", + "nul\u{0}byte", + "unicode-é-日本-🙂", + "line\u{2028}sep\u{2029}", + "ctl\u{1f}\u{7f}", + "&'", + "", +]; + +const DIFFICULTY_EDGES: &[u128] = &[ + 1, + 2, + 7, + u32::MAX as u128, + u64::MAX as u128, + u64::MAX as u128 + 1, + u128::MAX / 64, + u128::MAX / 16, +]; + +fn manifest_signing_key() -> ManifestSigningKey { + ManifestSigningKey::from_seed_hex(&"42".repeat(32)).unwrap() +} + +fn ledger_signing_key() -> ManifestSigningKey { + ManifestSigningKey::from_seed_hex(&"43".repeat(32)).unwrap() +} + +fn ctv_config() -> SettlementModeConfig { + SettlementModeConfig { + max_coinbase_settlement_outputs: 4, + max_direct_coinbase_outputs: 2, + max_fanout_recipients_per_transaction: 3, + reserved_coinbase_outputs: 0, + } +} + +fn prior_balances(rng: &mut Rng) -> Vec { + (0..rng.below(4)) + .map(|index| CarryForwardBalance { + recipient_id: format!("carry-{}-{index}", rng.pick(STRING_EDGES)), + order_key: format!("{:02}", rng.below(100)), + p2mr_program_hex: hex::encode([rng.below(256) as u8; 32]), + balance_sats: rng.below(20_000) as i128, + }) + .collect() +} + +/// A random window: sizes from empty to a few thousand, every string edge, +/// u128 difficulties at both ends, ineligible shares after the anchor, a +/// credit policy on some shares, and ascending, descending or shuffled order. +fn random_window(rng: &mut Rng, max_len: u64) -> (Vec, FoundBlock) { + let count = match rng.below(6) { + 0 => 0, + 1 => 1, + 2 => rng.below(12), + 3 => rng.below(200), + _ => rng.below(max_len), + }; + let anchor = 1_000_000 + rng.below(1_000) as i64; + let miners: Vec<(String, String, String)> = (0..1 + rng.below(9)) + .map(|index| { + ( + format!("miner-{}-{index}", rng.pick(STRING_EDGES)), + format!("{:02}", rng.below(100)), + hex::encode([rng.below(256) as u8; 32]), + ) + }) + .collect(); + let with_credit = rng.below(3) == 0; + let mut shares: Vec = (1..=count) + .map(|share_seq| { + let (miner_id, order_key, program) = rng.pick(&miners).clone(); + let job_issued_at_ms = anchor - rng.below(5_000) as i64 + rng.below(20) as i64 * 300; + AcceptedShare { + share_seq, + share_id: format!("share-{share_seq}-{}", rng.pick(STRING_EDGES)), + miner_id, + order_key, + p2mr_program_hex: program, + share_difficulty: if rng.below(4) == 0 { + *rng.pick(DIFFICULTY_EDGES) + } else { + 1 + rng.below(50) as u128 + }, + network_difficulty: *rng.pick(DIFFICULTY_EDGES), + template_height: rng.below(1_000_000), + job_id: format!("job-{}", rng.pick(STRING_EDGES)), + job_issued_at_ms, + accepted_at_ms: job_issued_at_ms + rng.below(1_500) as i64, + ntime: rng.below(u32::MAX as u64) as u32, + credit_policy: (with_credit && rng.below(2) == 0).then(|| "stale-grace".into()), + } + }) + .collect(); + match rng.below(3) { + 0 => {} + 1 => shares.reverse(), + _ => { + for index in (1..shares.len()).rev() { + let other = rng.below(index as u64 + 1) as usize; + shares.swap(index, other); + } + } + } + let network_difficulty = match rng.below(3) { + 0 => 1 + rng.below(3) as u128, + 1 => 1 + rng.below(200) as u128, + _ => *rng.pick(DIFFICULTY_EDGES), + }; + let found_block = FoundBlock { + block_height: 100 + rng.below(1_000), + coinbase_value_sats: 1 + rng.below(5_000_000_000), + network_difficulty, + anchor_job_issued_at_ms: anchor, + }; + (shares, found_block) +} + +fn geometries() -> Vec { + vec![ + Parallelism::serial(), + Parallelism::new(2, 1), + Parallelism::new(3, 2), + Parallelism::new(4, 7), + Parallelism::new(8, 64), + Parallelism::new(2, 4096), + ] +} + +fn error_text(error: &PrismError) -> String { + format!("{error:?}") +} + +fn build_serial( + ctv: bool, + shares: &[AcceptedShare], + found_block: &FoundBlock, + balances: &[CarryForwardBalance], +) -> Result { + if ctv { + build_audit_bundle_body_with_ctv_settlement_options( + shares, + found_block.clone(), + balances.to_vec(), + PayoutPolicy::day_one_default(), + 2_000, + ctv_config(), + Some(FanoutFeeRatePolicy::new(1_000, 12_000)), + Some("aaaaaaaa".into()), + vec!["11".repeat(32)], + &manifest_signing_key(), + &ledger_signing_key(), + ) + } else { + build_audit_bundle_body_with_coinbase_options( + shares, + found_block.clone(), + balances.to_vec(), + PayoutPolicy::day_one_default(), + Some("bbbbbbbb".into()), + vec!["22".repeat(32), "33".repeat(32)], + &manifest_signing_key(), + &ledger_signing_key(), + ) + } +} + +fn build_parallel( + ctv: bool, + shares: &[AcceptedShare], + found_block: &FoundBlock, + balances: &[CarryForwardBalance], + parallelism: Parallelism, +) -> Result { + if ctv { + build_audit_bundle_body_with_ctv_settlement_options_parallel( + shares, + found_block.clone(), + balances.to_vec(), + PayoutPolicy::day_one_default(), + 2_000, + ctv_config(), + Some(FanoutFeeRatePolicy::new(1_000, 12_000)), + Some("aaaaaaaa".into()), + vec!["11".repeat(32)], + &manifest_signing_key(), + &ledger_signing_key(), + parallelism, + ) + } else { + build_audit_bundle_body_with_coinbase_options_parallel( + shares, + found_block.clone(), + balances.to_vec(), + PayoutPolicy::day_one_default(), + Some("bbbbbbbb".into()), + vec!["22".repeat(32), "33".repeat(32)], + &manifest_signing_key(), + &ledger_signing_key(), + parallelism, + ) + } +} + +/// Every digest the refresh pipeline publishes for `(body, shares)`. +struct Digests { + canonical_bytes_sha256: String, + pipelined_audit_sha256: String, + share_array_sha256: String, + reward_manifest_sha256: String, + leaf_hex: String, +} + +fn digests(body: &AuditBundleBody, shares: &[AcceptedShare], parallelism: Parallelism) -> Digests { + let (prefix, share_array) = + CanonicalAuditHashPrefix::new_with_share_digest(shares, parallelism).unwrap(); + let canonical = canonical_audit_bundle_bytes_from_parts(body, shares).unwrap(); + Digests { + canonical_bytes_sha256: hex::encode(Sha256::digest(&canonical)), + pipelined_audit_sha256: prefix.finish_with(body, parallelism).unwrap(), + share_array_sha256: hex::encode(share_array), + reward_manifest_sha256: hex::encode(Sha256::digest( + canonical_reward_manifest_bytes(&body.reward_manifest).unwrap(), + )), + leaf_hex: body.audit_commitment_leaves_hex[0].clone(), + } +} + +fn check_window( + ctv: bool, + shares: &[AcceptedShare], + found_block: &FoundBlock, + balances: &[CarryForwardBalance], + label: &str, +) { + let serial = build_serial(ctv, shares, found_block, balances); + for parallelism in geometries() { + let parallel = build_parallel(ctv, shares, found_block, balances, parallelism); + match (&serial, ¶llel) { + (Err(expected), Err(actual)) => { + assert_eq!( + error_text(expected), + error_text(actual), + "{label} {parallelism:?}" + ); + continue; + } + (Err(expected), Ok(_)) => panic!("{label} {parallelism:?}: serial failed {expected:?}"), + (Ok(_), Err(actual)) => panic!("{label} {parallelism:?}: parallel failed {actual:?}"), + (Ok(_), Ok(_)) => {} + } + let (serial, parallel) = (serial.as_ref().unwrap(), parallel.as_ref().unwrap()); + assert_eq!(serial, parallel, "{label} {parallelism:?}: body differs"); + let serial_bytes = canonical_audit_bundle_bytes_from_parts(serial, shares).unwrap(); + let parallel_bytes = canonical_audit_bundle_bytes_from_parts(parallel, shares).unwrap(); + assert_eq!( + serial_bytes, parallel_bytes, + "{label} {parallelism:?}: bytes differ" + ); + let expected = Digests { + canonical_bytes_sha256: hex::encode(Sha256::digest(&serial_bytes)), + pipelined_audit_sha256: CanonicalAuditHashPrefix::new(shares) + .unwrap() + .finish(serial) + .unwrap(), + share_array_sha256: hex::encode(Sha256::digest(serde_json::to_vec(shares).unwrap())), + reward_manifest_sha256: hex::encode(Sha256::digest( + canonical_reward_manifest_bytes(&serial.reward_manifest).unwrap(), + )), + leaf_hex: prism_audit_commitment_leaf_hex( + &serial.reward_manifest, + &serial.payout_policy_manifest, + ) + .unwrap(), + }; + let actual = digests(parallel, shares, parallelism); + assert_eq!( + actual.canonical_bytes_sha256, expected.canonical_bytes_sha256, + "{label}" + ); + assert_eq!( + actual.pipelined_audit_sha256, expected.canonical_bytes_sha256, + "{label}" + ); + assert_eq!( + actual.pipelined_audit_sha256, expected.pipelined_audit_sha256, + "{label}" + ); + assert_eq!( + actual.share_array_sha256, expected.share_array_sha256, + "{label}" + ); + assert_eq!( + actual.reward_manifest_sha256, expected.reward_manifest_sha256, + "{label}" + ); + assert_eq!(actual.leaf_hex, expected.leaf_hex, "{label}"); + // The serial verifier accepts the parallel body and rebuilds its window. + verify_audit_parts(parallel, shares, &ledger_signing_key().public_key_hex()) + .unwrap_or_else(|error| panic!("{label} {parallelism:?}: verify {error:?}")); + let (header, window) = parallel.reward_manifest.clone().into_parts(); + let restored = restore_reward_manifest(header, shares, found_block).unwrap(); + assert_eq!( + restored.shares, window, + "{label} {parallelism:?}: restored window" + ); + } +} + +#[test] +fn randomized_windows_build_identical_bodies_bytes_and_digests() { + let mut rng = Rng(0x9E37_79B9_7F4A_7C15); + for round in 0..96 { + let (shares, found_block) = random_window(&mut rng, 2_000); + let balances = prior_balances(&mut rng); + let ctv = round % 4 == 3; + check_window( + ctv, + &shares, + &found_block, + &balances, + &format!("round {round} ({} shares, ctv {ctv})", shares.len()), + ); + } +} + +#[test] +fn boundary_windows_build_identical_bodies() { + let mut rng = Rng(0xD1B5_4A32_D192_ED03); + // Exact chunk boundaries for every geometry, and one share past each. + for count in [ + 1usize, 2, 3, 4, 7, 8, 14, 15, 63, 64, 65, 127, 128, 129, 4096, 4097, + ] { + let (_, mut found_block) = random_window(&mut rng, 1); + let shares: Vec = (1..=count as u64) + .map(|share_seq| AcceptedShare { + share_seq, + share_id: format!("share-{share_seq}"), + miner_id: format!("miner-{}", share_seq % 5), + order_key: format!("{:02}", share_seq % 5), + p2mr_program_hex: hex::encode([(share_seq % 5) as u8; 32]), + share_difficulty: if share_seq == 1 { + u128::from(u64::MAX) + 12_345 + } else { + 20 + }, + network_difficulty: u128::MAX, + template_height: 100, + job_id: "job".into(), + job_issued_at_ms: found_block.anchor_job_issued_at_ms - 10, + accepted_at_ms: found_block.anchor_job_issued_at_ms - 5, + ntime: 1, + credit_policy: (share_seq % 3 == 0).then(|| "stale-grace".into()), + }) + .collect(); + found_block.network_difficulty = (count.max(1) as u128 * 20) / 8; + for ctv in [false, true] { + check_window( + ctv, + &shares, + &found_block, + &[], + &format!("count {count} ctv {ctv}"), + ); + } + } +} + +#[test] +fn empty_window_prefix_and_digest_match_the_serial_path() { + for parallelism in geometries() { + let (prefix, digest) = + CanonicalAuditHashPrefix::new_with_share_digest(&[], parallelism).unwrap(); + assert_eq!(hex::encode(digest), hex::encode(Sha256::digest(b"[]"))); + let mut rng = Rng(7); + let (shares, found_block) = loop { + let (shares, found_block) = random_window(&mut rng, 50); + if !shares.is_empty() + && build_serial(false, &shares, &found_block, &[]).is_ok() + && shares.iter().all(|share| share.credit_policy.is_none()) + { + break (shares, found_block); + } + }; + let body = build_serial(false, &shares, &found_block, &[]).unwrap(); + // The empty window's prefix is the v1 schema; its suffix is the body's. + assert_eq!( + prefix.finish_with(&body, parallelism).unwrap(), + CanonicalAuditHashPrefix::new(&[]) + .unwrap() + .finish(&body) + .unwrap() + ); + } +} + +/// (name, canonical audit bytes sha256, share-array sha256, reward-manifest +/// sha256) recorded from the serial builders at 3.x.x 7b78ea6f, before the +/// parallel fast path existed. Both paths must still produce exactly these +/// digests; never re-record them from a build that contains the fast path. +const GOLDEN: [(&str, &str, &str, &str); 2] = [ + ( + "coinbase_options/2048", + "644daf335593c96fbb368310d726ed906df9cfbd59223013e7bee025a8aa88e9", + "f6090b4bf622adf7e06b7698c3e8e38be0803ad9e0f1737719314782d76b0942", + "5911f26eddf010cec907b9d3fafda7b2438247a07ebe7bdb784e60dbd8444f51", + ), + ( + "ctv_settlement/2048", + "71269b7579306219dd49bb1e3003e21563a50a5d234db3a8b2ab1c8bd43413b7", + "f6090b4bf622adf7e06b7698c3e8e38be0803ad9e0f1737719314782d76b0942", + "5911f26eddf010cec907b9d3fafda7b2438247a07ebe7bdb784e60dbd8444f51", + ), +]; + +fn golden_window() -> (Vec, FoundBlock) { + let mut rng = Rng(0x0BAD_5EED_0BAD_5EED); + let shares: Vec = (1..=2_048u64) + .map(|share_seq| { + let miner = share_seq % 7; + AcceptedShare { + share_seq, + share_id: format!( + "share-{share_seq}-{}", + STRING_EDGES[share_seq as usize % 10] + ), + miner_id: format!("miner-{miner}-{}", STRING_EDGES[miner as usize]), + order_key: format!("{miner:02}"), + p2mr_program_hex: hex::encode([miner as u8 + 1; 32]), + // The oldest share exceeds u64::MAX and is the partially counted + // window boundary; the rest keep the window economically sane. + share_difficulty: if share_seq == 1 { + u64::MAX as u128 + 12_345 + } else { + ((miner + 1) * (miner + 1) * (share_seq % 5 + 1)) as u128 + }, + network_difficulty: DIFFICULTY_EDGES[share_seq as usize % 8], + template_height: 100, + job_id: format!("job-{}", rng.below(1_000)), + job_issued_at_ms: 1_000 + share_seq as i64, + accepted_at_ms: 1_000 + share_seq as i64, + ntime: 1_800_000_000, + credit_policy: (share_seq % 5 == 0).then(|| "stale-grace".into()), + } + }) + .collect(); + let found_block = FoundBlock { + block_height: 101, + coinbase_value_sats: 500_000_000, + network_difficulty: 5_000, + anchor_job_issued_at_ms: 1_000 + 2_048, + }; + (shares, found_block) +} + +#[test] +fn golden_vectors_pin_both_paths() { + let (shares, found_block) = golden_window(); + let balances = []; + let mut mismatches = Vec::new(); + for (index, ctv) in [false, true].into_iter().enumerate() { + let (name, canonical, share_array, reward) = GOLDEN[index]; + let serial = build_serial(ctv, &shares, &found_block, &balances).unwrap(); + let serial_digests = digests(&serial, &shares, Parallelism::serial()); + for parallelism in geometries() { + let parallel = + build_parallel(ctv, &shares, &found_block, &balances, parallelism).unwrap(); + let actual = digests(¶llel, &shares, parallelism); + for (label, expected, value, serial_value) in [ + ( + "canonical_bytes_sha256", + canonical, + &actual.canonical_bytes_sha256, + &serial_digests.canonical_bytes_sha256, + ), + ( + "pipelined_audit_sha256", + canonical, + &actual.pipelined_audit_sha256, + &serial_digests.pipelined_audit_sha256, + ), + ( + "share_array_sha256", + share_array, + &actual.share_array_sha256, + &serial_digests.share_array_sha256, + ), + ( + "reward_manifest_sha256", + reward, + &actual.reward_manifest_sha256, + &serial_digests.reward_manifest_sha256, + ), + ] { + if value != expected { + mismatches.push(format!( + "{name} {label} {parallelism:?}: got {value}, pinned {expected}, serial path gives {serial_value}" + )); + } + } + } + } + assert!( + mismatches.is_empty(), + "golden mismatches:\n{}", + mismatches.join("\n") + ); +} + +fn vm_rss_mib() -> f64 { + let status = std::fs::read_to_string("/proc/self/status").unwrap_or_default(); + status + .lines() + .find_map(|line| line.strip_prefix("VmRSS:")) + .and_then(|rest| rest.trim().split(' ').next()) + .and_then(|kib| kib.parse::().ok()) + .map_or(0.0, |kib| kib / 1024.0) +} + +/// Memory probe for the builder at the refresh's shape (400k shares): builds +/// the tee, the body and the suffix `B275_RSS_CYCLES` times at +/// `B275_RSS_WORKERS` workers and prints the process RSS after each cycle. +/// Run under the heavy lock; compare against `MALLOC_ARENA_MAX` settings. +#[test] +#[ignore = "400k memory probe; run under the heavy test lock"] +fn rss_probe_400k() { + let workers: usize = std::env::var("B275_RSS_WORKERS") + .ok() + .and_then(|value| value.parse().ok()) + .unwrap_or(8); + let cycles: usize = std::env::var("B275_RSS_CYCLES") + .ok() + .and_then(|value| value.parse().ok()) + .unwrap_or(6); + let parallelism = if workers <= 1 { + Parallelism::serial() + } else { + Parallelism::new(workers, 4096) + }; + let (seed, found_block) = golden_window(); + let seed = seed[1].clone(); + let shares: Vec = (0..400_000u64) + .map(|index| { + let mut share = seed.clone(); + share.share_seq = index + 1; + share.share_id = format!("share-{index:07}-{}", "x".repeat(24)); + share.miner_id = format!("miner-{:04}", index % 2_000); + share.order_key = format!("{:04}", index % 2_000); + share.job_id = format!("job-{:06}", index % 500); + share.share_difficulty = 20; + share.job_issued_at_ms = found_block.anchor_job_issued_at_ms - 10; + share.accepted_at_ms = found_block.anchor_job_issued_at_ms - 5; + share + }) + .collect(); + let found_block = FoundBlock { + network_difficulty: 400_000 * 20 / 8, + ..found_block + }; + println!( + "RSS_PROBE workers={workers} after_window={:.0}", + vm_rss_mib() + ); + for cycle in 0..cycles { + let started = std::time::Instant::now(); + let (prefix, digest) = + CanonicalAuditHashPrefix::new_with_share_digest(&shares, parallelism).unwrap(); + let body = build_parallel(false, &shares, &found_block, &[], parallelism).unwrap(); + let audit = prefix.finish_with(&body, parallelism).unwrap(); + let peak = vm_rss_mib(); + drop(body); + println!( + "RSS_PROBE cycle={cycle} ms={} peak_mib={peak:.0} after_drop_mib={:.0} digest={} audit={}", + started.elapsed().as_millis(), + vm_rss_mib(), + &hex::encode(digest)[..8], + &audit[..8] + ); + } +} diff --git a/crates/qbit-prism/tests/reward_manifest_header.rs b/crates/qbit-prism/tests/reward_manifest_header.rs new file mode 100644 index 000000000..893d13407 --- /dev/null +++ b/crates/qbit-prism/tests/reward_manifest_header.rs @@ -0,0 +1,273 @@ +//! The normalized native audit row keeps only a `PrismRewardManifestHeader` +//! and rebuilds the counted window on read (#267). These tests prove the +//! rebuild is byte-identical, not merely equivalent, and that a header which +//! disagrees with its window is refused by name before any digest runs. + +use qbit_pool_builder::ManifestSigningKey; +use qbit_prism::{ + build_audit_bundle, canonical_audit_bundle_bytes, canonical_reward_manifest_bytes, + restore_reward_manifest, AcceptedShare, AuditBundle, CarryForwardBalance, FoundBlock, + PayoutPolicy, PrismError, PrismRewardManifest, PrismRewardManifestHeader, +}; +use serde::Deserialize; +use serde_json::Value; +use sha2::{Digest, Sha256}; + +const WINDOW_SHARES: u64 = 400; +const MINERS: u64 = 7; + +/// A few hundred shares from several miners, as in `audit_parts.rs`. The +/// oldest share's difficulty exceeds `u64::MAX`, so it is the partially +/// counted window boundary and its `u128` values exercise arbitrary-precision +/// JSON. `credit_policy_every` marks every n-th share `stale-grace`, which +/// selects the v1.1 schema and exercises the `credit_policy` field that +/// `CountedShare` omits when absent. +fn window_shares(credit_policy_every: Option) -> Vec { + (1..=WINDOW_SHARES) + .map(|share_seq| { + let miner = share_seq % MINERS; + let share_difficulty = if share_seq == 1 { + u128::from(u64::MAX) + 12_345 + } else { + u128::from((miner + 1) * (miner + 1) * (share_seq % 5 + 1)) + }; + let job_issued_at_ms = 1_000 + i64::try_from(share_seq).unwrap(); + AcceptedShare { + share_seq, + share_id: format!("share-{share_seq}"), + miner_id: format!("miner-{miner}"), + order_key: format!("{miner:02}"), + p2mr_program_hex: hex::encode([u8::try_from(miner + 1).unwrap(); 32]), + share_difficulty, + network_difficulty: 5_000, + template_height: 100, + job_id: format!("job-{job_issued_at_ms}"), + job_issued_at_ms, + accepted_at_ms: job_issued_at_ms, + ntime: 1_800_000_000, + credit_policy: credit_policy_every + .filter(|every| share_seq % every == 0) + .map(|_| "stale-grace".to_string()), + } + }) + .collect() +} + +fn found_block() -> FoundBlock { + FoundBlock { + block_height: 101, + coinbase_value_sats: 500_000_000, + network_difficulty: 5_000, + anchor_job_issued_at_ms: 1_000 + i64::try_from(WINDOW_SHARES).unwrap(), + } +} + +fn fixture_bundle(credit_policy_every: Option) -> AuditBundle { + build_audit_bundle( + window_shares(credit_policy_every), + found_block(), + vec![CarryForwardBalance { + recipient_id: "miner-2".to_string(), + order_key: "02".to_string(), + p2mr_program_hex: hex::encode([3_u8; 32]), + balance_sats: 4_800, + }], + PayoutPolicy::day_one_default(), + &ManifestSigningKey::from_seed_hex(&"42".repeat(32)).unwrap(), + &ManifestSigningKey::from_seed_hex(&"43".repeat(32)).unwrap(), + ) + .unwrap() +} + +fn sha256_hex(bytes: &[u8]) -> String { + hex::encode(Sha256::digest(bytes)) +} + +/// The value a normalized native row stores: the logical bundle minus the +/// top-level `shares` and minus `reward_manifest.shares`. +fn stored_shape(bundle: &AuditBundle) -> Value { + let mut stored = serde_json::to_value(bundle).unwrap(); + let body = stored.as_object_mut().unwrap(); + body.remove("shares").unwrap(); + body["reward_manifest"] + .as_object_mut() + .unwrap() + .remove("shares") + .unwrap(); + stored +} + +/// Criterion 1: the rebuilt manifest equals the original and the reassembled +/// bundle serializes to the original canonical bytes, byte for byte; a single +/// changed field in one rebuilt share changes the canonical sha256. +#[test] +fn restored_manifest_is_byte_identical_to_the_original() { + for credit_policy_every in [None, Some(13)] { + let original = fixture_bundle(credit_policy_every); + let original_bytes = canonical_audit_bundle_bytes(&original).unwrap(); + let (header, window) = original.reward_manifest.clone().into_parts(); + assert!( + !window.is_empty() && window.len() == header.included_share_count, + "the split window must be the counted window" + ); + + let restored = + restore_reward_manifest(header.clone(), &original.shares, &original.found_block) + .unwrap(); + assert_eq!(restored, original.reward_manifest); + assert_eq!( + restored.shares, window, + "rebuilt window differs from the split one" + ); + assert_eq!( + canonical_reward_manifest_bytes(&restored).unwrap(), + canonical_reward_manifest_bytes(&original.reward_manifest).unwrap() + ); + + let mut reassembled = original.clone(); + reassembled.reward_manifest = restored; + let reassembled_bytes = canonical_audit_bundle_bytes(&reassembled).unwrap(); + assert_eq!(reassembled, original); + assert!( + reassembled_bytes == original_bytes, + "canonical bytes differ after reconstruction" + ); + assert_eq!(sha256_hex(&reassembled_bytes), sha256_hex(&original_bytes)); + + // The split-and-join pair is lossless on its own as well. + let rejoined = header.clone().into_manifest(window.clone()); + assert_eq!(rejoined, original.reward_manifest); + + // The header's serde form round-trips and is what the stored row holds. + let stored = stored_shape(&original); + let stored_header: PrismRewardManifestHeader = + serde_json::from_value(stored["reward_manifest"].clone()).unwrap(); + assert_eq!(stored_header, header); + assert_eq!( + serde_json::to_value(&header).unwrap(), + stored["reward_manifest"], + "header serialization must equal the stored reward_manifest" + ); + + // Negative: one field of one rebuilt share moves the canonical sha256. + // The mutation lands on `reassembled`, whose window is the restored + // one, so this is about the window a read serves: the canonical + // digest a read checks last is sensitive to each of these fields of + // a rebuilt share. It is not a test of the rebuild itself; that is + // the equality with the original above. + for (field, mutate) in [ + ( + "counted_difficulty", + (|share: &mut qbit_prism::CountedShare| { + share.counted_difficulty += 1; + }) as fn(&mut qbit_prism::CountedShare), + ), + ("miner_id", |share| share.miner_id.push('x')), + ("credit_policy", |share| { + share.credit_policy = Some("stale-grace".to_string()); + }), + ] { + let mut tampered = reassembled.clone(); + let index = tampered.reward_manifest.shares.len() / 2; + mutate(&mut tampered.reward_manifest.shares[index]); + let tampered_bytes = canonical_audit_bundle_bytes(&tampered).unwrap(); + assert_ne!( + sha256_hex(&tampered_bytes), + sha256_hex(&original_bytes), + "changing {field} of one rebuilt share left the canonical sha256 unchanged" + ); + } + } +} + +/// Criterion 2: a stored header that disagrees with the rebuild fails as the +/// `reward_manifest` mismatch, not later at the canonical digest. +#[test] +fn header_mismatch_is_refused_by_name() { + let original = fixture_bundle(Some(11)); + let (header, _) = original.reward_manifest.clone().into_parts(); + let tampered: Vec<(&str, PrismRewardManifestHeader)> = vec![ + ("share_slice_digest_hex", { + let mut h = header.clone(); + h.share_slice_digest_hex = "00".repeat(32); + h + }), + ("counted_window_weight", { + let mut h = header.clone(); + h.counted_window_weight -= 1; + h + }), + ("included_share_count", { + let mut h = header.clone(); + h.included_share_count -= 1; + h + }), + ("entitlements", { + let mut h = header.clone(); + h.entitlements[0].weight += 1; + h + }), + ("newest_share_seq", { + let mut h = header.clone(); + h.newest_share_seq += 1; + h + }), + ]; + for (field, header) in tampered { + let error = restore_reward_manifest(header, &original.shares, &original.found_block) + .expect_err(&format!("a tampered {field} was accepted")); + assert!( + matches!( + error, + PrismError::AuditMismatch { + artifact: "reward_manifest" + } + ), + "tampered {field}: expected the reward_manifest mismatch, got {error}" + ); + } + // A window that is not the one the header was folded from is refused too, + // even when the header itself is untouched. + let short_window = &original.shares[1..]; + let error = restore_reward_manifest(header.clone(), short_window, &original.found_block) + .expect_err("a different window was accepted"); + assert!(matches!( + error, + PrismError::AuditMismatch { + artifact: "reward_manifest" + } + )); + // The untouched header against its own window still restores. + restore_reward_manifest(header, &original.shares, &original.found_block).unwrap(); +} + +/// EP-COMPAT, both directions. A full manifest cannot be read as a header +/// (it has `shares`, which the header denies), so a legacy row can never be +/// mistaken for a normalized one. And a binary that predates this change +/// cannot decode a normalized row as a bundle: `PrismRewardManifest.shares` +/// has no default, so the decode fails explicitly instead of serving a body +/// with an empty window. +#[test] +fn legacy_and_normalized_shapes_cannot_be_confused() { + let original = fixture_bundle(None); + let full_manifest = serde_json::to_value(&original.reward_manifest).unwrap(); + let error = PrismRewardManifestHeader::deserialize(&full_manifest) + .expect_err("a manifest with shares was decoded as a header"); + assert!( + error.to_string().contains("unknown field `shares`"), + "unexpected error: {error}" + ); + + let mut stored = stored_shape(&original); + stored["shares"] = serde_json::to_value(&original.shares).unwrap(); + let error = AuditBundle::deserialize(&stored) + .expect_err("an older reader decoded a normalized row as a full bundle"); + assert!( + error.to_string().contains("missing field `shares`"), + "unexpected error: {error}" + ); + + // The legacy stored shape still decodes as a full manifest. + let legacy: PrismRewardManifest = serde_json::from_value(full_manifest).unwrap(); + assert_eq!(legacy, original.reward_manifest); +} diff --git a/crates/qbit-prism/tests/support/window_corpus.rs b/crates/qbit-prism/tests/support/window_corpus.rs new file mode 100644 index 000000000..6b82443f5 --- /dev/null +++ b/crates/qbit-prism/tests/support/window_corpus.rs @@ -0,0 +1,945 @@ +//! The frozen payout-window corpus (#269), shared by the native replay +//! (`window_frozen_vectors`) and the serve-daemon gate (`window_daemon_gate`). +//! +//! `tests/fixtures/window_pipeline_parity/reference.json` was generated on +//! 2.x.x by the Python differential oracle and is never edited here; its +//! five inputs that the oracle derived from a seeded `random.Random` live in +//! the `inputs-unpinned.json` sidecar beside it. Every input is checked +//! against its frozen `input_sha256` before any output is compared, and every +//! output comparison is against the frozen bytes, never a re-serialization. +//! +//! `supplementary.json` holds the few cases the corpus never reaches (a whole +//! page expiring at exactly `window_weight`), exported from the same 2.x.x +//! oracle with the same entry shape and replayed under their own tally. +#![allow(dead_code)] + +use qbit_prism::window::{prepare_window_out_of_range, DeclaredWidth}; +use qbit_prism::AcceptedShare; +use serde::Deserialize; +use serde_json::{json, Value}; +use sha2::{Digest, Sha256}; +use std::collections::{BTreeSet, HashMap}; +use std::fmt; + +const REFERENCE_JSON: &str = + include_str!("../../../../tests/fixtures/window_pipeline_parity/reference.json"); +const UNPINNED_INPUTS_JSON: &str = + include_str!("../../../../tests/fixtures/window_pipeline_parity/inputs-unpinned.json"); +const SUPPLEMENTARY_JSON: &str = + include_str!("../../../../tests/fixtures/window_pipeline_parity/supplementary.json"); + +/// Pin of the frozen corpus file. The unpinned-inputs sidecar needs no pin of +/// its own: each of its documents is pinned by its `input_sha256` in here. +pub const REFERENCE_SHA256: &str = + "017c787d3b894d92702d65774e47224ce5a09838bcd1118549f740a21b406142"; +/// Pin of the supplementary cases, whose entries carry their own inputs. +pub const SUPPLEMENTARY_SHA256: &str = + "4a89967643938cfbec933b7f5582118e583245986eb2adf9e533d017ea252fc6"; +const REFERENCE_SCHEMA: &str = "qbit-prism-window-pipeline-parity-reference/v2"; +const INPUT_SCHEMA: &str = "qbit-prism-window-pipeline-parity-input/v1"; +const UNPINNED_SCHEMA: &str = "qbit-prism-window-pipeline-parity-unpinned-inputs/v1"; +const SUPPLEMENTARY_SCHEMA: &str = "qbit-prism-window-pipeline-parity-supplementary/v1"; +/// The 2.x.x commit whose oracle exported both the sidecar and the +/// supplementary cases. +const ORACLE_SOURCE_COMMIT: &str = "504846cc0b72e8f86ed17f896d4ccbbe196a31dc"; +const SUPPLEMENTARY_CASES: [&str; 4] = [ + "advance-page-expiry-at-weight", + "advance-page-expiry-at-weight-second-page", + "advance-page-expiry-one-above-weight", + "advance-page-expiry-one-below-weight", +]; +const UNPINNED_CASES: [&str; 5] = [ + "bulk-seeded", + "multi-page-interior-cutoff", + "page-boundary-511", + "page-boundary-512", + "page-boundary-513", +]; + +pub const FULL_REJECTIONS: [&str; 5] = [ + "duplicate_share_seq", + "duplicate_share_id", + "non_positive_difficulty", + "non_positive_window_weight", + "non_positive_page_size", +]; +pub const ADVANCE_REJECTIONS: [&str; 6] = [ + "anchor_regression", + "delta_non_positive_difficulty", + "delta_ineligible_at_anchor", + "delta_repeats_eligible_share", + "delta_not_append", + "delta_order_not_increasing", +]; + +/// A corpus case whose input leaves 3.x.x's declared integer widths (2.x.x's +/// `RUST_INTEGER_DOMAIN`): its frozen bytes came from Python's unbounded +/// integers and are not checkable here by design, so it is declined -- and +/// the decline itself is asserted. +pub struct OutOfDomain { + pub case: &'static str, + pub field: &'static str, + pub literal: &'static str, + pub width: DeclaredWidth, +} + +pub const OUT_OF_DOMAIN: [OutOfDomain; 2] = [ + OutOfDomain { + case: "difficulty-beyond-u128", + field: "window_weight", + // 2^128 + 2^64 + literal: "340282366920938463481821351505477763072", + width: DeclaredWidth::U128, + }, + OutOfDomain { + case: "ntime-beyond-u32", + field: "records[0].ntime", + // 2^32 + literal: "4294967296", + width: DeclaredWidth::U32, + }, +]; + +/// The counts both replays must reach. +pub const EXPECTED_TALLY: Tally = Tally { + cases: 35, + rejections: 12, + byte_compared: 21, + declined: 2, + with_advance_stats: 8, +}; + +/// The counts both replays must reach on the supplementary cases. +pub const EXPECTED_SUPPLEMENTARY_TALLY: Tally = Tally { + cases: 4, + rejections: 0, + byte_compared: 4, + declined: 0, + with_advance_stats: 4, +}; + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum Phase { + Full, + Advance(usize), +} + +impl fmt::Display for Phase { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + Phase::Full => formatter.write_str("full"), + Phase::Advance(index) => write!(formatter, "advances[{index}]"), + } + } +} + +pub enum Expectation { + Outputs, + Rejected { category: String, phase: Phase }, + Declined(&'static OutOfDomain), +} + +/// One corpus case: its frozen reference entry and its input document. +pub struct Case { + pub name: String, + pub entry: Value, + pub input: Value, + pub input_from_sidecar: bool, +} + +#[derive(Deserialize)] +pub struct TypedInput { + pub window_weight: u128, + pub page_size: usize, + pub snapshot: TypedStep, + pub advances: Vec, +} + +#[derive(Deserialize)] +pub struct TypedStep { + pub anchor_job_issued_at_ms: i64, + pub records: Vec, +} + +impl Case { + pub fn advances(&self) -> &[Value] { + self.input["advances"] + .as_array() + .expect("input advances is an array") + } + + pub fn out_of_domain(&self) -> Option<&'static OutOfDomain> { + OUT_OF_DOMAIN.iter().find(|entry| entry.case == self.name) + } + + pub fn expectation(&self) -> Expectation { + if let Some(declined) = self.out_of_domain() { + return Expectation::Declined(declined); + } + match self.entry.get("rejected").and_then(Value::as_str) { + None => Expectation::Outputs, + Some(category) => { + let phase = if FULL_REJECTIONS.contains(&category) { + Phase::Full + } else { + // The frozen entry records only the category: the phase + // is derived from the input. `assert_corpus_shape` holds + // every advance rejection to exactly one advance, so the + // refused step is that one and the full fold must succeed. + Phase::Advance(self.advances().len() - 1) + }; + Expectation::Rejected { + category: category.to_string(), + phase, + } + } + } + } + + /// The anchor of the case's final step, where its frozen window stands. + pub fn final_anchor(&self) -> &Value { + self.advances() + .last() + .map_or(&self.input["snapshot"]["anchor_job_issued_at_ms"], |step| { + &step["anchor_job_issued_at_ms"] + }) + } + + /// The spool tail bytes the frozen entry vouches for: the pinned literal, + /// or else `computed` when it hashes to the frozen `spool_tail_sha256`. + pub fn frozen_spool_tail(&self, computed: &[u8]) -> Option> { + let literal = self + .entry + .get("pinned_literals") + .and_then(|literals| literals.get("spool_tail")) + .and_then(Value::as_str); + match literal { + Some(literal) => Some(literal.as_bytes().to_vec()), + None => (self.entry["spool_tail_sha256"].as_str() + == Some(sha256_hex(computed).as_str())) + .then(|| computed.to_vec()), + } + } + + /// The typed parse of the input at 3.x.x's declared widths -- the same + /// strict serde parse the daemon applies to its request line. + pub fn typed(&self) -> Result { + serde_json::from_str(&python_json(&self.input)) + } + + /// The `full` prepare_window request exactly as the daemon receives it. + pub fn full_request(&self) -> Value { + json!({ + "request": "prepare_window", + "mode": "full", + "append_invalidation_epoch": 0, + "anchor_job_issued_at_ms": self.input["snapshot"]["anchor_job_issued_at_ms"].clone(), + "window_weight": self.input["window_weight"].clone(), + "page_size": self.input["page_size"].clone(), + "records": self.input["snapshot"]["records"].clone(), + }) + } + + pub fn advance_request(&self, index: usize, base_digest: &str) -> Value { + let step = &self.advances()[index]; + json!({ + "request": "prepare_window", + "mode": "advance", + "append_invalidation_epoch": 0, + "anchor_job_issued_at_ms": step["anchor_job_issued_at_ms"].clone(), + "base_digest": base_digest, + "records": step["records"].clone(), + }) + } +} + +/// Load both files, check the pins and every input's `input_sha256`, and +/// assert the corpus shape, before any output is compared. +pub fn load() -> Vec { + assert_eq!( + sha256_hex(REFERENCE_JSON.as_bytes()), + REFERENCE_SHA256, + "tests/fixtures/window_pipeline_parity/reference.json no longer matches its pin; \ + the frozen corpus changes only in a reviewed commit that updates the pin" + ); + let reference: Value = serde_json::from_str(REFERENCE_JSON).expect("reference.json parses"); + assert_eq!(reference["schema"], REFERENCE_SCHEMA); + assert_eq!(reference["default_page_size"], 512); + assert_eq!(reference["master_seed"], "0xb17e5eed"); + let sidecar: Value = + serde_json::from_str(UNPINNED_INPUTS_JSON).expect("inputs-unpinned.json parses"); + assert_eq!(sidecar["schema"], UNPINNED_SCHEMA); + assert_eq!(sidecar["source_commit"], ORACLE_SOURCE_COMMIT); + let sidecar_cases = sidecar["cases"] + .as_object() + .expect("sidecar cases is an object"); + let entries = reference["cases"] + .as_object() + .expect("reference cases is an object"); + + let mut cases = Vec::with_capacity(entries.len()); + let mut sidecar_used = BTreeSet::new(); + for (name, entry) in entries { + let pinned = entry + .get("pinned_literals") + .and_then(|literals| literals.get("input")); + let (input, input_from_sidecar) = match (pinned, sidecar_cases.get(name)) { + (Some(input), None) => (input.clone(), false), + (None, Some(input)) => { + sidecar_used.insert(name.as_str()); + (input.clone(), true) + } + (Some(_), Some(_)) => panic!("{name}: input is both pinned and in the sidecar"), + (None, None) => panic!("{name}: input is neither pinned nor in the sidecar"), + }; + assert_eq!(input["schema"], INPUT_SCHEMA, "{name}: input schema"); + assert_eq!(input["name"], name.as_str(), "{name}: input name"); + cases.push(Case { + name: name.clone(), + entry: entry.clone(), + input, + input_from_sidecar, + }); + } + + // input_sha256 first: a changed input invalidates every output check. + assert_input_hashes(&cases); + assert_corpus_shape(&cases, &sidecar_used, sidecar_cases.len()); + assert_integer_domain(&cases); + cases +} + +/// Load the supplementary cases, check their pin and every input's +/// `input_sha256`, and assert their shape, before any output is compared. +pub fn load_supplementary() -> Vec { + assert_eq!( + sha256_hex(SUPPLEMENTARY_JSON.as_bytes()), + SUPPLEMENTARY_SHA256, + "tests/fixtures/window_pipeline_parity/supplementary.json no longer matches its pin; \ + it changes only in a reviewed commit that re-exports it and updates the pin" + ); + let document: Value = + serde_json::from_str(SUPPLEMENTARY_JSON).expect("supplementary.json parses"); + assert_eq!(document["schema"], SUPPLEMENTARY_SCHEMA); + assert_eq!(document["source_commit"], ORACLE_SOURCE_COMMIT); + let entries = document["cases"] + .as_object() + .expect("supplementary cases is an object"); + let mut names: Vec<&str> = entries.keys().map(String::as_str).collect(); + names.sort_unstable(); + assert_eq!(names, SUPPLEMENTARY_CASES, "supplementary case names"); + + let cases: Vec = entries + .iter() + .map(|(name, entry)| { + let input = entry["pinned_literals"]["input"].clone(); + assert_eq!(input["schema"], INPUT_SCHEMA, "{name}: input schema"); + assert_eq!(input["name"], name.as_str(), "{name}: input name"); + assert!( + entry.get("rejected").is_none(), + "{name}: the oracle accepted every supplementary case" + ); + Case { + name: name.clone(), + entry: entry.clone(), + input, + input_from_sidecar: false, + } + }) + .collect(); + assert_input_hashes(&cases); + for case in &cases { + assert!( + prepare_window_out_of_range(&case.full_request()).is_none(), + "{}: supplementary inputs lie inside the declared widths", + case.name + ); + assert!( + !case.advances().is_empty(), + "{}: every supplementary case advances", + case.name + ); + } + cases +} + +fn assert_input_hashes(cases: &[Case]) { + let input_mismatches: Vec = cases + .iter() + .filter_map(|case| { + let recomputed = sha256_hex(python_json(&case.input).as_bytes()); + let frozen = case.entry["input_sha256"].as_str().unwrap_or(""); + (recomputed != frozen).then(|| { + format!( + "{}: input_sha256 frozen {frozen}, recomputed {recomputed}{}", + case.name, + if case.input_from_sidecar { + " (input from inputs-unpinned.json)" + } else { + " (pinned input)" + } + ) + }) + }) + .collect(); + assert!( + input_mismatches.is_empty(), + "{} corpus input(s) no longer reproduce their frozen input_sha256:\n{}", + input_mismatches.len(), + input_mismatches.join("\n") + ); +} + +fn assert_corpus_shape(cases: &[Case], sidecar_used: &BTreeSet<&str>, sidecar_len: usize) { + assert_eq!(cases.len(), 35, "corpus case count"); + let rejected: Vec<&Case> = cases + .iter() + .filter(|case| case.entry.get("rejected").is_some()) + .collect(); + assert_eq!(rejected.len(), 12, "rejection case count"); + assert_eq!(cases.len() - rejected.len(), 23, "success case count"); + let pinned = cases.iter().filter(|case| !case.input_from_sidecar).count(); + assert_eq!(pinned, 30, "pinned input count"); + assert_eq!( + sidecar_used.iter().copied().collect::>(), + UNPINNED_CASES, + "sidecar supplies exactly the five unpinned inputs" + ); + assert_eq!( + sidecar_len, + UNPINNED_CASES.len(), + "sidecar holds no extra cases" + ); + + let categories: BTreeSet<&str> = rejected + .iter() + .map(|case| { + case.entry["rejected"] + .as_str() + .expect("category is a string") + }) + .collect(); + let vocabulary: BTreeSet<&str> = FULL_REJECTIONS + .iter() + .chain(ADVANCE_REJECTIONS.iter()) + .copied() + .collect(); + assert_eq!( + categories, vocabulary, + "rejection categories cover the vocabulary" + ); + for case in &rejected { + let category = case.entry["rejected"].as_str().unwrap(); + if FULL_REJECTIONS.contains(&category) { + assert!( + case.advances().is_empty(), + "{}: full rejection with advances", + case.name + ); + } else { + // `Case::expectation` derives the refused step from this. + assert_eq!( + case.advances().len(), + 1, + "{}: advance rejection with other than exactly one advance", + case.name + ); + } + } + + // The frozen advance_stats is one entry per advance, so it is empty + // exactly when the case has no advances. + let mut with_stats = 0; + for case in cases + .iter() + .filter(|case| case.entry.get("rejected").is_none()) + { + for field in [ + "canonical_bytes_len", + "canonical_bytes_sha256", + "canonical_digest", + "record_count", + "record_stream_len", + "record_stream_sha256", + "spool_tail_len", + "spool_tail_sha256", + ] { + assert!( + !case.entry[field].is_null(), + "{}: frozen {field} missing", + case.name + ); + } + let stats = case.entry["advance_stats"] + .as_array() + .unwrap_or_else(|| panic!("{}: advance_stats is not a list", case.name)); + assert_eq!( + stats.len(), + case.advances().len(), + "{}: advance_stats length", + case.name + ); + if !stats.is_empty() { + with_stats += 1; + } + } + assert_eq!(with_stats, 8, "cases with advance_stats"); +} + +/// 3.x.x's width classifier agrees with the two-entry table on all 35 inputs. +fn assert_integer_domain(cases: &[Case]) { + let mut declined = 0; + for case in cases { + let found = prepare_window_out_of_range(&case.full_request()); + match (case.out_of_domain(), found) { + (None, None) => {} + (Some(expected), Some(found)) => { + declined += 1; + assert_eq!( + found.field, expected.field, + "{}: out-of-range field", + case.name + ); + assert_eq!( + found.literal, expected.literal, + "{}: out-of-range literal", + case.name + ); + assert_eq!( + found.width, expected.width, + "{}: out-of-range width", + case.name + ); + } + (expected, found) => panic!( + "{}: integer-domain classification differs from the table: table {:?}, \ + prepare_window_out_of_range {found:?}", + case.name, + expected.map(|entry| (entry.field, entry.width.name())) + ), + } + } + assert_eq!(declined, OUT_OF_DOMAIN.len(), "out-of-domain case count"); +} + +/// Byte outputs of one folded window, from either replay. +pub struct Outputs { + pub record_count: usize, + /// The canonical items stream, without the enclosing brackets. + pub canonical_items: Vec, + pub canonical_digest: String, + /// Per-record canonical fragments, in window order. + pub fragments: Vec>, + pub spool_tail: Vec, + /// `[added_rows, expired_rows, touched_pages]` per advance. + pub advance_stats: Vec<[usize; 3]>, +} + +pub enum Outcome { + Outputs(Box), + Rejected { + phase: Phase, + category: Option, + message: String, + }, + Declined { + field: String, + width: String, + detail: String, + }, +} + +impl Outcome { + fn describe(&self) -> String { + match self { + Outcome::Outputs(outputs) => format!("a window of {} records", outputs.record_count), + Outcome::Rejected { + phase, + category, + message, + } => format!("rejection {category:?} at {phase} ({message})"), + Outcome::Declined { + field, + width, + detail, + } => format!("decline of {field} at {width} ({detail})"), + } + } +} + +impl Expectation { + fn describe(&self) -> String { + match self { + Expectation::Outputs => "a window".to_string(), + Expectation::Rejected { category, phase } => format!("rejection {category} at {phase}"), + Expectation::Declined(entry) => { + format!("decline of {} at {}", entry.field, entry.width.name()) + } + } + } +} + +#[derive(Debug, Default, Clone, Copy, PartialEq, Eq)] +pub struct Tally { + pub cases: usize, + pub rejections: usize, + pub byte_compared: usize, + pub declined: usize, + pub with_advance_stats: usize, +} + +impl fmt::Display for Tally { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + write!( + formatter, + "{} cases, {} rejections, {} byte-compared, {} declined, {} with advance stats", + self.cases, self.rejections, self.byte_compared, self.declined, self.with_advance_stats + ) + } +} + +/// Every mismatch across the corpus, named by case and field, reported once. +#[derive(Default)] +pub struct Mismatches(Vec); + +impl Mismatches { + pub fn push(&mut self, case: &str, field: &str, detail: impl fmt::Display) { + self.0.push(format!("{case}: {field}: {detail}")); + } + + pub fn check( + &mut self, + case: &str, + field: &str, + expected: T, + actual: T, + ) { + if expected != actual { + self.push(case, field, format!("frozen {expected:?}, got {actual:?}")); + } + } + + /// Compare exact bytes, reporting the first differing offset with context. + pub fn check_bytes(&mut self, case: &str, field: &str, expected: &[u8], actual: &[u8]) { + if expected == actual { + return; + } + let offset = expected + .iter() + .zip(actual) + .position(|(left, right)| left != right) + .unwrap_or(expected.len().min(actual.len())); + let context = |bytes: &[u8]| { + let start = offset.saturating_sub(24); + let end = (offset + 40).min(bytes.len()); + String::from_utf8_lossy(&bytes[start.min(end)..end]).into_owned() + }; + self.push( + case, + field, + format!( + "frozen {} bytes, got {} bytes; first difference at byte {offset}: \ + frozen {:?}, got {:?}", + expected.len(), + actual.len(), + context(expected), + context(actual) + ), + ); + } + + /// Settle one case: its outcome against the corpus expectation. + pub fn settle(&mut self, case: &Case, outcome: Outcome, tally: &mut Tally) { + tally.cases += 1; + let name = case.name.as_str(); + match (case.expectation(), outcome) { + (Expectation::Outputs, Outcome::Outputs(outputs)) => { + tally.byte_compared += 1; + if !outputs.advance_stats.is_empty() { + tally.with_advance_stats += 1; + } + self.compare_outputs(case, &outputs); + } + ( + Expectation::Rejected { + category: expected, + phase: expected_phase, + }, + Outcome::Rejected { + phase, category, .. + }, + ) => { + tally.rejections += 1; + self.check(name, "rejection phase", expected_phase, phase); + self.check(name, "rejection category", Some(expected), category); + } + (Expectation::Declined(entry), Outcome::Declined { field, width, .. }) => { + tally.declined += 1; + self.check(name, "declined field", entry.field, field.as_str()); + self.check(name, "declined width", entry.width.name(), width.as_str()); + } + (expected, outcome) => self.push( + name, + "outcome", + format!( + "expected {}, got {}", + expected.describe(), + outcome.describe() + ), + ), + } + } + + fn compare_outputs(&mut self, case: &Case, outputs: &Outputs) { + let name = case.name.as_str(); + let entry = &case.entry; + let frozen_usize = |field: &str| entry[field].as_u64().map(|value| value as usize); + let frozen_str = |field: &str| entry[field].as_str().unwrap_or(""); + let pinned = |field: &str| entry.get("pinned_literals").and_then(|l| l.get(field)); + + self.check( + name, + "record_count", + frozen_usize("record_count"), + Some(outputs.record_count), + ); + + let mut canonical = Vec::with_capacity(outputs.canonical_items.len() + 2); + canonical.push(b'['); + canonical.extend_from_slice(&outputs.canonical_items); + canonical.push(b']'); + self.check( + name, + "canonical_bytes_len", + frozen_usize("canonical_bytes_len"), + Some(canonical.len()), + ); + self.check( + name, + "canonical_bytes_sha256", + frozen_str("canonical_bytes_sha256"), + sha256_hex(&canonical).as_str(), + ); + self.check( + name, + "canonical_digest", + frozen_str("canonical_digest"), + outputs.canonical_digest.as_str(), + ); + if let Some(literal) = pinned("canonical_bytes").and_then(Value::as_str) { + self.check_bytes( + name, + "pinned canonical_bytes", + literal.as_bytes(), + &canonical, + ); + } + let stream = outputs.fragments.join(&b"\n"[..]); + self.check( + name, + "record_stream_len", + frozen_usize("record_stream_len"), + Some(stream.len()), + ); + self.check( + name, + "record_stream_sha256", + frozen_str("record_stream_sha256"), + sha256_hex(&stream).as_str(), + ); + if let Some(literals) = pinned("record_jsons").and_then(Value::as_array) { + self.check( + name, + "pinned record_jsons count", + literals.len(), + outputs.fragments.len(), + ); + for (index, (literal, fragment)) in literals.iter().zip(&outputs.fragments).enumerate() + { + self.check_bytes( + name, + &format!("pinned record_jsons[{index}]"), + literal.as_str().unwrap_or_default().as_bytes(), + fragment, + ); + } + } + + self.check( + name, + "spool_tail_len", + frozen_usize("spool_tail_len"), + Some(outputs.spool_tail.len()), + ); + self.check( + name, + "spool_tail_sha256", + frozen_str("spool_tail_sha256"), + sha256_hex(&outputs.spool_tail).as_str(), + ); + if let Some(literal) = pinned("spool_tail").and_then(Value::as_str) { + self.check_bytes( + name, + "pinned spool_tail", + literal.as_bytes(), + &outputs.spool_tail, + ); + } + + let frozen_stats: Option> = + entry["advance_stats"].as_array().map(|steps| { + steps + .iter() + .map(|step| { + ["added_rows", "expired_rows", "touched_pages"] + .map(|field| step[field].as_u64().unwrap_or(u64::MAX) as usize) + }) + .collect() + }); + self.check( + name, + "advance_stats", + frozen_stats, + Some(outputs.advance_stats.clone()), + ); + } + + pub fn finish(self, what: &str, tally: Tally, expected: Tally) { + println!("{what}: {tally}"); + let mut mismatches = self.0; + if tally != expected { + mismatches.push(format!("corpus: tally: expected {expected}, got {tally}")); + } + assert!( + mismatches.is_empty(), + "{what}: {} mismatch(es) against the frozen corpus:\n{}", + mismatches.len(), + mismatches.join("\n") + ); + } +} + +pub fn sha256_hex(bytes: &[u8]) -> String { + hex::encode(Sha256::digest(bytes)) +} + +/// CPython's `json.dumps(value, sort_keys=True, separators=(",", ":"))`, +/// `ensure_ascii` included. Integer literals pass through verbatim +/// (`arbitrary_precision`), and keys are sorted here rather than relying on +/// serde_json's map order, which a `preserve_order` feature would change. +pub fn python_json(value: &Value) -> String { + let mut out = String::new(); + push_python_json(&mut out, value); + out +} + +fn push_python_json(out: &mut String, value: &Value) { + match value { + Value::Null => out.push_str("null"), + Value::Bool(flag) => out.push_str(if *flag { "true" } else { "false" }), + Value::Number(number) => out.push_str(number.as_str()), + Value::String(text) => push_python_string(out, text), + Value::Array(items) => { + out.push('['); + for (index, item) in items.iter().enumerate() { + if index > 0 { + out.push(','); + } + push_python_json(out, item); + } + out.push(']'); + } + Value::Object(map) => { + let mut keys: Vec<&String> = map.keys().collect(); + keys.sort(); + out.push('{'); + for (index, key) in keys.into_iter().enumerate() { + if index > 0 { + out.push(','); + } + push_python_string(out, key); + out.push(':'); + push_python_json(out, &map[key]); + } + out.push('}'); + } + } +} + +/// A test-only copy of `window.rs`'s private `push_python_json_escaped`: +/// everything outside 0x20..=0x7e escapes as lowercase `\uXXXX`, non-BMP as +/// a UTF-16 surrogate pair, exactly like CPython's `ensure_ascii` encoder. +fn push_python_string(out: &mut String, value: &str) { + out.push('"'); + for character in value.chars() { + match character { + '"' => out.push_str("\\\""), + '\\' => out.push_str("\\\\"), + '\u{08}' => out.push_str("\\b"), + '\t' => out.push_str("\\t"), + '\n' => out.push_str("\\n"), + '\u{0c}' => out.push_str("\\f"), + '\r' => out.push_str("\\r"), + ' '..='~' => out.push(character), + _ => { + let mut units = [0u16; 2]; + for unit in character.encode_utf16(&mut units) { + out.push_str(&format!("\\u{unit:04x}")); + } + } + } + } + out.push('"'); +} + +/// The per-record canonical fragments joined by `\n`. +pub fn record_stream(fragments: &[Vec]) -> Vec { + fragments.join(&b"\n"[..]) +} + +/// The frozen compact build-request tail the 2.x.x coordinator spooled -- +/// `,"compact_share_identities":,"compact_shares":}` with compact +/// separators and `ensure_ascii` -- a shape the daemon's build request still +/// accepts. Identities deduplicate in first-seen order; one row per record. +pub fn spool_tail(shares: &[AcceptedShare]) -> Vec { + let mut identity_indexes: HashMap<(&str, &str, &str), usize> = HashMap::new(); + let mut identities = String::new(); + let mut rows = String::new(); + for share in shares { + let identity = ( + share.miner_id.as_str(), + share.order_key.as_str(), + share.p2mr_program_hex.as_str(), + ); + let next_index = identity_indexes.len(); + let identity_index = *identity_indexes.entry(identity).or_insert_with(|| { + if next_index > 0 { + identities.push(','); + } + identities.push('['); + push_python_string(&mut identities, identity.0); + identities.push(','); + push_python_string(&mut identities, identity.1); + identities.push(','); + push_python_string(&mut identities, identity.2); + identities.push(']'); + next_index + }); + if !rows.is_empty() { + rows.push(','); + } + rows.push_str(&format!("[{},", share.share_seq)); + push_python_string(&mut rows, &share.share_id); + rows.push_str(&format!( + ",{identity_index},{},{},{},", + share.share_difficulty, share.job_issued_at_ms, share.accepted_at_ms + )); + match &share.credit_policy { + Some(policy) => push_python_string(&mut rows, policy), + None => rows.push_str("null"), + } + rows.push(']'); + } + format!(",\"compact_share_identities\":[{identities}],\"compact_shares\":[{rows}]}}") + .into_bytes() +} diff --git a/crates/qbit-prism/tests/window_daemon_gate.rs b/crates/qbit-prism/tests/window_daemon_gate.rs new file mode 100644 index 000000000..328843bf4 --- /dev/null +++ b/crates/qbit-prism/tests/window_daemon_gate.rs @@ -0,0 +1,510 @@ +//! The frozen payout-window corpus (#269) replayed through the real +//! `qbit-prism-build-audit-bundle --serve` daemon, one fresh process per +//! case: a `full` prepare_window, then one `advance` per delta, with the +//! coordinator's opaque items mirror rebuilt from the raw sections exactly as +//! the coordinator keeps it and checked against every step's digest. +//! Requests are written as JSON text lines from the input documents, so +//! literals wider than any Rust integer reach the daemon verbatim and must be +//! declined as `out_of_range`. +//! +//! Every non-empty window's frozen compact spool tail is then uploaded to the +//! same daemon as a build request, so the daemon's own decoder consumes the +//! frozen bytes; its build summary must equal the one built from the window +//! the daemon prepared from the full records. + +#[path = "support/window_corpus.rs"] +mod window_corpus; + +use qbit_prism::AcceptedShare; +use serde_json::value::RawValue; +use serde_json::Value; +use std::io::{BufRead, BufReader, Read, Write}; +use std::process::{Child, Command, Stdio}; +use std::sync::mpsc::{self, Receiver, Sender}; +use std::thread::{self, JoinHandle}; +use std::time::{Duration, Instant}; +use window_corpus::{ + python_json, sha256_hex, spool_tail, Case, Expectation, Mismatches, Outcome, Outputs, Phase, + Tally, +}; + +/// Per response; a hung daemon fails the case instead of hanging CI. +const RESPONSE_TIMEOUT: Duration = Duration::from_secs(60); +/// After stdin closes; a daemon with nothing left to read exits at once. +const EXIT_TIMEOUT: Duration = Duration::from_secs(10); +/// Window key of the build that uploads the frozen spool tail. It is no +/// content digest, so the daemon decodes the upload rather than serving the +/// prepared window it already holds. +const SPOOL_UPLOAD_KEY: &str = "frozen-spool-tail"; + +enum ReadRequest { + Line, + Exact(usize), +} + +struct Daemon { + child: Child, + stdin: Option>>, + reads: Sender, + responses: Receiver>>, + stderr: Option>, +} + +impl Daemon { + fn spawn() -> Daemon { + let mut child = Command::new(env!("CARGO_BIN_EXE_qbit-prism-build-audit-bundle")) + .arg("--serve") + .arg("--signing-key-seed-hex") + .arg("42".repeat(32)) + .arg("--ledger-signing-key-seed-hex") + .arg("43".repeat(32)) + .stdin(Stdio::piped()) + .stdout(Stdio::piped()) + .stderr(Stdio::piped()) + .spawn() + .expect("spawn the serve daemon"); + + // Writes and reads each run on a helper thread so the main thread + // only ever waits with a deadline. + let mut stdin = child.stdin.take().unwrap(); + let (stdin_tx, stdin_rx) = mpsc::channel::>(); + thread::spawn(move || { + for bytes in stdin_rx { + if stdin + .write_all(&bytes) + .and_then(|()| stdin.flush()) + .is_err() + { + break; + } + } + }); + let mut stdout = BufReader::new(child.stdout.take().unwrap()); + let (reads_tx, reads_rx) = mpsc::channel::(); + let (responses_tx, responses_rx) = mpsc::channel(); + thread::spawn(move || { + for request in reads_rx { + let result = match request { + ReadRequest::Line => { + let mut line = Vec::new(); + stdout.read_until(b'\n', &mut line).map(|_| line) + } + ReadRequest::Exact(len) => { + let mut bytes = vec![0u8; len]; + stdout.read_exact(&mut bytes).map(|()| bytes) + } + }; + if responses_tx.send(result).is_err() { + break; + } + } + }); + let mut stderr = child.stderr.take().unwrap(); + let stderr = thread::spawn(move || { + let mut text = String::new(); + let _ = stderr.read_to_string(&mut text); + text + }); + Daemon { + child, + stdin: Some(stdin_tx), + reads: reads_tx, + responses: responses_rx, + stderr: Some(stderr), + } + } + + fn read(&self, request: ReadRequest) -> Result, String> { + self.reads + .send(request) + .map_err(|_| "daemon stdout reader is gone".to_string())?; + match self.responses.recv_timeout(RESPONSE_TIMEOUT) { + Ok(Ok(bytes)) => Ok(bytes), + Ok(Err(error)) => Err(format!("reading daemon stdout: {error}")), + Err(_) => Err(format!("no daemon response within {RESPONSE_TIMEOUT:?}")), + } + } + + fn read_json_line(&self) -> Result { + let line = self.read(ReadRequest::Line)?; + if line.is_empty() { + return Err("daemon closed stdout".to_string()); + } + serde_json::from_slice(&line) + .map_err(|error| format!("{error}: {}", String::from_utf8_lossy(&line))) + } + + /// Send one request line exactly as given and read its response line. + fn send_line(&self, mut line: Vec) -> Result { + line.push(b'\n'); + self.stdin + .as_ref() + .expect("stdin is open") + .send(line) + .map_err(|_| "daemon stdin writer is gone".to_string())?; + self.read_json_line() + } + + fn request(&self, request: &Value) -> Result { + self.send_line(python_json(request).into_bytes()) + } + + /// Read one raw canonical-items section and its terminating newline. + fn read_section(&self, len: usize) -> Result, String> { + let mut bytes = self.read(ReadRequest::Exact(len + 1))?; + if bytes.pop() != Some(b'\n') { + return Err(format!( + "raw section of {len} bytes is not newline-terminated" + )); + } + Ok(bytes) + } + + /// Close stdin and require a clean exit within the timeout. + fn finish(mut self) -> Result<(), String> { + drop(self.stdin.take()); + let deadline = Instant::now() + EXIT_TIMEOUT; + let status = loop { + match self.child.try_wait() { + Ok(Some(status)) => break status, + Ok(None) if Instant::now() < deadline => thread::sleep(Duration::from_millis(10)), + Ok(None) => return Err("daemon did not exit after stdin closed".to_string()), + Err(error) => return Err(format!("waiting for the daemon: {error}")), + } + }; + let stderr = self + .stderr + .take() + .and_then(|handle| handle.join().ok()) + .unwrap_or_default(); + if status.success() { + Ok(()) + } else { + Err(format!("daemon exited with {status}; stderr: {stderr}")) + } + } +} + +impl Drop for Daemon { + fn drop(&mut self) { + let _ = self.child.kill(); + let _ = self.child.wait(); + } +} + +fn usize_field(response: &Value, field: &str) -> Result { + response[field] + .as_u64() + .map(|value| value as usize) + .ok_or_else(|| format!("response has no {field}: {response}")) +} + +fn str_field(response: &Value, field: &str) -> Result { + response[field] + .as_str() + .map(str::to_string) + .ok_or_else(|| format!("response has no {field}: {response}")) +} + +/// A response that is not `ok`: the three distinguishable prepare_window +/// outcomes, or a failure. +fn refusal(response: &Value, phase: Phase) -> Result { + let message = response["error"].as_str().unwrap_or_default().to_string(); + let category = response["rejection"].as_str().map(str::to_string); + if response["out_of_range"] == true { + return Ok(Outcome::Declined { + field: str_field(response, "field")?, + width: str_field(response, "width")?, + detail: message, + }); + } + let expected_flag = match phase { + Phase::Full => "fold_invalid", + Phase::Advance(_) => "fallback", + }; + if response[expected_flag] == true { + return Ok(Outcome::Rejected { + phase, + category, + message, + }); + } + Err(format!("unexpected response at {phase}: {response}")) +} + +/// The coordinator trusts its mirror only while it hashes to the digest the +/// daemon reports; hold every step to that. +fn check_mirror_digest( + mismatches: &mut Mismatches, + case: &Case, + step: Phase, + items: &[u8], + digest: &str, +) { + let mut framed = Vec::with_capacity(items.len() + 2); + framed.push(b'['); + framed.extend_from_slice(items); + framed.push(b']'); + let mirrored = sha256_hex(&framed); + if mirrored != digest { + mismatches.push( + &case.name, + &format!("{step} mirror digest"), + format!("share_snapshot_sha256 {digest}, sha256 of the mirrored items {mirrored}"), + ); + } +} + +fn exchange(daemon: &Daemon, case: &Case, mismatches: &mut Mismatches) -> Result { + let handshake = daemon.read_json_line()?; + if handshake["event"] != "handshake" || handshake["protocol"] != 2 { + return Err(format!("unexpected handshake: {handshake}")); + } + + let full = daemon.request(&case.full_request())?; + if full["ok"] != true { + return refusal(&full, Phase::Full); + } + let mut items = daemon.read_section(usize_field(&full, "window_items_len")?)?; + let mut digest = str_field(&full, "share_snapshot_sha256")?; + let mut record_count = usize_field(&full, "record_count")?; + check_mirror_digest(mismatches, case, Phase::Full, &items, &digest); + + let mut advance_stats = Vec::new(); + for index in 0..case.advances().len() { + let response = daemon.request(&case.advance_request(index, &digest))?; + if response["ok"] != true { + return refusal(&response, Phase::Advance(index)); + } + let appended = daemon.read_section(usize_field(&response, "appended_items_len")?)?; + let drop_bytes = usize_field(&response, "retained_drop_bytes")?; + if drop_bytes > items.len() { + return Err(format!( + "advances[{index}] retained_drop_bytes {drop_bytes} exceeds the {}-byte mirror", + items.len() + )); + } + items.drain(..drop_bytes); + items.extend_from_slice(&appended); + digest = str_field(&response, "share_snapshot_sha256")?; + record_count = usize_field(&response, "record_count")?; + check_mirror_digest(mismatches, case, Phase::Advance(index), &items, &digest); + advance_stats.push([ + usize_field(&response, "added_rows")?, + usize_field(&response, "expired_rows")?, + usize_field(&response, "touched_pages")?, + ]); + } + + // Split the mirrored stream into its records straight from the daemon's + // bytes: RawValue keeps each element's exact span. + let mut canonical = Vec::with_capacity(items.len() + 2); + canonical.push(b'['); + canonical.extend_from_slice(&items); + canonical.push(b']'); + let elements: Vec> = serde_json::from_slice(&canonical) + .map_err(|error| format!("mirrored items stream does not parse: {error}"))?; + let shares: Vec = elements + .iter() + .map(|element| serde_json::from_str(element.get())) + .collect::>() + .map_err(|error| format!("mirrored record does not parse: {error}"))?; + Ok(Outcome::Outputs(Box::new(Outputs { + record_count, + canonical_items: items, + canonical_digest: digest, + fragments: elements + .iter() + .map(|element| element.get().as_bytes().to_vec()) + .collect(), + spool_tail: spool_tail(&shares), + advance_stats, + }))) +} + +fn excerpt(value: &Value) -> String { + let text = value.to_string(); + match text.char_indices().nth(160) { + Some((cut, _)) => format!("{}...", &text[..cut]), + None => text, + } +} + +/// The first JSON path at which two build summaries differ. +fn first_difference(prepared: &Value, decoded: &Value, path: &str) -> Option { + match (prepared, decoded) { + (Value::Object(left), Value::Object(right)) => { + left.keys().chain(right.keys()).find_map(|key| { + first_difference( + left.get(key).unwrap_or(&Value::Null), + right.get(key).unwrap_or(&Value::Null), + &format!("{path}.{key}"), + ) + }) + } + (Value::Array(left), Value::Array(right)) if left.len() == right.len() => left + .iter() + .zip(right) + .enumerate() + .find_map(|(index, (left, right))| { + first_difference(left, right, &format!("{path}[{index}]")) + }), + _ if prepared == decoded => None, + _ => Some(format!( + "{path}: prepared window {}, frozen spool tail {}", + excerpt(prepared), + excerpt(decoded) + )), + } +} + +/// Build the case's window twice in the same daemon and require identical +/// summaries: once from the window the daemon prepared from the full records, +/// once from the frozen compact spool tail through the daemon's own upload +/// decoder. The summary's reward manifest lists every counted share with +/// each field the compact format carries (`share_seq`, `share_id`, +/// `miner_id`, `order_key`, `p2mr_program_hex`, `share_difficulty`, +/// `job_issued_at_ms`, `accepted_at_ms`, `credit_policy`) and none it does +/// not, so a decoder that drops, swaps or misindexes a field changes it. +/// Returns false for an empty window, which has no compact rows to upload. +fn check_spool_decoder(daemon: &Daemon, case: &Case, outputs: &Outputs) -> Result { + if outputs.record_count == 0 { + return Ok(false); + } + let tail = case.frozen_spool_tail(&outputs.spool_tail).ok_or( + "the entry pins no spool_tail literal and the computed tail does not hash to the \ + frozen spool_tail_sha256", + )?; + let shares: Vec = outputs + .fragments + .iter() + .map(|fragment| serde_json::from_slice(fragment)) + .collect::>() + .map_err(|error| format!("mirrored record does not parse: {error}"))?; + let oldest = shares + .iter() + .min_by_key(|share| share.share_seq) + .expect("a non-empty window has an oldest share"); + let newer_difficulty = shares + .iter() + .filter(|share| share.share_seq != oldest.share_seq) + .try_fold(0u128, |sum, share| sum.checked_add(share.share_difficulty)) + .ok_or("window difficulty total leaves u128")?; + // The final anchor admits every retained share. A requested weight + // (eight times the network difficulty) just above the newer shares' + // total counts each of them in full and the oldest one at least in + // part, so every share is listed, while the payout arithmetic stays far + // inside u128 even when the oldest share is 2^127 wide. + let found_block = format!( + "{{\"anchor_job_issued_at_ms\":{},\"block_height\":1,\ + \"coinbase_value_sats\":5000000000,\"network_difficulty\":{}}}", + python_json(case.final_anchor()), + newer_difficulty / 8 + 1 + ); + + let prepared = daemon.send_line( + format!( + "{{\"found_block\":{found_block},\"window_key\":{{\"share_snapshot_sha256\":\"{}\"}}}}", + outputs.canonical_digest + ) + .into_bytes(), + )?; + if prepared["ok"] != true || prepared["window_cache"]["hit"] != true { + return Err(format!( + "build from the prepared window: {}", + excerpt(&prepared) + )); + } + // The frozen tail is the request line's own suffix, closing brace + // included, exactly as the 2.x.x coordinator spooled it. + let mut upload = format!( + "{{\"found_block\":{found_block},\"window_key\":{{\"share_snapshot_sha256\":\ + \"{SPOOL_UPLOAD_KEY}\"}}" + ) + .into_bytes(); + upload.extend_from_slice(&tail); + let decoded = daemon.send_line(upload)?; + if decoded["ok"] != true || decoded["window_cache"]["hit"] != false { + return Err(format!( + "build from the frozen spool tail: {}", + excerpt(&decoded) + )); + } + if prepared["summary"] == decoded["summary"] { + Ok(true) + } else { + Err( + first_difference(&prepared["summary"], &decoded["summary"], "summary") + .unwrap_or_else(|| "build summaries differ".to_string()), + ) + } +} + +fn replay_all(what: &str, cases: &[Case], expected: Tally) { + let mut mismatches = Mismatches::default(); + let mut tally = Tally::default(); + let mut spool_decoded = 0; + for case in cases { + let daemon = Daemon::spawn(); + let outcome = exchange(&daemon, case, &mut mismatches); + if let Ok(Outcome::Outputs(outputs)) = &outcome { + match check_spool_decoder(&daemon, case, outputs) { + Ok(true) => spool_decoded += 1, + Ok(false) => {} + Err(error) => mismatches.push(&case.name, "frozen spool tail decode", error), + } + } + // Every outcome, rejections and declines included, leaves a daemon + // that exits cleanly once stdin closes. + if let Err(error) = daemon.finish() { + mismatches.push(&case.name, "daemon exit", error); + } + match outcome { + Ok(outcome) => mismatches.settle(case, outcome, &mut tally), + Err(error) => { + tally.cases += 1; + mismatches.push(&case.name, "daemon exchange", error); + } + } + } + // Every frozen non-empty window's spool tail must reach the decoder. + let decodable = cases + .iter() + .filter(|case| { + matches!(case.expectation(), Expectation::Outputs) + && case.entry["record_count"] + .as_u64() + .is_some_and(|count| count > 0) + }) + .count(); + println!( + "{what}: {spool_decoded} of {decodable} frozen spool tails decoded by the daemon and \ + built identically to its prepared window" + ); + if spool_decoded != decodable { + mismatches.push( + "corpus", + "spool tails decoded", + format!("expected {decodable}, got {spool_decoded}"), + ); + } + mismatches.finish(what, tally, expected); +} + +#[test] +fn frozen_corpus_replays_through_the_serve_daemon() { + replay_all( + "window_daemon_gate", + &window_corpus::load(), + window_corpus::EXPECTED_TALLY, + ); +} + +#[test] +fn supplementary_cases_replay_through_the_serve_daemon() { + replay_all( + "window_daemon_gate supplementary", + &window_corpus::load_supplementary(), + window_corpus::EXPECTED_SUPPLEMENTARY_TALLY, + ); +} diff --git a/crates/qbit-prism/tests/window_frozen_vectors.rs b/crates/qbit-prism/tests/window_frozen_vectors.rs new file mode 100644 index 000000000..2de03ce12 --- /dev/null +++ b/crates/qbit-prism/tests/window_frozen_vectors.rs @@ -0,0 +1,159 @@ +//! Native replay of the frozen payout-window corpus (#269). +//! +//! Every case of `tests/fixtures/window_pipeline_parity/reference.json` -- +//! generated on 2.x.x by the Python oracle -- is folded and advanced with +//! `qbit_prism::window::PayoutWindow` and compared against the frozen bytes. +//! The two cases outside the declared integer widths must fail the typed +//! parse; see `support/window_corpus.rs`. The supplementary cases replay the +//! same way under their own tally. + +#[path = "support/window_corpus.rs"] +mod window_corpus; + +use qbit_prism::window::{canonical_share_fragment, prepare_window_out_of_range, PayoutWindow}; +use std::any::Any; +use std::panic::{catch_unwind, AssertUnwindSafe}; +use window_corpus::{spool_tail, Case, Mismatches, Outcome, Outputs, Phase, Tally}; + +fn replay(case: &Case, mismatches: &mut Mismatches) -> Outcome { + let typed = match case.typed() { + Ok(typed) => typed, + Err(error) => { + // Only an out-of-width integer may fail the typed parse; the + // classifier names it, and `settle` holds it to the table. + let found = prepare_window_out_of_range(&case.full_request()); + return match found { + Some(found) => Outcome::Declined { + field: found.field, + width: found.width.name().to_string(), + detail: error.to_string(), + }, + None => Outcome::Rejected { + phase: Phase::Full, + category: None, + message: format!("typed parse failed in domain: {error}"), + }, + }; + } + }; + + let mut window = match PayoutWindow::from_full_snapshot( + typed.snapshot.records, + typed.snapshot.anchor_job_issued_at_ms, + typed.window_weight, + typed.page_size, + ) { + Ok(window) => window, + Err(error) => { + return Outcome::Rejected { + phase: Phase::Full, + category: error.rejection_category().map(str::to_string), + message: error.to_string(), + } + } + }; + let mut advance_stats = Vec::with_capacity(typed.advances.len()); + for (index, step) in typed.advances.into_iter().enumerate() { + let previous_items = window.canonical_items_bytes(); + match window.advance(step.records, step.anchor_job_issued_at_ms) { + Ok((advanced, stats, byte_delta)) => { + // The coordinator's mirror surgery lands on the new stream. + let mut mirror = previous_items + .get(byte_delta.retained_drop_bytes..) + .unwrap_or_default() + .to_vec(); + if byte_delta.retained_drop_bytes > previous_items.len() { + mismatches.push( + &case.name, + &format!("advances[{index}] retained_drop_bytes"), + format!( + "{} exceeds the previous {} bytes", + byte_delta.retained_drop_bytes, + previous_items.len() + ), + ); + } + mirror.extend_from_slice(&byte_delta.appended_items); + mismatches.check_bytes( + &case.name, + &format!("advances[{index}] byte delta"), + &advanced.canonical_items_bytes(), + &mirror, + ); + advance_stats.push([stats.added_rows, stats.expired_rows, stats.touched_pages]); + window = advanced; + } + Err(error) => { + return Outcome::Rejected { + phase: Phase::Advance(index), + category: error.rejection_category().map(str::to_string), + message: error.to_string(), + } + } + } + } + + let shares = window.shares_for_build(); + let canonical_items = window.canonical_items_bytes(); + let fragments: Vec> = shares.iter().map(canonical_share_fragment).collect(); + // Fragments encoded record by record must be the window's own items + // stream, exactly. + mismatches.check_bytes( + &case.name, + "fragments joined by ','", + &canonical_items, + &fragments.join(&b","[..]), + ); + Outcome::Outputs(Box::new(Outputs { + record_count: window.record_count(), + canonical_items, + canonical_digest: window.canonical_digest_hex(), + fragments, + spool_tail: spool_tail(&shares), + advance_stats, + })) +} + +fn panic_message(payload: &(dyn Any + Send)) -> String { + payload + .downcast_ref::<&str>() + .map(|message| message.to_string()) + .or_else(|| payload.downcast_ref::().cloned()) + .unwrap_or_else(|| "non-string panic payload".to_string()) +} + +/// Replay every case in isolation: a panic inside the engine (a +/// `debug_assert!`, say) becomes a mismatch naming its case, and the +/// remaining cases still run. +fn replay_all(what: &str, cases: &[Case], expected: Tally) { + let mut mismatches = Mismatches::default(); + let mut tally = Tally::default(); + for case in cases { + match catch_unwind(AssertUnwindSafe(|| replay(case, &mut mismatches))) { + Ok(outcome) => mismatches.settle(case, outcome, &mut tally), + Err(payload) => { + tally.cases += 1; + mismatches.push(&case.name, "panic", panic_message(payload.as_ref())); + } + } + } + mismatches.finish(what, tally, expected); +} + +#[test] +fn frozen_corpus_replays_through_the_native_window() { + replay_all( + "window_frozen_vectors", + &window_corpus::load(), + window_corpus::EXPECTED_TALLY, + ); +} + +#[test] +fn supplementary_cases_replay_through_the_native_window() { + replay_all( + "window_frozen_vectors supplementary", + &window_corpus::load_supplementary(), + window_corpus::EXPECTED_SUPPLEMENTARY_TALLY, + ); +} diff --git a/deny.toml b/deny.toml new file mode 100644 index 000000000..c5267c443 --- /dev/null +++ b/deny.toml @@ -0,0 +1,59 @@ +# cargo-deny configuration. CI runs `cargo deny check` with the version that +# .github/scripts/install-pinned-tool.sh pins; run the same locally with +# `cargo deny check` once cargo-deny is on PATH. +# +# Every ignore below names its reason. Add to a list only with one. + +[graph] +all-features = true + +[output] +feature-depth = 1 + +[advisories] +# RustSec advisories: any vulnerability, unmaintained crate, or yanked +# version in the lock file fails the check. +db-path = "~/.cargo/advisory-db" +db-urls = ["https://github.com/rustsec/advisory-db"] +yanked = "deny" +ignore = [] + +[licenses] +# Every crate in the graph must carry one of these licenses. The list is what +# the workspace's dependency tree actually uses; extend it with a reason. +allow = [ + "MIT", + "Apache-2.0", + "BSD-2-Clause", + "BSD-3-Clause", + "ISC", + "Zlib", + "Unicode-3.0", + # webpki-roots ships Mozilla's CA certificate bundle as data under the + # Community Data License Agreement (permissive); it reaches the graph + # through rustls in reqwest and sqlx. + "CDLA-Permissive-2.0", +] +confidence-threshold = 0.8 +exceptions = [] + +[bans] +# Duplicate versions of one crate are reported but do not fail the check; +# the workspace does not yet hold its tree to one version per crate. +multiple-versions = "warn" +# A dependency with no version requirement is refused, except the workspace's +# own path dependencies, which Cargo resolves to the one local copy. +wildcards = "deny" +allow-wildcard-paths = true +highlight = "all" +deny = [] +skip = [] +skip-tree = [] + +[sources] +# Only crates.io. A git or alternative-registry dependency needs a reason +# and an entry here. +unknown-registry = "deny" +unknown-git = "deny" +allow-registry = ["https://github.com/rust-lang/crates.io-index"] +allow-git = [] diff --git a/doc/mining.md b/doc/mining.md index bd8e9cecb..d2f7fe9a9 100644 --- a/doc/mining.md +++ b/doc/mining.md @@ -336,7 +336,7 @@ Practical starting point: - Start your initial share difficulty at roughly one tenth of your Bitcoin default - Use `CKPOOL_MINDIFF`, `CKPOOL_STARTDIFF`, and optionally `CKPOOL_MAXDIFF` to bound ckpool vardiff behavior -- For direct PRISM Stratum, keep the idle sweep enabled so a miner that receives too-hard work and produces no submitted or accepted shares can still step down after `PRISM_STRATUM_VARDIFF_RETARGET_SECONDS`; the sweep cadence is `PRISM_STRATUM_VARDIFF_IDLE_SWEEP_SECONDS`, and `0` disables it. +- For direct PRISM Stratum, each session re-evaluates vardiff on its own one-second timer, so a miner that receives too-hard work and produces no submitted or accepted shares still steps down after `PRISM_STRATUM_VARDIFF_RETARGET_SECONDS`, by at most `PRISM_STRATUM_VARDIFF_MAX_STEP_DOWN` per retarget; the native server has no separate idle-sweep setting. - Document a lower floor for small solo miners so shares do not become too sparse - Keep `CKPOOL_PUBLIC_DIFF_POLICY=explicit` for signet/mainnet-style deployments so startup fails before serving miners when policy values are missing diff --git a/doc/release-notes-1.1.0.md b/doc/release-notes-1.1.0.md index 06a72b7d8..8456e24f7 100644 --- a/doc/release-notes-1.1.0.md +++ b/doc/release-notes-1.1.0.md @@ -20,7 +20,7 @@ Release date: 2026-08-17 prefetch, and single-flight job builds that resolve or evict abandoned promises. - Makes payout-artifact builds incremental and debounced, and re-lands - event-driven artifact reuse behind the `PRISM_PAYOUT_ARTIFACT_REUSE` + event-driven artifact reuse behind the `PRISM_PAYOUT_ARTIFACT_REUSE` kill switch after the earlier anchor-scoped attempt was reverted. - Adds the reward-window leaderboard to the public dashboard API. `window=reward` ranks the canonical live PRISM work window using counted share difficulty @@ -57,7 +57,7 @@ The qbit mainnet genesis hash for this release line is unchanged: - Configuration is additive. No `.env.example` key was removed or renamed since 1.0.0. The two new behaviour switches are `PRISM_COINBASE_OUTPUT_POLICY`, which defaults to `canonical` and preserves - the 1.0.0 coinbase ordering, and `PRISM_PAYOUT_ARTIFACT_REUSE`, which + the 1.0.0 coinbase ordering, and `PRISM_PAYOUT_ARTIFACT_REUSE`, which defaults to `1` and can be set to `0` to fall back to full artifact builds. - The remaining new keys are PRISM tuning and capacity knobs covering block submit and landing timeouts, writer-lease TTL, tip-refresh fanout and worker diff --git a/doc/release-notes-2.0.0.md b/doc/release-notes-2.0.0.md index 8612214f7..24119cd43 100644 --- a/doc/release-notes-2.0.0.md +++ b/doc/release-notes-2.0.0.md @@ -1,8 +1,6 @@ # qbit-mining-bootstrap 2.0.0 Release Notes -This is the first 2.x release. Changes temporarily labelled 2.0.1 and 2.0.2 -during development are included here; those versions were not tagged or -published as GitHub releases. +Release date: pending ## Highlights @@ -28,11 +26,6 @@ published as GitHub releases. - Persists canonical audit-bundle bytes and durable audit publication order, improves writer-lease behavior under contention, and adds heap, allocator, component-cardinality, and lease-monitor diagnostics. -- Bounds candidate persistence, startup replay, and canonical audit finalization - with chunked storage and file-backed views; adds offline recovery of - accepted pending blocks. -- Resolves CKPool version masks after node readiness and refuses dynamic-mode - startup when no live block template can be obtained. - Moves the Python service images and CI to Python 3.14 and splits CI into independent lint, Python, Postgres, and Rust checks. @@ -63,20 +56,11 @@ have their own version tags. and operator endpoints, private. - The shipped Compose profile adds `prism-postgres-replica` and reads through `PRISM_PUBLIC_DATABASE_URL`. Compose defaults - `PRISM_PUBLIC_REPLICA_MODE=require`; provision the standby before cutover. - Leave the public DSN empty to derive the built-in standby URL from the - configured Postgres credentials, or set an explicit URL for a separate - standby. The coordinator keeps its primary database DSN. Direct launches - outside Compose default replica enforcement to `off`. -- The public psql backend also uses the public DSN. If a wrapper or custom - connection arguments are needed, set `PRISM_PUBLIC_PSQL_COMMAND` separately; - the coordinator's `PRISM_POSTGRES_PSQL_COMMAND` is not inherited by the - public service. + `PRISM_PUBLIC_REPLICA_MODE=require`; provision the standby and configure + the public DSN before cutover. The coordinator keeps its primary database + DSN. Direct launches outside Compose default replica enforcement to `off`. - Production needs an explicit `PRISM_POSTGRES_REPLICA_DATA_SOURCE` directory and read-only access to the shared audit files from the public API service. - Release-provenance checks require the replica path to be absolute and - non-overlapping with other enabled state sources before contacting Docker, - including normalized paths and filesystem aliases. Use a dedicated replication role and monitor the physical replication slot and retained WAL. The public standby is a read tier, not an automated failover target. Follow @@ -86,33 +70,20 @@ have their own version tags. with a default limit of 60 seconds; replay lag is reported separately. Outage responses may serve eligible cached data only within the documented staleness bounds, after which the service refuses the read with 503. -- The public HTTP server defaults to 64 active connections, a 10-second total - request-header deadline and a 10-second socket I/O idle timeout. Excess - connections close immediately; each response closes its connection. - `PRISM_PUBLIC_HTTP_MAX_CONNECTIONS` and `PRISM_PUBLIC_HTTP_TIMEOUT_SECONDS` - control these bounds. Review proxy upstream connection settings. -- The advertised Stratum URL must use `stratum+tcp` or `stratum+ssl`, include a - valid hostname and explicit port, and contain no credentials, path, query, - fragment or whitespace. Invalid endpoints now refuse startup. ### Ledger migration and audit history - Back up the ledger and stop the old coordinator before applying - `crates/qbit-prism/sql/001_share_ledger.sql`, followed by - `crates/qbit-prism/sql/002_candidate_bodies.sql`. Both are required, even - when configured to write legacy candidate bodies. The coordinator and - offline recovery command refuse a database without the second migration. - The first migration runs atomically, repairs partially seeded carry-forward - summaries, adds durable worker-difficulty state, and assigns + `crates/qbit-prism/sql/001_share_ledger.sql`. Despite its filename, this is + also the upgrade migration. It runs atomically, repairs partially seeded + carry-forward summaries, adds durable worker-difficulty state, and assigns `audit_publication_sequence` to historical confirmed and inactive blocks. - The migration takes exclusive table locks and builds a unique index non-concurrently. Schedule a maintenance window that allows interruption of reads as well as writes. With `PRISM_POSTGRES_INIT_SCHEMA=1`, the new coordinator applies the schema before opening listeners. For explicit - application, follow the fail-fast transaction requirements in - [the ledger operations guide](../docs/prism-ledger-ops.md) and apply the - candidate-body migration next, as described in - [the candidate storage guide](../docs/prism-candidate-storage.md). + application, use `ON_ERROR_STOP` and a single transaction as documented in + [the ledger operations guide](../docs/prism-ledger-ops.md). - New audit bundles are stored and served as their exact canonical bytes. Historical bundles retain the legacy reconstructed response until verified backfill publishes the canonical artifact. Run @@ -123,36 +94,7 @@ have their own version tags. for the complete backfill and replica-visibility procedure. - Use the documented ledger rollback procedure if reverting the software. The targeted audit-publication revert SQL is for a coordinated schema - rollback with all new-version writers stopped. Candidate-format rollback - must also follow the compatibility restrictions below. - -### Candidate storage and offline recovery - -- `PRISM_CANDIDATE_STORAGE_VERSION` defaults to `2`. Candidate bodies use - immutable 256 KiB chunks and bounded replay headers; existing version 1 - JSONB candidates remain readable. Candidate identity, timestamp normalization, - atomic share/outbox publication, and writer-session fencing are preserved. -- Canonical audit finalization uses replayable file-backed views and streamed - publication. Replay membership uses an exact PostgreSQL subset check through - bounded COPY writes. Spools, temporary tables, indexes, and candidate body - references have explicit retirement paths. -- `PRISM_BLOCK_REPLAY_PAGE_SIZE` accepts 1–1,024 descriptors. Smaller pages - increase database/RPC round trips; a row-count limit alone does not bound - the bytes in one candidate. Candidate spool reservations default to 4 GiB - per ledger instance; an empty `PRISM_CANDIDATE_SPOOL_DIR` selects the system - temporary directory. Provision and monitor that filesystem separately. -- Use `python3 -m lab.prism.recover_pending_blocks --help` for the offline - accepted-pending recovery command. Follow - [the ledger operations guide](../docs/prism-ledger-ops.md) for stopping the - writer, acquiring its lease, and reviewing recovery results. -- Qualify a compatible reader before enabling new-format writes. Keep the - additive schema during rollback. Once a new-format pending candidate exists, - an older reader that cannot hydrate it is unsafe. Use a compatible reader - with `PRISM_CANDIDATE_STORAGE_VERSION=1` to stop new-format writes, or a - separately verified drain/conversion procedure. Setting `1` does not remove - the migration requirement. Follow - [the candidate storage guide](../docs/prism-candidate-storage.md) for the - complete schema, capacity, rollout, and rollback procedure. + rollback with all new-version writers stopped. ### Configuration and runtime behavior @@ -171,19 +113,14 @@ have their own version tags. can preload recent converged values after restart. Resume is bounded assistance, not a guarantee that every active worker survives an abrupt exit with a retained difficulty. -- `PRISM_WINDOW_PIPELINE_RUST` defaults to `0`. When enabled with the daemon +- `PRISM_WINDOW_PIPELINE_RUST` defaults to `0`. When enabled with the daemon transport, unsupported integer ranges and daemon anomalies fall back to the Python pipeline for that materialization. Qualify a deployment before changing this switch; the real-daemon parity gate is part of CI. -- CKPool dynamic version-mask resolution runs after node readiness, retries - transient template failures, and aborts startup if no template is available. - A successful template without the field retains the configured fallback; - an explicit zero mask remains zero. `CKPOOL_VERSION_MASK_MODE=static` is the - explicit option for authorized prelaunch without template availability. - Rebuild service images for Python 3.14. The PRISM image sets `MALLOC_ARENA_MAX=2`; allocator telemetry defaults on, while heap census, tracemalloc, and malloc-trim controls default off. The optional - `PRISM_PYTHON_SWITCH_INTERVAL_SECONDS` override remains unset by default. + `PRISM_PYTHON_SWITCH_INTERVAL_SECONDS` override remains unset by default. Follow the capacity and allocator runbooks before tuning these controls. - Update consumers of removed `qbit_prism_shares_per_second` telemetry to rates derived from the monotonic share counters. Review @@ -199,8 +136,8 @@ have their own version tags. ## Changes Since v1.1.0 -This release includes the `2.x.x` changes after `v1.1.0` through -`504846c` (#258), along with release-promotion fixes in #246. +This release includes the 70 commits on `2.x.x` after `v1.1.0`, through +`700fae0` (#243). ### Coordinator ownership and concurrency validation @@ -237,9 +174,6 @@ This release includes the `2.x.x` changes after `v1.1.0` through verification tail latency (#214), and attribute lease-monitor stalls with explicit lateness responses (#233). - Cap escalated landing deadlines below the watchdog tolerance (#135). -- Allow one-candidate startup replay pages without truncating enumeration - (#256), and add offline recovery for accepted pending blocks (#259). -- Scope pending-fanout broadcast attempts to their own fanout (#292). ### Payout windows, audit artifacts, and memory @@ -258,14 +192,6 @@ This release includes the `2.x.x` changes after `v1.1.0` through - Replace builder pipe polling with readiness waits (#237), bound share serialization and lazy parsing across all paths (#239), bound database decoding (#238), and bound Rust window paging memory (#241). -- Release completed executor work and callback captures, avoiding traceback - cycles that retain finished job builds and delivery requests (#250). -- Release failed public-cache and address-validation requests (#252), and - clear finished cancellation frames while keeping each job-build waiter's - traceback separate (#253), without waiting for cyclic garbage collection. -- Bound candidate encoding, persistence, replay, payout membership checks, - and canonical audit finalization; add the candidate-body migration, spool - ownership, and crash/restart recovery gates (#258). ### Stratum, vardiff, and delivery health @@ -274,8 +200,6 @@ This release includes the `2.x.x` changes after `v1.1.0` through and stop overriding mainnet stale-share grace to zero (#225). - Remove the first-job admission lock convoy (#171) and evaluate delivery health by semantic work coverage (#221). -- Resolve CKPool version masks after node readiness, preserve static mode and - explicit zero masks, and validate retry settings before wallet setup (#257). ### Public dashboard API @@ -295,14 +219,9 @@ This release includes the `2.x.x` changes after `v1.1.0` through - Add always-on heap and component-cardinality telemetry (#232), bounded heap census and allocator controls, and a documented resident-memory soak bound (#234). -- Release sampled thread-frame references after each stall-probe sample, - including sampling caps and formatting errors, so completed workers' locals - are not retained until cyclic garbage collection (#248). - Make the interpreter switch interval tunable (#179), move service images and CI to Python 3.14 (#201), and shard lint, compile, Python, Postgres, and Rust CI checks while preserving the aggregate required check name (#243). -- Run push CI only for long-lived branches; PR branches are tested through - their pull-request merge result (#295). ## Operator Verification @@ -314,19 +233,7 @@ This release includes the `2.x.x` changes after `v1.1.0` through share accounting, and worker difficulty after reconnects before widening traffic. Keep production images digest-qualified. - Additional validation targets include `make test-prism-postgres-seed-guard` - and `make test-prism-public-read-replica`, plus - `make test-prism-postgres-candidate-storage` and - `make test-prism-postgres-native-ledger` for both candidate storage formats. - Capacity qualification and memory soak procedures are in - [the capacity readiness guide](../docs/prism-capacity-readiness.md). -- Candidate codec/storage experiments and actual mined-block crash/restart - gates are separate evidence. ARM64 experiments do not establish production - x86_64 performance or prove the cause of the exits reported in #255. - PostgreSQL and exceptional helpers still need capacity proportional to their - inputs within the documented limits. Coordinate production acceptance with - #254: at least two continuous hours in one process epoch, the specified risky - workloads, stable ownership after drain, and unchanged raw lease-failure - counters. This release does not raise lease thresholds, suppress alerts, or - change mining and payout rules. + and `make test-prism-public-read-replica`. Capacity qualification and memory + soak procedures are in [the capacity readiness guide](../docs/prism-capacity-readiness.md). Historical release notes remain in `doc/`. diff --git a/doc/release-notes-2.0.2.md b/doc/release-notes-2.0.2.md new file mode 100644 index 000000000..cb9903bd4 --- /dev/null +++ b/doc/release-notes-2.0.2.md @@ -0,0 +1,44 @@ +# qbit-mining-bootstrap 2.0.2 Release Notes + +Release date: pending + +## Candidate persistence and replay + +Candidate persistence and recovery use bounded codec and transport work instead +of serializing or decoding a complete payout window in the coordinator. The +candidate identity, timestamp normalization, atomic share/outbox publication and +writer-session fences remain part of the compatibility contract. + +Canonical audit finalization uses replayable file-backed views and streamed +artifact publication. Replay membership uses an exact PostgreSQL subset check +fed through bounded COPY writes with native backpressure. Temporary tables, +spools, file indexes and candidate body references have explicit retirement +paths; cyclic GC is not used as a recovery mechanism. + +The regression gates cover PostgreSQL and psql, cancellation, deadlines, retries, +writer-session fencing and an actual mined-block crash/restart. Separate large +codec experiments distinguish encoding, decoding, lazy materialization, native +buffering, reference-count destruction and cyclic GC. These experiments explain +possible blocking mechanisms; they do not prove the instruction responsible for +the production exits reported in #255. + +## Upgrade and rollback + +Follow the schema, capacity, rollout and rollback procedure in +[the candidate storage guide](../docs/prism-candidate-storage.md). Apply +`002_candidate_bodies.sql` (or start with `PRISM_POSTGRES_INIT_SCHEMA=1`) before +starting this release: the coordinator and the offline recovery command refuse +an unmigrated database for either storage version. Qualify a +compatible reader before enabling the new body representation. Keep the additive +schema during rollback. Once a new-format pending candidate exists, rolling back +to a reader that cannot hydrate it is unsafe; use the compatible reader with +new-format writes disabled, or a separately verified drain/conversion procedure. + +Coordinate deployment with #254's retention and self-check work. Production +acceptance requires at least two continuous hours in one process epoch, the +specified risky workloads, stable ownership after drain and unchanged raw lease +failure counters. This release does not increase lease thresholds, suppress +alerts or change mining and payout rules. + +The supported qbit mainnet binary pin remains v1.0.0 at +`7ebcddb622d6e639041f005a189b048ec2a221fe`. diff --git a/doc/release-notes-3.0.0-rc.1.md b/doc/release-notes-3.0.0-rc.1.md new file mode 100644 index 000000000..a8eda3067 --- /dev/null +++ b/doc/release-notes-3.0.0-rc.1.md @@ -0,0 +1,279 @@ +# qbit-mining-bootstrap 3.0.0-rc.1 Release Notes + +Release date: 2026-10-06 (release candidate; tag `v3.0.0-rc.1`, published as a +GitHub pre-release). + +This is the first release candidate of 3.0.0, the native Rust PRISM line. It +is the tree that #291's go/no-go evaluates on the PRISM production pair: two +frontends, each with its own `qbitd`, a PostgreSQL primary with one +asynchronous standby, and a public-read replica. It is not a production +release. Until #291 records a go, the production line stays 2.x.x (2.0.2). + +A candidate runs in pre-production with block submission and broadcasting off, +so it offers the network no found block and no transaction. A rehearsal uses +both holds, as the rehearsal procedure requires: + +- hold the ledger itself with `qbit-prism-server submission-hold set --reason + ...` (#664, migration 023) before any frontend starts on it, so that it + holds every frontend that connects; +- and start every frontend with `PRISM_BLOCK_SUBMIT_ENABLED=0` (#661), which + also keeps the CTV fanout broadcaster from starting. A frontend started + without it, on a ledger with no hold, offers every `pending` row it finds. + +See [the block submission kill switch](../docs/prism-configuration.md#block-submission-kill-switch) +and [the rehearsal procedure](../docs/prism-ledger-ops.md#block-submission-kill-switch-for-rehearsals). +Never leave either in place on a production frontend. + +## Highlights + +- Replaces the 2.x.x Python coordinator with `qbit-prism-server`, a native + Rust executable that runs Stratum, job refresh, block submission, CTV + broadcasting and HTTP; no Python runtime or builder subprocess remains in + the mining path. Operator work moves to subcommands, among them + `migrate`, `import-audits`, `self-check`, `candidates`, `fatal-state`, + `policy-transition`, `signing-transition`, `share-archive` and + `submission-hold` (#244, #360, #401, #419, #439, #464, #682). +- Several frontends can serve miners at once against one PostgreSQL writer + endpoint. Every instance coordinates through PostgreSQL, and only the + canonical share order and each job's accounting boundary are serialized: a + short transaction lock gives accepted shares one global `share_seq` order, + database constraints deduplicate proofs across instances, extranonces come + from a database sequence, expiring claims coordinate block candidates and + CTV broadcasts, and a database fingerprint refuses conflicting + configuration. A reconnecting miner can resume unexpired work on another + frontend (#244, #319, #397). +- Payout vectors exported from 2.x.x replay identically except for three + differences approved under decision D2: bootstrap work, which pays the + solver, is used only while there are no historical shares (2.x.x also used + it below its default three-miner gate); a proof that meets the network + target but not its share target is credited at network difficulty once its + block confirms on the active chain (2.x.x credited its share difficulty at + node acceptance); and a carry-only balance at or above the payout floor + can be paid in a bootstrap block. The 2.x.x per-worker credited-difficulty + overrides are not supported (#269, #302, #310, #316). +- `qbit_share_ledger` is partitioned by `RANGE (share_seq)` (migrations 016 + and 017): `migrate` makes the existing table the first partition behind a + validated bound, copying no rows and rebuilding no index, and every + frontend keeps empty lead partitions attached. Migration 013 trims the + ledger's secondary indexes, built concurrently on a populated ledger + (#144, #153, #369, #419). +- `share-archive` is the share retention path: it plans, seals, archives, + verifies, detaches, drops and restores whole partitions with the frontends + running, never deletes a share, and first seals each dependent audit's + canonical bytes, so blocks keep their published digests. A miner's + `last_share_at` reads `null` once its last share is archived, and + `/audit/share-window` returns no rows for an archived anchor. Stored native + audit bodies no longer grow with the payout window (#144, #267, #353, + #419). +- Migration 002's share-hash backfill runs after the migration transaction, + in short batches with a durable cursor, so `migrate` needs no statement + timeout above the default 15 s and resumes where it stopped. No frontend + starts until the backfill finishes, and meanwhile the + `share_hash_backfill_pending` capability fences earlier native builds off + (#582, #663, #669, #681). Migration 016 attributes block solvers in + bounded statements for the same reason (#672, #680). +- Found blocks are offered to the node before their audit lands, under one + durable reservation per block, so no frontend offers a block twice, even + after a crash. Unknown outcomes are reconciled from read-only chain + observations, a proven lost tip race settles `orphaned`, and a + `submitblock` that provably never ran (no connection, or `RPC_IN_WARMUP`) + is offered again (#391, #415, #424, #522, #526, #527, #566). +- A block found on work whose payout revision was superseded on the current + tip is captured and offered within a per-frontend overpay ceiling, with any + overpay recorded (migration 020) and carried as debt. Candidate and CTV + fanout claim leases are timed on the taker's monotonic clock, so a + database clock step no longer hands a live claim to another frontend + (migrations 021 and 022; #478, #504, #581, #646, #654, #657, #667, #674). +- `PRISM_BLOCK_SUBMIT_ENABLED=0` holds one frontend for rehearsals: found + blocks are enqueued but stay `pending`, `submitblock` is never called and + the CTV broadcaster does not start. `submission-hold set` holds every + frontend from the ledger, whatever its own setting (migration 023), + journaled with a reason, and `PrismBlockSubmissionHeld` pages when a + frontend holds submission (#291, #661, #664, #666, #676, #682). +- CTV fanouts are broadcast by several frontends through database claims. + The claim lane reads only due fanouts through a partial index (migration + 024), claims are handed back at shutdown, and a pass yields to its + frontend's pending template refresh for at most the replacement-build + budget. Every settlement mode now requires a pool fee (#433, #525, #534, + #535, #568, #573, #576, #668, #679). +- The HA reference (decision D3) is one primary plus one dedicated + asynchronous failover standby, separate from the public-read replica, with + a two-frontend overlay, `compose.prism-ha.yaml`. Failover can lose + acknowledged shares in the replication gap. An optional bounded wait lets + the standby flush a found block's rows before its offer, and, given + synchronized database clocks, blocks on work from a lost gap are refused + after a promotion. Promotion, fencing and the TCP load balancer are + operator-supplied, the last against a documented `/healthz` probe contract + (#281, #304, #331, #449, #529, #570, #619, #645). +- Stratum keeps the 2.x.x miner contract, including username syntax, BIP310 + version rolling, vardiff, the high-difficulty listener and the one-parent + stale grace (3 s by default). A per-source connection cap and per-session + budgets for malformed frames, unknown job IDs and authorize attempts are + new and off by default. A session opening during a same-tip rebuild storm + no longer waits for the whole fan-out, overtaken same-tip builds are still + delivered, and submits are answered while a session's rebuild is in flight. + A plain share the payout-revision fence refuses is answered `stale-job`, + not `ledger-confirmation-failed` (#244, #313, #468, #471, #598, #604, #616, + #621, #625, #642, #675, #698). +- Template refresh keeps its incremental path across retargets, builds on a + per-process worker pool (`PRISM_REFRESH_BUILD_THREADS`), and renews + readiness from a guarded tip poll, so a slow rebuild no longer refuses + current work as stale. After each block it lands, a frontend returns the + freed heap to the kernel (`PRISM_LANDING_MALLOC_TRIM_ENABLED`, default 1) + (#275, #501, #502, #600, #622, #627, #643). +- Coordinator metrics come from one native registry, with a generated + inventory covering both roles, and `/metrics` answers 200 with freshness + headers. The repository specifies the native alert rules and a review-only + diff for the deployed rules that maps each retired Python alert. The + native rules cover a stalled work refresh, a PostgreSQL outage and held + block submission (#277, #278, #279, #299, #308, #320, #534, #646, #676). +- The new `qbit-prism-load` harness drives real Stratum sessions against + real frontends, PostgreSQL with a streaming standby and a fake or real + regtest node, and reconciles the ledger exactly with what clients were + acknowledged. Presets model recipient weights, bursty arrival and rental + churn; a `faults` phase injects faults under load; `external` mode drives + deployed frontends through their load balancer; and #511's release + benchmark adds an A/B driver and a D1 comparison summarizer (#271, #342, + #521, #536, #539, #547, #554, #564, #590, #623, #635, #662). +- CI runs PRISM end to end. Required CI fails unless every gated test, live + regtest included, runs without skipping. A nightly workflow runs load + presets, real-node scenarios and Stratum fuzzing, and the `run-load` label + runs its preset set on a pull request; weekly jobs run the 2.x.x migration + lifecycle and a measured mainnet-shaped cutover on a real node, a 5.5 h + soak, and the shipped images under Compose with the HA overlay. The L3 + production-window matrix runs on a version-bump pull request, the + `release-candidate` label and `v*` tags, and a reduced matrix weekly. None + of these lanes is a required check (#322, #487, #544, #545, #549, #550, + #553, #565, #575, #587, #588, #594, #596, #597, #608). + +## Upgrading from 2.0.x + +Follow the [migration guide](../docs/prism-rust-migration.md). Native +migration is one-way, with no down-migrations (decision D5, #287, #366): + +> Before the first native share is acknowledged, restore the complete +> pre-migration database and artifact backup and restart the pinned old image +> in isolation from the migrated database. After the first native share is +> acknowledged, restoring an older database loses those accepted records. +> Any rollback that discards acknowledged history requires an explicit +> accounting reconciliation and recovery decision; it is not an ordinary +> image rollback. Keep every native frontend stopped while performing an +> isolated restore/recovery operation against its replacement database. + +- Drain pending candidates with a v2.0.1 or later image (v2.0.2 for a v2.0.2 + database), then stop every 2.x.x process; `migrate` takes v2.0.0 to v2.0.2 + sources and refuses an undrained one. Run `check-config`, `migrate` and + `import-audits`, whose two missing counts must be 0, after rehearsing on + an isolated restore. +- On a populated ledger the outage includes 002's share-hash backfill (35 to + over 80 minutes for 65.5M shares) and the online steps of 013, 017 and 024, + which can take hours. Earlier native builds are refused while the backfill is + pending; never record migration 2 by hand (#582, #669). +- The 102 retired 2.x.x settings are listed in + [`retired-settings.txt`](../crates/qbit-prism-server/src/config/retired-settings.txt). + [`check-config`](../docs/prism-configuration.md) and `run` name any still + set, and production mode refuses them (#361, #462). +- Production reads signing seeds from `_FILE` mounts, and every settlement + mode needs a pool fee with one recipient, 0 bps allowed (#535). +- Block totals, chart markers and the default `chain_state=active` view of + `/public/v1/blocks` count confirmed blocks only. Native block intents are + persisted before node acceptance and stay out of them until confirmed; + `chain_state=all` shows every candidate with its state. +- This candidate requires every native schema migration through 024. + `migrate` applies them, and every start refuses a database missing one. +- It is tested against qbit 1.0.0, the release + `.github/scripts/install-prism-qbit.sh` pins. +- The coordinator listener on 3341 now also serves `/public/v1` (2.0.x + answered 404); keep it private and send public traffic to `public-api` on + 3342, which needs no audit mount. Runbooks: + [ledger operations](../docs/prism-ledger-ops.md), + [HA](../docs/prism-ha-reference-architecture.md), + [alerts](../docs/prism-alert-migration.md) and + [go-live checks](../docs/mainnet-deployment.md#go-live-checks). + +## Verification + +A candidate is verified by the full suite (#557): + +- **CI** (`ci.yml`), on the version-bump pull request, and dispatched by hand + on the tag, since `ci.yml` has no tag trigger: lint, compile and Compose + validation; the Python and Rust unit tests; the PostgreSQL contract shards + and the proof that every gated test executed; the dependency audit; and the + Docker and real-`qbitd` image builds. +- **The L3 production-window matrix** (`prism-load-l3.yml`, suite `l3-full`: + every #473 cell, three repeats, on 32 vCPU runners). It runs by itself on + the version-bump pull request. The `v*` tag promotes that run when the + tagged commit's tree is the one the run measured, and runs the suite again + otherwise (#550, #608). +- **The load harness** (`prism-load-nightly.yml`), dispatched by hand on the + version-bump pull request and on the tag, with `preset=all live=true + weekly=true fuzz=true images=true bridging=true`: + - every nightly and manual preset, including #473's 400k and 500k window + cells at 1, 2 and 4 frontends, the 200k one-frontend cell and the + mainnet shapes; + - the bridging pair (#552); + - the live regtest nightly and weekly scenario lists (#521, #553, #575); + - the Stratum fuzz targets; + - the shipped images under the Compose `prism` profile with the HA + overlay (L6, #544). +- **The 5.5 h soak and the long fault set** (`preset=weekly`: `soak-weekly` + and `faults-long-real-node`). For this candidate the soak runs once, at + 3.x.x `c95febb3`. The only runtime change between that commit and this + candidate is #698 (#675). `faults-long-real-node` runs there and again on + the version-bump pull request. Their verdicts are in the evidence bundle. + +The evidence bundle on the `v3.0.0-rc.1` GitHub pre-release lists: +- every run, with the commit and tree it tested; +- each job's verdict and its artifacts; +- any failure, recorded as a named exception with its issue. + +#291 links to the bundle. Job and gate verdicts are correctness evidence for +#291. The measured rates are not D1 verdicts (#487 decision 5). + +## Known issues + +Open on `3.x.x` when this candidate was cut: + +- #695: at the 20x growth shape (8,000 sessions at a 1,000/s mean), share + appends queue on the global order lock and reach 420 to 675 shares/s, short + of the shape's rate. Today's mainnet shape (130 addresses) and the 5x shape + (650) kept up with their offered rates, with zero shortfall; their headroom + was not measured. This is a capacity ceiling for D1 (#477), not a + correctness defect: no acknowledged share was lost and reconciliation was + exact. +- #600: a 400k-window frontend's resident set ratchets up with block + landings, from allocator retention. #627 returns the freed heap after each + landing. Making jemalloc the allocator (#628) and judging the soak's memory + on per-cycle floors (#629) are open, and the weekly soak still records #600 + as an expected failure. +- #602: block landing and settlement hold the order lock, which can delay + share acknowledgements around a pool block. +- #622: new-tip delivery missed a 30 s bound with about 3,000 sessions at a + 1 s reanchor. #643 fixed the mechanism found; the issue stays open until + that scenario is re-run. Production reanchors every 60 s. +- #630: a payout-revision race during the post-offer landing raises + "offered candidate could not be processed" about twice per 30 minutes + under load. The ledger runbook says when those ALERT lines are benign and + when to act on them (#696). +- #670: `broadcast-ctv` cannot take over a dead frontend's fanout claim + within one pass. +- #688, #689 and #690: the public pending-fanouts query, the CTV claim lane + in a backlog, and fatal-state clearing each do work that grows with the + number of fanouts. +- #582 (the share-hash backfill on a production-sized ledger) and #604 + (first jobs queued behind rebuild storms) are fixed in code (#663, #680, + #681; #625) and stay open for #291's go/no-go sweep. +- #701: in `faults-long-real-node` at `c95febb3`, one public-API request + failed with a connection error during the block failover. It was the run's + only read-tier failure, and it fell outside the window in which the public + API refuses by design. #291 requires the public API to keep answering + through the failover drill. + +Before a go, every acceptance criterion of #291 must be met. When this +candidate was cut, the open ones included: +- the production-sized migration rehearsal with its timings; +- the D4 key-rotation rehearsal and the D5 isolated-restore rehearsal; +- the failover drill and the load-balancer exercise under mining load, with + `prism-public-api` answering `/public/v1` throughout; +- the two-hour soak against its criteria (#556); +- D1 throughput on the rehearsal host (#477). diff --git a/doc/release-notes-3.0.0-rc.2.md b/doc/release-notes-3.0.0-rc.2.md new file mode 100644 index 000000000..b77712610 --- /dev/null +++ b/doc/release-notes-3.0.0-rc.2.md @@ -0,0 +1,119 @@ +# qbit-mining-bootstrap 3.0.0-rc.2 Release Notes + +Release date: 2026-10-06 (release candidate; tag `v3.0.0-rc.2`, published as a +GitHub pre-release). + +This is the second release candidate of 3.0.0: 3.0.0-rc.1 plus migration 025, +the commit-gate answer fix for #716 and the operator tooling for legacy audit +bodies. It is the tree that #291's +go/no-go evaluates for production on the PRISM pair. It is not a production +release. Until #291 records a go, the production line stays 2.x.x (2.0.2). + +Everything in the [3.0.0-rc.1 notes](release-notes-3.0.0-rc.1.md) applies +(the highlights, the upgrade contract, the two submission holds for a +rehearsal, and how a candidate is verified), with the changes below. + +## Changes since 3.0.0-rc.1 + +- #708, fixed by #710: migration 025 checks legacy carry-forward chains per + payout program, as 2.x keeps the balances, not per case-sensitive label. A + miner who re-authorized in uppercase split one program into two labels on + union's ledger: 5,506 false mismatches under the old rule, 0 under the new + over 2,753,849 rows. Before 025, `self-check` and `fatal-state clear` + refused that ledger. #710 also changes + `scripts/prism-recovery-evidence.py`: the summarizer clears the old rule's + per-label findings, so a 2.x source and its migrated copy summarize the + same. Use this candidate's summarizer on both sides of a cutover. +- #716, fix A by #718: a share refused at the commit gate because chain-state + authority was briefly unavailable at a tip change now answers + `backend-rpc-unavailable` ("current chain state is unavailable"), or + `backend-database-unavailable` when the ledger database is what failed, + instead of `ledger-confirmation-failed`. So `PrismShareAppendFailures` no + longer fires for it. Such a share is still never credited; the + credit-preserving retry is #716's follow-up. +- Legacy audit bodies whose share-segment range digests were written with + Python's escaped JSON (non-ASCII worker names; 339 on union, heights + 53313-57440) import through canonical sidecars from + `scripts/prism_legacy_range_sidecars.py` (#715). Run it before + `import-audits`. #709 makes 3.x accept the escaped form after GA. +- #705: `scripts/prism-recovery-evidence.sql` streams with `FETCH_COUNT`; + the runbooks pass `-v FETCH_COUNT=10000` (#706). +- The load harness and CI, with no runtime change: + - the fault phase's read tier lets its scrapes in flight finish before it + stops `public-api` (#701, #702); + - the weekly reduced L3 runs #555's session points and #521's mainnet + shapes (#614); + - the weekly soak's resident-memory gate is real again (#714). #627's fix + for #600 was already in rc.1, whose soak passed every resident-memory + row. + +## Upgrading + +- **From 2.0.x:** as in the rc.1 notes, with three additions. + - This candidate requires every native schema migration through 025. + - Run `scripts/prism_legacy_range_sidecars.py` before `import-audits`, so + bodies with escaped range digests import and its two missing counts + reach 0. + - Export the recovery evidence with `FETCH_COUNT`, as the runbooks do. + Expect the export to take a long time on a production-sized ledger + (#712). +- **From 3.0.0-rc.1:** run `migrate` before starting any rc.2 frontend. An + rc.2 frontend refuses a database that lacks 025. + - 025 replaces a validator function and changes no stored row, so it is a + plain in-transaction migration, with no capability and no shutdown + proof. + - An rc.1 frontend accepts the new migration with a warning, keeps + starting and reads the corrected report. + - Because 011 defines the validator that 025 replaces, `migrate` runs 025 + again whenever it runs 011. + +## Verification + +The runtime changes between rc.1 and this candidate are: +- migration 025: its SQL, and its registration in `ledger/migration.rs`, + which adds 25 to the versions every start requires; +- #718's commit-gate answer in `coordinator/miner_submit.rs`. + +So the evidence is lean (#557): +- CI on the version-bump pull request, and dispatched on the tag; +- the L3 production-window matrix on the version-bump pull request, which + the tag promotes on a tree match (#608); +- a targeted nightly batch: `short-plan-fake-node`, `short-plan-real-node`, + `soak-smoke`, `faults-short-real-node` and `faults-failover-fake-node`; +- the live regtest weekly scenarios, including the 2.x.x migration lifecycle + and the measured mainnet-shaped cutover. + +rc.1's full nightly set and its 5.5 h soak carry over: +- **The soak** passed every gate. Its job was marked failed only by the + preset's "#600 looks fixed" tripwire, which #714 has since removed. +- **`faults-long-real-node`** is not re-run for this candidate. Its rc.1 + runs failed only on the harness artifact #701, which #702 fixes, with every + functional fault check passing. + +The evidence bundle on the `v3.0.0-rc.2` GitHub pre-release lists every run, +and #291 links to it. + +## Known issues + +As in the rc.1 notes, with these changes: + +- **Fixed since rc.1:** + - #708, by 025; + - #701, by #702; + - #716's reject reason and alert, by #718. The credit-preserving retry stays + open in #716. + - #600 is closed: rc.1 already had #627's fix. +- **Open since rc.1, not fixed in this candidate:** + - #711: a share append holds the order lock across about nine client round + trips, capping the cluster at about 890 shares/s locally and about 300/s + from the remote frontend on the pair. See #695. + - #712: the recovery-evidence export took 85 minutes at 65.9M shares, and + both exports sit inside the cutover outage. + - #713: `sslmode=verify-ca` still checks the host name, so it behaves like + `verify-full`. + - #704: the fenced failover-mid-landing drill test can time out when both + miners find a block during the held offer (a CI flake). + - #709: until it lands, legacy bodies with escaped range digests need the + canonical sidecars above. +- **Still open from rc.1:** #695, #602, #622, #630, #670 and #688 to #690. +- **Open for #291's go/no-go sweep:** #582 and #604, which are fixed in code. diff --git a/doc/release-notes-3.0.0-rc.3.md b/doc/release-notes-3.0.0-rc.3.md new file mode 100644 index 000000000..306440446 --- /dev/null +++ b/doc/release-notes-3.0.0-rc.3.md @@ -0,0 +1,90 @@ +# qbit-mining-bootstrap 3.0.0-rc.3 Release Notes + +Release date: 2026-10-06 (release candidate; tag `v3.0.0-rc.3`, published as a +GitHub pre-release). + +This is the third release candidate of 3.0.0: 3.0.0-rc.2 plus #719's shorter +order-lock hold for share appends (#711) and #721's parallel recovery-evidence +export (#712). It is the tree that #291's go/no-go evaluates for production +on the PRISM pair. It is not a production release. Until #291 records a go, +the production line stays 2.x.x (2.0.2). + +Everything in the [3.0.0-rc.1](release-notes-3.0.0-rc.1.md) and +[3.0.0-rc.2](release-notes-3.0.0-rc.2.md) notes applies, with the changes +below. + +## Changes since 3.0.0-rc.2 + +- #711, fix B by #719: a share append holds the global ORDER_LOCK for 3 + statements plus COMMIT, down from 9 client round trips. There is no + migration. N2 measured it on the pair, on a build whose runtime is #719's + head `82c80bb1`. That is rc.3 without #718's answer mapping, which doesn't + touch the lock hold: + - the D1 peak-second gate now passes, with a window p99 of 898 ms against + the 1,000 ms gate. rc.1 measured 1,479 ms on the same shape: an empty + window, with the spike truly offered. An earlier rc.1 run, whose spike + was under-offered, measured 1,116 ms; + - reconciliation is exact; + - the mean time each share append holds the global ordering lock + (`qbit_prism_database_order_lock_hold_seconds{holder="append"}`) is + 1.154 ms, against 1.473 ms on the rc.1 runtime in the same peak-shape run; + - the ceiling is 750 shares/s with the Stratum load balancer preferring the + primary's frontend (the pair's operating rule), flagged as 5% under 788; + - #291's failover drill under load, run 1, passes with zero acknowledged + shares lost. The run was an unplanned crash of the primary, then fencing, + promoting the standby and switching the writer, with 300 shares/s + through the real frontends. +- #712, in part by #721: `scripts/prism-recovery-evidence-parallel.py` exports + the recovery evidence in parallel, byte-identical to the serial export: 24 s + against 115 s with 8 jobs on a 6M-share copy. #721 also moves a `\gset` in + `scripts/prism-recovery-evidence.sql`, which leaves the serial export's + output unchanged. These are scripts; no runtime change. + +## Upgrading + +- **From 2.0.x:** as in the rc.2 notes. The schema is still migrations 2 to 25. + Export the recovery evidence with the + [parallel export](../scripts/prism-recovery-evidence-parallel.py), as the + [ledger runbook](../docs/prism-ledger-ops.md) describes, to shorten the + outage. +- **From 3.0.0-rc.2:** no migration; replace the binary on every frontend. + +## Verification + +The only runtime change between rc.2 and this candidate is #719's share append: +- `ledger/window.rs`; +- `check_writable` and `WRITABLE_COLUMNS` made visible from `ledger/connect.rs`; +- the import in `ledger.rs`. + +The evidence is the same set as rc.2's (#557): +- CI on the version-bump pull request, and dispatched on the tag; +- the L3 production-window matrix on the version-bump pull request, which the + tag promotes on a tree match (#608); +- the targeted nightly batch (`short-plan-fake-node`, `short-plan-real-node`, + `soak-smoke`, `faults-short-real-node`, `faults-failover-fake-node`); +- the live regtest weekly scenarios. + +rc.1's full nightly set and 5.5 h soak carry over. The evidence bundle on the +`v3.0.0-rc.3` GitHub pre-release lists every run, and #291 links to it. + +## Known issues + +As in the rc.2 notes, with these changes: + +- **#716 (open part):** at a tip change under load, a share admitted under a + tip-change lease can be refused when its commit gate finds the publication + authority busy. It is never credited, nothing is written and no ACKed + share is lost. Since #718 it is answered `backend-rpc-unavailable` + (code 20). It was seen: + - 4 times in about 149k shares in N2's rc.1 D1 sustained run on the pair + (500/s for 300 s, 7 tips); + - once in about 121k in the peak run on #719's build; + - once in about 150k in rc.2's L3 (3-blocks r2). + + The fix, a credit-preserving retry (#716 fix B), comes after the + candidates. +- **#711:** improved by fix B (#719). The issue stays open for fix A, and the + pair's ceiling is still flagged under 788 shares/s. +- **#712:** the parallel export (#721) shortens the evidence export. Taking + the source export off the critical path, and a parallel summarizer, are + still open. Both exports still sit inside the cutover outage. diff --git a/doc/release-notes-3.0.0-rc.4.md b/doc/release-notes-3.0.0-rc.4.md new file mode 100644 index 000000000..b6d0218a0 --- /dev/null +++ b/doc/release-notes-3.0.0-rc.4.md @@ -0,0 +1,102 @@ +# qbit-mining-bootstrap 3.0.0-rc.4 Release Notes + +Release date: 2026-10-06 (release candidate; tag `v3.0.0-rc.4`, published as a +GitHub pre-release). + +This is the fourth release candidate of 3.0.0: 3.0.0-rc.3 plus #723's +configurable Stratum listen backlog. It is the tree that #291's go/no-go +evaluates for production on the PRISM pair. It is not a production release. +Until #291 records a go, the production line stays 2.x.x (2.0.2). + +Everything in the [3.0.0-rc.1](release-notes-3.0.0-rc.1.md), +[3.0.0-rc.2](release-notes-3.0.0-rc.2.md) and +[3.0.0-rc.3](release-notes-3.0.0-rc.3.md) notes applies, with the changes +below. + +## Changes since 3.0.0-rc.3 + +- #723: both Stratum listeners (the primary and the high-difficulty one) + listen with a backlog of `PRISM_STRATUM_LISTEN_BACKLOG`, default 4096. + - **Before:** every native listener inherited mio's fixed backlog of 128, + a silent drop from 2.x.x's 1024. On the pair, N2's C5 run opened 400 + Stratum sessions at once through the load balancer. The accept queue + overflowed (`TcpExtListenOverflows` 175), the load balancer marked frontend + A down for about 4 s, and 158 of the 400 sessions spilled to the backup + frontend and stayed there. + - **The kernel caps the backlog at `net.core.somaxconn`** in the frontend's + network namespace. The primary Stratum listener's `PRISM listening` log + line records the backlog and the namespace's somaxconn, and a warning + names the cap when somaxconn is the lower. + - **The audit and operator HTTP listener and the public read service** now + listen with a fixed 1024. + - **`PRISM_STRATUM_LISTEN_BACKLOG` is a supported setting again.** It was + 2.x.x's setting and left the retired list; `check-config` validates it + (1 to 2147483647). + +## Upgrading + +- **From 2.0.x:** as in the rc.3 notes, and `PRISM_STRATUM_LISTEN_BACKLOG` is + read again. rc.1 to rc.3 did not read it. In production mode they refused it + as set but unread; elsewhere they warned, ignored it and listened with + mio's 128. Unset, it is 4096. A value carried over from 2.x.x overrides that + default (2.x.x's own default was 1024), and a value above 2147483647 now + fails `check-config`. +- **From 3.0.0-rc.3:** no migration; replace the binary on every frontend. + Keep `net.core.somaxconn` in each frontend's network namespace at least as + high as the backlog. +- **Rolling a frontend back to rc.3 or earlier:** first remove + `PRISM_STRATUM_LISTEN_BACKLOG` from its environment. This candidate's + `compose.yaml` always passes it (default 4096). Earlier native builds in + production mode refuse to start with a set-but-unread `PRISM_*` setting. + +## Verification + +The only runtime change between rc.3 and this candidate is #723's listener +binding and its setting: +- `listen.rs` (new), `server.rs`, `stratum.rs`, `lib.rs` and + `api/public_service.rs`; +- the setting's move from `config/retired-settings.txt` to + `config/native-settings.txt`. + +#723 also passes the setting through `compose.yaml` (default 4096) and +`.env.example`. It doesn't touch the order lock, the append, refresh or +settlement. So the pair evidence rc.3's notes cite still applies: the D1 +peak-second gate, the 750/s ceiling and the failover drill under load, run 1, +all measured on #719's build. + +#723's own proof in CI is its unit test: 300 concurrent connects against a +512 backlog with nothing accepting. None of this candidate's suites bursts +more than 128 connects through a load balancer, so the pair's C5 re-run is +the proof on the pair. Otherwise the evidence is the same set as rc.3's +(#557): +- CI on the version-bump pull request, and dispatched on the tag; +- the L3 production-window matrix on the version-bump pull request, which + the tag promotes on a tree match (#608); +- the targeted nightly batch: `short-plan-fake-node`, `short-plan-real-node`, + `soak-smoke`, `faults-short-real-node` and `faults-failover-fake-node`; +- the live regtest weekly scenarios. + +rc.1's full nightly set and 5.5 h soak carry over. The evidence bundle on the +`v3.0.0-rc.4` GitHub pre-release lists every run, and #291 links to it. + +## Known issues + +As in the rc.3 notes, with one addition (#724, the last item): +- **#716 (open part):** at a tip change under load, a share admitted under a + tip-change lease can be refused when its commit gate finds the publication + authority busy. It is never credited, nothing is written and no ACKed + share is lost. Since #718 it is answered `backend-rpc-unavailable` + (code 20). It was seen: + - 4 times in about 149k shares in N2's rc.1 D1 sustained run on the pair + (500/s for 300 s, 7 tips); + - once in about 121k in the peak run on #719's build; + - once in about 150k in rc.2's L3 (3-blocks r2). + + The fix, a credit-preserving retry (#716 fix B), comes after the + candidates. +- **#711:** improved by fix B (#719). The issue stays open for fix A, and the + pair's ceiling is still flagged under 788 shares/s. +- **#712:** the parallel export (#721) shortens the evidence export. Taking + the source export off the critical path, and a parallel summarizer, are + still open. Both exports still sit inside the cutover outage. +- **#724:** a live regtest test is intermittent in CI. diff --git a/doc/release-notes-3.0.0-rc.5.md b/doc/release-notes-3.0.0-rc.5.md new file mode 100644 index 000000000..effd16ce6 --- /dev/null +++ b/doc/release-notes-3.0.0-rc.5.md @@ -0,0 +1,447 @@ +# qbit-mining-bootstrap 3.0.0-rc.5 Release Notes + +Release date: 2026-10-08 (release candidate; tag `v3.0.0-rc.5`, published as a +GitHub pre-release). + +This is the fifth release candidate of 3.0.0: 3.0.0-rc.4 plus #737's fixes to +`fatal-state clear` and the carry-forward integrity report (#742, and #744's +migration 026), and two ways to shorten the cutover's outage: serving while +migration 002's share-hash backfill finishes (#743 and #746) and offline index +builds (#745). It also carries the cutover tooling merged after rc.4's tag +(#728, #732, #736 and #740) and a test-only fix (#747). It is the tree that +#291's go/no-go evaluates for production on the PRISM pair. It is not a +production release. Until #291 records a go, the production line stays 2.x.x +(2.0.2). + +Everything in the [3.0.0-rc.1](release-notes-3.0.0-rc.1.md), +[3.0.0-rc.2](release-notes-3.0.0-rc.2.md), +[3.0.0-rc.3](release-notes-3.0.0-rc.3.md) and +[3.0.0-rc.4](release-notes-3.0.0-rc.4.md) notes applies, with the changes +below. Unlike rc.4, an upgrade from the previous candidate runs +`qbit-prism-server migrate` first ([Upgrading](#upgrading)). + +## Changes since 3.0.0-rc.4 + +- #737, by #742: `fatal-state clear` accepts a tip that only grew during its + run, under a bound the operator sets, and the integrity report runs under a + statement timeout of its own. + - **Before:** the clear refused whenever the node's tip moved during its + run, a plain extension included. At production size a clear takes about a + minute, nearly all of it the integrity report, and tips arrive about once + a minute, so a drill cleared about one attempt in three. Its bound was a + fixed 120 s, and the report ran under the session's statement timeout, + 15 s by default, so the clear and `self-check` both needed + `PRISM_DATABASE_STATEMENT_TIMEOUT_MS=120000`. + - **F5, a tip that only grew:** the clear walks the node's new best block's + headers back to the captured height, where it must find the captured tip. + When the tip has moved by the end of its block and fanout checks, it first + asks the node each of their questions again. It still refuses, clearing + nothing, a reorganization, a moved tip while a pool block row lies above + the captured height, and a tip more than 1,000 blocks up. Each of these + refusals ends `retry fatal-state clear`. The event's `reconciliation` adds + `final_tip_hash` and `tip_extended`. + - **F2, `fatal-state clear --timeout-seconds`** (10 to 3600, default 120) + replaces the fixed bound, which the recovery must meet before its commit. + The closing tip check must end 5 s before the bound, so the `UPDATE` and + the event `INSERT` keep their 5 s. The commit runs outside the bound and + waits for its outcome, which under `synchronous_commit=remote_apply` + includes a standby's replay. A clear that runs out of its bound has + therefore always committed nothing, and the halt stays. + - **F1, the report's own statement timeout:** in the clear, the time left in + the bound less 5 s and one node RPC timeout (`PRISM_RPC_TIMEOUT_SECONDS`), + so PostgreSQL cancels a report that runs out of time before the bound + ends; in `self-check`, 300 s, in a read-only transaction. Neither command + needs `PRISM_DATABASE_STATEMENT_TIMEOUT_MS` raised. +- #737, by #744: migration 026 makes `qbit_carry_forward_integrity_report()` + compute each of its two expensive sets once (F4). It counted and listed each + separately, so it ran the validator and the drift check twice. + - At production size the report is estimated at about 29 s, down from about + 57 s. The estimate is the sum of the two checks' EXPLAIN ANALYZE times on a + production-sized ledger, 20.29 s and 9.14 s, not a timed 026 run. Locally, + on 400,000 carry rows, it took 1.21 s against 001's 2.30 s. + - 026 keeps 001's subqueries word for word and evaluates them in 001's + order, so the report is evaluated as 001's is. Like 025, it changes no + stored row and declares no capability. The recovery evidence export, + `scripts/prism-recovery-evidence.sql`, requires it, as startup does. + - `self-check`, `fatal-state clear` and the import admission gate all wait + for the report. The gate's header comment still says it takes about a + minute: the script is pinned by its sha256, so the comment waits for the + gate's next change. +- #582's cutover window, by #743 and #746: + `qbit-prism-server migrate --defer-share-hashes` lets frontends serve before + migration 002's share-hash backfill has finished, and + `qbit-prism-server backfill-share-hashes` finishes it after go-live. + - **Before:** no frontend started until the backfill had mapped every legacy + share's header, which on a production-sized ledger is most of `migrate`'s + time. So it sat inside the cutover's outage. + - **The deferred run** maps only the recent range: every accepted legacy + share within 1,000 template heights, the coinbase maturity, of the highest + one. It maps them before 013 drops the index their bounds read. It then + raises the backfill's fence to `share_hash_backfill_pending = 2`, which + permits serving with migration 2 unrecorded, and each start logs a warning + naming the backfill's cursor. + - **Measured on the pair:** in the first round of #291's rehearsal for this + candidate, the `qbit-prism-server migrate --defer-share-hashes` command, + on a copy of the production ledger, took 5 min 06 s end to end. It ran on + 3.x.x `539d6c91`, before #745, so 013's and 024's indexes were built + concurrently. + - **Why serving is safe:** a native share is credited only on a job whose + parent is the current tip or, within stale grace, the tip's parent, and + every copy of a legacy header has the same parent. So a credited native + share can repeat only a legacy header from the recent range, unless the + chain reorganizes more than 1,000 blocks below the legacy tip, and the + append's existing header check refuses that repeat as it would with the + whole mapping. + - **`qbit-prism-server backfill-share-hashes [--max-batch 5000] + [--statement-timeout-ms 2000] [--duty-cycle 0.5]`** maps the rest while + frontends serve, and records 2. Each batch maps at most `--max-batch` + `share_seq` values in one statement, in a transaction of its own. A + statement that outlasts `--statement-timeout-ms`, at most 5,000, is + cancelled and tried again at half its size, so no statement holds a + snapshot on the serving primary longer than that (#738). After a batch + that took `t`, the run rests `t * (1 - d) / d` with no transaction open, + `d` being `--duty-cycle`: as long again at the default 0.5. Stopped, it + resumes at its cursor. Plain `migrate` at fence 2 runs the same backfill + at the defaults, with or without `--offline-indexes`. + - **Before either records 2,** it checks every native share, under the same + throttle, and refuses any that repeats the header of an accepted legacy + share, naming it, and leaves the cursor, the fence and the missing record + as they are. A failure of the argument above is then a loud stop, not a + silent double credit. + - **The record of 2** is one short transaction, which waits at most 2 s for + the migration lock or the cursor table's lock and keeps to the throttle's + statement timeout. An attempt that waits longer is rolled back and tried + again, backing off from 2 s to 30 s, 20 times, about ten minutes in all. + Then the run stops, naming what held it, with 2 unrecorded, and a rerun + goes straight to the check and the record. After any failed attempt the + run first reads whether 2 is recorded, so an attempt that committed but + lost its reply ends the run as a success. + - **A rerun is safe:** on a database whose backfill has finished, + `backfill-share-hashes` maps nothing and succeeds with + `"already_complete": true`. + - **At fence 2,** `self-check` reports a `share_hash_backfill` field with a + warning, never as a failure: `"state": "pending"` with the cursor, or + `"state": "unknown"` with the reason when it cannot read the backfill. The + field is absent once the read succeeds with nothing pending. The recovery + evidence export exports the cursor, as one `share_hashes_deferred` record, + in place of the partial mapping, and the summary marks `share_hashes` + deferred. A complete export's summary is unchanged. + - **While the backfill is pending,** `share-archive restore` refuses at any + fence, and `detach` and `drop` refuse every share-ledger partition, + whatever its bounds: the double-credit check reads the native shares + through the attached ledger, so none may leave it. Each refusal names + `backfill-share-hashes`, or plain `migrate`, to finish the backfill + first. `plan`, `seal`, `archive` and `verify` are unaffected. + - Without `--defer-share-hashes`, `migrate` keeps rc.4's behaviour: the + whole backfill, then 2 recorded, before anything serves. +- #745: `qbit-prism-server migrate --offline-indexes [--index-build-workers N] + [--index-build-memory SIZE]` builds 013's and 024's indexes with a plain + parallel `CREATE INDEX` when nothing else runs against the database. + - **Before:** on a populated ledger, `migrate` built them with + `CREATE INDEX CONCURRENTLY`, which reads the share ledger twice and waits + out every older snapshot, even in the cutover with every frontend stopped. + - **The offline run** takes each of the two migrations in one transaction. + Before any DDL it refuses an instance that has not reported drained or + stopped and a live legacy writer lease. It then locks the tables + `ACCESS EXCLUSIVE` with a 5 s lock timeout, builds, drops, and records the + version as it commits. A transaction still open on a table ends the run at + the lock timeout with nothing changed, and an interrupted migration rolls + back whole. + - `N` defaults to 4, at most 1024. `SIZE` is a `maintenance_work_mem` as + PostgreSQL writes it, such as `2GB`; without it the run takes 2GB, or the + server's setting when that is higher. A build's sort uses about that much + in all, divided among its leader and workers. + - **With `--defer-share-hashes`,** as the cutover's migrate step runs them, + the recent range is mapped first, before 013 drops the index that serves + it, and 013 and 024 then build offline with the backfill pending. A gated + test runs both flags in one run on a populated 2.x.x source, then starts a + frontend that serves. + - The plan's estimate is about 3.7 min of outage down to 1.5 to 2 min, + unmeasured here; the rehearsal's measurement is in the cutover's go/no-go + record. + - The flag is `migrate`'s alone. Without it nothing changes, and a frontend + started with `PRISM_POSTGRES_INIT_SCHEMA=1` always builds concurrently. +- Cutover tooling, and a test fix, merged after rc.4's tag, with no runtime + change: + - #728 (#712): `scripts/prism-recovery-evidence.py --jobs N` summarizes the + recovery evidence in parallel, byte for byte as the serial summarizer + does. With 8 jobs it ran about 6.5 times as fast on a 3.25 GB export, + which puts the 45-minute summary of a 65.9M-share restore at an estimated + 7 minutes. + - #732 (#731): 3.x refuses the window-proof parts digest of every 2.x + audit-bundle.v2 body, which 2.x hashed in insertion order. + `scripts/prism_legacy_parts_digest_check.py` lists the rows + `import-audits` would refuse for it, 9,531 on the cutover drill's restored + tree, and `scripts/prism_legacy_range_sidecars_sharded.sh` writes their + canonical sidecars in parallel shards. + - #736 (#734): `scripts/prism_legacy_import_admission_gate.sh` is a go-live + gate for frontends that admit traffic while `import-audits` still runs. It + checks every fact `self-check` checks except two: legacy audit + completeness, which it reports as pending, and, since #746, the pending + share-hash backfill, which it doesn't check. Track the backfill with + `self-check`'s `share_hash_backfill` field or the cursor. + - #740 (#738): the gate runs its carry-forward integrity check on the hot + standby `PRISM_INTEGRITY_REPORT_DATABASE_URL` names, or skips it with + `PRISM_GATE_INTEGRITY=skip`. It refuses a standby that is not in recovery, + replicates another database or timeline, has `hot_standby_feedback` on, + serves a writer that waits for its apply, or is stale. Every query runs + under a 15 s statement timeout and a 5 s lock timeout; the report gets + 120 s on the writer and 600 s on a standby. + - #747 (#724), test-only: the async-promotion live drill opens its gap + session on the work the frontend has published, not on the prepared row + it saves first. Required CI had failed that way three times. + +## Upgrading + +- **From 2.0.x:** as in the rc.4 notes, with these changes. + - This candidate requires every native schema migration through 026. + - **The cutover's migrate step,** with every frontend and tool stopped, is + `qbit-prism-server migrate --defer-share-hashes --offline-indexes`. It + maps the recent range and permits serving, builds 013's and 024's indexes + offline with the backfill pending, and returns saying that 2 is deferred. + Start frontends only once it has returned. + - **Its evidence:** the migrated summary marks `share_hashes` deferred. + Before go-live, compare every other record, with + `jq 'del(.records.share_hashes)'` on both summaries. Check the share + hashes once 2 is recorded, with the + [migration guide's query](../docs/prism-rust-migration.md#recovery-evidence-while-the-backfill-is-pending), + on a standby with replay paused, never on the primary. Summarize with + `--jobs` (#728). + - On a source whose `share_seq` sequence lags behind its rows, the deferred + run moves the sequence up to the backfill's frozen end and logs that at + INFO, so a before-and-after comparison of `pg_sequences.last_value` + differs there by design. A healthy source, a promoted physical copy + included, is left untouched. + - Before `import-audits`, list the 2.x v2 bodies it would refuse and write + their canonical sidecars (#732). To admit traffic while it runs, gate each + frontend on `scripts/prism_legacy_import_admission_gate.sh` (#736, #740). + The gate doesn't check the pending share-hash backfill: `self-check`'s + `share_hash_backfill` field or the cursor shows it. + - **After go-live, finish the backfill** with + `qbit-prism-server backfill-share-hashes`, once frontends serve and + `self-check` passes. Run that `self-check` only while the share rate is + low, below about 200 shares/s: its integrity report, an estimated 29 s + since 026, holds one snapshot on the primary (#738). Above that, rely on + the admission gate, with its report on a hot standby + (`PRISM_INTEGRITY_REPORT_DATABASE_URL`). Then, for the backfill: + - Run it at a time of ordinary load. Its defaults suit a primary taking + up to a few hundred shares a second. + - Run it where it survives a disconnect, with `RUST_LOG=info` for its + progress lines. At the defaults it is estimated at three to four hours + for 65.9M shares, unmeasured beside live load. + - It keeps to #738's snapshot rule by itself: every statement it runs is + short and in a transaction of its own. Its writes can still slow share + appends, so watch the primary's order-lock hold + (`qbit_prism_database_order_lock_hold_seconds{holder="append"}`) and + share acknowledgement times while it runs. If they climb, stop it + (Ctrl-C) and run it again later, or with a lower `--duty-cycle`: it + resumes where it stopped. + - Start no `migrate` while it runs: the `migrate` would wait for it. + - Until it has recorded 2, `self-check` warns, and + `share-archive restore`, `detach` and `drop` refuse. + - It ends by printing `"recorded": true`. Running it again afterwards is + safe. + + See + [the runbook](../docs/prism-ledger-ops.md#finish-a-deferred-share-hash-backfill-after-go-live) + and + [serving with the backfill pending](../docs/prism-rust-migration.md#serving-with-the-backfill-pending-fence-2). +- **From 3.0.0-rc.4: run `qbit-prism-server migrate` before starting any rc.5 + frontend.** It applies 026 in the migration transaction, with no capability + and no shutdown proof, so rc.4 frontends need not be stopped first: they + accept 026 with a warning and read the same report. + - **A binary-only upgrade, as rc.4's was, no longer starts.** At the native + default, `PRISM_POSTGRES_INIT_SCHEMA=0`, an rc.5 frontend refuses the + database with `database schema is missing migration(s) 26; …`, naming + `qbit-prism-server migrate` as the remedy. One started with + `PRISM_POSTGRES_INIT_SCHEMA=1`, compose's default, applies 026 itself. + - **The evidence export:** this candidate's + `scripts/prism-recovery-evidence.sql` requires 026, so it refuses an rc.4 + database that `migrate` hasn't reached yet (`missing required native + migrations {26} (found …)`). Export such a database with rc.4's script, + or run `migrate` first. + - A database that rc.4 served has recorded 2, so `--defer-share-hashes` and + `backfill-share-hashes` have nothing to do there. +- **Rolling back to rc.4:** + - **After an upgrade from rc.4, or once 2 is recorded,** an rc.4 frontend + starts on the database and accepts 026 with a warning. First take + `--timeout-seconds` off any `fatal-state clear` command line, since rc.4 + doesn't know the flag. rc.4's `self-check` and clear need the statement + timeout override again, 026 or not (below). + - **While the fence is 2,** from `migrate --defer-share-hashes` until 2 is + recorded, no rc.4 binary can start or migrate on the database: + - an rc.4 start meets the backfill's cursor first, and refuses with + ``database is not ready: migration 2's share-hash backfill has not + finished (#582). … Run `qbit-prism-server migrate` to resume it from + there; …``. **Don't follow that advice with rc.4:** rc.4's `migrate` + refuses fence 2 too; + - rc.4's `migrate`, and an rc.4 start with `PRISM_POSTGRES_INIT_SCHEMA=1`, + refuse with `refusing to migrate a native database at schema migrations + … before any DDL: database declares share_hash_backfill_pending = 2, but + this server understands share_hash_backfill_pending 1 to 1 only: …`. + - **So a rollback to rc.4 from fence 2 has two ways only:** finish the + backfill with this candidate's `backfill-share-hashes` first, after which + rc.4 starts as above, or begin again from the pre-migrate backup or + physical copy of the source, migrated with rc.4, whose outage includes + the whole backfill. Once frontends have acknowledged a native share, that + restore loses those shares: it is the D5 boundary, with its accounting + reconciliation, not an image rollback + ([the recovery contract](release-notes-3.0.0.md#migration-and-recovery-d5)). +- **Operating rules still in force:** + - **#737's statement timeout for rc.4's tools.** rc.4's `self-check` and + `fatal-state clear` run the integrity report under + `PRISM_DATABASE_STATEMENT_TIMEOUT_MS`, 15 s by default. That is too short + at production size even with 026, whose report is estimated at about + 29 s. So whenever they are rc.4's, before this candidate's `migrate` has + applied 026 or after a rollback to rc.4, run them with + `PRISM_DATABASE_STATEMENT_TIMEOUT_MS=120000`, on that process only and + never in a frontend's environment. This candidate's run the report under + their own timeouts. + - **#738's snapshot rule.** Every share append updates the cluster row inside + the order lock, and a snapshot held on the primary keeps that row's dead + versions, so the lock's hold grows while the snapshot lasts. At about + 400 shares/s the order lock saturates after about 18 to 20 s of it. While + miners are live: + - hold no snapshot or transaction on the primary for more than about 5 s + at peak, or about 60 s below 200 shares/s; + - never set `hot_standby_feedback=on`, and keep heavy write I/O off the + WAL device; + - run long reads on a standby with replay paused: the evidence export, + and, above about 200 shares/s, the integrity report, which holds one + snapshot for its whole run. The admission gate does that through + `PRISM_INTEGRITY_REPORT_DATABASE_URL`; otherwise run `self-check` while + the share rate is low. + + `backfill-share-hashes` keeps to this rule by itself: its statement + timeout is at most 5 s. The go-live step above says when to run it. + - **`fatal-state clear --timeout-seconds <10 to 3600>`,** 120 by default, + sets the clear's bound. The clear refuses with `raise --timeout-seconds` + when less than 6 s plus one RPC timeout (21 s by default) of the bound is + left as the report would start. When it exceeds its bound (`fatal-state + recovery exceeded seconds before its commit`), or its closing tip + check cannot end 5 s before the bound, nothing was committed and the halt + stays: rerun, with a larger bound if it ran out of time. Only a connection + lost during the commit leaves the outcome unknown; then check + `fatal-state show` and the event table before rerunning. Its three chain + refusals, `the chain reorganized during fatal-state recovery`, `the tip + moved and a pool block lies above the captured height` and `the node's + tip moved blocks during fatal-state recovery`, only ask for a rerun + once the tip has settled. See + [fatal-state recovery](../docs/prism-ledger-ops.md#fatal-state-recovery). + +## Verification + +The runtime changes between rc.4 and this candidate are in `migrate`, the +start gate and operator commands: +- #742, merged as `539d6c91`: `fatal-state clear` and `self-check`'s report, + in `ledger/fatal_state.rs` and `tools.rs`; +- #743, merged as `d838cfd7`: the deferred backfill, fence 2 and the start + gate in `ledger/migration.rs` and `ledger/migration/share_hashes.rs`, the + online runner in `ledger/migration/online.rs`, `ledger/connect.rs`, the + archive refusals in `ledger/archive.rs`, and `tools.rs`; +- #744, merged as `1a4e353c`: migration 026, its SQL and its registration in + `ledger/migration.rs`, which adds 26 to the versions every start requires; +- #745, merged as `2d103c32`: the offline index builds in + `ledger/migration/online.rs`, their options through `ledger/connect.rs` and + `ledger/migration.rs`, and `tools.rs`; +- #746, merged as `1ede828f`: the throttled finish and the record of 2 in + `ledger/migration/share_hashes.rs`, `ledger/migration.rs` and + `ledger/migration/online.rs`, and `backfill-share-hashes` and + `self-check`'s field in `tools.rs`. + +The operator's evidence scripts change with them: #744 makes +`scripts/prism-recovery-evidence.sql` require 026, and #746 makes it, +`scripts/prism-recovery-evidence.py` and +`scripts/prism-recovery-evidence-parallel.py` export and summarize a pending +backfill's cursor as `share_hashes_deferred`. + +None of them touches the share append (`ledger/window.rs`), the order lock, +refresh or settlement. So the pair evidence rc.3's notes cite still applies to +the serving path: the D1 peak-second gate, the 750/s ceiling and the failover +drill under load, run 1, all measured on #719's build. What is new on the pair +is the cutover itself, and a backfill running on the primary beside live +load. + +The weekly measured mainnet-shaped cutover still runs plain `migrate`, which +now also applies 026 (#744). None of #742 to #746 adds the new flags or +`backfill-share-hashes` to it. So in CI the new paths are proved by their +gated PostgreSQL tests: +- `--defer-share-hashes` and `--offline-indexes` by `share_hash_backfill`'s + and `index_trim`'s, one of which runs both flags in one run, as the cutover + does; +- `backfill-share-hashes` by `share_hash_backfill`'s: the throttle, plain + `migrate` at the default throttle, the record's attempts, the double-credit + check and `self-check`'s field. The lib's `share_hashes::postgres_tests` + cover a record whose commit reply was lost and a cursor dropped while it + is read, and `migration_rollback`'s covers the export at fence 2, with the + parallel export byte for byte the serial one. `config_cli` and + `self_check_cli` pin the command line and the reports, and the Python tests + cover the export scripts; +- the clear by `fatal_state`'s, and 026 by `offer_lifecycle`'s equivalence of + its report with 001's. + +On the pair, #291's rehearsal of this candidate is the proof. Its first round +measured the `migrate --defer-share-hashes` command, on `539d6c91`, at +5 min 06 s end to end, with 013's and 024's indexes built concurrently +(above). The rehearsal's full measurements are recorded in the +cutover's go/no-go record: the recent range, the offline index builds, +`migrate` with both flags, an hour of throttled backfill beside live load, +and a `fatal-state clear` with F5 and 026. + +Otherwise the evidence is the same set as rc.4's (#557): +- CI on the version-bump pull request, and dispatched on the tag; +- the L3 production-window matrix on the version-bump pull request, which + the tag promotes on a tree match (#608); +- the targeted nightly batch: `short-plan-fake-node`, `short-plan-real-node`, + `soak-smoke`, `faults-short-real-node` and `faults-failover-fake-node`; +- the live regtest weekly scenarios. + +rc.1's full nightly set and 5.5 h soak carry over. The evidence bundle on the +`v3.0.0-rc.5` GitHub pre-release lists every run, and #291 links to it. + +## Known issues + +As in the rc.4 notes, with these changes: + +- **Fixed since rc.4:** #737's F1, F2, F4 and F5, by #742 and #744. +- **Closed since rc.4:** #582. Its fix has been in every candidate since rc.1, + and `migrate` has completed on a production-sized copy at the default + statement timeout. What remained, the backfill's time inside the outage, is + what `--defer-share-hashes` takes out. +- **Open since rc.4, not fixed in this candidate:** + - **#737 (open part):** F3, the clear still asks the node once per pool + block and once per deep-confirmed fanout, one call at a time (#690). And + the `/audit/carry-forward-integrity` and `/audit/ledger-integrity` routes + still run the report under `PRISM_DATABASE_STATEMENT_TIMEOUT_MS`. Since + 026 the report is estimated at about 29 s at production size, so they + would still time out at the default 15 s: don't poll them. + - **#738:** every share append updates the `qbit_prism_cluster` row, so a + long snapshot on the primary slows the append and, at high share rates, + saturates the order lock. The operating rule above applies. The fix, which + derives each append's clock from append-only state, comes in a later + candidate (planned for rc.6). + - **#739:** `backfill-ctv` refuses every legacy CTV set: 2.x stored each + set's digest over sorted keys, and 3.x checks it in struct order. The + cutover skips `backfill-ctv`. A census of a migrated production ledger + found no set for it to insert, and the frontends' maturity sweep already + rewrites mature blocks' artifacts. Proposed for rc.6. + - **#741:** CI runs the admission gate's real-schema tests as the + database's superuser, not as the frontend's role, which owns the schema + without being a superuser. A one-off run as such a role passed. Tooling + only. + - **#731 and #734, after the cutover:** 3.x still refuses 2.x's v2 + window-proof parts digest, so those bodies import through canonical + sidecars (#732). And production `self-check` still stops at legacy audit + completeness while `import-audits` runs, which the admission gate covers + (#736). +- **Still open from rc.4:** + - #716's open part and #711 are as the rc.4 notes describe them. #716's + open part is a share admitted under a tip-change lease and refused, never + credited, when its commit gate finds the publication authority busy. + - **#724:** #747 fixes one way the async-promotion live drill failed in CI: + a gap session that was issued the work published before the gap. Its + other intermittent answers, `stale-job` and `backend-rpc-unavailable`, + stay open. + - **#712:** the parallel export (#721) and the parallel summarizer (#728) + shorten the evidence step. Taking the source export off the critical path + is still open, and both exports still sit inside the cutover outage. diff --git a/doc/release-notes-3.0.0-rc.6.md b/doc/release-notes-3.0.0-rc.6.md new file mode 100644 index 000000000..7903aee0d --- /dev/null +++ b/doc/release-notes-3.0.0-rc.6.md @@ -0,0 +1,293 @@ +# qbit-mining-bootstrap 3.0.0-rc.6 Release Notes + +Release date: 2026-10-08 (release candidate; tag `v3.0.0-rc.6`, published as +a GitHub pre-release). + +This is the sixth release candidate of 3.0.0: 3.0.0-rc.5 plus #754's fix for +a 3.x regression in which the pool hashrate series timed out on large +ledgers, and #750's fix for #748, a finished share-hash backfill reported as +failed when its record's connection dropped. It also carries a test-only fix +for #724 (#753). It is the tree that #291's go/no-go evaluates for production +on the PRISM pair. It is not a production release. Until #291 records a go, +the production line stays 2.x.x (2.0.2). + +Everything in the [3.0.0-rc.1](release-notes-3.0.0-rc.1.md), +[3.0.0-rc.2](release-notes-3.0.0-rc.2.md), +[3.0.0-rc.3](release-notes-3.0.0-rc.3.md), +[3.0.0-rc.4](release-notes-3.0.0-rc.4.md) and +[3.0.0-rc.5](release-notes-3.0.0-rc.5.md) notes applies, with the changes +below. Unlike rc.5, an upgrade from the previous candidate replaces the +binary only: this candidate adds no migration ([Upgrading](#upgrading)). + +## Changes since 3.0.0-rc.5 + +- #754 fixes a 3.x regression where the pool hashrate series, + `/public/v1/hashrate-series`, timed out on large ledgers. It bounds both + of the series' ledger reads at both ends, so that, once the rollups have + run, no plan walks the ledger for them. Every candidate from rc.1 to rc.5 + has the regression: 2.x built the same read with bounded predicates. + - **Before:** the series takes most buckets from the rollup tables and + re-reads two kinds of rows from the share ledger: the boundary, which is + the current bucket and the leading partial one, and the tail, the shares + past the rollups' watermark. Neither read gave the planner an index + range it could use. The boundary bounded `accepted_at` with an `OR`, + plus a `NULL` guard on the range's start. The tail bounded `share_seq` + only from below, by the watermark, with the same guard on time and an + `OR` on the miner. + - **On a production-sized ledger,** 66M shares with the rollups current, + PostgreSQL walked whole indexes. rc.5's default pool request took 19 to + 40 s. With only the boundary fixed, as #754 first had it, the generic + plan's tail still walked `qbit_share_ledger_p0_accepted_recent_idx`, 66M + entries for no row, in 17 to 27 s. And a custom plan for a miner walked + the miner's whole history. So pool hashrate-series requests answered + 504, past the public API's read deadline, and big miners' charts were + slow. PRISM itself answers a read that outlasts + `PRISM_PUBLIC_READ_STATEMENT_TIMEOUT_SECONDS` with 503 `read_timeout`. + - **Now,** in `dashboard_hashrate_rollups.sql`, the boundary is two + disjoint slices, each bounded by `accepted_at` at both ends: a range + scan on `qbit_share_ledger_accepted_recent_idx`, or on the miner index + for a miner. `GREATEST` and `LEAST` drop a missing range start or first + full bucket, and without a range start the leading slice is empty. The + tail takes its shares from a subquery that bounds `share_seq` on both + sides, by the watermark and by the ledger's `max(share_seq)`, one + backward probe per partition. `OFFSET 0` fences it, so the time and + miner tests only filter its stream, and it reads only the shares past + the watermark: none while the rollups are current, the lag while they + catch up. + - **Measured** on a 6M-share copy of a 2.x ledger migrated to the native + schema, with a 600,000-share miner added: every request ran in 1 to + 17 ms, under generic, custom and auto plans, at `random_page_cost` 4 + and 1.1, for the pool, a small miner and the big miner, with and + without an anchor. On that copy, at `random_page_cost` 1.1, rc.5's + default request had taken 1,957 ms under the generic plan and 346 ms + under a custom one. On a production-sized ledger, 66M shares, the fixed + query took 9 to 23 ms for the pool and 2 to 15 ms for miners in all 20 + runs, across generic, custom (with and without an anchor) and auto + plans. rc.5's query took 15.8 to 25.2 s on the same requests, and the + outputs were identical in 7 of 7. + - **The output,** compared in 312 cases, matched rc.5's: every allowed + range and bucket pair, the pool, two miners and an unknown miner, a + smoothing context of 0 or 2 h, with and without an anchor, and the + watermark current, 200,000 shares behind or without a progress row. The + exceptions were unanchored cases whose range start slid while the slower + old query ran, which differed by one share, in the leading bucket only. + - **By design, the tail's cost follows the rollups' lag.** With no + progress row, before the rollups first run, it reads every share, as + rc.5's did. Alarm on the lag, + `qbit_prism_hashrate_rollup_watermark_lag_seconds`. + - It is a read-only change to the public read path: no migration, no + schema change and no write. It applies wherever `/public/v1` is served, + on the `public-api` service and on each frontend's coordinator listener. + The raw fallback, `dashboard_hashrate_series.sql`, read only when the + rollup tables are absent, is unchanged; both the 2.x and the 3.x + schemas have them. +- #748, by #750: `backfill-share-hashes`, and plain `migrate` at fence 2, no + longer report a finished backfill as failed when the record of 2 + committed but its connection was lost, as long as a fresh connection can + read the database in time (below). + - **Before:** after a failed attempt to record 2 while frontends serve, + the run read whether the attempt had recorded 2 all the same, on the + attempt's own connection. When that connection had dropped, and its + `COMMIT`'s reply with it, the read failed too, so a run whose `COMMIT` + had recorded 2 stopped and asked to be run again. That was safe, since + the rerun finds 2 recorded and says so, but it reported a finished + backfill as failed. The rc.5 notes' "an attempt that committed but lost + its reply ends the run as a success" held only while the attempt's + connection survived. + - **A lost connection** is one on which the attempt, or the read after it, + failed with a transport, TLS or protocol error, one the server ended + (SQLSTATE class 08, 57P01 to 57P05 or 25P03: a termination, a crash or + restart, a dropped database, an idle timeout), or one on which the + attempt got no reply within its own deadline. The read then runs on a + fresh connection from the same URL, search path and TLS, under the + record's 2 s lock timeout and the throttle's statement timeout. While the + server turns it away, it connects again after 250 ms, doubling to 2 s, + for up to 14 s at the defaults and 17 s at the 5 s cap. With 2 recorded + and the cursor gone, the run succeeds. If the fresh read fails too, the + run stops, naming each failure: on a dead network, for one, it gives up + at that deadline and stops as rc.5's run did. + - **Each attempt's own deadline** is its ten statements' statement and + lock timeouts plus 5 s: 45 s at the defaults, 75 s at the cap. The + server's own timeouts end a healthy attempt well within it, so an + attempt still unanswered is on a half-open connection, a network + partition say, whose `COMMIT` would otherwise wait out TCP's + retransmissions for some 15 minutes. It is abandoned, and its connection + taken as lost. + - **A lost connection that shows 2 unrecorded stops the run,** rather than + a retry, since the runners' lock, a session lock on that connection, + went with it: `refusing to try recording migration 2 again: an attempt + lost its connection, and on a fresh one 2 was not recorded when + checked. …`, or `… an attempt failed, and the connection was lost while + checking whether it had recorded 2 (…); on a fresh one 2 was not + recorded when checked. …`. A `COMMIT` still in flight there, waiting for + a synchronous standby say, can land moments later. Run + `backfill-share-hashes` again: it takes the lock, then finds 2 recorded + and says so, or goes straight to the double-credit check and the record. + - The batches, the throttle, the double-credit check, and the number of + record attempts and their backoff are rc.5's. So are the record of 2 + before anything serves, at fence 1, and `migrate --defer-share-hashes`, + which leaves 2 unrecorded. +- #753 (#724), test-only: the async-promotion live drill's new-tip shares + wait for work at the payout revision the final block's confirmation set. + - The drill's last check sends a share from each surviving session on + work for the tip the final block moved to, and requires it accepted and + credited once. It took the first job built on the new tip as that work. + The block moves the payout revision twice on that tip, though: when a + frontend first records the tip, and when the block's landing confirms + it. Work published between the two is superseded once the confirmation + commits, and its frontend answers a share on it `stale-job` until it + republishes (#752). Required CI had failed that way twice. + - Each session now waits for a job on work at the ledger's payout + revision, passing over jobs on superseded work or without their records. + The first job on superseded work the drill meets gets one share, which + must be refused, `stale-job` or, for a job retired past its retention, + `unknown-job`, and leave no ledger row: the original symptom, kept as a + check. One 30 s deadline bounds every step of the phase, share + submissions and ledger reads included. + +## Upgrading + +- **From 2.0.x, or from 3.0.0-rc.4:** as in the + [rc.5 notes](release-notes-3.0.0-rc.5.md#upgrading), with this candidate + in rc.5's place. That covers `migrate` for 026, which every start of this + candidate requires, as rc.5's does; the cutover's + `migrate --defer-share-hashes --offline-indexes`, and finishing the + backfill after go-live with `backfill-share-hashes`; rolling back to rc.4 + at fence 2; and the operating rules still in force: #737's statement + timeout for rc.4's tools, #738's snapshot rule and + `fatal-state clear --timeout-seconds`. From an earlier candidate, the + upgrade sections of each later candidate's notes apply in turn. +- **From 3.0.0-rc.5: no migration; replace the binary on every frontend and + tool, and on the `public-api` service.** This candidate requires the same + native schema migrations as rc.5, 2 to 26, declares the same capabilities + and understands the same share-hash fence values, 1 and 2. It adds no + setting and changes no evidence or gate script. + - **A backfill that rc.5 deferred,** at `share_hash_backfill_pending = 2`, + is served and finished as before: this candidate's frontends start + beside it, each logging the cursor, and its `backfill-share-hashes` + resumes at the cursor. An rc.5 run still going can finish, or be stopped + (Ctrl-C) and run again with this candidate. +- **A cutover that pinned rc.5 and hasn't gone live** can take rc.6 before + or after its `migrate` step. A database that rc.5's `migrate` migrated, + with or without `--defer-share-hashes` and `--offline-indexes`, is served + by rc.6 unchanged: there is no migration for it, no new capability, and + the same settings give the same configuration fingerprint. So start + rc.6's frontends on it with no second `migrate`, and finish the backfill + after go-live with rc.6's `backfill-share-hashes`, as the rc.5 notes + describe. rc.6 runs the cutover's + `migrate --defer-share-hashes --offline-indexes` as rc.5 does, since #750 + changes only the record of 2, which that run defers. The evidence and + admission-gate scripts are rc.5's, byte for byte. +- **Rolling back to rc.5:** replace the binary, at fence 2 too; nothing else + changes. rc.5 serves the same database, and its `backfill-share-hashes` + finishes a pending backfill. With it, #748 returns: after a lost + connection, its run can stop and ask to be run again when its `COMMIT` + had recorded 2, and the rerun then succeeds with + `"already_complete": true`; and an attempt on a half-open connection can + again wait out TCP's retransmissions, some 15 minutes. The pool hashrate + series times out again on a large ledger. + +## Verification + +The runtime changes between rc.5 and this candidate are in the public read +path and in the record of 2 at fence 2: +- #754, merged as `da02b1cc`: the pool hashrate series' two ledger reads, its + boundary and its tail, in `api/queries/dashboard_hashrate_rollups.sql`; +- #750, merged as `97bcd7e3`: the read after a failed record of 2 and each + attempt's deadline, in `ledger/migration/share_hashes.rs`, with its two + callers in `ledger/migration.rs` (`backfill-share-hashes`) and + `ledger/migration/online.rs` (plain `migrate`). It also updates the + migration guide's record paragraph and refusal table, and the ledger + runbook. + +#753, merged as `c8efdaef`, changes only the live +drill's support code, `tests/support/live_pg_async_live_miners.rs` and +`tests/support/live_share_client.rs`. + +None of them touches a migration, a capability or a setting, the share +append (`ledger/window.rs`), the order lock, refresh or settlement, and +#754 changes one read-only public query. So the pair evidence rc.3's notes +cite still applies to the serving path: the D1 peak-second gate, the 750/s +ceiling and the failover drill under load, run 1, all measured on #719's +build. For #291's rehearsal of rc.5: this candidate runs the cutover's +`migrate` as rc.5 does, and of the backfill's finish only the read after a +failed record of 2 and each attempt's deadline are new. + +In CI, the changes are proved by their gated tests: +- #754 by `api_2x`'s: the rollup series must equal the raw ledger series in + 78 cases, every range and bucket pair, context, anchor and subject, + before the rollups run, at a current watermark and with shares past it. + And on 20,000 rolled-up shares, half of them one miner's, five requests, + the pool's, a small miner's and the big miner's, anchored or not, run + under both plan modes and at `random_page_cost` 4 and 1.1, with the + rollups current and again after 300 shares land past the watermark. Each + plan may read at most 1,000 ledger shares, plus the lag in the second + run, counting filtered and recheck-dropped rows; must bound every ledger + index scan outside a `Limit` probe on one column from both sides; and + must run no sequential scan over ledger rows. It fails against rc.5's + query and against the boundary-only fix; +- #750 by the lib's `share_hashes::postgres_tests`, against a real + PostgreSQL, most through the crate's execution proxy: a `COMMIT` that + completes before its connection drops, fresh connects refused and then + told 57P03, a reply held past the attempt's deadline, a drop before the + `COMMIT`, a connection lost while checking a failed attempt, and a reply + held before any `COMMIT`, whose lost connection is dropped, never closed. + Unit tests pin which errors read as a lost connection, and the deadlines; +- #753 by the drill itself, in the live regtest suite that required CI + runs. + +Otherwise the evidence is the same set as rc.5's (#557): +- CI on the version-bump pull request, and dispatched on the tag; +- the L3 production-window matrix on the version-bump pull request, which + the tag promotes on a tree match (#608); +- the targeted nightly batch: `short-plan-fake-node`, `short-plan-real-node`, + `soak-smoke`, `faults-short-real-node` and `faults-failover-fake-node`; +- the live regtest weekly scenarios. + +rc.1's full nightly set and 5.5 h soak carry over. The evidence bundle on the +`v3.0.0-rc.6` GitHub pre-release lists every run, and #291 links to it. + +## Known issues + +As in the rc.5 notes, with these changes: + +- **Fixed since rc.5:** + - **#748, by #750:** rc.5's `backfill-share-hashes`, and plain `migrate` + at fence 2, could report a finished backfill as failed when the + record's connection dropped (above). + - **#724, on the test side:** #753 fixes the async-promotion live drill's + new-tip shares, as #747 fixed its drill race in rc.5. The product side + of the new-tip `stale-job` is #752, below. +- **Open since rc.5, not fixed in this candidate:** + - **#752, after the cutover:** the product behaviour behind #724's + `stale-job`, a burst of `stale-job` rejections after each pool block. + The block moves the payout revision twice on its new tip: when a + frontend first records the tip, and when the landing confirms the + block. Work a frontend publishes on the new tip between the two is + superseded once the confirmation commits, and the frontend refuses + shares on it, and on anything else it still publishes, as `stale-job` + until it republishes on the confirmed revision: the landing frontend at + once, the others at their next refresh tick (`PRISM_BLOCKPOLL_SECONDS`, + 2 s by default). So miners see slightly more `stale-job` rejections + around the pool's own blocks than on 2.x, which invalidates in-flight + work once per pool block, most of them on the frontends that didn't land + the block. Nothing is credited wrongly and no acknowledged share is + lost. `qbit_prism_stale_job_rejections_total{cause="payout_revision"}` + around each pool block shows them. +- **Still open from rc.5:** + - **#738, after the cutover:** as the rc.5 notes describe it, under the + same operating rule. Its fix, which derives each append's clock from + append-only state, is now planned after the cutover, as the first + capacity change after go-live, not for rc.6 as rc.5's notes said. At + production's current share rates it would change nothing measurable, + and shipping it before go-live would reopen the throughput + qualification, since it changes the money path's time semantics on the + hot path. The issue lists what would bring it forward, the share rate + and the order lock's load among them, and what to watch until then. + - **#739:** as the rc.5 notes describe it. It was proposed for rc.6 and is + not in this candidate. The cutover still skips `backfill-ctv`. + - **#724:** its other intermittent answer, `backend-rpc-unavailable` to a + lost acknowledged share's replay after the promotion, #716's commit-gate + class, stays open, and with it the issue. + - #737's open part, #741, #731 and #734, #716's open part, #711 and #712 + are as the rc.5 notes describe them. diff --git a/doc/release-notes-3.0.0.md b/doc/release-notes-3.0.0.md new file mode 100644 index 000000000..cb2fda758 --- /dev/null +++ b/doc/release-notes-3.0.0.md @@ -0,0 +1,309 @@ +# qbit-mining-bootstrap 3.0.0 Release Notes + +Release date: 2026-10-22 (tag `v3.0.0`, cut from the `3.0.x` branch). + +3.0.0 is 3.0.0-rc.6 plus #770's fix for #765, in which `import-audits` held +`SETTLEMENT_LOCK` while it sent and stored each legacy audit body, so a large +body stalled settlement and template refresh on the frontends serving beside +it, and #760's fix for #759, in which the node RPC client reused an idle +connection past qbitd's own idle close. Those two pull requests were prepared +as 3.0.0-rc.7 (#772), which was never tagged. `3.0.x` branches from `3.x.x` +at #770's merge, `4c134913`. `3.x.x` continues as 3.1 from 3.1.0-rc.1, which +adds the dual writer and migrations 027, 028 and 031; 3.0.0 has none of them. +#291 records the cutover qualification and the go/no-go for the production +rollout. + +Everything in the [3.0.0-rc.1](release-notes-3.0.0-rc.1.md), +[3.0.0-rc.2](release-notes-3.0.0-rc.2.md), +[3.0.0-rc.3](release-notes-3.0.0-rc.3.md), +[3.0.0-rc.4](release-notes-3.0.0-rc.4.md), +[3.0.0-rc.5](release-notes-3.0.0-rc.5.md) and +[3.0.0-rc.6](release-notes-3.0.0-rc.6.md) notes applies, with the changes +below; see also the +[development-line status](release-notes-3.x.x-development-line.md). Like +rc.6, an upgrade from the previous candidate replaces the binary only: 3.0.0 +adds no migration ([Upgrading](#upgrading)). + +## Changes since 3.0.0-rc.6 + +- #770 (#765): `import-audits` writes each legacy audit before it takes + `SETTLEMENT_LOCK`, holds the lock only to check the write guard and + commit, and bounds every phase of a row's transaction. + - **Before:** for each row, rc.6 took `SETTLEMENT_LOCK`, checked the write + guard, and only then encoded the row's UPDATE and sent it: the canonical + body as `bytea`, beside the audit's commitment and witness Merkle leaves + as JSON. The lock was held while the client encoded and sent the + statement and while the server stored it. Nothing bounded the send: the + session's `statement_timeout` (`PRISM_DATABASE_STATEMENT_TIMEOUT_MS`, + 15 s by default) starts only once the whole statement has arrived. In + #765, a legacy audit with a ~112 MB canonical body held the lock for + about 90 s, until it was killed, while the server waited for the rest + of the statement (`idle in transaction`, waiting on `ClientRead`). + Every other `SETTLEMENT_LOCK` user waited behind it: the frontends' + advisory-lock waits timed out, template refresh was deferred, and + settlement would have waited too. `docker stop` didn't stop it either: + in a container the command runs as PID 1, which ignores a SIGTERM it + has no handler for. + - **Now, per row:** the body is loaded, verified and canonicalized, and all + nine UPDATE parameters are encoded, on one blocking thread before any + transaction opens. The UPDATE is then sent and run on the row's own + connection, holding only that row's lock. Only then does the importer + poll for `SETTLEMENT_LOCK` with `pg_try_advisory_xact_lock`, every 25 ms + and never queued, check the write guard again under it, and commit. + Every other `SETTLEMENT_LOCK` holder still sees a row committed before + it took the lock, or not at all. The write guard also runs before the + write, so a halted cluster or a live legacy writer is refused before + anything is sent. + - **A lock holder that needs the row goes first.** A poll that misses the + lock also walks the lock holder's blockers (`pg_blocking_pids`, then + theirs in turn). If the holder waits on this row, directly or through + another session, the row is rolled back so the holder can proceed, and + the command fails naming the row. Other sessions waiting on the row, + such as a second import, just wait. + - **Bounds:** two new settings, read by `import-audits` only. + - `PRISM_IMPORT_AUDITS_WRITE_TIMEOUT_MS` (60000; 1000 to 600000) is the + write's transaction-local `idle_in_transaction_session_timeout`, + `statement_timeout` and `lock_timeout`. So the server ends a session + that stops sending mid-statement, and cancels an UPDATE that runs, or + waits for the row, that long. + - `PRISM_IMPORT_AUDITS_LOCK_TIMEOUT_MS` (5000; 100 to 600000) bounds the + wait for `SETTLEMENT_LOCK`. Once the lock is held, it is also the + transaction's idle and statement timeout, so a client or statement + that stalls under the lock loses its session, and the lock, within + that long. + - The row's connection also sets `client_connection_check_interval` to + 1 s where the server supports it (PostgreSQL 14 or later, on Linux). + An abandoned row is then rolled back within about a second, even while + the server is storing it. + - **A row that exceeds a bound** is rolled back, and the command exits 1 + with an error naming the row, `importing legacy audit `, and + its cause, such as `SETTLEMENT_LOCK stayed busy for 2000 ms; the row is + rolled back and nothing was stored`. Rows imported before it stay + imported, and a re-run resumes from it. + - **SIGTERM or SIGINT stops the command.** It stops before the next row, + or rolls back a row that has not been granted the lock, and exits 1 with + `import-audits stopped on request after importing audit bodies; …`. + A row that holds the lock finishes its commit first, and a stop that + lands once the last row is in ends as a finished import. A second signal + exits at once. + - **Each row it writes prints one line on stderr,** `imported`, or + `unchanged` for a row that changed after it was read, for example + `import-audits: block_hash= imported + parameter_bytes=block_hash:64,…,canonical_audit_bytes: + total_parameter_bytes= prepare_ms=… write_ms=… lock_wait_ms=… + lock_held_ms=… total_ms=…`. stdout keeps the completeness counts and + `Imported audit bodies`. + - **Measured** in a release build, over TLS through a TCP proxy, on a + synthetic legacy row with a 112.2 MB canonical body and a 37.5 MB + witness-leaves array. rc.6's order held `SETTLEMENT_LOCK` for 3.145 s: + 0.10 s encoding the statement, 3.01 s sending it and the server storing + it, about 0.15 s of that the send, and 28 ms for the guard and the + commit. 3.0.0 holds it for 13 ms. With the client's send + slowed to 16 MiB/s, the hold was 14.9 s against 16 ms. A session taking + the lock every 20 ms waited at most 3.13 s and 14.9 s beside rc.6's + order, and 3 ms and 17 ms beside 3.0.0's. + - **Why #765's send took so long is not proven:** it did not reproduce + here. The likely cause is #771, below. 3.0.0's `import-audits` + doesn't depend on it: whatever slows or stops the send now happens + before the lock and within the write timeout. The client also polls the + statement every 100 ms, which un-sticks #771's lost wakeup if it occurs. + - It changes `import-audits` only: the command, its transaction and two + setting names. There is no migration or schema change, and nothing the + frontends or the `public-api` service run changes. +- #760 (#759): every node RPC client keeps an idle connection for at most + 20 s. + - **Before:** the frontends' qbitd RPC client kept an idle kept-alive + connection for 60 s, and the public API's node client for reqwest's + default 90 s, while qbitd closes one after its `-rpcservertimeout`, 30 s + by default. A request written into a connection just as the node closes + it is never read, and fails after the connection existed: the plain + `transport failed` error, which nothing retries. On `submitblock` that + is an unknown outcome, and the block is never offered again. One + deployment, with `PRISM_CTV_BROADCASTER_POLL_SECONDS=30` (the default is + 10), logged the error from the CTV broadcaster, whose passes ran in step + with the 2 s refresh, on about one pass in eight to fifteen. + - **Now** every node client is built by `node_client_builder()`: the + frontends', the public API's and the load harness's. It keeps an idle + connection for at most 20 s (`POOL_IDLE_TIMEOUT`), so the client drops + it before the node can. A compile-time assert keeps the bound below + qbitd's 30 s default. Relay calls, `submitblock` included, share the + bounded pool as before. + - **Operator rule:** keep qbitd's `-rpcservertimeout`, and the idle + timeout of any proxy between a PRISM process and its node, above 20 s. + qbitd's default, 30 s, is. +- TLS dependencies, for the RustSec advisories `cargo deny` reports against + rc.6's lockfile: rustls 0.23.45 to 0.23.46 (RUSTSEC-2026-0346, -0347, + -0348 and -0349), rustls-webpki 0.103.15 to 0.103.16 (RUSTSEC-2026-0382) + and webpki-roots 1.0.9 to 1.0.10 (RUSTSEC-2026-0381), in the root and fuzz + lockfiles. They are patch releases within the versions rc.6 already + resolved, with no other dependency change. The serving binary's + PostgreSQL (sqlx) and HTTPS (reqwest) clients use them. + +## Upgrading + +- **From 2.0.x, or from 3.0.0-rc.4:** as in the + [rc.5 notes](release-notes-3.0.0-rc.5.md#upgrading), with 3.0.0 + in rc.5's place. That covers `migrate` for 026, which every start of + 3.0.0 requires, as rc.5's does; the cutover's + `migrate --defer-share-hashes --offline-indexes`, and finishing the + backfill after go-live with `backfill-share-hashes`; rolling back to rc.4 + at fence 2; and the operating rules still in force: #737's statement + timeout for rc.4's tools, #738's snapshot rule and + `fatal-state clear --timeout-seconds`. From an earlier candidate, the + upgrade sections of each later candidate's notes apply in turn. +- **From 3.0.0-rc.5:** replace the binary, as the + [rc.6 notes](release-notes-3.0.0-rc.6.md#upgrading) describe for rc.6; + everything below about rc.6 applies too. +- **From 3.0.0-rc.6: no migration; replace the binary on every frontend and + tool, and on the `public-api` service.** 3.0.0 requires the same + native schema migrations as rc.6, 2 to 26, declares the same capabilities + and understands the same share-hash fence values, 1 and 2. The same + settings give its frontends the same configuration fingerprint, so they + can replace rc.6's one at a time. It changes no evidence or admission-gate + script. + - **The two new settings are accepted by 3.0.0 binaries only.** rc.6's + `run` and `check-config` refuse a `PRISM_*` name they don't know in + production mode (`QBIT_CHAIN` main or mainnet, `QBIT_PRODUCTION` or + `QBIT_TOOLS_PRODUCTION`), and warn about it otherwise. So pass them only + to a 3.0.0 `import-audits` run, never through an environment that an + rc.6 frontend reads. + - **#760's node rule:** check that qbitd's `-rpcservertimeout`, and the + idle timeout of any proxy between a PRISM process (a frontend, a tool or + the `public-api` service) and its node, are above 20 s. + - **`import-audits` beside serving frontends** no longer needs the guard + #765 describes, one that kills it with SIGKILL once a settlement waiter + has waited a few seconds. Its hold is now the guard read and the commit, + and `docker stop` stops it. +- **Rolling back to rc.6:** first remove the two settings from every + environment an rc.6 binary will read, then replace the binary; nothing + else changes. With rc.6, #765 returns: run its `import-audits` beside + serving frontends only under the guard #765 describes. #759 returns too: + the node clients keep an idle connection for 60 s again, 90 s on the + public API. + +## Verification + +The runtime changes between rc.6 and 3.0.0 are in `import-audits`, in how +long the node clients keep an idle connection, and in the TLS crates' patch +releases above: +- #770, merged as `4c134913`: `import-audits` only, in + `ledger/migration.rs` (`Ledger::import_legacy_audits_with` and its per-row + transaction) and `tools.rs` (the command, its two settings, its stop + signal and its stderr line), with the two names in + `config/native-settings.txt` and a re-export in `ledger.rs`. It adds + `tests/import_audits_settlement_lock.rs` and its entries in + `test/prism-gated-tests.txt`, and updates the server's README, the ledger + runbook and the pool-acquire census. +- #760, merged as `12f1ae8e`: the node client's idle bound in `rpc.rs`, used + by the public API's node client (`api.rs`) and the load harness's, with + the node requirement in the migration guide. + +Neither touches a migration, a capability, the share append +(`ledger/window.rs`), the order lock or settlement. #760 changes only how +long each node client keeps an idle connection. So the serving evidence +rc.3's notes cite still applies: the D1 peak-second gate, the 750/s ceiling +and the failover drill under load, run 1, all measured on #719's build. + +In CI, the changes are proved by their tests: +- #770 by `import_audits_settlement_lock`'s 13 gated tests. They put the + importer's connections behind a relay that slows or stalls what the client + sends, which leaves the server in #765's state, and check: + - that a session taking `SETTLEMENT_LOCK` every 20 ms beside the import of + a 107 MB body with a 37.5 MB witness-leaves array waits less than 3 s, + while the throttled write takes longer than that (rc.6's order fails + it); + - that a stall mid-send and a stall under the lock each end within their + bound, and that a busy lock fails the row within its bound; + - that a lock holder waiting on the row goes first, directly or behind + another session; + - that a stop mid-send, mid-execution, before the grant, between rows and + after the last row each behaves as above; + - and, on the real command, its SIGTERM exit and its busy-lock output. +- #760 by `rpc_idle_close`: the real client against a loopback node that + drops a request arriving after its idle limit, as the race does. With a + bound below the node's, the next call opens a new connection and is + answered; with a bound past it, the call fails with the production error. + +Otherwise the evidence is the same set as rc.6's (#557): +- CI on the version-bump pull request, and dispatched on the tag; +- the L3 production-window matrix (`l3-full`) on the `v3.0.0` tag push. The + version-bump pull request targets `3.0.x`, where the matrix doesn't run + on pull requests, so the tag runs it rather than promoting a pull + request's run (#608); +- the targeted nightly batch: `short-plan-fake-node`, `short-plan-real-node`, + `soak-smoke`, `faults-short-real-node` and `faults-failover-fake-node`; +- the live regtest weekly scenarios. + +rc.1's full nightly set and 5.5 h soak carry over. The evidence bundle on the +`v3.0.0` GitHub release lists every run, and #291 links to it. + +## Known issues + +As in the rc.6 notes, with these changes: + +- **Fixed since rc.6:** + - **#765, by #770:** rc.6's `import-audits` held `SETTLEMENT_LOCK` while + it sent and stored each body, with nothing bounding the send, and ignored + `docker stop` (above). + - **#759, by #760:** the node clients kept an idle connection past qbitd's + idle close, so a call could fail in the close (above). +- **Open since rc.6, not fixed in 3.0.0:** + - **#771:** sqlx 0.8.6's rustls stream can lose the wakeup of a large write. + A large statement sent over TLS can then stall, with the server waiting + for the rest of it, until something closes the connection. It is the + likely cause of #765's hold. 3.0.0's `import-audits` works + around it (above), but other large statements from the serving binary + don't. The fix is upstream, launchbadge/sqlx#4251, which no 0.8.x + release carries; #771 tracks backporting it. + - **#762:** a `submitblock` whose deadline, + `PRISM_BLOCK_SUBMIT_RPC_TIMEOUT_SECONDS` (1 s by default), passes while + its connection is still opening is recorded as an unknown offer, not an + unsent one, so that block is never offered again. It needs no free + pooled connection when the call starts and a connect slower than the + deadline, which the 2 s refresh calls make unlikely. #760 leaves relay + calls on the bounded pool for this reason. + - **#757:** five alert-rule UIDs in `docs/prism-native-alert-rules.json` + exceed Grafana's 40-character limit, so provisioning the rule set as + published stops Grafana from starting. Until it's fixed, map the five + to short forms when rendering the rules; #757 lists them. + - **#758:** the two accepted-block revision-work warnings, + `PrismAcceptedRevisionWorkPending` and + `PrismAcceptedRevisionWorkTrackingUnknown`, fire after every pool block + on a frontend that has no miners and doesn't commit the settlements, a + backup frontend say, and the second latches until that frontend + restarts. Until it's fixed, silence the two on idle frontends. +- **Still open from rc.6,** as the rc.6 notes describe them: #752, #738, + #739, #724's other answer, #737's open part, #741, #731, #734, #716's open + part, #711 and #712. + +## Migration and recovery (D5) + +Native schema migration is one-way. No down-migrations are provided. Prefer +forward repair or a compatible native image retaining the latest durable state. +Both [migration](../docs/prism-rust-migration.md#recovery-and-rollback) and +[ledger operations](../docs/prism-ledger-ops.md#one-way-migration-and-isolated-restore-reconciliation) +runbooks contain the numbered reconciliation procedure and exact queries. + +> Before the first native share is acknowledged, restore the complete +> pre-migration database and artifact backup and restart the pinned old image +> in isolation from the migrated database. After the first native share is +> acknowledged, restoring an older database loses those accepted records. +> Any rollback that discards acknowledged history requires an explicit +> accounting reconciliation and recovery decision; it is not an ordinary +> image rollback. Keep every native frontend stopped while performing an +> isolated restore/recovery operation against its replacement database. + +Native `import-audits` is the canonical backfill for recoverable legacy bodies, +including those without a canonical sidecar. It verifies the original audit +digest, trusted ledger key and recorded coinbase. Rehearse it on an isolated +source restore before migrating the source. Production `self-check` fails when +audits lack shared stored bodies or canonical availability; native immutable +snapshots provide canonical reconstruction without a duplicate full byte copy. + +The one-shot commands `self-check`, `import-audits`, `backfill-ctv` and +`broadcast-ctv` no longer register a frontend heartbeat (#381). Earlier 3.x.x +builds left a `starting` row under a generated instance ID after every +invocation, which `fatal-state clear` then refused, and overwrote the live +frontend's status when the ID was configured. Rows those builds left behind are +not removed automatically; the fatal-state runbook's investigation still applies +to them. diff --git a/doc/release-notes-3.x.x-development-line.md b/doc/release-notes-3.x.x-development-line.md new file mode 100644 index 000000000..7e568223f --- /dev/null +++ b/doc/release-notes-3.x.x-development-line.md @@ -0,0 +1,157 @@ +# qbit-mining-bootstrap 3.x.x development line + +Release date: none. This is a development line, not a release. Its first +release is [3.0.0](release-notes-3.0.0.md), cut on the `3.0.x` branch. + +## Status + +3.0.0 is the production release of this line, from the `3.0.x` branch, which +starts at `4c134913`. Deploy 3.0.0 from its tag, not from `3.x.x`: `3.x.x` +continues as 3.1 (3.1.0-rc.1 onwards), which 3.0.0 doesn't include. The +previous production line is `2.x.x`, whose tip is version 2.0.2 (see +`release-notes-2.0.2.md`); the upgrade from it is in the 3.0.0 notes. + +This line exists so that three engineers can work in parallel on the native +Rust PRISM server without sharing one open pull request. Work is split into +workstreams A, B, and C as decided in #260 and tracked in #263. The README +carries a banner with the same statement. + +## What landed + +- PR #244, merged at `1398bbc1`: the multi-instance Rust PRISM server + (`qbit-prism-server`) replaces the Python coordinator runtime under + `lab/prism/`. Every instance coordinates through PostgreSQL; only the + canonical share order and each job's accounting boundary are serialized. +- PR #296: borrowed audit body build and verify entry points in `qbit-prism`. +- The integration of the 2.x.x tip v2.0.2 (`504846cc`, PR #258) into this + line, per #283. It brings: + - the ckpool version-mask probe (#257), with the settings + `CKPOOL_VERSION_MASK_PROBE_ATTEMPTS` and + `CKPOOL_VERSION_MASK_PROBE_RETRY_SECONDS` in `.env.example` and + `compose.yaml`, read by `docker/ckpool/ckpool-version-mask.py` and + `docker/ckpool/start-ckpool.sh`; + - the CI push-branch restriction (#295): push builds run only on `main`, + `1.x.x`, `2.x.x`, and `3.x.x`, while the sharded lint, compile, and test + layout from #243 is kept; + - the 2.0.0 and 2.0.2 release notes, as 2.x.x history; + - the schema file `crates/qbit-prism/sql/002_candidate_bodies.sql`, carried + byte-identical to v2.0.2 and digest-pinned by the upgrade tests (#285). + The Rust migrator enumerates its SQL files explicitly and applies + `001_share_ledger.sql` plus its own migrations; it never applies + `002_candidate_bodies.sql`. It accepts a #258 source after the drain + check, keeping the 002 objects and the capability row, refuses a + partial 001 (a leftover 001 object without `qbit_share_ledger`), a + partial 002 or a newer source before any DDL, and after 001 has run + compares what its `IF NOT EXISTS` left alone, sequences included, + against a scratch apply of the release SQL, refusing a drifted + definition transactionally; see `docs/prism-rust-migration.md`. +- `crates/qbit-prism-server/src/ledger.rs` is split into ownership + submodules, `ledger/candidates.rs`, `ledger/jobs.rs`, `ledger/window.rs`, + and `ledger/connect.rs`, with no behaviour change and no signature change, + so parallel workstream pull requests stop colliding on one file. +- One release identity per #284, currently `3.0.0` on the `3.0.x` branch, + after six release candidates (#557). The single source is + `[workspace.package].version` in the root `Cargo.toml`, which every crate + inherits with `version.workspace = true`, plus the `VERSION` file; bump the + two together. `scripts/check_version_skew.py` (also `make + check-version-skew`) fails when `VERSION` differs from the version Cargo + resolves for any workspace crate, and CI runs it in the Rust tests job that + the merge gate requires. +- #144: the share ledger is partitioned and has a retention path. Migration + 016 drops the two foreign keys onto `qbit_share_ledger(share_id)`, adds the + partition catalog (`qbit_prism_share_partitioning`, + `qbit_prism_share_partitions`) and its maintenance functions, and moves + block solver attribution onto `qbit_pool_blocks.solver_*`, backfilled for + existing blocks. Migration 017 converts `qbit_share_ledger` into a + `RANGE (share_seq)` partitioned table by attaching the release table as its + first partition behind a validated bound, so nothing is copied and no index + is rebuilt; it is applied inside the migration transaction on an empty + ledger and by a resumable online runner otherwise, and is recorded only + after the swap. `qbit-prism-server share-archive` is the retention path: + plan, seal, archive, verify, detach, drop and restore a whole partition, + never a `DELETE`, with the archive outside PostgreSQL as the copy of record. + Readers changed with it: the append path consults `qbit_prism_share_hashes` + first and bounds its ledger probe with `qbit_prism_share_probe_floor()`, + `share_id` uniqueness is per leaf with that table as the global authority, + and the four dashboard solver lookups read `qbit_pool_blocks` instead of + suffix-matching the ledger. Two behaviour changes are documented and + intended: a miner's `last_share_at` reads `null` once its last share has + been archived, and `/audit/share-window` for an anchor inside an archived + range returns no rows. See `docs/prism-share-ledger-partitioning.md`, + `docs/prism-ledger-ops.md` and `docs/prism-rust-migration.md`. +- #582: migration 002's share-hash backfill no longer runs inside the + migration transaction. As one statement it outlasted the statement timeout + on a production-sized ledger, so `migrate` could not complete. On a `2.x.x` + source with rows, the legacy shares' headers are now mapped after the + commit, in batches of consecutive `share_seq` that each take about half a + second, with a durable cursor (`qbit_prism_share_hash_backfill`). The + mapping is the single statement's, row for row, and an interrupted + `migrate` resumes at the cursor. 2 is recorded last, so every start, of + any native build, refuses the database until the backfill has finished. + A backfill started from #669 on also declares the capability + `share_hash_backfill_pending = 1`, in the transaction that creates its + cursor, and the transaction that records 2 removes it. Every earlier build + refuses that capability at connect, so recording 2 by hand does not let + one serve, and a declaration left without its cursor is refused (#669). + The fence covers only those backfills. One that a #582 build started + before #669 carries no declaration, and resuming it does not add one, so + recording 2 by hand could still let a pre-#582 build start with legacy + headers unmapped: let it finish. `migrate` needs no statement timeout + above the default 15 s. See `docs/prism-rust-migration.md`. + +The following 2.x.x changes were deliberately not carried, because the code +they fixed or configured no longer exists on this line: + +- The 2.x.x Python coordinator modules under `lab/prism/` and the tests that + import them. PRs #245, #248, #250, #252, #253, #256, #259, and #292 fixed + that runtime; their commits are ancestors of this line, but their Python + files are removed. Only `lab/prism/Dockerfile` remains. +- The `PRISM_BLOCK_REPLAY_PAGE_SIZE` knob. Its only reader was the Python + coordinator configuration, and the native lane claims one candidate at a + time. +- The `PRISM_CANDIDATE_STORAGE_VERSION`, `PRISM_CANDIDATE_SPOOL_DIR`, and + `PRISM_CANDIDATE_SPOOL_RESERVATION_BYTES` settings. No file on this line + reads them. +- The Python offline pending-block recovery command + (`lab.prism.recover_pending_blocks`) and its runbook steps. +- `docs/prism-candidate-storage.md`, which documented the Python candidate + storage lane end to end. + +## Not done yet + +- The Postgres-backed and live-regtest Rust suites need a PostgreSQL server and a `qbitd` binary; see + `test/prism-native-tests.sh`. They are not part of this note's verification. +- The tree reads 3.0.0 ([its notes](release-notes-3.0.0.md); the release + candidates' are [3.0.0-rc.1](release-notes-3.0.0-rc.1.md), + [3.0.0-rc.2](release-notes-3.0.0-rc.2.md), + [3.0.0-rc.3](release-notes-3.0.0-rc.3.md), + [3.0.0-rc.4](release-notes-3.0.0-rc.4.md), + [3.0.0-rc.5](release-notes-3.0.0-rc.5.md) and + [3.0.0-rc.6](release-notes-3.0.0-rc.6.md)). #291 keeps the rollout + record: the cutover qualification and the go/no-go. +- Native candidate recovery (#268) landed with #439. + +## Upgrade and rollback + +The supported production cutover is to 3.0.0, as its +[notes](release-notes-3.0.0.md#upgrading) and `docs/prism-rust-migration.md` +describe. The Rust migrator refuses to start against a live legacy Python +writer lease and against an undrained legacy block outbox, as a guard against +an accidental cutover. + +Decision D5 in #260 declares one-way native migration with forward repair and +isolated-restore reconciliation, without down-migrations. #287 supplies the +numbered procedures and exact queries in `docs/prism-rust-migration.md` and +`docs/prism-ledger-ops.md`, and enforces audit import through production +`self-check`. #291 records the rehearsal on production-sized history and its +restore and import timings. The [3.0.0 notes](release-notes-3.0.0.md) record +the same data-loss boundary: + +> Before the first native share is acknowledged, restore the complete +> pre-migration database and artifact backup and restart the pinned old image +> in isolation from the migrated database. After the first native share is +> acknowledged, restoring an older database loses those accepted records. +> Any rollback that discards acknowledged history requires an explicit +> accounting reconciliation and recovery decision; it is not an ordinary +> image rollback. Keep every native frontend stopped while performing an +> isolated restore/recovery operation against its replacement database. diff --git a/docker/bitcoin/Dockerfile b/docker/bitcoin/Dockerfile index 34eb82f72..e17204ea3 100644 --- a/docker/bitcoin/Dockerfile +++ b/docker/bitcoin/Dockerfile @@ -12,12 +12,20 @@ ARG BITCOIN_RELEASE_URL ARG BITCOIN_RELEASE_SHA256_AMD64 ARG BITCOIN_RELEASE_SHA256_ARM64 +# Package versions follow the image's Ubuntu release rather than pins that +# rot with every security update; the runtime and its base are not this +# workstream's to change (#288 owns container hardening). +# hadolint ignore=DL3008 RUN apt-get update && apt-get install -y --no-install-recommends \ ca-certificates \ curl \ tini \ && rm -rf /var/lib/apt/lists/* +# The pipes feed commands whose own exit status is what each step checks +# (`grep -q` and `sha256sum -c` under `set -e`); a `SHELL` with pipefail would +# change every RUN in this image, which is not this workstream's to do. +# hadolint ignore=DL4006 RUN set -eu; \ case "${TARGETARCH}" in \ amd64) release_arch="x86_64-linux-gnu"; release_sha256="${BITCOIN_RELEASE_SHA256_AMD64}" ;; \ diff --git a/docker/ckpool/Dockerfile b/docker/ckpool/Dockerfile index 5a25149ba..ae1550b28 100644 --- a/docker/ckpool/Dockerfile +++ b/docker/ckpool/Dockerfile @@ -8,6 +8,10 @@ ARG DEBIAN_FRONTEND=noninteractive ARG CKPOOL_GIT_URL ARG CKPOOL_GIT_REF +# Package versions follow the image's Ubuntu release rather than pins that +# rot with every security update; the runtime and its base are not this +# workstream's to change (#288 owns container hardening). +# hadolint ignore=DL3008 RUN apt-get update && apt-get install -y --no-install-recommends \ autoconf \ automake \ @@ -44,6 +48,10 @@ RUN ./autogen.sh \ FROM ubuntu:${UBUNTU_VERSION} ARG DEBIAN_FRONTEND=noninteractive +# Package versions follow the image's Ubuntu release rather than pins that +# rot with every security update; the runtime and its base are not this +# workstream's to change (#288 owns container hardening). +# hadolint ignore=DL3008 RUN apt-get update && apt-get install -y --no-install-recommends \ bash \ ca-certificates \ diff --git a/docker/qbit/Dockerfile b/docker/qbit/Dockerfile index ad1339708..4c805d649 100644 --- a/docker/qbit/Dockerfile +++ b/docker/qbit/Dockerfile @@ -5,6 +5,10 @@ ARG UBUNTU_VERSION=24.04 FROM ubuntu:${UBUNTU_VERSION} AS build-from-source ARG DEBIAN_FRONTEND=noninteractive +# Package versions follow the image's Ubuntu release rather than pins that +# rot with every security update; the runtime and its base are not this +# workstream's to change (#288 owns container hardening). +# hadolint ignore=DL3008 RUN apt-get update && apt-get install -y --no-install-recommends \ build-essential \ ca-certificates \ @@ -21,9 +25,14 @@ RUN apt-get update && apt-get install -y --no-install-recommends \ && rm -rf /var/lib/apt/lists/* WORKDIR /src/qbit +# qbit-src is a named build context (`--build-context qbit-src=...`), not a stage. +# hadolint ignore=DL3022 COPY --from=qbit-src . . RUN rm -rf build +# Clean, configure, build and install are separate layers on purpose: a failed +# build does not redo the configure step, and the install layer stays small. +# hadolint ignore=DL3059 RUN cmake -S . -B build \ -DCMAKE_BUILD_TYPE=Release \ -DBUILD_DAEMON=ON \ @@ -36,7 +45,9 @@ RUN cmake -S . -B build \ -DBUILD_BENCH=OFF \ -DBUILD_FUZZ_BINARY=OFF +# hadolint ignore=DL3059 RUN cmake --build build --target bitcoind bitcoin-cli -j"$(nproc)" +# hadolint ignore=DL3059 RUN cmake --install build --prefix /opt/qbit --component qbitd \ && cmake --install build --prefix /opt/qbit --component qbit-cli \ && if [ ! -x /opt/qbit/bin/qbitd ]; then \ @@ -53,6 +64,10 @@ RUN cmake --install build --prefix /opt/qbit --component qbitd \ FROM ubuntu:${UBUNTU_VERSION} AS runtime ARG DEBIAN_FRONTEND=noninteractive +# Package versions follow the image's Ubuntu release rather than pins that +# rot with every security update; the runtime and its base are not this +# workstream's to change (#288 owns container hardening). +# hadolint ignore=DL3008 RUN apt-get update && apt-get install -y --no-install-recommends \ ca-certificates \ libevent-core-2.1-7t64 \ diff --git a/docker/real-miner/Dockerfile b/docker/real-miner/Dockerfile index fbe674bf8..913c8fe60 100644 --- a/docker/real-miner/Dockerfile +++ b/docker/real-miner/Dockerfile @@ -8,6 +8,10 @@ ARG DEBIAN_FRONTEND=noninteractive ARG CPUMINER_GIT_URL ARG CPUMINER_GIT_REF +# Package versions follow the image's Ubuntu release rather than pins that +# rot with every security update; the runtime and its base are not this +# workstream's to change (#288 owns container hardening). +# hadolint ignore=DL3008 RUN apt-get update && apt-get install -y --no-install-recommends \ autoconf \ automake \ @@ -39,6 +43,10 @@ RUN ./autogen.sh \ FROM ubuntu:${UBUNTU_VERSION} ARG DEBIAN_FRONTEND=noninteractive +# Package versions follow the image's Ubuntu release rather than pins that +# rot with every security update; the runtime and its base are not this +# workstream's to change (#288 owns container hardening). +# hadolint ignore=DL3008 RUN apt-get update && apt-get install -y --no-install-recommends \ bash \ ca-certificates \ diff --git a/docs/README.md b/docs/README.md index b71f33a90..1f1881971 100644 --- a/docs/README.md +++ b/docs/README.md @@ -37,17 +37,38 @@ references. For public-facing navigation, prefer the short path below. This is useful for reviewers and operators who need implementation detail: -- [prism-ledger-ops.md](prism-ledger-ops.md): formal ledger invariants, - writer-lease behavior, compaction contract, and readiness probes. +- [prism-rust-migration.md](prism-rust-migration.md): coordinated cutover from + Python 2.x, native multi-instance operation, and rollback. +- [prism-ledger-ops.md](prism-ledger-ops.md): native ledger invariants, + transaction ordering, retention, and readiness probes. +- [prism-share-ledger-partitioning.md](prism-share-ledger-partitioning.md): + the design record for share ledger partitioning and retention (#144): the + online horizon, the sealing rule, the conversion, the archive format, and + every native reader against the partition key. +- [prism-native-metrics.md](prism-native-metrics.md): native cutover metric + inventory, measurement boundaries, and failure semantics. - [prism-overload-alerts.md](prism-overload-alerts.md): specification for - the PRISM overload and backlog alert rules (issue #188), including signal - inventory, thresholds and their provenance, and the ordering dependency on - #184. Not live: this repository holds no monitoring configuration. + historical Python overload and backlog alert rules (issue #188). Native + metrics are documented in [prism-native-metrics.md](prism-native-metrics.md); + this repository holds no live monitoring configuration. - [prism-capacity-readiness.md](prism-capacity-readiness.md): optional Stratum-to-Postgres qualification artifact and standalone validator contract. - [prism-coordinator-refactor/README.md](prism-coordinator-refactor/README.md): - completed coordinator decomposition, ownership map, validation evidence, and - stacked-PR publication plan. + historical Python coordinator decomposition and validation evidence. +- [prism-integration-test-gate.md](prism-integration-test-gate.md): the + shared gate every environment-dependent Rust test goes through, its + required-mode switch, the execution manifest CI proves against, and how to + add a gated test. +- [prism-e2e-load-ci.md](prism-e2e-load-ci.md): the end-to-end and load CI + lanes (#487 L0-L6): what each proves and does not, how to dispatch and read + a run, where its evidence lives and how to reproduce it locally. +- [prism-stratum-fuzzing.md](prism-stratum-fuzzing.md): the Stratum + cargo-fuzz targets, their invariants and nightly job, and the per-PR + property tests for the line codec and target conversions (#575). +- [prism-deleted-test-map.md](prism-deleted-test-map.md): every 2.x.x test + file absent on 3.x.x, grouped by area, with its native replacement or the + open issue that closes the gap. `scripts/check_deleted_test_map.py` fails CI + when a test it cites is gone or its counts drift from its rows. ## Public-Site Guidance diff --git a/docs/mainnet-deployment.md b/docs/mainnet-deployment.md index adb971a8c..9da2f6b07 100644 --- a/docs/mainnet-deployment.md +++ b/docs/mainnet-deployment.md @@ -19,6 +19,12 @@ Freeze these inputs before building: - explicit qbit and Bitcoin payout addresses for every enabled mining lane - the PRISM Postgres, signing, audit, difficulty, and CTV policies +Before tagging, run the PRISM release benchmark: the `throughput-20k-window-1fe` and +`throughput-400k-window-1fe-async` presets, previous release against the candidate, on the +reference host, with `scripts/prism_load_ab.py`. Attach both `comparison.md` +files to the release PR, and do not tag on a FAIL without a written +disposition ([`prism-release-benchmark.md`](prism-release-benchmark.md)). + Every deployed container must be an immutable, digest-qualified release artifact. Set all of these to references of the form `registry.example/image@sha256:<64 lowercase hex characters>`: @@ -71,7 +77,6 @@ QBIT_DATA_SOURCE=/srv/qbit-mining-bootstrap/mainnet/qbit BITCOIN_DATA_SOURCE=/srv/qbit-mining-bootstrap/mainnet/bitcoin PRISM_POSTGRES_DATA_SOURCE=/srv/qbit-mining-bootstrap/mainnet/postgres/data PRISM_POSTGRES_WAL_SOURCE=/srv/qbit-mining-bootstrap/mainnet/postgres/wal -PRISM_POSTGRES_REPLICA_DATA_SOURCE=/srv/qbit-mining-bootstrap/mainnet/postgres-replica/data PRISM_AUDIT_DATA_SOURCE=/srv/qbit-mining-bootstrap/mainnet/prism/audit ``` @@ -79,10 +84,42 @@ Create each directory before rendering Compose, then grant only the correspondin runtime the required access. Compose refuses to create missing production bind paths. -For the PRISM lane, release-provenance validation requires an absolute replica -data path distinct from the primary data, WAL, audit, and other enabled state -sources before contacting Docker. See [the replica runbook](prism-postgres-replica.md) -for provisioning the public read standby. +### PRISM signing secrets + +Each mining frontend reads its two signing seeds from files, not from the +environment (#260 D4). Keep the secrets directory outside the checkout and off +shared storage: + +```dotenv +PRISM_SECRETS_SOURCE=/srv/qbit-mining-bootstrap/mainnet/prism/secrets +``` + +The production override mounts it read-only at `/run/secrets/qbit-prism` in +`prism-coordinator`, and in `prism-coordinator-2` under the HA overlay. It +points `PRISM_MANIFEST_SIGNING_SEED_HEX_FILE` and +`PRISM_LEDGER_ATTESTATION_SIGNING_SEED_HEX_FILE` at the files below and clears +the direct seed variables; `scripts/check-env.sh` rejects nonblank direct +values. `prism-public-api` receives neither the mount nor a seed. The PRISM +image runs as UID and GID 10001, which must be able to read both files: + +```bash +secrets=/srv/qbit-mining-bootstrap/mainnet/prism/secrets +sudo install -d -m 0500 -o 10001 -g 10001 "$secrets" +printf '%s\n' "$PRISM_MANIFEST_SIGNING_SEED_HEX" | + sudo install -m 0400 -o 10001 -g 10001 /dev/stdin "$secrets/manifest-signing-seed-hex" +printf '%s\n' "$PRISM_LEDGER_ATTESTATION_SIGNING_SEED_HEX" | + sudo install -m 0400 -o 10001 -g 10001 /dev/stdin "$secrets/ledger-attestation-signing-seed-hex" +unset PRISM_MANIFEST_SIGNING_SEED_HEX PRISM_LEDGER_ATTESTATION_SIGNING_SEED_HEX +``` + +To require a bearer token on the operator listener, write it to +`operator-bearer-token` in the same directory with the same ownership and mode, +and set +`PRISM_OPERATOR_BEARER_TOKEN_FILE=/run/secrets/qbit-prism/operator-bearer-token`. +The built-in healthcheck sends it. The image entrypoint and Compose set both +core limits to zero. The Linux server also disables process dumpability before +reading credentials, covering host crash collectors that ignore those limits. +See [native configuration hardening](prism-configuration.md#mounted-signing-seeds). `PRISM_POSTGRES_WAL_SOURCE` is the primary's live WAL directory. Separating it from `PGDATA` permits an independent capacity and I/O boundary only when the two @@ -124,13 +161,10 @@ export COMPOSE_PROJECT_NAME="${COMPOSE_PROJECT_NAME:-qbit-mining-bootstrap}" default base Compose build context. Production commands pull immutable images and use `--no-build`; they do not build from that context. -Run the static gate for the enabled lanes, then render an explicit service -graph. The example enables all three lanes; remove profiles and services that -are not part of the deployment. +Render an explicit service graph. The example enables all three lanes; remove +profiles and services that are not part of the deployment. ```sh -MINING_LANES=ckpool,auxpow,prism make doctor - docker compose \ --project-name "$COMPOSE_PROJECT_NAME" \ --env-file config/upstream.env \ @@ -143,9 +177,11 @@ docker compose \ config --quiet qbitd ckpool bitcoind auxpow-stratum prism-postgres prism-coordinator prism-public-api ``` -Pull reviewed artifacts before stopping the prior release. Start operator -services with `--no-build` so the host cannot replace a reviewed artifact with -a locally built tag. On a fresh host, start only the nodes first: +Pull reviewed artifacts before running doctor or stopping the prior release. +Doctor runs the selected PRISM image's reader-configuration validator, so that +image must already exist even on a fresh host. Start operator services with +`--no-build` so the host cannot replace a reviewed artifact with a locally built +tag. On a fresh host, start only the nodes first: ```sh docker compose \ @@ -156,6 +192,8 @@ docker compose \ -f compose.production.yaml \ pull qbitd ckpool bitcoind auxpow-stratum prism-postgres prism-coordinator prism-public-api +MINING_LANES=ckpool,auxpow,prism make doctor + docker compose \ --project-name "$COMPOSE_PROJECT_NAME" \ --env-file config/upstream.env \ @@ -316,290 +354,86 @@ before exposing the listener. ## PRISM Gate -Production PRISM uses Postgres and sends a successful share response only after -the share transaction commits. A block-worthy share commits its complete block -intent in the same transaction. The in-memory candidate queue is only a wakeup; -pending work is replayed from Postgres after a restart. +Production Prism runs as the native `qbit-prism-server` executable. Several +physical frontends can write to the same PostgreSQL database through a shared +HA writer endpoint. The same canonical ledger and transaction ordering govern +all instances; there is no application writer election. A normal successful +share ACK follows a durable database commit, with block-worthy candidate intent +persisted atomically so another instance can recover it after a restart. + +For an existing Python deployment, complete the coordinated +[Rust cutover](prism-rust-migration.md) before admitting native traffic: drain and +stop every old writer/broadcaster, back up PostgreSQL and external audit +bodies/segments, run native `migrate`, `import-audits`, and `backfill-ctv`, then +verify preserved balances and audit hashes. Keep the same signing keys. Do not +run Python and Rust against the same database together. Before startup, set `PRISM_STRATUM_SHARE_DIFF`, `PRISM_STRATUM_VARDIFF_MIN_DIFF`, `PRISM_STRATUM_VARDIFF_START_DIFF`, and `PRISM_STRATUM_VARDIFF_MAX_DIFF` to explicit, reviewed, positive values. Production rejects missing values, the local-lab `1e-9` profile, and bounds that -do not satisfy `minimum <= start <= maximum`. This direct safety check does not -require a capacity artifact. - -Set the group-commit policy explicitly if the defaults are not appropriate: - -```dotenv -PRISM_SHARE_COMMIT_BATCH_SIZE=64 -PRISM_SHARE_COMMIT_LINGER_MILLISECONDS=5 -PRISM_SHARE_COMMIT_TIMEOUT_SECONDS=15 -``` - -Keep Postgres `fsync`, `full_page_writes`, and `synchronous_commit` enabled. -Measure acknowledgment latency under representative accepted-share load rather -than weakening database durability. - -Keep `PRISM_STRATUM_STALE_GRACE_SECONDS` at its bounded default (3 seconds) -on every chain, mainnet included. At zero, every in-flight prior-tip share is -rejected with Stratum error 21 at each block, which miners read as pool -failure rather than as stale work. Reward windows that contain a credited -prior-tip share publish the logical `qbit.prism.audit-bundle.v1.1` schema; -this release's verifier accepts it, so keep external mirrors and verifiers on -a release that does too. - -`PRISM_TEMPLATE_REFRESH_FAILURE_EXIT_SECONDS` bounds a persistent -PRISM template-refresh outage; it must be shorter than the operator alert and -response window. Coordination-only refresh deferrals remain outside that -ordinary failure budget, but only for -`PRISM_COORDINATION_BLOCKED_EXIT_SECONDS` (default 900 seconds). A continuous -streak beyond that separate bound fires the publication-progress watchdog and -restarts the coordinator. Alert on -`qbit_prism_template_refresh_coordination_blocked_age_seconds` before it -approaches the configured bound. - -`PRISM_TIP_REFRESH_EPOCH_FANOUT` is a staged rollout gate and defaults to `0`. -Leave it disabled for legacy refresh behavior. Enable it deliberately to make -each refresh wave converge on the latest observed tip epoch; payout and trust -publication fences remain authoritative. Monitor refresh-wave outcomes and -delivery coverage during rollout, and return the setting to `0` to roll back. - -PRISM's fast same-identity restart path requires the default native PostgreSQL -client (`PRISM_POSTGRES_NATIVE_CLIENT=auto` or `1`). Each coordinator holds a -dedicated PostgreSQL advisory guard for its writer ID and epoch and -periodically proves that isolated session live with a non-blocking check (the -session answers, still holds the advisory lock, and the committed lease row -still names it). Most beats run only that cheap read-only ownership proof: one -statement, no row lock and no `pg_stat_activity` scan. The full renewing -verification runs on the first beat and thereafter on any beat whose proof -reports the committed lease row inside the own-write authority margin (at or -above half the lease TTL), escalating on that same beat. So renewal is never -later than it was before the split, and for the whole of a long own fenced -write — when renewals are actually being skipped — every beat runs the same -fail-closed verification as before. The heartbeat never waits on the lease -tuple's row lock: fenced writes hold it for entire transactions, and -accepted-block persistence can legitimately exceed the guard's statement -timeout. It renews the lease TTL only while that tuple is uncontended -(`SKIP LOCKED`), so an idle coordinator — no fenced writes and the CTV -broadcaster disabled — still keeps its lease from expiring under a different -writer identity's expiry claim. If the committed row -is already expired and the renewal was lock-blocked, verification fails closed: -the skipped lock may be an in-flight expiry claim, and a stale committed token -read is not proof of liveness. The one exemption is a lock `pg_stat_activity` -attributes to this coordinator's own pooled backends — a fenced write -outlasting the TTL — because its exclusive tuple lock means no claim can be in -flight and its commit refreshes the lease before any queued claimant -re-evaluates its expiry CAS; hard-exiting there would roll back a valid write -and restart-loop on every similarly slow block. That exemption keeps only the -heartbeat alive: it assumes the write commits, so the external-side-effect -fence refuses guarded RPCs (without fencing the process) while renewal is -deferred behind the writer's own write — a rollback would hand the expired row -to a queued claimant — and broadcast passes or candidate-outbox replays simply -retry once the commit lands a renewal. A -replacement must acquire the guard and then wait one full silence interval -measured both from the lease row's last update and from its own guard -acquisition before its exact-session CAS, so a predecessor that just lost its -guard always has time to self-fence even when a long transaction left its -lease row looking stale. This prevents a live or paused twin from being fenced -merely because it is idle. A psql-only deployment cannot retain a session -guard, logs that fast adoption is disabled, and conservatively falls back to -the configured lease TTL. Keep generated session tokens in production; fixed -tokens remain a local test-only facility. - -#### Heartbeat timing is one validated policy - -The adoption silence, heartbeat interval, failure budget, monitor interval and -exit margin are terms of a single inequality — the old coordinator must be gone -before a replacement may compare-and-swap the lease row — owned by -`lab/prism/writer_lease_timing.py`. The defaults are derived from measurable -phase budgets rather than tuned individually: - -| term | value | where it comes from | -| --- | --- | --- | -| guard statement timeout | 0.50s | the guard session's server-side `statement_timeout` | -| heartbeat interval | 0.25s | idle gap between ownership proofs | -| scheduler slack | 0.50s | process-side scheduling delay, applied to both the heartbeat's stamping path and the monitor's own poll | -| failure budget | 1.25s | interval + statement timeout + slack | -| monitor interval | 0.05s | staleness poll | -| exit margin | 0.10s | hard-exit budget | -| exit envelope | 0.70s | exit margin + 2 x monitor interval + scheduler slack | -| adoption silence | 2.00s | must exceed 1.25 + 0.70 = 1.95s | -| server-proven cap | 1.30s | adoption silence - exit envelope | - -The exit envelope reserves the monitor's *own* lateness, not just its poll -granularity. The monitor is a Python thread in the same process, behind the -same GIL, as the heartbeat whose stalls the policy explicitly budgets for, so -it cannot be assumed to wake on time when the heartbeat does not. At the -shipped numbers, with a maximally late monitor: - -``` -1.30 (cap) + 0.05 (poll) + 0.50 (monitor lateness) + 0.10 (exit) = 1.95 < 2.00 -``` - -leaving one monitor poll of strict reserve before a successor may -compare-and-swap. - -Genuine writer failover therefore costs 2.0 seconds of adoption silence rather -than 1.0. That is the price of the guard session's own statement timeout: the -monitor can only trust completed round trips, so the envelope has to contain -one whole statement plus one idle interval plus process scheduling — and then, -separately, room for the monitor itself to be that late in noticing. It -replaces the temporary downstream 4.0-second pin. - -A combination that breaks the safety inequality is now **refused at startup**: -the coordinator hard-exits with the failing term named instead of printing a -warning and running with no exit-before-adoption guarantee. A combination that -is safe but leaves the staleness cap below the largest gap a healthy -coordinator can produce logs a `no tail-latency headroom` warning and starts — -that state cannot produce two writers, it only produces avoidable restarts. - -#### Reading a heartbeat exit - -Every heartbeat hard exit names which bound tripped (`Tripped: activity ... >= -failure budget` or `... >= adoption envelope cap`), the phase breakdown of the -last and worst verification attempts (`slot_wait` / `guard_sql` / `scheduler` -/ `total`), the monitor's own worst wake delay, and every term of the policy -that produced the decision. Guard-slot queueing, database round trips and -Python scheduling stalls are attributed separately, so an exit says whether to -look at PostgreSQL or at the coordinator process. - -The same attribution is exported with fixed metric cardinality: -`qbit_prism_lease_heartbeat_attempts_total{mode}` (`proof`, `renew`, `fence`), -`qbit_prism_lease_heartbeat_outcomes_total{outcome}`, -`qbit_prism_lease_heartbeat_phase_seconds{phase}` and its -`worst_phase_seconds` companion, -`qbit_prism_lease_heartbeat_activity_age_seconds`, -`qbit_prism_lease_heartbeat_server_proven_age_seconds`, -`qbit_prism_lease_heartbeat_monitor_wake_delay_seconds`, and -`qbit_prism_lease_heartbeat_policy_seconds{term}`. Alert on -`qbit_prism_lease_heartbeat_server_proven_age_seconds` approaching -`qbit_prism_lease_heartbeat_policy_seconds{term="server_proven_cap"}`, and on -`qbit_prism_lease_heartbeat_policy_seconds{term="stability_surplus"}` falling -to or below zero. - -Before each mutating qbitd or wallet RPC, the coordinator performs the same -bounded non-blocking exact-session verification on that advisory-guard -connection, so the fence never contends with an in-flight accepted-block -transaction on the lease tuple. This makes a lost -session or a host suspend/resume fail closed before the usual RPC path. It is a -preflight fence, not an atomic transaction across PostgreSQL and qbitd: a -process paused after verification and resumed after its database session was -lost could still reach the independent RPC target. Mutating RPCs do not use the -JSON-RPC client's transparent transport retry; durable block/CTV retry starts a -new fenced operation instead. Broadcast and block-submit RPCs deduplicate -identical payloads, which limits the effect, but separately built CTV fee -children can still conflict. Operators must not manually start a replacement -with the same writer identity when predecessor termination is uncertain, -especially during a PostgreSQL restart or network partition. Strict elimination -of that residual window requires qbitd/wallet RPCs to validate a fencing -generation supplied by the coordinator. - -### PRISM Allocator Settings - -The PRISM image sets `MALLOC_ARENA_MAX=2` (issue #226). This is a deliberate -allocator decision, recorded here so that a future change to it is a decision -too; `tests/test_prism_heap_census.py` pins the image's allocator environment. - -Why: on `union-mainnet` at pin `9c41894` the coordinator's resident set grew by -roughly 390 MB per hour across 45 hours of uptime with no restart, while the -process ran about 64 threads and its memory map was 943 anonymous regions, 658 -of them between 4 and 64 MiB. That band is the glibc per-thread malloc arena -signature (each arena grows in 64 MiB heaps), and glibc's default arena limit is -eight times the core count, 256 on the 32-core host, so every thread that ever -allocates keeps its own arena and its own retained free lists. Two arenas bound -that multiplication. The storm instrument reproduces the mechanism on any host -(`docs/prism-capacity-readiness.md`, "Allocator settings"): at the default a -64-thread probe creates 64 arenas and the 4-64 MiB region count climbs to the -mainnet shape; at `MALLOC_ARENA_MAX=2` the arena count stays at 2, and -`malloc_trim` can return the freed memory, which with 64 arenas it cannot, -because a per-thread heap's top is not something `malloc_trim` shrinks. - -Why it is safe before the testnet soak decides between 2 and 4: under the GIL -at most one Python thread allocates at a time, so arena-lock contention can only -come from native code that runs with the GIL released, which in this process is -libpq I/O on a handful of pooled connections; the audit-builder daemon the -coordinator spawns inherits the variable but is single-threaded; and the -failure mode of too few arenas is a measurable wait on share-ack latency -(`qbit_prism_share_ack_seconds`), whereas the failure mode of too many is the -17.7 GB resident set. The local storm drain ran at the same wall-clock under -the default, 2, and 4 on the development host. - -Override it per deployment rather than editing the image: +do not satisfy `minimum <= start <= maximum`. Mainnet supports the same bounded +one-parent `PRISM_STRATUM_STALE_GRACE_SECONDS` window as `2.x.x` (default 3). +These direct checks do not require an +optional capacity artifact. + +Keep PostgreSQL `fsync`, `full_page_writes`, and `synchronous_commit` enabled. +To preserve acknowledged shares across primary loss, also configure synchronous +replication and a failover policy promoting only a standby holding those durable +commits. An asynchronous replica alone can lose acknowledged work. Test the +actual HA endpoint, replication/storage policy, and miner retry behavior before +relying on that guarantee. + +Assign each instance a unique `PRISM_INSTANCE_ID` or use generated UUIDs. Payout, +CTV, chain, and signing configuration must agree; startup checks a shared +fingerprint. CPU and database limits can differ by host: ```dotenv -MALLOC_ARENA_MAX=4 -``` - -How that value reaches the process: the `prism-coordinator` service in -`compose.yaml` passes it through as `MALLOC_ARENA_MAX: ${MALLOC_ARENA_MAX:-2}`. -Compose has no `env_file:` for this service, so a dotenv value reaches the -container only through that line; the image's `ENV` is what applies when the -dotenv leaves the variable unset, and the passthrough's default is pinned equal -to the image default by `tests/test_prism_heap_census.py` so the two cannot -drift. A deployment that sets the coordinator's environment by some other -means must set the variable there. The public read service -(`prism-public-api`) runs from the same image and keeps the image default; it -is not part of the experiment. At build time `--build-arg MALLOC_ARENA_MAX=4` -changes the image default. Do not set it to `1`: that forces every thread onto -the main arena, whose `brk` heap shrinks only from its end. Confirm the -setting actually applied before reading any arena gauge: a census report's -`process.malloc_arena_count` cannot exceed the cap, and -`docker exec sh -c 'echo $MALLOC_ARENA_MAX'` -shows what the process started with. - -Do not set any other `MALLOC_*` variable, `GLIBC_TUNABLES`, or `PYTHONMALLOC` -in production without a soak. In particular never set an allocator variable to -an empty value: glibc reads `""` as `0`, and `MALLOC_MMAP_THRESHOLD_=0` would -route every allocation through `mmap`. `PYTHONMALLOC` stays unset so pymalloc -keeps objects of 512 bytes and under out of glibc; routing them through glibc -would enlarge the arenas this setting bounds. - -The heap census (`PRISM_HEAP_CENSUS`), its `tracemalloc` tracing -(`PRISM_HEAP_CENSUS_TRACEMALLOC`), the `malloc_trim` signal -(`PRISM_MALLOC_TRIM_SIGNAL`) and the periodic trim -(`PRISM_MALLOC_TRIM_INTERVAL_SECONDS`) are all off in production by default and -stay off unless an operator turns one on for a diagnosis window. Each is a -strict boolean or a floored interval validated at startup, each passes through -`compose.yaml` to the coordinator service (so a dotenv value reaches the -process), and `.env.example` lists them; the census writes under -`PRISM_AUDIT_DIR/heap-census` by default. Read -`docs/prism-capacity-readiness.md`, "Heap census, allocator control, and the -resident-set bound", before enabling any of them: the census holds the GIL for -the duration of one `gc.get_objects()` call, and `tracemalloc` costs every -allocation for as long as it traces. - -### PRISM Watchdog Exit Forensics - -The PRISM image has no entrypoint or init wrapper: its exec-form Dockerfile -command starts `python3 -m lab.prism.prism_coordinator` directly as container -PID 1, and neither Compose file overrides that command. Consequently a -watchdog `os._exit(1)` is delivered to Docker as status 1; there is no -repository wrapper that can translate it to zero. The development profile's -`restart: on-failure` depends on that status, while the production override's -`restart: unless-stopped` restarts unexpected status-0 and status-1 exits. - -Do not use a post-restart `.State.ExitCode` value as the prior watchdog exit -status. `docker inspect` exposes the container's current execution state, not -an invocation history, and can show zero after the replacement process is -already running. Capture the daemon's `die` event for the failing invocation -instead, and retain it with the incident record: - -```sh -docker events \ - --since '' \ - --until '' \ - --filter type=container \ - --filter container='' \ - --filter event=die \ - --format '{{json .Actor.Attributes}}' +PRISM_RUNTIME_WORKERS=4 +PRISM_JOB_BUILD_EXECUTOR_WORKERS=2 +PRISM_DATABASE_MAX_CONNECTIONS=16 +PRISM_DATABASE_STATEMENT_TIMEOUT_MS=15000 +PRISM_DATABASE_LOCK_TIMEOUT_MS=5000 +PRISM_DATABASE_ACQUIRE_PROBE_IDLE_MS=100 ``` -The event attributes contain the invocation's `exitCode`. The hard-exit path -does not flush stdout/stderr before arming termination because a full container -log pipe could otherwise suppress the watchdog itself. If no historical `die` -event was retained, a later running-state `ExitCode=0` cannot establish the -prior invocation's status; record the discrepancy as missing event-time -evidence. +These values are an example resource allocation, not a capacity qualification. +Budget database connections for all frontends together and measure ACK latency, +job delivery, and build/refresh work under representative load. The old Python +batch-writer, lease, subprocess, and refresh-scheduler variables were removed. + +Use `compose.prism-external-db.yaml` after the production overlay (and before +the HA overlay, if enabled) in the Compose file list for a managed +shared database, set `PRISM_DATABASE_URL` on each host, and render the effective +service graph before starting. `QBIT_RPC_HOST` is overridable; each frontend may +use a local synchronized qbitd or an explicitly configured external node. The +external database overlay removes the local Postgres dependency. With an +external qbitd, start the coordinator using `--no-deps` as described in the +migration guide. + +For Make entry points, set `COMPOSE_OVERLAY_FILES` to the ordered additional +files used by the pull/render/start commands, for example +`export COMPOSE_OVERLAY_FILES="compose.prism-external-db.yaml compose.prism-ha.yaml"`. +Doctor and `make up-prism-pool` then use those same overlays after the automatic +production overlay; HA must remain last. For a two-frontend Make launch, also +set `PRISM_COMPOSE_SERVICES="qbitd prism-coordinator prism-coordinator-2 prism-public-api"`. +Local databases are selected through service dependencies, so the external-DB +overlay does not start them. Explicit Docker commands still need the matching +`-f` flags; Make variables do not configure Docker itself. + +### Native process health and shutdown + +The image runs as UID 10001; its entrypoint disables core dumps and executes +`qbit-prism-server` as PID 1. SIGTERM closes admission +and initiates a bounded task drain; durable candidate/CTV intents remain +recoverable if the process exits before completing them. `/healthz` fails when +published work is stale, its payout revision is obsolete, or job delivery cannot +progress. Scrape each instance's native `/metrics` separately. Replace alerts +that depended on removed Python writer-lease or scheduler metrics. + +The production restart policy remains `unless-stopped`. For exit forensics, +retain Docker's `die` event at incident time: a running replacement's current +`.State.ExitCode` is not the previous process's exit status. ### CTV At Genesis @@ -613,6 +447,18 @@ For a 1 bit/vbyte launch floor, set the market-rate input to `1000`; the default preflight also rejects a configured rate below the node's relay or mempool fee floor. +Every settlement mode requires a pool fee (#525, #535), `PRISM_POOL_FEE_BPS=0` +included; without one the first sub-floor balance stops all work, so the +frontend refuses to start. `compose.yaml` sets no fee and no recipient by +default, and production refuses `.env.example`'s lab recipient, so enable the +fee and name exactly one reviewed recipient: + +```dotenv +PRISM_POOL_FEE_ENABLED=1 +PRISM_POOL_FEE_BPS=0 +PRISM_POOL_FEE_ADDRESS= +``` + When the committed parent transaction already pays its fee, broadcasting is walletless: @@ -637,12 +483,18 @@ A wallet is only required for optional positive CPFP sponsorship. ```sh MINING_LANES=ckpool,auxpow,prism make doctor - MINING_LANES=prism python3 scripts/prism-self-check.py --skip-live ``` -4. Pull immutable images. Start qbit and, when AuxPoW is enabled, Bitcoin without - the pool runtimes. Wait for exact chain, genesis, peer, header, IBD, and - template checks to pass. +4. Pull immutable images and validate the native Prism environment before + starting its listeners: + + ```sh + docker compose --env-file "$DEPLOY_ENV_FILE" -f compose.yaml -f compose.production.yaml \ + --profile prism run --rm --no-deps prism-coordinator qbit-prism-server check-config + ``` + + Start qbit and, when AuxPoW is enabled, Bitcoin without the pool runtimes. + Wait for exact chain, genesis, peer, header, IBD, and template checks to pass. 5. Initialize the fresh Postgres cluster with its separate live WAL path, then configure base backups, off-host WAL archiving, and any synchronous standby. 6. Start only the intended pool runtimes. Run `make prism-self-check` after @@ -650,7 +502,7 @@ A wallet is only required for optional positive CPFP sponsorship. 7. Connect controlled miners to each selected lane. Confirm subscribe, authorize, initial difficulty, fresh jobs, accepted work, reconnect behavior, and accounting visibility. -8. Exercise one child-process restart at a time and verify readiness and miner +8. Exercise one service restart at a time and verify readiness and miner recovery before admitting public traffic. ## Go-Live Checks @@ -669,6 +521,15 @@ Do not expose a lane until all applicable checks pass: - every successful PRISM share survives a coordinator restart exactly once - a pending winning candidate resumes safely after a forced process exit - walletless fee-bearing CTV fanout construction and broadcast have passed +- every PRISM frontend runs with `PRISM_BLOCK_SUBMIT_ENABLED` unset or `1`: + `check-config` prints no `WARNING: block submission is disabled` line and + `qbit_prism_block_submission_enabled` reads 1, so `PrismBlockSubmissionHeld` + is not paging. Set to `0`, that rehearsal kill switch keeps every found block + from the node; see + [prism-ledger-ops.md](prism-ledger-ops.md#block-submission-kill-switch-for-rehearsals) +- `qbit-prism-server submission-hold show` reports `"held": false`: a + cluster-wide hold holds every frontend whatever its own setting says; see + [holding the whole cluster](prism-ledger-ops.md#holding-the-whole-cluster-023-664) - AuxPoW uses explicit payout addresses and no helper payout wallet - backup restore, Postgres failover, disk alerts, and container restart policies have been exercised diff --git a/docs/prism-alert-migration.json b/docs/prism-alert-migration.json new file mode 100644 index 000000000..236c81906 --- /dev/null +++ b/docs/prism-alert-migration.json @@ -0,0 +1,1016 @@ +{ + "source_commit": "a007a14260548cb93e277882672d788fe9a3f95e", + "deployed_alerts": [ + { + "uid": "qbit-prism-metrics-nodata", + "title": "PrismMetricsNoData", + "external": false, + "disposition": "migrated", + "replacement": [ + "PrismMetricsNoData" + ], + "reason": "Native producer is populated; preserve the deployed signal with the explicit native bounds and guards in the specification." + }, + { + "uid": "qbit-prism-metrics-endpoint-down", + "title": "PrismMetricsEndpointDown", + "external": false, + "disposition": "migrated", + "replacement": [ + "PrismMetricsEndpointDown" + ], + "reason": "Native producer is populated; preserve the deployed signal with the explicit native bounds and guards in the specification." + }, + { + "uid": "qbit-prism-connected-clients", + "title": "PRISM Connected Clients", + "external": false, + "disposition": "migrated", + "replacement": [ + "PRISM Connected Clients" + ], + "reason": "Native producer is populated; preserve the deployed signal with the explicit native bounds and guards in the specification." + }, + { + "uid": "qbit-prism-stratum-nodata", + "title": "PrismStratumHealthNoData", + "external": true, + "disposition": "unchanged-external", + "replacement": [ + "PrismStratumHealthNoData" + ], + "reason": "No native replacement needed: deployment-owned Stratum sidecar rule and producer are retained byte-for-byte." + }, + { + "uid": "qbit-prism-stratum-down-warning", + "title": "PrismStratumUnavailableWarning", + "external": true, + "disposition": "unchanged-external", + "replacement": [ + "PrismStratumUnavailableWarning" + ], + "reason": "No native replacement needed: deployment-owned Stratum sidecar rule and producer are retained byte-for-byte." + }, + { + "uid": "qbit-prism-stratum-down", + "title": "PrismStratumUnavailable", + "external": true, + "disposition": "unchanged-external", + "replacement": [ + "PrismStratumUnavailable" + ], + "reason": "No native replacement needed: deployment-owned Stratum sidecar rule and producer are retained byte-for-byte." + }, + { + "uid": "qbit-prism-stratum-failure-rate", + "title": "PrismStratumCheckFailureRateHigh", + "external": true, + "disposition": "unchanged-external", + "replacement": [ + "PrismStratumCheckFailureRateHigh" + ], + "reason": "No native replacement needed: deployment-owned Stratum sidecar rule and producer are retained byte-for-byte." + }, + { + "uid": "qbit-prism-share-writer-backlog", + "title": "PrismShareWriterBacklog", + "external": false, + "disposition": "migrated", + "replacement": [ + "PrismShareAckP99High", + "PrismDatabasePoolWaitHigh" + ], + "reason": "Native shares are durably written before ACK; Python append-queue depth has no native counterpart. Replace latency/pressure intent, not queue semantics." + }, + { + "uid": "qbit-prism-share-writer-backlog-critical", + "title": "PrismShareWriterBacklogCritical", + "external": false, + "disposition": "migrated", + "replacement": [ + "PrismShareAckP99High", + "PrismRejectRatioByReasonHigh" + ], + "reason": "No native 100k volatile ACKed-share queue or overflow-loss threshold; use ACK/rejection impact, with no unqualified critical page." + }, + { + "uid": "qbit-prism-block-candidate-backlog", + "title": "PrismBlockCandidateBacklog", + "external": false, + "disposition": "migrated", + "replacement": [ + "PrismBlockCandidateBacklog" + ], + "reason": "Native producer is populated; preserve the deployed signal with the explicit native bounds and guards in the specification." + }, + { + "uid": "qbit-prism-shares-recovered-to-disk", + "title": "PrismSharesRecoveredToDisk", + "external": false, + "disposition": "no-replacement", + "replacement": [], + "reason": "No replacement: native has no local Python append spool/recovery-file producer; D3 spool follow-up is an open A decision." + }, + { + "uid": "qbit-prism-share-append-failures", + "title": "PrismShareAppendFailures", + "external": false, + "disposition": "migrated", + "replacement": [ + "PrismShareAppendFailures" + ], + "reason": "Native ledger-confirmation-failed reason IDs preserve the deployed any-failure signal, including a 60/60 low-volume write outage, without requiring the reject-ratio volume floor." + }, + { + "uid": "qbit-prism-shares-replayed", + "title": "PrismSharesReplayed", + "external": false, + "disposition": "no-replacement", + "replacement": [], + "reason": "No replacement: no native Python recovery-file replay counter or local spool." + }, + { + "uid": "qbit-prism-job-build-orphan-evicted", + "title": "PrismJobBuildOrphanEvictions", + "external": false, + "disposition": "no-replacement", + "replacement": [], + "reason": "No replacement: native job delivery has no Python executor orphan-eviction state." + }, + { + "uid": "qbit-prism-stats-reconcile-stale", + "title": "PrismAcceptedStatsReconcileStale", + "external": false, + "disposition": "no-replacement", + "replacement": [], + "reason": "No replacement: native process counters are not Python accepted-statistics reconciliation; rollup metrics deferred from #278." + }, + { + "uid": "qbit-prism-lease-proven-age", + "title": "PrismWriterLeaseServerProvenAgeNearCap", + "external": false, + "disposition": "no-replacement", + "replacement": [], + "reason": "No replacement: PostgreSQL coordination replaces the Python writer lease; server-proven-cap policy does not exist." + }, + { + "uid": "qbit-prism-lease-monitor-wake", + "title": "PrismWriterLeaseMonitorWakeDelayHigh", + "external": false, + "disposition": "migrated", + "replacement": [ + "PrismRuntimeLagHigh" + ], + "reason": "Same runtime-stall intent; native runtime wake, retained poll and progress-budget observations replace Python lease-monitor timing." + }, + { + "uid": "qbit-prism-lease-monitor-guarantee", + "title": "PrismWriterLeaseMonitorExitGuaranteeBreached", + "external": false, + "disposition": "no-replacement", + "replacement": [], + "reason": "No replacement: no native Python monitor exit-before-adoption guarantee or breach counter; runtime delay has its own alert." + }, + { + "uid": "qbit-prism-lease-surplus-invalid", + "title": "PrismWriterLeaseStabilitySurplusExhausted", + "external": false, + "disposition": "no-replacement", + "replacement": [], + "reason": "No replacement: no Python lease stability-surplus policy in the native runtime." + }, + { + "uid": "qbit-prism-preview-publish-latency", + "title": "PrismAcceptedPreviewPublicationLatencyHigh", + "external": false, + "disposition": "migrated", + "replacement": [ + "PrismAcceptedRevisionWorkPending" + ], + "reason": "Native pending age and build deadlines cover delayed/degraded delivery; histogram result labels preserve completed-path attribution. This is a frontend-local acceptance-to-mining.notify boundary, not the retired preview publication boundary." + }, + { + "uid": "qbit-prism-preview-publish-latency-crit", + "title": "PrismAcceptedPreviewPublicationLatencyCritical", + "external": false, + "disposition": "migrated", + "replacement": [ + "PrismAcceptedRevisionWorkPendingCritical" + ], + "reason": "Native pending age covers the acceptance-to-delivered-revision-work stall before a latency sample exists; the provisional 307-second critical floor is incident duration, not early-warning evidence." + }, + { + "uid": "qbit-prism-preview-publish-degraded", + "title": "PrismAcceptedPreviewPublicationDegraded", + "external": false, + "disposition": "migrated", + "replacement": [ + "PrismAcceptedRevisionWorkPending", + "PrismRevisionWorkBuildTimeouts" + ], + "reason": "Native pending age and build deadlines cover delayed/degraded delivery; histogram result labels preserve completed-path attribution. This is a frontend-local acceptance-to-mining.notify boundary, not the retired preview publication boundary." + }, + { + "uid": "qbit-prism-parent-preview-timeouts", + "title": "PrismAcceptedParentPreviewWaitTimeouts", + "external": false, + "disposition": "migrated", + "replacement": [ + "PrismRevisionWorkBuildTimeouts" + ], + "reason": "Native build-deadline events replace preview-wait timeouts; the two historical severities consolidate into one warning rule, with pending age providing sustained-stall critical severity." + }, + { + "uid": "qbit-prism-parent-preview-timeouts-crit", + "title": "PrismAcceptedParentPreviewWaitTimeoutsCritical", + "external": false, + "disposition": "migrated", + "replacement": [ + "PrismRevisionWorkBuildTimeouts" + ], + "reason": "Native build-deadline events replace preview-wait timeouts; the two historical severities consolidate into one warning rule, with pending age providing sustained-stall critical severity." + }, + { + "uid": "qbit-prism-unresolved-parent-depth", + "title": "PrismUnresolvedParentDepthNonzero", + "external": false, + "disposition": "no-replacement", + "replacement": [], + "reason": "No replacement: Python unresolved-parent transition cache and its depth/cap/age contract are absent; durable pending-candidate alerts do not claim equivalent semantics." + }, + { + "uid": "qbit-prism-unresolved-parent-near-cap", + "title": "PrismUnresolvedParentDepthNearCap", + "external": false, + "disposition": "no-replacement", + "replacement": [], + "reason": "No replacement: Python unresolved-parent transition cache and its depth/cap/age contract are absent; durable pending-candidate alerts do not claim equivalent semantics." + }, + { + "uid": "qbit-prism-unresolved-parent-at-cap", + "title": "PrismUnresolvedParentDepthAtCap", + "external": false, + "disposition": "no-replacement", + "replacement": [], + "reason": "No replacement: Python unresolved-parent transition cache and its depth/cap/age contract are absent; durable pending-candidate alerts do not claim equivalent semantics." + }, + { + "uid": "qbit-prism-unresolved-parent-age", + "title": "PrismUnresolvedParentAgeOverBudget", + "external": false, + "disposition": "no-replacement", + "replacement": [], + "reason": "No replacement: Python unresolved-parent transition cache and its depth/cap/age contract are absent; durable pending-candidate alerts do not claim equivalent semantics." + }, + { + "uid": "qbit-prism-unresolved-parent-age-crit", + "title": "PrismUnresolvedParentAgeCritical", + "external": false, + "disposition": "no-replacement", + "replacement": [], + "reason": "No replacement: Python unresolved-parent transition cache and its depth/cap/age contract are absent; durable pending-candidate alerts do not claim equivalent semantics." + }, + { + "uid": "qbit-prism-candidate-backlog-critical", + "title": "PrismBlockCandidateBacklogCritical", + "external": false, + "disposition": "migrated", + "replacement": [ + "PrismBlockCandidateBacklogCritical" + ], + "reason": "Native producer is populated; preserve the deployed signal with the explicit native bounds and guards in the specification." + }, + { + "uid": "qbit-prism-block-candidate-oldest", + "title": "PrismBlockCandidateOldestPending", + "external": false, + "disposition": "migrated", + "replacement": [ + "PrismBlockCandidateOldestPending" + ], + "reason": "Native producer is populated; preserve the deployed signal with the explicit native bounds and guards in the specification." + }, + { + "uid": "qbit-prism-candidate-oldest-critical", + "title": "PrismBlockCandidateOldestPendingCritical", + "external": false, + "disposition": "migrated", + "replacement": [ + "PrismBlockCandidateOldestPendingCritical" + ], + "reason": "Native producer is populated; preserve the deployed signal with the explicit native bounds and guards in the specification." + }, + { + "uid": "qbit-prism-candidate-metrics-unavailable", + "title": "PrismBlockCandidateMetricsUnavailable", + "external": false, + "disposition": "migrated", + "replacement": [ + "PrismBlockCandidateMetricsUnavailable" + ], + "reason": "Native producer is populated; preserve the deployed signal with the explicit native bounds and guards in the specification." + }, + { + "uid": "qbit-prism-cleanup-backlog-depth", + "title": "PrismCollapseCleanupBacklog", + "external": false, + "disposition": "no-replacement", + "replacement": [], + "reason": "No replacement: Python collapse-cleanup retry queue/backpressure producer is absent; do not reinterpret native pending candidate rows as cleanup state." + }, + { + "uid": "qbit-prism-cleanup-backlog-oldest", + "title": "PrismCollapseCleanupBacklogOldest", + "external": false, + "disposition": "no-replacement", + "replacement": [], + "reason": "No replacement: Python collapse-cleanup retry queue/backpressure producer is absent; do not reinterpret native pending candidate rows as cleanup state." + }, + { + "uid": "qbit-prism-cleanup-backlog-near-bound", + "title": "PrismCollapseCleanupBacklogNearBound", + "external": false, + "disposition": "no-replacement", + "replacement": [], + "reason": "No replacement: Python collapse-cleanup retry queue/backpressure producer is absent; do not reinterpret native pending candidate rows as cleanup state." + }, + { + "uid": "qbit-prism-cleanup-backpressure-active", + "title": "PrismCollapseCleanupBackpressure", + "external": false, + "disposition": "no-replacement", + "replacement": [], + "reason": "No replacement: Python collapse-cleanup retry queue/backpressure producer is absent; do not reinterpret native pending candidate rows as cleanup state." + }, + { + "uid": "qbit-prism-cleanup-backpressure-engaged", + "title": "PrismCollapseCleanupBackpressureEngaged", + "external": false, + "disposition": "no-replacement", + "replacement": [], + "reason": "No replacement: Python collapse-cleanup retry queue/backpressure producer is absent; do not reinterpret native pending candidate rows as cleanup state." + }, + { + "uid": "qbit-prism-semantic-work-coverage", + "title": "PrismSemanticWorkCoverageLoss", + "external": false, + "disposition": "migrated", + "replacement": [ + "PrismSemanticWorkCoverageLoss" + ], + "reason": "Native producer is populated; preserve the deployed signal with the explicit native bounds and guards in the specification." + }, + { + "uid": "qbit-prism-semantic-coverage-critical", + "title": "PrismSemanticWorkCoverageLossCritical", + "external": false, + "disposition": "migrated", + "replacement": [ + "PrismSemanticWorkCoverageLossCritical" + ], + "reason": "Native producer is populated; preserve the deployed signal with the explicit native bounds and guards in the specification." + }, + { + "uid": "qbit-prism-refresh-pending", + "title": "PrismRefreshPendingPastDeadline", + "external": false, + "disposition": "migrated", + "replacement": [ + "PrismTimeToUsableWorkHigh", + "PrismCurrentWorkGapHigh" + ], + "reason": "Observe the effect on usable work; native refresh has no Python pending-deadline state." + }, + { + "uid": "qbit-prism-refresh-pending-critical", + "title": "PrismRefreshPendingPastDeadlineCritical", + "external": false, + "disposition": "migrated", + "replacement": [ + "PrismTimeToUsableWorkHigh", + "PrismSemanticWorkCoverageLossCritical" + ], + "reason": "Coverage and first-work delay replace miner-impact intent; native thresholds require #291 measurement." + }, + { + "uid": "qbit-prism-metrics-snapshot-stale", + "title": "PrismMetricsSnapshotStale", + "external": false, + "disposition": "migrated", + "replacement": [ + "PrismMetricsSnapshotStale" + ], + "reason": "Native producer is populated; preserve the deployed signal with the explicit native bounds and guards in the specification." + }, + { + "uid": "qbit-prism-metrics-snapshot-unavailable", + "title": "PrismMetricsSnapshotUnavailable", + "external": false, + "disposition": "migrated", + "replacement": [ + "PrismMetricsSnapshotUnavailable" + ], + "reason": "Native producer is populated; preserve the deployed signal with the explicit native bounds and guards in the specification." + }, + { + "uid": "qbit-prism-metrics-snapshot-age", + "title": "PrismMetricsSnapshotAgeHigh", + "external": false, + "disposition": "migrated", + "replacement": [ + "PrismMetricsSnapshotAgeHigh" + ], + "reason": "Native producer is populated; preserve the deployed signal with the explicit native bounds and guards in the specification." + }, + { + "uid": "qbit-prism-public-staleness-refusal", + "title": "PrismPublicStalenessRefusalsIncreasing", + "external": false, + "disposition": "migrated", + "replacement": [ + "PrismPublicStalenessRefusalsIncreasing" + ], + "reason": "Native producer is populated; preserve the deployed signal with the explicit native bounds and guards in the specification." + }, + { + "uid": "qbit-prism-public-ledger-unready", + "title": "PrismPublicLedgerUnready", + "external": false, + "disposition": "migrated", + "replacement": [ + "PrismPublicLedgerUnready" + ], + "reason": "Native producer is populated; preserve the deployed signal with the explicit native bounds and guards in the specification." + }, + { + "uid": "qbit-prism-public-5xx-sustained", + "title": "PrismPublic5xxSustained", + "external": false, + "disposition": "migrated", + "replacement": [ + "PrismPublic5xxSustained" + ], + "reason": "Native producer is populated; preserve the deployed signal with the explicit native bounds and guards in the specification." + }, + { + "uid": "qbit-prism-backup-metrics-nodata", + "title": "PrismBackupMetricsNoData", + "external": true, + "disposition": "unchanged-external", + "replacement": [ + "PrismBackupMetricsNoData" + ], + "reason": "No native replacement needed: deployment-owned backup/node-exporter rule and producer are retained byte-for-byte." + }, + { + "uid": "qbit-prism-backup-collector-stale", + "title": "PrismBackupCollectorStale", + "external": true, + "disposition": "unchanged-external", + "replacement": [ + "PrismBackupCollectorStale" + ], + "reason": "No native replacement needed: deployment-owned backup/node-exporter rule and producer are retained byte-for-byte." + }, + { + "uid": "qbit-prism-backup-full-stale-warning", + "title": "PrismBackupFullStaleWarning", + "external": true, + "disposition": "unchanged-external", + "replacement": [ + "PrismBackupFullStaleWarning" + ], + "reason": "No native replacement needed: deployment-owned backup/node-exporter rule and producer are retained byte-for-byte." + }, + { + "uid": "qbit-prism-backup-full-stale-critical", + "title": "PrismBackupFullStaleCritical", + "external": true, + "disposition": "unchanged-external", + "replacement": [ + "PrismBackupFullStaleCritical" + ], + "reason": "No native replacement needed: deployment-owned backup/node-exporter rule and producer are retained byte-for-byte." + }, + { + "uid": "qbit-prism-backup-diff-stale-warning", + "title": "PrismBackupDifferentialStaleWarning", + "external": true, + "disposition": "unchanged-external", + "replacement": [ + "PrismBackupDifferentialStaleWarning" + ], + "reason": "No native replacement needed: deployment-owned backup/node-exporter rule and producer are retained byte-for-byte." + }, + { + "uid": "qbit-prism-backup-diff-stale-critical", + "title": "PrismBackupDifferentialStaleCritical", + "external": true, + "disposition": "unchanged-external", + "replacement": [ + "PrismBackupDifferentialStaleCritical" + ], + "reason": "No native replacement needed: deployment-owned backup/node-exporter rule and producer are retained byte-for-byte." + }, + { + "uid": "qbit-prism-backup-check-warning", + "title": "PrismBackupCheckWarning", + "external": true, + "disposition": "unchanged-external", + "replacement": [ + "PrismBackupCheckWarning" + ], + "reason": "No native replacement needed: deployment-owned backup/node-exporter rule and producer are retained byte-for-byte." + }, + { + "uid": "qbit-prism-backup-check-critical", + "title": "PrismBackupCheckCritical", + "external": true, + "disposition": "unchanged-external", + "replacement": [ + "PrismBackupCheckCritical" + ], + "reason": "No native replacement needed: deployment-owned backup/node-exporter rule and producer are retained byte-for-byte." + }, + { + "uid": "qbit-prism-backup-wal-stale-warning", + "title": "PrismBackupWalArchiveStaleWarning", + "external": true, + "disposition": "unchanged-external", + "replacement": [ + "PrismBackupWalArchiveStaleWarning" + ], + "reason": "No native replacement needed: deployment-owned backup/node-exporter rule and producer are retained byte-for-byte." + }, + { + "uid": "qbit-prism-backup-wal-stale-critical", + "title": "PrismBackupWalArchiveStaleCritical", + "external": true, + "disposition": "unchanged-external", + "replacement": [ + "PrismBackupWalArchiveStaleCritical" + ], + "reason": "No native replacement needed: deployment-owned backup/node-exporter rule and producer are retained byte-for-byte." + }, + { + "uid": "qbit-prism-backup-wal-fail-warning", + "title": "PrismBackupWalArchiveFailureWarning", + "external": true, + "disposition": "unchanged-external", + "replacement": [ + "PrismBackupWalArchiveFailureWarning" + ], + "reason": "No native replacement needed: deployment-owned backup/node-exporter rule and producer are retained byte-for-byte." + }, + { + "uid": "qbit-prism-backup-wal-fail-critical", + "title": "PrismBackupWalArchiveFailureCritical", + "external": true, + "disposition": "unchanged-external", + "replacement": [ + "PrismBackupWalArchiveFailureCritical" + ], + "reason": "No native replacement needed: deployment-owned backup/node-exporter rule and producer are retained byte-for-byte." + }, + { + "uid": "qbit-prism-backup-wal-space-warning", + "title": "PrismBackupWalFilesystemSpaceWarning", + "external": true, + "disposition": "unchanged-external", + "replacement": [ + "PrismBackupWalFilesystemSpaceWarning" + ], + "reason": "No native replacement needed: deployment-owned backup/node-exporter rule and producer are retained byte-for-byte." + }, + { + "uid": "qbit-prism-backup-wal-space-critical", + "title": "PrismBackupWalFilesystemSpaceCritical", + "external": true, + "disposition": "unchanged-external", + "replacement": [ + "PrismBackupWalFilesystemSpaceCritical" + ], + "reason": "No native replacement needed: deployment-owned backup/node-exporter rule and producer are retained byte-for-byte." + }, + { + "uid": "qbit-prism-backup-audit-stale-warning", + "title": "PrismBackupAuditSnapshotStaleWarning", + "external": true, + "disposition": "unchanged-external", + "replacement": [ + "PrismBackupAuditSnapshotStaleWarning" + ], + "reason": "No native replacement needed: deployment-owned backup/node-exporter rule and producer are retained byte-for-byte." + }, + { + "uid": "qbit-prism-backup-audit-stale-critical", + "title": "PrismBackupAuditSnapshotStaleCritical", + "external": true, + "disposition": "unchanged-external", + "replacement": [ + "PrismBackupAuditSnapshotStaleCritical" + ], + "reason": "No native replacement needed: deployment-owned backup/node-exporter rule and producer are retained byte-for-byte." + }, + { + "uid": "qbit-prism-backup-audit-slow-warning", + "title": "PrismBackupAuditDurationWarning", + "external": true, + "disposition": "unchanged-external", + "replacement": [ + "PrismBackupAuditDurationWarning" + ], + "reason": "No native replacement needed: deployment-owned backup/node-exporter rule and producer are retained byte-for-byte." + }, + { + "uid": "qbit-prism-backup-audit-slow-critical", + "title": "PrismBackupAuditDurationCritical", + "external": true, + "disposition": "unchanged-external", + "replacement": [ + "PrismBackupAuditDurationCritical" + ], + "reason": "No native replacement needed: deployment-owned backup/node-exporter rule and producer are retained byte-for-byte." + }, + { + "uid": "qbit-prism-backup-job-fail-warning", + "title": "PrismBackupScheduledJobFailureWarning", + "external": true, + "disposition": "unchanged-external", + "replacement": [ + "PrismBackupScheduledJobFailureWarning" + ], + "reason": "No native replacement needed: deployment-owned backup/node-exporter rule and producer are retained byte-for-byte." + }, + { + "uid": "qbit-prism-backup-job-fail-critical", + "title": "PrismBackupScheduledJobFailureCritical", + "external": true, + "disposition": "unchanged-external", + "replacement": [ + "PrismBackupScheduledJobFailureCritical" + ], + "reason": "No native replacement needed: deployment-owned backup/node-exporter rule and producer are retained byte-for-byte." + }, + { + "uid": "qbit-prism-backup-coverage-critical", + "title": "PrismBackupCombinedCoverageCritical", + "external": true, + "disposition": "unchanged-external", + "replacement": [ + "PrismBackupCombinedCoverageCritical" + ], + "reason": "No native replacement needed: deployment-owned backup/node-exporter rule and producer are retained byte-for-byte." + }, + { + "uid": "qbit-prism-backup-db-growth-warning", + "title": "PrismBackupDatabaseGrowthWarning", + "external": true, + "disposition": "unchanged-external", + "replacement": [ + "PrismBackupDatabaseGrowthWarning" + ], + "reason": "No native replacement needed: deployment-owned backup/node-exporter rule and producer are retained byte-for-byte." + }, + { + "uid": "qbit-prism-backup-db-growth-critical", + "title": "PrismBackupDatabaseGrowthCritical", + "external": true, + "disposition": "unchanged-external", + "replacement": [ + "PrismBackupDatabaseGrowthCritical" + ], + "reason": "No native replacement needed: deployment-owned backup/node-exporter rule and producer are retained byte-for-byte." + }, + { + "uid": "qbit-prism-backup-audit-growth-warning", + "title": "PrismBackupAuditGrowthWarning", + "external": true, + "disposition": "unchanged-external", + "replacement": [ + "PrismBackupAuditGrowthWarning" + ], + "reason": "No native replacement needed: deployment-owned backup/node-exporter rule and producer are retained byte-for-byte." + }, + { + "uid": "qbit-prism-backup-audit-growth-critical", + "title": "PrismBackupAuditGrowthCritical", + "external": true, + "disposition": "unchanged-external", + "replacement": [ + "PrismBackupAuditGrowthCritical" + ], + "reason": "No native replacement needed: deployment-owned backup/node-exporter rule and producer are retained byte-for-byte." + }, + { + "uid": "qbit-prism-backup-wal-growth-warning", + "title": "PrismBackupWalGenerationGrowthWarning", + "external": true, + "disposition": "unchanged-external", + "replacement": [ + "PrismBackupWalGenerationGrowthWarning" + ], + "reason": "No native replacement needed: deployment-owned backup/node-exporter rule and producer are retained byte-for-byte." + }, + { + "uid": "qbit-prism-backup-wal-growth-critical", + "title": "PrismBackupWalGenerationGrowthCritical", + "external": true, + "disposition": "unchanged-external", + "replacement": [ + "PrismBackupWalGenerationGrowthCritical" + ], + "reason": "No native replacement needed: deployment-owned backup/node-exporter rule and producer are retained byte-for-byte." + }, + { + "uid": "qbit-prism-coordinator-collector-stale", + "title": "PrismCoordinatorCollectorStale", + "external": true, + "disposition": "unchanged-external", + "replacement": [ + "PrismCoordinatorCollectorStale" + ], + "reason": "No native replacement needed: deployment-owned Docker lifecycle textfile rule and producer are retained byte-for-byte." + }, + { + "uid": "qbit-prism-coordinator-restart", + "title": "PrismCoordinatorUnexpectedRestart", + "external": true, + "disposition": "unchanged-external", + "replacement": [ + "PrismCoordinatorUnexpectedRestart" + ], + "reason": "No native replacement needed: deployment-owned Docker lifecycle textfile rule and producer are retained byte-for-byte." + }, + { + "uid": "qbit-prism-coordinator-restart-loop", + "title": "PrismCoordinatorRestartLoop", + "external": true, + "disposition": "unchanged-external", + "replacement": [ + "PrismCoordinatorRestartLoop" + ], + "reason": "No native replacement needed: deployment-owned Docker lifecycle textfile rule and producer are retained byte-for-byte." + } + ], + "retired_names": [ + { + "name": "qbit_prism_accepted_block_landing_phase_seconds", + "replacement": [], + "reason": "No replacement: accepted landing/preview publication is not proof-to-first-node-offer; landing phases are outside trimmed #278. First-offer timing is declared, rule deferred to A/#266." + }, + { + "name": "qbit_prism_accepted_block_landing_phase_seconds_count", + "replacement": [], + "reason": "No replacement: accepted landing/preview publication is not proof-to-first-node-offer; landing phases are outside trimmed #278. First-offer timing is declared, rule deferred to A/#266." + }, + { + "name": "qbit_prism_accepted_block_landing_phase_seconds_max", + "replacement": [], + "reason": "No replacement: accepted landing/preview publication is not proof-to-first-node-offer; landing phases are outside trimmed #278. First-offer timing is declared, rule deferred to A/#266." + }, + { + "name": "qbit_prism_accepted_block_landing_phase_seconds_sum", + "replacement": [], + "reason": "No replacement: accepted landing/preview publication is not proof-to-first-node-offer; landing phases are outside trimmed #278. First-offer timing is declared, rule deferred to A/#266." + }, + { + "name": "qbit_prism_accepted_block_preview_publication_seconds", + "replacement": [ + "qbit_prism_accepted_block_to_revision_work_seconds" + ], + "reason": "New frontend-local acceptance-to-successful-mining.notify boundary; closed result labels, different bucket ladder and restart/observation horizon are documented in prism-native-metrics.md." + }, + { + "name": "qbit_prism_accepted_block_preview_publication_seconds_bucket", + "replacement": [ + "qbit_prism_accepted_block_to_revision_work_seconds" + ], + "reason": "New frontend-local acceptance-to-successful-mining.notify boundary; closed result labels, different bucket ladder and restart/observation horizon are documented in prism-native-metrics.md." + }, + { + "name": "qbit_prism_accepted_block_preview_publication_seconds_count", + "replacement": [ + "qbit_prism_accepted_block_to_revision_work_seconds" + ], + "reason": "New frontend-local acceptance-to-successful-mining.notify boundary; closed result labels, different bucket ladder and restart/observation horizon are documented in prism-native-metrics.md." + }, + { + "name": "qbit_prism_accepted_parent_preview_wait_timeouts_total", + "replacement": [ + "qbit_prism_revision_work_build_timeouts_total" + ], + "reason": "Existing native build-deadline hits while accepted revision work awaits delivery; no new preview-wait mechanism or deadline is introduced." + }, + { + "name": "qbit_prism_accepted_parent_unresolved_depth_max", + "replacement": [], + "reason": "No replacement: Python unresolved-parent/preview-wait state machine is absent; pending candidate rows are a different contract." + }, + { + "name": "qbit_prism_accepted_parent_unresolved_oldest_seconds", + "replacement": [], + "reason": "No replacement: Python unresolved-parent/preview-wait state machine is absent; pending candidate rows are a different contract." + }, + { + "name": "qbit_prism_accepted_parent_unresolved_transitions", + "replacement": [], + "reason": "No replacement: Python unresolved-parent/preview-wait state machine is absent; pending candidate rows are a different contract." + }, + { + "name": "qbit_prism_block_candidate_cleanup_backpressure_active", + "replacement": [], + "reason": "No replacement: Python collapse/cleanup retry holders, pins and backpressure state have no native producer." + }, + { + "name": "qbit_prism_block_candidate_cleanup_backpressure_total", + "replacement": [], + "reason": "No replacement: Python collapse/cleanup retry holders, pins and backpressure state have no native producer." + }, + { + "name": "qbit_prism_block_candidate_cleanup_retry_backlog", + "replacement": [], + "reason": "No replacement: Python collapse/cleanup retry holders, pins and backpressure state have no native producer." + }, + { + "name": "qbit_prism_block_candidate_cleanup_retry_backlog_max", + "replacement": [], + "reason": "No replacement: Python collapse/cleanup retry holders, pins and backpressure state have no native producer." + }, + { + "name": "qbit_prism_block_candidate_cleanup_retry_oldest_seconds", + "replacement": [], + "reason": "No replacement: Python collapse/cleanup retry holders, pins and backpressure state have no native producer." + }, + { + "name": "qbit_prism_block_candidate_cleanup_retry_pending_share_holders", + "replacement": [], + "reason": "No replacement: Python collapse/cleanup retry holders, pins and backpressure state have no native producer." + }, + { + "name": "qbit_prism_block_candidate_cleanup_retry_terminal_outcome_pins", + "replacement": [], + "reason": "No replacement: Python collapse/cleanup retry holders, pins and backpressure state have no native producer." + }, + { + "name": "qbit_prism_block_candidate_collapse_total", + "replacement": [], + "reason": "No replacement: Python collapse/cleanup retry holders, pins and backpressure state have no native producer." + }, + { + "name": "qbit_prism_block_candidate_oldest_pending_seconds", + "replacement": [ + "qbit_prism_block_candidate_oldest_pending_seconds" + ], + "reason": "Same name retained by #278/#277; use native documented meaning and unknown/staleness guards." + }, + { + "name": "qbit_prism_block_candidates_pending", + "replacement": [ + "qbit_prism_block_candidates_pending" + ], + "reason": "Same name retained by #278/#277; use native documented meaning and unknown/staleness guards." + }, + { + "name": "qbit_prism_block_ledger_call_timeouts_total", + "replacement": [], + "reason": "No replacement: Python ledger timeout/submitter backoff series are absent; candidate age retains backlog intent without claiming cause." + }, + { + "name": "qbit_prism_block_submitter_retry_backoff_active", + "replacement": [], + "reason": "No replacement: Python ledger timeout/submitter backoff series are absent; candidate age retains backlog intent without claiming cause." + }, + { + "name": "qbit_prism_ledger_read_calls_total", + "replacement": [], + "reason": "No replacement: Python read-admission gate and execution timers are absent. Native database_pool_acquire_seconds aggregates observed checkout attempts at covered callers: metrics collection, instrumented ledger transactions and direct ledger queries, rollup transactions and share-partition maintenance. Public-role read pools, private audit HTTP endpoints and shared API read-model helpers, startup/migration gates, operator tools and telemetry-free compatibility wrappers remain excluded; see the acquisition guide for the source census and exclusion policy. Pool acquisition timing cannot replace query execution, advisory-lock waits or read-admission gate timing." + }, + { + "name": "qbit_prism_ledger_read_execute_seconds_total", + "replacement": [], + "reason": "No replacement: Python read-admission gate and execution timers are absent. Native database_pool_acquire_seconds aggregates observed checkout attempts at covered callers: metrics collection, instrumented ledger transactions and direct ledger queries, rollup transactions and share-partition maintenance. Public-role read pools, private audit HTTP endpoints and shared API read-model helpers, startup/migration gates, operator tools and telemetry-free compatibility wrappers remain excluded; see the acquisition guide for the source census and exclusion policy. Pool acquisition timing cannot replace query execution, advisory-lock waits or read-admission gate timing." + }, + { + "name": "qbit_prism_ledger_read_execute_timeouts_total", + "replacement": [], + "reason": "No replacement: Python read-admission gate and execution timers are absent. Native database_pool_acquire_seconds aggregates observed checkout attempts at covered callers: metrics collection, instrumented ledger transactions and direct ledger queries, rollup transactions and share-partition maintenance. Public-role read pools, private audit HTTP endpoints and shared API read-model helpers, startup/migration gates, operator tools and telemetry-free compatibility wrappers remain excluded; see the acquisition guide for the source census and exclusion policy. Pool acquisition timing cannot replace query execution, advisory-lock waits or read-admission gate timing." + }, + { + "name": "qbit_prism_ledger_read_gate_timeouts_total", + "replacement": [], + "reason": "No replacement: Python read-admission gate and execution timers are absent. Native database_pool_acquire_seconds aggregates observed checkout attempts at covered callers: metrics collection, instrumented ledger transactions and direct ledger queries, rollup transactions and share-partition maintenance. Public-role read pools, private audit HTTP endpoints and shared API read-model helpers, startup/migration gates, operator tools and telemetry-free compatibility wrappers remain excluded; see the acquisition guide for the source census and exclusion policy. Pool acquisition timing cannot replace query execution, advisory-lock waits or read-admission gate timing." + }, + { + "name": "qbit_prism_ledger_read_gate_wait_seconds_total", + "replacement": [], + "reason": "No replacement: Python read-admission gate and execution timers are absent. Native database_pool_acquire_seconds aggregates observed checkout attempts at covered callers: metrics collection, instrumented ledger transactions and direct ledger queries, rollup transactions and share-partition maintenance. Public-role read pools, private audit HTTP endpoints and shared API read-model helpers, startup/migration gates, operator tools and telemetry-free compatibility wrappers remain excluded; see the acquisition guide for the source census and exclusion policy. Pool acquisition timing cannot replace query execution, advisory-lock waits or read-admission gate timing." + }, + { + "name": "qbit_prism_metrics_snapshot_age_seconds", + "replacement": [ + "qbit_prism_metrics_snapshot_age_seconds" + ], + "reason": "Same name retained by #278/#277; use native documented meaning and unknown/staleness guards." + }, + { + "name": "qbit_prism_metrics_snapshot_available", + "replacement": [ + "qbit_prism_metrics_snapshot_available" + ], + "reason": "Same name retained by #278/#277; use native documented meaning and unknown/staleness guards." + }, + { + "name": "qbit_prism_metrics_snapshot_stale", + "replacement": [ + "qbit_prism_metrics_snapshot_stale" + ], + "reason": "Same name retained by #278/#277; use native documented meaning and unknown/staleness guards." + }, + { + "name": "qbit_prism_payout_artifact_events_total", + "replacement": [], + "reason": "No replacement: payout artifact/rescan/balance-read instrumentation is outside trimmed #278; no deployed rule to rewrite." + }, + { + "name": "qbit_prism_payout_window_full_rescan_seconds", + "replacement": [], + "reason": "No replacement: payout artifact/rescan/balance-read instrumentation is outside trimmed #278; no deployed rule to rewrite." + }, + { + "name": "qbit_prism_payout_window_full_rescan_seconds_count", + "replacement": [], + "reason": "No replacement: payout artifact/rescan/balance-read instrumentation is outside trimmed #278; no deployed rule to rewrite." + }, + { + "name": "qbit_prism_prior_balances_reads_total", + "replacement": [], + "reason": "No replacement: payout artifact/rescan/balance-read instrumentation is outside trimmed #278; no deployed rule to rewrite." + }, + { + "name": "qbit_prism_refresh_pending", + "replacement": [ + "qbit_prism_stratum_oldest_pending_initial_job_seconds", + "qbit_prism_stratum_current_tip_coverage_gap_seconds" + ], + "reason": "Replacement for alert intent only: time to usable work, not Python refresh-task state." + }, + { + "name": "qbit_prism_refresh_pending_age_seconds", + "replacement": [ + "qbit_prism_stratum_oldest_pending_initial_job_seconds", + "qbit_prism_stratum_current_tip_coverage_gap_seconds" + ], + "reason": "Replacement for alert intent only: time to usable work, not Python refresh-task state." + }, + { + "name": "qbit_prism_reorg_reconcile_errors_total", + "replacement": [], + "reason": "No replacement: Python reconciler pass/step/error series are outside trimmed native instrumentation; do not add undeployed historical rules." + }, + { + "name": "qbit_prism_reorg_reconcile_lookups_total", + "replacement": [], + "reason": "No replacement: Python reconciler pass/step/error series are outside trimmed native instrumentation; do not add undeployed historical rules." + }, + { + "name": "qbit_prism_reorg_reconcile_pass_seconds", + "replacement": [], + "reason": "No replacement: Python reconciler pass/step/error series are outside trimmed native instrumentation; do not add undeployed historical rules." + }, + { + "name": "qbit_prism_reorg_reconcile_pass_seconds_count", + "replacement": [], + "reason": "No replacement: Python reconciler pass/step/error series are outside trimmed native instrumentation; do not add undeployed historical rules." + }, + { + "name": "qbit_prism_reorg_reconcile_pass_seconds_sum", + "replacement": [], + "reason": "No replacement: Python reconciler pass/step/error series are outside trimmed native instrumentation; do not add undeployed historical rules." + }, + { + "name": "qbit_prism_reorg_reconcile_step_seconds", + "replacement": [], + "reason": "No replacement: Python reconciler pass/step/error series are outside trimmed native instrumentation; do not add undeployed historical rules." + }, + { + "name": "qbit_prism_reorg_reconcile_step_seconds_sum", + "replacement": [], + "reason": "No replacement: Python reconciler pass/step/error series are outside trimmed native instrumentation; do not add undeployed historical rules." + }, + { + "name": "qbit_prism_stratum_authorized_connections", + "replacement": [ + "qbit_prism_authorized_clients" + ], + "reason": "Native local authorized clients; update semantic-coverage joins." + }, + { + "name": "qbit_prism_stratum_semantic_current_work_ratio", + "replacement": [ + "qbit_prism_stratum_semantic_current_work_ratio" + ], + "reason": "Same name retained by #278/#277; use native documented meaning and unknown/staleness guards." + } + ] +} diff --git a/docs/prism-alert-migration.md b/docs/prism-alert-migration.md new file mode 100644 index 000000000..4f3e89d40 --- /dev/null +++ b/docs/prism-alert-migration.md @@ -0,0 +1,420 @@ +# Native alert migration (#279) + +This is the cutover consumer contract for the 3.x.x native runtime. The single +[metric inventory](prism-native-metrics.md) is generated from #278's registry. +The [historical overload appendix](prism-overload-alerts.md) remains historical; +its undeployed rules are not re-specified here. + +The input is `SwapLabsInc/qbit-tools`, `origin/main` at +`a007a14260548cb93e277882672d788fe9a3f95e`, file +`ansible/roles/qbit_monitoring_stack/templates/grafana-alert-rules-prism.yml.j2`. +The [frozen manifest](../tests/fixtures/prism-deployed-alerts.json) records its +SHA-256, all 78 definitions (including gated definitions), and all 53 referenced +`qbit_prism_*` tokens. Jinja values are deployment variables, not metric labels +or evidence that a gate is enabled. Backup, node-exporter lifecycle, and sidecar +producers remain external to the native inventory. + +## Review and apply the draft + +The [unified rules-file diff](prism-alert-rules-qbit-tools.patch) is the deployment +change for operator review. It is not applied by bootstrap and must not be used +with a 2.x.x image. No changes to qbit-tools are made by this work. + +```sh +# In a disposable copy of qbit-tools at a007a142 (or review/rebase against its newer main): +git apply --check /path/to/prism-alert-rules-qbit-tools.patch +# After operator review and coordinated native cutover: +git apply /path/to/prism-alert-rules-qbit-tools.patch +``` + +The draft preserves the 34 external definitions and their existing gates. +`qbit_monitoring_stack_prism_alerts_enabled` enables the native coordinator rules; +the public, connected-client and sidecar gates keep their meaning. Python +`overload_alerts_enabled` and `writer_lease_alerts_enabled` gates no longer +control native observability. The separate PostgreSQL HA group is required by +D3 and deliberately reports missing instrumentation. Provisioning `deleteRules` +removes every legacy UID that is not reused, plus each native UID disabled by +its current gate (including reused legacy UIDs); omission alone does not remove +an existing Grafana rule. This also removes previously enabled native rules on +a later gate change. External provisioning behavior is preserved. Native +critical-named bounds are warnings without page +labels until #291 qualifies the native thresholds; existing external paging is +preserved. Sidecar miner-impact and host-loss pages remain the outage net. + +Before cutover, render with the deployment's actual Ansible variables and +confirm the scrape selectors: `qbit-prism`, `qbit-prism-public-read`, and the new +primary exporter job `qbit-postgres-primary`, all with `network`. Exporter-owned +labels (`result`, `reason_id`, `collector`, `task`) come from the registry; +`job`, `instance`, and `network` are Prometheus target labels. Evaluate native +ratios per instance, never sum replicated candidate/RSS gauges across hosts. +The connected-client floor preserves the deployed network-wide total and only +evaluates when every configured coordinator has a fresh observation. +For a permitted rollback to 2.x.x, restore its template/gates and explicitly +delete any newly introduced native/D3 UIDs that should no longer run. This rules +change does not authorize database rollback or mixed-version writer operation. + +Regenerate after editing the JSON specifications or compatibility mappings: + +```sh +python3 scripts/generate_prism_metrics.py +python3 scripts/generate_prism_alerts.py --snapshot /path/to/read-only-snapshot +cargo +1.89.0 test --locked -p qbit-prism-server --test observability +python3 scripts/test_prism_alert_promql.py --promtool /path/to/promtool +``` + +## Truthful signals and threshold qualification + +Native body-based rules require `metrics_snapshot_available == 1` and +`metrics_snapshot_stale == 0`, using #277's effective freshness budget rather +than hard-coding a refresh interval. Snapshot unavailable/stale alerts and the +endpoint/no-series nets remain alertable on missing data. A fresh failed public +probe is distinct from a stale probe; `public_ledger_ready` already rejects both +failure and staleness, and the public role exposes `public_ledger_probe_age_seconds` +instead of the coordinator's snapshot gauge families. + +Candidate and RSS rules additionally require the relevant +`collector_available == 1`. Unknown, failed or expired collections render -1 and +fire the collector-unavailable rule; a successful zero remains healthy. Unknown +work gauges have their own rule. The pending-candidate rules +(`PrismBlockCandidateBacklog`, `PrismBlockCandidateOldestPending` and their +critical variants) count the four unfinished outbox states; since #415 +(migration 015) a block that lost a tip race leaves that set on its own once a +different block has `PRISM_CANDIDATE_ORPHAN_CONFIRMATIONS` (default 6) +confirmations at its height, so an orphan raises the oldest-pending age for a +few block intervals and then clears without operator action, and +`qbit_prism_block_candidates_orphaned_total` records completions observed by +the process after commit (cancellation or restart can lose an increment). +Since #493 the paging `PrismBlockCandidateOldestPendingCritical` reads +`qbit_prism_block_candidate_oldest_unacknowledged_seconds`, the oldest +unfinished row the node has not accepted (pre-offer rows, unknown offer +outcomes and definitive rejections other than a side-chain reply), so a +node-accepted lost race in reconciliation raises only the 15-second warning +until its orphan proof, a failing audit landing is reported by +`PrismBlockCandidateLandingFailed` (60 s) and paged by +`PrismBlockCandidateLandingFailedCritical` (307 s, one-minute dwell), and +`PrismBlockCandidateStuck` warns at 30 minutes on any unfinished row +automation is not finishing. Rollout order: upgrade the frontends before +applying the rules; the paging expression falls back to the all-unfinished +age on a target that does not export the new series yet, so the base +behaviour persists there rather than going silent. The reconciliation +backoff is ten seconds times the row's attempt count (pre-offer attempts +included), capped at an hour, so the sixth competitor confirmation at +60-second spacing is usually observed around the eighth retry, about nine +minutes in. Size those `for` clauses against that window when qualifying the +provisional 15/60 s bounds in #291: a backlog that persists past it is a row +automation cannot land, a rejected block or an unknown offer outcome, which +is operator work. +A rising orphan rate is a node or network signal (the qbitd first-seen question +in #413), not a coordinator stall; no rule is attached to the counter here. Runtime wake/poll/progress signals are evaluated +at scrape time and intentionally remain usable while the body publisher is +stale. The latest wake lag can reset within 100 ms of recovery, so the wake-delay +replacement also reads retained completed polls and the native progress budget. +The five-minute lookback retains an episode for evaluation; it is not the old +lease scheduler-slack policy or a reconstruction of wake frequency. + +The ACK p99 uses the native completed-response-write histogram. At least 100 +observations are required in five minutes so an idle miner does not create a +quantile alarm. Reject ratios preserve canonical `reason_id` and use accepted +plus canonical rejection decisions as their denominator; ACK timing is not a +durable-credit counter. The collector pool histogram records each started +`PgPool::acquire` attempt once: success when acquired, or failure on acquisition +error or cancellation, including the three-second collection deadline. Since #328 +instrumented coordinator ledger transactions record acquisition the same way. +Covered direct ledger queries, rollup transactions and share-partition maintenance +share that boundary. The [acquisition guide](prism-pool-acquire-metrics.md) lists +the source census and explicit exclusions: public-role read pools, private audit +HTTP endpoints and shared API read-model helpers, startup/migration gates, +operator tools and telemetry-free compatibility wrappers. The histogram aggregates +observed checkout attempts at covered callers. Durations measure elapsed pool +wait and exclude subsequent transaction work. The pool histogram is overlaid from +one live registry read per scrape, so new observations remain alertable when the +shared pool blocks collection and the health publisher. This rule requires a +successful scrape and the same per-instance sample minimum; cached-body freshness +and collector availability do not gate these live events. Cached gauges retain +their freshness guards and stale/missing snapshots retain their unknown-data +alerts. Deploy #351's live-histogram producer to every coordinator target before +activating the revised expression; older cached-histogram producers do not satisfy +its freshness contract. **Proof-to-first-offer waits are declared, +rule deferred to A/#266; advisory-lock waits are recorded since #328 with no rule written yet**; +neither has a firing rule. + +Numeric bounds without native measurement are explicitly **provisional, measure +in #291** in each rule's `basis` and provisioned description. This includes RSS +4 GiB (must be reconciled with container headroom), ACK 1 s, reject 5%, pool wait +0.5 s, backlog counts 2/5, oldest candidate ages 15/60 s, first-work/gap 15 s, +critical coverage 50%, and timing/sample windows. Native implementation evidence +supports the strict 95% coverage boundary, two-second blocked-poll budget, +30-second collector expiry, and #277 freshness calculation; it does not establish +production latency or memory SLOs. Preserve the 400k regression and 500k headroom +qualification targets in #291; this rule migration does not establish capacity. +#291 must exercise each signal, recovery, +startup, scrape loss, stale publisher and collector failure before cutover. + +`qbit_prism_public_requests_total` counts `/healthz`, `/metrics` and other routed +requests. **It must never feed a request-rate rule.** The public 5xx rule is a +response-error signal (including failed probes); there is no public request-rate +rule in this draft. This preserves the caveat in the D1 Sources evidence and +[#260's measured D1 addendum](https://github.com/Qbit-Org/qbit-mining-bootstrap/issues/260). + +## D3: deployment-provided primary/standby rules + +The [two PostgreSQL rules](prism-postgres-alert-rules.json) are emitted as the +separate `qbit-prism-postgres-ha-alerts` group in the diff. Required by D3: +**replay lag above 5 seconds for 1 minute**, and **zero replication rows for +`application_name='prism_standby_1'` for 1 minute**. Missing, failed and unknown +measurements also fire. The topology is one asynchronous standby: ACKs require +local durability and do not wait for replay; the loss window remains replication +lag or standby downtime. These alerts do not imply lossless failover. + +**Deployment-provided: requires postgres_exporter on the PRIMARY; verify presence +in #281/#291.** The dedicated failover standby is not the public read replica. +Never use `qbit_prism_public_replica_*` for this group. Its inventory is external +and deliberately excluded from the native-family test. + +The [query extension](prism-postgres-exporter-queries.yaml) observes the primary's +flushed (locally durable) WAL position and the dedicated standby's replay position. +It emits each position as exact high/low 32-bit components: Prometheus float64 +cannot represent every 64-bit LSN as one number. The rule compares the current +replay position lexicographically with the primary flushed position observed five +seconds earlier, with a one-minute dwell. A backlog of bytes alone has no time +unit; comparing an observed durable prefix over time supplies the five-second +budget without treating the last reported write latency as current backlog age. + +Require **one-second primary exporter scrapes** and one-second standby +`wal_receiver_status_interval` updates. Current samples must be at most two +seconds old, and the five-second historical samples at most seven seconds old; +missing history remains unknown. This sampled comparison has up to one scrape +interval of boundary uncertainty. Exporter query execution must refresh on each +scrape, without a long-lived statistics transaction/cache. #281/#291 must verify +that cadence, permissions (`pg_read_all_stats` or an appropriately scoped monitoring +role), scrape cost, and the actual exporter failure behavior before cutover. + +The diagnostic `pg_stat_replication_replay_lag` remains available but does not +feed the firing rule. PostgreSQL can report NULL when an idle standby has caught +up, and can retain a recent nonzero latency after it catches up. The query emits +zero only when exactly one streaming connection has a known replay position at +or beyond the primary's durable position; unknown positions, duplicate identity, +and NULL latency with outstanding WAL remain -1. This follows the documented +[PostgreSQL replay-lag semantics](https://www.postgresql.org/docs/16/monitoring-stats.html#MONITORING-PG-STAT-REPLICATION-VIEW). + +Expose `pg_stat_replication_{primary_flush_lsn_hi,primary_flush_lsn_lo,replay_lsn_hi,replay_lsn_lo,count,replay_lag,async}` +with `application_name="prism_standby_1"`. The count query always emits a row, +including zero when disconnected. The `async` gauge is 1 only for the required +asynchronous topology with no primary `synchronous_standby_names` requirement, +0 for a synchronous/quorum requirement (even on another standby), and -1 when unknown +or ambiguous; its absent, stale, negative or zero state triggers the D3 lag rule +without discarding valid measured WAL positions. `up`, `pg_up` and +`pg_exporter_last_scrape_error` must describe the same primary target, using +`job="qbit-postgres-primary"`, `instance`, and `network`; exactly one primary +exporter target per network is required. Query failure, exporter failure, +missing status, stale/negative positions and ambiguous identities remain +alertable. These are an explicit deployment query contract, not a claim that +stock postgres_exporter versions enable these names by default; the query stanza +is also included in the review-only diff. + +The disposable SQL test executes that exact query against one primary and one +asynchronous standby, feeds the observed positions into promtool, and checks +idle NULL, paused replay, one-minute dwell, recovery, synchronous misconfiguration, +asynchronous recovery, disconnection, and a query +run against the wrong database role. Separate PromQL fixtures cover failed or +missing exporter/query observations and precise LSN boundaries. No deployment +exporter is started by these tests; its integration remains a #281/#291 check. + +```sh +PRISM_TEST_PG_BIN_DIR=/path/to/postgresql/bin python3 scripts/test_prism_postgres_alerts.py --promtool /path/to/promtool +``` + + + + +## Every deployed alert + +78 definitions: **27 migrated, 34 unchanged external, 17 with no replacement**. +This includes definitions behind deployment Jinja flags; the snapshot does not record their effective values. + +| Deployed UID | Deployed alert | Native rule / disposition | Reason | +| --- | --- | --- | --- | +| `qbit-prism-metrics-nodata` | PrismMetricsNoData | `PrismMetricsNoData` (migrated) | Native producer is populated; preserve the deployed signal with the explicit native bounds and guards in the specification. | +| `qbit-prism-metrics-endpoint-down` | PrismMetricsEndpointDown | `PrismMetricsEndpointDown` (migrated) | Native producer is populated; preserve the deployed signal with the explicit native bounds and guards in the specification. | +| `qbit-prism-connected-clients` | PRISM Connected Clients | `PRISM Connected Clients` (migrated) | Native producer is populated; preserve the deployed signal with the explicit native bounds and guards in the specification. | +| `qbit-prism-stratum-nodata` | PrismStratumHealthNoData | `PrismStratumHealthNoData` (unchanged-external) | No native replacement needed: deployment-owned Stratum sidecar rule and producer are retained byte-for-byte. | +| `qbit-prism-stratum-down-warning` | PrismStratumUnavailableWarning | `PrismStratumUnavailableWarning` (unchanged-external) | No native replacement needed: deployment-owned Stratum sidecar rule and producer are retained byte-for-byte. | +| `qbit-prism-stratum-down` | PrismStratumUnavailable | `PrismStratumUnavailable` (unchanged-external) | No native replacement needed: deployment-owned Stratum sidecar rule and producer are retained byte-for-byte. | +| `qbit-prism-stratum-failure-rate` | PrismStratumCheckFailureRateHigh | `PrismStratumCheckFailureRateHigh` (unchanged-external) | No native replacement needed: deployment-owned Stratum sidecar rule and producer are retained byte-for-byte. | +| `qbit-prism-share-writer-backlog` | PrismShareWriterBacklog | `PrismShareAckP99High`, `PrismDatabasePoolWaitHigh` (migrated) | Native shares are durably written before ACK; Python append-queue depth has no native counterpart. Replace latency/pressure intent, not queue semantics. | +| `qbit-prism-share-writer-backlog-critical` | PrismShareWriterBacklogCritical | `PrismShareAckP99High`, `PrismRejectRatioByReasonHigh` (migrated) | No native 100k volatile ACKed-share queue or overflow-loss threshold; use ACK/rejection impact, with no unqualified critical page. | +| `qbit-prism-block-candidate-backlog` | PrismBlockCandidateBacklog | `PrismBlockCandidateBacklog` (migrated) | Native producer is populated; preserve the deployed signal with the explicit native bounds and guards in the specification. | +| `qbit-prism-shares-recovered-to-disk` | PrismSharesRecoveredToDisk | no replacement (no-replacement) | No replacement: native has no local Python append spool/recovery-file producer; D3 spool follow-up is an open A decision. | +| `qbit-prism-share-append-failures` | PrismShareAppendFailures | `PrismShareAppendFailures` (migrated) | Native ledger-confirmation-failed reason IDs preserve the deployed any-failure signal, including a 60/60 low-volume write outage, without requiring the reject-ratio volume floor. | +| `qbit-prism-shares-replayed` | PrismSharesReplayed | no replacement (no-replacement) | No replacement: no native Python recovery-file replay counter or local spool. | +| `qbit-prism-job-build-orphan-evicted` | PrismJobBuildOrphanEvictions | no replacement (no-replacement) | No replacement: native job delivery has no Python executor orphan-eviction state. | +| `qbit-prism-stats-reconcile-stale` | PrismAcceptedStatsReconcileStale | no replacement (no-replacement) | No replacement: native process counters are not Python accepted-statistics reconciliation; rollup metrics deferred from #278. | +| `qbit-prism-lease-proven-age` | PrismWriterLeaseServerProvenAgeNearCap | no replacement (no-replacement) | No replacement: PostgreSQL coordination replaces the Python writer lease; server-proven-cap policy does not exist. | +| `qbit-prism-lease-monitor-wake` | PrismWriterLeaseMonitorWakeDelayHigh | `PrismRuntimeLagHigh` (migrated) | Same runtime-stall intent; native runtime wake, retained poll and progress-budget observations replace Python lease-monitor timing. | +| `qbit-prism-lease-monitor-guarantee` | PrismWriterLeaseMonitorExitGuaranteeBreached | no replacement (no-replacement) | No replacement: no native Python monitor exit-before-adoption guarantee or breach counter; runtime delay has its own alert. | +| `qbit-prism-lease-surplus-invalid` | PrismWriterLeaseStabilitySurplusExhausted | no replacement (no-replacement) | No replacement: no Python lease stability-surplus policy in the native runtime. | +| `qbit-prism-preview-publish-latency` | PrismAcceptedPreviewPublicationLatencyHigh | `PrismAcceptedRevisionWorkPending` (migrated) | Native pending age and build deadlines cover delayed/degraded delivery; histogram result labels preserve completed-path attribution. This is a frontend-local acceptance-to-mining.notify boundary, not the retired preview publication boundary. | +| `qbit-prism-preview-publish-latency-crit` | PrismAcceptedPreviewPublicationLatencyCritical | `PrismAcceptedRevisionWorkPendingCritical` (migrated) | Native pending age covers the acceptance-to-delivered-revision-work stall before a latency sample exists; the provisional 307-second critical floor is incident duration, not early-warning evidence. | +| `qbit-prism-preview-publish-degraded` | PrismAcceptedPreviewPublicationDegraded | `PrismAcceptedRevisionWorkPending`, `PrismRevisionWorkBuildTimeouts` (migrated) | Native pending age and build deadlines cover delayed/degraded delivery; histogram result labels preserve completed-path attribution. This is a frontend-local acceptance-to-mining.notify boundary, not the retired preview publication boundary. | +| `qbit-prism-parent-preview-timeouts` | PrismAcceptedParentPreviewWaitTimeouts | `PrismRevisionWorkBuildTimeouts` (migrated) | Native build-deadline events replace preview-wait timeouts; the two historical severities consolidate into one warning rule, with pending age providing sustained-stall critical severity. | +| `qbit-prism-parent-preview-timeouts-crit` | PrismAcceptedParentPreviewWaitTimeoutsCritical | `PrismRevisionWorkBuildTimeouts` (migrated) | Native build-deadline events replace preview-wait timeouts; the two historical severities consolidate into one warning rule, with pending age providing sustained-stall critical severity. | +| `qbit-prism-unresolved-parent-depth` | PrismUnresolvedParentDepthNonzero | no replacement (no-replacement) | No replacement: Python unresolved-parent transition cache and its depth/cap/age contract are absent; durable pending-candidate alerts do not claim equivalent semantics. | +| `qbit-prism-unresolved-parent-near-cap` | PrismUnresolvedParentDepthNearCap | no replacement (no-replacement) | No replacement: Python unresolved-parent transition cache and its depth/cap/age contract are absent; durable pending-candidate alerts do not claim equivalent semantics. | +| `qbit-prism-unresolved-parent-at-cap` | PrismUnresolvedParentDepthAtCap | no replacement (no-replacement) | No replacement: Python unresolved-parent transition cache and its depth/cap/age contract are absent; durable pending-candidate alerts do not claim equivalent semantics. | +| `qbit-prism-unresolved-parent-age` | PrismUnresolvedParentAgeOverBudget | no replacement (no-replacement) | No replacement: Python unresolved-parent transition cache and its depth/cap/age contract are absent; durable pending-candidate alerts do not claim equivalent semantics. | +| `qbit-prism-unresolved-parent-age-crit` | PrismUnresolvedParentAgeCritical | no replacement (no-replacement) | No replacement: Python unresolved-parent transition cache and its depth/cap/age contract are absent; durable pending-candidate alerts do not claim equivalent semantics. | +| `qbit-prism-candidate-backlog-critical` | PrismBlockCandidateBacklogCritical | `PrismBlockCandidateBacklogCritical` (migrated) | Native producer is populated; preserve the deployed signal with the explicit native bounds and guards in the specification. | +| `qbit-prism-block-candidate-oldest` | PrismBlockCandidateOldestPending | `PrismBlockCandidateOldestPending` (migrated) | Native producer is populated; preserve the deployed signal with the explicit native bounds and guards in the specification. | +| `qbit-prism-candidate-oldest-critical` | PrismBlockCandidateOldestPendingCritical | `PrismBlockCandidateOldestPendingCritical` (migrated) | Native producer is populated; preserve the deployed signal with the explicit native bounds and guards in the specification. | +| `qbit-prism-candidate-metrics-unavailable` | PrismBlockCandidateMetricsUnavailable | `PrismBlockCandidateMetricsUnavailable` (migrated) | Native producer is populated; preserve the deployed signal with the explicit native bounds and guards in the specification. | +| `qbit-prism-cleanup-backlog-depth` | PrismCollapseCleanupBacklog | no replacement (no-replacement) | No replacement: Python collapse-cleanup retry queue/backpressure producer is absent; do not reinterpret native pending candidate rows as cleanup state. | +| `qbit-prism-cleanup-backlog-oldest` | PrismCollapseCleanupBacklogOldest | no replacement (no-replacement) | No replacement: Python collapse-cleanup retry queue/backpressure producer is absent; do not reinterpret native pending candidate rows as cleanup state. | +| `qbit-prism-cleanup-backlog-near-bound` | PrismCollapseCleanupBacklogNearBound | no replacement (no-replacement) | No replacement: Python collapse-cleanup retry queue/backpressure producer is absent; do not reinterpret native pending candidate rows as cleanup state. | +| `qbit-prism-cleanup-backpressure-active` | PrismCollapseCleanupBackpressure | no replacement (no-replacement) | No replacement: Python collapse-cleanup retry queue/backpressure producer is absent; do not reinterpret native pending candidate rows as cleanup state. | +| `qbit-prism-cleanup-backpressure-engaged` | PrismCollapseCleanupBackpressureEngaged | no replacement (no-replacement) | No replacement: Python collapse-cleanup retry queue/backpressure producer is absent; do not reinterpret native pending candidate rows as cleanup state. | +| `qbit-prism-semantic-work-coverage` | PrismSemanticWorkCoverageLoss | `PrismSemanticWorkCoverageLoss` (migrated) | Native producer is populated; preserve the deployed signal with the explicit native bounds and guards in the specification. | +| `qbit-prism-semantic-coverage-critical` | PrismSemanticWorkCoverageLossCritical | `PrismSemanticWorkCoverageLossCritical` (migrated) | Native producer is populated; preserve the deployed signal with the explicit native bounds and guards in the specification. | +| `qbit-prism-refresh-pending` | PrismRefreshPendingPastDeadline | `PrismTimeToUsableWorkHigh`, `PrismCurrentWorkGapHigh` (migrated) | Observe the effect on usable work; native refresh has no Python pending-deadline state. | +| `qbit-prism-refresh-pending-critical` | PrismRefreshPendingPastDeadlineCritical | `PrismTimeToUsableWorkHigh`, `PrismSemanticWorkCoverageLossCritical` (migrated) | Coverage and first-work delay replace miner-impact intent; native thresholds require #291 measurement. | +| `qbit-prism-metrics-snapshot-stale` | PrismMetricsSnapshotStale | `PrismMetricsSnapshotStale` (migrated) | Native producer is populated; preserve the deployed signal with the explicit native bounds and guards in the specification. | +| `qbit-prism-metrics-snapshot-unavailable` | PrismMetricsSnapshotUnavailable | `PrismMetricsSnapshotUnavailable` (migrated) | Native producer is populated; preserve the deployed signal with the explicit native bounds and guards in the specification. | +| `qbit-prism-metrics-snapshot-age` | PrismMetricsSnapshotAgeHigh | `PrismMetricsSnapshotAgeHigh` (migrated) | Native producer is populated; preserve the deployed signal with the explicit native bounds and guards in the specification. | +| `qbit-prism-public-staleness-refusal` | PrismPublicStalenessRefusalsIncreasing | `PrismPublicStalenessRefusalsIncreasing` (migrated) | Native producer is populated; preserve the deployed signal with the explicit native bounds and guards in the specification. | +| `qbit-prism-public-ledger-unready` | PrismPublicLedgerUnready | `PrismPublicLedgerUnready` (migrated) | Native producer is populated; preserve the deployed signal with the explicit native bounds and guards in the specification. | +| `qbit-prism-public-5xx-sustained` | PrismPublic5xxSustained | `PrismPublic5xxSustained` (migrated) | Native producer is populated; preserve the deployed signal with the explicit native bounds and guards in the specification. | +| `qbit-prism-backup-metrics-nodata` | PrismBackupMetricsNoData | `PrismBackupMetricsNoData` (unchanged-external) | No native replacement needed: deployment-owned backup/node-exporter rule and producer are retained byte-for-byte. | +| `qbit-prism-backup-collector-stale` | PrismBackupCollectorStale | `PrismBackupCollectorStale` (unchanged-external) | No native replacement needed: deployment-owned backup/node-exporter rule and producer are retained byte-for-byte. | +| `qbit-prism-backup-full-stale-warning` | PrismBackupFullStaleWarning | `PrismBackupFullStaleWarning` (unchanged-external) | No native replacement needed: deployment-owned backup/node-exporter rule and producer are retained byte-for-byte. | +| `qbit-prism-backup-full-stale-critical` | PrismBackupFullStaleCritical | `PrismBackupFullStaleCritical` (unchanged-external) | No native replacement needed: deployment-owned backup/node-exporter rule and producer are retained byte-for-byte. | +| `qbit-prism-backup-diff-stale-warning` | PrismBackupDifferentialStaleWarning | `PrismBackupDifferentialStaleWarning` (unchanged-external) | No native replacement needed: deployment-owned backup/node-exporter rule and producer are retained byte-for-byte. | +| `qbit-prism-backup-diff-stale-critical` | PrismBackupDifferentialStaleCritical | `PrismBackupDifferentialStaleCritical` (unchanged-external) | No native replacement needed: deployment-owned backup/node-exporter rule and producer are retained byte-for-byte. | +| `qbit-prism-backup-check-warning` | PrismBackupCheckWarning | `PrismBackupCheckWarning` (unchanged-external) | No native replacement needed: deployment-owned backup/node-exporter rule and producer are retained byte-for-byte. | +| `qbit-prism-backup-check-critical` | PrismBackupCheckCritical | `PrismBackupCheckCritical` (unchanged-external) | No native replacement needed: deployment-owned backup/node-exporter rule and producer are retained byte-for-byte. | +| `qbit-prism-backup-wal-stale-warning` | PrismBackupWalArchiveStaleWarning | `PrismBackupWalArchiveStaleWarning` (unchanged-external) | No native replacement needed: deployment-owned backup/node-exporter rule and producer are retained byte-for-byte. | +| `qbit-prism-backup-wal-stale-critical` | PrismBackupWalArchiveStaleCritical | `PrismBackupWalArchiveStaleCritical` (unchanged-external) | No native replacement needed: deployment-owned backup/node-exporter rule and producer are retained byte-for-byte. | +| `qbit-prism-backup-wal-fail-warning` | PrismBackupWalArchiveFailureWarning | `PrismBackupWalArchiveFailureWarning` (unchanged-external) | No native replacement needed: deployment-owned backup/node-exporter rule and producer are retained byte-for-byte. | +| `qbit-prism-backup-wal-fail-critical` | PrismBackupWalArchiveFailureCritical | `PrismBackupWalArchiveFailureCritical` (unchanged-external) | No native replacement needed: deployment-owned backup/node-exporter rule and producer are retained byte-for-byte. | +| `qbit-prism-backup-wal-space-warning` | PrismBackupWalFilesystemSpaceWarning | `PrismBackupWalFilesystemSpaceWarning` (unchanged-external) | No native replacement needed: deployment-owned backup/node-exporter rule and producer are retained byte-for-byte. | +| `qbit-prism-backup-wal-space-critical` | PrismBackupWalFilesystemSpaceCritical | `PrismBackupWalFilesystemSpaceCritical` (unchanged-external) | No native replacement needed: deployment-owned backup/node-exporter rule and producer are retained byte-for-byte. | +| `qbit-prism-backup-audit-stale-warning` | PrismBackupAuditSnapshotStaleWarning | `PrismBackupAuditSnapshotStaleWarning` (unchanged-external) | No native replacement needed: deployment-owned backup/node-exporter rule and producer are retained byte-for-byte. | +| `qbit-prism-backup-audit-stale-critical` | PrismBackupAuditSnapshotStaleCritical | `PrismBackupAuditSnapshotStaleCritical` (unchanged-external) | No native replacement needed: deployment-owned backup/node-exporter rule and producer are retained byte-for-byte. | +| `qbit-prism-backup-audit-slow-warning` | PrismBackupAuditDurationWarning | `PrismBackupAuditDurationWarning` (unchanged-external) | No native replacement needed: deployment-owned backup/node-exporter rule and producer are retained byte-for-byte. | +| `qbit-prism-backup-audit-slow-critical` | PrismBackupAuditDurationCritical | `PrismBackupAuditDurationCritical` (unchanged-external) | No native replacement needed: deployment-owned backup/node-exporter rule and producer are retained byte-for-byte. | +| `qbit-prism-backup-job-fail-warning` | PrismBackupScheduledJobFailureWarning | `PrismBackupScheduledJobFailureWarning` (unchanged-external) | No native replacement needed: deployment-owned backup/node-exporter rule and producer are retained byte-for-byte. | +| `qbit-prism-backup-job-fail-critical` | PrismBackupScheduledJobFailureCritical | `PrismBackupScheduledJobFailureCritical` (unchanged-external) | No native replacement needed: deployment-owned backup/node-exporter rule and producer are retained byte-for-byte. | +| `qbit-prism-backup-coverage-critical` | PrismBackupCombinedCoverageCritical | `PrismBackupCombinedCoverageCritical` (unchanged-external) | No native replacement needed: deployment-owned backup/node-exporter rule and producer are retained byte-for-byte. | +| `qbit-prism-backup-db-growth-warning` | PrismBackupDatabaseGrowthWarning | `PrismBackupDatabaseGrowthWarning` (unchanged-external) | No native replacement needed: deployment-owned backup/node-exporter rule and producer are retained byte-for-byte. | +| `qbit-prism-backup-db-growth-critical` | PrismBackupDatabaseGrowthCritical | `PrismBackupDatabaseGrowthCritical` (unchanged-external) | No native replacement needed: deployment-owned backup/node-exporter rule and producer are retained byte-for-byte. | +| `qbit-prism-backup-audit-growth-warning` | PrismBackupAuditGrowthWarning | `PrismBackupAuditGrowthWarning` (unchanged-external) | No native replacement needed: deployment-owned backup/node-exporter rule and producer are retained byte-for-byte. | +| `qbit-prism-backup-audit-growth-critical` | PrismBackupAuditGrowthCritical | `PrismBackupAuditGrowthCritical` (unchanged-external) | No native replacement needed: deployment-owned backup/node-exporter rule and producer are retained byte-for-byte. | +| `qbit-prism-backup-wal-growth-warning` | PrismBackupWalGenerationGrowthWarning | `PrismBackupWalGenerationGrowthWarning` (unchanged-external) | No native replacement needed: deployment-owned backup/node-exporter rule and producer are retained byte-for-byte. | +| `qbit-prism-backup-wal-growth-critical` | PrismBackupWalGenerationGrowthCritical | `PrismBackupWalGenerationGrowthCritical` (unchanged-external) | No native replacement needed: deployment-owned backup/node-exporter rule and producer are retained byte-for-byte. | +| `qbit-prism-coordinator-collector-stale` | PrismCoordinatorCollectorStale | `PrismCoordinatorCollectorStale` (unchanged-external) | No native replacement needed: deployment-owned Docker lifecycle textfile rule and producer are retained byte-for-byte. | +| `qbit-prism-coordinator-restart` | PrismCoordinatorUnexpectedRestart | `PrismCoordinatorUnexpectedRestart` (unchanged-external) | No native replacement needed: deployment-owned Docker lifecycle textfile rule and producer are retained byte-for-byte. | +| `qbit-prism-coordinator-restart-loop` | PrismCoordinatorRestartLoop | `PrismCoordinatorRestartLoop` (unchanged-external) | No native replacement needed: deployment-owned Docker lifecycle textfile rule and producer are retained byte-for-byte. | + +## Every historical name + +All **46** distinct tokens in the historical appendix are mapped below, including histogram suffixes and names retained by native producers. + +| Historical 2.x.x name | Native replacement / disposition | Reason | +| --- | --- | --- | +| `qbit_prism_accepted_block_landing_phase_seconds` | no replacement | No replacement: accepted landing/preview publication is not proof-to-first-node-offer; landing phases are outside trimmed #278. First-offer timing is declared, rule deferred to A/#266. | +| `qbit_prism_accepted_block_landing_phase_seconds_count` | no replacement | No replacement: accepted landing/preview publication is not proof-to-first-node-offer; landing phases are outside trimmed #278. First-offer timing is declared, rule deferred to A/#266. | +| `qbit_prism_accepted_block_landing_phase_seconds_max` | no replacement | No replacement: accepted landing/preview publication is not proof-to-first-node-offer; landing phases are outside trimmed #278. First-offer timing is declared, rule deferred to A/#266. | +| `qbit_prism_accepted_block_landing_phase_seconds_sum` | no replacement | No replacement: accepted landing/preview publication is not proof-to-first-node-offer; landing phases are outside trimmed #278. First-offer timing is declared, rule deferred to A/#266. | +| `qbit_prism_accepted_block_preview_publication_seconds` | `qbit_prism_accepted_block_to_revision_work_seconds` | New frontend-local acceptance-to-successful-mining.notify boundary; closed result labels, different bucket ladder and restart/observation horizon are documented in prism-native-metrics.md. | +| `qbit_prism_accepted_block_preview_publication_seconds_bucket` | `qbit_prism_accepted_block_to_revision_work_seconds` | New frontend-local acceptance-to-successful-mining.notify boundary; closed result labels, different bucket ladder and restart/observation horizon are documented in prism-native-metrics.md. | +| `qbit_prism_accepted_block_preview_publication_seconds_count` | `qbit_prism_accepted_block_to_revision_work_seconds` | New frontend-local acceptance-to-successful-mining.notify boundary; closed result labels, different bucket ladder and restart/observation horizon are documented in prism-native-metrics.md. | +| `qbit_prism_accepted_parent_preview_wait_timeouts_total` | `qbit_prism_revision_work_build_timeouts_total` | Existing native build-deadline hits while accepted revision work awaits delivery; no new preview-wait mechanism or deadline is introduced. | +| `qbit_prism_accepted_parent_unresolved_depth_max` | no replacement | No replacement: Python unresolved-parent/preview-wait state machine is absent; pending candidate rows are a different contract. | +| `qbit_prism_accepted_parent_unresolved_oldest_seconds` | no replacement | No replacement: Python unresolved-parent/preview-wait state machine is absent; pending candidate rows are a different contract. | +| `qbit_prism_accepted_parent_unresolved_transitions` | no replacement | No replacement: Python unresolved-parent/preview-wait state machine is absent; pending candidate rows are a different contract. | +| `qbit_prism_block_candidate_cleanup_backpressure_active` | no replacement | No replacement: Python collapse/cleanup retry holders, pins and backpressure state have no native producer. | +| `qbit_prism_block_candidate_cleanup_backpressure_total` | no replacement | No replacement: Python collapse/cleanup retry holders, pins and backpressure state have no native producer. | +| `qbit_prism_block_candidate_cleanup_retry_backlog` | no replacement | No replacement: Python collapse/cleanup retry holders, pins and backpressure state have no native producer. | +| `qbit_prism_block_candidate_cleanup_retry_backlog_max` | no replacement | No replacement: Python collapse/cleanup retry holders, pins and backpressure state have no native producer. | +| `qbit_prism_block_candidate_cleanup_retry_oldest_seconds` | no replacement | No replacement: Python collapse/cleanup retry holders, pins and backpressure state have no native producer. | +| `qbit_prism_block_candidate_cleanup_retry_pending_share_holders` | no replacement | No replacement: Python collapse/cleanup retry holders, pins and backpressure state have no native producer. | +| `qbit_prism_block_candidate_cleanup_retry_terminal_outcome_pins` | no replacement | No replacement: Python collapse/cleanup retry holders, pins and backpressure state have no native producer. | +| `qbit_prism_block_candidate_collapse_total` | no replacement | No replacement: Python collapse/cleanup retry holders, pins and backpressure state have no native producer. | +| `qbit_prism_block_candidate_oldest_pending_seconds` | `qbit_prism_block_candidate_oldest_pending_seconds` | Same name retained by #278/#277; use native documented meaning and unknown/staleness guards. | +| `qbit_prism_block_candidates_pending` | `qbit_prism_block_candidates_pending` | Same name retained by #278/#277; use native documented meaning and unknown/staleness guards. | +| `qbit_prism_block_ledger_call_timeouts_total` | no replacement | No replacement: Python ledger timeout/submitter backoff series are absent; candidate age retains backlog intent without claiming cause. | +| `qbit_prism_block_submitter_retry_backoff_active` | no replacement | No replacement: Python ledger timeout/submitter backoff series are absent; candidate age retains backlog intent without claiming cause. | +| `qbit_prism_ledger_read_calls_total` | no replacement | No replacement: Python read-admission gate and execution timers are absent. Native database_pool_acquire_seconds aggregates observed checkout attempts at covered callers: metrics collection, instrumented ledger transactions and direct ledger queries, rollup transactions and share-partition maintenance. Public-role read pools, private audit HTTP endpoints and shared API read-model helpers, startup/migration gates, operator tools and telemetry-free compatibility wrappers remain excluded; see the acquisition guide for the source census and exclusion policy. Pool acquisition timing cannot replace query execution, advisory-lock waits or read-admission gate timing. | +| `qbit_prism_ledger_read_execute_seconds_total` | no replacement | No replacement: Python read-admission gate and execution timers are absent. Native database_pool_acquire_seconds aggregates observed checkout attempts at covered callers: metrics collection, instrumented ledger transactions and direct ledger queries, rollup transactions and share-partition maintenance. Public-role read pools, private audit HTTP endpoints and shared API read-model helpers, startup/migration gates, operator tools and telemetry-free compatibility wrappers remain excluded; see the acquisition guide for the source census and exclusion policy. Pool acquisition timing cannot replace query execution, advisory-lock waits or read-admission gate timing. | +| `qbit_prism_ledger_read_execute_timeouts_total` | no replacement | No replacement: Python read-admission gate and execution timers are absent. Native database_pool_acquire_seconds aggregates observed checkout attempts at covered callers: metrics collection, instrumented ledger transactions and direct ledger queries, rollup transactions and share-partition maintenance. Public-role read pools, private audit HTTP endpoints and shared API read-model helpers, startup/migration gates, operator tools and telemetry-free compatibility wrappers remain excluded; see the acquisition guide for the source census and exclusion policy. Pool acquisition timing cannot replace query execution, advisory-lock waits or read-admission gate timing. | +| `qbit_prism_ledger_read_gate_timeouts_total` | no replacement | No replacement: Python read-admission gate and execution timers are absent. Native database_pool_acquire_seconds aggregates observed checkout attempts at covered callers: metrics collection, instrumented ledger transactions and direct ledger queries, rollup transactions and share-partition maintenance. Public-role read pools, private audit HTTP endpoints and shared API read-model helpers, startup/migration gates, operator tools and telemetry-free compatibility wrappers remain excluded; see the acquisition guide for the source census and exclusion policy. Pool acquisition timing cannot replace query execution, advisory-lock waits or read-admission gate timing. | +| `qbit_prism_ledger_read_gate_wait_seconds_total` | no replacement | No replacement: Python read-admission gate and execution timers are absent. Native database_pool_acquire_seconds aggregates observed checkout attempts at covered callers: metrics collection, instrumented ledger transactions and direct ledger queries, rollup transactions and share-partition maintenance. Public-role read pools, private audit HTTP endpoints and shared API read-model helpers, startup/migration gates, operator tools and telemetry-free compatibility wrappers remain excluded; see the acquisition guide for the source census and exclusion policy. Pool acquisition timing cannot replace query execution, advisory-lock waits or read-admission gate timing. | +| `qbit_prism_metrics_snapshot_age_seconds` | `qbit_prism_metrics_snapshot_age_seconds` | Same name retained by #278/#277; use native documented meaning and unknown/staleness guards. | +| `qbit_prism_metrics_snapshot_available` | `qbit_prism_metrics_snapshot_available` | Same name retained by #278/#277; use native documented meaning and unknown/staleness guards. | +| `qbit_prism_metrics_snapshot_stale` | `qbit_prism_metrics_snapshot_stale` | Same name retained by #278/#277; use native documented meaning and unknown/staleness guards. | +| `qbit_prism_payout_artifact_events_total` | no replacement | No replacement: payout artifact/rescan/balance-read instrumentation is outside trimmed #278; no deployed rule to rewrite. | +| `qbit_prism_payout_window_full_rescan_seconds` | no replacement | No replacement: payout artifact/rescan/balance-read instrumentation is outside trimmed #278; no deployed rule to rewrite. | +| `qbit_prism_payout_window_full_rescan_seconds_count` | no replacement | No replacement: payout artifact/rescan/balance-read instrumentation is outside trimmed #278; no deployed rule to rewrite. | +| `qbit_prism_prior_balances_reads_total` | no replacement | No replacement: payout artifact/rescan/balance-read instrumentation is outside trimmed #278; no deployed rule to rewrite. | +| `qbit_prism_refresh_pending` | `qbit_prism_stratum_oldest_pending_initial_job_seconds`, `qbit_prism_stratum_current_tip_coverage_gap_seconds` | Replacement for alert intent only: time to usable work, not Python refresh-task state. | +| `qbit_prism_refresh_pending_age_seconds` | `qbit_prism_stratum_oldest_pending_initial_job_seconds`, `qbit_prism_stratum_current_tip_coverage_gap_seconds` | Replacement for alert intent only: time to usable work, not Python refresh-task state. | +| `qbit_prism_reorg_reconcile_errors_total` | no replacement | No replacement: Python reconciler pass/step/error series are outside trimmed native instrumentation; do not add undeployed historical rules. | +| `qbit_prism_reorg_reconcile_lookups_total` | no replacement | No replacement: Python reconciler pass/step/error series are outside trimmed native instrumentation; do not add undeployed historical rules. | +| `qbit_prism_reorg_reconcile_pass_seconds` | no replacement | No replacement: Python reconciler pass/step/error series are outside trimmed native instrumentation; do not add undeployed historical rules. | +| `qbit_prism_reorg_reconcile_pass_seconds_count` | no replacement | No replacement: Python reconciler pass/step/error series are outside trimmed native instrumentation; do not add undeployed historical rules. | +| `qbit_prism_reorg_reconcile_pass_seconds_sum` | no replacement | No replacement: Python reconciler pass/step/error series are outside trimmed native instrumentation; do not add undeployed historical rules. | +| `qbit_prism_reorg_reconcile_step_seconds` | no replacement | No replacement: Python reconciler pass/step/error series are outside trimmed native instrumentation; do not add undeployed historical rules. | +| `qbit_prism_reorg_reconcile_step_seconds_sum` | no replacement | No replacement: Python reconciler pass/step/error series are outside trimmed native instrumentation; do not add undeployed historical rules. | +| `qbit_prism_stratum_authorized_connections` | `qbit_prism_authorized_clients` | Native local authorized clients; update semantic-coverage joins. | +| `qbit_prism_stratum_semantic_current_work_ratio` | `qbit_prism_stratum_semantic_current_work_ratio` | Same name retained by #278/#277; use native documented meaning and unknown/staleness guards. | + +## Native rule evidence + +The executable PromQL and Grafana conditions are in [prism-native-alert-rules.json](prism-native-alert-rules.json). +These are the only rewritten native rules; the historical appendix is not an active rule specification. + +| Rule | Threshold evidence | Producer contract | +| --- | --- | --- | +| PrismMetricsNoData | Preserves the deployed three-minute scrape coverage grace. | [registry.rs#L43](../crates/qbit-prism-server/src/metrics/registry.rs#L43), [metrics.rs#L39](../crates/qbit-prism-server/src/metrics.rs#L39) | +| PrismMetricsEndpointDown | Preserves the deployed three-minute scrape outage grace. | [metrics.rs#L39](../crates/qbit-prism-server/src/metrics.rs#L39) | +| PRISM Connected Clients | Preserves the deployed network-wide floor: sum all local native connection counts, below ten for three minutes; evaluate only with fresh measurements for every configured target. | [registry.rs#L45](../crates/qbit-prism-server/src/metrics/registry.rs#L45), [registry.rs#L78](../crates/qbit-prism-server/src/metrics/registry.rs#L78), [registry.rs#L79](../crates/qbit-prism-server/src/metrics/registry.rs#L79), [metrics.rs#L39](../crates/qbit-prism-server/src/metrics.rs#L39) | +| PrismMetricsSnapshotStale | #277 computes staleness at scrape time from max(3 * health refresh, 15s); no copied age constant. | [registry.rs#L79](../crates/qbit-prism-server/src/metrics/registry.rs#L79), [metrics.rs#L39](../crates/qbit-prism-server/src/metrics.rs#L39) | +| PrismMetricsSnapshotUnavailable | #277 computes staleness at scrape time from max(3 * health refresh, 15s); no copied age constant. | [registry.rs#L78](../crates/qbit-prism-server/src/metrics/registry.rs#L78), [metrics.rs#L39](../crates/qbit-prism-server/src/metrics.rs#L39) | +| PrismMetricsSnapshotAgeHigh | #277 computes staleness at scrape time from max(3 * health refresh, 15s); no copied age constant. | [registry.rs#L79](../crates/qbit-prism-server/src/metrics/registry.rs#L79), [registry.rs#L80](../crates/qbit-prism-server/src/metrics/registry.rs#L80), [metrics.rs#L39](../crates/qbit-prism-server/src/metrics.rs#L39) | +| PrismShareAckP99High | One-second histogram boundary; at least 100 completed ACKs in five minutes. Provisional, measure in #291. | [registry.rs#L55](../crates/qbit-prism-server/src/metrics/registry.rs#L55), [registry.rs#L78](../crates/qbit-prism-server/src/metrics/registry.rs#L78), [registry.rs#L79](../crates/qbit-prism-server/src/metrics/registry.rs#L79), [labels.rs#L15](../crates/qbit-prism-server/src/metrics/labels.rs#L15), [metrics.rs#L39](../crates/qbit-prism-server/src/metrics.rs#L39) | +| PrismLandingAckP99Slow | #602 R1 asks for a landing-window p99 at or under 500 ms and no acknowledgement over 2 s; 2.x.x shows a block-minute p99 of about 4-5 s against about 0.2 s otherwise. Three consecutive windows keep one slow landing (a cold cache, a one-off settlement) from ticketing. Provisional, measure in #291 and #602 step 3. | [registry.rs](../crates/qbit-prism-server/src/metrics/registry.rs), [landing_acks.rs](../crates/qbit-prism-server/src/metrics/landing_acks.rs), [landing.rs](../crates/qbit-prism-server/src/metrics/landing.rs), [share_observation.rs](../crates/qbit-prism-server/src/stratum/share_observation.rs) | +| PrismLandingAckP99High | 10 s is two thirds of the default share-commit timeout (PRISM_SHARE_COMMIT_TIMEOUT_SECONDS, 15 s), past which an append is answered ledger-outcome-unknown; rental proxies and some firmware resubmit or reconnect well before it. One window is enough. Provisional, measure in #291 and #602 step 3. | [registry.rs](../crates/qbit-prism-server/src/metrics/registry.rs), [landing_acks.rs](../crates/qbit-prism-server/src/metrics/landing_acks.rs), [landing.rs](../crates/qbit-prism-server/src/metrics/landing.rs), [share_observation.rs](../crates/qbit-prism-server/src/stratum/share_observation.rs) | +| PrismRejectRatioByReasonHigh | Five percent per canonical reason over five minutes, with at least 100 acceptance/rejection decisions, both counters live at scrape time (#581). Provisional, measure in #291. | [registry.rs#L54](../crates/qbit-prism-server/src/metrics/registry.rs#L54), [registry.rs#L60](../crates/qbit-prism-server/src/metrics/registry.rs#L60), [registry.rs#L155](../crates/qbit-prism-server/src/metrics/registry.rs#L155), [labels.rs#L78](../crates/qbit-prism-server/src/metrics/labels.rs#L78), [metrics.rs#L252](../crates/qbit-prism-server/src/metrics.rs#L252) | +| PrismPoolResubmissionRejectRatioHigh | Five percent of the pool's acceptance/rejection decisions over five minutes, from at least 500 decisions, for five minutes; the per-frontend, per-reason rule uses the same five percent. Provisional, measure in #291 and against 2.x.x's post-block reject spikes. | [registry.rs](../crates/qbit-prism-server/src/metrics/registry.rs), [labels.rs](../crates/qbit-prism-server/src/metrics/labels.rs), [share_observation.rs](../crates/qbit-prism-server/src/stratum/share_observation.rs) | +| PrismShareAppendFailures | Any ledger rejection whose confirmation failed or outcome unknown (ledger-confirmation-failed or ledger-outcome-unknown) preserves the deployed low-volume total-write-failure coverage; its input is live at scrape time since #581; measure alert volume in #291. | [registry.rs#L60](../crates/qbit-prism-server/src/metrics/registry.rs#L60), [registry.rs#L155](../crates/qbit-prism-server/src/metrics/registry.rs#L155), [labels.rs#L78](../crates/qbit-prism-server/src/metrics/labels.rs#L78) | +| PrismTimeToUsableWorkHigh | Authorization to successfully written first job; 15 seconds is a provisional bound, measure in #291. | [registry.rs#L61](../crates/qbit-prism-server/src/metrics/registry.rs#L61), [registry.rs#L62](../crates/qbit-prism-server/src/metrics/registry.rs#L62), [registry.rs#L78](../crates/qbit-prism-server/src/metrics/registry.rs#L78), [registry.rs#L79](../crates/qbit-prism-server/src/metrics/registry.rs#L79), [metrics.rs#L39](../crates/qbit-prism-server/src/metrics.rs#L39) | +| PrismCurrentWorkGapHigh | Native timer advances only while semantic coverage is strictly below 0.95; 15-second duration is provisional, measure in #291. | [registry.rs#L46](../crates/qbit-prism-server/src/metrics/registry.rs#L46), [registry.rs#L63](../crates/qbit-prism-server/src/metrics/registry.rs#L63), [registry.rs#L78](../crates/qbit-prism-server/src/metrics/registry.rs#L78), [registry.rs#L79](../crates/qbit-prism-server/src/metrics/registry.rs#L79), [metrics.rs#L39](../crates/qbit-prism-server/src/metrics.rs#L39) | +| PrismSemanticWorkCoverageLoss | Native semantic generation coverage; warning matches the strict 95-percent timer boundary. Critical 50 percent and three-minute dwell: Provisional, measure in #291. | [registry.rs#L46](../crates/qbit-prism-server/src/metrics/registry.rs#L46), [registry.rs#L64](../crates/qbit-prism-server/src/metrics/registry.rs#L64), [registry.rs#L78](../crates/qbit-prism-server/src/metrics/registry.rs#L78), [registry.rs#L79](../crates/qbit-prism-server/src/metrics/registry.rs#L79), [metrics.rs#L39](../crates/qbit-prism-server/src/metrics.rs#L39) | +| PrismSemanticWorkCoverageLossCritical | Native semantic generation coverage; warning matches the strict 95-percent timer boundary. Critical 50 percent and three-minute dwell: Provisional, measure in #291. | [registry.rs#L46](../crates/qbit-prism-server/src/metrics/registry.rs#L46), [registry.rs#L64](../crates/qbit-prism-server/src/metrics/registry.rs#L64), [registry.rs#L78](../crates/qbit-prism-server/src/metrics/registry.rs#L78), [registry.rs#L79](../crates/qbit-prism-server/src/metrics/registry.rs#L79), [metrics.rs#L39](../crates/qbit-prism-server/src/metrics.rs#L39) | +| PrismWorkRefreshStalledCritical | This frontend's template refresh has not succeeded for 120 seconds, so it has neither published nor revalidated work: job, share and block admission refuse once readiness is older than the health timeout, and while refreshes fail only a node poll that finds the published tip with its template within PRISM_TEMPLATE_MAX_AGE_SECONDS (120 s by default) renews it (#622), so by then plus the health timeout every miner on it is stalled whatever the cause; qbit_prism_tip_poll_age_seconds shows which side of that a paged frontend is on. That covers #525 on either settlement path, where a pool without a fee cannot settle sub-floor dust and every build fails with PayoutExceedsCandidateBalance, whether the dust meets a new tip or a same-parent rebuild; it also covers a refresh that fails before its tip observation or hangs. Semantic coverage cannot see this: it compares sessions with the latest published generation, which stays put while every refresh fails. A healthy refresh succeeds every poll interval (2 s by default, and within the health timeout for the frontend to be ready at all); 120 seconds is the default replacement-build lease (PRISM_TEMPLATE_REFRESH_FAILURE_EXIT_SECONDS). One-minute dwell: the gauge is monotonic during a stall. Not gated on connected miners: a frontend without working refreshes is stalled for every miner that connects. A node still syncing or short of peers also pages here, since it builds no work. -1 (not yet published) never fires; no-data and evaluation errors are OK because the snapshot rules own missing measurements, and a draining process stops its snapshot. Provisional, measure in #291. | [registry.rs](../crates/qbit-prism-server/src/metrics/registry.rs), [coordinator.rs](../crates/qbit-prism-server/src/coordinator.rs), [server.rs](../crates/qbit-prism-server/src/server.rs) | +| PrismWorkMetricsUnavailable | Unknown sentinels must never masquerade as healthy work coverage. | [registry.rs#L62](../crates/qbit-prism-server/src/metrics/registry.rs#L62), [registry.rs#L64](../crates/qbit-prism-server/src/metrics/registry.rs#L64), [registry.rs#L78](../crates/qbit-prism-server/src/metrics/registry.rs#L78), [registry.rs#L79](../crates/qbit-prism-server/src/metrics/registry.rs#L79), [metrics.rs#L39](../crates/qbit-prism-server/src/metrics.rs#L39) | +| PrismBlockCandidateBacklog | Native pending outbox rows include claimed/retry-delayed candidates; count bounds 2/5 and age bounds 15/60 seconds. Provisional, measure in #291. | [registry.rs#L66](../crates/qbit-prism-server/src/metrics/registry.rs#L66), [registry.rs#L70](../crates/qbit-prism-server/src/metrics/registry.rs#L70), [registry.rs#L78](../crates/qbit-prism-server/src/metrics/registry.rs#L78), [registry.rs#L79](../crates/qbit-prism-server/src/metrics/registry.rs#L79), [labels.rs#L18](../crates/qbit-prism-server/src/metrics/labels.rs#L18), [metrics.rs#L39](../crates/qbit-prism-server/src/metrics.rs#L39) | +| PrismBlockCandidateBacklogCritical | Native pending outbox rows include claimed/retry-delayed candidates; count bounds 2/5 and age bounds 15/60 seconds. Provisional, measure in #291. | [registry.rs#L66](../crates/qbit-prism-server/src/metrics/registry.rs#L66), [registry.rs#L70](../crates/qbit-prism-server/src/metrics/registry.rs#L70), [registry.rs#L78](../crates/qbit-prism-server/src/metrics/registry.rs#L78), [registry.rs#L79](../crates/qbit-prism-server/src/metrics/registry.rs#L79), [labels.rs#L18](../crates/qbit-prism-server/src/metrics/labels.rs#L18), [metrics.rs#L39](../crates/qbit-prism-server/src/metrics.rs#L39) | +| PrismBlockCandidateOldestPending | Native pending outbox rows include claimed/retry-delayed candidates; count bounds 2/5 and age bounds 15/60 seconds. Provisional, measure in #291. | [registry.rs#L67](../crates/qbit-prism-server/src/metrics/registry.rs#L67), [registry.rs#L70](../crates/qbit-prism-server/src/metrics/registry.rs#L70), [registry.rs#L78](../crates/qbit-prism-server/src/metrics/registry.rs#L78), [registry.rs#L79](../crates/qbit-prism-server/src/metrics/registry.rs#L79), [labels.rs#L18](../crates/qbit-prism-server/src/metrics/labels.rs#L18), [metrics.rs#L39](../crates/qbit-prism-server/src/metrics.rs#L39) | +| PrismBlockCandidateOldestPendingCritical | Pages only on a found block the node has not accepted (#493): pending and offer-reserved rows, offered rows whose one submitblock outcome is unknown (a transport failure or timeout, or a reservation whose call was lost with its frontend; not a row adopted on the node's own evidence that its block is active), and rows the node definitively rejected unless the reply names a side-chain block (inconclusive, duplicate, duplicate-inconclusive), read from the same database snapshot as the all-unfinished gauges. A rejection such as bad-txnmrklroot, high-hash, bad-cb-* or duplicate-invalid means the pool offered an invalid block, which is the emergency this rule exists for, and it keeps paging exactly as before. A node-accepted lost race sits in reconciliation until a different block has PRISM_CANDIDATE_ORPHAN_CONFIRMATIONS confirmations at its height; that is expected and never pages here. An unknown outcome keeps paging because receipt is unproven: the block may never have reached the node and is never offered again, so it is indistinguishable from an unoffered block until the chain proves an orphan or a later observation lands it. A landing that keeps failing after acceptance is owned by the landing-failed rules; every other unfinished row by the stuck warning. Mixed-version rollout: on a target that does not export the unacknowledged series yet, the expression falls back to the all-unfinished age, so the base paging behaviour (including its lost-race page) persists there until that frontend is upgraded; deploy frontends first. Native pending outbox rows include claimed/retry-delayed candidates; count bounds 2/5 and age bounds 15/60 seconds. Provisional, measure in #291. | [registry.rs#L67](../crates/qbit-prism-server/src/metrics/registry.rs#L67), [registry.rs#L70](../crates/qbit-prism-server/src/metrics/registry.rs#L70), [registry.rs#L78](../crates/qbit-prism-server/src/metrics/registry.rs#L78), [registry.rs#L79](../crates/qbit-prism-server/src/metrics/registry.rs#L79), [labels.rs#L18](../crates/qbit-prism-server/src/metrics/labels.rs#L18), [metrics.rs#L39](../crates/qbit-prism-server/src/metrics.rs#L39) | +| PrismBlockCandidateStuck | An unfinished candidate row older than 30 minutes (#493), read from the all-unfinished gauge: an offered row in reconciliation whose chain verdict is not arriving, a node-rejected block, or a landing that keeps failing after acceptance. A lost tip race is proven orphan within a few block intervals (six competitor confirmations at the 60-second target spacing plus the reconciliation retry backoff) and leaves this gauge well inside the threshold, aligned with PrismAcceptedBlockUnlanded. The reconciliation backoff is ten seconds times the row's attempt count, capped at an hour, and the count includes the pre-offer attempts, so the cumulative wait to the k-th retry is about 5k(k+1) seconds: the sixth competitor confirmation at 60-second spacing is reached around the eighth retry, about nine minutes, well inside 30 minutes, while a row that needed many pre-offer attempts starts its backoff higher and reaches 30 minutes in a few retries. Pre-offer, unknown-outcome and rejected rows already page at 60 seconds through PrismBlockCandidateOldestPendingCritical and a failing landing through the landing-failed rules; this warning is the non-paging catch-all for the rest, which is operator work (the row's last_error names the reason). Three-minute dwell like the other candidate warnings. Provisional, measure in #291. | [registry.rs#L67](../crates/qbit-prism-server/src/metrics/registry.rs#L67), [registry.rs#L70](../crates/qbit-prism-server/src/metrics/registry.rs#L70), [registry.rs#L78](../crates/qbit-prism-server/src/metrics/registry.rs#L78), [registry.rs#L79](../crates/qbit-prism-server/src/metrics/registry.rs#L79), [labels.rs#L18](../crates/qbit-prism-server/src/metrics/labels.rs#L18), [metrics.rs#L39](../crates/qbit-prism-server/src/metrics.rs#L39) | +| PrismBlockCandidateLandingFailed | An offered or reconciliation row whose audit landing has not succeeded after the offer (#493): the landing transaction is the only writer of the block's pool-block row, so its absence is the durable fact the collector reads, and a last_error naming a landing refusal (a prefix the coordinator and the collector share) covers a landed row that no longer authenticates; a retry that fails for a transient node or database reason, or a refused operator recovery, overwrites last_error but cannot make this age read zero while nothing has landed. The node accepted or rejected the block, the audit rebuild, its verification or the ledger landing refused, and the row is retried at the reconciliation backoff with read-only chain observations only. The age is measured from the offer reservation, the start of the post-offer lifecycle and the same instant the frontend's acceptance clock starts, so pre-offer retry backoff never counts toward it. An offered row counts before its first landing outcome (#510): a won block whose landing hangs in offered, waiting for build capacity with no deadline while its heartbeat keeps the claim, ages here instead of paging nowhere; a healthy offered row lands within the 60-second rebuild deadline, so it never reaches the three-minute warning dwell. Never a lost race: a lost race and a rejection land their audit before the chain verdict, so their pool-block row keeps this age at zero once landed. What is at risk is the block's audit, its pool-block row, the deferred solver credit and the next payout revision, which the successful landing writes; the block reward is on the chain regardless. Warning at 60 seconds for three minutes: a transient refusal (a rebuild deadline under a brief database stall) lands on the first retry ten seconds later and never reaches the dwell; the same shape as the pre-#493 candidate critical, without the page. Provisional, measure in #291. | [registry.rs#L67](../crates/qbit-prism-server/src/metrics/registry.rs#L67), [registry.rs#L70](../crates/qbit-prism-server/src/metrics/registry.rs#L70), [registry.rs#L78](../crates/qbit-prism-server/src/metrics/registry.rs#L78), [registry.rs#L79](../crates/qbit-prism-server/src/metrics/registry.rs#L79), [labels.rs#L18](../crates/qbit-prism-server/src/metrics/labels.rs#L18), [metrics.rs#L39](../crates/qbit-prism-server/src/metrics.rs#L39) | +| PrismBlockCandidateLandingFailedCritical | An offered or reconciliation row whose audit landing has not succeeded after the offer (#493): the landing transaction is the only writer of the block's pool-block row, so its absence is the durable fact the collector reads, and a last_error naming a landing refusal (a prefix the coordinator and the collector share) covers a landed row that no longer authenticates; a retry that fails for a transient node or database reason, or a refused operator recovery, overwrites last_error but cannot make this age read zero while nothing has landed. The node accepted or rejected the block, the audit rebuild, its verification or the ledger landing refused, and the row is retried at the reconciliation backoff with read-only chain observations only. The age is measured from the offer reservation, the start of the post-offer lifecycle and the same instant the frontend's acceptance clock starts, so pre-offer retry backoff never counts toward it. An offered row counts before its first landing outcome (#510): a won block whose landing hangs in offered, waiting for build capacity with no deadline while its heartbeat keeps the claim, ages here instead of paging nowhere; a healthy offered row lands within the 60-second rebuild deadline, so it never reaches the three-minute warning dwell. Never a lost race: a lost race and a rejection land their audit before the chain verdict, so their pool-block row keeps this age at zero once landed. What is at risk is the block's audit, its pool-block row, the deferred solver credit and the next payout revision, which the successful landing writes; the block reward is on the chain regardless. Pages at 307 seconds for one minute, the same floor and dwell as PrismAcceptedRevisionWorkPendingCritical: a landing that fails after the frontend's own active-chain observation already pages that rule at 307 s (the landed identity is a known wait with no revision to deliver), and a landing that fails before it now pages here at the same instant, so a won block whose ledger landing keeps failing is never silent whichever step refused. By 307 seconds the row has been retried about eight times at the reconciliation backoff (ten seconds times the attempt count), so the failure is persistent, never a single deadline. Gated like the other candidate rules on a fresh snapshot and an available collector; no-data and evaluation errors are OK because the unavailable warning owns the missing measurement. Provisional, measure in #291. | [registry.rs#L67](../crates/qbit-prism-server/src/metrics/registry.rs#L67), [registry.rs#L70](../crates/qbit-prism-server/src/metrics/registry.rs#L70), [registry.rs#L78](../crates/qbit-prism-server/src/metrics/registry.rs#L78), [registry.rs#L79](../crates/qbit-prism-server/src/metrics/registry.rs#L79), [labels.rs#L18](../crates/qbit-prism-server/src/metrics/labels.rs#L18), [metrics.rs#L39](../crates/qbit-prism-server/src/metrics.rs#L39) | +| PrismBlockCandidateMetricsUnavailable | Collector failure, cancellation or expiry after 30 seconds invalidates all four gauges, the unacknowledged and landing-failed ages included (#493). | [registry.rs#L66](../crates/qbit-prism-server/src/metrics/registry.rs#L66), [registry.rs#L67](../crates/qbit-prism-server/src/metrics/registry.rs#L67), [registry.rs#L70](../crates/qbit-prism-server/src/metrics/registry.rs#L70), [labels.rs#L18](../crates/qbit-prism-server/src/metrics/labels.rs#L18), [metrics.rs#L39](../crates/qbit-prism-server/src/metrics.rs#L39) | +| PrismBlockSubmissionHeld | qbit_prism_block_submission_enabled is 1 while a frontend offers found blocks, 0 while PRISM_BLOCK_SUBMIT_ENABLED or the cluster's hold holds them, and -1 while the switch is on but the frontend has not read the hold (#661, #664); a production frontend must never hold (docs/mainnet-deployment.md go-live checks). The switch is fixed at process start and the hold changes only by an operator command; -1 shows only until a frontend first reads the hold, which the one-minute dwell covers after a restart. A later failed read keeps the last value read, so a database outage does not page here, and the two-minute windows ride over a failed scrape, which would otherwise restart the dwell; #666. | [registry.rs#L80](../crates/qbit-prism-server/src/metrics/registry.rs#L80), [metrics.rs#L146](../crates/qbit-prism-server/src/metrics.rs#L146), [snapshots.rs#L126](../crates/qbit-prism-server/src/metrics/snapshots.rs#L126), [server.rs#L55](../crates/qbit-prism-server/src/server.rs#L55), [server.rs#L392](../crates/qbit-prism-server/src/server.rs#L392), [submission_hold.rs#L127](../crates/qbit-prism-server/src/ledger/submission_hold.rs#L127), [config.rs#L635](../crates/qbit-prism-server/src/config.rs#L635) | +| PrismDatabaseUnavailable | Database collector unavailable (failure, cancellation, or no success for COLLECTOR_STALE_AFTER = 30 s, collected every 10 s) for two minutes, from #581's disk-exhaustion run, where the collector held unavailable through a 200 s outage while every snapshot-gated rule flapped; provisional, measure in #291. | [registry.rs#L82](../crates/qbit-prism-server/src/metrics/registry.rs#L82), [labels.rs#L71](../crates/qbit-prism-server/src/metrics/labels.rs#L71), [metrics.rs#L23](../crates/qbit-prism-server/src/metrics.rs#L23), [collectors.rs#L123](../crates/qbit-prism-server/src/metrics/collectors.rs#L123) | +| PrismBlockOfferStandbyUnconfirmed | A found block was offered in the last 30 minutes without a confirmed copy on the dedicated failover standby (#529). With PRISM_OFFER_STANDBY_APPLICATION_NAME set, each found block's offer first waits, up to PRISM_OFFER_STANDBY_FLUSH_WAIT_MS (default 250 ms), for that standby's flush_lsn to reach the primary's WAL flush position, which covers the block's candidate row and offer reservation. This counts the offers where it did not: absent (no standby connected under that name), lagging (still behind at the bound) or failed (the position could not be read, for example a writer role without pg_monitor). The block was offered anyway, since the wait never holds a block. Until the standby catches up, losing the primary also loses that block's ledger rows, so a block that reached the chain would have no candidate, audit or pool-blocks record on the promoted primary. The frontend log names each such block hash at WARN (block offer standby wait unconfirmed). Check the D3 standby rules (disconnected or lagging) and the writer role's pg_monitor grant; after any failover, reconcile the logged hashes as the ledger operations guide describes. A healthy standby confirms within a few milliseconds, so one unconfirmed offer is already a signal: no dwell, and the 30-minute window keeps it visible. Warning only: no block was held and no share acknowledgement waits on it. The counter only moves when the wait is configured and a block is found; no-data and evaluation errors are OK because the snapshot rules own missing measurements. Provisional, measure in #291. | [registry.rs](../crates/qbit-prism-server/src/metrics/registry.rs), [coordinator.rs](../crates/qbit-prism-server/src/coordinator.rs), [standby_durability.rs](../crates/qbit-prism-server/src/ledger/standby_durability.rs) | +| PrismDatabasePoolWaitHigh | Collector, instrumented ledger transaction and selected direct-query pool-acquire p99 >0.5 seconds, combining success and failure, from at least ten observed acquisition attempts in five minutes; provisional, measure in #291. | [registry.rs#L68](../crates/qbit-prism-server/src/metrics/registry.rs#L68), [registry.rs#L70](../crates/qbit-prism-server/src/metrics/registry.rs#L70), [registry.rs#L78](../crates/qbit-prism-server/src/metrics/registry.rs#L78), [registry.rs#L79](../crates/qbit-prism-server/src/metrics/registry.rs#L79), [labels.rs#L16](../crates/qbit-prism-server/src/metrics/labels.rs#L16), [labels.rs#L18](../crates/qbit-prism-server/src/metrics/labels.rs#L18), [metrics.rs#L39](../crates/qbit-prism-server/src/metrics.rs#L39) | +| PrismDatabaseAdvisoryLockWaitHigh | Order and settlement advisory-lock p99 >1 second, combining success and failure, from at least ten observed waits for that lock in five minutes; #271 measured 15, 31 and 63 steady ORDER_LOCK waiters at one, two and four frontends without any native wait percentile, and PRISM_DATABASE_LOCK_TIMEOUT_MS bounds a single wait at five seconds by default; provisional, measure in #291/#447. | [registry.rs#L74](../crates/qbit-prism-server/src/metrics/registry.rs#L74), [labels.rs#L66](../crates/qbit-prism-server/src/metrics/labels.rs#L66), [labels.rs#L67](../crates/qbit-prism-server/src/metrics/labels.rs#L67), [connect.rs#L506](../crates/qbit-prism-server/src/ledger/connect.rs#L506), [connect.rs#L549](../crates/qbit-prism-server/src/ledger/connect.rs#L549) | +| PrismDatabaseAdvisoryLockFailuresCritical | Order and settlement advisory-lock failures >5% of that lock's waits over five minutes, from at least ten observed waits for that lock; #271 measured 15, 31 and 63 steady ORDER_LOCK waiters at one, two and four frontends and no native failure rate, and the default five-second PRISM_DATABASE_LOCK_TIMEOUT_MS turns a sustained wait into a failure; provisional, measure in #291/#447. | [registry.rs#L74](../crates/qbit-prism-server/src/metrics/registry.rs#L74), [labels.rs#L66](../crates/qbit-prism-server/src/metrics/labels.rs#L66), [labels.rs#L67](../crates/qbit-prism-server/src/metrics/labels.rs#L67), [connect.rs#L506](../crates/qbit-prism-server/src/ledger/connect.rs#L506), [connect.rs#L549](../crates/qbit-prism-server/src/ledger/connect.rs#L549) | +| PrismRuntimeLagHigh | Two-second native blocked-poll budget; five-minute evidence window and three-minute dwell replace the legacy lease-wake episode, provisional, measure in #291. | [registry.rs#L74](../crates/qbit-prism-server/src/metrics/registry.rs#L74), [registry.rs#L75](../crates/qbit-prism-server/src/metrics/registry.rs#L75), [registry.rs#L77](../crates/qbit-prism-server/src/metrics/registry.rs#L77), [labels.rs#L19](../crates/qbit-prism-server/src/metrics/labels.rs#L19), [metrics.rs#L39](../crates/qbit-prism-server/src/metrics.rs#L39) | +| PrismResidentMemoryHigh | Four GiB is an unqualified initial RSS bound; set against actual container headroom. Provisional, measure in #291. | [registry.rs#L70](../crates/qbit-prism-server/src/metrics/registry.rs#L70), [registry.rs#L73](../crates/qbit-prism-server/src/metrics/registry.rs#L73), [registry.rs#L78](../crates/qbit-prism-server/src/metrics/registry.rs#L78), [registry.rs#L79](../crates/qbit-prism-server/src/metrics/registry.rs#L79), [labels.rs#L18](../crates/qbit-prism-server/src/metrics/labels.rs#L18), [metrics.rs#L39](../crates/qbit-prism-server/src/metrics.rs#L39) | +| PrismProcessMetricsUnavailable | Process collector expires after 30 seconds or on failure; unsupported platforms are unknown. | [registry.rs#L70](../crates/qbit-prism-server/src/metrics/registry.rs#L70), [registry.rs#L73](../crates/qbit-prism-server/src/metrics/registry.rs#L73), [labels.rs#L18](../crates/qbit-prism-server/src/metrics/labels.rs#L18), [metrics.rs#L39](../crates/qbit-prism-server/src/metrics.rs#L39) | +| PrismPublicStalenessRefusalsIncreasing | Any refusal is an actual endpoint staleness-budget breach; five-minute window is provisional, measure in #291. | [public_service.rs#L309](../crates/qbit-prism-server/src/api/public_service.rs#L309), [public_service.rs#L320](../crates/qbit-prism-server/src/api/public_service.rs#L320) | +| PrismPublicLedgerUnready | Readiness already rejects missing/failed probes and ages >= max(3 * probe interval, 15s). | [public_service.rs#L309](../crates/qbit-prism-server/src/api/public_service.rs#L309), [public_service.rs#L320](../crates/qbit-prism-server/src/api/public_service.rs#L320) | +| PrismPublic5xxSustained | Any sustained 5xx response, preserving deployed failure intent; five-minute window is provisional, measure in #291. | [public_service.rs#L309](../crates/qbit-prism-server/src/api/public_service.rs#L309), [public_service.rs#L320](../crates/qbit-prism-server/src/api/public_service.rs#L320) | +| PrismAcceptedRevisionWorkPending | Provisional warning at more than one second, with no additional dwell: the #275 budget is a target, not a measured production percentile; measure in #291. Missing series on a configured target and failed scrapes also warn. Unknown tracking (a -1 age, the tracking-unknown gauge, or a missing unknown gauge) is owned by PrismAcceptedRevisionWorkTrackingUnknown, which dwells so a single failed refresh does not warn (#493); a known pending age beside unknown tracking still warns here and remains eligible for the critical rule. An offer the node accepted but has not reported active is an unlanded acceptance, excluded from this age and owned by PrismAcceptedBlockUnlanded (#493). | [registry.rs](../crates/qbit-prism-server/src/metrics/registry.rs), [landing.rs](../crates/qbit-prism-server/src/metrics/landing.rs), [stratum.rs](../crates/qbit-prism-server/src/stratum.rs) | +| PrismAcceptedRevisionWorkTrackingUnknown | Unknown or incomplete local landing tracking (#493 point 7): the pending age is -1 (no known wait remains but tracking is unknown), the tracking-unknown gauge is one beside any known age, or the unknown gauge is missing on a configured target during a mixed-version rollout. Two-minute dwell: a failed template refresh marks tracking unknown only until the next successful refresh, one poll interval later (two seconds by default), so a single transient refresh error cannot span the eight consecutive 15-second scrapes this dwell requires, while a lost or cancelled settlement COMMIT reply, an unsettled orphan verdict, bounded-tracking saturation and lost ordering history persist until terminal evidence or restart and still warn. No-data and evaluation errors alert so a missing measurement stays visible after the same dwell. Non-paging: the critical rule owns known stalls and gates out unknown ages itself; the pending warning owns the no-dwell budget signal. Measure the dwell in #291. | [registry.rs](../crates/qbit-prism-server/src/metrics/registry.rs), [landing.rs](../crates/qbit-prism-server/src/metrics/landing.rs), [stratum.rs](../crates/qbit-prism-server/src/stratum.rs) | +| PrismAcceptedRevisionWorkPendingCritical | Provisional critical floor of 307 seconds comes from the #413 incident duration, not an early-warning result. Gates (#493): the age must be known (not -1), the target scraped, and at least one authorized miner on the reporting frontend, because a frontend with no connected miners has no mining.notify write that can close its wait and would otherwise page after every pool block. The pending gauge is a live overlay rendered on every scrape, so it is not gated on the metrics snapshot: a database-pool stall that also stalls the health publisher must still page. The miner count is a snapshot family and may be a few refresh intervals old while the snapshot is stale; a false page on a frontend whose miners left inside that window is accepted in exchange for never missing a real stall. If every miner leaves a frontend during a stall, this page resolves and nothing else pages for that frontend: the coverage rules cannot fire with zero authorized clients and the connected-clients rule is a cluster-wide non-paging warning; the pending warning keeps reporting the age until a miner reconnects and receives work. One-minute dwell: the gauge is monotonic during a stall, so the dwell only absorbs a brief gate gap and stays well under the five-minute hold that delayed #413. No-data and evaluation errors are OK: the warning rules own missing or unknown measurements. The floor and dwell are literals in this specification, not deployment variables. A lost race on the offering frontend never counts here (#493 point 1): an offer the node accepted without an active-chain observation is an unlanded acceptance, excluded from the known pending age until an active-chain observation lands it or its orphan is proven, and reported by accepted_block_unlanded_seconds and PrismAcceptedBlockUnlanded instead. Measure thresholds and scrape/evaluation delay in #291; no operational deployment is claimed. | [registry.rs](../crates/qbit-prism-server/src/metrics/registry.rs), [landing.rs](../crates/qbit-prism-server/src/metrics/landing.rs), [stratum.rs](../crates/qbit-prism-server/src/stratum.rs) | +| PrismRevisionWorkBuildTimeouts | Any existing build deadline hit while locally observed accepted work awaits delivery; five-minute increase window is provisional, measure in #291. Successful builds and cancellation do not increment the counter. Only a known open wait counts (#493): while this frontend's tracking is unknown and no known wait remains, a deadline hit is an ordinary job-delivery failure, not a revision-work failure, and the tracking-unknown warning reports the unknown state instead. | [registry.rs](../crates/qbit-prism-server/src/metrics/registry.rs), [landing.rs](../crates/qbit-prism-server/src/metrics/landing.rs), [stratum.rs](../crates/qbit-prism-server/src/stratum.rs) | +| PrismAcceptedBlockUnlanded | An offer the node accepted (a null submitblock reply) that this frontend has not observed on the node's active chain for 30 minutes (#493 point 1). A lost tip race is settled as a proven orphan once a different block has PRISM_CANDIDATE_ORPHAN_CONFIRMATIONS (default 6) confirmations at its height: about six minutes at the 60-second target spacing plus the reconciliation retry backoff, so a routine lost race leaves this gauge well inside the threshold. The reconciliation backoff is ten seconds times the row's attempt count, capped at an hour, and the count includes the pre-offer attempts, so the cumulative wait to the k-th retry is about 5k(k+1) seconds: the sixth competitor confirmation at 60-second spacing is reached around the eighth retry, about nine minutes, well inside 30 minutes, while a row that needed many pre-offer attempts starts its backoff higher and reaches 30 minutes in a few retries. Such an acceptance is never part of the known pending age, so the revision-work critical cannot page on it; a winning offer leaves this gauge at its first active-chain observation and joins the known age on its original acceptance clock. Past 30 minutes the row is not being reconciled, the node is not advancing, or the orphan verdict keeps failing: operator work, warned, never paged; the cluster-wide candidate oldest-pending rules see the same reconciliation row. -1 (none tracked while identity tracking is saturated) is owned by the tracking-unknown warning and does not fire here. Three-minute dwell like the other warnings; no-data is OK because the pending warning already reports a missing landing family. Provisional; measure in #291. | [registry.rs](../crates/qbit-prism-server/src/metrics/registry.rs), [landing.rs](../crates/qbit-prism-server/src/metrics/landing.rs), [stratum.rs](../crates/qbit-prism-server/src/stratum.rs) | + + diff --git a/docs/prism-alert-pr-draft.md b/docs/prism-alert-pr-draft.md new file mode 100644 index 000000000..b3ac55391 --- /dev/null +++ b/docs/prism-alert-pr-draft.md @@ -0,0 +1,46 @@ +# Local PR draft — coordinator review required + +Target: `Qbit-Org/qbit-mining-bootstrap`, base `3.x.x`, head +`djh58/prism-b7-alert-migration`. No push or PR creation has occurred. + +Proposed title: `docs(prism): generate native metric inventory and migrate deployed alerts` + +Proposed body: + +The native runtime no longer emits the Python series consumed by deployed PRISM +alerts. This change generates the single 52-family inventory from the registry, +checks running `run` and `public-api` role scrapes in both directions, and maps +all 78 deployed definitions plus all 46 historical metric names. The deployment +artifact migrates 22 alert definitions, preserves 34 external definitions, and +records why 22 have no replacement; it renders 25 native/public rules and two +required primary/dedicated-standby rules alongside the external rules. + +The rules preserve unknown, stale and failed observations, retain the deployed +network-wide connected-client floor, and compare durable WAL progress over time +so healthy idle replication does not fire on NULL last-reported latency. The +review-only qbit-tools diff preserves external definitions and explicitly deletes +retired UIDs and native UIDs disabled by their deployment gates. No runtime +producer, VERSION, CHANGELOG, qbit-tools checkout or deployment was changed. + +Validation: all 12 ungated observability tests passed; promtool parsed all 27 +expressions and passed 38 synthetic scenarios plus nine scenarios using actual +SQL observations from disposable PostgreSQL 16 primary/async-standby clusters. +The patch applies cleanly and renders successfully across ten Jinja gate +combinations with exact external-definition preservation and correct UID deletion +sets. Formatting, whitespace and both generated-artifact checks passed. See +[verification](prism-alert-verification.md) and [review ledger](prism-alert-review-ledger.json) +for the executed/not-executed record and local review findings. + +Independent Fable and thermo reviews remain pending coordinator routing; GitHub +CI and bot lanes have not run because the branch is unpushed. Production exporter +integration, Grafana provisioning, the 400k regression/500k headroom load gates +and #291 qualification remain outstanding. Thresholds without native measurement +are explicitly provisional; first-offer and advisory-lock rules remain deferred. +The dedicated HA topology stays one primary plus one asynchronous standby, with +ACKs waiting for local durability only. Applying the deployment diff requires +separate operator review and the native cutover. + +Closes #279. + +Before publication, replace pending-lane statements with the actual independent +results and confirm the final reviewed SHA; do not infer approval from this draft. diff --git a/docs/prism-alert-review-ledger.json b/docs/prism-alert-review-ledger.json new file mode 100644 index 000000000..61cd85c64 --- /dev/null +++ b/docs/prism-alert-review-ledger.json @@ -0,0 +1,343 @@ +{ + "reviewed_base": "5355b69462663e9b9e666589147f74e7aeb268dc", + "reviewed_head": "3f4caa9896c1b8b412d6e8d75df0dd1f6f1bc278", + "stakes": "normal: alert docs/specs/test tooling, no runtime or financial-state mutation", + "lanes": { + "code-review-cli": { + "status": "completed", + "note": "local Codex CLI review command at medium effort; installed code-review skill unavailable", + "model": "gpt-6-astra", + "reasoning_effort": "medium", + "result": "No actionable regressions; generators checked, Cargo/PromQL not rerun by this lane" + }, + "codex-adv": { + "status": "completed", + "model": "gpt-5.6-sol", + "result": "Two major findings: idle standby false alert and network connected-client threshold changed to per-instance" + }, + "fable-adv": { + "status": "requested from coordinator, not executed locally" + }, + "thermo": { + "status": "pending coordinator routing", + "note": "Read installed /Users/dan/.claude/skills/thermo-nuclear-code-quality-review/SKILL.md after coordinator supplied exact path; no independent thermo lane executed locally" + }, + "github-ci-bots": { + "status": "not executed; no push or PR authorized" + }, + "codex-adv-corrections": { + "status": "completed", + "model": "gpt-5.6-sol", + "requested_diff": "3f4caa9..8e5e8d8", + "result": "Two major findings: disabled native UID survives (also found locally and corrected as d1d6c81); synchronous topology can look healthy (subsequently corrected as 2bf64f2)", + "final_async_correction_independent_review": "not executed; coordinator and Fable/thermo review remain pending" + } + }, + "findings": [ + { + "id": "coordinator-d3-idle-and-stalled-replay", + "source": "coordinator", + "claim": "NULL replay_lag is valid on healthy idle standby, and last-reported replay latency cannot establish current stalled-backlog age", + "file": "docs/prism-postgres-exporter-queries.yaml:9", + "disposition": "open", + "evidence": "PostgreSQL 16.14 disposable async primary/standby: healthy idle replay_lag=NULL with equal LSNs; paused replay 104-byte backlog with latency 0.00014 to 8.007 seconds; recovered equal LSNs retain latency 9.21 seconds", + "sources": [ + "coordinator", + "codex-adv" + ], + "local_status": "implemented-and-verified", + "local_commit": "8e5e8d8c8020de95863cb7a6e723093f8e177260", + "remaining": "Coordinator and independent review, then authorized publication; disposition remains open while unpushed", + "severity": "major" + }, + { + "id": "codex-adv-connected-client-network-floor", + "source": "codex-adv", + "claim": "Ten-client network-wide floor became a per-instance floor; two healthy instances with six each incorrectly alert twice", + "file": "docs/prism-native-alert-rules.json:55", + "disposition": "open", + "evidence": "Frozen deployed rule sums all qbit_prism_connected_clients for the network; native replacement compares each connections sample to 10", + "local_status": "implemented-and-verified", + "local_commit": "b85a13740f12c439596fe1243d48e940db700a3b", + "remaining": "Coordinator and independent review, then authorized publication; disposition remains open while unpushed", + "severity": "major" + }, + { + "id": "local-review-disabled-native-uid-survives", + "source": "local review", + "claim": "Disabled native gate omits a reused legacy UID but fixed deletion list leaves its persisted legacy rule active", + "file": "scripts/generate_prism_alerts.py:161", + "disposition": "open", + "local_status": "implemented-and-verified", + "local_commit": "d1d6c818a4dbe68731d0e8604b9eeee91b06deb2", + "evidence": "Added rendered UID-set regression failed on all-disabled settings before correction; all ten gate combinations pass after conditional native UID deletion; external definitions identical", + "remaining": "Coordinator and independent review, then authorized publication; disposition remains open while unpushed", + "sources": [ + "local review", + "codex-adv-corrections" + ], + "severity": "major" + }, + { + "id": "codex-corrective-sync-topology-healthy", + "source": "codex-adv-corrections", + "claim": "A streaming caught-up synchronous standby satisfies the D3 health predicate although the required topology is asynchronous", + "file": "docs/prism-postgres-exporter-queries.yaml:13 (reviewed 8e5e8d8)", + "disposition": "open", + "local_status": "implemented-and-verified", + "local_commit": "2bf64f2fbf6861320e05e62e07a9c741abd5953e", + "evidence": "Four synthetic PromQL cases returned healthy zero before correction; all pass with explicit fresh async gauge validation, and real PostgreSQL tests cover selected synchronous standby, synchronous requirement on another standby, and restored async configuration without hiding known WAL positions", + "remaining": "Final correction needs coordinator/independent review and authorized publication", + "severity": "major" + } + ], + "local_head": "2bf64f2fbf6861320e05e62e07a9c741abd5953e", + "publication": "not pushed; no PR; coordinator final review required", + "sql_evidence": { + "postgres": "postgres (PostgreSQL) 16.14 (Homebrew)", + "exporter": "not started; status inputs controlled by harness", + "wrong_role_query": "failed as expected", + "cases": [ + { + "name": "SQL healthy idle NULL lag has caught-up WAL proof", + "sample": 6, + "expect": [ + 0, + 0 + ], + "alerts": [ + false, + false + ], + "observation": { + "elapsed_seconds": 8.047, + "metrics": { + "application_name": "prism_standby_1", + "count": 1, + "async": 1, + "primary_flush_lsn_hi": 0, + "primary_flush_lsn_lo": 50429760, + "replay_lsn_hi": 0, + "replay_lsn_lo": 50429760, + "replay_lag": 0 + }, + "raw_replay_lag": null + } + }, + { + "name": "SQL recent write remains inside five-second allowance", + "sample": 10, + "expect": [ + 0, + 0 + ], + "alerts": [ + false, + false + ], + "observation": { + "elapsed_seconds": 12.126, + "metrics": { + "application_name": "prism_standby_1", + "count": 1, + "async": 1, + "primary_flush_lsn_hi": 0, + "primary_flush_lsn_lo": 50429864, + "replay_lsn_hi": 0, + "replay_lsn_lo": 50429760, + "replay_lag": 3.004356 + }, + "raw_replay_lag": 3.004356 + } + }, + { + "name": "SQL paused replay exceeds allowance but not dwell", + "sample": 13, + "expect": [ + 1, + 0 + ], + "alerts": [ + false, + false + ], + "observation": { + "elapsed_seconds": 15.142, + "metrics": { + "application_name": "prism_standby_1", + "count": 1, + "async": 1, + "primary_flush_lsn_hi": 0, + "primary_flush_lsn_lo": 50429864, + "replay_lsn_hi": 0, + "replay_lsn_lo": 50429760, + "replay_lag": 6.00784 + }, + "raw_replay_lag": 6.00784 + } + }, + { + "name": "SQL paused replay passes one-minute dwell", + "sample": 75, + "expect": [ + 1, + 0 + ], + "alerts": [ + true, + false + ], + "observation": { + "elapsed_seconds": 77.371, + "metrics": { + "application_name": "prism_standby_1", + "count": 1, + "async": 1, + "primary_flush_lsn_hi": 0, + "primary_flush_lsn_lo": 50429920, + "replay_lsn_hi": 0, + "replay_lsn_lo": 50429760, + "replay_lag": 67.459052 + }, + "raw_replay_lag": 67.459052 + } + }, + { + "name": "SQL resumed replay is caught up despite old reported latency", + "sample": 76, + "expect": [ + 0, + 0 + ], + "alerts": [ + false, + false + ], + "observation": { + "elapsed_seconds": 78.396, + "metrics": { + "application_name": "prism_standby_1", + "count": 1, + "async": 1, + "primary_flush_lsn_hi": 0, + "primary_flush_lsn_lo": 50429920, + "replay_lsn_hi": 0, + "replay_lsn_lo": 50429920, + "replay_lag": 0 + }, + "raw_replay_lag": 62.843997 + } + }, + { + "name": "SQL synchronous standby violates required asynchronous topology", + "sample": 78, + "expect": [ + 1, + 0 + ], + "alerts": null, + "observation": { + "elapsed_seconds": 80.498, + "metrics": { + "application_name": "prism_standby_1", + "count": 1, + "async": 0, + "primary_flush_lsn_hi": 0, + "primary_flush_lsn_lo": 50429920, + "replay_lsn_hi": 0, + "replay_lsn_lo": 50429920, + "replay_lag": 0 + }, + "raw_replay_lag": null + } + }, + { + "name": "SQL primary waiting for another standby also violates topology", + "sample": 80, + "expect": [ + 1, + 0 + ], + "alerts": null, + "observation": { + "elapsed_seconds": 82.595, + "metrics": { + "application_name": "prism_standby_1", + "count": 1, + "async": 0, + "primary_flush_lsn_hi": 0, + "primary_flush_lsn_lo": 50429920, + "replay_lsn_hi": 0, + "replay_lsn_lo": 50429920, + "replay_lag": 0 + }, + "raw_replay_lag": null + } + }, + { + "name": "SQL asynchronous topology restored", + "sample": 81, + "expect": [ + 0, + 0 + ], + "alerts": [ + false, + false + ], + "observation": { + "elapsed_seconds": 83.654, + "metrics": { + "application_name": "prism_standby_1", + "count": 1, + "async": 1, + "primary_flush_lsn_hi": 0, + "primary_flush_lsn_lo": 50429920, + "replay_lsn_hi": 0, + "replay_lsn_lo": 50429920, + "replay_lag": 0 + }, + "raw_replay_lag": null + } + }, + { + "name": "SQL disconnected standby is unknown and unavailable", + "sample": 83, + "expect": [ + 1, + 1 + ], + "alerts": null, + "observation": { + "elapsed_seconds": 85.833, + "metrics": { + "application_name": "prism_standby_1", + "count": 0, + "async": -1, + "primary_flush_lsn_hi": 0, + "primary_flush_lsn_lo": 50429920, + "replay_lsn_hi": -1, + "replay_lsn_lo": -1, + "replay_lag": -1 + }, + "raw_replay_lag": null + } + } + ] + }, + "checks": { + "observability": "executed-and-passed: 12 tests; none skipped", + "promql_synthetic": "executed-and-passed: 38 scenarios, 86 predicates, 12 alert states", + "promql_sql": "executed-and-passed: 9 scenarios, 18 predicates, 12 alert states", + "provisioning": "executed-and-passed: 10 gate combinations; patch check and application; external equality; conditional deletion", + "format_and_diff": "executed-and-passed", + "400k_regression_500k_headroom": "not executed; qualification targets preserved", + "live_exporter_grafana": "not executed; integration remains #281/#291" + }, + "verification_head": "2bf64f2fbf6861320e05e62e07a9c741abd5953e", + "review_outputs": { + "code-review-cli": "No actionable regressions were identified in the alert specifications, deployment-patch generator, or inventory tests. Both generators passed their consistency checks, including comparison against the frozen deployment snapshot; Cargo and PromQL tests were not rerun.", + "codex-adv": "FINDING 1: A healthy, caught-up PostgreSQL standby is reported as unknown and triggers the replay-lag alert during normal idle periods.\nFILE: docs/prism-postgres-exporter-queries.yaml:9\nSCENARIO: `pg_stat_replication` contains `prism_standby_1`, but PostgreSQL sets `replay_lag=NULL` after the caught-up standby has been idle \u2192 SQL converts it to `-1`; `docs/prism-postgres-alert-rules.json:6` treats `< 0` as failure and fires `PrismHAStandbyReplayLagHigh` after one minute despite `count=1` and `pg_up=1`.\nSEVERITY: major\n\nFINDING 2: The connected-client migration changes the threshold from a network-wide total to a per-instance minimum.\nFILE: docs/prism-native-alert-rules.json:55\nSCENARIO: Two fresh `qbit-prism` instances report six connections each \u2192 the deployed expression removed at `docs/prism-alert-rules-qbit-tools.patch:212` sums to 12 and remains healthy, while the replacement at patch line 224 evaluates `6 < 10` independently for both instances and emits two alerts after three minutes. All committed scenarios use a single instance, so this regression passes.\nSEVERITY: major", + "codex-adv-corrections": "FINDING 1: Gate-independent deletion generation leaves disabled, previously provisioned alerts active in Grafana.\nFILE: scripts/generate_prism_alerts.py:113\nSCENARIO: Grafana already contains `qbit-prism-connected-clients`; render with `qbit_monitoring_stack_prism_alert_connected_clients_enabled=false` \u2192 line 141 removes the rule from `groups`, but its UID remains in `kept_uids`, so no `deleteRules` entry is emitted and Grafana retains the active legacy alert. The deployment test still passes because line 91 compares against the same gate-independent UID set.\nSEVERITY: major\n\nFINDING 2: D3 reports a synchronous standby as healthy despite requiring an asynchronous topology.\nFILE: docs/prism-postgres-exporter-queries.yaml:13\nSCENARIO: Set `synchronous_standby_names='FIRST 1 (prism_standby_1)'` while the single standby is streaming and caught up \u2192 `count=1`, `known=true`, positions match, and both D3 alerts return healthy because `sync_state` is never queried. PRISM acknowledgements now wait for that standby and can stall when it disappears, contradicting the required asynchronous money-path behavior without any alert.\nSEVERITY: major" + } +} diff --git a/docs/prism-alert-rules-qbit-tools.patch b/docs/prism-alert-rules-qbit-tools.patch new file mode 100644 index 000000000..7174da944 --- /dev/null +++ b/docs/prism-alert-rules-qbit-tools.patch @@ -0,0 +1,1890 @@ +--- a/ansible/roles/qbit_monitoring_stack/templates/grafana-alert-rules-prism.yml.j2 ++++ b/ansible/roles/qbit_monitoring_stack/templates/grafana-alert-rules-prism.yml.j2 +@@ -1,114 +1,10 @@ +-{# +- Deploy-window tolerance (2026-07-26): a routine mining-pool deploy restarts +- the stratum coordinator that serves the docker-proxied PRISM metrics +- endpoint, and the 2026-07-26 mainnet #98 rollout showed the resulting +- scrape gap (~8s port window + 15s scrape interval + 1m evaluation lag) +- paging DatasourceNoData from any coordinator-scrape rule that treats +- NoData as alertable. Every rule below therefore declares an explicit +- no_data_state/exec_err_state. Coordinator-scrape threshold rules keep +- NoData/ExecErr = OK and rely on the dedicated coverage nets — one +- Alerting-on-NoData net per scrape surface (PrismMetricsNoData and +- PrismMetricsEndpointDown for the coordinator endpoint, +- PrismPublicLedgerUnready for the extracted read tier, +- PrismStratumHealthNoData for the sidecar, PrismBackupMetricsNoData for +- the backup collector) — so a planned restart resolves to OK without ever +- hiding a real outage, and those nets plus the coordinator threshold rules +- carry for: >= 3m to outlast a planned restart. The stratum-health sidecar +- probe rules are the deploy-independent real-outage net and deliberately +- keep their tighter 1m/2m windows (probe patience during lane restarts was +- fixed separately via #648). Severity=info rules stay routed to Slack +- (grouped under alert_class=prism-runtime): after the 2026-07-26 threshold +- fixes they only fire on sustained ledger failures or post-restart replay, +- both worth surfacing, and only critical page=true rules escalate to +- GoAlert. The backup staleness criticals detect an operation series that +- never succeeded through a collector-gated guard (collector series +- present, operation series absent) rather than NoData=Alerting, so a +- transient node-exporter scrape gap stays behind PrismBackupMetricsNoData's +- five-minute grace instead of bursting immediate criticals (2026-07-26). +- Where a guard shares an expression with an aggregated bool branch (WAL, +- audit, coverage), every branch is label_replace-tagged with a distinct +- label value inside the max(): a bare "max(...) or guard" lets a healthy +- 0-valued max() result shadow the guard in the PromQL union -- both +- collapse to the empty label signature and "or" keeps the left side -- so +- the missing-series critical would silently never fire (2026-07-29). +- Absence-guard tolerance (2026-07-31): at 15:17:53Z a backup-metrics +- collector run overlapped the qbit-prism-audit-backup run (93.9GB in ~31s, +- every 10 minutes), one probe hit its --timeout 5, and the collector +- published its textfile without the full/diff operation series. The guards +- did exactly what they were built to do and PrismBackupFullStaleCritical, +- PrismBackupDifferentialStaleCritical and PrismBackupCollectorStale all +- fired at 15:18Z against healthy backups. The immediate-guard design above +- stays: a truly-missing series must never hide behind NoData grace, so the +- fix is a short pending window rather than NoData=Alerting or a weaker +- guard. Every rule whose expression carries a collector-gated "unless" +- branch now uses qbit_monitoring_stack_prism_backup_alert_absence_guard_for +- (3m) instead of the shared immediate "for", so a single ~60s collector +- omission window can no longer page while a series that never succeeded +- still alerts within minutes. The age-based halves of those expressions are +- continuous, so +3m is immaterial against 10-day and 48-hour thresholds. +- PrismBackupCollectorStale deliberately keeps the immediate "for": it is +- the intended early, non-paging Slack signal that the collector itself is +- degraded. +- Chronic sidecar failure visibility (2026-07-31): PrismStratumUnavailable +- is a min()-over-lanes outage net gated on +- qbit_monitoring_stack_prism_alert_stratum_down_for, so a lane that fails a +- probe and recovers inside that window never notifies: 877 prism and 273 +- prism-highdiff sidecar check failures in a single day surfaced as only +- three pages. The sidecar +- now exports a cumulative qbit_stratum_health_check_failures_total counter +- (its qbit_stratum_health_check_failures gauge only holds the consecutive +- streak and resets on every recovery, so chronic flapping is invisible to +- increase()), and PrismStratumCheckFailureRateHigh reports a sustained +- failure rate as a non-paging severity=warning Slack signal. The paging +- contract is unchanged. +- Overload contract (2026-08-24): bootstrap #191/#188 makes stale complete +- metric snapshots scrapeable as HTTP 200, exports the exact unresolved-parent +- depth/cap, and specifies the landing, backlog, semantic-coverage, refresh, +- and snapshot alerts rendered below. Bootstrap #199 adds the direct +- accepted-block-to-preview histogram and its healthy/degraded result split. +- They stay behind an explicit pin gate: a pre-#184 build returns 503 for stale +- snapshots and cannot support the body rules honestly. The post-#199 rules +- are safe across a fast rollback to #196 because absent histogram series are +- explicitly OK. PrismMetricsEndpointDown is this deployment's PrismScrapeDown +- equivalent and fires after the existing deploy-safe three-minute window. +- Paging audit (2026-09-09, qbit-tools #1086): the coverage watchdogs +- (PrismMetricsEndpointDown, PrismStratumHealthNoData, +- PrismPublicLedgerUnready, PrismMetricsSnapshotUnavailable) fire on +- monitoring blindness rather than miner impact -- 61 of +- PrismMetricsEndpointDown's 63 fires in 45 days duplicated the Slack-only +- PrismMetricsNoData and 31 duplicated PrismStratumUnavailable -- so they +- are severity=warning Slack signals with no page label. The miner-facing +- outage pages through PrismStratumUnavailable and host loss through +- QbitNodeExporterDown. +- Writer-lease headroom (2026-08-31): bootstrap #214 exports its validated +- policy terms beside server-proven age and the monitor's worst wake delay. +- The rules below compare live values to the process's own policy instead of +- copying absolute timing defaults. They remain separately pin-gated and use +- NoData=OK so they stay inert during the coordinator restart and a rollback +- to the pre-#214 image. +- Coordinator restart evidence (2026-09-08, #1056): on 2026-09-06 the union +- coordinator went from 32 to 57 restart-policy restarts in under four hours +- while every rule above stayed green, because each exit plus its ~55s cold +- rebuild fit inside the three-minute coordinator-scrape NoData windows. The +- PrismCoordinator* rules read a host-side docker-inspect textfile from the +- PRISM host's node exporter instead of the coordinator scrape, so the +- evidence outlives the process. Coverage is one non-paging Slack rule with +- NoData=Alerting and a ten-minute pending window, so collector-versus-rule +- deploy order cannot page. The restart rules use increase() on a counter the +- collector seeds at 0 and publishes only from persisted state: a lost state +- file re-seeds downward to 0, which increase() treats as a reset, while +- extrapolation can only round an integer count up, so ">= N" never +- under-counts (negligible at the 15s scrape cadence). Deploys and lane +- restarts are manual Docker starts +- that never increment RestartCount, so planned work needs no maintenance +- flag; drills that crash the coordinator on purpose use a Grafana silence. ++{# Native 3.x.x cutover draft, bootstrap #279. Apply only with the native image. ++ Specification SHA-256: 9436a7246611e8a13adcdd2c242ea6d4bf2c6aae82db55913745395eefd11d3d ++ Generated from the reviewed a007a142 snapshot; backup, sidecar and lifecycle rules retain their producers. ++ Native numeric bounds marked provisional must be measured in #291 before production cutover. ++ The D3 group requires PRIMARY postgres_exporter query metrics; see bootstrap docs/prism-alert-migration.md. ++ Retired UIDs are explicitly deleted: omitting a provisioned rule would leave it active in Grafana. + #} +-{% set prism_connected_clients_threshold = 10 %} +-{% set prism_connected_clients_for = "3m" %} +-{% set prism_selector = 'job="qbit-prism",network="' ~ qbit_monitoring_stack_network ~ '"' %} +-{% set prism_public_read_selector = 'job="qbit-prism-public-read",network="' ~ qbit_monitoring_stack_network ~ '"' %} +-{% set prism_share_append_failures_sum = "sum(increase(qbit_prism_share_append_failures_total{" ~ prism_selector ~ "}[" ~ qbit_monitoring_stack_prism_alert_share_append_failures_window ~ "]))" %} +-{% set prism_share_append_attempts_sum = prism_share_append_failures_sum ~ " + sum(increase(qbit_prism_accepted_shares_total{" ~ prism_selector ~ "}[" ~ qbit_monitoring_stack_prism_alert_share_append_failures_window ~ "]))" %} + {% set prism_stratum_target_selector = 'job="qbit-stratum-health-sidecar",network="' ~ qbit_monitoring_stack_network ~ '",service="qbit-mining-pool",prism_capable="true"' %} + {% set prism_stratum_selector = 'job="qbit-stratum-health-sidecar",network="' ~ qbit_monitoring_stack_network ~ '",service="qbit-mining-pool",prism_capable="true",check=~"prism(-highdiff)?"' %} + {% set prism_backup_selector = 'job="qbit-node-exporter",network="' ~ qbit_monitoring_stack_network ~ '",service="' ~ qbit_monitoring_stack_prism_backup_alert_service ~ '",node="' ~ qbit_monitoring_stack_prism_backup_alert_node ~ '"' %} +@@ -118,53 +14,892 @@ + {% set prism_backup_database_growth_gib_per_day = "(sum(clamp_min(last_over_time(qbit_prism_backup_last_success_bytes{" ~ prism_backup_selector ~ ",operation=\"full\"}[1h]) - last_over_time(qbit_prism_backup_last_success_bytes{" ~ prism_backup_selector ~ ",operation=\"full\"}[1h] offset " ~ qbit_monitoring_stack_prism_backup_alert_growth_window ~ "), 0)) / " ~ qbit_monitoring_stack_prism_backup_alert_growth_window_days ~ " / 1073741824)" %} + {% set prism_backup_audit_growth_gib_per_day = "(sum(clamp_min(deriv(qbit_prism_backup_last_success_bytes{" ~ prism_backup_selector ~ ",operation=\"audit\"}[" ~ qbit_monitoring_stack_prism_backup_alert_growth_window ~ "]), 0)) * 86400 / 1073741824)" %} + {% set prism_backup_wal_growth_gib_per_day = "(sum(clamp_min(rate(qbit_prism_backup_wal_generated_bytes_total{" ~ prism_backup_selector ~ "}[" ~ qbit_monitoring_stack_prism_backup_alert_growth_window ~ "]), 0)) * 86400 / 1073741824)" %} +-{% set prism_share_writer_alert_rules = [ ++{# D3 deployment-provided queries.yaml stanza; install on the PRIMARY exporter: ++# Deployment-provided D3 query extension on the PRIMARY, not the public replica. ++# Configure through the deployment's supported SQL-query exporter mechanism; ++# stock collector versions differ. Verify the query metrics and exporter status ++# in #281/#291. Scrape this primary target every 1s; the rule compares positions ++# with the primary's durable WAL position observed 5s earlier. ++# pg_up, pg_exporter_last_scrape_error and Prometheus up must share this target. ++pg_stat_replication: ++ query: | ++ WITH primary_wal AS ( ++ SELECT pg_wal_lsn_diff(pg_current_wal_flush_lsn(), '0/0') AS durable_position ++ ), standby AS ( ++ SELECT COUNT(*) AS count, ++ BOOL_AND(sync_state = 'async') AS is_async, ++ COUNT(*) = 1 AND BOOL_AND(state = 'streaming' AND replay_lsn IS NOT NULL) AS known, ++ MIN(pg_wal_lsn_diff(replay_lsn, '0/0')) AS replay_position, ++ MAX(EXTRACT(EPOCH FROM replay_lag)) AS reported_lag ++ FROM pg_stat_replication ++ WHERE application_name = 'prism_standby_1' ++ ) ++ SELECT 'prism_standby_1' AS application_name, ++ count::double precision, ++ CASE WHEN count <> 1 THEN -1 ++ WHEN BTRIM(current_setting('synchronous_standby_names')) <> '' THEN 0 ++ ELSE COALESCE(is_async::int, -1) END::double precision AS async, ++ FLOOR(durable_position / 4294967296)::double precision AS primary_flush_lsn_hi, ++ MOD(durable_position, 4294967296)::double precision AS primary_flush_lsn_lo, ++ CASE WHEN known THEN FLOOR(replay_position / 4294967296) ELSE -1 END::double precision AS replay_lsn_hi, ++ CASE WHEN known THEN MOD(replay_position, 4294967296) ELSE -1 END::double precision AS replay_lsn_lo, ++ CASE WHEN NOT COALESCE(known, false) THEN -1 ++ WHEN replay_position >= durable_position THEN 0 ++ ELSE COALESCE(reported_lag, -1) END::double precision AS replay_lag ++ FROM primary_wal CROSS JOIN standby ++ metrics: ++ - application_name: ++ usage: LABEL ++ description: Dedicated failover standby application name ++ - count: ++ usage: GAUGE ++ description: Matching replication rows; zero if disconnected ++ - replay_lag: ++ usage: GAUGE ++ description: Diagnostic last-reported latency; zero only with caught-up WAL proof, -1 if unknown ++ - async: ++ usage: GAUGE ++ description: One async standby with no primary synchronous requirement is 1, invalid topology is 0, unknown or ambiguous is -1 ++ - primary_flush_lsn_hi: ++ usage: GAUGE ++ description: High 32 bits of primary durable WAL position, exact in Prometheus float64 ++ - primary_flush_lsn_lo: ++ usage: GAUGE ++ description: Low 32 bits of primary durable WAL position, exact in Prometheus float64 ++ - replay_lsn_hi: ++ usage: GAUGE ++ description: High 32 bits of known streaming standby replay position, or -1 ++ - replay_lsn_lo: ++ usage: GAUGE ++ description: Low 32 bits of known streaming standby replay position, or -1 ++ ++#} ++{% set prism_native_rules = [ + { + "uid": "qbit-prism-metrics-nodata", + "title": "PrismMetricsNoData", +- "expr": "count(qbit_prism_share_append_queue_depth{" ~ prism_selector ~ "})", +- "evaluator": "lt", +- "threshold": 1, +- "for": qbit_monitoring_stack_prism_alert_metrics_nodata_for, +- "severity": "warning", +- "service": "qbit-mining-pool", +- "labels": {"alert_class": "prism-runtime"}, +- "summary": "PRISM share-writer metrics are missing", +- "description": "Prometheus/Grafana has no PRISM queue-depth series for this network, so the durability alerts cannot evaluate.", ++ "expr": "((count by (job, instance, network) (qbit_prism_health_state{job=\"qbit-prism\",network=\"" ~ qbit_monitoring_stack_network ~ "\"})) or on(job, instance, network) (0 * up{job=\"qbit-prism\",network=\"" ~ qbit_monitoring_stack_network ~ "\"})) == bool 0", ++ "condition_type": "threshold", ++ "evaluator": "gt", ++ "threshold": 0, ++ "for": "3m", ++ "severity": "warning", ++ "service": "qbit-mining-pool", ++ "labels": { ++ "alert_class": "prism-runtime" ++ }, ++ "summary": "PrismMetricsNoData", ++ "description": "A configured coordinator target has no native health family; restore scrape visibility. Preserves the deployed three-minute scrape coverage grace.", + "no_data_state": "Alerting", + "exec_err_state": "Alerting" + }, + { + "uid": "qbit-prism-metrics-endpoint-down", + "title": "PrismMetricsEndpointDown", +- "expr": "min(up{" ~ prism_selector ~ "})", +- "evaluator": "lt", +- "threshold": 1, +- "for": qbit_monitoring_stack_prism_alert_endpoint_down_for, +- "severity": "warning", +- "service": "qbit-mining-pool", +- "labels": {"alert_class": "prism-runtime"}, +- "summary": "PRISM metrics endpoint is down", +- "description": "Prometheus cannot scrape the Tailnet-only PRISM metrics proxy. Restore the proxy/coordinator before relying on share-writer durability alerts. This is a scrape-visibility signal, not miner impact: it is Slack-only since the 2026-09-09 paging audit (qbit-tools #1086), and the miner-facing outage it usually accompanies is paged by PrismStratumUnavailable.", ++ "expr": "up{job=\"qbit-prism\",network=\"" ~ qbit_monitoring_stack_network ~ "\"} == bool 0", ++ "condition_type": "threshold", ++ "evaluator": "gt", ++ "threshold": 0, ++ "for": "3m", ++ "severity": "warning", ++ "service": "qbit-mining-pool", ++ "labels": { ++ "alert_class": "prism-runtime" ++ }, ++ "summary": "PrismMetricsEndpointDown", ++ "description": "Coordinator /metrics cannot be scraped; miner-impact paging remains with the sidecar. Preserves the deployed three-minute scrape outage grace.", + "no_data_state": "Alerting", + "exec_err_state": "Alerting" + }, + { + "uid": "qbit-prism-connected-clients", + "title": "PRISM Connected Clients", +- "connected_clients_rule": true, +- "expr": "sum(qbit_prism_connected_clients{" ~ prism_selector ~ "}) or (0 * sum(up{" ~ prism_selector ~ "} == 1))", +- "evaluator": "lt", +- "threshold": prism_connected_clients_threshold, +- "for": prism_connected_clients_for, +- "severity": "warning", +- "service": "qbit-mining-pool", +- "labels": {"alert_class": "prism-runtime", "job": "qbit-prism"}, +- "summary": "QBIT PRISM connected clients below threshold", +- "description": "The number of connected QBIT PRISM clients has remained below 10 for at least three minutes. Investigate the qbit-prism service, network connectivity, and exporter health. If the scrape succeeds but the connected-clients series itself is missing (exporter regression or metric rename), the rule evaluates as zero and still fires; only a full scrape gap during a planned coordinator restart resolves to OK, with endpoint-outage coverage owned by PrismMetricsNoData and PrismMetricsEndpointDown.", +- "no_data_state": "OK", +- "exec_err_state": "OK" +- }, ++ "expr": "(sum(qbit_prism_connections{job=\"qbit-prism\",network=\"" ~ qbit_monitoring_stack_network ~ "\"} and on(job, instance, network) (qbit_prism_metrics_snapshot_available{job=\"qbit-prism\",network=\"" ~ qbit_monitoring_stack_network ~ "\"} == 1) and on(job, instance, network) (qbit_prism_metrics_snapshot_stale{job=\"qbit-prism\",network=\"" ~ qbit_monitoring_stack_network ~ "\"} == 0) and on(job, instance, network) (up{job=\"qbit-prism\",network=\"" ~ qbit_monitoring_stack_network ~ "\"} == 1)) < bool " ~ (qbit_monitoring_stack_prism_alert_connected_clients_threshold | default(10)) ~ ") and on() (count(up{job=\"qbit-prism\",network=\"" ~ qbit_monitoring_stack_network ~ "\"}) == count(qbit_prism_connections{job=\"qbit-prism\",network=\"" ~ qbit_monitoring_stack_network ~ "\"} and on(job, instance, network) (qbit_prism_metrics_snapshot_available{job=\"qbit-prism\",network=\"" ~ qbit_monitoring_stack_network ~ "\"} == 1) and on(job, instance, network) (qbit_prism_metrics_snapshot_stale{job=\"qbit-prism\",network=\"" ~ qbit_monitoring_stack_network ~ "\"} == 0) and on(job, instance, network) (up{job=\"qbit-prism\",network=\"" ~ qbit_monitoring_stack_network ~ "\"} == 1)))", ++ "condition_type": "threshold", ++ "evaluator": "gt", ++ "threshold": 0, ++ "for": (qbit_monitoring_stack_prism_alert_connected_clients_for | default('3m')), ++ "severity": "warning", ++ "service": "qbit-mining-pool", ++ "labels": { ++ "alert_class": "prism-runtime" ++ }, ++ "summary": "PRISM Connected Clients", ++ "description": "Authorized and unauthenticated connections are included in the network total; six connections on each of two coordinators is twelve and remains healthy. Missing, failed or stale targets suppress the incomplete sum and remain covered by the scrape/snapshot alerts. Preserves the deployed network-wide floor: sum all local native connection counts, below ten for three minutes; evaluate only with fresh measurements for every configured target.", ++ "no_data_state": "OK", ++ "exec_err_state": "OK", ++ "connected_clients_rule": true ++ }, ++ { ++ "uid": "qbit-prism-metrics-snapshot-stale", ++ "title": "PrismMetricsSnapshotStale", ++ "expr": "qbit_prism_metrics_snapshot_stale{job=\"qbit-prism\",network=\"" ~ qbit_monitoring_stack_network ~ "\"} > bool 0", ++ "condition_type": "threshold", ++ "evaluator": "gt", ++ "threshold": 0, ++ "for": "3m", ++ "severity": "warning", ++ "service": "qbit-mining-pool", ++ "labels": { ++ "alert_class": "prism-runtime" ++ }, ++ "summary": "PrismMetricsSnapshotStale", ++ "description": "Snapshot stale means cached operational gauges cannot establish health. #277 computes staleness at scrape time from max(3 * health refresh, 15s); no copied age constant.", ++ "no_data_state": "Alerting", ++ "exec_err_state": "Alerting" ++ }, ++ { ++ "uid": "qbit-prism-metrics-snapshot-unavailable", ++ "title": "PrismMetricsSnapshotUnavailable", ++ "expr": "qbit_prism_metrics_snapshot_available{job=\"qbit-prism\",network=\"" ~ qbit_monitoring_stack_network ~ "\"} == bool 0", ++ "condition_type": "threshold", ++ "evaluator": "gt", ++ "threshold": 0, ++ "for": "3m", ++ "severity": "warning", ++ "service": "qbit-mining-pool", ++ "labels": { ++ "alert_class": "prism-runtime" ++ }, ++ "summary": "PrismMetricsSnapshotUnavailable", ++ "description": "Snapshot unavailable means cached operational gauges cannot establish health. #277 computes staleness at scrape time from max(3 * health refresh, 15s); no copied age constant.", ++ "no_data_state": "Alerting", ++ "exec_err_state": "Alerting" ++ }, ++ { ++ "uid": "qbit-prism-metrics-snapshot-age", ++ "title": "PrismMetricsSnapshotAgeHigh", ++ "expr": "(qbit_prism_metrics_snapshot_age_seconds{job=\"qbit-prism\",network=\"" ~ qbit_monitoring_stack_network ~ "\"} >= bool 0) * on(job, instance, network) (qbit_prism_metrics_snapshot_stale{job=\"qbit-prism\",network=\"" ~ qbit_monitoring_stack_network ~ "\"} == bool 1)", ++ "condition_type": "threshold", ++ "evaluator": "gt", ++ "threshold": 0, ++ "for": "3m", ++ "severity": "warning", ++ "service": "qbit-mining-pool", ++ "labels": { ++ "alert_class": "prism-runtime" ++ }, ++ "summary": "PrismMetricsSnapshotAgeHigh", ++ "description": "Snapshot agehigh means cached operational gauges cannot establish health. #277 computes staleness at scrape time from max(3 * health refresh, 15s); no copied age constant.", ++ "no_data_state": "Alerting", ++ "exec_err_state": "Alerting" ++ }, ++ { ++ "uid": "qbit-prism-share-ack-p99", ++ "title": "PrismShareAckP99High", ++ "expr": "((histogram_quantile(0.99, sum by (job, instance, network, le) (rate(qbit_prism_share_ack_seconds_bucket{job=\"qbit-prism\",network=\"" ~ qbit_monitoring_stack_network ~ "\"}[5m]))) > bool 1) and on(job, instance, network) (sum by (job, instance, network) (increase(qbit_prism_share_ack_seconds_count{job=\"qbit-prism\",network=\"" ~ qbit_monitoring_stack_network ~ "\"}[5m])) >= 100)) and on(job, instance, network) ((qbit_prism_metrics_snapshot_available{job=\"qbit-prism\",network=\"" ~ qbit_monitoring_stack_network ~ "\"} == 1) and on(job, instance, network) (qbit_prism_metrics_snapshot_stale{job=\"qbit-prism\",network=\"" ~ qbit_monitoring_stack_network ~ "\"} == 0))", ++ "condition_type": "threshold", ++ "evaluator": "gt", ++ "threshold": 0, ++ "for": "3m", ++ "severity": "warning", ++ "service": "qbit-mining-pool", ++ "labels": { ++ "alert_class": "prism-runtime" ++ }, ++ "summary": "PrismShareAckP99High", ++ "description": "Frame receipt to completed response write, both results combined; durable credit can precede a failed write. Replaces writer-queue pressure with miner-visible latency. One-second histogram boundary; at least 100 completed ACKs in five minutes. Provisional, measure in #291.", ++ "no_data_state": "OK", ++ "exec_err_state": "OK" ++ }, ++ { ++ "uid": "qbit-prism-landing-ack-p99-slow", ++ "title": "PrismLandingAckP99Slow", ++ "expr": "(qbit_prism_share_ack_slow_landing_windows{job=\"qbit-prism\",network=\"" ~ qbit_monitoring_stack_network ~ "\",p99_above_seconds=\"2\"} >= bool 3) and on(job, instance, network) ((qbit_prism_metrics_snapshot_available{job=\"qbit-prism\",network=\"" ~ qbit_monitoring_stack_network ~ "\"} == 1) and on(job, instance, network) (qbit_prism_metrics_snapshot_stale{job=\"qbit-prism\",network=\"" ~ qbit_monitoring_stack_network ~ "\"} == 0))", ++ "condition_type": "threshold", ++ "evaluator": "gt", ++ "threshold": 0, ++ "for": "1m", ++ "severity": "ticket", ++ "service": "qbit-mining-pool", ++ "labels": { ++ "alert_class": "prism-runtime" ++ }, ++ "summary": "PrismLandingAckP99Slow", ++ "description": "Share acknowledgement p99 exceeded 2 seconds in three consecutive landing windows on this frontend. A landing window opens when this frontend first observes a pool block's acceptance (its own submitblock acceptance, or a peer frontend's block on the active chain, seen before it is confirmed or already confirmed at the tip) and admits every share submission that arrives within the next 30 seconds; settlement, which holds ORDER_LOCK while every share append waits for it, runs inside that window (#602). Each window is judged 30 seconds after it stops admitting submissions: its nearest-rank acknowledgement p99 is over a bound exactly when more than one percent of its acknowledgements took longer, counted exactly by the process rather than read from buckets. The gauge holds its value until the next verdict, so the alert stays active until a landing window at or under the bound; a window with no acknowledgement leaves it unchanged. Per frontend. -1 (no verdict since start) never fires; no-data and evaluation errors are OK because the snapshot rules own missing measurements. #602 R1 asks for a landing-window p99 at or under 500 ms and no acknowledgement over 2 s; 2.x.x shows a block-minute p99 of about 4-5 s against about 0.2 s otherwise. Three consecutive windows keep one slow landing (a cold cache, a one-off settlement) from ticketing. Provisional, measure in #291 and #602 step 3.", ++ "no_data_state": "OK", ++ "exec_err_state": "OK" ++ }, ++ { ++ "uid": "qbit-prism-landing-ack-p99-high", ++ "title": "PrismLandingAckP99High", ++ "expr": "(qbit_prism_share_ack_slow_landing_windows{job=\"qbit-prism\",network=\"" ~ qbit_monitoring_stack_network ~ "\",p99_above_seconds=\"10\"} >= bool 1) and on(job, instance, network) ((qbit_prism_metrics_snapshot_available{job=\"qbit-prism\",network=\"" ~ qbit_monitoring_stack_network ~ "\"} == 1) and on(job, instance, network) (qbit_prism_metrics_snapshot_stale{job=\"qbit-prism\",network=\"" ~ qbit_monitoring_stack_network ~ "\"} == 0))", ++ "condition_type": "threshold", ++ "evaluator": "gt", ++ "threshold": 0, ++ "for": "1m", ++ "severity": "warning", ++ "service": "qbit-mining-pool", ++ "labels": { ++ "alert_class": "prism-runtime" ++ }, ++ "summary": "PrismLandingAckP99High", ++ "description": "Share acknowledgement p99 exceeded 10 seconds in the latest landing window on this frontend. A landing window opens when this frontend first observes a pool block's acceptance (its own submitblock acceptance, or a peer frontend's block on the active chain, seen before it is confirmed or already confirmed at the tip) and admits every share submission that arrives within the next 30 seconds; settlement, which holds ORDER_LOCK while every share append waits for it, runs inside that window (#602). Each window is judged 30 seconds after it stops admitting submissions: its nearest-rank acknowledgement p99 is over a bound exactly when more than one percent of its acknowledgements took longer, counted exactly by the process rather than read from buckets. The gauge holds its value until the next verdict, so the alert stays active until a landing window at or under the bound; a window with no acknowledgement leaves it unchanged. Per frontend. -1 (no verdict since start) never fires; no-data and evaluation errors are OK because the snapshot rules own missing measurements. 10 s is two thirds of the default share-commit timeout (PRISM_SHARE_COMMIT_TIMEOUT_SECONDS, 15 s), past which an append is answered ledger-outcome-unknown; rental proxies and some firmware resubmit or reconnect well before it. One window is enough. Provisional, measure in #291 and #602 step 3.", ++ "no_data_state": "OK", ++ "exec_err_state": "OK" ++ }, ++ { ++ "uid": "qbit-prism-reject-ratio", ++ "title": "PrismRejectRatioByReasonHigh", ++ "expr": "(((sum by (job, instance, network, reason_id) (increase(qbit_prism_rejections_total{job=\"qbit-prism\",network=\"" ~ qbit_monitoring_stack_network ~ "\"}[5m])) / on(job, instance, network) group_left() (sum by (job, instance, network) (increase(qbit_prism_accepted_shares_total{job=\"qbit-prism\",network=\"" ~ qbit_monitoring_stack_network ~ "\"}[5m])) + sum by (job, instance, network) (increase(qbit_prism_rejections_total{job=\"qbit-prism\",network=\"" ~ qbit_monitoring_stack_network ~ "\"}[5m])))) > bool 0.05) and on(job, instance, network) ((sum by (job, instance, network) (increase(qbit_prism_accepted_shares_total{job=\"qbit-prism\",network=\"" ~ qbit_monitoring_stack_network ~ "\"}[5m])) + sum by (job, instance, network) (increase(qbit_prism_rejections_total{job=\"qbit-prism\",network=\"" ~ qbit_monitoring_stack_network ~ "\"}[5m]))) >= 100))", ++ "condition_type": "threshold", ++ "evaluator": "gt", ++ "threshold": 0, ++ "for": "3m", ++ "severity": "warning", ++ "service": "qbit-mining-pool", ++ "labels": { ++ "alert_class": "prism-runtime" ++ }, ++ "summary": "PrismRejectRatioByReasonHigh", ++ "description": "Preserves reason_id on each alert; rejected includes failed response writes. Diagnose ledger-confirmation-failed, ledger-outcome-unknown, backend-rpc-unavailable (the node), backend-database-unavailable (the ledger database, #581), stale-job, duplicate-share and low-difficulty separately. Five percent per canonical reason over five minutes, with at least 100 acceptance/rejection decisions. Not gated on a fresh metrics snapshot: qbit_prism_rejections_total and qbit_prism_accepted_shares_total are rendered at scrape time since #581, so a database outage that stalls the health publisher does not freeze them. Provisional, measure in #291.", ++ "no_data_state": "OK", ++ "exec_err_state": "OK" ++ }, ++ { ++ "uid": "qbit-prism-pool-resubmission-reject-ratio", ++ "title": "PrismPoolResubmissionRejectRatioHigh", ++ "expr": "((sum by (network) (increase(qbit_prism_rejections_total{job=\"qbit-prism\",network=\"" ~ qbit_monitoring_stack_network ~ "\",reason_id=~\"duplicate-share|unknown-job|stale-job\"}[5m]) and on(job, instance, network) ((qbit_prism_metrics_snapshot_available{job=\"qbit-prism\",network=\"" ~ qbit_monitoring_stack_network ~ "\"} == 1) and on(job, instance, network) (qbit_prism_metrics_snapshot_stale{job=\"qbit-prism\",network=\"" ~ qbit_monitoring_stack_network ~ "\"} == 0))) / sum by (network) ((sum by (job, instance, network) (increase(qbit_prism_accepted_shares_total{job=\"qbit-prism\",network=\"" ~ qbit_monitoring_stack_network ~ "\"}[5m])) + sum by (job, instance, network) (increase(qbit_prism_rejections_total{job=\"qbit-prism\",network=\"" ~ qbit_monitoring_stack_network ~ "\"}[5m]))) and on(job, instance, network) ((qbit_prism_metrics_snapshot_available{job=\"qbit-prism\",network=\"" ~ qbit_monitoring_stack_network ~ "\"} == 1) and on(job, instance, network) (qbit_prism_metrics_snapshot_stale{job=\"qbit-prism\",network=\"" ~ qbit_monitoring_stack_network ~ "\"} == 0)))) > bool 0.05) and on(network) (sum by (network) ((sum by (job, instance, network) (increase(qbit_prism_accepted_shares_total{job=\"qbit-prism\",network=\"" ~ qbit_monitoring_stack_network ~ "\"}[5m])) + sum by (job, instance, network) (increase(qbit_prism_rejections_total{job=\"qbit-prism\",network=\"" ~ qbit_monitoring_stack_network ~ "\"}[5m]))) and on(job, instance, network) ((qbit_prism_metrics_snapshot_available{job=\"qbit-prism\",network=\"" ~ qbit_monitoring_stack_network ~ "\"} == 1) and on(job, instance, network) (qbit_prism_metrics_snapshot_stale{job=\"qbit-prism\",network=\"" ~ qbit_monitoring_stack_network ~ "\"} == 0))) >= 500)", ++ "condition_type": "threshold", ++ "evaluator": "gt", ++ "threshold": 0, ++ "for": "5m", ++ "severity": "warning", ++ "service": "qbit-mining-pool", ++ "labels": { ++ "alert_class": "prism-runtime" ++ }, ++ "summary": "PrismPoolResubmissionRejectRatioHigh", ++ "description": "Pool-wide share rejections for duplicate-share, unknown-job and stale-job, summed over every frontend with a fresh metrics snapshot, as a share of all acceptance and rejection decisions. These are the rejections slow acknowledgements and slow work delivery produce: a rental proxy resubmits a share it got no answer for and the copy is a duplicate (#602), and a miner still working an old job after a stall submits unknown or stale jobs (#598). Unlike PrismRejectRatioByReasonHigh it judges the pool, not one frontend, and the three reasons together, so rejections spread thinly across frontends and reasons still warn. Frontends with a missing or stale snapshot are left out of both sides of the ratio; no-data and evaluation errors are OK. Five percent of the pool's acceptance/rejection decisions over five minutes, from at least 500 decisions, for five minutes; the per-frontend, per-reason rule uses the same five percent. Provisional, measure in #291 and against 2.x.x's post-block reject spikes.", ++ "no_data_state": "OK", ++ "exec_err_state": "OK" ++ }, ++ { ++ "uid": "qbit-prism-share-append-failures", ++ "title": "PrismShareAppendFailures", ++ "expr": "(sum by (job, instance, network) (increase(qbit_prism_rejections_total{job=\"qbit-prism\",network=\"" ~ qbit_monitoring_stack_network ~ "\",reason_id=~\"ledger-confirmation-failed|ledger-outcome-unknown\"}[5m])) > bool 0)", ++ "condition_type": "threshold", ++ "evaluator": "gt", ++ "threshold": 0, ++ "for": "3m", ++ "severity": "warning", ++ "service": "qbit-mining-pool", ++ "labels": { ++ "alert_class": "prism-runtime" ++ }, ++ "summary": "PrismShareAppendFailures", ++ "description": "At least one share was rejected in the five-minute window because its ledger confirmation failed or outcome unknown (ledger-confirmation-failed or ledger-outcome-unknown). This preserves the low-volume total-write-failure signal from the deployed rule while retaining the native reason_id. Not gated on a fresh metrics snapshot (#581): qbit_prism_rejections_total is rendered at scrape time, so the rule holds through a database outage that stalls the health publisher instead of flapping with the snapshot's freshness.", ++ "no_data_state": "OK", ++ "exec_err_state": "OK" ++ }, ++ { ++ "uid": "qbit-prism-time-to-usable-work", ++ "title": "PrismTimeToUsableWorkHigh", ++ "expr": "((qbit_prism_stratum_oldest_pending_initial_job_seconds{job=\"qbit-prism\",network=\"" ~ qbit_monitoring_stack_network ~ "\"} > bool 15) and on(job, instance, network) (qbit_prism_stratum_pending_initial_jobs{job=\"qbit-prism\",network=\"" ~ qbit_monitoring_stack_network ~ "\"} > 0)) and on(job, instance, network) ((qbit_prism_metrics_snapshot_available{job=\"qbit-prism\",network=\"" ~ qbit_monitoring_stack_network ~ "\"} == 1) and on(job, instance, network) (qbit_prism_metrics_snapshot_stale{job=\"qbit-prism\",network=\"" ~ qbit_monitoring_stack_network ~ "\"} == 0))", ++ "condition_type": "threshold", ++ "evaluator": "gt", ++ "threshold": 0, ++ "for": "3m", ++ "severity": "warning", ++ "service": "qbit-mining-pool", ++ "labels": { ++ "alert_class": "prism-runtime" ++ }, ++ "summary": "PrismTimeToUsableWorkHigh", ++ "description": "Reports miners awaiting first usable work, including reauthorization; -1 is unknown and snapshot coverage is required. Authorization to successfully written first job; 15 seconds is a provisional bound, measure in #291.", ++ "no_data_state": "OK", ++ "exec_err_state": "OK" ++ }, ++ { ++ "uid": "qbit-prism-current-work-gap", ++ "title": "PrismCurrentWorkGapHigh", ++ "expr": "((qbit_prism_stratum_current_tip_coverage_gap_seconds{job=\"qbit-prism\",network=\"" ~ qbit_monitoring_stack_network ~ "\"} > bool 15) and on(job, instance, network) (qbit_prism_authorized_clients{job=\"qbit-prism\",network=\"" ~ qbit_monitoring_stack_network ~ "\"} > 0)) and on(job, instance, network) ((qbit_prism_metrics_snapshot_available{job=\"qbit-prism\",network=\"" ~ qbit_monitoring_stack_network ~ "\"} == 1) and on(job, instance, network) (qbit_prism_metrics_snapshot_stale{job=\"qbit-prism\",network=\"" ~ qbit_monitoring_stack_network ~ "\"} == 0))", ++ "condition_type": "threshold", ++ "evaluator": "gt", ++ "threshold": 0, ++ "for": "3m", ++ "severity": "warning", ++ "service": "qbit-mining-pool", ++ "labels": { ++ "alert_class": "prism-runtime" ++ }, ++ "summary": "PrismCurrentWorkGapHigh", ++ "description": "Current-generation usable work remains missing; replaces the operational intent of refresh deadline rules, not Python refresh task state. Native timer advances only while semantic coverage is strictly below 0.95; 15-second duration is provisional, measure in #291.", ++ "no_data_state": "OK", ++ "exec_err_state": "OK" ++ }, ++ { ++ "uid": "qbit-prism-semantic-work-coverage", ++ "title": "PrismSemanticWorkCoverageLoss", ++ "expr": "((qbit_prism_stratum_semantic_current_work_ratio{job=\"qbit-prism\",network=\"" ~ qbit_monitoring_stack_network ~ "\"} < bool " ~ (qbit_monitoring_stack_prism_alert_semantic_coverage_warning_ratio | default(0.95)) ~ ") and on(job, instance, network) (qbit_prism_stratum_semantic_current_work_ratio{job=\"qbit-prism\",network=\"" ~ qbit_monitoring_stack_network ~ "\"} >= 0) and on(job, instance, network) (qbit_prism_authorized_clients{job=\"qbit-prism\",network=\"" ~ qbit_monitoring_stack_network ~ "\"} > 0)) and on(job, instance, network) ((qbit_prism_metrics_snapshot_available{job=\"qbit-prism\",network=\"" ~ qbit_monitoring_stack_network ~ "\"} == 1) and on(job, instance, network) (qbit_prism_metrics_snapshot_stale{job=\"qbit-prism\",network=\"" ~ qbit_monitoring_stack_network ~ "\"} == 0))", ++ "condition_type": "threshold", ++ "evaluator": "gt", ++ "threshold": 0, ++ "for": (qbit_monitoring_stack_prism_alert_semantic_coverage_warning_for | default('3m')), ++ "severity": "warning", ++ "service": "qbit-mining-pool", ++ "labels": { ++ "alert_class": "prism-runtime" ++ }, ++ "summary": "PrismSemanticWorkCoverageLoss", ++ "description": "Unknown ratio -1 is excluded from coverage and reported separately; no-client ratio one is healthy. Native semantic generation coverage; warning matches the strict 95-percent timer boundary. Critical 50 percent and three-minute dwell: Provisional, measure in #291.", ++ "no_data_state": "OK", ++ "exec_err_state": "OK" ++ }, ++ { ++ "uid": "qbit-prism-semantic-coverage-critical", ++ "title": "PrismSemanticWorkCoverageLossCritical", ++ "expr": "((qbit_prism_stratum_semantic_current_work_ratio{job=\"qbit-prism\",network=\"" ~ qbit_monitoring_stack_network ~ "\"} < bool " ~ (qbit_monitoring_stack_prism_alert_semantic_coverage_critical_ratio | default(0.5)) ~ ") and on(job, instance, network) (qbit_prism_stratum_semantic_current_work_ratio{job=\"qbit-prism\",network=\"" ~ qbit_monitoring_stack_network ~ "\"} >= 0) and on(job, instance, network) (qbit_prism_authorized_clients{job=\"qbit-prism\",network=\"" ~ qbit_monitoring_stack_network ~ "\"} > 0)) and on(job, instance, network) ((qbit_prism_metrics_snapshot_available{job=\"qbit-prism\",network=\"" ~ qbit_monitoring_stack_network ~ "\"} == 1) and on(job, instance, network) (qbit_prism_metrics_snapshot_stale{job=\"qbit-prism\",network=\"" ~ qbit_monitoring_stack_network ~ "\"} == 0))", ++ "condition_type": "threshold", ++ "evaluator": "gt", ++ "threshold": 0, ++ "for": (qbit_monitoring_stack_prism_alert_semantic_coverage_critical_for | default('3m')), ++ "severity": "critical", ++ "service": "qbit-mining-pool", ++ "labels": { ++ "alert_class": "prism-runtime", ++ "page": "true" ++ }, ++ "summary": "PrismSemanticWorkCoverageLossCritical", ++ "description": "Unknown ratio -1 is excluded from coverage and reported separately; no-client ratio one is healthy. Native semantic generation coverage; warning matches the strict 95-percent timer boundary. Critical 50 percent and three-minute dwell: Provisional, measure in #291.", ++ "no_data_state": "OK", ++ "exec_err_state": "OK" ++ }, ++ { ++ "uid": "qbit-prism-work-refresh-stalled-critical", ++ "title": "PrismWorkRefreshStalledCritical", ++ "expr": "(qbit_prism_work_refresh_stalled_seconds{job=\"qbit-prism\",network=\"" ~ qbit_monitoring_stack_network ~ "\"} >= bool 120) and on(job, instance, network) ((qbit_prism_metrics_snapshot_available{job=\"qbit-prism\",network=\"" ~ qbit_monitoring_stack_network ~ "\"} == 1) and on(job, instance, network) (qbit_prism_metrics_snapshot_stale{job=\"qbit-prism\",network=\"" ~ qbit_monitoring_stack_network ~ "\"} == 0))", ++ "condition_type": "threshold", ++ "evaluator": "gt", ++ "threshold": 0, ++ "for": "1m", ++ "severity": "critical", ++ "service": "qbit-mining-pool", ++ "labels": { ++ "alert_class": "prism-runtime", ++ "page": "true" ++ }, ++ "summary": "PrismWorkRefreshStalledCritical", ++ "description": "This frontend's template refresh has not succeeded for 120 seconds, so it has neither published nor revalidated work: job, share and block admission refuse once readiness is older than the health timeout, and while refreshes fail only a node poll that finds the published tip with its template within PRISM_TEMPLATE_MAX_AGE_SECONDS (120 s by default) renews it (#622), so by then plus the health timeout every miner on it is stalled whatever the cause; qbit_prism_tip_poll_age_seconds shows which side of that a paged frontend is on. That covers #525 on either settlement path, where a pool without a fee cannot settle sub-floor dust and every build fails with PayoutExceedsCandidateBalance, whether the dust meets a new tip or a same-parent rebuild; it also covers a refresh that fails before its tip observation or hangs. Semantic coverage cannot see this: it compares sessions with the latest published generation, which stays put while every refresh fails. A healthy refresh succeeds every poll interval (2 s by default, and within the health timeout for the frontend to be ready at all); 120 seconds is the default replacement-build lease (PRISM_TEMPLATE_REFRESH_FAILURE_EXIT_SECONDS). One-minute dwell: the gauge is monotonic during a stall. Not gated on connected miners: a frontend without working refreshes is stalled for every miner that connects. A node still syncing or short of peers also pages here, since it builds no work. -1 (not yet published) never fires; no-data and evaluation errors are OK because the snapshot rules own missing measurements, and a draining process stops its snapshot. Provisional, measure in #291.", ++ "no_data_state": "OK", ++ "exec_err_state": "OK" ++ }, ++ { ++ "uid": "qbit-prism-work-metrics-unavailable", ++ "title": "PrismWorkMetricsUnavailable", ++ "expr": "((qbit_prism_stratum_semantic_current_work_ratio{job=\"qbit-prism\",network=\"" ~ qbit_monitoring_stack_network ~ "\"} < bool 0) + on(job, instance, network) (qbit_prism_stratum_oldest_pending_initial_job_seconds{job=\"qbit-prism\",network=\"" ~ qbit_monitoring_stack_network ~ "\"} < bool 0)) and on(job, instance, network) ((qbit_prism_metrics_snapshot_available{job=\"qbit-prism\",network=\"" ~ qbit_monitoring_stack_network ~ "\"} == 1) and on(job, instance, network) (qbit_prism_metrics_snapshot_stale{job=\"qbit-prism\",network=\"" ~ qbit_monitoring_stack_network ~ "\"} == 0))", ++ "condition_type": "threshold", ++ "evaluator": "gt", ++ "threshold": 0, ++ "for": "3m", ++ "severity": "warning", ++ "service": "qbit-mining-pool", ++ "labels": { ++ "alert_class": "prism-runtime" ++ }, ++ "summary": "PrismWorkMetricsUnavailable", ++ "description": "A published snapshot has not observed work delivery; inspect the publisher and delivery sampler. Unknown sentinels must never masquerade as healthy work coverage.", ++ "no_data_state": "OK", ++ "exec_err_state": "OK" ++ }, ++ { ++ "uid": "qbit-prism-block-candidate-backlog", ++ "title": "PrismBlockCandidateBacklog", ++ "expr": "(qbit_prism_block_candidates_pending{job=\"qbit-prism\",network=\"" ~ qbit_monitoring_stack_network ~ "\"} >= bool 2) and on(job, instance, network) ((qbit_prism_metrics_snapshot_available{job=\"qbit-prism\",network=\"" ~ qbit_monitoring_stack_network ~ "\"} == 1) and on(job, instance, network) (qbit_prism_metrics_snapshot_stale{job=\"qbit-prism\",network=\"" ~ qbit_monitoring_stack_network ~ "\"} == 0)) and on(job, instance, network) (qbit_prism_collector_available{job=\"qbit-prism\",network=\"" ~ qbit_monitoring_stack_network ~ "\",collector=\"database\"} == 1)", ++ "condition_type": "threshold", ++ "evaluator": "gt", ++ "threshold": 0, ++ "for": "3m", ++ "severity": "warning", ++ "service": "qbit-mining-pool", ++ "labels": { ++ "alert_class": "prism-runtime" ++ }, ++ "summary": "PrismBlockCandidateBacklog", ++ "description": "Per-scrape-target view of cluster-wide backlog; do not sum repeated database snapshots across coordinators. Unknown -1 belongs to the collector-unavailable alert. Native pending outbox rows include claimed/retry-delayed candidates; count bounds 2/5 and age bounds 15/60 seconds. Provisional, measure in #291.", ++ "no_data_state": "OK", ++ "exec_err_state": "OK" ++ }, ++ { ++ "uid": "qbit-prism-candidate-backlog-critical", ++ "title": "PrismBlockCandidateBacklogCritical", ++ "expr": "(qbit_prism_block_candidates_pending{job=\"qbit-prism\",network=\"" ~ qbit_monitoring_stack_network ~ "\"} >= bool 5) and on(job, instance, network) ((qbit_prism_metrics_snapshot_available{job=\"qbit-prism\",network=\"" ~ qbit_monitoring_stack_network ~ "\"} == 1) and on(job, instance, network) (qbit_prism_metrics_snapshot_stale{job=\"qbit-prism\",network=\"" ~ qbit_monitoring_stack_network ~ "\"} == 0)) and on(job, instance, network) (qbit_prism_collector_available{job=\"qbit-prism\",network=\"" ~ qbit_monitoring_stack_network ~ "\",collector=\"database\"} == 1)", ++ "condition_type": "threshold", ++ "evaluator": "gt", ++ "threshold": 0, ++ "for": "3m", ++ "severity": "warning", ++ "service": "qbit-mining-pool", ++ "labels": { ++ "alert_class": "prism-runtime" ++ }, ++ "summary": "PrismBlockCandidateBacklogCritical", ++ "description": "Per-scrape-target view of cluster-wide backlog; do not sum repeated database snapshots across coordinators. Unknown -1 belongs to the collector-unavailable alert. Native pending outbox rows include claimed/retry-delayed candidates; count bounds 2/5 and age bounds 15/60 seconds. Provisional, measure in #291.", ++ "no_data_state": "OK", ++ "exec_err_state": "OK" ++ }, ++ { ++ "uid": "qbit-prism-block-candidate-oldest", ++ "title": "PrismBlockCandidateOldestPending", ++ "expr": "(qbit_prism_block_candidate_oldest_pending_seconds{job=\"qbit-prism\",network=\"" ~ qbit_monitoring_stack_network ~ "\"} >= bool " ~ (qbit_monitoring_stack_prism_alert_block_candidate_age_warning_seconds | default(15)) ~ ") and on(job, instance, network) ((qbit_prism_metrics_snapshot_available{job=\"qbit-prism\",network=\"" ~ qbit_monitoring_stack_network ~ "\"} == 1) and on(job, instance, network) (qbit_prism_metrics_snapshot_stale{job=\"qbit-prism\",network=\"" ~ qbit_monitoring_stack_network ~ "\"} == 0)) and on(job, instance, network) (qbit_prism_collector_available{job=\"qbit-prism\",network=\"" ~ qbit_monitoring_stack_network ~ "\",collector=\"database\"} == 1)", ++ "condition_type": "threshold", ++ "evaluator": "gt", ++ "threshold": 0, ++ "for": (qbit_monitoring_stack_prism_alert_block_candidate_age_for | default('3m')), ++ "severity": "warning", ++ "service": "qbit-mining-pool", ++ "labels": { ++ "alert_class": "prism-runtime" ++ }, ++ "summary": "PrismBlockCandidateOldestPending", ++ "description": "Per-scrape-target view of cluster-wide backlog; do not sum repeated database snapshots across coordinators. Unknown -1 belongs to the collector-unavailable alert. Native pending outbox rows include claimed/retry-delayed candidates; count bounds 2/5 and age bounds 15/60 seconds. Provisional, measure in #291.", ++ "no_data_state": "OK", ++ "exec_err_state": "OK" ++ }, ++ { ++ "uid": "qbit-prism-candidate-oldest-critical", ++ "title": "PrismBlockCandidateOldestPendingCritical", ++ "expr": "((qbit_prism_block_candidate_oldest_unacknowledged_seconds{job=\"qbit-prism\",network=\"" ~ qbit_monitoring_stack_network ~ "\"} or on(job, instance, network) qbit_prism_block_candidate_oldest_pending_seconds{job=\"qbit-prism\",network=\"" ~ qbit_monitoring_stack_network ~ "\"}) >= bool " ~ (qbit_monitoring_stack_prism_alert_block_candidate_age_critical_seconds | default(60)) ~ ") and on(job, instance, network) ((qbit_prism_metrics_snapshot_available{job=\"qbit-prism\",network=\"" ~ qbit_monitoring_stack_network ~ "\"} == 1) and on(job, instance, network) (qbit_prism_metrics_snapshot_stale{job=\"qbit-prism\",network=\"" ~ qbit_monitoring_stack_network ~ "\"} == 0)) and on(job, instance, network) (qbit_prism_collector_available{job=\"qbit-prism\",network=\"" ~ qbit_monitoring_stack_network ~ "\",collector=\"database\"} == 1)", ++ "condition_type": "threshold", ++ "evaluator": "gt", ++ "threshold": 0, ++ "for": (qbit_monitoring_stack_prism_alert_block_candidate_age_for | default('3m')), ++ "severity": "critical", ++ "service": "qbit-mining-pool", ++ "labels": { ++ "alert_class": "prism-runtime", ++ "page": "true" ++ }, ++ "summary": "PrismBlockCandidateOldestPendingCritical", ++ "description": "Per-scrape-target view of cluster-wide backlog; do not sum repeated database snapshots across coordinators. Unknown -1 belongs to the collector-unavailable alert. Pages only on a found block the node has not accepted (#493): pending and offer-reserved rows, offered rows whose one submitblock outcome is unknown (a transport failure or timeout, or a reservation whose call was lost with its frontend; not a row adopted on the node's own evidence that its block is active), and rows the node definitively rejected unless the reply names a side-chain block (inconclusive, duplicate, duplicate-inconclusive), read from the same database snapshot as the all-unfinished gauges. A rejection such as bad-txnmrklroot, high-hash, bad-cb-* or duplicate-invalid means the pool offered an invalid block, which is the emergency this rule exists for, and it keeps paging exactly as before. A node-accepted lost race sits in reconciliation until a different block has PRISM_CANDIDATE_ORPHAN_CONFIRMATIONS confirmations at its height; that is expected and never pages here. An unknown outcome keeps paging because receipt is unproven: the block may never have reached the node and is never offered again, so it is indistinguishable from an unoffered block until the chain proves an orphan or a later observation lands it. A landing that keeps failing after acceptance is owned by the landing-failed rules; every other unfinished row by the stuck warning. Mixed-version rollout: on a target that does not export the unacknowledged series yet, the expression falls back to the all-unfinished age, so the base paging behaviour (including its lost-race page) persists there until that frontend is upgraded; deploy frontends first. Native pending outbox rows include claimed/retry-delayed candidates; count bounds 2/5 and age bounds 15/60 seconds. Provisional, measure in #291.", ++ "no_data_state": "OK", ++ "exec_err_state": "OK" ++ }, ++ { ++ "uid": "qbit-prism-block-candidate-stuck", ++ "title": "PrismBlockCandidateStuck", ++ "expr": "(qbit_prism_block_candidate_oldest_pending_seconds{job=\"qbit-prism\",network=\"" ~ qbit_monitoring_stack_network ~ "\"} >= bool 1800) and on(job, instance, network) ((qbit_prism_metrics_snapshot_available{job=\"qbit-prism\",network=\"" ~ qbit_monitoring_stack_network ~ "\"} == 1) and on(job, instance, network) (qbit_prism_metrics_snapshot_stale{job=\"qbit-prism\",network=\"" ~ qbit_monitoring_stack_network ~ "\"} == 0)) and on(job, instance, network) (qbit_prism_collector_available{job=\"qbit-prism\",network=\"" ~ qbit_monitoring_stack_network ~ "\",collector=\"database\"} == 1)", ++ "condition_type": "threshold", ++ "evaluator": "gt", ++ "threshold": 0, ++ "for": "3m", ++ "severity": "warning", ++ "service": "qbit-mining-pool", ++ "labels": { ++ "alert_class": "prism-runtime" ++ }, ++ "summary": "PrismBlockCandidateStuck", ++ "description": "Per-scrape-target view of cluster-wide backlog; do not sum repeated database snapshots across coordinators. Unknown -1 belongs to the collector-unavailable alert. An unfinished candidate row older than 30 minutes (#493), read from the all-unfinished gauge: an offered row in reconciliation whose chain verdict is not arriving, a node-rejected block, or a landing that keeps failing after acceptance. A lost tip race is proven orphan within a few block intervals (six competitor confirmations at the 60-second target spacing plus the reconciliation retry backoff) and leaves this gauge well inside the threshold, aligned with PrismAcceptedBlockUnlanded. The reconciliation backoff is ten seconds times the row's attempt count, capped at an hour, and the count includes the pre-offer attempts, so the cumulative wait to the k-th retry is about 5k(k+1) seconds: the sixth competitor confirmation at 60-second spacing is reached around the eighth retry, about nine minutes, well inside 30 minutes, while a row that needed many pre-offer attempts starts its backoff higher and reaches 30 minutes in a few retries. Pre-offer, unknown-outcome and rejected rows already page at 60 seconds through PrismBlockCandidateOldestPendingCritical and a failing landing through the landing-failed rules; this warning is the non-paging catch-all for the rest, which is operator work (the row's last_error names the reason). Three-minute dwell like the other candidate warnings. Provisional, measure in #291.", ++ "no_data_state": "OK", ++ "exec_err_state": "OK" ++ }, ++ { ++ "uid": "qbit-prism-candidate-landing-failed", ++ "title": "PrismBlockCandidateLandingFailed", ++ "expr": "(qbit_prism_block_candidate_oldest_landing_failed_seconds{job=\"qbit-prism\",network=\"" ~ qbit_monitoring_stack_network ~ "\"} >= bool 60) and on(job, instance, network) ((qbit_prism_metrics_snapshot_available{job=\"qbit-prism\",network=\"" ~ qbit_monitoring_stack_network ~ "\"} == 1) and on(job, instance, network) (qbit_prism_metrics_snapshot_stale{job=\"qbit-prism\",network=\"" ~ qbit_monitoring_stack_network ~ "\"} == 0)) and on(job, instance, network) (qbit_prism_collector_available{job=\"qbit-prism\",network=\"" ~ qbit_monitoring_stack_network ~ "\",collector=\"database\"} == 1)", ++ "condition_type": "threshold", ++ "evaluator": "gt", ++ "threshold": 0, ++ "for": "3m", ++ "severity": "warning", ++ "service": "qbit-mining-pool", ++ "labels": { ++ "alert_class": "prism-runtime" ++ }, ++ "summary": "PrismBlockCandidateLandingFailed", ++ "description": "Per-scrape-target view of cluster-wide backlog; do not sum repeated database snapshots across coordinators. Unknown -1 belongs to the collector-unavailable alert. An offered or reconciliation row whose audit landing has not succeeded after the offer (#493): the landing transaction is the only writer of the block's pool-block row, so its absence is the durable fact the collector reads, and a last_error naming a landing refusal (a prefix the coordinator and the collector share) covers a landed row that no longer authenticates; a retry that fails for a transient node or database reason, or a refused operator recovery, overwrites last_error but cannot make this age read zero while nothing has landed. The node accepted or rejected the block, the audit rebuild, its verification or the ledger landing refused, and the row is retried at the reconciliation backoff with read-only chain observations only. The age is measured from the offer reservation, the start of the post-offer lifecycle and the same instant the frontend's acceptance clock starts, so pre-offer retry backoff never counts toward it. An offered row counts before its first landing outcome (#510): a won block whose landing hangs in offered, waiting for build capacity with no deadline while its heartbeat keeps the claim, ages here instead of paging nowhere; a healthy offered row lands within the 60-second rebuild deadline, so it never reaches the three-minute warning dwell. Never a lost race: a lost race and a rejection land their audit before the chain verdict, so their pool-block row keeps this age at zero once landed. What is at risk is the block's audit, its pool-block row, the deferred solver credit and the next payout revision, which the successful landing writes; the block reward is on the chain regardless. Warning at 60 seconds for three minutes: a transient refusal (a rebuild deadline under a brief database stall) lands on the first retry ten seconds later and never reaches the dwell; the same shape as the pre-#493 candidate critical, without the page. Provisional, measure in #291.", ++ "no_data_state": "OK", ++ "exec_err_state": "OK" ++ }, ++ { ++ "uid": "qbit-prism-candidate-landing-failed-critical", ++ "title": "PrismBlockCandidateLandingFailedCritical", ++ "expr": "(qbit_prism_block_candidate_oldest_landing_failed_seconds{job=\"qbit-prism\",network=\"" ~ qbit_monitoring_stack_network ~ "\"} >= bool 307) and on(job, instance, network) ((qbit_prism_metrics_snapshot_available{job=\"qbit-prism\",network=\"" ~ qbit_monitoring_stack_network ~ "\"} == 1) and on(job, instance, network) (qbit_prism_metrics_snapshot_stale{job=\"qbit-prism\",network=\"" ~ qbit_monitoring_stack_network ~ "\"} == 0)) and on(job, instance, network) (qbit_prism_collector_available{job=\"qbit-prism\",network=\"" ~ qbit_monitoring_stack_network ~ "\",collector=\"database\"} == 1)", ++ "condition_type": "threshold", ++ "evaluator": "gt", ++ "threshold": 0, ++ "for": "1m", ++ "severity": "critical", ++ "service": "qbit-mining-pool", ++ "labels": { ++ "alert_class": "prism-runtime", ++ "page": "true" ++ }, ++ "summary": "PrismBlockCandidateLandingFailedCritical", ++ "description": "Per-scrape-target view of cluster-wide backlog; do not sum repeated database snapshots across coordinators. Unknown -1 belongs to the collector-unavailable alert. An offered or reconciliation row whose audit landing has not succeeded after the offer (#493): the landing transaction is the only writer of the block's pool-block row, so its absence is the durable fact the collector reads, and a last_error naming a landing refusal (a prefix the coordinator and the collector share) covers a landed row that no longer authenticates; a retry that fails for a transient node or database reason, or a refused operator recovery, overwrites last_error but cannot make this age read zero while nothing has landed. The node accepted or rejected the block, the audit rebuild, its verification or the ledger landing refused, and the row is retried at the reconciliation backoff with read-only chain observations only. The age is measured from the offer reservation, the start of the post-offer lifecycle and the same instant the frontend's acceptance clock starts, so pre-offer retry backoff never counts toward it. An offered row counts before its first landing outcome (#510): a won block whose landing hangs in offered, waiting for build capacity with no deadline while its heartbeat keeps the claim, ages here instead of paging nowhere; a healthy offered row lands within the 60-second rebuild deadline, so it never reaches the three-minute warning dwell. Never a lost race: a lost race and a rejection land their audit before the chain verdict, so their pool-block row keeps this age at zero once landed. What is at risk is the block's audit, its pool-block row, the deferred solver credit and the next payout revision, which the successful landing writes; the block reward is on the chain regardless. Pages at 307 seconds for one minute, the same floor and dwell as PrismAcceptedRevisionWorkPendingCritical: a landing that fails after the frontend's own active-chain observation already pages that rule at 307 s (the landed identity is a known wait with no revision to deliver), and a landing that fails before it now pages here at the same instant, so a won block whose ledger landing keeps failing is never silent whichever step refused. By 307 seconds the row has been retried about eight times at the reconciliation backoff (ten seconds times the attempt count), so the failure is persistent, never a single deadline. Gated like the other candidate rules on a fresh snapshot and an available collector; no-data and evaluation errors are OK because the unavailable warning owns the missing measurement. Provisional, measure in #291.", ++ "no_data_state": "OK", ++ "exec_err_state": "OK" ++ }, ++ { ++ "uid": "qbit-prism-candidate-metrics-unavailable", ++ "title": "PrismBlockCandidateMetricsUnavailable", ++ "expr": "(qbit_prism_collector_available{job=\"qbit-prism\",network=\"" ~ qbit_monitoring_stack_network ~ "\",collector=\"database\"} == bool 0) or on(job, instance, network) (qbit_prism_block_candidates_pending{job=\"qbit-prism\",network=\"" ~ qbit_monitoring_stack_network ~ "\"} < bool 0) or on(job, instance, network) (qbit_prism_block_candidate_oldest_pending_seconds{job=\"qbit-prism\",network=\"" ~ qbit_monitoring_stack_network ~ "\"} < bool 0) or on(job, instance, network) (qbit_prism_block_candidate_oldest_unacknowledged_seconds{job=\"qbit-prism\",network=\"" ~ qbit_monitoring_stack_network ~ "\"} < bool 0) or on(job, instance, network) (qbit_prism_block_candidate_oldest_landing_failed_seconds{job=\"qbit-prism\",network=\"" ~ qbit_monitoring_stack_network ~ "\"} < bool 0)", ++ "condition_type": "threshold", ++ "evaluator": "gt", ++ "threshold": 0, ++ "for": "3m", ++ "severity": "warning", ++ "service": "qbit-mining-pool", ++ "labels": { ++ "alert_class": "prism-runtime" ++ }, ++ "summary": "PrismBlockCandidateMetricsUnavailable", ++ "description": "Database candidate age/count are unknown, not zero; collector state is live at scrape time. Collector failure, cancellation or expiry after 30 seconds invalidates all four gauges, the unacknowledged and landing-failed ages included (#493).", ++ "no_data_state": "Alerting", ++ "exec_err_state": "Alerting" ++ }, ++ { ++ "uid": "qbit-prism-block-submission-held", ++ "title": "PrismBlockSubmissionHeld", ++ "expr": "(last_over_time(qbit_prism_block_submission_enabled{job=\"qbit-prism\",network=\"" ~ qbit_monitoring_stack_network ~ "\"}[2m]) != bool 1) and on(job, instance, network) (max_over_time(up{job=\"qbit-prism\",network=\"" ~ qbit_monitoring_stack_network ~ "\"}[2m]) == 1)", ++ "condition_type": "threshold", ++ "evaluator": "gt", ++ "threshold": 0, ++ "for": "1m", ++ "severity": "critical", ++ "service": "qbit-mining-pool", ++ "labels": { ++ "alert_class": "prism-runtime", ++ "page": "true" ++ }, ++ "summary": "PrismBlockSubmissionHeld", ++ "description": "This frontend holds block submission: its PRISM_BLOCK_SUBMIT_ENABLED is off (#661, the #291 rehearsal kill switch), or the cluster's block submission hold is set in the ledger (#664, qbit-prism-server submission-hold set), so it offers no block it finds to its node's submitblock and sends no CTV fanout. Its found blocks wait in the shared candidate outbox: a peer frontend that still submits offers them, and when every frontend is held, as under the cluster hold, none is offered. This rule pages a minute after a frontend is held, independently of whether it has found a block; the candidate rules page only once a found block has waited (PrismBlockCandidateOldestPendingCritical: an age of 60 seconds, then a three-minute dwell). A frontend that cannot yet show that it submits pages too (-1): its switch is on but it has not read the cluster's hold yet. A later failed read keeps the last one, so a database outage does not page here. The gauge is rendered at scrape time from the frontend's last health publication, so the rule reads its last sample in the past two minutes and is gated on a successful scrape in that window rather than on a fresh snapshot: a held frontend keeps paging through a stale snapshot or an occasional failed scrape. A missing endpoint, no data and evaluation errors are OK, because PrismMetricsEndpointDown and PrismMetricsNoData own them, and an image older than #661 exports no series. A rehearsal against the production network holds its frontends on purpose: scrape them under another job or network, or silence this rule with matchers on the rehearsal frontends' instance labels, never on the alert name alone, which would also hide a production frontend left held (docs/prism-ledger-ops.md, block submission kill switch for rehearsals). Remedy: when qbit-prism-server submission-hold show reports \"held\": true, clear the hold with submission-hold clear --reason once the cutover is verified (docs/prism-ledger-ops.md, holding the whole cluster); otherwise unset PRISM_BLOCK_SUBMIT_ENABLED or set it to 1, restart the frontend, and confirm check-config prints no block submission WARNING. #666, #664.", ++ "no_data_state": "OK", ++ "exec_err_state": "OK" ++ }, ++ { ++ "uid": "qbit-prism-database-unavailable", ++ "title": "PrismDatabaseUnavailable", ++ "expr": "(qbit_prism_collector_available{job=\"qbit-prism\",network=\"" ~ qbit_monitoring_stack_network ~ "\",collector=\"database\"} == bool 0) and on(job, instance, network) (up{job=\"qbit-prism\",network=\"" ~ qbit_monitoring_stack_network ~ "\"} == 1)", ++ "condition_type": "threshold", ++ "evaluator": "gt", ++ "threshold": 0, ++ "for": "2m", ++ "severity": "critical", ++ "service": "qbit-mining-pool", ++ "labels": { ++ "alert_class": "prism-runtime", ++ "page": "true" ++ }, ++ "summary": "PrismDatabaseUnavailable", ++ "description": "This frontend has had no complete successful observation of its ledger database for two minutes: the database collector, which reads PostgreSQL every 10 seconds through this frontend's own connection pool, in a read-only transaction with a 3-second deadline that includes the pool acquire and a 500 ms lock timeout, failed, was cancelled, or last succeeded more than 30 seconds ago. That covers PostgreSQL down or unreachable, and also a pool saturated, or a lock blocking the collector's reads, for two minutes. While it holds, the frontend refuses the shares and sessions it cannot record, mostly as backend-database-unavailable or ledger-confirmation-failed (#581), so every miner on it is stalled. Pages on exactly this state and says so: before #581 a PostgreSQL outage held only PrismBlockCandidateMetricsUnavailable, a warning titled for block-candidate metrics. Not this rule's: a database that still serves reads but refuses writes (a read-only standby or demoted primary, or a cluster halted by its fatal state) and a stall shorter than two minutes; both show as backend-database-unavailable refusals in PrismRejectRatioByReasonHigh. Collector state is live at scrape time and not gated on a fresh metrics snapshot, whose freshness flaps while the health publisher waits on the database. Gated on a successful scrape; a missing endpoint, no data and evaluation errors are OK because PrismMetricsEndpointDown and PrismMetricsNoData own them. Provisional, measure in #291.", ++ "no_data_state": "OK", ++ "exec_err_state": "OK" ++ }, ++ { ++ "uid": "qbit-prism-block-offer-standby-unconfirmed", ++ "title": "PrismBlockOfferStandbyUnconfirmed", ++ "expr": "(sum by (job, instance, network) (increase(qbit_prism_block_offer_standby_wait_total{job=\"qbit-prism\",network=\"" ~ qbit_monitoring_stack_network ~ "\",outcome=~\"absent|lagging|failed\"}[30m])) > bool 0) and on(job, instance, network) ((qbit_prism_metrics_snapshot_available{job=\"qbit-prism\",network=\"" ~ qbit_monitoring_stack_network ~ "\"} == 1) and on(job, instance, network) (qbit_prism_metrics_snapshot_stale{job=\"qbit-prism\",network=\"" ~ qbit_monitoring_stack_network ~ "\"} == 0))", ++ "condition_type": "threshold", ++ "evaluator": "gt", ++ "threshold": 0, ++ "for": "0s", ++ "severity": "warning", ++ "service": "qbit-mining-pool", ++ "labels": { ++ "alert_class": "prism-runtime" ++ }, ++ "summary": "PrismBlockOfferStandbyUnconfirmed", ++ "description": "A found block was offered in the last 30 minutes without a confirmed copy on the dedicated failover standby (#529). With PRISM_OFFER_STANDBY_APPLICATION_NAME set, each found block's offer first waits, up to PRISM_OFFER_STANDBY_FLUSH_WAIT_MS (default 250 ms), for that standby's flush_lsn to reach the primary's WAL flush position, which covers the block's candidate row and offer reservation. This counts the offers where it did not: absent (no standby connected under that name), lagging (still behind at the bound) or failed (the position could not be read, for example a writer role without pg_monitor). The block was offered anyway, since the wait never holds a block. Until the standby catches up, losing the primary also loses that block's ledger rows, so a block that reached the chain would have no candidate, audit or pool-blocks record on the promoted primary. The frontend log names each such block hash at WARN (block offer standby wait unconfirmed). Check the D3 standby rules (disconnected or lagging) and the writer role's pg_monitor grant; after any failover, reconcile the logged hashes as the ledger operations guide describes. A healthy standby confirms within a few milliseconds, so one unconfirmed offer is already a signal: no dwell, and the 30-minute window keeps it visible. Warning only: no block was held and no share acknowledgement waits on it. The counter only moves when the wait is configured and a block is found; no-data and evaluation errors are OK because the snapshot rules own missing measurements. Provisional, measure in #291.", ++ "no_data_state": "OK", ++ "exec_err_state": "OK" ++ }, ++ { ++ "uid": "qbit-prism-database-pool-wait", ++ "title": "PrismDatabasePoolWaitHigh", ++ "expr": "((histogram_quantile(0.99, sum by (job, instance, network, le) (rate(qbit_prism_database_pool_acquire_seconds_bucket{job=\"qbit-prism\",network=\"" ~ qbit_monitoring_stack_network ~ "\"}[5m]))) > bool 0.5) and on(job, instance, network) (sum by (job, instance, network) (increase(qbit_prism_database_pool_acquire_seconds_count{job=\"qbit-prism\",network=\"" ~ qbit_monitoring_stack_network ~ "\"}[5m])) >= 10)) and on(job, instance, network) (up{job=\"qbit-prism\",network=\"" ~ qbit_monitoring_stack_network ~ "\"} == 1)", ++ "condition_type": "threshold", ++ "evaluator": "gt", ++ "threshold": 0, ++ "for": "3m", ++ "severity": "warning", ++ "service": "qbit-mining-pool", ++ "labels": { ++ "alert_class": "prism-runtime" ++ }, ++ "summary": "PrismDatabasePoolWaitHigh", ++ "description": "Actual PgPool::acquire waits from metrics collection, instrumented ledger transactions and covered direct ledger queries, rollup transactions and share-partition maintenance, including share processing and health publication. Each started collector acquisition records one observation: success when acquired, or failure on acquisition error or cancellation, including the three-second collection deadline. Durations measure elapsed pool wait; BEGIN, query execution and advisory-lock waits are outside this timer. This is the aggregate of observed checkout attempts at covered callers. Public-role read pools, private audit HTTP endpoints and shared API read-model helpers, startup/migration gates, operator tools and telemetry-free compatibility wrappers remain excluded; see the acquisition guide for the source census and exclusion policy. Success and failure observations are combined. The histogram is read live at scrape time, so valid waits remain alertable during database collector failure or cached-body publication stalls; a successful scrape is required. Pool p99 >0.5 seconds from at least ten observed acquisition attempts in five minutes; provisional, measure in #291.", ++ "no_data_state": "OK", ++ "exec_err_state": "OK" ++ }, ++ { ++ "uid": "qbit-prism-database-advisory-lock-wait", ++ "title": "PrismDatabaseAdvisoryLockWaitHigh", ++ "expr": "((histogram_quantile(0.99, sum by (job, instance, network, lock, le) (rate(qbit_prism_database_advisory_lock_wait_seconds_bucket{job=\"qbit-prism\",network=\"" ~ qbit_monitoring_stack_network ~ "\",lock=~\"order|settlement\"}[5m]))) > bool 1) and on(job, instance, network, lock) (sum by (job, instance, network, lock) (increase(qbit_prism_database_advisory_lock_wait_seconds_count{job=\"qbit-prism\",network=\"" ~ qbit_monitoring_stack_network ~ "\",lock=~\"order|settlement\"}[5m])) >= 10)) and on(job, instance, network) ((qbit_prism_metrics_snapshot_available{job=\"qbit-prism\",network=\"" ~ qbit_monitoring_stack_network ~ "\"} == 1) and on(job, instance, network) (qbit_prism_metrics_snapshot_stale{job=\"qbit-prism\",network=\"" ~ qbit_monitoring_stack_network ~ "\"} == 0))", ++ "condition_type": "threshold", ++ "evaluator": "gt", ++ "threshold": 0, ++ "for": "3m", ++ "severity": "warning", ++ "service": "qbit-mining-pool", ++ "labels": { ++ "alert_class": "prism-runtime" ++ }, ++ "summary": "PrismDatabaseAdvisoryLockWaitHigh", ++ "description": "Order and settlement advisory transaction lock waits, combining success and failure. The timer covers the pg_advisory_xact_lock statement alone, including one database round trip; the surrounding transaction work is outside it. lock=\"migration\" is excluded because the migration advisory lock is taken only when a coordinator initializes the schema, where waiting behind another frontend for the whole online 013/016/017 migration window is the designed behaviour rather than a fault. Each lock is judged on its own waits; neither lock supplies the other's sample minimum. This family is rendered from the cached snapshot body and is not read live at scrape time, so an unpublished or stale body suppresses the rule instead of repeating an old percentile. Order and settlement advisory-lock p99 >1 second, combining success and failure, from at least ten observed waits for that lock in five minutes; #271 measured 15, 31 and 63 steady ORDER_LOCK waiters at one, two and four frontends without any native wait percentile, and PRISM_DATABASE_LOCK_TIMEOUT_MS bounds a single wait at five seconds by default; provisional, measure in #291/#447.", ++ "no_data_state": "OK", ++ "exec_err_state": "OK" ++ }, ++ { ++ "uid": "qbit-prism-database-advisory-lock-failures-critical", ++ "title": "PrismDatabaseAdvisoryLockFailuresCritical", ++ "expr": "((sum by (job, instance, network, lock) (increase(qbit_prism_database_advisory_lock_wait_seconds_count{job=\"qbit-prism\",network=\"" ~ qbit_monitoring_stack_network ~ "\",lock=~\"order|settlement\",result=\"failure\"}[5m])) / sum by (job, instance, network, lock) (increase(qbit_prism_database_advisory_lock_wait_seconds_count{job=\"qbit-prism\",network=\"" ~ qbit_monitoring_stack_network ~ "\",lock=~\"order|settlement\"}[5m])) > bool 0.05) and on(job, instance, network, lock) (sum by (job, instance, network, lock) (increase(qbit_prism_database_advisory_lock_wait_seconds_count{job=\"qbit-prism\",network=\"" ~ qbit_monitoring_stack_network ~ "\",lock=~\"order|settlement\"}[5m])) >= 10)) and on(job, instance, network) ((qbit_prism_metrics_snapshot_available{job=\"qbit-prism\",network=\"" ~ qbit_monitoring_stack_network ~ "\"} == 1) and on(job, instance, network) (qbit_prism_metrics_snapshot_stale{job=\"qbit-prism\",network=\"" ~ qbit_monitoring_stack_network ~ "\"} == 0))", ++ "condition_type": "threshold", ++ "evaluator": "gt", ++ "threshold": 0, ++ "for": "3m", ++ "severity": "critical", ++ "service": "qbit-mining-pool", ++ "labels": { ++ "alert_class": "prism-runtime", ++ "page": "true" ++ }, ++ "summary": "PrismDatabaseAdvisoryLockFailuresCritical", ++ "description": "Share of order or settlement advisory-lock waits that ended without a held lock. result=\"failure\" covers lock timeout (PRISM_DATABASE_LOCK_TIMEOUT_MS, five seconds by default), statement timeout, deadlock and connection error, and also cancellation of the waiting future: a dropped future such as a cancelled share append records failure from the wait guard rather than vanishing. The failure series does not exist until the first failure, so on a healthy coordinator the ratio is absent rather than zero and no-data stays OK instead of paging. Each series appears at its first observation, so the first failure after a restart is invisible to increase() and the ratio only rises once a later failure lands in the window. lock=\"migration\" is excluded for the same reason as the wait percentile: that lock is taken only at schema initialization. This family is rendered from the cached snapshot body, so an unpublished or stale body suppresses the rule. Order and settlement advisory-lock failures >5% of that lock's waits over five minutes, from at least ten observed waits for that lock; #271 measured 15, 31 and 63 steady ORDER_LOCK waiters at one, two and four frontends and no native failure rate, and the default five-second PRISM_DATABASE_LOCK_TIMEOUT_MS turns a sustained wait into a failure; provisional, measure in #291/#447.", ++ "no_data_state": "OK", ++ "exec_err_state": "OK" ++ }, ++ { ++ "uid": "qbit-prism-lease-monitor-wake", ++ "title": "PrismRuntimeLagHigh", ++ "expr": "(max_over_time(qbit_prism_runtime_lag_seconds{job=\"qbit-prism\",network=\"" ~ qbit_monitoring_stack_network ~ "\"}[5m]) > bool 2) + on(job, instance, network) (max by (job, instance, network) (max_over_time(qbit_prism_runtime_poll_lag_seconds{job=\"qbit-prism\",network=\"" ~ qbit_monitoring_stack_network ~ "\"}[5m])) > bool 2) + on(job, instance, network) (max by (job, instance, network) (qbit_prism_runtime_task_stalled{job=\"qbit-prism\",network=\"" ~ qbit_monitoring_stack_network ~ "\"}) > bool 0) + on(job, instance, network) (qbit_prism_runtime_lag_seconds{job=\"qbit-prism\",network=\"" ~ qbit_monitoring_stack_network ~ "\"} < bool 0)", ++ "condition_type": "threshold", ++ "evaluator": "gt", ++ "threshold": 0, ++ "for": "3m", ++ "severity": "warning", ++ "service": "qbit-mining-pool", ++ "labels": { ++ "alert_class": "prism-runtime" ++ }, ++ "summary": "PrismRuntimeLagHigh", ++ "description": "Runtime scheduling pressure replaces writer-lease monitor wake delay. Scrape-time runtime signals deliberately remain active during snapshot staleness; retained completed polls capture recovery even when latest wake lag drops within 100ms. Two-second native blocked-poll budget; five-minute evidence window and three-minute dwell replace the legacy lease-wake episode, provisional, measure in #291. A runtime sampler that never observes a tick remains unknown and alerts after the same dwell.", ++ "no_data_state": "OK", ++ "exec_err_state": "OK" ++ }, ++ { ++ "uid": "qbit-prism-resident-memory", ++ "title": "PrismResidentMemoryHigh", ++ "expr": "(qbit_prism_process_resident_memory_bytes{job=\"qbit-prism\",network=\"" ~ qbit_monitoring_stack_network ~ "\"} > bool 4294967296) and on(job, instance, network) ((qbit_prism_metrics_snapshot_available{job=\"qbit-prism\",network=\"" ~ qbit_monitoring_stack_network ~ "\"} == 1) and on(job, instance, network) (qbit_prism_metrics_snapshot_stale{job=\"qbit-prism\",network=\"" ~ qbit_monitoring_stack_network ~ "\"} == 0)) and on(job, instance, network) (qbit_prism_collector_available{job=\"qbit-prism\",network=\"" ~ qbit_monitoring_stack_network ~ "\",collector=\"process\"} == 1)", ++ "condition_type": "threshold", ++ "evaluator": "gt", ++ "threshold": 0, ++ "for": "3m", ++ "severity": "warning", ++ "service": "qbit-mining-pool", ++ "labels": { ++ "alert_class": "prism-runtime" ++ }, ++ "summary": "PrismResidentMemoryHigh", ++ "description": "Linux process VmRSS, not heap allocation; a real zero is valid and unsupported/failed/stale collection is -1. Four GiB is an unqualified initial RSS bound; set against actual container headroom. Provisional, measure in #291.", ++ "no_data_state": "OK", ++ "exec_err_state": "OK" ++ }, ++ { ++ "uid": "qbit-prism-process-metrics-unavailable", ++ "title": "PrismProcessMetricsUnavailable", ++ "expr": "(qbit_prism_collector_available{job=\"qbit-prism\",network=\"" ~ qbit_monitoring_stack_network ~ "\",collector=\"process\"} == bool 0) or on(job, instance, network) (qbit_prism_process_resident_memory_bytes{job=\"qbit-prism\",network=\"" ~ qbit_monitoring_stack_network ~ "\"} < bool 0)", ++ "condition_type": "threshold", ++ "evaluator": "gt", ++ "threshold": 0, ++ "for": "3m", ++ "severity": "warning", ++ "service": "qbit-mining-pool", ++ "labels": { ++ "alert_class": "prism-runtime" ++ }, ++ "summary": "PrismProcessMetricsUnavailable", ++ "description": "RSS cannot establish memory health until process collection succeeds. Process collector expires after 30 seconds or on failure; unsupported platforms are unknown.", ++ "no_data_state": "Alerting", ++ "exec_err_state": "Alerting" ++ }, ++ { ++ "uid": "qbit-prism-revision-work-pending", ++ "title": "PrismAcceptedRevisionWorkPending", ++ "expr": "clamp_max(((qbit_prism_accepted_block_revision_work_pending_seconds{job=\"qbit-prism\",network=\"" ~ qbit_monitoring_stack_network ~ "\"} > bool 1) or on(job, instance, network) (0 * up{job=\"qbit-prism\",network=\"" ~ qbit_monitoring_stack_network ~ "\"} + 1)) + on(job, instance, network) (up{job=\"qbit-prism\",network=\"" ~ qbit_monitoring_stack_network ~ "\"} == bool 0), 1)", ++ "condition_type": "threshold", ++ "evaluator": "gt", ++ "threshold": 0, ++ "for": "0s", ++ "severity": "warning", ++ "service": "qbit-mining-pool", ++ "labels": { ++ "alert_class": "prism-runtime" ++ }, ++ "summary": "PrismAcceptedRevisionWorkPending", ++ "description": "Acceptance is observed locally; completion requires an actual successful mining.notify write at the post-landing revision. Provisional warning at more than one second, with no additional dwell: the #275 budget is a target, not a measured production percentile; measure in #291. Missing series on a configured target and failed scrapes also warn. Unknown tracking (a -1 age, the tracking-unknown gauge, or a missing unknown gauge) is owned by PrismAcceptedRevisionWorkTrackingUnknown, which dwells so a single failed refresh does not warn (#493); a known pending age beside unknown tracking still warns here and remains eligible for the critical rule. An offer the node accepted but has not reported active is an unlanded acceptance, excluded from this age and owned by PrismAcceptedBlockUnlanded (#493).", ++ "no_data_state": "Alerting", ++ "exec_err_state": "Alerting" ++ }, ++ { ++ "uid": "qbit-prism-revision-work-unknown", ++ "title": "PrismAcceptedRevisionWorkTrackingUnknown", ++ "expr": "clamp_max((qbit_prism_accepted_block_revision_work_pending_seconds{job=\"qbit-prism\",network=\"" ~ qbit_monitoring_stack_network ~ "\"} < bool 0) + on(job, instance, network) ((qbit_prism_accepted_block_revision_work_tracking_unknown{job=\"qbit-prism\",network=\"" ~ qbit_monitoring_stack_network ~ "\"} > bool 0) or on(job, instance, network) (0 * up{job=\"qbit-prism\",network=\"" ~ qbit_monitoring_stack_network ~ "\"} + 1)), 1)", ++ "condition_type": "threshold", ++ "evaluator": "gt", ++ "threshold": 0, ++ "for": "2m", ++ "severity": "warning", ++ "service": "qbit-mining-pool", ++ "labels": { ++ "alert_class": "prism-runtime" ++ }, ++ "summary": "PrismAcceptedRevisionWorkTrackingUnknown", ++ "description": "Acceptance is observed locally; completion requires an actual successful mining.notify write at the post-landing revision. Unknown or incomplete local landing tracking (#493 point 7): the pending age is -1 (no known wait remains but tracking is unknown), the tracking-unknown gauge is one beside any known age, or the unknown gauge is missing on a configured target during a mixed-version rollout. Two-minute dwell: a failed template refresh marks tracking unknown only until the next successful refresh, one poll interval later (two seconds by default), so a single transient refresh error cannot span the eight consecutive 15-second scrapes this dwell requires, while a lost or cancelled settlement COMMIT reply, an unsettled orphan verdict, bounded-tracking saturation and lost ordering history persist until terminal evidence or restart and still warn. No-data and evaluation errors alert so a missing measurement stays visible after the same dwell. Non-paging: the critical rule owns known stalls and gates out unknown ages itself; the pending warning owns the no-dwell budget signal. Measure the dwell in #291.", ++ "no_data_state": "Alerting", ++ "exec_err_state": "Alerting" ++ }, ++ { ++ "uid": "qbit-prism-revision-work-pending-critical", ++ "title": "PrismAcceptedRevisionWorkPendingCritical", ++ "expr": "((qbit_prism_accepted_block_revision_work_pending_seconds{job=\"qbit-prism\",network=\"" ~ qbit_monitoring_stack_network ~ "\"} >= bool 307) and on(job, instance, network) (qbit_prism_accepted_block_revision_work_pending_seconds{job=\"qbit-prism\",network=\"" ~ qbit_monitoring_stack_network ~ "\"} >= 0)) and on(job, instance, network) (qbit_prism_authorized_clients{job=\"qbit-prism\",network=\"" ~ qbit_monitoring_stack_network ~ "\"} > 0) and on(job, instance, network) (up{job=\"qbit-prism\",network=\"" ~ qbit_monitoring_stack_network ~ "\"} == 1)", ++ "condition_type": "threshold", ++ "evaluator": "gt", ++ "threshold": 0, ++ "for": "1m", ++ "severity": "critical", ++ "service": "qbit-mining-pool", ++ "labels": { ++ "alert_class": "prism-runtime", ++ "page": "true" ++ }, ++ "summary": "PrismAcceptedRevisionWorkPendingCritical", ++ "description": "Acceptance is observed locally; completion requires an actual successful mining.notify write at the post-landing revision. Provisional critical floor of 307 seconds comes from the #413 incident duration, not an early-warning result. Gates (#493): the age must be known (not -1), the target scraped, and at least one authorized miner on the reporting frontend, because a frontend with no connected miners has no mining.notify write that can close its wait and would otherwise page after every pool block. The pending gauge is a live overlay rendered on every scrape, so it is not gated on the metrics snapshot: a database-pool stall that also stalls the health publisher must still page. The miner count is a snapshot family and may be a few refresh intervals old while the snapshot is stale; a false page on a frontend whose miners left inside that window is accepted in exchange for never missing a real stall. If every miner leaves a frontend during a stall, this page resolves and nothing else pages for that frontend: the coverage rules cannot fire with zero authorized clients and the connected-clients rule is a cluster-wide non-paging warning; the pending warning keeps reporting the age until a miner reconnects and receives work. One-minute dwell: the gauge is monotonic during a stall, so the dwell only absorbs a brief gate gap and stays well under the five-minute hold that delayed #413. No-data and evaluation errors are OK: the warning rules own missing or unknown measurements. The floor and dwell are literals in this specification, not deployment variables. A lost race on the offering frontend never counts here (#493 point 1): an offer the node accepted without an active-chain observation is an unlanded acceptance, excluded from the known pending age until an active-chain observation lands it or its orphan is proven, and reported by accepted_block_unlanded_seconds and PrismAcceptedBlockUnlanded instead. Measure thresholds and scrape/evaluation delay in #291; no operational deployment is claimed.", ++ "no_data_state": "OK", ++ "exec_err_state": "OK" ++ }, ++ { ++ "uid": "qbit-prism-revision-work-timeouts", ++ "title": "PrismRevisionWorkBuildTimeouts", ++ "expr": "increase(qbit_prism_revision_work_build_timeouts_total{job=\"qbit-prism\",network=\"" ~ qbit_monitoring_stack_network ~ "\"}[5m]) > bool 0", ++ "condition_type": "threshold", ++ "evaluator": "gt", ++ "threshold": 0, ++ "for": "0s", ++ "severity": "warning", ++ "service": "qbit-mining-pool", ++ "labels": { ++ "alert_class": "prism-runtime" ++ }, ++ "summary": "PrismRevisionWorkBuildTimeouts", ++ "description": "Acceptance is observed locally; completion requires an actual successful mining.notify write at the post-landing revision. Any existing build deadline hit while locally observed accepted work awaits delivery; five-minute increase window is provisional, measure in #291. Successful builds and cancellation do not increment the counter. Only a known open wait counts (#493): while this frontend's tracking is unknown and no known wait remains, a deadline hit is an ordinary job-delivery failure, not a revision-work failure, and the tracking-unknown warning reports the unknown state instead.", ++ "no_data_state": "Alerting", ++ "exec_err_state": "Alerting" ++ }, ++ { ++ "uid": "qbit-prism-accepted-block-unlanded", ++ "title": "PrismAcceptedBlockUnlanded", ++ "expr": "(qbit_prism_accepted_block_unlanded_seconds{job=\"qbit-prism\",network=\"" ~ qbit_monitoring_stack_network ~ "\"} >= bool 1800)", ++ "condition_type": "threshold", ++ "evaluator": "gt", ++ "threshold": 0, ++ "for": "3m", ++ "severity": "warning", ++ "service": "qbit-mining-pool", ++ "labels": { ++ "alert_class": "prism-runtime" ++ }, ++ "summary": "PrismAcceptedBlockUnlanded", ++ "description": "Acceptance is observed locally; an unlanded acceptance has no revision work to deliver yet. An offer the node accepted (a null submitblock reply) that this frontend has not observed on the node's active chain for 30 minutes (#493 point 1). A lost tip race is settled as a proven orphan once a different block has PRISM_CANDIDATE_ORPHAN_CONFIRMATIONS (default 6) confirmations at its height: about six minutes at the 60-second target spacing plus the reconciliation retry backoff, so a routine lost race leaves this gauge well inside the threshold. The reconciliation backoff is ten seconds times the row's attempt count, capped at an hour, and the count includes the pre-offer attempts, so the cumulative wait to the k-th retry is about 5k(k+1) seconds: the sixth competitor confirmation at 60-second spacing is reached around the eighth retry, about nine minutes, well inside 30 minutes, while a row that needed many pre-offer attempts starts its backoff higher and reaches 30 minutes in a few retries. Such an acceptance is never part of the known pending age, so the revision-work critical cannot page on it; a winning offer leaves this gauge at its first active-chain observation and joins the known age on its original acceptance clock. Past 30 minutes the row is not being reconciled, the node is not advancing, or the orphan verdict keeps failing: operator work, warned, never paged; the cluster-wide candidate oldest-pending rules see the same reconciliation row. -1 (none tracked while identity tracking is saturated) is owned by the tracking-unknown warning and does not fire here. Three-minute dwell like the other warnings; no-data is OK because the pending warning already reports a missing landing family. Provisional; measure in #291.", ++ "no_data_state": "OK", ++ "exec_err_state": "OK" ++ } ++] %} ++{% set prism_public_read_alert_rules = [ ++ { ++ "uid": "qbit-prism-public-staleness-refusal", ++ "title": "PrismPublicStalenessRefusalsIncreasing", ++ "expr": "increase(qbit_prism_public_staleness_refusals_total{job=\"qbit-prism-public-read\",network=\"" ~ qbit_monitoring_stack_network ~ "\"}[5m]) > bool 0", ++ "condition_type": "threshold", ++ "evaluator": "gt", ++ "threshold": 0, ++ "for": "3m", ++ "severity": "warning", ++ "service": "qbit-mining-pool", ++ "labels": { ++ "alert_class": "prism-runtime" ++ }, ++ "summary": "PrismPublicStalenessRefusalsIncreasing", ++ "description": "Cached public responses exceeded their route-specific budget. Any refusal is an actual endpoint staleness-budget breach; five-minute window is provisional, measure in #291.", ++ "no_data_state": "OK", ++ "exec_err_state": "OK" ++ }, ++ { ++ "uid": "qbit-prism-public-ledger-unready", ++ "title": "PrismPublicLedgerUnready", ++ "expr": "(((qbit_prism_public_ledger_ready{job=\"qbit-prism-public-read\",network=\"" ~ qbit_monitoring_stack_network ~ "\"} == bool 0) or on(job, instance, network) (0 * up{job=\"qbit-prism-public-read\",network=\"" ~ qbit_monitoring_stack_network ~ "\"} + 1)) + on(job, instance, network) (up{job=\"qbit-prism-public-read\",network=\"" ~ qbit_monitoring_stack_network ~ "\"} == bool 0)) > bool 0", ++ "condition_type": "threshold", ++ "evaluator": "gt", ++ "threshold": 0, ++ "for": "3m", ++ "severity": "warning", ++ "service": "qbit-mining-pool", ++ "labels": { ++ "alert_class": "prism-runtime" ++ }, ++ "summary": "PrismPublicLedgerUnready", ++ "description": "The public read probe is failed, unknown or stale, or its scrape target is down; its freshness owner does not emit the coordinator snapshot gauges. Readiness already rejects missing/failed probes and ages >= max(3 * probe interval, 15s). Missing readiness and failed scrapes on any configured target fire, even while another public instance is healthy.", ++ "no_data_state": "Alerting", ++ "exec_err_state": "Alerting" ++ }, ++ { ++ "uid": "qbit-prism-public-5xx-sustained", ++ "title": "PrismPublic5xxSustained", ++ "expr": "rate(qbit_prism_public_responses_total{job=\"qbit-prism-public-read\",network=\"" ~ qbit_monitoring_stack_network ~ "\",status=~\"5..\"}[5m]) > bool 0", ++ "condition_type": "threshold", ++ "evaluator": "gt", ++ "threshold": 0, ++ "for": "3m", ++ "severity": "warning", ++ "service": "qbit-mining-pool", ++ "labels": { ++ "alert_class": "prism-runtime" ++ }, ++ "summary": "PrismPublic5xxSustained", ++ "description": "This response counter includes health probes; it is an error signal and is not divided by public_requests_total. Any sustained 5xx response, preserving deployed failure intent; five-minute window is provisional, measure in #291.", ++ "no_data_state": "OK", ++ "exec_err_state": "OK" ++ } ++] %} ++{% set prism_database_alert_rules = [ ++ { ++ "uid": "qbit-prism-ha-standby-replay-lag", ++ "title": "PrismHAStandbyReplayLagHigh", ++ "expr": "clamp_max((max(up{job=\"qbit-postgres-primary\",network=\"" ~ qbit_monitoring_stack_network ~ "\"} != bool 1) or vector(1)) + (max(pg_up{job=\"qbit-postgres-primary\",network=\"" ~ qbit_monitoring_stack_network ~ "\"} != bool 1) or vector(1)) + (max(pg_exporter_last_scrape_error{job=\"qbit-postgres-primary\",network=\"" ~ qbit_monitoring_stack_network ~ "\"} != bool 0) or vector(1)) + ((count(up{job=\"qbit-postgres-primary\",network=\"" ~ qbit_monitoring_stack_network ~ "\"}) != bool 1) or vector(1)) + (max((pg_stat_replication_count{job=\"qbit-postgres-primary\",network=\"" ~ qbit_monitoring_stack_network ~ "\",application_name=\"prism_standby_1\"} < bool 0) + ((time() - timestamp(pg_stat_replication_count{job=\"qbit-postgres-primary\",network=\"" ~ qbit_monitoring_stack_network ~ "\",application_name=\"prism_standby_1\"})) < bool 0) + ((time() - timestamp(pg_stat_replication_count{job=\"qbit-postgres-primary\",network=\"" ~ qbit_monitoring_stack_network ~ "\",application_name=\"prism_standby_1\"})) > bool 2)) or vector(1)) + (max((pg_stat_replication_primary_flush_lsn_hi{job=\"qbit-postgres-primary\",network=\"" ~ qbit_monitoring_stack_network ~ "\",application_name=\"prism_standby_1\"} < bool 0) + ((time() - timestamp(pg_stat_replication_primary_flush_lsn_hi{job=\"qbit-postgres-primary\",network=\"" ~ qbit_monitoring_stack_network ~ "\",application_name=\"prism_standby_1\"})) < bool 0) + ((time() - timestamp(pg_stat_replication_primary_flush_lsn_hi{job=\"qbit-postgres-primary\",network=\"" ~ qbit_monitoring_stack_network ~ "\",application_name=\"prism_standby_1\"})) > bool 2)) or vector(1)) + (max((pg_stat_replication_primary_flush_lsn_lo{job=\"qbit-postgres-primary\",network=\"" ~ qbit_monitoring_stack_network ~ "\",application_name=\"prism_standby_1\"} < bool 0) + ((time() - timestamp(pg_stat_replication_primary_flush_lsn_lo{job=\"qbit-postgres-primary\",network=\"" ~ qbit_monitoring_stack_network ~ "\",application_name=\"prism_standby_1\"})) < bool 0) + ((time() - timestamp(pg_stat_replication_primary_flush_lsn_lo{job=\"qbit-postgres-primary\",network=\"" ~ qbit_monitoring_stack_network ~ "\",application_name=\"prism_standby_1\"})) > bool 2)) or vector(1)) + (max((pg_stat_replication_replay_lsn_hi{job=\"qbit-postgres-primary\",network=\"" ~ qbit_monitoring_stack_network ~ "\",application_name=\"prism_standby_1\"} < bool 0) + ((time() - timestamp(pg_stat_replication_replay_lsn_hi{job=\"qbit-postgres-primary\",network=\"" ~ qbit_monitoring_stack_network ~ "\",application_name=\"prism_standby_1\"})) < bool 0) + ((time() - timestamp(pg_stat_replication_replay_lsn_hi{job=\"qbit-postgres-primary\",network=\"" ~ qbit_monitoring_stack_network ~ "\",application_name=\"prism_standby_1\"})) > bool 2)) or vector(1)) + (max((pg_stat_replication_replay_lsn_lo{job=\"qbit-postgres-primary\",network=\"" ~ qbit_monitoring_stack_network ~ "\",application_name=\"prism_standby_1\"} < bool 0) + ((time() - timestamp(pg_stat_replication_replay_lsn_lo{job=\"qbit-postgres-primary\",network=\"" ~ qbit_monitoring_stack_network ~ "\",application_name=\"prism_standby_1\"})) < bool 0) + ((time() - timestamp(pg_stat_replication_replay_lsn_lo{job=\"qbit-postgres-primary\",network=\"" ~ qbit_monitoring_stack_network ~ "\",application_name=\"prism_standby_1\"})) > bool 2)) or vector(1)) + (max((pg_stat_replication_primary_flush_lsn_hi{job=\"qbit-postgres-primary\",network=\"" ~ qbit_monitoring_stack_network ~ "\",application_name=\"prism_standby_1\"} offset 5s < bool 0) + ((time() - timestamp(pg_stat_replication_primary_flush_lsn_hi{job=\"qbit-postgres-primary\",network=\"" ~ qbit_monitoring_stack_network ~ "\",application_name=\"prism_standby_1\"} offset 5s)) < bool 5) + ((time() - timestamp(pg_stat_replication_primary_flush_lsn_hi{job=\"qbit-postgres-primary\",network=\"" ~ qbit_monitoring_stack_network ~ "\",application_name=\"prism_standby_1\"} offset 5s)) > bool 7)) or vector(1)) + (max((pg_stat_replication_primary_flush_lsn_lo{job=\"qbit-postgres-primary\",network=\"" ~ qbit_monitoring_stack_network ~ "\",application_name=\"prism_standby_1\"} offset 5s < bool 0) + ((time() - timestamp(pg_stat_replication_primary_flush_lsn_lo{job=\"qbit-postgres-primary\",network=\"" ~ qbit_monitoring_stack_network ~ "\",application_name=\"prism_standby_1\"} offset 5s)) < bool 5) + ((time() - timestamp(pg_stat_replication_primary_flush_lsn_lo{job=\"qbit-postgres-primary\",network=\"" ~ qbit_monitoring_stack_network ~ "\",application_name=\"prism_standby_1\"} offset 5s)) > bool 7)) or vector(1)) + (max((pg_stat_replication_async{job=\"qbit-postgres-primary\",network=\"" ~ qbit_monitoring_stack_network ~ "\",application_name=\"prism_standby_1\"} < bool 0) + ((time() - timestamp(pg_stat_replication_async{job=\"qbit-postgres-primary\",network=\"" ~ qbit_monitoring_stack_network ~ "\",application_name=\"prism_standby_1\"})) < bool 0) + ((time() - timestamp(pg_stat_replication_async{job=\"qbit-postgres-primary\",network=\"" ~ qbit_monitoring_stack_network ~ "\",application_name=\"prism_standby_1\"})) > bool 2)) or vector(1)) + (max(pg_stat_replication_async{job=\"qbit-postgres-primary\",network=\"" ~ qbit_monitoring_stack_network ~ "\",application_name=\"prism_standby_1\"} != bool 1) or vector(1)) + (max((pg_stat_replication_replay_lsn_hi{job=\"qbit-postgres-primary\",network=\"" ~ qbit_monitoring_stack_network ~ "\",application_name=\"prism_standby_1\"} < bool on(job, instance, network, application_name) pg_stat_replication_primary_flush_lsn_hi{job=\"qbit-postgres-primary\",network=\"" ~ qbit_monitoring_stack_network ~ "\",application_name=\"prism_standby_1\"} offset 5s) + ((pg_stat_replication_replay_lsn_hi{job=\"qbit-postgres-primary\",network=\"" ~ qbit_monitoring_stack_network ~ "\",application_name=\"prism_standby_1\"} == bool on(job, instance, network, application_name) pg_stat_replication_primary_flush_lsn_hi{job=\"qbit-postgres-primary\",network=\"" ~ qbit_monitoring_stack_network ~ "\",application_name=\"prism_standby_1\"} offset 5s) * on(job, instance, network, application_name) (pg_stat_replication_replay_lsn_lo{job=\"qbit-postgres-primary\",network=\"" ~ qbit_monitoring_stack_network ~ "\",application_name=\"prism_standby_1\"} < bool on(job, instance, network, application_name) pg_stat_replication_primary_flush_lsn_lo{job=\"qbit-postgres-primary\",network=\"" ~ qbit_monitoring_stack_network ~ "\",application_name=\"prism_standby_1\"} offset 5s))) or vector(1)), 1)", ++ "condition_type": "threshold", ++ "evaluator": "gt", ++ "threshold": 0, ++ "for": "1m", ++ "severity": "warning", ++ "service": "qbit-mining-pool", ++ "labels": { ++ "alert_class": "prism-database-ha" ++ }, ++ "summary": "PrismHAStandbyReplayLagHigh", ++ "description": "D3: standby replay has not reached the primary durable WAL prefix observed five seconds earlier, or position/history/exporter observations are unknown, stale or failed, or the required async topology is invalid. Deployment-provided: requires primary SQL query metrics and one-second scrapes; verify in #281/#291. Idle caught-up positions are healthy even when PostgreSQL reports NULL replay_lag. The five-second comparison is sampled, with up to one scrape interval of timing uncertainty.", ++ "no_data_state": "Alerting", ++ "exec_err_state": "Alerting" ++ }, ++ { ++ "uid": "qbit-prism-ha-standby-disconnected", ++ "title": "PrismHAStandbyDisconnected", ++ "expr": "clamp_max((max(up{job=\"qbit-postgres-primary\",network=\"" ~ qbit_monitoring_stack_network ~ "\"} != bool 1) or vector(1)) + (max(pg_up{job=\"qbit-postgres-primary\",network=\"" ~ qbit_monitoring_stack_network ~ "\"} != bool 1) or vector(1)) + (max(pg_exporter_last_scrape_error{job=\"qbit-postgres-primary\",network=\"" ~ qbit_monitoring_stack_network ~ "\"} != bool 0) or vector(1)) + ((count(up{job=\"qbit-postgres-primary\",network=\"" ~ qbit_monitoring_stack_network ~ "\"}) != bool 1) or vector(1)) + (max((pg_stat_replication_count{job=\"qbit-postgres-primary\",network=\"" ~ qbit_monitoring_stack_network ~ "\",application_name=\"prism_standby_1\"} < bool 0) + ((time() - timestamp(pg_stat_replication_count{job=\"qbit-postgres-primary\",network=\"" ~ qbit_monitoring_stack_network ~ "\",application_name=\"prism_standby_1\"})) < bool 0) + ((time() - timestamp(pg_stat_replication_count{job=\"qbit-postgres-primary\",network=\"" ~ qbit_monitoring_stack_network ~ "\",application_name=\"prism_standby_1\"})) > bool 2)) or vector(1)) + (max(pg_stat_replication_count{job=\"qbit-postgres-primary\",network=\"" ~ qbit_monitoring_stack_network ~ "\",application_name=\"prism_standby_1\"} != bool 1) or vector(1)), 1)", ++ "condition_type": "threshold", ++ "evaluator": "gt", ++ "threshold": 0, ++ "for": "1m", ++ "severity": "warning", ++ "service": "qbit-mining-pool", ++ "labels": { ++ "alert_class": "prism-database-ha" ++ }, ++ "summary": "PrismHAStandbyDisconnected", ++ "description": "D3: the primary reports zero or ambiguous rows for prism_standby_1, or exporter/query observations are missing, stale or failed. Deployment-provided: requires the primary exporter and one-second scrapes; verify in #281/#291. This is the dedicated asynchronous HA standby, not the public read replica.", ++ "no_data_state": "Alerting", ++ "exec_err_state": "Alerting" ++ } ++] %} ++{% set prism_external_stratum_rules = [ + { + "uid": "qbit-prism-stratum-nodata", + "title": "PrismStratumHealthNoData", +@@ -230,752 +965,17 @@ + "value_annotations": ["check_failures_in_window"], + "no_data_state": "OK", + "exec_err_state": "OK" +- }, +- { +- "uid": "qbit-prism-share-writer-backlog", +- "title": "PrismShareWriterBacklog", +- "expr": "min_over_time(qbit_prism_share_append_queue_depth{" ~ prism_selector ~ "}[" ~ qbit_monitoring_stack_prism_alert_backlog_warning_window ~ "]) > " ~ qbit_monitoring_stack_prism_alert_backlog_warning_threshold, +- "condition_ref": "A", +- "evaluator": "gt", +- "threshold": qbit_monitoring_stack_prism_alert_backlog_warning_threshold, +- "for": qbit_monitoring_stack_prism_alert_immediate_for, +- "severity": "warning", +- "service": "qbit-mining-pool", +- "labels": {"alert_class": "prism-runtime"}, +- "summary": "PRISM share-writer backlog is sustained above the warning threshold", +- "description": "The ledger is falling behind. Check Postgres. Queued shares remain safe while the process lives; if needed, restart PRISM gracefully so it drains to Postgres or the fsynced recovery file. Avoid SIGKILL or force-removal while backlogged.", +- "no_data_state": "OK", +- "exec_err_state": "OK" +- }, +- { +- "uid": "qbit-prism-share-writer-backlog-critical", +- "title": "PrismShareWriterBacklogCritical", +- "expr": "min_over_time(qbit_prism_share_append_queue_depth{" ~ prism_selector ~ "}[" ~ qbit_monitoring_stack_prism_alert_backlog_critical_window ~ "]) > " ~ qbit_monitoring_stack_prism_alert_backlog_critical_threshold, +- "condition_ref": "A", +- "evaluator": "gt", +- "threshold": qbit_monitoring_stack_prism_alert_backlog_critical_threshold, +- "for": qbit_monitoring_stack_prism_alert_immediate_for, +- "severity": "critical", +- "service": "qbit-mining-pool", +- "labels": {"alert_class": "prism-runtime", "page": "true"}, +- "summary": "PRISM share-writer backlog is critical", +- "description": "The queue is approaching its 100000-share overflow boundary. Restore Postgres or perform a graceful PRISM restart immediately; an ungraceful death while deeply backlogged can lose already-acked shares.", +- "no_data_state": "OK", +- "exec_err_state": "OK" +- }, +- { +- "uid": "qbit-prism-block-candidate-backlog", +- "title": "PrismBlockCandidateBacklog", +- "expr": (("qbit_prism_block_candidates_pending{" ~ prism_selector ~ "} >= bool " ~ qbit_monitoring_stack_prism_alert_block_candidate_warning_threshold) if (qbit_monitoring_stack_prism_overload_alerts_enabled | bool) else ("min_over_time(qbit_prism_block_candidates_pending{" ~ prism_selector ~ "}[" ~ qbit_monitoring_stack_prism_alert_block_candidate_backlog_window ~ "]) > " ~ qbit_monitoring_stack_prism_alert_block_candidate_backlog_threshold)), +- "condition_ref": "A", +- "evaluator": "gt", +- "threshold": (0 if (qbit_monitoring_stack_prism_overload_alerts_enabled | bool) else qbit_monitoring_stack_prism_alert_block_candidate_backlog_threshold), +- "for": (qbit_monitoring_stack_prism_alert_block_candidate_count_for if (qbit_monitoring_stack_prism_overload_alerts_enabled | bool) else qbit_monitoring_stack_prism_alert_immediate_for), +- "severity": "warning", +- "service": "qbit-mining-pool", +- "labels": {"alert_class": "prism-runtime"}, +- "summary": "PRISM durable block-candidate backlog reached the warning threshold", +- "description": "The block submitter is falling behind candidate production: pending rows in the durable outbox have stayed at or above the warning threshold for the full window. The gauge is database-backed, so restarts do not reset it. Check oldest-pending age and collapse outcomes before intervening: a large fast-draining queue is healthy, while an old head is not. Avoid non-essential coordinator restarts while the backlog is high; pre-paginated bootstraps could fail closed above 1024 rows.", +- "no_data_state": "OK", +- "exec_err_state": "OK" +- }, +- { +- "uid": "qbit-prism-shares-recovered-to-disk", +- "title": "PrismSharesRecoveredToDisk", +- "expr": "increase(qbit_prism_shares_recovered_to_disk_total{" ~ prism_selector ~ "}[" ~ qbit_monitoring_stack_prism_alert_counter_window ~ "]) > 0", +- "condition_ref": "A", +- "evaluator": "gt", +- "threshold": 0, +- "for": qbit_monitoring_stack_prism_alert_immediate_for, +- "severity": "warning", +- "service": "qbit-mining-pool", +- "labels": {"alert_class": "prism-runtime"}, +- "summary": "PRISM is writing acked shares to its recovery file", +- "description": "A ledger outage, shutdown drain, or queue overflow entered the fsynced recovery path. Restore Postgres, keep the audit volume intact, and verify replay and queue recovery.", +- "no_data_state": "OK", +- "exec_err_state": "OK" +- }, +- { +- "uid": "qbit-prism-share-append-failures", +- "title": "PrismShareAppendFailures", +- "expr": "(" ~ prism_share_append_failures_sum ~ " > " ~ qbit_monitoring_stack_prism_alert_share_append_failures_threshold ~ ") or ((" ~ prism_share_append_failures_sum ~ " * ((" ~ prism_share_append_failures_sum ~ " / clamp_min(" ~ prism_share_append_attempts_sum ~ ", 1)) > bool " ~ qbit_monitoring_stack_prism_alert_share_append_failures_ratio ~ ") * (" ~ prism_share_append_failures_sum ~ " >= bool " ~ qbit_monitoring_stack_prism_alert_share_append_failures_ratio_floor ~ ")) > 0)", +- "condition_ref": "A", +- "evaluator": "gt", +- "threshold": qbit_monitoring_stack_prism_alert_share_append_failures_threshold, +- "for": qbit_monitoring_stack_prism_alert_share_append_failures_for, +- "severity": "info", +- "service": "qbit-mining-pool", +- "labels": {"alert_class": "prism-runtime"}, +- "summary": "PRISM ledger share appends are failing at a sustained rate", +- "description": "Across the PRISM fleet, more than " ~ qbit_monitoring_stack_prism_alert_share_append_failures_threshold ~ " shares in " ~ qbit_monitoring_stack_prism_alert_share_append_failures_window ~ " -- or more than " ~ ((qbit_monitoring_stack_prism_alert_share_append_failures_ratio * 100) | int) ~ " percent of ledger-write attempts with at least " ~ qbit_monitoring_stack_prism_alert_share_append_failures_ratio_floor ~ " failures, which also catches a total write outage on a low-volume network -- were in ledger group commits that failed before acknowledgement. Each failed submit is rejected back to its miner as a transient error, so this does not mean acknowledged shares were lost; lost-share risk surfaces through PrismSharesRecoveredToDisk and the backlog alerts. A sustained failure rate is the earliest signal that Postgres is rejecting or timing out writes: check Postgres health and writer logs before the in-memory queue grows.", +- "no_data_state": "OK", +- "exec_err_state": "OK" +- }, +- { +- "uid": "qbit-prism-shares-replayed", +- "title": "PrismSharesReplayed", +- "expr": "increase(qbit_prism_shares_replayed_total{" ~ prism_selector ~ "}[" ~ qbit_monitoring_stack_prism_alert_counter_window ~ "]) > 0", +- "condition_ref": "A", +- "evaluator": "gt", +- "threshold": 0, +- "for": qbit_monitoring_stack_prism_alert_immediate_for, +- "severity": "info", +- "service": "qbit-mining-pool", +- "labels": {"alert_class": "prism-runtime"}, +- "summary": "PRISM replayed recovery-file shares at startup", +- "description": "The prior shutdown left accepted shares outside Postgres. Confirm the ledger is healthy, queue depth returned to zero, and no recovery-file errors remain.", +- "no_data_state": "OK", +- "exec_err_state": "OK" +- }, +- { +- "uid": "qbit-prism-job-build-orphan-evicted", +- "title": "PrismJobBuildOrphanEvictions", +- "expr": "increase(qbit_prism_job_build_orphan_evicted_total{" ~ prism_selector ~ "}[" ~ qbit_monitoring_stack_prism_alert_counter_window ~ "]) > 0", +- "condition_ref": "A", +- "evaluator": "gt", +- "threshold": 0, +- "for": qbit_monitoring_stack_prism_alert_immediate_for, +- "severity": "warning", +- "service": "qbit-mining-pool", +- "labels": {"alert_class": "prism-runtime"}, +- "summary": "PRISM evicted an orphaned job-build flight", +- "description": "The coordinator admission sweep evicted a finished job-build flight whose completion never released its scheduler slot. Each eviction is a defused single-flight wedge that previously escalated into a blockpoll stall and a liveness-watchdog restart. Expect zero in steady state; if it fires, correlate the paired coordinator eviction log line with supersession churn.", +- "no_data_state": "OK", +- "exec_err_state": "OK" +- }, +- { +- "uid": "qbit-prism-stats-reconcile-stale", +- "title": "PrismAcceptedStatsReconcileStale", +- "expr": "max(qbit_prism_accepted_stats_reconcile_age_seconds{" ~ prism_selector ~ "}) > " ~ qbit_monitoring_stack_prism_alert_stats_reconcile_stale_seconds, +- "condition_ref": "A", +- "evaluator": "gt", +- "threshold": qbit_monitoring_stack_prism_alert_stats_reconcile_stale_seconds, +- "for": qbit_monitoring_stack_prism_alert_immediate_for, +- "severity": "warning", +- "service": "qbit-mining-pool", +- "labels": {"alert_class": "prism-runtime"}, +- "summary": "PRISM accepted-stats reconcile is stale", +- "description": "The accepted-share counters have not reconciled against the full ledger within the staleness budget. Bootstrap #110 serves maintained counters without blocking readers, so a persistently failing or wedged background reconcile no longer surfaces as caller errors and is visible only through this age gauge (qbit_prism_accepted_stats_reconcile_failures_total counts failed passes). Steady state keeps the age under PRISM_ACCEPTED_STATS_CACHE_SECONDS plus one aggregate duration; a monotonically growing value means the reconcile thread is wedged (usually a stuck database read) and the coordinator needs a restart.", +- "no_data_state": "OK", +- "exec_err_state": "OK" + } + ] %} +-{% if qbit_monitoring_stack_prism_writer_lease_alerts_enabled | bool %} +-{% if (qbit_monitoring_stack_prism_alert_lease_server_proven_cap_warning_ratio | float) <= 0 or (qbit_monitoring_stack_prism_alert_lease_server_proven_cap_warning_ratio | float) >= 1 %} +-{{ undef(hint="PRISM writer-lease server-proven cap warning ratio must be > 0 and < 1") }} ++{% set alert_rules = prism_native_rules if (qbit_monitoring_stack_prism_alerts_enabled | bool) else [] %} ++{% if (qbit_monitoring_stack_prism_alerts_enabled | bool) and (qbit_monitoring_stack_prism_stratum_alerts_enabled | bool) %} ++{% set alert_rules = alert_rules + prism_external_stratum_rules %} + {% endif %} +-{% if (qbit_monitoring_stack_prism_alert_lease_monitor_wake_warning_ratio | float) <= 0 or (qbit_monitoring_stack_prism_alert_lease_monitor_wake_warning_ratio | float) >= 1 %} +-{{ undef(hint="PRISM writer-lease monitor-wake warning ratio must be > 0 and < 1") }} +-{% endif %} +-{# The rule now reads a counter the coordinator labels by slack fraction, and it +- exports exactly "0.5", "0.8" and "1.0" (LEASE_MONITOR_LATE_WAKE_SLACK_FRACTIONS, +- upstream PR #233). A ratio outside that set would render a selector matching no +- series, which evaluates to no data and silently never fires -- worse than the +- gauge it replaces. Refuse to render instead. "1.0" is exported but excluded by +- the range guard above. It means 100% of scheduler slack, not the separate +- exit-guarantee boundary (slack plus one monitor interval). #} +-{% if (qbit_monitoring_stack_prism_alert_lease_monitor_wake_warning_ratio | float | string) not in ["0.5", "0.8"] %} +-{{ undef(hint="PRISM writer-lease monitor-wake warning ratio must be one of the coordinator's exported slack_fraction labels: 0.5 or 0.8") }} +-{% endif %} +-{% endif %} +-{% if (qbit_monitoring_stack_prism_alert_lease_monitor_wake_recovery_threshold | float) <= 0 or (qbit_monitoring_stack_prism_alert_lease_monitor_wake_recovery_threshold | float) >= 1 or (qbit_monitoring_stack_prism_alert_lease_monitor_wake_count_threshold | float) <= (qbit_monitoring_stack_prism_alert_lease_monitor_wake_recovery_threshold | float) %} +-{{ undef(hint="PRISM monitor-wake thresholds require 0 < recovery < 1 and firing > recovery") }} +-{% endif %} +-{# Grafana 11.5.2 forgets native recovery-threshold dimensions after KeepLast +- NoData/Error. Reconstruct a bounded hold from Prometheus history instead. +- A always returns numeric zero on a healthy, non-qualifying series; missing +- telemetry remains missing. The one-minute subquery matches rule cadence. #} +-{% set lease_wake_count = "sum without(slack_fraction) (increase(qbit_prism_lease_heartbeat_monitor_late_wakes_total{" ~ prism_selector ~ ",slack_fraction=\"" ~ (qbit_monitoring_stack_prism_alert_lease_monitor_wake_warning_ratio | float | string) ~ "\"}[" ~ qbit_monitoring_stack_prism_alert_lease_monitor_wake_recent_window ~ "]))" %} +-{% set lease_wake_frequent = "(" ~ lease_wake_count ~ " > " ~ qbit_monitoring_stack_prism_alert_lease_monitor_wake_count_threshold ~ ")" %} +-{% set lease_wake_held = "((max_over_time((" ~ lease_wake_count ~ ")[" ~ qbit_monitoring_stack_prism_alert_lease_monitor_wake_hold_window ~ ":1m]) > " ~ qbit_monitoring_stack_prism_alert_lease_monitor_wake_count_threshold ~ ") and (" ~ lease_wake_count ~ " >= " ~ qbit_monitoring_stack_prism_alert_lease_monitor_wake_recovery_threshold ~ "))" %} +-{% set lease_runbook = "https://github.com/SwapLabsInc/qbit-tools/blob/main/ansible/PRISM_LEASE_ALERTS.md" %} +-{# An empty optional query makes the entire Grafana 11.5.2 evaluation NoData. +- Supply a labelled -1 sentinel only where the primary counter has data; this +- preserves genuine primary NoData and makes missing context non-blocking. +- Match fallback by target identity so extra diagnostic labels cannot leave +- an exact-label sentinel that hides the real value from Grafana. #} +-{% set lease_guarantee_count = "increase(qbit_prism_lease_heartbeat_monitor_exit_guarantee_breaches_total{" ~ prism_selector ~ "}[" ~ qbit_monitoring_stack_prism_alert_lease_exit_guarantee_window ~ "])" %} +-{% set lease_wake_context_missing = "(" ~ lease_wake_count ~ " * 0 - 1)" %} +-{% set lease_guarantee_context_missing = "(" ~ lease_guarantee_count ~ " * 0 - 1)" %} +-{% set lease_peak_expr = "qbit_prism_lease_heartbeat_monitor_wake_delay_window_max_seconds{" ~ prism_selector ~ "} * 1000" %} +-{% set lease_peak_text = "Current five-minute peak: [[ if and $values.D.Labels (ge $values.D.Value 0.0) ]][[ printf \"%.0f\" $values.D.Value ]] ms[[ else ]]unavailable[[ end ]]. " %} +-{% set prism_writer_lease_alert_rules = [ +- { +- "uid": "qbit-prism-lease-proven-age", +- "title": "PrismWriterLeaseServerProvenAgeNearCap", +- "expr": "max_over_time(qbit_prism_lease_heartbeat_server_proven_age_seconds{" ~ prism_selector ~ "}[" ~ qbit_monitoring_stack_prism_alert_lease_server_proven_window ~ "]) >= bool on(job, instance) (qbit_prism_lease_heartbeat_policy_seconds{" ~ prism_selector ~ ",term=\"server_proven_cap\"} * " ~ qbit_monitoring_stack_prism_alert_lease_server_proven_cap_warning_ratio ~ ")", +- "condition_ref": "A", +- "evaluator": "gt", +- "threshold": 0, +- "for": qbit_monitoring_stack_prism_alert_lease_headroom_warning_for, +- "severity": "warning", +- "service": "qbit-mining-pool", +- "labels": {"alert_class": "prism-runtime"}, +- "summary": "PRISM writer-lease proof age is approaching its hard-exit cap", +- "description": "The newest completed guard round trip reached the configured warning fraction of the validated server-proven cap. Compare guard-slot, PostgreSQL, and scheduler phase metrics before a healthy coordinator exhausts its heartbeat envelope.", +- "no_data_state": "OK", +- "exec_err_state": "OK" +- }, +- { +- "uid": "qbit-prism-lease-monitor-wake", +- "title": "PrismWriterLeaseMonitorWakeDelayHigh", +- "expr": "(" ~ lease_wake_count ~ " and (" ~ lease_wake_frequent ~ " or " ~ lease_wake_held ~ ")) or (" ~ lease_wake_count ~ " * 0)", +- "condition_type": "threshold", +- "evaluator": "gt", +- "threshold": 0, +- "for": qbit_monitoring_stack_prism_alert_lease_monitor_wake_for, +- "severity": "warning", +- "service": "qbit-mining-pool", +- "labels": {"alert_class": "prism-runtime", "alert_signal": "lease-monitor"}, +- "context_queries": [ +- {"ref_id": "C", "expr": "max without(term) (qbit_prism_lease_heartbeat_policy_seconds{" ~ prism_selector ~ ",term=\"scheduler_slack\"}) * " ~ qbit_monitoring_stack_prism_alert_lease_monitor_wake_warning_ratio ~ " * 1000 or on(job, instance, network) " ~ lease_wake_context_missing}, +- {"ref_id": "D", "expr": lease_peak_expr ~ " or on(job, instance, network) " ~ lease_wake_context_missing} +- ], +- "summary": "[[ $labels.network ]] PRISM: frequent near-limit monitor wakes on [[ $labels.instance ]]", +- "description": "Approximately [[ printf \"%.1f\" $values.A.Value ]] wakes at least " ~ (((qbit_monitoring_stack_prism_alert_lease_monitor_wake_warning_ratio | float) * 100) | int | string) ~ "% of scheduler slack late in " ~ qbit_monitoring_stack_prism_alert_lease_monitor_wake_recent_window ~ ". Wake boundary: [[ if and $values.C.Labels (ge $values.C.Value 0.0) ]][[ printf \"%.0f\" $values.C.Value ]] ms[[ else ]]unavailable[[ end ]]. " ~ lease_peak_text ~ "Investigate coordinator scheduling pressure and guard phases. This frequency episode holds for up to " ~ qbit_monitoring_stack_prism_alert_lease_monitor_wake_hold_window ~ " after activity drops below the frequency threshold, and clears early when no observed wakes remain. Timing-guarantee breaches have a separate alert.", +- "annotations": {"runbook_url": lease_runbook}, +- "no_data_state": "KeepLast", +- "exec_err_state": "KeepLast" +- }, +- { +- "uid": "qbit-prism-lease-monitor-guarantee", +- "title": "PrismWriterLeaseMonitorExitGuaranteeBreached", +- "expr": lease_guarantee_count, +- "condition_type": "threshold", +- "evaluator": "gt", +- "threshold": 0, +- "for": "0m", +- "severity": "warning", +- "service": "qbit-mining-pool", +- "labels": {"alert_class": "prism-runtime", "alert_signal": "lease-monitor"}, +- "context_queries": [ +- {"ref_id": "C", "expr": "max without(term) (qbit_prism_lease_heartbeat_policy_seconds{" ~ prism_selector ~ ",term=\"max_guaranteed_monitor_lateness\"}) * 1000 or on(job, instance, network) " ~ lease_guarantee_context_missing}, +- {"ref_id": "D", "expr": lease_peak_expr ~ " or on(job, instance, network) " ~ lease_guarantee_context_missing} +- ], +- "summary": "[[ $labels.network ]] PRISM: monitor timing guarantee exceeded on [[ $labels.instance ]]", +- "description": "Approximately [[ printf \"%.1f\" $values.A.Value ]] exit-guarantee breaches observed in " ~ qbit_monitoring_stack_prism_alert_lease_exit_guarantee_window ~ ". Current policy limit: [[ if and $values.C.Labels (ge $values.C.Value 0.0) ]][[ printf \"%.0f\" $values.C.Value ]] ms[[ else ]]unavailable[[ end ]]. " ~ lease_peak_text ~ "Investigate immediately and correlate lease-exit logs. Exit-before-adoption was not guaranteed for these beats; this does not prove an unsafe write. A fatal event before the next scrape can escape this process-local counter.", +- "annotations": {"runbook_url": lease_runbook}, +- "no_data_state": "KeepLast", +- "exec_err_state": "KeepLast" +- }, +- { +- "uid": "qbit-prism-lease-surplus-invalid", +- "title": "PrismWriterLeaseStabilitySurplusExhausted", +- "expr": "qbit_prism_lease_heartbeat_policy_seconds{" ~ prism_selector ~ ",term=\"stability_surplus\"} <= bool 0", +- "condition_ref": "A", +- "evaluator": "gt", +- "threshold": 0, +- "for": qbit_monitoring_stack_prism_alert_lease_stability_surplus_for, +- "severity": "warning", +- "service": "qbit-mining-pool", +- "labels": {"alert_class": "prism-runtime"}, +- "summary": "PRISM writer-lease policy has no healthy tail-latency surplus", +- "description": "The validated safety policy is running with a server-proven cap at or below the largest healthy heartbeat gap. Split-brain safety remains fail-closed, but ordinary lawful tail latency can now restart a healthy coordinator; restore positive surplus immediately.", +- "no_data_state": "OK", +- "exec_err_state": "OK" +- } +-] %} +-{# The preview-publication boundaries double as histogram le-label +- selectors. A value that is not the exact %g rendering of an exported +- bucket boundary (lab/prism/block_candidates.py +- PRISM_ACCEPTED_BLOCK_PREVIEW_PUBLICATION_SECONDS_BUCKETS) would match no +- series, and with no_data_state OK the latency rule would go silently +- dead -- so fail the render instead. #} +-{% set prism_preview_bucket_labels = ["0.01", "0.05", "0.1", "0.25", "0.5", "1", "1.5", "2", "3", "4", "5", "7.5", "10", "20", "30", "60"] %} +-{% for prism_preview_boundary in [qbit_monitoring_stack_prism_alert_preview_publication_warning_seconds, qbit_monitoring_stack_prism_alert_preview_publication_critical_seconds] %} +-{% if (prism_preview_boundary | string) not in prism_preview_bucket_labels %} +-{{ undef(hint="prism preview publication boundary " ~ prism_preview_boundary ~ " is not the exact rendering of an exported histogram bucket boundary; use one of: " ~ (prism_preview_bucket_labels | join(", "))) }} +-{% endif %} +-{% endfor %} +-{% if qbit_monitoring_stack_prism_alert_preview_publication_breach_count < 1 or qbit_monitoring_stack_prism_alert_preview_publication_warning_breach_count < 1 %} +-{{ undef(hint="prism preview publication breach counts must be >= 1") }} +-{% endif %} +-{% if (qbit_monitoring_stack_prism_alert_cleanup_backlog_near_bound_ratio | float) <= 0 or (qbit_monitoring_stack_prism_alert_cleanup_backlog_near_bound_ratio | float) >= 1 %} +-{{ undef(hint="PRISM cleanup-backlog near-bound ratio must be > 0 and < 1") }} +-{% endif %} +-{% set prism_preview_published_increase = "sum by (job, instance) (increase(qbit_prism_accepted_block_preview_publication_seconds_bucket{" ~ prism_selector ~ ",result=\"published\",le=\"%s\"}[" ~ qbit_monitoring_stack_prism_alert_preview_publication_breach_window ~ "]))" %} +-{% set prism_overload_alert_rules = [ +- { +- "uid": "qbit-prism-preview-publish-latency", +- "title": "PrismAcceptedPreviewPublicationLatencyHigh", +- "expr": "((" ~ (prism_preview_published_increase | format("+Inf")) ~ ") - (" ~ (prism_preview_published_increase | format(qbit_monitoring_stack_prism_alert_preview_publication_warning_seconds)) ~ ")) > bool " ~ (qbit_monitoring_stack_prism_alert_preview_publication_warning_breach_count - 0.5), +- "condition_ref": "A", +- "evaluator": "gt", +- "threshold": 0, +- "for": qbit_monitoring_stack_prism_alert_immediate_for, +- "severity": "warning", +- "service": "qbit-mining-pool", +- "labels": {"alert_class": "prism-runtime"}, +- "summary": "PRISM accepted-block preview publication is slowing", +- "description": "Multiple publications strictly exceeded the warning boundary from definitive qbitd acceptance to payout-preview visibility, counted against the exact warning histogram boundary so the signal is independent of block cadence. Inspect accepted accounting, dequeue collapse, unresolved-parent age, and child preview waits before the fixed five-second boundary is reached.", +- "no_data_state": "OK", +- "exec_err_state": "OK" +- }, +- { +- "uid": "qbit-prism-preview-publish-latency-crit", +- "title": "PrismAcceptedPreviewPublicationLatencyCritical", +- "expr": "((" ~ (prism_preview_published_increase | format("+Inf")) ~ ") - (" ~ (prism_preview_published_increase | format(qbit_monitoring_stack_prism_alert_preview_publication_critical_seconds)) ~ ")) > bool " ~ (qbit_monitoring_stack_prism_alert_preview_publication_breach_count - 0.5), +- "condition_ref": "A", +- "evaluator": "gt", +- "threshold": 0, +- "for": qbit_monitoring_stack_prism_alert_immediate_for, +- "severity": "warning", +- "service": "qbit-mining-pool", +- "labels": {"alert_class": "prism-runtime"}, +- "summary": "PRISM accepted-block preview publication exceeded its contract", +- "description": "Multiple published-result publications strictly exceeded the " ~ qbit_monitoring_stack_prism_alert_preview_publication_critical_seconds ~ " s critical boundary (the fixed five-second child wait budget by default) inside the breach window. Counted against the exact histogram boundary, so any block cadence that can produce the configured breach count alerts identically -- there is no sample-size quantile to degenerate; a single slow publication (for example one back-to-back landing pair) does not alert. A pool finding fewer blocks per window than the breach count cannot reach this alert by construction: sustained slow publications there surface through the warning tier at its lower count, and fail-closed depth pages through PrismUnresolvedParentDepthAtCap; tune the per-network breach counts down if a chronically low-cadence deployment should alert on fewer breaches. The rule fires immediately once the count is met -- the window already integrates over time. Stop external load and restore accepted-accounting and publication progress before continuing.", +- "no_data_state": "OK", +- "exec_err_state": "OK" +- }, +- { +- "uid": "qbit-prism-preview-publish-degraded", +- "title": "PrismAcceptedPreviewPublicationDegraded", +- "expr": "sum(increase(qbit_prism_accepted_block_preview_publication_seconds_count{" ~ prism_selector ~ ",result=\"degraded\"}[" ~ qbit_monitoring_stack_prism_alert_preview_publication_window ~ "])) > bool 0", +- "condition_ref": "A", +- "evaluator": "gt", +- "threshold": 0, +- "for": qbit_monitoring_stack_prism_alert_immediate_for, +- "severity": "warning", +- "service": "qbit-mining-pool", +- "labels": {"alert_class": "prism-runtime"}, +- "summary": "PRISM published an accepted-block preview through its degraded path", +- "description": "At least one accepted block woke child work through fenced local preview retention instead of atomic payout-generation publication. Correlate publication failures, accepted accounting, and refresh health; require zero occurrences for mainnet promotion.", +- "no_data_state": "OK", +- "exec_err_state": "OK" +- }, +- { +- "uid": "qbit-prism-parent-preview-timeouts", +- "title": "PrismAcceptedParentPreviewWaitTimeouts", +- "expr": "increase(qbit_prism_accepted_parent_preview_wait_timeouts_total{" ~ prism_selector ~ "}[" ~ qbit_monitoring_stack_prism_alert_parent_preview_window ~ "]) > bool " ~ qbit_monitoring_stack_prism_alert_parent_preview_warning_threshold, +- "condition_ref": "A", +- "evaluator": "gt", +- "threshold": 0, +- "for": qbit_monitoring_stack_prism_alert_parent_preview_warning_for, +- "severity": "warning", +- "service": "qbit-mining-pool", +- "labels": {"alert_class": "prism-runtime"}, +- "summary": "PRISM child jobs are timing out behind parent accounting", +- "description": "At least one child job waited the full accepted-parent payout-preview budget and failed closed. Check unresolved-parent depth and age, then landing-class ledger call timeouts.", +- "no_data_state": "OK", +- "exec_err_state": "OK" +- }, +- { +- "uid": "qbit-prism-parent-preview-timeouts-crit", +- "title": "PrismAcceptedParentPreviewWaitTimeoutsCritical", +- "expr": "increase(qbit_prism_accepted_parent_preview_wait_timeouts_total{" ~ prism_selector ~ "}[" ~ qbit_monitoring_stack_prism_alert_parent_preview_window ~ "]) > bool " ~ qbit_monitoring_stack_prism_alert_parent_preview_critical_threshold, +- "condition_ref": "A", +- "evaluator": "gt", +- "threshold": 0, +- "for": qbit_monitoring_stack_prism_alert_parent_preview_critical_for, +- "severity": "warning", +- "service": "qbit-mining-pool", +- "labels": {"alert_class": "prism-runtime"}, +- "summary": "PRISM parent-preview timeouts are sustained", +- "description": "Parent bookkeeping is repeatedly blocking child job issuance. The count threshold is cadence-dependent; stop load, inspect the accounting lane, and re-baseline before mainnet use.", +- "no_data_state": "OK", +- "exec_err_state": "OK" +- }, +- { +- "uid": "qbit-prism-unresolved-parent-depth", +- "title": "PrismUnresolvedParentDepthNonzero", +- "expr": "qbit_prism_accepted_parent_unresolved_transitions{" ~ prism_selector ~ "} > bool 0", +- "condition_ref": "A", +- "evaluator": "gt", +- "threshold": 0, +- "for": qbit_monitoring_stack_prism_alert_unresolved_parent_nonzero_for, +- "severity": "warning", +- "service": "qbit-mining-pool", +- "labels": {"alert_class": "prism-runtime"}, +- "summary": "PRISM accepted-parent accounting is not draining", +- "description": "An unresolved accepted-parent transition has persisted beyond the normal landing tail. Check landing ledger timings and the block-accounting lane.", +- "no_data_state": "OK", +- "exec_err_state": "OK" +- }, +- { +- "uid": "qbit-prism-unresolved-parent-near-cap", +- "title": "PrismUnresolvedParentDepthNearCap", +- "expr": "qbit_prism_accepted_parent_unresolved_transitions{" ~ prism_selector ~ "} >= bool on(job, instance) (qbit_prism_accepted_parent_unresolved_depth_max{" ~ prism_selector ~ "} - 1)", +- "condition_ref": "A", +- "evaluator": "gt", +- "threshold": 0, +- "for": qbit_monitoring_stack_prism_alert_unresolved_parent_near_cap_for, +- "severity": "warning", +- "service": "qbit-mining-pool", +- "labels": {"alert_class": "prism-runtime"}, +- "summary": "PRISM unresolved-parent depth is one transition from its cap", +- "description": "One more landed transition can stop child-job admission. Reduce load and restore the accounting lane before the configured fence closes.", +- "no_data_state": "OK", +- "exec_err_state": "OK" +- }, +- { +- "uid": "qbit-prism-unresolved-parent-at-cap", +- "title": "PrismUnresolvedParentDepthAtCap", +- "expr": "qbit_prism_accepted_parent_unresolved_transitions{" ~ prism_selector ~ "} >= bool on(job, instance) qbit_prism_accepted_parent_unresolved_depth_max{" ~ prism_selector ~ "}", +- "condition_ref": "A", +- "evaluator": "gt", +- "threshold": 0, +- "for": qbit_monitoring_stack_prism_alert_unresolved_parent_at_cap_for, +- "severity": "critical", +- "service": "qbit-mining-pool", +- "labels": {"alert_class": "prism-runtime", "page": "true"}, +- "summary": "PRISM unresolved-parent depth reached its admission cap", +- "description": "Child job issuance is already failing closed with TemplateRefreshBlocked. Stop load and repair the accounting lane immediately.", +- "no_data_state": "OK", +- "exec_err_state": "OK" +- }, +- { +- "uid": "qbit-prism-unresolved-parent-age", +- "title": "PrismUnresolvedParentAgeOverBudget", +- "expr": "qbit_prism_accepted_parent_unresolved_oldest_seconds{" ~ prism_selector ~ "} > bool " ~ qbit_monitoring_stack_prism_alert_unresolved_parent_age_warning_seconds, +- "condition_ref": "A", +- "evaluator": "gt", +- "threshold": 0, +- "for": qbit_monitoring_stack_prism_alert_unresolved_parent_age_warning_for, +- "severity": "warning", +- "service": "qbit-mining-pool", +- "labels": {"alert_class": "prism-runtime"}, +- "summary": "PRISM oldest unresolved parent exceeded the preview budget", +- "description": "A timestamped accepted-parent transition is old enough to make child builds time out. The depth gauge is authoritative when this age gauge is -1.", +- "no_data_state": "OK", +- "exec_err_state": "OK" +- }, +- { +- "uid": "qbit-prism-unresolved-parent-age-crit", +- "title": "PrismUnresolvedParentAgeCritical", +- "expr": "qbit_prism_accepted_parent_unresolved_oldest_seconds{" ~ prism_selector ~ "} > bool " ~ qbit_monitoring_stack_prism_alert_unresolved_parent_age_critical_seconds, +- "condition_ref": "A", +- "evaluator": "gt", +- "threshold": 0, +- "for": qbit_monitoring_stack_prism_alert_unresolved_parent_age_critical_for, +- "severity": "warning", +- "service": "qbit-mining-pool", +- "labels": {"alert_class": "prism-runtime"}, +- "summary": "PRISM accepted-parent bookkeeping is critically old", +- "description": "The oldest timestamped transition has remained unresolved for more than the critical budget. Stop load and inspect landing/accounting progress.", +- "no_data_state": "OK", +- "exec_err_state": "OK" +- }, +- { +- "uid": "qbit-prism-candidate-backlog-critical", +- "title": "PrismBlockCandidateBacklogCritical", +- "expr": "qbit_prism_block_candidates_pending{" ~ prism_selector ~ "} >= bool " ~ qbit_monitoring_stack_prism_alert_block_candidate_critical_threshold, +- "condition_ref": "A", +- "evaluator": "gt", +- "threshold": 0, +- "for": qbit_monitoring_stack_prism_alert_block_candidate_count_for, +- "severity": "warning", +- "service": "qbit-mining-pool", +- "labels": {"alert_class": "prism-runtime"}, +- "summary": "PRISM durable block-candidate backlog is critical", +- "description": "Pending durable candidates have stayed at or above the testnet-derived critical threshold. Check oldest age, collapse throughput, submitter backoff, and qbitd latency; stop load if the queue is not draining.", +- "no_data_state": "OK", +- "exec_err_state": "OK" +- }, +- { +- "uid": "qbit-prism-block-candidate-oldest", +- "title": "PrismBlockCandidateOldestPending", +- "expr": "qbit_prism_block_candidate_oldest_pending_seconds{" ~ prism_selector ~ "} > bool " ~ qbit_monitoring_stack_prism_alert_block_candidate_age_warning_seconds, +- "condition_ref": "A", +- "evaluator": "gt", +- "threshold": 0, +- "for": qbit_monitoring_stack_prism_alert_block_candidate_age_for, +- "severity": "warning", +- "service": "qbit-mining-pool", +- "labels": {"alert_class": "prism-runtime"}, +- "summary": "PRISM oldest pending block candidate is not draining", +- "description": "The durable queue head is older than the warning budget. Prefer this cadence-independent age signal over candidate count and inspect submitter retry backoff.", +- "no_data_state": "OK", +- "exec_err_state": "OK" +- }, +- { +- "uid": "qbit-prism-candidate-oldest-critical", +- "title": "PrismBlockCandidateOldestPendingCritical", +- "expr": "qbit_prism_block_candidate_oldest_pending_seconds{" ~ prism_selector ~ "} > bool " ~ qbit_monitoring_stack_prism_alert_block_candidate_age_critical_seconds, +- "condition_ref": "A", +- "evaluator": "gt", +- "threshold": 0, +- "for": qbit_monitoring_stack_prism_alert_block_candidate_age_for, +- "severity": "critical", +- "service": "qbit-mining-pool", +- "labels": {"alert_class": "prism-runtime", "page": "true"}, +- "summary": "PRISM oldest pending block candidate is critically old", +- "description": "A durable candidate has remained pending for more than five minutes. Stop load and inspect collapse, node submission, and the accounting lane.", +- "no_data_state": "OK", +- "exec_err_state": "OK" +- }, +- { +- "uid": "qbit-prism-candidate-metrics-unavailable", +- "title": "PrismBlockCandidateMetricsUnavailable", +- "expr": "qbit_prism_block_candidates_pending{" ~ prism_selector ~ "} == bool -1", +- "condition_ref": "A", +- "evaluator": "gt", +- "threshold": 0, +- "for": qbit_monitoring_stack_prism_alert_block_candidate_unavailable_for, +- "severity": "warning", +- "service": "qbit-mining-pool", +- "labels": {"alert_class": "prism-runtime"}, +- "summary": "PRISM block-candidate store metrics are unavailable", +- "description": "The -1 sentinel means the coordinator could not read durable candidate state. Do not interpret count or age silence as an empty queue.", +- "no_data_state": "OK", +- "exec_err_state": "OK" +- }, +- { +- "uid": "qbit-prism-cleanup-backlog-depth", +- "title": "PrismCollapseCleanupBacklog", +- "expr": "qbit_prism_block_candidate_cleanup_retry_backlog{" ~ prism_selector ~ "} > bool 0", +- "condition_ref": "A", +- "evaluator": "gt", +- "threshold": 0, +- "for": qbit_monitoring_stack_prism_alert_cleanup_backlog_depth_for, +- "severity": "warning", +- "service": "qbit-mining-pool", +- "labels": {"alert_class": "prism-runtime"}, +- "summary": "PRISM collapsed candidates are owing in-memory cleanup", +- "description": "Terminal rows are holding cleanup retry records that the accounting lane is not finishing. A depth that survives ten minutes means one cleanup dependency is failing persistently; check the cleanup_retry_failed collapse outcome and the coordinator log for the named failing step.", +- "no_data_state": "OK", +- "exec_err_state": "OK" +- }, +- { +- "uid": "qbit-prism-cleanup-backlog-oldest", +- "title": "PrismCollapseCleanupBacklogOldest", +- "expr": "qbit_prism_block_candidate_cleanup_retry_oldest_seconds{" ~ prism_selector ~ "} > bool " ~ qbit_monitoring_stack_prism_alert_cleanup_backlog_oldest_seconds, +- "condition_ref": "A", +- "evaluator": "gt", +- "threshold": 0, +- "for": qbit_monitoring_stack_prism_alert_cleanup_backlog_oldest_for, +- "severity": "warning", +- "service": "qbit-mining-pool", +- "labels": {"alert_class": "prism-runtime"}, +- "summary": "PRISM oldest owed cleanup retry is not draining", +- "description": "The oldest cleanup retry record is past the warning budget, roughly seventeen consecutive failed attempts under the shipped backoff. Prefer this cadence-independent age signal over backlog depth and inspect the failing cleanup dependency.", +- "no_data_state": "OK", +- "exec_err_state": "OK" +- }, +- { +- "uid": "qbit-prism-cleanup-backlog-near-bound", +- "title": "PrismCollapseCleanupBacklogNearBound", +- "expr": "qbit_prism_block_candidate_cleanup_retry_backlog{" ~ prism_selector ~ "} >= bool on(job, instance) (qbit_prism_block_candidate_cleanup_retry_backlog_max{" ~ prism_selector ~ "} * " ~ qbit_monitoring_stack_prism_alert_cleanup_backlog_near_bound_ratio ~ ")", +- "condition_ref": "A", +- "evaluator": "gt", +- "threshold": 0, +- "for": qbit_monitoring_stack_prism_alert_cleanup_backlog_near_bound_for, +- "severity": "warning", +- "service": "qbit-mining-pool", +- "labels": {"alert_class": "prism-runtime"}, +- "summary": "PRISM cleanup-retry backlog is approaching its bound", +- "description": "The backlog is at or above the warning fraction of the coordinator's exported bound. At the bound the collapse stops bulk-terminalizing new rows and preserved siblings fall back to the slow per-row path. Restore the failing cleanup dependency before it engages.", +- "no_data_state": "OK", +- "exec_err_state": "OK" +- }, +- { +- "uid": "qbit-prism-cleanup-backpressure-active", +- "title": "PrismCollapseCleanupBackpressure", +- "expr": "qbit_prism_block_candidate_cleanup_backpressure_active{" ~ prism_selector ~ "} == bool 1", +- "condition_ref": "A", +- "evaluator": "gt", +- "threshold": 0, +- "for": qbit_monitoring_stack_prism_alert_cleanup_backpressure_active_for, +- "severity": "warning", +- "service": "qbit-mining-pool", +- "labels": {"alert_class": "prism-runtime"}, +- "summary": "PRISM collapse cleanup is at its retry-backlog bound", +- "description": "Bulk terminalization has stopped admitting new rows to protect coordinator memory. Declined rows remain durable and pending, draining one at a time with a submitblock, about six chain reads, and two ledger writes each. This is the intended degradation, not a second fault: fix the failing cleanup dependency rather than raising the bound.", +- "no_data_state": "OK", +- "exec_err_state": "OK" +- }, +- { +- "uid": "qbit-prism-cleanup-backpressure-engaged", +- "title": "PrismCollapseCleanupBackpressureEngaged", +- "expr": "sum by (job, instance) (increase(qbit_prism_block_candidate_cleanup_backpressure_total{" ~ prism_selector ~ "}[" ~ qbit_monitoring_stack_prism_alert_cleanup_backpressure_engaged_window ~ "])) > bool 0", +- "condition_ref": "A", +- "evaluator": "gt", +- "threshold": 0, +- "for": qbit_monitoring_stack_prism_alert_immediate_for, +- "severity": "warning", +- "service": "qbit-mining-pool", +- "labels": {"alert_class": "prism-runtime"}, +- "summary": "PRISM collapse cleanup backpressure engaged recently", +- "description": "The cleanup-retry backlog reached its bound at least once in the window even if it is not engaged now. Correlate with the backlog_deferred collapse outcome and the pending-candidate drain shape before treating the recovery as complete.", +- "no_data_state": "OK", +- "exec_err_state": "OK" +- }, +- { +- "uid": "qbit-prism-semantic-work-coverage", +- "title": "PrismSemanticWorkCoverageLoss", +- "expr": "(qbit_prism_stratum_semantic_current_work_ratio{" ~ prism_selector ~ "} < bool " ~ qbit_monitoring_stack_prism_alert_semantic_coverage_warning_ratio ~ ") * on(job, instance) (qbit_prism_stratum_authorized_connections{" ~ prism_selector ~ "} > bool 0)", +- "condition_ref": "A", +- "evaluator": "gt", +- "threshold": 0, +- "for": qbit_monitoring_stack_prism_alert_semantic_coverage_warning_for, +- "severity": "warning", +- "service": "qbit-mining-pool", +- "labels": {"alert_class": "prism-runtime"}, +- "summary": "PRISM miners are on semantically superseded work", +- "description": "Authorized clients are mining work that no longer matches the current template fingerprint and payout generation. Check refresh and job-delivery progress.", +- "no_data_state": "OK", +- "exec_err_state": "OK" +- }, +- { +- "uid": "qbit-prism-semantic-coverage-critical", +- "title": "PrismSemanticWorkCoverageLossCritical", +- "expr": "(qbit_prism_stratum_semantic_current_work_ratio{" ~ prism_selector ~ "} < bool " ~ qbit_monitoring_stack_prism_alert_semantic_coverage_critical_ratio ~ ") * on(job, instance) (qbit_prism_stratum_authorized_connections{" ~ prism_selector ~ "} > bool 0)", +- "condition_ref": "A", +- "evaluator": "gt", +- "threshold": 0, +- "for": qbit_monitoring_stack_prism_alert_semantic_coverage_critical_for, +- "severity": "critical", +- "service": "qbit-mining-pool", +- "labels": {"alert_class": "prism-runtime", "page": "true"}, +- "summary": "Most PRISM miners are on semantically superseded work", +- "description": "Semantic current-work coverage is below the critical ratio while clients remain authorized. Stop load and restore refresh/job delivery.", +- "no_data_state": "OK", +- "exec_err_state": "OK" +- }, +- { +- "uid": "qbit-prism-refresh-pending", +- "title": "PrismRefreshPendingPastDeadline", +- "expr": "(qbit_prism_refresh_pending{" ~ prism_selector ~ "} == bool 1) * on(job, instance) (qbit_prism_refresh_pending_age_seconds{" ~ prism_selector ~ "} > bool " ~ qbit_monitoring_stack_prism_alert_refresh_pending_warning_seconds ~ ")", +- "condition_ref": "A", +- "evaluator": "gt", +- "threshold": 0, +- "for": qbit_monitoring_stack_prism_alert_refresh_pending_warning_for, +- "severity": "warning", +- "service": "qbit-mining-pool", +- "labels": {"alert_class": "prism-runtime"}, +- "summary": "PRISM current-work refresh exceeded its health deadline", +- "description": "Template or payout work still needs publication/delivery beyond the configured default health deadline. Check semantic coverage and coordinator progress.", +- "no_data_state": "OK", +- "exec_err_state": "OK" +- }, +- { +- "uid": "qbit-prism-refresh-pending-critical", +- "title": "PrismRefreshPendingPastDeadlineCritical", +- "expr": "(qbit_prism_refresh_pending{" ~ prism_selector ~ "} == bool 1) * on(job, instance) (qbit_prism_refresh_pending_age_seconds{" ~ prism_selector ~ "} > bool " ~ qbit_monitoring_stack_prism_alert_refresh_pending_critical_seconds ~ ")", +- "condition_ref": "A", +- "evaluator": "gt", +- "threshold": 0, +- "for": qbit_monitoring_stack_prism_alert_refresh_pending_critical_for, +- "severity": "critical", +- "service": "qbit-mining-pool", +- "labels": {"alert_class": "prism-runtime", "page": "true"}, +- "summary": "PRISM current-work refresh is critically overdue", +- "description": "Current template or payout work has remained unpublished for more than the critical budget. Stop load and restore coordinator progress.", +- "no_data_state": "OK", +- "exec_err_state": "OK" +- }, +- { +- "uid": "qbit-prism-metrics-snapshot-stale", +- "title": "PrismMetricsSnapshotStale", +- "expr": "qbit_prism_metrics_snapshot_stale{" ~ prism_selector ~ "} == bool 1", +- "condition_ref": "A", +- "evaluator": "gt", +- "threshold": 0, +- "for": qbit_monitoring_stack_prism_alert_metrics_snapshot_for, +- "severity": "warning", +- "service": "qbit-mining-pool", +- "labels": {"alert_class": "prism-runtime"}, +- "summary": "PRISM is serving a stale complete metrics snapshot", +- "description": "The post-#184 endpoint remains scrapeable, but collection is not advancing. Treat body metrics as last-known-good and inspect the metrics refresh loop.", +- "no_data_state": "OK", +- "exec_err_state": "OK" +- }, +- { +- "uid": "qbit-prism-metrics-snapshot-unavailable", +- "title": "PrismMetricsSnapshotUnavailable", +- "expr": "qbit_prism_metrics_snapshot_available{" ~ prism_selector ~ "} == bool 0", +- "condition_ref": "A", +- "evaluator": "gt", +- "threshold": 0, +- "for": qbit_monitoring_stack_prism_alert_metrics_snapshot_for, +- "severity": "warning", +- "service": "qbit-mining-pool", +- "labels": {"alert_class": "prism-runtime"}, +- "summary": "PRISM has no complete metrics snapshot", +- "description": "Metrics warm-up has not produced a complete snapshot. This series may itself be absent during the 503 warm-up state; PrismMetricsEndpointDown remains the authoritative total-loss net; both are Slack-only coverage signals since the 2026-09-09 paging audit (qbit-tools #1086).", +- "no_data_state": "OK", +- "exec_err_state": "OK" +- }, +- { +- "uid": "qbit-prism-metrics-snapshot-age", +- "title": "PrismMetricsSnapshotAgeHigh", +- "expr": "qbit_prism_metrics_snapshot_age_seconds{" ~ prism_selector ~ "} > bool " ~ qbit_monitoring_stack_prism_alert_metrics_snapshot_age_seconds, +- "condition_ref": "A", +- "evaluator": "gt", +- "threshold": 0, +- "for": qbit_monitoring_stack_prism_alert_metrics_snapshot_for, +- "severity": "warning", +- "service": "qbit-mining-pool", +- "labels": {"alert_class": "prism-runtime"}, +- "summary": "PRISM metrics snapshot age exceeded its budget", +- "description": "The last complete metrics generation is old even if the endpoint still serves it. Raise this threshold if the deployment intentionally uses a refresh interval above fifteen seconds.", +- "no_data_state": "OK", +- "exec_err_state": "OK" +- } +-] %} +-{% set prism_public_read_alert_rules = [ +- { +- "uid": "qbit-prism-public-staleness-refusal", +- "title": "PrismPublicStalenessRefusalsIncreasing", +- "expr": "sum(increase(qbit_prism_public_staleness_refusals_total{" ~ prism_public_read_selector ~ "}[" ~ qbit_monitoring_stack_prism_public_read_alert_counter_window ~ "])) > 0", +- "condition_ref": "A", +- "evaluator": "gt", +- "threshold": 0, +- "for": qbit_monitoring_stack_prism_alert_immediate_for, +- "severity": "warning", +- "service": "qbit-mining-pool", +- "labels": {"alert_class": "prism-runtime"}, +- "summary": "PRISM public reads are exceeding an endpoint staleness budget", +- "description": "The extracted read tier refused at least one response because its cached age exceeded the route's fixed staleness contract. Check aggregate TTL/SWR settings and keep their combined window within the pinned 90-second budget.", +- "no_data_state": "OK", +- "exec_err_state": "OK" +- }, +- { +- "uid": "qbit-prism-public-ledger-unready", +- "title": "PrismPublicLedgerUnready", +- "expr": "min(qbit_prism_public_ledger_ready{" ~ prism_public_read_selector ~ "})", +- "evaluator": "lt", +- "threshold": 1, +- "for": qbit_monitoring_stack_prism_public_read_alert_ledger_unready_for, +- "severity": "warning", +- "service": "qbit-mining-pool", +- "labels": {"alert_class": "prism-runtime"}, +- "summary": "PRISM public read ledger readiness is unavailable", +- "description": "The extracted public read tier has reported ledger readiness other than 1, or its private metric is missing, throughout the deploy-safe grace window. Check the Tailscale scrape, read service, and primary Postgres readiness before trusting public responses. The public read tier is not the mining path, so this is a Slack-only readiness/scrape signal (2026-09-09 paging audit, qbit-tools #1086).", +- "no_data_state": "Alerting", +- "exec_err_state": "Alerting" +- }, +- { +- "uid": "qbit-prism-public-5xx-sustained", +- "title": "PrismPublic5xxSustained", +- "expr": "sum(rate(qbit_prism_public_responses_total{" ~ prism_public_read_selector ~ ",status=~\"5..\"}[" ~ qbit_monitoring_stack_prism_public_read_alert_5xx_window ~ "])) > 0", +- "condition_ref": "A", +- "evaluator": "gt", +- "threshold": 0, +- "for": qbit_monitoring_stack_prism_public_read_alert_5xx_for, +- "severity": "warning", +- "service": "qbit-mining-pool", +- "labels": {"alert_class": "prism-runtime"}, +- "summary": "PRISM public reads are returning sustained 5xx responses", +- "description": "The extracted read tier has continuously observed public 5xx responses across the configured window. Correlate staleness refusals, ledger readiness, read-tier logs, and Postgres health.", +- "no_data_state": "OK", +- "exec_err_state": "OK" +- } +-] %} +-{% set alert_rules = prism_share_writer_alert_rules if (qbit_monitoring_stack_prism_alerts_enabled | bool) else [] %} + {% if qbit_monitoring_stack_prism_public_read_alerts_enabled | bool %} + {% set alert_rules = alert_rules + prism_public_read_alert_rules %} + {% endif %} +-{% if (qbit_monitoring_stack_prism_alerts_enabled | bool) and (qbit_monitoring_stack_prism_overload_alerts_enabled | bool) %} +-{% set alert_rules = alert_rules + prism_overload_alert_rules %} +-{% endif %} +-{% if (qbit_monitoring_stack_prism_alerts_enabled | bool) and (qbit_monitoring_stack_prism_writer_lease_alerts_enabled | bool) %} +-{% set alert_rules = alert_rules + prism_writer_lease_alert_rules %} +-{% endif %} + {% if not (qbit_monitoring_stack_prism_alert_connected_clients_enabled | bool) %} + {% set alert_rules = alert_rules | rejectattr('connected_clients_rule', 'defined') | list %} +-{% endif %} +-{% if not (qbit_monitoring_stack_prism_stratum_alerts_enabled | bool) %} +-{% set alert_rules = alert_rules | rejectattr('prism_stratum_rule', 'defined') | list %} + {% endif %} + {% if qbit_monitoring_stack_prism_backup_alerts_effective | bool %} + {% set alert_rules = alert_rules + [ +@@ -1454,5 +1454,84 @@ + } + ] %} + {% endif %} ++ + {% set qbit_monitoring_stack_alert_rule_group_name = "qbit-prism-alerts" %} +-{% include "grafana-alert-rule-group.yml.j2" %} ++{% from "grafana-alert-rule-group.yml.j2" import render_alert_rule_group with context %} ++{% set native_partitions = namespace(standard=[], paging=[]) %} ++{% for rule in alert_rules %} ++{% if rule.severity == 'critical' and (rule.labels | default({})).get('page') == 'true' %} ++{% set native_partitions.paging = native_partitions.paging + [rule] %} ++{% else %} ++{% set native_partitions.standard = native_partitions.standard + [rule] %} ++{% endif %} ++{% endfor %} ++apiVersion: 1 ++deleteRules: ++ - orgId: 1 ++ uid: qbit-prism-share-writer-backlog ++ - orgId: 1 ++ uid: qbit-prism-share-writer-backlog-critical ++ - orgId: 1 ++ uid: qbit-prism-shares-recovered-to-disk ++ - orgId: 1 ++ uid: qbit-prism-shares-replayed ++ - orgId: 1 ++ uid: qbit-prism-job-build-orphan-evicted ++ - orgId: 1 ++ uid: qbit-prism-stats-reconcile-stale ++ - orgId: 1 ++ uid: qbit-prism-lease-proven-age ++ - orgId: 1 ++ uid: qbit-prism-lease-monitor-guarantee ++ - orgId: 1 ++ uid: qbit-prism-lease-surplus-invalid ++ - orgId: 1 ++ uid: qbit-prism-preview-publish-latency ++ - orgId: 1 ++ uid: qbit-prism-preview-publish-latency-crit ++ - orgId: 1 ++ uid: qbit-prism-preview-publish-degraded ++ - orgId: 1 ++ uid: qbit-prism-parent-preview-timeouts ++ - orgId: 1 ++ uid: qbit-prism-parent-preview-timeouts-crit ++ - orgId: 1 ++ uid: qbit-prism-unresolved-parent-depth ++ - orgId: 1 ++ uid: qbit-prism-unresolved-parent-near-cap ++ - orgId: 1 ++ uid: qbit-prism-unresolved-parent-at-cap ++ - orgId: 1 ++ uid: qbit-prism-unresolved-parent-age ++ - orgId: 1 ++ uid: qbit-prism-unresolved-parent-age-crit ++ - orgId: 1 ++ uid: qbit-prism-cleanup-backlog-depth ++ - orgId: 1 ++ uid: qbit-prism-cleanup-backlog-oldest ++ - orgId: 1 ++ uid: qbit-prism-cleanup-backlog-near-bound ++ - orgId: 1 ++ uid: qbit-prism-cleanup-backpressure-active ++ - orgId: 1 ++ uid: qbit-prism-cleanup-backpressure-engaged ++ - orgId: 1 ++ uid: qbit-prism-refresh-pending ++ - orgId: 1 ++ uid: qbit-prism-refresh-pending-critical ++{# Disabled native rules must be deleted too, including reused legacy UIDs. #} ++{% set emitted_uids = alert_rules | map(attribute='uid') | list %} ++{% for rule in prism_native_rules + prism_public_read_alert_rules %} ++{% if rule.uid not in emitted_uids %} ++ - orgId: 1 ++ uid: {{ rule.uid }} ++{% endif %} ++{% endfor %} ++groups: ++{% if native_partitions.standard %} ++{{ render_alert_rule_group('qbit-prism-alerts', qbit_monitoring_stack_grafana_alert_standard_evaluation_interval, native_partitions.standard) }} ++{% endif %} ++{% if native_partitions.paging %} ++{{ render_alert_rule_group('qbit-prism-alerts-paging', qbit_monitoring_stack_grafana_alert_paging_evaluation_interval, native_partitions.paging) }} ++{% endif %} ++{{ render_alert_rule_group('qbit-prism-postgres-ha-alerts', qbit_monitoring_stack_grafana_alert_standard_evaluation_interval, prism_database_alert_rules) }} diff --git a/docs/prism-alert-verification.md b/docs/prism-alert-verification.md new file mode 100644 index 000000000..12e2588e7 --- /dev/null +++ b/docs/prism-alert-verification.md @@ -0,0 +1,140 @@ +# Issue #279 implementation verification + +Workstream B, Dan (`djh58`); verified 2026-09-11 on branch +`djh58/prism-b7-alert-migration`, based on `5355b69462663e9b9e666589147f74e7aeb268dc` +(#308), which matched freshly fetched `origin/3.x.x` at task start. + +## Acceptance results + +| Acceptance | Result | Evidence | +| --- | --- | --- | +| `cargo +1.89.0 fmt --all --check` | Executed and passed | Final formatter check clean | +| `git diff --check` and staged diff check | Executed and passed | No whitespace errors, including the patch artifact | +| Inventory, both directions, both roles, ungated | Executed and passed | `inventory_scrapes_both_roles_in_both_directions`: real loopback HTTP listeners for the production role routers; coordinator startup and published states, public replica off and require; no PostgreSQL prerequisite or skip | +| Registry-generated inventory | Executed and passed | `inventory_is_generated_from_registry`; 38 coordinator + 14 public = 52 families | +| Rules versus inventory grep | Executed and passed | `native_rules_reference_only_inventory_families_for_their_role`; histogram suffixes resolve only to declared histogram families; public request counter and deferred first-offer/lock timing are rejected | +| Complete migration coverage | Executed and passed | All 46 historical metric tokens and 78 deployed definitions: 22 migrated, 22 no replacement, 34 external rules preserved | +| Entry-point documentation | Executed and passed | `PRISM.md`, crate README and the replacement-series section of `prism-rust-migration.md` updated; historical appendix links to the new contract | +| Deployment diff applies cleanly | Executed and passed | `git apply --check` and actual application in a disposable copy of the snapshot path; applied bytes equal generator output; original snapshot checksum unchanged | + +The required Cargo command was executed with +`CARGO_TARGET_DIR=/tmp/prism-b7-target`: + +```sh +cargo +1.89.0 test --locked -p qbit-prism-server --test observability -- --nocapture +``` + +All **12** observability tests passed, none ignored or skipped. The existing +`rust-tests` CI job runs `cargo test --locked --workspace --all-targets`, so these +tests execute without new environment gates. The existing collector/runtime +tests also verify real zero versus failed observations and blocked runtime HTTP +behavior. No production metric producer, name, label or version was changed. + +## Review corrections and consumer verification + +This report supersedes the original `3f4caa9` D3 verification: the first draft's +NULL replay-lag contract produced a false alert on a healthy idle standby. +The [review ledger](prism-alert-review-ledger.json) retains the findings, sources, +local commits and incomplete review lanes without claiming publication. + +| Finding | Local correction | Evidence | +| --- | --- | --- | +| Healthy idle NULL replay latency alerted; retained latency did not describe current backlog | `8e5e8d8`: use exact high/low durable WAL positions observed five seconds apart; retain explicit unknown/error states | Exact query tested on disposable PostgreSQL 16.14 primary plus one async standby; idle, paused replay, dwell, recovery and disconnect all pass | +| Deployed network-wide floor of ten clients became ten per instance | `b85a137`: sum only a complete set of fresh coordinator observations | Regression failed before correction; six plus six is healthy, six plus three alerts, partial stale data is unknown | +| Disabled gates omitted reused legacy UIDs without deleting persisted rules | `d1d6c81`: conditionally delete every disabled native UID | Regression failed before correction; ten rendered gate combinations pass, including all disabled and each gate toggled | +| Synchronous standby configuration could satisfy the async D3 health contract | `2bf64f2`: expose explicit async topology status and reject missing/stale/invalid values | Four synthetic regressions failed before correction; real SQL tests cover a selected synchronous standby, a primary waiting for another standby, and restoration to async | + +Executed and passed on corrective implementation `2bf64f2fbf6861320e05e62e07a9c741abd5953e`: + +- Both generators with `--check`, including the exact frozen deployment snapshot + and specification digest; `cargo +1.89.0 fmt --all --check` and whitespace checks. +- All 12 observability tests, covering both-role/both-direction inventory, rule + family references, migration completeness and real failed/zero collector states. +- Prometheus `promtool` 3.14.0 parsed all **27** authored expressions plus the two + D3 alert conditions used to test dwell. The retained synthetic fixtures passed + **38 scenarios / 86 predicate assertions**, plus **12 alert-state assertions**. + They cover native stale/unknown/zero signals, the network-wide client floor, + partial scrape loss, D3 failed/missing exporter and query status, stale history, + exact five-second boundaries, 64-bit LSN precision and low-word rollover. +- The exact SQL from `prism-postgres-exporter-queries.yaml` ran against two + disposable PostgreSQL **16.14** clusters with one dedicated **asynchronous** + standby. The resulting samples passed **9 PromQL scenarios / 18 predicate + assertions**, plus **12 alert-state assertions**. Idle caught-up SQL returned + zero only with equal WAL positions while raw `replay_lag` was NULL; paused + replay exceeded the allowance and one-minute dwell; recovery cleared with raw + latency still **62.843997 seconds**; disconnect returned count zero and unknown + positions/latency. Synchronous configuration of this standby or another + required standby returned explicit async status zero while preserving valid + WAL positions; restoring the asynchronous topology cleared the predicate. + Running the primary-only query on the standby failed as + expected. Both clusters were stopped and removed. The harness records real SQL + samples at nominal one-second cadence; exporter status inputs are controlled + by the harness, not a live postgres_exporter. +- The retained provisioning check applies the unified patch in a disposable + copy, compares resulting bytes to the generator, renders with Jinja2 and the + unchanged shared macro, then parses the provisioning YAML with PyYAML. All + **10** gate combinations passed. All enabled: **61** alerts (25 native/public, + 34 external, two D3) and **27** retired UID deletions. Additional disabled native + UIDs are deleted conditionally; all disabled: two mandatory D3 alerts and 52 + deletions. No duplicate UID, create/delete overlap, or altered external rule + dictionary was observed. Original snapshot bytes remain unchanged. + +Reproduce the added consumer checks: + +```sh +python3 scripts/test_prism_alert_promql.py --promtool /path/to/promtool +PRISM_TEST_PG_BIN_DIR=/path/to/postgresql/bin python3 scripts/test_prism_postgres_alerts.py --promtool /path/to/promtool +python3 scripts/test_prism_alert_deployment.py --snapshot /path/to/read-only-snapshot +``` + +The SQL/provisioning scripts require PyYAML; provisioning also requires Jinja2. +The static PromQL runner only needs Python and promtool. The inventory test +remains ungated in the existing Rust CI job. Provisioning deletion is validated +at the rendered contract level against Grafana's documented +[`deleteRules` mechanism](https://grafana.com/docs/grafana/latest/alerting/set-up/provision-alerting-resources/file-provisioning/); +no running Grafana instance was used. + +## Review lane status + +| Lane | Evidence and state | +| --- | --- | +| Local Codex CLI review, medium effort | Executed on `5355b69..3f4caa9` with gpt-6-astra; no actionable findings; this lane checked generators but did not rerun Cargo/PromQL | +| Local adversarial Codex | Executed on the original diff with gpt-5.6-sol; reported the idle-standby and client-floor findings above | +| Local corrective-diff adversarial review | Executed with gpt-5.6-sol on `3f4caa9..8e5e8d8`; independently reported gate deletion and synchronous-topology findings; both have local corrections and regression evidence | +| Independent Fable adversarial | Requested from coordinator; not executed locally; result pending | +| Thermo quality lane | Installed skill read at coordinator-provided path; independent execution pending coordinator routing | +| GitHub CI, Tenki and PR review bots | Not executed: branch unpushed and no PR opened | + +The main checkout's `AGENTS.md` was sought; the coordinator confirmed it is +absent. Review used the supplied session instructions and engineering-practices +EP-COMPAT, EP-OBSERVABILITY and EP-ERRORS. The local medium CLI review is recorded +as the actual command lane, not an unavailable `/code-review` skill invocation. +No unavailable independent lane is counted as complete. + +## Remaining coordinator and deployment work + +The coordinator must review the four findings, their local corrections and the +final diff (including the final async correction, which has not received a +further independent review); obtain or explicitly resolve the pending independent Fable/thermo +lanes; then authorize publishing and PR creation. No push, PR, production change, +or qbit-tools modification was performed. Issue #311 was left untouched. + +Not executed: production rendering with actual Ansible variables, live Grafana +provisioning/deletion, an actual postgres_exporter scrape/failure test, production +HA failover, qbitd integration, the **400k regression / 500k headroom** load checks, +or the #291 soak. Those capacity targets are preserved as qualification gates; +this documentation/test correction supplies no new load-capacity evidence. + +The D3 correction retains the approved five-second allowance and one-minute dwell +for one primary plus one asynchronous dedicated HA standby, separate from the +public read replica. It requires primary exporter query metrics, one-second +scrapes and standby feedback, exact WAL components, and explicit failed/missing/ +stale status; #281/#291 must verify permissions, collector cadence/cost, the +chosen exporter version and its real error behavior. ACKs still require local +durability without waiting for standby replay. Runtime metric producers, +VERSION and CHANGELOG were not changed. + +Native numeric thresholds without measurements remain **provisional, measure in +#291**; proof-to-first-offer and advisory-lock rules remain deferred to A/#266 +and #283. These are recorded integration/qualification dependencies, not +completed production checks or an authorization to deploy. diff --git a/docs/prism-async-promotion-gap-blocks.md b/docs/prism-async-promotion-gap-blocks.md new file mode 100644 index 000000000..46671f7b6 --- /dev/null +++ b/docs/prism-async-promotion-gap-blocks.md @@ -0,0 +1,697 @@ +# Blocks on gap-issued work after an asynchronous promotion (#619) + +This note investigates [#619](https://github.com/Qbit-Org/qbit-mining-bootstrap/issues/619). +Under the [D3 decision](https://github.com/Qbit-Org/qbit-mining-bootstrap/issues/260) +(one primary and one asynchronous standby), a promotion can lose the shares +of the last replication gap. A frontend that survives the promotion still holds +work whose payout window names those lost shares. It issues jobs on that work, +accepts a block proof on one of them, enqueues and offers the candidate, and +qbitd puts the block on chain. The landing then fails for good with +`window range incomplete`. + +The note covers both modes reported on #619, what is lost and how often, the +money at stake, the options, and a recommendation. It changes no product code. + +**Status: decided (d)** (operator, 2026-10-01), and implemented; see +[Decision and implementation](#decision-and-implementation). The analysis +below is the investigation as it stood before the decision. + +**Base.** `3.x.x` at `4f4b218d08b9f64a2ac0cc29ce97caf59cb50fb6`. Every +`path:line` citation is at that commit, relative to +`crates/qbit-prism-server/src/` unless it starts with `crates/` or `docs/`. + +**Evidence labels.** **Verified by running** means a command was run on +minidev-alex1 and its output is quoted below. **Read from code** means the +claim comes from reading the source at the base commit and was not executed. +Estimates are labelled as estimates. + +## Summary + +- **Both modes have one root cause.** Every check that admits work, issues a + job or enqueues a candidate compares versions that a promotion does not + change: the payout revision, the prior-balances digest, the template, the tip + and the readiness generation. The only share-history checks are + `probe_share_rows` probes, which test whether a `share_seq` *exists*. A + promotion can lose rows, and PostgreSQL's sequence then hands the lost + numbers out again to other shares. So a window captured on the old primary + stays authority on the promoted one. #466's timeline witness protects the + refresh's delta read (`ledger/window/snapshot_delta.rs:153-196`), but not + work that is already prepared, published or held. +- **Mode 2 is an authority bug, not a landing-predicate bug, and needs no + decision.** The landing predicate is right: it refuses to count reissued + rows as the window. What is wrong is upstream: + - the refresh keeps pre-promotion work published (`coordinator.rs:1165-1199`); + - the prepared-record repair re-creates that work on the promoted primary + after checking only that its endpoint `share_seq` values exist + (`ledger/jobs/compact_issued.rs:194-209`). +- **Both modes reproduce deterministically on this host without qbitd.** The + new `tests/b619_gap_work_after_promotion.rs` uses a real PostgreSQL 16 async + pair, a native `Coordinator` and the fake node. Both tests ended with + `window range incomplete: expected 15 shares, read 0` after the node + accepted the block, in 15 of 15 runs each. The promoted sequence resumed at + `share_seq` 34 every time, the same 34–46 reuse #474 B reported. +- **Money at stake per affected block.** Today the block is on chain and is + never landed. + - With direct payouts, the ledger never records its coinbase. The positive + carry-forward balances that coinbase paid out are paid again by later + landings, at most the as-issued float `F`. The block's accruals are never + credited. + - With CTV settlement on (the mainnet guide's setting), the covenant outputs + commit to fanout transactions that can only be rebuilt from the lost window. + There is no pool spend key, so those outputs stay unspendable until the + window is recovered from the old primary's storage. +- **Recommendation: option (d).** Revalidate the window inside the + transaction that uses it. This is (a)'s refusal, scoped by an exact + durability test rather than (c)'s blunt epoch fence. + - Refuse a block proof `stale-job`, before its enqueue, when the current + primary does not hold the window's last row as the window's own row. + - Stop issuing work prepared on another writer timeline. + - Make the repair check identity, not existence. + - No migration is needed. Do not take (b): the window is gone, so a landing + cannot be audited or settled without inventing numbers, and in CTV mode + it cannot unlock the covenant outputs anyway. + +## Background + +- **D3** ([#260](https://github.com/Qbit-Org/qbit-mining-bootstrap/issues/260), + decided 2026-09-11) sets `synchronous_standby_names=''`, so share ACKs never + wait for the standby. The accepted loss is the shares in the replication-lag + window. The decision comment puts that lag at tens of milliseconds, about 0 + to 1 share at today's rates and about 200 at the historic 2,000/s peak. It + equals the standby's downtime if the standby was out. +- **#529, #570 and #585** cover blocks found *in* the gap. #570 waits, within a + bound, for the standby to flush a found block's offer reservation before + `submitblock`, so a block offered from the old primary survives the + promotion with its candidate. #619 is a different case: the candidate is + enqueued and offered *on the promoted primary*, after the promotion, so its + own rows are never at risk. What the promoted primary lacks is the block's + *window*. +- **#466** repaired the refresh's retained-window delta read across replaced + history. `leaf_witness` reads the insertion timeline, + `left(pg_walfile_name(pg_current_wal_lsn()),8)`, in the same statement that + counts the range (`ledger/window/snapshot_delta.rs:157-196`). A changed + timeline sends the refresh to the full scan + (`ledger/window/snapshot_delta.rs:479-512`). This protects *new* work only. +- **#474 slice B (PR #620)** found #619 with live miners and a real qbitd. Its + gated drill (`live_regtest … async_live_miners`) now submits the block on + the gap-issued job after the promotion and requires it to be refused as + stale or to pay only promoted rows; it needs qbitd. + +## How the defect happens + +### Why lost sequence numbers come back (read from code, verified by running) + +- `share_seq` comes from the sequence `qbit_share_ledger_share_seq_seq` + (`crates/qbit-prism-server/migrations/016_share_ledger_partition_catalog.sql:487`). The INSERT at + `ledger/window.rs:855` takes its default. +- PostgreSQL WAL-logs a sequence 32 values ahead. A standby that replayed up + to the last logged value resumes, after promotion, at most 33 past the last + share it received. The old primary may have used numbers beyond that point. + The promoted primary then hands them out again to other shares. +- Every native ledger writer appends under `ORDER_LOCK`: the share append + (`ledger/window.rs:684-727`) and the deferred-share credit of settlement and + reconciliation (`ledger/blocks.rs:295`, `:514`, then `:677`). So + `share_seq` order is commit order. A physical standby replays a prefix of + the WAL, so the promoted ledger holds exactly the rows up to some + `share_seq`, and every later number is either absent or a *different* + share. +- **Verified by running.** The standby received shares 1..=20 and 80 more were + credited in the gap (21..=100). The promoted primary's first new share was + `share_seq` 34 in all 15 runs, and the timeline moved from `00000001` to + `00000002` in each of the 9 runs that recorded it. +- **Clock.** The ledger clock (`qbit_prism_cluster.ledger_clock_ms`) is + replicated. A reissued row's `accepted_at` is + `GREATEST(replicated clock, promoted host's clock)` (`ledger/window.rs:850`). + A window anchored in the gap has `anchor_ms` above every row the old primary + had credited by then (`ledger/window.rs:903`). Reissued rows therefore land + *after* the anchor, unless the promoted host's clock runs behind the old + primary's by more than the time since the anchor. + +### The checks that admit gap work (read from code) + +| Where | What it checks | What it misses | +| --- | --- | --- | +| Refresh, same template (`coordinator.rs:1165-1199`) | template fingerprint, fee, payout revision, balances digest, cached window inside `PRISM_PAYOUT_ARTIFACT_REANCHOR_SECONDS` (default 60 s, `config.rs:600`), and `current.bundle.is_some() \|\| share_seq unchanged` | The accepted cutoff is ignored whenever a bundle exists. The writer timeline is not read. Pre-promotion work stays published for up to the reanchor interval. | +| Refresh reuse of the cached window (`coordinator.rs:1240-1255`, `coordinator/refresh_window.rs:55-68`) | the same inputs plus accepted cutoff equality | No timeline. Reuse needs the cutoffs to coincide, so it is unlikely, not impossible. | +| Issuance authority (`coordinator/tip_observation.rs:205-222`, `:557-604`, `:641-746`) | the prepared identity, readiness generation, tip stamp and `identity.revision == revision` (`:734`) | Nothing about the share ledger. | +| `build_job` (`coordinator.rs:2898-2955`) | issues from the in-memory published `prepared` (`:2910`) | — | +| Issued-job save and repair (`coordinator/prepared_storage.rs:21-121`; `ledger/jobs/compact_issued.rs:194-209`) | A lost prepared record returns `PreparedMissing`. The repair re-inserts it from memory after `probe_share_rows(first, last)`. | The probe only checks that both `share_seq` values exist (`ledger/window.rs:1143-1157`). Reissued endpoints pass. | +| `save_compact_prepared` (`ledger/jobs/prepared.rs:224-237`) | the same endpoint existence probe | The same. Its own comment says it is "no substitute for read_window's count/digest checks". | +| Stratum submit (`stratum.rs:1528-1533`) | finds the job in the session's in-memory list | Never reads the job row, which was lost with the gap. | +| `submit_share` (`coordinator/miner_submit.rs:310-470`) | fee, tip, parent (`:368`), payout revision (`:417-439`), readiness | Nothing about the window's rows. The share's `job_issued_at` is the gap anchor (`:469`). | +| Candidate enqueue (`ledger/candidates.rs:651-667`) | `probe_share_rows(first, first)`. If missing, an ALERT log, then **enqueue anyway** "so the block reaches the node" | A reissued first row passes silently. A lost one is offered anyway. | +| Landing (`coordinator.rs:2101-2143` → `ledger/window.rs:268-344`) | Probes both endpoints (`:319`), then reads pages under `accepted AND share_seq … AND accepted_at<=anchor AND job_issued_at<=anchor` (`:1238-1240`), then count and digest (`:1428-1435`). | This is correct, and it is the first check that notices. The miss is reported as `Incomplete`, with `got: 0` synthesized when an endpoint is missing (`:319-325`). `classify_window_error` (`coordinator.rs:443-449`) then keeps the candidate in `reconciliation`. | + +### Mode 1: a job issued in the gap, held by a surviving session + +1. The frontend prepares work over gap shares. In the test, a template change + makes the refresh rebuild. The old primary saves the prepared record. +2. A session is issued a job on that work. Its row is saved on the old primary + only. +3. The promotion loses the gap: the shares, the prepared record and the job + row. +4. The session submits a block proof on the job. The job is found in session + memory, and `submit_share` checks pass: same parent, same revision. The + share append and the candidate enqueue commit on the promoted primary. The + enqueue's first-row probe fails and logs its ALERT, then enqueues anyway. +5. The offer reaches the node, which accepts it. The landing's endpoint probe + fails, so it reports `expected 15 shares, read 0`, and the candidate stays + in `reconciliation` with `offer_outcome='accepted'`. + +This happens even after the frontend has published fresh, correct work from +the promoted ledger. The held job stays submittable while its parent is the +tip and the revision is unchanged. Same-tip retention keeps retired jobs for +`PRISM_STRATUM_SAME_TIP_JOB_RETENTION_SECONDS` (30 s by default). + +### Mode 2: gap work issued to a new session after the promotion + +1. As in mode 1, the frontend holds gap work as its published work. +2. After the promotion, the promoted primary credits new shares that reuse + the gap window's numbers, 86..=100 in the test. +3. The next refresh sees the same template, revision and balances inside the + 60 s reanchor interval, so it keeps the gap work published + (`coordinator.rs:1165-1199`). +4. A brand-new session is issued a job on it. `persist_issued_job` finds the + prepared record missing. The repair's endpoint probe passes on the + reissued rows 86 and 100, so the repair writes the stale prepared record, + and then the job, onto the promoted primary. +5. A block on that job is accepted, enqueued (the first-row probe passes + silently) and offered. The landing's endpoint probe passes. The page read + returns 0 rows, because all 15 rows were credited after the window's + anchor. The result is `expected 15 shares, read 0`. + +This is exactly #619's second comment ("15 rows present, yet the landing read +0"). The repair step makes it worse: the promoted primary now durably holds a +prepared record that another frontend can resume from. + +### Why durable records are not the problem (read from code) + +A prepared record or job row that *did* replicate is consistent with the +promoted ledger: + +- its window's rows committed before the snapshot read them; +- the record was written after that read; +- WAL order and the prefix property then put those rows on the standby too. + +**Stale authority lives only in frontend memory:** published prepared work, +the cached window and session-held jobs. It also lives in the repair path, +which writes that memory back into the database. A fix therefore does not +have to touch stored formats. + +## Reproduction + +`crates/qbit-prism-server/tests/b619_gap_work_after_promotion.rs` held two +`#[ignore]`d tests, each naming #619, before the fix. They now assert the +fixed behaviour and are gated; the output below is the evidence from before +the fix. + +**The fixture:** + +- PostgreSQL 16 primary with one async standby on a physical slot, reached + through a replication relay the test can sever; +- a stable writer relay; +- one native `Coordinator` built by `support/fake_qbitd.rs`'s + `coordinator_config`, so the reanchor interval stays at its 60 s default; +- the in-process fake node, which accepts `submitblock` once + `accept_blocks()` is called. + +**The drill:** + +1. Credit 20 shares, then wait for the standby to replay them. +2. Sever the replication link and credit 80 more shares. +3. Move the template, so the refresh prepares the gap work, window 86..=100. +4. Issue one job on it. +5. Fence the writers, stop the old primary, `pg_promote(true,60)`, and route + the writer relay to the promoted standby. + +Each test asserts the outcome #619 needs, whichever option is chosen: + +- Mode 1: the block is either refused (`stale-job` or `unknown-job` with no + outbox row) or landed (`submitted` plus a `qbit_pool_blocks` row). +- Mode 2: a new session must be issued a job whose window reads whole on the + promoted ledger. A refusal is retried, with a refresh between, up to five + times, so a fix may refuse the gap work but may not starve the session. + The block on the issued job must then be refused or landed as in mode 1. + +**Verified by running** on minidev-alex1: + +``` +PRISM_TEST_PG_BIN_DIR=/usr/lib/postgresql/16/bin PRISM_TEST_REQUIRE_INTEGRATION=1 \ + cargo test --locked -p qbit-prism-server --test b619_gap_work_after_promotion \ + -- --ignored --nocapture --test-threads=1 +``` + +The command was run 15 times across three revisions of the test: 6 runs +before the timeline and lost-record evidence was added, 6 after, and 3 on the +committed test, whose mode 2 issuance check was tightened in review. Every +run failed both tests at the #619 assertion, in 3.6–6.5 s per run. Each run +of the committed test printed: + +``` +#619 drill: 20 shares replicated, 80 credited in the gap (150280 WAL bytes unreplicated); gap work window 86..=100 (15 shares, anchor …); its prepared record and the held job's row are gone; timeline 00000001 -> 00000002 +#619 mode 1: replicated through share_seq 20; gap 21..=100 lost; post-promotion credits reused share_seq Some(34)..=Some(46); held job's window 86..=100 (15 shares, …); fresh work's window 19..=46 (15 shares, …); block …: miner answered Ok(()); outbox Some(("reconciliation", Some("accepted"), Some("landing failed after the offer (node accepted the offer): window range incomplete: expected 15 shares, read 0; rows pruned or missing, or a different predicate (#268 owns recovery)"))); claim processing Some(Ok(())); qbit_pool_blocks row: false +#619 mode 2: replicated through share_seq 20; gap 21..=100 lost; post-promotion credits reused share_seq Some(34)..=Some(100); after the promotion the refresh kept the gap work published: true; issuance refusals before a job: []; new session's job … on window 86..=100 (15 shares, …) (issued job and repaired prepared record saved on the promoted primary: true); read on the promoted ledger: Err(Incomplete { expected: 15, got: 0 }); block …: miner answered Ok(()); outbox Some(("reconciliation", Some("accepted"), Some("landing failed after the offer (node accepted the offer): window range incomplete: expected 15 shares, read 0; …"))); …; qbit_pool_blocks row: false +test result: FAILED. 0 passed; 2 failed; 0 ignored; 0 measured; 0 filtered out; finished in 4.15s +``` + +Before the failing assertion, mode 2 also checks that each of the 15 rows now +at 86..=100 is a different share, credited after the gap work's anchor. Mode +1 also checks that the fresh work the frontend prepared after the promotion +reads whole, so #466 covers new work. + +**Not run on this host:** + +- #620's live drill (`live_regtest … async_live_miners`): there is no qbitd + here. +- Any CTV-settlement landing: the fixture runs with `ctv_enabled: false`. +- Load. + +The Stratum socket layer is not exercised either: the tests call the same +`MiningBackend` methods a session calls. The in-memory job lookup is read from +code (`stratum.rs:1528-1533`). The enqueue's ALERT is also read from code: the +test installs no log subscriber. The run shows the ledger held no +`share_seq` 86 at enqueue in mode 1, which is the condition of that branch. + +## What is lost, when, and how often + +**What.** A block found after the promotion on work whose window includes +shares the promoted primary did not receive. The block is valid and goes on +chain, but the pool can never land it. + +**When.** It needs both of these: + +1. Some frontend had prepared work after the standby's last received WAL + position, and that work is still issuable after the promotion. That means + jobs held by sessions until the parent changes, plus 30 s of same-tip + retention; or published work kept up to the 60 s reanchor interval while + the template does not change. +2. A block is found on that work before it retires. + +**How often (estimate, not measured).** Per promotion: + +``` +P ≈ P(gap work exists) × p × E / T +``` + +- `p` is the pool's share of network hashrate. +- `T` is the block interval. +- `E` is the time gap work keeps being hashed after the promotion. Miners + switch to the newest job, so `E` ends at the next refresh that publishes + other work: up to the 60 s reanchor while the template is stable, sooner + when the template changes. The table takes the upper range. + +The illustrative inputs are the ones #504 used: `p` = 30% and `T` = 60 s. + +| Standby state at the loss | P(gap work exists) | E | P per promotion | +| --- | --- | --- | --- | +| Streaming, lag `L` ≈ 10–50 ms, refresh every `R` ≈ 1–30 s | ≈ `L/R`: 0.03%–5% | 30–60 s (upper range) | ≈ 0.005%–1.5% | +| Lagging past the 5 s alert, or disconnected (the gap is its downtime) | ≈ 1 | 30–60 s (upper range) | ≈ 15%–30% | + +So a healthy failover rarely loses a block this way. A failover after the +standby fell behind loses one in roughly every 3 to 7. #291's drill will +measure the real lag, refresh cadence and promotion time on production +hardware, and this estimate should be recomputed from them. + +**Under D3 the accepted loss grows.** Today it is: acknowledged shares in the +last gap, plus blocks found in the gap that were never offered or whose offer +logged an unconfirmed standby wait. It should also name blocks found **after** +the promotion on work whose window lay in the gap. + +## Money at stake + +**Whose shares the coinbase pays.** The coinbase pays the window the old +primary had: the gap shares, acknowledged to their miners and lost with the +gap. It also pays the prior balances as issued (`ledger/candidates.rs:634-650` +writes that snapshot on the promoted primary at enqueue). + +**What the promoted ledger holds.** + +- Mode 1: none of the window's rows. +- Mode 2: 15 *other* shares at the same numbers. They were credited after the + promotion and are paid by later windows. + +Either way the landing correctly refuses to treat them as this block's window. + +**Today: offered, on chain, never landed.** There is no `qbit_pool_blocks` +row, no audit bundle and no carry-forward settlement. The candidate sits in +`reconciliation` and is never offered again. + +- **Direct payouts** (`PRISM_CTV_SETTLEMENT_ENABLED=0`). Miners hold their + coinbase amounts on chain. The ledger never learns of the block: + - **Overpay.** Positive as-issued balances that this coinbase paid out are + still owed on the promoted ledger, and later landings pay them again. Per + account, a coinbase pays at most `max(0, issued(m))` above its gross + (`ledger/divergence.rs:1-21`). So the unrecorded double payment is at most + `F`, the positive as-issued float. The ledger-ops guide sizes `F` at about + 2.9·10⁷ sats at 2,000 sub-floor miners. Later blocks' miners fund it, + because the coinbase allocates by candidate balance. Unlike a #478 + capture, no debt is recorded. + - **Underpay.** The block's accruals are never credited. Miners below the + payout floor lose the gross this block owed them. + - **No audit.** The coinbase's distribution cannot be verified by miners + and appears in no pool record. HA promotion step 6 already treats a block + on chain with no candidate as an explicit accounting-loss + reconciliation (`docs/prism-ha-reference-architecture.md:480-497`), but + it has no entry for a block that has a candidate yet cannot land. +- **CTV settlement on** (`docs/mainnet-deployment.md:460`). The settlement + routes these recipients through covenant outputs rather than direct coinbase + outputs (`crates/qbit-prism/src/settlement.rs:177-181`): + - recipients below `ctv_direct_floor`; + - recipients beyond the direct-output budget. + + Those outputs commit to fanout transactions, and "there is **no pool spend + key**" (`crates/qbit-prism/src/ctv.rs:1-12`). Fanout artifacts are persisted only + at landing, from the rebuilt window (`ledger/blocks.rs:246`, `:927-951`). + The job's bundle keeps only "presence of the original CTV fanout set, + without retaining its outputs" (`coordinator/compact_runtime.rs:27-36`). So + without the lost window nobody can build the fanout, and the covenant + amount stays unspendable. + + The window can still be recovered from the fenced old primary's storage, + if it survives (the window's rows on the old timeline), or from a frontend + whose cached refresh window still holds it. Otherwise that part of the + block's value is lost for good. +- **If the block is refused before its offer** (options a, c and d), the pool + loses that block's coinbase value: the block reward the pool would have + mined. Nothing reaches the chain and the ledger stays exact. + +Read from code, not run: the CTV path and the divergence bound. The fixture +ran with direct payouts and checked only that no `qbit_pool_blocks` row was +written. + +## Options + +Each option is judged on six points: + +- **EP-STATE:** a stale result must not win. Revalidate the authorizing + version in the transaction that publishes. +- **Miners:** what the miner sees. +- **Chain:** the consensus and coinbase effect. +- **Migration:** whether a schema change is needed. +- **Mode 2:** whether the option fixes it. +- **Tests:** the test plan. + +### (a) Refuse a block proof whose window is not durable on the current primary + +The check goes in the candidate enqueue, which runs inside the share-append +transaction under `ORDER_LOCK` (`ledger/window.rs:684-727`) or in +`enqueue_candidate_observed` for block-only proofs. It must refuse, not log +and enqueue anyway as `ledger/candidates.rs:658-666` does today. + +- **EP-STATE.** Correct if the test is exact. It runs on the primary that + will hold the candidate, in the writing transaction. A promotion between + `submit_share`'s checks and the enqueue is therefore still caught: the + stale proof cannot win. +- **Miners.** `stale-job` for that proof, and no credit for its share. Plain + shares on the same job can still be credited, because their value depends + on future windows, not this one. +- **Chain.** The block is never submitted, so the pool loses one coinbase. + The block is chain-valid but stays unpublished. +- **Migration.** None. `WindowRef` already carries `anchor_ms` and + `last_share_seq`. +- **Mode 2.** Not fixed on its own: a bad job is still issued, and only its + block is refused. +- **Tests.** + - Mode 1 of `b619_gap_work_after_promotion` flips to "refused before + enqueue". + - An out-of-order test: a proof that passed `submit_share` before the + promotion and enqueues after it is refused. + - A pruned-prefix case keeps today's "enqueue anyway" (see (d)). + +### (b) Land paying only rows the promoted primary holds and record the divergence + +- **Feasibility.** The coinbase is fixed. A landing would have to record the + block from its on-chain outputs and the as-issued balances, and treat the + window as whatever the promoted ledger holds: nothing, in both modes. + - Per-account gross cannot be recovered. On-chain amounts are a weighted + allocation of the reward by candidate balance + (`crates/qbit-prism/src/lib.rs:1405-1577`), and sub-floor accounts have none. + - So the settlement would have to invent a rule. One rule debits paid + accounts by their whole on-chain amount, which claws back acknowledged + work from miners. Another only zeroes the floats this coinbase paid, so + the block's accruals are dropped. + - The canonical audit cannot be produced at all. +- **EP-STATE.** It deliberately publishes a result whose authorizing state is + gone. The divergence would have to be recorded atomically with the + landing, as #478 does. +- **Miners.** The proof is accepted and the coinbase pays them. The block + shows as divergent and unauditable. +- **Chain.** Unchanged. +- **CTV.** It does not help. The covenant outputs still cannot be spent + without the window. +- **Migration.** Yes, migration 021: a new landing kind and divergence + records, changes to the public audit API (a block with no verifiable + window), and integrity-report changes. EP-COMPAT applies to all of them. +- **Mode 2.** Not fixed. +- **Tests.** A divergent landing with exact records. Integrity reports stay + explainable. CTV must be shown not to regress. + +**#478 is not a precedent.** #478 chose capture because the as-issued window +*was* in the ledger, so the block could land canonically with a bounded, +recorded debt. Here the window is gone. + +### (c) Fence work prepared under an earlier writer term + +The term is the PostgreSQL timeline, which #466 already reads. The work +records it at capture, as `LeafWitness` does: runtime only, never persisted. +Anything that issues, resumes or enqueues refuses work whose term differs +from the primary's current one. + +- **EP-STATE.** Correct only if every comparison runs in the transaction that + writes, for example the enqueue. A frontend-cached "current timeline" can + be stale. +- **False refusals.** It also refuses work prepared before the cut whose + window *did* replicate, which is most pre-promotion work. A planned, + fenced switchover with full replay changes the timeline too, so its blocks + for one job lifetime would be refused for no loss. +- **Miners.** `stale-job` on every proof on pre-promotion work. Sessions get + new work at the next refresh. +- **Chain.** As (a), plus false refusals. +- **Migration.** None in memory. Stored issued and prepared records need no + term, because a record that replicated is consistent with the promoted + ledger. +- **Mode 2.** Fixed, if the refresh reuse tests and issuance compare the + term. +- **Tests.** + - Both b619 tests flip. + - A fenced switchover counts the refusals it causes. + - A resumed-job test shows durable records are still honoured. + +### (d) Revalidate the window where it is used (recommended) + +This combines (a)'s outcome with (c)'s scoping, through one exact and cheap +test. + +**The durability test.** `share_seq` order is commit order, and the promoted +ledger is a commit-order prefix. So a window is held by the current primary +**if and only if** the primary holds the window's *last* row and that row is +the window's own. The first-row probe stays as the retention test: retention +removes only prefixes, and a missing last row is never retention. The test +can be made: + +1. **Without new data:** the last row must exist and satisfy the window's own + predicate, `accepted AND accepted_at<=anchor AND job_issued_at<=anchor`. + This is the predicate #466's `leaf_witness` already applies to an endpoint + (`ledger/window/snapshot_delta.rs:171-175`). It is one primary-key lookup, + pruned to one leaf. It cannot be fooled unless the promoted host's clock + runs behind the old primary's by more than the time from the anchor to the + reissue, which is seconds. The landing's count and digest stay as the + final proof. +2. **Exactly, still with no migration:** keep the last row's `share_id` in + the runtime `Prepared`, as it is captured from the snapshot, and pass it to + the enqueue. Stored records, which are self-consistent, fall back to (1). + +**Where the test applies:** + +- **Candidate enqueue** (`ledger/candidates.rs:651-667`). Refuse a candidate + whose last row fails (1) or (2), returning `stale-job` with a new + `StaleJobCause` and a WARN naming the block. Keep "enqueue anyway" only + for the pruned-prefix case it was written for. +- **Repair and prepared save** (`ledger/jobs/compact_issued.rs:199-208`, + `ledger/jobs/prepared.rs:224-237`). Replace the existence probe on + `last_share_seq` with the same test, so a stale record is never written + back. +- **Refresh reuse and issuance.** Carry the timeline captured with the window + in `Prepared` and `RefreshWindow`. Treat a timeline change as a reason to + rebuild in the same-template branch (`coordinator.rs:1165-1199`) and in + `RefreshWindow::reusable`. Have the issuance authority re-run the test when + the timeline differs. + +**Judged on the six points:** + +- **EP-STATE.** The authorizing state, the window's rows, is revalidated in + the transaction that writes the candidate, the prepared record or the + job. The timeline only decides when work is retired early, so a stale + cached timeline cannot let a stale result win. +- **Miners.** `stale-job` only for proofs on windows the primary really lost. + Pre-promotion work whose window replicated keeps working. New sessions get + fresh work. +- **Chain.** As (a), but only for genuinely lost windows. +- **Migration.** None. +- **Mode 2.** Fixed. +- **Tests.** + - Both b619 tests flip: mode 1 is refused before enqueue; in mode 2 the + refresh rebuilds and the new session's window reads whole. + - A fenced switchover with full replay: a held pre-switch job's block still + lands. + - The out-of-order test from (a). + - A unit test of the probe against a reissued row, a missing row and the + original row. + - An EP-ERRORS case: the probe's database error stays a definite pre-COMMIT + failure, via `enqueue_failed_before_commit`, and is never answered as + `stale-job`. + +### (e) Optional: publish only standby-durable work + +Before publishing new same-tip work, wait within a bound for the dedicated +standby's `flush_lsn` to reach the snapshot's WAL position. This reuses #570's +wait (#529's prototype measured p99 2.5–3.9 ms with a healthy standby), off +the share-ACK path. + +With a healthy standby, no issued window can then lie in a gap, which removes +the healthy-failover row of the frequency table. A lagging or absent standby +still produces gap work, so (e) supplements (d) and does not replace it. New +tips must never wait for it. + +## Recommendation + +1. **Fix mode 2 now, without a decision.** It is an authority bug. + - The refresh must rebuild across a writer-timeline change. + - Issuance must not hand out work prepared on another timeline unless the + window passes the durability test. + - The repair and `save_compact_prepared` must check identity, not + existence. +2. **Decide mode 1 as (d): refuse before the enqueue.** Refuse a block proof + `stale-job` when its window's last row is not the window's own on the + current primary, using the test in the enqueue transaction. + - The block is lost to the pool, but nothing unlandable reaches the chain. + The ledger stays exact, and in CTV mode no covenant output is stranded. + - The test refuses only genuinely lost windows, so planned switchovers and + replicated pre-promotion work are unaffected. + - It needs no migration, and it fits #474 B's contract: "resume or + truthfully refuse original work under its existing authority". + - (b) is rejected: it cannot be computed or audited without the window, + and it strands CTV outputs anyway. +3. **Amend the accepted D3 loss** in `docs/prism-ha-reference-architecture.md` + and #291. Add: "blocks found after a promotion on work whose window lay in + the replication gap, refused `stale-job`". +4. **Add a runbook entry** for blocks that already slipped through: a + candidate in `reconciliation` with `window range incomplete` after a + promotion. + - Classify it as an accounting-loss reconciliation, as step 6 does. + - In CTV mode, recover the window rows from the fenced old primary to + rebuild the fanout before that storage is reused. + - Consider (e) once #291 measures the healthy-failover exposure. + +The trade-off for the decision owner: (d) gives up, rarely, a block that is +valid on chain and that today pays its miners (direct mode). In return it +keeps the books and the covenant outputs sound. + +If production ran direct payouts only, a #478-style divergent capture could be +argued instead. It would still need a landing that is not canonical, a +migration and an accounting rule chosen without the window. + +## Other defects found + +- **Misattributed landing error (EP-OBSERVABILITY).** After a promotion, + `classify_window_error` (`coordinator.rs:443-449`) says "rows pruned or + missing, or a different predicate", and a digest mismatch says + "corruption". Neither names replaced or lost history, which is what + happened. *Fixed with (d):* a failed landing read probes the window first + and, when this primary does not hold it, says `window not held by this + primary … (#619 …)`. +- **Mode 2's silent enqueue.** The enqueue's ALERT + (`ledger/candidates.rs:658-666`) fires only when the first row is missing. + A reissued first row passes silently, so mode 2 has no pre-offer signal at + all. *Fixed with (d):* the enqueue checks the last row first and refuses + with a WARN; the ALERT remains for a pruned prefix only. +- **Same-template reuse ignores cutoff regression** (read from code). The + same-template reuse test ignores the accepted cutoff whenever a bundle + exists (`coordinator.rs:1176`). After a promotion the ledger's cutoff can + sit below the published window's last row (46 against 100 in mode 1's run), + and on an unchanged template the work would still be kept. This is part of + mode 2's cause. *Covered by (d):* within one timeline the cutoff never + regresses, and a timeline change now forces the rebuild. +- **Constant `writer_epoch`** (not fixed). `qbit_share_ledger.writer_epoch` is always + written as 0 (`ledger/window.rs:855`). It looks like a writer term but + carries none. Do not mistake it for one in a fix. + +## Decision and implementation + +The operator chose option (d) on 2026-10-01, together with the mode 2 +authority fix; not (b) and not (e). No migration and no setting were added. + +- **Writer timeline.** `WriterTimeline` (`ledger/window.rs`) is #466's + `left(pg_walfile_name(pg_current_wal_lsn()),8)`, read in the transaction + that scans a refresh window, in the refresh probe and in the clock and + revision read issuance already makes, so fresh work costs no extra round + trip. `Prepared` carries the timeline its window was read on; work + reconstructed from a stored record carries none. It is runtime-only. +- **Durability test.** `probe_window_holding` checks, in one statement, that + the window's last row exists and matches the window's own predicate + (`accepted`, accepted and issued by the anchor), and whether its first row + exists. Missing first row with the last row held is `PrefixPruned` + (retention); anything else that fails is `NotHeld`. +- **Mode 1.** The candidate enqueue (`persist_prepared_candidate`) runs the + test inside the transaction that would write the candidate and returns the + typed `WindowNotHeld`, rolling back the share it carried. `submit_share` + answers it `stale-job`. The ledger counts it at the refusal under the new + `qbit_prism_stale_job_rejections_total{cause="window_not_held"}`, so a + refusal that finishes after the miner's acknowledgement deadline is counted + too, and logs a WARN naming the block and #619. +- **Mode 2.** The refresh rebuilds when the probe's timeline differs from the + published work's, labelled under the new + `qbit_prism_refresh_seconds{trigger="writer_timeline"}`, and never reuses a + cached window across a timeline. Issuance (`work_authority_in_epoch`) + admits work from another timeline, or with no timeline, only while the + window passes the test; a refusal there is deferred with the cause + `window_not_held …`, not `payout snapshot stale`. `save_compact_prepared` + and the issued-job repair require the test before writing a prepared record. +- **Landing text.** A landing whose window read fails probes the window and, + when it is `NotHeld`, records `window not held by this primary …` instead of + pruning or corruption. + +**Precondition: synchronized database clocks.** The durability test compares +time, not identity. A share reissued under a lost number fails it only because +the promoted host stamps it after the gap work's anchor. If the promoted +host's clock lags the old primary's by more than the time from the last gap +work to the first reissued share (seconds), a reissued row passes, the block +is offered, and it ends in `reconciliation` with a digest mismatch: the +pre-fix outcome. The HA reference and the ledger runbook state this +precondition. `candidate_window_switch` pins it: a reissued last row stamped +before the anchor passes the test, and only the landing's digest refuses the +window. + +The identity guard of option (d)(2) narrows the precondition but does not +remove it: + +- It would carry the last row's `share_id` in the runtime `Prepared` and + compare it in the enqueue. That needs no migration. +- It would exact-match all work prepared in memory: held jobs and published + work, which are modes 1 and 2. +- Work reconstructed from a stored record has no `share_id` to compare + without a migration, so it would keep the time test. Such a record holds a + gap window only if a release without this fix re-created it. + +The guard is not built here. Take it if clock synchronization across the +database hosts cannot be assured. + +**A window with no rows at all.** One whose newest row is absent is now refused +`stale-job` rather than enqueued anyway. For work young enough to be mined, +retention never drops a window's newest row, so this is lost history. + +**Verified by running** (minidev-alex1, PostgreSQL 16). The four tests in +`b619_gap_work_after_promotion.rs` cover mode 1, mode 2, a fenced switchover +with full replay, and a durable job resumed after an async promotion. + +- **With the fix:** all four pass in 5 of 5 runs, plus 5 earlier runs on the + final product code. +- **Against the product code of `06703682`:** in 5 of 5 runs, mode 1's gap + block is offered and stuck in `reconciliation` with `window range + incomplete: expected 15 shares, read 0`, and mode 2's gap work is issued + after the promotion. The switchover and resume blocks land there too; those + two tests fail only because the `window_not_held` series does not exist yet. +- **The enqueue's two cases** are in `candidate_window_switch`: a pruned + prefix still publishes with its ALERT, and a missing or reissued last row + is refused. `window_switch_tests` pins the landing's `window not held` + text for a last row reissued after the enqueue. diff --git a/docs/prism-b4-stratum-admission.md b/docs/prism-b4-stratum-admission.md new file mode 100644 index 000000000..3e0a9db6c --- /dev/null +++ b/docs/prism-b4-stratum-admission.md @@ -0,0 +1,189 @@ +# Stratum subscription admission: B4 first slice + +This first slice of [#276](https://github.com/Qbit-Org/qbit-mining-bootstrap/issues/276) +allocates the existing four-byte extranonce only on the first valid +`mining.subscribe`. Connect-only sockets, health probes, configuration and +authorization before subscription consume no session sequence value. Repeated +or pipelined subscriptions return the same eight-character lowercase hex value. +The backend allocator and wire width are unchanged. + +A failed allocation returns `backend-database-unavailable` (`database +unavailable`, #581) and leaves the connection unsubscribed. An immediate +failure can be retried on that connection. A timeout returns +`session allocation timed out`, labelled by what the allocation was waiting +on when the session's deadline passed (#655): `backend-database-unavailable` +while it waited on the ledger database, which the native coordinator's +allocation always does, and `backend-rpc-unavailable` for a backend that +names no database step. A sequence value already consumed before the timeout +is not published or reused. + +The initial-job lifetime now begins during connection setup, before the later +subscription allocation. The base runtime began that lifetime after eager +allocation completed. +Allocation time therefore counts toward the connection's initial-job lifetime. +The configured duration and timer cutoff are retained, with no reset or extension +on subscribe, failure or retry. A full allocation timeout can exhaust that +lifetime and close the connection after its error response; recovery reconnects +and obtains a fresh value. Delivery deadlines, retained-work expiry, original +worker permits, share grace and candidate/revision checks retain their existing +behavior. + +Connection failures now emit structured tracing events with an `error` field. +No new request-rate or IP policy, configuration setting, allocator migration or +wire change is part of this slice. + +## Evidence + +The socket tests use the production listener and request loop. Five regressions +cover no allocation before subscribe; malformed and repeated subscriptions; +immediate allocator failure and retry; timeout after a consumed allocation; and +structured tracing for an actual TCP reset. On `3.x.x` base `f316eb9`, the +independent focused suite passed 64 tests: 36 library, five readiness, five new +socket and 18 existing protocol cases. +All passed; none were ignored in that invocation. + +An explicit PostgreSQL test admits and closes 10,000 real sockets without +subscription, synchronizing each session's entry and exit. It delegates allocation +to the real Ledger implementation in a unique disposable schema. Before this +change, `(last_value, is_called)` advanced from `(1, false)` to `(10000, true)`. +After the change it stays `(1, false)`. A subsequent subscription through the +same listener returns `00000001` and changes the sequence to `(1, true)`. + +The PostgreSQL test is ignored in ordinary runs and must be invoked explicitly +in database CI. Explicit invocation without the required DSN fails; it does not +silently pass or skip. Use a disposable database: + +```sh +LC_ALL=C PRISM_TEST_DATABASE_URL="$DISPOSABLE_POSTGRES_URL" \ + cargo +1.89.0 test --locked -j2 -p qbit-prism-server \ + --test stratum_admission_postgres -- --ignored --exact \ + ten_thousand_unsubscribed_connections_do_not_advance_postgres_sequence --nocapture +``` + +The focused validation was: + +```sh +LC_ALL=C PRISM_TEST_DATABASE_URL="$DISPOSABLE_POSTGRES_URL" \ + cargo +1.89.0 test --locked -j2 -p qbit-prism-server \ + --lib --test stratum_protocol --test stratum_admission --test readiness_rpc \ + -- --test-threads=2 --nocapture +``` + +These results qualify this slice; they do not claim a fresh all-targets run. + +## Remaining work and scope + +Wrap-safe allocation remains pending. The current sequence is still `NO CYCLE`; +subscribe floods can still consume it. Migration009 must cover active reservations, +unexpired issued jobs, resumed jobs and concurrent frontends before reuse is safe. +A wider wire format requires an explicit compatibility decision. + +Trusted proxy and edge address handling remain P2 follow-ups in +[#262](https://github.com/Qbit-Org/qbit-mining-bootstrap/issues/262); load +balancer policy itself belongs to +[#281](https://github.com/Qbit-Org/qbit-mining-bootstrap/issues/281). The +per-source cap, malformed-frame budget, bounded unknown-job lookups and the +authorization budget are the second slice, below. + +# Second slice: per-source cap and per-session budgets + +This slice adds one admission cap keyed on the observed peer address and three +per-session rate budgets. Every one of them defaults to disabled, so a default +deployment behaves exactly as the first slice did. Valid `mining.submit` +traffic is never charged against any budget, and no share acceptance, landing +or refresh behavior changes. + +## What each limit does + +`PRISM_STRATUM_MAX_CONNECTIONS_PER_IP` bounds concurrent connections from one +observed peer address. The decision is taken in the accept loop, from the +address the accepted socket reports, before the global permit is taken and +before any session task is spawned: a refused source costs one hash lookup and +no database round trip. The refused socket is dropped with no JSON-RPC +response, the same shape as the existing global limit. The registry holds weak +references and prunes dead keys on every decision, so it is bounded by live +connections rather than by every address ever seen. The ordinary and +high-difficulty listeners share one registry, so a miner using both ports +spends two slots from the same address budget. + +The three budgets are plain per-session counters over one shared fixed window, +`PRISM_STRATUM_SESSION_BUDGET_INTERVAL_SECONDS` (default 60). A budget is a +rate, never a session lifetime total: a whole window's allowance is admitted as +a burst and refills in full at the next window. Exceeding one answers the +request and then closes the connection, recording the reason on +`qbit_prism_stratum_connection_refusals_total`. + +- `PRISM_STRATUM_MAX_MALFORMED_FRAMES_PER_INTERVAL` counts malformed **frames**, + not bytes, and includes the malformed non-submit frames that were previously + answered and forgotten at no accounting cost. One frame is one charge + whatever it weighs. +- `PRISM_STRATUM_MAX_UNKNOWN_JOBS_PER_INTERVAL` counts every `mining.submit` + whose job is not held by the session and whose ledger resume lookup misses. + Each such submit costs exactly one lookup, so the budget bounds the ledger + work one session can drive: a flood of unknown-job submits produces at most + the budget plus one lookup per interval before the disconnect, whether the + IDs repeat or not. There is deliberately no negative cache. A resume miss is + not proof that the ID is bogus: besides honest staleness after a tip or + payout-revision change, a resume returns a miss when its publication stamp + changes mid-resume, when issuance authority loses a lease or tip race during + revalidation, and while this frontend is momentarily behind on the parent or + payout revision. Each of those clears on the miner's next share, so every + submit must re-query, and each counts as a miss against the budget. That is + also why the default stays 0: with the budget off, behavior is exactly as + before, and an operator who enables it must size it above the worst honest + miss rate rather than rely on a cache. +- `PRISM_STRATUM_MAX_AUTHORIZE_ATTEMPTS_PER_INTERVAL` counts `mining.authorize` + attempts, charged before the address validation RPC, which is what bounds the + `validateaddress` calls one connection can drive by cycling usernames. + Re-authorizing to switch usernames stays supported: within the budget it + behaves exactly as before. + +## Recommended starting values + +These are starting points for a frontend at the default +`PRISM_STRATUM_MAX_CONNECTIONS=384`, not tuned limits. Raise anything a real +deployment brushes against; every one of them refuses a legitimate miner if it +is set below an honest rate. + +| Setting | Recommended start | Why | +| --- | --- | --- | +| `PRISM_STRATUM_SESSION_BUDGET_INTERVAL_SECONDS` | 60 | Long enough that a burst is averaged out, short enough that a spent budget recovers within a miner's patience. | +| `PRISM_STRATUM_MAX_MALFORMED_FRAMES_PER_INTERVAL` | 64 | An honest miner sends none. 64 tolerates a broken proxy or a firmware bug without ejecting the rig on the first bad frame. | +| `PRISM_STRATUM_MAX_UNKNOWN_JOBS_PER_INTERVAL` | 256 | Must sit well above an honest miner's in-flight submits per interval, because in the worst case every one of them misses: after a tip or payout-revision change, or through a run of transient resume races, all work a rig still holds arrives as unknown. At the default 15-second vardiff target a connection submits about 4 shares a minute, and a connection holds at most 64 retained jobs (`PRISM_STRATUM_SAME_TIP_JOB_RETENTION_PER_CONNECTION`); 256 covers several whole-retention flushes in one window. | +| `PRISM_STRATUM_MAX_AUTHORIZE_ATTEMPTS_PER_INTERVAL` | 32 | Normal sessions authorize once or twice. 32 leaves room for a client that re-authorizes on every reconnect attempt inside one connection. | +| `PRISM_STRATUM_MAX_CONNECTIONS_PER_IP` | 0 | See the runbook rule below. | + +## Runbook rule for the per-source cap + +**Leave `PRISM_STRATUM_MAX_CONNECTIONS_PER_IP` at 0 unless the deployment is +known to preserve miner source addresses.** + +The cap keys on the observed last hop. This repository has no PROXY-protocol +support and ships no Stratum load balancer: it is operator-supplied +([#281](https://github.com/Qbit-Org/qbit-mining-bootstrap/issues/281), +`docs/prism-ha-reference-architecture.md`). Behind a terminating TCP proxy, +behind NAT egress, or behind a Docker userland proxy, many or all miners share +one key, and any cap below the global limit throttles the whole pool. The +failure mode is silent from the miner's side: an accepted connection closed +with no response. Before enabling it, confirm the topology delivers real client +addresses end to end, for example a pass-through L4 balancer or direct +exposure with `network_mode: host`. + +When it is safe to enable, size it with reconnect-storm headroom of about twice +steady state. A per-source slot is released only when the session task ends, so +after a tip change or a balancer failover a farm reconnecting every rig at once +transiently needs roughly double its steady-state slots; a cap tuned to steady +state ejects legitimate reconnects. Note that a single NAT'd farm can present +hundreds of rigs from one address, so start from the observed maximum per +address and add the headroom, never from a guess. + +Because a metric label may never carry a peer address, a refusal is counted as +`qbit_prism_stratum_connection_refusals_total{reason="ip_limit"}` and the +address appears only in the matching structured tracing event, which also +carries the configured limit and the listener name. Search the log when that +counter rises; a misconfigured cap is otherwise invisible per address by +design. + +Load balancer policy, PROXY protocol and trusted-edge address handling are out +of scope here: document, do not implement. **This slice does not complete +#276's admission story beyond the criteria listed in that issue.** diff --git a/docs/prism-b8-miner-parity.md b/docs/prism-b8-miner-parity.md new file mode 100644 index 000000000..0ae04a019 --- /dev/null +++ b/docs/prism-b8-miner-parity.md @@ -0,0 +1,167 @@ +# Native miner decision parity (B8) + +Reference: `2.x.x` commit `95ffe063846d51f83999a66cc654da5f7476fdef`, +`tests/test_prism_retained_jobs.py`, `tests/test_prism_hot_path.py`, +[`tip_refresh.py`](https://github.com/Qbit-Org/qbit-mining-bootstrap/blob/95ffe063846d51f83999a66cc654da5f7476fdef/lab/prism/tip_refresh.py) and [`job_delivery.py`](https://github.com/Qbit-Org/qbit-mining-bootstrap/blob/95ffe063846d51f83999a66cc654da5f7476fdef/lab/prism/job_delivery.py). + +The native tests execute `Coordinator::submit`, `refresh_once`, job construction, +persistence and resume. Narrow `SubmitLedger` and `WorkLedger` interfaces replace +only storage I/O in ungated tests. Production implementations delegate to the +existing ledger operations, including transactional `append_at_revision`. +Loopback RPC fixtures, controlled completion gates and a paused monotonic clock +exercise the real decision code. Socket tests use real Coordinator persistence +and resume; there is no successful in-memory resume substitute. + +## Restored behavior + +Detected chain state fences candidates immediately. Share credit selects one +coherent view containing the prepared payout and published tip's hash, +predecessor, observation age and publication provenance. Older asynchronous +observations and parent lookups cannot overwrite a newer view. Submit fallback RPC does not invent a published +transition or turn an unavailable chain lookup into a stale-share rejection. +After fallback RPC completes, admission captures the current publication pair +again; only a publication matching the returned hash can supply grace provenance. +The live CTV fee check still rejects an already obsolete fee before fallback RPC. + +Long-poll replies wake refresh. A changed valid hash triggers fresh chain-info +verification without waiting for work construction or persistence. Verification +is ordered when its request starts, so a later observation can supersede it. +It detects the chain tip without publishing work or refreshing full-readiness +timestamps. Failed or malformed verification revokes admission; a notification +channel failure instead leaves last-good readiness to age and retries with +shutdown-aware backoff. Same-hash notifications require no verification RPC. + +The last published work retains ordinary authority for +`PRISM_SUBMIT_TIP_MAX_AGE_SECONDS` (default 10). A detected departure also grants +the pinned legacy replacement-build lease, bounded by +`PRISM_TEMPLATE_REFRESH_FAILURE_EXIT_SECONDS` (default 120) from the first +departure. Either budget can preserve authority. Failed refreshes and newer +detected tips do not renew the lease; returning to the published tip clears it. +Zero submit-cache age forces RPC and disables this lease. A lease cannot revive +an older same-parent payout snapshot. This restores the setting's miner-credit +budget; native process supervision and template refresh retries are unchanged. + +After successful publication, exactly-one-parent share grace starts at each +connection's first successful replacement delivery. Failed or pending delivery +keeps grace open. Same-tip deliveries preserve the first anchor; a real tip flip +reanchors it. Startup baseline and submit-only observation cannot open grace. +A coherent published retention hint survives cache expiry or cache age zero; +credit classification still performs its own authoritative checks. + +Active eviction retains original context, worker, target and version mask in a +session-local graveyard. Its same-tip TTL starts at burial and its same-tip cap +is the configured count N. Previous-parent entries use delivery grace and are +removed when a known predecessor proves them ineligible. The graveyard's hard +3N cap accommodates the old active set, old graveyard and new same-tip graveyard; +including the unchanged active N bound, each session holds at most 4N entries. +Original username admission permits survive while retained work can credit and +are reused on same-connection reauthorization. Expiry, actual capacity eviction, +payout replacement and disconnect release them. Reconnect still uses the exact +worker ownership, same-parent, payout and absolute-expiry checks in Coordinator. +Absolute expiry is checked again after an awaited resume returns. +The connection timer checks both active and retained work before applying its +initial-work timeout. Expiry of an older resumed job cannot disconnect a mature +connection while newer retained work can still earn credit. Once both sets are +empty, the existing timeout and username-permit cleanup still apply. + +Accepted records preserve issued economics and original-worker deduplication. +Eligible prior-parent shares use the stale-grace policy and the current durable +revision. Published-build lease shares retain ordinary policy. Neither exception +allows an old-parent or old-revision block candidate. Readiness revocation during +an awaited lookup rejects before persistence, and the ledger's transaction +remains the final cross-frontend revision fence. + +## Prepared dependency lifetime + +Issued work now retains its original shared prepared record atomically through +its absolute reconnect deadline. Expired metadata can be extended; a physically +pruned dependency is repaired from the original typed record under bounded +serialization capacity. Original worker, target, policy, coinbase suffix and +payout revision remain unchanged. See [the dependency lifetime contract and +focused regression evidence](prism-b8-prepared-dependency.md). + +## Ungated regression coverage + +| Boundary | Executable coverage | +| --- | --- | +| Frozen issued economics; grace delivery and exact deadline; startup; skipped/reorg tips; old-parent block-only rejection; same-parent payout replacement | `coordinator/miner_tests/credit.rs` | +| Cache reuse, no observation, aged/zero cache, fallback failure, parent RPC failure | `coordinator/miner_tests/observations.rs` | +| Slow observation and parent completions; coherent selection; readiness revocation; revision change before append | `coordinator/miner_tests/interleavings.rs` | +| Coupled payout/lease selection; real publication during fallback; unrelated publication rejection; CTV fee rejection before failed fallback | `coordinator/miner_tests/admission_races.rs` | +| Actual long-poll loop during blocked refresh; fresh verification ordering; failed/malformed verification; same-hash RPC count; shutdown during wait, verification and backoff | `coordinator/miner_tests/blockwait.rs` | +| Zero-grace published authority; nonrenewing divergence lease; expiry; return/redeparture; candidate-only observation | `coordinator/miner_tests/published_lease.rs` | +| Real refresh orchestration at 1/32/128 clients with identical RPC counts; build/persist/publish gates; repeated refresh failure; stale-vs-unavailable resume; obsolete payout issuance | `coordinator/miner_tests/refresh.rs` | +| Durable dependency expiry; delayed publication; return/redeparture; original bootstrap repair; concurrent repair and cancellation | `coordinator/miner_tests/prepared_expiry.rs` | +| Actual socket delivery, original-worker retained dedup/weight, unknown-before-node-RPC, absolute expiry crossing resume | `stratum/stale_grace_tests.rs` | +| Mature-session timer after resumed-job expiry, retained-job credit and final cleanup; unauthenticated/never-usable timeout controls | `stratum/session_timer_tests.rs` | +| Zero/aged cache undelivered grace, real reconnect ownership and parent controls, TTL/capacity/permit lifecycle, unrelated-parent pruning, hard retention bound | `stratum/retained_tests.rs` | + +All paths are under `crates/qbit-prism-server/src/`. The prepared-dependency +change adds ten ungated regressions to the earlier miner-parity coverage. The config CLI test also covers both restored settings' defaults, +fractional values, zero behavior and invalid inputs. Existing protocol tests +exercise permit reuse across reauthorization, timer expiry, payout replacement, +and disconnect with active capacity eviction. + +The grace negative control replaces the production grace predicate with false: +`observed_tip_pending_prepared_and_revision_still_credits_exact_prior_parent` +fails with stale-job. Restoring the predicate makes the same test pass. +Readiness revocation, stale resume classification and obsolete payout issuance +also reproduced as failing real-Coordinator tests before their fixes. + +The review regressions were run against unchanged pre-fix production code on +the rebased branch. They reproduced an old payout borrowing a newer payout's +lease, publication-during-fallback returning stale-job, and a discarded long-poll +notification admitting an old-tip candidate while real refresh awaited persistence. +The payout race replays the handoff between two real refresh-built publications +at the existing locks; it executes the actual submit and final revision fence. +Restoring the fixes passes all 52 Coordinator miner tests, including the ten +new admission and long-poll cases. + +## Qualification + +Run the native suite with Rust 1.89 and the lockfile. The database and node +fixtures must be disposable; each database test uses an isolated schema. + +```sh +CARGO_TARGET_DIR=/tmp/prism-b8-target \ +PRISM_TEST_DATABASE_URL="$DISPOSABLE_POSTGRES_URL" \ +QBITD_BIN="$LOCAL_REGTEST_QBITD" \ +PRISM_TEST_PG_BIN_DIR="$LOCAL_POSTGRES_BIN" \ + cargo +1.89.0 test --locked -j2 -p qbit-prism-server --all-targets \ + -- --test-threads=2 --nocapture +``` + +With the same disposable environment, run the two explicit PostgreSQL acceptance +targets as CI does. They fail when the database configuration is missing: + +```sh +cargo +1.89.0 test --locked -j2 -p qbit-prism-server \ + --test issued_job_dependency -- --ignored --test-threads=2 +cargo +1.89.0 test --locked -j2 -p qbit-prism-server \ + --test stratum_admission_postgres -- --ignored --exact \ + ten_thousand_unsubscribed_connections_do_not_advance_postgres_sequence +``` + +Final shipping qualification also runs `--workspace --all-targets`, exercising +the separately landed payout-vector, window-corpus and SQL-window-oracle tests. +The commit-specific result below is historical; final-head results belong in the +PR's qualification record rather than being inferred from an older run. + +A full qualification must report the gated database, live node and physical +failover tests as exercised, rather than counting their environment-skipped +early returns as proof. In the high-difficulty live test, a newly connected miner +can receive still-published work during replacement construction. The invalid +future-timestamp case therefore waits for delivery of the confirmed block's +parent before submitting; the existing no-ACK/no-credit and terminal outbox +assertions remain unchanged. + +Qualification on 2026-09-10 after rebasing onto `3.x.x` +`f39442755448b1483bd5e183d8d972c7e461b997`, at runtime commit +`551b741cb4a1b7b8cb9729936b5b9f6929d60942` (historical runtime), with all three +disposable integration gates enabled: +208 tests passed, zero failed, zero ignored, across 20 target summaries. This +includes all six candidate-lease database tests, 29 ledger database tests, +eight real qbitd regtest cases, two physical PostgreSQL failover tests, five +readiness cases and 18 Stratum protocol tests. No environment-skip message was +emitted. The live high-difficulty case credited exactly 1,000,000 network work +and rejected the invalid future-timestamp block without ACK or share credit. diff --git a/docs/prism-b8-prepared-dependency.md b/docs/prism-b8-prepared-dependency.md new file mode 100644 index 000000000..bd42d30ca --- /dev/null +++ b/docs/prism-b8-prepared-dependency.md @@ -0,0 +1,84 @@ +# Issued-work prepared dependency lifetime + +A miner job may be issued while published tip A remains authoritative during a +failed replacement build. Its shared prepared record must remain readable until +that job's original reconnect deadline. An initial TTL calculated when A was +built cannot guarantee this: same-tip reuse, delayed publication, late divergence +and a tip returning before departing again can all make valid issuance occur +later. With default intervals, prepared A could expire at t165 while a job issued +at t170 remained valid until t200; immediate reconnect incorrectly returned an +unknown job. + +`Ledger::save_issued_job` atomically guards the prepared dependency and saves the +compact issued record under the existing settlement lock and current payout +revision fence. The dependency identifies its original storage key, parent and +payout revision separately from that current revision. The child's JSON reference +and deadline must match the typed arguments. Its absolute expiry is fixed before +persistence; waiting, repair and exact duplicate retries cannot extend it. +Mismatching immutable data and elapsed or unrepresentable deadlines fail. + +The common path reads only dependency metadata with `FOR KEY SHARE`, which blocks +concurrent deletion while permitting a non-key update. It avoids loading or +serializing the prepared payload. A dependency already outliving the issued job +receives no expiry update. Otherwise its expiry advances monotonically to the +child deadline plus 60 seconds, avoiding an expiry update for every miner. +Row locking itself can still cause disk writes; this is not a zero-WAL claim. +[PostgreSQL 16 locking documentation](https://www.postgresql.org/docs/16/explicit-locking.html) +describes these row-lock semantics. + +A missing dependency returns `PreparedMissing` after releasing the transaction, +without saving a child. Coordinator serializes the exact original typed record +only on this cold path. An `Arc` retains that record and shares its snapshot and +bundle with live prepared work; there is no eager serialized JSON cache. Resume +also keeps the original record, including bootstrap `bundle: null` and its +original coinbase suffix, even if reconstructing miner work requires a bundle. +A per-`Arc` mutex coalesces concurrent repairs for that live prepared +record, and the existing build-slot semaphore bounds serialization. Repair +coalescing is per in-memory record, not a global storage-key promise. Both owned guards live through the actual +blocking work when its asynchronous waiter is canceled. A follower retries +compact persistence before doing heavy work. Repair rechecks issuance admission, +retains the original child deadline, and verifies any competing existing prepared +row exactly; it never replaces a conflicting payload. The repair and child save +commit together. + +Garbage collection retains its 4096-row batch limit and now rechecks expiry in the +outer `DELETE` predicate. If a candidate ID was selected before a saver renewed +its row, PostgreSQL re-evaluates that predicate against the updated row after +waiting for the lock. If deletion wins first, issuance observes a missing record +and repairs it before saving the child. This relies on the production default +Read Committed isolation. [PostgreSQL 16 transaction isolation documentation](https://www.postgresql.org/docs/16/transaction-iso.html) +describes the recheck behavior. + +Persisted JSON and schema remain compatible with existing records. This is a +behavior restoration, not the upcoming compact-window representation in #273. +All native frontends sharing this jobs table must use the corrected GC query; +an older frontend's unconditional outer deletion could still remove a renewed +row. Once issuance stops, existing retention and the bounded renewal headroom +allow the prepared record to expire; issued leases are never renewed. + +## Focused verification + +`coordinator/miner_tests/prepared_expiry.rs` contains ten ungated tests using real +Coordinator refresh, build, persistence, resume and submit decisions with loopback +RPC and controlled storage/serialization boundaries. The memory ledger now +filters reads by expiry and models the atomic dependency save. The initial +expiry/reconnect regression failed against the old production persistence path +and passed after this change. + +Coverage includes expired and physically deleted prepared rows, reuse and delayed +publication, returning/redeparting tips, original/current revision separation, +unchanged target/worker/version mask, independent candidate fencing, bootstrap +identity after resume, fixed child expiry, competing repairs, duration overflow, +coalesced repair, and cancellation retaining capacity through blocking completion. +The historical native library run passed 107 tests, with no ignored or skipped +tests; final-head counts belong in the PR qualification record: + +```sh +CARGO_TARGET_DIR=/tmp/prism-b8-expiry-target \ + cargo +1.89.0 test --locked -j2 -p qbit-prism-server --lib -- --test-threads=2 +``` + +Real PostgreSQL transaction and GC concurrency qualification is recorded separately +by `tests/issued_job_dependency.rs`; the ungated memory fixture does not establish +PostgreSQL lock semantics. Full database/node/physical-failover qualification is +required again after integration and the final base update. diff --git a/docs/prism-candidate-storage.md b/docs/prism-candidate-storage.md deleted file mode 100644 index cb2abbad4..000000000 --- a/docs/prism-candidate-storage.md +++ /dev/null @@ -1,391 +0,0 @@ -# Bounded candidate persistence and replay - -This guide covers the 2.x.x work for [#255](https://github.com/Qbit-Org/qbit-mining-bootstrap/issues/255) -and its release dependency on [#254](https://github.com/Qbit-Org/qbit-mining-bootstrap/issues/254). -It includes the complete cursor enumeration introduced by PR #256 and the -offline accepted-pending recovery command introduced by PR #259. - -## Storage and resource settings - -Apply `001_share_ledger.sql` followed by `002_candidate_bodies.sql` before -starting this release, whichever storage version it writes. -`PRISM_POSTGRES_INIT_SCHEMA=1` performs this at startup; the migrations are -idempotent. Replay headers and terminal outbox statements use the migrated -columns for both storage versions, so the coordinator and the offline recovery -command refuse a database without `002`; `PRISM_CANDIDATE_STORAGE_VERSION=1` -stops version 2 writes but does not make an unmigrated database usable. -Existing version 1 JSONB outbox rows remain intact -and readable. New version 2 rows reference a sealed body and carry a small replay -header; the old `candidate` column is null for those rows. - -| Setting or boundary | Default / limit | -| --- | --- | -| `PRISM_CANDIDATE_STORAGE_VERSION` | `2`; set `1` for compatible-reader rollout or rollback | -| `PRISM_CANDIDATE_SPOOL_DIR` | Empty selects the system temporary directory; choose a persistent-capacity mount for large workloads | -| `PRISM_CANDIDATE_SPOOL_RESERVATION_BYTES` | 4 GiB of admitted candidate body and index files per ledger instance | -| `PRISM_BLOCK_REPLAY_PAGE_SIZE` | 1,024 descriptors; supported range 1–1,024 | -| Replay header page | 256 KiB and the configured row cap | -| Stored chunk / read page | 256 KiB per chunk; at most three chunks per read | -| Index transfer | 256 page entries or 64 field spans per statement | -| Shared helper admission | One legacy, record-normalization or finalization helper at a time per process; deadline/cancellation also apply while awaiting admission | -| Legacy conversion / JSONB comparison helper | 4 GiB address-space limit, 600-second deadline, 64 KiB retained diagnostics | -| Oversized audit record helper | 4 GiB address-space limit, 300-second deadline, 1 MiB metadata response ceiling | -| Finalization SQL transport | 4 GiB statement spool, one helper under the writer gate, 4 GiB helper address space, 600 seconds or the shorter operation deadline | - -Replay retains at most 1,024 queued descriptors. When admission fills, enumeration -pauses before registering further credit floors or payout previews. The existing -enumeration-owed gate keeps job builds blocked until the queue drains and the -remaining durable rows are handled, including when the last SQL page was exhausted. - -Hydration limits ordinary native JSON decodes and the aggregate metadata skeleton -to 1 MiB. Larger arrays, objects and strings remain spool-backed views with disk -indexes. String decoding carries incomplete escapes and surrogate pairs across -slices; block submission streams its encoded body with an exact content length. -Consensus-bounded metadata needed by existing consumers is decoded in slices and -joined explicitly. This is a bound on coordinator work units, not a claim that -arbitrary metadata or PostgreSQL/helper memory is independent of input size. - -The codec preserves the historical identity bytes, including neutralizing only -the pending share's acknowledgment timestamp. Append retries compare the digest -and the actual body, using isolated JSONB comparison when different encodings -may be equivalent. Standalone intent persistence retains its historical -digest-only retry rule. Staging and sealing grant no share credit; publication -and credit remain one synchronous, fenced commit. - -Candidate spool reservations include the disk index and are released explicitly -on close and failed hydration. Disk exhaustion and helper capacity/deadline -failures leave valid durable candidates pending for retry. They must not be -classified as corrupt bodies. Monitor the spool filesystem and container memory -separately: reservations do not bound filesystem page cache. - -Finalization retains the existing atomic SQL. Its text streams to a temporary -file; an isolated `psql` process owns the complete statement buffer. PostgreSQL -still parses that complete statement, so server memory and temporary-file -capacity remain size-dependent. This transport preserves the native ledger's -exact DSN, private schema options, writer-session guards and operation deadlines. -The fixed helper limits are implementation constants, not environment settings. - -## Problem and evidence - -A streamed candidate identity hash did not bound the surrounding pipeline. -[Candidate construction](https://github.com/Qbit-Org/qbit-mining-bootstrap/blob/d39280070517a1bd66e09d5cfe8cd330a2be1f2f/lab/prism/block_candidates.py#L1145) -materialized the share sequence and validated the entire intent with `json.dumps`; -[append](https://github.com/Qbit-Org/qbit-mining-bootstrap/blob/d39280070517a1bd66e09d5cfe8cd330a2be1f2f/lab/prism/share_ledger.py#L4223) -subsequently encoded the nested intent through `_jsonb_literal`; -[startup replay](https://github.com/Qbit-Org/qbit-mining-bootstrap/blob/d39280070517a1bd66e09d5cfe8cd330a2be1f2f/lab/prism/share_ledger.py#L4580) -decoded full intents inside a single `json_agg`. -Limiting candidate count, or returning one row per candidate, still permits one -large native JSON decoder call. The downstream reproducibility check also read -an aggregate audit window and built a Python ID set, and canonical audit -finalization reparsed complete bundles. - -The original portable experiment used CPython 3.14.7, 375,000 synthetic shares -and approximately 232 MB per candidate. It reproduced long observer pauses with -cyclic GC disabled. This establishes the codec mechanism; it is neither a valid -mined-block fixture nor proof of the instruction responsible for a production -exit. Production runs x86_64. Recorded one-candidate replay and retirement remain -plausible contributors to the observed exits, which had little recorded GC time. - -The membership experiment below separately measures the downstream omission -check against PostgreSQL 16.15, psycopg 3.2.13 and CPython 3.14.7 in ARM64 Docker. -The fixture contains 375,000 actual ledger rows and runs a real writer-session -proof actor alongside the operation. Runs were serial; each result is one run, -not a distribution or a production capacity estimate. -The first runs used default allocator settings. Repeats with the coordinator's -`MALLOC_ARENA_MAX=2` policy are listed separately. - -| Phase | GC | Wall time | Largest observer delay | Process high-water RSS | -| --- | --- | ---: | ---: | ---: | -| Bounded membership | disabled | 0.449 s | 5.8 ms | 48 MiB | -| Prior aggregate read | disabled | 1.432 s | 309.9 ms | 903 MiB | -| Bounded membership | enabled | 0.465 s | 7.5 ms | 48 MiB | -| Prior aggregate read | enabled | 1.554 s | 352.5 ms | 903 MiB | -| Bounded membership, 1,024 rows | disabled | 0.009 s | 1.6 ms | 48 MiB | -| Bounded membership, two arenas | disabled | 0.453 s | 7.3 ms | 48 MiB | -| Prior aggregate, two arenas | disabled | 1.487 s | 318.7 ms | 903 MiB | -| Bounded membership, two arenas | enabled | 0.482 s | 13.5 ms | 48 MiB | -| Prior aggregate, two arenas | enabled | 1.483 s | 350.3 ms | 903 MiB | - -After adding native COPY backpressure, fresh two-arena runs measured 0.453 s -(GC disabled) and 0.508 s (enabled), with maximum observer delays of 15.0 ms and -70.9 ms respectively and the same 48 MiB high-water RSS. Their aggregate -controls measured 320.7 ms and 323.9 ms observer delays and 903 MiB high-water -RSS. Maximum bounded-phase heartbeat wake lateness was 31.4/54.1 ms, with -102.9/76.2 ms proof calls. These single-run variations are recorded rather than -attributed to a production cause; they are not a worst-case latency guarantee. - -There were no proof errors. Maximum proof duration was 8.4 ms for the large -GC-disabled bounded phase versus 320.8 ms for the aggregate phase. GC-enabled -proof maxima were 23.8 ms and 357.7 ms. The enabled aggregate run also recorded -a 28.7 ms generation-0 collection. Destruction of its returned row graph took -25.8 ms separately. These measurements must not be added together to infer a -production pause, and RSS includes the interpreter and native client. -The improved harness also separates heartbeat wake lateness from proof-call -duration: in the two-arena GC-enabled run, the aggregate phase delayed a wake -by 298.4 ms even though its slowest proof call took only 6.0 ms. Bounded -membership had 9.9 ms wake lateness and a 42.6 ms proof-call maximum. A short -database round trip therefore does not by itself establish timely scheduling. - -A separate audit experiment writes a 231,202,664-byte JSON artifact containing -375,000 synthetic shares and scans, walks, streams, and retires its actual -`CanonicalAuditBundleView`. Three artifacts run sequentially; the baseline -performs one decode of their combined array, matching the aggregate boundary. -It uses CPython 3.14.7 ARM64 Docker and `MALLOC_ARENA_MAX=2`. - -| Audit phase (three candidates) | GC | Largest observer delay | High-water RSS | -| --- | --- | ---: | ---: | -| Streamed scan/walk/encode/retirement | disabled | 8.5 ms | 32 MiB | -| Whole aggregate decode | disabled | 1,449 ms | 2.15 GiB | -| Whole-graph reference-count retirement | disabled | 142 ms | — | -| Streamed scan/walk/encode/retirement | enabled | 21.3 ms | 32 MiB | -| Whole aggregate decode | enabled | 1,473 ms | 2.15 GiB | -| Whole-graph reference-count retirement | enabled | 139 ms | — | - -The GC-enabled aggregate recorded one 117 ms collection. Explicit view retirement -took less than 0.14 ms in each run and released both weakly observed owners before -cyclic GC. A separate traced single-candidate run peaked at 1.35 MB for scan, -1.33 MB for lazy iteration and 2.14 MB for encoding; traced wall times are excluded -from the timing comparison. A 1,024-share control also passes. This probe has an -observer thread, not a native lease actor; use the membership and full pipeline -qualification for heartbeat behavior. - -Independent PostgreSQL backpressure qualification also froze a disposable server -for six seconds during COPY. The default Linux `LibpqWriter` accepted 60 MiB while -frozen and grew client RSS by 49 MiB. Per-chunk flushing kept RSS flat and blocked -until the server resumed; both preserved all 3,532,045 rows. A trigger-throttled -consumer confirmed statement-timeout rollback, connection reuse and producer -cancellation. Unfrozen throughput was comparable (4.66–4.88 s default versus -4.81–4.87 s flushing). This measures native transport allocation separately from -Python JSON allocation. The committed slow-consumer regression fails with the -prior writer and passes with explicit flushing. - -## Native candidate pipeline qualification - -`tests/perf/candidate_storage.py` creates a private disposable PostgreSQL schema -and exercises the production prepare, append, exact retry, header enumeration, -hydration, lazy walk, close, terminalization and janitor paths. A real -writer-session proof actor runs beside each phase. The fixture supplies 375,000 -synthetic shares and 232,787,365 canonical bytes per candidate, under CPython -3.14.7, PostgreSQL 16, psycopg 3.2.13 and ARM64 Docker with `MALLOC_ARENA_MAX=2`. -These measurements follow the final spool-view integration and are separate -from the real mined-block gate. - -| Candidates | GC | Largest observer delay | High-water RSS | Largest proof call | Largest heartbeat wake lateness | -| --- | --- | ---: | ---: | ---: | ---: | -| 3 | disabled | 28.0 ms | 103 MiB | 5.3 ms | 23.8 ms | -| 3 | enabled | 5.9 ms | 101 MiB | 3.9 ms | 7.6 ms | -| 1 | disabled | 4.6 ms | 101 MiB | 3.9 ms | 6.7 ms | -| 3 × 20 generations, 1,024 shares | disabled | 11.1 ms | 59 MiB | 7.4 ms | 7.9 ms | - -All four runs passed without proof errors. Each explicit close returned spool -reservations to zero and released the weakly observed body owner. The larger -delay in the GC-disabled run is retained as observed; these serial single runs -are not a worst-case guarantee or production-equivalent timings. - -```sh -python tests/perf/candidate_storage.py \ - --database-url "$TEST_DATABASE_URL" --shares 375000 --candidates 3 --gc off -``` - -Repeat with `--gc on`, with `--candidates 1`, and with -`--shares 1024 --candidates 3 --generations 20 --gc off` for repeated ownership -retirement without cyclic GC. This harness creates and drops its own private -schema; supply a disposable database and run timing measurements serially. - -## Reproducibility check - -A replayed candidate is reproducible only when its recorded share IDs cover -every ID in the durable window at its declared anchor and difficulty. Order, -duplicate IDs, and extra recorded IDs do not change this historical subset -rule. Missing durable IDs still reject replay. The check does not substitute a -new ledger window for the recorded candidate window or grant share credit. - -`candidate_window.py` streams IDs through PostgreSQL COPY into a private -transactional temporary table, then returns one boolean from an exact text -anti-join. COPY writes are at most 16 KiB and flush the native output buffer before the -producer advances, with cancellation/deadline checks at -most 256 records apart. Oversized individual strings are escaped and encoded in -4,096-character pieces. PostgreSQL join memory can spill under an 8 MiB -`work_mem`; the operation uses a read slot and never takes the writer-lease -lock. Failure rolls back the temporary table, and the candidate's existing retry -starts from its replayable immutable source. The psql fallback spools its input -and caps captured diagnostic text at 64 KiB. In-memory ledger compatibility uses -an on-disk SQLite ID index with a 2 MiB page cache. Its digest index narrows -lookups, then compares the actual ID bytes from file spans; a digest collision -cannot satisfy membership. - -## Running qualification - -Use disposable databases only. The membership harness deliberately replaces -share rows in the supplied database. Initialize the ledger schema first, then -run in the same Python image as the coordinator: - -```sh -python tests/perf/candidate_window_membership.py \ - --database-url "$TEST_DATABASE_URL" --shares 375000 --gc off \ - --prepare-disposable-fixture -``` - -For the standalone audit experiment: - -```sh -python tests/perf/canonical_audit_retirement.py \ - --shares 375000 --candidates 3 --gc off --baseline -``` - -Repeat with `--gc on` and with `--shares 1024`. Use `--trace` in a separate -single-candidate run to measure allocations without conflating tracing overhead -with ordinary timing. Run large measurements serially. -Record image/source IDs, architecture, Python and native-client versions, raw -output, bytes per phase, observer delays, proof latency/errors, GC events, -allocation/ownership counts and retained RSS after drain. Separate lazy decode, -reference-count retirement and forced-GC controls; force GC only in the isolated -qualification process. - -The live Stratum gate also has a primed, wallet-free fixture that first submits -three ordinary valid shares, then solves a real block. It uses the pinned qbit -functional helpers and binary. Historical node mocktime keeps ASERT difficulty -above the ordinary-share target. Only this disposable fixture permits the old -template timestamp; coordinator/database clocks and all lease budgets stay -unchanged. - -```sh -QBIT_PRISM_LIVE_POSTGRES=1 \ -QBIT_PRISM_LIVE_NATIVE_POSTGRES=1 \ -QBIT_PRISM_LIVE_PRIMED_WINDOW=1 \ -QBIT_PRISM_LIVE_AUDIT_API=1 \ -bash test/test-prism-stratum-regtest-live.sh -``` - -Run the gate in Docker with access to its Docker daemon, the pinned qbit -binaries and functional helper tree, Rust tools, PostgreSQL client, and psycopg. -Use a unique `QBIT_PRISM_POSTGRES_CONTAINER` when several workspaces run gates. -For abrupt restart qualification, also set `QBIT_PRISM_LIVE_RESTART_REPLAY=1`. -That test entry point verifies the share/outbox commit through an independent -connection, exits before the first node offer, stops every miner, and starts the -normal coordinator. The successor must recover and finalize the exact persisted -hash/digest/share once. No production fault-injection switch is added. - -## Design choices - -```mermaid -flowchart LR - Live[Live immutable candidate view] --> Codec[Stream validation and identity hash] - Codec --> Stage[Stage immutable body chunks] - Stage --> Commit[Atomic fenced share and outbox publication] - Commit --> Headers[Bounded replay headers] - Headers --> Hydrate[Bounded reads into owned spool] - Legacy[Legacy pending JSON] --> Helper[Isolated compatibility helper] - Helper --> Hydrate - Hydrate --> Land[Verify and finalize with lazy audit views] - Land --> Terminal[Fenced terminal state and body detachment] - Terminal --> Reap[Bounded physical cleanup] -``` - - -Small candidate-count pages are useful admission controls, but a byte bound must -also cover each native decoder call. Returning one complete JSON row per -candidate leaves the single-candidate failure mode intact. Moving work to a -Python thread does not isolate a C codec that holds the interpreter lock, and -disabling cyclic GC does not bound reference-count destruction. - -The implementation separates durable candidate identity from its transport and -retained representation. Immutable body chunks and small replay metadata permit -bounded reads and retries without retaining nested share dictionaries. An -additive format also preserves the old pending rows for compatible recovery. -Staging a body must not credit its share: the existing atomic, fenced share/outbox -publication remains the visibility boundary. Cleanup first detaches terminal -work, then reclaims unreachable body data in bounded operations. - -An alternative normalized child table with one SQL row per share would simplify -some range queries, but duplicates the canonical intent model, makes exact legacy -serialization harder to preserve, and still needs a policy for oversized records -and non-share collections. Chunked canonical bytes retain the identity contract -and separate transport sizing from mining rules, at the cost of chunk/index -integrity checks, spool capacity and a compatible-reader rollback floor. - -Pure-Python JSON is useful in the standalone recovery tool, but its cooperative -scheduling does not bound its object graph or retirement cost. Isolated helpers -are appropriate for legacy or oversized materialization that cannot be streamed -through the supported codec. They require explicit deadlines, capacity limits, -shutdown ownership and recoverable resource-pressure behavior. The ordinary path -must not fall back to whole-window decoding in the coordinator when such a -helper or its spool is unavailable. - -Dual-writing the entire old JSON body would ease binary rollback but retain the -original encoding and allocation boundary. A larger lease threshold or suppressed -alert would leave that boundary unchanged and is not part of the fix. - -## Validation contract - -| Area | Required assertion | -| --- | --- | -| Identity | Old and new canonical hashes are byte-identical across ordinary, Unicode, escaped, empty and large windows; pending timestamps normalize identically. | -| Exact retry | Equivalent JSONB numeric/key-order representations retain historical retry behavior; different content is refused even under an injected digest collision. | -| Malformed input | Invalid field types, forbidden JSONB text, malformed encodings, missing chunks, bad offsets and digest/count mismatches fail closed without publishing partial work. Resource pressure remains distinguishable from corrupt data. | -| Atomicity | Upload/seal alone credits no share. Share credit and the outbox reference appear in the same fenced commit. A lost acknowledgment replays exactly once. | -| Fencing | A stale writer/session cannot publish, mutate sealed content or terminalize another session's work. Recheck after hydration/comparison, including takeover races. | -| Replay | Cursor enumeration remains complete across equal timestamps, byte-truncated pages, more candidates than one page, one huge candidate and several huge candidates. Oversized work cannot silently disappear. | -| Cancellation and retries | Interrupt encoding, upload, hydration, helper work, finalization and cleanup; preserve durable pending work, release private resources and allow a fresh retry. | -| Shutdown and ownership | Join workers/helpers, close owned file/index handles and release retired representations before cyclic GC, including exceptions and repeated generations. Do not close a source still owned by legitimate work. | -| Stale cleanup | Preserve active-chain accepted candidates and accepted-parent accounting rules. Detach terminal references under the fence, then reclaim unreachable data without a long writer-lease lock. | -| Mining and payouts | Real Stratum shares and a mined block pass normal verification/accounting, including abrupt process exit before the first node offer; Python/Rust parity and the ordinary lease gates remain green. | - -Performance qualification records peak bytes per codec/transport call, file-index -buffering, native output buffering, admitted spool bytes, helper count and memory, -retained representations after drain, observer scheduling and real lease-proof -behavior separately. Exercise at least 375,000 shares per candidate and both one- -and three-candidate replay, GC enabled/disabled and small-window controls. Stress -many recipients, small audit segments and an oversized individual record as well -as ordinary large windows. A fixed number of candidates or records is not a byte -bound. Allocation tracing belongs in separate runs from ordinary timing. - -## Coordination and production acceptance - -#254 owns daemon sequence retention, matched self-check adoption, fallback -window construction, exception ownership, and their process metrics. Candidate -changes must consume its replayable byte-backed sequence interface and preserve -its payout-generation, balance-check and accepted-parent fences. Review the -combined ownership graph after both branches land; passing the candidate tests -alone does not qualify #254's independent paths. - -Before enabling new storage in production, rehearse schema upgrade, legacy -pending replay, partial staging cleanup, exact retries, stale-session refusal, -shutdown, and rollback against a representative backup on x86_64. Capacity-plan -spool space and helper memory from measured candidate sizes and solve/retirement -rates. A spool reservation is a disk admission bound, not a bound on filesystem -page cache; include helper RSS, container file memory and file-descriptor counts -in the capacity review. Resource pressure must leave valid pending work recoverable. - -Use an additive rollout: deploy and qualify the compatible reader before -enabling chunked writes. Record pending counts, candidate identity hashes, -credited-share state and body integrity before and after the migration rehearsal. -An interrupted upload must expose neither partial candidate work nor credited -shares without their outbox entry. Lost acknowledgments must be safe to retry -under the original exact comparison and writer-session fences. - -Once a chunked outbox body exists, the rollback floor is the compatible reader -with new-format writes disabled. Repinning directly to the investigated -production revision is unsafe: it cannot replay the new representation. A -further rollback requires either draining the pending work or a separately -verified conversion with the writer stopped, followed by identity and durability -checks. Keep the additive body tables during rollback; dual-writing the full -legacy JSON would restore the encoding boundary this change removes. - -Production acceptance requires the #254 minimum **two-hour continuous -observation** in one process epoch after any approved restart. Include at least -two naturally scheduled self-checks, daemon fallback and recovery, live solves, -real pending-candidate replay, accepted-parent accounting, both public Stratum -lanes and a scheduled backup. Extend the observation when a required workload -has not occurred. - -Require no unexplained restart, no new 0.8/1.0 late-wake or exit-guarantee-breach -increments, no hidden durable hard exit, no hash/payout divergence, no lost or -duplicated credit, complete replay, and stable ownership/spool/backlog levels -after drain. Record 0.5 diagnostics, GC distributions and Python/native/Rust -memory separately. Alert silence is not evidence that raw counters remained -unchanged. Any recurrence or stale-session write fails acceptance. - -Deployment and production restarts require the normal operator-approved pin -workflow. This implementation and its Docker experiments do not deploy anything, -change production configuration, or establish a proven production root cause. diff --git a/docs/prism-capacity-readiness.md b/docs/prism-capacity-readiness.md index c1f430d6f..f353ca369 100644 --- a/docs/prism-capacity-readiness.md +++ b/docs/prism-capacity-readiness.md @@ -6,8 +6,9 @@ restart, mainnet readiness, or CI. Compose, `make doctor`, `prism-self-check`, and the coordinator do not consume an artifact or any `PRISM_CAPACITY_*` environment variables. -The bootstrap repository ships the strict -`qbit-prism-capacity-evidence/v2` validator, but it does not ship a production +The bootstrap repository ships the native `qbit-prism-server +capacity-evidence` command for the strict `qbit-prism-capacity-evidence/v3` +format, but it does not ship a production qualification runner. An operator may use the format after capturing the complete miner-facing path: valid Stratum submission, share validation, ACK, and durable Postgres commit. Process health and a schema-only database benchmark @@ -37,17 +38,38 @@ Do not improve benchmark numbers by weakening durability. ## Bound PRISM Configuration -Qualification schema `qbit-prism-capacity-evidence/v2` binds: - -- share difficulty and vardiff enablement -- vardiff minimum, start, maximum, target, retarget, step, EWMA, tolerance, and - idle-sweep values -- share commit batch size, linger, and timeout -- Stratum send timeout - -The validator enforces `minimum <= start <= maximum`. Changing any bound value -requires a new qualification run. The exact `1e-9` local-lab difficulty is -rejected. +Schema `qbit-prism-capacity-evidence/v3` binds only settings the native server +reads at startup, so the artifact names the values the measured binary ran +with. Record each value from the measured coordinator's environment: + +- `PRISM_STRATUM_SHARE_DIFF` and `PRISM_STRATUM_VARDIFF` (`0` or `1`) +- vardiff minimum, start, maximum, target, retarget, step up, step down, EWMA, + and tolerance +- `PRISM_SHARE_COMMIT_TIMEOUT_SECONDS` and `PRISM_STRATUM_SEND_TIMEOUT_SECONDS` +- `PRISM_RUNTIME_WORKERS`: Tokio worker threads, `1..=1024` +- `PRISM_DATABASE_MAX_CONNECTIONS`: the PostgreSQL pool that share commits + draw from, `4..=1024` +- `PRISM_STRATUM_MAX_CONNECTIONS`: accepted miner connections, from `1` to the + platform's Tokio semaphore permit limit +- `PRISM_JOB_BUILD_EXECUTOR_WORKERS`: blocking workers that build job bundles, + `1..=PRISM_RUNTIME_WORKERS + 8` + +The validator requires exactly this set, enforces `minimum <= start <= maximum` +and the native startup ranges above, and rejects the exact `1e-9` local-lab +difficulty. Changing a bound value requires a new qualification run. The set +does not describe frontend/resource topology or prove multi-instance scaling; +retain the complete measured environment and topology beside the image and +revision, database profile, raw miner output, and reconciliation results. + +Version 1 and 2 artifacts are rejected with an error naming the retired schema, +and a pre-rewrite artifact does not qualify the native binary. Any other +`qbit-prism-capacity-evidence/` version is rejected as unsupported. Version 2 +bound Python-runtime knobs the native server never read. Evidence or an +`--expect` flag that names one is rejected with the key named: + +- `PRISM_STRATUM_VARDIFF_IDLE_SWEEP_SECONDS` +- `PRISM_SHARE_COMMIT_BATCH_SIZE` +- `PRISM_SHARE_COMMIT_LINGER_MILLISECONDS` ## Load-Run Contract @@ -85,7 +107,7 @@ by normal standalone validation. The CLI test-only override exists solely so aut tests can validate the example's structure: ```bash -python3 scripts/prism_capacity_evidence.py \ +qbit-prism-server capacity-evidence \ tests/fixtures/prism-capacity-evidence.json \ --allow-example-evidence-for-tests ``` @@ -95,11 +117,11 @@ Never use that override in a deployment command or runtime environment. ## Validate Qualification Evidence Pass the independently configured policy and subject together with every bound -runtime value. The shell names below are local validator inputs, not coordinator -environment variables: +value. The `CAPACITY_*`, subject, and profile shell names are local validator +inputs; each `--expect` value is the measured coordinator's native setting: ```bash -python3 scripts/prism_capacity_evidence.py /path/to/capacity-evidence.json \ +qbit-prism-server capacity-evidence /path/to/capacity-evidence.json \ --forecast-peak-shares-per-second "$CAPACITY_FORECAST_SHARES_PER_SECOND" \ --ack-p99-limit-milliseconds "$CAPACITY_ACK_P99_LIMIT_MILLISECONDS" \ --max-age-seconds "$CAPACITY_EVIDENCE_MAX_AGE_SECONDS" \ @@ -118,11 +140,12 @@ python3 scripts/prism_capacity_evidence.py /path/to/capacity-evidence.json \ --expect PRISM_STRATUM_VARDIFF_MAX_STEP_DOWN="$PRISM_STRATUM_VARDIFF_MAX_STEP_DOWN" \ --expect PRISM_STRATUM_VARDIFF_EWMA_ALPHA="$PRISM_STRATUM_VARDIFF_EWMA_ALPHA" \ --expect PRISM_STRATUM_VARDIFF_RETARGET_TOLERANCE="$PRISM_STRATUM_VARDIFF_RETARGET_TOLERANCE" \ - --expect PRISM_STRATUM_VARDIFF_IDLE_SWEEP_SECONDS="$PRISM_STRATUM_VARDIFF_IDLE_SWEEP_SECONDS" \ - --expect PRISM_SHARE_COMMIT_BATCH_SIZE="$PRISM_SHARE_COMMIT_BATCH_SIZE" \ - --expect PRISM_SHARE_COMMIT_LINGER_MILLISECONDS="$PRISM_SHARE_COMMIT_LINGER_MILLISECONDS" \ --expect PRISM_SHARE_COMMIT_TIMEOUT_SECONDS="$PRISM_SHARE_COMMIT_TIMEOUT_SECONDS" \ - --expect PRISM_STRATUM_SEND_TIMEOUT_SECONDS="$PRISM_STRATUM_SEND_TIMEOUT_SECONDS" + --expect PRISM_STRATUM_SEND_TIMEOUT_SECONDS="$PRISM_STRATUM_SEND_TIMEOUT_SECONDS" \ + --expect PRISM_RUNTIME_WORKERS="$PRISM_RUNTIME_WORKERS" \ + --expect PRISM_DATABASE_MAX_CONNECTIONS="$PRISM_DATABASE_MAX_CONNECTIONS" \ + --expect PRISM_STRATUM_MAX_CONNECTIONS="$PRISM_STRATUM_MAX_CONNECTIONS" \ + --expect PRISM_JOB_BUILD_EXECUTOR_WORKERS="$PRISM_JOB_BUILD_EXECUTOR_WORKERS" ``` No startup or deployment path runs this validator automatically. If an operator @@ -137,480 +160,945 @@ database/hardware profile, forecast, or ACK objective changes. Issue #226 recorded a coordinator whose resident set grew by roughly 390 MB per hour over 45 hours of uptime with no restart, no OOM kill, and a memory map made of hundreds of 4-64 MiB anonymous regions -- the shape of glibc per-thread -malloc arenas across the process's 64 threads. Nothing on `/metrics` could say -whether that growth was live Python objects, pymalloc fragmentation, or glibc -arena fragmentation. The families below are the always-on instrument for that -question. They are cheap (every reading is a counter load, a `len()`, or one -`mallinfo2` call; never a heap walk, never `gc.get_objects()`, never -`tracemalloc`) and fixed-cardinality (no label carries a job id, tip hash, -generation, worker name, or any other value that grows with traffic). They are -not the census, the allocator experiment, or the fix; those are #226's second -part. +malloc arenas across the process's 64 threads. The families this section used +to describe were the always-on instrument for that question on the Python +coordinator. That runtime was removed in #244 and the families went with it; +what the native server exports in their place is the table further down. ### Interpreter and allocator families -Rendered by `lab/prism/metrics.py` from `lab/prism/process_telemetry.py`, in -this order, after the progress-health block: - -| Family | Type | Meaning | -| --- | --- | --- | -| `qbit_prism_process_allocated_blocks` | gauge | `sys.getallocatedblocks()`: memory blocks the CPython allocator currently holds. A live-object proxy: it rises with retention and stays flat under pure allocator fragmentation. | -| `qbit_prism_process_gc_trigger_count{generation}` | gauge | `gc.get_count()`: the collector's per-generation trigger counters, compared against `gc.get_threshold()` to decide when to collect. Generation 0 is allocations minus deallocations. **Not** a count of retained objects. CPython 3.13 made the collector incremental, so the third entry is unused and reads `0`. | -| `qbit_prism_process_gc_collections_total{generation}` | counter | `gc.get_stats()`: collector passes per generation since start. | -| `qbit_prism_process_gc_collected_objects_total{generation}` | counter | Objects the collector freed per generation since start. | -| `qbit_prism_process_gc_uncollectable_objects_total{generation}` | counter | Objects found unreachable but not freeable. Growth here is a leak the collector already knows about. | -| `qbit_prism_process_threads` | gauge | `threading.active_count()`: each live thread is a candidate glibc arena. | -| `qbit_prism_process_malloc_info_available` | gauge | 1 when glibc `mallinfo2` is bound and `PRISM_MALLOC_TELEMETRY` is on; 0 otherwise. | -| `qbit_prism_process_malloc_arena_bytes` | gauge | `mallinfo2.arena`: heap space glibc obtained from the kernel for its arenas, main heap plus every per-thread arena heap, excluding mmapped chunks. | -| `qbit_prism_process_malloc_in_use_bytes` | gauge | `mallinfo2.uordblks`: bytes in allocated chunks across every arena. | -| `qbit_prism_process_malloc_free_bytes` | gauge | `mallinfo2.fordblks`: bytes in free chunks glibc retains rather than returning. | -| `qbit_prism_process_malloc_mmapped_bytes` | gauge | `mallinfo2.hblkhd`: bytes glibc served directly through `mmap`. | - -The `generation` label is the closed set `0`, `1`, `2`. An interpreter that -reports fewer generations renders `-1` for the missing one rather than -shrinking the series set. - -**`mallinfo2` is glibc-only.** The symbol exists in glibc 2.33 and later. It is -absent on musl, on macOS, and on older glibc; the older int-typed `mallinfo` is -deliberately not used as a fallback because its 32-bit fields wrap on a -multi-gigabyte heap. On a platform without it the collector binds once, remembers -the absence, never raises and never logs on a scrape, and renders -`qbit_prism_process_malloc_info_available 0` with all four byte gauges at `-1`. -It never renders zero bytes: a zero arena on a 17 GB process is a lie an operator -would act on. The family set is identical on every platform, so a dashboard built -against one reads the same series on another; only the availability gauge and -the sentinel values differ. - -`PRISM_MALLOC_TELEMETRY` (default `1`, strict boolean, validated at startup by the -coordinator config loader) switches the `mallinfo2` call off. The call walks -glibc's free lists under each arena lock; that is negligible on a healthy heap, -but it is the one reading here that is not a plain counter load, so an operator -can stop it without a redeploy. Off renders exactly like an absent symbol. The -interpreter families have no switch: they are pure counter loads and are always -on. - -The switch is passed through in `compose.yaml` and documented in `.env.example`, -so a restart with a changed `.env` is enough on the compose stack. The mainnet -deployment sets the coordinator's environment from its own configuration rather -than from this file, so turning the switch off there needs the same variable -added on that side. The renderer resolves the switch once, at construction, and -falls back to the shipped default if the value has become invalid since startup --- a telemetry knob must not be able to take the coordinator down from a metrics -thread. +Retired. The `qbit_prism_process_allocated_blocks`, `qbit_prism_process_gc_*`, +`qbit_prism_process_threads` and `qbit_prism_process_malloc_*` families, with +the `PRISM_MALLOC_TELEMETRY` switch, read CPython allocator and collector +state and glibc `mallinfo2` from the Python coordinator's process-telemetry +module. The native server has no interpreter, no cycle collector, and does not +bind `mallinfo2`, so none of those readings has a Rust analogue and none is +planned. `PRISM_MALLOC_TELEMETRY` is not read by the native server and is no +longer in `compose.yaml` or `.env.example`. ### Component-cardinality families -Two families with a closed `component` label set, rendered after the heap -block: +Retired. `qbit_prism_component_entries{component}` and +`qbit_prism_component_bytes{component}` were `len()` and byte readings over +the Python coordinator's in-process structures, with the `component` label set +pinned by two constants in its metrics module. The payout window, candidate +registries and job caches they counted now live in PostgreSQL or in Rust +structures the registry does not yet size. #278's cutover-minimum scope does +not size them either, and defers its per-worker series to P2 (#262); #279 owns +the inventory that decides which component gauges the native server carries. A `qbit_prism_component_*` series +does not appear in a native `/metrics` body. + +### Process telemetry on the native server + +The registry landed by #308 exports one process reading and the state needed +to trust it. Names are the ones in `docs/prism-native-metrics.md`; every family +carries the `qbit_prism_` prefix. | Family | Type | Meaning | | --- | --- | --- | -| `qbit_prism_component_entries{component}` | gauge | Entries retained by each in-process component: one `len()` or `Queue.qsize()` over the owning structure. | -| `qbit_prism_component_bytes{component}` | gauge | Bytes retained by each byte-sized payload: the cached payout window's canonical JSON, the daemon window mirror, and the share-window serialization compact strings. Every value is resident bytes except `share_window_serialization_spool`, which is an on-disk temporary file and is *not* part of RSS. | - -The `component` values are fixed by `PRISM_COMPONENT_ENTRY_KINDS` and -`PRISM_COMPONENT_BYTE_KINDS` in `lab/prism/metrics.py` and pinned by the family -test. They cover the payout window (pages, records, the armed artifact's shares, -the preview and invalidation maps, the append-invalidation stamps, the in-flight -scan anchors), the single-slot share-window serialization cache, job contexts -and the bundle cache, the evicted-job graveyard and its same-tip index, the -candidate lanes and every hash-keyed candidate registry (outstanding, -tip-observed, counted abandonments, preview stamps, ancestor re-drives, terminal -outcomes, disposition flights, waiting retries, dequeued hashes, accounted -blocks), reconcile flights and the trusted-tip memo, the pending-share commit -floor, and the daemon mirror (records in a mirror-backed window and the -compiler's mirror of the daemon's uploaded-window LRU). - -The cached payout window is accounted exactly once, so the byte family can be -summed for attribution. Its two backings are mutually exclusive: on the Python -path the window reports under `payout_window_pages`, `payout_window_records`, -and `payout_window_canonical_json` with the daemon-mirror components at zero; -on the Rust path (`PRISM_WINDOW_PIPELINE_RUST=1`, which is what mainnet runs) -the mirror *is* the cached window, so it reports under -`daemon_window_mirror_records` and `daemon_window_mirror_canonical_items` with -all three `payout_window_*` components at zero. Which pair is non-zero is how -you read the active backing off `/metrics`. `daemon_uploaded_windows` is a -separate structure -- the compiler's LRU of windows already uploaded to the -daemon -- and is never part of that either/or. - -There is no hashrate-rollup gauge because there is no in-process rollup buffer: -the rollup maintenance loop folds shares in PostgreSQL through -`advance_hashrate_rollups` and retains nothing between passes. +| `process_resident_memory_bytes` | gauge | `VmRSS` from `/proc/self/status`, in bytes, refreshed by the process collector every ten seconds. `-1` when the read failed, the platform is unsupported, or the observation is older than 30 seconds. | +| `collector_available{collector="process"}` | gauge | 1 only while the latest completed process collection succeeded and is at most 30 seconds old. Read RSS only when this is 1. | +| `collector_success{collector="process"}` | gauge | Latest attempt succeeded (1), failed (0), or none completed yet (-1). | +| `collector_age_seconds{collector="process"}` | gauge | Monotonic age of the last successful process observation; `-1` before the first success. A failure does not refresh it. | +| `runtime_workers` | gauge | Configured Tokio worker threads (`PRISM_RUNTIME_WORKERS`; empty chooses the available parallelism). | +| `runtime_lag_seconds` | gauge | Latest wake lateness of the 100 ms runtime sampler; `-1` before its first tick. | +| `runtime_poll_lag_seconds{task}` | gauge | Largest active poll, or a completed poll retained for 60 to 61 seconds, per monitored task. | +| `runtime_progress_age_seconds{task}` | gauge | Oldest active explicit operation's time since progress; zero when idle. | +| `runtime_task_stalled{task}` | gauge | 1 while a poll exceeds two seconds or an explicit operation exceeds its budget. | +| `metrics_snapshot_available`, `metrics_snapshot_stale`, `metrics_snapshot_age_seconds` | gauge | The #277 freshness block for the cached body itself. | + +Every gauge here is a scrape-time render of cached observations; the scrape +performs no I/O. Read the `x-prism-metrics-state` header (`fresh`, `stale` or +`unavailable`) before trusting a body, as before. What the native block does +not have, and who owns it: thread and open-file-descriptor gauges were on +#278's original list but fall outside its cutover-minimum scope, which keeps +RSS alone from the process block; allocator arena, +in-use, free and mmapped byte gauges are in no open issue, so a soak that +needs the retention-versus-fragmentation split has to raise it on #279 rather +than expect it. ### What to look at first when RSS climbs -1. **Is it retention or fragmentation?** Plot - `qbit_prism_process_allocated_blocks` and - `qbit_prism_process_malloc_in_use_bytes` against - `qbit_prism_process_resident_memory_bytes`. All three rising together is - retention: something is holding objects. RSS rising while allocated blocks - and in-use bytes stay flat is fragmentation *or* an mmapped retention -- see - step 2 before deciding. -2. **If fragmentation, whose?** `qbit_prism_process_malloc_arena_bytes minus - qbit_prism_process_malloc_in_use_bytes` (equivalently the free-bytes gauge) - growing with `qbit_prism_process_threads` high is glibc arena fragmentation, - the shape #226 captured. Arena flat while RSS rises points at pymalloc or at - the Rust daemon child, which this process's gauges do not see. - - **Check `qbit_prism_process_malloc_mmapped_bytes` before concluding - fragmentation.** glibc serves allocations above its mmap threshold with - `mmap` and records them in `hblkhd`, which this gauge exports — they are - *not* in `uordblks`/`..._in_use_bytes`. So a large retained native buffer - makes both step 1's in-use gauge and this step's arena figure look flat while - RSS climbs, which reads exactly like fragmentation and is in fact retention. - Rising mmapped bytes means step 1's answer was retention after all. -3. **If retention, which component?** Read `qbit_prism_component_entries` and - `qbit_prism_component_bytes` on one panel. The component whose slope matches - RSS is the one to census. A registry keyed by block hash that only ever grows - (terminal outcomes, preview stamps, accounted blocks) is the first suspect - after a candidate storm; the payout window and serialization bytes are the - first suspects after a run of full rescans. -4. **Is the collector keeping up?** A rising - `qbit_prism_process_gc_uncollectable_objects_total` is a reference cycle with - a finalizer the collector cannot break — that is retention the collector - cannot fix on its own, and it is the one GC series that counts objects. - `qbit_prism_process_gc_collections_total` flattening while allocation - continues means collections have stopped happening at all. Do **not** read - `qbit_prism_process_gc_trigger_count` as a backlog of retained objects: it is - the counter CPython compares against its thresholds, it resets on every - collection, and its third entry is unused on the shipped interpreter. - -Every gauge here is a scrape-time snapshot of the same process; read the -`X-Prism-Metrics-State` header before trusting a stale document. - -### Stack-sample frame ownership - -The stall probe retains formatted text only. `sys._current_frames()` includes -the sampler's own frame, so leaving its dictionary in a local variable creates -a cycle that can retain every sampled thread's locals until cyclic GC. This -includes completed payout-window workers and their share data; limiting the -number of printed threads or frames does not bound that retained heap (#247). -`StallProbe.capture_stacks()` releases the snapshot and traversal references in -`finally`, including when formatting fails or a sampling limit is reached. It -does not clear another thread's live execution frame. - -`StallProbeFrameOwnershipTests` in `tests/test_prism_lease_monitor_stalls.py` -checks this with real worker threads and weak references while cyclic GC is -disabled for the test. Payloads must disappear after workers finish, even while -the probe's last sample or a formatting-error traceback remains alive. GC stays -enabled according to the application's existing policy in production. - -Passing that ownership regression does not establish that the remaining #236 -stalls are resolved. After deployment, compare the native-ledger/window/job -pipeline at 350k–400k shares under concurrent work and a sustained soak. Record -coordinator RSS, per-generation GC-pause histogram deltas, late-wake counter -deltas, restart count, and mining-work availability; measure the Rust builder's -memory separately. Lifetime maximum pauses and a stack captured after the -monitor resumes cannot attribute an individual late wake to a collection. +1. **Is the reading real?** `qbit_prism_collector_available{collector="process"}` + must be 1 and the `x-prism-metrics-state` header `fresh`. RSS renders `-1` + when the collector failed or its observation aged out; a drop to `-1` is a + collection failure, not a release of memory. +2. **Is it load or time?** Plot `qbit_prism_process_resident_memory_bytes` + against `qbit_prism_connections`, `qbit_prism_authorized_clients`, the rate + of `qbit_prism_accepted_shares_total`, and + `qbit_prism_block_candidates_pending`. RSS that moves with connections or + the candidate backlog and returns when they do is working set; RSS that + climbs at flat load is the #226 shape and is what the bound below catches. +3. **Is the runtime keeping up?** `qbit_prism_runtime_task_stalled{task}` at 1, + `qbit_prism_runtime_poll_lag_seconds{task}` in whole seconds, or + `qbit_prism_database_pool_acquire_seconds` shifting into its upper buckets + during the climb points at a blocked worker holding buffers rather than at + a leak. The incident-6 lesson is that such a process still answers + `/healthz` from a surviving worker, so the runtime gauges are the tell. +4. **Retention or fragmentation?** The native registry cannot say. The Python + lane split RSS into live objects, allocator in-use bytes, free bytes and + mmapped bytes; the native server exports RSS alone. The binary uses the + system allocator, glibc malloc in the shipped image, so arena mechanics + still apply to its worker and blocking-pool threads, but nothing on this + branch measures them. Record the RSS series and the correlated series + above and attach them to the finding; do not infer an allocator setting + from RSS alone. ## Heap Census, Allocator Control, and the Resident-Set Bound -Issue #226's second part. The always-on families above say *whether* the -resident set is growing through retention or through fragmentation; this -section is the instrument that says *what* is retained and *which allocator -setting* bounds the fragmentation, and the automated check that a fix has to -pass. Everything here is off by default, operator-triggered, bounded, and -unreachable from any request path: no HTTP route, no Stratum message, and no -`/metrics` scrape can start a heap walk or a trim. `tests/test_prism_heap_census.py` -pins each of those properties. +Issue #226's second part. On the Python coordinator this section was the +instrument that said *what* was retained (a `gc.get_objects()` census on +`SIGUSR1`) and *which allocator setting* bounded the fragmentation +(`malloc_trim` on `SIGRTMIN+1`, the `MALLOC_ARENA_MAX` experiment), plus the +automated bound a fix had to pass. The census and the allocator controls were +CPython and glibc instruments wired into the Python process and left with it +in #244. The bound and the soak that produces its input are +runtime-independent and are re-anchored below. ### Running a census -The census is `lab/prism/process_telemetry.py`'s `HeapCensus`. It is armed only -when the coordinator starts with `PRISM_HEAP_CENSUS=1`; without that the signal -is not registered, no worker thread exists, and the census entry point is inert -even if called. Armed, it responds to `SIGUSR1`: - -```sh -# Compose stack -docker kill --signal=SIGUSR1 -# Bare process -kill -USR1 -``` - -The signal handler writes one byte to a pipe and returns; a dedicated thread -(`prism-heap-census`) does the walk. That is the shape `SIGUSR2`'s -`faulthandler` dump follows, with the work moved off the handler so a second -signal cannot re-enter a lock the handler holds. Signals that arrive within -`PRISM_HEAP_CENSUS_MIN_INTERVAL_SECONDS` (default `60`) of the last census are -logged as suppressed, so a repeated `kill` cannot stack walks. - -What the walk costs, and why it is never automatic: it is one -`gc.get_objects()` call followed by one pass over every tracked object taking -its type and `sys.getsizeof()`. The `gc.get_objects()` call allocates one -pointer per tracked object and holds the GIL until it returns; nothing can -interrupt it, and on a heap of tens of millions of objects it is a stall of -seconds on every thread. The pass that follows is what the walk cap bounds. -Take a census in a low-load window, and expect a share-ack latency blip in -`qbit_prism_share_ack_seconds` when you do. - -The switches and bounds, all validated at startup (a value outside its range -refuses startup rather than promising a bound it does not keep). Each passes -through `compose.yaml` to the `prism-coordinator` service and is listed in -`.env.example`, so a value in the deploy dotenv reaches the process; compose -has no `env_file:` for the service, and a variable without a passthrough line -would silently never arrive: - -| Variable | Default | Meaning | -| --- | --- | --- | -| `PRISM_HEAP_CENSUS` | `0` | Arm `SIGUSR1`. Strict boolean. | -| `PRISM_HEAP_CENSUS_DIR` | `$PRISM_AUDIT_DIR/heap-census` | Where reports are written. Created on first census. | -| `PRISM_HEAP_CENSUS_TOP_N` | `40` (max `200`) | Entries in each of the four lists: types by count, types by shallow bytes, `tracemalloc` sites by line, sites by file. | -| `PRISM_HEAP_CENSUS_MAX_BYTES` | `262144` (`4096` to `4194304`) | Output ceiling per report. Met by halving top-N until the JSON fits, never by cutting the document. | -| `PRISM_HEAP_CENSUS_MAX_SECONDS` | `10` (max `120`) | Walk cap for the per-object pass; the report says when it was hit. | -| `PRISM_HEAP_CENSUS_MIN_INTERVAL_SECONDS` | `60` | Signals closer together than this coalesce. | -| `PRISM_HEAP_CENSUS_TRACEMALLOC` | `0` | Start `tracemalloc` (one frame) at startup so the census can name allocation sites. Requires `PRISM_HEAP_CENSUS=1`. | - -`tracemalloc` is the second, separate opt-in because its cost is standing, not -per census: every allocation pays for the trace for as long as it runs, and the -trace itself is resident memory proportional to live blocks. Use it for a -diagnosis window, not steady state. Without it the census still names the -retained *types*; with it the census also names the *sites*. - -Reports land as `heap-census--.json`, written to a temporary -name and renamed into place. At most 32 are kept per directory (oldest removed -first), so with the default byte ceiling the census can occupy at most 8 MiB -of the audit volume. +Retired. The heap census, its `SIGUSR1` arming and the `PRISM_HEAP_CENSUS*` +settings applied to the retired Python coordinator (#244). The native server +registers no census signal and reads none of those variables, `compose.yaml` +and `.env.example` no longer pass them, and #288's acceptance includes a CI +check that fails if one reappears in `docs/` as live. The native server +installs handlers for `SIGTERM` and `SIGINT` only. In the shipped container it +is PID 1 with no init, so the kernel drops `SIGUSR1` and the signal does +nothing; run as a bare process or under an init, the default action +terminates it. Do not send it. ### Reading a census -A report is one JSON document: - -- `process`: the same readings `/metrics` exports (`resident_memory_bytes`, - `allocated_blocks`, `threads`, the GC counters, the `mallinfo2` bytes) plus - what the scrape cannot afford: `malloc_arena_count` and the split of glibc's - free bytes into `malloc_free_bin_bytes` (interior free chunks) and - `malloc_free_top_bytes` (the free space at the end of each arena's heaps), - and `anonymous_map`, the private-anonymous region histogram in the same - 4-64 MiB bands the mainnet capture was reported in. -- `walk`: `tracked_objects`, `objects_walked`, `distinct_types`, `truncated`, - and the seconds spent in `gc.get_objects()` and in the whole pass. -- `types_by_count` and `types_by_bytes`: top-N `{type, count, shallow_bytes}`. - Shallow bytes are `sys.getsizeof()`; a `dict` of 60k share records is one - large dict plus 60k small dicts plus their strings, so read the two lists - together. -- `tracemalloc`: `{"tracing": false}` or the top-N sites by line and by file - with bytes and block counts, plus the traced total and peak. - -Read it in this order: - -1. **Retention or fragmentation?** Compare `process.resident_memory_bytes` - with `process.malloc_in_use_bytes` and `process.allocated_blocks` against - an earlier census (or against the `/metrics` history). In-use bytes and - allocated blocks rising with RSS is retention; RSS rising over a flat - in-use is fragmentation. On the mainnet capture the map alone said - fragmentation, but only a census on that process can prove there is no - retention underneath it. -2. **If retention, of what?** `types_by_count` names the type; the first - census after startup is the baseline and the type whose count grows across - censuses is the leak. With tracing on, `tracemalloc.by_line` names the - allocation site. A site inside the payout-window materialization is an - issue #228 finding; a registry keyed by block hash is a candidate-lane - finding. -3. **If fragmentation, can a trim reach it?** `malloc_free_bin_bytes` is what - `malloc_trim` returns to the kernel, in every arena. `malloc_free_top_bytes` - across `malloc_arena_count` arenas is what it cannot: glibc trims a - per-thread heap's top only when the owning thread frees a chunk of 64 KiB - or more while the top exceeds `M_TRIM_THRESHOLD`, and that threshold - floats up to 64 MiB as large chunks are freed. Free bytes parked in the - tops of many arenas are the case `MALLOC_ARENA_MAX` fixes and a trim does - not; free bytes in bins are the case a trim fixes. -4. **Was the walk complete?** `walk.truncated` true means the type lists are - over a prefix of the heap. Raise `PRISM_HEAP_CENSUS_MAX_SECONDS` and take - another census in a quieter window rather than reading a partial census - as a whole one. +Retired with the census. The report format (`process`, `walk`, +`types_by_count`, `types_by_bytes`, `tracemalloc`) described CPython heap +state and has no native producer. ### `malloc_trim` -`MallocTrimmer` calls glibc `malloc_trim(0)` and reports the readings around -it through the same collector `/metrics` uses: RSS, arena, in-use and free -bytes before and after, the deltas, glibc's own released flag, and the call's -duration, on one log line (`prism malloc_trim ...`). A trim that works shows -`in_use_delta=0` (it frees nothing the program holds), a negative -`resident_delta`, and `released=True`. +The signal-driven trimmer, `PRISM_MALLOC_TRIM_SIGNAL` and +`PRISM_MALLOC_TRIM_INTERVAL_SECONDS` were a Python-process instrument (#244) and are retired. +The native server exposes no trim signal, and `SIGRTMIN+1` is unhandled there, +so the same warning as for `SIGUSR1` applies. -Two ways to run it, both off by default: - -| Variable | Default | Meaning | -| --- | --- | --- | -| `PRISM_MALLOC_TRIM_SIGNAL` | `0` | Arm `SIGRTMIN+1` (`docker kill --signal=RTMIN+1`, `kill -RTMIN+1`). Strict boolean. | -| `PRISM_MALLOC_TRIM_INTERVAL_SECONDS` | `0` (off; minimum `60`) | Periodic trim on the census thread, paced by this interval. | - -Both go through the same worker thread and the same -`PRISM_HEAP_CENSUS_MIN_INTERVAL_SECONDS` gate, so signals cannot stack trims -either. The design call is to offer both: the signal is the instrument (does -RSS return when asked?), and the paced periodic action is the candidate fix -for interior fragmentation once the soak shows it returns memory. The floor -exists because of what the call does: `malloc_trim` takes every arena's lock -in turn while it consolidates and walks that arena's free lists, and a thread -allocating in that arena at that moment waits. On a fragmented multi-gigabyte -heap that is a stall of tens to hundreds of milliseconds across whichever of -the 64 threads happen to allocate, not a wedge; it is not something to run -every second on a latency-sensitive process. Run the signal form in a -low-load window first and read the logged duration before choosing an -interval. - -What it cannot do: shrink the tops of per-thread heaps (see "Reading a -census"). With many arenas, `malloc_trim` consolidates each arena's free -chunks *into* its top and then cannot return that top, so RSS does not move; -with `MALLOC_ARENA_MAX=2` the same free chunks stay interleaved with other -threads' live chunks in two arenas, and the trim returns them. That is -measured, not inferred: the storm instrument below shows it on any glibc host. +Instead the native frontend that lands a block trims once, after the landing +(#600): glibc keeps the heap a landing's window rebuild frees, so without it +a 400k-window frontend's resident set climbs with each landing to a plateau of +about 3.3 GB. `PRISM_LANDING_MALLOC_TRIM_ENABLED` (default `1`) controls it; +[landing memory per frontend](prism-storage-sizing.md#landing-memory-per-frontend) +has the measurements and the memory limit to set. ### Allocator settings and the storm instrument -The image sets `MALLOC_ARENA_MAX=2`; `docs/mainnet-deployment.md` records the -decision, the rationale, and how to override it per deployment. The -experiment behind it is `tests/prism_candidate_storm.py` with three additions: - -- `--heap-report` reads the resident set, `allocated_blocks`, thread count, - the `mallinfo2` bytes, the arena count, the free-bytes split (tops versus - bins), and the anonymous-map histogram at every phase boundary of the - storm, then after the storm's objects are released and collected, then - after one `malloc_trim`. The release-and-collect reading is #185's - post-storm drain measurement, re-runnable on any host. -- `--arena-probe-threads N` (with `--arena-probe-rounds`) runs `N` threads that - each allocate medium buffers (4-120 KiB: over pymalloc's 512-byte ceiling, - under glibc's initial 128 KiB `mmap` threshold) and meet at a barrier before - exiting, which is how glibc gives each of them an arena. It reproduces the - coordinator's long-lived-thread shape; the storm rig itself is single- - threaded. -- `--allocator-experiment default,2,4` runs the instrument once per - `MALLOC_ARENA_MAX` setting in a child process (glibc reads the variable once - at start) and prints every child's heap section side by side; - `--allocator-mmap-threshold BYTES` pins `MALLOC_MMAP_THRESHOLD_` in the - children as an extra axis. - -```sh -python3 tests/prism_candidate_storm.py --decide --drain-per-row \ - --arena-probe-threads 64 --allocator-experiment default,2,4 > arena-experiment.json -``` - -Read the mechanism, not the absolutes: memory absolutes vary by up to an order -of magnitude across hosts, and the instrument prints `python`, `libc`, -`machine` and `cpu_count` beside every number for that reason. What should -transfer: `malloc_arena_count` after the probe reaches the thread count at -the default and stops at the cap otherwise; `anonymous_regions_4mib_to_64mib` -climbs into the mainnet shape at the default; `allocated_blocks` at -`after_release_gc` returns to `start` (no Python retention from the storm); -and `malloc_trim` returns memory at the cap and not at the default. The drain's -`wall_seconds` and the `phase_seconds` are the latency comparison across -settings. +Retired. The image no longer sets `MALLOC_ARENA_MAX`, `compose.yaml` and +`.env.example` carry no allocator variable, and the candidate-storm rig under +`tests/` that produced the arena experiment was deleted with the Python lane +(#244). The `MALLOC_ARENA_MAX=1` setting in +`docs/prism-payout-artifact-measurement.md` applies to that test process on +the operator's host, not to the coordinator image. The native allocator +experiment is #600's: its weekly soak's resident-set failure was glibc keeping +the heap each block landing frees, answered by the post-landing trim above; +`MALLOC_ARENA_MAX=2` only halved that growth and slowed a full 400k refresh +from 7.5 s to 13 s. ### The resident-set bound -The stated bound: **after a one-hour warm-up, the resident set must stay within -2.0x the warm-up peak for the rest of a 24 h soak at ordinary load.** The -warm-up peak is the baseline because the first hour materializes the payout -window and runs the first full rescan; the #226 slope (390 MB/h from a 125-145 -MiB start) breaches this bound in its third hour, and a post-storm excursion -like #185's 613 MiB drains back under it. The check is automated, not a graph -someone reads: +The stated bound is unchanged: **after a one-hour warm-up, the resident set +must stay within 2.0x the warm-up peak for the rest of a 24 h soak at ordinary +load.** The warm-up peak is the baseline because the first hour materializes +the payout window and runs the first full rescan; the #226 slope (390 MB/h from +a 125-145 MiB start) breaches this bound in its third hour, and a post-storm +excursion like #185's 613 MiB drains back under it. The check is automated, +not a graph someone reads. The Python tool that computed the verdict left with +#244; the same verdict is one `awk` pass over the sample file: ```sh -python3 -m lab.prism.process_telemetry rss-bound --samples soak-rss.csv \ - --warmup-seconds 3600 --multiple 2.0 --min-span-seconds 82800 +sort -t, -k1,1n soak-rss.csv | awk -F, -v warmup=3600 -v multiple=2.0 -v min_span=82800 -v max_gap=360 ' + function numeric(s) { return s ~ /^[ \t]*-?([0-9]+\.?[0-9]*|\.[0-9]+)[ \t]*$/ } + /^[ \t]*(#|$)/ { next } + NF != 2 || !numeric($1) || !numeric($2) { bad = $0; unusable = 1; exit } + $2 < 0 { next } + prev != "" && $1 - prev > max_gap { gap = $1 - prev; gap_at = $1; exit } + { prev = $1 + if (t0 == "") t0 = $1 + if ($1 - t0 <= warmup) { if ($2 > base) base = $2; next } + post++; last = $1 + if ($2 > peak) { peak = $2; peak_at = $1 } + if (!breach && $2 > base * multiple) breach = $1 } + END { + if (unusable) { print "unusable input: row \"" bad "\" is not seconds,rss_bytes"; exit 2 } + if (gap) { printf "unusable input: %d s between the samples at %d and %d, soak samples every 300 s\n", gap, prev, gap_at; exit 2 } + if (base == "" || !post) { print "unusable input: no warm-up or no post-warm-up samples"; exit 2 } + if (last - t0 < min_span) { printf "unusable input: series spans %d s, soak needs %d s\n", last - t0, min_span; exit 2 } + printf "baseline=%d bound=%d peak=%d peak_at=%d first_breach_at=%s\n", base, base * multiple, peak, peak_at, breach ? breach : "none" + exit breach ? 1 : 0 }' ``` The input is one `seconds,rss_bytes` line per sample (absolute epoch seconds -are fine; `#` comments are skipped; `-1` samples are ignored). The command -prints the verdict as JSON (baseline, bound, peak and its time, the first -breach time, and the least-squares slope over the post-warm-up samples) and -exits `0` on pass, `1` on fail, `2` on unusable input. `--max-slope-bytes-per-hour` -adds a slope guard for a leak slow enough to stay under the multiple inside 24 -hours; `--min-span-seconds` refuses a series shorter than the soak. +are fine). `#` comments and blank lines are skipped and `-1` samples are +ignored; any other row that is not exactly two numeric fields is unusable +input (exit `2`), so a truncated row during an excursion cannot pass as a +skipped one, as it could not in the Python tool. The field count is exact, +which is stricter than the Python tool: it read the first two fields and +ignored the rest, so `82800,300,000` judged as 300 bytes and `0,100,garbage` +set the baseline. The capture loop below writes exactly two fields, and a row +with a third was corrupted somewhere between the loop and the judge, so it is +refused rather than read. The samples are sorted by timestamp before they +are judged, as the Python tool sorted them, so concatenated partial logs of +one run judge the same as a single file; comment and blank lines are still +skipped wherever they sort, and a malformed row still exits `2`. The same sort +is why two runs must never share a file, which step 3 below prevents with a +directory per run. The warm-up runs from the earliest sample and includes a +sample taken exactly one hour after it, as it did in the Python tool; only +later samples are judged against the bound. The command prints the baseline, +the bound, the post-warm-up peak and its time, and the first breach time, and +exits `0` on pass, `1` on fail, `2` on unusable input. The span floor is the +Python tool's default: `min_span=82800` refuses a series that spans less than +23 hours from its first sample to its last, not one shorter than the soak. The +tool's source recorded the hour of slack as tolerance for a late first sample; +the run itself is still the 24 h that step 2 below asks for, and step 3's loop +ends the run only when its samples span that, by writing a marker that step 6 +requires before this check is run, so the hour of slack does not pass a run +that was cut short in its last hour, however it was cut. The span floor +sees only the first and last samples, so it cannot see a hole between them: +two samples 23 hours apart satisfy it, and so does a series with an hour +missing after the warm-up. `max_gap=360` refuses a series in which two +consecutive samples are more than 360 seconds apart, as unusable input (exit +`2`) naming the interval and both timestamps, because an excursion that rose +and drained inside such a hole is not in the file, and a verdict over the +rest would be a pass over evidence the run never took. The tolerance is the +five-minute cadence plus a minute of slack for the three reads an iteration +of the capture loop makes before it sleeps, of which only the `curl` is +bounded; a series taken in cadence has no interval near it. A `-1` row is +skipped before the interval is measured, as it is skipped everywhere else, so +a hole bridged by `-1` rows every five minutes is still a hole: those rows +are samples without a value and say nothing about what RSS did between the +usable ones. By the same rule a single `-1` row between two five-minute +samples opens a 600 s gap; the capture loop below never writes one, it stops +the run instead, so a file refused on that account was not the loop's. The +Python tool had no gap rule, so this is a tightening over it of the same +kind as the exact field count: its verdict passed a series with an +hour-long hole after the warm-up, and a faithful port would have too. The +slope guard the Python tool offered (a leak slow enough to stay under the +multiple inside 24 hours) has no replacement in the runbook; take it from +the RSS series on the deployment's dashboard, whose rules #279 owns. When it fails: the first breach time says whether the growth is the steady -slope (breach hours in) or an excursion (breach right after a storm or a -rescan burst). Take a census at the breach and read it as above; if the -census names a retained type, that type's owner is the fix; if it names free -bytes parked in arena tops, the arena cap is the fix and the soak is rerun at -the other setting. Do not raise the multiple to make a soak pass. +slope (breach hours in) or an excursion (breach right after a candidate storm +or a rescan burst). There is no census to take at the breach; the evidence is +the RSS series, the correlated series from "What to look at first when RSS +climbs" at the breach time, and the full `/metrics` body captured there. Do +not raise the multiple to make a soak pass. ### Testnet 24 h soak runbook (deferred to the operator) -This soak has not been run. Nothing here was verified against a live host; -the instrument, the thresholds and the procedure are what this package -delivers, and the numbers are what the soak produces. - -1. **Build** the release image (which now carries `MALLOC_ARENA_MAX=2`) and - record its digest. -2. **Run three soaks in sequence**, each a fresh coordinator process for at - least 24 h at ordinary testnet load with the same miner population, with - these values in the deploy dotenv (each passes through `compose.yaml` to - the `prism-coordinator` service; a deployment that sets the coordinator's - environment another way sets them there): - - `MALLOC_ARENA_MAX=2` (the image default), - - `MALLOC_ARENA_MAX=4`, - - `MALLOC_ARENA_MAX=2` with `PRISM_MALLOC_TRIM_INTERVAL_SECONDS=900`. - - Set `PRISM_HEAP_CENSUS=1` and `PRISM_MALLOC_TRIM_SIGNAL=1` on all three; - leave `PRISM_HEAP_CENSUS_TRACEMALLOC` off unless a census names growth - without naming a site. Before trusting a run, confirm the setting reached - the process and the census is armed: +This soak has not been run against the native server. Nothing here was +verified against a live host; the thresholds and the procedure are what this +document delivers, and the numbers are what the soak produces. It is the +memory-bound evidence for the promotion decision and is distinct from #291's +two-hour cutover soak, which reads its own criteria from the same registry. + +1. **Build** the coordinator image the deployment will run and record its + image ID. Substitute the deployment's env files for the repository + examples once, in the function; the same Compose invocation then builds + the image and resolves its name, so a `PRISM_COORDINATOR_IMAGE` set only + in an env file is the image inspected, not the default. The name is read + from the resolved service record because `config --images` also lists the + images of the services the coordinator depends on; the read needs + `python3` on the operator host: + + ```sh + compose() { + docker compose --env-file config/upstream.env.example --env-file .env.example \ + -f compose.yaml --profile prism "$@" + } + compose build prism-coordinator + image=$(compose config --format json \ + | python3 -c 'import json, sys; print(json.load(sys.stdin)["services"]["prism-coordinator"]["image"])') + docker image inspect --format '{{.Id}}' "$image" + ``` + +2. **Run** one fresh coordinator process for at least 24 h at ordinary testnet + load with the same miner population throughout. The three + `MALLOC_ARENA_MAX` runs the Python lane called for are retired with the + allocator experiment above; there is one configuration to soak, the one + the deployment ships. Before trusting a run, confirm the process collector + is publishing and the body is fresh. The audit port is bound to the + container's loopback and compose publishes only the Stratum port, so the + read runs inside the container, where the server is PID 1 and the image + carries `curl` (`3341` is the default `PRISM_AUDIT_PORT`; substitute the + deployment's value): ```sh c= - docker exec "$c" sh -c 'echo MALLOC_ARENA_MAX=$MALLOC_ARENA_MAX PRISM_HEAP_CENSUS=$PRISM_HEAP_CENSUS' - docker logs "$c" 2>&1 | grep 'prism heap census armed' + docker exec "$c" curl -sS --max-time 5 -D - http://127.0.0.1:3341/metrics \ + | tr -d '\r' \ + | awk ' + $0 == "x-prism-metrics-state: fresh" { fresh = 1 } + $0 == "qbit_prism_collector_available{collector=\"process\"} 1" { up = 1 } + END { + if (fresh && up) { print "ready"; exit 0 } + if (!fresh) why = "x-prism-metrics-state is not fresh" + if (!up) why = why (why ? ", " : "") "process collector gauge is not 1" + print "not ready: " why; exit 1 }' ``` - Record the exact environment beside each run. -3. **Capture every 5 minutes** for the whole soak. The audit port is bound to - the container's loopback and compose does not publish it, so both reads - run inside the container, where the coordinator is PID 1 and the `lab` - package is on the path (`3341` is the default `PRISM_AUDIT_PORT`; - substitute the deployment's value): + Both lines are required, `x-prism-metrics-state: fresh` and the process + collector gauge at 1: a `stale` or `unavailable` body is a cached render + whose process gauge can still read 1, so the gauge alone proves nothing. + Do not start the run, and do not trust its RSS series, until the check + prints `ready` and exits `0`. Record the deploy dotenv (secrets redacted) + and the image ID beside each run. +3. **Capture every 5 minutes** for the whole soak: RSS from `/proc/1/status` + for the bound, and the correlated series for the reading order above. Both + reads run inside the container; the parsing runs on the host. The fence + defines `capture`, which takes the samples in a loop; the end of step 4 + calls it, once the two snapshot functions it also calls are defined. + Before every sample it checks that the clock has not moved backward since + the previous sample ended, that no more than 360 s have passed since + the previous one started and that it is still reading the process the run + started with, after the sample's reads it reads the identity again and + then the clock, which must not precede this sample's start or be more than + 360 s past the time the previous sample's reads ended, and it stops the run + as invalid when any of these checks fails or one of its appends to the run's files + does. The first samples taken 3,600 s, 43,200 s and 86,400 s or more after + the first also take that hour's snapshots, below and in step 4, between the + sample's appends and its second identity read, and a snapshot that fails + stops the run the same way. It ends the run itself, as complete, after the + first sample taken 86,400 s or more after the first, once that sample has + passed every check, by writing `soak-complete` and returning `0`. + + Set `expected_authorized_clients` to the positive number of authorized + clients in the ordinary-load population. Every sample must match it and, + after the first establishes a baseline, show at least one new accepted + share ACK since the previous sample. Keep ordinary load producing accepted + shares in each five-minute interval; a cumulative histogram left by earlier + traffic or rejected-only traffic cannot qualify an idle run. The count + checks population size, so the operator must also keep the individual + miners unchanged. ```sh c= - while true; do - docker exec "$c" python3 -m lab.prism.process_telemetry rss-sample --pid 1 >> soak-rss.csv - docker exec "$c" python3 -c 'import sys, urllib.request; sys.stdout.write(urllib.request.urlopen("http://127.0.0.1:3341/metrics", timeout=5).read().decode())' \ - | grep -E '^qbit_prism_process_(resident_memory_bytes|allocated_blocks|threads|malloc_(arena|in_use|free)_bytes) ' \ - | sed "s/^/$(date +%s) /" >> soak-metrics.log - sleep 300 - done + expected_authorized_clients= + run=soak-$(date -u +%Y%m%dT%H%M%SZ) + process() { + docker inspect --format '{{.State.Status}} {{.State.StartedAt}} {{.RestartCount}}' "$c" + } + invalid() { + tee -a "$run/soak-invalid" >&2 + } + capture() { + prev= + prev_end= + start= + prev_accepted= + snapped=0 + mkdir "$run" || return + if ! awk -v expected="$expected_authorized_clients" 'BEGIN { exit !(expected ~ /^[0-9]+$/ && expected + 0 > 0) }'; then + echo "$(date -u +%FT%TZ): soak invalid, expected_authorized_clients must be a positive integer" | invalid + return 1 + fi + if ! first=$(process) || [ -z "$first" ]; then + echo "$(date -u +%FT%TZ): soak invalid, could not read the coordinator process identity at the start of the run" | invalid + return 1 + fi + while true; do + now=$(date +%s) + if [ -n "$prev_end" ] && [ "$now" -lt "$prev_end" ]; then + echo "$(date -u +%FT%TZ): soak invalid, the clock moved backward from the previous sample's end at $prev_end to $now" | invalid + return 1 + fi + if [ -n "$prev" ] && [ $((now - prev)) -gt 360 ]; then + echo "$(date -u +%FT%TZ): soak invalid, no sample for $((now - prev)) s at $now, the last one was at $prev" | invalid + return 1 + fi + prev=$now + if [ -z "$start" ]; then start=$now; fi + if ! current=$(process) || [ -z "$current" ]; then + echo "$(date -u +%FT%TZ): soak invalid, could not read the coordinator process identity before the sample at $now" | invalid + return 1 + fi + if ! echo "$now $current" >> "$run/soak-process.log"; then + echo "$(date -u +%FT%TZ): soak invalid, could not append the reading at $now to $run/soak-process.log" | invalid + return 1 + fi + if [ "$current" != "$first" ]; then + { + echo "$(date -u +%FT%TZ): soak invalid, the coordinator is not the process the run started with" + echo " at start: $first" + echo " now: $current" + } | invalid + return 1 + fi + rc=0 + body=$(docker exec "$c" cat /proc/1/status) || rc=$? + rss=$(printf '%s\n' "$body" | awk -v now="$now" ' + /^VmRSS:/ && $2 ~ /^[0-9]+$/ && $3 == "kB" { n++; row = now "," $2 * 1024 } + END { if (n == 1) print row }') + case $rc:$rss in + 0:"$now",[0-9]*) + echo "$rss" >> "$run/soak-rss.csv" || { + echo "$(date -u +%FT%TZ): soak invalid, could not append the sample at $now to $run/soak-rss.csv" | invalid + return 1 + } ;; + *) + { + echo "$(date -u +%FT%TZ): soak invalid, no RSS sample at $now" + if [ "$rc" -ne 0 ]; then echo " docker exec exited $rc"; fi + echo " read from /proc/1/status:" + printf '%s\n' "$body" | sed 's/^/ /' + } | invalid + return 1 ;; + esac + rc=0 + metrics=$(docker exec "$c" curl -sS --max-time 5 -D - http://127.0.0.1:3341/metrics) || rc=$? + metrics=$(printf '%s\n' "$metrics" | tr -d '\r') + why=$(printf '%s\n' "$metrics" | awk -v rc="$rc" -v previous="$prev_accepted" -v expected="$expected_authorized_clients" ' + function canonical(n) { sub(/^0+/, "", n); return n == "" ? "0" : n } + function increased(n, p) { + n = canonical(n); p = canonical(p) + return length(n) > length(p) || (length(n) == length(p) && ("n" n) > ("n" p)) + } + /^x-prism-metrics-state:/ { state = $0 } + $0 == "x-prism-metrics-state: fresh" { fresh = 1 } + /^qbit_prism_collector_available\{collector="process"\} / { up = $0 } + $0 == "qbit_prism_collector_available{collector=\"process\"} 1" { up_ok = 1 } + /^qbit_prism_process_resident_memory_bytes[ {]/ { rss = $0; if ($NF ~ /^[0-9]+$/) rss_ok = 1 } + $1 == "qbit_prism_share_ack_seconds_count{result=\"accepted\"}" { + accepted_n++; accepted = $2; accepted_ok = NF == 2 && $2 ~ /^[0-9]+$/ + } + $1 == "qbit_prism_authorized_clients" { + clients_n++; clients = $2; clients_ok = NF == 2 && $2 ~ /^[0-9]+$/ + } + END { + if (rc != 0) print "docker exec exited " rc + else if (!fresh) print (state ? "state header read \"" state "\"" : "no x-prism-metrics-state header") + else if (!up_ok) print (up ? "process collector gauge read \"" up "\"" : "no qbit_prism_collector_available{collector=\"process\"} sample") + else if (!rss_ok) print (rss ? "RSS gauge read \"" rss "\"" : "no qbit_prism_process_resident_memory_bytes sample") + else if (accepted_n != 1 || !accepted_ok) print "qbit_prism_share_ack_seconds_count{result=\"accepted\"} must have one nonnegative integer sample" + else if (previous != "" && !increased(accepted, previous)) print "accepted share ACK count did not increase from " previous " to " accepted + else if (clients_n != 1 || !clients_ok) print "qbit_prism_authorized_clients must have one nonnegative integer sample" + else if (canonical(clients) != canonical(expected)) print "authorized client count " clients " does not match expected " expected }') + if [ -n "$why" ]; then + { + echo "$(date -u +%FT%TZ): soak invalid, no metrics sample at $now" + echo " $why" + echo " headers read from /metrics:" + printf '%s\n' "$metrics" | awk 'NF == 0 { exit } { print " " $0 }' + } | invalid + return 1 + fi + prev_accepted=$(printf '%s\n' "$metrics" | awk '$1 == "qbit_prism_share_ack_seconds_count{result=\"accepted\"}" { print $2 }') + lines=$(printf '%s\n' "$metrics" \ + | grep -E '^(x-prism-metrics-state:|qbit_prism_(share_ack_seconds(_bucket|_sum|_count)|process_resident_memory_bytes|collector_available|collector_age_seconds|runtime_lag_seconds|runtime_task_stalled|runtime_poll_lag_seconds|database_pool_acquire_seconds(_bucket|_sum|_count)|connections|authorized_clients|accepted_shares_total|block_candidates_pending)[ {])' \ + | sed "s/^/$now /") + printf '%s\n' "$lines" >> "$run/soak-metrics.log" || { + echo "$(date -u +%FT%TZ): soak invalid, could not append the sample at $now to $run/soak-metrics.log" | invalid + return 1 + } + for hour in 1 12 24; do + if [ "$hour" -gt "$snapped" ] && [ $((now - start)) -ge $((hour * 3600)) ]; then + snapped=$hour + name=h$(printf '%02d' "$hour").txt + if ! why=$(share_ack_snapshot "$run/share-ack-$name" 2>&1); then + { + echo "$(date -u +%FT%TZ): soak invalid, no hour-$hour share-ack snapshot at $now" + printf '%s\n' "$why" | sed 's/^/ /' + } | invalid + return 1 + fi + if [ "$hour" -ne 12 ] && ! why=$(full_metrics_snapshot "$run/metrics-$name" 2>&1); then + { + echo "$(date -u +%FT%TZ): soak invalid, no hour-$hour full metrics snapshot at $now" + printf '%s\n' "$why" | sed 's/^/ /' + } | invalid + return 1 + fi + fi + done + if ! after=$(process) || [ -z "$after" ]; then + echo "$(date -u +%FT%TZ): soak invalid, could not read the coordinator process identity after the sample at $now" | invalid + return 1 + fi + if [ "$after" != "$first" ]; then + { + echo "$(date -u +%FT%TZ): soak invalid, the coordinator changed while the sample at $now was read" + echo " at start: $first" + echo " after read: $after" + } | invalid + return 1 + fi + end=$(date +%s) + if [ "$end" -lt "$now" ]; then + echo "$(date -u +%FT%TZ): soak invalid, the clock moved backward during the sample from $now to $end" | invalid + return 1 + fi + if [ -n "$prev_end" ] && [ $((end - prev_end)) -gt 360 ]; then + echo "$(date -u +%FT%TZ): soak invalid, the reads for the sample at $now ended at $end, $((end - prev_end)) s after the previous sample's ended at $prev_end" | invalid + return 1 + fi + prev_end=$end + if [ $((now - start)) -ge 86400 ]; then + printf '%s\n' "$(date -u +%FT%TZ): soak complete, the samples from $start to $now span $((now - start)) s" > "$run/soak-complete.tmp" \ + && mv "$run/soak-complete.tmp" "$run/soak-complete" || { + echo "$(date -u +%FT%TZ): soak invalid, could not write $run/soak-complete after the sample at $now" | invalid + return 1 + } + return 0 + fi + sleep 300 + done + } ``` - Also keep the share-ack histogram (`qbit_prism_share_ack_seconds`) at - hours 1, 12 and 24 for the latency comparison between settings. -4. **Census** at hour 1 (the baseline) and hour 24 with - `docker kill --signal=SIGUSR1 "$c"`, and once more at any breach. The - coordinator logs `prism heap census written path=...`; the report lands on - the audit volume inside the container, so copy it out with - `docker cp "$c":/var/lib/qbit-prism/audit/heap-census ./heap-census-` - (substitute `PRISM_AUDIT_DIR` if it was changed). Docker accepts both - `SIGUSR1` and `RTMIN+1` as `--signal` values and delivers them; both were - checked against a container on the development host. -5. **Trim** once at hour 23 with `docker kill --signal=RTMIN+1 "$c"` on the - runs without the periodic trim, and record the logged `prism malloc_trim` - line (`docker logs "$c" 2>&1 | grep 'prism malloc_trim'`): - `resident_delta`, `free_delta`, `seconds`. -6. **Judge** each run: + The bound is evidence about one process. `compose.yaml` gives + `prism-coordinator` `restart: on-failure`, so a coordinator that exits + mid-soak is started again by Compose inside the same container, and a loop + that only read `/proc/1/status` would go on sampling the new PID 1 without + a word: the CSV would splice several short lifetimes into one 23 h span, + each restart's low RSS and reset share counters would hide the growth, and + the bound, which assumes one process, could pass. The container is the + same across a Compose restart, so `docker inspect` on `$c` is the right + probe: `StartedAt` moves on any restart, `RestartCount` counts the ones the + policy made, and `Status` catches a process that exited and was not + restarted. Every identity probe must exit successfully and return a + nonempty value; a failed or empty probe invalidates the run, including + the initial probe before any samples are taken. + `soak-process.log` keeps one reading per sample, so a later + reader can show the run was one process. That reading is taken before the + sample's two reads, so it catches a restart since the previous sample; a + restart between the check and the reads would record the replacement's + RSS and metrics under the original identity in `soak-process.log`. Every + later iteration's check would catch that, but the last iteration of a + valid run has no later iteration, because `capture` ends the run after + the sample that completes the 24 h, so a restart in that window on the + last sample would leave a directory with no `soak-invalid` that reads as + a single-process run. So the identity is read again after the two reads, + and a change ends the run as invalid; that sample's rows are already in + the files, which is fine, because the marker keeps the directory from + being judged, as step 6 says. The completion check comes after this + one, so `soak-complete` is written only when the sample that completes + the 24 h has passed it. + Any change during the soak invalidates the run: keep the run directory, + whose `soak-invalid` holds the message, attach `docker logs "$c"` (the + container keeps the exited process's output), and start over from step 2. + + A run that starts over must not write into the files of the run it + replaces. The judge sorts the samples by timestamp before it reads them, + so a `soak-rss.csv` shared by two runs would take its first hour, and so + its baseline, from the earlier run, judge the later run against that + baseline, and satisfy the span floor with the earlier run's first sample + and the later run's last; the two logs would splice the same way. So + every run writes into its own directory, named for its UTC start time, + and `capture` returns on the `mkdir` that creates it: `mkdir` without + `-p` refuses a name that already exists, and on that refusal `capture` + returns with `mkdir`'s status before its loop has run, rather than + append to whatever the name holds. The earlier run's directory stays as + it was, which is the record the restart rule asks to keep. + + A missing sample invalidates the run for the same reason. The bound assumes + an unbroken five-minute series: its span floor only checks the first and + last timestamps, so two samples 23 h apart satisfy it, and a gap between + them can hide an excursion that drained back before the next sample. A + loop that wrote nothing on a failed read would leave exactly that gap, and + the judge's malformed-row rule cannot catch it because no row is written + at all; its gap rule would, but a day later, without the failed read's + output. So each iteration writes exactly one `seconds,bytes` row or stops: + the body is read into a variable first, because the exit status of a + pipeline is its last command's without `pipefail`, which not every + operator shell sets; the row is printed only when the body carries one + `VmRSS:` line in kB; and an empty or malformed result, whether from a + `docker exec` that failed (its exit status is printed) or a body without + `VmRSS:` (the body is printed), ends the run. A transient `docker exec` + failure is therefore not skipped: the run is invalid and starts over from + step 2. + + An append that fails is a missing sample the loop itself caused, and it + invalidates the run for the same reason. The run directory can fill, or + the filesystem under it can return an I/O error, at any hour of the soak, + and a shell that is not running under `set -e` goes on past a failed + redirection as if it had succeeded, so a loop that did not look at the + status of its appends would keep reading and keep failing to write. Once + the rows already on disk spanned the 82,800 s the judge's floor asks for, + they would pass the judge, and the hours after the disk filled, with the + process readings and metrics lines that should explain them, would not + be in the record. So the loop checks the status of each of its three + appends, to `soak-process.log`, `soak-rss.csv` and `soak-metrics.log`, + and stops the run as invalid on the first that fails, naming the file + and the sample time. The metrics lines are read into a variable before + they are appended, for the reason the body and the response are: the + status of a pipeline is its last command's, which here would be `sed`, + and what a failed write does to `sed`'s exit status is each + implementation's own affair, whereas the shell builtin `printf` reports + one as `1` in `sh`, `bash` and `zsh` alike, so the same guard reads all + three appends in the shell's own terms. The run is invalid and starts + over from step 2. + + A gap between samples invalidates the run for the same reason, and it is + the hole no read can report. A host that is suspended for an hour, or a + `docker inspect` or `docker exec ... cat` that stalls for one, leaves the + series the same hole a missing sample would; neither call is bounded, only + the `curl` carries `--max-time`. Until now the iteration after the stall + wrote a valid row and the loop went on, the judge's span floor saw the + first and last timestamps and nothing between, and an excursion that rose + and drained inside the hole was gone from the record. So the loop keeps + the previous iteration's `now` and compares each new one with it before + anything is written for the iteration: past 360 s it prints the interval, + its own timestamp and the last sample's to stderr and stops, so the run's + files end at the last sample taken in cadence. The tolerance is the 300 s of sleep plus a + minute for the three reads, and it is the same `max_gap=360` the judge + above holds the file to, so a run judged from its file alone, or a file + whose loop was ended some other way, is held to the same interval. The + run is invalid and starts over from step 2. + + The last sample's reads are the hole that check cannot see. A row is + stamped with the `now` read before the sample's reads, so a + `docker inspect` or `docker exec ... cat` that stalls inside them shows + only in the next iteration's `now`, which is what the check above + compares, and the last iteration of a valid run has no next iteration, + because `capture` ends the run after the sample that completes the 24 h. + Until now a stall of an hour in that sample's reads passed both identity + checks, because the process had not changed, and the completion check + read the sample's stale `now`: the run ended with a row stamped 86,400 s + after the first that described the process an hour later, a marker that + gave that stamp as the end of the span, and nothing in the directory to + tell it from a run in cadence. So the clock is read again after the + post-read identity check, and each sample's end is held to the previous + sample's end as its start is held to the previous start: past 360 s it + prints the sample's time, when its reads ended and how long after the + previous sample's that was, to stderr, and stops, before the completion + check can run. The 300 s of sleep lie inside that interval, so the check + leaves the reads the same minute the tolerance above does, and holds the + last sample's reads to it with no next iteration needed; a hole in the + sleep itself is still seen first by the check above, before anything is + read for the iteration. The run is invalid and starts over from step 2. + + Every consecutive clock reading must also be nondecreasing: the next + sample's start cannot precede the previous sample's end, and a sample's + end cannot precede its own start, including the first and final samples. + A backward step invalidates the run before another sample or completion + marker can be written. Keep the invalid directory for diagnosis and + restart from step 2; sorting its CSV cannot make the timing trustworthy. + + The metrics sample is held to the same rule. The correlated series is what + reads an RSS excursion back at its own five-minute sample, so a scrape + that fails, or whose header says `stale` or `unavailable`, would leave + that series with a hole at the very interval the RSS series may need + explained. The response is read into a variable the same way, and the + filtered lines are appended only when the exec succeeded, the body carries + the line `x-prism-metrics-state: fresh` and the line + `qbit_prism_collector_available{collector="process"} 1`, and the RSS gauge + family, `qbit_prism_process_resident_memory_bytes`, the series the reading + order starts from, carries a nonnegative integer value; anything else + prints the time, the first condition that failed (the exit status, the + state header actually seen or its absence, the process collector gauge + line actually seen or its absence, or the RSS gauge line actually seen or + its absence) and the header lines to stderr, and ends the run. The header + alone does not make the RSS value usable: it describes the age of the last + snapshot publication, and a fresh snapshot can carry a failed process + collection. The collector families are overlaid at scrape time, and when + the last successful process collection is absent or older than 30 + seconds, the server renders the process collector gauge at 0 and + `qbit_prism_process_resident_memory_bytes` at `-1` under a header that + still says `fresh`. That sample has no RSS value for the reading order to + start from, so it is unusable and the run is invalid. The same fresh scrape + must contain exactly one nonnegative integer accepted ACK count and + authorized-client gauge. The accepted count must increase after the first + sample, and the authorized count must match `expected_authorized_clients`; + missing, malformed, frozen or decreasing evidence invalidates the run + before completion can be published. Both counts are retained in + `soak-metrics.log`, together with every share-ACK bucket, sum and count for + both results from that same scrape. The other families + in the filter are not required, because a histogram bucket or a + `runtime_task_stalled{task}` series can legitimately be absent from a + given scrape. The step-2 gate proves the body fresh and the collector + publishing once, at the start; this check proves both, a usable RSS value, + the configured population size and accepted-share progress at every later + sample. + + Every stop path writes its message the same way, through `invalid`, + which copies what it is given to `$run/soak-invalid` and to stderr, and + then returns `1` from `capture`. The message on the terminal serves the + operator who is watching it and nobody else. A loop left with `break` + ends with status `0`, which is what a caller reads as success, and the + iteration that finds a bad metrics sample has already written its RSS + row, so a stop after 23 hours used to leave a directory whose CSV + satisfies the span floor and nothing, in the directory or in the status, + to say the run was cut short; a wrapper that ran the fence as a script, + or an operator who kept the directory and lost the shell's scrollback, + would have judged it and passed it. So the directory records why it is + invalid on its own, and the status says that it is: `capture` returns + `1` only through `invalid`, and `0` only once it has written the + completion marker the next paragraph describes, `soak-complete`. The fence + defines `capture`, and step 4 calls it, in the operator's interactive shell, + which is why the stop paths `return` rather than `exit`, which would end + the shell that steps 4 to 7 read `$run` from, and why `run=` is set + outside the function. When the fault is the disk itself the marker may + not be written; `tee` says so on stderr and still copies the message + there, so it reaches the operator either way, and the run starts over + from step 2 as before. + + A valid run ends with a marker too, for the same reason. The judge's + span floor is 82,800 s, an hour short of the soak, and a loop that ran + until something outside it ended the run left that hour to whatever + did: a `SIGHUP` from a closed terminal, a `SIGTERM` from a host going + down or an operator's interrupt in the 24th hour runs none of the stop + paths, writes no `soak-invalid`, and leaves a CSV whose first and last + samples are 23 h apart, which the floor accepts. Nothing in that + directory told it apart from a run the operator had ended after 24 h, + and a directory judged on its files alone passed. So the loop keeps the + time of its first sample, the first row of the CSV, and ends the run + itself: after the first sample taken 86,400 s or more after it, once + that sample's reads, appends, hour-24 snapshots, both identity checks and + the check on the time its reads ended have passed, it writes one line, the time and the + span the samples cover, to `$run/soak-complete.tmp`, renames that to + `$run/soak-complete`, and returns `0`. The marker is published by the + rename, so it appears whole or not at all. A redirect straight to the + marker's name would create the file before the first byte is written, + and a write that then failed, on a full disk, could keep `invalid` from + writing `soak-invalid` on the same disk in the same moment, leaving an + empty `soak-complete` and no `soak-invalid`, which is the one directory + the gate must never accept. The rename is a single operation inside the + run directory: a write or rename that fails, or an interruption between + the two, leaves at most `soak-complete.tmp`, which step 6 does not read, + and a write or rename that fails also ends the run as invalid through + `invalid`, like a failed append, so the run is invalid whether or not + that message reaches the disk. Step 6 reads the two markers before the + CSV: `soak-invalid` makes the run invalid whatever else the directory + holds, neither marker is a run cut short before it was complete, and + only `soak-complete` with no `soak-invalid` reaches the bound check. The + marker is written by the loop that took every sample in the directory + and by nothing else: `capture` returns on the `mkdir` that finds the + directory already there, so a marker an earlier run left cannot be + found by a later one, and an operator does not write one by hand. A run + cut short is invalid and starts over from step 2, keeping its directory + as the restart rule asks. + + `VmRSS` in `/proc/1/status` is the field the registry's process collector + reads, so the CSV and the gauge agree up to collector cadence. The log also + carries the runtime and pool series item 3 of the reading order cites, so a + breach found after the run can be read back at its own five-minute sample + instead of from the hour-1 or hour-24 snapshot. The per-cadence share-ACK + histograms support interval latency comparisons. `capture` also keeps + cumulative share-ACK snapshots at hours 1, 12 and 24 for inspection, + through this function: ```sh - python3 -m lab.prism.process_telemetry rss-bound --samples soak-rss.csv \ - --warmup-seconds 3600 --multiple 2.0 --min-span-seconds 82800 + share_ack_snapshot() ( + snapshot=$1 + metrics=$(docker exec "$c" curl -fsS --max-time 5 -D - http://127.0.0.1:3341/metrics) || { + echo "share-ack snapshot failed: metrics scrape failed" >&2 + exit 1 + } + lines=$(printf '%s\n' "$metrics" | awk ' + BEGIN { in_headers = 1 } + { sub(/\r$/, "") } + in_headers { + headers = headers $0 "\n" + if (tolower($0) == "x-prism-metrics-state: fresh") fresh = 1 + if ($0 == "") in_headers = 0 + next + } + /^qbit_prism_share_ack_seconds_(bucket|sum|count)[{ ]/ { histogram = histogram $0 "\n" } + $1 == "qbit_prism_share_ack_seconds_count{result=\"accepted\"}" { + accepted_n++; accepted_ok = NF == 2 && $2 ~ /^[0-9]+$/ && $2 + 0 > 0 + } + END { + if (!fresh || histogram == "" || accepted_n != 1 || !accepted_ok) exit 1 + printf "%s%s", headers, histogram + }') || { + echo "share-ack snapshot failed: fresh state header and positive accepted ACK count required" >&2 + exit 1 + } + if ! printf '%s\n' "$lines" > "$snapshot.tmp" || ! mv "$snapshot.tmp" "$snapshot"; then + echo "share-ack snapshot failed: could not publish $snapshot" >&2 + exit 1 + fi + ) ``` - Pass: exit `0` on every run you intend to ship, and share-ack p99 at - hour 24 within the alert threshold already configured for the deployment, - with no `prism malloc_trim` duration above 500 ms. Fail: any exit `1`, or a - latency regression between the arena settings that the operator would - alert on. Between 2 and 4, ship the one that passes with the larger margin - under the bound; if both pass and the latency is indistinguishable, keep - the image default of 2. -7. **Record** on issue #226: the three verdict JSON documents, the census - reports, the trim lines, the share-ack histograms, the image digest, and - the glibc version inside the image (`ldd --version`), because the trim - behaviour above was measured on glibc 2.39 and the image carries a newer - one. -8. **Re-run #185's drain** on the fixed build: + `capture` calls it with `"$run/share-ack-h01.txt"`, + `"$run/share-ack-h12.txt"` and `"$run/share-ack-h24.txt"` in the first + samples taken 3,600 s, 43,200 s and 86,400 s or more after its first. + Each file keeps the response headers and the histogram from the same + successful scrape. A cached response can return HTTP 200 with a `stale` + or `unavailable` state, so the function requires the `fresh` header and a + positive integer accepted ACK count before writing anything. A rejected-only + histogram is not accepted-share latency evidence; the capture loop separately + checks that accepted ACKs keep progressing. It publishes the snapshot by + renaming a completed temporary file; a failed write leaves no partial snapshot at the final + name. Use only the final `.txt` files for snapshot inspection. If any + call fails, the run lacks usable latency evidence, so `capture` ends it + as invalid through `invalid`: keep its directory and repeat the soak from + step 2. + +4. **Snapshot** the full `/metrics` body, headers included, at hour 1 (the + baseline), hour 24, and at any breach: + + ```sh + full_metrics_snapshot() ( + snapshot=$1 + if ! docker exec "$c" curl -fsS --max-time 5 -D - http://127.0.0.1:3341/metrics > "$snapshot.tmp"; then + echo "full metrics snapshot failed: metrics scrape or temporary-file write failed" >&2 + exit 1 + fi + if ! awk ' + { sub(/\r$/, "") } + $0 == "" { exit } + tolower($0) == "x-prism-metrics-state: fresh" { fresh = 1 } + END { exit !fresh } + ' "$snapshot.tmp"; then + echo "full metrics snapshot failed: fresh state header required" >&2 + exit 1 + fi + if ! mv "$snapshot.tmp" "$snapshot"; then + echo "full metrics snapshot failed: could not publish $snapshot" >&2 + exit 1 + fi + ) + ``` + + `capture` calls it with `"$run/metrics-h01.txt"` and + `"$run/metrics-h24.txt"` in the samples that take the hour-1 and hour-24 + share-ack snapshots, after them; at each breach call it with a distinct + name such as `"$run/metrics-breach-$(date +%s).txt"`. + For a manual breach snapshot while `capture` is running, use a second + shell, set `c` to the same container and `run` to the absolute path of the + existing run directory, and define `full_metrics_snapshot` there before + calling it. + The function requires a successful scrape and a `fresh` response header, + then renames the temporary file to publish the complete headers and body + unchanged. HTTP errors, transport failures, cached responses and failed + writes or renames return nonzero and publish no new final snapshot. Use + only the final `.txt` files as evidence; a `.tmp` file may be partial or + stale. If a scheduled snapshot fails, `capture` ends the run as invalid + through `invalid`; keep the run directory and repeat the soak from step 2. + If a manual breach snapshot fails, record the failure in + `$run/soak-invalid`, keep the directory, and repeat the soak from step 2. + + This replaces the census step: there is no heap walk on the native server, + and the body at the breach is what the correlated reading works from. + + The loop schedules the hour snapshots against its first sample's + timestamp. It takes them after appending the sample and before checking + process identity and read timing again, so a restart or delay during a + snapshot can invalidate the run. A failed scheduled snapshot writes + `soak-invalid` with the snapshot function's diagnostic; `soak-complete` + follows successful hour-24 snapshots. Define `capture`, + `share_ack_snapshot` and `full_metrics_snapshot` in the same shell before + starting the run: + + ```sh + capture + ``` +5. **Trim** is retired with `malloc_trim`; there is nothing to send at hour 23. +6. **Judge** each run with the `awk` bound check above against its own + `$run/soak-rss.csv`, and only once this gate has accepted the run + directory: ```sh - python3 tests/prism_candidate_storm.py --decide --drain-per-row --heap-report + completed() { + if [ -e "$run/soak-invalid" ]; then + echo "not judged: $run/soak-invalid says why the run is invalid" >&2 + return 1 + fi + if [ ! -f "$run/soak-complete" ]; then + echo "not judged: $run has no soak-complete, capture did not end the run after 24 h of samples" >&2 + return 1 + fi + cat "$run/soak-complete" + } + completed ``` - and record `start` against `after_release_gc` and `after_malloc_trim`, - with the platform line, on #185. + The gate prints the completion line and exits `0` for a directory that + holds `soak-complete` and no `soak-invalid`; then run the bound check + inside the run directory, or substitute the path, since the check names + `soak-rss.csv`. It exits `1`, and the run is not judged whatever its CSV + would say, when the directory holds `soak-invalid`, because `capture` + stopped the run on a gap between samples, a process change, a missing + RSS sample, an append that failed, a metrics scrape that failed, was not + fresh, had its process collector unavailable or carried no usable RSS + value, an hour snapshot that failed, or a completion marker it could not + write, and the marker says + which; and when the directory holds no `soak-complete`, because then + `capture` did not end the run itself after 24 h of samples, so whatever + did, a signal, a reboot, a closed terminal or an interrupt, did so before + the run was complete, however far its CSV reaches. Either run is invalid + and is run again from step 2. The marker is `capture`'s to write; do not + write one by hand. + For latency, use `soak-metrics.log`: subtract the previous cadence's + bucket and count values from the hour-1 cadence and, separately, from the + hour-24 cadence. Match series by their labels and buckets by `le`, never + line order; require complete, unchanged bucket boundaries, including + `+Inf`, and counts that agree with `+Inf`. Compute p99 from each interval's + cumulative bucket deltas, summing accepted and rejected results before + taking the quantile. A single hour snapshot is cumulative since process + start; its p99 can hide a slow final interval behind earlier fast ACKs. + Missing or malformed evidence, counter decreases/resets, inconsistent + bucket deltas or fewer than 100 interval observations make latency + **inconclusive**, so the run cannot pass the combined verdict. + + These are adjacent-sample intervals, nominally 300 s with up to 60 s of + read overhead and scrape-time variation, not exact Prometheus `[5m]` + windows. The shipped `PrismShareAckP99High` rule in + `docs/prism-native-alert-rules.json` takes p99 over `rate(..._bucket[5m])`, + combines both results per target, requires `increase(..._count[5m]) >= 100` + and a fresh, available snapshot, and fires only after p99 stays **above + 1 second** for `3m`. Use the deployment's actual rule and archived + Prometheus evaluations to establish that alert verdict; two soak samples + cannot establish the exact five-minute estimate or its dwell. + + Pass: RSS check exit `0` and usable hour-1 and hour-24 interval evidence, + with hour-24 p99 within the deployment's configured ACK threshold. + Fail: RSS check exit `1` or hour-24 interval p99 above that threshold. + Record both interval p99 values, observation counts, elapsed times and the + threshold; report any alert firing separately from this interval verdict. + Record every + `qbit_prism_runtime_task_stalled` sample at 1 with its timestamp; a stall + that coincides with an RSS excursion is the first thing to explain. +7. **Record** on issue #291, which owns cutover qualification: the verdict + line, the run directory (`soak-process.log`, `soak-rss.csv`, + `soak-metrics.log`, `soak-complete`, the three share-ack histograms, and + the hour-1, hour-24 and breach snapshots), the image ID, and the redacted + deploy dotenv. The glibc version inside the image + (`docker exec "$c" ldd --version`) still belongs in the record, because + the process uses it as its allocator. +8. **The post-storm drain re-run** on #185 is retired; the storm rig was a + Python test deleted in #244, and #185's drain measurement is the Python + lane's record. + +## Payout-window JSONB storage ceiling + +Independent of any load-test record, the payout window has a hard storage +ceiling. PostgreSQL refuses a JSONB container whose elements exceed 268,435,455 +bytes, and three native writes still embed the whole window, so each grows +linearly with the share count. `cargo test -p qbit-prism-server --test +jsonb_ceiling_gate` measures them against 25% of that limit (67,108,863 bytes) +and fails when the set of crossing writes changes in either direction. + +| Path | Column written | Window copies | At 400,000 shares | Removed by | +| --- | --- | --- | --- | --- | +| refresh | `qbit_prism_jobs.payload` | 3 | refused by PostgreSQL (measured) | #273 | +| enqueue | `qbit_block_candidate_outbox.candidate` | 2 | refused by PostgreSQL (measured) | #265 | +| landing | `qbit_pool_audit_bundles.audit_bundle` | 1 | 235 MB, 3.5x the gate threshold (measured) | #267 | + +The legacy audit import (`import-audits`) was a fourth such write, refused by +PostgreSQL at 400,000 shares. #265 removed it. The import now stores only +`canonical_audit_bytes`, which is `bytea`, so PostgreSQL's 1 GiB value limit +bounds it rather than the JSONB ceiling. It writes no JSONB value. + +The host, gate commit, PostgreSQL version (16.15) and build mode (debug) behind +these measurements are recorded under "Baseline, measured at the base commit +plus the gate's own test files" and "At 400,000 shares" in +`docs/prism-payout-artifact-measurement.md`. + +Sizes are uncompressed JSONB containers. A refused write is reported as refused, +never with a size. After the enqueue write is refused, the gate writes a +window-free substitute outbox row itself so the later phases stay measurable; +the report labels that row `SUBSTITUTE` and never counts it as the enqueue write. +See "JSONB ceiling gate and 400k-share +baselines" in `docs/prism-payout-artifact-measurement.md` for the 50,000-, +100,000-, 200,000- and 400,000-share measurements and how to reproduce them. A +capacity qualification run should record the payout window size it exercised, +because a window near 400,000 shares reaches this ceiling before it reaches any +throughput limit. diff --git a/docs/prism-compact-runtime-e2e-history.md b/docs/prism-compact-runtime-e2e-history.md new file mode 100644 index 000000000..5703e1bbe --- /dev/null +++ b/docs/prism-compact-runtime-e2e-history.md @@ -0,0 +1,160 @@ +# Compact runtime end-to-end qualification history + +This records the independent test slice and its integration at the named +historical commits. Current review and qualification status is maintained in +[PR #397](https://github.com/Qbit-Org/qbit-mining-bootstrap/pull/397). + +## Independent baseline, 2026-09-15 + +Base: `cff94df5` (the inline runtime, after #377). The test slice touches only +`tests/compact_runtime_e2e.rs` and its new dedicated support directory. +Cargo automatically discovers the target. Gate-manifest registration remains +with the activation owner. + +The run used a newly initialized disposable PostgreSQL 16.14 instance with +`fsync=on`, `full_page_writes=on`, and `synchronous_commit=on`, checked by the +fixture. Each case creates and drops a unique schema; cases run serially because +the ledger advisory locks are database-wide. The nonempty window contains 16 +accepted rows loaded by the existing WindowPlan. All prepared and issued data +comes from real public Coordinator calls against the existing FakeNode. + +```sh +PRISM_TEST_REQUIRE_INTEGRATION=1 \ +PRISM_TEST_DATABASE_URL=postgresql://USER@127.0.0.1:PORT/postgres \ +cargo test --locked -p qbit-prism-server --test compact_runtime_e2e \ + -- --nocapture --test-threads=1 +``` + +Compilation passed. The baseline test result is intentionally **red**: +7 passed, 6 failed, 0 ignored (4 passing runtime cases plus 3 inherited +WindowPlan unit tests). All six failures report exactly: +`runtime refresh still wrote inline prepared storage: no compact format_version`. + +| Runtime test | Inline baseline result | +| --- | --- | +| `refresh_issue_resume_preserves_original_work_and_compact_storage` | A/B independent publication, exact wire/bundle/input equality, unchanged refresh and wrong-worker rejection pass; compact shape fails | +| `bootstrap_resume_keeps_each_workers_original_payout` | Two distinct worker payouts and exact cross-frontend resume pass; compact shape fails | +| `real_socket_reconnect_resumes_original_entropy_mask_and_submits_once` | Real socket subscribe/configure/authorize/notify, reconnect on B with different session entropy, original-mask submit, duplicate refusal, one accepted row and original header hash pass; compact shape fails | +| `resume_expiry_does_not_slide_and_expired_work_is_a_miss` | Pass | +| `malformed_issued_inputs_are_errors_and_missing_work_is_a_miss` | Pass: zero/oversized/invalid targets, nonpositive/string difficulty, bad entropy; missing job is a miss | +| `compact_reference_corruption_is_an_error_and_blobs_survive_collection` | Blocked at compact prerequisite; the corruption/collection assertions have not executed on this base | +| `missing_prepared_dependency_repairs_original_record_without_renewing_identity` | Blocked at compact prerequisite; original-record repair and retention assertions await activation | +| `resumed_compact_work_authenticates_retained_share_rows` | Blocked at compact prerequisite; middle-share corruption rejection awaits activation | +| `cancelled_issued_save_releases_sql_resources_without_publishing` | Pass: a real SQL advisory-lock waiter is observed before cancellation, a subsequent save succeeds, cancelled job stays absent | +| `unknown_issued_commit_is_observed_and_reconciled_without_reissuing` | Pass: existing PostgreSQL wire proxy withholds COMMIT acknowledgement, original durable job resumes through B, exactly one INSERT and unchanged expiry | + +The compact assertions use `Ledger::compact_prepared`, the public validating +reader, and canonical audit/manifest serializers. They compare the original +window, watermark, payout revision, template, balance digest, builder policy, +keys and audit hashes, and reject recursively embedded share arrays. A fixture +AFTER ROW trigger records actual committed job-row writes and their maximum +uncompressed JSONB size; this is scoped to job rows, not a global write inventory +or a WAL measurement. Missing measurements fail rather than becoming zero. + +## Remaining qualification + +- Core qualification owns internal admission/read/rebuild/cleanup deadline + proofs and the 400k/500k memory/WAL runs. This 16-share fixture makes no scale + claims and does not qualify replacement-lease authority by cross-frontend + substitution. +- Full deep-review lanes require the final qualified activation diff and are + launched by the coordinator. This test branch is intended for integration + into that activation PR; no standalone PR or overall merge readiness is claimed. + +## Approved-helper follow-up + +After preserving the independent baseline commit, the coordinator approved +integrating support-only `c52a7572f31096f589848682ebe5aa306f354d52` for its +FakeNode controls and PostgreSQL returned-row observer. No shared helper was +edited by this test slice. + +`delayed_old_refresh_cannot_replace_new_tip_publication_or_resume_retired_work` +passes: A's old template response is held at the approved HTTP barrier, B +publishes an advanced node tip, A's delayed refresh fails, and both frontends +refuse the old issued job after A publishes its own new-tip replacement. +The A/B resume test now also requires exactly 16 actual returned share rows, +using the shared proxy's DataRow/CommandComplete accounting and excluding +endpoint probes and metadata. This assertion follows the compact-storage +prerequisite and therefore remains unqualified on the inline base. + +The complete helper-backed inline run reports **8 passed, 6 expected failures, +0 ignored**, including the three inherited WindowPlan unit tests. All six +failures remain at the compact-format prerequisite. Clippy passes with +`cargo clippy --locked -p qbit-prism-server --test compact_runtime_e2e -- -D warnings`. + +## Integrated functional run, 2026-09-15 + +Activation base: `a154eb5969e1fe64b180914c099dc257b701cf50`, plus approved +COMMIT-pause helper `487827d`. Only the new E2E +files were replayed onto this exact committed head; production, shared support, +and gate-manifest files remain the activation owner's versions. The original +inline-red commits and evidence remain preserved on their separate branch. + +The completed suite reports **18 passed, 0 failed, 0 ignored** in the same durable +PostgreSQL 16.14 fixture: 15 runtime cases and 3 inherited WindowPlan unit tests. +Strict Clippy also passes. The previously blocked corruption, reference-row +authentication, blob retention, and original-dependency repair cases now execute +and pass. Observed prepared-row maxima are 1,518 uncompressed JSONB bytes for +the 16-share window and 1,146 bytes for the empty bootstrap window. + +`PreparedBundle` is now treated solely as submission metadata. On an independent +blocking test task, the oracle reads original `StoredCompactPrepared` inputs +and actual `Ledger::read_window(..., BalanceSource::AsIssued)` rows, then invokes +the public borrowing native-audit builders. It verifies BOTH original stored +audit/coinbase-manifest hashes and the issued coinbase against the rebuilt +artifact. Builder options come from the stored record, and the fixture signing +seeds must match its recorded public keys. Setup also reads the actual referenced +window outside measurement brackets; it never assumes retained snapshot arrays. + +Bootstrap workers prove distinct actual coinbases and synthetic payout shares, +and each coinbase is independently reconstructed. The real-socket replacement +listener advertises a harder target and negotiates no rolling mask; the old +job still submits under its original target, entropy, and mask exactly once. +That strengthened socket case is included in the final full integrated run. +The 16-row fixture does not claim enabled-CTV or large-window qualification. + +Two additional cases exercise original expiry across real PostgreSQL waits. +Issued persistence waits behind SETTLEMENT past its one-second expiry, then +refuses publication after the lock is released without changing the prepared +reservation. Resume reaches an observed AccessShareLock wait on the actual +share table and returns an expired-job miss while the blocking lock is still +held. Both cases subsequently issue/resume fresh work successfully, proving +recovery after the queued SQL operations are released. These are public runtime +tests, not claims about internal build-permit or blocking-owner cancellation. + +The final two tests use the owner's `ExecutionProxy::pause_after_commit` helper +to hold an actual completed COMMIT reply while the database row is already +visible and transaction locks are free. An unchanged cached refresh completes +during that wait and preserves the original publication, payload, and expiry; +the original job is then delivered and resumed successfully. A genuinely newer +tip publication during the same wait causes the old persistence to refuse +delivery after the reply is released, and both frontends reject the retired +job while accepting fresh work. The committed old row stays unchanged: database +commit alone does not authorize miner delivery after revocation. + +The final delta was then replayed onto the newer integration head +`a9158b1cd68cd83ddd5c9c4449d03f327af21e95`, including the owner's later compact +ledger guards and runtime admission tests. The same complete E2E command again +passed **18/18** with no skips (9.65 seconds), and target Clippy again passed +with warnings denied. This is functional evidence for these 15 runtime cases; +the separate scale qualification and coordinated full review remain required. + +## Gate IDs for activation-owner registration + +```text +qbit-prism-server::compact_runtime_e2e::bootstrap_resume_keeps_each_workers_original_payout +qbit-prism-server::compact_runtime_e2e::cancelled_issued_save_releases_sql_resources_without_publishing +qbit-prism-server::compact_runtime_e2e::compact_reference_corruption_is_an_error_and_blobs_survive_collection +qbit-prism-server::compact_runtime_e2e::delayed_old_refresh_cannot_replace_new_tip_publication_or_resume_retired_work +qbit-prism-server::compact_runtime_e2e::issued_expiry_during_sql_wait_does_not_publish_or_renew_original_identity +qbit-prism-server::compact_runtime_e2e::malformed_issued_inputs_are_errors_and_missing_work_is_a_miss +qbit-prism-server::compact_runtime_e2e::missing_prepared_dependency_repairs_original_record_without_renewing_identity +qbit-prism-server::compact_runtime_e2e::real_socket_reconnect_resumes_original_entropy_mask_and_submits_once +qbit-prism-server::compact_runtime_e2e::refresh_issue_resume_preserves_original_work_and_compact_storage +qbit-prism-server::compact_runtime_e2e::resume_expiry_does_not_slide_and_expired_work_is_a_miss +qbit-prism-server::compact_runtime_e2e::resume_expiry_includes_blocked_share_read_and_releases_resources +qbit-prism-server::compact_runtime_e2e::resumed_compact_work_authenticates_retained_share_rows +qbit-prism-server::compact_runtime_e2e::superseding_publication_during_completed_commit_wait_refuses_old_delivery +qbit-prism-server::compact_runtime_e2e::unchanged_refresh_during_completed_commit_wait_preserves_original_authority +qbit-prism-server::compact_runtime_e2e::unknown_issued_commit_is_observed_and_reconciled_without_reissuing +``` diff --git a/docs/prism-configuration.md b/docs/prism-configuration.md new file mode 100644 index 000000000..9f80d71aa --- /dev/null +++ b/docs/prism-configuration.md @@ -0,0 +1,232 @@ +# Native PRISM configuration + +Run `qbit-prism-server check-config` before starting a frontend. The command +validates configuration without connecting to PostgreSQL or the node. It +checks the mining, public API, and background rollup settings together. +Malformed API numbers and booleans fail validation instead of selecting a +fallback value. + +Every environment name beginning with `PRISM_` that the native process does +not support is listed in one diagnostic, including names whose values are +empty. Values are never included in this diagnostic. It is a warning in lab +mode and an error when `QBIT_PRODUCTION=1`, `QBIT_TOOLS_PRODUCTION=1`, or +`QBIT_CHAIN=mainnet` (also `main`) selects production mode. Settings belonging +to Compose or a replica bootstrap script should be passed to those tools, +not exported into the native server process. + +Starting the coordinator runs the same check: `qbit-prism-server` with no +subcommand (the `run` serve path) refuses to start in production, naming each +unsupported variable, and warns in lab mode. The check runs before the +process parses its configuration or contacts the node, so a 2.x.x environment +is reported instead of a downstream startup failure. Unset the reported names +in the unit or Compose environment, or move them to the tool that reads them; +a name that belongs to no tool should be removed. `check-config` remains the +preflight, because it reports every invalid setting it covers without +starting a frontend; the public reader's own database settings are checked +separately by `check-public-database-config`. + +The supported names live in +[`native-settings.txt`](../crates/qbit-prism-server/src/config/native-settings.txt). +The retired 2.x.x names live in +[`retired-settings.txt`](../crates/qbit-prism-server/src/config/retired-settings.txt). +The diagnostic also catches unknown names absent from either inventory. +Conditional settings remain recognized when their feature is disabled. + +## Mounted signing seeds + +Production frontends read signing seeds from files: + +```sh +PRISM_MANIFEST_SIGNING_SEED_HEX_FILE=/run/secrets/qbit-prism/manifest-signing-seed-hex +PRISM_LEDGER_ATTESTATION_SIGNING_SEED_HEX_FILE=/run/secrets/qbit-prism/ledger-attestation-signing-seed-hex +``` + +Each file contains one 32-byte hexadecimal seed. A trailing newline is +accepted. The process must be able to read the mounted files as its non-root +UID; provision ownership and permissions before starting it. Keep the +trusted public key in `PRISM_LEDGER_WRITER_PUBLIC_KEY_HEX`; it must match the +ledger seed, and the two signing keys must differ. + +Nonempty direct `PRISM_MANIFEST_SIGNING_SEED_HEX` and +`PRISM_LEDGER_ATTESTATION_SIGNING_SEED_HEX` values are rejected in production. +Outside production, either the direct value or its `_FILE` form is accepted; +setting both fails rather than choosing one silently. Empty, unreadable, +malformed, or oversized secret files also fail. Development test seeds still +require `PRISM_ALLOW_TEST_SIGNING_SEEDS=1`, which production rejects. + +This implements decision D4 in #260: signing keys remain on every mining +frontend, with mounted delivery, a non-root image, and disabled core dumps. +On Linux, the server disables process dumpability before loading credentials; +this also prevents dumps piped to a host collector, for which +[Linux ignores `RLIMIT_CORE`](https://man7.org/linux/man-pages/man5/core.5.html). +The key rotation rehearsal required before cutover is tracked by #291. + +## Database-only commands and audit import + +`migrate`, `import-audits`, and `backfill-ctv` load database configuration +without reading either signing seed. They still require +`PRISM_DATABASE_URL`; connection and instance budgets use the same validation +as the server. `backfill-share-hashes`, which finishes a deferred share-hash +backfill after go-live, reads `PRISM_DATABASE_URL` alone. Audit import and CTV backfill additionally require the public +trust pin `PRISM_LEDGER_WRITER_PUBLIC_KEY_HEX` to verify stored artifacts. +Commands that build or sign work continue to require signing configuration. + +For audit import, `--root` overrides `PRISM_AUDIT_DIR`. Without `--root`, +`PRISM_AUDIT_DIR` selects the directory containing legacy audit bodies. +An empty value is treated as unset. When neither is set, the ledger importer +resolves stored body URIs as before. + +## Operator listener and health refresh + +`PRISM_OPERATOR_BEARER_TOKEN` optionally protects the operator listener on +port 3341. Its `_FILE` form reads a mounted token instead. Tokens must contain at +least 16 visible ASCII characters with no whitespace. Use exactly one form, +and configure probes and monitoring clients with the matching bearer +token. The built-in `healthcheck` command reads that token for operator +probes; `healthcheck --public-api` probes the independent public role without +sending it. Probes refuse redirects. + +`PRISM_HEALTH_REFRESH_SECONDS` controls the health publisher cadence as well +as the snapshot and `self-check` heartbeat staleness budgets, both +`max(3 * PRISM_HEALTH_REFRESH_SECONDS, 15)` seconds. It must be a whole number +from 1 through 86400 seconds (default 2). +The public API remains a separate process and does not need signing seeds. + +## Public audit artifact admission + +`/public/v1/artifacts/` serves a block's audit by digesting and parsing +its sealed canonical bytes or, for an unsealed native block, by rebuilding the +share window; both hold the whole window in memory and outlive the read +connection. `PRISM_PUBLIC_AUDIT_REBUILD_CONCURRENCY` bounds how many run at +once in one process, a whole number from 1 through 64 (default 1). It is +independent of `PRISM_POSTGRES_READ_CONCURRENCY`, so a larger read pool admits +no more rebuilds. A request waits for a rebuild slot within its own +`PRISM_PUBLIC_READ_STATEMENT_TIMEOUT_SECONDS` deadline and gets 503 +`read_timeout` when that runs out. + +`PRISM_PUBLIC_AUDIT_ARTIFACT_MAX_IN_FLIGHT` caps the audit artifact requests +running or waiting for a slot, a whole number from 1 through 4096 (default 32). +With the response cache enabled (the default), identical concurrent requests +share one computation and one place under the cap. A request past the cap is refused at once with 503 +`audit_artifact_busy` and `Retry-After: 5`, after two indexed point lookups and +before any audit read. CTV manifests served from the same route are never +counted or refused. Both settings apply to the public service and to the +operator listener's copy of the public routes; a value outside its range stops +startup with ` must be ..`. See +[prism-ledger-ops.md](prism-ledger-ops.md#public-audit-artifact-admission) and +[prism-storage-sizing.md](prism-storage-sizing.md#native-audit-body-size). + +## Share ledger partition maintenance + +`qbit_share_ledger` is partitioned by `share_seq` and has no DEFAULT +partition, so an append whose sequence value has run past the last attached +bound is refused by PostgreSQL. Every frontend keeps the lead attached by +calling `qbit_prism_share_partition_ensure()` once at startup and then every +`PRISM_SHARE_PARTITION_ENSURE_INTERVAL_SECONDS`, a whole number from 1 +through 86400 seconds (default 60). The call is serialized per schema, so +running several frontends creates each partition once, and it creates nothing +while the lead is intact; a run that creates partitions logs at info, and one +that fails logs at warning and is retried on the next tick without ending the +task. + +A server whose startup call fails does not start: an instance that cannot +maintain its partitions must not accept shares until its lead runs out. The +width of a partition and how many are kept ahead of the sequence are database +settings, not environment settings; they live in the one row of +`qbit_prism_share_partitioning` (default 2^24 rows and 4 partitions, about +67 million rows of headroom). See +[prism-share-ledger-partitioning.md](prism-share-ledger-partitioning.md) for +the design record and +[prism-ledger-ops.md](prism-ledger-ops.md#share-ledger-partitions-and-retention) +for the operator procedure. `qbit_prism_share_ledger_partition_lead_rows` +in `/metrics` reports the remaining headroom; see +[prism-native-metrics.md](prism-native-metrics.md). + +## Stratum per-source cap and per-session budgets + +Five settings bound what one source address or one connection can cost. Every +one defaults to the value that keeps today's behavior, and `check-config` +rejects an out-of-range value rather than falling back. + +| Setting | Default | Accepted range | Failure when rejected | +| --- | --- | --- | --- | +| `PRISM_STRATUM_MAX_CONNECTIONS_PER_IP` | 0 (disabled) | 0 through the semaphore capacity | `PRISM_STRATUM_MAX_CONNECTIONS_PER_IP exceeds semaphore capacity` | +| `PRISM_STRATUM_SESSION_BUDGET_INTERVAL_SECONDS` | 60 | finite, above 0, at most 3600 | `PRISM_STRATUM_SESSION_BUDGET_INTERVAL_SECONDS must be finite, positive and at most 3600` | +| `PRISM_STRATUM_MAX_MALFORMED_FRAMES_PER_INTERVAL` | 0 (disabled) | 0 through 1000000 | `PRISM_STRATUM_MAX_MALFORMED_FRAMES_PER_INTERVAL must be at most 1000000` | +| `PRISM_STRATUM_MAX_UNKNOWN_JOBS_PER_INTERVAL` | 0 (disabled) | 0 through 1000000 | `PRISM_STRATUM_MAX_UNKNOWN_JOBS_PER_INTERVAL must be at most 1000000` | +| `PRISM_STRATUM_MAX_AUTHORIZE_ATTEMPTS_PER_INTERVAL` | 0 (disabled) | 0 through 1000000 | `PRISM_STRATUM_MAX_AUTHORIZE_ATTEMPTS_PER_INTERVAL must be at most 1000000` | + +A value that is not a number fails as `invalid `; an empty value selects +the default. The three budgets are rates measured over the shared interval, not +session lifetime totals: a full window's allowance is admitted as a burst and +refills whole at the next window. Valid `mining.submit` traffic is never +charged against any of them. + +The per-source cap keys on the address this listener observed, which is the +last hop. Leave it at 0 unless the deployment preserves miner source addresses. +See [prism-b4-stratum-admission.md](prism-b4-stratum-admission.md) for the +runbook, the recommended starting values and the topologies that make a cap +unsafe. + +## Stratum listen backlog + +`PRISM_STRATUM_LISTEN_BACKLOG` (default 4096) is how many connections the kernel +queues on each Stratum listener before the accept loop takes them. It accepts 1 +through 2147483647, the range `listen(2)` takes. `check-config` rejects an +out-of-range value with `PRISM_STRATUM_LISTEN_BACKLOG must be between 1 and +2147483647`, and a value that is not a number with `invalid +PRISM_STRATUM_LISTEN_BACKLOG`; an empty value selects the default. It applies to the primary and +high-difficulty listeners alike. A burst of miners reconnecting together, as +when a cutover re-points every miner, a frontend restarts or a load balancer +fails over, is held in that queue. Past it, the kernel drops their SYNs +(`TcpExtListenOverflows`), the miners retry a second later, and a load balancer +that observes layer-4 errors can mark the frontend down. The kernel caps the +value at `net.core.somaxconn` in the frontend's network namespace without an +error, so raise that too if it is lower. The frontend's `PRISM listening` line +logs both `listen_backlog` and `somaxconn`, and a warning names the cap when +somaxconn is the lower. + +Before this setting the native listeners used mio's fixed backlog of 128; the +2.x.x runtime read the same name with a default of 1024. The operator and +public HTTP listeners use a fixed backlog of 1024. + +## Block submission kill switch + +`PRISM_BLOCK_SUBMIT_ENABLED` (boolean, default `1`) exists for rehearsals +against a real node and a restored ledger (#291). Set to `0`, a frontend sends +its node no found block and no transaction: + +- the submit loop claims no candidate. A found block is still validated, + credited and enqueued, but its outbox row stays `pending` and `submitblock` + is never called; +- the CTV fanout broadcaster does not start, whatever + `PRISM_CTV_BROADCASTER_ENABLED` says, and `broadcast-ctv` refuses before it + connects. That broadcaster is PRISM's only sender of `sendrawtransaction` + and `submitpackage`, and its only user of the CPFP wallet. + +The frontend's node client also refuses, before sending, every call that would +relay a block or a transaction. Templates, readiness, share acceptance and the +database writes continue as usual. The setting is read at startup and applies +per frontend; it is not part of the cluster fingerprint. `0`, `false`, `no` +and `off` all turn it off. While it is `0`, `check-config` and `self-check` +lead with a warning. See +[the rehearsal procedure](prism-ledger-ops.md#block-submission-kill-switch-for-rehearsals) +for checking it on running frontends and for what to do with held blocks +afterwards. Never leave it at `0` on a production frontend: a block that +frontend finds is never offered, and `PrismBlockSubmissionHeld` pages a minute +after the frontend starts. + +A ledger can also hold block submission for every frontend that connects to +it, whatever this setting says: `qbit-prism-server submission-hold set +--reason ...` (migration 023, #664). `check-config` never reads the database, +so it only says so; `submission-hold show` and `self-check` report the hold. +See [holding the whole cluster](prism-ledger-ops.md#holding-the-whole-cluster-023-664). + +## Preventing stale guidance + +CI runs `python3 scripts/check_prism_settings.py`. It checks the native name +inventory against source references and rejects retired settings in +`scripts/check-env.sh`, `.env.example`, `docs/`, `doc/`, and operator READMEs. +An explicitly historical Markdown mention +can use a same-line `retired-setting` HTML annotation naming that one setting. +This exception cannot suppress a shell check or other lines of documentation. diff --git a/docs/prism-coordinator-refactor/README.md b/docs/prism-coordinator-refactor/README.md index a75de1e09..814b3a557 100644 --- a/docs/prism-coordinator-refactor/README.md +++ b/docs/prism-coordinator-refactor/README.md @@ -1,5 +1,10 @@ # PRISM Coordinator Refactor +Historical record of the Python coordinator. The Rust rewrite replaces the +modules and validation commands described below; see the +[native server](../../crates/qbit-prism-server/README.md) and +[migration guide](../prism-rust-migration.md) for the current implementation. + Status: **complete** and organized as a nine-PR review stack. No required roadmap item remains. @@ -44,3 +49,5 @@ are in [Validation](validation.md). - [Stacked PRs](stacked-prs.md): publication order and reconstruction rules. - [A1 audit artifacts](a1-audit-artifacts.md): durable storage contract. - [B3 decision](b3-decision.md): finalization concurrency evidence. +- [Window reference](window-ref.md): `WindowRef` and `Ledger::read_window` + design for the native server (#264). diff --git a/docs/prism-coordinator-refactor/validation.md b/docs/prism-coordinator-refactor/validation.md index 2f77d0525..d3218d79d 100644 --- a/docs/prism-coordinator-refactor/validation.md +++ b/docs/prism-coordinator-refactor/validation.md @@ -57,15 +57,14 @@ Run when a milestone touches persistence, payout, shares, audit, candidates, vardiff, or finalization: ```sh -make test-prism-postgres-ledger +make test-prism-postgres cargo test -p qbit-prism ``` At the concurrency/finalization and final milestones, run: ```sh -make test-prism-stratum-regtest-live -make test-prism-stratum-postgres-regtest-live +make test-prism-regtest ``` If qbitd, qbit-cli, PostgreSQL, Docker, Rust, credentials, or another real @@ -121,7 +120,7 @@ root; Docker-backed checks ran via `sudo`. | --- | --- | | PRISM Python discovery (`test_prism_*.py`) | 1,601 passed | | full Python discovery | 1,956 passed, 2 skipped (ckpool binary absent); `GIT_CONFIG_GLOBAL=/dev/null` was scoped only to this command | -| PostgreSQL ledger integration | passed via `sudo bash test/test-prism-postgres-ledger.sh`, including the typed share-replay assertions, the #120 carry-forward equivalence/drift checks, and the three A1 publication gates (transition parity, migration M0-M11, cross-process fencing C1-C3) | +| PostgreSQL ledger integration | passed via `sudo bash test/test-prism-postgres-ledger.sh` (the 2.x.x script; on 3.x.x the native suite is `make test-prism-postgres`), including the typed share-replay assertions, the #120 carry-forward equivalence/drift checks, and the three A1 publication gates (transition parity, migration M0-M11, cross-process fencing C1-C3) | | Rust workspace | 222 passed (`cargo test --locked --workspace --all-targets`), including the audit CLI canonicalizer round-trip regressions | | Docker Python compile | passed | | Docker Ruff (`E4,E7,E9,F`) | 54 findings, all F821 on lazily evaluated type annotations that deliberately reference coordinator-owned types without importing them (never evaluated at runtime under `from __future__ import annotations`); every removable finding was cleaned in the final ownership layer | @@ -130,6 +129,7 @@ root; Docker-backed checks ran via `sudo`. | structural and target diff audit | passed; see [Structure](structure.md) | `make test-prism-stratum-regtest-live` and -`make test-prism-stratum-postgres-regtest-live` are `UNAVAILABLE`: their +`make test-prism-stratum-postgres-regtest-live` (2.x.x targets; on 3.x.x both +are `make test-prism-regtest`) are `UNAVAILABLE`: their doctor stops at `Required executable not found: qbitd` before either live test can run. `qbitd` and `qbit-cli` are absent from the host path. diff --git a/docs/prism-coordinator-refactor/window-ref.md b/docs/prism-coordinator-refactor/window-ref.md new file mode 100644 index 000000000..d6f2edb8b --- /dev/null +++ b/docs/prism-coordinator-refactor/window-ref.md @@ -0,0 +1,1260 @@ +# Window Reference Design (`WindowRef` and `Ledger::read_window`) + +Decision: **persisted job and candidate rows reference the payout window; +they never copy it.** A `WindowRef` names the window's anchor, the digest of +the prior balances it was built on and, when the window has shares, an +immutable range of `qbit_share_ledger` rows with its digest. +`Ledger::read_window` rebuilds the window from those rows or fails. +Migrations 007 (#265, `qbit_block_candidate_outbox`) and 008 (#273, +`qbit_prism_jobs`) embed the reference in the row's JSON document and add the +same six typed columns. Everything else a rebuild needs is stored with the +reference, never re-derived from local configuration. `qbit-prism` has the +borrowed parts API (#296, merged) so builds and verification stop cloning the +window. This is the pattern `qbit_prism_audit_snapshots` already applies to +`AuditBundle.shares`. + +Status: design record for #264 (PR 1 of 3), part of #261 under #260, revised +after two adversarial reviews. The first confirmed the citations, both digest +definitions, the range predicate and the immutability trigger; the second +confirmed every earlier fix, that the CHECK accepts exactly three states and +the 131 citations it checked, and found one blocker, five should-fix items +and six nits, all resolved below. The operator has confirmed the shape +below; djh58's agreement (#273) is required before merge, and the one +question left for Anatolie (#283) is narrowed below. The parts API (PR 2) is +merged as #296. Code citations are at 3.x.x `41a2afd`, after #283 landed the +`ledger.rs` split (#300); `srv/` is `crates/qbit-prism-server/`. + +## Problem + +PostgreSQL rejects a JSONB container whose elements exceed 268,435,455 bytes +(measured). The production window from #254 is 372,257 shares, about 232 MB +canonical, about 679 bytes per share as JSONB (measured). On PostgreSQL 16.15 +with 372,000 production-shaped shares, one share array (216,972,363 bytes of +text, 583 B per share) fits; two arrays and three arrays fail with `total size +of jsonb object elements exceeds the maximum of 268435455 bytes`, and one +single-window insert into `qbit_prism_jobs` took 6.23 s and wrote 44,752,760 +bytes of WAL (measured). Every window-carrying document at the base holds more +than one copy: + +| Document | Copies | Where | +| --- | --- | --- | +| `StoredPrepared` (`srv/src/coordinator.rs:46`) | 3 | `snapshot.shares`, `bundle.shares`, `bundle.reward_manifest.shares`; written at `:641` | +| `Candidate` (`srv/src/ledger/candidates.rs:4`) | 2 | `bundle.shares`, `bundle.reward_manifest.shares`; written at `:182-183` | +| legacy audit import (`srv/src/ledger/migration.rs:117`) | 2 + 2 | the inline bundle as JSONB, and both copies again in `canonical_audit_bytes` (bytea) | +| landed audit body (`srv/src/ledger/blocks.rs:140`) | 1 | `reward_manifest.shares` survives `remove("shares")` | + +The claim phase writes no new window value. Of the 17 JSONB columns at the +base, the window reaches three: `qbit_prism_jobs.payload` at refresh, +`qbit_block_candidate_outbox.candidate` at enqueue, and +`qbit_pool_audit_bundles.audit_bundle` at landing and import. + +## `WindowRef` + +```rust +// qbit-prism-server; goes in `ledger/window.rs`, the module #283 landed (#300) +#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)] +pub struct WindowRef { + pub anchor_ms: i64, // Snapshot.anchor_ms, the predicate's cutoff + #[serde(with = "hex32")] // lowercase 64-hex in JSON, not an integer array + pub prior_balances_digest: [u8; 32], // qbit_prism::prior_balances_digest(&snapshot.prior_balances) + pub shares: Option, // None: no ledger share falls inside the difficulty window +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)] +pub struct ShareRange { + pub first_share_seq: u64, // inclusive, >= 1 + pub last_share_seq: u64, // inclusive, >= first_share_seq + pub share_count: u64, // rows matched by the window predicate, >= 1, <= last - first + 1 + #[serde(with = "hex32")] // lowercase 64-hex in JSON, not an integer array + pub snapshot_sha256: [u8; 32], // sha256(serde_json::to_vec(&shares)), same bytes as qbit_prism_audit_snapshots +} +``` + +A reference is built from a `Snapshot` (`srv/src/ledger/window.rs:10-16`): +`anchor_ms` is copied, the digests are computed as in [Digests](#digests), +and `shares` is `None` when `snapshot.shares` is empty, otherwise the +sequence numbers of its first and last share, `shares.len()` and the snapshot +digest. Rust holds the digests as `[u8; 32]`. A derived `Serialize` would +write each as a JSON array of 32 integers, so both digest fields carry +`#[serde(with = "hex32")]`, a small adapter module in `ledger/window.rs` +(the server's `hex = "0.4"` is declared without its `serde` feature, +`srv/Cargo.toml:18`). It serializes with `hex::encode`, which is lowercase, +and deserializes only a string of exactly 64 characters from `[0-9a-f]`, +rejecting uppercase, so the JSON form equals the column's CHECKed text byte +for byte and the payload/column comparison below is exact. Whichever of #265 +and #273 lands first adds the module, with a round-trip test and rejection +tests for uppercase, short, long and non-hex input. `share_count` is added to the issue's fields +because a range can contain rejected rows and rows stamped after the anchor, +so `last - first + 1` is not the count; it also turns a pruned range into a +cheap typed error before any hashing, as `srv/src/ledger/audit.rs:63-66` +already does for audit snapshots. `payout_revision` stays outside the +reference: the `qbit_prism_jobs.payout_revision` column for 008 +(`srv/migrations/002_multi_instance.sql:52`) and the `Candidate.payout_revision` +field for 007 (`srv/src/ledger/candidates.rs:8`); `read_window` returns the +current value for the caller's fence ([Revision fence and reorgs](#revision-fence-and-reorgs)). + +**An empty window is routine, not an edge case.** `refresh_once` +(`srv/src/coordinator.rs:468`) sets `bundle = None` when `snapshot.shares` is +empty (`:556-557`) and stores that `StoredPrepared` on every refresh +(`:627-641`, `:655-663`). A miner who resumes on that row (`:1498-1510`) or is +issued a job from it (`:1343-1356`) gets a bundle from `build_bundle` with +`Some(worker)`, which fabricates one synthetic share, `share_seq: 1`, +`share_id: "bootstrap-share"`, `job_id: "bootstrap-job"`, from the worker's +address and program, the anchor and the template's `curtime` (`:727-745`). +Landing already special-cases that window through `inline_shares` +(`srv/src/ledger/audit.rs:124-131`). `shares: None` is the empty window; the +bootstrap share is never in the ledger, so it is never in a range, and a +candidate found on such a job stores it inline +([Stored bundle inputs](#stored-bundle-inputs)). + +### Relation to `audit_body_ref` + +`crates/qbit-prism/src/lib.rs:80-81` glob-exports `audit_body_ref`, whose +private file-format types `AuditBodyRef` +(`crates/qbit-prism/src/audit_body_ref.rs:36-44`) and +`AuditWindowCompletenessProof` (`:58-68`) fix the vocabulary this record +reuses: + +| `audit_body_ref` | `WindowRef` | Note | +| --- | --- | --- | +| `first_share_seq`, `last_share_seq`, `share_count` | the same names in `ShareRange` | same meaning: inclusive bounds and the matched count | +| `share_slice_digest_hex` | not reused; `snapshot_sha256` instead | it hashes `CountedShare` fields (`crates/qbit-prism/src/lib.rs:2616-2634`), a build output (`:915`, carried at `:705`) that omits `network_difficulty`, `template_height`, `job_id` and `ntime`. A reference must authenticate the builder's *input* before the build runs, so `snapshot_sha256` is over `AcceptedShare` JSON, the bytes `qbit_prism_audit_snapshots` already stores | +| `bundle_without_shares`, a `Value` with the top-level `shares` removed | `AuditBundleBody`, typed | the shape landing already stores (`srv/src/ledger/blocks.rs:136-140`); it still embeds `reward_manifest.shares`, so the [caveat](#bundle-ownership) applies to the v2 file format too | + +### Columns (migration 007 and migration 008) + +Both migrations add the same columns, types and CHECK; 007 to +`qbit_block_candidate_outbox`, 008 to `qbit_prism_jobs`. + +| Column | Type | Meaning | +| --- | --- | --- | +| `window_anchor_ms` | `bigint` | `anchor_ms` | +| `window_prior_balances_sha256` | `text` | lowercase 64-hex | +| `window_first_share_seq` | `bigint` | `shares.first_share_seq` | +| `window_last_share_seq` | `bigint` | `shares.last_share_seq` | +| `window_share_count` | `bigint` | `shares.share_count` | +| `window_snapshot_sha256` | `text` | lowercase 64-hex, the `qbit_prism_audit_snapshots` convention (`srv/migrations/002_multi_instance.sql:115`) | + +Three states, and the CHECK accepts exactly these: + +| State | anchor and prior digest | four range columns | Rows | +| --- | --- | --- | --- | +| no window | both NULL | all NULL | per-worker `StoredJob` rows (`srv/src/coordinator.rs:58-67`); terminal outbox rows, whose `candidate` is already NULL (`crates/qbit-prism/sql/001_share_ledger.sql:97-104`) and whose window columns 007's terminal UPDATE also NULLs ([Immutability and retention](#immutability-and-retention)); every row written before 007/008 | +| empty window | both set | all NULL | a prepared row or candidate whose window has no ledger share | +| range | both set | all set, constrained | the common case | + +```sql +ALTER TABLE qbit_block_candidate_outbox -- 007; 008 repeats this on qbit_prism_jobs + ADD COLUMN IF NOT EXISTS window_anchor_ms bigint, + ADD COLUMN IF NOT EXISTS window_prior_balances_sha256 text, + ADD COLUMN IF NOT EXISTS window_first_share_seq bigint, + ADD COLUMN IF NOT EXISTS window_last_share_seq bigint, + ADD COLUMN IF NOT EXISTS window_share_count bigint, + ADD COLUMN IF NOT EXISTS window_snapshot_sha256 text; +ALTER TABLE qbit_block_candidate_outbox ADD CONSTRAINT qbit_block_candidate_outbox_window_check CHECK ( + -- no window + (num_nulls(window_anchor_ms, window_prior_balances_sha256) = 2 + AND num_nulls(window_first_share_seq, window_last_share_seq, + window_share_count, window_snapshot_sha256) = 4) + OR (num_nonnulls(window_anchor_ms, window_prior_balances_sha256) = 2 + AND window_prior_balances_sha256 ~ '^[0-9a-f]{64}$' + AND ( + -- empty window + num_nulls(window_first_share_seq, window_last_share_seq, + window_share_count, window_snapshot_sha256) = 4 + -- range + OR (num_nonnulls(window_first_share_seq, window_last_share_seq, + window_share_count, window_snapshot_sha256) = 4 + AND window_first_share_seq >= 1 + AND window_last_share_seq >= window_first_share_seq + AND window_share_count BETWEEN 1 AND window_last_share_seq - window_first_share_seq + 1 + AND window_snapshot_sha256 ~ '^[0-9a-f]{64}$')))); +``` + +- **Every state is distinguishable by two columns.** `window_anchor_ms IS + NULL` is "no window"; an anchor with `window_first_share_seq IS NULL` is + "empty window"; `window_first_share_seq IS NOT NULL` is "range". The + `num_nulls`/`num_nonnulls` guards matter because a CHECK that evaluates to + NULL passes; a plain conjunction would accept a row with only some columns + of a group set. +- **A legacy row cannot be confused with an empty window.** `ADD COLUMN` + leaves every pre-007/008 row all-NULL, and an empty window always carries + the anchor and the prior digest. So a `prepared:` row with a NULL anchor is + a pre-008 row, a cache miss ([Compatibility](#compatibility)); one with an + anchor and no range is a post-008 empty window. On the outbox a pending row + with a NULL anchor is a pre-007 row, which 007 refuses to leave behind. +- No new index; retention (D6) may add one on `window_first_share_seq`. + `ADD CONSTRAINT` has no `IF NOT EXISTS`, so each migration guards it to stay + re-runnable like 002 to 005. +- Not an FK to `qbit_prism_audit_snapshots`: that would tie job and candidate + lifetime to audit retention and to the bootstrap inline-share special case + (`srv/src/ledger/audit.rs:124-131`). The columns cost about 150 bytes of + payload per row. + +### Payload and columns + +The reference lives in both places on both tables, uniformly: + +- The row's small JSON document embeds `window: WindowRef`, digests as + lowercase hex through the `hex32` adapter ([`WindowRef`](#windowref)): the + `Candidate` for 007 (`srv/src/ledger/candidates.rs:4-14`), the + `StoredPrepared` for 008 (`srv/src/coordinator.rs:46-55`). For 007, + `candidate_sha256` keeps covering the whole document including the + reference, as it covers the candidate today + (`srv/src/ledger/candidates.rs:182-183` at enqueue, `:73-79` at claim). +- The six columns are a typed duplicate for the CHECK and the future + retention index, written in the same statement as the document. +- Decode reads the document, then compares it with the columns returned by + the same statement: the claim `UPDATE … RETURNING` + (`srv/src/ledger/candidates.rs:160`) and `Ledger::job` + (`srv/src/ledger/jobs.rs:34-38`) gain the six columns. Any disagreement, + including a document with a range and NULL range columns, is a decode + error, surfaced like corruption (`Decode` below). + +## `Ledger::read_window` + +```rust +pub struct Window { + pub shares: Vec, // ascending share_seq, exactly share_count rows; empty for shares: None + pub prior_balances: Vec, // by BalanceSource; digest-checked against the reference either way + pub payout_revision: i64, // the current revision, read in the same snapshot; claim's probe hint and B's eligibility input, never copied into a resumed Snapshot +} + +pub enum BalanceSource { + Current, // qbit_current_carry_forward_balances(): a claim of a candidate that isn't leased + AsIssued, // the qbit_prism_balance_snapshots row keyed by the reference's digest: a resume of published work, or a leased candidate's audit +} + +#[derive(Debug, thiserror::Error)] // the server crate already depends on thiserror 2 (srv/Cargo.toml) +pub enum WindowError { + #[error("window range incomplete: expected {expected} shares, read {got}")] + Incomplete { expected: u64, got: u64 }, // range pruned or missing, or a different predicate + #[error("prior balances changed since the reference was written")] + PriorBalancesChanged { expected: [u8; 32], actual: [u8; 32] }, // balances moved, not corrupt; the caller decides + #[error("as-issued balance snapshot missing")] + BalanceSnapshotMissing { digest: [u8; 32] }, // AsIssued only: pruned or never written; a failure, not a miss + #[error("window snapshot digest mismatch")] + SnapshotDigestMismatch { expected: [u8; 32], actual: [u8; 32] }, + #[error("window database error: {0}")] + Database(#[from] sqlx::Error), // includes SQLSTATE 57014, the statement timeout + #[error("window decode error: {0}")] + Decode(#[source] anyhow::Error), // share_from_row, hex, or payload/column disagreement: corruption +} + +impl Ledger { + pub async fn read_window( + &self, + window: &WindowRef, + balances: BalanceSource, + ) -> Result; +} +``` + +`WindowError` converts into `anyhow::Error` with `?` at the `anyhow`-based +call sites (`process_candidate_inner` and `resume_job` both return `anyhow` +results); callers match on the variant first, because the variant decides +the caller action ([Errors and callers](#errors-and-callers)). + +### Read + +`Ledger` has one pool, the primary (`srv/src/ledger.rs:43-46`); there is no +replica pool, and `read_window` must never be given one, because the balances +and the revision must be current for the fence. One transaction on that pool, +`SET TRANSACTION ISOLATION LEVEL REPEATABLE READ READ ONLY` as its first +statement, so the revision, the balances and every page of the range are one +consistent snapshot; a landing that commits mid-read cannot yield a revision +that disagrees with the balances. In order: + +1. `SELECT payout_revision FROM qbit_prism_cluster WHERE singleton`. +2. The balances, by `BalanceSource`. `Current` reads them through the + `read_prior_balances` statement `Ledger::snapshot` uses + (`srv/src/ledger/window.rs:244-252`), then checks the prior-balances digest, + else `PriorBalancesChanged`. `AsIssued` reads `SELECT balances FROM + qbit_prism_balance_snapshots WHERE prior_balances_digest = $1` instead. No + row is `BalanceSnapshotMissing`, and a row whose sorted set doesn't hash to + its key is `Decode`: stored rows are immutable, so they can only disagree + through corruption. `AsIssued` never returns `PriorBalancesChanged`. It costs microseconds, + so a reference whose balances moved never pays for the window. The digest + sorts internally (`prior_balances_digest`, `crates/qbit-prism/src/lib.rs`), + but `AuditBundle.prior_balances` serializes in vector order, and the query + has no outer `ORDER BY`. The function orders its own rows + (`crates/qbit-prism/sql/001_share_ledger.sql:1544`), but SQL doesn't + guarantee that order through a table expression. So `read_prior_balances` + itself sorts the vector with the digest's comparator: `order_key`, then + `recipient_id`, then `p2mr_program_hex`, compared as bytes. Refresh and every rebuild share that function, so both hand the builder the + same vector. The as-issued read sorts with the same comparator, and + `save_job` stores the set already sorted. An outer SQL `ORDER BY` alone wouldn't do, because it follows the + database collation, not the byte order the digest uses. A #265 unit test + requires any permutation of the same balances to give identical canonical + bytes after that sort. +3. If `window.shares` is `None`: commit and return `Window { shares: vec![], + prior_balances, payout_revision }` without touching `qbit_share_ledger`. +4. Otherwise an existence probe on `first_share_seq` and `last_share_seq`, + two primary-key lookups; a missing row is `Incomplete` in one round trip + before any page is read. +5. The range, in **ascending keyset pages of 4096**, mandatory, inside the + same transaction: + +```sql +-- SELECT_SHARE (srv/src/ledger.rs:40) with the predicate of read_range (srv/src/ledger/audit.rs:150) +SELECT share_seq,share_id,miner_id,payout_order_key,encode(p2mr_program,'hex') AS program, + share_difficulty::text AS difficulty,network_difficulty::text AS network_difficulty, + template_height,job_id,job_issued_at,accepted_at,ntime,credit_policy + FROM qbit_share_ledger + WHERE accepted AND share_seq > $cursor AND share_seq <= $last + AND accepted_at<=to_timestamp($anchor::double precision/1000) + AND job_issued_at<=to_timestamp($anchor::double precision/1000) + ORDER BY share_seq LIMIT 4096; -- $cursor starts at first - 1 and advances to each page's last share_seq +``` + + `Ledger::snapshot` is the precedent for keyset paging over this predicate + (`srv/src/ledger/window.rs:217-218`), not for the direction: it pages descending + from the cutoff and reverses at `:232` because it learns the start only + when the weight is spent. `read_window` knows both bounds, so it pages + ascending, rows arrive in canonical order and the digest streams. +6. Per page, `shares.len()` may not exceed `share_count` (more rows is also + `Incomplete`: the reference was built with a different predicate); after + the last page it must equal it. Then the snapshot digest, else + `SnapshotDigestMismatch`. + +`read_window` never returns a partial window. Every error leaves the caller +with no `Window` at all. Paging keeps the raw row buffer under about 2 MB, +gives each page its own statement timeout and lets the digest stream: the +hasher takes `[`, each share's compact JSON separated by `,`, then `]`, which +is byte-identical to `serde_json::to_vec(&shares)`, so no 233 to 260 MB +buffer exists. + +### Digests + +`snapshot_sha256` is SHA-256 over `serde_json::to_vec(&shares)`: a compact +JSON array of `AcceptedShare` values in ascending `share_seq`, fields in +declaration order (`crates/qbit-prism/src/lib.rs:184-199`), `credit_policy` +omitted when `None`, `u128` difficulties as JSON numbers. These are the bytes +`persist_audit_snapshot` hashes at `srv/src/ledger/audit.rs:120` and +`materialize_audit_row` checks at `:68`, so `window_snapshot_sha256` equals +the `qbit_prism_audit_snapshots.snapshot_sha256` landing later inserts for the +same window. Streaming the serializer into the +hasher, as above, produces the same bytes. + +**Two share digests, not interchangeable.** `snapshot_sha256` hashes the +native `AcceptedShare` serialization above. `PayoutWindow::canonical_digest_hex` +(`crates/qbit-prism/src/window.rs`), which #274 names, hashes a different, +Python-compatible canonical form with sorted keys, so the two differ for the +same shares; a compiled one-share comparison on #273 shows it. The reference +keeps `snapshot_sha256`, because it equals +`qbit_prism_audit_snapshots.snapshot_sha256` and landing's +`share_snapshot_sha256`, and existing audit hashes don't change. #274's +instruction to "use `canonical_digest_hex` for the slice digest instead of +`serde_json::to_vec(shares)`" is amended accordingly: its incremental window +must produce the native `snapshot_sha256`, or carry both. + +**Cost model for #274.** A flat SHA-256 can't drop shares from the front of a +sliding window. Every advance that expires old shares rehashes the whole +window, the 0.7 to 1.4 s per non-cached refresh that the cost table budgets. +Caching each share's serialized bytes removes the serialization, not the +hash. #274's gate, under 1 s wall at 400,000 shares with a one-share delta, +therefore needs a measured native encoding and hash path. If a flat rehash +misses it, the reference digest has to become chunked, for example SHA-256 +over the digests of fixed `share_seq` pages so only the edge pages rehash. +That is a versioned change to `WindowRef` that #274 owns, and the landed-audit +recovery check would then compute the flat digest separately. + +`prior_balances_digest` is the digest already carried as +`ledger_window_attestation.prior_balances_digest_hex` +(`crates/qbit-prism/src/lib.rs:706`), computed at build (`:1519`) and checked +at verify (`:2585`). Its definition, `prior_balances_digest_hex` +(`:2651-2672`): sort the balances by `(order_key, recipient_id, +p2mr_program_hex)`, then for each balance feed SHA-256 with `recipient_id`, +`order_key` and `p2mr_program_hex` as a big-endian `u64` length followed by the +UTF-8 bytes (`update_string`, `:2674-2677`), then `balance_sats` as a +big-endian `i128` (`update_i128`, `:2687-2689`). The function is private +today; the parts API exports it as +`prior_balances_digest(&[CarryForwardBalance]) -> [u8; 32]`, so the server +computes the reference and `read_window` checks it with the same code. + +### Threads, concurrency and deadlines + +**`read_window` owns its blocking hand-offs.** It is `async`. The runtime +thread issues the statements and receives each page as sqlx wire buffers; it +runs no serde, no hashing and no `share_from_row`. For each page it moves the +rows, the running `Vec` and the hasher into one +`tokio::task::spawn_blocking` task, which maps the rows (`share_from_row`, +`srv/src/ledger/window.rs:263-283`: two `u128` parses, five required `String`s plus +an optional sixth for `credit_policy`, two timestamps), updates the hash and +hands the state back; the balances digest runs the same way. **Callers must +not wrap `read_window` in `spawn_blocking`**; they await it on the runtime. +Dropping the future between pages rolls the transaction back and returns the +connection; an in-flight page task finishes its at most 4096 rows detached, +so the blocking work a drop can leave behind is one page, about 5 to 10 ms. +The rebuild that follows is the caller's blocking task, held in +`AbortOnDropHandle` as today (`srv/src/coordinator.rs:998`). + +**Concurrency.** `read_window` acquires nothing; each call holds one pool +connection with an open snapshot for the 2 to 5 s estimated below. Bundle +builds are bounded by `build_slots` (`srv/src/coordinator.rs:82`, created at +`:300` from `PRISM_JOB_BUILD_EXECUTOR_WORKERS`, default +`min(runtime_workers, 4)` and as low as 1, `srv/src/config.rs:365-370`; +acquired at `srv/src/coordinator.rs:712` and `:997`). The caller acquires its +`build_slots` permit **before** `read_window` and holds it across the read +plus the rebuild, so at most `build_workers` windows are materialized at +once. That bound alone doesn't protect the pool. +`PRISM_DATABASE_MAX_CONNECTIONS` defaults to 16 but may be as low as 4 +(`srv/src/config.rs:358-364`), and `build_workers` may exceed it, up to +`runtime_workers + 8` (`:365-370`). The pool is shared, with a 15 s acquire +timeout (`srv/src/ledger/connect.rs:25-27`), with share appends and the +candidate-lease heartbeat (`srv/src/coordinator.rs:887-907`). Four rebuilds +of distinct `storage_key`s could then hold all four connections for their +whole multi-page reads: an append or a heartbeat waits out the acquire +timeout, and a failed heartbeat drops recoverable claim work. So +`read_window` also takes a permit from a `window_reads` semaphore sized +`clamp(database_max_connections - 2, 1, build_workers)`. The caller acquires +it after its `build_slots` permit and releases it as soon as `read_window` +returns, before the rebuild, so at least two connections always stay free +for appends and heartbeats. Waiting for it counts against the caller's +deadline. A dedicated pool was rejected: it would add connections beyond the +budget operators size the database for. **Under that permit the rebuild calls the `build_audit_bundle_body*` +builders directly, or a `build_bundle` variant that takes the held permit, +never `build_bundle` itself**: `build_bundle` acquires its own permit +(`:712`), so with one worker a nested acquisition would wait on itself. The +claim path already calls the builders directly (`:997-1031`). Resume is per +reconnecting miner (`resume_job`, `:1447-1470`) and every miner of one job +reads the same `prepared:` row (`:1466`), so #273 coalesces concurrent +resumes of the same `storage_key` behind one in-flight rebuild, for example a +single-flight map next to the `Prepared` cache: later miners await the first +rebuild's `Arc` and `Arc` instead of reading the window again. The entry shares only the slim resumed form built from that pair (below) and, for an empty window, the as-issued balances each waiter's bootstrap build reads: +each waiter still runs its own checks on its own job (worker, target, +extranonce, version mask, absolute expiry and final authority), and +cancelling a waiter neither renews a job nor hands a bootstrap bundle to +another miner. A blocking build owns its `build_slots` permit until it +finishes: the permit moves into the `spawn_blocking` closure, because +cancelling the awaiting task or dropping `AbortOnDropHandle` +(`srv/src/coordinator.rs:998`) can't stop a closure that is already running. + +**A resumed job keeps no window.** `build_workers` bounds how many rebuilds +run at once, not how many rebuilt windows stay resident. A session keeps up +to 64 jobs per connection (`srv/src/stratum.rs:189`), so resumed jobs that +each held their `Arc` and `Arc` could pin about +0.5 GB apiece. Once the rebuild has produced the job's coinbase, the resumed +`Prepared` keeps everything submit and the candidate need except the window: +- the decoded `StoredPrepared` itself, whose stored inputs the candidate + persists: `payout_policy`, the whole nested `ctv`, `audit_builder_version`, + `signer_keys`, `coinbase_suffix`, `template_sha256`, the `WindowRef`, and the + snapshot scalars `anchor_ms`, `payout_revision` and `share_seq`, all O(1); +- the template and the fee; +- the body's non-window fields (`found_block` and the coinbase manifest); +- for an empty window, the miner's own `bootstrap_share`; +- the as-issued `prior_balances`, O(recipients), which a `leased` enqueue + writes back ([Immutability and retention](#immutability-and-retention)). + +The candidate takes its stored inputs from there, never from local +configuration ([Stored bundle inputs](#stored-bundle-inputs)). Submit reads +nothing that depends on the window (`srv/src/coordinator.rs:1621-1666`, +`:1695`), and the whole-bundle clone at `:1696` goes with #265. A resumed window never reaches async code. The rebuild's `spawn_blocking` +closure builds the coinbase and the slim resumed form, then drops +`Snapshot.shares` and the body's `reward_manifest.shares` inside the same +closure, so only the slim form crosses back, and the single-flight entry +shares only that. Freeing about 0.5 GB of vectors and their heap-backed +fields on a runtime worker would stall candidate-lease heartbeats and share +processing. The same rule covers every other release of a rebuilt window: +the claim's rebuilt `CandidateClaim` parts after landing, and a replaced +local `Prepared`, go to `spawn_blocking(move || drop(…))` instead of being +dropped on the runtime. Windows then exist only inside rebuilds, and the +`k × 0.5 GB` bound in the cost table holds however many resumed jobs sessions +keep. Locally issued jobs share their generation's `Prepared` (`:1392-1395`), +window included, for as long as Stratum retains them +(`srv/src/stratum.rs:639-641`), as today; bounding how many local +generations that keeps resident is an acceptance note on #273, not part of +this contract. Each generation holds its window once. `Prepared` and `JobContext` +(`srv/src/coordinator.rs:26-43`) keep `snapshot: Arc`, which owns +the only `Vec`, and replace `bundle: Arc` and +`Option>` (`:29`, `:35`) with `Arc` (#296). +For an empty window, `JobContext` also keeps `bootstrap_share: +Option`, the synthetic share its per-worker build fabricated +(`:727-745`, reached from `:1343-1350`). Today that share lives only in +`JobContext.bundle.shares`, and the body keeps just a `CountedShare`, which +lacks fields the stored share needs, so submit copies this one verbatim into +`Candidate.bootstrap_share`. +The body carries every field the coordinator reads from them (`found_block` +at `:1648` and `:1663`, the coinbase manifest) and no shares; the +whole-bundle clone into the candidate (`:1696`) goes with #265. An owned +`AuditBundle` beside the snapshot would need a second `Vec`, because +`into_bundle` takes it by value and two `Arc`s do not share storage: about +0.75 GB per 400k rebuild instead of 0.5 GB. Canonical bytes and verification +use `canonical_audit_bundle_bytes_from_parts` and `verify_audit_parts` over +`(&body, &snapshot.shares)`; no owned bundle is assembled, at claim either: first-time landing takes the same parts (below). For an empty window the entry carries `bundle: None` and the as-issued +`prior_balances` from `read_window`, O(recipients), until every waiter has +built its bundle, and each +resuming miner builds its own bootstrap bundle, because sharing one would put +miner A's bootstrap share in miner B's job. It does so under its own +`build_slots` permit, taken after the entry's permit is released, by calling +the builders directly: the synthetic share is the one `build_bundle` +fabricates from the worker today (`:727-745`, reached from `:1498-1510`), and +every other input comes from `StoredPrepared` (template, anchor, +`payout_policy`, `ctv`) and the issued `payout_revision` from `StoredPrepared`, plus the as-issued balances the entry carries. +It never calls `build_bundle` itself, which reads `config.ctv_enabled`, +`payout_policy`, `ctv_direct_floor` and `ctv_config` (`:755`, `:760-762`) and +would reintroduce the drift [Stored bundle inputs](#stored-bundle-inputs) +forbids. + +**First-time landing.** A claim whose block has no landed audit lands from +the same parts. `CandidateClaim` holds the rebuilt `Arc` and +the `Arc>` that `read_window` returned, never an assembled +`AuditBundle`. Before its transaction opens, `land_candidate_checked` +(`srv/src/ledger/blocks.rs:55-157`) does all of its whole-window work in one +`spawn_blocking` closure over those `Arc`s. That closure replaces the clone at +`:61`, and nothing serializes the window on the runtime: + +- `verify_audit_parts` produces the report (`:63-67`); +- `serde_json::value::to_raw_value(&body)` gives the stored body as a + `Box`, so the bind at `:141-142` copies bytes instead of + serializing. The body has no top-level `shares`, so the whole-bundle + `to_value` and `remove("shares")` at `:136-140` go. Until #267 normalizes + `reward_manifest.shares` out of the stored body, it is still the largest + value landing writes, as today, and the bind's copy of it into SQLx's + argument buffer is the one whole-body step left on the runtime: a single + contiguous copy with no per-share work; +- `audit_body_byte_len` comes from `write_canonical_audit_bundle_from_parts` + into a counting writer instead of the full canonical bytes at `:143`; +- the commitment and witness leaves and the payout accounts (`:145-147`) are + serialized there too; +- the share snapshot's ordering check and digest + (`srv/src/ledger/audit.rs:115-120`) run there, and for a non-empty window + the digest must equal the reference's `snapshot_sha256`. + +The transaction binds only those values. `persist_audit_snapshot` takes the +digest and the reference's range instead of `&AuditBundle`; a bootstrap +bundle still writes `inline_shares` from the stored `bootstrap_share` +(`:124-131`). It keeps its fence, the re-read of the range under exact +equality (`:132-138`), but not its reader: `read_range` (`:144-152`) is one +unpaged `fetch_all` that maps every row and compares the whole window on the +runtime. The re-read uses the paged reader `read_window` uses, inside the +same transaction, and compares each page with its slice of the claim's +shares, so runtime work stays per page, as in `read_window`. None of this +waits for #267: the parts entry points are #296's. + +**Deadlines.** `read_window` takes no deadline of its own. Each statement, +so each page, runs under the connection's `statement_timeout`, 15 s by +default (`srv/src/ledger/connect.rs:23-33`) and overridable through +`PRISM_DATABASE_STATEMENT_TIMEOUT_MS` up to 600000 ms (`:11-24`); a timeout +surfaces as `Database` with SQLSTATE 57014, and pool acquisition adds its own +15 s (`:27`). The worst case is bounded by `pages × statement_timeout` plus +page CPU, about 98 pages at 400k; the expected total is 2 to 5 s, which the +#264 harness must measure before #265 and #273 rely on it. The claim puts +one `tokio::time::timeout` around `read_window` plus the rebuild; a resume +relies on its caller's existing end-to-end deadline: + +| Caller | Deadline | On expiry | +| --- | --- | --- | +| claim (#265) | **60 s** around `read_window` plus the rebuild only, the work this record adds. The steps after it keep their existing deadlines: observation, landing, lease renewal and `submitblock`, whose RPC timeout `PRISM_BLOCK_SUBMIT_RPC_TIMEOUT_SECONDS` may be set up to 86,400 s (`srv/src/config.rs:428`, bounded at `:138-145`). A 60 s bound around them would cancel an in-flight submission and reschedule it as a reconstruction timeout. The candidate lease is not a deadline: its heartbeat renews it every 30 s for as long as processing runs (`srv/src/coordinator.rs:887-907`), and only a failed renewal drops the work (`:950-963`). Against the estimate, 60 s is twelve times the 5 s upper bound and four statement timeouts at the 15 s default, so a rebuild that needs longer is a harness finding, not a reason to raise it | fail the attempt through `retry_candidate` (`srv/src/ledger/candidates.rs:114`) with an alert; the lease was renewed within the last 30 s of a 120 s term, so the `claim_expires_at > clock_timestamp()` condition holds and the row is rescheduled after `LEAST(60, attempt_count)` seconds (`:118`), never abandoned | +| resume (#273) | **one end-to-end deadline, the caller's existing one**: `resume_job`'s only caller already wraps it in `timeout(initial_job_timeout_seconds, …)` and maps expiry to the backend error "job resume timed out" (`srv/src/stratum.rs:1227-1232`). That deadline covers admission (the `build_slots` and `window_reads` permits), the wait on the single-flight entry, `read_window`, the rebuild and any bootstrap build. There is no inner timeout: a shorter one would reject work that completes inside today's allowance (30 s by default, `srv/src/config.rs:195`), which would be a policy change, not parity | the backend error "job resume timed out", as today; never `unknown-job` | + +### Errors and callers + +| Outcome | Meaning | Claim (#265) | Resume (#273) | Landing (#265, #267) | Import (#265) | +| --- | --- | --- | --- | --- | --- | +| stored `audit_builder_version` or `signer_keys` differ from this binary's (caller check before `read_window`, not a variant) | a builder or signing keys this binary does not have built the reference; the upgrade and rotation refusals ([Builder version](#builder-version), [Signing keys](#signing-keys)) keep a drained deployment from reaching it | `retry_candidate` (`srv/src/ledger/candidates.rs:114`) with an alert naming both values; never abandon, never rebuild with the current builder or keys | cache miss: `Ok(None)`; the next refresh writes a job at the current version | not a caller; landing uses the claim's parts | n/a | +| `Window.payout_revision != row revision` (caller check, not a variant) | the revision moved since the reference was written: a landing, a reorg, a resettlement, or the pool's own block reaching the tip | a hint only, like the cached tip and revision at `srv/src/coordinator.rs:972-973`: run `observe_candidate` (`:978`) and finish through `finish_candidate_at_revision` (`:979-988`) only when the block is not active and the revision or parent changed; an active block continues and lands at the observed revision (`:1039-1046`); #289 owns old-epoch candidates | not a miss by itself: B's published-work check decides eligibility, and eligible work rebuilds as issued ([Revision fence and reorgs](#revision-fence-and-reorgs)) | not a caller; landing keeps its own fence (`srv/src/ledger/blocks.rs:124-127`) | n/a | +| `PriorBalancesChanged` | the balances moved: the candidate is superseded, or it is the pool's own block, already landed | not reached when `qbit_pool_audit_bundles` already holds the block's audit, because the claim then treats landing as done, from the landed row's columns authenticated against the block's coinbase, without `read_window`, and continues to observe, renew and submit ([Revision fence and reorgs](#revision-fence-and-reorgs)); otherwise run `observe_candidate`: not active and changed is `finish_candidate_at_revision`; active is `retry_candidate` with an alert, the outcome today's `prior == bundle.prior_balances` failure has (`srv/src/ledger/blocks.rs:128-132`, reaching `submit_loop`'s retry at `srv/src/coordinator.rs:1123-1131`) | not reached for published work, which reads its as-issued balance set, not the current one ([Revision fence and reorgs](#revision-fence-and-reorgs)) | as above | n/a | +| `Incomplete` | rows pruned or missing; D6 violated, or a wrong predicate | fail the attempt through `retry_candidate` (`srv/src/ledger/candidates.rs:114`) with the error in `last_error`; never abandon automatically, #268 owns recovery | a failure, not a miss: return `Err`, which reaches the miner as a backend error through the existing path (`srv/src/stratum.rs:1227-1232`), with an alert. `unknown-job` (`:1251-1255`) stays for work that is absent, expired or deliberately incompatible | as above | the legacy window is not in the ledger: keep the bytes-only import | +| `BalanceSnapshotMissing` | `AsIssued` only: the job's balance snapshot was pruned or never written | n/a: claims read `Current` | a failure, not a miss: `Err` through the backend-error path, with an alert, as for `Incomplete` | n/a | n/a | +| `SnapshotDigestMismatch`, `Decode` | corruption, a reference built from different bytes, or payload/column disagreement | same as `Incomplete`, with an alert | same as `Incomplete` | as above | same as `Incomplete` | +| `Database` (incl. 57014) | transient | propagate: `submit_loop` hands the error to `retry_candidate` (`srv/src/coordinator.rs:1123-1131`), which releases the claim and reschedules the row after `LEAST(60, attempt_count)` seconds (`srv/src/ledger/candidates.rs:118`); lease expiry recovers the row only if that write itself fails | propagate; the reconnect fails and retries | as above | propagate | +| landing equality failure after a successful `read_window` (`srv/src/ledger/audit.rs:132-138`) | the landing transaction read a different range than the claim did, which immutability forbids | like `SnapshotDigestMismatch`: `retry_candidate` with an alert, never abandon; the error already reaches `submit_loop`'s retry path (`srv/src/coordinator.rs:1123-1131`) | n/a | the check stays | n/a | +| caller deadline expired | `read_window` plus the rebuild outran the deadline in the table above | 60 s: `retry_candidate` with an alert | the caller's own `initial_job_timeout_seconds`: the backend error "job resume timed out" (`srv/src/stratum.rs:1232`), as today | n/a | n/a | +| empty window (`shares: None`) | not an error | rebuild from `vec![bootstrap_share]` | rebuild the bootstrap bundle per miner with the builders directly, from the stored policy inputs, never `build_bundle` | lands through `inline_shares` (`srv/src/ledger/audit.rs:124-131`), unchanged | n/a | + +## Revision fence and reorgs + +Share rows survive a reorg: the immutability trigger below forbids UPDATE and +DELETE, so the range is always rebuildable and `Incomplete` never means "the +chain moved". What moves is the revision and, sometimes, the balances, and +neither is a supersession test on its own. `observe_chain_view` bumps +`payout_revision` on every tip with more work, the pool's own block included +(`srv/src/ledger/window.rs:66-68`); an ambiguous `submitblock` timeout makes that +routine, because the attempt fails with "block submission outcome +unresolved" (`srv/src/coordinator.rs:1085-1092`, `:1107`) while the block can +still reach the tip, and the timed-out request is never resent +(`srv/tests/rpc_deadlines.rs:196-222`). A rule of "any revision inequality is +superseded" would therefore abandon the pool's accepted blocks. The digest +alone cannot detect a reorg either: a landing that a reorg undoes can return +the balances to their earlier values. So each caller applies its own rule, +and `read_window` reads the revision in the same snapshot as the balances so +that the comparison and the digest describe one moment: + +- **Claim (#265) keeps today's rule; `read_window` never decides + supersession.** The candidate is finished only when `observe_candidate` + reports it **not active** and the revision or the parent tip changed + (`srv/src/coordinator.rs:975-979`), the outcome being + `finish_candidate_at_revision` (`:979-988`); the comment at `:975-977` + says why: an already-active block still needs its audit recovered. An + **active** candidate is rebuilt from `read_window` and lands at the + observed, newer revision with the digest-checked balances, as `:1039-1046` + does today. The guard is `active_candidate_can_land_at_proven_new_chain_revision` + (`srv/tests/ledger_postgres.rs:505-539`): it bumps the revision after the + claim, shows that landing at the candidate's revision fails, and lands at + the observed one. If `qbit_pool_audit_bundles` **already holds the block's + audit**, because an earlier claim landed it and lost its lease after + landing (`srv/src/coordinator.rs:1041-1043`, `:1067-1069`) or a reconcile + has confirmed it, the balances have legitimately moved. The claim then skips + `read_window` and treats landing as done, once the landed row's columns are + authenticated against the block the candidate found instead of a stored + digest. The candidate keeps no + expected `audit_bundle_sha256`: computing one at submit would serialize the + whole bundle on the share path, which [Stored bundle inputs](#stored-bundle-inputs) + rules out. The block is the expectation, and every check is O(1): + + - the landed `coinbase_tx_hex` equals the coinbase in the candidate's + `block_hex`, parsed with the segwit-aware parser that + `codec::witness_merkle_leaves_from_block` uses; + - `audit_commitment_root_hex` (`crates/qbit-prism/src/lib.rs:975`) over the + landed `audit_commitment_leaves_hex` equals that coinbase's witness + reserved value, which the builder sets to the audit commitment root + (`:1711-1724`). The single leaf commits to the reward manifest and the + payout policy manifest (`:960`); + - `share_snapshot_sha256` equals `window_snapshot_sha256`. For an empty + window that column is NULL under the three-state CHECK, so the comparison + is against `sha256(serde_json::to_vec(&[bootstrap_share]))`, computed from + the stored `bootstrap_share`: the digest `persist_audit_snapshot` writes + for inline shares; + - `found_block_bits` matches the header, as today's landing checks for an + existing audit (`srv/src/ledger/blocks.rs:95-117`). + + Only then does the claim treat landing as done. It skips the rebuild and + `land_candidate` and continues through the rest of today's flow, never + straight to finishing. A landed audit doesn't mean a submitted block: an + inactive block lands (`srv/src/coordinator.rs:1067-1069`) before the lease + renewal and `submitblock` (`:1084-1092`), so an earlier claim can land and + lose its lease before it submits. The claim observes the block (`:1039`, + `:1070`) and finishes it only where today's code does: an active block + (`:1040-1048`), or an inactive one whose revision or parent moved + (`:1050-1063`, `:1071-1080`). Otherwise it renews the lease and calls + `submitblock` (`:1084-1092`), as today. It never calls `materialize_audit_row` + (`srv/src/ledger/audit.rs:42-87`) on the submit loop. That function reads the + whole range in one unpaged query, serializes and clones it on the runtime, + and builds an owned `AuditBundle`: at 400k, two share arrays that the + cooperative 60 s deadline cannot interrupt. If #265 ever needs the landed body + on this path, it needs #267's parts-based, paged reader under + `spawn_blocking` first, which makes #267 a prerequisite for that path. + + `PriorBalancesChanged` on an active candidate with no landed audit has the + outcome today's landing check has when + `prior == bundle.prior_balances` fails (`srv/src/ledger/blocks.rs:128-132`): + the error propagates to `submit_loop`, which logs "candidate remains + recoverable" and calls `retry_candidate` (`srv/src/coordinator.rs:1123-1131`); + the attempt fails with an alert and is never abandoned. +- **Resume (#273) keeps published work's authority; revision equality no + longer decides it.** Today `srv/src/coordinator.rs:1483-1488` requires the + prepared row's revision to equal the current one. B8 (djh58, #273) keeps + work published at revision R0 eligible during a bounded replacement lease + after the ledger reaches R1, and this record must not undo that: + 1. B's published-identity and lease check decides whether the stored job + is still eligible. Only the published job gains the lease, and only + inside it; an arbitrary older job at the same parent never does. + Ineligible work is a cache miss, `Ok(None)`. **Interim + constraint:** until B8 and #289 define how an active leased block lands + against R0's balances, the lease covers only revision changes that + leave `prior_balances_digest` unchanged, and a balance change ends it at + once. An active leased block then carries the current balances and lands + through today's path. + 2. Eligible work is rebuilt with its **as-issued** economics: the balances + it was published with, not the current ones. `read_window(…, BalanceSource::AsIssued)` does this + ([`Ledger::read_window`](#ledgerread_window)): it takes the balances from + the job's balance snapshot (below) instead of + `qbit_current_carry_forward_balances()`, and checks them against the + reference's `prior_balances_digest`. It still reads the current revision in the same transaction, for the + caller's own fence, but that revision no longer has to equal the job's. + The rebuilt `Snapshot` keeps the issued revision, + `StoredPrepared.payout_revision`, never `Window.payout_revision`. The job + wire and any candidate take their revision from it, so a block found on + R0 work carries R0. `observe_candidate` then sees the change and takes the + superseded path instead of retrying `PriorBalancesChanged`. + 3. Submission honours the same lease. Today submit rejects work when the + frontend's current work or the job's revision differs from the ledger's + (`srv/src/coordinator.rs:1632-1644`), apart from parent-stale work + inside the stale grace. That grace work is credited as a share + (`:1669`), but a candidate is enqueued only when the block passes and the + work isn't stale (`:1672`), so its found block is dropped. B8's lease + therefore decides those checks at submit exactly as it decides + eligibility at resume: work the lease covers is not stale because the + revision changed, and a block found on it is enqueued as a candidate + carrying R0, with its as-issued window and inputs. Work outside the + lease keeps today's handling. + 4. A leased candidate is submitted before any terminal disposition. The + candidate records that the lease covered it, as a `leased` flag that + `candidate_sha256` covers. Otherwise the claim's pre-submit supersession + checks would finish it before `submitblock` and discard the block: the + cached-hint path (`srv/src/coordinator.rs:978-988`), the post-rebuild + check (`:1050-1063`) and the post-land check (`:1071-1080`). So would the + pre-submit landing (`:1067-1069`), which checks R0's balances against + R1. A leased candidate skips all four and submits before any + rebuild. `submitblock` needs only the stored block (`block_bytes`), so the + claim calls neither `read_window` nor the builders first; a + `BalanceSource::Current` read would return `PriorBalancesChanged` + against the candidate's R0 digest and retry forever. It renews its lease, + calls `submitblock` (`:1084-1092`), and only then meets + `observe_candidate` (`:1095`). Whether or not the block is active, the claim then lands its audit + before any terminal disposition. It rebuilds with + `BalanceSource::AsIssued`, skipped when the audit has already landed as + for any claim, and lands through today's path, + `land_candidate_at_revision` at the observed revision + (`srv/src/coordinator.rs:1041-1043`). The interim constraint in step 1 + allows that, because the balances equal the current ones. This is + today's order with the RPC moved first: today an inactive block lands + before `submitblock` (`:1065-1069`), so a late acceptance stays + reconcilable (`:1052-1054`). Only then does the claim take today's + outcome (`:1095-1107`). An active block finishes as submitted. On an + inactive block, a string result, `duplicate` included, finishes it with + the node's reason. A null result, which can still describe a known + side-chain block (`:1093-1094`), leaves it pending through + `retry_candidate` (`:1123-1131`). A retry resubmits the stored block with + its audit already landed, so no answer from the node can leave the block + without payout records. A balance change after enqueue is + the case every candidate has today: `retry_candidate` with an alert, + never abandoned. A rule for landing across a balance change is B8's and + #289's to define before the lease may bridge one. Every other candidate keeps today's + fences. + + **As-issued balances.** Once R1 lands, the current view can't return R0's + balances. Rebuilding them from `qbit_payout_carry_forward` + (`crates/qbit-prism/sql/001_share_ledger.sql:130-147`) would mean replaying + which blocks were active at R0, and reorgs change that. So `save_job` + stores the balances a job was built with in an immutable + `qbit_prism_balance_snapshots` row keyed by `prior_balances_digest`, in the + same transaction as the job row and only if the key is new (`ON CONFLICT DO + NOTHING`). A revision that leaves the balances unchanged writes nothing. + The row grows with the number of recipients with a balance, not with the + window. In as-issued mode a missing row is `BalanceSnapshotMissing`, a failure, not a miss. Retention deletes a row once no job row and no non-terminal + `leased` outbox row carries its digest + ([Immutability and retention](#immutability-and-retention)); other claims + read `Current`. +- **Landing** keeps its own fences unchanged: `require_revision`, the + `payout_revision` equality and `prior == bundle.prior_balances` + (`srv/src/ledger/blocks.rs:91-94,120-132`), and `persist_audit_snapshot`'s + re-read of the range under exact equality (`srv/src/ledger/audit.rs:132-138`), + paged rather than today's single `fetch_all` + ([Threads, concurrency and deadlines](#threads-concurrency-and-deadlines)). + `read_window` authenticates a rebuild for the caller; it does not replace + the landing transaction's checks. + +## Immutability and retention + +A `WindowRef` is valid only while its rows exist unchanged. Today that holds +because `qbit_share_ledger` rows are immutable (the statement-level trigger at +`srv/migrations/002_multi_instance.sql:127-135` raises on UPDATE, DELETE and +TRUNCATE, so no pruning exists), `share_seq` is assigned by the insert under +`ORDER_LOCK` (`srv/src/ledger/window.rs:99,169`) so no row can later appear inside +a written range, and the anchor predicate is stable because `accepted_at` and +`job_issued_at` never change and the ledger clock only moves forward (`:164`). + +`finish_candidate_at_revision` sets `candidate=NULL` in its terminal UPDATE +(`srv/src/ledger/blocks.rs:214`); after 007 the same UPDATE also sets the six +window columns NULL, so a terminal row is "no window" and the floor below +considers non-terminal rows only. It must also NULL `block_bytes`, the +`bytea` column `block_hex` moves to ([Stored bundle inputs](#stored-bundle-inputs)). +Before 007 the block left the row with `candidate`; without this, the outbox +would keep every submitted or abandoned block forever. `block_hash` stays for +duplicate detection. + +This extends the Python-era invariant "audit bodies and share segments remain +digest-checked and reconstructable" ([Invariants](invariants.md), Audit +artifacts; the contract is [A1 audit artifacts](a1-audit-artifacts.md)) to +native rows. Proposed wording for the native server: *a share window +referenced by any live row is reconstructable: `qbit_share_ledger` rows are +immutable, and no retention job removes a row at or above the smallest +`window_first_share_seq` of any non-terminal outbox row, any job row still present, expired or not, or any `qbit_prism_audit_snapshots` row without `inline_shares`.* + +D6 (#260) must honour that floor. A future prune runs in one transaction that +takes `SETTLEMENT_LOCK` and then `ORDER_LOCK`, the order every existing +dual-lock site uses (`srv/src/ledger/connect.rs:52-53`, +`srv/src/ledger/window.rs:196-197`, `srv/src/ledger/blocks.rs:181-182,248-249`), +so it cannot form an advisory-lock cycle with a refresh snapshot or a +candidate finalization. Those are the locks under which references are written +(`save_job` under `SETTLEMENT_LOCK`, `srv/src/ledger/jobs.rs:14`; enqueue under +`ORDER_LOCK`, `srv/src/ledger/window.rs:99` and +`srv/src/ledger/candidates.rs:29`). The +prune computes the floor, and deletes strictly below it. It must also keep enough history for any window a later `Ledger::snapshot` +can select, and a horizon computed from today's difficulty isn't enough: a +rise after the prune moves the next window's start earlier +(`srv/src/ledger/window.rs:190-232`), into rows already deleted. `snapshot` +wouldn't fail then. Its scan stops when the requested weight is reached or no +rows remain (`:216-231`), so it would publish an underweight window that +silently drops eligible shares. So D6 needs both of these: + +- **A ratcheted horizon.** The prune keeps every row the window would need at + a durable maximum difficulty, stored in `qbit_prism_cluster`, that rises + with every higher difficulty observed and falls only when an operator + lowers it, times a stated safety factor. +- **An explicit failure.** The prune records the `share_seq` it deleted below + in `qbit_prism_cluster`. `snapshot` returns an error instead of a window + when its scan reaches that recorded floor before the requested weight, so + the refresh publishes no work rather than short payouts. A pool whose + history was never pruned still gets the short window it legitimately has. + +A `read_window` that began before a prune commits still sees its snapshot; one that begins after it fails +with `Incomplete` at the existence probe. + +**In-flight windows.** The floor covers written references only. +`Ledger::snapshot` fixes its cutoff and releases both locks when its first +transaction commits (`srv/src/ledger/window.rs:195-208`). The page scan +(`:212-233`), the digest, the bundle and `save_job` +(`srv/src/coordinator.rs:560-668`) all come after that, while no row +references the window yet. Miners get the work only after `save_job` commits +(`:686-705`), so the gap ends there. A prune inside the gap, after a fall in +difficulty shrinks the next selectable window, could delete rows of the +window being built, and `save_job` would then publish a reference to rows +that are gone. So D6 adds a reservation: + +- `Ledger::snapshot`'s first transaction, still under both locks, inserts a + reservation row holding its cutoff and an expiry longer than any refresh + can take. The window start isn't known yet, so the row's `first_share_seq` + stays NULL until the scan ends and the refresh fills it in. +- `save_job` deletes the reservation in its own transaction, where the job + row's reference takes over. It publishes nothing, a cache miss, if the + reservation has expired or the window's `first_share_seq` row is gone, + checked with the existence probe `read_window` uses; a prune only removes a + prefix, so that row's presence means the whole range is present. +- The prune, holding both locks, skips its run while any unexpired + reservation still has a NULL `first_share_seq`, and otherwise lowers its + floor to the smallest reserved `first_share_seq`. + +An age-based horizon was rejected: how far back in time a window reaches +depends on the pool's hashrate, so no fixed age is safe. Until D6 lands nothing is pruned, and none of this is needed. + +**Template and balance-snapshot rows.** These are pruned today, not by D6. +`prune_expired_jobs` (`srv/src/ledger/jobs.rs:43-45`) is one unlocked +`DELETE` of up to 4,096 expired job rows. Templates change on every new +transaction set and at every reanchor (60 s by default, +`srv/src/config.rs:433`), so each distinct template becomes a new, possibly +multi-megabyte row. Without cleanup the table would grow for as long as the +frontend runs. WAL volume is no higher than today's, when every `save_job` +writes the template inside its payload, but storage would never shrink. + +So #273 turns the prune into one transaction under `SETTLEMENT_LOCK` and +then `ORDER_LOCK`, the established order. The first is the lock `save_job` +holds while it inserts the template, the balance snapshot and the job row; +the second is the lock enqueue holds while it writes a `leased` candidate +that refers to a balance snapshot. The prune runs three statements: + +1. It deletes a batch of job rows expired for longer than the grace below + and returns the template digests that batch carried. +2. It deletes the `qbit_prism_templates` rows among those digests that no + remaining job row references. +3. It deletes every `qbit_prism_balance_snapshots` row that no remaining job + row and no non-terminal `leased` outbox row references, not only those the + batch carried. A snapshot kept by a `leased` candidate has no job row left + to name it once its job expires, so a batch-scoped statement would never + look at it again after the candidate turns terminal. The scan stays small: + the table holds only snapshots that live jobs or pending `leased` + candidates reference, plus the ones this statement deletes. + +A single statement with a data-modifying CTE wouldn't do, because its outer +`NOT EXISTS` still sees the rows the CTE deletes. The lock keeps a prune from +deleting a row between a `save_job`'s insert-or-reuse and its job insert, and +a later `save_job` just inserts a deleted row again. Indexes on the job row's `template_sha256` and `window_prior_balances_sha256`, and +on the outbox's `window_prior_balances_sha256`, keep the checks cheap. + +**Enqueue re-establishes what the candidate references.** The locks order a +prune and an enqueue but don't decide which runs first. A submission can pass +its job's expiry check and then lose to a prune that deletes the job row and +its balance snapshot before enqueue takes `ORDER_LOCK`; today +`persist_candidate` (`srv/src/ledger/candidates.rs:169-200`) checks neither. +So the enqueue transaction, under `ORDER_LOCK` and before it writes the +candidate: + +- re-inserts a `leased` candidate's balance snapshot, `ON CONFLICT DO + NOTHING`, from the as-issued `prior_balances` its job carries + (`Snapshot.prior_balances`, `srv/src/ledger/window.rs:15`, or the slim + resumed form), after checking they hash to `prior_balances_digest`, so a + prune that ran first costs nothing; +- probes a non-empty window's `first_share_seq` row with the existence probe + `read_window` uses. A prune only removes a prefix, so the row's presence + means the whole range is there, and the committed outbox row then holds + the floor above. + +Shares can't be written back, so a failed probe is prevented rather than +repaired: an expired job row holds the floor until the prune deletes it, and +the prune deletes a job row only once it has been expired for a grace that +#273 sets well above the share path's latency. If the probe fails anyway, +enqueue still publishes the candidate, because the block must reach the +node, and raises an alert; its claim then meets `Incomplete`, which #268 +recovers. + +## Stored bundle inputs + +**Invariant: a rebuild reads no local configuration for any field that +reaches the signed bundle.** Today's claim rebuild takes `found_block`, +`prior_balances`, `payout_policy`, `ctv_fanout_fee_policy` and +`witness_merkle_leaves_hex` from the stored bundle but `config.ctv_direct_floor` +and `config.ctv_config` from local configuration +(`srv/src/coordinator.rs:1004-1027`), and `config.ctv_enabled` chooses the +builder (`:1003`). The fee policy is revalidated on every refresh because +relay floors change (`:525-527`). The cluster fingerprint +(`srv/src/config.rs:479-491`, pinned once in `qbit_prism_cluster` by +`configure`, `srv/src/ledger/connect.rs:128-140`) keeps live instances on one policy, +so the drift is across time, a fingerprint reset between the refresh that +built the row and its claim, not across instances. If a claiming frontend +re-derives any signed input differently, the canonical bundle changes: +`srv/src/ledger/blocks.rs:101-105` raises "existing block audit differs from +candidate", or a non-matching body lands. #265 removes the drift by storing +every builder input except the window: + +| Builder input (`crates/qbit-prism/src/lib.rs:1753-1764`) | At refresh (`srv/src/coordinator.rs`) | At claim today | With `WindowRef` | Size | +| --- | --- | --- | --- | --- | +| `shares` | `snapshot.shares` (`srv/src/coordinator.rs:747`) or the bootstrap share (`:727-745`) | `source.shares` | `read_window`, or `vec![bootstrap_share]` | the window, never stored | +| `found_block` | from the template (`:719-726`) | `source.found_block` | **stored verbatim** in the candidate JSON | 4 scalars | +| `prior_balances` | `snapshot.prior_balances` (`:759`) | `source.prior_balances` | `read_window`: the current set for a claim, the as-issued set from `qbit_prism_balance_snapshots` for a resume of published work or a leased candidate's audit ([Revision fence and reorgs](#revision-fence-and-reorgs)); digest-checked and sorted with the digest's comparator ([Read](#read)) | per recipient with a balance; stored once per distinct set, keyed by its digest, never in the job or candidate row | +| `payout_policy` | `config.payout_policy` (`:760`) | `source.payout_policy` | **stored verbatim** | O(1) | +| `direct_floor_sats`, `settlement_config`, `ctv_fanout_fee_policy` | `config.ctv_direct_floor`, `config.ctv_config`, `fee` (`:761-763`) | `config.*` (`:1009-1010`), the drift, and `source.ctv_fanout_fee_policy` | **stored verbatim** as one nested `ctv: Option<{ direct_floor_sats, settlement_config, fanout_fee_policy }>`, whose presence also selects the CTV builder in place of `config.ctv_enabled` (`:755`, `:1003`) | O(1) | +| `coinbase_script_sig_suffix_hex` | `Some(suffix)` (`:764`) | `claim.candidate.coinbase_suffix_hex` (`:992`) | already stored, **required after 007**: submit always writes `Some` (`:1697`); the `Option` with `serde(default)` (`srv/src/ledger/candidates.rs:10-11`) exists for pre-007 rows, so post-007 decode rejects `None`, and the `else` branch that lands the stored bundle when the suffix is absent (`srv/src/coordinator.rs:1036-1038`) is removed with the bundle | O(1) | +| `witness_merkle_leaves_hex` | `codec::witness_merkle_leaves_hex` over the template's transactions (`:749-750`) | `source.witness_merkle_leaves_hex` | **re-derived** from the stored `block_hex` by `codec::witness_merkle_leaves_from_block`, below | not stored twice | +| signing keys | `config.manifest_seed`, `config.ledger_seed` (`:751-752`) | `config.*` (`:1001-1002`) | local: the seeds are the signer, not a field; both public keys are **stored** as `signer_keys` and checked ([Signing keys](#signing-keys)) | about 130 B | +| builder logic | the linked `qbit-prism` | the linked `qbit-prism`, possibly a newer release | **stored** as `audit_builder_version` and checked, never re-derived ([Builder version](#builder-version)) | 2 B | + +`audit_commitment_leaves_hex` is a builder output, not an input. The +`bootstrap_share: Option` that a candidate found on an empty +window carries (about 600 B) follows the same rule: its fields come from the +worker's identity and the template the issuing frontend saw, which the +claiming frontend does not have, and it sits in `shares` and +`reward_manifest.shares`, so re-deriving it is exactly the drift the invariant +forbids. Submit sets it when `context.prepared.bundle.is_none()` +(`srv/src/coordinator.rs:27`; the `None` arm at `:1343` is where the +per-worker bootstrap bundle was built). `deferred_share` +(`srv/src/ledger/candidates.rs:12-13`) is the precedent for one inline share. Decode +requires `bootstrap_share.is_some() == window.shares.is_none()` and +`found_block.anchor_job_issued_at_ms == window.anchor_ms`, because +`persist_audit_snapshot` takes the anchor from the bundle +(`srv/src/ledger/audit.rs:123`) while `read_window` takes it from the +reference. + +**Witness leaves.** No server code splits a block into transactions today; +the only derivation is from the template (`srv/src/codec.rs:253-275`, used at +`srv/src/coordinator.rs:749-750`). Re-deriving from `block_hex` is sound +because `assemble_submission` appends the template's transactions verbatim +after the coinbase (`srv/src/codec.rs:490-497`). #265 adds +`codec::witness_merkle_leaves_from_block(&[u8]) -> Result>`: it +skips the 80-byte header and the compact-size transaction count, as landing +does (`srv/src/ledger/blocks.rs:68-78`), parses past the coinbase with a +segwit-aware transaction parser (`parse_transaction`, +`crates/qbit-prism/src/ctv.rs:1629`, is private today and the candidate for +reuse) and `double_sha256`s each remaining transaction. Its test asserts +equality with the template-derived leaves for a block assembled from that +template. Storing the leaves instead was rejected: one 64-hex string per +transaction scales with the block and would break the row-size bound below. + +Every stored input is O(1), about 1 KB in total, and `candidate_sha256` +covers all of them. The one input that scales is `block_hex`, already stored +today (`srv/src/ledger/candidates.rs:6`): it grows with the block, hex doubling the +serialized size, not with the window. So #265 moves `block_hex` out of the JSONB document into a `bytea` column, `block_bytes`, on the same row: +the JSONB value stays under 1 MB without restating #265's criterion, and the row still carries the block, which consensus bounds, not the window. The +bytes stay authenticated. The candidate JSON keeps `block_sha256`, the +SHA-256 of the `bytea`, which `candidate_sha256` covers. Claim decode checks +both that digest and that the block's 80-byte header double-SHA-256s to +`block_hash`, so altered or inconsistently written bytes fail as `Decode` +before any witness derivation, landing or submission. + +### Builder version + +Stored inputs reproduce a bundle only under the builder that first built it. +A release can change what the builders emit for the same inputs: a new reward +rule, a new manifest field, a canonical-schema change. It would then rebuild a +surviving reference into a different bundle. A pending candidate's coinbase +already commits to the old one, so its claim would fail the audit checks on +every retry, or a retrospectively changed body would land. The `schema` +strings (`AUDIT_BUNDLE_SCHEMA_V1` and `_V1_1`, +`crates/qbit-prism/src/lib.rs:96-97`) don't capture this, because the share +shape chooses them (`:1556-1560`), not the builder logic. The golden test +(`crates/qbit-prism/tests/audit_cli.rs:353`, digests at `:497-521`) detects +such a change for its fixture, but it cannot rebuild the old output. + +- **Version.** `qbit-prism` gains `pub const AUDIT_BUILDER_VERSION: u16`, + starting at 1, in the PR that adds 007 (#265). Both documents store it as + `audit_builder_version` next to the other stored inputs: the `Candidate` + for 007, where `candidate_sha256` covers it, and `StoredPrepared` for 008. +- **Bump rule.** Any change that alters `canonical_audit_bundle_bytes` for + the same inputs bumps the constant in the same commit. The golden test keys + its expected digests by the constant, so a new output is recorded as a new + entry next to the old one, not by overwriting it, and review sees both. +- **Rebuild check.** A rebuild compares the stored version with the binary's + before it calls `read_window`. A mismatch is never rebuilt with the current + builder ([Errors and callers](#errors-and-callers)). +- **Upgrade refusal.** A release that bumps the constant ships a migration + step, with or without DDL. Its refusal predicate counts `state='pending'` + outbox rows whose `audit_builder_version` differs from the new value, and + the operator drains them with the old frontends through the 007 procedure + ([Compatibility](#compatibility)). Job rows need no refusal: a mismatched + prepared job is a cache miss, and the next refresh writes a new one. +- **Rejected alternative.** Keeping a frozen copy of each old builder, so old + references rebuild under it, would let a release skip the drain. This + record drains and refuses instead: the outbox drains in minutes, and every + retained builder is more signed-output code to audit. + +### Signing keys + +The seeds stay local, because they are the signer, not a field. But the +bundle embeds both public keys and their signatures +(`build_profiled_signed_manifest` and `build_ledger_window_attestation`, +`crates/qbit-prism/src/lib.rs:54` and `:1507`), so a rebuild reproduces the +original bytes only under the original keys. With unchanged seeds it does, as +the golden test's digests for fixed seeds show. The cluster fingerprint binds +both public keys (`srv/src/config.rs:479-491`), and `configure` pins it once +and refuses a mismatch (`srv/src/ledger/connect.rs:128-148`). The code has no reset +path, so rotating either key takes an operator resetting +`qbit_prism_cluster.config_fingerprint`. A candidate still pending across +that reset would be rebuilt and signed with the new keys, into bytes its +coinbase does not commit to. + +- **Stored.** Both documents store `signer_keys: { manifest_key_hex, + ledger_key_hex }`, the public keys the building frontend signed with, about + 130 B; `candidate_sha256` covers them for 007. +- **Rebuild check.** It runs next to the builder-version check, before + `read_window`. A mismatch is never rebuilt with the local keys + ([Errors and callers](#errors-and-callers)). +- **Rotation procedure.** A rotation follows the 007 order + ([Compatibility](#compatibility)): with the old-key frontends running, + drain the outbox; stop **every** old-key frontend; only then check that no + `state='pending'` row remains; reset the fingerprint; start the new-key + frontends. The check has to come after the stop, because a frontend still + running could find a block after it and enqueue a candidate with the old + keys, which no new-key frontend can rebuild. +- **Rotation refusal.** #265 extends `configure`: when it writes a + fingerprint onto a reset (NULL) one, it refuses in the same transaction + while any `state='pending'` outbox row stores `signer_keys` other than the + local pair. Prepared jobs signed with the old keys are cache misses. +- **Writer fence.** A one-time check can't stop a writer that runs later, and + today's candidate enqueue is fenced only by `ORDER_LOCK` and the revision + (`srv/src/ledger/candidates.rs:29`), not by the fingerprint. So #265 and + #273 re-read `qbit_prism_cluster.config_fingerprint` with `FOR SHARE` inside + every enqueue and `save_job` transaction, and refuse the write, with an + alert, when it isn't the writer's own. `FOR SHARE` conflicts with + `configure`'s `FOR UPDATE` (`srv/src/ledger/connect.rs:128-148`), so a + write and a reset can't interleave: either the write commits first and the + refusal above sees it, or the reset does and the write fails. A frontend + left running with old keys then fails loudly instead of creating rows no + new-key frontend can rebuild. The fence makes a procedure violation + visible; the stop-first order is what keeps old-key work from existing at + all. +- **Rejected alternative.** A versioned signer that keeps old seeds to + reproduce old signatures would keep retired private keys online, which + defeats a rotation made because a key leaked. + +## Compatibility + +- **Rows written before 007.** Terminal outbox rows already have + `candidate IS NULL` and are untouched. 007 refuses to apply while any + `state='pending'` row still holds an inline candidate (`candidate ? 'bundle'`), + naming the rows: one-way per D5, modelled on the `version < 3` + pending-shape check at `srv/src/ledger/connect.rs:74-75`. After 007 there + is no compatibility decode: a pending row with a NULL `window_anchor_ms` is + a hard claim error telling the operator to stop the pre-007 frontend. +- **Rows written before 008.** Legacy inline prepared rows stay in place. + Resume treats a payload with `snapshot` or `bundle` keys, or a NULL + `window_anchor_ms` on a `prepared:` row, as a cache miss (`Ok(None)`) + because prepared work is regenerable; the rows expire under their TTL and + `prune_expired_jobs` (`srv/src/ledger/jobs.rs:43-44`) removes them. This is + unambiguous now: an empty window is not NULL. +- **Mixed versions.** 3.x.x does not support a pre-007/008 and a post-007/008 + frontend on one database: the old frontend would keep writing inline + documents the new claim path rejects and would fail to decode a reference + job on cross-frontend resume. The one-way model is the one `Ledger::connect` + already enforces for the legacy writer lease (`srv/src/ledger/connect.rs:66`) and + [prism-rust-migration](../prism-rust-migration.md) documents; #285 gates + startup on the schema version. +- **Upgrade procedure.** A `state='pending'` outbox row is cleared only by a + frontend claiming and landing it: the CHECK at + `crates/qbit-prism/sql/001_share_ledger.sql:97-104` keeps `candidate` + non-NULL until the row is terminal, and `retry_candidate` + (`srv/src/ledger/candidates.rs:114-126`) only backs off, never abandons. So "refuse + while pending inline rows exist" plus "apply with every frontend stopped" + would deadlock. There are two starting points. From a **native pre-007 + 3.x.x deployment**, the native `submit_loop` drains the outbox, step 1 + below. From **production 2.x.x**, the 2.x.x submitter drains it before the + cutover, and leftovers are refused twice: by the `version < 3` gate + (`srv/src/ledger/connect.rs:74-75`) on the first native connect and by #285's + 006; #265's migration test covers "a 2.x.x schema with only terminal + rows". The order, with the 2.x.x frontends doing step 1 and any repeat in + that case, is: + 1. With the pre-007 frontends **running**, drain the outbox. `submit_loop` + (`srv/src/coordinator.rs:1112-1135`) polls every 100 ms and claims each + pending row; a row whose revision or parent is superseded is finished by + `finish_candidate_at_revision` (`:979-988`); a failed attempt is retried + after `LEAST(60, attempt_count)` seconds (`srv/src/ledger/candidates.rs:118`). A row + that keeps failing is #268's escape hatch; the procedure never abandons + it. No switch stops candidate production while keeping `submit_loop`, so + a block can be found at any moment until the frontends stop, and a zero + count taken while they run proves nothing. + 2. Stop every frontend. From here nothing creates, claims or finishes a + candidate. + 3. Only now verify `SELECT count(*) FROM qbit_block_candidate_outbox WHERE + state='pending'` is 0. If it is not, a block was found between the drain + and the stop: start the pre-007 frontends again, let `submit_loop` drain + it, and repeat from step 2. + 4. Start one post-008 frontend. `Ledger::connect` applies migrations in one + transaction (the base schema at `srv/src/ledger/connect.rs:77-80`, versioned + steps after it), so it applies 006 (#285), then 007, then 008, each + refusal predicate running in that transaction and naming the rows it + found. Then start the rest; #285's startup gate keeps a pre-007/008 + frontend from joining. 007's refusal is the backstop: if a pending + inline row is present anyway, the migration transaction fails naming it, + nothing is applied, and the operator returns to step 1 with the pre-007 + frontends. +- **Refusal predicates compose.** 006 (#285) refuses on its own pending + shape (#258's storage-version-2); 007 refuses on `EXISTS(SELECT 1 FROM + qbit_block_candidate_outbox WHERE state='pending' AND candidate ? 'bundle')`. + Each predicate runs inside its own `version < N` step of the migration + transaction, in migration order and one-way, never unconditionally at + startup. The pending-shape check at `srv/src/ledger/connect.rs:74-75` is the + model for the shape only: it sits inside `if version.unwrap_or(0) < 3` + (`:49-95`), runs before the base schema (`:77-80`) on the first native + connect after 2.x.x, and never runs at version 7 or above, so #265 leaves + it alone. +- **Canonical hashes.** Under one `AUDIT_BUILDER_VERSION` and one pair of + signing keys, a bundle rebuilt from `read_window` through the borrowing + builders is field-for-field the bundle built at refresh, so + `canonical_audit_bundle_bytes` and `audit_bundle_sha256` are unchanged; the + golden test (`crates/qbit-prism/tests/audit_cli.rs:353`) pins them for its + fixture. Across a bump or a key rotation, [Builder version](#builder-version) + and [Signing keys](#signing-keys) apply. + +## Where the code lives + +#283 landed the `ledger.rs` split (#300), and `srv/src/ledger/window.rs` +exists and belongs to workstream B. It already holds `Snapshot` +(`srv/src/ledger/window.rs:10-16`), `AppendResult` (`:4-7`), the `append*` +family (`:74-185`), `snapshot` (`:190-241`), `observe_chain_view` (`:21-72`), +`read_prior_balances` (`:244-252`), `share_header_hash` (`:254-261`) and +`share_from_row` (`:263-283`). `WindowRef`, `ShareRange`, `Window`, +`WindowError` and `read_window` go there too: every one of them is B-owned +code that only B-owned callers and A's #265 claim path use, and the file +already owns the window read they extend. + +`SELECT_SHARE` needs no move. It stays a `const` in the module root +(`srv/src/ledger.rs:40`), where `window.rs` and `ledger/audit.rs` both reach +it through `use super::*`. `read_range` likewise stays in A-owned +`ledger/audit.rs` (`srv/src/ledger/audit.rs:144-152`). + +One question is left, and it is narrower than before the split: whether +`read_window` reuses `read_range` from `audit.rs` as it stands, or +`read_range` moves into `window.rs` so both readers share one paged +implementation. Only the second edits an A-owned file. **Pending confirmation +with Anatolie on #283**; this record does not claim that choice is settled. + +## Bundle ownership + +`AuditBundle` is unchanged. #296 (merged into 3.x.x as `f316eb9`) added to +`crates/qbit-prism/src/lib.rs`: + +| Entry point | Purpose | +| --- | --- | +| `AuditBundleBody` | every `AuditBundle` field except the top-level `shares`; `#[serde(deny_unknown_fields)]`, so a full bundle cannot decode as a body | +| `build_audit_bundle_body`, `build_audit_bundle_body_with_coinbase_script_sig_suffix`, `build_audit_bundle_body_with_coinbase_options`, `build_audit_bundle_body_with_ctv_settlement_options`, each taking `&[AcceptedShare]` | borrowing builders mirroring the four `build_audit_bundle*` functions (`crates/qbit-prism/src/lib.rs:1567`, `:1611`, `:1658`, `:1753`) | +| `verify_audit_parts(&AuditBundleBody, &[AcceptedShare], …)` and its `verify_audit_parts*` variants | verification without an owned bundle | +| `canonical_audit_bundle_bytes_from_parts(&AuditBundleBody, &[AcceptedShare])`, `write_canonical_audit_bundle_from_parts` | canonical bytes without splicing | +| `AuditBundleBody::into_bundle(Vec)`, `AuditBundle::into_parts()` | lossless conversion both ways | +| `prior_balances_digest(&[CarryForwardBalance]) -> [u8; 32]` | the private `prior_balances_digest_hex` (`crates/qbit-prism/src/lib.rs:2670`) made public for `WindowRef` | + +An earlier draft called the body `AuditBody`; it is `AuditBundleBody` so it +reads next to `AuditBundle` and is not mistaken for the `audit_body_ref` file +format. Both paths serialize through one private `AuditBundleRef<'a>`, so the +canonical bytes are identical by construction; the four existing +`build_audit_bundle*` functions became one-line wrappers with no clone, and +no server file changed in #296. No serde `rc` feature in `qbit-prism`. + +Not `Arc<[AcceptedShare]>`: #296 records why (serde `rc`, the `&durable == +shares` equality at `srv/src/ledger/audit.rs:135`, and no saving while +`reward_manifest.shares` stays). + +**Caveat.** `AuditBundleBody` still embeds `reward_manifest.shares` +(`crates/qbit-prism/src/lib.rs:234`): one `CountedShare` per counted share, +about 556 B each (computed), about 222 MB at 400k. Storing an +`AuditBundleBody` is not a fix for the JSONB ceiling; the body alone crosses +it at about 483k counted shares. #265 and #273 store a `WindowRef` and +rebuild the bundle with the borrowing builders; #267 normalizes the stored +body. The #264 head itself saves no memory; the saving arrives when #265 and +#273 stop holding owned bundles. + +## Expected cost at 400k shares + +Estimates from the measured facts in [Problem](#problem) plus three computed +sizes: an `AcceptedShare` about 594 B as compact JSON (581 B measured +production-shaped), a `CountedShare` about 556 B, a native production-shaped +share about 650 B. The #264 scale harness replaces every estimate. + +| Quantity | Today (3 copies in a job row, 2 in a candidate) | With `WindowRef` | +| --- | --- | --- | +| rows read per rebuild | 1 JSONB row | 400,000 share rows in 98 pages + 1 cluster row + one row per recipient with a balance + 2 probe rows | +| bytes read | 700 to 780 MB JSONB; fails above 268 MB per container (measured at 372k) | about 175 MB on the wire | +| one share array as JSON | 233 to 260 MB | never materialized: the digest streams | +| decode memory | a 470 to 780 MB `serde_json::Value`, then the typed document | about 250 MB for `Vec` plus one 2 MB page of raw rows | +| decode time | JSONB parse of the whole document (fails at production scale) | 0.4 to 0.8 s CPU for row mapping; 1 to 3 s database and transfer | +| hash time | none at claim; `candidate_sha256` over the document | 0.5 to 0.9 s to serialize 233 to 260 MB; 0.2 to 0.5 s SHA-256 | +| total per rebuild | n/a (fails) | 2 to 5 s wall, 1.1 to 2.2 s on blocking threads; the 60 s claim deadline is twelve times the upper bound | +| empty window rebuild | n/a | 2 statements, no share rows, microseconds of hashing; the bootstrap bundle has one share | +| k concurrent rebuilds | n/a | k ≤ `build_workers` (default at most 4): at most `clamp(database_max_connections - 2, 1, build_workers)` pool connections, held only during `read_window`, and about k × 0.5 GB peak, held only during the rebuild because resumed jobs keep no window (one window, owned by the `Snapshot`, plus the rebuilt body's `reward_manifest.shares`; `Prepared` holds no second copy, see [Threads, concurrency and deadlines](#threads-concurrency-and-deadlines)); k = 1 per `storage_key` under the single-flight map | +| WAL per write, measured | 44,752,760 B for one single-window insert into `qbit_prism_jobs` at 372k | under 1 KB for the six columns (about 150 B payload), plus at most one 8 KB full-page image | +| WAL per write, derived | about 134 MB per refresh at three copies and 90 MB per candidate at two, when under the ceiling; #273 estimates 220 MB per refresh for production data at 3:1 | as above | +| refresh CPU added | none | one serialization and SHA-256 of the window per non-cached refresh, 0.7 to 1.4 s on a blocking thread; landing already pays it at `srv/src/ledger/audit.rs:120` | + +The window's share of WAL per refresh falls by more than four orders of +magnitude. The read cost is the price of not storing the window; it equals one +`Ledger::snapshot` scan and is paid only on claim and cross-frontend resume, +not on every refresh. + +## How #265 and #273 use this + +| Step | #265, migration 007, candidates and import | #273, migration 008, prepared work and job rows | +| --- | --- | --- | +| columns | the six on `qbit_block_candidate_outbox`, with `window_prior_balances_sha256` indexed for retention | the six on `qbit_prism_jobs`; per-worker `StoredJob` rows keep them NULL; plus an indexed `template_sha256` on prepared rows and an index on `window_prior_balances_sha256`, both for retention | +| document | `Candidate` (`srv/src/ledger/candidates.rs:4-14`) drops `bundle: AuditBundle` for `window: WindowRef`, `bootstrap_share: Option`, `found_block`, `payout_policy`, `ctv: Option<{ direct_floor_sats, settlement_config, fanout_fee_policy }>`, `audit_builder_version`, `signer_keys` and a `leased` flag ([Revision fence and reorgs](#revision-fence-and-reorgs)); it keeps `block_hash`, `block_hex` (moved to a `bytea` column and authenticated by a new `block_sha256`, [Stored bundle inputs](#stored-bundle-inputs)), `job_id`, `payout_revision`, `deferred_share` and `coinbase_suffix_hex`, now required. The rebuilt parts, `Arc` and the window's shares, live in `CandidateClaim` (`srv/src/ledger/candidates.rs:17-20`), absent until the rebuild fills them where `srv/src/coordinator.rs:1032-1034` replaces `candidate.bundle` today; landing reads them there instead of `claim.candidate.bundle.*` (`srv/src/ledger/blocks.rs:61-153`) and never assembles an `AuditBundle` ([Threads, concurrency and deadlines](#threads-concurrency-and-deadlines)), and `observe_candidate` reads the stored `found_block.block_height` from the candidate instead of through the bundle (`srv/src/coordinator.rs:829`) | `StoredPrepared` (`srv/src/coordinator.rs:46-55`) drops `snapshot` and `bundle` for `window: WindowRef` plus the `Snapshot` scalars the reference does not carry: `share_seq`, the newest sequence at snapshot time (`srv/src/ledger/window.rs:236`), which the cached-work equivalence test compares (`srv/src/coordinator.rs:612`), and `payout_revision`, which `save_job` fences (`srv/src/ledger/jobs.rs:16-23`); resume no longer requires it to equal the current revision ([Revision fence and reorgs](#revision-fence-and-reorgs)). It stores the policy inputs `build_bundle` reads from `config` at `:760-762`: `payout_policy` and the same nested `ctv` field, which also covers the `config.ctv_enabled` choice at `:755` and absorbs today's `fee` (`:50`; `fee_policy` is `None` when CTV is off, `:319-322`). It stores `audit_builder_version` and `signer_keys` too, and keeps `fingerprint`, `generation`, `parent_of_tip` and `coinbase_suffix`. The `template`, the full `getblocktemplate` result with every transaction's hex, moves out of the payload into an immutable `qbit_prism_templates` row keyed by the SHA-256 of its bytes, written once per distinct template (`ON CONFLICT DO NOTHING`) in `save_job`'s transaction; `StoredPrepared` keeps `template_sha256`, and so does a new column on the job row that retention uses ([Immutability and retention](#immutability-and-retention)), and a resume fetches the row, checks the digest and decodes it in `spawn_blocking`. A consensus-valid template can carry megabytes of transaction hex, so without this the payload breaks #273's 1 MB whole-value criterion however small the window is; #273 qualifies the bound with a large valid template. A cross-frontend resume still builds the bundle the issuing frontend would have built. `Prepared` (`:32-43`) and `JobContext` (`:26-30`) keep the in-memory `snapshot: Arc`, the only owned window, and hold `Arc` instead of `Arc` (`:29`, `:35`), built by borrowing (`:747` stops cloning), so neither the local frontend nor a resume keeps a second copy of the window. `Prepared` also keeps `window: WindowRef`, the reference `refresh_once` computed or the resume decoded, so both documents take the same small value | +| write | submit (`srv/src/coordinator.rs:1691-1699`) clones `Prepared.window`, the `WindowRef` `refresh_once` already computed, a few hundred bytes, so a found block never re-digests the window on the share path (the 0.7 to 1.4 s the cost table budgets once per refresh), and sets `bootstrap_share` from `JobContext.bootstrap_share` when `context.prepared.bundle.is_none()` (`:27`; the `None` arm at `:1343` is where the per-worker bootstrap bundle was built); persist (`srv/src/ledger/candidates.rs:182-183`) serializes and digests the small row before the append transaction opens and writes the columns with it | `refresh_once` (`srv/src/coordinator.rs:468`, the store at `:627-641`) computes the reference once per non-cached refresh and keeps it as `Prepared.window`, which submit clones (#265) and a resume takes from `StoredPrepared`; `save_job` writes the columns and a payload with no `shares` key; an empty snapshot writes an empty-window reference | +| read | claim decode (`srv/src/ledger/candidates.rs:73-79`) checks the digest and the columns, and that the `bytea` block hashes to the candidate's `block_sha256` with its header hashing to `block_hash` | resume (`srv/src/coordinator.rs:1447-1470`) decodes the small payload inline, so `:1469-1470` no longer needs `spawn_blocking` for it; the template fetched by `template_sha256` is decoded in `spawn_blocking` | +| fence | `process_candidate_inner` (`srv/src/coordinator.rs:966`) keeps `:972-979`; `Window.payout_revision != candidate.payout_revision` is a second hint for the same `observe_candidate` probe and never a supersession by itself ([Revision fence and reorgs](#revision-fence-and-reorgs)) | B's published-identity and lease check replaces the revision equality at `srv/src/coordinator.rs:1483-1488`; eligible work rebuilds in as-issued mode, and only ineligible work is `Ok(None)`; the same lease decides staleness at submit (`srv/src/coordinator.rs:1632-1644`, `:1672`), so a block found on leased work is enqueued with R0 ([Revision fence and reorgs](#revision-fence-and-reorgs)) | +| rebuild | under a `build_slots` permit (`:997`) and the 60 s deadline around `read_window` and the rebuild: if `qbit_pool_audit_bundles` already holds the block's audit, authenticate that row against the block's coinbase ([Revision fence and reorgs](#revision-fence-and-reorgs)), skip `read_window`, the rebuild and `land_candidate`, never call `materialize_audit_row`, and continue to observe, renew the lease and `submitblock` as today; else await `read_window(…, BalanceSource::Current)` (a `leased` candidate submits before any rebuild and rebuilds only with `AsIssued`, [Revision fence and reorgs](#revision-fence-and-reorgs)), re-derive the witness leaves from `block_hex`, run `build_audit_bundle_body_*(&window.shares, …)` directly in `spawn_blocking` (`:998-1031`; never `build_bundle`, which takes a second permit at `:712`), and put the body and `window.shares` in `CandidateClaim` as parts for landing, never `into_bundle` ([Threads, concurrency and deadlines](#threads-concurrency-and-deadlines)) | under one `build_slots` permit, the single-flight entry for the `storage_key`, within the caller's end-to-end deadline: await `read_window(…, BalanceSource::AsIssued)`, move its `Vec` into a `Snapshot` whose `payout_revision` is `StoredPrepared.payout_revision`, not `Window.payout_revision`, and rebuild `Prepared` as `(Arc, Arc)` through the borrowing builders called directly, never `build_bundle`, or use the local incremental window once #274 lands; once the job's coinbase is built, keep only the slim resumed `Prepared`, with no window ([Threads, concurrency and deadlines](#threads-concurrency-and-deadlines)) | +| empty window | build over `&[bootstrap_share]` | the single-flight entry carries `bundle: None`; each miner gets its own bootstrap bundle, built with the builders directly under a fresh `build_slots` permit taken after the entry's permit is released, from the synthetic share `build_bundle` fabricates from the worker today (`:727-745`, reached from `:1498-1510`), the stored template, anchor, `payout_policy` and `ctv`, the as-issued balances the entry keeps until every waiter has built, and the issued `payout_revision` from `StoredPrepared`; never `build_bundle`, which reads `config` for signed fields (`:755`, `:760-762`) | +| import | `srv/src/ledger/migration.rs:117` stores `canonical_audit_bytes` plus non-share metadata and no inline body. The same PR makes both readers decode `canonical_audit_bytes`, digest-checked against `audit_bundle_sha256` and under `spawn_blocking` (a two-copy body is about 470 MB at 400k), before any `body_uri` fallback, as `audit_canonical_bytes` already does (`srv/src/ledger/audit.rs:12-23`): `Ledger::audit_bundle` (`:92-107`), which today returns only the JSON column, so `backfill-ctv` would stop on imported rows with "import legacy audits first" (`srv/src/ledger/migration.rs:133-136`); and the bundle endpoint's fallback (`srv/src/api/read_models.rs:196-231`), which today reads only `body_uri`, so every frontend would still need the legacy filesystem. Keeping the inline body instead would bring back the two-copy JSONB document this design removes | n/a | +| must not | serialize a share array into the outbox; re-digest the window at submit instead of cloning `Prepared.window`; hydrate a landed audit on the submit loop through `materialize_audit_row`; finish a recovered claim without checking the landed coinbase and audit root against `block_hex`; finish a claim only because its audit has landed; finish or land a `leased` candidate before `submitblock`; finish a submitted `leased` candidate before its audit has landed; rebuild a `leased` candidate with `BalanceSource::Current`, or before `submitblock`; drop the rebuilt `CandidateClaim` parts on a runtime thread; hand landing an assembled `AuditBundle`, or clone, serialize, digest or re-read the window unpaged on a runtime thread while landing; use block bytes whose `block_sha256` or header hash doesn't match; leave `block_bytes` set on a terminal row; hold `ORDER_LOCK` across `read_window`; call `read_window` without a `window_reads` permit; decode inline candidates on a post-007 schema; read local configuration for any stored input; rebuild a reference whose `audit_builder_version` or `signer_keys` differ from this binary's; accept a new fingerprint in `configure` while a pending row stores other `signer_keys`; enqueue a candidate without re-reading `config_fingerprint` `FOR SHARE` in the same transaction; write a candidate without first re-inserting a `leased` candidate's balance snapshot and probing its window's `first_share_seq` row in the same transaction; wrap `read_window` in `spawn_blocking`; call `build_bundle` under a held `build_slots` permit; leave an imported audit readable only through `body_uri` | write a share array into `payload`; call `read_window` without a `window_reads` permit; wrap `read_window` in `spawn_blocking`; run whole-window serde on a runtime thread; call `build_bundle` for any resume rebuild, empty window included (it takes a second permit and reads `config` for signed fields); hold an owned `AuditBundle` beside the `Snapshot` in `Prepared` or `JobContext`; resume a job whose `audit_builder_version` or `signer_keys` differ from this binary's; rebuild published work with the current balances; let any job but the published one gain the replacement lease; turn a decode, digest, database or deadline failure into `unknown-job`; keep the full template in the prepared payload; insert a template row outside `save_job`'s transaction, or a balance-snapshot row outside it or a `leased` enqueue's, or prune them without `SETTLEMENT_LOCK` then `ORDER_LOCK`; scope the balance-snapshot prune to an expired batch's digests; delete a job row before it has been expired for the grace; drop a resumed job's as-issued `prior_balances` before its candidate is enqueued; give a resumed `Snapshot` the current revision instead of `StoredPrepared.payout_revision`; run `save_job` without re-reading `config_fingerprint` `FOR SHARE` in its transaction; keep `Snapshot.shares` or `reward_manifest.shares` in a resumed job once its coinbase is built; drop a rebuilt window on a runtime thread; issue an empty-window `JobContext` without its synthetic `bootstrap_share`; drop a stored input from a resumed job before its candidate is enqueued; mark leased work stale at submit on a revision change, or drop a block found on it; bridge a change of `prior_balances_digest` with the lease before B8 and #289 define how such a block lands; release a `build_slots` permit before its blocking build finishes; change the `save_job` revision fence (`srv/src/ledger/jobs.rs:16-23`) or the cached-work reuse conditions (`srv/src/coordinator.rs:528-533`) | +| text to amend on merge | "reconstructs … through `Ledger::read_window` in `spawn_blocking`": `read_window` is awaited, only the builder runs in `spawn_blocking`; the closed field list ("stores the `WindowRef` … plus … not the bundle"): it is the [Stored bundle inputs](#stored-bundle-inputs) table, `found_block`, `payout_policy`, `ctv`, `bootstrap_share`, `audit_builder_version`, `signer_keys` and the required `coinbase_suffix_hex`; and "outbox row under 1 MB", met by moving `block_hex` into a `bytea` column | the five-field reference list: it is `anchor_ms`, `prior_balances_digest` and an optional range of four (`first_share_seq`, `last_share_seq`, `share_count`, `snapshot_sha256`) | + +**#267, audit bodies.** Uses `AuditBundleBody`, `verify_audit_parts` and +`canonical_audit_bundle_bytes_from_parts` so `ledger/audit.rs` and +`api/read_models.rs` verify and serve without materializing a bundle, and +normalizes `reward_manifest.shares` out of the stored body; the +`share_snapshot_sha256` FK stays. + +## Open questions + +- **djh58 (#273).** The four contract points on #273 are answered here: + published work keeps its authority and rebuilds with as-issued balances, + reconstruction failures stay distinct from missing work under one + end-to-end deadline, both share digests are named with #274's reuse + instruction amended, and the template leaves the prepared payload. Still to + confirm: the `qbit_prism_balance_snapshots` and `qbit_prism_templates` + shapes. Legacy inline prepared rows expire as cache misses, a stop-all + upgrade policy for the development line that C's migration procedure and + tests must state, not a production compatibility guarantee. The 0.7 to + 1.4 s of digest CPU per non-cached refresh is an interim cost to measure, + not an approved budget. Also for djh58 and #289: how an active leased block + lands against as-issued balances after a balance change. Until that rule + exists, the lease covers only revision changes that leave + `prior_balances_digest` unchanged. +- **Anatolie (#283).** Narrowed by #300, not settled. `ledger/window.rs` + exists under B and already holds `share_from_row`, so `WindowRef`, + `read_window`, `Window` and `WindowError` land there with no cross-workstream + edit, and `SELECT_SHARE` stays in the module root. What is left is only + `read_range`: whether `read_window` reuses it where it is, in A-owned + `ledger/audit.rs`, or it moves into `window.rs` so both readers share one + paged implementation. Only the move edits an A-owned file; either A signs + off on that one edit or #265 keeps calling `audit.rs`. +- **D1 (#260).** The cost section assumes 400k shares. A single share array + has 3 to 13 % headroom under the JSONB ceiling there, and an + `AuditBundleBody` crosses it at about 483k counted shares; a target above + that makes #267 a P0 alongside #265 and #273. +- **D6 (#260).** Retention must never prune below the floor above, must keep a ratcheted horizon and record its floor so `snapshot` fails rather than publish an underweight window, must take + `SETTLEMENT_LOCK` then `ORDER_LOCK`, the established order, while it + computes and deletes, must honour the in-flight reservations ([Immutability and retention](#immutability-and-retention)), must count an expired job row in the floor until the prune deletes it, and must + narrow the immutability trigger only for that job. It may add the + `window_first_share_seq` index. +- **#265, #285 and #287.** Whether 007's refusal should also consider live + pre-007 instances heartbeating in `qbit_prism_instances` (#285's startup + gate), and whether the legacy audit import registers a + `qbit_prism_audit_snapshots` row through `read_window` when the history + exists or stays bytes-only (#287). diff --git a/docs/prism-deleted-test-cases.txt b/docs/prism-deleted-test-cases.txt new file mode 100644 index 000000000..0418b7909 --- /dev/null +++ b/docs/prism-deleted-test-cases.txt @@ -0,0 +1,23 @@ +# Deleted cases expanded individually in prism-deleted-test-map.md. +# Independent of the map rows; these are the unittest methods in the pinned +# tests/test_prism_pending_block_recovery.py, excluding its test_config helper. +# Regenerate the names below from the fixed revision: +# git show 504846cc0b72e8f86ed17f896d4ccbbe196a31dc:tests/test_prism_pending_block_recovery.py | sed -n 's/^ def \(test_[^(]*\)(.*/\1/p' + +test_plan_orders_parent_before_child_without_loading_payloads +test_missing_pending_parent_rejects_plan +test_duplicate_missing_abandoned_and_inactive_fail_closed +test_submitted_without_complete_accounting_is_not_skipped +test_payload_identity_height_and_header_are_verified +test_chunked_candidate_row_is_hydrated_not_misclassified +test_unsupported_storage_version_and_unsealed_body_fail_closed +test_apply_finalizes_only_after_normal_accounting_and_checks_durable_result +test_accounting_failure_and_lost_lease_leave_outbox_pending +test_recovery_config_preserves_lease_policy_and_signing_identity +test_runner_never_offers_new_blocks +test_canonical_bytes_identity_and_parser_behavior_match_stdlib +test_codec_restores_after_error +test_exclusive_guard_refuses_live_writer_and_reacquires_after_release +test_reader_is_read_only_and_loads_only_selected_candidate_with_cooperative_codec +test_legacy_only_schema_is_refused_by_the_reader_and_the_ledger +test_real_finalizer_persists_confirms_publishes_and_completes_outbox diff --git a/docs/prism-deleted-test-files.txt b/docs/prism-deleted-test-files.txt new file mode 100644 index 000000000..30f40138c --- /dev/null +++ b/docs/prism-deleted-test-files.txt @@ -0,0 +1,147 @@ +# Deleted test files from the pinned native PRISM cutover comparison. +# Independent of the rows in prism-deleted-test-map.md. +# Regenerate the paths below from these fixed revisions: +# git diff --name-only --diff-filter=D 504846cc0b72e8f86ed17f896d4ccbbe196a31dc 8dccc7494753dd83171929fc4bf02fd2ada6b45c -- tests/ test/ + +test/test-prism-postgres-candidate-storage.sh +tests/fixtures/fake_serve_builder.py +tests/perf/candidate_storage.py +tests/perf/candidate_window_membership.py +tests/perf/canonical_audit_retirement.py +tests/perf/serialize_block_share_cost.py +tests/perf/stratum_session_concurrency_load.py +tests/perf/window_builder_pipe_latency.py +tests/perf/window_ledger_decode_latency.py +tests/perf/window_paging_allocation.py +tests/perf/window_pipeline_gil_scaling.py +tests/prism_candidate_storm.py +tests/prism_concurrency_harness.py +tests/prism_coordinator_test_support.py +tests/prism_landing_harness.py +tests/prism_postgres_a1_gate.py +tests/prism_postgres_a1_migration_gate.py +tests/prism_postgres_a1_process_gate.py +tests/prism_postgres_a1_revert_gate.py +tests/prism_postgres_candidate_gate.py +tests/prism_public_read_replica_gate.py +tests/prism_regtest_crash_coordinator.py +tests/prism_regtest_share_miner.py +tests/prism_vardiff_test_support.py +tests/prism_window_rows_reference.py +tests/prism_worker_difficulty_store_gate.py +tests/test_ctv_broadcaster.py +tests/test_ctv_broadcaster_daemon.py +tests/test_prism_accepted_block_accounting.py +tests/test_prism_accepted_preview_publication_latency.py +tests/test_prism_accepted_preview_telemetry.py +tests/test_prism_admission_lock.py +tests/test_prism_allocator_experiment.py +tests/test_prism_ancestor_redrive.py +tests/test_prism_audit_api.py +tests/test_prism_audit_artifacts.py +tests/test_prism_audit_bundle_view.py +tests/test_prism_audit_http.py +tests/test_prism_audit_root_scan.py +tests/test_prism_backfill_audit_bundle_canonical.py +tests/test_prism_background_services.py +tests/test_prism_block_candidate_collapse.py +tests/test_prism_block_candidates.py +tests/test_prism_block_finalization.py +tests/test_prism_bounded_canonical_finalization.py +tests/test_prism_bounded_executor.py +tests/test_prism_builder_pipe_transport.py +tests/test_prism_candidate_codec.py +tests/test_prism_candidate_helper.py +tests/test_prism_candidate_replay_bounds.py +tests/test_prism_candidate_schema_refusal.py +tests/test_prism_candidate_spool_integration.py +tests/test_prism_candidate_spool_view.py +tests/test_prism_candidate_storage_races.py +tests/test_prism_candidate_storm.py +tests/test_prism_candidate_window.py +tests/test_prism_capacity_evidence.py +tests/test_prism_completion_callbacks.py +tests/test_prism_compose_profile.py +tests/test_prism_concurrency_harness.py +tests/test_prism_coordinator_config.py +tests/test_prism_coordinator_config_loading.py +tests/test_prism_coordinator_job_cache.py +tests/test_prism_coordinator_metrics.py +tests/test_prism_coordinator_shutdown.py +tests/test_prism_ctv_refresh_priority.py +tests/test_prism_ctv_runtime.py +tests/test_prism_dequeue_stale_skip.py +tests/test_prism_direct_stratum.py +tests/test_prism_endpoint_registry.py +tests/test_prism_first_job_latency.py +tests/test_prism_heap_census.py +tests/test_prism_hot_path.py +tests/test_prism_immutable_refresh_artifacts.py +tests/test_prism_incremental_payout_window.py +tests/test_prism_initial_job_delivery.py +tests/test_prism_issue_224_cadence.py +tests/test_prism_issue_224_core.py +tests/test_prism_issue_224_landing.py +tests/test_prism_issue_224_ledger.py +tests/test_prism_job_build_exception_retention.py +tests/test_prism_job_build_promises.py +tests/test_prism_job_build_retention.py +tests/test_prism_job_builder.py +tests/test_prism_job_delivery.py +tests/test_prism_landing_append_epoch.py +tests/test_prism_landing_budget_watchdog.py +tests/test_prism_landing_harness_model.py +tests/test_prism_landing_interleavings.py +tests/test_prism_lease_heartbeat_tail_latency.py +tests/test_prism_lease_heartbeat_under_fenced_write.py +tests/test_prism_lease_monitor_stalls.py +tests/test_prism_lease_orphaned_lock.py +tests/test_prism_lease_renewal_vs_expiry_claim.py +tests/test_prism_lease_takeover.py +tests/test_prism_metrics.py +tests/test_prism_observability.py +tests/test_prism_payout_state.py +tests/test_prism_payout_window_daemon_recenter.py +tests/test_prism_pending_block_recovery.py +tests/test_prism_process_telemetry.py +tests/test_prism_progress_health.py +tests/test_prism_public_dashboard_api.py +tests/test_prism_public_read_service.py +tests/test_prism_reconnect_backpressure.py +tests/test_prism_refresh_retry_pacing.py +tests/test_prism_reorg_reconciler.py +tests/test_prism_replay_enumeration_gate.py +tests/test_prism_representative_independence.py +tests/test_prism_retained_jobs.py +tests/test_prism_reward_window_invariant.py +tests/test_prism_self_check.py +tests/test_prism_settlement_balance_comparison.py +tests/test_prism_share_ledger.py +tests/test_prism_share_submission.py +tests/test_prism_share_writer.py +tests/test_prism_share_writer_service.py +tests/test_prism_singleflight_exception_retention.py +tests/test_prism_statement_spool.py +tests/test_prism_stratum_restart_bind.py +tests/test_prism_stratum_session.py +tests/test_prism_streamed_block_hex_consumers.py +tests/test_prism_streamed_submitblock.py +tests/test_prism_tip_publication_boundary.py +tests/test_prism_tip_refresh.py +tests/test_prism_tip_refresh_delivery.py +tests/test_prism_tip_refresh_epoch.py +tests/test_prism_tip_refresh_validation.py +tests/test_prism_vardiff.py +tests/test_prism_vardiff_durable_resume.py +tests/test_prism_vardiff_fast_arrival.py +tests/test_prism_vardiff_resume.py +tests/test_prism_vardiff_service.py +tests/test_prism_window_pipeline_rust.py +tests/test_prism_worker_difficulty_store.py +tests/test_prism_writer_lease_policy.py +tests/test_public_dashboard_api_contract.py +tests/test_window_paging_allocation_harness.py +tests/test_window_pipeline_parity.py +tests/test_window_pipeline_parity_gate.py +tests/window_pipeline_parity.py +tests/window_pipeline_parity_gate.py diff --git a/docs/prism-deleted-test-map.md b/docs/prism-deleted-test-map.md new file mode 100644 index 000000000..af5a33f95 --- /dev/null +++ b/docs/prism-deleted-test-map.md @@ -0,0 +1,417 @@ +# Deleted 2.x.x tests and their 3.x.x replacements + +This map lists every test file that exists on `origin/2.x.x` (504846c) and is absent on the 3.x.x base it was built against (8dccc74, the merge of #316), with the 3.x.x test that covers the same behaviour, or the open issue that will. All 142 files were deleted by the native PRISM server cutover (#244). This is the "deleted-test map" the readiness review (#260) asked #286 to check in; every open gap row links the issue that closes it. + +Regenerate the file list with: + +``` +git diff --name-status --diff-filter=D 504846c 8dccc74 -- tests/ test/ +``` + +Both ends are pinned on purpose: `origin/2.x.x` has moved past 504846c and adds test files of its own, and this branch removes eight `test/test-prism-*.sh` wrappers on its own; neither is a cutover deletion and neither is a row here. + +The same pinned deletion set is checked in as [prism-deleted-test-files.txt](prism-deleted-test-files.txt), with its generation command in the header. The offline check compares file-row names against this manifest, so it also works in shallow checkouts without either historical commit. + +The 17 case rows under `tests/test_prism_pending_block_recovery.py` are checked against [prism-deleted-test-cases.txt](prism-deleted-test-cases.txt), generated from that file's test methods at the same pinned 2.x.x revision. This separate inventory keeps individual cases from disappearing or being renamed while the file counts still match. + +How the rows were derived: each 2.x.x file was read at 504846c (module docstring, test names, test bodies by behaviour cluster), then the native suites were searched and read: `crates/qbit-prism-server/tests/*.rs` (with `support/` and `observability/`), `#[cfg(test)]` modules under `crates/qbit-prism-server/src/`, `crates/qbit-prism/tests/*.rs`, `crates/qbit-prism/src/`, and the Python tests that remain under `tests/`. A native test is cited only where its body asserts the behaviour. Many cited native tests are gated on `PRISM_TEST_DATABASE_URL`, `PRISM_TEST_PG_BIN_DIR` or `QBITD_BIN`; they take those inputs through the shared gate described in [prism-integration-test-gate.md](prism-integration-test-gate.md), and the `prism-native-postgres` job proves from the gate's manifest that each of them executed. The nine PostgreSQL-gated decision-D2 coordinator tests from #316 (`crates/qbit-prism-server/src/coordinator/d2_bootstrap_tests.rs` and `d2_below_target_tests.rs`) are cited where they assert a deleted test's behaviour or its documented 3.x.x replacement rule (`docs/prism-rust-migration.md`, "Payout differences from 2.x.x (decision D2)"). + +Status values: + +- **full**: every behaviour the 2.x.x file asserted has a named 3.x.x test. +- **partial**: some behaviours have a native test; the row names what is not covered, and links the issue when one covers the remainder. Sub-behaviours whose subject no longer exists are marked retired inside the row. A partial row with no "not covered" clause has nothing left uncovered: everything it does not cite is retired, and it is partial rather than full only because of that. +- **open gap**: no native test; the linked issue closes the gap. +- **needs triage**: no native test and no issue that closes it, or the mapping is unclear. +- **retired**: the subject no longer exists in 3.x.x (Python-runtime mechanics with no native counterpart); neither a gap nor a replacement. + +In the replacement column `path::name` is a Rust test function (`#[test]` or `#[tokio::test]`) or a remaining Python `def test_`, with `path` relative to the repository root. A test that a macro generates has no `fn` line of its own, so it is named in prose beside its file, not as `path::name`. + +Issues are named in two ways, and the difference is checked (see [Keeping this map current](#keeping-this-map-current)). A link, written `→ [#N](…)` where it owns a remainder, names an **open** issue the row depends on: the issue that owns behaviour no native test covers, or one the row's reason leans on. A plain `#N` is history: the closed issue or merged pull request the cited tests came from, or the issue that closed without a behaviour. A closed issue is never the owner of an uncovered behaviour. + +The readiness-review issues this map first linked (#266, #271, #272, #280, #285, #287 and #288) have all closed, by pull requests that did not touch this map. Their rows were reconciled on 2026-09-17 at a1937054 (#448): each names the tests that landed, retires what has no native subject with the reason, and writes what is still uncovered as "not covered, no open issue". #270 closed on 2026-09-18 after #438 and #459 landed the native storm scenarios; the rows below cite that evidence, retire the capacity-closed condition under owner decision D-B, and leave the remaining gaps explicitly unowned. A needs triage row still links #447 where its reason depends on it. #278 closed on 2026-09-17, a day after this reconciliation first linked it, and the closed-issue check caught it on the next run; its row was re-read and now names it as history. [#418](https://github.com/Qbit-Org/qbit-mining-bootstrap/issues/418) (the P2 follow-up that carried #268's remaining recover work) is closed too; where prose links it, it marks the origin of the `candidates_recover` tests, not a gap. #278 is mentioned where its trimmed cutover-minimum set named a missing metric family, as history. + +## Summary + +| Rows | Full | Partial | Open gap | Needs triage | Retired | +| --- | --- | --- | --- | --- | --- | +| 142 | 6 | 80 | 0 | 0 | 56 | + +No open gap rows remain. The five open gap rows, and one open gap case under `tests/test_prism_pending_block_recovery.py`, linked #271, #272, #280 and #285, which have all closed; each was reconciled on 2026-09-17 (#448) and is now partial. Partial rows whose uncovered remainder is owned by an open issue link it inline, and what no open issue owns is written as "not covered, no open issue". + +Sections and row counts: + +- Share ledger and accounting (12) +- Block candidates: landing, finalization, recovery, storm and collapse, reorg (12) +- Bounded candidate storage (#255 chunked bodies, #258 storage version 2) (12) +- Writer lease, lease heartbeat and the concurrency harnesses (11) +- PostgreSQL ops gates: migration, revert, cross-process, read replica, worker-difficulty store (7) +- Payout window, payout state, window pipeline parity and issue #224 telemetry (18) +- Audit bundles, artifacts, audit HTTP and canonical backfill (6) +- CTV fanout and CPFP broadcaster (3) +- Stratum sessions, admission, job delivery, retained jobs and the submit hot path (11) +- Vardiff and durable difficulty resume (6) +- Job builder, template and tip refresh, refresh liveness (14) +- Coordinator configuration, shutdown, background services, self-check, capacity evidence and Compose (10) +- Metrics, observability, process telemetry and progress health (7) +- Public dashboard API and the public read tier (4) +- Performance instruments (tests/perf) (9) + +## Keeping this map current + +This map is evidence for the cutover go/no-go (#291), so it is checked, not trusted. The rule the checks serve: **the pull request that closes an issue this map links updates the map in the same change.** Name the tests that landed and move the row to full or partial; retire what has no native subject, with the reason; and where behaviour is still uncovered, link an open issue whose body names it, or write "not covered, no open issue". + +`python3 scripts/check_deleted_test_map.py` runs in the required `Lint, compile, and Compose validation` job, offline, and fails when: + +- a deleted file from the pinned manifest has no file row, a file row names a file outside that manifest, or the manifest is unreadable, empty or contains duplicates; +- a case from the pinned case manifest has no case row, a case row names a case outside that manifest, or the case manifest is unreadable, empty or contains duplicates; +- a `path::name` reference names a file that is not in the repository, a function that is not in that file, or a function that is not a test; +- a table line does not parse, a status is not one the legend defines, a row's text does not lead with its status, a full or partial row cites no test, or an open gap row links no issue; +- the summary table or the section row counts differ from the rows, or the needs triage index differs from the needs triage rows; +- an issue link does not point at this repository's issue of the same number, or an owner arrow introduces an issue number with a bare reference or malformed link. + +The offline check imports Python tests for discovery but does not execute fixtures or test bodies. The required Python unit test shards separately require a successful execution of every Python citation assigned to that shard, using the normal unittest run. Module, class, case and async fixture skips, test-body skips and expected failures do not count as successful parity evidence. Uncited optional tests may still skip. A local offline `ok` establishes map consistency; the required test jobs establish execution. + +`python3 scripts/check_deleted_test_map.py --check-issues` also asks GitHub for the state of every issue the map depends on (any issue link inside a table row, and any `→` link anywhere) and fails when one is closed. It runs as the `deleted-test-map-owner-issues` job on every pull request and every push to `3.x.x`. That job is deliberately not required: an issue closing must not block an unrelated pull request. A red run there is the prompt to reconcile the row it names. It exits 2, not 0, when GitHub cannot be asked (network, rate limit, unexpected reply): an unknown state is never read as open. The job rides on CI and not on a schedule because a `schedule` or `workflow_dispatch` trigger only fires from the default branch, and this map lives on `3.x.x`; run the whole workflow by hand with `gh workflow run ci.yml --ref 3.x.x`, or the check alone with `GH_TOKEN="$(gh auth token)" python3 scripts/check_deleted_test_map.py --check-issues`. + +`python3 scripts/check_deleted_test_map.py --summary` prints the row counts and the rows that are still open, which is what a sign-off comment on #291 should quote. + +What the checks cannot see: whether a cited test's body still asserts what the row says, and an ownership claim written as a plain `#N`. Both are review's job, which is why a plain `#N` must only ever be history. + +## Share ledger and accounting + +The 2.x.x `PsqlShareLedger` contracts, the Python share writer thread and the pure submission decisions. The native ledger appends one share per `ORDER_LOCK` transaction; the writer lease, psql/psycopg backends, statement spool and recovery journal have no native counterpart. + +| 2.x.x file | Covered | 3.x.x replacement or status | +| --- | --- | --- | +| `tests/test_prism_share_ledger.py` | partial | partial: covered by `crates/qbit-prism-server/tests/ledger_postgres.rs::concurrent_instances_have_one_commit_order_and_stable_snapshots`, `crates/qbit-prism-server/tests/ledger_postgres.rs::global_duplicates_idempotence_and_config_fencing`, `crates/qbit-prism-server/tests/ledger_postgres.rs::candidate_outbox_is_atomic_and_claims_recover_after_owner_loss`, `crates/qbit-prism-server/tests/ledger_postgres.rs::simultaneous_block_only_enqueue_has_one_winner`, `crates/qbit-prism-server/tests/ledger_postgres.rs::deferred_share_survives_confirmation_crash_and_is_credited_once`, `crates/qbit-prism-server/tests/ledger_postgres.rs::candidate_dispatch_prioritizes_fresh_work_and_services_oldest_due_fairly`, `crates/qbit-prism-server/tests/ledger_postgres.rs::reconciliation_watches_only_recent_blocks_and_one_mature_checkpoint`, `crates/qbit-prism-server/tests/ledger_postgres.rs::verified_landing_reconstructs_audit_and_reorgs_are_revision_fenced`, `crates/qbit-prism-server/tests/ledger_postgres.rs::legacy_audit_import_validates_envelope_hash_and_pinned_key`, `crates/qbit-prism-server/tests/ledger_postgres.rs::ctv_artifacts_wait_for_maturity_and_claims_are_fenced`, `crates/qbit-prism-server/tests/ledger_postgres.rs::migration_refuses_an_active_legacy_writer`, `crates/qbit-prism-server/tests/window_read_oracle.rs::window_matches_the_oracle_across_page_boundaries`, `crates/qbit-prism-server/tests/window_read_oracle.rs::rows_on_the_anchor_are_credited_and_rows_after_it_are_not`, `crates/qbit-prism-server/tests/window_read_oracle.rs::the_row_that_crosses_the_weight_boundary_is_credited`, `crates/qbit-prism-server/tests/window_read_oracle.rs::rejected_rows_are_excluded_and_consume_no_weight`, `crates/qbit-prism/src/lib.rs::anchor_requires_job_issue_and_accept_time`, `crates/qbit-prism/src/lib.rs::credit_policy_is_preserved_on_counted_shares`, `crates/qbit-prism-server/tests/support/ledger_2x.rs::native_publication_ordinal_survives_reactivation_and_excludes_rejected_candidates`, `crates/qbit-prism-server/tests/support/ledger_2x.rs::canonical_2x_sidecar_import_preserves_exact_bytes_and_fails_closed`, `crates/qbit-prism-server/tests/support/ledger_2x.rs::audit_range_query_uses_deadline_remaining_after_delayed_snapshot`, `crates/qbit-prism-server/tests/api_database.rs::shared_database_serves_all_contracts_and_global_reward_ranks`, `crates/qbit-prism-server/tests/api_database.rs::accepted_public_blocks_and_earnings_follow_confirmed_chain_state`, `crates/qbit-prism-server/tests/api_2x.rs::public_service_is_read_only_and_bounds_http_database_work`, `crates/qbit-prism-server/tests/observability/registry.rs::pending_candidate_count_and_age_move_together_without_database_prerequisites`, `crates/qbit-prism-server/tests/ledger_deadline_atomicity.rs::issued_job_late_expiry_rolls_back`, `crates/qbit-prism-server/tests/ledger_deadline_atomicity.rs::landing_rows_roll_back_on_late_failure`, `crates/qbit-prism-server/tests/ledger_deadline_atomicity.rs::backfill_item_failure_is_atomic`, `crates/qbit-prism-server/tests/ledger_deadline_atomicity.rs::import_item_failure_preserves_completed_items`, `crates/qbit-prism-server/tests/ledger_deadline_atomicity.rs::incomplete_window_is_typed_error`, `crates/qbit-prism-server/tests/ledger_deadline_atomicity.rs::incomplete_audit_snapshot_returns_no_body`, `crates/qbit-prism-server/tests/ledger_deadline_atomicity.rs::conflicting_audit_preserves_original`, `crates/qbit-prism-server/tests/ledger_single_execution.rs::candidate_terminal_timeout_executes_once`, `crates/qbit-prism-server/tests/ledger_single_execution.rs::fanout_journal_timeout_executes_once`, `crates/qbit-prism-server/tests/ledger_single_execution.rs::candidate_backoff_requires_new_claim`, `crates/qbit-prism-server/tests/ledger_single_execution.rs::broadcast_retry_requires_new_claim` (the native rules that replace the 2.x.x deadline, batch, retry and segment cases, documented in `docs/prism-ledger-ops.md`, "Retry, replay and deadline contract", and asserted against PostgreSQL: an issued-job save whose absolute deadline passes during a row-lock wait publishes nothing and renews nothing; a landing or a CTV repair whose last row fails leaves no row of that block; a failed legacy import keeps its completed items and a rerun imports only the unfinished ones; a window or audit read over missing share rows is an error, never a partial or repaired body; a different audit for a landed hash is refused and the original kept; a candidate terminal write and a CTV attempt-journal insert are each sent once across a statement timeout, a lost reply and a lost COMMIT reply; a later attempt needs a fresh claim); not covered, no open issue: pending-candidate cursor pages and bulk abandon, the job-issue delta read; the final-partial-batch decode deadline and between-batch progress stamp, `append_batch`, the `retry_safe` read retry and the audit share-segment slot lifecycle (gap backfill, quarantine, conflicting duplicate) are retired (the native ledger has no share batch writer, no automatic read retry, and writes no segment files; its one automatic re-execution is the share append re-run once after a no-partition refusal, `crates/qbit-prism-server/tests/share_partitions_postgres.rs::an_append_past_the_last_bound_attaches_the_lead_and_lands_the_share`), as are the writer-lease, psql/psycopg backend, operation-gate, TTL-cache and row-decoding clusters | +| `tests/test_prism_statement_spool.py` | retired | retired — the psql statement spool (large finalization statements streamed to a psql subprocess under the Python writer lease) has no native counterpart; sqlx binds parameters and there is no writer lease | +| `tests/test_prism_landing_append_epoch.py` | retired | retired — the append-invalidation epoch counter and per-anchor stamp history are Python coordinator state; native windows are anchored on the monotone ledger clock and fenced by payout revision, so the forfeiture path does not exist | +| `tests/test_prism_landing_budget_watchdog.py` | retired | retired — no in-process block-work watchdog or coordinator-local writer-lock admission queue; native budgets are PostgreSQL `lock_timeout`/`statement_timeout` session settings and renewed candidate claims | +| `tests/test_prism_landing_interleavings.py` | partial | partial: covered by `crates/qbit-prism-server/src/coordinator/window_switch_tests.rs::a_parent_landing_late_diverges_the_child_which_lands_as_issued_with_additive_balances` (two distinct found blocks, a parent and its child, are offered and accepted on two frontends with both landings held; the parent lands and confirms while the child is still unlanded, then the child lands; both rows end `submitted`, neither block is offered twice, each block has its own `qbit_pool_audit_bundles` row, and the integrity report, which names a confirmed block without its audit row, is clean after both), `crates/qbit-prism-server/tests/support/candidates_recover.rs::recover_apply_lands_the_listed_blocks_and_verifies_completed_ones_idempotently` (two distinct blocks land back to back in one run, and each ends `confirmed` with its audit body and payout entries), `crates/qbit-prism-server/tests/ledger_single_execution.rs::distinct_hashes_hold_independent_claims` (a second hash is claimed, landed and finished while the first hash's claim is held and its outbox row is locked; the held row is untouched); the superseded-envelope report and the writer-lease gate on restoring that envelope are retired — the native landing inserts the block's audit row, keyed by block hash, in the transaction that inserts the pool block and before confirmation, so there is no evidence pointer or publication floor to fall behind (`/audit/latest` is a per-request query over confirmed blocks, not stored state, and nothing gates on the publication ordinal, so the 2.x.x ordinal-and-floor assertions are retired with the floor) and no writer lease; the native rules in their place are `crates/qbit-prism-server/tests/ledger_single_execution.rs::superseded_landing_reports_failure` (a landing at a superseded payout revision is an error that writes no block, audit, payout, carry or fanout row and leaves the claim unchanged) and `crates/qbit-prism-server/tests/ledger_single_execution.rs::restored_authority_requires_fresh_claim` (a claim that expired during a halt is refused by every operation once authority returns; only a new claim finishes the block) | +| `tests/test_prism_replay_enumeration_gate.py` | retired | retired — the coordinator-local writer convoy a replay page could queue behind is gone; native claims one pending row per query with `FOR UPDATE SKIP LOCKED` | +| `tests/prism_window_rows_reference.py` | full | `crates/qbit-prism-server/tests/window_read_oracle.rs::window_matches_the_oracle_across_page_boundaries`, `crates/qbit-prism-server/tests/window_read_oracle.rs::the_row_that_crosses_the_weight_boundary_is_credited`, `crates/qbit-prism-server/tests/window_read_oracle.rs::a_u128_maximum_difficulty_is_decoded_and_closes_the_window`, `crates/qbit-prism-server/tests/window_read_oracle.rs::rows_on_the_anchor_are_credited_and_rows_after_it_are_not`, `crates/qbit-prism-server/tests/window_read_oracle.rs::the_anchor_barrier_holds_at_large_millisecond_values`, `crates/qbit-prism-server/tests/window_read_oracle.rs::writer_rows_follow_integer_milliseconds_past_2_43`, `crates/qbit-prism-server/tests/window_read_oracle.rs::rejected_rows_are_excluded_and_consume_no_weight`, `crates/qbit-prism-server/tests/window_read_oracle.rs::empty_windows_return_no_shares` | +| `tests/test_prism_reward_window_invariant.py` | full | full: covered by `crates/qbit-prism-server/tests/window_multiplier_invariant.rs::every_window_multiplier_literal_matches_the_constant`, which enumerates the multiplier literals across `crates/qbit-prism-server/src/**` (Rust and SQL alike) and `crates/qbit-prism/sql/*.sql`, pins every one to `qbit_prism::PRISM_WINDOW_MULTIPLIER` — reporting all disagreeing sites at once — and asserts the discovered site set is exactly the expected one, so a new spelling or a silently unmatched site fails. It covers the three `qbit_audit_share_window` multipliers and the dashboard `audit_network_difficulty::numeric * 8::numeric` the Python test pinned, plus sites it never reached: the two dashboard `qbit_prism_window` binds, the archive horizon's inline SQL, and the `window_multiplier`/`requested_window_weight` read models. The Python coordinator and `share_ledger.py` copies it also pinned are retired subjects. Not pinned: a Rust site that spells the multiplier as a `checked_mul(8)` argument rather than one of the scanned idioms; the server's three such sites derive it from the constant today | +| `tests/test_prism_settlement_balance_comparison.py` | partial | partial: covered by `crates/qbit-prism-server/tests/offer_lifecycle.rs::pending_divergent_landing_refuses_while_the_post_offer_landing_succeeds` (an intervening block moves one miner's carried balance from 1000 to 1700; the pending landing is refused with "candidate prior balances differ from current canonical balances" and writes no pool block; the same claim lands as issued once it has been offered); not covered, no open issue: a membership divergence at that fence (a program added to or missing from the current balances; `crates/qbit-prism-server/src/ledger/blocks.rs` compares one `qbit_prism::prior_balances_digest` of the as-issued set with the current one, and the test changes only an amount), and a block landed under a re-spelled payout address between issue and landing: the 2.x.x relabel tolerance has no native counterpart, labels are the address as typed at authorize and the digest covers `recipient_id` and `order_key`, so by reading the source that is a digest divergence for a pending row, the opposite of the 2.x.x rule, and no native test drives it; the D2c test `crates/qbit-prism-server/src/coordinator/d2_bootstrap_tests.rs::a_bootstrap_block_still_pays_carried_forward_balances` asserts only that a seeded carry is carried into the bootstrap bundle and paid, not the comparison; the `settlement_balances_by_program` projection itself (program-hex case, split identities) is retired — `qbit_current_carry_forward_balances()` returns one row per program with zero totals dropped and split identities summed in SQL, and the Rust replay of the same rule is asserted by `crates/qbit-prism/src/lib.rs::current_carry_forward_balances_replays_active_deltas` Also `crates/qbit-prism-server/tests/offer_lifecycle.rs::a_divergent_offered_landing_warns_with_both_digests_and_keeps_its_as_issued_accounts` asserts the offered-and-moved landing's warning, both digests and the provenance marker. | +| `tests/test_prism_share_writer.py` | partial | partial: covered by `crates/qbit-prism-server/tests/stratum_protocol.rs::multiple_miners_unique_extranonces_durable_accept_and_duplicate_rejection`, `crates/qbit-prism-server/tests/support/live_highdiff.rs::observability_grace_counter_moves_only_after_real_durable_credit`, `crates/qbit-prism-server/tests/support/live_highdiff.rs::real_highdiff_block_only_proof_waits_for_active_chain_credit`, `crates/qbit-prism-server/tests/postgres_failover.rs::acknowledged_shares_survive_synchronous_primary_loss_and_pool_reconnect`, `crates/qbit-prism-server/tests/observability/stratum.rs::accepted_share_with_failed_tcp_response_has_no_ack_or_rejection`, `crates/qbit-prism-server/tests/ledger_postgres.rs::concurrent_instances_have_one_commit_order_and_stable_snapshots`, `crates/qbit-prism-server/tests/ledger_postgres.rs::candidate_outbox_is_atomic_and_claims_recover_after_owner_loss`; not covered: group-commit batching and linger, the recovery journal and its replay/quarantine, backlog overflow, the landing-fence interplay of predating appends (all Python writer-thread mechanisms with no native subject) | +| `tests/test_prism_share_writer_service.py` | retired | retired — Python S3 typed-port service (symbol identity, floor/anchor registry, admission tokens, journal replay) has no native counterpart; native writes one share per transaction through `Ledger::append_at_revision` | +| `tests/test_prism_share_submission.py` | partial | partial: covered by `crates/qbit-prism-server/tests/stratum_protocol.rs::multiple_miners_unique_extranonces_durable_accept_and_duplicate_rejection`, `crates/qbit-prism-server/tests/stratum_protocol.rs::negotiation_reauthorization_retains_original_job_worker_and_grace_expires`, `crates/qbit-prism-server/tests/stratum_protocol.rs::same_parent_payout_replacement_retains_the_superseded_job_for_block_submission`, `crates/qbit-prism-server/tests/readiness_rpc.rs::another_frontend_payout_revision_captures_the_same_parent_block_but_fences_its_share`, `crates/qbit-prism-server/tests/stratum_protocol.rs::highdiff_floor_still_forwards_valid_block_below_share_target`, `crates/qbit-prism-server/tests/support/live_highdiff.rs::real_highdiff_block_only_proof_waits_for_active_chain_credit`, `crates/qbit-prism-server/src/coordinator/d2_below_target_tests.rs::block_only_proof_is_credited_network_work_only_once_the_block_is_active`, `crates/qbit-prism-server/src/coordinator/d2_below_target_tests.rs::share_passing_block_proof_is_credited_its_assigned_difficulty_at_once` (both block routes driven through `MiningBackend::submit` against PostgreSQL: the block-only route waits for active-chain credit, the share-passing route is credited and enqueued at once), `crates/qbit-prism-server/tests/support/live_highdiff.rs::observability_grace_counter_moves_only_after_real_durable_credit`, `crates/qbit-prism-server/tests/observability/stratum.rs::complete_submit_frame_to_successful_ack_excludes_partial_frame_and_post_ack_hint`, `crates/qbit-prism-server/tests/observability/stratum.rs::failed_rejection_response_counts_the_decision_without_an_ack`, `crates/qbit-prism-server/src/stratum/stale_grace_tests.rs::unknown_job_rejects_before_any_node_rpc_in_real_request_and_coordinator_resume`, `crates/qbit-prism-server/src/stratum/stale_grace_tests.rs::same_tip_delivery_does_not_slide_grace_but_real_flip_reanchors` (a missing context rejects as `unknown-job` and a known prior-tip job past its grace as `stale-job`, both read off the wire), `crates/qbit-prism-server/src/stratum/stale_grace_tests.rs::same_tip_capacity_eviction_survives_beyond_one_second_and_keeps_original_worker` (a retained same-tip context earns ordinary credit with no credit policy), `crates/qbit-prism-server/src/stratum/retained_tests.rs::local_graveyard_cannot_authorize_another_connections_worker_or_old_parent`, `crates/qbit-prism-server/src/stratum/retained_tests.rs::published_tip_capacity_is_reselected_after_awaited_resume` (a cross-connection resume is same-tip only: `resume_job` misses once the tip has moved, and a resume that finishes after the tip moved rejects as `stale-job` with no record, while the same connection's own prior-tip job is credited under `stale-grace`), `crates/qbit-prism-server/src/coordinator/miner_tests/credit.rs::observed_tip_pending_prepared_and_revision_still_credits_exact_prior_parent` (a block-passing proof inside stale grace is recorded under the `stale-grace` policy with no candidate); not covered, no open issue: the `mining.submit` validation order, a username that differs from the authorized one rejecting as `unauthorized-worker` before a wrong extranonce2 width rejects as `invalid-extranonce` (both checks live in the `mining.submit` arm of `crates/qbit-prism-server/src/stratum.rs`; no native test sends either frame; never listed in #280); `RecentShareIndex` is retired (dedup is the durable `qbit_prism_share_hashes` table) | + +## Block candidates: landing, finalization, recovery, storm and collapse, reorg + +The found-block path from durable candidate to landed pool block. The native outbox claims one pending row per query with a fenced claim token; the 2.x.x replay pages, set-oriented collapse selector and in-process lanes are gone, but the storm scenarios they defended are not (#270). + +| 2.x.x file | Covered | 3.x.x replacement or status | +| --- | --- | --- | +| `tests/test_prism_block_candidates.py` | partial | partial: covered by `crates/qbit-prism-server/tests/ledger_postgres.rs::candidate_outbox_is_atomic_and_claims_recover_after_owner_loss`, `crates/qbit-prism-server/tests/ledger_postgres.rs::candidate_renewal_requires_a_live_pending_token`, `crates/qbit-prism-server/tests/ledger_postgres.rs::candidate_dispatch_prioritizes_fresh_work_and_services_oldest_due_fairly`, `crates/qbit-prism-server/tests/ledger_postgres.rs::simultaneous_block_only_enqueue_has_one_winner`, `crates/qbit-prism-server/tests/ledger_postgres.rs::active_candidate_can_land_at_proven_new_chain_revision`, `crates/qbit-prism-server/tests/ledger_postgres.rs::verified_landing_reconstructs_audit_and_reorgs_are_revision_fenced`, `crates/qbit-prism-server/tests/ledger_postgres.rs::deferred_share_survives_confirmation_crash_and_is_credited_once`, `crates/qbit-prism-server/tests/ledger_postgres.rs::late_acceptance_after_abandoned_claim_recovers_deferred_credit`, `crates/qbit-prism-server/tests/ledger_postgres.rs::stale_candidate_active_proof_cannot_overwrite_a_newer_reorg`, `crates/qbit-prism-server/src/coordinator/candidate_lease_tests.rs::landing_transaction_renews_across_expiries_and_terminal_contention_finishes_once`, `crates/qbit-prism-server/src/coordinator/candidate_lease_tests.rs::saturated_build_capacity_keeps_the_lease_until_one_candidate_confirms`, `crates/qbit-prism-server/src/coordinator/candidate_lease_tests.rs::canceled_or_lost_renewal_cannot_resume_queued_work`, `crates/qbit-prism-server/src/coordinator/candidate_lease_tests.rs::submit_loop_shutdown_drops_waiting_work_and_allows_recovery`, `crates/qbit-prism-server/src/coordinator/candidate_lease_tests.rs::token_loss_between_heartbeats_is_fenced_immediately_before_submitblock`, `crates/qbit-prism-server/src/coordinator/candidate_lease_tests.rs::early_chain_probe_skips_only_proven_stale_work_and_recovers_active_blocks`, `crates/qbit-prism-server/tests/support/live_highdiff.rs::real_highdiff_block_only_proof_waits_for_active_chain_credit`, `crates/qbit-prism-server/tests/stratum_protocol.rs::highdiff_floor_still_forwards_valid_block_below_share_target`, `crates/qbit-prism-server/tests/stratum_protocol.rs::multiple_miners_unique_extranonces_durable_accept_and_duplicate_rejection`, `crates/qbit-prism-server/tests/rpc_deadlines.rs::configuration_keeps_general_and_submit_deadlines_independent`, `crates/qbit-prism-server/tests/rpc_deadlines.rs::ordinary_read_can_outlive_submit_deadline_and_submit_is_never_retried`, `crates/qbit-prism-server/tests/readiness_rpc.rs::observed_readiness_failure_closes_cached_work_and_candidate_settlement`, `crates/qbit-prism-server/tests/support/ledger_2x.rs::native_publication_ordinal_survives_reactivation_and_excludes_rejected_candidates`, `crates/qbit-prism-server/tests/live_regtest.rs::real_two_server_mining_failover_audit_and_reorg`, `crates/qbit-prism-server/src/coordinator/d2_below_target_tests.rs::block_only_proof_is_credited_network_work_only_once_the_block_is_active`, `crates/qbit-prism-server/src/coordinator/d2_below_target_tests.rs::block_only_proof_with_a_lost_submitblock_reply_is_credited_once_by_reconciliation`, `crates/qbit-prism-server/src/coordinator/d2_below_target_tests.rs::block_only_credit_survives_a_reorg_that_disconnects_its_block`, `crates/qbit-prism-server/src/coordinator/d2_below_target_tests.rs::block_only_proof_the_node_rejects_fails_the_submission_without_credit`, `crates/qbit-prism-server/src/coordinator/d2_below_target_tests.rs::share_passing_block_proof_is_credited_its_assigned_difficulty_at_once` (the below-target cluster under the documented D2b rule, against PostgreSQL: durable intent before the offer, no credit until the block is active, one lost-ack block finalized by reconciliation, a published credit surviving a disconnecting reorg, a refused block earning nothing, a share-passing solve credited and enqueued at once), `crates/qbit-prism-server/tests/offer_latency.rs::offer_reaches_the_node_within_the_bound_while_builders_and_landing_are_held`, `crates/qbit-prism-server/src/coordinator/window_switch_tests.rs::leased_candidate_submits_first_then_lands_as_issued_before_any_terminal_outcome`, `crates/qbit-prism-server/src/coordinator/candidate_lease_tests.rs::crash_after_offer_before_landing_recovers_on_another_frontend_without_a_second_offer`, `crates/qbit-prism-server/src/coordinator/candidate_lease_tests.rs::accepted_landing_failure_reaches_reconciliation_with_a_reason_never_abandoned` (the offer-before-landing order of #391, replacing the three 2.x.x ordering tests #266 named by line, `test_submitter_offers_block_before_writer_admission_with_rpc_deadline`, `test_block_submitted_before_audit_bundle_build` and `test_block_candidate_submits_before_full_audit_persistence`: with every builder permit taken and the settlement lock held, `submitblock` reaches the node within 250 ms of the claim, exactly once, the row is `offered` with no audit row until the hold is released, and the same attempt then lands; a leased candidate has no audit row while its `submitblock` is in flight; a crash or a landing failure after the offer is finished by a later claim without a second `submitblock`; `saturated_build_capacity_keeps_the_lease_until_one_candidate_confirms` above now asserts one `submitblock` and no audit row while build capacity is held, not zero submissions), `crates/qbit-prism-server/src/coordinator/window_switch_tests.rs::a_parent_landing_late_diverges_the_child_which_lands_as_issued_with_additive_balances`, `crates/qbit-prism-server/tests/ledger_single_execution.rs::distinct_hashes_hold_independent_claims` (interleaved distinct-hash landings: a parent and its child found on two frontends are both offered before either lands, the parent lands and confirms while the child's landing is held, then the child lands, with two `submitblock` calls in total and an audit row for each block; a hash lands and finishes while a sibling's outbox row is locked, and the sibling's claim is untouched); not covered, no open issue: ancestor/descendant landing deferral, retry backoff cap and clear, malformed durable row quarantine, post-accept clean-job refresh, the cleanup-retry backlog bound cluster (#270 closed after the native storm scenarios landed), the below-target intent-failure retry slot, the moved-tip replay tail and the three-blocks-in-succession lost-ack cadence; the Python subprocess builder, B1 owner decomposition, in-memory queue lanes, floor holders, watchdog budgets, worker pools, startup pagination and the superseded live-envelope restore under a writer-lease proof are retired | +| `tests/test_prism_block_candidate_collapse.py` | partial | partial: `crates/qbit-prism-server/tests/candidate_storm.rs::siblings_behind_a_decided_height_drain_once_each_at_a_constant_node_cost` asserts the exact terminal sibling set, and `crates/qbit-prism-server/tests/candidate_storm.rs::claim_lanes_and_the_dispatch_probe_keep_their_indexes_over_unfinished_due_rows` asserts indexed claim lanes over the unfinished storm alongside retained history. This storm suite asserts the dispatch probe indexed at baseline and records its storm-size plan. The pre-#432 EXISTS probe could use a sequential scan at storm size; `candidate_dispatch_probe` now checks the ordered probe against exact unfinished/retained populations, including direct/prepared execution and churn. Restart no-double-offer safety is covered by `crates/qbit-prism-server/tests/candidate_storm_restart.rs::restart_between_claim_and_terminalization_offers_every_block_exactly_once`. The selector-specific cases and `S - oracle_abandoned == {}` retire while the set-oriented selector remains out of scope: there is no selected set S. Their safety purpose is carried by the durable no-resubmission lifecycle and the per-hash node counter, not an assertion of Python oracle equivalence. Cleanup-dependency recovery and offer provenance now have the small-population native cases listed in [Native storm evidence](#native-storm-evidence). This does not establish Python selector-oracle equivalence; failure injection at each chain RPC of one candidate observation is covered by `crates/qbit-prism-server/src/ledger/candidates/storm_fault_tests.rs::every_pre_offer_chain_observation_fault_leaves_the_row_unoffered` and `crates/qbit-prism-server/src/ledger/candidates/storm_fault_tests.rs::every_post_offer_chain_observation_fault_keeps_one_offer_and_reconciles`, which fault each of the six calls `observe_candidate` makes (`getblockchaininfo` and `getnetworkinfo` twice each, `getblockhash`, `getbestblockhash`), keyed by method and occurrence within the drain, with an RPC error and with a differing well-formed answer, before the offer and after it; `crates/qbit-prism-server/src/ledger/candidates/storm_fault_tests.rs::a_failed_block_hash_read_never_abandons_a_block_the_node_already_holds` covers the pre-offer `getblockhash` error over a block the node already holds, and `crates/qbit-prism-server/src/ledger/candidates/storm_fault_tests.rs::a_competitor_at_the_candidates_height_orphans_only_at_the_threshold` the post-offer competitor at five and at six confirmations of `PRISM_CANDIDATE_ORPHAN_CONFIRMATIONS`; not covered there: RPC timeouts, malformed JSON and partial reads, which `crates/qbit-prism-server/tests/rpc_deadlines.rs` and the RPC client's own tests own; `submitblock` reply classification, which `crates/qbit-prism-server/src/coordinator/storm_evidence_tests.rs` owns; and the operator `recover` path's `getblockheader`, which is not reachable from `process_candidate` | +| `tests/prism_candidate_storm.py` | retired | retired — the aggregate-replay measurement instrument and selector oracle have no native counterpart; the scenarios are mapped in `tests/test_prism_candidate_storm.py` below, rather than porting the Python harness | +| `tests/test_prism_candidate_storm.py` | partial | partial: live drain cost and exact sibling terminal sets are covered by `crates/qbit-prism-server/tests/candidate_storm.rs::siblings_behind_a_decided_height_drain_once_each_at_a_constant_node_cost` and `crates/qbit-prism-server/tests/candidate_storm.rs::drained_rows_cost_the_same_statements_at_the_baseline_and_storm_cardinalities`; admission by `crates/qbit-prism-server/tests/candidate_storm.rs::admission_serves_new_work_first_while_the_due_backlog_is_still_owed`; real process death, recovery and one `submitblock` per hash by `crates/qbit-prism-server/tests/candidate_storm_restart.rs::restart_between_claim_and_terminalization_offers_every_block_exactly_once`; held-lease offer/dispatch budgeting by `crates/qbit-prism-server/tests/candidate_storm_restart.rs::a_held_candidate_lease_costs_the_drain_an_offer_and_never_a_wakeup`. These use 100 candidates by default and 3,120 with `PRISM_TEST_STORM_CANDIDATES=3120`; wall time is recorded, not asserted. The four cleanup dependencies and distinct retry/lease/replay offer provenance are covered by [Native storm evidence](#native-storm-evidence), using small fixed populations. Post-restart wire-statement cost is covered by `crates/qbit-prism-server/tests/candidate_storm_restart.rs::a_relaunched_frontend_recovers_one_row_at_the_same_statement_cost_at_both_cardinalities`, which measures the statements a **relaunched process** spends recovering and finishing one row whose claimant died, at 24 rows and at the run's own cardinality in one test process, asserted equal and against a pinned 64; it attributes executions to that row by statement identity through `tests/support/ledger_execution_proxy.rs` — a relaunched frontend cannot be quiesced, so a wall-clock bracket would charge it for the background loops — and proves the attribution by requiring the classifier to select nothing at all from an idle window. The capacity-closed condition is retired under owner decision D-B on #270 (2026-09-18): no native condition closes capacity on a pending candidate. The native pre-offer terminal refusal is covered by `crates/qbit-prism-server/src/coordinator/storm_evidence_tests.rs::a_pre_offer_supersession_terminalizes_without_any_call`. Not covered, no open issue: the per-distinct-height probe bound is not a native guarantee (the native path probes each sibling). Python marker-cardinality, selector-evidence gauges, preserved wakeups, drain-local transition reporting, selector fault-vocabulary, queued-holder view and inline/deferred submitter accounting retire with those mechanisms | +| `tests/test_prism_block_finalization.py` | partial | partial: covered by `crates/qbit-prism-server/src/coordinator/candidate_lease_tests.rs::landing_transaction_renews_across_expiries_and_terminal_contention_finishes_once`, `crates/qbit-prism-server/tests/ledger_postgres.rs::verified_landing_reconstructs_audit_and_reorgs_are_revision_fenced`; not covered: the candidate interarrival histogram (no native series); the Python port-surface AST tests are retired | +| `tests/test_prism_accepted_block_accounting.py` | partial | partial: covered by `crates/qbit-prism-server/tests/ledger_postgres.rs::candidate_dispatch_prioritizes_fresh_work_and_services_oldest_due_fairly`, `crates/qbit-prism-server/tests/support/live_highdiff.rs::real_highdiff_block_only_proof_waits_for_active_chain_credit`, `crates/qbit-prism-server/src/coordinator/d2_below_target_tests.rs::block_only_proof_with_a_lost_submitblock_reply_is_credited_once_by_reconciliation` (a transport-failed `submitblock` is not definitive acceptance: the block stays `prepared`, uncredited and unacknowledged until reconciliation observes it); not covered: none with a native subject; the Python accounting-lane queue, spillover ordering and dispatch-quota mechanics are retired | +| `tests/test_prism_ancestor_redrive.py` | retired | retired — the retained-retry loop, replay-pass short-circuits and separate preview publication it re-drives do not exist natively; landing confirms and bumps the revision in one transaction and failed rows are re-claimed from the outbox | +| `tests/test_prism_dequeue_stale_skip.py` | partial | partial: covered by `crates/qbit-prism-server/src/coordinator/candidate_lease_tests.rs::early_chain_probe_skips_only_proven_stale_work_and_recovers_active_blocks` and `crates/qbit-prism-server/tests/candidate_storm.rs::siblings_behind_a_decided_height_drain_once_each_at_a_constant_node_cost` (exact terminal sibling set at storm cardinality, rather than execution of the retired Python selector oracle); [Native storm evidence](#native-storm-evidence) additionally covers failed/shifting chain observation and offer provenance; failure injection at each chain RPC of one candidate observation is covered by `crates/qbit-prism-server/src/ledger/candidates/storm_fault_tests.rs::every_pre_offer_chain_observation_fault_leaves_the_row_unoffered` and `crates/qbit-prism-server/src/ledger/candidates/storm_fault_tests.rs::every_post_offer_chain_observation_fault_keeps_one_offer_and_reconciles`, which fault each of the six calls `observe_candidate` makes (`getblockchaininfo` and `getnetworkinfo` twice each, `getblockhash`, `getbestblockhash`), keyed by method and occurrence within the drain, with an RPC error and with a differing well-formed answer, before the offer and after it; `crates/qbit-prism-server/src/ledger/candidates/storm_fault_tests.rs::a_failed_block_hash_read_never_abandons_a_block_the_node_already_holds` covers the pre-offer `getblockhash` error over a block the node already holds, and `crates/qbit-prism-server/src/ledger/candidates/storm_fault_tests.rs::a_competitor_at_the_candidates_height_orphans_only_at_the_threshold` the post-offer competitor at five and at six confirmations of `PRISM_CANDIDATE_ORPHAN_CONFIRMATIONS`; not covered there: RPC timeouts, malformed JSON and partial reads, which `crates/qbit-prism-server/tests/rpc_deadlines.rs` and the RPC client's own tests own; `submitblock` reply classification, which `crates/qbit-prism-server/src/coordinator/storm_evidence_tests.rs` owns; and the operator `recover` path's `getblockheader`, which is not reachable from `process_candidate` | +| `tests/test_prism_pending_block_recovery.py` | partial | partial: covered by `crates/qbit-prism-server/tests/ledger_postgres.rs::active_candidate_can_land_at_proven_new_chain_revision`, `crates/qbit-prism-server/tests/ledger_postgres.rs::late_acceptance_after_abandoned_claim_recovers_deferred_credit`, `crates/qbit-prism-server/tests/ledger_postgres.rs::candidate_renewal_requires_a_live_pending_token`, `crates/qbit-prism-server/tests/ledger_postgres.rs::candidate_outbox_is_atomic_and_claims_recover_after_owner_loss`, `crates/qbit-prism-server/src/coordinator/candidate_lease_tests.rs::early_chain_probe_skips_only_proven_stale_work_and_recovers_active_blocks`, `crates/qbit-prism-server/src/coordinator/candidate_lease_tests.rs::canceled_or_lost_renewal_cannot_resume_queued_work`, `crates/qbit-prism-server/src/coordinator/d2_below_target_tests.rs::block_only_proof_is_credited_network_work_only_once_the_block_is_active` (claim, `process_candidate` and the terminal `submitted` state driven by hand, credit landing with confirmation), `crates/qbit-prism-server/tests/support/candidates_cli.rs::list_shows_every_unfinished_state_without_claiming_or_loading_payloads`, `crates/qbit-prism-server/tests/support/candidates_cli.rs::list_separates_parked_rows_from_retrying_rows_and_shows_unknown_storage_versions`, `crates/qbit-prism-server/tests/support/candidates_cli.rs::abandon_terminalizes_a_pending_row_exactly_as_supersession_does`, `crates/qbit-prism-server/tests/support/candidates_cli.rs::abandon_refuses_every_offered_state_and_leaves_the_row_byte_identical`, `crates/qbit-prism-server/tests/support/candidates_cli.rs::abandon_refuses_a_live_foreign_claim_and_succeeds_once_it_expires`, `crates/qbit-prism-server/tests/support/candidates_cli.rs::abandon_refuses_a_terminal_row_and_a_pending_row_whose_block_landed` (#268's `candidates list` and `candidates abandon`: a read-only inventory that loads no payload column and takes no claim, and the one operator abandon, which applies to `pending` alone, refuses a live claim, a landed block and every offered state, and calls the node zero times), `crates/qbit-prism-server/tests/support/candidates_recover.rs::recover_plan_orders_parent_before_child_without_claiming_or_loading_payloads`, `crates/qbit-prism-server/tests/support/candidates_recover.rs::recover_plan_fails_closed_on_missing_terminal_inactive_legacy_and_unlisted_parents`, `crates/qbit-prism-server/tests/support/candidates_recover.rs::recover_apply_lands_the_listed_blocks_and_verifies_completed_ones_idempotently`, `crates/qbit-prism-server/tests/support/candidates_recover.rs::recover_apply_refuses_a_live_foreign_claim_and_leaves_the_row_untouched`, `crates/qbit-prism-server/tests/support/candidates_recover.rs::recover_apply_enforces_the_deadline_and_leaves_the_candidate_recoverable` (#418's `candidates recover`: a plan that orders parent before child, claims nothing and loads no payload, fails closed on every listed hash, an `--apply` that lands through the real rebuild and skips completed blocks idempotently, refuses a live foreign claim, honours one deadline, and never calls `submitblock`), `crates/qbit-prism-server/tests/support/ledger_2x.rs::unknown_storage_version_row_is_parked_and_not_reclaimed_at_lease_expiry`, `crates/qbit-prism-server/tests/support/ledger_2x.rs::frozen_258_source_refuses_pending_v1_and_v2_rows_and_migrates_terminal_rows`, `crates/qbit-prism-server/tests/support/ledger_2x.rs::newer_storage_version_or_capability_is_refused_at_migrate_and_at_connect`, `crates/qbit-prism-server/tests/support/ledger_2x.rs::startup_without_initialize_requires_the_current_schema_version`, `crates/qbit-prism-server/tests/support/candidates_cli.rs::abandon_refuses_unsupported_storage_versions_without_changing_evidence` (#285's storage-version and schema refusals: the claim lane parks a version-2 or unknown-version row with the reason in `last_error` and `next_attempt_at = infinity` and does not re-claim it at lease expiry, `migrate` refuses a pending version-2 row and a newer declared version before any DDL, a start without `initialize` refuses a 001-only database naming the missing migrations, and `abandon` refuses versions 2 and 3 with exit 7 and the row byte-identical, as `recover` does for a version-2 row in the plan test above); not covered, no open issue: the reader half of the legacy-schema refusal — `candidates list` and `candidates recover` planning open a read-only pool without the startup schema gate, and no test runs either against a database that lacks the native migrations (#285 closed without it); the cooperative-JSON codec, chunked-body hydration and Python config-derivation cases are retired (per-case mapping below) | +| `tests/test_prism_reorg_reconciler.py` | partial | partial: covered by `crates/qbit-prism-server/tests/ledger_postgres.rs::verified_landing_reconstructs_audit_and_reorgs_are_revision_fenced`, `crates/qbit-prism-server/tests/ledger_postgres.rs::stale_candidate_active_proof_cannot_overwrite_a_newer_reorg`, `crates/qbit-prism-server/tests/ledger_postgres.rs::reconciliation_watches_only_recent_blocks_and_one_mature_checkpoint`, `crates/qbit-prism-server/tests/ledger_postgres.rs::late_acceptance_after_abandoned_claim_recovers_deferred_credit`, `crates/qbit-prism-server/tests/support/ledger_2x.rs::native_publication_ordinal_survives_reactivation_and_excludes_rejected_candidates`, `crates/qbit-prism-server/tests/readiness_rpc.rs::observed_readiness_failure_closes_cached_work_and_candidate_settlement`, `crates/qbit-prism-server/tests/live_regtest.rs::real_two_server_mining_failover_audit_and_reorg`, `crates/qbit-prism-server/src/coordinator/d2_below_target_tests.rs::block_only_credit_survives_a_reorg_that_disconnects_its_block` (a reorg observed by `refresh_once` marks the pool block inactive and keeps its published credit); not covered: a stranded prepared row (owner lost, no outbox row) being rejected or confirmed by the reconcile pass; the trust memo, flight admission and serialized-adapter mechanics are retired | +| `tests/prism_regtest_crash_coordinator.py` | partial | partial: covered by `crates/qbit-prism-server/src/coordinator/candidate_lease_tests.rs::submit_loop_shutdown_drops_waiting_work_and_allows_recovery`, `crates/qbit-prism-server/src/coordinator/candidate_lease_tests.rs::canceled_or_lost_renewal_cannot_resume_queued_work`, `crates/qbit-prism-server/tests/ledger_postgres.rs::candidate_outbox_is_atomic_and_claims_recover_after_owner_loss`; a real native process killed between durable candidate publication and its first offer is covered by `crates/qbit-prism-server/tests/candidate_storm_restart.rs::a_kill_before_the_first_offer_recovers_as_one_known_offer`, which kills a real `qbit-prism-server run` child parked inside the pre-offer `getblockhash` — after the durable claim, before `Ledger::reserve_offer` commits — and proves the relaunched process offers that block exactly once with known evidence. The kill is deterministic and lives in the required-CI database suite rather than inside the node-gated regtest harness (owner decision D-C): the property belongs to the durable candidate outbox, not to the node, so nothing it asserts needs `qbitd`. Not covered: the same kill observed against a live regtest node, and the rest of the Python harness's coordinator-crash surface | +| `tests/prism_regtest_share_miner.py` | full | `crates/qbit-prism-server/tests/live_regtest.rs::real_two_server_mining_failover_audit_and_reorg` | + +### `tests/test_prism_pending_block_recovery.py`, case by case + +The row above stays **partial**. The cases enumerated here are those of the +file at **504846c** (v2.0.2, #258), the `origin/2.x.x` revision this document +pins throughout: seventeen test methods plus the `test_config` helper. Three of +them — `test_chunked_candidate_row_is_hydrated_not_misclassified`, +`test_unsupported_storage_version_and_unsealed_body_fail_closed` and +`test_legacy_only_schema_is_refused_by_the_reader_and_the_ledger` — exist only +from #258 (v2.0.2) onward, so a checkout of v2.0.1 (`95ffe06`) shows fourteen +cases and none of those three. Read the file at 504846c, not at the v2.0.1 +commit #268's own reference link uses. + +The `2.x.x` file mixed the offline recovery runner with a cooperative JSON +codec and a native PostgreSQL reader, and the three retired, moved and were +closed separately. #268 shipped `candidates list` and `candidates abandon`; its +P2 follow-up +[#418](https://github.com/Qbit-Org/qbit-mining-bootstrap/issues/418) shipped +`recover --apply`, whose tests live in +`crates/qbit-prism-server/tests/support/candidates_recover.rs` (module +`candidates_recover` of the `ledger_postgres` binary), so every plan-level +case below is covered. #285 (closed by #321) shipped the storage-version +refusals and the startup schema gate, whose tests live in +`crates/qbit-prism-server/tests/support/ledger_2x.rs` (module `two_x` of the +same binary), so the storage-version cases are covered too. What remains is +the reader half of the legacy-schema case: the read-only `candidates list` +and `candidates recover` planning connect without the startup schema gate, +and no test runs them against an unmigrated database. + +| 2.x.x case | Covered | 3.x.x replacement or status | +| --- | --- | --- | +| `test_plan_orders_parent_before_child_without_loading_payloads` | full | `crates/qbit-prism-server/tests/support/candidates_recover.rs::recover_plan_orders_parent_before_child_without_claiming_or_loading_payloads` (seeded pending candidates given out of order print in height order, parent before child; every row is byte-identical afterwards, the dispatch sequence is unmoved, and the plan statement described through `pool.describe()` projects neither `candidate` nor `block_bytes`; works while the cluster is halted), `crates/qbit-prism-server/tests/support/candidates_cli.rs::list_shows_every_unfinished_state_without_claiming_or_loading_payloads` (the same no-payload projection for the inventory) | +| `test_missing_pending_parent_rejects_plan` | full | `crates/qbit-prism-server/tests/support/candidates_recover.rs::recover_plan_fails_closed_on_missing_terminal_inactive_legacy_and_unlisted_parents` (an unlisted unfinished parent refuses the whole plan with exit 10 and a missing row with exit 2, every row left byte-identical), `crates/qbit-prism-server/tests/support/candidates_cli.rs::abandon_refuses_a_terminal_row_and_a_pending_row_whose_block_landed` (the same missing-row exit status for `abandon`) | +| `test_duplicate_missing_abandoned_and_inactive_fail_closed` | full | `crates/qbit-prism-server/tests/support/candidates_recover.rs::recover_plan_fails_closed_on_missing_terminal_inactive_legacy_and_unlisted_parents` (duplicates, 33 hashes and a malformed hash are usage errors before any connection is opened; a missing row exits 2, an abandoned row 4, a block not on the active chain 9; every refusal leaves every row byte-identical), `crates/qbit-prism-server/tests/support/candidates_cli.rs::abandon_refuses_a_terminal_row_and_a_pending_row_whose_block_landed` (an already-terminal row is refused and never re-applied by `abandon`) | +| `test_submitted_without_complete_accounting_is_not_skipped` | full | `crates/qbit-prism-server/tests/support/candidates_recover.rs::recover_plan_fails_closed_on_missing_terminal_inactive_legacy_and_unlisted_parents` (a `submitted` row without its audit row is refused with exit 4 rather than skipped as complete; `candidates abandon` never selects a `submitted` row at all) | +| `test_payload_identity_height_and_header_are_verified` | full | `crates/qbit-prism-server/tests/support/candidates_recover.rs::recover_plan_fails_closed_on_missing_terminal_inactive_legacy_and_unlisted_parents` (a stored height that disagrees with the node's is refused with exit 10), `crates/qbit-prism-server/tests/support/candidates_recover.rs::recover_apply_lands_the_listed_blocks_and_verifies_completed_ones_idempotently` (real candidates land only through the claim lane's own decode and authentication and the node verification), `crates/qbit-prism-server/tests/ledger_postgres.rs::candidate_outbox_is_atomic_and_claims_recover_after_owner_loss` (the claim authenticates the document digest, the block digest and the window reference before any rebuild) | +| `test_chunked_candidate_row_is_hydrated_not_misclassified` | retired | retired — `3.x.x` never imports a #258 chunked body, so there is no hydration to assert; such a row is drained by the pinned `2.x.x` image before migration. Refusal by version replaces it: `migrate` refuses a pending version-2 row by hash and `storage_version` with the drain procedure, and a version-2 row that appears after cutover is parked by the claim lane, in `crates/qbit-prism-server/tests/support/ledger_2x.rs::frozen_258_source_refuses_pending_v1_and_v2_rows_and_migrates_terminal_rows`; `candidates recover` refuses one with exit 7 and the evidence preserved in `crates/qbit-prism-server/tests/support/candidates_recover.rs::recover_plan_fails_closed_on_missing_terminal_inactive_legacy_and_unlisted_parents` | +| `test_unsupported_storage_version_and_unsealed_body_fail_closed` | partial | partial: covered by `crates/qbit-prism-server/tests/support/ledger_2x.rs::unknown_storage_version_row_is_parked_and_not_reclaimed_at_lease_expiry` (a `storage_version = 3` row is not claimed; it is parked with the version named in `last_error`, `next_attempt_at = infinity` and no claim token, neither instance re-claims it once its lease has expired, ordinary work still drains, and resetting `next_attempt_at` re-offers it), `crates/qbit-prism-server/tests/support/ledger_2x.rs::frozen_258_source_refuses_pending_v1_and_v2_rows_and_migrates_terminal_rows` (`migrate` refuses a pending version-2 row before any DDL, and one inserted after cutover is parked the same way), `crates/qbit-prism-server/tests/support/ledger_2x.rs::newer_storage_version_or_capability_is_refused_at_migrate_and_at_connect` (a declared `candidate_storage_version = 3` is refused before any DDL and at connect, with and without `initialize`), `crates/qbit-prism-server/tests/support/candidates_recover.rs::recover_plan_fails_closed_on_missing_terminal_inactive_legacy_and_unlisted_parents` (`recover` refuses a version-2 row with exit 7 and the whole plan with it), `crates/qbit-prism-server/tests/support/candidates_cli.rs::abandon_refuses_unsupported_storage_versions_without_changing_evidence` (`abandon` refuses versions 2 and 3, parked or not and even under an expired claim, with exit 7 and the row byte-identical), `crates/qbit-prism-server/tests/support/candidates_cli.rs::list_separates_parked_rows_from_retrying_rows_and_shows_unknown_storage_versions` (the parked row is listed with its `storage_version` and the `last_error` naming it, distinctly from a retrying row); the unsealed-body and missing-body sub-cases are retired: no native code reads a chunk body, so a version-2 row is refused by its version whatever its body state | +| `test_apply_finalizes_only_after_normal_accounting_and_checks_durable_result` | full | `crates/qbit-prism-server/tests/support/candidates_recover.rs::recover_apply_lands_the_listed_blocks_and_verifies_completed_ones_idempotently` (a real pending candidate and a real reconciliation candidate land through the real rebuild and `land_candidate_at_revision`, the pending one adopted first; the rows end `submitted` with confirmed pool blocks, audit rows and payout entries; a second `--apply` prints `verified ... already complete` and changes nothing; exactly the listed hashes are touched), `crates/qbit-prism-server/tests/support/candidates_cli.rs::abandon_terminalizes_a_pending_row_exactly_as_supersession_does` (the durable result decides the outcome — the statement's own `rows_affected` does, and the diagnosis that produces the message runs after it and gates nothing — and the finished row compares equal to one the epoch-supersession statement abandoned) | +| `test_accounting_failure_and_lost_lease_leave_outbox_pending` | full | `crates/qbit-prism-server/tests/support/candidates_recover.rs::recover_apply_enforces_the_deadline_and_leaves_the_candidate_recoverable` (a landing paused inside the database past `--timeout-seconds` exits 11, the claim is released with the deadline reason in `last_error`, no pool block was committed, and the same allowlist then succeeds with an adequate deadline), `crates/qbit-prism-server/tests/support/candidates_recover.rs::recover_apply_refuses_a_live_foreign_claim_and_leaves_the_row_untouched` (a row another instance holds is refused with exit 5 and left byte-identical, and a claimed row this binary cannot authenticate is released with the reason in `last_error`, its schedule and evidence untouched, never parked or abandoned), `crates/qbit-prism-server/tests/support/candidates_cli.rs::abandon_terminalizes_a_pending_row_exactly_as_supersession_does` (a halted cluster and a live legacy writer lease each leave the row byte-identical), `crates/qbit-prism-server/tests/support/candidates_cli.rs::abandon_refuses_every_offered_state_and_leaves_the_row_byte_identical` (every refusal leaves it byte-identical) | +| `test_recovery_config_preserves_lease_policy_and_signing_identity` | retired | retired — `list` and `abandon` read `PRISM_DATABASE_URL` and nothing else, while `recover --apply` loads the frontend configuration and is fenced by the durable claim, not a writer lease; the Python config derivation itself has no native counterpart. That none of the three registers an instance is asserted in every `candidates_cli` and `candidates_recover` test | +| `test_runner_never_offers_new_blocks` | full | `crates/qbit-prism-server/tests/support/candidates_cli.rs::list_shows_every_unfinished_state_without_claiming_or_loading_payloads`, `crates/qbit-prism-server/tests/support/candidates_cli.rs::list_separates_parked_rows_from_retrying_rows_and_shows_unknown_storage_versions`, `crates/qbit-prism-server/tests/support/candidates_cli.rs::abandon_terminalizes_a_pending_row_exactly_as_supersession_does`, `crates/qbit-prism-server/tests/support/candidates_cli.rs::abandon_refuses_every_offered_state_and_leaves_the_row_byte_identical`, `crates/qbit-prism-server/tests/support/candidates_cli.rs::abandon_refuses_a_live_foreign_claim_and_succeeds_once_it_expires`, `crates/qbit-prism-server/tests/support/candidates_cli.rs::abandon_refuses_a_terminal_row_and_a_pending_row_whose_block_landed` — each runs the real binary against a recorder in front of the fake node and asserts it made no RPC call of any kind; `crates/qbit-prism-server/tests/support/candidates_recover.rs::recover_plan_orders_parent_before_child_without_claiming_or_loading_payloads`, `crates/qbit-prism-server/tests/support/candidates_recover.rs::recover_plan_fails_closed_on_missing_terminal_inactive_legacy_and_unlisted_parents`, `crates/qbit-prism-server/tests/support/candidates_recover.rs::recover_apply_lands_the_listed_blocks_and_verifies_completed_ones_idempotently`, `crates/qbit-prism-server/tests/support/candidates_recover.rs::recover_apply_refuses_a_live_foreign_claim_and_leaves_the_row_untouched`, `crates/qbit-prism-server/tests/support/candidates_recover.rs::recover_apply_enforces_the_deadline_and_leaves_the_candidate_recoverable` — each runs the real binary against a scripted fake node that counts every RPC method and asserts zero `submitblock` calls (and, in plan mode, only `getblockheader` and `getblockhash`) | +| `test_canonical_bytes_identity_and_parser_behavior_match_stdlib` | retired | retired — the cooperative JSON codec is Python-only | +| `test_codec_restores_after_error` | retired | retired — as above | +| `test_exclusive_guard_refuses_live_writer_and_reacquires_after_release` | full | `crates/qbit-prism-server/tests/support/candidates_cli.rs::abandon_terminalizes_a_pending_row_exactly_as_supersession_does` (the operator abandon refuses while a legacy writer lease is live and succeeds once it is gone), `crates/qbit-prism-server/tests/support/candidates_recover.rs::recover_apply_refuses_a_live_foreign_claim_and_leaves_the_row_untouched` (`recover --apply` refuses a row another instance holds with exit 5, leaves it byte-identical, and succeeds once the claim expires — the durable claim is the native guard, not a writer lease), `crates/qbit-prism-server/tests/ledger_postgres.rs::migration_refuses_an_active_legacy_writer` | +| `test_reader_is_read_only_and_loads_only_selected_candidate_with_cooperative_codec` | partial | partial: covered by `crates/qbit-prism-server/tests/support/candidates_cli.rs::list_shows_every_unfinished_state_without_claiming_or_loading_payloads`, which runs the inventory on a `default_transaction_read_only=on` pool while the cluster is halted, leaves every row byte-identical, consumes no dispatch slot and projects neither payload column; the cooperative-codec half is retired | +| `test_legacy_only_schema_is_refused_by_the_reader_and_the_ledger` | partial | partial: the ledger half is covered by `crates/qbit-prism-server/tests/support/ledger_2x.rs::startup_without_initialize_requires_the_current_schema_version` (a 001-only database started without `initialize` is refused at connect, naming the required migrations and `qbit-prism-server migrate`, and no DDL runs; a native database missing one migration is refused the same way, naming it); not covered, no open issue: the reader half — `candidates list` and `candidates recover` planning open a read-only pool without the startup schema gate, and no test runs either against a database that lacks the native migrations (#285 closed without it) | +| `test_real_finalizer_persists_confirms_publishes_and_completes_outbox` | full | `crates/qbit-prism-server/tests/ledger_postgres.rs::verified_landing_reconstructs_audit_and_reorgs_are_revision_fenced`, `crates/qbit-prism-server/src/coordinator/d2_below_target_tests.rs::block_only_proof_is_credited_network_work_only_once_the_block_is_active`, `crates/qbit-prism-server/tests/support/candidates_recover.rs::recover_apply_lands_the_listed_blocks_and_verifies_completed_ones_idempotently` (the operator command finishes the rows through the real `land_candidate_at_revision` and `finish_candidate_at_revision`: confirmed pool blocks, audit rows, payout entries and `submitted` outbox rows) | + +## Bounded candidate storage (#255 chunked bodies, #258 storage version 2) + +The 2.x.x chunked candidate body store, its codec, spool views and streamed consumers. The native outbox stores one JSONB `Candidate`, so the Python storage mechanics are retired; what survives is the migrator's contract for the 002 chunk schema and version-2 rows and the startup schema gate, both asserted in `crates/qbit-prism-server/tests/support/ledger_2x.rs` (#285, closed by #321). + +| 2.x.x file | Covered | 3.x.x replacement or status | +| --- | --- | --- | +| `test/test-prism-postgres-candidate-storage.sh` | retired | retired — Docker/psql wrapper for the Python chunked-body gate; native PostgreSQL tests are driven by `test/prism-native-tests.sh` | +| `tests/prism_postgres_candidate_gate.py` | partial | partial: the migration-time contract for the 002 chunk schema and version-2 rows is covered by `crates/qbit-prism-server/tests/support/ledger_2x.rs::frozen_258_source_refuses_pending_v1_and_v2_rows_and_migrates_terminal_rows` (over the frozen 001 + 002: pending version-1 and version-2 rows are both refused by hash before any DDL and the version-2 row alone still blocks; terminal rows of both versions migrate with `retired_body_id` kept; the declared version 2 is recorded as source evidence; native writers persist, claim, land and finish version-1 rows under 002's dual-format rule; a version-2 row inserted after cutover is parked), `crates/qbit-prism-server/tests/support/ledger_2x.rs::partial_002_source_is_refused_naming_the_missing_object` (body tables without the capability row, the reverse, and a missing 002 trigger are each refused by name; re-applying the release 002 makes the source acceptable), `crates/qbit-prism-server/tests/support/ledger_2x.rs::drifted_002_object_is_refused_on_a_258_source` (a replaced 002 function body or a disabled chunk guard trigger refuses the migration), `crates/qbit-prism-server/tests/support/ledger_2x.rs::newer_storage_version_or_capability_is_refused_at_migrate_and_at_connect`, `crates/qbit-prism-server/tests/support/ledger_2x.rs::pre_006_native_schema_on_a_258_source_refuses_a_pending_v2_row_before_any_ddl`, `crates/qbit-prism-server/tests/support/ledger_2x.rs::live_sql_stays_pinned_to_the_frozen_2x_release` (the carried `002_candidate_bodies.sql` stays byte-identical to the v2.0.2 release); the Python chunked body store itself is retired: staging and publication, header pages, hydration and the spool, the seal and retirement triggers driven through the writer, and the janitor have no native caller, because the native outbox never writes or reads a chunk body | +| `tests/test_prism_candidate_codec.py` | retired | retired — the chunked candidate codec, spool index and v1 identity oracle are Python-only; the native outbox stores one JSONB `Candidate` with a serde-derived `candidate_sha256` | +| `tests/test_prism_candidate_helper.py` | retired | retired — `LegacyCandidateHelper` psql child-process lifecycle no longer exists | +| `tests/test_prism_candidate_replay_bounds.py` | retired | retired — replay header pages and descriptor backpressure are gone; the native outbox claims one row per query | +| `tests/test_prism_candidate_spool_integration.py` | retired | retired — lazy spool-backed candidate fields are Python-only | +| `tests/test_prism_candidate_spool_view.py` | retired | retired — spool views, streamed strings and helper supervision are Python-only | +| `tests/test_prism_bounded_canonical_finalization.py` | partial | partial: covered by `crates/qbit-prism/tests/audit_parts.rs::parts_path_matches_owned_path_byte_for_byte`, `crates/qbit-prism/tests/audit_parts.rs::builders_never_copy_the_window`, `crates/qbit-prism/tests/audit_parts.rs::parts_verifier_reads_the_lent_window`, `crates/qbit-prism/tests/audit_parts.rs::audit_bundle_body_rejects_a_full_bundle`, `crates/qbit-prism/tests/audit_cli.rs::build_audit_bundle_serve_mode_caches_parsed_windows`, `crates/qbit-prism-server/tests/ledger_postgres.rs::verified_landing_reconstructs_audit_and_reorgs_are_revision_fenced`, `crates/qbit-prism-server/tests/support/ledger_2x.rs::canonical_2x_sidecar_import_preserves_exact_bytes_and_fails_closed`; not covered: the Python compiler/store seam (streamed builder input, bounded view, tampered or foreign artifact rejection at publication, inode binding, scratch-pressure classification, generation release), retired with the filesystem audit store | +| `tests/test_prism_streamed_submitblock.py` | retired | retired — native `submitblock` sends the in-memory `Candidate.block_hex`; no on-disk block spool exists (deadline behaviour is in `crates/qbit-prism-server/tests/rpc_deadlines.rs::ordinary_read_can_outlive_submit_deadline_and_submit_is_never_retried`) | +| `tests/test_prism_streamed_block_hex_consumers.py` | retired | retired — streamed string views over hydrated bodies are Python-only | +| `tests/test_prism_candidate_storage_races.py` | partial | partial: covered by `crates/qbit-prism-server/tests/ledger_postgres.rs::candidate_outbox_is_atomic_and_claims_recover_after_owner_loss`, `crates/qbit-prism-server/tests/ledger_postgres.rs::simultaneous_block_only_enqueue_has_one_winner`; not covered: chunk/page seal races, orphan-body retirement races, the JSONB content comparator and the retired-body restage, all retired with the chunked body store | +| `tests/test_prism_candidate_schema_refusal.py` | partial | partial: the ledger's rule is covered by `crates/qbit-prism-server/tests/support/ledger_2x.rs::startup_without_initialize_requires_the_current_schema_version` (an unmigrated 001-only database is refused at a start without `initialize`, naming the required migrations, and no DDL runs), `crates/qbit-prism-server/tests/support/ledger_2x.rs::startup_without_initialize_refuses_a_pre_006_native_schema_with_009`, `crates/qbit-prism-server/tests/support/ledger_2x.rs::migrated_database_without_its_capability_declaration_is_refused_at_connect_and_at_migrate` (a schema whose capability row or table never landed is refused with and without `initialize`, unchanged), `crates/qbit-prism-server/tests/support/ledger_2x.rs::partial_002_source_is_refused_naming_the_missing_object` (body tables without the capability row, and the reverse, refused at `migrate`), `crates/qbit-prism-server/tests/support/ledger_2x.rs::newer_storage_version_or_capability_is_refused_at_migrate_and_at_connect` (the migrated schema is accepted and a declared version 3 or an unknown capability is refused before any DDL and at connect), `crates/qbit-prism-server/tests/support/ledger_2x.rs::native_candidate_version_two_is_refused_without_claiming_or_migrating`; not covered, no open issue: the recovery reader applying the same rule, and the plan refusing before it reads a candidate or calls the node — `candidates list` and `candidates recover` planning open a read-only pool without the startup schema gate, and no test runs either against an unmigrated or newer-capability database (#285 closed without it); the per-`PRISM_CANDIDATE_STORAGE_VERSION` sub-cases are retired (native writers store version 1 only) | + +## Writer lease, lease heartbeat and the concurrency harnesses + +3.x.x has no singleton writer lease: migration 002 rejects writes to `qbit_ledger_writer_lease`, instances coordinate through advisory locks and per-instance heartbeat rows, and the only lease that survives is the per-candidate claim (`claim_token`, renewal in `ledger/candidates.rs`). Rows below are mapped onto that claim lease where the property carries over; heartbeat timing policy and the deterministic Python harnesses are retired. + +| 2.x.x file | Covered | 3.x.x replacement or status | +| --- | --- | --- | +| `tests/prism_concurrency_harness.py` | retired | retired — Python baton-scheduler/virtual-clock/FakePostgres harness; native interleavings run on real PostgreSQL with `pg_advisory_xact_lock` triggers (`crates/qbit-prism-server/src/coordinator/candidate_lease_tests.rs`) | +| `tests/test_prism_concurrency_harness.py` | retired | retired — self-tests of the Python harness (SQL classifier, port signatures of `PsqlShareLedger`, scheduler determinism, fake tuple/advisory-lock model); none of those subjects exist natively | +| `tests/prism_landing_harness.py` | retired | retired — Python landing-tail harness over `BlockCandidateService`/`FakePostgres`; native landing runs against a real database | +| `tests/test_prism_landing_harness_model.py` | retired | retired — self-tests of the fake landing model; the one production property it modelled (publication ordinal survives reactivation, rejected rows get none) is asserted natively in `crates/qbit-prism-server/tests/support/ledger_2x.rs::native_publication_ordinal_survives_reactivation_and_excludes_rejected_candidates` | +| `tests/test_prism_lease_heartbeat_tail_latency.py` | partial | partial: covered by `crates/qbit-prism-server/src/coordinator/candidate_lease_tests.rs::landing_transaction_renews_across_expiries_and_terminal_contention_finishes_once`, `crates/qbit-prism-server/src/coordinator/candidate_lease_tests.rs::token_loss_between_heartbeats_is_fenced_immediately_before_submitblock`, `crates/qbit-prism-server/src/coordinator/candidate_lease_tests.rs::canceled_or_lost_renewal_cannot_resume_queued_work`; not covered: the #212 heartbeat timing policy, scheduler-stall envelope and the "unsafe policy refuses to arm" check (Python heartbeat service, no native subject) | +| `tests/test_prism_lease_heartbeat_under_fenced_write.py` | partial | partial: covered by `crates/qbit-prism-server/tests/ledger_postgres.rs::candidate_processing_lock_allows_renewal_and_blocks_expired_takeover`, `crates/qbit-prism-server/src/coordinator/candidate_lease_tests.rs::landing_transaction_renews_across_expiries_and_terminal_contention_finishes_once`, `crates/qbit-prism-server/src/coordinator/candidate_lease_tests.rs::canceled_or_lost_renewal_cannot_resume_queued_work`; not covered: guard query-slot serialisation and the pre-fix queueing-renewal control (Python guard connection, no native subject) | +| `tests/test_prism_lease_monitor_stalls.py` | partial | partial: covered by `crates/qbit-prism-server/tests/observability/runtime.rs::blocked_critical_poll_degrades_real_http_while_another_worker_remains_healthy`, `crates/qbit-prism-server/tests/observability/runtime.rs::idle_slow_async_work_and_cancelled_operations_do_not_leave_false_stalls`, `crates/qbit-prism-server/src/metrics/runtime.rs::progress_is_monotonic_per_operation_and_idle_is_not_stalled`, `crates/qbit-prism-server/src/metrics/runtime.rs::completed_poll_buckets_expire_with_at_most_one_extra_second`, `crates/qbit-prism-server/src/metrics/runtime.rs::completed_poll_survives_expiry_of_a_larger_predecessor`; not covered: GC-pause histograms, stack-sample stall probe, heartbeat-monitor lateness hard-exit (Python interpreter mechanics, no native subject) | +| `tests/test_prism_lease_orphaned_lock.py` | partial | partial: covered by `crates/qbit-prism-server/tests/ledger_postgres.rs::candidate_processing_lock_allows_renewal_and_blocks_expired_takeover`, `crates/qbit-prism-server/tests/ledger_postgres.rs::candidate_mutations_reject_a_token_taken_over_while_waiting_for_its_row`, `crates/qbit-prism-server/src/coordinator/candidate_lease_tests.rs::canceled_or_lost_renewal_cannot_resume_queued_work`; not covered: that a startup or claim blocked by an orphaned (never-committed) transaction fails within the configured `lock_timeout` rather than waiting indefinitely; no native test exercises `PRISM_DATABASE_LOCK_TIMEOUT_MS` | +| `tests/test_prism_lease_renewal_vs_expiry_claim.py` | full | (on the per-candidate claim lease) `crates/qbit-prism-server/tests/ledger_postgres.rs::candidate_renewal_requires_a_live_pending_token`, `crates/qbit-prism-server/tests/ledger_postgres.rs::candidate_mutations_reject_a_token_taken_over_while_waiting_for_its_row`, `crates/qbit-prism-server/src/coordinator/candidate_lease_tests.rs::landing_transaction_renews_across_expiries_and_terminal_contention_finishes_once`, `crates/qbit-prism-server/src/coordinator/candidate_lease_tests.rs::canceled_or_lost_renewal_cannot_resume_queued_work` | +| `tests/test_prism_lease_takeover.py` | partial | partial: covered by `crates/qbit-prism-server/tests/ledger_postgres.rs::candidate_outbox_is_atomic_and_claims_recover_after_owner_loss`, `crates/qbit-prism-server/tests/ledger_postgres.rs::candidate_renewal_requires_a_live_pending_token`, `crates/qbit-prism-server/tests/ledger_postgres.rs::candidate_processing_lock_allows_renewal_and_blocks_expired_takeover`, `crates/qbit-prism-server/tests/ledger_postgres.rs::migration_refuses_an_active_legacy_writer`; not covered: same-identity fast adoption, adoption-silence measured from guard acquisition, lost-CAS re-observation and writer-epoch fencing (no fast adoption or epochs natively; a claim is taken only after expiry) | +| `tests/test_prism_writer_lease_policy.py` | retired | retired — the #212 five-term heartbeat timing inequality and its phase attribution belong to the Python `LedgerLeaseHeartbeatService`; the native candidate lease is a fixed constant whose only validated bound (1..=600 s) is asserted in `crates/qbit-prism-server/tests/ledger_postgres.rs::candidate_renewal_requires_a_live_pending_token` | + +## PostgreSQL ops gates: migration, revert, cross-process, read replica, worker-difficulty store + +The non-discovered gates that `test/test-prism-postgres-*.sh` drove on 2.x.x. Native equivalents live in `ledger_postgres.rs`, `support/ledger_2x.rs`, `migration_rollback.rs`, `rollups_postgres.rs` and `support/public_replica.rs`. + +| 2.x.x file | Covered | 3.x.x replacement or status | +| --- | --- | --- | +| `tests/prism_postgres_a1_gate.py` | partial | partial: covered by `crates/qbit-prism-server/tests/support/ledger_2x.rs::native_publication_ordinal_survives_reactivation_and_excludes_rejected_candidates`, `crates/qbit-prism-server/tests/rollups_postgres.rs::competing_frontends_fold_each_share_once_including_late_clocked_work`, `crates/qbit-prism-server/tests/rollups_postgres.rs::failed_bucket_write_rolls_back_the_watermark_and_every_other_grain`, `crates/qbit-prism-server/tests/api_2x.rs::chart_rollups_match_raw_boundaries_tail_and_missing_progress`, `crates/qbit-prism-server/tests/api_2x.rs::block_views_markers_and_network_estimate_keep_2x_contracts`; not covered: PostgreSQL-versus-memory ledger parity and writer-lease identity binding (both retired: no in-memory ledger or writer lease in 3.x.x), the ordinal allocator-gap and durable-floor cases (no `audit_publication_sequence_floor` in the native ledger), block-marker future-dated and below-range exclusion | +| `tests/prism_postgres_a1_migration_gate.py` | partial | partial: covered by `crates/qbit-prism-server/tests/support/ledger_2x.rs::legacy_2x_upgrade_repairs_partial_carry_seed_and_preserves_shared_state` (existing publication ordinals are unchanged by the native upgrade), `crates/qbit-prism-server/tests/support/ledger_2x.rs::legacy_2x_failed_native_migration_rolls_back_seed_repair_and_all_ddl`, `crates/qbit-prism-server/tests/support/ledger_2x.rs::source_state_table_is_the_pinned_data`, `crates/qbit-prism-server/tests/support/ledger_2x.rs::frozen_pre_258_source_refuses_pending_v1_cleanly_and_migrates_terminal_rows`, `crates/qbit-prism-server/tests/support/ledger_2x.rs::partial_001_source_is_refused_before_any_ddl_naming_the_objects_present`, `crates/qbit-prism-server/tests/support/ledger_2x.rs::drifted_pre_258_source_is_refused_naming_each_object_and_rolls_back`, `crates/qbit-prism-server/tests/ledger_postgres.rs::migration_refuses_undrained_legacy_block_candidates` (the native source-schema matrix: the accepted and refused source states are pinned data, a release-schema source migrates, a `qbit_pool_blocks` or ordinal sequence restored on its own is refused before any DDL, and a release column, index or sequence whose definition differs is refused by name with the whole migration rolled back), `crates/qbit-prism-server/tests/ledger_postgres.rs::fixture_databases_isolate_advisory_keys_while_one_fixture_still_contends` (a migration queued behind the held migration advisory key is seen ungranted in `pg_locks`, does not end while the key is held, and finishes once it is released), `crates/qbit-prism-server/tests/support/session_sequence.rs::migration_009_preserves_preexisting_jobs_and_runs_once_for_two_frontends` (two frontends migrating at once both start and each migration is recorded once); not covered, no open issue: the ordinal block of `001_share_ledger.sql`, which the native migrator still applies to every source — its refusal of a wrong ordinal column, sequence, index or constraint definition, its in-place validation of a `NOT VALID` ordinal constraint, the ordinal allocator at the bigint maximum, its backfill of a pre-ordinal `qbit_pool_blocks` (M0-M8) and its non-positive and duplicate ordinal refusals (M9, M10, and the rejecting advisory waiter) (#285 closed without them; its drift tests use other release objects because 001's own guard refuses these first). No supported source release reaches the backfill or those two refusals (v2.0.0 to v2.0.2 ship one identical 001 that already carries the ordinal column, sequence, unique index and check constraint), but the migrator classifies any ledger without 002 objects as pre-#258 and lets 001 backfill it instead of refusing a pre-v2.0.0 source, so the path is live and untested | +| `tests/prism_postgres_a1_process_gate.py` | partial | partial: covered by `crates/qbit-prism-server/tests/ledger_postgres.rs::concurrent_instances_have_one_commit_order_and_stable_snapshots`, `crates/qbit-prism-server/tests/postgres_failover.rs::acknowledged_shares_survive_synchronous_primary_loss_and_pool_reconnect`, `crates/qbit-prism-server/tests/live_regtest.rs::real_two_server_mining_failover_audit_and_reorg`; not covered: publication-ordinal assignment order under cross-process confirmation contention (C1) and the two-process confirmation-then-publication interleavings (C2/C3); the filesystem `AuditArtifactStore` flock authority those cases drive is retired | +| `tests/prism_postgres_a1_revert_gate.py` | partial | partial: covered by `crates/qbit-prism-server/tests/migration_rollback.rs::legacy_ordinal_revert_refuses_native_schema_without_removing_columns`; not covered, no open issue: the script's legacy-schema paths (revert round trip, dependent-object refusal with full rollback, no-op and missing-table cases), which still ship in `crates/qbit-prism/sql/` for 2.x.x databases and have no 3.x.x test (#287 closed with decision D5, a one-way migration, instead); the recovery path that replaced the revert is `crates/qbit-prism-server/tests/migration_rollback.rs::frozen_2x_backup_restore_reconciles_before_ack_and_exposes_post_ack_loss` | +| `tests/prism_public_read_replica_gate.py` | full | `crates/qbit-prism-server/tests/support/public_replica.rs::read_only_public_service_tracks_real_standby_stream_and_refuses_promotion`, `crates/qbit-prism-server/tests/api_2x.rs::public_service_is_read_only_and_bounds_http_database_work`, `crates/qbit-prism-server/src/api/public_service.rs::replica_liveness_uses_heartbeat_and_ages_failed_probes` | +| `tests/prism_worker_difficulty_store_gate.py` | partial | partial: covered by `crates/qbit-prism-server/tests/support/ledger_2x.rs::durable_worker_hints_order_by_shared_evidence_and_keep_original_ttl`, `crates/qbit-prism-server/tests/support/ledger_2x.rs::legacy_2x_upgrade_repairs_partial_carry_seed_and_preserves_shared_state`; not covered: memory-versus-PostgreSQL parity (retired: no memory store), the schema CHECK-constraint refusals at the database boundary, and the index-backed plan assertion | +| `tests/test_prism_worker_difficulty_store.py` | partial | partial: covered by `crates/qbit-prism-server/tests/support/ledger_2x.rs::durable_worker_hints_order_by_shared_evidence_and_keep_original_ttl`, `crates/qbit-prism-server/tests/stratum_protocol.rs::idle_retarget_lowers_reconnect_difficulty_without_renewing_evidence`, `crates/qbit-prism-server/tests/stratum_protocol.rs::resumed_difficulty_is_lane_scoped_ttl_bounded_and_explicit_requests_win`; not covered: the Python SQL-string and schema-string contracts (retired), the memory store and `load_recent` bulk read (retired: native reads per key), lower-only correction on a missing row being a no-op, and prune of the exact cutoff boundary | + +## Payout window, payout state, window pipeline parity and issue #224 telemetry + +The Python payout-state owner, the incremental payout artifact, the Rust daemon pipe and the parity oracle. The native server folds the full window per snapshot in-process; the frozen parity corpus under `tests/fixtures/window_pipeline_parity/` is replayed natively by `window_frozen_vectors.rs` and `window_daemon_gate.rs`. + +| 2.x.x file | Covered | 3.x.x replacement or status | +| --- | --- | --- | +| `tests/test_prism_payout_state.py` | partial | partial: covered by `crates/qbit-prism-server/tests/readiness_rpc.rs::another_frontend_payout_revision_captures_the_same_parent_block_but_fences_its_share`, `crates/qbit-prism-server/tests/stratum_protocol.rs::same_parent_payout_replacement_retains_the_superseded_job_for_block_submission`, `crates/qbit-prism-server/tests/readiness_rpc.rs::observed_readiness_failure_closes_cached_work_and_candidate_settlement`, `crates/qbit-prism-server/tests/ledger_postgres.rs::verified_landing_reconstructs_audit_and_reorgs_are_revision_fenced`, `crates/qbit-prism-server/tests/ledger_postgres.rs::deferred_share_survives_confirmation_crash_and_is_credited_once`, `crates/qbit-prism-server/tests/ledger_postgres.rs::late_acceptance_after_abandoned_claim_recovers_deferred_credit`, `crates/qbit-prism/src/lib.rs::difficulty_change_uses_found_block_network_difficulty_for_window_width`, `crates/qbit-prism-server/src/coordinator/candidate_lease_tests.rs::saturated_build_capacity_keeps_the_lease_until_one_candidate_confirms`; not covered: incremental artifact reuse, re-arm/backoff/debounce, periodic self-check and re-center, the 5/8..2x staleness band, accepted-parent preview waits and budgets, startup replay enumeration gate, landing observability metric names (all Python payout-state mechanics with no native counterpart; the native coordinator snapshots the whole window per refresh) | +| `tests/test_prism_incremental_payout_window.py` | partial | partial: covered by `crates/qbit-prism/src/window.rs::crossing_row_is_retained`, `crates/qbit-prism/src/window.rs::eligibility_uses_both_timestamps`, `crates/qbit-prism/src/window.rs::duplicate_and_zero_difficulty_are_rejected`, `crates/qbit-prism/src/window.rs::advance_rejections_mirror_python`, `crates/qbit-prism/src/window.rs::advance_matches_full_rebuild_and_reports_byte_surgery`, `crates/qbit-prism/src/window.rs::advance_shares_untouched_pages_and_pages_large_deltas_linearly`, `crates/qbit-prism/src/window.rs::advance_pages_a_multi_page_delta_onto_a_full_page_within_bound`, `crates/qbit-prism/src/window.rs::full_snapshot_paging_after_the_cutoff_allocates_only_retained_records`, `crates/qbit-prism/src/window.rs::digest_items_and_payout_totals_are_independent_of_page_size`, `crates/qbit-prism/src/window.rs::fragment_escapes_match_python_json_dumps`, `crates/qbit-prism/tests/window_frozen_vectors.rs::frozen_corpus_replays_through_the_native_window`; not covered: the 10,000-seed randomized incremental-vs-full-rescan property (native has one hand-built advance case plus the 35+4 frozen cases), and the Python chunked JSON encoder helpers (`iter_json_*_text_chunks`, `release_share_list_incrementally`, `share_array_json_view`), which are Python-only | +| `tests/test_prism_candidate_window.py` | retired | retired — the COPY-streamed disk-window membership check ([`candidate_window.py`](https://github.com/Qbit-Org/qbit-mining-bootstrap/blob/504846cc0b72e8f86ed17f896d4ccbbe196a31dc/lab/prism/candidate_window.py)) has no native subject: candidates are signed ledger attestations persisted in the PostgreSQL outbox and `land_candidate` verifies the signed bundle, revision and prior balances rather than share membership | +| `tests/test_prism_payout_window_daemon_recenter.py` | retired | retired — Python coordinator self-check/re-center of a cached daemon window; the native server folds the full window per snapshot and spawns no daemon | +| `tests/test_prism_window_pipeline_rust.py` | partial | partial: covered by `crates/qbit-prism/tests/audit_cli.rs::prepare_window_declares_out_of_range_values_and_survives_them`, `crates/qbit-prism/tests/audit_cli.rs::prepare_window_advance_ignores_the_append_invalidation_epoch_tag`, `crates/qbit-prism/tests/audit_cli.rs::build_audit_bundle_serve_mode_caches_parsed_windows`, `crates/qbit-prism/tests/window_daemon_gate.rs::frozen_corpus_replays_through_the_serve_daemon`, `crates/qbit-prism/tests/window_daemon_gate.rs::supplementary_cases_replay_through_the_serve_daemon`; not covered: the coordinator-side `PRISM_WINDOW_PIPELINE_RUST` switch, degrade-to-Python on daemon death/protocol mismatch/contention, deadline stamping after the daemon lock, miscount refusal at materialization, poisoned-daemon retirement (Python coordinator mechanics; the native server does not use the daemon) | +| `tests/test_window_pipeline_parity.py` | partial | partial: covered by `crates/qbit-prism/tests/window_frozen_vectors.rs::frozen_corpus_replays_through_the_native_window`, `crates/qbit-prism/tests/window_frozen_vectors.rs::supplementary_cases_replay_through_the_native_window`, `crates/qbit-prism/tests/window_daemon_gate.rs::frozen_corpus_replays_through_the_serve_daemon`, `crates/qbit-prism/tests/window_daemon_gate.rs::supplementary_cases_replay_through_the_serve_daemon`, `crates/qbit-prism/src/window.rs::rejection_categories_mirror_the_parity_vocabulary`, `crates/qbit-prism/src/window.rs::declared_widths_admit_the_edge_and_refuse_one_past_it`, `crates/qbit-prism/src/window.rs::difficulty_totals_that_leave_u128_are_declined_not_saturated`; not covered: the adapter registry (Python-only) and the 17 divergence-detection self-tests that drive perturbed backends to prove the harness fails where it should | +| `tests/test_window_pipeline_parity_gate.py` | retired | retired — the env-selected parity adapter gate is gone; both native replays run ungated under `cargo test` against the prebuilt daemon binary and refuse a handshake other than protocol 2 | +| `tests/window_pipeline_parity.py` | full | `crates/qbit-prism/tests/support/window_corpus.rs` (corpus loader, pins and comparator; no test function of its own), `crates/qbit-prism/tests/window_frozen_vectors.rs::frozen_corpus_replays_through_the_native_window`, `crates/qbit-prism/tests/window_daemon_gate.rs::frozen_corpus_replays_through_the_serve_daemon` | +| `tests/window_pipeline_parity_gate.py` | retired | retired — strict CI entry point for the Python adapter selection; superseded by the two ungated native replays | +| `tests/test_window_paging_allocation_harness.py` | retired | retired — safety tests for the deleted perf driver `tests/perf/window_paging_allocation.py`; the #240 paging property itself is pinned by `crates/qbit-prism/src/window.rs::full_snapshot_paging_keeps_aggregate_capacity_linear_across_sizes` | +| `tests/fixtures/fake_serve_builder.py` | retired | retired — fake daemon fixture for Python coordinator tests; native tests drive the real `qbit-prism-build-audit-bundle --serve` binary | +| `tests/test_prism_issue_224_cadence.py` | retired | retired — Python reconcile-intent mechanics (balance reread vs window rescan) have no native counterpart, every native snapshot reads carry live; the #224 dense-cadence reject-window measurement is the `--cadence dense` phase of the on-demand `crates/qbit-prism-load` harness (`crates/qbit-prism-load/src/cadence.rs`, #342), a measurement that asserts no bound | +| `tests/test_prism_issue_224_core.py` | retired | retired — reconcile intents and fixed-cardinality reconcile attribution are Python payout-state telemetry; revision-fenced reconciliation is natively pinned by `crates/qbit-prism-server/tests/ledger_postgres.rs::verified_landing_reconstructs_audit_and_reorgs_are_revision_fenced` | +| `tests/test_prism_issue_224_landing.py` | partial | partial: `crates/qbit-prism-server/tests/landing_metrics.rs::real_offer_and_peer_proof_finish_once_only_after_notify_delivery`, `crates/qbit-prism-server/tests/landing_metrics.rs::actual_build_deadline_counts_once_and_later_notify_is_degraded`, and `crates/qbit-prism-server/tests/observability/cardinality.rs::every_http_family_and_closed_label_tuple_stays_bounded_under_varied_inputs` cover the native acceptance-to-delivery boundary and closed results. Python per-phase preview attribution is retired; native observation-horizon and lost-evidence limits are documented in `docs/prism-native-metrics.md`. Not covered, no open issue: unconditional exactly-once reporting across a process restart or unavailable evidence, retired on 2026-09-22 by the #458 acceptance reconciliation in favour of the bounded process-local contract: at most one sample per tracked block, exactly one once its committed revision and a qualifying `mining.notify` write are both locally known; a fresh process starts with no local observations, may re-observe immature confirmed blocks from its own proof clock and never reconstructs a prior interval; a proven orphan closes without a delivery sample; a lost settlement reply, failed terminal read or saturation stays unknown (tracking-unknown gauge, pending age -1), never a healthy zero. A cross-restart or cluster-wide unique count would need durable observation metadata that #458 decided not to add. | +| `tests/test_prism_issue_224_ledger.py` | partial | partial: covered by `crates/qbit-prism-server/tests/window_read_oracle.rs::window_matches_the_oracle_across_page_boundaries`, `crates/qbit-prism-server/tests/window_read_oracle.rs::the_row_that_crosses_the_weight_boundary_is_credited`, `crates/qbit-prism-server/tests/window_read_oracle.rs::rows_on_the_anchor_are_credited_and_rows_after_it_are_not`, `crates/qbit-prism-server/tests/window_read_oracle.rs::rejected_rows_are_excluded_and_consume_no_weight`; not covered, no open issue: gate-wait versus execution attribution of the window and prior-balance reads, per read operation (the native waits are recorded since #334, per lock, and for pool acquisition, by outcome and not per read, by `crates/qbit-prism-server/tests/ledger_wait_metrics.rs::advisory_lock_and_pool_waits_are_recorded_by_outcome`; no native series times a read's execution, and #278, whose cutover-minimum set named the lock-wait series, closed without a per-read one) | +| `tests/test_prism_accepted_preview_telemetry.py` | retired | retired — the Python accepted-preview telemetry contract module (vocabularies, recorder, renderer) does not exist natively | +| `tests/test_prism_accepted_preview_publication_latency.py` | partial | partial: `crates/qbit-prism-server/tests/landing_metrics.rs::failed_template_and_second_acceptance_keep_oldest_wait_until_delivery`, `crates/qbit-prism-server/tests/landing_metrics.rs::cancelled_refresh_keeps_acceptance_without_fabricating_timeout_or_delivery`, and `crates/qbit-prism-server/tests/landing_metrics.rs::delayed_settlement_reply_cannot_publish_an_older_acceptance_at_a_newer_revision` exercise stalls, supersession, cancellation and reordered settlement/delivery. The native endpoint is actual mining.notify write; Python preview-cache publication and its scheduling mechanisms are retired. Not covered, no open issue: unconditional exactly-once latency reporting across a process restart or unavailable evidence, retired on 2026-09-22 by the #458 acceptance reconciliation (see the `tests/test_prism_issue_224_landing.py` row for the accepted process-local contract); the Python preview-cache latency that survived a coordinator restart has no native subject, since a restarted frontend begins a new observation horizon and reports unknown rather than a reconstructed or zero interval. | +| `tests/test_prism_builder_pipe_transport.py` | retired | retired — Python non-blocking pipe client for the daemon ([`bundle_compiler.py`](https://github.com/Qbit-Org/qbit-mining-bootstrap/blob/700fae0d666443b8a050e0143258c74e02be0926/lab/prism/bundle_compiler.py)); the native server folds windows in-process and spawns no daemon, and the daemon protocol itself is exercised over real pipes with a response deadline by `crates/qbit-prism/tests/window_daemon_gate.rs` | + +## Audit bundles, artifacts, audit HTTP and canonical backfill + +Audit bodies live in PostgreSQL in 3.x.x and are served content-addressed by `/public/v1/artifacts`; the on-disk audit root, its scans, its share-segment slot repair and the Python streaming bundle view are retired. The canonical backfill maps to `import-audits`, which #287 made an enforced step: `self-check` reports the audit rows that lack a stored body or canonical bytes and fails on them in production mode (`crates/qbit-prism-server/tests/audit_completeness.rs`). + +| 2.x.x file | Covered | 3.x.x replacement or status | +| --- | --- | --- | +| `tests/test_prism_audit_api.py` | partial | partial: covered by `crates/qbit-prism-server/tests/api_database.rs::shared_database_serves_all_contracts_and_global_reward_ranks`, `crates/qbit-prism-server/tests/api_contract.rs::validation_preserves_public_and_operator_error_shapes`, `crates/qbit-prism-server/tests/api_contract.rs::health_reads_runtime_snapshot_without_database_access`, `crates/qbit-prism-server/src/api.rs::expired_health_snapshot_returns_503_even_when_last_snapshot_was_ready`, `crates/qbit-prism-server/tests/api_contract.rs::metrics_publication_preserves_samples_and_restores_fresh_headers`, `crates/qbit-prism-server/src/api/metrics_snapshot/tests.rs::expired_publication_fails_closed_through_the_router_and_recovers`; not covered: 400 on a malformed fanout txid or commitment hash, 404 on an unknown commitment leaf, 404 on `/audit/latest` before any evidence exists, and the exact field values of `/miners/{id}/status`, `/owed-balances`, `/audit/share-window`, `/audit/carry-forward-integrity` (only HTTP 200 is asserted natively) | +| `tests/test_prism_audit_artifacts.py` | partial | partial: covered by `crates/qbit-prism-server/tests/support/ledger_2x.rs::native_publication_ordinal_survives_reactivation_and_excludes_rejected_candidates`, `crates/qbit-prism-server/tests/support/ledger_2x.rs::canonical_2x_sidecar_import_preserves_exact_bytes_and_fails_closed`, `crates/qbit-prism-server/tests/api_2x.rs::content_addressed_bytes_and_legacy_fallback_headers_are_exact`, `crates/qbit-prism-server/tests/ledger_postgres.rs::verified_landing_reconstructs_audit_and_reorgs_are_revision_fenced`, `crates/qbit-prism/tests/audit_cli.rs::audit_clis_accept_compact_body_ref_bundle`, `crates/qbit-prism/tests/audit_cli.rs::audit_clis_accept_v2_range_proof_bundle`, `crates/qbit-prism-server/tests/ledger_deadline_atomicity.rs::incomplete_audit_snapshot_returns_no_body`, `crates/qbit-prism-server/tests/ledger_deadline_atomicity.rs::conflicting_audit_preserves_original` (the native rule that replaces segment repair, documented in `docs/prism-ledger-ops.md`, "Native snapshot rejection and legacy segments": an audit whose share rows are missing is a read error, with no body, no canonical bytes and no change to the stored audit or snapshot row, and it serves again once the rows are back; a different signed audit for a landed hash is refused and the original block, audit, snapshot, carry and revision are unchanged); the audit share-segment slot lifecycle (gap backfill from the ledger, quarantine when the ledger has no rows, conflicting duplicate raises) is retired: a native audit row names an immutable share range in `qbit_share_ledger` and is reconstructed on read, so there is no segment slot to merge, repair or quarantine; the on-disk store mechanics (root/symlink/fsync/inode authority, subprocess verifier, legacy evidence proof tokens, retention and pruning, about 80 tests) are retired with the filesystem audit root | +| `tests/test_prism_audit_bundle_view.py` | partial | partial: covered by `crates/qbit-prism-server/tests/audit_reconstruction_permit.rs::a_queued_reconstruction_keeps_its_permit_after_its_caller_is_dropped`, `crates/qbit-prism-server/tests/audit_reconstruction_permit.rs::a_second_reconstruction_waits_for_the_permit_and_both_rows_hydrate`, `crates/qbit-prism-server/tests/audit_reconstruction_permit.rs::failed_and_cancelled_reconstructions_release_the_permit`, which pin the bound that stands in for the streaming view: concurrent window-sized reconstructions are limited by the shared permit, and the permit travels with the blocking job past a dropped caller, coming back exactly once when the job ends — on success, on failure and on cancellation; the imported-audit twin of the same bound is `crates/qbit-prism-server/tests/support/ledger_2x.rs::imported_audit_decode_limit_outlives_a_dropped_request`; the artifact route's own admission — a dedicated rebuild limit separate from the read concurrency, an in-flight cap that refuses before any audit read, and `pool-summary` staying inside its read deadline under artifact load — is pinned by `crates/qbit-prism-server/tests/artifact_admission.rs::rebuilds_are_bounded_independently_of_the_read_concurrency`, `crates/qbit-prism-server/tests/artifact_admission.rs::an_over_cap_artifact_request_is_refused_before_any_audit_read` and `crates/qbit-prism-server/tests/artifact_admission.rs::artifact_route_load_keeps_pool_summary_within_its_read_deadline` (#469); not covered, no open issue: the bounded audit-body read itself — `materialize_audit_row` still hydrates the whole share array for an unsealed row, so no native test reads a body without holding a window-sized structure; #267's artifact-route bounding closed with the admission above and sealing (#144) rather than a streaming read; also not covered: that the job keeps the permit for its whole duration rather than only until it starts, which no one-share fixture can observe without a hook | +| `tests/test_prism_audit_http.py` | retired | retired — Python `AuditHttpFacade` thread and listener lifecycle (start/stop races, join timeouts, bind failure) has no native subject; the one metrics-freshness contract it also pinned lives at `crates/qbit-prism-server/tests/api_contract.rs::metrics_publication_preserves_samples_and_restores_fresh_headers` and `crates/qbit-prism-server/src/api/metrics_snapshot/tests.rs::age_uses_monotonic_time_and_exceeds_the_budget_strictly` | +| `tests/test_prism_audit_root_scan.py` | retired | retired — audit bodies live in PostgreSQL; there is no long-lived audit root directory descriptor or readdir scan in 3.x.x (the only filesystem path is the one-shot `import-audits --root`) | +| `tests/test_prism_backfill_audit_bundle_canonical.py` | partial | partial: covered by `crates/qbit-prism-server/tests/support/ledger_2x.rs::canonical_2x_sidecar_import_preserves_exact_bytes_and_fails_closed`, `crates/qbit-prism-server/tests/ledger_postgres.rs::legacy_audit_import_validates_envelope_hash_and_pinned_key`, `crates/qbit-prism-server/tests/migration_rollback.rs::frozen_2x_cli_import_restores_all_body_layouts_and_serves_exact_artifacts` (the `import-audits` CLI publishes a sidecar-only, an external-body and an inline row, and `/public/v1/artifacts` serves each one's exact canonical bytes after the files are removed), `crates/qbit-prism-server/tests/audit_completeness.rs::import_cli_reports_remaining_audits_on_success_and_failure` (a rerun imports 0; an unrecoverable body exits nonzero with the remaining-work counts on stdout), `crates/qbit-prism-server/tests/ledger_deadline_atomicity.rs::import_item_failure_preserves_completed_items` (rows are walked in block-hash order, a failing row leaves the completed row imported and later rows untouched, a rerun imports exactly the unfinished rows), `crates/qbit-prism-server/tests/support/audit_acquire_metrics.rs::legacy_import_pages_release_before_writes_and_audit_decode` (one row per read), `crates/qbit-prism-server/tests/support/ledger_2x.rs::corrupt_imported_canonical_bytes_are_refused_not_served_from_the_legacy_file` (stored canonical bytes that fail their digest or shape are a read error, not served from the legacy file), `crates/qbit-prism-server/src/tools.rs::audit_root_env_and_cli_precedence` (`--root`, falling back to a non-blank `PRISM_AUDIT_DIR`, the flag winning; `ImportAudits` has no other option); not covered, no open issue: refusal of a row whose `block_hash` or `audit_bundle_sha256` is not 64 hex characters (`legacy_canonical_sidecar` in `crates/qbit-prism-server/src/ledger/migration.rs`), and use of the inline body when the row's `body_uri` is unreadable (#287 closed without either); retired: reconstruction of an absent segment from database share rows with its digest-failure and incomplete-range refusals (native import reads share parts only from the body and its segment files, and `docs/prism-ledger-ops.md` declares ledger gap backfill not a native lifecycle), dry-run mode, the `--start-after`/`--limit`/`--batch-size` checkpoint options (the native cursor is the `canonical_audit_bytes IS NULL` predicate), writer-lease preflight and lease release ordering (no writer lease in 3.x.x), scanning past a failed row (native import stops at the first failing row), the `qbit.prism.audit-bundle-canonical-backfill.v1` summary document, and the Python adapter binding and argparse surface | + +## CTV fanout and CPFP broadcaster + +The 2.x.x broadcaster daemon and its coordinator-facing runtime. Natively the broadcaster is a Tokio task with claims fenced in the ledger. + +| 2.x.x file | Covered | 3.x.x replacement or status | +| --- | --- | --- | +| `tests/test_ctv_broadcaster.py` | partial | partial: covered by `crates/qbit-prism/src/broadcast.rs::build_cpfp_child_spends_anchor_and_funding`, `crates/qbit-prism/src/broadcast.rs::build_cpfp_child_rejects_fee_at_or_above_funding`, `crates/qbit-prism/src/broadcast.rs::build_cpfp_child_rejects_zero_fee`, `crates/qbit-prism/src/broadcast.rs::build_cpfp_child_rejects_funding_equal_to_anchor`, `crates/qbit-prism/src/broadcast.rs::fanout_settlement_status_tracks_chain_state`, `crates/qbit-prism-server/tests/ledger_postgres.rs::ctv_artifacts_wait_for_maturity_and_claims_are_fenced`, `crates/qbit-prism-server/tests/live_regtest.rs::real_ctv_maturity_broadcast_and_confirmation`, `crates/qbit-prism-server/tests/support/live_ctv_cpfp.rs::real_cpfp_recovers_reserved_funding_after_owner_crash`; not covered: the fake-node status matrix for the server broadcaster (idempotent when already in mempool, unexpected parent spender is not confirmed, transient scan failure does not poison spender coverage, zero-fee anchored fanout refused without CPFP fee, positive fee without wallet rejected not raised) and the whole fee-bump path, which has no native counterpart; no candidate issue | +| `tests/test_ctv_broadcaster_daemon.py` | partial | partial: covered by `crates/qbit-prism-server/tests/ledger_postgres.rs::ctv_artifacts_wait_for_maturity_and_claims_are_fenced`, `crates/qbit-prism-server/tests/api_database.rs::shared_database_serves_all_contracts_and_global_reward_ranks`, `crates/qbit-prism-server/tests/rpc_deadlines.rs::ordinary_read_can_outlive_submit_deadline_and_submit_is_never_retried`, `crates/qbit-prism-server/tests/rpc_deadlines.rs::general_deadline_covers_streamed_body_and_releases_the_client`, `crates/qbit-prism-server/tests/rpc_deadlines.rs::wallet_selection_preserves_proxy_prefix_and_replaces_existing_wallet`; not covered: the claim lane skipping rows whose backoff has not elapsed (only the pending read feeds are asserted), immature rows skipping node probes, per-pass row limit and yielding, CPFP fee configured without `PRISM_CTV_BROADCASTER_WALLET` failing before any RPC, connection reuse and retry of an idempotent read after a stale drop; no candidate issue | +| `tests/test_prism_ctv_runtime.py` | retired | retired — Python `CtvRuntimeService` owner boundary (writer-admission context, lazy daemon handoff, live-config rebuild, patch seams) does not exist; the native loop is `broadcaster::run` on a Tokio interval with claims fenced in the database | + +## Stratum sessions, admission, job delivery, retained jobs and the submit hot path + +Session and admission behaviour is covered by `stratum_protocol.rs`, `stratum_admission*.rs` and `observability/stratum.rs`; the stale-grace anchoring, retained-job and submit-time tip selection cases that #280 owned are covered by the ungated suites #313 added, `crates/qbit-prism-server/src/coordinator/miner_tests/` and `crates/qbit-prism-server/src/stratum/stale_grace_tests.rs` with `retained_tests.rs`. + +| 2.x.x file | Covered | 3.x.x replacement or status | +| --- | --- | --- | +| `tests/test_prism_direct_stratum.py` | partial | partial: covered by `crates/qbit-prism-server/tests/stratum_codec.rs::witness_coinbase_round_trip_and_header_endianness`, `crates/qbit-prism-server/tests/stratum_codec.rs::extranonce_split_ignores_identical_output_and_witness_bytes`, `crates/qbit-prism-server/tests/stratum_codec.rs::transaction_parser_rejects_truncation_and_noncanonical_sizes`, `crates/qbit-prism-server/tests/stratum_codec.rs::merkle_branches_use_txids_instead_of_wtxids`, `crates/qbit-prism-server/tests/stratum_codec.rs::version_mask_advertisement_is_authoritative`, `crates/qbit-prism-server/tests/stratum_codec.rs::targets_preserve_network_block_solutions_above_listener_floor`, `crates/qbit-prism/src/lib.rs::audit_bundle_coinbase_witness_commitment_uses_consensus_leaves_and_audit_nonce`; not covered: the share target when no listener floor is configured (network cap alone) is not asserted in isolation | +| `tests/test_prism_stratum_session.py` | retired | retired — Python `StratumSessionService`/`SessionRegistry`/`SessionRuntime` owner boundary (symbol identity, registry snapshots, thread handoff); the one behavioural clause that survives (per-username capacity) is covered by `crates/qbit-prism-server/tests/stratum_protocol.rs::per_username_capacity_preserves_prior_authorization_on_failed_reauthorize` | +| `tests/test_prism_stratum_restart_bind.py` | partial | partial: `server::run` (`crates/qbit-prism-server/src/server.rs`) binds both Stratum listeners, primary and high difficulty, before `Coordinator::new`, so a startup that loses the bind race registers no instance row (#460); bind-before-recovery is covered by `crates/qbit-prism-server/tests/stratum_restart_bind.rs::held_primary_port_fails_startup_before_registration`, `crates/qbit-prism-server/tests/stratum_restart_bind.rs::held_highdiff_port_fails_startup_before_registration`, `crates/qbit-prism-server/tests/stratum_restart_bind.rs::without_a_highdiff_port_only_the_primary_listener_is_bound` and `crates/qbit-prism-server/tests/stratum_restart_bind.rs::a_configured_highdiff_port_is_bound_and_accepts`, and listeners accepting during recovery by `crates/qbit-prism-server/tests/stratum_restart_bind.rs::listeners_are_bound_while_the_node_is_slow`; not covered: bind retry against a draining predecessor, retry abort on shutdown and zero-retry fail-fast, which native has no subject for (`run` binds once), and port release when a shutdown arrives during the node readiness wait, which needs a production change because no signal handler exists until every task is spawned; both are follow-ups with no issue yet | +| `tests/test_prism_reconnect_backpressure.py` | partial | partial: covered by `crates/qbit-prism-server/tests/stratum_protocol.rs::delivery_stats_follow_admission_progress_generation_and_disconnect`, `crates/qbit-prism-server/tests/stratum_protocol.rs::admission_timeout_records_failure_and_shutdown_releases_session_stats`, `crates/qbit-prism-server/tests/observability/stratum.rs::pending_initial_work_moves_and_cancellation_does_not_fabricate_share_events`, `crates/qbit-prism-server/src/metrics/snapshots.rs::coverage_gap_preserves_strict_95_percent_boundary_and_ages_until_recovery`, `crates/qbit-prism-server/tests/stratum_admission.rs::timed_out_consumed_allocation_returns_error_then_reconnect_uses_fresh_id`; not covered: the global connection cap under a storm (`connection_limit` drop path has no test), health failing closed on a sustained coverage gap or stalled initial delivery (the `job-delivery-stalled` rule in `publish_health`, `crates/qbit-prism-server/src/server.rs`, has no test), delivery priority ordering initial > new-tip > same-tip, and a 640-client storm not displacing the tip builder | +| `tests/test_prism_admission_lock.py` | retired | retired — pins the Python coordinator `RLock` split (delivery anchor, pending-queue bookkeeping, coverage census) and its lock order; the native session is per-connection async state with no shared coordinator lock | +| `tests/test_prism_initial_job_delivery.py` | partial | partial: covered by `crates/qbit-prism-server/tests/stratum_protocol.rs::fast_arrival_retries_failed_paired_delivery_and_only_proven_difficulty_resumes`, `crates/qbit-prism-server/tests/stratum_protocol.rs::delivery_stats_follow_admission_progress_generation_and_disconnect`, `crates/qbit-prism-server/tests/observability/stratum.rs::pending_initial_work_moves_and_cancellation_does_not_fabricate_share_events`, `crates/qbit-prism-server/tests/stratum_protocol.rs::same_parent_payout_replacement_retains_the_superseded_job_for_block_submission`; not covered: health turning non-green after a stalled delivery deadline and on sustained coverage loss, the 250-client reconnect storm sharing one build, reauthorization superseding a pending identity mid-build, and startup prewarm (no native prewarm exists) | +| `tests/test_prism_first_job_latency.py` | partial | partial: covered by `crates/qbit-prism-server/tests/config_cli.rs::large_resource_budgets_fail_instead_of_truncating_or_panicking` (worker-count knob bounds); not covered, no open issue: no native test bounds first-notify latency for sessions joining under tip churn, or time to usable work after a new tip; the `crates/qbit-prism-load` harness binary reports both on demand (`time_to_reconnect_milliseconds` and `time_to_usable_work` in `load-harness-report.json`) and asserts no bound (#271 closed with the harness, not a test); retired: publication-priority build subscription, retryable payout-gate non-admission, share-window serialization cache and hashrate-ordered fanout, Python job-build-pipeline mechanics with no native counterpart | +| `tests/test_prism_job_delivery.py` | retired | retired — Python S2 `JobDeliveryRuntime` facade: symbol identity, descriptor routing, legacy retained-state tuple conversion | +| `tests/test_prism_retained_jobs.py` | partial | partial: covered by `crates/qbit-prism-server/tests/stratum_protocol.rs::negotiation_reauthorization_retains_original_job_worker_and_grace_expires`, `crates/qbit-prism-server/tests/stratum_protocol.rs::same_parent_payout_replacement_retains_the_superseded_job_for_block_submission`, `crates/qbit-prism-server/tests/readiness_rpc.rs::another_frontend_payout_revision_captures_the_same_parent_block_but_fences_its_share`, `crates/qbit-prism-server/tests/stratum_protocol.rs::multiple_miners_unique_extranonces_durable_accept_and_duplicate_rejection`, `crates/qbit-prism-server/tests/stratum_protocol.rs::reconnect_on_another_listener_preserves_original_entropy_mask_and_expiry`, `crates/qbit-prism-server/tests/stratum_protocol.rs::retained_username_capacity_reclaims_evicted_and_replaced_jobs`, `crates/qbit-prism-server/tests/support/live_highdiff.rs::observability_grace_counter_moves_only_after_real_durable_credit`, `crates/qbit-prism-server/tests/support/live_highdiff.rs::observed_tip_advance_fences_old_prepared_jobs_and_health`, `crates/qbit-prism-server/src/coordinator/miner_tests/credit.rs::grace_uses_delivery_not_first_seen_and_closes_after_exact_boundary` (grace runs from this connection's delivery: a share is accepted 3 s after a delivery made 100 s after the tip was published and refused one nanosecond later), `crates/qbit-prism-server/src/stratum/retained_tests.rs::undelivered_grace_survives_zero_or_expired_submit_cache_and_active_eviction`, `crates/qbit-prism-server/src/stratum/stale_grace_tests.rs::same_tip_delivery_does_not_slide_grace_but_real_flip_reanchors` (a tip flip reanchors retained-job grace to delivery: an evicted job stays creditable 20 s after the flip until replacement work reaches the connection, then for the grace interval, then is unknown; a same-tip redelivery keeps the first anchor), `crates/qbit-prism-server/src/coordinator/miner_tests/credit.rs::startup_baseline_opens_no_grace_but_a_later_real_transition_does`, `crates/qbit-prism-server/src/stratum/stale_grace_tests.rs::failed_replacement_keeps_retired_job_until_actual_delivery_then_expires`, `crates/qbit-prism-server/src/coordinator/miner_tests/credit.rs::observed_tip_pending_prepared_and_revision_still_credits_exact_prior_parent` (the window stays open until the connection receives new-tip work; the credit is `stale-grace` at the current payout revision, with no candidate), `crates/qbit-prism-server/src/coordinator/miner_tests/observations.rs::no_observation_falls_back_to_rpc_without_opening_grace`, `crates/qbit-prism-server/src/coordinator/miner_tests/published_lease.rs::candidate_only_observation_cannot_publish_authority_or_open_stale_grace` (a tip seen only by the submit's own RPC read, or only by a candidate probe, opens no grace), `crates/qbit-prism-server/src/stratum/stale_grace_tests.rs::same_tip_capacity_eviction_survives_beyond_one_second_and_keeps_original_worker`, `crates/qbit-prism-server/src/coordinator/miner_tests/credit.rs::same_tip_retained_work_credits_normally_and_reauthorization_dedups_original_worker` (an evicted same-tip job is credited 2 s after eviction at its original difficulty with no credit policy, and after reauthorization the share ID and the duplicate rejection stay with the original worker), `crates/qbit-prism-server/src/coordinator/miner_tests/observations.rs::parent_rpc_failure_during_grace_is_backend_unavailable`, `crates/qbit-prism-server/src/stratum/stale_grace_tests.rs::unknown_job_rejects_before_any_node_rpc_in_real_request_and_coordinator_resume`, `crates/qbit-prism-server/src/coordinator/miner_tests/interleavings.rs::slow_parent_lookup_cannot_overwrite_a_newer_tip_or_parent`, `crates/qbit-prism-server/src/coordinator/miner_tests/credit.rs::skipped_tip_uses_chain_parent_and_rejects_two_blocks_back`, `crates/qbit-prism-server/src/stratum/retained_tests.rs::graveyard_removes_known_unrelated_parent_instead_of_extending_grace`, `crates/qbit-prism-server/src/stratum/retained_tests.rs::graveyard_burial_ttl_and_same_tip_capacity_keep_original_identity_bounded`, `crates/qbit-prism-server/src/stratum/retained_tests.rs::rapid_publications_keep_the_graveyard_bounded_before_its_next_prune`, `crates/qbit-prism-server/src/stratum/retained_tests.rs::published_tip_capacity_preserves_prior_work_when_current_work_resumes_without_delivery` (the session graveyard: grace decided by the chain parent when a tip is skipped, a same-tip TTL with an exact boundary, a same-tip cap, a hard bound, and grace-eligible prior-tip work surviving capacity pressure), `crates/qbit-prism-server/src/stratum/retained_tests.rs::local_graveyard_cannot_authorize_another_connections_worker_or_old_parent`, `crates/qbit-prism-server/src/stratum/retained_tests.rs::published_tip_capacity_is_reselected_after_awaited_resume` (a cross-connection resume needs the same username and the same tip, and never borrows stale grace), `crates/qbit-prism-server/src/coordinator/miner_tests/resume_inputs.rs::stored_inputs_resume_unchanged_with_original_work_and_expiry`, `crates/qbit-prism-server/tests/compact_runtime_e2e.rs::real_socket_reconnect_resumes_original_entropy_mask_and_submits_once` (a resumed job keeps the original `extranonce1`, coinbase halves and version mask while the replacement socket holds a different `extranonce1`); not covered, no open issue: a block candidate built from a resumed job carrying the original `extranonce1` (by reading the source, `submission_candidate` in `crates/qbit-prism-server/src/coordinator/miner_submit.rs` assembles the coinbase suffix from the job's own `extranonce1` and receives no session state, so there is no per-connection extranonce to record, but no native test builds a candidate from a resumed job), and a submit whose username differs from the authorized full username rejecting as `unauthorized-worker` (the check is in the `mining.submit` arm of `crates/qbit-prism-server/src/stratum.rs`; no native test sends the frame; never listed in #280); the accepted-block pool-close cap and the pool-wide graveyard (lookup cost among 4,096 entries, one connection's burial evicting another's) are retired, the native graveyard being per-session state, and the `qbit_prism_block_solves_dropped_total` and `qbit_prism_evicted_job_*` families have no native counterpart | +| `tests/test_prism_hot_path.py` | partial | partial: covered by `crates/qbit-prism-server/src/coordinator/miner_tests/interleavings.rs::submit_selection_cannot_mix_old_hash_with_new_observation_provenance`, `crates/qbit-prism-server/src/coordinator/miner_tests/interleavings.rs::late_chain_info_cannot_roll_back_a_newer_observation` (submit-time tip selection is one coherent view: rewriting the observed tip, its parent and its baseline flag while a submit is parked does not change how the share is credited, and a slow chain read cannot roll a newer observation back), `crates/qbit-prism-server/src/coordinator/miner_tests/observations.rs::observed_tip_serves_submit_without_node_rpc`, `crates/qbit-prism-server/src/coordinator/miner_tests/observations.rs::no_observation_falls_back_to_rpc_without_opening_grace`, `crates/qbit-prism-server/src/coordinator/miner_tests/observations.rs::stale_observation_falls_back_to_rpc_and_does_not_invent_transition`, `crates/qbit-prism-server/src/coordinator/miner_tests/observations.rs::zero_max_age_forces_per_share_rpc_even_with_fresh_observation` (the fallback ladder, each case asserting the exact node-RPC list: none on a fresh observation, one `getbestblockhash` with no observation, with an aged one, and per share at a zero maximum age), `crates/qbit-prism-server/src/coordinator/miner_tests/stale_causes.rs::parent_without_grace_precedes_payout_revision_drift`, `crates/qbit-prism-server/src/stratum/retained_tests.rs::published_tip_capacity_is_reselected_after_awaited_resume`, `crates/qbit-prism-server/tests/support/live_highdiff.rs::observed_tip_advance_fences_old_prepared_jobs_and_health` (a job on the prior tip rejects as `stale-job` against the observed tip: one stale-cause count and no record, no node RPC, and the same fence against a real node), `crates/qbit-prism-server/src/coordinator/miner_tests/published_lease.rs::detected_replacement_preserves_published_credit_with_zero_grace_and_current_revision`, `crates/qbit-prism-server/src/coordinator/miner_tests/published_lease.rs::later_detected_tips_and_failed_refresh_do_not_renew_the_first_divergence`, `crates/qbit-prism-server/src/coordinator/miner_tests/refresh.rs::failed_real_refresh_retains_published_credit_until_lease_expires` (published work keeps ordinary credit without an RPC while a detected replacement is unbuilt; the lease runs from the first departure, is not renewed by later tips or failed refreshes, and expires into an RPC read), `crates/qbit-prism-server/src/coordinator/miner_tests/refresh.rs::real_refresh_build_persist_publish_and_client_job_delivery_have_constant_rpc_count`, `crates/qbit-prism-server/src/coordinator/miner_tests/observations.rs::refresh_observation_rpc_count_is_independent_of_client_submit_count` (the node-RPC list of a real `refresh_once` is identical at 1, 32 and 128 clients, and per-client job build, persistence and submit add none), `crates/qbit-prism-server/tests/readiness_rpc.rs::observed_readiness_failure_closes_cached_work_and_candidate_settlement` (an untrusted chain view stops job issue, resume and credit); not covered, no open issue: tip publication staying prompt while a burst of submits is parked in the ledger append (the deleted 600-client case bounded it at 0.5 s; native submits read-lock `prepared` and `observed_tip` and publication write-locks them in `crates/qbit-prism-server/src/coordinator/publication_authority.rs`, single-submit interleavings exist, but no native test drives a burst or bounds the wait, and the `crates/qbit-prism-load` harness reports `time_to_usable_work` without asserting a bound; never listed in #280); the debug-logging cluster (per-share and per-job stdout prints), the lock-set cluster (Python `RLock` isolation: lazy lock-set install, vardiff-lock waits, once-per-submit lock acquisition) and the queued-fanout driver whose live trust check the last case exercised are retired | +| `tests/prism_coordinator_test_support.py` | retired | retired — shared fixtures (`FakeLedger`, `FakeRpc`, `ObservedRLock`) for the Python coordinator suites; the native suites carry their own `Backend`/`Client` fixtures in `stratum_protocol.rs` and `support/stratum_admission.rs` | + +## Vardiff and durable difficulty resume + +Retarget math, fast-arrival convergence and the durable worker-difficulty row. The Python persistence lane, idle-sweep executor and vardiff metric families have no native counterpart. + +| 2.x.x file | Covered | 3.x.x replacement or status | +| --- | --- | --- | +| `tests/test_prism_vardiff.py` | partial | partial: covered by `crates/qbit-prism-server/tests/stratum_codec.rs::vardiff_uses_actual_accepted_work_and_limits_idle_step`, `crates/qbit-prism-server/tests/stratum_protocol.rs::idle_retarget_lowers_reconnect_difficulty_without_renewing_evidence`, `crates/qbit-prism-server/tests/stratum_protocol.rs::fast_arrival_retries_failed_paired_delivery_and_only_proven_difficulty_resumes`, `crates/qbit-prism-server/tests/stratum_protocol.rs::highdiff_floor_still_forwards_valid_block_below_share_target`, `crates/qbit-prism-server/tests/stratum_codec.rs::targets_preserve_network_block_solutions_above_listener_floor`, `crates/qbit-prism-server/tests/stratum_protocol.rs::resumed_difficulty_is_lane_scoped_ttl_bounded_and_explicit_requests_win`, `crates/qbit-prism-server/tests/capacity.rs::binds_every_configuration_and_checks_vardiff_math`; not covered: the step-up retarget keeping the old job submittable at its stamped difficulty (native `deliver_job` retains prior jobs but no test submits against the retired job), the step-down flushing with `clean_jobs` (native sends non-clean in both directions; behaviour differs, untested), retarget build failure rolling the difficulty back without advertising (native `restore_retarget` untested); the idle-sweep bounded-executor and busy-client-skip clusters (roughly 20 tests) are Python-only executor mechanics | +| `tests/test_prism_vardiff_durable_resume.py` | partial | partial: covered by `crates/qbit-prism-server/tests/support/ledger_2x.rs::durable_worker_hints_order_by_shared_evidence_and_keep_original_ttl`, `crates/qbit-prism-server/tests/stratum_protocol.rs::fast_arrival_retries_failed_paired_delivery_and_only_proven_difficulty_resumes`, `crates/qbit-prism-server/tests/stratum_protocol.rs::idle_retarget_lowers_reconnect_difficulty_without_renewing_evidence`, `crates/qbit-prism-server/tests/stratum_protocol.rs::resumed_difficulty_is_lane_scoped_ttl_bounded_and_explicit_requests_win`; not covered: the periodic prune loop in `coordinator.rs` (only the ledger prune statement is tested), the retention kill switch preventing durable writes, and store-failure fallback to the in-memory cache; the write-queue coalescing, bounded shutdown drain and prune-batching clusters (about 15 tests) are Python persistence-lane mechanics and retired | +| `tests/test_prism_vardiff_fast_arrival.py` | partial | partial: covered by `crates/qbit-prism-server/src/vardiff.rs::fast_arrival_uses_one_bounded_evidence_gate_and_normal_steps_after_delivery`, `crates/qbit-prism-server/src/vardiff.rs::normal_arrivals_do_not_repeatedly_test_the_initial_noise_gate`, `crates/qbit-prism-server/tests/stratum_protocol.rs::fast_arrival_retries_failed_paired_delivery_and_only_proven_difficulty_resumes`, `crates/qbit-prism-server/tests/config_cli.rs::reconnect_and_initial_convergence_settings_are_bounded_and_optional`; not covered: an explicit `d=` request landing mid-retarget outranking the initial step, the `md=` floor never undercut by an initial step, and the arrival metrics (`qbit_prism_vardiff_initial_retargets_total`, `..._high_diff_arrival_*`), which have no native family | +| `tests/test_prism_vardiff_resume.py` | partial | partial: covered by `crates/qbit-prism-server/tests/stratum_protocol.rs::resumed_difficulty_is_lane_scoped_ttl_bounded_and_explicit_requests_win`, `crates/qbit-prism-server/tests/stratum_protocol.rs::idle_retarget_lowers_reconnect_difficulty_without_renewing_evidence`, `crates/qbit-prism-server/tests/stratum_protocol.rs::fast_arrival_retries_failed_paired_delivery_and_only_proven_difficulty_resumes`, `crates/qbit-prism-server/tests/config_cli.rs::reconnect_and_initial_convergence_settings_are_bounded_and_optional`; not covered: the in-memory `retained_difficulties` LRU bound and eviction, the `PRISM_STRATUM_VARDIFF_RESUME=0` kill switch restoring lane-start reconnects, a re-authorize of a live session never re-resuming, a failed username reservation neither resuming nor retaining, and the resume-outcome metrics (`qbit_prism_vardiff_resume_total`, `..._retained_sessions`, `..._lane_accepted_shares_*`), which have no native family | +| `tests/test_prism_vardiff_service.py` | retired | retired — Python `VardiffService` owner: window capture/reset stamps and service-owned idle metrics; the equivalent window reset lives inside `Vardiff::retarget` and is exercised by `crates/qbit-prism-server/src/vardiff.rs::fast_arrival_uses_one_bounded_evidence_gate_and_normal_steps_after_delivery` | +| `tests/prism_vardiff_test_support.py` | retired | retired — shared fixtures (`FakeLedger`, `RecordingLedger`, `FakeRpc` variants, `SubmitRpc`) for the Python coordinator domain suites | + +## Job builder, template and tip refresh, refresh liveness + +The 2.x.x job-build executor, bundle cache and tip-refresh fanout. Natively one serialized `refresh_once` builds shared work and each session stamps it; promise, executor and epoch mechanics are retired, and the refresh-liveness scenarios (representative independence, CTV yield to a newer tip) are covered by `crates/qbit-prism-server/tests/refresh_liveness.rs` (#375, closed by #433). + +| 2.x.x file | Covered | 3.x.x replacement or status | +| --- | --- | --- | +| `tests/test_prism_job_builder.py` | partial | partial: covered by `crates/qbit-prism-server/tests/ledger_postgres.rs::concurrent_instances_have_one_commit_order_and_stable_snapshots`, `crates/qbit-prism-server/tests/window_read_oracle.rs::rows_on_the_anchor_are_credited_and_rows_after_it_are_not`, `crates/qbit-prism/src/lib.rs::anchor_requires_job_issue_and_accept_time`, `crates/qbit-prism-server/tests/stratum_codec.rs::witness_coinbase_round_trip_and_header_endianness`, `crates/qbit-prism-server/tests/stratum_codec.rs::extranonce_split_ignores_identical_output_and_witness_bytes`, `crates/qbit-prism-server/tests/stratum_protocol.rs::multiple_miners_unique_extranonces_durable_accept_and_duplicate_rejection`, `crates/qbit-prism-server/tests/stratum_protocol.rs::reconnect_on_another_listener_preserves_original_entropy_mask_and_expiry`, `crates/qbit-prism-server/tests/readiness_rpc.rs::observed_readiness_failure_closes_cached_work_and_candidate_settlement`, `crates/qbit-prism-server/tests/readiness_rpc.rs::cached_ctv_work_revalidates_live_floors_and_fences_old_underfunded_jobs`, `crates/qbit-prism-server/tests/support/live_highdiff.rs::observed_tip_advance_fences_old_prepared_jobs_and_health`, `crates/qbit-prism-server/tests/stratum_protocol.rs::admission_timeout_records_failure_and_shutdown_releases_session_stats`, `crates/qbit-prism-server/src/coordinator/d2_bootstrap_tests.rs::an_empty_ledger_pays_the_solver_through_a_synthetic_bootstrap_share`, `crates/qbit-prism-server/src/coordinator/d2_bootstrap_tests.rs::accepted_shares_below_the_2xx_readiness_gate_still_pay_the_window`, `crates/qbit-prism-server/src/coordinator/d2_bootstrap_tests.rs::a_single_share_from_another_miner_takes_the_whole_coinbase` (the bootstrap selection through `refresh_once` and `build_job`: a per-solver synthetic bootstrap share only while the window is empty, the shared window bundle from the first accepted share on), `crates/qbit-prism-server/src/coordinator/miner_tests/refresh.rs::real_refresh_build_persist_publish_and_client_job_delivery_have_constant_rpc_count` (one refresh serves 1, 32 and 128 workers with an identical node-RPC list, per-client job building issues no node RPC, and the prepared bundle is stored once with one issued record per client; #280, closed by #313); not covered, no open issue: reorg-reconcile chain-cache reuse per tip; the 2.x.x ready latch (`PRISM_MIN_READY_MINERS`, per-worker collection bundles until three miners are ready) is retired by decision D2a, whose 3.x.x rule the D2a tests above assert; the Python spool, serve-daemon fallback, executor priority scheduling, cache bounding and subprocess-builder crash clusters are retired | +| `tests/test_prism_job_build_exception_retention.py` | retired | retired — Python traceback/refcount cycle regressions of a ThreadPoolExecutor job flight; no native equivalent | +| `tests/test_prism_job_build_promises.py` | retired | retired — single-flight promise lifecycle of the Python job-build executor; native refresh is one serialized `refresh_once` under `refresh_lock` with no joiner promises | +| `tests/test_prism_job_build_retention.py` | retired | retired — Python cyclic-GC release of completed flights | +| `tests/test_prism_coordinator_job_cache.py` | partial | partial: covered by `crates/qbit-prism-server/tests/readiness_rpc.rs::observed_readiness_failure_closes_cached_work_and_candidate_settlement`, `crates/qbit-prism-server/tests/readiness_rpc.rs::another_frontend_payout_revision_captures_the_same_parent_block_but_fences_its_share`, `crates/qbit-prism-server/tests/readiness_rpc.rs::cached_ctv_work_revalidates_live_floors_and_fences_old_underfunded_jobs`; not covered: none with a native subject; the moved-symbol identity, descriptor routing, executor naming and terminal shutdown cases are retired | +| `tests/test_prism_tip_publication_boundary.py` | partial | partial: covered by `crates/qbit-prism-server/tests/readiness_rpc.rs::observed_readiness_failure_closes_cached_work_and_candidate_settlement`, `crates/qbit-prism-server/tests/readiness_rpc.rs::cached_ctv_work_revalidates_live_floors_and_fences_old_underfunded_jobs`, `crates/qbit-prism-server/tests/support/live_highdiff.rs::observed_tip_advance_fences_old_prepared_jobs_and_health`; not covered: a tip flip between bundle build and publication (the `ready_tip` "tip changed during node readiness proof" guard has no interleaving test), a payout mutation before publication keeping the previous authority | +| `tests/test_prism_tip_refresh.py` | retired | retired — R1/P1 owner-boundary symbol identity, private aliases and descriptor routing of the Python decomposition | +| `tests/test_prism_tip_refresh_delivery.py` | partial | partial: covered by `crates/qbit-prism-server/tests/stratum_protocol.rs::negotiation_reauthorization_retains_original_job_worker_and_grace_expires`, `crates/qbit-prism-server/tests/stratum_protocol.rs::same_parent_payout_replacement_retains_the_superseded_job_for_block_submission`, `crates/qbit-prism-server/tests/stratum_protocol.rs::delivery_stats_follow_admission_progress_generation_and_disconnect`, `crates/qbit-prism-server/tests/readiness_rpc.rs::observed_readiness_failure_closes_cached_work_and_candidate_settlement`, `crates/qbit-prism-server/tests/readiness_rpc.rs::another_frontend_payout_revision_captures_the_same_parent_block_but_fences_its_share`, `crates/qbit-prism-server/tests/support/live_highdiff.rs::observed_tip_advance_fences_old_prepared_jobs_and_health`, `crates/qbit-prism-server/tests/ledger_postgres.rs::verified_landing_reconstructs_audit_and_reorgs_are_revision_fenced`, `crates/qbit-prism-server/tests/ledger_postgres.rs::reconciliation_watches_only_recent_blocks_and_one_mature_checkpoint`, `crates/qbit-prism-server/tests/live_regtest.rs::real_two_server_mining_failover_audit_and_reorg`, `crates/qbit-prism-server/src/api.rs::expired_health_snapshot_returns_503_even_when_last_snapshot_was_ready`, `crates/qbit-prism-server/src/coordinator/miner_tests/refresh.rs::real_refresh_build_persist_publish_and_client_job_delivery_have_constant_rpc_count` (one stored prepared bundle serves 1, 32 and 128 per-client job builds, and the node-RPC list of the refresh does not change with the client count), `crates/qbit-prism-server/src/coordinator/miner_tests/interleavings.rs::late_chain_info_cannot_roll_back_a_newer_observation`, `crates/qbit-prism-server/src/coordinator/miner_tests/blockwait.rs::changed_notification_after_a_newer_poll_verifies_while_real_refresh_is_blocked` (a slow poll cannot regress a newer long-poll observation, and the overtaken refresh fails), `crates/qbit-prism-server/src/coordinator/miner_tests/blockwait.rs::notification_failure_keeps_last_good_readiness_and_cancels_retry_backoff`, `crates/qbit-prism-server/src/coordinator/miner_tests/blockwait.rs::failed_or_malformed_verification_revokes_admission_and_wakes_refresh`, `crates/qbit-prism-server/src/coordinator/miner_tests/blockwait.rs::verification_started_after_notification_still_loses_to_a_later_chain_poll`, `crates/qbit-prism-server/src/coordinator/miner_tests/blockwait.rs::same_hash_notification_has_no_verification_rpc_and_wait_is_cancellable`, `crates/qbit-prism-server/src/coordinator/miner_tests/blockwait.rs::fresh_verification_rpc_is_cancellable_without_publishing_work` (the long-poll failure loop: a failed `waitfornewblock` keeps the task alive and the last good readiness, wakes polling and cancels its backoff on shutdown; a failed or malformed verification revokes admission and the loop waits again; the long poll detects a tip without publishing work), `crates/qbit-prism-server/src/stratum/stale_grace_tests.rs::failed_replacement_keeps_retired_job_until_actual_delivery_then_expires` (a failed replacement build leaves the old job creditable until a replacement is actually delivered, then its grace expires), `crates/qbit-prism-server/tests/stratum_protocol.rs::resumed_expiry_does_not_release_other_retained_work_during_build_failure` (a hundred failed builds later the same socket still receives its next job); not covered, no open issue: reuse of the reorg-reconcile `chain_cache` memo per tip (natively reconciliation runs only inside `refresh_once` under its lock, so the 2.x.x single-flight cases have no separate subject, but no test touches the memo); the template-refresh failure process exit is retired (`PRISM_TEMPLATE_REFRESH_FAILURE_EXIT_SECONDS` now bounds the replacement-build lease, whose clock rule `crates/qbit-prism-server/src/coordinator/miner_tests/refresh.rs::failed_real_refresh_retains_published_credit_until_lease_expires` asserts), as are the unsupported-long-poll classification with its watchdog heartbeat, the Python fanout executor and handler-thread cleanup | +| `tests/test_prism_tip_refresh_epoch.py` | partial | partial: covered by `crates/qbit-prism-server/tests/stratum_protocol.rs::delivery_stats_follow_admission_progress_generation_and_disconnect`, `crates/qbit-prism-server/tests/stratum_protocol.rs::same_parent_payout_replacement_retains_the_superseded_job_for_block_submission`; not covered: none with a native subject; epoch minting, fixpoint, wave outcome vocabulary, wave pass budget and the trust-blocked outcome of the Python epoch fanout are retired | +| `tests/test_prism_tip_refresh_validation.py` | partial | partial: covered by `crates/qbit-prism-server/tests/readiness_rpc.rs::observed_readiness_failure_closes_cached_work_and_candidate_settlement`, `crates/qbit-prism-server/tests/readiness_rpc.rs::another_frontend_payout_revision_captures_the_same_parent_block_but_fences_its_share`, `crates/qbit-prism-server/tests/support/live_highdiff.rs::observed_tip_advance_fences_old_prepared_jobs_and_health`, `crates/qbit-prism-server/src/coordinator/d2_below_target_tests.rs::block_only_credit_survives_a_reorg_that_disconnects_its_block` (reconciliation after a reorg rebuilds the next block's payout from post-reorg state), `crates/qbit-prism-server/src/coordinator/miner_tests/refresh.rs::real_refresh_build_persist_publish_and_client_job_delivery_have_constant_rpc_count` (the chain is validated once per refresh whatever the client count: the node-RPC list, one `getblockheader` included, is identical at 1, 32 and 128 clients and per-client job builds add none), `crates/qbit-prism-server/src/coordinator/miner_tests/blockwait.rs::changed_notification_after_a_newer_poll_verifies_while_real_refresh_is_blocked`, `crates/qbit-prism-server/src/coordinator/miner_tests/compact_authority.rs::cold_proof_rejects_changed_economics_tip_and_stale_template_without_a_lease` (a newer tip seen after the work is built and before it is persisted or published cancels publication at the persistence gate: the reservation is refused with nothing saved and nothing published, and a real refresh overtaken while persisting fails; natively nothing interrupts the bundle build itself, so the 2.x.x per-build-phase cancellation is retired), `crates/qbit-prism-server/src/coordinator/miner_tests/refresh_window.rs::economic_drift_during_fee_probe_defers_until_next_refresh`, `crates/qbit-prism-server/src/coordinator/miner_tests/refresh_window.rs::build_wait_rechecks_payout_revision_drift` (a payout change during a refresh never publishes the older state: the pass either rereads and publishes the latest revision or fails unsaved, and the next pass publishes the latest); the Python fanout executor queue, per-client delivery gate and coordination budget are retired | +| `tests/test_prism_immutable_refresh_artifacts.py` | partial | partial: covered by `crates/qbit-prism-server/tests/support/live_highdiff.rs::observed_tip_advance_fences_old_prepared_jobs_and_health`, `crates/qbit-prism-server/tests/readiness_rpc.rs::observed_readiness_failure_closes_cached_work_and_candidate_settlement`, `crates/qbit-prism-server/tests/stratum_codec.rs::witness_coinbase_round_trip_and_header_endianness`; not covered: the "tip changed between chain observation and template" fetch coherence guard has no test; the snapshot/artifact object-invariant mismatch cases are moot with one immutable `Prepared` | +| `tests/test_prism_refresh_retry_pacing.py` | partial | partial: covered by `crates/qbit-prism-server/tests/readiness_rpc.rs::cached_ctv_work_revalidates_live_floors_and_fences_old_underfunded_jobs`, `crates/qbit-prism-server/tests/readiness_rpc.rs::observed_readiness_failure_closes_cached_work_and_candidate_settlement` (same-tip reuse of prepared work); not covered: failed-pass holdoff spacing with jitter and same-tip `getblocktemplate` skipping; neither knob (`PRISM_TIP_REFRESH_FAILURE_HOLDOFF_SECONDS`, `PRISM_TEMPLATE_CACHE_SECONDS`) exists natively, the retry cadence is the fixed `PRISM_BLOCKPOLL_SECONDS` tick and every pass refetches the template | +| `tests/test_prism_representative_independence.py` | partial | partial: covered by `crates/qbit-prism-server/tests/readiness_rpc.rs::observed_readiness_failure_closes_cached_work_and_candidate_settlement` (shared work is built by `refresh_once` with no connected client, then stamped per worker), the twelve `disconnect_test!` cases of `crates/qbit-prism-server/tests/refresh_liveness.rs` (`bootstrap_disconnect_*` on an empty window and `window_disconnect_*` on a credited one; the macro generates them, so they have no `fn` line to cite: the session that connected first is lost before the refresh, during the chain observation, the template fetch or the window read, after the prepared-work commit, or after publication, and each time the shared work is rebuilt with the same payout window, the remaining sessions receive the new job and have a share accepted, a later authorization reuses the publication with no rebuild, and on an empty window each session's bootstrap work names that session's own worker and payout address, never the lost session's), `crates/qbit-prism-server/tests/refresh_liveness.rs::every_session_disconnects_then_new_tip_work_serves_a_new_identity` (every session disconnects, a new tip is published with nobody connected, and a newcomer is served that tip's work and has a share accepted; #375, closed by #433); not covered, no open issue: a session that authorizes while a same-tip publication is in flight still receiving that publication's job; the representative target, its poll-start reselection and the collection-mode cases (collection identity absent or ineligible, the collection marker and the ready latch) are retired — the shared publication is built with no client identity, and decision D2a replaced collection mode with per-solver bootstrap work whose identity binding the `bootstrap_disconnect_*` cases assert; the 2.x.x reselect-on-reauthorization loop and cached collection bundle have no native counterpart (bootstrap work is rebuilt per call) | +| `tests/test_prism_ctv_refresh_priority.py` | partial | partial: covered by `crates/qbit-prism-server/tests/refresh_liveness.rs::ctv_yields_after_current_chunk_and_a_later_pass_finishes_remaining_rows` (a refresh to a newer tip is held at `getblocktemplate`, after its chain observation and before its parent lookup, while one fanout completion is held: the pass ends after that row with `tip_refresh_yields_total` 1 and no claim on the rest, a later pass finishes the remaining rows, a same-tip refresh before a further pass yields nothing, and the yield, chunk-row and chunk-duration series carry only `le` labels in a fixed set of 19 series), `crates/qbit-prism-server/tests/refresh_liveness.rs::new_tip_serves_work_while_ctv_completion_is_held_then_ctv_yields` (the same case with the newer tip already published and serving work), `crates/qbit-prism-server/tests/refresh_liveness.rs::ctv_chunk_metrics_count_an_attempt_whose_completion_failed_to_persist`, `crates/qbit-prism-server/tests/refresh_liveness.rs::ctv_settles_after_the_replacement_build_budget_when_publication_keeps_failing` (a detected tip whose refresh fails before publication still defers the pass, for at most the replacement-build budget), `crates/qbit-prism-server/tests/refresh_liveness.rs::ctv_settles_when_the_observed_tip_predates_the_pass`, `crates/qbit-prism-server/src/coordinator/miner_tests/observations.rs::only_an_observation_newer_than_a_pass_supersedes_its_tip`; not covered, no open issue: a pending tip surviving an older refresh that finishes late (natively the "tip observation superseded before work publication" check in `crates/qbit-prism-server/src/coordinator/tip_observation.rs`, which has no test; the `TipState::observe` sequence guard beside it is tested by `crates/qbit-prism-server/src/coordinator/miner_tests/interleavings.rs::late_chain_info_cannot_roll_back_a_newer_observation` and `crates/qbit-prism-server/src/coordinator/miner_tests/blockwait.rs::verification_started_after_notification_still_loses_to_a_later_chain_poll`), and the per-pass `PRISM_CTV_BROADCASTER_LIMIT` bound in `crates/qbit-prism-server/src/broadcaster.rs`, which no test sets (#375 closed without either); the pending token itself, the daemon `chunk_size` wiring and active-refresh cancellation are retired (natively pending is derived on each check from the observed and published tip, and a chunk is one claimed fanout) | + +## Coordinator configuration, shutdown, background services, self-check, capacity evidence and Compose + +Configuration is validated eagerly in `Config::from_env` and checked by `config_cli.rs`; `self-check`, `healthcheck` and `capacity-evidence` are native subcommands; the Compose contract is still tested in Python under `tests/`. + +| 2.x.x file | Covered | 3.x.x replacement or status | +| --- | --- | --- | +| `tests/test_prism_coordinator_config.py` | partial | partial: covered by `crates/qbit-prism-server/tests/config_cli.rs::valid_regtest_and_production_configuration_are_checked_without_services`, `crates/qbit-prism-server/tests/config_cli.rs::large_resource_budgets_fail_instead_of_truncating_or_panicking`, `crates/qbit-prism-server/tests/config_cli.rs::reconnect_and_initial_convergence_settings_are_bounded_and_optional`, `crates/qbit-prism-server/tests/config_cli.rs::production_rejects_test_seeds_and_reused_or_mismatched_trust_keys`, `crates/qbit-prism-server/tests/config_cli.rs::mainnet_aliases_allow_bounded_stale_grace_and_enforce_production_rules`, `crates/qbit-prism-server/tests/config_cli.rs::mainnet_requires_a_valid_genesis_pin_before_startup`, `crates/qbit-prism-server/tests/config_cli.rs::production_flags_reject_regtest_and_readiness_budgets_are_validated`, `crates/qbit-prism-server/tests/config_cli.rs::retired_writer_sessions_and_invalid_backend_urls_fail_before_startup`, `crates/qbit-prism-server/tests/config_cli.rs::ctv_fee_premiums_are_validated_before_automatic_or_explicit_fee_work`, `crates/qbit-prism-server/tests/config_cli.rs::run_rejects_invalid_highdiff_settings_before_connecting`, `crates/qbit-prism-server/src/config.rs::mainnet_genesis_pin_is_required_and_matches_the_node`, `crates/qbit-prism-server/tests/readiness_rpc.rs::public_rpc_readiness_requires_synced_headers_and_configured_peer_floor`, `crates/qbit-prism-server/src/readiness.rs::template_age_boundary_zero_and_future_time_match_legacy_policy`, `crates/qbit-prism-server/src/coordinator/fee_policy_tests.rs::configured_ctv_fee_cannot_bypass_either_live_relay_floor`, `crates/qbit-prism-server/src/coordinator/fee_policy_tests.rs::estimated_ctv_fee_obeys_the_same_floor_and_preserves_the_premium`, `crates/qbit-prism-server/src/coordinator/fee_policy_tests.rs::unavailable_or_invalid_live_floors_fail_closed`, `crates/qbit-prism-server/src/coordinator.rs::exact_fee_estimate_rejects_zero_negative_and_malformed_values`, `crates/qbit-prism-server/tests/stratum_codec.rs::version_mask_advertisement_is_authoritative`, `crates/qbit-prism-server/tests/stratum_codec.rs::targets_preserve_network_block_solutions_above_listener_floor`, `crates/qbit-prism-server/tests/stratum_codec.rs::extranonce_split_ignores_identical_output_and_witness_bytes`, `crates/qbit-prism-server/tests/stratum_codec.rs::vardiff_uses_actual_accepted_work_and_limits_idle_step`, `crates/qbit-prism-server/tests/authorization_rpc.rs::authorization_fallback_requires_a_definitive_address_result`, `crates/qbit-prism-server/tests/stratum_protocol.rs::negotiation_reauthorization_retains_original_job_worker_and_grace_expires`, `crates/qbit-prism-server/tests/stratum_protocol.rs::resumed_difficulty_is_lane_scoped_ttl_bounded_and_explicit_requests_win`, `crates/qbit-prism-server/tests/stratum_protocol.rs::multiple_miners_unique_extranonces_durable_accept_and_duplicate_rejection`, `crates/qbit-prism-server/tests/stratum_protocol.rs::per_username_capacity_preserves_prior_authorization_on_failed_reauthorize`, `crates/qbit-prism-server/tests/stratum_protocol.rs::reauthorization_retains_original_username_capacity_until_timer_expiry`, `crates/qbit-prism-server/tests/stratum_protocol.rs::reauthorization_disconnect_reclaims_current_and_retained_username_capacity`, `crates/qbit-prism/src/lib.rs::day_one_floor_formula_matches_pq_spend_cost`, `crates/qbit-prism/src/lib.rs::fixed_floor_override_replaces_formula_floor`, `crates/qbit-prism/src/lib.rs::pool_fee_first_without_a_pool_fee_policy_is_rejected`, `crates/qbit-prism/src/lib.rs::unknown_coinbase_output_policy_values_are_rejected`; not covered: env-level validation of `PRISM_POOL_FEE_*`, `PRISM_PAYOUT_*` aliases, `PRISM_COINBASE_TAG` and `PRISM_COINBASE_OUTPUT_POLICY` (implemented in `Config::from_env`, no test), the mainnet CTV explicit-fee-rate refusal (no test), the production `PRISM_STRATUM_MAX_CONNECTIONS > 0` and lab-difficulty refusals in `from_env`, the testnet default username fallback, payout-address cache LRU/TTL and concurrent single-flight, `mining.suggest_difficulty` (implemented, untested), the global connection limit, GBT signet rules and template-fingerprint clock-field exclusion (untested); the Python switch-interval, session-guard, replay-page-size, heartbeat-capability and accept-thread cases (about 25 tests) are retired with the Python runtime | +| `tests/test_prism_coordinator_config_loading.py` | retired | retired — Python loader/RPC-seam decomposition (mapping-based loader, frozen `LifecycleConfig`, deferred validation, module re-exports) with no native counterpart; native settings are validated eagerly in `Config::from_env` | +| `tests/test_prism_coordinator_shutdown.py` | partial | partial: covered by `crates/qbit-prism-server/src/coordinator/candidate_lease_tests.rs::submit_loop_shutdown_drops_waiting_work_and_allows_recovery`, `crates/qbit-prism-server/tests/stratum_protocol.rs::admission_timeout_records_failure_and_shutdown_releases_session_stats`, `crates/qbit-prism-server/src/ledger/instances.rs::startup_rows_are_inactive_not_live`; not covered: SIGTERM/SIGINT handling and the 30 s drain-then-abort path of `server::run`, refusal of new Stratum submits during shutdown (no pool-closed rejection exists natively); the writer-lease heartbeat, watchdog-exit, lease-release and single-writer cases (about 50 tests) are retired with the single-writer lease | +| `tests/test_prism_background_services.py` | partial | partial: covered by `crates/qbit-prism-server/tests/rollups_postgres.rs::competing_frontends_fold_each_share_once_including_late_clocked_work`, `crates/qbit-prism-server/tests/rollups_postgres.rs::failed_bucket_write_rolls_back_the_watermark_and_every_other_grain`, `crates/qbit-prism-server/src/metrics/runtime.rs::progress_is_monotonic_per_operation_and_idle_is_not_stalled`, `crates/qbit-prism-server/tests/observability/runtime.rs::blocked_critical_poll_degrades_real_http_while_another_worker_remains_healthy`, `crates/qbit-prism-server/tests/observability/runtime.rs::idle_slow_async_work_and_cancelled_operations_do_not_leave_false_stalls`, `crates/qbit-prism-server/src/server.rs::publication_guard_finishes_before_slow_maintenance`; not covered: the rollup `run` loop itself (backlog drain vs interval sleep, warn-and-retry on error, clean stop on shutdown), and process exit on a critical task failure (`server::run` `join_next`); the Python thread registry and watchdog hard-exit cases are retired | +| `tests/test_prism_bounded_executor.py` | partial | partial: covered by `crates/qbit-prism-server/src/coordinator/candidate_lease_tests.rs::saturated_build_capacity_keeps_the_lease_until_one_candidate_confirms`, `crates/qbit-prism-server/src/coordinator/candidate_lease_tests.rs::canceled_or_lost_renewal_cannot_resume_queued_work`, `crates/qbit-prism-server/src/coordinator/candidate_lease_tests.rs::submit_loop_shutdown_drops_waiting_work_and_allows_recovery`, `crates/qbit-prism-server/tests/config_cli.rs::large_resource_budgets_fail_instead_of_truncating_or_panicking`; not covered: queue-bound accounting (active count excluded from the bound); the Python GC-retention and re-export cases are retired | +| `tests/test_prism_completion_callbacks.py` | retired | retired — CPython `Future` done-callback and frame-retention mechanics with no native counterpart | +| `tests/test_prism_singleflight_exception_retention.py` | retired | retired — CPython traceback reference cycles in Python single-flight owners; the native cache single-flight is exercised by `crates/qbit-prism-server/src/api/response_cache.rs::stale_entry_returns_immediately_while_one_refresh_replaces_it` but retention is not a Rust concern | +| `tests/test_prism_self_check.py` | partial | partial: covered by `crates/qbit-prism-server/src/ledger/instances.rs::empty_table_is_not_live`, `crates/qbit-prism-server/src/ledger/instances.rs::stale_rows_do_not_count`, `crates/qbit-prism-server/src/ledger/instances.rs::startup_rows_are_inactive_not_live`, `crates/qbit-prism-server/src/ledger/instances.rs::inclusive_freshness_boundary_and_unready_servers_are_live`, `crates/qbit-prism-server/src/ledger/instances.rs::single_live_instance_warns`, `crates/qbit-prism-server/src/ledger/instances.rs::future_dated_row_is_unknown`, `crates/qbit-prism-server/src/ledger/instances.rs::failed_heartbeat_connection_is_not_zero_or_healthy`, `crates/qbit-prism-server/src/ledger/instances.rs::heartbeat_sql_observes_empty_stale_and_missing_table`, `crates/qbit-prism-server/tests/healthcheck_cli.rs::audit_disabled_healthcheck_uses_only_readiness_on_primary_or_highdiff`, `crates/qbit-prism-server/tests/healthcheck_cli.rs::invalid_or_unready_stratum_responses_fail_the_actual_health_command`, `crates/qbit-prism-server/tests/stratum_protocol.rs::highdiff_floor_still_forwards_valid_block_below_share_target`, `crates/qbit-prism-server/src/config.rs::mainnet_genesis_pin_is_required_and_matches_the_node`, `crates/qbit-prism-server/tests/readiness_rpc.rs::public_rpc_readiness_requires_synced_headers_and_configured_peer_floor`, `crates/qbit-prism-server/src/coordinator/fee_policy_tests.rs::unavailable_or_invalid_live_floors_fail_closed`, `crates/qbit-prism-server/tests/config_cli.rs::mainnet_aliases_allow_bounded_stale_grace_and_enforce_production_rules`, `tests/test_check_env_production_gate.py::test_production_mainnet_prelaunch_requires_explicit_authorization`, `tests/test_check_env_production_gate.py::test_mainnet_requires_canonical_explicit_chain_selector`, `tests/test_check_env_production_gate.py::test_mainnet_requires_pinned_genesis_hash`, `tests/test_check_env_production_gate.py::test_production_git_provider_requires_exact_commit_pin`, `tests/test_check_env_production_gate.py::test_mainnet_provenance_passes_without_capacity_evidence_and_verifies_pin`, `tests/test_check_env_production_gate.py::test_production_rejects_unsafe_prism_difficulty_profiles`, `tests/test_check_env_production_gate.py::test_mainnet_accepts_bounded_stale_grace`, `tests/test_check_env_production_gate.py::test_production_auxpow_requires_explicit_payouts`, `tests/test_check_env_production_gate.py::test_ctv_rejects_non_positive_explicit_fee_rate_before_docker_check`, `tests/test_makefile_lifecycle.py::test_deployment_env_replaces_repository_env_in_make`; not covered: mainnet pre-launch mode (non-fatal IBD while launch is unauthorized), highdiff-probe failure modes (rejected authorize, connection closed without difficulty), and self-check-level coinbase-output-policy / pool-fee-first static checks; the ready-miner threshold (`PRISM_MIN_READY_MINERS`) is retired by decision D2a, and `crates/qbit-prism-server/src/coordinator/d2_bootstrap_tests.rs::accepted_shares_below_the_2xx_readiness_gate_still_pay_the_window` asserts the gate is not restored | +| `tests/test_prism_capacity_evidence.py` | partial | partial: covered by `crates/qbit-prism-server/tests/capacity.rs::qualifies_exact_two_times_capacity_and_reconciled_identifiers`, `crates/qbit-prism-server/tests/capacity.rs::examples_need_explicit_opt_in`, `crates/qbit-prism-server/tests/capacity.rs::qualification_requires_all_external_bindings`, `crates/qbit-prism-server/tests/capacity.rs::validates_time_uuid_and_subject`, `crates/qbit-prism-server/tests/capacity.rs::binds_every_configuration_and_checks_vardiff_math`, `crates/qbit-prism-server/tests/capacity.rs::requires_durability_forecast_and_latency_limits`, `crates/qbit-prism-server/tests/capacity.rs::every_fault_phase_must_be_nontrivial_and_sustain_two_times_forecast`, `crates/qbit-prism-server/tests/capacity.rs::detects_share_loss_hash_mismatch_and_phase_totals`, `crates/qbit-prism-server/tests/capacity.rs::strict_objects_and_numbers_reject_ambiguous_claims`, `crates/qbit-prism-server/tests/capacity.rs::loader_rejects_duplicate_keys_at_any_depth`, `crates/qbit-prism-server/tests/capacity.rs::bound_configuration_uses_only_native_runtime_settings` (the bound key set equals the fixture's, and every key is in the native settings inventory and none in the retired one), `crates/qbit-prism-server/tests/capacity.rs::rejects_retired_schemas_and_configuration_keys_by_name` (v1 and v2 artifacts and the three 2.x.x-only configuration keys are refused by name, in the artifact and in `--expect`); not covered, no open issue: the `capacity-evidence` CLI subprocess requiring `--allow-example-evidence-for-tests` (library-level opt-in only) | +| `tests/test_prism_compose_profile.py` | partial | partial: covered by `tests/test_mining_compose_profiles.py::test_each_mining_profile_has_an_exact_service_graph`, `tests/test_mainnet_compose_contract.py::test_prism_services_receive_configured_database_and_read_limits`, `tests/test_mainnet_compose_contract.py::test_prism_services_receive_the_complete_rpc_endpoint_override`, `tests/test_mainnet_compose_contract.py::test_public_read_tier_serves_from_the_standby`, `tests/test_mainnet_compose_contract.py::test_public_read_tier_waits_on_the_standby_not_the_primary`, `tests/test_mainnet_compose_contract.py::test_replica_storage_is_separate_from_the_primary`, `tests/test_mainnet_compose_contract.py::test_operator_services_have_restart_policies`, `tests/test_mainnet_compose_contract.py::test_prism_runtime_has_no_capacity_evidence_dependency`, `tests/test_mainnet_compose_contract.py::test_host_port_bindings_are_explicit`, `crates/qbit-prism-server/tests/healthcheck_cli.rs::invalid_or_unready_stratum_responses_fail_the_actual_health_command`, `crates/qbit-prism-server/src/api.rs::expired_health_snapshot_returns_503_even_when_last_snapshot_was_ready`, `tests/test_prism_container_hardening.py::test_role_healthchecks_match_their_listeners` (both lab services run the native `healthcheck` subcommand), `tests/test_prism_container_hardening.py::test_native_processes_receive_only_native_prism_settings` (no `PRISM_*` name outside the native inventory reaches either lab service), `crates/qbit-prism-server/tests/config_cli.rs::unread_environment_reports_every_name_without_values` (`check-config` names every retired setting and refuses it in production), `tests/test_prism_container_hardening.py::test_public_api_is_seedless`, `tests/test_mainnet_compose_contract.py::test_public_read_tier_is_seedless` (no signing seed or operator token on the public API in the lab and production renders; no volume in the production render), `tests/test_prism_public_credentials.py::test_missing_and_empty_public_dsn_preserve_supported_defaults` (in the lab render and every overlay without an external database the public API's default DSN names the standby and replica mode is `require`), `tests/test_mainnet_compose_contract.py::test_operator_state_uses_explicit_host_bind_mounts` (one audit bind mount on the production coordinator; the public API runs `qbit-prism-server public-api` and mounts nothing), `crates/qbit-prism-server/src/tools.rs::audit_root_env_and_cli_precedence` (`import-audits` resolves `PRISM_AUDIT_DIR` when `--root` is absent, and `--root` wins); not covered, no open issue: the public API receiving no writer or schema-initialisation settings (the ledger-writer key and session token and the init-schema switch are still native settings, so the native-settings-only test cannot catch them, and no remaining test asserts their absence from `prism-public-api`), lab-render pass-through of the native coordinator and public-api settings (an override reaching the container, including `PRISM_PUBLIC_STRATUM_URL` and the advertised Stratum ports on the public API), the lab named audit volume, the `nofile` limit overrides, the published public-api port, the qbitd lab defaults (launch-readiness variables unset, `QBIT_NODE_EXTRA_ARG` as one argv entry), the pg_hba mount and rules, the replica service shape and healthcheck, and the replica-bootstrap script text; the public-api read-only audit mount (the native public API serves audits from PostgreSQL), the coordinator-carries-no-public-knobs check (the native coordinator serves `/public/v1` in the combined role and reads those settings) and 33 of the 66 `PRISM_*` coordinator settings the 2.x.x test asserted (now in the retired-settings inventory) are retired | + +## Metrics, observability, process telemetry and progress health + +The native registry ships the cutover-minimum series (#308, `docs/prism-native-metrics.md`). Families with no native producer are listed per row; glibc and Python-GC instruments are retired. + +| 2.x.x file | Covered | 3.x.x replacement or status | +| --- | --- | --- | +| `tests/test_prism_metrics.py` | partial | partial: covered by `crates/qbit-prism-server/tests/observability/registry.rs::histogram_buckets_are_cumulative_and_rejects_are_closed`, `crates/qbit-prism-server/tests/observability/registry.rs::idle_http_scrape_has_unique_help_and_type_for_every_family`, `crates/qbit-prism-server/src/metrics/registry.rs::wrong_family_operations_fail_before_corrupting_exposition`; not covered: the payout-window build-phase, block-submitter/backoff, cleanup-backlog, collapse, accepted-preview attribution, ledger-read-gate and accepted-parent-unresolved-depth families (no native producer; no candidate issue); the Python renderer byte-parity, removed re-export, mallinfo/component-cardinality and GC/lease-monitor stall tests are retired with the Python runtime | +| `tests/test_prism_coordinator_metrics.py` | partial | partial: covered by `crates/qbit-prism-server/tests/observability/stratum.rs::complete_submit_frame_to_successful_ack_excludes_partial_frame_and_post_ack_hint`, `crates/qbit-prism-server/tests/observability/registry.rs::pending_candidate_count_and_age_move_together_without_database_prerequisites`, `crates/qbit-prism-server/tests/observability_database.rs::collector_uses_real_schema_pending_rows_and_failed_read_semantics`, `crates/qbit-prism-server/src/metrics/snapshots.rs::coverage_gap_preserves_strict_95_percent_boundary_and_ages_until_recovery`, `crates/qbit-prism-server/tests/observability/registry.rs::startup_scrape_renders_registry_without_fabricating_publication`, `crates/qbit-prism-server/src/api/metrics_snapshot/tests.rs::expired_publication_fails_closed_through_the_router_and_recovers`, `crates/qbit-prism-server/tests/api_contract.rs::health_reads_runtime_snapshot_without_database_access`, `crates/qbit-prism-server/src/metrics/snapshots.rs::known_legacy_health_fields_follow_pinned_types_without_inventing_missing_values`, `crates/qbit-prism-server/tests/observability/runtime.rs::blocked_critical_poll_degrades_real_http_while_another_worker_remains_healthy`, `crates/qbit-prism-server/src/server.rs::publication_guard_finishes_before_slow_maintenance`, `crates/qbit-prism-server/src/coordinator/candidate_lease_tests.rs::saturated_build_capacity_keeps_the_lease_until_one_candidate_confirms` and `crates/qbit-prism-server/src/coordinator/candidate_lease_tests.rs::crash_after_offer_before_landing_recovers_on_another_frontend_without_a_second_offer` (`block_submit_seconds`, the proof-to-first-offer latency in #278's cutover-minimum set, is populated since #391: the normal path records exactly one first-offer sample, and the frontend that finishes a crashed offer records none); not covered: job-build/cache/payout-phase/tip-refresh IPC histograms, per-worker share and rejection counters, CTV broadcaster progress, audit-artifact storage gauges, coordinator-lock counters, vardiff idle-sweep families; the thread-heartbeat watchdog (pause names, hard exit, retry-wait slices) is retired with the Python thread model | +| `tests/test_prism_observability.py` | partial | partial: covered by `crates/qbit-prism-server/src/api/metrics_snapshot/tests.rs::age_uses_monotonic_time_and_exceeds_the_budget_strictly`, `crates/qbit-prism-server/src/api/metrics_snapshot/tests.rs::missing_snapshot_is_unavailable_even_after_the_budget`, `crates/qbit-prism-server/src/api/metrics_snapshot/tests.rs::expired_publication_fails_closed_through_the_router_and_recovers`, `crates/qbit-prism-server/tests/api_contract.rs::metrics_freshness_headers_cover_both_roles_before_publication`, `crates/qbit-prism-server/tests/api_contract.rs::metrics_publication_preserves_samples_and_restores_fresh_headers`, `crates/qbit-prism-server/tests/api_contract.rs::health_reads_runtime_snapshot_without_database_access`, `crates/qbit-prism-server/src/api.rs::expired_health_snapshot_returns_503_even_when_last_snapshot_was_ready`, `crates/qbit-prism-server/tests/observability/runtime.rs::wall_clock_fields_do_not_change_monotonic_health_or_mask_a_base_failure`, `crates/qbit-prism-server/src/metrics/snapshots.rs::coverage_gap_preserves_strict_95_percent_boundary_and_ages_until_recovery`, `crates/qbit-prism-server/src/metrics/snapshots.rs::collector_expiry_and_failure_preserve_last_success_time_without_fabricating_values`, `crates/qbit-prism-server/src/metrics/snapshots.rs::out_of_order_collections_and_cancelled_older_attempt_cannot_replace_newer_values`, `crates/qbit-prism-server/tests/observability/stratum.rs::pending_initial_work_moves_and_cancellation_does_not_fabricate_share_events`; not covered: the semantic-coverage delivery-health readiness decisions (sustained semantic gap fails closed after the deadline, genuine first-job starvation, rapid tip churn stays ready, overload/reject-storm/cap-saturated reasons), the semantic-currency adapter (payout/session generation disqualification), and `metrics_collection_failures_total` with the invalid-payload class | +| `tests/test_prism_process_telemetry.py` | retired | retired — glibc `mallinfo2` binding, Python GC generation set and `PRISM_MALLOC_TELEMETRY` no longer exist; the native process collector reads `/proc/self/status` (`crates/qbit-prism-server/tests/observability/registry.rs::process_collector_reads_real_input_and_distinguishes_failure_from_zero`) | +| `tests/test_prism_heap_census.py` | retired | retired — Python heap census, `tracemalloc`, `malloc_trim`, allocator environment pins and the RSS-bound CLI are Python-runtime instruments with no native subject | +| `tests/test_prism_allocator_experiment.py` | retired | retired — glibc arena probe and the storm instrument's `--heap-report` driver are Python-runtime tooling deleted with `tests/prism_candidate_storm.py` | +| `tests/test_prism_progress_health.py` | partial | partial: covered by `crates/qbit-prism-server/src/metrics/runtime.rs::progress_is_monotonic_per_operation_and_idle_is_not_stalled`, `crates/qbit-prism-server/tests/observability/runtime.rs::idle_slow_async_work_and_cancelled_operations_do_not_leave_false_stalls`, `crates/qbit-prism-server/tests/observability/runtime.rs::wall_clock_fields_do_not_change_monotonic_health_or_mask_a_base_failure`, `crates/qbit-prism-server/src/server.rs::publication_progress_uses_configured_freshness_budget`, `crates/qbit-prism-server/src/server.rs::publication_guard_finishes_before_slow_maintenance`, `crates/qbit-prism-server/tests/api_contract.rs::health_reads_runtime_snapshot_without_database_access`, `crates/qbit-prism-server/src/api.rs::expired_health_snapshot_returns_503_even_when_last_snapshot_was_ready`, `crates/qbit-prism-server/tests/support/live_highdiff.rs::observed_tip_advance_fences_old_prepared_jobs_and_health`, `crates/qbit-prism-server/src/readiness.rs::template_age_boundary_zero_and_future_time_match_legacy_policy`, `crates/qbit-prism-server/tests/readiness_rpc.rs::observed_readiness_failure_closes_cached_work_and_candidate_settlement`, `crates/qbit-prism-server/tests/stratum_protocol.rs::delivery_stats_follow_admission_progress_generation_and_disconnect`; not covered: per-client delivery proof as the readiness criterion (a registered job is not proof before the socket send, payout change without replacement delivery, partial fanout stays pending), the fixed reason vocabulary and order, the `refresh_pending`/`tip_poll_age`/`current_generation_delivery_age`/`bundle_build_oldest_age` gauges, and the "new tip observed but not published for N seconds" deadline | + +## Public dashboard API and the public read tier + +All 13 `/public/v1` routes, the OpenAPI document and the fixtures are at parity per the readiness review; the rows record what the native contract tests assert and what they do not. + +| 2.x.x file | Covered | 3.x.x replacement or status | +| --- | --- | --- | +| `tests/test_prism_public_dashboard_api.py` | partial | partial: covered by `crates/qbit-prism-server/tests/api_contract.rs::validation_preserves_public_and_operator_error_shapes`, `crates/qbit-prism-server/tests/api_contract.rs::cache_etag_and_cors_are_usable_by_dashboards`, `crates/qbit-prism-server/tests/api_2x.rs::block_views_markers_and_network_estimate_keep_2x_contracts`, `crates/qbit-prism-server/tests/api_2x.rs::chart_rollups_match_raw_boundaries_tail_and_missing_progress`, `crates/qbit-prism-server/tests/api_2x.rs::public_service_is_read_only_and_bounds_http_database_work`, `crates/qbit-prism-server/tests/api_2x.rs::content_addressed_bytes_and_legacy_fallback_headers_are_exact`, `crates/qbit-prism-server/tests/api_database.rs::shared_database_serves_all_contracts_and_global_reward_ranks`, `crates/qbit-prism-server/tests/api_database.rs::accepted_public_blocks_and_earnings_follow_confirmed_chain_state`, `crates/qbit-prism-server/src/api/charts.rs::network_rate_is_exact_nullable_and_nonnegative`, `crates/qbit-prism-server/src/api/charts.rs::dual_rates_keep_raw_credit_and_gap_aware_trailing_rates`, `crates/qbit-prism-server/src/api/charts.rs::only_v2_or_smoothed_series_trim_a_partial_leading_bucket`, `crates/qbit-prism-server/src/api/public.rs::qbit_scaled_units_are_exact`, `crates/qbit-prism-server/src/api/public.rs::small_hashrates_remain_decimal_strings`, `crates/qbit-prism-server/src/api/response_cache.rs::stale_entry_returns_immediately_while_one_refresh_replaces_it`, `crates/qbit-prism-server/src/api/public_service.rs::every_sql_statement_uses_the_remaining_request_budget`, `crates/qbit-prism-server/tests/api_config.rs::malformed_api_settings_fail_without_silent_fallback` (an out-of-range or non-numeric pool fee, minimum payout, cache lifetime, cache flag or read timeout fails startup naming the setting), `crates/qbit-prism-server/tests/api_config.rs::operator_token_file_and_nondefault_settings_are_parsed_at_startup` (the pool fee, `PRISM_PUBLIC_MINIMUM_PAYOUT_BITS` taking precedence over `PRISM_PAYOUT_MIN_OUTPUT_BITS`, the read timeout, the cache kill switch, and the default and artifact cache lifetimes), `crates/qbit-prism-server/tests/config_cli.rs::malformed_public_timeout_fails_check_config`; not covered, no open issue: `PRISM_PUBLIC_READ_STATEMENT_TIMEOUT_SECONDS=0` disabling the deadline, the high-diff endpoint derivation and the `PRISM_PUBLIC_STRATUM_HIGHDIFF_URL` override, `PRISM_PUBLIC_STRATUM_URL` and the pool fee as rendered in the mining-configuration body, the valid-value fallback to `PRISM_PAYOUT_MIN_OUTPUT_BITS` and the legacy `PRISM_PAYOUT_MIN_OUTPUT_SATS`, the aggregate-class cache lifetime settings and the kill switch's effect on response headers (#288 closed without these), concurrent-miss coalescing and non-cacheable coalescing, cache-key hash-case canonicalization, expired-before-fresh cache reaping, pending-fanout pagination past 1,000 rows, non-witness txid fallback, the cpfp-anchor spendability rule, miner row builders not inventing values, pending-maturity netting; the clamp-and-tolerate pool-fee parsing and the skip-an-invalid-minimum-payout fallback are retired (strict startup parsing replaces them); the memory-ledger cases are retired (no in-memory ledger natively) | +| `tests/test_prism_public_read_service.py` | partial | partial: covered by `crates/qbit-prism-server/tests/api_2x.rs::public_service_is_read_only_and_bounds_http_database_work`, `crates/qbit-prism-server/tests/support/public_replica.rs::read_only_public_service_tracks_real_standby_stream_and_refuses_promotion`, `crates/qbit-prism-server/src/api/public_service.rs::replica_liveness_uses_heartbeat_and_ages_failed_probes`, `crates/qbit-prism-server/src/api/public_service.rs::every_sql_statement_uses_the_remaining_request_budget`, `crates/qbit-prism-server/src/api/public_service/metrics_tests.rs::metrics_headers_follow_probe_age_not_database_success_or_wall_time`, `crates/qbit-prism-server/src/api/response_cache.rs::database_outage_serves_only_fresh_bounded_entries_and_never_reads_origin`, `crates/qbit-prism-server/src/api/response_cache.rs::stale_entry_returns_immediately_while_one_refresh_replaces_it`, `crates/qbit-prism-server/src/api/response_cache.rs::healthy_origin_refreshes_an_entry_that_exceeded_its_budget`, `crates/qbit-prism-server/tests/api_contract.rs::metrics_freshness_headers_cover_both_roles_before_publication`, `crates/qbit-prism-server/tests/api_config.rs::public_and_coordinator_rpc_settings_match_in_isolated_processes`; not covered: exhaustive per-route staleness-budget publication and refusal (only `/public/v1/blocks` = 15 s is asserted), the unrecognised `PRISM_PUBLIC_REPLICA_MODE` and missing `PRISM_PUBLIC_STRATUM_URL` startup refusals, the replica gauges and refusal counters on `/metrics`, the replay-lag response header, database-outage recovery dropping the degraded headers, per-route "every database-backed route degrades and the rest do not"; the registry-agreement, contract-equality-with-dispatch, no-writer-lock landmine, coordinator-subtraction and owed-balance read-model cases are retired (native has one handler set, a read-only pool and no writer lock) | +| `tests/test_public_dashboard_api_contract.py` | partial | partial: covered by `crates/qbit-prism-server/tests/api_contract.rs::two_x_fixtures_preserve_versioned_row_shapes_and_chart_counts`, `crates/qbit-prism-server/tests/api_contract.rs::validation_preserves_public_and_operator_error_shapes`; not covered: OpenAPI route and schema-tag declarations, the retired-nomenclature scan, fixture field conventions, pagination and earnings/reward-share/ETA arithmetic across the 18 fixtures, compact-bits difficulty derivation in fixtures, cache-header and env-knob documentation checks (no native test reads `docs/public-dashboard-api-v1.openapi.yaml`; no candidate issue) | +| `tests/test_prism_endpoint_registry.py` | partial | partial: covered by `crates/qbit-prism-server/tests/api_2x.rs::public_service_is_read_only_and_bounds_http_database_work`, `crates/qbit-prism-server/tests/support/public_replica.rs::read_only_public_service_tracks_real_standby_stream_and_refuses_promotion`, `crates/qbit-prism-server/tests/api_contract.rs::validation_preserves_public_and_operator_error_shapes`, `crates/qbit-prism-server/tests/api_contract.rs::metrics_freshness_headers_cover_both_roles_before_publication`; not covered: the per-route staleness-budget derivation table and the pinned route counts; the registry-versus-routing drift checks and the `do_GET`-only AST scan are retired with [`endpoint_registry.py`](https://github.com/Qbit-Org/qbit-mining-bootstrap/blob/700fae0d666443b8a050e0143258c74e02be0926/lab/prism/endpoint_registry.py) | + +## Performance instruments (tests/perf) + +Measurement drivers, not assertions. Each measured a Python-runtime term except the socket-level Stratum load driver, whose successor is the on-demand `crates/qbit-prism-load` harness (#342) beside the asserted share-append floor in `crates/qbit-prism-server/tests/throughput_floor.rs` (#323), and the Rust paging allocation study, whose property is now a unit test. The `tests/perf/*.md` notes remain on HEAD. + +| 2.x.x file | Covered | 3.x.x replacement or status | +| --- | --- | --- | +| `tests/perf/candidate_storage.py` | retired | retired — probes the Python chunked candidate store (#255 `prepare`/append/hydrate/retry with a writer-session proof actor), which the native JSONB candidate row replaced; the native storage-scale evidence is the different `crates/qbit-prism-server/tests/jsonb_ceiling_gate.rs` ratchet | +| `tests/perf/candidate_window_membership.py` | retired | retired — Python bounded-membership versus `json_agg` codec experiment with a Python lease-guard actor; neither the codec nor the lease guard exists natively | +| `tests/perf/canonical_audit_retirement.py` | retired | retired — Python `CanonicalAuditBundleView` codec/ownership and `tracemalloc` experiment; the Python audit view does not exist natively | +| `tests/perf/serialize_block_share_cost.py` | retired | retired — measured CPython interpreter CPU of `direct_stratum.serialize_block` per accepted share; the native codec only assembles block hex for block-passing submissions (`crates/qbit-prism-server/tests/stratum_codec.rs::witness_coinbase_round_trip_and_header_endianness` asserts empty `block_hex` otherwise) | +| `tests/perf/stratum_session_concurrency_load.py` | partial | partial: covered by `crates/qbit-prism-server/tests/throughput_floor.rs::share_append_throughput_floor` (asserts that 1, 2 and 4 concurrent appenders each sustain the configured shares/s floor through `Ledger::append` against PostgreSQL, a level below it being measured once more so that only a second miss fails (#683), the durable write path the driver's `--ledger postgres` mode included; it speaks no Stratum), `crates/qbit-prism-load/tests/harness.rs::a_reconnect_across_several_failed_attempts_reports_the_whole_outage` and `crates/qbit-prism-load/tests/harness.rs::an_aborted_run_withholds_the_artifact_and_removes_a_stale_one` (the successor instrument's reconnect timing and its refusal to report after an abort, which the driver's `--selftest` smoked); the driver asserted nothing about the coordinator's performance and was never collected or run by CI (its on-demand `--selftest` only checked that the instrument ran: four sessions established and shares accepted against the in-process coordinator, capture and replay, abort refusal), the two harness tests above run against a fake Stratum server, and its socket-level subject (paced and saturating submit load, reconnect storm, ACK latency, rejections by reason, CPU) is measured on demand by the `crates/qbit-prism-load` harness binary, which drives real `qbit-prism-server run` processes over real Stratum sockets and is not a test; retired: coordinator `RLock` hold attribution, heartbeat-watchdog misses and the GIL-wait probe (no coordinator lock, heartbeat watchdog or GIL natively; the harness samples `ORDER_LOCK` and `SETTLEMENT_LOCK` waiters instead) | +| `tests/perf/window_builder_pipe_latency.py` | retired | retired — measured the Python-to-Rust `--serve` pipe transport, which the in-process native window fold removed | +| `tests/perf/window_ledger_decode_latency.py` | retired | retired — measured Python `json_agg` decode time and GIL-induced monitor-thread lateness; no GIL or Python decode path exists natively | +| `tests/perf/window_paging_allocation.py` | partial | partial: covered by `crates/qbit-prism/src/window.rs::full_snapshot_paging_keeps_aggregate_capacity_linear_across_sizes`, `crates/qbit-prism/src/window.rs::full_snapshot_paging_after_the_cutoff_allocates_only_retained_records`, `crates/qbit-prism/src/window.rs::advance_pages_a_multi_page_delta_onto_a_full_page_within_bound`, `crates/qbit-prism/src/window.rs::advance_shares_untouched_pages_and_pages_large_deltas_linearly`; not covered: measured builder RSS and wall time at 210k/400k shares (the `--serve` daemon it drove is retired; no native RSS measurement of the window fold) | +| `tests/perf/window_pipeline_gil_scaling.py` | retired | retired — GIL-held versus GIL-released scaling of the Python materialization pipeline; no GIL exists natively | + +## Needs triage + +Rows with no native test and no open issue that closes them, re-read on 2026-09-17 at a1937054 (#448); each Reason ends with what that re-read found. Each needs an owner to decide between a native test, an issue, or retirement. + +| 2.x.x file | Area | Reason | +| --- | --- | --- | + +## Native storm evidence + +The #270 tests exercise the native lifecycle rather than the retired selector. +The four cleanup tests in +`crates/qbit-prism-server/src/ledger/candidates/storm_fault_tests.rs` use small +populations with neighbouring rows to check fault isolation: + +- `a_failed_or_shifting_chain_observation_leaves_the_drain_recoverable`: failed + or incoherent chain observation retains offer metadata and permits recovery. +- `a_lost_landing_acknowledgement_rolls_back_and_the_next_attempt_lands`: a lost + write acknowledgement rolls back the landing; recovery does not resubmit. +- `a_lost_terminal_commit_acknowledgement_stays_terminal`: a committed terminal + outcome survives its lost acknowledgement and cannot be claimed again. +- `a_lost_renewal_acknowledgement_cancels_only_that_attempt`: failed renewal + cancels the pre-offer attempt; the next legal attempt submits exactly once. + +The provenance tests in +`crates/qbit-prism-server/src/coordinator/storm_evidence_tests.rs` distinguish +row state and ownership from an actual counted node call: + +- `a_pre_offer_retry_records_no_call_and_leaves_the_row_pending` +- `a_held_and_renewed_lease_before_any_call_is_not_offer_evidence` +- `a_pre_offer_supersession_terminalizes_without_any_call` +- `a_replay_adoption_after_restart_records_an_unknown_offer_with_no_call` +- `offer_path_records_a_counted_call_with_a_durable_timestamp_and_reply` +- `a_known_call_keeps_its_call_evidence_through_orphan_terminalization` +- `an_interrupted_reservation_orphans_unknown_without_inventing_a_call` + +The supersession case does not exercise the old capacity-closed selector +condition: there is no equivalent native condition. Replay adoption uses a +second frontend after claim expiry; the separate restart integration test kills +a real process. An interrupted reservation may terminalize as `orphaned` with +an unknown outcome and no call timestamp or reply. These small-population cases +do not claim a 3,120-candidate repetition of each fault, per-height chain-probe +deduplication, post-restart SQL statement-cost measurement, or exhaustive failure +injection at every chain RPC. diff --git a/docs/prism-dispatch-probe-evidence.md b/docs/prism-dispatch-probe-evidence.md new file mode 100644 index 000000000..27b61a1ae --- /dev/null +++ b/docs/prism-dispatch-probe-evidence.md @@ -0,0 +1,73 @@ +# Candidate dispatch probe (#432) + +The dispatch probe now selects at most one due row from an ordered derived +query before allocating a sequence slot. It keeps both advancing +`clock_timestamp()` predicates and the existing unfinished partial index. +The probe grants no candidate ownership; the atomic claim, fresh/oldest lane +policy, parameters, result shape and schema are unchanged. + +The motivating PG16.14 synthetic investigation pinned base +`b3832e9c877703712fbf33a71bec3c297817358a` and executed 456 plans. At 3,120 +backed-off unfinished rows, growing retained history from 50,000 to 100,000 +made the original probe reject 53,120 and 103,120 visible rows, respectively. +The ordered query rejected 3,120 in both fixtures. Scan buffer accesses were +1,601/2,953 for the original and 213/214 for the ordered query. Those measured +accesses were shared hits, not physical disk reads. Live-claimed and +sparse-last layouts sometimes cost the ordered query more buffer accesses. + +The separate `candidate_dispatch_probe` regression target executes the +canonical Rust builder against real PostgreSQL. It records the query and its +SHA-256, server/planner settings, exact counts, relation sizes, and +`EXPLAIN (ANALYZE, BUFFERS, SETTINGS, TIMING OFF, FORMAT JSON)` output: + +- Exactly 24, 100 and 3,120 total unfinished rows, each including all four + unfinished states, alongside 50,000 and 100,000 retained rows. +- Busy, backed-off, live-claimed, expired and sparse-last populations, using + default planner methods/costs and direct plus parameter-free prepared SQL. +- Churn with an old snapshot retaining obsolete tuple/index versions, 100,000 + rows moved into and out of the unfinished set, delete/reinsert scattering, + stale statistics, then ordinary `VACUUM (ANALYZE)`. +- Empty, terminal and positive-infinity polls consume no slots; each positive + probe allocates one slot and leaves candidate records unchanged. +- A clock-substituted canonical-query model checks exact due/expiry + microsecond boundaries, separately from actual advancing-clock execution + in an old transaction. Sequence allocation survives transaction rollback. +- Native empty-outbox `claim_candidate` timeout, cancellation and + connection-loss controls check error propagation and pool recovery. A sent + probe can consume a sequence slot after its caller is cancelled, and a lost + response after allocation leaves an unknown outcome; this control does not + establish absence of replay or gap-free sequence allocation. + +The first qualification on PG16.14 used tree +`d00602bc17c5403a7d1c73f1915c6fe97f7d8571` (published as commit `c510b436`) +and canonical query SHA-256 +`24ffe2248cf3f9ea6100deac143402f9fae6c49783687499b57681328a5040a0`. +It executed 72 plans (36 direct and 36 prepared), plus six preparatory +executions per prepared case. All used the unfinished index without a Sort. +With an old snapshot and stale statistics, +3,120 visible unfinished rows still incurred about 11,800 buffer accesses. +Thus the assertions cover these plan shapes and visible rows; they impose no +latency/page threshold or universal physical-work bound. Shared reads can +be served by the operating-system cache and are not proof of disk I/O. + +Reproduce on a **disposable** PG16 database whose role can create databases: + +```sh +export CARGO_BUILD_JOBS=2 +export PRISM_TEST_DATABASE_URL=postgresql://USER@127.0.0.1:PORT/postgres +export PRISM_TEST_REQUIRE_INTEGRATION=1 +export PRISM_TEST_GATE_MANIFEST=/tmp/dispatch-probe-gate.txt +cargo test --locked -p qbit-prism-server --test candidate_dispatch_probe -- --test-threads=1 --nocapture +``` + +Record `git rev-parse HEAD` with the output. Each test owns and drops its own +fixture database. The normal CI shard discovery includes this target, and +its four gate identities are registered in `test/prism-gated-tests.txt`. +CI runs the four database-isolated tests concurrently; the command above +serializes the local qualification, so their resource conditions differ. +The existing admission, lifecycle, claim fairness/owner-loss and storm/restart +targets provide native runtime qualification separately; storm sizes 100 and +3,120 refer to their existing configured sibling counts, not the exact total +unfinished counts of this probe matrix. The merged storm test behavior and +assertions are unchanged. No cold-cache latency, universal optimizer choice, or exact +concurrent clock trace equivalence is claimed. diff --git a/docs/prism-e2e-load-ci.md b/docs/prism-e2e-load-ci.md new file mode 100644 index 000000000..cf6c3708b --- /dev/null +++ b/docs/prism-e2e-load-ci.md @@ -0,0 +1,597 @@ +# PRISM end-to-end and load CI lanes (#487) + +This page covers each CI lane that runs PRISM end to end or under load. For +each lane it says what the lane proves and what it does not. It also covers +how to dispatch a run, how to read its summary, where the evidence is kept and +how to reproduce a run locally. #487 plans the lanes L0 to L6. This page is +updated as each lane lands (#546). A lane that has not landed has a short +section marked **not yet running**, which names the issue that owns it. + +The checked list of scenarios is `test/e2e-scenarios.toml`. For each scenario +it gives the lane that runs it, the issue that owns it, its pass criteria and +whether it runs today. `scripts/check_e2e_scenarios.py` holds that list to +what the workflows run. The manifest names lanes by trigger (`pr`, `nightly`, +`dispatch`, `weekly`, `L6`) rather than by #487's numbers: + +| #487 lane | Where it runs today | Manifest lane | Status | +|---|---|---|---| +| L0 existing CI | `ci.yml`, every PR and every push to `main`, `1.x.x`, `2.x.x`, `3.x.x` | `pr` | running, required | +| L1 E2E smoke | `ci.yml`'s `prism-native-postgres` shards | `pr` | running, required | +| L2 nightly load | `prism-load-nightly.yml`: `run`, `bridging`, `live-nightly`, `stratum-fuzz`, `evidence` | `nightly`, `dispatch` | running; trend rows and a report-only regression rule with provisional thresholds (#551); repeats not yet (#549) | +| L3 production-window matrix | `prism-load-l3.yml`: `l3-full` on the version bump, the `release-candidate` label, a `v*` tag and dispatch; `l3-reduced` weekly | `L3` (and `dispatch` for the presets) | running on pull requests and tags; dispatch and the weekly run need the file on `main` | +| L4 real-node scenarios | `live-nightly` variants and the Sunday `live-weekly` job | `nightly` (and weekly, below) | running; the 2,000-wallet case **not yet** (#604, #622) | +| L5 soak and chaos | the Saturday soak (#575) is not L5 | `weekly` | **not yet running** (#556) | +| L6 shipped images | `prism-load-nightly.yml`'s `shipped-images` job | `L6` | running | + +A GitHub schedule runs only from the default branch's copy of a workflow, and +only a workflow whose file is on that branch can be dispatched. So each change +to a lane lands on `3.x.x` first and is then mirrored to `main`. The schedules +build and test `3.x.x`. None of the lanes in `prism-load-nightly.yml` or +`prism-load-l3.yml` is a required check. + +## L0: existing CI + +**Runs:** `ci.yml` on every PR, as its merge result, and on pushes to the +long-lived branches (`main`, `1.x.x`, `2.x.x`, `3.x.x`); a push to a feature +branch with no PR runs nothing. The jobs are lint and compile, Python +unit tests in four shards (including `tests/test_check_e2e_scenarios.py`), +`cargo test`, and the four required `prism-native-postgres` shards against a +`postgres:16` service and the pinned regtest `qbitd`. The +`prism-integration-proof` job uses `scripts/check_gate_manifest.py` to show +that every id in `test/prism-gated-tests.txt` ran and did not skip +([prism-integration-test-gate.md](prism-integration-test-gate.md)). + +**Proves:** the unit, component and contract tests, and the `live_regtest` +baseline on a real node: two frontends mining, failover, audit and a reorg; +CTV maturity; CPFP recovery. No gated test can skip silently. + +**Does not prove:** anything about rate, latency or memory at scale. The +runners have 2 vCPU and the builds are debug. + +## L1: end-to-end smoke + +**Runs:** gated tests inside the same required shards (#487 decision 2): + +- `qbit-prism-load::load_smoke` runs the `pr-smoke` preset: a debug frontend + with 100 sessions over 20 addresses, 3 tips and 30 s of churn. +- `qbit-prism-load::real_node` runs `real-node-smoke` against a ramped real + regtest node. +- `qbit-prism-load::ctv_settlement` runs the harness with `--ctv-settlement` + (#548): 400 addresses overflow the direct-output cap, so the landed block + pays 12 directly and the rest through one CTV fanout chunk, counted in the + side report's `settlement` block. +- `live_regtest::weighted_recipients_tests::real_weighted_recipients_pay_exact_pplns_outputs_through_fanout_spend` + is #524's wallets, mined through to spent outputs. + +**Proves:** the harness still drives the server end to end on every change. +Every session gets work on every tip. The ledger reconciles exactly with what +the clients were acknowledged, against both the fake node and a real node. The +weighted payouts are paid and spent on chain. + +**Does not prove:** throughput or latency budgets. The build is debug, the +runner has 2 vCPU, and the gates are about correctness, not speed. + +## L2: nightly load + +**Runs:** `prism-load-nightly.yml` at 02:17 UTC every night. It also runs on +dispatch, and on a PR when a maintainer applies the `run-load` label (#549); +the label runs against the PR's merge result, on `pull_request`. It has these +parts: + +- `run`: every preset whose `schedule` is `nightly` + (`scripts/prism_load_matrix.py nightly`). Each preset runs in its own job on + the runner it names (8 vCPU today), from release binaries that one `build` + job made at a commit pinned by the `plan` job. The presets are the D1 20k + fixture with and without 500 addresses, #473's 400k one-frontend cell, + #275's tip delivery, the `mainnet-shape-130/650-addresses` shapes, rental + churn, and the bridging pair `short-plan-{fake,real}-node`. +- `bridging`: the bridging pair again, both on one runner, with the + real-minus-fake row ([below](#bridging-lane-552)). +- `live-nightly`: the opt-in `#[ignore]` real-node variants listed in + `test/prism-nightly-gated-tests.txt`. They are the qbitd `-reindex` crash, + the 130-payee weighted recipients, the dense-cadence soak, a full PostgreSQL + disk, wall-clock jumps, the soak smoke and #553's short set of chain events + under 2,000 Stratum sessions (L4, below). The job uses the same gate + manifest check to show each one ran. +- `stratum-fuzz`: the cargo-fuzz Stratum targets for 20 minutes, gated on no + crash ([prism-stratum-fuzzing.md](prism-stratum-fuzzing.md)). +- `guard` and `report`: a night on which `3.x.x` has not moved is cancelled, + never passed. A failed scheduled run opens or comments on the one + `prism-load-nightly-failure` issue. +- `evidence` (scheduled runs only) and `evidence-preview` (a `run-load` PR or + a dispatch, read-only): the trend rows, the regression rule and evidence + promotion ([below](#trend-regression-rule-and-evidence-promotion-551)). + +**Proves:** each preset completes, reconciles exactly, loses no acknowledged +share and has no durability finding. It has zero shortfall in each gated +phase, and meets any tip-delivery budget or #473 D1 rule the preset sets. All +of this is on one named runner class, on one commit. + +**Does not prove:** + +- **A regression, as a gate.** The regression rule (#551) compares each + number with its trailing baseline and files one tracking issue, but it is + report-only and its thresholds are provisional until #542's measured + variance is checked in. Each preset still runs once a night; the repeats + are #549. +- **A D1 verdict** (#487 decision 5). The rates hold only on the runner class + they ran on, and CI runners are not the rehearsal host (#477). + +### Bridging lane (#552) + +**Runs:** the `bridging` job, on the nightly schedule, on the `run-load` +label, and on dispatch with `bridging: true`. It runs `short-plan-fake-node` +and `short-plan-real-node` one after the other on one 8 vCPU runner, through +`prism-load-run.sh`. The order alternates with the run number, and the row +records it. The two presets differ only in `--node`: `--plan short`, 2 +frontends, 100 sessions, a 20k window at 50 shares/s and 3 external tips. +The matrix still runs each half on its own VM as well. +`scripts/prism_load_bridge.py` then writes `bridge-row.json` (schema +`qbit.prism.load-bridge.v1`) and a table in the job summary. Each row is real +minus fake: + +- `client_ack_latency` p50 and p99, per phase; +- time to usable work: the slowest session's time on each external tip, p50 + and max over the tips; +- peak RSS (`VmHWM`) of each frontend and of the frontends summed; +- the pool node's `submitblock` and `getblocktemplate` latency through the + harness's relay, and the pool qbitd's peak RSS. These exist only on the + real node. The fake side and the difference read `unknown`, never 0. + +A figure either side could not measure is `unknown`, and a footnote gives the +reason. If either side failed its harness or its gate, the row is marked +**Not trusted** and says why, and the job fails once the row and the artifact +`prism-load-bridging` are written. + +**Proves:** what the real node costs on this runner class, at this commit, +for this plan, without the VM-to-VM spread that separate jobs include. + +**Does not prove:** a D1 verdict or a D1 rate (#487 decisions 1 and 5). +The plan runs at 50 shares/s. Each run is one sample, so a single night's +difference carries the run-to-run noise, which #542 measures. + +### Trend, regression rule and evidence promotion (#551) + +**Runs:** the `evidence` job, on every scheduled run of +`prism-load-nightly.yml` (the nightly, the Sunday weekly run and the Saturday +soak). It is the only job in the workflow with `contents: write` (with +`issues: write` and `actions: read`). A `run-load` PR and a dispatch get +`evidence-preview` instead, with a read-only token. It applies the same rule +to its own rows against the recorded trend and writes only to the job +summary. Both jobs call `.github/actions/prism-load-evidence`, which the L3 +tag-push job will call too once #550's tag hook lands (after #608). + +**Trend rows.** Each preset job builds its row on every event with +`scripts/prism_load_trend.py row` and uploads it as the artifact +`prism-load-trend-` (7 days). Only `evidence` records rows. A row is +one JSON object, schema `qbit.prism.load-trend-row.v1`: + +| Field | What | +|---|---| +| `kind` | `preset` (a preset run), `missing` (a planned preset whose job, at its latest attempt, left no row), `job` (a non-preset job's outcome: `bridging` as L2, `live-nightly` and `live-weekly` as L4, `stratum-fuzz` as `fuzz`, `shipped-images` as L6), or `promotion` (an asset promoted after its row was written) | +| `run_id`, `run_attempt`, `run_url`, `event`, `ref`, `commit`, `recorded_at` | the run, the commit its job checked out, and when the row was built (UTC) | +| `lane`, `preset`, `preset_sha256` | L2 for the nightly presets, `weekly` for the Saturday soak (#575, not L5); the preset file's hash, so a changed preset starts a new series | +| `runner_class`, `host` | the runner label and `host.json`'s CPU, memory, kernel, filesystem, write cache and FUA | +| `fsync` | `pg_test_fsync`'s fdatasync ops/s and µs/op, and its band | +| `outcome` | the harness and gate exit codes, `pass`, `fail` or `no verdict`, and the build provenance | +| `headline` | per phase: achieved rate, client ACK p50/p99 (ms, client clock), server ACK mean and bucketed p99 (ms, receipt to response write, including the share append), ORDER_LOCK max and mean waiters, peak frontend RSS (MiB), shortfall, refused valid shares, missing shares; per run: tip-to-last-notify p99 (ms) and durability findings | +| `artifact`, `promotion`, `asset_url` | the run's artifact (name, ID, URL), and its release asset once promoted | + +A number the run did not measure is `null`, never 0. A row of another schema +is skipped by every reader, not guessed at. + +**The branch.** `ci-evidence` is an orphan branch holding +`trend//YYYY-MM.jsonl` (the month of `recorded_at`) and a README. It is +append-only: rows are added, never edited, and a push is never forced. The +first write creates it. `scripts/prism_load_trend.py append` refuses any row +whose event is not `schedule`, or a `push` of a `refs/tags/v*` tag. The +action refuses to write from any other event too. Each attempt fetches the +tip and skips rows already there. The identity is kind, run, attempt and +preset; for a job row it is job, attempt, outcome and commit (each job exports the attempt it ran in). It then commits on top +and pushes, and retries a rejected or unconfirmed push from a fresh fetch, up +to 5 attempts in 300 s. So the nightly and the soak writing at once lose no +row, and a re-run never duplicates one. + +**Promotion.** Bundles that must outlive the 90-day artifacts become release +assets: + +- the Saturday soak's (`weekly`) bundle, and any L5 run's, always; +- an L2 run's bundle when it is flagged, meaning the rule found a regressed + number or the gate failed; +- a tag's full-suite bundles (L3), all of them, once the tag hook lands. + +Monthly bundles go to the rolling `ci-evidence/YYYY-MM` pre-release, which +is created when missing and never marked latest. A tag's bundles go to that +tag's release. An asset is the run's artifact, fetched by its exact ID and +tarred as `--run-attempt.tar.gz`, so `--clobber` only +ever replaces the same bundle on a retry. The row records the asset URL read +back from the release. A promotion that fails is recorded as `failed` in the +row, and the job fails. A re-run that later promotes it adds a `promotion` +row. To keep a run cited in an issue or a doc, a maintainer runs +`scripts/prism_load_trend.py cite --artifact-id N --lane L2 --preset NAME +--repository Qbit-Org/qbit-mining-bootstrap [--append]`. + +**The regression rule** (`scripts/prism_load_regress.py`) is **report-only**. +Its thresholds live in `test/prism-load-regression.toml`, which is marked +`provisional`. + +- **Series.** A series is one lane (L2 today), preset and preset sha256, + runner class and fsync band (`fast` under 1,000 µs/op, `medium` under + 4,000, `slow` above). Only runs whose harness exited 0 count, each at its + latest attempt. +- **Limit.** A number regresses when it is worse than the median of the last + 14 such runs (at least 5) by more than + `max(k × spread, min_relative_change × |median|, min_absolute_change)`, + with `k = 3` and per-number floors. +- **Spread.** The spread is #542's measured coefficient of variation over + every run (`all.cv` in `scripts/prism_load_probe.py variance`'s document, + schema `qbit.prism.runner-probe-variance.v1`). It is read from the group + for the run's own #542 fsync band, or else the class's all-band group, when + that document is checked in as `test/prism-load-variance.json`. Otherwise + it is the baseline's scaled MAD, which is provisional. +- **Where #542 does not apply.** #542 measures the rate, the client ACK + p50/p99 and the tip-to-last-notify p99, so the server ACK, lock waiters + and RSS stay provisional. A variance document of another schema or version + is refused by name. When #542's VM-to-VM spread is over 2× its run-to-run + spread, the summary points at #511's same-VM A/B instead. +- **Commit range.** A regressed number is reported with the range from the + last good run's commit to the first bad run's. Flagged runs stay out of + later baselines, so a lasting regression keeps its range. After 14 flagged + runs in a row, the new level is taken as accepted. +- **Unknowns.** An unmeasured number, a run with no fsync cost, a short + series, a harness exit other than 0 and a planned preset with no row each + read **unknown** or **not evaluated**, with the reason, never as within. +- **Where it reports.** The verdict goes to the job summary and the artifact + `prism-load-regression-verdict` (90 days). On a regression, `evidence` + opens or updates the one `prism-load-regression` issue. Its body is the + latest flagged run's table, with the commit range and both runs' numbers + side by side, and each flagged run adds a comment. It never pages and + never blocks a merge. + +**Proves:** every scheduled run since #551 has trend rows, and PR and +dispatch runs record nothing. It also shows whether a number moved beyond +the (provisional) noise allowance of its own series, and between which +commits. + +**Does not prove:** a D1 verdict (decision 5), or a regression gate. The rule +never fails a job. Its thresholds stay provisional until #542's measured +document is checked in and `provisional` is set to false. + +### Runner classes and noise floor (#541, #542) + +`.github/workflows/prism-load-runner-probe.yml` (#541) is dispatch-only. It +measures the Blacksmith classes: `pg_test_fsync` where the clusters live, the +build, seeding and run times, peak host memory, shortfall, and the +tip-to-last-notify p99. It runs `short-plan-20k-window-1fe` and +`throughput-20k-window-1fe` on each class, and its collate job writes one +table (artifact `prism-probe-table`). + +**Not yet running; owner #542:** at least 10 repeats per class for the +run-to-run and VM-to-VM spread, and the runner class chosen for each lane. + +## L3: production-window matrix + +**Runs:** `prism-load-l3.yml` (#550). The suites are checked in at +`crates/qbit-prism-load/presets/suites.toml`, separate from each preset's +`schedule`, so a preset can be nightly and in L3 at once: + +| Suite | Presets | Repeats | Runner | When | +|---|---|---|---|---| +| `l3-full` | #447's matrix as #473 ran it: 200k fe1, 400k fe1/2/4 async, 400k fe2 sync, `throughput-400k-window-2fe-async-3-blocks`, `dense-cadence-400k-window-{1,2}fe-async`, 500k fe1/2/4 | 3 (33 jobs) | 32 vCPU | the version-bump PR, `release-candidate`, a `v*` tag, dispatch | +| `l3-reduced` | 400k at 1, 2 and 4 frontends, async; `session-point-{5000,10000}-sessions-400k-2fe-async` (#555); `mainnet-shape-{130,650,2600}-addresses` and `rental-churn-bursts-and-storms` (#521) | 1 (9 jobs) | 32 vCPU | Monday 06:13 UTC on `3.x.x`, dispatch | + +The runner is the 32 vCPU class until #542 picks one per lane. The triggers: + +- **The version bump:** a PR into `3.x.x` or `main` that changes `VERSION`, + from this repository, on every push. The `changes` job checks the PR's file + list. A trigger-level `paths` filter would also swallow the label event. +- **`release-candidate`:** a maintainer applying the label to a PR into + `3.x.x` or `main`, including a fork's. As with `run-load`, only the + `labeled` event starts it. +- **A `v*` tag**, on either line. The tag first looks for the newest finished + same-repository PR run of `l3-full` whose verdict is complete for the tagged + commit's tree (`git rev-parse HEAD^{tree}`, recorded by the plan job). If it + finds one, it promotes it: it carries that run's tables and verdict, links + the run, and fails if that run failed. Otherwise it runs the full suite + (`scripts/prism_l3_promote.py`). A lookup that fails reruns the suite. A + tag reads the workflow file at the tagged commit, so a tag cut before this + file reached its line starts nothing. +- **Dispatch:** `suite` and `ref`, or `tag_dry_run`, which acts as a tag push + on `ref` without pushing a tag. Like every dispatch, it needs the file on + `main`. +- **Weekly:** `l3-reduced` on `3.x.x`. Like every schedule it runs from + `main`'s copy of the file. + +The workflow follows L2's pattern. The plan job pins one commit. One job +builds the release binaries once. Each preset repeat runs in its own job, +through `prism-load-run.sh` and held to its preset's gates, with at most eight +jobs at once. Then the `collate` job runs `qbit-prism-load-compare --collate` +(#511's summarizer) and writes #473's document tables to the job summary. +They have one table per D1 phase, with a row per preset: its sessions, +window, frontends, replication and plan, then #511's cells (n, target, +achieved, shortfall, refused valid shares, unanswered submits, ACK p50 and +p99 against the limit, `ORDER_LOCK` waiters, the D1 verdict). A table of each +preset's own gates follows. The job also writes a verdict: + +- `complete`: every planned run reached its gate; +- `passed`: every planned run passed it. + +A planned run whose job left no artifact, whose report names another commit, +or which ran a preset file other than the checked-in one is listed and fails +the suite. It is never dropped. A failed build lists every run as never run +and writes no verdict. Concurrency is per PR and label (a new push cancels +the older push's run), per tag, and one group for the weekly run. The +workflow is never a required check. Cost: `l3-full` is about 660 runner-minutes +(about $42) a release candidate, and `l3-reduced` about 200 runner-minutes +(about $13) a week. + +**Proves:** on one commit and one runner class, #447's production-window +cells, including the found-block path at 400k and dense cadence, complete, +reconcile exactly, lose no acknowledged share and hold their gates, with +three repeats for each cell. Its correctness rows count toward #291. A tag +has L3 evidence for exactly its tree. + +**Does not prove:** + +- **A D1 verdict** (#487 decision 5). The D1 column applies #473's rule on + the CI runner class, not on the rehearsal host. +- **A regression.** Nothing compares against a baseline. The A/B release + benchmark on the reference host (#511, [prism-release-benchmark.md](prism-release-benchmark.md)) + and #551's trend do that. +- **#473's flush and build controls** and the 20k dense attempt (exit 6): + they are not presets. + +**Capacity-envelope points (#555).** `l3-reduced` also runs the 5,000 and +10,000 session points. They are `throughput-400k-window-2fe-async` (400k +window, 2 async frontends, the D1 plan) at 5,000 and 10,000 sessions, with +admission sessions per frontend + 16 as the 2,000-session cell ran it. The +10,000 point's D1 `burst` phase, 2,000 shares/s for 60 s, is the burst +variant. #521's mainnet shapes and rental churn run beside them (decision +11). These are capacity-envelope evidence, never D1 verdicts (decision 5): +their rows carry no D1 verdict, and they gate reconciliation and a +`steady_state` that places every offer. The 10,000-recipient point has no +preset yet; the harness has had CTV fanout settlement since #548 +(`--ctv-settlement`, #644), so it can now be written. The 50,000-recipient +follow-up runs only if the 10,000-recipient point costs more than 25% more +per recipient than the same preset at 1,000 recipients (one fanout chunk), +on the same runner class in the same week. The cost is measured as window +rebuild (the tip-to-last-notify p99) or payout revision (a landing to every +session holding the new revision's work, median over landings). The trigger +is evaluated once that point runs. + +## L4: real-node scenarios + +**Runs today:** + +- the nightly `live-nightly` variants above; +- the Sunday 03:43 UTC `live-weekly` job, which runs + `test/prism-weekly-gated-tests.txt` in release mode: #545's 2.x.x migration + lifecycle, #575's measured cutover of a mainnet-shaped 2.x.x ledger and + #553's long set of chain events under 2,000 sessions. + +#553 runs #521's fixtures unchanged under a load of 2,000 share-only Stratum +sessions (`tests/support/live_session_load.rs`), which adds its own checks: +every acknowledged share durable, only refusals a correct miner can earn, +and every session given work on each lasting tip within 15 s, with #481's +time to usable work reported per frontend. Nightly, at 100 shares/s: +scenarios 1 and 2, scenario 4's crash case, two frontends on two nodes that +disagree about the tip, a node that loses its peer, 5 minutes of external +tips, and #598's guard (2,000 sessions on one frontend at a 1 s reanchor, in +debug). Weekly, at 400 shares/s: scenario 4's other four cases, scenario 5's +20-minute soak and 45 minutes of external tips. + +**Proves:** each listed scenario's own assertions on a real regtest node and +PostgreSQL 16, and that every listed id executed. + +**Does not prove:** #521 scenario 7 at 2,000 wallets, which fails on #621 in +debug and #622 in release and runs only by hand until both are fixed. + +**Faults under load (#554)** run in the load harness's `faults` phase +(`crates/qbit-prism-load/README.md`, "Faults under load"), one fault at a +time while the sessions keep mining, each held to its criteria in the +manifest: + +- per PR, `faults-pr-smoke` (`qbit-prism-load::faults`): a SIGTERM drain with + an offer in flight and a `SETTLEMENT_LOCK` holder, on the fake node; +- nightly, `faults-short-real-node`: every fault but the full WAL volume, on + the real node with 500 sessions: the slow database, pool exhaustion, the + lock holder, the frontend SIGKILL, the drain, the rolling restart, the + reconnect storm, the restart over a candidate backlog, the primary lost + with its async standby promoted, the fenced switch and a found block across + a failover; +- weekly, on the Saturday selection beside the soak, `faults-long-real-node`: + every fault, the full WAL volume included, with 2,000 sessions on a 16 vCPU + runner; +- on dispatch, `faults-failover-fake-node`: the five database and landing + faults on the fake node. + +They prove D3's loss policy under load (only acknowledged shares in the +replication gap are lost, each listed), that the frontends serve the promoted +primary within 30 s without a restart, that a block mid-landing lands once +across a failover, and that a full WAL volume acknowledges nothing it cannot +keep. They do not prove a real network partition: the relays and endpoints +stand in for one. + +## L5: soak and chaos + +**Not yet running; owner #556:** the two-hour soak with randomised faults, +judged against #291's criteria. + +L5 is not the Saturday 05:41 UTC `soak-weekly` job (#575, manifest lane +`weekly`). That job runs one server lifetime of about 5.5 h with no faults. It +is gated on RSS and descriptor trends, connection drift, bounded WAL, zero +lost shares and exact reconciliation. [prism-soak.md](prism-soak.md) has its +gates. + +## L6: shipped images + +**Runs:** the `shipped-images` job (#544) on the Sunday schedule, on dispatch +(`images: true`), and on a PR into `3.x.x` or a `3.x.x` push that changes a +path in `WATCHED_PATHS` in `scripts/prism_shipped_image_lane.py`. The job +builds the images with `docker compose build` and brings up `compose.yaml` + +`compose.prism-ha.yaml` under the `prism` profile. cpuminer-opt and +`qbit-prism-miner` then mine against it. + +**Proves:** the checks in the driver's `CHECKS`: + +- both frontends are ready; +- #281 criterion 1 holds across the frontends; +- each client finds a block and its audit bundle verifies; +- the ledger reconciles exactly; +- the public API lists every found block. + +**Does not prove:** anything about capacity. The load is a few miners. + +**Reading a failure:** an error outside the checks leaves the checks after it +`not reached`. The job summary and `l6-report.json` (`failed_phase`) name the +phase it stopped in: `node`, `stack`, `resume`, `load`, `quiesce`, `ledger` or +`blocks`. A qbitd RPC that times out names its method. In the `node` phase, +qbitd's `createwallet` takes about 25-30 s on the runner, which is mostly the +create-time P2MR keys. If the call outlasts its 30 s client timeout, the lane +waits for `listwallets` to show the wallet instead of failing (#637). +`createwallet` plus that wait is bounded at 300 s. The report's `wallet` +records which way the wallet arrived and how long it took. + +## Dispatching a run + +Dispatch runs the default branch's copy of the workflow. `ref` picks the +commit that is built and run: + +```sh +# The nightly set against a branch (the live variants are on by default). +gh workflow run prism-load-nightly.yml -f preset=nightly -f ref=my-branch + +# Named presets only, without the live variants. +gh workflow run prism-load-nightly.yml \ + -f preset=short-plan-fake-node,short-plan-real-node -f ref=3.x.x -f live=false +``` + +The inputs are: + +- `preset`: comma-separated names, `nightly`, `all` (nightly plus manual) or + `weekly` (the soak); +- `ref`; +- `tip_last_notify_p99_budget_ms` and `max_shortfall`: empty keeps each + preset's own budget; +- `live`, `weekly`, `fuzz`, `images` and `bridging`: add those jobs. + +A one-off configuration is a checked-in preset on a branch, dispatched with +that branch as `ref`. The harness refuses a flag the preset pins if it is given +again, so a result always matches a reviewed file. + +To run the nightly set against a PR, apply the `run-load` label. A later push +needs the label applied again. The runner probe is dispatched the same way +(`gh workflow run prism-load-runner-probe.yml -f classes=8,16`). + +The production-window matrix is dispatched from its own workflow, once the +file is on `main`: + +```sh +# The full suite on a ref. +gh workflow run prism-load-l3.yml -f suite=l3-full -f ref=my-branch + +# What a v* tag on this ref would do: promote a matching PR run, or run l3-full. +gh workflow run prism-load-l3.yml -f ref=v3.0.0-rc.1 -f tag_dry_run=true +``` + +To run `l3-full` on a PR, apply `release-candidate`. A PR that changes +`VERSION` runs it on every push. + +## Reading a job summary and `gate.md` + +A preset job's summary starts with `### on ` and the commit. +Next comes the gate's verdict, `### : PASS` or `FAIL`, followed by a +table: + +| Column | Meaning | +|---|---| +| Check | Every run gets the exit code, lost acknowledged shares, durability findings and a shortfall row per gated phase. A row is added for each gate the preset sets (see the harness README's gate table) | +| Observed | what the side report says, in the unit shown | +| Budget | the preset's budget, or the dispatch override | +| Result | `pass`; `**FAIL**`; or `info` for a row reported but not gated (ungated phases, the D1 burst) | + +A gated figure the report could not measure fails its check and gives the +reason. An ungated one reads `unmeasured: <reason>`. Neither is ever shown as +a pass or as zero. The run's exit codes are: + +- `harness-exit-code`: 0 means the run completed and reconciled. 2 to 8 each + have one meaning in the + [exit-code table](../crates/qbit-prism-load/README.md#exit-codes): 4 is a + durability loss, 6 an abort, 8 a contradicted premise. +- `gate-exit-code`: 0 means pass and 1 means fail. Anything else (2, or no + file) means the gate could not reach a verdict. The nightly does not count + that commit as tested and runs it again the next night. + +D1 presets add #473's verdict table. The folded sections hold `host.json` +(runner label, CPU, memory, kernel, and the mount options and write cache of +the cluster's disk) and `pg_test_fsync`. Compare two runs only when both of +these match. + +For more detail, the side report `load-harness-report.json` has every phase's +`client_ack_latency`, `shortfall`, `processes` (per-frontend CPU and RSS) and +`reconciliation`. It also has `time_to_usable_work.tips[]` and, for a +real-node run, the `node` block. The harness README's +[Honest values](../crates/qbit-prism-load/README.md#honest-values) section +defines each field. + +## Where evidence lives + +| Where | What | Kept | Status | +|---|---|---|---| +| Actions artifact `prism-load-<preset>` | the run directory: `load-harness-report.json`, `gate.md`, both exit codes, `host.json`, `pg_test_fsync.txt`, `logs/`, and the soak's samples and report for a soak | 90 days | running | +| `prism-load-bridging` | both runs' directories (`fake/`, `real/`) and `bridge-row.json` | 90 days | running | +| `prism-live-nightly`, `prism-live-weekly`, `prism-stratum-fuzz`, `prism-shipped-images` | gate manifests and test logs; fuzz logs and crashing inputs; `l6-report.json` and the Compose logs | 30 days | running | +| `prism-load-tested-commit` | the commit the last nightly carried to a verdict (the guard reads it) | 90 days | running | +| `prism-l3-run-<preset>-r<repeat>` | one L3 run's directory, as `prism-load-<preset>` | 90 days | running | +| `prism-l3-plan`, `prism-l3-collated` | the plan (commit, tree, matrix) and a tag's promotion decision; the collated tables, `verdict.json` and, on a promoted tag, `promotion.json` | 90 days | running | +| `prism-l3-verdict-<tree>` | the verdict a tag's promotion looks up by tree | 90 days | running | +| `prism-load-trend-<preset>` | the preset run's trend row (every event) | 7 days | running | +| `prism-load-regression-verdict` | the regression rule's verdict JSON and summary | 90 days | running | +| `ci-evidence` branch | `trend/<lane>/YYYY-MM.jsonl`: one JSON line per preset run, job outcome, missing preset or later promotion, from scheduled runs (and `v*` tag runs once L3's tag hook lands) | permanent | running (#551) | +| Release assets | the rolling `ci-evidence/YYYY-MM` pre-release: soaks, flagged L2 runs and cited runs; a tag's release: its full-suite jobs | permanent | running for the nightly workflow (#551); the tag's release follows L3's tag hook, and the bundle for a release candidate is #557 | + +A run cited in [prism-throughput-measurements.md](prism-throughput-measurements.md) +or in an issue can be promoted with `scripts/prism_load_trend.py cite` +([above](#trend-regression-rule-and-evidence-promotion-551)); link the asset. + +## Reproducing a run locally + +`.github/scripts/prism-load-run.sh` is exactly what a `run` job executes: + +```sh +cargo build --locked --release -p qbit-prism-server -p qbit-prism-load +# Only a real-node preset (--node qbitd) needs this. It exports nothing when +# run like this, so set QBITD_BIN yourself. +bash .github/scripts/install-prism-qbit.sh "$HOME/prism-qbit" +export QBITD_BIN="$HOME/prism-qbit/qbit-1.0.0/bin/qbitd" # the path it prints +PG_BIN_DIR=/usr/lib/postgresql/16/bin RUNNER_LABEL=local \ + .github/scripts/prism-load-run.sh short-plan-fake-node load-out +``` + +The script needs PostgreSQL 16 server binaries (`PG_BIN_DIR`). The harness +builds and removes its own cluster under `PRISM_LOAD_TMPDIR`, which defaults +to `/tmp/pload`. Extra arguments go to the harness, and only flags the preset +does not pin are accepted, for example `--allow-dirty-tree` for a modified +checkout. The output directory then has the same files as the CI artifact. +Compare against CI only on a host of the same class; `host.json` says what +yours is. + +To reproduce the other jobs: + +- **A gated or live test.** An id in the gated lists is + `<package>::<binary>::<test path>`. Split it the way the workflow does, and + pass only the test path to the filter: + `PRISM_TEST_DATABASE_URL=… PRISM_TEST_PG_BIN_DIR=… PRISM_TEST_REQUIRE_INTEGRATION=1 cargo test -p <package> --test <binary> -- --ignored --exact <test path>`. + Leave out `--ignored` for an id from `test/prism-gated-tests.txt`. Check + that the output says `1 passed`: a filter that matches nothing also exits + 0. +- **L6.** Run + `python3 scripts/prism_shipped_image_lane.py prepare|build|run --work DIR [--out OUT]`, + in that order. The script's docstring lists what it needs. +- **The soak.** See [prism-soak.md](prism-soak.md#running-one). + +## Keeping this page current + +When a lane lands, the PR that lands it replaces the lane's **not yet +running** section with what it runs, what it proves and what it does not. The +same PR updates the table at the top and the manifest's `[lanes]` entry. diff --git a/docs/prism-ha-functional-qualification.md b/docs/prism-ha-functional-qualification.md new file mode 100644 index 000000000..58d72b846 --- /dev/null +++ b/docs/prism-ha-functional-qualification.md @@ -0,0 +1,178 @@ +# Bounded HA functional preparation for #281 + +This procedure reuses the existing HA overlay, readiness harness, managed +PostgreSQL driver and Coordinator/Stratum tests. It prepares the three remaining +[#281](https://github.com/Qbit-Org/qbit-mining-bootstrap/issues/281) live gates. +**A local pass does not close any of those gates**, establish D1 capacity, or +replace the owner-coordinated [#291](https://github.com/Qbit-Org/qbit-mining-bootstrap/issues/291) +rehearsal. Do not run a competing full rehearsal or change acceptance thresholds. + +## Inventory and exact remaining acceptance + +| Existing capability | What it actually provides | Remaining live evidence | +| --- | --- | --- | +| [#304](https://github.com/Qbit-Org/qbit-mining-bootstrap/pull/304), `compose.prism-ha.yaml` | Two frontend services, distinct IDs/ports, per-frontend RPC wiring, one writer DSN; four-stack CI renders and self-check instance census. | Start the **actual overlay** with two healthy frontends; retain work issued by A, reconnect through the operator endpoint to B and accept that original share exactly once. A new job after reconnect alone is insufficient. | +| [#331](https://github.com/Qbit-Org/qbit-mining-bootstrap/pull/331), `prism_ha_readiness_probe.py` | Deterministic health classification and rise/fall state machine, with no network I/O or routing. | Observe the **operator TCP endpoint** preserve ordinary tip/payout rebuilds, eject sustained unavailability within its stated bound, reject new admission when all backends are down, and require recovery successes. | +| [#449](https://github.com/Qbit-Org/qbit-mining-bootstrap/pull/449), HA reference | Approved D3: one primary and **one dedicated asynchronous failover standby**; public reads use a separate replica. Documents fencing, stable writer authority, reconciliation and primary-side alerts. | Run #291's overlay failover drill under mining load: positively fence all old writer connections, promote the eligible standby, move the stable endpoint, reconcile ACK/accounting and record replication gap/loss, recovery time and alert delivery. | +| `compact_runtime_e2e` | Real Coordinator methods, Stratum sockets, small PG-backed windows, controlled local RPC node, original issued-work authority/expiry and unknown-commit coverage. | Its passing socket resume test is functional evidence; it neither launches Compose nor uses an operator LB or a real qbitd. | +| `qbit-prism-load::cluster::ManagedPostgres` | Disposable PG primary and named async/sync standby, explicit durability and physical slot. | The existing load driver has no operator frontend LB or managed promotion phase. This preparation does not replace it with a new mining simulator. | +| `postgres_failover.rs` | Existing synchronous ledger-loss/reconnect fixture. | Its zero-loss assertion belongs to that synchronous fixture and cannot be applied to approved asynchronous D3. | + +The authority for the deployment settings and unchanged live acceptance is the +[HA reference](prism-ha-reference-architecture.md#281-acceptance-evidence). +The bundled public-read replica is **not** the dedicated failover candidate. + +## Resource contract + +Agree on a slot with concurrent performance work before any build, container or +PG run. Use at most two build jobs. The functional command below creates two +loopback-only PostgreSQL 16 processes, 32 connections per server, seven selected +small existing tests, two ledger pools, one local writer proxy and eleven initial +ledger shares. The existing tests use controlled RPC fixtures on loopback. +No containers, external database, real node, public-read workload or production +access is involved. There is no existing-database URL argument. + +The wrapper creates a private, short `/tmp/b281-*` parent and passes it as the +child's `TMPDIR`. Every managed cluster lives below that exact new parent. +It never stops a process/container by name, scans for another user's databases, +or runs global Docker cleanup. Its resource census records the owned parent, +child PID, child exit status and exact PG paths. PG shutdown must be verified +with `pg_ctl status` and disappearance of `postmaster.pid` before deleting data. +Unknown cleanup retains the owned directory and fails the command. SIGTERM and +Ctrl-C use the wrapper's owned-resource cleanup path. SIGKILL or machine loss +requires inspecting that recorded parent; absence of +an artifact is not proof of cleanup. + +## Run the reusable preparation + +The first two commands are lightweight and start no services. Compose output is +captured privately, validated using explicit fixture values and reduced to an +allowlist; never print resolved Compose configuration containing real secrets. +The render exercises nondefault IDs, separate RPC URLs, health/high-difficulty listener ports, +loopback host bindings and a shared writer DSN through the real Compose merge. +It does not claim those values reached running frontend containers. + +```sh +python3 -m unittest tests.test_prism_ha_qualification tests.test_prism_ha_readiness_probe +python3 scripts/prism_ha_qualification.py render +``` + +After the resource slot is available, build the example and the **existing** +test executables. Keep Cargo's JSON receipts so the selected binaries are exact; +do not select a possibly stale executable with a wildcard. These are debug +functional builds, not performance measurements. + +```sh +B281_BUILD_DIR=$(mktemp -d /tmp/b281-build-XXXXXXXX) +CARGO_BUILD_JOBS=2 cargo build --locked -p qbit-prism-load --example ha_qualification +CARGO_BUILD_JOBS=2 cargo test --locked -p qbit-prism-server \ + --test compact_runtime_e2e --no-run --message-format=json > "$B281_BUILD_DIR/runtime-build.jsonl" +CARGO_BUILD_JOBS=2 cargo test --locked -p qbit-prism-server \ + --lib --no-run --message-format=json > "$B281_BUILD_DIR/ack-build.jsonl" +B281_RUNTIME_TESTS=$(python3 -c 'import json,sys; print(next(r["executable"] for r in map(json.loads,open(sys.argv[1])) if r.get("reason")=="compiler-artifact" and r.get("executable") and r["target"]["name"]=="compact_runtime_e2e"))' "$B281_BUILD_DIR/runtime-build.jsonl") +B281_ACK_TESTS=$(python3 -c 'import json,sys; print(next(r["executable"] for r in map(json.loads,open(sys.argv[1])) if r.get("reason")=="compiler-artifact" and r.get("executable") and r["target"]["name"]=="qbit_prism_server"))' "$B281_BUILD_DIR/ack-build.jsonl") +python3 scripts/prism_ha_qualification.py run \ + --pg-bin-dir "$(pg_config --bindir)" \ + --example-bin target/debug/examples/ha_qualification \ + --runtime-tests "$B281_RUNTIME_TESTS" --ack-tests "$B281_ACK_TESTS" \ + --out "$B281_BUILD_DIR/evidence" +``` + +Both test executable arguments are required by the wrapper and the example; +omitting either fails argument parsing before a PostgreSQL process can start. +Select a PG16 `--pg-bin-dir` explicitly if `pg_config` names another version. +If `CARGO_TARGET_DIR` is customized, select the example from that build's target +directory too. Use the wrapper so startup failures also receive an owned-resource +census and cleanup; invoking the example alone omits that outer cleanup guard. + +`functional/ha-functional.json`, individual test logs under `functional/`, +`functional-process.log` and `resource-census.json` are local artifacts. Preserve them together with the +checkout SHA, build receipts and binary hashes when handing off evidence. +The wrapper has a 480-second execution ceiling plus a bounded termination/cleanup +period; the example bounds its exercise to 180 seconds. Neither extends a share +or issued-job deadline inside the runtime to force success. + +## What the functional phase proves + +The four selected `compact_runtime_e2e` tests exercise original-work socket +resume with one credited share, stale-publication fencing, expiry during a +blocked read, and reconciliation of an unknown issued-job commit. Three existing +`coordinator::commit_reconcile_tests` exercise a queued append refused at its +original share deadline, a confirmed in-flight commit accepted within grace, +and an unknown answer at deadline plus grace whose commit lands later. The +selected test names and logs are recorded. Missing inputs or selecting zero +tests fails the run; a skipped test is not evidence. + +The async ledger phase uses the real `Ledger` and managed PG driver: + +1. Validate PG16, `fsync=on`, `full_page_writes=on`, writer + `synchronous_commit=on`, and exactly one named streaming async standby on + the **primary**. Both differently named ledger pools use one unchanged DSN + through a loopback TCP endpoint. +2. Append eight shares and wait for the standby to replay the captured primary + flush LSN. Stop only that owned standby, then append three more shares. + Record the successful `Ledger::append` returns and the unreplicated WAL byte + gap. These returns are **not miner wire ACK measurements**. Time-based replay + lag while disconnected stays null/unknown, never zero. +3. Stop the exact owned primary and confirm both an already-open writer socket + fails and a new direct connection is refused. Verify the primary is stopped + **before** restarting/promoting the dedicated standby. No restart supervisor + exists for the test primary. This is a process fence for a local fixture; + production requires the operator's partition-safe fencing authority. +4. Verify recovery before promotion and writer role afterward; only then + retarget the stable endpoint. The two original pools reconnect without a + DSN change. Reconcile all eleven acknowledged IDs against the promoted DB: + eight survive and the deliberately unreplicated three are reported missing. + Do not retry those missing IDs and relabel them as recovered history. +5. Retry a **surviving** ID and verify no second credit; append one fresh share + on the new primary. Record fence-to-promotion time and the interval between + observed successful appends. Mining-endpoint downtime remains unmeasured. + Capture replication status from the promoted primary, where the lack of a + replacement standby is explicit. Stop all owned PG processes, close the + proxy and remove only verified-stopped owned directories. + +The injected three-share loss is a positive control showing that reconciliation +detects asynchronous loss. It is neither a production loss forecast nor an +acceptance budget, and must not weaken the full rehearsal's accounting criteria. + +## Operator components still required for the live gates + +Do not convert this example's writer proxy into a production failover service. +Before an actual overlay run, the rehearsal owner must supply a disposable +isolated Compose network, exact project/container/volume ownership, separate +node wiring, a stable writer endpoint with connection draining, fencing and +promotion authority, the dedicated standby, and the actual operator TCP LB. +Provide any public reader as a separately named member; never route reads to +the dedicated standby. Confirm the render's nondefault settings in each live +`check-config`, health `instance_id`, RPC target and self-check instance census. +Keep container IDs and volumes in the resource manifest before perturbations; +stop/remove only those IDs and verify the final census matches the baseline. + +For the TCP gate, retain the real operator configuration and timestamped probe, +routing, connection and submit events. Use the existing +[readiness state machine](prism-ha-readiness-probe-harness.md) as an expected +trace: HTTP 200 **and** JSON `ok: true`, two-second start cadence, one-second +total timeout, fall six and rise two. Exercise new tips and payout revisions, +then prolonged work loss, all-backends-down and recovery. Preserve established +connections during ordinary rebuilds. The current sustained-observation bound +is 13 seconds (28 seconds including the default stale-publication guard); record +actual scheduling and publication settings. A replayed simulator trace cannot +certify real routing. Longer rebuilds require reviewed policy, not a hidden +threshold increase. + +For failover, capture offered, miner-ACKed, rejected, unknown/no-response and +committed IDs before and after the authority change, including duplicate retries. +Record pre-loss primary flush and standby receive/replay LSNs, the acknowledged +unreplicated gap, measured downtime and any unmeasurable interval. Preserve the +original ACK deadline across dependent work and keep late durable/unknown +outcomes distinct. Observe the **dedicated standby on the current primary** via +the operator's [PostgreSQL exporter and D3 rules](prism-alert-migration.md#d3-deployment-provided-primarystandby-rules): +healthy streaming, lag, disconnection, stale/missing exporter data and post-promotion +no-standby state. Native public-replica metrics describe a different target and +cannot substitute for this signal. This preparation samples primary SQL only; +it does not deploy or certify exporter refresh, scrape cadence or alert delivery. + +If the operator LB, fencing/endpoint component or rehearsal host is unavailable, +retain the reusable preparation and record that component as missing. Leave all +three live criteria open. Report every acceptance row as executed, failed or +unexecuted, with its exact command/artifact and the reason for any gap. diff --git a/docs/prism-ha-readiness-probe-harness.md b/docs/prism-ha-readiness-probe-harness.md new file mode 100644 index 000000000..41084182f --- /dev/null +++ b/docs/prism-ha-readiness-probe-harness.md @@ -0,0 +1,22 @@ +# HA readiness probe harness + +This standalone simulator exercises the documented operator-supplied Stratum +TCP load-balancer readiness contract without network access, deployment +services, or production traffic. The defaults are the documented policy: +2-second monotonic probe cadence, 1-second outer timeout, six consecutive +failures to mark a frontend down, and two consecutive successes to allow new +sessions again. Initial state is explicitly `unknown`; failures, timeouts, +unreachable endpoints, malformed responses, non-200 status, and `ok != true` +never count as healthy. + +`tests/test_prism_ha_readiness_probe.py` provides positive and negative +controls, including transient rebuild preservation, bounded ejection, recovery +gating, timeout handling, and validation of non-default thresholds. It is a +deterministic health-check and hysteresis simulator, not an integration, +deployment, live load-balancer, or performance qualification. + +The [bounded #281 functional procedure](prism-ha-functional-qualification.md) +reuses this simulator alongside the existing real PostgreSQL and Stratum +fixtures. It records the remaining operator TCP endpoint, actual-overlay resume +and full failover gates separately; a simulated readiness trace does not satisfy +any live routing acceptance criterion. diff --git a/docs/prism-ha-reference-architecture.md b/docs/prism-ha-reference-architecture.md new file mode 100644 index 000000000..c6672f13f --- /dev/null +++ b/docs/prism-ha-reference-architecture.md @@ -0,0 +1,740 @@ +# PRISM HA Compose and database reference architecture + +This is the deployment contract for [#281](https://github.com/Qbit-Org/qbit-mining-bootstrap/issues/281), +after the 2026-09-10 scope trim and the [approved D3 decision](https://github.com/Qbit-Org/qbit-mining-bootstrap/issues/260#issuecomment-5635989715), +its [addendum](https://github.com/Qbit-Org/qbit-mining-bootstrap/issues/260#issuecomment-5636035280) +and the [#281 policy confirmation](https://github.com/Qbit-Org/qbit-mining-bootstrap/issues/281#issuecomment-5635994877). +The selected policy is **one primary plus one dedicated asynchronous failover +standby**, separate from the public-read replica. Writer sessions use +`synchronous_commit=on` and the primary uses `synchronous_standby_names=''`: +positive share ACKs require local WAL durability and do not wait for the standby. +Primary loss can lose acknowledged shares in the replication gap, and the +rows of blocks found in it; blocks found after a promotion on work whose +window lay in the replication gap are refused `stale-job`, given +NTP-synchronized database clocks. This is **not +lossless failover and has no guaranteed lag bound**. An optional bounded wait +keeps a found block's rows on the standby before its offer; see +[Found blocks](#found-blocks-a-bounded-standby-flush-before-the-offer-529). + +This separate document owns the complete frontend-to-database architecture. +[prism-postgres-replica.md](prism-postgres-replica.md) remains the detailed guide +to provisioning the existing public read replica and managing its slot. +The supplied overlay is a local two-frontend deployment, not a production HA +certification: its default node and database still share a Compose host. +Compact cross-frontend resume landed in [#397](https://github.com/Qbit-Org/qbit-mining-bootstrap/pull/397), +and [#273 is closed with runtime qualification evidence](https://github.com/Qbit-Org/qbit-mining-bootstrap/issues/273#issuecomment-5704427645). +That does not establish real-overlay reconnect/resume, promotion under load or +operator load-balancer behavior. Those live #281/#291 gates remain open in the +[acceptance evidence matrix](#281-acceptance-evidence). + +## Seven deployment requirements + +| # | Requirement | Exact wiring/settings | Delivery | +| --- | --- | --- | --- | +| 1 | Two active mining frontends | `prism-coordinator` and `prism-coordinator-2`; `PRISM_INSTANCE_ID=prism-frontend-1` / `prism-frontend-2`; identical `PRISM_DATABASE_URL`, payout configuration and signing keys; shared database job/extranonce authority. | **Provided here:** runtime, compact resume and two-service overlay. **Operator-supplied:** placement in independent failure domains and matching secret distribution. Real-overlay resume qualification remains open. | +| 2 | Highly available Stratum TCP endpoint | Operator endpoint sends TCP to frontend ports `3340` / `3343` on the Compose host by default; probe each frontend's own `/healthz`, with the hysteresis below. Publish that endpoint in `PRISM_PUBLIC_STRATUM_URL`. | **Operator-supplied:** TCP load balancer, endpoint redundancy, routing and firewall. No LB service is shipped. | +| 3 | A separate node per frontend | Set `PRISM_HA_RPC_HOST_1=node-1` and `PRISM_HA_RPC_HOST_2=node-2`, or set each `PRISM_HA_RPC_URL_1/2` to its node's complete HTTP(S) URL. Same chain/genesis and consensus policy on both. | **Provided here:** independent per-service RPC wiring. **Operator-supplied:** both independent production nodes, authentication and placement. Both lab defaults use bundled `qbitd`. | +| 4 | PostgreSQL primary plus one dedicated asynchronous failover standby, with separate public reads | Primary: `fsync=on`, `full_page_writes=on`, `wal_level=replica`, `max_wal_senders=10`, `max_replication_slots=10`, `synchronous_standby_names=''`; writer sessions: `synchronous_commit=on`. HA standby: `hot_standby=on`, its own physical slot and `application_name=prism_standby_1`. Optional found-block standby wait: `PRISM_OFFER_STANDBY_APPLICATION_NAME=prism_standby_1` on both frontends and `GRANT pg_monitor TO prism_writer` ([#529](#found-blocks-a-bounded-standby-flush-before-the-offer-529)). `prism-public-api` uses the separate public-read DSN in `PRISM_PUBLIC_DATABASE_URL` with `PRISM_PUBLIC_REPLICA_MODE=require`. | **Provided here:** lab primary, asynchronous **public-read** replica, slot bootstrap and checked public API. **Operator-supplied:** dedicated HA standby, D3 configuration/monitoring, independent storage/failure domains, backups and capacity. | +| 5 | Promotion | After fencing and checking the eligible standby, `SELECT pg_promote(wait => true, wait_seconds => 60)`; verify `pg_is_in_recovery() = false` and the required history before moving the writer endpoint. | **Operator-supplied:** decision authority, failure detection, promotion execution and rehearsal. Procedure below; no automatic promotion service. | +| 6 | Fence the old primary | Power/storage fencing or enforced network isolation that stops **all existing and new** writer connections, including both frontends and settlement workers; disable old-primary restart automation. Keep the fence until rejoin as a standby. | **Operator-supplied:** fencing mechanism and positive confirmation. Changing DNS or stopping one frontend is insufficient. | +| 7 | Stable authoritative writer endpoint | Both frontends use the same `PRISM_DATABASE_URL=postgresql://prism_writer:<secret>@prism-writer.internal:5432/qbit?sslmode=verify-full`; the endpoint routes only to the unfenced primary. | **Provided here:** shared DSN pass-through and external-DB overlay. **Operator-supplied:** endpoint ownership, TLS, failover routing, connection draining and fencing. The bundled `prism-postgres` name is not a failover endpoint. | + +D3's primary and dedicated failover standby are two HA members. A separate +public-read replica is an additional database member, not a second approved +failover candidate. Keep public queries off the dedicated standby so they cannot +delay its replay. The bundled lab's primary/public-replica pair does not supply +this complete production topology, and the HA overlay does not provision it. + +## Compose stacks and configuration path + +Use Docker Compose **2.24.4 or newer** for +[`!override` support](https://docs.docker.com/reference/compose-file/merge/#replace-value). +Apply HA last: + +```sh +# Bundled lab database, one bundled node, two frontends: +docker compose -f compose.yaml -f compose.prism-ha.yaml --profile prism config --quiet +docker compose -f compose.yaml -f compose.prism-ha.yaml --profile prism up -d + +# Production image/storage restrictions, bundled database: +docker compose -f compose.yaml -f compose.production.yaml \ + -f compose.prism-ha.yaml --profile prism config --quiet + +# Operator database endpoint: +docker compose -f compose.yaml -f compose.prism-external-db.yaml \ + -f compose.prism-ha.yaml --profile prism config --quiet + +# Production restrictions and operator database endpoint: +docker compose -f compose.yaml -f compose.production.yaml \ + -f compose.prism-external-db.yaml -f compose.prism-ha.yaml --profile prism config --quiet +``` + +All four stacks are supported and validated in CI on pushes and PRs. Production +requires the existing explicit image and storage variables; its invalid-image +and missing-bind-source defaults remain effective on **both** frontends. +External DB stacks require an explicit writer DSN, and should explicitly set the +public read DSN and `PRISM_PUBLIC_REPLICA_MODE=require`: the external-DB overlay's +existing fallback otherwise uses the writer with replica checks `off`. +Do not activate `local-database` when using external DBs. + +The second service shares the base coordinator template through a YAML anchor. +Its `prism-ha-overlay-required` profile is an implementation detail: do not enable +it directly. The HA overlay enables it with the ordinary `prism` profile and +replaces host port mappings. Production and external-DB overrides share their +coordinator settings through anchors too. A cross-file `extends` would resolve +before overlays merge and silently lose some effective production settings. +Without HA, the existing `prism` deployment starts only one coordinator. + +Run `check-config` in **each** service before serving traffic. Shared keys, +chain/genesis, fee and payout policy must match; the existing cluster fingerprint +enforces that agreement. IDs must be distinct and stable across restarts; each +passes the existing `Config::from_env` validation (at most 128 characters, no +control characters or reserved `prepared:` prefix). Duplicate IDs overwrite the +same heartbeat row and cannot establish HA. Do not scale either named service +with `--scale`, which would duplicate its configured ID and published ports. + +### Every HA setting reaches its consumer + +Compose uses shell variables ahead of `--env-file`/`.env`; later env files win +over earlier files, and later Compose overlays win on the same service keys. +These are interpolation knobs, not new application environment readers: + +| Operator input | Default | Container/effective consumer | +| --- | --- | --- | +| `PRISM_HA_INSTANCE_ID_1`, `PRISM_HA_INSTANCE_ID_2` | `prism-frontend-1`, `prism-frontend-2` | `PRISM_INSTANCE_ID` → existing `Config::from_env().instance_id` → `Coordinator` / `Ledger` / health and heartbeat. HA supersedes the global `PRISM_INSTANCE_ID`. | +| `PRISM_HA_RPC_HOST_1`, `PRISM_HA_RPC_HOST_2` | global `QBIT_RPC_HOST`, or `qbitd` | `QBIT_RPC_HOST` → existing `rpc_connection_from_env()` → `Config.rpc_url` → coordinator RPC client. A nonempty per-frontend setting wins; otherwise the global setting is preserved. | +| `PRISM_HA_RPC_URL_1`, `PRISM_HA_RPC_URL_2` | global `QBIT_RPC_URL`, or empty | `QBIT_RPC_URL` → same existing reader. A nonempty URL wins over host **and** `QBIT_RPC_PORT`; blank uses `http://<host>:<QBIT_RPC_PORT>/`. A nonempty per-frontend setting wins; otherwise the global URL is preserved. To use a per-frontend host when a global URL exists, provide a complete per-frontend URL; a blank override falls back to the global URL. `QBIT_RPC_USER/PASSWORD` and the fallback port remain shared; different node credentials require a service environment override. | +| `PRISM_DATABASE_URL` (existing) | bundled `prism-postgres:5432`, default DB/user `qbit` | Same full DSN on both services → `Config.database_url` validation → `Ledger::connect`; no HA-specific DSN or second environment reader. Database credentials are inherited from the base. | +| `PRISM_HA_STRATUM_PORT_HOST_1`, `PRISM_HA_STRATUM_PORT_HOST_2` | existing `PRISM_STRATUM_PORT_HOST` or `3340`; `3343` | Docker publishes to the common `PRISM_STRATUM_PORT` (default `3340`), read by existing `StratumConfig`. Host mappings do not alter the listener. | +| `PRISM_HA_HIGHDIFF_PORT_HOST_1`, `PRISM_HA_HIGHDIFF_PORT_HOST_2` | existing `PRISM_STRATUM_HIGHDIFF_PORT_HOST` or `127.0.0.1:0`; `127.0.0.1:0` | Docker allocates separate loopback host ports; target is `PRISM_STRATUM_HIGHDIFF_PORT` or `4334`. The actual listener remains disabled unless its existing runtime setting enables it. Choose distinct explicit host ports when enabling it. | +| `PRISM_HA_HEALTH_PORT_HOST_1`, `PRISM_HA_HEALTH_PORT_HOST_2` | `127.0.0.1:3341`, `127.0.0.1:3344` | Docker publishes the common `PRISM_HA_AUDIT_PORT` target; the effective `PRISM_AUDIT_PORT` is read by existing API configuration and `healthcheck`. | +| `PRISM_HA_AUDIT_BIND` | `0.0.0.0` in HA | `PRISM_AUDIT_BIND` → existing API listener reader; this HA knob supersedes the base `PRISM_AUDIT_BIND`, including the loopback value in `.env.example`, so published probes work. An explicit loopback-only bind defeats host forwarding. Use a reachable bind for HTTP probes. | +| `PRISM_HA_AUDIT_PORT` | `3341` | Supersedes base `PRISM_AUDIT_PORT` (including `0`) on both services and their published targets → existing `Config.audit_port` / API listener and `healthcheck`. Must be nonzero. | + +The HTTP health listener is required by this overlay's external LB contract. +Audit-disabled deployments are unsupported with this overlay; they need an +explicit operator override of both environment and port mappings, plus their own +readiness contract. Set the HA port knob when migrating a custom audit port. + +Only health ports default to host loopback; Stratum defaults publish on all host +interfaces. Permit the health ports only from the operator's management/LB +network. The audit listener exposes more than health: do not publish it as a +public endpoint. For a remote operator TCP load balancer bind the health mapping to the intended +private host address, and enforce the corresponding network ACL. + +Changing RPC targets does not remove the inherited `depends_on: qbitd`; the +bundled node still starts in these stacks. Operators deploying independently +managed nodes can override dependencies in their placement configuration. + +Safe configuration verification (use fixture values, never log real secrets). +Both fixture URLs are explicit so inherited credentialed RPC URLs cannot appear +in the projection: + +```sh +PRISM_HA_INSTANCE_ID_1=verify-east PRISM_HA_INSTANCE_ID_2=verify-west \ +PRISM_HA_RPC_HOST_1=node-east.internal PRISM_HA_RPC_HOST_2=node-west.internal \ +PRISM_HA_RPC_URL_1=http://node-east.internal:19452/ \ +PRISM_HA_RPC_URL_2=http://node-west.internal:19452/ \ +docker compose -f compose.yaml -f compose.prism-ha.yaml --profile prism config \ + --format json | jq '.services | with_entries(select(.key | startswith("prism-coordinator"))) | + map_values({ports, environment: (.environment | + {PRISM_INSTANCE_ID, QBIT_RPC_HOST, QBIT_RPC_URL, PRISM_AUDIT_BIND, PRISM_AUDIT_PORT})})' + +docker compose -f compose.yaml -f compose.prism-ha.yaml --profile prism \ + exec prism-coordinator qbit-prism-server check-config +docker compose -f compose.yaml -f compose.prism-ha.yaml --profile prism \ + exec prism-coordinator-2 qbit-prism-server check-config +``` + +For image qualification also render with `config --resolve-image-digests --quiet` using +available registry images, or start the built images and inspect health IDs, +effective RPC targets and port mappings through the real startup path. A plain +render alone does not demonstrate that a node answers RPC or that a coordinator +is healthy. Review credentials privately if a complete RPC URL contains any. + +## D3: approved asynchronous replication and ACK policy + +The following describes PostgreSQL commit semantics with `fsync=on` and +`full_page_writes=on`. It assumes the selected replication wait finishes +normally. **Cancellation is a separate qualification concern below.** + +| `synchronous_commit` | ACK guarantee with `synchronous_standby_names='FIRST 1 (prism_standby_1)'` | Standby down | Current PRISM writer session | +| --- | --- | --- | --- | +| `off` | No local WAL-flush guarantee; acknowledged shares can disappear after a primary crash. | Does not wait. | Replaced with `on`. | +| `local` | Local WAL is durable; no replica durability guarantee. | Does not wait. | Replaced with `on`. | +| `remote_write` | Local WAL durable, standby OS has written WAL; standby OS/power failure can lose it. | Waits for the named standby. | Replaced with `on`. | +| `on` | WAL flushed on primary and named standby; survives primary loss if the standby's storage survives. | Waits for the named standby. | Preserved. | +| `remote_apply` | As `on`, plus the commit is replayed and visible to standby queries. | Waits for standby replay. | Preserved if set when the connection opens. | + +With `synchronous_standby_names=''`, every non-`off` level only waits for local +flush, and standby loss does not block replication-related ACK progress. +[PostgreSQL 16 commit semantics](https://www.postgresql.org/docs/16/runtime-config-wal.html#GUC-SYNCHRONOUS-COMMIT). + +The runtime pool's existing `after_connect` explicitly keeps `remote_apply` or +sets `on`, and refuses `fsync=off` / `full_page_writes=off`. A role default of +`local`, `off`, or `remote_write` therefore does **not** select that mode for PRISM. +`self-check.durability` queries `pg_settings` through this same configured ledger +pool, so it observes the session value actually used, not just a server default. +For `remote_apply`, set the writer role/database default before reconnecting all +writer pools; existing connections do not retroactively receive a role default. + +### Selected policy and optional synchronous qualification + +**Asynchronous availability is selected.** Keep `synchronous_standby_names=''` +and `synchronous_commit=on`. Standby loss does not add a replication wait to +positive ACKs. During standby downtime, and after promotion until a replacement +is caught up, locally durable writes have no failover-copy guarantee. The loss +exposure is the unreplicated WAL gap, which can grow throughout standby downtime; +five-second alerting does not cap it. D3's rough expected local-link lag was an +estimate, not a measurement or service guarantee. #291 must record real lag, +ACK/accounting reconciliation, promotion time and frontend recovery time. + +Strict synchronous operation remains deferred behind D3's two preconditions: +(1) close and qualify the synchronous-wait cancellation/ACK gap across the actual +runtime paths, including the remaining limits below; (2) provide either a second +standby or an approved manual degrade procedure, so standby maintenance does not +stall mining. The landed share-commit guard addresses part of (1), not both +preconditions. No strict or automatic-degrade policy is approved here. + +Provision a stable `application_name=prism_standby_1` from the start. The D3 +addendum permits an optional **qualification** flip on the primary, without a +restart, once that intended standby is streaming. For a server whose settings +are managed through `ALTER SYSTEM`, execute these as separate statements outside +a transaction; an operator-managed configuration must use its owning mechanism: + +```sql +-- Optional #291 experiment, with synchronous ACK caveats below: +ALTER SYSTEM SET synchronous_standby_names = 'FIRST 1 (prism_standby_1)'; +SELECT pg_reload_conf(); +SHOW synchronous_standby_names; + +-- Restore the approved asynchronous policy after the experiment: +ALTER SYSTEM SET synchronous_standby_names = ''; +SELECT pg_reload_conf(); +SHOW synchronous_standby_names; +``` + +Record the effective settings, replication state, ACK latency and throughput +under load before/during/after the flip, including standby-down behavior and +return to async. A reload request is not evidence of effective configuration; +verify the `SHOW` results and standby identity. Existing D3 alerts intentionally +flag a non-async topology, so record that expected observation during the drill. +This experiment does not establish strict no-unreplicated-ACK durability or +change the approved production policy. PostgreSQL documents the +[reloadable standby-name setting](https://www.postgresql.org/docs/16/runtime-config-replication.html#GUC-SYNCHRONOUS-STANDBY-NAMES). + +### Found blocks: a bounded standby flush before the offer (#529) + +The replication gap can hold a found block's candidate row and its offer +reservation as well as shares. A block offered to the node whose rows are then +lost with the primary is on chain with no candidate, audit or `qbit_pool_blocks` +record on the promoted primary, and no code path lands an on-chain block without +its candidate row ([#529](https://github.com/Qbit-Org/qbit-mining-bootstrap/issues/529)). + +Set `PRISM_OFFER_STANDBY_APPLICATION_NAME=prism_standby_1` on every frontend to +close the window between the offer and the standby copy, without changing D3: + +- After a found block's offer reservation commits, the offering frontend reads + the primary's `pg_current_wal_flush_lsn()`, which covers the candidate row and + the reservation. It then polls `pg_stat_replication` until the `flush_lsn` + of every standby connected under that name (`streaming`, or `catchup` after + a reconnect) reaches it, and only then calls `submitblock`. `flush_lsn` is + what `synchronous_commit=on` would wait for: everything the standby flushed + survives its promotion. `sent_lsn` would not (the drill and the pair test + fail with it); `write_lsn` differs only across a crash of the standby's own + host, which the tests do not exercise. A second standby sharing the name can + delay the wait to the bound, never confirm it. +- The wait is bounded by `PRISM_OFFER_STANDBY_FLUSH_WAIT_MS` (default 250, `0` + turns it off, at most 10000). It never holds the block: a standby that is not + connected is not waited for, and at the bound the block is offered anyway. + Every offer without a confirmed copy logs the block hash at WARN + (`block offer standby wait unconfirmed`, with its `outcome`, `waited_us` and, + when lagging, `lag_bytes`) for the reconciliation in + [promotion step 6](#promotion-fencing-and-the-stable-writer-endpoint). +- `synchronous_standby_names` stays `''`: no commit waits for the standby, and + share appends and their ACKs are unchanged. Block-only proofs are the one + exception: their ACK already waits for their candidate's disposition, so a + lagging standby delays those ACKs by the bound. +- The writer role must read other sessions' replication positions: + `GRANT pg_monitor TO prism_writer;` (`pg_read_all_stats` alone is enough). + Without it PostgreSQL shows only the WAL sender's pid, and every wait reports + `failed`. `self-check` refuses a configured wait whose role cannot read the + positions, or whose name does not match exactly one connected standby + reporting its flush position; + `check-config` prints the effective setting. +- `qbit_prism_block_offer_standby_wait_total{outcome=confirmed|absent|lagging|failed}` + counts the waits, and `PrismBlockOfferStandbyUnconfirmed` warns for 30 minutes + after any offer that was not `confirmed`. + +Measured with the prototype on loopback PostgreSQL 16, a debug build and a noisy +shared host, so read as relative A/B: a healthy standby confirmed in 0.9 ms p50 +and at most 4 ms p99; proof-to-`submitblock` latency and share ACK latency were +unchanged; a standby that was down cost at most 2.5 ms per offer; a standby +lagging past a 250 ms bound held every offer for the bound (251 to 262 ms). The +gated drill `a_found_block_offered_after_the_standby_flush_survives_a_replication_cut_at_submitblock` +in [`live_pg_offer_standby.rs`](../crates/qbit-prism-server/tests/support/live_pg_offer_standby.rs) +cuts replication the moment the block reaches the node, then fences, promotes +and moves the endpoint: the promoted primary holds the reservation and the block +lands once. Without the wait, or polling `sent_lsn`, the standby has no row for +the offered block at the cut. + +With the wait on, the accepted asynchronous loss is: acknowledged shares in the +last replication gap; found blocks whose offer logged an unconfirmed standby +wait (standby down, behind at the bound, or unreadable); and found blocks never +offered before the primary was lost, since no frontend can reserve an offer on a +dead primary; and found blocks whose offering frontend crashed or lost its +claim between the reservation and `submitblock`. The wait widens that last +window from one database round trip to at most the bound (250 ms by default, +10 s at most): recovery treats such a reservation as delivery unknown and never +offers the block, as it always has. That is the price of keeping the +reservation on the standby, which lets a held attempt record its outcome on +the promoted primary. The wait does not make D3 synchronous and is not a strict +durability claim. + +Since #619 the accepted asynchronous loss also names blocks found after a +promotion on work whose window lay in the replication gap. Such a block is +refused `stale-job` before its enqueue, counted under +`qbit_prism_stale_job_rejections_total{cause="window_not_held"}`, and never +offered: only its reward is lost. See +[work from a lost replication gap](prism-ledger-ops.md#work-from-a-lost-replication-gap-619). + +**Precondition: synchronized database clocks.** The refusal compares time, +not row identity: a share the promoted primary reissues under a lost number +fails the check only because the promoted host stamps it later than the gap +work's anchor. Keep the primary and the standby NTP-synchronized. A promoted +host whose clock lags the old primary's by more than the time from the last +gap work to the first reissued share (seconds) can let such a block pass, be +offered, and end in `reconciliation` with a window digest mismatch, as before +#619; step 6 reconciles it. + +### Current timeout/cancellation behavior and remaining limits + +The ledger sets `statement_timeout=15000` ms by default +(`PRISM_DATABASE_STATEMENT_TIMEOUT_MS`, valid `1..600000`) and +`lock_timeout=5000` ms (`PRISM_DATABASE_LOCK_TIMEOUT_MS`). PostgreSQL can cancel +a synchronous-replication wait **after local commit**, emit a warning and finish +the commit without replica confirmation; disconnects can also leave an unknown +commit outcome. A timeout therefore is not proof of rollback or a reliable +"no positive ACK" policy. See PostgreSQL's +[synchronous-wait cancellation handling](https://github.com/postgres/postgres/blob/REL_16_STABLE/src/backend/replication/syncrep.c). + +[#333](https://github.com/Qbit-Org/qbit-mining-bootstrap/pull/333) landed #324's +commit reconciliation. The current [share-ACK path](../crates/qbit-prism-server/src/coordinator/miner_submit.rs) +still passes a one-shot commit gate immediately before COMMIT. For a share-pass +append without a block candidate, at `PRISM_SHARE_COMMIT_TIMEOUT_SECONDS` the ACK +path closes the gate if COMMIT has not been sent: the append rolls back +and the miner gets `ledger-confirmation-failed`. If COMMIT is already in +flight, the ACK path waits a further `share_commit_grace` (5 s) for its reply. +Only a severity-ERROR reply counts as a rollback; any other failure, or no +reply by then, is answered `ledger-outcome-unknown` and logged with the share +ID. A sync-rep guard also answers `ledger-outcome-unknown`, not accepted, when +COMMIT took at least the ledger sessions' effective `statement_timeout`, +because that may be a synchronous-replication wait cancelled after local +commit. This is a duration-based guard, not a direct replica-confirmation check. +Share-pass appends carrying a block candidate are never refused by the ACK deadline; +they follow the same deadline and grace as a plain share and use the same result +classifier. Block-only proofs instead wait for their credit/candidate disposition +up to the share deadline. See the [commit reconciliation tests](../crates/qbit-prism-server/src/coordinator/miner_tests/commit_reconcile.rs) +and [writer session setup](../crates/qbit-prism-server/src/ledger/connect.rs). + +**`statement_timeout` never ends a synchronous-replication wait (#529).** +PostgreSQL disables the statement timeout before it commits: +`finish_xact_command()` calls `disable_statement_timeout()` and only then +`CommitTransactionCommand()`, whose `SyncRepWaitForLSN()` does the waiting. Only +a query cancel (`pg_cancel_backend`), a backend termination, a configuration +reload that empties `synchronous_standby_names`, or the standby's return ends +it. Measured on PostgreSQL 16 with `synchronous_standby_names='FIRST 1 (prism_standby_1)'`, +the standby stopped and `statement_timeout=2s`: a transaction that ran +`SET LOCAL synchronous_commit=on` waited in COMMIT on `IPC/SyncRep` for 1124 s +(18.7 minutes) until `pg_cancel_backend`, then returned the "canceling wait for +synchronous replication" warning and `COMMIT`. Its row stayed invisible to other +sessions for the whole wait. So the duration guard above describes a +cancellation that `statement_timeout` never makes: in the optional #291 sync +experiment with the standby down, writer backends block with their locks held +until something cancels them. The share-ACK path still answers within +`PRISM_SHARE_COMMIT_TIMEOUT_SECONDS` plus `share_commit_grace` +(`ledger-outcome-unknown`), but the backend does not return. **A statement +timeout cannot be the duration guard.** Bound a synchronous experiment with an +explicit cancel (for example an operator watchdog that cancels backends whose +`wait_event` is `SyncRep` past a limit) or by returning to async. For the same +reason, `SET LOCAL synchronous_commit` for found blocks alone is not a safe +alternative to the bounded wait above: it also needs a non-empty +`synchronous_standby_names`, which changes D3. + +**Remaining limits:** a block-only ACK can follow a locally visible credit whose +sync-rep wait was cancelled; its poll does not observe the crediting transaction's +replication wait. A share-pass COMMIT whose wait is cancelled before +`statement_timeout` (for example by `pg_cancel_backend`) can also evade the +duration guard. PostgreSQL may return a warning and local success in that case. +These paths prevent a blanket no-unreplicated-ACK claim for optional sync mode; +they do not change the approved async loss model. + +#291 must exercise the actual PRISM share-ACK path with the standby stopped, +through and beyond these timeouts, including lost client responses. Its +standby-down run must cover block-only proofs as well. **Strict standby-down +ACK behavior is not certified by selecting `on` alone.** Resolve any required +runtime changes with the ledger owner before adopting strict synchronous +operation. Increasing a finite timeout only postpones the question; do not claim +an infinite wait. + +### Exact database provisioning choices + +Provision the dedicated HA standby with a replication role, scoped HBA +rule and base-backup procedure adapted from [the public-replica runbook](prism-postgres-replica.md). +Use separate primary/standby storage and a separate HA slot. From initial async +provisioning, assign a unique `application_name=prism_standby_1` in the standby's existing +`primary_conninfo` (retain its host, credentials and TLS settings), and set: + +```conf +# Primary and dedicated HA standby, so the latter can later be the writer: +fsync = on +full_page_writes = on +wal_level = replica +max_wal_senders = 10 +max_replication_slots = 10 +hot_standby = on + +# Dedicated HA standby only, alongside standby.signal and its primary_conninfo: +primary_slot_name = 'prism_ha_standby' # example: operator must provision this slot +wal_receiver_status_interval = 1s # required by the alert observation contract + +# Primary: approved D3 asynchronous policy: +synchronous_standby_names = '' +``` + +Set the writer role's default to the approved level: +`ALTER ROLE prism_writer IN DATABASE qbit SET synchronous_commit = 'on';`. +Reconnect writer pools after changing role defaults. Reload server configuration +with `SELECT pg_reload_conf()` where applicable; restart for start-only settings. +Do not enable the synchronous name until the standby is streaming and capable +of satisfying the policy, or even bootstrap/writer startup commits can wait. + +`application_name`, not the slot name or container name, is what +`synchronous_standby_names` matches. The shipped bootstrap does not assign that +explicit name; operators must configure and verify it on the dedicated HA standby. +Use a different application name and slot for the public-read replica. +Never use a wildcard that could count an unintended replication client. +[PostgreSQL replication configuration](https://www.postgresql.org/docs/16/runtime-config-replication.html). + +The bundled **public-read** replica's physical slot is `prism_public_replica`, selected by +`PRISM_POSTGRES_REPLICATION_SLOT` in the base Compose environment and consumed +by `config/prism-postgres/replica-entrypoint.sh` (`pg_basebackup --slot`, then +`primary_slot_name`). It is not the dedicated HA standby's slot. Provision a separate +physical slot for that standby (the template uses `prism_ha_standby`); each slot +retains WAL independently and does not make replication synchronous. Monitor +`active`, `restart_lsn`, retained bytes, WAL disk space and `wal_status`. +The shipped `max_slot_wal_keep_size=-1` default does not cap retention. Operators +must size and approve a cap (for example `max_slot_wal_keep_size='16GB'` only +after measuring WAL rate and outage budget); exceeding it may require a fresh +base backup. Do not drop a live standby's slot to clear disk pressure. +[PostgreSQL slot retention settings](https://www.postgresql.org/docs/16/runtime-config-replication.html#GUC-MAX-SLOT-WAL-KEEP-SIZE). + +Verify from an authorized operator connection to the primary: + +```sql +SHOW synchronous_standby_names; +SELECT application_name, state, sync_state, sent_lsn, write_lsn, flush_lsn, replay_lsn +FROM pg_stat_replication; +SELECT slot_name, active, restart_lsn, wal_status, + pg_wal_lsn_diff(pg_current_wal_lsn(), restart_lsn) AS retained_bytes +FROM pg_replication_slots; +``` + +For approved D3 expect empty `synchronous_standby_names`, exactly one matching +`prism_standby_1` row in `state='streaming'` / `sync_state='async'`, and its expected +active slot. The separately named public replica can have its own streaming row. +During the optional synchronous experiment, the HA row should become `sync`. +Also inspect the writer session's effective durability through self-check, +not only an administrator's +`SHOW synchronous_commit`, whose role/session defaults may differ. + +### Dedicated HA standby alerting + +D3 requires alerts for replay lag above **5 seconds** and standby disconnection. +Use the current [D3 alert contract](prism-alert-migration.md#d3-deployment-provided-primarystandby-rules), +[PostgreSQL rule source](prism-postgres-alert-rules.json) and +[primary query extension](prism-postgres-exporter-queries.yaml). These are +**operator-provided primary PostgreSQL exporter/custom-query metrics**, not +native PRISM metrics or a provisioned monitoring deployment. Never substitute +`qbit_prism_public_replica_*`: those describe the separate public-read target. + +The rules use a one-minute dwell and one-second primary scrapes/standby status +updates. The lag rule compares the HA standby's replay position with the +primary's durable WAL position sampled five seconds earlier; it does not treat +PostgreSQL's last reported `replay_lag` as a current backlog-age bound. An idle +caught-up standby may report NULL lag. Missing/stale/failed observations, +ambiguous identity and invalid topology remain alertable, not healthy. Verify +permissions, query refresh/cost, scrape cadence and exporter failure behavior in +#291 using the linked contract. An alert threshold is not an enforced loss bound. +[PostgreSQL replication statistics semantics](https://www.postgresql.org/docs/16/monitoring-stats.html#MONITORING-PG-STAT-REPLICATION-VIEW). + +## Promotion, fencing and the stable writer endpoint + +This is an operator procedure, not automatic failover. Follow it under the +approved asynchronous D3 policy, with an authoritative decision maker outside the +primary/failover pair; the pair alone does not supply a partition-safe election. + +1. Record the incident and pause new mining admission if required by the ACK + policy. Capture acknowledged-share evidence and available WAL positions. +2. **Fence the old primary first.** Confirm power/storage isolation or network + isolation covering existing connections and every writer. Disable its restart + automation. If positive fencing cannot be established, do not promote. +3. Verify the single eligible standby: correct cluster/history, recovery active, + received/replayed WAL positions and storage health. In a planned switch, quiesce + writes and wait for replay through the captured primary flush LSN. After an + unplanned loss, only claim the approved replication guarantee; an asynchronous + or degraded interval requires explicit accounting-loss reconciliation. +4. Promote on that standby with `SELECT pg_promote(wait => true, wait_seconds => 60)` + and confirm `pg_is_in_recovery() = false`. A timeout needs inspection, not a + second competing promotion. Apply the approved no-standby ACK policy. +5. Move `prism-writer.internal`/the managed writer endpoint to this sole primary. + Flush or drain existing database proxy connections and reconnect both PRISM + pools as needed; DNS changes do not retarget established sockets. Verify both + frontends use the same writer, matching keys/fingerprint and preserved share + history, then resume admission under the approved policy. A found block that + was mid-landing on a surviving frontend lands within seconds of the move: + that frontend retries its failed claim release until the new primary + answers (#529). A claim held by a frontend lost with the old primary waits + for its 120 s candidate lease. +6. **Reconcile found blocks offered without a standby copy.** Collect every + `block offer standby wait unconfirmed` hash that either frontend logged + before the primary was lost, at least back to the last time the standby was + known to be caught up (all of them when unsure; there are few). For each, + read its candidate on the new primary + (`SELECT state FROM qbit_block_candidate_outbox WHERE block_hash = '<hash>'`) + and ask the node whether the block is on its active chain + (`getblockheader <hash>`, `confirmations` of at least 1). A candidate that + survived needs nothing: the ordinary recovery lands it once. A block with no + candidate that is not on the active chain lost only its reward, like a block + found in the gap and never offered; record it. A block **with no candidate + that is on the active chain** paid the balances its coinbase commits to while + the promoted ledger still carries them, so the next landing would pay them + again: treat it as the explicit accounting-loss reconciliation of step 3, + with the frontend log line, the block and its coinbase as evidence. + `recover` cannot land it; it refuses a hash with no candidate row. The same + check covers blocks found while `PRISM_OFFER_STANDBY_APPLICATION_NAME` was + unset, whose offers log nothing: compare the node's recent blocks carrying + the pool's coinbase tag with `qbit_block_candidate_outbox` instead. Also + collect each `block refused stale-job before its offer` WARN (#619): a block + found after the promotion on work whose window lay in the gap, never + offered, so only its reward was lost. A candidate in `reconciliation` + whose `last_error` starts `window not held by this primary` (or, from an + older release, `window range incomplete: … read 0`) was offered before the + refusal existed and is on chain: reconcile it as an accounting loss, and + with CTV settlement on export its window's rows from the fenced old primary + before step 7 reuses that storage, as + [the ledger runbook](prism-ledger-ops.md#work-from-a-lost-replication-gap-619) + describes. +7. Rejoin the fenced former primary only after a verified rewind or fresh base + backup of the new primary. Create/reconcile the physical slot on the new + primary; PostgreSQL 16 physical slots are not automatically transferred by + this Compose setup. Re-establish the single named standby and verify its + async streaming/replay status and monitoring. Keep `synchronous_standby_names=''` + under approved D3; record the interval without a caught-up failover copy. +8. Reconfigure or rebuild the **separate public-read replica** to follow the new + primary, and verify its read endpoint and freshness. Do not direct public reads + to the dedicated failover standby. With `PRISM_PUBLIC_REPLICA_MODE=require`, + the public process refuses a promoted writer. Deliberately choosing `off` to + read the writer is a separate recorded operator action. + +The bundled replica's healthcheck requires recovery and will mark a promoted +instance unhealthy. Its bootstrap entrypoint also refuses to restart a complete +cluster lacking `standby.signal`. For a lab drill, reconfigure the promoted +service to use the stock PostgreSQL entrypoint and a primary healthcheck before +restart; never empty or re-bootstrap the promoted data directory. Production +database role transitions belong to the operator's HA system. +[PostgreSQL promotion and fencing guidance](https://www.postgresql.org/docs/16/warm-standby-failover.html). + +## Operator TCP load-balancer readiness contract + +Check each **mining frontend**, not the public API, PostgreSQL port, aggregate +pool endpoint or a TCP-connect-only check. No authentication or signing key is +needed for these probes: + +- Preferred: `GET http://<frontend-management-address>:<health-host-port>/healthz`. + Success requires HTTP **200** and JSON boolean **`ok: true`**. Do not use just + HTTP status or `ready`: the stale-snapshot guard can clear `ok` independently. + The body includes `schema=qbit.prism.audit-health.v1`, `instance_id`, `ready`, + `status` and `snapshot_age_seconds`. +- Local equivalent: `qbit-prism-server healthcheck` exits 0 only for success. + `--url http://.../healthz` selects a remote HTTP target; its internal HTTP timeout + is 3 seconds. Do not pass `--public-api` for mining routing. +- When the audit listener is disabled (`PRISM_AUDIT_PORT=0`), the subcommand uses + Stratum instead: send `{"id":1,"method":"mining.get_health","params":[]}\n` to + that frontend's enabled listener, without subscribe/authorize. Require one + newline-terminated JSON frame at most 4096 bytes, matching `id=1`, null `error` + and `result.ready=true`, within 3 seconds. HTTP is preferred for the shipped + overlay because it also checks the published-health freshness guard. + +`ready` is an instantaneous ability to provide current work: prepared work must +match the observed node tip and database payout revision, the tip poll must be +younger than `PRISM_HEALTH_TIP_POLL_MAX_AGE_SECONDS` (default 15), and an enabled +CTV policy must satisfy the current fee floor. The tip poll is the last refresh +that published or revalidated work, or the last refresh-grade node poll that found +the published tip with its template still within `PRISM_TEMPLATE_MAX_AGE_SECONDS` +and, under CTV settlement, a relay floor read no older than the poll that the +published fee still meets (#622), so a slow rebuild on a responsive node does not +clear readiness; +`qbit_prism_tip_poll_age_seconds` exports its age. The server can also clear readiness +for stalled job delivery. A new tip **or payout-revision change** invalidates +prepared work until rebuilding and publishing its replacement; this normal +transition clears readiness even when the process and TCP listener are alive. + +Recommended operator policy, subject to #291's measured rebuild durations: + +| Parameter | Value / behavior | +| --- | --- | +| Probe start interval | Every **2 seconds**, per frontend, using a monotonic scheduler. No overlapping probes; do not add a full interval after completion. | +| HTTP timeout | **1 second** total; timeouts, malformed JSON, non-200 or `ok != true` count as failures. A local command adapter must enforce the same outer deadline. | +| Fall threshold | **6 consecutive failures**. A successful probe resets the failure count. Until this threshold, preserve the prior routing state. | +| Rise threshold | **2 consecutive successes** before a down/starting backend accepts new sessions. A failure resets the success count. | +| Routing action | Mark down for **new TCP sessions**; do not kill established sessions solely because of an ordinary rebuild. If all are down, do not silently route to known-unready backends. | + +Six starts span ten seconds, so an observed rebuild outage **shorter than ten +seconds** does not by itself eject a previously up backend. A continuously +failing endpoint is ejected within **13 seconds** (at most 2 seconds to the +first start, five further intervals and a 1-second final timeout). This is an +endpoint-observation bound, not a guarantee that every 500k-share rebuild meets +that budget. Measure the latter before sign-off; a longer ordinary rebuild must +lead to an explicitly reviewed threshold and revised bound. + +HTTP health is published every `PRISM_HEALTH_REFRESH_SECONDS` (2 seconds by +default in the binary, 5 in Compose). If publication stalls, the +HTTP handler rejects a snapshot older than `max(15, 3 * PRISM_HEALTH_REFRESH_SECONDS)` +seconds (default 15); `ready` alone need not reflect that rejection. With these +defaults, stale-publication detection plus LB hysteresis has a conservative +**28-second** bound. If that environment setting is customized outside Compose, +recompute the bound. Probe schedulers must honor the stated cadence, deadlines +and scheduling budget for either bound to hold. The Compose container check +remains its existing 5-second/3-retry diagnostic; it is not the operator TCP load balancer's +routing policy and Docker does not implement this TCP endpoint for the operator. + +Qualification: generate new tips and payout-revision changes under load, record +readiness transitions and routing decisions, confirm ordinary rebuilds stay +within budget, then hold work unavailable and confirm ejection by the stated +bound and re-entry after two successes. This is the #186 carry-over to #291. + +## Self-check live instances + +`qbit-prism-server self-check` adds `live_instances` to its existing JSON report. +It snapshots `qbit_prism_instances` through a **read-only connection using the +same resolved writer DSN**, before the existing self-check initializes its own +coordinator. That coordinator writes no heartbeat: the diagnostic is not a +frontend, so it never appears in its own sample, leaves no row behind when it +exits, and does not touch the row of a running frontend that shares its +`PRISM_INSTANCE_ID`. No new configuration reader, migration or HA election is +introduced. + +The server attempts to publish a heartbeat every **2 seconds**. `observed_at` is +one PostgreSQL `clock_timestamp()` sample through `PRISM_DATABASE_URL`; counts +describe that instant, before the rest of self-check runs. Ages and the fixed +**15-second inclusive** freshness window use that database clock. A fresh +`qbit.prism.audit-health.v1` row with boolean `ready` counts as live even when +`ready=false` during a rebuild. This is process liveness, not mining readiness +or a proof of distinct failure domains. Startup/stopped rows are listed as +inactive; they do not manufacture redundancy. Old rows remain listed as stale. +After a frontend has been stopped and its ID permanently retired, an operator +may run `DELETE FROM qbit_prism_instances WHERE instance_id = '<retired-id>';` +against the writer. Retirement is manual, never automatic; do not remove a +running frontend's row to hide a liveness problem. + +| Observation | `status` / `count` | Meaning | +| --- | --- | --- | +| Empty table | `empty` / `0` | Successful read, no instance observations. | +| Only old rows | `stale` / `0` | Heartbeats exist but exceed the window. | +| Only fresh startup/stopped rows | `inactive` / `0` | No observed running server heartbeat. | +| One fresh server row | `observed` / `1` | `single_instance=true` and an HA warning. A single instance must not present itself as HA. | +| Two fresh server rows | `observed` / `2` | IDs and rows listed; inspect readiness separately. | +| Future-dated or unrecognized fresh rows | `unknown` / `null` | Count is not asserted; inspect `unknown_instances`. | +| Failed query/connect, or 5-second read deadline exceeded | `failed` / `null` | No sample timestamp or instance list; self-check emits `ok=false` and exits unsuccessfully, without logging a credentialed DSN. | + +The report always uses the same `qbit.prism.self-check.v2` shape. A heartbeat +read failure does not skip the remaining local checks: the command completes +its report with `ok=false`, null unavailable fields, and a nonzero exit. Local +check failures also produce that shape and a nonzero exit; successful local +checks cannot mask a failed heartbeat read. `single_instance` is null when no +live rows are known (or the observation is unknown/failed), true for exactly one, +and false for two or more. + +Successful observations include `instance_ids`, `instances`, `stale_instances`, +`inactive_instances` and `unknown_instances`. Fewer than two known live rows +sets `ha_warning`. The existing top-level successful `ok` covers the command's +local checks; it is **not** an HA certification. [#412](https://github.com/Qbit-Org/qbit-mining-bootstrap/pull/412) +removed operator-command heartbeat registration, including the old transient +`starting` write. Self-check no longer manufactures or overwrites a frontend row. + +The `ledger::instances::live_instance_tests` unit tests exercise +empty/stale/fresh/boundary/future/startup rows and a failed connection. The SQL +test `ledger::instances::live_instance_tests::heartbeat_sql_observes_empty_stale_and_missing_table` +creates a connection-local temporary table and verifies an empty result, an old +heartbeat and a missing-table error without modifying deployment rows. Export +`PRISM_TEST_DATABASE_URL` for a **disposable** database, then run: + +```sh +( + set -eu + name=ledger::instances::live_instance_tests::heartbeat_sql_observes_empty_stale_and_missing_table + cargo test --locked -p qbit-prism-server --lib -- --list --exact "${name}" | + grep -Fqx "${name}: test" + out="$(PRISM_TEST_REQUIRE_INTEGRATION=1 cargo test --locked -p qbit-prism-server \ + --lib -- --exact "${name}" --nocapture 2>&1)" || { printf '%s\n' "${out}"; exit 1; } + printf '%s\n' "${out}" + printf '%s\n' "${out}" | grep -Fq 'test result: ok. 1 passed; 0 failed;' +) +``` + +Record the step as passed only when the subshell exits zero. It fails when the +name does not enumerate exactly that test, when the run does not report one +pass (a `0 passed` result is a failure), and, because +`PRISM_TEST_REQUIRE_INTEGRATION=1` turns the gate's skip into a failure, when +`PRISM_TEST_DATABASE_URL` is unset, empty or unreachable. + +For a full-command SQL failure probe, use a **disposable** database +and a DSN whose `options=-csearch_path=missing_ha_probe` hides the heartbeat +table; `self-check` must return `live_instances.status=failed`, `count=null`, +`observed_at=null` and a nonzero exit, not an empty or healthy cluster. The SQL +test samples directly through the reader; the operator command also leaves +frontend heartbeat registration untouched. + +## #281 acceptance evidence + +This matrix maps the six #281 acceptance items to landed evidence as of +`3.x.x` commit `31d20f464be9e465254b878253f56d9b69f92f66`. Test references describe +existing coverage and linked historical execution, not tests rerun by this +reconciliation. No live gate is completed by documentation or issue closure alone. + +| #281 acceptance item | Landed evidence | Remaining qualification / status | +| --- | --- | --- | +| Two healthy overlay coordinators; reconnect to either and accept the resumed share | [#304](https://github.com/Qbit-Org/qbit-mining-bootstrap/pull/304) (`984b3b44`) supplies the overlay; [#397](https://github.com/Qbit-Org/qbit-mining-bootstrap/pull/397) (`0cb9c233`) activates compact resume. [#273 closing evidence](https://github.com/Qbit-Org/qbit-mining-bootstrap/issues/273#issuecomment-5704427645) records PostgreSQL/runtime qualification; [`compact_runtime_e2e.rs`](../crates/qbit-prism-server/tests/compact_runtime_e2e.rs) includes `real_socket_reconnect_resumes_original_entropy_mask_and_submits_once`. | **Lab exercise automated; deployment open.** #487's lane L6 ([#544](https://github.com/Qbit-Org/qbit-mining-bootstrap/issues/544), `shipped-images` in [`prism-load-nightly.yml`](../.github/workflows/prism-load-nightly.yml), driven by [`prism_shipped_image_lane.py`](../scripts/prism_shipped_image_lane.py)) brings up this overlay from the shipped images weekly and on every Dockerfile or Compose change, issues work on frontend 1, has frontend 2 accept it once and refuse another worker and replays on either frontend, and checks the ledger credits it once. That runs on one bundled node and database with no load balancer; route/reconnect through the operator's load balancer on the production placement is still open. | +| Compose config validated in CI | #304; [CI's `Validate PRISM HA Compose stacks`](../.github/workflows/ci.yml) renders all four combinations and asserts distinct IDs/ports, one writer DSN and reachable health bindings. | **Landed.** Rendering does not establish healthy containers, RPC reachability or production placement. | +| Seven requirements, exact settings and provided/operator markings | #304 and the [requirements table](#seven-deployment-requirements), reconciled here with the approved D3 comments. | **Landed documentation.** Production provisioning and effective configuration still require verification. | +| Self-check lists live instance count and IDs | #304; [#362](https://github.com/Qbit-Org/qbit-mining-bootstrap/pull/362) consolidates the reader/heartbeat contract; #412 removes tool heartbeats. [`live_instance_tests`](../crates/qbit-prism-server/src/ledger/instances.rs), including `heartbeat_sql_observes_empty_stale_and_missing_table`; [#408](https://github.com/Qbit-Org/qbit-mining-bootstrap/pull/408) hardens the qualification command. | **Landed.** Capture two distinct fresh frontend IDs in the actual overlay drill; count alone is not proof of separate failure domains. | +| #291 failover drill on this overlay under D3 | Procedure above; [#333](https://github.com/Qbit-Org/qbit-mining-bootstrap/pull/333) (`e13051cf`) supplies commit reconciliation/guard coverage. [`postgres_failover.rs`](../crates/qbit-prism-server/tests/postgres_failover.rs) has `acknowledged_shares_survive_synchronous_primary_loss_and_pool_reconnect`, a disposable **synchronous** fixture. [`live_pg_failover.rs`](../crates/qbit-prism-server/tests/support/live_pg_failover.rs) (#521) runs this procedure against a disposable **asynchronous** D3 pair under real regtest mining with a found block mid-landing: a cut replication link loses exactly the unreplicated shares, counted and printed, and the held block lands once on the promoted primary, within 30 s of the promotion and with an `unknown` offer outcome (#529). [`live_pg_offer_standby.rs`](../crates/qbit-prism-server/tests/support/live_pg_offer_standby.rs) (#529) cuts replication at a found block's `submitblock` with the bounded standby flush wait on: the block survives promotion and lands once. | **Open:** actual async promotion/fencing/writer-endpoint drill on the overlay under mining load, measured replication gap/loss and promotion/recovery time, accounting reconciliation, dedicated-standby alert checks, and the qualified sync flip/back experiment. The disposable fixtures cannot qualify the real overlay, fencing or endpoint. | +| LB preserves ordinary rebuilds and ejects prolonged unavailability within the stated bound | #304 documents readiness/hysteresis; [#331](https://github.com/Qbit-Org/qbit-mining-bootstrap/pull/331) (`2914a629`) supplies the [deterministic simulator](prism-ha-readiness-probe-harness.md) and [`test_prism_ha_readiness_probe.py`](../tests/test_prism_ha_readiness_probe.py). | **Open:** real operator TCP load balancer under tip/revision churn and prolonged unavailability; record routing, ejection and recovery timings. The simulator neither deploys nor qualifies that load balancer. | + +#291 must report observed acknowledged-share loss (including zero if observed), +the unreplicated interval and reconciliation under **asynchronous D3**. Historical +zero-loss wording is not a guarantee supplied by this policy. Runtime capability, +synthetic tests and rendered Compose do not establish zero outage or cutover readiness. + +## Cutover checklist + +- [ ] Verify and record the approved async D3 settings, distinct HA/public replica + identities/slots, standby-down ACK behavior and the interval after promotion + before a replacement is caught up; qualify dedicated-standby alert observations. +- [ ] Deploy independent nodes, frontends and database storage/failure domains; + validate both effective configurations and the stable writer/public endpoints. +- [ ] Confirm the operator TCP load balancer implements the per-frontend probe contract and + measured hysteresis; verify two distinct fresh heartbeat IDs. +- [ ] Run the addendum's optional sync flip/back qualification with ACK latency and + throughput evidence; retain both strict-mode preconditions and exercise timeout, + non-timeout cancellation, block-only and ambiguous-commit cases. +- [ ] **Exercise database promotion under mining load before production traffic**: + fence the old primary, promote, move the writer endpoint, verify acknowledged + share accounting and both frontend recovery, then rejoin the one standby. +- [ ] Complete #291 using D1's separate dimensions: 500k-share headroom (400k + regression), 2,000 shares/s for a minute, 500/s for five minutes, 2,000 sessions + and dense block cadence; public read load still needs measurement. Do not infer + one dimension from another. +- [ ] Verify real-overlay cross-frontend job resume using the landed #273 runtime; + record the actual failover drill results and async loss/reconciliation evidence. diff --git a/docs/prism-integration-test-gate.md b/docs/prism-integration-test-gate.md new file mode 100644 index 000000000..2ac9f7e42 --- /dev/null +++ b/docs/prism-integration-test-gate.md @@ -0,0 +1,245 @@ +# Integration test gate + +Every Rust test whose execution depends on an environment input takes that +input through one gate, the `qbit-prism-test-gate` crate in +`crates/qbit-prism-test-gate`. The gate applies one decision table, records +what it decided in an execution manifest, and prints a skip line with a fixed +prefix when it lets a test return early. This page is the contract: the +switch, the manifest, the expected list, and how to add a gated test. Issue +#286 introduced it after 59 native tests were found to report `passed` +without running. + +## Inputs + +| variable | what it carries | tests that need it | +| --- | --- | --- | +| `PRISM_TEST_DATABASE_URL` | a disposable PostgreSQL the test may write to (each test creates and drops its own schema; `api_2x`, `ledger_postgres`, `audit_body_normalization`, `ledger_deadline_atomicity`, `candidate_window_migration`, `ledger_single_execution`, `readiness_rpc`, `window_read_oracle` and `window_reference` create and drop a whole database per fixture instead, because PostgreSQL scopes advisory locks to a database, so their role needs `CREATEDB` and the URL must name a non-template database such as `postgres`) | ledger, API, rollups, readiness, authorization, migration, JSONB ceiling, window oracle, candidate lease, the D2 payout rules, the explicit `#[ignore]` runs, and the live suite | +| `PRISM_TEST_PG_BIN_DIR` | a directory holding `initdb`, `pg_ctl` and `pg_basebackup` | the physical failover and public read replica tests, which run their own clusters | +| `QBITD_BIN` | a `qbitd` executable | the live regtest suite (`live_regtest`, with its high-difficulty, CTV/CPFP, weighted-recipient and two-node modules) | + +An empty or whitespace-only value counts as unset. A value that is not valid +Unicode is an error, never "absent": treating it as unset would let a +deliberately configured run skip a test without a word. + +## The decision table + +| input set and non-empty | switch | result | +| --- | --- | --- | +| yes | any | run | +| no | `PRISM_TEST_REQUIRE_INTEGRATION=1` | fail, naming the missing variable and the test | +| no | `GITHUB_JOB=prism-native-postgres` | fail, same | +| no | neither | print one skip line and return | + +`PRISM_TEST_REQUIRE_INTEGRATION` accepts `1` (require), `0` or unset (do not +require); any other value is an error, so a typo cannot pass silently. The +switch is consulted before the job id, so a run that sets both is reported as +demanded by the switch. + +`GITHUB_JOB` is set by GitHub to the running job's id. Matching it means a +database outage in the `prism-native-postgres` job surfaces as a failure even +if the job's own environment were edited. Keying on `CI` would be wrong: +GitHub sets `CI=true` in every job, including `rust-tests`, which runs the +whole workspace with no database. + +Explicit `#[ignore]` runs are always required. The native job selects ten +`#[ignore]` gated tests in three targets with `--ignored`: the 10,000-connection +admission test in `stratum_admission_postgres` (by `--exact` name), both tests +in `observability_database`, and all seven issued/prepared job dependency +contracts in `issued_job_dependency` (with `--test-threads=2`). They use the +gate's `required_*` entry points, which never skip: a missing input fails them +whatever the switch says, since a vacuous pass is exactly what selecting them +explicitly tried to avoid. All ten are in the expected list. Measurement runs, +such as the full-size ratchet and baseline sweep in `jsonb_ceiling_gate`, stay +`#[ignore]` and opt-in; CI never selects them, so they are not listed. The +130-recipient nightly case of `live_regtest`'s weighted-recipient module is +opt-in the same way until a nightly job selects it. + +The nightly `live-nightly` job in `prism-load-nightly.yml` runs the +`live_regtest` `#[ignore]` variants listed in +[test/prism-nightly-gated-tests.txt](../test/prism-nightly-gated-tests.txt) +with `--ignored --exact`, and proves each executed with the same checker. Two +of them need host tools besides the gate's inputs, found on `PATH` and failing +the test when absent: the disk-exhaustion variant (#575) mounts a small ext4 +image without root, so it needs `fuse2fs`, `mkfs.ext4`, `fusermount3` and +`/dev/fuse`; the clock-jump variant runs the servers and its own PostgreSQL +under libfaketime, found next to the `faketime` wrapper (the `faketime` and +`libfaketime` packages). Its private clusters use `PRISM_TEST_PG_BIN_DIR`. + +The table is a pure function, `qbit_prism_test_gate::decide`, over injected +values, with unit tests for every row in the crate itself. + +## The skip line + +A skipped test prints exactly one line on the process's real standard error, +past libtest's capture, so it is visible with or without `--nocapture`: + +```text +[prism-test-gate] skipped <package>::<binary>::<test path>: PRISM_TEST_DATABASE_URL is unset or empty; set PRISM_TEST_DATABASE_URL to run it, or PRISM_TEST_REQUIRE_INTEGRATION=1 to fail instead +``` + +The prefix `[prism-test-gate] skipped` is fixed. `scripts/check_gate_manifest.py` +fails the native job if it appears in the job's log, and also if any of the +per-file skip messages the gate replaced reappears (`skipping ... set +PRISM_TEST_DATABASE_URL`, `PRISM_TEST_DATABASE_URL not set`, `set +PRISM_TEST_DATABASE_URL for ...`, `SKIPPED: jsonb_ceiling_gate`). + +## Test identity + +A gated test is identified as `<package>::<binary>::<test path>`: + +- `<package>` is the Cargo package, for example `qbit-prism-server`; +- `<binary>` is the test binary, the file stem of an integration test + (`ledger_postgres`) or the crate name for `#[cfg(test)]` modules in the + library (`qbit_prism_server`); +- `<test path>` is the test's module path and name inside that binary, as + `cargo test` prints it (`two_x::legacy_2x_upgrade_repairs_partial_carry_seed_and_preserves_shared_state`). + +The first two come from the `gate::site!()` macro at the call site. The third +is the name libtest gives the thread it runs each test on, which is why the +gate must be called from the test's own thread: at the top of the test body, +or from a fixture helper the test body calls before spawning anything. A +call from an unnamed or runtime worker thread is an error that names the +thread. + +## The manifest + +When `PRISM_TEST_GATE_MANIFEST=<path>` is set, every decision appends one +line to that file: + +```text +executed qbit-prism-server::ledger_postgres::candidate_renewal_requires_a_live_pending_token +skipped qbit-prism-server::live_regtest::real_two_server_mining_failover_audit_and_reorg +failed qbit-prism-server::postgres_failover::acknowledged_shares_survive_synchronous_primary_loss_and_pool_reconnect +``` + +Each line is one `O_APPEND` write, so parallel test threads and the several +test binaries of one `cargo test` run share the file safely. A test that +asks the gate more than once (a fixture opened in a loop) writes more than +one line; the checker counts it once. An unwritable manifest fails the test. + +## The expected list and the proof + +[test/prism-gated-tests.txt](../test/prism-gated-tests.txt) lists every gated +test the `prism-native-postgres` job must execute, one id per line, sorted. +The number of test ids in that file is the minimum count; see the file for +the current list and the `check-gate-manifest` success line from +[scripts/check_gate_manifest.py](../scripts/check_gate_manifest.py) for the +count verified in a green run. + +The job has four shards, each with its own PostgreSQL service and qbitd. +[scripts/run_rust_test_shard.py](../scripts/run_rust_test_shard.py) discovers +all workspace targets from Cargo metadata and assigns them longest first, +each to the shard with the least estimated time so far. The estimates are +measured durations for the few heavy targets and 20 seconds for the rest; each +shard's job summary lists its per-command durations for the next re-measure. +Each test binary stays intact, preserving its fixtures and process-wide locks. +New workspace targets join automatically. +The three explicit `--ignored` targets run only on the shards owning them. +Use `--shard-index 0 --shard-count 4 --dry-run` to inspect a shard's commands. + +Each shard's PostgreSQL service keeps its data directory on a tmpfs, so a +busy disk on the shared runner host cannot stall its commits; `fsync`, +`full_page_writes` and `synchronous_commit` stay on. The disposable clusters +some tests start from `PRISM_TEST_PG_BIN_DIR` stay on disk. The service logs +every statement that runs over a second and every lock wait over a second, and +the runner prints that log when the job ends. The job sets +`PRISM_DATABASE_STATEMENT_TIMEOUT_MS=60000` for the ledgers the tests open in +process: production keeps its 15 s default, but these 2 vCPU runners take 1 to +4 s for the heaviest statements and, in a slow spell, more than 15 s. The +servers and operator tools the tests spawn drop inherited `PRISM_` settings and +keep the production default. `authorization_rpc` re-runs its own test binary in +a cleared environment and forwards the setting to it. Tests of timeout +behaviour set their own values. + +Each shard uploads its manifest and test log as `prism-gate-shard-<index>`. +The `prism-integration-proof` job requires every shard to succeed, downloads +and combines their manifests and logs, then runs + +```sh +python3 scripts/check_gate_manifest.py \ + --manifest prism-gate-manifest.txt \ + --expected test/prism-gated-tests.txt \ + --log prism-native-tests.log +``` + +which fails when any expected test has no `executed` line, any `skipped` or +`failed` line appears, fewer distinct tests executed than the list holds, a +test executed that is not in the list, or the log shows a skip line. It +prints the manifest to the log and to the job summary, and the job uploads +the combined manifest as the `prism-gate-manifest` artifact. Every shard sets +`PRISM_TEST_REQUIRE_INTEGRATION=1`, so an input that goes missing fails the +tests themselves before the checker runs. + +`rust-tests` runs the same workspace with no inputs, where every gated test +prints its skip line and passes; nothing there asserts execution, which is +why both the native matrix and its combined proof are in the required +`checks` list; `rust-tests` is not the proof. + +## Running locally + +The public-reader credential tests require PostgreSQL to reject missing or +incorrect passwords for their temporary reader roles. A supplied +`PRISM_TEST_DATABASE_URL` must point to a disposable cluster that enforces +password authentication for those roles; an unrestricted TCP `trust` rule +invalidates these tests. The runner configures this for clusters it creates, +while allowing its bootstrap role to connect without a password for setup. + +`test/prism-native-tests.sh` provisions a private PostgreSQL cluster (or uses +`PRISM_TEST_DATABASE_URL`), resolves `PRISM_TEST_PG_BIN_DIR` and `QBITD_BIN`, +and exports the switch once every input a mode needs is present, so a local +run is as non-vacuous as CI: + +| invocation | what runs | required mode | +| --- | --- | --- | +| `make test-prism-postgres` (`test/prism-native-tests.sh`) | the whole workspace, the three explicit `--ignored` targets, then the manifest check | when `qbitd` and the PostgreSQL server tools are found; otherwise the manifest is printed and the run says which input is missing | +| `make test-prism-regtest` (`... live`) | `live_regtest` | always (the mode refuses to start without `qbitd`) | +| `make test-prism-public-read-replica` (`... replica`) | `postgres_failover` | always | +| `test/prism-native-tests.sh cargo-args <args>` | the `cargo test` you name | always | + +Without the script, a bare `cargo test --workspace --all-targets` prints one +skip line per gated test and passes. Set `PRISM_TEST_REQUIRE_INTEGRATION=1` +to turn those skips into failures, for example to prove that a test really +depends on its input. + +## Adding a gated test + +1. Take the input through the gate at the top of the test body, or in the + fixture helper the body calls first: + + ```rust + use qbit_prism_test_gate as gate; + + #[tokio::test] + async fn my_ledger_contract() -> anyhow::Result<()> { + let Some(url) = gate::database_url(gate::site!())? else { + return Ok(()); + }; + // ... + } + ``` + + `gate::pg_bin_dir`, `gate::qbitd_and_database_url` and the general + `gate::inputs(site, &[Input::...])` cover the other inputs; + `gate::required_database_url` is for explicitly selected `#[ignore]` tests. + Tests that return `()` call `.expect("integration gate")` instead of `?`. +2. Never read the variables directly. `scripts/check_gate_env_reads.py`, run + in the lint job, fails on any Rust source line outside the gate crate that + holds one of the variable names as a string literal. +3. Add the test's id to `test/prism-gated-tests.txt`, keeping the file + sorted. The id is what the gate writes; the easiest way to get it is to + run the test with `PRISM_TEST_GATE_MANIFEST` set and copy the line. A + gated test that runs without being listed fails the native job, so the + list cannot silently drift. +4. If the test lives under `crates/qbit-prism`, nothing else changes: the + native job runs the whole workspace, and the crate already has the gate + as a dev-dependency. +5. Renaming or removing a gated test means changing the list too. + +## Related + +- [prism-deleted-test-map.md](prism-deleted-test-map.md): every 2.x.x test + file absent on 3.x.x, with its native replacement or the issue that closes + the gap. +- `crates/qbit-prism-test-gate/src/lib.rs`: the gate, its decision table and + its unit tests; `crates/qbit-prism-test-gate/tests/process.rs` drives the + runtime path through a child process. diff --git a/docs/prism-ledger-ops.md b/docs/prism-ledger-ops.md index 21c50d1f1..0c9ac00aa 100644 --- a/docs/prism-ledger-ops.md +++ b/docs/prism-ledger-ops.md @@ -1,869 +1,3873 @@ -# PRISM Ledger Operations Contract - -This contract is the public operating model for the qbit PRISM share ledger. -PRISM means Payouts, Rewards, and Integrity Settlement Manifest. -It is intentionally narrower than a full production deployment guide: it -defines the invariants the implementation and regtest evidence rely on, and it -names the hardening layers that can be added without changing reward semantics. - -## Canonical Write Path - -Accepted shares must enter exactly one ordered log: `qbit_share_ledger`. -Stratum frontends may scale horizontally, but they do not insert independent -share sequences. The supported topology is: - -1. Stratum frontend validates miner identity, share shape, job id, and target. -2. Frontend submits the accepted share to the bounded group-commit writer and - waits for Postgres to commit it. -3. One logical ledger writer owns the Postgres writer lease and inserts shares - into `qbit_share_ledger`. -4. PRISM's TIDES-style reward windows, audit exports, payout policy, and reorg - reversal read from that same canonical log. - -The coordinator runs the frontend and writer in one process. Every accepted -share receives one monotonic `share_seq`, and all reward windows are derived -from that ordering. Stratum success is sent only after the transaction commits; -worker counters and vardiff accounting advance at the same boundary. A full -queue, commit error, or commit timeout returns no success. Miners may retry the -same submission safely. - -The writer batches up to `PRISM_SHARE_COMMIT_BATCH_SIZE` shares for at most -`PRISM_SHARE_COMMIT_LINGER_MILLISECONDS` before committing. The defaults are 64 -shares and 5 ms. `PRISM_SHARE_COMMIT_TIMEOUT_SECONDS` bounds queue admission. -Once admitted, a client waits for a definite commit outcome; the watchdog -restarts a wedged writer instead of returning an ambiguous timeout. Tune the linger against measured ACK -latency, but do not weaken Postgres durability settings to reduce it. A batch -flushes immediately when it contains a block candidate so the normal linger -does not consume that candidate's tip-race budget. - -## Writer Lease and Replay - -The writer lease is stored in `qbit_ledger_writer_lease`. A writer is identified -by `(writer_id, writer_epoch, writer_session_token)`. A process may refresh only -the exact session token it acquired; another process with the same writer id and -epoch is still fenced until the existing lease expires. During startup, a -replacement process with the same writer id and epoch waits and retries until -that predecessor lease expires, then acquires a fresh session token. A different -writer id or epoch is treated as a conflicting active writer and fails fast. - -`share_id` is globally unique. Replaying an exact share payload is idempotent -and returns its original sequence without inserting another row. Reusing the ID -with any different payout, difficulty, job, timestamp, nonce, or credit policy -fails the complete batch. After failover, the replacement writer resumes at the -next database sequence value and stale writers are rejected before insert. - -The ledger is single-writer, not active-active. A replacement writer takes the -lease after expiry. Active-active insertion would create ambiguous ordering and -is outside the accepted PRISM contract. - -Graceful coordinator shutdown closes Stratum admission and all background -writer admission first, then waits for admitted share batches, accepted-block -finalization, CTV status updates, and payout/reorg mutations to finish. It -releases the exact-session writer lease immediately after that writer barrier; -client socket delivery, obsolete job fanout, and executor/thread cleanup drain -after release and therefore cannot delay a replacement writer. SIGTERM only -closes admission and wakes the serve loop; the barrier and database work run -outside the signal handler. - -`PRISM_WRITER_QUIESCENCE_TIMEOUT_SECONDS` bounds the writer barrier and defaults -to 15 seconds. This leaves time for conservative synchronous Postgres flushing -while remaining well below the default 60-second lease TTL. If the timeout -expires, the coordinator logs each still-active writer component and -deliberately does not release the lease; process termination and TTL fencing -then preserve the single-writer invariant. Do not shorten this below the -durable flush time of the deployed Postgres system. - -Shutdown emits structured JSON log events named `shutdown_start`, -`writer_quiescence`, `lease_release_attempt`, `lease_release`, -`lease_release_withheld`, and `non_writer_drain`. Prometheus exposes the same -path through `qbit_prism_shutdowns_total`, -`qbit_prism_shutdown_writer_quiescence_seconds`, -`qbit_prism_shutdown_writer_quiescence_total`, -`qbit_prism_shutdown_lease_release_attempts_total`, -`qbit_prism_shutdown_lease_release_total`, -`qbit_prism_shutdown_lease_release_seconds`, -`qbit_prism_shutdown_sigterm_to_lease_release_seconds`, -`qbit_prism_shutdown_non_writer_drain_seconds`, and -`qbit_prism_shutdown_release_withheld_total`. - -### Heartbeat Monitor Lateness and In-Process Stalls - -The writer-lease heartbeat's timing policy (`lab/prism/writer_lease_timing.py`) -guarantees that a coordinator whose guard session has died hard-exits before a -replacement may adopt the lease, *provided* the monitor thread that takes that -decision wakes no later than `scheduler_slack + monitor_interval` after its -poll is due (0.55 s at the shipped numbers; the module docstring derives this -as inequality (4)). That proviso is an assumption about the process, not a -property of the policy. Issue #227 recorded a 0.648 s monitor wake delay on -`union-mainnet` with the host idle: the stall was in-process, an interpreter -held by a multi-second payout-window rescan. - -Two consequences are exported and one is decided. - -**Attribution that survives GIL stalls.** Every guard statement returns its own -server-side execution time, so `guard_sql` in -`qbit_prism_lease_heartbeat_phase_seconds{phase}` and its `worst_` twin is now -server execution, the new `guard_client_resume` phase is the part of the round -trip during which PostgreSQL had already answered and this process had not -resumed, and `scheduler_delay` is the whole process-side residual -(`guard_slot_wait + guard_sql + scheduler_delay = total`; `guard_client_resume` -is the in-round-trip share of `scheduler_delay`). `guard_sql` is now bounded by the -server rather than by the client: each statement can run for at most the 0.5 s -server-enforced statement timeout, and a renewing verification may lawfully run -two statements inside one guard slot (the attribution recheck), so `guard_sql` -sums both and can legitimately reach about 1.0 s on a two-statement attempt -(`WRITER_LEASE_VERIFICATION_MAX_STATEMENTS` × 0.5 s) and cannot exceed that; the -`(N stmt)` count in the attempt summary says which case applies. Read a large -`guard_client_resume` as "this process, not PostgreSQL". - -**The monitor's lateness as re-armable signals.** -`qbit_prism_lease_heartbeat_monitor_wake_delay_seconds` stays the lifetime -high-water mark for dashboard continuity. Alert on the new families instead, -all fixed-cardinality: - -- `qbit_prism_lease_heartbeat_monitor_wake_lateness_seconds` (histogram; every - poll's lateness, closed bucket set); -- `qbit_prism_lease_heartbeat_monitor_late_wakes_total{slack_fraction}` with - `slack_fraction` in `0.5`, `0.8`, `1.0` (wakes at least that fraction of - `scheduler_slack` late; a counter, so `increase(...[5m]) > 0` on the `1.0` - series re-expresses the wake-delay alert without a never-resetting gauge); -- `qbit_prism_lease_heartbeat_monitor_wake_delay_window_max_seconds` (the worst - lateness in the trailing 300 s; falls back to zero once a stall ages out); -- `qbit_prism_lease_heartbeat_monitor_wake_delay_record_age_seconds` (how old - the lifetime record is, so an exit message's figure can be dated); -- `qbit_prism_lease_heartbeat_monitor_exit_guarantee_breaches_total` and - `qbit_prism_lease_heartbeat_monitor_worst_exit_guarantee_overrun_seconds` - (wakes later than `max_guaranteed_monitor_lateness`, and the worst amount - by which the exit could have landed after the adoption edge); -- `qbit_prism_lease_heartbeat_policy_seconds{term="max_guaranteed_monitor_lateness"}` - (the bound itself, for alert expressions); -- `qbit_prism_lease_heartbeat_stall_probe_samples_total` and - `..._stall_probe_suppressed_total` (stack samples the stall probe took, and - triggers it refused under its rate limit); -- `qbit_prism_process_gc_pause_seconds{generation}` (histogram), - `qbit_prism_process_gc_last_pause_seconds{generation}` and - `qbit_prism_process_gc_max_pause_seconds{generation}`: cyclic-collector - pause *durations* from `gc.callbacks`. These complement, and do not replace, - the `qbit_prism_process_gc_*` count families from issue #226, which report - how many passes ran and what they freed but not how long any pass held the - interpreter. - -**What a breach means and the decided response.** A wake later than -`scheduler_slack` (0.50 s) is a *lateness-beyond-slack breach*: the assumption -the policy budgets was exceeded. A wake later than -`max_guaranteed_monitor_lateness` (0.55 s) is an *exit-guarantee breach*: on -that beat, had the guard session died, this process could have outlived the -successor's adoption edge by the overrun. The monitor does **not** tighten its -cap or hard-exit on a breach (that would restart healthy coordinators under a -transient stall, issue #212's failure mode, without moving the exit earlier on -the beat that breached). It accepts the residual because every effect that -could escape is fenced: every ledger write joins the exact-session lease CTE -and matches nothing once adoption rewrites the session token, and every -external effect runs behind a synchronous exact-session verification that -hard-exits the moment the row names another session. A breach is therefore -counted, its overrun kept, and one structured warning is logged -(`prism coordinator: WARNING ledger lease heartbeat monitor wake delay ... exceeded -the ... scheduler slack`), rate-limited to at most 3 per 60 s and never from a -blocking write. A wake at least half a slack late additionally takes a stack -sample of the running threads, under the same 3-per-60 s limit -(`LEASE_MONITOR_STALL_PROBE_MAX_SAMPLES_PER_WINDOW` / -`LEASE_MONITOR_STALL_PROBE_WINDOW_SECONDS` in `writer_lease_timing.py`), so the -instrument cannot turn a stall into a stall storm. - -**Operator action on a lateness-beyond-slack warning.** - -1. Read the warning: it states whether exit-before-adoption still held, the - overrun if not, the phase attribution of the last and worst attempts, the - worst GC pause and its generation, and the sampled thread stacks. The - stacks name the frame that held the interpreter; a payout-window rescan - (`reconcile_invalidation`) or a large collection are the known causes. -2. Confirm it was in-process rather than host contention: PSI, steal, and - cgroup throttling on the host should be quiet. If they are not, the host is - the problem and the policy's slack is not the lever. -3. Check `qbit_prism_lease_heartbeat_monitor_exit_guarantee_breaches_total`. - An isolated breach with no coincident guard loss changed nothing; the - fences carried it. Repeated breaches mean the workload has outgrown the - interpreter's ability to schedule the monitor, and the durable fix is the - out-of-process watchdog tracked in issue #130 — not a larger slack, which - only lengthens failover, and not a shorter cap, which manufactures issue - #212. -4. Reduce the stall source: if the rescan attribution - (`qbit_prism_payout_window_full_rescan_seconds`) shows multi-second full - rescans, that is the workload to bound. If the worst GC pause is the stall, - the heap growth tracked by the issue #226 families is the lead. -5. Do not disable the heartbeat, do not raise only the failure budget, and do - not weaken PostgreSQL durability. - -### Orphaned Locks and Bounded Startup Acquisition - -A coordinator that vanishes without closing its sockets — network partition, -`SIGSTOP`, VM pause — can leave a Postgres backend idle in transaction, still -holding the `qbit_ledger_writer_lease` row lock its landing CTE took. Without -countermeasures the successor's startup lease upsert queues behind that lock, -inside ledger construction and before the watchdog arms, until kernel TCP -keepalive teardown (hours at OS defaults): a full-pool availability outage. -Settlement correctness is unaffected — the outbox row stays pending and -replays — only availability is at stake. - -Two independent layers bound this. First, every Postgres session the -coordinator opens (the pooled native client, the dedicated lease-guard -session, and the psql subprocess backend) carries session guards: -`idle_in_transaction_session_timeout` -(`PRISM_POSTGRES_IDLE_IN_TRANSACTION_TIMEOUT_SECONDS`, default 15) makes the -server abort an orphaned transaction and release its locks, and the -server-side keepalive GUCs (`PRISM_POSTGRES_TCP_KEEPALIVES_IDLE_SECONDS`, -`PRISM_POSTGRES_TCP_KEEPALIVES_INTERVAL_SECONDS`, -`PRISM_POSTGRES_TCP_KEEPALIVES_COUNT`, defaults 30/10/3) bound the server's -teardown of a socket toward a vanished client at 30 + 3x10 = 60 seconds, the -backstop where the idle-in-transaction timer does not apply. - -Second, the startup lease upsert and adoption CAS each run under a bounded -lock deadline (`PRISM_LEDGER_LEASE_ACQUIRE_LOCK_TIMEOUT_SECONDS`, default 5) -with a bounded retry (`PRISM_LEDGER_LEASE_ACQUIRE_ATTEMPTS`, default 5). Each -timed-out attempt is logged with its attempt number and the underlying error. -The retry budget (5x5s = 25s) is sized to outlast the idle-in-transaction -timeout — which runs on the blocking backend's own clock, from when its -transaction went idle rather than from when the successor started retrying — -so an orphaned lock is normally reaped mid-budget and startup self-heals -without operator action. An acquisition that never completes within the budget -fails construction with a `RuntimeError`, exiting the process visibly for the -supervisor to restart. That error names the lock conflict as the likeliest -cause but does not assert it: a connect timeout, an exhausted connection-pool -slot, and a server that is merely overloaded all expire the same deadline, so -read the chained cause it quotes before assuming a stuck transaction. Waiting -for a *live* holder's lease TTL to expire is unchanged — that outer wait is -intended failover behaviour; only the per-statement lock wait is bounded. - -Arming that deadline widens the orphan shape slightly, and the trade is -deliberate. A deadline is what makes the native client wrap a statement in an -explicit transaction whose `COMMIT` is a separate client message, so the -startup acquire is now transaction-scoped where it used to be plain -autocommit. A coordinator that vanishes mid-acquire can therefore orphan the -lease-row lock itself — on a path that previously could not produce one, -because autocommit leaves no transaction for the server to hold open. The two -orphans are not comparable in cost: the one this fix removes was bounded only -by TCP keepalive teardown, hours at OS defaults, while the one it introduces -is bounded by `PRISM_POSTGRES_IDLE_IN_TRANSACTION_TIMEOUT_SECONDS` (default -15) and delays a successor by at most that. It is also why the session guard -is not optional and why disarming it is rejected at construction: without it -the acquisition deadline would trade an unbounded wait for an unbounded wait -of its own making. - -The session guards carry three deployment caveats. The guards travel as -libpq startup options, so native connections now always set the `options` -connect parameter: a deployment routing `PRISM_DATABASE_URL` through a -connection pooler that rejects startup options (older PgBouncer builds) will -fail at connect rather than silently drop them, visibly at coordinator -startup. The default compose topology connects directly to `prism-postgres` -and is unaffected. The psql-subprocess backend delivers the same guards -through `PGOPTIONS`, so a wrapper script standing in for `psql` -(`PRISM_POSTGRES_PSQL_COMMAND`) that does not forward its environment drops -them without any error. On a Unix-socket DSN the `tcp_keepalives_*` GUCs are -ignored — accepted and inert — leaving -`PRISM_POSTGRES_IDLE_IN_TRANSACTION_TIMEOUT_SECONDS` as the guard that still -applies; it is the one that covers the orphaned lease-row lock in any case. - -## Block Candidate Outbox - -A block-worthy share transaction also inserts an immutable intent into -`qbit_block_candidate_outbox`. The intent contains the complete block, template -context, reward inputs, and extranonce fields required to finish audit and -submission. The share and intent become visible atomically before Stratum -success. - -The bounded live-candidate queue is only a wakeup path. Queue saturation -coalesces wakeups; it cannot delete an outbox row. Recovery restores pending -rows in batches into a separate, lower-priority replay queue, without doing -per-row database accounting. Live discoveries therefore always outrank restart -work, while an older replay stalled in accounting cannot hide later durable -rows. The pre-accept startup recovery pass is best-effort under a slow ledger: -if its database budget expires, the coordinator finishes starting and the -block-submitter loop retries every durable pending row with ordinary backoff. -Because job builds stay blocked until every pending candidate is known, the -startup enumeration must be provably untruncated: a full batch re-queries -with a doubled window (capped at 1024 rows) until the outbox returns fewer -rows than requested. If the cap is ever hit, the gate stays closed while the -restored batch drains and the submitter loop re-enumerates the remainder. -Before qbitd can observe a candidate, the coordinator installs a short in-memory -prospective-payout barrier; this prevents startup prewarm from issuing child -work from the old balance base without falsely claiming that the block landed. - -Once a durable candidate is dequeued, its qbit `submitblock` RPC is the fast -lane: it runs before the attempt-marker write, accepted-block writer admission, -audit construction, or payout publication. The node result and same-hash lease -then transfer to an independent, height-prioritized accounting lane. A full -primary handoff spills to a result-preserving overflow queue; it never turns an -already-offered block back into a raw-submit retry. `block_submitter` and -`block_accounting` expose independent phase heartbeats, so slow accounting does -not delay later node offers or disguise the phase that stopped progressing. - -`PRISM_BLOCK_SUBMIT_RPC_TIMEOUT_SECONDS` bounds the fast-lane RPC (default 1 -second). `PRISM_BLOCK_SUBMIT_DB_TIMEOUT_SECONDS` gives each later Postgres -statement and local ledger gate a fresh deadline (default 1 second); direct -outbox reads and mutations additionally use a single-flight wrapper so a -driver that ignores its deadline cannot accumulate retry threads. Timeouts -leave the row pending and enter the ordinary candidate backoff. - -Landing-path observability lives on `/metrics`: -`qbit_prism_block_ledger_calls_total` / `_call_timeouts_total` / -`_call_budget_seconds` / `_call_last_duration_seconds` / -`_call_max_duration_seconds` (labelled by `call_class`, `fast` vs -`landing`), `qbit_prism_accepted_parent_unresolved_transitions` and -`_unresolved_oldest_seconds`, -`qbit_prism_accepted_parent_preview_wait_timeouts_total`, -`qbit_prism_prior_balances_reads_total` / `_read_last_seconds` / -`_read_max_seconds`, and `qbit_prism_startup_phase_seconds{phase=...}`. - -Read-slot ledger operations additionally split local admission from server -execution, labelled by `operation`: -`qbit_prism_ledger_read_calls_total`, -`qbit_prism_ledger_read_gate_wait_seconds_total` / `_max` and -`qbit_prism_ledger_read_gate_timeouts_total` (coordinator-local admission — -waiting for `PRISM_POSTGRES_READ_CONCURRENCY`; a gate timeout means no -statement was ever sent), against -`qbit_prism_ledger_read_execute_seconds_total` / `_max` and -`qbit_prism_ledger_read_execute_timeouts_total` (PostgreSQL, including the -tail a cancelled statement spends returning). One duration covering both is -what made the #211 exhaustion unattributable from a scrape: the replay -enumeration reported `exceeded 5s` while its statement deadline was barely -touched and the database showed no blocked backends. Read the two halves -before widening any budget — a rising gate series is contention in this -process, a rising execute series is the database. - -Since #224 the same split covers the payout-window and prior-balances reads -under fixed operation names: `payout_window_snapshot`, -`payout_window_delta` and `prior_balances_after_pool_block` on the read -slot, and `current_prior_balances` on the **writer lock**, where the -landing's prior-balances check has always taken it. The `operation` label -is closed (`PRISM_LEDGER_READ_OPERATIONS` in -`lab/prism/accepted_preview_telemetry.py`); any other name folds into -`other`. The operator validation contract for these series, the landing -phase / reconcile caller / full-rescan families, and the 4 s and 5 s -accepted-preview boundaries is in `docs/prism-overload-alerts.md`, -"Issue #224". - -Pending block-candidate enumeration -(`pending_block_candidate_rows`, the `replay-outbox-query` phase) is a -read-only single-snapshot statement and takes the bounded read slot, not the -writer lock (#211). It is therefore unaffected by an accounting write holding -the writer gate. The page it returns is advisory: a candidate can land or be -terminalized between the snapshot and the caller's decision, so every terminal -transition re-checks under the writer lease — `mark_block_candidates_abandoned` -re-asks `qbit_pool_blocks` inside its fenced `UPDATE`, and the single-hash -terminal updates additionally require `state = 'pending'`. - -`PRISM_BLOCK_REPLAY_PAGE_SIZE` controls the number of candidate payloads in -each cursor page during startup and forced ancestor re-drive (default 1024, -allowed range 1–1024). Set it to `1` to reduce combined JSON decoding pauses -when pending candidates contain large payout windows (issue #255). The walk -still visits every page and requires a short page, including an empty final -page when necessary, before declaring enumeration complete. Timeouts, missing -cursors, and cleanup backpressure still leave the job-build gate closed. - -This is an opt-in mitigation, not a byte or memory bound: one candidate can -still exceed the lease timing envelope, and restored candidates remain queued -for accounting. Smaller pages add database and chain-RPC round trips. Ordinary -steady-state polls and the legacy cursorless fallback retain their existing -batch limits. Neither lease protection nor accounting rules change. After -deploying with the setting, check enumeration logs for `limit=1`, completion -of pending accepted-block accounting, and full subscribe/authorize/notify -health on both Stratum listeners. Restore `1024` to undo the tuning. - -For already accepted blocks that still cannot complete during normal replay, -`python3 -m lab.prism.recover_pending_blocks` provides a temporary offline -recovery command. Run it from an image containing the command, using the -managed coordinator's environment, signing keys, network and audit volumes. -PostgreSQL and qbitd must remain available. - -1. Inspect the exact hashes with a metadata-only plan: +# PRISM Ledger Operations + +This is the operating contract for the native Rust Prism server. The database +retains the original accounting tables and functions while additive migrations +provide active-active application instances. For the upgrade procedure, see +[Rust migration](prism-rust-migration.md). + +## Share commit and ordering + +All instances insert accepted shares into `qbit_share_ledger` in one PostgreSQL +database. A short transaction-scoped advisory lock orders insertion and snapshot +creation. PostgreSQL assigns the canonical `share_seq`; sequence gaps are +permitted, and sequence order is authoritative. Local arrival timestamps, +frontend counters, and per-worker summaries do not determine reward order. + +The submission path validates identity, job, header, and target before durable +admission. A normal successful Stratum response follows the database commit, +with `synchronous_commit=on`. A share meeting the network target also persists +its complete candidate intent atomically with the share. Connection loss after +commit can lose the reply without losing the share. + +An exact replay returns the existing share without another reward credit. +Conflicting reuse of a share identifier fails. The native global proof-hash +registry prevents the same newly submitted proof receiving separate credits +under different usernames or on different servers. Historical rows are retained +unchanged during migration, including any duplicates accepted by older code. + +There is no Python batch queue or process-wide ledger writer lease. All healthy +instances may append; transaction locks protect a shared ordering boundary. +Database connection and statement/lock limits bound resource use. Session +extranonce allocation also uses the shared database sequence and never cycles. + +## Snapshot and payout boundary + +`qbit_prism_window(anchor_job_issued_at, window_weight)` selects eligible shares +newest first by `share_seq`, counting a partial oldest share when the requested +weight is reached. Both `job_issued_at` and `accepted_at` must be no later than +the anchor. The audit wrapper fixes the reward weight to eight times network +difficulty. + +Native snapshots record a database-coordinated monotonic anchor, share range, +and payout revision. Share acceptance and snapshot creation use the same +ordering lock, so equal wall-clock timestamps or host clock differences cannot +let a later share enter an earlier snapshot. Published jobs bind their snapshot; +later accepted work cannot change their committed coinbase. + +Payout-changing operations serialize under the settlement transaction lock. +They advance the shared payout revision, and job publication checks that its +revision remains current. A cluster fingerprint binds genesis, ledger and +manifest public keys, reward multiplier, payout policy, and CTV policy. A +mismatched instance fails startup. Coordinators can use different local resource +limits and synchronized qbit nodes on the same chain. + +Compact issued-job hot writes use a per-Coordinator collector: at most 128 +admitted children (pending plus active), 64 children per transaction, and one +active batch including cancellation cleanup. Collection dwells for up to 1 ms +from the oldest admission when storage is available, flushing earlier when full +or a deadline requires it. Time behind another batch and admission backpressure +remain inside the original persistence deadline. These bounds are initial +engineering choices, not evidence of a latency target or a speedup. + +Groups share the complete original compact dependency identity and the expected +current revision and parent; each Coordinator is bound to its own ledger/frontend. +Only small owned child metadata is queued. Prepared reservations, inline/direct +single-job APIs, and missing-dependency cold repair keep their existing paths. +Compact hot batches and single-child repair take cluster `FOR KEY SHARE`, +prepared `FOR KEY SHARE`, and template/balance `FOR KEY SHARE` locks in that +order when those rows exist. Direct compact prepared storage also takes cluster +`FOR KEY SHARE` before its authority checks and blob writes. These persistence transactions do +not acquire `SETTLEMENT_LOCK` or `ORDER_LOCK`. +Shared revision, configuration, writable-state and dependency checks happen +after the row waits. Children and any retention extension through the largest +child expiry plus existing headroom commit atomically; original reservation +identity and each child's absolute expiry never change. + +The cluster row's lock matrix (#479): + +| Who | Lock on the cluster row | +|---|---| +| Authority writers (`payout_revision`, `config_fingerprint`, `fatal_error`) | `FOR UPDATE`, then the `UPDATE` | +| Job persistence cohorts, repair and prepared storage | `FOR KEY SHARE` | +| Blob GC | `FOR UPDATE` | +| Share append clock and snapshot anchor (`ledger_clock_ms`) | `FOR NO KEY UPDATE` (the `UPDATE`'s own lock) | +| Candidate, offer and append revision readers | `FOR SHARE` | + +`FOR KEY SHARE` conflicts with `FOR UPDATE`, so a cohort still serialises +against every authority change and against blob GC, but it does not conflict +with the clock `UPDATE`: shares and cohorts no longer queue behind each other +on the clock, which no persistence path reads. A **bare `UPDATE` of an +authority column is forbidden**: it takes only `FOR NO KEY UPDATE` and passes a +held cohort fence. Every authority writer takes the row `FOR UPDATE` in the +same transaction before it writes (`lock_cluster_authority`, or its own +`SELECT … FOR UPDATE`). The `*_waits_for_a_job_cohort_fence` tests drive each +production writer through its API against a held fence: configure, tip +observation, the settlement, reconcile and orphan revision bumps, both halts, +fatal-state clear and the policy transition (`ledger_postgres`), and the +signing transition (`signing_transition`) and orphan settlement +(`offer_lifecycle`). `authority_writers_lock_the_cluster_row_first` scans the +source for a bare one. Mutable authority is checked after acquiring the +shared fence and stays fenced until commit. + +**Upgrading from a 3.x.x build before #517.** Such a build's authority writers +are bare `UPDATE`s, which pass the new cohorts' `FOR KEY SHARE`. Do not mix the +two: drain and stop every older frontend, collector and one-shot tool +(coordinated, as for any fence change) before starting a build with this +fence. The schema is unchanged. + +**MultiXact use.** A clock `UPDATE` that overlaps a cohort's fence now +proceeds instead of waiting. It leaves a locker-plus-updater MultiXact on the +singleton row, and the lock is carried to the new row version. On PostgreSQL +16, 1,000 clock `UPDATE`s used no MultiXact IDs with no locker and 2,000 with +one `FOR KEY SHARE` holder open. At 500 shares/s with a fraction *f* of +appends overlapping a cohort, that is about 86 M·*f* a day, which brings the +anti-wraparound MultiXact vacuum forward to about every 4.6/*f* days at the +default `autovacuum_multixact_freeze_max_age`. This is far cheaper than the +wait it replaces, and it is not a correctness issue. To do: in the next D1 +run, record `pg_stat_slru` (`MultiXactOffset`, `MultiXactMember`) and +`mxid_age(datminmxid)` before and after, and note the measured rate here. Coordinator +readiness, publication generation, epoch and lease checks still run before and +after storage; a durable undelivered row grants no delivery authority. + +The blob collector takes `SETTLEMENT_LOCK` → `ORDER_LOCK` → cluster `FOR UPDATE`, +then starts fresh blob-key and reference scans. The exclusive fence excludes +compact reservation and repair through commit, including repair that reuses +surviving orphan blobs whose bytes were already checked. Blob row locks alone +cannot refresh an earlier cross-table reference snapshot. Every surviving job +and candidate balance reference retains its blob, even after job/claim expiry. +Job expiry remains a separate statement with its outer renewal recheck. + +| Path | Lock order relevant to compact persistence | +| --- | --- | +| Compact prepared / issued / batch | Cluster SHARE → dependency/blob writes or locks → commit; no later advisory acquisition | +| Blob collector | SETTLEMENT → ORDER → cluster UPDATE → fresh scans/deletes | +| Candidate enqueue | ORDER → cluster SHARE → balance blob → candidate | +| Share append with candidate | ORDER → cluster clock UPDATE → balance blob → candidate | +| Chain observation | SETTLEMENT → cluster UPDATE | +| Accounting/recovery | SETTLEMENT → ORDER → authority UPDATE (some also take instance/lease table fences) | +| Configuration | Cluster UPDATE → configuration checks/write | + +The collector must acquire ORDER **before** its exclusive row fence: a candidate +can already hold ORDER while waiting for the cluster row. Compact writers never +reacquire either advisory lock. Native authority writers do not lock compact job +or blob rows and then wait for their cluster fence; ordinary ledger-clock updates +can still cause legitimate row contention. This removes unrelated advisory +serialization, not contention with actual authority changes. + +**Coordinated upgrade required:** stop all frontends and collectors, replace all +binaries with this protocol, then restart them together. An old collector is +unsafe with a new unlocked repair writer. The schema and compact format 1 are +unchanged, but arbitrary mixed-version rolling operation is unsupported. Follow +the existing migration/candidate-drain requirements when crossing older formats. + +The three-second stub acceptance builds work first and proves runtime return and +exact durable children/dependencies while a transaction still owns only the +settlement advisory lock. Hot singleton/cohort, cold repair with surviving or +missing blobs, and one/two frontends are checked; the direct 64-child API also +proves a single atomic commit. Separate races synchronize on observed database +waits, not assumed sleep durations. Direct prepared persistence has the same +stub acceptance, but fresh publication still observes the chain under settlement +locking. This is neither full-refresh independence nor evidence for the +one-second/2,000-session budget. Historical measurements remain attributed to +their original revisions in `tests/perf/b275_persistence_measure.md`. Row waits +are not settlement advisory time; storage time still includes pool and locks, +and unmeasured commit time remains unknown. + +A conflicting child fails its whole transaction, including renewal. Other +groups can succeed independently; no SQL failure is silently replayed. Canceled +queued children are discarded. Once a batch is active, it continues while any +member still waits: an individually canceled child's immutable metadata may +commit undelivered alongside live peers, with its original expiry unchanged. +Cancellation of all active members interrupts the shared attempt; before COMMIT +this rolls back every child, including the singleton case. The minimum original +deadline and earliest child expiry still govern the whole atomic batch, even +when the earliest member has canceled. These limits can fail otherwise live +peers; avoiding that would require a different transaction partition or a retry. +A batch-local statement limit preserves stricter session +settings and otherwise caps statements at 15 seconds or the remaining original +deadline. An interrupted attempt drains rollback before starting the next batch; +cleanup is bounded at 16 seconds and discards an unresponsive connection. +Dropping the Coordinator closes admissions and resolves pending waiters. +COMMIT already started means an uncertain outcome after cancellation or lost +acknowledgement, never proof of rollback. Reconciliation must use the exact +original IDs, payloads and expiries. Committed, undelivered children retain their +dependency through their original expiries with the existing renewal headroom, +including when every caller cancels during COMMIT. Cancellation does not delete +committed metadata, undo its retention, or restart its expiry; normal pruning +still applies. An unexpired committed child also keeps its payload's +`extranonce1` referenced, preventing session allocation from reusing that value +until the child's original expiry even when its caller has canceled. +A durable row may remain undelivered after +authority revocation: the original caller still revalidates after persistence, +and Stratum never sends work before successful durable commit and revalidation. + +The batch debug event records actual shared storage-attempt elapsed time and +cardinality, including pool and lock waits; cleanup is separate. Concurrent pool +and advisory wait totals overlap and cannot be subtracted to infer exclusive +service time. The one-second delivery, two-frontend non-regression, and lock-free +criteria of #275 remain unqualified by this batching change. + +A pool with no historical shares issues solver-paid bootstrap work. There is no +three-miner gate. A network-valid candidate below its assigned share target is +stored without ordinary share credit. Active-chain confirmation inserts its +deferred share at network difficulty; a losing candidate receives no credit. + +## Durable block candidates + +`qbit_block_candidate_outbox` stores complete candidate evidence before a node +submission can be lost to a process crash. Workers claim unfinished rows with +expiring, token-fenced database claims. Multiple instances may process different +candidates; a stale claimant cannot overwrite a successor's accounting result. +Terminal candidates retain the evidence needed for replay identity while large +payloads are released. Deferred below-share-target credit is tied to the same +durable candidate lifecycle. + +Since migration 011 a claim runs in two phases, the node offer first and the +accounting after it, and the row records where the block is between them: + +| State | Meaning | +| --- | --- | +| `pending` | Durable and never offered. The only state a claim may offer from, and the only one that may still be abandoned: a block proven superseded before it was ever offered. | +| `offer_reserved` | The claim took the durable reservation immediately before its one `submitblock` call. The row is the unique reservation per block hash: once it commits, no claim on any frontend offers the block again, this frontend included after a crash. The one exception is a call that provably never ran on the node: the connection the request would have been written to was never established (connection refused, a name or route failure, the connect timeout, #522), or the node answered it with `RPC_IN_WARMUP` (`-28`), which qbitd returns before it dispatches any call while it is still starting (#526). Both rules assume the RPC URL reaches one node: an endpoint that retries a failed POST on another backend is not supported, because it can submit a block twice by itself. That attempt returns its own reservation to `pending` in one claim-fenced transaction. The offer columns are cleared, `last_error` starts with `offer not sent`, and the row is offered again after the ordinary pending backoff. | +| `offered` | The node's answer is recorded in `offer_outcome` (`accepted`, `rejected` with the node's reply in `offer_reply`, or `unknown`) with the call time; the audit is still to be landed. | +| `reconciliation` | Offered, and automation could not finish it: an unknown outcome (a transport failure or timeout after the connection was established, a reservation whose call was lost with its frontend, or a pre-011 attempt 011 quarantined), a node rejection, a landing that failed after acceptance, a node or database error after the offer, or a block not on the active chain yet. `last_error` holds the reason. Retried `min(3600, 10 × attempt_count)` s apart with read-only chain observations only, never another `submitblock`, and never abandoned; settled terminal as `orphaned` once the chain proves a competitor at its height. | +| `orphaned` | Terminal (migration 015, #415). Reachable from the three offer states only. One coherent read-only observation proved a *different* block active at the candidate's height with at least `PRISM_CANDIDATE_ORPHAN_CONFIRMATIONS` confirmations (default 6, counted as `tip_height − height + 1`), after the row's audit landed: the lost tip race of the 2026-09-16 mainnet orphan stall (#413). `last_error` names the competitor, the height, the confirmations and the tip. Like the other terminal states, the row releases its document, block bytes and window reference, so terminal history does not pin candidate payloads or balance snapshots. It preserves its offer metadata and reason; the landed audit and pool-block row retain the accounting evidence. Never claimed again, never offered again, and no longer counted by `qbit_prism_block_candidates_pending` / `qbit_prism_block_candidate_oldest_pending_seconds`; `qbit_prism_block_candidates_orphaned_total` counts completions observed by this process after commit and may undercount if cancellation or restart intervenes. The block's `qbit_pool_blocks` row is marked `inactive` and keeps its landed audit, so a later reorg that reactivates the block is confirmed and credited (deferred share included) by the ordinary reorg reconciler, from that preserved evidence, without this row ever reopening. `orphaned` describes the completed outbox processing decision, not the block's permanent chain status. | +| `submitted` | The block was proven on the active chain and its audit landed. The document, the block bytes and the window reference are released; the offer record stays. A recovered reservation, whose call's answer was never recorded, lands with `offer_outcome = unknown` (#529). | +| `abandoned` | Reachable from `pending` only. | + +### A lost revision race after the offer (#630) + +After an offer, a frontend can log either of these ERROR lines: + +``` +ALERT: offered candidate could not land; kept for reconciliation, never offered again +ALERT: offered candidate could not be processed; kept for reconciliation, never offered again +``` + +When the reason ends `payout revision changed while observing chain state`, +the line records a lost race, not a failure. The reason starts `landing +failed after the offer (…)` on the first line and `post-offer processing +failed (…)` on the second. The parentheses hold the offer's outcome: usually +`node accepted the offer`, or `offer outcome unknown: …` for a recovered +reservation or an adopted block. + +- **What happened.** Each post-offer write is fenced on the payout revision + its claim observed: the landing, the finish as `submitted`, and the + settlement as `orphaned`. The revision moves whenever a frontend records a + new chain tip, or confirms, deactivates or matures a block, and on a policy + transition or a fatal-state clear. An accepted own block becomes the node's + tip, so a template refresh on either frontend records that tip and, once the + audit has landed, confirms the block, credits its deferred share and bumps + the revision again. If such a bump commits between a claim's observation and + its write, the write is refused, and the row is settled in `reconciliation` + with that reason. +- **Why it is benign.** The refused write changed nothing. The row keeps its + offer record and is never offered again. Its next claim comes at the + `next_attempt` that `candidates list` shows, the `reconciliation` backoff + above. It observes again, lands the audit if it had not landed, and settles + the row `submitted`, or `orphaned` once the chain proves a competitor. The + block's confirmation and deferred-share credit commit once, whichever side + commits them first. A retry that loses the race again logs the line again, + and waits longer. +- **What to do.** Nothing, as long as the row finishes. Run + `qbit-prism-server candidates list` once its `next_attempt` has passed; the + block should be gone. Act only if it is still listed after a few retries, or + its `last_error` names another reason. Then handle it as any other + `reconciliation` row. Never try to abandon it: `candidates abandon` refuses + an offered row (exit 3). +- **Alerts.** While it lasts, a lost landing counts as landing-failed, a lost + finish as pending only, and a recovered reservation as unacknowledged, for + its unknown outcome. The candidate alerts measure a row's age from when it + was created or reserved, not from the race, and none fires before a row has + been unfinished for at least 3 minutes. So a freshly found block whose race + clears on the next retry or two stays under all of them. A row that was + already old, or that has many attempts behind it (its retry waits + `10 × attempt_count` s), can fire one anyway. If one fires, look at the row. + +#630 tracks retrying the write in place, so that a lost race stops logging an +ALERT. + +### Candidate commands + +Three operator commands read and finish the rows in the state table above. +Neither `list` nor `abandon` builds a node client, reads a signing key, loads +the server configuration or starts a listener, so neither can offer a block. +`list` needs only `PRISM_DATABASE_URL`, as `fatal-state show` does. `abandon` writes +an ordinary ledger row, so it uses the one-shot tool connection and the +stored-data settings behind it — `PRISM_DATABASE_URL`, `PRISM_INSTANCE_ID` +(generated when unset) and `PRISM_DATABASE_MAX_CONNECTIONS`. `recover` is the +one of the three that reads the node, and is described after the other two. + +```sh +qbit-prism-server candidates list [--json] [--limit <1..10000, default 100>] +qbit-prism-server candidates abandon --block-hash <64 lowercase hex> --reason "<nonblank explanation>" [--unsafe-database-clock-expiry] +qbit-prism-server candidates recover --block-hash <64 lowercase hex> [--block-hash <hash> ...] [--apply] [--timeout-seconds <1..3600, default 600>] [--unsafe-database-clock-expiry] +``` + +`list` prints unfinished rows up to the selected limit — `pending`, `offer_reserved`, `offered` +and `reconciliation` — oldest due first. That is the oldest-due claim lane's +own ordering, so the row a server works next is the first line and parked +rows sort last. It is not ordered by height, which is read out of the +candidate document and may be unknown. + +``` +block_hash state height attempts next_attempt claim sv last_error +7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a pending 184021 0 2026-09-16T15:03:10.792447+00:00 - 1 - +5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f offered 184020 2 2026-09-16T15:03:28.793326+00:00 expired 1 - +3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c offer_reserved 184022 1 2026-09-16T15:03:43.793265+00:00 prism-frontend-b until 2026-09-16T15:03:50.793265+00:00 1 - +9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e reconciliation 184018 7 2026-09-16T15:04:08.793284+00:00 - 1 submitblock reply was lost with the offering frontend; awaiting an autho…(truncated) +d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4 pending 184019 3 parked - 1 block digest did not authenticate against candidate_sha256 +b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1 pending - 1 parked - 3 candidate storage_version 3 is not supported by this server; only versio…(truncated) +``` + +| Column | Meaning | +| --- | --- | +| `block_hash` | The whole 64-character hash, so it can be pasted straight into `candidates abandon --block-hash`. | +| `state` | The lifecycle state from the table above. Terminal rows are finished work and never appear. | +| `height` | `candidate->'found_block'->>'block_height'`, extracted server-side. `-` (text) or `null` (`--json`) when the document holds no readable height — an unknown-`storage_version` row, for example. Never `0`. | +| `attempts` | `attempt_count`: how many claims the row has taken, not how many offers were made. | +| `next_attempt` | `next_attempt_at`, or `parked` when it is `infinity`. | +| `claim` | `<claim_instance_id> until <claim_expires_at>` for a claim the database clock still calls live, `expired` for one past that expiry, `-` for none. The expiry is the database clock's estimate only: a frontend takes a claimed row over once it has watched the claim go unrenewed for its whole lease (see [Candidate claim leases and database clock steps](#candidate-claim-leases-and-database-clock-steps-021-581)). `--json` keeps the stale holder and expiry. | +| `sv` | `storage_version`. Anything other than `1` is a row this server cannot decode. | +| `last_error` | Why the last attempt stopped. Truncated in text mode only, and marked `…(truncated)` when it is; `--json` prints it whole. | + +**Parked is not abandoned.** A parked row has `next_attempt_at = 'infinity'` +and a `last_error`, which is exactly what the `parked` cell means. It is still +unfinished: the claim lane moved it out of the retry schedule because this +server could not decode or validate it, and it keeps its state, its document, +its block bytes and its window reference. It is operator work, not a retry +loop, and it is not terminal until something finishes it. + +`list` never loads a payload. `candidate` and `block_bytes` are not in its +projection, so the command stays usable on a production-sized row. It opens a +`default_transaction_read_only=on` pool with one connection, a 15 s statement +timeout and a 5 s lock timeout, which is why it takes no claim and why it +keeps working while the cluster is halted — precisely when it is needed. An +empty list is a **success**: the command prints `no unfinished candidates` (or +an empty `candidates` array) and exits zero, so a `set -e` runbook can wait for +exactly that. `--json` prints one +`{"schema":"qbit.prism.candidates.list.v1","candidates":[...],"limit":100,"truncated":false}` document with +every field untruncated and every unknown value as `null`. The inventory is limited +to `--limit` rows (default 100, maximum 10000). Both modes fetch one extra row to +detect omitted candidates: JSON sets `truncated: true`, while text mode writes a +warning to stderr. Parked rows sort last and may be among the omitted rows. Raise +`--limit` to inspect more; inventories larger than 10000 require a read-only +database query. A full page is complete only when `truncated` is false. These +fields describe the query's snapshot, not rows arriving after it. + +`abandon` finishes one `pending` row with `storage_version = 1` **whose +document this release could replay**. Other storage versions are refused +atomically and their evidence is preserved, and so is a version-1 row holding a +pre-migration `2.x.x` document: the native claim lane parks one of those at +version 1 rather than rewriting it, so the version alone does not say who wrote +the row. The statement's shape test is the migrator's own — a native document +carries `payout_revision` and `block_hash` beside either an inline `bundle` +(pre-007) or a `window` reference (007 and later) — and anything else is +refused with exit 8 and every column left as it is. That block is still owed to +[the legacy drain](prism-rust-migration.md#two-drains-one-for-each-era), which +is the only thing that can finish it. `pending` is +the only state it touches, because it is the only unfinished state from which +no `submitblock` can yet +have been made: a row in `offer_reserved`, `offered` or `reconciliation` is the +record that a call may already have happened, and discarding it would discard +that record. The rule is the statement's `WHERE` clause, not a check the +command makes first, so a row that moves between reading and writing is still +refused. Before evaluating these predicates, `abandon` takes the settlement +lock shared with candidate landing. If a landing is still in flight when its +claim expires, the command waits, then sees the committed accounting and +refuses with exit 6. A successful abandon releases the document, the block +bytes and the six window columns, clears the claim, sets `completed_at`, and writes the +operator's `--reason` into `last_error` — exactly the columns the offline +epoch supersession writes. `next_attempt_at` is deliberately left as it is: no +lane selects a terminal row, so its value is inert, and an `infinity` left +there stays as evidence that the row had been parked. + +| Exit | Outcome | Message | +| --- | --- | --- | +| 0 | Abandoned. | `abandoned <hash>: <reason>` | +| 1 | Configuration or database failure, including the two fences below. | The underlying error, as for every other command. | +| 2 | No such row. | `no candidate row for <hash>` | +| 3 | Offered to the node; never abandonable. | `candidate <hash> is in state <state>; it was offered to the node and is never abandoned. Its block may already have been submitted. Leave it to reconciliation` | +| 4 | Already terminal. | `candidate <hash> is already <submitted\|abandoned\|orphaned>; nothing to do` | +| 5 | Held by a live claim: one that changed while the command watched it, or, with `--unsafe-database-clock-expiry`, one the database clock calls live. | `candidate <hash> is held by <instance>, whose claim changed while this command watched it (the holder renewed it, or a frontend took the row over), so it is live (database clock estimate: until <expiry>); retry once the holder has released it`, or with the flag `candidate <hash> is held by <instance> until <expiry>; retry after the claim expires` | +| 6 | Pending, but its block has landed. | `candidate <hash> is pending but its block is already in qbit_pool_blocks; reconcile it before abandoning — abandoning would discard landed accounting` | +| 7 | Unsupported storage version; evidence preserved. | Names the version and directs legacy rows to the pinned `2.x.x` drain, newer formats to a compatible release. | +| 8 | A pre-migration `2.x.x` document parked at `storage_version = 1`; evidence preserved. | `candidate <hash> holds a pre-migration 2.x.x document at storage_version 1; evidence preserved. This release cannot replay it, and abandoning it would discard the block the legacy drain still owes: drain it with the pinned 2.x.x image, never an operator abandon` | + +Codes 3, 6, 7 and 8 protect offer, accounting, storage-format and legacy-era +evidence. Code 4 +is kept distinct from code 2 so that re-running a successful abandon reads as +"nothing to do" rather than as a lost row. + +A claim is over for `abandon` as it is for a frontend (#581): only once the +command has itself watched it go unrenewed for its whole lease, on its own +monotonic clock. A holder's writes are fenced on its token alone, so a +`claim_expires_at` the database clock has passed does not mean the holder +stopped, and after a forward step it would otherwise abandon a block its live +holder is about to offer. On a claimed row the command prints `candidate +<hash> is claimed by <instance>; waiting <N> s for the claim to go unrenewed +for its whole lease`, waits once (up to one lease: 120 s, or 600 s for a claim +taken before 021, and 0 s for a revoked one), and abandons the row by compare +and set on the version it watched. A claim that changed meanwhile, renewed or +taken over, is live, and its refusal (exit 5) is the answer. +`--unsafe-database-clock-expiry` restores the pre-021 rule instead: an +**expired** claim, `claim_expires_at` before the database clock, is not a live +claim, so a supported row whose owner died is abandonable without waiting. +That is wrong by the size of any database clock step, so use it only when the +holder is known to be gone and the clock has not stepped. Under that rule code +5 reflects whether the claim was live at one database timestamp captured after +acquiring the settlement lock. The UPDATE and refusal diagnosis share that +timestamp, so expiry between them still reports a claim refusal rather than an +internal consistency error; a fresh invocation can abandon the now-expired row. Unsupported versions report +code 7, and an unreplayable version-1 document code 8, both ahead of claim +status: a claim expires on its own, and neither a storage format nor a document +shape becomes replayable by waiting. A pending row whose block has landed always +reports code 6, because the accounting must be reconciled. + +`abandon` connects as a one-shot tool and writes through the ordinary ledger +write transaction, so it is fenced twice, both reported as exit 1: + +- **A halted cluster.** While `qbit_prism_cluster.fatal_error` is set, + `abandon` is refused at connect with `cluster halted: ...`, exactly as a + frontend would be, and the write guard would refuse it again. Use + [fatal-state recovery](#fatal-state-recovery) first. `list` is unaffected and + stays available throughout. +- **A live legacy Python writer lease.** While a `qbit_ledger_writer_lease` row + has not expired, the write fails with `live legacy Python writer lease`. This + is what stops an operator abandon racing the `2.x.x` writer during a cutover. + +Like the other one-shot commands, `abandon` writes no heartbeat: it leaves no +`qbit_prism_instances` row for `fatal-state clear` to refuse, and a live +frontend that shares its configured `PRISM_INSTANCE_ID` keeps its status and +session-owner token untouched. + +`recover` lands a native-era block the node has already accepted and the +claim lane cannot finish — a window whose rebuild exceeds the lane's 60 s +rebuild deadline, for example, so the row sits in `reconciliation` retrying +forever. `abandon` is correctly refused for every state the node may already +have been offered, so before #418 such a row had no operator path. `recover` +lands it at the proven chain revision, under an explicit allowlist, through +the same in-process machinery the coordinator uses +(`land_candidate_at_revision`, driven the way `process_candidate_inner` +drives it for an offered row), and it never calls `submitblock`. It starts no +listener. The invariant is #268's: no operator command may discard or +duplicate the effect of a `submitblock` that may already have been made. +`recover` lands an already-accepted block; it never offers one. `abandon` +stays `pending`-only; `recover` does not widen it and adds no `--force`, and +it adds no candidate state and no column. + +`recover` is the only one of the three that reads the node, and what it +loads depends on the mode. The plan needs `PRISM_DATABASE_URL` plus the node +RPC settings — `QBIT_RPC_URL` (or `QBIT_RPC_HOST`/`QBIT_RPC_PORT`), +`QBIT_RPC_USER`, `QBIT_RPC_PASSWORD` and `PRISM_RPC_TIMEOUT_SECONDS` — and no +signing seed, no chain setting and no instance ID. `--apply` needs the +frontend's full configuration, exactly as `self-check`, `broadcast-ctv` and +`fatal-state clear` do: database, qbit RPC, `QBIT_CHAIN`/genesis, payout +policy and the signing seeds, because the audit is rebuilt from the +candidate's stored inputs and signed with this frontend's seeds, and the +command verifies the node's genesis/chain and the cluster fingerprint at +connect. Run it from a frontend's environment. Setting `PRISM_INSTANCE_ID` +(for example `operator-recovery-INC-123`) is recommended, so that +`candidates list` names the recovery as the claim holder while it runs. +Frontends do not need to be stopped: the durable claim is the fence. A row +another instance holds is refused (exit 5), and while the recovery holds a +row no frontend's claim lane can take it. (`2.x.x` required stopping the +coordinator because its runner used the writer lease; the native command +does not.) `--apply` connects as a one-shot tool (`Ledger::connect_tool`, +#412), as `abandon` does: no heartbeat, no `qbit_prism_instances` row left +behind, a halted cluster refused at connect with `cluster halted: ...` +(exit 1), and every write refused while a legacy Python writer lease is live +(exit 1). + +**Plan.** Without `--apply` the command plans and writes nothing, ever. It +reads the outbox rows for the listed hashes on a +`default_transaction_read_only=on` pool with one connection, the shape +`list` uses, so taking a claim is impossible by construction. It loads no +payload: `candidate` and `block_bytes` are not in the projection, and the +stored height is extracted server-side exactly as `list` extracts it. It +calls the node read-only — `getblockheader` for each hash and `getblockhash` +at its height — to prove each block is on the active chain and to learn its +parent, and it prints the selection ordered by height ascending, parent +before child, one table and one summary line on stdout: + +``` +block_hash height state parent claim action +<hash> 184018 reconciliation <parent hash> - recover +<hash> 184019 pending <parent hash> prism-frontend-b until 2026-09-16T15:03:50+00:00 recover +<hash> 184020 submitted <parent hash> - complete +plan: 2 to recover, 1 already complete; rerun with --apply to land them +``` + +`action` is `recover` for an unfinished row the node proves active, or +`complete` for a `submitted` row whose accounting is proven — a `confirmed` +`qbit_pool_blocks` row and an audit row — which `--apply` verifies and +skips. `claim` is rendered as in `list`: `<instance> until <expiry>`, +`expired` or `-`. A live claim is reported in the plan but does not fail it; +it is a moment-in-time fact, and `--apply`'s own claim is the fence. When +nothing is left to recover the summary reads `plan: nothing to recover; +every listed block is already complete`. The plan exits zero whenever the +selection is valid. + +The selection fails closed: every listed hash must pass, or the whole plan is +refused. The plan reports every problem it finds on stderr, one line each, +and exits with the code of the first one, in argument order. The allowlist +itself is checked at the entry boundary, before any connection is opened: +one to 32 hashes (the bound of the `2.x.x` runner, #259), each 64 lowercase +hex characters, no duplicates. Zero hashes or an out-of-range +`--timeout-seconds` is refused by the argument parser (exit 2, as for every +command); a duplicate, more than 32 or a malformed hash is refused by the +command itself (exit 1). There is no "recover everything" mode. + +**Apply.** `--apply` runs the plan first, and a refused plan applies nothing. +Then, for each planned block in height order: + +1. A block the plan marked `complete` rechecks its current outbox state, + confirmed accounting, audit presence and active-chain membership before + printing `verified <hash> at height <h>: already complete` on stdout and + skipping it idempotently. A changed or unproven result stops the command + without claiming that block or attempting later blocks. +2. Otherwise the command prints `recovering <hash> at height <h> from + <state>` and takes the durable, token-fenced claim on that row by hash, + through the existing claim mechanism: `attempt_count` increments, + `claim_instance_id` and `claim_expires_at` name this process, and the + ordinary lease heartbeat renews the claim while the work runs. The claim + statement's `WHERE` is the safety property — an unfinished state, no live + claim, `storage_version = 1` and a document this release wrote, the same + shape test `abandon` uses. Another holder's claim is over only once this + command has itself watched it go unrenewed for its whole lease, as a + frontend does (#581): it prints `candidate <hash> is claimed by <instance>; + waiting <N> s for the claim to go unrenewed for its whole lease`, waits + once when the deadline leaves room, and takes the row over by compare and + set on the version it watched. A holder that renews meanwhile is live, and + its refusal (exit 5) is the answer. `--unsafe-database-clock-expiry` + restores the pre-021 rule instead, `claim_expires_at` before the database + clock; it is wrong by the size of any database clock step (a forward step + takes a live holder's row), so use it only when the holder is known to be + gone and the database clock has not stepped. A row parked with `next_attempt_at = + 'infinity'` is claimable here, because a parked row is operator work, + but its schedule is left untouched. +3. It decodes and authenticates the row exactly as the claim lane does + (document digest, block digest, window columns, header hashes to + `block_hash`), then verifies the block against the node: it is on the + active chain at its height, the node's header height equals the + candidate's recorded height, and the node's `previousblockhash` equals + the parent in the candidate's block bytes. +4. A `pending` row is first adopted into `reconciliation` with the node's + evidence, exactly as the coordinator's pre-offer probe adopts an active + pending block. From that commit on, no claim on any frontend can ever + offer it, whatever happens next. +5. It runs the normal post-offer landing: builder admission, the audit + rebuild from the as-issued balance snapshot (or the current balances when + they still hash to the reference), signature and coinbase verification, + the durable range proof, and `land_candidate_at_revision` at the chain + revision observed immediately before the landing transaction. An audit + that already landed is authenticated against the block and not rebuilt + over. +6. It observes the chain again and finishes the row as `submitted` at a + revision proven now — the same `finish_candidate_at_revision` the + coordinator uses — which confirms the pool block and credits any deferred + share, then prints `recovered <hash> at height <h>`. + +The final line is `recovered N, verified M already complete`, and the command +exits 0. Rerunning the same allowlist after a success prints only the +`verified` lines and exits 0. + +**Deadline.** `--timeout-seconds N` (1 to 3600, default 600) is one deadline +carried across the whole operation: the plan's database read, every node RPC +call, the coordinator connection, and each candidate's claim, window read, +audit rebuild, landing transaction and confirmation, including revalidation +of already-complete blocks. A verification timeout exits 11 with +`exceeded (verifying); candidate <hash> was not verified`; it takes no claim +on that block. When the deadline expires during recovery the +command attempts a bounded release of the in-flight candidate's claim and +exits 11 when cleanup confirms release. If cleanup fails or times out, the +command exits 1 and reports that the claim may remain until its lease expires. +During landing, if the token no longer matches, it exits 1 and reports that +the candidate may have completed or changed owners; inspect the row before retrying. + +**Failure and resume.** On any failure the command stops at that candidate +and attempts to release its claim (bounded). A confirmed release records the +reason in the row's `last_error`, +leaves the row in whatever unfinished state it is in — a pending row that +was adopted stays `reconciliation`, and a landed audit stays landed and is +reused by the next attempt — leaves `next_attempt_at` untouched, and exits +nonzero. Later blocks in the plan are not attempted. Repeating the same +allowlist resumes: finished blocks are verified and skipped, and the failed +one is retried. + +Messages are written to stderr; only the exit-0 output above is on stdout. +Codes shared with `abandon` keep their meaning. + +| Exit | Outcome | Message | +| --- | --- | --- | +| 0 | Plan printed, or every listed block recovered or verified complete. | The stdout described above. | +| 1 | Configuration, database or node failure, including unconfirmed claim cleanup, a halted cluster, a live legacy Python writer lease, a node that is unreachable or not caught up, and an allowlist the command refuses (a duplicate, more than 32 or a malformed hash). | The underlying error, as for every other command. | +| 2 | A listed hash has no outbox row. | `no candidate row for <hash>` | +| 4 | A listed row is terminal and cannot be recovered. | `candidate <hash> is already abandoned; its evidence was released and it cannot be recovered`, `candidate <hash> is already orphaned; its candidate payload was released and its accounting remains in the ledger. Leave chain changes to reconciliation`, or `candidate <hash> is submitted but its accounting is not proven complete (<what is missing>); inspect qbit_pool_blocks and qbit_pool_audit_bundles before retrying` | +| 5 | A listed row is held by a live claim (`--apply` only; the plan reports the holder). | `candidate <hash> is held by <instance>, whose claim this command must watch go unrenewed for <N> more seconds of its lease (database clock estimate: until <expiry>); the deadline leaves no room for that, or the holder renewed while this command waited and is live. Retry with a longer --timeout-seconds, or once the holder has released it`; with `--unsafe-database-clock-expiry`, `candidate <hash> is held by <instance> until <expiry>; retry after the claim expires` | +| 7 | Unsupported storage version; evidence preserved. | Names the version, as for `abandon`. | +| 8 | A pre-migration `2.x.x` document parked at `storage_version = 1`; evidence preserved. | As for `abandon`, ending in `drain it with the pinned 2.x.x image` | +| 9 | Not on the active chain: the node does not hold the block at its height. Nothing to recover; `recover` never offers. | `candidate <hash> is not on the active chain (<node detail>); nothing to recover. recover never offers a block: a block the node never accepted stays with the coordinator (or, while pending, may be abandoned); a block a reorg removed stays in reconciliation` | +| 10 | Selection refused: an unfinished parent is not in the allowlist, or the stored height is unreadable or disagrees with the node. | `candidate <hash> has an unfinished parent <parent> (<state>) that is not in the allowlist; add --block-hash <parent> so it lands first` or `candidate <hash> is stored at height <h> but the node holds it at height <node height>` | +| 11 | The deadline expired; cleanup confirmed release, or this attempt no longer holds the claim. Unconfirmed cleanup exits 1 instead. | `recovery deadline of <N> seconds exceeded (landing); candidate <hash> was left recoverable and its claim released`; planning and connecting say `nothing was claimed`; a claim-phase timeout says `candidate <hash> is no longer claimed by this attempt` after cleanup succeeds. | +| 12 | Landing refused or failed; the candidate was left recoverable with the reason in `last_error`. | `recovery of <hash> stopped: <reason>; the candidate was left recoverable` | + +Codes 3 (offered; never abandonable) and 6 (pending but landed) do not occur +for `recover`: every unfinished state is recoverable, and a landed audit is +exactly what an idempotent landing reuses. + +A native recovery, step by step: + +1. Plan the exact hashes. Take them from `candidates list`, run + `candidates recover --block-hash <hash> ...` without `--apply`, and add + every unfinished parent the plan names (exit 10) until it exits zero and + the table shows what you expect. +2. Apply from a frontend's environment, with a named `PRISM_INSTANCE_ID` + such as `operator-recovery-INC-123`, by rerunning the same allowlist with + `--apply`. Frontends may stay up. +3. Check the exit status and the final line (`recovered N, verified M + already complete`). On a nonzero exit read the stderr line and the row's + `last_error`, resolve the cause, and rerun the same allowlist to resume: + finished blocks are verified and skipped, the failed one is retried. +4. Verify with `candidates list` — the recovered rows are gone, because + `submitted` rows are finished work — and with the audit and dashboard + reads for the landed blocks. + +`recover` is the one command of the three that takes an allowlist — at most +32 hashes, named one by one, never "everything" — and none of the three +offers a block. + +**Offer phase.** The minimum before the one `submitblock` call: the proof was +validated at Stratum admission and the row authenticated at claim (document +digest, block digest, window reference); an ordinary candidate passes the +cached staleness screen, where a stale cached tip or payout revision triggers +one authoritative chain probe that abandons a proven-superseded block or, for +a block the chain already holds, adopts the row into `reconciliation` with the +node's evidence and lands it without any offer; a leased candidate (#350) +skips the screen and is offered as issued. The request is prepared, the +reservation is committed, the strictly-live claim token is renewed +immediately before the bounded RPC, the call is made and its outcome +recorded. Nothing here waits for a builder permit or a window read, and on +the cached fast path nothing here takes the settlement lock; only the +authoritative probe a stale cached tip or revision triggers observes the +chain, which records the observation under `SETTLEMENT_LOCK` before the +reservation. That screen is kept: it is what keeps proven-superseded backlog +off the node. + +**Queue and measurement.** Each frontend runs one submit loop, and it +processes claims serially: a claim's whole post-offer phase, its landing +included, completes before the loop takes the next claim, so a block found +while another is being settled waits in the outbox to be claimed, and on a +single frontend every block admitted in that time waits behind it. The offer +phase bound above is measured from the claim, on the cached fast path, with +isolated sequential blocks (`tests/offer_latency.rs`, which holds the builder +permits and the landing while it measures); it excludes the time a candidate +spends in the outbox before it is claimed and establishes neither +proof-to-offer latency under load nor concurrent throughput, both of which +include the serial loop's queue delay. A stale cached tip or revision never +shortens that path: the authoritative probe keeps its chain observation under +`SETTLEMENT_LOCK` and its abandon-or-adopt decision under the claim. The +ordinary payout-revision screen applies to every never-offered candidate; an +issued lease (#350) is its one approved exception. That queue has a material +economic window: the offer precedes the accounting confirmation, so an +ordinary candidate found on the offered block's tip and queued while that +block is being settled carries the revision current at its admission, and +when the settlement's confirmation advances the payout revision the screen +abandons the queued candidate as superseded work, even though its parent is +still the active tip. The screen stays: valid proof of work on a superseded +payout snapshot is not authorization to offer it, and only an issued lease is +exempt. More frontends can reduce the queue delay, with no throughput +guarantee; removing it, by settling one claim's post-offer phase while the +next claim is offered, is a separate concurrency change and not part of 011. + +**Post-offer phase.** Builder admission, the audit rebuild from the as-issued +balance snapshot (or, for a row enqueued without one, from the current +balances when they still hash to the reference), signature and coinbase +verification, the durable range proof, the landing at the chain revision +observed after the rebuild, and a fresh observation of the chain before the +row is finished as `submitted`. A failure here (a stored builder version or +signer pair this binary cannot rebuild, a window read failure, a landed audit +that does not authenticate against the block, a landing refused by its fences, +a node or database error, a block not on the active chain) settles the row in +`reconciliation` with its reason and every piece of evidence. No post-offer +error returns a row to `pending` or abandons it. Only when that settlement +itself cannot be written does the error propagate, and the row keeps its +reservation or offer record for the next claim. + +**Recovery.** Every unfinished state is claimable once its lease is over, on +any frontend (a frontend that has watched the claim go unrenewed for its +whole lease takes it over, #581), and counts toward the candidate backlog and +retention. A +recovered `offer_reserved` row is delivery unknown: the call may or may not +have been made, so it is never offered again even when the node reports the +block unknown; it lands its audit and waits in `reconciliation` for the chain. +A recovered `offered` or `reconciliation` row lands its audit, or +authenticates an already-landed audit against the block rather than rebuilding +over it, and is finished once the block is active. A duplicate offer is +therefore impossible across crashes and takeovers. The price is that a crash +between the reservation commit and the call loses that delivery; the row +reports it as unknown and reconciles rather than retrying. + +A frontend whose attempt fails releases its claim at once, so the row does +not wait for the lease. When that release fails too, because the database +stopped answering (a PostgreSQL failover drops the pool's connections with +the old primary), the frontend retries the same token-fenced release about +once a second, alongside its other work, until the database answers or the +lease ends, and once more at shutdown (#529). The release runs only after the +failed attempt has stopped, and changes nothing once another frontend has +taken the row. The frontend logs `candidate claim release deferred` when it +starts retrying, and `deferred candidate claim release succeeded` or +`... expired with its lease` when it stops. A claim whose frontend was lost +with the old primary still waits for its 120 s lease. + +**A graceful shutdown mid-offer (#578).** From just before the reservation +until the node's answer is recorded (or an unsent reservation is returned to +`pending`), a graceful shutdown does not drop the attempt: the submit loop +keeps driving it, bounded by the send's deadline, the standby wait's bound +and the lease's bound for each of the reservation, the renewal before the +send and the recording, and logs `shutdown waits for a found block's offer +in flight`. Everything the submit loop does after the shutdown signal +(this wait, handing back its claim and the deferred releases) finishes +within 20 s, inside the server's 30 s shutdown, and no new row is claimed +once the signal arrived. Only then does it stop; +the landing that follows is left to the next claim, which lands the `offered` +row without another offer. Without this, a restart between the reservation +and the call left an `offer_reserved` row that was never sent and is never +offered again. A crash in the same window still recovers as delivery unknown. + +**A call that never connected (#522).** When `submitblock` fails before its +connection is established, the request provably never reached the node. That +covers connection refused (the node or its RPC server restarting), a name or +route failure, and the 5-second connect timeout. The attempt that holds the +reservation returns the row to `pending`, and the next claim runs the +pre-offer phase again: an ordinary candidate whose parent was superseded +meanwhile is abandoned there, and a leased candidate is offered without that +screen, as every leased candidate is (#350). Nothing else is treated this way. A reset, closed connection, lost +reply or timeout after the connection was established, and any HTTP or +JSON-RPC error, stay unknown and are never offered again, because the request +may have been written. A crash or a lost claim between the failed connect and +the row update leaves `offer_reserved`. Recovery treats that row as delivery +unknown, since nothing durable proves the call was not made. Such an attempt +records no outcome and no first-offer sample. While the node refuses, the +row stays `pending` and counts toward +`qbit_prism_block_candidate_oldest_unacknowledged_seconds`, like any block the +node has not accepted. It never counts as an unknown outcome. + +**Parked candidates.** A claim parks a row it must not use instead of +retrying it. Two cases park: an unsupported `storage_version`, or a version-1 +row without its JSONB body (#285); and, since #387, a supported row whose +persisted input fails the claim's validation. That covers a missing window +reference or an inline pre-007 document, and JSON that is not a supported +candidate. It covers a `candidate_sha256` or document block hash that +disagrees with the row, and window columns that disagree with the document +or cannot hold its range. It also covers missing, truncated or mis-hashed +block bytes, an empty or non-hex coinbase suffix, stored inputs that +contradict the window reference, and an offer outcome this binary does not +know, persisted on a claimed unfinished row. (A claim selects only the four +unfinished states, so a row in an unknown state is never claimed and never +reaches this path.) The decode runs after the claim commits, and parking is +one `UPDATE` in a follow-up transaction behind the writer fence. That update +matches the row's database block hash, the claim's token and the state the +claim selected, so a replacement owner's claim, or a row that has since moved +to another state, is never parked. It clears `claim_token`, +`claim_instance_id` and `claim_expires_at`, records the reason in +`last_error` and sets `next_attempt_at` to `infinity`. The state, document, +digest, block bytes, window columns and offer record stay unchanged, and +`attempt_count` keeps the claim's increment. A validation reason reads +`candidate <block_hash>: validation <kind>: <diagnosis>`, at most 1024 bytes, +where `<kind>` is one of `lifecycle`, `window_reference`, `document`, +`document_digest`, `document_identity`, `window_columns`, `block`, +`coinbase_suffix` or `reference_invariants`. The claim still fails, so the +submit loop logs `candidate polling failed`. The ledger logs the full +diagnosis with one of three messages, and only the first means the row is +parked: + +| Log message | Meaning | +| --- | --- | +| `parked a candidate that failed validation; operator action required` | The update matched exactly one row and its commit succeeded. | +| `a candidate failed validation and was not parked` | Nothing was committed: the transaction could not begin, the writer fence refused, the claim was no longer this attempt's, or the transaction failed before its commit. A row still held by that claim is claimed again after its lease and parking is retried. | +| `a candidate failed validation and whether it was parked is unknown; inspect the row` | The commit returned an error, so the outcome is unknown. | + +Nothing else parks a row. A claim or decode failure that is not one of these +validation failures is not evidence about the stored candidate: a database or +connection error while claiming, an executor or join failure, a column that +cannot be read, or an error serializing the parsed document. Such a claim +parks nothing and is left to expire and be retried. A failure of the parking +transaction itself is reported as in the table above; in particular, a +parking commit error can leave the row durably parked with an unknown +outcome. + +Find parked rows and their reasons with: + +```sql +SELECT block_hash, state, storage_version, attempt_count, last_error, updated_at +FROM qbit_block_candidate_outbox +WHERE state IN ('pending', 'offer_reserved', 'offered', 'reconciliation') + AND next_attempt_at = 'infinity' +ORDER BY updated_at; +``` + +Nothing unparks a row automatically. No binary resets `next_attempt_at`, +whether it is the one that parked the row or an earlier release. Reverting +the binary therefore leaves every parked row parked, because a claim selects +only due rows. Recovery is explicit operator work. `candidates recover` +(above) is the one command that claims a parked row, and only to land a +block the node already holds: it authenticates the row exactly as the claim +lane does and leaves the schedule untouched, so it does not make a row that +failed validation pass. First preserve the +row, from a backup or with `SELECT to_jsonb(o) FROM +qbit_block_candidate_outbox o WHERE block_hash = '<hash>'`. Then establish +why the stored evidence disagrees. Never edit the document, its digest, the +block bytes or the window columns to make a row pass: they are what the +claim authenticates. Once the cause is resolved, an operator may return the +row to the claim lanes with `UPDATE qbit_block_candidate_outbox SET +next_attempt_at = clock_timestamp() WHERE block_hash = '<hash>' AND +next_attempt_at = 'infinity'`. A row that still fails validation is parked +again with a fresh reason. Resetting the schedule of an `offer_reserved`, +`offered` or `reconciliation` row never grants another offer (see +**Recovery** above). + +**Timing.** `proof_observed_at_ms` is the enqueuing frontend's wall clock as +the locally validated block proof entered the coordinator; `offered_at_ms` is +the offering frontend's wall clock immediately before its `submitblock` call. +`qbit_prism_block_submit_seconds` observes their difference once per block, +on the frontend that made the call, after the call returned; that difference +includes the wait in the outbox before the claim, which the claim-to-offer +bound above excludes. The reservation +time is never substituted for the call time; a row without a proof time +(written before 011, or by a bare enqueue) yields no sample; a negative +interval is clock skew between the two hosts and yields none; a recovery on +another frontend records none. A crash between the call and the outcome +commit may lose that block's sample with the process (unless a scrape had +already read it), and the recovery never emits one, so the histogram is at +most once per block, not exactly once across crashes. + +**Rolling upgrade and quiesce.** A pre-011 frontend keeps a row `pending` +through its whole attempt and offers it after landing, or before landing for +a leased candidate; a post-011 frontend would reserve and offer such a row +again. The two must never share an outbox. Stop every pre-011 frontend and +let their claims expire (at most the 120 s lease) before migrating. Keep +supervisors and automatic restarts disabled throughout the cutover. Every +`qbit_prism_instances` row must explicitly report `stopped` or `drained`; +011 names and refuses any other instance, even with an empty outbox or a +stale heartbeat. Heartbeat age proves only that reporting stopped, not that +an idle or paused frontend cannot resume. Resolve blockers through graceful +shutdown; do not edit or delete instance evidence to bypass the check. +The scan holds a table lock through migration commit to serialize heartbeat +registration and updates. 011 also refuses any pending row with a live claim, +refuses an attempted +pending row that lacks its block bytes or window reference, quarantines every +pending row a pre-011 frontend attempted as `reconciliation` with an unknown +outcome (recovered without any offer, confirmed if the chain holds the block, +otherwise kept with its evidence for the operator), leaves never-attempted +rows pending, and declares the `candidate_offer_lifecycle` capability, which +a pre-011 binary refuses at connect and a post-011 binary requires at every +start and before any migration DDL: a database at 011 whose declaration is +missing or holds another value (a selectively restored capability table) is +refused with nothing changed, the refusal names the remedy, and the server +never repairs the declaration itself. 011 replaces only the lifecycle rules it +knows (001's state and payload rules, or #258's dual-format rule), recognised +by their definitions rather than their names; every other CHECK constraint on +the outbox that references neither `state` nor `completed_at` (by the +catalog's column dependencies) is kept exactly as it is, and one that +references either column, or a foreign constraint under one of 011's own +names, is refused by name with nothing changed, for the operator to drop and, +if it still holds for the offer lifecycle, re-create after the migration. +This conservative catalog policy writes no synthetic rows and executes no +operator predicate; it does not prove a kept CHECK compatible with later +writes, and a kept CHECK that rejects the quarantine or a lifecycle write +fails that write's transaction as any CHECK would. Signer rotation is +refused while any unfinished row stores other keys, in every unfinished +state. + +**Startup fence (012).** The instance-table lock alone cannot reject an old +startup queued behind migration. Migration 012 adds a database CHECK that +requires `candidate_offer_lifecycle: 1` in every `starting` heartbeat. This +binary writes that marker; a pre-011 binary cannot finish registering even +if it checked capabilities before the cutover. The constraint is checked +after the lock wait, including an upsert that would reuse a stopped instance +ID. Shutdown and historical health evidence are retained. Migration 012 and +011 commit together on a pre-011 database. A database already at 011 must +also stop all frontends gracefully before applying 012; it uses the same +explicit shutdown check. Restart with this binary after cutover. The new +`instance_offer_startup = 1` capability rejects earlier binaries at their +ordinary startup gate, and this binary refuses a missing or changed +012 declaration. Recovery exports require the same schema and declaration. + +**Orphan disposition upgrade (015).** Stop all frontends gracefully before +applying 015, including frontends already using the offer lifecycle. Keep +supervisors and automatic restarts disabled until migration finishes, and +restart only binaries that understand `candidate_orphan_disposition`. As with +011/012, every recorded instance must explicitly report `stopped` or `drained`; +an old heartbeat is not proof that a frontend cannot resume. The migration +refuses active instances before replacing the outbox constraints, and uses the +configured database lock timeout. Run it during a maintenance window: replacing +and validating CHECK constraints requires an exclusive outbox lock. +The capability gate runs at connect and does **not** evict an older frontend +that is already connected. Migration 012's startup marker proves support for +the offer lifecycle, not for the later orphan disposition; it cannot fence a +pre-015 startup already waiting on the migration lock. The shutdown and restart +procedure is therefore required, including disabling automatic restarts. +Migration 015 refuses modified named lifecycle rules and additional CHECKs that +reference `state` or `completed_at`, leaving the schema unchanged on refusal. +Resolve those constraints explicitly before retrying; do not bypass the check. + +PostgreSQL and qbitd do not share a transaction. Accounting effects are +idempotent and claim-fenced. Do not infer active-chain acceptance from a +socket write or a missing RPC reply: an offered block is confirmed only by a +fresh active-chain observation. + +## Candidate claim leases and database clock steps (021, #581) + +Every outbox deadline used to be the database clock: `next_attempt_at` +(`clock_timestamp()` + backoff) and `claim_expires_at` (`clock_timestamp()` + +lease), compared with `clock_timestamp()` by the claim lanes and by every +write the holder made. A database clock step moved all of them by the step. +Forward, a live claim looked expired at once, another frontend took the row +while the holder was still offering its block, and the holder's next write +was refused. Backward, a dead holder's row waited lease + step, and a retry +waited backoff + step (the #586 clock-jump test's stranded reconciliation +row, two hours). Since 021 neither decision reads the database clock's +estimate of a lease, and a retry the clock stepped back over is due at once. + +**Leases.** A claim's version is (`claim_token`, `claim_renewals`): a claim +writes renewals 0, every renewal adds one. Its lease is +`claim_lease_seconds`, written by the claim and by every renewal (1 to 600, +120 for the submit loop and for `candidates recover`). + +- Every claim poll (`Ledger::claim_survey_sql`) reads the version and lease + of every claimed unfinished row, and the frontend starts its own monotonic + clock for a version the first time a reply shows it. That reply arrives + after the commit that wrote the version, which comes after the instant the + holder started the renewal its own lease is measured from, so the holder's + lease always ends first, whatever either wall clock or the database clock + reads. +- A frontend takes a claimed row over only once it has watched the same + version for the whole lease, and only by compare and set on that version + (`FOR UPDATE SKIP LOCKED`, so a landing transaction holding the row is + never taken mid-commit). Any renewal, release or other takeover since + changes the version, and the takeover changes nothing. A takeover runs + before the dispatch probe and the claim lanes and takes no scheduling slot; + the lanes and the probe serve unclaimed rows only. +- A claimed row is taken over whatever its `next_attempt_at`, unless it is + parked (`infinity`). Its schedule is the one it was claimed with, on the + clock of that moment, and nothing rewrites it while the claim lasts, while + every renewal rewrites `updated_at`: after a backward step and one renewal, + neither says the row is due, and a dead holder's row would otherwise wait + out the step. The lanes claim only due rows, so only a `candidates recover` + claim of a row still in backoff can be taken over before its backoff ends. +- The holder's writes are fenced on its token alone. Its lease is its own + monotonic deadline (`with_tracked_heartbeat`: the start of its last + successful renewal plus the lease), which stops its work before any + observer's clock can end the lease. A renewal that times out on lock + contention only proves the token still holds the row; it never extends the + deadline. +- `claim_expires_at` is still written, as the database clock's estimate of + the lease's end, for `candidates list` and the operator. No decision reads + it, except `candidates abandon` and `candidates recover` with + `--unsafe-database-clock-expiry`. + +What this costs: a frontend that starts while a dead holder's claim is in the +table times it from its own first reply, so that takeover comes up to one +lease (120 s) later than the claim's true end, never earlier. A claim taken +before 021, which recorded no lease, is timed as the 600-second maximum. +Monotonic clocks are not wall clocks: a holder whose host was suspended past +its lease resumes believing its lease is live, exactly as before 021, and +its token fence and the offer-before-landing lifecycle (a recovered +`offer_reserved` row is never offered again) keep that safe. + +**Retries.** Every statement that schedules a retry writes `updated_at` from +the same database clock as `next_attempt_at`. A row whose last write is later +than the clock now proves the clock stepped back since, so its deadline is +due at once: each claim poll reschedules such unclaimed rows to the clock (and +logs `candidate retries were last written later than the database clock reads +now, so the clock stepped back`). `candidates recover`'s release keeps the +row's schedule, which its claim's own write may have hidden a step from, but +never further ahead than the row's own backoff (`min(60, attempt_count)` +seconds, `min(3600, 10 * attempt_count)` in reconciliation). A backward step +therefore delays a retry by less than its own backoff, never by the step. A +forward step makes a retry due early by up to its backoff; retries are pacing, +not a safety property. A row parked for the operator (`next_attempt_at = +'infinity'`) is never made due. A deadline set far ahead without a later +`updated_at` (a test or operator hold) stays held by the claim polls, but a +`candidates recover` release pulls it in to the row's own backoff. + +**Revoking a claim.** `ledger::revoke_candidate_claims` sets +`claim_lease_seconds = 0` (and moves `claim_expires_at` into the past), so the +next frontend to read the claim takes it over at once. The holder's token +stays until then. The lease tests use it where waiting out a real lease is not +deterministic. An operator may run the same statement only once the holder is +known to be gone (its process stopped, or its host down for good): + +```sql +UPDATE qbit_block_candidate_outbox +SET claim_lease_seconds = 0, + claim_expires_at = LEAST(claim_expires_at, clock_timestamp() - interval '1 second') +WHERE block_hash = '<hash>' AND claim_token IS NOT NULL; +``` + +**Upgrade (offline, per D5).** 021 adds the two columns, the +`qbit_block_candidate_outbox_claimed_idx` partial index the claim poll reads +claimed rows through, and the `candidate_claim_observed_lease = 1` +capability. A pre-021 frontend takes claims over by the database clock and +never writes the columns a post-021 frontend times, so the two must never +run together. As for 018: stop every earlier frontend and one-shot tool, +disable automatic restarts, and let every `qbit_prism_instances` row report +`stopped` or `drained`; the migrator refuses any other instance before +applying 021 and holds the registration lock through the commit. The +capability refuses older binaries at every later connect; it cannot evict one +that is already running. Start only post-021 binaries after the commit. +Removing the capability or the columns is not a supported downgrade. + +CTV fanout claims follow the same contract since 022; see the next section. + +## CTV fanout claim leases and database clock steps (022, #654) + +Before 022 a CTV fanout claim was decided as candidate claims were before +021: the claim lane took a fanout once `claim_expires_at` was before +`clock_timestamp()` and its `next_broadcast_attempt_at` was due, and every +write the holder made required `claim_expires_at` in the future. A forward +database clock step made a live claim look expired: another frontend took the +fanout over and broadcast the identical transaction (the node deduplicates +it), and the holder's settlement write was refused and the attempt retried. +A backward step made a dead holder's claim wait lease + step, and an attempt +or a confirmation check scheduled before the step wait its delay + step. +Nothing was broadcast twice in a way the node does not deduplicate; the cost +was liveness. Since 022 a fanout claim is timed exactly as a candidate claim +is (previous section): + +- **Leases.** A claim's version is (`claim_token`, `claim_renewals`) on + `qbit_ctv_fanout_artifacts`, and its lease `claim_lease_seconds` (1 to + 600; the broadcaster takes 120 at every claim and renewal). Every fanout + claim poll (`Ledger::fanout_claim_survey_sql`) reads every claimed fanout's + version through the `qbit_ctv_fanout_artifacts_claimed_idx` partial index, + and a frontend takes a claimed fanout over only once it has watched the same + version for the whole lease on its own monotonic clock, by compare and set + on that version (`FOR UPDATE SKIP LOCKED`, so a holder's fenced write that + holds the row is never taken over mid-commit). The takeover runs before + the lane, which takes only unclaimed, due fanouts. A claimed fanout is taken + over whatever its schedule (the one it was due at when claimed, on the + clock of that moment), unless it is held at `infinity`. +- **Holder.** Every write the holder makes (renewal, scan progress, CPFP + reservation, package and wallet cleanup, settlement and the hand-backs) is + fenced on its token alone. Its lease is its own monotonic deadline: one + attempt runs for at most 90 seconds, less than the 120-second lease every + renewal takes, and every renewal is sent inside the attempt, so the attempt + ends before any observer can end the lease of any version it wrote. Its + completion after that deadline is a compare and set on the token, refused + once a takeover has replaced it. +- `claim_expires_at` is still written, as the database clock's estimate of + the lease's end, for operators. No decision reads it. + +What this costs is what it costs candidates: a frontend that starts while a +dead holder's claim is in the table times it from its own first poll, so that +takeover comes up to one lease (120 s) later than the claim's true end, never +earlier. The one-shot `broadcast-ctv` times claims from its own first poll +too, so within one pass it takes over only a revoked claim; a running +frontend's broadcaster takes over a dead one's. To push a dead holder's +fanout with `broadcast-ctv` while no frontend is running, revoke its claim +first, as below. A claim taken before 022, which recorded no lease, is timed +as the 600-second maximum. + +**Attempts and checks.** Every statement that schedules +`next_broadcast_attempt_at` writes `updated_at` from the same database clock. +Each fanout claim poll makes due, at once, every unclaimed fanout the lane +could take whose last write is later than the clock now, which proves the +clock stepped back (it logs `CTV fanout attempts were last scheduled later +than the database clock reads now, so the clock stepped back`). A claim handed +back at shutdown (`release_fanout_claim`) keeps its schedule, but never later +than the clock: the fanout was due when it was claimed, and the release's own +`updated_at` would otherwise hide a step from every claim poll. A backward +step therefore delays an attempt or a confirmation check by less than its own +delay, never by the step; a forward step makes them due early. A fanout held +at `infinity` is never made due, and a schedule set far ahead without a later +`updated_at` (a test or operator hold) stays held while the fanout is +unclaimed. A claimed fanout is held only at `infinity`: its own schedule is no +hold, because after a backward step it reads ahead although its holder died, +so the takeover does not wait for it. + +**Revoking a claim.** `ledger::revoke_fanout_claims` sets +`claim_lease_seconds = 0` (and moves `claim_expires_at` into the past), so the +next frontend to read the claim takes it over at once; the holder's token +stays until then. The fanout tests use it where waiting out a real lease is +not deterministic. An operator may run the same statement only once the +holder is known to be gone: + +```sql +UPDATE qbit_ctv_fanout_artifacts +SET claim_lease_seconds = 0, + claim_expires_at = LEAST(claim_expires_at, clock_timestamp() - interval '1 second') +WHERE fanout_txid = '<txid>' AND claim_token IS NOT NULL; +``` + +**Upgrade (offline, per D5).** 022 adds the two columns, the +`qbit_ctv_fanout_artifacts_claimed_idx` partial index and the +`fanout_claim_observed_lease = 1` capability. A pre-022 frontend or +`broadcast-ctv` takes fanout claims over by the database clock and never +writes the columns a post-022 one times, so the two must never run together. +Upgrade exactly as for 021, and together with it when both are pending: stop +every earlier frontend and one-shot tool, disable automatic restarts, and let +every `qbit_prism_instances` row report `stopped` or `drained`; the migrator +refuses any other instance before applying 022 and holds the registration +lock through the commit. The capability refuses older binaries at every later +connect; it cannot evict one that is already running. Start only post-022 +binaries after the commit. Removing the capability or the columns is not a +supported downgrade. + +### The claim lane and settled history (024, #668) + +Settled fanouts stay in `qbit_ctv_fanout_artifacts`: a fanout's row remains +after its 1,000th confirmation, and mainnet adds about 65,000 to 130,000 a +year at #521's block rate. Before 024 the claim lane's selection matched them +on its status and schedule predicates and dropped them only after reading +them, a sequential scan of the whole table on every claim. A broadcaster pass +claims once per fanout it attempts, up to `PRISM_CTV_BROADCASTER_LIMIT`. + +Since 024 the lane (`Ledger::fanout_lane_sql`) gathers its candidates first, +into one array, and looks only those up by primary key: +- the due fanouts among those it may still claim, which are every fanout + still to broadcast or check (`broadcastable`, `broadcast_submitted`, + `failed`) and every confirmed one under 1,000 deep. They are read from the + partial index `qbit_ctv_fanout_artifacts_lane_idx`, keyed by + `next_broadcast_attempt_at`, as two ranges of it: never attempted (`NULL`) + and scheduled at or before `statement_timestamp()`; +- the newest deep fanout, the checkpoint reconciliation keeps watching, + through `qbit_prism_fanout_checkpoint_idx`. + +It then applies its full predicate and order to them, as before. Due is read +at `statement_timestamp()` so the index can serve it as a bound, which only +means a fanout falling due during the claim statement itself waits for the +next poll. The candidate query repeats the index's predicate, which must stay +identical to it, or the planner cannot use the index. + +The survey every claim poll opens with (#654) reads the fanouts scheduled +later than now through the broadcast index (`settlement_status`, +`next_broadcast_attempt_at`): the watched fanouts waiting for their next +check, never the settled ones, whose schedules are past. +`ledger_postgres::fanout_lane_plan` plans and runs both statements, prepared, +over 20,000 settled and 500 watched fanouts. It fails if either plans a +sequential scan of the table, if the lane does not read its index or visits +more rows than the few due, or if the survey visits more than the watched +fanouts. + +Measured on PostgreSQL 16 with a warm cache, the lane statement prepared, per +claim, with 7,500 watched fanouts: several times the 1,000 to 2,000 that +today's block rate keeps under 1,000 deep. + +| settled fanouts | due now | before 024 | after 024 | +| ---: | ---: | ---: | ---: | +| 100,000 | 377 | 63 ms, sequential scan | 1.6 ms | +| 100,000 | 7,502, all | 67 ms, sequential scan | 28 ms | +| 1,000,000 | 377 | 357 to 372 ms, sequential scan | 1.8 to 2.0 ms | +| 1,000,000 | 7,502, all | 412 ms, sequential scan | 38 ms | + +After 024 a claim costs what the due fanouts cost: one index entry and one +primary-key lookup each, then a sort for the first. The settled history only +deepens the primary key. A confirmed fanout under 1,000 deep is checked again +5 seconds after each check, so a broadcaster that cannot check every watched +fanout that often finds most of them due, and each claim then reads them all, +as in the second and fourth rows. The survey took under 6 ms in the same +runs. + +**Upgrade.** 024 is additive, as 019 and 020 are: no capability and no +shutdown proof, and it needs no frontend stopped. A binary that does not know +the index never reads it. It is applied online, as 013 is: on an existing +ledger `migrate` (or a start with `PRISM_POSTGRES_INIT_SCHEMA=1`) builds the +index with `CREATE INDEX CONCURRENTLY` after the migration transaction +commits, on its +own connection without statement or lock timeouts, and records 24 once the +index is valid. No write to `qbit_ctv_fanout_artifacts` waits for the build, +a found block's landing and the broadcasters of frontends still running +included. The build reads the table twice and waits for the transactions +already open to finish: about 0.6 s for 1,000,000 settled fanouts on a warm +cache, against 0.2 s for a plain `CREATE INDEX`. Until 24 is recorded every start +of a 024 binary refuses the database, as for any missing migration. An +interrupted build leaves an invalid index that the next run drops and +builds again. A fresh or empty source applies 024 inside the migration +transaction. With every frontend and tool stopped, +`migrate --offline-indexes` builds it with a plain `CREATE INDEX` in one +transaction instead, as it builds 013's +([migration 013](prism-rust-migration.md#migration-013-the-share-ledger-index-trim-applied-online)). + +## Chain observation epoch upgrade (018) + +Migration 018 adds `qbit_prism_cluster.chain_epoch` and declares +`chain_observation_epoch = 1`. The counter changes atomically with every +accepted chain checkpoint and payout revision. Accounting-only revision +updates do not change it. Equal-work observations bind their transition to +the original epoch before node I/O, so a peer's A-to-C-to-A round trip cannot +be mistaken for accounting-only drift. Fresh retries retain that original +epoch; cancellation, unknown COMMIT outcomes and failed publication cannot +rearm them. Lower-work refusals retain the previous local tip without +restoring consumed retry authority. + +Epochs order fresh observation attempts against committed cluster changes; +they do not timestamp node choices that occurred between polls. A new +observation started after a completed peer round trip can establish a new +transition from the currently accepted predecessor, using the current epoch. +That is distinct from retaining an older in-flight witness across the round +trip. Subsequent unchanged polls cannot repeat the replacement. + +A scheduler tick or wake permits at most one immediate fresh retry after a +definite accounting-only refusal. Another refusal returns to normal polling, +and shutdown prevents the extra attempt. The original witness epoch and all +publication checks still apply; repeated concurrent interference has no +unconditional two-second completion guarantee. + +This is an **offline development-line upgrade**, not a rolling upgrade. +Stop every earlier frontend and one-shot writer, disable automatic restarts, +and finish or cancel any old startup already past its capability check. +Every registered instance must explicitly report `stopped` or `drained`; +do not delete instance rows or treat heartbeat expiry as shutdown. The +migrator holds the registration lock through the schema commit and refuses +active instances before applying 018. That database check cannot discover an +unregistered old tool or evict an already connected process: excluding all +old writers is an operator prerequisite. Start only epoch-aware binaries +after commit. No production deployment is implied by this development change. + +Existing checkpoint and accounting values are preserved; epoch begins at zero +because old processes and their observations no longer exist at the offline +boundary. Restart, accounting, policy changes and fatal-state recovery never +reset it. The capability refuses older binaries on subsequent connection, +including operator tools that use the same gate. Removing the capability or +resetting the counter is not a supported downgrade. Restoring an older full +backup requires all writers stopped and the existing accounting-reconciliation +procedure; post-upgrade shares and candidates must not be silently discarded. + +Recovery evidence includes every nonzero epoch in `chain_checkpoint`, while +the zero default preserves pre/post-migration evidence equivalence. Restore +the whole cluster checkpoint, never its epoch independently. Issued balances, +WindowRef, prepared/job/candidate payloads, protocol and payout/candidate +revision fences are unchanged. A cold conflicting equal-work frontend still +waits for convergence or more work; this introduces no authoritative-node +setting. Migration numbers 016/017 belong to the separate share-partitioning +change; 018 is independently required on this branch. + +## Blocks, balances, and reorgs + +Core durable tables remain: + +| Table | Purpose | +| --- | --- | +| `qbit_share_ledger` | Canonical accepted share history | +| `qbit_pool_blocks` | Pool blocks and active-chain/maturity state | +| `qbit_pool_payout_entries` | Per-recipient payout records | +| `qbit_payout_carry_forward` | Auditable balance deltas | +| `qbit_pool_audit_bundles` | Canonical audit metadata and stored body | +| `qbit_ctv_fanout_sets` | Committed fanout sets | +| `qbit_ctv_fanout_artifacts` | Transactions, maturity, and broadcast state | + +Current balances replay active confirmed carry-forward deltas. Zero net balances +need no current row; negative balances remain visible debt offsetting future +rewards. `qbit_carry_forward_integrity_report()` checks stored prior, candidate, +and carry values against replay. The legacy operator report added +`audit_head_sha256` over active carry rows; the SQL function alone does not +calculate it. The recovery export below reproduces that exact head. Preserve it +with independent release/recovery records. + +At production size the report takes about a minute, longer than the 15 s +`PRISM_DATABASE_STATEMENT_TIMEOUT_MS` gives every ledger statement by default. +`self-check` runs it in a read-only transaction under a statement timeout of its +own, 300 s, and `fatal-state clear` under the time its bound leaves it +([fatal-state recovery](#recovery-commands)), so neither needs that setting +raised (#737). The `/audit/carry-forward-integrity` and `/audit/ledger-integrity` +routes still run it under the setting. + +The report holds one snapshot for its whole run, about a minute at production +size today. On a primary serving miners every share updates the cluster row, so +at high share rates that snapshot piles up dead versions of the row and slows +the share append (#738); below about 200 shares/s it's harmless. At higher +rates, run the report on a hot standby with `hot_standby_feedback` off and its +replay paused instead, as the admission gate's +`PRISM_INTEGRITY_REPORT_DATABASE_URL` does (#740; see +[drain and migrate](prism-rust-migration.md#drain-and-migrate)), or run +`self-check` while the share rate is low. + +A landing writes the block's payout and carry rows from the immutable payout +manifest of the audit its coinbase commits to, whatever the canonical balances +are when it lands, and since migration 011 marks the block with that audit's +digest (`qbit_pool_blocks.as_issued_audit_sha256`) in the same transaction. A +pending, never-offered candidate whose reference balances are no longer +current is refused (it is superseded work); an offered block lands as issued. +Current balances still sum the active per-block `(gross − onchain)` deltas, so +a block that lands after another block moved a miner's balance adds its own +issued deltas, and a miner paid a carried balance on chain by both blocks +carries the difference as visible debt. No commitment is rebuilt. Each block +whose work predates a landing pays that landing's paydown again, so a carry can +be paid a third time by a third such block; see +[block capture and payout divergence](#block-capture-and-payout-divergence-478) +for how every such landing is recorded and bounded. `qbit_carry_forward_integrity_mismatches()` validates a +marked block against that manifest: the arithmetic of every manifest account, +fee recipients included; each carry row field by field against the matching +miner account; and the payout entries against every account. A marked block +whose audit or manifest is missing is a finding by itself. Blocks landed before +011 keep the sequential rule. `qbit_carry_forward_current_drift()` is unchanged. + +Since migration 025 (#708), the sequential rule follows each payout program's +chain, as 2.x keeps it: `qbit_current_carry_forward_balances()` sums a +program's rows whatever label they carry. +- **One program, two labels.** Stratum keeps the address as the miner typed + it, and bech32 is case-insensitive, so one program can be paid under an + uppercase and a lowercase label. +- **What 025 fixed.** On union, one program's balance runs across both + labels. Before 025, the rule partitioned by + `(miner_id, payout_order_key, p2mr_program)` and reported 5,506 false + mismatches there, which `self-check` and `fatal-state clear` refuse. Per + program it finds none over all 2,753,849 active rows. +- **Labels are untouched.** The labels stay as written; the as-issued + manifests name them too. +- **The recovery export.** The summarizer re-checks the legacy chain per + program from the exported active carry rows. It clears only the findings + 2.x's per-label rule makes on a chain that holds, so a 2.x source and its + migrated copy summarize alike. +- **Migration order.** 025 replaces the validator 011 defines, so `migrate` + runs 025 again whenever it runs 011. + +Since migration 026 (#737), `qbit_carry_forward_integrity_report()` runs the +validator and `qbit_carry_forward_current_drift()` once each. Before 026 it +ran each twice, once to count its rows and once to list them. +- **The time.** On union's 2,753,849 active rows EXPLAIN ANALYZE put the + validator at 20.29 s and the drift check at 9.14 s, and the report took + about 57 s. Once each is estimated at about 29 s, their sum, not a timed + run. `self-check`, `fatal-state clear` and the import admission gate all + wait for it. +- **The same report.** 026 keeps 001's subqueries word for word, runs them + in 001's order and takes each count from its listing, so the report is + evaluated the same way as 001's. Findings that tie on the listing's sort + keys come out in the order those subqueries leave them, as in 001: a + block's findings without a carry row share its height and a NULL + `carry_forward_seq`. + +Coinbase maturity is 1,000 blocks: a height-H payout becomes mature only at tip +height H+1,000 or later. An immature disconnected block is marked inactive, so +its balances stop contributing; it can reactivate. Terminal reversal preserves +audit history and marks payout/carry rows reversed. A mature disconnect sets a +shared fatal state and stops ordinary accounting until investigated. Other +instances must not continue with a different interpretation of that event. + +## Block capture and payout divergence (#478) + +Every own block that lands bumps the payout revision twice on the same tip: +once when a frontend observes the block as the new tip, and once when the +block's landing is confirmed and its payout changes the canonical balances. +Work issued between the two carries the earlier revision. Before #478 a block +found on such work was refused `stale-job` at submit and a pending one was +abandoned at offer, so a valid block on the active tip was dropped with its +whole coinbase. + +A block-bearing proof on the active tip is now captured whatever its payout +revision. Its share is deferred: it is credited only if the block confirms, +never against the current revision. Plain shares on superseded work are still +refused. Work a Stratum session retired at a same-parent payout replacement +is kept for a block only: it holds no username reservation, it gets no stale +grace, parent grace or deferred share, and it is dropped once its parent is +no longer the tip. A block found on it is captured, and its share is refused +`stale-job`. + +### The overpay a captured block can cause + +A captured block's coinbase pays the prior balances its work was issued with. +If a landing has paid those balances down by the time the captured block's +rows start to count, the miner is paid twice against one balance. The additive +balance carries the difference as debt. The ledger stays exact: on-chain +amounts equal the coinbases, and `qbit_carry_forward_integrity_report()` shows +no mismatch or drift, because a marked as-issued landing is validated against +its own manifest. + +Debt is realized when a landed block's rows start to count, at its +confirmation. Per account `m`, it is the increase in `max(0, −balance(m))` at +that moment. + +The payout policy never pays an account more than its candidate balance. So a +block's on-chain amount minus its gross is at most `max(0, issued(m))`, and +the debt the block creates for `m` is at most `max(0, issued(m))`. This holds +whatever confirms before or after it. Summed over accounts, a block's realized +debt is at most `F`, the positive as-issued float: the sum of +`max(0, issued(m))` over the accounts of its own as-issued balances. A bound +computed from the balances at the offer is not enough. A later block that is +current at its own offer can confirm first and pay the same carry out, which +leaves the captured block to realize its whole float. + +The offer reservation first reads the payout revision `FOR SHARE`. Every +revision bump updates that row, so no bump commits between that read and the +reservation's commit: a block whose revision is still current there is +reserved current. For a block whose revision was already superseded, the +reservation reads the block's own immutable as-issued snapshot and computes +`F`. It then records the decision and reserves the block, or refuses it, in +one transaction. + +- **Within the ceiling.** If `F` is at most + `PRISM_CAPTURE_OVERPAY_CEILING_BPS` of the block's coinbase value (default + 100 bps, 1%), the block is reserved for its one offer. +- **Over the ceiling.** The block is abandoned without being offered. The + bound and the ceiling go in `last_error`, a WARN log is written, and + `qbit_prism_capture_offer_decisions_total{decision="abandoned_ceiling"}` + counts it. Its miner is answered `stale-job`, counted under + `payout_revision`, as base answered such a proof at submit. Whenever the + pool's positive carry float exceeds the ceiling, captures are refused. The + pool then loses those blocks exactly as it did before #478. +- **Unknown.** If the bound cannot be computed, for example because of a + database error or a missing as-issued snapshot, nothing is reserved and + nothing is abandoned. The offer fails and is retried like any other offer + failure. +- **Off.** `PRISM_CAPTURE_OVERPAY_CEILING_BPS=0` is the pre-#478 behaviour: + - Such a proof is refused `stale-job` at submit. + - A pending block whose revision moved is abandoned at offer with + `payout revision or parent superseded`. + - The abandonment is recorded as decision `disabled`, with no bound + computed, and counted as `abandoned_disabled`. This holds on the ordinary + pre-offer path and when a reservation races the bump. + +A captured block abandoned before its offer because its parent is no longer +the tip (a lost race) is answered `stale-job` too, counted under +`parent_grace`. An abandonment before the offer is the pool's own staleness +decision on a block the ledger wrote correctly, so it never answers +`ledger-confirmation-failed`. That answer feeds the share-append failure +warning. + +The ceiling bounds, in every order of confirmations, each block whose payout +revision was already superseded when its offer was reserved. It does not +bound: + +- a block reserved current that lands divergent because an earlier landing + confirmed after its reservation, which is the pre-#478 double-payment + class; +- a leased candidate (#350), which is reserved without the check, as base + offers it before any supersession check. + +Both are recorded at their confirmation, without an offer decision. + +### Every divergence is recorded + +Migration 020 adds `qbit_prism_payout_divergences`, with one row per block +that either had an offer decision or diverged at its confirmation: + +- The offer decision: `offered`, `abandoned` by the ceiling, or `disabled` + because capture is off. The row also holds the observed revision, the bound + `F` (none for `disabled`) and the ceiling in sats. +- The confirmation: the digest of the canonical balances the block's rows + met, `divergent_accounts`, `overpay_sats` (the debt this confirmation + created), `overpaid_debt_after_sats` and `pool_debt_after_sats`. + `divergent_accounts` counts every account whose balance moved between + issue and confirmation, whether it was overpaid or underpaid. The overpaid + ones are the rows in `qbit_prism_payout_divergence_accounts`. + +The confirmation is recorded in the transaction that makes the rows count: +the settlement's confirmation or the chain reconciler's, in height order when +several blocks confirm together. A re-confirmation after a reorg replaces +these fields. `qbit_prism_payout_divergence_accounts` has one row per overpaid +account, with its as-issued and current priors, gross, on-chain, overpay and +resulting debt. + +For the blocks that count now, the recorded `overpay_sats` sum to the debt +those confirmations created. The debt fields are read from the current +balances. They equal that sum until an indebted miner's later gross repays +part of it; after that they are lower. + +The integrity report served at `/audit/carry-forward-integrity`, and printed +by `self-check`, carries a `payout_divergence` line from +`qbit_prism_payout_divergence_report()`. The line never fails a check. It +shows: + +- the confirmed divergent landings and their total overpay + (`divergent_landings`, `overpay_sats`); +- the offers abandoned by the ceiling (`offers_abandoned_by_ceiling`); +- the offers refused because capture is off (`offers_refused_capture_off`); +- the debtor count, the total debt and the largest debt, from the current + balances (`debtor_count`, `debt_sats`, `largest_debt_sats`); +- the live positive float, `positive_float_sats` (#506): the sum of + `max(0, balance)` over the current balances, one per account. It is the `F` + that work issued against these balances would be held to if it were + captured. See [Sizing the ceiling](#sizing-the-ceiling). + +The line is read in one statement, so the float and the debt fields come from +the same snapshot. Sats amounts are decimal strings. `positive_float_sats` is +added by the frontend, not by migration 020, so a line served by a frontend +without #506 does not have it. A missing field means that frontend cannot +report the float; it does not mean the float is zero. + +The metrics: + +- `qbit_prism_capture_offer_decisions_total{decision}` counts the offer + decisions. +- `qbit_prism_divergent_landings_total` and + `qbit_prism_divergent_landing_overpay_sats_total` count divergent + confirmations and their debt, on the frontend that committed them. +- `qbit_prism_carry_forward_debt_sats` is read from the canonical balances + after every balance change and every full refresh a frontend commits. + +### Accepted cost and its bound + +A miner's debt is recovered only from that miner's later gross. A miner who +stops mining keeps it, and it is typically the miner whose carry the earlier +landing paid out. The remaining miners then bear it as claims the pool cannot +fund from that miner. This is an accepted cost of capturing blocks that were +previously dropped whole, within these bounds: + +- **Per captured block, in every order:** the debt it creates is at most its + positive as-issued float `F`. The offer holds `F` to the ceiling, so it is at + most 1% of the block's coinbase value by default. `F` is the positive carry + float of the pool when the block's work was issued. It grows with the + number of miners holding a balance below the payout floor (14,720 sats by + default): about 2.9·10⁷ sats at 2,000 such miners, against a 5·10⁷-sat + default ceiling. See [Sizing the ceiling](#sizing-the-ceiling). +- **Per day, at the #224 cadence** (35 own blocks in a trailing hour, pairs + 18 to 20 s apart; at most 840 own blocks a day): at most 840 times the + ceiling, 8.4 coinbase values at the default. That worst case needs every + own block to be a capture and each capture to pay its whole float out + again. A capture needs an own parent and a block found between that + parent's observation and the delivery of its post-landing work. That window + is `W`, about 1 s at the #275 bar. The realized figure is + `qbit_prism_divergent_landing_overpay_sats_total` and the report line. +- **What it replaces:** each capture keeps a coinbase the pool previously lost + entirely. At the ceiling, a capture still keeps at least 99% of its value + for the pool. + +The same double payment also happens without capture, for a block offered +before an earlier landing confirmed. Such a confirmation is recorded the same +way, without an offer decision, and is not held to the ceiling. + +### Sizing the ceiling + +The default of 100 bps is a starting point, not a value known to fit a given +pool. A capture is refused whenever its float `F` exceeds the ceiling, so a +ceiling below the pool's positive carry float refuses every capture and the +pool loses those blocks as it did before #478. + +- **Check it against the live float** before relying on capture, and again + as the miner count grows. The live float is `positive_float_sats` on the + report's `payout_divergence` line. It equals the sum of `balance_sats` over + `/owed-balances` and + `SELECT sum(owed_balance_sats) FROM qbit_current_owed_balances()`. Each + offer decision also records the `F` it was checked against, in + `qbit_prism_payout_divergences.overpay_bound_sats`, next to + `overpay_ceiling_sats`. +- **Compare it with the ceiling in sats.** A block's ceiling is + `floor(coinbase_value_sats × PRISM_CAPTURE_OVERPAY_CEILING_BPS / 10000)`. + Work issued now can be captured if `positive_float_sats` is at most that. + The smallest setting that admits it is + `max(1, ceil(positive_float_sats × 10000 / coinbase_value_sats))` bps, + because 0 turns capture off. If that exceeds 10000, the largest accepted + setting, no setting admits the float. The line + shows no headroom because the ceiling depends on the next block's coinbase + value, subsidy plus fees, and on each frontend's setting, and the database + holds neither. Use the lowest coinbase value you expect, since a block with + fewer fees gets a lower ceiling. +- **Leave margin above the float you read.** A capture is held to the float + as it was when its work was issued, before the landing that superseded it. + A landing pays some balances out and accrues others, so the float in a + report can differ from the `F` of the next capture. Size against the + highest float you see across landings, or against the recorded + `overpay_bound_sats`, and add margin for miner growth. +- **Example.** Each miner below the payout floor carries under 14,720 sats. + At 2,000 such miners the float is at most about 2.9·10⁷ sats, inside the + 5·10⁷-sat default (1% of a 5·10⁹-sat coinbase). At about 3,400 miners all + near the floor it can exceed the default on its own, and the default then + refuses every capture. +- **Raise the ceiling if captures are abandoned by it.** Captures refused by + the ceiling show as + `qbit_prism_capture_offer_decisions_total{decision="abandoned_ceiling"}`, + as `offers_abandoned_by_ceiling` on the report's `payout_divergence` line, + and as a WARN log naming the bound and the ceiling. Raise + `PRISM_CAPTURE_OVERPAY_CEILING_BPS` above the recorded bounds, on every + frontend, and restart them all. The setting accepts up to 10000 (100%). + The per-capture and per-day bounds above scale with it. +- `F` is conservative. A block superseded only by a balance-neutral bump, + such as a maturity or fatal-clear bump, can create no more debt than a + current block, but it is still held to `F`. + +### Known record limitations + +These affect the divergence records and counters only. The balances, the +report's `debt_sats` and the `qbit_prism_carry_forward_debt_sats` gauge are +read from the canonical balances and stay exact. + +- **Reorg order.** The chain reconciler confirms and deactivates in one pass, + in height order. If one reconciliation confirms an own block below an own + block it deactivates, the lower block's confirmation is recorded against + balances that still count the higher block. Its `overpay_sats` can then be + too low (the deactivated block accrued to the account) or too high (it paid + the account down). This needs own blocks on both branches of a reorg. +- **Observed revision.** For a superseded block, the offer reads the revision + and computes `F` in one transaction and records the decision in a second. + A bump between them leaves `offer_observed_payout_revision` one revision + behind the one current at the decision. `F` does not depend on it; the + column is the revision at which the supersession was observed. + +### Upgrade and operations notes + +- **Rolling upgrade.** A frontend without #478 abandons a claimed captured + block with `payout revision or parent superseded`, exactly as before, and + its deferred share stays uncredited. The miner, waiting on a new frontend, + is answered `stale-job`. A frontend without #478 that lands or confirms a + block writes the as-issued rows and the ALERT log line, but no + `qbit_prism_payout_divergences` row and no metrics. That includes a + captured block a new frontend reserved. The records and metrics are + complete only once every frontend runs this release. Until then the + report's `divergent_landings` and `overpay_sats` can undercount, while + `debt_sats`, read from the balances, stays exact. Capture is reliable only + once every frontend runs this release. +- **The ceiling is per frontend.** `PRISM_CAPTURE_OVERPAY_CEILING_BPS` is read + by each frontend at startup and is not part of the cluster fingerprint. + Each offer is decided by the frontend that claimed it. Set the same value + for every frontend (the Compose file passes one value to all of them). To + change it, including to turn capture off with 0, restart every frontend. + Until the last one restarts, the others keep deciding with the old value. +- **Fatal-state clear.** The clear bumps the revision without moving the tip, + so a pre-fatal pending block on a still-current tip is now offered after the + clear, within the ceiling, instead of being abandoned. The clear already + requires stopped frontends and a clean integrity report. +- **Rejection causes.** A block-bearing proof from ordinary work on the + active tip is captured, not refused, so it no longer counts under + `stale_job_rejections_total{cause="payout_revision"}`. Answers still counted + there or under `parent_grace`: + - retired, block-only work: its plain shares, and the share of a block + captured from it, under `payout_revision`, or under `parent_grace` after a + tip change. Before #478 these were answered `unknown-job` with no cause + series. + - a capture abandoned before its offer: under `payout_revision` for a + ceiling refusal or capture off, under `parent_grace` for a lost race. +- **Acknowledgement.** A block captured from ordinary work defers its share: + the miner is acknowledged when the block confirms and the share is + credited, within the block-only acknowledgement bound (60 s), as for every + block-only proof, and the session's next work waits for that answer. A + block found on block-only work is enqueued with no deferred share, and its + share is refused `stale-job` as soon as the enqueue settles, because nothing + will ever credit it. + +## Audit storage and retention + +Native accepted-block audits store the non-share bundle fields and a +`share_snapshot_sha256` reference to `qbit_prism_audit_snapshots`. A snapshot +records the canonical share interval, anchor, count, and digest. Its shares are +reconstructed from the immutable ledger. Bootstrap snapshots may retain their +synthetic share inline. The reader verifies the reconstructed share digest and +canonical bundle SHA before returning a logical v1/v1.1 bundle. A row whose +shares have been archived is served from stored canonical bytes instead: before +a partition leaves the ledger every audit whose snapshot intersects it is +sealed, the artifact rebuilt from the still-online shares and checked against +its advertised `audit_bundle_sha256`, and the bytes kept in +`canonical_audit_bytes`, exactly as an imported legacy audit keeps them. +Readers prefer stored bytes over reconstruction whatever the row's shape, so +the artifact route, the audit bundle route and the operator tools keep serving +the block under its published digest. The snapshot metadata row stays: it +records the range and digest those bytes were proved against. + +Share UPDATE, DELETE, and TRUNCATE are prohibited, on the partitioned parent +and on every leaf. Migration 017 installs the immutability trigger on the +parent, and `qbit_prism_share_partition_create` installs it on each partition +it creates, because PostgreSQL does not clone a statement trigger to a +partition. Removing a share could break both future accounting and already +published audit hashes, and nothing in the native runtime deletes one: shares +are immutable and are never deleted. + +Retention is not deletion. Space is reclaimed a whole partition at a time, +after the partition has been sealed, written to an archive outside PostgreSQL, +and verified against the live rows: `DETACH PARTITION ... CONCURRENTLY`, then +`DROP TABLE` of the detached relation, with the archive as the copy of record +from then on. `qbit-prism-server share-archive` is that path and the only one; +there is still no row-level prune or share-compaction command, and no `DELETE` +runs on the ledger. Keep the canonical share history and all referenced +snapshot rows online until that procedure clears them, and keep the archive +under the same retention plan as the database backups. The procedure is +[below](#share-ledger-partitions-and-retention); the decision behind it is +decision D6 in [the design record](prism-share-ledger-partitioning.md). + +### Public audit artifact admission + +An artifact read of a native block is window-sized work: an unsealed block is +rebuilt from its share range, a sealed one has its stored bytes digested and +parsed, and neither result fits the public response cache at production window +sizes, so every distinct request repeats it. +`PRISM_PUBLIC_AUDIT_REBUILD_CONCURRENCY` (default 1) is how many of those run at +once, and it is deliberately not derived from `PRISM_POSTGRES_READ_CONCURRENCY`: +a larger read pool serves more dashboard reads without admitting a second +window into memory *on this route*. The limit does not extend to +`/public/v1/blocks/<hash>/settlement-artifacts`, which falls back to the +audit-bundle reader for a block with no CTV fanout set and rebuilds or decodes +the window under the shared imported-audit decode limit, still sized from +`PRISM_POSTGRES_READ_CONCURRENCY`; that route admits as many windows as the read +concurrency allows. Moving it under the artifact limit would change another +route's behaviour and is left as a follow-up. A request waits for its slot inside its own read deadline +(`PRISM_PUBLIC_READ_STATEMENT_TIMEOUT_SECONDS`, default 20 s) and answers 503 +`read_timeout` if the deadline runs out first, exactly as a slow query does. + +`PRISM_PUBLIC_AUDIT_ARTIFACT_MAX_IN_FLIGHT` (default 32) caps how many audit +artifact requests may run or queue at once. Past the cap a request is refused +immediately, after the route's two indexed point lookups and before any audit +read, with HTTP 503, the error code `audit_artifact_busy` and `Retry-After: 5` +— the only `Retry-After` the server sends. The refusal is never cached +(`Cache-Control: no-store`) and is counted in +`qbit_prism_public_audit_artifact_refusals_total` on the public service's +`/metrics`. CTV fanout manifests, which the same route serves from stored JSON, +are neither counted nor refused. With the response cache enabled (the default), +identical concurrent requests for one artifact collapse into a single +computation and occupy one place under the cap, so the cap bounds distinct +artifacts, not clients. A sustained stream of refusals means +either an unusually wide fan-out of distinct blocks or a rebuild concurrency +too low for the traffic; raise the cap only after checking that the memory bound +in [prism-storage-sizing.md](prism-storage-sizing.md#memory-bound-on-artifact-reads) +still holds. + +Imported historical external audits retain their verified canonical bytes; +they are not silently rewritten into references to potentially incomplete +legacy share history. Import preserves their published canonical SHA. Legacy +body-ref and v2 segment formats remain supported by the Rust offline loaders. +Back up external bodies and their segments until import and restore validation +are complete, and retain the original backup under the migration retention plan. + +## CTV recovery and broadcasting + +Committed fanout artifacts are durable database records. Broadcasting requires +a mature active parent and a current claim. Workers record transaction/package +outcomes and retry state so another instance can resume after a crash. Failed, +reorged, or completed artifacts remain visible through the audit/public API. + +To reconstruct missing artifact sets from verified database audits: + +```sh +qbit-prism-server backfill-ctv +``` + +Run `import-audits` first for historical file-backed bodies. Backfill verifies +the trusted ledger key, canonical digest, and recorded coinbase before repairing +rows; matching existing artifacts are idempotent. It processes stored audits, +replacing the former Python tool's individual-path and block-filter CLI. + +To process a single batch using the normal broadcaster policy: + +```sh +qbit-prism-server broadcast-ctv +``` + +The integrated periodic worker uses `PRISM_CTV_BROADCASTER_ENABLED=1`. An optional +CPFP wallet and fee configuration must be consistent with the intended operating +policy. Durable claims coordinate work across instances, timed on each +frontend's monotonic clock rather than the database clock (see +[CTV fanout claim leases](#ctv-fanout-claim-leases-and-database-clock-steps-022-654)); +node RPCs may still +receive an identical transaction more than once after a lost reply. The one-shot +`broadcast-ctv` verifies the node, the schema and the cluster fingerprint like a +frontend but registers no heartbeat: its claims are fenced by their own claim +tokens, so it never competes with a frontend's broadcaster for the same fanout +and leaves no `qbit_prism_instances` row for fatal-state recovery to refuse. + +Confirmed fanouts are observed every five seconds until 1,000 confirmations; +afterward the latest deep checkpoint is checked every 60 seconds. A shallow +fanout disconnect returns the transaction to broadcast work. Disconnection of a +deep checkpoint halts the shared cluster for explicit reconciliation. + +Between claimed fanouts a frontend's periodic pass yields to its own template +refresh: it stops at the fanout boundary while the detected tip is unpublished, +or when a tip observed after the pass began differs from the tip the pass +started from, counted by +`qbit_prism_ctv_fanout_broadcaster_tip_refresh_yields_total`, and the remaining +rows stay claimable for a later pass. The unpublished-tip yield lasts at most +`PRISM_TEMPLATE_REFRESH_FAILURE_EXIT_SECONDS` (default 120) from the first +departure, the same replacement-build budget as the published-tip lease, so a +refresh that keeps failing before publication cannot strand settlement. An +observation older than the pass never yields, so a refresh that has not caught +up with the node cannot strand it either, and same-tip polls never yield. + +Without transaction indexing, the broadcaster uses a durable block-scan cursor +and chain anchor. `PRISM_CTV_SPEND_SCAN_BLOCKS` bounds each pass (default 32, +range 1–256); a reorg resets the cursor. The node must retain the historical +blocks needed by that scan. A pruned/unavailable block range cannot be treated +as proof that a fanout is unspent. + +For positive CPFP sponsorship, use a dedicated wallet. Broadcasters that create +or recover unsigned packages must reach the same sponsorship wallet RPC service; +unrelated wallets with the same name cannot sign each other's reserved inputs. +Other nodes can replay an already signed package without opening that wallet. +The database reserves +funding before wallet locking and preserves the exact signed child before +submission. A replacement process recovers the reservation and replays that +same package rather than selecting fresh funding after a lost reply. Funding +is not unlocked until the node observes it spent, including a mempool spend. +Automatic replacement fee bumps and abandoned-reservation release are not +implemented; retain and reconcile the durable reservation when handling those +cases manually. + +## Block submission kill switch for rehearsals + +A rehearsal (#291) runs production frontends against a restored copy of the +mainnet ledger and a real mainnet node. Hold the restored ledger itself with +`submission-hold set` before any frontend starts on it +([below](#holding-the-whole-cluster-023-664)), so that it holds every frontend +that connects to it, and start every frontend of the rehearsal with +`PRISM_BLOCK_SUBMIT_ENABLED=0` as well, so that no found block and no +transaction it handles can reach the network: + +| Path | With `PRISM_BLOCK_SUBMIT_ENABLED=0` | +| --- | --- | +| Found blocks | Validated, credited and enqueued as usual, and the outbox row stays `pending`. The submit loop claims no row at all, so `submitblock` is never called, and no row, restored ones included, is reserved, offered, settled or abandoned. A block proof below the miner's share target is credited only once its block confirms, so it is answered `ledger-outcome-unknown` after `PRISM_SHARE_COMMIT_TIMEOUT_SECONDS`, as with a stuck outbox. | +| CTV fanouts | The periodic broadcaster does not start, even with `PRISM_CTV_BROADCASTER_ENABLED=1`, and `broadcast-ctv` exits non-zero before it connects. No `sendrawtransaction`, `submitpackage` or CPFP wallet call is made. | +| Everything else | Unchanged: templates, tip polling, readiness, Stratum, share acknowledgement, rollups and every write to the rehearsal database. `candidates recover` still lands blocks the node has already accepted; it never offers one. | + +Inside the process, the claim's processing and the broadcaster pass refuse +under the switch as well, and the frontend's node client, its wallet clients +included, refuses every call that would relay a block or a transaction +(`submitblock`, `sendrawtransaction`, `submitpackage` and the wallet's own +sends) before sending it. So no other caller reaches the network either. The +switch is per frontend and is read at startup; `0`, `false`, `no` and `off` +all turn it off. A frontend started without it offers every `pending` row it +can claim, held ones included, so set it on every frontend that shares the +rehearsal database, and restart a frontend to change it. As an independent +second layer, give the rehearsal frontends a node RPC user that cannot call +`submitblock`, `sendrawtransaction` or `submitpackage`, for example with +Bitcoin Core's `-rpcwhitelist` where the qbitd build supports it. + +Verify the mode before admitting miners: + +- `check-config` prints `WARNING: block submission is disabled by + PRISM_BLOCK_SUBMIT_ENABLED: ...` as its first line, followed by a second + warning when a configured CTV broadcaster is held. It still exits 0, in + production mode too, because a rehearsal on mainnet runs in production mode. +- `self-check` reports `"block_submission": {"enabled": false, ...}` for its + own environment and repeats the warning on stderr. Its + `live_instances.instances[].status.block_submission_enabled` is each running + frontend's own value, read from its heartbeat; every one must be `false`. A + frontend that is still starting is listed under `inactive_instances` until + its first health publication, so run `self-check` again once it is up. +- `/healthz` carries `block_submission_enabled`, and `/metrics` reads + `qbit_prism_block_submission_enabled 0` from startup. +- `qbit-prism-server submission-hold show` reports `"held": true`, and each + frontend's heartbeat carries `"held": true` in `block_submission_hold` + ([below](#holding-the-whole-cluster-023-664)). +- Each frontend logs a warning at startup that ends `the submit loop claims no + candidate`, and, when a broadcaster was configured, one that it is held. + +A held block looks like any other pending candidate: +`qbit_prism_block_candidates_pending` and +`qbit_prism_block_candidate_oldest_pending_seconds` count and age it, +`candidates list` shows it `pending`, and the pending-candidate alerts fire as +they would for a stuck outbox. `PrismBlockSubmissionHeld` pages a minute after +each held frontend starts, independently of any found block, because +`qbit_prism_block_submission_enabled` is not 1 (#666). All of that is expected +during a rehearsal. Scrape the rehearsal's frontends under another job or +network, or silence `PrismBlockSubmissionHeld` and the pending-candidate alerts +for the rehearsal, so they do not reach production paging. Scope such a silence +with matchers on the rehearsal frontends' `instance` labels, never on the alert +name alone: that would also hide a production frontend left held, which is what +`PrismBlockSubmissionHeld` exists to page. + +After the rehearsal, do not start a frontend with submission enabled against +the rehearsal database while held rows remain. That frontend would offer each +one to its node, and a leased candidate skips the pre-offer staleness screen. +Discard the database, or first abandon each held row with `candidates +abandon`. Production frontends run with the setting unset or `1`; the +[go-live checks](mainnet-deployment.md#go-live-checks) confirm it. + +### Holding the whole cluster (023, #664) + +`PRISM_BLOCK_SUBMIT_ENABLED` holds one frontend, and only one started with it. +A hold stored in the ledger holds every frontend that connects to that ledger, +whatever its own setting says, so a frontend started without the variable on +a rehearsal database still sends its node nothing: + +```sh +qbit-prism-server submission-hold set --reason "#291 rehearsal on a restored ledger" +qbit-prism-server submission-hold show +qbit-prism-server submission-hold clear --reason "<why>" [--offer-pending-candidates] +``` + +While the hold is set: + +- No frontend claims a block candidate, in any state, so `submitblock` is + never called and every row stays as it is. That includes `offered` and + `reconciliation` rows: the landing, crediting and fanouts of a block already + sent wait for the hold to clear. `candidates recover` still lands a block + the node has already accepted, and never offers one. +- An offer's reservation reads the hold in its own transaction, `FOR SHARE` + on the hold's own row, which `set` locks `FOR UPDATE`. A hold therefore + commits either before a reservation, which is then refused and leaves its + row `pending`, or after one that already reserved, whose single offer + completes. Nothing else locks that row and the read takes no lock on the + cluster row, so it adds no wait behind a landing or blob GC: a leased + candidate's reservation still locks nothing there. +- No frontend's CTV broadcaster claims or sends a fanout, and `broadcast-ctv` + refuses; confirmation tracking of fanouts already sent waits too. A hold set + during an attempt hands its claim back unattempted before the send; a send + already past that check completes. +- Everything else runs as usual, as under `PRISM_BLOCK_SUBMIT_ENABLED=0`. + +Running frontends obey a hold at their next claim, within a submit-loop tick, +and report it at their next health publication: + +- Each logs `the cluster holds block submission` once, and logs again when + the hold is cleared. +- `/healthz` and the heartbeat keep `block_submission_enabled` as the + frontend's own setting, so a frontend started without the variable still + shows, and carry the hold as `block_submission_hold`: `held`, `reason`, + `set_at` and `set_by`, as the frontend last read it. `held` is null until + its first read succeeds; a later failed read keeps the last one, so a + database outage does not report the frontend as held. +- `qbit_prism_block_submission_enabled` reads 0 while the setting or the hold + holds blocks back, and -1 while the setting is on but the frontend has not + yet read the hold. While a cluster is held, `PrismBlockSubmissionHeld` therefore pages for + every frontend: scope a rehearsal's or a cutover's alerting as + [above](#block-submission-kill-switch-for-rehearsals). +- `self-check` reports the hold as `submission_hold` and warns on stderr. + `check-config` never reads the database and says where the hold is reported. + +`set` and `clear` need only `PRISM_DATABASE_URL`, and both require `--reason`. +They use the operator connection, which works while frontends run and on a +halted cluster, and print the resulting hold as JSON. `set` keeps a hold +already in force, with its original reason. `clear` is refused while any +candidate is `pending`, because every frontend with submission enabled would +then offer those blocks; pass `--offer-pending-candidates` only when they +should be offered. `show` opens the database read-only, gives up after 15 +seconds, and also reads a ledger from before 023, which reports +`"schema_supports_hold": false`. + +Each `set` that holds the cluster and each `clear` that releases it is +journaled in the append-only `qbit_prism_submission_hold_events`, with who ran +it, why, and how many candidates were pending. A `set` that finds a hold in +force, a `clear` that finds none and a refused `clear` change nothing and +record nothing. `show` prints the latest event as `last_event`; the whole +history is: + +```sql +SELECT event_id, action, recorded_at, operator_identity, database_role, + reason, pending_candidates +FROM qbit_prism_submission_hold_events +ORDER BY event_id; +``` + +Migration 023 adds the hold's row and its journal and declares +`block_submission_hold = 1`, so a binary older than the hold, which would +ignore it, refuses the database at connect. 023 is applied offline in the same +stopped window as the rest of `migrate`, like 021 and 022. + +**Rehearsal.** Restore, run `migrate`, then `submission-hold set` before any +frontend starts, and confirm `"held": true` with `show`. Never clear the hold +on a rehearsal database: discard the database afterwards. + +**Cutover.** The hold can also keep a freshly migrated production ledger from +the network until the frontends are verified. Run `submission-hold set` after +`migrate`, in the stopped window. Start and verify the frontends. Then run +`submission-hold clear --reason ...`, which a drained source allows because it +has no pending candidates, and confirm `"held": false` with `show` before +admitting miners. The [go-live checks](mainnet-deployment.md#go-live-checks) +require it. + +## Retry, replay and deadline contract + +The ledger never replays SQL automatically. After a `statement_timeout` +(`PRISM_DATABASE_STATEMENT_TIMEOUT_MS`, default 15000), a `lock_timeout` +(`PRISM_DATABASE_LOCK_TIMEOUT_MS`, default 5000), a closed connection or a lost +acknowledgement, the error reaches the caller and the ledger neither re-sends +the statement nor re-runs the transaction. This is distinct from normal +traversal: a paged read sends its page query repeatedly by design, and audit +materialization reads the snapshot row and its share range through separate +pool checkouts. A statement cancelled or a socket closed before `COMMIT` +aborts the open transaction, so the claim and state that authorized the call +remain as they were; that holds only when the transaction did not commit. A +`CommandComplete` received before `COMMIT` proves execution, not durability. A +`COMMIT` whose acknowledgement is lost leaves the outcome unknown to the +caller: the transaction may or may not have committed. The caller's next +observation must come from the durable claim and state, never from a replay. +Every later attempt is a separate public operation with its own claim. + +| Operation (public API) | Class | Automatic re-execution | What runs later, and who authorizes it | Coverage | +| --- | --- | --- | --- | --- | +| Candidate terminal disposition: `finish_candidate`, `finish_candidate_at_revision` | never-retried mutation | none | The same live claim may invoke again after a reported error; otherwise the worker calls `retry_candidate`. A consumed claim is rejected and writes nothing. | dynamic: `candidate_terminal_timeout_executes_once` | +| Candidate backoff: `retry_candidate` | explicit later operation | none | Releases the claim, records the error and advances `next_attempt_at` by `min(60, attempt_count)` seconds, or `min(3600, 10 × attempt_count)` for a `reconciliation` row. It does not re-run the failed write and never changes the row's lifecycle state. A fresh `claim_candidate` after that time is the next attempt, with a new token; the old token stays rejected. | dynamic: `candidate_backoff_requires_new_claim`; `offer_lifecycle::every_unfinished_state_is_claimable_retained_and_reserved_only_once` | +| Offer lifecycle: `reserve_offer`, `record_offer`, `release_unsent_offer`, `adopt_active_candidate`, `reconcile_candidate` | never-retried mutations, claim-fenced | none | Each is one state transition of the live claim's row (`pending` to `offer_reserved`, `offer_reserved` to `offered`, `offer_reserved` back to `pending` after a `submitblock` that never established its connection (#522) or that the node answered from its warmup (#526), `pending` to `reconciliation`, an offered state to `reconciliation`) and is rejected once the claim is lost or the row has moved on. A reported error after `reserve_offer` leaves the reservation in place: the next claim, on any frontend, treats the row as delivery unknown and never calls `submitblock` for it. | `offer_lifecycle::every_unfinished_state_is_claimable_retained_and_reserved_only_once`, `candidate_lease_tests::token_loss_between_heartbeats_is_fenced_immediately_before_submitblock`, `candidate_lease_tests::crash_after_offer_before_landing_recovers_on_another_frontend_without_a_second_offer`, `candidate_lease_tests::offer_not_sent_tests::*` (#522, #526), `offer_not_executed_tests` (#526), `rpc_not_sent`, `live_regtest::node_outage_tests::node_killed_before_submitblock_lands_the_block_once_after_restart`, `live_regtest::node_outage_tests::node_restarted_into_warmup_before_submitblock_lands_the_block_once` (#526), `live_regtest::lost_reply_tests::lost_submitblock_reply_owner_killed_survivor_lands_the_block_once` (#474: the owning frontend killed with the node's acceptance unheard) | +| CTV attempt journal: `finish_fanout` | never-retried mutation; one journal row per authorized claim | none | Every recorded attempt, `failed` included, releases the claim and schedules `next_broadcast_attempt_at` (10 s per attempt, at most 3600 s). A fresh `claim_fanout` after that time is the next attempt and its own journal row; the old token stays rejected. | dynamic: `fanout_journal_timeout_executes_once`, `broadcast_retry_requires_new_claim` | +| Claims: `claim_candidate`, `claim_fanout`, `renew_candidate_claim`, `renew_fanout_claim` | public reinvocation | none | One token per block hash or fanout. Distinct hashes have independent leases. Candidate terminal dispositions acquire the shared `SETTLEMENT_LOCK`, then `ORDER_LOCK`. The cited test verifies that a disposition does not touch a sibling candidate's locked outbox row; it does not establish concurrent execution of dispositions. | dynamic: `distinct_hashes_hold_independent_claims` | +| Landing: `land_candidate`, `land_candidate_at_revision` | dependency reread, idempotent for an identical audit | none | Re-invocation re-reads the stored audit digest and header bits and accepts only an identical audit. A superseded payout revision is a reported error with no block, audit, payout, carry or fanout row written; recovery at a proven newer revision is an explicit call. | dynamic: `superseded_landing_reports_failure`; existing `ledger_postgres::active_candidate_can_land_at_proven_new_chain_revision` | +| Expired or lost claim across a halt | never revived | none | Clearing `fatal_error` restores authority for new work only; the token that expired during the halt is still rejected by every operation and a new claim is required. | dynamic: `restored_authority_requires_fresh_claim` | +| Share append: `append`, `append_at_revision` | public reinvocation, idempotent by share identity | none | A miner or frontend resubmission is a new call; the share identity and proof-hash registry return the existing row without a second credit. | inventory: `ledger_postgres::global_duplicates_idempotence_and_config_fencing`, `postgres_failover` | +| Page traversal: `read_window`, `snapshot`, audit materialization | normal page traversal | none | A page is not a retry. `read_window` pages inside one `REPEATABLE READ READ ONLY` transaction and reports an incomplete range as an error instead of a partial window. `snapshot` fixes its anchor and revision in one short transaction, then pages immutable rows at or before that anchor in a second one. Audit materialization reads the snapshot row and the share range through separate checkouts; the share count, snapshot digest and bundle digest authenticate the result, and a mismatch is an error. | inventory: `window_reference`, `window_read_oracle` | +| Dependency reread or repair: `save_issued_job` with a repair payload, `backfill_ctv`, `import_legacy_audits` | dependency reread/repair | none | Cold-path calls that re-read the durable dependency and verify its identity, or rebuild a missing row idempotently. None replays a failed write. | inventory: `issued_job_dependency`, `ledger_postgres::ctv_artifacts_wait_for_maturity_and_claims_are_fenced`, `ledger_postgres::legacy_audit_import_validates_envelope_hash_and_pinned_key` | +| Reconciliation: `pool_blocks_for_reconcile`, `reconcile_blocks`, `reconcile_blocks_at_revision`, `observe_fanout` | public reinvocation, revision fenced | none | Periodic calls. `pool_blocks_for_reconcile` is one query. A stale expected revision (for `observe_fanout`, when the observation carries one) is an error that changes nothing. | inventory: `ledger_postgres::verified_landing_reconstructs_audit_and_reorgs_are_revision_fenced` | +| Session reservation release: `SessionId::release`, drop cleanup | best-effort cleanup | none | A lost cleanup reply leaves the outcome unknown: the reservation may have been deleted or retained. A retained reservation is reclaimed once its owner is recorded as stopped. | inventory: `ledger_postgres::session_sequence` | + +"Dynamic" rows are exercised by +`crates/qbit-prism-server/tests/ledger_single_execution.rs` against a disposable +PostgreSQL. The two timeout tests route one ledger pool through a test-only +protocol proxy that frames both directions of the wire and counts the +targeted statement in `Execute` and `Query` frames across every connection, +including a same-text replay the aborted transaction rejects and a second +copy of the statement inside one simple `Query` frame. Server rejections are +recorded separately from executions and every one in the observed window +must be the targeted statement's own failure, so a replay under a different +statement text, which the server refuses at `Parse` without any `Execute`, +also fails the test. An execution whose statement text the proxy did not +learn fails the test instead of escaping the count. The targeted statement +is identified while it runs by a marker NOTICE from a statement-level +fixture trigger on the durable table, not by its SQL text or its position. +They seed a real server `statement_timeout` on the terminal write and a lost +acknowledgement before and after `COMMIT`, and read the durable outcome back +directly; a re-invocation after a lost acknowledgement is checked to run on +a connection other than the one the fault closed. The durability of the +lost-`COMMIT` case is proven by the proxy observing the server's `COMMIT` +completion and by reading the state back, not by the caller's view. +"Inventory" rows are a static review of the code, backed by the existing +tests named; they were not re-tested for this contract. + +```sh +PRISM_TEST_DATABASE_URL=postgresql://test_user@127.0.0.1:5432/test_db \ + cargo test --locked -p qbit-prism-server --test ledger_single_execution -- --nocapture +``` + +### Deadlines + +The legacy writer's final-partial-batch and progress-between-batches deadline +workflow has no native counterpart. There is no batch writer: each accepted +share commits in its own transaction, so no deadline can expire inside a +batch and no batch is retried as a unit. Two native mechanisms stand in for +it: + +- Issued-job late expiry. `save_issued_job` checks the job's absolute + deadline after taking the shared settlement lock and before locking its + dependency row, and again just before commit, so a deadline that elapsed + while waiting on row locks is a reported error that retrying cannot reset, + and the job is not saved. The runtime pruner runs every two seconds with + no overlapping batches. First, `prune_expired_jobs` removes at most 4096 + expired rows as a separate statement, without advisory locks and with the + configured database timeout. Its outer expiry recheck preserves concurrent + renewals. Then `prune_unreferenced_blobs` starts a fresh five-second deadline, + takes `SETTLEMENT_LOCK`, `ORDER_LOCK`, then cluster `FOR UPDATE`, checks the + shared writable fence, and only then inspects up to 256 template keys and + 256 balance keys with fresh reference scans. References from + every surviving job and every candidate retain their blobs, regardless of + job or claim expiry. Per-table cursors advance only after commit, skip live + prefixes and wrap to find later orphans even when no jobs expire. A halted + cluster still permits the original expiry statement but refuses blob GC. + Blob failure or cancellation rolls back both blob deletes and leaves the + cursors unchanged; it does not undo previously committed expiry. Successful + nonzero counts are logged per phase, and failures warn on each pruning tick. +- Per-item import failure and restart. `import-audits` processes one legacy + audit at a time, each verified, and its statement's parameters encoded, + off the runtime threads, and stored in its own transaction. A failing item + stops the command with an error that names it; earlier items stay + imported, there is no partially imported item, and rerunning the command + resumes with the rows that still lack canonical bytes. +- `import-audits` and `SETTLEMENT_LOCK` (#765). Each row's UPDATE is sent + and run first, holding only that row's lock; only then does the command + poll for `SETTLEMENT_LOCK` (`pg_try_advisory_xact_lock`, never queued), + check the write guard under it and commit; the guard also runs before the + write, so a halted cluster is refused before anything is sent. Settlement + waits for that check and the commit, never for a legacy body to be sent or + stored. The one exception is a lock holder that needs the very row being + imported (a re-landing of that block filling its `found_block_bits`): it + waits for that row's UPDATE to finish, and then goes first, because a poll + that misses the lock and finds the lock's holder waiting for the importer + rolls the row back. A stopped or killed importer's row is rolled back + within a second even mid-UPDATE, because its connection sets + `client_connection_check_interval` (PostgreSQL 14 or later on Linux; + elsewhere the server finds out at the write timeout). Transaction-local timeouts bound each phase: + `PRISM_IMPORT_AUDITS_WRITE_TIMEOUT_MS` (60 s) the write, and + `PRISM_IMPORT_AUDITS_LOCK_TIMEOUT_MS` (5 s) the wait for the lock and, + once it is held, any statement or client pause, after which PostgreSQL + ends the session and the lock with it. A row that exceeds one is rolled + back. SIGTERM (`docker stop`) or SIGINT stops the command before the next + row, or rolls back a row that has not been granted the lock, and it exits + 1; a second signal exits at once. Each imported row prints one + `import-audits: block_hash=...` line on + stderr with every parameter's encoded size and the prepare, write, lock + wait, lock hold and total times. + +### A ledger future cancelled inside `BEGIN` (#482) + +Every D1 run of #447 logged PostgreSQL's `WARNING: there is already a +transaction in progress` on the frontend, 10,303 times across the 33 runs on +`a1937054`: 8,431 in `slow_database`, 1,866 in the drain after it, five in +flush-control phases, and one in the `steady_state` of +`d1-dense400k-fe1-async-r1`, a phase with no share-commit cancellation. The +warning is what PostgreSQL says to a `BEGIN` that arrives on a connection +whose transaction block is already open. Every run reconciled exactly. This +section names the trigger, records why nothing could commit or stay locked +across it, and describes the guard that now keeps such a connection out of +the pool. + +**The trigger.** SQLx 0.8.6's `PgTransactionManager::begin` +(`sqlx-postgres/src/transaction.rs`) queues the `BEGIN`, awaits the reply +with `wait_until_ready`, and only then increments `transaction_depth`. The +guard it drops on cancellation calls `start_rollback`, which does nothing at +depth 0. A future dropped between the write and the reply therefore hands +its `PoolConnection` back with `transaction_depth == 0` while the server has +opened a transaction: the pool's release ping (`Sync`, then +`wait_until_ready`) consumes the `BEGIN` reply and offers the connection as +idle, and the next checkout's `BEGIN` lands inside the leftover transaction. +`crates/qbit-prism-server/tests/ledger_cancelled_begin.rs` reproduces this +deterministically by holding the `BEGIN` reply at the wire proxy and +cancelling the future there; against the code before this guard, +`pg_stat_activity` showed the cancelled checkout's backend `idle in +transaction` with `query = BEGIN` while it sat in the pool, the proxy saw +`BEGIN`, `BEGIN` on that connection with no `ROLLBACK` between, the next +transaction ran on the same backend pid, and SQLx relayed the warning while +it ran. The futures that can be dropped there: + +- the persist task aborted at `share_commit_timeout` (`AppendTask::abort`, + the #324 mechanism), logged as `share commit deadline passed before COMMIT + was sent`: the bulk of the `slow_database` and drain occurrences; +- the persist task aborted because the submit handler itself was dropped + (`AppendTask::drop`), which is what a miner disconnect during a submit + does. Nothing is logged for it, so it is the likeliest reading of the one + `steady_state` occurrence; +- the metrics collector's three-second deadline around its snapshot + transaction (`metrics database collection unavailable` with `deadline + exceeded` in the log), which runs every ten seconds in every phase; +- session allocation under the subscribe timeout (`session allocation timed + out`), and any other `tokio::time::timeout` around a ledger transaction. + +**Why no advisory lock can be held across it.** `ORDER_LOCK` and +`SETTLEMENT_LOCK` are taken by `pg_advisory_xact_lock` inside a transaction +that has completed its `BEGIN`, so the connection is at depth 1 from then on. +Dropping the `Transaction` at any later await queues `ROLLBACK` +(`start_rollback`), and SQLx's pool return flushes the queued protocol and +waits for its replies before the connection is released, so a lock taken by +a cancelled append is released before the pool can offer that connection. +The one await that cannot queue a rollback is the `BEGIN` reply itself, and +a transaction cancelled there has run nothing but `BEGIN`: it holds no lock +and contains no statement. The tests assert the absence of both keys in +`pg_locks` after the cancellation. + +**What the leftover could still do.** Nothing on the share path: the next +append's `BEGIN` joined the empty transaction, and its own `COMMIT` or +rollback ended it with nothing foreign inside. The hazard was a statement +that runs without a transaction of its own, on a connection checked out +directly: the heartbeat, a session-reservation `DELETE`, a difficulty +record, the landed-audit-bits `UPDATE`. Inside the leftover such a +statement is reported successful but is neither durable nor visible until +the connection's next transaction user ends the block, and a later dropped +`Transaction` on that connection rolls it back; its row locks are held while +the connection sits idle in the pool; an error in it leaves the block +aborted, and the pool would keep re-offering that connection until SQLx's +default `max_lifetime` of thirty minutes; and the collector's +`transaction_timestamp()` ages would be measured from the leftover `BEGIN`. +None of this was observed in #447, and the exact reconciliation says the +share path was never affected; it is why the outcome is guarded rather than +only documented. + +**The guard.** Two places, one per form of `BEGIN`: + +- `ledger::shielded_begin` runs `Transaction::begin` on an already + checked-out connection in a task of its own and joins it. `Ledger::begin` + (every owned transaction: session allocation, configuration, divergence, + archive, the startup migration) and the metrics collector use it. A caller + dropped or aborted mid-`BEGIN` detaches from the task; the task still reads + the reply, and the transaction it then drops queues the `ROLLBACK` that the + pool return flushes. The connection returns clean and serves the next + transaction; the wire shows `BEGIN`, `ROLLBACK`, `BEGIN`. +- The share append borrows its connection through the admission guard + (`AppendConnection`), so its `BEGIN` cannot be moved into a task. The guard + now records whether `BEGIN` completed, and when it did not, its drop + retires the connection: `PoolConnection::detach` leaves the pool + synchronously, size and permit included, so the pool opens a replacement + and can never see that connection again, and the socket is closed on the + runtime the append ran on (a close that is never polled, at shutdown, + drops the raw connection, which closes the socket and makes the server + abort whatever it held). Retirement was chosen over a rollback issued from + the drop because a drop must not block and must never return a connection + whose server side may be in a transaction: a rollback would have to run on + the captured runtime before the connection re-enters the pool, could not + run at all while that runtime is shutting down, and would then need the + close as its fallback anyway. The cost is one reconnect per append + cancelled inside `BEGIN`, on the order of three hundred per D1 run in + `slow_database` and none in steady state. The append opens its + transaction with `BEGIN; SET LOCAL plan_cache_mode = force_generic_plan` + (#479) as one simple query. SQLx reads every reply of that batch through + its `ReadyForQuery` before `begin` returns, so the guard records + completion only once the whole batch is in: a cancel after the `BEGIN` + reply but before the end of the batch retires the connection as well. The + test proves it by also holding only the batch's closing `ReadyForQuery`. + +On the successful path nothing changes: an append is the same ten round +trips on one connection, followed by SQLx's own release ping. Since #479 they +carry eleven server statements: the opening simple query is `BEGIN` plus +`SET LOCAL plan_cache_mode = force_generic_plan`, which applies to every +statement of the append transaction and makes the share_id probe planned once +per connection, and `pg_stat_statements` shows the `SET`. The test prints the +trace after the retirement so the count can be compared with the trace +recorded before the guard. + +**Rejected alternative.** A pool-level `after_release` hook in the ledger's +pool builder that rolls back any open transaction would cover every +cancellation source in one place, but SQLx exposes no client-side +transaction status: `PgConnection::in_transaction` is crate-private, and +`Connection::is_in_transaction` reports SQLx's own depth, which is exactly +the number that is wrong here. Detection therefore needs a statement round +trip on every release, in addition to the release ping, and an unconditional +`ROLLBACK` would warn in the server log on every release. That is roughly one +extra round trip per checkout on a path that costs ten statements and a ping +today, paid as connection occupancy in the pool-bound `slow_database` phase, +with the `ORDER_LOCK` hold time unchanged. `idle_in_transaction_session_timeout` +was rejected too: it reaps an idle leftover only after its timeout, does +nothing under load, where the leftover is reused within milliseconds, and +would end legitimate transactions that idle between statements. + +**Residual.** `partitions::ensure_with_metrics` and the rollup writer open +their transactions with a plain `Transaction::begin` on a fresh checkout. The +former runs inside the persist task on the missing-partition retry path and +could be aborted mid-`BEGIN` there; it needs a detached partition lead first. +The issued-job batch writer's `BatchConnection` runs in the batcher task and +is cancelled only at shutdown. + +```sh +test/prism-native-tests.sh cargo-args --locked -p qbit-prism-server \ + --test ledger_cancelled_begin -- --nocapture +``` + +### Native snapshot rejection and legacy segments + +A native audit body references an immutable share snapshot in +`qbit_prism_audit_snapshots`. Landing rejects a snapshot that is empty, out of +canonical order, or different from the ledger's rows for its range, before any +obligation is recorded. Materialization verifies the share count, the +reconstructed share digest and the canonical bundle digest, and returns an +error instead of a repaired or substituted body. Imported canonical bytes are +checked on both read paths: raw canonical-byte serving requires the declared +digest and a JSON object, and materialization into a logical body +additionally rejects an audit envelope and parses the bytes as a flat bundle. + +The legacy `audit-body-ref` and v2 segment envelopes are still parsed by the +shared audit parser and the offline loaders, for import and verification. The +legacy segment lifecycle (gap backfill from the ledger, quarantine of a +segment when the ledger has no rows, and the conflicting-duplicate raise) +belonged to the retired filesystem audit store and is not a native lifecycle: +natively, a body is either reconstructed exactly from immutable rows or served +from imported canonical bytes, and a mismatch is a read error to investigate, +never a repair or quarantine step. + +## HA database and shutdown + +Point every instance at the same writable primary endpoint. Do not route ledger +queries to a lagging read replica or distribute writes among independent +PostgreSQL primaries. Sum `PRISM_DATABASE_MAX_CONNECTIONS` across frontends and +reserve capacity for migrations, monitoring, backup, and failover administration. + +The cluster records its highest observed cumulative chain work. A node that is +still synchronizing, follows a lower-work tip, or disagrees at equal work cannot +advance accounting. Share commits are fenced by the current revision as well. +This prevents a lagging node from reversing another frontend's accepted blocks. +For manual regtest invalidation/reconsideration, extend the intended branch past +the previous work record before expecting the pool to resume; do not clear the +record to accommodate a lagging production node. + +Keep PostgreSQL `fsync=on`, `full_page_writes=on`, and +`synchronous_commit=on`. A local durable commit protects against a coordinator +crash. Zero acknowledged-share loss on database-primary failure additionally +requires synchronous standby flush and a promotion policy restricted to a +standby containing acknowledged commits. An HA endpoint does not establish this +by itself. Test primary failure and client reconnection using the actual +replication, proxy, and storage configuration. + +### Found blocks and the failover standby (#529) + +Under asynchronous D3 a found block's candidate row and offer reservation can +be lost with the primary after the block was offered, leaving it on chain with +no ledger record. With `PRISM_OFFER_STANDBY_APPLICATION_NAME` set to the +dedicated failover standby's `application_name` (and `pg_monitor` granted to the +writer role), each frontend waits after the reservation commits, up to +`PRISM_OFFER_STANDBY_FLUSH_WAIT_MS` (default 250 ms), for that standby's +`flush_lsn` to cover the block's rows, then offers it. Shares are unaffected. +The wait never holds the block: when the standby is not connected, is still +behind at the bound, or its position cannot be read, the block is offered at +once or at the bound, and the frontend logs a WARN line whose message starts +`block offer standby wait unconfirmed`, with the fields `block` (the hash), +`standby`, `outcome` (`absent`, `lagging` or `failed`), `waited_us`, and +`lag_bytes` or `error`. A confirmed wait is logged at INFO with its `waited_us`. + +`qbit_prism_block_offer_standby_wait_total{outcome}` counts every wait and +`PrismBlockOfferStandbyUnconfirmed` warns on any that did not confirm. +`self-check` refuses the setting when the writer role cannot read replication +positions or the name matches no single connected standby reporting its flush +position. + +**After a failover, reconcile the logged hashes.** Collect every hash logged +`block offer standby wait unconfirmed` before the primary was lost, and follow +[promotion step 6](prism-ha-reference-architecture.md#promotion-fencing-and-the-stable-writer-endpoint): +a hash whose candidate survived on the promoted primary needs nothing; one with +no candidate that is not on the node's active chain lost only its reward; one +with no candidate that **is** on the active chain is an accounting-loss +reconciliation, because its coinbase paid balances the promoted ledger still +carries. `candidates recover` cannot land it: it refuses a hash with no +candidate row. The HA reference's +[found-block section](prism-ha-reference-architecture.md#found-blocks-a-bounded-standby-flush-before-the-offer-529) +has the measurements and the accepted-loss statement. + +### Work from a lost replication gap (#619) + +An asynchronous promotion can also lose the end of the share history that +work was read from: shares credited in the gap, after the standby's last +received WAL. The promoted sequence then hands their `share_seq` values out +again to other shares. Every frontend tags the work it prepares with the +writer timeline its window was read on (PostgreSQL increments it at every +promotion), and every check below revalidates on the primary that would hold +the result: + +- **Refresh.** A changed timeline replaces published work at the next poll, + even on an unchanged template, revision and balances. The rebuild is + counted under `qbit_prism_refresh_seconds{trigger="writer_timeline"}` and + logged as `refresh published` with `trigger=writer_timeline`. +- **Issuance and resume.** Work read on another timeline, or reconstructed + from a stored record, is issued only while the primary holds its window's + last row as the window's own row. Until the refresh rebuilds, a new session + is sent no job: nothing is answered on the wire, and the session retries its + build on its timer and at the next publication. The frontend logs `job + preparation deferred` (or `job persistence deferred`) with `window_not_held: + the work's window is not held by this primary after a writer timeline change + (#619)`. A prepared record is never written back, by the repair or by + `save_compact_prepared`, unless its window is held. +- **Found blocks.** The candidate enqueue, in the transaction that would write + the candidate, refuses a block whose window this primary does not hold. The + miner is answered `stale-job`, the share the proof carried is not credited, + `qbit_prism_stale_job_rejections_total{cause="window_not_held"}` counts it + and a WARN names the block: `block refused stale-job before its offer: this + primary does not hold the window its coinbase pays, which a promotion lost or + reissued (#619)`. Nothing is enqueued or offered. The ledger counts the + refusal itself, so one that finishes after the miner's acknowledgement + deadline is counted too; that miner was answered `ledger-outcome-unknown`. + A block whose window replicated, including on work prepared before a planned + switchover or on a job resumed after the promotion, lands as before. + +**Precondition: synchronized database clocks.** The check compares the window's +last row with the window's anchor by time: the row must be accepted and issued +no later than the anchor. A share reissued after the promotion is stamped with +the promoted host's clock, so it fails the check only while that clock does not +lag the old primary's by more than the time from the last gap work to the first +reissued share, a few seconds. Keep every database host NTP-synchronized. If the +promoted host's clock lags further, a block on gap work can pass the check, be +offered, and end in `reconciliation` with a window digest mismatch, as before +#619; reconcile it as below. + +A refused block is a lost reward, like a block found in the gap and never +offered. Record each WARN's hash with the failover's reconciliation. The design +note [`prism-async-promotion-gap-blocks.md`](prism-async-promotion-gap-blocks.md) +has the analysis and the options that were weighed. + +**A candidate already in `reconciliation` after a promotion.** A block offered +before this check existed, or by a frontend still running an older release +during a rolling upgrade, can be on chain while its window is not held. Its +landing fails with a `last_error` that starts `window not held by this +primary` (older releases: `window range incomplete: … read 0`; under the clock +precondition above: a window digest mismatch). The row is retried read-only and never offered again, and +`abandon` refuses it, because it is not `pending`. Treat it as an +accounting-loss reconciliation, as +[promotion step 6](prism-ha-reference-architecture.md#promotion-fencing-and-the-stable-writer-endpoint) +does for a block on chain with no candidate: + +1. Confirm the block is on the node's active chain (`getblockheader <hash>`, + `confirmations` of at least 1) and record its coinbase. +2. Record the divergence. The coinbase paid the lost window and the as-issued + prior balances, which the promoted ledger still carries, so the next + landings pay those balances again, up to the block's positive as-issued + float. The block's accruals (gross it carried for accounts below the payout + floor) are never credited. +3. **With CTV settlement on,** the coinbase's covenant outputs can only be + spent by fanout transactions built from the lost window, and there is no + pool spend key. No tool rebuilds a fanout from exported rows today, and + `candidates recover` cannot land a window the primary does not hold, so + recovering those outputs needs engineering work; record the block for it. + What the operator can do now is keep that recovery possible: do not + rewind the fenced old primary, rebuild it from the new primary or start it + as a writer until the window's rows (the candidate's + `window_first_share_seq..window_last_share_seq`, on the old timeline) have + been copied out of an isolated copy of its data directory. Without those + rows the covenant amount stays unspendable. + +SIGTERM closes listener admission and asks tasks to drain before the database +pool closes. The native server bounds shutdown drain to 30 seconds; unfinished +candidate/CTV intents remain in PostgreSQL and become reclaimable after their +claims expire. There is no legacy writer-lease release barrier. Observe each +frontend's health before restoring traffic after a restart. + +Backups require the database, signing-key recovery material, and any unimported +external audit bodies/segments. Use independent base backups plus WAL archives +for point-in-time recovery; replication is not a replacement for backups. +Restore into isolation and verify share order, audit hashes, carry-forward +integrity, CTV state, and API reads before declaring recovery complete. + +## One-way migration and isolated-restore reconciliation + +Decision D5 in #260 chooses one-way native migration with forward repair. No +down-migrations are provided for native 002–010, including 006/007/008. Never +remove native guards to run a legacy writer. The publication-ordinal revert +refuses native schemas and names this recovery path. The data-loss boundary, +matching the [unreleased 3.0.0 release notes](../doc/release-notes-3.0.0.md), is: + +> Before the first native share is acknowledged, restore the complete +> pre-migration database and artifact backup and restart the pinned old image +> in isolation from the migrated database. After the first native share is +> acknowledged, restoring an older database loses those accepted records. +> Any rollback that discards acknowledged history requires an explicit +> accounting reconciliation and recovery decision; it is not an ordinary +> image rollback. Keep every native frontend stopped while performing an +> isolated restore/recovery operation against its replacement database. + +1. **Preconditions and stop boundary.** Pin the source/native image digests and + retain keys, configuration, database backups/WAL and every external audit + body/segment. Before an actual cutover, drain legacy candidates with the + supported source release as described in the [migration guide](prism-rust-migration.md#supported-2xx-source-schemas). + Stop all Python/native frontends, public services and CTV/candidate workers; + block their access to the isolated restore. Keep the latest database intact. + Treat an unknown first-ACK boundary as post-ACK. Lost replies and background + settlement can leave committed work even before a successful native ACK. +2. **Numbered pre-migration canonical backfill rehearsal.** Restore a source + copy and artifact backup in isolation, then run native `migrate` and + `import-audits --root /var/lib/qbit-prism/audit` there. This is the native + canonical backfill: it reconstructs bytes from recoverable inline or + external bodies when the sidecar never existed, preserving the published + SHA and verifying the trusted ledger key and coinbase. Require zero + `missing_stored_bodies` and `missing_canonical_bytes` before migrating the + real source. A corrupt present sidecar fails closed. Restore missing + evidence and rerun import; do not invent replacement history. This uses + the native equivalent and does not run the `2.x.x` command blocked by + #174/#176. Record timings and history size; use a production-sized copy. +3. **Save and compare exact accounting evidence.** Configure protected libpq + services `prism-source`, `prism-current` and `prism-restore` for distinct + databases with the correct ledger `search_path`. With the source drained: + + ```sh + umask 077 + PGSERVICE=prism-source pg_dump --format=custom --file=pre-native.dump + PGSERVICE=prism-source psql -XqAt -v ON_ERROR_STOP=1 -v FETCH_COUNT=10000 \ + -f scripts/prism-recovery-evidence.sql > source.rows.jsonl + python3 scripts/prism-recovery-evidence.py source.rows.jsonl > source.summary.json + ``` + + The [exact SQL](../scripts/prism-recovery-evidence.sql) exports ordered share + rows, block/publication order, audit SHAs, payouts, carry rows, candidate and + CTV state in one read-only snapshot. The [streaming summarizer](../scripts/prism-recovery-evidence.py) + records counts and digests and reproduces the legacy `audit_head_sha256`; + compare that head to the mirrored pre-cutover report. It refuses incomplete + exports and carry mismatches/drift. It checks legacy carry rows per payout + program (#708): a finding of `2.x.x`'s per-label rule that the program's + chain clears is named in a `note:` on stderr and left out of the summary. + Require `unfinished_candidates` zero. Protect the evidence as accounting + data and budget disk for the share export. + `FETCH_COUNT` makes psql print rows in batches instead of buffering each whole + result, which took 63 GB and was OOM-killed at 65.9M shares (#705). The script + defaults it to 10000 since #705; the flag covers copies from before that. + + The serial export took 85 minutes at 65.9M shares (#712). The + [parallel export](../scripts/prism-recovery-evidence-parallel.py) prints the + same rows byte for byte, so the summaries and every `cmp` in this runbook are + unchanged. It can replace the `psql` line of any export here: + + ```sh + umask 077 + PGSERVICE=prism-source python3 scripts/prism-recovery-evidence-parallel.py \ + --jobs 8 > source.rows.jsonl + ``` + + A coordinator session runs the script's checks and exports its snapshot. + `--jobs` sessions import that snapshot. They export the shares in + `share_seq` ranges, the share hashes in header-hash ranges, and the other + kinds as the script does. Each range's sort may use `--work-mem` (default + 256MB) before it spills to disk. Size `--jobs` to the database host's idle + cores; the export opens one connection more. No session may prompt for a + password, so keep it in the service's password file. Parts wait in + `--work-dir` (default: the current directory) until they can be printed in + order, so budget the export's size there as well. If any session fails, + the output has no completion marker, so the summarizer refuses it. On a + 6M-share copy (4.6 GB of rows, a shared 16-core host), 8 jobs took 24 s + against 115 s serially on the `2.x.x` source, and 12 s against 57 s once + migrated. + + The summarizer parses in parallel as well: `--jobs` worker processes + (default: the CPU count, at most 8) parse and canonicalize newline-aligned + ranges of the rows, and one process applies every check in file order. Its + summary, notes and refusals are those of `--jobs 1`, the serial summarizer, + byte for byte. From a range a worker cannot read as a text-mode read does + (invalid UTF-8, a read error) on, it reads the rows serially from the start, + so even a decoding error is the serial one. Size `--jobs` to the host's + idle cores. The serial pass ran at about 29 MB/s on the 65.9M-share restore + (about 80 GB, so about 45 minutes). On 3.25 GB of share and header rows on + a 16-core host, 8 jobs took 5.7 s against 36 s serially, and 12 jobs 4.4 s. +4. **Exercise the isolated pre-ACK restore.** Provision a separate empty + database with compatible PostgreSQL, roles and extensions. Restore the full + database and artifact backup, never over the current database: + + ```sh + pg_restore --exit-on-error --single-transaction \ + --dbname='service=prism-restore' pre-native.dump + PGSERVICE=prism-restore psql -XqAt -v ON_ERROR_STOP=1 -v FETCH_COUNT=10000 \ + -f scripts/prism-recovery-evidence.sql > restored.rows.jsonl + python3 scripts/prism-recovery-evidence.py restored.rows.jsonl > restored.summary.json + cmp source.summary.json restored.summary.json + ``` + + Only exact equality permits testing the pinned old image on the isolated + legacy database, with mining and broadcasting disabled. Stop it again before + migrating the restore. For a post-ACK incident, continue to step 6 even if + this pre-migration comparison passes. +5. **Repair forward and enforce import.** Verify `PRISM_DATABASE_URL` targets + the isolated restore and set the original trusted + `PRISM_LEDGER_WRITER_PUBLIC_KEY_HEX`; no signing seed is needed for import. + Run `time qbit-prism-server migrate --offline-indexes` then + `time qbit-prism-server import-audits --root /var/lib/qbit-prism/audit`. + Nothing else runs against the isolated restore, so `--offline-indexes` + builds 013's and 024's indexes with a plain `CREATE INDEX` + ([migration 013](prism-rust-migration.md#migration-013-the-share-ledger-index-trim-applied-online)). + If import stops on a legacy body with `audit body ref hash mismatch` + because a range's shares carry non-ASCII text, + [prove it and write its sidecar](prism-rust-migration.md#legacy-bodies-whose-range-digests-use-pythons-escaped-json-709), + then rerun import. Repeat the export as + `migrated.rows.jsonl`/`migrated.summary.json` and require + `cmp source.summary.json migrated.summary.json` to pass. Both import + completeness counts must be zero. Native snapshot-backed audits count as + canonical available without duplicating their share window as stored bytes. + Availability counts do not replace digest/signature verification. Run + `backfill-ctv` only after this comparison and explain any intentional CTV + repair delta in later evidence. With `migrate --defer-share-hashes`, the + migrated summary marks `share_hashes` deferred: compare every other record + with `jq 'del(.records.share_hashes)'` on both summaries, and the share + hashes once + [`backfill-share-hashes` has recorded 2 after go-live](#finish-a-deferred-share-hash-backfill-after-go-live) + ([Recovery evidence while the backfill is pending](prism-rust-migration.md#recovery-evidence-while-the-backfill-is-pending)). +6. **Reconcile all post-ACK differences.** Export `prism-current` using the same + SQL to `current.rows.jsonl`/`current.summary.json`. Compare against the + restored summary and inspect the ordered rows for every changed digest. + Run these queries against current state, supplying the source summary's + integer `last_share_seq` as the psql variable `source_last_share_seq`: + + ```sql + SELECT share_seq, share_id, miner_id, share_difficulty, credit_policy + FROM qbit_share_ledger + WHERE accepted AND share_seq > :source_last_share_seq ORDER BY share_seq; + SELECT state, count(*) FROM qbit_block_candidate_outbox GROUP BY state ORDER BY state; + SELECT qbit_carry_forward_integrity_report(); + ``` + + A native share tail is accepted history the older restore would lose. Also + reconcile block states, the carry head/balances, payouts and CTV broadcasts + against chain evidence; share counts alone are insufficient. Preserve the + current database and prefer forward repair with a compatible native image. + Any plan discarding acknowledged history requires an explicit accounting + owner's recovery decision. Do not copy rows ad hoc into the legacy schema. +7. **Verify service and record the rehearsal.** Sample exact canonical artifact + bytes/SHA/ETag through the isolated public API, then repeat through the actual + public edge after cutover. Use the exact `curl`, `shasum`, and ETag commands + in [recovery step 5](prism-rust-migration.md#recovery-and-rollback); require no + missing-canonical fallback header. Include old history, sidecar-only and + reconstructed imports. Resolve stale missing edge cache entries before + admission. Run production `self-check` with the intended services available; + its JSON must show zero audit-completeness and integrity failures and its + exit status must be zero. Lab mode reports incomplete audits without making + that condition fatal; production mode refuses them. Retain every duration, + row count, comparison, artifact sample and image digest. #291 rehearses this + on production-sized history and retains the exact data-loss boundary when + completing the 3.0.0 release notes before approving rollout. + +## Finish a deferred share-hash backfill after go-live + +`qbit-prism-server migrate --defer-share-hashes` takes migration 2's +share-hash backfill out of the cutover's outage: it maps the legacy shares +within 1,000 template heights of the tip, permits serving at +`share_hash_backfill_pending = 2`, and leaves the rest of the legacy ledger +unmapped +([Serving with the backfill pending](prism-rust-migration.md#serving-with-the-backfill-pending-fence-2)). +Until the rest is mapped and 2 recorded, `self-check` warns, and +`share-archive restore`, and `detach` and `drop` of every partition, refuse: +archiving waits for 2 (`plan`, `seal`, `archive` and `verify` are +unaffected). Finish it once frontends serve, at a time of ordinary load. + +1. **Check the state.** `self-check` reports `share_hash_backfill` with + `"state": "pending"`, `fence` 2, the cursor (`next_seq`, `end_seq`, + `remaining_seqs`) and the recent range. `"state": "unknown"` means it + could not read the backfill: the `error` says why. Without the field there + is nothing to do. +2. **Choose the throttle.** The defaults suit a primary taking up to a few + hundred shares a second: batches of at most 5,000 `share_seq`, each + statement cancelled after 2 s and retried at half its size, and a rest as + long as each batch took. Every native share updates the cluster row inside + `ORDER_LOCK`, and a snapshot held on the primary keeps that row's dead + versions, so `ORDER_LOCK` saturates after 18 to 20 seconds of held horizon + at about 400 shares a second (#738). Each of the backfill's statements, + its double-credit check's and its record's included, runs in a transaction + of its own and is cancelled at the statement timeout, which is at most 5 s. + If even the smallest batch times out, rerun with a smaller `--max-batch` + first, and only then a larger `--statement-timeout-ms`. Lower `--duty-cycle` (down to 0.01) to + spare the primary more; the run takes longer in proportion. +3. **Run it** where it survives a disconnect, with the cutover's + `PRISM_DATABASE_URL`: + + ```sh + RUST_LOG=info qbit-prism-server backfill-share-hashes \ + --max-batch 5000 --statement-timeout-ms 2000 --duty-cycle 0.5 \ + 2>backfill-share-hashes.log + ``` + + At the defaults expect about 5,500 `share_seq` a second at the measured + cost per row, three to four hours for 65.9M shares. A `migrate` started + meanwhile waits for it on the runners' lock: run none. +4. **Watch it.** + - `share-hash backfill progress` every ten seconds in its log: `next_seq`, + `end_seq`, `mapped`, `percent`, `seqs_per_second` and `remaining_s`. + - The cursor, from any session: + `SELECT next_seq, end_seq, updated_at FROM qbit_prism_share_hash_backfill;` + - On the primary, `qbit_prism_database_order_lock_hold_seconds{holder="append"}` + and the percentiles of `qbit_prism_share_ack_seconds{result="accepted"}`, + against the same hour without it, and the run's own backend in + `pg_stat_activity`: its `xact_start` within a few seconds of now, and its + `backend_xmin` moving on with every batch, the attempts to record 2 + included. + - If `ORDER_LOCK` holds or share acknowledgements climb, stop it (Ctrl-C or + SIGTERM). Each batch committed with the cursor, so only the batch in + flight is lost. Run it again later, or with a lower `--duty-cycle`; it + resumes at `next_seq`. +5. **It ends** by checking every native share for a header an accepted legacy + share holds, throttled the same way, and recording 2 in one short + transaction. That transaction waits at most 2 s for the migration lock or + the cursor table's lock, and is tried again after a backoff for about ten + minutes; a refusal that something held either through every attempt leaves + 2 unrecorded, and a rerun once it is released goes straight to the check + and the record. An attempt that recorded 2 but lost its reply ends the run + as a success: the run reads that 2 is recorded before it would try again, + on a fresh connection if the attempt's own one was lost. If the lost + connection shows 2 unrecorded when checked, the run stops, as the runners' + lock went with that connection: run it again. A COMMIT still in flight + there, waiting for a synchronous standby say, can land moments later, and + the rerun then finds 2 recorded; otherwise it records 2. + A run that records 2 prints one JSON object with `"recorded": true`. A + rerun on a database whose backfill finished already prints + `"already_complete": true` and succeeds. If it refuses with `native share <id> ... + repeats header <h> ... so that header was credited twice`, frontends may + keep serving: report the double credit and reconcile it before anything + records 2, and never record 2 by hand. Its other refusals are in + [the refusal table](prism-rust-migration.md#serving-with-the-backfill-pending-fence-2). +6. **Afterwards** `self-check` no longer reports the field, the archive steps + work again, and the share hashes can be compared with the source's + ([Recovery evidence while the backfill is pending](prism-rust-migration.md#recovery-evidence-while-the-backfill-is-pending)), + on a standby with replay paused, never on the primary. + +Plain `qbit-prism-server migrate` at fence 2 runs the same backfill at the +default throttle, after any other pending migration. + +## Share ledger indexes + +`qbit_share_ledger` is append-only and every insert maintains every index, +so an index nobody scans, or INCLUDE payload nobody reads, is write +amplification without a reader. Migration 013 (#153) trimmed the secondary +indexes to the native query set below. The table is what to check against +before adding an index or a query that reads the ledger. + +Migration 017 (#144) changed where that set lives, not what is in it. The +ledger is a partitioned table now, so the parent carries the primary key and +the four secondary indexes as partitioned indexes and every leaf carries one +index of its own for each of them, adopted from the release table for +`qbit_share_ledger_p0` and created with the partition for every cell after it. +The definitions below are the parent's; `pg_stat_user_indexes` reports the +leaves, one row per partition per index, and that is where sizes and scan +counts are read. Each leaf's indexes are cache-resident while the partition is +hot and are never touched again once it is not. + +The one index that could not survive the conversion is the global +`UNIQUE (share_id)`: a unique index on a partitioned table must include the +partition key, and `share_id` is not it. +`qbit_share_ledger_share_id_key` is now the leaf index +`qbit_share_ledger_p0_share_id_key`, and every partition created afterwards +gets its own `<partition>_share_id_key` from +`qbit_prism_share_partition_create`. The authority on `share_id` uniqueness +across the whole ledger is `qbit_prism_share_hashes`, which the native append +already wrote one row of per share, in the same transaction as the ledger row, +keyed by the header hash with `UNIQUE (share_id)`. It is not partitioned, its +rows are never removed, it stays O(1) per share, and the append path consults +it first: a header hash it holds under the same `share_id` is an exact replay, +under another `share_id` it is the cross-identity duplicate refused as before, +and a header hash it does not hold is a new share. + +| Index | Definition | Native readers | Plan | +| --- | --- | --- | --- | +| `qbit_share_ledger_pkey` | `(share_seq)` | every `share_seq` walk that projects share rows: the payout page walk of `snapshot`, the audit range reads, `qbit_prism_window`'s ranking pass, the rollup batch in `rollups.sql`, the latest-share probe | index scan, then the heap for the projected columns | +| `<partition>_share_id_key` | `(share_id)`, unique, one per leaf, no partitioned parent | the replay comparison in the append path, the block-only reconciliation probes, the vardiff evidence lookup, `share_accepted_at_ms` | index scan on the leaves a bounded probe leaves after pruning | +| `qbit_share_ledger_accepted_seq_walk_idx` (013) | `(share_seq DESC) INCLUDE (job_issued_at, accepted_at, share_difficulty) WHERE accepted` | `qbit_prism_window`'s newest-first page walk (pool snapshot, reward leaderboard), the landing durable-range count under the settlement lock, `max(share_seq)`, the rollup boundary and tail passes | index-only | +| `qbit_share_ledger_accepted_recent_idx` | `(accepted_at DESC) INCLUDE (share_difficulty, miner_id, share_seq) WHERE accepted` | pool hashrate series, leaderboard window, pool snapshot rollups, the miner summary's pool figure, evidence counts | index-only | +| `qbit_share_ledger_accepted_miner_history_idx` (013) | `(miner_id, accepted_at DESC) INCLUDE (share_difficulty, share_seq, share_id) WHERE accepted` | miner share summary, worker rows (`share_id` carries the worker name), miner hashrate series and rollups (`share_seq` against the watermark) | index-only | +| `qbit_share_ledger_accepted_block_suffix_idx` | `((lower(right(share_id, 64))), accepted_at DESC, share_seq DESC) INCLUDE (miner_id, share_difficulty, network_difficulty) WHERE accepted AND length(share_id) >= 65` | the block-solver lookup in blocks, leaderboard, reward leaderboard and pool snapshot | index scan, one row per block | + +Dropped by 013, with no native reader: + +- `qbit_share_ledger_accepted_seq_window_idx`, `share_seq DESC` with seven + INCLUDE columns. Every `share_seq` walk was planned on the primary key, so + the payload never earned an index-only read; the narrow replacement is the + one the planner takes. +- `qbit_share_ledger_accepted_miner_recent_idx`: its `payout_order_key` + INCLUDE had no reader. +- `qbit_share_ledger_accepted_window_idx`, `(job_issued_at, share_seq DESC)`: + `job_issued_at` is only ever a filter on a `share_seq` walk. +- `qbit_share_ledger_template_height_idx`: no native query filters on + `template_height`. `qbit_shares_since_template_height`, the 001 operator + replay function, is its only caller and now reads the primary key with a + filter; if a native consumer appears, restoring it is one + `CREATE INDEX CONCURRENTLY ... ON qbit_share_ledger (template_height, share_seq) WHERE accepted`. + +`WHERE accepted` stays on every partial index. The native writers only +insert accepted rows, but every reader excludes rejected rows by contract +(the window and rollup tests write `accepted = false` rows to prove it), and +the predicate is what lets the frozen `qbit_prism_window` walk stay +index-only without carrying the column. + +**The probe floor rule for any new `share_id` query.** A `share_id` lookup +with no `share_seq` bound has to descend one leaf index per attached +partition, because uniqueness is per leaf and the executor cannot prune on a +column that is not the partition key. Every native `share_id` read of the +ledger therefore carries +`share_seq >= qbit_prism_share_probe_floor()`, a `STABLE` function that +returns the lower bound of the attached partition two below the one the next +`share_seq` lands in, read from `qbit_prism_share_partitions`, so PostgreSQL +prunes to at most three leaves holding rows (plus the empty lead) at executor +start, whatever width each partition was created with. The proof is +`Subplans Removed` in the plan of the query. Carry the same bound in any new query that looks a +share up by `share_id`; if the question is only whether a header was ever +credited, read `qbit_prism_share_hashes` instead and do not touch the ledger +at all. An unbounded probe stays correct and gets slower with every partition: +the design spike measured 31 buffers against 8. + +The one place that still probes without the bound is the append path's replay +comparison, and only on a miss: when `qbit_prism_share_hashes` says the header +was credited under this `share_id` but the bounded probe does not find the row, +the append probes once more without the bound, so a replay of any online row is +still compared field by field. A row that has left the online ledger cannot be +compared and is refused as the duplicate it is +(`duplicate-share: header already credited globally, and its share is +archived`). The coordinator's replays are seconds old, so that refusal is +unreachable in practice and safe if reached. + +One invariant this leaves is worth stating plainly. A row inserted around the +append path, by a direct `INSERT` from an operator or a harness, with a +`share_id` already present in another partition is not refused by PostgreSQL. +The native writers cannot create one; `share-archive plan` and the archive +verification report any `share_id` present in more than one attached leaf. + +Known full scan: the boundary and tail passes of +`dashboard_hashrate_rollups.sql` bound `accepted_at` through CTE values the +planner cannot estimate, so each is a full index-only scan (of +`accepted_seq_walk_idx` after 013, of `accepted_recent_idx` before it). That +predates 013 and is a query change, not an index change. + +### Measuring the trim on production + +Run these on the production database after 013 and before scheduling #144, +in this order. The `ANALYZE` comes first: the statistics captured for #144 +were hundreds of times below the row count, and every plan is provisional +until they are current. + +```sql +ANALYZE qbit_share_ledger; + +-- Statistics after the ANALYZE, against the real row count. +SELECT c.reltuples::bigint, s.n_live_tup, (SELECT count(*) FROM qbit_share_ledger) AS rows +FROM pg_class c JOIN pg_stat_user_tables s ON s.relid = c.oid +WHERE c.relname = 'qbit_share_ledger'; + +-- Whether vacuum has run, so the visibility map is set and index-only scans +-- do not fall back to the heap; stats_reset bounds the scan counts below. +SELECT s.last_vacuum, s.last_autovacuum, s.last_analyze, s.last_autoanalyze, + s.n_tup_ins, s.n_dead_tup, age(c.relfrozenxid) AS frozen_age, + pg_postmaster_start_time(), d.stats_reset +FROM pg_stat_user_tables s JOIN pg_class c ON c.oid = s.relid +JOIN pg_stat_database d ON d.datname = current_database() +WHERE s.relname = 'qbit_share_ledger'; + +-- Every index with its size and scan count; a zero-scan index has had no +-- reader since stats_reset. +SELECT indexrelname, pg_size_pretty(pg_relation_size(indexrelid)) AS size, + idx_scan, idx_tup_read, idx_tup_fetch +FROM pg_stat_user_indexes WHERE relname = 'qbit_share_ledger' +ORDER BY pg_relation_size(indexrelid) DESC; + +-- The gate: the payout page walk as the pool snapshot runs it, with the +-- network difficulty of the moment. "Heap Fetches" near zero on +-- qbit_share_ledger_accepted_seq_walk_idx means the covering index earns +-- index-only reads; large Heap Fetches mean the visibility map is cold +-- (VACUUM qbit_share_ledger, then run it again). +EXPLAIN (ANALYZE, BUFFERS) +SELECT count(*), sum(counted_difficulty) +FROM qbit_prism_window(clock_timestamp(), (<network difficulty> * 8)::numeric); +``` + +Record the index sizes before and after 013, the scan counts, the +`Heap Fetches` lines of the page walk, and the share acknowledgement latency +histogram from `/metrics` before and after, in #153 and #144. + +## Share ledger partitions and retention + +Nothing ever left `qbit_share_ledger`, and everything that touches it paid for +that: every insert maintained the primary key, a global `UNIQUE (share_id)` and +four secondary indexes at ever-growing depth, and vacuum, base backups, restore +drills and cache hit rates all degraded with the lifetime share count. What +reads the table is narrow. The payout window reads the newest slice bounded by +difficulty, the audit of a landed block reads its own range, the dashboard +aggregates cover at most the last 24 hours, and a handful of probes by +`share_id` run seconds after the share was written. Since #267 a landed block's +audit is rebuilt from the ledger at read time rather than stored, so no aged +share row is needed online except by the audits that still depend on it. + +Migration 017 (#144) gives the ledger a shape whole partitions can leave from, +and decision D6 of #260 says when one may: shares are immutable and are never +deleted, and +retention is detach-and-archive of a whole partition after the audits that +depend on it have been sealed. This section is the operator procedure. The +reasoning, the reader inventory and the archive format specification are in +[the design record](prism-share-ledger-partitioning.md), and the conversion +itself is in +[the migration guide](prism-rust-migration.md#migration-017-the-share-ledger-partition-conversion-applied-online). + +### The layout + +The ledger is `RANGE (share_seq)`. `share_seq` is the routing key every ordered +read already carries, a `bigserial` routing key cannot fail, and +`PRIMARY KEY (share_seq)` survives on the parent. There is no DEFAULT +partition: a missing partition would otherwise silently collect rows that a +later ATTACH would have to move. + +Partitions are cells of a grid. Each new one is `partition_rows` wide and +starts where the highest attached one ends, so with the width unchanged cell k +covers `[k*rows, (k+1)*rows)`; `rows` is `partition_rows` in +`qbit_prism_share_partitioning`. The default is 2^24 = 16,777,216 rows, about +7.5 GB of heap plus about 20 GB of indexes at the production row shape, five +days at 39 shares per second and nine hours at 500. Names are +`qbit_share_ledger_p<n>`, n one above the highest number the catalog has ever +recorded in any state (`qbit_prism_share_partition_next_number()`): a name is +never reused, and never derived from the width. Changing the width affects +partitions created afterwards only; bounds already attached stay as they are, +and the next partition starts at the last bound with the new width. The +probe floor (`qbit_prism_share_probe_floor()`, below) is read from the +attached bounds, not from the width, so raising the width does not widen the +probes over the narrower partitions still attached. The +release table becomes `qbit_share_ledger_p0`, `[MINVALUE, bound)`, and spans +as many cells as it needs. + +Two catalog tables record the settings and what happened to each partition. +PostgreSQL's `pg_inherits` remains the authority on what is attached and with +which bounds; the catalog adds what happened to a partition after it left, and +every `share-archive` command cross-checks the two. + +`qbit_prism_share_partitioning`, one row: + +| Column | Meaning | +| --- | --- | +| `singleton` | always true, the primary key of the one row | +| `partition_rows` | the grid width in rows (default 16,777,216, accepted range 1,048,576 to 1,073,741,824) | +| `lead_partitions` | how many empty partitions to keep attached ahead of the sequence (default 4, accepted range 1 to 64) | +| `conversion_bound` | the exclusive upper bound chosen for the release table when 017 prepared the conversion | +| `converted_at` | when the swap completed; NULL until then, and the maintenance function does nothing while it is NULL | +| `updated_at` | when this row last changed | + +`qbit_prism_share_partitions`, one row per partition: + +| Column | Meaning | +| --- | --- | +| `partition_name` | the relation name, the primary key | +| `lower_seq`, `upper_seq` | the recorded bounds; `lower_seq` is NULL for MINVALUE, which only `qbit_share_ledger_p0` has | +| `state` | `attached`, `detached` or `dropped` | +| `attached_at` | when the partition joined the ledger | +| `sealed_at` | when the last audit row depending on this partition got its stored canonical bytes | +| `archive_uri`, `archive_manifest_sha256` | where the archive was written and the SHA-256 of its manifest | +| `archive_rows`, `archive_rows_sha256` | the row count in the archive and the SHA-256 of the uncompressed row stream | +| `archived_at`, `archive_verified_at` | when the archive was written and when it was last re-read and compared | +| `detached_at`, `dropped_at` | when the partition left the ledger and when its relation was dropped | + +The table's own CHECK constraints hold the lifecycle: an archive URI implies a +manifest digest and an `archived_at`, a verification implies an archive, a +`detached` state implies `detached_at`, a `dropped` state implies both +timestamps, and any state other than `attached` implies +`archive_verified_at IS NOT NULL`. A partition cannot be recorded as having +left without having been archived and verified first. + +### Lead partitions + +`qbit_prism_share_partition_ensure()` keeps `lead_partitions` empty partitions +attached above the next `share_seq`, about 67 M rows of headroom at the +defaults. Every instance calls it at startup and then every +`PRISM_SHARE_PARTITION_ENSURE_INTERVAL_SECONDS`, default 60. The call takes +the online migration runner's advisory lock, so it is serialized per schema, +two frontends never race on one partition name, and no partition is created +while a conversion is swapping the table. It creates +nothing when the lead is intact and returns the number it created. + +A new partition is a standalone table `LIKE` the parent with a validated bound +`CHECK`, its own `UNIQUE (share_id)` and the immutability trigger, and is then +attached. The attach takes only SHARE UPDATE EXCLUSIVE on the parent: appends +and reads continue. A name already held by any relation is refused and never +adopted: + +``` +refusing to create share ledger partition qbit_share_ledger_p7: a relation already holds that name +``` + +Should an insert ever find no partition for its `share_seq`, PostgreSQL +refuses it with SQLSTATE 23514 and the append path runs `ensure` once and +retries the share, so a lead that ran out costs one retry rather than a lost +share. That is the recovery, not the plan: if it is reached, the ensure task is +not running on any instance, or the lead is too small for the share rate. +Alert on the distance between the highest attached `upper_seq` and +`qbit_prism_share_next_seq()`, and raise `lead_partitions` rather than relying +on the retry: + +```sql +SELECT (SELECT max(upper_seq) FROM qbit_prism_share_partitions WHERE state = 'attached') + - qbit_prism_share_next_seq() AS lead_rows, + (SELECT lead_partitions * partition_rows FROM qbit_prism_share_partitioning WHERE singleton) AS target_rows; +``` + +### The online horizon + +A share row stays online while any of these holds. All five are properties of a +whole partition, `share-archive plan` checks each one and names its blocker, +and a partition leaves only when all five clear. + +1. **It can still be in a payout window.** Its `share_seq` is at or above the + floor of the current window taken at four times the requested weight, + `qbit_prism_window(clock_timestamp(), 8 * D * 4)` with D the network + difficulty the operator supplies, so a difficulty rise of up to 4x between + two retention runs cannot reach into archived history. +2. **It is younger than the retention age**, 30 days by default. The longest + dashboard read of raw rows is 24 hours, so this covers every one of them + with margin. +3. **The hashrate rollup watermark has not reached its newest row.** While + `qbit_hashrate_rollup_progress` is behind the partition's newest committed + `share_seq`, or the share sequence has not yet passed the partition's upper + bound, the permanent rollup tables do not yet hold its whole contribution. + The mark is the newest row, not the bound: the sweep advances only to rows + that committed, and a `share_seq` an append drew and rolled back is never + folded. +4. **A landed block's audit still depends on it**, that is, an audit row whose + share snapshot intersects the partition and that has no stored + `canonical_audit_bytes`. Sealing clears this condition. +5. **An unfinished block candidate or a deferred share references it.** + +### Sealing, per D6 + +Before a partition is detached, every audit row whose snapshot intersects it is +sealed: the canonical artifact is rebuilt from the still-online shares, its +digest is checked against the advertised `audit_bundle_sha256`, and the bytes +are stored in `canonical_audit_bytes`, exactly as imported legacy audits are +stored since #325. The block then keeps serving under its advertised digest +after its shares are gone, through the artifact route, the audit bundle route +and the operator tools alike, because readers prefer stored bytes over +reconstruction whatever the row's shape. + +The price is the artifact size back per archived block, about 111 MB +uncompressed at 100,000 shares, TOAST-compressed in the column. The +alternative, pinning every landed block's window online forever, would have +left cost scaling with blocks found, which is what this work exists to stop. +Sealing is idempotent and can run ahead of any detach, so run it early and +keep it out of the critical path of the detach. + +Two documented reader changes follow from a detach, and both are intended: + +- A miner's `last_share_at` in the miner share summary is the newest share + within the online horizon, so a miner whose last share is older than the + retention age reads `null`. +- `/audit/share-window?anchor=` for an anchor inside an archived range returns + `rows: []`. The shares are in the archive and in every sealed artifact whose + window covers them. An anchor-to-archive index is deferred. + +Everything else that was lifetime-scoped keeps working across a detach. Block +solver attribution moved onto `qbit_pool_blocks.solver_*` in migration 016, +written at landing and backfilled for every existing block. +`accepted_share_count` and `distinct_miner_count` in `/audit/latest-evidence` +and `GET /public/v1/hashrate-series?range=all` are served from the permanent +rollup tables plus the raw tail above the watermark, which condition 3 keeps +online until the sweep has folded it. + +### The operator procedure + +`qbit-prism-server share-archive <command>` runs as the operator against the +primary, with the frontends running. Nothing here needs a maintenance window. + +| Command | Effect | +| --- | --- | +| `plan --network-difficulty D [--retention-days N] [--window-multiple M] [--check-duplicates]` | every partition with its bounds, row count, age, and each of the five conditions above with its blocker named; nothing is changed | +| `seal <partition>` | stores canonical bytes for every audit row whose snapshot intersects the partition and has none, verifying each against its advertised digest; records `sealed_at` when none is left | +| `archive <partition> --dir <root> [--force]` | writes `<root>/qbit_share_ledger/<partition>/<manifest-sha256>/rows.ndjson.gz` and `manifest.json`, records the URI, digests and row count; refused while the share sequence has not passed the partition, since appends could still land in it, and refused out of order, so the chain of manifests stays contiguous, and refused over a predecessor whose archive is not verified, so every link is to a certified manifest; `--force` writes an archive again, clearing its verification and that of every later archive, which must then be written and verified again in order, each over its verified predecessor, and is refused once a later archived partition has left the ledger | +| `verify <partition> --dir <root>` | re-reads the archive, checks both digests and that the manifest chains, without a gap, to the nearest archived partition, whose own archive has to be verified, and, while the partition is attached, streams the live rows again and compares; records `archive_verified_at` for that full comparison, and only once the share sequence has passed the partition | +| `detach <partition> --network-difficulty D [--retention-days N] [--window-multiple M] [--check-duplicates]` | requires every plan condition, sealed, archived and verified, and counts the live rows against the archive again; `DETACH PARTITION ... CONCURRENTLY`, or FINALIZE for an attempt interrupted after PostgreSQL marked the partition detach-pending, held to every condition again because the mark does not tell a `share-archive detach` from a statement run by hand; a pending partition that fails one is brought back with the `FINALIZE` and `ATTACH PARTITION` statements the refusal prints. The table stays as a standalone relation. A partition PostgreSQL no longer holds, whether an earlier run's catalog update was lost or the DDL ran by hand, is only reconciled into the catalog as sealed and verified; otherwise it is attached again under its recorded bounds and sealed first | +| `drop <partition> --dir <root>` | requires `detached`, sealed and verified; reads the recorded archive back from disk, checking both digests against the catalog, and counts the rows against it again; `DROP TABLE`; the archive is the copy of record | +| `restore <manifest> --dir <root> [--attach]` | recreates the partition table from the archive, verifies count and digests, and optionally attaches it under its recorded bounds | + +`plan` also reports the attached partition count, the lead ahead of the +sequence, and any `share_id` present in more than one attached leaf. While a +partition carries the detach-pending mark PostgreSQL hides its rows from every +new query of the parent, including the payout window, so `plan` counts the +hidden rows into that condition itself; the rollup sweep cannot, so never run +`DETACH PARTITION` by hand on a partition whose rows are not yet folded. + +A session that retires the oldest partition, with the network difficulty of the +moment and the default retention age: + +```sh +ARCHIVE_ROOT=/var/lib/qbit-prism/share-archive + +qbit-prism-server share-archive plan --network-difficulty 402304 --retention-days 30 +qbit-prism-server share-archive seal qbit_share_ledger_p0 +qbit-prism-server share-archive archive qbit_share_ledger_p0 --dir "$ARCHIVE_ROOT" +qbit-prism-server share-archive verify qbit_share_ledger_p0 --dir "$ARCHIVE_ROOT" +qbit-prism-server share-archive detach qbit_share_ledger_p0 --network-difficulty 402304 --retention-days 30 +qbit-prism-server share-archive drop qbit_share_ledger_p0 --dir "$ARCHIVE_ROOT" +``` + +Run `plan` again after `drop` and keep its output with the run. `verify` +compares the archive against the live rows only while the partition is still +attached, so the order above is the order that gets that comparison: archive, +verify, then detach. Both refuse a partition the share sequence has not passed, +because a comparison of a partition that can still receive rows proves nothing +about the rows still to come; and since ledger rows are immutable, `detach` and +`drop` each count the rows against the archive again before acting, so an +append that committed after the comparison is refused rather than lost. `drop` +also reads the archive back from disk under `--dir` right before `DROP TABLE`: +`archive_verified_at` proves the archive was complete when it was compared, not +that its files are still there, and a copy of record that went missing or was +altered in between is refused rather than made the only copy. +Between `detach` and `drop` the rows are still on disk +under the standalone relation and can be read directly by name, which is the +last chance to look at them without a restore; do not collapse those two steps +into one to save a command. Read the catalog back afterwards: + +```sql +SELECT partition_name, lower_seq, upper_seq, state, sealed_at, archived_at, + archive_verified_at, detached_at, dropped_at, archive_rows, + archive_uri, archive_manifest_sha256, archive_rows_sha256 +FROM qbit_prism_share_partitions ORDER BY upper_seq; + +-- What PostgreSQL actually holds attached, which is the authority. +SELECT c.relname, pg_get_expr(c.relpartbound, c.oid) AS bounds +FROM pg_inherits i JOIN pg_class c ON c.oid = i.inhrelid +WHERE i.inhparent = 'qbit_share_ledger'::regclass +ORDER BY c.relname; +``` + +### What to keep outside PostgreSQL + +The archive is the copy of record for every share that has left the ledger, so +the archive root is accounting data. Keep it off the database volume, back it +up with the same retention and encryption as the database backups, and keep it +in the isolated-restore evidence set described in +[one-way migration and isolated-restore reconciliation](#one-way-migration-and-isolated-restore-reconciliation). +A share range that has left the online ledger is proved from its manifest +chain, not from a database export. + +The layout under the root: + +``` +<root>/qbit_share_ledger/<partition_name>/<manifest-sha256>/ + rows.ndjson.gz gzip; one JSON object per line, share_seq ascending + manifest.json UTF-8, no trailing newline +``` + +Every `archive` run writes into a new version directory named after the +SHA-256 of the manifest it holds, and the catalog points at that version only +once both files and their directory entries are durable: `archive_uri` is the +full path of the recorded `manifest.json` and `archive_manifest_sha256` is the +directory's name. A partition directory can hold more than one version after +`--force` or an interrupted run; the earlier ones stay on disk so that a failed +catalog update cannot destroy the copy of record, but only the version the +catalog records is that copy, and `verify` reads that one. An archive written +before this version scheme, with both files directly under the partition +directory, is still read by `verify`, which checks its digest against the +catalog as for any other. + +A row is the whole ledger row with a fixed key order: difficulties are decimal +strings (`numeric(78,0)`), timestamps are microseconds since the epoch (exact +for `timestamptz`), the P2MR program is hex. + +``` +{"share_seq":1,"share_id":"alice:…","miner_id":"alice","payout_order_key":"…", + "p2mr_program_hex":"…","share_difficulty":"1","network_difficulty":"100", + "template_height":100,"job_id":"job","job_issued_at_us":1000000, + "accepted_at_us":1758000000000000,"ntime":1,"accepted":true,"reject_reason":null, + "credit_policy":null,"writer_id":"…","writer_epoch":0} +``` + +The manifest carries `schema` (`qbit.prism.share-archive.v1`), the recorded +bounds, what the rows hold (`row_count`, first and last `share_seq`, first and +last `accepted_at_us`; zero rows is a valid archive), the digests and sizes of +both the uncompressed stream and the file (`rows_sha256`, `rows_gz_sha256`, +`rows_bytes`, `rows_gz_bytes`), the chain link, the +`qbit_prism_schema_migrations` versions at archive time, and who created it +when. The full field list is in +[the design record](prism-share-ledger-partitioning.md#archive-format-v1). + +**The manifest chain.** `previous_manifest_sha256` and `previous_upper_seq` +point at the archived partition with the next-lower `upper_seq`, and are null +only for the first. A gap between one manifest's `previous_upper_seq` and its +own `lower_seq` means a partition is missing from the chain; `archive` refuses +to write such a manifest and `verify` refuses to certify one, so partitions are +archived in `upper_seq` order. The manifest's own SHA-256 is what the catalog +and the next manifest record, so a manifest cannot be rewritten without +breaking both: `archive --force` clears the verification of every later +archive along with its own, and each has to be written again with `--force`, +in `upper_seq` order, and verified again. That order is enforced, not +assumed: a link proves one hop, and `verify` checks only the manifest's own +link, so both `archive` and `verify` refuse a partition whose nearest archived +predecessor has no `archive_verified_at`. A verification therefore stands for +the whole chain below it, and a repair proceeds from the rewritten partition +up, verify then archive then verify; a later partition cannot be written +against a middle manifest whose own link is obsolete, certified on that one +hop, and detached over a chain that can then never be repaired. The rewrite +is refused once a later archived partition has been detached or dropped, +because that partition can no longer be archived from its live rows; bring +the missing files back from a copy of the archive root instead. + +**Verifying an archive by hand.** `rows_sha256` is the SHA-256 of the +uncompressed byte stream, so a verifier streams the file without materializing +it. Start from the version directory the catalog records, which is the +`archive_manifest_sha256` from the catalog read above (the `archive` command +prints the same value); the off-host copy has no catalog to ask, so carry the +digest along with it: + +```sh +MANIFEST_SHA256=… # archive_manifest_sha256 for qbit_share_ledger_p0 +cd "$ARCHIVE_ROOT/qbit_share_ledger/qbit_share_ledger_p0/$MANIFEST_SHA256" +gzip -dc rows.ndjson.gz | sha256sum # must equal the manifest's rows_sha256 +sha256sum rows.ndjson.gz # must equal the manifest's rows_gz_sha256 +gzip -dc rows.ndjson.gz | wc -l # must equal the manifest's row_count +sha256sum manifest.json # must equal $MANIFEST_SHA256, the directory's name, + # and the next manifest's previous_manifest_sha256 +gzip -dc rows.ndjson.gz | head -1 # the first row; its share_seq is first_share_seq +gzip -dc rows.ndjson.gz | tail -1 # the last row; its share_seq is last_share_seq +``` + +Do this on the archive copy that was written off the database host, not only on +the one the tool just produced; the point of the check is the copy that will +outlive the rows. + +### Restoring a partition + +`share-archive restore <manifest> --dir <root>` recreates the partition table +from the archive and verifies count and digests. With `--attach` it also +attaches it back under its recorded bounds, which makes the rows readable +through `qbit_share_ledger` again. Restore into an isolated database for +inspection wherever the question does not require the production ledger. +Attaching back into production is for reconciliation, not for routine reads: +the partition's rows are outside the online horizon by construction, so every +dashboard query that reaches them pays for a leaf that nothing else needs. +After an inspection, run `verify` and then `detach` and `drop` again rather +than leaving the partition attached. + +### Measuring before and after + +Acceptance criterion 4 of #144 is this same set of measurements on a +production-sized copy, before and after 017, taken by the operator. It needs +production access. Run `ANALYZE qbit_share_ledger` first, both times: the +statistics captured for #144 were hundreds of times below the row count, and +every plan is provisional until they are current. After 017 that statement +analyzes the parent and every attached leaf. + +**Insert latency.** The share acknowledgement histogram +`qbit_prism_share_ack_seconds{result="accepted"}` in `/metrics` is the +end-to-end bound on a share: `mining.submit` frame arrival to completed +response write. Scrape it on every coordinator before the conversion and again +after, over comparable load, and compare percentiles from the bucket counts +rather than the average. These are elapsed ACK bounds, not measured ledger +deadlines, so read them alongside +`qbit_prism_database_pool_acquire_seconds` and +`qbit_prism_database_advisory_lock_wait_seconds` to tell an ingest change from +a database one. + +**Vacuum duration.** The point of the conversion is that vacuum stops scanning +lifetime history, so measure it per partition and against the recorded +duration of the old whole-table run: + +```sql +\timing on +VACUUM (VERBOSE, ANALYZE) qbit_share_ledger_p0; +VACUUM (VERBOSE, ANALYZE) qbit_share_ledger_p1; +``` + +Record the elapsed time, the index scan lines and the page counts of each, and +the sum against the single pre-017 `VACUUM (VERBOSE, ANALYZE) qbit_share_ledger`. + +**Sizes and scan counts per partition.** After 017 there is one row per leaf +per index, which is where a partition that no reader touches becomes visible: + +```sql +SELECT c.relname AS partition, + pg_size_pretty(pg_total_relation_size(c.oid)) AS total_with_indexes, + pg_size_pretty(pg_relation_size(c.oid)) AS heap, + s.n_live_tup, s.last_vacuum, s.last_autovacuum, s.last_analyze +FROM pg_inherits i +JOIN pg_class c ON c.oid = i.inhrelid +LEFT JOIN pg_stat_user_tables s ON s.relid = c.oid +WHERE i.inhparent = 'qbit_share_ledger'::regclass +ORDER BY c.relname; + +SELECT relname AS partition, indexrelname, + pg_size_pretty(pg_relation_size(indexrelid)) AS size, + idx_scan, idx_tup_read, idx_tup_fetch +FROM pg_stat_user_indexes +WHERE relname LIKE 'qbit\_share\_ledger\_p%' +ORDER BY relname, pg_relation_size(indexrelid) DESC; +``` + +`idx_scan` is counted since `stats_reset`; read +`SELECT stats_reset FROM pg_stat_database WHERE datname = current_database()` +with it. + +**The page walk.** The payout window read is the query the conversion must not +regress. Run it as the pool snapshot runs it, with the network difficulty of +the moment, before and after: + +```sql +EXPLAIN (ANALYZE, BUFFERS) +SELECT count(*), sum(counted_difficulty) +FROM qbit_prism_window(clock_timestamp(), (<network difficulty> * 8)::numeric); +``` + +Record the total buffers and the `Heap Fetches` line on +`qbit_share_ledger_p<k>_accepted_seq_walk_idx`. Heap fetches near zero mean the +covering index is earning index-only reads; large heap fetches mean the +visibility map is cold, so vacuum the leaf and run it again. After 017 the walk +should touch the newest leaves only, and the older ones should not appear in +the plan at all. + +**The bounded `share_id` probe.** This is the plan that proves the probe floor +is doing its job. `Subplans Removed` must be present and must account for every +leaf outside the floor's three: + +```sql +EXPLAIN (ANALYZE, BUFFERS) +SELECT share_seq, share_id, accepted_at +FROM qbit_share_ledger +WHERE share_id = '<a recent share_id>' + AND share_seq >= qbit_prism_share_probe_floor(); +``` + +For contrast, and to keep the number that justifies the rule, run the same +query without the `share_seq` predicate and record its buffer count. It must be +the one that grows with the attached partition count. + +#### Container evidence + +Container evidence (PostgreSQL 16.15) is lock-semantics and phase-shape +evidence on a synthetic ledger, not a production absolute. It belongs here so +the production numbers above have something to be read against. + +Taken on 2026-09-16 in Docker on an Apple M-series laptop: a synthetic +ledger of 1,000,000 production-shaped rows (1,000 MB with its indexes, 500 +miners), `VACUUM (ANALYZE)` before each read, insert latency from 2,000 +single-row inserts timed in PL/pgSQL with `clock_timestamp()`. + +| Measurement | Before 017 | After 017 | Notes | +| --- | --- | --- | --- | +| Prepare, validate, swap | | 2.3 ms, 47 ms, 16.7 ms | the validation scan is the only term that grows with the table (about 300 MB of heap here); the swap renamed, created the parent, adopted five indexes, attached and created two lead partitions | +| Insert latency p50 / p95 / p99 | 0.031 / 0.048 / 0.066 ms | 0.032 / 0.047 / 0.069 ms into the release partition; 0.025 / 0.038 / 0.061 ms into a fresh lead partition | a hot leaf's indexes are small and cache-resident | +| `VACUUM (ANALYZE)` duration | 0.32 s | 0.44 s for every partition; 0.02 s for one lead partition | retention bounds the set vacuum visits | +| Page-walk plan, `qbit_prism_window` over about 200k shares | 15,809 shared buffers | 17,353 shared buffers | the same recursive page walk through the partitioned parent | +| Bounded `share_id` probe | | 8 shared buffers over the three attached leaves | `Subplans Removed` appears once the floor clears a partition; at 1 M rows the floor is still 0 | +| Unbounded `share_id` probe | | 6 shared buffers | one index descent per attached leaf; grows with the attached count, which is the cost the bound removes | + +## Offline pool-fee and CTV fee-rate changes + +`qbit-prism-server policy-transition --to /protected/next.env` changes the +cluster's pinned fee policy after **every frontend has stopped**. It supports +pool-fee enablement, recipient and basis-point changes, and explicit/automatic +CTV fee rates and premiums. Signing keys, genesis, username fallback, output +ordering, payout thresholds, CTV enablement and settlement layout must stay +unchanged. Signing-key rotation has its own command, +[`signing-transition`](#signing-key-rotation); multi-epoch verification +remains deferred. + +1. Preserve the current configuration, signing material and database backup. + Prepare a protected env file containing the target overrides. For example, + for an already-enabled pool fee and CTV settlement: + + ```dotenv + PRISM_POOL_FEE_BPS=200 + PRISM_CTV_FANOUT_FEE_MARKET_RATE_BITS_PER_1000_WEIGHT=2000 + PRISM_CTV_FANOUT_FEE_PREMIUM_BPS=12000 + ``` + + These are examples, not market-rate recommendations. The command validates + the target CTV fee against the node's current relay and mempool floors; it + can replace an old rate that is already below those floors. Mainnet still + requires an explicit rate. Rates can change again before restart. + + Every target must keep a pool fee, in either settlement mode (#525, #535): + `PRISM_POOL_FEE_ENABLED=1` with a recipient, `PRISM_POOL_FEE_BPS=0` + allowed. Without one, sub-floor dust cannot be settled and the first + balance below the payout floor fails every work build, so `run`, + `check-config`, `self-check` and this command's target all refuse it. The + current configuration is still read when it lacks a fee, so this command + is also how a cluster that ran without one enables it. A frontend that ran + before #535 with the fee off, including one started from an older + `.env.example` or from `compose.yaml`'s fee-off default, refuses to start + until it does: + stop every frontend and run this command with a target such as + + ```dotenv + PRISM_POOL_FEE_ENABLED=1 + PRISM_POOL_FEE_BPS=0 + PRISM_POOL_FEE_ADDRESS=<reviewed P2MR fee address> + ``` + + then set the same values in the frontends' environment. A 0-bps fee pays + nothing and adds no coinbase output until dust must be swept. + +2. Disable automatic restarts and stop every frontend and standalone candidate + or CTV worker. Graceful SIGTERM closes miner admission and drains workers; + in the bundled stack use + `docker compose stop --timeout 45 prism-coordinator prism-coordinator-2`. + Confirm every `qbit_prism_instances.status.state` is `stopped`. A stale, + `starting`, `draining`, `drained`, unknown or live heartbeat is refused by + instance name. There is no force flag. If a crashed instance cannot record + its own stopped marker, escalate for reviewed recovery; do not manufacture + markers with SQL. Keep the qbit node and database available. + +3. With the **current** configuration in the command's environment, run: + + ```sh + qbit-prism-server migrate + qbit-prism-server policy-transition --to /protected/next.env + ``` + + Migration `014` adds the immutable transition journal (`013` is used for + the share ledger index trim). The command creates no frontend heartbeat. + The target file overlays current `PRISM_*` and `QBIT_*` variables; omitted + values retain their current values, and an empty value clears an optional + setting. It supports dotenv quoting and `export`, never shell execution. + Other stack variables are ignored. The target database URL must be unchanged. + Both effective policies are checked with the normal configuration parser + and pool-fee address resolution. The current fingerprint must match the + database, and the target must actually change the policy. + +4. Save the returned JSON. Under the settlement and ordering locks, with + concurrent frontend registrations excluded, one transaction updates the + fingerprint, increments `payout_revision` once, and journals the old/new + public policies, revisions, database login, instance snapshot and candidate + counts. It abandons unoffered pending candidates with `epoch-superseded` and + releases their retained window/block payloads. A pending candidate with an + already-landed block is refused until reconciled. Offered, offer-reserved + and reconciliation candidates retain their original bytes, policy and keys; + their old claims are fenced and they can resume immediately on restart + without another offer. Parked rows stay parked for operator recovery. + Existing audit and CTV manifest bytes are unchanged. Recovery evidence + exports include the transition journal and its allocation sequence. + + A failure before commit leaves the policy and candidate dispositions intact. + A timeout or lost commit response can mean success: inspect + `qbit_prism_policy_transitions` and the cluster fingerprint/revision before + retrying. Repeating a committed transition with the old environment fails + the current-fingerprint check and does not increment the revision again. + The journal contains public keys, never signing seeds or credentials. + +5. Apply the effective target configuration to **all** frontends, run the normal + `check-config`, then restart and verify health before restoring miner traffic. + Restarting with the old policy is rejected and names the active payout + revision. Miners disconnect during the stop and receive fresh work after + reconnecting; revision-bound jobs prepared before the transition cannot be + issued. Historical verification continues to use the unchanged public keys. + +## Signing-key rotation + +`qbit-prism-server signing-transition --confirm` resets the pinned cluster +fingerprint so that frontends configured with new signing keys can pin theirs. +It is the guarded form of the reset the rotation rehearsal (#291) performs by +hand, and it leaves a durable record. The fingerprint is the SHA-256 of the +policy document, which includes both public keys (`ledger_key`, +`manifest_key`), so any key change is a fingerprint change, and +[`policy-transition`](#offline-pool-fee-and-ctv-fee-rate-changes) keeps +refusing it. Migration `019` adds the immutable journal +`qbit_prism_signing_transitions`; run `qbit-prism-server migrate` first. + +The one outcome the command exists to exclude is a reset while a frontend +still holds a claim, an offer reservation or a fresh heartbeat. Its checks and +its one write happen in one transaction that holds the settlement and order +locks, the instance registration lock, and the cluster row `FOR UPDATE`, the +same row lock every `configure` takes and every candidate writer reads `FOR +SHARE`: a concurrent startup or write serialises behind it and then sees the +reset. There is no force flag. Without `--confirm` the command prints what it +would check and do, reads neither its configuration nor the database, +changes nothing and exits non-zero. + +1. Preserve the current configuration, both old signing seeds, their public + keys and a database backup. Generate the new key pair; keep the old public + keys (they also end up in the journal): every bundle signed before the + rotation verifies only with them. + +2. Disable automatic restarts and stop every frontend and one-shot tool. + Confirm the candidate outbox is drained: the command refuses while any + candidate is `pending`, `offer_reserved`, `offered` or in `reconciliation`, + whatever keys it carries (stricter than `configure`, which only refuses + other keys), and names the counts, states and first block hashes. Let + offered candidates land or settle through their existing recovery + procedures; `candidates abandon` handles a pending one. + +3. With the **current** (old-key) environment, the one the cluster is pinned + to, run: + ```sh - python3 -m lab.prism.recover_pending_blocks \ - --block-hash "$PARENT_HASH" --block-hash "$CHILD_HASH" + qbit-prism-server migrate + qbit-prism-server signing-transition --confirm ``` - This takes no writer lease and reads no candidate payloads. Every hash must - be on the active chain and have a pending outbox row, or already have - submitted/confirmed accounting and persisted audit evidence. Include any - pending parent; the command sorts the selection by height. -2. Stop the normal coordinator and its restart supervisor. Launch a fresh - container with the same managed configuration, replacing its normal - coordinator command with the command above plus `--apply`. Recovery refuses - a live writer lease, acquires a fresh guarded session, and runs the normal - lease heartbeat and accepted-block accounting. It loads one selected - candidate at a time, verifies its stored identity and chain header, and - completes the outbox only after the normal finalizer succeeds. It starts - no Stratum listeners and makes no `submitblock` calls. -3. Check for a final `success` event and exit status zero. Failure stops the - run; repeat the same allowlist to resume after investigating. Completed - blocks are verified and skipped. The processing and cleanup deadline is - 600 seconds after construction (`--timeout-seconds`, range 1–3600). -4. Restart the managed coordinator, verify the selected blocks' accounting - and audit evidence, and check full subscribe/authorize/notify health on - both Stratum listeners. - -Recovery uses Python's cooperative JSON codecs only in its fresh process to -reduce long interpreter stalls while the heartbeat runs. This costs throughput -and still holds one complete candidate in memory; it does not replace the -bounded-payload work in #255. Lease timings, signing identity, and payout rules -remain unchanged. Logs include JSON progress events with schema -`qbit.prism.pending-block-recovery.v1` alongside normal coordinator logs. - -Alert before the landing deadline is exhausted, not after: page when -`qbit_prism_prior_balances_read_max_seconds` exceeds ~20% of the -landing budget or the poll budget, when any -`qbit_prism_block_ledger_call_timeouts_total{call_class="landing"}` -increment occurs, and when -`qbit_prism_accepted_parent_unresolved_oldest_seconds` exceeds the -preview wait budget. The #188 prior-balances read crossed the one-second -line silently over several weeks; these series exist so that growth is a -ticket, not an outage. - -Deadlines are split by call class. The poll-class budget above covers only -cheap outbox polls and fast-lane-adjacent calls. The landing-class -accounting tail — persisting an accepted block, reading prior balances, -confirming it, and rejecting the prepared state of a terminal candidate — -runs each statement under `PRISM_BLOCK_LANDING_DB_TIMEOUT_SECONDS` -(default 30 seconds) starting with the first attempt. After an observed -landing timeout the next attempt for the same block hash doubles its -budget up to `PRISM_BLOCK_LANDING_DB_TIMEOUT_MAX_SECONDS` (default 120 -seconds); only ledger-originated deadlines count as landing timeouts — -a node RPC timing out inside the landing tail neither escalates the -next PostgreSQL budget nor increments the landing-timeout series; retries stay paced by the candidate backoff, server-side -cancellation is confirmed by the ledger backends (the pooled session is -rolled back or replaced, never reused mid-cancel), and the stuck-call and -coordination watchdogs remain the overall bound. Startup replay of a -pending accepted candidate re-enters the same landing-class scope, and -the gating startup outbox enumeration both runs with the landing budget -and records under `call_class="landing"`, so an enumeration timeout -fires the landing-timeout alert rather than inflating the fast-call -budget gauge. A -landing-class operation must never start at the poll budget: a -structurally slow landing under a one-second ceiling is statement-canceled -on every attempt and can never converge (issue #188). Contended -submit-path locks are acquired in heartbeat slices and identify the lock in a -periodic diagnostic controlled by `PRISM_BLOCK_SUBMIT_LOCK_WAIT_LOG_SECONDS` -(default 5 seconds). At most two timeout-ignoring RPC workers and two -timeout-ignoring ledger workers may remain detached. If either bounded worker -pool remains exhausted for `PRISM_BLOCK_SUBMIT_STUCK_CALL_EXIT_SECONDS` -(default 30 seconds), the coordinator requests shutdown and exits nonzero so -the supervisor replaces the poisoned process; durable outbox rows remain -pending for replay. One detached call does not interrupt the healthy raw lane -while the other bounded slot can still make progress. - -Successful submissions become `submitted`; candidates that definitively lose -their tip race or fail validation become `abandoned`. If the process exits -after `submitblock` but before the attempt marker or terminal outbox update, -restart resubmits the same bytes. qbit's accepted-duplicate response is a -successful landing signal; block-hash-keyed ledger persistence and the -finalize-only registry keep accounting and terminal side effects exactly once. -Restart can also recognize the candidate as the active tip and complete the -same idempotent confirmation path. Exact miner resubmissions observe an -existing terminal outbox state in the same durable pre-submit transaction: -`submitted` coalesces to success and `abandoned` stays rejected before any new -node offer. Exact share replays return the original row as not newly inserted, -so process-local worker and vardiff counters are not credited twice. -Transient RPC, audit, and ledger outcomes remain pending and retry with an -exponential delay starting at 250 milliseconds and capped at 30 seconds. They -do not increment terminal abandonment counters. An abandonment is counted only -after any prepared payout state is rejected and the false disposition is fixed; -if cleanup fails, the candidate remains pending and can still converge to -submitted on later chain evidence. Replay carries the database row's block hash -separately from candidate JSON, so malformed payloads can be quarantined using -the authoritative outbox key instead of replaying forever. -If qbit has already returned the candidate outcome but durable outbox -finalization fails, replay resumes only that finalization step with the same -bounded pacing. It does not call `submitblock` again, recount an accepted block, -rebuild or republish audit evidence, or reacquire a share-writer floor already -released after the known outcome. - -The block submitter heartbeat carries its current phase, including replay -query, node RPC, lock admission, audit, persistence, and finalization. A stale -watchdog diagnostic therefore reports a label such as -`block_submitter:replay-outbox-query` instead of only the thread name. - -When a network-valid hash is below a listener's advertised share target, the -coordinator first stores a candidate-only intent, submits it synchronously, and -links share credit only if the block lands. This closes the submit-to-credit -crash window without crediting a below-target hash that loses its tip race. -Terminal outbox rows retain the intent digest but clear the large block/template -body, bounding permanent outbox storage while preserving exact-replay checks. - -Production deployments must use the Postgres-backed ledger. The in-memory ledger -exists only for local/regtest proof runs and requires an explicit -`PRISM_ALLOW_MEMORY_LEDGER=1` opt-in. - -## Reward Query Semantics - -`qbit_prism_window(anchor_job_issued_at, window_weight)` returns accepted shares -in descending `share_seq` order until the requested weight is filled. This is -the TIDES-style reward-window primitive inside PRISM. The oldest included share -is partially counted when needed. Eligibility requires both: - -- `job_issued_at <= anchor_job_issued_at` -- `accepted_at <= anchor_job_issued_at` - -That second condition freezes the block view and prevents an old-job share that -arrives after the found-block anchor from entering the published payout split. - -`qbit_audit_share_window(anchor_job_issued_at, network_difficulty)` is the -public audit wrapper. It fixes the TIDES-style window multiplier at 8x network -difficulty and returns the counted difficulty for every included share. - -Accepted rows may carry a nullable `credit_policy`. Normal shares leave it -empty; `stale-grace` marks a prior-tip share credited by the coordinator's short -stale-grace policy. Reward-window queries still count these rows because they -are accepted shares, while audits can distinguish them from normal current-tip -shares. Audit bundles containing a credited row use -the logical `qbit.prism.audit-bundle.v1.1` schema, and the window is enabled by -default on every chain, so external auditors must run a release that accepts it. - -Deployments that run with `PRISM_POSTGRES_INIT_SCHEMA=0` must apply -`crates/qbit-prism/sql/001_share_ledger.sql` before starting any upgraded -coordinator. The file is the cumulative, idempotent schema initializer and -migration path, despite its `001` name. Skipping it can break share inserts, -reward-window calls, pool-block confirmation/reactivation, and audit evidence -publication because required columns, functions, and the durable publication -ordinal will be missing. - -Apply the file in a single transaction and stop the PRISM share writer -first. The script enforces this itself with a `BEGIN`/`COMMIT` wrapper, so -even a plain autocommit `psql -f` runs as one transaction — but pass -`--single-transaction` (or `-1`) anyway, together with `ON_ERROR_STOP`, so -the intent is explicit and a failure cannot strand an open transaction: + + The command refuses when the pinned fingerprint is not this environment's: + the journal must record the keys being retired. It refuses a halted + cluster: clear the fatal state with the current keys first, because + `fatal-state clear` compares the pinned fingerprint. It refuses while any + registered frontend is live. A row is quiescent when its status is + `stopped`, or when its heartbeat is older, by the database clock, than the + freshness window `self-check` uses: three `PRISM_HEALTH_REFRESH_SECONDS`, + never less than fifteen seconds. Run the command with the frontends' + cadence in its environment. A killed frontend never writes `stopped`; its + row stops refusing once its heartbeat is stale. A `starting` or `running` + row with a fresh heartbeat, a status this binary cannot read, or a + future-dated heartbeat (an unknown age) refuses by instance name. If the + fingerprint is already unset, the command reports the last journal row and + writes nothing, so a lost commit response is resolved by running it again. + That rerun assumes nothing pinned a fingerprint in between, which is why + restarts stay disabled from step 2 until step 5: if an old-key process did + start after a reset that had in fact committed, the rerun finds the old + fingerprint pinned again, refuses while that process is live, and otherwise + resets a second time and writes a second journal row. After an ambiguous + outcome, inspect `qbit_prism_signing_transitions` and + `qbit_prism_cluster.config_fingerprint` before rerunning. + +4. Save the returned JSON: the journal row (`transition_id`, the old + fingerprint, the old policy document with both old public keys, every + instance row with its measured heartbeat age and the window applied, the + unchanged payout revision, the database login) and the next steps. The + payout revision does not change: a rotation is not a payout-policy change. + Existing audit and CTV manifest bytes are unchanged. 120 seconds is the + ceiling for the whole command (the node calls, the transaction and its + commit); it does not bound a lock wait. Each wait for the settlement and + order locks, the instance table or the cluster row ends after + `PRISM_DATABASE_LOCK_TIMEOUT_MS` (5 seconds by default), and any one + statement after `PRISM_DATABASE_STATEMENT_TIMEOUT_MS` (15 seconds by + default). A failure or timeout before commit leaves the fingerprint pinned + and the journal without a row. On `canceling statement due to lock + timeout`, something still holds the cluster row, the instance table or + those locks, which is what a running frontend or tool does: confirm every + frontend and tool is stopped and run the command again. Do not raise the + timeout blindly. + +5. Start the new-key frontends. The cluster fingerprint is unknown until the + first of them configures: **the first `configure` after the reset pins the + fingerprint, whatever keys it brings.** An old-key frontend or one-shot + tool started by mistake pins the old fingerprint again; that is safe (the + new-key frontends refuse with `cluster configuration fingerprint mismatch`) + and the command can be run again. Once a new-key frontend has pinned, + old-key starts are refused with the same message, and `signing-transition` + in the old environment refuses because the pinned fingerprint is no longer + its own. `fatal-state clear` again works with the new environment. + +6. Do not run `backfill-ctv` or `import-audits` with the new key against + audits signed before the rotation: every ledger-attestation verifier checks + one configured key, and both commands abort on the first old-key row. + Multi-epoch verification remains deferred; the journal's `previous_policy` + is the anchor for it. The public API and the read tier verify no + signatures and are unaffected. + +**Halted while the fingerprint is unset.** The fatal-state writers (a mature +pool block or a deep CTV fanout disconnected) are not fenced on the +fingerprint, and a partitioned-but-alive old frontend is indistinguishable +from a killed one once its heartbeat is stale. If such a process halts the +cluster between the reset and the first new-key pin, `fatal-state clear` +refuses (the fingerprint is missing) and no frontend can start (halted). The +exposure is the same as the by-hand reset's. Recovery, as the database owner: +restore the old fingerprint from the journal row, taking the row lock first +as every authority write must, +`BEGIN; SELECT singleton FROM qbit_prism_cluster WHERE singleton FOR UPDATE; UPDATE qbit_prism_cluster SET config_fingerprint=(SELECT previous_fingerprint FROM qbit_prism_signing_transitions ORDER BY transition_id DESC LIMIT 1) WHERE singleton; COMMIT`, +make sure the process that wrote the halt is really gone, run +`fatal-state clear` with the **old** key environment, then run +`signing-transition --confirm` again; the journal keeps both rows. + +## Fatal-state recovery + +A disconnected mature pool block or deep confirmed CTV fanout records a shared +fatal state in `qbit_prism_cluster.fatal_error`. The message names the +`block_hash` or `fanout_txid`, says `manual reconciliation required`, and names +`qbit-prism-server fatal-state clear --reason <text>` as the recovery command. Every +ledger write transaction then fails with `cluster halted: ...`, and commands +that open the ledger for writing fail at startup. Only the audited command below +ends the halt; there is no public API route and no force flag. + +Clearing restores authority for new work only. It does not approve or forgive +accounting, and claims that expired during the halt still require fresh claims. + +### Recovery commands ```sh -psql "$PRISM_DATABASE_URL" --single-transaction -v ON_ERROR_STOP=1 \ - -f crates/qbit-prism/sql/001_share_ledger.sql -``` - -Before that wrapper existed, a per-statement autocommit apply could commit -the carry-forward summary triggers before the summary seeding block later in -the file ran. If the apply was interrupted in that gap, or a still-running -writer confirmed or reversed a block in it, the summary received only those -post-trigger deltas: every miner's balance was silently under-reported by -their full pre-upgrade carry, and the seed's emptiness guard locked that -partial state in permanently. The schema's seed guard now compares the -summary against the carry history it summarizes and repairs a partial -summary on the next apply. The writer must still be stopped during the -apply: concurrent mutations block on the apply's table locks and then fire -the freshly (re)created triggers after it commits, which invites long -lock waits and deadlocks even though the apply itself is atomic. Note that -combining `--single-transaction` with the script's own `BEGIN`/`COMMIT` -wrapper makes psql print two harmless warnings ("there is already a -transaction in progress" / "there is no transaction in progress"); the apply -is still exactly one transaction. - -### Audit publication ordering migration - -Existing databases must receive the new `audit_publication_sequence` migration. -It creates and validates a bigint sequence, adds the nullable pool-block column, -deterministically backfills confirmed and inactive rows by `found_at` and -`block_hash`, adds a unique index and state constraint, advances the allocator -beyond every retained ordinal, and replaces confirmation/reactivation functions -so each new durable confirmation receives an ordinal. Exact confirmation replay -preserves its prior ordinal. Historical inactive rows are backfilled so later -reactivation can retain that already-published ordinal without allocating a new -audit publication. - -The migration includes `ALTER TABLE` operations that require PostgreSQL's -`ACCESS EXCLUSIVE` table lock. They wait for existing readers and writers and -can interrupt new reads as well as writes while held. A serialized phase takes -a transaction-scoped advisory lock and a `SHARE ROW EXCLUSIVE` lock on -`qbit_pool_blocks`, and the unique index is built non-concurrently. Treat the -whole migration as read-impacting: stop the old coordinator or use a reviewed -maintenance window, take the normal database backup, and apply the file with -`ON_ERROR_STOP` inside a single transaction (see the apply requirements above), -using the same database role and schema search path as PRISM. For example: +qbit-prism-server migrate +qbit-prism-server fatal-state show +qbit-prism-server fatal-state clear --reason "<nonblank explanation>" [--timeout-seconds <10 to 3600>] +``` + +Apply migration 010 with `migrate` before recovery. It adds the +`qbit_prism_fatal_state_events` audit table, works while the cluster is halted, +and does not register a frontend. Confirm on the writer: + +```sql +SELECT version FROM qbit_prism_schema_migrations WHERE version = 10; +``` + +`fatal-state show` reads PostgreSQL only. It neither starts nor registers an +instance and needs no signing configuration. It prints JSON with `fatal_error`, +`set_at`, `block_hash`, `fanout_txid`, and `halted`, and exits nonzero while +halted and zero otherwise. `set_at` is null for a state recorded before +migration 010, whose set time is unknown. A database read failure is also a +nonzero exit, so keep the JSON with the exit status. + +`fatal-state clear` uses the normal server configuration (database, qbit RPC, +chain/genesis, payout, and signing settings) to verify cluster identity. Under +the settlement, ordering, and instance locks it refuses unless: + +- every stored `qbit_prism_instances` row has `status.state` `stopped` or + `drained`; +- no live legacy writer lease exists; +- the node's chain still contains every mature pool block and every deep + confirmed fanout checkpoint, and still holds the tip the command captured + (below); and +- normal block reconciliation leaves no unresolved disconnection and the + carry-forward integrity report passes. + +Every check is made against the tip the command captures from the node when it +starts. The chain may grow during the run, which takes about a minute at +production size: blocks on top of the captured tip leave every block at or +below its height as it was, so every observation stands, and the command goes +on and records the new tip (#737). It accepts a moved tip only when the node's +new best block leads back, header by header, to the captured tip at the +captured height. When the tip has already moved by the time the block and +fanout checks end, the command first repeats each of their node queries, since +a chain that went over to a fork during them and came back onto a longer chain +would leave answers read on the fork. It refuses, clearing nothing: + +- when the chain reorganized during the run: an answer asked again differs, or + the new best block descends from another block at the captured height, is + not above that height, or is unknown to the node; +- when the tip moved while a pool block row lies above the captured height, + since a new block could be that one; and +- when the tip moved more than 1,000 blocks. + +Each of these refusals ends with `retry fatal-state clear`. + +Once its database connection is open, the recovery, its node calls and the +integrity report included, must reach its commit within `--timeout-seconds` +(120 by default, 10 to 3600). The integrity report runs under a statement timeout of its +own, the time left in that bound less what is kept for the closing tip check and +the commit, 5 s plus one node RPC timeout (`PRISM_RPC_TIMEOUT_SECONDS`, 15 s by +default), whatever `PRISM_DATABASE_STATEMENT_TIMEOUT_MS` gives the other +statements, so that setting needs no override. With less than 6 s plus one RPC +timeout of the bound left when the report would start (21 s by default), the +command refuses with `raise --timeout-seconds`. A report that runs out of time +is cancelled by PostgreSQL before the bound ends, so no statement and no lock +outlives the failed command. The closing tip check must itself end 5 s before +the bound, however many headers a moved tip makes it walk on a slow node; +otherwise the command refuses with nothing committed (`the closing tip check +did not finish 5 s before the fatal-state recovery bound`), so the `UPDATE` and +the event `INSERT` keep their 5 s. The commit itself runs outside the bound: +once it is sent, the command waits for its outcome, which under +`synchronous_commit=remote_apply` includes a standby's replay. + +The clear and its audit `INSERT` commit in one transaction. Failures before +commit roll both back and leave the cluster halted; a lost response during +commit requires checking the durable state before retrying. The event records `fatal_error`, +`fatal_error_set_at`, `reason`, `operator_identity` (PostgreSQL `session_user`), +`database_role` (`current_user`), `cleared_at`, the `instances` snapshot, and +`reconciliation`: `genesis_hash`; `tip_hash` and `tip_height`, the captured +tip; `final_tip_hash`, the node's best block at the check just before the +commit, which descends from the captured tip (its headers lead back to it), and +`tip_extended`, whether that differs from the captured tip; `blocks_checked`, +`deep_fanouts_checked`, and `integrity`. + +The event identifies a database login, not a person. Prefer an individual +PostgreSQL login for `clear`. When a shared login is unavoidable, put the +incident ID, operator identity, and evidence reference in the reason. + +### 1. Preserve evidence first + +Collect evidence before stopping, restarting, or reconfiguring anything, and +store it with the incident record: + +1. The `fatal-state show` JSON and exit status. +2. The disconnected block or fanout record, pool blocks at the affected heights, + and their stored audit bundle digests. +3. The current qbit chain from every node the frontends use: tip hash, height, + chainwork, and the confirmations of the named block or fanout. +4. The complete carry-forward integrity report and any reconciliation output. + +Query the writer endpoint, not a replica, in a read-only session (for example +`PGOPTIONS='-c default_transaction_read_only=on' psql`). For a fanout, use its +parent block's height as `<affected_height>`. + +```sql +SELECT fatal_error, updated_at, payout_revision, best_tip_hash, + best_tip_height, best_chainwork +FROM qbit_prism_cluster WHERE singleton; + +SELECT block_hash, block_height, parent_hash, coinbase_txid, chain_state, + maturity_state, matured_at, inactive_since, disconnected_at +FROM qbit_pool_blocks WHERE block_hash = '<block_hash>'; + +SELECT fanout_txid, block_hash, chunk_index, settlement_status, + confirmed_block_hash, confirmed_block_height, confirmed_depth, updated_at +FROM qbit_ctv_fanout_artifacts WHERE fanout_txid = '<fanout_txid>'; + +SELECT b.block_hash, b.block_height, b.chain_state, b.maturity_state, + a.audit_bundle_sha256 +FROM qbit_pool_blocks b +LEFT JOIN qbit_pool_audit_bundles a USING (block_hash) +WHERE b.block_height >= <affected_height> +ORDER BY b.block_height, b.block_hash +LIMIT 200; + +SELECT fanout_txid, confirmed_block_hash, confirmed_block_height, + confirmed_depth +FROM qbit_ctv_fanout_artifacts +WHERE settlement_status = 'confirmed' AND confirmed_depth >= 1000 +ORDER BY confirmed_block_height DESC, fanout_txid DESC +LIMIT 20; + +SELECT qbit_carry_forward_integrity_report(); +``` ```sh -psql "$PRISM_DATABASE_URL" --single-transaction -v ON_ERROR_STOP=1 \ - -f crates/qbit-prism/sql/001_share_ledger.sql -``` - -With `PRISM_POSTGRES_INIT_SCHEMA=1`, coordinator construction applies the same -script before listeners open. The migration is rerunnable and tested across -fresh, legacy, partial, concurrent, malformed, and bigint-boundary states; it -fails closed instead of accepting a conflicting sequence, column, index, or -constraint definition. - -`qbit_shares_since_template_height(min_template_height)` supports operational -replay and frontend recovery. It returns accepted shares at or above the -template height in ascending `share_seq` order and excludes rejected shares. - -## Payout and Reorg State - -Accepted pool blocks are persisted in `qbit_pool_blocks`, with payout rows in -`qbit_pool_payout_entries`, carried balances in `qbit_payout_carry_forward`, and -audit bundles in `qbit_pool_audit_bundles`. The direct coordinator durably -persists the compact candidate intent before calling `submitblock`, then builds, -verifies, and persists the full audit and payout state after the block becomes -active. A definitive pre-acceptance rejection terminalizes the outbox row and -creates no prepared payout state. - -Immature disconnects are first quarantined as `chain_state='inactive'`, which -removes their carry-forward balances from current owed totals without mutating -historical shares or payout rows. If the block returns to the active chain, the -coordinator reactivates it. The terminal reversal path is the fenced ledger -wrapper `PsqlShareLedger.reverse_immature_block(...)`, backed by -`qbit_reverse_immature_pool_block(...)`; it marks the block, payout entries, and -carried balances as reversed. Mature rows, and rows already height-mature at the -supplied active tip, must not be reversed by that path. qbit coinbase maturity -is 1000 blocks, so operators must not mark pool payouts mature before -`block_height + 1000`. - -## Publication Ordinal Rollback And Schema Revert - -Schema initialization adds a durable publication ordinal to accepted pool -blocks: `qbit_pool_blocks.audit_publication_sequence`, allocated from -`qbit_audit_publication_sequence_seq` at the durable prepared -> confirmed -boundary, unique across the table, and required by a validated CHECK -constraint on every confirmed row. Historical confirmed and inactive rows are -backfilled deterministically in `(found_at, block_hash)` order the first time -the migration runs. - -Rolling back coordinator code does not require reverting this schema. The -`qbit_pool_blocks_assign_publication_ordinal` BEFORE trigger assigns the next -ordinal to any confirming write that omits it, so a pre-ordinal coordinator -that confirms with a plain `chain_state` UPDATE keeps satisfying the -constraint. That trigger is the primary rollback path; prefer it. - -`crates/qbit-prism/sql/001_share_ledger_revert_audit_publication_sequence.sql` -is the second path, for the cases the trigger cannot cover: rolling back more -than one release, or aligning a live ledger with a restored pre-migration -base backup. It returns `qbit_pool_blocks` to its pre-ordinal shape by -dropping, in order, the CHECK constraint, the assignment trigger and its -function, the unique index, the column, and the sequence, then restores the -pre-ordinal `qbit_confirm_pool_block` body. The file is one transaction -serialized behind the same advisory lock as the forward migration: any -failure aborts the whole revert, and re-running it after the failure is -resolved -- or on an already-reverted or never-migrated schema -- is safe. An -object the migration did not create, such as an operator view over the -ordinal column, fails the revert loudly instead of being dropped. - -Before applying the revert: - -1. Stop the coordinator and anything else writing to the ledger. The revert - takes an ACCESS EXCLUSIVE lock on `qbit_pool_blocks`, so it stalls behind - a live writer. Worse, a post-migration coordinator left running (or - restarted afterwards) confirms blocks by assigning the ordinal explicitly: - every such confirmation fails with `column "audit_publication_sequence" - does not exist` the moment the revert commits, and a post-migration - coordinator restarted with schema initialization enabled immediately - re-applies the forward migration, silently undoing the revert. -2. Take a base backup, or verify the continuous WAL archive covers this - point. The revert discards every assigned publication ordinal; nothing can - recover them afterwards except that backup. - -What is permanently lost: the recorded confirmed-publication order. The -ordinal is allocated when a block durably reaches `confirmed` and is reused -across exact replay and inactive -> confirmed reactivation, so it is the only -record of the order in which blocks were actually published. After the -revert, publication currency falls back to what pre-ordinal code used to -select current evidence -- `found_at`/`block_height` read order -- which is -not that durable publication order. Re-applying `001_share_ledger.sql` later -re-adds the column and backfills deterministically by -`(found_at, block_hash)`, but that is a fresh assignment: it does not -reproduce the ordinals observed before the revert, and external consumers -that recorded pre-revert ordinals will see the sequence renumbered. - -Apply the revert with the ledger role whose `search_path` selects the PRISM -schema, inside a single transaction, and stop on the first error (the revert -script wraps itself in one `BEGIN`/`COMMIT` and relies on it, matching the -forward script): +qbit-cli getblockchaininfo +qbit-cli getblockheader <block_hash> +qbit-cli getrawtransaction <fanout_txid> true <confirmed_block_hash> +``` + +A header `confirmations` of -1 means the block is not in that node's active +chain. + +### 2. Stop every frontend + +Stop every `run` frontend gracefully with SIGTERM (bundled stacks: +`docker compose stop --timeout 45 prism-coordinator prism-coordinator-2`). Shutdown closes +admission and drains tasks and sessions for up to 30 seconds. Only then does the +server record `stopped`, and only if no session guard remains. If that marker +fails, the process exits with an error and its row keeps its previous status. +Keep deployment supervisors and automatic restart (Compose restart policies, +systemd units, orchestrators, HA managers) disabled until the final +verification. + +Inspect every instance row: + +```sql +SELECT instance_id, started_at, heartbeat_at, + round(extract(epoch FROM clock_timestamp() - heartbeat_at)::numeric, 1) + AS age_seconds, + status->>'state' AS state, status->>'schema' AS schema, + status->'ready' AS ready +FROM qbit_prism_instances +ORDER BY instance_id +LIMIT 200; +``` + +Every row must show `stopped` or `drained`. A running frontend's row holds its +health payload (`qbit.prism.audit-health.v1`) and no `state`. A `starting` row +never became ready. Only `run` frontends write rows: the one-shot commands +`self-check`, `import-audits`, `backfill-ctv` and `broadcast-ctv` pass the same +startup gates (schema and capabilities, the halt guard, the cluster +fingerprint) without registering, so a normal or failed exit leaves nothing +behind, and a run under a live frontend's `PRISM_INSTANCE_ID` leaves that row, +including its session-owner token, untouched. A `starting` row under a +generated UUID that an earlier 3.x.x build's one-shot command left behind is +still refused and still needs the investigation below; nothing removes it +automatically. Stale does not mean stopped: a heartbeat older than the +`self-check` window (`max(3 * PRISM_HEALTH_REFRESH_SECONDS, 15)` seconds) shows +only that reporting stopped. The process may be hung, paused, cut off from +PostgreSQL, or on an unreachable host, and +may resume. `clear` therefore rejects missing, `starting`, unready, unknown, +and old live states, however old the heartbeat. + +Resolve each blocker by finding the process for that `instance_id` and stopping +it gracefully so it records its own marker. Do not insert, update, or delete +`qbit_prism_instances` rows, move `heartbeat_at`, or set or clear `fatal_error` +with SQL. The crashed-owner step in +[retained reservations](prism-session-sequence.md#retained-reservations-and-recovery) +is a different procedure and does not authorize a marker for this recovery. If +an instance cannot record `stopped` itself, stop and escalate for a separately +reviewed decision. + +### 3. Reconcile before clearing + +Establish whether the halt reflects the canonical chain. Compare the evidence +from independent nodes with `best_tip_hash` and `best_chainwork`. A node on a +lower-work or minority branch is repaired at the node, never by clearing. + +If the block or fanout really is disconnected, resolve the chain and accounting +discrepancy (affected payouts, carry-forward balances, and fanout settlement) +through a separately reviewed reconciliation before running `clear`. This +runbook provides no SQL for that change. `clear` is not approval to forgive a +still-disconnected mature payout: it refuses while any mature pool block or deep +checkpoint is missing from the chain, and success means only that its checks +passed. Record the review reference, then repeat the evidence queries. + +### 4. Clear the fatal state + +Run `clear` from a host with the frontends' normal configuration, using the +operator's own database login: ```sh -psql "$PRISM_DATABASE_URL" \ - --single-transaction \ - --set ON_ERROR_STOP=1 \ - -f crates/qbit-prism/sql/001_share_ledger_revert_audit_publication_sequence.sql -``` - -The full round trip -- migrate, revert, confirm pre-ordinal-style, re-apply, -re-backfill without double assignment -- is exercised by -`tests/prism_postgres_a1_revert_gate.py` inside -`test/test-prism-postgres-ledger.sh`. - -## CTV Fanout Artifact Repair - -Schema initialization is also the idempotent repair path for deployed PRISM -databases. In particular, it drops the old `NOT NULL` constraint from -`qbit_ctv_fanout_artifacts.anchor_vout` so fee-bearing, anchorless CTV fanouts -can persist with `anchor_vout = NULL`. - -The same schema init path also adds bounded broadcast-attempt summaries to -`qbit_ctv_fanout_artifacts`. Operators can tune retained detail rows with -`PRISM_CTV_BROADCAST_ATTEMPT_DETAIL_LIMIT`; the summary columns retain total -attempt count, latest package/result/error context, and per-status counts after -old detail rows are no longer retained. - -If a block was mined while the old constraint was still present, backfill the -missing fanout artifact rows from the persisted audit bundle, a local -`prism-live-audit-bundle-*.json` envelope, or a local audit body file: - -```bash -PRISM_DATABASE_URL='postgres://...' \ -python3 -m lab.prism.backfill_ctv_fanouts --db-block-height 21883 -``` - -The repair tool also accepts `--db-block-hash <hash>` or local JSON paths. Local -paths may be full v1 bundles, live envelopes, legacy compact audit body refs, -or `qbit.prism.audit-bundle.v2` proof bodies; the tool follows envelope -`body_uri` pointers and reads `bundle_without_shares` from compact bodies -because CTV backfill does not need share rows. For a local candidate/final -audit bundle whose filename does not include the block hash, pass -`--path-block-hash <hash>`. The tool runs schema init by default before -backfilling and then calls the same fenced -`persist_ctv_fanout_manifest_set` path as the coordinator. Stop the active -coordinator or otherwise ensure the repair process can acquire the ledger -writer lease before running a backfill. - -## Compaction and Archive Contract - -The ledger is append-only for reward correctness. Compaction is allowed only as -an archive-first operation, and only after proving it cannot change any future -window, audit, maturity, or reorg answer. - -Before deleting any hot rows, an operator must: - -1. Export the candidate `share_seq` prefix to durable archive storage. -2. Record row count, first and last sequence, and a cryptographic hash of the - exported rows in canonical order. -3. Prove no unresolved pool block, audit bundle, immature payout, or future 8x - PRISM reward window can reference the candidate rows. -4. Re-run representative `qbit_prism_window`, - `qbit_audit_share_window`, and `qbit_shares_since_template_height` queries - before and after the dry run and verify identical results. - -No public harness currently deletes ledger rows. Until an archive proof exists, -production deployments should retain the full canonical share log. - -For operator disk planning and permanent-vs-ephemeral data categories, see -[`docs/prism-storage-sizing.md`](prism-storage-sizing.md). - -## Throughput Evidence - -`make test-prism-postgres-throughput` is an opt-in capacity harness. It creates -a temporary Postgres container, bulk-inserts synthetic accepted shares through -the canonical schema, records observed shares/sec, and stores -`EXPLAIN (ANALYZE, BUFFERS, FORMAT JSON)` output for the audit-window query. - -Useful environment variables: - -- `QBIT_PRISM_THROUGHPUT_SHARES`: number of synthetic rows to insert. -- `QBIT_PRISM_MIN_SHARES_PER_SEC`: optional failing threshold. -- `QBIT_PRISM_THROUGHPUT_REPORT`: JSON report path. - -The throughput harness measures schema/query capacity. It is separate from the -lab ledger adapter's execution backends described below. - -## Hot-Path Execution Backends - -The lab ledger adapter (`PsqlShareLedger`) supports two interchangeable -execution backends over the same SQL and the same durability contract: - -- **`psycopg-pool` (production default):** a persistent pooled psycopg client. - The coordinator's share-writer group commits (`PRISM_SHARE_COMMIT_BATCH_SIZE` - / `PRISM_SHARE_COMMIT_LINGER_MILLISECONDS`) execute as one round trip on a - long-lived connection instead of one `psql` fork+connect per statement; the - batch statement's own `set_config('synchronous_commit', 'on', true)` keeps - the Stratum ACK boundary at the database commit. Reads share a pool bounded - by `PRISM_POSTGRES_READ_CONCURRENCY` plus one writer slot. Read-only - statements retry once after a lost connection; mutations do not re-execute - automatically because a lost response cannot prove that PostgreSQL did not - commit the first execution. -- **`psql-subprocess` (fallback):** the legacy zero-Python-dependency backend - that shells out one `psql` per statement. It remains fully supported for - regtest portability and as the operational escape hatch. - -`PRISM_POSTGRES_NATIVE_CLIENT` selects the backend: `auto` (default) uses the -pooled client when psycopg is importable and a `postgres://` DSN is available -(from `PRISM_DATABASE_URL` or inside `PRISM_POSTGRES_PSQL_COMMAND`), `1` -requires it, and `0` forces the subprocess fallback. The coordinator startup -line reports the active backend as `ledger_execution=`. - -Readiness and health counters (`accepted_share_stats`) are maintained -incrementally by the single lease-holding writer and reconciled against the -database once per `PRISM_ACCEPTED_STATS_CACHE_SECONDS` (default 60) instead of -running `count(*) / count(DISTINCT miner_id)` aggregates every few seconds. - -`make test-prism-postgres-native-ledger` is the opt-in end-to-end validation -for the pooled backend: it provisions a temporary Postgres container and -exercises schema init, lease guards, concurrent batched appends, duplicate -replay, cached stats, and cross-backend read consistency with the `psql` -fallback. Run it on any docker host with `psycopg` installed before rolling -the pooled backend into an environment. - -It does not simulate live Stratum miner swarms, reconnect storms, malformed -client messages, or stale-share bursts across job changes. Those remain separate -operator-load concerns if production needs coverage beyond ledger/query -capacity. - -## Operator Readiness - -Optional capacity qualification and its versioned evidence contract are -documented in [PRISM capacity qualification](prism-capacity-readiness.md). - -`make prism-self-check` is the PRISM operator readiness probe. It resolves the -same Compose environment as `make up-prism-pool`, then emits PASS/WARN/FAIL -rows for: - -- qbit RPC reachability, chain identity, IBD state, and peer count. -- the configured genesis hash against `getblockhash 0` when - `QBIT_EXPECTED_GENESIS_HASH` is set. -- PRISM coordinator `/healthz` from inside the coordinator container. -- miner-facing Stratum TCP reachability. -- Postgres readiness for the canonical ledger. -- PRISM signing/key environment and forbidden production bypass flags. -- audit/archive path writability. -- basic mining configuration such as share difficulty, vardiff bounds, pool - fee configuration, CTV fanout fee sourcing, and minimum ready miners. - -The command is safe to run repeatedly. It exits non-zero when any FAIL row is -present. Use `python3 scripts/prism-self-check.py --skip-live` to validate only -static configuration before the profile is running. - -On mainnet, an explicit -`QBIT_MAINNET_LAUNCH_READINESS_CHECKS_ENABLED=0` keeps the probe useful before -launch: `qbit.ibd`, a missing initial `stratum.highdiff_floor` difficulty -notification, and a below-threshold `coordinator.ready_miners` result are WARN -rows, while all other checks remain active and fatal. Set the flag to `1` for -launch; an omitted flag is also strict, and a malformed value fails closed. - -For operator runs, `make up-prism-pool` requires Postgres plus -`PRISM_MANIFEST_SIGNING_SEED_HEX`, -`PRISM_LEDGER_ATTESTATION_SIGNING_SEED_HEX`, and the trusted -`PRISM_LEDGER_WRITER_PUBLIC_KEY_HEX`. The two seed values are 32-byte hex -Ed25519 signing seeds. The ledger public key is the verifying key derived from -the ledger attestation seed and must be distributed through an operator-trusted -channel, not copied from the bundle being verified. Keep -`PRISM_ALLOW_MEMORY_LEDGER`, `PRISM_ALLOW_TEST_SIGNING_SEEDS`, -`PRISM_ALLOW_BUNDLE_EMBEDDED_LEDGER_KEY`, and -`PRISM_ALLOW_FIXED_LEDGER_SESSION_TOKEN` disabled outside local tests. - -Mainnet configuration is always treated as production configuration, even when -the separate production toggle is omitted. It must select the chain explicitly -with `QBIT_CHAIN_FLAG=-chain=main` and pin the final release genesis hash in -`QBIT_EXPECTED_GENESIS_HASH`. The live readiness probe normalizes the configured -`mainnet` name to the `main` name returned by qbit RPC, then verifies height zero -against the pin. - -Production builds using the git source provider must set `QBIT_GIT_COMMIT` to a -full 40-character object ID. The environment doctor verifies that the resolved -checkout is at that exact commit instead of trusting a mutable branch or tag. -Stale-credit grace (`PRISM_STRATUM_STALE_GRACE_SECONDS`, default 3) stays -enabled in production, mainnet included; reward windows that contain a credited -prior-tip share publish the logical `qbit.prism.audit-bundle.v1.1` schema, so -keep the deployed verifier and accounting release on a version that accepts it. - -The parent-chain selector is checked independently: `BITCOIN_CHAIN` and -`BITCOIN_CHAIN_FLAG` must be an exact pair, including `mainnet` with -`-chain=main`. When a production configuration selects a non-regtest parent for -AuxPoW, both `QBIT_MINER_ADDRESS` and `BITCOIN_MINER_ADDRESS` must be explicit; -automatic wallet-derived payout addresses are rejected. - -Mainnet CTV settlement requires an operator-reviewed positive -`PRISM_CTV_FANOUT_FEE_MARKET_RATE_BITS_PER_1000_WEIGHT`. On non-mainnet networks, -if it is omitted, the live readiness probe requires `estimatesmartfee` to return -a positive rate before the pool is considered ready. A new chain, or a chain -producing only empty blocks, does not have the confirmed transaction history -needed for empirical fee estimation. A wallet fallback fee does not populate -that history and is not a substitute for the explicit CTV fanout rate. +set -o pipefail +qbit-prism-server fatal-state clear \ + --reason "<incident>: <operator>; <block or fanout> reconciled per <review>; evidence <ref>" \ + | tee fatal-state-clear.json +``` + +Save the success JSON with the incident record, along with the audit event: + +```sql +SELECT cleared_at, operator_identity, database_role, reason, fatal_error, + fatal_error_set_at, instances, reconciliation +FROM qbit_prism_fatal_state_events +ORDER BY cleared_at DESC +LIMIT 5; +``` + +A validation or reconciliation failure clears nothing and writes no event. Correct the reported blocker +and rerun deliberately; do not loop. Three refusals report a chain that moved +during the run rather than a blocker, and only ask for a rerun: `the chain +reorganized during fatal-state recovery`, `the tip moved and a pool block lies +above the captured height`, and `the node's tip moved <n> blocks during +fatal-state recovery`. Rerun once the node's tip has settled; if a +reorganization recurs, investigate the chain first (step 3). A tip that only grew +during the run does not refuse. When the integrity report did not finish in the +time the bound left it, the command says to raise `--timeout-seconds`: rerun +with a larger bound. When the command exceeded its bound (`fatal-state recovery +exceeded <n> seconds before its commit`), nothing was committed and the halt +stays: rerun, with a larger `--timeout-seconds` if it ran out of time. The commit +runs outside the bound, so only a connection that drops during the commit leaves +the outcome unknown: then check `fatal-state show` and the event table before +rerunning. + +### 5. Restart and verify + +1. Confirm `qbit-prism-server fatal-state show` exits zero, and save its JSON. +2. Record the ledger head before admitting traffic: + + ```sql + SELECT share_seq, accepted_at FROM qbit_share_ledger + WHERE accepted ORDER BY share_seq DESC LIMIT 1; + ``` + +3. Re-enable supervisors and start the frontends. Each `/healthz` must return + 200 (see [health checks](#health-diagnostics-and-validation)), and each + instance row must carry a fresh, ready health payload. +4. Confirm new accepted shares. The step 2 query must return a higher + `share_seq` with a later `accepted_at`, and each frontend's + `qbit_prism_accepted_shares_total` and + `qbit_prism_share_ack_seconds_count{result="accepted"}` must increase. +5. Keep the evidence, reconciliation reference, clear JSON, audit event, and + these checks together. + +If a frontend reports `cluster halted` again, a new fatal state was recorded. +Start again from evidence collection. + +## Health, diagnostics, and validation + +`/healthz` returns 200 only when the process has fresh work for its observed tip +and current payout revision and job delivery can progress; otherwise it returns +503. The HTTP handler reads a published snapshot and fails closed when that +snapshot becomes stale. Database or node outages therefore cannot keep an old +green response indefinitely. A tracked task polling beyond two seconds, or an +health/metrics publication exceeding the existing freshness budget, also +returns 503 with +`ok=false`, `ready=false`, and `status="runtime-stalled"`; the corresponding +`qbit_prism_runtime_task_stalled{task="..."}` is 1. This live check requires a +surviving runtime worker to serve the probe; a completed long poll retains +metric evidence without keeping readiness failed. The publication progress +guard ends before heartbeat/prune maintenance; their asynchronous waits do not +mark a just-published snapshot runtime-stalled. + +The coordinator health payload also supplies three known 2.x compatibility +aliases: `ledger_backend` is `postgres-native` for the PostgreSQL backend, +`accepted_block` is whether `found_block_count` is positive, and +`accepted_block_count` copies that count. `ready_miner_count` and `max_blocks` +remain unmapped because their native sources have not been agreed. + +`/metrics` exports native process health, accepted/rejected share and block +counters, runtime workers, connections, pending builds, current-work delivery +coverage, and delivery outcomes. It also includes share ACK latency and reject +reasons, initial-work waits, candidate backlog, collector pool waits and status, +runtime lag, and RSS; the full [native family inventory](prism-native-metrics.md) +identifies the timing families declared without samples. Scrape every instance +with its own label; process counters reset after restart. Dashboard accounting +is read from PostgreSQL across instances. Detailed Python queue, writer lease, watchdog, +and incremental-refresh metrics no longer describe this runtime. + +The coordinator (`run`) serves the last complete metrics publication and adds +three gauges at scrape time: + +| Gauge | Meaning | +| --- | --- | +| `qbit_prism_metrics_snapshot_available` | `0` before the first publication; `1` afterward, including when stale | +| `qbit_prism_metrics_snapshot_stale` | `1` when missing or older than the freshness budget; otherwise `0` | +| `qbit_prism_metrics_snapshot_age_seconds` | Monotonic age in seconds; `-1` before the first publication | + +The coordinator uses the same budget as `/healthz`: +`max(3 * PRISM_HEALTH_REFRESH_SECONDS, 15)` seconds. The setting is read as an +unsigned whole number of seconds from 1 through 86400, defaulting to 2 when +absent; invalid values fail startup. Compose supplies 5 by default. Both values +give a 15-second freshness budget. The publisher ticks at this configured +interval, so publication cadence and the staleness budget stay aligned. +Once the age exceeds that budget, a scrape sets `qbit_prism_health_state` to `0` +while retaining the other cached samples. Collector age/availability and runtime +state are overlaid from memory at scrape time; this does not refresh the cached +body timestamp or a collector's last-success timestamp. +Scraping neither renews the publication age nor queries the database. + +Both `run` and `public-api` return HTTP 200 for `/metrics`, including missing +and stale observations. Inspect the freshness signals and `/healthz` rather +than treating a successful scrape as readiness. GET and HEAD responses carry: + +- `Cache-Control: no-store`. +- `X-Prism-Metrics-State: fresh`, `stale`, or `unavailable`. +- `Age`: elapsed whole seconds, rounded down, including `0`; omitted when unknown. +- `Warning: 110 qbit-prism "metrics snapshot is stale; serving last complete payload"` + only when the state is `stale`. + +The public role derives these headers from its existing readiness probe age, +also exported as `qbit_prism_public_ledger_probe_age_seconds`. Its budget is +`max(3 * PRISM_PUBLIC_READINESS_PROBE_INTERVAL_SECONDS, 15)` seconds; the probe +interval defaults to 5 seconds. Before the first completed probe the state is +`unavailable`. A recent failed probe is still `fresh`, with +`qbit_prism_public_ledger_ready 0`; freshness does not mean the database is ready. +The public role keeps its existing metrics body, without the coordinator's +three snapshot gauges. + +Useful checks: + +```sh +qbit-prism-server check-config +qbit-prism-server healthcheck --url http://127.0.0.1:3341/healthz +curl --silent --show-error --include --max-time 5 http://127.0.0.1:3341/metrics +qbit-prism-server self-check +bash test/prism-native-tests.sh +QBITD_BIN=/path/to/qbitd bash test/prism-native-tests.sh live +``` + +The database test wrapper starts a private local cluster unless +`PRISM_TEST_DATABASE_URL` is supplied. Its default mode runs the whole workspace +and the three explicit `--ignored` database targets, as CI does, and checks the +integration gate's manifest so no gated test passes without running (see +[the integration test gate](prism-integration-test-gate.md)). Live tests add +actual qbitd regtest and bounded CPU mining. Use an isolated database for tests. The native builder +benchmark measures CPU build/verify work, not end-to-end accepted-share capacity; +see [measurement](prism-payout-artifact-measurement.md) and +[optional qualification](prism-capacity-readiness.md). + +The physical failover test uses disposable PostgreSQL primary/synchronous +standby processes and two ledger clients through a stable TCP endpoint. It +checks survival of acknowledged IDs, deduplication, and resumed writes after +immediate primary loss and promotion: + +```sh +PRISM_TEST_PG_BIN_DIR=/usr/lib/postgresql/16/bin \ + cargo test --locked -p qbit-prism-server --test postgres_failover -- --nocapture +``` + +The test skips unless that server-tool directory is provided. Its test proxy +and promotion sequence do not replace validation of a production HA manager. + +The live regtest suite (#521) runs the documented promotion procedure against a +disposable primary with one **asynchronous** standby while a found block is +mid-landing (`live_regtest` tests `pg_failover_tests::*`, which need +`QBITD_BIN`, `PRISM_TEST_DATABASE_URL` and `PRISM_TEST_PG_BIN_DIR`). With replication cut before the +failover, the shares committed in the gap are lost and counted, and every +earlier share survives; after a fenced switch that replays through the old +primary's flush LSN, none is lost. Either way the block is offered once and +lands on the promoted primary. When the offering frontend loses the node's +answer with the old primary, the reservation is recovered without another +`submitblock`, usually once its 120-second candidate lease expires, and the +submitted row keeps no offer outcome. + +The collector failure/recovery test requires a disposable PostgreSQL database +and is ignored by ordinary `--all-targets` runs. Invoke it explicitly when +running without the wrapper: + +```sh +PRISM_TEST_DATABASE_URL=postgresql://test_user@127.0.0.1:5432/test_db \ + cargo test --locked -p qbit-prism-server --test observability_database -- --ignored +``` + +It verifies valid zero values, a blocked query, pool exhaustion, and recovery; +never point test fixtures at a production database. diff --git a/docs/prism-metric-metadata.json b/docs/prism-metric-metadata.json new file mode 100644 index 000000000..a3bc55821 --- /dev/null +++ b/docs/prism-metric-metadata.json @@ -0,0 +1,432 @@ +{ + "native": { + "refresh_window_acquisitions_total": { + "labels": { + "outcome": "WindowAcquisition" + }, + "status": "Counted once per refresh snapshot by the coordinator's ledger, at the decision between the delta path and the full newest-first scan. `advanced` is the delta path: the retired window's rows were reused for the fresh anchor, cutoff and network difficulty, including a retarget in either direction and a multi-page delta. Every other outcome names why the full scan ran: `no_prior` (no retired window was exclusively owned, as at the first refresh or after a cancelled build kept a reference), `empty_prior` (the retired window had no rows), `tail_mismatch` (the retired window's last row was not its accepted cutoff, which only legacy or raw rows with a future timestamp at the top of the ledger can cause), `anchor_regressed`, `cutoff_regressed`, `delta_too_large` (more than 262,144 new sequence slots), `no_evidence` (the retired window had no single-leaf witness, as across a partition boundary), `leaf_changed` (the retained range's partition was detached, reattached or restored, or the writer timeline changed, before the delta was read), `margin_too_large` (a heavier target needed more than 16 older pages), `partial` (history ran out before the target, so the full scan decides), `witness_changed` (the merged range's leaf, incarnation or timeline differed when re-read after the pages, including a margin that crossed into another leaf), `count_mismatch` (the anchored row count of the merged range differed, as after a retroactive insert or newly eligible timestamp) and `invariant` (the merged window failed its own ordering or crossing check, logged at error). Every outcome is initialized to zero. A high full-scan share is a cost signal, never an error; `invariant` is a defect signal." + }, + "refresh_seconds": { + "labels": { + "trigger": "RefreshTrigger", + "acquisition": "RefreshAcquisition" + }, + "status": "Observed once per refresh that published rebuilt work, from the refresh's entry (after the poll or block-wait wake) to its publication, on the coordinator's monotonic clock. A refresh that reused the published work unchanged, or that failed, records nothing. `trigger` names what invalidated the published work, by the label's own precedence when several inputs changed on one poll: `initial` (nothing published yet), then `writer_timeline` (the database writer's timeline moved, a promotion: #619), `tip` (the template parent changed), `revision` (the payout revision changed), `balances` (the carry-forward balances changed), `reanchor` (the cached window aged past `PRISM_PAYOUT_ARTIFACT_REANCHOR_SECONDS`, or no cached window was held while work was published), `shares` (new shares with no cached bundle), `fee` (the relay fee floor changed) and `template` (any other template or publication change, including a cached window that no longer matches the published reference). The reuse test itself checks these inputs in a different order; the label is a classification of the same inputs, never a decision. On a same-template poll the revision, balances and share inputs are the ledger probe's; on a new template the ledger is not probed at entry, and the label compares the published work with the window the rebuild used, so a revision or balance change that had landed by the rebuild's admission is named ahead of `template`. `acquisition` says where the window came from: `delta` or `full` for a fresh ledger snapshot by that path, `cached` when the previous refresh's window was reused and only the body was rebuilt. Series appear on their first observation. Default buckets." + }, + "capture_offer_decisions_total": { + "labels": { + "decision": "CaptureDecision" + }, + "status": "#478 block capture. `offered`: the block's positive as-issued float (the most debt its confirmation can create, in any order) was within `PRISM_CAPTURE_OVERPAY_CEILING_BPS` of the coinbase value, and the block was reserved for its one offer, decided in the reservation's transaction. `abandoned_ceiling`: the float exceeded the ceiling and the pending block was abandoned; the WARN log names the bound and the ceiling, and `qbit_prism_payout_divergences` keeps both. `abandoned_disabled`: the ceiling is 0, so the block was abandoned as before #478 (recorded as decision `disabled`). The miner of an abandoned capture is answered `stale-job`, never `ledger-confirmation-failed`. A bound that cannot be computed abandons nothing: the offer fails and is retried. Process-local; every series starts at zero. No firing rule." + }, + "block_proof_ack_capped_total": { + "labels": { + "path": "BlockAckPath" + }, + "status": "Counted once per block-bearing submission answered before its outcome was known (#574). `block_only`: a captured (#478) or below-target block proof whose deferred share waits for its own frontend's landing, or for its duplicate probe and candidate enqueue, past `share_commit_timeout` itself (`PRISM_SHARE_COMMIT_TIMEOUT_SECONDS`, 15 s by default; it was at least 60 s before #574). `share`: a share-pass append carrying a found block that had not committed by the share deadline, or by its grace when COMMIT was already in flight, the same bounds as a plain share; it is never refused, so it runs on; and a share-pass append that captured its block because the payout revision moved before its commit (#657), whose deferred share then waits for its block's landing as a `block_only` proof's does, up to the same share deadline. Either is answered `ledger-outcome-unknown` (Stratum error 20, \"share outcome is not yet known\"), the same reply as any share still pending at its deadline, so it also counts in `qbit_prism_rejections_total{reason_id=\"ledger-outcome-unknown\"}` and feeds PrismShareAppendFailures; the WARN log names the block hash, share id, phase and elapsed_ms. Only the wait ends at the bound: the probe, enqueue, append and landing run on, so the block is still offered and landed and its share is credited once when that commits. Process-local; both series start at zero. No firing rule of its own." + }, + "block_offer_standby_wait_total": { + "labels": { + "outcome": "StandbyWaitOutcome" + }, + "status": "Counted once per found-block offer when PRISM_OFFER_STANDBY_APPLICATION_NAME is set and PRISM_OFFER_STANDBY_FLUSH_WAIT_MS is positive, after the offer reservation commits and immediately before submitblock (#529). `confirmed`: the flush_lsn in pg_stat_replication of every standby connected under that name reached the primary's WAL flush position read after the reservation, so the candidate row and reservation survive the standby's promotion. `absent`: no standby connected under that name (streaming or catching up), so the offer did not wait. `lagging`: the standby was still behind at the bound; the WARN log carries lag_bytes. `failed`: the positions could not be read (a database error, no answer before the bound, or a writer role without pg_read_all_stats, which pg_monitor includes). Every outcome is offered; every outcome but `confirmed` logs the block hash at WARN for post-failover reconciliation. The wait itself is logged per block as waited_us. Process-local; every series starts at zero. Warns through PrismBlockOfferStandbyUnconfirmed." + }, + "divergent_landings_total": { + "status": "Counted at the confirmation, when a landed block's rows start to count: it is divergent when those rows meet canonical balances other than the as-issued ones its coinbase commits to (#478 captures, and the pre-existing case of a block whose work predates an earlier confirmation). Each is recorded in `qbit_prism_payout_divergences`, in the confirming transaction, with its per-account overpay; a frontend without #478 records and counts nothing. Process-local; starts at zero. No firing rule." + }, + "divergent_landing_overpay_sats_total": { + "status": "The sum over committed divergent confirmations of the carry-forward debt each created: per account, the increase in max(0, -balance) when the block's rows start to count. Unrecovered while the indebted miner earns no further gross; see the ledger operations guide for the accepted-cost bound. Process-local; starts at zero. No firing rule." + }, + "carry_forward_debt_sats": { + "status": "Read from the canonical balances after every confirmation or deactivation this frontend commits and at every full refresh snapshot; zero when no account is negative. Process-local. No firing rule." + }, + "ctv_fanout_broadcaster_tip_refresh_yields_total": { + "replaces": ["qbit_prism_ctv_fanout_broadcaster_tip_refresh_yields_total"] + }, + "ctv_fanout_broadcaster_chunk_rows": { + "replaces": ["qbit_prism_ctv_fanout_broadcaster_chunk_rows"], + "status": "Values are rows, not seconds: instead of the default time ladder this histogram exports the single finite bound `le=\"1\"` plus `+Inf`, so `_count` is the number of claimed fanout attempts and `_sum` equals `_count`. The 2.x.x ladder (1, 2, 5, 10, 25, 50, 100) is not exported; a query for a higher `le` matches no series." + }, + "ctv_fanout_broadcaster_chunk_seconds": { + "replaces": ["qbit_prism_ctv_fanout_broadcaster_chunk_seconds"], + "status": "Default histogram ladder in seconds; one observation per claimed attempt, recorded even when persisting the attempt's completion fails." + }, + "accepted_shares_total": { + "replaces": [ + "qbit_prism_accepted_shares_total" + ], + "status": "Process-local counter; legacy canonical ledger count was persistent. Read from the Stratum listeners at scrape time and overlaid on cached metric bodies (#581), so the share-refusal rules see acceptances stop while the health publisher, and with it the cached body, waits on an unavailable database." + }, + "share_ack_seconds": { + "replaces": [ + "qbit_prism_share_ack_seconds" + ], + "labels": { + "result": "AckResult" + }, + "status": "Uses the default histogram ladder plus 15 and 20 seconds; these are elapsed ACK bounds, not measured ledger deadlines." + }, + "share_ack_landing_window_seconds": { + "labels": { + "result": "AckResult" + }, + "status": "Each observation is also in `share_ack_seconds`, on the same ladder: steady state is `share_ack_seconds` minus this family, bucket by bucket. A landing window opens when this frontend first observes a pool block's acceptance, its own `submitblock` acceptance, or a peer frontend's block on the active chain, first seen either before it is confirmed or already confirmed at the tip (a buried block first seen confirmed, as after a restart, opens none; a restart whose tip is a pool block opens one), and admits every submission that arrives within 30 seconds of it; an acceptance while a window admits extends it. A submission in flight at the acceptance is in the window. Both series start at zero. Fed by PrismLandingAckP99Slow and PrismLandingAckP99High through `share_ack_slow_landing_windows`." + }, + "share_ack_slow_landing_windows": { + "labels": { + "p99_above_seconds": "LandingAckBound" + }, + "status": "A landing window (see `share_ack_landing_window_seconds`) is judged 30 seconds after it stops admitting submissions, so late acknowledgements can complete, or at the next acceptance if that comes first. Its nearest-rank p99 is over a bound exactly when more than one percent of its acknowledgements took longer than the bound, which the process counts exactly rather than reading from buckets. A window with no acknowledgement has no verdict and leaves the count unchanged; a window at or under the bound resets it to zero. -1 until the first verdict since process start. Per frontend: each frontend judges the landings it observed. PrismLandingAckP99Slow tickets at three for the 2-second bound, PrismLandingAckP99High warns at one for the 10-second bound." + }, + "rejections_total": { + "replaces": [ + "qbit_prism_rejections_total" + ], + "labels": { + "reason_id": "RejectReason" + }, + "status": "Present unknown or empty IDs map to unrecognised; missing IDs and explicit internal-error retain internal-error. Normalization does not change the protocol response. backend-rpc-unavailable names the node, backend-database-unavailable the ledger database (#581): a refusal on a path that reads both names whichever failed, and a timeout or aged-out readiness names the ledger database while the work it waited for was still waiting on it (#655). A published-work lease whose commit gate cannot read or prove its chain authority before COMMIT is answered the same way as readiness it cannot prove (#716), so contention with no failed dependency counts as backend-rpc-unavailable. Incremented per refusal and overlaid on cached metric bodies at scrape time (#581), so refusals during a database outage are counted while the cached body is stale." + }, + "stale_shares_total": { + "replaces": [ + "qbit_prism_stale_shares_total" + ] + }, + "duplicate_shares_total": { + "replaces": [ + "qbit_prism_duplicate_shares_total" + ] + }, + "low_difficulty_shares_total": { + "replaces": [ + "qbit_prism_low_difficulty_shares_total" + ] + }, + "grace_credited_shares_total": { + "replaces": [ + "qbit_prism_grace_credited_shares_total" + ] + }, + "stratum_pending_initial_jobs": { + "replaces": [ + "qbit_prism_stratum_pending_initial_jobs" + ] + }, + "stratum_oldest_pending_initial_job_seconds": { + "replaces": [ + "qbit_prism_stratum_oldest_pending_initial_job_seconds" + ] + }, + "stratum_request_delivery_wait_seconds": { + "status": "Default histogram ladder. One observation per waiting request, when its session handles it, whether the delivery it waited for was announced or failed; none when the session ends first. A delivery is the admission, build and persistence of the session's next job and its announcement. A frame that is not a JSON object is answered at once and never waits. A submit queued behind a waiting request is read only after it, and `share_ack_seconds` starts at that read, so this family, not the ACK histogram, carries that wait. Registered at startup: every bucket, `_sum` and `_count` reads 0 until the first wait. No rule of its own." + }, + "job_delivery_cancellations_total": { + "status": "Process-local counter, 0 at startup. A delivery is counted once, at its end, in exactly one of `job_delivery_successes_total`, `job_delivery_failures_total` and this family (#621). Counted here: a delivery still being admitted, built or persisted, or built but not yet announced, when its session ends, which since #621 a session notices while a delivery is in flight. A delivery whose announcement write fails still counts as a failure. No rule of its own." + }, + "job_delivery_failures_total": { + "status": "A build, admission or persistence error or deadline, or a failed announcement write. A delivery its session abandons before the announcement counts in `job_delivery_cancellations_total` instead (#621)." + }, + "stratum_current_tip_coverage_gap_seconds": { + "replaces": [ + "qbit_prism_stratum_current_tip_coverage_gap_seconds" + ] + }, + "stratum_semantic_current_work_ratio": { + "replaces": [ + "qbit_prism_stratum_semantic_current_work_ratio" + ], + "status": "-1 before observation; one when no clients are authorized." + }, + "block_submit_seconds": { + "replaces": [ + "qbit_prism_block_submit_seconds" + ], + "status": "Observed once per block by the offering frontend after its one submitblock call returned, from the enqueuing frontend's proof-observation wall clock to the offering frontend's wall clock immediately before the send (A/#266). No sample for a row without a proof time, for a negative interval (host clock skew) or on a recovery; a crash between the call and its outcome commit may lose the sample. No firing rule." + }, + "block_candidates_pending": { + "replaces": [ + "qbit_prism_block_candidates_pending" + ], + "status": "Counts every unfinished outbox state: pending, offer_reserved, offered and reconciliation (A/#266). Excludes rows settled as proven orphans (#415, migration 015): those are terminal, so a lost tip race leaves these gauges once a different block has PRISM_CANDIDATE_ORPHAN_CONFIRMATIONS confirmations at its height. Since #493 the paging rule reads block_candidate_oldest_unacknowledged_seconds instead, so an acknowledged lost race in reconciliation warns but never pages; this gauge keeps its all-unfinished meaning and feeds the 15-second warning and the 30-minute stuck warning." + }, + "block_candidate_oldest_pending_seconds": { + "replaces": [ + "qbit_prism_block_candidate_oldest_pending_seconds" + ], + "status": "Counts every unfinished outbox state: pending, offer_reserved, offered and reconciliation (A/#266). Excludes rows settled as proven orphans (#415, migration 015): those are terminal, so a lost tip race leaves these gauges once a different block has PRISM_CANDIDATE_ORPHAN_CONFIRMATIONS confirmations at its height. Since #493 the paging rule reads block_candidate_oldest_unacknowledged_seconds instead, so an acknowledged lost race in reconciliation warns but never pages; this gauge keeps its all-unfinished meaning and feeds the 15-second warning and the 30-minute stuck warning." + }, + "block_candidate_oldest_unacknowledged_seconds": { + "status": "Read in the same database snapshot as the two all-unfinished gauges (#493): the unfinished rows the node has not accepted are pending and offer_reserved rows, offered or reconciliation rows whose one submitblock outcome is unknown (a transport failure or timeout, or a reservation whose call was lost with its frontend; a row adopted on the node's own evidence that its block is active carries the adoption reply and is excluded), and rows the node definitively rejected unless the reply names a side-chain block (inconclusive, duplicate, duplicate-inconclusive: a lost tip race). A node-accepted lost race awaiting its orphan proof and a landing that keeps failing after acceptance are unfinished but accepted, so they age only block_candidate_oldest_pending_seconds (and the landing-failed gauge). PrismBlockCandidateOldestPendingCritical pages on this gauge, falling back to the all-unfinished age on a target that does not export it yet; PrismBlockCandidateStuck warns on the all-unfinished age at 30 minutes. -1 with the other candidate gauges when the collector fails." + }, + "block_candidate_oldest_landing_failed_seconds": { + "status": "Read in the same database snapshot as the other candidate gauges (#493 review M1): the oldest offered or reconciliation row whose landing has not committed (the landing transaction is the only writer of the block's qbit_pool_blocks row, so its absence is the durable fact) or whose last_error carries the landing-failed prefix the coordinator and the collector share (a landed row that no longer authenticates): a row the node accepted or rejected whose audit rebuild, verification or ledger landing keeps refusing and is retried at the reconciliation backoff. Since #510 an offered row counts before its first landing outcome too: a won block whose landing hangs in offered (waiting for build capacity, which has no deadline, while its heartbeat keeps the claim) ages here instead of reading zero, and a healthy offered row lands within the 60-second rebuild deadline, well inside the warning dwell. A retry that fails for a transient node or database reason, or a refused operator recovery, overwrites last_error but not the missing pool-block row, so the age never reads zero while nothing has landed. Measured from offer_reserved_at, the start of the post-offer lifecycle, so pre-offer retry backoff never counts toward it. Zero for a lost race and for a rejection once its audit has landed, because the landing runs before the chain verdict and writes the pool-block row. PrismBlockCandidateLandingFailed warns at 60 s for 3 m and PrismBlockCandidateLandingFailedCritical pages at 307 s for 1 m, the same floor and dwell the revision-work critical applies when the landing fails after the frontend's own active-chain observation, so a won block whose landing keeps failing is never silent. -1 with the other candidate gauges when the collector fails." + }, + "blocks_total": { + "status": "Process-local count of first block confirmations observed after a successful settlement or reconciliation commit. Reconciliation includes a previously orphaned candidate that later becomes active; the durable audit publication marker prevents repeat counting on later disconnect/reconnect cycles. Cancellation or restart after commit can lose an increment." + }, + "block_candidates_orphaned_total": { + "status": "Counts offered-candidate orphan settlements whose successful completion this instance observed after commit (#415): its landed audit is durable, and one coherent tip observation showed a different block active at its height with at least PRISM_CANDIDATE_ORPHAN_CONFIRMATIONS confirmations. May undercount if cancellation or restart occurs after the database commit and before the process records completion. Never counts a failed observation; a reorg that later reactivates the block is credited by the reorg reconciler without touching this counter. Process-local; starts at zero. No firing rule." + }, + "node_peers": { + "status": "Recorded only when the readiness attempt reached getnetworkinfo and it answered, which is never on regtest, during initial block download, or while blocks and headers disagree; observation adds no RPC call. -1 means the latest attempt did not learn the count, including RPC failure; the gauge never keeps an earlier reading. An answered zero is a real zero, and it is below every configured PRISM_MIN_PEERS floor." + }, + "node_initial_block_download": { + "status": "Recorded whenever the latest getblockchaininfo answered with the boolean, including the 1 that then makes readiness fail. -1 means startup, a failed call, or a field that was not a boolean." + }, + "node_observation_age_seconds": { + "status": "Stamped when the call answered, not when the whole readiness check passed, so a reachable but unready node is distinguishable from an unreachable one. Computed from a monotonic clock at scrape time. Peers and the sync flag come from the latest attempt to start, so a slower older attempt landing late cannot roll them back, while the age keeps the latest answer any attempt received." + }, + "hashrate_rollup_watermark_lag_seconds": { + "status": "The stored watermark is a share sequence, not a time; a caught-up pass is one that advanced the watermark within its batch bound, leaving no unfolded share. A pass that lost the race to another frontend leaves the stamp unchanged, so a frontend that keeps losing reads high: alert on the minimum across instances. -1 runs from loop start until its first such pass, and PRISM_HASHRATE_ROLLUP_ENABLED=0 leaves the family declared with no sample at all." + }, + "work_refresh_stalled_seconds": { + "status": "Stamped when Coordinator::refresh_once succeeds (#525), whether it published new work or revalidated the current work; readiness, which job, share and block admission require within the health timeout, is renewed by that success and, since #622, also by a refresh-grade node poll that finds the published tip; `qbit_prism_tip_poll_age_seconds` is its age, so a frontend whose rebuilds stall while its polls answer reads young there and grows here. A healthy frontend reads at most about one poll interval. A refresh that keeps failing, on a new tip or the same parent (for example PayoutExceedsCandidateBalance when sub-floor dust exists without a pool fee, on either settlement path), or that hangs, grows it without bound, while semantic coverage keeps reading one because sessions still hold the latest published generation. Counted from coordinator start until the first success, so a restart into a stall still grows; -1 only before the health publisher first runs. Pages through PrismWorkRefreshStalledCritical." + }, + "tip_poll_age_seconds": { + "status": "Computed when the health publisher runs, from the monotonic stamp readiness keeps (#622). A refresh renews it when it publishes or revalidates work. A node poll renews it, to when the poll was sent, only when no newer observation has found another tip, it found the published tip, no readiness revocation raced it, the published template is within PRISM_TEMPLATE_MAX_AGE_SECONDS, readiness was not already revoked, no refresh has since found the cluster on a heavier chain, and, under CTV settlement, a relay floor read no older than the poll is recorded and the published work's fee meets it: the refresh's own poll at its start, once that refresh's chain observation and reconciliation agree and it has read the relay floor, and the block wait's, which polls on an unchanged notification only once readiness is at least one interval old and at most once per interval, with PRISM_BLOCKWAIT_ENABLED: a third of PRISM_HEALTH_TIP_POLL_MAX_AGE_SECONDS and at most 5 s (5 s at the default 15 s; a short timeout polls that often), which is also how long the idle block wait waits for a new block. That poll is one readiness check: getblockchaininfo, getnetworkinfo on a public chain (the peer floor) and, under CTV settlement, getmempoolinfo (the relay floor); a failed floor read renews nothing. A healthy frontend reads at most about one poll interval. At PRISM_HEALTH_TIP_POLL_MAX_AGE_SECONDS job preparation defers with `tip polling stale`, share and block admission answer backend-rpc-unavailable and /healthz reports unready. -1 means readiness is unavailable: never established, or revoked by a failed node observation, a stale template, an unavailable fee floor or, under CTV settlement, a relay floor read above the published work's fee (a refresh's or the block wait's), until a refresh restores it; it is never a reading of zero. Read it with `qbit_prism_work_refresh_stalled_seconds`: young here and old there is a stalled rebuild on a responsive node." + }, + "job_preparation_deferrals_total": { + "labels": { + "reason": "JobDeferral" + }, + "status": "Counted once per `job preparation deferred` WARN, with the first check that refused the preparation (#622): `tip_polling_stale` (readiness older than PRISM_HEALTH_TIP_POLL_MAX_AGE_SECONDS and no replacement lease), `tip_polling_unavailable` (readiness revoked), `new_tip_pending` (a newer tip is observed and its work is not published yet), `work_retired` (the authority the work was read with is retired, logged as `payout snapshot stale` for a payout revision or prior-balance change, including a same-tip build a later publication overtook without keeping them, a newer tip's published work replacing its parent, or a lease or publication the work no longer matches, and as `window_not_held` when this primary does not hold the work's window after a writer timeline change (#619)), `fee_floor` (the job's CTV fee is below the live relay floor, or the floor is unknown) and `other` (readiness changed during admission, a database or node error, or a build failure). Tip authority is checked before the payout revision, so `tip_polling_stale` can mask retired work behind it. The session retries on its next timer tick or publication, so one session can count many. Process-local; every series starts at zero." + }, + "share_ledger_partition_lead_rows": { + "status": "max(upper_seq) over the attached rows of qbit_prism_share_partitions minus the next share_seq, read by the database collector with the pending-candidate counts (#144). -1 means the collector's last attempt failed or the ledger is not partitioned yet; it is never a report of exhausted headroom. Falling steadily means the maintenance task is not attaching the lead: the next append past the last bound is refused and retried once after attaching it." + }, + "process_resident_memory_bytes": { + "replaces": [ + "qbit_prism_process_resident_memory_bytes" + ] + }, + "database_pool_acquire_seconds": { + "labels": { + "result": "Outcome" + }, + "status": "Client-observed `PgPool::acquire` time: waiting for a pool permit, the idle-connection liveness ping and, when the pool grows, connection setup; excludes transaction BEGIN and the queries that follow. Recorded by the metrics collector, instrumented coordinator ledger transactions and covered direct ledger queries, rollup transactions and share-partition maintenance. The histogram aggregates observed checkout attempts at covered callers. Public-role read pools, private audit HTTP endpoints and shared API read-model helpers, startup/migration gates, operator tools and telemetry-free compatibility wrappers remain excluded; see the acquisition guide for the source census and exclusion policy. `failure` includes acquire errors, the 15-second acquire timeout and acquisitions abandoned by cancellation, recorded with the elapsed wait. Buckets, count and sum are read together from the live registry on each scrape, independently of cached-body publication; scraping does not create observations or renew snapshot freshness." + }, + "database_advisory_lock_wait_seconds": { + "labels": { + "lock": "LockKind", + "result": "Outcome" + }, + "status": "Client-observed duration of the `pg_advisory_xact_lock` statement, including one database round trip, recorded by the coordinator's ledger for the migration, order and settlement locks; the migration lock is taken only when the coordinator initializes the schema. `failure` includes lock timeout (`PRISM_DATABASE_LOCK_TIMEOUT_MS`, default 5 seconds), statement timeout, deadlock and connection errors, and waits abandoned by cancellation. The CPFP funding lock is not observed (#328). Series appear on their first observation, so a restart's first failure is not visible to `increase()`." + }, + "database_order_lock_hold_seconds": { + "labels": { + "holder": "OrderLockHolder" + }, + "status": "Measured by the coordinator's ledger from the instant `pg_advisory_xact_lock` returned for `ORDER_LOCK` to the instant the holding transaction's COMMIT or ROLLBACK returned (#602); a transaction abandoned by an error ends its hold as it unwinds, a moment before its queued ROLLBACK releases the lock. `append` is the share append, which every miner's acknowledgement waits behind; `first_confirmation` is the offering frontend's settlement that confirmed its block for the first time and `settlement` every other one, including an abandon; `reconcile` is the reorg reconciler, which runs on every template build and can also confirm a block; `first_confirmation` and `settlement` are per attempt, so a settlement that rolls back and is retried records twice; `prepared` is the refresh's window snapshot that prepared work is built from; `candidate_insert` a block-only candidate's insert; `cleanup` blob garbage collection; `fatal_state` fatal-state recovery; `operator` the policy and signing transitions and archive verify and restore, when run with a registry. The schema cutover's hold and archive's statement-scoped append drain are not observed. #602's R2 budget is 250 ms for every holder but `append`. Own ladder from 1 ms to 10 s. Series appear on their first observation. No firing rule of its own." + }, + "collector_available": { + "labels": { + "collector": "Collector" + } + }, + "collector_success": { + "labels": { + "collector": "Collector" + } + }, + "collector_age_seconds": { + "labels": { + "collector": "Collector" + } + }, + "runtime_poll_lag_seconds": { + "labels": { + "task": "TaskKind" + } + }, + "runtime_progress_age_seconds": { + "labels": { + "task": "TaskKind" + } + }, + "runtime_task_stalled": { + "labels": { + "task": "TaskKind" + } + }, + "connections": { + "replaces": [ + "qbit_prism_connected_clients", + "qbit_prism_stratum_active_connections" + ] + }, + "authorized_clients": { + "replaces": [ + "qbit_prism_stratum_authorized_connections" + ] + }, + "authorized_with_current_work": { + "replaces": [ + "qbit_prism_stratum_clients_with_current_tip_jobs" + ] + }, + "pending_job_builds": { + "replaces": [ + "qbit_prism_job_delivery_queue_depth" + ], + "status": "Delivery count replaces the operational intent of queue depth, not its implementation." + }, + "runtime_lag_seconds": { + "replaces": [ + "qbit_prism_lease_heartbeat_monitor_wake_delay_window_max_seconds" + ], + "status": "Runtime-stall intent replaces lease wake delay; no native writer lease." + }, + "metrics_snapshot_available": { + "replaces": [ + "qbit_prism_metrics_snapshot_available" + ] + }, + "metrics_snapshot_stale": { + "replaces": [ + "qbit_prism_metrics_snapshot_stale" + ] + }, + "metrics_snapshot_age_seconds": { + "replaces": [ + "qbit_prism_metrics_snapshot_age_seconds" + ] + }, + "blocks_total": { + "replaces": [ + "qbit_prism_blocks_accepted_total" + ], + "status": "Native confirmed-block process counter; not the legacy node-acceptance accounting boundary." + }, + "stratum_connection_refusals_total": { + "labels": { + "reason": "ConnectionRefusalReason" + }, + "status": "`global_limit` counts a newly accepted socket closed because `PRISM_STRATUM_MAX_CONNECTIONS` permits were exhausted; `username_limit` counts a `mining.authorize` refused by `PRISM_STRATUM_MAX_CONNECTIONS_PER_USERNAME`; `ip_limit` counts a newly accepted socket closed at `PRISM_STRATUM_MAX_CONNECTIONS_PER_IP`, before any permit, task or query. `malformed_frame_budget`, `unknown_job_budget` and `authorize_budget` count a session disconnected for exceeding `PRISM_STRATUM_MAX_MALFORMED_FRAMES_PER_INTERVAL`, `PRISM_STRATUM_MAX_UNKNOWN_JOBS_PER_INTERVAL` or `PRISM_STRATUM_MAX_AUTHORIZE_ATTEMPTS_PER_INTERVAL`, once per session. Same-username reauthorization, reuse of a retained username permit and every disabled limit never count. Refusals are not share rejections, and no series carries a peer address. Process-local; every series starts at zero." + }, + "stratum_connection_limit": { + "status": "Set from `PRISM_STRATUM_MAX_CONNECTIONS` when a Stratum listener starts. The primary and high-difficulty listeners share this limit and `qbit_prism_connections`." + }, + "stale_job_rejections_total": { + "labels": { + "cause": "StaleJobCause" + }, + "status": "Each series counts one existing stale-job decision: resumed-job absolute expiry, CTV relay-fee floor, stale parent or failed stale-grace parent check, and payout-revision mismatch, attributed in that execution order. `window_not_held` (#619) counts a block-bearing proof the candidate enqueue refused before COMMIT because the current primary does not hold the window its coinbase pays: the window's last row is absent or is another share, as after an asynchronous promotion lost the end of the history the work was read from. Nothing was enqueued or offered and the proof's share was not credited; each refusal also logs a WARN naming the block. The ledger counts it at the refusal itself, so a refusal that finishes after the miner's acknowledgement deadline, whose miner was answered `ledger-outcome-unknown`, is counted too. The wire reason and message are unchanged and still counted by `qbit_prism_rejections_total{reason_id=\"stale-job\"}`. Stale-grace credit is not a rejection. Since #478 a block-bearing proof from ordinary work on the active tip is captured whatever its payout revision, so it is no longer counted under `payout_revision`. Still counted: block-only work a same-parent payout replacement retired (its plain shares, and the share of a block captured from it) under `payout_revision`, or `parent_grace` after a tip change; and a capture abandoned before its offer, under `payout_revision` for a ceiling refusal or capture off and `parent_grace` for a lost race. Since #675 `payout_revision` also counts a share-pass submission that passed every submit check, a stale-grace share included, and that the append's payout-revision fence then refused before any write, because a settlement moved the revision first: a plain share, or a block-bearing one with capture off, whose lost block a WARN names. It is counted when the miner is answered `stale-job`, so a block-bearing append still waiting at its acknowledgement deadline (answered `ledger-outcome-unknown`) is not. The series does not separate fence refusals from submit-check ones. Process-local; every series starts at zero." + }, + "accepted_block_to_revision_work_seconds": { + "labels": { + "result": "RevisionWorkResult" + }, + "replaces": [ + "qbit_prism_accepted_block_preview_publication_seconds" + ], + "status": "Frontend-local monotonic timing; no inference of a peer node acceptance timestamp. Every result ends at successful mining.notify write, not prepared publication or proof-to-first-offer. Superseded means a later payout revision arrived before delivery and measures through replacement-work delivery. The pending gauge retains unresolved delivery age. Finite buckets: 0.25, 0.5, 1, 2.5, 5, 10, 30, 60, 120, 300, 307 and 600 seconds." + }, + "accepted_block_revision_work_pending_seconds": { + "status": "Computed at scrape time and overlaid on cached metric bodies. A second acceptance never resets the oldest known unresolved delivery wait; a failed build does not clear it. Unknown intervals are exposed independently by accepted_block_revision_work_tracking_unknown, so they cannot mask a separate measured stall. Lost ordering history makes pending age unknown. An offer the node accepted without an active-chain observation is an unlanded acceptance (#493): excluded here and reported by accepted_block_unlanded_seconds until an active-chain observation lands it on its original acceptance clock or a proven orphan closes it." + }, + "accepted_block_revision_work_tracking_unknown": { + "status": "Process-local boolean, zero at a fresh start and one for incomplete observation, lost target knowledge or bounded tracking saturation. May be one alongside a positive known pending age; never treat a known age as proof that all tracking is complete. No labels." + }, + "accepted_block_unlanded_seconds": { + "status": "Computed at scrape time and overlaid on cached metric bodies (#493). Set by the offering frontend at a definitive null submitblock reply; an active-chain observation (the post-offer landing observation, a reconcile proof or a settlement) lands the block, which then joins accepted_block_revision_work_pending_seconds on its original acceptance clock; a committed proven orphan closes it without a histogram sample. An unlanded acceptance never contributes to the known pending age, never attributes a build deadline and is not unknown tracking: a lost tip race is a known state. -1 only when none is tracked and identity tracking is saturated, so an unlanded acceptance may be untracked; a tracked unlanded age stays visible beside saturation, and the tracking-unknown gauge is one either way. While the terminal probe (the ten-second collector read that carries peer orphan evidence) fails or is cancelled, an unlanded-only frontend reads tracking unknown and a -1 pending age, because the lost race's orphan evidence is unreadable; this gauge keeps its age, and the next successful read restores the known state. Warned by PrismAcceptedBlockUnlanded at 30 minutes, never paged." + }, + "revision_work_build_timeouts_total": { + "replaces": [ + "qbit_prism_accepted_parent_preview_wait_timeouts_total" + ], + "status": "Counts actual existing deadline hits, once per operation; success, cancellation and ordinary errors are not timeout events. No deadline or fallback policy is introduced. Only a known open wait attributes a deadline hit here (#493): while tracking is unknown and no known wait remains (a lost or cancelled settlement COMMIT reply, an unsettled orphan verdict, bounded-tracking saturation or lost ordering history), a deadline hit is an ordinary job_delivery_failures_total event, accepted_block_revision_work_tracking_unknown reports the unknown state and the pending gauge stays -1; it is not counted here and does not stamp a wait recovered after the deadline degraded; a wait recovered before the deadline counts and is degraded as usual." + } + }, + "public": { + "qbit_prism_public_requests_total": { + "type": "counter", + "labels": "none", + "role": "public-api", + "meaning": "Every routed request, including /healthz and /metrics; never use in a request-rate rule." + }, + "qbit_prism_public_ledger_ready": { + "type": "gauge", + "labels": "none", + "role": "public-api", + "meaning": "Schema/read probe succeeded and remains within its freshness budget; zero before a probe, after failure, or when stale. Does not include the separate replica heartbeat refusal." + }, + "qbit_prism_public_ledger_probe_age_seconds": { + "type": "gauge", + "labels": "none", + "role": "public-api", + "meaning": "Monotonic age of the last completed readiness probe (including failure); -1 before the first attempt." + }, + "qbit_prism_public_staleness_refusals_total": { + "type": "counter", + "labels": "none", + "role": "public-api", + "meaning": "Responses refused for exceeding an endpoint cache-age budget." + }, + "qbit_prism_public_replica_refusals_total": { + "type": "counter", + "labels": "none", + "role": "public-api", + "meaning": "Database-reading routes refused by the public replica contract." + }, + "qbit_prism_public_audit_artifact_refusals_total": { + "type": "counter", + "labels": "none", + "role": "public-api", + "meaning": "Audit artifact requests refused with 503 audit_artifact_busy because PRISM_PUBLIC_AUDIT_ARTIFACT_MAX_IN_FLIGHT audit artifacts were already in flight. Public-api role only: the mining process serves the same route on its operator listener and applies the same cap there, but has no public-service metrics owner, so those refusals are uncounted." + }, + "qbit_prism_public_degraded_responses_total": { + "type": "counter", + "labels": "none", + "role": "public-api", + "meaning": "Cached HIT responses served during database unavailability." + }, + "qbit_prism_public_database_outage_refusals_total": { + "type": "counter", + "labels": "none", + "role": "public-api", + "meaning": "Uncached database reads refused during database unavailability." + }, + "qbit_prism_public_responses_total": { + "type": "counter", + "labels": "`status` (HTTP status code)", + "role": "public-api", + "meaning": "HTTP responses by status, including health probes; appears after the first response." + }, + "qbit_prism_public_cache_total": { + "type": "counter", + "labels": "`state=HIT,MISS,STALE,BYPASS`", + "role": "public-api", + "meaning": "Public route cache outcomes; appears after a public route request." + }, + "qbit_prism_public_replica_in_recovery": { + "type": "gauge", + "labels": "none", + "role": "public-api (replica=require)", + "meaning": "Last observed public read database recovery state; zero if never observed. Not the dedicated HA standby." + }, + "qbit_prism_public_replica_heartbeat_age_seconds": { + "type": "gauge", + "labels": "none", + "role": "public-api (replica=require)", + "meaning": "Last observed public WAL-receiver heartbeat age plus monotonic time since observation; -1 if unknown/disconnected." + }, + "qbit_prism_public_replica_replay_lag_seconds": { + "type": "gauge", + "labels": "none", + "role": "public-api (replica=require)", + "meaning": "Time since the last replayed transaction plus observation age; can increase on an idle primary and is not a byte-backlog or HA failover-loss measurement; -1 if unknown." + }, + "qbit_prism_public_replica_apply_backlog_bytes": { + "type": "gauge", + "labels": "none", + "role": "public-api (replica=require)", + "meaning": "Last observed receive-LSN minus replay-LSN bytes; -1 if unknown; may remain cached after probe failure." + }, + "qbit_prism_public_replica_max_lag_seconds": { + "type": "gauge", + "labels": "none", + "role": "public-api (replica=require)", + "meaning": "Configured public read replica maximum lag/heartbeat-age budget in seconds." + } + } +} diff --git a/docs/prism-metrics-cardinality-privacy.md b/docs/prism-metrics-cardinality-privacy.md new file mode 100644 index 000000000..5787a62e5 --- /dev/null +++ b/docs/prism-metrics-cardinality-privacy.md @@ -0,0 +1,117 @@ +# Native metrics cardinality and privacy acceptance (#278) + +The run-role wire contract is tested against real `/metrics` responses, with an +independent expected set in +`crates/qbit-prism-server/tests/observability/contract.rs`. Expectations are not +generated from registry descriptors or the inventory document. The census +compares every family name and type, requires one HELP and TYPE per family, and +compares every sample name and complete label tuple. Duplicate samples fail. + +The current bound is **64 families and 679 series**, including histogram +`_bucket`, `_sum`, `_count` and `le="+Inf"` series. Startup has **244 series**: +first-offer, advisory-lock and refresh metadata exist, but their 434 derived +series remain absent until observations occur, and the hashrate rollup lag declares metadata +without a sample until its loop starts, which it never does while +`PRISM_HASHRATE_ROLLUP_ENABLED=0`. The three node gauges are registered unknown +at startup and carry no labels. All 14 rejection reasons, two ACK outcomes, +two pool outcomes, six lock/outcome pairs, two collectors, ten task kinds, six +connection refusal reasons, four stale-job causes and three revision-work results are covered. Bucket bounds +are independently pinned, including ACK's 15/20-second buckets and CTV chunk +rows' one-row bucket and the landing ladder from 0.25 through 600 seconds. + +#275's refresh-path accounting adds two families: fourteen +`qbit_prism_refresh_window_acquisitions_total{outcome}` counters, present at +zero from startup, and the `qbit_prism_refresh_seconds{trigger,acquisition}` +histogram, whose 24 closed label pairs (eight triggers by three acquisitions) +are reserved at startup and render 336 series once every pair has been +observed. Both label sets are closed enums; no tip hash, height, job or miner +enters them. + +#458 adds exactly four families and 48 startup/populated series: 45 histogram +series for `result=published|degraded|superseded`, a pending gauge, a separate +unlabeled tracking-unknown gauge and one timeout counter. #493 adds three +unlabeled gauges present from startup, `qbit_prism_accepted_block_unlanded_seconds` +and the database collector's `qbit_prism_block_candidate_oldest_unacknowledged_seconds` +and `qbit_prism_block_candidate_oldest_landing_failed_seconds`. Block identities remain inside the bounded observation state; +there is no frontend, worker, block, height or job label. The real offer/delivery +privacy assertion is `landing_metrics::lost_offer_reply_starts_at_active_proof_and_is_not_a_work_build_timeout`. +#600's post-landing `malloc_trim` adds three unlabeled families: the +`qbit_prism_landing_malloc_trim_released_bytes_total` counter (zero at startup), +the `qbit_prism_landing_malloc_trim_resident_bytes` gauge (-1 at startup) and +the `qbit_prism_landing_malloc_trim_seconds` histogram, declared at startup and +sampled from the first trim: 2 startup series and 16 populated ones. +Adjacent node/rollup additions are outside this branch's census and must be +combined explicitly when integrating another metrics change. + +`every_http_family_and_closed_label_tuple_stays_bounded_under_varied_inputs` +drives every public enum variant and repeats varied values, missing and arbitrary +reasons, successful zero observations, failed collections and cancelled +collections. Neither observation values nor failure states may introduce names +or labels. Unknown-only landing age remains -1; its separate boolean unknown +gauge can coexist with a known pending age, distinct from a fresh successful zero. The node +gauges are driven through every combination a readiness attempt can report, +including the answered zero peer count. This is a +run-role qualification; public-api family and label contracts are unchanged. + +Privacy checks cover two producer-to-render paths: + +- `stratum_hashes_heights_jobs_and_miners_never_enter_metrics_exposition` seeds + synthetic parent hashes, coinbase heights, job IDs and miner names through the + production TCP listener and codec. It verifies issued work contains the input, + accepted submissions reach the fixture backend, and identifier-bearing backend + errors reach the unchanged protocol response. Metrics must show the accepted + and rejected events while exposing none of those identifiers or proof hashes. +- `candidate_identifiers_and_heights_never_enter_collector_http_metrics` inserts + synthetic hashes, heights, job IDs and miner IDs into the real migrated outbox + in a disposable PostgreSQL schema. The real collector and HTTP renderer must + expose aggregate counts/age only, including after a failed collection. It + verifies the stored input and resulting count so an unused fixture cannot pass. + +Separately, `census_and_privacy_hold_through_unavailable_fresh_and_stale_http_snapshots` +checks the complete census, cache headers and unknown age semantics through the +real API router in all three freshness states. Its seeded health payload checks +separation between health and metrics publication: `/metrics` does not read +`state.health`. That absence is not evidence of a third producer privacy path; +the Stratum and candidate-collector tests provide that evidence. + +Heights are forbidden in metadata, names and labels. Numeric samples are compared +across deliberately different source heights: a series that follows both input +heights fails. A legitimate count or elapsed value coincidentally equal to one +height is not treated as disclosure. Negative controls demonstrate rejection of +new families, extra labels, changed types, missing enum/bucket series, duplicate +samples, textual identifiers and numeric height attribution. + +## Issue acceptance mapping + +| #278 criterion | Native evidence and disposition | +| --- | --- | +| Registry, HELP/TYPE and movement | Delivered foundation; existing `observability/registry.rs` movement/histogram tests and `observability/inventory.rs` role scrapes remain in place. The new census pins the complete run-role wire set. | +| Closed rejection enum and bounded name/label census | Covered by `observability/cardinality.rs`, with actual HTTP rendering, every enum variant, repeated input variation and negative controls. | +| No hash/height disclosure in metrics | Covered by `observability/privacy.rs` and `observability/database_privacy.rs`; the latter requires disposable PostgreSQL and runs in the existing required `observability_database -- --ignored` CI suite. | +| Blocked critical work degrades native health | Existing `blocked_critical_poll_degrades_real_http_while_another_worker_remains_healthy` blocks a tracked poll for three seconds and verifies live HTTP health/metrics failure and recovery. | +| Cached OK cannot mask failure | Existing `expired_publication_fails_closed_through_the_router_and_recovers` expires cached health/metrics and checks HTTP freshness and recovery; the blocked-poll test covers failure overriding a still-fresh cached OK. | +| Wall-clock changes cannot change native freshness | Existing `wall_clock_fields_do_not_change_monotonic_health_or_mask_a_base_failure` and `age_uses_monotonic_time_and_exceeds_the_budget_strictly` cover wall fields and the monotonic stale boundary. | +| Exact 2.x `/healthz` shape | **Intentionally not identical; do not mark exact parity fulfilled.** `known_legacy_health_fields_follow_pinned_types_without_inventing_missing_values` pins only supported aliases using `health_2x_known_fields.json`. `ready_miner_count` was accepted-share participants, and `max_blocks` was the legacy pool-close cap; neither has a native equivalent. Native backend/health contracts retain their documented differences. | + +The issue's combined legacy health checkbox must be split when recording +acceptance: the native failure/freshness criteria have evidence; the obsolete +exact-shape clause is an intentional difference. This qualification does not add +per-worker families, change readiness, alter acquisition deadlines, or expand +the separate SQL acquisition coverage work. The node and rollup-lag families +added by the #278 hardening remainder are label-less and carry no per-worker, +thread, file-descriptor or alert-rule additions; they are documented in +[the native inventory](prism-native-metrics.md). + +## Running the focused checks + +```sh +CARGO_BUILD_JOBS=2 cargo test -p qbit-prism-server --test observability +CARGO_BUILD_JOBS=2 cargo test -p qbit-prism-server --test stratum_protocol metrics_privacy +CARGO_BUILD_JOBS=2 cargo test -p qbit-prism-server --lib api::metrics_snapshot::tests +# Use only a disposable PostgreSQL database configured through the test gate: +CARGO_BUILD_JOBS=2 cargo test -p qbit-prism-server --test observability_database -- --ignored +``` + +See [the integration test gate](prism-integration-test-gate.md) for database +configuration. Run the existing native health and snapshot tests when verifying +the health acceptance mapping; none requires exact legacy response equality. diff --git a/docs/prism-metrics-histogram-consumers.md b/docs/prism-metrics-histogram-consumers.md new file mode 100644 index 000000000..17f17e309 --- /dev/null +++ b/docs/prism-metrics-histogram-consumers.md @@ -0,0 +1,162 @@ +# Reading native event histograms + +Event histogram values are elapsed **seconds** supplied by the event producer, +except `qbit_prism_ctv_fanout_broadcaster_chunk_rows`, which counts rows (see +[Reading the CTV chunk-row histogram](#reading-the-ctv-chunk-row-histogram)). +The registry converts `Duration` to seconds and records it once per hook call; +it does not infer a wait from a configured timeout. The producer owns the timing +boundary, outcome, and whether cancellation produces an observation. Pool +acquisition and advisory-lock waits are separate measurements, not the complete +share acknowledgement or database operation duration. + +The default finite bucket upper bounds are 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1, +2.5, 5, 10, and 30 seconds, followed by `+Inf`. Only +`qbit_prism_share_ack_seconds` and its landing-window subset add 15 and 20 +seconds, and only the chunk-row histogram and the `ORDER_LOCK` hold histogram +(see [Landing-window ACKs and ORDER_LOCK holds](#landing-window-acks-and-order_lock-holds-602)) +replace the ladder. Buckets are cumulative and include +their upper bound: an observation of exactly 5 seconds increments `le="5"` and +every larger bucket; an observation greater than 5 and at most 10 seconds first +appears in `le="10"`. An ACK of exactly 15 seconds first appears in `le="15"`, +just over 15 seconds in `le="20"`, exactly 20 seconds in `le="20"`, and just over +20 seconds in `le="30"`. First-offer, pool-acquisition and advisory-lock +histograms retain their original ladder: exactly 15 seconds first appears in +`le="30"`. A nominal +timeout does not determine a bucket: use the measured elapsed time, which can +include scheduling delay. + +## Share ACK deadlines, cost and consumer compatibility + +The 15-second bound matches the default `PRISM_SHARE_COMMIT_TIMEOUT_SECONDS`; +20 seconds matches that default plus the five-second reconciliation grace for +an ordinary share whose COMMIT was already in flight. Configuration can move +the commit deadline, and candidate-bearing appends have separate rules. These +fixed buckets do not adapt to either. ACK timing uses Tokio's monotonic clock +from complete submit-frame receipt through the successful response write, +including validation and response overhead; it does not measure only the ledger +commit. A bucket crossing alone cannot identify a commit failure or an unknown +ledger outcome. Inspect `result` and the rejection/late-confirmation counters +alongside latency; failed response writes produce no ACK observation. + +For a fully populated scrape, there are 11 histogram label combinations: two +ACK results, one first offer, two pool outcomes and six lock/outcome pairs. +Adding these bounds to the shared ladder would add 22 series. Restricting them +to ACKs adds **four series per process** (two bounds × two results), including +at startup: ACK exposition grows from 28 to 32 series. All existing buckets, +`+Inf`, `_sum`, `_count`, family names and labels remain. Together with the +single new `rejections_total{reason_id="unrecognised"}` series, the #278 change +adds **five series per process**; scrape labels multiply the cost across targets. + +Classic [`histogram_quantile`](https://prometheus.io/docs/prometheus/latest/querying/functions/#histogram_quantile) +still interpolates within buckets. The old +(10, 30] interval becomes (10, 15], (15, 20] and (20, 30], reducing its maximum +width from 20 to 10 seconds. Quantile values and decisions at some thresholds +change: if all observations are 14 seconds, p99 estimates change from 29.8 to +14.95 seconds, which changes a comparison against 15 seconds. This is still an +estimate, not the observed p99. The checked-in `PrismShareAckP99High` expression +combines both results per instance and uses the existing one-second bound; +the threshold and expression stay as written, but upper-tail estimates change. +Other histogram families keep their previous interpolation. + +For exact counts relative to an elapsed-time boundary, subtract cumulative +buckets: `le="20" - le="15"` counts ACKs in (15, 20], and `_count - le="20"` +counts ACKs strictly over 20 seconds (use matching labels and rate windows). +Keep accepted and rejected outcomes separate when interpreting those counts. +Do not treat missing new buckets on an old instance as zero. During mixed-version +aggregation, use only the common ladder (exclude `le="15"` and `le="20"`) or +separate versions; use the new bounds only after every contributing instance +has supplied them throughout the query's rate window. This also applies to the +first rate window following a restart into the new version. + +The registry's family selector is shared by observation and rendering. Consumers +of `metrics::BUCKETS`, including live pool-snapshot tests and pool-alert fixtures, +continue to use the default ladder. The rendered event fixture pins all existing +samples plus the five additive series; HTTP inventory tests retain all 46 +coordinator and 14 public families. See the [native inventory](prism-native-metrics.md) +for reason-label compatibility. + +## Landing-window ACKs and ORDER_LOCK holds (#602) + +`qbit_prism_share_ack_landing_window_seconds{result}` holds the +`qbit_prism_share_ack_seconds` observations whose submission arrived within 30 +seconds after a pool block acceptance the frontend observed; each is recorded +in both families, on the same ladder. Steady state is therefore the difference, +bucket by bucket and with matching labels and rate windows: + +```promql +sum by (le) (rate(qbit_prism_share_ack_seconds_bucket[1h])) +- +sum by (le) (rate(qbit_prism_share_ack_landing_window_seconds_bucket[1h])) +``` + +A landing window holds a few hundred to a few thousand acknowledgements and +landings are hours apart, so a five-minute `histogram_quantile` over the +landing family is mostly empty. Judge landings with +`qbit_prism_share_ack_slow_landing_windows{p99_above_seconds}` instead: the +process counts, per window, how many acknowledgements exceeded 2 and 10 +seconds, which gives the nearest-rank p99 against each bound exactly, and +publishes how many consecutive windows exceeded it. + +`qbit_prism_database_order_lock_hold_seconds{holder}` has its own ladder: +0.001, 0.0025, 0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1, 2.5, 5 and 10 +seconds, then `+Inf`. An append holds `ORDER_LOCK` for milliseconds, and #602's +budget for every other holder is 250 ms (`le="0.25"`). `first_confirmation` and +`orphan` appear about once per block (a settlement attempt that rolls back and +is retried records a second hold), so read their `_sum` increases per landing +rather than a quantile. `reconcile` runs on every template build as well as at +landings, so its landing holds are mixed with routine ones. A hold runs from the grant of the lock to the end of its transaction, +so it includes the transaction's own statements and its COMMIT round trip; +the wait for the lock is `qbit_prism_database_advisory_lock_wait_seconds`. + +## Reading the CTV chunk-row histogram + +`qbit_prism_ctv_fanout_broadcaster_chunk_rows` keeps its 2.x.x name and type +but counts **rows**, not seconds. A native broadcaster chunk is one claimed +fanout, so every observation is exactly 1 and the ladder is the single finite +bound `le="1"` followed by `+Inf`: `_count` is the number of claimed attempts +and `_sum` equals `_count`. Read attempt throughput from `rate(..._count[5m])`; +`histogram_quantile` over this family is an interpolation artifact, not a +measurement. The 2.x.x ladder (1, 2, 5, 10, 25, 50, 100) is not exported, so a +query for `le="10"` matches no native series, and a mixed-version aggregation +must use only `le="1"`, `+Inf`, `_sum` and `_count`. The companion +`qbit_prism_ctv_fanout_broadcaster_chunk_seconds` histogram uses the default +time ladder and records one observation per attempt, including an attempt +whose completion persistence failed. + +## Reading small waits + +For waits below 10 ms, use `_sum / _count` to calculate an **average**, not a +percentile. The first bucket groups all observations at or below 10 ms, so it +cannot resolve a distribution within that interval. A histogram quantile there +is an estimate from the bucket boundaries, not a measured sub-10-ms percentile. + +For example, this five-minute average advisory-lock wait preserves lock kind and +outcome while combining instances: + +```promql +sum by (lock, result) ( + rate(qbit_prism_database_advisory_lock_wait_seconds_sum[5m]) +) +/ +sum by (lock, result) ( + rate(qbit_prism_database_advisory_lock_wait_seconds_count[5m]) +) +``` + +The result is seconds per recorded observation; multiply by 1000 for +milliseconds. A zero observation count means no average is available, not that +the average wait was zero. Keep missing series and a zero denominator distinct +from a measured zero-duration wait. Inspect success and failure outcomes +separately when the question depends on whether acquisition completed. + +Closed event-label observations and increments allocate no heap memory after +instance and platform mutex initialization, including the first observation of a +previously hidden histogram. Construction, rendering, platform mutex +initialization, and arbitrary owned labels are outside this guarantee. The test +performs a setup render to initialize the mutex without recording any events. +The scoped allocation-count regression test also covers concurrent recording; +it does not measure throughput or establish a throughput improvement. + +```sh +cargo test --locked --release -p qbit-prism-server --test metrics_allocation +``` diff --git a/docs/prism-native-alert-rules.json b/docs/prism-native-alert-rules.json new file mode 100644 index 000000000..e2f1572e0 --- /dev/null +++ b/docs/prism-native-alert-rules.json @@ -0,0 +1,1140 @@ +{ + "network_placeholder": "__NETWORK__", + "rules": [ + { + "uid": "qbit-prism-metrics-nodata", + "title": "PrismMetricsNoData", + "role": "run", + "expr": "((count by (job, instance, network) (qbit_prism_health_state{job=\"qbit-prism\",network=\"__NETWORK__\"})) or on(job, instance, network) (0 * up{job=\"qbit-prism\",network=\"__NETWORK__\"})) == bool 0", + "condition_type": "threshold", + "evaluator": "gt", + "threshold": 0, + "for": "3m", + "severity": "warning", + "service": "qbit-mining-pool", + "labels": { + "alert_class": "prism-runtime" + }, + "summary": "PrismMetricsNoData", + "description": "A configured coordinator target has no native health family; restore scrape visibility. Preserves the deployed three-minute scrape coverage grace.", + "no_data_state": "Alerting", + "exec_err_state": "Alerting", + "basis": "Preserves the deployed three-minute scrape coverage grace.", + "producer_refs": [ + "crates/qbit-prism-server/src/metrics/registry.rs#L43", + "crates/qbit-prism-server/src/metrics.rs#L39" + ] + }, + { + "uid": "qbit-prism-metrics-endpoint-down", + "title": "PrismMetricsEndpointDown", + "role": "run", + "expr": "up{job=\"qbit-prism\",network=\"__NETWORK__\"} == bool 0", + "condition_type": "threshold", + "evaluator": "gt", + "threshold": 0, + "for": "3m", + "severity": "warning", + "service": "qbit-mining-pool", + "labels": { + "alert_class": "prism-runtime" + }, + "summary": "PrismMetricsEndpointDown", + "description": "Coordinator /metrics cannot be scraped; miner-impact paging remains with the sidecar. Preserves the deployed three-minute scrape outage grace.", + "no_data_state": "Alerting", + "exec_err_state": "Alerting", + "basis": "Preserves the deployed three-minute scrape outage grace.", + "producer_refs": [ + "crates/qbit-prism-server/src/metrics.rs#L39" + ] + }, + { + "uid": "qbit-prism-connected-clients", + "title": "PRISM Connected Clients", + "role": "run", + "expr": "(sum(qbit_prism_connections{job=\"qbit-prism\",network=\"__NETWORK__\"} and on(job, instance, network) (qbit_prism_metrics_snapshot_available{job=\"qbit-prism\",network=\"__NETWORK__\"} == 1) and on(job, instance, network) (qbit_prism_metrics_snapshot_stale{job=\"qbit-prism\",network=\"__NETWORK__\"} == 0) and on(job, instance, network) (up{job=\"qbit-prism\",network=\"__NETWORK__\"} == 1)) < bool 10) and on() (count(up{job=\"qbit-prism\",network=\"__NETWORK__\"}) == count(qbit_prism_connections{job=\"qbit-prism\",network=\"__NETWORK__\"} and on(job, instance, network) (qbit_prism_metrics_snapshot_available{job=\"qbit-prism\",network=\"__NETWORK__\"} == 1) and on(job, instance, network) (qbit_prism_metrics_snapshot_stale{job=\"qbit-prism\",network=\"__NETWORK__\"} == 0) and on(job, instance, network) (up{job=\"qbit-prism\",network=\"__NETWORK__\"} == 1)))", + "condition_type": "threshold", + "evaluator": "gt", + "threshold": 0, + "for": "3m", + "severity": "warning", + "service": "qbit-mining-pool", + "labels": { + "alert_class": "prism-runtime" + }, + "summary": "PRISM Connected Clients", + "description": "Authorized and unauthenticated connections are included in the network total; six connections on each of two coordinators is twelve and remains healthy. Missing, failed or stale targets suppress the incomplete sum and remain covered by the scrape/snapshot alerts. Preserves the deployed network-wide floor: sum all local native connection counts, below ten for three minutes; evaluate only with fresh measurements for every configured target.", + "no_data_state": "OK", + "exec_err_state": "OK", + "basis": "Preserves the deployed network-wide floor: sum all local native connection counts, below ten for three minutes; evaluate only with fresh measurements for every configured target.", + "connected_clients_rule": true, + "producer_refs": [ + "crates/qbit-prism-server/src/metrics/registry.rs#L45", + "crates/qbit-prism-server/src/metrics/registry.rs#L78", + "crates/qbit-prism-server/src/metrics/registry.rs#L79", + "crates/qbit-prism-server/src/metrics.rs#L39" + ] + }, + { + "uid": "qbit-prism-metrics-snapshot-stale", + "title": "PrismMetricsSnapshotStale", + "role": "run", + "expr": "qbit_prism_metrics_snapshot_stale{job=\"qbit-prism\",network=\"__NETWORK__\"} > bool 0", + "condition_type": "threshold", + "evaluator": "gt", + "threshold": 0, + "for": "3m", + "severity": "warning", + "service": "qbit-mining-pool", + "labels": { + "alert_class": "prism-runtime" + }, + "summary": "PrismMetricsSnapshotStale", + "description": "Snapshot stale means cached operational gauges cannot establish health. #277 computes staleness at scrape time from max(3 * health refresh, 15s); no copied age constant.", + "no_data_state": "Alerting", + "exec_err_state": "Alerting", + "basis": "#277 computes staleness at scrape time from max(3 * health refresh, 15s); no copied age constant.", + "producer_refs": [ + "crates/qbit-prism-server/src/metrics/registry.rs#L79", + "crates/qbit-prism-server/src/metrics.rs#L39" + ] + }, + { + "uid": "qbit-prism-metrics-snapshot-unavailable", + "title": "PrismMetricsSnapshotUnavailable", + "role": "run", + "expr": "qbit_prism_metrics_snapshot_available{job=\"qbit-prism\",network=\"__NETWORK__\"} == bool 0", + "condition_type": "threshold", + "evaluator": "gt", + "threshold": 0, + "for": "3m", + "severity": "warning", + "service": "qbit-mining-pool", + "labels": { + "alert_class": "prism-runtime" + }, + "summary": "PrismMetricsSnapshotUnavailable", + "description": "Snapshot unavailable means cached operational gauges cannot establish health. #277 computes staleness at scrape time from max(3 * health refresh, 15s); no copied age constant.", + "no_data_state": "Alerting", + "exec_err_state": "Alerting", + "basis": "#277 computes staleness at scrape time from max(3 * health refresh, 15s); no copied age constant.", + "producer_refs": [ + "crates/qbit-prism-server/src/metrics/registry.rs#L78", + "crates/qbit-prism-server/src/metrics.rs#L39" + ] + }, + { + "uid": "qbit-prism-metrics-snapshot-age", + "title": "PrismMetricsSnapshotAgeHigh", + "role": "run", + "expr": "(qbit_prism_metrics_snapshot_age_seconds{job=\"qbit-prism\",network=\"__NETWORK__\"} >= bool 0) * on(job, instance, network) (qbit_prism_metrics_snapshot_stale{job=\"qbit-prism\",network=\"__NETWORK__\"} == bool 1)", + "condition_type": "threshold", + "evaluator": "gt", + "threshold": 0, + "for": "3m", + "severity": "warning", + "service": "qbit-mining-pool", + "labels": { + "alert_class": "prism-runtime" + }, + "summary": "PrismMetricsSnapshotAgeHigh", + "description": "Snapshot agehigh means cached operational gauges cannot establish health. #277 computes staleness at scrape time from max(3 * health refresh, 15s); no copied age constant.", + "no_data_state": "Alerting", + "exec_err_state": "Alerting", + "basis": "#277 computes staleness at scrape time from max(3 * health refresh, 15s); no copied age constant.", + "producer_refs": [ + "crates/qbit-prism-server/src/metrics/registry.rs#L79", + "crates/qbit-prism-server/src/metrics/registry.rs#L80", + "crates/qbit-prism-server/src/metrics.rs#L39" + ] + }, + { + "uid": "qbit-prism-share-ack-p99", + "title": "PrismShareAckP99High", + "role": "run", + "expr": "((histogram_quantile(0.99, sum by (job, instance, network, le) (rate(qbit_prism_share_ack_seconds_bucket{job=\"qbit-prism\",network=\"__NETWORK__\"}[5m]))) > bool 1) and on(job, instance, network) (sum by (job, instance, network) (increase(qbit_prism_share_ack_seconds_count{job=\"qbit-prism\",network=\"__NETWORK__\"}[5m])) >= 100)) and on(job, instance, network) ((qbit_prism_metrics_snapshot_available{job=\"qbit-prism\",network=\"__NETWORK__\"} == 1) and on(job, instance, network) (qbit_prism_metrics_snapshot_stale{job=\"qbit-prism\",network=\"__NETWORK__\"} == 0))", + "condition_type": "threshold", + "evaluator": "gt", + "threshold": 0, + "for": "3m", + "severity": "warning", + "service": "qbit-mining-pool", + "labels": { + "alert_class": "prism-runtime" + }, + "summary": "PrismShareAckP99High", + "description": "Frame receipt to completed response write, both results combined; durable credit can precede a failed write. Replaces writer-queue pressure with miner-visible latency. One-second histogram boundary; at least 100 completed ACKs in five minutes. Provisional, measure in #291.", + "no_data_state": "OK", + "exec_err_state": "OK", + "basis": "One-second histogram boundary; at least 100 completed ACKs in five minutes. Provisional, measure in #291.", + "producer_refs": [ + "crates/qbit-prism-server/src/metrics/registry.rs#L55", + "crates/qbit-prism-server/src/metrics/registry.rs#L78", + "crates/qbit-prism-server/src/metrics/registry.rs#L79", + "crates/qbit-prism-server/src/metrics/labels.rs#L15", + "crates/qbit-prism-server/src/metrics.rs#L39" + ] + }, + { + "uid": "qbit-prism-landing-ack-p99-slow", + "title": "PrismLandingAckP99Slow", + "role": "run", + "expr": "(qbit_prism_share_ack_slow_landing_windows{job=\"qbit-prism\",network=\"__NETWORK__\",p99_above_seconds=\"2\"} >= bool 3) and on(job, instance, network) ((qbit_prism_metrics_snapshot_available{job=\"qbit-prism\",network=\"__NETWORK__\"} == 1) and on(job, instance, network) (qbit_prism_metrics_snapshot_stale{job=\"qbit-prism\",network=\"__NETWORK__\"} == 0))", + "condition_type": "threshold", + "evaluator": "gt", + "threshold": 0, + "for": "1m", + "severity": "ticket", + "service": "qbit-mining-pool", + "labels": { + "alert_class": "prism-runtime" + }, + "summary": "PrismLandingAckP99Slow", + "description": "Share acknowledgement p99 exceeded 2 seconds in three consecutive landing windows on this frontend. A landing window opens when this frontend first observes a pool block's acceptance (its own submitblock acceptance, or a peer frontend's block on the active chain, seen before it is confirmed or already confirmed at the tip) and admits every share submission that arrives within the next 30 seconds; settlement, which holds ORDER_LOCK while every share append waits for it, runs inside that window (#602). Each window is judged 30 seconds after it stops admitting submissions: its nearest-rank acknowledgement p99 is over a bound exactly when more than one percent of its acknowledgements took longer, counted exactly by the process rather than read from buckets. The gauge holds its value until the next verdict, so the alert stays active until a landing window at or under the bound; a window with no acknowledgement leaves it unchanged. Per frontend. -1 (no verdict since start) never fires; no-data and evaluation errors are OK because the snapshot rules own missing measurements. #602 R1 asks for a landing-window p99 at or under 500 ms and no acknowledgement over 2 s; 2.x.x shows a block-minute p99 of about 4-5 s against about 0.2 s otherwise. Three consecutive windows keep one slow landing (a cold cache, a one-off settlement) from ticketing. Provisional, measure in #291 and #602 step 3.", + "no_data_state": "OK", + "exec_err_state": "OK", + "basis": "#602 R1 asks for a landing-window p99 at or under 500 ms and no acknowledgement over 2 s; 2.x.x shows a block-minute p99 of about 4-5 s against about 0.2 s otherwise. Three consecutive windows keep one slow landing (a cold cache, a one-off settlement) from ticketing. Provisional, measure in #291 and #602 step 3.", + "producer_refs": [ + "crates/qbit-prism-server/src/metrics/registry.rs", + "crates/qbit-prism-server/src/metrics/landing_acks.rs", + "crates/qbit-prism-server/src/metrics/landing.rs", + "crates/qbit-prism-server/src/stratum/share_observation.rs" + ] + }, + { + "uid": "qbit-prism-landing-ack-p99-high", + "title": "PrismLandingAckP99High", + "role": "run", + "expr": "(qbit_prism_share_ack_slow_landing_windows{job=\"qbit-prism\",network=\"__NETWORK__\",p99_above_seconds=\"10\"} >= bool 1) and on(job, instance, network) ((qbit_prism_metrics_snapshot_available{job=\"qbit-prism\",network=\"__NETWORK__\"} == 1) and on(job, instance, network) (qbit_prism_metrics_snapshot_stale{job=\"qbit-prism\",network=\"__NETWORK__\"} == 0))", + "condition_type": "threshold", + "evaluator": "gt", + "threshold": 0, + "for": "1m", + "severity": "warning", + "service": "qbit-mining-pool", + "labels": { + "alert_class": "prism-runtime" + }, + "summary": "PrismLandingAckP99High", + "description": "Share acknowledgement p99 exceeded 10 seconds in the latest landing window on this frontend. A landing window opens when this frontend first observes a pool block's acceptance (its own submitblock acceptance, or a peer frontend's block on the active chain, seen before it is confirmed or already confirmed at the tip) and admits every share submission that arrives within the next 30 seconds; settlement, which holds ORDER_LOCK while every share append waits for it, runs inside that window (#602). Each window is judged 30 seconds after it stops admitting submissions: its nearest-rank acknowledgement p99 is over a bound exactly when more than one percent of its acknowledgements took longer, counted exactly by the process rather than read from buckets. The gauge holds its value until the next verdict, so the alert stays active until a landing window at or under the bound; a window with no acknowledgement leaves it unchanged. Per frontend. -1 (no verdict since start) never fires; no-data and evaluation errors are OK because the snapshot rules own missing measurements. 10 s is two thirds of the default share-commit timeout (PRISM_SHARE_COMMIT_TIMEOUT_SECONDS, 15 s), past which an append is answered ledger-outcome-unknown; rental proxies and some firmware resubmit or reconnect well before it. One window is enough. Provisional, measure in #291 and #602 step 3.", + "no_data_state": "OK", + "exec_err_state": "OK", + "basis": "10 s is two thirds of the default share-commit timeout (PRISM_SHARE_COMMIT_TIMEOUT_SECONDS, 15 s), past which an append is answered ledger-outcome-unknown; rental proxies and some firmware resubmit or reconnect well before it. One window is enough. Provisional, measure in #291 and #602 step 3.", + "producer_refs": [ + "crates/qbit-prism-server/src/metrics/registry.rs", + "crates/qbit-prism-server/src/metrics/landing_acks.rs", + "crates/qbit-prism-server/src/metrics/landing.rs", + "crates/qbit-prism-server/src/stratum/share_observation.rs" + ] + }, + { + "uid": "qbit-prism-reject-ratio", + "title": "PrismRejectRatioByReasonHigh", + "role": "run", + "expr": "(((sum by (job, instance, network, reason_id) (increase(qbit_prism_rejections_total{job=\"qbit-prism\",network=\"__NETWORK__\"}[5m])) / on(job, instance, network) group_left() (sum by (job, instance, network) (increase(qbit_prism_accepted_shares_total{job=\"qbit-prism\",network=\"__NETWORK__\"}[5m])) + sum by (job, instance, network) (increase(qbit_prism_rejections_total{job=\"qbit-prism\",network=\"__NETWORK__\"}[5m])))) > bool 0.05) and on(job, instance, network) ((sum by (job, instance, network) (increase(qbit_prism_accepted_shares_total{job=\"qbit-prism\",network=\"__NETWORK__\"}[5m])) + sum by (job, instance, network) (increase(qbit_prism_rejections_total{job=\"qbit-prism\",network=\"__NETWORK__\"}[5m]))) >= 100))", + "condition_type": "threshold", + "evaluator": "gt", + "threshold": 0, + "for": "3m", + "severity": "warning", + "service": "qbit-mining-pool", + "labels": { + "alert_class": "prism-runtime" + }, + "summary": "PrismRejectRatioByReasonHigh", + "description": "Preserves reason_id on each alert; rejected includes failed response writes. Diagnose ledger-confirmation-failed, ledger-outcome-unknown, backend-rpc-unavailable (the node), backend-database-unavailable (the ledger database, #581), stale-job, duplicate-share and low-difficulty separately. Five percent per canonical reason over five minutes, with at least 100 acceptance/rejection decisions. Not gated on a fresh metrics snapshot: qbit_prism_rejections_total and qbit_prism_accepted_shares_total are rendered at scrape time since #581, so a database outage that stalls the health publisher does not freeze them. Provisional, measure in #291.", + "no_data_state": "OK", + "exec_err_state": "OK", + "basis": "Five percent per canonical reason over five minutes, with at least 100 acceptance/rejection decisions, both counters live at scrape time (#581). Provisional, measure in #291.", + "producer_refs": [ + "crates/qbit-prism-server/src/metrics/registry.rs#L54", + "crates/qbit-prism-server/src/metrics/registry.rs#L60", + "crates/qbit-prism-server/src/metrics/registry.rs#L155", + "crates/qbit-prism-server/src/metrics/labels.rs#L78", + "crates/qbit-prism-server/src/metrics.rs#L252" + ] + }, + { + "uid": "qbit-prism-pool-resubmission-reject-ratio", + "title": "PrismPoolResubmissionRejectRatioHigh", + "role": "run", + "expr": "((sum by (network) (increase(qbit_prism_rejections_total{job=\"qbit-prism\",network=\"__NETWORK__\",reason_id=~\"duplicate-share|unknown-job|stale-job\"}[5m]) and on(job, instance, network) ((qbit_prism_metrics_snapshot_available{job=\"qbit-prism\",network=\"__NETWORK__\"} == 1) and on(job, instance, network) (qbit_prism_metrics_snapshot_stale{job=\"qbit-prism\",network=\"__NETWORK__\"} == 0))) / sum by (network) ((sum by (job, instance, network) (increase(qbit_prism_accepted_shares_total{job=\"qbit-prism\",network=\"__NETWORK__\"}[5m])) + sum by (job, instance, network) (increase(qbit_prism_rejections_total{job=\"qbit-prism\",network=\"__NETWORK__\"}[5m]))) and on(job, instance, network) ((qbit_prism_metrics_snapshot_available{job=\"qbit-prism\",network=\"__NETWORK__\"} == 1) and on(job, instance, network) (qbit_prism_metrics_snapshot_stale{job=\"qbit-prism\",network=\"__NETWORK__\"} == 0)))) > bool 0.05) and on(network) (sum by (network) ((sum by (job, instance, network) (increase(qbit_prism_accepted_shares_total{job=\"qbit-prism\",network=\"__NETWORK__\"}[5m])) + sum by (job, instance, network) (increase(qbit_prism_rejections_total{job=\"qbit-prism\",network=\"__NETWORK__\"}[5m]))) and on(job, instance, network) ((qbit_prism_metrics_snapshot_available{job=\"qbit-prism\",network=\"__NETWORK__\"} == 1) and on(job, instance, network) (qbit_prism_metrics_snapshot_stale{job=\"qbit-prism\",network=\"__NETWORK__\"} == 0))) >= 500)", + "condition_type": "threshold", + "evaluator": "gt", + "threshold": 0, + "for": "5m", + "severity": "warning", + "service": "qbit-mining-pool", + "labels": { + "alert_class": "prism-runtime" + }, + "summary": "PrismPoolResubmissionRejectRatioHigh", + "description": "Pool-wide share rejections for duplicate-share, unknown-job and stale-job, summed over every frontend with a fresh metrics snapshot, as a share of all acceptance and rejection decisions. These are the rejections slow acknowledgements and slow work delivery produce: a rental proxy resubmits a share it got no answer for and the copy is a duplicate (#602), and a miner still working an old job after a stall submits unknown or stale jobs (#598). Unlike PrismRejectRatioByReasonHigh it judges the pool, not one frontend, and the three reasons together, so rejections spread thinly across frontends and reasons still warn. Frontends with a missing or stale snapshot are left out of both sides of the ratio; no-data and evaluation errors are OK. Five percent of the pool's acceptance/rejection decisions over five minutes, from at least 500 decisions, for five minutes; the per-frontend, per-reason rule uses the same five percent. Provisional, measure in #291 and against 2.x.x's post-block reject spikes.", + "no_data_state": "OK", + "exec_err_state": "OK", + "basis": "Five percent of the pool's acceptance/rejection decisions over five minutes, from at least 500 decisions, for five minutes; the per-frontend, per-reason rule uses the same five percent. Provisional, measure in #291 and against 2.x.x's post-block reject spikes.", + "producer_refs": [ + "crates/qbit-prism-server/src/metrics/registry.rs", + "crates/qbit-prism-server/src/metrics/labels.rs", + "crates/qbit-prism-server/src/stratum/share_observation.rs" + ] + }, + { + "uid": "qbit-prism-share-append-failures", + "title": "PrismShareAppendFailures", + "role": "run", + "expr": "(sum by (job, instance, network) (increase(qbit_prism_rejections_total{job=\"qbit-prism\",network=\"__NETWORK__\",reason_id=~\"ledger-confirmation-failed|ledger-outcome-unknown\"}[5m])) > bool 0)", + "condition_type": "threshold", + "evaluator": "gt", + "threshold": 0, + "for": "3m", + "severity": "warning", + "service": "qbit-mining-pool", + "labels": { + "alert_class": "prism-runtime" + }, + "summary": "PrismShareAppendFailures", + "description": "At least one share was rejected in the five-minute window because its ledger confirmation failed or outcome unknown (ledger-confirmation-failed or ledger-outcome-unknown). This preserves the low-volume total-write-failure signal from the deployed rule while retaining the native reason_id. Not gated on a fresh metrics snapshot (#581): qbit_prism_rejections_total is rendered at scrape time, so the rule holds through a database outage that stalls the health publisher instead of flapping with the snapshot's freshness.", + "no_data_state": "OK", + "exec_err_state": "OK", + "basis": "Any ledger rejection whose confirmation failed or outcome unknown (ledger-confirmation-failed or ledger-outcome-unknown) preserves the deployed low-volume total-write-failure coverage; its input is live at scrape time since #581; measure alert volume in #291.", + "producer_refs": [ + "crates/qbit-prism-server/src/metrics/registry.rs#L60", + "crates/qbit-prism-server/src/metrics/registry.rs#L155", + "crates/qbit-prism-server/src/metrics/labels.rs#L78" + ] + }, + { + "uid": "qbit-prism-time-to-usable-work", + "title": "PrismTimeToUsableWorkHigh", + "role": "run", + "expr": "((qbit_prism_stratum_oldest_pending_initial_job_seconds{job=\"qbit-prism\",network=\"__NETWORK__\"} > bool 15) and on(job, instance, network) (qbit_prism_stratum_pending_initial_jobs{job=\"qbit-prism\",network=\"__NETWORK__\"} > 0)) and on(job, instance, network) ((qbit_prism_metrics_snapshot_available{job=\"qbit-prism\",network=\"__NETWORK__\"} == 1) and on(job, instance, network) (qbit_prism_metrics_snapshot_stale{job=\"qbit-prism\",network=\"__NETWORK__\"} == 0))", + "condition_type": "threshold", + "evaluator": "gt", + "threshold": 0, + "for": "3m", + "severity": "warning", + "service": "qbit-mining-pool", + "labels": { + "alert_class": "prism-runtime" + }, + "summary": "PrismTimeToUsableWorkHigh", + "description": "Reports miners awaiting first usable work, including reauthorization; -1 is unknown and snapshot coverage is required. Authorization to successfully written first job; 15 seconds is a provisional bound, measure in #291.", + "no_data_state": "OK", + "exec_err_state": "OK", + "basis": "Authorization to successfully written first job; 15 seconds is a provisional bound, measure in #291.", + "producer_refs": [ + "crates/qbit-prism-server/src/metrics/registry.rs#L61", + "crates/qbit-prism-server/src/metrics/registry.rs#L62", + "crates/qbit-prism-server/src/metrics/registry.rs#L78", + "crates/qbit-prism-server/src/metrics/registry.rs#L79", + "crates/qbit-prism-server/src/metrics.rs#L39" + ] + }, + { + "uid": "qbit-prism-current-work-gap", + "title": "PrismCurrentWorkGapHigh", + "role": "run", + "expr": "((qbit_prism_stratum_current_tip_coverage_gap_seconds{job=\"qbit-prism\",network=\"__NETWORK__\"} > bool 15) and on(job, instance, network) (qbit_prism_authorized_clients{job=\"qbit-prism\",network=\"__NETWORK__\"} > 0)) and on(job, instance, network) ((qbit_prism_metrics_snapshot_available{job=\"qbit-prism\",network=\"__NETWORK__\"} == 1) and on(job, instance, network) (qbit_prism_metrics_snapshot_stale{job=\"qbit-prism\",network=\"__NETWORK__\"} == 0))", + "condition_type": "threshold", + "evaluator": "gt", + "threshold": 0, + "for": "3m", + "severity": "warning", + "service": "qbit-mining-pool", + "labels": { + "alert_class": "prism-runtime" + }, + "summary": "PrismCurrentWorkGapHigh", + "description": "Current-generation usable work remains missing; replaces the operational intent of refresh deadline rules, not Python refresh task state. Native timer advances only while semantic coverage is strictly below 0.95; 15-second duration is provisional, measure in #291.", + "no_data_state": "OK", + "exec_err_state": "OK", + "basis": "Native timer advances only while semantic coverage is strictly below 0.95; 15-second duration is provisional, measure in #291.", + "producer_refs": [ + "crates/qbit-prism-server/src/metrics/registry.rs#L46", + "crates/qbit-prism-server/src/metrics/registry.rs#L63", + "crates/qbit-prism-server/src/metrics/registry.rs#L78", + "crates/qbit-prism-server/src/metrics/registry.rs#L79", + "crates/qbit-prism-server/src/metrics.rs#L39" + ] + }, + { + "uid": "qbit-prism-semantic-work-coverage", + "title": "PrismSemanticWorkCoverageLoss", + "role": "run", + "expr": "((qbit_prism_stratum_semantic_current_work_ratio{job=\"qbit-prism\",network=\"__NETWORK__\"} < bool 0.95) and on(job, instance, network) (qbit_prism_stratum_semantic_current_work_ratio{job=\"qbit-prism\",network=\"__NETWORK__\"} >= 0) and on(job, instance, network) (qbit_prism_authorized_clients{job=\"qbit-prism\",network=\"__NETWORK__\"} > 0)) and on(job, instance, network) ((qbit_prism_metrics_snapshot_available{job=\"qbit-prism\",network=\"__NETWORK__\"} == 1) and on(job, instance, network) (qbit_prism_metrics_snapshot_stale{job=\"qbit-prism\",network=\"__NETWORK__\"} == 0))", + "condition_type": "threshold", + "evaluator": "gt", + "threshold": 0, + "for": "3m", + "severity": "warning", + "service": "qbit-mining-pool", + "labels": { + "alert_class": "prism-runtime" + }, + "summary": "PrismSemanticWorkCoverageLoss", + "description": "Unknown ratio -1 is excluded from coverage and reported separately; no-client ratio one is healthy. Native semantic generation coverage; warning matches the strict 95-percent timer boundary. Critical 50 percent and three-minute dwell: Provisional, measure in #291.", + "no_data_state": "OK", + "exec_err_state": "OK", + "basis": "Native semantic generation coverage; warning matches the strict 95-percent timer boundary. Critical 50 percent and three-minute dwell: Provisional, measure in #291.", + "producer_refs": [ + "crates/qbit-prism-server/src/metrics/registry.rs#L46", + "crates/qbit-prism-server/src/metrics/registry.rs#L64", + "crates/qbit-prism-server/src/metrics/registry.rs#L78", + "crates/qbit-prism-server/src/metrics/registry.rs#L79", + "crates/qbit-prism-server/src/metrics.rs#L39" + ] + }, + { + "uid": "qbit-prism-semantic-coverage-critical", + "title": "PrismSemanticWorkCoverageLossCritical", + "role": "run", + "expr": "((qbit_prism_stratum_semantic_current_work_ratio{job=\"qbit-prism\",network=\"__NETWORK__\"} < bool 0.5) and on(job, instance, network) (qbit_prism_stratum_semantic_current_work_ratio{job=\"qbit-prism\",network=\"__NETWORK__\"} >= 0) and on(job, instance, network) (qbit_prism_authorized_clients{job=\"qbit-prism\",network=\"__NETWORK__\"} > 0)) and on(job, instance, network) ((qbit_prism_metrics_snapshot_available{job=\"qbit-prism\",network=\"__NETWORK__\"} == 1) and on(job, instance, network) (qbit_prism_metrics_snapshot_stale{job=\"qbit-prism\",network=\"__NETWORK__\"} == 0))", + "condition_type": "threshold", + "evaluator": "gt", + "threshold": 0, + "for": "3m", + "severity": "critical", + "service": "qbit-mining-pool", + "labels": { + "alert_class": "prism-runtime", + "page": "true" + }, + "summary": "PrismSemanticWorkCoverageLossCritical", + "description": "Unknown ratio -1 is excluded from coverage and reported separately; no-client ratio one is healthy. Native semantic generation coverage; warning matches the strict 95-percent timer boundary. Critical 50 percent and three-minute dwell: Provisional, measure in #291.", + "no_data_state": "OK", + "exec_err_state": "OK", + "basis": "Native semantic generation coverage; warning matches the strict 95-percent timer boundary. Critical 50 percent and three-minute dwell: Provisional, measure in #291.", + "producer_refs": [ + "crates/qbit-prism-server/src/metrics/registry.rs#L46", + "crates/qbit-prism-server/src/metrics/registry.rs#L64", + "crates/qbit-prism-server/src/metrics/registry.rs#L78", + "crates/qbit-prism-server/src/metrics/registry.rs#L79", + "crates/qbit-prism-server/src/metrics.rs#L39" + ] + }, + { + "uid": "qbit-prism-work-refresh-stalled-critical", + "title": "PrismWorkRefreshStalledCritical", + "role": "run", + "expr": "(qbit_prism_work_refresh_stalled_seconds{job=\"qbit-prism\",network=\"__NETWORK__\"} >= bool 120) and on(job, instance, network) ((qbit_prism_metrics_snapshot_available{job=\"qbit-prism\",network=\"__NETWORK__\"} == 1) and on(job, instance, network) (qbit_prism_metrics_snapshot_stale{job=\"qbit-prism\",network=\"__NETWORK__\"} == 0))", + "condition_type": "threshold", + "evaluator": "gt", + "threshold": 0, + "for": "1m", + "severity": "critical", + "service": "qbit-mining-pool", + "labels": { + "alert_class": "prism-runtime", + "page": "true" + }, + "summary": "PrismWorkRefreshStalledCritical", + "description": "This frontend's template refresh has not succeeded for 120 seconds, so it has neither published nor revalidated work: job, share and block admission refuse once readiness is older than the health timeout, and while refreshes fail only a node poll that finds the published tip with its template within PRISM_TEMPLATE_MAX_AGE_SECONDS (120 s by default) renews it (#622), so by then plus the health timeout every miner on it is stalled whatever the cause; qbit_prism_tip_poll_age_seconds shows which side of that a paged frontend is on. That covers #525 on either settlement path, where a pool without a fee cannot settle sub-floor dust and every build fails with PayoutExceedsCandidateBalance, whether the dust meets a new tip or a same-parent rebuild; it also covers a refresh that fails before its tip observation or hangs. Semantic coverage cannot see this: it compares sessions with the latest published generation, which stays put while every refresh fails. A healthy refresh succeeds every poll interval (2 s by default, and within the health timeout for the frontend to be ready at all); 120 seconds is the default replacement-build lease (PRISM_TEMPLATE_REFRESH_FAILURE_EXIT_SECONDS). One-minute dwell: the gauge is monotonic during a stall. Not gated on connected miners: a frontend without working refreshes is stalled for every miner that connects. A node still syncing or short of peers also pages here, since it builds no work. -1 (not yet published) never fires; no-data and evaluation errors are OK because the snapshot rules own missing measurements, and a draining process stops its snapshot. Provisional, measure in #291.", + "no_data_state": "OK", + "exec_err_state": "OK", + "basis": "This frontend's template refresh has not succeeded for 120 seconds, so it has neither published nor revalidated work: job, share and block admission refuse once readiness is older than the health timeout, and while refreshes fail only a node poll that finds the published tip with its template within PRISM_TEMPLATE_MAX_AGE_SECONDS (120 s by default) renews it (#622), so by then plus the health timeout every miner on it is stalled whatever the cause; qbit_prism_tip_poll_age_seconds shows which side of that a paged frontend is on. That covers #525 on either settlement path, where a pool without a fee cannot settle sub-floor dust and every build fails with PayoutExceedsCandidateBalance, whether the dust meets a new tip or a same-parent rebuild; it also covers a refresh that fails before its tip observation or hangs. Semantic coverage cannot see this: it compares sessions with the latest published generation, which stays put while every refresh fails. A healthy refresh succeeds every poll interval (2 s by default, and within the health timeout for the frontend to be ready at all); 120 seconds is the default replacement-build lease (PRISM_TEMPLATE_REFRESH_FAILURE_EXIT_SECONDS). One-minute dwell: the gauge is monotonic during a stall. Not gated on connected miners: a frontend without working refreshes is stalled for every miner that connects. A node still syncing or short of peers also pages here, since it builds no work. -1 (not yet published) never fires; no-data and evaluation errors are OK because the snapshot rules own missing measurements, and a draining process stops its snapshot. Provisional, measure in #291.", + "producer_refs": [ + "crates/qbit-prism-server/src/metrics/registry.rs", + "crates/qbit-prism-server/src/coordinator.rs", + "crates/qbit-prism-server/src/server.rs" + ] + }, + { + "uid": "qbit-prism-work-metrics-unavailable", + "title": "PrismWorkMetricsUnavailable", + "role": "run", + "expr": "((qbit_prism_stratum_semantic_current_work_ratio{job=\"qbit-prism\",network=\"__NETWORK__\"} < bool 0) + on(job, instance, network) (qbit_prism_stratum_oldest_pending_initial_job_seconds{job=\"qbit-prism\",network=\"__NETWORK__\"} < bool 0)) and on(job, instance, network) ((qbit_prism_metrics_snapshot_available{job=\"qbit-prism\",network=\"__NETWORK__\"} == 1) and on(job, instance, network) (qbit_prism_metrics_snapshot_stale{job=\"qbit-prism\",network=\"__NETWORK__\"} == 0))", + "condition_type": "threshold", + "evaluator": "gt", + "threshold": 0, + "for": "3m", + "severity": "warning", + "service": "qbit-mining-pool", + "labels": { + "alert_class": "prism-runtime" + }, + "summary": "PrismWorkMetricsUnavailable", + "description": "A published snapshot has not observed work delivery; inspect the publisher and delivery sampler. Unknown sentinels must never masquerade as healthy work coverage.", + "no_data_state": "OK", + "exec_err_state": "OK", + "basis": "Unknown sentinels must never masquerade as healthy work coverage.", + "producer_refs": [ + "crates/qbit-prism-server/src/metrics/registry.rs#L62", + "crates/qbit-prism-server/src/metrics/registry.rs#L64", + "crates/qbit-prism-server/src/metrics/registry.rs#L78", + "crates/qbit-prism-server/src/metrics/registry.rs#L79", + "crates/qbit-prism-server/src/metrics.rs#L39" + ] + }, + { + "uid": "qbit-prism-block-candidate-backlog", + "title": "PrismBlockCandidateBacklog", + "role": "run", + "expr": "(qbit_prism_block_candidates_pending{job=\"qbit-prism\",network=\"__NETWORK__\"} >= bool 2) and on(job, instance, network) ((qbit_prism_metrics_snapshot_available{job=\"qbit-prism\",network=\"__NETWORK__\"} == 1) and on(job, instance, network) (qbit_prism_metrics_snapshot_stale{job=\"qbit-prism\",network=\"__NETWORK__\"} == 0)) and on(job, instance, network) (qbit_prism_collector_available{job=\"qbit-prism\",network=\"__NETWORK__\",collector=\"database\"} == 1)", + "condition_type": "threshold", + "evaluator": "gt", + "threshold": 0, + "for": "3m", + "severity": "warning", + "service": "qbit-mining-pool", + "labels": { + "alert_class": "prism-runtime" + }, + "summary": "PrismBlockCandidateBacklog", + "description": "Per-scrape-target view of cluster-wide backlog; do not sum repeated database snapshots across coordinators. Unknown -1 belongs to the collector-unavailable alert. Native pending outbox rows include claimed/retry-delayed candidates; count bounds 2/5 and age bounds 15/60 seconds. Provisional, measure in #291.", + "no_data_state": "OK", + "exec_err_state": "OK", + "basis": "Native pending outbox rows include claimed/retry-delayed candidates; count bounds 2/5 and age bounds 15/60 seconds. Provisional, measure in #291.", + "producer_refs": [ + "crates/qbit-prism-server/src/metrics/registry.rs#L66", + "crates/qbit-prism-server/src/metrics/registry.rs#L70", + "crates/qbit-prism-server/src/metrics/registry.rs#L78", + "crates/qbit-prism-server/src/metrics/registry.rs#L79", + "crates/qbit-prism-server/src/metrics/labels.rs#L18", + "crates/qbit-prism-server/src/metrics.rs#L39" + ] + }, + { + "uid": "qbit-prism-candidate-backlog-critical", + "title": "PrismBlockCandidateBacklogCritical", + "role": "run", + "expr": "(qbit_prism_block_candidates_pending{job=\"qbit-prism\",network=\"__NETWORK__\"} >= bool 5) and on(job, instance, network) ((qbit_prism_metrics_snapshot_available{job=\"qbit-prism\",network=\"__NETWORK__\"} == 1) and on(job, instance, network) (qbit_prism_metrics_snapshot_stale{job=\"qbit-prism\",network=\"__NETWORK__\"} == 0)) and on(job, instance, network) (qbit_prism_collector_available{job=\"qbit-prism\",network=\"__NETWORK__\",collector=\"database\"} == 1)", + "condition_type": "threshold", + "evaluator": "gt", + "threshold": 0, + "for": "3m", + "severity": "warning", + "service": "qbit-mining-pool", + "labels": { + "alert_class": "prism-runtime" + }, + "summary": "PrismBlockCandidateBacklogCritical", + "description": "Per-scrape-target view of cluster-wide backlog; do not sum repeated database snapshots across coordinators. Unknown -1 belongs to the collector-unavailable alert. Native pending outbox rows include claimed/retry-delayed candidates; count bounds 2/5 and age bounds 15/60 seconds. Provisional, measure in #291.", + "no_data_state": "OK", + "exec_err_state": "OK", + "basis": "Native pending outbox rows include claimed/retry-delayed candidates; count bounds 2/5 and age bounds 15/60 seconds. Provisional, measure in #291.", + "producer_refs": [ + "crates/qbit-prism-server/src/metrics/registry.rs#L66", + "crates/qbit-prism-server/src/metrics/registry.rs#L70", + "crates/qbit-prism-server/src/metrics/registry.rs#L78", + "crates/qbit-prism-server/src/metrics/registry.rs#L79", + "crates/qbit-prism-server/src/metrics/labels.rs#L18", + "crates/qbit-prism-server/src/metrics.rs#L39" + ] + }, + { + "uid": "qbit-prism-block-candidate-oldest", + "title": "PrismBlockCandidateOldestPending", + "role": "run", + "expr": "(qbit_prism_block_candidate_oldest_pending_seconds{job=\"qbit-prism\",network=\"__NETWORK__\"} >= bool 15) and on(job, instance, network) ((qbit_prism_metrics_snapshot_available{job=\"qbit-prism\",network=\"__NETWORK__\"} == 1) and on(job, instance, network) (qbit_prism_metrics_snapshot_stale{job=\"qbit-prism\",network=\"__NETWORK__\"} == 0)) and on(job, instance, network) (qbit_prism_collector_available{job=\"qbit-prism\",network=\"__NETWORK__\",collector=\"database\"} == 1)", + "condition_type": "threshold", + "evaluator": "gt", + "threshold": 0, + "for": "3m", + "severity": "warning", + "service": "qbit-mining-pool", + "labels": { + "alert_class": "prism-runtime" + }, + "summary": "PrismBlockCandidateOldestPending", + "description": "Per-scrape-target view of cluster-wide backlog; do not sum repeated database snapshots across coordinators. Unknown -1 belongs to the collector-unavailable alert. Native pending outbox rows include claimed/retry-delayed candidates; count bounds 2/5 and age bounds 15/60 seconds. Provisional, measure in #291.", + "no_data_state": "OK", + "exec_err_state": "OK", + "basis": "Native pending outbox rows include claimed/retry-delayed candidates; count bounds 2/5 and age bounds 15/60 seconds. Provisional, measure in #291.", + "producer_refs": [ + "crates/qbit-prism-server/src/metrics/registry.rs#L67", + "crates/qbit-prism-server/src/metrics/registry.rs#L70", + "crates/qbit-prism-server/src/metrics/registry.rs#L78", + "crates/qbit-prism-server/src/metrics/registry.rs#L79", + "crates/qbit-prism-server/src/metrics/labels.rs#L18", + "crates/qbit-prism-server/src/metrics.rs#L39" + ] + }, + { + "uid": "qbit-prism-candidate-oldest-critical", + "title": "PrismBlockCandidateOldestPendingCritical", + "role": "run", + "expr": "((qbit_prism_block_candidate_oldest_unacknowledged_seconds{job=\"qbit-prism\",network=\"__NETWORK__\"} or on(job, instance, network) qbit_prism_block_candidate_oldest_pending_seconds{job=\"qbit-prism\",network=\"__NETWORK__\"}) >= bool 60) and on(job, instance, network) ((qbit_prism_metrics_snapshot_available{job=\"qbit-prism\",network=\"__NETWORK__\"} == 1) and on(job, instance, network) (qbit_prism_metrics_snapshot_stale{job=\"qbit-prism\",network=\"__NETWORK__\"} == 0)) and on(job, instance, network) (qbit_prism_collector_available{job=\"qbit-prism\",network=\"__NETWORK__\",collector=\"database\"} == 1)", + "condition_type": "threshold", + "evaluator": "gt", + "threshold": 0, + "for": "3m", + "severity": "critical", + "service": "qbit-mining-pool", + "labels": { + "alert_class": "prism-runtime", + "page": "true" + }, + "summary": "PrismBlockCandidateOldestPendingCritical", + "description": "Per-scrape-target view of cluster-wide backlog; do not sum repeated database snapshots across coordinators. Unknown -1 belongs to the collector-unavailable alert. Pages only on a found block the node has not accepted (#493): pending and offer-reserved rows, offered rows whose one submitblock outcome is unknown (a transport failure or timeout, or a reservation whose call was lost with its frontend; not a row adopted on the node's own evidence that its block is active), and rows the node definitively rejected unless the reply names a side-chain block (inconclusive, duplicate, duplicate-inconclusive), read from the same database snapshot as the all-unfinished gauges. A rejection such as bad-txnmrklroot, high-hash, bad-cb-* or duplicate-invalid means the pool offered an invalid block, which is the emergency this rule exists for, and it keeps paging exactly as before. A node-accepted lost race sits in reconciliation until a different block has PRISM_CANDIDATE_ORPHAN_CONFIRMATIONS confirmations at its height; that is expected and never pages here. An unknown outcome keeps paging because receipt is unproven: the block may never have reached the node and is never offered again, so it is indistinguishable from an unoffered block until the chain proves an orphan or a later observation lands it. A landing that keeps failing after acceptance is owned by the landing-failed rules; every other unfinished row by the stuck warning. Mixed-version rollout: on a target that does not export the unacknowledged series yet, the expression falls back to the all-unfinished age, so the base paging behaviour (including its lost-race page) persists there until that frontend is upgraded; deploy frontends first. Native pending outbox rows include claimed/retry-delayed candidates; count bounds 2/5 and age bounds 15/60 seconds. Provisional, measure in #291.", + "no_data_state": "OK", + "exec_err_state": "OK", + "basis": "Pages only on a found block the node has not accepted (#493): pending and offer-reserved rows, offered rows whose one submitblock outcome is unknown (a transport failure or timeout, or a reservation whose call was lost with its frontend; not a row adopted on the node's own evidence that its block is active), and rows the node definitively rejected unless the reply names a side-chain block (inconclusive, duplicate, duplicate-inconclusive), read from the same database snapshot as the all-unfinished gauges. A rejection such as bad-txnmrklroot, high-hash, bad-cb-* or duplicate-invalid means the pool offered an invalid block, which is the emergency this rule exists for, and it keeps paging exactly as before. A node-accepted lost race sits in reconciliation until a different block has PRISM_CANDIDATE_ORPHAN_CONFIRMATIONS confirmations at its height; that is expected and never pages here. An unknown outcome keeps paging because receipt is unproven: the block may never have reached the node and is never offered again, so it is indistinguishable from an unoffered block until the chain proves an orphan or a later observation lands it. A landing that keeps failing after acceptance is owned by the landing-failed rules; every other unfinished row by the stuck warning. Mixed-version rollout: on a target that does not export the unacknowledged series yet, the expression falls back to the all-unfinished age, so the base paging behaviour (including its lost-race page) persists there until that frontend is upgraded; deploy frontends first. Native pending outbox rows include claimed/retry-delayed candidates; count bounds 2/5 and age bounds 15/60 seconds. Provisional, measure in #291.", + "producer_refs": [ + "crates/qbit-prism-server/src/metrics/registry.rs#L67", + "crates/qbit-prism-server/src/metrics/registry.rs#L70", + "crates/qbit-prism-server/src/metrics/registry.rs#L78", + "crates/qbit-prism-server/src/metrics/registry.rs#L79", + "crates/qbit-prism-server/src/metrics/labels.rs#L18", + "crates/qbit-prism-server/src/metrics.rs#L39" + ] + }, + { + "uid": "qbit-prism-block-candidate-stuck", + "title": "PrismBlockCandidateStuck", + "role": "run", + "expr": "(qbit_prism_block_candidate_oldest_pending_seconds{job=\"qbit-prism\",network=\"__NETWORK__\"} >= bool 1800) and on(job, instance, network) ((qbit_prism_metrics_snapshot_available{job=\"qbit-prism\",network=\"__NETWORK__\"} == 1) and on(job, instance, network) (qbit_prism_metrics_snapshot_stale{job=\"qbit-prism\",network=\"__NETWORK__\"} == 0)) and on(job, instance, network) (qbit_prism_collector_available{job=\"qbit-prism\",network=\"__NETWORK__\",collector=\"database\"} == 1)", + "condition_type": "threshold", + "evaluator": "gt", + "threshold": 0, + "for": "3m", + "severity": "warning", + "service": "qbit-mining-pool", + "labels": { + "alert_class": "prism-runtime" + }, + "summary": "PrismBlockCandidateStuck", + "description": "Per-scrape-target view of cluster-wide backlog; do not sum repeated database snapshots across coordinators. Unknown -1 belongs to the collector-unavailable alert. An unfinished candidate row older than 30 minutes (#493), read from the all-unfinished gauge: an offered row in reconciliation whose chain verdict is not arriving, a node-rejected block, or a landing that keeps failing after acceptance. A lost tip race is proven orphan within a few block intervals (six competitor confirmations at the 60-second target spacing plus the reconciliation retry backoff) and leaves this gauge well inside the threshold, aligned with PrismAcceptedBlockUnlanded. The reconciliation backoff is ten seconds times the row's attempt count, capped at an hour, and the count includes the pre-offer attempts, so the cumulative wait to the k-th retry is about 5k(k+1) seconds: the sixth competitor confirmation at 60-second spacing is reached around the eighth retry, about nine minutes, well inside 30 minutes, while a row that needed many pre-offer attempts starts its backoff higher and reaches 30 minutes in a few retries. Pre-offer, unknown-outcome and rejected rows already page at 60 seconds through PrismBlockCandidateOldestPendingCritical and a failing landing through the landing-failed rules; this warning is the non-paging catch-all for the rest, which is operator work (the row's last_error names the reason). Three-minute dwell like the other candidate warnings. Provisional, measure in #291.", + "no_data_state": "OK", + "exec_err_state": "OK", + "basis": "An unfinished candidate row older than 30 minutes (#493), read from the all-unfinished gauge: an offered row in reconciliation whose chain verdict is not arriving, a node-rejected block, or a landing that keeps failing after acceptance. A lost tip race is proven orphan within a few block intervals (six competitor confirmations at the 60-second target spacing plus the reconciliation retry backoff) and leaves this gauge well inside the threshold, aligned with PrismAcceptedBlockUnlanded. The reconciliation backoff is ten seconds times the row's attempt count, capped at an hour, and the count includes the pre-offer attempts, so the cumulative wait to the k-th retry is about 5k(k+1) seconds: the sixth competitor confirmation at 60-second spacing is reached around the eighth retry, about nine minutes, well inside 30 minutes, while a row that needed many pre-offer attempts starts its backoff higher and reaches 30 minutes in a few retries. Pre-offer, unknown-outcome and rejected rows already page at 60 seconds through PrismBlockCandidateOldestPendingCritical and a failing landing through the landing-failed rules; this warning is the non-paging catch-all for the rest, which is operator work (the row's last_error names the reason). Three-minute dwell like the other candidate warnings. Provisional, measure in #291.", + "producer_refs": [ + "crates/qbit-prism-server/src/metrics/registry.rs#L67", + "crates/qbit-prism-server/src/metrics/registry.rs#L70", + "crates/qbit-prism-server/src/metrics/registry.rs#L78", + "crates/qbit-prism-server/src/metrics/registry.rs#L79", + "crates/qbit-prism-server/src/metrics/labels.rs#L18", + "crates/qbit-prism-server/src/metrics.rs#L39" + ] + }, + { + "uid": "qbit-prism-candidate-landing-failed", + "title": "PrismBlockCandidateLandingFailed", + "role": "run", + "expr": "(qbit_prism_block_candidate_oldest_landing_failed_seconds{job=\"qbit-prism\",network=\"__NETWORK__\"} >= bool 60) and on(job, instance, network) ((qbit_prism_metrics_snapshot_available{job=\"qbit-prism\",network=\"__NETWORK__\"} == 1) and on(job, instance, network) (qbit_prism_metrics_snapshot_stale{job=\"qbit-prism\",network=\"__NETWORK__\"} == 0)) and on(job, instance, network) (qbit_prism_collector_available{job=\"qbit-prism\",network=\"__NETWORK__\",collector=\"database\"} == 1)", + "condition_type": "threshold", + "evaluator": "gt", + "threshold": 0, + "for": "3m", + "severity": "warning", + "service": "qbit-mining-pool", + "labels": { + "alert_class": "prism-runtime" + }, + "summary": "PrismBlockCandidateLandingFailed", + "description": "Per-scrape-target view of cluster-wide backlog; do not sum repeated database snapshots across coordinators. Unknown -1 belongs to the collector-unavailable alert. An offered or reconciliation row whose audit landing has not succeeded after the offer (#493): the landing transaction is the only writer of the block's pool-block row, so its absence is the durable fact the collector reads, and a last_error naming a landing refusal (a prefix the coordinator and the collector share) covers a landed row that no longer authenticates; a retry that fails for a transient node or database reason, or a refused operator recovery, overwrites last_error but cannot make this age read zero while nothing has landed. The node accepted or rejected the block, the audit rebuild, its verification or the ledger landing refused, and the row is retried at the reconciliation backoff with read-only chain observations only. The age is measured from the offer reservation, the start of the post-offer lifecycle and the same instant the frontend's acceptance clock starts, so pre-offer retry backoff never counts toward it. An offered row counts before its first landing outcome (#510): a won block whose landing hangs in offered, waiting for build capacity with no deadline while its heartbeat keeps the claim, ages here instead of paging nowhere; a healthy offered row lands within the 60-second rebuild deadline, so it never reaches the three-minute warning dwell. Never a lost race: a lost race and a rejection land their audit before the chain verdict, so their pool-block row keeps this age at zero once landed. What is at risk is the block's audit, its pool-block row, the deferred solver credit and the next payout revision, which the successful landing writes; the block reward is on the chain regardless. Warning at 60 seconds for three minutes: a transient refusal (a rebuild deadline under a brief database stall) lands on the first retry ten seconds later and never reaches the dwell; the same shape as the pre-#493 candidate critical, without the page. Provisional, measure in #291.", + "no_data_state": "OK", + "exec_err_state": "OK", + "basis": "An offered or reconciliation row whose audit landing has not succeeded after the offer (#493): the landing transaction is the only writer of the block's pool-block row, so its absence is the durable fact the collector reads, and a last_error naming a landing refusal (a prefix the coordinator and the collector share) covers a landed row that no longer authenticates; a retry that fails for a transient node or database reason, or a refused operator recovery, overwrites last_error but cannot make this age read zero while nothing has landed. The node accepted or rejected the block, the audit rebuild, its verification or the ledger landing refused, and the row is retried at the reconciliation backoff with read-only chain observations only. The age is measured from the offer reservation, the start of the post-offer lifecycle and the same instant the frontend's acceptance clock starts, so pre-offer retry backoff never counts toward it. An offered row counts before its first landing outcome (#510): a won block whose landing hangs in offered, waiting for build capacity with no deadline while its heartbeat keeps the claim, ages here instead of paging nowhere; a healthy offered row lands within the 60-second rebuild deadline, so it never reaches the three-minute warning dwell. Never a lost race: a lost race and a rejection land their audit before the chain verdict, so their pool-block row keeps this age at zero once landed. What is at risk is the block's audit, its pool-block row, the deferred solver credit and the next payout revision, which the successful landing writes; the block reward is on the chain regardless. Warning at 60 seconds for three minutes: a transient refusal (a rebuild deadline under a brief database stall) lands on the first retry ten seconds later and never reaches the dwell; the same shape as the pre-#493 candidate critical, without the page. Provisional, measure in #291.", + "producer_refs": [ + "crates/qbit-prism-server/src/metrics/registry.rs#L67", + "crates/qbit-prism-server/src/metrics/registry.rs#L70", + "crates/qbit-prism-server/src/metrics/registry.rs#L78", + "crates/qbit-prism-server/src/metrics/registry.rs#L79", + "crates/qbit-prism-server/src/metrics/labels.rs#L18", + "crates/qbit-prism-server/src/metrics.rs#L39" + ] + }, + { + "uid": "qbit-prism-candidate-landing-failed-critical", + "title": "PrismBlockCandidateLandingFailedCritical", + "role": "run", + "expr": "(qbit_prism_block_candidate_oldest_landing_failed_seconds{job=\"qbit-prism\",network=\"__NETWORK__\"} >= bool 307) and on(job, instance, network) ((qbit_prism_metrics_snapshot_available{job=\"qbit-prism\",network=\"__NETWORK__\"} == 1) and on(job, instance, network) (qbit_prism_metrics_snapshot_stale{job=\"qbit-prism\",network=\"__NETWORK__\"} == 0)) and on(job, instance, network) (qbit_prism_collector_available{job=\"qbit-prism\",network=\"__NETWORK__\",collector=\"database\"} == 1)", + "condition_type": "threshold", + "evaluator": "gt", + "threshold": 0, + "for": "1m", + "severity": "critical", + "service": "qbit-mining-pool", + "labels": { + "alert_class": "prism-runtime", + "page": "true" + }, + "summary": "PrismBlockCandidateLandingFailedCritical", + "description": "Per-scrape-target view of cluster-wide backlog; do not sum repeated database snapshots across coordinators. Unknown -1 belongs to the collector-unavailable alert. An offered or reconciliation row whose audit landing has not succeeded after the offer (#493): the landing transaction is the only writer of the block's pool-block row, so its absence is the durable fact the collector reads, and a last_error naming a landing refusal (a prefix the coordinator and the collector share) covers a landed row that no longer authenticates; a retry that fails for a transient node or database reason, or a refused operator recovery, overwrites last_error but cannot make this age read zero while nothing has landed. The node accepted or rejected the block, the audit rebuild, its verification or the ledger landing refused, and the row is retried at the reconciliation backoff with read-only chain observations only. The age is measured from the offer reservation, the start of the post-offer lifecycle and the same instant the frontend's acceptance clock starts, so pre-offer retry backoff never counts toward it. An offered row counts before its first landing outcome (#510): a won block whose landing hangs in offered, waiting for build capacity with no deadline while its heartbeat keeps the claim, ages here instead of paging nowhere; a healthy offered row lands within the 60-second rebuild deadline, so it never reaches the three-minute warning dwell. Never a lost race: a lost race and a rejection land their audit before the chain verdict, so their pool-block row keeps this age at zero once landed. What is at risk is the block's audit, its pool-block row, the deferred solver credit and the next payout revision, which the successful landing writes; the block reward is on the chain regardless. Pages at 307 seconds for one minute, the same floor and dwell as PrismAcceptedRevisionWorkPendingCritical: a landing that fails after the frontend's own active-chain observation already pages that rule at 307 s (the landed identity is a known wait with no revision to deliver), and a landing that fails before it now pages here at the same instant, so a won block whose ledger landing keeps failing is never silent whichever step refused. By 307 seconds the row has been retried about eight times at the reconciliation backoff (ten seconds times the attempt count), so the failure is persistent, never a single deadline. Gated like the other candidate rules on a fresh snapshot and an available collector; no-data and evaluation errors are OK because the unavailable warning owns the missing measurement. Provisional, measure in #291.", + "no_data_state": "OK", + "exec_err_state": "OK", + "basis": "An offered or reconciliation row whose audit landing has not succeeded after the offer (#493): the landing transaction is the only writer of the block's pool-block row, so its absence is the durable fact the collector reads, and a last_error naming a landing refusal (a prefix the coordinator and the collector share) covers a landed row that no longer authenticates; a retry that fails for a transient node or database reason, or a refused operator recovery, overwrites last_error but cannot make this age read zero while nothing has landed. The node accepted or rejected the block, the audit rebuild, its verification or the ledger landing refused, and the row is retried at the reconciliation backoff with read-only chain observations only. The age is measured from the offer reservation, the start of the post-offer lifecycle and the same instant the frontend's acceptance clock starts, so pre-offer retry backoff never counts toward it. An offered row counts before its first landing outcome (#510): a won block whose landing hangs in offered, waiting for build capacity with no deadline while its heartbeat keeps the claim, ages here instead of paging nowhere; a healthy offered row lands within the 60-second rebuild deadline, so it never reaches the three-minute warning dwell. Never a lost race: a lost race and a rejection land their audit before the chain verdict, so their pool-block row keeps this age at zero once landed. What is at risk is the block's audit, its pool-block row, the deferred solver credit and the next payout revision, which the successful landing writes; the block reward is on the chain regardless. Pages at 307 seconds for one minute, the same floor and dwell as PrismAcceptedRevisionWorkPendingCritical: a landing that fails after the frontend's own active-chain observation already pages that rule at 307 s (the landed identity is a known wait with no revision to deliver), and a landing that fails before it now pages here at the same instant, so a won block whose ledger landing keeps failing is never silent whichever step refused. By 307 seconds the row has been retried about eight times at the reconciliation backoff (ten seconds times the attempt count), so the failure is persistent, never a single deadline. Gated like the other candidate rules on a fresh snapshot and an available collector; no-data and evaluation errors are OK because the unavailable warning owns the missing measurement. Provisional, measure in #291.", + "producer_refs": [ + "crates/qbit-prism-server/src/metrics/registry.rs#L67", + "crates/qbit-prism-server/src/metrics/registry.rs#L70", + "crates/qbit-prism-server/src/metrics/registry.rs#L78", + "crates/qbit-prism-server/src/metrics/registry.rs#L79", + "crates/qbit-prism-server/src/metrics/labels.rs#L18", + "crates/qbit-prism-server/src/metrics.rs#L39" + ] + }, + { + "uid": "qbit-prism-candidate-metrics-unavailable", + "title": "PrismBlockCandidateMetricsUnavailable", + "role": "run", + "expr": "(qbit_prism_collector_available{job=\"qbit-prism\",network=\"__NETWORK__\",collector=\"database\"} == bool 0) or on(job, instance, network) (qbit_prism_block_candidates_pending{job=\"qbit-prism\",network=\"__NETWORK__\"} < bool 0) or on(job, instance, network) (qbit_prism_block_candidate_oldest_pending_seconds{job=\"qbit-prism\",network=\"__NETWORK__\"} < bool 0) or on(job, instance, network) (qbit_prism_block_candidate_oldest_unacknowledged_seconds{job=\"qbit-prism\",network=\"__NETWORK__\"} < bool 0) or on(job, instance, network) (qbit_prism_block_candidate_oldest_landing_failed_seconds{job=\"qbit-prism\",network=\"__NETWORK__\"} < bool 0)", + "condition_type": "threshold", + "evaluator": "gt", + "threshold": 0, + "for": "3m", + "severity": "warning", + "service": "qbit-mining-pool", + "labels": { + "alert_class": "prism-runtime" + }, + "summary": "PrismBlockCandidateMetricsUnavailable", + "description": "Database candidate age/count are unknown, not zero; collector state is live at scrape time. Collector failure, cancellation or expiry after 30 seconds invalidates all four gauges, the unacknowledged and landing-failed ages included (#493).", + "no_data_state": "Alerting", + "exec_err_state": "Alerting", + "basis": "Collector failure, cancellation or expiry after 30 seconds invalidates all four gauges, the unacknowledged and landing-failed ages included (#493).", + "producer_refs": [ + "crates/qbit-prism-server/src/metrics/registry.rs#L66", + "crates/qbit-prism-server/src/metrics/registry.rs#L67", + "crates/qbit-prism-server/src/metrics/registry.rs#L70", + "crates/qbit-prism-server/src/metrics/labels.rs#L18", + "crates/qbit-prism-server/src/metrics.rs#L39" + ] + }, + { + "uid": "qbit-prism-block-submission-held", + "title": "PrismBlockSubmissionHeld", + "role": "run", + "expr": "(last_over_time(qbit_prism_block_submission_enabled{job=\"qbit-prism\",network=\"__NETWORK__\"}[2m]) != bool 1) and on(job, instance, network) (max_over_time(up{job=\"qbit-prism\",network=\"__NETWORK__\"}[2m]) == 1)", + "condition_type": "threshold", + "evaluator": "gt", + "threshold": 0, + "for": "1m", + "severity": "critical", + "service": "qbit-mining-pool", + "labels": { + "alert_class": "prism-runtime", + "page": "true" + }, + "summary": "PrismBlockSubmissionHeld", + "description": "This frontend holds block submission: its PRISM_BLOCK_SUBMIT_ENABLED is off (#661, the #291 rehearsal kill switch), or the cluster's block submission hold is set in the ledger (#664, qbit-prism-server submission-hold set), so it offers no block it finds to its node's submitblock and sends no CTV fanout. Its found blocks wait in the shared candidate outbox: a peer frontend that still submits offers them, and when every frontend is held, as under the cluster hold, none is offered. This rule pages a minute after a frontend is held, independently of whether it has found a block; the candidate rules page only once a found block has waited (PrismBlockCandidateOldestPendingCritical: an age of 60 seconds, then a three-minute dwell). A frontend that cannot yet show that it submits pages too (-1): its switch is on but it has not read the cluster's hold yet. A later failed read keeps the last one, so a database outage does not page here. The gauge is rendered at scrape time from the frontend's last health publication, so the rule reads its last sample in the past two minutes and is gated on a successful scrape in that window rather than on a fresh snapshot: a held frontend keeps paging through a stale snapshot or an occasional failed scrape. A missing endpoint, no data and evaluation errors are OK, because PrismMetricsEndpointDown and PrismMetricsNoData own them, and an image older than #661 exports no series. A rehearsal against the production network holds its frontends on purpose: scrape them under another job or network, or silence this rule with matchers on the rehearsal frontends' instance labels, never on the alert name alone, which would also hide a production frontend left held (docs/prism-ledger-ops.md, block submission kill switch for rehearsals). Remedy: when qbit-prism-server submission-hold show reports \"held\": true, clear the hold with submission-hold clear --reason once the cutover is verified (docs/prism-ledger-ops.md, holding the whole cluster); otherwise unset PRISM_BLOCK_SUBMIT_ENABLED or set it to 1, restart the frontend, and confirm check-config prints no block submission WARNING. #666, #664.", + "no_data_state": "OK", + "exec_err_state": "OK", + "basis": "qbit_prism_block_submission_enabled is 1 while a frontend offers found blocks, 0 while PRISM_BLOCK_SUBMIT_ENABLED or the cluster's hold holds them, and -1 while the switch is on but the frontend has not read the hold (#661, #664); a production frontend must never hold (docs/mainnet-deployment.md go-live checks). The switch is fixed at process start and the hold changes only by an operator command; -1 shows only until a frontend first reads the hold, which the one-minute dwell covers after a restart. A later failed read keeps the last value read, so a database outage does not page here, and the two-minute windows ride over a failed scrape, which would otherwise restart the dwell; #666.", + "producer_refs": [ + "crates/qbit-prism-server/src/metrics/registry.rs#L80", + "crates/qbit-prism-server/src/metrics.rs#L146", + "crates/qbit-prism-server/src/metrics/snapshots.rs#L126", + "crates/qbit-prism-server/src/server.rs#L55", + "crates/qbit-prism-server/src/server.rs#L392", + "crates/qbit-prism-server/src/ledger/submission_hold.rs#L127", + "crates/qbit-prism-server/src/config.rs#L635" + ] + }, + { + "uid": "qbit-prism-database-unavailable", + "title": "PrismDatabaseUnavailable", + "role": "run", + "expr": "(qbit_prism_collector_available{job=\"qbit-prism\",network=\"__NETWORK__\",collector=\"database\"} == bool 0) and on(job, instance, network) (up{job=\"qbit-prism\",network=\"__NETWORK__\"} == 1)", + "condition_type": "threshold", + "evaluator": "gt", + "threshold": 0, + "for": "2m", + "severity": "critical", + "service": "qbit-mining-pool", + "labels": { + "alert_class": "prism-runtime", + "page": "true" + }, + "summary": "PrismDatabaseUnavailable", + "description": "This frontend has had no complete successful observation of its ledger database for two minutes: the database collector, which reads PostgreSQL every 10 seconds through this frontend's own connection pool, in a read-only transaction with a 3-second deadline that includes the pool acquire and a 500 ms lock timeout, failed, was cancelled, or last succeeded more than 30 seconds ago. That covers PostgreSQL down or unreachable, and also a pool saturated, or a lock blocking the collector's reads, for two minutes. While it holds, the frontend refuses the shares and sessions it cannot record, mostly as backend-database-unavailable or ledger-confirmation-failed (#581), so every miner on it is stalled. Pages on exactly this state and says so: before #581 a PostgreSQL outage held only PrismBlockCandidateMetricsUnavailable, a warning titled for block-candidate metrics. Not this rule's: a database that still serves reads but refuses writes (a read-only standby or demoted primary, or a cluster halted by its fatal state) and a stall shorter than two minutes; both show as backend-database-unavailable refusals in PrismRejectRatioByReasonHigh. Collector state is live at scrape time and not gated on a fresh metrics snapshot, whose freshness flaps while the health publisher waits on the database. Gated on a successful scrape; a missing endpoint, no data and evaluation errors are OK because PrismMetricsEndpointDown and PrismMetricsNoData own them. Provisional, measure in #291.", + "no_data_state": "OK", + "exec_err_state": "OK", + "basis": "Database collector unavailable (failure, cancellation, or no success for COLLECTOR_STALE_AFTER = 30 s, collected every 10 s) for two minutes, from #581's disk-exhaustion run, where the collector held unavailable through a 200 s outage while every snapshot-gated rule flapped; provisional, measure in #291.", + "producer_refs": [ + "crates/qbit-prism-server/src/metrics/registry.rs#L82", + "crates/qbit-prism-server/src/metrics/labels.rs#L71", + "crates/qbit-prism-server/src/metrics.rs#L23", + "crates/qbit-prism-server/src/metrics/collectors.rs#L123" + ] + }, + { + "uid": "qbit-prism-block-offer-standby-unconfirmed", + "title": "PrismBlockOfferStandbyUnconfirmed", + "role": "run", + "expr": "(sum by (job, instance, network) (increase(qbit_prism_block_offer_standby_wait_total{job=\"qbit-prism\",network=\"__NETWORK__\",outcome=~\"absent|lagging|failed\"}[30m])) > bool 0) and on(job, instance, network) ((qbit_prism_metrics_snapshot_available{job=\"qbit-prism\",network=\"__NETWORK__\"} == 1) and on(job, instance, network) (qbit_prism_metrics_snapshot_stale{job=\"qbit-prism\",network=\"__NETWORK__\"} == 0))", + "condition_type": "threshold", + "evaluator": "gt", + "threshold": 0, + "for": "0s", + "severity": "warning", + "service": "qbit-mining-pool", + "labels": { + "alert_class": "prism-runtime" + }, + "summary": "PrismBlockOfferStandbyUnconfirmed", + "description": "A found block was offered in the last 30 minutes without a confirmed copy on the dedicated failover standby (#529). With PRISM_OFFER_STANDBY_APPLICATION_NAME set, each found block's offer first waits, up to PRISM_OFFER_STANDBY_FLUSH_WAIT_MS (default 250 ms), for that standby's flush_lsn to reach the primary's WAL flush position, which covers the block's candidate row and offer reservation. This counts the offers where it did not: absent (no standby connected under that name), lagging (still behind at the bound) or failed (the position could not be read, for example a writer role without pg_monitor). The block was offered anyway, since the wait never holds a block. Until the standby catches up, losing the primary also loses that block's ledger rows, so a block that reached the chain would have no candidate, audit or pool-blocks record on the promoted primary. The frontend log names each such block hash at WARN (block offer standby wait unconfirmed). Check the D3 standby rules (disconnected or lagging) and the writer role's pg_monitor grant; after any failover, reconcile the logged hashes as the ledger operations guide describes. A healthy standby confirms within a few milliseconds, so one unconfirmed offer is already a signal: no dwell, and the 30-minute window keeps it visible. Warning only: no block was held and no share acknowledgement waits on it. The counter only moves when the wait is configured and a block is found; no-data and evaluation errors are OK because the snapshot rules own missing measurements. Provisional, measure in #291.", + "no_data_state": "OK", + "exec_err_state": "OK", + "basis": "A found block was offered in the last 30 minutes without a confirmed copy on the dedicated failover standby (#529). With PRISM_OFFER_STANDBY_APPLICATION_NAME set, each found block's offer first waits, up to PRISM_OFFER_STANDBY_FLUSH_WAIT_MS (default 250 ms), for that standby's flush_lsn to reach the primary's WAL flush position, which covers the block's candidate row and offer reservation. This counts the offers where it did not: absent (no standby connected under that name), lagging (still behind at the bound) or failed (the position could not be read, for example a writer role without pg_monitor). The block was offered anyway, since the wait never holds a block. Until the standby catches up, losing the primary also loses that block's ledger rows, so a block that reached the chain would have no candidate, audit or pool-blocks record on the promoted primary. The frontend log names each such block hash at WARN (block offer standby wait unconfirmed). Check the D3 standby rules (disconnected or lagging) and the writer role's pg_monitor grant; after any failover, reconcile the logged hashes as the ledger operations guide describes. A healthy standby confirms within a few milliseconds, so one unconfirmed offer is already a signal: no dwell, and the 30-minute window keeps it visible. Warning only: no block was held and no share acknowledgement waits on it. The counter only moves when the wait is configured and a block is found; no-data and evaluation errors are OK because the snapshot rules own missing measurements. Provisional, measure in #291.", + "producer_refs": [ + "crates/qbit-prism-server/src/metrics/registry.rs", + "crates/qbit-prism-server/src/coordinator.rs", + "crates/qbit-prism-server/src/ledger/standby_durability.rs" + ] + }, + { + "uid": "qbit-prism-database-pool-wait", + "title": "PrismDatabasePoolWaitHigh", + "role": "run", + "expr": "((histogram_quantile(0.99, sum by (job, instance, network, le) (rate(qbit_prism_database_pool_acquire_seconds_bucket{job=\"qbit-prism\",network=\"__NETWORK__\"}[5m]))) > bool 0.5) and on(job, instance, network) (sum by (job, instance, network) (increase(qbit_prism_database_pool_acquire_seconds_count{job=\"qbit-prism\",network=\"__NETWORK__\"}[5m])) >= 10)) and on(job, instance, network) (up{job=\"qbit-prism\",network=\"__NETWORK__\"} == 1)", + "condition_type": "threshold", + "evaluator": "gt", + "threshold": 0, + "for": "3m", + "severity": "warning", + "service": "qbit-mining-pool", + "labels": { + "alert_class": "prism-runtime" + }, + "summary": "PrismDatabasePoolWaitHigh", + "description": "Actual PgPool::acquire waits from metrics collection, instrumented ledger transactions and covered direct ledger queries, rollup transactions and share-partition maintenance, including share processing and health publication. Each started collector acquisition records one observation: success when acquired, or failure on acquisition error or cancellation, including the three-second collection deadline. Durations measure elapsed pool wait; BEGIN, query execution and advisory-lock waits are outside this timer. This is the aggregate of observed checkout attempts at covered callers. Public-role read pools, private audit HTTP endpoints and shared API read-model helpers, startup/migration gates, operator tools and telemetry-free compatibility wrappers remain excluded; see the acquisition guide for the source census and exclusion policy. Success and failure observations are combined. The histogram is read live at scrape time, so valid waits remain alertable during database collector failure or cached-body publication stalls; a successful scrape is required. Pool p99 >0.5 seconds from at least ten observed acquisition attempts in five minutes; provisional, measure in #291.", + "no_data_state": "OK", + "exec_err_state": "OK", + "basis": "Collector, instrumented ledger transaction and selected direct-query pool-acquire p99 >0.5 seconds, combining success and failure, from at least ten observed acquisition attempts in five minutes; provisional, measure in #291.", + "producer_refs": [ + "crates/qbit-prism-server/src/metrics/registry.rs#L68", + "crates/qbit-prism-server/src/metrics/registry.rs#L70", + "crates/qbit-prism-server/src/metrics/registry.rs#L78", + "crates/qbit-prism-server/src/metrics/registry.rs#L79", + "crates/qbit-prism-server/src/metrics/labels.rs#L16", + "crates/qbit-prism-server/src/metrics/labels.rs#L18", + "crates/qbit-prism-server/src/metrics.rs#L39" + ] + }, + { + "uid": "qbit-prism-database-advisory-lock-wait", + "title": "PrismDatabaseAdvisoryLockWaitHigh", + "role": "run", + "expr": "((histogram_quantile(0.99, sum by (job, instance, network, lock, le) (rate(qbit_prism_database_advisory_lock_wait_seconds_bucket{job=\"qbit-prism\",network=\"__NETWORK__\",lock=~\"order|settlement\"}[5m]))) > bool 1) and on(job, instance, network, lock) (sum by (job, instance, network, lock) (increase(qbit_prism_database_advisory_lock_wait_seconds_count{job=\"qbit-prism\",network=\"__NETWORK__\",lock=~\"order|settlement\"}[5m])) >= 10)) and on(job, instance, network) ((qbit_prism_metrics_snapshot_available{job=\"qbit-prism\",network=\"__NETWORK__\"} == 1) and on(job, instance, network) (qbit_prism_metrics_snapshot_stale{job=\"qbit-prism\",network=\"__NETWORK__\"} == 0))", + "condition_type": "threshold", + "evaluator": "gt", + "threshold": 0, + "for": "3m", + "severity": "warning", + "service": "qbit-mining-pool", + "labels": { + "alert_class": "prism-runtime" + }, + "summary": "PrismDatabaseAdvisoryLockWaitHigh", + "description": "Order and settlement advisory transaction lock waits, combining success and failure. The timer covers the pg_advisory_xact_lock statement alone, including one database round trip; the surrounding transaction work is outside it. lock=\"migration\" is excluded because the migration advisory lock is taken only when a coordinator initializes the schema, where waiting behind another frontend for the whole online 013/016/017 migration window is the designed behaviour rather than a fault. Each lock is judged on its own waits; neither lock supplies the other's sample minimum. This family is rendered from the cached snapshot body and is not read live at scrape time, so an unpublished or stale body suppresses the rule instead of repeating an old percentile. Order and settlement advisory-lock p99 >1 second, combining success and failure, from at least ten observed waits for that lock in five minutes; #271 measured 15, 31 and 63 steady ORDER_LOCK waiters at one, two and four frontends without any native wait percentile, and PRISM_DATABASE_LOCK_TIMEOUT_MS bounds a single wait at five seconds by default; provisional, measure in #291/#447.", + "no_data_state": "OK", + "exec_err_state": "OK", + "basis": "Order and settlement advisory-lock p99 >1 second, combining success and failure, from at least ten observed waits for that lock in five minutes; #271 measured 15, 31 and 63 steady ORDER_LOCK waiters at one, two and four frontends without any native wait percentile, and PRISM_DATABASE_LOCK_TIMEOUT_MS bounds a single wait at five seconds by default; provisional, measure in #291/#447.", + "producer_refs": [ + "crates/qbit-prism-server/src/metrics/registry.rs#L74", + "crates/qbit-prism-server/src/metrics/labels.rs#L66", + "crates/qbit-prism-server/src/metrics/labels.rs#L67", + "crates/qbit-prism-server/src/ledger/connect.rs#L506", + "crates/qbit-prism-server/src/ledger/connect.rs#L549" + ] + }, + { + "uid": "qbit-prism-database-advisory-lock-failures-critical", + "title": "PrismDatabaseAdvisoryLockFailuresCritical", + "role": "run", + "expr": "((sum by (job, instance, network, lock) (increase(qbit_prism_database_advisory_lock_wait_seconds_count{job=\"qbit-prism\",network=\"__NETWORK__\",lock=~\"order|settlement\",result=\"failure\"}[5m])) / sum by (job, instance, network, lock) (increase(qbit_prism_database_advisory_lock_wait_seconds_count{job=\"qbit-prism\",network=\"__NETWORK__\",lock=~\"order|settlement\"}[5m])) > bool 0.05) and on(job, instance, network, lock) (sum by (job, instance, network, lock) (increase(qbit_prism_database_advisory_lock_wait_seconds_count{job=\"qbit-prism\",network=\"__NETWORK__\",lock=~\"order|settlement\"}[5m])) >= 10)) and on(job, instance, network) ((qbit_prism_metrics_snapshot_available{job=\"qbit-prism\",network=\"__NETWORK__\"} == 1) and on(job, instance, network) (qbit_prism_metrics_snapshot_stale{job=\"qbit-prism\",network=\"__NETWORK__\"} == 0))", + "condition_type": "threshold", + "evaluator": "gt", + "threshold": 0, + "for": "3m", + "severity": "critical", + "service": "qbit-mining-pool", + "labels": { + "alert_class": "prism-runtime", + "page": "true" + }, + "summary": "PrismDatabaseAdvisoryLockFailuresCritical", + "description": "Share of order or settlement advisory-lock waits that ended without a held lock. result=\"failure\" covers lock timeout (PRISM_DATABASE_LOCK_TIMEOUT_MS, five seconds by default), statement timeout, deadlock and connection error, and also cancellation of the waiting future: a dropped future such as a cancelled share append records failure from the wait guard rather than vanishing. The failure series does not exist until the first failure, so on a healthy coordinator the ratio is absent rather than zero and no-data stays OK instead of paging. Each series appears at its first observation, so the first failure after a restart is invisible to increase() and the ratio only rises once a later failure lands in the window. lock=\"migration\" is excluded for the same reason as the wait percentile: that lock is taken only at schema initialization. This family is rendered from the cached snapshot body, so an unpublished or stale body suppresses the rule. Order and settlement advisory-lock failures >5% of that lock's waits over five minutes, from at least ten observed waits for that lock; #271 measured 15, 31 and 63 steady ORDER_LOCK waiters at one, two and four frontends and no native failure rate, and the default five-second PRISM_DATABASE_LOCK_TIMEOUT_MS turns a sustained wait into a failure; provisional, measure in #291/#447.", + "no_data_state": "OK", + "exec_err_state": "OK", + "basis": "Order and settlement advisory-lock failures >5% of that lock's waits over five minutes, from at least ten observed waits for that lock; #271 measured 15, 31 and 63 steady ORDER_LOCK waiters at one, two and four frontends and no native failure rate, and the default five-second PRISM_DATABASE_LOCK_TIMEOUT_MS turns a sustained wait into a failure; provisional, measure in #291/#447.", + "producer_refs": [ + "crates/qbit-prism-server/src/metrics/registry.rs#L74", + "crates/qbit-prism-server/src/metrics/labels.rs#L66", + "crates/qbit-prism-server/src/metrics/labels.rs#L67", + "crates/qbit-prism-server/src/ledger/connect.rs#L506", + "crates/qbit-prism-server/src/ledger/connect.rs#L549" + ] + }, + { + "uid": "qbit-prism-lease-monitor-wake", + "title": "PrismRuntimeLagHigh", + "role": "run", + "expr": "(max_over_time(qbit_prism_runtime_lag_seconds{job=\"qbit-prism\",network=\"__NETWORK__\"}[5m]) > bool 2) + on(job, instance, network) (max by (job, instance, network) (max_over_time(qbit_prism_runtime_poll_lag_seconds{job=\"qbit-prism\",network=\"__NETWORK__\"}[5m])) > bool 2) + on(job, instance, network) (max by (job, instance, network) (qbit_prism_runtime_task_stalled{job=\"qbit-prism\",network=\"__NETWORK__\"}) > bool 0) + on(job, instance, network) (qbit_prism_runtime_lag_seconds{job=\"qbit-prism\",network=\"__NETWORK__\"} < bool 0)", + "condition_type": "threshold", + "evaluator": "gt", + "threshold": 0, + "for": "3m", + "severity": "warning", + "service": "qbit-mining-pool", + "labels": { + "alert_class": "prism-runtime" + }, + "summary": "PrismRuntimeLagHigh", + "description": "Runtime scheduling pressure replaces writer-lease monitor wake delay. Scrape-time runtime signals deliberately remain active during snapshot staleness; retained completed polls capture recovery even when latest wake lag drops within 100ms. Two-second native blocked-poll budget; five-minute evidence window and three-minute dwell replace the legacy lease-wake episode, provisional, measure in #291. A runtime sampler that never observes a tick remains unknown and alerts after the same dwell.", + "no_data_state": "OK", + "exec_err_state": "OK", + "basis": "Two-second native blocked-poll budget; five-minute evidence window and three-minute dwell replace the legacy lease-wake episode, provisional, measure in #291.", + "producer_refs": [ + "crates/qbit-prism-server/src/metrics/registry.rs#L74", + "crates/qbit-prism-server/src/metrics/registry.rs#L75", + "crates/qbit-prism-server/src/metrics/registry.rs#L77", + "crates/qbit-prism-server/src/metrics/labels.rs#L19", + "crates/qbit-prism-server/src/metrics.rs#L39" + ] + }, + { + "uid": "qbit-prism-resident-memory", + "title": "PrismResidentMemoryHigh", + "role": "run", + "expr": "(qbit_prism_process_resident_memory_bytes{job=\"qbit-prism\",network=\"__NETWORK__\"} > bool 4294967296) and on(job, instance, network) ((qbit_prism_metrics_snapshot_available{job=\"qbit-prism\",network=\"__NETWORK__\"} == 1) and on(job, instance, network) (qbit_prism_metrics_snapshot_stale{job=\"qbit-prism\",network=\"__NETWORK__\"} == 0)) and on(job, instance, network) (qbit_prism_collector_available{job=\"qbit-prism\",network=\"__NETWORK__\",collector=\"process\"} == 1)", + "condition_type": "threshold", + "evaluator": "gt", + "threshold": 0, + "for": "3m", + "severity": "warning", + "service": "qbit-mining-pool", + "labels": { + "alert_class": "prism-runtime" + }, + "summary": "PrismResidentMemoryHigh", + "description": "Linux process VmRSS, not heap allocation; a real zero is valid and unsupported/failed/stale collection is -1. Four GiB is an unqualified initial RSS bound; set against actual container headroom. Provisional, measure in #291.", + "no_data_state": "OK", + "exec_err_state": "OK", + "basis": "Four GiB is an unqualified initial RSS bound; set against actual container headroom. Provisional, measure in #291.", + "producer_refs": [ + "crates/qbit-prism-server/src/metrics/registry.rs#L70", + "crates/qbit-prism-server/src/metrics/registry.rs#L73", + "crates/qbit-prism-server/src/metrics/registry.rs#L78", + "crates/qbit-prism-server/src/metrics/registry.rs#L79", + "crates/qbit-prism-server/src/metrics/labels.rs#L18", + "crates/qbit-prism-server/src/metrics.rs#L39" + ] + }, + { + "uid": "qbit-prism-process-metrics-unavailable", + "title": "PrismProcessMetricsUnavailable", + "role": "run", + "expr": "(qbit_prism_collector_available{job=\"qbit-prism\",network=\"__NETWORK__\",collector=\"process\"} == bool 0) or on(job, instance, network) (qbit_prism_process_resident_memory_bytes{job=\"qbit-prism\",network=\"__NETWORK__\"} < bool 0)", + "condition_type": "threshold", + "evaluator": "gt", + "threshold": 0, + "for": "3m", + "severity": "warning", + "service": "qbit-mining-pool", + "labels": { + "alert_class": "prism-runtime" + }, + "summary": "PrismProcessMetricsUnavailable", + "description": "RSS cannot establish memory health until process collection succeeds. Process collector expires after 30 seconds or on failure; unsupported platforms are unknown.", + "no_data_state": "Alerting", + "exec_err_state": "Alerting", + "basis": "Process collector expires after 30 seconds or on failure; unsupported platforms are unknown.", + "producer_refs": [ + "crates/qbit-prism-server/src/metrics/registry.rs#L70", + "crates/qbit-prism-server/src/metrics/registry.rs#L73", + "crates/qbit-prism-server/src/metrics/labels.rs#L18", + "crates/qbit-prism-server/src/metrics.rs#L39" + ] + }, + { + "uid": "qbit-prism-public-staleness-refusal", + "title": "PrismPublicStalenessRefusalsIncreasing", + "role": "public-api", + "expr": "increase(qbit_prism_public_staleness_refusals_total{job=\"qbit-prism-public-read\",network=\"__NETWORK__\"}[5m]) > bool 0", + "condition_type": "threshold", + "evaluator": "gt", + "threshold": 0, + "for": "3m", + "severity": "warning", + "service": "qbit-mining-pool", + "labels": { + "alert_class": "prism-runtime" + }, + "summary": "PrismPublicStalenessRefusalsIncreasing", + "description": "Cached public responses exceeded their route-specific budget. Any refusal is an actual endpoint staleness-budget breach; five-minute window is provisional, measure in #291.", + "no_data_state": "OK", + "exec_err_state": "OK", + "basis": "Any refusal is an actual endpoint staleness-budget breach; five-minute window is provisional, measure in #291.", + "producer_refs": [ + "crates/qbit-prism-server/src/api/public_service.rs#L309", + "crates/qbit-prism-server/src/api/public_service.rs#L320" + ] + }, + { + "uid": "qbit-prism-public-ledger-unready", + "title": "PrismPublicLedgerUnready", + "role": "public-api", + "expr": "(((qbit_prism_public_ledger_ready{job=\"qbit-prism-public-read\",network=\"__NETWORK__\"} == bool 0) or on(job, instance, network) (0 * up{job=\"qbit-prism-public-read\",network=\"__NETWORK__\"} + 1)) + on(job, instance, network) (up{job=\"qbit-prism-public-read\",network=\"__NETWORK__\"} == bool 0)) > bool 0", + "condition_type": "threshold", + "evaluator": "gt", + "threshold": 0, + "for": "3m", + "severity": "warning", + "service": "qbit-mining-pool", + "labels": { + "alert_class": "prism-runtime" + }, + "summary": "PrismPublicLedgerUnready", + "description": "The public read probe is failed, unknown or stale, or its scrape target is down; its freshness owner does not emit the coordinator snapshot gauges. Readiness already rejects missing/failed probes and ages >= max(3 * probe interval, 15s). Missing readiness and failed scrapes on any configured target fire, even while another public instance is healthy.", + "no_data_state": "Alerting", + "exec_err_state": "Alerting", + "basis": "Readiness already rejects missing/failed probes and ages >= max(3 * probe interval, 15s).", + "producer_refs": [ + "crates/qbit-prism-server/src/api/public_service.rs#L309", + "crates/qbit-prism-server/src/api/public_service.rs#L320" + ] + }, + { + "uid": "qbit-prism-public-5xx-sustained", + "title": "PrismPublic5xxSustained", + "role": "public-api", + "expr": "rate(qbit_prism_public_responses_total{job=\"qbit-prism-public-read\",network=\"__NETWORK__\",status=~\"5..\"}[5m]) > bool 0", + "condition_type": "threshold", + "evaluator": "gt", + "threshold": 0, + "for": "3m", + "severity": "warning", + "service": "qbit-mining-pool", + "labels": { + "alert_class": "prism-runtime" + }, + "summary": "PrismPublic5xxSustained", + "description": "This response counter includes health probes; it is an error signal and is not divided by public_requests_total. Any sustained 5xx response, preserving deployed failure intent; five-minute window is provisional, measure in #291.", + "no_data_state": "OK", + "exec_err_state": "OK", + "basis": "Any sustained 5xx response, preserving deployed failure intent; five-minute window is provisional, measure in #291.", + "producer_refs": [ + "crates/qbit-prism-server/src/api/public_service.rs#L309", + "crates/qbit-prism-server/src/api/public_service.rs#L320" + ] + }, + { + "uid": "qbit-prism-revision-work-pending", + "title": "PrismAcceptedRevisionWorkPending", + "role": "run", + "expr": "clamp_max(((qbit_prism_accepted_block_revision_work_pending_seconds{job=\"qbit-prism\",network=\"__NETWORK__\"} > bool 1) or on(job, instance, network) (0 * up{job=\"qbit-prism\",network=\"__NETWORK__\"} + 1)) + on(job, instance, network) (up{job=\"qbit-prism\",network=\"__NETWORK__\"} == bool 0), 1)", + "condition_type": "threshold", + "evaluator": "gt", + "threshold": 0, + "for": "0s", + "severity": "warning", + "service": "qbit-mining-pool", + "labels": { + "alert_class": "prism-runtime" + }, + "summary": "PrismAcceptedRevisionWorkPending", + "description": "Acceptance is observed locally; completion requires an actual successful mining.notify write at the post-landing revision. Provisional warning at more than one second, with no additional dwell: the #275 budget is a target, not a measured production percentile; measure in #291. Missing series on a configured target and failed scrapes also warn. Unknown tracking (a -1 age, the tracking-unknown gauge, or a missing unknown gauge) is owned by PrismAcceptedRevisionWorkTrackingUnknown, which dwells so a single failed refresh does not warn (#493); a known pending age beside unknown tracking still warns here and remains eligible for the critical rule. An offer the node accepted but has not reported active is an unlanded acceptance, excluded from this age and owned by PrismAcceptedBlockUnlanded (#493).", + "no_data_state": "Alerting", + "exec_err_state": "Alerting", + "basis": "Provisional warning at more than one second, with no additional dwell: the #275 budget is a target, not a measured production percentile; measure in #291. Missing series on a configured target and failed scrapes also warn. Unknown tracking (a -1 age, the tracking-unknown gauge, or a missing unknown gauge) is owned by PrismAcceptedRevisionWorkTrackingUnknown, which dwells so a single failed refresh does not warn (#493); a known pending age beside unknown tracking still warns here and remains eligible for the critical rule. An offer the node accepted but has not reported active is an unlanded acceptance, excluded from this age and owned by PrismAcceptedBlockUnlanded (#493).", + "producer_refs": [ + "crates/qbit-prism-server/src/metrics/registry.rs", + "crates/qbit-prism-server/src/metrics/landing.rs", + "crates/qbit-prism-server/src/stratum.rs" + ] + }, + { + "uid": "qbit-prism-revision-work-unknown", + "title": "PrismAcceptedRevisionWorkTrackingUnknown", + "role": "run", + "expr": "clamp_max((qbit_prism_accepted_block_revision_work_pending_seconds{job=\"qbit-prism\",network=\"__NETWORK__\"} < bool 0) + on(job, instance, network) ((qbit_prism_accepted_block_revision_work_tracking_unknown{job=\"qbit-prism\",network=\"__NETWORK__\"} > bool 0) or on(job, instance, network) (0 * up{job=\"qbit-prism\",network=\"__NETWORK__\"} + 1)), 1)", + "condition_type": "threshold", + "evaluator": "gt", + "threshold": 0, + "for": "2m", + "severity": "warning", + "service": "qbit-mining-pool", + "labels": { + "alert_class": "prism-runtime" + }, + "summary": "PrismAcceptedRevisionWorkTrackingUnknown", + "description": "Acceptance is observed locally; completion requires an actual successful mining.notify write at the post-landing revision. Unknown or incomplete local landing tracking (#493 point 7): the pending age is -1 (no known wait remains but tracking is unknown), the tracking-unknown gauge is one beside any known age, or the unknown gauge is missing on a configured target during a mixed-version rollout. Two-minute dwell: a failed template refresh marks tracking unknown only until the next successful refresh, one poll interval later (two seconds by default), so a single transient refresh error cannot span the eight consecutive 15-second scrapes this dwell requires, while a lost or cancelled settlement COMMIT reply, an unsettled orphan verdict, bounded-tracking saturation and lost ordering history persist until terminal evidence or restart and still warn. No-data and evaluation errors alert so a missing measurement stays visible after the same dwell. Non-paging: the critical rule owns known stalls and gates out unknown ages itself; the pending warning owns the no-dwell budget signal. Measure the dwell in #291.", + "no_data_state": "Alerting", + "exec_err_state": "Alerting", + "basis": "Unknown or incomplete local landing tracking (#493 point 7): the pending age is -1 (no known wait remains but tracking is unknown), the tracking-unknown gauge is one beside any known age, or the unknown gauge is missing on a configured target during a mixed-version rollout. Two-minute dwell: a failed template refresh marks tracking unknown only until the next successful refresh, one poll interval later (two seconds by default), so a single transient refresh error cannot span the eight consecutive 15-second scrapes this dwell requires, while a lost or cancelled settlement COMMIT reply, an unsettled orphan verdict, bounded-tracking saturation and lost ordering history persist until terminal evidence or restart and still warn. No-data and evaluation errors alert so a missing measurement stays visible after the same dwell. Non-paging: the critical rule owns known stalls and gates out unknown ages itself; the pending warning owns the no-dwell budget signal. Measure the dwell in #291.", + "producer_refs": [ + "crates/qbit-prism-server/src/metrics/registry.rs", + "crates/qbit-prism-server/src/metrics/landing.rs", + "crates/qbit-prism-server/src/stratum.rs" + ] + }, + { + "uid": "qbit-prism-revision-work-pending-critical", + "title": "PrismAcceptedRevisionWorkPendingCritical", + "role": "run", + "expr": "((qbit_prism_accepted_block_revision_work_pending_seconds{job=\"qbit-prism\",network=\"__NETWORK__\"} >= bool 307) and on(job, instance, network) (qbit_prism_accepted_block_revision_work_pending_seconds{job=\"qbit-prism\",network=\"__NETWORK__\"} >= 0)) and on(job, instance, network) (qbit_prism_authorized_clients{job=\"qbit-prism\",network=\"__NETWORK__\"} > 0) and on(job, instance, network) (up{job=\"qbit-prism\",network=\"__NETWORK__\"} == 1)", + "condition_type": "threshold", + "evaluator": "gt", + "threshold": 0, + "for": "1m", + "severity": "critical", + "service": "qbit-mining-pool", + "labels": { + "alert_class": "prism-runtime", + "page": "true" + }, + "summary": "PrismAcceptedRevisionWorkPendingCritical", + "description": "Acceptance is observed locally; completion requires an actual successful mining.notify write at the post-landing revision. Provisional critical floor of 307 seconds comes from the #413 incident duration, not an early-warning result. Gates (#493): the age must be known (not -1), the target scraped, and at least one authorized miner on the reporting frontend, because a frontend with no connected miners has no mining.notify write that can close its wait and would otherwise page after every pool block. The pending gauge is a live overlay rendered on every scrape, so it is not gated on the metrics snapshot: a database-pool stall that also stalls the health publisher must still page. The miner count is a snapshot family and may be a few refresh intervals old while the snapshot is stale; a false page on a frontend whose miners left inside that window is accepted in exchange for never missing a real stall. If every miner leaves a frontend during a stall, this page resolves and nothing else pages for that frontend: the coverage rules cannot fire with zero authorized clients and the connected-clients rule is a cluster-wide non-paging warning; the pending warning keeps reporting the age until a miner reconnects and receives work. One-minute dwell: the gauge is monotonic during a stall, so the dwell only absorbs a brief gate gap and stays well under the five-minute hold that delayed #413. No-data and evaluation errors are OK: the warning rules own missing or unknown measurements. The floor and dwell are literals in this specification, not deployment variables. A lost race on the offering frontend never counts here (#493 point 1): an offer the node accepted without an active-chain observation is an unlanded acceptance, excluded from the known pending age until an active-chain observation lands it or its orphan is proven, and reported by accepted_block_unlanded_seconds and PrismAcceptedBlockUnlanded instead. Measure thresholds and scrape/evaluation delay in #291; no operational deployment is claimed.", + "no_data_state": "OK", + "exec_err_state": "OK", + "basis": "Provisional critical floor of 307 seconds comes from the #413 incident duration, not an early-warning result. Gates (#493): the age must be known (not -1), the target scraped, and at least one authorized miner on the reporting frontend, because a frontend with no connected miners has no mining.notify write that can close its wait and would otherwise page after every pool block. The pending gauge is a live overlay rendered on every scrape, so it is not gated on the metrics snapshot: a database-pool stall that also stalls the health publisher must still page. The miner count is a snapshot family and may be a few refresh intervals old while the snapshot is stale; a false page on a frontend whose miners left inside that window is accepted in exchange for never missing a real stall. If every miner leaves a frontend during a stall, this page resolves and nothing else pages for that frontend: the coverage rules cannot fire with zero authorized clients and the connected-clients rule is a cluster-wide non-paging warning; the pending warning keeps reporting the age until a miner reconnects and receives work. One-minute dwell: the gauge is monotonic during a stall, so the dwell only absorbs a brief gate gap and stays well under the five-minute hold that delayed #413. No-data and evaluation errors are OK: the warning rules own missing or unknown measurements. The floor and dwell are literals in this specification, not deployment variables. A lost race on the offering frontend never counts here (#493 point 1): an offer the node accepted without an active-chain observation is an unlanded acceptance, excluded from the known pending age until an active-chain observation lands it or its orphan is proven, and reported by accepted_block_unlanded_seconds and PrismAcceptedBlockUnlanded instead. Measure thresholds and scrape/evaluation delay in #291; no operational deployment is claimed.", + "producer_refs": [ + "crates/qbit-prism-server/src/metrics/registry.rs", + "crates/qbit-prism-server/src/metrics/landing.rs", + "crates/qbit-prism-server/src/stratum.rs" + ] + }, + { + "uid": "qbit-prism-revision-work-timeouts", + "title": "PrismRevisionWorkBuildTimeouts", + "role": "run", + "expr": "increase(qbit_prism_revision_work_build_timeouts_total{job=\"qbit-prism\",network=\"__NETWORK__\"}[5m]) > bool 0", + "condition_type": "threshold", + "evaluator": "gt", + "threshold": 0, + "for": "0s", + "severity": "warning", + "service": "qbit-mining-pool", + "labels": { + "alert_class": "prism-runtime" + }, + "summary": "PrismRevisionWorkBuildTimeouts", + "description": "Acceptance is observed locally; completion requires an actual successful mining.notify write at the post-landing revision. Any existing build deadline hit while locally observed accepted work awaits delivery; five-minute increase window is provisional, measure in #291. Successful builds and cancellation do not increment the counter. Only a known open wait counts (#493): while this frontend's tracking is unknown and no known wait remains, a deadline hit is an ordinary job-delivery failure, not a revision-work failure, and the tracking-unknown warning reports the unknown state instead.", + "no_data_state": "Alerting", + "exec_err_state": "Alerting", + "basis": "Any existing build deadline hit while locally observed accepted work awaits delivery; five-minute increase window is provisional, measure in #291. Successful builds and cancellation do not increment the counter. Only a known open wait counts (#493): while this frontend's tracking is unknown and no known wait remains, a deadline hit is an ordinary job-delivery failure, not a revision-work failure, and the tracking-unknown warning reports the unknown state instead.", + "producer_refs": [ + "crates/qbit-prism-server/src/metrics/registry.rs", + "crates/qbit-prism-server/src/metrics/landing.rs", + "crates/qbit-prism-server/src/stratum.rs" + ] + }, + { + "uid": "qbit-prism-accepted-block-unlanded", + "title": "PrismAcceptedBlockUnlanded", + "role": "run", + "expr": "(qbit_prism_accepted_block_unlanded_seconds{job=\"qbit-prism\",network=\"__NETWORK__\"} >= bool 1800)", + "condition_type": "threshold", + "evaluator": "gt", + "threshold": 0, + "for": "3m", + "severity": "warning", + "service": "qbit-mining-pool", + "labels": { + "alert_class": "prism-runtime" + }, + "summary": "PrismAcceptedBlockUnlanded", + "description": "Acceptance is observed locally; an unlanded acceptance has no revision work to deliver yet. An offer the node accepted (a null submitblock reply) that this frontend has not observed on the node's active chain for 30 minutes (#493 point 1). A lost tip race is settled as a proven orphan once a different block has PRISM_CANDIDATE_ORPHAN_CONFIRMATIONS (default 6) confirmations at its height: about six minutes at the 60-second target spacing plus the reconciliation retry backoff, so a routine lost race leaves this gauge well inside the threshold. The reconciliation backoff is ten seconds times the row's attempt count, capped at an hour, and the count includes the pre-offer attempts, so the cumulative wait to the k-th retry is about 5k(k+1) seconds: the sixth competitor confirmation at 60-second spacing is reached around the eighth retry, about nine minutes, well inside 30 minutes, while a row that needed many pre-offer attempts starts its backoff higher and reaches 30 minutes in a few retries. Such an acceptance is never part of the known pending age, so the revision-work critical cannot page on it; a winning offer leaves this gauge at its first active-chain observation and joins the known age on its original acceptance clock. Past 30 minutes the row is not being reconciled, the node is not advancing, or the orphan verdict keeps failing: operator work, warned, never paged; the cluster-wide candidate oldest-pending rules see the same reconciliation row. -1 (none tracked while identity tracking is saturated) is owned by the tracking-unknown warning and does not fire here. Three-minute dwell like the other warnings; no-data is OK because the pending warning already reports a missing landing family. Provisional; measure in #291.", + "no_data_state": "OK", + "exec_err_state": "OK", + "basis": "An offer the node accepted (a null submitblock reply) that this frontend has not observed on the node's active chain for 30 minutes (#493 point 1). A lost tip race is settled as a proven orphan once a different block has PRISM_CANDIDATE_ORPHAN_CONFIRMATIONS (default 6) confirmations at its height: about six minutes at the 60-second target spacing plus the reconciliation retry backoff, so a routine lost race leaves this gauge well inside the threshold. The reconciliation backoff is ten seconds times the row's attempt count, capped at an hour, and the count includes the pre-offer attempts, so the cumulative wait to the k-th retry is about 5k(k+1) seconds: the sixth competitor confirmation at 60-second spacing is reached around the eighth retry, about nine minutes, well inside 30 minutes, while a row that needed many pre-offer attempts starts its backoff higher and reaches 30 minutes in a few retries. Such an acceptance is never part of the known pending age, so the revision-work critical cannot page on it; a winning offer leaves this gauge at its first active-chain observation and joins the known age on its original acceptance clock. Past 30 minutes the row is not being reconciled, the node is not advancing, or the orphan verdict keeps failing: operator work, warned, never paged; the cluster-wide candidate oldest-pending rules see the same reconciliation row. -1 (none tracked while identity tracking is saturated) is owned by the tracking-unknown warning and does not fire here. Three-minute dwell like the other warnings; no-data is OK because the pending warning already reports a missing landing family. Provisional; measure in #291.", + "producer_refs": [ + "crates/qbit-prism-server/src/metrics/registry.rs", + "crates/qbit-prism-server/src/metrics/landing.rs", + "crates/qbit-prism-server/src/stratum.rs" + ] + } + ] +} diff --git a/docs/prism-native-metrics.md b/docs/prism-native-metrics.md new file mode 100644 index 000000000..0f0532463 --- /dev/null +++ b/docs/prism-native-metrics.md @@ -0,0 +1,714 @@ +# Native metric inventory + +The coordinator (`run`) owns one process-local registry shared by Stratum, +background collectors, and the runtime monitor. `/metrics` renders cached +observations, with the pool-acquisition histogram, collector measurements, +#277 freshness, collector age/availability, the landing and node ages, the +accepted-share counter and `qbit_prism_rejections_total` (#581), and runtime state +read at scrape time. `qbit_prism_rejected_shares_total` and the stale, duplicate and +low-difficulty totals still come from the cached body. +Scraping performs no database, node, or filesystem I/O. Public-api metrics retain +their existing contract. + +The generated table below is the sole inventory for both roles: **48 coordinator +families and 15 public families**. Names, types and meanings for `run` come from +[registry.rs](../crates/qbit-prism-server/src/metrics/registry.rs#L42), with bounded +label values from [labels.rs](../crates/qbit-prism-server/src/metrics/labels.rs#L15). +[Compatibility annotations](prism-metric-metadata.json) add replacement names +and the pre-existing public producer contract. Regenerate with +`python3 scripts/generate_prism_metrics.py`; the ungated observability tests +scrape real HTTP listeners for both roles and both public replica modes, compare +families in both directions, and check the generator for drift. + +Every coordinator family has HELP and TYPE +metadata, including families declared without samples. The public role retains +its existing untyped exposition, except +`qbit_prism_public_audit_artifact_refusals_total`, which carries both; the +inventory records the counter/gauge intent from its producer. Public response/cache label sets are lazy and appear after a +request; replica gauges appear only with `PRISM_PUBLIC_REPLICA_MODE=require`. +Default histogram boundaries in seconds are 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, +1, 2.5, 5, 10, 30, and +Inf. `qbit_prism_share_ack_seconds` adds 15 and 20. +`qbit_prism_ctv_fanout_broadcaster_chunk_rows` counts rows, not seconds, and +exports the single finite bound 1; every other histogram uses the default +ladder. Histograms also export `_sum` and `_count`. See the +[histogram consumer guide](prism-metrics-histogram-consumers.md) for the five +added series per process, elapsed-time attribution, quantile changes, +mixed-version queries and the chunk-row histogram. +Bucket samples add `le`; the table lists producer labels. `job`, `instance` and +`network` are deployment scrape labels, not native metric dimensions. See the +[deployed-alert migration and draft diff](prism-alert-migration.md) for the +consumer contract and threshold evidence. + +Before the first complete health/metrics publication, scrapes expose the full +initialized registry while snapshot availability remains 0, snapshot age remains +-1, and `x-prism-metrics-state` remains `unavailable` without an `Age` header. +This includes declared histogram metadata without samples for unwired producers; +rendering the startup registry does not create a publication timestamp. + +## Cutover inventory + +<!-- generated-inventory:start --> +<!-- Run: python3 scripts/generate_prism_metrics.py --> + +| Family | Type | Labels | Role | Meaning / status | 2.x.x name replaced | +| --- | --- | --- | --- | --- | --- | +| `qbit_prism_accepted_block_revision_work_pending_seconds` | gauge | none | run | Monotonic age of the oldest known locally observed acceptance awaiting revision work delivery; -1 when only unknown tracking remains, zero when none; each frontend tracks every block it observes, and a frontend with no connected miners keeps waiting. Computed at scrape time and overlaid on cached metric bodies. A second acceptance never resets the oldest known unresolved delivery wait; a failed build does not clear it. Unknown intervals are exposed independently by accepted_block_revision_work_tracking_unknown, so they cannot mask a separate measured stall. Lost ordering history makes pending age unknown. An offer the node accepted without an active-chain observation is an unlanded acceptance (#493): excluded here and reported by accepted_block_unlanded_seconds until an active-chain observation lands it on its original acceptance clock or a proven orphan closes it. | none | +| `qbit_prism_accepted_block_revision_work_tracking_unknown` | gauge | none | run | Whether any local landing observation is unknown or incomplete; independent of known pending delivery age. Process-local boolean, zero at a fresh start and one for incomplete observation, lost target knowledge or bounded tracking saturation. May be one alongside a positive known pending age; never treat a known age as proof that all tracking is complete. No labels. | none | +| `qbit_prism_accepted_block_to_revision_work_seconds` | histogram | `result=published,degraded,superseded` | run | Frontend-local definitive acceptance observation to first successful mining.notify write carrying compatible post-landing payout work, in seconds; each frontend samples every block from its own observation, so a sum across instances counts one block once per frontend. Frontend-local monotonic timing; no inference of a peer node acceptance timestamp. Every result ends at successful mining.notify write, not prepared publication or proof-to-first-offer. Superseded means a later payout revision arrived before delivery and measures through replacement-work delivery. The pending gauge retains unresolved delivery age. Finite buckets: 0.25, 0.5, 1, 2.5, 5, 10, 30, 60, 120, 300, 307 and 600 seconds. | `qbit_prism_accepted_block_preview_publication_seconds` | +| `qbit_prism_accepted_block_unlanded_seconds` | gauge | none | run | Monotonic age of the oldest definitive submitblock acceptance this frontend has not yet observed on the node's active chain, such as a lost tip race until its orphan proof; never part of the known pending delivery age; zero when none, -1 when none is tracked and identity tracking is saturated; while the terminal probe that carries peer orphan evidence fails, the pending gauge reads -1 and the tracking-unknown gauge one even if only unlanded acceptances are open. Computed at scrape time and overlaid on cached metric bodies (#493). Set by the offering frontend at a definitive null submitblock reply; an active-chain observation (the post-offer landing observation, a reconcile proof or a settlement) lands the block, which then joins accepted_block_revision_work_pending_seconds on its original acceptance clock; a committed proven orphan closes it without a histogram sample. An unlanded acceptance never contributes to the known pending age, never attributes a build deadline and is not unknown tracking: a lost tip race is a known state. -1 only when none is tracked and identity tracking is saturated, so an unlanded acceptance may be untracked; a tracked unlanded age stays visible beside saturation, and the tracking-unknown gauge is one either way. While the terminal probe (the ten-second collector read that carries peer orphan evidence) fails or is cancelled, an unlanded-only frontend reads tracking unknown and a -1 pending age, because the lost race's orphan evidence is unreadable; this gauge keeps its age, and the next successful read restores the known state. Warned by PrismAcceptedBlockUnlanded at 30 minutes, never paged. | none | +| `qbit_prism_accepted_shares_total` | counter | none | run | Shares accepted by this instance since process start. Process-local counter; legacy canonical ledger count was persistent. Read from the Stratum listeners at scrape time and overlaid on cached metric bodies (#581), so the share-refusal rules see acceptances stop while the health publisher, and with it the cached body, waits on an unavailable database. | `qbit_prism_accepted_shares_total` | +| `qbit_prism_authorized_clients` | gauge | none | run | Current local authorized Stratum connections. | `qbit_prism_stratum_authorized_connections` | +| `qbit_prism_authorized_missing_current_work` | gauge | none | run | Authorized connections missing the current semantic work generation. | none | +| `qbit_prism_authorized_with_current_work` | gauge | none | run | Authorized connections holding the current semantic work generation. | `qbit_prism_stratum_clients_with_current_tip_jobs` | +| `qbit_prism_block_candidate_oldest_landing_failed_seconds` | gauge | none | run | Oldest cluster-wide time since the offer reservation of an offered or reconciliation row whose audit landing has not committed (no pool-block row) or whose last error names a landing refusal; zero when none, or -1 when unknown. Read in the same database snapshot as the other candidate gauges (#493 review M1): the oldest offered or reconciliation row whose landing has not committed (the landing transaction is the only writer of the block's qbit_pool_blocks row, so its absence is the durable fact) or whose last_error carries the landing-failed prefix the coordinator and the collector share (a landed row that no longer authenticates): a row the node accepted or rejected whose audit rebuild, verification or ledger landing keeps refusing and is retried at the reconciliation backoff. Since #510 an offered row counts before its first landing outcome too: a won block whose landing hangs in offered (waiting for build capacity, which has no deadline, while its heartbeat keeps the claim) ages here instead of reading zero, and a healthy offered row lands within the 60-second rebuild deadline, well inside the warning dwell. A retry that fails for a transient node or database reason, or a refused operator recovery, overwrites last_error but not the missing pool-block row, so the age never reads zero while nothing has landed. Measured from offer_reserved_at, the start of the post-offer lifecycle, so pre-offer retry backoff never counts toward it. Zero for a lost race and for a rejection once its audit has landed, because the landing runs before the chain verdict and writes the pool-block row. PrismBlockCandidateLandingFailed warns at 60 s for 3 m and PrismBlockCandidateLandingFailedCritical pages at 307 s for 1 m, the same floor and dwell the revision-work critical applies when the landing fails after the frontend's own active-chain observation, so a won block whose landing keeps failing is never silent. -1 with the other candidate gauges when the collector fails. | none | +| `qbit_prism_block_candidate_oldest_pending_seconds` | gauge | none | run | Oldest cluster-wide pending candidate age, or -1 when unknown. Counts every unfinished outbox state: pending, offer_reserved, offered and reconciliation (A/#266). Excludes rows settled as proven orphans (#415, migration 015): those are terminal, so a lost tip race leaves these gauges once a different block has PRISM_CANDIDATE_ORPHAN_CONFIRMATIONS confirmations at its height. Since #493 the paging rule reads block_candidate_oldest_unacknowledged_seconds instead, so an acknowledged lost race in reconciliation warns but never pages; this gauge keeps its all-unfinished meaning and feeds the 15-second warning and the 30-minute stuck warning. | `qbit_prism_block_candidate_oldest_pending_seconds` | +| `qbit_prism_block_candidate_oldest_unacknowledged_seconds` | gauge | none | run | Oldest cluster-wide candidate age the node has not accepted: pending and offer-reserved rows, offered rows whose one submitblock outcome is unknown (not a row adopted on the node's active-chain evidence), and rows the node rejected unless the reply names a side-chain block; zero when every unfinished row was accepted, or -1 when unknown. Read in the same database snapshot as the two all-unfinished gauges (#493): the unfinished rows the node has not accepted are pending and offer_reserved rows, offered or reconciliation rows whose one submitblock outcome is unknown (a transport failure or timeout, or a reservation whose call was lost with its frontend; a row adopted on the node's own evidence that its block is active carries the adoption reply and is excluded), and rows the node definitively rejected unless the reply names a side-chain block (inconclusive, duplicate, duplicate-inconclusive: a lost tip race). A node-accepted lost race awaiting its orphan proof and a landing that keeps failing after acceptance are unfinished but accepted, so they age only block_candidate_oldest_pending_seconds (and the landing-failed gauge). PrismBlockCandidateOldestPendingCritical pages on this gauge, falling back to the all-unfinished age on a target that does not export it yet; PrismBlockCandidateStuck warns on the all-unfinished age at 30 minutes. -1 with the other candidate gauges when the collector fails. | none | +| `qbit_prism_block_candidates_orphaned_total` | counter | none | run | Offered block candidates this instance settled as proven orphans since process start. Counts offered-candidate orphan settlements whose successful completion this instance observed after commit (#415): its landed audit is durable, and one coherent tip observation showed a different block active at its height with at least PRISM_CANDIDATE_ORPHAN_CONFIRMATIONS confirmations. May undercount if cancellation or restart occurs after the database commit and before the process records completion. Never counts a failed observation; a reorg that later reactivates the block is credited by the reorg reconciler without touching this counter. Process-local; starts at zero. No firing rule. | none | +| `qbit_prism_block_candidates_pending` | gauge | none | run | Cluster-wide nonterminal candidate count, or -1 when unknown. Counts every unfinished outbox state: pending, offer_reserved, offered and reconciliation (A/#266). Excludes rows settled as proven orphans (#415, migration 015): those are terminal, so a lost tip race leaves these gauges once a different block has PRISM_CANDIDATE_ORPHAN_CONFIRMATIONS confirmations at its height. Since #493 the paging rule reads block_candidate_oldest_unacknowledged_seconds instead, so an acknowledged lost race in reconciliation warns but never pages; this gauge keeps its all-unfinished meaning and feeds the 15-second warning and the 30-minute stuck warning. | `qbit_prism_block_candidates_pending` | +| `qbit_prism_block_offer_standby_wait_total` | counter | `outcome=confirmed,absent,lagging,failed` | run | Found-block offers that waited for the failover standby's WAL flush first (#529, PRISM_OFFER_STANDBY_APPLICATION_NAME), by how the wait ended; every one was offered. Counted once per found-block offer when PRISM_OFFER_STANDBY_APPLICATION_NAME is set and PRISM_OFFER_STANDBY_FLUSH_WAIT_MS is positive, after the offer reservation commits and immediately before submitblock (#529). `confirmed`: the flush_lsn in pg_stat_replication of every standby connected under that name reached the primary's WAL flush position read after the reservation, so the candidate row and reservation survive the standby's promotion. `absent`: no standby connected under that name (streaming or catching up), so the offer did not wait. `lagging`: the standby was still behind at the bound; the WARN log carries lag_bytes. `failed`: the positions could not be read (a database error, no answer before the bound, or a writer role without pg_read_all_stats, which pg_monitor includes). Every outcome is offered; every outcome but `confirmed` logs the block hash at WARN for post-failover reconciliation. The wait itself is logged per block as waited_us. Process-local; every series starts at zero. Warns through PrismBlockOfferStandbyUnconfirmed. | none | +| `qbit_prism_block_proof_ack_capped_total` | counter | `path=share,block_only` | run | Block-bearing submissions answered ledger-outcome-unknown because their acknowledgement bound (#574: share_commit_timeout, plus the share grace for an append whose COMMIT was in flight) passed before the append, enqueue or landing finished, by the wait that was cut short; the block and any deferred share are still landed and credited. Counted once per block-bearing submission answered before its outcome was known (#574). `block_only`: a captured (#478) or below-target block proof whose deferred share waits for its own frontend's landing, or for its duplicate probe and candidate enqueue, past `share_commit_timeout` itself (`PRISM_SHARE_COMMIT_TIMEOUT_SECONDS`, 15 s by default; it was at least 60 s before #574). `share`: a share-pass append carrying a found block that had not committed by the share deadline, or by its grace when COMMIT was already in flight, the same bounds as a plain share; it is never refused, so it runs on; and a share-pass append that captured its block because the payout revision moved before its commit (#657), whose deferred share then waits for its block's landing as a `block_only` proof's does, up to the same share deadline. Either is answered `ledger-outcome-unknown` (Stratum error 20, "share outcome is not yet known"), the same reply as any share still pending at its deadline, so it also counts in `qbit_prism_rejections_total{reason_id="ledger-outcome-unknown"}` and feeds PrismShareAppendFailures; the WARN log names the block hash, share id, phase and elapsed_ms. Only the wait ends at the bound: the probe, enqueue, append and landing run on, so the block is still offered and landed and its share is credited once when that commits. Process-local; both series start at zero. No firing rule of its own. | none | +| `qbit_prism_block_submission_enabled` | gauge | none | run | Whether this frontend offers found blocks to its node and lets its CTV fanout broadcaster send: 1 normally, 0 while PRISM_BLOCK_SUBMIT_ENABLED is off (#291) or the cluster holds block submission (#664), so block candidates stay pending and no fanout is sent, -1 while the switch is on but the frontend has not yet read the cluster's hold; a later failed read keeps the last one. | none | +| `qbit_prism_block_submit_seconds` | histogram | none | run | Locally validated block proof to first node offer; requires the offer owner's timestamp boundary. Observed once per block by the offering frontend after its one submitblock call returned, from the enqueuing frontend's proof-observation wall clock to the offering frontend's wall clock immediately before the send (A/#266). No sample for a row without a proof time, for a negative interval (host clock skew) or on a recovery; a crash between the call and its outcome commit may lose the sample. No firing rule. | `qbit_prism_block_submit_seconds` | +| `qbit_prism_blocks_total` | counter | none | run | Blocks confirmed by this instance since process start. Native confirmed-block process counter; not the legacy node-acceptance accounting boundary. | `qbit_prism_blocks_accepted_total` | +| `qbit_prism_capture_offer_decisions_total` | counter | `decision=offered,abandoned_ceiling,abandoned_disabled` | run | Offer decisions this instance made for pending blocks on the current tip whose payout revision was superseded (#478 block capture), by decision. #478 block capture. `offered`: the block's positive as-issued float (the most debt its confirmation can create, in any order) was within `PRISM_CAPTURE_OVERPAY_CEILING_BPS` of the coinbase value, and the block was reserved for its one offer, decided in the reservation's transaction. `abandoned_ceiling`: the float exceeded the ceiling and the pending block was abandoned; the WARN log names the bound and the ceiling, and `qbit_prism_payout_divergences` keeps both. `abandoned_disabled`: the ceiling is 0, so the block was abandoned as before #478 (recorded as decision `disabled`). The miner of an abandoned capture is answered `stale-job`, never `ledger-confirmation-failed`. A bound that cannot be computed abandons nothing: the offer fails and is retried. Process-local; every series starts at zero. No firing rule. | none | +| `qbit_prism_carry_forward_debt_sats` | gauge | none | run | Sum of negative carry-forward balances, in sats, read from the canonical balances at this instance's latest balance change or full refresh, or -1 before one. Read from the canonical balances after every confirmation or deactivation this frontend commits and at every full refresh snapshot; zero when no account is negative. Process-local. No firing rule. | none | +| `qbit_prism_collector_age_seconds` | gauge | `collector=database,process` | run | Monotonic age of the last successful collector observation, or -1 before success. | none | +| `qbit_prism_collector_available` | gauge | `collector=database,process` | run | Whether a collector has a complete successful observation. | none | +| `qbit_prism_collector_success` | gauge | `collector=database,process` | run | Whether the latest collector attempt succeeded, or -1 before an attempt. | none | +| `qbit_prism_connections` | gauge | none | run | Current local Stratum connections. | `qbit_prism_connected_clients`, `qbit_prism_stratum_active_connections` | +| `qbit_prism_ctv_fanout_broadcaster_chunk_rows` | histogram | none | run | Fanouts attempted per native broadcaster chunk; each chunk contains one row. Values are rows, not seconds: instead of the default time ladder this histogram exports the single finite bound `le="1"` plus `+Inf`, so `_count` is the number of claimed fanout attempts and `_sum` equals `_count`. The 2.x.x ladder (1, 2, 5, 10, 25, 50, 100) is not exported; a query for a higher `le` matches no series. | `qbit_prism_ctv_fanout_broadcaster_chunk_rows` | +| `qbit_prism_ctv_fanout_broadcaster_chunk_seconds` | histogram | none | run | Claimed fanout attempt duration including status persistence, in seconds. Default histogram ladder in seconds; one observation per claimed attempt, recorded even when persisting the attempt's completion fails. | `qbit_prism_ctv_fanout_broadcaster_chunk_seconds` | +| `qbit_prism_ctv_fanout_broadcaster_tip_refresh_yields_total` | counter | none | run | CTV passes deferred at a fanout boundary for a newer or unpublished tip. | `qbit_prism_ctv_fanout_broadcaster_tip_refresh_yields_total` | +| `qbit_prism_database_advisory_lock_wait_seconds` | histogram | `lock=migration,order,settlement`; `result=success,failure` | run | Database advisory transaction lock wait by lock and outcome. Client-observed duration of the `pg_advisory_xact_lock` statement, including one database round trip, recorded by the coordinator's ledger for the migration, order and settlement locks; the migration lock is taken only when the coordinator initializes the schema. `failure` includes lock timeout (`PRISM_DATABASE_LOCK_TIMEOUT_MS`, default 5 seconds), statement timeout, deadlock and connection errors, and waits abandoned by cancellation. The CPFP funding lock is not observed (#328). Series appear on their first observation, so a restart's first failure is not visible to `increase()`. | none | +| `qbit_prism_database_order_lock_hold_seconds` | histogram | `holder=append,candidate_insert,first_confirmation,settlement,reconcile,orphan,prepared,cleanup,fatal_state,operator` | run | ORDER_LOCK hold from the grant of the lock to the end of the transaction holding it, by holder, in seconds (#602). Measured by the coordinator's ledger from the instant `pg_advisory_xact_lock` returned for `ORDER_LOCK` to the instant the holding transaction's COMMIT or ROLLBACK returned (#602); a transaction abandoned by an error ends its hold as it unwinds, a moment before its queued ROLLBACK releases the lock. `append` is the share append, which every miner's acknowledgement waits behind; `first_confirmation` is the offering frontend's settlement that confirmed its block for the first time and `settlement` every other one, including an abandon; `reconcile` is the reorg reconciler, which runs on every template build and can also confirm a block; `first_confirmation` and `settlement` are per attempt, so a settlement that rolls back and is retried records twice; `prepared` is the refresh's window snapshot that prepared work is built from; `candidate_insert` a block-only candidate's insert; `cleanup` blob garbage collection; `fatal_state` fatal-state recovery; `operator` the policy and signing transitions and archive verify and restore, when run with a registry. The schema cutover's hold and archive's statement-scoped append drain are not observed. #602's R2 budget is 250 ms for every holder but `append`. Own ladder from 1 ms to 10 s. Series appear on their first observation. No firing rule of its own. | none | +| `qbit_prism_database_pool_acquire_seconds` | histogram | `result=success,failure` | run | Actual database pool acquisition wait by outcome. Client-observed `PgPool::acquire` time: waiting for a pool permit, the idle-connection liveness ping and, when the pool grows, connection setup; excludes transaction BEGIN and the queries that follow. Recorded by the metrics collector, instrumented coordinator ledger transactions and covered direct ledger queries, rollup transactions and share-partition maintenance. The histogram aggregates observed checkout attempts at covered callers. Public-role read pools, private audit HTTP endpoints and shared API read-model helpers, startup/migration gates, operator tools and telemetry-free compatibility wrappers remain excluded; see the acquisition guide for the source census and exclusion policy. `failure` includes acquire errors, the 15-second acquire timeout and acquisitions abandoned by cancellation, recorded with the elapsed wait. Buckets, count and sum are read together from the live registry on each scrape, independently of cached-body publication; scraping does not create observations or renew snapshot freshness. | none | +| `qbit_prism_divergent_landing_overpay_sats_total` | counter | none | run | Carry-forward debt, in sats, created by the divergent confirmations this instance committed. The sum over committed divergent confirmations of the carry-forward debt each created: per account, the increase in max(0, -balance) when the block's rows start to count. Unrecovered while the indebted miner earns no further gross; see the ledger operations guide for the accepted-cost bound. Process-local; starts at zero. No firing rule. | none | +| `qbit_prism_divergent_landings_total` | counter | none | run | Confirmations this instance committed whose landed rows started to count on canonical balances other than their as-issued prior balances. Counted at the confirmation, when a landed block's rows start to count: it is divergent when those rows meet canonical balances other than the as-issued ones its coinbase commits to (#478 captures, and the pre-existing case of a block whose work predates an earlier confirmation). Each is recorded in `qbit_prism_payout_divergences`, in the confirming transaction, with its per-account overpay; a frontend without #478 records and counts nothing. Process-local; starts at zero. No firing rule. | none | +| `qbit_prism_duplicate_shares_total` | counter | none | run | Duplicate share rejections. | `qbit_prism_duplicate_shares_total` | +| `qbit_prism_grace_credited_shares_total` | counter | none | run | Durably accepted shares credited by stale grace. | `qbit_prism_grace_credited_shares_total` | +| `qbit_prism_hashrate_rollup_watermark_lag_seconds` | gauge | none | run | Monotonic time since this frontend last completed a caught-up hashrate rollup pass, or -1 before its first; no sample when the rollup is disabled. The stored watermark is a share sequence, not a time; a caught-up pass is one that advanced the watermark within its batch bound, leaving no unfolded share. A pass that lost the race to another frontend leaves the stamp unchanged, so a frontend that keeps losing reads high: alert on the minimum across instances. -1 runs from loop start until its first such pass, and PRISM_HASHRATE_ROLLUP_ENABLED=0 leaves the family declared with no sample at all. | none | +| `qbit_prism_health_state` | gauge | none | run | Whether this instance is ready to serve mining work. | none | +| `qbit_prism_job_delivery_cancellations_total` | counter | none | run | Local job deliveries abandoned before their announcement because their session ended: the miner disconnected or broke the protocol, or the listener shut down (#621); counted neither as successes nor as failures. Process-local counter, 0 at startup. A delivery is counted once, at its end, in exactly one of `job_delivery_successes_total`, `job_delivery_failures_total` and this family (#621). Counted here: a delivery still being admitted, built or persisted, or built but not yet announced, when its session ends, which since #621 a session notices while a delivery is in flight. A delivery whose announcement write fails still counts as a failure. No rule of its own. | none | +| `qbit_prism_job_delivery_failures_total` | counter | none | run | Failed local job deliveries. A build, admission or persistence error or deadline, or a failed announcement write. A delivery its session abandons before the announcement counts in `job_delivery_cancellations_total` instead (#621). | none | +| `qbit_prism_job_delivery_successes_total` | counter | none | run | Successful local job deliveries. | none | +| `qbit_prism_job_preparation_deferrals_total` | counter | `reason=tip_polling_stale,tip_polling_unavailable,new_tip_pending,work_retired,fee_floor,other` | run | Job preparations this frontend deferred (each logged as job preparation deferred), by the first check that refused them. Counted once per `job preparation deferred` WARN, with the first check that refused the preparation (#622): `tip_polling_stale` (readiness older than PRISM_HEALTH_TIP_POLL_MAX_AGE_SECONDS and no replacement lease), `tip_polling_unavailable` (readiness revoked), `new_tip_pending` (a newer tip is observed and its work is not published yet), `work_retired` (the authority the work was read with is retired, logged as `payout snapshot stale` for a payout revision or prior-balance change, including a same-tip build a later publication overtook without keeping them, a newer tip's published work replacing its parent, or a lease or publication the work no longer matches, and as `window_not_held` when this primary does not hold the work's window after a writer timeline change (#619)), `fee_floor` (the job's CTV fee is below the live relay floor, or the floor is unknown) and `other` (readiness changed during admission, a database or node error, or a build failure). Tip authority is checked before the payout revision, so `tip_polling_stale` can mask retired work behind it. The session retries on its next timer tick or publication, so one session can count many. Process-local; every series starts at zero. | none | +| `qbit_prism_landing_malloc_trim_released_bytes_total` | counter | none | run | Resident bytes this frontend's post-landing malloc_trim calls returned to the kernel (#600): the process resident set just before each trim minus just after it, never negative; allocation on other threads during a trim can hide part of its release. | none | +| `qbit_prism_landing_malloc_trim_resident_bytes` | gauge | none | run | Process resident bytes right after this frontend's latest post-landing trim, its resident floor after the landing, or -1 before the first trim or when procfs could not be read. | none | +| `qbit_prism_landing_malloc_trim_seconds` | histogram | none | run | Duration of each malloc_trim(0) this frontend ran after a block landing released its rebuilt window (#600), in seconds; glibc builds only, none while PRISM_LANDING_MALLOC_TRIM_ENABLED=0. | none | +| `qbit_prism_late_confirmed_shares_total` | counter | none | run | Shares accepted after the share commit deadline once their in-flight ledger commit was confirmed. | none | +| `qbit_prism_low_difficulty_shares_total` | counter | none | run | Low difficulty share rejections. | `qbit_prism_low_difficulty_shares_total` | +| `qbit_prism_metrics_snapshot_age_seconds` | gauge | none | run | Monotonic age of the metrics snapshot, or -1 before the first publication. | `qbit_prism_metrics_snapshot_age_seconds` | +| `qbit_prism_metrics_snapshot_available` | gauge | none | run | Whether a complete metrics snapshot has been published. | `qbit_prism_metrics_snapshot_available` | +| `qbit_prism_metrics_snapshot_stale` | gauge | none | run | Whether the metrics snapshot is missing or exceeds the health freshness budget. | `qbit_prism_metrics_snapshot_stale` | +| `qbit_prism_node_initial_block_download` | gauge | none | run | Whether the node reported initial block download in the latest answered getblockchaininfo, or -1 when unknown. Recorded whenever the latest getblockchaininfo answered with the boolean, including the 1 that then makes readiness fail. -1 means startup, a failed call, or a field that was not a boolean. | none | +| `qbit_prism_node_observation_age_seconds` | gauge | none | run | Monotonic age of the last answered getblockchaininfo, or -1 before one; it grows while the node is unreachable. Stamped when the call answered, not when the whole readiness check passed, so a reachable but unready node is distinguishable from an unreachable one. Computed from a monotonic clock at scrape time. Peers and the sync flag come from the latest attempt to start, so a slower older attempt landing late cannot roll them back, while the age keeps the latest answer any attempt received. | none | +| `qbit_prism_node_peers` | gauge | none | run | Node peer connections from the latest node observation, or -1 when unknown. Recorded only when the readiness attempt reached getnetworkinfo and it answered, which is never on regtest, during initial block download, or while blocks and headers disagree; observation adds no RPC call. -1 means the latest attempt did not learn the count, including RPC failure; the gauge never keeps an earlier reading. An answered zero is a real zero, and it is below every configured PRISM_MIN_PEERS floor. | none | +| `qbit_prism_pending_job_builds` | gauge | none | run | Current local pending job deliveries. Delivery count replaces the operational intent of queue depth, not its implementation. | `qbit_prism_job_delivery_queue_depth` | +| `qbit_prism_process_open_fds` | gauge | none | run | Process open file descriptors counted in procfs, the collector's own read included, or -1 when unknown. | none | +| `qbit_prism_process_resident_memory_bytes` | gauge | none | run | Process resident memory bytes from procfs, or -1 when unknown. | `qbit_prism_process_resident_memory_bytes` | +| `qbit_prism_public_audit_artifact_refusals_total` | counter | none | public-api | Audit artifact requests refused with 503 audit_artifact_busy because PRISM_PUBLIC_AUDIT_ARTIFACT_MAX_IN_FLIGHT audit artifacts were already in flight. Public-api role only: the mining process serves the same route on its operator listener and applies the same cap there, but has no public-service metrics owner, so those refusals are uncounted. | `qbit_prism_public_audit_artifact_refusals_total` | +| `qbit_prism_public_cache_total` | counter | `state=HIT,MISS,STALE,BYPASS` | public-api | Public route cache outcomes; appears after a public route request. | `qbit_prism_public_cache_total` | +| `qbit_prism_public_database_outage_refusals_total` | counter | none | public-api | Uncached database reads refused during database unavailability. | `qbit_prism_public_database_outage_refusals_total` | +| `qbit_prism_public_degraded_responses_total` | counter | none | public-api | Cached HIT responses served during database unavailability. | `qbit_prism_public_degraded_responses_total` | +| `qbit_prism_public_ledger_probe_age_seconds` | gauge | none | public-api | Monotonic age of the last completed readiness probe (including failure); -1 before the first attempt. | `qbit_prism_public_ledger_probe_age_seconds` | +| `qbit_prism_public_ledger_ready` | gauge | none | public-api | Schema/read probe succeeded and remains within its freshness budget; zero before a probe, after failure, or when stale. Does not include the separate replica heartbeat refusal. | `qbit_prism_public_ledger_ready` | +| `qbit_prism_public_replica_apply_backlog_bytes` | gauge | none | public-api (replica=require) | Last observed receive-LSN minus replay-LSN bytes; -1 if unknown; may remain cached after probe failure. | `qbit_prism_public_replica_apply_backlog_bytes` | +| `qbit_prism_public_replica_heartbeat_age_seconds` | gauge | none | public-api (replica=require) | Last observed public WAL-receiver heartbeat age plus monotonic time since observation; -1 if unknown/disconnected. | `qbit_prism_public_replica_heartbeat_age_seconds` | +| `qbit_prism_public_replica_in_recovery` | gauge | none | public-api (replica=require) | Last observed public read database recovery state; zero if never observed. Not the dedicated HA standby. | `qbit_prism_public_replica_in_recovery` | +| `qbit_prism_public_replica_max_lag_seconds` | gauge | none | public-api (replica=require) | Configured public read replica maximum lag/heartbeat-age budget in seconds. | `qbit_prism_public_replica_max_lag_seconds` | +| `qbit_prism_public_replica_refusals_total` | counter | none | public-api | Database-reading routes refused by the public replica contract. | `qbit_prism_public_replica_refusals_total` | +| `qbit_prism_public_replica_replay_lag_seconds` | gauge | none | public-api (replica=require) | Time since the last replayed transaction plus observation age; can increase on an idle primary and is not a byte-backlog or HA failover-loss measurement; -1 if unknown. | `qbit_prism_public_replica_replay_lag_seconds` | +| `qbit_prism_public_requests_total` | counter | none | public-api | Every routed request, including /healthz and /metrics; never use in a request-rate rule. | `qbit_prism_public_requests_total` | +| `qbit_prism_public_responses_total` | counter | `status` (HTTP status code) | public-api | HTTP responses by status, including health probes; appears after the first response. | `qbit_prism_public_responses_total` | +| `qbit_prism_public_staleness_refusals_total` | counter | none | public-api | Responses refused for exceeding an endpoint cache-age budget. | `qbit_prism_public_staleness_refusals_total` | +| `qbit_prism_refresh_seconds` | histogram | `trigger=initial,tip,revision,balances,reanchor,shares,template,fee,writer_timeline`; `acquisition=delta,full,cached` | run | Template refresh from entry to published work, in seconds, by what triggered the rebuild and how its window was acquired. Observed once per refresh that published rebuilt work, from the refresh's entry (after the poll or block-wait wake) to its publication, on the coordinator's monotonic clock. A refresh that reused the published work unchanged, or that failed, records nothing. `trigger` names what invalidated the published work, by the label's own precedence when several inputs changed on one poll: `initial` (nothing published yet), then `writer_timeline` (the database writer's timeline moved, a promotion: #619), `tip` (the template parent changed), `revision` (the payout revision changed), `balances` (the carry-forward balances changed), `reanchor` (the cached window aged past `PRISM_PAYOUT_ARTIFACT_REANCHOR_SECONDS`, or no cached window was held while work was published), `shares` (new shares with no cached bundle), `fee` (the relay fee floor changed) and `template` (any other template or publication change, including a cached window that no longer matches the published reference). The reuse test itself checks these inputs in a different order; the label is a classification of the same inputs, never a decision. On a same-template poll the revision, balances and share inputs are the ledger probe's; on a new template the ledger is not probed at entry, and the label compares the published work with the window the rebuild used, so a revision or balance change that had landed by the rebuild's admission is named ahead of `template`. `acquisition` says where the window came from: `delta` or `full` for a fresh ledger snapshot by that path, `cached` when the previous refresh's window was reused and only the body was rebuilt. Series appear on their first observation. Default buckets. | none | +| `qbit_prism_refresh_window_acquisitions_total` | counter | `outcome=advanced,no_prior,empty_prior,tail_mismatch,anchor_regressed,cutoff_regressed,delta_too_large,no_evidence,leaf_changed,margin_too_large,partial,witness_changed,count_mismatch,invariant` | run | Refresh payout-window snapshots by acquisition outcome: advanced by the delta path, or the reason the full scan ran. Counted once per refresh snapshot by the coordinator's ledger, at the decision between the delta path and the full newest-first scan. `advanced` is the delta path: the retired window's rows were reused for the fresh anchor, cutoff and network difficulty, including a retarget in either direction and a multi-page delta. Every other outcome names why the full scan ran: `no_prior` (no retired window was exclusively owned, as at the first refresh or after a cancelled build kept a reference), `empty_prior` (the retired window had no rows), `tail_mismatch` (the retired window's last row was not its accepted cutoff, which only legacy or raw rows with a future timestamp at the top of the ledger can cause), `anchor_regressed`, `cutoff_regressed`, `delta_too_large` (more than 262,144 new sequence slots), `no_evidence` (the retired window had no single-leaf witness, as across a partition boundary), `leaf_changed` (the retained range's partition was detached, reattached or restored, or the writer timeline changed, before the delta was read), `margin_too_large` (a heavier target needed more than 16 older pages), `partial` (history ran out before the target, so the full scan decides), `witness_changed` (the merged range's leaf, incarnation or timeline differed when re-read after the pages, including a margin that crossed into another leaf), `count_mismatch` (the anchored row count of the merged range differed, as after a retroactive insert or newly eligible timestamp) and `invariant` (the merged window failed its own ordering or crossing check, logged at error). Every outcome is initialized to zero. A high full-scan share is a cost signal, never an error; `invariant` is a defect signal. | none | +| `qbit_prism_rejected_shares_total` | counter | none | run | Shares rejected by this instance since process start. | none | +| `qbit_prism_rejections_total` | counter | `reason_id=stale-job,duplicate-share,low-difficulty,malformed-submit,unauthorized-worker,unknown-job,invalid-extranonce,invalid-ntime-or-nonce,backend-rpc-unavailable,backend-database-unavailable,internal-error,pool-closed,ledger-confirmation-failed,ledger-outcome-unknown,unrecognised` | run | Share rejections by canonical bounded reason ID. Present unknown or empty IDs map to unrecognised; missing IDs and explicit internal-error retain internal-error. Normalization does not change the protocol response. backend-rpc-unavailable names the node, backend-database-unavailable the ledger database (#581): a refusal on a path that reads both names whichever failed, and a timeout or aged-out readiness names the ledger database while the work it waited for was still waiting on it (#655). A published-work lease whose commit gate cannot read or prove its chain authority before COMMIT is answered the same way as readiness it cannot prove (#716), so contention with no failed dependency counts as backend-rpc-unavailable. Incremented per refusal and overlaid on cached metric bodies at scrape time (#581), so refusals during a database outage are counted while the cached body is stale. | `qbit_prism_rejections_total` | +| `qbit_prism_revision_work_build_timeouts_total` | counter | none | run | Existing build deadlines actually hit while a known accepted-block revision work wait is open on this frontend; deadlines hit while only unknown tracking remains are not counted. Counts actual existing deadline hits, once per operation; success, cancellation and ordinary errors are not timeout events. No deadline or fallback policy is introduced. Only a known open wait attributes a deadline hit here (#493): while tracking is unknown and no known wait remains (a lost or cancelled settlement COMMIT reply, an unsettled orphan verdict, bounded-tracking saturation or lost ordering history), a deadline hit is an ordinary job_delivery_failures_total event, accepted_block_revision_work_tracking_unknown reports the unknown state and the pending gauge stays -1; it is not counted here and does not stamp a wait recovered after the deadline degraded; a wait recovered before the deadline counts and is degraded as usual. | `qbit_prism_accepted_parent_preview_wait_timeouts_total` | +| `qbit_prism_runtime_lag_seconds` | gauge | none | run | Latest observed runtime sampler wake lateness, or -1 before the first observation. Runtime-stall intent replaces lease wake delay; no native writer lease. | `qbit_prism_lease_heartbeat_monitor_wake_delay_window_max_seconds` | +| `qbit_prism_runtime_poll_lag_seconds` | gauge | `task=refresh,submit,block_wait,broadcast,rollup,health_publisher,stratum_listener,stratum_session,collector,share_partitions` | run | Maximum active poll duration or completed poll duration retained for 60 to 61 seconds, by task. | none | +| `qbit_prism_runtime_progress_age_seconds` | gauge | `task=refresh,submit,block_wait,broadcast,rollup,health_publisher,stratum_listener,stratum_session,collector,share_partitions` | run | Oldest active operation time since progress; zero when idle. | none | +| `qbit_prism_runtime_task_stalled` | gauge | `task=refresh,submit,block_wait,broadcast,rollup,health_publisher,stratum_listener,stratum_session,collector,share_partitions` | run | Whether an active poll or operation exceeds its progress budget. | none | +| `qbit_prism_runtime_workers` | gauge | none | run | Configured Tokio runtime worker threads. | none | +| `qbit_prism_share_ack_landing_window_seconds` | histogram | `result=accepted,rejected` | run | The share_ack_seconds observations whose submission arrived within 30 seconds after a pool block acceptance this frontend observed, by outcome (#602); steady state is share_ack_seconds minus this family. Each observation is also in `share_ack_seconds`, on the same ladder: steady state is `share_ack_seconds` minus this family, bucket by bucket. A landing window opens when this frontend first observes a pool block's acceptance, its own `submitblock` acceptance, or a peer frontend's block on the active chain, first seen either before it is confirmed or already confirmed at the tip (a buried block first seen confirmed, as after a restart, opens none; a restart whose tip is a pool block opens one), and admits every submission that arrives within 30 seconds of it; an acceptance while a window admits extends it. A submission in flight at the acceptance is in the window. Both series start at zero. Fed by PrismLandingAckP99Slow and PrismLandingAckP99High through `share_ack_slow_landing_windows`. | none | +| `qbit_prism_share_ack_seconds` | histogram | `result=accepted,rejected` | run | Complete mining.submit frame arrival to completed response write, by outcome. Uses the default histogram ladder plus 15 and 20 seconds; these are elapsed ACK bounds, not measured ledger deadlines. | `qbit_prism_share_ack_seconds` | +| `qbit_prism_share_ack_slow_landing_windows` | gauge | `p99_above_seconds=2,10` | run | Consecutive most recent closed landing windows on this frontend whose share acknowledgement p99 exceeded the bound in seconds (#602); -1 before a landing window holding an acknowledgement has closed. A landing window (see `share_ack_landing_window_seconds`) is judged 30 seconds after it stops admitting submissions, so late acknowledgements can complete, or at the next acceptance if that comes first. Its nearest-rank p99 is over a bound exactly when more than one percent of its acknowledgements took longer than the bound, which the process counts exactly rather than reading from buckets. A window with no acknowledgement has no verdict and leaves the count unchanged; a window at or under the bound resets it to zero. -1 until the first verdict since process start. Per frontend: each frontend judges the landings it observed. PrismLandingAckP99Slow tickets at three for the 2-second bound, PrismLandingAckP99High warns at one for the 10-second bound. | none | +| `qbit_prism_share_ledger_partition_lead_rows` | gauge | none | run | Rows of attached share ledger partition headroom above the next share_seq, or -1 when unknown. max(upper_seq) over the attached rows of qbit_prism_share_partitions minus the next share_seq, read by the database collector with the pending-candidate counts (#144). -1 means the collector's last attempt failed or the ledger is not partitioned yet; it is never a report of exhausted headroom. Falling steadily means the maintenance task is not attaching the lead: the next append past the last bound is refused and retried once after attaching it. | none | +| `qbit_prism_stale_job_rejections_total` | counter | `cause=resume_expired,fee_floor,parent_grace,payout_revision,window_not_held` | run | Stale-job share rejections by the internal decision that refused them. Each series counts one existing stale-job decision: resumed-job absolute expiry, CTV relay-fee floor, stale parent or failed stale-grace parent check, and payout-revision mismatch, attributed in that execution order. `window_not_held` (#619) counts a block-bearing proof the candidate enqueue refused before COMMIT because the current primary does not hold the window its coinbase pays: the window's last row is absent or is another share, as after an asynchronous promotion lost the end of the history the work was read from. Nothing was enqueued or offered and the proof's share was not credited; each refusal also logs a WARN naming the block. The ledger counts it at the refusal itself, so a refusal that finishes after the miner's acknowledgement deadline, whose miner was answered `ledger-outcome-unknown`, is counted too. The wire reason and message are unchanged and still counted by `qbit_prism_rejections_total{reason_id="stale-job"}`. Stale-grace credit is not a rejection. Since #478 a block-bearing proof from ordinary work on the active tip is captured whatever its payout revision, so it is no longer counted under `payout_revision`. Still counted: block-only work a same-parent payout replacement retired (its plain shares, and the share of a block captured from it) under `payout_revision`, or `parent_grace` after a tip change; and a capture abandoned before its offer, under `payout_revision` for a ceiling refusal or capture off and `parent_grace` for a lost race. Since #675 `payout_revision` also counts a share-pass submission that passed every submit check, a stale-grace share included, and that the append's payout-revision fence then refused before any write, because a settlement moved the revision first: a plain share, or a block-bearing one with capture off, whose lost block a WARN names. It is counted when the miner is answered `stale-job`, so a block-bearing append still waiting at its acknowledgement deadline (answered `ledger-outcome-unknown`) is not. The series does not separate fence refusals from submit-check ones. Process-local; every series starts at zero. | none | +| `qbit_prism_stale_shares_total` | counter | none | run | Shares rejected as stale or unknown jobs. | `qbit_prism_stale_shares_total` | +| `qbit_prism_stratum_connection_limit` | gauge | none | run | Configured global Stratum connection limit, not currently available permits; -1 before a listener starts. Set from `PRISM_STRATUM_MAX_CONNECTIONS` when a Stratum listener starts. The primary and high-difficulty listeners share this limit and `qbit_prism_connections`. | none | +| `qbit_prism_stratum_connection_refusals_total` | counter | `reason=global_limit,username_limit,ip_limit,malformed_frame_budget,unknown_job_budget,authorize_budget` | run | Stratum connections refused by an admission limit or disconnected by a per-session budget, by closed reason. `global_limit` counts a newly accepted socket closed because `PRISM_STRATUM_MAX_CONNECTIONS` permits were exhausted; `username_limit` counts a `mining.authorize` refused by `PRISM_STRATUM_MAX_CONNECTIONS_PER_USERNAME`; `ip_limit` counts a newly accepted socket closed at `PRISM_STRATUM_MAX_CONNECTIONS_PER_IP`, before any permit, task or query. `malformed_frame_budget`, `unknown_job_budget` and `authorize_budget` count a session disconnected for exceeding `PRISM_STRATUM_MAX_MALFORMED_FRAMES_PER_INTERVAL`, `PRISM_STRATUM_MAX_UNKNOWN_JOBS_PER_INTERVAL` or `PRISM_STRATUM_MAX_AUTHORIZE_ATTEMPTS_PER_INTERVAL`, once per session. Same-username reauthorization, reuse of a retained username permit and every disabled limit never count. Refusals are not share rejections, and no series carries a peer address. Process-local; every series starts at zero. | none | +| `qbit_prism_stratum_current_tip_coverage_gap_seconds` | gauge | none | run | Continuous age of native current-generation coverage below 95 percent, or -1 before observation. | `qbit_prism_stratum_current_tip_coverage_gap_seconds` | +| `qbit_prism_stratum_oldest_pending_initial_job_seconds` | gauge | none | run | Oldest first usable work wait, or -1 before observation. | `qbit_prism_stratum_oldest_pending_initial_job_seconds` | +| `qbit_prism_stratum_pending_initial_jobs` | gauge | none | run | Authorized clients awaiting first usable work, or -1 before observation. | `qbit_prism_stratum_pending_initial_jobs` | +| `qbit_prism_stratum_rebuild_lane_waiters` | gauge | none | run | Local job rebuilds waiting for a rebuild-lane permit; sessions whose work is still on the published parent and payout revision queue here, first jobs never do. | none | +| `qbit_prism_stratum_request_delivery_wait_seconds` | histogram | none | run | Complete Stratum request frame to the end of its session's own job delivery, for a request that waited for that delivery (#621): every JSON object frame but mining.submit and mining.get_health, that is mining.subscribe, mining.authorize, mining.configure, mining.suggest_difficulty, mining.extranonce.subscribe, any other method, or a missing or non-string method; the session reads nothing after it meanwhile, so a submit sent behind it waits as long. Submits and health probes are answered during a delivery and never wait. Default histogram ladder. One observation per waiting request, when its session handles it, whether the delivery it waited for was announced or failed; none when the session ends first. A delivery is the admission, build and persistence of the session's next job and its announcement. A frame that is not a JSON object is answered at once and never waits. A submit queued behind a waiting request is read only after it, and `share_ack_seconds` starts at that read, so this family, not the ACK histogram, carries that wait. Registered at startup: every bucket, `_sum` and `_count` reads 0 until the first wait. No rule of its own. | none | +| `qbit_prism_stratum_semantic_current_work_ratio` | gauge | none | run | Fraction of authorized connections with current semantic work; one when no clients are authorized. -1 before observation; one when no clients are authorized. | `qbit_prism_stratum_semantic_current_work_ratio` | +| `qbit_prism_tip_poll_age_seconds` | gauge | none | run | Monotonic age of this frontend's readiness proof, the node's tip as its last refresh or refresh-grade poll on the published tip found it, which job, share and block admission require within the health timeout; -1 while readiness is unavailable or before the coordinator publishes it. Computed when the health publisher runs, from the monotonic stamp readiness keeps (#622). A refresh renews it when it publishes or revalidates work. A node poll renews it, to when the poll was sent, only when no newer observation has found another tip, it found the published tip, no readiness revocation raced it, the published template is within PRISM_TEMPLATE_MAX_AGE_SECONDS, readiness was not already revoked, no refresh has since found the cluster on a heavier chain, and, under CTV settlement, a relay floor read no older than the poll is recorded and the published work's fee meets it: the refresh's own poll at its start, once that refresh's chain observation and reconciliation agree and it has read the relay floor, and the block wait's, which polls on an unchanged notification only once readiness is at least one interval old and at most once per interval, with PRISM_BLOCKWAIT_ENABLED: a third of PRISM_HEALTH_TIP_POLL_MAX_AGE_SECONDS and at most 5 s (5 s at the default 15 s; a short timeout polls that often), which is also how long the idle block wait waits for a new block. That poll is one readiness check: getblockchaininfo, getnetworkinfo on a public chain (the peer floor) and, under CTV settlement, getmempoolinfo (the relay floor); a failed floor read renews nothing. A healthy frontend reads at most about one poll interval. At PRISM_HEALTH_TIP_POLL_MAX_AGE_SECONDS job preparation defers with `tip polling stale`, share and block admission answer backend-rpc-unavailable and /healthz reports unready. -1 means readiness is unavailable: never established, or revoked by a failed node observation, a stale template, an unavailable fee floor or, under CTV settlement, a relay floor read above the published work's fee (a refresh's or the block wait's), until a refresh restores it; it is never a reading of zero. Read it with `qbit_prism_work_refresh_stalled_seconds`: young here and old there is a stalled rebuild on a responsive node. | none | +| `qbit_prism_work_refresh_stalled_seconds` | gauge | none | run | Monotonic time since this frontend's last successful template refresh, which publishes or revalidates its work, or since start before the first; -1 before the coordinator publishes it. Stamped when Coordinator::refresh_once succeeds (#525), whether it published new work or revalidated the current work; readiness, which job, share and block admission require within the health timeout, is renewed by that success and, since #622, also by a refresh-grade node poll that finds the published tip; `qbit_prism_tip_poll_age_seconds` is its age, so a frontend whose rebuilds stall while its polls answer reads young there and grows here. A healthy frontend reads at most about one poll interval. A refresh that keeps failing, on a new tip or the same parent (for example PayoutExceedsCandidateBalance when sub-floor dust exists without a pool fee, on either settlement path), or that hangs, grows it without bound, while semantic coverage keeps reading one because sessions still hold the latest published generation. Counted from coordinator start until the first success, so a restart into a stall still grows; -1 only before the health publisher first runs. Pages through PrismWorkRefreshStalledCritical. | none | + +<!-- generated-inventory:end --> + +Reject and task labels come from the closed enums rendered in the table above. +Present but unrecognised rejection reason IDs, including an empty string, map +to `unrecognised`; arbitrary input cannot create more than the 14 closed reason +series. Explicit `internal-error` and missing (`None`) reason IDs retain the +`internal-error` label. Every reason counter is initialized to zero, so the first +event is observable by `increase()`. The fallback counts an actual rejection; +it is a label-drift signal, never an accepted share or proof of health. + +Normalization happens only in the submit observation path. The current +reason-less username-limit refusal belongs to `mining.authorize`, so it is +counted by connection-refusal telemetry and never by share-rejection or ACK +metrics. Missing reasons supplied by a submit backend retain their prior +classification. Protocol error codes, messages and reason metadata are unchanged. + +Reason-filtered consumers of `internal-error` now exclude present unknown IDs; +use `unrecognised` to identify reasons the enum does not recognize. Summing all +reasons still counts the same rejections, and the checked-in per-reason rejection +ratio includes the new label automatically. Its grouping can now split a former +`internal-error` total, so the per-reason alert decision can change. The explicit +ledger-confirmation reason filter retains its two existing reasons. No alert +rules are added by this follow-up. + +Session futures and server-owned background futures are monitored. Only the +health publisher currently registers an explicit operation-progress budget, +covering health and metrics publication only. It uses the same effective +`max(3 * PRISM_HEALTH_REFRESH_SECONDS, 15)` seconds as #277 freshness and ends +immediately after publication; later heartbeat/prune waits are outside that +deadline. Those futures still receive ordinary blocked-poll monitoring. +Idle socket reads and asynchronous waits are not blocked polls. A blocked poll +can degrade live `/healthz` and `health_state` when a surviving worker serves the +probe; a completed long poll retains timing evidence but does not keep readiness +failed. An entirely blocked runtime cannot serve HTTP. + +## Failure and compatibility contract + +A failed response write is not an ACK. A durable share can be credited while its +ACK write fails; cancellation before a rejection or durable acceptance produces +neither event. Existing accepted/rejected totals keep their original Stratum +accounting boundary. The grace hook does not reinterpret the caller's grace +hint; it uses the coordinator's stale decision after the database confirms credit. + +Share acknowledgements follow the ledger outcome (#324). A share-pass append +whose COMMIT was already in flight at `PRISM_SHARE_COMMIT_TIMEOUT_SECONDS` can +still be accepted within `share_commit_grace` (5 s); each such acceptance +increments `qbit_prism_late_confirmed_shares_total`. A share-pass submission +carrying a found-block candidate gets the same deadline and grace (#574: never +less than a plain share) but is never refused, so it runs on past an unknown +answer; its late acceptances are counted the same way. The counter is keyed on when the append itself finished, not on when the +acknowledgement was processed. Three outcomes are answered +`ledger-outcome-unknown`, never `ledger-confirmation-failed`: a COMMIT still in +flight after the grace period, a COMMIT failure other than a severity-ERROR +reply, and, under the sync-rep guard, a COMMIT that took at least the ledger +sessions' `statement_timeout`, because that may be a synchronous-replication +wait cancelled after local commit. Such a share may still be credited, and the +warning log names its `share_id`. Unknown answers are rejections under the +protocol, so they count in `qbit_prism_rejected_shares_total`, +`qbit_prism_rejections_total{reason_id="ledger-outcome-unknown"}` and +`qbit_prism_share_ack_seconds{result="rejected"}`. Block-only proofs (a +captured #478 proof or a below-target block, whose share is deferred to its +block's landing) wait for their candidate's disposition up to +`PRISM_SHARE_COMMIT_TIMEOUT_SECONDS` itself (#574; it was at least 60 s +before, and a captured proof's miner could submit nothing else while its +landing ran). A share-pass block whose payout revision moved between its +submit check and its commit is captured by its append instead of refused, and +waits for its block the same way (#657); with capture off +(`PRISM_CAPTURE_OVERPAY_CEILING_BPS=0`) it is refused as before, answered +`stale-job` like a plain share whose revision moved (#675). A +block-bearing submission still +pending at its bound (a block-only proof at the share deadline, a +block-carrying append at the share deadline, or after its grace when COMMIT +was in flight) is +answered with the reply above: Stratum error 20, "share outcome is not yet +known", `reason_id` `ledger-outcome-unknown`. It also increments +`qbit_prism_block_proof_ack_capped_total{path}` and the warning log adds +`elapsed_ms` to the block hash. Only the wait ends there. The duplicate probe, +the candidate enqueue, the append and the landing run on, so the block is +still offered and the share is credited once when it commits. +`share_commit_grace` is not an environment variable. + +Collectors run every ten seconds. Database collection uses a read-only, +repeatable-read transaction with a three-second overall deadline, a two-second +statement timeout, and a 500 ms lock timeout. Failure or cancellation makes the +measurements unknown (-1), sets success/availability to zero, and leaves the +last-success timestamp intact. Observations expire after 30 seconds even if no +new attempt finishes. A newer collection attempt supersedes an older result; +late completion or cancellation cannot replace the newer publication. A real +zero count, age, or RSS remains valid after successful collection. + +Pool timing pre-registers both result labels at count zero. Each started +collector acquisition records one observation: success when acquired, or failure +on acquisition error or cancellation, including the three-second overall +deadline. Since #328 instrumented coordinator ledger transactions record +acquisition the same way, including one abandoned by cancellation. Selected direct +ledger queries now use the same boundary, including payout-revision reads and +heartbeats. Rollup transactions and share-partition maintenance also record +acquisitions. [The acquisition guide](prism-pool-acquire-metrics.md) lists the +source census and explicit exclusions: public-role read pools, private audit +HTTP endpoints and shared API read-model helpers, startup/migration gates, +operator tools and telemetry-free compatibility wrappers. The histogram aggregates +observed checkout attempts at covered callers. Duration is the monotonic elapsed pool wait until +acquisition completes or is cancelled; subsequent transaction work is excluded. +Collector status also records collection failure or cancellation separately. +Candidate count and age describe database time; this is not a monotonic latency +measurement. A/#266 must update the pending predicate if outbox states change. + +`qbit_prism_share_ledger_partition_lead_rows` is read in the same transaction +as those two, so a collection failure leaves all three unknown together. It is +the number of `share_seq` values the attached share ledger partitions still +cover above the next one the sequence will hand out (#144): about 67 million +with the default width and lead, several hours of appends at any rate this pool +has run at. It falls as shares are appended and steps back up each time the +maintenance task attaches a partition, so a value that only falls means the +task is not running or its calls are failing, which +`PRISM_SHARE_PARTITION_ENSURE_INTERVAL_SECONDS` in +[prism-configuration.md](prism-configuration.md) describes. Reaching zero does +not lose a share: the append that finds no partition attaches the lead itself +and retries once. A never-partitioned ledger and a failed read both read -1; +neither is a report of exhausted headroom. + +The pending predicate is the four unfinished outbox states (`pending`, +`offer_reserved`, `offered`, `reconciliation`). Since #415 (migration 015) a +reconciliation row whose block lost a tip race does not stay in that set +forever: once its audit has landed and one coherent tip observation shows a +*different* block active at its height with at least +`PRISM_CANDIDATE_ORPHAN_CONFIRMATIONS` confirmations (default 6, counted from +the observed tip as `tip_height - height + 1`), the post-offer settlement marks +the row terminal as `orphaned`, in the same transaction as its reason and its +block's `inactive` chain state. Both pending gauges then stop counting the row. +The process increments `qbit_prism_block_candidates_orphaned_total` after it +observes a successful commit; cancellation or restart between commit and +recording can lose that increment. The row releases its document, block bytes +and window reference while preserving offer metadata and the reason for the +operator surface (#268); it is never claimed or offered again. A failed observation settles nothing, so +unknown stays distinct from zero: the row waits in reconciliation and the +gauges report -1 only when the collector itself fails. A later reorg that +reactivates the block is credited by the ordinary reorg reconciler from the +landed audit, deferred share included, without reopening the row or touching +the counter. Rows a frontend cannot land (a rebuild refused for good, a missing +balance snapshot) are not orphans and stay in the pending set with their reason +in `last_error`; that backlog is operator work. + +The paging rule does not read that all-unfinished age (#493). The same +snapshot also reports `qbit_prism_block_candidate_oldest_unacknowledged_seconds`, +the oldest age among unfinished rows the node has not accepted: `pending` and +`offer_reserved` rows, offered rows whose one `submitblock` outcome is +`unknown` (a row adopted on the node's own active-chain evidence carries the +adoption reply and is excluded), and rows the node definitively rejected +unless the reply names a side-chain block (`inconclusive`, `duplicate`, +`duplicate-inconclusive`, a lost tip race). `PrismBlockCandidateOldestPendingCritical` +pages on it at 60 seconds, so a found block that is not reaching the node and +a block the node refused (`bad-*`, `high-hash`, `duplicate-invalid`: the pool +built an invalid block) still page exactly as before, and an unknown outcome +keeps paging because receipt is unproven and the block is never offered +again. A node-accepted lost race is unfinished but accepted: it ages only the +all-unfinished gauge, which the 15-second warning still reports and +`PrismBlockCandidateStuck` warns on at 30 minutes without paging. A landing +that keeps failing after the offer is reported by its own gauge, +`qbit_prism_block_candidate_oldest_landing_failed_seconds`, the time since +the offer reservation of the oldest offered or reconciliation row whose landing has not committed (the landing transaction +is the only writer of the block's `qbit_pool_blocks` row, so its absence is +the durable fact, and a transient retry failure or a refused operator +recovery that overwrites `last_error` cannot zero it) or whose `last_error` +carries the landing-failed prefix the coordinator and the collector share (a +landed row that no longer authenticates): `PrismBlockCandidateLandingFailed` warns +at 60 seconds for three minutes and `PrismBlockCandidateLandingFailedCritical` +pages at 307 seconds for one minute, the floor and dwell the revision-work +critical applies when the landing fails after the frontend's own active-chain +observation, so a won block whose ledger landing keeps failing is never +silent whichever step refused. An offered row counts before its first landing +outcome (#510), so a won block whose landing hangs waiting for build capacity +ages that gauge instead of reading zero; a healthy offered row lands within +the 60-second rebuild deadline. A lost race and a rejection land their audit +before the chain verdict and leave that gauge at zero. All four gauges read -1 together when the collector fails. Rollout +order: frontends first, then rules; until a target exports the unacknowledged +series the paging expression falls back to its all-unfinished age, so the +base behaviour (its lost-race page included) persists on that target. + +The existing coordinator families retain their names and types; their complete +contracts appear in the generated table above. The #277 `metrics_snapshot_available`, +`metrics_snapshot_stale`, and `metrics_snapshot_age_seconds` names, labels, +HELP/TYPE lines, headers, and atomic body/timestamp publication remain unchanged. +Deployment alert compatibility was checked against the frozen `SwapLabsInc/qbit-tools` +templates at `a007a14260548cb93e277882672d788fe9a3f95e`. The seven overlapping +families keep their names/types/labels. The [alert migration](prism-alert-migration.md) +replaces legacy rules and joins on retired `stratum_authorized_connections`. The existing +public-role `qbit_prism_public_requests_total` includes `/healthz`, `/metrics`, +and other routed requests, so its rate is not a dashboard-only request rate. + +The coordinator adds the three health compatibility aliases whose native sources +are known; the fixture deliberately identifies unmapped legacy fields: `ready_miner_count` (accepted-share participants) and `max_blocks` (the 2.x accepted-block pool-close cap) have no native health equivalents. + +No new configuration setting is introduced. Worker slots, per-worker series, +payout-build duration remains outside the trimmed metrics scope; the node gauges +and the rollup-lag series in the inventory above were added by the #278 hardening +remainder and still add no setting. #458 adds the landing observation contract below. The subsequent +[#278 cardinality/privacy qualification](prism-metrics-cardinality-privacy.md) +pins the complete run-role wire census and tests identifier-bearing inputs +through Stratum and the candidate collector without expanding the family set. + +## Accepted block to delivered revision work (#458) + +`qbit_prism_accepted_block_to_revision_work_seconds` begins when this frontend +classifies its actual `submitblock` reply as definitive acceptance, or first +obtains a coherent active-chain proof for the block. A recovered unknown offer +starts at that proof, never at an invented earlier acceptance time. A peer +frontend starts at its own proof during reconciliation; these frontend-local +intervals must not be presented as a shared node-acceptance clock. Mature +historical blocks do not start new observations. + +`published` ends after `write_all` successfully writes the complete first +`mining.notify` frame whose job carries the associated post-landing revision. +This is socket-write completion, not an acknowledgement from mining hardware. +Prepared publication, job construction, issuance persistence, first node offer +and a failed socket write are not completion. The local settlement observer +uses the revision from the same committed transaction, including a no-op +confirmation or reactivation. A peer proof associates the first revision this +frontend can prove includes that block; it does not reconstruct an unseen +original confirmation revision. An in-flight local settlement cannot be +replaced by this peer-proof association. If its reply is delayed, actual write +and revision-observation clocks are retained so later association preserves +which event happened first. + +A definitive `submitblock` acceptance is not an active-chain observation: the +block may have lost a tip race (#493). Such an acceptance is tracked as +**unlanded**, reported by `qbit_prism_accepted_block_unlanded_seconds` (the +age of the oldest one; zero when none; -1 only when none is tracked and +identity tracking is saturated) and never part of the known pending age or of build-deadline +attribution, because it has no revision work to deliver. The post-offer +landing observation, a reconcile proof or a settlement observation lands it: +the same identity then joins the known pending age on its original +acceptance clock, so the histogram still measures from the definitive +acceptance. A committed proven orphan closes it without a sample. Only the +offering frontend ever holds an unlanded acceptance; a peer starts at its own +active-chain proof. An unlanded acceptance is a known state, not unknown +tracking, with one deliberate exception: while the terminal probe that +carries peer orphan evidence fails or is cancelled, an unlanded-only +frontend reads tracking unknown and a -1 pending age, because the lost +race's orphan evidence is unreadable; the unlanded gauge keeps the age and +the next successful read restores the known state. A build deadline +attributed through another known wait never marks an unlanded block +degraded: it has no revision work yet. + +`superseded` means a later revision was observed before delivery of the target +revision. Its histogram interval still ends at actual replacement-work delivery, +never at the revision observation. The pending gauge retains the original +acceptance age through supersession and a second accepted block, until a +successful write of compatible current replacement work. A late write of an +already superseded revision cannot clear that wait. Thus stalls are visible +before the first completed histogram sample. The gauge is computed from a +monotonic clock at render time and overlaid on cached metric bodies. +With no connected miners there is no qualifying notify delivery: pending age +continues to rise even when prepared work is ready, and the warning rule +reports it. The metric does not reinterpret an empty listener as successful +miner delivery. Every frontend observes every active immature block from its +own reconcile proof, so each frontend carries its own wait for the same block +and a sum across instances counts one block once per frontend; the paging rule +therefore requires an authorized miner on the reporting frontend (#493). + +`qbit_prism_revision_work_build_timeouts_total` increments only in the existing +Stratum job-build timeout branch, once for that operation when a known +accepted-block wait accepted before the build was open at its start or when +the deadline fired. Its budget includes the existing admission wait. Only +known waits attribute a deadline (#493): while this frontend's tracking is +unknown and no known wait remains (a lost or cancelled settlement COMMIT +reply, an unsettled orphan verdict, bounded-tracking saturation or lost +ordering history), a build deadline is an ordinary +`qbit_prism_job_delivery_failures_total` event; the tracking-unknown gauge +reports the unknown state, the pending gauge stays -1, and the deadline is +neither counted here nor stamps a wait recovered after it `degraded`. A wait +recovered by a proven first confirmation before the deadline is counted and +`degraded` as usual. +Success, ordinary errors, cancellation, node-offer timeouts and issuance or +socket-write deadlines are not counted as job-build timeouts. A subsequent +qualifying delivery is `degraded` if an applicable build deadline was hit before +that write. A timeout of an older operation cannot degrade a later acceptance. +Native refresh currently has no overall deadline and this instrumentation adds +none; no refresh deadline or fallback-delivery policy is inferred from elapsed +time. The histogram differs from both proof-to-first-offer and the #275/#444 +harness approximation, which do not observe this actual socket boundary. + +Tracking is process-local and bounded to 4,096 accepted block identities, +4,096 revisions and 4,096 delivery timestamps while delivery remains unresolved. +Repeated writes at the same revision share a timestamp until another acceptance +arrives, preserving the first eligible write for each acceptance. Completed identities +are retained as deduplication tombstones until a coherent mature accounting +checkpoint retires their height. A monotonic height watermark then rejects +delayed proofs even after the identity is released; unresolved intervals are +never retired by that checkpoint. Revision events are released when all known +deliveries resolve. Nothing is evicted to make a duplicate look new. +Exceeding a bound sets `accepted_block_revision_work_tracking_unknown` to +**1 until restart**. Missing identities have no fabricated observations; known +pending ages remain visible after identity-only saturation. Lost revision or +delivery history additionally sets pending age to -1 and suppresses terminal +samples because supersession or the first delivery can no longer be proven. +Within this horizon, +a tracked block has at most one terminal sample, and exactly one once its +revision and qualifying delivery are both known. Process restart loses +unresolved intervals and deduplication history, and begins new local proof +clocks rather than replaying peer timestamps. If a local settlement commits but +its exact-revision reply is lost or cancelled, tracking is unknown and the interval +stays unresolved: a current-revision proof cannot safely reconstruct that +original revision. A later proven first confirmation can recover an attempt +that did not commit; an already-confirmed replay cannot invent the lost fact. +The same unknown boundary applies when a peer confirms a locally accepted +block during audit landing, before the local settlement observer starts. Only +a newly observed already-confirmed peer block may begin at its current local +proof revision; a later proof cannot upgrade an earlier unresolved acceptance. +Universal exactly-once reporting beyond these knowledge/capacity boundaries +requires durable observation metadata and is not claimed here. + +Unknown tracking is independent of known pending age: an unknown interval A +cannot hide a measured interval B crossing the critical threshold. Pending age +is -1 when no known interval remains but tracking is unknown, and zero only +when the observed empty state is known. The separate unknown gauge stays one +even while a known interval's positive age is reported. A rejected transaction +before COMMIT does not invent a lost outcome. Operator recovery uses the same +committed revision evidence as ordinary settlement, and a proven first +confirmation remains authoritative regardless of observer arrival order. + +A successfully committed proven orphan closes its delivery wait without a +histogram sample: it has no eligible delivery target. Its identity remains a +deduplication tombstone through the same mature watermark, including if later +chain reconciliation reactivates it; this does not change payout credit or +block-confirmation counting. The shared ledger observer arms uncertainty only +at the existing orphan COMMIT boundary, so a rejected precommit verdict leaves +the measured wait intact. An in-flight, lost or cancelled orphan COMMIT cannot +be relabeled as successful delivery or upgraded by a later revision proof. + +Each frontend discovers peer-settled orphans through the existing ten-second +metrics collector cadence. One extra read-only lookup checks at most 4,096 +locally unresolved hashes against the terminal outbox state; it is skipped +when that set is empty and shares the existing three-second total collection +deadline and read-only snapshot. No background loop, accounting lock, schema, +write, or work-publication dependency is added. Positive durable orphan +evidence closes only a still-tracked identity, even if its original COMMIT +reply was lost; negative evidence never guesses the original outcome. +A failed or cancelled terminal read preserves pending state and exposes +tracking uncertainty, and an older completion cannot erase a newer failure. +Before the next successful collection, the accepting frontend still reports +its locally known wait; this is a local knowledge boundary, not the peer's +orphan commit time. + +A failed or cancelled refresh preserves known pending age. If no unresolved +acceptance is known, that failed observation yields -1 rather than a fabricated +healthy zero; a later successful refresh restores the known empty zero. An +older cancelled refresh cannot overwrite a newer completed observation. No +block, job, worker, frontend or height label is emitted. + +There are **five concrete alert rules**: `PrismAcceptedRevisionWorkPending` +warns above one second and for a missing pending series or a failed scrape, +`PrismAcceptedRevisionWorkTrackingUnknown` warns for unknown tracking (a -1 +age, the tracking-unknown gauge or a missing unknown gauge) after a two-minute +dwell, `PrismAcceptedRevisionWorkPendingCritical` uses the provisional +307-second incident-duration floor, `PrismRevisionWorkBuildTimeouts` warns on +a five-minute increase, and `PrismAcceptedBlockUnlanded` warns when an +unlanded acceptance is 30 minutes old for three minutes. The budget and +timeout warnings have zero additional dwell and alert on no data, so a +missing measurement is always visible. The unknown warning dwells (#493 +point 7): a failed template refresh marks tracking unknown only until the +next successful refresh, one poll interval later, so a single transient +refresh error never spans the eight consecutive 15-second scrapes the dwell +requires, while a lost settlement reply, an unsettled orphan verdict, +saturation or lost ordering persist and still warn. The paging +critical is gated (#493): the age must be known, the target scraped and at +least one miner authorized on that frontend, with a one-minute dwell and +no-data and evaluation errors treated as OK. It is deliberately not gated on +the metrics snapshot: the pending gauge is a live overlay rendered on every +scrape, and a database-pool stall that also stalls the health publisher must +still page. The miner count is a snapshot family, so while the snapshot is +stale the gate uses a count a few refresh intervals old; a false page on a +frontend whose miners left inside that window is accepted over a missed +stall. The dwell only absorbs a brief gate gap on a gauge that is monotonic +during a stall and stays well under the five-minute hold that delayed #413. +If every miner leaves a frontend during a stall, this page resolves and +nothing else pages for that frontend: the coverage rules cannot fire with +zero authorized clients and the connected-clients rule is a cluster-wide +non-paging warning, so only the pending warning keeps reporting the age until +a miner reconnects and receives work. A lost race on the offering frontend +never pages (#493 point 1): its acceptance is unlanded, outside the known +age, until an active-chain observation lands it or its orphan is proven. A +routine lost race is proven within a few block intervals (six competitor +confirmations at the 60-second target spacing plus the reconciliation retry +backoff), well inside the 30-minute unlanded warning; past that the row is +not being reconciled, the node is not advancing or the orphan verdict keeps +failing, which is warned operator work, never a page. The one-second warning +is a budget target; 307 seconds is not evidence of early detection. Scrape and +evaluation intervals still add delay. Thresholds must be measured in #291; +this repository change neither applies the generated external patch nor +claims operational deployment. Five historical definitions consolidate into +these native rules, with the two timeout severities sharing one warning. + +The required PostgreSQL/fake-node/socket tests are in +`crates/qbit-prism-server/tests/landing_metrics.rs`; state ordering, saturation, +unknown measurements and live cached-body overlays are in +`crates/qbit-prism-server/src/metrics/landing/tests.rs`. The independent census +is 51 run-role families, 214 startup series and 312 populated series on this +base, with exactly these four new families relative to #450; #493 adds the +unlabeled unlanded gauge and the unlabeled unacknowledged and landing-failed +candidate ages. + +## Diagnosing Stratum admission saturation + +These signals observe the existing limits; they do not add limits or change +readiness. + +- `qbit_prism_stratum_connection_limit` is the configured + `PRISM_STRATUM_MAX_CONNECTIONS`, shared by the primary and high-difficulty + listeners, or -1 until a listener starts. It is capacity, not free permits: + compare it with `qbit_prism_connections` to see the remaining room. +- `qbit_prism_stratum_connection_refusals_total{reason="global_limit"}` + increases when a new TCP connection is closed at the limit. The miner sees an + accepted connection closed without any JSON-RPC response. A rising count with + `qbit_prism_connections` at the limit is admission saturation. +- `reason="username_limit"` increases when `mining.authorize` returns + `[20,"too many connections for username",null]` under + `PRISM_STRATUM_MAX_CONNECTIONS_PER_USERNAME`. The connection stays open and + keeps any earlier authorization. Work still retained for that username keeps + its slot until it expires, so a quick reconnect can be refused briefly. + Same-username reauthorization is never counted. +- `reason="ip_limit"` increases when a newly accepted socket is closed at + `PRISM_STRATUM_MAX_CONNECTIONS_PER_IP`. Like `global_limit` the miner sees an + accepted connection closed with no JSON-RPC response, and the decision is + taken at accept, before any permit, session task or database query. The label + never names the address; the matching structured tracing event carries + `peer`, so search the log when this rises. Both listeners share one budget + per address. See + [prism-b4-stratum-admission.md](prism-b4-stratum-admission.md) before + raising the cap. +- `reason="malformed_frame_budget"`, `"unknown_job_budget"` and + `"authorize_budget"` increase once per session, when a per-session rate + budget disconnects it. The session receives its error response first, then + the close. A rise with no operator change means either a misbehaving client + or a budget set below an honest rate: an unknown-job burst is normal + immediately after a tip or payout-revision change, so compare it with + `qbit_prism_rejections_total{reason_id="unknown-job"}` before lowering + anything. +- Reconnects with no refusal increase, and connections below the limit, point + to the transport path or the miner rather than server admission. +- Readiness (`qbit_prism_health_state`, `mining.get_health`) says whether this + instance can serve work to miners already connected. It does not reserve + spare connection capacity: a ready instance at its limit still refuses new + sockets. + +Both counters are process-local, start at zero for every reason so `increase()` +sees the first refusal, and appear in the cached body with the other Stratum +series at its next publication. Neither count is a share rejection. + +## Follow-up ownership + +First-offer timing is populated by A/#266's offer owner. It remains declared +without samples until an actual first-offer observation occurs; this startup +state is included in the cardinality qualification. +Since #328, instrumented coordinator ledger transactions record into +`database_pool_acquire_seconds`, and the coordinator's migration, order and +settlement advisory-lock acquisitions record into +`database_advisory_lock_wait_seconds`; the CPFP funding lock is not timed. +Covered non-transaction ledger queries are timed through `Ledger::acquire`; +rollup and share-partition maintenance use the same acquisition timer. The +[acquisition guide](prism-pool-acquire-metrics.md) records the source census, +explicit exclusions and CI guard added for #352. The histogram counts observed +checkout attempts at covered callers, not all process acquisitions. The native alert +specification attaches no firing rule to first-offer timing: `block_submit_seconds` +is observed at most once per found block (#391), too sparse for a rate or +percentile alarm. Order and settlement advisory-lock waits are covered by +`PrismDatabaseAdvisoryLockWaitHigh` and `PrismDatabaseAdvisoryLockFailuresCritical`; +`lock="migration"` has no rule because that lock is taken only at schema +initialization, where waiting out an online migration is expected. +`PrismDatabasePoolWaitHigh` describes both collector and instrumented ledger +acquisitions, including cancellation observations from #328 and #345. The collector +acquires from the same pool as the ledger, so pool exhaustion can fail collection +while the histogram records valid waits. The health publisher also awaits this +pool, so #351 overlays the live pool histogram on every scrape, including before +the first publication and while the cached body is stale. All buckets, count and +sum come from one registry read; rendering does not create observations or renew +the cached-body timestamp. This rule requires a successful scrape (`up == 1`) +and the existing per-instance minimum of ten observations in five minutes, +independently of collector or cached-body availability. Failed or missing scrapes +suppress it, and frozen counts age out of the sample window. Cached gauges retain +their freshness gates and stale/missing snapshots retain their unknown-data alerts. +Deploy the live-histogram producer to every coordinator target before activating +this expression: older producers still cache pool observations, so their successful +scrapes alone cannot establish this contract. The landing order for #336 is #334, +then #343, then #351; the first two have landed. +#328 references #278 rather than closing it. + +The #280 rebase must preserve the narrow hooks in `stratum::request` (complete +frame time, rejection decision, and successful write) and the durable +`MiningBackend::submit` branch in `coordinator.rs`; its extracted +`coordinator/miner_submit.rs` inherits the latter. No metric family is waiting +for #280 to become populated. + +The coverage-gap threshold preserves the strict `< 0.95` boundary of the +[2.x producer](https://github.com/Qbit-Org/qbit-mining-bootstrap/blob/504846cc0b72e8f86ed17f896d4ccbbe196a31dc/lab/prism/observability.py#L317). +Its native observed boundary is authorized clients holding current-generation +work (semantic coverage); it is a metrics-only timer and does not change health +readiness decisions. This preserves the legacy alert threshold/reset semantics. + +Collector gauges, the pool histogram and, since #581, `qbit_prism_accepted_shares_total` +and `qbit_prism_rejections_total` are refreshed from one registry read at scrape +time, including when the health publisher stalls. This does not renew #277's +cached-body publication timestamp or the collector's last-success time. Because +the two share counters stay current while the cached body is stale, +`PrismShareAppendFailures` and `PrismRejectRatioByReasonHigh` are no longer gated +on a fresh snapshot, and a PostgreSQL outage, which stalls the publisher, holds +them instead of flapping with the snapshot's freshness. The outage itself pages +on `PrismDatabaseUnavailable` (`qbit_prism_collector_available{collector="database"}`, +two minutes, gated on a successful scrape only). + +Registry injection is required by `ApiState::new`, `Coordinator::new`, and +`run_listener`. Primary and high-difficulty listeners explicitly receive the same +registry; listener configuration no longer creates its own telemetry state. +Four legacy block-processing rejection labels (`candidate-audit-mismatch`, +`submitblock-rejected`, `block-stale`, `ledger-confirmation-superseded`) have no +producer in this slice and are excluded until their owning follow-up wires them. + +The ACK clock starts when this session reads a complete frame from its buffered +socket, not when bytes first reach the kernel. A pipelined second submit queued +behind earlier request work is therefore not a measure of the miner's full wait. +`runtime_lag_seconds` reports only the latest sampler tick, which can return to +near zero within 100 ms after recovery; completed tracked polls retain their +maximum separately in `runtime_poll_lag_seconds`. The health `ledger_backend` +alias uses `postgres-native`, matching the other native API responses, while the +existing `backend` field retains its storage-engine value `postgres`. + +## Reading the node gauges and the rollup lag + +`qbit_prism_node_peers`, `qbit_prism_node_initial_block_download` and +`qbit_prism_node_observation_age_seconds` come from one observation site, the +readiness chain check, and record only what that attempt already fetched. They +add no RPC call, so a check that short-circuits before `getnetworkinfo` - +regtest, a node in initial block download, or a node whose blocks and headers +disagree - leaves the peer count unknown at -1. Unknown never means the last +reading: a refused `getnetworkinfo` reports -1 rather than the peer count of an +earlier attempt, so the gauge cannot advertise a peer floor nobody confirmed. An +answered zero is a real zero, and it is below every configured `PRISM_MIN_PEERS`. + +The sync flag is recorded whenever `getblockchaininfo` answers with the boolean, +including the 1 that then makes readiness fail, and reads -1 only when that call +failed or answered with something that is not a boolean. The observation age is +stamped on that answered call rather than on a fully successful check, so a node +that answers but is not ready keeps a small age while an unreachable node's age +grows without bound. It is monotonic and computed at scrape time. Concurrent +refresh, block-wait, candidate and broadcaster observations are ordered by the +instant each attempt began, so a slower older attempt landing late cannot roll +back a newer observation. That ordering governs the peer count and the sync +flag, which belong to one attempt; the age instead keeps the latest answer any +attempt received, so a superseded attempt whose call answered last still proves +the node was reachable then. + +`qbit_prism_hashrate_rollup_watermark_lag_seconds` is not derived from the +stored watermark: `qbit_hashrate_rollup_progress.last_share_seq` is a share +sequence, not a time, and its `updated_at` uses the database clock. The series +is instead the monotonic time since this frontend last completed a rollup pass +that advanced the watermark without filling its batch bound, which is the pass +that left no unfolded share behind. An idle pool scans nothing and still +advances, so it stays caught up; a backlog, a failing pass or a stopped loop +grows the lag without bound. Competing frontends race, and the loser advances +nothing, so a frontend that keeps losing reads high while the cluster is healthy: +alert on the minimum across instances, never on a single one. The series reads +-1 from loop start until its first caught-up pass, and while +`PRISM_HASHRATE_ROLLUP_ENABLED=0` the family is declared without any sample, +because a frontend that never runs the rollup has no lag to report. diff --git a/docs/prism-overload-alerts.md b/docs/prism-overload-alerts.md index d1e602a47..b0755796e 100644 --- a/docs/prism-overload-alerts.md +++ b/docs/prism-overload-alerts.md @@ -1,5 +1,12 @@ # PRISM Overload and Backlog Alert Specification +Historical specification for the removed Python coordinator. Its scheduler, +delivery, and backlog metric names do not describe the native runtime. Use the +current [native metric inventory](prism-native-metrics.md) and +[deployed-alert migration](prism-alert-migration.md) when adapting deployment +alerts for Rust. The remaining document records the original rationale; +its undeployed rules are not the native alert specification. + Issue #188. This document specifies the alerting rules for PRISM's landing-path, delivery, and backlog overload classes. It is a specification only. @@ -70,7 +77,7 @@ unresolved-transitions count. What the gauge task adds and corrects: **Adds the cap.** Before it, the configured unresolved-depth cap -(`PRISM_ACCEPTED_PARENT_UNRESOLVED_DEPTH_MAX`, default 8, in +(`PRISM_ACCEPTED_PARENT_UNRESOLVED_DEPTH_MAX` <!-- retired-setting: PRISM_ACCEPTED_PARENT_UNRESOLVED_DEPTH_MAX -->, default 8, in `lab/prism/coordinator_config.py`) was readable only from process configuration, so no rule could express "at or near the cap" without hard-coding a deployment-specific constant. Exporting the cap as @@ -298,7 +305,7 @@ Severity convention: `warning` means investigate within the working day; ``` - **`for`:** `1m` (warning), `5m` (critical) - **Interpretation / first action:** 15 s is the default - `PRISM_HEALTH_PENDING_REFRESH_MAX_AGE_SECONDS`, i.e. the point at which + `PRISM_HEALTH_PENDING_REFRESH_MAX_AGE_SECONDS` <!-- retired-setting: PRISM_HEALTH_PENDING_REFRESH_MAX_AGE_SECONDS -->, i.e. the point at which the coordinator's own health evaluator considers the refresh overdue. Firing here means the process is already reporting itself degraded on this axis. If the deployment overrides that environment variable, the @@ -415,8 +422,8 @@ Before these rules are trusted on mainnet: 2. Recompute the *cadence-dependent* thresholds only — the critical timeout count in rule 1 and both candidate-count thresholds in rule 4. 3. Leave the configuration-derived thresholds alone unless the underlying - defaults change; if `PRISM_ACCEPTED_PARENT_UNRESOLVED_DEPTH_MAX`, - `PRISM_HEALTH_PENDING_REFRESH_MAX_AGE_SECONDS`, or the metrics refresh + defaults change; if `PRISM_ACCEPTED_PARENT_UNRESOLVED_DEPTH_MAX` <!-- retired-setting: PRISM_ACCEPTED_PARENT_UNRESOLVED_DEPTH_MAX -->, + `PRISM_HEALTH_PENDING_REFRESH_MAX_AGE_SECONDS` <!-- retired-setting: PRISM_HEALTH_PENDING_REFRESH_MAX_AGE_SECONDS -->, or the metrics refresh interval is overridden in deployment, update rules 2, 6, and 7 respectively. 4. Confirm the target build contains #184 — that a stale complete snapshot @@ -490,7 +497,7 @@ reads as "two filled an empty backlog, four were declined". ### Configuration knob -`PRISM_BLOCK_CANDIDATE_CLEANUP_RETRY_BACKLOG_MAX` (default +`PRISM_BLOCK_CANDIDATE_CLEANUP_RETRY_BACKLOG_MAX` <!-- retired-setting: PRISM_BLOCK_CANDIDATE_CLEANUP_RETRY_BACKLOG_MAX --> (default `DEFAULT_BLOCK_CANDIDATE_CLEANUP_RETRY_BACKLOG_MAX = 4096`, loaded into `BlockConfig.candidate_cleanup_retry_backlog_max` in `lab/prism/coordinator_config.py`). Startup refuses a non-integer, a @@ -687,8 +694,9 @@ warning or the 5 s accepted-parent child wait budget still-exact accepted-share window (`force_prior_balances_read=True`, `force_full_window_rescan=False`) instead of the O(window) oracle rescan. This rests on the verified invariant that reconciliation never writes - `qbit_share_ledger` (proved in `test/test-prism-postgres-ledger.sh`, - `share-ledger-identity=inactive+reactivate+mature`). Every other full + `qbit_share_ledger` (proved on 2.x.x in `test/test-prism-postgres-ledger.sh`, + `share-ledger-identity=inactive+reactivate+mature`; the 3.x.x native ledger + suite runs through `make test-prism-postgres`). Every other full rescan trigger is unchanged, including the fail-closed one below. - The landing, the reconciler and the payout-window oracle record fixed-cardinality attribution families (below). Every label is a closed @@ -853,7 +861,7 @@ sum by (reason, path) (increase(qbit_prism_payout_window_full_rescan_seconds_cou During normal forward progress expect `cold_start` once per process start, one of the `periodic_self_check*` reasons at most about once per -`PRISM_PAYOUT_ARTIFACT_FULL_RESCAN_SECONDS` (default 3600 s), and every +`PRISM_PAYOUT_ARTIFACT_FULL_RESCAN_SECONDS` <!-- retired-setting: PRISM_PAYOUT_ARTIFACT_FULL_RESCAN_SECONDS --> (default 3600 s), and every other cell, `reconcile_invalidation` included, at zero. A successful #224 reread has **no family of its own**: it is one `qbit_prism_ledger_read_calls_total{operation="current_prior_balances"}` diff --git a/docs/prism-payout-artifact-measurement.md b/docs/prism-payout-artifact-measurement.md index 2698d892a..5e5db9e3e 100644 --- a/docs/prism-payout-artifact-measurement.md +++ b/docs/prism-payout-artifact-measurement.md @@ -1,485 +1,491 @@ -# PRISM payout-artifact before/after measurement +# Measure native Prism performance -This is a production runbook for a human operator. Every database statement is -read-only. Do not run it from a development workspace, and do not reset -`pg_stat_statements`: compare cumulative snapshots by `queryid` instead. +Measure the Rust deployment with the intended payout window, miner population, +database, and frontend count. The removed Python incremental-window scheduler, +subprocess builder, and its event names are not measurement targets for the +native server. A faster synthetic builder result alone does not demonstrate +miner-facing capacity or HA durability. -Use matching observation windows (one hour minimum; 24 hours preferred) before -and after rollout. Record the rollout commit and image digest, UTC start/end, -tip count, accepted-share count, connected-miner range, and network difficulty. -Keep all raw CSV and log extracts with the rollout evidence. +## Native builder benchmark -## 1. Capture `pg_stat_statements` +The built-in benchmark builds and verifies actual signed audit bundles in +process using synthetic shares: -Run both captures immediately before and after each observation window. Save -the reset timestamp beside each CSV; any change in `stats_reset` invalidates -the subtraction (restart the observation window instead of comparing across a -Postgres restart or statistics reset): - -```sql -SELECT now() AT TIME ZONE 'UTC' AS captured_at_utc, stats_reset -FROM pg_stat_statements_info; +```sh +cargo run --locked --release -p qbit-prism-server -- benchmark \ + --shares 100000 --miners 100 --iterations 20 \ + --output-json /tmp/prism-native-builder.json ``` -The capture keeps the overall top 25 plus every known full-window, delta, and -carry-balance query even when one falls outside the top-N at one boundary: +Record the commit, binary/image digest, compiler/build mode, CPU allocation, +memory limit, and exact dimensions. Output schema +`qbit.prism.native-builder-benchmark.v1` reports: + +- `build_and_verify_p50_ms` and `build_and_verify_p99_ms` +- `canonical_audit_bytes` +- `shares`, `miners`, `iterations`, and `engine` + +The timings include native construction and verification. They exclude Stratum +networking, PostgreSQL commits, node RPC, multi-instance contention, and durable +CTV broadcasting. The benchmark uses direct settlement with synthetic data; +measure the configured CTV path in a real integration run as well. A percentile +from few iterations is descriptive, not a well-sampled latency tail. + +## Miner-facing load measurement + +Use controlled miners or a load generator that submits valid work to the actual +Stratum listeners. Keep load bounded and run against isolated test infrastructure +before a production canary. Measure at least: + +| Measurement | Why it matters | +| --- | --- | +| Valid submissions, successful ACKs, unique committed proofs | Establishes throughput and accounting reconciliation | +| ACK p50/p95/p99 and maximum | Captures database, target validation, and routing latency | +| Subscribe/authorize to initial difficulty/job | Detects reconnect admission or delivery stalls | +| Tip observation to fresh job delivery | Measures usable mining work under refresh load | +| RSS and CPU per frontend, peak concurrent builders | Shows actual resource use and window memory cost | +| Database lock waits, pool utilization, statement latency | Reveals the shared database bottleneck | +| WAL bytes and table/index growth | Quantifies durability/storage cost | +| Candidate submit-to-active and active-to-accounted latency | Separates node acceptance from durable settlement | +| Reclaimed candidate/CTV work after interruption | Exercises recovery rather than process uptime alone | + +Capture steady state, reconnect bursts, slow database service, a frontend +restart, and actual HA database failover if zero acknowledged-share loss is a +requirement. Reconcile unique ACKed proofs against database identifiers after +in-flight transactions have resolved. Do not count exact duplicate resubmissions +as additional accepted work. + +Record the same workload against one and several frontends, with total miner +load and total CPU allocation stated separately. More frontends provide routing +and process redundancy, but every share still crosses one PostgreSQL commit and +ordering boundary. Report measured scaling; do not infer it from thread count. + +The retained [optional v3 qualification validator](prism-capacity-readiness.md) +checks a strict evidence record. The repository's synthetic builder benchmark +does not generate a qualification artifact or replace the complete load runner. + +## Compare database query costs + +If `pg_stat_statements` is enabled in the reviewed database profile, capture +cumulative counters at the start and end of matching observation intervals. +Do not reset shared production statistics. A reset invalidates the subtraction: ```sql -COPY ( - WITH normalized AS ( - SELECT - queryid, - calls, - total_exec_time, - mean_exec_time, - rows, - shared_blks_hit, - shared_blks_read, - shared_blk_read_time, - temp_blks_read, - temp_blks_written, - regexp_replace(query, E'\\s+', ' ', 'g') AS normalized_query - FROM pg_stat_statements - WHERE dbid = ( - SELECT oid FROM pg_database WHERE datname = current_database() - ) - ), ranked AS ( - SELECT *, row_number() OVER (ORDER BY total_exec_time DESC) AS total_rank - FROM normalized - ) - SELECT - now() AT TIME ZONE 'UTC' AS captured_at_utc, - queryid, - calls, - total_exec_time, - mean_exec_time, - rows, - shared_blks_hit, - shared_blks_read, - shared_blk_read_time, - temp_blks_read, - temp_blks_written, - left(normalized_query, 500) AS query - FROM ranked - WHERE total_rank <= 25 - OR normalized_query LIKE '%WITH RECURSIVE pages AS (%' - OR ( - normalized_query LIKE '%qbit_share_ledger%' - AND normalized_query LIKE '%accepted_at > to_timestamp%' - AND normalized_query LIKE '%job_issued_at > to_timestamp%' - ) - OR normalized_query LIKE '%qbit_current_carry_forward_balances()%' - ORDER BY total_exec_time DESC -) TO STDOUT WITH (FORMAT CSV, HEADER TRUE); -``` - -For each `queryid`, subtract the opening snapshot from the closing snapshot. -Report calls, total execution time, shared blocks read, shared-block read time, -and temp blocks. Calculate the interval mean as -`total_exec_time_delta / calls_delta`. Keep separate filtered views for the -pages oracle, delta query, and carry aggregate. If a captured queryid is absent -from the explicit filters at either boundary, mark it not comparable and fix -the capture before the next window; top-N membership alone must never imply a -zero counter. - -Expected after rollout: the pages CTE runs only for cold start, explicit -reconcile/correction fallback, invariant fallback, and periodic self-check. It -should disappear from steady-state top consumers. - -The carry-balance aggregate should likewise be limited to cold publication or -forced payout mutations. Normal and debounced artifact logs should report -`prior_balances_source:"published"`. - -## 2. Explain the full-rescan oracle - -Take `anchor_ms`, `window_shares`, and network difficulty from one coordinator -build event. Set `window_weight` to `16 * network_difficulty`. Use the same -window weight before and after rollout. Run the following in `psql`, supplying -integer variables `anchor_ms` and `window_weight`: - -`EXPLAIN ANALYZE` executes the query. Prefer a production replica with -representative data; otherwise run off-peak with operator approval and retain -the statement timeout below. It is read-only, but it can still consume material -I/O and CPU. +SELECT now() AT TIME ZONE 'UTC' AS captured_at_utc, stats_reset +FROM pg_stat_statements_info; -```sql -BEGIN READ ONLY; -SET LOCAL statement_timeout = '5min'; - -EXPLAIN (ANALYZE, BUFFERS, FORMAT TEXT) -WITH RECURSIVE pages AS ( - SELECT page.min_share_seq, - page.page_weight, - page.page_weight AS cumulative_weight - FROM LATERAL ( - SELECT min(page_rows.share_seq) AS min_share_seq, - COALESCE(sum(page_rows.share_difficulty), 0)::numeric AS page_weight - FROM ( - SELECT ledger.share_seq, ledger.share_difficulty - FROM qbit_share_ledger ledger - WHERE ledger.accepted - AND ledger.job_issued_at <= - to_timestamp((:anchor_ms)::double precision / 1000.0) - AND ledger.accepted_at <= - to_timestamp((:anchor_ms)::double precision / 1000.0) - ORDER BY ledger.share_seq DESC - LIMIT 4096 - ) page_rows - ) page - UNION ALL - SELECT page.min_share_seq, - page.page_weight, - pages.cumulative_weight + page.page_weight - FROM pages - CROSS JOIN LATERAL ( - SELECT min(page_rows.share_seq) AS min_share_seq, - COALESCE(sum(page_rows.share_difficulty), 0)::numeric AS page_weight - FROM ( - SELECT ledger.share_seq, ledger.share_difficulty - FROM qbit_share_ledger ledger - WHERE ledger.accepted - AND ledger.job_issued_at <= - to_timestamp((:anchor_ms)::double precision / 1000.0) - AND ledger.accepted_at <= - to_timestamp((:anchor_ms)::double precision / 1000.0) - AND ledger.share_seq < pages.min_share_seq - ORDER BY ledger.share_seq DESC - LIMIT 4096 - ) page_rows - ) page - WHERE pages.cumulative_weight < (:window_weight)::numeric - AND pages.min_share_seq IS NOT NULL -), -page_cutoff AS ( - SELECT min(min_share_seq) AS min_share_seq - FROM pages - WHERE min_share_seq IS NOT NULL -), -ranked AS ( - SELECT ledger.*, - sum(ledger.share_difficulty) OVER ( - ORDER BY ledger.share_seq DESC - ROWS BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW - )::numeric AS cumulative_difficulty - FROM qbit_share_ledger ledger - CROSS JOIN page_cutoff - WHERE ledger.accepted - AND ledger.job_issued_at <= - to_timestamp((:anchor_ms)::double precision / 1000.0) - AND ledger.accepted_at <= - to_timestamp((:anchor_ms)::double precision / 1000.0) - AND ledger.share_seq >= page_cutoff.min_share_seq -), -rows AS ( - SELECT * - FROM ranked - WHERE cumulative_difficulty - share_difficulty - < (:window_weight)::numeric -) -SELECT count(*), min(share_seq), max(share_seq), sum(share_difficulty) -FROM rows; - -ROLLBACK; +SELECT queryid, calls, total_exec_time, rows, shared_blks_hit, + shared_blks_read, temp_blks_read, temp_blks_written, query +FROM pg_stat_statements +WHERE dbid = (SELECT oid FROM pg_database WHERE datname = current_database()) +ORDER BY total_exec_time DESC; ``` -Capture execution/planning time, recursive loops, shared hit/read blocks, read -I/O time, and temp blocks. The isolated full oracle may remain expensive; the -success criterion is that its call frequency collapses. - -After rollout, also explain one normal delta query from -`pg_stat_statements`. Confirm both timestamp-range branches use indexes, the -returned rows approximate `delta_rows`, and no recursive pages CTE appears. - -## 3. Extract artifact events - -Use the exact UTC observation range. Preserve the original timestamped log and -also extract parseable coordinator payloads: +Capture every query row or explicitly retain all relevant ledger, snapshot, +carry-forward, audit, and outbox queries at both boundaries. Absence from a top-N +list does not mean a zero counter. For matching query IDs, compute differences +in calls, execution time, reads, and temporary I/O; interval mean execution time +is `total_exec_time_delta / calls_delta`. + +Use the actual native query with representative arguments when inspecting a +plan. `EXPLAIN ANALYZE` executes the query, so use an isolated restored database +for expensive window/audit probes. Keep a statement timeout and read-only +transaction. Native reward windows use eight times network difficulty; the old +Python prefetch's sixteen-times scan is not the current reward policy. + +## Storage and audit response measurement + +Record `pg_total_relation_size` deltas for the ledger, proof registry, audit +snapshots, non-share audit bodies, job records, and payout/CTV tables. Count +native snapshot references separately from imported legacy inline bundles. +Measure full public audit serialization time and response bytes for large +windows; normalization reduces persistent duplication but the compatible public +response still contains the logical share array. + +Verify sampled reconstructed bundles against their canonical SHA, the trusted +ledger public key, and recorded coinbase. A reduced storage footprint is useful +only when old and new audits remain independently reproducible. + +## Comparison record + +Retain raw measurements and a concise record containing revision/image, +configuration, database profile and durability settings, workload, observation +start/end, frontend/resource counts, latency distributions, unique-commit +reconciliation, storage/WAL growth, and recovery outcomes. State omissions and +sample sizes. Compare matched workloads and label predictions separately from +observed results. + +## JSONB ceiling gate and 400k-share baselines + +PostgreSQL refuses a JSONB container whose elements exceed 268,435,455 bytes. +Two native writes still embed the whole payout window and therefore grow +linearly with the share count, so the payout window has a hard storage ceiling +that arrives well before any capacity limit. The landing write was a third until +#267 normalized the stored audit body; it is now flat in the window. The gate at +`crates/qbit-prism-server/tests/jsonb_ceiling_gate.rs` measures those writes and +holds the line while they are being removed. The legacy audit import was a +fourth until #265. It now stores only the canonical bytes, as `bytea`, and +writes no JSONB value. + +### Running it + +Reduced sizes, which is what `cargo test --locked -p qbit-prism-server +--all-targets` and the `prism-native-postgres` CI job already run. The +database-free `rust-tests` job skips it by name +(`--skip jsonb_ceiling_ratchet_at_reduced_sizes`), because with `CI` set and no +database the gate fails instead of skipping: ```sh -rg '"event": "payout_artifact_' "$PRISM_COORDINATOR_LOG" \ - | sed -E 's/^.*prism coordinator: //' \ - > "$PRISM_ARTIFACT_EVENTS" -jq -e . "$PRISM_ARTIFACT_EVENTS" >/dev/null +PRISM_TEST_DATABASE_URL=postgresql://user@127.0.0.1:5432/postgres \ + cargo test --locked -p qbit-prism-server --test jsonb_ceiling_gate ``` -### Build duration p50/p95 +Full size (400,000 shares) against a disposable cluster the repository script +starts and cleans up itself: ```sh -jq -s ' - def percentile($p): - sort as $v - | if length == 0 then null else $v[((length * $p | ceil) - 1)] end; - [.[] - | select(.event == "payout_artifact_built") - | .duration_seconds] - | { - count: length, - p50_seconds: percentile(0.50), - p95_seconds: percentile(0.95), - max_seconds: (if length == 0 then null else max end) - } -' "$PRISM_ARTIFACT_EVENTS" +test/prism-native-tests.sh cargo-args \ + --locked -p qbit-prism-server --test jsonb_ceiling_gate \ + -- --ignored --nocapture --exact jsonb_ceiling_ratchet_at_full_size ``` -Repeat grouped by `.window_build_mode`. Report `incremental`, `full_rescan`, -`self_check_match`, `self_check_mismatch`, and -`incremental_self_check_failed` separately. A self-check mismatch is a release -blocker and must be investigated even though runtime resets to the full oracle. - -### Time spent building during publication - -Set `PRISM_OBSERVATION_SECONDS` to the exact interval length. Include actual -builds, debounced retags, publication aborts, and cached found-block fallbacks -so the duty calculation cannot hide remaining publication work: - -```sh -jq -s --argjson seconds "$PRISM_OBSERVATION_SECONDS" ' - [.[] - | select(.event == "payout_artifact_built" - or .event == "payout_artifact_build_debounced" - or .event == "payout_artifact_build_aborted" - or .event == "payout_artifact_found_block_cached") - | select(.during_publication == true) - | (.duration_seconds // 0)] - | { - publication_build_seconds: (add // 0), - observation_seconds: $seconds, - duty_cycle_fraction: ((add // 0) / $seconds), - duty_cycle_percent: (((add // 0) / $seconds) * 100) - } -' "$PRISM_ARTIFACT_EVENTS" -``` +Always name the test: `--ignored` without a filter also starts +`jsonb_ceiling_baseline_sweep` in the same process, against the same cluster. -The after-rollout target is less than 20% of wall-clock. The legacy baseline -does not label every synchronous scan consistently; record that limitation and -cross-check adjacent `payout_artifact_installed` events with -`during_publication:true`. +The full-size run measures the reduced pair first, then 400,000 shares. It needs +the reduced pair's projections to attribute the writes PostgreSQL refuses at +full size, so budget its wall clock on top. -Also retain counts and bounded-work fields: +Full size against a PostgreSQL you already have: ```sh -jq -s ' - group_by(.event + ":" + (.window_build_mode // "legacy")) - | map({ - key: (.[0].event + ":" + (.[0].window_build_mode // "legacy")), - count: length, - delta_rows: ([.[].delta_rows // 0] | add), - touched_pages: ([.[].touched_pages // 0] | add) - }) -' "$PRISM_ARTIFACT_EVENTS" +PRISM_TEST_DATABASE_URL=postgresql://user@127.0.0.1:5432/postgres \ + cargo test --locked -p qbit-prism-server --test jsonb_ceiling_gate \ + -- --ignored --nocapture jsonb_ceiling_ratchet_at_full_size ``` -## 4. Sidecar failure rate and reconcile symptom - -Adjust the sidecar expression only if its deployed message differs, and keep -sample matching lines: +The baseline sweep that produced the table below. It also runs the reduced pair, +5,000 and 20,000, for the CI projection it checks its own sizes against, and +skips a size the list already contains: ```sh -PRISM_SIDECAR_FAILURES=$(rg -c \ - 'mining\.subscribe.*(fail|timeout|timed out|error)' \ - "$PRISM_SIDECAR_LOG" || true) -awk -v failures="$PRISM_SIDECAR_FAILURES" \ - -v seconds="$PRISM_OBSERVATION_SECONDS" \ - 'BEGIN { printf "failures=%d rate_per_hour=%.3f\n", failures, failures * 3600 / seconds }' - -PRISM_RECONCILE_TIMEOUTS=$(rg -c \ - 'reconcile prefetch join exceeded 20s' \ - "$PRISM_COORDINATOR_LOG" || true) -awk -v failures="$PRISM_RECONCILE_TIMEOUTS" \ - -v seconds="$PRISM_OBSERVATION_SECONDS" \ - 'BEGIN { printf "events=%d rate_per_hour=%.3f\n", failures, failures * 3600 / seconds }' +PRISM_JSONB_GATE_BASELINE_SIZES=50000,100000,200000 \ +PRISM_TEST_DATABASE_URL=postgresql://user@127.0.0.1:5432/postgres \ + cargo test --locked -p qbit-prism-server --test jsonb_ceiling_gate \ + -- --ignored --nocapture jsonb_ceiling_baseline_sweep ``` -Before rollout, confirm the sidecar calculation reproduces the known 50–115 -failures/hour range. Expected after rollout: approximately zero background -sidecar failures, zero normal-operation reconcile-prefetch timeouts, and -`PrismStratumCheckFailureRateHigh` clears after its lookback interval. - -## 5. Comparison record - -| Metric | Before | After | Target | -|---|---:|---:|---:| -| Artifact build p50 / p95 (seconds) | | | Incremental scales with delta | -| Publication build duty cycle | | | <20% | -| Full pages-CTE calls/hour | | | Cold/correction/check only | -| Pages-CTE execution time/hour | | | Near-zero background | -| Pages-CTE shared blocks read/hour | | | Near-zero background | -| Carry-balance aggregate calls/hour | | | Cold/payout mutation only | -| Reconcile-prefetch >20s events/hour | | | 0 | -| Sidecar subscribe failures/hour | 50–115 | | Approximately 0 background | -| `PrismStratumCheckFailureRateHigh` | Firing | | Clear | - -Do not claim the production win from local simulation. Until both captures are -complete, report these as expected effects and attach the raw before/after -evidence to the rollout record. - -## 6. Full-rescan attribution (issue #228) - -Every `payout_artifact_built` and `payout_artifact_build_debounced` line now -says why the build was requested. Read these fields together: - -| Field | Meaning | -|---|---| -| `invalidation_cause` | The payout-state source cause the candidate was captured with, verbatim (`external_tip`, `accepted_block_preview`, `accepted_block_preview_withdrawn`, `direct_block_uncertain`, `payout_only`, `startup`); `null` for a background preparation, which carries no candidate. | -| `force_origin` | What marked the build as a payout mutation: `caller_flag` (the caller passed `force_full_window_rescan`), `reserved_cause` (the captured cause was `accepted_block_preview_withdrawn` and this preparation consumed it), `caller_flag+reserved_cause`, or `none`. | -| `cause_inherited` | `true` when the capture found a cause another preparation had already acted on. The source tuple is deliberately sticky (candidate supersession and the finalizer's pending-cause read depend on it); before this change every such capture re-forced the full oracle. An inherited cause never forces anything now, so a `true` here with `force_origin: "none"` is the expected shape. | -| `payout_mutated` | Balances were re-read from the ledger because pool-block, payout-entry or carry rows moved (a reorg, a maturity, a withdrawal), by either flag below. | -| `prior_balances_read_forced` | The caller passed `force_prior_balances_read` (the reorg reconciler after a confirmed mutation, issue #224): the accepted-share ledger is append-only, so the window advances from its delta and only the carry aggregate is reread. Never a full rescan. | -| `window_oracle_forced` | The caller passed `force_full_window_rescan`: the fail-closed full oracle, which also rereads balances. In production that is the preview-withdrawal path and the reconciler's error path after an uncertain mutation. | -| `self_check_deferred` | An overdue periodic self-check was left for the next build off the tip-refresh path because this build's cached window was inside the accepted staleness bound (below). | -| `self_check_recentered` | The periodic self-check re-centered an oversized window at the live weight (issue #207). | -| `phase_seconds` | The phase split of this build, see section 7. | - -Which builds run the full oracle (`window_build_mode: "full_rescan"`), by -`full_rescan_reason`: - -| Reason | Trigger | Expected rate | -|---|---|---| -| `cold_start` | No cached window (process start, or reuse re-enabled). | Once per process. | -| `late_visible_append` | The append-invalidation epoch moved: a share committed below an already-walked anchor. The epoch check is the guard that keeps a failed reorg check from resuming append-only state; it is unchanged. | Rare; alert if sustained. | -| `snapshot_window_weight_out_of_band`, `anchor_regression`, `window_pipeline_mode_changed`, `incremental_invariant_failed`, `window_mirror_divergence`, `window_value_out_of_range`, `delta_api_unavailable` | Genuine epoch breaks detected inside materialization. | Rare. | -| `window_daemon_state_lost` | The Rust daemon no longer holds the base digest after a respawn or LRU eviction. Recovery is one oracle read that re-prepares the daemon. | Tracks daemon restarts. | -| `window_daemon_busy`, `window_daemon_unavailable`, `window_value_out_of_range`, `window_mirror_divergence` (after a `self_check_recentered: true` build) | A periodic self-check adopted a window the daemon could not be prepared for at the time, and the reason names what declined the preparation; the next advance rebuilt through the oracle. A deliberate coordinator adoption, never reported as daemon state loss (issue #207). | Rare. | -| `reconcile_invalidation` | A caller passed `force_full_window_rescan`: the preview-withdrawal path and the reconciler's error path after an uncertain mutation. Since PR #230, reorg reconciliation and payout maturity ask for a prior-balances reread instead (`prior_balances_read_forced: true`, an incremental build), so this reason no longer tracks block cadence. | Withdrawals and reconciler errors only. | - -Self-check builds (`self_check_match` / `self_check_mismatch`) report -`periodic_self_check`, with `self_check_recentered: true` when the cached -weight sat more than 25% above the live snapshot weight and the window was -re-centered at the live weight. With the Rust pipeline on, the re-center -prepares the daemon for the adopted digest in the same build, so the next -build advances incrementally; a second full ledger snapshot on the following -build is the regression issue #207 fixed. The rescan reason vocabulary is -closed (`lab/prism/accepted_preview_telemetry.py`, exported as -`qbit_prism_payout_window_full_rescan_seconds{reason,path}`), so the -re-center is a flag on the build rather than a reason of its own. - -`payout_artifact_tip_refresh_cached` marks a routine candidate that found the -preparation lock busy and reused the armed window instead of queueing (see -section 8). It counts under `qbit_prism_payout_artifact_events_total{event="tip_refresh_cached"}` -once the tip-refresh exporter lists the event; until then count it from logs. - -Per-hour full-rescan count from the extract in section 3: - -```sh -jq -r ' - select(.event == "payout_artifact_built" and .window_build_mode == "full_rescan") - | "\(.full_rescan_reason) \(.force_origin) \(.cause_inherited)" -' "$PRISM_ARTIFACT_EVENTS" | sort | uniq -c +| Variable | Default | Meaning | +| --- | --- | --- | +| `PRISM_TEST_DATABASE_URL` | none | PostgreSQL to test against. Missing while `CI` is set is a failure, never a skip. | +| `PRISM_JSONB_GATE_TARGET_SHARES` | `400000` | share count the reduced run projects to | +| `PRISM_JSONB_GATE_N1` | `5000` | smaller reduced size | +| `PRISM_JSONB_GATE_N2` | `20000` | larger reduced size | +| `PRISM_JSONB_GATE_STATEMENT_TIMEOUT_MS` | `600000` | exported as `PRISM_DATABASE_STATEMENT_TIMEOUT_MS` before the first `Ledger::connect` | +| `PRISM_JSONB_GATE_BASELINE_SIZES` | `50000,100000,200000` | sizes for the baseline sweep: at least two, strictly increasing, each dividing the window weight; checked before anything runs | + +Every share count must divide the window weight (8,000,000) exactly, and +`0 < n1 < n2 <= target` must hold. A missing variable takes the default; an +empty, malformed, zero, negative or out-of-range value fails loudly and is never +replaced by the default. + +### What it measures + +The gate drives five phases against a real PostgreSQL 16 with an in-process fake +node. No phase submits a block, so none of them needs `QBITD_BIN`. + +| Phase | Call | Write | +| --- | --- | --- | +| refresh | `Coordinator::refresh_once` | `qbit_prism_jobs.payload` | +| enqueue | `Ledger::append(share, Some(candidate))` | `qbit_block_candidate_outbox.candidate` | +| claim | `Ledger::claim_candidate` | lease columns only | +| landing | `Ledger::land_candidate` then `finish_candidate` | `qbit_pool_audit_bundles.audit_bundle` | +| import | `Ledger::import_legacy_audits` | `qbit_pool_audit_bundles.audit_bundle` and `canonical_audit_bytes` | + +After every phase it discovers every JSONB column through +`information_schema.columns` (17 at this base commit, all present only after +`Ledger::connect(.., true)` has applied `001_share_ledger.sql` and migrations +002-005) and records, per row: + +- `pg_column_size(col)`, the TOAST-compressed stored size; +- `octet_length(col::text)`, the JSON text form; +- `pg_column_size(col::text::jsonb)`, the **uncompressed** JSONB container. The + 268,435,455-byte ceiling applies to the uncompressed container, so the + compressed on-disk size must not be what a threshold compares against. A + computed datum is never TOAST-compressed, so the round trip through `text` + reports the real container size. Measured, it comes out at 1.07 times + `octet_length(col::text)` and about 27 to 35 times `pg_column_size(col)` for + share-heavy documents on the synthetic fixture: 27.4 for the 50,000-share + refresh payload, 34.7 for the 400,000-share landing body. + +A value is attributed to the phase whose write created its row or changed its +value, compared by `md5(col::text)`. Attributing by `xmin` alone would be wrong: +the claim UPDATE touches only the lease columns, but the new tuple version +carries the unchanged TOAST pointer forward, so `xmin` alone blames claim for +the candidate that enqueue wrote. The gate prints every such tuple rewrite as an +`attribution:` line so the distinction stays visible. + +### How the ratchet works + +A write **crosses** when its size at the target exceeds 67,108,863 bytes, 25% of +the hard limit, or when PostgreSQL refuses it outright. The reduced run fits +`size(n) = a + b*n` through its two sizes and projects to the target. A negative +slope fails the gate instead of quietly passing, and so does an intercept more +negative than 5% of the `n1` measurement: that write grew faster than linearly +between the two sizes, and a straight line would understate it. The measured +window-carrying writes sit under 0.3%. A positive intercept, up to the `n1` size +itself for a constant-size write, is a fixed per-write overhead and is allowed. +The full-size run compares the measured size directly. + +A refusal is a result rather than a crash, but PostgreSQL's ceiling error names +neither the table nor the column, and the server adds no context that would. +The gate therefore attributes every refusal by projection: among the columns the +refused phase writes, exactly one has to project past 90% of the hard limit at +the refused share count, and it has to be the write the gate expected there. No +candidate, two candidates, or a single candidate that is not the expected write +each fail the gate, naming what was found and what was expected, so a second +oversized write in the same phase can never be recorded as the one already +known. The projections come from the reduced pair, which is why the full-size +run and the baseline sweep both run that pair first; at the reduced sizes +themselves a refusal at `n2` is attributed from the `n1` measurement, and that scaled measurement stays +the write's projection for the full-size run and the sweep, so a violation +already refused at `n2` is still explained at the target. A refusal at `n1` +fails the gate, since nothing smaller was accepted to attribute it from. The ratchet row says how: `refused at n=400000 (attributed by +projection)`, and never a byte count. +Until a refusal is attributed, the measurement table shows it by phase +only, under `(not attributed)`, never under the column its call site +assumed; each run attributes before it prints. + +The gate passes only when the set of crossing writes equals `KNOWN_VIOLATIONS` +exactly. An unlisted write that crosses fails it as a new violation. A listed +write that stops crossing also fails it, with a message naming the entry to +delete, so #265, #267 and #273 are forced to shrink the list as they land. The +full table is printed on every run, passing or failing. + +### Baseline, measured at the base commit plus the gate's own test files + +All numbers below are **measured**, unless a cell says "projected". Recorded on +2026-09-10 by the gate as committed in `0c62751`, which is base commit +`1398bbc1f6e00972c53595bc6f27a6e57adf386c` plus only the three new test files, +with `cargo test` in **debug** (rustc 1.97.1) against a local disposable +PostgreSQL **16.15** cluster with stock settings, on a single host: 8 vCPU +(Intel Core, Haswell-class) and 22 GiB RAM, no swap. One host, one table; do not +mix in numbers from another machine. + +Later commits changed how the gate reports and validates: refusals, the +substitute row, units, unknown values, the fit checks and the settings checks. +They did not change what the pipeline writes or how a JSONB size is measured, +so the sizes here stand. The one measurement they did change is the claim +phase's peak RSS, now reset before claim; see the note below the +per-phase table. + +Largest JSONB write per path. "Uncompressed" is `pg_column_size(col::text::jsonb)`, +the size the 268,435,455-byte ceiling applies to; "stored" is the +TOAST-compressed `pg_column_size(col)`. The stored size varies by a few bytes +from run to run: the payload embeds a random `storage_key` and schema UUID, so +the pglz output differs. The gate thresholds on the uncompressed size, which is +byte-identical across runs and hosts. + +| Shares | Path | Column | Uncompressed B | Text B | Stored B | +| ---: | --- | --- | ---: | ---: | ---: | +| 50,000 | refresh | `qbit_prism_jobs.payload` | 94,694,334 | 88,230,523 | 3,460,299 | +| 50,000 | enqueue | `qbit_block_candidate_outbox.candidate` | 61,934,708 | 57,742,014 | 2,160,865 | +| 50,000 | claim | none | - | - | - | +| 50,000 | landing | `qbit_pool_audit_bundles.audit_bundle` | 29,173,666 | 27,252,045 | 861,409 | +| 50,000 | import | `qbit_pool_audit_bundles.audit_bundle` | 61,933,626 | 57,740,951 | 2,160,601 | +| 100,000 | refresh | `qbit_prism_jobs.payload` | 189,894,214 | 176,380,550 | 6,923,912 | +| 100,000 | enqueue | `qbit_block_candidate_outbox.candidate` | 124,334,648 | 115,392,039 | 4,315,575 | +| 100,000 | claim | none | - | - | - | +| 100,000 | landing | `qbit_pool_audit_bundles.audit_bundle` | 58,773,666 | 54,402,069 | 1,707,202 | +| 100,000 | import | `qbit_pool_audit_bundles.audit_bundle` | 124,333,566 | 115,390,976 | 4,315,311 | +| 200,000 | refresh | `qbit_prism_jobs.payload` | **refused** | - | - | +| 200,000 | enqueue | `qbit_block_candidate_outbox.candidate` | 248,734,408 | 230,992,039 | 8,626,824 | +| 200,000 | claim | none | - | - | - | +| 200,000 | landing | `qbit_pool_audit_bundles.audit_bundle` | 117,573,546 | 108,902,069 | 3,399,850 | +| 200,000 | import | `qbit_pool_audit_bundles.audit_bundle` | 248,733,326 | 230,990,976 | 8,626,557 | + +The claim phase writes no JSONB value at any size; it updates only the lease +columns. That row is a measured zero-copy result, not a missing measurement. + +At 200,000 shares the prepared job is already refused: + +```text +error returned from database: total size of jsonb object elements exceeds the +maximum of 268435455 bytes ``` -## 7. Build phase attribution (issue #228) - -`qbit_prism_payout_window_build_phase_seconds{phase=...,outcome=...}` is a -histogram over the same buckets as the other payout timings, observed once -per build for each of a closed phase set, under a closed `outcome`: -`completed`, `debounced`, `aborted` (a lost generation race, an anchor past -the audit ceiling, an empty window) or `failed` (the build died with the -ledger). Every exit observes, so a build slow enough to lose its generation -race keeps the time it measured instead of vanishing from the family; read -the `aborted` and `failed` cells first during an incident. No per-tip, -per-generation or per-reason label exists in this family. - -| `phase` | What it measures | -|---|---| -| `ledger_read` | Every PostgreSQL read the build issued: the full oracle, the delta query, the periodic self-check's oracle read, and the carry-balance aggregate when balances were not reused. | -| `record_conversion` | In-process work on records: folding into pages, canonical JSON and digests, mirror verification. | -| `daemon_prepare` | Round trips to the Rust daemon (`prepare_window` full or advance), including any window upload. | -| `lock_wait` | Time queued for the payout-state preparation lock before the build could start. | - -The same split is on each build line as `phase_seconds`. The phases account -for the build's `duration_seconds` up to artifact construction and the cheap -accepted-share count. A build whose `ledger_read` dominates is a database -problem; one whose `lock_wait` dominates is queued behind another build; one -whose `daemon_prepare` dominates is the daemon or its upload. - -## 8. Accepted staleness on the tip-refresh path (issue #228) - -`PRISM_TIP_REFRESH_WINDOW_STALENESS_SECONDS` (60 s, in `lab/prism/payout_state.py`) -is the staleness the coordinator accepts for a routine candidate build, the -one a tip refresh publishes for a new tip. Two rules key on it: - -- If the preparation lock is busy and an armed window exists whose anchor is - no older than the bound (and whose payout generation, append epoch and - balances are current, and whose difficulty is inside the snapshot-weight - band), the candidate reuses that window instead of queueing behind the - lock holder. This is the found-block degrade of issue #188 applied to the - tip-refresh path. -- If a periodic self-check is overdue when a routine candidate builds, and - the cached window's anchor is within the bound of the anchor being built, - the check is deferred, but never for more than one staleness bound past - its due time. The grace is measured from the last self-check attempt, not - from the last build, so candidates arriving faster than the bound cannot - push it out. A deferral also marks the check owed: the next payout - publication requests a background preparation, which normally runs the - check off the critical path well inside the grace. If nothing else runs - it, the first routine build after the grace runs it on the critical path. - -Why 60 s: it is one background re-anchor interval. In steady state the -re-anchor keeps the armed window younger than this, so a window inside the -bound is one the delivery path is already serving to new jobs as fresh, and -a fanout that reuses it declares an anchor no older than reuse itself -declares. Past the bound the wave pays the (incremental) build: the -re-anchor has already flagged that window for replacement, and carrying it -into a new payout generation would let a stalled background walker pin a -stale anchor for every following generation. It sits well inside the 300 s -audit ceiling that bounds how far any served window may trail the live -ledger. - -What this does not bound: the reorg reconciler acquires the preparation lock -before it captures its source, so a wave still waits for a background build -that holds the lock through a periodic self-check. That wait is one oracle -read at the self-check cadence (two in 48 h on mainnet), not one per -invalidation. - -The bookkeeping behind `cause_inherited` is one integer, the highest source -generation any candidate preparation has acted on, plus a one-shot -entitlement carried by the tuple a reservation hands back to its caller. A -capture acts only above that mark; the reserver's own preparation acts once -by presenting its reserved tuple, whatever happened at other generations in -between. Nothing accumulates, so there is nothing to prune or to watch. - -## 9. Deliberate re-center versus daemon state loss (issue #207) - -| Signal | Deliberate re-center | Genuine daemon eviction or restart | -|---|---|---| -| Self-check build line | `full_rescan_reason: "periodic_self_check"`, `self_check_recentered: true` | not applicable | -| Next build, daemon prepared | `window_build_mode: "incremental"`, no reason | `full_rescan_reason: "window_daemon_state_lost"` | -| Next build, daemon could not be prepared at adoption | the daemon's outcome at adoption: `window_daemon_busy`, `window_daemon_unavailable`, `window_value_out_of_range` or `window_mirror_divergence` | `window_daemon_state_lost` | -| Daemon counters | one extra `window_prepares` at the self-check build | `spawns` increments on a restart | - -Both recoveries take the existing path: one full oracle read that -re-prepares the daemon, after which the cache is a daemon mirror again. The -byte-exact parity of the window, the 5/8 band floor, the 2x ceiling and the ->1.25x re-center threshold are unchanged. - -## 10. Testnet soak thresholds (issue #228) - -Operator-run; the deterministic equivalents are in -`tests/test_prism_payout_state.py` (`PayoutWindowRescanBoundTests`) and -`tests/test_prism_payout_window_daemon_recenter.py`. Run a 24 h soak on -testnet with `PRISM_WINDOW_PIPELINE_RUST=1` and extract events as in section -3, then: - -```sh -jq -s ' - [.[] | select(.event == "payout_artifact_built")] as $built - | { - full_rescans: ($built | map(select(.window_build_mode == "full_rescan")) | length), - reconcile_invalidation: ($built | map(select(.full_rescan_reason == "reconcile_invalidation")) | length), - daemon_state_lost: ($built | map(select(.full_rescan_reason == "window_daemon_state_lost")) | length), - inherited_forces: ($built | map(select(.cause_inherited == true and .force_origin != "none")) | length), - p95_build_seconds: ($built | map(.duration_seconds) | sort | .[((length * 0.95 | ceil) - 1)]) - } -' "$PRISM_ARTIFACT_EVENTS" +The enqueued candidate and the imported audit body are at 248.7 MB there, inside +the limit by 7%. One more doubling puts both over it. + +Wall time, WAL and peak resident set, per phase. WAL is +`pg_wal_lsn_diff` around the one operation and includes full-page writes, so it +moves with checkpoint timing rather than purely with the payload. Peak RSS is +`VmHWM` of the **test process** (the gate plus the native library, not the +PostgreSQL backend), with `/proc/self/clear_refs` resetting the high-water mark +at the start of each phase. On an OS without a resettable `VmHWM` (macOS) the +gate prints `not measured on this OS` for every phase instead of a number. When +these tables and the 400,000-share table below were recorded, the claim phase +did not reset the mark, so its peak RSS spans enqueue and claim together; the +gate now resets it before claim too. + +**WAL and stored sizes here are measured on synthetic shares and are +optimistic.** The fixture repeats five miner identities and a padded `miner_id`, +so the window compresses about 29:1 in TOAST +(`uncompressed / pg_column_size(col)` = 27.4 to 34.7 across these runs), while +production data is assumed to compress about 3:1. That 3:1 is the assumption +stated in issue #273, not a measurement. Do not carry these WAL or stored figures into +a production estimate without restating that ratio. The gate prints the same +caveat on every run. The uncompressed sizes, which are what the JSONB ceiling +applies to and what the gate thresholds on, are unaffected. + +| Shares | Phase | Result | Seconds | WAL B | Peak RSS KiB | +| ---: | --- | --- | ---: | ---: | ---: | +| 50,000 | refresh | ran | 17.66 | 3,704,648 | 655,244 | +| 50,000 | enqueue | ran | 14.91 | 2,501,664 | 1,028,772 | +| 50,000 | claim | ran | 11.25 | 896 | 1,101,840 | +| 50,000 | landing | ran | 33.05 | 1,040,592 | 1,082,872 | +| 50,000 | import | ran | 50.25 | 4,617,296 | 1,248,072 | +| 100,000 | refresh | ran | 35.73 | 7,419,792 | 1,638,612 | +| 100,000 | enqueue | ran | 30.06 | 4,638,512 | 1,924,400 | +| 100,000 | claim | ran | 22.13 | 896 | 2,149,764 | +| 100,000 | landing | ran | 67.16 | 2,023,304 | 2,088,428 | +| 100,000 | import | ran | 101.06 | 9,385,664 | 2,293,936 | +| 200,000 | refresh | refused | 74.10 | 1,145,896 | 3,230,884 | +| 200,000 | enqueue | ran | 60.03 | 9,252,776 | 3,984,264 | +| 200,000 | claim | ran | 44.70 | 78,856 | 4,435,528 | +| 200,000 | landing | ran | 133.45 | 12,776,016 | 4,312,596 | +| 200,000 | import | ran | 203.34 | 18,528,496 | 4,862,096 | + +Whole-pipeline wall clock: 177.6 s at 50,000, 357.6 s at 100,000 and 702.9 s at +200,000. The fixture load itself is a small part of that: 15,000 rows/s, so +3.3 s, 6.5 s and 13.5 s respectively. + +`qbit_pool_audit_bundles.canonical_audit_bytes` is bytea, so the JSONB ceiling +does not apply to it; it is reported only. After import it holds 55,640,585 B at +50,000 shares, 111,190,610 B at 100,000 and 222,590,610 B at 200,000, against +268,435,456 B, which is 25% of the 1 GiB varlena limit. Projected to 400,000 +shares it is ~445 MB, over that reporting line but still inside the varlena +limit. + +#### At 400,000 shares + +Measured on the same 8 vCPU / 22 GiB host as the table above, which is close to +its limit at this size (see the allocator note at the end of this subsection). +A run on a larger, dedicated host should be recorded in its own host-labelled +table rather than merged into this one. + +The full-size run completes and the ratchet holds: every phase is reached, and +the crossing set is exactly the four known violations. Three of the four writes +are refused by PostgreSQL outright, each with + +```text +error returned from database: total size of jsonb object elements exceeds the +maximum of 268435455 bytes ``` -Pass criteria for the 24 h window, before any mainnet rollout: - -| Measure | Threshold | -|---|---| -| Full rescans per hour (`full_rescans / 24`) | at most 2 | -| `reconcile_invalidation` full rescans | 0 | -| `window_daemon_state_lost` full rescans | at most the daemon `spawns` count over the window | -| `inherited_forces` | 0 | -| p95 `duration_seconds` of `payout_artifact_built` | at most 3 s | -| Tip-refresh waves with elapsed above 3 s whose outcome is `fanout_superseded` | none attributable to a payout build (`phase_seconds.lock_wait` under 1 s on every build during the wave) | +| Path | Result | Uncompressed B | Text B | Stored B | Seconds | WAL B | Peak RSS KiB | +| --- | --- | ---: | ---: | ---: | ---: | ---: | ---: | +| refresh, `qbit_prism_jobs.payload` | refused | - | - | - | 128.28 | 692,280 | 5,094,428 | +| enqueue, `qbit_block_candidate_outbox.candidate` | refused | - | - | - | 111.91 | 23,264 | 4,149,832 | +| claim, of the gate-written substitute row | ran, writes no JSONB | - | - | - | 0.01 | 808 | 4,150,152 | +| landing, `qbit_pool_audit_bundles.audit_bundle` | ran | 235,173,306 | 217,902,069 | 6,785,157 | 266.81 | 7,305,224 | 4,584,516 | +| import, `qbit_pool_audit_bundles.audit_bundle` | refused | - | - | - | 399.19 | 166,864 | 5,610,676 | + +Whole pipeline: 1,222.9 s, of which 27.7 s is the fixture load (400,000 rows at +14,465 rows/s, 585.7 B per serialized share, 223.4 MiB of window). Peak RSS of +the test process was 5.6 GiB. `canonical_audit_bytes` is never written at this +size because the import that would write it is refused first. + +Two notes on reading the 400,000-share table. The enqueue write is refused, and +the gate reports it only as refused: `rejected` in its size columns, +`refused at n=400000 (attributed by projection)` in the ratchet, and +PostgreSQL's error text underneath. +After the refusal the gate writes a window-free **substitute** outbox row itself, +so that the claim, landing and import writes stay measurable instead of being +reported as unreached. The substitute is not the enqueue write and never enters +the ratchet; the report lists it on its own line labelled `SUBSTITUTE`. In the +run recorded here it measured 14,880 B uncompressed, a figure that run's report, +produced before the substitute was labelled, printed in the enqueue row. The +claim row above claimed that substitute, so its seconds, WAL and peak RSS +describe claiming a window-free row, not a production candidate. And the landing +write survives at 400,000 shares only because it carries +one window copy rather than two or three: at 235 MB it is inside the hard limit +but 3.5 times over the gate threshold, and one more doubling of the window puts +it over the ceiling as well. + +The measured landing size confirms the projections: 235,173,306 B measured +against 233,066,529 B projected from the CI pair (0.90% low) and 236,373,666 B +projected from 50,000 and 100,000 shares (0.51% high). + +**Run the full-size test on a host with memory to spare.** At 400,000 shares the +test process needs several GiB and PostgreSQL needs several more at the same +time. Without +`MALLOC_ARENA_MAX=1 MALLOC_MMAP_THRESHOLD_=65536 MALLOC_TRIM_THRESHOLD_=65536` +the test process retained about 7 GiB of freed allocations by the landing phase; +with them it peaked at 5.6 GiB and finished. Set those for the full-size run on +any machine that is not dedicated to it, and budget for PostgreSQL alongside. +Nothing in CI is affected: CI runs only the reduced sizes. + +The reduced-size gate itself, at its default 5,000 and 20,000 shares, pinned to +two cores with `taskset -c 0,1` and `CARGO_BUILD_JOBS=2` on the same host, took +**88.3 s** of test time (17.8 s at 5,000, 70.4 s at 20,000). In CI it runs in +the `prism-native-postgres` job, a 2 vCPU runner with `timeout-minutes: 20` +(`.github/workflows/ci.yml`). That job's test step took about 4m41s at `3.x.x` +`f316eb9`, before this gate existed. The 88.3 s comes from the recording host, +not a CI runner, so treat the sum as an estimate of the headroom, not a +measurement of it. + +Fit stability. The baseline sweep computes and checks this itself, per write. It +runs the reduced CI pair (5,000 and 20,000) alongside its own sizes, projects +every write to 400,000 shares from that pair and again from the largest pair of +sweep sizes at which **that write** was accepted, and fails when the two differ +by more than 5% of the larger-pair projection. The pair has to be chosen per +write because the writes are refused at different sizes - refresh is already +refused at 200,000 while the other three are not - so no single pair fits all +four. A write accepted at fewer than two sweep sizes fails the sweep by name +rather than going unchecked, and a CI projection below 1 MiB is reported as not +compared, because a relative difference that small is fixed per-write overhead +and not the window's growth. The table below was recorded before the check was +per write, against 50,000 and 100,000 - the largest pair at which all four +writes were accepted - and the two projections agree to well inside the +tolerance. Re-running the same sweep now pairs enqueue, import and landing at +100,000 and 200,000, where the measurements above show them still accepted; +refresh, refused at 200,000, keeps 50,000 and 100,000: + +| Path | From 5,000 and 20,000 | From 50,000 and 100,000 | Difference | +| --- | ---: | ---: | ---: | +| refresh | 756,772,922 B | 761,093,494 B | 0.57% | +| enqueue | 494,920,433 B | 498,734,288 B | 0.76% | +| import | 494,919,351 B | 498,733,206 B | 0.76% | +| landing | 233,066,529 B | 236,373,666 B | 1.40% | + +### After #265's import change + +The baseline above predates #265, whose import stores the canonical audit +bytes instead of an inline JSONB body. With that change, at 400,000 shares the +import writes no JSONB value: it ran in 129.5 s with 18.7 MB of WAL, and +`canonical_audit_bytes` holds 445,390,610 B (bytea, reported only). The ratchet +holds with three known violations. This run was measured on a different host +from the baseline (Apple M-series, debug build, PostgreSQL 16), so compare its +timings with the tables above only loosely. + +### Fixture + +The window is loaded by `crates/qbit-prism-server/tests/support/window_fixture.rs` +with one server-side `INSERT ... SELECT FROM generate_series` per 50,000 rows, +then the `qbit_prism_share_hashes` backfill migration 002 performs. Shares are a +pure function of the share index: no wall clock, no OS randomness, and the same +parameters always produce byte-identical shares. The row shape follows the #258 +measurement fixture; `miner_id` is padded so one serialized `AcceptedShare` is +exactly 581 bytes at share index 1 and averages 583.7 to 584.9 bytes over a +window, against the ~581 bytes per share measured on the production window in +#254 (372,257 shares, ~232 MB canonical). + +Two deviations from #258 are deliberate. `share_difficulty` is scaled to +`8,000,000 / n` so exactly `n` shares fill the payout window, instead of #258's +21-digit value; keeping that value would require a network difficulty of ~5e24 +at 400,000 shares, which the 24-bit compact-bits mantissa cannot express +exactly, so the window length would stop being an exact function of `n`. And the +share index inside `share_id` is zero-padded to a fixed width. Both cost bytes +that the `miner_id` padding gives back, which is why the serialized share size +is the same at every window size and the two-point linear fit is sound. diff --git a/docs/prism-pool-acquire-metrics.md b/docs/prism-pool-acquire-metrics.md new file mode 100644 index 000000000..57d15be3e --- /dev/null +++ b/docs/prism-pool-acquire-metrics.md @@ -0,0 +1,255 @@ +# Pool acquisition timing + +`qbit_prism_database_pool_acquire_seconds` measures a connection checkout, +from polling the acquisition to its success, error or cancellation. This +includes waiting for a pool slot, opening or validating a connection, and +checkout hooks. SQL execution and the `BEGIN` round trip occur afterward and +are outside this duration. The unit is seconds. + +The `result="success"` label means the checkout succeeded; a later query error +does not change that observation. An acquisition cancelled while waiting +records one `result="failure"` observation with its elapsed duration. Creating +and dropping an acquisition future without polling it records nothing. +An acquisition still waiting has not yet contributed to `_count`. + +Ledger, collector, rollup and partition checkouts use `metrics::time_pool_acquire`, alongside +the metrics recording hooks, and Tokio's monotonic clock. +In an unpaused runtime this measures real elapsed time. In a paused test +runtime it follows the runtime's controlled clock; ledger checkouts now share +the collector's existing behavior in those tests. Constructing an unpolled +future or acquiring without a metrics handle reads no observation clock. +Advisory-lock observations retain their separate standard monotonic clock. + +## Coverage + +In addition to the existing ledger transaction and metrics collector +acquisitions, these paths use the shared checkout timer: + +- `payout_revision` and `release_session_owner_reservations`; +- `SessionId::release` and its spawned drop cleanup; +- `worker_difficulty` and `share_accepted_at_ms`; +- `cpfp_package` and `retired_cpfp_funding`; +- `heartbeat` and the attached ledger's `fatal_state` read; +- `audit_bundle`'s initial representation read and its native snapshot/range + reconstruction reads; +- the existing-audit probe before candidate landing and each page of its + durable-range proof, released before the page's blocking comparison; +- `landed_audit` and `record_landed_audit_bits`; +- the durable-range proof's inline/bootstrap existence probe, newest eligible + boundary query, and oldest eligible boundary query for partial history; +- `pool_blocks_for_reconcile`; +- `migration_source`, the one-row pages of `import_legacy_audits` (including + the final empty read), and `backfill_ctv`'s initial audit list; +- `job`, `compact_prepared`, and `prune_expired_jobs`; +- `WorkLedger::now_ms`, `chain_observation_state`, and `persist_block_only`'s + initial duplicate probe and credit/disposition polls; +- the server's rollup loop, once per valid `advance` attempt; +- partition attachment at server startup, each maintenance tick, and the fresh + transaction after an append is refused with SQLSTATE 23514; +- candidate heartbeat's live-token read after renewal contention and its + terminal-state read after work cancellation; +- `apply_online_migration`'s startup checkout, before the connection is detached. + +Each pending online migration records one checkout when metrics are attached. +Timing ends before detach: the runner's session settings, advisory-lock waits, concurrent +index builds and drops, and the detached connection's lifetime are excluded. +An error or cancellation after checkout does not add another observation or +change that checkout's success. The transaction that records the migration +uses the detached connection and is not another pool checkout. + +The job readers and expiry statement each record one checkout. The compact +reader releases it before waiting for blocking decoding and hashing. Expiry +releases it before the separate blob cleanup transaction; the expiry cutoff, +bounded selection, renewal recheck, blob lock ordering and cleanup deadline +are unchanged. A missing job or zero deleted rows is a successful checkout; +SQL or decoding failures after acquisition do not relabel that checkout. + +The migration-source lookup keeps schema resolution and the provenance read +on one checkout. Acquiring through an existing connection or transaction is +not another pool checkout. Import reads release before filesystem and audit +verification work, then acquire separately for each write transaction. + +Recording requires an attached metrics handle. Operator-only connections and +ledgers created without telemetry continue to work without observations. + +Session reservations retain their creating ledger's metrics handle. Explicit +release and background drop cleanup each time only their own checkout, before +the unchanged token-fenced DELETE. A successful explicit release disarms drop +cleanup and records once. An error or cancellation retains the existing drop +fallback: if that task runs, it attempts and records a separate checkout. +Dropping an unpolled release future records nothing for that release, but still +spawns its background cleanup when a Tokio runtime is current. Without a current +runtime, drop retains the reservation and records nothing. An unpolled background +task records nothing; runtime shutdown during checkout records one failure. SQL waits and errors +after checkout keep the successful observation. Failed cleanup can retain a +reservation for the existing owner cleanup/reclamation paths; timing adds no +retry, deadline or change to reservation ownership. + +Each durable-range boundary statement records one checkout and releases it +before any proof refusal, blocking fold or subsequent query. Bootstrap uses +only its existence probe; a non-inline range uses its existing page checkouts +and the newest boundary query, plus the oldest query only for partial history. +SQL failures and boundary refusals after checkout retain a success observation. +These probes add no transaction and do not time the SQL or the proof itself. + +Each landed-audit lookup or bits update records one checkout and releases its +connection at the statement boundary, before decoding the row or returning. +Found and missing rows, a filled legacy NULL, and zero-row/idempotent updates +all retain a successful checkout. SQL errors or cancellation after checkout +do too; cancellation while acquiring records one failure. The read's +authentication contract and the update's `found_block_bits IS NULL` predicate +are unchanged. Failure or cancellation after a write is sent can still leave +its commit outcome unknown; retries retain the same idempotence. + +Ledger audit reconstruction records one checkout for the initial representation +read, one for a native snapshot, and one for a non-inline share range. Native +inline snapshots therefore record two successes and range-backed snapshots +three; imported canonical bytes, legacy inline bodies and missing audit rows +record one. Each connection is released at its statement boundary, before row +decoding, blocking reconstruction or the next acquisition. Missing snapshots, +incomplete ranges, SQL errors and digest/decoding refusals retain the successes +already observed. Cancelling a pending checkout records one failure; cancelling +SQL or the later reconstruction does not relabel completed checkouts. The +snapshot authority, anchored range, canonical bytes and digest checks are +unchanged. + +The coordinator database clock and chain-observation state each record one +checkout. The latter retains its one-statement revision/epoch/tip snapshot. +Each block-only duplicate probe and each credit/disposition poll records its +own checkout, inside the +original acknowledgement deadline. The two block-only probes are not issued-job +reads. Credit and disposition still share one SQL statement and MVCC snapshot. +Connections are released at the statement boundary, before enqueue or poll +sleep. Missing rows, duplicates and terminal dispositions retain successful +checkout observations; SQL errors or cancellation after acquisition do too. +Cancellation while acquiring records a failure. Before enqueue a read failure +is definite; after enqueue a failed or missing read does not prove whether the +durable candidate earned credit. Timing changes neither that distinction nor +the enqueue's existing handling of an unknown outcome. + +The server passes its existing metrics handle to the rollup loop. Each valid +rollup attempt records one checkout before `BEGIN`; `SET LOCAL`, the single +rollup statement and `COMMIT` reuse that connection and add no observations. +A real batch and an empty batch both record success. Invalid batches and +unpolled futures record nothing. Pool errors and cancellation during checkout +record failure; later SQL errors or cancellation retain the completed success +and its duration. Shutdown drops the in-flight attempt using the existing +select. Timing changes no transaction ownership, statement timeout, tick +scheduling or error propagation. Cancellation after `COMMIT` starts can still +be committed: the existing watermark and idempotence semantics reconcile that +outcome. The public pool-only `rollups::advance` and `rollups::run` wrappers +remain compatible and record nothing. + +Partition maintenance uses the same boundary: checkout completes before +`BEGIN`, the existing 30-second local statement timeout, the ensure function, +and `COMMIT`. An intact lead and newly attached partitions both record one +success. Pool errors and cancellation during checkout record one failure; +SQL errors, SQL cancellation and shutdown after checkout retain its success +and duration. The public `partitions::ensure` and `partitions::run` wrappers +still pass no metrics. The server passes its registry at startup and on each +tick; an append's SQLSTATE 23514 fallback passes its ledger's metrics handle. +That fallback records three checkouts: the refused append transaction, the +partition transaction, and the one retry. It does not double-count +`Ledger::begin`, restart the enclosing deadline, or change the SQL, rollback, +retry count or share-credit semantics. As before, failure after sending +`COMMIT` can have an unknown commit outcome. + +The candidate live-token and terminal-state probes each record their own +checkout inside the original `timeout` future. The first probe still uses +the remaining lease budget; the second still uses the existing reconciliation +timeout after dropping work. SQL predicates, token/expiry decisions and +original renewal errors are unchanged. Acquisition failure or cancellation +records failure; missing/nonterminal rows and SQL errors after acquisition +retain checkout success. A failed probe supplies no new evidence of ownership +or completion. These probes add samples to the same aggregate, without adding +labels or a second observation to the renewal transaction's `Ledger::begin`. + +Polls contribute actual acquisition attempts to the same aggregate histogram +as the other covered callers. Expanding coverage can change its percentile: +a pending candidate can produce many fast acquisitions when the pool has spare +capacity, even if settlement is stalled. Each poll waits for its checkout and +SQL before the existing 50 ms sleep, so a saturated pool slows those attempts +and their observed waits increase. The pool-wait alert's five-minute p99 and +ten-attempt minimum describe this combined population, not distinct submissions, +per-caller latency or settlement latency. Adding fast samples can lower the +aggregate p99; this coverage change does not change the alert's grouping, +threshold or minimum count. + +Rollup attempts also join this aggregate population, including no-work ticks. +Their checkout counts describe attempts, not folded shares or completed +rollups; fast rollup checkouts can lower the aggregate percentile too. +Partition ticks and fallback attachment attempts join it too, including +no-work ensures. Their samples count checkouts, not created partitions or +successful share appends. + +Landed-audit reads and bits writes also join this aggregate population, +including misses and idempotent retries. Their samples describe checkout +attempts, not newly landed blocks or successful SQL statements. + +## Source census and exclusions + +The census for [#352](https://github.com/Qbit-Org/qbit-mining-bootstrap/issues/352) +is checked in CI by the existing Python test shards, which automatically +discover `tests/test_check_prism_pool_acquires.py`. Its baseline test verifies +the repository inventory alongside the negative controls. Locally, +`python3 scripts/check_prism_pool_acquires.py` checks the same inventory; +`--census` prints current call sites and source lines without updating the +reviewed classifications in `scripts/prism_pool_acquires.json`. Each entry +has an exact expected occurrence count and a policy with an owner and reason. +Adding a site, adding another occurrence of an existing site, removing a site, +or removing/disabling its timer fails the guard until reviewed. + +The scanner walks the server crate's production module graph, including its +binaries and platform branches, and excludes test-only modules. It tokenizes +comments and SQL strings as opaque text. Direct implicit executors (including +`raw_sql`), explicit `acquire`/`begin` variants, aliases and calls to typed +pool/generic executor/reader helpers are inventoried. Mutable SQLx executor +borrows execute on an existing connection; nested acquisitions remain separate +sites. `Transaction::begin` on a checkout, migration connection reborrows and +window/range readers must not be counted again. The report counts source +syntax, including helper forwarding, rather than runtime attempts; adding its +categories together does not give a number of pool acquisitions. + +This is a conservative source review gate, not Rust type analysis or a +whole-program proof. New pooling abstractions, type aliases or generated code +need review of the scanner's coverage as well as classification. Tests mutate +the real source tree to prove detection of new implicit and explicit pool +calls, renamed pool handles, generic/helper calls, stale entries and removed +timers; a borrowed-connection control adds no fresh checkout site. +Imported pool type aliases and ordinary `type` aliases enroll helper callers; +`include!` fails closed until the generated Rust source is explicitly supported. + +The following exclusions are intentional and remain visible in the inventory: + +| Population | Owner and reason | +| --- | --- | +| Startup schema, capability, provenance and coordinator session-setting checks | Ledger startup/migrations: gates before serving. Existing timed startup transactions and online migration checkouts still contribute. | +| Generic migration `Acquire` helpers | Ledger migrations: a pool argument at startup acquires; a borrowed transaction during migration does not. Their callers are inventoried too. | +| Candidate inventory/recovery reader, fatal-state inspection and self-check | Operator tools: dedicated diagnostic pools or explicit operator calls, outside the frontend work loops. | +| Public HTTP reads and readiness probes | Public API: isolated read pools; the public role's metric export policy requires a separate decision. | +| Private audit HTTP endpoints and shared API read-model helpers | Operator API: private routes can retain the run-role pool, while public dispatch substitutes its isolated read pool. These HTTP queries remain excluded even when served by the run process. | +| Pool-only audit, partition and rollup wrappers, including `lead_rows` | Compatibility APIs: no attached metrics owner. The server uses metrics-aware maintenance; its headroom collector reads within an already timed transaction. | + +Pool creation and idle connection replenishment are not acquisition attempts +observed by this helper. Ledgers created without telemetry record nothing. +Accordingly the histogram is the aggregate of **observed checkout attempts at +covered callers**, not all process acquisitions, all use of a database role, +request throughput or an unbiased sample of every pool operation. The metric +type, labels, buckets and result semantics have not changed. + +## Adding a caller + +Use `Ledger::acquire()` inside the caller's existing deadline, then run the +unchanged query on the returned connection. Release the checkout at the +original statement boundary, before another transaction, page or unrelated +work. For a single statement, a temporary such as +`query.fetch_one(&mut *ledger.acquire().await?).await` keeps that scope small. +Do not wrap a whole query or transaction in the acquisition timer, and do not +add a second observation around `Ledger::begin()`. + +The histogram's cumulative buckets describe completed acquisition durations. +For waits below its first finite bucket (10 ms), `_sum / _count` gives an +average, not a percentile. Compare rates over the same interval, handle a +zero observation count, and retain appropriate instance/result grouping. +See [histogram consumer guidance](prism-metrics-histogram-consumers.md) for +query examples and bucket-resolution limits. diff --git a/docs/prism-postgres-alert-rules.json b/docs/prism-postgres-alert-rules.json new file mode 100644 index 000000000..272a3ea25 --- /dev/null +++ b/docs/prism-postgres-alert-rules.json @@ -0,0 +1,40 @@ +{ + "rules": [ + { + "uid": "qbit-prism-ha-standby-replay-lag", + "title": "PrismHAStandbyReplayLagHigh", + "expr": "clamp_max((max(up{job=\"qbit-postgres-primary\",network=\"__NETWORK__\"} != bool 1) or vector(1)) + (max(pg_up{job=\"qbit-postgres-primary\",network=\"__NETWORK__\"} != bool 1) or vector(1)) + (max(pg_exporter_last_scrape_error{job=\"qbit-postgres-primary\",network=\"__NETWORK__\"} != bool 0) or vector(1)) + ((count(up{job=\"qbit-postgres-primary\",network=\"__NETWORK__\"}) != bool 1) or vector(1)) + (max((pg_stat_replication_count{job=\"qbit-postgres-primary\",network=\"__NETWORK__\",application_name=\"prism_standby_1\"} < bool 0) + ((time() - timestamp(pg_stat_replication_count{job=\"qbit-postgres-primary\",network=\"__NETWORK__\",application_name=\"prism_standby_1\"})) < bool 0) + ((time() - timestamp(pg_stat_replication_count{job=\"qbit-postgres-primary\",network=\"__NETWORK__\",application_name=\"prism_standby_1\"})) > bool 2)) or vector(1)) + (max((pg_stat_replication_primary_flush_lsn_hi{job=\"qbit-postgres-primary\",network=\"__NETWORK__\",application_name=\"prism_standby_1\"} < bool 0) + ((time() - timestamp(pg_stat_replication_primary_flush_lsn_hi{job=\"qbit-postgres-primary\",network=\"__NETWORK__\",application_name=\"prism_standby_1\"})) < bool 0) + ((time() - timestamp(pg_stat_replication_primary_flush_lsn_hi{job=\"qbit-postgres-primary\",network=\"__NETWORK__\",application_name=\"prism_standby_1\"})) > bool 2)) or vector(1)) + (max((pg_stat_replication_primary_flush_lsn_lo{job=\"qbit-postgres-primary\",network=\"__NETWORK__\",application_name=\"prism_standby_1\"} < bool 0) + ((time() - timestamp(pg_stat_replication_primary_flush_lsn_lo{job=\"qbit-postgres-primary\",network=\"__NETWORK__\",application_name=\"prism_standby_1\"})) < bool 0) + ((time() - timestamp(pg_stat_replication_primary_flush_lsn_lo{job=\"qbit-postgres-primary\",network=\"__NETWORK__\",application_name=\"prism_standby_1\"})) > bool 2)) or vector(1)) + (max((pg_stat_replication_replay_lsn_hi{job=\"qbit-postgres-primary\",network=\"__NETWORK__\",application_name=\"prism_standby_1\"} < bool 0) + ((time() - timestamp(pg_stat_replication_replay_lsn_hi{job=\"qbit-postgres-primary\",network=\"__NETWORK__\",application_name=\"prism_standby_1\"})) < bool 0) + ((time() - timestamp(pg_stat_replication_replay_lsn_hi{job=\"qbit-postgres-primary\",network=\"__NETWORK__\",application_name=\"prism_standby_1\"})) > bool 2)) or vector(1)) + (max((pg_stat_replication_replay_lsn_lo{job=\"qbit-postgres-primary\",network=\"__NETWORK__\",application_name=\"prism_standby_1\"} < bool 0) + ((time() - timestamp(pg_stat_replication_replay_lsn_lo{job=\"qbit-postgres-primary\",network=\"__NETWORK__\",application_name=\"prism_standby_1\"})) < bool 0) + ((time() - timestamp(pg_stat_replication_replay_lsn_lo{job=\"qbit-postgres-primary\",network=\"__NETWORK__\",application_name=\"prism_standby_1\"})) > bool 2)) or vector(1)) + (max((pg_stat_replication_primary_flush_lsn_hi{job=\"qbit-postgres-primary\",network=\"__NETWORK__\",application_name=\"prism_standby_1\"} offset 5s < bool 0) + ((time() - timestamp(pg_stat_replication_primary_flush_lsn_hi{job=\"qbit-postgres-primary\",network=\"__NETWORK__\",application_name=\"prism_standby_1\"} offset 5s)) < bool 5) + ((time() - timestamp(pg_stat_replication_primary_flush_lsn_hi{job=\"qbit-postgres-primary\",network=\"__NETWORK__\",application_name=\"prism_standby_1\"} offset 5s)) > bool 7)) or vector(1)) + (max((pg_stat_replication_primary_flush_lsn_lo{job=\"qbit-postgres-primary\",network=\"__NETWORK__\",application_name=\"prism_standby_1\"} offset 5s < bool 0) + ((time() - timestamp(pg_stat_replication_primary_flush_lsn_lo{job=\"qbit-postgres-primary\",network=\"__NETWORK__\",application_name=\"prism_standby_1\"} offset 5s)) < bool 5) + ((time() - timestamp(pg_stat_replication_primary_flush_lsn_lo{job=\"qbit-postgres-primary\",network=\"__NETWORK__\",application_name=\"prism_standby_1\"} offset 5s)) > bool 7)) or vector(1)) + (max((pg_stat_replication_async{job=\"qbit-postgres-primary\",network=\"__NETWORK__\",application_name=\"prism_standby_1\"} < bool 0) + ((time() - timestamp(pg_stat_replication_async{job=\"qbit-postgres-primary\",network=\"__NETWORK__\",application_name=\"prism_standby_1\"})) < bool 0) + ((time() - timestamp(pg_stat_replication_async{job=\"qbit-postgres-primary\",network=\"__NETWORK__\",application_name=\"prism_standby_1\"})) > bool 2)) or vector(1)) + (max(pg_stat_replication_async{job=\"qbit-postgres-primary\",network=\"__NETWORK__\",application_name=\"prism_standby_1\"} != bool 1) or vector(1)) + (max((pg_stat_replication_replay_lsn_hi{job=\"qbit-postgres-primary\",network=\"__NETWORK__\",application_name=\"prism_standby_1\"} < bool on(job, instance, network, application_name) pg_stat_replication_primary_flush_lsn_hi{job=\"qbit-postgres-primary\",network=\"__NETWORK__\",application_name=\"prism_standby_1\"} offset 5s) + ((pg_stat_replication_replay_lsn_hi{job=\"qbit-postgres-primary\",network=\"__NETWORK__\",application_name=\"prism_standby_1\"} == bool on(job, instance, network, application_name) pg_stat_replication_primary_flush_lsn_hi{job=\"qbit-postgres-primary\",network=\"__NETWORK__\",application_name=\"prism_standby_1\"} offset 5s) * on(job, instance, network, application_name) (pg_stat_replication_replay_lsn_lo{job=\"qbit-postgres-primary\",network=\"__NETWORK__\",application_name=\"prism_standby_1\"} < bool on(job, instance, network, application_name) pg_stat_replication_primary_flush_lsn_lo{job=\"qbit-postgres-primary\",network=\"__NETWORK__\",application_name=\"prism_standby_1\"} offset 5s))) or vector(1)), 1)", + "condition_type": "threshold", + "evaluator": "gt", + "threshold": 0, + "for": "1m", + "severity": "warning", + "service": "qbit-mining-pool", + "labels": { + "alert_class": "prism-database-ha" + }, + "summary": "PrismHAStandbyReplayLagHigh", + "description": "D3: standby replay has not reached the primary durable WAL prefix observed five seconds earlier, or position/history/exporter observations are unknown, stale or failed, or the required async topology is invalid. Deployment-provided: requires primary SQL query metrics and one-second scrapes; verify in #281/#291. Idle caught-up positions are healthy even when PostgreSQL reports NULL replay_lag. The five-second comparison is sampled, with up to one scrape interval of timing uncertainty.", + "no_data_state": "Alerting", + "exec_err_state": "Alerting" + }, + { + "uid": "qbit-prism-ha-standby-disconnected", + "title": "PrismHAStandbyDisconnected", + "expr": "clamp_max((max(up{job=\"qbit-postgres-primary\",network=\"__NETWORK__\"} != bool 1) or vector(1)) + (max(pg_up{job=\"qbit-postgres-primary\",network=\"__NETWORK__\"} != bool 1) or vector(1)) + (max(pg_exporter_last_scrape_error{job=\"qbit-postgres-primary\",network=\"__NETWORK__\"} != bool 0) or vector(1)) + ((count(up{job=\"qbit-postgres-primary\",network=\"__NETWORK__\"}) != bool 1) or vector(1)) + (max((pg_stat_replication_count{job=\"qbit-postgres-primary\",network=\"__NETWORK__\",application_name=\"prism_standby_1\"} < bool 0) + ((time() - timestamp(pg_stat_replication_count{job=\"qbit-postgres-primary\",network=\"__NETWORK__\",application_name=\"prism_standby_1\"})) < bool 0) + ((time() - timestamp(pg_stat_replication_count{job=\"qbit-postgres-primary\",network=\"__NETWORK__\",application_name=\"prism_standby_1\"})) > bool 2)) or vector(1)) + (max(pg_stat_replication_count{job=\"qbit-postgres-primary\",network=\"__NETWORK__\",application_name=\"prism_standby_1\"} != bool 1) or vector(1)), 1)", + "condition_type": "threshold", + "evaluator": "gt", + "threshold": 0, + "for": "1m", + "severity": "warning", + "service": "qbit-mining-pool", + "labels": { + "alert_class": "prism-database-ha" + }, + "summary": "PrismHAStandbyDisconnected", + "description": "D3: the primary reports zero or ambiguous rows for prism_standby_1, or exporter/query observations are missing, stale or failed. Deployment-provided: requires the primary exporter and one-second scrapes; verify in #281/#291. This is the dedicated asynchronous HA standby, not the public read replica.", + "no_data_state": "Alerting", + "exec_err_state": "Alerting" + } + ] +} diff --git a/docs/prism-postgres-exporter-queries.yaml b/docs/prism-postgres-exporter-queries.yaml new file mode 100644 index 000000000..7e66b4b77 --- /dev/null +++ b/docs/prism-postgres-exporter-queries.yaml @@ -0,0 +1,57 @@ +# Deployment-provided D3 query extension on the PRIMARY, not the public replica. +# Configure through the deployment's supported SQL-query exporter mechanism; +# stock collector versions differ. Verify the query metrics and exporter status +# in #281/#291. Scrape this primary target every 1s; the rule compares positions +# with the primary's durable WAL position observed 5s earlier. +# pg_up, pg_exporter_last_scrape_error and Prometheus up must share this target. +pg_stat_replication: + query: | + WITH primary_wal AS ( + SELECT pg_wal_lsn_diff(pg_current_wal_flush_lsn(), '0/0') AS durable_position + ), standby AS ( + SELECT COUNT(*) AS count, + BOOL_AND(sync_state = 'async') AS is_async, + COUNT(*) = 1 AND BOOL_AND(state = 'streaming' AND replay_lsn IS NOT NULL) AS known, + MIN(pg_wal_lsn_diff(replay_lsn, '0/0')) AS replay_position, + MAX(EXTRACT(EPOCH FROM replay_lag)) AS reported_lag + FROM pg_stat_replication + WHERE application_name = 'prism_standby_1' + ) + SELECT 'prism_standby_1' AS application_name, + count::double precision, + CASE WHEN count <> 1 THEN -1 + WHEN BTRIM(current_setting('synchronous_standby_names')) <> '' THEN 0 + ELSE COALESCE(is_async::int, -1) END::double precision AS async, + FLOOR(durable_position / 4294967296)::double precision AS primary_flush_lsn_hi, + MOD(durable_position, 4294967296)::double precision AS primary_flush_lsn_lo, + CASE WHEN known THEN FLOOR(replay_position / 4294967296) ELSE -1 END::double precision AS replay_lsn_hi, + CASE WHEN known THEN MOD(replay_position, 4294967296) ELSE -1 END::double precision AS replay_lsn_lo, + CASE WHEN NOT COALESCE(known, false) THEN -1 + WHEN replay_position >= durable_position THEN 0 + ELSE COALESCE(reported_lag, -1) END::double precision AS replay_lag + FROM primary_wal CROSS JOIN standby + metrics: + - application_name: + usage: LABEL + description: Dedicated failover standby application name + - count: + usage: GAUGE + description: Matching replication rows; zero if disconnected + - replay_lag: + usage: GAUGE + description: Diagnostic last-reported latency; zero only with caught-up WAL proof, -1 if unknown + - async: + usage: GAUGE + description: One async standby with no primary synchronous requirement is 1, invalid topology is 0, unknown or ambiguous is -1 + - primary_flush_lsn_hi: + usage: GAUGE + description: High 32 bits of primary durable WAL position, exact in Prometheus float64 + - primary_flush_lsn_lo: + usage: GAUGE + description: Low 32 bits of primary durable WAL position, exact in Prometheus float64 + - replay_lsn_hi: + usage: GAUGE + description: High 32 bits of known streaming standby replay position, or -1 + - replay_lsn_lo: + usage: GAUGE + description: Low 32 bits of known streaming standby replay position, or -1 diff --git a/docs/prism-postgres-replica.md b/docs/prism-postgres-replica.md index 50c2c7b6a..1c4e7d667 100644 --- a/docs/prism-postgres-replica.md +++ b/docs/prism-postgres-replica.md @@ -1,8 +1,16 @@ # PRISM Postgres read replica -`prism-public-api` — the public dashboard and audit read tier extracted from -the coordinator in issue #145 — answers from a hot standby, not from the -coordinator's primary. This document covers how that standby is provisioned in +For two mining frontends, the operator TCP load-balancer contract and the approved +D3 promotion/ACK policy, see the [HA reference architecture](prism-ha-reference-architecture.md). +D3 selects one primary plus one **dedicated asynchronous failover standby**, +separate from this public-read replica. ACKs require local WAL durability without +waiting for the standby; primary loss can lose the replication gap. The bundled +public-read replica does not provision the dedicated HA standby. + +`prism-public-api` runs `qbit-prism-server public-api` and serves public reads +from a hot standby in the default Compose topology. The native process uses a +separate SQL pool, requires no signing keys, and serves canonical audit bytes +from PostgreSQL without an artifact filesystem mount. This document covers how that standby is provisioned in compose, what a production deployment has to do by hand instead, and how to operate the replication slot it depends on. @@ -94,23 +102,12 @@ connection from host ...`. | Variable | Default | Effect | | --- | --- | --- | | `PRISM_POSTGRES_REPLICATION_SLOT` | `prism_public_replica` | Physical slot the standby creates and streams through. | -| `PRISM_POSTGRES_REPLICA_DATA_SOURCE` | `prism-postgres-replica-data` | Volume or host path backing the standby's data directory. Release-provenance validation requires an absolute host path that neither overlaps nor aliases any other enabled state source (including ancestors, descendants and symlinks). Pre-create it before production startup. | -| `PRISM_PUBLIC_DATABASE_URL` | Empty (derived by the public service) | Read-only DSN `prism-public-api` uses. Empty builds a URL for `prism-postgres-replica:5432` using `PRISM_POSTGRES_USER`, `PRISM_POSTGRES_PASSWORD`, and `PRISM_POSTGRES_DB`. An explicit URL overrides that default. Compose passes an explicit URL as `PRISM_DATABASE_URL`; otherwise the public service percent-encodes the separate credentials when constructing its URL. The coordinator keeps its separate primary DSN. | -| `PRISM_PUBLIC_PSQL_COMMAND` | Empty | Optional command for the public psql backend. Compose passes it as `PRISM_POSTGRES_PSQL_COMMAND` only to the public service; empty derives `psql` from `PRISM_PUBLIC_DATABASE_URL`. The coordinator keeps its own `PRISM_POSTGRES_PSQL_COMMAND`. | +| `PRISM_POSTGRES_REPLICA_DATA_SOURCE` | `prism-postgres-replica-data` | Volume or host path backing the standby's data directory. | +| `PRISM_PUBLIC_DATABASE_URL` | `postgresql://qbit:change-this@prism-postgres-replica:5432/qbit` | Read-only DSN `prism-public-api` uses. Compose passes it into the container as `PRISM_DATABASE_URL`; it is a separate operator knob because `PRISM_DATABASE_URL` names the primary, which the coordinator needs it to. | +| `PRISM_PUBLIC_POSTGRES_PASSWORD` | Empty | Optional dedicated public-reader password, passed by Compose as `PGPASSWORD`. Used when the public DSN omits its password; never defaults to `PRISM_POSTGRES_PASSWORD`. | | `PRISM_PUBLIC_REPLICA_MODE` | `require` in compose, `off` in code | `require` refuses replica-backed routes unless the backing server is in recovery with a live replication stream. `off` serves whatever the DSN names — the behaviour before a standby existed. | | `PRISM_PUBLIC_REPLICA_MAX_LAG_SECONDS` | `60` | How long the standby's replication stream may be silent before its answers stop counting as current. | -Leave `PRISM_PUBLIC_DATABASE_URL` empty for the built-in standby so the public -connection follows the configured Postgres credentials. For an external standby -or separate read credentials, set the full URL explicitly and update it when -those credentials change. - -With `PRISM_POSTGRES_NATIVE_CLIENT=psql`, the public service must still connect -to the standby. Leave `PRISM_PUBLIC_PSQL_COMMAND` empty to use the public DSN, -or set a separate command that reaches the replica if a wrapper or extra psql -arguments are required. The coordinator's primary command is never inherited -by the public service. - `PRISM_PUBLIC_REPLICA_MODE` defaults to `off` in the code and `require` in the shipped compose. That split is deliberate: merging the replica work must not 503 a deployment that has not provisioned a standby yet, but anyone who brings @@ -118,6 +115,137 @@ the stack up from this repository's compose gets the enforced contract without having to opt in. Cut over by provisioning the standby, pointing `PRISM_PUBLIC_DATABASE_URL` at it, and then setting `require`. +### Public-reader credentials + +For a dedicated reader, set `PRISM_PUBLIC_DATABASE_URL` to that role's complete +DSN, including its own password (percent-encode reserved URL characters): + +```dotenv +PRISM_PUBLIC_DATABASE_URL=postgresql://prism_reader:<encoded-reader-password>@standby.internal:5432/qbit?sslmode=verify-full +``` + +Alternatively, omit the password from the DSN and set +`PRISM_PUBLIC_POSTGRES_PASSWORD` to the reader's raw password. In `.env` files, +single-quote that value so Compose preserves dollar signs and comment characters +literally; do not percent-encode this separate carrier. For example, these +synthetic values preserve the password `reader$literal #:@/383`: + +```dotenv +PRISM_PUBLIC_DATABASE_URL=postgresql://prism_reader@standby.internal:5432/qbit?sslmode=verify-full +PRISM_PUBLIC_POSTGRES_PASSWORD='reader$literal #:@/383' +``` + +If the password contains a single quote, use the percent-encoded DSN form above +so the `.env` file also remains valid for the shell-based preflight scripts. +Set the passwordless reader DSN as well: the carrier alone does not select a +reader role, and a default DSN with an embedded bootstrap password still takes +precedence. +Compose maps these two settings to `PRISM_DATABASE_URL` and `PGPASSWORD` inside +`prism-public-api`. +The native SQLx driver loads PostgreSQL environment defaults before applying +the DSN; a DSN password takes precedence over `PGPASSWORD`. The public process +receives no extra bootstrap password, even when production, external-database +or HA overlays are applied. Keep reader and bootstrap passwords distinct. + +Create the reader on the primary so the role and grants reach its physical +standby. Grant `CONNECT` on the database, `USAGE` on the application schema and +`SELECT` on the tables used by the public read models. It must not own the +database or tables, inherit a writer role, or have superuser, replication, +create-role or create-database privileges. Reconcile read grants when migrations +add public tables. In `require` mode it also needs `pg_read_all_stats` so the +readiness probe can see `pg_stat_wal_receiver.last_msg_receipt_time`; this +monitoring grant does not permit replication or application writes. Role +provisioning remains an operator task. + +With password authentication required by PostgreSQL, an omitted, empty or +incorrect reader password leaves `/healthz` unready with an authentication +error; database-backed public reads return 503 and +`qbit-prism-server healthcheck --public-api` exits unsuccessfully. The listener +stays up to report the failure and retry its readiness probe. It does not borrow +`PRISM_POSTGRES_PASSWORD`. The HTTP healthcheck needs no database password of +its own. Non-database public metadata can still be served while unready. + +Compatibility: an unset or empty `PRISM_PUBLIC_DATABASE_URL` retains the base +Compose lab DSN with the bootstrap password embedded; the external-database +overlay instead retains its fallback to `PRISM_DATABASE_URL` and replica mode +`off`. These defaults do not provide a dedicated-reader boundary. For an +external fallback DSN without a password, explicitly supply the intended +public credential with `PRISM_PUBLIC_POSTGRES_PASSWORD`. The former implicit +bootstrap `PGPASSWORD` is no longer supplied. A direct binary or `docker run` +invocation still uses `PRISM_DATABASE_URL` and SQLx's native PostgreSQL settings; +the `PRISM_PUBLIC_*` credential names above are Compose inputs only. Supply only +reader credentials to that process, including any `PGPASSWORD` or password file. +The shipped Compose service sets `PGPASSWORD` to empty when its reader carrier +is unset, so it does not select a password file as an implicit replacement. + +#### Preflight and production policy + +`public-api` and `check-public-database-config` use the same SQLx options parser +and production policy. The existing `DatabaseConfig` contract rejects the +`change-this` marker anywhere in a production DSN. Public-reader validation also +rejects that marker in the **effective decoded password**, including a URI or +query password, `PGPASSWORD`, or a native SQLx password-file entry. An unused +carrier does not override or invalidate an explicit non-default DSN password. +The `qbit` role name is permitted; the lab password remains permitted outside +production. This is a default-credential check, not a password-strength or +database-role privilege check. + +Production is selected by the public process's `QBIT_PRODUCTION`, +`QBIT_TOOLS_PRODUCTION`, or a `QBIT_CHAIN` of `main`/`mainnet`, following +`DatabaseConfig`. A missing, empty, malformed, unsupported-scheme, or invalid +SQLx-options **effective DSN** fails before connecting or opening the listener, +with a value-free configuration error. Missing, empty, or incorrect **passwords** +are not rejected solely for their absence: PostgreSQL may use another supported +authentication mechanism. Password authentication failures remain readiness +failures as described above. Validation neither connects to PostgreSQL nor +proves authentication, grants, schema availability, or replication health. + +`scripts/check-env.sh` now runs this native validator for the Prism lane through +the selected Compose public service, using the image and effective environment +that deployment will use. `make doctor` and `make up-prism-pool` prepare tagged +lab images before validation, including rebuilding an older image that lacks +the validator. Production and digest-pinned artifacts must be pulled explicitly +before doctor, following the mainnet runbook. The standalone script likewise +requires a prepared image. The check starts no service dependencies and publishes no ports; +its one-shot container is removed on exit. A missing/old image or failed +launcher is a failed preflight, never a passing or skipped credential check. +For Make entry points, use the same additional overlays in deployment order: + +```sh +export COMPOSE_OVERLAY_FILES="compose.prism-external-db.yaml compose.prism-ha.yaml" +MINING_LANES=prism make doctor +make up-prism-pool +``` + +Make inserts `compose.production.yaml` before these overlays when either +production flag or the main chain selects production. Set +`PRISM_COMPOSE_SERVICES="qbitd prism-coordinator prism-coordinator-2 prism-public-api"` +to start both HA frontends. Database dependencies follow the selected overlay. +For standalone preflight, pass every overlay explicitly, for example from the +repository root (relative `--compose-file` paths use the caller's directory): + +```sh +DEPLOY_ENV_FILE=/absolute/path/deployment.env bash scripts/check-env.sh \ + --compose-file compose.production.yaml \ + --compose-file compose.prism-external-db.yaml \ + --compose-file compose.prism-ha.yaml +``` + +`--public-reader-only` runs just this credential configuration preflight; it +does not replace the other deployment checks. The preflight preserves Compose's +environment precedence, explicit empty overrides, and base/external DSN +fallbacks, without sourcing shell example defaults into the container. It +uses only the public service's reader carrier and does not add a bootstrap or +writer password. For direct binary execution, export the reader's +`PRISM_DATABASE_URL` and, if needed, its `PGPASSWORD` or `PGPASSFILE`, then run +`qbit-prism-server check-public-database-config` before `public-api`. +Parser warnings are suppressed because they can contain configuration values. +When a native password file does not authenticate, check its existence, +ownership, restrictive permissions, and matching reader entry locally without +printing its contents. Authentication tests require a disposable PostgreSQL +fixture with password authentication enabled; a trust-auth fixture cannot +exercise rejected passwords. + ### What the bound actually bounds The enforced number is the **walreceiver heartbeat age**, not replay lag. @@ -144,8 +272,8 @@ rather than an indefinite 200. Routes that read no replica state are not gated — today that is `/public/v1/mining-configuration`, assembled from environment, and the content-addressed `/public/v1/artifacts/{sha256}`, whose body either hashes to -the requested digest or does not. The classification comes from -`lab/prism/endpoint_registry.py`, not a hand-kept list. +the requested digest or does not. The native route classification preserves these two freshness-independent +responses; artifact bytes still undergo content-hash verification. | `PRISM_POSTGRES_REPLICA_BASEBACKUP_ATTEMPTS` | `60` | Bootstrap retry budget. | | `PRISM_POSTGRES_REPLICA_BASEBACKUP_RETRY_SECONDS` | `5` | Delay between bootstrap attempts. | @@ -233,18 +361,22 @@ path for a slot invalidated with `wal_status = 'lost'`. ## Failover semantics -The replica is a **read scaling tier, not a failover target**. - -- It is read-only. Writes fail with `cannot execute INSERT in a read-only - transaction`. -- There is **no automated promotion anywhere in this repository** — no - `pg_promote` call, no `promote_trigger_file`, no cluster manager. A standby - only becomes writable if an operator promotes it by hand. -- Promotion is not a supported operation for this standby. The coordinator's - writer lease, ledger attestation, and audit invariants are all defined - against the primary; promoting the read replica does not produce a valid - PRISM primary. Losing the primary is a primary-restore problem, and - `docs/prism-ledger-ops.md` is the relevant runbook. -- Because promotion is out of scope, the standby's healthcheck fails when - `pg_is_in_recovery()` stops returning `t`. A promoted standby is a - misconfiguration, and it is meant to be loud. +The supplied public replica provides read scaling. Its default asynchronous +replication does not guarantee preservation of acknowledged shares after loss +of the primary. No cluster manager or automatic promotion is configured by +these Compose files. + +Native mining frontends can recover through a stable HA writer endpoint after +promotion of the eligible dedicated standby. Follow the approved async D3 +policy, fencing and reconciliation in the [HA reference architecture](prism-ha-reference-architecture.md#promotion-fencing-and-the-stable-writer-endpoint); +acknowledged shares absent from the promoted replica can be lost. Preserve the +signing configuration and reconcile the surviving accounting history; there is +no application writer lease to elect. The [HA migration guide](prism-rust-migration.md#ha-durability-and-failover) +also describes the distinct requirements for a no-loss policy, which D3 does not select. + +A public process configured with `PRISM_PUBLIC_REPLICA_MODE=require` refuses a +server after it is promoted: it is now a writer rather than the required +standby. Repoint the public read endpoint at the separate public-read replica, or +explicitly use `off` to serve bounded reads from the new primary. Keep mining +and settlement connections on the authoritative writer endpoint, and keep public +queries off the dedicated HA standby. diff --git a/docs/prism-prepared-qualification.md b/docs/prism-prepared-qualification.md new file mode 100644 index 000000000..3420c5d87 --- /dev/null +++ b/docs/prism-prepared-qualification.md @@ -0,0 +1,146 @@ +# Prepared-work qualification preparation + +This is preparation for #273, not refresh/resume acceptance evidence. At +`3.x.x` base `12fda8119be0d7b115036a3047453d8ac0704e68`, the coordinator still persists and +decodes inline `StoredPrepared`. The additive compact storage API is not its +runtime path. No 400k or 500k database run, WAL budget, retention guarantee, +or canonical A-to-B rebuild result is established by this preparation. + +## Runnable preparation + +```sh +cargo +1.98.1 test --locked -p qbit-prism-server \ + --test prepared_qualification_contract +``` + +The target reuses [WindowPlan](../crates/qbit-prism-server/tests/support/window_fixture.rs) +from #264. Both 400,000 and 500,000 divide its existing window weight exactly; +the tests check production-shaped serialized rows at sequence-width boundaries, +the midpoint, and the final row without allocating either full window. The +size band applies to the sampled average, matching the fixture's average-byte +contract rather than imposing that band on each row. +They test the failure behavior of the reusable +[qualification assertions](../crates/qbit-prism-server/tests/support/prepared_work_assertions.rs). +Synthetic byte strings and byte counts in these unit tests are guard inputs, +never measured production evidence. There are no ignored or database-gated +acceptance tests here and no new manifest IDs. + +## Share rows versus the literal key + +#273 literally requires the stored payload to have no `shares` key. The existing +stored format does contain that key: `CompactPrepared.window` embeds +`WindowRef.shares: Option<ShareRange>`. This preparation explicitly adopts a +**no-materialized-share-rows** guard; it does not establish unchanged literal-key +acceptance or close that checkbox. Final qualification and reconciliation of +the literal wording remain open on #273. + +`assert_no_materialized_shares` rejects every `shares` key except at the exact +root `window.shares` path. That exception requires the complete, existing +`WindowRef` serialization: integer `anchor_ms`, 64-character lowercase-hex +`prior_balances_digest`, and `shares` as either null (empty window) or an object +with exactly `first_share_seq`, `last_share_seq`, `share_count`, and +`snapshot_sha256`. Sequence bounds must be positive, ordered and fit SQL bigint; +the positive count may not exceed the inclusive span (gaps are valid), and the +snapshot digest must be 64-character lowercase hex. Missing, extra or malformed +metadata fields fail. Arrays at `window.shares`, hidden arrays, and any `shares` +key elsewhere fail, including nulls and metadata-shaped objects. + +The guard reuses `WindowRef` deserialization and requires exact serialization +round-trip equality, since the existing deserializer otherwise ignores unknown +fields and defaults a missing optional range. It checks payload structure, not +digest authenticity or the full prepared-record contract. No stored field, +format version, runtime serializer or upstream schema changes here. + +The serializer trace is `Ledger::save_compact_prepared` to +[`encode_record`](../crates/qbit-prism-server/src/ledger/jobs/prepared.rs), which +calls `serde_json::to_value(record)` and adds `original_expires_at_ms` without +renaming or removing the range. The pure tests build actual `CompactPrepared` +records from empty and nonempty `Snapshot` references and use that serialization +plus expiry envelope. They exercise both valid shapes and legacy/hidden-array +negative controls without claiming a database write or runtime activation. + +## Runtime binding for the integration owner + +Reuse [the existing fake node and coordinator configuration](../crates/qbit-prism-server/tests/support/fake_qbitd.rs) +and the two-coordinator pattern in +[candidate_window_qualification.rs](../crates/qbit-prism-server/tests/candidate_window_qualification.rs). +Use a dedicated disposable loopback PostgreSQL primary, serial execution, +and a fresh schema for each size. A fresh schema alone does not isolate +server-wide WAL or database-wide advisory locks. Start no background submit, +refresh, or collector loops during the measurement bracket. + +1. Create frontends A and B against the same schema and fake node, with + distinct instance IDs. Load `WindowPlan::new(n).load(...)` once, before + measurement, and use `verify_round_trip` on first/middle/last rows. +2. Capture the insert LSN immediately before and after awaiting A's actual + `Coordinator::refresh_once`. A failed refresh is a failed operation; do + not seed a substitute job, omit the failure, or report zero WAL. Record + the published snapshot length and require exactly `n`. +3. Observe every JSONB column written by that refresh using the catalog/write + attribution approach in [jsonb_ceiling_gate.rs](../crates/qbit-prism-server/tests/jsonb_ceiling_gate.rs). + Its `Inventory` is file-private: sharing it requires a separate coordinated + extraction, not copying a second inventory or weakening the all-pipeline + ratchet. Pass the largest uncompressed value to `assert_refresh_measurements`; + compressed `pg_column_size` alone does not prove the whole-value bound. + Inspect the actual persisted prepared payload with `assert_no_materialized_shares`. + Record its range-metadata exception explicitly; this is not proof of the + literal no-`shares`-key criterion. +4. Refresh B outside A's WAL bracket so B has its own publication authority. + Build and persist an issued job on A through `MiningBackend::build_job` and + `persist_issued_job`, then call `MiningBackend::resume_job` on B with the + same worker and ID. Require `Some`, not a cache miss. Compare the real + canonical audit bytes with `assert_canonical_equality`; also compare job + ID, original worker, target, difficulty, extranonces and version mask. + Preserve the issued absolute deadline through the existing caller timeout. + The future body representation must use `canonical_audit_bundle_bytes_from_parts` + with the reconstructed shares, not native-window or legacy-window digests. +5. Repeat A's unchanged refresh separately and require cached prepared identity + and unchanged stored payload. Exercise a revision change between snapshot + construction and save through the existing coordinator test boundary: the + save must fail without publishing replacement work. A cached run is not a + substitute for the measured non-cached refresh budget. + Cached no-write reuse may legitimately observe zero WAL; report it separately + with unchanged prepared identity/payload and no observed writes. Do not pass + it to `assert_refresh_measurements`, which requires a logged prepared write + and a positive WAL delta. Neither missing measurements nor a zero delta + alongside an observed logged write can qualify the non-cached refresh. +6. Reuse the issued-dependency/compact-storage race tests for retention: both + GC-before-repair and repair-before-GC must preserve referenced template, + balance and share data through the original issued deadline. A pending + candidate remains a reference after its claim lease expires; after its + terminal transition, a zero-job sweep must reclaim unreferenced blobs. + Include unsupported-format and malformed-record errors without rewriting + their expected outcomes. These tests belong to the runtime/repair owners; + the preparation target does not simulate them. + +Activation requires replacing coordinator refresh's inline serialization and +resume's inline decode with the compact storage/reader APIs, preserving final +authority and deadline checks, and integrating issued dependency repair. +The private revision-interleaving and JSONB observation seams also need owner +wiring. Coordinate any newly executable database test ID with the manifest +owner; never reuse an existing ID or register this preparation as acceptance. + +## Measurement record + +The [#330 WAL helper](../scripts/measure_postgres_wal.py) establishes explicit +operation outcome, server-wide attribution and missing-measurement handling. +It brackets SQL through separate `psql` calls, so it cannot itself bracket a +Rust coordinator refresh. Use the same attribution discipline, with +`pg_current_wal_insert_lsn()` around the awaited Rust operation and +`pg_wal_lsn_diff(after,before)` for integer bytes. Do not attribute fixture +loading, frontend initialization, B's refresh or mining to A's refresh. + +Record the exact source/base SHAs, PostgreSQL version, share count, observed +share rows, separate metadata/probe/page query counts, JSONB bytes, WAL bytes, +refresh/resume wall time and outcome. Record `fsync`, `full_page_writes`, +`wal_compression`, checkpoint placement and first-seen versus reused template +and balance keys. Missing measurements remain unavailable, not zero or pass. +Both limits are strict decimal byte bounds: under 1,000,000 JSONB bytes and +under 5,000,000 WAL bytes for the isolated refresh. + +Run 400k regression and 500k headroom serially, then a large valid template +case. Include its first insert in WAL; do not hide it behind a warmed key. +Report actual OS-supported RSS separately from unsupported/unavailable RSS. +Replication and socket delivery need their own qualification: the reused +fake-node harness is two in-process coordinators, not physical hosts or a +socket-level miner test. diff --git a/docs/prism-prepared-window-reference-plan.md b/docs/prism-prepared-window-reference-plan.md new file mode 100644 index 000000000..994089c85 --- /dev/null +++ b/docs/prism-prepared-window-reference-plan.md @@ -0,0 +1,386 @@ +# Prepared work window reference: implementation checkpoint + +This is the current local integration plan, updated after the user's +[contract signoff pinned to bed6ad8](https://github.com/Qbit-Org/qbit-mining-bootstrap/pull/297#issuecomment-5638877264) +and [issue273 confirmation](https://github.com/Qbit-Org/qbit-mining-bootstrap/issues/273#issuecomment-5638878822). +The B-owned schema/reader implementation and measured results are recorded in +[the foundation report](prism-window-reference-foundation.md). Final runtime +integration still requires A265's actual candidate interfaces. + +Issue [#273](https://github.com/Qbit-Org/qbit-mining-bootstrap/issues/273), +workstream B. The approved replacement lease bridges a revision change only +while `prior_balances_digest` remains unchanged. Changed balances end that +lease immediately. Broader protection belongs with B8/#289. Approval is for +implementation, not full miner-behavior parity or a cutover waiver: qualification +must measure affected work and verify prompt replacement-job delivery. + +## Verified dependency state + +Checked 2026-09-11 against GitHub and current `3.x.x` at +`d39cf621ce25fea4d5a1e86b02cf6c3c66949ce5`. This child includes that base +additively through `48a5923` (merged PR313), `c6bce3a` (merged PR297), +and `b480ea1` (merged PR319): + +| Dependency | Observed state | Consequence | +| --- | --- | --- | +| [PR313](https://github.com/Qbit-Org/qbit-mining-bootstrap/pull/313) | Merged as `0af13ec9b405731ce4eb93fb473352c94972a730`, included | No remaining PR313 dependency; its original worktree is untouched. | +| [Issue264](https://github.com/Qbit-Org/qbit-mining-bootstrap/issues/264) | Closed by the design merge | No longer a design-merge blocker. B's exact reader/types are implemented locally. | +| [PR297](https://github.com/Qbit-Org/qbit-mining-bootstrap/pull/297) | Head `406b27344f53d9d84b68004adcae05b2f0f46f88` merged as `82a543d`; user signoff pins `bed6ad8` | The unchanged-balances restriction remains. Record later retention/candidate details without inferring a new cutover waiver or choosing an unmeasured retention grace. | +| [Alex's response](https://github.com/Qbit-Org/qbit-mining-bootstrap/pull/297#issuecomment-5637165179) | All four B objections answered; the user approved the narrowed contract | Implementation is not waiting for that signoff. | +| Borrowing builders and verification | Present in `crates/qbit-prism/src/lib.rs`; `AuditBundleBody`, four borrowed builders, parts verification/canonical serialization, public `prior_balances_digest` | Reuse these APIs. `AuditBundleBody.reward_manifest.shares` remains large and must never enter prepared JSONB. | +| Shared window reader | `d7526fc` implements the exact types, `read_window` and migration008 under local B authorization | Foundation remains usable; no A-owned `audit::read_range` edit. Local authorization does not imply A/C implementation review. | +| Builder version | `AUDIT_BUILDER_VERSION` absent | Latest design assigns its introduction and frozen-vector versioning to A's #265. B must not invent a second constant. | +| Candidate representation | `Candidate.bundle` and submit still require owned `AuditBundle` | Latest design's slim resumed jobs cannot preserve candidate reconstruction without #265 or explicit coordinated integration. | +| [A265 import PR325](https://github.com/Qbit-Org/qbit-mining-bootstrap/pull/325) | Open at `ff74abb0da65f15ddcd25689328a075f7ab2a1b4` | This supplies canonical import/read changes, not the candidate fields, builder constant, witness parser or 007. Import is not a missing B prerequisite. | +| [Source-schema PR321](https://github.com/Qbit-Org/qbit-mining-bootstrap/pull/321) | Open at `798860c1a79a05a536a9512208f1960220ff4ef9` | Adds 006/startup capabilities, not 007 or a slim candidate. Coordinate its runner with 008 when integrated; do not copy A-owned code here. | +| Migrations / [PR319](https://github.com/Qbit-Org/qbit-mining-bootstrap/pull/319) | Merged as `d39cf621ce25fea4d5a1e86b02cf6c3c66949ce5`; runner applies 002 through 005, 008 and 009 by membership | 009 is included unchanged from current base. No 006 or 007 is copied. Candidate-aware GC needs A's 007 fields and retention predicate. | + +The landed B276 runner and migration009 are now included through current +`3.x.x`, without a whole open-branch cherry-pick. The only production runner +addition is 008 membership. Migration009 still creates +`qbit_prism_jobs_extranonce1_expiry_idx` on +`(lower(payload->>'extranonce1'), expires_at)` and leaves existing jobs intact. +The regular B migration test now uses the landed 009 SQL, installs either +predecessor under `MIGRATION_LOCK`, and runs/restarts the real combined runner. +Both migration tests require the exact set `[2, 3, 4, 5, 6, 8, 9]`. Existing job +fields must match exactly, with only 008's seven new null columns added. +Do not duplicate or edit 009. Coordinate the final 006/007 manifest with A/C. + +No candidate, landing, import, deployment, metrics, CI, or incremental-window +change is authorized by this plan. + +### Post-signoff delta: b3e8ba9 + +The delta does not change the balance restriction, window API, or GC contract. +It explicitly preserves `JobContext.bootstrap_share: Option<AcceptedShare>` +for local empty-window jobs after replacing the full bundle with a body. The +original synthetic share must reach `Candidate.bootstrap_share` verbatim; a +`CountedShare` lacks required fields. This is an A/B handoff requirement for +the body-only conversion, not authorization to edit A's implementation here. + +Locally retained jobs keep each generation's `Prepared`, including its window, +until Stratum releases them. Memory accounting must include all retained local +generations as well as bounded concurrent rebuilds; the single-current-window +claim is invalid. A generation cap, eviction change, and acceptable total RSS +remain qualification/contract decisions, not an inferred approval to alter +retained miner work. The later record also names SQLx's whole-body bind copy on +landing; A265 owns its 400k stall measurement and any follow-up decision. + +### Landed follow-ups through 406b273 + +The empty-window singleflight entry keeps as-issued balances until every +waiter's bootstrap build finishes. The slim resumed job also retains those +balances for leased enqueue repair. A's enqueue re-inserts the digest-checked +balance snapshot inside its `ORDER_LOCK` transaction; that previously open +ordering question is now answered in the record and should not be asked again. +The concrete Rust enqueue input/exports remain unavailable in PR325. + +The record now calls for an expiry grace before physical job deletion, and for +expired-but-not-deleted jobs to retain D6's share floor. B must set and qualify +that grace above measured share-path latency without extending the child's +absolute eligibility deadline. The exact grace value is not specified or +implemented here. A still enqueues and alerts on an unexpectedly missing share +prefix so the block reaches the node; B does not change that candidate contract. +Leased dispatch submits first, then lands the audit before any terminal result, +even for an inactive block; null inactive results stay recoverable. These are +A-owned implementation requirements, not missing design answers. + +The additional cancellation note on #273 is implemented independently in B's +reader: accumulated state and completed vectors awaiting transaction commit +have a guard that hands destruction to a blocking thread on cancellation/error. +This also covers a completed blocking page whose awaiting future disappears. +Successful return transfers the vectors to the caller, which retains its own +off-thread cleanup obligation. The foundation report records its tests. + +## Proposed schema and storage contract + +Migration `008_prepared_window_reference.sql` belongs to B273. It adds: + +| Table | Columns / constraints | +| --- | --- | +| `qbit_prism_jobs` | `window_anchor_ms bigint`, `window_prior_balances_sha256 text`, `window_first_share_seq bigint`, `window_last_share_seq bigint`, `window_share_count bigint`, `window_snapshot_sha256 text`, `template_sha256 text` | +| `qbit_prism_templates` | `template_sha256 text PRIMARY KEY`, `template_bytes bytea NOT NULL` | +| `qbit_prism_balance_snapshots` | `prior_balances_digest text PRIMARY KEY`, `balances bytea NOT NULL` | + +All digest text is exactly 64 lowercase hex characters. The six window columns +use PR297's explicit `num_nulls`/`num_nonnulls` CHECK, accepting exactly: + +1. All six NULL: issued child rows and legacy inline prepared rows. +2. Anchor and balance digest present, four range columns NULL: empty window. +3. All six present: first >= 1, last >= first, and 1 <= count <= last-first+1. + +The reference in the payload and the SQL columns must agree exactly. The +template digest is present only on new prepared rows; malformed partial state +is a decode error, not a legacy miss. Index both the job template digest and +balance digest for bounded GC reference checks. Keep B276's issued +`extranonce1` at the same top-level JSON path and preserve its expression index. + +Template bytes are the original serialized template, hashed before storage; +their retrieval verifies the digest before decoding. Balance bytes contain the +as-issued vector in the bytewise order used by `prior_balances_digest` +(`order_key`, `recipient_id`, `p2mr_program_hex`). Refresh and reconstruction +use the same ordering. The balance key authenticates the semantic sorted set; +it is not a SHA-256 of those stored bytes. Insertion checks equal existing +bytes after `ON CONFLICT DO NOTHING`; a conflicting immutable row is an error. +Reject updates to either store's digest or content in the database; explicit +GC deletion remains allowed. Validate the insertion digest before publishing +the dependent prepared record, and validate it again on reconstruction. +Bytea keeps both external stores out of the whole-refresh JSONB size limit, +including a large recipient set. These column names and byte encoding are +locally implemented interfaces awaiting A/C integration review. + +`StoredPrepared` contains only the window reference, original `anchor_ms`, +`share_seq` and `payout_revision`, template digest, parent identity, +fingerprint, generation, original coinbase suffix, complete payout/CTV/fee +inputs, builder version, signer public keys, and original audit hashes. +It contains no snapshot, bundle, share array, prior-balance array, or template. +The optional `window.shares` value is a small range object, never an array. + +`Prepared` holds immutable repair inputs and the shared local snapshot/body. +Builds borrow `Snapshot.shares`, using the landed parts APIs. The design now +also requires each resumed job to release both `Snapshot.shares` and +`reward_manifest.shares` after constructing its coinbase. This requires the +coordinated candidate representation listed above; silently dropping shares +while the current submit path clones an owned bundle is not a valid adapter. + +## Shared window API + +Use PR297's approved signature and types, implemented in B-owned `ledger/window.rs`: + +```rust +pub async fn read_window( + &self, + window: &WindowRef, + balances: BalanceSource, +) -> Result<Window, WindowError>; +``` + +`WindowRef` carries `anchor_ms`, `prior_balances_digest: [u8; 32]`, and +`Option<ShareRange>`. The range contains first/last sequence, count, and the +native snapshot digest. Digest serde accepts only lowercase 64-hex strings. +`BalanceSource::{Current, AsIssued}` separates ordinary candidate claims from +original published economics (resume and leased-candidate audit). +`Window.payout_revision` is the +current transaction's fence; a resumed snapshot retains its stored original +revision. Candidate admission continues using its distinct fences. + +The reader uses one primary `REPEATABLE READ READ ONLY` transaction. Read the +revision and verified balance source first, return directly for an empty +range, probe endpoints, then fetch ascending keyset pages of at most 4096 +rows using the existing accepted/sequence/anchor predicate. Check count and +stream SHA-256 of native `AcceptedShare` JSON array bytes. This is the existing +audit snapshot digest; it is different from `PayoutWindow::canonical_digest_hex` +and from the digest of counted-share outputs. + +Row mapping, balance decoding, and hashing run in the reader's own +`spawn_blocking` hand-offs. Do not move or change A-owned `audit::read_range` +without agreement. Its current implementation is private and unpaged, so +merely calling it cannot meet this API's paging contract. + +Preserve typed `Incomplete`, `PriorBalancesChanged`, +`BalanceSnapshotMissing`, `SnapshotDigestMismatch`, `Database`, and `Decode` +errors. The as-issued source never substitutes current balances and never +reports a corrupt or absent balance blob as a missing job. + +### Coherent eligibility input (B-owned) + +`Ledger::payout_state() -> Result<PayoutState, WindowError>` reads +`{ payout_revision, prior_balances_digest }` from one primary repeatable-read +snapshot. It checks the same fatal/read-only guards as `payout_revision`, reads +only current balance rows, and computes the existing semantic digest off the +runtime. It neither reads share history nor consults the as-issued store. +It sets no private timeout, acquires no build permit, and returns observations, +not an admission decision. This helper is independently implemented and tested; +its existence does not imply that runtime callers use the narrowed policy yet. + +At integration, B's WorkLedger/SubmitLedger adapters must supply this coherent +pair, not independently awaited revision and digest lookups. The issued digest +comes from the authenticated `Prepared.window`, computed once at refresh, never +from current balances or a per-submit re-hash of the window. A lease requires +the exact published identity, a live lease and equal balance digests. A known +balance mismatch yields an ineligible resume (`Ok(None)`) or the corresponding +stale-work submit result; a failed state read remains backend-unavailable. +`read_window(AsIssued)` continues to reconstruct original economics even after +balances move: changing it to a current-balance check would break audit recovery. + +Recheck the publication/lease after the state read and all later waits, and +revalidate the coherent pair after rebuild/bootstrap before returning work. +Save/repair and append retain their final transactional current-revision +checks; an observation is not authority to commit after that revision changes. +Reuse the original operation deadline across these steps. Do not add a full +window read or unmeasured current-balance scan to every ordinary share: any +revision-keyed reuse of the small pair needs explicit freshness/fence tests. + +## Save, repair, and collection + +Prepared save inserts/reuses template and balance blobs and inserts the job +in one transaction under existing `SETTLEMENT_LOCK`, with the current-revision +and configuration-fingerprint fences. Immutable conflict checks cover blobs, +payload, reference columns, parent, and original revision. + +Extend `PreparedDependency` and its repair input to carry the external blob +identities and exact immutable bytes. The hot issued-save path reads compact +dependency metadata. If any dependency is missing, it rolls back and lets +the coordinator prepare repair inputs outside transaction locks. The cold +retry restores template, balances, prepared record, and child atomically. +Original prepared economics remain R0 even when admission succeeds under R1. +The child's absolute expiry is computed once and never renewed by repair, +lock waits, cancellation, or reconnect. Preserve PR313's dependency headroom, +post-wait authority checks, and immutable conflicts. + +GC runs two independent phases at the existing two-second cadence, without +overlapping batches: + +1. `prune_expired_jobs` deletes at most 4096 expired jobs in its original + lock-free statement, under the configured database timeout. The indexed + key selection preserves the outer `clock_timestamp()` expiry recheck after + a concurrent renewal's row-lock wait. No additional retention grace is added. + This statement commits and releases all row locks before blob GC starts. +2. `prune_unreferenced_blobs` starts a fresh five-second deadline and acquires + `SETTLEMENT_LOCK` **then `ORDER_LOCK`**, checks the shared writable fence, + and pages at most 256 keys from each blob table before filtering references. + Separate DELETE statements remove selected templates without any surviving + job reference, and selected balances without any surviving job **or candidate** + reference. Claim expiry does not affect retention, and unexpected terminal + references also retain their blobs until explicitly detached. + +Each blob table has an in-memory key cursor that advances past retained keys +only after commit and wraps after an empty page. Both pages run with zero +expired jobs, so terminal candidates' detached balances and other orphans are +eventually revisited even behind a live prefix. The sweep reads metadata, not +blob payloads or shares. Blob failure/cancellation rolls back both blob deletes +and leaves cursors unchanged; already committed job expiry stays committed. +A halted cluster permits the original expiry statement but stops protected blob +GC at the writable fence. + +The settlement lock serializes save/repair with blob GC; the order lock +serializes leased-candidate enqueue with blob GC. No path may reverse this order +or acquire advisory locks while retaining expiry's row locks. If repair goes +first, blob GC observes its references; if GC goes first, repair atomically +restores the original blobs/prepared/child. The lock-free expiry statement's +post-wait recheck protects renewals independently. A modifying CTE with an outer +`NOT EXISTS` sees the wrong statement snapshot, so it cannot replace the +separate delete statements. Claim expiry/retry must not remove a nonterminal +candidate's protection. No share-ledger pruning is added. + +Blob GC uses A265's landed typed outbox digest column and index; it checks all +surviving references instead of inferring references from bundled payloads or +filtering on claim state. Per landed `406b273`, +A's enqueue re-inserts the digest-checked as-issued balances under `ORDER_LOCK` +before committing its reference, including when GC went first. A probes the +share prefix but still enqueues with an alert if it is missing. The exact Rust +input and immutable-conflict implementation remain A-owned. B cannot promise +preservation based on an unlocked earlier read or invent a different outcome. + +## Resume admission, capacity, and cancellation + +The existing Stratum `initial_job_timeout_seconds` wrapper remains the one +deadline, normally 30 seconds. It includes job/dependency reads, admission, +build/read permits, singleflight waiting, decode, reconstruction, and +per-miner bootstrap. Do not add an inner 25-second timeout or reset a deadline +at a phase boundary. + +Rebuild concurrency uses `build_slots`; window reads additionally use +`clamp(database_max_connections - 2, 1, build_workers)` capacity shared with +A's reads. The blocking closure owns its build permit until actual exit, even +if the async waiter is cancelled. Avoid nested acquisition by calling the +borrowed builders directly under the held permit. + +A bounded in-flight map keyed by immutable prepared identity shares only the +snapshot/body reconstruction. Each waiter retains its worker, target, +difficulty, extranonce, version mask, absolute expiry, and authority checks. +Empty windows coalesce only the empty snapshot/balances; each miner builds +its own synthetic bootstrap share with stored policy inputs. No completed +entry becomes an unbounded window cache. Define cancellation and error +cleanup so abandoned entries cannot retain windows, waiters, or capacity. + +Recheck authority after every admission-relevant async wait. Eligible original +published R0 work can survive the bounded R1 replacement lease only while +the current and issued balance digests match; arbitrary older same-parent work +cannot inherit it. Broader protection across changed balances remains #289. +Before returning, verify original issued +expiry again. Missing/expired/incompatible work is `Ok(None)`; corruption, +database errors, reconstruction mismatches, and deadline exhaustion retain +the existing truthful backend-error path. + +## Qualification matrix + +| Area | Required evidence | +| --- | --- | +| Window shape | Empty/range round trips; native and legacy digest distinction; uppercase, malformed, oversized sequence, bad-count, and partial-column rejection; payload/column equality. | +| Reader | Exact ascending range with excluded/rejected/after-anchor rows; incomplete endpoints/interior; digest mismatch; corrupt/missing as-issued balance bytes; database/decode errors; stable RR snapshot; empty range reads no ledger rows. | +| Economics | Signed frozen fixture and A-to-B rebuild equality for normal, bootstrap, CTV, prior-only recipient, and changed-current-balances cases; preserve original R0 audit hashes and candidate fences. | +| Admission | Same-digest revision swap preserves the published lease; changed digest rejects immediately. Coherent revision/digest under concurrent balance changes; authority/revision/parent/lease and absolute expiry move at each permit, shared entry, DB read, blocking build, bootstrap, and final return; arbitrary R0 cannot borrow a lease; failed reads stay errors. | +| Capacity | One build worker cannot deadlock; cancelled blocking builder retains permit until closure exit; shared reader pool leaves two connections; leader cancellation/failure and follower cancellation clean up; waiters cannot exchange identity/economics/expiry. | +| Durability | Both GC/repair orders, both GC/enqueue orders under settlement then order; no dangling children; immutable conflicts; original absolute expiry; expired-job batch bound; pending leased candidate retains balances through claim/retry expiry, then a later zero-job sweep removes its orphan after terminal completion; unrelated live jobs/candidates remain protected. | +| Migration | Fresh and pre-008 rows under real runner; exact three-state CHECK; legacy inline prepared miss; new malformed payload error; 008/009 coexistence and restart; missing numbered migration must not be skipped by a higher version. | +| Regression | Existing readiness, Stratum, PR313 issued-dependency and coordinator miner suites, unchanged assertions; workspace all-target compilation on combined base. | +| Scale | Real PostgreSQL 400k refresh then A-to-B resume with exact signed equality; 500k headroom; large valid template; all refresh JSONB values below 1 MB; row-read counts, RSS including retained local generations, wall time, measured WAL, affected-work count and prompt replacement delivery after balance changes. | + +Use the landed `tests/support/window_fixture.rs` production-shaped fixture, +which accepts both 400k and 500k. Add a B-specific refresh/resume test so +unrelated candidate/import JSONB limitations do not masquerade as B results. +Do not weaken the existing all-pipeline ratchet to make B green. + +For WAL, load the fixture before measurement, ensure no concurrent writers on +the disposable primary, record `pg_current_wal_insert_lsn()` immediately before +and after the awaited refresh, and calculate `pg_wal_lsn_diff(after,before)`. +Record PostgreSQL version, fsync/full-page-write and WAL-compression settings, +checkpoint placement, first-seen versus reused template/balance keys, and +whether asynchronous standby replay is included (it must not be called +commit latency). Run the decided one-dedicated-asynchronous-standby shape +separately for replication qualification. The target is under 5 MB per refresh; +a first-seen large valid template can itself challenge that budget and must +be reported, never excluded without explanation. + +Count actual share rows returned by the reader, separately from metadata, +balance/template reads, endpoint probes, and page query count. Report the +400k and 500k cold read separately from shared followers. Measure process RSS +with an OS-supported sampler; the landed harness's Linux `/proc` RSS path is +not evidence on macOS. Report wall time per refresh and per A-to-B resume, +retained-memory behavior after the final waiter, and timeout outcomes. + +## Upgrade and review boundaries + +This development line uses a stop-all migration: drain the pre-007 outbox, +stop every old frontend, apply the coordinated ordered migrations, then start +one new frontend before the rest. Legacy inline prepared rows are cache misses +and expire normally. This is not a rolling-upgrade compatibility promise. + +This is high-stakes work because it changes payout reconstruction and stored +schema. The completed foundation may be published as a draft before full +deep-review, with incomplete runtime and qualification items explicit. The +eventual integrated change still requires the full review battery and accurate +reporting of unavailable lanes. Draft visibility is not approval to mark it +ready or merge; no VERSION or CHANGELOG bump is part of this work. + +## Initial execution evidence (historical) + +The following results predate `d7526fc` and the user's contract signoff. Current +implementation and tests are in the foundation report; full B273 refresh/resume +acceptance and A integration remain incomplete. + +- Default `cargo check --workspace --all-targets --locked`: stopped before + compilation; Cargo 1.84 cannot parse dependency `base64ct 1.8.3` requiring + edition 2024. No dependency or lockfile change was made. +- `cargo +1.89.0 check --workspace --all-targets --locked`: passed on the + unchanged PR313 base. +- PostgreSQL discovered through `pg_config --bindir`: + `/opt/homebrew/Cellar/postgresql@16/16.14/bin` (PostgreSQL 16.14). +- Pinned qbitd available at `~/bin/qbit-v1.0.0/bin/qbitd`; presence does not + establish that live-node tests ran. +- Baseline tests on unchanged PR313 code, with Rust 1.89.0: + +| Command | Result | +| --- | --- | +| `RUSTUP_TOOLCHAIN=1.89.0 PRISM_TEST_PG_BIN_DIR=/opt/homebrew/Cellar/postgresql@16/16.14/bin bash test/prism-native-tests.sh cargo-args --locked -p qbit-prism-server --test readiness_rpc --test stratum_protocol` | 5 readiness tests and 22 Stratum protocol tests passed; none ignored. The runner created and removed its own disposable PostgreSQL cluster. | +| `RUSTUP_TOOLCHAIN=1.89.0 PRISM_TEST_PG_BIN_DIR=/opt/homebrew/Cellar/postgresql@16/16.14/bin bash test/prism-native-tests.sh cargo-args --locked -p qbit-prism-server --test issued_job_dependency -- --ignored --test-threads=2` | All 7 explicitly gated PostgreSQL dependency tests ran and passed; none skipped. | +| `cargo +1.89.0 test --locked -p qbit-prism-server --lib coordinator::miner_tests` | All 52 selected tests passed; 77 unrelated library tests filtered out. | +| `cargo +1.89.0 test --locked -p qbit-prism --test audit_parts` | All 7 tests passed, including byte-for-byte owning/borrowed equivalence and borrowed-window verification. | + +These are prerequisite/baseline results, not B273 implementation acceptance. +The later foundation implements 008 and the reader, including reader-only +400k/500k measurements and actual 008/009 coexistence checks. Compact runtime +prepared work, full refresh/resume, large-template/WAL/standby qualification, +combined A/B/C integration, and deep-review remain pending; no remote review or +bot result is implied by these baseline tests. diff --git a/docs/prism-public-operator-endpoint-split.md b/docs/prism-public-operator-endpoint-split.md index 1d83a623a..5ab695af4 100644 --- a/docs/prism-public-operator-endpoint-split.md +++ b/docs/prism-public-operator-endpoint-split.md @@ -1,310 +1,130 @@ -# PRISM Public / Operator Endpoint Split +# Prism public and operator endpoints -This document maps every endpoint the PRISM HTTP surface serves and classifies -each one as **public read surface**, **operator surface**, or -**payout-affecting**. It is the prerequisite for issue #145, which moves the -public read tier out of the coordinator process. +The Rust runtime keeps the `2.x.x` public read tier as a separate process: -It exists so a reviewer can disagree with one classification without re-reading -the HTTP layer. Each row carries the evidence its classification rests on, and -the classification itself is declared in -[`lab/prism/endpoint_registry.py`](../lab/prism/endpoint_registry.py) and pinned -against real dispatch by `tests/test_prism_endpoint_registry.py`. Nothing here -changes request handling; this is the boundary written down. - -## Result - -**Exactly the 13 `/public/v1/*` routes are extracted. Everything else stays.** - -That is a narrower answer than "move the read traffic", and the narrowing is the -substance of this document. Three findings drove it. - -### 1. The boundary is already written down, and it excludes `/audit/*` - -`PRISM.md` states that the coordinator "exposes a private audit/ops listener and -a dashboard-safe public API from the same process", lists `/healthz`, -`/metrics`, `/owed-balances` and every `/audit/*` route as private/internal, -instructs operators not to expose `/audit/*` to the internet, and concludes: -"operators can expose only `/public/v1` through a reverse proxy or dashboard -frontend." - -`docs/public-dashboard-api/README.md` repeats it as a contract — "the public API -must not expose `/audit/*`, `/metrics`, `/healthz`, operator controls, raw -private sockets, credentials, or unrestricted internal manifests" — and -`tests/test_public_dashboard_api_contract.py` enforces it by asserting the -OpenAPI document contains no `/audit/` path. - -The issue asks that published audit evidence be served from the artifact files -rather than through the coordinator. It already is, and through a public route: -`/public/v1/artifacts/{sha256}` resolves bodies from `body_uri` on disk when the -inline column is NULL, verified against the requested sha256, and -`/public/v1/blocks/{block_hash}/settlement-artifacts` links to it. **The -evidence content is public; the `/audit/*` routes are not.** They are internal -tooling over the same data: raw `sats` units, no pagination, full -broadcast-attempt histories, `parent_coinbase_tx_hex` and `manifest_set_json` -broadcast material, and in one case a caller-controlled unbounded window. - -So the `PRISM_PUBLIC_ARTIFACT_CACHE_*` knobs move with the extracted service -because `/public/v1/artifacts/{sha256}` moves — which is exactly what the issue -predicted, and it lands without widening the public surface. - -### 2. Two nominally public routes take the writer lock today - -This is the finding that most changes the shape of the work. - -`/public/v1/miners/{recipient_id}` — the most-polled route on the surface — -resolves an owed balance by calling `current_owed_balances()`, which returns -*every* recipient's balance under the writer lock and then filters in Python. -`/public/v1/blocks/{block_hash}/settlement-artifacts` falls through to -`audit_bundle()` under the writer lock for direct-coinbase blocks. - -Both must be severed before the extraction is worth anything. Moving the process -without fixing these would move the GIL contention but keep the lock contention. - -### 3. Some routes that look public are operator surfaces - -The issue predicted the drift count and the health probes. The sweep found more: -`/owed` and `/owed-balances`, the `/payouts/{recipient_id}/status` alias, and -the entire `/audit/*` family. `/miners/{recipient_id}/status` is not even listed -in `PRISM.md`'s endpoint inventory and has no consumer outside tests — it is -legacy, superseded by `/public/v1/miners/{recipient_id}`. - -## The three axes - -### Audience - -- **Public read** — the documented, versioned dashboard contract under - `/public/v1`. -- **Operator** — process state, or the private audit/ops tooling. -- **Payout-affecting** — live payout obligations: the state the settlement - machinery itself acts on. Out of scope for extraction by the terms of the - issue. - -### Ledger access - -How the response reaches durable state. This decides what may move to a replica, -and it is the axis that matters for the money path. - -| Tier | Mechanism | Consequence | -| --- | --- | --- | -| `WRITER_LOCK` | `_operation_gate(self._lock, "writer lock")` in `share_ledger.py` | Serializes against the lease-holding writer. Public volume here is share-ingest backpressure. | -| `READ_SLOT` | `_run_read_json` → `_operation_gate(self._read_semaphore, "read slot")`, bounded by `PRISM_POSTGRES_READ_CONCURRENCY` (default 4) | Primary Postgres, but not the writer lock. Replica-eligible as written. | -| `ARTIFACT_FILE` | The audit artifact directory (`PRISM_AUDIT_DIR`) | Already the durable public record. No database involved. | -| `PROCESS_STATE` | In-memory coordinator counters, gauges, snapshots | Cannot leave the process at all. | -| `QBIT_RPC` | `coordinator.rpc.call(...)` | A node read, not coordinator state. The extracted service gets its own client. | - -The writer-lock reads reached from HTTP, verified against the `PsqlShareLedger` -in `lab/prism/share_ledger.py` (`def` line, then the line holding the gate): - -| Ledger method | Definition | Gate | Reached by | -| --- | --- | --- | --- | -| `current_owed_balances` | `:4161` | `:4172` | `/owed`, `/owed-balances`, `/miners/{id}/status`, **`/public/v1/miners/{id}`** | -| `carry_forward_integrity_report` | `:4259` | `:4261` | `/audit/carry-forward-integrity` | -| `audit_share_window` | `:4327` | `:4353` | `/audit/share-window` | -| `audit_block_payouts` | `:4365` | `:4383` | `/audit/blocks/{h}/payouts`, `/audit/block/{h}` | -| `recipient_payout_history` | `:4389` | `:4418` | `/miners/{id}/status` | -| `audit_bundle` | `:4906` | `:4927` | `/audit/blocks/{h}/bundle`, **`/public/v1/blocks/{h}/settlement-artifacts`** | -| `audit_bundle_by_commitment` | `:4931` | `:4958` | `/audit/commitments/{leaf}/bundle` | - -The SQL these run is read-only — the stored functions they call -(`qbit_current_owed_balances`, `qbit_prism_window`, `qbit_audit_block_fanouts`, -`qbit_fanout_status`, `qbit_audit_share_window`) are declared `STABLE` in -`crates/qbit-prism/sql/001_share_ledger.sql`. The cost is the in-process gate, -not a database lock, which is what makes these replica-eligible once the read is -routed off the writer. No HTTP route on any path performs an `INSERT`, `UPDATE`, -`DELETE` or advisory lock. - -`public_api.py` also carries pre-read-model fallbacks that reach -`recipient_payout_history` and `ledger.all_shares()` when a `dashboard_*` method -is absent. On the production `PsqlShareLedger` every read model exists, so those -branches are unreachable; they matter only to the in-memory ledger used when -`PRISM_ALLOW_MEMORY_LEDGER=1`. The extracted service must keep that true, or the -fallbacks would reintroduce writer-lock reads on public routes. - -## Operator surface — retained - -| Path(s) | Ledger access | Why operator | -| --- | --- | --- | -| `/healthz` | `PROCESS_STATE` | Liveness of this instance: job-build progress, tip refresh, writer-lease state. Answered from a background-refreshed in-memory snapshot (`PRISM_HEALTH_REFRESH_SECONDS`, default 5s), overlaid with live progress on every request so a cached `ok=true` cannot mask a failed refresh. Wired to the compose healthcheck (`compose.yaml`, pinned by `tests/test_prism_compose_profile.py`) and to `make prism-self-check`. | -| `/metrics` | `PROCESS_STATE` | Prometheus exposition of this process's counters. Serves only the complete cached snapshot (`PRISM_METRICS_REFRESH_SECONDS`, default 5s). | -| `/audit/latest` | `ARTIFACT_FILE` | The operator's view of the live evidence envelope. Public consumers reach the same evidence content-addressed via `/public/v1/artifacts/{sha256}`. | -| `/audit/share-window` | `WRITER_LOCK` | Raw per-share rows for a caller-supplied anchor. The window is caller-controlled and unbounded. The public equivalent is the aggregated reward leaderboard. | -| `/audit/blocks/{block_hash}/payouts`, `/audit/block/{block_hash}` | `WRITER_LOCK` | Per-recipient payout rows in raw sats with `maturity_state`/`chain_state`. The short form is a legacy alias for the same handler. Public per-miner slice: `/public/v1/miners/{id}/payouts`. | -| `/audit/blocks/{block_hash}/bundle` | `WRITER_LOCK` + `ARTIFACT_FILE` | Full inline bundle plus `coinbase_tx_hex`. The bundle body itself is public, content-addressed. | -| `/audit/commitments/{commitment_leaf_hex}/bundle` | `WRITER_LOCK` + `ARTIFACT_FILE` | The same bundle by commitment leaf. No public commitment-leaf lookup exists. | -| `/audit/blocks/{block_hash}/ctv-fanout-manifest-set` | `READ_SLOT` | Full CTV recovery payload including `parent_coinbase_tx_hex` and `manifest_set_json` — broadcast material. | -| `/audit/blocks/{block_hash}/ctv-fanouts` | `READ_SLOT` | Raw artifact dicts with `fanout_tx_hex` and sats fields. | -| `/audit/fanouts/pending` | `READ_SLOT` | The broadcaster's work queue with full attempt histories. | -| `/audit/fanouts/{fanout_txid}/status` | `READ_SLOT` | Raw status including `submit_result` and attempt detail. | - -## Payout-affecting — retained, out of scope - -| Path(s) | Ledger access | Why | -| --- | --- | --- | -| `/audit/carry-forward-integrity`, `/audit/ledger-integrity` | `WRITER_LOCK` | The drift count (`current_drift_count`), mismatch list, and the `audit_head_sha256` operators are told to mirror after payout-affecting blocks. Reports whether the payout ledger disagrees with itself. Runs two statements under one held writer gate — the worst route on the surface to expose to uncontrolled volume. | -| `/owed`, `/owed-balances` | `WRITER_LOCK` | Every recipient's outstanding balance: the obligations the next settlement will pay. O(recipients) under the writer lock. There is no public all-recipients dump. | -| `/miners/{recipient_id}/status`, `/payouts/{recipient_id}/status` | `WRITER_LOCK` | The pre-dashboard payout-operations view in raw sats. Both aliases reach one handler. Superseded for public use by `/public/v1/miners/{recipient_id}`, which reports bits. | - -## Public read surface — extracted - -| Path | Ledger access | Extraction blocker | -| --- | --- | --- | -| `/public/v1/pool-summary` | `READ_SLOT` + `QBIT_RPC` | None. Needs its own qbit RPC client. | -| `/public/v1/blocks` | `READ_SLOT` | None. | -| `/public/v1/leaderboard` | `READ_SLOT` + `QBIT_RPC` | None. `window=reward` needs network difficulty from RPC. | -| `/public/v1/hashrate-series` | `READ_SLOT` | None. The widest ledger scan on the surface, and the strongest single argument for moving this traffic off the primary. | -| `/public/v1/mining-configuration` | none | Falls back to `getattr(coordinator, "port")` / `getattr(coordinator, "bind")` when the public URL env vars are unset. `PRISM_PUBLIC_STRATUM_URL` / `PRISM_STRATUM_PORT` must be supplied explicitly. | -| `/public/v1/miners/{recipient_id}` | `WRITER_LOCK` + `READ_SLOT` + `QBIT_RPC` | `owed_balance_for_recipient()` calls `current_owed_balances()`, a writer-lock read returning every recipient's balance, filtered in Python. Needs a per-recipient owed-balance read that does not fence. | -| `/public/v1/miners/{recipient_id}/earnings` | `READ_SLOT` | None. | -| `/public/v1/miners/{recipient_id}/payouts` | `READ_SLOT` | None. | -| `/public/v1/miners/{recipient_id}/workers` | `READ_SLOT` | None. | -| `/public/v1/blocks/{block_hash}/settlement-artifacts` | `WRITER_LOCK` + `READ_SLOT` + `ARTIFACT_FILE` | The CTV path is read-slot, but a direct-coinbase block falls through to `direct_coinbase_settlement_payload()` → `audit_bundle()` under the writer lock. Take that branch from the artifact file. | -| `/public/v1/fanouts/pending` | `READ_SLOT` | None. | -| `/public/v1/fanouts/{fanout_txid}` | `READ_SLOT` | None. | -| `/public/v1/artifacts/{sha256}` | `READ_SLOT` + `ARTIFACT_FILE` | None. Already resolves the body from `body_uri` on disk when the inline column is NULL. Carries the `PRISM_PUBLIC_ARTIFACT_CACHE_*` knobs. | - -## What makes this an extraction rather than a rewrite - -`public_api.py` reaches the coordinator through a small, complete surface: - -- `coordinator.ledger` — 24 references, 10 direct methods plus 7 duck-typed - `getattr` read models -- `coordinator.rpc` — 3 references: `getblockchaininfo`, `getblocktemplate`, - `getnetworkinfo` -- `getattr(coordinator, "port")` and `getattr(coordinator, "bind")`, both with - environment fallbacks already in place - -There is no other coordinator state on the public path. The extracted service -supplies its own object with `.ledger` and `.rpc`, and the dashboard code is -unchanged. - -## Completeness - -The repository contains exactly one HTTP-serving surface outside tests: the -`AuditHttpFacade` listener in `lab/prism/audit_http.py`, fed by -`_CoordinatorAuditHttp` and `public_api.dispatch`. `make_audit_handler` in -`prism_coordinator.py` is a compatibility factory over the same facade, not a -second surface. The coordinator binds only the Stratum listener(s) and the audit -HTTP port. No Rust crate serves HTTP. Every route string in `tests/`, `test/`, -`docs/` and `doc/` falls inside the union of the routes classified here; the -only exception is a test-only `/control` route in the fake qbit RPC server. - -## Staleness is a contract - -The precedent is in `lab/prism/observability.py`: the cached `/metrics` endpoint -returns 503 rather than serving an unbounded-stale snapshot. - -- `MINIMUM_HEALTH_STALE_SECONDS = 15.0` -- `stale_after = max(3 * refresh_seconds, MINIMUM_HEALTH_STALE_SECONDS)` -- `/healthz` returns 503 with `snapshot_age_seconds` once over budget -- `/metrics` returns `503 if stale else 200` and always emits - `qbit_prism_metrics_snapshot_age_seconds`, `qbit_prism_metrics_snapshot_stale` - and `qbit_prism_metrics_snapshot_available` - -**Landed.** Each extracted endpoint states the staleness it tolerates, surfaces -the observed age to callers, and refuses rather than serving beyond its budget. -The budgets live in the `max_staleness_seconds` field of `Endpoint` in -`lab/prism/endpoint_registry.py`, derived rather than hand-picked: - -``` -budget = max(3 * (cache_ttl_seconds + underlying_cache_seconds), - MINIMUM_PUBLIC_STALE_SECONDS) +```sh +qbit-prism-server public-api ``` -with `MINIMUM_PUBLIC_STALE_SECONDS = 15`, matching the precedent's floor. The -derivation is documented in that module's docstring and pinned by -`tests/test_prism_endpoint_registry.py`, which asserts the budgets against the -formula and against `public_cache_policy` itself, so a changed cache default -cannot silently leave a budget behind. - -Three sources of staleness compound and must all be counted: - -1. The shared in-process response cache (`PublicResponseCache`), whose TTL is - already per-route via `public_cache_policy`. Counted. -2. The pool reward-window aggregate cache in the ledger - (`_pool_reward_window_aggregate`, `PRISM_PUBLIC_REWARD_WINDOW_CACHE_SECONDS`, - default 30s), which sits underneath every miner page. Counted, and it is the - only route where two caches sit in series: `/public/v1/miners/{recipient_id}` - at 105s versus 15s for the other plain row reads. -3. Replica lag, once a replica exists. Contributes 0 today because no replica - exists; when one lands it becomes another `underlying_cache_seconds` term - rather than a new mechanism. - -On the wire, every extracted response carries -`X-Prism-Staleness-Budget-Seconds` beside the existing `Age` header. Past the -budget the service returns 503 with `Cache-Control: no-store` and an ordinary -`prism.dashboard.error.v1` body naming both the budget and the observed age — -error code `upstream_unavailable`, already in the documented enum, rather than a -new error schema. - -At the documented cache defaults the budgets are 15s for the plain row reads -(`/blocks`, `/leaderboard`, `/miners/{id}/earnings`, `/miners/{id}/payouts`, -`/blocks/{h}/settlement-artifacts`, `/fanouts/pending`, `/fanouts/{txid}`), 90s -for the pool-wide aggregates (`/pool-summary`, `/hashrate-series`, -`/miners/{id}/workers`), 105s for `/miners/{recipient_id}`, and 900s for -`/mining-configuration`. - -`/public/v1/artifacts/{sha256}` is the exception, and says so explicitly through -the registry's `immutable_content` flag rather than by omitting a budget: content -is immutable and content-addressed, so a body that hashes to the requested -sha256 is correct at any age. It reports `X-Prism-Staleness-Budget-Seconds: -unbounded` and never refuses for staleness. The only observable staleness is the -404→200 transition for an artifact published moments ago, which no budget would -fix. +Its default port is 3342. Configure `PRISM_DATABASE_URL` for that process's +read pool and `PRISM_PUBLIC_STRATUM_URL` for the advertised mining endpoint. +Compose supplies the read DSN through `PRISM_PUBLIC_DATABASE_URL`, independently +of the mining writer DSN. This process requires no signing keys and does not +migrate the schema, allocate mining sessions, or claim accounting work. -## Health is a contract +The public role serves `/public/v1/*`, `/healthz`, and `/metrics`. Operator +aliases under `/audit`, legacy `/miners` and `/payouts`, and private status +routes remain on the combined coordinator listener (default 3341). Keep that +listener bound to a trusted interface. -The coordinator's `/healthz` answers one question: is this instance failing to -deliver work to the miners it has admitted? It fails closed (HTTP 503, with -`initial-delivery-stalled` in `unhealthy_reasons`) for two conditions, both -measured against `PRISM_STRATUM_INITIAL_JOB_TIMEOUT_SECONDS`: +The independent public listener's unauthenticated `/healthz` reports readiness, +not database driver diagnostics. Database failures use these fixed `error` +strings; they never include raw SQLx messages, DSNs, credentials, role names, +database names, hostnames, addresses, or server-provided SQL identifiers: -- **Genuine first-job starvation.** An authorized miner has waited the full - deadline without any delivered work - (`oldest_genuinely_pending_initial_job_age_seconds`). -- **Sustained semantic current-work loss.** Fewer than 95% of authorized miners - hold work matching the current template fingerprint, payout generation, and - their current connection, authorization, and difficulty generations - (`semantic_current_work_ratio`) for longer than the deadline - (`semantic_current_work_gap_age_seconds`), with no exact-tip work covering - the gap either. - -Exact-tip churn by itself stays diagnostic. When the observed tip advances -faster than fanout, `current_tip_job_coverage` drops and -`current_tip_coverage_gap_age_seconds` grows while miners still hold the latest -published work; the 2026-08-31 incident read 4 of 33 miners on the exact tip -with all 33 semantically current. Those fields keep reporting the churn, and -exact coverage still corroborates `overload` and the -`connection-capacity-saturated` and `stale-unknown-rejection-storm` reasons, -each of which needs an independent pressure signal. It no longer fails health -on its own (issue #216). - -Router-facing mining readiness, whether a load balancer should send new miners -to this instance, is a distinct signal to be added by issue #186. It is not -derivable from `/healthz`, and this document does not classify it. - -## Out of scope - -Anything payout-affecting stays on the coordinator: the writer lease, share -acknowledgement, block landing, settlement, and the surfaces listed above. The -extracted service is read-only — it opens no write path to the ledger and holds -no lease. That is now structural rather than aspirational: it is constructed -without any `PRISM_LEDGER_WRITER_*` input, receives none of the signing or -block-submission configuration in compose, mounts the audit artifact volume -`:ro`, and refuses to start on the in-memory ledger (whose missing `dashboard_*` -read models would send `public_api`'s fallbacks back onto writer-lock reads). -`tests/test_prism_public_read_service.py` drives every extracted route against a -ledger that raises if any writer-lock read is reached. - -The two blockers named above are removed. `dashboard_miner_owed_balance_bits` -and `dashboard_direct_coinbase_settlement` on `PsqlShareLedger` answer both -formerly-fenced public reads through `_run_read_json`, and `public_api` prefers -them through the same duck-typed pattern used by every other read model, keeping -the old paths as in-memory-ledger fallbacks. - -Replica wiring is tracked separately. It needs `wal_level`, a replication slot, -and a standby service in the Postgres setup; today `prism-postgres` has a -dedicated WAL volume (`POSTGRES_INITDB_WALDIR`) but no replication -configuration. Until that lands, the extracted service reads the primary — a -separate process with its own connection pool, which already removes the GIL -contention, but not yet a separate database. +| Failure category | Public `error` | Operator checks | +| --- | --- | --- | +| Authentication | `database authentication failed` | Public-reader credentials and database authentication rules | +| Access | `database access denied` | Public-reader schema, table, and monitoring grants | +| Connection | `database connection failed` | Database availability, network, TLS, database name, connection limits | +| Configuration | `database connection configuration is invalid` | Public-reader connection configuration | +| Schema | `native public read schema is incomplete` | Schema migrations and public-reader `search_path` | +| Timeout | `database probe timed out` | Database availability, network, connection limits, pool availability, load, query latency | +| Cancellation | `database readiness query was canceled` | Database statement deadlines and operator query cancellations | +| Other query/driver failure | `database readiness query failed` | Database service logs and readiness query compatibility | + +The public process emits bounded `public readiness probe failed` warnings for +failed database probes, with fixed `category`, `phase` (`schema`, `replica`, or +`probe` for the whole-probe timeout), and `action` fields. The event policy is: + +- The first failed probe emits one warning immediately. +- A change of failure category emits one warning immediately, even within the + reminder interval. A phase change alone does not count as a category change. +- An unchanged failure emits at most one reminder per 60 seconds, measured with + monotonic time from the last warning decision. The first completed failed + probe at or after that boundary emits the reminder; missed intervals do not + produce a burst of catch-up events. +- After an unhealthy episode, the first healthy probe emits one + `public readiness probe recovered` event at INFO. Further healthy probes and + a healthy startup are silent. A new failure after recovery warns immediately; + flapping therefore preserves every observed failure/recovery transition. + +An episode begins with a database probe failure from the categories above. +Recovery requires a fresh successful probe that also passes the existing +replica readiness policy. A successful query while replica policy still refuses +health does not announce recovery or reset the warning budget. Replica-only +refusals without a preceding database probe failure remain visible through the +existing health and metrics surfaces and do not start a diagnostic episode. +These events describe completed probes; they do not create a second observer +for startup or snapshot staleness between probes. + +The readiness snapshot and diagnostic state are published together and unlocked +before calling the synchronous logging sink. A blocked sink can delay the probe +task, but health readers can still see its freshly published result; the existing +staleness policy continues to apply if subsequent probes cannot complete. There +is no additional background task, event queue, or configuration setting. + +Set `RUST_LOG=qbit_prism_server::api::public_service=info` to collect warnings +and recovery events, or use `=warn` to collect only warnings. Filters that exclude +these levels hide the events; filtered events still advance the policy state, +and enabling a filter does not replay them. The process continues to use its +existing `RUST_LOG` reader and default filter. + +Classification uses SQLx variants and recognized SQLSTATE values; +unknown errors remain failures without guessing their cause. PostgreSQL reports +both an expired `statement_timeout` and an operator cancellation as SQLSTATE +`57014`; both use the cancellation category with deadline/cancellation guidance. +If the whole-probe timer wins instead, the category is timeout. These labels +describe the observed failure, not a guess based on localized driver text. +The warning does not format the raw error, +its source chain, or even an unrecognized SQLSTATE. Consult access-controlled +database logs and deployment configuration for detailed investigation; do not +publish those sources or enable verbose driver logging on a public diagnostics +surface. This boundary applies to readiness diagnostics, not every runtime log. + +Categories describe the error that reaches the probe, not necessarily the first +underlying cause. SQLx retries refused connections and PostgreSQL connection +errors `53300` (connection limit) and `57P03` (cannot connect now). The existing +five-second probe deadline expires before the pool's acquisition deadline, so +these outages report timeout, as does an exhausted pool. Check reachability and +connection limits as well as query latency for timeout warnings. Non-retried +connection errors, such as a missing database, can report connection directly. +No retry or deadline policy is changed by this diagnostic categorization. + +Compatibility: this intentionally reduces database error detail in the existing +string-valued `error` field. The `qbit.prism.public-read-health.v1` schema, all +response fields/types, and successful response shape are unchanged. Failures +still return HTTP 503 with `ok=false`; database probe failures also retain +`database_ready=false`. Starting, stale, replica refusal, GET/HEAD, and HTTP +healthcheck behavior are unchanged, as is the five-second probe deadline. +Existing replica refusal messages still take precedence in the public response; +the warning records any underlying database probe failure. No client should +depend on the former raw driver text. The combined coordinator listener and +its authentication policy are outside this diagnostic change. + +This page classifies HTTP endpoints. Operator commands are not endpoints and +are not listed here: `migrate`, `import-audits`, `backfill-ctv`, +`policy-transition`, `share-archive` and the rest run from a shell against the +primary and reach neither listener. `share-archive`, the share ledger retention +path, is documented in +[ledger operations](prism-ledger-ops.md#share-ledger-partitions-and-retention). + +One operator route's answer depends on retention. `/audit/share-window?anchor=` +reads the online share ledger, so an anchor inside a range whose partition has +been archived returns no rows rather than an error. Those shares are in the +archive and in every sealed artifact whose window covers them; an +anchor-to-archive index is deferred. Public artifact and audit bundle routes +are unaffected, because a block whose shares have been archived is served from +its stored canonical bytes under the same digest. + +Read concurrency, SQL statement deadlines, cache capacity, and stale response +budgets are bounded independently of mining. Replica mode additionally checks +recovery state and WAL receiver heartbeat age; promotion or a stale stream +closes replica-dependent routes. Content-addressed canonical bytes remain +verified against their requested digest. All shared evidence is in PostgreSQL; +the public role needs no audit filesystem mount. + +See the [public API contract](public-dashboard-api/README.md), +[replica operations](prism-postgres-replica.md), and +[coordinated Rust migration](prism-rust-migration.md) for parameters, readiness +headers, database privileges, and deployment examples. diff --git a/docs/prism-refresh-probe.md b/docs/prism-refresh-probe.md new file mode 100644 index 000000000..1a9ca8023 --- /dev/null +++ b/docs/prism-refresh-probe.md @@ -0,0 +1,166 @@ +# Coherent refresh economics probes + +Issue [#434](https://github.com/Qbit-Org/qbit-mining-bootstrap/issues/434) +combines the refresh loop's paired payout-state and accepted-cutoff reads. +It saves one SQL execution and one checkout per pair. It does **not** reduce +the number of full balance reads or semantic balance hashes. + +## Read and publication contract + +The private `RefreshProbe` contains a `PayoutState` and an accepted share +sequence. A successful observation uses one connection and one repeatable-read +transaction: + +1. Begin and set repeatable-read isolation without overriding read-only settings. +2. Read revision in the original standalone SELECT, retaining the existing + primary, writable-session and fatal-state guards before accessing shares. +3. Read accepted cutoff and current balances in one SELECT. The cutoff uses + the same `ACCEPTED_CUTOFF_SQL` constant as anchored snapshot selection; the + balance projection shares `PRIOR_BALANCE_SQL` with public payout reads. + A left join returns one explicitly marked placeholder for empty balances. + Remove that placeholder, decode and compute the existing semantic digest + on a blocking thread. All row vectors are also destroyed there. +4. Commit before returning the observation. + +The first data SELECT establishes the snapshot. A concurrent atomic update of +revision, cutoff and balances therefore produces a complete old or new triple. +The previous independent cutoff read could observe shares committed after the +payout-state snapshot; coalescing deliberately makes that selection coherent. +Shares committed after selection are observed on a later refresh. + +Public `Ledger::payout_state()` remains share-free. It does not read accepted +metadata, share payloads or stored balance blobs. Both methods use the existing +current-balance SQL and the same digest decoder. No schema, digest recipe, +stored balance order or as-issued identity changes are required. + +Refresh takes another probe after waiting for build admission. The admission +owner accompanies blocking decode/hash work, so cancelling the async caller +cannot release that capacity before the actual blocking cleanup finishes. +Database, decode, task and commit-response errors remain errors, including a +lost response to a commit that the server executed. No error becomes valid +zero state or a successful mismatch. + +The final idle payout-state equality check remains in place, as do all compact +reservation/publication proofs. The probe is selection metadata, not an +anchored window or publication authority. New shares after window capture do +not veto publication of that immutable selection. Original readiness proofs, +anchors, expiry and deadlines retain their existing boundaries. + +## Counted work + +Counts cover payout probes and separate accepted-cutoff probes only. They +exclude chain observation/reconciliation, fee RPC, database clocks, anchored +snapshots/share pages, persistence, connection validation and other hashes. +One successful payout or combined probe executes five statements: BEGIN, SET, +revision SELECT, balance/cutoff SELECT and COMMIT. These are SQL executions, not a +claim about network packet counts. + +| Path without publication-lock retry | Prior SQL / checkouts / balance hashes | Combined probe SQL / checkouts / balance hashes | +| --- | --- | --- | +| Unchanged empty or nonempty work | 11 / 3 / 2 | 10 / 2 / 2 | +| First build, no cached window | 21 / 5 / 4 | 20 / 4 / 4 | +| Build with a cached window | 27 / 7 / 5 | 25 / 5 / 5 | +| Each extra publication-lock proof | +5 / +1 / +1 | +5 / +1 / +1 | + +The public refresh regression observes actual PostgreSQL Execute/Query and +DataRow frames. For both unchanged empty and nonempty work with one balance +recipient it measured 11 component executions and two returned balance rows +before integration, and 10 executions and two returned balance rows after it. +It also asserts that the same prepared object remains published and no share +payload is returned. Parse frames do not count as executions. + +The private probe regression independently measures one checkout through the +existing acquisition metric and one actual digest callback per invocation, +with five executions and max(N,1)+1 DataRows for N=0, N=1 and N=2. +The empty case includes one outer-join placeholder; nonempty cases repeat only +cutoff metadata alongside each existing balance row. Full-path checkout/hash +totals in the table are derived from these observations and the call graph; +the build rows are call-graph counts, not an instrumented full-build benchmark. + +The proxy logs full-refresh elapsed wall time separately. The helper logs its +own elapsed wall time, which includes SQL, transfer and blocking work. Neither +small correctness run supplies meaningful latency percentiles or a production +speedup claim, and nested or separate-process timings must not be added. + +## Prior bounded cost investigation + +The earlier investigation used PostgreSQL 16.14 with fsync on, 32 MB shared +buffers, no parallel query workers, C collation, a private Unix socket and a +five-second statement timeout. It used a reduced share fixture with 500,000 +rows and the existing balance table/function. Balance fixtures had one active +partition per nonzero program group (M=N). This was not production data or the +full migrated schema. + +A libpq driver executed the baseline 11-statement idle sequence and the +10-statement first-SELECT-coalesced sequence proposed before the guard repair. +Those timings do not measure the final guard-first query layout. Ten samples followed warm-up, or five at +100,000 recipients, with alternating order. The Rust digest microtest used +the exact shared digest function and encoding helpers, compiled in release +mode on arm64 macOS; its timer excluded SQLx decoding, row transfer and input +string destruction. + +| Returned balance groups N | Baseline DB-client p50 ms | Coalesced DB-client p50 ms | One sorted digest p50 ms | +| ---: | ---: | ---: | ---: | +| 0 | 0.361 | 0.254 | 0.0002 | +| 1 | 0.384 | 0.273 | 0.0007 | +| 100 | 0.398 | 0.371 | 0.0453 | +| 1,000 | 1.835 | 1.778 | 0.3048 | +| 10,000 | 14.342 | 14.463 | 2.3996 | +| 100,000 | 173.741 | 173.618 | 23.7845 | + +These historical samples suggest a possible overhead reduction at small N, +not a measured speedup for the final query layout or a large-N CPU reduction. Both sequences transfer and hash balances twice. SQL grouping cost +also depends on active/inactive partition rows M, which can exceed N, and +sorts may spill. The semantic digest still sorts bytewise independently of +database collation. No revision-only cache is valid: supported balance changes +can occur without a payout revision bump. + +## Regression evidence and limits + +| Scenario | Evidence | +| --- | --- | +| Same-revision amount, recipient, order-key and program changes | Real PostgreSQL private-probe test checks the native digest after each change; Coordinator tests rebuild while preserving old reservation balances | +| Concurrent revision, cutoff and balance update | First-SELECT advisory gate returns the old coherent triple; the same SQL under READ COMMITTED is an explicit failing control | +| Balance or revision drift during build admission | Gated Coordinator tests require a fresh snapshot after the wait | +| Fatal/read-only/missing cluster state plus locked share history | Standalone guard returns `RowNotFound`; the old combined first SELECT is a negative control that times out with SQLSTATE `57014` | +| Checkout, metadata and balance cancellation or SQL timeout | Real PostgreSQL tests recover the single connection and preserve error categories | +| Running hash cancelled | A blocking gate retains build admission until cleanup, while transaction rollback releases its connection | +| Lost or cancelled commit acknowledgement | Wire faults and a commit-response gate prevent successful probe return and verify recovery | +| Original anchor, selected shares, immutable storage order and foreign resume | Existing `refresh_window_split`, `window_reference` and `window_balance_order` suites retained | +| Out-of-order refresh and publication waits | Existing readiness/tip and compact authority proofs retained; older-refresh regression remains green | + +The first-query guard matters under compound faults. The independent baseline +measured fatal cluster state plus an ACCESS EXCLUSIVE share-table lock: the +standalone guard returned no row in 8.919 ms, while the former combined SELECT +waited and timed out with SQLSTATE `57014` after 212.254 ms. PostgreSQL resolves +and locks referenced relations before evaluating the WHERE guard, so an SQL +conditional cannot preserve this ordering. The regression runs the former query +as a negative control and requires the repaired probe and public payout-state +path to return `RowNotFound` while the lock remains held, also for read-only and +missing cluster state. Timing is logged, not asserted against a flaky threshold. + +After the guard succeeds, cutoff and balance access now share one statement. +Their relative error ordering under simultaneous failures is not claimed to +match the old separate reads (which decoded balances before reading cutoff). +Both paths still propagate failure and retain the original caller deadline; +no error authorizes reuse or publishes work. This patch preserves the concrete +first-guard contract, not exact equivalence for every compound error. + +The deterministic fee-RPC regression records the unchanged two-observation +behavior: if same-revision balances change between the early and late probe, +the refresh returns `payout state changed during work reuse`, creates no new +reservation, and the next refresh builds fresh economics. + +A single late economic probe could halve matching-idle balance scans, but is +not enabled here. A successful late mismatch would instead attempt the normal +fresh build in that refresh. It would no longer observe errors or transient +drift entirely before that final read. Those are explicit behavior changes; +coalescing alone is not evidence that removing the early read is equivalent. + +Run the focused checks through `test/prism-native-tests.sh cargo-args` with +PostgreSQL 16, `CARGO_BUILD_JOBS=2` and a private `CARGO_TARGET_DIR`: the library +`ledger::window::payout_state::tests` and `coordinator::miner_tests::refresh_window` +filters, and the `refresh_window_split`, `window_reference`, and +`window_balance_order` integration targets. Added database cases are registered +in `test/prism-gated-tests.txt`. Explicit scale tests and large load benchmarks +are not required for this bounded correctness evidence. diff --git a/docs/prism-rejections.md b/docs/prism-rejections.md index 20b343ada..af02a4ade 100644 --- a/docs/prism-rejections.md +++ b/docs/prism-rejections.md @@ -3,10 +3,13 @@ PRISM rejection reason IDs are stable machine-readable strings. Operator logs, Prometheus metrics, Stratum error data, ledger rejected-row fixtures, and future dashboard/API surfaces should use these IDs instead of parsing human messages. +This reference includes legacy IDs retained for historical records. The +[native metric inventory](prism-native-metrics.md) lists the current closed +share-rejection label set. | Reason ID | Meaning | | --- | --- | -| `stale-job` | The submitted share or block candidate was built on an obsolete qbit tip/template outside the stale-grace credit window. | +| `stale-job` | The submitted share or block candidate no longer has valid job authority, including an obsolete tip/template outside stale grace, a payout revision that moved before the share's commit, or an expired/replaced publication lease refused before COMMIT. | | `duplicate-share` | The same miner submitted a share with an already-seen header. | | `low-difficulty` | The submitted share did not satisfy the miner's assigned share target. | | `malformed-submit` | The `mining.submit` payload could not be parsed or assembled. | @@ -14,20 +17,32 @@ dashboard/API surfaces should use these IDs instead of parsing human messages. | `unknown-job` | The job ID was unknown or no longer active for that client. | | `invalid-extranonce` | The extranonce field had an invalid shape for this coordinator. | | `invalid-ntime-or-nonce` | `ntime` or `nonce` was not a 4-byte hex string. | -| `candidate-audit-mismatch` | The final PRISM audit bundle did not match the submitted coinbase. | -| `submitblock-rejected` | qbit `submitblock` rejected the candidate or did not advance to the submitted block. | -| `backend-rpc-unavailable` | A backend RPC dependency was unavailable while classifying a submission, including stale-grace parent lookup. | +| `candidate-audit-mismatch` | Legacy taxonomy; no native share producer. The final PRISM audit bundle did not match the submitted coinbase. | +| `submitblock-rejected` | Legacy taxonomy; no native share producer. qbit `submitblock` rejected the candidate or did not advance to the submitted block. | +| `backend-rpc-unavailable` | A backend RPC dependency was unavailable while classifying a submission, including stale-grace parent lookup. Also chain authority the frontend could not prove (`current chain state is unavailable`) when no database failure explains it, including a published-work lease whose commit gate could not read or prove that authority immediately before COMMIT (#716; see "Commit-gate refusals"). | +| `backend-database-unavailable` | A ledger database statement failed, or did not answer in time, while authorizing a session, a job or a submission (#581, #655): a database session allocation (`database unavailable`), issued-job persistence or resume (persistence that outlived the issued job's own deadline while it waited on the database included, whichever of that deadline and the statement it cut short failed first), the payout revision and payout state a submission is checked against (`current payout state is unavailable`), and readiness that aged out because the latest template refresh failed on the database (`current chain state is unavailable`). A Stratum session's own deadline that passes while the coordinator is still waiting on the database lands here too, with the session's message (`session allocation timed out`, `job persistence timed out`, `job resume timed out`, `initial job delivery timed out`; #655): the coordinator names each step it waits on the database for, and the session reads that at its deadline. For issued-job persistence those steps are its clock and revision reads and its batch or repair write; a compact repair's lock, build admission and encoding are not (a persistence that outlives the issued job's own deadline stays the database's whatever it was doing, as #581 decided). So does readiness that aged out while the refresh in flight has outlived its deadline, the health timeout, and is still waiting on the database, although it has not failed yet (`current chain state is unavailable`). So does a published-work lease's commit gate that cannot prove its authority while either holds (#716). A lock held past its bound, a full connection pool and PostgreSQL down land here; so does a cluster halted by its fatal state or a read-only database, whose payout read returns no row. A refusal on a path that also reads the node is attributed to whichever dependency failed, or, for a timeout, to the one the work was still waiting on. Not here: corrupt or changed stored data and a failed blocking task (`backend-rpc-unavailable`, as before), and a session timeout that passes while the work waits on anything the coordinator does not name as the database, such as the node, CPU work or admission behind other work (`backend-rpc-unavailable`, as before #655). | | `internal-error` | An internal coordinator failure prevented normal classification. | | `pool-closed` | The coordinator was no longer accepting shares. | -| `block-stale` | The block candidate height was stale against the active qbit tip. | -| `ledger-confirmation-failed` | The ledger did not confirm a block that qbit appeared to accept. | +| `block-stale` | Legacy taxonomy; no native share producer. The block candidate height was stale against the active qbit tip. | +| `ledger-confirmation-failed` | The ledger did not record the share. For share-pass submissions, the commit was not sent or was rolled back. A share whose payout revision moved between its submit check and its commit is not answered here: the append's fence refuses it before any write as `stale-job`, cause `payout_revision` (#675), and while capture is on (`PRISM_CAPTURE_OVERPAY_CEILING_BPS` > 0) a share-pass submission that carries a found block is not refused at all: its block is captured and its answer follows the block's, as for a block-only proof (#657). Nor is a share whose published-work lease's commit gate could not read or prove its authority: since #716 that is answered as admission answers readiness it cannot prove (see "Commit-gate refusals"). For block-only proofs, the block was not on the active chain when its candidate was abandoned, refused by the node after the offer, or settled as a proven orphan (#415); a later reorg or late landing can still credit it. | +| `ledger-outcome-unknown` | The ledger outcome was not known by the acknowledgement deadline; the share may still be credited (logged with `share_id`). | -The coordinator exposes these IDs in: +The native coordinator exposes its produced reason IDs in: - Stratum JSON-RPC error data as `{"reason_id": "<id>"}` when a rejection is classified. - Prometheus as `qbit_prism_rejections_total{reason_id="<id>"}`. +For share telemetry, present unknown or empty IDs normalize to the bounded +`unrecognised` label. This is a metrics-only fallback, not a new protocol +response: the numeric error, message and reason metadata stay unchanged. +Explicit `internal-error` and missing (`None`) IDs retain the `internal-error` +label. The existing reason-less username-limit refusal is an authorization +event and never enters share-rejection telemetry. See the +[native inventory](prism-native-metrics.md) for the per-reason alert implications; +an `unrecognised` count indicates a real rejection with a reason outside the +known taxonomy, not a healthy or absent event. + For ready-pool refreshes, share validation follows the last qbit tip for which the refresh path has a coherent, final-validated replacement bundle ready to fan out. @@ -93,28 +108,126 @@ divergence (later detected tips do not renew it). If refresh or reconciliation stalls beyond that budget, submits fall back to the live RPC read instead of accepting shares against a frozen snapshot. +## Commit-gate refusals + +A share admitted under the published-work replacement lease must still hold +that original authority immediately before COMMIT. If its publication has +been replaced or its fixed lease/resume deadline expired, a completed typed +gate refusal produces the existing `stale-job` response: +numeric code **21**, message **`stale job`**. This is the same authority check +used before persistence, performed again after database waits. It changes +classification, not payout eligibility, stale grace, candidate authority or +the share acknowledgement deadline. No new credit is recorded for this refusal. + +Gate closure alone does **not** prove stale work. Authority the gate cannot +read or prove is answered as admission answers readiness it cannot prove +(#716): code **20**, message **`current chain state is unavailable`**. The +reason is `backend-rpc-unavailable`, or `backend-database-unavailable` when +the latest template refresh failed on the ledger database or the refresh in +flight is still blocked on it (#581, #655). That covers: +- an authority lock the gate could not take without waiting, because the tip + poll, a fee or readiness write, or the publication itself held or was + queued for it; +- unhealthy or unknown readiness; +- an unavailable tip observation. + +Before #716 these were `ledger-confirmation-failed`. Nothing is written, and +neither a stale race nor a ledger failure is proven. A share acknowledgement +deadline that closes the gate before COMMIT, or a refusal that records no +closure, remains `ledger-confirmation-failed` (code **20**). The first +closure wins: if a lease fence closed the gate and its ROLLBACK is still in +flight at the deadline, the fence's answer stands. + +For example, a non-refresh probe may observe a return to the published +parent while its cached observation has aged out. That does not prove the +unchanged publication or the original lease expired, so the refused append +is answered `current chain state is unavailable`, not `stale-job`. The winning +closure cause is fixed atomically: a subsequent tip change cannot turn a +timeout or backend refusal into an expected stale race. Once COMMIT has +started, later lease changes cannot change its result: confirmed credit stays +accepted, a definite rollback stays failed, and an unresolved outcome stays +`ledger-outcome-unknown`. An identical share already durably credited retains +its duplicate outcome even if a later gate refuses. + +If the append has not returned by the acknowledgement deadline, the existing +timeout/unknown handling still applies; gate state alone does not replace a +completed ledger result, including its immutable-row duplicate check. + +**Telling the cases apart in logs.** A gate refusal that isn't a proven stale +lease logs INFO `share commit gate refused before COMMIT`, with a `closure` +field: +- `authority-unavailable`: the gate could not read or prove its authority. + - Within seconds of a tip change, this is ordinary lock contention with + that tip's poll, fee and readiness writes, or publication. + - Away from tip changes, or alongside stale readiness, it is a readiness + or tip-observation loss; investigate that as you would the same answer + at admission. +- `deadline-or-cancelled` or `unspecified`: a strict refusal. + +A WARN `share persistence failed` is a real ledger failure. That includes +`share commit deadline passed before COMMIT was sent`: a share that waited +out its acknowledgement deadline. +`PrismShareAppendFailures` counts only `ledger-confirmation-failed` and +`ledger-outcome-unknown`, so contention at a tip change no longer fires it. +These refusals count under `backend-rpc-unavailable`, or +`backend-database-unavailable` as above, in `qbit_prism_rejections_total`. +The credit-preserving fix, which retries the append once the authority is +readable instead of refusing, remains open in #716. + +The load harness follows the stable reason ID, including for mixed producer +versions and saved reports: + +| Wire reason | Harness class before and after this change | +| --- | --- | +| `stale-job` | Expected race; excluded from `rejected_valid_shares`, never counted as accepted credit. | +| `ledger-confirmation-failed` | Backend refusal; remains in `rejected_valid_shares`, including the historical message `share was not committed because its commit gate closed`. | +| `backend-rpc-unavailable`, `backend-database-unavailable` | Backend refusal; remains in `rejected_valid_shares`. Since #716 they also cover a lease whose commit gate could not read or prove its authority (`current chain state is unavailable`). | +| `ledger-outcome-unknown` | Backend/uncertain outcome; remains in `rejected_valid_shares` and reconciliation. | +| Missing or unrecognised gate reason | Unknown; no stale exemption from the message alone. | + +Older producers used the generic gate-closed message for both stale authority +and backend causes. Neither that message nor proximity to a tip change can +retroactively distinguish them. The raw #447 matrix reports are unavailable +for this change, so no historical verdict recomputation is claimed; the +recorded verdicts and D1 rule remain unchanged. Any future comparison must +name its source reports and show the old/new policy explicitly, preserving +ambiguous failures unless independent causal evidence resolves them. + Rejections are counted, never logged per share or written to the ledger. -Diagnose reject spikes from `qbit_prism_rejections_total`, the -`qbit_prism_job_build_seconds` histogram, and the -`qbit_prism_tip_refresh_seconds` histograms. Per-share/per-job stdout logging -exists only behind `PRISM_HOT_PATH_LOG=1` for debugging and must stay off in -production. Prepared fanout passes validate tip and chain-view trust once per +Diagnose reject spikes from `qbit_prism_rejections_total{reason_id}` and +`qbit_prism_share_ack_seconds{result="rejected"}`, then use the stale-job +causes below. The native runtime has no +per-share or per-job stdout logging; it does not read the retired Python +`PRISM_HOT_PATH_LOG` <!-- retired-setting: PRISM_HOT_PATH_LOG --> debugging switch. Prepared fanout passes validate tip and chain-view trust once per pass (minting a validation token; per-client deliveries consult only in-memory token state) plus a post-fanout re-validation, so per-client RPC round trips never return to the delivery path. -Additional private metrics relevant to attribution and grace behavior: - -- `qbit_prism_grace_credited_shares_total` -- `qbit_prism_vardiff_idle_retargets_total` -- `qbit_prism_worker_submitted_shares_total{worker="<bounded-label>"}` -- `qbit_prism_worker_accepted_shares_total{worker="<bounded-label>"}` -- `qbit_prism_worker_grace_credited_shares_total{worker="<bounded-label>"}` -- `qbit_prism_worker_rejections_total{worker="<bounded-label>",reason_id="<id>"}` -- `qbit_prism_evicted_job_contexts{class="same_tip|stale_grace"}` -- `qbit_prism_evicted_job_submits_total{outcome="accepted_same_tip|credited_stale_grace"}` -- `qbit_prism_evicted_job_expirations_total{class="same_tip|stale_grace"}` -- `qbit_prism_evicted_job_capacity_evictions_total{scope="connection"}` - -`PRISM_WORKER_METRICS_LIMIT` caps distinct worker labels. New workers beyond -the cap aggregate into `_other`. +## Stale-job causes + +Every `stale-job` rejection keeps its wire reason, numeric code 21 and message. +`qbit_prism_stale_job_rejections_total{cause}` records which of the five +decisions below refused the share, once, at that decision. The share observation +counts the same rejection once as `reason_id="stale-job"`. Lease admission, +lease revalidation and commit-gate refusals use that coarse reason without a +label in this five-cause series; its sum is not a total of all stale responses. +`unknown-job` has no cause series. Stale-grace credit is an accepted +share, counted by `qbit_prism_grace_credited_shares_total`, not a cause. + +The decisions run in this order, and a share that would fail several is +counted only under the first one it reaches. No later check runs just to choose +a cause, and backend-unavailable answers are never stale-job causes. + +| Cause | Decision | What to check | +| --- | --- | --- | +| `resume_expired` | Stratum found the job's absolute resume lease expired after restoring it for a reconnected miner, before submitting to the coordinator. Message `stale job`. | Reconnect churn and how long miners take to resubmit old work: `qbit_prism_connections`, `qbit_prism_stratum_connection_refusals_total` and the retention settings `PRISM_STRATUM_SAME_TIP_JOB_RETENTION_SECONDS` and `PRISM_STRATUM_STALE_GRACE_SECONDS`, whichever is longer. | +| `fee_floor` | The coordinator found the job's CTV fanout fee below the live relay floor, or the floor unavailable. Message `job CTV fee is below the current relay floor`. | Node relay policy and mempool minimum fee, then whether replacement work reached miners: `qbit_prism_authorized_missing_current_work` and `qbit_prism_stratum_current_tip_coverage_gap_seconds`. | +| `parent_grace` | The job's parent is not the published tip, and stale grace was unavailable, expired, or the job's parent is not the new tip's immediate parent. Block-only work (#478) is counted here after a tip change without any grace check, and so is a capture abandoned before its offer because its parent is no longer the tip. Message `stale job`. | Tip freshness and delivery: `qbit_prism_stratum_semantic_current_work_ratio`, `qbit_prism_authorized_missing_current_work`, `qbit_prism_stratum_current_tip_coverage_gap_seconds`, `qbit_prism_job_delivery_failures_total` and `qbit_prism_grace_credited_shares_total`. | +| `payout_revision` | The parent is current, but either the job's payout snapshot or the current published payout snapshot disagrees with the durable payout revision, and no share-lease exception applies. Since #478 a block-bearing proof from ordinary work on the active tip is captured instead and is not counted here. Counted here: block-only work, which a same-parent payout replacement retired (its plain shares, and the share of a block captured from it), and a capture abandoned before its offer by the overpay ceiling or with capture off. Also counted here, after every check above passed (a stale-grace share included): a share-pass submission whose revision a settlement moved before the append's fence read it under `ORDER_LOCK`, refused there before any write (#675). That is a plain share, or a block-bearing one with capture off, whose lost block a WARN names; with capture on its block is captured instead (#657). It is counted when the miner is answered `stale-job`; a block-bearing append still waiting at its acknowledgement deadline was answered `ledger-outcome-unknown` instead and is not. Message `stale job`. | Payout revision changes and replacement delivery: `qbit_prism_pending_job_builds`, `qbit_prism_job_delivery_successes_total`, `qbit_prism_job_delivery_failures_total` and the coverage series above. | +| `window_not_held` | A block-bearing proof passed every check above, but the candidate enqueue, in the transaction that would write the candidate, found that the current primary does not hold the window its coinbase pays: the window's last row is absent or is another share credited after the window's anchor (#619). After an asynchronous promotion that lost the end of the replication gap, a session can still hold a job on work read from the lost history. The block is never enqueued or offered, and the share it carried is not credited. Message `stale job`; a WARN names the block and #619. The ledger counts the refusal itself, so one that finishes after the miner's acknowledgement deadline (the miner then saw `ledger-outcome-unknown`) is counted too. | A database promotion: the refresh logs and `qbit_prism_refresh_seconds{trigger="writer_timeline"}`, then the HA reference's promotion steps. See [the #619 design note](prism-async-promotion-gap-blocks.md). | + +The native registry has no per-worker, evicted-job, job-build or tip-refresh +histograms; `docs/prism-native-metrics.md` is the complete inventory. + +The native runtime does not read the retired Python worker-label cap +`PRISM_WORKER_METRICS_LIMIT` <!-- retired-setting: PRISM_WORKER_METRICS_LIMIT -->. diff --git a/docs/prism-release-benchmark.md b/docs/prism-release-benchmark.md new file mode 100644 index 000000000..9402a92fe --- /dev/null +++ b/docs/prism-release-benchmark.md @@ -0,0 +1,254 @@ +# PRISM release benchmark + +Before a release is tagged, its build is compared with the previous +release's on the reference host, on the release preset pair. Each preset is +run as interleaved A/B repeats and held to #473's D1 rule (#511, S4 of +#487). This page names the reference host, the presets and the command. It +also says how to read the result. + +## The command + +```sh +python3 scripts/prism_load_ab.py \ + --base <previous release> --candidate <release candidate> \ + --preset throughput-20k-window-1fe --out ab-throughput-20k-window-1fe +``` + +That one command reproduces #479 STEP 1's steady-state comparison of two +commits with `--base 5d0042f6 --candidate <commit>` (three interleaved +repeats by default). + +The driver, `scripts/prism_load_ab.py`: + +1. **Builds each ref separately.** Each gets a release build in its own + clean detached worktree under `<out>/builds/`. Every run uses that + build's own harness and server, because the harness refuses another + build's server. +2. **Takes the host's benchmark lock.** It holds `/tmp/qbit-prism-bench.lock` + from the first build to the comparison, so two series never share the + host. A second invocation on the same `--out` is refused + outright, whatever lock file it names. After each run the driver, a child subreaper, stops + anything the run left below it, including the servers and PostgreSQL that + leave the harness's process group with `setsid()`, and it does the same + for an interrupted build before releasing the locks. A process that + outlives SIGKILL (stuck in uninterruptible I/O) is waited out, with the + locks still held, before anything else runs. A host where the + driver cannot become a subreaper is refused, dry runs included. +3. **Records `pg_test_fsync`.** It runs `-s 5` on the filesystem the harness + builds its clusters on, once before the series and once after. A probe + that fails, or prints no one-write fdatasync figure, stops the series as + bad input (exit 2), with the probe's output kept. +4. **Alternates the order.** Odd repeats run the base first and even + repeats the candidate first. +5. **Gates each run on load.** Before each run it waits a 60 s cooldown and + then until the 1-minute load average is under 3. It gives up after an + hour and exits 3; it never runs on a loaded host. +6. **Samples the host.** During each run it records the load average and + `MemAvailable` every 10 s. It also snapshots the process table at the + start and end of each run, by command name only, since another process's + command line may carry a secret. +7. **Enforces a ceiling.** A run still going after 2,400 s is killed. +8. **Summarizes.** It calls `qbit-prism-load-compare` and writes the result + to `<out>/comparison.md`. The exit status is the comparison's: 0 pass, + 1 fail, 2 bad inputs. + +`--dry-run` builds both refs, prints each build's resolved command line and +runs nothing. It takes the lock too, since its builds load the host. Printed +and in `manifest.json`, a command line's URLs carry no password: an external +`--database-url`'s is redacted as the harness redacts it. + +`--resume` continues an interrupted series from `<out>/manifest.json`. It +must run on the same host (hostname, vCPUs, memory and kernel) with every +setting unchanged except `--resume`, `--skip-build`, `--dry-run` and +`--compare-bin`. An interrupted run's harness, server and cluster are +stopped before the driver exits, on Ctrl-C, SIGTERM or SIGHUP alike. The summarizer refuses a series in which +either build is short of its repeats. + +`--skip-build` reuses only builds the driver itself made: it records each +build's commit and binary digests beside the binaries, and refuses a +worktree whose binaries were built for another commit, copied in, or left +by a build that failed. `--resume` likewise refuses a build whose binaries are +not the ones the recorded runs used. + +`python3 scripts/prism_load_ab.py --help` lists the +settings. + +### Every flag comes from the preset + +The preset is read from the checkout the driver runs in. It pins every flag +the result depends on, so both builds run the same workload whatever their +harness defaults (see #497, which moved `--stratum-max-pending-initial-jobs` +from 2,016 to 128). Both refusals below happen before anything runs: + +- **A build's harness has an unpinned result flag.** The pair is refused. +- **A build's harness predates a flag the preset pins.** For example, + `5d0042f6` has no `--stratum-max-pending-initial-jobs`. The driver leaves + the flag off only when `crates/qbit-prism-load/legacy-flags.json` says + that build ran that value anyway. For this flag the old value was + `ceil(sessions / frontends) + 16`, at least 128, so `throughput-20k-window-1fe`'s 2,016 + qualifies. Any other value refuses the pair. +- **A build's harness lost a flag the preset pins.** The table records the + commit that introduced each flag (`introduced`). A build whose history + has that commit but whose harness lacks the flag removed it, and the pair + is refused: the legacy rule speaks only for builds older than the flag. + +`tests/legacy_flags.rs` holds the table to the harness's own defaults. +`tests/test_prism_load_ab.py` checks that every D1 preset resolves against +`5d0042f6`'s flag list. + +## The release preset pair + +| Preset | What it measures | Runner class | +|---|---|---| +| `throughput-20k-window-1fe` | #271's D1 plan over the 20k window, 2,000 sessions on one frontend, async standby. This is #479 STEP 1's configuration. | 8 vCPU | +| `throughput-400k-window-1fe-async` | The same plan over #473's 400k production window. | 8 vCPU | + +These are the `d1-20k` and `d1-400k` presets #511 asks for. #559 renamed +them; the old names still resolve through `presets/aliases.txt` for one +release. + +`throughput-500k-window-1fe-async` is the 500k cell, and +`tip-delivery-2000-miners-400k-2fe-retarget` is #275's tip-delivery +benchmark. Run them with the same command when a release touches the window +or tip paths. The tip-delivery preset pins workload flags that `5d0042f6`'s +harness cannot express, so the driver refuses it against that base. + +Both presets gate `steady_state` only. The 2,000 shares/s `burst` is reported +beside it, not gated: #473 found no configuration that meets it, and on the +reference host the disk caps it for every build. + +## Reading the comparison + +`comparison.md` has three parts: + +- **Host facts** and both flush-cost readings. +- **A run table** in run order, with the load average before and during + each run. +- **One table per D1 phase**, with a row per build. + +A build **meets** the D1 rule in a phase only when every repeat passes all +of these: + +- the harness exited 0; +- `shortfall == 0` and no valid share was refused; +- no submit went unanswered; +- the client's ACK p99 stayed within the run's validator limit. + +A run killed at the ceiling, or one that exited non-zero, is kept out of +the medians and fails its build's verdict. A figure a report does not carry +shows as "n/a" and fails the rule; it is never counted as zero. A report +naming a revision other than its build's is refused rather than pooled. + +Every run is also held to the preset's own gates, as +`qbit-prism-load-gate` holds a nightly run: reconciliation, durability, and +any tip-delivery or churn budget the preset sets. Both builds must report the +same target rate and, within 5%, the same phase length in every phase, +every phase reported as completed, and every counted run must report +every phase any run reported and the +preset's pinned frontends, sessions, submits in flight per session, plan, +window size, replication (observed at entry and after the load) and ACK p99 +limit, with every frontend launched +with the pinned server settings (runtime workers, database connections, +commit timeout, block poll, initial-job admission, pool fee), and with the arrival, +population, template, pool-fee, churn and node settings the report states reading +as this harness renders the preset's, with the population its seed draws +and each phase's offers on its pinned arrival clock, second by second and +placed as the population asks (a build whose harness +predates one of those flags is exempt from it, since the legacy table +already proved it ran the pinned value); a build that reads a pinned rate differently +fails the comparison rather than passing on less load. The same holds for +the rest of the planned workload, even when every run agrees: each phase's +database delay, seen to be paid, and frontend restarts (the reconnect phase's drained restart +with two or more frontends, the mid-flight kill's relaunch, with submits in +flight at the kill), the scheduled blocks, each a distinct block accepted by +the node between its slot and the next (under +the dense cadence, the pinned landing budget and the landings it schedules), +each launched frontend by the harness's name for it, the seeded ledger's rows +(a retargeting node's older history included) and +average share size, every pinned session holding work at once when the load +began (the distinct count the harness's start-up gate read; a harness older +than that count, a connection for each), an external database's endpoint and +connection options, +the completed reconnects, +the memory floor (every check of it read), a durable database, the pinned +external and churn tips (each a distinct tip), the +template bits the fake node served after every tip it drove, and the samplers, each sampling at its pinned interval on +every launched frontend. + +**The verdict is PASS** when the candidate meets the rule in every gated +phase, every candidate run passes the preset's gates, and the targets +agree. When the base meets it and the candidate does not, the comparison +calls it a **regression**. The medians' differences are printed under each +phase. + +## One build across a suite (L3, #550) + +`qbit-prism-load-compare --collate <plan.json> --runs-dir <dir> --presets-dir +crates/qbit-prism-load/presets --verdict-out verdict.json` renders one build's +run of a preset suite (`presets/suites.toml`) instead of two builds' runs of +one preset. `.github/workflows/prism-load-l3.yml` calls it after the +production-window matrix. It uses the same row as the A/B table, so a cell +reads the same in both: #473's rule over the repeats in the medians, "n/a" +for a figure a report does not carry, and every run held to its preset's +gates. Each row leads with the preset's sessions, window, frontends, +replication and plan, as #473's tables do. A preset with no D1 verdict table +(the capacity-envelope points and the mainnet shapes) is measured in the same +columns but gets no D1 verdict. + +A run counts only when its artifact is present, its report names the plan's +commit, it ran the checked-in preset byte for byte (SHA-256) and it was built +from a clean tree. Exit 0 when every planned run reached its gate and passed +it, 1 when not, 2 when the inputs cannot be read. The collation does not +compare against a base. The A/B series on the reference host stays the +release's regression check, and L3 is the production-window evidence beside +it, on the CI runner class. + +## Reference host + +Rates depend heavily on what a durable commit costs. The share append +commits inside `ORDER_LOCK`, so the flush cost is serialized per share: + +- #473's laptop flushed without forcing the drive cache (about 34 µs per + commit) and found no 500 shares/s gap between builds. +- #479 then found that gap on a Linux host with real flushes. + +A comparison is only evidence on a host whose flushes are real. + +The reference host is **`ref-linux-479`**, the Linux host #479 STEP 1 ran +on: + +| Fact | Value | +|---|---| +| Machine | shared Linux development VM, 22 vCPU, 41 GiB RAM, one SATA SSD shared with other work | +| PostgreSQL | 16, managed by the harness: a fresh primary and async standby per run, `fsync`, `synchronous_commit` and `full_page_writes` on, `wal_sync_method = fdatasync` | +| **Flush cost** | `pg_test_fsync -s 5` on the clusters' filesystem, one 8 kB write: **fdatasync 3,467 ops/s = 288 µs**, open_datasync 3,937 ops/s = 254 µs, fsync 2,123 ops/s = 471 µs | +| Load gate | 1-minute load under 3, nothing else holding the benchmark lock | +| What the builds did there | `5d0042f6` sustained 500 shares/s in `steady_state` (shortfall 0 / 1,213 / 0); `8fc94faf` placed 420.8–475.1. `burst` was disk-capped at 470–601 shares/s for `5d0042f6` | + +The 288 µs is compiled into the summarizer (`REFERENCE_FDATASYNC_USECS` in +`crates/qbit-prism-load/src/compare.rs`). Every comparison says whether both +of its `pg_test_fsync` readings fall within 2× of it. A reading outside that +band is printed in bold, and that series' rates are not comparable with the +reference's. A preset with an external `--database-url` commits on the +database host's storage, which this probe does not measure, so its series is +placed in no flush class. #479 did not record the VM's hostname; every series records its +own in `manifest.json` (`host.hostname`). + +A replacement reference host needs three things: + +- the same class of machine; +- its `pg_test_fsync` fdatasync figure updated in the table above and in + `REFERENCE_FDATASYNC_USECS`; +- an A/A series (the same commit as base and candidate) to show its spread. + +## Release checklist line + +> Run `scripts/prism_load_ab.py` with `throughput-20k-window-1fe` and `throughput-400k-window-1fe-async`, +> previous release as base and the release candidate as candidate, on the +> reference host, and attach both `comparison.md` files to the release PR. +> Do not tag on a FAIL without a written disposition. + +For the first 3.x.x release the previous release (2.x.x) has no Rust server. +The base is `5d0042f6`, #271's measured build, as it was for #479. +[`mainnet-deployment.md`](mainnet-deployment.md#release-inputs) carries this +line. diff --git a/docs/prism-rust-migration.md b/docs/prism-rust-migration.md new file mode 100644 index 000000000..4f413df54 --- /dev/null +++ b/docs/prism-rust-migration.md @@ -0,0 +1,2497 @@ +# Migrate Prism to Rust and multiple instances + +The Rust server replaces all Python Prism runtime and operator scripts from the +`2.x.x` branch. Stratum, public/audit HTTP routes, signed audit formats, payout rules, and existing +PostgreSQL accounting history remain compatible. The runtime uses native threads +and allows multiple frontends to write to one shared PostgreSQL database. + +This is a coordinated cutover. **Do not run Python and Rust coordinators against +the same database at the same time.** The additive native migration checks the +legacy writer lease and prevents a Python process from acquiring it afterward. +Stopping the Python services and retaining a recoverable backup are required +before migration. + +## Prepare the deployment + +1. Pin the source revision and native image digest. Build and test the native + binary and validate the reviewed environment with `qbit-prism-server + check-config`. Keep the same manifest signing seed, ledger attestation seed, + trusted ledger public key, chain, payout policy, pool fee, and CTV policy. +2. Choose the shared PostgreSQL writer endpoint. All frontends need access to + that same database and a synchronized qbitd on the same chain. Configure TLS, + credentials, connection limits, and replication through the database's + existing operating procedures. +3. Give each frontend a unique `PRISM_INSTANCE_ID` or use generated UUIDs. + Configure `PRISM_RUNTIME_WORKERS`, `PRISM_JOB_BUILD_EXECUTOR_WORKERS`, and + `PRISM_DATABASE_MAX_CONNECTIONS` per host. Budget total database connections + across all frontends and administrative clients. +4. Record a baseline: latest accepted `share_seq`, accepted share count, pool + blocks and pending candidates, carry-forward integrity report and + `audit_head_sha256`, CTV states, and representative audit SHA values. Save + canonical audit downloads and independently verify them against the trusted + ledger key and on-chain coinbase. +5. Back up external audit bodies **and every referenced segment**, including + legacy compact body refs and v2 proof-body segments. A live evidence envelope + alone is not an audit backup. Preserve recorded file paths or plan a mount at + the original absolute path for import. Include the `2.x.x` canonical + `prism-audit-bundle-canonical-*.json.gz` sidecars. +6. **Pre-migration canonical backfill rehearsal (required).** On an isolated + restore of the source database and artifact backup, run the native `migrate` + and `import-audits` commands in the recovery procedure below. Native import + is also the canonical backfill: if a sidecar never existed, it reconstructs + bytes from a recoverable inline bundle or external body/segments, verifies + the trusted ledger signature, recorded coinbase, and existing published SHA, + then stores the bytes. Both completeness counts must be zero before the + production migration. A missing body or a digest mismatch stops cutover; + recover the original evidence rather than changing the published SHA. This + uses the native equivalent, not the `2.x.x` canonical-backfill command, so + it does not depend on #174/#176 or mutate historical public content on the + source. Record restore, migrate, import, verification times and history size + on a production-sized copy; #291 supplies the release rehearsal evidence. + [`make prism-cutover-rehearsal`](#rehearse-the-cutover-on-a-snapshot) runs + this rehearsal on a `pg_dump` and records those times, the longest lock + each step held, and a pass/fail check of every invariant below. + +## Drain and migrate + +1. Remove the Python frontends from new connection routing, stop the Python +coordinators, public read services, and standalone CTV broadcaster daemons, and wait for admitted +writes and candidate accounting to finish. Inspect pending candidate state and +resolve it while the old runtime is still available. Confirm every old process +has stopped and its writer lease has been released or expired. A paused old +process is not a completed shutdown. + +2. Take the final consistent database backup after draining, along with the audit +filesystem backup and securely retained key/configuration recovery material. +Retain the old pinned image for recovery. Rehearse restoring this backup into an +isolated database before the production change. + +3. Run the following with the reviewed operator environment exported, using the +new executable and the existing database: + +```sh +qbit-prism-server check-config +qbit-prism-server migrate +qbit-prism-server import-audits --root /var/lib/qbit-prism/audit +``` + +4. Before admitting native traffic, compare the source and migrated recovery + evidence using the numbered procedure below, before running + `qbit-prism-server backfill-ctv`; record any intentional CTV repair delta + separately afterward. `import-audits` must report + `missing_stored_bodies: 0` and `missing_canonical_bytes: 0`. Run production + `self-check` with the intended services available and retain its JSON and + exit status. A missing body or canonical representation makes it fail in + production; lab mode reports the counts without failing for incompleteness. + Native normalized audits use their stored metadata and immutable snapshot + for canonical reconstruction, so they do not require a duplicate full byte + copy. These counts check availability; import and artifact reads authenticate + the contents. Fetch representative historical artifacts through the actual + public edge, including the oldest history, a sidecar-only import and a + reconstructed body. Require the exact SHA and matching ETag, as below. + +Nothing on the serving path reads a legacy row's canonical bytes. So +frontends can admit traffic before `import-audits` finishes. From 3.0.0 an +import beside them holds `SETTLEMENT_LOCK` only to check the write guard and +commit each row. It sends and stores the row first, within +`PRISM_IMPORT_AUDITS_WRITE_TIMEOUT_MS`; `PRISM_IMPORT_AUDITS_LOCK_TIMEOUT_MS` +bounds its wait for the lock and its hold. SIGTERM stops it (#765). Through +rc.6 it held the lock while it sent and stored each body, which stalls +settlement and template refresh for as long, so run an earlier candidate's +import beside serving frontends only under a guard that kills it with +SIGKILL once a settlement waiter has waited a few seconds: as a container's +PID 1 it ignores SIGTERM. Until `import-audits` finishes, +production `self-check` fails at audit completeness and skips its other checks +(#734). `scripts/prism_legacy_import_admission_gate.sh` checks every other fact +`self-check` checks, against the frontend's own environment on each host. It +reports legacy completeness as pending, and fails on any incomplete native +audit. Until a legacy row is imported, `/public/v1/artifacts/<sha>` answers 500 +for an audit whose 2.x body can't be read on that host (#735), and the block's +audit view answers 404. + +The gate's carry-forward integrity check runs the same report `self-check` +runs. At production size it took about a minute before migration 026 (#737). +Since 026 it is estimated at about half a minute, the sum of its two checks' +EXPLAIN ANALYZE times (20.29 s and 9.14 s), not a timed run. It holds one +snapshot for as long. On a primary with live miners every share updates the +cluster row, so a long snapshot there piles up dead row versions and slows +the share append's lock (#738). Below about 200 shares/s that's harmless. Above it, set +`PRISM_INTEGRITY_REPORT_DATABASE_URL` to a hot standby and the gate runs the +report there. That standby must have `hot_standby_feedback` off, or the report +holds back the writer all the same. Pause its replay first +(`SELECT pg_wal_replay_pause();`, until `pg_get_wal_replay_pause_state()` reads +`paused`; `pg_wal_replay_resume()` afterwards), or set +`max_standby_streaming_delay` and `max_standby_archive_delay` to -1 there, so no +recovery conflict cancels the report. Never pause a standby the writer waits on +for apply (`synchronous_commit = remote_apply`): every commit would stall. The +found-block offer standby (#529) waits only for flush, which a pause doesn't +stop. The gate refuses a URL that isn't in recovery, isn't a replica of this +writer's database on the writer's timeline, has feedback on, serves a writer +that waits for apply, or whose last replayed transaction is older than +`PRISM_INTEGRITY_REPORT_MAX_AGE_SECONDS` (default 300; the frontends' heartbeats +keep a live writer committing). It records both WAL positions. Finish every +query on the standby before `pg_wal_replay_resume()`: after a long pause, the +replayed backlog cancels queries running at the resume or just after it. Or set `PRISM_GATE_INTEGRITY=skip`: +the gate says so in a WARN line, and you record the report from wherever it +ran. Every query the gate runs has a 15 s statement timeout and a 5 s lock +timeout; the report gets 120 s on the writer and 600 s on a standby. + +`migrate` applies the existing schema and additive native migration under a +transaction lock. It preserves share sequence/history, balances, audit metadata, +and settlement rows. New tables hold shared configuration/revision, instance +heartbeats, expiring jobs, immutable audit snapshots, and durable claim state. +Migration 3 retains `2.x.x` publication ordinals, retained worker difficulty, +hashrate rollups, and their watermark. Migration 6 pins the accepted `2.x.x` +source schema, records what was migrated, and declares the schema capability +every later start checks. The base schema and native migrations apply in one +transaction, including carry-forward summary repair, except 002's share-hash +backfill and migrations 013, 017 and 024. On a ledger with rows, 002 maps the +legacy shares' headers after the commit, in bounded batches, and records 2 +once every one is mapped +([below](#migration-002s-share-hash-backfill-applied-online)). With +`migrate --defer-share-hashes`, only the legacy shares near the tip are mapped +before the database serves, and `backfill-share-hashes` maps the rest while +frontends serve, in throttled batches +([Serving with the backfill pending](#serving-with-the-backfill-pending-fence-2)). +Migration 013, the share ledger index trim, builds its indexes after the +commit with `CREATE INDEX CONCURRENTLY` +([below](#migration-013-the-share-ledger-index-trim-applied-online)), and +migration 024 builds the CTV fanout claim lane's index the same way (#668). +With every frontend and tool stopped, `migrate --offline-indexes` builds both +with a plain `CREATE INDEX` in one transaction instead, which reads each table +once and waits for no other transaction. +Migration 017, the share ledger partition conversion, validates its bound and +swaps the table after the commit on a dedicated connection +([below](#migration-017-the-share-ledger-partition-conversion-applied-online)). +Where the ledger is empty all three stay inside the migration transaction, +under the cutover locks that exclude writers (002's backfill then has nothing +to map); otherwise each records its version only after its last change. The +migration is +idempotent; a refusal due to an old active writer, an unsupported source +schema, or unresolved legacy work must be resolved before admitting native +traffic. + +### Two drains, one for each era + +**Legacy rows and native rows have different drains.** Every `2.x.x` candidate +must be drained with the pinned legacy image *before* migrating, as in step 1 +above and in [the drain requirement](#supported-2xx-source-schemas); the +migrator refuses while any remains. The native `qbit-prism-server candidates` +commands act on native-era rows only — they cannot read a chunked v2 body or a +pre-migration v1 document, and running them neither satisfies nor bypasses the +migrator's drain check. + +Use them after the cutover, on rows this release wrote: + +```sh +qbit-prism-server candidates list +qbit-prism-server candidates abandon --block-hash <hash> --reason "<nonblank explanation>" +qbit-prism-server candidates recover --block-hash <hash> [--block-hash <hash> ...] [--apply] +``` + +`candidates list` exits zero on an empty list, so it is the check a `set -e` +runbook waits on before stopping native frontends; `candidates abandon` applies +to a `pending` row only and refuses everything the node may already have been +offered. `candidates recover` lands a native-era block the node has already +accepted, under an explicit allowlist, and never offers one; like the other +two it cannot read a chunked v2 body or a pre-migration v1 document (exit 7 +and exit 8, evidence preserved), so it neither satisfies nor bypasses the +migrator's drain check. All three are documented in full under +[Candidate commands](prism-ledger-ops.md#candidate-commands). A pre-migration +row reaching a native frontend is parked by the claim lane with a `last_error` +naming its `storage_version`; `candidates list` shows it, but the answer is +still the legacy drain, with the pinned `2.x.x` image, never an operator +abandon. + +That rule is enforced, not only documented. A `2.x.x` v1 document is parked at +`storage_version = 1`, the same version the native writer uses, so `abandon` +tests the document rather than the version: it changes a row only when the +`candidate` carries `payout_revision` and `block_hash` beside an inline +`bundle` or a `window` reference — the predicate +`refuse_undrained_outbox` classifies native rows with, above. A legacy document +is refused atomically with exit 8 and keeps every column, so an operator +sweeping a stalled outbox cannot delete the evidence this drain still needs. + +### Supported 2.x.x source schemas + +The minimum supported source release is **v2.0.1** (`95ffe06`). A v2.0.0 +(`f6854a0`) database has the identical schema and migrates as the same source +state, but drain it with the v2.0.1 or later image, because the offline +recovery command below first shipped in v2.0.1. The newest supported source is +**v2.0.2** (`504846c`, #258). The SQL those releases applied is frozen +byte-for-byte under `crates/qbit-prism-server/tests/fixtures/schema_2x/`. The +migrator classifies the database before it runs any DDL, and after +`001_share_ledger.sql` has run it checks that what 001 left alone is the +release definition: + +| Source state | Evidence | Verdict | +| --- | --- | --- | +| fresh | no `qbit_share_ledger`, no other object `001_share_ledger.sql` creates, no `002_candidate_bodies.sql` object: an empty database, with or without objects of the operator's own | accept | +| partial 001 | no `qbit_share_ledger`, but some table, sequence, index, trigger or function that 001 creates is present: a selective restore, or part of the schema installed by hand | refuse before any DDL, naming the objects present; restore the full pre-migration backup or migrate into an empty database | +| pre-#258 (v2.0.0, v2.0.1) | `001_share_ledger.sql` only: no `qbit_prism_schema_capabilities`, no `002_candidate_bodies.sql` object | accept after the drain check | +| #258 applied (v2.0.2) | `candidate_storage_version = 2` and every `002_candidate_bodies.sql` object present | accept after the drain check | +| partial 002 | some 002 objects or the capability row, but not all (v2.0.2 applies 001 and 002 as two script calls, and a restart between them leaves this) | refuse, naming the missing object; finish 002 with the v2.0.2 release (`PRISM_POSTGRES_INIT_SCHEMA=1`) or restore the backup | +| newer | `candidate_storage_version > 2`, or a capability this release does not know | refuse before any DDL; a newer PRISM release wrote the database. A native database also gets a capability check before later DDL; after migration 6, its candidate version must be 1, the format the native claim lane can process | +| native collision | a table, sequence, index, trigger, function or column that a native migration (`002_multi_instance.sql` to `017_share_ledger_partitions.sql`) creates and the `2.x.x` release does not is already present, in an empty database or a `2.x.x` one, or a reserved relation name is held by a relation of another kind (a view, an index backing an operator's constraint): a leftover of an earlier native attempt, a selective restore, or something installed by hand | refuse before any DDL, naming the objects; nothing is dropped; restore the full pre-migration backup, or check what the objects hold and remove them, then migrate again | +| drifted 001 | a 001 (or 002) object whose definition, after 001 has run, differs from the frozen release: a table, column, index, sequence or named constraint that 001's `IF NOT EXISTS` skipped, or any 002 object, with a dropped constraint, a changed type, nullability or default, a different index definition, an altered sequence (a lowered maximum, a different increment), a table or sequence made `UNLOGGED` (or temporary), a release constraint left `NOT VALID` (other than the pinned `qbit_share_ledger_credit_policy_check`), a release foreign key whose enforcement triggers were disabled, row-level security enabled or forced on a release table or a policy on one, a child table created with `INHERITS` on a release table or a release table made a child or partition of another, a replaced function body, a disabled trigger or a trigger the release does not create on a release table | refuse transactionally, naming each object and what differs; the migration rolls back and the database is unchanged; restore the pre-migration backup or bring the database to the release schema with the `2.x.x` release, then migrate again | + +Migration requires visible `qbit_` relations and functions to resolve in +`current_schema()`, where its unqualified DDL creates objects. An empty first +schema followed by a legacy ledger schema in `search_path` is refused before +any DDL, preventing a parallel ledger from hiding accounting history. +Set the intended ledger schema first and ensure no `qbit_` objects resolve +from other schemas. A later empty schema or unrelated operator objects are +allowed. + +**The release definitions.** They are not a stored fingerprint: before any +DDL touches the source, inside the migration transaction, the migrator opens +a savepoint, creates a scratch schema, applies the frozen 001 there (plus +`002_candidate_bodies.sql` for a #258 source), reads every table, column, +constraint, index, trigger, function and sequence that produced, and rolls +the savepoint back, so the comparison is exact for the PostgreSQL version in +use and nothing from the scratch apply survives. Both checks below use that +one reading. + +**The fresh check.** A database without `qbit_share_ledger` and without any +002 object is fresh only if it has none of the other objects 001 creates. +Anything else, a `qbit_pool_blocks` restored on its own, a leftover +`qbit_audit_publication_sequence_seq`, a hand-created function, is a partial +001: 001's `IF NOT EXISTS` would keep it exactly as it is, so the migration +is refused before any DDL, naming what is present: + +``` +refusing to migrate a partial 001 source before any DDL: the database has no qbit_share_ledger but +holds 4 object(s) that the 2.x.x release's 001_share_ledger.sql creates (table qbit_pool_blocks; +index qbit_pool_blocks_audit_publication_sequence_idx on qbit_pool_blocks; index +qbit_pool_blocks_maturity_idx on qbit_pool_blocks; index qbit_pool_blocks_public_recent_idx on +qbit_pool_blocks), so it is neither an empty database nor a 2.x.x ledger, and 001's IF NOT EXISTS +would keep those objects whatever they hold. Nothing was changed. Restore the full pre-migration +backup, or migrate into an empty database +``` + +Objects the release does not create, an operator's own table for instance, +do not disqualify a fresh database; they are kept and logged at warning +level. + +**The native-collision check.** The same scratch apply continues with the +native migrations, in the order the migrator applies them, and is read +again; the second reading minus the first is every table, sequence, index, +trigger and function a native migration creates and the release does not +(`qbit_prism_schema_migrations`, which the migrator creates itself, is never +in it), and, per release table, every column a native migration adds to it +(the claim columns 002 adds to the outbox and the fanout artifacts, for +instance). A `2.x.x` database or an empty one that already has one of those +objects or columns is refused before any DDL, whatever it holds: `CREATE +TABLE IF NOT EXISTS`, `ADD COLUMN IF NOT EXISTS` and their kin would keep it +as it is, the migration would then record a schema it did not build, and a +writer would fail only afterwards. A reserved name held by a relation of +another kind is the same collision, because `IF NOT EXISTS` looks at the +name alone: a view under a native table's name, or an operator table whose +unique constraint is named `qbit_prism_candidate_claim_idx` and whose index +therefore owns that name, would make 002 skip the outbox index it meant to +create and leave candidate polling to scan the outbox; each is named by +what holds the name (`view qbit_prism_jobs`, `index +qbit_prism_candidate_claim_idx backing constraint +qbit_prism_candidate_claim_idx on operator_notes`). For example, a leftover +`qbit_prism_cluster(singleton boolean PRIMARY KEY)` on an otherwise empty +database is refused like this: + +``` +refusing to migrate a native collision source before any DDL: the empty database already holds +1 object(s) that the native migrations create and the 2.x.x release does not (table +qbit_prism_cluster), so a native migration's IF NOT EXISTS would keep each such table, sequence, +index, trigger, function or column whatever it holds and the migration would record a schema it +did not build. Nothing was changed. Restore the full pre-migration backup, or check what those +objects hold and remove them yourself, then migrate again +``` + +A column already on a release table under a reserved name, a `claim_token +integer` on the outbox say, is named as `column +qbit_block_candidate_outbox.claim_token`. Nothing is dropped for you. An +object or column in both readings, the capability table on a #258 source +for instance, or the outbox's `storage_version` there (the release 002 +added it; on a pre-#258 source 006 adds it, so it is reserved), stays +governed by the release checks. + +Migration 017 reserves a family of names rather than a list. The partitions of +the share ledger are named `qbit_share_ledger_p<n>` in creation order, and their +indexes take that name as a prefix, so any relation in the ledger's schema +matching `qbit_share_ledger_p<n>` or `qbit_share_ledger_p<n>_<suffix>` is +refused before the conversion takes its first step, whatever kind of relation +it is. The names the swap itself takes are `qbit_share_ledger_p0` for the +converted release table and `qbit_share_ledger_p0_pkey`, +`qbit_share_ledger_p0_share_id_key`, +`qbit_share_ledger_p0_accepted_recent_idx`, +`qbit_share_ledger_p0_accepted_block_suffix_idx`, +`qbit_share_ledger_p0_accepted_seq_walk_idx` and +`qbit_share_ledger_p0_accepted_miner_history_idx` for its six indexes. An +operator's own relation under one of those names is named in the refusal with +its kind and is never adopted; rename it out of the `qbit_share_ledger_p` +prefix, or move it aside, and migrate again. + +**The release-schema check.** 001 is the idempotent schema every `2.x.x` +start re-applied, so the migrator applies it next: that repairs everything +001 re-asserts (its functions, its triggers, the columns and constraints it +alters or re-adds by name). What `CREATE TABLE IF NOT EXISTS`, `ADD COLUMN IF +NOT EXISTS` and `CREATE INDEX IF NOT EXISTS` skip is checked after it, before +`002_multi_instance.sql` or any later native migration alters those tables. +It runs on a fresh database too, where 001 has just created everything and +it passes trivially. Each object needs an equivalent in the schema the +migrator runs in: for every table and sequence its persistence (the release +creates ordinary logged relations, and an `UNLOGGED` or temporary table is +one whose rows PostgreSQL truncates after a crash, so it is drift whatever +its columns say; a sequence follows its table's persistence on PostgreSQL 15 +and later); for every table its row-level security, both flags (`ENABLE ROW +LEVEL SECURITY` and `FORCE ROW LEVEL SECURITY`) and every policy on it, by +name, with its command, whether it is permissive or restrictive, its roles +and its `USING` and `WITH CHECK` expressions (the release creates none of +this; a policy that hides rows from the migrate role, or forced security +with no policy at all, would make the drain check see an empty outbox and +the native claim lane never see the legacy rows it left behind, so a policy +on a release table is drift, not an extra: `table qbit_block_candidate_outbox +differs: expected row-level security disabled, found enabled and forced`, +`policy hide_pending on qbit_block_candidate_outbox: FOR ALL USING ((state +<> 'pending'::text)); the release has no row-level security policy on this +table`); for every table its inheritance, both ways (the release creates +none: a child created with `INHERITS (qbit_share_ledger)` has its rows +included in every query of the ledger, the share reads included, without +the release constraints ever checking them, and a release table attached as +a partition of or inheriting from another table is no longer the relation +the release defined, so either is drift: `table qbit_share_ledger differs: +expected no child table, found child table(s) qbit_share_ledger_2025`; +inheritance among the operator's own tables is theirs to keep); column +type, NOT NULL, default, identity and generated status, collation; +constraints per table by definition, validation state and the enabled state +of the internal triggers that enforce them; index definitions and validity; +trigger definitions and enabled state; function arguments, result, language, body, +volatility and settings; and for every sequence, the ones behind `bigserial` +columns and the explicit `qbit_audit_publication_sequence_seq` alike, its +data type, start, increment, minimum, maximum, cache and cycle. The value a +sequence has reached is data and is not compared, so a ledger whose share +sequence has advanced migrates and keeps its position. Schema qualification, +column order, comments and the names of auto-generated constraints are +ignored. A constraint's validation state is compared: a release constraint +that is `NOT VALID` in the source, a foreign key or CHECK dropped and +re-added without checking the rows that were there, is drift (`constraint +qbit_block_candidate_outbox_share_id_fkey on qbit_block_candidate_outbox is +NOT VALID; the release validates it`), because those rows may be orphaned +or otherwise invalid. The one exemption is pinned: +`qbit_share_ledger_credit_policy_check`, which the release 001 itself adds +`NOT VALID` to a `qbit_share_ledger` upgraded from before the column +existed and never validates, is accepted in either state. A test derives +that list from the frozen release SQL, so it cannot drift from the release. +The internal triggers that enforce a foreign key are compared the same way: +a superuser's `ALTER TABLE ... DISABLE TRIGGER ALL` during a bulk load +leaves the constraint's definition and validation state as they were while +no new row is checked against it, and 001 never re-enables a trigger it did +not create, so it is drift (`constraint +qbit_block_candidate_outbox_share_id_fkey on qbit_block_candidate_outbox +differs: enforcement triggers expected 4 enabled, found 4 disabled`). +A trigger the release does not create on a release table is drift too, not +an extra: it can refuse or rewrite rows the migrator and the native writers +put there. A guard that rejects `writer_epoch = +0`, say, lets the `2.x.x` writer's leased epochs through and fails every +native share insert, which writes epoch 0, so it is named with what it runs +(`trigger operator_epoch_guard on qbit_share_ledger: CREATE TRIGGER +operator_epoch_guard BEFORE INSERT ON qbit_share_ledger FOR EACH ROW EXECUTE +FUNCTION operator_reject_epoch_zero(); the release has no trigger on this +table`); a trigger on a table the release does not create is the operator's +own and is kept. Rewrite rules on release tables are compared by definition +and enabled state too. An extra rule is refused even when disabled: an +`ON INSERT DO INSTEAD` rule can suppress or redirect native inserts before +triggers run. Rules on operator-only tables are kept. +An extra constraint on a release table is also drift: +`CHECK (writer_epoch > 0)` accepts legacy shares but rejects native shares, +so migration refuses it by name and definition, regardless of validation or +enforcement state. Foreign keys from operator tables into release tables are +also refused, including owners in other schemas: their internal triggers can +block native updates or deletes, such as pruning old fanout broadcast attempts. +Constraints involving only operator tables are kept. +An extra `NOT NULL` column on a release table without a default, identity +or generated expression is also drift: native inserts omit it and would +fail. Extra defaults, identities and generated expressions are also refused: +PostgreSQL can evaluate them during native writes, and migration cannot prove +that they will succeed (for example, a generated `10 / writer_epoch` fails +for native epoch-zero shares). Only plain nullable extra columns remain +accepted, and their types must not be domains. Domain defaults, constraints +and nullability can reject an omitted value even when the column itself +looks nullable. Columns on operator-only tables are unaffected. All extra +indexes on release tables are refused, including plain nonunique indexes: an access method or custom operator +class can execute code that rejects native writes. Unique, expression and +partial indexes can also constrain or evaluate writes. The migrator reports +the index without removing it; review and remove extra release-table indexes +before migrating. Indexes on operator-only tables remain accepted. +Accepted columns and extra tables, functions and sequences are kept and +logged at warning level. A missing or different object refuses the migration: + +``` +refusing to migrate a drifted 001 source: after 001_share_ledger.sql ran, the database does not +match the v2.0.x release schema (v2.0.1, 001_share_ledger.sql), 3 object(s) differ (column +qbit_pool_blocks.parent_hash differs: expected NOT NULL, found nullable; sequence +qbit_share_ledger_share_seq_seq differs: maximum expected 9223372036854775807, found 1000000; +missing constraint qbit_block_candidate_outbox_share_id_fkey on qbit_block_candidate_outbox: +FOREIGN KEY (share_id) REFERENCES qbit_share_ledger(share_id)). Nothing was changed: the migration +rolled back. Restore the pre-migration backup, or bring the database to the release schema with the +2.x.x release (v2.0.1 or later; v2.0.2 for a #258 database) and take a new backup, then migrate again +``` + +The check needs the migrate role to hold `CREATE` on the database, for the +scratch schema; without it `migrate` refuses and names the grant. The +migration record is unchanged by this check: `pre_258` still means the +database is equivalent to the v2.0.1 release schema. + +`3.x.x` never applies `002_candidate_bodies.sql` and does not import #258's +chunked candidate bodies. On a #258 source it keeps the 002 tables and +triggers; native writers store version 1 JSONB candidates, which 002's +dual-format rule accepts. After validating and draining the source, 006 +records its former capability value of 2 in the source metadata and changes +the live declaration to 1. Version 2 is accepted as legacy source evidence, +never as native runtime support. + +**#258's rollback floor.** Once a `storage_version = 2` outbox row exists, a +writer that predates v2.0.2 must not run against the database (see the header +of `002_candidate_bodies.sql`). The same floor applies to this cutover from the +other side: `3.x.x` cannot replay a v2 row either, so every v2 row must be +drained by the v2.0.2 coordinator before migration. Never roll a #258 database +back below v2.0.2 to drain it. + +**The drain requirement covers v1 and v2 rows.** A pending v1 candidate +(JSONB body without the native `payout_revision`, `bundle` and `block_hash` +fields) and a pending v2 candidate (`candidate` NULL, body in the chunk +tables) both cannot be replayed by Rust. `migrate` refuses them +transactionally, without changing the schema, and names the blocking rows: + +``` +legacy Python block outbox is not drained: 2 pending 2.x.x candidate row(s) cannot be +replayed natively (block_hash=... storage_version=1 ...; block_hash=... storage_version=2 ...). +Drain them with the pinned 2.x.x release before migrating: ... +``` + +The check reads outbox rows, not the capability row: 002 declares +`candidate_storage_version = 2` whatever the writer stored, so the row proves +002 ran, not that v2 work is pending. Drain with the pinned `2.x.x` image: + +1. Start the `2.x.x` coordinator (v2.0.2 for a #258 database, v2.0.1 or later + otherwise) and let its block submitter finish every pending candidate; it + replays every durable pending row on start. +2. For a block that is already accepted on the active chain but cannot complete + through normal replay, follow the + [pinned `2.x.x` offline recovery procedure](https://github.com/Qbit-Org/qbit-mining-bootstrap/blob/504846cc0b72e8f86ed17f896d4ccbbe196a31dc/docs/prism-ledger-ops.md#L372-L399) + from the `2.x.x` image, with the managed coordinator and its restart + supervisor stopped. Select the exact hashes with `--block-hash` and inspect + the read-only plan first; repeat the same allowlist with `--apply` to drain. + + That command was removed from `3.x.x` with the Python runtime; it only + exists in the `2.x.x` image. +3. Confirm `qbit_block_candidate_outbox` has no `pending` rows, then repeat the + backup and migration boundary. Do not delete pending rows merely to bypass + this check. + +A v2 row that reaches the native claim lane anyway (one written after the +drain, for instance) is parked, not retried: the lane records why in +`last_error`, releases the claim, and sets `next_attempt_at` to `infinity`, so +no lease expiry offers it again. Find parked rows with +`SELECT block_hash, storage_version, last_error FROM qbit_block_candidate_outbox +WHERE state = 'pending' AND next_attempt_at = 'infinity'` and drain them with +the `2.x.x` image as above. That replay advice applies to the legacy pending +rows being drained before migration. After 011, changing `next_attempt_at` +only reschedules reconciliation for a reserved or offered row; it never +grants another offer. Follow [the offer lifecycle recovery guidance](prism-ledger-ops.md) +for those rows and retain their evidence. + +**A database an earlier 3.x.x build migrated before 006.** `3.x.x` is a +development line with no supported production upgrade, but a database an +earlier `3.x.x` build brought to native schema 3, 4 or 5 can still hold a +pending v2 row: that build's drain check used the v1-only predicate, which +never counted a row whose `candidate` is NULL. `migrate` therefore runs the +same column-aware drain check on a native schema 3, 4 or 5 before 004, 005 +and 006, and refuses before any DDL when such a row is pending, naming the +rows as above: + +``` +refusing to apply migration 006 to a native database at schema migrations 2, 3, 4, 5, 8, 9: an earlier +3.x.x build migrated it before the drain rule covered these rows, and the legacy Python block outbox is +not drained: 1 pending 2.x.x candidate row(s) cannot be replayed natively (block_hash=... +storage_version=2 ...). Nothing was changed. Restore the pre-migration 2.x.x backup and drain them with +the pinned 2.x.x release ... +``` + +Before the drain check or native DDL, an existing outbox `storage_version` +column must match `integer NOT NULL DEFAULT 1`, with no identity or generated +expression. Migration 006 uses `ADD COLUMN IF NOT EXISTS`, so it would keep a +malformed column that breaks subsequent native candidate inserts or claims. +A mismatch refuses the upgrade, names the differing properties, and leaves +the database unchanged. An absent column is accepted and created by 006. +The `qbit_prism_migration_source` name must also be absent when migration 6 +is not recorded. A partial, complete or populated metadata table is refused +before native DDL: 006 must create its own record, never retain unverified +provenance through `IF NOT EXISTS`. Restore the full backup, or review and +move the existing object aside before retrying; migration removes nothing. +The pre-006 native outbox must have neither row-level security (`ENABLE` or +`FORCE`) nor policies, including inactive policies. The drain check reads +these settings from the catalog before counting pending candidates, so a +policy cannot hide a legacy row and make the database appear drained. +Refusal preserves the rows and security settings for operator review. + +Native pending rows the earlier build wrote carry the native fields and are +not counted; a v2 body, a `body_id`, a `storage_version` other than 1, or a +v1 body without the native fields is. The remedy is the one in the recovery +section below: the `2.x.x` release is not supported against a native schema +(its revert script refuses one, and this guide never points it at a migrated +database), so restore the pre-migration `2.x.x` backup, drain there with the +pinned image, take a new backup and migrate again. If native traffic was +admitted after the earlier migration, that restore discards it and needs the +reconciliation decision the recovery section describes. + +The capability rows of that database are checked before the drain check, so +`migrate` never applies 004, 005 and 006 to a database a newer release wrote: +one that declares `candidate_storage_version > 2` or a capability this +release does not know is refused before any DDL, the "newer" verdict of the +table above, and nothing is recorded for it. A database already at 6 gets +the runtime check before 008 and 009: any candidate version above 1 is +unsupported, even 2. The remedy is the startup gate's: upgrade the server. + +``` +refusing to migrate a native database at schema migrations 2, 3, 4, 5, 8, 9 before any DDL: database declares +candidate_storage_version = 3, but this server understands candidate_storage_version 1 to 2 only: a newer +PRISM release wrote this database; upgrade the server before starting it here +``` + +A migration record with 3 and not 2 is refused before any DDL as well. Every +native build records both in the one transaction that applies them, so such +a record was edited or restored selectively; every start refuses the gap, +and `migrate` neither re-runs `002_multi_instance.sql` on a database at 3 +nor records it unseen, which would vouch for objects it never checked, so +004 to 009 are not applied above it either. Restore the full pre-migration +backup, or, once every object 002 creates is verified present, record it by +hand (`INSERT INTO qbit_prism_schema_migrations(version) VALUES(2)`) and +migrate again. + +Migration history itself is validated before its rows are trusted at startup +or migration. `qbit_prism_schema_migrations` must retain its native logged-table, +integer primary-key and timestamp definitions, without extra columns, +constraints, indexes, triggers, rules, inheritance or row security. A fresh +or 2.x.x source must not already hold this table without migration 3. Refusal +leaves the database unchanged; restore the full backup or review and repair +the history table (move a non-native object aside) before retrying. + +**What was migrated.** After a successful migration +`qbit_prism_migration_source` holds one row: the accepted source state +(`pre_258`, `258_applied`, `fresh`, or `native` for a database that was +already on the Rust schema), the `2.x.x` release and commit that source +corresponds to, the capability value the source declared, the highest +schema migration recorded before this one, and which instance migrated it. `migrate` prints it, +every start logs it, and a repeated `migrate` never rewrites it. +Once migration 6 is recorded, both the table and its readable singleton row +are required at startup and before applying later migrations. Missing or +malformed metadata is refused without committing later DDL or migration +versions. Restore the full backup, including the original source record; +the migrator never invents provenance for an already-migrated database. + +**Startup gate.** Every start reads `qbit_prism_schema_migrations` and +`qbit_prism_schema_capabilities`, with or without +`PRISM_POSTGRES_INIT_SCHEMA`. This release requires every native migration +it ships, from 2 on (`REQUIRED_SCHEMA_VERSIONS` in +`crates/qbit-prism-server/src/ledger/migration.rs` lists them), each checked +on its own rather than as a high-water mark: a later migration being present +never stands in for an earlier missing migration. Stop all +older frontends before applying 011; it refuses live pre-upgrade claims and +quarantines previously attempted candidates for reconciliation without +another offer. See [the offer lifecycle upgrade procedure](prism-ledger-ops.md) +for the quiesce and recovery steps. Stop all frontends again before applying +015 and restart only upgraded binaries; its capability gate runs at connect +and does not evict older processes already serving the database. Keep automatic +restarts disabled throughout the cutover. 013 is recorded only once its online +index builds have completed, so a start after an interrupted build is +refused until `migrate` finishes them. 017 is recorded the same way and for +the same reason: it is recorded only after the swap has made the release table +the first partition of the partitioned parent, so a start after an interrupted +validation or an interrupted swap is refused until `migrate` has resumed and +finished the conversion. Migration 018 adds the durable chain epoch +and declares `chain_observation_epoch = 1`. Apply 018 only after stopping every old frontend, +one-shot writer and paused startup, with restarts disabled, then start only +epoch-aware binaries. The registered-instance shutdown check does not evict a +connected writer or discover an unregistered old tool. See the +[offline epoch upgrade contract](prism-ledger-ops.md#chain-observation-epoch-upgrade-018). +The capability refuses older binaries at subsequent connects; deleting it or +resetting the epoch is not a supported downgrade. A preexisting undeclared +epoch column or capability causes migration refusal and transaction rollback. +Migration 019 adds the immutable signing-key rotation journal +(`qbit_prism_signing_transitions`). It is additive and declares no capability: +a binary that does not know the table never reads or writes it. See +[signing-key rotation](prism-ledger-ops.md#signing-key-rotation). +Migration 020 adds the payout-divergence evidence of #478 block capture +(`qbit_prism_payout_divergences`, `qbit_prism_payout_divergence_accounts`) and +its report function (`qbit_prism_payout_divergence_report()`). It is additive +and declares no capability. See +[block capture and payout divergence](prism-ledger-ops.md#block-capture-and-payout-divergence-478). +Migration 021 adds `claim_renewals` and `claim_lease_seconds` to the +candidate outbox, a partial index over its claimed rows, and declares +`candidate_claim_observed_lease = 1`: a claim is taken over only once the +taker has watched it go unrenewed for its lease on its own monotonic clock, +so a database clock step no longer ends or stretches a lease (#581). A +pre-021 binary takes claims over by the database clock, so 021 is applied +offline exactly as 018 is, and the capability refuses older binaries at +later connects. See +[candidate claim leases and database clock steps](prism-ledger-ops.md#candidate-claim-leases-and-database-clock-steps-021-581). +Migration 022 does the same for CTV fanout claims (#654): it adds +`claim_renewals` and `claim_lease_seconds` to `qbit_ctv_fanout_artifacts`, a +partial index over its claimed rows, and declares +`fanout_claim_observed_lease = 1`. A pre-022 binary takes fanout claims over +by the database clock, so 022 is applied offline as 021 is. See +[CTV fanout claim leases and database clock steps](prism-ledger-ops.md#ctv-fanout-claim-leases-and-database-clock-steps-022-654). +Migration 023 adds the cluster-wide block submission hold (#664): the +one-row table `qbit_prism_submission_hold`, its append-only journal +`qbit_prism_submission_hold_events`, and the capability +`block_submission_hold = 1`. A pre-023 binary would claim and offer +candidates under a hold, so 023 is applied offline as 021 is, in the +cutover's stopped window, and the capability refuses older binaries at later +connects. See +[holding the whole cluster](prism-ledger-ops.md#holding-the-whole-cluster-023-664). +Migration 024 adds `qbit_ctv_fanout_artifacts_lane_idx`, a partial index over +the fanouts the CTV claim lane may still claim, keyed by their schedule, so a +claim reads only the fanouts due now and no longer the settled fanout history +(#668). It is additive, as 019 and 020 are: no capability and no shutdown +proof, and it is not applied offline. A binary that does not know the index +never reads it. It is applied online, as 013 is: on an existing ledger +`migrate` builds it with `CREATE INDEX CONCURRENTLY` after the migration +transaction commits, so no write to `qbit_ctv_fanout_artifacts` waits for the +build, a found block's landing included, and records 24 once it is valid. See +[the claim lane](prism-ledger-ops.md#the-claim-lane-and-settled-history-024-668). +Migration 025 replaces `qbit_carry_forward_integrity_mismatches()` (#708): +rows landed before 011 are checked per payout program, as 2.x keeps their +balances, instead of per case-sensitive label, so one program paid under two +spellings of its bech32 address is one chain. On union's ledger the label +rule found 5,506 false mismatches in such a program, which `self-check` and +`fatal-state clear` refuse; the program rule finds none over all 2,753,849 +active rows. It changes no stored row and is applied in the migration +transaction, with no capability and no shutdown proof: an earlier binary +accepts the unknown migration with a warning and reads the corrected report. +See [blocks, balances, and reorgs](prism-ledger-ops.md#blocks-balances-and-reorgs). +Migration 026 replaces `qbit_carry_forward_integrity_report()` (#737) so +that it runs `qbit_carry_forward_integrity_mismatches()` and +`qbit_carry_forward_current_drift()` once each instead of twice. It keeps +001's subqueries word for word, evaluates each once in 001's order, and +takes each count from its listing. On union's ledger EXPLAIN ANALYZE put the +two checks at 20.29 s and 9.14 s, and the report took about 57 s; once each +is estimated at about 29 s, their sum, not a timed run. The report is +evaluated the same way as 001's, by the same listing subqueries, so findings +that tie on the listing's sort keys come out in the order those subqueries +leave them, as in 001, whose own order for them is not fixed either. +`migrate` runs 026 again whenever it runs 001, as it runs 025 again after +011. Like 025, it changes no stored row and is applied in the migration +transaction, with no capability and no shutdown proof: an earlier binary +accepts the unknown migration with a warning and reads the same report. +While a share-hash backfill that this release started is pending on a +populated `2.x.x` source, the database declares +`share_hash_backfill_pending = 1` (#669), raised to 2 once +`migrate --defer-share-hashes` has permitted serving. It is declared with the +backfill's cursor and removed with it when 2 is recorded, so every earlier +build that checks capabilities is refused until then, and every build before +rc.5 refuses 2. A backfill that a #582 build +started carries no declaration (see +[the backfill](#migration-002s-share-hash-backfill-applied-online)). +A database missing any required migration is refused +at connect, naming the gap, before any accounting statement runs, and so is +one declaring a +capability or a runtime `candidate_storage_version` other than 1. A native +version-2 declaration is refused before the server can claim and park a +newer writer's candidate; a #258 migration records its source version of 2 +separately and declares runtime version 1 within the migration transaction. +The capability relation must be an ordinary table; views, materialized +views, foreign tables and partitioned relations cannot substitute for it. +Its catalog kind is checked before any capability rows are read, at startup +and before migration DDL. Restore the original table from the full backup +if a selective restore replaced it. +The declaration must also belong to the current schema, where the migration +history was verified: `search_path` resolves an unqualified name past a +schema that lost its table to a later schema's, and that declaration +describes another ledger. Startup refuses it, naming both schemas, instead +of trusting a version-1 row from elsewhere; `migrate` already refuses any +`qbit_` object resolved from another schema before any DDL. +A database at 6 must declare its capabilities: 006 created +`qbit_prism_schema_capabilities` and its `candidate_storage_version` row and +nothing native removes them, so a start refuses a database missing either, +naming the remedy, rather than reading the absence as a legacy state, and +`migrate` refuses the same database before any DDL. +The capability table must also have row-level security disabled, including +its `FORCE` flag. Startup and migration check the catalog before trusting +its rows, because a policy could hide an unknown capability while exposing +the required candidate-version declaration. Nothing disables a policy +automatically; operators must review the table before retrying. +A migration this release does not know is accepted with a warning that names +it: native migrations are additive, and a release whose format an older +binary must not touch declares a capability. With the native default +`PRISM_POSTGRES_INIT_SCHEMA=0` that means a newer binary refuses to start +until `migrate` has run, instead of failing later in the claim path. + +`import-audits` processes database rows whose audit body is external. It resolves +full v1/v1.1 bodies, legacy body refs, and v2 proof bodies, verifies segment +references, the trusted ledger signature, recorded coinbase, and canonical +bundle hash, then stores the logical body in PostgreSQL. Original files and +published hashes are retained. Canonical gzip sidecars are verified against +their uncompressed hash and stored byte-for-byte in the database. Content-addressed +HTTP responses preserve these exact bytes. This makes history available on every frontend +without a shared filesystem. + +`--root` resolves relative body paths and restricts imports to that directory; +it does **not** rewrite an absolute URI to a different root. Mount old absolute +paths at their recorded locations. Keep referenced segment paths readable as +well. Import failure is a reason to repair the missing/corrupt artifact before +continuing; a matching hash without recoverable bytes is insufficient. A legacy +body that fails only because a range digest was written as escaped JSON is +intact; [prove it and write its sidecar](#legacy-bodies-whose-range-digests-use-pythons-escaped-json-709). +So is a 2.x v2 body that fails with `window proof share_parts_digest_hex +mismatch`; [list those rows and write their sidecars](#2x-v2-bodies-whose-window-proof-parts-digest-3x-recomputes-differently-731). + +`backfill-ctv` scans verified stored bundles and restores missing fanout records. +Run import first. It is idempotent for matching existing records and replaces the +old Python repair command's individual-path/block selection interface. + +Neither `import-audits` nor `backfill-ctv` registers a frontend. Both pass the +frontend's startup gates (schema version, declared capabilities, migration +source and the halt refusal) and then write only the rows they repair, so a run +leaves no `qbit_prism_instances` heartbeat for fatal-state recovery to refuse +and never overwrites a running frontend's row. `self-check` and `broadcast-ctv` +behave the same way. + +### Legacy bodies whose range digests use Python's escaped JSON (#709) + +`1.x` and `2.x` hashed each `segment_range` and `segment_prefix` share part +over Python's `json.dumps`, which writes non-ASCII text as `\uXXXX` escapes; +3.x recomputes that digest over raw UTF-8. A range whose shares carry +non-ASCII text, usually a worker name in `miner_id`, therefore fails in 3.x +although its content is intact, and `import-audits` stops on the body with +`audit body ref hash mismatch`. When it stops this way, +`scripts/prism_legacy_range_sidecars.py` proves such bodies and writes the +canonical sidecars import reads before a body. For each row it inlines the +failing ranges from their slot files and runs the 3.x canonicalizer on the +result, which requires the canonical bundle to hash to the row's +`audit_bundle_sha256`. In a checkout of the release that runs import, build +the canonicalizer and export the rows import has not stored yet from the +database it ran against (`prism-restore` in a rehearsal). Run the tool once to +classify them and again with `--write`, then rerun import: + +```sh +cargo build --release --locked -p qbit-prism --bin qbit-prism-audit-canonicalize +PGSERVICE=prism-restore psql -Xq -v ON_ERROR_STOP=1 -c "COPY ( + SELECT block_hash, body_uri, audit_bundle_sha256 FROM qbit_pool_audit_bundles + WHERE canonical_audit_bytes IS NULL AND share_snapshot_sha256 IS NULL + AND audit_bundle IS NULL AND body_uri IS NOT NULL ORDER BY block_hash +) TO STDOUT WITH (FORMAT csv, HEADER)" > legacy-rows.csv +python3 scripts/prism_legacy_range_sidecars.py --rows legacy-rows.csv \ + --audit-root /var/lib/qbit-mining-pool/prism/audit \ + --canonicalizer target/release/qbit-prism-audit-canonicalize \ + --sidecar-dir /var/lib/qbit-prism/audit +python3 scripts/prism_legacy_range_sidecars.py --rows legacy-rows.csv \ + --audit-root /var/lib/qbit-mining-pool/prism/audit \ + --canonicalizer target/release/qbit-prism-audit-canonicalize \ + --sidecar-dir /var/lib/qbit-prism/audit --write +time qbit-prism-server import-audits --root /var/lib/qbit-prism/audit +``` + +- `--sidecar-dir` must be the directory passed to `import-audits --root`; + import looks for `prism-audit-bundle-canonical-<block>-<digest>.json.gz` + there, and resolves a relative body URI against it, as the tool does. + `--audit-root` is where the audit tree is mounted, and it replaces the + recorded `--uri-prefix`, by default `/var/lib/qbit-mining-pool/prism/audit`, + in body and slot URIs. Run the tool where the tree is at its recorded + paths, as above, so that only the bodies import refuses get sidecars. +- The export leaves out rows with an inline `audit_bundle`: import reads + those from the database, not from `body_uri`. Every exported row goes + through the canonicalizer, so when the affected heights are known, narrow + the export to them through `qbit_pool_blocks`. +- Each run reports every row in `legacy-range-sidecars.jsonl`, and the first + run writes nothing. A run exits 0 only when every row is `ok-in-3x` (it + verifies as it is), `proven` (`sidecar-written` with `--write`) or + `sidecar-ok` (an existing sidecar that holds the digest's bytes). Any other + status, such as a body the canonicalizer still refuses, exits 3: restore + that evidence instead. +- The tool never replaces a sidecar. An existing one that does not hold the + digest's bytes is reported as `sidecar-bad` and named on stderr; import + would stop on it too. A new sidecar is published by hard-linking a complete + file, so the sidecar directory's filesystem must support hard links; import + never sees a partial one. Import re-verifies every sidecar against the + digest, the recorded coinbase and the trusted ledger key. + +The post-GA follow-up is #709: 3.x accepting the escaped form, so that these +bodies verify without sidecars. + +### 2.x v2 bodies whose window-proof parts digest 3.x recomputes differently (#731) + +2.x wrote each audit-bundle.v2 body's `share_parts_digest_hex` over its share +parts in insertion order; 3.x recomputes it with sorted keys. 3.x therefore +refuses every such body, and `import-audits` stops on the first one with +`window proof share_parts_digest_hex mismatch`. The body is intact. Import +reads a canonical sidecar before the body, so only a v2 body without one in the +import root reaches the check. 2.x has written a sidecar beside each new v2 +body only since #173 reached a deployment. Union's bodies from before +2026-08-24, about 9,500 of them, have none. +`scripts/prism_legacy_parts_digest_check.py` lists those rows without +reading any slot, and `scripts/prism_legacy_range_sidecars_sharded.sh` writes +their sidecars with `scripts/prism_legacy_range_sidecars.py` in parallel +shards. With `legacy-rows.csv` exported and the canonicalizer built as in the +previous section: + +```sh +python3 scripts/prism_legacy_parts_digest_check.py --rows legacy-rows.csv \ + --audit-root /var/lib/qbit-mining-pool/prism/audit \ + --sidecar-dir /var/lib/qbit-prism/audit \ + --affected-csv refused-rows.csv --jobs 8 > parts-digest.jsonl +scripts/prism_legacy_range_sidecars_sharded.sh refused-rows.csv \ + /var/lib/qbit-mining-pool/prism/audit /var/lib/qbit-prism/audit \ + target/release/qbit-prism-audit-canonicalize scripts/prism_legacy_range_sidecars.py \ + legacy-sidecar-shards 8 +python3 scripts/prism_legacy_parts_digest_check.py --rows legacy-rows.csv \ + --audit-root /var/lib/qbit-mining-pool/prism/audit \ + --sidecar-dir /var/lib/qbit-prism/audit --jobs 8 > /dev/null +time qbit-prism-server import-audits --root /var/lib/qbit-prism/audit +``` + +- The checker reads each body once. A row whose sidecar is in `--sidecar-dir` + is `has-sidecar` and its body is not read. That status means only that the + file is there: import verifies the sidecar's bytes and stops on a bad one. + The sidecar tool checks an existing sidecar (`sidecar-ok` or `sidecar-bad`) + for every row it is given. `2x-order-only` and `matches-neither` rows are + the ones import refuses for this digest: `--affected-csv` lists them in the + tool's `--rows` format, and stderr counts them as `import_refuses`. +- The checker exits 0 only when no row is refused for this digest, no body is + missing or unreadable, and no v2 body is for another digest. Restore any + `body-missing`, `body-unreadable` or `body-digest-differs` evidence instead. + It checks nothing else: a body-ref or plain bundle is `no-window-proof`, and + rows that import refuses for other reasons, such as the escaped range + digests above, need the previous section's classification run. +- The runner splits the rows round robin into shards, each with its own CSV, + work directory and report, and runs the tool with `--write` on each. Its + last argument is the number of shards, 8 by default. The output directory + must be new or empty and outside the import root. The run exits 0 only when + every shard exits 0 and no temporary file is left in the import root. A + rerun verifies the sidecars already written (`sidecar-ok`) instead of + rewriting them. +- The bodies are immutable, so both steps can run ahead of the cutover + against a copy of the tree, writing into the directory import will read. + In the window, rerun them for the rows added since. + +The post-cutover follow-up is #731: 3.x recomputing the digest in 2.x's +order, so that these bodies verify without sidecars. + +### Migration 002's share-hash backfill, applied online + +Migration 002 creates `qbit_prism_share_hashes`, the header authority the +native append consults before it credits a share, and maps the header of every +accepted `2.x.x` share whose ID ends in 64 hex digits. Where legacy +worker-scoped IDs repeat a header, the earliest `share_seq` wins; case is +folded; rejected rows and IDs without a hex header are not mapped. Until #582 +the mapping was one `INSERT ... SELECT DISTINCT ON` inside the migration +transaction, under `PRISM_DATABASE_STATEMENT_TIMEOUT_MS`. At the default 15 s +it failed from about a million shares. Its cost per share rose with the ledger +(37 µs at 1.03M shares, 78 µs at 4.13M), so the production ledger's 65.5M +shares outgrew even the 600 s maximum. + +On a fresh deployment or an empty `2.x.x` source there is nothing to map, and +2 is recorded in the migration transaction. On a `2.x.x` source with rows, +002's file is still applied in the transaction, but 2 is not recorded there. +The transaction also creates the backfill's cursor, +`qbit_prism_share_hash_backfill`, one row covering every `share_seq` the ledger +holds: + +| Column | Meaning | +| --- | --- | +| `start_seq` | the ledger's first `share_seq` when the transaction committed | +| `next_seq` | the legacy shares from `start_seq` up to it are mapped | +| `end_seq` | the ledger held no row at or above it when the transaction committed | +| `started_at`, `updated_at` | when the cursor was created and last advanced | + +After the commit, and before 013, 017 and 024, `migrate` maps the legacy shares in +batches of consecutive `share_seq`. Each batch is 002's statement restricted to +its range: one statement, in a transaction that also advances the cursor. The +first batch covers 10,000 `share_seq`; later ones double or halve toward half a +second each, between 1,000 and 50,000. So every statement stays far inside the +server's statement timeout whatever the ledger's size, and no setting has to be +raised for `migrate`. The cap matters after a gap in `share_seq` or a run of +rejected shares, where batches are cheap and grow. A batch that outlasts the +timeout all the same, on a much slower disk, is rolled back and retried at +half its size; only an operator's cancel stops the run. The last transaction +waits for the migration lock and checks that the cursor has passed every row +the ledger holds. It then drops the cursor table and records 2. + +The mapping is the single statement's, row for row. Batches run in ascending +`share_seq`, so the earliest share of a repeated header is mapped by the batch +that holds it. A later copy, in a later batch, meets `ON CONFLICT DO NOTHING`. +The batches also skip the per-row foreign-key check the single statement paid: +016 has dropped that key by the time they run. + +**Readiness: the server does not serve during the backfill**, unless +`migrate --defer-share-hashes` has mapped its recent range and permitted +serving, as +[Serving with the backfill pending](#serving-with-the-backfill-pending-fence-2) +describes. Otherwise, until 2 is recorded, every start refuses the database. That is this release's start and +every earlier native build's, since each requires 2. The `2.x.x` writer is +fenced by 002's lease trigger, which committed with the transaction. So no share +can be appended, and none credited against a legacy header not yet mapped. The +backfill is part of the cutover's outage, as 013's builds and 017's validation +are on a populated ledger. A start, `self-check` and `import-audits` all refuse +with: + +``` +database is not ready: migration 2's share-hash backfill has not finished (#582). +The legacy shares from share_seq 1 up to 30001 are mapped in +qbit_prism_share_hashes and those from 30001 up to 4129153 are not, and until +every one is mapped a share could be credited twice. Run `qbit-prism-server +migrate` to resume it from there, or `qbit-prism-server migrate +--defer-share-hashes` to map only the legacy shares within 1000 template +heights of the highest one and serve while `qbit-prism-server +backfill-share-hashes` maps the rest; every start refuses the database until +one of them has finished +``` + +A backfill that a build before #669 started is refused the same way, but its +remedy names plain `migrate` alone: `--defer-share-hashes` refuses it. One +that a `--defer-share-hashes` run claimed and left before its recent range +was mapped is refused naming that run's retry first +([Serving with the backfill pending](#serving-with-the-backfill-pending-fence-2)). + +They refuse the same way while the cursor table exists, even if 2 has been +recorded by hand, and so does the recovery evidence export +(`scripts/prism-recovery-evidence.sql`). Once serving is permitted, at +`share_hash_backfill_pending = 2`, the export reads the database as every start +does, and marks the share hashes deferred +([Recovery evidence while the backfill is pending](#recovery-evidence-while-the-backfill-is-pending)). + +**Earlier builds are fenced off while the backfill is pending (#669).** A +build from before #582 knows nothing of the cursor. Its `migrate` refuses 3 +without 2 as an edited record, and its remedy is to record 2 once 002's +objects are present. On its own, that would let it serve with legacy headers +unmapped. So the database also declares the capability +`share_hash_backfill_pending = 1` while the backfill is pending: + +- the migration transaction that creates the cursor declares it, and nothing + declares it later; +- the transaction that records 2 removes it with the cursor. It is the one + capability native code removes, because it describes a state, not a format. + That transaction first reads the declaration again under the migration lock. + At a value other than 1 or 2, a newer release declared it while the runner + mapped between holds of that lock, and the runner stops: the cursor, the + declaration and the record are that release's to finish; +- between the two, `migrate --defer-share-hashes` raises it to 2 when it + permits serving, which the starts of rc.5 and later accept and every + earlier build refuses. + +An earlier build cannot remove the fence, so the fence must never coexist +with an earlier build's runner, and that is why it is declared only with a new +cursor. Migrations serialize on the migration lock, and no earlier build's +`migrate` passes its capability check once the fence has committed. So any +runner an earlier build started belongs to a cursor an earlier build created, +which carries no fence. Declared later, by a resume, a fence could meet such a +runner already past its check, still mapping or queued for the runners' lock. +That runner would record 2 and leave the fence behind. The price: a backfill +that a #582 build (before #669) started stays unfenced to its end. Resume it +with this release, and don't record 2 by hand. + +How earlier builds react to a fenced backfill, once each checks declared +capabilities at connect and at `migrate`: +- **Every start** of an earlier build refuses the capability, so recording 2 + by hand lets nothing serve. This release refuses the cursor itself, and so + does the evidence export. +- **A build from before #582** still answers its first `migrate` with the + 3-without-2 refusal and its remedy. Following that remedy changes nothing: + its next `migrate` refuses the capability. +- **A #582 build** refuses the capability at once. + +A declaration whose cursor is gone can only be left by dropping the cursor by +hand. Every `migrate` of this release refuses it, and so does every start once +the record is otherwise complete. Restore the full pre-migration backup. If +the mapping is verified complete instead, remove the declaration. This query +counts the legacy headers that are not mapped to their earliest accepted +share; the count must be 0: + +```sql +SELECT count(*) AS unmapped +FROM (SELECT DISTINCT ON (lower(right(share_id, 64))) + lower(right(share_id, 64)) AS header, share_id + FROM qbit_share_ledger + WHERE accepted AND share_id ~ '[0-9a-fA-F]{64}$' + ORDER BY lower(right(share_id, 64)), share_seq) expected +LEFT JOIN qbit_prism_share_hashes mapped + ON mapped.header_hash = expected.header AND mapped.share_id = expected.share_id +WHERE mapped.header_hash IS NULL; +``` + +**Progress.** No frontend runs while the backfill does, so there is no +Prometheus series for it. `migrate` logs the backfill's start, a progress line +every ten seconds (`next_seq`, `end_seq`, rows mapped, `share_seq` per second +and the estimated seconds left) and its completion. Any session can read the +cursor: + +```sql +SELECT start_seq, next_seq, end_seq, + round(100.0 * (next_seq - start_seq) / greatest(end_seq - start_seq, 1), 1) AS percent, + started_at, updated_at +FROM qbit_prism_share_hash_backfill; +``` + +**Interrupting and resuming.** Each batch commits with its cursor, so an +interrupted `migrate` loses only the batch in flight. That holds for a killed +process, a lost connection, and a cancelled or timed-out statement. Run +`migrate` again. The migration transaction finds 3 recorded without 2 and the +cursor table present, so it applies nothing, and the backfill resumes at +`next_seq`; 013, 017 and 024 follow. A record with 3 and not 2 but no cursor table +is still refused as an edited record. + +Resuming only goes forward. The migration transaction has committed, so the +database is native, and the `2.x.x` writer cannot return to it. To go back, +restore the pre-migration backup as in +[Recovery and rollback](#recovery-and-rollback). No native share can have been +acknowledged before 2 is recorded, so the whole backfill falls before the ACK +boundary. + +**Refusals, and what each one means.** + +| Refusal | What to do | +| --- | --- | +| `refusing to migrate before any DDL: a table named qbit_prism_share_hash_backfill already exists, and migration 2 creates its share-hash backfill's progress table under that name. Nothing was changed. Check what it holds, then rename or move it aside and migrate again` | a `2.x.x` source holds a relation under the cursor's name; check what it holds, rename or move it aside, and migrate again | +| `migration 2: backfilling qbit_prism_share_hashes for share_seq <a> to <b>; every earlier batch committed, so migrate again to resume from share_seq <a>` | the context on a failed batch statement: cancelled, timed out, or a lost connection. Migrate again | +| `refusing to continue migration 2: its share-hash backfill's cursor is no longer at share_seq <a>, where this run, which holds the runner lock, left it; migrate again` | something other than the runner moved the cursor. Migrate again, which resumes from where the cursor is | +| `a <kind> named qbit_prism_share_hash_backfill holds the name of migration 2's share-hash backfill progress table; check what it holds, then rename or move it aside` | on a native database, a relation that is not the cursor table holds its name; move it aside, then start or migrate again | +| `qbit_prism_share_hash_backfill has no row: migration 2 created it with its cursor and nothing deletes the row, so it was edited or restored selectively. Restore the full backup` | the cursor table was edited; restore the full backup | +| `database declares share_hash_backfill_pending = 1, but migration 2's share-hash backfill cursor qbit_prism_share_hash_backfill is gone. Only the transaction that records 2 removes them, and it removes both, so the cursor was dropped by hand and legacy shares may be unmapped (#669). Restore the full pre-migration backup and migrate again, or, once qbit_prism_share_hashes is verified to map every accepted share whose ID ends in 64 hex digits (...), remove the declaration with DELETE FROM qbit_prism_schema_capabilities WHERE capability='share_hash_backfill_pending' and migrate again` | from `migrate`, and from a start once every migration is recorded: the cursor was dropped by hand while the backfill was pending. Restore the full pre-migration backup, or verify the mapping with the query above and remove the declaration. At `= 2` frontends may have served since, so the message offers no backup: verify the mapping | +| `refusing to record migration 2: while its share-hash backfill ran, a newer PRISM release declared share_hash_backfill_pending = <n>, but this server understands share_hash_backfill_pending 1 to 2 only. That release finishes the backfill; upgrade the server before starting or migrating here` | a newer release's `migrate` ran while this release's runner was mapping. Upgrade the server and migrate with the newer release | +| `refusing to continue migration 2: its share-hash backfill's progress table is gone, but 2 is not recorded. Only the transaction that records 2 drops it, so it was dropped by hand. Restore the full backup` | the cursor table was dropped by hand during a run; restore the full backup | + +**Plan for the backfill on a large ledger.** The backfill writes every legacy +share's header once more. The mapping takes about 490 bytes per share with its +two indexes (4.0 GB at 8.1M shares), so budget about 32 GB for the production +ledger's 65.5M shares, plus the WAL that comes with it. + +Its rate depends more on PostgreSQL's memory and checkpoint settings than on +anything in `migrate`. Both indexes take random inserts. Once they no longer +fit in `shared_buffers`, frequent checkpoints make most inserts write a full +page image. These measurements are from a mainnet-shaped ledger of 8.25M +shares, on PostgreSQL 16 with `fsync` on, on a shared 16-core host: + +| Settings | Backfill | Rate | WAL | +| --- | --- | --- | --- | +| server defaults (128 MB `shared_buffers`, 1 GB `max_wal_size`) | 348 to 413 s | about 40,000 `share_seq`/s until the indexes outgrew the cache at about 5M shares, about 13,000/s after | 16 GB | +| `shared_buffers = 4GB`, `max_wal_size = 32GB`, `checkpoint_timeout = 30min` | 251 s | a steady 30,000 to 40,000/s | 5.8 GB | + +The batch sort is not the cost: 64 MB of `work_mem` changed nothing. + +For 65.5M shares, expect roughly 35 to 40 minutes with settings like the +second row, and 80 minutes or more at the defaults. So give the database the +larger settings for the cutover window: `shared_buffers` a quarter of memory, +`max_wal_size` in tens of gigabytes, a long `checkpoint_timeout`, and the disk +for the WAL. Restore them afterwards if they are not the server's normal ones. +`make prism-cutover-rehearsal` runs its private cluster at the defaults, so +the time it reports for the backfill is the pessimistic one. + +#### Serving with the backfill pending (fence 2) + +On a production-sized ledger the backfill is most of `migrate`'s time, as +**Plan for the backfill on a large ledger** above estimates. Run +`qbit-prism-server migrate --defer-share-hashes` to take it out of the +cutover's outage. The flag is `migrate`'s alone, never an environment setting: +compose starts frontends with `PRISM_POSTGRES_INIT_SCHEMA=1`, and they run the +same migration at their start. Plain `migrate` behaves exactly as above. + +**What the deferred run does.** + +1. The migration transaction creates the cursor and declares + `share_hash_backfill_pending = 1`, as plain `migrate`'s does. It also + claims the backfill for the deferral before anything is mapped: it sets the + cursor's `deferred_at`, a column the code adds. On a cursor that an earlier + run left at fence 1, the claim is set the same way, and a retry keeps the + first claim's time. +2. After the commit, in the slot the backfill takes otherwise, it maps the + *recent range*: every accepted legacy share whose template height is + within 1,000, the coinbase maturity, of the highest one. + - The range's lowest height is `H_min`, the highest template height of an + accepted legacy share less 1,000, and at least 0. + - Its first share is `S_min`, the first `share_seq` of an accepted share + at or above `H_min`. + - It runs 002's batches over `[S_min, end_seq)`, restricted to + `template_height >= H_min`, in ascending `share_seq`. 2.x's + `qbit_share_ledger_template_height_idx` serves the bounds, so the range + is mapped before 013 drops that index. +3. One transaction then records `recent_min_height` (`H_min`) and + `recent_start_seq` (`S_min`, NULL when the ledger has no accepted share) + beside the cursor, and raises the fence to + `share_hash_backfill_pending = 2`. It refuses if the ledger holds a row past + `end_seq`: nothing should have appended. + - The same transaction keeps every native share at or above `end_seq`. + Native shares draw their `share_seq` from the sequence that + `qbit_share_ledger.share_seq` owns, found by that column. If the + sequence's next value would be below `end_seq`, it is moved to hand out + `end_seq` next (`setval(<sequence>, end_seq - 1, true)`), and an INFO + line names its old and new `last_value` and `is_called`. A sequence can + lag behind the rows after explicit inserts, or after a restore that left + it uncalled. A sequence that does not lag is not written to, so its + `last_value` and `is_called` stay exactly as the source had them. A + promoted physical copy is ahead already. +4. 013, 017 and 024 follow as usual, with the cursor present. With + `--offline-indexes` beside the flag, as a cutover's migrate step can run + them, 013 and 024 build offline + ([migration 013](#migration-013-the-share-ledger-index-trim-applied-online)). +5. `migrate` returns and says that 2 is deferred, naming `next_seq` and + `end_seq`. The cursor has not moved, and 2 is not recorded. + +The range is about 40,000 to 60,000 shares at the ledger's rate, so this takes +seconds. A rerun of `--defer-share-hashes` after an interruption maps the +range again, or, once the fence is 2, applies only what else is pending. + +**A deferred run that stops before serving is permitted** leaves the cursor +at fence 1 with its claim. From there, only the operator's `migrate` maps +anything: + +- `qbit-prism-server migrate --defer-share-hashes` again maps the range and + permits serving, as a first run does; +- plain `qbit-prism-server migrate`, chosen explicitly, maps the whole backfill + and records 2 before anything serves, as without the flag. + +Every other connect that migrates refuses the database, as every start does, +and changes nothing. That includes a frontend that compose starts with +`PRISM_POSTGRES_INIT_SCHEMA=1`, a tool and an operator command. Such a +connect runs neither the whole backfill, which would take hours while the +operator's retry waited for the runners' lock, nor 013, which would drop the +index that the range reads. A connect that planned the backfill before the +claim, and then waited for the runners' lock, reads the claim again once it +holds the lock, and refuses the same way. Without a claim, a backfill at +fence 1 is run to its end by every connect, as before. + +**Why serving is safe with the rest unmapped.** A native share is credited +only on a job this cluster issued, whose parent is the current tip or, within +stale grace, the tip's parent. Its ID ends in the block hash, which commits to +that parent. Every copy of a legacy header has the same parent, so the same +template height. A credited native share can therefore repeat only a legacy +header whose parent can be the tip at or after the cutover: one in the recent +range, unless the chain reorganized more than 1,000 blocks below the legacy +tip, which coinbase maturity does not survive either. Every such header is +mapped before anything serves, so the append's header check refuses a repeat +exactly as it would with the whole mapping. Each header in the range maps to +its earliest copy: the range holds all of a header's copies. Nothing but that +check reads `qbit_prism_share_hashes` at runtime; the window, balances, +settlement and the broadcaster never do. + +**Which starts are allowed.** + +| The database holds | A start, `self-check`, `import-audits` and operator commands | +| --- | --- | +| the cursor and `share_hash_backfill_pending = 1` | refused, naming the cursor and both remedies | +| the cursor, `share_hash_backfill_pending = 1` and a deferred run's claim | refused, naming the deferred run that stopped and its retry, `migrate --defer-share-hashes`; a connect that migrates maps nothing | +| the cursor and no fence (a backfill a build before #669 started) | refused; `--defer-share-hashes` refuses it too, and plain `migrate` finishes it | +| the cursor and `share_hash_backfill_pending = 2` | allowed without 2 recorded. Each start logs a warning naming `next_seq` and `end_seq` | +| the fence at 1 or 2 without the cursor | refused: the cursor was dropped by hand | + +No build before rc.5 serves beside the pending backfill or finishes it. An +earlier release candidate, rc.1 to rc.4, meets the cursor first at a start +and refuses with the backfill's own message, `database is not ready: +migration 2's share-hash backfill has not finished (#582) …`. That message's +advice to run `migrate` holds only for rc.5 and later: the earlier build's +`migrate`, and its start with `PRISM_POSTGRES_INIT_SCHEMA=1`, refuse the fence +before any DDL (`… understands share_hash_backfill_pending 1 to 1 only …`). +Finish the backfill with rc.5 or later, or go back to the pre-migrate backup. + +**Who finishes it: `backfill-share-hashes`, or plain `migrate`.** A frontend +started with `PRISM_POSTGRES_INIT_SCHEMA=1`, a tool or an operator command +migrates at its start, but leaves a backfill at fence 2 as it is. Once +frontends serve, run `qbit-prism-server backfill-share-hashes` +([below](#finishing-the-backfill-while-frontends-serve-backfill-share-hashes)), +whose throttle the operator sets. Plain `qbit-prism-server migrate`, without +the flag, runs the same backfill at the default throttle, after whatever else +is pending. Either: + +- maps `[next_seq, end_seq)` while frontends serve, in batches that keep to a + throttle: each batch is one statement of at most 5,000 `share_seq` in a + transaction of its own, cancelled after 2 s and then tried again at half its + size, and the run rests between batches as long as each took. No statement + holds a snapshot on the primary longer than the statement timeout (#738). + rc.4's batches, up to 50,000 `share_seq` under the pool's 15 s, back to back, + run only before anything serves. A native share whose header a batch in + flight is mapping, which the argument above rules out, waits for that batch + and then fails on the header's uniqueness, uncredited + (`ledger-confirmation-failed`); a resubmission is refused as already + credited; +- checks, before it maps anything, that recording 2 at fence 2 will find 017's + conversion bound. A plain `migrate` that planned the backfill at fence 1, + ahead of 017, can find the fence at 2 once it holds the runners' lock: a + deferred run raised it while this one waited. If 017 has not recorded the + bound, the run refuses at once, rather than at the record after hours of + batches; rerun `migrate`, which then applies 017 first; +- **never extends the end.** Every row at or above the cursor's `end_seq` is a + native share, which mapped its own header in the transaction that appended + it. The end is never read again from `max(share_seq)`. Under the migration + lock, the record step checks that `end_seq` is at or below + `qbit_prism_share_partitioning.conversion_bound`, under which the release + table holds the legacy shares. Native shares go into that table too, until + the sequence passes the bound; +- **refuses a double credit before it records 2.** It looks for a native share + that repeats the header of an accepted legacy share below `end_seq`. Such a + share was credited while its header was not yet mapped, and the batch that + reached the legacy copy met the native mapping and said nothing. The run + then refuses, naming the share, and leaves the cursor, the fence and the + missing record of 2 as they are. The mapping is complete by then, so no + later share can repeat a legacy header, and frontends keep serving. The + check probes 2.x's `qbit_share_ledger_accepted_block_suffix_idx` for each + native share, and the share ID index for a bare 64-digit legacy ID in lower + or upper case; a bare ID in mixed case is the one form it does not find. + It reads the native shares from `end_seq` up to the highest `share_seq` at + its start, before the migration lock, in chunks of consecutive `share_seq` + that keep to the same throttle as the batches: each chunk one short + statement in a transaction of its own under the throttle's statement + timeout, read again at half its size if it outlasts it, with the same rests + between chunks. Shares appended after the check starts need no reading: + every legacy header is mapped by then, so the append refuses a repeat as + already credited; +- drops the cursor and the fence and records 2, in one short transaction. + While frontends serve it waits at most 2 s for the migration lock, which + every migrating start takes, and for the cursor table's lock, which every + reader of the cursor holds briefly. An attempt that waits longer is rolled + back, holding nothing, and tried again after a backoff, from 2 s doubling + to 30 s, 20 times, about ten minutes in all; the run then stops with 2 + unrecorded, and a rerun goes straight to the double-credit check and the + record. After any failed attempt the run first reads whether the attempt + recorded 2 all the same, its COMMIT's reply lost: with 2 recorded and the + cursor gone the backfill is done, and the run says so and succeeds. It + reads on the attempt's own connection, unless that connection is lost: + dropped or reset, ended by the server (a restart, a crash, a failover, an + idle timeout), or silent past the attempt's own deadline, its statements' + timeouts and a few seconds, as a partition leaves it. Then it reads on a + fresh connection from the same URL under the same short timeouts, + connecting again after a backoff from 250 ms while the server turns it + away, and closes that connection again. A lost connection that showed 2 + unrecorded stops the run instead of a retry: the runners' lock, which the + run holds on its connection, went with it, and a rerun takes it again and + goes straight to the double-credit check and the record. + +An interrupted run resumes at `next_seq`. `migrate --defer-share-hashes` on a +backfill at fence 2 leaves it as it is. + +**Share archive is refused while pending.** `share-archive restore` refuses +while the cursor exists, at any fence: an attached restore maps the headers of +the rows it restores. So do `share-archive detach` and `drop` of every +partition, whatever its bounds. The backfill reads the legacy shares below +`end_seq` from the attached ledger, and before it records 2 the double-credit +check reads every native share from `end_seq` up through the attached ledger +too: a native partition that left the ledger would hide its shares from that +check. Each refusal names `backfill-share-hashes` as the remedy, and plain +`migrate`, which finishes it as well; archive the partition once 2 is +recorded. `share-archive plan`, `seal`, `archive` and `verify` are unaffected. + +**Progress.** The cursor query above reads the same while the rest is pending; +`recent_min_height` and `recent_start_seq` say what the recent range covered, +and `deferred_at` when a deferred run first claimed the backfill. `self-check` +reports the pending backfill in its `share_hash_backfill` field, and the run +logs its progress +([below](#finishing-the-backfill-while-frontends-serve-backfill-share-hashes)). + +**Refusals.** + +| Refusal | What to do | +| --- | --- | +| `refusing to defer migration 2's share-hash backfill: a build before #669 started it, so the database declares no share_hash_backfill_pending fence, ...` | the backfill carries no fence, and a resume never declares one. Run plain `migrate`, which finishes it before anything serves | +| `refusing to permit serving with migration 2's share-hash backfill pending: the share ledger's end is share_seq <a>, not the end <b> the backfill was planned to, ...` | a writer appended around every gate before serving was permitted. Nothing serves. Run plain `migrate`, which maps every row and records 2 | +| `database is not ready: migration 2's share-hash backfill has not finished (#582). ...`, saying that `qbit-prism-server migrate --defer-share-hashes` claimed it and stopped before its recent range was mapped | a deferred run stopped before it permitted serving; every start and every other connect that migrates refuses until the operator's `migrate` has run. Run `migrate --defer-share-hashes` again, or plain `migrate` to map everything before anything serves | +| `refusing to permit serving with migration 2's share-hash backfill pending: the share_seq sequence <s> hands out <n> next, below the backfill's end <e>, ...` | moving the sequence up to the end did not hold. Nothing serves. Run `migrate --defer-share-hashes` again | +| `refusing to permit serving with migration 2's share-hash backfill pending: no sequence is owned by qbit_share_ledger.share_seq, ...` | the ledger's `share_seq` column lost its sequence. Nothing serves. Run plain `migrate`, which maps every row and records 2 | +| `refusing to finish migration 2's share-hash backfill before mapping anything: it permits serving now, ... The share-hash fence changed while this run waited for the runner lock. ...` | a deferred run raised the fence to 2 while this run waited, and stopped before 017. Nothing was mapped. Rerun `migrate`, which applies 017 before it finishes the backfill | +| `migration 2: backfilling qbit_prism_share_hashes for share_seq <a> to <b> while frontends serve; every earlier batch committed. Resume from share_seq <a> with qbit-prism-server backfill-share-hashes` | a throttled batch failed, from `backfill-share-hashes` or plain `migrate`: a lost connection or a cancel, or, with `even a batch of <n> share_seq, the smallest, outlasted its <ms> ms statement timeout`, a batch that kept timing out down to the smallest size. Run `backfill-share-hashes`, for the latter with a smaller `--max-batch`, or else a larger `--statement-timeout-ms`, at most 5000 (the snapshot rule below); `migrate` takes no throttle flags | +| `refusing to wait any longer to record migration 2: something held MIGRATION_LOCK or the share-hash cursor through <n> attempts over about <t>. ...` | every batch is done and no native share repeats a legacy header, but something held the migration lock, or a transaction that read the cursor stayed open, through every attempt. 2 is not recorded and frontends keep serving. Find the holder in `pg_locks`, then run `backfill-share-hashes` again, which goes straight to the check and the record | +| `refusing to try recording migration 2 again: an attempt failed (...), and so did reading whether it had recorded 2 all the same. ...` or `... an attempt failed (...)[; the connection was lost while checking whether it had recorded 2 (...)], and reading whether it had recorded 2 on a fresh connection failed too. ...` | an attempt failed, and the database could not be read, on a fresh connection either when the attempt's own one was lost, before the run could tell whether the attempt committed. Run `backfill-share-hashes` again: it finds 2 recorded and says so, or goes straight to the check and the record | +| `refusing to try recording migration 2 again: an attempt lost its connection, and on a fresh one 2 was not recorded when checked. ...` or `... an attempt failed, and the connection was lost while checking whether it had recorded 2 (...); on a fresh one 2 was not recorded when checked. ...` | the attempt's connection was lost, by the attempt or by the read after it, and the runners' lock went with it. 2 was not recorded when the run checked, and frontends keep serving. A COMMIT still in flight on the lost connection, waiting for a synchronous standby say, can land moments later. Run `backfill-share-hashes` again: it takes the lock, then finds 2 recorded and says so, or goes straight to the check and the record | +| `refusing to record migration 2: after an attempt to record it failed, 2 is recorded, but its share-hash backfill's cursor ... still exists` or `... the cursor ... is gone, but 2 is not recorded` | only the transaction that records 2 drops the cursor, in the same commit, so the migration history or the cursor was edited while the run recorded 2. Find out which before changing anything; for a cursor dropped by hand, restore the full backup | +| `migration 2: finding the last native share, ...` or `migration 2: checking the native shares from share_seq <a> to <b> for a header an accepted legacy share holds, before recording 2. ...` | a statement of the double-credit check failed, a chunk at its smallest size if it timed out. Every batch committed and 2 is not recorded. Run `backfill-share-hashes` again, which goes straight to the check | +| `refusing to record migration 2: native share <id>, at share_seq <n>, repeats header <h> of an accepted legacy share below share_seq <end>, so that header was credited twice. ...` | the safety argument above failed for that header. Frontends may keep serving. Report it, and reconcile the double credit before anything records 2; never record 2 by hand over it | +| `refusing to record migration 2: its share-hash backfill's end, share_seq <a>, lies above the conversion bound <b> ...` | the cursor or the partitioning catalog was edited. Restore the full backup | +| `refusing to restore a share archive while migration 2's share-hash backfill is pending: ...`, `refusing to detach <partition> while migration 2's share-hash backfill is pending: ...`, `refusing to drop <partition> ...` | finish the backfill with `backfill-share-hashes` or plain `migrate`, then the archive step | + +#### Finishing the backfill while frontends serve: `backfill-share-hashes` + +`qbit-prism-server backfill-share-hashes` maps the rest of a backfill that +`migrate --defer-share-hashes` left serving, and records 2. Run it after +go-live, once frontends serve and `self-check` passes. It needs only +`PRISM_DATABASE_URL`, connects as the operator does, passes the start gate as +every start does, and registers no instance. Until it has recorded 2, +`share-archive restore`, `detach` and `drop` refuse, whatever the partition, +so archiving waits for it. On a production ledger it runs for hours, so run it +in a session that survives a disconnect, with `RUST_LOG=info` for its progress +lines: + +```sh +RUST_LOG=info qbit-prism-server backfill-share-hashes 2>backfill-share-hashes.log +``` + +It prints what it did once 2 is recorded, as one JSON object: the rows it +`mapped`, the `share_seq` values `[next_seq, end_seq)` it covered and their +count `seqs`, `elapsed_ms`, `"recorded": true` and `"already_complete": +false`. On a database whose backfill finished already there is nothing to +map, and it succeeds all the same, with `"already_complete": true`, +`"mapped": 0` and no range, so a retry after a lost reply is safe. + +**The snapshot rule (#738).** Every native share append updates the +`qbit_prism_cluster` row inside `ORDER_LOCK`. A snapshot held anywhere on the +primary keeps that row's dead versions, so each append's update costs more the +longer it is held: at about 400 shares a second `ORDER_LOCK` saturates after +18 to 20 seconds of held horizon, and at about 130 a second it takes about five +minutes. So every statement the backfill issues on the primary is short, each +in a transaction of its own, never one long snapshot: a batch statement is +cancelled at its statement timeout, 2 s by default, and the run holds no +transaction between batches. The double-credit check reads the native shares +the same way, and so does the transaction that records 2. The statement +timeout is at most 5 s for that reason; the default is 2 s. + +**The throttle.** + +| Flag | Default | What it bounds | +| --- | --- | --- | +| `--max-batch` | 5000 | the `share_seq` values one batch maps, at most as many legacy shares: about half a second at the 83 to 94 µs a row measured on a mainnet-shaped ledger. 1 to 50,000. Batches start at the smaller of 10,000 and this, and grow or shrink toward half a second within it, as rc.4's do within 1,000 to 50,000 | +| `--statement-timeout-ms` | 2000 | each batch statement, each statement of the double-credit check and each of the record's, set for its own transaction (`SET LOCAL`). A batch that outlasts it is rolled back and tried again at half its size, down to the smaller of 1,000 and `--max-batch`, where a timeout stops the run instead. 1 to 5,000: no statement may hold a snapshot on the serving primary longer (#738) | +| `--duty-cycle` | 0.5 | the share of the time batches take: after a batch that took `t`, the run rests `t * (1 - d) / d` outside any transaction, so at 0.5 as long as the batch took, at 0.25 three times as long. 0.01 to 1 | + +Each is checked before anything connects. A lower duty cycle spares the +primary and takes longer in proportion. A larger batch makes fewer +transactions, not shorter snapshots. Plain `migrate` runs the defaults. At +the defaults and that per-row cost, a batch takes about half a second and so +does its rest, about 5,500 `share_seq` a second: three to four hours for a +65.9M-share ledger, an estimate the pair rehearsal is to measure beside live +load. + +**What it does, and what it refuses.** + +1. It refuses, before it maps anything, a database whose backfill does not + permit serving. The start gate refuses one at fence 1, naming + `migrate --defer-share-hashes` and plain `migrate`, and one a build before + #669 started, naming plain `migrate`. A database whose backfill has finished + has nothing to map: the command says so and succeeds. +2. It takes the runners' lock, as every online migration does, and holds it to + its end. A `migrate` or a second `backfill-share-hashes` started meanwhile + waits for it. Under the lock it reads the fence again and refuses any value + but 2, and checks 017's conversion bound, as plain `migrate` does. +3. It maps `[next_seq, end_seq)` in throttled batches. Each batch commits with + the cursor, so an interrupted run, a killed process or a lost connection + loses only the batch in flight; run it again to resume at `next_seq`. +4. It checks every native share for a double credit, throttled, and records 2 + in one short transaction, retried behind a held lock for about ten minutes, + as above. + +**Watching it.** + +- Its log: `share-hash backfill batches throttled` once, naming the throttle, + then `share-hash backfill progress` every ten seconds, with `next_seq`, + `end_seq`, the rows `mapped`, the `percent` of the range this run started + with, the current `batch_seqs`, `seqs_per_second` over the whole run, rests + included, and the estimated seconds left, `remaining_s`. A batch that outlasted its timeout logs a + warning naming its retry size. The double-credit check logs its own progress + and its end, then `share-hash backfill complete; migration 2 recorded`. +- Any session can read the cursor with the query above: a one-row read. + `updated_at` moves with every batch. +- `self-check` reports `share_hash_backfill` with `"state": "pending"`, the + fence, `start_seq`, `next_seq`, `end_seq`, `remaining_seqs`, the recent + range and `updated_at`, and a warning line; neither fails the check. Both + are gone once 2 is recorded. A backfill self-check cannot read is reported + as `"state": "unknown"` with the error, and a warning, never as done. +- A record attempt that waited out its lock timeout logs a warning naming the + attempt and its backoff. +- On the primary, the run's backend holds no transaction longer than a few + seconds: in `pg_stat_activity` its `xact_start` stays within a few seconds + of now, and its `backend_xmin` moves on with every batch, the record's + attempts included, which wait at most 2 s for a lock. If + `qbit_prism_database_order_lock_hold_seconds{holder="append"}` or the share + acknowledgement percentiles from `qbit_prism_share_ack_seconds` rise while it + runs, stop it (Ctrl-C, or SIGTERM) and run it again later, or with a lower + `--duty-cycle`. Stopping loses only the batch in flight. + +**Interrupting and resuming.** Stopping it is always safe, and so is running +it again. A run that stops in the double-credit check finds every batch done +and goes straight to the check. + +#### Recovery evidence while the backfill is pending + +At `share_hash_backfill_pending = 2` the recovery evidence export +(`scripts/prism-recovery-evidence.sql`, and the parallel export) reads the +database, as every start does. The mapping is partial, so it exports no +`share_hashes` row. It exports one `share_hashes_deferred` record instead, +the cursor: `start_seq`, `next_seq`, `end_seq`, `recent_min_height` and +`recent_start_seq`. It reads the cursor last, after the integrity report, in +the export's snapshot, and no other part of the export reads the cursor +table. So the export holds that table's lock only until it commits, right +after, and the transaction that records 2, which waits at most 2 s an attempt +for it, is never held up for long. If 2 was recorded after the export's +snapshot, the cursor is gone by the time it is read, and the export fails: +export again. The summarizer marks the kind deferred: the summary's +`records.share_hashes` has count 0 and a `deferred` object holding the cursor, +and a `note: share_hashes deferred` on stderr says so. Every other record is +summarized as before. At fence 1 the export refuses the cursor, as startup +does, naming `migrate --defer-share-hashes` to permit serving and plain +`migrate` to finish the backfill. A cursor that a build before #669 started +declares no fence, and the export refuses it naming plain `migrate` alone. + +So right after `migrate --defer-share-hashes`, before anything serves, the +migrated summary equals the source's but for that kind. `cmp` fails on it, and +the comparison leaves it out: + +```sh +diff <(jq -S 'del(.records.share_hashes)' source.summary.json) \ + <(jq -S 'del(.records.share_hashes)' migrated.summary.json) +``` + +Compare the share hashes once 2 is recorded. Before frontends serve, the +summary is then the source's whole. After they have served, the mapping also +holds the native shares' headers, so check the legacy part instead. The query +counts the legacy headers, below the cursor's `end_seq` from the deferred +summary, that are not mapped to their earliest accepted legacy share; the count +must be 0: + +```sql +SELECT count(*) AS unmapped +FROM (SELECT DISTINCT ON (lower(right(share_id, 64))) + lower(right(share_id, 64)) AS header, share_id + FROM qbit_share_ledger + WHERE accepted AND share_id ~ '[0-9a-fA-F]{64}$' AND share_seq < :end_seq + ORDER BY lower(right(share_id, 64)), share_seq) expected +LEFT JOIN qbit_prism_share_hashes mapped + ON mapped.header_hash = expected.header AND mapped.share_id = expected.share_id +WHERE mapped.header_hash IS NULL; +``` + +It reads the whole legacy ledger in one snapshot, so with miners live run it on +a standby with `hot_standby_feedback` off and replay paused, never on the +primary (#738). + +### Migration 013: the share ledger index trim, applied online + +Migration 013 (#153) trims the secondary indexes of `qbit_share_ledger` to +what the native readers use. It replaces +`qbit_share_ledger_accepted_seq_window_idx`, a `share_seq DESC` index carrying +seven INCLUDE columns that no native read ever fetched from it, with +`qbit_share_ledger_accepted_seq_walk_idx`, whose INCLUDE is exactly what the +newest-first page walk and the landing count read; replaces +`qbit_share_ledger_accepted_miner_recent_idx` with +`qbit_share_ledger_accepted_miner_history_idx`, without the unread +`payout_order_key`; and drops `qbit_share_ledger_accepted_window_idx` and +`qbit_share_ledger_template_height_idx`, which no native query uses. The +[index inventory](prism-ledger-ops.md#share-ledger-indexes) maps every index +to its readers and lists the measurements to take afterwards. + +On a fresh deployment or an empty 2.x.x source it is applied inside the +migration transaction while the cutover locks exclude writers. Existing +native ledgers always apply it outside that transaction, even when no +shares are visible: the first share may still be uncommitted, and native +writers do not take the migration lock. Populated 2.x.x sources also use +this online path. `CREATE INDEX CONCURRENTLY` cannot run in a transaction +block, and a plain `CREATE INDEX` holds a SHARE lock on the ledger for the +whole build, so every append would queue behind it. Inside the migration +transaction the file is then applied only to the scratch schema, so the +migrator learns the index definitions as this PostgreSQL renders them. After +the commit, on a dedicated connection with no statement or lock timeout and +under a session-level advisory lock keyed by the ledger's schema, `migrate` +(or a start with `PRISM_POSTGRES_INIT_SCHEMA=1`) builds each new index with +`CREATE INDEX CONCURRENTLY`, drops each replaced one with `DROP INDEX +CONCURRENTLY`, and records 13 last. Appends continue throughout. A +concurrent build waits for the transactions that were already using the +table, so a long payout-window read delays it without blocking anything +else; two frontends starting together build once, the second waiting on the +lock and finding the version recorded. + +Until 13 is recorded, every start refuses the database, naming it, like any +other required migration. That is the resumable state: an interrupted build +leaves an invalid index behind, still maintained by every insert, and the +next `migrate` drops it and builds again; an index built before the +interruption is kept when it is valid and matches the declared definition. A +valid index under one of the two new names with any other definition, or a +relation of another kind under one of the six names, is refused by name and +left alone, as with every native collision: + +``` +refusing to apply migration 13: index qbit_share_ledger_accepted_seq_walk_idx on qbit_share_ledger +already exists with a different definition (CREATE INDEX qbit_share_ledger_accepted_seq_walk_idx ON +qbit_share_ledger USING btree (share_seq)); the migration declares CREATE INDEX +qbit_share_ledger_accepted_seq_walk_idx ON qbit_share_ledger USING btree (share_seq DESC) INCLUDE +(job_issued_at, accepted_at, share_difficulty) WHERE accepted. The migration is not recorded and +nothing was changed by it. Check what that index serves, then rename or drop it and migrate again +``` + +Plan for the build on a large ledger. Both replacements are built before +anything is dropped, so the transient footprint is the two new indexes on +top of the existing ones; on the production ledger measured for #144 the two +replaced indexes held 44 GB, and the replacements are a fraction of that, but +keep that headroom free. A concurrent build reads the table twice. Run +`migrate` from a shell where a multi-hour command is acceptable, rather than +relying on a starting frontend, whose readiness stays down until the build +completes. Then run `ANALYZE qbit_share_ledger` and take the measurements the +inventory lists before scheduling #144. + +When nothing else runs against the database, as in the cutover's migrate step +or on an isolated restore, `qbit-prism-server migrate --offline-indexes` builds +013's indexes, and 024's, with a plain `CREATE INDEX` instead. It runs the same +plan in one transaction per migration. Before any DDL it takes the migration +lock and then, waiting at most 5 s for any other lock, refuses any instance +that has not reported `drained` or `stopped` and a live legacy writer lease, +and locks the table `ACCESS EXCLUSIVE`. Then it builds each index from its +definition as this PostgreSQL renders it, drops the replaced ones with a plain +`DROP INDEX`, checks the declared set and records 13 as it commits. A plain +build reads the table once, may use parallel workers and waits for no other +transaction. Every append and read of the table waits for the whole run, so +start frontends only after `migrate` returns: one that starts during the run +fails its startup. A transaction still open on the table, a missed writer or a +running export, makes the migration fail at the lock timeout before it changes +anything. An interrupted migration rolls back whole, leaving no invalid index +and nothing dropped; the migrations the run recorded before it stay recorded. +Plain `migrate` then builds concurrently and records without stopping +frontends, which is also the way to finish a migration whose indexes an earlier +run left built but unrecorded while frontends serve. The flag is `migrate`'s +alone: a frontend started with `PRISM_POSTGRES_INIT_SCHEMA=1` always builds +concurrently. Nothing after 013 depends on how its indexes were built: 017's +conversion looks only for each index on the table, and PostgreSQL renders a +plain build's definition exactly as a concurrent one's. + +`--index-build-workers` (default 4) caps the parallel workers of each build. +They come out of the server's `max_worker_processes` and +`max_parallel_workers`. A build's sort takes about `maintenance_work_mem` in +all: PostgreSQL treats it as the limit of the whole build and divides it among +the leader and the workers, at least 32 MB each, or launches fewer workers. +The run sets it for its transaction to `--index-build-memory`, a size as +PostgreSQL writes it (`512MB`, `2GB`), or without that flag to 2GB unless the +server's setting is higher. Size it to the memory the database host has free +beside the server's own. Without free slots or memory a build uses fewer +workers, or none, which is slower but no less correct. Before it builds, the +run logs the workers it asked for and those settings, and each build's +`index built` line gives its time. + +### Migration 016: the partition catalog and solver attribution + +Migration 016 (#144) is everything the partition conversion needs before the +table changes shape, and it is transactional: 017's SQL is the functions 016 +installs, so 016 must have committed before the online runner starts. It does +four things. + +It drops the two foreign keys onto `qbit_share_ledger(share_id)`, the 001 +outbox key and the 002 `qbit_prism_share_hashes` key. A unique index on a +partitioned table must include the partition key, so the global +`UNIQUE (share_id)` cannot survive 017 and nothing can reference `share_id` +afterwards. Both keys would also pin partitions: PostgreSQL re-validates +inbound foreign keys at DETACH, and the outbox is never pruned, so a single +retained outbox row would pin the partition holding its block's solving share +forever. The outbox keeps its own `UNIQUE (share_id)`, and +`qbit_prism_share_hashes` keeps its `UNIQUE (share_id)` and header-hash primary +key and becomes the global uniqueness authority the append path consults first. +The schema rule from here on is that **no object may carry a foreign key to +`qbit_share_ledger`**. + +It creates the partition catalog, `qbit_prism_share_partitioning` (the one +settings row: grid width, lead count, the conversion bound and when the swap +completed) and `qbit_prism_share_partitions` (per partition: bounds, state, +and the sealed, archived, verified, detached and dropped timestamps with the +archive URI and digests). Both are documented column by column in +[the ledger operations guide](prism-ledger-ops.md#share-ledger-partitions-and-retention). + +It creates the maintenance and conversion functions: +`qbit_prism_share_next_seq()`, `qbit_prism_share_probe_floor()`, +`qbit_prism_share_partition_create()`, +`qbit_prism_share_partition_ensure()`, and the three steps of the conversion, +`qbit_prism_share_ledger_convert_prepare()`, +`qbit_prism_share_ledger_convert_validate()` and +`qbit_prism_share_ledger_convert_swap()`. Defining them here is what lets the +online runner and the transactional empty-ledger path run the same code. + +Finally it moves block solver attribution onto `qbit_pool_blocks`. Four +dashboard queries found each block's solving share by suffix-matching every +block's hash against the ledger on every request, and after a detach that +lookup would silently blank a historical block's solver. The columns +(`solver_miner_id`, `solver_share_id`, `solver_share_difficulty`, +`solver_network_difficulty`) are added by 016, written at landing from now on, +and read by the queries; for the blocks found before 016, the migrator fills +them right after 016's file, in the same transaction. The backfill is one +`accepted_block_suffix_idx` probe per block without a solver recorded, exactly +the lookup those queries performed on every request. The number of probes +follows the pool's block count, not the share count, but each probe is two +random reads (an index leaf and a heap page). On a cold cache their cost grows +with the ledger. + +Until #672 the probes were one statement in 016's file, under the migration +transaction's statement timeout. On a mainnet-shaped ledger with all 13,566 +blocks, read from a fresh restore on a quiet disk, that statement took: + +| Ledger | Warm cache | Cold cache | +| --- | --- | --- | +| 8.25M shares | 3.0 s | 6.7 s | +| 16.5M shares | 7.4 s | 11.2 s | + +On a busy disk it took about 30 s, and the 15 s default refused it. The +migrator now makes the same probes right after the file, in the same +transaction, in statements of at most 128 blocks taken in `block_hash` order +(`ledger/migration/block_solvers.rs`). The attribution and the locks are the +single statement's. On the same ledgers the longest of those 107 statements +took 71 to 592 ms. With a concurrent writer saturating the disk, the single +statement took 14.0 s at 8.25M, one second short of the default, and the +longest batched one took 0.47 s. No statement comes near the timeout whatever +the cache, so `migrate` needs no statement timeout above the default for 016. + +### Migration 017: the share ledger partition conversion, applied online + +Migration 017 (#144) converts `qbit_share_ledger` into a partitioned table, +`RANGE (share_seq)`, with the release table attached as its first partition. +Nothing is copied and no index is rebuilt: with a validated +`CHECK (share_seq < bound)` on the release table, `ATTACH PARTITION` proves the +bound from the constraint and adopts every existing index, so the swap is +catalog work and takes milliseconds whatever the table holds. The layout it +produces, the retention path it enables and the per-partition index set are in +[the ledger operations guide](prism-ledger-ops.md#share-ledger-partitions-and-retention); +the reasoning is in +[the design record](prism-share-ledger-partitioning.md). + +On a fresh deployment or an empty `2.x.x` source the migration file is applied +as written, inside the migration transaction, under the cutover locks that +exclude writers; a fresh deployment's first partition is one grid cell. +Everywhere else, including an existing native ledger whose first share is still +uncommitted, the same three steps run after the commit, each in its own +transaction on a dedicated connection with no statement timeout, resumable from +whatever stage the database is in, and 16 is recorded last. Until then every +start refuses the database, as for every other required migration. The +migration transaction cannot do this work: its statement timeout and its lock +hold would not survive a validation scan that runs for hours, and the swap's +lock has to be taken with a short timeout and retried rather than queued. + +**1. Prepare.** `CHECK (share_seq < bound) NOT VALID` on the release table, +where `bound` is the first grid boundary at least two partition widths above +the next `share_seq`. That is 33.5 M rows of headroom at the default width: +nineteen hours at 500 shares per second, ten days at 39. The constraint is +enforced on new rows at once, and adding it takes the table lock only for the +catalog change, which is why it is taken with the same short timeout and +retries as the swap. The sequence is never restarted: rows keep landing in the +release table until it reaches the bound, then in the next cell, so `share_seq` +stays gap-free. + +**2. Validate.** `ALTER TABLE ... VALIDATE CONSTRAINT` for every `CHECK` the +table still carries as NOT VALID, in name order. That is the bound, and the +release's `qbit_share_ledger_credit_policy_check` where 001 left it pending on +a ledger upgraded from before the column existed, which is the one NOT VALID +constraint the migrator tolerates on a source. Each is one scan of the table +under SHARE UPDATE EXCLUSIVE, which blocks neither appends nor reads, and each +runs for hours on a production-sized ledger. ATTACH refuses a partition whose +constraint is NOT VALID while the parent's is validated, so both have to be +validated before the swap, not just the bound. + +**Re-bounding.** A run that is interrupted and resumed later may find the +sequence has eaten the headroom. `prepare` is therefore called again at the +start of the pending stage and of the validated stage: it returns the existing +bound while at least one partition width of headroom is left, and otherwise +drops the constraint, pending or validated, bounds further out and records the +new bound, which then has to be validated again. That is the one case where a validation scan has to start +over, and it is why an interrupted conversion is better resumed promptly than +left for a week. + +**3. Swap.** One transaction of catalog work: rename the table and its six +indexes to their `_p0` names; create the parent `LIKE` the release table, with +the same columns, defaults and `CHECK` constraints under the same names, which +is what ATTACH matches on; give the parent the primary key and the four +secondary indexes under the release names; move the sequence's ownership and +the grants across; `ATTACH PARTITION ... FROM (MINVALUE) TO (bound)`, which +proves the bound from the validated constraint and adopts every existing index; +recreate `qbit_shares_since_template_height`, the one release function bound to +the old row type by OID; install the immutability trigger on the parent, +because PostgreSQL does not clone a statement trigger to a partition; record the +catalog; create the lead partitions. Measured on the container at 16 ms with +5,000 rows, and none of its terms is a function of the row count. The swap needs +ACCESS EXCLUSIVE, so it is requested with a two-second `lock_timeout` and +retried for up to ten minutes: a long read delays the swap instead of holding +every append queued behind an ACCESS EXCLUSIVE request. Each retry logs the +wait so far. The function also counts the indexes on the partition afterwards +and refuses if ATTACH built one instead of adopting it, because a build there +would hold that lock for hours. + +**Resumability.** The runner reads the ledger's stage afresh before each step +and after it, from the relation kind of `qbit_share_ledger`, the validation +state of `qbit_share_ledger_p0_bound` and `converted_at` in the catalog: + +| Stage found | What it means | What the run does | +| --- | --- | --- | +| plain | an ordinary table with no bound | refuses the reserved names, prepares the bound, then validates and swaps | +| pending | the bound is on the table and enforced on new rows, not yet validated | prepares again (which re-bounds only if the headroom is gone), validates every pending `CHECK`, then swaps | +| validated | the bound is validated | prepares again (which re-bounds and validates again only if the headroom is gone), then swaps | +| converted | the parent is partitioned and the catalog records the conversion | ensures the lead partitions and records 16 | + +Every name the run would take is checked while the ledger is still plain, +before the first step, so a refusal at that point has changed nothing. A run +that resumes from the pending or validated stage does not repeat that sweep, +because the bound is already on the table; the swap re-checks the seven `_p0` +names itself and refuses without changing anything if one has been taken since. +The version is recorded in its own transaction under the migration lock, after +the stage has been re-read and found still converted. A version another +instance recorded meanwhile is not applied again. + +**Refusals, and what each one means.** From the runner: + +| Refusal | What to do | +| --- | --- | +| `refusing to apply migration 16: table qbit_share_ledger_p0 already hold names the share ledger partitions take. The migration is not recorded and nothing was changed by it. Check what they hold, then rename or move them aside and migrate again` | every relation named `qbit_share_ledger_p<k>` or `qbit_share_ledger_p<k>_<suffix>` is listed with its kind; rename or move each aside, then migrate again | +| `migration 16: swapping the share ledger could not take its lock on qbit_share_ledger within 600 s; a transaction has held the table throughout (a long read, an open writer). Let it finish, or stop the frontends, and migrate again` | also raised for `preparing the partition bound` and `checking the partition bound`; find the holder in `pg_stat_activity` and `pg_locks`, let it finish or stop the frontends, and migrate again. Nothing after the last completed step was changed | +| `qbit_share_ledger does not exist` | the runner was pointed at a schema without the ledger; check `search_path` and the database | +| `qbit_share_ledger is a partitioned table but qbit_prism_share_partitioning does not record a conversion; the migrator did not partition it. Check what did, then restore the full backup or move the table aside and migrate again` | something other than this migrator partitioned the ledger; the runner will not adopt it | +| `qbit_share_ledger is a relation of kind v, not a table; the migrator cannot partition it` | a view or other relation holds the ledger's name; restore the table from the full backup | +| `refusing to record migration 16: qbit_share_ledger is no longer the converted table this run left; migrate again` | the ledger changed between the conversion and the recording; nothing is recorded, and the next run plans afresh from what it finds | +| `validating the share ledger bound for migration 16; migrate again to resume` | the context on a failed or cancelled validation scan; the pending constraint stays, and the next `migrate` resumes from the pending stage | + +From the SQL functions that do the work: + +| Refusal | What to do | +| --- | --- | +| `share ledger is already converted` | raised by `prepare` and `swap` when `converted_at` is already set. The runner reads the stage first and never reaches them in that state, so this appears only when the functions are called by hand | +| `qbit_share_ledger is not a plain table` | raised by `prepare`, `validate` and `swap` when the ledger is already partitioned or is not a table at all; see the runner refusals above | +| `qbit_share_ledger_p0_bound exists with an unexpected definition: <definition>` | a constraint under the bound's name that is not `share_seq < <n>`; inspect it, drop it if it is not the migrator's, and migrate again | +| `qbit_share_ledger_p0_bound is missing; run qbit_prism_share_ledger_convert_prepare() first` | the swap was called out of order; `migrate` calls the three in order, so this appears only when the functions are called by hand | +| `qbit_share_ledger_p0_bound is not validated; run qbit_prism_share_ledger_convert_validate() first` | the same, one stage later | +| `qbit_share_ledger carries a NOT VALID CHECK constraint (<names>); run qbit_prism_share_ledger_convert_validate() first` | a `CHECK` was added NOT VALID after the validation step ran; validate it, or drop it if it is not wanted, then migrate again | +| `qbit_share_ledger_p0_bound reads <n> but the catalog recorded <m>` | the constraint and `qbit_prism_share_partitioning.conversion_bound` disagree, so one of them was edited; restore the full backup rather than reconciling them by hand | +| `the share sequence has reached the conversion bound <n>; prepare again with more headroom` | the sequence passed the bound before the swap ran, which needs a full partition width of shares to land during the validation scan. The runner re-prepares only from the plain and pending stages, so a `migrate` from the validated stage repeats this refusal; drop `qbit_share_ledger_p0_bound` so that the next run prepares a new bound, and expect the validation scan again | +| `refusing to convert the share ledger: a relation already holds the reserved name qbit_share_ledger_p0_pkey` | one of the seven `_p0` names is taken; rename or move that relation aside and migrate again | +| `refusing to convert the share ledger: index qbit_share_ledger_accepted_recent_idx is not on qbit_share_ledger` | one of the six release indexes is missing or sits on another table, so 013 did not finish or an index was moved; repair the index set and migrate again | +| `attaching qbit_share_ledger_p0 built 1 new index(es) instead of adopting the existing ones` | a release index did not match the parent's definition, so ATTACH built one; the transaction rolls back, nothing is converted, and the index definitions have to be reconciled with 013's declarations first | +| `refusing to create share ledger partition qbit_share_ledger_p7: a relation already holds that name` | from `qbit_prism_share_partition_create`, at conversion time or later from the periodic ensure; rename or move that relation aside | +| `share ledger is converted but no partition is attached in qbit_prism_share_partitions` | from the ensure function: the catalog has no attached row to extend from, so it was edited or selectively restored; restore the full backup | + +And from the scratch apply, which proves before any DDL that the migration file +converts the table: `migration 16: qbit_share_ledger is missing before it`, +`migration 16: qbit_share_ledger already has partitions before it`, +`migration 16: qbit_share_ledger is missing after it`, and +`migration 16: qbit_share_ledger_p0 is not a partition of qbit_share_ledger +after it`. These refuse the binary's own migration file, not the source +database, and mean the file and the runner have gone out of step. + +**Plan for the conversion on a large ledger.** Budget one full scan of the +table per pending `CHECK` constraint, which is hours on a production-sized +ledger: the bound always, and the release's +`qbit_share_ledger_credit_policy_check` as well where 001 left it NOT VALID. +Appends and reads continue throughout, at SHARE UPDATE EXCLUSIVE. The swap +itself is milliseconds, but it needs ACCESS EXCLUSIVE, so allow for up to ten +minutes of `lock_timeout` retries and stop long-running reports for that +window if the retries are being exhausted. Run `migrate` from a shell where a +multi-hour command is acceptable, not from a starting frontend, whose readiness +stays down until the conversion completes. Unlike 013, this migration needs no +extra disk and rebuilds no index: it copies no rows and every index is adopted +by ATTACH. Run `ANALYZE qbit_share_ledger` afterwards and take the before and +after measurements the +[operations runbook](prism-ledger-ops.md#measuring-before-and-after) lists, +which are acceptance criterion 4 of #144. + +**One-way, per decision D5.** No revert script ships. Reverting means a second +cutover: rebuilding a global `UNIQUE (share_id)` over the whole table under +ACCESS EXCLUSIVE, measured at 43 s over 5.5 M rows in the design spike and +proportional to the table, and that rebuild fails at the end on the first +duplicate `share_id` accepted across two leaves. Recovery is the +[isolated-restore reconciliation](prism-ledger-ops.md#one-way-migration-and-isolated-restore-reconciliation) +of #287, as for every other native migration. + +## Bring up native instances + +Start one instance first and verify readiness before admitting controlled miners: + +```sh +qbit-prism-server run +# In a separate shell with the same environment: +qbit-prism-server healthcheck --url http://127.0.0.1:3341/healthz +qbit-prism-server self-check +``` + +For the supplied external database Compose overlay: + +```sh +# Public reads may use the writer, or a separate standby endpoint with mode=require. +export PRISM_PUBLIC_DATABASE_URL="$PRISM_DATABASE_URL" +export PRISM_PUBLIC_REPLICA_MODE=off +docker compose -f compose.yaml -f compose.prism-external-db.yaml \ + --profile prism up -d qbitd prism-coordinator prism-public-api +``` + +Set `PRISM_DATABASE_URL` to the external writer endpoint on every host. The +overlay removes dependencies on the local primary and replica containers. +Set `PRISM_PUBLIC_STRATUM_URL` to the advertised mining endpoint. The public +service uses its own read pool and does not require signing keys or an audit +filesystem mount. To read a standby, set `PRISM_PUBLIC_DATABASE_URL` to it and +`PRISM_PUBLIC_REPLICA_MODE=require`; health then requires a recovering server +and a fresh replication stream. Replay lag is reported separately. +Use an up-to-date Docker Compose release supporting `!override`. Production +operators should combine their pinned-image/storage production configuration +with this overlay **last**, and render `docker compose ... config` before start. + +`QBIT_RPC_HOST` is overridable. With an already managed external qbitd, set its +RPC connection and start only the coordinator with `up -d --no-deps +prism-coordinator`. The local qbitd dependency then does not launch another node. +The `make up-prism-pool` helper manages the local node/database/public-service stack; use the +explicit Compose overlay commands for the external database topology. + +Add the remaining instances with identical signing and payout configuration. +Keep TCP connection routing stable while a miner is connected; reconnects may +land on another instance. Existing TCP sessions do not move between processes. +The shared extranonce allocator, durable job records, proof deduplication, and +canonical ledger provide cross-instance accounting continuity. + +Check these before restoring ordinary traffic: + +- Every frontend is healthy and follows the intended qbit chain/tip. +- Controlled miners subscribe, authorize, receive difficulty/jobs, submit shares, + and recover after reconnect or a frontend restart. +- Database share order and counts reconcile with unique committed proofs; + duplicate submission on another instance gives no second credit. +- Baseline audit downloads have the same canonical hashes; carry-forward + integrity and CTV state agree with the pre-migration records. +- Public dashboard and audit routes return the expected data on every instance. +- Candidate and CTV claim recovery work after a process interruption. + +A subsequent rollout between compatible **Rust** versions may drain and replace +one frontend at a time. Review each version's schema compatibility separately. +Run the new release's `migrate` first; frontends still on the previous release +keep starting on the newer schema while its migrations are additive and it +declares no capability they do not understand, so they can be drained and +replaced one at a time. A release that declares a new capability needs every +frontend stopped first. + +## HA durability and failover + +Multiple frontends remove dependence on one Prism process. The database still +has one authoritative primary/writer endpoint. All accounting writes and reads +must reach that endpoint, rather than independent primaries or lagging read +replicas. The separately served public dashboard may use a checked read replica; +it never makes accounting or settlement decisions. + +Keep `fsync=on`, `full_page_writes=on`, and `synchronous_commit=on`. The last +setting makes a local commit durable but does not create replication. If the +requirement is no loss of acknowledged shares after primary storage loss, +configure synchronous replication and promote only a standby holding all those +durable commits. An asynchronous failover can lose already acknowledged work. +Avoid silently degrading the required synchronous policy during an outage. +The bundled public-read replica uses asynchronous replication by default; its +presence alone does not provide this lossless accounting failover guarantee. + +Exercise the actual database failover endpoint under controlled mining load. +Reconcile ACKed proof IDs with the promoted database and inspect retry/dedupe, +payout, and candidate outcomes. Use independent backups and WAL archives even +when replication is synchronous; replicas can reproduce operator mistakes. + +The repository also includes a disposable physical PostgreSQL failover test: + +```sh +PRISM_TEST_PG_BIN_DIR=/usr/lib/postgresql/16/bin \ + cargo test --locked -p qbit-prism-server --test postgres_failover -- --nocapture +``` + +It checks two ledger clients' acknowledged IDs through a stable TCP proxy, +synchronous standby promotion after immediate primary loss, duplicate replay, +and continued writes. Run it with installed PostgreSQL server tools as an +unprivileged user. This test complements production-specific failover drills. + +## Configuration and observable changes + +Retained interfaces include miner username syntax, BIP310 version rolling, +vardiff and high-difficulty listeners, the payout/CTV policy variables, public +`/public/v1` routes, audit route aliases, canonical bundle hashes, and legacy +`*_SATS` monetary aliases. Keep explicit mainnet share/vardiff bounds. As on +`2.x.x`, mainnet permits bounded one-parent stale-share grace (default 3 seconds). +The grace interval starts when that client receives replacement work; stale +blocks are never submitted. Retained vardiff hints are shared by listener and +exact username, with accepted-work evidence controlling their expiry. + +Native submit classification also preserves the published-tip authority used by +`2.x.x` at `95ffe063846d51f83999a66cc654da5f7476fdef`. A detected tip immediately +fences block candidates. Miner share credit continues against the last published +work while replacement is prepared, including when stale grace is zero. Its +ordinary freshness budget is `PRISM_SUBMIT_TIP_MAX_AGE_SECONDS` (default 10); +zero forces a live tip RPC for each share. A detected replacement may extend +published authority through `PRISM_TEMPLATE_REFRESH_FAILURE_EXIT_SECONDS` +(default 120), measured from the first departure. Repeated detections and failed +refresh attempts do not restart that deadline. Once both budgets have expired, +submit falls back to the node; RPC failures remain backend-unavailable. + +Successful work publication opens one-parent stale grace after the startup +baseline. The deadline starts separately for each connection's first delivery +of that tip, and same-tip refreshes do not slide it. An undelivered replacement +keeps eligible retained work in a bounded per-connection graveyard; +absolute reconnect-job expiry is never extended. Share credit retains the +original issued worker, target, network difficulty and policy. Both ordinary +published-work credit and stale-grace credit still commit under the current +transactional payout revision. These are restorations of miner behavior, with +ungated real-coordinator decision and socket/session regression tests; they do +not relax current-chain candidate submission checks. +See the [miner decision parity reference](prism-b8-miner-parity.md) for retained +work bounds, the regression coverage map and disposable qualification commands. + +`PRISM_USERNAME_FALLBACK_ADDRESS` applies when validation explicitly identifies +an invalid address or a recognized address type that Prism cannot pay. RPC +failures and malformed validation responses reject authorization without +substituting another payout recipient. + +Mainnet `check-config` and `run` require `QBIT_EXPECTED_GENESIS_HASH` to contain +the trusted 64-hex genesis hash. Startup compares it to the connected node; +production flags reject regtest. Public-chain readiness also requires completed +initial block download, matching block/header heights, and at least +`PRISM_MIN_PEERS` connected peers (default 1). Templates must satisfy +`PRISM_TEMPLATE_MAX_AGE_SECONDS` (default 120). A failed readiness observation +closes mining readiness until a fresh valid poll succeeds. + +General node and wallet RPCs use `PRISM_RPC_TIMEOUT_SECONDS` (15 seconds). +The client keeps an idle node connection for at most 20 seconds (#759). Keep +qbitd's `-rpcservertimeout`, and the idle timeout of any proxy in front of the +node, above 20 seconds; qbitd's default is 30. Otherwise a call can be written +into the idle close and fail after the connection existed, which `submitblock` +records as an unknown outcome. +`PRISM_BLOCK_SUBMIT_RPC_TIMEOUT_SECONDS` (1 second) bounds only `submitblock`; +ambiguous submission results retain the durable candidate for reconciliation. +Candidate claims renew throughout processing, including waits for build workers. +Losing the lease cancels the attempt; the durable row remains recoverable. +Fresh candidates take priority over retries, with periodic oldest-due selection +to keep older recovery work moving. +Wallet selection preserves any configured `QBIT_RPC_URL` proxy path prefix. +The independent public service retains its separate public read deadline. + +Canonical payout changes retire prior jobs even when the parent tip is +unchanged. Miners receive replacement work with `clean_jobs=true`; ordinary +same-tip refreshes keep valid retained jobs, and previous-parent share grace +remains bounded by each connection's notification time. +Reauthorizing a connection preserves the original payout identity of retained +work. When `PRISM_STRATUM_MAX_CONNECTIONS_PER_USERNAME` is enabled, that work +also keeps its original username's capacity slot until it expires or is discarded. + +CTV fee policies, including explicit rates, are checked against the node's live +`minrelaytxfee` and `mempoolminfee` before building payout artifacts. A configured +rate or discounted premium below these floors fails the build; correct the fee +policy before admitting mining work. + +Removed implementation settings include Python writer IDs/epochs/session leases, +share batch/linger queues, psql/native-client selection, subprocess builders, +incremental window schedulers, refresh rollout gates, and their watchdog knobs. +Remove them from the operator environment; they do not tune the Rust runtime. +The supported settings are documented in [.env.example](../.env.example) and the +[native server README](../crates/qbit-prism-server/README.md). + +The combined audit/operator HTTP port remains 3341; the separate native +`public-api` role preserves public port 3342. `/healthz` retains its readiness role; +process metrics describe native work rather than the old Python scheduler. +Migrate deployed alerts using the [native inventory](prism-native-metrics.md) +and [complete alert migration and deployment diff](prism-alert-migration.md). +Queue-pressure intent moves to `qbit_prism_share_ack_seconds` and +`qbit_prism_rejections_total{reason_id}`; Python lease wake delay moves to +`qbit_prism_runtime_lag_seconds`, retained `qbit_prism_runtime_poll_lag_seconds` +and `qbit_prism_runtime_task_stalled`. Refresh impact uses +`qbit_prism_stratum_oldest_pending_initial_job_seconds` and +`qbit_prism_stratum_current_tip_coverage_gap_seconds`; pending-candidate age and +count retain their names. Guard body-based rules with #277's +`qbit_prism_metrics_snapshot_available` / `qbit_prism_metrics_snapshot_stale` +and candidate count/age and RSS rules with `qbit_prism_collector_available` so +unknown -1 is never healthy zero. The pool-wait histogram is read live at scrape +time; its rule requires `up == 1` and sufficient per-instance observations instead +of cached-body or collector availability. Deploy that producer on every target +before activating this rule. The first-offer histogram is declared without samples +(A/#266); advisory-lock waits are recorded since #328; rules for both remain +deferred. D3's dedicated standby alerts require the +primary's deployment-provided PostgreSQL exporter, not public read replica data. +Dashboard totals continue to derive from the shared database. The native +`self-check` emits structured JSON and fails nonzero on an error instead of +printing the Python checker's old PASS/WARN/FAIL table. + +The newer block-marker, dual-rate chart, network-hashrate, and reorg-view +contracts are retained. Native block intents are persisted before acceptance, +so the default active block view, chart markers, and found-block totals include +confirmed blocks only. Use `chain_state=all` to inspect every candidate with its +explicit state. Previously landed blocks that disconnect appear as `reversed` +with their disconnection time, while remaining eligible for native reactivation. +Earnings and payout histories remain confirmed-only. + +Hashrate history uses the existing three rollup grains and raw tail. Any +frontend may advance the shared watermark; an atomic comparison prevents +concurrent passes from double-counting. `PRISM_HASHRATE_ROLLUP_ENABLED`, +`PRISM_HASHRATE_ROLLUP_INTERVAL_SECONDS` (15), and +`PRISM_HASHRATE_ROLLUP_BATCH_SHARES` (50000, maximum 100000) retain their roles. + +Bootstrap now pays the solver only when there are no historical shares; there +is no three-miner readiness gate. A network-valid proof below its assigned share +target is credited at network difficulty only after its block is confirmed on +the active chain. These deliberate accounting simplifications were approved +under decision D2 in #260, and belong in the operator release notes. Their +payout effect is recorded in +[Payout differences from 2.x.x (decision D2)](#payout-differences-from-2xx). + +`2.x.x` also read `PRISM_STRATUM_SHARE_WEIGHT` (default 1) and <!-- retired-setting: PRISM_STRATUM_SHARE_WEIGHT --> +`PRISM_STRATUM_SHARE_WEIGHTS_JSON` (default empty), a JSON object keyed by miner <!-- retired-setting: PRISM_STRATUM_SHARE_WEIGHTS_JSON --> +username or payout address. A worker with an entry in that object was credited +`max(1, override)` instead of its share target's difficulty; the default weight +only filled the job's delivered `share_weight` field. `3.x.x` does not support +these per-worker credited-difficulty overrides. With the `2.x.x` defaults, which +set no overrides, both versions credit a share at its share target's difficulty, +so crediting is unchanged. A deployment that set either variable must remove it +before cutover, because `3.x.x` ignores it. + +<a id="payout-differences-from-2xx"></a> + +## Payout differences from 2.x.x (decision D2) + +`crates/qbit-prism/fixtures/vectors/` holds money-path vectors exported from +`2.x.x` at `504846cc0b72e8f86ed17f896d4ccbbe196a31dc`. They were computed by the `2.x.x` engine +and rule code, not by hand. `crates/qbit-prism/tests/money_path_vectors.rs` +replays every case through this engine. Window clipping, carry-only recipients, +pool fees, dust recompute, remainder ties, and CTV chunking match `2.x.x` +exactly. Every other difference is one of the entries below, under decision D2 of issue #260. Each +differing case stores both the `2.x.x` and `3.x.x` payout and names its entry +by anchor. The test fails if the anchors the vectors use differ from the +entries below, and it pins every vector file's sha256, so a vector changes only +through a re-export in a reviewed commit that updates the pin. All amounts are +in sats. +All vectors use the day-one floor of 14720 sats. + +<a id="d2a-bootstrap-pooling"></a> + +### D2a: bootstrap pooling + +- **Vector:** `bootstrap_transition.json`, case + `below-gate-with-other-miners-shares`. + - miner-a has a 30-difficulty share and miner-b a 20-difficulty share in the ledger. + - miner-b solves a 500000000-sat block. +- **2.x.x:** miner-b is paid 500000000. + - Two distinct miners is below the `PRISM_MIN_READY_MINERS` gate (default 3). <!-- retired-setting: PRISM_MIN_READY_MINERS --> + - So the job is a collection job: one synthetic solver share, and the solver + is paid the whole coinbase. +- **3.x.x:** miner-a is paid 300000000 and miner-b 200000000. + - The window is proportional as soon as any share exists. +- **Reason:** `crates/qbit-prism-server/src/coordinator.rs` (bundle selection + after the ledger snapshot) builds the solver-only bundle only when + `snapshot.shares` is empty. There is no readiness gate. +- **Matching cases:** `at-readiness-gate` (three miners, prior balances kept on + both) and `empty-ledger` (the solver is paid everything on both) match `2.x.x`. +- **D2 item:** bootstrap pooling. + +<a id="d2b-below-target-credit"></a> + +### D2b: below-target block credit + +- **Vector:** `below_target_credit.json`, cases `block-only-proof-accepted`, + `block-only-proof-confirmed-after-reconciliation`, and + `block-only-proof-reorged-after-acceptance`. + - The listener floor holds the share target at difficulty 4000000. + - Network difficulty is 1000000. + - miner-a submits a proof that meets the network target but not the share + target. +- **Credited amount:** + - **2.x.x:** the assigned share difficulty, 4000000. + - **3.x.x:** network difficulty, 1000000. +- **Next block's payout** (8000000 window; miner-b has 2000000 + 2000000, miner-c 1000000): + - **2.x.x:** miner-a 250000000, miner-b 187500000, miner-c 62500000. + - The 9000000 of credited work overfills the window. + - miner-b's oldest share counts only 1000000. + - **3.x.x:** miner-a 83333334, miner-b 333333333, miner-c 83333333. + - The window holds 6000000. +- **Timing:** + - **2.x.x:** credits at node acceptance of the block. + - **3.x.x:** credits inside the transaction that marks the block confirmed on the + active chain. + - That is either after acceptance or during reconciliation. + - The Stratum acknowledgement waits for that credit row. + - A block that is abandoned instead fails the submission with "block-only + proof was not accepted on the active chain". + - The acknowledgement waits at most `share_commit_timeout` (#574; it was + `max(60 s, share_commit_timeout)` before), and a candidate still pending + then is answered + `ledger-outcome-unknown` while its credit can still land. + - On both versions a credited share survives a later reorg. + - **Modeling assumption:** the `3.x.x` next-block payout assumes the + deferred credit lands before the next block's anchor, because the credit + row is stamped at confirmation, not at submission. +- **Reason:** + - `crates/qbit-prism-server/src/coordinator/miner_submit.rs` credits `network` difficulty + when `share_pass` is false, and holds the share as the candidate's + `deferred_share` until the credit row exists. + - `crates/qbit-prism-server/src/ledger/blocks.rs` (`credit_deferred_share`) + appends it only on confirmation. +- **Matching cases:** `block-only-proof-rejected` (no credit on either) and + `share-and-block-proof-control` (a share-passing proof is credited at the + assigned difficulty on both) match `2.x.x`. +- **D2 item:** below-target credit. + +<a id="d2c-prior-balances-during-bootstrap"></a> + +### D2c: prior balances during bootstrap + +**Status:** approved on 2026-09-11 under decision D2 in #260. This entry is +recorded separately from D2a because it was approved on its own. + +- **Vector:** `bootstrap_transition.json`, case + `bootstrap-carry-only-account-at-or-above-floor`. + - The share window is empty. + - miner-old has a 20000-sat carry and no share. + - miner-b solves a 500000000-sat block. +- **2.x.x:** miner-b is paid 500000000. + - miner-old is not in the payout manifest and is not paid in this block. + - Its 20000-sat balance waits for a later block. +- **3.x.x:** miner-old is paid 19999, and carries 1. + - miner-b is paid 499980001, and carries 19999. + - The engine allocates the coinbase over both accounts' candidate balances. +- **Reason:** + - The `2.x.x` collection bundle passes `prior_balances=[]` + (`lab/prism/job_bundle.py` `build_collection_bundle`, which is `2.x.x` + code at `504846cc0b72e8f86ed17f896d4ccbbe196a31dc` and does not exist on + `3.x.x`). + - `crates/qbit-prism-server/src/coordinator.rs` `build_bundle` passes + `snapshot.prior_balances` in bootstrap too. + - So a carry-only account at or above the floor can be paid in a bootstrap + block. +- **D2 item:** D2c, prior balances during bootstrap. + +## Retained mining work + +Same-connection active eviction preserves the original worker, target and +version mask, including after reauthorization. The configured per-connection +retention count N bounds both the existing active set and the same-tip graveyard. +A previous parent can temporarily own its former active set plus graveyard; +the graveyard has a conservative total cap of 3N (4N including active work). +Same-tip graveyard TTL starts at eviction. Previous-parent entries require a +published transition and delivery-based grace; unrelated parents and absolute +resume expiries are removed. Original username admission permits remain held +while this retained work can credit, and are released on actual expiry, capacity +eviction, payout replacement or disconnect. Reauthorization reuses the same +connection's permit. Cross-connection resume still requires the exact original +worker, the current published template's parent and payout revision, and an +unexpired absolute job deadline. The durable current-revision fence remains +separate from those original published payout inputs. + +## Recovery and rollback + +D5 declares a **one-way migration with forward repair and isolated-restore +reconciliation**. There are no down-migrations for native 002–010, including +006/007/008, or permission to remove migration guards. The legacy publication +ordinal revert refuses native schemas and points here. The following boundary +matches the [unreleased 3.0.0 release notes](../doc/release-notes-3.0.0.md): + +> Before the first native share is acknowledged, restore the complete +> pre-migration database and artifact backup and restart the pinned old image +> in isolation from the migrated database. After the first native share is +> acknowledged, restoring an older database loses those accepted records. +> Any rollback that discards acknowledged history requires an explicit +> accounting reconciliation and recovery decision; it is not an ordinary +> image rollback. Keep every native frontend stopped while performing an +> isolated restore/recovery operation against its replacement database. + +The ACK boundary does not prove that nothing else committed: a committed share +can lose its reply, and candidate/CTV work can progress before admission. Always +compare the complete evidence and retain the latest database. Prefer forward +repair or a compatible native image preserving all durable history. + +An interrupted `migrate` is before this boundary as long as migration 2 is not +recorded and `share_hash_backfill_pending` is not 2. On a populated source, 2 is +recorded only after +[002's share-hash backfill](#migration-002s-share-hash-backfill-applied-online), +and every start refuses the database until then, so no native share can have +been acknowledged. Resume it with `migrate`, or restore the pre-migration +backup, as for any other pre-ACK failure. Once `migrate --defer-share-hashes` +has raised the fence to 2, frontends may serve with 2 unrecorded, and the +boundary is their first ACK, as on any served database +([Serving with the backfill pending](#serving-with-the-backfill-pending-fence-2)). + +1. **Establish isolation and retain evidence.** Stop every native frontend, + public service, candidate worker and CTV broadcaster, and confirm no old + Python writer is running. Block their network access to the restore. Keep + the latest database and WAL, the drained pre-migration database backup, all + external bodies/segments, pinned images, and key/configuration recovery + material. Never restore over the current database. Record whether any native + share was acknowledged; an unknown boundary is treated as post-ACK. +2. **Capture the drained source baseline before migration.** Configure named + libpq services `prism-source`, `prism-current`, and `prism-restore` for the + distinct databases, with the intended ledger schema first in `search_path`. + Keep credentials in protected libpq files, not the evidence. The export is a + single repeatable-read, read-only transaction. Reserve disk for ordered share + history; the summary hashes it a row at a time. With writers still stopped: + + ```sh + umask 077 + PGSERVICE=prism-source pg_dump --format=custom --file=pre-native.dump + PGSERVICE=prism-source psql -XqAt -v ON_ERROR_STOP=1 -v FETCH_COUNT=10000 \ + -f scripts/prism-recovery-evidence.sql > source.rows.jsonl + python3 scripts/prism-recovery-evidence.py source.rows.jsonl > source.summary.json + ``` + + `FETCH_COUNT` makes psql print rows in batches instead of buffering each + whole result, which took 63 GB and was OOM-killed at 65.9M shares (#705). + The script defaults it to 10000 since #705; the flag covers copies from + before that. + + The serial export took 85 minutes at 65.9M shares (#712). The + [parallel export](../scripts/prism-recovery-evidence-parallel.py) prints the + same rows byte for byte, so the summaries and every `cmp` below are + unchanged. It can replace the `psql` line of any export in this procedure: + + ```sh + umask 077 + PGSERVICE=prism-source python3 scripts/prism-recovery-evidence-parallel.py \ + --jobs 8 > source.rows.jsonl + ``` + + A coordinator session runs the script's checks and exports its snapshot. + `--jobs` sessions import that snapshot. They export the shares in + `share_seq` ranges, the share hashes in header-hash ranges, and the other + kinds as the script does. Each range's sort may use `--work-mem` (default + 256MB) before it spills to disk. Size `--jobs` to the database host's idle + cores; the export opens one connection more. No session may prompt for a + password, so keep it in the service's password file. Parts wait in + `--work-dir` (default: the current directory) until they can be printed in + order, so budget the export's size there as well. If any session fails, + the output has no completion marker, so the summarizer refuses it. On a + 6M-share copy (4.6 GB of rows, a shared 16-core host), 8 jobs took 24 s + against 115 s serially on the `2.x.x` source, and 12 s against 57 s once + migrated. + + The summarizer parses in parallel as well: `--jobs` worker processes + (default: the CPU count, at most 8) parse and canonicalize newline-aligned + ranges of the rows, and one process applies every check in file order. Its + summary, notes and refusals are those of `--jobs 1`, the serial summarizer, + byte for byte. From a range a worker cannot read as a text-mode read does + (invalid UTF-8, a read error) on, it reads the rows serially from the start, + so even a decoding error is the serial one. Size `--jobs` to the host's + idle cores. The serial pass ran at about 29 MB/s on the 65.9M-share restore + (about 80 GB, so about 45 minutes). On 3.25 GB of share and header rows on + a 16-core host, 8 jobs took 5.7 s against 36 s serially, and 12 jobs 4.4 s. + + Keep the filesystem backup from the same drained boundary. Require + `unfinished_candidates` zero and both integrity mismatch counts zero. The + summary records ordered share counts/digests, accepted count and high-water + sequence, block/publication order, audit SHA identities, payout/carry and CTV + state. `audit_head_sha256` is the exact `2.x.x` active carry hash chain. The + SQL integrity function alone does **not** calculate that head; the companion + Python summarizer does. Compare it with the previously mirrored legacy head. + The checked-in SQL contains the exact reconciliation queries. The summarizer + checks legacy carry rows per payout program, as 025's validator does + (#708): a finding of `2.x.x`'s per-label rule that the program's chain + clears is named in a `note:` on stderr and left out of the summary, so the + source summarizes as its migrated copy will. +3. **Restore and verify the pre-ACK recovery path.** Provision a new empty + database behind `prism-restore` on an isolated host with the matching + PostgreSQL version, roles and extensions. Restore the full backup (not a + selected table dump) and its filesystem backup at the original paths: + + ```sh + pg_restore --exit-on-error --single-transaction \ + --dbname='service=prism-restore' pre-native.dump + PGSERVICE=prism-restore psql -XqAt -v ON_ERROR_STOP=1 -v FETCH_COUNT=10000 \ + -f scripts/prism-recovery-evidence.sql > restored.rows.jsonl + python3 scripts/prism-recovery-evidence.py restored.rows.jsonl > restored.summary.json + cmp source.summary.json restored.summary.json + ``` + + Require exact equality. Only after it passes may the pinned old image be + started against this isolated legacy database for recovery verification. + Keep miner routing and broadcast access disabled. Stop it before rehearsing + the native migration. Never point the old image at a native schema. +4. **Rehearse forward migration and canonical repair.** Set the native tool's + `PRISM_DATABASE_URL` to the isolated restore, verify that endpoint, mount the + recovered artifacts, and supply the original trusted + `PRISM_LEDGER_WRITER_PUBLIC_KEY_HEX`. The import does not need signing seeds: + + ```sh + time qbit-prism-server migrate --offline-indexes + time qbit-prism-server import-audits --root /var/lib/qbit-prism/audit + PGSERVICE=prism-restore psql -XqAt -v ON_ERROR_STOP=1 -v FETCH_COUNT=10000 \ + -f scripts/prism-recovery-evidence.sql > migrated.rows.jsonl + python3 scripts/prism-recovery-evidence.py migrated.rows.jsonl > migrated.summary.json + cmp source.summary.json migrated.summary.json + ``` + + Import must exit zero with both completeness counts zero; it reconstructs + canonical bytes when recoverable bodies have no sidecar. A present corrupt + sidecar is an error, not permission to use a different body. It commits one + verified audit at a time; after restoring missing evidence, rerun it safely. + Check equality before running `backfill-ctv`, which can intentionally repair + missing CTV rows; retain and explain that repair's later evidence delta. + After `migrate --defer-share-hashes`, the migrated summary marks + `share_hashes` deferred, so compare every other record, and the share + hashes once 2 is recorded, as + [Recovery evidence while the backfill is pending](#recovery-evidence-while-the-backfill-is-pending) + describes. +5. **Verify canonical reads and the public edge.** On the isolated public API, + fetch stored artifact SHA values from the audit export. Compare downloaded + bytes with the source backup, their SHA-256, and the quoted SHA ETag. After + actual cutover repeat through the configured public edge, not only origin: + + ```sh + # PUBLIC_EDGE is the reviewed HTTPS public endpoint; SHA is a recorded audit SHA. + curl --fail --silent --show-error -D artifact.headers \ + "$PUBLIC_EDGE/public/v1/artifacts/$SHA" -o artifact.json + printf '%s artifact.json\n' "$SHA" | shasum -a 256 -c - + rg -i "^etag: \"$SHA\"" artifact.headers + ``` + + Require no `x-prism-artifact-canonical-state: missing` fallback header. + Successful canonical responses omit that header. If the edge still serves a cached + missing response, invalidate that affected cache entry and verify again + before admission. Published canonical bytes and hashes must not change. +6. **Reconcile the post-ACK boundary.** Export `prism-current` with the same SQL + and summarizer into `current.rows.jsonl` and `current.summary.json`. Compare + with `restored.summary.json`, retaining the ordered JSONL when a digest + differs. The following exact query on the current database identifies the + accepted share tail beyond the source high-water mark; substitute the + recorded integer as the psql variable `source_last_share_seq`: + + ```sql + SELECT share_seq, share_id, miner_id, share_difficulty, credit_policy + FROM qbit_share_ledger + WHERE accepted AND share_seq > :source_last_share_seq + ORDER BY share_seq; + SELECT state, count(*) FROM qbit_block_candidate_outbox GROUP BY state ORDER BY state; + SELECT qbit_carry_forward_integrity_report(); + ``` + + Tail counts alone cannot prove equality: reconcile changed block states, + carry balances/head, payouts, pending candidates and CTV broadcasts against + chain evidence too. A nonempty native tail demonstrates records an older + restore would discard. Preserve that database and repair forward. Do not + copy rows into the legacy schema or resume miners until the accounting owner + has approved a documented plan preserving or explicitly reconciling every + difference, including any proposed loss of acknowledged history. +7. **Complete the rehearsal and release gate.** Record backup/restore/migration/ + import/verification durations, row counts, retained evidence locations, + artifact samples, image digests and the measured outage window. With the + intended native services available, run production `self-check`; require + zero completeness and integrity counts and successful readiness. #291 must + repeat this procedure on production-sized history and retain this exact data + loss boundary when completing `doc/release-notes-3.0.0.md` before rollout is approved. + +CI runs steps 2 to 7 every week on a scaled 2.x.x ledger built from the frozen +release SQL: a pre-ACK isolated restore, `migrate` and `import-audits` with +exact evidence equality, both native servers mining their own blocks on a +real regtest node over the migrated history, `self-check`, and the post-ACK +boundary (the tail query above names exactly the native shares, the isolated +restore still equals the source, and the legacy revert refuses). The scenario +is `migration_lifecycle_tests` in `crates/qbit-prism-server/tests/live_regtest.rs`, +listed in `test/prism-weekly-gated-tests.txt`. It does not replace #291's +rehearsal on production-sized history. + +## Rehearse the cutover on a snapshot + +`make prism-cutover-rehearsal` rehearses the whole cutover on a `pg_dump` of +the drained `2.x.x` database, on a private host, and prints a pass/fail +report (#575). It never connects to the database the dump came from, or to any +database but the one it creates: + +1. It creates a private PostgreSQL cluster with `initdb` under `WORKDIR` + (`fsync`, `full_page_writes` and `synchronous_commit` on, listening on + loopback only) and restores the dump into it: a custom, directory or tar + archive with `pg_restore --no-owner --no-privileges`, a plain dump with + `psql` (take a plain dump with `--no-owner --no-privileges`: the cluster + has none of the source's roles). The cluster accepts only a random + password it keeps to itself, and is stopped and removed at the end. +2. On the restored source it takes the [recovery evidence](#recovery-and-rollback) + (`scripts/prism-recovery-evidence.{sql,py}`) and the invariants below. +3. It runs `check-config`, `migrate` and `import-audits --root <AUDIT_ROOT>` + with this build's `qbit-prism-server`, exactly as + [Drain and migrate](#drain-and-migrate) runs them, timing each and + sampling `pg_locks` for the statements they run. +4. It takes the evidence and the invariants again and compares them with the + source's. Then it starts one frontend on the migrated ledger and checks + readiness, every address's balances through `/public/v1/miners/<address>`, + a first Stratum job (`mining.subscribe`, `mining.authorize` with an + address from the ledger, `mining.notify`) and `self-check`. + +The checks, each reported as PASS or FAIL: + +| Check | Holds when | +| --- | --- | +| restore reproduces the source | (per-PR test) the restored copy's evidence equals the database the dump was taken from | +| source drained | the source evidence has no unfinished candidate | +| check-config, migrate | the command exits 0; a refusal is reported with its error line | +| import-audits complete | the import exits 0 with `missing_stored_bodies` and `missing_canonical_bytes` both 0 | +| recovery evidence unchanged | the evidence summary after `migrate` and `import-audits` equals the source's, record for record, including `audit_head_sha256` | +| row counts and sums unchanged | every `2.x.x` table has the same row count and the same sums: accepted shares and their difficulty, the highest `share_seq`, stale-grace rows, blocks by chain and maturity state, audit identities, gross, on-chain and carried sats, payout entries, outbox states, CTV rows, the hashrate rollups row by row and their watermark, retained worker difficulty, the writer lease, and every balance recomputed from the carry rows (`qbit_recomputed_carry_forward_balances()`) | +| balances unchanged (SQL) | every address's owed, lifetime and pending-maturity balance, computed with the public API's own SQL, is unchanged | +| payout window unchanged | the window is not empty, and the native snapshot a frontend builds its next job from holds exactly the shares `2.x.x`'s `qbit_prism_window` held, in the same order and with the same credit policy, at the latest network difficulty the ledger records | +| window prior balances unchanged | the snapshot's prior balances equal `2.x.x`'s `qbit_current_carry_forward_balances()` | +| frontend ready, first job issued | `/healthz` answers 200 and the frontend sends `mining.notify` | +| API balances equal the source | every address's balances served by the running frontend equal the source's | +| self-check, audit completeness | `self-check` exits 0 and reports both completeness counts 0 | +| history unchanged after the frontend ran | once the frontend has started, reconciled and stopped, every fact above except retained worker difficulty (which it prunes past its TTL) is still the source's | + +Run it from a checkout of the release to be deployed, with a Rust toolchain +and PostgreSQL 16's server binaries: + +```sh +make prism-cutover-rehearsal \ + DUMP=pre-native.dump \ + LEDGER_KEY=<the trusted PRISM_LEDGER_WRITER_PUBLIC_KEY_HEX> \ + AUDIT_ROOT=/var/lib/qbit-prism/audit \ + STATEMENT_TIMEOUT_MS=<the PRISM_DATABASE_STATEMENT_TIMEOUT_MS migrate will use> \ + REPORT=rehearsal.json WORKDIR=/srv/rehearsal +``` + +| Setting | Meaning | +| --- | --- | +| `DUMP` | required: the dump to rehearse | +| `LEDGER_KEY` | required: the trusted ledger public key the history was signed with, which `import-audits` verifies every body against | +| `AUDIT_ROOT` | the external audit bodies, segments and canonical sidecars, mounted at their recorded paths as for the real import; without it only inline history can import, and the completeness check fails on any external body | +| `SCHEMA` | the ledger schema; by default the one schema holding `qbit_share_ledger` | +| `STATEMENT_TIMEOUT_MS` | the statement timeout the real `migrate` will run with; a statement that exceeds it fails the rehearsal as it would fail the cutover. Default: the server's 15000 | +| `ENV_FILE` | the reviewed production environment the real cutover will use (`NAME=value` lines): its synced qbitd, chain, genesis pin, signing seeds, trusted ledger key and pool fee. Every command then runs with it, with only the database URL, the listen addresses and ports, and the instance ID replaced, so `check-config` and `self-check` check the production configuration. A synced node's tip is past the snapshot's, so the frontend may mature blocks the source left immature: the API check then compares owed and lifetime balances only. By default the rehearsal runs a lab configuration with test signing keys, against a node it serves from the dump itself: each confirmed pool block at its height and a synthetic hash at every other height, with the highest confirmed block as the tip, so the reconciler sees the recorded chain and matures nothing `2.x.x` had not | +| `REPORT` | also write the report as JSON: steps, locks, the longest statements, checks, row counts and database sizes. Relative paths are resolved from where `make` runs | +| `WORKDIR` | where the private cluster lives while it runs (default `TMPDIR`); budget the restored database twice over, plus the audit root | +| `PG_BIN_DIR` | PostgreSQL 16 server binaries (default `pg_config --bindir`) | + +The frontend runs with test signing keys and pins its own cluster fingerprint +on the restored copy, which is discarded; nothing it signs leaves the host and +no miner connects to it but the rehearsal's one Stratum session. The report +can name addresses and balances: keep it with the rest of the rehearsal +evidence. It exits non-zero when any check fails. + +**Reading the report.** Each step has its wall time and the longest lock it +held. The lock sampler polls `pg_locks` every 10 ms on a side connection for +the commands' own sessions (`application_name=prism-cutover-rehearsal`). A +hold is continuous: a lock released and taken again counts as two holds. A +hold shorter than one interval shows as 0 ms, and a very short one can be +missed. `migrate` is split by what it was running: the migration transaction +(`001` and native `002` to `023`, `025` and `026`), 002's share-hash backfill, 013's and 024's +concurrent index builds, and 017's prepare, validate and swap. The transaction holds the +cutover locks, ACCESS EXCLUSIVE on `qbit_share_ledger` among them, for its +whole length; the backfill's batches hold only ACCESS SHARE on it. The report +also lists the statements `migrate` spent the most sampled time in. + +**Rehearsing without a snapshot.** `make prism-cutover-rehearsal-dump +OUT=mainnet.dump SCALE=1 DENSITY=0.0625` writes a mainnet-shaped `2.x.x` +database from the frozen release SQL and dumps it, with its audit bodies in +`mainnet.audits` and the ledger key to pass as `LEDGER_KEY` printed in its +summary. Its shape is #521's aggregates of mainnet `2.x.x` from 2026-07-15 to +2026-09-28, seeded and deterministic: + +- 257 payout identities over 76 days in three eras, from 4 to about 130 of + them a day; +- a whale holding 82-98% of a day's work on most days over a Zipf(1.1) tail; + worker difficulty over three orders of magnitude; hobby rigs whose credit + accrues under the payout floor; +- 5-minute share rates with mean, maximum and coefficient of variation per + era of 8/92/1.24, 25/394/0.65 and 1.3/17/0.77 shares per second; +- 13,566 blocks at mainnet's work per block, with signed audit bundles in the + three historical body layouts, PPLNS carry-forward chains and payout + entries, the newest blocks immature (pending payouts) and some inactive, + reversed and rejected; +- CTV fanout rows for the last 40 days, terminal v1 and (after the #258 + upgrade) v2 outbox rows, abandoned candidates, caught-up hashrate rollups, + retained worker difficulty, an expired writer lease, and the + `qbit_share_ledger_credit_policy_check` left NOT VALID as on a ledger + upgraded from before that column. + +`make prism-cutover-rehearsal-generated SCALE=1 DENSITY=0.0625` does both in +one command: it writes the dump under `WORKDIR` (a temporary directory, +removed afterwards unless `KEEP=1`) and rehearses it with the generator's +ledger key, taking `STATEMENT_TIMEOUT_MS` and `REPORT` as above. Use it to +rehearse a branch that changes a migration. + +`SCALE=3` multiplies the identities, the rate and the blocks, which is #521's +growth shape. `DENSITY` keeps that fraction of the share rows, and each kept +row carries `1/DENSITY` times a share's difficulty. The work per identity per +5 minutes, the window weight and every balance therefore keep mainnet's +shape, while the row-proportional costs (the 013 and 017 scans, 002's +share-hash backfill and the evidence export) scale with the rows the report +states. At `DENSITY=1` it is mainnet's 65.5M rows, about 105 GB before +migrating. + +In CI, `migration_rollback`'s +`cutover_rehearsal_tests::mainnet_shaped_2x_dump_rehearses_the_cutover_and_keeps_every_balance` +runs this path on every pull request: a mainnet-shaped ledger of about 25,000 +rows and 40 blocks, dumped with `pg_dump` and rehearsed as above, with a +payout window larger than one 4,096-row page. The weekly +`live_regtest` scenario +`migration_lifecycle_tests::weekly_mainnet_shaped_2x_ledger_cuts_over_measured_mines_and_restores_in_isolation` +writes the mainnet shape at 1/16 of the rows, about 4.1M, over a real regtest +chain. It cuts over with the same measured steps and checks, at the server's +default statement timeout, and requires 002's share-hash backfill to have run +as its own measured step (#582); then it mines and reconciles as the scenario +above does. diff --git a/docs/prism-session-sequence.md b/docs/prism-session-sequence.md new file mode 100644 index 000000000..3451dfd78 --- /dev/null +++ b/docs/prism-session-sequence.md @@ -0,0 +1,105 @@ +# Four-byte session allocation (migration 009) + +The first valid `mining.subscribe` reserves one four-byte `extranonce1`. +Connect-only traffic consumes no sequence values, and repeated subscriptions on +the same connection retain their original value. The wire response and stored +job payloads retain their previous format. + +The sequence cycles through 1–4,294,967,295. Each allocation attempt atomically +inserts a reservation keyed by its numeric value, or replaces a reservation +whose owner explicitly reports `{"state":"stopped"}`. Before committing, it +also checks for unexpired job rows with that hexadecimal `extranonce1`, including +pre-009 rows. A collision rolls back that attempt and advances to the next +candidate. There is no global allocator lock and no transaction spans multiple +attempts. After 1,024 occupied candidates, the client receives Stratum error 20 +with reason `session-allocation-exhausted` and can retry subscription; no partial +subscription is published. + +Each Ledger process incarnation has a random owner token, recorded in its +reservations and heartbeat status. A stopped marker must match that token as +well as the instance ID, so another process using the same instance ID cannot +make live reservations reclaimable. Reporting stopped atomically closes local +allocation admission and requires zero pending or active session guards; a +shutdown that has not finished cancelling sessions cannot publish the marker. +All clones of the Ledger share that admission state. + +The session owns a non-cloneable guard. Disconnect and task cancellation schedule +deletion of only that guard's reservation token; a late cleanup cannot delete a +replacement reservation. Unexpired persisted jobs independently block reuse +after the guard is released. Reservations do **not** expire based on elapsed +time: a connected miner's current in-memory job can outlive its database row on +an unchanged tip. Neither a stale heartbeat nor an absent instance row proves +that a session has ended. + +## Upgrade and compatibility + +Stop and drain **all pre-009 frontends before applying migration 009**. The old +binary calls unchecked `nextval`, so an old and a new frontend must not run +together after cycling is enabled. This is a coordinated stop/drain upgrade, +not a mixed-version rolling upgrade. No job row is rewritten and the old binary +can still read the job schema and four-byte wire format, but must not allocate +sessions against the migrated sequence. + +Start one or more new frontends with the existing migration runner enabled. +Its transaction advisory lock serializes concurrent migrations, and 009 is +recorded after the 002–006 and 008 migrations. Number 007 remains reserved +for another workstream; the runner checks applied versions individually, so +009 does not conceal a lower-numbered gap when that migration is integrated, +and a database at 2, 3, 4, 5, 8 and 9 still receives 006 on its next migrate. +Every start also requires each of those migrations (the startup gate in +`docs/prism-rust-migration.md`), so a frontend with initialization disabled +refuses a database missing 009. Before starting one, verify on the writer: + +```sql +SELECT version FROM qbit_prism_schema_migrations WHERE version = 9; +``` + +The result must contain 9. Migration DDL and its version record commit atomically; +the job lookup index can briefly block writes during creation. A lock or statement +timeout rolls the migration back and can be retried. Two new frontends can +migrate/start concurrently under the existing migration lock. + +## Retained reservations and recovery + +Normal disconnects release reservations. Normal server shutdown drains/aborts +sessions before publishing `stopped`, which also makes any missed cleanup +eligible for reuse at allocation. A crash, a lost commit response, a failed +cleanup, or runtime termination can retain reservations conservatively. + +For a crashed owner, first prove that its instance ID is not running **anywhere** +and prevent that process from restarting during the operation. On the writer, +mark that owner stopped (substitute the verified instance ID): + +```sql +UPDATE qbit_prism_instances +SET status = jsonb_set(status, '{state}', '"stopped"'::jsonb), + heartbeat_at = clock_timestamp() +WHERE instance_id = 'verified-stopped-instance'; +``` + +Preserve `session_owner_token` in that status: the allocator will reclaim only +that process incarnation's reservations as candidates are encountered, while +retaining protection from unexpired jobs. If the heartbeat row is absent or +there are older incarnations, their reservations stay protected. To remove all +reservations for the verified-stopped instance ID immediately, under the same +precondition that **none** of its processes can still run or restart: + +```sql +DELETE FROM qbit_prism_session_reservations AS r +WHERE r.instance_id = 'verified-stopped-instance' + AND NOT EXISTS ( + SELECT 1 FROM qbit_prism_jobs AS j + WHERE lower(j.payload->>'extranonce1') = lpad(to_hex(r.extranonce1), 8, '0') + AND j.expires_at > clock_timestamp() + ); +``` + +Retention consumes IDs, not every successful connection over the process's +lifetime: ordinary disconnect cleanup frees them. With the default 384-connection +cap, losing every active reservation in 100 crashes per day would retain 38,400 +IDs per day; exhausting 4,294,967,295 values at that rate takes about 306 years. +That illustration excludes additional failed disconnect cleanups, which can +accumulate during outages and should be recovered using the procedure above. +If automatic crash reclamation becomes necessary, use renewable leases with +session and job-write fencing; increasing a heartbeat-age threshold is not a +safe substitute. diff --git a/docs/prism-share-append-admission.md b/docs/prism-share-append-admission.md new file mode 100644 index 000000000..e6b1b8fdd --- /dev/null +++ b/docs/prism-share-append-admission.md @@ -0,0 +1,66 @@ +# Share append admission + +Share append transactions acquire bounded admission **before** checking out a +database connection. For a pool of `N` connections, at most `N - 1` appends can +hold connections, including connections still being cleaned up after an error +or cancellation. Frontend construction supports pools of at least two +connections, so even the smallest frontend pool leaves one connection available +to other work. A caller-supplied one-connection pool still permits one append; +it cannot provide concurrent read headroom. + +The cap follows pool identity. Ledger clones and separately constructed ledger +handles assigned the same public pool share one allowance. Separate pools using +the same database URL have separate allowances. A weak registry retains no idle +pool after its outstanding appends and cleanup finish. + +This prevents appenders waiting for `ORDER_LOCK` from consuming every frontend +connection needed by refresh. It does not reserve an exclusive refresh connection: +other database users can consume the remaining capacity, and work that needs an +accounting lock must still wait for that lock. Pool size, lock ordering and SQL +authority checks are unchanged. + +Candidate preparation precedes admission. The submission's original deadline +still covers admission and persistence; queueing and the existing single retry +for a missing partition create no new deadline. A cancelled queued append has +not started SQL. After checkout, the transaction borrows a connection guard that +keeps admission until SQLx has returned or closed that connection. Dropping a +transaction only queues rollback, so releasing admission at caller cancellation +would be too early. The same rule covers an uncertain COMMIT reply; admission +does not classify that outcome, retry it, or suppress cleanup warnings. An +aborted append keeps its server-side `ORDER_LOCK` queue position until the lock +is granted and the queued rollback runs, exactly as before; abort does not free +the slot promptly. The guard spawns that cleanup on the runtime handle captured +at admission, so with the ledger's own pool (no `min_connections`, the +default) a future dropped from a thread outside the runtime context still +cleans up without panicking while the runtime lives; on a shut-down runtime +the cleanup is cancelled and the floated connection and permit are released by +ownership. A caller-supplied pool with `min_connections > 0` still reaches +SQLx's own off-context spawn after the cleanup is queued and panics there +exactly once, as it did before admission existed; that cleanup still finishes. +Off-runtime drops of SQLx guards are not supported in general. + +One failure mode changes: the pool's `acquire_timeout` used to fail an append +with `PoolTimedOut` when every connection was busy. Admission now waits for a +permit with no bound of its own, so an append that its caller follows rather +than refuses at the acknowledgement deadline (a candidate-bearing share) waits +for its turn instead of failing after that timeout. Checkout after admission is +still bounded by `acquire_timeout`. + +The missing-partition retry releases the first attempt through that cleanup, +attaches the partition lead outside append admission, and reacquires for its +single retry. Revision and pre-COMMIT fences still run inside the transaction. +An exact match to an already durable share keeps its existing duplicate outcome. +Miner rejection identifiers and successful-credit deduplication are unchanged. + +`append_admission` integration tests hold the actual advisory key on durable +PostgreSQL and queue more appends than the frontend pool can hold. They check +control-read progress, cancellation on and off the runtime, shared-pool handles, +revision/gate changes, lost COMMIT replies, durable replay and missing-partition +recovery. Pool checkout telemetry continues to measure actual checkout; it does +not include the new admission wait, so checkout saturation under-reports append +queueing. + +External-tip latency (node tip observation through automatic refresh, issuance, +persistence and first client delivery under concurrent share load) is a separate +measurement with its own warm-up, coverage and reconciliation rules; it is not +the explicit-refresh delivery benchmark. diff --git a/docs/prism-share-ledger-partitioning.md b/docs/prism-share-ledger-partitioning.md new file mode 100644 index 000000000..832f988ba --- /dev/null +++ b/docs/prism-share-ledger-partitioning.md @@ -0,0 +1,401 @@ +# Share ledger partitioning and retention (design record, #144) + +This record resolves decision **D6** of #260 (retention and archival of +shares and audit bodies) and specifies what migrations 016 and 017, the +`share-archive` operator command and the reader changes of #144 do. The +first slice, the index trim (#153, migration 013), is recorded in +[prism-ledger-ops.md](prism-ledger-ops.md#share-ledger-indexes). The +migration mechanics are in +[prism-rust-migration.md](prism-rust-migration.md#migration-017-the-share-ledger-partition-conversion-applied-online), +the operator procedure in +[prism-ledger-ops.md](prism-ledger-ops.md#share-ledger-partitions-and-retention). + +## Why + +`qbit_share_ledger` is one row per accepted share and nothing ever left it. +Every insert maintained the primary key, a global `UNIQUE (share_id)` and +four secondary indexes at ever-growing depth; vacuum, base backups, restore +drills and cache hit rates all degraded with lifetime share count. What +reads the table is narrow: the payout window (newest slice, difficulty +bounded), the audit range of each landed block, dashboard aggregates over +the last 24 hours, and a handful of probes by `share_id` seconds after the +share was written. Nothing needs an aged share row online, except the +audit of a block whose window it belongs to, and since #267 that audit is +rebuilt from the ledger at read time. + +## Decision D6 + +**Online horizon.** A share row stays online while any of these holds: + +1. it can still be in a payout window: its `share_seq` is at or above the + floor of the current window taken at four times the requested weight + (`qbit_prism_window(clock_timestamp(), 8 * D * 4)`, D the network + difficulty the operator supplies), so a difficulty rise of up to 4x + between two retention runs cannot reach into archived history; +2. it is younger than the retention age (default 30 days), which covers + every dashboard read of raw rows (the longest is 24 hours) with margin; +3. the hashrate rollup watermark (`qbit_hashrate_rollup_progress`) has not + passed it, so the permanent rollup tables hold its contribution; +4. a landed block's audit still depends on it: an audit row whose share + snapshot intersects the partition and that has no stored + `canonical_audit_bytes` (see the next point); +5. an unfinished block candidate or deferred share references it. + +Everything above is a property of a whole partition, checked by +`share-archive plan`, and a partition leaves only when all five clear. + +**Archived blocks are served from stored canonical bytes (option 2 of the +D6 discussion).** Before a partition is detached, every audit row whose +snapshot intersects it is *sealed*: the canonical artifact is rebuilt from +the still-online shares, its digest checked against the advertised +`audit_bundle_sha256`, and the bytes stored in `canonical_audit_bytes`, +exactly as imported legacy audits are stored since #325. Readers prefer +stored bytes over reconstruction whatever the row's shape, so the artifact +route, the audit bundle route and the operator tools keep serving the +block with its advertised digest after its shares are gone. The price is +the artifact size back per archived block (about 111 MB uncompressed at +100,000 shares, TOAST-compressed in the column); the alternative, pinning +every landed block's window online forever, would have left cost scaling +with blocks found, against the point of this work. Sealing is idempotent +and can run ahead of any detach. The snapshot metadata row +(`qbit_prism_audit_snapshots`) stays: it records the range and digest the +bytes were proved against. + +**Archive format.** A detached partition is written, before it is +detached, as newline-delimited canonical JSON rows (every column of the +table, exact timestamps in microseconds) compressed with gzip, plus a +manifest carrying the bounds, counts, digests and a hash-chain link to the +previous archived partition. The format is specified +[below](#archive-format-v1). A restore recreates the partition table from +the archive, verifies count and digest, and can attach it back for +inspection or service. + +**Retention is detach-and-archive, never delete.** No `DELETE` runs on the +ledger; the immutability triggers stay on the parent and on every leaf. +Space is reclaimed by `DETACH PARTITION ... CONCURRENTLY` followed, after +the archive has been re-read and verified against the live rows, by +`DROP TABLE` of the detached table, which reads the archive back once more +first. The archive is the copy of record from +then on, and the catalog row (`qbit_prism_share_partitions`) keeps the +bounds, the archive location, the digests and the timestamps of each step. + +## The shape + +- **`RANGE (share_seq)`.** `share_seq` is the routing key every ordered + read already carries, `bigserial` routing cannot fail, and `PRIMARY KEY + (share_seq)` survives on the parent. No DEFAULT partition: a missing + partition would otherwise silently collect rows that a later ATTACH + would have to move. +- **A grid of cells.** Partition width is `partition_rows` in + `qbit_prism_share_partitioning` (default 2^24 = 16,777,216 rows; about + 7.5 GB of heap and 20 GB of indexes at the production row shape, five + days at 39 shares/s, nine hours at 500 shares/s). Each new partition is + one width wide and starts where the highest attached one ends, so with + the width unchanged cell k covers `[k*rows, (k+1)*rows)`. Names are + `qbit_share_ledger_p<n>`, n from a counter over every name the catalog + has recorded (`qbit_prism_share_partition_next_number()`), so a name is + never reused and never depends on the width. Changing the width affects + partitions created afterwards. The release table becomes + `qbit_share_ledger_p0`, `[MINVALUE, bound)`, spanning as many cells as + it needs. +- **Lead partitions.** `qbit_prism_share_partition_ensure()` keeps + `lead_partitions` (default 4, about 67 M rows) attached above the next + `share_seq`. The server calls it at startup and every + `PRISM_SHARE_PARTITION_ENSURE_INTERVAL_SECONDS` (default 60) from every + instance; the call is serialized per schema and creates nothing when the + lead is intact. A new partition is a standalone table `LIKE` the parent + with a validated bound `CHECK`, its own `UNIQUE (share_id)` and the + immutability trigger, then attached (SHARE UPDATE EXCLUSIVE on the + parent: appends and reads continue). Should an insert ever find no + partition (SQLSTATE 23514), the append path runs `ensure` once and + retries the share. +- **Per-leaf indexes** are the release set after 013: the primary key, + `accepted_recent_idx`, `accepted_block_suffix_idx`, + `accepted_seq_walk_idx`, `accepted_miner_history_idx`, plus the leaf's + `UNIQUE (share_id)`. Each leaf's indexes are cache-resident while it is + hot and are never touched again once it is not. +- **The catalog.** `qbit_prism_share_partitioning` (one row: width, lead, + the conversion bound and timestamp) and `qbit_prism_share_partitions` + (per partition: bounds, `attached` / `detached` / `dropped`, sealed, + archived, verified, detached and dropped timestamps, archive URI and + digests). PostgreSQL's `pg_inherits` is the authority on what is + attached; the catalog records what happened to a partition afterwards + and is cross-checked by every command. + +## `share_id` uniqueness after partitioning + +A unique index on a partitioned table must include the partition key, so +the global `UNIQUE (share_id)` cannot exist on the parent. Three tiers +replace it: + +1. **`qbit_prism_share_hashes` is the global authority.** The native + append already writes one row per share there, keyed by the header + hash with `UNIQUE (share_id)`, in the same transaction as the ledger + row. It is not partitioned, its rows are never removed, and it stays + O(1) per share. The append now consults it *first*: a header hash it + holds under another `share_id` is the cross-identity duplicate, refused + as before. It then probes the ledger for the `share_id` with the bound + of tier 3 (at most three leaves), which finds an exact replay or a + payload mismatch of any recent row, including rows written around the + append path, which the hash table does not know. Only when the hash + table holds the header under this `share_id` and the bounded probe + misses is the probe repeated without the bound, so a replay of any + online row is still matched exactly; a replay whose row has left the + online ledger is refused as the duplicate it is. The coordinator's + replays are seconds old, so that last case is unreachable in practice + and safe if reached. + Rejected rows reserve their exact IDs separately in + `qbit_prism_rejected_share_ids`, without reserving their header hashes. + Verification records those IDs and sequences atomically with + `archive_verified_at`, before detach is permitted; fresh imports register + them atomically with attachment. These records survive detach and drop. + Append uses the retained sequence to compare an online rejected row and + refuses its ID when the row is archived. Unregistered legacy rejected rows + remain covered by a lookup below the conversion bound. +2. **Per-leaf `UNIQUE (share_id)`** on every partition, created explicitly + by the partition procedure (a `PARTITION OF` table would get none). +3. **Bounded probes.** Every `share_id` lookup on the ledger (the replay + comparison, the block-only reconciliation probes, the vardiff evidence + lookup) carries + `share_seq >= qbit_prism_share_probe_floor()`: the lower bound of the + attached partition two below the one the next `share_seq` lands in, + read from the catalog, so PostgreSQL prunes to at most three leaves + holding rows (plus the empty lead) at executor start (verified: + "Subplans Removed" in the plan) instead of descending one `share_id` + index per attached partition. The floor follows the bounds actually + attached, not `partition_rows`: subtracting two current widths from + the sequence would span every narrower partition still attached once + the width was raised. A probe without the bound costs one index + descent per partition per share, which the design spike measured at + 31 buffers against 8; the two lookups that fall back to an unbounded + probe on a miss (the credited replay above, and the vardiff evidence + lookup) do so only for a row older than the floor, at least two full + partitions of rows, off the share path. + +The invariant this leaves is stated plainly: a row inserted *around* the +append path (a direct `INSERT` by an operator or a harness) with a +`share_id` already present in another partition is not refused by +PostgreSQL. The archive verification and the `plan` command report such +duplicates across attached partitions; the native writers cannot create +one. + +The two inbound foreign keys onto `share_id` (the 001 outbox key and the +002 `qbit_prism_share_hashes` key) are dropped by 016: a foreign key onto a +partitioned parent's non-key column cannot exist, and PostgreSQL +re-validates inbound keys at DETACH, so a single retained outbox row would +pin its block's partition forever. Rule from here on: **no object may +carry a foreign key to `qbit_share_ledger`**. + +## The conversion (migration 017) + +Shape 2 of the design spike, priced by prototype and measured again here: +the release table is attached as the first partition, nothing is copied, +no index is rebuilt. + +1. **Prepare.** `CHECK (share_seq < bound) NOT VALID` on the release + table, `bound` the first grid boundary at least two partition widths + above the next `share_seq` (33.5 M rows: nineteen hours at 500 + shares/s, ten days at 39). The constraint is enforced on new rows at + once. The sequence is never restarted: rows keep landing in the release + table until it reaches the bound, then in the next cell, so `share_seq` + stays gap-free. Every name the swap takes is checked before this step. +2. **Validate.** `ALTER TABLE ... VALIDATE CONSTRAINT`: one scan, SHARE + UPDATE EXCLUSIVE, appends and reads continue; hours on the production + ledger. If a run is interrupted and resumed after the sequence has come + within one width of the bound, the bound is moved out and validated + again. +3. **Swap.** One transaction: rename the table and its six indexes to + their `_p0` names; create the parent `LIKE` the release table (same + columns, defaults, `CHECK` constraints under the same names, which is + what ATTACH matches on), give it the primary key and the four secondary + indexes under the release names, move the sequence's ownership and the + grants; `ATTACH PARTITION ... FROM (MINVALUE) TO (bound)`, which proves + the bound from the validated constraint and adopts every existing index + (the function refuses if any index was built instead); recreate + `qbit_shares_since_template_height`, the one release function bound to + the old row type; install the immutability trigger on the parent; + record the catalog; create the lead partitions. Measured on the + container: 16 ms at 5,000 rows and none of its terms is a function of + the row count. The ACCESS EXCLUSIVE lock is requested with a two-second + `lock_timeout` and retried for up to ten minutes, so a long read delays + the swap without holding the appends queued behind the request. + +Where the ledger is empty (a fresh deployment, an empty 2.x.x source) the +three steps run inside the migration transaction under the cutover locks; +a fresh deployment's first partition is one cell. Everywhere else they run +after the commit on a dedicated connection, resumable from whatever stage +the database is in, and 16 is recorded last. Until then every start +refuses the database. + +**One-way, per D5.** The revert is a second cutover: it must rebuild a +global `UNIQUE (share_id)` over the whole table under ACCESS EXCLUSIVE +(43 s at 5.5 M rows in the spike, proportional to the table), and it fails +at the end on the first duplicate `share_id` accepted across two leaves. +No revert script ships; recovery is the isolated-restore reconciliation of +#287. + +## Every native reader, and the partition key + +| Reader | Predicate | After 017 | +| --- | --- | --- | +| `Ledger::snapshot` cutoff, `max(share_seq)` | ordered walk | ordered append, newest leaf first, stops at the first row | +| `Ledger::snapshot` page walk, `qbit_prism_window` pages, `qbit_audit_share_window` | `share_seq < cursor ORDER BY share_seq DESC LIMIT 4096` | pruned by the cursor; older leaves never executed | +| landing durable-range proof, `read_range`, `read_range_paged`, `probe_share_rows`, `durable_range_exists`, the landing count | `share_seq` range | pruned to the leaves the range covers | +| `rollups.sql` batch | `share_seq > watermark` | pruned | +| `dashboard_hashrate_rollups.sql` tail | `share_seq > watermark` | pruned; the boundary pass keeps its documented full index-only scan (a query change, not this one) | +| replay comparison in `append_in`, block-only reconciliation probes, `share_accepted_at_ms` | `share_id = $1` | consult `qbit_prism_share_hashes` first, or carry `share_seq >= qbit_prism_share_probe_floor()`: at most three leaves | +| `dashboard_leaderboard.sql`, `dashboard_pool_snapshot.sql`, `dashboard_miner_share_summary.sql`, `dashboard_miner_worker_rows.sql`, `dashboard_hashrate_series.sql` (raw fallback) | `accepted_at` range, 3 h to 24 h | one `accepted_recent_idx` / `miner_history_idx` descent per attached leaf, empty for every leaf outside the range: O(attached partitions) buffer reads per request, a few hundred at most with retention in place; shown not to need the key | +| block solver lookups in blocks, leaderboard, reward leaderboard, pool snapshot | `lower(right(share_id, 64)) = block_hash` | no longer read the ledger: `qbit_pool_blocks.solver_*`, written at landing and backfilled by 016 | +| `latest_evidence` lifetime counts | none | served from the permanent rollup tables plus the raw tail above the watermark | +| `tools.rs` latest miner | ordered walk | newest leaf | +| `read_schema_ready.sql` | `to_regclass` | a partitioned parent resolves | + +## Lifetime-scoped consumers and what a detach changes + +The design spike's nine Python consumers reconcile to these native ones: + +- **Block solver attribution** (four dashboard queries): moved onto + `qbit_pool_blocks` by 016 and written at landing. Unaffected by detach. +- **`accepted_share_count` and `distinct_miner_count`** in + `/audit/latest-evidence`: read from `qbit_hashrate_rollup_pool` and + `qbit_hashrate_rollup_miner` at the daily grain plus the raw tail, which + the rollup sweep folds before any partition can leave (rule 3 above). + Unaffected by detach. +- **`GET /public/v1/hashrate-series?range=all`**: served from the rollups + whenever the rollup tables exist (always, on 3.x.x). Unaffected. +- **A miner's `last_share_at`** in the miner share summary: the newest + share within the online horizon. A miner whose last share is older than + the retention age reads `null`. Documented change. +- **`/audit/share-window?anchor=`** for an anchor inside an archived + range: `rows: []`. The shares are in the archive and in every sealed + artifact whose window covers them. Documented change; an anchor-to- + archive index is deferred. +- **Pool readiness** never depended on a lifetime miner count on 3.x.x + (`PRISM_MIN_READY_MINERS` is retired), <!-- retired-setting: PRISM_MIN_READY_MINERS --> + so the spike's collection-mode regression has no native counterpart. +- **Audit reconstruction** of a landed block: sealed before detach (D6). + +## Retention procedure + +`qbit-prism-server share-archive <command>` runs as the operator against +the primary, with the frontends running: + +| Command | Effect | +| --- | --- | +| `plan --network-difficulty D [--retention-days N] [--window-multiple M] [--check-duplicates]` | every partition with its bounds, row count, age, and each of the five conditions above with its blocker named; nothing is changed | +| `seal <partition>` | stores canonical bytes for every audit row whose snapshot intersects the partition and has none, verifying each against its advertised digest; records `sealed_at` when none is left | +| `archive <partition> --dir <root> [--force]` | writes `<root>/qbit_share_ledger/<partition>/<manifest-sha256>/rows.ndjson.gz` and `manifest.json`, records the URI, digests and row count; refused while the share sequence has not passed the partition, since appends could still land in it, and refused out of order, so the chain of manifests stays contiguous, and refused over a predecessor whose archive is not verified, so every link is to a certified manifest; `--force` writes an archive again, clearing its verification and that of every later archive, which must then be written and verified again in order, each over its verified predecessor, and is refused once a later archived partition has left the ledger | +| `verify <partition> --dir <root>` | re-reads the archive, checks both digests and that the manifest chains, without a gap, to the nearest archived partition, whose own archive has to be verified, and, while the partition is attached, streams the live rows again and compares; records `archive_verified_at` only for that full comparison, and only once the share sequence has passed the partition, so a verify after the detach reports but never counts as the proof the detach required | +| `detach <partition> --network-difficulty D [--retention-days N] [--window-multiple M] [--check-duplicates]` | requires every plan condition, sealed, archived and verified, and counts the live rows against the archive again; `DETACH PARTITION ... CONCURRENTLY`, or FINALIZE for an attempt interrupted after PostgreSQL marked the partition detach-pending, held to every condition again because the mark does not tell a `share-archive detach` from a statement run by hand; the table stays as a standalone relation | +| `drop <partition> --dir <root>` | requires `detached` and verified; reads the recorded archive back from disk, checking both digests against the catalog, and counts the rows against it again; `DROP TABLE`; the archive is the copy of record | +| `restore <manifest> --dir <root> [--attach]` | recreates the partition table from the archive, verifies count and digests, and optionally attaches it under its recorded bounds | + +`plan` also reports the attached partition count, the lead ahead of the +sequence, and, with `--check-duplicates` (one pass over every attached +leaf, so not the default), any `share_id` present in more than one attached +leaf; `detach` with that flag refuses while such a duplicate exists. A +partition that is not attached gets `not_applicable` for each condition +rather than `clear`, and a condition whose evidence is missing (no rollup +watermark row, a catalog row without its relation) is reported as unknown, +never as clear. Every command prints JSON. `restore` creates the table, +loads the rows, re-streams the digest and, with `--attach`, attaches and +records the catalog in one transaction, so a failed restore leaves nothing +behind; `detach`, `drop` and the restore run their DDL without statement or +lock timeouts, as the migration runner does. + +While a partition carries the detach-pending mark, PostgreSQL hides its rows +from every new query of the parent. The payout window is read from the +parent, so `plan` counts the hidden partition's accepted rows into that +condition itself: a window that ran out of visible rows before reaching its +weight still reaches into the partition and blocks it. A pending partition +that fails a condition is brought back first; the refusal prints the +`FINALIZE` and `ATTACH PARTITION` statements that do it. The rollup sweep +reads the parent too and cannot fold rows it does not see, so a partition +whose rows are not yet folded must never be detached by hand: the watermark +passes them while they are hidden and nothing can show it afterwards. + +## Archive format v1 + +``` +<root>/qbit_share_ledger/<partition_name>/<manifest-sha256>/ + rows.ndjson.gz gzip; one JSON object per line, share_seq ascending + manifest.json the record below, UTF-8, no trailing newline +``` + +Each write creates a new version directory. The catalog switches to it only +after both files and their directory entries are durable. Prior versions stay +on disk, including after `--force`, so a failed catalog update cannot destroy +the recorded copy. Verification also accepts existing v1 archives in the +original layout without the digest directory. + +A row is the whole ledger row with a fixed key order: + +``` +{"share_seq":1,"share_id":"alice:…","miner_id":"alice","payout_order_key":"…", + "p2mr_program_hex":"…","share_difficulty":"1","network_difficulty":"100", + "template_height":100,"job_id":"job","job_issued_at_us":1000000, + "accepted_at_us":1758000000000000,"ntime":1,"accepted":true,"reject_reason":null, + "credit_policy":null,"writer_id":"…","writer_epoch":0} +``` + +Difficulties are decimal strings (`numeric(78,0)`), timestamps are +microseconds since the epoch (exact for `timestamptz`), the program is +hex. `rows_sha256` is the SHA-256 of the uncompressed byte stream, so a +verifier can stream the file without materializing it. + +The manifest: + +| Field | Meaning | +| --- | --- | +| `schema` | `qbit.prism.share-archive.v1` | +| `partition_name`, `lower_seq` (null for MINVALUE), `upper_seq` | the recorded bounds | +| `row_count`, `first_share_seq`, `last_share_seq`, `first_accepted_at_us`, `last_accepted_at_us` | what the rows hold; zero rows is a valid archive | +| `rows_sha256`, `rows_gz_sha256`, `rows_bytes`, `rows_gz_bytes` | digests and sizes of the stream and of the file | +| `previous_manifest_sha256`, `previous_upper_seq` | the chain: the archived partition with the next-lower `upper_seq`, or null for the first; a gap between `previous_upper_seq` and `lower_seq` means a partition is missing from the chain, which `archive` refuses to write and `verify` refuses to certify; both also refuse a link to a predecessor whose archive is not verified, so the chain is written and certified in order and a verification stands for every link below it | +| `schema_versions` | `qbit_prism_schema_migrations` at archive time | +| `created_at`, `created_by` | UTC timestamp and the operator tool's instance id | + +The manifest's own SHA-256 is what the catalog and the next manifest +record, and a manifest is accepted only when it is the canonical encoding +of itself (re-serializing it reproduces the bytes on disk), so a hand +edit is detected before any digest is compared. `created_by` is the +operator process's `PRISM_INSTANCE_ID`. Signing manifests with the ledger +key is a possible extension; the public artifact digests already commit +to every share a block paid on. + +## Measurements + +Container evidence (PostgreSQL 16.15 in Docker on an Apple M-series +laptop, a synthetic ledger of 1,000,000 production-shaped rows, 1,000 MB +with its indexes, `VACUUM (ANALYZE)` before each read; 2,000 single-row +inserts timed from PL/pgSQL with `clock_timestamp()`): + +| Measurement | Before 017 | After 017 | Notes | +| --- | ---: | ---: | --- | +| prepare (bound `NOT VALID`) | | 2.3 ms | catalog only | +| validate (one scan) | | 47 ms | SHARE UPDATE EXCLUSIVE; scales with heap size, about 300 MB here | +| swap | | 16.7 ms | rename, parent, five indexes adopted, attach, lead partitions; none of its terms is a function of the row count | +| insert latency p50 / p95 / p99 | 0.031 / 0.048 / 0.066 ms | 0.032 / 0.047 / 0.069 ms into the release partition; 0.025 / 0.038 / 0.061 ms into a fresh lead partition | a hot leaf's indexes are small and cache-resident | +| `VACUUM (ANALYZE)` of the ledger | 0.32 s | 0.44 s for all partitions; 0.02 s for one lead partition | retention keeps the set vacuum has to visit bounded | +| page walk, `qbit_prism_window` over 200k shares | 15,809 shared buffers | 17,353 shared buffers | same plan shape through the partitioned parent | +| bounded `share_id` probe | | 8 shared buffers over three leaves | `Subplans Removed` appears once the floor clears a partition (shown in the prototype at 100 M) | + +They are lock-semantics and phase-shape evidence, not production +absolutes. The production run of the same measurements +(insert latency percentiles from `/metrics`, `VACUUM (VERBOSE)` duration, +`pg_stat_user_indexes` sizes, the page-walk `EXPLAIN`) before and after +017 on a production-sized copy is acceptance criterion 4 of #144 and is +listed as a runbook in the operations guide; it needs production access +and is left to the operator. + +## Deferred + +- Pruning `qbit_prism_share_hashes` to the replay horizon (it stays + lifetime-scoped by design, about 150 B per share, and is the uniqueness + authority). +- Dropping `accepted_block_suffix_idx` once production confirms every + block carries its solver columns (its only readers were the four + queries 016 rewrote). +- An anchor-to-archive index for `/audit/share-window`. +- Bounding the dashboard `accepted_at` scans by `share_seq` from the + catalog (`accepted_at` is monotone in `share_seq` for native rows). diff --git a/docs/prism-soak.md b/docs/prism-soak.md new file mode 100644 index 000000000..cb4ed63ab --- /dev/null +++ b/docs/prism-soak.md @@ -0,0 +1,353 @@ +# Long soak (#575 item 2) + +Before #575 the longest automated PRISM run was about twelve minutes. This +page covers the three pieces that hold a server to hours and days instead: + +1. **`--plan soak`** in the load harness (`qbit-prism-load`): one server + lifetime on regtest, looping checked-in presets' workloads for hours, with + forced share-ledger partition rollovers and the operator's retention cycle + running under the load. +2. **The weekly soak**, the `soak-weekly` preset, run by + `.github/workflows/prism-load-nightly.yml` once a week for about 5.5 h. +3. **`qbit-prism-soak-report`**, the same gates applied hourly to a live + deployment (testnet4) through Prometheus and a read-only database URL. + +All three write the same per-sample record and are judged by the same code +(`crates/qbit-prism-load/src/soak.rs`), so a regtest soak and a multi-day +testnet4 soak mean the same thing by "passed". + +## What a soak is held to + +Each sample records, per server process, resident memory, open file +descriptors and threads; for the database, connections per client, +connections idle in a transaction for over a minute, `pg_wal` size, the next +`share_seq`, every share-ledger partition with its state and size, the +twenty largest other relations, and the payout divergence rows (#478); and +the share ACK latency since the previous sample. A value that could not be +read is `null` with the reason beside it, never zero, and a gate whose inputs +are unknown fails (EP-OBSERVABILITY). + +| Gate | Holds | +|---|---| +| resident memory slope | the peak of each whole `rss_trend_window_minutes` window after the warm-up, per process; the least-squares slope over at least `min_trend_windows` peaks, at most `rss_slope_mib_per_hour_max` | +| resident memory against the warm-up peak | the bound of [prism-capacity-readiness.md](prism-capacity-readiness.md#the-resident-set-bound): after the warm-up, at most `rss_warmup_peak_multiple_max` (2.0) times the warm-up's peak | +| open file descriptors slope | the same fit, at most `fd_slope_per_hour_max` | +| database connections | per client, never above `pool_connections_max` | +| database connection drift | mean of the last quarter of post-warm-up samples minus the first quarter's, at most `pool_connections_drift_max` | +| idle in transaction over 60 s | zero at every sample | +| WAL size | `pg_wal` never above `wal_bytes_max` | +| share partition rollovers | partition bounds the share sequence crossed, at least `min_rollovers` | +| share partitions archived and dropped | partitions that left the ledger through `share-archive`, at least `min_archive_cycles` | +| one server lifetime | one PID per process for the whole soak (`one_lifetime`) | +| payout divergences | no new row during the soak | +| acknowledged shares in the ledger | deployments only; see below | + +On regtest the harness's own gate still applies on top: exit 0, which is an +exact reconciliation of every offered, acknowledged and committed share, no +durability finding, no shortfall in any phase, and the tip-delivery budgets. + +**A known resident-memory failure.** `rss_expected_failure` names an issue, +or is `null`. While it names `<issue>`, the resident-memory rows of every +process are an expected failure as a group. A measured failure is reported as +`expected failure (<issue>)` and does not fail the soak. A soak in which +every resident-memory row passed fails with `<issue> looks fixed`, so the key +goes back to `null` and the gate is real again. It is the group, not each row, +because one frontend's warm-up ratio can pass while its slope, or the other +frontend's rows, fail. An unknown reading still fails, and no other gate is +affected. + +Every checked-in soak now sets `null`. `soak-weekly` named #600 until #627: +its first run found a 400k-window frontend's resident set ratcheting up with +block landings to about 3.3 GB. With #627's post-landing `malloc_trim`, the +5.5 h run of 2026-10-06 (at `c95febb3`) passed every resident-memory row: + +| Frontend | Peak vs warm-up peak | Window-peak slope | +|---|---|---| +| fe-0 | 1.17x | +8.65 MiB/h | +| fe-1 | 1.13x | -15.75 MiB/h | + +Memory and descriptors are fitted only over *steady* samples, those taken +outside rental churn, so the population being compared is the same from +sample to sample; the connection gates use every sample. + +Resident memory is fitted over window peaks, not over every sample, because +a block landing on a 400k window is a step, not a trend: in the first +`soak-short` runs each landing's audit build lifted a frontend's resident +set by about 800 MiB (520 to 1,300 MiB) for ten minutes or more, and a second +one to 2.2 GiB before it fell back to 0.9 GiB. A least-squares line through +those samples reads thousands of MiB/h on a healthy server. A window at +least as long as the landing cadence holds a landing in every peak alike, +so landings shift the envelope and a leak tilts it. Descriptors have no such +steps and are fitted over every steady sample. + +The server counts its own descriptors (`qbit_prism_process_open_fds`, beside +`qbit_prism_process_resident_memory_bytes`): it makes itself non-dumpable at +start, which closes `/proc/<pid>/fd` to every other process of its user, so +the gauge is the only way to read the count without privileges. + +## `--plan soak` + +A soak preset is an ordinary preset with `"--plan": "soak"` and a `soak` +block. Its own flags are the server, cluster and population of the one +lifetime: frontends, sessions, window, pool fee, template bits, database and +runtime sizes. The `soak.presets` it names contribute only their workload +flags (`WORKLOAD_FLAGS` in `soak.rs`: plan, rates, arrival, tips, blocks, +churn, storms), and each is run with the phases its own run would drive, +with three changes: no phase restarts or kills a frontend (the reconnect +phase keeps its client reconnects; a looped preset with `--mid-flight-kill` +is refused); the deliberately degraded `slow_database` phase is left out, +since its backlog is its own measurement and, looped, it has to drain before +the next cycle starts, which it can outlast (a first `soak-short` run aborted +there, 16 submits still outstanding at the 25 s drain limit); and every +phase is named `c<cycle>.<preset>.<phase>`. Whole +cycles are planned while they fit in `soak.minutes`. A looped workload that +could connect more sessions than the soak preset's listeners are sized for is +refused before anything starts. + +| `soak` key | Meaning | +|---|---| +| `minutes` | scheduled load; whole cycles only | +| `presets` | the looped presets, in order | +| `sample_seconds` | seconds between samples | +| `rollover_minutes` | when to roll the share ledger into its next partition | +| `rollover_margin_rows` | rows left below the bound by a rollover | +| `archive_every_minutes` | minutes between retention passes | +| `archive_window_multiple`, `archive_retention_days` | `share-archive --window-multiple` and `--retention-days` | +| `partition_ensure_interval_seconds` | `PRISM_SHARE_PARTITION_ENSURE_INTERVAL_SECONDS` for every frontend | +| `gates` | the table above | + +**Rollover.** A partition is 2^24 rows, days of a soak's load. At each +`rollover_minutes` the harness advances the share sequence to +`rollover_margin_rows` below the bound of the partition the sequence is in +(`setval`, which only ever raises it here, as the partition tests do); the +live load then crosses the bound itself into the lead partition, and the +frontends' maintenance attaches a new lead within +`partition_ensure_interval_seconds`. `share_seq` already has gaps, since a +rolled-back append's value is never reused, and every reader walks it in +order rather than by count. + +**Retention.** Every `archive_every_minutes` the harness runs the operator's +own commands against the live cluster, as +[the retention procedure](prism-share-ledger-partitioning.md#retention-procedure) +describes: `share-archive seal`, `archive` and `verify` for every partition +the sequence has passed, then `plan`, and `detach` and `drop` for each +partition it reports eligible. A command that fails, or runs past ten +minutes and is killed, is an event with its output (`soak-events.jsonl`) and +fails the soak. At the end every partition that left the ledger is read back +for the run's reconciliation, beside the live ledger: a detached one as it +stands, a dropped one restored from its archive with `share-archive +restore`, not attached. An acknowledged share has to be in PostgreSQL or in +an archive that restores row for row. + +The run writes `soak-samples.jsonl`, `soak-events.jsonl` and, from the +gate, `soak-report.md` (the verdict and a table of the samples) beside +`load-harness-report.json`, whose `soak` section lists the rollovers, the +retention steps and errors, the dropped and restored partitions. + +### The presets + +| Preset | Schedule | Load | Rollovers / archived | Purpose | +|---|---|---|---|---| +| `soak-weekly` | weekly | 13 cycles of 25.5 min (mainnet-shape-130-addresses, rental-churn-bursts-and-storms, tip-delivery-2000-miners-400k-2fe-retarget) = 5.5 h | 4 forced, gated >= 3 / >= 2 | the weekly CI soak | +| `soak-short` | manual | 2 cycles of tip-delivery-2000-miners-400k-2fe-retarget = 19 min, no block landing | 1 forced, gated >= 1; the 400k-window partition sealed, archived and verified under load, not dropped | a quick soak, and the run the leak mutant must fail | +| `soak-smoke` | manual | 4 cycles of pr-smoke = 3.6 min | 2 / 2 | the nightly opt-in gated test `soak_smoke`, proving the machinery end to end | + +`soak-weekly` runs the mainnet-shape-130-addresses cluster: two frontends, +400 sessions over 130 addresses, a 400k window, retargeting tips and a +200 bps fee. The tip-delivery segment therefore runs its 133 shares/s and +eight tips 45 s apart over those 400 sessions, not the 2,000 of its own +preset: one lifetime has one population. + +Its bounds: resident memory's envelope (the peak of each 25.5 min cycle) +at most 16 MiB/h after a one-hour warm-up and within 2x the warm-up peak, +descriptors at most 2/h, at most 24 connections +per frontend (16 in the pool, the rest the server's single-connection side +pools) drifting by at most 2, `pg_wal` at most 2 GiB (the managed cluster's +`max_wal_size` is PostgreSQL's 1 GB default, and the async standby's slot +can hold more while it catches up). A leak of 1 KiB per accepted share at the +soak's average of about 73 shares/s is about 128 MiB/h per frontend, eight +times the slope bound. The first 5.5 h run (2026-10-06, above) stayed inside +these bounds, but its +8.65 MiB/h slope leaves less than a 2x margin on one +sample. Read a later slope failure against #628 and #629 (allocator, and +per-cycle floors instead of landing-transient peaks) before calling it a +regression. + +`soak-short` cannot be that sensitive. In its first twenty minutes a healthy +server's resident set still grows, as its caches fill, by 0.6 to 0.9 GiB/h +(measured on this preset's workload), so its envelope bound is 1.5 GiB/h and +it catches a gross leak only: the test-only mutant below keeps 16 KiB per +accepted share, about 3.7 GiB/h per frontend at soak-short's rate. + +`soak-smoke`'s memory and descriptor bounds are deliberately loose: four +minutes of a debug server is not a trend. It proves the sampling, rollover, +retention, restore and gating end to end, every night, through the opt-in +gated test `soak_smoke` (four minutes was too much to add to every PR); +`soak-short` and `soak-weekly` hold the trends. Every PR runs the gated test +`soak_sampling` instead: in a few seconds against the PR shard's database it +samples a fresh PRISM schema and the test process, forces a rollover, and +holds the samples to the soak gates. + +### Running one + +```sh +cargo build --locked --release -p qbit-prism-server -p qbit-prism-load +bash .github/scripts/prism-load-run.sh soak-short out/soak-short +``` + +The gate's verdict, including the soak table, is `out/soak-short/gate.md`; +the exit status is the gate's. + +**Proving the gate bites.** `qbit-prism-server` has a test-only feature, +`soak-leak-mutant`, that keeps 16 KiB per accepted share for the life of the +process. No build or image of this repository enables it. Build it into a +separate target directory and run `soak-short` against it; the +resident-memory slope gate fails: + +```sh +CARGO_TARGET_DIR=target-mutant cargo build --locked --release \ + -p qbit-prism-server --features qbit-prism-server/soak-leak-mutant +target/release/qbit-prism-load --preset crates/qbit-prism-load/presets/soak-short.json \ + --server-bin target-mutant/release/qbit-prism-server \ + --allow-unverified-server-revision --out out/soak-leak +target/release/qbit-prism-load-gate --report out/soak-leak/load-harness-report.json \ + --preset crates/qbit-prism-load/presets/soak-short.json --exit-code 0 +``` + +## The weekly CI job + +`prism-load-nightly.yml` has a third schedule, Saturday 05:41 UTC, that +selects `weekly` (`scripts/prism_load_matrix.py weekly`), which is +`soak-weekly`. It reuses the workflow's release build, runs the preset with +`prism-load-run.sh` on an 8 vCPU runner and uploads the run directory. The +preset's timeout is 355 min, under the 6 h job limit. The soak has its own +concurrency group, so it never holds the nightly queue, and the live regtest +variants do not run on its schedule. Dispatch it by hand with +`preset: soak-weekly`. + +The nightly's skip-unchanged guard (#549) does not apply to the soak: it runs +every Saturday whether or not `3.x.x` moved. A soak measures drift over one +server lifetime and its bounds are still being calibrated, so another lifetime +on the same commit is a second sample, not a repeat; and the commit the guard +compares with is the one the nightly presets tested, not the soak. A failed +soak opens or comments on the `prism-load-nightly-failure` issue like any +scheduled run. + +**Cost.** About 5 h 40 min of one 8 vCPU runner a week plus the shared +release build, roughly 350 runner-minutes, about $5.60 a week at $0.016 per +8 vCPU minute, or about $24 a month. + +## A multi-day testnet4 soak + +The regtest soak cannot see real miners, a real node or days of time. +`qbit-prism-soak-report` applies the same gates to a live deployment, hourly, +from outside it: it needs only a Prometheus that scrapes the PRISM servers' +`/metrics` and a database URL. Nothing about the deployment is in the +repository; the endpoints come from flags or the environment. + +### Before you start + +- **Prometheus** scrapes every PRISM server of the deployment. Each server's + series carry a label that names the process (`instance` by default, + `--instance-label` otherwise); `--selector` narrows the metrics to the + deployment's servers, e.g. `--selector 'job="prism"'`. +- **A read-only database role** on the PRISM database. `pg_monitor` lets it + read `pg_wal`'s size and `pg_read_all_stats` the other sessions' states; + without them the WAL and idle-in-transaction gates read unknown and fail, + by design. The ledger is read through its parent table, so partitions + attached after the grant need no grant of their own. + + ```sql + CREATE ROLE prism_soak LOGIN PASSWORD '...'; + GRANT CONNECT ON DATABASE <db> TO prism_soak; + GRANT USAGE ON SCHEMA public TO prism_soak; + GRANT SELECT ON ALL TABLES IN SCHEMA public TO prism_soak; + GRANT SELECT ON ALL SEQUENCES IN SCHEMA public TO prism_soak; + GRANT pg_monitor, pg_read_all_stats TO prism_soak; + ``` + +- **One lifetime, one configuration.** Deploy the release under test, record + its image ID and the deploy environment (secrets redacted), and do not + restart or reconfigure it during the soak. A restart is detected (the + acknowledged-share counter resets) and restarts the trends from the new + process; the checked-in testnet4 gates do not fail on it + (`one_lifetime: false`), so note every intentional restart in the soak log. +- **Ordinary load throughout**: the testnet4 miners that normally mine, with + the same population for the whole soak. The + [24 h resident-set runbook](prism-capacity-readiness.md#testnet-24-h-soak-runbook-deferred-to-the-operator) + in the capacity readiness record is the stricter, shorter procedure for the + memory bound alone; a multi-day soak here does not replace it. + +### Start it + +Build the reporter from the release's commit, on any host that can reach +Prometheus and the database: + +```sh +cargo build --locked --release -p qbit-prism-load --bin qbit-prism-soak-report +export PRISM_SOAK_PROMETHEUS_URL=<prometheus base URL> +export PRISM_SOAK_DATABASE_URL=<read-only database URL> +mkdir -p soak-testnet4-$(date -u +%Y%m%d) +target/release/qbit-prism-soak-report \ + --out soak-testnet4-$(date -u +%Y%m%d) --selector 'job="prism"' --once +``` + +The first `--once` sample checks the plumbing: every row of +`soak-report.md` should read a value, not `unknown`. Then either leave it +running in the foreground (it samples every `--interval-seconds`, 3600 by +default, until stopped): + +```sh +nohup target/release/qbit-prism-soak-report --out soak-testnet4-<date> \ + --selector 'job="prism"' >> soak-testnet4-<date>/reporter.log 2>&1 & +``` + +or run `--once` hourly from cron or a systemd timer with the same `--out`; +each run resumes from the samples file, so the soak's history is the file's. +`--once` exits 0 when every gate passes, 1 when one fails and 2 when its +inputs cannot be read, so a timer's failure state is the verdict. + +### Read it + +`soak-report.md` is rewritten after every sample: the verdict table over +every sample so far, then up to 48 evenly spaced samples. The checked-in +gates, `crates/qbit-prism-load/soak-gates/testnet4.json` (`--gates` for +another file), use a one-hour warm-up like the resident-set bound, need at +least twelve post-warm-up samples before a trend can pass, and hold resident +memory's envelope (the peak of each 4 h window, at least four of them, so +the trend can pass from about 17 h in) to 4 MiB/h and 2x the warm-up peak, descriptors to 0.5/h, connections +to 40 per client with a drift of at most 2, and `pg_wal` to 4 GiB. They ask +for no rollover or archive, since testnet4's share rate does not fill a +2^24-row partition in days; a soak that runs the retention procedure can +raise `min_rollovers` and `min_archive_cycles` in its own gates file. + +Connections are attributed to a client by `application_name` when it has +one, else by role and address. The servers do not set an application name, +so on a deployment one client is one server host's role; raise +`pool_connections_max` when several servers share a host and a role. + +**Acknowledged shares in the ledger.** The reporter cannot see the miners' +side, so it compares the servers' own count of acknowledged shares +(`qbit_prism_accepted_shares_total`, summed over processes, across restarts) +with the accepted rows the ledger holds for the same span. The first +sample's readings are the baseline. Each later sample adds the counters' +growth since the previous reading to the acknowledged side, and the ledger's +accepted rows between the two readings' scrape times (Prometheus's +`timestamp()`, the earliest among the processes) to the committed side, +counted while those rows are still live, so retention cannot take them out +of the count. Processes scraped a little after the earliest may be ahead by +the rows of that spread, so the check is `committed + spread >= +acknowledged`. A counter that reset since its last reading +(`resets()`) is a restart: the new process's count is all new, and what the +old one acknowledged after its last reading cannot be counted; the report +counts such restarts. That can hide a loss inside the uncounted stretch; it +cannot invent one. A sample that could not read the counters or the ledger +advances neither side, and the next one covers the stretch. The regtest +soak's reconciliation, which knows every share each client was told was +accepted, is the exact version. + +A process whose series stops appearing stays in the samples with every +figure unknown, and its trends fail on the first sample that cannot read +it: a process gone dark is not judged on what it did before. + +When the soak ends, attach `soak-report.md`, `soak-samples.jsonl`, the image +ID and the deploy environment to the release's tracking issue. diff --git a/docs/prism-storage-sizing.md b/docs/prism-storage-sizing.md index 9297632ef..4ce2b0950 100644 --- a/docs/prism-storage-sizing.md +++ b/docs/prism-storage-sizing.md @@ -1,320 +1,425 @@ -# PRISM Storage and VM Sizing +# PRISM Storage and Resource Planning + +Native Prism keeps the canonical ledger, block accounting, audit metadata, +normalized audit snapshots, and CTV recovery state in PostgreSQL. Multiple +frontends use this shared database; new accepted-block audits do not require a +shared local filesystem. + +## What is retained + +| Data | Storage and lifetime | +| --- | --- | +| Accepted shares | Immutable PostgreSQL rows, never updated, deleted or truncated. Online while the share is inside the online horizon, then archived outside PostgreSQL one whole partition at a time and the partition dropped | +| Proof identity (`qbit_prism_share_hashes`) | Permanent immutable PostgreSQL rows, one per share, about 150 B each. Not partitioned and never removed: it is the global `share_id` authority the append path consults first | +| Blocks, payouts, carry-forward, maturity/reorg state | Permanent PostgreSQL accounting history | +| Native audit share snapshots | Permanent range/count/anchor/digest metadata referencing immutable shares | +| Native audit bodies | Non-share JSON plus snapshot reference in PostgreSQL: the logical bundle minus its top-level `shares` and minus `reward_manifest.shares`, both rebuilt on read. Rows landed before 3.x.x #267 still hold `reward_manifest.shares` and stay readable as they are. Before the shares a body depends on are archived, the body is sealed: its full canonical artifact is stored in `canonical_audit_bytes` and served from there instead of being rebuilt, which costs the artifact size back per archived block, about 22 MB at a 20,000-share window and about 111 MB at 100,000 (`audit_body_byte_len` in the table below), TOAST-compressed in the column | +| Imported legacy audits | Digest-checked canonical bytes (bytea) plus non-share metadata in PostgreSQL; an inline body survives only on inline-only rows; original backup retained | +| CTV manifests, transactions, outcomes | Durable PostgreSQL recovery and audit state | +| CPFP funding reservations and signed child packages | Durable recovery records; retain until reconciled | +| Pending block candidates and deferred credit | Durable until resolved; retained identity/status afterward | +| Jobs and prepared snapshots | Expiring shared records for reconnect/job recovery | +| Per-process HTTP caches and socket state | Reconstructible memory | +| qbit/Bitcoin chain data | Separate node storage, sized independently | + +Accepted ledger rows cannot be updated, deleted, or truncated. Native audits +reconstruct their exact share slice from those rows, rebuild the counted-share +window of `reward_manifest` from that slice (the stored header must match the +rebuilt one field for field), and verify the snapshot digest and the canonical +bundle hash of the whole body. Overlapping windows therefore share ledger +storage instead of embedding another copy of every share in every +accepted-block audit. + +Retention of that history is detach-and-archive, never deletion. There is no +row-level prune or share-compaction command, and no `DELETE` runs on the +ledger. `qbit-prism-server share-archive` retires a whole partition once no +online reader can still need it: seal the audits that depend on it, write the +partition to an archive outside PostgreSQL, verify the archive against the live +rows, detach, and only then drop the detached relation, with the archive as the +copy of record from then on. Size and retain that archive as accounting data, +not as a cache. The procedure, the online horizon it checks and the archive +format are in +[ledger operations](prism-ledger-ops.md#share-ledger-partitions-and-retention). + +### Archived share partitions + +A partition at the default width holds 2^24 = 16,777,216 rows, about 7.5 GB of +heap plus about 20 GB of indexes at the production row shape. Archiving it +writes newline-delimited canonical JSON, one object per ledger row with every +column, gzipped. Uncompressed that JSON is larger than the heap it came from, +since numerics and timestamps become decimal strings and the program becomes +hex; gzip on rows of this shape should recover roughly 5:1. Treat that as an +estimate to be replaced by a measurement, not a planning number: the manifest +of the first real `archive` run records `rows_bytes` and `rows_gz_bytes` and +gives the exact ratio for this pool's row shape. + +The archive is an additional copy, not a saving on the database volume until +the partition is dropped. Budget both for the interval between `archive` and +`drop`, plus whatever the archive retention policy keeps off-host. + +### Native audit body size + +Measured with `cargo test -p qbit-prism-server --test audit_body_normalization +-- --ignored measure_landing_body_and_settlement_lock_hold` (release build, one +landed block, the synthetic five-recipient window of +`tests/support/window_fixture.rs` at about 581 B per share): + +| Window (counted shares) | Stored `audit_bundle` as JSON text | Stored on disk, `pg_column_size` (TOAST-compressed) | `audit_body_byte_len` (full canonical artifact) | +| ---: | ---: | ---: | ---: | +| 20,000, landed before #267 | 10,901,855 B | 353,091 B | 22,250,397 B | +| 20,000, landed since #267 | 12,949 B | 3,087 B | 22,250,397 B | +| 100,000, landed before #267 | 54,401,879 B | 1,707,134 B | 111,190,422 B | +| 100,000, landed since #267 | 12,972 B | 3,088 B | 111,190,422 B | + +Before #267 the stored body kept `reward_manifest.shares`, one record per +counted share, so it grew by about 545 B of JSON text per share: the production +window of 372,257 shares measured about 235 MB as one JSONB value (issue #267), +88% of PostgreSQL's 268,435,455-byte limit, and the reduced-size JSONB ceiling +gate projected 233 MB at 400,000 synthetic shares. The body landed since #267 +does not grow with the window. Its residual size is O(distinct miners and payout accounts), +not a fixed bound: `reward_manifest.entitlements`, the payout policy manifest's +accounts, `prior_balances`, and the CTV fanout manifests scale with the number +of distinct payout programs, and the synthetic window above has five. A pool +paying thousands of distinct programs stores proportionally more per block. +The synthetic window also compresses about 27:1 in TOAST, so the on-disk column +is optimistic for production data; the JSON text column is not. + +Two lengths describe one native audit and neither is derivable from the other: +`audit_body_byte_len` is the length of the full canonical artifact, both share +copies included, exactly as `/public/v1/artifacts/<sha256>` serves it; +`sum(pg_column_size(audit_bundle))` (the storage query below) is the stored, +compressed length of the non-share body. + +Serving a native audit rebuilds the window on every request: the range read, +the counted-window fold, and the canonical hash. The same harness measured that +read at 1.0 s for 20,000 and 5.4 s for 100,000 shares (release build, one +request, no contention); budget CPU and the public read deadline for it. + +### Memory bound on artifact reads + +`/public/v1/artifacts/<sha256>` holds a whole window in memory per read, +whichever shape the row has: an unsealed native block is rebuilt from its share +range, and a sealed one has its stored canonical bytes digested and parsed. +Since #267's admission half, both run under +`PRISM_PUBLIC_AUDIT_REBUILD_CONCURRENCY` (default 1), which is independent of +`PRISM_POSTGRES_READ_CONCURRENCY`: raising the read concurrency admits more +ordinary reads and **no** more window-sized reads *of this route*, and its +transient memory is -This document sizes the PRISM pool storage footprint for operator planning. It -separates hot Postgres, audit artifacts, qbit/Bitcoin chain data, Docker -overhead, WAL, and backups because those are different capacity problems. - -## Scope - -PRISM stores permanent accounting data in Postgres and writes public audit -artifacts under `PRISM_AUDIT_DIR`. - -Base Compose uses named volumes for local development: - -- `prism-postgres-data` at `/var/lib/postgresql/data` -- `prism-postgres-wal` at `/var/lib/postgresql/wal` -- `prism-audit-data` at `/var/lib/qbit-prism/audit` -- `qbit-data` at `/var/lib/qbit` - -Production Compose replaces these with pre-created absolute bind mounts supplied -through `PRISM_POSTGRES_DATA_SOURCE`, `PRISM_POSTGRES_WAL_SOURCE`, -`PRISM_AUDIT_DATA_SOURCE`, and `QBIT_DATA_SOURCE`. The data and live WAL sources -must be distinct paths and should be monitored as separate capacity boundaries. - -The current production storage target is: - -- keep the canonical accepted-share ledger in Postgres, or in a verified - immutable archive when a future archive flow exists -- keep block, payout, carry-forward, maturity, reorg, and settlement state - durable -- keep canonical audit artifact hashes in Postgres -- keep large exact audit artifact bodies outside hot Postgres once the artifact - externalization work lands - -## Pilot Baseline - -A small pilot deployment measured on 2026-06-30 showed active PRISM state at -about 1.2 GB before long-term growth. The useful planning signal is the -composition of that active state: - -| Active PRISM item | Size / count | -| --- | ---: | -| Postgres volume | about 195 MB | -| Database size | about 140 MB | -| Audit file volume | about 989 MB | -| qbit data volume | about 61 MB | -| Active PRISM total | about 1.2 GB | - -The active table breakdown was: - -| Table | Size / rows | -| --- | ---: | -| `qbit_pool_audit_bundles` | 126 MB, 571 rows | -| `qbit_share_ledger` | 3.7 MB, 5,624 accepted shares | -| payout, carry-forward, and block tables | about 2.5 MB combined | - -The accepted-share ledger was not the disk driver in that pilot. Inline audit -bundle bodies dominated the hot database footprint, which is why newer artifact -externalization and share-segment formats matter for production planning. - -## Growth Model - -qbit targets 60-second aggregate block spacing, so the network produces about -1,440 blocks per day. PRISM vardiff targets about one accepted share per active -worker every 15 seconds. +```text +artifact_read_bytes = PRISM_PUBLIC_AUDIT_REBUILD_CONCURRENCY x peak_bytes_per_read +``` -Use: +That bound is the artifact route's alone. `/public/v1/blocks/<hash>/settlement-artifacts` +falls back to the audit-bundle reader when the block has no CTV fanout set, and +that reader still rebuilds or decodes the same window under the shared +imported-audit decode limit, which is sized from `PRISM_POSTGRES_READ_CONCURRENCY`. +Its bound is therefore `PRISM_POSTGRES_READ_CONCURRENCY x peak_bytes_per_read`, +and raising the read concurrency does raise it; size a public process for both. +Moving that route under the artifact limit is a behaviour change on another +route and is not part of #267's admission half. + +What the bound covers is the work admitted under the permit: the range read and +fold, the canonical serialization, and the stored bytes' digest and parse. It +does not cover the response body, which stays alive after the permit is +released until the client has drained it, so a read's resident footprint +outlasts its permit by about one more copy of the artifact. + +CPU outside the permit matters as much as memory. Serving an audit artifact used +to hash the whole body twice more on a runtime worker thread — once to check the +returned bytes against the requested address, once for the ETag. One SHA-256 +pass over a 111 MB artifact measures 1.6-1.8 s in a debug build, so at 50 +requests per second those passes saturated the public process's worker threads: +a task that only sleeps overshot by 23.5 s, the node RPC behind +`/public/v1/pool-summary` timed out, and that route failed its 20 s deadline +even though the read pool was idle (p99 pool-summary latency 0.107 s in the same +run). Both passes are gone. The bytes are digested against `audit_bundle_sha256` +inside the blocking job, under the permit; the row is found by that column, so +the route compares it to the requested address as a string; and the ETag is that +same address rather than a fresh hash of the body. The verification is the same +digest it always was, and responses are byte-identical. After the change, the +same load holds pool-summary at 0.118-0.364 s maximum with no failed sample and +the sleeping probe's overshoot at 0.32-0.44 s. A public process still needs CPU +headroom for the rebuild itself, which is what the permit bounds. + +The legacy fallback body — an audit row with neither canonical bytes nor a share +snapshot, served through the audit-bundle reader — is still hashed on a runtime +thread for its ETag, because it has no proven content address to carry. It is a +pre-#267 import shape, and it is not admitted by the rebuild limit either. + +`peak_bytes_per_read` is a multiple of the canonical artifact length, which is +`audit_body_byte_len` in the table above. Measured at a 5,000-share window by +`cargo test -p qbit-prism-server --test artifact_admission -- --nocapture +artifact_route_load_keeps_pool_summary_within_its_read_deadline` (debug build, +process peak resident memory across one uncontended read, `VmHWM` reset before +each): the canonical artifact was 5,570,372 B (**1,114 B per share**, measured), +one rebuild peaked about **34 MiB** above the resident baseline (about 6x the +artifact) and one sealed decode about **11 MiB** (about 2x). Those multiples are +what the bound below extrapolates; they are measured only at that window and in +a debug build, and the test process also holds the response body. + +Derived, not measured, at the sizes this pool plans for: at 1,114 B per share a +400,000-share window is a **445 MB** canonical artifact, so one rebuild is on +the order of **2.5 GB** of transient memory and one sealed decode on the order +of **1 GB**, times the rebuild concurrency. No measurement at 400,000 shares +exists; run the `#[ignore]` variant +(`artifact_route_load_at_a_large_window`, `PRISM_ARTIFACT_ADMISSION_SHARES`) on +a machine with that headroom before raising the setting. The default of 1 keeps +one window in flight per public process; the in-flight cap +(`PRISM_PUBLIC_AUDIT_ARTIFACT_MAX_IN_FLIGHT`, default 32) bounds only how many +requests may queue for it, not memory, because a queued request holds neither a +read connection nor a window. + +Legacy external body refs and v2 segment files remain readable by the offline +Rust tools. During [migration](prism-rust-migration.md), import these into the +shared database before relying on another frontend to serve old audits. Preserve +all referenced bodies and segments in the migration backup. A small live-evidence +envelope or a stored SHA cannot replace missing artifact bytes. + +### Refresh build memory per frontend + +Since #502 the refresh build serializes the share array and folds the counted +shares in chunks on the frontend's builder pool (`PRISM_REFRESH_BUILD_THREADS`, +default three quarters of the host's cores clamped to 1..4, `0` for the serial +build, at most 64), and it holds more memory per frontend process than the +serial build did. The trade is accepted. #502's own confirmation pair on its +merge base measured per-frontend RSS after the tips at 328-342 MiB before and +568-608 MiB after, and peak RSS at 557 MiB before and 669 MiB after; an +independent 400,000-share probe measured 251-254 MiB retained after the drop +with the serial build against 284-330 MiB with eight workers. Budget roughly +0.5 GiB more retained memory on a two-frontend host. + +Two things hold that memory. The pool's threads keep the counted-share strings +and chunk buffers in their own allocator arenas, one per thread, so the retained +part grows with the thread count (at 400,000 shares, sixteen threads retained +about 130 MiB more per frontend than four for the same refresh time); the +thread count is the knob that bounds it, and on a memory-tight host +`MALLOC_ARENA_MAX` (for example `MALLOC_ARENA_MAX=2` in the frontend's +environment) caps glibc's arenas whatever the thread count. And the build slot +(`PRISM_JOB_BUILD_EXECUTOR_WORKERS`) is released before the previous refresh's counted-share +body is freed, so the slot count no longer bounds the number of live +counted-share bodies exactly: expect one extra body per slot, transiently, on +top of the bound the slot gives. + +### Landing memory per frontend + +Landing a block rebuilds its payout window and audit on the frontend that +lands it. At a 400,000-share window the live heap goes from about 260 MiB to a +1.6 GiB peak and back within the landing (#600). glibc keeps what the landing +frees in the arenas of the threads that did the work and does not return it, +so without a trim the landing frontend's resident set climbs with each of its +landings to a plateau of 3.2–3.3 GB after four to six of them, with a +high-water mark of 3.9 GB, while its live heap stays at about 250 MiB. A +frontend that has not landed a block stays at 0.6–0.7 GB. + +Since #600 the frontend that landed calls `malloc_trim(0)` once the landing's +transaction has ended and its rebuilt window is released. The trim runs on +the blocking thread that released the window, never on a runtime worker and +never under a lock or a database transaction, and returns every free page in +every arena to the kernel. It locks each glibc arena in turn while it does, +so a thread that allocates from that arena meanwhile, a runtime worker +included, waits for it: `qbit_prism_landing_malloc_trim_seconds` bounds that +pause. It is on by default; +`PRISM_LANDING_MALLOC_TRIM_ENABLED=0` turns it off. glibc builds only (the +image and release binaries); elsewhere it does nothing. +`qbit_prism_landing_malloc_trim_seconds`, +`qbit_prism_landing_malloc_trim_released_bytes_total` and +`qbit_prism_landing_malloc_trim_resident_bytes` (the resident set right after +the latest trim, the frontend's floor) report each trim. + +Measured with the same binary, trim off and on, two 30-minute runs of each +over the weekly soak's 400k-window two-frontend shape (six blocks per run, +pinned to four cores). The floor is the lowest resident set in each +10-minute cycle; landings are the blocks that frontend rebuilt and landed, +its trims with the trim on and its post-landing work deliveries, which can +miss one, with it off (#600): + +| trim | frontend | landings | cycle floors (MiB) | peak (MiB) | +| --- | --- | ---: | --- | ---: | +| off | fe-1 | 4 | 420, 2,110, 2,540 | 3,166 | +| off | fe-1 | 5 | 419, 1,374, 2,026 | 2,635 | +| off | fe-0 | 1 | 354, 634, 1,349 | 1,436 | +| off | fe-0 | 0 | 356, 614, 632 | 709 | +| on | fe-0 | 6 | 426, 579, 593 | 1,367 | +| on | fe-0 | 3 | 412, 538, 551 | 745 | +| on | fe-1 | 3 | 387, 605, 685 | 1,642 | +| on | fe-1 | 0 | 447, 651, 674 | 766 | + +Each trim took 62–100 ms and returned about 1 GB, leaving the frontend at +about 400 MiB. With the trim on, the landing frontend's accepted block +reached miners as post-landing work in 5.4–6.2 s, against 5.2–8.4 s with it +off, and its block offers took 31–105 ms against 27–99 ms. Share +acknowledgements slower than 50 ms or 250 ms were no more frequent within +6 s of a trim than elsewhere in the run. + +Set the per-frontend memory limit from the peak, not the floor, and leave +headroom for a larger landing: + +| 400k window, per frontend | memory limit | RSS warning | +| --- | ---: | ---: | +| trim on (default) | 3 GiB | 2.5 GiB | +| trim off | 5 GiB | 4 GiB | + +Scale both roughly linearly with the window: at 200k the floor without a trim +plateaued at 1.0–1.06 GB. Two frontends on one host need twice the limit plus +PostgreSQL. `PrismResidentMemoryHigh` fires at 4 GiB, the trim-off warning: +with a 3 GiB limit, lower it to 2.5 GiB or the frontend is killed before it +fires. + +## Estimate from measured ingest + +Estimate share growth from accepted share rate, not miner hashrate alone: ```text -accepted_shares_per_second = active_workers / 15 -accepted_shares_per_year = accepted_shares_per_second * 31,536,000 -hot_share_ledger_bytes = accepted_shares_per_year * 1.2 KB -pool_blocks_per_day = 1,440 * pool_share_of_blocks +accepted_shares_per_second ≈ active_workers / target_share_interval_seconds +accepted_shares_per_year = accepted_shares_per_second × 31,536,000 +annual_ledger_bytes = accepted_shares_per_year × measured_bytes_per_share ``` -The measured share-ledger cost is about 0.7 to 0.8 KB per accepted share across -pilot data and local schema probes. Use 1.2 KB per share for planning to cover -indexes, bloat, and operational headroom. - -The estimates below assume the optimized Postgres pattern: hot Postgres keeps -the canonical ledger, block/payout/carry rows, and artifact metadata, not the -full audit JSONB body for every block. If exact artifact bodies remain local, -provision artifact storage separately. - -Externalizing exact current artifacts reduces hot Postgres size. It does not -reduce total retained bytes unless artifacts are compressed or replaced by a -future reduced/window proof format. - -## One-Year Hot Postgres Scenarios - -| Scenario | Active workers | Shares/sec | Pool share of blocks | Blocks/day | Estimated hot Postgres live data/year | Practical VM disk if backups/artifacts are not local | -| --- | ---: | ---: | ---: | ---: | ---: | ---: | -| Testnet / pilot | 15 | 1 | 1% | 14 | about 38 GB | 250-500 GB | -| Early public | 150 | 10 | 5% | 72 | about 386 GB | 1.5-2 TB | -| Serious pool | 1,500 | 100 | 20% | 288 | about 4.1 TB | 8-10 TB | -| Very large | 7,500 | 500 | 50% | 720 | about 22 TB | 45-60 TB | - -If local artifact bodies, local physical backups, or WAL archives are kept on -the same VM, increase the practical disk target: - -| Scenario | Practical disk with local artifacts/backups | -| --- | ---: | -| Testnet / pilot | 500 GB-1 TB | -| Early public | 4-8 TB | -| Serious pool | 15-25 TB or dedicated database storage | -| Very large | dedicated storage architecture, not a single small VM | - -## Permanent Data - -Keep these forever in hot storage or a verified immutable archive: - -- accepted shares with exact `share_seq` order -- `share_id`, miner/payout identity, payout order key, and P2MR program -- share difficulty/work amount and network difficulty context -- job/template metadata, `job_issued_at`, `accepted_at`, and proof identifiers -- block reward-window anchors -- block, payout, carry-forward, maturity, reorg, and reversal rows -- payout policy inputs, fee policy, minimum payout/floor settings, and - settlement-mode decisions -- canonical audit artifact hashes and verifier/schema versions -- coinbase txid/hex and manifest hashes needed to match chain evidence -- CTV/fanout settlement artifacts needed for third-party verification or - broadcast recovery - -Do not prune `qbit_share_ledger` until the archive contract in -`docs/prism-ledger-ops.md` is implemented and verified. - -## Archiveable Or Ephemeral Data - -These can have bounded retention once the policy is explicit: - -- rejected-share details after operational/debug retention -- non-canonical candidate audit artifacts that never became accepted blocks -- dashboard caches and rollups that can be regenerated or are intentionally - lower resolution over time -- verbose broadcast attempts after terminal status, if latest status, error - summary, attempt count, and final artifacts are retained. The live schema - now stores this summary on `qbit_ctv_fanout_artifacts` and caps retained - detail rows with `PRISM_CTV_BROADCAST_ATTEMPT_DETAIL_LIMIT`. -- Docker images, Docker build cache, and old non-active TIDES volumes after - backup or decommission confirmation - -Inactive PRISM blocks are not deletion candidates simply because they are -inactive. They can reactivate after a reorg. - -## Artifact Storage Strategy - -The live coordinator stores accepted-block audit artifacts with these rules: - -1. Exact artifact externalization: Postgres stores hash/pointer metadata while - every reader still resolves the body to the logical - `qbit.prism.audit-bundle.v1` expected by public API and verifier callers. - New rows also store `audit_body_byte_len` so the canonical row has the - artifact hash, stored byte size, schema version, and body pointer. -2. Share-segment proof bodies: when `PRISM_AUDIT_SHARE_SEGMENT_SIZE` is - positive, new external body files use `qbit.prism.audit-bundle.v2`. The v2 - body keeps non-share bundle sections inline, stores a - `qbit.prism.window-completeness-proof.v1`, and points at stable - `qbit.prism.audit-share-segment.v1` segment-slot files with per-range hashes. - This removes the old repeated inline tail/partial-window share arrays while - preserving the same reconstructed v1 bundle hash. -3. Legacy body refs: older compact files using `qbit.prism.audit-body-ref.v1` - remain readable. The Rust `qbit-prism-audit-verify` and - `qbit-prism-audit-canonicalize` tools accept full v1 bundles, legacy body - refs, and v2 proof bodies, verifying referenced segment/range hashes before - reconstructing the canonical v1 bundle. - -`prism-live-audit-bundle-*.json` files are now small operator envelopes that -point at the canonical body URI. They are not the durable audit body. Retention -may prune live envelopes and old candidates; it must not prune -`prism-audit-bundle-body-*` or `prism-audit-share-segment-*` unless those -artifacts have first been archived and the resolver policy is explicit. Stable -`prism-audit-share-segment-slot-*` files may grow as adjacent ranges are first -referenced, so old body files verify their original range hash against the -selected slice rather than against the whole mutable slot file. - -CTV broadcast retries are bounded separately from audit bodies: - -- `PRISM_CTV_BROADCAST_ATTEMPT_DETAIL_LIMIT` keeps only the newest N detailed - rows per fanout in `qbit_ctv_fanout_broadcast_attempts`. -- `qbit_ctv_fanout_artifacts` retains total attempt count, per-status counts, - first/last timestamps, last package txids/hexes, last submit result, last - error, and retry backoff state. -- `rejected` and `failed` attempts move a fanout to terminal `failed`, and - terminal failed fanouts are not selected for broadcast work. - -## VM Recommendations - -For near-term production or a pilot after audit artifact externalization: - -- Use at least 2 TB NVMe if audit bodies and backups are externalized. -- Use 4 TB if keeping a local artifact mirror. -- Use 4-8 TB if compact artifact bodies remain local and the pool is expected - to grow beyond pilot traffic. - -For a serious public pool: - -- Treat Postgres as a dedicated service or dedicated volume. -- Plan 10 TB or more for hot Postgres at roughly 100 accepted shares/sec. -- Keep immutable audit artifacts and backups in separate object or cold - storage. -- Treat artifact availability separately from artifact integrity. Hashes prove - fetched bytes are correct; backups and restore tests prove bytes remain - available. - -## Postgres Durability And Recovery - -Postgres always uses write-ahead logging (WAL). WAL is the database transaction -log, not a second application-level share queue: changes are recorded in WAL -before their modified table pages are written. Keep `fsync=on`, -`full_page_writes=on`, and `synchronous_commit=on`. Disabling any of those to -reduce share-accept latency weakens the meaning of a committed share and is not -a production tuning option. - -The production Compose contract sets `POSTGRES_INITDB_WALDIR` to the separately -mounted live WAL path. That setting is honored only when the official Postgres -entrypoint creates a fresh cluster. It does not move `pg_wal` for an existing -cluster. Verify the initialized cluster before admitting shares: +At a 15-second target, 150 active workers produce about ten accepted shares per +second. Vardiff bounds, hashrate changes, reconnects, and listener profiles can +change the actual rate; measure it under the intended configuration. + +The following is arithmetic using a **planning assumption** of 1.2 kB per +accepted share including indexes. It is not a measured native capacity claim: + +| Accepted shares/second | Shares/year | Ledger bytes/year at 1,200 bytes/share | +| ---: | ---: | ---: | +| 1 | 31.5 million | 37.8 GB | +| 10 | 315 million | 378 GB | +| 100 | 3.15 billion | 3.78 TB | +| 500 | 15.8 billion | 18.9 TB | + +Native proof deduplication adds indexed rows; measure its footprint alongside +the ledger. Non-share audit sections, per-recipient payouts/carry rows, CTV +transactions, and imported legacy inline bodies add independent growth. The +number of recipients and pool-found blocks can matter as much as the share rate. +Do not multiply just one table's row size and call it the database requirement. + +Measure a representative interval after migration and under load. Record table +and index growth, WAL generation, pending outbox size, expiring job storage, +backup size, and process peak RSS. Include cold startup and simultaneous job +refresh across all frontends. Large historical reward windows still require +memory and CPU to construct or export a full logical audit, even though their +persistent shares are normalized. + +Provision headroom for peak ingest, autovacuum, index growth, restore staging, +backups, and the time needed to respond to disk alerts. Estimate replicas and +backup retention separately; they are additional copies, not free capacity. + +## Local and external database deployments + +Local development Compose uses: + +- `prism-postgres-data` for PostgreSQL data +- `prism-postgres-wal` for live WAL +- `prism-audit-data` for legacy artifact access/import +- `qbit-data` for the qbit node + +Production Compose maps these to pre-created absolute paths through +`PRISM_POSTGRES_DATA_SOURCE`, `PRISM_POSTGRES_WAL_SOURCE`, +`PRISM_AUDIT_DATA_SOURCE`, and `QBIT_DATA_SOURCE`. The audit mount remains useful +for migration and archived operator artifacts; the native runtime does not +write repeated live audit bodies there for every block. + +For several physical frontends, use the external-database Compose overlay and +one HA writer endpoint. Size the external database's data, WAL, replicas, and +backup archive according to its deployment profile. Frontend hosts primarily +need CPU, memory, logs, image storage, and any locally managed qbit chain data. +A separate qbit node per frontend has its own chain-storage requirement. +CTV recovery without transaction indexing needs retained historical blocks for +its durable bounded scan; account for that when choosing a node pruning policy. + +The native runtime uses `PRISM_RUNTIME_WORKERS` and a bounded +`PRISM_JOB_BUILD_EXECUTOR_WORKERS` CPU builder limit. Increasing threads cannot +remove PostgreSQL commit or storage bottlenecks. Sum +`PRISM_DATABASE_MAX_CONNECTIONS` across frontends before setting the database +connection budget; reserve operational connections as well. + +## Durability, WAL, and recovery + +Keep `fsync=on`, `full_page_writes=on`, and `synchronous_commit=on`. WAL is the +PostgreSQL transaction log, not a second application share queue. Durable share +ACKs depend on successful commit; disabling these settings to improve benchmark +numbers changes that guarantee. + +The production local database sets `POSTGRES_INITDB_WALDIR` to the separately +mounted WAL path. This setting applies only when creating a fresh cluster and +does not relocate an existing `pg_wal`. Inspect the actual filesystem link and +mounts after initialization as well as the SQL settings: ```sql SHOW data_directory; +SHOW fsync; +SHOW full_page_writes; +SHOW synchronous_commit; +SHOW synchronous_standby_names; SELECT pg_current_wal_lsn(), pg_walfile_name(pg_current_wal_lsn()); ``` -Also verify that `readlink -f /var/lib/postgresql/data/pg_wal` inside the -container resolves to `/var/lib/postgresql/wal` and that the container mount -table maps that target to `PRISM_POSTGRES_WAL_SOURCE`; SQL alone cannot prove -the storage separation. Losing either the live data path or live WAL path makes -the primary unavailable, so snapshotting only one is not a recoverable backup. - -A commit on one Postgres primary protects against a coordinator restart, but it -does not by itself protect against loss of that primary and its storage. Choose -the recovery objective explicitly: - -- For a non-zero recovery point objective, use encrypted off-host base backups - plus continuous WAL archiving and document the maximum acceptable data loss. -- For zero loss of acknowledged commits after primary storage failure, add a - synchronous standby on independent failure-domain storage. Configure - `synchronous_standby_names` and keep `synchronous_commit=on` so commit waits - for the selected standby to durably flush WAL. Do not silently fall back to - asynchronous replication when that guarantee is required. -- Replicas do not replace backups. Operator error, corruption, and accidental - deletion can replicate immediately, so retain independent point-in-time - recovery material. - -Point-in-time recovery requires a compatible physical base backup and every WAL -segment from that backup through the requested recovery time. Store both -encrypted outside the database host, apply a tested retention policy, and run a -scheduled restore drill into an isolated Postgres instance. A drill is complete -only after schema checks, ledger continuity checks, artifact-pointer checks, and -an application read test pass. - -Live WAL and archived WAL are different storage classes. Postgres writes active -segments to `pg_wal`; an archive command copies completed segments to the -independent point-in-time-recovery archive. The separate Compose WAL mount holds -the former, not the latter. - -WAL consumes disk according to write volume and checkpoint behavior, not just -accepted-share row size. Healthy archived segments can be recycled from live -WAL; a failed archive command or an inactive replication slot can retain live -WAL without bound and fill its filesystem. Measure WAL generated during a -production-like load test, reserve several times the measured peak between -operator response windows, and alert on: - -- time and bytes since the last successful archived WAL segment -- `pg_wal` bytes and growth rate -- replication slot retained bytes and standby replay/flush lag -- base-backup age and the last successful restore drill -- synchronous standby count whenever zero acknowledged-share loss is required - -Treat recovery as a release gate: perform at least one full backup, primary-loss -simulation, point-in-time restore, and application verification before accepting -shares with economic value. - -## Monitoring - -Track at least: +A primary-local durable commit survives a frontend crash. To preserve every +acknowledged share after loss of the database primary, configure synchronous +replication on independent storage and fail over only to a standby containing +those flushed commits. An asynchronous standby or a highly available DNS name +does not establish this recovery point objective. + +Keep encrypted off-host base backups plus continuous WAL archives and conduct +isolated restore drills. Replicas do not replace recovery history: accidental +changes and corruption can replicate. A restore drill should verify schema, +share order, representative canonical audit hashes, carry-forward integrity, +CTV state, and application reads. Include unimported external audit artifacts +and signing-key recovery material in the recovery plan. + +Live WAL and archived WAL are distinct. Failed archiving or an inactive +replication slot can retain live WAL without bound. Measure peak WAL generation +and reserve enough capacity for the incident response interval. Alert on live +WAL growth, retained slot bytes, archive failures, standby flush/replay lag, +synchronous standby availability, backup age, and the last successful restore. + +## Read-only storage inspection ```sql SELECT pg_size_pretty(pg_database_size(current_database())); -SELECT - relname, - n_live_tup::bigint AS est_rows, - pg_size_pretty(pg_total_relation_size(relid)) AS total +SELECT relname, n_live_tup::bigint AS estimated_rows, + pg_size_pretty(pg_total_relation_size(relid)) AS total_with_indexes FROM pg_stat_user_tables ORDER BY pg_total_relation_size(relid) DESC; -SELECT count(*) FROM qbit_share_ledger WHERE accepted; +SELECT count(*) AS audit_count, + count(*) FILTER (WHERE share_snapshot_sha256 IS NOT NULL) AS native_references, + count(*) FILTER (WHERE audit_bundle IS NULL AND body_uri IS NOT NULL) AS external_bodies, + coalesce(sum(pg_column_size(audit_bundle)), 0) AS stored_audit_json_bytes +FROM qbit_pool_audit_bundles; -SELECT - count(*) AS bundles, - pg_size_pretty(sum(pg_column_size(audit_bundle))::bigint) AS inline_jsonb -FROM qbit_pool_audit_bundles -WHERE audit_bundle IS NOT NULL; +SELECT count(*) AS snapshots, + coalesce(sum(share_count), 0) AS referenced_shares_with_reuse +FROM qbit_prism_audit_snapshots; + +SELECT slot_name, active, + pg_wal_lsn_diff(pg_current_wal_lsn(), restart_lsn) AS retained_bytes +FROM pg_replication_slots; ``` -Filesystem checks: +`referenced_shares_with_reuse` counts references across snapshots, not unique +stored share rows. `n_live_tup` is an estimate and can lag; exact counts can be +expensive on a large ledger. `stored_audit_json_bytes` is the compressed +non-share body; the full logical audit each native row serves is +`audit_body_byte_len` bytes, orders of magnitude larger, and +`sum(audit_body_byte_len)` is the size of the artifacts the public route can be +asked for, not of anything stored. + +Inspect data/WAL/legacy artifact filesystems and container image/log usage: ```sh -df -h / df -h "$PRISM_POSTGRES_DATA_SOURCE" "$PRISM_POSTGRES_WAL_SOURCE" "$PRISM_AUDIT_DATA_SOURCE" -du -sh "$PRISM_POSTGRES_DATA_SOURCE" "$PRISM_POSTGRES_WAL_SOURCE" "$PRISM_AUDIT_DATA_SOURCE" -du -sh /var/lib/docker /var/lib/containerd docker system df -docker volume ls ``` -Add alerts for: - -- hot Postgres disk over 70% and 85% -- WAL or backup archive growth exceeding expected daily ingest -- audit artifact storage growth exceeding the block/artifact model -- `qbit_pool_audit_bundles` rows whose inline body remains large after artifact - externalization lands -- missing or hash-mismatched artifact objects -- `/metrics` gauges `qbit_prism_audit_artifact_bytes` and - `qbit_prism_audit_artifact_files`, split by body, share segment, live bundle, - candidate, and other artifact kinds -- CTV retry pressure via `qbit_prism_ctv_fanouts_failed` and the fanout-row - broadcast summary fields +Use explicit retention for logs, expired operational records, and image caches. +Do not delete inactive pool blocks, referenced shares, snapshots, or canonical +settlement evidence as routine cleanup. Aged shares leave through +`share-archive` and its checks, one partition at a time, and never through a +`DELETE`. Track storage growth and restore time against the measured ingest +model rather than relying on the former Python pilot's repeated-audit-file +footprint. diff --git a/docs/prism-stratum-fuzzing.md b/docs/prism-stratum-fuzzing.md new file mode 100644 index 000000000..7b624e16f --- /dev/null +++ b/docs/prism-stratum-fuzzing.md @@ -0,0 +1,128 @@ +# Stratum fuzzing and property tests + +Issue #575 item 4. Three cargo-fuzz targets search the Stratum front door for +hours; a small property-test suite holds the same contracts on every PR. + +## What runs where + +| What | Where | When | Budget | +| --- | --- | --- | --- | +| `tests/stratum_properties.rs` (proptest) | `rust-tests` CI job (`cargo test --workspace`) and the database job's workspace run | every PR | well under a second of run time; one more test binary to build | +| `stratum_session`, `stratum_lines`, `codec_parsers` (cargo-fuzz) | `stratum-fuzz` job of `prism-load-nightly.yml` | nightly and on dispatch | 20 minutes, all three at once | +| the same targets | a developer machine | on demand | as long as you like | + +The property tests read no environment input, so they are not gated and not +listed in `test/prism-gated-tests.txt`; see +[the integration test gate](prism-integration-test-gate.md). + +## The seam + +The targets drive the production per-connection loop, +`qbit_prism_server::stratum::serve_connection`, over an in-memory pipe +(`tokio::io::duplex`). `run_listener` calls the same function with an accepted +socket's halves after `set_nodelay`, so framing, timeouts, buffer bounds, +budgets and shutdown are the production ones. Admission (the global, per-source +and per-username limits taken at accept) and `StratumConfig::validate` stay with +the caller; the harness validates its configuration itself. + +The backend is `FuzzBackend` in `crates/qbit-prism-server/fuzz/src/backend.rs`: +no database, node or network. It builds real jobs from real payout manifests, +remembers every job it built, and credits a share by the coordinator's gate +(not block-only, current tip or inside stale grace, meets the share target, not +a duplicate). Everything the coordinator decides against PostgreSQL is out of +scope here; the ledger and live suites cover it. + +## Targets + +- **`stratum_lines`**: arbitrary bytes, cut into arbitrary fragments, into one + connection. Byte 0 picks the message bound (256 B to 16 KiB), byte 1 the + malformed-frame budget, byte 2 the fragment sizes; the rest is the stream. +- **`stratum_session`**: a script of requests and backend events. Each line is + a frame, sent after placeholders such as `$job`, `$en2` and `$nonce` are + replaced with live values (a nonce that meets the job's target, when one is + near), or an event: `#refresh` (new tip), `#revision` (same-parent payout + replacement), `#reconnect`, `#fail-build`, `#fail-submit`, `#pipeline`, + `#sleep`, `#again`. An optional `#cfg` first line sets the listener: message + bound, extranonce2 size, stale grace, retention, per-connection job cap, + per-username limit, the three session budgets, vardiff and its retarget + timing, difficulties, network bits, template transactions and the + version-rolling mask. The backend keeps retained vardiff evidence like the + ledger does, so reconnects resume a difficulty. `fuzz/src/script.rs` + documents the grammar. +- **`codec_parsers`**: the codec's transaction, block, coinbase-split, compact + target, difficulty, hex and template parsers on arbitrary bytes. + +## Invariants + +A run fails, and libFuzzer saves the input, when any of these breaks: + +- **No panic**, in the session task or the harness. +- **Bounded memory.** A counting global allocator fails an iteration whose + heap grows past a fixed allowance plus a small multiple of its input, and + libFuzzer's `-malloc_limit_mb` and `-rss_limit_mb` catch single huge + allocations and leaks. No line the server writes may exceed four times the + message bound plus 16 KiB. +- **Framing.** Every complete frame within the bound is answered exactly once, + in order, with the frame's own `id` (any JSON type, compared exactly); a frame + that is not a JSON object gets a null-id `malformed-submit`; a frame over the + bound gets one null-id `Stratum message exceeds size limit` and the + connection closes; a trailing partial frame is dropped. How the stream is + fragmented changes none of it. The server closes only for an oversize frame or + a spent budget, and writes nothing after that answer. +- **Documented errors only.** Every error is `[code, message, data]` with a + reason ID from [the rejection reference](prism-rejections.md) and its code + (21 stale or unknown job, 22 duplicate, 23 low difficulty, 20 otherwise), or + one of the reason-less refusals (spent authorize or unknown-job budget, + username limit). `internal-error` never appears, and `backend-rpc-unavailable` + only after an injected fault. The answers that are fully determined by the + connection state and the fields are predicted exactly: submit before + subscribe and authorize, short or non-string params, another username, a + wrong-size extranonce2, a wrong-size ntime or nonce, a job never built. +- **Honest work.** Every `mining.notify` names a job the backend built for this + connection's extranonce1, after subscribe and authorize, matches it field for + field, and follows a `set_difficulty` equal to its share difficulty. + `build_job` is never asked for a non-finite or non-positive difficulty. +- **No credit for an invalid share.** The backend's credits and the submits + answered `true` correspond one to one, in order. Each credited share is + re-verified from the job as built and the raw wire fields, by code that shares + nothing with `Job::assemble_submission` but SHA-256: extranonce2 size, 4-byte + hex time and nonce, time not before `mintime`, version bits inside the mask + the job was persisted with, and the rebuilt header's hash at or below the + job's share target. The credit goes to the worker the job was issued to, and + never to work that a same-parent payout replacement delivered on the same + connection retired to block-only (#478). +- **Parsers.** Accepted transactions strip their witness idempotently, coinbase + splits reassemble, compact targets decode into range and re-encode, and a + higher difficulty never gives a larger target. + +## Running locally + +```sh +rustup toolchain install nightly-2026-09-24 --profile minimal +cargo install cargo-fuzz --locked --version 0.13.2 +cd crates/qbit-prism-server/fuzz +cargo +nightly-2026-09-24 fuzz run stratum_session corpus/stratum_session seeds/stratum_session -- \ + -dict=stratum.dict -max_len=16384 -rss_limit_mb=2048 -malloc_limit_mb=512 -timeout=30 +``` + +`corpus/` and `artifacts/` are ignored by git. To reproduce a finding, pass the +artifact instead of the corpus directories, with `PRISM_FUZZ_TRACE=1` to print +every frame and line. `bash .github/scripts/prism-stratum-fuzz.sh <corpus> +<output> [seconds]` runs exactly what the nightly job runs. + +`cargo test` in `fuzz/` (stable toolchain) replays every seed through its +target's checks without libFuzzer, checks that each session seed reaches the +state it is named for, and fails when `seeds/` drifts from the generator in +`fuzz/tests/seeds.rs`; `PRISM_FUZZ_REGENERATE_SEEDS=1 cargo test` rewrites it. +The seeds are the requests and codec vectors of `tests/stratum_protocol.rs` and +`tests/stratum_codec.rs`. + +## Nightly job + +`stratum-fuzz` in `prism-load-nightly.yml` restores the previous night's +corpus from the Actions cache, runs `.github/scripts/prism-stratum-fuzz.sh` for +1,200 seconds per target on an 8 vCPU runner, merges each corpus down to the +inputs that add coverage, saves it under a new cache key, and uploads the logs +and any crashing, leaking, timing-out or out-of-memory inputs. It fails when any +target found one. The script pins the nightly toolchain and the cargo-fuzz +release (by sha256) it installs. diff --git a/docs/prism-throughput-measurements.md b/docs/prism-throughput-measurements.md new file mode 100644 index 000000000..9b8d1ed37 --- /dev/null +++ b/docs/prism-throughput-measurements.md @@ -0,0 +1,1932 @@ +# PRISM throughput measurements (#271, #447) + +This document records throughput matrices measured by the `qbit-prism-load` +harness. The harness drives real `qbit-prism-server` frontends over real +Stratum sockets, with real proof of work, against a PostgreSQL primary and +standby that it manages. None of these numbers sets a CI floor. None is a +capacity claim for other hardware. + +It has two parts, and **no table mixes them**: + +- **The #271 matrix** on base `5d0042f6`, on a shared Linux VM: the 20k and + 100k windows. It is everything from + [What the matrix established](#what-the-matrix-established) to + [What was not measured](#what-was-not-measured), and within it every number + comes from that one host and that one build. +- **[The production-window matrix (#447)](#production-window-matrix-on-a1937054-447)** + on base `a1937054`, on a laptop: the 200k, 400k and 500k windows that #271 + could not run, found blocks under load and dense cadence. It has its own + host table, method notes and tables. + +Several statements in the #271 part were true of its base and are no longer +true of `3.x.x`: the three large windows run, found blocks land at 400k, and +dense cadence has been measured at D1 at the production window. Each carries +a pointer to the #447 part rather than being rewritten, because it is still +an accurate record of what that base did. + +This document does not change `docs/prism-capacity-readiness.md`, which #303 +owns. The raw side reports, artifacts, frontend logs and host samples are kept +outside the repository, and this document cites them by run id. + +## What the matrix established + +- **No configuration reaches 500 shares/s.** + - At the 20k window, every configuration's median `steady_state` rate fell + between 238 and 304 shares/s against the 500 shares/s target. Individual + runs ranged from 235 to 320 shares/s across three repeats. + - The 2,000 shares/s burst achieved no more. + - **Adding frontends does not add throughput.** Summed over all frontends, + CPU stayed at or below 0.65 cores in every phase. + - **`ORDER_LOCK` waiters saturate every pool.** The waiter queue peaked at + the frontends' total database connections minus the holder: 15, 31 and 63 + at 1, 2 and 4 frontends. It averaged 85–94 % of that in `steady_state`. + The share append takes that one advisory lock, so appends serialise on it. +- **Synchronous replication costs 12–18 % against asynchronous, consistently.** + - Median `steady_state` was 290, 304 and 288 shares/s async against 238, 251 + and 254 shares/s sync, at 1, 2 and 4 frontends. That is 18 %, 17 % and + 12 %. + - Every sync run was slower than every async run at the same frontend count. +- **200k, 400k and 500k are all refused by the JSONB ceiling (#273)** at + startup, with the verbatim refusal in [Blocked sizes](#blocked-sizes). On + `a1937054` all three run; see + [The blocked sizes now run](#the-blocked-sizes-now-run). + - **At 400k**, #265 (found-block candidates) would refuse next, but it cannot + be reached. See [The 400k result](#the-400k-result), which is what #271 + stays open for. +- **100k at 4 frontends does not fit on this host.** + - The async attempt stopped itself at the harness memory floor, at + `MemAvailable` 6,092 MiB against the 6,144 MiB floor, 64 s into + `steady_state`. + That run is the load-bearing evidence for this conclusion. + - The sync attempt was stopped from outside by the coordinator's memory guard + about 3 minutes in, so it shows only that free memory fell past an external + threshold, not that the harness declined to continue. It is reported for + completeness rather than relied on. + - At 1 frontend, 100k fits and the ceiling holds at five times the window: + 285 shares/s async and 238 shares/s sync. +- **ACK latency now counts acknowledgements only.** + - The median `slow_database` ACK p50 is 12.2–15.3 s on this base. On + `2e94136c` the same column read 0.4 ms, because refusal response times were + blended into acknowledgement latency. + - Those earlier figures are kept under `base-2e94136` for comparison. The + harness fix is why they moved; see + [Comparison with the earlier harness bases](#comparison-with-the-earlier-harness-bases). +- **Repeats.** The 20k tables have three repeats per configuration. The 100k + tables have two, which cannot identify an outlier. +- **Other results.** + - Every run's capacity evidence is refused, as expected. + - No completed run on this base recorded an ACK/commit divergence (#324), an + unknown-outcome commit or a durability finding. + - The replication premise agreed at entry and after the load in every run. +- **The dense-cadence block was dropped.** At D1 the harness correctly refuses + to start the dense phase; see [Dense cadence](#dense-cadence). #447 + attempted it again; see [Dense cadence at D1](#dense-cadence-at-d1). + +## Host + +Every table in this document comes from this one host. No table mixes hosts. + +| Fact | Value | +|---|---| +| Host | alexdevbox2, a shared development VM | +| CPU | Intel Core Processor (Haswell, no TSX), 8 vCPU, x86_64 | +| RAM | 22.9 GiB (`MemTotal` 24,026,912 KiB), no swap, no cgroup CPU or memory limit | +| Disk | 200 GiB QEMU virtual disk, ext4 root; the managed clusters live on the same disk | +| OS | Ubuntu 24.04.4 LTS, Linux 6.8.0-106-generic | +| PostgreSQL | 16.15 (Ubuntu 16.15-0ubuntu0.24.04.1), managed by the harness: one primary and one standby per run | +| Durability | `fsync=on`, `full_page_writes=on`, `synchronous_commit=on`, read back from the primary for every run | +| Build | release profile for both `qbit-prism-server` and `qbit-prism-load`, built from `5d0042f629f3271ef6743ec9d859cb0c35018f31` with Rust 1.98.1 stable | +| Frontend settings | harness defaults: `PRISM_RUNTIME_WORKERS=2`, `PRISM_DATABASE_MAX_CONNECTIONS=16`, `PRISM_SHARE_COMMIT_TIMEOUT_SECONDS=15`, `PRISM_BLOCKPOLL_SECONDS=2`. `PRISM_STRATUM_MAX_PENDING_INITIAL_JOBS` was the harness's then-fixed `(sessions_per_frontend + 16)`, at least 128 (2,016 for 2,000 sessions on one frontend, 1,016 on two, 516 on four; the 200-session premise probes, `--frontends 2 --sessions 200`, ran the floor, 128), not always the server default of 128 the harness has launched since `--stratum-max-pending-initial-jobs` was added. Reproducing a 2,000-session row needs that value passed explicitly; a premise-probe row reproduces with the flag omitted. Each side report's `frontend_environment[].environment` carries the value its run used | + +**The host was shared.** Other users' agent sessions, their `git` processes +and two unrelated PostgreSQL containers ran on it throughout. For each run the +harness driver records three things: the 1-minute load average and +`MemAvailable` just before the run (after a 60 s cooldown), the same two every +10 s during the run, and a process snapshot at each end. The load average and +lowest `MemAvailable` are reported beside each run's numbers. Before a run, the +load average was 0.8–2.8 across the whole matrix. The load average during a +run includes the run's own PostgreSQL, frontends and client. + +## Method + +- **Plan.** D1 (`--plan d1`) runs these phases in order: + - a 30 s `warm_up`, which is not in the artifact; + - `steady_state` at 500 shares/s for 300 s; + - `burst` at 2,000 shares/s for 60 s; + - `reconnect` at 500 shares/s for 60 s, with one drained frontend restart; + - `slow_database` at 500 shares/s for 60 s, behind a 10 ms one-way proxy + delay. +- **Load shape.** 2,000 Stratum sessions, round-robin across the frontends, + each with one outstanding submit at a time. +- **Validator inputs.** Forecast peak 2,000 shares/s; ACK p99 limit 1,000 ms. +- **Memory floor.** `--min-mem-available-mib 6144`, above the external + memory guard's threshold of 25 % of `MemTotal` (5,866 MiB). +- **Order and repeats.** + - Runs were strictly one at a time, with a 60 s cooldown between them. + - **20k: three repeats per configuration**, interleaved. One pass runs async + at 1, 2 and 4 frontends, then sync at 1, 2 and 4, and the next pass repeats + that order. + - **100k: two repeats per configuration**, at 1 and 4 frontends only, by + coordinator decision. +- **Cells.** Each cell shows the median of the repeats, with the minimum and + maximum in parentheses. +- **Two repeats cannot identify an outlier.** A two-run cell's "median" is the + mean of the two runs, and its range is just those two runs. It carries less + confidence than a three-run cell. At 20k, the `steady_state` rate's spread + across three repeats reached 14.4–14.5 % of the median at 1 and 4 + frontends, async. +- **Units.** + - Rates are shares per second. + - ACK latency is measured on the client, from writing the submit to reading + its response, in milliseconds on the client's monotonic clock. + - On this base, ACK latency covers **accepted shares only**. Refusal latency + is reported separately. +- **Phase abbreviations.** In the tables, *steady* is `steady_state`, *reconn* + is `reconnect` and *slowdb* is `slow_database`. +- **Achieved and offered.** "Achieved" is acknowledged shares per second. + "Offered" is what the client actually sent. Sessions wait for their previous + response, so offered falls below the target when responses are slow. +- **Advisory-lock waits are sampled.** The harness polls `pg_locks` every + 10 ms, so waiter-seconds is a lower bound. The `pg_stat_statements` columns + give the calls and total execution time of the one normalized advisory-lock + statement during the phase. That execution time includes time spent waiting + for the lock. +- **Frontend CPU** is the mean number of cores over the phase. *Sum* adds all + frontends together; *max* is the busiest single frontend. +- **Memory.** **RSS** is the maximum sampled resident set. **VmHWM** is the + kernel's peak. +- **Teardown tails.** A submit still outstanding when the run's 25 s drain + expires, and which PostgreSQL then held, is reported as a no-response commit + with `window_ended: true`. This is not a divergence, and the run still exits + 0. Tails are listed per run below. + +## Replication premise probes + +Two short probes ran before the matrix to check the replication premise +against a real managed cluster. Both used 2 frontends, 200 sessions, the short +plan at 50 shares/s, and the 20k window. + +| Run id | Exit | Declared | Observed at entry | Observed after load | Agreed | Share difficulty agreed | +|---|---|---|---|---|---|---| +| w11-premise-fe2-async-short50 | 0 | async | async | async | true | true | +| w11-premise-fe2-sync-short50 | 0 | sync | sync | sync | true | true | + +In both probes, every one of these was 0: no-response commits, ACK/commit +divergences, unknown-outcome commits, durability findings, submits outstanding +at stop, harness-bug rejections, and unaccounted offers in any phase. + +## 20k window: D1 matrix (three repeats) + +All 18 runs exited 0. Every artifact is refused; see +[Capacity-evidence verdicts](#capacity-evidence-verdicts). + +### Rate and ACK latency + +| fe | repl | n | steady achieved/offered (target) /s | steady ACK p50 / p99 / max ms | burst achieved/offered (target) /s | burst ACK p50 / p99 / max ms | reconn achieved/offered (target) /s | reconn ACK p50 / p99 / max ms | slowdb achieved/offered (target) /s | slowdb ACK p50 / p99 / max ms | +|---|---|---|---|---|---|---|---|---|---|---| +| 1 | async | 3 | 290 (278–320) / 350 (344–366) (500) | 6,186 (5,679–6,351) / 8,638 (8,497–10,794) / 12,935 (12,456–15,649) | 278 (274–300) / 598 (552–610) (2000) | 6,422 (5,983–6,552) / 11,000 (10,217–12,056) / 15,857 (15,567–18,902) | 301 (289–303) / 364 (342–368) (500) | 6,100 (5,804–6,410) / 9,234 (7,498–9,836) / 12,572 (7,862–13,578) | 3.9 (2.3–4.1) / 322 (316–322) (500) | 12,619 (11,931–12,802) / 30,364 (30,306–30,485) / 30,506 (30,396–30,519) | +| 2 | async | 3 | 304 (299–313) / 384 (370–385) (500) | 6,089 (5,763–6,190) / 8,462 (7,559–9,123) / 15,463 (13,500–16,614) | 267 (232–308) / 762 (687–960) (2000) | 6,670 (5,878–7,344) / 9,767 (9,479–12,920) / 18,133 (15,564–20,134) | 283 (264–339) / 372 (343–383) (500) | 6,001 (5,462–6,104) / 7,581 (6,419–10,832) / 15,925 (8,942–15,951) | 2.8 (2.6–3.9) / 413 (407–416) (500) | 14,705 (14,595–14,716) / 32,323 (29,989–33,613) / 32,746 (29,995–33,646) | +| 4 | async | 3 | 288 (266–307) / 410 (379–412) (500) | 6,359 (5,987–6,762) / 7,930 (7,562–9,909) / 15,365 (14,176–19,114) | 251 (244–295) / 1,283 (1,018–1,603) (2000) | 7,483 (6,390–7,552) / 9,450 (8,971–10,878) / 20,011 (19,365–21,943) | 271 (253–302) / 416 (412–428) (500) | 6,350 (5,692–6,599) / 7,230 (7,077–9,155) / 10,674 (9,594–13,569) | 5.9 (5.8–6.2) / 414 (412–418) (500) | 15,346 (14,152–16,512) / 29,014 (28,204–34,085) / 29,848 (29,304–34,658) | +| 1 | sync | 3 | 238 (235–238) / 325 (320–327) (500) | 7,321 (7,251–7,441) / 11,675 (11,524–11,840) / 18,166 (17,557–19,077) | 231 (227–243) / 711 (566–860) (2000) | 7,910 (7,607–7,970) / 13,538 (13,237–14,494) / 21,370 (20,040–21,437) | 249 (239–262) / 333 (289–357) (500) | 7,275 (7,202–7,391) / 8,207 (8,152–8,286) / 8,414 (8,319–8,865) | 4 (1.7–4.2) / 320 (310–322) (500) | 12,204 (9,188–12,727) / 30,334 (19,019–30,397) / 30,386 (29,935–30,407) | +| 2 | sync | 3 | 251 (248–256) / 352 (349–361) (500) | 7,217 (7,075–7,222) / 10,768 (10,306–11,477) / 16,944 (16,869–17,402) | 225 (210–238) / 810 (779–864) (2000) | 7,740 (7,430–7,839) / 12,999 (11,216–13,980) / 20,713 (19,475–21,194) | 261 (256–262) / 350 (348–382) (500) | 6,900 (5,567–7,060) / 8,114 (7,833–8,492) / 8,540 (7,945–9,009) | 3.9 (3.8–4.5) / 433 (396–440) (500) | 13,931 (13,493–14,296) / 34,143 (30,221–34,234) / 34,279 (32,038–35,252) | +| 4 | sync | 3 | 254 (248–264) / 382 (378–393) (500) | 7,240 (6,810–7,283) / 10,790 (10,270–10,926) / 20,286 (18,911–20,851) | 224 (218–253) / 1,270 (876–1,274) (2000) | 7,565 (7,542–7,994) / 13,115 (10,448–13,997) / 20,959 (19,470–21,341) | 245 (239–272) / 411 (400–442) (500) | 7,430 (6,553–7,502) / 9,306 (9,042–10,434) / 12,416 (11,171–17,749) | 5.1 (4.8–5.3) / 421 (407–442) (500) | 14,644 (14,517–16,404) / 29,148 (28,942–29,359) / 29,696 (29,625–30,034) | + +### Refusal latency + +A refusal's latency runs from writing the submit to reading the refusal. The +count of refusals is in the last parentheses. + +| fe | repl | steady refusal p50 / p99 ms (refusals) | burst refusal p50 / p99 ms (refusals) | reconn refusal p50 / p99 ms (refusals) | slowdb refusal p50 / p99 ms (refusals) | +|---|---|---|---|---|---| +| 1 | async | 0.5 (0.5–0.5) / 9,191 (8,288–10,859) (18,149 (13,592–19,660)) | 0.5 (0.4–0.5) / 95 (6–484) (19,205 (15,111–20,144)) | 0.5 (0.4–0.5) / 500 (290–500) (3,978 (2,329–4,466)) | 0.5 (0.5–0.5) / 29,716 (29,643–29,741) (19,105 (18,721–19,186)) | +| 2 | async | 0.5 (0.5–0.5) / 8,248 (8,080–9,950) (24,352 (16,808–25,340)) | 0.5 (0.4–0.5) / 338 (154–417) (31,782 (22,773–41,533)) | 0.5 (0.5–0.5) / 378 (344–483) (4,697 (2,646–5,385)) | 0.5 (0.4–0.5) / 29,754 (29,496–29,812) (23,881 (23,609–24,183)) | +| 4 | async | 0.5 (0.5–0.6) / 7,753 (6,192–9,963) (34,002 (31,267–36,658)) | 0.5 (0.5–0.5) / 216 (101–323) (61,910 (43,422–81,547)) | 0.5 (0.5–0.5) / 454 (413–470) (8,739 (7,499–9,506)) | 0.5 (0.5–0.5) / 28,284 (26,028–29,031) (24,214 (24,104–24,419)) | +| 1 | sync | 0.5 (0.5–0.5) / 12,953 (10,222–13,062) (26,826 (24,822–26,983)) | 0.5 (0.5–0.5) / 441 (416–479) (28,834 (19,416–37,990)) | 0.5 (0.4–0.5) / 500 (500–500) (4,291 (2,372–7,072)) | 0.5 (0.5–0.5) / 29,665 (29,249–35,612) (18,927 (17,392–19,099)) | +| 2 | sync | 0.5 (0.5–0.5) / 8,754 (8,099–9,603) (30,307 (30,304–31,484)) | 0.5 (0.5–0.5) / 315 (149–405) (35,091 (32,452–39,242)) | 0.4 (0.4–0.5) / 470 (182–492) (5,614 (5,268–7,224)) | 0.5 (0.5–0.5) / 30,011 (29,874–42,087) (24,777 (22,995–25,751)) | +| 4 | sync | 0.5 (0.5–0.5) / 7,786 (7,375–9,838) (40,349 (34,140–41,688)) | 0.5 (0.5–0.5) / 105 (100–413) (60,969 (39,077–63,358)) | 0.5 (0.5–0.5) / 466 (356–471) (9,942 (7,667–12,179)) | 0.5 (0.5–0.5) / 25,199 (22,889–25,914) (24,964 (24,130–26,174)) | + +### Rejections by `(code, reason_id, message)` + +Counts per phase. No harness-bug rejection class appeared in any run. + +| fe | repl | phase | `20` `backend-rpc-unavailable` "current chain state is unavailable" | `20` `backend-rpc-unavailable` "current payout state is unavailable" | `20` `ledger-confirmation-failed` "share was not confirmed by the database" | `21` `stale-job` "stale job" | +|---|---|---|---|---|---|---| +| 1 | async | steady | 16,149 (11,592–17,659) | 0 (0–0) | 0 (0–0) | 2,000 (2,000–2,001) | +| 1 | async | burst | 19,205 (15,111–20,144) | 0 (0–0) | 0 (0–0) | 0 (0–0) | +| 1 | async | reconn | 3,978 (2,329–4,466) | 0 (0–0) | 0 (0–0) | 0 (0–0) | +| 1 | async | slowdb | 15,204 (14,828–15,935) | 1,600 (568–2,520) | 2,293 (1,381–2,683) | 0 (0–0) | +| 2 | async | steady | 22,352 (14,808–23,340) | 0 (0–0) | 0 (0–0) | 2,000 (2,000–2,000) | +| 2 | async | burst | 31,782 (22,773–41,533) | 0 (0–0) | 0 (0–0) | 0 (0–0) | +| 2 | async | reconn | 4,697 (2,646–5,385) | 0 (0–0) | 0 (0–0) | 0 (0–0) | +| 2 | async | slowdb | 21,611 (21,299–21,754) | 7 (5–15) | 2,112 (1,993–2,877) | 0 (0–0) | +| 4 | async | steady | 31,878 (29,267–34,426) | 0 (0–0) | 0 (0–0) | 2,124 (2,000–2,232) | +| 4 | async | burst | 61,910 (43,422–81,547) | 0 (0–0) | 0 (0–0) | 0 (0–0) | +| 4 | async | reconn | 8,739 (7,499–9,506) | 0 (0–0) | 0 (0–0) | 0 (0–0) | +| 4 | async | slowdb | 21,011 (20,800–21,208) | 74 (54–102) | 3,312 (2,842–3,334) | 0 (0–0) | +| 1 | sync | steady | 24,826 (22,131–24,983) | 0 (0–0) | 0 (0–0) | 2,000 (2,000–2,691) | +| 1 | sync | burst | 28,834 (19,416–37,990) | 0 (0–0) | 0 (0–0) | 0 (0–0) | +| 1 | sync | reconn | 4,291 (2,372–7,072) | 0 (0–0) | 0 (0–0) | 0 (0–0) | +| 1 | sync | slowdb | 15,203 (15,041–15,463) | 1,806 (235–2,403) | 1,694 (1,493–2,080) | 0 (0–0) | +| 2 | sync | steady | 28,305 (28,229–29,483) | 0 (0–0) | 0 (0–0) | 2,002 (2,001–2,075) | +| 2 | sync | burst | 35,091 (32,452–39,242) | 0 (0–0) | 0 (0–0) | 0 (0–0) | +| 2 | sync | reconn | 5,614 (5,268–7,224) | 0 (0–0) | 0 (0–0) | 0 (0–0) | +| 2 | sync | slowdb | 22,867 (20,119–23,805) | 14 (11–153) | 2,731 (958–2,865) | 0 (0–0) | +| 4 | sync | steady | 37,944 (32,140–39,353) | 0 (0–0) | 0 (0–0) | 2,335 (2,000–2,405) | +| 4 | sync | burst | 60,969 (39,077–63,358) | 0 (0–0) | 0 (0–0) | 0 (0–0) | +| 4 | sync | reconn | 9,942 (7,667–12,179) | 0 (0–0) | 0 (0–0) | 0 (0–0) | +| 4 | sync | slowdb | 22,063 (21,733–23,314) | 98 (10–257) | 2,644 (2,387–2,762) | 0 (0–0) | + +### ORDER_LOCK, SETTLEMENT_LOCK and `pg_stat_statements` lock time + +| fe | repl | steady ORDER_LOCK max / mean waiters · waiter-s ≥ | steady SETTLEMENT_LOCK max / mean · waiter-s ≥ | steady lock stmt calls / total exec ms | burst ORDER_LOCK max / mean · waiter-s ≥ | burst SETTLEMENT_LOCK max / mean · waiter-s ≥ | burst lock stmt calls / total exec ms | reconn ORDER_LOCK max / mean · waiter-s ≥ | reconn SETTLEMENT_LOCK max / mean · waiter-s ≥ | reconn lock stmt calls / total exec ms | slowdb ORDER_LOCK max / mean · waiter-s ≥ | slowdb SETTLEMENT_LOCK max / mean · waiter-s ≥ | slowdb lock stmt calls / total exec ms | +|---|---|---|---|---|---|---|---|---|---|---|---|---|---| +| 1 | async | 15 (15–15) / 13.05 (12.9–13.12) · 3,912 (3,864–3,932) | 15 (15–15) / 0.93 (0.92–1.09) · 279 (276–328) | 95,540 (91,994–104,284) / 4,208,315 (4,207,615–4,224,209) | 15 (15–15) / 12.56 (12.54–12.82) · 752 (752–768) | 15 (15–15) / 1.25 (1.13–1.33) · 74.8 (67.4–79.8) | 18,208 (17,828–19,945) / 835,980 (831,176–839,664) | 15 (15–15) / 12.71 (12.67–14.29) · 762 (759–857) | 15 (0–15) / 1.13 (0–1.16) · 67.6 (0–68.5) | 19,357 (18,522–20,076) / 835,521 (831,370–861,671) | 15 (15–15) / 9.41 (7.44–10.89) · 564 (447–650) | 0 (0–7) / 0 (0–0.06) · 0 (0–3.4) | 1,429 (954–2,500) / 563,953 (446,703–658,015) | +| 2 | async | 31 (31–31) / 28.19 (27.68–28.34) · 8,449 (8,292–8,498) | 31 (31–31) / 1.46 (1.43–2.01) · 440 (430–606) | 99,342 (97,721–102,244) / 8,922,586 (8,918,861–8,957,885) | 31 (31–31) / 27.04 (26.05–27.61) · 1,620 (1,558–1,655) | 31 (15–31) / 2.35 (1.59–3.17) · 142 (95–192) | 18,370 (16,097–20,437) / 1,762,222 (1,762,028–1,771,431) | 31 (31–31) / 27.41 (24.33–28.1) · 1,642 (1,448–1,683) | 16 (16–31) / 1.49 (1.32–4.28) · 89 (79–264) | 18,912 (18,836–22,436) / 1,747,394 (1,726,697–1,776,260) | 31 (31–31) / 17.86 (17.54–18.26) · 1,075 (1,053–1,097) | 1 (1–1) / 0.08 (0.06–0.08) · 4.9 (3.3–5) | 1,874 (1,665–2,421) / 1,044,616 (1,022,458–1,071,911) | +| 4 | async | 63 (63–63) / 58.98 (56.43–59.22) · 17,681 (16,914–17,754) | 32 (32–63) / 1.92 (1.87–4.11) · 579 (567–1,234) | 94,969 (88,053–101,606) / 18,325,162 (18,207,141–18,382,415) | 63 (63–63) / 57.88 (57.08–59.11) · 3,470 (3,422–3,542) | 32 (16–39) / 2.42 (1.66–2.86) · 145 (101–171) | 17,099 (16,634–18,701) / 3,655,179 (3,620,239–3,675,014) | 63 (63–63) / 59.09 (58.04–59.84) · 3,541 (3,482–3,587) | 17 (16–32) / 1.3 (0.88–2.01) · 78 (52–120) | 18,340 (17,262–20,745) / 3,660,660 (3,641,823–3,673,196) | 63 (63–63) / 43.63 (40.56–47.39) · 2,622 (2,438–2,846) | 3 (3–16) / 0.85 (0.56–1.57) · 50.9 (33.7–94.1) | 1,611 (1,130–1,689) / 1,841,510 (1,699,275–2,001,131) | +| 1 | sync | 15 (15–15) / 12.77 (12.75–12.79) · 3,827 (3,821–3,833) | 15 (15–15) / 1.19 (1.19–1.2) · 358 (357–359) | 79,490 (79,355–80,731) / 4,234,493 (4,228,749–4,241,389) | 15 (15–15) / 12.38 (12.23–13.71) · 742 (731–822) | 15 (15–15) / 1.49 (0.17–1.59) · 88.5 (9.8–96.4) | 14,922 (13,900–15,261) / 841,107 (837,476–842,681) | 15 (15–15) / 14.27 (12.95–14.27) · 856 (776–856) | 1 (0–15) / 0 (0–1.33) · 0 (0–80) | 16,279 (15,722–16,686) / 867,636 (867,267–868,674) | 15 (15–15) / 9.62 (4.09–10.74) · 577 (247–640) | 3 (0–6) / 0.01 (0–0.04) · 0.8 (0–2.5) | 1,534 (1,014–1,548) / 580,148 (247,282–645,757) | +| 2 | sync | 31 (31–31) / 27.39 (27.3–27.64) · 8,214 (8,179–8,286) | 31 (31–31) / 2.28 (2.05–2.28) · 684 (618–689) | 83,968 (82,651–85,085) / 8,954,420 (8,926,213–8,961,739) | 31 (31–31) / 25.71 (25.62–26.65) · 1,538 (1,534–1,597) | 31 (31–31) / 3.3 (2.68–3.56) · 198 (161–213) | 15,719 (14,614–15,741) / 1,760,972 (1,752,960–1,773,282) | 31 (31–31) / 28.63 (27.66–28.74) · 1,715 (1,658–1,722) | 16 (15–17) / 0.84 (0.83–1.5) · 50.1 (49.4–89.2) | 17,483 (17,071–17,677) / 1,784,937 (1,766,441–1,787,743) | 31 (31–31) / 17.87 (9.84–18.69) · 1,074 (592–1,122) | 6 (1–16) / 0.26 (0.1–3.7) · 15 (6–226) | 2,004 (1,204–2,036) / 1,047,327 (780,684–1,115,924) | +| 4 | sync | 63 (63–63) / 56.82 (56.4–56.83) · 17,039 (16,914–17,042) | 63 (48–63) / 4.02 (3.53–4.29) · 1,207 (1,059–1,286) | 84,492 (83,895–87,578) / 18,293,714 (18,183,425–18,345,445) | 63 (63–63) / 53.34 (51.87–54.69) · 3,191 (3,105–3,277) | 63 (32–63) / 7.12 (4.83–7.37) · 432 (290–444) | 15,340 (15,148–16,101) / 3,604,921 (3,575,778–3,665,108) | 63 (63–63) / 58.91 (53.91–59.85) · 3,532 (3,230–3,588) | 18 (17–48) / 1.45 (1.02–5.08) · 87 (61–305) | 15,893 (15,737–18,598) / 3,646,332 (3,568,878–3,686,966) | 63 (63–63) / 41.48 (36.92–43.28) · 2,496 (2,228–2,604) | 10 (3–18) / 1.2 (0.76–1.79) · 72 (45–106) | 1,131 (1,050–2,035) / 2,014,481 (1,631,188–2,032,333) | + +### Frontend CPU, memory and host load + +| fe | repl | steady CPU cores (sum / max fe) | burst CPU cores (sum / max fe) | reconn CPU cores (sum / max fe) | slowdb CPU cores (sum / max fe) | RSS max MiB (max fe) | VmHWM MiB (max fe) | lowest MemAvailable MiB | load1 before run | load1 during mean / max | +|---|---|---|---|---|---|---|---|---|---|---| +| 1 | async | 0.37 (0.37–0.39) / 0.37 (0.37–0.39) | 0.42 (0.42–0.42) / 0.42 (0.42–0.42) | 0.38 (0.36–0.39) / 0.38 (0.36–0.39) | 0.12 (0.12–0.13) / 0.12 (0.12–0.13) | 694 (633–731) | 694 (633–731) | 17,789 (16,813–18,050) | 1.42 (1.23–1.73) | 3.37 (2.77–3.67) / 4.55 (4.53–4.75) | +| 2 | async | 0.42 (0.42–0.43) / 0.21 (0.21–0.22) | 0.49 (0.47–0.55) / 0.26 (0.24–0.29) | 0.45 (0.43–0.45) / 0.23 (0.22–0.23) | 0.19 (0.17–0.19) / 0.11 (0.09–0.11) | 655 (617–657) | 655 (617–708) | 16,273 (15,464–16,907) | 0.96 (0.96–1.67) | 3.87 (3–3.92) / 5.49 (4.24–5.97) | +| 4 | async | 0.48 (0.45–0.49) / 0.12 (0.12–0.13) | 0.65 (0.57–0.71) / 0.2 (0.16–0.2) | 0.48 (0.44–0.49) / 0.13 (0.12–0.13) | 0.2 (0.2–0.2) / 0.07 (0.07–0.08) | 689 (647–693) | 689 (647–693) | 13,683 (12,942–14,295) | 1.93 (0.81–2.39) | 3.65 (3.28–4.44) / 5.14 (4.8–6.31) | +| 1 | sync | 0.33 (0.32–0.33) / 0.33 (0.32–0.33) | 0.4 (0.38–0.43) / 0.4 (0.38–0.43) | 0.31 (0.31–0.32) / 0.31 (0.31–0.32) | 0.12 (0.12–0.15) / 0.12 (0.12–0.15) | 638 (550–704) | 660 (550–755) | 17,313 (16,536–17,728) | 1.98 (1.4–2.75) | 3.12 (2.9–3.52) / 4.42 (4.16–4.92) | +| 2 | sync | 0.37 (0.36–0.38) / 0.19 (0.18–0.19) | 0.46 (0.46–0.46) / 0.24 (0.23–0.24) | 0.38 (0.37–0.4) / 0.2 (0.19–0.21) | 0.19 (0.18–0.2) / 0.11 (0.1–0.12) | 662 (639–713) | 662 (639–713) | 16,310 (16,188–16,668) | 1.68 (1.35–2.6) | 3.47 (3.34–3.75) / 5.14 (4.73–5.29) | +| 4 | sync | 0.42 (0.42–0.43) / 0.11 (0.11–0.12) | 0.6 (0.53–0.61) / 0.16 (0.15–0.17) | 0.44 (0.41–0.45) / 0.12 (0.11–0.12) | 0.2 (0.18–0.21) / 0.06 (0.06–0.07) | 628 (624–657) | 628 (624–657) | 13,721 (12,982–14,399) | 1.58 (1.05–1.7) | 3.49 (2.93–3.53) / 4.66 (3.83–4.84) | + +### Reconnects, time to usable work and outcomes + +The fake node mints three external tips 6 s apart, at heights 101, 102 and +103, early in each run. The time-to-usable-work cell counts only notifies that +arrive while their tip is still the tip. + +- **Tips 101 and 102.** No session received usable work before the tip was + replaced, in any run: 0 of 2,000 sessions, so no latency exists to report. +- **Tip 103, which is never replaced.** The cell gives the time until all + 2,000 sessions had work. + +| fe | repl | reconnect phase: completed / failed · reconnect p50 / max ms | tip 103: all 2,000 sessions with work, ms | ACK/commit divergences (#324) | unknown-outcome commits | durability findings | teardown tail (window-ended no-response commits) | harness-bug rejections | exit codes | +|---|---|---|---|---|---|---|---|---|---| +| 1 | async | 13 (13–13) / 0 (0–0) · 12,642 (12,318–14,439) / 20,815 (20,422–20,917) | 42,384 (35,235–48,100) | 0, 0, 0 | 0, 0, 0 | 0, 0, 0 | 0, 4, 4 | 0, 0, 0 | 0, 0, 0 | +| 2 | async | 1,012 (1,012–1,013) / 772 (383–819) · 4,322 (3,954–6,626) / 16,145 (13,629–16,639) | 45,562 (39,632–45,952) | 0, 0, 0 | 0, 0, 0 | 0, 0, 0 | 0, 1, 7 | 0, 0, 0 | 0, 0, 0 | +| 4 | async | 513 (512–513) / 351 (233–461) · 3,657 (3,447–3,767) / 20,979 (16,213–25,233) | 59,908 (40,505–61,786) | 0, 0, 0 | 0, 0, 0 | 0, 0, 0 | 17, 12, 11 | 0, 0, 0 | 0, 0, 0 | +| 1 | sync | 13 (13–13) / 0 (0–0) · 16,115 (15,430–16,650) / 25,777 (24,380–26,068) | 53,878 (49,593–65,362) | 0, 0, 0 | 0, 0, 0 | 0, 0, 0 | 0, 5, 4 | 0, 0, 0 | 0, 0, 0 | +| 2 | sync | 1,012 (1,011–1,013) / 593 (275–760) · 4,231 (4,160–4,561) / 22,028 (16,802–25,100) | 55,188 (48,676–57,626) | 0, 0, 0 | 0, 0, 0 | 0, 0, 0 | 0, 8, 3 | 0, 0, 0 | 0, 0, 0 | +| 4 | sync | 513 (512–513) / 424 (307–498) · 3,763 (3,703–3,923) / 22,711 (19,051–28,382) | 65,256 (49,357–66,652) | 0, 0, 0 | 0, 0, 0 | 0, 0, 0 | 0, 0, 0 | 0, 0, 0 | 0, 0, 0 | + +Every `reconnect` phase lasted 60.0–60.001 s. At 2 and 4 frontends each +includes one drained frontend restart; at 1 frontend the phase recorded no +restart. So a restart still running at the phase deadline did not inflate the +rate denominator on this host. + +### Per run + +| run id | harness run tag | started (UTC) | exit | steady achieved /s · p99 ms | burst achieved /s · p99 ms | reconn achieved /s · p99 ms | slowdb achieved /s · p99 ms | lowest MemAvailable MiB | load1 before | load1 during mean | +|---|---|---|---|---|---|---|---|---|---|---| +| w11-20k-fe1-async-r1 | 99cc308d | 2026-09-15T13:38:22 | 0 | 320.4 · 8497.0 | 299.9 · 10217.0 | 301.4 · 9234.5 | 2.3 · 30485.4 | 18050 | 1.42 | 2.77 | +| w11-20k-fe2-async-r1 | dcd15dc1 | 2026-09-15T13:48:16 | 0 | 313.4 · 8462.3 | 307.5 · 9479.4 | 338.6 · 6418.7 | 2.6 · 29988.6 | 16907 | 0.96 | 3.0 | +| w11-20k-fe4-async-r1 | 62c943a5 | 2026-09-15T13:58:39 | 0 | 307.4 · 7561.6 | 294.8 · 8970.8 | 302.5 · 7076.7 | 5.9 · 28203.7 | 13683 | 0.81 | 3.28 | +| w11-20k-fe1-sync-r1 | c090b02a | 2026-09-15T14:09:05 | 0 | 237.5 · 11840.0 | 227.2 · 13537.9 | 249.1 · 8207.0 | 1.7 · 19019.2 | 16536 | 2.75 | 3.52 | +| w11-20k-fe2-sync-r1 | aad3e400 | 2026-09-15T14:19:28 | 0 | 250.7 · 10767.7 | 238.2 · 12998.8 | 260.7 · 7833.4 | 3.9 · 34143.0 | 16188 | 2.6 | 3.47 | +| w11-20k-fe4-sync-r1 | cdce683d | 2026-09-15T14:29:54 | 0 | 247.8 · 10789.7 | 253.3 · 10447.8 | 239.1 · 9042.4 | 5.1 · 29147.5 | 12982 | 1.05 | 3.49 | +| w11-20k-fe1-async-r2 | bb0a5d65 | 2026-09-15T14:40:20 | 0 | 290.0 · 8638.3 | 277.7 · 11000.5 | 289.1 · 9835.7 | 4.1 · 30306.0 | 16813 | 1.23 | 3.37 | +| w11-20k-fe2-async-r2 | 04d10be0 | 2026-09-15T14:50:41 | 0 | 303.7 · 9122.7 | 232.4 · 12920.0 | 264.5 · 10832.5 | 2.8 · 33613.2 | 15464 | 0.96 | 3.87 | +| w11-20k-fe4-async-r2 | 897e8d7a | 2026-09-15T15:01:06 | 0 | 265.6 · 9908.9 | 243.8 · 10878.3 | 253.3 · 9154.9 | 6.2 · 34084.7 | 12942 | 1.93 | 4.44 | +| w11-20k-fe1-sync-r2 | 6416bac2 | 2026-09-15T15:11:37 | 0 | 237.8 · 11523.8 | 242.6 · 14494.5 | 239.1 · 8151.7 | 4.2 · 30396.8 | 17313 | 1.98 | 2.9 | +| w11-20k-fe2-sync-r2 | 8680e498 | 2026-09-15T15:21:59 | 0 | 256.3 · 10305.7 | 224.8 · 11215.6 | 261.8 · 8491.8 | 4.5 · 30220.6 | 16310 | 1.35 | 3.34 | +| w11-20k-fe4-sync-r2 | 44935228 | 2026-09-15T15:32:22 | 0 | 253.7 · 10926.3 | 218.4 · 13115.1 | 245.3 · 9305.7 | 5.3 · 29358.6 | 14399 | 1.7 | 3.53 | +| w11-20k-fe1-async-r3 | f7a684b4 | 2026-09-15T15:42:50 | 0 | 278.5 · 10794.0 | 274.0 · 12055.6 | 302.7 · 7498.1 | 3.9 · 30363.6 | 17789 | 1.73 | 3.67 | +| w11-20k-fe2-async-r3 | b6779a8b | 2026-09-15T15:53:15 | 0 | 299.2 · 7558.8 | 267.2 · 9767.3 | 282.7 · 7581.3 | 3.9 · 32323.3 | 16273 | 1.67 | 3.92 | +| w11-20k-fe4-async-r3 | 9808f2fe | 2026-09-15T16:03:40 | 0 | 288.3 · 7929.6 | 251.0 · 9450.4 | 270.8 · 7229.5 | 5.8 · 29013.5 | 14295 | 2.39 | 3.65 | +| w11-20k-fe1-sync-r3 | ddcb51f9 | 2026-09-15T16:14:08 | 0 | 235.0 · 11674.8 | 230.6 · 13237.0 | 261.6 · 8286.1 | 4.0 · 30334.5 | 17728 | 1.4 | 3.12 | +| w11-20k-fe2-sync-r3 | 5a000353 | 2026-09-15T16:24:39 | 0 | 248.2 · 11477.3 | 210.4 · 13980.1 | 256.3 · 8113.7 | 3.8 · 34234.4 | 16668 | 1.68 | 3.75 | +| w11-20k-fe4-sync-r3 | 0a8bc48a | 2026-09-15T16:35:00 | 0 | 263.7 · 10270.5 | 224.2 · 13996.8 | 272.5 · 10434.0 | 4.8 · 28942.1 | 13721 | 1.58 | 2.93 | + +### Calibration probes + +These probes use 2 frontends, async replication, 2,000 sessions, the 20k +window, and the short plan: `warm_up` 30 s, `steady_state` 60 s, `reconnect` +60 s and `slow_database` 60 s. Both exited 0. They find the rate this host +sustains without shortfall. That rate is the one the dense short run uses. + +| run id | exit | steady achieved/target /s · ACK p50 / p99 ms | reconn achieved/target /s · ACK p99 ms | slowdb achieved/target /s · ACK p99 ms | teardown tail | lowest MemAvailable MiB | load1 before | +|---|---|---|---|---|---|---|---| +| w11-cal-fe2-async-short-r200 | 0 | 200 / 200 · 4.1 / 5,485 | 200 / 200 · 2,141 | 4.05 / 200 · 29,625 | 0 | 17,319 | 1.11 | +| w11-cal-fe2-async-short-r100 | 0 | 100 / 100 · 4.4 / 4,859 | 100 / 100 · 3,427 | 3.92 / 100 · 28,664 | 3 | 16,997 | 0.97 | + +At 200 shares/s the ACK median is about 4 ms, but p99 is still about 5 s. +Rate is therefore not the only source of the multi-second tail. + +## 100k window: D1 matrix (two repeats) + +**Every cell in this section is two runs, or one.** Two repeats cannot tell an +outlier from the configuration's real behaviour. The medians below are the +mean of two runs, and their ranges are only those two runs. The block runs 1 +and 4 frontends only; the frontend dimension is settled at 20k. + +### 1 frontend + +All four runs exited 0. + +| fe | repl | n | steady achieved/offered (target) /s | steady ACK p50 / p99 / max ms | burst achieved/offered (target) /s | burst ACK p50 / p99 / max ms | reconn achieved/offered (target) /s | reconn ACK p50 / p99 / max ms | slowdb achieved/offered (target) /s | slowdb ACK p50 / p99 / max ms | +|---|---|---|---|---|---|---|---|---|---|---| +| 1 | async | 2 | 285 (278–293) / 366 (360–373) (500) | 5,946 (5,792–6,101) / 8,560 (8,416–8,704) / 16,464 (15,732–17,195) | 331 (325–336) / 428 (410–446) (2000) | 6,076 (6,045–6,108) / 8,820 (6,729–10,912) / 12,694 (7,492–17,896) | 265 (258–272) / 405 (398–412) (500) | 5,895 (5,724–6,066) / 10,012 (9,563–10,460) / 16,021 (15,090–16,952) | 2.15 (2.1–2.2) / 408 (408–408) (500) | 11,772 (11,529–12,016) / 30,404 (30,362–30,446) / 30,475 (30,462–30,489) | +| 1 | sync | 2 | 238 (233–243) / 354 (351–357) (500) | 6,904 (6,796–7,012) / 11,546 (11,398–11,695) / 21,175 (20,712–21,639) | 252 (240–265) / 608 (504–712) (2000) | 7,364 (7,186–7,542) / 8,367 (8,275–8,458) / 12,585 (8,576–16,595) | 222 (220–224) / 373 (362–384) (500) | 7,030 (6,758–7,303) / 11,936 (10,842–13,030) / 16,519 (14,978–18,059) | 2.2 (2.2–2.2) / 386 (368–405) (500) | 12,048 (12,001–12,094) / 30,203 (30,015–30,391) / 30,259 (30,016–30,503) | + +| fe | repl | steady refusal p50 / p99 ms (refusals) | burst refusal p50 / p99 ms (refusals) | reconn refusal p50 / p99 ms (refusals) | slowdb refusal p50 / p99 ms (refusals) | +|---|---|---|---|---|---| +| 1 | async | 0.5 (0.5–0.5) / 9,520 (9,429–9,610) (24,347 (24,075–24,619)) | 0.45 (0.4–0.5) / 428 (414–441) (5,830 (4,430–7,231)) | 0.5 (0.5–0.5) / 500 (500–500) (8,420 (7,564–9,275)) | 0.5 (0.5–0.5) / 29,042 (28,996–29,088) (24,352 (24,319–24,385)) | +| 1 | sync | 0.5 (0.5–0.5) / 10,639 (10,457–10,821) (34,606 (33,985–35,227)) | 0.5 (0.5–0.5) / 172 (2–342) (21,349 (14,369–28,329)) | 0.45 (0.4–0.5) / 431 (361–501) (9,050 (8,535–9,565)) | 0.5 (0.5–0.5) / 29,355 (29,166–29,544) (23,040 (21,919–24,160)) | + +| fe | repl | phase | `20` `backend-rpc-unavailable` "current chain state is unavailable" | `20` `backend-rpc-unavailable` "current payout state is unavailable" | `20` `ledger-confirmation-failed` "share was not confirmed by the database" | `21` `stale-job` "stale job" | +|---|---|---|---|---|---|---| +| 1 | async | steady | 22,090 (21,830–22,351) | 0 (0–0) | 0 (0–0) | 2,256 (2,245–2,268) | +| 1 | async | burst | 5,830 (4,430–7,231) | 0 (0–0) | 0 (0–0) | 0 (0–0) | +| 1 | async | reconn | 8,420 (7,564–9,275) | 0 (0–0) | 0 (0–0) | 0 (0–0) | +| 1 | async | slowdb | 22,402 (22,371–22,433) | 186 (123–249) | 1,764 (1,703–1,825) | 0 (0–0) | +| 1 | sync | steady | 31,860 (30,814–32,907) | 0 (0–0) | 462 (0–925) | 2,283 (2,246–2,320) | +| 1 | sync | burst | 21,349 (14,369–28,329) | 0 (0–0) | 0 (0–0) | 0 (0–0) | +| 1 | sync | reconn | 9,050 (8,535–9,565) | 0 (0–0) | 0 (0–0) | 0 (0–0) | +| 1 | sync | slowdb | 20,891 (19,570–22,212) | 394 (307–482) | 1,754 (1,641–1,867) | 0 (0–0) | + +| fe | repl | steady ORDER_LOCK max / mean waiters · waiter-s ≥ | steady SETTLEMENT_LOCK max / mean · waiter-s ≥ | steady lock stmt calls / total exec ms | burst ORDER_LOCK max / mean · waiter-s ≥ | burst lock stmt calls / total exec ms | reconn ORDER_LOCK max / mean · waiter-s ≥ | reconn lock stmt calls / total exec ms | slowdb ORDER_LOCK max / mean · waiter-s ≥ | slowdb lock stmt calls / total exec ms | +|---|---|---|---|---|---|---|---|---|---|---| +| 1 | async | 15 (15–15) / 12.51 (12.43–12.59) · 3,743 (3,720–3,767) | 15 (15–15) / 0.82 (0.74–0.9) · 245 (220–269) | 93,876 (91,229–96,523) / 4,002,732 (4,001,888–4,003,575) | 15 (15–15) / 13.665 (13.04–14.29) · 819 (782–857) | 19,658 (19,657–19,658) / 860,499 (859,203–861,795) | 15 (15–15) / 11.49 (11.32–11.66) · 686 (676–696) | 17,896 (17,456–18,335) / 765,458 (757,805–773,112) | 15 (15–15) / 6.08 (6.02–6.14) · 366 (363–370) | 1,606 (1,495–1,717) / 365,958 (362,107–369,810) | +| 1 | sync | 15 (15–15) / 12.13 (11.97–12.29) · 3,629 (3,583–3,676) | 15 (15–15) / 1.11 (1.01–1.21) · 333 (304–361) | 79,740 (78,116–81,365) / 4,009,606 (3,992,401–4,026,810) | 15 (15–15) / 13.9 (13.52–14.28) · 834 (811–857) | 16,097 (15,644–16,550) / 866,260 (865,305–867,216) | 15 (15–15) / 11.045 (10.94–11.15) · 661 (655–666) | 14,698 (13,961–15,435) / 757,729 (756,733–758,725) | 15 (15–15) / 7.005 (6.52–7.49) · 422 (393–451) | 1,584 (1,526–1,642) / 421,802 (392,756–450,848) | + +| fe | repl | steady CPU cores | burst CPU cores | reconn CPU cores | slowdb CPU cores | RSS max MiB | VmHWM MiB | lowest MemAvailable MiB | load1 before run | load1 during mean / max | +|---|---|---|---|---|---|---|---|---|---|---| +| 1 | async | 0.43 (0.43–0.43) | 0.39 (0.39–0.39) | 0.435 (0.43–0.44) | 0.2 (0.2–0.2) | 2,741 (2,520–2,962) | 2,794 (2,619–2,969) | 13,240 (12,566–13,913) | 1.16 (0.9–1.42) | 3.14 (2.58–3.7) / 4.11 (3.03–5.19) | +| 1 | sync | 0.385 (0.38–0.39) | 0.385 (0.36–0.41) | 0.39 (0.39–0.39) | 0.21 (0.2–0.22) | 2,640 (2,506–2,775) | 2,770 (2,637–2,903) | 13,253 (12,636–13,870) | 1.35 (1.24–1.46) | 2.705 (2.62–2.79) / 3.63 (3.36–3.9) | + +| fe | repl | reconnect phase: completed / failed · reconnect p50 / max ms | tip 103: all 2,000 sessions with work, ms | ACK/commit divergences (#324) | unknown-outcome commits | durability findings | teardown tail | harness-bug rejections | exit codes | +|---|---|---|---|---|---|---|---|---|---| +| 1 | async | 13 (13–13) / 0 (0–0) · 12,340 (11,440–13,239) / 21,815 (14,972–28,658) | 57,227; 60,727 | 0, 0 | 0, 0 | 0, 0 | 0, 0 | 0, 0 | 0, 0 | +| 1 | sync | 13 (13–13) / 0 (0–0) · 11,766 (11,575–11,956) / 18,643 (18,068–19,219) | 71,326; 66,854 | 0, 0 | 0, 0 | 0, 0 | 0, 0 | 0, 0 | 0, 0 | + +As at 20k, no session had work at tips 101 or 102 before they were replaced. + +| run id | harness run tag | started (UTC) | exit | steady achieved /s · p99 ms | burst achieved /s · p99 ms | reconn achieved /s · p99 ms | slowdb achieved /s · p99 ms | lowest MemAvailable MiB | load1 before | load1 during mean | +|---|---|---|---|---|---|---|---|---|---|---| +| w11-100k-fe1-async-r1 | 7547cde1 | 2026-09-15T17:19:59 | 0 | 293.1 · 8416.5 | 336.0 · 6728.6 | 272.0 · 9563.3 | 2.2 · 30445.7 | 12566 | 0.9 | 3.7 | +| w11-100k-fe1-sync-r1 | 53fc52cd | 2026-09-15T17:34:19 | 0 | 243.3 · 11397.7 | 264.8 · 8274.8 | 224.3 · 10842.0 | 2.2 · 30015.0 | 12636 | 1.24 | 2.79 | +| w11-100k-fe1-async-r2 | 4e3210b4 | 2026-09-15T17:44:30 | 0 | 277.6 · 8704.1 | 325.1 · 10912.2 | 257.9 · 10459.8 | 2.1 · 30362.0 | 13913 | 1.42 | 2.58 | +| w11-100k-fe1-sync-r2 | 68a24d41 | 2026-09-15T17:54:41 | 0 | 233.3 · 11694.6 | 240.1 · 8458.5 | 219.9 · 13029.8 | 2.2 · 30391.0 | 13870 | 1.46 | 2.62 | + +### 4 frontends: did not fit on this host + +`w11-100k-fe4-async-r1` (harness run tag `a9f991a8`) exited 6. The harness +stopped the run itself 64 s into `steady_state` with the message `MemAvailable +fell to 6092 MiB, below the 6144 MiB floor`, and withheld the artifact. + +- **Memory.** The four frontends' VmHWM reached 2,390, 1,682, 1,687 and + 2,410 MiB, about 8.2 GiB together, and was still rising. +- **Host.** `MemAvailable` fell from 19,939 MiB just before the run to + 5,865 MiB at the last 10 s sample. +- **Load.** The 1-minute load average was 1.45 before the run. +- **Rates before the stop.** `warm_up` achieved 354 shares/s at ACK p99 + 6,462 ms. The partial `steady_state` achieved 266 shares/s at ACK p99 + 9,385 ms. + +`w11-100k-fe4-sync-r1` exited 6 after 177 s, with no side report. + +- **How it ended.** The coordinator's external memory guard sent it SIGTERM. + That guard's log, kept on the coordinator's machine rather than this host, + records the decision and the reading it acted on: + + ```text + 2026-09-15T18:22:05Z avail=6360MB total=23463MB pct=27 min_pct=26 running=4 + 2026-09-15T18:22:07Z avail=5686MB total=23463MB pct=24 min_pct=24 running=4 + 2026-09-15T18:22:07Z BELOW FLOOR: SIGTERM qbit-prism-load on alexdevbox2 + ``` + + The harness logged `qbit-prism-load: terminated; tearing down` at 18:22:08. + +- **This is weaker evidence than its async sibling, and the difference + matters.** The async attempt stopped *itself* at the harness's own floor, so + it demonstrates the harness declining to measure what it cannot measure. This + run was stopped from outside, so it demonstrates only that free memory fell + past an external threshold. The conclusion -- 100k at 4 frontends does not fit + on this host -- rests on the async run. + +- **Why the external guard won.** The harness floor is 6,144 MiB and the guard's + is 25 % of 23,463 MiB, about 5,866 MiB: a gap of roughly 280 MiB. Memory fell + from 11,658 MiB to 5,686 MiB in under a minute, and at that gradient 280 MiB + is a few seconds. The two thresholds are too close together for this + workload, and the harness should have been given more room to stop itself + first. That is a fact about how these measurements were set up, not about + PRISM. +- **Memory.** Before the run, `MemAvailable` was 19,871 MiB and the 1-minute + load average 0.51. In the last minute, 10 s samples went from 11,658 MiB + through 9,652, 8,819, 7,911 and 7,925 to 7,303 MiB; the signal arrived + before the next sample. + +No further 4-frontend repeat was attempted. A configuration that does not fit +needs no second demonstration. + +## Dense cadence + +**The dense-cadence block was dropped.** It is the input to #291's soak, not +a #271 gate, and it was not worth more host time. It was not retried at D1 or +at the calibrated sustained rate. The one attempt produced a finding: **at D1 +on this host, the `dense_cadence` phase cannot start.** The harness refuses to +publish numbers under a delay the phase does not declare, and that refusal is +correct behaviour. + +`w11-dense20k-fe1-async-r1` (D1 with `--cadence dense --scheduled-blocks 15`, +1 frontend, async, 20k window) exited 6. It completed every D1 phase, then the +harness aborted before the dense phase with this reason: + +```text +1201 submit(s) offered before the dense_cadence phase were still outstanding 25.0 s later, so its 0 ms delay could not be applied without them finishing under it +``` + +The dense phase runs at 0 ms database delay and follows `slow_database`, +which runs at 10 ms. The harness changes the proxy delay only once every submit +offered under the previous delay has settled. Otherwise those submits would +finish under the new delay while keeping the old phase's stamp. If they have +not settled within the 25 s phase-boundary limit, the harness aborts the run. + +`slow_database` left about 1,200 submits outstanding, and they did not drain +inside that limit. Its accepted-share ACK p99 on this host was 28–34 s in 17 +of the 18 runs at 20k, and 19 s in the other. + +The same condition appears at the end of the ordinary runs. In 15 of the 20 +runs at 20k (including the calibration probes), submits were still +outstanding when the 25 s stop drain expired, up to 1,116 of them. The earlier +dense runs on `2e94136c` completed only because that harness did not yet have +this check. + +Everything else in the aborted run was clean: + +- the premise agreed; +- there was no divergence, no durability finding and no teardown tail; +- the D1 phases matched the 20k matrix (steady 319 shares/s at p99 8,256 ms). + +The dense runs at 2 and 4 frontends, the repeats, and the short-plan run at +the calibrated 200 shares/s were not run. + +## Blocked sizes + +All three used 1 frontend, async, D1 and 2,000 sessions, with one attempt +each. Each exited 3 at startup: `load-fe-0 did not become ready within 120s +(listening=true)`. The only refusal kind the harness found in the frontend log +was the JSONB ceiling. The side report is the only output; there is no +artifact. + +| Size | Run id | Exit | Refusal text (frontend log, verbatim) | Matches | Lowest MemAvailable MiB | Unblocked by | +|---|---|---|---|---|---|---| +| 200k | w11-200k-fe1-async-r1 | 3 | `template refresh deferred error=error returned from database: total size of jsonb object elements exceeds the maximum of 268435455 bytes` | 8 | 13,124 | #273 | +| 400k | w11-400k-fe1-async-r1 | 3 | `template refresh deferred error=error returned from database: total size of jsonb object elements exceeds the maximum of 268435455 bytes` | 3 | 9,517 | #273; then #265 for found-block candidates | +| 500k | w11-500k-fe1-async-r1 | 3 | `template refresh deferred error=error returned from database: total size of jsonb array elements exceeds the maximum of 268435455 bytes` | 1 | 6,575 | #273; #265 refuses candidates from 400k | + +At 500k the refusal names `jsonb array elements`; at 200k and 400k it names +`jsonb object elements`. At 500k, seeding took the host to 6,575 MiB +`MemAvailable`, which is 431 MiB above the harness floor and about 700 MiB +above the external memory guard. + +## The 400k result + +**#271 stays open for this result.** (It has since closed, and #447 made the +re-run this section asks for. The run got past startup, and the #265 refusal +expected below did not occur; see +[Found blocks at 400k](#found-blocks-at-400k).) + +- **Run.** `w11-400k-fe1-async-r1`: 1 frontend, async, D1, 2,000 sessions, + `--window-shares 400000`. It exited 3 (blocked) after 156 s. +- **Startup.** The frontend never became ready: `load-fe-0 did not become + ready within 120s (listening=true)`. +- **Refusal.** The frontend log shows the refusal three times, verbatim: + + ```text + template refresh deferred error=error returned from database: total size of jsonb object elements exceeds the maximum of 268435455 bytes + ``` + +- **Harness classification.** It recorded `blocked.blocked: true`, found only + the `jsonb_ceiling` refusal kind, and wrote a startup-only side report and + no artifact. +- **Memory.** The lowest `MemAvailable` sampled was 9,517 MiB. The load + average was 0.98 before the run. + +Two issues stand between this size and a measurement: + +1. **#273, the JSONB ceiling.** Persisting the prepared job exceeds + PostgreSQL's 268,435,455-byte JSONB container limit. The template refresh + is deferred, so no frontend ever serves work. +2. **#265, found-block candidates at 400k.** The audit bundle written at + landing is refused at this window. **That refusal was not observed here, + and cannot be while #273 stands.** No work is served, so no share or + candidate is produced and nothing reaches the landing path. This run is + therefore evidence of the #273 block only; it is not yet evidence about + #265. + +Once #273 lands, the same command should be re-run. It is expected to get +past startup and reach the #265 refusal. If it does not, the plan changes. + +## Capacity-evidence verdicts + +The validator used forecast peak 2,000 shares/s and ACK p99 limit 1,000 ms. +Every completed run's artifact was refused with the same first error, `capacity +run did not acknowledge every offered valid share`, which is the expected +verdict. The next table shows why beyond that first error: the slowest artifact +phase is always `slow_database`, at 2–6 shares/s, with ACK p99 of 19–34 s. + +| Window | fe | repl | runs | valid | slowest artifact phase /s (min–max) | worst artifact ACK p99 ms (min–max) | suggested forecast (min–max) | suggested ACK p99 limit ms | +|---|---|---|---|---|---|---|---|---| +| 20k | 1 | async | 3 | 0 | 2.3–4.1 | 30,306–30,485 | 1.14–2.03 | 15,000 | +| 20k | 2 | async | 3 | 0 | 2.6–3.9 | 29,989–33,613 | 1.32–1.93 | 15,000 | +| 20k | 4 | async | 3 | 0 | 5.8–6.2 | 28,204–34,085 | 2.90–3.11 | 15,000 | +| 20k | 1 | sync | 3 | 0 | 1.7–4.2 | 19,019–30,397 | 0.83–2.09 | 15,000 | +| 20k | 2 | sync | 3 | 0 | 3.8–4.5 | 30,221–34,234 | 1.90–2.25 | 15,000 | +| 20k | 4 | sync | 3 | 0 | 4.8–5.3 | 28,942–29,359 | 2.38–2.67 | 15,000 | +| 100k | 1 | async | 2 | 0 | 2.1–2.2 | 30,362–30,446 | 1.03–1.09 | 15,000 | +| 100k | 1 | sync | 2 | 0 | 2.2–2.2 | 30,015–30,391 | 1.08–1.09 | 15,000 | + +Refusals make up 23–46 % of offered valid shares per run. They are mostly +`backend-rpc-unavailable` "current chain state is unavailable"; see the +rejection tables. The 1,000 ms ACK p99 limit is exceeded in every phase of +every run, including `steady_state`, where p99 is 7.6–11.8 s. The runs on the +aborted, blocked and did-not-fit paths withheld their artifacts, so they carry +no verdict. + +## Comparison with the earlier harness bases + +Earlier runs of the same matrix on this host are kept under their own bases. +They are evidence that the harness fixes changed what they were meant to +change. They are **not** a like-for-like comparison of the server, because the +server also changed between those bases: + +- from `2e94136c` to this base, 64 files under `crates/qbit-prism-server/src` + changed; +- from `840449b` to this base, 52 files changed. + +| Measure (20k, D1) | `2e94136c` (18 runs) | `840449b` (5 runs, r1 only) | `5d0042f6` (18 runs) | +|---|---|---|---| +| Harness ACK latency | accepted and refused shares blended | accepted only (H3) | accepted only | +| Proxy delay changed only after the previous phase settled (H2) | no | yes | yes | +| slow_database ACK p50 ms, median per configuration | 0.4 in every configuration | 11,219–20,980 | 12,204–15,346 | +| burst ACK p50 ms, median per configuration | 0.5–5,118 | 5,695–6,946 | 6,422–7,910 | +| reconnect ACK p99 ms, median per configuration | 18,553–29,753 | 6,083–13,930 | 7,230–9,306 | +| slow_database achieved /s, median per configuration | 2.9–31.9 | 3.0–5.1 | 2.8–5.9 | +| steady_state achieved /s, median per configuration | 286–346 | 286–340 | 238–304 | +| runs exiting 5 | 12 of 18 | 0 of 5 | 0 of 18 | + +What the table shows: + +- **ACK p50 under refusals.** On `2e94136c` the harness counted refusal + response times as acknowledgement latency. + - Refusals return in about 0.5 ms and outnumber acknowledgements in + `slow_database`, so that phase's ACK p50 read 0.4 ms in every + configuration. + - The burst ACK p50 read about 1 ms in five of six configurations. + - Once the p50 counts acknowledgements only (H3), `slow_database` is 12–15 s + and burst is 6–8 s, which is what the accepted shares actually waited. + + The change is the harness fix, not a change in the server's behaviour. The + blended figures stay under `base-2e94136` for this comparison. +- **Reconnect and slow_database.** The high reconnect ACK p99 and the higher + `slow_database` rates on `2e94136c` are consistent with submits from the + previous phase finishing under the next phase's stamp. That is what H2 + prevents, and both measures moved to the levels seen on `840449b` and here. +- **Exit 5.** The 12 exit-5 runs on `2e94136c` counted ACK/commit divergences + under the harness of that time. On this base a no-response commit at + teardown is classified as a window-ended tail rather than a divergence. No + completed run on this base recorded a divergence of either kind. +- **Steady-state rate.** It is lower on this base than on either earlier base, + with sync lower by 40–60 shares/s. The load average during runs was also + higher: a mean of 2.8–3.9 here against 1.6–3.0 on `840449b`. Because both + the server and the host's background load differ, this document does not + attribute the drop to either one. + +The `840449b` set has one run for each of five configurations; its cells are +ranges over those single runs, not medians. + +## Metric gaps (#278, #279) + +The server's native registry now exports `qbit_prism_share_ack_seconds`, +`qbit_prism_rejections_total`, the advisory-lock-wait and pool-acquire +histograms, and runtime lag and RSS gauges. The harness still had to measure +the following from outside, or could not resolve them from the server's own +series: + +1. **The cause of "current chain state is unavailable" is not identifiable.** + Three different submit-admission checks return the same `(code 20, + backend-rpc-unavailable, "current chain state is unavailable")`: + - no chain poll has completed; + - the last poll is older than the health timeout, without a share lease; + - the readiness generation changed during admission. + + Neither the refusal nor `qbit_prism_rejections_total{reason_id}` says which + check fired. At this load it is the largest refusal class, about 4k–62k + per phase at 20k, so the capacity result's main refusal cannot be attributed + from metrics. (#278, reject reasons.) +2. **The ACK histogram does not resolve the latencies measured here.** The + `qbit_prism_share_ack_seconds` buckets above 1 s are 2.5, 5, 10 and 30 s. + Accepted-share ACKs at this load sit between 5 and 10 s, and + `slow_database` sits at about 30 s. Server-side p99 is therefore only known + to lie between two buckets. The client-measured p99 in this document is the + only precise figure. (#278, share-ACK latency.) +3. **There is no per-tip time-to-usable-work series.** The server exposes + gauges such as `qbit_prism_stratum_oldest_pending_initial_job_seconds` and + `qbit_prism_stratum_current_tip_coverage_gap_seconds`. It has no + distribution of how long sessions waited for work on each tip. The harness + measured that on the client: 35–71 s for all 2,000 sessions, and zero + sessions served within a tip's 6 s life at tips 101 and 102. A gauge's + instantaneous oldest value cannot reproduce those two facts. (#278, time + to usable work.) +4. **Reconnect timing is client-side only.** Time to reconnect was measured + from the client's close across failed attempts. No server series covers it. + (#278.) +5. **Advisory-lock attribution needed `pg_locks` sampling.** + `qbit_prism_database_advisory_lock_wait_seconds{lock}` gives each wait's + duration by lock. The waiter queue depth (15, 31 and 63 here, equal to the + pool size) came from 10 ms `pg_locks` sampling. `pg_stat_statements` cannot + separate the locks because MIGRATION, ORDER and SETTLEMENT share one + normalized statement text. (#278, database pool and lock waits.) +6. **Alert inventory (#279).** None of the conditions this matrix hit have an + alert this document could cite: a sustained ACK p99 far above the 1,000 ms + limit, a refusal ratio of 23–46 %, or `MemAvailable` falling to the floor + at 100k with 4 frontends. Whether the deployed rules migrated under #279 + cover them should be checked against the #278 registry inventory. + +## Reproducing a run + +Build both binaries in release mode from the base commit. Then run the harness +with PostgreSQL 16 server binaries on the host. `<pg-bin>` is the directory +holding `initdb` and `postgres`, and `<out>` is any empty directory. + +```sh +cargo build --locked --release -p qbit-prism-load -p qbit-prism-server +``` + +Every run uses the same base command. `<args>` is the run's argument list from +the table below. + +```sh +target/release/qbit-prism-load \ + --server-bin target/release/qbit-prism-server \ + --pg-bin-dir <pg-bin> \ + <args> \ + --out <out> +``` + +| Run ids | `<args>` | +|---|---| +| `w11-premise-fe2-{async,sync}-short50` | `--frontends 2 --sessions 200 --window-shares 20000 --replication {async,sync} --plan short --rate 50 --forecast-peak-shares-per-second 2000 --ack-p99-limit-ms 1000 --min-mem-available-mib 6144` | +| `w11-20k-fe{1,2,4}-{async,sync}-r{1,2,3}` | `--frontends {1,2,4} --sessions 2000 --window-shares 20000 --replication {async,sync} --plan d1 --forecast-peak-shares-per-second 2000 --ack-p99-limit-ms 1000 --min-mem-available-mib 6144` | +| `w11-cal-fe2-async-short-r{200,100}` | `--frontends 2 --sessions 2000 --window-shares 20000 --replication async --plan short --rate {200,100} --forecast-peak-shares-per-second 2000 --ack-p99-limit-ms 1000 --min-mem-available-mib 6144` | +| `w11-100k-fe1-{async,sync}-r{1,2}`, `w11-100k-fe4-{async,sync}-r1` | `--frontends {1,4} --sessions 2000 --window-shares 100000 --replication {async,sync} --plan d1 --forecast-peak-shares-per-second 2000 --ack-p99-limit-ms 1000 --min-mem-available-mib 6144` | +| `w11-dense20k-fe1-async-r1` | `--frontends 1 --sessions 2000 --window-shares 20000 --replication async --plan d1 --cadence dense --scheduled-blocks 15 --forecast-peak-shares-per-second 2000 --ack-p99-limit-ms 1000 --min-mem-available-mib 6144` | +| `w11-{200,400,500}k-fe1-async-r1` | `--frontends 1 --sessions 2000 --window-shares {200000,400000,500000} --replication async --plan d1 --forecast-peak-shares-per-second 2000 --ack-p99-limit-ms 1000 --min-mem-available-mib 6144` | + +Runs were driven one at a time, detached from any interactive session, with a +60 s cooldown and a whole-run ceiling of 2,400 s (1,500 s for the blocked +sizes). The harness exit code is the run's result: + +| Exit | Meaning | +|---|---| +| 0 | completed | +| 3 | blocked | +| 4 | durability loss | +| 5 | ACK/commit divergence | +| 6 | aborted | +| 7 | harness-bug rejections | +| 8 | contradicted premise | + +## What was not measured + +This list is the #271 matrix's. #447 has since measured dense cadence at D1 +and the three blocked sizes; its own list is +[What was not measured (#447)](#what-was-not-measured-447). + +- **Dense cadence.** The block was dropped after its first run showed the + dense phase cannot start at D1 on this host. +- **100k at 4 frontends.** The configuration does not fit on this host, so + no repeat was attempted. +- **100k at 2 frontends, and a third 100k repeat.** Both were cut from the + plan; the 20k block settles the frontend dimension. + +## Production-window matrix on `a1937054` (#447) + +This part records the runs #271 could not make: the 200k, 400k and 500k +windows, the frontend dimension at 400k, one synchronous comparison, found +blocks under load, and dense cadence. It is a different host and a different +build from everything above, so it has its own host table, method notes and +tables. **No table in this part contains a run from the #271 host, and no +table above contains a run from this one.** + +### What the production-window matrix established + +- **200k, 400k and 500k all run.** Each exited 0 at one frontend with #271's + argument lists unchanged, so #273 holds; see + [The blocked sizes now run](#the-blocked-sizes-now-run). +- **Every ordinary run reconciled exactly.** All 22 runs of the rate matrix + exited 0, with no ACK/commit divergence, no unknown-outcome commit, no + durability finding, no acknowledgement lost inside a run and no harness-bug + rejection. One run has a teardown tail, as defined under [Method](#method): + `d1-400k-fe4-async-r3` (7, in `slow_database`). Those submits were still + outstanding when the 25 s stop drain expired, PostgreSQL holds them, and the + harness reports them with `window_ended: true` and exits 0. The runs outside + the medians are the two flush-control repeats that exited 5 and the 20k + dense attempt that exited 6; see [Runs outside the + medians](#runs-outside-the-medians). +- **The host is a laptop, and it is not the #291 rehearsal host,** which had + not been named. Its commits do not force the drive cache, and that decides + the next two results; see [What this host changes](#what-this-host-changes). +- **500 shares/s for five minutes: the rate was sustained in all 22 repeats, + with a median ACK p99 of 13–33 ms.** + - No token of the target went unplaced in any repeat, at any window, at 1, + 2 or 4 frontends, async or sync. + - By the strict rule four of the eight configurations read "met". The + other four each miss it on one to three shares of about 150,000, refused + at the commit gate when a tip's work arrived; see + [D1 verdict](#d1-verdict). +- **With a forced flush the same configuration sustains about 117 shares/s.** + The [flush control](#the-flush-control) changes one PostgreSQL setting and + nothing else. It misses 500 shares/s by a factor of four, in three repeats. + The result above is therefore what this build does when a commit costs + tens of microseconds, and it is not a prediction for the rehearsal host. + What one WAL flush costs there decides it, and `pg_test_fsync` measures + that in two minutes. +- **2,000 shares/s for one minute: not met by any configuration.** + - The median `burst` rate is 1,468–1,623 shares/s. + - **Adding frontends adds `ORDER_LOCK` waiters, not throughput:** 14.4, 30.3 + and 62.4 waiters on average at 1, 2 and 4 frontends, with all frontends + together using at most 0.48 cores. This is #271's finding at the + production window. + - The window size does not move it: 1,558, 1,623 and 1,550 shares/s at + 200k, 400k and 500k. The build control below adds 1,621 at 20k. +- **#271's 238–304 shares/s was its host, and its build has the higher + ceiling here.** The [build control](#the-build-control) runs #271's 20k + argument list on this host on both builds, three interleaved repeats each. + - Both sustain 500 shares/s, so the build is not what took #271's figure to + 500. + - In `burst` #271's build acknowledged 1,979–1,994 shares/s with + `ORDER_LOCK` unsaturated, and the current build 1,559–1,649 with it + saturated: about a fifth lower. The current build is more than ten times + better on ACK p99 at 500 shares/s, 17–32 ms against 418–446 ms. + - "Build" is the harness and the server together; both cross runs were + refused, so which of the two moved is not known. +- **Synchronous replication shows no cost here,** and that says little. At + 400k and 2 frontends, sync sustained 500 shares/s and its `burst` range + overlaps its async pair's. The standby shares the primary's disk and + neither forces a flush. +- **Found blocks land at 400k under load.** Three scheduled blocks were + accepted, no refusal was logged, and `candidates list` showed no unfinished + row afterwards. The #265 refusal #271 expected did not occur; see + [Found blocks at 400k](#found-blocks-at-400k). +- **Dense cadence ran at D1 at the production window,** in three runs of + three. While own blocks landed 9 to 20 s apart the pool acknowledged + 399–415 of the 500 shares/s offered, and every session had work for the new + tip about 3 s after each landing. 44 of 45 scheduled blocks landed; one, + found 9 s after the previous landing, was refused as a stale job. The + harness's own per-landing figures are empty on this build, because they key + on a rejection message the server no longer sends; see + [Dense cadence at D1](#dense-cadence-at-d1). +- **Time to usable work after a tip splits in two,** 3–8 s or 20–37 s + depending on the configuration, and the cause was not investigated; see + [Time to usable work after a tip](#time-to-usable-work-after-a-tip). +- **Memory is not a constraint on this build.** A frontend held at most + 1,869 MiB at a 500k window, at 4 frontends during `warm_up`, and about + 1,075 MiB at 1 frontend. The host never had less than 60,959 MiB + reclaimable. +- **The harness under-reported frontend CPU about 42 times on Apple + silicon.** It is fixed in the change that adds this part, and the CPU + figures here are converted; see + [Harness observations](#harness-observations-447). + +### Host (#447) + +Every table in this part comes from this one host. No table mixes it with +the #271 host above. + +| Fact | Value | +|---|---| +| Host | Anatolies-MacBook-Pro, a developer's laptop in interactive use. **It is not the #291 rehearsal host**, which had not been named when these runs were made; see [What this host changes](#what-this-host-changes) | +| CPU | Apple M5 Max, 18 cores (6 "Super" and 12 "Performance"), arm64. The core types are asymmetric and macOS decides placement | +| RAM | 128 GiB (`hw.memsize` 137,438,953,472), no swap in use, no memory or CPU limit | +| Disk | Apple SSD AP2048Z, 2 TB, APFS; the managed clusters live on the same disk | +| OS | macOS 26.6.2 (25G83), Darwin 25.6.0 | +| PostgreSQL | 16.15 (Homebrew), managed by the harness: one primary and one standby per run | +| Durability | `fsync=on`, `full_page_writes=on`, `synchronous_commit=on`, read back from the primary for every run. **`wal_sync_method=open_datasync`, the macOS default, which does not force the drive's write cache** | +| Build | release profile for both `qbit-prism-server` and `qbit-prism-load`, built from `a1937054ac50765c82222de5c7ac27edfd4925ac` with Rust 1.98.1 stable, in a clean detached worktree that nothing else touched | +| Frontend settings | harness defaults, unchanged from #271: `PRISM_RUNTIME_WORKERS=2`, `PRISM_DATABASE_MAX_CONNECTIONS=16`, `PRISM_SHARE_COMMIT_TIMEOUT_SECONDS=15`, `PRISM_BLOCKPOLL_SECONDS=2` | +| Power | AC power, **except for about 35 minutes on battery** on 2026-09-18, from about 17:11 to between 17:34 and 17:48 UTC. `d1-400k-fe2-async-blocks3-r1` and `d1-dense20k-fe1-async-r1` ran wholly on battery; `d1-dense400k-fe1-async-r1` started on battery and ended on AC; `d1-400k-fe2-sync-r3` ended on battery, and the battery still read its held 80 % a minute after the run, which puts the switch in the run's last minutes, after `steady_state` and `burst` had ended. `pmset` reports `powermode 0` for both sources, so Low Power Mode was off. The power source and `pmset -g therm` were recorded before and after every D1 run (not the premise probe, which predates the recording), and no run recorded a thermal or performance warning | + +**The host was shared.** It is a workstation. The window server, a browser, a +chat client, other coding-agent sessions and two unrelated PostgreSQL +instances ran throughout, outside this measurement's control. The driver +records the 1-minute load average and a memory sample just before each run +(after a 60 s cooldown) and every 10 s during it, and a process snapshot at +each end, as #271's driver did. Both are reported beside each run. Before a +run the 1-minute load average was 1.56–8.15 across the matrix, on 18 +cores; #271's was 0.8–2.8 on 8. + +### What this host changes + +The difference from the #291 rehearsal host is not only size. These facts +about this host decide how far its numbers travel, and the first one decides +the verdict. + +- **Commits do not force the drive cache here, and the 500 shares/s result + depends on that.** + - PostgreSQL on macOS defaults to `wal_sync_method=open_datasync`. On macOS + neither `O_DSYNC` nor `fsync()` forces the drive's write cache; only + `fsync_writethrough` (`F_FULLFSYNC`) does. The harness sets `fsync=on` and + reads it back, but it does not set the sync method. + - `pg_test_fsync` on this disk, with nothing else running: `open_datasync` + 28,992 ops/s (34 µs per 8 kB write) against `fsync_writethrough` + 284 ops/s (3,517 µs). That is a factor of about 100. + - The share append commits while it holds `ORDER_LOCK`, so the cost of a + commit is inside the serialised section. + - [The flush control](#the-flush-control) measures the consequence: with a + forced flush the same configuration sustains about 117 shares/s, not 500. + - So a rate in this part is what this build does **when a commit costs tens + of microseconds**. It is not a prediction for a host whose commits wait + for a flush. What the rehearsal host's commits cost is the first thing to + measure there, and `pg_test_fsync` measures it in two minutes. +- **The harness memory floor does not operate on macOS.** + `--min-mem-available-mib` reads `MemAvailable` from `/proc/meminfo`. That + file does not exist here, and the harness then skips the check without + saying so (`run.rs`, the 1 s memory check). The flag was still passed, + unchanged, so the argument lists match #271's. The driver carried a guard + instead: every 10 s it summed free, inactive and speculative pages from + `vm_stat`, and it would have sent SIGTERM below 16 GiB. It never fired. That + sum is an approximation and is **not** `MemAvailable`; macOS publishes no + equivalent, so it is not comparable with the Linux figures above. +- **Frontend CPU was under-reported about 42 times, and is converted here.** + The harness read `proc_pid_rusage` CPU times as nanoseconds. On Apple + silicon they are Mach ticks of 125/3 ns. A probe that burned 0.2500 s of + CPU read 0.0060 s through the harness's arithmetic and 0.2500 s after + scaling. The harness is fixed in the change that adds this part, with a + test that fails against the old arithmetic. The runs recorded here were + made with the unfixed build, so every CPU figure in this part is the + reported value multiplied by 125/3, and the tables say so. +- **Peak RSS is a sampled maximum, not a kernel peak.** Linux's `VmHWM` has no + macOS equivalent, and the harness labels the figure accordingly. A spike + between two 1 s samples is not in it. +- **`pg_stat_statements` was not loaded,** so the lock-statement columns are + `n/a`. The harness looks for `pg_stat_statements.so` and macOS ships a + `.dylib`. +- **The side report's host block is mostly `null`** (`cpu_model`, `kernel`, + `memory_total_kib`), because the harness reads those from `/proc`. The host + table above was taken from `sysctl`, `system_profiler` and `sw_vers`. + +Apart from the CPU units, none of these is wrong output: the harness targets +Linux, and on macOS it reports what it cannot measure as unknown rather than +as zero. + +### Method (#447) + +The method is #271's, with these differences. + +- **Run ids** carry the prefix `d1-` instead of `w11-`. The argument lists of + the three blocked sizes are unchanged from + `w11-{200,400,500}k-fe1-async-r1`. +- **Cluster root.** The harness builds its cluster under `$TMPDIR`. With the + macOS default, PostgreSQL refuses to start: `Unix-domain socket path ... is + too long (maximum 103 bytes)`, and the harness exits 2. The driver + therefore set `TMPDIR=/tmp/d1t`. Nothing else in the environment was + changed. +- **Whole-run ceiling** 3,600 s. No run reached it. +- **Order and repeats.** + - Runs were strictly one at a time, with a 60 s cooldown between them. + - The three blocked sizes ran first, on 2026-09-17. They are also the first + repeat of the 1-frontend async cells at 400k and 500k. + - Everything else ran on 2026-09-18, interleaved as #271's was. At 400k one + pass runs async at 1, 2 and 4 frontends and then sync at 2, and the next + pass repeats that order; at 500k a pass is async at 1, 2 and 4. + - **Three repeats per configuration** in the rate matrix and the flush + control. 200k, the found-block run and the 20k dense attempt are one run + each; dense cadence at 400k is two runs at 1 frontend and one at 2. + - Synchronous replication was measured at 2 frontends, the topology #291 + describes. +- **D1 verdict.** D1 (#260) sets rates only, so "met" is defined from the + harness's own accounting rather than from a rate comparison; the definition + is printed above the [verdict tables](#d1-verdict). + +### D1 verdict + +D1 (#260) sets rates only: "2,000/s sustained for one minute, 500/s for five +minutes". It sets no acknowledgement fraction and no latency, so ACK p99 is +shown beside each verdict, against the limit the run's validator used, and is +**not** part of it. The two phases are judged separately. + +**Definition.** A repeat **meets** a phase when all three hold for that phase: +`shortfall == 0` (the open-loop scheduler placed every token of the target +rate; a token is a shortfall when no session could take it because each was +still waiting for its previous answer, run.rs:2059-2068); +`rejected_valid_shares == 0` (the server refused no share the harness holds +valid, that is, no rejection outside the races the server is entitled to lose, +run.rs:2534-2548); and no submit of the phase went unanswered +(`rejections.no_response_by_phase`). A configuration is **met** only when +EVERY repeat in its medians meets it, and the count says how many did. A null +in any of the three gives no verdict, never a pass. + +**Why not `achieved >= target`.** The scheduler generates `floor(elapsed * +rate)` tokens and stops at the deadline, so it places 149,999 tokens in a 300 +s phase at 500/s: a phase that acknowledged every one reads 499.997 shares/s. +Achieved also excludes the entitled races, which are listed in their own +column. Rates that would round to their target are printed to three decimals +so none reads as the target. + +#### The 500 shares/s phase (`steady_state`, 300 s) + +| window | fe | repl | plan | n | target /s | offered /s | achieved /s | shortfall tokens | rejected valid shares, per run | unanswered submits, per run | entitled-race rejections, per run | verdict | ACK p99 ms | ACK p99 within the validator limit | ORDER_LOCK max / mean waiters | same-configuration runs outside the medians | +|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---| +| 200k | 1 | async | d1 | 1 | 500 | 499.997 | 499.997 | 0 | 0 | 0 | 0 | **met** (1 of 1) | 20.5 | 1 of 1 (limit 1,000 ms) | 15 / 0.07 | – | +| 400k | 1 | async | d1 | 3 | 500 | 499.997 (499.995–499.997) | 499.4 (499.3–499.4) | 0 (0–0) | 0, 1, 0 | 0, 0, 0 | 193, 199, 171 | **not met** (2 of 3): valid shares refused in 1 of 3 repeats, at most 1 per run | 18.5 (8.6–25.1) | 3 of 3 (limit 1,000 ms) | 15 (15–15) / 0.08 (0.07–0.09) | – | +| 400k | 2 | async | d1 | 3 | 500 | 499.995 (499.995–499.997) | 499.3 (499.3–499.3) | 0 (0–0) | 1, 1, 1 | 0, 0, 0 | 201, 205, 203 | **not met** (0 of 3): valid shares refused in 3 of 3 repeats, at most 1 per run | 23.2 (11.4–37.5) | 3 of 3 (limit 1,000 ms) | 31 (31–31) / 0.16 (0.12–0.24) | – | +| 400k | 4 | async | d1 | 3 | 500 | 499.995 (499.995–499.997) | 499.995 (499.995–499.997) | 0 (0–0) | 0, 0, 0 | 0, 0, 0 | 0, 0, 0 | **met** (3 of 3) | 15 (13.2–16.1) | 3 of 3 (limit 1,000 ms) | 63 (62–63) / 0.22 (0.21–0.23) | – | +| 400k | 2 | sync | d1 | 3 | 500 | 499.995 (499.995–499.997) | 499.995 (499.995–499.997) | 0 (0–0) | 0, 0, 0 | 0, 0, 0 | 0, 0, 0 | **met** (3 of 3) | 33 (9–2,560) | 2 of 3 (limit 1,000 ms) | 31 (30–31) / 0.21 (0.15–1.39) | – | +| 400k | 2 | async | d1 +3 blocks | 1 | 500 | 499.995 | 482.5 | 0 | 12 | 0 | 5251 | **not met** (0 of 1): valid shares refused in 1 of 1 repeats, at most 12 per run | 51.1 | 1 of 1 (limit 1,000 ms) | 31 / 0.17 | – | +| 400k | 1 | async | d1 +dense(15) | 2 | 500 | 498.828 (497.66–499.997) | 498.2 (497–499.3) | 351 (0–701) | 9, 2 | 0, 0 | 187, 201 | **not met** (0 of 2): shortfall in 1 of 2 repeats, at most 701 per run; valid shares refused in 2 of 2 repeats, at most 9 per run | 1,963 (22–3,903) | 1 of 2 (limit 1,000 ms) | 15 (15–15) / 0.64 (0.1–1.17) | – | +| 400k | 2 | async | d1 +dense(15) | 1 | 500 | 499.995 | 499.3 | 0 | 1 | 0 | 196 | **not met** (0 of 1): valid shares refused in 1 of 1 repeats, at most 1 per run | 19.8 | 1 of 1 (limit 1,000 ms) | 31 / 0.17 | – | +| 400k | 1 | async | d1 +wal_sync_method=fsync_writethrough | 1 | 500 | 250.1 | 117.3 | 74,981 | 38170 | 0 | 1671 | **not met** (0 of 1): shortfall in 1 of 1 repeats, at most 74,981 per run; valid shares refused in 1 of 1 repeats, at most 38,170 per run | 38,296 | 0 of 1 (limit 1,000 ms) | 15 / 14.11 | d1-400k-fe1-async-wt-r1 (exit 5), d1-400k-fe1-async-wt-r3 (exit 5) | +| 500k | 1 | async | d1 | 3 | 500 | 499.997 (499.995–499.997) | 499.4 (499.2–499.4) | 0 (0–0) | 0, 1, 1 | 0, 0, 0 | 193, 189, 247 | **not met** (1 of 3): valid shares refused in 2 of 3 repeats, at most 1 per run | 13.3 (13.2–26.7) | 3 of 3 (limit 1,000 ms) | 15 (15–15) / 0.08 (0.07–0.09) | – | +| 500k | 2 | async | d1 | 3 | 500 | 499.997 (499.995–499.997) | 499.3 (499.3–499.3) | 0 (0–0) | 2, 3, 3 | 0, 0, 0 | 198, 212, 203 | **not met** (0 of 3): valid shares refused in 3 of 3 repeats, at most 3 per run | 17.4 (8.1–28.1) | 3 of 3 (limit 1,000 ms) | 31 (31–31) / 0.15 (0.09–0.19) | – | +| 500k | 4 | async | d1 | 3 | 500 | 499.997 (499.995–499.997) | 499.997 (499.995–499.997) | 0 (0–0) | 0, 0, 0 | 0, 0, 0 | 0, 0, 0 | **met** (3 of 3) | 13.3 (13–22.5) | 3 of 3 (limit 1,000 ms) | 63 (63–63) / 0.18 (0.16–0.26) | – | +| 20k | 1 | async | d1 | 0 | n/a | n/a | n/a | n/a | n/a | n/a | n/a | no verdict: no run in the medians | n/a | n/a | n/a | d1-dense20k-fe1-async-r1 (exit 6) | + +#### The 2,000 shares/s phase (`burst`, 60 s) + +| window | fe | repl | plan | n | target /s | offered /s | achieved /s | shortfall tokens | rejected valid shares, per run | unanswered submits, per run | entitled-race rejections, per run | verdict | ACK p99 ms | ACK p99 within the validator limit | ORDER_LOCK max / mean waiters | same-configuration runs outside the medians | +|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---| +| 200k | 1 | async | d1 | 1 | 2000 | 1,558.5 | 1,558.5 | 26,490 | 0 | 0 | 0 | **not met** (0 of 1): shortfall in 1 of 1 repeats, at most 26,490 per run | 1,886 | 0 of 1 (limit 1,000 ms) | 15 / 14.41 | – | +| 400k | 1 | async | d1 | 3 | 2000 | 1,623 (1,503.1–1,755.6) | 1,623 (1,503.1–1,755.6) | 22,622 (14,663–29,816) | 0, 0, 0 | 0, 0, 0 | 0, 0, 0 | **not met** (0 of 3): shortfall in 3 of 3 repeats, at most 29,816 per run | 4,026 (3,357–4,397) | 0 of 3 (limit 1,000 ms) | 15 (15–15) / 14.36 (14.36–14.38) | – | +| 400k | 2 | async | d1 | 3 | 2000 | 1,510.2 (1,468.2–1,521.7) | 1,490.2 (1,445.6–1,521.7) | 29,384 (28,698–31,908) | 1203, 1356, 0 | 0, 0, 0 | 0, 0, 0 | **not met** (0 of 3): shortfall in 3 of 3 repeats, at most 31,908 per run; valid shares refused in 2 of 3 repeats, at most 1,356 per run | 2,608 (2,464–3,099) | 0 of 3 (limit 1,000 ms) | 31 (31–31) / 30.33 (30.31–30.35) | – | +| 400k | 4 | async | d1 | 3 | 2000 | 1,504.2 (1,423–1,662.7) | 1,504.2 (1,423–1,662.7) | 29,746 (20,237–34,621) | 0, 0, 0 | 0, 0, 0 | 0, 0, 0 | **not met** (0 of 3): shortfall in 3 of 3 repeats, at most 34,621 per run | 1,882 (1,634–1,972) | 0 of 3 (limit 1,000 ms) | 63 (63–63) / 62.38 (62.32–62.39) | – | +| 400k | 2 | sync | d1 | 3 | 2000 | 1,504.7 (1,402.8–1,616.5) | 1,504.7 (1,402.8–1,616.5) | 29,714 (23,008–35,830) | 0, 0, 0 | 0, 0, 0 | 0, 0, 0 | **not met** (0 of 3): shortfall in 3 of 3 repeats, at most 35,830 per run | 1,812 (1,667–1,849) | 0 of 3 (limit 1,000 ms) | 31 (31–31) / 30.47 (30.44–30.47) | – | +| 400k | 2 | async | d1 +3 blocks | 1 | 2000 | 1,600 | 1,600 | 23,998 | 0 | 0 | 0 | **not met** (0 of 1): shortfall in 1 of 1 repeats, at most 23,998 per run | 1,688 | 0 of 1 (limit 1,000 ms) | 31 / 30.43 | – | +| 400k | 1 | async | d1 +dense(15) | 2 | 2000 | 1,491.5 (1,455.9–1,527) | 1,490.8 (1,454.6–1,527) | 30,510 (28,377–32,643) | 83, 0 | 0, 0 | 0, 0 | **not met** (0 of 2): shortfall in 2 of 2 repeats, at most 32,643 per run; valid shares refused in 1 of 2 repeats, at most 83 per run | 4,786 (4,378–5,194) | 0 of 2 (limit 1,000 ms) | 15 (15–15) / 14.35 (14.34–14.36) | – | +| 400k | 2 | async | d1 +dense(15) | 1 | 2000 | 1,492.7 | 1,466.6 | 30,439 | 1561 | 0 | 0 | **not met** (0 of 1): shortfall in 1 of 1 repeats, at most 30,439 per run; valid shares refused in 1 of 1 repeats, at most 1,561 per run | 2,255 | 0 of 1 (limit 1,000 ms) | 31 / 30.32 | – | +| 400k | 1 | async | d1 +wal_sync_method=fsync_writethrough | 1 | 2000 | 762.2 | 107.2 | 74,266 | 39297 | 0 | 2 | **not met** (0 of 1): shortfall in 1 of 1 repeats, at most 74,266 per run; valid shares refused in 1 of 1 repeats, at most 39,297 per run | 39,017 | 0 of 1 (limit 1,000 ms) | 15 / 13.68 | d1-400k-fe1-async-wt-r1 (exit 5), d1-400k-fe1-async-wt-r3 (exit 5) | +| 500k | 1 | async | d1 | 3 | 2000 | 1,550 (1,481.4–1,561.4) | 1,550 (1,481.4–1,561.4) | 26,996 (26,313–31,114) | 0, 0, 0 | 0, 0, 0 | 0, 0, 0 | **not met** (0 of 3): shortfall in 3 of 3 repeats, at most 31,114 per run | 4,744 (3,828–4,870) | 0 of 3 (limit 1,000 ms) | 15 (15–15) / 14.38 (14.37–14.38) | – | +| 500k | 2 | async | d1 | 3 | 2000 | 1,521.7 (1,521.6–1,547.8) | 1,504.3 (1,496.5–1,522.9) | 28,695 (27,130–28,700) | 1516, 1041, 1496 | 0, 0, 0 | 0, 0, 0 | **not met** (0 of 3): shortfall in 3 of 3 repeats, at most 28,700 per run; valid shares refused in 3 of 3 repeats, at most 1,516 per run | 2,289 (2,179–3,083) | 0 of 3 (limit 1,000 ms) | 31 (31–31) / 30.33 (30.32–30.33) | – | +| 500k | 4 | async | d1 | 3 | 2000 | 1,468 (1,401.4–1,514.5) | 1,468 (1,401.4–1,514.5) | 31,921 (29,131–35,917) | 0, 0, 0 | 0, 0, 0 | 0, 0, 0 | **not met** (0 of 3): shortfall in 3 of 3 repeats, at most 35,917 per run | 1,879 (1,854–1,978) | 0 of 3 (limit 1,000 ms) | 63 (63–63) / 62.35 (62.34–62.39) | – | +| 20k | 1 | async | d1 | 0 | n/a | n/a | n/a | n/a | n/a | n/a | n/a | no verdict: no run in the medians | n/a | n/a | n/a | d1-dense20k-fe1-async-r1 (exit 6) | + +**Which rows are the rate matrix.** The eight rows whose plan column reads +`d1` alone, 22 repeats in all, are the rate matrix, and the reading below is +about them. The tables also hold the other D1-plan runs, each marked in its +plan column and discussed in its own section: the found-block run +(`+3 blocks`), dense cadence (`+dense(15)`) and the flush control +(`+wal_sync_method=fsync_writethrough`). Landing blocks and forcing a flush +both cost the rate phases something, and their rows say how much. The `20k` +row with no run in its medians is the dense attempt that aborted. + +**Reading the 500 shares/s table.** + +- **The rate was sustained in every repeat of every ordinary configuration.** + `shortfall` is 0 in all 22 repeats: every token of the 500 shares/s target + was placed, at 200k, 400k and 500k, at 1, 2 and 4 frontends, async and + sync. No submit went unanswered in the phase. ACK p99 was 8–38 ms in 21 of + the 22; one synchronous repeat reached 2,560 ms and is the only one above + the 1,000 ms limit. +- **The configurations that read "not met" miss it on one to three shares.** + The strict rule needs every valid share acknowledged. Nine repeats, all at + 1 or 2 frontends async, refused between one and three shares of about + 150,000, 14 shares in all. Every one is `ledger-confirmation-failed`: 12 + "share was not + committed because its commit gate closed" and 2 "share was not confirmed by + the database". + - **They come from one event per run,** the arrival of work for the last + external tip. The fake node mints tips 6, 12 and 18 s into the 30 s + `warm_up`. When work for the last one reaches the sessions, the jobs they + hold go stale at once: 171–247 `stale-job` rejections, which are races + the server is entitled to lose, and at most three shares caught at the + commit gate as the epoch changes. + - **Where that event lands decides the verdict.** At 4 frontends, and at 2 + frontends under synchronous replication, all 2,000 sessions had work for + the last tip 3.3–7.8 s after it was minted, inside `warm_up`, so + `steady_state` sees neither the stale jobs nor the refused share and + reads "met". At 1 and 2 frontends async it took 19.9–36.7 s, which is 8 + to 25 s into `steady_state`. The rows differ in when a tip's work + arrived, not in the rate they sustained. + - **Why the arrival time splits in two was not investigated,** and it is a + result of its own; see + [Time to usable work after a tip](#time-to-usable-work-after-a-tip). + - **The rule was not relaxed to absorb this.** It was fixed before these + runs, and a refusal count matters in general: a refusal returns in half a + millisecond, so a run that refused a quarter of its shares, as #271's + did, can still show no shortfall. Instead each "not met" carries its + reason and its size. + - **What it leaves open** is whether a share refused at the commit gate + during a tip change is a race the server is entitled to lose, as + `stale-job` is, or a refusal of valid work. The harness classes it as the + second, by its `reason_id`. That is a question for the owners of #429's + gate and of the harness's classifier. + +**Reading the 2,000 shares/s table.** + +- **No configuration meets it.** The median `burst` rate is between 1,468 and + 1,623 shares/s. Between a fifth and a quarter of the target's tokens could + not be placed, and the median ACK p99 is 1.8–4.7 s. +- **Adding frontends adds waiters, not throughput.** At 400k `ORDER_LOCK` + averaged 14.4, 30.3 and 62.4 waiters at 1, 2 and 4 frontends, which is each + topology's database connections minus the holder. The rate did not rise + with them: 1,623, 1,490 and 1,504 shares/s. This is #271's finding, now at + the production window and at more than five times its rate. +- **The frontends are idle while it happens.** All frontends together used at + most 0.48 cores in `burst`, in any run. +- **The window size does not move the ceiling.** 200k, 400k and 500k at 1 + frontend gave 1,558, 1,623 and 1,550 shares/s. +- **At 2 frontends the saturated phase also refuses work.** In five of the + six async repeats at 400k and 500k, `burst` refused 1,041–1,516 shares with + `backend-rpc-unavailable`, "current chain state is unavailable", 1.1–1.7 % + of what it offered. It is the class that made up 23–46 % of offers in + #271's runs. Here it appears in no other D1 rate phase of any ordinary run. + +### Time to usable work after a tip + +The fake node mints three external tips during `warm_up`, 6 s apart, at +heights 101, 102 and 103. Tips 101 and 102 are each replaced after 6 s; tip +103 is never replaced. A tip counts as served in a run only when all 2,000 +sessions had work for it while it was still the tip. Each cell gives the +number of repeats that served the tip and, for those, the time until the last +session had work: median, with minimum and maximum. + +| window | fe | repl | n | tip 101 (6 s life): repeats that served it | tip 102 (6 s life): repeats that served it | tip 103 (never replaced): repeats that served it | +|---|---|---|---|---|---|---| +| 200k | 1 | async | 1 | 1 of 1: 5.4 s | 1 of 1: 5.5 s | 1 of 1: 5.6 s | +| 400k | 1 | async | 3 | 0 of 3 | 0 of 3 | 3 of 3: 21.6 (19.9–22.0) s | +| 400k | 2 | async | 3 | 0 of 3 | 0 of 3 | 3 of 3: 35.1 (33.7–36.7) s | +| 400k | 4 | async | 3 | 0 of 3 | 2 of 3: 4.2 (4.0–4.4) s | 3 of 3: 3.8 (3.7–7.8) s | +| 400k | 2 | sync | 3 | 3 of 3: 3.4 (3.3–3.4) s | 3 of 3: 3.3 (3.3–3.4) s | 3 of 3: 3.3 (3.3–3.3) s | +| 500k | 1 | async | 3 | 0 of 3 | 0 of 3 | 3 of 3: 23.8 (22.8–25.9) s | +| 500k | 2 | async | 3 | 0 of 3 | 0 of 3 | 3 of 3: 27.5 (26.7–27.8) s | +| 500k | 4 | async | 3 | 2 of 3: 4.9 (4.7–5.1) s | 3 of 3: 4.4 (4.4–4.9) s | 3 of 3: 4.5 (4.4–5.0) s | + +- **The time splits in two, and the split follows the configuration, not the + repeat.** At 4 frontends, and at 2 frontends under synchronous replication, + work reaches every session 3.3–7.8 s after a tip. At 1 and 2 frontends + async the last tip takes 19.9–36.7 s, and tips 101 and 102 are replaced + before any session has work for them, in every repeat. +- **It is not drift.** The configurations were interleaved within each pass, + and each one kept its own value across its three repeats, which were made + 30 to 40 minutes apart and in two cases a day apart. +- **It is not the window alone.** The same 1-frontend async configuration + that takes about 22 s here served every own-block landing of the + [dense-cadence phase](#dense-cadence-at-d1) in 3.0–3.7 s, at the same + window and rate. The slow case is specific to external tips arriving in + the first seconds of load. +- **The cause was not investigated.** The frontend log line `template refresh + deferred error=tip observation superseded` appears only in runs where a tip + was replaced before it was served. (One 500k 4-frontend repeat left a tip + unserved without logging it.) That restates the symptom and does not + explain it. In particular nothing + here explains why synchronous replication, with the same 2 frontends and + the same offered rate, serves every tip in 3.3 s while its async pair + takes about ten times as long. #271 saw the slow case only: 35–71 s at + 20k, and no tip served within its 6 s life. +- **Why it matters here.** It decides which phase the tip-change rejections + fall in, and so which rows of the [D1 verdict](#d1-verdict) read "met". + +### The blocked sizes now run + +All three sizes that #271 recorded as refused now run. The argument lists are +unchanged from `w11-{200,400,500}k-fe1-async-r1`: 1 frontend, async, D1, +2,000 sessions. + +| Size | Run id | Exit | JSONB refusals in the frontend log | steady achieved /s · ACK p99 ms | burst achieved /s · ACK p99 ms | frontend RSS max MiB (sampled) | lowest reclaimable-approx MiB | +|---|---|---|---|---|---|---|---| +| 200k | d1-200k-fe1-async-r1 | 0 | 0 | 499.997 · 20.5 | 1,558 · 1,886 | 650 | 62,338 | +| 400k | d1-400k-fe1-async-r1 | 0 | 0 | 499.4 · 18.5 | 1,623 · 4,026 | 928 | 62,537 | +| 500k | d1-500k-fe1-async-r1 | 0 | 0 | 499.4 · 13.2 | 1,561 · 4,744 | 1,075 | 62,600 | + +- **Exit 0 at every size, so #273 holds.** Each frontend became ready and + served work, no frontend log holds the JSONB refusal, and + `blocked.blocked` is `false` in all three side reports. On #271's base all + three exited 3 at startup. +- **Every run reconciled exactly:** no ACK/commit divergence, no + unknown-outcome commit, no durability finding, and no committed share whose + acknowledgement was lost. (At 200k, 17 `slow_database` submits were still + unanswered when the stop drain expired; PostgreSQL holds none of them.) +- **Memory is no longer the constraint it was.** One frontend holds about + 1 GiB at a 500k window. On #271's base one frontend held 2.7 GiB at 100k, + and seeding 500k left that host 431 MiB above its floor. The two hosts and + builds differ, so this shows that the window fits here, not by how much + #273 reduced it. +- **These three runs are also the first repeat of the 1-frontend cells** at + 400k and 500k in the tables below. + +### The flush control + +This control is beyond the steps #447 lists. It exists because the host's +commits do not force the drive cache, and the verdict above cannot be read +without knowing what that is worth. + +- **What differs.** The control is the 1-frontend async 400k configuration, + argument list unchanged. The one difference is a line in the cluster's + `postgresql.conf`: `wal_sync_method = fsync_writethrough`. It was added by + pointing `--pg-bin-dir` at a directory of links to the real PostgreSQL + binaries in which `initdb` is wrapped to append that line. The standby is + made by `pg_basebackup`, which copies the primary's configuration with its + data directory. The harness was not changed. +- **It reached the runtime.** Each run's `database-profile.json` records + `wal_sync_method: fsync_writethrough`, and during the first run the live + primary reported the same value with source `configuration file`. +- **Same host, same build, same day** as its ordinary pair, + `d1-400k-fe1-async-r{1,2,3}`. + +| run id | exit | steady achieved /s (target 500) | steady shortfall tokens | steady ACK p50 / p99 ms | burst achieved /s (target 2000) | reconn achieved /s (target 500) | steady ORDER_LOCK mean waiters | committed shares whose answer was lost | divergences / unknown-outcome / durability findings | +|---|---|---|---|---|---|---|---|---|---| +| d1-400k-fe1-async-wt-r1 | 5 | 112.6 | 72,359 | 10,194 / 40,408 | 114.2 | 134.9 | 14.07 | 1 | 0 / 0 / 0 | +| d1-400k-fe1-async-wt-r2 | 0 | 117.3 | 74,981 | 10,323 / 38,296 | 107.2 | 122.5 | 14.11 | 0 | 0 / 0 / 0 | +| d1-400k-fe1-async-wt-r3 | 5 | 117.9 | 75,649 | 10,338 / 37,519 | 111.0 | 133.2 | 14.11 | 2 | 0 / 0 / 0 | + +- **With a forced flush, the 500 shares/s phase is missed by a factor of + about four.** `steady_state` acknowledged 112.6, 117.3 and 117.9 + shares/s, against a median of 499.4 for its pair. About half of the + target's tokens could not be placed at all, and accepted shares waited a + median of about 10 s for their ACK. +- **The 2,000 shares/s phase achieves no more**, because the ceiling is the + same serialised commit: `ORDER_LOCK` averaged 14 waiters of a possible 15 + in `steady_state` already, where the pair averaged 0.08. +- **So the ordinary runs' 500 shares/s result is a property of cheap commits, + not only of this build.** `pg_test_fsync` puts a forced flush on this disk + at + 3.5 ms against 34 µs without one. A commit that waits for one sits inside + `ORDER_LOCK`, and nothing a frontend does runs in parallel with it. +- **What this does and does not say about other hardware.** + - It is not a prediction for the rehearsal host. An Apple laptop SSD takes + milliseconds to honour `F_FULLFSYNC`; a server device with power-loss + protection can take tens of microseconds, and a network volume can take + more than either. + - It does say which number decides the outcome there: what one WAL flush + costs. `pg_test_fsync` on the rehearsal host's WAL volume gives it in two + minutes, before any harness run. + - #271's host, a Linux VM on a virtual disk, sustained 238–304 shares/s. + That sits between this host's two figures, which is where a commit cost + between the two would put it. The builds differ as well, so this is + consistent with the explanation and does not prove it. +- **Two of the three runs exit 5, and nothing was lost in either.** In + `reconnect`, a client-initiated reconnect waits up to 25 s for its + outstanding submit before it closes the socket. The `reconnect` ACK p99 was + 56, 37 and 33 s in the three runs, all above that limit, so the wait + expired for one submit in the first run and two in the third, and + PostgreSQL holds all three shares. The harness classes each as + transport-indeterminate: the share is present and only its acknowledgement + is missing. None of the three runs has an ACK/commit divergence, an + unknown-outcome commit or a durability finding. Exit-5 runs are kept out of + the medians, so the control's row in the tables is the second run alone; + the table above gives all three. + +### The build control + +This control is also beyond the steps #447 lists. #271 measured 238–304 +shares/s and this part measures 500, and between the two both the host and +the build changed. #271 declined to attribute its own base-to-base +differences for the same reason. This control changes one of the two: it runs +#271's 20k argument list, `w11-20k-fe1-async`, unchanged, on this host, on +both builds. + +- **Both builds were built the same way:** release profile, Rust 1.98.1, each + in its own clean detached worktree, `5d0042f6` being the base of the #271 + matrix above. +- **The runs alternate:** current build, then #271's, three times, on the + same afternoon and on AC power. +- **"Build" means the harness and the server together.** Two cross runs were + attempted to separate them, and neither could run; see below. + +This is the one table in this part that holds more than one build, as #271's +[comparison with its earlier +bases](#comparison-with-the-earlier-harness-bases) +does, and every row says which binaries it ran. + +| run id | binaries | exit | steady achieved /s (target 500) | steady shortfall tokens | steady ACK p50 / p99 ms | burst achieved /s (target 2000) | burst shortfall tokens | burst ACK p50 / p99 ms | burst ORDER_LOCK max / mean waiters | last external tip: all 2,000 sessions with work, s | +|---|---|---|---|---|---|---|---|---|---|---| +| d1-20k-fe1-async-r1 | `a1937054` harness and server (this matrix) | 0 | 499.997 | 0 | 1.6 / 28 | 1,558.5 | 26,487 | 1,261 / 3,871 | 15 / 14.35 | 0.7 | +| d1-20k-fe1-async-r2 | `a1937054` harness and server (this matrix) | 0 | 499.997 | 0 | 1.6 / 17 | 1,649.3 | 21,042 | 1,216 / 3,339 | 15 / 14.34 | 0.7 | +| d1-20k-fe1-async-r3 | `a1937054` harness and server (this matrix) | 0 | 499.997 | 0 | 1.1 / 32 | 1,621.2 | 22,729 | 1,237 / 1,861 | 15 / 14.41 | 0.7 | +| d1-20k-fe1-async-build5d0042f6-r1 | `5d0042f6` harness and server (#271's build) | 0 | 499.997 | 0 | 1.4 / 431 | 1,978.9 | 1,266 | 154 / 1,652 | 15 / 9.58 | 1.3 | +| d1-20k-fe1-async-build5d0042f6-r2 | `5d0042f6` harness and server (#271's build) | 8 | 499.997 | 0 | 1.4 / 418 | 1,994.2 | 348 | 1 / 987 | 15 / 5.04 | 1.3 | +| d1-20k-fe1-async-build5d0042f6-r3 | `5d0042f6` harness and server (#271's build) | 0 | 499.997 | 0 | 1.5 / 446 | 1,992.5 | 448 | 80 / 1,167 | 15 / 10.28 | 1.4 | + +- **#271's 238–304 shares/s was its host.** Its own build sustains 500 + shares/s here in all three runs, with no shortfall and no refused share. At + 20k the two builds do not differ at all on the 500 shares/s phase's rate. +- **The current build's 2,000 shares/s ceiling is about a fifth lower than + #271's build's, on the same host.** + - #271's build acknowledged 1,979–1,994 shares/s in `burst` and placed + 98.9–99.7 % of the target, with `ORDER_LOCK` averaging 5–10 waiters of a + possible 15: the lock was not saturated. + - The current build acknowledged 1,559–1,649 with the lock saturated at + 14.3–14.4. + - The two ranges are about 330 shares/s apart and neither set's spread + exceeds 100, so three interleaved repeats are enough to call them + different. + - By the strict rule #271's build does not meet the phase either: 348–1,266 + of its 120,000 tokens went unplaced. +- **The current build is more than ten times better on latency at 500 + shares/s:** `steady_state` ACK p99 of 17–32 ms against 418–446 ms. +- **It also serves a new tip sooner at this window:** 0.7–0.9 s for every + session against 1.3–1.5 s. #271's host took 35–71 s on the same build, so + that + figure, too, was its host. +- **Whether the server or the harness moved could not be measured.** Both + cross runs were attempted and both were refused at startup within three + seconds, each correctly: + - the `a1937054` harness driving the `5d0042f6` server: the harness had + migrated the database to its own schema, and the older server refused it, + `database declares capability candidate_offer_lifecycle = 1, which this + server does not understand: a newer PRISM release wrote this database`; + - the `5d0042f6` harness driving the `a1937054` server: `migration 011 + requires every earlier instance to report drained or stopped; offending + instances: load-seed`. + + So "build" above stays the harness and the server together. The harness + drives the load, and a client that offered more slowly would lower the + ceiling without any change in the server; this control cannot rule that + out. +- **What this does not show.** It is one window, 20k, at one frontend. The + production windows were not run on #271's build, which could not run them. + The cause of the lower ceiling was not looked for; 50 commits separate + the two builds. Whether a fifth of the peak rate is a fair price + for the latency is a judgement for the people deciding D1, and this + section only puts the two numbers side by side. +- **One of #271's build's runs exited 8** + (`d1-20k-fe1-async-build5d0042f6-r2`). + Its `slow_database` phase acknowledged no share at all in its minute, and + that harness then withholds the artifact because the phase has no ACK + latency to state. The phase serves about 3 shares/s on either build, so an + empty minute is within its reach. The run's `steady_state` and `burst` + completed in the ordinary way and are in the table. + +### Found blocks at 400k + +`d1-400k-fe2-async-blocks3-r1` is the ordinary 2-frontend async 400k +configuration with `--scheduled-blocks 3`, so three own blocks are found and +submitted during `steady_state`. It ran with `--keep-artifacts`, so the run's +own database could be inspected afterwards. + +- **All three blocks landed.** The phase reports `scheduled_blocks: 3`. The + node accepted all three submissions, at heights 104, 105 and 106, and + recorded three pool tip changes. The run exited 0, `blocked.blocked` is + `false`, and it reconciled exactly. +- **The refusal #271 expected did not happen.** #271 predicted that once #273 + landed, this size would get past startup and reach #265's refusal of the + audit bundle at landing. It reached the landing path three times and no + refusal was logged. This is the first time #265's candidate path and #267's + landing have been exercised under load at the production window. + - The frontend logs hold one candidate-related line: `candidate polling + failed error=pool timed out while waiting for an open connection`, once, + on `load-fe-1`. It falls in the last seconds of `slow_database`, almost + three minutes after the phase with the landings ended, so it is the + delayed database starving the pool and not a landing that failed. +- **No unfinished candidate was left behind.** After the run, the kept primary + was restarted and `qbit-prism-server candidates list --limit 10000` (#268) + was run against it. It printed `no unfinished candidates`; the JSON form + holds 0 rows and `truncated: false`. +- **The database agrees, so the empty list is not an empty database.** The + run's `qbit_block_candidate_outbox` holds exactly three rows. All three + are `state = submitted` with `offer_outcome = accepted`, one attempt each + and no `last_error`, and each completed 5.8–6.4 s after it was created, + inside `steady_state`. +- **What three landings cost.** Each landing bumps the payout revision, and + every session's job goes stale at once. + - `steady_state` acknowledged 482.5 shares/s against about 499.3 in the + same configuration without landings. The scheduler still placed the whole + target (`shortfall` 0), so the difference is rejected shares, not offers + that could not be made. + - 5,251 of those rejections are races the server is entitled to lose: 4,901 + `stale-job` and 350 `unknown-job`. + - 12 are refusals of shares the harness holds valid, all + `ledger-confirmation-failed`: 6 "share was not committed because its + commit gate closed" and 6 "share was not confirmed by the database". + PostgreSQL holds none of the 12, so none is a divergence. + - ACK p99 rose to 51 ms, from a median of 23 ms. + +One run cannot show spread. It shows that the path works at this window and +what a landing costs in rejected work; the dense-cadence phase is the +measurement of that cost per landing. + +Per run: own blocks the scheduler asked for (`phases[].scheduled_blocks`), +what the fake node recorded for `submitblock` (`node.submissions`), the pool +tips that resulted (`node.tip_changes` with origin `pool`), client failures of +kind `scheduled-block`, and frontend log refusals (`blocked.log_matches`). +Per-run values; nothing here is a median. + +| run id | status | blocks scheduled (by phase) | node submitblock: calls / accepted / rejected | node rejection reasons | pool tip changes | client scheduled-block failures | frontend log refusals (kind × lines) | +|---|---|---|---|---|---|---|---| +| d1-400k-fe2-async-blocks3-r1 | ok | steady: 3 | 3 / 3 / 0 | – | 3 | 0 | job_deferred × 3000; refresh_deferred × 4 | + +### Dense cadence at D1 + +#271 dropped this block after one attempt showed the phase could not start +at D1 on its host. #447 asked for it to be attempted again at D1, and measured +at a reduced rate if it still could not start. + +**It started at D1 at the production window, three times out of three.** + +| run id | fe | landed | payout-revision bumps | achieved / offered /s (target 500) | shortfall tokens | entitled-race rejections | refused valid shares | rejections per landing | all 2,000 sessions with new-tip work, s after a landing: median (min–max) | ACK p50 / p99 ms | rejections the harness attributes to a landing | +|---|---|---|---|---|---|---|---|---|---|---|---| +| d1-dense400k-fe1-async-r1 | 1 | 14 of 15 | 28 | 415.2 / 500.0 | 0 | 19,262 | 1,101 | 1,454 | 3.12 (2.98–3.71) | 1.1 / 381 | 0 | +| d1-dense400k-fe1-async-r2 | 1 | 15 of 15 | 30 | 409.7 / 500.0 | 0 | 20,596 | 1,079 | 1,445 | 3.13 (3.02–3.47) | 1.1 / 392 | 0 | +| d1-dense400k-fe2-async-r1 | 2 | 15 of 15 | 30 | 399.4 / 500.0 | 0 | 23,016 | 1,139 | 1,610 | 3.19 (2.96–3.51) | 1.2 / 353 | 0 | + +All three are D1 runs with `--cadence dense --scheduled-blocks 15`, async, +2,000 sessions, 400k window: the phase is 240 s at 500 shares/s, with own +blocks landing 9, 19, 9, 18 and 20 s apart. Nothing here is a median across +runs. + +- **A landing costs about three seconds of work.** After every one of the + 44 landings, all 2,000 sessions had work for the new tip 3.0–3.7 s later, + including across the 9 s gaps, and no landing's tip was replaced before it + was served. At 500 shares/s that is about 1,550 shares mined on a job the + landing made stale, and the phase rejected 1,445–1,610 per landing. +- **So the pool acknowledges 399–415 of every 500 shares/s offered while + blocks land at this cadence**, with the whole target placed (`shortfall` + 0). About 95 % of the rejections are races the server is entitled to lose, + `stale-job` and `unknown-job`. +- **About 1,100 per run are refusals of valid work.** Nearly all are + `backend-rpc-unavailable`, "current chain state is unavailable"; 27–44 per + run are `ledger-confirmation-failed`. PostgreSQL holds none of them. +- **One scheduled block of 45 never reached the node.** In the first run the + block found 9 s after the previous landing was answered `unknown-job`, + "stale job", 14 ms after it was sent: the job it was mined on had already + been retired. The harness counts it as a rejected landing, correctly. What + it means for a real block found seconds after another is a question for + the owners of the found-block path; this document only records that it + happened once in 45. +- **The first of the three ran partly on battery** (see [Host](#host-447)). + Its repeat on AC power gives the same figures. + +**The harness's own dense-cadence figures are empty on this build, and that +is a harness gap, not a result.** The section attributes to a landing only +rejections whose message is `new payout work is pending` or `new tip work is +pending`. At this base the server no longer sends the first at all, and the +second did not occur. So the section below reports 0 rebuild-pending +rejections, no window, and a `null` budget for #291, in runs that rejected +more than 20,000 shares around their landings. The table above is taken from +the same reports' phase entry, rejection tally and time-to-new-tip-work +block, which do not depend on that attribution. Until the harness attributes +`stale-job` and `unknown-job` rejections to landings, its proposed #291 +budget should not be read as "no rejected work". + +**At 20k the phase still cannot start, by a small margin.** +`d1-dense20k-fe1-async-r1` repeats #271's attempt with the argument list +unchanged, and exited 6 with the same reason: + +```text +20 submit(s) offered before the dense_cadence phase were still outstanding 25.0 s later, so its 0 ms delay could not be applied without them finishing under it +``` + +#271's host left 1,201 outstanding. Every D1 phase before the abort +completed: `steady_state` placed its whole target and refused nothing, and +`burst` reached 1,643 shares/s. The condition is +the same one: `slow_database` is offered 500 shares/s and serves about 3, so +some submits are still waiting when the 25 s boundary limit expires. On this +host the three 400k attempts started and the 20k attempt missed by 20 +submits. The margin is thin, and the phase starting should not be relied on. + +Per run; nothing here is a median, because a percentile of per-landing windows +cannot be pooled across runs from the summaries alone. Windows are +milliseconds on the harness's monotonic clock; the per-landing distributions +are counts of shares, one sample per landing and frontend +(cadence.rs:678-696). `landings` counts landings that were given a window +(`dense_cadence.landings`), `landed` those the node accepted +(`landing_outcomes.landed`). + +| run id | status | ran | landing budget / attempts / landings with a window | outcomes: landed / rejected / never produced / accepted without node submission / no response | schedule slots over budget | bumps: observed / attributed / unattributed | rebuild-pending rejections: in phase / attributed / unattributed | revision sampler coverage | no-landing reason | +|---|---|---|---|---|---|---|---|---|---| +| d1-dense20k-fe1-async-r1 | NOT COMPLETED (exit 6); not in medians | false | n/a | n/a | n/a | n/a | n/a | n/a | n/a | +| d1-dense400k-fe1-async-r1 | ok | true (phase completed: true) | 15 / 15 / 14 | 14 / 1 / 0 / 0 / 0 | 0 | 28 / 28 / 0 | 0 / 0 / 0 | 1 / blind: false | – | +| d1-dense400k-fe1-async-r2 | ok | true (phase completed: true) | 15 / 15 / 15 | 15 / 0 / 0 / 0 / 0 | 0 | 30 / 30 / 0 | 0 / 0 / 0 | 1 / blind: false | – | +| d1-dense400k-fe2-async-r1 | ok | true (phase completed: true) | 15 / 15 / 15 | 15 / 0 / 0 / 0 / 0 | 0 | 30 / 30 / 0 | 0 / 0 / 0 | 1 / blind: false | – | + +Summaries over every landing and frontend (`dense_cadence.summaries.overall`): + +| run id | status | combined rebuild-pending window ms: p50 / p99 / max (samples) | combined rebuild-pending rejections per landing per frontend: p50 / p99 / max (samples) | rejected before new-revision work per landing per frontend: p50 / p99 / max (samples) | lost valid shares: total / found in PostgreSQL | #291 budget ms: window p99 / recommended | +|---|---|---|---|---|---|---| +| d1-dense400k-fe1-async-r1 | ok | n/a / n/a / n/a (0) | 0 / 0 / 0 (14) | 0 / 0 / 0 (14) | 0 / 0 | n/a / n/a | +| d1-dense400k-fe1-async-r2 | ok | n/a / n/a / n/a (0) | 0 / 0 / 0 (15) | 0 / 0 / 0 (15) | 0 / 0 | n/a / n/a | +| d1-dense400k-fe2-async-r1 | ok | n/a / n/a / n/a (0) | 0 / 0 / 0 (30) | 0 / 0 / 0 (27) | 0 / 0 | n/a / n/a | + +"New-revision work" is an approximation the harness labels as such: the first +`clean_jobs` notify at or after the bump stands in for the first job at the +new revision (cadence.rs:1221-1228). + +Why a cell is `n/a`, or what qualifies it: + +- dense phase did not run: "the run ended before the dense_cadence phase could + run, so there is no schedule, no landing and no window to report"; aborted: + "20 submit(s) offered before the dense_cadence phase were still outstanding + 25.0 s later, so its 0 ms delay could not be applied without them finishing + under it" — d1-dense20k-fe1-async-r1 +- `summaries.overall.combined_rebuild_pending_window_duration_millis`: no + samples were recorded — d1-dense400k-fe1-async-r1; + d1-dense400k-fe1-async-r2; d1-dense400k-fe2-async-r1 + +### Measurements + +Every table below is generated from the side reports by one script and none +is typed by hand. The flush control shares these tables with its ordinary +pair, in its own rows: its plan column reads +`d1 +wal_sync_method=fsync_writethrough`, and no median mixes the two. The +build control's runs, on either build, are not in these tables; they are +under [The build control](#the-build-control). Source line references in this +part +are to `a1937054`. + +**Configuration groups.** + +Grouping uses the report's own fields: `window.requested_window_shares`, +`topology.frontends`, `premise.replication.declared`, `topology.plan`, and, so +that only true repeats share a row, `topology.sessions`, each phase's +`target_rate_shares_per_second`, `dense_cadence.cadence`, the scheduled-block +count, `mid_flight_kill.ran`, the primary's effective `wal_sync_method` +(`database-profile.json`), the commit the binaries were built from and whether +the server binary was that build's own (`driver.json`). A run that differs in +any of these is never a repeat of another, and its plan column says how it +differs. Runs are in `driver.json` `started_at_utc` order. + +| window | fe | repl | plan | sessions | target /s steady / burst / reconn / slowdb | scheduled blocks | n in medians | runs in medians (run order) | completed with a finding (same group) | not completed (same window/fe/repl/plan) | +|---|---|---|---|---|---|---|---|---|---|---| +| 200k | 1 | async | d1 | 2000 | 500 / 2000 / 500 / 500 | 0 | 1 | d1-200k-fe1-async-r1 | – | – | +| 400k | 1 | async | d1 | 2000 | 500 / 2000 / 500 / 500 | 0 | 3 | d1-400k-fe1-async-r1, d1-400k-fe1-async-r2, d1-400k-fe1-async-r3 | – | – | +| 400k | 2 | async | d1 | 2000 | 500 / 2000 / 500 / 500 | 0 | 3 | d1-400k-fe2-async-r1, d1-400k-fe2-async-r2, d1-400k-fe2-async-r3 | – | – | +| 400k | 4 | async | d1 | 2000 | 500 / 2000 / 500 / 500 | 0 | 3 | d1-400k-fe4-async-r1, d1-400k-fe4-async-r2, d1-400k-fe4-async-r3 | – | – | +| 400k | 2 | sync | d1 | 2000 | 500 / 2000 / 500 / 500 | 0 | 3 | d1-400k-fe2-sync-r1, d1-400k-fe2-sync-r2, d1-400k-fe2-sync-r3 | – | – | +| 400k | 2 | async | d1 +3 blocks | 2000 | 500 / 2000 / 500 / 500 | 3 | 1 | d1-400k-fe2-async-blocks3-r1 | – | – | +| 400k | 1 | async | d1 +dense(15) | 2000 | 500 / 2000 / 500 / 500 | 15 | 2 | d1-dense400k-fe1-async-r1, d1-dense400k-fe1-async-r2 | – | – | +| 400k | 2 | async | d1 +dense(15) | 2000 | 500 / 2000 / 500 / 500 | 15 | 1 | d1-dense400k-fe2-async-r1 | – | – | +| 400k | 1 | async | d1 +wal_sync_method=fsync_writethrough | 2000 | 500 / 2000 / 500 / 500 | 0 | 1 | d1-400k-fe1-async-wt-r2 | d1-400k-fe1-async-wt-r1, d1-400k-fe1-async-wt-r3 | – | +| 500k | 1 | async | d1 | 2000 | 500 / 2000 / 500 / 500 | 0 | 3 | d1-500k-fe1-async-r1, d1-500k-fe1-async-r2, d1-500k-fe1-async-r3 | – | – | +| 500k | 2 | async | d1 | 2000 | 500 / 2000 / 500 / 500 | 0 | 3 | d1-500k-fe2-async-r1, d1-500k-fe2-async-r2, d1-500k-fe2-async-r3 | – | – | +| 500k | 4 | async | d1 | 2000 | 500 / 2000 / 500 / 500 | 0 | 3 | d1-500k-fe4-async-r1, d1-500k-fe4-async-r2, d1-500k-fe4-async-r3 | – | – | +| 20k | 1 | async | d1 | n/a | n/a | n/a | 0 | – | – | d1-dense20k-fe1-async-r1 | + +#### Rate and ACK latency + +| window | fe | repl | plan | n | steady achieved/offered (target) /s | steady ACK p50 / p99 / max ms | burst achieved/offered (target) /s | burst ACK p50 / p99 / max ms | reconn achieved/offered (target) /s | reconn ACK p50 / p99 / max ms | slowdb achieved/offered (target) /s | slowdb ACK p50 / p99 / max ms | +|---|---|---|---|---|---|---|---|---|---|---|---|---| +| 200k | 1 | async | d1 | 1 | 499.997 / 499.997 (500) | 1.1 / 20.5 / 630 | 1,558 / 1,558 (2000) | 1,289 / 1,886 / 2,241 | 499.983 / 499.983 (500) | 1.1 / 128 / 610 | 3.2 / 329 (500) | 10,015 / 30,348 / 30,429 | +| 400k | 1 | async | d1 | 3 | 499 (499–499) / 499.997 (499.995–499.997) (500) | 1.1 (1.1–1.7) / 18.5 (8.6–25.1) / 574 (546–636) | 1,623 (1,503–1,756) / 1,623 (1,503–1,756) (2000) | 1,228 (1,145–1,318) / 4,026 (3,357–4,397) / 10,033 (9,626–11,348) | 499.983 (499.975–499.983) / 499.983 (499.975–499.983) (500) | 1.2 (1.1–1.7) / 780 (705–1,015) / 1,189 (1,132–1,390) | 1.7 (1.7–1.7) / 417 (412–422) (500) | 10,413 (10,216–10,483) / 30,010 (29,993–30,514) / 30,024 (30,020–30,516) | +| 400k | 2 | async | d1 | 3 | 499 (499–499) / 499.995 (499.995–499.997) (500) | 1.1 (1.1–1.1) / 23.2 (11.4–37.5) / 640 (621–726) | 1,490 (1,446–1,522) / 1,510 (1,468–1,522) (2000) | 1,340 (1,324–1,368) / 2,608 (2,464–3,099) / 5,882 (5,828–6,596) | 499.983 (499.975–499.983) / 499.983 (499.975–499.983) (500) | 1.1 (1.1–1.1) / 870 (845–940) / 1,275 (1,261–1,334) | 2.4 (2.4–2.6) / 423 (422–430) (500) | 14,825 (14,596–16,225) / 29,710 (29,651–29,861) / 29,816 (29,774–29,863) | +| 400k | 4 | async | d1 | 3 | 499.995 (499.995–499.997) / 499.995 (499.995–499.997) (500) | 1.1 (1.1–1.2) / 15 (13.2–16.1) / 566 (473–606) | 1,504 (1,423–1,663) / 1,504 (1,423–1,663) (2000) | 1,346 (1,213–1,411) / 1,882 (1,634–1,972) / 2,016 (1,784–4,093) | 499.975 (499.967–499.983) / 499.983 (499.975–499.983) (500) | 1.2 (1.1–1.7) / 1,309 (1,102–1,381) / 1,674 (1,503–1,771) | 4.4 (3.6–4.4) / 439 (423–443) (500) | 16,148 (15,393–18,004) / 41,191 (28,282–41,425) / 41,791 (28,918–42,701) | +| 400k | 2 | sync | d1 | 3 | 499.995 (499.995–499.997) / 499.995 (499.995–499.997) (500) | 1.2 (1.2–1.3) / 33 (9–2,560) / 603 (505–2,660) | 1,505 (1,403–1,617) / 1,505 (1,403–1,617) (2000) | 1,342 (1,253–1,432) / 1,812 (1,667–1,849) / 1,991 (1,790–2,079) | 499.983 (499.983–499.983) / 499.983 (499.983–499.983) (500) | 1.2 (1.2–1.2) / 1,265 (966–1,355) / 1,620 (1,381–1,692) | 2.4 (2.2–2.5) / 420 (418–440) (500) | 14,485 (13,379–14,996) / 29,708 (29,369–29,751) / 29,848 (29,414–29,925) | +| 400k | 2 | async | d1 +3 blocks | 1 | 482 / 499.995 (500) | 1.2 / 51.1 / 779 | 1,600 / 1,600 (2000) | 1,247 / 1,688 / 1,946 | 499.983 / 499.983 (500) | 1.1 / 828 / 1,246 | 3.8 / 428 (500) | 13,698 / 29,334 / 29,725 | +| 400k | 1 | async | d1 +dense(15) | 2 | 498 (497–499) / 498.828 (497.66–499.997) (500) | 1.1 (1.1–1.1) / 1,963 (22–3,903) / 2,966 (532–5,401) | 1,491 (1,455–1,527) / 1,491 (1,456–1,527) (2000) | 1,318 (1,287–1,348) / 4,786 (4,378–5,194) / 11,300 (10,871–11,729) | 499.983 (499.983–499.983) / 499.983 (499.983–499.983) (500) | 1.1 (1.1–1.2) / 887 (775–999) / 1,373 (1,192–1,554) | 1.7 (1.7–1.7) / 422 (421–422) (500) | 10,342 (10,226–10,459) / 30,133 (30,019–30,247) / 30,133 (30,019–30,247) | +| 400k | 2 | async | d1 +dense(15) | 1 | 499 / 499.995 (500) | 1.1 / 19.8 / 641 | 1,467 / 1,493 (2000) | 1,368 / 2,255 / 5,706 | 499.983 / 499.983 (500) | 1.1 / 1,008 / 1,350 | 2.5 / 423 (500) | 15,072 / 29,621 / 29,927 | +| 400k | 1 | async | d1 +wal_sync_method=fsync_writethrough | 1 | 117 / 250 (500) | 10,323 / 38,296 / 60,775 | 107 / 762 (2000) | 7,330 / 39,017 / 66,057 | 122 / 309 (500) | 8,134 / 36,512 / 38,573 | 1.8 / 406 (500) | 11,532 / 30,386 / 30,390 | +| 500k | 1 | async | d1 | 3 | 499 (499–499) / 499.997 (499.995–499.997) (500) | 1.2 (1.1–1.2) / 13.3 (13.2–26.7) / 606 (598–711) | 1,550 (1,481–1,561) / 1,550 (1,481–1,561) (2000) | 1,253 (1,251–1,324) / 4,744 (3,828–4,870) / 12,042 (11,052–12,149) | 499.975 (499.975–499.983) / 499.975 (499.975–499.983) (500) | 1.1 (1.1–1.4) / 865 (766–887) / 1,245 (1,184–1,258) | 1.8 (1.6–1.8) / 416 (416–417) (500) | 10,859 (10,040–11,039) / 30,029 (30,023–30,285) / 30,040 (30,029–30,293) | +| 500k | 2 | async | d1 | 3 | 499 (499–499) / 499.997 (499.995–499.997) (500) | 1.1 (1.1–1.1) / 17.4 (8.1–28.1) / 586 (515–612) | 1,504 (1,496–1,523) / 1,522 (1,522–1,548) (2000) | 1,320 (1,302–1,323) / 2,289 (2,179–3,083) / 6,167 (6,140–7,119) | 499.975 (499.975–499.983) / 499.975 (499.975–499.983) (500) | 1.1 (1.1–1.1) / 885 (867–904) / 1,291 (1,265–1,302) | 2.3 (2.1–2.3) / 425 (423–425) (500) | 14,019 (13,030–14,253) / 29,834 (29,569–29,926) / 29,927 (29,896–29,994) | +| 500k | 4 | async | d1 | 3 | 499.997 (499.995–499.997) / 499.997 (499.995–499.997) (500) | 1.1 (1.1–1.2) / 13.3 (13–22.5) / 616 (513–646) | 1,468 (1,401–1,514) / 1,468 (1,401–1,514) (2000) | 1,388 (1,332–1,423) / 1,879 (1,854–1,978) / 2,175 (1,964–4,257) | 499.983 (499.983–499.983) / 499.983 (499.983–499.983) (500) | 1.1 (1.1–1.2) / 1,513 (1,449–1,815) / 1,852 (1,823–2,124) | 2.4 (2.3–4.5) / 440 (431–444) (500) | 14,785 (14,498–17,890) / 27,345 (26,762–40,747) / 27,995 (26,924–40,831) | + +#### Refusal latency + +A refusal's latency runs from writing the submit to reading the refusal. The +count of refusals (`client_rejection_latency.samples`) is in the last +parentheses. + +| window | fe | repl | plan | n | steady refusal p50 / p99 ms (refusals) | burst refusal p50 / p99 ms (refusals) | reconn refusal p50 / p99 ms (refusals) | slowdb refusal p50 / p99 ms (refusals) | +|---|---|---|---|---|---|---|---|---| +| 200k | 1 | async | d1 | 1 | n/a / n/a (0) | n/a / n/a (0) | n/a / n/a (0) | 0.2 / 29,898 (19,524) | +| 400k | 1 | async | d1 | 3 | 168 (166–190) / 408 (363–433) (193 (171–200)) | n/a / n/a (0 (0–0)) | n/a / n/a (0 (0–0)) | 0.2 (0.2–0.2) / 28,452 (28,400–28,905) (24,945 (24,594–25,237)) | +| 400k | 2 | async | d1 | 3 | 200 (186–203) / 453 (449–456) (204 (202–206)) | 0.1 (0.1–0.1) [n=2] / 0.2 (0.2–0.2) [n=2] (1,203 (0–1,356)) | n/a / n/a (0 (0–0)) | 0.2 (0.2–0.2) / 29,123 (28,846–29,137) (25,258 (25,145–25,635)) | +| 400k | 4 | async | d1 | 3 | n/a / n/a (0 (0–0)) | n/a / n/a (0 (0–0)) | n/a / n/a (0 (0–0)) | 0.2 (0.2–0.2) / 23,334 (22,863–34,044) (25,943 (25,193–26,026)) | +| 400k | 2 | sync | d1 | 3 | n/a / n/a (0 (0–0)) | n/a / n/a (0 (0–0)) | n/a / n/a (0 (0–0)) | 0.2 (0.2–0.2) / 29,023 (28,925–29,870) (25,076 (24,957–26,028)) | +| 400k | 2 | async | d1 +3 blocks | 1 | 0.3 / 293 (5,263) | n/a / n/a (0) | n/a / n/a (0) | 0.2 / 29,660 (25,431) | +| 400k | 1 | async | d1 +dense(15) | 2 | 194 (194–195) / 1,763 (463–3,063) (200 (196–203)) | 0.1 [n=1] / 0.2 [n=1] (41.5 (0–83)) | n/a / n/a (0 (0–0)) | 0.2 (0.2–0.2) / 28,589 (28,524–28,654) (25,192 (25,151–25,232)) | +| 400k | 2 | async | d1 +dense(15) | 1 | 202 / 441 (197) | 0.1 / 0.3 (1,561) | n/a / n/a (0) | 0.2 / 29,249 (25,222) | +| 400k | 1 | async | d1 +wal_sync_method=fsync_writethrough | 1 | 0.2 / 60,470 (39,841) | 0.2 / 55,357 (39,299) | 0.2 / 19,773 (11,191) | 0.2 / 28,547 (24,265) | +| 500k | 1 | async | d1 | 3 | 198 (192–229) / 459 (428–559) (193 (190–248)) | n/a / n/a (0 (0–0)) | n/a / n/a (0 (0–0)) | 0.2 (0.2–0.2) / 28,757 (28,752–28,868) (24,889 (24,879–24,924)) | +| 500k | 2 | async | d1 | 3 | 200 (188–201) / 453 (433–475) (206 (200–215)) | 0.1 (0.1–0.1) / 0.3 (0.2–3.7) (1,496 (1,041–1,516)) | n/a / n/a (0 (0–0)) | 0.2 (0.2–0.2) / 28,882 (28,868–29,495) (25,336 (25,251–25,349)) | +| 500k | 4 | async | d1 | 3 | n/a / n/a (0 (0–0)) | n/a / n/a (0 (0–0)) | n/a / n/a (0 (0–0)) | 0.2 (0.2–0.2) / 23,088 (22,219–23,482) (25,784 (25,714–26,476)) | + +Why a cell is `n/a`, or what qualifies it: + +- `client_rejection_latency`: no samples were recorded — 26 runs, including + d1-200k-fe1-async-r1 (steady, burst, reconn); d1-400k-fe1-async-r1 (burst, + reconn); d1-400k-fe1-async-r2 (burst, reconn) + +#### Rejections by `(code, reason_id, message)` + +Counts per phase, over the classes found in the data. No harness-bug rejection +class appeared in any run. +A class with no row in a completed phase's tally is a counted 0: the tally is +built from every submit record and lists a class only where it occurred +(run.rs:2665-2698). + +| window | fe | repl | plan | n | phase | `20` `backend-rpc-unavailable` "current chain state is unavailable" | `20` `backend-rpc-unavailable` "current payout state is unavailable" | `20` `ledger-confirmation-failed` "share was not committed because its commit gate closed" | `20` `ledger-confirmation-failed` "share was not confirmed by the database" | `21` `stale-job` "stale job" | `21` `unknown-job` "stale job" | +|---|---|---|---|---|---|---|---|---|---|---|---| +| 200k | 1 | async | d1 | 1 | steady | 0 | 0 | 0 | 0 | 0 | 0 | +| 200k | 1 | async | d1 | 1 | burst | 0 | 0 | 0 | 0 | 0 | 0 | +| 200k | 1 | async | d1 | 1 | reconn | 0 | 0 | 0 | 0 | 0 | 0 | +| 200k | 1 | async | d1 | 1 | slowdb | 15,599 | 548 | 0 | 3,377 | 0 | 0 | +| 200k | 1 | async | d1 | 1 | warm_up | 0 | 0 | 3 | 1 | 0 | 0 | +| 400k | 1 | async | d1 | 3 | steady | 0 (0–0) | 0 (0–0) | 0 (0–1) | 0 (0–0) | 193 (171–199) | 0 (0–0) | +| 400k | 1 | async | d1 | 3 | burst | 0 (0–0) | 0 (0–0) | 0 (0–0) | 0 (0–0) | 0 (0–0) | 0 (0–0) | +| 400k | 1 | async | d1 | 3 | reconn | 0 (0–0) | 0 (0–0) | 0 (0–0) | 0 (0–0) | 0 (0–0) | 0 (0–0) | +| 400k | 1 | async | d1 | 3 | slowdb | 22,979 (22,628–23,277) | 272 (219–340) | 0 (0–0) | 1,688 (1,626–1,747) | 0 (0–0) | 0 (0–0) | +| 400k | 1 | async | d1 | 3 | warm_up | 0 (0–0) | 0 (0–0) | 0 (0–0) | 0 (0–2) | 0 (0–0) | 0 (0–0) | +| 400k | 2 | async | d1 | 3 | steady | 0 (0–0) | 0 (0–0) | 1 (1–1) | 0 (0–0) | 203 (201–205) | 0 (0–0) | +| 400k | 2 | async | d1 | 3 | burst | 1,203 (0–1,356) | 0 (0–0) | 0 (0–0) | 0 (0–0) | 0 (0–0) | 0 (0–0) | +| 400k | 2 | async | d1 | 3 | reconn | 0 (0–0) | 0 (0–0) | 0 (0–0) | 0 (0–0) | 0 (0–0) | 0 (0–0) | +| 400k | 2 | async | d1 | 3 | slowdb | 23,167 (22,735–23,302) | 3 (0–4) | 0 (0–0) | 1,974 (1,956–2,897) | 0 (0–0) | 0 (0–0) | +| 400k | 2 | async | d1 | 3 | warm_up | 0 (0–0) | 0 (0–0) | 0 (0–0) | 0 (0–1) | 0 (0–0) | 0 (0–0) | +| 400k | 4 | async | d1 | 3 | steady | 0 (0–0) | 0 (0–0) | 0 (0–0) | 0 (0–0) | 0 (0–0) | 0 (0–0) | +| 400k | 4 | async | d1 | 3 | burst | 0 (0–0) | 0 (0–0) | 0 (0–0) | 0 (0–0) | 0 (0–0) | 0 (0–0) | +| 400k | 4 | async | d1 | 3 | reconn | 0 (0–0) | 0 (0–0) | 0 (0–0) | 0 (0–0) | 0 (0–0) | 0 (0–0) | +| 400k | 4 | async | d1 | 3 | slowdb | 23,955 (22,762–24,032) | 20 (11–23) | 0 (0–0) | 1,983 (1,968–2,408) | 0 (0–0) | 0 (0–0) | +| 400k | 4 | async | d1 | 3 | warm_up | 0 (0–0) | 0 (0–0) | 3 (3–7) | 125 (118–268) | 200 (143–221) | 0 (0–0) | +| 400k | 2 | sync | d1 | 3 | steady | 0 (0–0) | 0 (0–0) | 0 (0–0) | 0 (0–0) | 0 (0–0) | 0 (0–0) | +| 400k | 2 | sync | d1 | 3 | burst | 0 (0–0) | 0 (0–0) | 0 (0–0) | 0 (0–0) | 0 (0–0) | 0 (0–0) | +| 400k | 2 | sync | d1 | 3 | reconn | 0 (0–0) | 0 (0–0) | 0 (0–0) | 0 (0–0) | 0 (0–0) | 0 (0–0) | +| 400k | 2 | sync | d1 | 3 | slowdb | 23,116 (22,990–24,048) | 5 (4–73) | 0 (0–0) | 1,956 (1,907–1,962) | 0 (0–0) | 0 (0–0) | +| 400k | 2 | sync | d1 | 3 | warm_up | 0 (0–0) | 0 (0–0) | 1 (1–4) | 1 (1–1) | 0 (0–1) | 0 (0–0) | +| 400k | 2 | async | d1 +3 blocks | 1 | steady | 0 | 0 | 6 | 6 | 4,901 | 350 | +| 400k | 2 | async | d1 +3 blocks | 1 | burst | 0 | 0 | 0 | 0 | 0 | 0 | +| 400k | 2 | async | d1 +3 blocks | 1 | reconn | 0 | 0 | 0 | 0 | 0 | 0 | +| 400k | 2 | async | d1 +3 blocks | 1 | slowdb | 21,557 | 12 | 0 | 3,862 | 0 | 0 | +| 400k | 1 | async | d1 +dense(15) | 2 | steady | 0 (0–0) | 0 (0–0) | 1 (0–2) | 4.5 (0–9) | 194 (187–201) | 0 (0–0) | +| 400k | 1 | async | d1 +dense(15) | 2 | burst | 41.5 (0–83) | 0 (0–0) | 0 (0–0) | 0 (0–0) | 0 (0–0) | 0 (0–0) | +| 400k | 1 | async | d1 +dense(15) | 2 | reconn | 0 (0–0) | 0 (0–0) | 0 (0–0) | 0 (0–0) | 0 (0–0) | 0 (0–0) | +| 400k | 1 | async | d1 +dense(15) | 2 | slowdb | 23,227 (23,187–23,267) | 268 (264–272) | 0 (0–0) | 1,697 (1,692–1,701) | 0 (0–0) | 0 (0–0) | +| 400k | 1 | async | d1 +dense(15) | 2 | dense_cadence | 1,055 (1,035–1,074) | 0 (0–0) | 8 (7–9) | 27.5 (20–35) | 17,030 (16,461–17,599) | 2,899 (2,801–2,997) | +| 400k | 1 | async | d1 +dense(15) | 2 | warm_up | 0 (0–0) | 0 (0–0) | 0 (0–0) | 1.5 (0–3) | 0 (0–0) | 0 (0–0) | +| 400k | 2 | async | d1 +dense(15) | 1 | steady | 0 | 0 | 1 | 0 | 196 | 0 | +| 400k | 2 | async | d1 +dense(15) | 1 | burst | 1,561 | 0 | 0 | 0 | 0 | 0 | +| 400k | 2 | async | d1 +dense(15) | 1 | reconn | 0 | 0 | 0 | 0 | 0 | 0 | +| 400k | 2 | async | d1 +dense(15) | 1 | slowdb | 22,813 | 5 | 0 | 2,404 | 0 | 0 | +| 400k | 2 | async | d1 +dense(15) | 1 | dense_cadence | 1,097 | 0 | 19 | 23 | 21,504 | 1,512 | +| 400k | 2 | async | d1 +dense(15) | 1 | warm_up | 0 | 0 | 0 | 2 | 0 | 0 | +| 400k | 1 | async | d1 +wal_sync_method=fsync_writethrough | 1 | steady | 37,008 | 7 | 845 | 310 | 1,671 | 0 | +| 400k | 1 | async | d1 +wal_sync_method=fsync_writethrough | 1 | burst | 39,297 | 0 | 0 | 0 | 2 | 0 | +| 400k | 1 | async | d1 +wal_sync_method=fsync_writethrough | 1 | reconn | 11,190 | 0 | 0 | 0 | 1 | 0 | +| 400k | 1 | async | d1 +wal_sync_method=fsync_writethrough | 1 | slowdb | 22,306 | 299 | 0 | 1,660 | 0 | 0 | +| 400k | 1 | async | d1 +wal_sync_method=fsync_writethrough | 1 | warm_up | 0 | 0 | 0 | 1,419 | 0 | 0 | +| 500k | 1 | async | d1 | 3 | steady | 0 (0–0) | 0 (0–0) | 1 (0–1) | 0 (0–0) | 193 (189–247) | 0 (0–0) | +| 500k | 1 | async | d1 | 3 | burst | 0 (0–0) | 0 (0–0) | 0 (0–0) | 0 (0–0) | 0 (0–0) | 0 (0–0) | +| 500k | 1 | async | d1 | 3 | reconn | 0 (0–0) | 0 (0–0) | 0 (0–0) | 0 (0–0) | 0 (0–0) | 0 (0–0) | +| 500k | 1 | async | d1 | 3 | slowdb | 22,920 (22,919–22,964) | 234 (210–245) | 0 (0–0) | 1,726 (1,724–1,750) | 0 (0–0) | 0 (0–0) | +| 500k | 1 | async | d1 | 3 | warm_up | 0 (0–0) | 0 (0–0) | 0 (0–0) | 1 (0–1) | 0 (0–0) | 0 (0–0) | +| 500k | 2 | async | d1 | 3 | steady | 0 (0–0) | 0 (0–0) | 2 (2–2) | 1 (0–1) | 203 (198–212) | 0 (0–0) | +| 500k | 2 | async | d1 | 3 | burst | 1,496 (1,041–1,516) | 0 (0–0) | 0 (0–0) | 0 (0–0) | 0 (0–0) | 0 (0–0) | +| 500k | 2 | async | d1 | 3 | reconn | 0 (0–0) | 0 (0–0) | 0 (0–0) | 0 (0–0) | 0 (0–0) | 0 (0–0) | +| 500k | 2 | async | d1 | 3 | slowdb | 23,375 (23,284–23,396) | 20 (14–72) | 0 (0–0) | 1,933 (1,889–1,953) | 0 (0–0) | 0 (0–0) | +| 500k | 2 | async | d1 | 3 | warm_up | 0 (0–0) | 0 (0–0) | 0 (0–0) | 1 (0–1) | 0 (0–0) | 0 (0–0) | +| 500k | 4 | async | d1 | 3 | steady | 0 (0–0) | 0 (0–0) | 0 (0–0) | 0 (0–0) | 0 (0–0) | 0 (0–0) | +| 500k | 4 | async | d1 | 3 | burst | 0 (0–0) | 0 (0–0) | 0 (0–0) | 0 (0–0) | 0 (0–0) | 0 (0–0) | +| 500k | 4 | async | d1 | 3 | reconn | 0 (0–0) | 0 (0–0) | 0 (0–0) | 0 (0–0) | 0 (0–0) | 0 (0–0) | +| 500k | 4 | async | d1 | 3 | slowdb | 23,799 (23,740–24,500) | 6 (5–8) | 0 (0–0) | 1,969 (1,968–1,979) | 0 (0–0) | 0 (0–0) | +| 500k | 4 | async | d1 | 3 | warm_up | 0 (0–0) | 0 (0–0) | 7 (4–11) | 2 (2–3) | 6 (3–51) | 0 (0–0) | + +#### ORDER_LOCK, SETTLEMENT_LOCK and `pg_stat_statements` lock time + +Waiters are sampled from `pg_locks` every 10 ms, so waiter-seconds is a lower +bound (`waiter_seconds_estimate`). The statement columns are `calls` and +`total_exec_time` (ms) of the one normalized advisory-lock statement, which +the MIGRATION, ORDER and SETTLEMENT locks share. + +| window | fe | repl | plan | n | steady ORDER_LOCK max / mean waiters · waiter-s ≥ | steady SETTLEMENT_LOCK max / mean · waiter-s ≥ | steady lock stmt calls / total exec ms | burst ORDER_LOCK max / mean · waiter-s ≥ | burst SETTLEMENT_LOCK max / mean · waiter-s ≥ | burst lock stmt calls / total exec ms | reconn ORDER_LOCK max / mean · waiter-s ≥ | reconn SETTLEMENT_LOCK max / mean · waiter-s ≥ | reconn lock stmt calls / total exec ms | slowdb ORDER_LOCK max / mean · waiter-s ≥ | slowdb SETTLEMENT_LOCK max / mean · waiter-s ≥ | slowdb lock stmt calls / total exec ms | +|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---| +| 200k | 1 | async | d1 | 1 | 15 / 0.07 · 20.4 | 1 / 0.00074 · 0.2 | n/a / n/a | 15 / 14.41 · 865 | 1 / 0.00037 · 0.022 | n/a / n/a | 15 / 0.53 · 31 | 1 / 0.00037 · 0.022 | n/a / n/a | 15 / 8.43 · 506 | 1 / 0.04 · 2.7 | n/a / n/a | +| 400k | 1 | async | d1 | 3 | 15 (15–15) / 0.08 (0.07–0.09) · 23.5 (23.1–29.2) | 1 (1–1) / 0.00078 (0.00075–0.00086) · 0.2 (0.2–0.3) | n/a / n/a | 15 (15–15) / 14.36 (14.36–14.38) · 862 (862–863) | 1 (1–1) / 0.00037 (0.00018–0.00055) · 0.022 (0.011–0.033) | n/a / n/a | 15 (15–15) / 0.51 (0.49–0.63) · 29.7 (28.9–37.2) | 1 (0–1) / 0.00056 (0–0.0011) · 0.033 (0–0.1) | n/a / n/a | 15 (15–15) / 6.14 (6.14–6.35) · 369 (368–381) | 1 (1–1) / 0.05 (0.05–0.05) · 3 (2.9–3) | n/a / n/a | +| 400k | 2 | async | d1 | 3 | 31 (31–31) / 0.16 (0.12–0.24) · 49.8 (34.9–69.8) | 3 (1–4) / 0.0024 (0.0021–0.0026) · 0.7 (0.6–0.8) | n/a / n/a | 31 (31–31) / 30.33 (30.31–30.35) · 1,820 (1,818–1,821) | 1 (1–2) / 0.0037 (0.0022–0.0039) · 0.2 (0.1–0.2) | n/a / n/a | 31 (31–31) / 1.21 (1.21–1.23) · 71.4 (71.4–72.6) | 2 (1–2) / 0.003 (0.0024–0.0039) · 0.2 (0.1–0.2) | n/a / n/a | 31 (31–31) / 17.81 (17.14–17.94) · 1,070 (1,031–1,078) | 2 (2–3) / 0.2 (0.14–0.3) · 12.2 (8.1–18) | n/a / n/a | +| 400k | 4 | async | d1 | 3 | 63 (62–63) / 0.22 (0.21–0.23) · 64.4 (62.6–70.3) | 5 (4–5) / 0.01 (0.01–0.01) · 2.6 (2.3–2.7) | n/a / n/a | 63 (63–63) / 62.38 (62.32–62.39) · 3,743 (3,739–3,744) | 2 (2–3) / 0.01 (0.01–0.02) · 0.7 (0.6–1.3) | n/a / n/a | 63 (63–63) / 3.1 (2.61–3.1) · 183 (153–183) | 7 (5–8) / 0.02 (0.01–0.04) · 1.2 (0.8–2.4) | n/a / n/a | 63 (63–63) / 33.83 (29.12–36.4) · 2,042 (1,748–2,187) | 7 (6–7) / 1.81 (1.54–1.84) · 108 (92–110) | n/a / n/a | +| 400k | 2 | sync | d1 | 3 | 31 (30–31) / 0.21 (0.15–1.39) · 63 (44–421) | 2 (2–2) / 0.0028 (0.0022–0.0033) · 0.8 (0.6–1) | n/a / n/a | 31 (31–31) / 30.47 (30.44–30.47) · 1,828 (1,826–1,828) | 1 (1–1) / 0.0013 (0.00092–0.0017) · 0.1 (0.1–0.1) | n/a / n/a | 31 (31–31) / 1.52 (1.15–1.7) · 90 (68–101) | 2 (2–2) / 0.0028 (0.0024–0.0039) · 0.2 (0.1–0.2) | n/a / n/a | 31 (31–31) / 17.71 (16.54–17.98) · 1,061 (996–1,081) | 2 (1–3) / 0.23 (0.08–0.26) · 14 (4.8–15.6) | n/a / n/a | +| 400k | 2 | async | d1 +3 blocks | 1 | 31 / 0.17 · 51.4 | 2 / 0.0025 · 0.7 | n/a / n/a | 31 / 30.43 · 1,826 | 1 / 0.0017 · 0.1 | n/a / n/a | 31 / 1.18 · 69.8 | 2 / 0.0022 · 0.1 | n/a / n/a | 31 / 22.18 · 1,331 | 2 / 0.12 · 7.4 | n/a / n/a | +| 400k | 1 | async | d1 +dense(15) | 2 | 15 (15–15) / 0.64 (0.1–1.17) · 202 (33–371) | 1.5 (1–2) / 0.0011 (0.00097–0.0012) · 0.3 (0.3–0.3) | n/a / n/a | 15 (15–15) / 14.35 (14.34–14.36) · 861 (860–862) | 1 (1–1) / 0.00064 (0.00055–0.00074) · 0.038 (0.033–0.044) | n/a / n/a | 15 (15–15) / 0.58 (0.53–0.62) · 34.2 (31.6–36.8) | 1.5 (1–2) / 0.0011 (0.0011–0.0011) · 0.1 (0.1–0.1) | n/a / n/a | 15 (15–15) / 6.15 (6.14–6.16) · 369 (368–370) | 1 (1–1) / 0.05 (0.05–0.05) · 2.9 (2.8–2.9) | n/a / n/a | +| 400k | 2 | async | d1 +dense(15) | 1 | 31 / 0.17 · 51.4 | 2 / 0.0024 · 0.7 | n/a / n/a | 31 / 30.32 · 1,819 | 2 / 0.0028 · 0.2 | n/a / n/a | 31 / 1.35 · 80.2 | 3 / 0.01 · 0.3 | n/a / n/a | 31 / 17.67 · 1,061 | 2 / 0.26 · 15.4 | n/a / n/a | +| 400k | 1 | async | d1 +wal_sync_method=fsync_writethrough | 1 | 15 / 14.11 · 4,234 | 1 / 0.0012 · 0.4 | n/a / n/a | 15 / 13.68 · 821 | 1 / 0.0045 · 0.3 | n/a / n/a | 15 / 13.57 · 815 | 1 / 0.00057 · 0.033 | n/a / n/a | 15 / 6.5 · 391 | 1 / 0.05 · 2.8 | n/a / n/a | +| 500k | 1 | async | d1 | 3 | 15 (15–15) / 0.08 (0.07–0.09) · 24.6 (21–27.3) | 2 (1–2) / 0.00071 (0.00063–0.00074) · 0.2 (0.2–0.2) | n/a / n/a | 15 (15–15) / 14.38 (14.37–14.38) · 863 (862–863) | 1 (1–1) / 0.00037 (0.00018–0.00037) · 0.022 (0.011–0.022) | n/a / n/a | 15 (15–15) / 0.53 (0.52–0.56) · 31.5 (30.5–33) | 1 (1–1) / 0.00074 (0.00019–0.00093) · 0.044 (0.01–0.1) | n/a / n/a | 15 (15–15) / 6.15 (6.08–6.16) · 369 (365–371) | 1 (1–1) / 0.05 (0.05–0.05) · 2.8 (2.7–2.9) | n/a / n/a | +| 500k | 2 | async | d1 | 3 | 31 (31–31) / 0.15 (0.09–0.19) · 46.6 (27.5–62.2) | 3 (2–3) / 0.0025 (0.0021–0.0039) · 0.7 (0.6–1.2) | n/a / n/a | 31 (31–31) / 30.33 (30.32–30.33) · 1,820 (1,819–1,820) | 2 (1–2) / 0.0031 (0.0029–0.0033) · 0.2 (0.2–0.2) | n/a / n/a | 31 (31–31) / 1.15 (1.13–1.22) · 69.1 (66.8–72.8) | 1 (1–1) / 0.0024 (0.002–0.003) · 0.2 (0.1–0.2) | n/a / n/a | 31 (31–31) / 16.61 (16.11–16.92) · 998 (969–1,018) | 3 (2–3) / 0.24 (0.22–0.28) · 14.4 (13.3–16.4) | n/a / n/a | +| 500k | 4 | async | d1 | 3 | 63 (63–63) / 0.18 (0.16–0.26) · 56.6 (49.9–83.5) | 4 (4–5) / 0.01 (0.01–0.01) · 2.5 (2.2–2.6) | n/a / n/a | 63 (63–63) / 62.35 (62.34–62.39) · 3,741 (3,740–3,744) | 2 (2–3) / 0.02 (0.01–0.02) · 1 (0.7–1.1) | n/a / n/a | 63 (63–63) / 3.69 (3.17–4.84) · 218 (187–287) | 4 (3–5) / 0.02 (0.01–0.02) · 0.9 (0.6–0.9) | n/a / n/a | 63 (63–63) / 29.74 (28.26–33.1) · 1,790 (1,706–1,987) | 7 (7–7) / 1.37 (1.19–1.84) · 82 (72–110) | n/a / n/a | + +Why a cell is `n/a`, or what qualifies it: + +- `advisory_lock_statement` is null: `database.pg_stat_statements` = + 'unavailable' — 27 runs, including d1-200k-fe1-async-r1 (steady, burst, + reconn, slowdb); d1-400k-fe1-async-r1 (steady, burst, reconn, slowdb); + d1-400k-fe1-async-r2 (steady, burst, reconn, slowdb) + +#### Frontend CPU, memory and host load + +CPU is the mean number of cores over the phase (`processes[].cpu_cores_mean`): +*sum* adds all frontends, *max fe* is the busiest one. RSS is the largest +`rss_kib_max` (a 1 s sampled resident set) over every phase and frontend, in +MiB. + +**CPU columns are converted.** macOS CPU unit defect in the harness: +`processes[].cpu_seconds` and `cpu_cores_mean` come from `proc_pid_rusage` +`ri_user_time + ri_system_time` divided by 1e9 as if nanoseconds +(measure.rs:267-271). On Apple Silicon those fields are Mach absolute-time +ticks (`mach_timebase_info` 125/3 on the measurement host), so the reported +cores are understated by 125/3 = 41.67x. Values from Apple-silicon reports +built from `a1937054` (before the harness fix) are multiplied by 125/3 here; +the reports themselves are unchanged, and their as-reported per-frontend +`cpu_cores_mean` spans 0.0003–0.0194 cores. + +| window | fe | repl | plan | n | steady CPU cores, reported × 125/3 (sum / max fe) | burst CPU cores, reported × 125/3 (sum / max fe) | reconn CPU cores, reported × 125/3 (sum / max fe) | slowdb CPU cores, reported × 125/3 (sum / max fe) | RSS max MiB (max fe; sampled maximum) | lowest reclaimable-approx MiB (driver vm_stat: free + inactive + speculative pages; not MemAvailable) | load1 before run | load1 during mean / max | +|---|---|---|---|---|---|---|---|---|---|---|---|---| +| 200k | 1 | async | d1 | 1 | 0.18 / 0.18 | 0.36 / 0.36 | 0.18 / 0.18 | 0.05 / 0.05 | 650 | 62,338 | 7.53 | 4.98 / 8.38 | +| 400k | 1 | async | d1 | 3 | 0.21 (0.2–0.21) / 0.21 (0.2–0.21) | 0.39 (0.39–0.4) / 0.39 (0.39–0.4) | 0.2 (0.2–0.21) / 0.2 (0.2–0.21) | 0.06 (0.05–0.07) / 0.06 (0.05–0.07) | 928 (885–928) | 62,812 (62,537–66,918) | 3.53 (1.56–6.95) | 3.98 (2.76–4.57) / 4.76 (3.83–6.64) | +| 400k | 2 | async | d1 | 3 | 0.27 (0.27–0.28) / 0.14 (0.13–0.14) | 0.46 (0.46–0.46) / 0.23 (0.23–0.23) | 0.31 (0.3–0.31) / 0.17 (0.17–0.17) | 0.07 (0.07–0.08) / 0.04 (0.04–0.04) | 846 (843–888) | 62,495 (61,580–64,049) | 4.13 (2.91–4.77) | 5.03 (4.31–5.06) / 7.2 (7.04–7.86) | +| 400k | 4 | async | d1 | 3 | 0.36 (0.36–0.37) / 0.09 (0.09–0.09) | 0.41 (0.4–0.43) / 0.11 (0.1–0.13) | 0.39 (0.38–0.41) / 0.12 (0.11–0.12) | 0.09 (0.08–0.1) / 0.03 (0.02–0.03) | 875 (874–890) | 62,354 (60,959–63,017) | 4.27 (3.18–4.32) | 4.87 (4.57–5.61) / 7.03 (6.27–8.31) | +| 400k | 2 | sync | d1 | 3 | 0.25 (0.25–0.26) / 0.13 (0.13–0.13) | 0.33 (0.33–0.34) / 0.17 (0.16–0.17) | 0.3 (0.29–0.31) / 0.17 (0.16–0.17) | 0.07 (0.07–0.08) / 0.04 (0.04–0.04) | 1,176 (1,160–1,296) | 63,642 (63,343–65,361) | 5.62 (2.62–6.57) | 3.92 (3.12–3.92) / 5.34 (4.42–6.1) | +| 400k | 2 | async | d1 +3 blocks | 1 | 0.44 / 0.25 | 0.44 / 0.22 | 0.27 / 0.14 | 0.06 / 0.03 | 3,315 | 61,940 | 3.82 | 4.53 / 6.72 | +| 400k | 1 | async | d1 +dense(15) | 2 | 0.21 (0.2–0.22) / 0.21 (0.2–0.22) | 0.39 (0.38–0.39) / 0.39 (0.38–0.39) | 0.2 (0.2–0.21) / 0.2 (0.2–0.21) | 0.06 (0.06–0.06) / 0.06 (0.06–0.06) | 3,824 (3,504–4,143) | 64,293 (61,789–66,796) | 3.57 (2.8–4.34) | 5.08 (4.25–5.9) / 7.5 (6.25–8.74) | +| 400k | 2 | async | d1 +dense(15) | 1 | 0.28 / 0.14 | 0.46 / 0.23 | 0.3 / 0.17 | 0.07 / 0.05 | 3,442 | 65,862 | 3.97 | 4.95 / 6.56 | +| 400k | 1 | async | d1 +wal_sync_method=fsync_writethrough | 1 | 0.13 / 0.13 | 0.16 / 0.16 | 0.15 / 0.15 | 0.07 / 0.07 | 883 | 65,832 | 4.26 | 4.31 / 5.64 | +| 500k | 1 | async | d1 | 3 | 0.22 (0.22–0.22) / 0.22 (0.22–0.22) | 0.36 (0.36–0.37) / 0.36 (0.36–0.37) | 0.22 (0.22–0.22) / 0.22 (0.22–0.22) | 0.06 (0.06–0.06) / 0.06 (0.06–0.06) | 1,075 (1,072–1,079) | 64,763 (62,600–67,311) | 4.68 (4.13–5.01) | 4.49 (4.49–4.91) / 6.35 (6.22–6.89) | +| 500k | 2 | async | d1 | 3 | 0.29 (0.29–0.3) / 0.15 (0.15–0.15) | 0.48 (0.47–0.48) / 0.24 (0.24–0.24) | 0.28 (0.27–0.29) / 0.14 (0.14–0.15) | 0.07 (0.07–0.08) / 0.04 (0.04–0.05) | 1,037 (1,036–1,156) | 64,014 (63,704–68,106) | 3.97 (2.54–3.98) | 5.32 (5.22–6.03) / 7.58 (7.22–8.65) | +| 500k | 4 | async | d1 | 3 | 0.41 (0.41–0.43) / 0.1 (0.1–0.11) | 0.39 (0.39–0.42) / 0.1 (0.1–0.11) | 0.46 (0.41–0.48) / 0.14 (0.12–0.16) | 0.1 (0.09–0.12) / 0.03 (0.02–0.03) | 1,868 (1,864–1,869) | 63,840 (61,964–64,919) | 4.7 (3.23–7.92) | 5.48 (4.52–5.95) / 8.49 (6–9.57) | + +Why a cell is `n/a`, or what qualifies it: + +- No kernel peak-RSS (`VmHWM`) column: `processes[].peak_rss_kib` is null in + every report here (`peak_rss_source`: sampled max; measure.rs:278-304). The + RSS column is the only resident-set figure these reports carry +- No `MemAvailable` column: `phases[].min_mem_available_kib` is null in every + report here (the harness reads /proc/meminfo, measure.rs:1192-1202, which + macOS lacks). The reclaimable-approx column is the driver's own + approximation and is not comparable with the Linux `MemAvailable` figures in + earlier tables + +#### Reconnects, time to usable work and outcomes + +The time-to-usable-work cell is `all_sessions_milliseconds` of the last +external tip (the one no later external tip replaces), and only when every +session had work on it. The outcome columns list per-run values in run order; +they cover the runs in the medians and then, starred, the runs of the same +group that completed with a finding (those are in no median). + +| window | fe | repl | plan | n | reconnect phase: completed / failed · reconnect p50 / max ms | tip 103: all 2,000 sessions with work, ms | earlier tips: sessions with usable work before replacement | ACK/commit divergences (#324) | unknown-outcome commits | durability findings | teardown tail (window-ended no-response commits) | mid-run no-response commits | harness-bug rejections | submits outstanding when the stop drain expired | exit codes | +|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---| +| 200k | 1 | async | d1 | 1 | 13 / 0 · 5.2 / 7.8 | 5,582 | 101: 2,000 of 2,000; 102: 2,000 of 2,000 | 0 | 0 | 0 | 0 | 0 | 0 | 17 | 0 | +| 400k | 1 | async | d1 | 3 | 13 (13–13) / 0 (0–0) · 5.8 (5.8–6.5) / 7.2 (7–7.8) | 21,557 (19,895–21,958) | 101: 0 (0–0) of 2,000; 102: 0 (0–0) of 2,000 | 0, 0, 0 | 0, 0, 0 | 0, 0, 0 | 0, 0, 0 | 0, 0, 0 | 0, 0, 0 | 0, 0, 0 | 0, 0, 0 | +| 400k | 2 | async | d1 | 3 | 1,013 (1,013–1,013) / 1,000 (997–1,000) · 2,792 (2,764–2,867) / 2,849 (2,817–2,929) | 35,061 (33,713–36,684) | 101: 0 (0–0) of 2,000; 102: 0 (0–0) of 2,000 | 0, 0, 0 | 0, 0, 0 | 0, 0, 0 | 0, 0, 0 | 0, 0, 0 | 0, 0, 0 | 0, 0, 0 | 0, 0, 0 | +| 400k | 4 | async | d1 | 3 | 513 (513–513) / 495 (491–500) · 2,705 (2,646–2,784) / 2,739 (2,680–2,820) | 3,832 (3,704–7,809) | 101: 0 (0–0) of 2,000; 102: 2,000 (1,500–2,000) of 2,000 | 0, 0, 0 | 0, 0, 0 | 0, 0, 0 | 0, 0, 7 | 0, 0, 0 | 0, 0, 0 | 0, 71, 367 | 0, 0, 0 | +| 400k | 2 | sync | d1 | 3 | 1,013 (1,013–1,013) / 994 (990–995) · 2,775 (2,686–2,935) / 2,828 (2,737–2,997) | 3,294 (3,282–3,296) | 101: 2,000 (2,000–2,000) of 2,000; 102: 2,000 (2,000–2,000) of 2,000 | 0, 0, 0 | 0, 0, 0 | 0, 0, 0 | 0, 0, 0 | 0, 0, 0 | 0, 0, 0 | 0, 233, 0 | 0, 0, 0 | +| 400k | 2 | async | d1 +3 blocks | 1 | 1,013 / 1,000 · 2,791 / 2,854 | 34,379 | 101: 0 of 2,000; 102: 0 of 2,000 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | +| 400k | 1 | async | d1 +dense(15) | 2 | 13 (13–13) / 0 (0–0) · 5.9 (5.8–6.1) / 50.1 (8.1–92.1) | 21,635 (21,388–21,883) | 101: 0 (0–0) of 2,000; 102: 0 (0–0) of 2,000 | 0, 0 | 0, 0 | 0, 0 | 0, 0 | 0, 0 | 0, 0 | 0, 0 | 0, 0 | +| 400k | 2 | async | d1 +dense(15) | 1 | 1,013 / 995 · 2,822 / 2,888 | 35,733 | 101: 0 of 2,000; 102: 0 of 2,000 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | +| 400k | 1 | async | d1 +wal_sync_method=fsync_writethrough | 1 | 13 / 5 · 10,858 / 67,020 | 445,150 | 101: 0 of 2,000; 102: 0 of 2,000 | 0, 0*, 0* | 0, 0*, 0* | 0, 0*, 0* | 0, 0*, 0* | 0, 1*, 2* | 0, 0*, 0* | 0, 1347*, 0* | 0, 5*, 5* | +| 500k | 1 | async | d1 | 3 | 13 (13–13) / 0 (0–0) · 5.6 (5.6–5.8) / 8 (8–431) | 23,842 (22,808–25,945) | 101: 0 (0–0) of 2,000; 102: 0 (0–0) of 2,000 | 0, 0, 0 | 0, 0, 0 | 0, 0, 0 | 0, 0, 0 | 0, 0, 0 | 0, 0, 0 | 0, 0, 0 | 0, 0, 0 | +| 500k | 2 | async | d1 | 3 | 1,013 (1,013–1,013) / 1,000 (997–1,000) · 3,533 (3,430–3,562) / 3,592 (3,487–3,625) | 27,527 (26,732–27,836) | 101: 0 (0–0) of 2,000; 102: 0 (0–0) of 2,000 | 0, 0, 0 | 0, 0, 0 | 0, 0, 0 | 0, 0, 0 | 0, 0, 0 | 0, 0, 0 | 0, 0, 0 | 0, 0, 0 | +| 500k | 4 | async | d1 | 3 | 513 (513–513) / 500 (493–500) · 3,365 (3,358–3,390) / 3,399 (3,394–3,427) | 4,461 (4,401–4,976) | 101: 2,000 (1,500–2,000) of 2,000; 102: 2,000 (2,000–2,000) of 2,000 | 0, 0, 0 | 0, 0, 0 | 0, 0, 0 | 0, 0, 0 | 0, 0, 0 | 0, 0, 0 | 0, 373, 0 | 0, 0, 0 | + +`*` marks a run that completed with a finding (exit 4, 5, 7 or 8). It is in no +median. + +#### Per run + +Every run with a side report, in start order. A run that is not `ok` is in no +median. A phase that did not complete prints `incomplete` instead of its +partial numbers. + +| run id | harness run tag | started (UTC) | exit | status | steady achieved /s · p99 ms | burst achieved /s · p99 ms | reconn achieved /s · p99 ms | slowdb achieved /s · p99 ms | load1 before | load1 during mean | lowest reclaimable-approx MiB (driver vm_stat; not MemAvailable) | +|---|---|---|---|---|---|---|---|---|---|---|---| +| d1-200k-fe1-async-r1 | dae611bb | 2026-09-17T19:29:56 | 0 | ok | 500.0 · 20.5 | 1558.5 · 1886.5 | 500.0 · 127.9 | 3.2 · 30348.2 | 7.53 | 4.98 | 62338 | +| d1-400k-fe1-async-r1 | c5fe5d1e | 2026-09-17T19:40:07 | 0 | ok | 499.4 · 18.5 | 1623.0 · 4025.9 | 500.0 · 779.5 | 1.7 · 29993.0 | 6.95 | 4.57 | 62537 | +| d1-500k-fe1-async-r1 | 375b5df2 | 2026-09-17T19:50:04 | 0 | ok | 499.4 · 13.2 | 1561.4 · 4744.3 | 500.0 · 887.0 | 1.8 · 30023.1 | 5.01 | 4.49 | 62600 | +| d1-400k-fe1-async-wt-r1 | 0256a9e8 | 2026-09-18T15:08:56 | 5 | FINDING (exit 5: ACK/commit divergence); not in medians | 112.6 · 40408.0 | 114.2 · 12133.6 | 134.9 · 56277.0 | 1.5 · 30223.8 | 4.67 | 3.96 | 61102 | +| d1-400k-fe2-async-r1 | 653cee64 | 2026-09-18T15:19:38 | 0 | ok | 499.3 · 37.5 | 1490.2 · 2607.7 | 500.0 · 939.8 | 2.4 · 29861.3 | 4.77 | 5.06 | 61580 | +| d1-400k-fe4-async-r1 | 150f1ca7 | 2026-09-18T15:29:46 | 0 | ok | 500.0 · 15.0 | 1662.7 · 1634.3 | 500.0 · 1101.6 | 3.6 · 28281.8 | 3.18 | 4.87 | 62354 | +| d1-400k-fe2-sync-r1 | e0b7127c | 2026-09-18T15:40:21 | 0 | ok | 500.0 · 32.8 | 1402.8 · 1848.9 | 500.0 · 1354.8 | 2.5 · 29708.4 | 6.57 | 3.92 | 63343 | +| d1-400k-fe1-async-r2 | 0e75a6bd | 2026-09-18T15:50:28 | 0 | ok | 499.3 · 25.1 | 1503.1 · 4397.1 | 500.0 · 1015.5 | 1.7 · 30514.1 | 3.53 | 3.98 | 62812 | +| d1-400k-fe2-async-r2 | 30ec5c4e | 2026-09-18T16:00:24 | 0 | ok | 499.3 · 23.2 | 1445.6 · 2463.8 | 500.0 · 869.6 | 2.6 · 29650.8 | 2.91 | 5.03 | 62495 | +| d1-400k-fe4-async-r2 | a953b44a | 2026-09-18T16:10:33 | 0 | ok | 500.0 · 16.1 | 1423.0 · 1971.9 | 500.0 · 1309.1 | 4.4 · 41425.4 | 4.32 | 5.61 | 60959 | +| d1-400k-fe2-sync-r2 | f229a920 | 2026-09-18T16:21:30 | 0 | ok | 500.0 · 8.8 | 1616.5 · 1666.7 | 500.0 · 966.3 | 2.4 · 29751.3 | 2.62 | 3.12 | 65361 | +| d1-400k-fe1-async-r3 | fce088fe | 2026-09-18T16:32:07 | 0 | ok | 499.4 · 8.6 | 1755.6 · 3357.1 | 500.0 · 705.2 | 1.7 · 30010.0 | 1.56 | 2.76 | 66918 | +| d1-400k-fe2-async-r3 | 5d224baa | 2026-09-18T16:42:06 | 0 | ok | 499.3 · 11.4 | 1521.7 · 3098.8 | 500.0 · 845.5 | 2.4 · 29710.0 | 4.13 | 4.31 | 64049 | +| d1-400k-fe4-async-r3 | 67dd5efc | 2026-09-18T16:52:13 | 0 | ok | 500.0 · 13.2 | 1504.2 · 1881.8 | 500.0 · 1381.3 | 4.4 · 41190.9 | 4.27 | 4.57 | 63017 | +| d1-400k-fe2-sync-r3 | f561b568 | 2026-09-18T17:03:10 | 0 | ok | 500.0 · 2559.8 | 1504.7 · 1812.2 | 500.0 · 1264.9 | 2.2 · 29369.3 | 5.62 | 3.92 | 63642 | +| d1-400k-fe2-async-blocks3-r1 | eaa07995 | 2026-09-18T17:13:34 | 0 | ok | 482.5 · 51.1 | 1600.0 · 1688.4 | 500.0 · 827.7 | 3.8 · 29334.1 | 3.82 | 4.53 | 61940 | +| d1-dense20k-fe1-async-r1 | 1e81170b | 2026-09-18T17:24:04 | 6 | NOT COMPLETED (exit 6); not in medians | 500.0 · 13.9 | 1643.0 · 2077.7 | 500.0 · 146.0 | 3.3 · 30516.7 | 3.95 | 3.95 | 64576 | +| d1-dense400k-fe1-async-r1 | 33bf557e | 2026-09-18T17:34:12 | 0 | ok | 497.0 · 3903.5 | 1454.6 · 5193.5 | 500.0 · 999.2 | 1.7 · 30247.2 | 4.34 | 5.9 | 61789 | +| d1-500k-fe2-async-r1 | e981913b | 2026-09-18T17:48:57 | 0 | ok | 499.3 · 17.4 | 1496.5 · 2179.2 | 500.0 · 867.0 | 2.3 · 29833.6 | 3.98 | 5.32 | 63704 | +| d1-500k-fe4-async-r1 | f5991b2e | 2026-09-18T17:59:20 | 0 | ok | 500.0 · 22.5 | 1468.0 · 1854.3 | 500.0 · 1814.6 | 2.3 · 26762.1 | 4.7 | 5.95 | 61964 | +| d1-500k-fe1-async-r2 | 8fedf497 | 2026-09-18T18:10:11 | 0 | ok | 499.4 · 26.7 | 1481.4 · 4869.6 | 500.0 · 865.0 | 1.8 · 30285.0 | 4.68 | 4.91 | 64763 | +| d1-500k-fe2-async-r2 | 37fd4356 | 2026-09-18T18:20:17 | 0 | ok | 499.3 · 28.1 | 1504.3 · 3083.1 | 500.0 · 885.0 | 2.3 · 29568.7 | 2.54 | 5.22 | 64014 | +| d1-500k-fe4-async-r2 | 5d2c0def | 2026-09-18T18:30:41 | 0 | ok | 500.0 · 13.3 | 1401.4 · 1978.1 | 500.0 · 1512.7 | 4.5 · 40746.6 | 3.23 | 4.52 | 63840 | +| d1-500k-fe1-async-r3 | 623b7af4 | 2026-09-18T18:42:05 | 0 | ok | 499.2 · 13.3 | 1550.0 · 3828.1 | 500.0 · 765.6 | 1.6 · 30029.2 | 4.13 | 4.49 | 67311 | +| d1-500k-fe2-async-r3 | 9a5fd144 | 2026-09-18T18:52:10 | 0 | ok | 499.3 · 8.1 | 1522.9 · 2289.4 | 500.0 · 903.9 | 2.1 · 29926.1 | 3.97 | 6.03 | 68106 | +| d1-500k-fe4-async-r3 | 5668ffea | 2026-09-18T19:02:35 | 0 | ok | 500.0 · 13.0 | 1514.5 · 1879.0 | 500.0 · 1449.2 | 2.4 · 27345.3 | 7.92 | 5.48 | 64919 | +| d1-400k-fe1-async-wt-r2 | 87d986a0 | 2026-09-18T19:13:24 | 0 | ok | 117.3 · 38296.3 | 107.2 · 39016.9 | 122.5 · 36511.8 | 1.8 · 30385.8 | 4.26 | 4.31 | 65832 | +| d1-400k-fe1-async-wt-r3 | d155ab56 | 2026-09-18T19:23:41 | 5 | FINDING (exit 5: ACK/commit divergence); not in medians | 117.9 · 37518.8 | 111.0 · 36759.9 | 133.2 · 32761.5 | 1.9 · 30299.1 | 2.53 | 4.0 | 68032 | +| d1-dense400k-fe1-async-r2 | c494d03d | 2026-09-18T19:35:07 | 0 | ok | 499.3 · 21.9 | 1527.0 · 4378.1 | 500.0 · 774.7 | 1.7 · 30019.4 | 2.8 | 4.25 | 66796 | +| d1-dense400k-fe2-async-r1 | 2c3228ca | 2026-09-18T19:49:05 | 0 | ok | 499.3 · 19.8 | 1466.6 · 2255.1 | 500.0 · 1008.2 | 2.5 · 29620.5 | 3.97 | 4.95 | 65862 | + +#### Phases outside the four tabulated ones (per run) + +| run id | status | phase | completed | achieved / offered (target) /s | ACK p50 / p99 / max ms | in artifact | +|---|---|---|---|---|---|---| +| d1-200k-fe1-async-r1 | ok | warm_up | true | 499.8 / 500.0 (500) | 1.1 / 378.0 / 644.5 | false | +| d1-400k-fe1-async-r1 | ok | warm_up | true | 499.9 / 500.0 (500) | 1.3 / 4.0 / 25.4 | false | +| d1-500k-fe1-async-r1 | ok | warm_up | true | 499.9 / 500.0 (500) | 1.3 / 4.0 / 15.7 | false | +| d1-400k-fe1-async-wt-r1 | FINDING (exit 5: ACK/commit divergence); not in medians | warm_up | true | 181.6 / 244.9 (500) | 9204.6 / 17245.4 / 17274.3 | false | +| d1-400k-fe2-async-r1 | ok | warm_up | true | 500.0 / 500.0 (500) | 2.5 / 89.6 / 150.5 | false | +| d1-400k-fe4-async-r1 | ok | warm_up | true | 489.3 / 500.0 (500) | 1.4 / 1225.6 / 1269.0 | false | +| d1-400k-fe2-sync-r1 | ok | warm_up | true | 499.9 / 500.0 (500) | 1.2 / 351.6 / 569.3 | false | +| d1-400k-fe1-async-r2 | ok | warm_up | true | 500.0 / 500.0 (500) | 1.3 / 3.1 / 16.4 | false | +| d1-400k-fe2-async-r2 | ok | warm_up | true | 500.0 / 500.0 (500) | 2.5 / 39.1 / 56.3 | false | +| d1-400k-fe4-async-r2 | ok | warm_up | true | 486.0 / 500.0 (500) | 2.0 / 1771.6 / 1796.1 | false | +| d1-400k-fe2-sync-r2 | ok | warm_up | true | 499.9 / 500.0 (500) | 1.2 / 372.3 / 595.9 | false | +| d1-400k-fe1-async-r3 | ok | warm_up | true | 500.0 / 500.0 (500) | 1.1 / 3.3 / 22.5 | false | +| d1-400k-fe2-async-r3 | ok | warm_up | true | 499.9 / 500.0 (500) | 2.2 / 6.4 / 21.0 | false | +| d1-400k-fe4-async-r3 | ok | warm_up | true | 488.3 / 500.0 (500) | 1.5 / 1295.2 / 1331.9 | false | +| d1-400k-fe2-sync-r3 | ok | warm_up | true | 499.8 / 500.0 (500) | 1.3 / 395.2 / 646.6 | false | +| d1-400k-fe2-async-blocks3-r1 | ok | warm_up | true | 500.0 / 500.0 (500) | 2.4 / 8.5 / 17.1 | false | +| d1-dense20k-fe1-async-r1 | NOT COMPLETED (exit 6); not in medians | warm_up | true | 499.8 / 500.0 (500) | 1.2 / 296.1 / 490.2 | false | +| d1-dense400k-fe1-async-r1 | ok | warm_up | true | 500.0 / 500.0 (500) | 1.3 / 3.4 / 20.8 | false | +| d1-dense400k-fe1-async-r1 | ok | dense_cadence | true | 415.2 / 500.0 (500) | 1.1 / 381.0 / 1042.6 | false | +| d1-500k-fe2-async-r1 | ok | warm_up | true | 500.0 / 500.0 (500) | 2.4 / 37.6 / 86.8 | false | +| d1-500k-fe4-async-r1 | ok | warm_up | true | 499.4 / 500.0 (500) | 1.6 / 461.4 / 763.4 | false | +| d1-500k-fe1-async-r2 | ok | warm_up | true | 499.9 / 500.0 (500) | 1.3 / 4.7 / 27.9 | false | +| d1-500k-fe2-async-r2 | ok | warm_up | true | 499.9 / 500.0 (500) | 2.4 / 31.0 / 82.7 | false | +| d1-500k-fe4-async-r2 | ok | warm_up | true | 498.1 / 500.0 (500) | 1.3 / 371.5 / 604.8 | false | +| d1-500k-fe1-async-r3 | ok | warm_up | true | 500.0 / 500.0 (500) | 1.2 / 3.7 / 28.4 | false | +| d1-500k-fe2-async-r3 | ok | warm_up | true | 499.9 / 500.0 (500) | 2.5 / 10.5 / 22.6 | false | +| d1-500k-fe4-async-r3 | ok | warm_up | true | 499.5 / 500.0 (500) | 1.3 / 441.2 / 684.5 | false | +| d1-400k-fe1-async-wt-r2 | ok | warm_up | true | 150.5 / 197.8 (500) | 8109.3 / 16762.1 / 16793.1 | false | +| d1-400k-fe1-async-wt-r3 | FINDING (exit 5: ACK/commit divergence); not in medians | warm_up | true | 148.5 / 197.7 (500) | 13054.0 / 17673.0 / 17704.8 | false | +| d1-dense400k-fe1-async-r2 | ok | warm_up | true | 499.9 / 500.0 (500) | 1.2 / 11.8 / 138.6 | false | +| d1-dense400k-fe1-async-r2 | ok | dense_cadence | true | 409.7 / 500.0 (500) | 1.1 / 391.6 / 811.8 | false | +| d1-dense400k-fe2-async-r1 | ok | warm_up | true | 499.9 / 500.0 (500) | 2.4 / 87.1 / 146.3 | false | +| d1-dense400k-fe2-async-r1 | ok | dense_cadence | true | 399.4 / 500.0 (500) | 1.2 / 352.7 / 669.8 | false | + +### Capacity-evidence verdicts (#447) + +Validator inputs (`validator.forecast_used` shares/s, +`validator.ack_p99_limit_used_milliseconds`): forecast peak 2000 shares/s, ACK +p99 limit 1000 ms. Runs in the medians only. + +| Window | fe | repl | plan | runs | artifact written | valid | slowest artifact phase /s (min–max) | worst artifact ACK p99 ms (min–max) | suggested forecast (min–max) | suggested ACK p99 limit ms | +|---|---|---|---|---|---|---|---|---|---|---| +| 200k | 1 | async | d1 | 1 | 1 | 0 | 3.2 | 30,348 | 1.58 | 15,000 | +| 400k | 1 | async | d1 | 3 | 3 | 0 | 1.7 | 29,993–30,514 | 0.83–0.85 | 15,000 | +| 400k | 2 | async | d1 | 3 | 3 | 0 | 2.4–2.6 | 29,651–29,861 | 1.19–1.32 | 15,000 | +| 400k | 4 | async | d1 | 3 | 3 | 0 | 3.6–4.4 | 28,282–41,425 | 1.78–2.22 | 15,000 | +| 400k | 2 | sync | d1 | 3 | 3 | 0 | 2.2–2.5 | 29,369–29,751 | 1.10–1.23 | 15,000 | +| 400k | 2 | async | d1 +3 blocks | 1 | 1 | 0 | 3.8 | 29,334 | 1.92 | 15,000 | +| 400k | 1 | async | d1 +dense(15) | 2 | 2 | 0 | 1.7 | 30,019–30,247 | 0.82–0.84 | 15,000 | +| 400k | 2 | async | d1 +dense(15) | 1 | 1 | 0 | 2.5 | 29,621 | 1.23 | 15,000 | +| 400k | 1 | async | d1 +wal_sync_method=fsync_writethrough | 1 | 1 | 0 | 1.8 | 38,296 | 0.92 | 15,000 | +| 500k | 1 | async | d1 | 3 | 3 | 0 | 1.6–1.8 | 30,023–30,285 | 0.81–0.89 | 15,000 | +| 500k | 2 | async | d1 | 3 | 3 | 0 | 2.1–2.3 | 29,569–29,926 | 1.06–1.17 | 15,000 | +| 500k | 4 | async | d1 | 3 | 3 | 0 | 2.3–4.5 | 26,762–40,747 | 1.14–2.25 | 15,000 | + +First entry of `validator.verdict.error_chain` per refused artifact, with its +counts folded to `N`: + +- `capacity run did not acknowledge every offered valid share: offered=N + acknowledged=N rejected=N` — 27 runs + +Runs outside the medians carry no row above. Their `validator` block says: + +- `d1-400k-fe1-async-wt-r1`: artifact_written = true, valid = false +- `d1-dense20k-fe1-async-r1`: artifact_written = false, withheld_reason: `the + run aborted: 20 submit(s) offered before the dense_cadence phase were still + outstanding 25.0 s later, so its 0 ms delay could not be applied without + them finishing under it` +- `d1-400k-fe1-async-wt-r3`: artifact_written = true, valid = false + +### Runs outside the medians + +**Runs that completed with a finding.** + +Exit 4, 5, 7 and 8 runs completed every phase but carry a finding, so none of +their numbers is in a median above. They are in the per-run table, flagged. + +| run id | window | fe | repl | plan | started (UTC) | exit | meaning | finding, from the report | +|---|---|---|---|---|---|---|---|---| +| d1-400k-fe1-async-wt-r1 | 400k | 1 | async | d1 | 2026-09-18T15:08:56 | 5 | ACK/commit divergence | ACK/commit divergences: 0; unknown-outcome commits: 0; mid-run no-response commits: 1; harness-bug rejections: 0 | +| d1-400k-fe1-async-wt-r3 | 400k | 1 | async | d1 | 2026-09-18T19:23:41 | 5 | ACK/commit divergence | ACK/commit divergences: 0; unknown-outcome commits: 0; mid-run no-response commits: 2; harness-bug rejections: 0 | + +**Runs that did not complete.** + +These runs did not complete (harness exit 2, 3 or 6, an exit the harness does +not document, a run the driver ended, or a phase with `completed: false`). +None of their numbers is in any median. The reason text is the report's own, +verbatim except that terminal colour codes are removed; `(argv)` marks a +configuration value the reduced report does not carry, read from the driver's +recorded command line instead. + +| run id | window | fe | repl | plan | started (UTC) | exit | meaning | ended_by | phases completed | reason, from the report | frontend log refusals (kind × lines) | why this script excluded it | +|---|---|---|---|---|---|---|---|---|---|---|---|---| +| d1-dense20k-fe1-async-r1 | 20k | 1 | async | d1 | 2026-09-18T17:24:04 | 6 | aborted | harness | warm_up, steady_state, burst, reconnect, slow_database | aborted: `20 submit(s) offered before the dense_cadence phase were still outstanding 25.0 s later, so its 0 ms delay could not be applied without them finishing under it` | none | harness exit 6 (aborted); report.aborted is set | + +### Replication premise (#447) + +One short probe ran before the matrix, as in #271: 2 frontends, 200 sessions, +the short plan at 50 shares/s, the 20k window. + +| Run id | Exit | Declared | Observed at entry | Observed after load | Agreed | Share difficulty agreed | +|---|---|---|---|---|---|---| +| d1-premise-fe2-async-short50 | 0 | async | async | async | true | true | + +The synchronous probe was not repeated. The premise is checked inside every +run, and it agreed in all of them: 34 runs declared `async` and 3 declared +`sync`; each observed its declared mode at entry and again after the load, and +every run's share difficulty agreed. + +### Harness observations (#447) + +Each of these came out of a run. Only the first is changed here. + +1. **Frontend CPU was read in the wrong unit on Apple silicon (fixed).** + `measure.rs` divided `proc_pid_rusage`'s `ri_user_time + ri_system_time` + by 1e9 as if they were nanoseconds. They are Mach absolute-time ticks, + 125/3 ns each on this host, so every `cpu_seconds` and `cpu_cores_mean` + was about 42 times too small. The change that adds this part scales by + `mach_timebase_info` and adds two tests; one of them burns a known amount + of CPU and fails against the old arithmetic. Nothing compared the reading + with CPU really spent before. Linux hosts, including #271's, were never + affected. +2. **The memory floor is skipped without a word on macOS.** When + `MemAvailable` cannot be read the 1 s check does nothing, the run proceeds + unguarded, and the side report's `min_mem_available_kib` is `null`. The + `null` is honest; the silence is not. A refusal at entry, or a line in the + report saying the floor did not operate, would be. +3. **`pg_stat_statements` is never loaded on macOS.** The harness preloads it + when `pg_stat_statements.so` exists in `pkglibdir`; Homebrew ships + `pg_stat_statements.dylib`. Every lock-statement cell in this part is + `n/a` for that reason. +4. **The default `TMPDIR` on macOS makes the managed cluster unstartable.** + PostgreSQL logs `Unix-domain socket path ... is too long (maximum 103 + bytes)` and the harness exits 2 with `pg_ctl failed`, without that line, + because the cluster root and its log are removed with the failure. +5. **The dense-cadence attribution keys on a message the server no longer + sends.** See [Dense cadence at D1](#dense-cadence-at-d1). +6. **PostgreSQL warns `there is already a transaction in progress` in every + D1 run's frontend log.** That is `BEGIN` arriving on a connection that + already has a transaction open. It appears 10,303 times across 33 of the 33 + D1 runs: 8,431 in `slow_database` and 1,866 in the drain after it, where + appends are being cancelled at the share-commit deadline (`share commit + deadline passed before COMMIT was sent`). The other 6 are elsewhere. 5 are + in `warm_up` of `d1-400k-fe1-async-wt-r1`, `steady_state` of + `d1-400k-fe1-async-wt-r2`, phases that logged the same cancellations. 1 is + in `steady_state` of `d1-dense400k-fe1-async-r1`, a phase that logged no + such cancellation, so the deadline is not the only trigger. That pattern + suggests a cancelled attempt + leaving its transaction open for the connection's next user. Every run in + which it appears reconciled exactly, so no effect on the ledger was + observed. Its cause and its consequences were not investigated here, and + nothing in the repository mentions it. + +### Reproducing a run (#447) + +Build both binaries in release mode from `a1937054`, in a clean checkout that +nothing else modifies. Then run the harness with PostgreSQL 16 server binaries +and a short `TMPDIR`: + +```sh +cargo build --locked --release -p qbit-prism-load -p qbit-prism-server +TMPDIR=/tmp/d1t target/release/qbit-prism-load \ + --server-bin target/release/qbit-prism-server \ + --pg-bin-dir <pg-bin> \ + <args> \ + --out <out> +``` + +Every run carries the same tail, written `<common>` below: `--sessions 2000 +--plan d1 --forecast-peak-shares-per-second 2000 --ack-p99-limit-ms 1000 +--min-mem-available-mib 6144`. + +| Run ids | `<args>` | +|---|---| +| `d1-premise-fe2-async-short50` | `--frontends 2 --sessions 200 --window-shares 20000 --replication async --plan short --rate 50 --forecast-peak-shares-per-second 2000 --ack-p99-limit-ms 1000 --min-mem-available-mib 6144` | +| `d1-{200,400,500}k-fe1-async-r1` | `--frontends 1 --window-shares {200000,400000,500000} --replication async <common>` | +| `d1-400k-fe{1,2,4}-async-r{1,2,3}`, `d1-500k-fe{1,2,4}-async-r{1,2,3}` | `--frontends {1,2,4} --window-shares {400000,500000} --replication async <common>` | +| `d1-400k-fe2-sync-r{1,2,3}` | `--frontends 2 --window-shares 400000 --replication sync <common>` | +| `d1-400k-fe1-async-wt-r{1,2,3}` | as `d1-400k-fe1-async-r*`, with `--pg-bin-dir` pointing at the wrapper directory described under [The flush control](#the-flush-control) | +| `d1-400k-fe2-async-blocks3-r1` | `--frontends 2 --window-shares 400000 --replication async --scheduled-blocks 3 --keep-artifacts <common>` | +| `d1-dense20k-fe1-async-r1` | `--frontends 1 --window-shares 20000 --replication async --cadence dense --scheduled-blocks 15 <common>` | +| `d1-dense400k-fe{1,2}-async-r*` | `--frontends {1,2} --window-shares 400000 --replication async --cadence dense --scheduled-blocks 15 <common>` | +| `d1-20k-fe1-async-r{1,2,3}` | `--frontends 1 --window-shares 20000 --replication async <common>`, which is #271's `w11-20k-fe1-async-r*` argument list | +| `d1-20k-fe1-async-build5d0042f6-r{1,2,3}` | the same, with both binaries built from `5d0042f6` in its own clean worktree | +| `d1-20k-fe1-async-cross-{newharness-oldserver,oldharness-newserver}-r1` | the same, with `--server-bin` from the other build and `--allow-unverified-server-revision`; both were refused at startup, see [The build control](#the-build-control) | + +Runs were driven one at a time, detached from any interactive session, with a +60 s cooldown and a whole-run ceiling of 3,600 s that no run reached. The +longest took 790 s. The exit codes are those in +[Reproducing a run](#reproducing-a-run) above. + +### What was not measured (#447) + +- **The #291 rehearsal host.** It had not been named. Every rate in this part + belongs to a laptop whose commits do not force the drive cache, and the + flush control shows how much that is worth. These runs settle that the + production window runs, reconciles and lands blocks on this build. They do + not settle what rate the rehearsal host sustains. +- **Synchronous replication beyond one configuration,** and on a standby that + does not share the primary's disk. The managed standby is on the same SSD + and, like the primary, does not force a flush, so the absence of a + synchronous cost here says little about a deployment. +- **Spread for the found-block run,** which is one run. +- **200k beyond one run at one frontend.** It was a blocked size to clear, not + a configuration to characterise. +- **A reduced-rate dense run.** It was planned as the fallback if the phase + could not start at D1. The phase started at D1 at the production window, so + the fallback was not needed. +- **Which of the server and the harness lowered the 2,000 shares/s ceiling** + between `5d0042f6` and `a1937054`. Both cross runs were refused, correctly; + see [The build control](#the-build-control). +- **#271's build at the production windows.** It cannot run them. diff --git a/docs/prism-window-reference-foundation.md b/docs/prism-window-reference-foundation.md new file mode 100644 index 000000000..1c5ad1df0 --- /dev/null +++ b/docs/prism-window-reference-foundation.md @@ -0,0 +1,402 @@ +# B273 window reference foundation + +This local slice adds the schema and authenticated reader approved after the +[initial plan](prism-prepared-window-reference-plan.md). It does **not** enable +compact prepared jobs or finish issue273. Authorization covers B-owned code +only and is not A/C owner approval. + +This branch includes current `3.x.x` at +`d39cf621ce25fea4d5a1e86b02cf6c3c66949ce5` additively: `48a5923` includes +PR313's merge `0af13ec`, `c6bce3a` includes PR297's merge `82a543d`, and +`b480ea1` includes PR319's merge `d39cf62`. +PR313 and the #264 design dependency are no longer open blockers. The earlier +dependency updates remain in history. The original PR313 worktree is untouched; +this slice adds no release bump. + +The API remains compatible with [PR297's approved bed6ad8 contract](https://github.com/Qbit-Org/qbit-mining-bootstrap/blob/bed6ad8888d6bd58ec9fc96fed8b5ba409b63cee/docs/prism-coordinator-refactor/window-ref.md). +The [posted user signoff](https://github.com/Qbit-Org/qbit-mining-bootstrap/pull/297#issuecomment-5638877264) +authorizes the narrower lease: revision changes are bridged only while the +balance digest is unchanged; changed balances end the lease immediately. +It does not waive affected-work/replacement-delivery qualification or establish +full miner parity. Final resume/submit and leased-candidate wiring remain +unimplemented here, pending the exact A265 interfaces listed below. + +## Implemented contract + +- Public `WindowRef`, `ShareRange`, `Window`, `BalanceSource` and `WindowError` + are exported from `ledger`. Digests serialize as exactly 64 lowercase hex + characters. Invalid sequence bounds and counts fail before database access. +- `Ledger::read_window(&WindowRef, BalanceSource)` uses one primary + `REPEATABLE READ READ ONLY` transaction. It reads revision and balances, + probes endpoints, then reads ascending keyset pages of at most 4096 rows. + Decode and hash work runs in `spawn_blocking`. Native share JSON is streamed + into SHA-256 without allocating a serialized copy of the entire window. +- A `BlockingDrop` guard transfers accumulated page state and vectors awaiting + commit to a blocking thread when the reader is cancelled or fails. A page + task's abandoned result carries the same guard. This implements the later + [cancellation note on #273](https://github.com/Qbit-Org/qbit-mining-bootstrap/issues/273#issuecomment-5639213901) + without changing `read_window`'s API. After successful return the caller owns + the window and its off-thread cleanup obligation. +- Current balances are digest checked; as-issued balances are decoded from + the immutable snapshot identified by their semantic digest. Digest + validation uses a bytewise `(order_key, recipient_id, p2mr_program_hex)` + clone, while each path returns the source vector order unchanged so legacy + persisted snapshots retain their exact balance ordering. The returned + revision is current, never an implicit replacement for a job's original + revision. +- Missing endpoints/count mismatches, changed current balances, missing or + corrupt as-issued balances, native digest mismatches, SQL failures, and decode + failures keep their specified error variants. An empty reference never + queries share history. There is no partial-window result or cache-miss mapping. +- Migration008 creates the exact no-window/empty/range column states, + lowercase digest checks, immutable template/balance bytea stores, and job + digest indexes. Updates changing blob keys or contents are rejected; deletion + remains available to the future coordinated GC. Legacy payloads and issued + top-level `extranonce1` are untouched. +- The migration runner adds 008 to the landed PR319 membership approach. + It applies 008 even when 009 is already installed, and 009 when only 008 + is installed. No 006, 007, candidate or audit reader implementation was + imported. Migration009 is unchanged from base. Existing migration assertions + verify the full applied version set and exact preservation of every original + job field, with only 008's seven new null columns added. +- The later B-owned `Ledger::payout_state()` returns a `PayoutState` containing + current revision and current balance digest from one repeatable-read primary + snapshot. It retains `payout_revision`'s fatal/read-only guards, reads no share + history or as-issued blob, and decodes/hashes/drops balances in a blocking + closure. It has no private deadline or implicit eligibility decision. + `WindowRef`, `Window`, `WindowError` and `read_window` signatures are unchanged. +- B's tests use the shared `qbit_prism_test_gate` introduced by merged PR322. + All 14 regular B database cases are in `test/prism-gated-tests.txt`; explicit + scale qualifications use the required-input gate. Test fixture + modulo expressions follow the current pinned toolchain's Clippy rules. + +The bytea column contracts are `qbit_prism_templates(template_sha256, +template_bytes)` and `qbit_prism_balance_snapshots(prior_balances_digest, +balances)`. The latter contains native serialized `CarryForwardBalance` JSON; +its key is `qbit_prism::prior_balances_digest`, **not** the hash of those bytes. +Future writers must authenticate content and compare immutable conflicts in +the same save/repair transaction. Schema immutability alone is not that writer. + +## Post-signoff qualification + +The coherent-state follow-up was tested on the additive PR313 dependency +`91294e7`, with Rust 1.89.0 and disposable PostgreSQL 16.14 discovered through +`pg_config --bindir`. Workspace all-target compilation, formatting and +`git diff --check` passed. The regular database command below passed 223 tests: + +| Suite | Passed | +| --- | --- | +| Server library, including PR313's nine restored D2 coordinator cases | 142 | +| `ledger_postgres` / `migration_rollback` | 29 / 1 | +| `readiness_rpc` / `stratum_protocol` | 5 / 22 | +| `window_read_oracle` | 8 | +| `window_reference` | 16; the two explicit qualification tests remained ignored | + +The four new `window_reference::payout_state` database cases passed both alone +and in the regression run: + +- Revision-only change preserves the balance identity; changed balances alter + it, without requiring share history or the as-issued store. +- A test-only database view gates the revision result before the balance + statement starts. Another transaction commits a new revision and balance set + during that wait; the reader returns the old coherent pair, and the next read + sees the new pair. Blocking the balance statement itself is insufficient to + distinguish transaction isolation from that statement's own snapshot. +- Empty balances remain valid; fatal state, read-only configuration, missing + current-balance storage and numeric decode overflow remain truthful errors. +- SQLSTATE 57014 and cancellation during the balance query release the only + connection, which a subsequent state read successfully reuses. + +The coherence test was checked with a temporary local mutation replacing +`REPEATABLE READ` with `READ COMMITTED`. It failed with the expected +`payout eligibility mixed two MVCC snapshots` error. The mutation was restored +in a `finally` cleanup; the four focused cases were then rerun on the real code. + +The first compile caught the new test fixture's `u64` balance field; it was +corrected to the existing `CarryForwardBalance.balance_sats: i128` contract +before the passing runs. Existing as-issued signed-equality and miner assertions +were not changed. The new tests exercise the helper, not runtime lease +revocation or candidate dispatch. No credit-policy success is inferred from +these regression results. + +This follow-up did not rerun the ignored 400k/500k reader, actual external 009 +SQL test, or issued-dependency suite; their earlier evidence remains below. +Full refresh/resume, large-template/WAL, async standby, live-qbit, GC/enqueue +races and full deep-review remain unrun for this integration. No production +access, push or public approval was performed. + +## Current-base draft checkpoint + +At `a6e3482`, the base was `3.x.x` at `82a543d`; Rust is now the repository's +installed pin 1.98.1, with disposable PostgreSQL 16.14 located through +`pg_config --bindir`. +The base merge retained upstream versions of every conflicted file; no +foundation changes existed in those files. No release metadata was changed. + +The selected real-database regression run passed 225 tests: 144 library, +29 ledger, 1 migration rollback, 5 readiness, 22 Stratum, 8 window oracle and +16 window reference. Three explicit qualifications were ignored in that run. +All 13 regular window-reference database cases recorded `executed`; the shared +manifest checker passed against the B-only expected/actual subset. That scoped +check is not a claim that the entire CI execution manifest has been qualified. +Workspace all-target compilation and Clippy with `-D warnings` passed. Existing +miner assertions are unchanged. The two new guard unit tests prove cancelled +payloads drop on a different thread and successful handoff preserves ownership. + +The full 400k/500k reader and issued-dependency evidence below remains +historical; it was not rerun for this draft checkpoint. The landed 009 SQL is +now covered by the regular combined-base suite below. Runtime +refresh/resume, candidate-aware GC, large-template/WAL, async standby and +cutover qualification remain incomplete. Full deep-review is deferred until +after draft publication; this document does not report missing lanes as green. + +### Combined base with landed PR319 + +After the additive `b480ea1` merge of `3.x.x` at `d39cf62`, the regular suite +passed **236 tests**: 144 library, 37 ledger, 1 migration rollback, 6 readiness, +1 Stratum admission, 22 Stratum protocol, 8 window oracle and 17 window +reference. Three qualifications remained ignored: the admission capacity +measurement and B's two explicit scale measurements. All 14 regular B database +cases recorded execution and passed the scoped manifest check. +Formatting, workspace/all-target Clippy with `-D warnings`, the direct gate +environment-read check (135 Rust files), and `git diff --check` passed after +the combined-base test adjustments. The earlier all-target `cargo check` +passed before this merge; all-target Clippy compiled the final combined base. + +The landed 009 SQL now runs in a regular B test in both orders with 008, using +the actual membership runner and restart. The first combined run exposed +PR319's whole-row comparison expecting no new columns; its expected result now +includes exactly 008's seven null columns and still compares every original +field, including payload/extranonce/expiry. Both existing migration tests now +require `[2, 3, 4, 5, 6, 8, 9]`; no assertion was reduced to a maximum version or +subset. Migration009 and the session/candidate implementation are unchanged. + +```sh +PRISM_TEST_GATE_MANIFEST=/tmp/window-reference-gates.txt \ + bash test/prism-native-tests.sh cargo-args --locked -p qbit-prism-server \ + --lib --test window_reference --test readiness_rpc --test stratum_protocol \ + --test stratum_admission_postgres --test window_read_oracle \ + --test ledger_postgres --test migration_rollback +``` + +### Late cancellation at 400k + +The explicit `cancellation::cancelling_400k_read_after_96_pages_measures_runtime_stall` +test ran at `a6e3482` on disposable PostgreSQL 16.14 and Rust 1.98.1. It loads the +production-shaped 400,000-row fixture, streams 96 complete pages (393,216 +shares), then gates the next page's projected share ID with a transaction-level +advisory lock. A single-thread Tokio runtime aborts and joins the reader while +a 1 ms timer measures its scheduling gaps, then verifies the sole reader +connection can be reused. + +For this test-only connection, bitmap/sequence scans and sorting are disabled +and `EXPLAIN` must contain only streaming nodes. The ordered view and this +check prevent a sort from evaluating the gate before the first page has been +returned. An earlier unconstrained sample was rejected for that reason; it is +not evidence of late cancellation. This is a cleanup measurement, not a +production query-plan or read-throughput benchmark. + +The validated sample reported 27 microseconds from abort to joined cancellation, +a maximum 5,418 microsecond gap for the 1 ms ticker, and successful connection +recovery. Test wall time was 29.00 s including fixture setup/read/cleanup; +`/usr/bin/time -l` around the command reported 31.97 s and 380,977,152 bytes +maximum RSS. That RSS is command-level (including build/fixture effects), not +per-phase reader residency. No new timing threshold, full 30 s resume result, +WAL result, or 500k cancellation qualification is claimed. + +```sh +PRISM_TEST_GATE_MANIFEST=/tmp/window-cancel-gate.txt \ + bash test/prism-native-tests.sh cargo-args --locked -p qbit-prism-server \ + --test window_reference \ + cancellation::cancelling_400k_read_after_96_pages_measures_runtime_stall \ + -- --ignored --exact --nocapture +``` + +## Initial foundation qualification (d7526fc) + +Rust 1.89.0, PostgreSQL 16.14 discovered through `pg_config --bindir`, macOS. +Every database run used the existing disposable-cluster runner; no production +database was accessed. The default Cargo 1.84 remains unsuitable for the +locked dependencies, so no lockfile or global toolchain change was used. + +| Check | Result | +| --- | --- | +| Workspace all-target `cargo check --locked` | Passed | +| `cargo fmt --all -- --check`, `git diff --check` | Passed | +| Server library tests | 133 passed, including 4 new reference unit tests | +| `ledger_postgres` | 29 passed | +| `migration_rollback` | 1 passed | +| `readiness_rpc` | 5 passed, assertions unchanged | +| `stratum_protocol` | 22 passed, assertions unchanged | +| `window_read_oracle` | 8 passed | +| `window_reference` | 12 passed: 9 database cases and 3 existing fixture checks; 2 explicit qualifications separately gated | +| Ignored `issued_job_dependency` suite | All 7 ran and passed | +| Actual PR319 migration009, 008 then 009 and 009 then 008 | Explicit test ran and passed | +| Explicit production-shaped reader test | 400k and 500k passed in both debug and optimized builds | + +The PostgreSQL cases cover native row equality across page boundaries and +filtered sequence gaps; signed audit equality with original balances after a +revision change; empty-range reads without a share table; malformed stored +balances and numeric decode overflow; missing history, count and digest errors; +SQLSTATE 57014 timeout and cancellation cleanup with one connection; concurrent +revision/balance/history changes across a blocked read; all 64 partial-null +column states; immutable blobs; old-schema upgrade and repeated application. + +The first broader run found the existing migration test's expected maximum +version 5. After replacing it with the exact applied set `[2, 3, 4, 5, 8]`, +the full selected regression run passed. No miner assertion was relaxed. + +Reproduce the regular database run with: + +```sh +RUSTUP_TOOLCHAIN=1.89.0 bash test/prism-native-tests.sh cargo-args \ + --locked -p qbit-prism-server --lib --test window_reference \ + --test readiness_rpc --test stratum_protocol --test window_read_oracle \ + --test ledger_postgres --test migration_rollback +``` + +The external 009 fixture was read from PR319 commit +`3a7ba469c9318791d5fb366359ca406397f3065b`. It was not copied into the repository. +The historical reproduction at `d7526fc` set `PRISM_TEST_MIGRATION_009` +to that reviewed SQL file and ran (current HEAD uses the landed regular test): + +```sh +RUSTUP_TOOLCHAIN=1.89.0 bash test/prism-native-tests.sh cargo-args \ + --locked -p qbit-prism-server --test window_reference \ + reviewed_009_sql_coexists_with_008_in_both_orders -- --ignored --exact +``` + +This proves SQL coexistence under the migration lock and the real 008 runner, +not compilation or runtime integration of the entire PR319 branch. + +### Large reader measurement + +The separate `production_shaped_reader_at_400k_and_500k` test uses the landed +production-shaped fixture and checks every reconstructed native row. It times +only the awaited reader after fixture load and expected-digest construction. +Returned-share counts exclude revision/balance queries and endpoint probes; +page query counts are not separately instrumented. + +| Build | Shares / returned share rows | Reader wall time | +| --- | --- | --- | +| Debug | 400,000 / 400,000 | 32.692 s | +| Debug | 500,000 / 500,000 | 21.078 s | +| Optimized (`--release`) | 400,000 / 400,000 | 16.144 s | +| Optimized (`--release`) | 500,000 / 500,000 | 24.453 s | + +These are single local samples, not latency percentiles or an end-to-end +deadline result. Debug 400k exceeded 30 seconds; even the optimized results +leave material read/build/permit budgeting to validate in integration. No +latency scaling conclusion follows from the non-monotonic debug measurements. +The native digests matched between builds: + +- 400k: `d9a8ea1fd49b977193b218eb6ce8f7dac36c046df4145e54cb066d43b6322e75` +- 500k: `91b976807eb19516ac6d723153297d2b42cd871b0ed047501f0a13fd74cc8791` + +The debug command was wrapped in macOS `/usr/bin/time -l`: 194.26 seconds +total wall time and 635,486,208 bytes maximum resident set size. That is a +command-level measurement including fixture creation, the expected vectors, +and reconstructed vectors, not per-phase reader RSS or retained production +memory. The optimized test took 128.29 seconds excluding its 78-second build; +optimized RSS was not measured. Neither run measured WAL or standby replay. + +```sh +RUSTUP_TOOLCHAIN=1.89.0 bash test/prism-native-tests.sh cargo-args \ + --locked --release -p qbit-prism-server --test window_reference \ + production_shaped_reader_at_400k_and_500k -- --ignored --exact --nocapture +``` + +## Remaining integration hooks + +1. Obtain A265's concrete interfaces below and A/C integration review. User + signoff on the narrowed contract has already been given; it is not a blocker. + A's `audit::read_range` is unchanged. No placeholder builder version or + candidate adapter exists here. +2. Switch `StoredPrepared`/`Prepared`, WorkLedger and refresh to references and + stored policy/signer/hash inputs. Authenticate fetched template bytes before + decoding, compare payload/reference columns, and preserve exact original + signed hashes. Wire coherent `payout_state` through WorkLedger/SubmitLedger + admission, comparing against the issued digest with publication/lease/expiry + rechecks after waits and the unchanged transaction revision fence. State-read + failure remains an error; known mismatch is ineligible. Define the legacy + prepared-row miss at that runtime cutover. +3. Insert/reuse template, balance snapshot and prepared job atomically; extend + PR313 cold repair to restore all dependencies and the child with immutable + conflict checks and the child's original absolute expiry. Add transactional, + job/blob GC under `SETTLEMENT_LOCK` then `ORDER_LOCK`: expire at most 4096 + jobs beyond a qualified post-expiry grace, prune their unreferenced templates, then sweep balances unreferenced by + any job or nonterminal leased candidate. The balance sweep also runs when + zero jobs expire, so candidate-retained orphans disappear after terminal + completion. See the plan's GC/repair/enqueue race matrix. This foundation + does not change `save_job` or runtime GC. +4. Add shared reader capacity, build permits, bounded singleflight, original + deadline, and per-waiter authority/expiry checks in callers. The reader has + no permits or private timeout by design. Release full rebuilt windows inside + the blocking closure before resumed jobs enter sessions, using A265's + candidate integration. Preserve the original synthetic bootstrap share. +5. Qualify real 400k refresh plus frontend-A-to-B resume, 500k headroom, a large + valid template, all refresh JSONB values below 1 MB, and measured refresh WAL + against the 5 MB target. Large reader tests do not establish those gates. + Record dedicated asynchronous standby results, per-phase RSS/read counts, + retained local generations, affected work, and replacement delivery latency. + +### Exact A265 dependencies at this checkpoint + +Inspection of `3.x.x` at `d39cf62` and A265's open +[PR325 at ff74abb](https://github.com/Qbit-Org/qbit-mining-bootstrap/pull/325) +(`ff74abb0da65f15ddcd25689328a075f7ab2a1b4`) still finds +`Candidate.bundle: AuditBundle`, an optional suffix, no candidate window +columns, no `AUDIT_BUILDER_VERSION`, and no +`codec::witness_merkle_leaves_from_block`. PR325 supplies canonical import/read +work and explicitly leaves enqueue for A's next PR; it is not a missing B +import prerequisite. #264 is closed and PR297 is merged, so neither is a design +blocker. The following remain #265 implementation handoffs, not unanswered +design objections or a request for another user signoff: + +| A-owned interface | Needed by B | +| --- | --- | +| `qbit_prism::AUDIT_BUILDER_VERSION: u16` and versioned frozen vectors | Persist/check the actual builder version before reconstructing prepared work; no B-local substitute constant. | +| Slim candidate fields: `window`, `found_block`, `payout_policy`, nested optional `ctv { direct_floor_sats, settlement_config, fanout_fee_policy }`, `audit_builder_version`, `signer_keys { manifest_key_hex, ledger_key_hex }`, `bootstrap_share`, `leased`, required `coinbase_suffix_hex`, original `payout_revision`, `job_id`, `deferred_share`, block identity/digest and bytes | Agree concrete shared Rust types/exports and construction API. B must retain these original inputs in the resumed job after releasing its full window/body; the old owned-bundle submit constructor cannot do that. | +| Migration007's six outbox window columns, especially indexed `window_prior_balances_sha256`; authenticated `leased` flag and nonterminal-state predicate | GC must join actual typed references. Current states are `pending`, `submitted`, `abandoned`; A must confirm the retention predicate with 007, including claimed/retrying pending rows. Do not invent a SQL `leased` column when the record only specifies a JSON flag. | +| Enqueue under `ORDER_LOCK`, receiving as-issued balances for digest-checked re-insertion and probing the share prefix | `406b273` answers the GC-first ordering case: reinsert balances before committing the reference, and enqueue/alert even if the share prefix is unexpectedly absent. A's concrete Rust method/input and immutable-conflict behavior are still needed; B does not invent them. | +| Leased candidate dispatch: submit stored bytes first; rebuild with `AsIssued` and land before any terminal result, whether active or inactive; null inactive results and changed-balance landing failures remain recoverable | Required by the landed follow-up record while preserving issued revision. B's current independent candidate fences remain unchanged until A integration. | +| Parts-based claim/landing and `codec::witness_merkle_leaves_from_block(&[u8]) -> Result<Vec<String>>` | Candidate reconstructs from immutable original inputs without forcing B to keep an owned `AuditBundle` or a full resumed window. No edits to A's audit/read-range path here. | + +B's `qbit_prism_templates(template_sha256, template_bytes)` and +`qbit_prism_balance_snapshots(prior_balances_digest, balances)` shapes already +exist in 008. A/C must review their encoding/conflict and retention integration; +that review does not require B to invent candidate types or copy migration007. + +The open source-schema [PR321](https://github.com/Qbit-Org/qbit-mining-bootstrap/pull/321) +at `798860c1a79a05a536a9512208f1960220ff4ef9` supplies migration006 and startup +capabilities, not the missing 007/candidate shape. Its runner will need the +minimal 008 membership integration when combined; it is not copied here. +Its latest delta strengthens source-schema sequence/partial-001 detection and +adds gate records; it introduces none of the missing candidate interfaces. + +### b3e8ba9 delta and decisions still open + +The newer head preserves the exact synthetic `JobContext.bootstrap_share` for +empty-window local jobs when converting to body-only storage. It also corrects +memory accounting: each retained local generation keeps its full window, even +after another generation is published. Both are recorded integration facts; +the current full-bundle representation is unchanged. A retention cap, eviction +policy, and acceptable total RSS need separate qualification/review rather than +being inferred from the bed6ad8 signoff. A265 owns the additional measurement of +SQLx's contiguous whole-body bind copy on the runtime during first landing. +The landed `406b273` follow-up also retains as-issued balances in the slim +resumed job for enqueue repair and in an empty-window singleflight entry for +each bootstrap build. Physical deletion waits a B273-selected expiry grace; +that grace must be qualified above share-path latency and cannot extend job +eligibility. No grace value or retention/eviction policy is selected here. + +At runtime cutover, drain old outbox work and stop all old frontends before the +coordinated migration/start procedure. This development policy is not a rolling +compatibility guarantee. + +Full deep-review remains pending against the eventual integrated change: +medium code-review, thermo, Fable and Codex adversarial lanes, and GitHub/CI/bot +review have not run for this slice. Local implementation review and tests do +not establish those lanes. The foundation is intended for draft publication +before that review, with all incomplete runtime/qualification items explicit. +Draft visibility does not authorize ready-for-review, merge-to-base or deployment. diff --git a/docs/public-dashboard-api-v1.openapi.yaml b/docs/public-dashboard-api-v1.openapi.yaml index 3dd0a086e..fb5ff2f69 100644 --- a/docs/public-dashboard-api-v1.openapi.yaml +++ b/docs/public-dashboard-api-v1.openapi.yaml @@ -241,8 +241,8 @@ paths: default: active description: > Omit or pass `active` for the `prism.dashboard.blocks.v1` response, - which hides reorged-out blocks exactly as before this parameter - existed. `all` includes every recorded pool block and `reversed` + which includes currently confirmed blocks and hides unaccepted native + candidate records. `all` includes every recorded pool block and `reversed` returns only blocks the pool once landed that a chain reorganization later disconnected; both return `prism.dashboard.blocks.v2` rows carrying `chain_state` and @@ -851,9 +851,9 @@ components: type: integer minimum: 0 description: > - Every pool block not reversed by a chain reorganization. Excludes - `reversed` blocks by design; it is not the sum of the per-state - counters below. + Every currently confirmed pool block. Native candidate records are + persisted before node acceptance and are excluded until confirmed; + this is not the sum of the per-state counters below. prism_blocks_total: type: integer minimum: 0 @@ -1090,7 +1090,7 @@ components: block_count: type: integer minimum: 1 - description: Every non-reversed block found in the bucket, listed or not. + description: Every confirmed block found in the bucket, listed or not. blocks: type: array maxItems: 3 @@ -1130,7 +1130,7 @@ components: total_blocks: type: integer minimum: 0 - description: All non-reversed found blocks inside the range. + description: All confirmed found blocks inside the range. points: type: array items: diff --git a/docs/public-dashboard-api/README.md b/docs/public-dashboard-api/README.md index 3d81361ef..4fffd3d8a 100644 --- a/docs/public-dashboard-api/README.md +++ b/docs/public-dashboard-api/README.md @@ -8,7 +8,7 @@ The contract is source-of-truth for both sides: - `../public-dashboard-api-v1.openapi.yaml` defines `/public/v1` endpoints. - `fixtures/*.json` are mock responses the dashboard can render before a live backend exists. -- `tests/test_public_dashboard_api_contract.py` keeps the fixtures and public +- `crates/qbit-prism-server/tests/api_contract.rs` and the PostgreSQL API tests keeps the fixtures and public naming conventions from drifting. ## Architecture @@ -27,20 +27,25 @@ The dashboard app must not query Postgres, qbit RPC, private command sockets, or internal audit endpoints directly. Its only stable data dependency should be the sanitized `/public/v1` API described by this contract. -In deployment, `/public/v1` is served by its own process — the -`prism-public-api` service (`python3 -m lab.prism.public_read_service`), on -`PRISM_PUBLIC_API_PORT` (default `3342`). It is no longer served by the -`prism-coordinator` audit HTTP listener, which now answers only `/audit/*`, -`/healthz`, `/metrics`, `/owed*`, and the operator miner/payout status routes; a -`/public/v1` request to the coordinator returns its ordinary -`{"error": "unknown endpoint"}` 404. - -The split exists because public read traffic scales with public interest rather -than with hashrate. Served in-process it shared the GIL that acknowledges shares -and lands blocks, and the primary Postgres connection the lease-holding writer -commits through. The extracted tier reads through bounded read slots only, never -acquires a writer lease, and depends on Postgres rather than on the coordinator, -so it keeps serving across coordinator restarts. +In deployment, run `qbit-prism-server public-api` as the independent +`prism-public-api` service on `PRISM_PUBLIC_API_PORT` (default `3342`). It serves +only `/public/v1`, its own `/healthz`, and `/metrics`. The combined Rust `serve` +role also retains the public routes alongside the operator API for existing +integrations; production proxies should use the independent public service. + +The public service has its own bounded PostgreSQL read pool, sets read-only +sessions, never migrates schema, and does not claim a writer identity. It keeps +serving across mining coordinator restarts. Set its `PRISM_DATABASE_URL` to the +public read database and explicitly set `PRISM_PUBLIC_STRATUM_URL`, because the +read service has no mining listener from which to infer that address. +Its node RPC settings use the same defaults as the coordinator: +`http://127.0.0.1:18452/`, username `qbit`, and password `change-this`. +Configure `QBIT_RPC_HOST` and `QBIT_RPC_PORT`, or set `QBIT_RPC_URL` to override +the complete endpoint. `QBIT_RPC_USER` and `QBIT_RPC_PASSWORD` set credentials. +Both readiness modes verify the required native read schema before allowing +database-backed reads. Apply migrations on the writer and let them replay to +the standby before starting the public service. Hashrate rollups remain +optional because charts can read the canonical share ledger directly. Operators can expose only that path from the pool service, or place a dashboard/web proxy in front of it. The ownership boundary stays the same: pool @@ -48,38 +53,13 @@ read models live here; dashboard rendering lives outside the pool process. `prism-public-api` also serves its own `/healthz` and `/metrics` for that process; those are operator surfaces and must not be exposed publicly. -The HTTP server admits up to `PRISM_PUBLIC_HTTP_MAX_CONNECTIONS` connections -(default 64, range 1–1,024); it immediately closes excess connections without -queuing workers. `PRISM_PUBLIC_HTTP_TIMEOUT_SECONDS` (default 10, finite and -positive) bounds the complete request-line/header phase and socket I/O idle -time. Each response closes its connection, including requests asking for -HTTP/1.1 keep-alive. Size proxy upstream connections accordingly. Database and -public dispatch deadlines remain separate from this network timeout. - -Startup requires `PRISM_PUBLIC_STRATUM_URL` to contain a `stratum+tcp` or -`stratum+ssl` scheme, a valid hostname (or bracketed IPv6 address), and an -explicit port from 1 to 65,535. Credentials, paths, queries, fragments and -whitespace are refused before opening the database or HTTP listener. The same -validation applies to an explicit `PRISM_PUBLIC_STRATUM_HIGHDIFF_URL` when -`PRISM_STRATUM_HIGHDIFF_PORT` enables that advertised endpoint; an empty override -continues to derive its URL from the primary endpoint. - ## Caching Successful `GET /public/v1` responses are safe to cache briefly. The service emits conservative browser caching (`Cache-Control: public, max-age=0, must-revalidate`) plus shared-cache headers for CDNs such as Vercel. Dynamic dashboard read models default to a 5-second shared-cache TTL with 30 seconds of -`stale-while-revalidate` before applying the route limit. Both shared-cache TTL -and stale-while-revalidate are clamped to the route's -published staleness budget. For example, a default plain read uses a 5-second -TTL and at most 10 additional stale seconds within its 15-second budget. -Background refreshes share four worker slots across all response caches in the -process. When they are busy, eligible stale responses remain available within -their existing window; additional refresh work is not queued. A later request -can retry after capacity returns. - -The pool-wide aggregate read models — +`stale-while-revalidate`. The pool-wide aggregate read models — `GET /public/v1/pool-summary`, `GET /public/v1/hashrate-series`, and `GET /public/v1/miners/{recipient_id}/workers` — are expensive to recompute and default to a 30-second shared-cache TTL instead. @@ -117,10 +97,9 @@ servable until the window ends. The in-process window is clamped so a stale-served `Age` never exceeds the route's staleness budget (below); past the window the next request blocks and recomputes as before. Error responses use `Cache-Control: no-store` and are not cached by that origin cache. -Miner pages additionally share one briefly cached pool-wide reward-window -aggregate (`PRISM_PUBLIC_REWARD_WINDOW_CACHE_SECONDS`, default 30 seconds, 0 -disables), so requests for different miners reuse a single recursive -reward-window scan instead of each re-running it. +Miner pages calculate reward-window aggregates directly in PostgreSQL and +share the same response cache as the other public routes. The former Python +`PRISM_PUBLIC_REWARD_WINDOW_CACHE_SECONDS` <!-- retired-setting: PRISM_PUBLIC_REWARD_WINDOW_CACHE_SECONDS --> inner-cache setting is obsolete. Every origin computation that takes a ledger read slot — the immutable artifact route included, on a cold request — runs under one per-request @@ -132,6 +111,17 @@ that has long since hung up, and returns `503` with error code `read_timeout` and `Cache-Control: no-store`. The refusal is never cached, so the next request — or a stale-while-revalidate refresh — retries the origin. +`/public/v1/artifacts/{sha256}` also bounds how many block-audit artifacts it +will rebuild or decode at once, because each holds a whole payout window in +memory. Past that bound a request is refused immediately, before any audit +read, with `503`, the error code `audit_artifact_busy`, `Retry-After: 5` and +`Cache-Control: no-store`; retrying after the stated delay is the intended +client behaviour, and the refusal, like every other error here, is never +cached. CTV fanout manifests, which this route also serves, are never refused +this way. Requests for one artifact that arrive together share a single +computation, so a client that sees this code is competing with requests for +*other* artifacts, not with itself. + ## Staleness Every `/public/v1` response states how old an answer that route is willing to @@ -144,12 +134,10 @@ worse than an honest one. sha256 is correct at any age and this route never refuses for staleness. - `Age` — the observed age of the response actually served, as before. -When the observed age exceeds the budget, the service returns **503** with -`Cache-Control: no-store` and an ordinary `prism.dashboard.error.v1` body whose -message names both the budget and the observed age. Clients should treat this -as "the data behind this route is too old to answer with", not as a new error -schema — the error code is `upstream_unavailable`, already in the documented -enum. +When a cached response exceeds the budget, a healthy service discards it and +recomputes before answering. During a database outage it returns **503** with +`Cache-Control: no-store` and an ordinary `prism.dashboard.error.v1` body with +error code `upstream_unavailable`. Budgets are derived from the caches that sit under each route rather than hand-picked: @@ -159,11 +147,10 @@ budget = max(3 * (cache_ttl_seconds + underlying_cache_seconds), 15) ``` `cache_ttl_seconds` is the route's own shared-response TTL and -`underlying_cache_seconds` is any second cache stacked beneath it — today only -the pool reward-window aggregate (`PRISM_PUBLIC_REWARD_WINDOW_CACHE_SECONDS`, -default 30s) under `/public/v1/miners/{recipient_id}`. The factor of three and -the 15-second floor match the existing precedent for PRISM's cached `/metrics` -endpoint. At the documented defaults this yields: +`underlying_cache_seconds` retains the historical 30-second allowance for +`/public/v1/miners/{recipient_id}` to preserve the 2.x header contract, even +though Rust computes that aggregate directly in PostgreSQL. The factor of +three and 15-second floor retain the existing budgets: | Route | Budget | | --- | --- | @@ -183,15 +170,13 @@ endpoint. At the documented defaults this yields: | `/public/v1/artifacts/{sha256}` | unbounded | The budgets are constants derived from the documented cache defaults — the -`max_staleness_seconds` values in `lab/prism/endpoint_registry.py` — and there +`staleness_budget` function in `crates/qbit-prism-server/src/api/public_service.rs` — and there is no environment knob that raises a budget. An operator who raises one of the cache TTL knobs that do exist (`PRISM_PUBLIC_CACHE_TTL_SECONDS`, `PRISM_PUBLIC_AGGREGATE_CACHE_TTL_SECONDS`, -`PRISM_PUBLIC_CONFIG_CACHE_TTL_SECONDS`, or -`PRISM_PUBLIC_REWARD_WINDOW_CACHE_SECONDS`) above its route's budget will see -that route begin refusing with 503 rather than quietly serving older data. -Raising a TTL past its budget therefore also requires changing the registry -constant in the same change; otherwise leave the defaults alone. The +`PRISM_PUBLIC_CONFIG_CACHE_TTL_SECONDS`) above its route's budget will see +the origin refresh at the budget boundary rather than serve older data. +The `PRISM_PUBLIC_ARTIFACT_CACHE_*` TTLs are exempt: the artifact route is content-addressed and never refuses for staleness. @@ -286,10 +271,9 @@ Chain reorganizations happen, and hiding them entirely made the pool's block history look cleaner than the chain it mines. `GET /public/v1/blocks` therefore takes a `chain_state` filter: -- Omitted or `chain_state=active` — exactly the pre-filter behavior and the - `prism.dashboard.blocks.v1` schema tag: every recorded pool block except - those a reorganization reversed. Existing consumers see a byte-compatible - response. +- Omitted or `chain_state=active` — confirmed blocks only, using the + `prism.dashboard.blocks.v1` row shape. Prepared and never-accepted native + candidate records are excluded. - `chain_state=all` — every recorded pool block, including reversed ones. - `chain_state=reversed` — only blocks the pool once landed that a chain reorganization later disconnected. @@ -308,7 +292,7 @@ response with a reversed block. `GET /public/v1/pool-summary` surfaces the same information in aggregate: `pool.blocks_reversed_total` and `pool.blocks_inactive_total` count the reversed and currently-inactive pool blocks. `blocks_found_total` is -unchanged and keeps excluding reversed blocks, so it is not the sum of the +the count of currently confirmed blocks, so it is not the sum of the per-state counters. These are **additive fields on `prism.dashboard.pool-summary.v1`** (added in 2.x): pool-summary takes no request parameter, so the repo's param-gated versioning precedent does not @@ -351,7 +335,7 @@ range's chart renders are rejected with `400 bad_request`: 1w allows 5m/1h/1d, 1m allows 1h/1d, 6m and all allow 1d only. The response (`prism.dashboard.block-markers.v1`) reports `total_blocks` — all -non-reversed found blocks in range — and `points` containing only buckets with +confirmed found blocks in range — and `points` containing only buckets with at least one found block, ascending by timestamp. Each point carries the bucket's full `block_count`, at most its 3 most recent blocks (`found_at` descending, height breaking ties) with `height`, `hash`, and `found_at`, and @@ -423,43 +407,19 @@ re-serialization step. Audit bundles written before canonical-byte persistence was introduced retain the legacy reconstructed response until the verified backfill publishes their canonical artifact. -Operators can verify the historical range while the coordinator is live: - -```sh -python3 -m lab.prism.backfill_audit_bundle_canonical --dry-run -``` - -Before publishing, stop the coordinator and confirm it no longer owns the -PostgreSQL writer lease. Publish mode requires exclusive lease ownership and -fails instead of waiting behind a live same-identity coordinator. Schema repair -is unrelated to this filesystem-only rollout, so the recommended publish -command disables it explicitly: +For migration, drain the Python coordinator, preserve its database and audit +artifacts, then run the native migration and `import-legacy-audits` commands described +in [the migration guide](../prism-rust-migration.md). Imported canonical bytes +are stored in PostgreSQL so physical replicas and every frontend receive the +same artifact identity. Native audits reconstruct exact bytes from immutable +share ranges rather than storing overlapping full share windows repeatedly. -```sh -python3 -m lab.prism.backfill_audit_bundle_canonical --no-init-schema -``` - -The command releases its exact writer lease and closes its database and -artifact-store resources on success or failure, so the coordinator can restart -without waiting for the backfill lease TTL. It pages in stable block-hash order -and reports `last_checkpoint`; pass that value to `--start-after` to resume a -bounded rollout. The checkpoint advances over failed rows. Their block hashes, -advertised digests, failure kinds, and reasons remain in the JSON -`failed_rows` array, but resuming after the checkpoint will skip them. After -correcting a failure, retry from before its block hash or rerun from the -beginning; already published rows are idempotent no-ops. - -On a read-replica deployment, the ledger row remains the visibility authority. -If the row has replayed but the canonical file is absent, the service uses the -legacy reconstructed response with `Cache-Control: no-store` and -`X-Prism-Artifact-Canonical-State: missing`; it is never admitted to the origin -cache or an immutable CDN cache. A corrupt canonical file fails closed rather -than being disguised as legacy history. If shared storage receives the file -before the replica replays its row, the artifact returns a non-cacheable `404` -until replay catches up; the file alone never exposes an uncommitted artifact. -This immutable route remains exempt from the ordinary freshness refusal. -Replica readiness is bounded by the existing WAL-receiver heartbeat contract, -not by replay-lag position. +If canonical bytes for a legacy row are unavailable, its reconstructed response +uses `Cache-Control: no-store` and `X-Prism-Artifact-Canonical-State: missing`. +It never enters the origin or immutable CDN cache. Corrupt present bytes fail +closed. The database row remains the visibility authority: an artifact whose +row has not replayed returns a non-cacheable `404`. Immutable artifacts remain +exempt from ordinary replica freshness refusals. Direct-coinbase blocks return the same settlement-artifacts wrapper with `settlement_mode: direct_coinbase` and `fanouts: []`. A `404` means no public @@ -499,3 +459,17 @@ for its own frontend. Those are not required public API surfaces for PRISM dashboard v1. Template fragments are an Ocean implementation detail, and CSV exports can be generated from the paginated JSON read models or added later as a thin convenience layer without changing the core dashboard contract. + +## Rust candidate-state compatibility + +The Rust coordinator persists candidate records before the node accepts them. +For this reason, the default block list, block markers, and found-block counters +include only confirmed blocks; earnings and balances likewise require active +chain confirmation. This deliberately tightens the older Python non-reversed +predicate so rejected work cannot appear as a successful found block. + +`chain_state=all` remains the complete recorded history. A previously confirmed +block that is disconnected appears publicly as `reversed`, with its disconnect +time; the ledger can still reactivate it if that branch becomes active again. +Never-confirmed inactive candidates remain `inactive`, with no disconnect time. +The reversed and inactive summary counters follow these same public states. diff --git a/docs/refresh-window-split.md b/docs/refresh-window-split.md new file mode 100644 index 000000000..efe693148 --- /dev/null +++ b/docs/refresh-window-split.md @@ -0,0 +1,224 @@ +# Template and payout-window refresh + +The refresh loop retains one immutable, anchored `Snapshot` and its native +`WindowRef`. A transaction or fee-policy change builds a new template, +coinbase, and compact prepared record using those same window inputs. It does +not select the accepted-share window again or recompute its native digest. + +As-is window reuse is invalidated by a changed accepted-share cutoff, payout revision, +prior-balance digest, network difficulty, or the existing +`PRISM_PAYOUT_ARTIFACT_REANCHOR_SECONDS` interval. The interval starts before the +snapshot read; rebuilding templates does not renew it. The accepted-cutoff +probe reads the latest accepted sequence without selecting share payloads; +it shares its exact SQL with the snapshot cutoff. Production appends serialize +under the ordering barrier, assign the acceptance timestamp from the ledger +clock, and reject future job timestamps, so a newly committed accepted row is +eligible at the next snapshot barrier. Rejected rows do not invalidate it. + +Invalidating cached inputs does not by itself replace usable published work. +As before this split, accepted shares with an unchanged template wait for the +next template or economic change, or the original reanchor deadline; that +build captures the latest eligible shares. This preserves the existing +publication cadence under a busy share stream instead of adding a full read, +generation change, and miner update on every poll. The first accepted share +still immediately replaces an empty window. Revision, balance, and tip changes +continue through their existing refresh and publication checks. + +Build inputs are selected after waiting for build admission, with a fresh +cutoff, payout state, and reanchor-age check. Shares committed after that +selection belong to the next window, as they did after the baseline snapshot +read. The reanchor interval triggers another read; it is not a replacement for +the existing absolute work deadline. Tip, readiness, revision, and balance +identity are still revalidated before publication. + +`Prepared`, issued jobs, persisted records, and resumed jobs still contain +compact references rather than accepted-share arrays. The cache is owned only +by the serialized refresh loop. Its steady retained memory is one snapshot: +O(window shares + prior-balance recipients), including each share's owned +strings. Builders still need their existing transient counted-share and +artifact allocations. Old cache ownership is released on the blocking executor under build admission +before reading a replacement; an active blocking build retains its own admitted +inputs until cleanup ends. Keeping old issued jobs does not retain those rows. +No idle build permit is retained. + +The serialized refresh loop owns cached inputs even if a later reservation is +cancelled. Build admission covers actual build cleanup, then ends before database +reservation waits so existing work can still resume. Cached inputs never confer +publication authority. Publication still uses the existing reservation and +publication authority +boundary, with fresh tip, readiness, revision, and balance checks. A failed publication may leave reusable cached inputs, but fast reuse of published +work requires its exact window reference to match the cache. Original issued balances, +references, and absolute expirations are unchanged. Native share-array hashes +and canonical audit hashes continue to use their existing serializers; the +Python-compatible `PayoutWindow` digest is not substituted for either. + +## Issue #274 scope and evidence + +This implements the September 10 refresh split. It removes repeated full-window +SQL reads during template churn; it does not introduce the +incremental `PayoutWindow` engine or a new hash recipe. The bounded delta +acquisition below separately reduces payload reads when reanchoring. Existing borrowing +builders still fold and hash the captured inputs for each changed template. +Those remaining costs need the existing 400k/500k harness and the agreed refresh +budget to determine whether the conditional incremental-engine work is needed. + +`refresh_window_split` exercises the real Coordinator and PostgreSQL through the +existing execution proxy: three transaction changes two seconds apart return +zero additional accepted-share rows; a share on every unchanged-template poll +does not replace published work, and the next template or original reanchor +captures all latest eligible shares in one read. Revisions and difficulty +changes trigger fresh reads immediately. It also verifies as-issued resume, +compact storage and canonical audit reconstruction, rejection of a delayed +refresh after a newer tip observation, and cancellation during a completed +COMMIT response wait. Unit regressions cover same-revision +balance changes, the empty-to-nonempty transition, and release of retired rows +while old prepared work remains alive. Admission regressions hold the sole +build slot while the cutoff changes or the original interval expires. A delayed +snapshot COMMIT response checks that read latency does not renew the interval; +a share committed during reservation enters the next build without rewriting +the already-selected window. + +The unchanged frozen money/window corpus and compact runtime tests remain the +compatibility gates. No sub-second one-share-delta result, production timing, +24-hour memory soak, or measured per-frontend RSS is claimed by this change. +The original incremental/differential criteria depend on the conditional engine +work. The 24-hour memory criterion remains unmeasured; bounded cache ownership +is a structural property, not a soak result. + +## Bounded delta acquisition (#275) + +A rebuild still captures a fresh anchor, accepted cutoff, payout revision and +prior balances under the original settlement/ordering locks. When the refresh +loop exclusively owns the retired window, it moves that window's shares into +acquisition under build admission. A shared `Arc`, including one still held by +cancelled blocking work, takes the full-read path. Hashing, reward folding, +publication authority and the as-is reuse predicate are unchanged. + +The candidate replays the full reader's newest-first saturating weight fold, +for the fresh target weight, over three sources: the eligible rows appended +above the retained cutoff, read in ascending pages of 4096 rows (a delta may +span several pages, up to 262,144 sequence slots); the retained rows; and, +when the fresh target is heavier than the retained rows reach, a margin of +older eligible rows read newest-first below the retained first row, up to 16 +pages. A changed network difficulty is therefore not a fallback: a lighter +target retires rows from the old end, a heavier one pulls the margin in, and +the crossing row is re-derived either way. Trimming precedes appending, so +obsolete rows never inflate the retained vector, and a large retirement +releases excess capacity; with a margin one exactly sized vector is assembled +instead. Decoding, merging and destruction stay on the blocking executor. + +The reads run in one READ COMMITTED transaction, as the full scan's do: each +page and each witness statement sees its own snapshot, and consistency comes +from the anchored predicate rather than from snapshot isolation. Ledger rows +never change, eligibility is `accepted_at <= anchor AND job_issued_at <= +anchor`, and the anchor is issued by an `UPDATE` of the cluster singleton that +every append also updates, so an append in flight at the anchor had already +committed with an earlier clock and every later append carries a clock +strictly above it. A row that appears between two statements is either above +the cutoff and ineligible, or a retroactive INSERT into an old slot or a row +whose timestamps became eligible, and the final count catches both; the +partition and timeline witness is re-read in that same final statement. + +Memory at the bounds: the retained window, the whole delta and the whole +margin are alive together until the merge, and with a margin the exactly +sized merged vector is allocated while all three exist. At about 600 B per +decoded production share and 216 B of inline element per row, a 400k window +(240 MB steady) peaks at about 640 MB with the delta and margin both at their +bounds (157 MB and 39 MB) plus the 200 MB assembly duplicate, and a 500k +window at about 720 MB; the full scan's own transient is its vector doubling +slack (108 MB at 500k) plus one page. A per-block retarget moves the crossing +row by a fraction of a percent of the window (about 3,200 rows at 400k for a +0.8% step; the measured margins were a few thousand rows), so the 16-page +margin bound covers a long run of upward steps while keeping the margin's +share of the transient small; a larger jump takes the full scan. The delta +bound is about half an hour of production shares, so a frontend that missed +refreshes for that long still advances. Admission is a permit count, not +bytes, so nothing but these two constants bounds the transient. + +The merged window is checked before it is trusted. It must be one strictly +ascending sequence with no duplicate, and it must cross exactly at its first +row: the fold over every row reaches zero and the fold without the first row +does not, the same boundary landing re-derives. A walk that runs out of +history before the target is refused as partial, because the count proof +below cannot see rows older than the first row; the full scan decides whether +such a window is legitimately partial. `WindowRef` is then the digest of +exactly this window, so no margin or delta row can leak into a snapshot that +landing would refuse as a superset or as under-coverage. + +Endpoint presence by itself is insufficient: two live endpoints can surround a +detached interior partition. Acquisition therefore records private evidence that +the full retained suffix was in one live PostgreSQL leaf, including that leaf's +`pg_inherits` tuple incarnation and current WAL insertion timeline. A later +advance requires the same timeline and live leaf for the retained range and new +cutoff. Detached, detach-pending, restored or replaced leaves invalidate that +proof; cross-partition windows use the full scan. + +Even an unchanged leaf does not prove an unchanged eligible set. Sequence gaps +are normal after rollbacks and crash recovery, and immutable history permits an +INSERT into an unused old sequence. Future timestamps can also become eligible. +After trimming, one statement checks both the live leaf incarnation and the +number of currently eligible rows from the proposed first row through the fresh +cutoff. Within that writer history, retained rows are still immutable members of +that leaf; equal count proves there are no omitted eligible members in that range. Since this suffix +already reaches the weight, older rows cannot affect the full reader's answer. +A changed count falls back to a full scan at the same fresh anchor. + +The subset proof depends on the existing immutable-history triggers. An operator +repair that disables or bypasses those triggers can change payloads without +changing the count or leaf incarnation. Such a repair requires restarting every +frontend to discard retained windows before trusting another refresh. Supported +partition restoration invalidates the acquisition evidence normally. + +Both leaf checks also read the current WAL insertion timeline in the same +transaction as share acquisition. A supported asynchronous promotion can lose +acknowledged rows and replace their sequence values while preserving leaf OID, +catalog incarnation and count, including an interior row whose newest neighbor +survives. The changed timeline rejects that retained vector and takes the full +read. Ordinary pool rotation or reconnection to the same history keeps the +evidence usable. Query errors remain errors; they never authorize reuse. + +This history proof uses the existing [D3 promotion and rejoin +contract](prism-ha-reference-architecture.md#promotion-fencing-and-the-stable-writer-endpoint): +one eligible standby, positive fencing of the old writer, and verified rewind or +a fresh base backup before rejoining the former primary. Its later promotion +advances the timeline again. Timeline numbers are not globally unique: +independently promoted sibling copies can choose the same number, but D3 does +not permit them as competing or replacement writers. Isolated restore and PITR +follow the existing [D5 stopped-frontend recovery +boundary](prism-rust-migration.md#recovery-and-rollback), which discards this +process-local evidence. These are existing deployment boundaries; no additional +frontend restart or privilege grant is required for D3 failover. + +This check is an **O(window) metadata scan**, not O(delta) database work. It avoids +transferring and decoding retained payloads but still pays the count cost and +recomputes every existing digest. The optimization also falls back for empty or +short (partial) history, decreasing cutoffs or anchors, absent acquisition +evidence, deltas spanning more than 262,144 sequence slots and margins of more +than 16 pages. Default partitions span 16,777,216 sequence slots; a 400k window +crosses a boundary for roughly 2.4% of uniformly distributed dense positions, +an arithmetic illustration rather than a measured production eligibility rate. +Every refresh snapshot counts its outcome in +`qbit_prism_refresh_window_acquisitions_total{outcome}` and every published +rebuild observes `qbit_prism_refresh_seconds{trigger,acquisition}` and logs one +`refresh published` line with the acquisition's row and page counts, so the +production eligibility rate is measured rather than assumed. + +Real PostgreSQL differential tests compare serialized snapshots and `WindowRef` +at the same fresh anchor, including crossing rows, retargets in both +directions, multi-page deltas and margins, the size bounds, seeded random walks +over target and appended shares, revision/balance changes, real post-INSERT +rollback gaps, retroactive inserts, future eligibility and partition +detach/reattach, and prove that a delta-assembled window passes the landing +re-derivation while its superset and its under-covering subset fail it. An +adversarial differential contributed by the independent review walks a +legacy-style fixture (sequence gaps, rejected rows, a partition boundary, +retargets of up to 8x either way, rows committed between the anchor and the +extension read, corrupted retained inputs) and requires every advanced window +to equal the full reader's and land, and every refusal to be predicted and +named; a coordinator-level test lands a found block through the real +coordinator on a window the delta path built after a retarget. Coordinator proxy tests distinguish actual payload +rows from metadata and retain their semantic and as-issued assertions. The +experiment's performance screen is at least 15% end-to-end improvement over the +exact base across at least three balanced 400k pairs, with exact durable delivery +identities and no worse memory bound. This screen is separate from the unchanged +one-second goal; no subsecond claim follows from a payload-read improvement. diff --git a/docs/releasing.md b/docs/releasing.md index f86dc51f4..859185544 100644 --- a/docs/releasing.md +++ b/docs/releasing.md @@ -1,14 +1,19 @@ # Publishing a release -`main` is the release promotion branch; `2.x.x` is the development branch for -the 2.x series. A version is published only when its tag and GitHub release -exist. A version bump or merged preparation PR alone does not publish it. +`main` is the release promotion branch. `3.x.x` is the development branch for +the 3.x series, now carrying 3.1; `2.x.x` is the maintenance branch of the +2.x series. A release cut from an earlier point of a development branch gets +its own branch at that cutoff, as `3.0.x` does for 3.0.0. A version is +published only when its tag and GitHub release exist. A version bump or +merged preparation PR alone does not publish it. ## Prepare one release candidate -Keep one pending release version across `VERSION`, the `qbit-prism` and -`qbit-pool-builder` package versions in their `Cargo.toml` files and `Cargo.lock`, -and `doc/release-notes-<version>.md`. Ordinary development PRs contribute to +Keep one pending release version across `VERSION`, `[workspace.package].version` +in the root `Cargo.toml` (which every crate inherits), both `Cargo.lock` files +(the root and `crates/qbit-prism-server/fuzz`), and +`doc/release-notes-<version>.md`. `scripts/check_version_skew.py` checks the +first three agree. Ordinary development PRs contribute to that pending release's notes; they do not each create a patch release. Choose the next version after reconciling the last published release and any identifiers already distributed to operators. @@ -20,32 +25,49 @@ Their changes and migration requirements are consolidated in [the 2.0.0 notes](../doc/release-notes-2.0.0.md). Do not create retrospective 2.0.1 or 2.0.2 tags for those preparation commits. +For 3.0.0, the cutoff is `4c134913` on `3.x.x`, the last commit before the +3.1 dual writer. `3.0.x` starts there and takes the version bump, the 3.0.0 +notes and the release's dependency fixes (#776, merged as `aabe470f`). Unlike +2.0.0, `v3.0.0` was tagged and published on `3.0.x` at `aabe470f`, before the +promotion. The promotion into `main` (#778) then gives `main` the tagged tree, +plus this guide and a README banner that doesn't name a branch; it adds no +tag. The 3.0.0-rc.N pre-releases were tagged on `3.x.x`. 3.0.0-rc.7 was +prepared (#772) but never tagged, and its changes ship in 3.0.0. The L3 +matrix for the release is recorded in #779. + Merge the intended development cutoff into the promotion branch and retain -fixes already made during promotion review. Check package versions together, +fixes already made during promotion review. The repository allows only squash +and rebase merges, so the promotion lands on `main` as one squashed commit. Check package versions together, run Docker lint and the relevant test gates, and review upgrade and rollback instructions. Leave the release date out while publication is unscheduled; set the actual release date in a reviewed commit before the final merge. ## Publish the reviewed main commit -After the promotion PR passes required checks and is merged, identify the -exact resulting commit on `main`. A squash or rebase merge produces a different +Tag the promoted `main` commit unless the release was already tagged on its +release branch, as 3.0.0 was. After the promotion PR passes required checks +and is merged, identify the exact resulting commit on `main`. A squash or rebase merge produces a different commit from the promotion branch head. Verify the resulting commit's checks, version metadata and dated release notes before tagging it. -For 2.0.0, after confirming that `v2.0.0` does not already exist, use the -reviewed main commit in place of `<release-main-sha>`: +After confirming that the tag doesn't already exist, use the reviewed main +commit in place of `<release-main-sha>`, and the version in place of +`<version>` (as `v2.0.0` and `v3.0.0` were): ```sh git fetch origin main --tags -git tag -s v2.0.0 <release-main-sha> -m 'qbit-mining-bootstrap 2.0.0' -git verify-tag v2.0.0 -git push origin refs/tags/v2.0.0 -git show v2.0.0:doc/release-notes-2.0.0.md > /tmp/qbit-release-2.0.0-notes.md -gh release create v2.0.0 --verify-tag --title 'qbit-mining-bootstrap 2.0.0' \ - --notes-file /tmp/qbit-release-2.0.0-notes.md +git tag -s v<version> <release-main-sha> -m 'qbit-mining-bootstrap <version>' +git verify-tag v<version> +git push origin refs/tags/v<version> +git show v<version>:doc/release-notes-<version>.md > /tmp/qbit-release-notes.md +gh release create v<version> --verify-tag --title 'qbit-mining-bootstrap <version>' \ + --notes-file /tmp/qbit-release-notes.md ``` +Pushing a `v*` tag also runs the L3 production-window matrix +(`prism-load-l3.yml`), which promotes the promotion PR's run on the same tree +or runs `l3-full` (#608). + Run publication commands only when authorized to publish. If a tag or release already exists, inspect its target and contents instead of replacing it. Use notes from the tagged tree, then verify the GitHub release points to that @@ -55,7 +77,9 @@ release version. ## Reconcile the development branch After promotion, open a synchronization PR from a branch that merges the -released `main` commit into the latest `2.x.x`. Preserve subsequent development +released `main` commit into the latest development branch (`3.x.x` after +3.0.0, `2.x.x` after 2.0.0), and into a release branch such as `3.0.x` that +will take patch releases. Preserve subsequent development and resolve metadata and notes deliberately; do not reset or force-push the shared branch. Run its checks before merging. diff --git a/docs/window-reader-lifetime.md b/docs/window-reader-lifetime.md new file mode 100644 index 000000000..acbb58919 --- /dev/null +++ b/docs/window-reader-lifetime.md @@ -0,0 +1,119 @@ +# Window reader admission lifetime + +Refs #273, reader cancellation and error-classification gates from +[the activation review](https://github.com/Qbit-Org/qbit-mining-bootstrap/issues/273#issuecomment-5671702658). + +## Completion boundary + +`read_window_with_permit` puts the caller's permit in one shared +`ReadAdmission`. The read future, each blocking mapping, and every guarded +payload retain that same admission. A blocking mapping transfers ownership +to its output before returning; a cleanup owns the payload before the +completion reference in field drop order. The final reference releases the +permit only after all of those owners finish. + +Cancelling the future stops further paging and drops its transaction. +An already submitted blocking page can still finish. Its accumulated state, +the decoded balances, and any vectors waiting for commit retain admission +through destruction on blocking workers. Cleanup tasks can run concurrently +and finish in either order without admitting another read early. Errors can +return before cleanup finishes; the permit still bounds that outstanding work. + +On success, the authenticated vectors transfer to the caller and the read +permit is released before the caller's rebuild. The caller then owns those +vectors and their cleanup obligation. There is no additional semaphore +acquisition, retry, background paging loop, or timeout in the reader. + +The private paging helper owns its cursor and consumer alongside the state, +then returns only guarded state. The final transfer has no await and restores +the state guard before dropping the consumer. The window reader's consumer +captures only the share count; the public paging helper continues to drop its +consumer on the runtime after successful paging. + +Cleanup captures one struct so its field order also applies when runtime +shutdown discards a queued blocking task without executing the closure. +Separate moved closure captures have an unspecified destruction order, even +if explicit `drop` calls would order them when the closure runs. + +## Caller and compatibility audit + +The signatures of `read_window`, `read_window_with_permit`, +`read_range_paged`, and `payout_state` are unchanged, as are stored bytes, +window digests, ordering, and error variants. + +| Path | Contract retained | +| --- | --- | +| `Coordinator::rebuild_claim_parts` → coordinator reader → `Ledger::read_window_with_permit` | Build admission precedes read admission; the existing outer rebuild deadline includes read admission and I/O. The returned window moves immediately into the blocking builder. | +| `hydrate_compact_inputs` → `WorkLedger` → `Ledger::read_window_with_permit` | As-issued balances, existing outer deadline, original issued revision and absolute expiry remain caller-owned. The adapter's publication and eligibility checks remain in place. | +| Direct `Ledger::read_window` users | Same snapshot, bounds, count, digest, and corruption checks; no implicit admission requirement is added. | +| Public `read_range_paged` consumers | The same generic state/consumer API and result remain available; row mapping and accumulated-state cleanup still run off the runtime. | +| Candidate landing after reconstruction | `land_candidate_checked` authenticates rebuilt parts, verifies durable rows through its separate `verify_durable_range` reader, then applies its existing claim and revision fences. Its implementation is unchanged. | +| `WorkLedger::payout_state` | Revision and balance digest still come from one primary repeatable-read snapshot. A blocking task failure is `TaskFailed`; malformed balances are `Decode`. | + +The read result does not itself authorize publication or credit. Existing +caller checks and transactional revision fences retain that responsibility; +the current observed revision never replaces the issued operation identity. + +## Regression evidence + +The focused `ledger::window` unit suite covers: + +- cancellation while mapping is held, followed by a separately held output + destructor; a second admission remains pending through both; +- two simultaneous payload destructors completing in different order, with + admission retained until both finish; +- mapping failure and panic retaining admission until input cleanup ends; +- successful transfer releasing admission without destroying the result; +- exactly-once payload destruction and permit recovery; +- runtime timer progress and spare blocking-worker progress while cleanup or + mapping is held; +- mapping panic as `TaskFailed` and mapping decode errors as `Decode`; +- admission held during cleanup after the captured runtime has shut down, + including a payload destructor that panics. + +The PostgreSQL `window_reference::payout_state` tests exercise the production +balance decoder with a stored numeric overflow (`Decode`) and valid empty +balances (the canonical empty digest). + +The existing database cancellation regression in `candidate_window_migration` +also exercises the real reader after one accumulated page while the next +database page is gated. Its single-blocking-worker setup proves queued +cleanup ownership; the controlled multi-worker unit cases prove completion +ordering independently of blocking-pool scheduling. + +Run the focused unit suite: + +```sh +cargo test --locked -p qbit-prism-server --lib ledger::window:: +``` + +Run the caller and storage regressions on a disposable local database: + +```sh +env -u PRISM_TEST_DATABASE_URL test/prism-native-tests.sh cargo-args --locked \ + -p qbit-prism-server --lib --test window_reference --test window_read_oracle \ + --test window_balance_order --test candidate_window_migration \ + --test candidate_window_switch --test candidate_window_qualification \ + --test ledger_deadline_atomicity -- --nocapture --test-threads=1 +``` + +Qualification on 2026-09-14: the focused unit suite passed 15 tests and Clippy +passed for all server targets with warnings denied. The regression selection +passed 267 library tests and 62 integration tests, with 6 explicit ignores. +The first parallel run hit a PostgreSQL lock timeout in +`ascending_pages_match_native_rows_and_filter_gaps`; the complete +`window_reference` suite then passed serially (19 tests, 2 ignores). The +serial invocation above avoids concurrent fixture migration lock contention; +no production timeout or connection configuration was changed. + +Follow-up qualification on 2026-09-15 after the paging and blocking-handoff +review fixes passed all 15 focused tests, 267 library tests and 62 integration +tests on a fresh disposable PostgreSQL, with the same 6 explicit ignores. +The serial run completed without failures; the real payout corruption and +reader cancellation tests executed. Clippy with warnings denied, formatting, +and whitespace checks passed. Independent verification of the resulting PR +head remains a separate review step. + +These changes satisfy the reader ownership and error-classification gates. +They do not activate compact runtime work or establish the separate +replacement-lease and first-build readiness gates. diff --git a/lab/auxpow/Dockerfile b/lab/auxpow/Dockerfile index 0ff3eeef2..78e93f3f5 100644 --- a/lab/auxpow/Dockerfile +++ b/lab/auxpow/Dockerfile @@ -4,6 +4,8 @@ FROM python:3.14-slim-trixie WORKDIR /app +# qbit-src is a named build context (`--build-context qbit-src=...`), not a stage. +# hadolint ignore=DL3022 COPY --from=qbit-src test/functional /opt/qbit-functional COPY lab/auxpow/*.py /app/lab/auxpow/ diff --git a/lab/prism/Dockerfile b/lab/prism/Dockerfile index 63a63aec9..4a3e846dd 100644 --- a/lab/prism/Dockerfile +++ b/lab/prism/Dockerfile @@ -1,60 +1,51 @@ # syntax=docker/dockerfile:1.7 -FROM python:3.14-slim-trixie - -WORKDIR /app - -RUN apt-get update \ - && apt-get install -y --no-install-recommends \ - ca-certificates \ - cargo \ - postgresql-client \ - rustc \ - && rm -rf /var/lib/apt/lists/* - -# Persistent pooled Postgres client for the share-ledger hot path. The psql -# binary stays installed as the fallback backend (PRISM_POSTGRES_NATIVE_CLIENT=0). -RUN pip install --no-cache-dir "psycopg[binary]==3.2.*" - +# The builder is the toolchain pinned in rust-toolchain.toml, by version and by +# the multi-arch index digest of that tag on Docker Hub; the docker-builds CI +# job asserts the stage's cargo reports the pin, and +# scripts/check_toolchain_pin.py holds the version here to the pin. +FROM rust:1.98.1-bookworm@sha256:9a73a5088750b4c95158ab26629c854c3d6fc4b173cb7bc8079ad252d8ed7bfa AS build +WORKDIR /build COPY Cargo.toml Cargo.lock ./ COPY crates/ ./crates/ -COPY lab/auxpow/ ./lab/auxpow/ -COPY lab/prism/ ./lab/prism/ - -ENV PYTHONPATH=/app - -# Release profile plus a direct-binary path: the coordinator shells out to -# these tools on the job build hot path, so it must not pay `cargo run` -# build-graph checks per invocation nor debug-profile execution time. -RUN cargo build --locked --release -p qbit-prism --bins - -ENV PRISM_TOOL_BIN_DIR=/app/target/release - -# Issue #226: glibc malloc arena policy, set deliberately rather than left to -# glibc. The mainnet coordinator runs about 64 threads and its memory map is -# hundreds of 4-64 MiB anonymous regions -- the per-thread arena signature -- -# on a host where glibc's default arena limit is 8 x cores = 256, so every -# thread that ever mallocs owns an arena and that arena's retained free lists. -# Two arenas is the default because it is safe before the soak decides -# between 2 and 4: under the GIL at most one Python thread allocates at a -# time, so arena-lock contention can only come from GIL-released native code -# (libpq I/O on a handful of pooled connections), and the audit-builder -# daemon this process spawns inherits the setting but is single-threaded. The -# failure mode of too few arenas is measurable lock wait on /metrics share-ack -# latency; the failure mode of too many is the 17.7 GB resident set. Override -# at build time with --build-arg MALLOC_ARENA_MAX=4 or at run time with -# -e MALLOC_ARENA_MAX=4; docs/mainnet-deployment.md records the decision and -# docs/prism-capacity-readiness.md the soak that revisits it. -# -# PYTHONMALLOC is deliberately unset: pymalloc keeps objects of 512 bytes and -# under out of glibc entirely, and routing them through glibc would enlarge -# the very arenas this bounds. No other MALLOC_* variable is set here. Never -# set one to an empty value: glibc parses "" as 0, and MALLOC_MMAP_THRESHOLD_=0 -# would send every allocation through mmap. tests/test_prism_heap_census.py -# pins exactly this allocator environment. -ARG MALLOC_ARENA_MAX=2 -ENV MALLOC_ARENA_MAX=${MALLOC_ARENA_MAX} - -# Keep Python as PID 1 so watchdog os._exit(1) reaches Docker unchanged. Any -# future wrapper must exec the coordinator or explicitly propagate its status. -CMD ["python3", "-m", "lab.prism.prism_coordinator"] +RUN --mount=type=cache,target=/usr/local/cargo/registry \ + --mount=type=cache,target=/usr/local/cargo/git \ + cargo build --locked --release -p qbit-prism-server -p qbit-prism --bins + +# The runtime base is pinned by the multi-arch index digest of bookworm-slim; +# move the digest deliberately to take a Debian point or security release. +# Packages still resolve from the archive at build time: Debian drops +# superseded versions, so exact apt pins would break the next rebuild. +FROM debian:bookworm-slim@sha256:88200866dfff7ea7f5cbcb6ec7c8a701889efe6fe859fe64d6990e4b07ea4171 +# hadolint ignore=DL3008 +RUN apt-get update \ + && apt-get install -y --no-install-recommends ca-certificates curl \ + && rm -rf /var/lib/apt/lists/* \ + && groupadd --system --gid 10001 prism \ + && useradd --system --uid 10001 --gid prism --home-dir /nonexistent \ + --no-create-home --shell /usr/sbin/nologin prism \ + && install -d -m 0750 -o prism -g prism /var/lib/qbit-prism/audit +WORKDIR /app +COPY --from=build /build/target/release/qbit-prism-server /usr/local/bin/ +COPY --from=build /build/target/release/qbit-prism-miner /usr/local/bin/ +COPY --from=build /build/target/release/qbit-prism-audit-canonicalize /usr/local/bin/ +COPY --from=build /build/target/release/qbit-prism-audit-verify /usr/local/bin/ +COPY --from=build /build/target/release/qbit-prism-build-audit-bundle /usr/local/bin/ +COPY --from=build /build/target/release/qbit-prism-reorg-verify /usr/local/bin/ +ENV RUST_LOG=info +EXPOSE 3340 3341 +STOPSIGNAL SIGTERM +# A numeric UID lets orchestrators verify the non-root policy. Mounted secrets +# and the audit directory must be readable by UID/GID 10001. +USER 10001:10001 +# Mining frontends hold signing seeds in memory (#260 D4), so a core dump could +# write them to disk. Lowering the soft and hard limits here holds under plain +# `docker run` too; exec keeps the command as PID 1 to receive SIGTERM. +ENTRYPOINT ["/bin/sh", "-c", "ulimit -c 0 && exec \"$@\"", "qbit-prism"] +# Probes the operator listener, sending PRISM_OPERATOR_BEARER_TOKEN when set, +# or Stratum when that listener is disabled. The public-api role overrides it +# with `healthcheck --public-api`; run qbit-prism-miner and one-shot tools with +# --no-healthcheck. +HEALTHCHECK --interval=5s --timeout=5s --start-period=15s --retries=3 \ + CMD ["qbit-prism-server", "healthcheck"] +CMD ["qbit-prism-server"] diff --git a/lab/prism/__init__.py b/lab/prism/__init__.py deleted file mode 100644 index 268ab3ecb..000000000 --- a/lab/prism/__init__.py +++ /dev/null @@ -1 +0,0 @@ -"""PRISM pool lab helpers.""" diff --git a/lab/prism/accepted_preview_telemetry.py b/lab/prism/accepted_preview_telemetry.py deleted file mode 100644 index 129078e8b..000000000 --- a/lab/prism/accepted_preview_telemetry.py +++ /dev/null @@ -1,723 +0,0 @@ -"""Shared telemetry contract for accepted-preview latency attribution. - -Why this module exists ----------------------- -Issue #224: ``PrismAcceptedPreviewPublicationLatencyHigh`` fired on Union -mainnet while the coordinator stayed healthy. The publication histogram -(``qbit_prism_accepted_block_preview_publication_seconds``) proves that a -tail exists between definitive node acceptance and the visible payout -preview, but nothing on ``/metrics`` says *which stretch of the landing* -owns a sample above the 4 s warning or the 5 s child wait budget. Logs -around the alert showed reconcile-driven full payout-window rescans and -publication-blocked retries, which is correlated evidence, not -attribution. - -Three implementation lanes fix that in parallel: the core landing path, -landing attribution, and ledger attribution. They must agree on metric -names, label vocabularies, snapshot shapes, and neutral record types -before any of them instruments a call site, or the families they export -drift and the operator gets three partial answers. This module is that -agreement. It is deliberately behavior-neutral: nothing here is called on -a hot path yet, nothing here changes payout, reconciliation, -finalization, candidate, ledger, queue, lock, timeout, publication, or -retry behavior, and the renderer exports every family from an empty -owner so the series exist from process start. - -Contract at a glance --------------------- -Every vocabulary below is a closed tuple. A metric label may only ever be -one of its members: the recorder raises on an unknown programmer-chosen -label (phase, caller, step, path) exactly as -:meth:`ReorgReconcilerService.record_lookup` does, and *normalizes* the two -values that arrive as runtime strings (a payout-window rescan reason, a -ledger read operation name) onto a fixed ``other`` member rather than -growing a series. Block hashes, heights, miner IDs, exception text and -caller-supplied operation strings never become labels; the per-publication -diagnostic record is where a hash or height may appear, bounded by a -small ring. - -``PRISM_ACCEPTED_LANDING_PHASES`` - Sub-phases of one accepted-block landing, in the order the landing - runs them (``BlockFinalizationService._land_and_confirm_block_candidate`` - and the accounting lane around it): - - ``lane_wait`` - definitive node acceptance stamped on the submitter thread - (``_note_accepted_block_preview_acceptance``) to the accounting - lane starting the landing task. - ``balance_lock_wait`` - waiting for ``_payout_balance_mutation_lock`` under - ``_block_submitter_lock(..., "payout-balance-mutation")``. - ``reconcile`` - the inline ``ensure_reorg_reconciled_for_tip`` stretch, the - ``reorg-reconcile`` heartbeat phase. - ``prior_balances_check`` - ``_stamped_prior_balances_match_current`` and the live - ``current_prior_balances`` comparison before the verified preview. - This is a prior-balances *reread*; it is never a payout-window - rescan (see the rescan family below). - ``chain_probe`` - the active-chain reads: ``pool-block-state``, ``tip-height-rpc`` - and ``_block_candidate_chain_probe``. - ``preview_prepare`` - materializing the compact preview from the issued job summary or - deriving the verified preview from the audit bundle. - ``preview_publish`` - ``_publish_accepted_block_payout_preview`` itself, both the - generation publication and the degraded fenced branch. - - Family: ``qbit_prism_accepted_block_landing_phase_seconds{phase}`` - as ``_sum`` / ``_count`` / ``_max``, the same shape as - ``qbit_prism_block_finalization_phase_seconds``. - -``PRISM_REORG_RECONCILE_CALLERS`` / ``PRISM_REORG_RECONCILE_STEPS`` - Which path asked for a reconcile pass and where the pass spent its - wall-clock. Callers: ``landing`` (the in-lock - ``ensure_reorg_reconciled_for_tip(..., _coalesce_same_tip=False)``), - ``post_confirm`` (the forced-publish - ``reconcile_prism_pool_blocks_once(_force_publish=True)`` after a - durable confirmation), ``tip_refresh``, ``job_build`` (spelled exactly - as ``PRISM_REORG_RECONCILE_LOOKUP_PATHS`` spells them, so the two - families join), and ``other`` for tools, benchmarks and periodic - passes. Steps follow ``ReorgReconcilerService.reconcile``: - ``admission_wait`` (the serialized adapter's writer admission plus the - payout-balance mutation lock, added because that wait is the one - stretch a tip-refresh caller spends behind a landing and it is not a - step of the pass itself), ``watch_query`` (``reorg_watch_blocks``), - ``chain_probe`` (``getblockcount`` / ``getblockhash`` per watched or - stranded row), ``mutations`` (``mark_pool_block_inactive``, - ``reactivate_pool_block``, ``reject_prepared_block``), - ``candidate_prepare`` (``prepared_candidate``, which is where a - ``reconcile_invalidation`` full rescan actually executes) and - ``publish`` (``publish_candidate``). - - Families: ``qbit_prism_reorg_reconcile_pass_seconds{caller}`` and - ``qbit_prism_reorg_reconcile_step_seconds{caller,step}``, each as - ``_sum`` / ``_count`` / ``_max``. The pass count by caller is the - pass family's ``_count``; pass minus the sum of its steps is the - unattributed remainder. - -``PRISM_PAYOUT_WINDOW_FULL_RESCAN_REASONS`` / ``..._PATHS`` - Full payout-window oracle rescans by the reason that forced them and - the window pipeline that executed the fold. The reason vocabulary is - the set of literals ``lab.prism.payout_state`` hands to - ``_full_payout_window_materialization(reason=...)`` or records as the - periodic self-check reason; it lives here only because that module - keeps them as scattered literals rather than a tuple, and - ``tests.test_prism_accepted_preview_telemetry`` parses that module to - fail the moment a literal appears that this tuple does not carry. - ``normalize_payout_window_full_rescan_reason`` maps anything else to - ``other``. Paths: ``daemon`` (the Rust ``--serve`` window pipeline) and - ``in_process`` (the Python fold, including the legacy whole-snapshot - read for ledgers without the incremental record contract). - - Family: ``qbit_prism_payout_window_full_rescan_seconds{reason,path}`` - as ``_sum`` / ``_count`` / ``_max``. A prior-balances reread is *not* - an observation of this family: it is a ledger read operation - (``current_prior_balances``) and, on the landing, the - ``prior_balances_check`` phase. Wave 1 must never encode the two as - one event. - -``PRISM_LEDGER_READ_OPERATIONS`` - The closed operation vocabulary for the existing - ``qbit_prism_ledger_read_*`` families, which already split - coordinator-local admission (``gate_wait``) from PostgreSQL execution - (``execute``) per operation. ``pending_block_candidate_rows`` is the - one operation shipped today; ``payout_window_snapshot`` - (``snapshot_at_job_issue``), ``payout_window_delta`` - (``snapshot_between_job_issues``), ``current_prior_balances`` and - ``prior_balances_after_pool_block`` are the names the ledger lane must - use when it routes those statements through the attributed read path. - ``fold_ledger_read_stats`` merges any other operation name into - ``other`` before rendering so the label stays bounded even if a call - site drifts. - -``AcceptedPreviewPublicationDiagnostic`` - One bounded per-publication record for structured logging or the - small in-process ring (``PRISM_ACCEPTED_PREVIEW_DIAGNOSTICS_CAPACITY`` - entries, oldest evicted). It carries the block hash and height that - the metrics must not, the publication result, the acceptance-to- - publication interval, the per-phase seconds, the reconcile caller, the - rescan reason/path/seconds and the landing's ledger gate/execute - totals. Every field is a fixed-vocabulary string, a number, or None. - -Verified invariant: reconciliation and the share ledger -------------------------------------------------------- -A reconcile pass may change pool-block and payout balance state. Its -three mutators -- ``mark_pool_block_inactive``, ``reactivate_pool_block`` -and ``reject_prepared_block`` -- call ``qbit_mark_pool_block_inactive``, -``qbit_reactivate_pool_block`` and ``qbit_reject_prepared_pool_block``, -which update ``qbit_pool_blocks``, ``qbit_pool_payout_entries``, -``qbit_payout_carry_forward``, ``qbit_ctv_fanout_artifacts`` and the -writer-lease row. Neither those functions, the ``qbit_confirm_pool_block`` -and ``qbit_reverse_immature_pool_block`` helpers they call, nor any other -statement in ``crates/qbit-prism/sql/001_share_ledger.sql`` updates or -deletes from ``qbit_share_ledger``; the only writers of that table are the -share-append inserts in ``lab.prism.share_ledger``. So a reconcile pass -can move the prior-balances base the next preview must observe, but it -cannot change the contents of an accepted-share window at a fixed anchor. - -The optimization that follows from this -- answering a -``reconcile_invalidation`` with a prior-balances reread instead of a full -window rescan -- relies on exactly that invariant. It is **not** -implemented in this wave. This module only makes sure the two events are -distinguishable once it is. - -Handoff -------- -Every lane obtains the shared owner with -``ensure_accepted_preview_telemetry(runtime)`` where ``runtime`` is the -coordinator (the object every service already holds as ``self.runtime`` / -``self._runtime`` / ``self._coordinator``); the renderer obtains it from -its port the same way, so no coordinator attribute or ``_ensure_*`` method -needs adding and the lanes do not collide in ``prism_coordinator.py``. -""" - -from __future__ import annotations - -from collections import deque -from contextlib import contextmanager -from dataclasses import dataclass, field, replace as dataclass_replace -import math -import threading -import time -from types import MappingProxyType -from typing import Any, Callable, Iterator, Mapping - - -# --- closed vocabularies --------------------------------------------------- - -LANDING_PHASE_LANE_WAIT = "lane_wait" -LANDING_PHASE_BALANCE_LOCK_WAIT = "balance_lock_wait" -LANDING_PHASE_RECONCILE = "reconcile" -LANDING_PHASE_PRIOR_BALANCES_CHECK = "prior_balances_check" -LANDING_PHASE_CHAIN_PROBE = "chain_probe" -LANDING_PHASE_PREVIEW_PREPARE = "preview_prepare" -LANDING_PHASE_PREVIEW_PUBLISH = "preview_publish" -PRISM_ACCEPTED_LANDING_PHASES = ( - LANDING_PHASE_LANE_WAIT, - LANDING_PHASE_BALANCE_LOCK_WAIT, - LANDING_PHASE_RECONCILE, - LANDING_PHASE_PRIOR_BALANCES_CHECK, - LANDING_PHASE_CHAIN_PROBE, - LANDING_PHASE_PREVIEW_PREPARE, - LANDING_PHASE_PREVIEW_PUBLISH, -) - -RECONCILE_CALLER_LANDING = "landing" -RECONCILE_CALLER_POST_CONFIRM = "post_confirm" -RECONCILE_CALLER_TIP_REFRESH = "tip_refresh" -RECONCILE_CALLER_JOB_BUILD = "job_build" -RECONCILE_CALLER_OTHER = "other" -PRISM_REORG_RECONCILE_CALLERS = ( - RECONCILE_CALLER_LANDING, - RECONCILE_CALLER_POST_CONFIRM, - RECONCILE_CALLER_TIP_REFRESH, - RECONCILE_CALLER_JOB_BUILD, - RECONCILE_CALLER_OTHER, -) - -RECONCILE_STEP_ADMISSION_WAIT = "admission_wait" -RECONCILE_STEP_WATCH_QUERY = "watch_query" -RECONCILE_STEP_CHAIN_PROBE = "chain_probe" -RECONCILE_STEP_MUTATIONS = "mutations" -RECONCILE_STEP_CANDIDATE_PREPARE = "candidate_prepare" -RECONCILE_STEP_PUBLISH = "publish" -PRISM_REORG_RECONCILE_STEPS = ( - RECONCILE_STEP_ADMISSION_WAIT, - RECONCILE_STEP_WATCH_QUERY, - RECONCILE_STEP_CHAIN_PROBE, - RECONCILE_STEP_MUTATIONS, - RECONCILE_STEP_CANDIDATE_PREPARE, - RECONCILE_STEP_PUBLISH, -) - -FULL_RESCAN_REASON_OTHER = "other" -# Mirrors the literals in lab.prism.payout_state (see the module docstring -# and the AST drift test). Order is render order: the landing-forced reason -# first, then cache-lifecycle reasons, daemon reasons, incremental fallbacks, -# the periodic self-check family, and the fold-in bucket last. -PRISM_PAYOUT_WINDOW_FULL_RESCAN_REASONS = ( - "reconcile_invalidation", - "cold_start", - "cache_invalidated", - "late_visible_append", - "anchor_regression", - "snapshot_window_weight_out_of_band", - "window_pipeline_mode_changed", - "window_mirror_divergence", - "window_daemon_unavailable", - "window_daemon_busy", - "window_daemon_state_lost", - "window_value_out_of_range", - "delta_api_unavailable", - "incremental_invariant_failed", - "periodic_self_check", - "periodic_self_check_failed", - "periodic_self_check_balance_check_failed", - FULL_RESCAN_REASON_OTHER, -) - -FULL_RESCAN_PATH_DAEMON = "daemon" -FULL_RESCAN_PATH_IN_PROCESS = "in_process" -PRISM_PAYOUT_WINDOW_FULL_RESCAN_PATHS = ( - FULL_RESCAN_PATH_DAEMON, - FULL_RESCAN_PATH_IN_PROCESS, -) - -LEDGER_READ_OPERATION_PENDING_BLOCK_CANDIDATE_ROWS = "pending_block_candidate_rows" -LEDGER_READ_OPERATION_PAYOUT_WINDOW_SNAPSHOT = "payout_window_snapshot" -LEDGER_READ_OPERATION_PAYOUT_WINDOW_DELTA = "payout_window_delta" -LEDGER_READ_OPERATION_CURRENT_PRIOR_BALANCES = "current_prior_balances" -LEDGER_READ_OPERATION_PRIOR_BALANCES_AFTER_POOL_BLOCK = ( - "prior_balances_after_pool_block" -) -LEDGER_READ_OPERATION_OTHER = "other" -PRISM_LEDGER_READ_OPERATIONS = ( - "observe_block_candidate_outbox_row", - "pending_block_candidate_headers", - "read_block_candidate_body_manifest", - "read_block_candidate_body_page", - "read_block_candidate_body_pages", - "read_block_candidate_body_spans", - LEDGER_READ_OPERATION_PENDING_BLOCK_CANDIDATE_ROWS, - LEDGER_READ_OPERATION_PAYOUT_WINDOW_SNAPSHOT, - LEDGER_READ_OPERATION_PAYOUT_WINDOW_DELTA, - LEDGER_READ_OPERATION_CURRENT_PRIOR_BALANCES, - LEDGER_READ_OPERATION_PRIOR_BALANCES_AFTER_POOL_BLOCK, - LEDGER_READ_OPERATION_OTHER, -) - -# How many per-publication diagnostic records the in-process ring keeps. -# Sized for an operator reading the last hour of a dense accepted-tip -# cadence (the #224 alert hour held 35 accepted blocks), not as a log. -PRISM_ACCEPTED_PREVIEW_DIAGNOSTICS_CAPACITY = 64 - - -def normalize_payout_window_full_rescan_reason(reason: object) -> str: - """Bound a payout_state reason string to the closed vocabulary.""" - if isinstance(reason, str) and reason in PRISM_PAYOUT_WINDOW_FULL_RESCAN_REASONS: - return reason - return FULL_RESCAN_REASON_OTHER - - -def normalize_ledger_read_operation(operation: object) -> str: - """Bound a ledger read operation name to the closed vocabulary.""" - if isinstance(operation, str) and operation in PRISM_LEDGER_READ_OPERATIONS: - return operation - return LEDGER_READ_OPERATION_OTHER - - -def fold_ledger_read_stats( - stats_by_operation: Mapping[str, Mapping[str, float | int]], -) -> dict[str, dict[str, float | int]]: - """Merge out-of-contract operation names into ``other`` for rendering. - - Operations in :data:`PRISM_LEDGER_READ_OPERATIONS` pass through as - copies, so the shipped ``pending_block_candidate_rows`` series render - exactly as before. Anything else is summed into ``other`` -- ``*_max`` - fields take the maximum, every other field adds -- so an operation - name the ledger lane did not take from this contract still cannot open - a new series. - """ - folded: dict[str, dict[str, float | int]] = {} - for operation, stats in stats_by_operation.items(): - key = normalize_ledger_read_operation(operation) - target = folded.get(key) - if target is None: - folded[key] = dict(stats) - continue - for name, value in stats.items(): - if name.endswith("_max"): - target[name] = max(float(target.get(name, 0.0)), float(value)) - else: - target[name] = target.get(name, 0) + value - return folded - - -def empty_duration_stats() -> dict[str, float | int]: - """The sum/count/max cell every family here accumulates into.""" - return {"count": 0, "sum": 0.0, "max": 0.0} - - -def _accumulate(stats: dict[str, float | int], seconds: float) -> None: - elapsed = float(seconds) - if not math.isfinite(elapsed) or elapsed < 0.0: - elapsed = 0.0 - stats["count"] = int(stats["count"]) + 1 - stats["sum"] = float(stats["sum"]) + elapsed - stats["max"] = max(float(stats["max"]), elapsed) - - -def _publication_results() -> tuple[str, ...]: - # Imported lazily: block_candidates is the B1 owner module and will - # import this contract from its landing paths; a module-level import - # here would make that a cycle. - from lab.prism.block_candidates import ( - PRISM_ACCEPTED_BLOCK_PREVIEW_PUBLICATION_RESULTS, - ) - - return PRISM_ACCEPTED_BLOCK_PREVIEW_PUBLICATION_RESULTS - - -def _bounded_seconds(value: object) -> float: - seconds = float(value) # type: ignore[arg-type] - if not math.isfinite(seconds) or seconds < 0.0: - return 0.0 - return seconds - - -# --- neutral data types ---------------------------------------------------- - - -@dataclass(frozen=True, slots=True) -class AcceptedPreviewPublicationDiagnostic: - """One bounded record of an accepted block's preview publication. - - This is the only place in the contract where a block hash or height - may appear. It is a value for a structured log line or the diagnostics - ring, never a metric label. Construction validates every vocabulary - field and clamps every duration, so a record that exists is already - safe to log. - """ - - block_hash: str - block_height: int | None - result: str - acceptance_to_publication_seconds: float - phase_seconds: Mapping[str, float] = field(default_factory=dict) - reconcile_caller: str | None = None - full_rescan_reason: str | None = None - full_rescan_path: str | None = None - full_rescan_seconds: float = 0.0 - ledger_gate_wait_seconds: float = 0.0 - ledger_execute_seconds: float = 0.0 - recorded_monotonic: float | None = None - - def __post_init__(self) -> None: - block_hash = str(self.block_hash).strip().lower() - if not block_hash: - raise ValueError("diagnostic block_hash must be non-empty") - object.__setattr__(self, "block_hash", block_hash) - if self.block_height is not None: - object.__setattr__(self, "block_height", int(self.block_height)) - if self.result not in _publication_results(): - raise ValueError( - f"unknown accepted preview publication result: {self.result!r}" - ) - phases: dict[str, float] = { - phase: 0.0 for phase in PRISM_ACCEPTED_LANDING_PHASES - } - for phase, seconds in dict(self.phase_seconds).items(): - if phase not in phases: - raise ValueError(f"unknown accepted landing phase: {phase!r}") - phases[phase] = _bounded_seconds(seconds) - object.__setattr__(self, "phase_seconds", MappingProxyType(phases)) - if ( - self.reconcile_caller is not None - and self.reconcile_caller not in PRISM_REORG_RECONCILE_CALLERS - ): - raise ValueError( - f"unknown reorg reconcile caller: {self.reconcile_caller!r}" - ) - if self.full_rescan_reason is not None: - object.__setattr__( - self, - "full_rescan_reason", - normalize_payout_window_full_rescan_reason(self.full_rescan_reason), - ) - if ( - self.full_rescan_path is not None - and self.full_rescan_path not in PRISM_PAYOUT_WINDOW_FULL_RESCAN_PATHS - ): - raise ValueError( - f"unknown payout window full rescan path: {self.full_rescan_path!r}" - ) - for name in ( - "acceptance_to_publication_seconds", - "full_rescan_seconds", - "ledger_gate_wait_seconds", - "ledger_execute_seconds", - ): - object.__setattr__(self, name, _bounded_seconds(getattr(self, name))) - if self.recorded_monotonic is not None: - object.__setattr__( - self, "recorded_monotonic", float(self.recorded_monotonic) - ) - - def log_fields(self) -> dict[str, object]: - """Flat JSON-serializable fields for ``_payout_artifact_log`` style lines.""" - fields: dict[str, object] = { - "block_hash": self.block_hash, - "block_height": self.block_height, - "result": self.result, - "acceptance_to_publication_seconds": round( - self.acceptance_to_publication_seconds, 6 - ), - "reconcile_caller": self.reconcile_caller, - "full_rescan_reason": self.full_rescan_reason, - "full_rescan_path": self.full_rescan_path, - "full_rescan_seconds": round(self.full_rescan_seconds, 6), - "ledger_gate_wait_seconds": round(self.ledger_gate_wait_seconds, 6), - "ledger_execute_seconds": round(self.ledger_execute_seconds, 6), - } - for phase in PRISM_ACCEPTED_LANDING_PHASES: - fields[f"phase_{phase}_seconds"] = round(self.phase_seconds[phase], 6) - return fields - - def summary(self) -> str: - """One line naming the owner of the interval, for plain prints.""" - phases = " ".join( - f"{phase}={self.phase_seconds[phase]:.3f}s" - for phase in PRISM_ACCEPTED_LANDING_PHASES - ) - rescan = ( - f" rescan={self.full_rescan_reason}/{self.full_rescan_path}" - f"={self.full_rescan_seconds:.3f}s" - if self.full_rescan_reason is not None - else "" - ) - return ( - f"hash={self.block_hash} height={self.block_height} " - f"result={self.result} " - f"total={self.acceptance_to_publication_seconds:.3f}s {phases}" - f"{rescan} ledger_gate_wait={self.ledger_gate_wait_seconds:.3f}s " - f"ledger_execute={self.ledger_execute_seconds:.3f}s" - ) - - -# --- the shared owner ------------------------------------------------------ - - -class AcceptedPreviewTelemetry: - """Fixed-cardinality accumulators plus the bounded diagnostics ring. - - One lock guards every cell, the same model as the B1 owner's - publication histogram and the finalization service's phase metrics: - recorders take it for one dictionary update, ``snapshot`` copies every - cell under it and returns plain dicts the renderer may read without - it. Every cell of every closed product exists from construction, so an - empty owner renders every series at zero and a scrape never sees a - family appear mid-life. - """ - - def __init__( - self, - *, - diagnostics_capacity: int = PRISM_ACCEPTED_PREVIEW_DIAGNOSTICS_CAPACITY, - monotonic: Callable[[], float] | None = None, - ) -> None: - capacity = int(diagnostics_capacity) - if capacity <= 0: - raise ValueError("diagnostics_capacity must be positive") - self._monotonic = monotonic or time.monotonic - self._lock = threading.Lock() - self._landing_phases = { - phase: empty_duration_stats() for phase in PRISM_ACCEPTED_LANDING_PHASES - } - self._reconcile_passes = { - caller: empty_duration_stats() for caller in PRISM_REORG_RECONCILE_CALLERS - } - self._reconcile_steps = { - (caller, step): empty_duration_stats() - for caller in PRISM_REORG_RECONCILE_CALLERS - for step in PRISM_REORG_RECONCILE_STEPS - } - self._full_rescans = { - (reason, path): empty_duration_stats() - for reason in PRISM_PAYOUT_WINDOW_FULL_RESCAN_REASONS - for path in PRISM_PAYOUT_WINDOW_FULL_RESCAN_PATHS - } - self._diagnostics: deque[AcceptedPreviewPublicationDiagnostic] = deque( - maxlen=capacity - ) - self._diagnostics_recorded = 0 - - # -- recorders --------------------------------------------------------- - - def observe_landing_phase(self, phase: str, seconds: float) -> None: - if phase not in self._landing_phases: - raise ValueError(f"unknown accepted landing phase: {phase!r}") - with self._lock: - _accumulate(self._landing_phases[phase], seconds) - - @contextmanager - def landing_phase(self, phase: str) -> Iterator[None]: - """Time one landing sub-phase; records on every exit, raising or not.""" - if phase not in self._landing_phases: - raise ValueError(f"unknown accepted landing phase: {phase!r}") - started = self._monotonic() - try: - yield - finally: - self.observe_landing_phase(phase, self._monotonic() - started) - - def observe_reconcile_pass(self, caller: str, seconds: float) -> None: - if caller not in self._reconcile_passes: - raise ValueError(f"unknown reorg reconcile caller: {caller!r}") - with self._lock: - _accumulate(self._reconcile_passes[caller], seconds) - - def observe_reconcile_step(self, caller: str, step: str, seconds: float) -> None: - key = (caller, step) - if key not in self._reconcile_steps: - if caller not in PRISM_REORG_RECONCILE_CALLERS: - raise ValueError(f"unknown reorg reconcile caller: {caller!r}") - raise ValueError(f"unknown reorg reconcile step: {step!r}") - with self._lock: - _accumulate(self._reconcile_steps[key], seconds) - - @contextmanager - def reconcile_step(self, caller: str, step: str) -> Iterator[None]: - """Time one reconcile step for one caller; records on every exit.""" - if (caller, step) not in self._reconcile_steps: - if caller not in PRISM_REORG_RECONCILE_CALLERS: - raise ValueError(f"unknown reorg reconcile caller: {caller!r}") - raise ValueError(f"unknown reorg reconcile step: {step!r}") - started = self._monotonic() - try: - yield - finally: - self.observe_reconcile_step(caller, step, self._monotonic() - started) - - def observe_payout_window_full_rescan( - self, - reason: object, - path: str, - seconds: float, - ) -> None: - """Record one executed full oracle rescan. - - ``reason`` is the payout_state string and is normalized; ``path`` - is chosen by the recording code and must be a vocabulary member. - A prior-balances reread must not be recorded here. - """ - if path not in PRISM_PAYOUT_WINDOW_FULL_RESCAN_PATHS: - raise ValueError(f"unknown payout window full rescan path: {path!r}") - key = (normalize_payout_window_full_rescan_reason(reason), path) - with self._lock: - _accumulate(self._full_rescans[key], seconds) - - def record_publication_diagnostic( - self, - diagnostic: AcceptedPreviewPublicationDiagnostic, - ) -> AcceptedPreviewPublicationDiagnostic: - """Retain one publication record, stamping it if the caller did not.""" - if not isinstance(diagnostic, AcceptedPreviewPublicationDiagnostic): - raise TypeError("expected an AcceptedPreviewPublicationDiagnostic") - if diagnostic.recorded_monotonic is None: - diagnostic = dataclass_replace( - diagnostic, recorded_monotonic=self._monotonic() - ) - with self._lock: - self._diagnostics.append(diagnostic) - self._diagnostics_recorded += 1 - return diagnostic - - # -- readers ----------------------------------------------------------- - - def snapshot(self) -> dict[str, Any]: - """Copied cells for one render; keys are the closed products.""" - with self._lock: - return { - "landing_phases": { - phase: dict(stats) - for phase, stats in self._landing_phases.items() - }, - "reconcile_passes": { - caller: dict(stats) - for caller, stats in self._reconcile_passes.items() - }, - "reconcile_steps": { - key: dict(stats) for key, stats in self._reconcile_steps.items() - }, - "full_rescans": { - key: dict(stats) for key, stats in self._full_rescans.items() - }, - "diagnostics_retained": len(self._diagnostics), - "diagnostics_recorded": self._diagnostics_recorded, - } - - def diagnostics_snapshot( - self, - ) -> tuple[AcceptedPreviewPublicationDiagnostic, ...]: - """Retained publication records, oldest first.""" - with self._lock: - return tuple(self._diagnostics) - - -_OWNER_ATTRIBUTE = "_accepted_preview_telemetry" -_OWNER_BOOTSTRAP_LOCK = threading.Lock() - - -def ensure_accepted_preview_telemetry(owner: object) -> AcceptedPreviewTelemetry: - """The one telemetry owner attached to ``owner`` (normally the coordinator). - - Idempotent and race-free: the first caller from any thread creates - the instance under a module bootstrap lock, every later caller gets - that same instance. State lives in the owner's instance ``__dict__`` - exactly as the coordinator's lazily built renderer does, so no - coordinator attribute, descriptor or ``_ensure_*`` method has to be - added for the lanes or the renderer to share it. - """ - try: - state = vars(owner) - except TypeError as exc: - raise TypeError( - "accepted preview telemetry needs an owner with an instance " - f"__dict__, got {type(owner).__name__}" - ) from exc - telemetry = state.get(_OWNER_ATTRIBUTE) - if telemetry is not None: - return telemetry - with _OWNER_BOOTSTRAP_LOCK: - telemetry = state.get(_OWNER_ATTRIBUTE) - if telemetry is None: - telemetry = AcceptedPreviewTelemetry() - state[_OWNER_ATTRIBUTE] = telemetry - return telemetry - - -__all__ = [ - "AcceptedPreviewPublicationDiagnostic", - "AcceptedPreviewTelemetry", - "FULL_RESCAN_PATH_DAEMON", - "FULL_RESCAN_PATH_IN_PROCESS", - "FULL_RESCAN_REASON_OTHER", - "LANDING_PHASE_BALANCE_LOCK_WAIT", - "LANDING_PHASE_CHAIN_PROBE", - "LANDING_PHASE_LANE_WAIT", - "LANDING_PHASE_PREVIEW_PREPARE", - "LANDING_PHASE_PREVIEW_PUBLISH", - "LANDING_PHASE_PRIOR_BALANCES_CHECK", - "LANDING_PHASE_RECONCILE", - "LEDGER_READ_OPERATION_CURRENT_PRIOR_BALANCES", - "LEDGER_READ_OPERATION_OTHER", - "LEDGER_READ_OPERATION_PAYOUT_WINDOW_DELTA", - "LEDGER_READ_OPERATION_PAYOUT_WINDOW_SNAPSHOT", - "LEDGER_READ_OPERATION_PENDING_BLOCK_CANDIDATE_ROWS", - "LEDGER_READ_OPERATION_PRIOR_BALANCES_AFTER_POOL_BLOCK", - "PRISM_ACCEPTED_LANDING_PHASES", - "PRISM_ACCEPTED_PREVIEW_DIAGNOSTICS_CAPACITY", - "PRISM_LEDGER_READ_OPERATIONS", - "PRISM_PAYOUT_WINDOW_FULL_RESCAN_PATHS", - "PRISM_PAYOUT_WINDOW_FULL_RESCAN_REASONS", - "PRISM_REORG_RECONCILE_CALLERS", - "PRISM_REORG_RECONCILE_STEPS", - "RECONCILE_CALLER_JOB_BUILD", - "RECONCILE_CALLER_LANDING", - "RECONCILE_CALLER_OTHER", - "RECONCILE_CALLER_POST_CONFIRM", - "RECONCILE_CALLER_TIP_REFRESH", - "RECONCILE_STEP_ADMISSION_WAIT", - "RECONCILE_STEP_CANDIDATE_PREPARE", - "RECONCILE_STEP_CHAIN_PROBE", - "RECONCILE_STEP_MUTATIONS", - "RECONCILE_STEP_PUBLISH", - "RECONCILE_STEP_WATCH_QUERY", - "empty_duration_stats", - "ensure_accepted_preview_telemetry", - "fold_ledger_read_stats", - "normalize_ledger_read_operation", - "normalize_payout_window_full_rescan_reason", -] diff --git a/lab/prism/audit_artifacts.py b/lab/prism/audit_artifacts.py deleted file mode 100644 index f28ce7bf6..000000000 --- a/lab/prism/audit_artifacts.py +++ /dev/null @@ -1,6154 +0,0 @@ -#!/usr/bin/env python3 -"""Filesystem ownership, verification, and publication for PRISM audit data. - -This module deliberately has no coordinator dependency. The coordinator owns -block-finalization sequencing and the ledger owns database authorization; this -store is the sole authority for paths and filesystem mutation below the audit -root. -""" - -from __future__ import annotations - -import copy -from contextlib import contextmanager, nullcontext -import fcntl -import gzip -import hashlib -import hmac -import io -import json -import os -import re -import selectors -import signal -import stat -import subprocess -import threading -import time -import uuid -import zlib -from dataclasses import dataclass -from pathlib import Path -from collections.abc import Sequence -from typing import Any, Callable, Iterable, Iterator, Mapping - -from lab.prism.audit_bundle_view import ( - ArtifactSource, - CanonicalArtifactError, - CanonicalAuditBundleView, - RawJsonRecord, - iter_json_byte_chunks, - json_chunks_sha256_and_size, - materialize_json, - spool_json_document, -) -from lab.prism.prism_tools import prism_tool_command - - -AUDIT_BODY_REF_SCHEMA = "qbit.prism.audit-body-ref.v1" -AUDIT_BUNDLE_V2_SCHEMA = "qbit.prism.audit-bundle.v2" -AUDIT_SHARE_SEGMENT_SCHEMA = "qbit.prism.audit-share-segment.v1" -AUDIT_WINDOW_COMPLETENESS_PROOF_SCHEMA = ( - "qbit.prism.window-completeness-proof.v1" -) -LIVE_ENVELOPE_SCHEMA = "qbit.prism.live-audit-bundle-envelope.v1" -LIVE_EVIDENCE_SCHEMA = "qbit.prism.live-stratum-evidence.v1" -DEFAULT_AUDIT_SHARE_SEGMENT_SIZE = 10_000 -DEFAULT_VERIFIER_TIMEOUT_SECONDS = 60.0 -MAX_VERIFIER_OUTPUT_BYTES = 1024 * 1024 -VERIFICATION_REPORT_SCHEMA = "qbit.prism.audit-verification-report.v1" -VERIFICATION_IDENTITY_SCHEMA = "qbit.prism.audit-verification-identity.v1" -LEGACY_VERIFICATION_UNAVAILABLE_SCHEMA = ( - "qbit.prism.legacy-verification-unavailable.v1" -) -# The canonical bundle is stored compressed, but its public identity stays -# the SHA-256 of the *uncompressed* canonical JSON. Level 9 with mtime=0 -# keeps stdlib gzip reproducible for a given compressor; republication checks -# the uncompressed payload so a compressor change cannot redefine identity. -CANONICAL_BUNDLE_GZIP_LEVEL = 9 - -_HEX_64 = r"[0-9a-f]{64}" -_TOKEN_32 = r"[0-9a-f]{32}" -_BODY_RE = re.compile( - rf"\Aprism-audit-bundle-body-(?P<block>{_HEX_64})-(?P<digest>{_HEX_64})\.json\Z" -) -_SHARE_CONTENT_RE = re.compile( - rf"\Aprism-audit-share-segment-(?P<first>[1-9][0-9]*)-" - rf"(?P<last>[1-9][0-9]*)-(?P<digest>{_HEX_64})\.json\Z" -) -_SHARE_SLOT_RE = re.compile( - r"\Aprism-audit-share-segment-slot-(?P<first>[1-9][0-9]*)-" - r"(?P<last>[1-9][0-9]*)\.json\Z" -) -_CANONICAL_BUNDLE_RE = re.compile( - rf"\Aprism-audit-bundle-canonical-(?P<block>{_HEX_64})-" - rf"(?P<digest>{_HEX_64})\.json\.gz\Z" -) -_LIVE_RE = re.compile( - rf"\Aprism-live-audit-bundle-(?P<height>0|[1-9][0-9]*)-" - rf"(?P<block>{_HEX_64})\.json\Z" -) -_CANDIDATE_RE = re.compile( - rf"\A\.prism-live-audit-bundle-candidate-(?P<block>{_HEX_64})-" - rf"(?P<token>{_TOKEN_32})\.json\.tmp\Z" -) -_LEGACY_CANDIDATE_RE = re.compile( - rf"\Aprism-live-audit-bundle-candidate-(?P<block>{_HEX_64})\.json\Z" -) -_LEGACY_HIDDEN_CANDIDATE_RE = re.compile( - rf"\A\.prism-live-audit-bundle-candidate-(?P<block>{_HEX_64})\.json\.tmp\Z" -) -_PUBLICATION_LOCK_NAME = ".prism-audit-publication.lock" -_PUBLICATION_THREAD_LOCKS_GUARD = threading.Lock() -_PUBLICATION_THREAD_LOCKS: dict[tuple[int, int], threading.RLock] = {} -_PUBLICATION_LOCAL_OWNERS: dict[tuple[int, int], tuple[int, int]] = {} - - -def _publication_thread_lock(identity: tuple[int, int]) -> threading.RLock: - """Share a local lock across store instances opened on the same inode.""" - - with _PUBLICATION_THREAD_LOCKS_GUARD: - lock = _PUBLICATION_THREAD_LOCKS.get(identity) - if lock is None: - lock = threading.RLock() - _PUBLICATION_THREAD_LOCKS[identity] = lock - return lock - - -def _reject_cross_store_local_owner( - identity: tuple[int, int], - *, - thread_id: int, - store_id: int, -) -> None: - """Reject same-thread nesting through a second store before it can block.""" - - with _PUBLICATION_THREAD_LOCKS_GUARD: - local_owner = _PUBLICATION_LOCAL_OWNERS.get(identity) - if local_owner is not None: - owner_thread, owner_store = local_owner - if owner_thread == thread_id and owner_store != store_id: - raise RuntimeError( - "audit publication guard is already held by another store " - "on this thread" - ) - - -def _canonical_hex(value: object, *, name: str, expected_bytes: int = 32) -> str: - text = str(value) - if len(text) != expected_bytes * 2: - raise ValueError(f"{name} must be {expected_bytes} bytes of hex") - try: - bytes.fromhex(text) - except ValueError as exc: - raise ValueError(f"{name} must be hexadecimal") from exc - return text.lower() - - -def _sha256_bytes(payload: bytes) -> str: - return hashlib.sha256(payload).hexdigest() - - -def _json_bytes(payload: Mapping[str, Any], *, sort_keys: bool = False) -> bytes: - return json.dumps( - payload, - separators=(",", ":"), - sort_keys=sort_keys, - ).encode("utf-8") - - -def canonical_audit_bundle_bytes( - final_bundle: dict[str, Any], - canonicalizer: Callable[[dict[str, Any]], bytes] | None = None, -) -> bytes: - if canonicalizer is None: - raise RuntimeError("J1 canonical bundle capability is required") - canonical = canonicalizer(final_bundle) - return canonical.encode() if isinstance(canonical, str) else bytes(canonical) - - -def gzip_canonical_bundle_bytes(payload: bytes) -> bytes: - """Compress canonical bundle bytes reproducibly. - - A BytesIO sink carries no name, so gzip emits no FNAME field; with mtime=0 - the header is constant and equal inputs are reproducible for a given - compressor implementation. - """ - - sink = io.BytesIO() - with gzip.GzipFile( - fileobj=sink, - mode="wb", - compresslevel=CANONICAL_BUNDLE_GZIP_LEVEL, - mtime=0, - ) as handle: - handle.write(payload) - return sink.getvalue() - - -def gunzip_canonical_bundle_bytes(payload: bytes) -> bytes: - return gzip.decompress(payload) - - -class CanonicalAuditBundleCorrupt(RuntimeError): - """A canonical bundle artifact is present but unusable. - - Distinct from a clean miss so public readers can report the damage and fail - closed instead of silently serving a non-byte-exact reconstruction. - """ - - -class _AuditShareSegmentConflict(RuntimeError): - """A stable-slot merge saw two different payloads for one share_seq.""" - - -@dataclass(frozen=True) -class AuditArtifactConfig: - root: Path - evidence_path: Path - live_bundle_retention: int = 5 - candidate_retention_seconds: int = 24 * 60 * 60 - share_segment_size: int = 0 - verifier_timeout_seconds: float = DEFAULT_VERIFIER_TIMEOUT_SECONDS - - -@dataclass(frozen=True) -class AuditPublicationIdentity: - """Ordering token assigned while P1's balance serializer is held.""" - - sequence: int - block_height: int - block_hash: str - - def __post_init__(self) -> None: - if isinstance(self.sequence, bool) or not isinstance(self.sequence, int): - raise ValueError("publication sequence must be an integer") - if self.sequence < 0: - raise ValueError("publication sequence must be non-negative") - if isinstance(self.block_height, bool) or not isinstance( - self.block_height, - int, - ): - raise ValueError("publication block height must be an integer") - if self.block_height < 0: - raise ValueError("publication block height must be non-negative") - canonical_block_hash = _canonical_hex( - self.block_hash, - name="publication block hash", - ) - if self.block_hash != canonical_block_hash: - raise ValueError("publication block hash must be canonical") - - def to_json(self) -> dict[str, object]: - return { - "sequence": self.sequence, - "block_height": self.block_height, - "block_hash": self.block_hash, - } - - -@dataclass(frozen=True) -class OwnedCandidateArtifact: - path: Path - block_hash: str - token: str - store_token: str - - -@dataclass(frozen=True) -class VerifiedAuditBundle: - candidate: OwnedCandidateArtifact - report: Mapping[str, Any] - literal_sha256: str - byte_length: int - device: int - inode: int - mtime_ns: int - canonical_copy_eligible: bool - verification_identity: Mapping[str, Any] - - -@dataclass(frozen=True) -class PublishedAuditBodyRef: - path: Path - body_uri: str - audit_bundle_sha256: str - - -@dataclass(frozen=True) -class AuditPublication: - identity: AuditPublicationIdentity - envelope_path: Path - evidence: Mapping[str, Any] - published: bool - # True when this superseded publication restored its own missing live - # envelope. Recorded instead of logging inside the store's locked region; - # the caller emits the diagnostic after every lock scope has exited. - superseded_envelope_written: bool = False - - -@dataclass(frozen=True) -class RetentionResult: - live_removed: int = 0 - candidate_removed: int = 0 - errors: int = 0 - - -@dataclass(frozen=True) -class _FileIdentity: - device: int - inode: int - mode: int - size: int - mtime_ns: int - - @classmethod - def from_stat(cls, value: os.stat_result) -> _FileIdentity: - return cls( - value.st_dev, - value.st_ino, - value.st_mode, - value.st_size, - value.st_mtime_ns, - ) - - def matches(self, value: os.stat_result) -> bool: - return ( - self.device == value.st_dev - and self.inode == value.st_ino - and self.mode == value.st_mode - and self.size == value.st_size - and self.mtime_ns == value.st_mtime_ns - and stat.S_ISREG(value.st_mode) - ) - - -@dataclass(frozen=True) -class _LegacyProofToken: - identity: AuditPublicationIdentity - evidence_file: _FileIdentity - evidence_sha256: str - envelope_file: _FileIdentity - envelope_sha256: str - - -class AuditArtifactStore: - """Own every audit artifact path and filesystem mutation for one root.""" - - def __init__( - self, - config: AuditArtifactConfig, - *, - canonicalizer: Callable[[dict[str, Any]], bytes] | None = None, - verifier: Callable[..., dict[str, Any]] | None = None, - wall_time: Callable[[], float] = time.time, - read_only: bool = False, - ) -> None: - """Own one audit artifact root. - - ``read_only`` opens the root for reading published bodies and nothing - else. The publication lock is an ordinary file opened O_RDWR and - created on demand, so acquiring it needs a writable filesystem; the - extracted public read tier mounts this volume ``:ro`` precisely so it - cannot write, and would fail to construct a store at all. A read-only - store therefore skips the lock, and every publication path refuses - rather than running unserialized. - """ - live_bundle_retention = int(config.live_bundle_retention) - candidate_retention_seconds = int(config.candidate_retention_seconds) - share_segment_size = int(config.share_segment_size) - verifier_timeout_seconds = float(config.verifier_timeout_seconds) - if share_segment_size < 0: - raise ValueError("audit share segment size must be non-negative") - if verifier_timeout_seconds <= 0: - raise ValueError("verifier timeout must be positive") - configured_root = Path(config.root).expanduser().absolute() - if not read_only: - configured_root.mkdir(parents=True, exist_ok=True) - root_stat = configured_root.lstat() - if not stat.S_ISDIR(root_stat.st_mode) or stat.S_ISLNK(root_stat.st_mode): - raise RuntimeError("audit artifact root must be a non-symlink directory") - root = configured_root.resolve(strict=True) - configured_evidence = Path(config.evidence_path).expanduser().absolute() - if not read_only: - configured_evidence.parent.mkdir(parents=True, exist_ok=True) - evidence_path = configured_evidence.parent.resolve(strict=True) / configured_evidence.name - self._read_only = bool(read_only) - self._root = root - self._evidence_path = evidence_path - self._root_fd, self._root_identity = self._open_directory_authority(root) - try: - if read_only: - # No lock file, and no fd that could ever authorize a write. - self._publication_lock_fd = -1 - self._publication_lock_identity = None - self._publication_lifecycle_lock = threading.RLock() - self._publication_inode_thread_lock = threading.RLock() - else: - ( - self._publication_lock_fd, - self._publication_lock_identity, - ) = self._open_publication_lock_authority( - root, - self._root_fd, - self._root_identity, - ) - self._publication_lifecycle_lock = threading.RLock() - self._publication_inode_thread_lock = _publication_thread_lock( - ( - self._publication_lock_identity.device, - self._publication_lock_identity.inode, - ) - ) - self._publication_guard_owner: int | None = None - self._publication_guard_depth = 0 - ( - self._evidence_parent_fd, - self._evidence_parent_identity, - ) = self._open_directory_authority(evidence_path.parent) - except BaseException: - publication_lock_fd = getattr(self, "_publication_lock_fd", -1) - if publication_lock_fd >= 0: - os.close(publication_lock_fd) - self._publication_lock_fd = -1 - os.close(self._root_fd) - self._root_fd = -1 - raise - self._live_bundle_retention = live_bundle_retention - self._candidate_retention_seconds = candidate_retention_seconds - self._share_segment_size = share_segment_size - self._verifier_timeout_seconds = verifier_timeout_seconds - self._canonicalizer = canonicalizer - self._verifier = verifier - self._wall_time = wall_time - self._lock = threading.RLock() - self._closed = False - self._store_token = uuid.uuid4().hex - self._active_candidates: dict[str, tuple[Path, _FileIdentity | None]] = {} - self._latest_evidence: dict[str, Any] | None = None - self._current_envelope: Path | None = None - self._current_identity: AuditPublicationIdentity | None = None - self._evidence_state = "absent" - self._compatibility_evidence_override = False - self._invalidated_legacy_identity: AuditPublicationIdentity | None = None - self._legacy_proof_token: _LegacyProofToken | None = None - self._load_current_evidence() - - @property - def root(self) -> Path: - return self._root - - @property - def evidence_path(self) -> Path: - return self._evidence_path - - @property - def share_segment_size(self) -> int: - return self._share_segment_size - - @property - def live_bundle_retention(self) -> int: - return self._live_bundle_retention - - @property - def candidate_retention_seconds(self) -> int: - return self._candidate_retention_seconds - - @property - def verifier_timeout_seconds(self) -> float: - return self._verifier_timeout_seconds - - @staticmethod - def _directory_identity(path: Path) -> tuple[int, int]: - value = path.lstat() - if not stat.S_ISDIR(value.st_mode) or stat.S_ISLNK(value.st_mode): - raise RuntimeError("audit artifact parent must be a non-symlink directory") - return value.st_dev, value.st_ino - - @staticmethod - def _open_directory_authority(path: Path) -> tuple[int, tuple[int, int]]: - fd = os.open( - path, - os.O_RDONLY - | getattr(os, "O_DIRECTORY", 0) - | getattr(os, "O_NOFOLLOW", 0), - ) - try: - value = os.fstat(fd) - if not stat.S_ISDIR(value.st_mode): - raise RuntimeError("audit artifact parent is not a directory") - identity = (value.st_dev, value.st_ino) - if AuditArtifactStore._directory_identity(path) != identity: - raise RuntimeError("audit artifact parent identity changed") - return fd, identity - except RuntimeError: - os.close(fd) - raise - except OSError as exc: - os.close(fd) - raise RuntimeError( - "audit artifact parent authority is invalid" - ) from exc - except BaseException: - os.close(fd) - raise - - @staticmethod - def _open_publication_lock_authority( - root: Path, - root_fd: int, - root_identity: tuple[int, int], - ) -> tuple[int, _FileIdentity]: - flags = os.O_RDWR | getattr(os, "O_NOFOLLOW", 0) - created = False - try: - try: - fd = os.open( - _PUBLICATION_LOCK_NAME, - flags | os.O_CREAT | os.O_EXCL, - 0o600, - dir_fd=root_fd, - ) - created = True - except FileExistsError: - fd = os.open(_PUBLICATION_LOCK_NAME, flags, dir_fd=root_fd) - except OSError as exc: - raise RuntimeError( - "audit publication lock cannot be opened safely" - ) from exc - try: - value = os.fstat(fd) - identity = _FileIdentity.from_stat(value) - if not stat.S_ISREG(value.st_mode): - raise RuntimeError("audit publication lock is not a regular file") - linked = os.stat( - _PUBLICATION_LOCK_NAME, - dir_fd=root_fd, - follow_symlinks=False, - ) - if not identity.matches(linked): - raise RuntimeError("audit publication lock identity changed") - root_value = os.fstat(root_fd) - if (root_value.st_dev, root_value.st_ino) != root_identity: - raise RuntimeError("audit artifact root authority is invalid") - if AuditArtifactStore._directory_identity(root) != root_identity: - raise RuntimeError("audit artifact root identity changed") - if created: - os.fsync(fd) - os.fsync(root_fd) - return fd, identity - except RuntimeError: - os.close(fd) - raise - except OSError as exc: - os.close(fd) - raise RuntimeError( - "audit publication lock authority is invalid" - ) from exc - except BaseException: - os.close(fd) - raise - - def close(self) -> None: - lock = getattr(self, "_lock", None) - publication_lock = getattr(self, "_publication_lifecycle_lock", None) - if lock is None: - self._close_unlocked() - return - if publication_lock is None: - with lock: - self._close_unlocked() - return - with publication_lock: - if ( - self._publication_guard_owner == threading.get_ident() - and self._publication_guard_depth > 0 - ): - raise RuntimeError( - "cannot close audit artifact store inside publication guard" - ) - with lock: - self._close_unlocked() - - def _close_unlocked(self) -> None: - if getattr(self, "_closed", False): - return - self._closed = True - for field in ( - "_publication_lock_fd", - "_root_fd", - "_evidence_parent_fd", - ): - fd = getattr(self, field, -1) - if isinstance(fd, int) and fd >= 0: - try: - os.close(fd) - except OSError: - pass - setattr(self, field, -1) - - def __del__(self) -> None: - try: - self.close() - except Exception: - pass - - def _validate_publication_lock_identity( - self, - *, - root: Path | None = None, - root_fd: int | None = None, - root_identity: tuple[int, int] | None = None, - lock_fd: int | None = None, - lock_identity: _FileIdentity | None = None, - ) -> None: - if getattr(self, "_read_only", False): - raise RuntimeError( - "audit artifact store is read-only: it holds no publication lock" - ) - root = self._root if root is None else root - root_fd = self._root_fd if root_fd is None else root_fd - root_identity = ( - self._root_identity if root_identity is None else root_identity - ) - lock_fd = self._publication_lock_fd if lock_fd is None else lock_fd - lock_identity = ( - self._publication_lock_identity - if lock_identity is None - else lock_identity - ) - if getattr(self, "_closed", False) or root_fd < 0 or lock_fd < 0: - raise RuntimeError("audit artifact store is closed") - try: - if self._directory_identity(root) != root_identity: - raise RuntimeError("audit artifact root identity changed") - root_value = os.fstat(root_fd) - if (root_value.st_dev, root_value.st_ino) != root_identity: - raise RuntimeError("audit artifact root authority is invalid") - value = os.fstat(lock_fd) - if not lock_identity.matches(value): - raise RuntimeError("audit publication lock authority is invalid") - linked = os.stat( - _PUBLICATION_LOCK_NAME, - dir_fd=root_fd, - follow_symlinks=False, - ) - except OSError as exc: - raise RuntimeError("audit publication lock identity changed") from exc - if not lock_identity.matches(linked): - raise RuntimeError("audit publication lock identity changed") - - @contextmanager - def publication_order_guard(self) -> Iterator[None]: - """Serialize ordinal allocation/publication across threads/processes.""" - - if getattr(self, "_read_only", False): - raise RuntimeError( - "audit artifact store is read-only: publication is not available" - ) - with self._publication_order_guard(validate_on_exit=True): - yield - - @contextmanager - def _publication_order_guard( - self, - *, - validate_on_exit: bool, - ) -> Iterator[None]: - thread_id = threading.get_ident() - initial_identity = self._publication_lock_identity - _reject_cross_store_local_owner( - (initial_identity.device, initial_identity.inode), - thread_id=thread_id, - store_id=id(self), - ) - with self._publication_lifecycle_lock: - stable_identity = self._publication_lock_identity - _reject_cross_store_local_owner( - (stable_identity.device, stable_identity.inode), - thread_id=thread_id, - store_id=id(self), - ) - process_lock = self._publication_inode_thread_lock - with process_lock: - with self._publication_order_guard_locked( - validate_on_exit=validate_on_exit - ): - yield - - @contextmanager - def _publication_order_guard_locked( - self, - *, - validate_on_exit: bool, - ) -> Iterator[None]: - """Guard body with lifecycle and current inode thread locks held.""" - - thread_id = threading.get_ident() - if self._publication_guard_owner == thread_id: - self._publication_guard_depth += 1 - try: - self._validate_publication_lock_identity() - yield - if validate_on_exit: - self._validate_publication_lock_identity() - finally: - self._publication_guard_depth -= 1 - return - if self._publication_guard_owner is not None: - raise RuntimeError("audit publication guard ownership is inconsistent") - self._validate_publication_lock_identity() - lock_fd = self._publication_lock_fd - root = self._root - root_fd = self._root_fd - root_identity = self._root_identity - lock_identity = self._publication_lock_identity - local_key = (lock_identity.device, lock_identity.inode) - with _PUBLICATION_THREAD_LOCKS_GUARD: - local_owner = _PUBLICATION_LOCAL_OWNERS.get(local_key) - if local_owner is not None: - owner_thread, owner_store = local_owner - if owner_thread == thread_id and owner_store != id(self): - raise RuntimeError( - "audit publication guard is already held by another store " - "on this thread" - ) - raise RuntimeError("audit publication guard local ownership is inconsistent") - fcntl.flock(lock_fd, fcntl.LOCK_EX) - registered = False - try: - with _PUBLICATION_THREAD_LOCKS_GUARD: - local_owner = _PUBLICATION_LOCAL_OWNERS.get(local_key) - if local_owner is not None: - raise RuntimeError( - "audit publication guard local ownership is inconsistent" - ) - _PUBLICATION_LOCAL_OWNERS[local_key] = (thread_id, id(self)) - registered = True - self._publication_guard_owner = thread_id - self._publication_guard_depth = 1 - self._validate_publication_lock_identity( - root=root, - root_fd=root_fd, - root_identity=root_identity, - lock_fd=lock_fd, - lock_identity=lock_identity, - ) - yield - if validate_on_exit: - self._validate_publication_lock_identity( - root=root, - root_fd=root_fd, - root_identity=root_identity, - lock_fd=lock_fd, - lock_identity=lock_identity, - ) - finally: - self._publication_guard_depth = 0 - self._publication_guard_owner = None - if registered: - with _PUBLICATION_THREAD_LOCKS_GUARD: - if _PUBLICATION_LOCAL_OWNERS.get(local_key) == ( - thread_id, - id(self), - ): - del _PUBLICATION_LOCAL_OWNERS[local_key] - fcntl.flock(lock_fd, fcntl.LOCK_UN) - - def _require_publication_order_guard(self) -> None: - if ( - self._publication_guard_owner != threading.get_ident() - or self._publication_guard_depth <= 0 - ): - raise RuntimeError("audit publication order guard is required") - self._validate_publication_lock_identity() - - @contextmanager - def _prepared_publication_order_guard( - self, - *, - root: Path, - root_fd: int, - root_identity: tuple[int, int], - lock_fd: int, - lock_identity: _FileIdentity, - ) -> Iterator[None]: - current_identity = self._publication_lock_identity - if ( - lock_identity.device == current_identity.device - and lock_identity.inode == current_identity.inode - ): - self._validate_publication_lock_identity( - root=root, - root_fd=root_fd, - root_identity=root_identity, - lock_fd=lock_fd, - lock_identity=lock_identity, - ) - yield - self._validate_publication_lock_identity( - root=root, - root_fd=root_fd, - root_identity=root_identity, - lock_fd=lock_fd, - lock_identity=lock_identity, - ) - return - if root_identity == self._root_identity: - raise RuntimeError( - "audit publication lock changed within the current root" - ) - process_lock = _publication_thread_lock( - (lock_identity.device, lock_identity.inode) - ) - if not process_lock.acquire(blocking=False): - raise RuntimeError("new audit publication guard is busy") - flocked = False - try: - try: - fcntl.flock(lock_fd, fcntl.LOCK_EX | fcntl.LOCK_NB) - except BlockingIOError as exc: - raise RuntimeError("new audit publication guard is busy") from exc - flocked = True - self._validate_publication_lock_identity( - root=root, - root_fd=root_fd, - root_identity=root_identity, - lock_fd=lock_fd, - lock_identity=lock_identity, - ) - yield - self._validate_publication_lock_identity( - root=root, - root_fd=root_fd, - root_identity=root_identity, - lock_fd=lock_fd, - lock_identity=lock_identity, - ) - finally: - if flocked: - fcntl.flock(lock_fd, fcntl.LOCK_UN) - process_lock.release() - - def _validate_root_identity(self) -> None: - if getattr(self, "_closed", False) or self._root_fd < 0: - raise RuntimeError("audit artifact store is closed") - try: - current = self._directory_identity(self._root) - except (FileNotFoundError, OSError) as exc: - raise RuntimeError("audit artifact root identity changed") from exc - if current != self._root_identity: - raise RuntimeError("audit artifact root identity changed") - value = os.fstat(self._root_fd) - if (value.st_dev, value.st_ino) != self._root_identity: - raise RuntimeError("audit artifact root authority is invalid") - - def _validate_evidence_parent_identity(self) -> None: - if getattr(self, "_closed", False) or self._evidence_parent_fd < 0: - raise RuntimeError("audit artifact store is closed") - try: - current = self._directory_identity(self._evidence_path.parent) - except (FileNotFoundError, OSError) as exc: - raise RuntimeError("audit evidence parent identity changed") from exc - if current != self._evidence_parent_identity: - raise RuntimeError("audit evidence parent identity changed") - value = os.fstat(self._evidence_parent_fd) - if (value.st_dev, value.st_ino) != self._evidence_parent_identity: - raise RuntimeError("audit evidence parent authority is invalid") - - def _owned_parent_fd(self, path: Path) -> int | None: - path = Path(path).absolute() - try: - parent = path.parent.resolve(strict=True) - except OSError: - parent = path.parent - if parent == self._root: - return self._root_fd - if parent == self._evidence_path.parent: - return self._evidence_parent_fd - return None - - def duplicate_root_directory_fd(self) -> int: - with self._lock: - self._validate_root_identity() - return os.dup(self._root_fd) - - def _scan_root_names(self) -> list[str]: - """Enumerate the audit root through a fresh directory description. - - ``os.listdir(self._root_fd)`` reads through the long-lived pinned - descriptor. Some kernels and overlay filesystems serve a stale or - truncated enumeration through a directory description that has - already been read once and mutated since: on an aarch64 Docker host - a conflict snapshot written moments earlier was absent from the next - scan, so quarantine deduplication wrote a second copy and metrics - and retention saw an empty root. Reopening ``.`` relative to the - pinned descriptor yields a fresh description of the very same inode - -- authority still flows from ``_root_fd``, never from a pathname -- - and the identity is re-verified before and after the enumeration. - """ - self._validate_root_identity() - fd = os.open( - ".", - os.O_RDONLY | getattr(os, "O_DIRECTORY", 0), - dir_fd=self._root_fd, - ) - try: - value = os.fstat(fd) - if (value.st_dev, value.st_ino) != self._root_identity: - raise RuntimeError("audit artifact root authority is invalid") - names = os.listdir(fd) - finally: - os.close(fd) - self._validate_root_identity() - return names - - def _owned_lstat(self, path: Path) -> os.stat_result: - fd = self._owned_parent_fd(path) - if fd is None: - raise RuntimeError("audit stat target has no directory authority") - return os.stat(Path(path).name, dir_fd=fd, follow_symlinks=False) - - def _owned_open(self, path: Path, flags: int, mode: int = 0o777) -> int: - fd = self._owned_parent_fd(path) - if fd is None: - mutation_flags = ( - os.O_WRONLY - | os.O_RDWR - | os.O_CREAT - | os.O_EXCL - | getattr(os, "O_TRUNC", 0) - ) - if flags & mutation_flags: - raise RuntimeError("audit mutation target has no directory authority") - raise RuntimeError("audit read target has no directory authority") - return os.open(Path(path).name, flags, mode, dir_fd=fd) - - def _owned_unlink(self, path: Path) -> None: - fd = self._owned_parent_fd(path) - if fd is None: - raise RuntimeError("audit unlink target has no directory authority") - os.unlink(Path(path).name, dir_fd=fd) - - def _owned_replace(self, source: Path, target: Path) -> None: - source_fd = self._owned_parent_fd(source) - target_fd = self._owned_parent_fd(target) - if source_fd is None or target_fd is None: - raise RuntimeError("audit replace target has no directory authority") - os.replace( - Path(source).name, - Path(target).name, - src_dir_fd=source_fd, - dst_dir_fd=target_fd, - ) - - def _owned_link(self, source: Path, target: Path) -> None: - source_fd = self._owned_parent_fd(source) - target_fd = self._owned_parent_fd(target) - if source_fd is None or target_fd is None: - raise RuntimeError("audit link target has no directory authority") - os.link( - Path(source).name, - Path(target).name, - src_dir_fd=source_fd, - dst_dir_fd=target_fd, - follow_symlinks=False, - ) - - def _read_owned_regular_bytes( - self, - path: Path, - ) -> tuple[bytes, os.stat_result]: - parent_fd = self._owned_parent_fd(path) - if parent_fd is not None: - self._validate_owned_parent(path) - fd = self._owned_open( - path, - os.O_RDONLY | getattr(os, "O_NOFOLLOW", 0), - ) - result = self._read_regular_bytes_fd(fd) - if parent_fd is not None: - self._validate_owned_parent(path) - return result - - def _validate_owned_parent(self, path: Path) -> None: - path = Path(path).absolute() - try: - parent = path.parent.resolve(strict=True) - except OSError: - parent = path.parent - if parent == self._root: - self._validate_root_identity() - elif parent == self._evidence_path.parent: - self._validate_evidence_parent_identity() - else: - raise RuntimeError("audit target has no directory authority") - - def reconfigure( - self, - *, - root: Path | None = None, - evidence_path: Path | None = None, - live_bundle_retention: int | None = None, - candidate_retention_seconds: int | None = None, - share_segment_size: int | None = None, - verifier_timeout_seconds: float | None = None, - canonicalizer: Callable[[dict[str, Any]], bytes] | None = None, - ) -> None: - if ( - self._publication_guard_owner == threading.get_ident() - and self._publication_guard_depth > 0 - ): - raise RuntimeError( - "cannot reconfigure audit artifact store inside publication guard" - ) - retired_root_fd: int | None = None - retired_publication_lock_fd: int | None = None - with self._publication_order_guard(validate_on_exit=False): - ( - retired_root_fd, - retired_publication_lock_fd, - ) = self._reconfigure_under_publication_guard( - root=root, - evidence_path=evidence_path, - live_bundle_retention=live_bundle_retention, - candidate_retention_seconds=candidate_retention_seconds, - share_segment_size=share_segment_size, - verifier_timeout_seconds=verifier_timeout_seconds, - canonicalizer=canonicalizer, - ) - for retired_fd in (retired_publication_lock_fd, retired_root_fd): - if retired_fd is not None: - try: - os.close(retired_fd) - except OSError: - pass - - def _reconfigure_under_publication_guard( - self, - *, - root: Path | None, - evidence_path: Path | None, - live_bundle_retention: int | None, - candidate_retention_seconds: int | None, - share_segment_size: int | None, - verifier_timeout_seconds: float | None, - canonicalizer: Callable[[dict[str, Any]], bytes] | None, - ) -> tuple[int | None, int | None]: - self._require_publication_order_guard() - with self._lock: - if self._closed: - raise RuntimeError("audit artifact store is closed") - if root is not None and self._active_candidates: - raise RuntimeError( - "cannot reconfigure audit root while candidates are active" - ) - size = self._share_segment_size - if share_segment_size is not None: - size = int(share_segment_size) - if size < 0: - raise ValueError( - "audit share segment size must be non-negative" - ) - verifier_timeout = self._verifier_timeout_seconds - if verifier_timeout_seconds is not None: - verifier_timeout = float(verifier_timeout_seconds) - if verifier_timeout <= 0: - raise ValueError("verifier timeout must be positive") - new_root = self._root - new_root_fd: int | None = None - new_root_identity = self._root_identity - new_publication_lock_fd: int | None = None - new_publication_lock_identity = self._publication_lock_identity - new_evidence_path = self._evidence_path - new_evidence_parent_fd: int | None = None - new_evidence_parent_identity = self._evidence_parent_identity - try: - if root is not None: - candidate_root = Path(root).expanduser().absolute() - candidate_root.mkdir(parents=True, exist_ok=True) - new_root = candidate_root.resolve(strict=True) - new_root_fd, new_root_identity = ( - self._open_directory_authority(new_root) - ) - ( - new_publication_lock_fd, - new_publication_lock_identity, - ) = self._open_publication_lock_authority( - new_root, - new_root_fd, - new_root_identity, - ) - if evidence_path is not None: - candidate_evidence = Path(evidence_path).expanduser().absolute() - candidate_evidence.parent.mkdir(parents=True, exist_ok=True) - new_evidence_path = ( - candidate_evidence.parent.resolve(strict=True) - / candidate_evidence.name - ) - ( - new_evidence_parent_fd, - new_evidence_parent_identity, - ) = self._open_directory_authority( - new_evidence_path.parent - ) - except BaseException: - for prepared_fd in ( - new_publication_lock_fd, - new_root_fd, - new_evidence_parent_fd, - ): - if prepared_fd is not None: - os.close(prepared_fd) - raise - new_live_bundle_retention = ( - self._live_bundle_retention - if live_bundle_retention is None - else int(live_bundle_retention) - ) - new_candidate_retention_seconds = ( - self._candidate_retention_seconds - if candidate_retention_seconds is None - else int(candidate_retention_seconds) - ) - prepared_guard = ( - self._prepared_publication_order_guard( - root=new_root, - root_fd=new_root_fd, - root_identity=new_root_identity, - lock_fd=new_publication_lock_fd, - lock_identity=new_publication_lock_identity, - ) - if new_root_fd is not None - and new_publication_lock_fd is not None - else nullcontext() - ) - old_root = self._root - old_root_fd = self._root_fd - old_root_identity = self._root_identity - old_publication_lock_fd = self._publication_lock_fd - old_publication_lock_identity = self._publication_lock_identity - old_publication_inode_thread_lock = ( - self._publication_inode_thread_lock - ) - old_evidence_path = self._evidence_path - old_evidence_parent_fd = self._evidence_parent_fd - old_evidence_parent_identity = self._evidence_parent_identity - old_latest_evidence = self._latest_evidence - old_current_envelope = self._current_envelope - old_current_identity = self._current_identity - old_evidence_state = self._evidence_state - old_compatibility_override = self._compatibility_evidence_override - old_invalidated_legacy_identity = self._invalidated_legacy_identity - old_legacy_proof_token = self._legacy_proof_token - old_live_retention = self._live_bundle_retention - old_candidate_retention = self._candidate_retention_seconds - old_share_segment_size = self._share_segment_size - old_verifier_timeout = self._verifier_timeout_seconds - old_canonicalizer = self._canonicalizer - try: - with prepared_guard: - # Final old-authority validation is the linearization - # boundary. After this succeeds, only the prepared new - # authority may influence commit or rollback. - self._validate_publication_lock_identity( - root=old_root, - root_fd=old_root_fd, - root_identity=old_root_identity, - lock_fd=old_publication_lock_fd, - lock_identity=old_publication_lock_identity, - ) - if new_root_fd is not None: - self._root = new_root - self._root_fd = new_root_fd - self._root_identity = new_root_identity - assert new_publication_lock_fd is not None - self._publication_lock_fd = new_publication_lock_fd - self._publication_lock_identity = ( - new_publication_lock_identity - ) - self._publication_inode_thread_lock = _publication_thread_lock( - ( - new_publication_lock_identity.device, - new_publication_lock_identity.inode, - ) - ) - if new_evidence_parent_fd is not None: - self._evidence_path = new_evidence_path - self._evidence_parent_fd = new_evidence_parent_fd - self._evidence_parent_identity = ( - new_evidence_parent_identity - ) - if root is not None or evidence_path is not None: - self._latest_evidence = None - self._current_envelope = None - self._current_identity = None - self._evidence_state = "absent" - self._compatibility_evidence_override = False - self._invalidated_legacy_identity = None - self._legacy_proof_token = None - self._live_bundle_retention = new_live_bundle_retention - self._candidate_retention_seconds = ( - new_candidate_retention_seconds - ) - self._share_segment_size = size - self._verifier_timeout_seconds = verifier_timeout - if canonicalizer is not None: - self._canonicalizer = canonicalizer - if root is not None or evidence_path is not None: - # Readers must see either the complete old authority/cache - # or the complete new authority/cache. Both old and new - # process guards remain held across switch and reload. - self._reload_current_evidence_locked() - if new_evidence_parent_fd is not None: - try: - os.close(old_evidence_parent_fd) - except OSError: - pass - return ( - old_root_fd if new_root_fd is not None else None, - ( - old_publication_lock_fd - if new_publication_lock_fd is not None - else None - ), - ) - except BaseException: - self._root = old_root - self._root_fd = old_root_fd - self._root_identity = old_root_identity - self._publication_lock_fd = old_publication_lock_fd - self._publication_lock_identity = old_publication_lock_identity - self._publication_inode_thread_lock = ( - old_publication_inode_thread_lock - ) - self._evidence_path = old_evidence_path - self._evidence_parent_fd = old_evidence_parent_fd - self._evidence_parent_identity = old_evidence_parent_identity - self._latest_evidence = old_latest_evidence - self._current_envelope = old_current_envelope - self._current_identity = old_current_identity - self._evidence_state = old_evidence_state - self._compatibility_evidence_override = ( - old_compatibility_override - ) - self._invalidated_legacy_identity = ( - old_invalidated_legacy_identity - ) - self._legacy_proof_token = old_legacy_proof_token - self._live_bundle_retention = old_live_retention - self._candidate_retention_seconds = old_candidate_retention - self._share_segment_size = old_share_segment_size - self._verifier_timeout_seconds = old_verifier_timeout - self._canonicalizer = old_canonicalizer - for prepared_fd in ( - new_publication_lock_fd, - new_root_fd, - new_evidence_parent_fd, - ): - if prepared_fd is not None: - try: - os.close(prepared_fd) - except OSError: - pass - raise - - def publication_sequence_floor(self) -> int: - with self._lock: - if not self._directory_authority_is_current(): - return 0 - return ( - self._current_identity.sequence - if self._current_identity is not None - and self._evidence_state in {"valid", "legacy_proven"} - else 0 - ) - - def legacy_evidence_identity(self) -> AuditPublicationIdentity | None: - with self._lock: - if not self._directory_authority_is_current(): - return None - if ( - self._publication_guard_owner == threading.get_ident() - and self._publication_guard_depth > 0 - and not self._compatibility_evidence_override - ): - self._reload_current_evidence_locked() - if self._evidence_state not in { - "legacy", - "legacy_unproven", - } or self._current_identity is None: - return None - return self._current_identity - - def _directory_authority_is_current(self) -> bool: - try: - self._validate_root_identity() - self._validate_evidence_parent_identity() - return True - except RuntimeError: - # A transient pathname replacement revokes authority while it is - # present, but the pinned fd/cache can safely recover if the exact - # original directory inode is restored. Malformed evidence uses - # the separate sticky `invalid` state. - return False - - @staticmethod - def artifact_kind(name: str) -> str: - if _BODY_RE.fullmatch(name): - return "body" - segment = _SHARE_CONTENT_RE.fullmatch(name) or _SHARE_SLOT_RE.fullmatch(name) - if segment: - return ( - "share_segment" - if int(segment.group("first")) <= int(segment.group("last")) - else "other" - ) - if _CANONICAL_BUNDLE_RE.fullmatch(name): - # Classified as its own kind so retention never treats a canonical - # bundle as a live envelope or an expendable candidate. - return "canonical_bundle" - if ( - _CANDIDATE_RE.fullmatch(name) - or _LEGACY_CANDIDATE_RE.fullmatch(name) - or _LEGACY_HIDDEN_CANDIDATE_RE.fullmatch(name) - ): - return "candidate" - if _LIVE_RE.fullmatch(name): - return "live_bundle" - return "other" - - def metrics_snapshot(self) -> dict[str, dict[str, int] | int]: - metrics: dict[str, dict[str, int] | int] = { - kind: {"files": 0, "bytes": 0} - for kind in ( - "body", - "share_segment", - "canonical_bundle", - "live_bundle", - "candidate", - "other", - ) - } - metrics["scan_error"] = 0 - try: - self._validate_root_identity() - paths = [self._root / name for name in self._scan_root_names()] - except (OSError, RuntimeError): - metrics["scan_error"] = 1 - return metrics - for path in paths: - if path.name == _PUBLICATION_LOCK_NAME: - continue - try: - value = self._owned_lstat(path) - if not stat.S_ISREG(value.st_mode): - continue - except RuntimeError: - metrics["scan_error"] = 1 - break - except OSError: - metrics["scan_error"] = 1 - continue - kind = self.artifact_kind(path.name) - bucket = metrics[kind] - assert isinstance(bucket, dict) - bucket["files"] += 1 - bucket["bytes"] += value.st_size - try: - self._validate_root_identity() - except RuntimeError: - metrics["scan_error"] = 1 - return metrics - - def issue_candidate(self, *, block_hash: str) -> OwnedCandidateArtifact: - block_hash = _canonical_hex(block_hash, name="block_hash") - with self._lock: - self._validate_root_identity() - for _attempt in range(16): - token = uuid.uuid4().hex - path = self._root / ( - f".prism-live-audit-bundle-candidate-{block_hash}-{token}.json.tmp" - ) - try: - self._owned_lstat(path) - except FileNotFoundError: - candidate = OwnedCandidateArtifact( - path=path, - block_hash=block_hash, - token=token, - store_token=self._store_token, - ) - self._active_candidates[token] = (path, None) - return candidate - raise RuntimeError("could not allocate an absent audit candidate path") - - def _require_candidate( - self, - candidate: OwnedCandidateArtifact, - ) -> tuple[Path, _FileIdentity | None]: - self._validate_root_identity() - if candidate.store_token != self._store_token: - raise RuntimeError("candidate does not belong to this audit store") - if not _CANDIDATE_RE.fullmatch(candidate.path.name): - raise RuntimeError("candidate has an invalid owned filename") - if candidate.path.parent != self._root: - raise RuntimeError("candidate escapes the audit artifact root") - current = self._active_candidates.get(candidate.token) - if current is None or current[0] != candidate.path: - raise RuntimeError("candidate is no longer active") - return current - - def adopt_created_candidate( - self, - candidate: OwnedCandidateArtifact, - ) -> None: - raise RuntimeError( - "pathname adoption is forbidden; transfer the exact open compiler inode" - ) - - def adopt_compiler_candidate( - self, - candidate: OwnedCandidateArtifact, - *, - path: Path, - value: os.stat_result, - ) -> None: - """Transfer the exact still-open inode created exclusively by J1.""" - - with self._lock: - expected_path, identity = self._require_candidate(candidate) - if identity is not None or Path(path) != expected_path: - raise RuntimeError("compiler candidate transfer is invalid") - transferred = _FileIdentity.from_stat(value) - current = self._owned_lstat(expected_path) - if not transferred.matches(current): - raise RuntimeError("compiler candidate identity changed before transfer") - self._active_candidates[candidate.token] = (expected_path, transferred) - - def write_compatibility_candidate( - self, - candidate: OwnedCandidateArtifact, - bundle: Mapping[str, Any], - ) -> Path: - payload = _json_bytes(bundle) - with self._lock: - path, identity = self._require_candidate(candidate) - if identity is not None: - raise RuntimeError("candidate already exists") - created = False - try: - fd = self._owned_open( - path, - os.O_WRONLY | os.O_CREAT | os.O_EXCL | getattr(os, "O_NOFOLLOW", 0), - 0o600, - ) - created = True - opened = os.fstat(fd) - if not stat.S_ISREG(opened.st_mode): - raise RuntimeError("candidate is not a regular file") - candidate_identity = _FileIdentity.from_stat(opened) - with self._lock: - self._require_candidate(candidate) - self._active_candidates[candidate.token] = ( - path, - candidate_identity, - ) - with os.fdopen(fd, "wb") as handle: - try: - handle.write(payload) - handle.flush() - os.fsync(handle.fileno()) - finally: - finalized_identity = _FileIdentity.from_stat( - os.fstat(handle.fileno()) - ) - with self._lock: - current = self._active_candidates.get(candidate.token) - if current == (path, candidate_identity): - self._active_candidates[candidate.token] = ( - path, - finalized_identity, - ) - self._validate_root_identity() - return path - except BaseException: - if created: - self._unlink_candidate_if_same(candidate, allow_unadopted=True) - else: - self.release_candidate(candidate) - raise - - def release_candidate(self, candidate: OwnedCandidateArtifact) -> None: - """Release an uncreated reservation without acquiring delete authority.""" - - with self._lock: - self._require_candidate(candidate) - self._active_candidates.pop(candidate.token, None) - - def _unlink_candidate_if_same( - self, - candidate: OwnedCandidateArtifact, - *, - allow_unadopted: bool = False, - ) -> None: - with self._lock: - self._validate_root_identity() - current = self._active_candidates.get(candidate.token) - if current is None or current[0] != candidate.path: - return - expected = current[1] - try: - value = self._owned_lstat(candidate.path) - except FileNotFoundError: - self._active_candidates.pop(candidate.token, None) - return - if not stat.S_ISREG(value.st_mode): - self._active_candidates.pop(candidate.token, None) - return - if expected is not None and not expected.matches(value): - self._active_candidates.pop(candidate.token, None) - return - if expected is None and not allow_unadopted: - return - self._remove_identity_safe( - candidate.path, - _FileIdentity.from_stat(value), - ) - self._active_candidates.pop(candidate.token, None) - - def discard_candidate(self, candidate: OwnedCandidateArtifact) -> None: - with self._lock: - current = self._active_candidates.get(candidate.token) - if current is not None and current[1] is None: - # Reservation alone never grants deletion authority. J1 must - # return and transfer the exact created inode first. - self._active_candidates.pop(candidate.token, None) - return - self._unlink_candidate_if_same(candidate) - - @staticmethod - def verified_canonical_bundle_path( - candidate_bundle_path: Path, - report: Mapping[str, Any], - ) -> Path | None: - expected = str(report["audit_bundle_sha256_hex"]).lower() - payload, _value = AuditArtifactStore.read_regular_bytes( - Path(candidate_bundle_path) - ) - return ( - Path(candidate_bundle_path) - if _sha256_bytes(payload) == expected - else None - ) - - def verify_candidate( - self, - candidate: OwnedCandidateArtifact, - *, - coinbase_tx_hex: str, - expected_coinbase_value_sats: int, - trusted_writer_public_key_hex: str, - trust_source: str = "configured", - expected_block_height: int | None = None, - verifier: Callable[..., dict[str, Any]] | None = None, - ) -> VerifiedAuditBundle: - key = _canonical_hex( - trusted_writer_public_key_hex, - name="ledger writer public key", - ) - coinbase_tx_hex = _canonical_hex_bytes( - coinbase_tx_hex, - name="coinbase_tx_hex", - ) - with self._lock: - path, identity = self._require_candidate(candidate) - if identity is None: - raise RuntimeError( - "candidate compiler inode was not transferred before verification" - ) - before = self._owned_lstat(path) - if not identity.matches(before): - raise RuntimeError("candidate identity changed before verification") - before_bytes, before_descriptor_stat = self._read_owned_regular_bytes(path) - if not identity.matches(before_descriptor_stat): - raise RuntimeError("candidate identity changed before verification") - literal_before = _sha256_bytes(before_bytes) - verify = verifier or self._verifier or self.verify_bundle - snapshot_fd, verification_path = self._open_verification_snapshot( - before_bytes - ) - try: - report = verify( - verification_path, - coinbase_tx_hex, - key, - expected_coinbase_value_sats=expected_coinbase_value_sats, - expected_block_height=expected_block_height, - ) - finally: - os.close(snapshot_fd) - after_bytes, after = self._read_owned_regular_bytes(path) - literal_after = _sha256_bytes(after_bytes) - if ( - not identity.matches(after) - or before.st_size != after.st_size - or before.st_mtime_ns != after.st_mtime_ns - or not hmac.compare_digest(literal_before, literal_after) - ): - raise RuntimeError("candidate changed during verification") - normalized = self._validate_verifier_report( - report, - coinbase_tx_hex=coinbase_tx_hex, - expected_coinbase_value_sats=expected_coinbase_value_sats, - expected_block_height=expected_block_height, - ) - expected_digest = str(normalized["audit_bundle_sha256_hex"]) - verification_identity = self.build_verification_identity( - trust_source=trust_source, - trusted_writer_public_key_hex=key, - literal_sha256=literal_after, - literal_byte_len=after.st_size, - report=normalized, - ) - verified = VerifiedAuditBundle( - candidate=candidate, - report=copy.deepcopy(normalized), - literal_sha256=literal_after, - byte_length=after.st_size, - device=after.st_dev, - inode=after.st_ino, - mtime_ns=after.st_mtime_ns, - canonical_copy_eligible=hmac.compare_digest( - literal_after, - expected_digest, - ), - verification_identity=verification_identity, - ) - self._validate_root_identity() - return verified - - def require_current_verified_candidate( - self, - verified: VerifiedAuditBundle, - candidate: OwnedCandidateArtifact, - ) -> None: - """Reject reuse of a successful result across ephemeral candidates.""" - - with self._lock: - path, identity = self._require_candidate(candidate) - if verified.candidate != candidate or identity is None: - raise RuntimeError("verified audit result belongs to another candidate") - if ( - identity.device != verified.device - or identity.inode != verified.inode - or path != candidate.path - ): - raise RuntimeError("verified audit result candidate identity changed") - - @staticmethod - def build_verification_identity( - *, - trust_source: str, - trusted_writer_public_key_hex: str, - literal_sha256: str, - literal_byte_len: int, - report: Mapping[str, Any], - ) -> dict[str, Any]: - if trust_source not in {"configured", "embedded_test_only"}: - raise RuntimeError("audit verifier trust source is invalid") - normalized_report = AuditArtifactStore._normalize_report_identity(report) - byte_len = literal_byte_len - if isinstance(byte_len, bool) or not isinstance(byte_len, int) or byte_len < 0: - raise RuntimeError("audit verifier literal byte length is invalid") - base = { - "schema": VERIFICATION_IDENTITY_SCHEMA, - "trust_source": trust_source, - "ledger_writer_public_key_hex": _canonical_hex( - trusted_writer_public_key_hex, - name="ledger writer public key", - ), - "literal_sha256_hex": _canonical_hex( - literal_sha256, - name="verified literal sha256", - ), - "literal_byte_len": byte_len, - "report": normalized_report, - } - return { - **base, - "identity_sha256_hex": _sha256_bytes( - _json_bytes(base, sort_keys=True) - ), - } - - @staticmethod - def _normalize_verification_identity( - value: object, - *, - report: Mapping[str, Any], - ) -> dict[str, Any]: - if not isinstance(value, dict): - raise RuntimeError("audit verification identity is required") - normalized = AuditArtifactStore.build_verification_identity( - trust_source=str(value.get("trust_source") or ""), - trusted_writer_public_key_hex=str( - value.get("ledger_writer_public_key_hex") or "" - ), - literal_sha256=str(value.get("literal_sha256_hex") or ""), - literal_byte_len=value.get("literal_byte_len"), - report=report, - ) - if value != normalized: - raise RuntimeError("audit verification identity mismatch") - return normalized - - @staticmethod - def _legacy_verification_marker( - *, - identity: AuditPublicationIdentity, - report: Mapping[str, Any], - ) -> dict[str, Any]: - normalized_report = AuditArtifactStore._normalize_report_identity(report) - return { - "schema": LEGACY_VERIFICATION_UNAVAILABLE_SCHEMA, - "reason": "evidence-predates-deterministic-verification-identity", - "block_hash": identity.block_hash, - "block_height": identity.block_height, - "audit_publication_sequence": identity.sequence, - "report_identity_sha256_hex": _sha256_bytes( - _json_bytes(normalized_report, sort_keys=True) - ), - } - - @staticmethod - def _normalize_legacy_verification_marker( - value: object, - *, - identity: AuditPublicationIdentity, - report: Mapping[str, Any], - ) -> dict[str, Any]: - if not isinstance(value, dict): - raise RuntimeError("legacy audit verification marker is required") - expected = AuditArtifactStore._legacy_verification_marker( - identity=identity, - report=report, - ) - if value != expected: - raise RuntimeError("legacy audit verification marker mismatch") - return expected - - @staticmethod - def _loaded_evidence_state( - payload: Mapping[str, Any], - identity: AuditPublicationIdentity, - ) -> str: - if identity.sequence == 0: - return "legacy" - verification = payload.get("audit_verification_identity") - if ( - isinstance(verification, dict) - and verification.get("schema") - == LEGACY_VERIFICATION_UNAVAILABLE_SCHEMA - ): - # The on-disk ledger proof is a hint only. The coordinator must - # revalidate exact active ledger state on every process start. - return "legacy_unproven" - return "valid" - - def _open_verification_snapshot(self, payload: bytes) -> tuple[int, Path]: - path = self._root / f".prism-audit-verification-{uuid.uuid4().hex}.tmp" - identity: _FileIdentity | None = None - read_fd: int | None = None - try: - fd = self._owned_open( - path, - os.O_WRONLY | os.O_CREAT | os.O_EXCL | getattr(os, "O_NOFOLLOW", 0), - 0o600, - ) - identity = _FileIdentity.from_stat(os.fstat(fd)) - with os.fdopen(fd, "wb") as handle: - try: - handle.write(payload) - handle.flush() - os.fsync(handle.fileno()) - finally: - identity = _FileIdentity.from_stat(os.fstat(handle.fileno())) - read_fd = self._owned_open( - path, - os.O_RDONLY | getattr(os, "O_NOFOLLOW", 0), - ) - if not identity.matches(os.fstat(read_fd)): - raise RuntimeError("verification snapshot identity changed") - if not self._remove_identity_safe(path, identity): - raise RuntimeError("verification snapshot path was replaced") - self._fsync_directory(self._root) - return read_fd, Path(f"/dev/fd/{read_fd}") - except BaseException: - if read_fd is not None: - os.close(read_fd) - if identity is not None: - self._remove_identity_safe(path, identity) - raise - - @staticmethod - def _validate_verifier_report( - report: object, - *, - coinbase_tx_hex: str, - expected_coinbase_value_sats: int, - expected_block_height: int | None, - ) -> dict[str, Any]: - if not isinstance(report, dict): - raise RuntimeError("audit verifier returned a non-object report") - result = AuditArtifactStore._normalize_report_identity(report) - if not hmac.compare_digest(result["coinbase_tx_hex"], coinbase_tx_hex): - raise RuntimeError("audit verifier coinbase does not match submitted coinbase") - if int(result["coinbase_value_sats"]) != int(expected_coinbase_value_sats): - raise RuntimeError("audit verifier coinbase value does not match expected value") - if ( - expected_block_height is not None - and result["block_height"] != int(expected_block_height) - ): - raise RuntimeError("audit verifier block height does not match expected height") - return result - - @staticmethod - def _normalize_report_identity(report: Mapping[str, Any]) -> dict[str, Any]: - result = dict(report) - if result.get("schema") != VERIFICATION_REPORT_SCHEMA: - raise RuntimeError("audit verifier report schema is invalid") - block_height = result.get("block_height") - if isinstance(block_height, bool) or not isinstance(block_height, int): - raise RuntimeError("audit verifier block height is invalid") - if block_height < 0: - raise RuntimeError("audit verifier block height is invalid") - result["audit_bundle_sha256_hex"] = _canonical_hex( - result.get("audit_bundle_sha256_hex"), - name="audit bundle sha256", - ) - result["coinbase_tx_hex"] = _canonical_hex_bytes( - result.get("coinbase_tx_hex"), - name="report coinbase_tx_hex", - ) - report_value = result.get("coinbase_value_sats") - if isinstance(report_value, bool) or not isinstance(report_value, int): - raise RuntimeError("audit verifier coinbase value is required") - if result["coinbase_value_sats"] < 0: - raise RuntimeError("audit verifier coinbase value is invalid") - for key in ( - "reward_manifest_sha256_hex", - "payout_policy_manifest_sha256_hex", - "prism_audit_commitment_leaf_hex", - "audit_commitment_root_hex", - "coinbase_txid", - "coinbase_wtxid", - "coinbase_manifest_sha256_hex", - ): - result[key] = _canonical_hex(result.get(key), name=key) - for key in ( - "min_output_sats", - "onchain_output_count", - "accrued_account_count", - ): - value = result.get(key) - if isinstance(value, bool) or not isinstance(value, int) or value < 0: - raise RuntimeError(f"audit verifier {key} is invalid") - return result - - def verify_bundle( - self, - bundle_path: Path, - coinbase_tx_hex: str, - ledger_writer_public_key_hex: str, - *, - expected_coinbase_value_sats: int, - expected_block_height: int | None = None, - ) -> dict[str, Any]: - del expected_block_height - inherited_fds: tuple[int, ...] = () - match = re.fullmatch(r"/dev/fd/(?P<fd>[0-9]+)", str(bundle_path)) - if match is not None: - inherited_fds = (int(match.group("fd")),) - process = subprocess.Popen( - prism_tool_command("qbit-prism-audit-verify") - + [ - str(bundle_path), - "--coinbase-tx-hex", - coinbase_tx_hex, - "--ledger-writer-public-key-hex", - ledger_writer_public_key_hex, - "--expected-coinbase-value-sats", - str(expected_coinbase_value_sats), - ], - stdout=subprocess.PIPE, - stderr=subprocess.PIPE, - pass_fds=inherited_fds, - start_new_session=True, - ) - outputs = {"stdout": bytearray(), "stderr": bytearray()} - selector = selectors.DefaultSelector() - assert process.stdout is not None - assert process.stderr is not None - for name, stream in ( - ("stdout", process.stdout), - ("stderr", process.stderr), - ): - os.set_blocking(stream.fileno(), False) - selector.register(stream, selectors.EVENT_READ, name) - deadline = time.monotonic() + self._verifier_timeout_seconds - try: - while selector.get_map() or process.poll() is None: - remaining_seconds = deadline - time.monotonic() - if remaining_seconds <= 0: - self._kill_verifier_group(process) - process.wait() - raise RuntimeError("qbit-prism-audit-verify timed out") - events = selector.select(min(0.05, remaining_seconds)) - for key, _mask in events: - try: - chunk = os.read(key.fileobj.fileno(), 64 * 1024) - except (BlockingIOError, InterruptedError): - continue - if not chunk: - selector.unregister(key.fileobj) - continue - output = outputs[str(key.data)] - remaining = MAX_VERIFIER_OUTPUT_BYTES + 1 - len(output) - if remaining > 0: - output.extend(chunk[:remaining]) - if len(output) > MAX_VERIFIER_OUTPUT_BYTES: - self._kill_verifier_group(process) - process.wait() - raise RuntimeError( - "qbit-prism-audit-verify output exceeded limit" - ) - process.wait() - except BaseException: - if process.poll() is None: - self._kill_verifier_group(process) - process.wait() - raise - finally: - selector.close() - process.stdout.close() - process.stderr.close() - stdout = bytes(outputs["stdout"]) - stderr = bytes(outputs["stderr"]) - if process.returncode != 0: - raise RuntimeError( - "qbit-prism-audit-verify failed: " - + stderr.decode(errors="replace").strip() - ) - try: - report = json.loads(stdout) - except (UnicodeDecodeError, json.JSONDecodeError) as exc: - raise RuntimeError("qbit-prism-audit-verify returned invalid JSON") from exc - if not isinstance(report, dict): - raise RuntimeError("qbit-prism-audit-verify returned a non-object report") - return report - - @staticmethod - def _kill_verifier_group(process: subprocess.Popen[bytes]) -> None: - try: - os.killpg(process.pid, signal.SIGKILL) - except (ProcessLookupError, PermissionError, AttributeError): - try: - process.kill() - except (ProcessLookupError, PermissionError): - pass - - @staticmethod - def trusted_writer_key( - configured_key: str | None, - bundle: Mapping[str, Any], - *, - allow_embedded_test_key: bool, - ) -> str: - if configured_key is not None: - return _canonical_hex( - configured_key, - name="configured ledger writer public key", - ) - if not allow_embedded_test_key: - raise RuntimeError("configured ledger writer public key is required") - try: - embedded = bundle["ledger_window_attestation"]["signature"][ - "public_key_hex" - ] - except (KeyError, TypeError) as exc: - raise RuntimeError("audit bundle has no embedded test writer key") from exc - return _canonical_hex(embedded, name="bundle ledger public key") - - def live_envelope_path(self, *, block_height: int, block_hash: str) -> Path: - block_hash = _canonical_hex(block_hash, name="block_hash") - if isinstance(block_height, bool) or not isinstance(block_height, int): - raise ValueError("block_height must be an integer") - if block_height < 0: - raise ValueError("block_height must be non-negative") - return self._root / ( - f"prism-live-audit-bundle-{block_height}-{block_hash}.json" - ) - - def body_path(self, block_hash: str, audit_bundle_sha256: str) -> Path: - block_hash = _canonical_hex(block_hash, name="block_hash") - digest = _canonical_hex( - audit_bundle_sha256, - name="audit_bundle_sha256", - ) - return self._root / f"prism-audit-bundle-body-{block_hash}-{digest}.json" - - def canonical_bundle_path( - self, - block_hash: str, - audit_bundle_sha256: str, - ) -> Path: - """Owned path for the compressed canonical bundle of one publication. - - The digest in the name is the public SHA-256 over the *uncompressed* - canonical JSON, so the artifact is content-addressed by the identity - readers already advertise. - """ - - block_hash = _canonical_hex(block_hash, name="block_hash") - digest = _canonical_hex( - audit_bundle_sha256, - name="audit_bundle_sha256", - ) - self._validate_root_identity() - return self._root / ( - f"prism-audit-bundle-canonical-{block_hash}-{digest}.json.gz" - ) - - def resolve_owned_path(self, body_uri: object) -> Path: - self._validate_root_identity() - raw = Path(str(body_uri)).expanduser() - if not raw.is_absolute() and ( - len(raw.parts) != 1 or raw.name in {".", ".."} - ): - raise RuntimeError( - f"audit bundle body path escapes audit body store: {body_uri}" - ) - path = raw if raw.is_absolute() else self._root / raw - path = path.absolute() - try: - path = path.parent.resolve(strict=True) / path.name - except OSError as exc: - raise RuntimeError( - f"audit bundle body path is not resolvable: {body_uri}" - ) from exc - if path.parent != self._root: - raise RuntimeError( - f"audit bundle body path escapes audit body store: {body_uri}" - ) - if self.artifact_kind(path.name) not in {"body", "share_segment"}: - raise RuntimeError(f"audit bundle body path is not owned: {body_uri}") - self._validate_root_identity() - return path - - def publish_success( - self, - *, - identity: AuditPublicationIdentity, - publication_floor_sequence: int, - report: Mapping[str, Any], - persistence: Mapping[str, Any], - evidence: Mapping[str, Any], - verification_identity: Mapping[str, Any], - created_at: str, - restore_superseded_envelope: bool = False, - ) -> AuditPublication: - self._require_publication_order_guard() - block_hash = _canonical_hex(identity.block_hash, name="block_hash") - if block_hash != identity.block_hash: - raise ValueError("publication identity block_hash must be canonical") - if identity.sequence <= 0 or identity.block_height < 0: - raise ValueError("publication identity is invalid") - if ( - isinstance(publication_floor_sequence, bool) - or not isinstance(publication_floor_sequence, int) - or publication_floor_sequence < 0 - ): - raise ValueError("publication floor sequence must be a non-negative integer") - if identity.sequence > publication_floor_sequence: - raise RuntimeError( - "audit publication sequence exceeds the durable ledger floor" - ) - envelope_path = self.live_envelope_path( - block_height=identity.block_height, - block_hash=block_hash, - ) - with self._lock: - self._validate_root_identity() - self._validate_evidence_parent_identity() - if not self._compatibility_evidence_override: - invalidated_legacy = self._invalidated_legacy_identity - self._reload_current_evidence_locked() - if ( - invalidated_legacy is not None - and self._evidence_state in {"legacy", "legacy_unproven"} - and self._current_identity == invalidated_legacy - ): - # Explicit invalidation remains repairable only while disk - # still names the same unproven legacy publication. A peer - # valid publication always wins this reconciliation. - self._latest_evidence = None - self._current_envelope = None - self._current_identity = None - self._evidence_state = "invalid" - else: - self._invalidated_legacy_identity = None - current = self._current_identity - if self._evidence_state in {"legacy", "legacy_unproven"}: - raise RuntimeError( - "current audit evidence lacks a validated publication identity" - ) - if current is not None: - exact = current == identity - if ( - self._evidence_state == "legacy_proven" - and identity.sequence <= current.sequence - ): - raise RuntimeError( - "legacy audit evidence is never exact-replay-equivalent" - ) - if identity.sequence == current.sequence and not exact: - raise RuntimeError("audit publication identity conflict") - if identity.sequence < current.sequence: - # A superseded ordinal never becomes the current evidence - # reference, but the live envelope is this block's only - # published evidence pointer and lives on its own - # height+hash path. Two distinct blocks landing inside one - # confirm->publish window arrive here in ordinal order, so - # skipping the write erased the earlier block's public - # audit trail permanently and silently. - # - # Only the caller knows whether it is the live writer for - # this landing. A process whose writer lease was taken over - # must stay fenced out of the shared audit root entirely: - # its in-memory report may be stale, and publishing a stale - # report as a block's public evidence is worse than leaving - # the envelope missing. Such callers never opt in. - if not restore_superseded_envelope: - return AuditPublication( - identity=identity, - envelope_path=envelope_path, - evidence=copy.deepcopy(dict(evidence)), - published=False, - ) - return self._write_superseded_live_envelope_locked( - identity=identity, - envelope_path=envelope_path, - report=report, - persistence=persistence, - evidence=evidence, - created_at=created_at, - ) - if exact and self._latest_evidence is not None: - expected_envelope = self._build_live_envelope( - identity=identity, - report=report, - persistence=persistence, - created_at=created_at, - ) - replay_annotations = copy.deepcopy(dict(evidence)) - # These global counters are observational annotations, not - # immutable block identity. Reuse the originally durable - # values so an outbox replay after more shares arrive does - # not strand an otherwise exact publication. - for annotation in ( - "accepted_share_count", - "distinct_miner_count", - ): - if annotation in self._latest_evidence: - replay_annotations[annotation] = ( - self._latest_evidence[annotation] - ) - replay_confirmation = replay_annotations.get("confirmation") - current_confirmation = self._latest_evidence.get("confirmation") - if ( - isinstance(replay_confirmation, dict) - and isinstance(current_confirmation, dict) - and "confirmed_count" in current_confirmation - and "confirmed_count" in replay_confirmation - ): - # The confirm disposition says how *this* call landed, - # not what the block is. The flip that first published - # this evidence reported a fresh confirmation (1); - # every later replay of the same durable row reports - # the idempotent disposition (2) -- which is the point - # of that split (issue #61). Reuse the originally - # durable value so an exact replay is not stranded as - # a payload conflict. The publication ordinal inside - # the same confirmation is block identity and is - # compared unchanged: a replay that returned a - # different ordinal still conflicts. - replay_confirmation["confirmed_count"] = ( - current_confirmation["confirmed_count"] - ) - replay_persistence = copy.deepcopy(dict(persistence)) - current_persistence = self._latest_evidence.get( - "persistence" - ) - if isinstance(current_persistence, dict) and ( - "share_count" in current_persistence - ): - # persist_accepted_block reports the current global - # accepted-share count. It is observational and may - # advance between a committed block and outbox replay. - replay_persistence["share_count"] = ( - current_persistence["share_count"] - ) - replay_evidence = self._normalized_durable_evidence( - identity=identity, - envelope_path=envelope_path, - report=report, - persistence=replay_persistence, - evidence=replay_annotations, - verification_identity=verification_identity, - ) - if self._latest_evidence != replay_evidence: - raise RuntimeError( - "exact audit publication replay payload conflict" - ) - envelope_present = True - evidence_present = True - try: - disk_envelope_bytes, _value = ( - self._read_owned_regular_bytes(envelope_path) - ) - except FileNotFoundError: - envelope_present = False - else: - self._match_existing_live_envelope( - disk_envelope_bytes, - expected_envelope, - ) - try: - disk_evidence_bytes, _value = ( - self._read_owned_regular_bytes(self._evidence_path) - ) - except FileNotFoundError: - evidence_present = False - else: - try: - disk_evidence = json.loads(disk_evidence_bytes) - except (UnicodeDecodeError, json.JSONDecodeError): - # The evidence path is mutable publication state, - # not operator-owned preservation authority. A - # ledger-proven exact replay may repair malformed - # bytes while retaining the exact immutable - # envelope checked above. - evidence_present = False - else: - if disk_evidence != replay_evidence: - try: - self._validate_evidence(disk_evidence) - except ( - OSError, - RuntimeError, - TypeError, - ValueError, - ): - evidence_present = False - else: - raise RuntimeError( - "existing live audit evidence conflicts" - ) - if envelope_present and evidence_present: - self._fsync_directory(envelope_path.parent) - self._fsync_directory(self._evidence_path.parent) - return AuditPublication( - identity=identity, - envelope_path=envelope_path, - evidence=copy.deepcopy(self._latest_evidence), - published=False, - ) - if identity.sequence != publication_floor_sequence: - raise RuntimeError( - "audit publication repair is behind the durable ledger floor" - ) - # A missing exact entry is repaired by the same atomic - # primitives as a first publication. Preserve the durable - # observational annotations while falling through. - persistence = replay_persistence - evidence = replay_annotations - if identity.sequence != publication_floor_sequence: - raise RuntimeError( - "audit publication is behind the durable ledger floor" - ) - envelope = self._build_live_envelope( - identity=identity, - report=report, - persistence=persistence, - created_at=created_at, - ) - try: - old_envelope, _old_envelope_stat = self._read_owned_regular_bytes( - envelope_path - ) - except FileNotFoundError: - old_envelope = None - reuse_envelope = False - if old_envelope is not None: - self._match_existing_live_envelope(old_envelope, envelope) - reuse_envelope = True - durable_evidence = self._normalized_durable_evidence( - identity=identity, - envelope_path=envelope_path, - report=report, - persistence=persistence, - evidence=evidence, - verification_identity=verification_identity, - ) - try: - old_evidence, _old_evidence_stat = ( - self._read_owned_regular_bytes(self._evidence_path) - ) - except FileNotFoundError: - old_evidence = None - reuse_evidence = False - if old_evidence is not None: - try: - existing_evidence = json.loads(old_evidence) - except (UnicodeDecodeError, json.JSONDecodeError): - # Invalid bytes at the mutable evidence path may be - # repaired by this ledger-proven publication. - pass - else: - if existing_evidence == durable_evidence: - reuse_evidence = True - else: - try: - _payload, _path, existing_identity = ( - self._validate_evidence(existing_evidence) - ) - except (OSError, RuntimeError, TypeError, ValueError): - # Canonical JSON can still be structurally or - # cryptographically invalid. It has the same - # repair policy as malformed bytes. - pass - else: - if identity.sequence <= existing_identity.sequence: - raise RuntimeError( - "existing live audit evidence conflicts" - ) - if not reuse_envelope: - published_envelope_identity = self._write_mutable_json( - envelope_path, - envelope, - sort_keys=True, - ) - else: - published_envelope_identity = None - self._fsync_directory(envelope_path.parent) - published_evidence_identity: _FileIdentity | None = None - try: - if reuse_evidence: - self._fsync_directory(self._evidence_path.parent) - else: - published_evidence_identity = self._write_mutable_json( - self._evidence_path, - durable_evidence, - sort_keys=True, - ) - self._validate_root_identity() - self._validate_evidence_parent_identity() - self._require_publication_order_guard() - except BaseException: - if published_evidence_identity is not None: - try: - self._rollback_mutable_replace( - self._evidence_path, - old_evidence, - published_evidence_identity, - allow_detached_authority=True, - ) - except BaseException: - pass - if reuse_envelope: - pass - elif old_envelope is None: - assert published_envelope_identity is not None - self._remove_identity_safe( - envelope_path, - published_envelope_identity, - allow_detached_authority=True, - ) - self._fsync_pinned_directory(envelope_path.parent) - else: - assert published_envelope_identity is not None - self._rollback_mutable_replace( - envelope_path, - old_envelope, - published_envelope_identity, - allow_detached_authority=True, - ) - raise - self._require_publication_order_guard() - self._latest_evidence = copy.deepcopy(durable_evidence) - self._current_envelope = envelope_path - self._current_identity = identity - self._evidence_state = "valid" - self._compatibility_evidence_override = False - self._invalidated_legacy_identity = None - self._legacy_proof_token = None - try: - self.prune_best_effort() - except Exception: - # Retention is maintenance after the evidence reference is durable. - pass - return AuditPublication( - identity=identity, - envelope_path=envelope_path, - evidence=copy.deepcopy(durable_evidence), - published=True, - ) - - def _write_superseded_live_envelope_locked( - self, - *, - identity: AuditPublicationIdentity, - envelope_path: Path, - report: Mapping[str, Any], - persistence: Mapping[str, Any], - evidence: Mapping[str, Any], - created_at: str, - ) -> AuditPublication: - """Persist the live envelope for a ledger-proven superseded ordinal. - - ``publish_success`` has already fenced ``identity.sequence`` at or below - the durable publication floor, so this ordinal was durably allocated by - the ledger and is unique to this block. The envelope path is keyed by - height and hash, so writing it cannot disturb the current evidence - reference, which must keep naming the highest published sequence. - """ - self._require_publication_order_guard() - envelope = self._build_live_envelope( - identity=identity, - report=report, - persistence=persistence, - created_at=created_at, - ) - try: - existing_envelope_bytes, _existing_stat = ( - self._read_owned_regular_bytes(envelope_path) - ) - except FileNotFoundError: - pass - else: - # A same-hash stale replay always finds its own envelope already - # present. This path only restores missing evidence -- but - # accepting arbitrary present bytes would silently retain a - # truncated or corrupt envelope forever, so present bytes are held - # to the main publish path's immutable-content standard, and a - # match is made durable exactly like a reused envelope there. - self._match_existing_live_envelope(existing_envelope_bytes, envelope) - self._fsync_directory(envelope_path.parent) - return AuditPublication( - identity=identity, - envelope_path=envelope_path, - evidence=copy.deepcopy(dict(evidence)), - published=False, - ) - written_identity = self._write_mutable_json( - envelope_path, - envelope, - sort_keys=True, - ) - try: - self._validate_root_identity() - self._validate_evidence_parent_identity() - self._require_publication_order_guard() - except BaseException: - try: - self._remove_identity_safe( - envelope_path, - written_identity, - allow_detached_authority=True, - ) - except BaseException: - pass - try: - self._fsync_pinned_directory(envelope_path.parent) - except BaseException: - pass - raise - return AuditPublication( - identity=identity, - envelope_path=envelope_path, - evidence=copy.deepcopy(dict(evidence)), - published=False, - superseded_envelope_written=True, - ) - - def _match_existing_live_envelope( - self, - existing_bytes: bytes, - envelope: Mapping[str, Any], - ) -> None: - """Hold present envelope bytes to the immutable-content standard. - - ``created_at`` is the only field a lawful republication may vary; any - other divergence, and unparseable or non-object bytes, are conflicts - because a live envelope is immutable once published. - """ - try: - existing_envelope = json.loads(existing_bytes) - except (UnicodeDecodeError, json.JSONDecodeError) as exc: - raise RuntimeError("existing live audit envelope is invalid") from exc - if not isinstance(existing_envelope, dict): - raise RuntimeError("existing live audit envelope is invalid") - immutable_existing = { - key: value - for key, value in existing_envelope.items() - if key != "created_at" - } - immutable_new = { - key: value for key, value in envelope.items() if key != "created_at" - } - if immutable_existing != immutable_new: - raise RuntimeError("existing live audit envelope conflicts") - - def _build_live_envelope( - self, - *, - identity: AuditPublicationIdentity, - report: Mapping[str, Any], - persistence: Mapping[str, Any], - created_at: str, - ) -> dict[str, Any]: - normalized_report = self._normalize_report_identity(report) - if normalized_report["block_height"] != identity.block_height: - raise RuntimeError("audit report block height does not match publication") - report_digest = _canonical_hex( - normalized_report.get("audit_bundle_sha256_hex"), - name="audit report digest", - ) - persistence_digest = _canonical_hex( - persistence.get("audit_bundle_sha256"), - name="persistence digest", - ) - if not hmac.compare_digest(report_digest, persistence_digest): - raise RuntimeError("audit report and persistence digests differ") - body_uri = str(persistence.get("body_uri") or "") - if body_uri: - body_path = self.resolve_owned_path(body_uri) - expected_body_path = self.body_path(identity.block_hash, persistence_digest) - if body_path != expected_body_path: - raise RuntimeError( - "persistence body URI does not match publication identity" - ) - body_uri = str(body_path) - return { - "schema": LIVE_ENVELOPE_SCHEMA, - "block_hash": identity.block_hash, - "block_height": identity.block_height, - "audit_bundle_sha256": persistence_digest, - "body_uri": body_uri, - "body_filename": Path(body_uri).name if body_uri else None, - "coinbase_txid": normalized_report.get("coinbase_txid"), - "coinbase_manifest_sha256": normalized_report.get( - "coinbase_manifest_sha256_hex" - ), - "coinbase_tx_hex": normalized_report.get("coinbase_tx_hex"), - "coinbase_value_sats": normalized_report.get("coinbase_value_sats"), - "created_at": created_at, - } - - def _normalized_durable_evidence( - self, - *, - identity: AuditPublicationIdentity, - envelope_path: Path, - report: Mapping[str, Any], - persistence: Mapping[str, Any], - evidence: Mapping[str, Any], - verification_identity: Mapping[str, Any], - ) -> dict[str, Any]: - normalized_report = self._normalize_report_identity(report) - normalized_verification = self._normalize_verification_identity( - dict(verification_identity), - report=normalized_report, - ) - normalized_persistence = copy.deepcopy(dict(persistence)) - normalized_persistence["audit_bundle_sha256"] = _canonical_hex( - normalized_persistence.get("audit_bundle_sha256"), - name="persistence digest", - ) - body_uri = str(normalized_persistence.get("body_uri") or "") - if body_uri: - normalized_persistence["body_uri"] = str(self.resolve_owned_path(body_uri)) - durable = copy.deepcopy(dict(evidence)) - durable.update( - { - "schema": LIVE_EVIDENCE_SCHEMA, - "block_hash": identity.block_hash, - "block_height": identity.block_height, - "audit_bundle_path": str(envelope_path), - "audit_publication_identity": identity.to_json(), - "audit_report": normalized_report, - "audit_verification_identity": normalized_verification, - "persistence": normalized_persistence, - "coinbase_txid": normalized_report["coinbase_txid"], - "coinbase_manifest_sha256_hex": normalized_report[ - "coinbase_manifest_sha256_hex" - ], - "coinbase_tx_hex": normalized_report["coinbase_tx_hex"], - "coinbase_value_sats": normalized_report["coinbase_value_sats"], - } - ) - return durable - - def latest_evidence(self) -> dict[str, Any] | None: - with self._lock: - if not self._directory_authority_is_current(): - return None - if self._compatibility_evidence_override: - return copy.deepcopy(self._latest_evidence) - if self._evidence_state == "legacy_unproven": - return None - if self._latest_evidence is not None: - return copy.deepcopy(self._latest_evidence) - if self._evidence_state == "invalid": - return None - self._load_current_evidence() - with self._lock: - if not self._directory_authority_is_current(): - return None - return copy.deepcopy(self._latest_evidence) - - def set_latest_evidence_for_compatibility( - self, - payload: Mapping[str, Any] | None, - ) -> None: - with self._lock: - self._compatibility_evidence_override = True - self._invalidated_legacy_identity = None - self._legacy_proof_token = None - if payload is None: - self._latest_evidence = None - self._current_envelope = None - self._current_identity = None - self._evidence_state = "absent" - return - copied = copy.deepcopy(dict(payload)) - try: - latest, envelope, identity = self._validate_evidence(copied) - except (OSError, RuntimeError, TypeError, ValueError): - self._latest_evidence = copied - self._current_envelope = None - self._current_identity = None - self._evidence_state = "invalid" - return - self._latest_evidence = latest - self._current_envelope = envelope - self._current_identity = identity - self._evidence_state = self._loaded_evidence_state(latest, identity) - - def _load_current_evidence(self) -> None: - with self._lock: - self._load_current_evidence_locked() - - def _load_current_evidence_locked(self) -> None: - try: - self._validate_evidence_parent_identity() - evidence_bytes, _value = self._read_owned_regular_bytes(self._evidence_path) - except FileNotFoundError: - self._latest_evidence = None - self._current_envelope = None - self._current_identity = None - self._evidence_state = "absent" - return - except (OSError, RuntimeError): - self._latest_evidence = None - self._current_envelope = None - self._current_identity = None - self._evidence_state = "invalid" - return - try: - payload = json.loads(evidence_bytes.decode("utf-8")) - parsed = self._validate_evidence(payload) - except (OSError, UnicodeDecodeError, json.JSONDecodeError, RuntimeError, TypeError, ValueError): - self._latest_evidence = None - self._current_envelope = None - self._current_identity = None - self._evidence_state = "invalid" - return - self._latest_evidence, self._current_envelope, self._current_identity = parsed - self._evidence_state = self._loaded_evidence_state( - self._latest_evidence, - self._current_identity, - ) - - def _reload_current_evidence_locked(self) -> None: - """Reload peer state while retaining an exact in-process legacy proof.""" - - prior_token = self._legacy_proof_token - self._load_current_evidence_locked() - if ( - prior_token is not None - and self._evidence_state == "legacy_unproven" - and self._current_identity == prior_token.identity - ): - try: - current_token = self._capture_legacy_proof_token( - prior_token.identity - ) - except (OSError, RuntimeError): - current_token = None - if current_token == prior_token: - self._evidence_state = "legacy_proven" - return - self._legacy_proof_token = None - - def _capture_legacy_proof_token( - self, - identity: AuditPublicationIdentity, - ) -> _LegacyProofToken: - if self._current_envelope is None: - raise RuntimeError("legacy audit envelope is unavailable") - evidence_bytes, evidence_stat = self._read_owned_regular_bytes( - self._evidence_path - ) - envelope_bytes, envelope_stat = self._read_owned_regular_bytes( - self._current_envelope - ) - return _LegacyProofToken( - identity=identity, - evidence_file=_FileIdentity.from_stat(evidence_stat), - evidence_sha256=_sha256_bytes(evidence_bytes), - envelope_file=_FileIdentity.from_stat(envelope_stat), - envelope_sha256=_sha256_bytes(envelope_bytes), - ) - - def _validate_evidence( - self, - payload: object, - ) -> tuple[dict[str, Any], Path, AuditPublicationIdentity]: - if not isinstance(payload, dict) or payload.get("schema") != LIVE_EVIDENCE_SCHEMA: - raise RuntimeError("invalid evidence schema") - self._validate_root_identity() - raw_block_hash = payload.get("block_hash") - block_hash = _canonical_hex(raw_block_hash, name="block_hash") - if raw_block_hash != block_hash: - raise RuntimeError("evidence block hash is not canonical") - raw_block_height = payload.get("block_height") - if ( - isinstance(raw_block_height, bool) - or not isinstance(raw_block_height, int) - or raw_block_height < 0 - ): - raise RuntimeError("evidence block height is invalid") - block_height = raw_block_height - raw_envelope_path = payload.get("audit_bundle_path") - if not isinstance(raw_envelope_path, str): - raise RuntimeError("evidence envelope path is not canonical") - envelope_path = Path(raw_envelope_path).absolute() - expected_path = self.live_envelope_path( - block_height=block_height, - block_hash=block_hash, - ) - if envelope_path != expected_path or raw_envelope_path != str(expected_path): - raise RuntimeError("evidence envelope path is not canonical") - envelope_bytes, _envelope_stat = self._read_owned_regular_bytes(envelope_path) - envelope = json.loads(envelope_bytes.decode("utf-8")) - if ( - not isinstance(envelope, dict) - or envelope.get("schema") != LIVE_ENVELOPE_SCHEMA - or envelope.get("block_hash") != block_hash - or isinstance(envelope.get("block_height"), bool) - or not isinstance(envelope.get("block_height"), int) - or envelope.get("block_height") != block_height - ): - raise RuntimeError("evidence envelope identity mismatch") - report = payload.get("audit_report") - persistence = payload.get("persistence") - if not isinstance(report, dict) or not isinstance(persistence, dict): - raise RuntimeError("evidence report and persistence are required") - normalized_report = self._normalize_report_identity(report) - if report != normalized_report: - raise RuntimeError("evidence audit report is not canonical") - if normalized_report["block_height"] != block_height: - raise RuntimeError("evidence report block height mismatch") - report_digest = _canonical_hex( - normalized_report.get("audit_bundle_sha256_hex"), - name="audit report digest", - ) - raw_persistence_digest = persistence.get("audit_bundle_sha256") - persistence_digest = _canonical_hex( - raw_persistence_digest, - name="persistence digest", - ) - raw_envelope_digest = envelope.get("audit_bundle_sha256") - envelope_digest = _canonical_hex( - raw_envelope_digest, - name="envelope digest", - ) - if ( - raw_persistence_digest != persistence_digest - or raw_envelope_digest != envelope_digest - ): - raise RuntimeError("evidence digest is not canonical") - if not ( - hmac.compare_digest(report_digest, persistence_digest) - and hmac.compare_digest(report_digest, envelope_digest) - ): - raise RuntimeError("evidence digest mismatch") - raw_persistence_body_uri = persistence.get("body_uri") - raw_envelope_body_uri = envelope.get("body_uri") - if not isinstance(raw_persistence_body_uri, str) or not isinstance( - raw_envelope_body_uri, - str, - ): - raise RuntimeError("evidence body URI is not canonical") - persistence_body_uri = raw_persistence_body_uri - envelope_body_uri = raw_envelope_body_uri - if persistence_body_uri: - body_path = self.resolve_owned_path(persistence_body_uri) - expected_body_path = self.body_path(block_hash, persistence_digest) - if body_path != expected_body_path: - raise RuntimeError("evidence body URI identity mismatch") - canonical_body_uri = str(body_path) - if persistence_body_uri != canonical_body_uri: - raise RuntimeError("evidence body URI is not canonical") - persistence_body_uri = canonical_body_uri - if envelope_body_uri: - canonical_envelope_body_uri = str( - self.resolve_owned_path(envelope_body_uri) - ) - if envelope_body_uri != canonical_envelope_body_uri: - raise RuntimeError("envelope body URI is not canonical") - envelope_body_uri = canonical_envelope_body_uri - if ( - envelope_body_uri != persistence_body_uri - or envelope.get("body_filename") - != (Path(persistence_body_uri).name if persistence_body_uri else None) - ): - raise RuntimeError("evidence and envelope body URIs differ") - coinbase_bindings = ( - ("coinbase_txid", "coinbase_txid"), - ("coinbase_manifest_sha256_hex", "coinbase_manifest_sha256"), - ("coinbase_tx_hex", "coinbase_tx_hex"), - ("coinbase_value_sats", "coinbase_value_sats"), - ) - for report_key, envelope_key in coinbase_bindings: - expected_value = normalized_report[report_key] - evidence_value = payload.get(report_key) - envelope_value = envelope.get(envelope_key) - if report_key == "coinbase_value_sats": - if ( - isinstance(evidence_value, bool) - or not isinstance(evidence_value, int) - or isinstance(envelope_value, bool) - or not isinstance(envelope_value, int) - ): - raise RuntimeError("evidence coinbase value is invalid") - elif report_key == "coinbase_tx_hex": - canonical_evidence_value = _canonical_hex_bytes( - evidence_value, - name="evidence coinbase_tx_hex", - ) - canonical_envelope_value = _canonical_hex_bytes( - envelope_value, - name="envelope coinbase_tx_hex", - ) - if ( - evidence_value != canonical_evidence_value - or envelope_value != canonical_envelope_value - ): - raise RuntimeError("evidence coinbase identity is not canonical") - evidence_value = canonical_evidence_value - envelope_value = canonical_envelope_value - else: - canonical_evidence_value = _canonical_hex( - evidence_value, - name=report_key, - ) - canonical_envelope_value = _canonical_hex( - envelope_value, - name=envelope_key, - ) - if ( - evidence_value != canonical_evidence_value - or envelope_value != canonical_envelope_value - ): - raise RuntimeError("evidence coinbase identity is not canonical") - evidence_value = canonical_evidence_value - envelope_value = canonical_envelope_value - if evidence_value != expected_value or envelope_value != expected_value: - raise RuntimeError("evidence coinbase identity mismatch") - identity_payload = payload.get("audit_publication_identity") - if not isinstance(identity_payload, dict): - # Older durable evidence remains readable but starts at sequence 0. - identity = AuditPublicationIdentity(0, block_height, block_hash) - else: - raw_sequence = identity_payload.get("sequence") - raw_identity_height = identity_payload.get("block_height") - if ( - isinstance(raw_sequence, bool) - or not isinstance(raw_sequence, int) - or isinstance(raw_identity_height, bool) - or not isinstance(raw_identity_height, int) - ): - raise RuntimeError("invalid evidence publication identity") - identity = AuditPublicationIdentity( - raw_sequence, - raw_identity_height, - _canonical_hex( - identity_payload.get("block_hash"), - name="publication block hash", - ), - ) - if ( - identity.sequence < 0 - or identity.block_height != block_height - or identity.block_hash != block_hash - or identity_payload != identity.to_json() - ): - raise RuntimeError("invalid evidence publication identity") - normalized_payload = copy.deepcopy(payload) - verification_payload = payload.get("audit_verification_identity") - if identity.sequence > 0: - if ( - isinstance(verification_payload, dict) - and verification_payload.get("schema") - == LEGACY_VERIFICATION_UNAVAILABLE_SCHEMA - ): - normalized_verification = ( - self._normalize_legacy_verification_marker( - verification_payload, - identity=identity, - report=normalized_report, - ) - ) - else: - normalized_verification = self._normalize_verification_identity( - verification_payload, - report=normalized_report, - ) - normalized_payload["audit_verification_identity"] = ( - normalized_verification - ) - elif verification_payload is not None: - # Sequence-zero evidence is the only supported legacy form. If a - # producer supplied a verification identity anyway, validate it - # rather than allowing an unchecked field to become durable state. - normalized_payload["audit_verification_identity"] = ( - self._normalize_verification_identity( - verification_payload, - report=normalized_report, - ) - ) - self._validate_root_identity() - self._validate_evidence_parent_identity() - return normalized_payload, envelope_path, identity - - def adopt_legacy_publication_identity( - self, - identity: AuditPublicationIdentity, - *, - publication_floor_sequence: int, - ) -> None: - """Record a ledger proof that must be revalidated after every restart.""" - - self._require_publication_order_guard() - if ( - isinstance(publication_floor_sequence, bool) - or not isinstance(publication_floor_sequence, int) - or publication_floor_sequence < 0 - ): - raise ValueError("publication floor sequence must be a non-negative integer") - with self._lock: - self._validate_root_identity() - self._validate_evidence_parent_identity() - if not self._compatibility_evidence_override: - self._load_current_evidence_locked() - if self._evidence_state not in { - "legacy", - "legacy_unproven", - } or self._latest_evidence is None: - return - if identity.sequence > publication_floor_sequence: - raise RuntimeError( - "legacy audit publication sequence exceeds the durable ledger floor" - ) - if identity.sequence < publication_floor_sequence: - self.invalidate_unprovable_legacy_evidence() - return - if ( - identity.sequence <= 0 - or identity.block_hash != self._latest_evidence.get("block_hash") - or identity.block_height - != int(self._latest_evidence.get("block_height", -1)) - ): - raise RuntimeError("legacy evidence ledger identity mismatch") - upgraded = copy.deepcopy(self._latest_evidence) - upgraded["audit_publication_identity"] = identity.to_json() - report = upgraded.get("audit_report") - if not isinstance(report, dict): - raise RuntimeError("legacy audit report is unavailable") - upgraded["audit_verification_identity"] = ( - self._legacy_verification_marker( - identity=identity, - report=report, - ) - ) - old_evidence, _old_stat = self._read_owned_regular_bytes( - self._evidence_path - ) - published_identity = self._write_mutable_json( - self._evidence_path, - upgraded, - sort_keys=True, - ) - try: - self._validate_root_identity() - self._validate_evidence_parent_identity() - except BaseException: - self._rollback_mutable_replace( - self._evidence_path, - old_evidence, - published_identity, - allow_detached_authority=True, - ) - raise - self._latest_evidence = upgraded - self._current_identity = identity - self._evidence_state = "legacy_proven" - self._invalidated_legacy_identity = None - self._legacy_proof_token = self._capture_legacy_proof_token(identity) - - def invalidate_unprovable_legacy_evidence(self) -> None: - """Drop ordering/pin authority while allowing a new durable repair.""" - - with self._lock: - if self._evidence_state not in {"legacy", "legacy_unproven"}: - return - self._invalidated_legacy_identity = self._current_identity - self._legacy_proof_token = None - self._latest_evidence = None - self._current_envelope = None - self._current_identity = None - self._evidence_state = "invalid" - - def prune_best_effort( - self, - *, - keep_live_path: Path | None = None, - ) -> RetentionResult: - try: - with self.publication_order_guard(): - with self._lock: - if not self._compatibility_evidence_override: - # A different coordinator process may have published a - # newer reference. Reconcile it under the same flock - # immediately before deriving live-deletion pins. - self._reload_current_evidence_locked() - return self._prune_best_effort_guarded( - keep_live_path=keep_live_path, - ) - except (OSError, RuntimeError): - return RetentionResult(errors=1) - - def _prune_best_effort_guarded( - self, - *, - keep_live_path: Path | None = None, - ) -> RetentionResult: - self._require_publication_order_guard() - live_removed = 0 - candidate_removed = 0 - errors = 0 - with self._lock: - live_authorized = self._directory_authority_is_current() - pins = ( - { - path - for path in (self._current_envelope,) - if path is not None - } - if live_authorized - and self._evidence_state in {"valid", "legacy_proven"} - else set() - ) - if keep_live_path is not None: - raw_keep = Path(keep_live_path).absolute() - candidate_keep = raw_keep.parent.resolve() / raw_keep.name - if ( - candidate_keep.parent == self._root - and _LIVE_RE.fullmatch(candidate_keep.name) - ): - pins.add(candidate_keep) - active_paths = {entry[0] for entry in self._active_candidates.values()} - live_retention = self._live_bundle_retention - candidate_retention = self._candidate_retention_seconds - evidence_state = self._evidence_state if live_authorized else "invalid" - try: - self._validate_root_identity() - entries = sorted( - (self._root / name for name in self._scan_root_names()), - key=lambda value: value.name, - ) - except (OSError, RuntimeError): - return RetentionResult(errors=1) - live: list[tuple[int, int, str, Path, _FileIdentity]] = [] - now = self._wall_time() - for path in entries: - try: - value = self._owned_lstat(path) - except RuntimeError: - errors += 1 - return RetentionResult(0, candidate_removed, errors) - except (FileNotFoundError, OSError): - errors += 1 - continue - if not stat.S_ISREG(value.st_mode): - continue - live_match = _LIVE_RE.fullmatch(path.name) - if live_match: - live.append( - ( - value.st_mtime_ns, - int(live_match.group("height")), - live_match.group("block"), - path, - _FileIdentity.from_stat(value), - ) - ) - continue - if not ( - _CANDIDATE_RE.fullmatch(path.name) - or _LEGACY_CANDIDATE_RE.fullmatch(path.name) - or _LEGACY_HIDDEN_CANDIDATE_RE.fullmatch(path.name) - ): - continue - if path in active_paths: - continue - if candidate_retention != 0 and now - value.st_mtime <= candidate_retention: - continue - try: - with self._lock: - if not self._directory_authority_is_current(): - errors += 1 - break - current_active_paths = { - entry[0] for entry in self._active_candidates.values() - } - if path in current_active_paths: - continue - if self._unlink_scanned_owned( - path, - _FileIdentity.from_stat(value), - ): - candidate_removed += 1 - except (OSError, RuntimeError): - errors += 1 - if evidence_state not in {"valid", "legacy_proven"}: - return RetentionResult(0, candidate_removed, errors) - live.sort(key=lambda item: (item[0], item[1], item[2], item[3].name), reverse=True) - retained = 0 - for _mtime, _height, _block, path, identity in live: - if path in pins: - continue - if live_retention < 0 or retained < max(live_retention - len(pins), 0): - retained += 1 - continue - try: - with self._lock: - self._require_publication_order_guard() - if not self._directory_authority_is_current(): - errors += 1 - break - if path == self._current_envelope: - continue - if self._unlink_scanned_owned( - path, - identity, - require_all_authorities=True, - ): - live_removed += 1 - except (OSError, RuntimeError): - errors += 1 - try: - self._validate_root_identity() - self._validate_evidence_parent_identity() - except RuntimeError: - errors += 1 - return RetentionResult(live_removed, candidate_removed, errors) - - def _unlink_scanned_owned( - self, - path: Path, - identity: _FileIdentity, - *, - require_all_authorities: bool = False, - ) -> bool: - if require_all_authorities and not self._directory_authority_is_current(): - raise RuntimeError("audit directory authority changed before live prune") - return self._remove_identity_safe(path, identity) - - def _remove_identity_safe( - self, - path: Path, - identity: _FileIdentity, - *, - allow_detached_authority: bool = False, - ) -> bool: - if not allow_detached_authority: - self._validate_owned_parent(path) - try: - current = self._owned_lstat(path) - except FileNotFoundError: - return False - if not stat.S_ISREG(current.st_mode): - # Hard-link restoration is unavailable for directories and is not - # portable for symlinks. Never relocate an unowned nonregular - # replacement merely to discover that cleanup lacks authority. - return False - if not identity.matches(current): - # A replacement that was already present at the precheck is not - # ours to relocate, even temporarily. - return False - quarantine = path.with_name(f".{path.name}.{uuid.uuid4().hex}.cleanup") - try: - self._owned_replace(path, quarantine) - except FileNotFoundError: - return False - try: - moved = self._owned_lstat(quarantine) - if identity.matches(moved): - self._owned_unlink(quarantine) - return True - # A replacement won the race. Restore it when the original name is - # free; otherwise preserve it under the quarantine name. - try: - self._owned_link(quarantine, path) - except OSError: - return False - self._owned_unlink(quarantine) - return False - except BaseException: - # Never turn an uncertain identity into deletion authority. - raise - - def _write_mutable_json( - self, - path: Path, - payload: Mapping[str, Any], - *, - sort_keys: bool, - ) -> _FileIdentity: - body = _json_bytes(payload, sort_keys=sort_keys) - return self._write_mutable_bytes(path, body) - - def _write_mutable_bytes(self, path: Path, payload: bytes) -> _FileIdentity: - return self._write_mutable_chunks(path, lambda: (payload,)) - - def _write_mutable_chunks( - self, - path: Path, - chunks_factory: Callable[[], Iterable[bytes]], - ) -> _FileIdentity: - """Replace a mutable owned file with a chunk stream, atomically. - - ``chunks_factory`` returns a fresh bounded iterator each call, so a - whole-window payload is never held as one bytes object; the prior - bytes are still captured for rollback (bounded by the existing - file: a share slot or an evidence document). - """ - path = Path(path).absolute() - resolved_parent = path.parent.resolve(strict=True) - if not ( - resolved_parent == self._root - or path == self._evidence_path - ): - raise RuntimeError("mutable audit target is outside the owned store") - with self._lock: - if path == self._evidence_path: - self._validate_evidence_parent_identity() - else: - self._validate_root_identity() - old_bytes: bytes | None - try: - old_bytes, _old_stat = self._read_owned_regular_bytes(path) - except FileNotFoundError: - old_bytes = None - tmp_path = path.with_name(f".{path.name}.{uuid.uuid4().hex}.tmp") - temp_identity: _FileIdentity | None = None - published = False - try: - fd = self._owned_open( - tmp_path, - os.O_WRONLY | os.O_CREAT | os.O_EXCL | getattr(os, "O_NOFOLLOW", 0), - 0o600, - ) - temp_identity = _FileIdentity.from_stat(os.fstat(fd)) - with os.fdopen(fd, "wb") as handle: - try: - for chunk in chunks_factory(): - handle.write(chunk) - handle.flush() - os.fsync(handle.fileno()) - finally: - temp_identity = _FileIdentity.from_stat( - os.fstat(handle.fileno()) - ) - self._owned_replace(tmp_path, path) - published = True - try: - self._fsync_directory(path.parent) - if path == self._evidence_path: - self._validate_evidence_parent_identity() - else: - self._validate_root_identity() - except BaseException: - self._rollback_mutable_replace( - path, - old_bytes, - temp_identity, - allow_detached_authority=True, - ) - raise - return temp_identity - finally: - if not published and temp_identity is not None: - self._remove_identity_safe( - tmp_path, - temp_identity, - allow_detached_authority=True, - ) - - def _rollback_mutable_replace( - self, - path: Path, - old_bytes: bytes | None, - published_identity: _FileIdentity, - *, - allow_detached_authority: bool = False, - ) -> None: - removed = self._remove_identity_safe( - path, - published_identity, - allow_detached_authority=allow_detached_authority, - ) - if not removed: - # A replacement won the race. Preserve it and do not restore stale - # bytes over an inode this operation never owned. - return - if old_bytes is None: - if allow_detached_authority: - self._fsync_pinned_directory(path.parent) - else: - self._fsync_directory(path.parent) - return - rollback = path.with_name(f".{path.name}.{uuid.uuid4().hex}.rollback") - rollback_identity: _FileIdentity | None = None - try: - fd = self._owned_open( - rollback, - os.O_WRONLY | os.O_CREAT | os.O_EXCL | getattr(os, "O_NOFOLLOW", 0), - 0o600, - ) - rollback_identity = _FileIdentity.from_stat(os.fstat(fd)) - with os.fdopen(fd, "wb") as handle: - try: - handle.write(old_bytes) - handle.flush() - os.fsync(handle.fileno()) - finally: - rollback_identity = _FileIdentity.from_stat( - os.fstat(handle.fileno()) - ) - try: - self._owned_link(rollback, path) - except FileExistsError: - # Preserve a concurrent replacement rather than overwriting it. - return - if allow_detached_authority: - self._fsync_pinned_directory(path.parent) - else: - self._fsync_directory(path.parent) - finally: - if rollback_identity is not None: - self._remove_identity_safe( - rollback, - rollback_identity, - allow_detached_authority=allow_detached_authority, - ) - - def _write_immutable_bytes(self, path: Path, payload: bytes) -> None: - self._write_immutable_chunks(path, lambda: (payload,), len(payload)) - - def _write_immutable_chunks( - self, - path: Path, - chunks_factory: Callable[[], Iterable[bytes]], - size: int | None, - ) -> None: - """Publish an immutable owned artifact from a replayable chunk stream. - - ``chunks_factory`` must return the same bounded chunks on every call: - the stream is consumed once to write the temporary inode and again, - chunk by chunk against the existing file, whenever an equal artifact - may already be published. ``size`` is the stream's total byte count - when known in advance (None derives it from the stream). - """ - if getattr(self, "_read_only", False): - raise RuntimeError( - "audit artifact store is read-only: publication is not available" - ) - path = Path(path).absolute() - if ( - path.parent.resolve(strict=True) != self._root - or self.artifact_kind(path.name) - not in {"body", "share_segment", "canonical_bundle"} - ): - raise RuntimeError( - "immutable audit target is not an owned body, segment, or " - "canonical bundle" - ) - with self._lock: - self._validate_root_identity() - try: - existing = self._owned_lstat(path) - except FileNotFoundError: - existing = None - if existing is not None: - if not stat.S_ISREG(existing.st_mode) or not self._file_matches_chunks( - path, - chunks_factory, - size, - ): - raise RuntimeError( - f"existing audit artifact does not match payload at {path}" - ) - # The prior writer may have failed between link/rename and its - # directory fsync. An idempotent retry is the durability - # repair boundary, even though no bytes need to change. - self._fsync_directory(path.parent) - return - tmp_path = path.with_name(f".{path.name}.{uuid.uuid4().hex}.tmp") - temp_identity: _FileIdentity | None = None - linked_by_this_call = False - try: - fd = self._owned_open( - tmp_path, - os.O_WRONLY | os.O_CREAT | os.O_EXCL | getattr(os, "O_NOFOLLOW", 0), - 0o600, - ) - temp_identity = _FileIdentity.from_stat(os.fstat(fd)) - with os.fdopen(fd, "wb") as handle: - try: - written = 0 - for chunk in chunks_factory(): - handle.write(chunk) - written += len(chunk) - handle.flush() - os.fsync(handle.fileno()) - finally: - temp_identity = _FileIdentity.from_stat( - os.fstat(handle.fileno()) - ) - if size is not None and written != size: - raise RuntimeError( - f"audit artifact stream size changed while publishing {path}" - ) - try: - self._owned_link(tmp_path, path) - linked_by_this_call = True - except FileExistsError: - if not self._file_matches_chunks(path, chunks_factory, size): - raise RuntimeError( - f"existing audit artifact does not match payload at {path}" - ) - try: - self._fsync_directory(path.parent) - except BaseException: - if linked_by_this_call: - # Revoke only the inode linked by this call. A racing - # replacement is never deleted or overwritten. - self._remove_identity_safe( - path, - temp_identity, - allow_detached_authority=True, - ) - try: - self._fsync_directory(path.parent) - except BaseException: - # Preserve the original durability failure; the - # retry path always fsyncs an equal existing file. - pass - raise - self._validate_root_identity() - finally: - if temp_identity is not None: - self._remove_identity_safe( - tmp_path, - temp_identity, - allow_detached_authority=True, - ) - - def _fsync_directory(self, path: Path) -> None: - path = Path(path).absolute() - if path == self._root: - self._validate_root_identity() - os.fsync(self._root_fd) - self._validate_root_identity() - return - if path == self._evidence_path.parent: - self._validate_evidence_parent_identity() - os.fsync(self._evidence_parent_fd) - self._validate_evidence_parent_identity() - return - raise RuntimeError("audit fsync target has no pinned authority") - - def _fsync_pinned_directory(self, path: Path) -> None: - path = Path(path).absolute() - if path == self._root: - os.fsync(self._root_fd) - return - if path == self._evidence_path.parent: - os.fsync(self._evidence_parent_fd) - return - raise RuntimeError("audit fsync target has no pinned authority") - - def file_matches_bytes(self, path: Path, expected: bytes) -> bool: - return self._file_matches_chunks(path, lambda: (expected,), len(expected)) - - def _file_matches_chunks( - self, - path: Path, - chunks_factory: Callable[[], Iterable[bytes]], - size: int | None, - ) -> bool: - """Compare an owned file with a chunk stream without reading it whole. - - Bounded by one chunk at a time; the descriptor identity is checked - before and after the comparison exactly as the whole-file reader - does, so a file mutated mid-compare reports a mismatch. - """ - try: - parent_fd = self._owned_parent_fd(path) - if parent_fd is not None: - self._validate_owned_parent(path) - fd = self._owned_open( - path, - os.O_RDONLY | getattr(os, "O_NOFOLLOW", 0), - ) - except OSError: - return False - try: - before = os.fstat(fd) - if not stat.S_ISREG(before.st_mode): - return False - if size is not None and before.st_size != size: - return False - offset = 0 - equal = True - for chunk in chunks_factory(): - if not equal: - break - actual = _pread_exact(fd, offset, len(chunk)) - if not hmac.compare_digest(actual, chunk): - equal = False - offset += len(chunk) - if equal and offset != before.st_size: - equal = False - after = os.fstat(fd) - if ( - before.st_dev != after.st_dev - or before.st_ino != after.st_ino - or before.st_size != after.st_size - or before.st_mtime_ns != after.st_mtime_ns - or before.st_ctime_ns != after.st_ctime_ns - ): - return False - if parent_fd is not None: - self._validate_owned_parent(path) - return equal - except OSError: - return False - finally: - os.close(fd) - - @staticmethod - def file_sha256_hex(path: Path) -> str: - payload, _value = AuditArtifactStore.read_regular_bytes(path) - return _sha256_bytes(payload) - - @staticmethod - def read_regular_bytes(path: Path) -> tuple[bytes, os.stat_result]: - fd = os.open( - path, - os.O_RDONLY | getattr(os, "O_NOFOLLOW", 0), - ) - return AuditArtifactStore._read_regular_bytes_fd(fd) - - @staticmethod - def _read_regular_bytes_fd(fd: int) -> tuple[bytes, os.stat_result]: - try: - before = os.fstat(fd) - if not stat.S_ISREG(before.st_mode): - raise OSError("audit artifact is not a regular file") - chunks: list[bytes] = [] - while True: - chunk = os.read(fd, 1024 * 1024) - if not chunk: - break - chunks.append(chunk) - after = os.fstat(fd) - if ( - before.st_dev != after.st_dev - or before.st_ino != after.st_ino - or before.st_size != after.st_size - or before.st_mtime_ns != after.st_mtime_ns - or before.st_ctime_ns != after.st_ctime_ns - ): - raise OSError("audit artifact changed while reading") - return b"".join(chunks), after - finally: - os.close(fd) - - # External body and share-segment capability used by the ledger. Database - # lease checks remain in the ledger; paths, encodings, and bytes stay here. - - def canonical_audit_bundle_bytes(self, final_bundle: Mapping[str, Any]) -> bytes: - if isinstance(final_bundle, CanonicalAuditBundleView): - # The view *is* the canonical artifact, so its bytes are the - # canonical bytes by definition. This whole-body copy is the - # compatibility lane for callers that still want one bytes - # object; the bounded publication path streams - # ``final_bundle.iter_bytes()`` instead. - return b"".join(final_bundle.iter_bytes()) - return canonical_audit_bundle_bytes(final_bundle, self._canonicalizer) - - def canonical_audit_body_bytes_for_sha( - self, - final_bundle: Mapping[str, Any], - audit_bundle_sha256: str, - ) -> bytes: - expected = _canonical_hex( - audit_bundle_sha256, - name="audit_bundle_sha256", - ) - if isinstance(final_bundle, CanonicalAuditBundleView): - actual = final_bundle.sha256_hex - if not hmac.compare_digest(actual, expected): - raise RuntimeError( - f"audit bundle sha256 mismatch: expected {expected}, got {actual}" - ) - return b"".join(final_bundle.iter_bytes()) - body = self.canonical_audit_bundle_bytes(final_bundle) - actual = _sha256_bytes(body) - if not hmac.compare_digest(actual, expected): - raise RuntimeError( - f"audit bundle sha256 mismatch: expected {expected}, got {actual}" - ) - return body - - # Bounded canonical-source helpers (issue #255). The verified candidate - # artifact is the authority; these open it under the store's directory - # authority and expose it as a positional-read source so digests, gzip - # publication and logical comparison stream through bounded chunks. - - def _open_owned_artifact_source(self, path: Path) -> ArtifactSource: - """Open an owned regular file as a bounded positional-read source.""" - path = Path(path) - parent_fd = self._owned_parent_fd(path) - if parent_fd is not None: - self._validate_owned_parent(path) - fd = self._owned_open( - path, - os.O_RDONLY | getattr(os, "O_NOFOLLOW", 0), - ) - try: - source = ArtifactSource(fd, path=path) - except CanonicalArtifactError as exc: - raise OSError(str(exc)) from exc - if parent_fd is not None: - try: - self._validate_owned_parent(path) - except BaseException: - # The descriptor is retired here, not by a finalizer that a - # retained traceback could postpone. - source.close() - raise - return source - - def _scan_owned_artifact( - self, - path: Path, - *, - lazy_paths: Sequence[tuple[str, ...]], - ) -> CanonicalAuditBundleView: - """Scan an owned JSON artifact into a bounded view (closed by caller).""" - parent_fd = self._owned_parent_fd(path) - if parent_fd is not None: - self._validate_owned_parent(path) - fd = self._owned_open( - Path(path), - os.O_RDONLY | getattr(os, "O_NOFOLLOW", 0), - ) - view = CanonicalAuditBundleView.scan( - fd, - path=Path(path), - lazy_paths=lazy_paths, - ) - if parent_fd is not None: - try: - self._validate_owned_parent(path) - except BaseException: - # Retire the artifact and index descriptors now; a retained - # traceback must not keep them open until collection. - view.close() - raise - return view - - def _require_logical_match( - self, - source: ArtifactSource, - final_bundle: Mapping[str, Any], - ) -> None: - """Full logical comparison of a canonical source with ``final_bundle``. - - A view scanned from this very inode (same identity and digest) is - the artifact by construction. Any other bundle -- a dictionary from - a compatibility builder, or a view over a different file -- is - compared member by member against a bounded scan of the source, with - the window-sized arrays streamed record by record, which preserves - the whole-document equality check without ``json.loads`` of the - candidate. - """ - if ( - isinstance(final_bundle, CanonicalAuditBundleView) - and final_bundle.identity == source.identity - and hmac.compare_digest(final_bundle.sha256_hex, source.sha256_hex()) - ): - return - fd = os.dup(source.fileno()) - try: - scanned = CanonicalAuditBundleView.scan(fd, path=source.path) - except CanonicalArtifactError as exc: - raise RuntimeError("canonical audit candidate is not valid JSON") from exc - try: - if scanned != final_bundle: - raise RuntimeError( - "canonical audit candidate does not match logical bundle" - ) - finally: - scanned.close() - - def stored_canonical_bundle_sha256( - self, - block_hash: str, - audit_bundle_sha256: str, - ) -> str | None: - """Verify a stored canonical bundle by streaming, without its bytes. - - Returns the verified uncompressed digest, None when nothing is - stored, and raises :class:`CanonicalAuditBundleCorrupt` for damage, - exactly like :meth:`read_canonical_audit_bundle` -- but the artifact - is decompressed and hashed one bounded chunk at a time. - """ - - block_hash = _canonical_hex(block_hash, name="block_hash") - digest = _canonical_hex( - audit_bundle_sha256, - name="audit_bundle_sha256", - ) - path = self.canonical_bundle_path(block_hash, digest) - try: - source = self._open_owned_artifact_source(path) - except FileNotFoundError: - self._validate_root_identity() - return None - except OSError as exc: - raise CanonicalAuditBundleCorrupt( - f"canonical audit bundle is not retrievable at {path}: {exc}" - ) from exc - try: - hasher = hashlib.sha256() - try: - with gzip.GzipFile( - fileobj=_ChunkReader(source), - mode="rb", - ) as handle: - while True: - piece = handle.read(SCAN_CHUNK_BYTES_LOCAL) - if not piece: - break - hasher.update(piece) - source.verify_identity() - except (OSError, EOFError, zlib.error, CanonicalArtifactError) as exc: - raise CanonicalAuditBundleCorrupt( - f"canonical audit bundle does not decompress at {path}: {exc}" - ) from exc - actual = hasher.hexdigest() - if not hmac.compare_digest(actual, digest): - raise CanonicalAuditBundleCorrupt( - f"canonical audit bundle hash mismatch at {path}: " - f"expected {digest}, got {actual}" - ) - finally: - source.close() - self._validate_root_identity() - return actual - - def write_canonical_audit_bundle_from_source( - self, - block_hash: str, - audit_bundle_sha256: str, - source: ArtifactSource, - ) -> Path: - """Publish the canonical bundle behind ``source`` by streaming it. - - Same contract as :meth:`write_canonical_audit_bundle` -- verified - digest before any store side effect, reproducible compression, - immutable publication, and a post-publication re-verification -- - without ever holding the uncompressed or compressed body as one - bytes object. Republishing is a durable no-op when the stored - artifact decompresses to the advertised digest. - """ - - block_hash = _canonical_hex(block_hash, name="block_hash") - digest = _canonical_hex( - audit_bundle_sha256, - name="audit_bundle_sha256", - ) - actual = source.sha256_hex() - if not hmac.compare_digest(actual, digest): - raise RuntimeError( - f"audit bundle sha256 mismatch: expected {digest}, got {actual}" - ) - path = self.canonical_bundle_path(block_hash, digest) - with self._lock: - try: - stored = self.stored_canonical_bundle_sha256(block_hash, digest) - except CanonicalAuditBundleCorrupt: - # Preserve the immutable writer's existing conflict behavior - # for damaged files. It will refuse to replace their bytes. - stored = None - if stored is not None: - self._fsync_directory(path.parent) - self._validate_root_identity() - return path - self._write_immutable_chunks( - path, - lambda: _gzip_canonical_chunks(source.iter_bytes()), - None, - ) - self._fsync_directory(path.parent) - published = self.stored_canonical_bundle_sha256(block_hash, digest) - if published is None: - raise RuntimeError( - f"canonical audit bundle is absent after publication at {path}" - ) - self._validate_root_identity() - return path - - # Canonical bundle persistence. The public digest of a publication is the - # SHA-256 of these uncompressed bytes; keeping them on disk means a reader - # never has to trust a reconstruction to answer for that identity. - - def write_canonical_audit_bundle( - self, - block_hash: str, - audit_bundle_sha256: str, - canonical_bytes: bytes, - ) -> Path: - """Publish the exact canonical bundle bytes behind an advertised digest. - - Verifies the uncompressed digest before anything reaches the store, - compresses reproducibly, publishes immutably, then reopens the - published artifact and verifies it again. Republishing identical - bytes is a durable no-op; a conflicting artifact fails loudly. - """ - - block_hash = _canonical_hex(block_hash, name="block_hash") - digest = _canonical_hex( - audit_bundle_sha256, - name="audit_bundle_sha256", - ) - payload = bytes(canonical_bytes) - actual = _sha256_bytes(payload) - if not hmac.compare_digest(actual, digest): - raise RuntimeError( - f"audit bundle sha256 mismatch: expected {digest}, got {actual}" - ) - path = self.canonical_bundle_path(block_hash, digest) - with self._lock: - try: - existing_payload = self.read_canonical_audit_bundle( - block_hash, - digest, - ) - except CanonicalAuditBundleCorrupt: - # Preserve the immutable writer's existing conflict behavior - # for damaged files. It will refuse to replace their bytes. - existing_payload = None - if existing_payload is not None and hmac.compare_digest( - existing_payload, - payload, - ): - # The public identity covers the uncompressed canonical bytes, - # not a particular zlib encoding. A retry after a compressor - # upgrade is therefore the same durability-repair boundary as - # a byte-identical gzip republication. - self._fsync_directory(path.parent) - self._validate_root_identity() - return path - - # The immutable writer is the publication boundary: it links - # atomically and refuses to overwrite an artifact that does not - # represent this canonical payload. - self._write_immutable_bytes(path, gzip_canonical_bundle_bytes(payload)) - # `_write_immutable_bytes` already fsyncs the directory; restating - # the durability boundary here keeps it explicit for this artifact. - self._fsync_directory(path.parent) - published = self.read_canonical_audit_bundle(block_hash, digest) - if published is None: - raise RuntimeError( - f"canonical audit bundle is absent after publication at {path}" - ) - if not hmac.compare_digest(published, payload): - raise CanonicalAuditBundleCorrupt( - f"canonical audit bundle bytes changed during publication at {path}" - ) - self._validate_root_identity() - return path - - def read_canonical_audit_bundle( - self, - block_hash: str, - audit_bundle_sha256: str, - ) -> bytes | None: - """Return the stored canonical bytes, or None when none were stored. - - Damage raises `CanonicalAuditBundleCorrupt` rather than reporting a - miss, so a caller that falls back to reconstruction can log the loss - instead of silently absorbing it. - """ - - block_hash = _canonical_hex(block_hash, name="block_hash") - digest = _canonical_hex( - audit_bundle_sha256, - name="audit_bundle_sha256", - ) - path = self.canonical_bundle_path(block_hash, digest) - try: - compressed, _value = self._read_owned_regular_bytes(path) - except FileNotFoundError: - self._validate_root_identity() - return None - except OSError as exc: - # A symlink refused by O_NOFOLLOW, a non-regular file, or a file - # mutated mid-read is damage, not absence. - raise CanonicalAuditBundleCorrupt( - f"canonical audit bundle is not retrievable at {path}: {exc}" - ) from exc - try: - payload = gunzip_canonical_bundle_bytes(compressed) - except (OSError, EOFError, zlib.error) as exc: - raise CanonicalAuditBundleCorrupt( - f"canonical audit bundle does not decompress at {path}: {exc}" - ) from exc - actual = _sha256_bytes(payload) - if not hmac.compare_digest(actual, digest): - raise CanonicalAuditBundleCorrupt( - f"canonical audit bundle hash mismatch at {path}: " - f"expected {digest}, got {actual}" - ) - self._validate_root_identity() - return payload - - def _owned_external_body_bytes( - self, - body_uri: object, - *, - expected_sha256: str, - block_hash: str | None = None, - ) -> tuple[Path, bytes]: - """Resolve and read an owned external body, binding its filename identity.""" - - path = self.resolve_owned_path(body_uri) - match = _BODY_RE.fullmatch(path.name) - if match is None: - raise RuntimeError("body URI is not an owned canonical body") - if match.group("digest") != expected_sha256: - raise RuntimeError("body URI digest does not match expected digest") - if block_hash is not None and match.group("block") != block_hash: - raise RuntimeError("body URI block hash does not match expected block") - payload, _value = self._read_owned_regular_bytes(path) - return path, payload - - def literal_canonical_bundle_bytes( - self, - body_uri: object, - *, - expected_sha256: object, - ) -> bytes | None: - """Return a pre-v2 external body that already *is* the canonical bundle. - - Bodies written before the compact layouts stored the canonical JSON - verbatim. When the file digests to the advertised identity it is the - artifact we want, so take it directly instead of parsing and - reserializing a bundle that would only have to hash back to the same - bytes. Returns None when the body is a compact reference instead. - """ - - expected = _canonical_hex( - expected_sha256, - name="audit_bundle_sha256", - ) - _path, payload = self._owned_external_body_bytes( - body_uri, - expected_sha256=expected, - ) - if not hmac.compare_digest(_sha256_bytes(payload), expected): - return None - self._validate_root_identity() - return payload - - def canonical_bundle_bytes_from_external_body( - self, - *, - block_hash: str, - audit_bundle_sha256: str, - body_uri: object, - load_missing_range: Callable[..., list[Any]] | None = None, - ) -> bytes: - """Recover the canonical bytes for an already published body. - - Built for the historical backfill: it takes the pre-v2 literal-body - shortcut when it applies, and otherwise reconstructs the logical bundle - from the compact reference, repairing share segments that have since - left the disk through `load_missing_range`. Recovered shares are - canonicalized and checked against the digest the body itself already - commits to, and the assembled bundle is checked against the advertised - bundle digest, so database fallback bytes are never trusted on their - own. - """ - - block_hash = _canonical_hex(block_hash, name="block_hash") - expected = _canonical_hex( - audit_bundle_sha256, - name="audit_bundle_sha256", - ) - path, body_bytes = self._owned_external_body_bytes( - body_uri, - expected_sha256=expected, - block_hash=block_hash, - ) - if hmac.compare_digest(_sha256_bytes(body_bytes), expected): - self._validate_root_identity() - return body_bytes - try: - body = json.loads(body_bytes.decode("utf-8")) - except (UnicodeDecodeError, json.JSONDecodeError) as exc: - raise RuntimeError( - f"audit bundle body is not valid JSON at {path}: {exc}" - ) from exc - schema = body.get("schema") if isinstance(body, dict) else None - if schema == AUDIT_BODY_REF_SCHEMA: - bundle = self.resolve_audit_body_ref( - body, - expected_sha256=expected, - body_uri=str(path), - load_missing_range=load_missing_range, - ) - elif schema == AUDIT_BUNDLE_V2_SCHEMA: - bundle = self.resolve_audit_bundle_v2( - body, - expected_sha256=expected, - body_uri=str(path), - load_missing_range=load_missing_range, - ) - else: - raise RuntimeError( - f"audit bundle body is not a canonical bundle source at {path}" - ) - canonical = self.canonical_audit_bundle_bytes(bundle) - actual = _sha256_bytes(canonical) - if not hmac.compare_digest(actual, expected): - raise RuntimeError( - f"audit bundle body hash mismatch at {path}: " - f"expected {expected}, got {actual}" - ) - self._validate_root_identity() - return canonical - - def _read_share_part_with_recovery( - self, - part: Mapping[str, Any], - *, - parent_body_uri: object, - load_missing_range: Callable[..., list[Any]] | None, - ) -> list[Any]: - """Read one share part, repairing a segment that has left the disk. - - The digest the compact body already commits to is the only authority - for a recovered range, so a callback result is accepted solely because - it canonicalizes back to that digest. - """ - - try: - return self.read_audit_share_segment( - part, - parent_body_uri=parent_body_uri, - ) - except (OSError, RuntimeError) as exc: - if load_missing_range is None: - raise - unavailable = exc - kind = str(part.get("kind") or "") - digest_key = { - "segment": "sha256", - "segment_range": "range_sha256", - "segment_prefix": "prefix_sha256", - }.get(kind) - if digest_key is None: - raise unavailable - try: - expected = _canonical_hex(part.get(digest_key), name="share segment digest") - first_share_seq = int(part.get("first_share_seq") or 0) - last_share_seq = int(part.get("last_share_seq") or 0) - share_count = int(part.get("share_count") or 0) - except (TypeError, ValueError) as exc: - raise RuntimeError( - f"audit bundle body is not valid JSON at {parent_body_uri}: " - f"invalid share part: {exc}" - ) from unavailable - try: - recovered = list( - load_missing_range( - first_share_seq=first_share_seq, - last_share_seq=last_share_seq, - ) - ) - except _AuditShareSegmentConflict: - raise - except Exception as exc: - raise RuntimeError( - f"audit bundle body is not retrievable at {parent_body_uri}: " - f"share segment {part.get('body_uri')} could not be recovered: {exc}" - ) from unavailable - self._validate_share_range(first_share_seq, last_share_seq, recovered) - if len(recovered) != share_count: - raise RuntimeError( - f"audit bundle body is not valid JSON at {parent_body_uri}: " - f"recovered share segment {part.get('body_uri')} share count mismatch" - ) - actual = _sha256_bytes( - self.storage_json_bytes( - self.audit_share_segment_payload( - first_share_seq=first_share_seq, - last_share_seq=last_share_seq, - shares=recovered, - ) - ) - ) - if not hmac.compare_digest(actual, expected): - raise RuntimeError( - f"audit bundle body hash mismatch at {parent_body_uri}: " - f"recovered share segment {part.get('body_uri')} " - f"expected {expected}, got {actual}" - ) - return recovered - - @staticmethod - def storage_json_bytes(payload: Mapping[str, Any]) -> bytes: - return _json_bytes(payload) - - def audit_share_segment_payload( - self, - *, - first_share_seq: int, - last_share_seq: int, - shares: Sequence[Any], - ) -> dict[str, Any]: - return { - "schema": AUDIT_SHARE_SEGMENT_SCHEMA, - "first_share_seq": first_share_seq, - "last_share_seq": last_share_seq, - "share_count": len(shares), - "shares": shares, - } - - @staticmethod - def _storage_json_chunks(payload: Mapping[str, Any]) -> Callable[[], Iterator[bytes]]: - """A replayable factory of :meth:`storage_json_bytes` in bounded chunks. - - The stream reproduces ``storage_json_bytes(payload)`` exactly while - any lazy share sequence inside ``payload`` is encoded batch by batch - instead of being materialized; a plain list of records encodes the - same way. - """ - return lambda: iter_json_byte_chunks(payload) - - def write_audit_share_segment( - self, - *, - first_share_seq: int, - last_share_seq: int, - shares: Sequence[Any], - ) -> tuple[str, str]: - self._validate_share_range(first_share_seq, last_share_seq, shares) - segment = self.audit_share_segment_payload( - first_share_seq=first_share_seq, - last_share_seq=last_share_seq, - shares=shares, - ) - chunks = self._storage_json_chunks(segment) - digest, size = json_chunks_sha256_and_size(chunks()) - path = self._root / ( - f"prism-audit-share-segment-{first_share_seq}-{last_share_seq}-{digest}.json" - ) - if not _SHARE_CONTENT_RE.fullmatch(path.name): - raise RuntimeError("invalid audit share segment bounds") - self._write_immutable_chunks(path, chunks, size) - self._validate_root_identity() - return str(path), digest - - def write_audit_share_segment_range( - self, - *, - segment_first_share_seq: int, - segment_last_share_seq: int, - first_share_seq: int, - last_share_seq: int, - shares: Sequence[Any], - load_missing_range: Callable[..., list[Any]] | None = None, - ) -> tuple[str, str]: - if not shares: - raise RuntimeError("audit share segment range cannot be empty") - if ( - segment_first_share_seq <= 0 - or segment_last_share_seq < segment_first_share_seq - or first_share_seq < segment_first_share_seq - or last_share_seq > segment_last_share_seq - ): - raise RuntimeError("audit share range is outside its segment slot") - self._validate_share_range(first_share_seq, last_share_seq, shares) - path = self._root / ( - "prism-audit-share-segment-slot-" - f"{segment_first_share_seq}-{segment_last_share_seq}.json" - ) - if not _SHARE_SLOT_RE.fullmatch(path.name): - raise RuntimeError("invalid audit share segment slot bounds") - incoming = self.audit_share_segment_payload( - first_share_seq=first_share_seq, - last_share_seq=last_share_seq, - shares=shares, - ) - # Digest the incoming range in bounded chunks. Completed 10k slots - # dominate normal block persistence; byte equality lets them return - # without parsing, merging, deep-copying, or serializing the existing - # slot tree, and the chunk stream is replayed rather than held whole. - incoming_chunks = self._storage_json_chunks(incoming) - range_digest, incoming_size = json_chunks_sha256_and_size(incoming_chunks()) - with self._lock: - self._validate_root_identity() - if self._file_matches_chunks(path, incoming_chunks, incoming_size): - self._fsync_directory(path.parent) - return str(path), range_digest - existing_bytes: bytes | None = None - merged: list[Any] | None = shares - try: - value = self._owned_lstat(path) - except FileNotFoundError: - value = None - if value is not None: - if not stat.S_ISREG(value.st_mode): - raise RuntimeError( - f"existing audit share segment is not regular at {path}" - ) - try: - existing_bytes, existing_descriptor = self._read_owned_regular_bytes( - path - ) - if ( - existing_descriptor.st_dev != value.st_dev - or existing_descriptor.st_ino != value.st_ino - or existing_descriptor.st_mtime_ns != value.st_mtime_ns - ): - raise OSError("share segment identity changed") - existing = json.loads(existing_bytes) - except (OSError, UnicodeDecodeError, json.JSONDecodeError) as exc: - raise RuntimeError( - f"existing audit share segment is not valid JSON at {path}" - ) from exc - if ( - not isinstance(existing, dict) - or existing.get("schema") != AUDIT_SHARE_SEGMENT_SCHEMA - or not isinstance(existing.get("shares"), list) - ): - raise RuntimeError( - f"existing audit share segment has invalid schema at {path}" - ) - merged = self.merge_audit_share_ranges( - existing["shares"], - shares, - segment_path=path, - load_missing_range=load_missing_range, - ) - if merged is None: - # Keep prior body_uri references valid at the stable slot - # path, while the new bundle uses an immutable - # incoming-only segment. - fresh_uri, fresh_digest = self.write_audit_share_segment( - first_share_seq=first_share_seq, - last_share_seq=last_share_seq, - shares=shares, - ) - if fresh_digest != range_digest: - raise RuntimeError( - "audit share segment range hash changed during fallback" - ) - assert existing_bytes is not None - try: - self.quarantine_audit_share_segment( - path, - expected_bytes=existing_bytes, - ) - except (OSError, RuntimeError): - # The stable slot already preserves the old range, and - # the immutable file preserves the incoming one. A - # quarantine snapshot failure must not wedge block - # finalization. - pass - return fresh_uri, range_digest - segment_first = int(merged[0]["share_seq"]) - segment_last = int(merged[-1]["share_seq"]) - if ( - segment_first == first_share_seq - and segment_last == last_share_seq - and len(merged) == len(shares) - ): - segment_chunks = incoming_chunks - else: - segment = self.audit_share_segment_payload( - first_share_seq=segment_first, - last_share_seq=segment_last, - shares=merged, - ) - segment_chunks = self._storage_json_chunks(segment) - if existing_bytes is None or not _bytes_equal_chunks( - existing_bytes, - segment_chunks(), - ): - # Streamed chunk by chunk: a slot never becomes one bytes - # object here, whatever a single record's size. - self._write_mutable_chunks(path, segment_chunks) - else: - self._fsync_directory(path.parent) - self._validate_root_identity() - return str(path), range_digest - - @staticmethod - def _validate_share_range( - first_share_seq: int, - last_share_seq: int, - shares: Sequence[Any], - ) -> None: - if first_share_seq <= 0 or last_share_seq < first_share_seq or not shares: - raise RuntimeError("audit share range bounds are invalid") - # Walked once, record by record: a lazy slice streams from the - # artifact and never becomes a list of sequence numbers. - previous: int | None = None - first: int | None = None - try: - for share in shares: - share_seq = int(share["share_seq"]) - if first is None: - first = share_seq - elif previous is not None and previous + 1 != share_seq: - raise RuntimeError("audit share range is not exactly contiguous") - previous = share_seq - except (KeyError, TypeError, ValueError) as exc: - raise RuntimeError("audit share range has invalid share_seq") from exc - if first != first_share_seq or previous != last_share_seq: - raise RuntimeError("audit share range is not exactly contiguous") - - @staticmethod - def _is_share_sequence(value: object) -> bool: - """True for a list or any other replayable sequence of records.""" - return isinstance(value, Sequence) and not isinstance( - value, - (str, bytes, bytearray), - ) - - @staticmethod - def _contiguous_share_start(shares: Sequence[Any]) -> int | None: - """First share_seq when every record is a dict in strict +1 order. - - One streaming pass; None reproduces the historical "fall back to the - inline body-ref layout" verdict for a window that is not a - contiguous dictionary sequence. - """ - first: int | None = None - previous: int | None = None - for share in shares: - if not isinstance(share, dict): - return None - try: - share_seq = int(share["share_seq"]) - except (KeyError, TypeError, ValueError): - return None - if first is None: - first = share_seq - elif previous is not None and previous + 1 != share_seq: - return None - previous = share_seq - return first - - @staticmethod - def _iter_segment_slots( - first_seq: int, - count: int, - segment_size: int, - ) -> Iterator[tuple[int, int, int, int]]: - """(index, end_index, slot_start, slot_end) for a contiguous window. - - Because contiguity has been verified, record ``i`` carries sequence - ``first_seq + i``; the slot boundaries follow arithmetically without - a second pass over the records. - """ - index = 0 - while index < count: - seq = first_seq + index - slot_start = ((seq - 1) // segment_size) * segment_size + 1 - slot_end = slot_start + segment_size - 1 - end = min(count, index + (slot_end - seq + 1)) - yield index, end, slot_start, slot_end - index = end - - def merge_audit_share_ranges( - self, - existing_shares: list[Any], - incoming_shares: list[Any], - *, - segment_path: Path, - load_missing_range: Callable[..., list[Any]] | None = None, - ) -> list[Any] | None: - if not existing_shares: - return list(incoming_shares) - existing_by_seq = self.audit_shares_by_seq( - existing_shares, - segment_path=segment_path, - ) - incoming_by_seq = self.audit_shares_by_seq( - incoming_shares, - segment_path=segment_path, - ) - for share_seq, incoming in incoming_by_seq.items(): - existing = existing_by_seq.get(share_seq) - if existing is not None and existing != incoming: - raise _AuditShareSegmentConflict( - "existing audit share segment conflicts at share_seq " - f"{share_seq} in {segment_path}" - ) - merged_by_seq = {**existing_by_seq, **incoming_by_seq} - ordered_seqs = sorted(merged_by_seq) - if any(current + 1 != nxt for current, nxt in zip(ordered_seqs, ordered_seqs[1:])): - # A skipped finalize can leave an otherwise valid slot behind the - # incoming range. Repair that hole from the append-only ledger. - gap_before, gap_after = next( - (current, nxt) - for current, nxt in zip(ordered_seqs, ordered_seqs[1:]) - if current + 1 != nxt - ) - missing_first = gap_before + 1 - missing_last = gap_after - 1 - durable_shares: list[Any] | None = None - if load_missing_range is not None: - try: - durable_shares = load_missing_range( - first_share_seq=missing_first, - last_share_seq=missing_last, - ) - except _AuditShareSegmentConflict: - raise - except Exception: - # This read is a recovery aid; its failure must not turn - # the original slot gap into another permanent finalize - # failure. - durable_shares = None - if durable_shares is not None: - try: - durable_by_seq = self.audit_shares_by_seq( - durable_shares, - segment_path=segment_path, - require_contiguous=False, - ) - except _AuditShareSegmentConflict: - raise - except RuntimeError: - durable_by_seq = {} - if sorted(durable_by_seq) == list(range(missing_first, missing_last + 1)): - merged_by_seq.update(durable_by_seq) - ordered_seqs = sorted(merged_by_seq) - if not any( - current + 1 != nxt - for current, nxt in zip(ordered_seqs, ordered_seqs[1:]) - ): - return [merged_by_seq[share_seq] for share_seq in ordered_seqs] - # The incoming range is independently complete and remains usable. - # The caller preserves this slot and gives that range a fresh URI. - return None - return [merged_by_seq[share_seq] for share_seq in ordered_seqs] - - def quarantine_audit_share_segment( - self, - segment_path: Path, - *, - expected_bytes: bytes, - ) -> Path: - segment_path = Path(segment_path) - with self._lock: - self._validate_root_identity() - conflict_prefix = f"{segment_path.name}.conflict-" - for name in sorted(self._scan_root_names()): - if not name.startswith(conflict_prefix): - continue - if self.file_matches_bytes(self._root / name, expected_bytes): - return self._root / name - timestamp = time.time_ns() - quarantine_path = segment_path.with_name( - f"{segment_path.name}.conflict-{timestamp}" - ) - while True: - try: - self._owned_lstat(quarantine_path) - except FileNotFoundError: - break - timestamp += 1 - quarantine_path = segment_path.with_name( - f"{segment_path.name}.conflict-{timestamp}" - ) - # Snapshot rather than move: previously finalized bundles retain - # this stable slot URI, so it must remain readable across crashes - # and retries. - self._write_mutable_bytes(quarantine_path, expected_bytes) - return quarantine_path - - @staticmethod - def audit_shares_by_seq( - shares: list[Any], - *, - segment_path: Path, - require_contiguous: bool = True, - ) -> dict[int, Any]: - by_seq: dict[int, Any] = {} - for share in shares: - if not isinstance(share, dict): - raise RuntimeError( - f"audit share segment has invalid share payload at {segment_path}" - ) - try: - share_seq = int(share["share_seq"]) - except (KeyError, TypeError, ValueError) as exc: - raise RuntimeError( - f"audit share segment has invalid share_seq at {segment_path}" - ) from exc - existing = by_seq.get(share_seq) - if existing is not None and existing != share: - raise _AuditShareSegmentConflict( - "audit share segment has duplicate conflicting share_seq " - f"{share_seq} at {segment_path}" - ) - by_seq[share_seq] = share - ordered = sorted(by_seq) - if require_contiguous and any( - current + 1 != nxt for current, nxt in zip(ordered, ordered[1:]) - ): - raise RuntimeError( - f"audit share segment has non-contiguous share_seq values at {segment_path}" - ) - return by_seq - - @staticmethod - def _spool_share_parts( - parts: Iterable[Mapping[str, Any]], - scratch: list[CanonicalAuditBundleView] | None, - ) -> Sequence[Any]: - """The share-part index as a replayable on-disk sequence. - - Parts are written to an unlinked scratch document as they are - produced -- an inline part's lazy slice streams batch by batch -- - and read back lazily, so a small configured segment size cannot - turn the index into a window-scaled list. The scratch view is - appended to ``scratch`` for the caller to close; without a scratch - list it retires with the last reference to the sequence. - """ - - def chunks() -> Iterator[bytes]: - yield b'{"share_parts":[' - first = True - for part in parts: - if not first: - yield b"," - first = False - yield from iter_json_byte_chunks(part) - yield b"]}" - - view = spool_json_document(chunks(), lazy_paths=(("share_parts",),)) - if scratch is not None: - scratch.append(view) - return view["share_parts"] - - @staticmethod - def share_parts_digest_hex(parts: Sequence[Any]) -> str: - """``sha256(storage_json_bytes({"share_parts": parts}))``, streamed.""" - digest, _size = json_chunks_sha256_and_size( - iter_json_byte_chunks({"share_parts": parts}) - ) - return digest - - def _iter_share_range_parts( - self, - shares: Sequence[Any], - first_seq: int, - *, - load_missing_range: Callable[..., list[Any]] | None, - ) -> Iterator[dict[str, Any]]: - segment_size = self._share_segment_size - for index, end, segment_start, segment_end in self._iter_segment_slots( - first_seq, - len(shares), - segment_size, - ): - chunk = shares[index:end] - chunk_first = first_seq + index - chunk_last = first_seq + end - 1 - uri, digest = self.write_audit_share_segment_range( - segment_first_share_seq=segment_start, - segment_last_share_seq=segment_end, - first_share_seq=chunk_first, - last_share_seq=chunk_last, - shares=chunk, - load_missing_range=load_missing_range, - ) - yield { - "kind": "segment_range", - "segment_first_share_seq": segment_start, - "segment_last_share_seq": segment_end, - "first_share_seq": chunk_first, - "last_share_seq": chunk_last, - "share_count": end - index, - "range_sha256": digest, - "body_uri": uri, - } - - def audit_share_range_parts( - self, - shares: Sequence[Any], - *, - load_missing_range: Callable[..., list[Any]] | None = None, - scratch: list[CanonicalAuditBundleView] | None = None, - ) -> Sequence[Any] | None: - """Slot-range parts for a contiguous window, one bounded slice each. - - Accepts a list or a lazy artifact sequence. The window is walked - once to prove contiguity, then each slot's records are handed to the - segment writer as a slice bounded by the configured segment size; - no list of sequence numbers or of records spanning the whole window - is built, and the parts themselves are spooled to disk as they are - produced and returned as a replayable sequence. - """ - if self._share_segment_size <= 0: - return None - if not self._is_share_sequence(shares): - return None - first_seq = self._contiguous_share_start(shares) - if first_seq is None: - return None - return self._spool_share_parts( - self._iter_share_range_parts( - shares, - first_seq, - load_missing_range=load_missing_range, - ), - scratch, - ) - - def _iter_share_parts( - self, - shares: Sequence[Any], - first_seq: int, - ) -> Iterator[dict[str, Any]]: - segment_size = self._share_segment_size - for index, end, segment_start, segment_end in self._iter_segment_slots( - first_seq, - len(shares), - segment_size, - ): - chunk = shares[index:end] - chunk_first = first_seq + index - chunk_last = first_seq + end - 1 - if ( - end - index == segment_size - and chunk_first == segment_start - and chunk_last == segment_end - ): - uri, digest = self.write_audit_share_segment( - first_share_seq=segment_start, - last_share_seq=segment_end, - shares=chunk, - ) - yield { - "kind": "segment", - "first_share_seq": segment_start, - "last_share_seq": segment_end, - "share_count": end - index, - "sha256": digest, - "body_uri": uri, - } - else: - yield { - "kind": "inline", - "first_share_seq": chunk_first, - "last_share_seq": chunk_last, - "share_count": end - index, - # A lazy slice stays lazy here; the spool encoder - # materializes it batch by batch. - "shares": chunk, - } - - def audit_share_parts( - self, - shares: Sequence[Any], - *, - scratch: list[CanonicalAuditBundleView] | None = None, - ) -> Sequence[Any] | None: - if self._share_segment_size <= 0: - return None - if not self._is_share_sequence(shares): - return None - first_seq = self._contiguous_share_start(shares) - if first_seq is None: - return None - return self._spool_share_parts(self._iter_share_parts(shares, first_seq), scratch) - - def audit_body_ref( - self, - *, - block_hash: str, - audit_bundle_sha256: str, - final_bundle: Mapping[str, Any], - scratch: list[CanonicalAuditBundleView] | None = None, - ) -> dict[str, Any] | None: - if self._share_segment_size <= 0: - return None - shares = final_bundle.get("shares") - if not self._is_share_sequence(shares) or not shares: - return None - parts = self.audit_share_parts(shares, scratch=scratch) - if parts is None or not any(part.get("kind") == "segment" for part in parts): - return None - without_shares = {key: value for key, value in final_bundle.items() if key != "shares"} - return { - "schema": AUDIT_BODY_REF_SCHEMA, - "block_hash": block_hash, - "audit_bundle_sha256": audit_bundle_sha256, - "audit_bundle_schema": str(final_bundle.get("schema") or ""), - "share_count": len(shares), - "share_segment_size": self._share_segment_size, - "shares_key_index": list(final_bundle).index("shares"), - "bundle_without_shares": without_shares, - "share_parts": parts, - } - - def audit_bundle_v2( - self, - *, - block_hash: str, - audit_bundle_sha256: str, - final_bundle: Mapping[str, Any], - load_missing_range: Callable[..., list[Any]] | None = None, - scratch: list[CanonicalAuditBundleView] | None = None, - ) -> dict[str, Any] | None: - if self._share_segment_size <= 0: - return None - shares = final_bundle.get("shares") - if not self._is_share_sequence(shares) or not shares: - return None - parts = self.audit_share_range_parts( - shares, - load_missing_range=load_missing_range, - scratch=scratch, - ) - if parts is None: - return None - proof: dict[str, Any] = { - "schema": AUDIT_WINDOW_COMPLETENESS_PROOF_SCHEMA, - "share_segment_size": self._share_segment_size, - "first_share_seq": int(shares[0]["share_seq"]), - "last_share_seq": int(shares[-1]["share_seq"]), - "share_count": len(shares), - # The parts serialization is hashed incrementally from the - # on-disk index; it equals the reader's whole-encoding digest. - "share_parts_digest_hex": self.share_parts_digest_hex(parts), - "share_parts": parts, - } - reward = final_bundle.get("reward_manifest") - if isinstance(reward, dict): - for key in ( - "anchor_job_issued_at_ms", - "anchor_share_seq", - "newest_share_seq", - "oldest_share_seq", - "included_share_count", - "requested_window_weight", - "counted_window_weight", - "share_slice_digest_hex", - ): - if key in reward: - proof[key] = reward[key] - return { - "schema": AUDIT_BUNDLE_V2_SCHEMA, - "block_hash": block_hash, - "audit_bundle_sha256": audit_bundle_sha256, - "logical_audit_bundle_schema": str(final_bundle.get("schema") or ""), - "share_count": len(shares), - "shares_key_index": list(final_bundle).index("shares"), - "bundle_without_shares": { - key: value for key, value in final_bundle.items() if key != "shares" - }, - "share_window_proof": proof, - } - - def externalize_audit_body( - self, - block_hash: str, - audit_bundle_sha256: str, - final_bundle: Mapping[str, Any], - ) -> str: - block_hash = _canonical_hex(block_hash, name="block_hash") - digest = _canonical_hex( - audit_bundle_sha256, - name="audit_bundle_sha256", - ) - path = self.body_path(block_hash, digest) - if isinstance(final_bundle, CanonicalAuditBundleView): - if not hmac.compare_digest(final_bundle.sha256_hex, digest): - raise RuntimeError( - "audit bundle sha256 mismatch: " - f"expected {digest}, got {final_bundle.sha256_hex}" - ) - final_bundle.verify_identity() - self._write_immutable_chunks( - path, - final_bundle.iter_bytes, - final_bundle.byte_length, - ) - else: - body = self.canonical_audit_body_bytes_for_sha(final_bundle, digest) - self._write_immutable_bytes(path, body) - self._validate_root_identity() - return str(path) - - def prepare_external_audit_body( - self, - payload: Mapping[str, Any], - final_bundle: Mapping[str, Any], - *, - body_uri: str | None, - canonical_bundle_path: Path | None = None, - load_missing_range: Callable[..., list[Any]] | None = None, - ) -> str | None: - """Publish the canonical bundle and its external body for one block. - - Bounded end to end (issue #255): the canonical candidate is opened - under the store's directory authority and digested, compared and - compressed through bounded chunks; a :class:`CanonicalAuditBundleView` - scanned from that very inode is accepted by identity, any other - logical bundle is compared member by member with the window arrays - streamed; compact share segments are cut from bounded slices; the - body document is encoded chunk by chunk; and every verification of - an existing or freshly published body streams the same way. The - digest, signature and coinbase checks, exact inode identity binding - and immutable-publication semantics are unchanged. - """ - block_hash = _canonical_hex(payload["block_hash"], name="block_hash") - expected = _canonical_hex( - payload["audit_bundle_sha256"], - name="audit_bundle_sha256", - ) - if body_uri is None: - return None - body_path = self.resolve_owned_path(body_uri) - canonical_path = self.body_path(block_hash, expected) - if body_path != canonical_path: - raise RuntimeError( - "existing audit bundle body pointer does not match canonical external path: " - f"{body_uri}" - ) - canonical: ArtifactSource | None = None - owned_canonical = False - canonical_bytes: bytes | None = None - source_identity: _FileIdentity | None = None - # Scratch views (the on-disk share-part index) retire with this call. - scratch: list[CanonicalAuditBundleView] = [] - try: - if canonical_bundle_path is not None: - source_path = Path(canonical_bundle_path) - try: - canonical = self._open_owned_artifact_source(source_path) - except OSError as exc: - raise RuntimeError( - f"canonical audit bundle is not retrievable at {source_path}: {exc}" - ) from exc - owned_canonical = True - source_identity = _FileIdentity( - canonical.identity.device, - canonical.identity.inode, - canonical.identity.mode, - canonical.identity.size, - canonical.identity.mtime_ns, - ) - actual = canonical.sha256_hex() - if not hmac.compare_digest(actual, expected): - raise RuntimeError( - f"audit bundle sha256 mismatch: expected {expected}, got {actual}" - ) - # Compact storage is derived from final_bundle, not necessarily - # from the supplied canonical path. Bind those logical inputs - # independently -- by identity for a view of this inode, - # otherwise by a streamed member-by-member comparison. - self._require_logical_match(canonical, final_bundle) - elif isinstance(final_bundle, CanonicalAuditBundleView): - if not hmac.compare_digest(final_bundle.sha256_hex, expected): - raise RuntimeError( - "audit bundle sha256 mismatch: " - f"expected {expected}, got {final_bundle.sha256_hex}" - ) - final_bundle.verify_identity() - canonical = final_bundle.source - else: - canonical_bytes = self.canonical_audit_body_bytes_for_sha( - final_bundle, - expected, - ) - # Persist the exact bytes the public digest is taken over, reusing - # the source already in hand. This runs before the compact body so - # every path through this call publishes it, including the ones - # that return early on an existing body. The caller's lease - # preflight has already completed, so this is still a fenced write. - if canonical is not None: - self.write_canonical_audit_bundle_from_source( - block_hash, - expected, - canonical, - ) - else: - assert canonical_bytes is not None - self.write_canonical_audit_bundle(block_hash, expected, canonical_bytes) - storage = self.audit_bundle_v2( - block_hash=block_hash, - audit_bundle_sha256=expected, - final_bundle=final_bundle, - load_missing_range=load_missing_range, - scratch=scratch, - ) - if storage is None: - storage = self.audit_body_ref( - block_hash=block_hash, - audit_bundle_sha256=expected, - final_bundle=final_bundle, - scratch=scratch, - ) - body_size: int | None - if storage is not None: - body_chunks = self._storage_json_chunks(storage) - body_size = None - elif canonical is not None: - body_chunks = canonical.iter_bytes - body_size = canonical.identity.size - else: - assert canonical_bytes is not None - literal = canonical_bytes - body_chunks = lambda: (literal,) # noqa: E731 - replayable single chunk - body_size = len(literal) - try: - existing = self._owned_lstat(body_path) - except FileNotFoundError: - existing = None - if existing is not None: - if not stat.S_ISREG(existing.st_mode): - raise RuntimeError( - f"existing audit bundle body is not regular at {body_path}" - ) - if storage is not None and self._file_matches_chunks( - body_path, - body_chunks, - body_size, - ): - if self._compact_body_reconstructs_to( - body_path, - expected=expected, - final_bundle=final_bundle, - ): - self._fsync_directory(body_path.parent) - return str(body_path) - raise RuntimeError( - f"existing audit bundle body does not match payload at {body_path}" - ) - if storage is None and self._literal_body_matches_sha(body_path, expected): - self._fsync_directory(body_path.parent) - return str(body_path) - # Layout upgrades may represent the same canonical logical - # bundle with different storage bytes. Verify by a streamed - # comparison against the logical bundle (a compact body) or by - # digest (a literal body). - if not self._existing_body_matches( - body_path, - expected=expected, - final_bundle=final_bundle, - ): - raise RuntimeError( - f"existing audit bundle body does not match payload at {body_path}" - ) - self._fsync_directory(body_path.parent) - return str(body_path) - self._write_immutable_chunks(body_path, body_chunks, body_size) - if canonical_bundle_path is not None and source_identity is not None: - source_after = self._owned_lstat(Path(canonical_bundle_path)) - if not source_identity.matches(source_after): - raise RuntimeError( - "canonical audit bundle identity changed after publication" - ) - # The compact storage bytes were derived from the already verified - # logical bundle. Validate the exact destination here, streaming - # the segments against the logical window. - if storage is not None: - valid = self._compact_body_reconstructs_to( - body_path, - expected=expected, - final_bundle=final_bundle, - ) - else: - valid = self._literal_body_matches_sha(body_path, expected) - if not valid: - raise RuntimeError("published audit bundle body failed digest verification") - self._validate_root_identity() - return str(body_path) - finally: - for view in scratch: - view.close() - if owned_canonical and canonical is not None: - canonical.close() - - def _literal_body_matches_sha(self, body_path: Path, expected: str) -> bool: - """True when the owned body file digests to ``expected``, streamed.""" - try: - source = self._open_owned_artifact_source(body_path) - except OSError: - return False - try: - return hmac.compare_digest(source.sha256_hex(), expected) - except (OSError, CanonicalArtifactError): - return False - finally: - source.close() - - def _existing_body_matches( - self, - body_path: Path, - *, - expected: str, - final_bundle: Mapping[str, Any], - ) -> bool: - """A published body of any layout represents ``final_bundle``. - - Compact layouts are compared record by record through their - segments; a literal body is verified by its digest. Nothing is - reconstructed into a whole-window object graph. - """ - if self._compact_body_reconstructs_to( - body_path, - expected=expected, - final_bundle=final_bundle, - ): - return True - return self._literal_body_matches_sha(body_path, expected) - - # Every window-, transaction- or recipient-scaled member of a compact - # body stays lazy: the header mirrors the artifact's lazy paths and the - # share-part index scales with the window over the segment size. - _BODY_LAZY_PATHS: tuple[tuple[str, ...], ...] = ( - ("bundle_without_shares", "shares"), - ("bundle_without_shares", "reward_manifest", "shares"), - ("bundle_without_shares", "reward_manifest", "entitlements"), - ("bundle_without_shares", "witness_merkle_leaves_hex"), - ("bundle_without_shares", "audit_commitment_leaves_hex"), - ("bundle_without_shares", "prior_balances"), - ("bundle_without_shares", "payout_policy_manifest", "accounts"), - ("bundle_without_shares", "payout_policy_manifest", "onchain_entitlements"), - ("bundle_without_shares", "settlement_mode_decision", "direct_recipients"), - ("bundle_without_shares", "settlement_mode_decision", "fanout_chunks"), - ("bundle_without_shares", "ctv_fanout_manifest_set", "manifests"), - ("share_window_proof", "share_parts"), - ("share_parts",), - ) - - def _compact_body_reconstructs_to( - self, - body_path: Path, - *, - expected: str, - final_bundle: Mapping[str, Any], - ) -> bool: - """The compact body at ``body_path`` reconstructs exactly to ``final_bundle``. - - Streams instead of reconstructing: the body is scanned with its - window-sized members left lazy, the header is compared member by - member (the reward manifest's counted shares record by record), and - every share part is read one bounded segment at a time and matched - against the corresponding records of the logical window. The same - wrapper identity, proof and part validations apply as in the - reconstruction readers; no recovery callback is consulted, so a - segment missing from disk is a mismatch exactly as before. - """ - try: - body = self._scan_owned_artifact(body_path, lazy_paths=self._BODY_LAZY_PATHS) - except (OSError, RuntimeError, CanonicalArtifactError): - return False - try: - schema = body.get("schema") - if schema == AUDIT_BODY_REF_SCHEMA: - return self._body_ref_matches( - body, - body_path=body_path, - expected=expected, - final_bundle=final_bundle, - ) - if schema == AUDIT_BUNDLE_V2_SCHEMA: - return self._bundle_v2_matches( - body, - body_path=body_path, - expected=expected, - final_bundle=final_bundle, - ) - return False - except ( - OSError, - RuntimeError, - TypeError, - ValueError, - UnicodeDecodeError, - json.JSONDecodeError, - ): - return False - finally: - body.close() - - def _header_matches( - self, - body: Mapping[str, Any], - final_bundle: Mapping[str, Any], - ) -> bool: - without_shares = body.get("bundle_without_shares") - if not isinstance(without_shares, Mapping): - return False - expected_without = { - key: value for key, value in final_bundle.items() if key != "shares" - } - if without_shares != expected_without: - return False - keys = list(final_bundle) - expected_index = keys.index("shares") if "shares" in keys else len(keys) - return int(body.get("shares_key_index", len(without_shares))) == expected_index - - def _parts_match_window( - self, - parts: Sequence[Any], - *, - body_path: Path, - final_bundle: Mapping[str, Any], - allow_inline: bool, - ) -> tuple[int, int | None, int | None] | None: - """Match ordered share parts against the logical window, streaming. - - Returns (count, first_share_seq, last_share_seq) of the matched - records, or None on any mismatch. Each segment part is read - bounded by the segment size; inline parts are bounded by the body. - """ - expected_shares = final_bundle.get("shares") - if not self._is_share_sequence(expected_shares): - return None - expected_iter = iter(expected_shares) - count = 0 - first_seq: int | None = None - last_seq: int | None = None - previous_last_share_seq: int | None = None - for part in parts: - if isinstance(part, RawJsonRecord): - # An inline part above the in-process limit: the part is - # the unit here, so it is decoded whole (documented). - part = materialize_json(part) - if not isinstance(part, Mapping): - return None - kind = part.get("kind") - segment_view: CanonicalAuditBundleView | None = None - if kind in {"segment", "segment_range", "segment_prefix"}: - segment_view, part_shares = self._open_share_segment_slice( - part, - parent_body_uri=str(body_path), - ) - elif ( - allow_inline - and kind == "inline" - and self._is_share_sequence(part.get("shares")) - ): - inline = part["shares"] - if len(inline) != int(part.get("share_count") or 0): - return None - self._validate_share_range( - int(part.get("first_share_seq") or 0), - int(part.get("last_share_seq") or 0), - inline, - ) - part_shares = inline - else: - return None - try: - first_share_seq = int(part.get("first_share_seq") or 0) - last_share_seq = int(part.get("last_share_seq") or 0) - if ( - previous_last_share_seq is not None - and first_share_seq <= previous_last_share_seq - ): - return None - previous_last_share_seq = last_share_seq - for record in part_shares: - try: - expected_record = next(expected_iter) - except StopIteration: - return None - if record != expected_record: - return None - if isinstance(record, dict): - try: - seq = int(record.get("share_seq") or 0) - except (TypeError, ValueError): - seq = 0 - if first_seq is None: - first_seq = seq - last_seq = seq - count += 1 - finally: - if segment_view is not None: - segment_view.close() - try: - next(expected_iter) - except StopIteration: - return count, first_seq, last_seq - return None - - def _body_ref_matches( - self, - body: Mapping[str, Any], - *, - body_path: Path, - expected: str, - final_bundle: Mapping[str, Any], - ) -> bool: - declared = _canonical_hex( - body.get("audit_bundle_sha256"), - name="audit_bundle_sha256", - ) - self._validate_body_wrapper_identity(body, str(body_path), declared) - if not hmac.compare_digest(declared, expected): - return False - parts = body.get("share_parts") - if not self._is_share_sequence(parts) or not self._header_matches(body, final_bundle): - return False - matched = self._parts_match_window( - parts, - body_path=body_path, - final_bundle=final_bundle, - allow_inline=True, - ) - if matched is None: - return False - return matched[0] == int(body.get("share_count") or 0) - - def _bundle_v2_matches( - self, - body: Mapping[str, Any], - *, - body_path: Path, - expected: str, - final_bundle: Mapping[str, Any], - ) -> bool: - declared = _canonical_hex( - body.get("audit_bundle_sha256"), - name="audit_bundle_sha256", - ) - self._validate_body_wrapper_identity(body, str(body_path), declared) - if not hmac.compare_digest(declared, expected): - return False - proof = body.get("share_window_proof") - if not isinstance(proof, dict): - return False - if proof.get("schema") != AUDIT_WINDOW_COMPLETENESS_PROOF_SCHEMA: - return False - if int(body.get("share_count") or 0) != int(proof.get("share_count") or 0): - return False - parts = proof.get("share_parts") - if not self._is_share_sequence(parts): - return False - expected_parts_digest = str(proof.get("share_parts_digest_hex") or "").lower() - actual_parts_digest = self.share_parts_digest_hex(parts) - if expected_parts_digest != actual_parts_digest: - return False - if not self._header_matches(body, final_bundle): - return False - matched = self._parts_match_window( - parts, - body_path=body_path, - final_bundle=final_bundle, - allow_inline=False, - ) - if matched is None: - return False - count, first_seq, last_seq = matched - if count != int(proof.get("share_count") or 0): - return False - if count and ( - (first_seq or 0) != int(proof.get("first_share_seq") or 0) - or (last_seq or 0) != int(proof.get("last_share_seq") or 0) - ): - return False - reward_manifest = final_bundle.get("reward_manifest") - copied_proof_fields = ( - "anchor_job_issued_at_ms", - "anchor_share_seq", - "newest_share_seq", - "oldest_share_seq", - "included_share_count", - "requested_window_weight", - "counted_window_weight", - "share_slice_digest_hex", - ) - if isinstance(reward_manifest, Mapping): - for field in copied_proof_fields: - if (field in proof) != (field in reward_manifest) or proof.get( - field - ) != reward_manifest.get(field): - return False - elif any(field in proof for field in copied_proof_fields): - return False - return True - - def validate_canonical_source( - self, - path: Path, - expected_sha256: str, - final_bundle: Mapping[str, Any] | None = None, - ) -> None: - expected = _canonical_hex( - expected_sha256, - name="audit_bundle_sha256", - ) - try: - source = self._open_owned_artifact_source(Path(path)) - except OSError as exc: - raise RuntimeError( - f"canonical audit bundle is not retrievable at {path}: {exc}" - ) from exc - try: - actual = source.sha256_hex() - if not hmac.compare_digest(actual, expected): - raise RuntimeError( - f"audit bundle sha256 mismatch: expected {expected}, got {actual}" - ) - if final_bundle is not None: - self._require_logical_match(source, final_bundle) - finally: - source.close() - self._validate_root_identity() - - def audit_body_byte_len( - self, - body_uri: object | None, - final_bundle: Mapping[str, Any], - canonical_bundle_path: Path | None = None, - ) -> int: - if body_uri: - value = self._owned_regular_size(self.resolve_owned_path(body_uri)) - self._validate_root_identity() - return value - if canonical_bundle_path is not None: - value = self._owned_regular_size(Path(canonical_bundle_path)) - self._validate_root_identity() - return value - if isinstance(final_bundle, CanonicalAuditBundleView): - final_bundle.verify_identity() - value = final_bundle.byte_length - else: - value = len(self.canonical_audit_bundle_bytes(final_bundle)) - self._validate_root_identity() - return value - - def _owned_regular_size(self, path: Path) -> int: - """Byte length of an owned regular file from its descriptor, unread.""" - source = self._open_owned_artifact_source(path) - try: - return source.identity.size - finally: - source.close() - - def read_external_body( - self, - body_uri: object, - *, - expected_sha256: object | None = None, - ) -> dict[str, object] | None: - if not body_uri: - return None - try: - path = self.resolve_owned_path(body_uri) - body_match = _BODY_RE.fullmatch(path.name) - if body_match is None: - raise RuntimeError("body URI is not an owned canonical body") - if expected_sha256 is not None and body_match.group("digest") != _canonical_hex( - expected_sha256, - name="audit_bundle_sha256", - ): - raise RuntimeError("body URI digest does not match expected digest") - body_bytes, _value = self._read_owned_regular_bytes(path) - except (OSError, RuntimeError, TypeError, ValueError, OverflowError) as exc: - raise RuntimeError( - f"audit bundle body is not retrievable at {body_uri}: {exc}" - ) from exc - try: - body = json.loads(body_bytes.decode("utf-8")) - except (UnicodeDecodeError, json.JSONDecodeError) as exc: - raise RuntimeError( - f"audit bundle body is not valid JSON at {body_uri}: {exc}" - ) from exc - try: - if isinstance(body, dict) and body.get("schema") == AUDIT_BODY_REF_SCHEMA: - resolved = self.resolve_audit_body_ref( - body, - expected_sha256=expected_sha256, - body_uri=body_uri, - ) - self._validate_root_identity() - return resolved - if isinstance(body, dict) and body.get("schema") == AUDIT_BUNDLE_V2_SCHEMA: - resolved = self.resolve_audit_bundle_v2( - body, - expected_sha256=expected_sha256, - body_uri=body_uri, - ) - self._validate_root_identity() - return resolved - if expected_sha256: - expected = _canonical_hex( - expected_sha256, - name="audit_bundle_sha256", - ) - actual = _sha256_bytes(body_bytes) - if not hmac.compare_digest(actual, expected): - raise RuntimeError( - f"audit bundle body hash mismatch at {body_uri}: " - f"expected {expected}, got {actual}" - ) - self._validate_root_identity() - return body - except RuntimeError: - raise - except (TypeError, ValueError, OverflowError) as exc: - raise RuntimeError( - f"audit bundle body is not valid JSON at {body_uri}: {exc}" - ) from exc - - def external_body_matches_sha(self, body_path: Path, expected: str) -> bool: - try: - self.read_external_body(str(body_path), expected_sha256=expected) - except ( - OSError, - RuntimeError, - TypeError, - ValueError, - OverflowError, - UnicodeDecodeError, - json.JSONDecodeError, - ): - return False - return True - - def external_body_available_for_sha(self, body_uri: object, expected: str) -> bool: - try: - expected = _canonical_hex(expected, name="audit_bundle_sha256") - self.read_external_body(body_uri, expected_sha256=expected) - except ( - OSError, - RuntimeError, - TypeError, - ValueError, - OverflowError, - UnicodeDecodeError, - json.JSONDecodeError, - ): - return False - return True - - def resolve_audit_body_ref( - self, - body_ref: Mapping[str, Any], - *, - expected_sha256: object | None, - body_uri: object, - verify_digest: bool = True, - load_missing_range: Callable[..., list[Any]] | None = None, - ) -> dict[str, object]: - expected = ( - _canonical_hex(expected_sha256, name="audit_bundle_sha256") - if expected_sha256 - else None - ) - declared = _canonical_hex( - body_ref.get("audit_bundle_sha256"), - name="audit_bundle_sha256", - ) - self._validate_body_wrapper_identity(body_ref, body_uri, declared) - if expected and not hmac.compare_digest(declared, expected): - raise RuntimeError( - f"audit bundle body hash mismatch at {body_uri}: expected {expected}, got {declared}" - ) - without_shares = body_ref.get("bundle_without_shares") - parts = body_ref.get("share_parts") - if not isinstance(without_shares, dict) or not isinstance(parts, list): - raise RuntimeError( - f"audit bundle body is not valid JSON at {body_uri}: invalid body reference" - ) - shares: list[Any] = [] - previous_last_share_seq: int | None = None - for part in parts: - if not isinstance(part, dict): - raise RuntimeError( - f"audit bundle body is not valid JSON at {body_uri}: invalid share part" - ) - kind = part.get("kind") - if kind in {"segment", "segment_range", "segment_prefix"}: - part_shares = self._read_share_part_with_recovery( - part, - parent_body_uri=body_uri, - load_missing_range=load_missing_range, - ) - elif kind == "inline" and isinstance(part.get("shares"), list): - inline = part["shares"] - if len(inline) != int(part.get("share_count") or 0): - raise RuntimeError( - f"audit bundle body is not valid JSON at {body_uri}: inline share count mismatch" - ) - self._validate_share_range( - int(part.get("first_share_seq") or 0), - int(part.get("last_share_seq") or 0), - inline, - ) - part_shares = inline - else: - raise RuntimeError( - f"audit bundle body is not valid JSON at {body_uri}: invalid share part kind" - ) - first_share_seq = int(part.get("first_share_seq") or 0) - last_share_seq = int(part.get("last_share_seq") or 0) - if ( - previous_last_share_seq is not None - and first_share_seq <= previous_last_share_seq - ): - raise RuntimeError( - f"audit bundle body is not valid JSON at {body_uri}: share parts overlap or are out of order" - ) - previous_last_share_seq = last_share_seq - shares.extend(part_shares) - if len(shares) != int(body_ref.get("share_count") or 0): - raise RuntimeError( - f"audit bundle body is not valid JSON at {body_uri}: share count mismatch" - ) - bundle = self._insert_shares( - without_shares, - shares, - int(body_ref.get("shares_key_index", len(without_shares))), - ) - if verify_digest: - actual = _sha256_bytes(self.canonical_audit_bundle_bytes(bundle)) - if not hmac.compare_digest(actual, declared): - raise RuntimeError( - f"audit bundle body hash mismatch at {body_uri}: expected {declared}, got {actual}" - ) - return bundle - - def resolve_audit_bundle_v2( - self, - body: Mapping[str, Any], - *, - expected_sha256: object | None, - body_uri: object, - verify_digest: bool = True, - load_missing_range: Callable[..., list[Any]] | None = None, - ) -> dict[str, object]: - expected = ( - _canonical_hex(expected_sha256, name="audit_bundle_sha256") - if expected_sha256 - else None - ) - declared = _canonical_hex( - body.get("audit_bundle_sha256"), - name="audit_bundle_sha256", - ) - self._validate_body_wrapper_identity(body, body_uri, declared) - if expected and not hmac.compare_digest(declared, expected): - raise RuntimeError( - f"audit bundle body hash mismatch at {body_uri}: expected {expected}, got {declared}" - ) - without_shares = body.get("bundle_without_shares") - proof = body.get("share_window_proof") - if not isinstance(without_shares, dict) or not isinstance(proof, dict): - raise RuntimeError( - f"audit bundle body is not valid JSON at {body_uri}: invalid v2 body" - ) - if proof.get("schema") != AUDIT_WINDOW_COMPLETENESS_PROOF_SCHEMA: - raise RuntimeError( - f"audit bundle body is not valid JSON at {body_uri}: invalid proof schema" - ) - if int(body.get("share_count") or 0) != int(proof.get("share_count") or 0): - raise RuntimeError( - f"audit bundle body is not valid JSON at {body_uri}: proof share count mismatch" - ) - parts = proof.get("share_parts") - if not isinstance(parts, list): - raise RuntimeError( - f"audit bundle body is not valid JSON at {body_uri}: missing share parts" - ) - expected_parts_digest = str(proof.get("share_parts_digest_hex") or "").lower() - actual_parts_digest = _sha256_bytes( - self.storage_json_bytes({"share_parts": parts}) - ) - if expected_parts_digest != actual_parts_digest: - raise RuntimeError( - f"audit bundle body is not valid JSON at {body_uri}: share parts digest mismatch" - ) - shares: list[Any] = [] - for part in parts: - if not isinstance(part, dict): - raise RuntimeError( - f"audit bundle body is not valid JSON at {body_uri}: invalid share part" - ) - shares.extend( - self._read_share_part_with_recovery( - part, - parent_body_uri=body_uri, - load_missing_range=load_missing_range, - ) - ) - if len(shares) != int(proof.get("share_count") or 0): - raise RuntimeError( - f"audit bundle body is not valid JSON at {body_uri}: share count mismatch" - ) - if shares and ( - int(shares[0].get("share_seq") or 0) - != int(proof.get("first_share_seq") or 0) - or int(shares[-1].get("share_seq") or 0) - != int(proof.get("last_share_seq") or 0) - ): - raise RuntimeError( - f"audit bundle body is not valid JSON at {body_uri}: proof range mismatch" - ) - bundle = self._insert_shares( - without_shares, - shares, - int(body.get("shares_key_index", len(without_shares))), - ) - reward_manifest = bundle.get("reward_manifest") - copied_proof_fields = ( - "anchor_job_issued_at_ms", - "anchor_share_seq", - "newest_share_seq", - "oldest_share_seq", - "included_share_count", - "requested_window_weight", - "counted_window_weight", - "share_slice_digest_hex", - ) - if isinstance(reward_manifest, dict): - for field in copied_proof_fields: - if (field in proof) != (field in reward_manifest) or proof.get( - field - ) != reward_manifest.get(field): - raise RuntimeError( - f"audit bundle body is not valid JSON at {body_uri}: proof {field} mismatch" - ) - elif any(field in proof for field in copied_proof_fields): - raise RuntimeError( - f"audit bundle body is not valid JSON at {body_uri}: proof has no reward manifest" - ) - if verify_digest: - actual = _sha256_bytes(self.canonical_audit_bundle_bytes(bundle)) - if not hmac.compare_digest(actual, declared): - raise RuntimeError( - f"audit bundle body hash mismatch at {body_uri}: expected {declared}, got {actual}" - ) - return bundle - - def _validate_body_wrapper_identity( - self, - body: Mapping[str, Any], - body_uri: object, - declared_digest: str, - ) -> None: - path = self.resolve_owned_path(body_uri) - match = _BODY_RE.fullmatch(path.name) - if match is None: - raise RuntimeError("compact audit body path is invalid") - block_hash = _canonical_hex(body.get("block_hash"), name="block_hash") - if ( - block_hash != match.group("block") - or declared_digest != match.group("digest") - ): - raise RuntimeError("compact audit body identity mismatch") - - @staticmethod - def _insert_shares( - without_shares: Mapping[str, Any], - shares: list[Any], - shares_key_index: int, - ) -> dict[str, object]: - bundle: dict[str, object] = {} - inserted = False - for index, (key, value) in enumerate(without_shares.items()): - if index == shares_key_index: - bundle["shares"] = shares - inserted = True - bundle[str(key)] = value - if not inserted: - bundle["shares"] = shares - return bundle - - def _resolve_share_segment_path( - self, - part: Mapping[str, Any], - ) -> tuple[Path, str, re.Match[str]]: - """Validate a share part's kind, owned path and filename bounds.""" - body_uri = part.get("body_uri") - kind = str(part.get("kind") or "") - path = self.resolve_owned_path(body_uri) - if self.artifact_kind(path.name) != "share_segment": - raise RuntimeError("not an owned share segment") - if kind == "segment": - match = _SHARE_CONTENT_RE.fullmatch(path.name) - if match is None: - raise RuntimeError("immutable share segment path is invalid") - if ( - int(match.group("first")) != int(part.get("first_share_seq") or 0) - or int(match.group("last")) != int(part.get("last_share_seq") or 0) - ): - raise RuntimeError("immutable share segment path bounds mismatch") - elif kind in {"segment_range", "segment_prefix"}: - match = _SHARE_SLOT_RE.fullmatch(path.name) - if match is None: - # An irreparable stable-slot merge publishes the incoming - # range at an immutable content-addressed URI instead of - # rewriting the slot; its filename bounds are the exact - # declared range. - content_match = _SHARE_CONTENT_RE.fullmatch(path.name) - if content_match is None: - raise RuntimeError("share segment slot path is invalid") - if ( - int(content_match.group("first")) - != int(part.get("first_share_seq") or 0) - or int(content_match.group("last")) - != int(part.get("last_share_seq") or 0) - ): - raise RuntimeError( - "immutable share segment range path bounds mismatch" - ) - match = content_match - else: - declared_slot_first = int( - part.get("segment_first_share_seq") - or match.group("first") - ) - declared_slot_last = int( - part.get("segment_last_share_seq") - or match.group("last") - ) - if ( - declared_slot_first != int(match.group("first")) - or declared_slot_last != int(match.group("last")) - ): - raise RuntimeError("share segment slot path bounds mismatch") - else: - raise RuntimeError("invalid share part kind") - return path, kind, match - - def _open_share_segment_slice( - self, - part: Mapping[str, Any], - *, - parent_body_uri: object, - ) -> tuple[CanonicalAuditBundleView, Sequence[Any]]: - """:meth:`read_audit_share_segment` with the segment left on disk. - - Performs the same path, digest, header, contiguity, count and range - checks, but the segment is scanned into a bounded view whose share - array stays lazy, and the selected range comes back as a slice of - that array. The caller closes the returned view. Used by the - publication-time reconstruction check so no segment -- and no single - oversized record inside one -- is loaded whole (#255). - """ - body_uri = part.get("body_uri") - kind = str(part.get("kind") or "") - if kind not in {"segment", "segment_range", "segment_prefix"}: - raise RuntimeError( - f"audit bundle body is not valid JSON at {parent_body_uri}: " - "invalid share part kind" - ) - try: - path, kind, match = self._resolve_share_segment_path(part) - view = self._scan_owned_artifact(path, lazy_paths=(("shares",),)) - except (OSError, RuntimeError, CanonicalArtifactError) as exc: - raise RuntimeError( - f"audit bundle body is not retrievable at {parent_body_uri}: " - f"share segment {body_uri}: {exc}" - ) from exc - try: - if kind == "segment": - expected = str(part.get("sha256") or "").lower() - actual = view.sha256_hex - if not hmac.compare_digest(actual, expected): - raise RuntimeError( - f"audit bundle body hash mismatch at {parent_body_uri}: " - f"share segment {body_uri} expected {expected}, got {actual}" - ) - segment_shares = view.get("shares") - if ( - view.get("schema") != AUDIT_SHARE_SEGMENT_SCHEMA - or not self._is_share_sequence(segment_shares) - ): - raise RuntimeError( - f"audit bundle body is not valid JSON at {parent_body_uri}: " - f"invalid share segment {body_uri}" - ) - try: - segment_first = int(view.get("first_share_seq")) - segment_last = int(view.get("last_share_seq")) - segment_count = int(view.get("share_count")) - self._validate_share_range( - segment_first, - segment_last, - segment_shares, - ) - except (TypeError, ValueError, RuntimeError) as exc: - raise RuntimeError( - f"audit bundle body is not valid JSON at {parent_body_uri}: " - f"share segment {body_uri} header mismatch" - ) from exc - if segment_count != len(segment_shares): - raise RuntimeError( - f"audit bundle body is not valid JSON at {parent_body_uri}: " - f"share segment {body_uri} share count mismatch" - ) - if kind == "segment": - if ( - segment_first != int(match.group("first")) - or segment_last != int(match.group("last")) - ): - raise RuntimeError("immutable share segment header bounds mismatch") - elif ( - segment_first < int(match.group("first")) - or segment_last > int(match.group("last")) - ): - raise RuntimeError("share segment header escapes slot bounds") - first_share_seq = int(part.get("first_share_seq") or 0) - last_share_seq = int(part.get("last_share_seq") or 0) - # The header check above proved strict +1 contiguity from - # segment_first, so the requested range is an index range. - if first_share_seq > last_share_seq: - selected: Sequence[Any] = [] - elif ( - first_share_seq < segment_first - or last_share_seq > segment_last - ): - raise RuntimeError( - f"audit bundle body is not valid JSON at {parent_body_uri}: " - f"share segment {body_uri} does not contain requested range" - ) - else: - selected = segment_shares[ - first_share_seq - segment_first : last_share_seq - segment_first + 1 - ] - if len(selected) != int(part.get("share_count") or 0): - raise RuntimeError( - f"audit bundle body is not valid JSON at {parent_body_uri}: " - f"share segment {body_uri} share count mismatch" - ) - if kind in {"segment_range", "segment_prefix"}: - key = "range_sha256" if kind == "segment_range" else "prefix_sha256" - actual, _size = json_chunks_sha256_and_size( - iter_json_byte_chunks( - self.audit_share_segment_payload( - first_share_seq=first_share_seq, - last_share_seq=last_share_seq, - shares=selected, - ) - ) - ) - expected = str(part.get(key) or "").lower() - if not hmac.compare_digest(actual, expected): - raise RuntimeError( - f"audit bundle body hash mismatch at {parent_body_uri}: " - f"share segment range {body_uri} expected {expected}, got {actual}" - ) - self._validate_root_identity() - return view, selected - except BaseException: - view.close() - raise - - def read_audit_share_segment( - self, - part: Mapping[str, Any], - *, - parent_body_uri: object, - ) -> list[Any]: - body_uri = part.get("body_uri") - kind = str(part.get("kind") or "") - if kind not in {"segment", "segment_range", "segment_prefix"}: - raise RuntimeError( - f"audit bundle body is not valid JSON at {parent_body_uri}: " - "invalid share part kind" - ) - try: - path, kind, match = self._resolve_share_segment_path(part) - segment_bytes, _value = self._read_owned_regular_bytes(path) - except (OSError, RuntimeError) as exc: - raise RuntimeError( - f"audit bundle body is not retrievable at {parent_body_uri}: " - f"share segment {body_uri}: {exc}" - ) from exc - if kind == "segment": - expected = str(part.get("sha256") or "").lower() - actual = _sha256_bytes(segment_bytes) - if not hmac.compare_digest(actual, expected): - raise RuntimeError( - f"audit bundle body hash mismatch at {parent_body_uri}: " - f"share segment {body_uri} expected {expected}, got {actual}" - ) - try: - segment = json.loads(segment_bytes) - except (UnicodeDecodeError, json.JSONDecodeError) as exc: - raise RuntimeError( - f"audit bundle body is not valid JSON at {parent_body_uri}: " - f"share segment {body_uri}: {exc}" - ) from exc - if ( - not isinstance(segment, dict) - or segment.get("schema") != AUDIT_SHARE_SEGMENT_SCHEMA - or not isinstance(segment.get("shares"), list) - ): - raise RuntimeError( - f"audit bundle body is not valid JSON at {parent_body_uri}: " - f"invalid share segment {body_uri}" - ) - segment_shares = segment["shares"] - try: - segment_first = int(segment.get("first_share_seq")) - segment_last = int(segment.get("last_share_seq")) - segment_count = int(segment.get("share_count")) - self._validate_share_range( - segment_first, - segment_last, - segment_shares, - ) - except (TypeError, ValueError, RuntimeError) as exc: - raise RuntimeError( - f"audit bundle body is not valid JSON at {parent_body_uri}: " - f"share segment {body_uri} header mismatch" - ) from exc - if segment_count != len(segment_shares): - raise RuntimeError( - f"audit bundle body is not valid JSON at {parent_body_uri}: " - f"share segment {body_uri} share count mismatch" - ) - if kind == "segment": - assert match is not None - if ( - segment_first != int(match.group("first")) - or segment_last != int(match.group("last")) - ): - raise RuntimeError("immutable share segment header bounds mismatch") - else: - assert match is not None - if ( - segment_first < int(match.group("first")) - or segment_last > int(match.group("last")) - ): - raise RuntimeError("share segment header escapes slot bounds") - selected = self.select_audit_share_segment_range( - segment_shares, - first_share_seq=int(part.get("first_share_seq") or 0), - last_share_seq=int(part.get("last_share_seq") or 0), - parent_body_uri=parent_body_uri, - body_uri=body_uri, - ) - if len(selected) != int(part.get("share_count") or 0): - raise RuntimeError( - f"audit bundle body is not valid JSON at {parent_body_uri}: " - f"share segment {body_uri} share count mismatch" - ) - if kind in {"segment_range", "segment_prefix"}: - key = "range_sha256" if kind == "segment_range" else "prefix_sha256" - actual = _sha256_bytes( - self.storage_json_bytes( - self.audit_share_segment_payload( - first_share_seq=int(part.get("first_share_seq") or 0), - last_share_seq=int(part.get("last_share_seq") or 0), - shares=selected, - ) - ) - ) - expected = str(part.get(key) or "").lower() - if not hmac.compare_digest(actual, expected): - raise RuntimeError( - f"audit bundle body hash mismatch at {parent_body_uri}: " - f"share segment range {body_uri} expected {expected}, got {actual}" - ) - elif kind != "segment": - raise RuntimeError( - f"audit bundle body is not valid JSON at {parent_body_uri}: invalid share part kind" - ) - self._validate_root_identity() - return selected - - @staticmethod - def select_audit_share_segment_range( - shares: list[Any], - *, - first_share_seq: int, - last_share_seq: int, - parent_body_uri: object, - body_uri: object, - ) -> list[Any]: - selected: list[Any] = [] - previous_seq: int | None = None - for share in shares: - if not isinstance(share, dict): - raise RuntimeError( - f"audit bundle body is not valid JSON at {parent_body_uri}: " - f"share segment {body_uri} has invalid share" - ) - share_seq = int(share.get("share_seq") or 0) - if previous_seq is not None and previous_seq + 1 != share_seq: - raise RuntimeError( - f"audit bundle body is not valid JSON at {parent_body_uri}: " - f"share segment {body_uri} is not contiguous" - ) - previous_seq = share_seq - if first_share_seq <= share_seq <= last_share_seq: - selected.append(share) - if selected: - if int(selected[0].get("share_seq") or 0) != first_share_seq: - selected = [] - elif int(selected[-1].get("share_seq") or 0) != last_share_seq: - selected = [] - if not selected and first_share_seq <= last_share_seq: - raise RuntimeError( - f"audit bundle body is not valid JSON at {parent_body_uri}: " - f"share segment {body_uri} does not contain requested range" - ) - return selected - - -SCAN_CHUNK_BYTES_LOCAL = 256 * 1024 - - -class _ChunkReader(io.RawIOBase): - """Sequential read adapter over an :class:`ArtifactSource`.""" - - def __init__(self, source: ArtifactSource) -> None: - super().__init__() - self._source = source - self._offset = 0 - - def readable(self) -> bool: - return True - - def readinto(self, buffer: Any) -> int: - view = memoryview(buffer) - chunk = self._source.pread(self._offset, len(view)) - view[: len(chunk)] = chunk - self._offset += len(chunk) - return len(chunk) - - -def _gzip_canonical_chunks(chunks: Iterable[bytes]) -> Iterator[bytes]: - """:func:`gzip_canonical_bundle_bytes` over a chunk stream, chunk by chunk. - - Same compressor, level, header and (absent) mtime, so equal inputs give - the same compressed bytes as the whole-body helper; deflate output does - not depend on how the input was split. - """ - - sink = io.BytesIO() - with gzip.GzipFile( - fileobj=sink, - mode="wb", - compresslevel=CANONICAL_BUNDLE_GZIP_LEVEL, - mtime=0, - ) as handle: - for chunk in chunks: - handle.write(chunk) - if sink.tell(): - piece = sink.getvalue() - sink.seek(0) - sink.truncate() - yield piece - piece = sink.getvalue() - if piece: - yield piece - - -def _bytes_equal_chunks(expected: bytes, chunks: Iterable[bytes]) -> bool: - """``expected == b"".join(chunks)`` without joining the chunks.""" - offset = 0 - view = memoryview(expected) - for chunk in chunks: - end = offset + len(chunk) - if end > len(view) or not hmac.compare_digest(bytes(view[offset:end]), chunk): - return False - offset = end - return offset == len(view) - - -def _pread_exact(fd: int, offset: int, size: int) -> bytes: - """Read exactly ``size`` bytes at ``offset``; short at end of file.""" - pieces: list[bytes] = [] - remaining = size - while remaining > 0: - try: - chunk = os.pread(fd, remaining, offset) - except InterruptedError: - continue - if not chunk: - break - pieces.append(chunk) - offset += len(chunk) - remaining -= len(chunk) - return b"".join(pieces) if len(pieces) != 1 else pieces[0] - - -def _canonical_hex_bytes(value: object, *, name: str) -> str: - text = str(value) - if len(text) % 2: - raise ValueError(f"{name} must contain complete hexadecimal bytes") - try: - bytes.fromhex(text) - except ValueError as exc: - raise ValueError(f"{name} must be hexadecimal") from exc - return text.lower() diff --git a/lab/prism/audit_bundle_view.py b/lab/prism/audit_bundle_view.py deleted file mode 100644 index ad9a9b63f..000000000 --- a/lab/prism/audit_bundle_view.py +++ /dev/null @@ -1,2366 +0,0 @@ -#!/usr/bin/env python3 -"""Bounded views over an immutable canonical audit artifact (issue #255). - -The canonical audit bundle the Rust builder writes for a found block carries -two window-sized arrays: the top-level ``shares`` (one record per accepted -share in the payout window) and ``reward_manifest.shares`` (one record per -counted share). Loading that artifact with ``json.load`` materialized both -as Python dictionaries inside the lease-bearing coordinator process, and -every downstream consumer -- the logical comparison against the candidate -file, compact-body segmentation, reconstruction verification, and the -accepted-block persistence payload -- parsed or encoded the whole window -again. - -This module keeps the verified artifact file as the sole authority and -exposes it through bounded objects: - -* :class:`CanonicalAuditBundleView` is an immutable mapping of the bundle's - top-level members. Small members (found block, balances, policy and - coinbase manifests) are parsed eagerly; the window-sized arrays are - exposed as :class:`LazyRecordSequence` objects that decode records on - demand from the file through bounded read windows. -* :class:`LazyRecordSequence` is a replayable ``collections.abc.Sequence`` - of dictionaries: every iteration re-reads the artifact, random access - seeks to a checkpoint and scans at most one stride of records, and no - window-sized list, tuple, string or dictionary is ever built. Equality - against a list or another lazy sequence streams both sides record by - record. -* :func:`iter_json_chunks` encodes a payload that embeds such sequences into - the exact ``json.dumps(payload, separators=(",", ":"))`` text in bounded - chunks, so compact bodies and share segments can be written and digested - without a whole-window string. - -The scanner never rejects a valid document because of its size: a record -larger than the scan window grows that one decode to the record's own size -and the window is trimmed again afterwards, so an oversized single field is -isolated to its own decode rather than refused. - -The byte contract is the Rust typed canonical field order. Nothing here -sorts keys or otherwise re-canonicalizes; the v1 candidate-identity JSON -(sorted keys, ``default=str``) is a different encoder with a different -digest and is never derived from these views. -""" - -from __future__ import annotations - -from contextlib import contextmanager - -import codecs -from collections.abc import Mapping, Sequence -from dataclasses import dataclass -import hashlib -import json -import os -from pathlib import Path -import re -import stat -import struct -import subprocess -import sys -import tempfile -import threading -import time -from typing import Any, Callable, Iterable, Iterator -import weakref - -from lab.prism.share_json_stream import iter_json_array_text_chunks - - -# Bytes read per positional read while scanning or replaying an artifact. -# Also the ceiling on decoded text retained between records for ordinary -# records; one oversized record grows the window to its own size only. -SCAN_CHUNK_BYTES = 256 * 1024 -# Records between byte-offset checkpoints of a lazy sequence. Random access -# seeks to the nearest checkpoint and skips at most this many records. -RECORD_INDEX_STRIDE = 256 -# Records per ``json.dumps`` call inside the streaming encoder. -JSON_BATCH_RECORDS = 256 -# Target size of one yielded encoder chunk. -JSON_CHUNK_CHARS = 64 * 1024 -# Member paths of a canonical audit bundle whose length scales with the -# window (one entry per accepted or counted share), with the block's -# transactions (one leaf per transaction) or with recipient cardinality -# (balances, accounts, entitlements, settlement recipients and fanout -# manifests). Recipient cardinality is itself bounded only by the window, -# so none of these is a byte bound; every one stays on disk. What remains -# eager is fixed-size: the found block, policy, attestation, digests and -# the signed coinbase manifest, whose outputs are capped by configuration. -CANONICAL_BUNDLE_LAZY_PATHS: tuple[tuple[str, ...], ...] = ( - ("shares",), - ("reward_manifest", "shares"), - ("reward_manifest", "entitlements"), - ("witness_merkle_leaves_hex",), - ("audit_commitment_leaves_hex",), - ("prior_balances",), - ("payout_policy_manifest", "accounts"), - ("payout_policy_manifest", "onchain_entitlements"), - ("settlement_mode_decision", "direct_recipients"), - ("settlement_mode_decision", "fanout_chunks"), - ("ctv_fanout_manifest_set", "manifests"), -) -# Largest single JSON value the coordinator decodes in-process with one -# ``raw_decode`` call. A record above this limit is normalized by an -# isolated helper process instead (see RawJsonRecord); it is never -# rejected. Values above the limit are also counted in the scan statistics. -RECORD_DECODE_SOFT_LIMIT_BYTES = 1024 * 1024 -# Members of an isolated record that are kept in the coordinator as plain -# values: any member whose compact encoding is at most this many bytes. -RAW_RECORD_MEMBER_LIMIT_BYTES = 4096 -# Wall-clock ceiling for one isolated record normalization, admission wait -# included. -RAW_RECORD_HELPER_TIMEOUT_SECONDS = 300.0 -# Isolated helper processes admitted at once: each one may hold a whole -# record (and two encodings of it), so the slots bound helper memory. -RAW_RECORD_HELPER_SLOTS = 1 -RAW_RECORD_HELPER_MEMORY_BYTES = 4 * 1024 * 1024 * 1024 -RAW_RECORD_HELPER_HEADER_BYTES = 1024 * 1024 -# Supervisor poll interval while a helper runs or a slot is awaited. -RAW_RECORD_HELPER_POLL_SECONDS = 0.05 -# Bytes of a helper's diagnostics retained (the tail) for the error report. -RAW_RECORD_HELPER_DIAGNOSTIC_BYTES = 64 * 1024 - -_WHITESPACE = re.compile(r"[ \t\n\r]*") -# Strict JSON token grammar (RFC 8259 plus the standard decoder's NaN and -# Infinity constants). The quantifiers are possessive: when a window ends -# inside a long token the match must fail in one pass instead of -# backtracking through every prefix, which would allocate a regex state -# stack proportional to the window. -_WS = r"[ \t\n\r]*+" -_STRING_TOKEN = r'"(?:[^"\\\x00-\x1f]++|\\(?:["\\/bfnrt]|u[0-9a-fA-F]{4}))*+"' -_NUMBER_TOKEN = r"-?(?:0|[1-9][0-9]*+)(?:\.[0-9]++)?+(?:[eE][+-]?[0-9]++)?+" -_SCALAR_TOKEN = rf"(?:{_STRING_TOKEN}|{_NUMBER_TOKEN}|true|false|null|NaN|Infinity|-Infinity)" -# One complete JSON object with no nested containers: every share record -# in a canonical bundle. Matching it validates and skips the record at C -# speed without building a dictionary; anything it does not match (a -# record cut by the window, a nested value, or a malformed one) goes to -# the streaming validator below, which reports malformed input. -_FLAT_OBJECT = re.compile( - r"\{" - + _WS - + rf"(?:{_STRING_TOKEN}{_WS}:{_WS}{_SCALAR_TOKEN}" - + rf"(?:{_WS},{_WS}{_STRING_TOKEN}{_WS}:{_WS}{_SCALAR_TOKEN})*+)?+" - + _WS - + r"\}" -) -_NON_STRING_SCALAR = re.compile( - rf"(?:{_NUMBER_TOKEN}|true|false|null|NaN|Infinity|-Infinity)\Z" -) -# Characters that end a string's ordinary run: the closing quote, an -# escape, or a control character the grammar forbids. -_STRING_STRUCTURAL = re.compile(r'["\\\x00-\x1f]') -_SCALAR_END = re.compile(r"[,\]}: \t\n\r\"{\[]") -_ESCAPE_TOKEN = re.compile(r'["\\/bfnrt]|u[0-9a-fA-F]{4}') -# Characters that can extend a JSON number token past a decoded prefix. -_NUMBER_CONTINUATION = frozenset("0123456789.eE+-") -_DECODER = json.JSONDecoder() -_COMPACT_SEPARATORS = (",", ":") -# Checkpoint entries buffered before they are written to the index file. -_INDEX_FLUSH_BYTES = 4096 - - -def _close_fd(fd: int) -> None: - try: - os.close(fd) - except OSError: - pass - - -def _scratch_fd(purpose: str) -> int: - """An unlinked temporary file's descriptor, or resource pressure.""" - try: - handle = tempfile.TemporaryFile() - except OSError as exc: - raise ArtifactResourcePressure( - f"cannot create the {purpose} scratch file: {exc}" - ) from exc - try: - return os.dup(handle.fileno()) - except OSError as exc: - raise ArtifactResourcePressure( - f"cannot duplicate the {purpose} scratch descriptor: {exc}" - ) from exc - finally: - handle.close() - - -def _pwrite_all(fd: int, data: bytes, offset: int, *, purpose: str) -> None: - view = memoryview(data) - while view: - try: - written = os.pwrite(fd, view, offset) - except InterruptedError: - continue - except OSError as exc: - raise ArtifactResourcePressure( - f"cannot write the {purpose} scratch file: {exc}" - ) from exc - view = view[written:] - offset += written - - -class _CheckpointIndex: - """Fixed-width record-offset checkpoints for the lazy arrays of one scan. - - Entries are 8-byte little-endian byte offsets appended while scanning - and read back with positional reads, so the index scales with the - window on disk, never in memory: an unlinked temporary file holds it - (the share-window spool's approach) and its descriptor closes with the - last owner. There is no in-memory fallback: when the scratch file - cannot be created or written, :class:`ArtifactResourcePressure` is - raised so the caller retries later rather than growing the process. - """ - - __slots__ = ( - "__weakref__", - "_count", - "_fd", - "_finalizer", - "_pending", - "_written", - ) - - ENTRY = struct.Struct("<Q") - - def __init__(self) -> None: - self._count = 0 - self._written = 0 - self._pending = bytearray() - self._fd = _scratch_fd("checkpoint index") - self._finalizer = weakref.finalize(self, _close_fd, self._fd) - - @property - def count(self) -> int: - return self._count - - def append(self, offset: int) -> None: - self._pending += self.ENTRY.pack(int(offset)) - if len(self._pending) >= _INDEX_FLUSH_BYTES: - self.flush() - self._count += 1 - - def flush(self) -> None: - if not self._pending: - return - if not self._finalizer.alive: - raise CanonicalArtifactError("canonical audit artifact index is closed") - _pwrite_all( - self._fd, - bytes(self._pending), - self._written * self.ENTRY.size, - purpose="checkpoint index", - ) - self._written += len(self._pending) // self.ENTRY.size - self._pending = bytearray() - - def get(self, index: int) -> int: - if not 0 <= index < self._count: - raise IndexError("checkpoint index out of range") - self.flush() - if not self._finalizer.alive: - raise CanonicalArtifactError("canonical audit artifact index is closed") - entry = os.pread(self._fd, self.ENTRY.size, index * self.ENTRY.size) - if len(entry) != self.ENTRY.size: - raise CanonicalArtifactError("canonical audit artifact index is truncated") - return int(self.ENTRY.unpack(entry)[0]) - - def close(self) -> None: - self._finalizer() - - -class ArtifactResourcePressure(Exception): - """A scratch resource (checkpoint index, record spool, helper) is unavailable. - - Raised instead of degrading to an unbounded in-memory structure. The - artifact itself is intact, so this is deliberately neither a - :class:`CanonicalArtifactError` nor a :class:`json.JSONDecodeError`: - callers must not classify it as corruption or mismatch, and the - operation is retry-safe once the pressure passes. - """ - - -class CanonicalArtifactError(ValueError): - """The artifact cannot be used: identity changed, closed, or not a file.""" - - -class CanonicalArtifactSyntaxError(CanonicalArtifactError, json.JSONDecodeError): - """The artifact is not a well-formed JSON document. - - Also a :class:`json.JSONDecodeError`, so callers that classified the - historical ``json.load`` failure keep classifying this one the same way. - """ - - def __init__(self, msg: str, doc: str = "", pos: int = 0) -> None: - json.JSONDecodeError.__init__(self, msg, doc, pos) - - def __str__(self) -> str: - return str(self.msg) - - -@dataclass(frozen=True) -class ArtifactIdentity: - """The stat identity a view binds to; mirrors the store's file identity.""" - - device: int - inode: int - mode: int - size: int - mtime_ns: int - - @classmethod - def from_stat(cls, value: os.stat_result) -> ArtifactIdentity: - return cls( - value.st_dev, - value.st_ino, - value.st_mode, - value.st_size, - value.st_mtime_ns, - ) - - def matches(self, value: os.stat_result) -> bool: - return ( - stat.S_ISREG(value.st_mode) - and self.device == value.st_dev - and self.inode == value.st_ino - and self.mode == value.st_mode - and self.size == value.st_size - and self.mtime_ns == value.st_mtime_ns - ) - - -class ArtifactSource: - """One open regular file read only through bounded positional reads. - - Owns its descriptor: the file stays readable after its path is unlinked - or replaced, and every read re-checks the descriptor's identity so a - truncated or rewritten inode is reported rather than silently reread. - The descriptor closes when the source is closed or garbage collected. - """ - - def __init__(self, fd: int, *, path: Path | None = None) -> None: - value = os.fstat(fd) - if not stat.S_ISREG(value.st_mode): - os.close(fd) - raise CanonicalArtifactError("canonical audit artifact is not a regular file") - self._fd = fd - self.path = path - self.identity = ArtifactIdentity.from_stat(value) - self._finalizer = weakref.finalize(self, os.close, fd) - - @property - def closed(self) -> bool: - return not self._finalizer.alive - - def close(self) -> None: - self._finalizer() - - def fileno(self) -> int: - if self.closed: - raise CanonicalArtifactError("canonical audit artifact source is closed") - return self._fd - - def verify_identity(self) -> None: - if self.closed: - raise CanonicalArtifactError("canonical audit artifact source is closed") - if not self.identity.matches(os.fstat(self._fd)): - raise CanonicalArtifactError("canonical audit artifact changed after it was scanned") - - def pread(self, offset: int, size: int) -> bytes: - """Read up to ``size`` bytes at ``offset``; empty at end of file.""" - fd = self.fileno() - pieces: list[bytes] = [] - remaining = size - while remaining > 0: - try: - chunk = os.pread(fd, remaining, offset) - except InterruptedError: - continue - if not chunk: - break - pieces.append(chunk) - offset += len(chunk) - remaining -= len(chunk) - return b"".join(pieces) if len(pieces) != 1 else pieces[0] - - def iter_bytes(self, *, chunk_bytes: int = SCAN_CHUNK_BYTES) -> Iterator[bytes]: - """The complete file in bounded chunks; identity checked at both ends.""" - self.verify_identity() - offset = 0 - while True: - chunk = self.pread(offset, chunk_bytes) - if not chunk: - break - offset += len(chunk) - yield chunk - self.verify_identity() - if offset != self.identity.size: - raise CanonicalArtifactError("canonical audit artifact size changed while reading") - - def sha256_hex(self, *, chunk_bytes: int = SCAN_CHUNK_BYTES) -> str: - digest = hashlib.sha256() - for chunk in self.iter_bytes(chunk_bytes=chunk_bytes): - digest.update(chunk) - return digest.hexdigest() - - -class ScanStats: - """Exact allocation maxima observed while scanning one artifact. - - ``max_value_chars`` is the largest single JSON value handed to one - ``raw_decode`` call (a lazy-array record or an eager member); - ``window_high_water_chars`` is the largest decoded text window held at - any moment (at most one read chunk beyond the largest value); - ``oversized_values`` counts values above - :data:`RECORD_DECODE_SOFT_LIMIT_BYTES`. - """ - - __slots__ = ( - "checkpoint_entries", - "isolated_records", - "max_value_chars", - "oversized_values", - "values_decoded", - "window_high_water_chars", - ) - - def __init__(self) -> None: - self.max_value_chars = 0 - self.oversized_values = 0 - self.values_decoded = 0 - self.window_high_water_chars = 0 - self.checkpoint_entries = 0 - # Records normalized by the isolated helper instead of in-process. - self.isolated_records = 0 - - def note_value(self, chars: int) -> None: - """Record one value's size: decoded text characters, or bytes when - the value was skipped structurally (equal for ASCII artifacts).""" - self.values_decoded += 1 - if chars > self.max_value_chars: - self.max_value_chars = chars - if chars > RECORD_DECODE_SOFT_LIMIT_BYTES: - self.oversized_values += 1 - - def note_window(self, chars: int) -> None: - if chars > self.window_high_water_chars: - self.window_high_water_chars = chars - - def as_dict(self) -> dict[str, int]: - return { - "max_value_chars": self.max_value_chars, - "oversized_values": self.oversized_values, - "values_decoded": self.values_decoded, - "window_high_water_chars": self.window_high_water_chars, - "checkpoint_entries": self.checkpoint_entries, - "isolated_records": self.isolated_records, - } - - -class _Cursor: - """A decoded text window over an artifact, advanced record by record. - - ``text`` holds a bounded decoded window starting at ``window_start`` - bytes into the file; ``pos`` is the character index of the next token. - Reads happen only when the parser needs more text, feed the optional - hasher, and invoke the optional checkpoint callable so a cancellation - check runs at least once per chunk. - """ - - __slots__ = ( - "_ascii", - "_checkpoint", - "_chunk_bytes", - "_decoder", - "_hasher", - "_next_read", - "_source", - "_window_start", - "eof", - "generation", - "pos", - "stats", - "text", - ) - - def __init__( - self, - source: ArtifactSource, - offset: int, - *, - chunk_bytes: int, - hasher: Any | None = None, - checkpoint: Callable[[], None] | None = None, - stats: ScanStats | None = None, - ) -> None: - self._source = source - self._chunk_bytes = max(1, int(chunk_bytes)) - self._hasher = hasher - self._checkpoint = checkpoint - self._decoder = codecs.getincrementaldecoder("utf-8")() - self._window_start = int(offset) - self._next_read = int(offset) - self._ascii = True - self.text = "" - self.pos = 0 - self.eof = False - # Bumped whenever consumed text is dropped, so a caller can tell - # whether a slice of ``text`` taken before a skip is still valid. - self.generation = 0 - self.stats = stats if stats is not None else ScanStats() - - @property - def byte_pos(self) -> int: - """Byte offset in the file of ``text[pos]``.""" - if self._ascii: - return self._window_start + self.pos - return self._window_start + len(self.text[: self.pos].encode("utf-8")) - - def _append(self, piece: str) -> None: - if not piece: - return - if self._ascii and not piece.isascii(): - self._ascii = False - self.text = self.text + piece if self.text else piece - self.stats.note_window(len(self.text)) - - def fill(self, min_ahead: int) -> None: - """Ensure ``min_ahead`` characters follow ``pos`` unless at EOF.""" - while not self.eof and len(self.text) - self.pos < min_ahead: - chunk = self._source.pread(self._next_read, self._chunk_bytes) - if self._checkpoint is not None: - self._checkpoint() - if not chunk: - self.eof = True - try: - tail = self._decoder.decode(b"", final=True) - except UnicodeDecodeError as exc: - raise CanonicalArtifactSyntaxError( - f"canonical audit artifact is not valid UTF-8: {exc}" - ) from exc - self._append(tail) - break - self._next_read += len(chunk) - if self._hasher is not None: - self._hasher.update(chunk) - try: - self._append(self._decoder.decode(chunk)) - except UnicodeDecodeError as exc: - raise CanonicalArtifactSyntaxError( - f"canonical audit artifact is not valid UTF-8: {exc}" - ) from exc - - def trim(self, *, force: bool = False) -> None: - """Drop consumed text once it exceeds one chunk; keeps windows bounded.""" - if self.pos < self._chunk_bytes and not force: - return - if self.pos == 0: - return - self._window_start = self.byte_pos - self.text = self.text[self.pos :] - self.pos = 0 - self._ascii = self.text.isascii() - self.generation += 1 - - def skip_ws(self) -> None: - while True: - self.pos = _WHITESPACE.match(self.text, self.pos).end() - if self.pos < len(self.text) or self.eof: - return - self.fill(1) - - def peek(self) -> str: - self.fill(1) - return self.text[self.pos] if self.pos < len(self.text) else "" - - def expect(self, char: str) -> None: - if self.peek() != char: - raise CanonicalArtifactSyntaxError( - f"canonical audit artifact is malformed at byte {self.byte_pos}: " - f"expected {char!r}" - ) - self.pos += 1 - - def at_eof(self) -> bool: - self.fill(1) - return self.eof and self.pos >= len(self.text) - - def decode_value(self) -> Any: - """Decode one JSON value at ``pos`` (whitespace already skipped). - - A value cut by the window boundary extends the window until it fits; - the growth is geometric so an oversized field costs a handful of - reads, and nothing beyond that one value is retained. - - Token completion is checked explicitly for numbers: ``raw_decode`` - happily returns the prefix ``1`` of a window ending in ``1e+`` or - ``1.`` without touching the window end, so a number is complete - only when the character after it cannot continue a number (or the - file has ended). A continuation character with the whole file in - view is a malformed number, exactly as the standard decoder reports - for the complete document. - """ - growth = self._chunk_bytes - while True: - self.fill(1) - try: - value, end = _DECODER.raw_decode(self.text, self.pos) - except json.JSONDecodeError as exc: - if self.eof: - raise CanonicalArtifactSyntaxError( - f"canonical audit artifact is malformed: {exc}" - ) from exc - self.fill(len(self.text) - self.pos + growth) - growth *= 2 - continue - if end >= len(self.text): - if not self.eof: - # A number or literal may continue in the next chunk. - self.fill(len(self.text) - self.pos + growth) - growth *= 2 - continue - elif ( - isinstance(value, (int, float)) - and not isinstance(value, bool) - and self.text[end] in _NUMBER_CONTINUATION - ): - if not self.eof: - self.fill(len(self.text) - self.pos + growth) - growth *= 2 - continue - raise CanonicalArtifactSyntaxError( - f"canonical audit artifact is malformed at byte {self.byte_pos}: " - "invalid number" - ) - self.stats.note_value(end - self.pos) - self.pos = end - return value - - def skip_value(self) -> None: - """Validate and advance past one JSON value without retaining it. - - A flat record that fits the window is validated and skipped by one - strict-grammar regex match. Anything else -- a record straddling - the window, a nested value, a string or scalar of any size -- is - walked by a streaming token validator window by window, so the text - held never exceeds about two read chunks even for a value far larger - than the chunk. The validation is the standard decoder's: token - syntax, escapes, control characters, separators, container balance - and termination all fail closed here, at scan time. - """ - char = self.peek() - if char == "{": - match = _FLAT_OBJECT.match(self.text, self.pos) - if match is not None: - self.stats.note_value(match.end() - self.pos) - self.pos = match.end() - return - start = self.byte_pos - self._walk_value() - self.stats.note_value(self.byte_pos - start) - - def _malformed(self, detail: str) -> CanonicalArtifactSyntaxError: - return CanonicalArtifactSyntaxError( - f"canonical audit artifact is malformed at byte {self.byte_pos}: {detail}" - ) - - def _advance_window(self) -> None: - """Consume the whole window and read the next chunk.""" - self.pos = len(self.text) - self.trim(force=True) - self.fill(1) - if self.eof and self.pos >= len(self.text): - raise CanonicalArtifactSyntaxError( - "canonical audit artifact is malformed: unterminated value" - ) - - def _walk_value(self) -> None: - """Streaming validator for one JSON value starting at ``pos``.""" - stack: list[str] = [] - state = "value" - while True: - self.skip_ws() - char = self.peek() - if state == "value": - if char == '"': - self.pos += 1 - self._skip_string_body() - state = "after" - elif char == "{": - self.pos += 1 - stack.append("O") - state = "key_or_end" - elif char == "[": - self.pos += 1 - stack.append("A") - state = "value_or_end" - else: - self._skip_scalar() - state = "after" - elif state == "value_or_end": - if char == "]": - self.pos += 1 - stack.pop() - state = "after" - else: - state = "value" - continue - elif state == "key_or_end": - if char == "}": - self.pos += 1 - stack.pop() - state = "after" - elif char == '"': - self.pos += 1 - self._skip_string_body() - state = "colon" - else: - raise self._malformed("expected a member name or '}'") - elif state == "key": - if char != '"': - raise self._malformed("expected a member name") - self.pos += 1 - self._skip_string_body() - state = "colon" - elif state == "colon": - if char != ":": - raise self._malformed("expected ':'") - self.pos += 1 - state = "value" - else: # after a complete value - if not stack: - return - top = stack[-1] - if char == ",": - self.pos += 1 - state = "key" if top == "O" else "value" - elif char == "]" and top == "A": - self.pos += 1 - stack.pop() - elif char == "}" and top == "O": - self.pos += 1 - stack.pop() - elif char == "": - raise CanonicalArtifactSyntaxError( - "canonical audit artifact is malformed: unterminated value" - ) - else: - raise self._malformed("expected ',' or a closing bracket") - if self.pos >= self._chunk_bytes: - self.trim() - - def _skip_string_body(self) -> None: - """Validate to just past the closing quote; ``pos`` is inside the string.""" - while True: - match = _STRING_STRUCTURAL.search(self.text, self.pos) - if match is None: - self._advance_window() - continue - char = match.group() - self.pos = match.end() - if char == '"': - return - if char != "\\": - self.pos -= 1 - raise self._malformed("control character in string") - # An escape needs up to five more characters; make them visible - # before validating so a window boundary cannot split it. - self.fill(5) - escape = _ESCAPE_TOKEN.match(self.text, self.pos) - if escape is None: - raise self._malformed("invalid escape sequence") - self.pos = escape.end() - if self.pos >= self._chunk_bytes: - self.trim() - - def _skip_scalar(self) -> None: - """Validate one number or literal token; it may span windows.""" - start = self.byte_pos - pieces: list[str] = [] - while True: - match = _SCALAR_END.search(self.text, self.pos) - if match is None: - if self.eof: - pieces.append(self.text[self.pos :]) - self.pos = len(self.text) - break - pieces.append(self.text[self.pos :]) - self._advance_window() - continue - pieces.append(self.text[self.pos : match.start()]) - self.pos = match.start() - break - token = "".join(pieces) - if not token: - raise CanonicalArtifactSyntaxError( - f"canonical audit artifact is malformed at byte {start}: expected a value" - ) - if _NON_STRING_SCALAR.match(token) is None: - raise CanonicalArtifactSyntaxError( - f"canonical audit artifact is malformed at byte {start}: invalid token {token[:32]!r}" - ) - - -def _is_plain(value: Any, depth: int) -> bool: - """True when ``json.dumps`` can encode ``value`` without a lazy member.""" - if value is None or isinstance(value, (str, int, float, bool)): - return True - if depth <= 0: - return isinstance(value, (dict, list, tuple)) - if isinstance(value, dict): - return all(_is_plain(item, depth - 1) for item in value.values()) - if isinstance(value, (list, tuple)): - return all(_is_plain(item, depth - 1) for item in value) - return False - - -class RawJsonDocument: - """A pre-encoded JSON value: a byte span of a source emitted verbatim. - - The streaming encoder copies its bytes chunk by chunk, so a whole - canonical artifact can be embedded in a larger JSON document (the - legacy inline persistence lane) without decoding or re-encoding it. - """ - - __slots__ = ("_end", "_source", "_start") - - def __init__(self, source: ArtifactSource, *, start: int = 0, end: int | None = None) -> None: - self._source = source - self._start = int(start) - self._end = int(source.identity.size if end is None else end) - - @property - def byte_length(self) -> int: - return self._end - self._start - - def iter_byte_chunks(self, *, chunk_bytes: int = SCAN_CHUNK_BYTES) -> Iterator[bytes]: - self._source.verify_identity() - offset = self._start - while offset < self._end: - chunk = self._source.pread(offset, min(chunk_bytes, self._end - offset)) - if not chunk: - raise CanonicalArtifactError("canonical audit artifact span is truncated") - offset += len(chunk) - yield chunk - self._source.verify_identity() - - def iter_text_chunks(self, *, chunk_bytes: int = SCAN_CHUNK_BYTES) -> Iterator[str]: - decoder = codecs.getincrementaldecoder("utf-8")() - try: - for chunk in self.iter_byte_chunks(chunk_bytes=chunk_bytes): - text = decoder.decode(chunk) - if text: - yield text - tail = decoder.decode(b"", final=True) - except UnicodeDecodeError as exc: - raise CanonicalArtifactSyntaxError( - f"canonical audit artifact is not valid UTF-8: {exc}" - ) from exc - if tail: - yield tail - - -class StreamedJsonString: - """A JSON string value whose text is produced chunk by chunk. - - The streaming encoder escapes each chunk as it goes, so a canonical - encoding of a recipient-scaled member can be embedded as a *string* - member of a larger document (the CTV recovery payload's - ``manifest_set_json``) without ever holding the text whole. Chunks are - Python text, so a boundary never splits a code point and chunk-wise - escaping concatenates exactly. - """ - - __slots__ = ("_chunks_factory",) - - def __init__(self, chunks_factory: Callable[[], Iterable[str]]) -> None: - self._chunks_factory = chunks_factory - - def iter_text_chunks(self) -> Iterator[str]: - return iter(self._chunks_factory()) - - def iter_encoded_chunks(self) -> Iterator[str]: - yield '"' - for chunk in self._chunks_factory(): - if chunk: - yield json.dumps(chunk)[1:-1] - yield '"' - - -class MappedSequence(Sequence): - """A replayable sequence applying ``transform`` to another sequence's items. - - Iteration and indexing apply the transform on demand, so a derived - per-item structure (the CTV recovery artifacts derived from lazy - manifests) never exists as a whole list. - """ - - __slots__ = ("_base", "_transform") - - def __init__(self, base: Sequence[Any], transform: Callable[[Any], Any]) -> None: - self._base = base - self._transform = transform - - def __len__(self) -> int: - return len(self._base) - - def __iter__(self) -> Iterator[Any]: - for item in self._base: - yield self._transform(item) - - def __getitem__(self, index: int | slice) -> Any: - if isinstance(index, slice): - return MappedSequence(self._base[index], self._transform) - return self._transform(self._base[index]) - - def __eq__(self, other: object) -> bool: - if other is self: - return True - return _sequences_equal(self, other) - - def __ne__(self, other: object) -> bool: - result = self.__eq__(other) - if result is NotImplemented: - return result # type: ignore[return-value] - return not result - - __hash__ = None # type: ignore[assignment] - - -class RawJsonRecord(Mapping): - """One JSON value normalized outside the coordinator's decoder. - - Built for records above :data:`RECORD_DECODE_SOFT_LIMIT_BYTES`: an - isolated helper process decodes the record and returns its compact - encodings (original key order, and sorted keys for equality) plus the - members small enough to keep, which are spooled to an unlinked - temporary file. The coordinator never calls ``json.loads`` on the - record. Small members read like a mapping; a member above - :data:`RAW_RECORD_MEMBER_LIMIT_BYTES` is reachable only through the - streamed encodings and raises :class:`CanonicalArtifactError` when - indexed. Equality with a mapping or another record compares the - sorted-key encodings record by record; the streaming encoder emits the - original-order encoding verbatim. - """ - - __slots__ = ( - "__weakref__", - "_is_object", - "_keys", - "_members", - "_omitted", - "_ordered", - "_sorted", - "_spool", - ) - - def __init__( - self, - spool: ArtifactSource, - *, - ordered: tuple[int, int], - sorted_span: tuple[int, int], - keys: Sequence[str], - members: Mapping[str, Any], - omitted: Iterable[str], - is_object: bool, - ) -> None: - self._spool = spool - self._ordered = ordered - self._sorted = sorted_span - self._keys = tuple(keys) - self._members = dict(members) - self._omitted = frozenset(omitted) - self._is_object = bool(is_object) - - @property - def is_object(self) -> bool: - return self._is_object - - @property - def omitted_members(self) -> frozenset[str]: - return self._omitted - - @property - def encoded_length(self) -> int: - return self._ordered[1] - self._ordered[0] - - def __getitem__(self, key: str) -> Any: - if key in self._members: - return self._members[key] - if key in self._omitted: - raise CanonicalArtifactError( - f"record member {key!r} exceeds the in-process decode limit; " - "it is available only through the streamed encodings" - ) - raise KeyError(key) - - def __iter__(self) -> Iterator[str]: - return iter(self._keys) - - def __len__(self) -> int: - return len(self._keys) - - def __contains__(self, key: object) -> bool: - return key in self._members or key in self._omitted - - def __repr__(self) -> str: - return f"RawJsonRecord(members={list(self._keys)}, bytes={self.encoded_length})" - - def ordered_text_chunks(self) -> Iterator[str]: - return RawJsonDocument( - self._spool, - start=self._ordered[0], - end=self._ordered[1], - ).iter_text_chunks() - - def sorted_text_chunks(self) -> Iterator[str]: - return RawJsonDocument( - self._spool, - start=self._sorted[0], - end=self._sorted[1], - ).iter_text_chunks() - - def __eq__(self, other: object) -> bool: - if other is self: - return True - if isinstance(other, RawJsonRecord): - return _text_streams_equal(self.sorted_text_chunks(), other.sorted_text_chunks()) - if isinstance(other, Mapping) or ( - isinstance(other, Sequence) and not isinstance(other, (str, bytes, bytearray)) - ) or other is None or isinstance(other, (str, int, float, bool)): - try: - other_chunks = iter_json_chunks(other, sort_keys=True) - except (TypeError, ValueError): - return False - return _text_streams_equal(self.sorted_text_chunks(), other_chunks) - return NotImplemented - - def __ne__(self, other: object) -> bool: - result = self.__eq__(other) - if result is NotImplemented: - return result # type: ignore[return-value] - return not result - - __hash__ = None # type: ignore[assignment] - - def close(self) -> None: - self._spool.close() - - -def _text_streams_equal(left: Iterable[str], right: Iterable[str]) -> bool: - """Compare two text streams without joining either.""" - left_iter = iter(left) - right_iter = iter(right) - left_buffer = "" - right_buffer = "" - left_done = right_done = False - while True: - while not left_buffer and not left_done: - try: - left_buffer = next(left_iter) - except StopIteration: - left_done = True - while not right_buffer and not right_done: - try: - right_buffer = next(right_iter) - except StopIteration: - right_done = True - if left_done or right_done: - return left_done and right_done and not left_buffer and not right_buffer - size = min(len(left_buffer), len(right_buffer)) - if left_buffer[:size] != right_buffer[:size]: - return False - left_buffer = left_buffer[size:] - right_buffer = right_buffer[size:] - - -def _helper_main(argv: Sequence[str]) -> int: - """Isolated record normalization: stdin record -> header + encodings. - - Runs in a child process so a record of any size is decoded outside the - lease-bearing coordinator. Exit status 2 marks a malformed record. - """ - import resource - - resource.setrlimit(resource.RLIMIT_AS, (RAW_RECORD_HELPER_MEMORY_BYTES, RAW_RECORD_HELPER_MEMORY_BYTES)) - member_limit = RAW_RECORD_MEMBER_LIMIT_BYTES - if len(argv) >= 2 and argv[0] == "--normalize-record": - member_limit = int(argv[1]) - elif argv != ["--normalize-record"]: - sys.stderr.write("usage: --normalize-record [member-limit-bytes]\n") - return 64 - raw = sys.stdin.buffer.read() - try: - value = json.loads(raw) - except (UnicodeDecodeError, ValueError) as exc: - sys.stderr.write(f"malformed record: {exc}\n") - return 2 - ordered = json.dumps(value, separators=_COMPACT_SEPARATORS).encode("utf-8") - sorted_text = json.dumps(value, sort_keys=True, separators=_COMPACT_SEPARATORS).encode("utf-8") - keys: list[str] = [] - members: dict[str, Any] = {} - omitted: list[str] = [] - is_object = isinstance(value, dict) - if is_object: - for key, item in value.items(): - keys.append(str(key)) - if len(json.dumps(item, separators=_COMPACT_SEPARATORS)) <= member_limit: - members[str(key)] = item - else: - omitted.append(str(key)) - header = json.dumps( - { - "is_object": is_object, - "keys": keys, - "members": members, - "omitted": omitted, - "ordered_size": len(ordered), - "sorted_size": len(sorted_text), - }, - separators=_COMPACT_SEPARATORS, - ).encode("utf-8") - out = sys.stdout.buffer - out.write(header + b"\n") - out.write(ordered) - out.write(sorted_text) - out.flush() - return 0 - - -class HelperAdmission: - """Bounded admission slots shared by isolated helper processes. - - Every helper kind that decodes a whole document outside the coordinator - (the record normalizer here, the legacy candidate helper) can share one - instance so their combined memory stays bounded by the slot count. - Waiting is supervised by the caller's deadline/cancellation check. - """ - - __slots__ = ("_semaphore", "slots") - - def __init__(self, slots: int = RAW_RECORD_HELPER_SLOTS) -> None: - self.slots = max(1, int(slots)) - self._semaphore = threading.BoundedSemaphore(self.slots) - - def acquire( - self, - check: Callable[[], None], - *, - poll_seconds: float = RAW_RECORD_HELPER_POLL_SECONDS, - ) -> None: - """Take a slot; ``check`` runs between polls and raises to give up.""" - while not self._semaphore.acquire(timeout=max(0.001, float(poll_seconds))): - check() - - def release(self) -> None: - self._semaphore.release() - - @contextmanager - def hold(self, check: Callable[[], None]) -> Iterator[None]: - self.acquire(check) - try: - check() - yield - finally: - self.release() - - -RECORD_HELPER_ADMISSION = HelperAdmission() - - -def _feed_helper_stdin(process: subprocess.Popen[bytes], source: ArtifactSource, start: int, end: int) -> list[BaseException]: - errors: list[BaseException] = [] - assert process.stdin is not None - try: - offset = start - while offset < end: - chunk = source.pread(offset, min(SCAN_CHUNK_BYTES, end - offset)) - if not chunk: - raise CanonicalArtifactError("canonical audit artifact span is truncated") - process.stdin.write(chunk) - offset += len(chunk) - process.stdin.flush() - except BrokenPipeError: - pass - except BaseException as exc: # surfaced by the supervisor - errors.append(exc) - finally: - try: - process.stdin.close() - except OSError: - pass - return errors - - -def _read_exact(stream: Any, size: int) -> Iterator[bytes]: - remaining = size - while remaining > 0: - chunk = stream.read(min(SCAN_CHUNK_BYTES, remaining)) - if not chunk: - raise ArtifactResourcePressure("record helper output ended early") - remaining -= len(chunk) - yield chunk - - -class _HelperOutput: - """What the stdout reader thread collected: header facts and the spool.""" - - __slots__ = ("error", "header", "no_header", "spool_fd") - - def __init__(self) -> None: - self.header: dict[str, Any] | None = None - self.no_header = False - self.spool_fd: int | None = None - self.error: BaseException | None = None - - -def _read_helper_output(stream: Any, output: _HelperOutput) -> None: - """Read the header line, then stream both encodings into a scratch file.""" - try: - header_line = stream.readline(RAW_RECORD_HELPER_HEADER_BYTES + 1) - if len(header_line) > RAW_RECORD_HELPER_HEADER_BYTES: - raise ArtifactResourcePressure("record helper metadata exceeds its byte limit") - if not header_line: - output.no_header = True - return - try: - header = json.loads(header_line) - parsed = { - "ordered_size": int(header["ordered_size"]), - "sorted_size": int(header["sorted_size"]), - "keys": [str(key) for key in header["keys"]], - "members": dict(header["members"]), - "omitted": [str(key) for key in header["omitted"]], - "is_object": bool(header["is_object"]), - } - except (ValueError, KeyError, TypeError) as exc: - raise ArtifactResourcePressure(f"record helper header is malformed: {exc}") from exc - output.spool_fd = _scratch_fd("record spool") - offset = 0 - for chunk in _read_exact(stream, parsed["ordered_size"] + parsed["sorted_size"]): - _pwrite_all(output.spool_fd, chunk, offset, purpose="record spool") - offset += len(chunk) - output.header = parsed - except BaseException as exc: # classified by the supervisor - output.error = exc - - -def _drain_helper_stderr(stream: Any, tail: bytearray, limit: int) -> None: - """Drain diagnostics so a noisy child never blocks; keep only the tail.""" - try: - while True: - chunk = stream.read(8192) - if not chunk: - return - tail.extend(chunk) - if len(tail) > limit: - del tail[:-limit] - except OSError: - return - - -def normalize_record_isolated( - source: ArtifactSource, - start: int, - end: int, - *, - member_limit: int = RAW_RECORD_MEMBER_LIMIT_BYTES, - timeout_seconds: float = RAW_RECORD_HELPER_TIMEOUT_SECONDS, - cancellation: Callable[[], None] | None = None, - admission: HelperAdmission | None = None, -) -> RawJsonRecord: - """Normalize the record at ``source[start:end)`` in a helper process. - - The record bytes stream to the helper through a pipe and its encodings - stream back into an unlinked scratch file, so the coordinator holds at - most one read chunk of the record at any time. Every protocol step -- - the wait for an admission slot, feeding stdin, reading the header and - the encodings, draining diagnostics -- runs under one deadline - (``timeout_seconds`` from the call) and the optional ``cancellation`` - check, which raises to abandon the helper. On every exit the child is - killed if still alive, reaped, and its pipes closed; a scratch - descriptor not handed to the record is closed. No ``preexec_fn`` is - used: the coordinator is multithreaded. - - A malformed record is a :class:`CanonicalArtifactSyntaxError`. A helper - that cannot be admitted or spawned in time, exits abnormally, times out, - or a scratch file that cannot be written is - :class:`ArtifactResourcePressure`: retryable, never corruption. - """ - deadline = time.monotonic() + float(timeout_seconds) - - def check() -> None: - if cancellation is not None: - cancellation() - if time.monotonic() >= deadline: - raise ArtifactResourcePressure("record helper exceeded its deadline") - - slots = admission if admission is not None else RECORD_HELPER_ADMISSION - slots.acquire(check) - try: - return _run_record_helper(source, start, end, member_limit=member_limit, check=check) - finally: - slots.release() - - -def _run_record_helper( - source: ArtifactSource, - start: int, - end: int, - *, - member_limit: int, - check: Callable[[], None], -) -> RawJsonRecord: - check() - command = [sys.executable, "-m", __name__, "--normalize-record", str(int(member_limit))] - try: - process = subprocess.Popen( - command, - stdin=subprocess.PIPE, - stdout=subprocess.PIPE, - stderr=subprocess.PIPE, - close_fds=True, - ) - except OSError as exc: - raise ArtifactResourcePressure(f"cannot start the record helper: {exc}") from exc - assert process.stdin is not None and process.stdout is not None and process.stderr is not None - output = _HelperOutput() - feeder_errors: list[BaseException] = [] - diagnostics = bytearray() - threads = [ - threading.Thread( - target=lambda: feeder_errors.extend(_feed_helper_stdin(process, source, start, end)), - name="prism-audit-record-helper-feed", - daemon=True, - ), - threading.Thread( - target=_read_helper_output, - args=(process.stdout, output), - name="prism-audit-record-helper-output", - daemon=True, - ), - threading.Thread( - target=_drain_helper_stderr, - args=(process.stderr, diagnostics, RAW_RECORD_HELPER_DIAGNOSTIC_BYTES), - name="prism-audit-record-helper-stderr", - daemon=True, - ), - ] - try: - for thread in threads: - thread.start() - while process.poll() is None: - check() - if output.error is not None or feeder_errors: - break - time.sleep(RAW_RECORD_HELPER_POLL_SECONDS) - if process.poll() is None: - # A transport error while the child still runs: stop it now so - # the pipe threads unblock, then report the error below. - process.kill() - process.wait() - for thread in threads: - while thread.is_alive(): - check() - thread.join(RAW_RECORD_HELPER_POLL_SECONDS) - tail = diagnostics.decode("utf-8", "replace").strip() - if process.returncode == 2: - raise CanonicalArtifactSyntaxError( - "canonical audit artifact record is malformed: " + tail - ) - if process.returncode != 0: - raise ArtifactResourcePressure( - f"record helper exited with status {process.returncode}: {tail}" - ) - if feeder_errors: - raise feeder_errors[0] - if output.error is not None: - raise output.error - if output.no_header or output.header is None or output.spool_fd is None: - raise ArtifactResourcePressure("record helper produced no header: " + tail) - header = output.header - spool_fd = output.spool_fd - output.spool_fd = None - try: - spool = ArtifactSource(spool_fd, path=None) - except BaseException: - _close_fd(spool_fd) - raise - return RawJsonRecord( - spool, - ordered=(0, header["ordered_size"]), - sorted_span=(header["ordered_size"], header["ordered_size"] + header["sorted_size"]), - keys=header["keys"], - members=header["members"], - omitted=header["omitted"], - is_object=header["is_object"], - ) - finally: - if process.poll() is None: - try: - process.kill() - except OSError: - pass - try: - process.wait() - except OSError: - pass - for stream in (process.stdin, process.stdout, process.stderr): - try: - stream.close() - except OSError: - pass - for thread in threads: - thread.join(timeout=5.0) - if output.spool_fd is not None: - _close_fd(output.spool_fd) - - -def streamed_sha256_json_hex(value: Any) -> str: - """``sha256(json.dumps(value, sort_keys=True, separators=(",", ":")))``, streamed. - - The ledger's ``sha256_json_hex`` over a value that may embed lazy - sequences or isolated records, without building the encoding whole. - """ - digest = hashlib.sha256() - for chunk in iter_json_byte_chunks(value, sort_keys=True): - digest.update(chunk) - return digest.hexdigest() - - -def materialize_json(value: Any) -> Any: - """A plain JSON object graph for consumers that need one. - - Lazy sequences become lists, views become dictionaries, and an isolated - record is decoded in-process from its ordered encoding. This is the - explicit boundary for statement builders that must embed a whole - recipient-scaled member; it is never applied to a window-sized member - on the finalization path. - """ - if isinstance(value, RawJsonRecord): - return json.loads("".join(value.ordered_text_chunks())) - if isinstance(value, RawJsonDocument): - return json.loads("".join(value.iter_text_chunks())) - if isinstance(value, StreamedJsonString): - return "".join(value.iter_text_chunks()) - if not isinstance(value, (Mapping, Sequence)): - # A streamed scalar from another bounded view family (a candidate - # spool string or raw token): text when it has some, else its JSON. - if callable(getattr(value, "iter_text_chunks", None)): - return "".join(value.iter_text_chunks()) - if callable(getattr(value, "iter_encoded_chunks", None)): - return json.loads("".join(value.iter_encoded_chunks())) - if isinstance(value, Mapping): - return {str(key): materialize_json(item) for key, item in value.items()} - if isinstance(value, (str, bytes, bytearray)) or not isinstance(value, Sequence): - return value - return [materialize_json(item) for item in value] - - -class _Coalescer: - __slots__ = ("_chunk_chars", "_pending", "_pending_chars") - - def __init__(self, chunk_chars: int) -> None: - self._chunk_chars = int(chunk_chars) - self._pending: list[str] = [] - self._pending_chars = 0 - - def push(self, text: str) -> str | None: - if not text: - return None - self._pending.append(text) - self._pending_chars += len(text) - if self._pending_chars < self._chunk_chars: - return None - return self.flush() - - def flush(self) -> str | None: - if not self._pending: - return None - chunk = self._pending[0] if len(self._pending) == 1 else "".join(self._pending) - self._pending = [] - self._pending_chars = 0 - return chunk - - -def _encode_record_batches( - records: Iterable[Any], - *, - sort_keys: bool, - batch_records: int, - chunk_chars: int, -) -> Iterator[str]: - """Array body (no brackets) of a record stream, isolated records verbatim. - - Plain records are encoded ``batch_records`` at a time through one - ``json.dumps``; an isolated :class:`RawJsonRecord` streams its own - encoding between batches. - """ - batch: list[Any] = [] - first = True - - def flush() -> Iterator[str]: - nonlocal batch, first - if not batch: - return - text = json.dumps(batch, sort_keys=sort_keys, separators=_COMPACT_SEPARATORS)[1:-1] - batch = [] - if not first: - text = "," + text - first = False - yield text - - for record in records: - if isinstance(record, RawJsonRecord): - yield from flush() - if not first: - yield "," - first = False - yield from (record.sorted_text_chunks() if sort_keys else record.ordered_text_chunks()) - continue - if not _is_plain(record, 2): - yield from flush() - if not first: - yield "," - first = False - yield from _encode_pieces( - record, - sort_keys=sort_keys, - batch_records=batch_records, - chunk_chars=chunk_chars, - ) - continue - batch.append(record) - if len(batch) >= batch_records: - yield from flush() - yield from flush() - - -def _encode_pieces( - value: Any, - *, - sort_keys: bool, - batch_records: int, - chunk_chars: int, -) -> Iterator[str]: - if isinstance(value, RawJsonRecord): - yield from (value.sorted_text_chunks() if sort_keys else value.ordered_text_chunks()) - return - if isinstance(value, RawJsonDocument): - if sort_keys: - raise TypeError("a pre-encoded document cannot be re-sorted") - yield from value.iter_text_chunks() - return - if isinstance(value, StreamedJsonString): - yield from value.iter_encoded_chunks() - return - if not isinstance(value, (Mapping, Sequence)) and callable( - getattr(value, "iter_encoded_chunks", None) - ): - # A streamed scalar from another bounded view family (a candidate - # spool string or raw token) is emitted verbatim; its encoding does - # not depend on key order. Lazy containers take the generic Mapping - # and Sequence routes below, member by member. - yield from value.iter_encoded_chunks() - return - if isinstance(value, Mapping): - keys = list(value) - if any(not isinstance(key, str) for key in keys): - yield json.dumps( - value if isinstance(value, dict) else dict(value), - sort_keys=sort_keys, - separators=_COMPACT_SEPARATORS, - ) - return - if sort_keys: - keys.sort() - yield "{" - first = True - for key in keys: - yield ("" if first else ",") + json.dumps(key) + ":" - first = False - yield from _encode_pieces( - value[key], - sort_keys=sort_keys, - batch_records=batch_records, - chunk_chars=chunk_chars, - ) - yield "}" - return - if isinstance(value, (str, bytes, bytearray)) or not isinstance(value, Sequence): - yield json.dumps(value, sort_keys=sort_keys, separators=_COMPACT_SEPARATORS) - return - yield "[" - if isinstance(value, (LazyRecordSequence, LazyRecordSlice, MappedSequence)): - yield from _encode_record_batches( - value, - sort_keys=sort_keys, - batch_records=batch_records, - chunk_chars=chunk_chars, - ) - elif isinstance(value, (list, tuple)) and _is_plain(value, 2): - yield from iter_json_array_text_chunks( - value, - sort_keys=sort_keys, - batch_records=batch_records, - chunk_chars=chunk_chars, - ) - else: - first = True - for item in value: - if not first: - yield "," - first = False - yield from _encode_pieces( - item, - sort_keys=sort_keys, - batch_records=batch_records, - chunk_chars=chunk_chars, - ) - yield "]" - - -def iter_json_chunks( - value: Any, - *, - sort_keys: bool = False, - batch_records: int = JSON_BATCH_RECORDS, - chunk_chars: int = JSON_CHUNK_CHARS, -) -> Iterator[str]: - """``json.dumps(value, separators=(",", ":"), sort_keys=...)`` in bounded text chunks. - - Mappings are encoded member by member, lazy record sequences and plain - lists batch by batch (each ``json.dumps`` covers at most - ``batch_records`` items), isolated records and pre-encoded documents - verbatim from their scratch files, and every other value through one - ``json.dumps`` bounded by that value's own size. Concatenating the - chunks reproduces the compact encoding byte for byte; every chunk is - ASCII unless a pre-encoded document carries raw UTF-8. - """ - coalescer = _Coalescer(chunk_chars) - for piece in _encode_pieces( - value, - sort_keys=sort_keys, - batch_records=batch_records, - chunk_chars=chunk_chars, - ): - chunk = coalescer.push(piece) - if chunk is not None: - yield chunk - chunk = coalescer.flush() - if chunk is not None: - yield chunk - - -def iter_json_byte_chunks( - value: Any, - *, - sort_keys: bool = False, - batch_records: int = JSON_BATCH_RECORDS, - chunk_chars: int = JSON_CHUNK_CHARS, -) -> Iterator[bytes]: - """:func:`iter_json_chunks`, UTF-8 encoded chunk by chunk.""" - for chunk in iter_json_chunks( - value, - sort_keys=sort_keys, - batch_records=batch_records, - chunk_chars=chunk_chars, - ): - yield chunk.encode("utf-8") - - -def json_chunks_sha256_and_size(chunks: Iterable[bytes]) -> tuple[str, int]: - """Digest and byte count of a chunk stream without joining it.""" - digest = hashlib.sha256() - size = 0 - for chunk in chunks: - digest.update(chunk) - size += len(chunk) - return digest.hexdigest(), size - - -def _sequences_equal(left: Sequence[Any], right: Any) -> bool: - if isinstance(right, (str, bytes, bytearray)) or not isinstance(right, Sequence): - return NotImplemented # type: ignore[return-value] - if len(left) != len(right): - return False - right_iter = iter(right) - for item in left: - try: - other = next(right_iter) - except StopIteration: - return False - if item != other: - return False - try: - next(right_iter) - except StopIteration: - return True - return False - - -class LazyRecordSequence(Sequence): - """A replayable, read-only sequence of JSON records inside an artifact. - - Records live between ``start`` (the byte after the opening bracket) and - ``end`` (the byte after the closing bracket) of one JSON array in the - source. Iteration streams the array through bounded windows; indexing - seeks to the checkpoint preceding the index and skips forward. Slices - with unit step return a :class:`LazyRecordSlice` over the same file; - other slices return a list bounded by the slice itself. - """ - - __slots__ = ( - "__weakref__", - "_cancellation", - "_chunk_bytes", - "_count", - "_end", - "_index", - "_index_base", - "_source", - "_start", - "_stats", - "_stride", - ) - - def __init__( - self, - source: ArtifactSource, - *, - start: int, - end: int, - count: int, - index: _CheckpointIndex, - index_base: int, - stride: int = RECORD_INDEX_STRIDE, - chunk_bytes: int = SCAN_CHUNK_BYTES, - stats: ScanStats | None = None, - cancellation: Callable[[], None] | None = None, - ) -> None: - self._source = source - self._start = int(start) - self._end = int(end) - self._count = int(count) - self._index = index - self._index_base = int(index_base) - self._stride = max(1, int(stride)) - self._chunk_bytes = int(chunk_bytes) - self._stats = stats if stats is not None else ScanStats() - # Runs once per read chunk of every lazy read and before each - # isolated-helper step; raises to abandon the work. - self._cancellation = cancellation - - def _decode_record(self, cursor: _Cursor) -> Any: - """Decode the record at the cursor, isolating an oversized one. - - The record is first skipped structurally (bounded), which yields its - byte span. A record within the in-process limit is decoded from the - window text when it is still held, otherwise from one positional - read of exactly that span; a larger record goes to the isolated - helper and comes back as a :class:`RawJsonRecord`. - """ - cursor.skip_ws() - text_start = cursor.pos - generation = cursor.generation - start = cursor.byte_pos - cursor.skip_value() - end = cursor.byte_pos - size = end - start - if size <= RECORD_DECODE_SOFT_LIMIT_BYTES: - if cursor.generation == generation: - text = cursor.text[text_start : cursor.pos] - else: - text = self._source.pread(start, size).decode("utf-8") - try: - return json.loads(text) - except ValueError as exc: - raise CanonicalArtifactSyntaxError( - f"canonical audit artifact record is malformed: {exc}" - ) from exc - self._stats.isolated_records += 1 - return normalize_record_isolated( - self._source, start, end, cancellation=self._cancellation - ) - - @property - def source(self) -> ArtifactSource: - return self._source - - @property - def byte_range(self) -> tuple[int, int]: - return self._start, self._end - - def __len__(self) -> int: - return self._count - - def __repr__(self) -> str: - return f"LazyRecordSequence(count={self._count}, bytes={self._end - self._start})" - - def _cursor(self, offset: int) -> _Cursor: - return _Cursor( - self._source, - offset, - chunk_bytes=self._chunk_bytes, - checkpoint=self._cancellation, - ) - - def iter_range(self, start: int, stop: int) -> Iterator[Any]: - """Records ``start`` (inclusive) to ``stop`` (exclusive), streamed.""" - start = max(0, int(start)) - stop = min(self._count, int(stop)) - if start >= stop: - return - self._source.verify_identity() - block = start // self._stride - cursor = self._cursor(self._index.get(self._index_base + block)) - index = block * self._stride - while index < start: - cursor.skip_ws() - cursor.skip_value() - cursor.skip_ws() - cursor.expect(",") - cursor.trim() - index += 1 - while index < stop: - yield self._decode_record(cursor) - index += 1 - cursor.trim() - if index < self._count: - cursor.skip_ws() - cursor.expect(",") - if stop == self._count: - cursor.skip_ws() - cursor.expect("]") - self._source.verify_identity() - - def __iter__(self) -> Iterator[Any]: - return self.iter_range(0, self._count) - - def __getitem__(self, index: int | slice) -> Any: - if isinstance(index, slice): - start, stop, step = index.indices(self._count) - if step == 1: - return LazyRecordSlice(self, start, max(start, stop)) - return [self[position] for position in range(start, stop, step)] - if isinstance(index, bool) or not isinstance(index, int): - raise TypeError("lazy record indices must be integers or slices") - if index < 0: - index += self._count - if not 0 <= index < self._count: - raise IndexError("lazy record index out of range") - for record in self.iter_range(index, index + 1): - return record - raise IndexError("lazy record index out of range") - - def batches(self, size: int) -> Iterator[LazyRecordSlice]: - """Consecutive slices of at most ``size`` records.""" - size = max(1, int(size)) - for start in range(0, self._count, size): - yield LazyRecordSlice(self, start, min(self._count, start + size)) - - def __eq__(self, other: object) -> bool: - if other is self: - return True - return _sequences_equal(self, other) - - def __ne__(self, other: object) -> bool: - result = self.__eq__(other) - if result is NotImplemented: - return result # type: ignore[return-value] - return not result - - __hash__ = None # type: ignore[assignment] - - -class LazyRecordSlice(Sequence): - """A bounded window of a :class:`LazyRecordSequence`, streamed on demand.""" - - __slots__ = ("_parent", "_start", "_stop") - - def __init__(self, parent: LazyRecordSequence, start: int, stop: int) -> None: - self._parent = parent - self._start = int(start) - self._stop = int(stop) - - def __len__(self) -> int: - return self._stop - self._start - - def __repr__(self) -> str: - return f"LazyRecordSlice({self._start}:{self._stop})" - - def __iter__(self) -> Iterator[Any]: - return self._parent.iter_range(self._start, self._stop) - - def __getitem__(self, index: int | slice) -> Any: - length = len(self) - if isinstance(index, slice): - start, stop, step = index.indices(length) - if step == 1: - return LazyRecordSlice( - self._parent, - self._start + start, - self._start + max(start, stop), - ) - return [self[position] for position in range(start, stop, step)] - if isinstance(index, bool) or not isinstance(index, int): - raise TypeError("lazy record indices must be integers or slices") - if index < 0: - index += length - if not 0 <= index < length: - raise IndexError("lazy record index out of range") - return self._parent[self._start + index] - - def __eq__(self, other: object) -> bool: - if other is self: - return True - return _sequences_equal(self, other) - - def __ne__(self, other: object) -> bool: - result = self.__eq__(other) - if result is NotImplemented: - return result # type: ignore[return-value] - return not result - - __hash__ = None # type: ignore[assignment] - - -def is_lazy_sequence(value: object) -> bool: - return isinstance(value, (LazyRecordSequence, LazyRecordSlice)) - - -def _scan_array( - cursor: _Cursor, - *, - source: ArtifactSource, - index: _CheckpointIndex, - stride: int, - chunk_bytes: int, - cancellation: Callable[[], None] | None = None, -) -> Any: - cursor.skip_ws() - if cursor.peek() != "[": - return cursor.decode_value() - cursor.pos += 1 - start = cursor.byte_pos - index_base = index.count - count = 0 - cursor.skip_ws() - if cursor.peek() == "]": - cursor.pos += 1 - return LazyRecordSequence( - source, - start=start, - end=cursor.byte_pos, - count=0, - index=index, - index_base=index_base, - stride=stride, - chunk_bytes=chunk_bytes, - stats=cursor.stats, - cancellation=cancellation, - ) - while True: - cursor.skip_ws() - if count % stride == 0: - index.append(cursor.byte_pos) - cursor.skip_value() - count += 1 - cursor.trim() - cursor.skip_ws() - char = cursor.peek() - if char == ",": - cursor.pos += 1 - continue - if char == "]": - cursor.pos += 1 - break - raise CanonicalArtifactSyntaxError( - f"canonical audit artifact is malformed at byte {cursor.byte_pos}: " - "expected ',' or ']'" - ) - return LazyRecordSequence( - source, - start=start, - end=cursor.byte_pos, - count=count, - index=index, - index_base=index_base, - stride=stride, - chunk_bytes=chunk_bytes, - stats=cursor.stats, - cancellation=cancellation, - ) - - -def _parse_object( - cursor: _Cursor, - path: tuple[str, ...], - *, - lazy_paths: Sequence[tuple[str, ...]], - source: ArtifactSource, - index: _CheckpointIndex, - stride: int, - chunk_bytes: int, - cancellation: Callable[[], None] | None = None, -) -> dict[str, Any]: - cursor.skip_ws() - cursor.expect("{") - members: dict[str, Any] = {} - cursor.skip_ws() - if cursor.peek() == "}": - cursor.pos += 1 - return members - while True: - cursor.skip_ws() - if cursor.peek() != '"': - raise CanonicalArtifactSyntaxError( - f"canonical audit artifact is malformed at byte {cursor.byte_pos}: " - "expected a member name" - ) - key = cursor.decode_value() - cursor.skip_ws() - cursor.expect(":") - member_path = path + (str(key),) - if member_path in lazy_paths: - value = _scan_array( - cursor, - source=source, - index=index, - stride=stride, - chunk_bytes=chunk_bytes, - cancellation=cancellation, - ) - elif any( - len(lazy) > len(member_path) and lazy[: len(member_path)] == member_path - for lazy in lazy_paths - ): - cursor.skip_ws() - if cursor.peek() == "{": - value = _parse_object( - cursor, - member_path, - lazy_paths=lazy_paths, - source=source, - index=index, - stride=stride, - chunk_bytes=chunk_bytes, - cancellation=cancellation, - ) - else: - value = cursor.decode_value() - else: - cursor.skip_ws() - value = cursor.decode_value() - members[str(key)] = value - cursor.trim() - cursor.skip_ws() - char = cursor.peek() - if char == ",": - cursor.pos += 1 - continue - if char == "}": - cursor.pos += 1 - return members - raise CanonicalArtifactSyntaxError( - f"canonical audit artifact is malformed at byte {cursor.byte_pos}: " - "expected ',' or '}'" - ) - - -class CanonicalAuditBundleView(Mapping): - """Bounded, immutable mapping view of one canonical audit artifact. - - Construct with :meth:`scan`, which reads the file exactly once through - bounded windows, digests the raw bytes as it goes, parses every member - outside ``lazy_paths`` eagerly and indexes the lazy arrays. The view - binds the file identity it scanned; every later lazy read re-checks it. - - Equality with a dictionary (either operand order) compares member by - member, streaming the lazy arrays record by record, so callers keep the - full logical comparison the dictionary path performed without building - a window-sized object graph. - """ - - def __init__( - self, - source: ArtifactSource, - members: dict[str, Any], - *, - sha256_hex: str, - lazy_paths: tuple[tuple[str, ...], ...], - scan_stats: ScanStats | None = None, - index: _CheckpointIndex | None = None, - ) -> None: - self._source = source - self._members = members - self._sha256_hex = sha256_hex - self._lazy_paths = lazy_paths - self._scan_stats = scan_stats if scan_stats is not None else ScanStats() - self._index = index - - @classmethod - def scan( - cls, - fd: int, - *, - path: Path | None = None, - cancellation: Callable[[], None] | None = None, - chunk_bytes: int = SCAN_CHUNK_BYTES, - stride: int = RECORD_INDEX_STRIDE, - lazy_paths: Sequence[tuple[str, ...]] = CANONICAL_BUNDLE_LAZY_PATHS, - ) -> CanonicalAuditBundleView: - """Scan the artifact behind ``fd``; the view takes ownership of ``fd``. - - ``cancellation`` runs once per read chunk so a cancelled build stops - within one bounded read; the lazy sequences the view hands out keep - it, so later lazy reads and isolated-helper runs honour it as well. - The whole file is consumed so the returned digest covers every - byte, and trailing garbage is rejected. - """ - source = ArtifactSource(fd, path=path) - try: - index = _CheckpointIndex() - except BaseException: - source.close() - raise - try: - hasher = hashlib.sha256() - stats = ScanStats() - cursor = _Cursor( - source, - 0, - chunk_bytes=chunk_bytes, - hasher=hasher, - checkpoint=cancellation, - stats=stats, - ) - lazy = tuple(tuple(str(part) for part in item) for item in lazy_paths) - members = _parse_object( - cursor, - (), - lazy_paths=lazy, - source=source, - index=index, - stride=stride, - chunk_bytes=chunk_bytes, - cancellation=cancellation, - ) - cursor.skip_ws() - if not cursor.at_eof(): - raise CanonicalArtifactSyntaxError( - "canonical audit artifact has trailing data after the bundle" - ) - source.verify_identity() - if cursor.byte_pos != source.identity.size: - raise CanonicalArtifactError( - "canonical audit artifact size does not match its scan" - ) - index.flush() - stats.checkpoint_entries = index.count - except BaseException: - index.close() - source.close() - raise - return cls( - source, - members, - sha256_hex=hasher.hexdigest(), - lazy_paths=lazy, - scan_stats=stats, - index=index, - ) - - @classmethod - def scan_path( - cls, - path: Path, - *, - cancellation: Callable[[], None] | None = None, - chunk_bytes: int = SCAN_CHUNK_BYTES, - stride: int = RECORD_INDEX_STRIDE, - lazy_paths: Sequence[tuple[str, ...]] = CANONICAL_BUNDLE_LAZY_PATHS, - ) -> CanonicalAuditBundleView: - """Open ``path`` without following symlinks and scan it.""" - fd = os.open(path, os.O_RDONLY | getattr(os, "O_NOFOLLOW", 0)) - return cls.scan( - fd, - path=Path(path), - cancellation=cancellation, - chunk_bytes=chunk_bytes, - stride=stride, - lazy_paths=lazy_paths, - ) - - # Mapping protocol. - - def __getitem__(self, key: str) -> Any: - return self._members[key] - - def __iter__(self) -> Iterator[str]: - return iter(self._members) - - def __len__(self) -> int: - return len(self._members) - - def __contains__(self, key: object) -> bool: - return key in self._members - - def __repr__(self) -> str: - return ( - f"CanonicalAuditBundleView(sha256={self._sha256_hex}, " - f"bytes={self.byte_length}, members={list(self._members)})" - ) - - def __eq__(self, other: object) -> bool: - if other is self: - return True - if not isinstance(other, Mapping): - return NotImplemented - if len(self._members) != len(other): - return False - for key, value in self._members.items(): - if key not in other: - return False - if value != other[key]: - return False - return True - - def __ne__(self, other: object) -> bool: - result = self.__eq__(other) - if result is NotImplemented: - return result # type: ignore[return-value] - return not result - - __hash__ = None # type: ignore[assignment] - - # Artifact binding. - - @property - def source(self) -> ArtifactSource: - return self._source - - @property - def path(self) -> Path | None: - return self._source.path - - @property - def identity(self) -> ArtifactIdentity: - return self._source.identity - - @property - def sha256_hex(self) -> str: - return self._sha256_hex - - @property - def byte_length(self) -> int: - return self._source.identity.size - - @property - def lazy_paths(self) -> tuple[tuple[str, ...], ...]: - return self._lazy_paths - - @property - def scan_stats(self) -> ScanStats: - """Allocation maxima observed by the initial scan of this artifact.""" - return self._scan_stats - - def verify_identity(self) -> None: - self._source.verify_identity() - - def iter_bytes(self, *, chunk_bytes: int = SCAN_CHUNK_BYTES) -> Iterator[bytes]: - return self._source.iter_bytes(chunk_bytes=chunk_bytes) - - def key_index(self, key: str) -> int: - return list(self._members).index(key) - - def without(self, *keys: str) -> dict[str, Any]: - """Ordered copy of the members minus ``keys``; values are shared.""" - return { - key: value for key, value in self._members.items() if key not in keys - } - - def close(self) -> None: - """Retire the artifact and index descriptors this view owns. - - Lazy sequences handed out earlier keep their own references to the - source and index and fail closed once these are gone. - """ - if self._index is not None: - self._index.close() - self._source.close() - - @property - def closed(self) -> bool: - return self._source.closed - - -def compare_streaming(left: Any, right: Any) -> bool: - """``left == right`` where either side may embed lazy sequences or views.""" - return bool(left == right) - - -def spool_json_document( - chunks: Iterable[bytes], - *, - lazy_paths: Sequence[tuple[str, ...]], - stride: int = RECORD_INDEX_STRIDE, -) -> CanonicalAuditBundleView: - """Write a JSON document to an unlinked scratch file and scan it lazily. - - Used for derived metadata that would otherwise become a window-scaled - list -- the compact body's share-part index -- so it stays replayable - on disk with the same bounded readers as the artifact itself. - """ - fd = _scratch_fd("json spool") - try: - offset = 0 - for chunk in chunks: - _pwrite_all(fd, chunk, offset, purpose="json spool") - offset += len(chunk) - except BaseException: - _close_fd(fd) - raise - return CanonicalAuditBundleView.scan(fd, lazy_paths=lazy_paths, stride=stride) - - -__all__ = [ - "ArtifactIdentity", - "ArtifactResourcePressure", - "ArtifactSource", - "CANONICAL_BUNDLE_LAZY_PATHS", - "CanonicalArtifactError", - "CanonicalArtifactSyntaxError", - "CanonicalAuditBundleView", - "HelperAdmission", - "JSON_BATCH_RECORDS", - "JSON_CHUNK_CHARS", - "LazyRecordSequence", - "LazyRecordSlice", - "MappedSequence", - "RAW_RECORD_HELPER_DIAGNOSTIC_BYTES", - "RAW_RECORD_HELPER_POLL_SECONDS", - "RAW_RECORD_HELPER_SLOTS", - "RAW_RECORD_HELPER_TIMEOUT_SECONDS", - "RAW_RECORD_MEMBER_LIMIT_BYTES", - "RECORD_DECODE_SOFT_LIMIT_BYTES", - "RECORD_HELPER_ADMISSION", - "RECORD_INDEX_STRIDE", - "RawJsonDocument", - "RawJsonRecord", - "SCAN_CHUNK_BYTES", - "ScanStats", - "StreamedJsonString", - "compare_streaming", - "is_lazy_sequence", - "iter_json_byte_chunks", - "iter_json_chunks", - "json_chunks_sha256_and_size", - "materialize_json", - "normalize_record_isolated", - "spool_json_document", - "streamed_sha256_json_hex", -] - - -if __name__ == "__main__": - raise SystemExit(_helper_main(sys.argv[1:])) diff --git a/lab/prism/audit_http.py b/lab/prism/audit_http.py deleted file mode 100644 index 9f5e4ff14..000000000 --- a/lab/prism/audit_http.py +++ /dev/null @@ -1,706 +0,0 @@ -"""Operator audit/ops HTTP routing and bounded server lifecycle for PRISM. - -The public ``/public/v1`` read surface is served by its own process -(``lab/prism/public_read_service.py``), so nothing here imports -``lab.prism.public_api``: an unknown ``/public/v1`` request reaches this -listener's ordinary unknown-endpoint 404. -""" - -from __future__ import annotations - -from dataclasses import dataclass -from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer -import json -import threading -import time -from typing import Mapping, Protocol -import urllib.parse - - -class MetricsResponse(Protocol): - """What ``/metrics`` needs from a cached metrics read (issue #184). - - Structural on purpose: this module stays free of the observability policy - that decides the state and names the headers, and only writes what it is - handed. - """ - - @property - def status(self) -> int: ... - - @property - def body(self) -> str: ... - - def response_headers(self) -> Mapping[str, str]: ... - - -class AuditHttpPort(Protocol): - """Purpose-specific application reads exposed to the HTTP facade.""" - - def cached_health_payload(self) -> tuple[int, Mapping[str, object]]: ... - - def cached_metrics_payload(self) -> MetricsResponse: ... - - def latest_evidence_payload(self) -> Mapping[str, object] | None: ... - - def owed_balances_payload(self) -> Mapping[str, object]: ... - - def carry_forward_integrity_payload(self) -> Mapping[str, object]: ... - - def miner_status_payload(self, recipient_id: str) -> Mapping[str, object]: ... - - def ledger_backend(self) -> str: ... - - def audit_share_window( - self, - *, - anchor_job_issued_at_ms: int, - network_difficulty: int, - ) -> list[dict[str, object]]: ... - - def audit_block_payouts(self, *, block_hash: str) -> list[dict[str, object]]: ... - - def audit_ctv_fanouts(self, *, block_hash: str) -> list[dict[str, object]]: ... - - def audit_ctv_fanout_manifest_set( - self, - *, - block_hash: str, - ) -> Mapping[str, object] | None: ... - - def ctv_fanout_status( - self, - *, - fanout_txid: str, - ) -> Mapping[str, object] | None: ... - - def pending_ctv_fanout_statuses( - self, - *, - limit: int, - ) -> list[dict[str, object]]: ... - - def audit_bundle( - self, - *, - block_hash: str, - ) -> Mapping[str, object] | None: ... - - def audit_bundle_by_commitment( - self, - *, - commitment_leaf_hex: str, - ) -> Mapping[str, object] | None: ... - - -@dataclass(frozen=True) -class AuditHttpConfig: - bind: str - port: int - thread_name: str = "prism-audit-http" - join_timeout_seconds: float = 1.0 - - def __post_init__(self) -> None: - if not self.bind: - raise ValueError("audit HTTP bind must not be empty") - if self.port < 0 or self.port > 65_535: - raise ValueError("audit HTTP port must be between 0 and 65535") - if not self.thread_name: - raise ValueError("audit HTTP thread name must not be empty") - if self.join_timeout_seconds < 0: - raise ValueError("audit HTTP join timeout must be nonnegative") - - -@dataclass(frozen=True) -class AuditHttpState: - lifecycle: str - bound_address: tuple[str, int] | None - thread_alive: bool - - -class _BoundedThreadingHttpServer(ThreadingHTTPServer): - daemon_threads = True - block_on_close = False - allow_reuse_address = True - - def __init__( - self, - server_address: tuple[str, int], - handler_type: type[BaseHTTPRequestHandler], - ) -> None: - self.ready = threading.Event() - self._startup_lock = threading.Lock() - self._startup_cancelled = False - self._serve_committed = False - super().__init__(server_address, handler_type) - - def service_actions(self) -> None: - self.ready.set() - - def serve_unless_startup_cancelled(self, *, poll_interval: float) -> None: - """Commit atomically to serving or honor startup cancellation.""" - - with self._startup_lock: - if self._startup_cancelled: - return - self._serve_committed = True - self.serve_forever(poll_interval=poll_interval) - - def cancel_startup(self) -> bool: - """Cancel an uncommitted start and report whether serving committed. - - A True result means ``serve_forever()`` runs (or already ran), so - ``BaseServer.shutdown()`` will be answered; a False result means the - serve thread is guaranteed to bail out before entering the loop, so - blocking ``shutdown()`` must never be called for this server. - """ - - with self._startup_lock: - self._startup_cancelled = True - return self._serve_committed - - -class AuditHttpFacade: - """Own exact route behavior plus the listener socket and server thread.""" - - def __init__( - self, - port: AuditHttpPort, - config: AuditHttpConfig | None = None, - ) -> None: - self.port = port - self.config = config - self._lifecycle_lock = threading.Lock() - self._server: _BoundedThreadingHttpServer | None = None - self._thread: threading.Thread | None = None - self._serve_exited: threading.Event | None = None - self._lifecycle = "new" - self._handler_type = self._make_handler_type() - - def handler_type(self) -> type[BaseHTTPRequestHandler]: - return self._handler_type - - def state(self) -> AuditHttpState: - with self._lifecycle_lock: - return self._state_locked() - - def start(self) -> AuditHttpState: - config = self.config - if config is None: - raise RuntimeError("audit HTTP lifecycle requires configuration") - with self._lifecycle_lock: - if self._thread is not None and self._thread.is_alive(): - return self._state_locked() - server = _BoundedThreadingHttpServer( - (config.bind, config.port), - self._handler_type, - ) - serve_exited = threading.Event() - thread = threading.Thread( - target=self._serve, - args=(server, serve_exited), - name=config.thread_name, - daemon=True, - ) - self._server = server - self._thread = thread - self._serve_exited = serve_exited - self._lifecycle = "starting" - try: - thread.start() - except Exception: - server.server_close() - self._server = None - self._thread = None - self._serve_exited = None - self._lifecycle = "stopped" - raise - try: - if not server.ready.wait(1.0): - raise RuntimeError("audit HTTP server did not enter its serve loop") - except Exception: - self._request_serve_exit_bounded( - server, - thread, - serve_exited, - config.join_timeout_seconds, - ) - server.server_close() - thread.join(timeout=config.join_timeout_seconds) - with self._lifecycle_lock: - if self._server is server: - self._server = None - self._thread = None - self._serve_exited = None - self._lifecycle = ( - "stopped" if not thread.is_alive() else "stop_timeout" - ) - elif ( - self._server is None - and self._thread is None - and not thread.is_alive() - ): - self._lifecycle = "stopped" - raise - with self._lifecycle_lock: - if self._server is not server or self._thread is not thread: - raise RuntimeError("audit HTTP server stopped during startup") - if not thread.is_alive(): - self._lifecycle = "exited" - raise RuntimeError("audit HTTP server exited during startup") - self._lifecycle = "running" - return self._state_locked() - - def _serve( - self, - server: _BoundedThreadingHttpServer, - serve_exited: threading.Event, - ) -> None: - try: - server.serve_unless_startup_cancelled(poll_interval=0.05) - finally: - try: - server.server_close() - finally: - serve_exited.set() - with self._lifecycle_lock: - if self._server is server: - self._server = None - self._thread = None - self._serve_exited = None - if self._lifecycle in {"starting", "running"}: - self._lifecycle = "exited" - - @staticmethod - def _request_serve_exit_bounded( - server: _BoundedThreadingHttpServer, - thread: threading.Thread, - serve_exited: threading.Event, - timeout: float, - ) -> None: - """Ask the serve loop to exit without ever blocking unboundedly. - - ``BaseServer.shutdown()`` waits for ``serve_forever()`` to exit, so - calling it directly can block forever when a stop races startup - before the serve loop commits (the documented old-layer gap). - Cancel the startup window atomically first; only a committed loop is - woken, from a disposable daemon thread this caller never joins, and - the wait for the loop's exit is bounded by ``timeout``. - """ - - if not server.cancel_startup(): - # The serve thread is guaranteed to observe the cancellation - # before entering serve_forever(); there is no loop to wake. - return - if not thread.is_alive(): - return - threading.Thread( - target=server.shutdown, - name=f"{thread.name}-shutdown", - daemon=True, - ).start() - serve_exited.wait(timeout) - - def stop(self) -> bool: - with self._lifecycle_lock: - server = self._server - thread = self._thread - serve_exited = self._serve_exited - if server is None or thread is None: - self._lifecycle = "stopped" - return True - self._lifecycle = "stopping" - join_timeout = self.config.join_timeout_seconds if self.config else 1.0 - deadline = time.monotonic() + join_timeout - if serve_exited is not None: - self._request_serve_exit_bounded( - server, - thread, - serve_exited, - max(0.0, deadline - time.monotonic()), - ) - server.server_close() - thread.join(timeout=max(0.0, deadline - time.monotonic())) - stopped = not thread.is_alive() - with self._lifecycle_lock: - if stopped and self._server is server: - self._server = None - self._thread = None - self._serve_exited = None - self._lifecycle = "stopped" if stopped else "stop_timeout" - return stopped - - def _state_locked(self) -> AuditHttpState: - server = self._server - thread = self._thread - return AuditHttpState( - lifecycle=self._lifecycle, - bound_address=( - None - if server is None - else (str(server.server_address[0]), int(server.server_address[1])) - ), - thread_alive=bool(thread is not None and thread.is_alive()), - ) - - def _make_handler_type(self) -> type[BaseHTTPRequestHandler]: - facade = self - - class AuditHandler(BaseHTTPRequestHandler): - server_version = "QbitPrismAudit/0.1" - - def do_GET(self) -> None: - parsed = urllib.parse.urlparse(self.path) - path = parsed.path.rstrip("/") or "/" - query = urllib.parse.parse_qs(parsed.query) - try: - if path == "/healthz": - status, payload = facade.port.cached_health_payload() - self.write_json(status, payload) - return - if path == "/metrics": - metrics = facade.port.cached_metrics_payload() - self.write_text( - metrics.status, - metrics.body, - "text/plain; version=0.0.4", - # Freshness is stated in the response metadata as - # well as the body, so a scraper or an operator can - # tell a stale payload from a current one without - # parsing Prometheus text (issue #184). - headers=metrics.response_headers(), - ) - return - if path == "/audit/latest": - payload = facade.port.latest_evidence_payload() - if payload is None: - self.write_json( - 404, - {"error": "no PRISM evidence has been produced"}, - ) - else: - self.write_json(200, payload) - return - if path in {"/owed", "/owed-balances"}: - self.write_json(200, facade.port.owed_balances_payload()) - return - if path in { - "/audit/carry-forward-integrity", - "/audit/ledger-integrity", - }: - self.write_json( - 200, - facade.port.carry_forward_integrity_payload(), - ) - return - if path.startswith("/miners/") and path.endswith("/status"): - recipient_id = urllib.parse.unquote( - path.removeprefix("/miners/").removesuffix("/status") - ) - self.write_json( - 200, - facade.port.miner_status_payload(recipient_id), - ) - return - if path.startswith("/payouts/") and path.endswith("/status"): - recipient_id = urllib.parse.unquote( - path.removeprefix("/payouts/").removesuffix("/status") - ) - self.write_json( - 200, - facade.port.miner_status_payload(recipient_id), - ) - return - if path == "/audit/share-window": - self.handle_share_window(query) - return - if path.startswith("/audit/blocks/") and path.endswith( - "/payouts" - ): - block_hash = path.removeprefix( - "/audit/blocks/" - ).removesuffix("/payouts") - self.handle_block_payouts(block_hash) - return - if path.startswith("/audit/blocks/") and path.endswith( - "/ctv-fanouts" - ): - block_hash = path.removeprefix( - "/audit/blocks/" - ).removesuffix("/ctv-fanouts") - self.handle_block_ctv_fanouts(block_hash) - return - if path.startswith("/audit/blocks/") and path.endswith( - "/ctv-fanout-manifest-set" - ): - block_hash = path.removeprefix( - "/audit/blocks/" - ).removesuffix("/ctv-fanout-manifest-set") - self.handle_block_ctv_fanout_manifest_set(block_hash) - return - if path == "/audit/fanouts/pending": - self.handle_pending_ctv_fanouts(query) - return - if path.startswith("/audit/fanouts/") and path.endswith( - "/status" - ): - fanout_txid = path.removeprefix( - "/audit/fanouts/" - ).removesuffix("/status") - self.handle_ctv_fanout_status(fanout_txid) - return - if path.startswith("/audit/commitments/") and path.endswith( - "/bundle" - ): - commitment_leaf_hex = path.removeprefix( - "/audit/commitments/" - ).removesuffix("/bundle") - self.handle_commitment_bundle(commitment_leaf_hex) - return - if path.startswith("/audit/block/"): - block_hash = path.removeprefix("/audit/block/") - self.handle_block_payouts(block_hash) - return - if path.startswith("/audit/blocks/") and path.endswith( - "/bundle" - ): - block_hash = path.removeprefix( - "/audit/blocks/" - ).removesuffix("/bundle") - self.handle_block_bundle(block_hash) - return - self.write_json(404, {"error": "unknown endpoint"}) - except ValueError as exc: - self.write_json(400, {"error": str(exc)}) - except Exception as exc: - self.write_json(500, {"error": str(exc)}) - - def handle_share_window(self, query: dict[str, list[str]]) -> None: - anchor_raw = self.first_query_value( - query, - "anchor_job_issued_at_ms", - "anchor", - ) - difficulty_raw = self.first_query_value( - query, - "network_difficulty", - ) - if anchor_raw is None or difficulty_raw is None: - raise ValueError( - "anchor_job_issued_at_ms and network_difficulty are required" - ) - rows = facade.port.audit_share_window( - anchor_job_issued_at_ms=int(anchor_raw), - network_difficulty=int(difficulty_raw), - ) - self.write_json( - 200, - { - "schema": "qbit.prism.audit-share-window.v1", - "ledger_backend": facade.port.ledger_backend(), - "rows": rows, - }, - ) - - def handle_block_payouts(self, block_hash: str) -> None: - block_hash = self.clean_hash(block_hash) - rows = facade.port.audit_block_payouts(block_hash=block_hash) - if not rows: - self.write_json( - 404, - { - "error": "unknown PRISM block", - "block_hash": block_hash, - }, - ) - return - self.write_json( - 200, - { - "schema": "qbit.prism.audit-block-payouts.v1", - "ledger_backend": facade.port.ledger_backend(), - "block_hash": block_hash, - "rows": rows, - }, - ) - - def handle_block_ctv_fanouts(self, block_hash: str) -> None: - block_hash = self.clean_hash(block_hash, name="block hash") - rows = facade.port.audit_ctv_fanouts(block_hash=block_hash) - if not rows: - self.write_json( - 404, - { - "error": "unknown CTV fanout block", - "block_hash": block_hash, - }, - ) - return - self.write_json( - 200, - { - "schema": "qbit.prism.audit-ctv-fanouts.v1", - "ledger_backend": facade.port.ledger_backend(), - "block_hash": block_hash, - "rows": rows, - }, - ) - - def handle_block_ctv_fanout_manifest_set(self, block_hash: str) -> None: - block_hash = self.clean_hash(block_hash, name="block hash") - payload = facade.port.audit_ctv_fanout_manifest_set( - block_hash=block_hash - ) - if payload is None: - self.write_json( - 404, - { - "error": "unknown CTV fanout block", - "block_hash": block_hash, - }, - ) - return - self.write_json(200, payload) - - def handle_ctv_fanout_status(self, fanout_txid: str) -> None: - fanout_txid = self.clean_hash(fanout_txid, name="fanout txid") - payload = facade.port.ctv_fanout_status(fanout_txid=fanout_txid) - if payload is None: - self.write_json( - 404, - { - "error": "unknown CTV fanout", - "fanout_txid": fanout_txid, - }, - ) - return - self.write_json(200, payload) - - def handle_pending_ctv_fanouts( - self, - query: dict[str, list[str]], - ) -> None: - limit_raw = self.first_query_value(query, "limit") - limit = int(limit_raw) if limit_raw is not None else 100 - rows = facade.port.pending_ctv_fanout_statuses(limit=limit) - self.write_json( - 200, - { - "schema": "qbit.prism.pending-ctv-fanouts.v1", - "ledger_backend": facade.port.ledger_backend(), - "count": len(rows), - "rows": rows, - }, - ) - - def handle_block_bundle(self, block_hash: str) -> None: - block_hash = self.clean_hash(block_hash, name="block hash") - payload = facade.port.audit_bundle(block_hash=block_hash) - if payload is None: - self.write_json( - 404, - { - "error": "unknown PRISM block", - "block_hash": block_hash, - }, - ) - return - self.write_json(200, payload) - - def handle_commitment_bundle(self, commitment_leaf_hex: str) -> None: - commitment_leaf_hex = self.clean_hash( - commitment_leaf_hex, - name="audit commitment leaf", - ) - payload = facade.port.audit_bundle_by_commitment( - commitment_leaf_hex=commitment_leaf_hex - ) - if payload is None: - self.write_json( - 404, - { - "error": "unknown PRISM audit commitment", - "audit_commitment_leaf_hex": commitment_leaf_hex, - }, - ) - return - self.write_json(200, payload) - - def write_json( - self, - status: int, - payload: object, - headers: dict[str, str] | None = None, - ) -> None: - # The one content-addressed payload this listener used to - # serve (#154's /public/v1/artifacts/{sha256} RawJsonBody) - # left with the public read surface, so no route reaching - # here carries pre-serialized bytes and this module needs no - # public-API import at all. - body = json.dumps(payload, sort_keys=True).encode() + b"\n" - try: - self.send_response(status) - self.send_header("Content-Type", "application/json") - self.send_header("Content-Length", str(len(body))) - for key, value in (headers or {}).items(): - self.send_header(key, value) - self.end_headers() - self.wfile.write(body) - except (BrokenPipeError, ConnectionResetError): - # The client (typically a health checker with a short - # timeout) hung up before the response was written; - # nothing to salvage. - return - - def write_text( - self, - status: int, - payload: str, - content_type: str, - headers: Mapping[str, str] | None = None, - ) -> None: - body = payload.encode() - try: - self.send_response(status) - self.send_header("Content-Type", content_type) - self.send_header("Content-Length", str(len(body))) - for key, value in (headers or {}).items(): - self.send_header(key, value) - self.end_headers() - self.wfile.write(body) - except (BrokenPipeError, ConnectionResetError): - return - - def log_message(self, format: str, *args: object) -> None: - return - - @staticmethod - def first_query_value( - query: dict[str, list[str]], - *keys: str, - ) -> str | None: - for key in keys: - values = query.get(key) - if values: - return values[0] - return None - - @staticmethod - def clean_hash(value: str, *, name: str = "block hash") -> str: - value = urllib.parse.unquote(value).strip() - if len(value) != 64 or any( - char not in "0123456789abcdefABCDEF" for char in value - ): - raise ValueError(f"{name} must be 64 hex characters") - return value.lower() - - return AuditHandler - - -__all__ = [ - "AuditHttpConfig", - "AuditHttpFacade", - "AuditHttpPort", - "AuditHttpState", - "MetricsResponse", -] diff --git a/lab/prism/backfill_audit_bundle_canonical.py b/lab/prism/backfill_audit_bundle_canonical.py deleted file mode 100644 index 1c4fae8ae..000000000 --- a/lab/prism/backfill_audit_bundle_canonical.py +++ /dev/null @@ -1,1117 +0,0 @@ -#!/usr/bin/env python3 -"""Backfill immutable canonical audit-bundle artifacts for existing rows. - -The live publication path owns one immutable canonical artifact per -``(block_hash, audit_bundle_sha256)``: the exact byte sequence the advertised -digest was computed over, published once and never rewritten. Rows written -before that artifact existed still advertise a digest with nothing canonical -behind it. Three shapes are in the table today: - -* a pre-v2 literal external body, whose ``body_uri`` file *is* the canonical - byte sequence -- recovering it is a copy, not a re-serialization; -* a modern compact/v2 external body, which stores the bundle without its - shares plus content-addressed share segments, and has to be reconstructed - through the artifact store before it can be canonicalized; -* an inline ``qbit_pool_audit_bundles.audit_bundle`` JSONB body, the oldest - layout, which is canonicalized with the repository's own canonicalizer. - -This tool walks those rows in primary-key order, recovers the canonical bytes -for each, and verifies the advertised digest *before* anything reaches the -filesystem. It never rewrites a digest: a row whose recovered bytes do not -hash to the advertised value is reported on stderr, left untouched, and turned -into a nonzero exit status once the requested range has been processed. - -Every publication is preceded, immediately, by the ledger's pre-publication -preflight (writer-lease refresh plus current row identity confirmation), so a -writer that has lost its lease -- or a row that changed underneath this scan -- -cannot publish. - -The run is idempotent and resumable. Rows are paged by ``block_hash`` (the -primary key), the last visited hash is reported as ``last_checkpoint``, and -``--start-after`` resumes from it. Revisiting an already published row is -safe: its canonical artifact is recognized and skipped. Failed row identities -and reasons are retained in the machine-readable ``failed_rows`` summary. A -checkpoint advances past failures, so retry them by starting before the first -failed hash (or by rerunning the idempotent scan from the beginning). - -Publish mode owns the coordinator writer lease exclusively. Operators must -stop the coordinator before publishing; the command refuses lease waits rather -than sitting behind a live same-identity owner. Once the run finishes or -raises, it releases the exact lease and closes both ledger and artifact-store -resources before returning. -""" - -from __future__ import annotations - -import argparse -import hashlib -import hmac -import inspect -import json -import os -import sys -from dataclasses import asdict, dataclass -from pathlib import Path -from typing import Any, Callable, Iterator, Mapping, Sequence - -if not __package__: - sys.path.insert(0, str(Path(__file__).resolve().parents[2])) - -from lab.prism.audit_artifacts import ( - AuditArtifactConfig, - AuditArtifactStore, - DEFAULT_AUDIT_SHARE_SEGMENT_SIZE, -) -from lab.prism.backfill_ctv_fanouts import psql_command_from_env -from lab.prism.bundle_compiler import canonical_bundle_bytes -from lab.prism.coordinator_config import ( - DEFAULT_LEASE_ACQUIRE_ATTEMPTS, - DEFAULT_LEASE_ACQUIRE_LOCK_TIMEOUT_SECONDS, - DEFAULT_POSTGRES_IDLE_IN_TRANSACTION_TIMEOUT_SECONDS, - DEFAULT_POSTGRES_TCP_KEEPALIVES_COUNT, - DEFAULT_POSTGRES_TCP_KEEPALIVES_IDLE_SECONDS, - DEFAULT_POSTGRES_TCP_KEEPALIVES_INTERVAL_SECONDS, - env, - env_nonnegative_int, - env_positive_float, - env_positive_int, -) -from lab.prism.share_ledger import PsqlShareLedger, canonical_hex - - -BACKFILL_SUMMARY_SCHEMA = "qbit.prism.audit-bundle-canonical-backfill.v1" -DEFAULT_BATCH_SIZE = 100 - -# Every capability this tool needs from phase 2 is resolved by name at -# construction, first match wins, and the resolved names are reported in the -# summary. Reconciling a final signature difference is then a one-line edit -# here plus a summary line that says exactly what the run bound to. -READ_CANONICAL_CANDIDATES = ("read_canonical_audit_bundle",) -WRITE_CANONICAL_CANDIDATES = ("write_canonical_audit_bundle",) -LITERAL_BODY_CANDIDATES = ( - "literal_canonical_bundle_bytes", - "read_literal_audit_body", - "read_verified_literal_audit_body", - "read_pre_v2_audit_body", - "read_literal_external_audit_body", -) -RECONSTRUCT_CANDIDATES = ( - "canonical_bundle_bytes_from_external_body", - "reconstruct_external_audit_body", - "read_external_body", -) -LEDGER_PREFLIGHT_CANDIDATES = ( - "preflight_canonical_bundle_publication", - "preflight_audit_bundle_publication", - "confirm_audit_bundle_publication", - "refresh_lease_and_confirm_audit_bundle", -) -LEDGER_SHARE_RANGE_CANDIDATES = ( - "load_audit_share_ledger_range", - "_load_audit_share_ledger_range", -) -LEDGER_READ_JSON_CANDIDATES = ("_run_read_json", "_run_json") -PREFLIGHT_DIGEST_KEYWORDS = ( - "audit_bundle_sha256", - "digest", - "audit_bundle_sha256_hex", -) -# Whatever the reconstruction helper hands its missing-range callback, one of -# these carries the segment digest the recovered range must reproduce. -SEGMENT_DIGEST_KEYS = ( - "expected_sha256", - "range_sha256", - "prefix_sha256", - "sha256", - "segment_sha256", -) - - -class CanonicalBackfillMismatch(RuntimeError): - """Recovered bytes do not reproduce the advertised digest.""" - - -class CanonicalBackfillUnrecoverable(RuntimeError): - """No source for this row yielded bytes that could be verified.""" - - -class LedgerPreflightRefused(RuntimeError): - """The ledger declined to authorize this publication.""" - - -class MissingCanonicalCapability(RuntimeError): - """A required phase-2 method is not present on the injected object.""" - - -class BackfillWriterLeaseUnavailable(RuntimeError): - """Publish mode could not take exclusive ownership of the writer lease.""" - - -def _sha256_hex(payload: bytes) -> str: - return hashlib.sha256(payload).hexdigest() - - -def _accepts_keyword(func: Callable[..., Any], name: str) -> bool: - """Report whether ``func`` takes ``name`` as a keyword argument. - - The phase-2 signatures are assumed, not observed, so every optional - keyword this tool would like to pass is probed rather than assumed. An - unintrospectable callable (a C builtin, an exotic mock) is given the - benefit of the doubt: the documented signature is the contract. - """ - - try: - signature = inspect.signature(func) - except (TypeError, ValueError): - return True - for parameter in signature.parameters.values(): - if parameter.kind is inspect.Parameter.VAR_KEYWORD: - return True - if parameter.name == name and parameter.kind in ( - inspect.Parameter.POSITIONAL_OR_KEYWORD, - inspect.Parameter.KEYWORD_ONLY, - ): - return True - return False - - -def _resolve_capability( - owner: object, - candidates: Sequence[str], - *, - description: str, -) -> tuple[str, Callable[..., Any]]: - for name in candidates: - value = getattr(owner, name, None) - if callable(value): - return name, value - raise MissingCanonicalCapability( - f"{description} is unavailable on {type(owner).__name__}; " - f"tried: {', '.join(candidates)}" - ) - - -def _sql_text_literal(value: str) -> str: - return "'" + str(value).replace("'", "''") + "'" - - -def audit_bundle_page_sql( - *, - start_after: str | None, - limit: int, - text_literal: Callable[[str], str] = _sql_text_literal, -) -> str: - """One bounded, resumable page of ``qbit_pool_audit_bundles``. - - ``block_hash`` is the table's primary key, so ordering by it is stable and - ``block_hash > <cursor>`` resumes exactly where the previous page stopped. - The range predicate and the ordering share the column's collation, so the - pagination stays self-consistent whatever that collation is. - """ - - if limit <= 0: - raise ValueError("audit bundle page limit must be positive") - predicate = ( - "" if start_after is None else f" WHERE block_hash > {text_literal(start_after)}\n" - ) - return f""" -SELECT COALESCE(json_agg(json_build_object( - 'block_hash', page.block_hash, - 'audit_bundle_sha256', page.audit_bundle_sha256, - 'body_uri', page.body_uri, - 'audit_bundle', page.audit_bundle -) ORDER BY page.block_hash ASC), '[]'::json) -FROM ( - SELECT block_hash, audit_bundle_sha256, body_uri, audit_bundle - FROM qbit_pool_audit_bundles -{predicate} ORDER BY block_hash ASC - LIMIT {int(limit)} -) AS page; -""" - - -@dataclass(frozen=True) -class AuditBundleRow: - block_hash: str - audit_bundle_sha256: str - body_uri: str | None - audit_bundle: dict[str, Any] | None - - -@dataclass(frozen=True) -class RecoveredCanonical: - canonical_bytes: bytes - source: str - - -class CanonicalArtifactAdapter: - """The phase-2 canonical artifact surface, bound by name in one place. - - Phase 2 owns ``read_canonical_audit_bundle`` / - ``write_canonical_audit_bundle``, a verified pre-v2 literal-body reader, - and a verified external-body reconstruction helper that accepts a - missing-range callback. This adapter is the only place that names them, - so a final signature difference is reconciled here and nowhere else. - """ - - def __init__(self, store: object) -> None: - self._store = store - self._read_name, self._read = _resolve_capability( - store, - READ_CANONICAL_CANDIDATES, - description="canonical audit bundle reader", - ) - self._write_name, self._write = _resolve_capability( - store, - WRITE_CANONICAL_CANDIDATES, - description="canonical audit bundle writer", - ) - self._literal_name, self._literal = _resolve_capability( - store, - LITERAL_BODY_CANDIDATES, - description="verified pre-v2 literal audit body reader", - ) - self._reconstruct_name, self._reconstruct = _resolve_capability( - store, - RECONSTRUCT_CANDIDATES, - description="verified external audit body reconstruction helper", - ) - self._canonicalize_name, self._canonicalize = _resolve_capability( - store, - ("canonical_audit_bundle_bytes",), - description="audit bundle canonicalizer", - ) - self._literal_takes_digest = _accepts_keyword(self._literal, "expected_sha256") - self._reconstruct_takes_digest = _accepts_keyword( - self._reconstruct, - "expected_sha256", - ) - self._reconstruct_takes_callback = _accepts_keyword( - self._reconstruct, - "load_missing_range", - ) - - def resolved_interface(self) -> dict[str, str]: - return { - "read_canonical": self._read_name, - "write_canonical": self._write_name, - "literal_body_reader": self._literal_name, - "external_body_reconstruction": self._reconstruct_name, - "canonicalizer": self._canonicalize_name, - } - - def read_canonical(self, block_hash: str, digest: str) -> bytes | None: - payload = self._read(block_hash, digest) - return None if payload is None else bytes(payload) - - def write_canonical(self, block_hash: str, digest: str, payload: bytes) -> Any: - return self._write(block_hash, digest, payload) - - def read_literal_body(self, body_uri: object, digest: str) -> bytes | None: - """Return the pre-v2 body bytes verbatim, or None when it is not one. - - A compact or v2 wrapper is not the canonical byte sequence, so the - reader is expected to decline it rather than hand back wrapper bytes. - """ - - if self._literal_takes_digest: - payload = self._literal(body_uri, expected_sha256=digest) - else: - payload = self._literal(body_uri) - return None if payload is None else bytes(payload) - - def canonical_bytes_from_external_body( - self, - block_hash: str, - body_uri: object, - digest: str, - load_missing_range: Callable[..., list[Any]], - ) -> bytes | None: - if self._reconstruct_name == "canonical_bundle_bytes_from_external_body": - payload = self._reconstruct( - block_hash=block_hash, - audit_bundle_sha256=digest, - body_uri=body_uri, - load_missing_range=load_missing_range, - ) - return None if payload is None else bytes(payload) - kwargs: dict[str, Any] = {} - if self._reconstruct_takes_digest: - kwargs["expected_sha256"] = digest - if self._reconstruct_takes_callback: - kwargs["load_missing_range"] = load_missing_range - bundle = self._reconstruct(body_uri, **kwargs) - if bundle is None: - return None - if not isinstance(bundle, dict): - raise CanonicalBackfillUnrecoverable( - "external audit body reconstruction did not return a bundle object" - ) - return self.canonical_bytes(bundle) - - def canonical_bytes(self, bundle: Mapping[str, Any]) -> bytes: - return bytes(self._canonicalize(dict(bundle))) - - def segment_storage_bytes( - self, - *, - first_share_seq: int, - last_share_seq: int, - shares: list[Any], - ) -> bytes: - """Canonicalize a recovered share range with the store's own encoder.""" - - payload = self._store.audit_share_segment_payload( - first_share_seq=first_share_seq, - last_share_seq=last_share_seq, - shares=shares, - ) - return bytes(self._store.storage_json_bytes(payload)) - - -class AuditBundleLedgerAdapter: - """The PostgreSQL surface this backfill needs, bound by name in one place.""" - - def __init__(self, ledger: object) -> None: - self._ledger = ledger - self._read_json_name, self._read_json = _resolve_capability( - ledger, - LEDGER_READ_JSON_CANDIDATES, - description="ledger read-slot JSON runner", - ) - self._share_range_name, self._share_range = _resolve_capability( - ledger, - LEDGER_SHARE_RANGE_CANDIDATES, - description="ledger audit share range reader", - ) - self._preflight_name, self._preflight = _resolve_capability( - ledger, - LEDGER_PREFLIGHT_CANDIDATES, - description="ledger publication preflight", - ) - self._preflight_digest_keyword = next( - ( - keyword - for keyword in PREFLIGHT_DIGEST_KEYWORDS - if _accepts_keyword(self._preflight, keyword) - ), - PREFLIGHT_DIGEST_KEYWORDS[0], - ) - text_literal = getattr(ledger, "_text_literal", None) - self._text_literal = ( - text_literal if callable(text_literal) else _sql_text_literal - ) - - def resolved_interface(self) -> dict[str, str]: - return { - "read_json": self._read_json_name, - "share_range_reader": self._share_range_name, - "publication_preflight": self._preflight_name, - "preflight_digest_keyword": self._preflight_digest_keyword, - } - - def page(self, *, start_after: str | None, limit: int) -> list[AuditBundleRow]: - sql = audit_bundle_page_sql( - start_after=start_after, - limit=limit, - text_literal=self._text_literal, - ) - rows = self._read_json(sql) - if not isinstance(rows, list): - raise RuntimeError("audit bundle page did not return a JSON array") - page: list[AuditBundleRow] = [] - for row in rows: - if not isinstance(row, Mapping): - raise RuntimeError("audit bundle page returned a non-object row") - bundle = row.get("audit_bundle") - page.append( - AuditBundleRow( - block_hash=str(row.get("block_hash") or ""), - audit_bundle_sha256=str(row.get("audit_bundle_sha256") or ""), - body_uri=( - None if row.get("body_uri") is None else str(row["body_uri"]) - ), - audit_bundle=dict(bundle) if isinstance(bundle, Mapping) else None, - ) - ) - return page - - def load_share_ledger_range( - self, - *, - first_share_seq: int, - last_share_seq: int, - ) -> list[Any]: - shares = self._share_range( - first_share_seq=first_share_seq, - last_share_seq=last_share_seq, - ) - if not isinstance(shares, list): - raise RuntimeError("audit share ledger range did not return a list") - return shares - - def preflight_publication(self, *, block_hash: str, digest: str) -> Any: - """Refresh the writer lease and confirm this row's identity. - - Called immediately before every filesystem publication. A refusal -- - raised, reported through an ``error`` key, or an explicit falsy - ``confirmed`` -- withholds the publication. - """ - - result = self._preflight( - block_hash=block_hash, - **{self._preflight_digest_keyword: digest}, - ) - if isinstance(result, Mapping): - error = result.get("error") - if error: - raise LedgerPreflightRefused(str(error)) - confirmed = result.get("confirmed") - if confirmed is not None and not confirmed: - raise LedgerPreflightRefused( - "ledger preflight did not confirm the audit bundle row identity" - ) - elif result is False: - raise LedgerPreflightRefused( - "ledger preflight did not confirm the audit bundle row identity" - ) - return result - - -@dataclass -class BackfillCounters: - visited: int = 0 - written: int = 0 - already_present: int = 0 - dry_run: int = 0 - literal_freebie: int = 0 - reconstructed: int = 0 - inline_canonicalized: int = 0 - db_segment_reloads: int = 0 - mismatch: int = 0 - errors: int = 0 - - -class CanonicalBundleBackfill: - """Recover and publish canonical bytes for existing audit bundle rows.""" - - def __init__( - self, - *, - artifacts: CanonicalArtifactAdapter, - ledger: AuditBundleLedgerAdapter, - dry_run: bool = False, - batch_size: int = DEFAULT_BATCH_SIZE, - limit: int | None = None, - start_after: str | None = None, - stderr: Any = None, - ) -> None: - batch_size = int(batch_size) - if batch_size <= 0: - raise ValueError("batch_size must be positive") - if limit is not None: - limit = int(limit) - if limit <= 0: - raise ValueError("limit must be positive when given") - self._artifacts = artifacts - self._ledger = ledger - self._dry_run = bool(dry_run) - self._batch_size = batch_size - self._limit = limit - self._start_after = start_after - self._stderr = sys.stderr if stderr is None else stderr - self._counters = BackfillCounters() - self._checkpoint = start_after - self._failed_rows: list[dict[str, str | None]] = [] - - def run(self) -> dict[str, Any]: - for row in self._iter_rows(): - self._counters.visited += 1 - # The checkpoint advances over every visited row, including the - # ones this run refuses to publish. Those are reported loudly and - # make the run exit nonzero; silently rewinding to them on the next - # resume would stall the scan on a row that needs an operator. - self._checkpoint = row.block_hash - self._process_row(row) - return self.summary() - - def summary(self) -> dict[str, Any]: - counters = asdict(self._counters) - return { - "schema": BACKFILL_SUMMARY_SCHEMA, - **counters, - "last_checkpoint": self._checkpoint, - "start_after": self._start_after, - "batch_size": self._batch_size, - "limit": self._limit, - "dry_run_mode": self._dry_run, - "exit_code": self.exit_code(), - "failed_rows": [dict(failure) for failure in self._failed_rows], - "interface": { - "artifacts": self._artifacts.resolved_interface(), - "ledger": self._ledger.resolved_interface(), - }, - } - - def exit_code(self) -> int: - return 1 if (self._counters.mismatch or self._counters.errors) else 0 - - def _iter_rows(self) -> Iterator[AuditBundleRow]: - cursor = self._start_after - remaining = self._limit - while remaining is None or remaining > 0: - page_size = ( - self._batch_size if remaining is None else min(self._batch_size, remaining) - ) - page = self._ledger.page(start_after=cursor, limit=page_size) - if not page: - return - for row in page: - yield row - next_cursor = page[-1].block_hash - # The scan only terminates while the cursor strictly advances. A - # page whose last row carries no usable key would otherwise reissue - # the same query forever; the primary key makes that unreachable in - # a healthy database, so it is reported rather than tolerated. - if not next_cursor or (cursor is not None and next_cursor <= cursor): - self._counters.errors += 1 - self._report( - "error", - page[-1], - "audit bundle page did not advance the resume checkpoint", - ) - return - cursor = next_cursor - if remaining is not None: - remaining -= len(page) - if len(page) < page_size: - return - - def _report(self, kind: str, row: AuditBundleRow, reason: str) -> None: - self._failed_rows.append( - { - "kind": kind, - "block_hash": row.block_hash or None, - "audit_bundle_sha256": row.audit_bundle_sha256 or None, - "reason": reason, - } - ) - print( - f"prism canonical audit bundle backfill {kind} " - f"block={row.block_hash or '<missing>'} " - f"digest={row.audit_bundle_sha256 or '<missing>'} " - f"reason={reason}", - file=self._stderr, - flush=True, - ) - - def _process_row(self, row: AuditBundleRow) -> None: - try: - block_hash = canonical_hex( - row.block_hash, - name="block_hash", - expected_bytes=32, - ) - digest = canonical_hex( - row.audit_bundle_sha256, - name="audit_bundle_sha256", - expected_bytes=32, - ) - except (ValueError, TypeError) as exc: - self._counters.errors += 1 - self._report("error", row, f"row identity is not canonical: {exc}") - return - - try: - if self._already_published(block_hash, digest): - self._counters.already_present += 1 - return - recovered = self._recover(row, digest) - actual = _sha256_hex(recovered.canonical_bytes) - if not hmac.compare_digest(actual, digest): - raise CanonicalBackfillMismatch( - f"recovered {recovered.source} bytes hash to {actual}" - ) - except CanonicalBackfillMismatch as exc: - self._counters.mismatch += 1 - self._report("mismatch", row, str(exc)) - return - except Exception as exc: # noqa: BLE001 - one bad row must not end the range - self._counters.errors += 1 - self._report("error", row, f"{type(exc).__name__}: {exc}") - return - - if recovered.source == "literal": - self._counters.literal_freebie += 1 - elif recovered.source == "reconstructed": - self._counters.reconstructed += 1 - else: - self._counters.inline_canonicalized += 1 - - if self._dry_run: - self._counters.dry_run += 1 - return - - try: - # Preflight is the last thing before the filesystem is touched: - # the lease is refreshed and the row identity re-confirmed with no - # verification work left to run in between. - self._ledger.preflight_publication(block_hash=block_hash, digest=digest) - self._artifacts.write_canonical( - block_hash, - digest, - recovered.canonical_bytes, - ) - except Exception as exc: # noqa: BLE001 - report and keep scanning - self._counters.errors += 1 - self._report("error", row, f"{type(exc).__name__}: {exc}") - return - self._counters.written += 1 - - def _already_published(self, block_hash: str, digest: str) -> bool: - """Report whether a verified canonical artifact already exists. - - An unreadable artifact is treated as absent -- publication is - idempotent, so recovering and republishing is the useful answer. An - artifact that reads back with the wrong digest is not absent; it is a - corrupt publication, and it is reported rather than overwritten. - """ - - try: - existing = self._artifacts.read_canonical(block_hash, digest) - except Exception: # noqa: BLE001 - fall through to recovery - return False - if existing is None: - return False - actual = _sha256_hex(existing) - if not hmac.compare_digest(actual, digest): - raise CanonicalBackfillMismatch( - f"existing canonical artifact hashes to {actual}" - ) - return True - - def _recover(self, row: AuditBundleRow, digest: str) -> RecoveredCanonical: - reasons: list[str] = [] - if row.body_uri: - literal = self._recover_literal(row, digest, reasons) - if literal is not None: - return literal - canonical = self._reconstruct(row, digest, reasons) - if canonical is not None: - actual = _sha256_hex(canonical) - if not hmac.compare_digest(actual, digest): - raise CanonicalBackfillMismatch( - f"reconstructed external body canonicalizes to {actual}" - ) - return RecoveredCanonical(canonical, "reconstructed") - if row.audit_bundle is not None: - canonical = self._artifacts.canonical_bytes(row.audit_bundle) - actual = _sha256_hex(canonical) - if not hmac.compare_digest(actual, digest): - raise CanonicalBackfillMismatch( - f"inline bundle canonicalizes to {actual}" - ) - return RecoveredCanonical(canonical, "inline") - if not row.body_uri: - reasons.append("row has neither an inline bundle nor an external body") - else: - reasons.append("external body did not yield the advertised bytes") - raise CanonicalBackfillUnrecoverable("; ".join(reasons)) - - def _recover_literal( - self, - row: AuditBundleRow, - digest: str, - reasons: list[str], - ) -> RecoveredCanonical | None: - """Promote a pre-v2 literal body verbatim when it already verifies. - - These bytes *are* the canonical serialization the digest was computed - over, so they are copied, never parsed and re-emitted: re-serializing - would substitute this process's encoder for the one that produced the - advertised digest. - """ - - try: - literal = self._artifacts.read_literal_body(row.body_uri, digest) - except Exception as exc: # noqa: BLE001 - a compact body is not a failure - reasons.append(f"literal body reader: {type(exc).__name__}: {exc}") - return None - if literal is None: - return None - actual = _sha256_hex(literal) - if hmac.compare_digest(actual, digest): - return RecoveredCanonical(literal, "literal") - # Not a mismatch on its own: a compact or v2 wrapper never hashes to - # the bundle digest, and reconstruction is the path that reads it. - reasons.append(f"external body bytes hash to {actual}") - return None - - def _reconstruct( - self, - row: AuditBundleRow, - digest: str, - reasons: list[str], - ) -> bytes | None: - try: - return self._artifacts.canonical_bytes_from_external_body( - row.block_hash, - row.body_uri, - digest, - self._load_missing_range, - ) - except CanonicalBackfillMismatch: - raise - except Exception as exc: # noqa: BLE001 - recorded, then inline is tried - reasons.append( - f"external body reconstruction: {type(exc).__name__}: {exc}" - ) - return None - - def _load_missing_range( - self, - *, - first_share_seq: int, - last_share_seq: int, - **extra: Any, - ) -> list[Any]: - """Recover an absent on-disk share segment from the append-only ledger. - - The range is read from PostgreSQL, canonicalized with the store's own - segment encoder, and checked against the segment digest the bundle - embeds. A range that does not reproduce that digest is refused here, - before it can reach a reconstruction the final bundle digest would - then have to catch. - """ - - first_share_seq = int(first_share_seq) - last_share_seq = int(last_share_seq) - shares = self._ledger.load_share_ledger_range( - first_share_seq=first_share_seq, - last_share_seq=last_share_seq, - ) - try: - sequences = [int(share["share_seq"]) for share in shares] - except (KeyError, TypeError, ValueError) as exc: - raise CanonicalBackfillUnrecoverable( - "ledger share range has an invalid share_seq" - ) from exc - if sequences != list(range(first_share_seq, last_share_seq + 1)): - raise CanonicalBackfillUnrecoverable( - "ledger share range " - f"{first_share_seq}-{last_share_seq} is incomplete or out of order" - ) - segment_bytes = self._artifacts.segment_storage_bytes( - first_share_seq=first_share_seq, - last_share_seq=last_share_seq, - shares=shares, - ) - expected = _expected_segment_digest(extra) - if expected is not None: - actual = _sha256_hex(segment_bytes) - if not hmac.compare_digest(actual, expected): - raise CanonicalBackfillMismatch( - f"ledger share range {first_share_seq}-{last_share_seq} " - f"canonicalizes to {actual}, not the embedded {expected}" - ) - self._counters.db_segment_reloads += 1 - return shares - - -def _expected_segment_digest(extra: Mapping[str, Any]) -> str | None: - """Pull the embedded segment digest out of the callback's extra kwargs.""" - - for key in SEGMENT_DIGEST_KEYS: - value = extra.get(key) - if value: - return str(value).lower() - part = extra.get("part") - if isinstance(part, Mapping): - for key in SEGMENT_DIGEST_KEYS: - value = part.get(key) - if value: - return str(value).lower() - return None - - -def build_audit_store_from_env( - environ: Mapping[str, str] | None = None, - *, - read_only: bool = False, -) -> AuditArtifactStore: - """Open the audit artifact root the way every PRISM entry point does.""" - - source = os.environ if environ is None else environ - evidence_path = Path( - env("PRISM_EVIDENCE_PATH", "prism-live-evidence.json", environ=source) - ) - root = Path(env("PRISM_AUDIT_DIR", str(evidence_path.parent), environ=source)) - return AuditArtifactStore( - AuditArtifactConfig( - root=root, - evidence_path=evidence_path, - share_segment_size=env_nonnegative_int( - "PRISM_AUDIT_SHARE_SEGMENT_SIZE", - DEFAULT_AUDIT_SHARE_SEGMENT_SIZE, - environ=source, - ), - ), - canonicalizer=canonical_bundle_bytes, - read_only=read_only, - ) - - -def build_parser() -> argparse.ArgumentParser: - parser = argparse.ArgumentParser( - description=( - "Backfill immutable canonical audit-bundle artifacts for existing " - "qbit_pool_audit_bundles rows." - ) - ) - parser.add_argument( - "--start-after", - default=None, - help="resume after this block hash checkpoint (see last_checkpoint)", - ) - parser.add_argument( - "--batch-size", - type=int, - default=env_positive_int( - "PRISM_AUDIT_CANONICAL_BACKFILL_BATCH_SIZE", - DEFAULT_BATCH_SIZE, - ), - help="rows fetched per page", - ) - parser.add_argument( - "--limit", - type=int, - default=None, - help="stop after visiting this many rows in this run", - ) - parser.add_argument( - "--dry-run", - action="store_true", - help="recover and verify every row without publishing anything", - ) - parser.add_argument( - "--psql-command", - help="psql command; defaults to PRISM_POSTGRES_PSQL_COMMAND/PRISM_DATABASE_URL", - ) - parser.add_argument( - "--writer-id", - default=os.environ.get( - "PRISM_LEDGER_WRITER_ID", - "prism-audit-canonical-backfill", - ), - ) - parser.add_argument( - "--writer-epoch", - type=int, - default=int(os.environ.get("PRISM_LEDGER_WRITER_EPOCH", "1")), - ) - parser.add_argument( - "--writer-session-token", - default=os.environ.get("PRISM_LEDGER_WRITER_SESSION_TOKEN"), - ) - parser.add_argument( - "--lease-ttl-seconds", - type=float, - default=float(os.environ.get("PRISM_LEDGER_LEASE_TTL_SECONDS", "60")), - ) - parser.add_argument( - "--lease-acquire-lock-timeout-seconds", - type=float, - default=env_positive_float( - "PRISM_LEDGER_LEASE_ACQUIRE_LOCK_TIMEOUT_SECONDS", - DEFAULT_LEASE_ACQUIRE_LOCK_TIMEOUT_SECONDS, - ), - ) - parser.add_argument( - "--lease-acquire-attempts", - type=int, - default=env_positive_int( - "PRISM_LEDGER_LEASE_ACQUIRE_ATTEMPTS", - DEFAULT_LEASE_ACQUIRE_ATTEMPTS, - ), - ) - parser.add_argument( - "--postgres-idle-in-transaction-timeout-seconds", - type=float, - default=env_positive_float( - "PRISM_POSTGRES_IDLE_IN_TRANSACTION_TIMEOUT_SECONDS", - DEFAULT_POSTGRES_IDLE_IN_TRANSACTION_TIMEOUT_SECONDS, - ), - ) - parser.add_argument( - "--postgres-tcp-keepalives-idle-seconds", - type=int, - default=env_positive_int( - "PRISM_POSTGRES_TCP_KEEPALIVES_IDLE_SECONDS", - DEFAULT_POSTGRES_TCP_KEEPALIVES_IDLE_SECONDS, - ), - ) - parser.add_argument( - "--postgres-tcp-keepalives-interval-seconds", - type=int, - default=env_positive_int( - "PRISM_POSTGRES_TCP_KEEPALIVES_INTERVAL_SECONDS", - DEFAULT_POSTGRES_TCP_KEEPALIVES_INTERVAL_SECONDS, - ), - ) - parser.add_argument( - "--postgres-tcp-keepalives-count", - type=int, - default=env_positive_int( - "PRISM_POSTGRES_TCP_KEEPALIVES_COUNT", - DEFAULT_POSTGRES_TCP_KEEPALIVES_COUNT, - ), - ) - parser.add_argument( - "--no-init-schema", - action="store_true", - help="do not run the idempotent PRISM schema repair before backfilling", - ) - return parser - - -def ledger_from_args( - args: argparse.Namespace, - store: AuditArtifactStore, -) -> PsqlShareLedger: - return PsqlShareLedger( - psql_command=psql_command_from_env(args.psql_command), - writer_id=args.writer_id, - writer_epoch=args.writer_epoch, - writer_session_token=args.writer_session_token, - lease_retry_sleep=( - None if args.dry_run else _refuse_publish_writer_lease_wait - ), - initialize_schema=not args.no_init_schema and not args.dry_run, - read_only=args.dry_run, - lease_ttl_seconds=args.lease_ttl_seconds, - lease_acquire_lock_timeout_seconds=args.lease_acquire_lock_timeout_seconds, - lease_acquire_attempts=args.lease_acquire_attempts, - postgres_idle_in_transaction_timeout_seconds=( - args.postgres_idle_in_transaction_timeout_seconds - ), - postgres_tcp_keepalives_idle_seconds=args.postgres_tcp_keepalives_idle_seconds, - postgres_tcp_keepalives_interval_seconds=( - args.postgres_tcp_keepalives_interval_seconds - ), - postgres_tcp_keepalives_count=args.postgres_tcp_keepalives_count, - audit_artifact_store=store, - audit_bundle_canonicalizer=canonical_bundle_bytes, - ) - - -def _refuse_publish_writer_lease_wait(wait_seconds: float) -> None: - """Refuse to wait behind a live same-identity coordinator lease.""" - - raise BackfillWriterLeaseUnavailable( - "publish-mode canonical audit-bundle backfill requires exclusive " - "writer-lease ownership; stop the coordinator before publishing " - f"(lease acquisition requested a {float(wait_seconds):g}s wait)" - ) - - -def _cleanup_backfill_resources( - *, - ledger: PsqlShareLedger | None, - store: AuditArtifactStore | None, - publish_mode: bool, -) -> list[str]: - """Release the lease, then close every constructed resource.""" - - failures: list[str] = [] - if ledger is not None: - if publish_mode: - try: - released = ledger.release_writer_lease() - if released is False: - raise RuntimeError( - "the exact backfill writer lease was no longer held" - ) - except Exception as exc: # noqa: BLE001 - close must still run - failures.append( - "writer lease release failed: " - f"{type(exc).__name__}: {exc}" - ) - try: - ledger.close() - except Exception as exc: # noqa: BLE001 - store close must still run - failures.append(f"ledger close failed: {type(exc).__name__}: {exc}") - if store is not None: - try: - store.close() - except Exception as exc: # noqa: BLE001 - report after all cleanup - failures.append( - f"audit artifact store close failed: {type(exc).__name__}: {exc}" - ) - return failures - - -def main(argv: list[str] | None = None) -> int: - args = build_parser().parse_args(argv) - if args.limit is not None and args.limit <= 0: - raise SystemExit("--limit must be positive") - if args.batch_size <= 0: - raise SystemExit("--batch-size must be positive") - try: - start_after = ( - None - if args.start_after is None - else canonical_hex(args.start_after, name="start_after", expected_bytes=32) - ) - except ValueError as exc: - raise SystemExit(f"--start-after is invalid: {exc}") from exc - - store: AuditArtifactStore | None = None - ledger: PsqlShareLedger | None = None - summary: dict[str, Any] | None = None - primary_error: BaseException | None = None - cleanup_failures: list[str] = [] - try: - store = build_audit_store_from_env(read_only=args.dry_run) - # Bind the canonical artifact API before opening the database: a - # missing phase-2 capability fails without first taking a writer lease. - artifacts = CanonicalArtifactAdapter(store) - try: - ledger = ledger_from_args(args, store) - except Exception as exc: - if args.dry_run: - raise - raise SystemExit( - "publish-mode canonical audit-bundle backfill could not acquire " - "exclusive writer-lease ownership; stop the coordinator before " - f"publishing and retry: {exc}" - ) from exc - backfill = CanonicalBundleBackfill( - artifacts=artifacts, - ledger=AuditBundleLedgerAdapter(ledger), - dry_run=args.dry_run, - batch_size=args.batch_size, - limit=args.limit, - start_after=start_after, - ) - summary = backfill.run() - except BaseException as exc: - primary_error = exc - finally: - cleanup_failures = _cleanup_backfill_resources( - ledger=ledger, - store=store, - publish_mode=not args.dry_run, - ) - if primary_error is not None: - for failure in cleanup_failures: - primary_error.add_note(f"backfill cleanup: {failure}") - raise primary_error - if cleanup_failures: - raise RuntimeError("; ".join(cleanup_failures)) - assert summary is not None - print(json.dumps(summary, indent=2, sort_keys=True)) - return int(summary["exit_code"]) - - -if __name__ == "__main__": - raise SystemExit(main()) diff --git a/lab/prism/backfill_ctv_fanouts.py b/lab/prism/backfill_ctv_fanouts.py deleted file mode 100644 index 31433acfe..000000000 --- a/lab/prism/backfill_ctv_fanouts.py +++ /dev/null @@ -1,387 +0,0 @@ -#!/usr/bin/env python3 -"""Backfill PRISM CTV fanout artifact rows from audit artifacts.""" - -from __future__ import annotations - -import argparse -import json -import os -import re -import shlex -import sys -from dataclasses import dataclass -from pathlib import Path -from typing import Any - -if not __package__: - sys.path.insert(0, str(Path(__file__).resolve().parents[2])) - -from lab.prism.coordinator_config import ( - DEFAULT_LEASE_ACQUIRE_ATTEMPTS, - DEFAULT_LEASE_ACQUIRE_LOCK_TIMEOUT_SECONDS, - DEFAULT_POSTGRES_IDLE_IN_TRANSACTION_TIMEOUT_SECONDS, - DEFAULT_POSTGRES_TCP_KEEPALIVES_COUNT, - DEFAULT_POSTGRES_TCP_KEEPALIVES_IDLE_SECONDS, - DEFAULT_POSTGRES_TCP_KEEPALIVES_INTERVAL_SECONDS, - env_positive_float, - env_positive_int, -) -from lab.prism.share_ledger import ( - AUDIT_BODY_REF_SCHEMA, - AUDIT_BUNDLE_V2_SCHEMA, - PsqlShareLedger, - canonical_hex, - require_mapping, - sha256_json_hex, -) - - -HEX64_RE = re.compile(r"(?<![0-9a-fA-F])([0-9a-fA-F]{64})(?![0-9a-fA-F])") - - -@dataclass(frozen=True) -class CtvFanoutBackfillInput: - source: str - block_hash: str - manifest_set: dict[str, Any] - manifest_set_sha256: str - - -def load_json_file(path: Path) -> Any: - try: - return json.loads(path.read_text(encoding="utf-8")) - except json.JSONDecodeError as exc: - raise SystemExit(f"{path}: invalid JSON: {exc}") from exc - - -def resolve_local_audit_payload(payload: dict[str, Any], *, source_path: Path) -> dict[str, Any]: - schema = payload.get("schema") - if schema == "qbit.prism.live-audit-bundle-envelope.v1": - body_uri = payload.get("body_uri") - if not body_uri: - raise SystemExit(f"{source_path}: audit envelope has no body_uri") - body_path = Path(str(body_uri)).expanduser() - if not body_path.is_absolute(): - body_path = source_path.parent / body_path - body_payload = load_json_file(body_path) - if not isinstance(body_payload, dict): - raise SystemExit(f"{body_path}: top-level JSON must be an object") - return resolve_local_audit_payload(body_payload, source_path=body_path) - if schema == AUDIT_BODY_REF_SCHEMA: - bundle_without_shares = payload.get("bundle_without_shares") - if not isinstance(bundle_without_shares, dict): - raise SystemExit(f"{source_path}: compact audit body has no bundle_without_shares") - return bundle_without_shares - if schema == AUDIT_BUNDLE_V2_SCHEMA: - bundle_without_shares = payload.get("bundle_without_shares") - if not isinstance(bundle_without_shares, dict): - raise SystemExit(f"{source_path}: audit-bundle.v2 has no bundle_without_shares") - return bundle_without_shares - return payload - - -def infer_block_hash_from_path(path: Path) -> str | None: - matches = HEX64_RE.findall(path.name) - if not matches: - return None - return canonical_hex(matches[-1], name="block_hash", expected_bytes=32) - - -def block_hash_from_payload(payload: dict[str, Any]) -> str | None: - raw = payload.get("block_hash") - if raw is None: - found_block = payload.get("found_block") - if isinstance(found_block, dict): - raw = found_block.get("block_hash") - if raw is None: - return None - return canonical_hex(str(raw), name="block_hash", expected_bytes=32) - - -def ctv_manifest_source(payload: dict[str, Any]) -> tuple[dict[str, Any], str | None]: - if isinstance(payload.get("audit_bundle"), dict): - payload = require_mapping(payload["audit_bundle"], "audit_bundle") - - if isinstance(payload.get("ctv_fanout_manifest_set"), dict): - return require_mapping(payload["ctv_fanout_manifest_set"], "ctv_fanout_manifest_set"), None - if isinstance(payload.get("manifest_set"), dict): - manifest_set_sha256 = payload.get("manifest_set_sha256") - return require_mapping(payload["manifest_set"], "manifest_set"), ( - None if manifest_set_sha256 is None else str(manifest_set_sha256) - ) - if isinstance(payload.get("manifests"), list): - return payload, None - raise ValueError("payload does not contain a CTV fanout manifest set") - - -def backfill_input_from_payload( - payload: dict[str, Any], - *, - source: str, - block_hash: str | None = None, - manifest_set_sha256: str | None = None, -) -> CtvFanoutBackfillInput: - payload_block_hash = block_hash_from_payload(payload) - resolved_block_hash = block_hash or payload_block_hash - if resolved_block_hash is None: - raise ValueError("block hash is required for CTV fanout backfill") - resolved_block_hash = canonical_hex(resolved_block_hash, name="block_hash", expected_bytes=32) - - manifest_set, payload_manifest_set_sha256 = ctv_manifest_source(payload) - resolved_manifest_set_sha256 = manifest_set_sha256 or payload_manifest_set_sha256 or sha256_json_hex(manifest_set) - resolved_manifest_set_sha256 = canonical_hex( - resolved_manifest_set_sha256, - name="manifest_set_sha256", - expected_bytes=32, - ) - return CtvFanoutBackfillInput( - source=source, - block_hash=resolved_block_hash, - manifest_set=manifest_set, - manifest_set_sha256=resolved_manifest_set_sha256, - ) - - -def backfill_input_from_path( - path: Path, - *, - block_hash: str | None = None, - manifest_set_sha256: str | None = None, -) -> CtvFanoutBackfillInput: - payload = load_json_file(path) - if not isinstance(payload, dict): - raise SystemExit(f"{path}: top-level JSON must be an object") - payload = resolve_local_audit_payload(payload, source_path=path) - inferred_hash = block_hash or block_hash_from_payload(payload) or infer_block_hash_from_path(path) - try: - return backfill_input_from_payload( - payload, - source=str(path), - block_hash=inferred_hash, - manifest_set_sha256=manifest_set_sha256, - ) - except ValueError as exc: - raise SystemExit(f"{path}: {exc}") from exc - - -def psql_command_from_env(explicit: str | None) -> str: - if explicit: - return explicit - psql_command = os.environ.get("PRISM_POSTGRES_PSQL_COMMAND", "") - if psql_command: - return psql_command - database_url = os.environ.get("PRISM_DATABASE_URL", "") - if database_url: - return f"psql {shlex.quote(database_url)}" - raise SystemExit("set PRISM_POSTGRES_PSQL_COMMAND or PRISM_DATABASE_URL, or pass --psql-command") - - -def db_block_hashes_for_height(ledger: PsqlShareLedger, height: int) -> list[str]: - if height < 0: - raise SystemExit("--db-block-height values must be non-negative") - rows = ledger._run_json( - f""" -SELECT COALESCE(json_agg(block_hash ORDER BY found_at DESC), '[]'::json) -FROM qbit_pool_blocks -WHERE block_height = {int(height)} - AND chain_state <> 'reversed'; -""" - ) - if not isinstance(rows, list): - raise RuntimeError("block-height lookup returned non-array JSON") - return [ - canonical_hex(str(block_hash), name="block_hash", expected_bytes=32) - for block_hash in rows - ] - - -def backfill_input_from_db(ledger: PsqlShareLedger, *, block_hash: str) -> CtvFanoutBackfillInput: - block_hash = canonical_hex(block_hash, name="block_hash", expected_bytes=32) - row = ledger.audit_bundle(block_hash=block_hash) - if row is None: - raise SystemExit(f"no audit bundle found for block {block_hash}") - audit_bundle = row.get("audit_bundle") - if not isinstance(audit_bundle, dict): - raise SystemExit(f"audit bundle body is unavailable for block {block_hash}") - try: - return backfill_input_from_payload( - audit_bundle, - source=f"db:{block_hash}", - block_hash=block_hash, - ) - except ValueError as exc: - raise SystemExit(f"db:{block_hash}: {exc}") from exc - - -def build_parser() -> argparse.ArgumentParser: - parser = argparse.ArgumentParser( - description="Backfill qbit_ctv_fanout_artifacts from PRISM CTV audit artifacts." - ) - parser.add_argument( - "paths", - nargs="*", - type=Path, - help="local audit bundle, audit API payload, or CTV manifest-set JSON files", - ) - parser.add_argument( - "--path-block-hash", - help="block hash for a single path when it cannot be inferred from JSON or filename", - ) - parser.add_argument( - "--path-manifest-set-sha256", - help="manifest-set sha256 for a single path; defaults to canonical JSON sha256", - ) - parser.add_argument("--db-block-hash", action="append", default=[], help="load an audit bundle from DB by block hash") - parser.add_argument( - "--db-block-height", - action="append", - type=int, - default=[], - help="load audit bundle(s) from DB by block height", - ) - parser.add_argument("--psql-command", help="psql command; defaults to PRISM_POSTGRES_PSQL_COMMAND/PRISM_DATABASE_URL") - parser.add_argument("--writer-id", default=os.environ.get("PRISM_LEDGER_WRITER_ID", "prism-ctv-fanout-backfill")) - parser.add_argument("--writer-epoch", type=int, default=int(os.environ.get("PRISM_LEDGER_WRITER_EPOCH", "1"))) - parser.add_argument("--writer-session-token", default=os.environ.get("PRISM_LEDGER_WRITER_SESSION_TOKEN")) - parser.add_argument( - "--lease-ttl-seconds", - type=float, - default=float(os.environ.get("PRISM_LEDGER_LEASE_TTL_SECONDS", "60")), - ) - parser.add_argument( - "--lease-acquire-lock-timeout-seconds", - type=float, - default=env_positive_float( - "PRISM_LEDGER_LEASE_ACQUIRE_LOCK_TIMEOUT_SECONDS", - DEFAULT_LEASE_ACQUIRE_LOCK_TIMEOUT_SECONDS, - ), - ) - parser.add_argument( - "--lease-acquire-attempts", - type=int, - default=env_positive_int( - "PRISM_LEDGER_LEASE_ACQUIRE_ATTEMPTS", - DEFAULT_LEASE_ACQUIRE_ATTEMPTS, - ), - ) - parser.add_argument( - "--postgres-idle-in-transaction-timeout-seconds", - type=float, - default=env_positive_float( - "PRISM_POSTGRES_IDLE_IN_TRANSACTION_TIMEOUT_SECONDS", - DEFAULT_POSTGRES_IDLE_IN_TRANSACTION_TIMEOUT_SECONDS, - ), - ) - parser.add_argument( - "--postgres-tcp-keepalives-idle-seconds", - type=int, - default=env_positive_int( - "PRISM_POSTGRES_TCP_KEEPALIVES_IDLE_SECONDS", - DEFAULT_POSTGRES_TCP_KEEPALIVES_IDLE_SECONDS, - ), - ) - parser.add_argument( - "--postgres-tcp-keepalives-interval-seconds", - type=int, - default=env_positive_int( - "PRISM_POSTGRES_TCP_KEEPALIVES_INTERVAL_SECONDS", - DEFAULT_POSTGRES_TCP_KEEPALIVES_INTERVAL_SECONDS, - ), - ) - parser.add_argument( - "--postgres-tcp-keepalives-count", - type=int, - default=env_positive_int( - "PRISM_POSTGRES_TCP_KEEPALIVES_COUNT", - DEFAULT_POSTGRES_TCP_KEEPALIVES_COUNT, - ), - ) - parser.add_argument( - "--no-init-schema", - action="store_true", - help="do not run the idempotent PRISM schema repair before backfilling", - ) - return parser - - -def ledger_from_args(args: argparse.Namespace) -> PsqlShareLedger: - return PsqlShareLedger( - psql_command=psql_command_from_env(args.psql_command), - writer_id=args.writer_id, - writer_epoch=args.writer_epoch, - writer_session_token=args.writer_session_token, - initialize_schema=not args.no_init_schema, - lease_ttl_seconds=args.lease_ttl_seconds, - lease_acquire_lock_timeout_seconds=args.lease_acquire_lock_timeout_seconds, - lease_acquire_attempts=args.lease_acquire_attempts, - postgres_idle_in_transaction_timeout_seconds=args.postgres_idle_in_transaction_timeout_seconds, - postgres_tcp_keepalives_idle_seconds=args.postgres_tcp_keepalives_idle_seconds, - postgres_tcp_keepalives_interval_seconds=args.postgres_tcp_keepalives_interval_seconds, - postgres_tcp_keepalives_count=args.postgres_tcp_keepalives_count, - ) - - -def main(argv: list[str] | None = None) -> int: - args = build_parser().parse_args(argv) - if len(args.paths) != 1 and (args.path_block_hash or args.path_manifest_set_sha256): - raise SystemExit("--path-block-hash and --path-manifest-set-sha256 require exactly one path") - - path_inputs = [ - backfill_input_from_path( - path, - block_hash=args.path_block_hash, - manifest_set_sha256=args.path_manifest_set_sha256, - ) - for path in args.paths - ] - db_hashes = [ - canonical_hex(block_hash, name="db_block_hash", expected_bytes=32) - for block_hash in args.db_block_hash - ] - if not path_inputs and not db_hashes and not args.db_block_height: - raise SystemExit("provide at least one path, --db-block-hash, or --db-block-height") - - ledger = ledger_from_args(args) - - for height in args.db_block_height: - height_hashes = db_block_hashes_for_height(ledger, height) - if not height_hashes: - raise SystemExit(f"no non-reversed PRISM blocks found at height {height}") - db_hashes.extend(height_hashes) - - inputs = path_inputs + [ - backfill_input_from_db(ledger, block_hash=block_hash) - for block_hash in dict.fromkeys(db_hashes) - ] - results = [] - for item in inputs: - result = ledger.persist_ctv_fanout_manifest_set( - block_hash=item.block_hash, - manifest_set=item.manifest_set, - manifest_set_sha256=item.manifest_set_sha256, - ) - results.append( - { - "source": item.source, - "block_hash": item.block_hash, - "manifest_set_sha256": item.manifest_set_sha256, - **result, - } - ) - - print( - json.dumps( - { - "schema": "qbit.prism.ctv-fanout-backfill.v1", - "input_count": len(inputs), - "results": results, - }, - indent=2, - sort_keys=True, - ) - ) - return 0 - - -if __name__ == "__main__": - raise SystemExit(main()) diff --git a/lab/prism/background_services.py b/lab/prism/background_services.py deleted file mode 100644 index e0bb9b621..000000000 --- a/lab/prism/background_services.py +++ /dev/null @@ -1,2285 +0,0 @@ -"""Named lifecycle registry for PRISM process-level background loops. - -The registry deliberately has no ``start_all`` operation. The coordinator -starts named services at the existing recovery boundaries, while this module -owns the start-once state and the exact thread handles used during shutdown. - -This module also owns :class:`LedgerLeaseHeartbeatService`, the sole owner of -the ledger writer-lease heartbeat/monitor threads, their freshness state, and -the synchronous exact-session external-effect fence. Because its startup is -conditional, creates two coupled threads, and waits for the first proof, it is -started directly at the serve boundary rather than modeled as two independent -registry specifications. - -:class:`WatchdogService` owns watchdog policy: the coordination-blocked -streak, coordination-versus-publication classification, the liveness and -fatal-stop decisions, failure detail/diagnostics, and the bounded fresh-thread -shutdown/lease-release attempt followed by an unconditional process exit. -Generic liveness heartbeat registration/record/pause maps stay coordinator -runtime state and reach the watchdog through ports. Both WatchdogService.run -and the lease service trigger the dynamic coordinator ``_watchdog_hard_exit`` -seam so per-instance monkeypatches intercept every exit path; the coordinator -delegate routes back into :meth:`WatchdogService.hard_exit`. -""" - -from __future__ import annotations - -from dataclasses import dataclass -import gc -import inspect -import os -import queue -import sys -import threading -import time -from typing import Any, Callable, Iterable - -from lab.prism.coordinator_shutdown import ShutdownInProgress -from lab.prism.process_telemetry import PRISM_GC_GENERATIONS -from lab.prism.share_ledger import ( - DEFAULT_WRITER_LEASE_ADOPTION_SILENCE_SECONDS, - WRITER_LEASE_VERIFICATION_MAX_STATEMENTS, - WriterLeaseRenewalDeferred, -) -from lab.prism.writer_lease_timing import ( - WRITER_LEASE_HEARTBEAT_SCHEDULER_SLACK_SECONDS, - LEASE_HEARTBEAT_MODE_FENCE, - LEASE_HEARTBEAT_MODE_PROOF, - LEASE_HEARTBEAT_MODE_RENEW, - LEASE_HEARTBEAT_MODES, - LEASE_HEARTBEAT_OUTCOME_DEFERRED, - LEASE_HEARTBEAT_OUTCOME_FAILED, - LEASE_HEARTBEAT_OUTCOME_PROVEN, - LEASE_HEARTBEAT_OUTCOME_RENEWAL_DUE, - LEASE_HEARTBEAT_OUTCOME_RENEWED, - LEASE_HEARTBEAT_OUTCOMES, - LEASE_MONITOR_LATE_WAKE_SLACK_FRACTIONS, - LEASE_MONITOR_STALL_PROBE_MAX_SAMPLES_PER_WINDOW, - LEASE_MONITOR_STALL_PROBE_TRIGGER_SLACK_FRACTION, - LEASE_MONITOR_STALL_PROBE_WINDOW_SECONDS, - LEASE_MONITOR_WAKE_DELAY_BUCKETS, - LEASE_MONITOR_WAKE_DELAY_WINDOW_SECONDS, - LEASE_MONITOR_WAKE_DELAY_WINDOW_SLICES, - UNATTRIBUTED_PHASES, - WriterLeaseHeartbeatPolicy, - WriterLeaseVerificationAttempt, - WriterLeaseVerificationPhases, -) - -# GC pause histogram buckets (issue #227). A closed set, like every other -# bucket vocabulary on /metrics. Distinct from the WP1a gc families in -# lab.prism.process_telemetry, which export gc.get_stats() counts (passes, -# objects collected, uncollectable) and say nothing about how long a pass -# held the interpreter; these are the pause durations, measured from the -# collector's own start/stop callbacks. -PRISM_GC_PAUSE_SECONDS_BUCKETS = ( - 0.001, - 0.0025, - 0.005, - 0.01, - 0.025, - 0.05, - 0.1, - 0.25, - 0.5, - 1.0, - 2.5, -) - -# Bounds on one stall-probe stack sample, so a sample of a 64-thread -# process stays a few kilobytes: the innermost frames of each thread, and -# at most this many threads. -LEASE_MONITOR_STALL_PROBE_MAX_FRAMES_PER_THREAD = 12 -LEASE_MONITOR_STALL_PROBE_MAX_THREADS = 64 - - -@dataclass(frozen=True, slots=True) -class BackgroundServiceSpec: - """Immutable construction and shutdown policy for one background loop.""" - - name: str - thread_name: str - target: Callable[[], None] - daemon: bool - join_timeout: float - watchdog_monitored: bool - registration_identity: object | None = None - - def __post_init__(self) -> None: - if not self.name: - raise ValueError("background service name must not be empty") - if not self.thread_name: - raise ValueError("background service thread name must not be empty") - if self.join_timeout < 0: - raise ValueError("background service join timeout must be nonnegative") - - -@dataclass(frozen=True, slots=True) -class BackgroundServiceSnapshot: - """Read-only lifecycle state returned without exposing registry records.""" - - specification: BackgroundServiceSpec - started: bool - thread: threading.Thread | None - - -@dataclass(slots=True) -class _BackgroundServiceRecord: - specification: BackgroundServiceSpec - started: bool = False - thread: threading.Thread | None = None - start_hook_completed: bool = False - - -ThreadFactory = Callable[..., threading.Thread] - - -class BackgroundServiceRegistry: - """Start named process services once and retain their drain handles.""" - - def __init__( - self, - specifications: Iterable[BackgroundServiceSpec] = (), - *, - thread_factory: ThreadFactory = threading.Thread, - ) -> None: - self._lock = threading.Lock() - self._thread_factory = thread_factory - self._records: dict[str, _BackgroundServiceRecord] = {} - self._thread_names: set[str] = set() - for specification in specifications: - self.register(specification) - - def register(self, specification: BackgroundServiceSpec) -> None: - """Register one service without starting it.""" - with self._lock: - self._register_locked(specification) - - def register_if_absent(self, specification: BackgroundServiceSpec) -> bool: - """Atomically install an equivalent dynamic service at most once. - - Returns true when this call registered the service. Concurrent callers - describing the same lifecycle and registration identity receive false; - a conflicting reuse of either name still fails explicitly. - """ - with self._lock: - existing = self._records.get(specification.name) - if existing is not None: - if self._equivalent(existing.specification, specification): - return False - raise ValueError( - "incompatible background service registration for name: " - f"{specification.name}" - ) - self._register_locked(specification) - return True - - def _register_locked(self, specification: BackgroundServiceSpec) -> None: - if specification.name in self._records: - raise ValueError( - f"background service is already registered: {specification.name}" - ) - if specification.thread_name in self._thread_names: - raise ValueError( - "background service thread name is already registered: " - f"{specification.thread_name}" - ) - self._records[specification.name] = _BackgroundServiceRecord( - specification=specification - ) - self._thread_names.add(specification.thread_name) - - @staticmethod - def _equivalent( - existing: BackgroundServiceSpec, - candidate: BackgroundServiceSpec, - ) -> bool: - target_equivalent = ( - existing.target is candidate.target - if existing.registration_identity is None - and candidate.registration_identity is None - else existing.registration_identity == candidate.registration_identity - and existing.registration_identity is not None - and candidate.registration_identity is not None - ) - return bool( - target_equivalent - and existing.name == candidate.name - and existing.thread_name == candidate.thread_name - and existing.daemon == candidate.daemon - and existing.join_timeout == candidate.join_timeout - and existing.watchdog_monitored == candidate.watchdog_monitored - ) - - def contains(self, name: str) -> bool: - with self._lock: - return name in self._records - - def service_names(self) -> tuple[str, ...]: - with self._lock: - return tuple(self._records) - - def snapshot(self, name: str) -> BackgroundServiceSnapshot: - with self._lock: - record = self._records[name] - return BackgroundServiceSnapshot( - specification=record.specification, - started=record.started, - thread=record.thread, - ) - - def start( - self, - name: str, - *, - on_started: Callable[[BackgroundServiceSpec], None] | None = None, - ) -> threading.Thread: - """Start a named service once, returning the same thread thereafter.""" - with self._lock: - record = self._records[name] - if record.thread is not None: - if not record.start_hook_completed and on_started is not None: - on_started(record.specification) - record.start_hook_completed = True - return record.thread - specification = record.specification - thread = self._thread_factory( - target=specification.target, - name=specification.thread_name, - daemon=specification.daemon, - ) - thread.start() - record.thread = thread - record.started = True - if on_started is None: - record.start_hook_completed = True - else: - # The live thread remains registered for shutdown if this - # nonblocking side-effect fails. A later start call retries - # only the hook and never creates a second worker. - on_started(specification) - record.start_hook_completed = True - return thread - - def threads_to_drain(self) -> tuple[tuple[threading.Thread, float], ...]: - """Return started threads in stable registration/shutdown order.""" - with self._lock: - return tuple( - (record.thread, record.specification.join_timeout) - for record in self._records.values() - if record.started and record.thread is not None - ) - - def watchdog_service_names(self, *, started_only: bool = False) -> tuple[str, ...]: - """Derive watchdog keys from the same records used to start loops.""" - with self._lock: - return tuple( - record.specification.name - for record in self._records.values() - if record.specification.watchdog_monitored - and (record.started or not started_only) - ) - - -@dataclass(frozen=True, slots=True) -class WatchdogPorts: - """Dynamic process-supervision capabilities used by the watchdog owner. - - ``publication_state``, ``hard_exit``, ``overdue_heartbeats``, and - ``publication_failure_expired`` resolve coordinator methods at call time - so per-instance monkeypatches intercept them; the interval/budget/enabled - reads stay live per iteration for focused timing overrides. - """ - - wait_for_stop: Callable[[float], bool] - interval_seconds: Callable[[], float] - fatal_exit_requested: Callable[[], bool] - publication_state: Callable[[float], tuple[str | None, float, float, float]] - hard_exit: Callable[[str], None] - liveness_enabled: Callable[[], bool] - overdue_heartbeats: Callable[[float], list[str]] - liveness_timeout_seconds: Callable[[], float] - coordination_budget_seconds: Callable[[], float] - publication_budget_seconds: Callable[[], float] - ensure_job_cache_state: Callable[[], None] - publication_failure_expired: Callable[[float], bool] - publication_divergence_since: Callable[[], float | None] - lease_release_timeout_seconds: Callable[[], float] - shutdown_controller: Callable[[], Any] - request_shutdown: Callable[[], None] - release_ledger_lease: Callable[[float], bool] - lease_failure_reason: Callable[[], str | None] - exit_process: Callable[[int], None] - log: Callable[[str], None] - - -class WatchdogService: - """Own watchdog classification, streak state, and bounded emergency exit. - - Progress-health continues to own the publication-divergence timestamps; - the classification reads a locked snapshot through a port and rechecks it - while claiming an exit. Lock order is the service's coordination lock, - then the progress-health lock — progress-health must never call the - watchdog while holding its own lock. - """ - - def __init__(self, ports: WatchdogPorts) -> None: - self._ports = ports - self._coordination_blocked_lock = threading.Lock() - self._coordination_blocked_since_monotonic: float | None = None - self._publication_watchdog_exit_claimed = False - self.failure_detail: str | None = None - - def record_coordination_blocked_refresh(self, now: float) -> None: - with self._coordination_blocked_lock: - if self._publication_watchdog_exit_claimed: - return - if self._coordination_blocked_since_monotonic is None: - self._coordination_blocked_since_monotonic = now - - def clear_coordination_blocked_streak(self) -> None: - with self._coordination_blocked_lock: - # Once the watchdog claims an expired generation, its hard exit is - # the terminal action. Otherwise a refresh completing between the - # deadline check and os._exit could appear to cancel an exit that - # already won the streak lock. - if not self._publication_watchdog_exit_claimed: - self._coordination_blocked_since_monotonic = None - - def coordination_blocked_streak_age_seconds( - self, - now: float | None = None, - ) -> float: - now = time.monotonic() if now is None else now - with self._coordination_blocked_lock: - started = self._coordination_blocked_since_monotonic - return 0.0 if started is None else max(0.0, now - started) - - def coordination_blocked_streak_expired(self, now: float) -> bool: - budget = float(self._ports.coordination_budget_seconds()) - with self._coordination_blocked_lock: - started = self._coordination_blocked_since_monotonic - return bool( - started is not None - and (budget <= 0 or now - started >= budget) - ) - - def publication_watchdog_state( - self, - now: float, - ) -> tuple[str | None, float, float, float]: - """Arbitrate coordination and ordinary publication deadlines. - - The final decision is serialized with coordination streak changes. - A streak recorded while the ordinary publication check is in flight - therefore owns the longer coordination deadline; once either deadline - is claimed, later refresh results cannot cancel the terminal exit. - """ - coordination_budget = float(self._ports.coordination_budget_seconds()) - publication_budget = float(self._ports.publication_budget_seconds()) - self._ports.ensure_job_cache_state() - - # This preflight keeps the common healthy case cheap. Its result is - # revalidated under both state locks before an ordinary exit is - # claimed, so a concurrent publication cannot leave a stale verdict. - publication_expired = self._ports.publication_failure_expired(now) - with self._coordination_blocked_lock: - started = self._coordination_blocked_since_monotonic - if started is not None: - age = max(0.0, now - started) - if coordination_budget <= 0 or age >= coordination_budget: - self._publication_watchdog_exit_claimed = True - return ( - "coordination", - age, - coordination_budget, - publication_budget, - ) - return None, age, coordination_budget, publication_budget - - if publication_expired: - # Recheck the divergence timestamp under the progress-health - # lock (inside the port) while the coordination lock is held. - divergence_since = self._ports.publication_divergence_since() - publication_expired = bool( - publication_budget > 0 - and divergence_since is not None - and now - divergence_since >= publication_budget - ) - if publication_expired: - self._publication_watchdog_exit_claimed = True - return ( - "publication", - 0.0, - coordination_budget, - publication_budget, - ) - return None, 0.0, coordination_budget, publication_budget - - def run(self) -> None: - while True: - if self._ports.wait_for_stop(self._ports.interval_seconds()): - if self._ports.fatal_exit_requested(): - self._ports.log( - "prism coordinator: fatal block-work restart requested; " - "exiting non-zero even if the main thread is blocked" - ) - self._ports.exit_process(1) - return - now = time.monotonic() - ( - publication_failure, - coordination_age, - coordination_budget, - publication_budget, - ) = self._ports.publication_state(now) - if publication_failure == "coordination": - self.failure_detail = ( - "prism coordinator: publication-progress watchdog firing; " - "template refresh remained coordination-blocked past the " - f"coordination budget={coordination_budget:g}s " - f"streak_age={coordination_age:.3f}s" - ) - self._ports.hard_exit("coordination") - if publication_failure == "publication": - self.failure_detail = ( - "prism coordinator: publication-progress watchdog firing; " - "current tip/generation remained unpublished past the " - f"template refresh failure budget={publication_budget:g}s" - ) - self._ports.hard_exit("publication") - overdue = ( - self._ports.overdue_heartbeats(now) - if self._ports.liveness_enabled() - else [] - ) - if overdue: - self.failure_detail = ( - "prism coordinator: liveness watchdog firing; unresponsive " - f"subsystems={overdue} " - f"timeout={self._ports.liveness_timeout_seconds():g}s" - ) - # Queued shares have not been acknowledged. Miners reconnect - # and retry them after restart; exact-payload replay is - # idempotent if Postgres committed just before this exit. - self._ports.hard_exit("liveness") - - def hard_exit( - self, - reason: str, - *, - timeout_seconds: float | None = None, - ) -> None: - """Bound a fresh-thread lease release, then terminate unconditionally.""" - try: - if timeout_seconds is None: - timeout_seconds = float( - self._ports.lease_release_timeout_seconds() - ) - deadline = time.monotonic() + max(0.0, timeout_seconds) - release_thread = threading.Thread( - target=self._release_ledger_lease, - args=(reason, deadline), - name="prism-watchdog-lease-release", - daemon=True, - ) - release_thread.start() - release_thread.join(max(0.0, deadline - time.monotonic())) - finally: - # Nothing, including timeout logging or thread-start failure, may - # extend or suppress the watchdog's terminal action. Exiting also - # discards the in-memory vardiff retention store (see - # vardiff_service.SessionDifficultyStore), so the post-restart - # reconnect wave is not smoothed by difficulty resume. - self._ports.exit_process(1) - - def _release_ledger_lease(self, reason: str, deadline: float) -> bool: - """Use only the shutdown controller and a fresh DB connection. - - Do not call ``shutdown`` here: cancellation takes the coordinator - control-plane lock, which may belong to the subsystem that triggered - the watchdog. Closing writer admission plus the controller's - tracked-writer barrier retains the graceful path's release-withheld - invariant without that lock. - - Any best-effort diagnostic is emitted only after lease handling on this - daemon worker. A blocked container log pipe may park the worker, but it - cannot precede the release attempt or extend the caller's hard deadline. - """ - controller = self._ports.shutdown_controller() - self._ports.request_shutdown() - if not controller.begin_shutdown(f"watchdog_{reason}"): - handled = controller.wait_for_lease_handling() - self._exit_diagnostic(reason, lease_handled=handled) - return handled - - quiesced, _elapsed, _blockers = controller.wait_for_writer_quiescence( - max(0.0, deadline - time.monotonic()) - ) - if not quiesced: - self._exit_diagnostic(reason, lease_handled=False) - return False - if time.monotonic() >= deadline: - self._exit_diagnostic(reason, lease_handled=False) - return False - handled = self._ports.release_ledger_lease(deadline) - self._exit_diagnostic(reason, lease_handled=handled) - return handled - - def _exit_diagnostic(self, reason: str, *, lease_handled: bool) -> None: - """Best-effort logging after the watchdog's safety-critical work.""" - detail = ( - self._ports.lease_failure_reason() - if reason == "lease_heartbeat" - else self.failure_detail - ) - try: - self._ports.log( - (detail or f"prism coordinator: {reason} watchdog firing") - + ". Exiting non-zero so the restart policy recovers the process. " - + f"lease_handled={lease_handled}" - ) - except Exception: - pass - - -def guard_session_verifier(ledger: object) -> Callable[..., object] | None: - """Return the guarded-session liveness check for ``ledger``. - - Prefer ``verify_writer_lease_guard_session``: it proves the dedicated - guard session is live and renews the lease TTL only via SKIP LOCKED, - so it can never queue behind the row lock a long fenced transaction - (``persist_accepted_block``) holds until commit and then die on the - guard's statement timeout, yet an idle coordinator still keeps its - lease from expiring under different-identity claimants. - ``renew_writer_lease_heartbeat`` is the legacy embedder spelling and - does wait on the lease tuple; it is only a fallback for ledgers that - predate the non-blocking verification. - """ - verify = getattr(ledger, "verify_writer_lease_guard_session", None) - if verify is not None: - return verify - return getattr(ledger, "renew_writer_lease_heartbeat", None) - - -def guard_session_prover(ledger: object) -> Callable[..., object] | None: - """Return the cheap read-only ownership proof for ``ledger``, if any. - - ``prove_writer_lease_guard_session`` answers only "does this backend - still hold the advisory guard, and does the committed lease row still - name this exact session?" on one non-blocking statement. It is the - frequent half of the issue #212 split: the heartbeat needs ownership - several times per adoption-silence window, but the lease TTL only needs - a writer-side refresh well before it lapses, and paying for renewal - (a ``SKIP LOCKED`` row lock, a ``pg_stat_activity`` locker attribution, - and sometimes a second statement) on every beat is what pushed the - frequent statement toward the guard's 500ms statement timeout during a - rapid-block burst. - - A ledger that does not offer it — an embedder's, or a test fake — - simply keeps running the full verification on every beat, which is the - behavior that predates the split. - """ - return getattr(ledger, "prove_writer_lease_guard_session", None) - - -def verifier_callbacks(verify: Callable[..., object]) -> frozenset[str]: - """Which progress callbacks ``verify`` accepts. - - Resolved once per loop rather than per beat: a ledger's verification - signature cannot change under a running heartbeat, and - ``inspect.signature`` is far too expensive for a 0.25s cadence. - """ - try: - parameters = inspect.signature(verify).parameters - except (TypeError, ValueError): - return frozenset() - return frozenset( - name - for name in ( - "on_query_start", - "on_statement_progress", - "on_statement_end", - "on_statement_server_seconds", - ) - if name in parameters - ) - - -def _print_line(line: str) -> None: - """Default log port for the monitor's diagnostics: a flushed print. - - Only ever invoked from the monitor's diagnostics thread - (:meth:`LedgerLeaseHeartbeatService._run_diagnostics`), never from the - monitor thread itself. The monitor is the thread that must notice a - dead guard session and hard-exit before a successor adopts, so a full - container log pipe stalling this flush must stall a throwaway thread, - not the thread that decides the exit. - """ - try: - print(line, flush=True) - except Exception: - pass - - -class _WindowRateLimiter: - """At most ``max_events`` admissions per ``window_seconds``, then refuse. - - A fixed window keyed on the first admission, not a token bucket: the - property the stall probe needs is a hard ceiling on the number of - samples any interval can produce, so that a stall can never be turned - into a stall storm by the instrument that reports it. - """ - - __slots__ = ("max_events", "window_seconds", "_window_started", "_admitted") - - def __init__(self, *, max_events: int, window_seconds: float) -> None: - self.max_events = int(max_events) - self.window_seconds = float(window_seconds) - self._window_started: float | None = None - self._admitted = 0 - - def admit(self, now: float) -> bool: - if ( - self._window_started is None - or now - self._window_started >= self.window_seconds - ): - self._window_started = now - self._admitted = 0 - if self._admitted >= self.max_events: - return False - self._admitted += 1 - return True - - -class MonitorWakeTelemetry: - """The heartbeat monitor's own lateness, as signals that can re-arm. - - Issue #227: the only export of the monitor's lateness was a lifetime - high-water mark. Every stall smaller than the record was invisible, the - alert built on it could fire once per process lifetime, and a hard-exit - message quoted a possibly hours-old worst case. This keeps the lifetime - maximum for dashboard continuity and adds the signals an alert can - actually be expressed on: - - * a fixed-bucket histogram of every wake's lateness; - * counters of wakes at or above 0.5x, 0.8x and 1.0x the policy's - scheduler slack (closed label set); - * a rolling-window maximum that falls back once a stall ages out; - * the age of the current lifetime record. - - Written only by the monitor thread; read by the metrics thread. No - lock: every field is a Python int or float, so a scrape sees either the - value before or after one ``observe``, never a torn one. - """ - - __slots__ = ( - "_buckets", - "_bucket_counts", - "_count", - "_sum", - "lifetime_max_seconds", - "_record_monotonic", - "_late", - "_slice_seconds", - "_slice_index", - "_slice_max", - ) - - def __init__( - self, - *, - buckets: tuple[float, ...] = LEASE_MONITOR_WAKE_DELAY_BUCKETS, - window_seconds: float = LEASE_MONITOR_WAKE_DELAY_WINDOW_SECONDS, - window_slices: int = LEASE_MONITOR_WAKE_DELAY_WINDOW_SLICES, - ) -> None: - self._buckets = tuple(float(bucket) for bucket in buckets) - # Cumulative, in Prometheus's own convention: bucket i counts every - # observation <= its upper bound. - self._bucket_counts = [0 for _ in self._buckets] - self._count = 0 - self._sum = 0.0 - self.lifetime_max_seconds = 0.0 - self._record_monotonic: float | None = None - self._late = { - fraction: 0 for fraction in LEASE_MONITOR_LATE_WAKE_SLACK_FRACTIONS - } - slices = max(1, int(window_slices)) - self._slice_seconds = float(window_seconds) / slices - # A ring of per-slice maxima keyed by absolute slice number, so an - # old slice is recognised as stale rather than read as current. - self._slice_index = [-1 for _ in range(slices)] - self._slice_max = [0.0 for _ in range(slices)] - - @property - def buckets(self) -> tuple[float, ...]: - return self._buckets - - def observe( - self, - wake_delay_seconds: float, - *, - slack_seconds: float, - now: float, - ) -> None: - delay = max(0.0, float(wake_delay_seconds)) - self._count += 1 - self._sum += delay - for index, bound in enumerate(self._buckets): - if delay <= bound: - self._bucket_counts[index] += 1 - for fraction in self._late: - if delay >= float(fraction) * float(slack_seconds): - self._late[fraction] += 1 - if delay > self.lifetime_max_seconds: - self.lifetime_max_seconds = delay - self._record_monotonic = now - absolute = int(now // self._slice_seconds) - slot = absolute % len(self._slice_max) - if self._slice_index[slot] != absolute: - self._slice_index[slot] = absolute - self._slice_max[slot] = delay - elif delay > self._slice_max[slot]: - self._slice_max[slot] = delay - - def window_max_seconds(self, now: float) -> float: - """Largest lateness observed inside the trailing window.""" - oldest = int(now // self._slice_seconds) - len(self._slice_max) + 1 - worst = 0.0 - for index, value in zip(self._slice_index, self._slice_max): - if index >= oldest and value > worst: - worst = value - return worst - - def record_age_seconds(self, now: float) -> float: - """Seconds since the lifetime maximum was last raised (0 before any).""" - if self._record_monotonic is None: - return 0.0 - return max(0.0, now - self._record_monotonic) - - def snapshot(self, now: float) -> dict[str, object]: - """Fixed-key view for the metrics surface.""" - return { - "buckets": { - bound: int(count) - for bound, count in zip(self._buckets, self._bucket_counts) - }, - "count": int(self._count), - "sum": float(self._sum), - "late_wakes": dict(self._late), - "lifetime_max_seconds": float(self.lifetime_max_seconds), - "window_max_seconds": self.window_max_seconds(now), - "record_age_seconds": self.record_age_seconds(now), - } - - -class GcPauseTelemetry: - """Cyclic-collector pause durations per generation, from ``gc.callbacks``. - - Issue #227's stall attribution needs to know whether the interpreter - was inside a collection when the monitor could not run. The WP1a - families (``qbit_prism_process_gc_*`` in ``process_telemetry``) export - ``gc.get_stats()`` — how many passes ran and what they freed — and - cannot say how long any pass held the interpreter. This is the missing - duration: the collector calls back at the start and stop of every - pass with the generation, and the difference is the pause. - - The callback must never raise (an exception in a GC callback is - reported as unraisable and can hide the pass) and must never take a - lock: a collection can start on any thread, including one that holds - a lock the callback would need. Every field is a plain int or float, - so the metrics thread reads them lock-free like the rest of the - heartbeat's counters. - - ``generation`` is CPython's closed three-value set. Anything else - (there is nothing else on CPython) is ignored rather than allowed to - grow the label set. - """ - - def __init__( - self, - *, - clock: Callable[[], float] = time.perf_counter, - buckets: tuple[float, ...] = PRISM_GC_PAUSE_SECONDS_BUCKETS, - ) -> None: - self._clock = clock - self._buckets = tuple(float(bucket) for bucket in buckets) - self._bucket_counts = { - generation: [0 for _ in self._buckets] - for generation in PRISM_GC_GENERATIONS - } - self._count = {generation: 0 for generation in PRISM_GC_GENERATIONS} - self._sum = {generation: 0.0 for generation in PRISM_GC_GENERATIONS} - self._last = {generation: 0.0 for generation in PRISM_GC_GENERATIONS} - self._max = {generation: 0.0 for generation in PRISM_GC_GENERATIONS} - self._started_at: float | None = None - self._started_generation: str | None = None - self._installed = False - - @property - def buckets(self) -> tuple[float, ...]: - return self._buckets - - @property - def installed(self) -> bool: - return self._installed - - def install(self) -> None: - """Register on ``gc.callbacks`` once; idempotent.""" - if self._installed: - return - gc.callbacks.append(self._callback) - self._installed = True - - def uninstall(self) -> None: - if not self._installed: - return - self._installed = False - try: - gc.callbacks.remove(self._callback) - except ValueError: - pass - - def _callback(self, phase: str, info: dict[str, Any]) -> None: - try: - if phase == "start": - self._started_at = self._clock() - self._started_generation = str(info.get("generation", "")) - elif phase == "stop": - started = self._started_at - if started is None: - return - self._started_at = None - generation = str( - info.get("generation", self._started_generation or "") - ) - self.record(generation, self._clock() - started) - except Exception: - # Never let telemetry disturb a collection. - self._started_at = None - - def record(self, generation: str, pause_seconds: float) -> None: - """Account one pause to ``generation``; unknown generations are dropped. - - Ordering matters because nothing locks against the scrape thread: - the count is bumped *before* the buckets here and read *after* - them in :meth:`snapshot`, so whichever way a scrape interleaves it - sees every ``le`` bucket at or below ``+Inf``. The other order - renders a bucket larger than the count, which Prometheus ingests - as a malformed histogram (the same discipline as - :meth:`MonitorWakeTelemetry.observe`). - """ - if generation not in self._count: - return - pause = max(0.0, float(pause_seconds)) - self._count[generation] += 1 - self._sum[generation] += pause - counts = self._bucket_counts[generation] - for index, bound in enumerate(self._buckets): - if pause <= bound: - counts[index] += 1 - self._last[generation] = pause - if pause > self._max[generation]: - self._max[generation] = pause - - def worst(self) -> tuple[str | None, float]: - """The longest pause seen and its generation (``None`` before any).""" - generation: str | None = None - worst = 0.0 - for candidate in PRISM_GC_GENERATIONS: - if self._count[candidate] and ( - generation is None or self._max[candidate] > worst - ): - generation = candidate - worst = self._max[candidate] - return generation, worst - - def snapshot(self) -> dict[str, dict[str, object]]: - """Fixed-key view: one entry per generation, each with fixed keys.""" - return { - generation: { - "buckets": { - bound: int(count) - for bound, count in zip( - self._buckets, self._bucket_counts[generation] - ) - }, - "count": int(self._count[generation]), - "sum": float(self._sum[generation]), - "last_seconds": float(self._last[generation]), - "max_seconds": float(self._max[generation]), - } - for generation in PRISM_GC_GENERATIONS - } - - -class StallProbe: - """Rate-limited stack sample of the running threads during a stall. - - When the monitor wakes at least half a scheduler slack late, the - question is *what held the interpreter*. ``sys._current_frames()`` - answers it at the only moment it can be asked — the instant the - monitor gets the GIL back — at a cost of one frame walk per thread. - - The rate limit is load-bearing, not decorative. A stall storm is by - definition many late wakes in a row, and an instrument that paid a - frame walk plus a log line for each of them would deepen the storm it - is reporting. ``LEASE_MONITOR_STALL_PROBE_MAX_SAMPLES_PER_WINDOW`` - samples per ``LEASE_MONITOR_STALL_PROBE_WINDOW_SECONDS`` is the ceiling; - every trigger beyond it is counted as suppressed and costs nothing. - """ - - def __init__( - self, - *, - max_samples_per_window: int = LEASE_MONITOR_STALL_PROBE_MAX_SAMPLES_PER_WINDOW, - window_seconds: float = LEASE_MONITOR_STALL_PROBE_WINDOW_SECONDS, - max_threads: int = LEASE_MONITOR_STALL_PROBE_MAX_THREADS, - max_frames_per_thread: int = LEASE_MONITOR_STALL_PROBE_MAX_FRAMES_PER_THREAD, - ) -> None: - self._limiter = _WindowRateLimiter( - max_events=max_samples_per_window, - window_seconds=window_seconds, - ) - self._max_threads = int(max_threads) - self._max_frames = int(max_frames_per_thread) - self.samples_total = 0 - self.suppressed_total = 0 - self.last_sample: dict[str, object] | None = None - - @property - def max_samples_per_window(self) -> int: - return self._limiter.max_events - - @property - def window_seconds(self) -> float: - return self._limiter.window_seconds - - def admit(self, now: float) -> bool: - """O(1): may a sample be taken now? Counts the refusals. - - This is the only part of the probe the monitor thread runs; the - frame walk itself is deferred to the diagnostics thread through - :meth:`capture`. - """ - if self._limiter.admit(now): - return True - self.suppressed_total += 1 - return False - - def capture(self, *, now: float, wake_delay_seconds: float) -> str: - """Walk the other threads' stacks for an already-admitted sample.""" - stacks = self.capture_stacks() - self.samples_total += 1 - self.last_sample = { - "monotonic": float(now), - "wake_delay_seconds": float(wake_delay_seconds), - "stacks": stacks, - } - return stacks - - def sample(self, *, now: float, wake_delay_seconds: float) -> str | None: - """Admit and capture in one step, or ``None`` when rate-limited.""" - if not self.admit(now): - return None - return self.capture(now=now, wake_delay_seconds=wake_delay_seconds) - - def capture_stacks(self) -> str: - """One line per other thread: name, ident, innermost frames first. - - Walks the frame objects directly — ``f_code.co_name``, the basename - of ``f_code.co_filename``, ``f_lineno``, ``f_back`` — and never - touches ``linecache``. ``traceback.extract_stack`` would stat every - file and read every source line (milliseconds, and megabytes kept - in ``linecache``) for text this sample does not print; even - ``StackSummary.extract(lookup_lines=False)`` still calls - ``linecache.checkcache`` per file. Nothing here reads a file. - """ - current = threading.get_ident() - names = {thread.ident: thread.name for thread in threading.enumerate()} - frame = walker = None - frames = sys._current_frames() - try: - lines: list[str] = [] - listed = 0 - for ident, frame in frames.items(): - if ident == current: - continue - if listed >= self._max_threads: - lines.append( - f"... {len(frames) - 1 - listed} more threads not listed" - ) - break - listed += 1 - entries: list[str] = [] - walker = frame - while walker is not None and len(entries) < self._max_frames: - code = walker.f_code - entries.append( - f"{code.co_name}@{os.path.basename(code.co_filename)}" - f":{walker.f_lineno}" - ) - walker = walker.f_back - lines.append( - f"thread {names.get(ident, '?')} ({ident}): " + " <- ".join(entries) - ) - return "\n".join(lines) - finally: - # The snapshot includes this frame: keeping it forms a cycle - # retaining every sampled thread's locals until cyclic GC. Drop - # both the snapshot and traversal references, even on an error or - # a sampling cap. The loop can bind frame to this frame even when - # skipping the current thread. Never clear live frames themselves. - frames.clear() - frame = walker = None - - -def _default_monotonic() -> float: - """Read ``time.monotonic`` at call time, not at class-definition time. - - The lease lifecycle's documented test seam is patching ``monotonic`` on - the shared ``time`` module object. A dataclass default captured when this - module is imported would keep reading the real clock behind such a patch, - which is exactly the kind of silent divergence between test and - production that the phase attribution exists to prevent. - """ - return time.monotonic() - - -@dataclass(frozen=True) -class LedgerLeaseHeartbeatPorts: - """Call-time-resolved coordinator seams for the lease heartbeat owner. - - Every callable resolves live coordinator state (ledger identity, test - timing overrides, patched hard-exit methods) when invoked, preserving the - documented monkeypatch seams. ``lease_hard_exit`` routes through the - coordinator's ``_ledger_lease_heartbeat_hard_exit`` delegate so instance - patches intercept every internal failure path; the real exit body then - reaches process termination only through the dynamic coordinator - ``_watchdog_hard_exit`` port (PR 80 redirects that seam). - """ - - ledger: Callable[[], object] - heartbeat_seconds: Callable[[], float] - failure_seconds: Callable[[], float] - monitor_seconds: Callable[[], float] - exit_timeout_seconds: Callable[[], float] - external_fence_timeout_seconds: Callable[[], float] - lease_hard_exit: Callable[..., None] - watchdog_hard_exit: Callable[..., None] - heartbeat_loop: Callable[[], None] - monitor_loop: Callable[[], None] - # Every interval this service measures about itself reads this clock, so - # a deterministic test can drive the whole envelope — idle waits, phase - # attribution, staleness, the monitor's own lateness — by advancing a - # virtual clock instead of sleeping and hoping. Production passes - # nothing and gets time.monotonic. - monotonic: Callable[[], float] = _default_monotonic - # The process-side scheduling allowance, resolved like every other - # timing term rather than read from a module constant: it appears on - # both sides of the safety inequality (the heartbeat's stamping path - # and the monitor's own poll), so a deployment or a scaled-down test - # that changes it must change what the policy validates. - scheduler_slack_seconds: Callable[[], float] = ( - lambda: WRITER_LEASE_HEARTBEAT_SCHEDULER_SLACK_SECONDS - ) - # Where the monitor's stall diagnostics go (issue #227). Invoked only - # from the diagnostics thread the service spawns after the exit - # decision, never from the monitor thread, so a blocking default is - # safe here; see LedgerLeaseHeartbeatService._run_diagnostics. Tests - # substitute a list append to assert on the structured warning. - log: Callable[[str], None] = _print_line - - -class LedgerLeaseHeartbeatService: - """Sole owner of ledger-lease heartbeat state, threads, and fencing. - - Owns the success/progress/server-proven timestamps and their timing - envelope, guarded-session verifier selection, the heartbeat and monitor - threads, startup readiness/fail-closed state, the stop event, the failure - reason, the exit-thread exemption, and the synchronous exact-session - external-effect fence. It owns no writer-admission or lease-release-claim - state; that remains with ``CoordinatorShutdownController``. - """ - - def __init__(self, ports: LedgerLeaseHeartbeatPorts) -> None: - self._ports = ports - self.freshness_lock = threading.Lock() - self.failure_lock = threading.Lock() - self.last_success_monotonic: float | None = None - self.last_progress_monotonic: float | None = None - self.last_server_proven_monotonic: float | None = None - self.failed: threading.Event | None = None - self.ready: threading.Event | None = None - self.stop_event: threading.Event | None = None - self.thread: threading.Thread | None = None - self.monitor_thread: threading.Thread | None = None - self.exit_thread: threading.Thread | None = None - self.failure_reason: str | None = None - self.failure_has_traceback = False - # Phase attribution (issue #212). Written by whichever thread ran the - # attempt and published as one immutable snapshot, so the monitor can - # quote the last attempt in a hard-exit reason without locking. - self.last_phases: WriterLeaseVerificationPhases = UNATTRIBUTED_PHASES - self.worst_phases: WriterLeaseVerificationPhases = UNATTRIBUTED_PHASES - self.attempt_counts: dict[str, int] = { - mode: 0 for mode in LEASE_HEARTBEAT_MODES - } - self.outcome_counts: dict[str, int] = { - outcome: 0 for outcome in LEASE_HEARTBEAT_OUTCOMES - } - # The monitor's own lateness. It decides the hard exit, so a stalled - # monitor thread consumes envelope exactly like a stalled heartbeat - # thread and must be attributable separately from database latency. - # This is the lifetime high-water mark, kept for the existing gauge - # and exit message; the histogram, threshold counters, rolling - # maximum and record age that issue #227 asked for live in - # ``monitor_wakes``. - self.monitor_wake_delay_seconds = 0.0 - self.monitor_wakes = MonitorWakeTelemetry() - # Issue #227's response to lateness beyond slack: a wake later than - # the policy's max_guaranteed_monitor_lateness is a beat on which - # exit-before-adoption was not guaranteed. Counted, with the worst - # overrun kept, and warned about (rate-limited, never blocking). - self.exit_guarantee_breaches = 0 - self.worst_exit_guarantee_overrun_seconds = 0.0 - self.last_slack_breach: dict[str, object] | None = None - self.suppressed_slack_breach_warnings = 0 - self._slack_breach_warnings = _WindowRateLimiter( - max_events=LEASE_MONITOR_STALL_PROBE_MAX_SAMPLES_PER_WINDOW, - window_seconds=LEASE_MONITOR_STALL_PROBE_WINDOW_SECONDS, - ) - self.stall_probe = StallProbe() - # The most recent diagnostics thread (stack walk, formatting and - # the log write all happen there, after the exit decision); kept so - # a test can join it before asserting on what it logged. - self.diagnostics_thread: threading.Thread | None = None - # Installed on gc.callbacks by start() and removed by stop(), so a - # coordinator that never arms the heartbeat registers nothing. - self.gc_pauses = GcPauseTelemetry() - - def record_success(self, renewal_started_monotonic: float) -> None: - """Advance the conservative freshness edge without regression.""" - with self.freshness_lock: - last_success = self.last_success_monotonic - if ( - last_success is None - or renewal_started_monotonic > last_success - ): - self.last_success_monotonic = renewal_started_monotonic - - def record_phases(self, phases: WriterLeaseVerificationPhases) -> None: - """Publish one attempt's phase breakdown for exits and metrics.""" - self.last_phases = phases - if phases.total_seconds >= self.worst_phases.total_seconds: - self.worst_phases = phases - counts = self.attempt_counts - if phases.mode in counts: - counts[phases.mode] += 1 - outcomes = self.outcome_counts - if phases.outcome in outcomes: - outcomes[phases.outcome] += 1 - - def record_progress(self) -> None: - """Stamp mid-verification progress for the staleness monitor only. - - Fires from inside verify_writer_lease_guard_session when its - attribution recheck runs a second statement: the first statement's - round trip completed, proving the heartbeat machinery is making - progress. Deliberately separate from the success timestamp — the - verification has not yet reached a verdict, so nothing that treats - success as authorization freshness may observe this mark. Only the - heartbeat monitor reads it, so a lawful two-statement verification - is not mistaken for a wedged heartbeat while a genuinely stuck - statement still ages toward the failure budget unimpeded. - """ - self.last_progress_monotonic = self._ports.monotonic() - - def record_server_proven( - self, - proven_monotonic: float | None = None, - ) -> None: - """Stamp a completed server round trip: progress plus the envelope edge. - - Client-side progress marks (attempt start, query-slot acquisition) - keep a lawfully slow verification alive under the monitor, but they - can postdate a silent guard-session death by up to one idle - interval, so they cannot carry the adoption-envelope guarantee - (hard exit completes before a replacement's silence window can - elapse). Only a response actually received from PostgreSQL — a - completed statement round trip or a whole verification — proves - the session was alive approximately now, so only these sites stamp - the server-proven edge the monitor's envelope cap measures from. - - ``proven_monotonic`` is the *send* edge of the statement whose - response just arrived: a response proves the session was alive at - some instant between the statement leaving this process and the - answer arriving, and the conservative end of that interval is when - it left. Stamping receipt time instead would let scheduler delay - between the response and this call make the session look fresher - than PostgreSQL proved, spending adoption envelope that the exit - ordering argument budgets for the monitor's own poll. Omitting the - argument keeps the old receipt-time behavior for callers that - cannot observe the send edge. - - The stamp never regresses: the heartbeat and a concurrent external - fence both publish here, and an older send edge arriving second - must not undo a newer proof. - """ - now = self._ports.monotonic() - self.last_progress_monotonic = now - proven = now if proven_monotonic is None else float(proven_monotonic) - with self.freshness_lock: - last_proven = self.last_server_proven_monotonic - if last_proven is None or proven > last_proven: - self.last_server_proven_monotonic = proven - - def adoption_silence_seconds(self) -> float: - """Resolve the ledger's adoption-silence window for timing checks. - - PsqlShareLedger keeps the value on a private attribute with no - public alias; read both spellings so operator overrides actually - reach the misconfiguration warning and the monitor's envelope cap - instead of silently comparing against the compiled default. - """ - ledger = self._ports.ledger() - for name in ( - "lease_adoption_silence_seconds", - "_lease_adoption_silence_seconds", - ): - value = getattr(ledger, name, None) - if value: - return float(value) - return float(DEFAULT_WRITER_LEASE_ADOPTION_SILENCE_SECONDS) - - def activity_age_seconds(self) -> float: - """Age of the newest monitor-visible heartbeat activity mark. - - The success timestamp is deliberately the call-start edge (authority - freshness must never look newer than what PostgreSQL proved), so it - alone cannot answer "is the heartbeat advancing?": back-to-back - verifications that each lawfully approach the statement budget age - the freshest success edge by two call durations plus the idle - interval, and a budget sized for one statement would hard-exit a - healthy sole writer. Staleness enforcers therefore measure silence - from the newest of the success edge and the progress mark, which - carries attempt starts, query-slot acquisition, statement round - trips, and completions. A verification that stops advancing stops - stamping, so a wedged statement still ages out on the same budget. - """ - last_success = float( - self.last_success_monotonic - if self.last_success_monotonic is not None - else self._ports.monotonic() - ) - last_progress = float( - self.last_progress_monotonic - if self.last_progress_monotonic is not None - else last_success - ) - return max(0.0, self._ports.monotonic() - max(last_success, last_progress)) - - def server_proven_age_seconds(self) -> float: - """Age of the newest completed guard round trip. - - The envelope edge: unlike the client-side activity marks this can - never postdate a silent guard-session death, so it is what the - monitor's adoption cap measures from. - """ - last_server_proven = float( - self.last_server_proven_monotonic - if self.last_server_proven_monotonic is not None - else self._ports.monotonic() - ) - return max(0.0, self._ports.monotonic() - last_server_proven) - - def policy(self) -> WriterLeaseHeartbeatPolicy: - """Resolve the five coupled timings as one validated policy. - - Built from live ports and the ledger's own adoption silence, so an - operator override reaches the same inequality the compiled defaults - satisfy rather than being compared against a stale constant. - """ - return WriterLeaseHeartbeatPolicy( - adoption_silence_seconds=self.adoption_silence_seconds(), - heartbeat_interval_seconds=float(self._ports.heartbeat_seconds()), - failure_budget_seconds=float(self._ports.failure_seconds()), - monitor_interval_seconds=float(self._ports.monitor_seconds()), - exit_margin_seconds=float(self._ports.exit_timeout_seconds()), - scheduler_slack_seconds=float( - self._ports.scheduler_slack_seconds() - ), - ) - - def snapshot(self) -> dict[str, object]: - """Read-only heartbeat attribution for the metrics surface. - - Every key is fixed and every nested map has a fixed key set, so the - rendered metric cardinality cannot grow with traffic. - """ - policy = self.policy() - last = self.last_phases - worst = self.worst_phases - thread = self.thread - now = self._ports.monotonic() - return { - "running": bool(thread is not None and thread.is_alive()), - "activity_age_seconds": self.activity_age_seconds(), - "server_proven_age_seconds": self.server_proven_age_seconds(), - "monitor_wake_delay_seconds": self.monitor_wake_delay_seconds, - # Issue #227: the re-armable signals. Bucket, fraction and - # generation vocabularies are closed sets from - # writer_lease_timing / process_telemetry. - "monitor_wakes": self.monitor_wakes.snapshot(now), - "exit_guarantee_breaches": int(self.exit_guarantee_breaches), - "worst_exit_guarantee_overrun_seconds": float( - self.worst_exit_guarantee_overrun_seconds - ), - "stall_probe": { - "samples": int(self.stall_probe.samples_total), - "suppressed": int(self.stall_probe.suppressed_total), - }, - "slack_breach_warnings_suppressed": int( - self.suppressed_slack_breach_warnings - ), - "gc_pauses": self.gc_pauses.snapshot(), - "attempts": dict(self.attempt_counts), - "outcomes": dict(self.outcome_counts), - "last_phase_seconds": last.phase_seconds(), - "worst_phase_seconds": worst.phase_seconds(), - "last_statement_count": last.statement_count, - "policy_seconds": { - "adoption_silence": policy.adoption_silence_seconds, - "heartbeat_interval": policy.heartbeat_interval_seconds, - "failure_budget": policy.failure_budget_seconds, - "monitor_interval": policy.monitor_interval_seconds, - "exit_margin": policy.exit_margin_seconds, - "guard_statement_timeout": ( - policy.guard_statement_timeout_seconds - ), - "scheduler_slack": policy.scheduler_slack_seconds, - "exit_envelope": policy.exit_envelope_seconds, - "server_proven_cap": policy.server_proven_cap_seconds, - "max_healthy_server_gap": policy.max_healthy_server_gap_seconds, - "stability_surplus": policy.stability_surplus_seconds, - "max_guaranteed_monitor_lateness": ( - policy.max_guaranteed_monitor_lateness_seconds - ), - }, - } - - def start(self) -> threading.Thread | None: - """Arm both coupled threads and wait for the first liveness proof.""" - existing = self.thread - if existing is not None and existing.is_alive(): - return existing - ledger = self._ports.ledger() - guard_required = bool( - getattr(ledger, "writer_lease_guard_required", False) - ) - if not bool( - getattr(ledger, "writer_lease_fast_adoption_capable", False) - ): - if guard_required: - # The session token published to the lease row promises a - # guarded heartbeat session, but the advisory guard is already - # gone (for example the dedicated connection dropped between - # ledger construction and serve()). Continuing would let a - # replacement adopt the lease after one silence window while - # this process keeps running as a fenced zombie. - self._ports.lease_hard_exit( - "prism coordinator: writer lease guard was lost before " - "the lease heartbeat started; hard-exiting so a " - "replacement does not adopt while this process is live", - include_traceback=False, - ) - return None - verify = guard_session_verifier(ledger) - if verify is None: - if guard_required: - self._ports.lease_hard_exit( - "prism coordinator: writer lease guard requires a " - "heartbeat but the ledger cannot verify the guarded " - "session; hard-exiting instead of running unguarded", - include_traceback=False, - ) - return None - # One validated policy, resolved before anything is armed. An unsafe - # combination is refused outright rather than warned about and run: - # the whole content of a violation is "this process may still be - # live when a replacement adopts the lease", which is the split-brain - # the guard exists to prevent, and running anyway trades a loud - # startup failure for a silent double-writer window. Refusing here — - # before the threads exist and before serve() admits a writer — is - # the fail-closed reading. - policy = self.policy() - violations = policy.violations() - if violations: - self._ports.lease_hard_exit( - "prism coordinator: refusing to start the ledger lease " - "heartbeat; its timing policy cannot guarantee hard exit " - "before replacement adoption (" - + "; ".join(violations) - + f"). Policy: {policy.describe()}", - include_traceback=False, - ) - return None - for advisory in policy.advisories(): - # Safe but unstable: no double-writer window, but ordinary - # verification tail latency will hard-exit a healthy - # coordinator. A lab or test policy is allowed to make that - # trade deliberately, so this stays a warning. - print( - "prism coordinator: ledger lease heartbeat timing has no " - f"tail-latency headroom: {advisory}. Policy: " - f"{policy.describe()}", - flush=True, - ) - armed_started_monotonic = self._ports.monotonic() - self.last_success_monotonic = armed_started_monotonic - self.last_server_proven_monotonic = armed_started_monotonic - self.failed = threading.Event() - self.ready = threading.Event() - self.stop_event = threading.Event() - # GC pause durations are process-wide but the stall attribution is - # this service's question, so the collector callbacks are armed - # with the threads and disarmed with them (see stop()). - self.gc_pauses.install() - thread = threading.Thread( - target=self._ports.heartbeat_loop, - name="prism-ledger-lease-heartbeat", - daemon=True, - ) - self.thread = thread - thread.start() - monitor_thread = threading.Thread( - target=self._ports.monitor_loop, - name="prism-ledger-lease-heartbeat-monitor", - daemon=True, - ) - monitor_thread.start() - self.monitor_thread = monitor_thread - failure_seconds = policy.failure_budget_seconds - # The startup fencing deadline measures silence from the newest - # monitor-visible activity, not from arming: a first verification - # that is legally slower than the whole budget end-to-end but - # demonstrably advancing (queue wait, then a statement round trip - # near the guard's statement timeout) is not a wedged first beat. - # A genuinely stuck first statement stamps nothing and ages out on - # the same budget the monitor enforces. - while not self.ready.wait(0.01): - if self.failed.is_set(): - return None - if self.activity_age_seconds() < failure_seconds: - continue - self._ports.lease_hard_exit( - "prism coordinator: initial ledger lease heartbeat did not " - "complete before startup fencing deadline " - f"({self.last_phases.summary()})", - include_traceback=False, - ) - return None - return thread - - def hard_exit(self, message: str, *, include_traceback: bool) -> None: - with self.failure_lock: - failed = self.failed - if failed is None: - failed = threading.Event() - self.failed = failed - if failed.is_set(): - return - failed.set() - # This thread now proceeds to _watchdog_hard_exit and stays parked - # there until os._exit; it can never issue another renewal. The - # release worker must not join it, or a heartbeat/monitor-driven - # exit deadlocks against its own release and burns the exit budget - # before the fresh-connection lease release can run. - self.exit_thread = threading.current_thread() - # Never write to stdout/stderr before arming the hard-exit path. A full - # container log pipe can block a flush forever, leaving writer - # admission open and the old process alive. Keep the detail in memory - # for tests/embedders whose patched hard-exit returns; the real process - # exits and Docker's die event is the authoritative diagnostic. - self.failure_reason = message - self.failure_has_traceback = include_traceback - self._ports.watchdog_hard_exit( - "lease_heartbeat", - timeout_seconds=float(self._ports.exit_timeout_seconds()), - ) - - def run_guard_attempt( - self, - call: Callable[..., object], - callbacks: frozenset[str], - mode: str, - ) -> object: - """Run one guarded call, attributing its phases, and publish them. - - Every progress mark this installs feeds the same two clocks the - monitor reads, and the attempt object turns the same marks into the - phase breakdown an operator needs to see in the exit reason: - queue wait for the guard's serialized slot, guard SQL across the - round trips, and the residual — Python scheduling and GIL - contention — that neither of those explains. - - Progress marks feed only the staleness monitor; authority freshness - still moves exclusively on the conservative success edge the caller - records. The attempt-start and slot-acquisition marks are - client-side and can postdate a silent guard-session death by at - most one idle interval, so they carry only the liveness budget. The - adoption envelope is enforced separately by the monitor's - server-proven cap, which measures from completed round trips at - their conservative send edge. During any hang the external-effect - fence also queues behind the hung statement and fails closed, so no - new external effect authorizes while the monitor ages out. - - Raises whatever the guarded call raises, after publishing the - phases of the failed attempt — the exit reason is built from them. - """ - attempt = WriterLeaseVerificationAttempt( - mode, - monotonic=self._ports.monotonic, - ) - - def on_query_start() -> None: - attempt.slot_acquired() - self.record_progress() - - def on_statement_end() -> None: - self.record_server_proven(attempt.statement_completed()) - - kwargs: dict[str, Callable[..., None]] = {} - if "on_query_start" in callbacks: - kwargs["on_query_start"] = on_query_start - if "on_statement_end" in callbacks: - kwargs["on_statement_end"] = on_statement_end - elif "on_statement_progress" in callbacks: - # The legacy spelling: fires only between a verification's two - # statements, which is enough to keep a lawful attribution - # recheck from looking wedged but carries no phase timing. - kwargs["on_statement_progress"] = self.record_server_proven - if "on_statement_server_seconds" in callbacks: - # Issue #227: the statement's own server-side execution time, - # so a GIL stall between PostgreSQL answering and this thread - # resuming is attributed to this process rather than to SQL. - # Timing only; the proven edge above stays the send edge. - kwargs["on_statement_server_seconds"] = ( - attempt.statement_server_seconds - ) - try: - result = call(**kwargs) - except BaseException: - self.record_phases(attempt.finish(LEASE_HEARTBEAT_OUTCOME_FAILED)) - raise - self.record_phases(attempt.finish(self._attempt_outcome(mode, result))) - # Completion is a real server response even when the call reported - # no per-statement marks; stamp the conservative edge so a verifier - # without callbacks still keeps the envelope fed. - self.record_server_proven(attempt.proven_edge_monotonic) - return result - - @staticmethod - def _attempt_outcome(mode: str, result: object) -> str: - """Classify one guarded call for attribution and metrics.""" - if not isinstance(result, dict): - return ( - LEASE_HEARTBEAT_OUTCOME_PROVEN - if mode == LEASE_HEARTBEAT_MODE_PROOF - else LEASE_HEARTBEAT_OUTCOME_RENEWED - ) - if mode == LEASE_HEARTBEAT_MODE_PROOF: - if result.get("lease_renewal_due"): - return LEASE_HEARTBEAT_OUTCOME_RENEWAL_DUE - return LEASE_HEARTBEAT_OUTCOME_PROVEN - if result.get("renewal_deferred_to_own_write"): - return LEASE_HEARTBEAT_OUTCOME_DEFERRED - try: - renewed = int(result.get("renewed_count", 0)) - except (TypeError, ValueError): - renewed = 0 - if renewed > 0: - return LEASE_HEARTBEAT_OUTCOME_RENEWED - return LEASE_HEARTBEAT_OUTCOME_PROVEN - - def heartbeat_loop(self) -> None: - """Keep proving the guarded session live on an isolated DB path. - - The heartbeat must never wait on the lease tuple that fenced writes - hold for whole transactions, or a long ``persist_accepted_block`` - would time out the heartbeat statement and hard-exit a healthy - coordinator (the block-39416 restart loop). It still renews the - lease TTL whenever that tuple is uncontended, so an idle coordinator - (no fenced writes, CTV broadcaster disabled) does not let - ``lease_expires_at`` lapse into different-identity expiry claims. - - Ownership and renewal run at different rates (issue #212). Each beat - asks the cheap read-only ownership question — one non-blocking - statement, no row lock and no ``pg_stat_activity`` scan — and the - full renewing verification runs on the first beat and thereafter - only on a beat whose proof reports the committed row inside the - own-write authority margin. That escalation happens on the *same* - beat, so renewal is never later than it was before the split, and - while renewals are actually being skipped (a long own fenced write) - every beat runs the same fail-closed verification it always did. - A ledger with no proof method keeps running the full verification - on every beat. - """ - ledger = self._ports.ledger() - verify = guard_session_verifier(ledger) - if verify is None: - return - prove = guard_session_prover(ledger) - verify_callbacks = verifier_callbacks(verify) - prove_callbacks = ( - verifier_callbacks(prove) if prove is not None else frozenset() - ) - interval_seconds = float(self._ports.heartbeat_seconds()) - heartbeat_stop = self.stop_event - if heartbeat_stop is None: - heartbeat_stop = threading.Event() - self.stop_event = heartbeat_stop - # The first beat renews: startup readiness must prove the renewing - # path works, not merely that the session answers. - renew_this_beat = True - while not heartbeat_stop.is_set(): - beat_started_monotonic = self._ports.monotonic() - # An advancing verification is monitor-visible activity, and the - # attempt start is its first client-side mark. - self.record_progress() - try: - if prove is not None and not renew_this_beat: - proof = self.run_guard_attempt( - prove, - prove_callbacks, - LEASE_HEARTBEAT_MODE_PROOF, - ) - renew_this_beat = not isinstance(proof, dict) or bool( - proof.get("lease_renewal_due") - ) - if renew_this_beat or prove is None: - self.record_progress() - self.run_guard_attempt( - verify, - verify_callbacks, - LEASE_HEARTBEAT_MODE_RENEW, - ) - # A landed renewal (or a deferral behind this writer's - # own in-flight write) puts the answer back in the - # committed row: the next beat's proof re-reads it and - # escalates again if renewal is still outstanding. - renew_this_beat = False - except Exception: - self._ports.lease_hard_exit( - "prism coordinator: ledger lease heartbeat failed; " - "hard-exiting so this process cannot outlive its " - "fast-adoptable session " - f"({self.last_phases.summary()})", - include_traceback=True, - ) - return - # The session was proven live no earlier than the beat's start. - # Using that conservative edge prevents a delayed response from - # making local freshness look newer than what PostgreSQL proved. - self.record_success(beat_started_monotonic) - ready = self.ready - if ready is not None: - ready.set() - if heartbeat_stop.wait(interval_seconds): - return - - def monitor_loop(self) -> None: - # One resolved policy for the whole loop: the monitor's two bounds - # and the exit reason all quote the same inequality. - policy = self.policy() - failure_seconds = policy.failure_budget_seconds - monitor_seconds = policy.monitor_interval_seconds - heartbeat_stop = self.stop_event - if heartbeat_stop is None: - heartbeat_stop = threading.Event() - self.stop_event = heartbeat_stop - # The adoption-envelope cap: the client-side progress marks that - # keep a lawfully slow verification alive can postdate a silent - # guard-session death by up to one idle interval, which with a - # silence sized at interval + budget would let the hard exit land - # at or after a replacement's CAS eligibility. Server-proven marks - # cannot postdate the death, so a second bound measured from them - # restores exit-before-adoption regardless of client marks. What - # the cap must leave free inside the silence is the whole exit - # envelope: the hard-exit budget, one poll of granularity, this - # thread's own budgeted lateness, and one further poll of strict - # reserve (see WriterLeaseHeartbeatPolicy.exit_envelope_seconds). - # Reserving only the poll intervals — as this did before the - # scheduler-slack term was added — lets a maximally late monitor - # begin the exit after the successor is already eligible. The cap - # is floored at the failure budget so a deliberately tiny (advised) - # silence degrades to the plain liveness bound instead of killing - # lawful single-statement verifications. - server_cap_seconds = policy.server_proven_cap_seconds - while True: - wait_started_monotonic = self._ports.monotonic() - if heartbeat_stop.wait(monitor_seconds): - return - # The monitor decides the hard exit, so its own lateness is - # spent envelope exactly like a slow statement is. Attributing - # it separately is what lets an operator tell "PostgreSQL was - # slow" from "this process could not get scheduled". - wake_delay = max( - 0.0, - self._ports.monotonic() - wait_started_monotonic - monitor_seconds, - ) - if wake_delay > self.monitor_wake_delay_seconds: - self.monitor_wake_delay_seconds = wake_delay - # Issue #227: the lifetime maximum above is a signal that can - # fire once per process. Feed the histogram, the slack-fraction - # counters, the rolling maximum and the record age, and decide - # whether the documented response is due. All of that is O(1) - # arithmetic; anything heavier (the stack walk, the formatting, - # the log write) comes back as a deferred job and runs only - # *after* the exit decision below, and never on this thread. - # The instrument must not extend what it measures: work done - # here would consume the very envelope it is reporting, and - # work done between polls is not measured as lateness at all. - diagnostics = self._observe_monitor_wake(wake_delay, policy) - # Mid-verification progress counts: the budget — sized for one - # statement plus one idle wait plus scheduler slack so it stays - # under the adoption silence — cannot absorb a whole multi-step - # verification, and each demonstrable step (attempt start, slot - # acquisition, statement round trip, completion) is the - # advancing-heartbeat evidence this monitor watches for. A - # wedged statement records nothing and still ages out. - age_seconds = self.activity_age_seconds() - server_age_seconds = self.server_proven_age_seconds() - if ( - age_seconds < failure_seconds - and server_age_seconds < server_cap_seconds - ): - if diagnostics is not None: - self._run_diagnostics(diagnostics) - continue - # Exiting: the deferred diagnostics are dropped. The exit reason - # below already carries the wake-delay figures and the worst GC - # pause, and nothing may run ahead of the exit. - # Name the bound that tripped and the phase that consumed it. - # Issue #212 was unresolvable in production precisely because - # the exit said only how stale the heartbeat was, never where - # the time went. - if age_seconds >= failure_seconds: - tripped = ( - f"activity {age_seconds:.3f}s >= failure budget " - f"{failure_seconds:.3f}s" - ) - else: - tripped = ( - f"server-proven {server_age_seconds:.3f}s >= adoption " - f"envelope cap {server_cap_seconds:.3f}s" - ) - # The wake-delay figures are the lifetime record, the trailing - # window's worst and the record's age, so an operator can tell - # a stall that just happened from one hours old (issue #227). - exit_now = self._ports.monotonic() - gc_generation, gc_worst = self.gc_pauses.worst() - self._ports.lease_hard_exit( - "prism coordinator: ledger lease heartbeat stopped making " - f"progress for {age_seconds:.3f}s " - f"(server-proven {server_age_seconds:.3f}s); hard-exiting " - "before its session becomes fast-adoptable. " - f"Tripped: {tripped}. Last attempt: " - f"{self.last_phases.summary()}. Worst attempt: " - f"{self.worst_phases.summary()}. Monitor wake delay: " - f"{self.monitor_wake_delay_seconds:.3f}s lifetime record " - f"({self.monitor_wakes.record_age_seconds(exit_now):.1f}s old), " - f"{self.monitor_wakes.window_max_seconds(exit_now):.3f}s in the " - f"last {LEASE_MONITOR_WAKE_DELAY_WINDOW_SECONDS:g}s, " - f"{self.exit_guarantee_breaches} exit-guarantee breach(es). " - f"Worst GC pause: {gc_worst:.3f}s " - f"(generation {gc_generation if gc_generation is not None else 'none'}). " - f"Policy: {policy.describe()}", - include_traceback=False, - ) - return - - def _observe_monitor_wake( - self, - wake_delay: float, - policy: WriterLeaseHeartbeatPolicy, - ) -> Callable[[], None] | None: - """Account one monitor wake in O(1); return the deferred diagnostics. - - The decision issue #227 asked for, argued in the - ``writer_lease_timing`` module docstring: lateness beyond the slack - is *accepted* rather than turned into a tightened cap (which would - hard-exit healthy coordinators under a transient stall — issue - #212 — without moving the exit earlier on the beat that breached), - because every effect that could escape during the un-covered - interval is fenced at the row or behind a synchronous - exact-session verification. What changes is that the breach is no - longer invisible: it is counted, its overrun past the - exit-guarantee bound is kept, one structured warning names it, and - a stack sample says what held the interpreter. - - Three tiers, all keyed on the policy's ``scheduler_slack``: - - * at or above half the slack: take a rate-limited stack sample; - * above the slack: the budgeted assumption was exceeded — record - the breach and warn (rate-limited); - * above ``max_guaranteed_monitor_lateness`` (slack plus one - monitor interval): exit-before-adoption was not guaranteed for - this beat — count it and keep the worst overrun. - - This runs on the monitor thread *before* the staleness decision, so - it does arithmetic only: the histogram, the counters, the - rate-limit admissions and a handful of float reads. Everything that - walks frames, formats text or writes a log line is returned as a - closure for :meth:`monitor_loop` to hand to - :meth:`_run_diagnostics` after the exit decision — or to drop, when - the decision is to exit. The instrument must not extend the - envelope it reports on. - """ - now = self._ports.monotonic() - slack = float(policy.scheduler_slack_seconds) - self.monitor_wakes.observe(wake_delay, slack_seconds=slack, now=now) - if wake_delay < LEASE_MONITOR_STALL_PROBE_TRIGGER_SLACK_FRACTION * slack: - return None - probe_admitted = self.stall_probe.admit(now) - if wake_delay <= slack: - if not probe_admitted: - return None - last_summary = self.last_phases.summary() - return lambda: self._emit_probe_sample( - now=now, - wake_delay=wake_delay, - slack=slack, - last_summary=last_summary, - ) - bound = policy.max_guaranteed_monitor_lateness_seconds - overrun = policy.exit_guarantee_overrun_seconds(wake_delay) - if overrun > 0.0: - self.exit_guarantee_breaches += 1 - if overrun > self.worst_exit_guarantee_overrun_seconds: - self.worst_exit_guarantee_overrun_seconds = overrun - # The figures the warning quotes are read now, at the wake, not - # when the diagnostics thread gets around to formatting them. - server_proven_age = self.server_proven_age_seconds() - activity_age = self.activity_age_seconds() - window_max = self.monitor_wakes.window_max_seconds(now) - breaches = self.exit_guarantee_breaches - self.last_slack_breach = { - "monotonic": now, - "wake_delay_seconds": float(wake_delay), - "overrun_seconds": float(overrun), - "server_proven_age_seconds": server_proven_age, - "stacks": None, - } - warning_admitted = self._slack_breach_warnings.admit(now) - if not warning_admitted: - self.suppressed_slack_breach_warnings += 1 - if not probe_admitted and not warning_admitted: - return None - return lambda: self._emit_slack_breach( - now=now, - wake_delay=wake_delay, - slack=slack, - bound=bound, - overrun=overrun, - breaches=breaches, - server_proven_age=server_proven_age, - activity_age=activity_age, - window_max=window_max, - policy=policy, - probe_admitted=probe_admitted, - warning_admitted=warning_admitted, - ) - - def _run_diagnostics(self, job: Callable[[], None]) -> None: - """Run deferred wake diagnostics on a throwaway daemon thread. - - Never on the monitor thread. The monitor's poll spacing is one - interval plus its lateness plus whatever it does between polls, - and that last term is not measured as lateness, so it has to stay - O(1): a frame walk, string formatting or a print here would be - unmeasured envelope consumption on the thread that decides the - hard exit. The rate limits upstream (probe and warning admissions) - bound how many of these threads a stall storm can create. - """ - thread = threading.Thread( - target=job, - name="prism-ledger-lease-monitor-diagnostics", - daemon=True, - ) - self.diagnostics_thread = thread - try: - thread.start() - except Exception: - pass - - def _emit_probe_sample( - self, - *, - now: float, - wake_delay: float, - slack: float, - last_summary: str, - ) -> None: - """Diagnostics thread: an admitted half-slack sample and its log line.""" - stacks = self.stall_probe.capture(now=now, wake_delay_seconds=wake_delay) - self._ports.log( - "prism coordinator: ledger lease heartbeat monitor woke " - f"{wake_delay:.3f}s late (scheduler slack {slack:.3f}s); " - "stall probe sampled the running threads. Last attempt: " - f"{last_summary}. Threads:\n{stacks}" - ) - - def _emit_slack_breach( - self, - *, - now: float, - wake_delay: float, - slack: float, - bound: float, - overrun: float, - breaches: int, - server_proven_age: float, - activity_age: float, - window_max: float, - policy: WriterLeaseHeartbeatPolicy, - probe_admitted: bool, - warning_admitted: bool, - ) -> None: - """Diagnostics thread: the sample and the structured breach warning.""" - stacks: str | None = None - if probe_admitted: - stacks = self.stall_probe.capture(now=now, wake_delay_seconds=wake_delay) - breach = self.last_slack_breach - if breach is not None and breach.get("monotonic") == now: - breach["stacks"] = stacks - if not warning_admitted: - return - if overrun > 0.0: - verdict = ( - "exit-before-adoption was NOT guaranteed for this beat: " - f"the worst-case exit lands {overrun:.3f}s after the " - f"adoption edge (bound {bound:.3f}s)" - ) - else: - verdict = ( - "exit-before-adoption still held for this beat, with the " - "strictness reserve consumed " - f"(bound {bound:.3f}s)" - ) - gc_generation, gc_worst = self.gc_pauses.worst() - self._ports.log( - "prism coordinator: WARNING ledger lease heartbeat monitor wake " - f"delay {wake_delay:.3f}s exceeded the {slack:.3f}s scheduler " - "slack the timing policy budgets (issue #227 lateness-beyond-" - f"slack breach; {breaches} exit-guarantee " - f"breach(es) this process); {verdict}. The stall was in-process " - "(this thread could not be scheduled); every ledger write is " - "session-token fenced and every external effect is behind an " - "exact-session verification, so no effect escapes, but the " - "process may have outlived the adoption edge had its guard " - "session died. " - f"server_proven_age={server_proven_age:.3f}s " - f"activity_age={activity_age:.3f}s " - f"window_max={window_max:.3f}s " - f"worst_gc_pause={gc_worst:.3f}s" - f"(generation {gc_generation if gc_generation is not None else 'none'}). " - f"Last attempt: {self.last_phases.summary()}. Worst attempt: " - f"{self.worst_phases.summary()}. Policy: {policy.describe()}." - + (f" Threads:\n{stacks}" if stacks is not None else "") - ) - - def stop(self, *, deadline: float | None = None) -> bool: - heartbeat_stop = self.stop_event - if heartbeat_stop is None: - return True - heartbeat_stop.set() - self.gc_pauses.uninstall() - threads = (self.thread, self.monitor_thread) - # A heartbeat or monitor thread that armed the hard exit is blocked in - # _watchdog_hard_exit joining the very release worker that runs this - # method. It has already stopped renewing forever, so treat it as - # stopped instead of deadlocking against it until the deadline. - exit_thread = self.exit_thread - if deadline is None: - deadline = ( - self._ports.monotonic() - + DEFAULT_WRITER_LEASE_ADOPTION_SILENCE_SECONDS - ) - for thread in threads: - if ( - thread is None - or thread is threading.current_thread() - or thread is exit_thread - ): - continue - thread.join(max(0.0, deadline - self._ports.monotonic())) - return all( - thread is None - or thread is threading.current_thread() - or thread is exit_thread - or not thread.is_alive() - for thread in threads - ) - - def require_fresh_lease_for_external_side_effect(self, component: str) -> None: - """Synchronously verify the exact guarded session before an RPC effect. - - The periodic heartbeat is an early process-liveness detector, not an - authorization oracle: CLOCK_MONOTONIC may not advance across host - suspend, and a PostgreSQL connection object does not necessarily know - its server session died until the next I/O. Every external mutation - therefore performs a bounded non-blocking verification on the session - holding the advisory guard. It must not wait on the lease row: a long - fenced transaction (``persist_accepted_block``) holds that tuple's - row lock until commit, and queueing behind it would time out and - fence a healthy coordinator (the TTL is renewed opportunistically via - SKIP LOCKED instead). A daemon worker bounds a dead network path even - when the driver's server-side statement timeout cannot. - - PostgreSQL and qbitd are independent systems, so this remains a - preflight fence rather than an atomic transaction with the subsequent - RPC. The advisory session remains held across the RPC in the ordinary - case; see the deployment guide for the residual post-check pause risk. - - A verification that reports ``renewal_deferred_to_own_write`` proves - liveness but not authority: the writer's own fenced write holds the - expired lease row, and only its commit guarantees the lease survives. - That is enough for the heartbeat, which merely keeps a healthy - process from self-fencing, but not for an RPC — a rollback would let - a queued claimant take the lease while the effect is in flight. This - fence then raises :class:`WriterLeaseRenewalDeferred` without arming - the hard exit; callers already retry deferred work on their own - cadence. - """ - ledger = self._ports.ledger() - if not bool( - getattr(ledger, "writer_lease_guard_required", False) - ): - return - verify = guard_session_verifier(ledger) - if verify is None: - self._ports.lease_hard_exit( - "prism coordinator: refusing external side effect from " - f"{component}; exact-session verification is unavailable", - include_traceback=False, - ) - raise ShutdownInProgress( - f"writer lease guard verification is unavailable before {component}" - ) - - # The configured fence timeout budgets one statement; the - # verification may lawfully run a second inside the same guarded - # slot (the attribution recheck), and killing that recheck at the - # single-statement deadline would hard-exit a coordinator the - # recheck was about to prove healthy. Two moderately slow but - # in-deadline statements must both fit. - timeout_seconds = WRITER_LEASE_VERIFICATION_MAX_STATEMENTS * max( - 0.0, - float(self._ports.external_fence_timeout_seconds()), - ) - # The guarded session serializes this verification behind the periodic - # heartbeat and any concurrent fence, and the execution budget above - # tracks the guard's statement timeout. Waiting for the serialized - # query slot must not consume that budget, or a fence arriving during - # an in-flight renewal hard-exits a coordinator whose session is still - # healthy. The queue wait gets the heartbeat failure budget instead: a - # guard session that cannot complete queued work within it is already - # being declared dead by the heartbeat monitor. - queue_timeout_seconds = max( - timeout_seconds, - float(self._ports.failure_seconds()), - ) - query_started = threading.Event() - callbacks = verifier_callbacks(verify) - verify_reports_query_start = "on_query_start" in callbacks - # This verification holds the guard's serialized slot for its whole - # statement sequence, so the periodic heartbeat queues behind it - # and records nothing meanwhile. Its lawful second statement (the - # attribution recheck) can therefore age the heartbeat's last - # success past the monitor's failure budget — the same hazard the - # heartbeat loop's own recheck had — so this caller must feed the - # monitor its completed round trips too. - verify_reports_statement_end = "on_statement_end" in callbacks - verify_reports_statement_progress = ( - not verify_reports_statement_end - and "on_statement_progress" in callbacks - ) - verify_reports_server_seconds = ( - "on_statement_server_seconds" in callbacks - ) - # The fence is a guard-slot caller like the heartbeat, so its phases - # belong in the same attribution: a fence that spent its budget - # queueing behind an in-flight renewal reads very differently from - # one that spent it inside PostgreSQL. - attempt = WriterLeaseVerificationAttempt( - LEASE_HEARTBEAT_MODE_FENCE, - monotonic=self._ports.monotonic, - ) - if not verify_reports_query_start: - # A verification that cannot report when its query slot was - # acquired keeps the previous behavior: the whole budget covers - # both. - query_started.set() - - outcome: queue.Queue[BaseException | None] = queue.Queue(maxsize=1) - verification_result: list[object] = [] - verification_started_monotonic = attempt.started_monotonic - - def on_query_start() -> None: - attempt.slot_acquired() - query_started.set() - - def on_statement_end() -> None: - self.record_server_proven(attempt.statement_completed()) - - def verify_exact_session() -> None: - try: - verify_kwargs: dict[str, Callable[..., None]] = {} - if verify_reports_query_start: - verify_kwargs["on_query_start"] = on_query_start - if verify_reports_statement_end: - verify_kwargs["on_statement_end"] = on_statement_end - elif verify_reports_statement_progress: - verify_kwargs["on_statement_progress"] = ( - self.record_server_proven - ) - if verify_reports_server_seconds: - verify_kwargs["on_statement_server_seconds"] = ( - attempt.statement_server_seconds - ) - result = verify(**verify_kwargs) - except BaseException as exc: - outcome.put(exc) - else: - verification_result.append(result) - outcome.put(None) - - try: - thread = threading.Thread( - target=verify_exact_session, - name="prism-ledger-lease-external-fence", - daemon=True, - ) - thread.start() - except BaseException as exc: - self._ports.lease_hard_exit( - "prism coordinator: refusing external side effect from " - f"{component}; could not start exact-session verification", - include_traceback=False, - ) - raise ShutdownInProgress( - f"writer lease guard verification could not start before {component}" - ) from exc - - def refuse(detail: str) -> str: - """Fail closed, naming the phase that consumed the fence budget.""" - phases = attempt.finish(LEASE_HEARTBEAT_OUTCOME_FAILED) - self.record_phases(phases) - return ( - "prism coordinator: refusing external side effect from " - f"{component}; {detail} ({phases.summary()})" - ) - - queue_deadline = verification_started_monotonic + queue_timeout_seconds - while not query_started.is_set() and thread.is_alive(): - remaining = queue_deadline - self._ports.monotonic() - if remaining <= 0.0: - self._ports.lease_hard_exit( - refuse( - "exact-session verification could not start within " - f"{queue_timeout_seconds:g}s" - ), - include_traceback=False, - ) - raise ShutdownInProgress( - f"writer lease guard verification timed out before {component}" - ) - thread.join(min(0.01, remaining)) - thread.join(timeout_seconds) - if thread.is_alive(): - self._ports.lease_hard_exit( - refuse( - "exact-session verification exceeded " - f"{timeout_seconds:g}s" - ), - include_traceback=False, - ) - raise ShutdownInProgress( - f"writer lease guard verification timed out before {component}" - ) - - try: - error = outcome.get_nowait() - except queue.Empty as exc: - self._ports.lease_hard_exit( - refuse("exact-session verification returned no result"), - include_traceback=False, - ) - raise ShutdownInProgress( - f"writer lease guard verification failed before {component}" - ) from exc - if error is not None: - self._ports.lease_hard_exit( - refuse("exact-session verification failed"), - include_traceback=False, - ) - raise ShutdownInProgress( - f"writer lease guard verification failed before {component}" - ) from error - - # Use the call-start time so scheduler delay never makes this success - # appear fresher than the database response actually proves. The - # envelope edge takes the conservative send edge of the newest - # completed round trip for the same reason. - self.record_success(verification_started_monotonic) - self.record_server_proven(attempt.proven_edge_monotonic) - - # A verification that proved liveness only because the writer's own - # fenced write holds the expired lease row is not authority for an - # external side effect: that survival argument assumes the write - # commits, and a rollback would hand the row to a queued - # different-identity claimant while the RPC is in flight. Refuse the - # side effect without fencing the process — the session is healthy - # and the write's commit will land the next renewal; callers retry - # on their own cadence (broadcast interval, candidate outbox replay). - result = verification_result[0] if verification_result else None - self.record_phases( - attempt.finish( - self._attempt_outcome(LEASE_HEARTBEAT_MODE_FENCE, result) - ) - ) - if isinstance(result, dict) and result.get( - "renewal_deferred_to_own_write" - ): - raise WriterLeaseRenewalDeferred( - f"withholding {component}: writer lease renewal is deferred " - "behind this coordinator's own in-flight fenced write; " - "retry after that transaction commits and a renewal lands" - ) - - -class _LeaseHeartbeatStateField: - """Descriptor routing a legacy coordinator attribute to lease ownership. - - The lease service holds the only copy of each mutable field; the - coordinator class keeps the historical private attribute names readable - and writable for tests and embedders. Reads and writes construct the - lazy service on first touch so pre-service fixture assignments are - adopted rather than duplicated. - """ - - def __init__(self, attribute: str): - self.attribute = attribute - - def __get__(self, instance: Any, owner: type[Any] | None = None) -> Any: - if instance is None: - return self - return getattr( - instance._ensure_lease_heartbeat_service(), self.attribute - ) - - def __set__(self, instance: Any, value: Any) -> None: - setattr( - instance._ensure_lease_heartbeat_service(), self.attribute, value - ) diff --git a/lab/prism/block_candidates.py b/lab/prism/block_candidates.py deleted file mode 100644 index 3880de25f..000000000 --- a/lab/prism/block_candidates.py +++ /dev/null @@ -1,8967 +0,0 @@ -"""Durable PRISM block-candidate codec, replay queue, and submitter service. - -B1 owner module: the node-offer fast lane, the same-hash disposition guard, -the accounting actor and its handoff queues, startup/outbox replay and -quarantine, retry merge/pacing/finalize-only state, acceptance evidence, -block-work liveness heartbeats, and the bounded DB/RPC call workers all live -here. Post-offer accepted-block finalization (``submit_block_candidate``, -``_submit_block_candidate_serialized``, ``_land_and_confirm_block_candidate``) -stays coordinator-owned at this layer and is reached through the injected -coordinator reference; a later layer moves it to its own finalization owner. -This module never imports ``prism_coordinator``. -""" - -from __future__ import annotations - -import dataclasses -import inspect -import itertools -import json -import math -import queue -import threading -import time -import traceback -from contextlib import AbstractContextManager, contextmanager -from dataclasses import dataclass, field, replace as dataclass_replace -from types import SimpleNamespace -from collections.abc import Mapping, Sequence -from typing import Any, Callable, Iterable, Iterator, Protocol - -from lab.prism import direct_stratum -from lab.prism.audit_bundle_view import MappedSequence -from lab.prism.accepted_preview_telemetry import ( - LANDING_PHASE_LANE_WAIT, - LANDING_PHASE_PREVIEW_PREPARE, - LANDING_PHASE_PREVIEW_PUBLISH, - PRISM_ACCEPTED_LANDING_PHASES, - RECONCILE_CALLER_LANDING, - AcceptedPreviewPublicationDiagnostic, - ensure_accepted_preview_telemetry, -) -from lab.prism.coordinator_config import ( - BLOCK_LANDING_DB_TIMEOUT_WATCHDOG_FRACTION, - DEFAULT_ACCEPTED_PARENT_REDRIVE_ATTEMPT_MAX, - DEFAULT_ACCEPTED_PARENT_REDRIVE_DEFER_THRESHOLD, - DEFAULT_BLOCK_CANDIDATE_CLEANUP_RETRY_BACKLOG_MAX, - DEFAULT_BLOCK_LANDING_DB_TIMEOUT_MAX_SECONDS, - DEFAULT_BLOCK_LANDING_DB_TIMEOUT_SECONDS, - DEFAULT_BLOCK_SUBMIT_DB_TIMEOUT_SECONDS, - DEFAULT_BLOCK_SUBMIT_LOCK_WAIT_LOG_SECONDS, - DEFAULT_BLOCK_SUBMIT_RPC_TIMEOUT_SECONDS, - DEFAULT_BLOCK_SUBMIT_STUCK_CALL_EXIT_SECONDS, - DEFAULT_PRISM_OBSERVED_TIP_ACCEPT_WINDOW_SECONDS, - DEFAULT_PRISM_WATCHDOG_TIMEOUT_SECONDS, - MAX_BLOCK_CANDIDATE_CLEANUP_RETRY_BACKLOG_MAX, - MAX_BLOCK_REPLAY_PAGE_SIZE, -) -from lab.prism.candidate_codec import ( - CandidateBodyIntegrityError, - CandidateCodecError, - PreparedCandidateIntent, - prepare_candidate_intent, -) -from lab.prism.candidate_store import ( - HEADER_PAGE_MAX_BYTES, - HEADER_PAGE_MAX_ROWS, - CandidateBodyRef, - CandidateBodyUnavailable, - DurableCandidateDescriptor, -) -from lab.prism.coordinator_shutdown import ShutdownInProgress -from lab.prism.job_bundle import PRISM_JOB_BUILD_SECONDS_BUCKETS -from lab.prism.job_delivery import PrismJobContext -from lab.prism.rpc import JsonRpc -from lab.prism.share_ledger import LedgerOperationTimeout, PendingShare -from lab.prism.stratum_session import ClientState, WorkerIdentity - - -# Block candidates queue to a dedicated submitter thread so the miner's share -# ack never waits on audit/submitblock after the share and intent commit. The -# bound limits RAM; overflow only coalesces a wakeup because Postgres retains -# the authoritative pending candidate. -MAX_PENDING_BLOCK_CANDIDATES = 32 -# Startup replay must observe every durable pending candidate before child -# job builds unblock (issue #188 fix 4): a truncated enumeration could hide -# the active parent whose payout transition is not yet armed. A full batch -# therefore re-queries with a doubled window until the result is provably -# untruncated. This cap bounds one enumeration pass's memory; at the cap the -# gate simply stays closed while the queued batch drains, and the submitter -# loop re-enumerates the shrinking remainder. -MAX_BLOCK_REPLAY_ENUMERATION_ROWS = MAX_BLOCK_REPLAY_PAGE_SIZE -# Issue #255. Metadata-first replay keeps at most this many small durable -# descriptors in memory; rows beyond it register their accepted-parent and -# credit-floor obligations and stay in the outbox, which a later poll -# re-reads once the window drains. A descriptor is a few hundred bytes, so -# the window is well inside the 4 MiB header/queue budget. -MAX_BLOCK_REPLAY_DESCRIPTORS_IN_MEMORY = 1024 -# Rows per metadata-first replay page: the configured replay page size is -# additionally clamped to the #255 header page cap. -BLOCK_REPLAY_HEADER_PAGE_ROWS = min(MAX_BLOCK_REPLAY_ENUMERATION_ROWS, HEADER_PAGE_MAX_ROWS) -# Minimum spacing between two janitor steps on the submitter loop. -BLOCK_CANDIDATE_BODY_JANITOR_INTERVAL_SECONDS = 5.0 -# Ancestor re-drive bookkeeping (issue #190) is keyed by block hash and -# dropped the moment the blocking transition resolves; this bound only -# guards against a pathological stream of distinct never-resolving -# ancestors leaking entries forever. Eviction is oldest-first, and every -# deferral re-inserts its ancestor's record and its child's blocking -# entry, so for a wedge the still-blocking pair is the last thing evicted. -MAX_ANCESTOR_REDRIVE_TRACKED_HASHES = 64 - - -@dataclass -class _AncestorRedriveRecord: - """Per-ancestor re-drive bookkeeping (issue #190). - - One record holds every fact the mechanism tracks about an ancestor, so - recency (the record's insertion position), the armed request, and the - exhaustion latch can never disagree the way parallel registries could. - Mutated only under ``coordinator.lock``. - """ - - # Consecutive finalization deferrals since the last armed pass. Frozen - # while a request is armed-but-unconsumed: deferrals during that window - # prove nothing the armed pass will not already act on, and counting - # them would let the next attempts arm on single deferrals right after - # consumption, exhausting the cap before the first pass's adoption had - # a chance to resolve the ancestor. - streak: int = 0 - # Re-drive passes armed against the per-ancestor cap. - attempts: int = 0 - # Armed passes the submitter actually consumed. The resolved counter - # keys off this, not ``attempts``: a transition that resolved through - # the ordinary landing tail while a request sat unconsumed is the - # mechanism standing by, not succeeding. - consumed: int = 0 - # A forced durable-replay pass is waiting for the submitter loop. - armed: bool = False - # The cap is spent; deferrals fall back to exactly the pre-#190 - # behavior with the publication-progress watchdog as the backstop. - exhausted: bool = False - - -DEFAULT_BLOCK_CANDIDATE_RETRY_INITIAL_SECONDS = 0.25 -DEFAULT_BLOCK_CANDIDATE_RETRY_MAX_SECONDS = 30.0 -# The primary accounting handoff queue must be bounded or the documented -# result-preserving spillover ordering can never engage; the overflow queue -# stays unbounded by design so an already-offered block is never converted -# back into a raw-submit retry. -DEFAULT_BLOCK_ACCOUNTING_QUEUE_DEPTH = 8 -# How many definitive-acceptance accounting tasks the lane may dispatch back -# to back while non-accepted work is waiting. The accepted lane exists so a -# decided block's accounting never queues behind stale replay (see -# _enqueue_block_accounting_task), but "never behind" must not become "stale -# replay never runs": a stale accounting task that is starved leaves its -# durable outbox row pending, its offer-time accepted-block payout-preview -# barrier armed, and its disposition lease held, which is a different bug -# rather than a fixed one. The quota therefore states the fairness bound -# outright -- a non-accepted task at the head of its lane waits behind at -# most this many accepted services, whatever the accepted arrival rate. -# -# Four, not one: one accepted tip can produce more than one accepted-evidence -# task (the winner, plus a same-height idempotent replay of it), and a quota -# of one would interleave a stale task's ~6 RPCs, two ledger writes and a -# payout-balance mutation into the middle of that acceptance tail -- the very -# interleaving the lane was added to remove. Four covers a normal tip's tail -# while still bounding stale delay at four accepted services. -BLOCK_ACCOUNTING_ACCEPTED_DISPATCH_QUOTA = 4 -# A collapsed row whose post-write cleanup failed has no durable replay -# source left, so the accounting lane is the only thing that can ever finish -# it. Draining it purely from that lane's idle branch made it hostage to the -# lane staying idle: sustained ordinary accounting traffic, or a -# continuously replenished invalid-candidate quarantine queue, starved the -# retry for as long as the traffic lasted. The lane therefore also offers -# one bounded retry after at most this many completed work items, where a -# work item is one finished accounting task from any of the handoff queues -# or one finished quarantine item. The offer runs at most one hash's -# still-owed steps and the cadence resets on every offer -- including one -# that finds nothing due -- so neither lane can starve the other. -DEFAULT_BLOCK_ACCOUNTING_CLEANUP_RETRY_WORK_ITEMS = 8 -# Issue #198. A cleanup-retry record is the only replay source a durably -# terminal row has left, so the registry never sheds one; under a systemic -# cleanup fault it therefore grows by one record per row the collapse wins. -# At the observed 312,000-candidate storm that is an unbounded in-memory -# backlog -- each record retaining its exact pending-share floor holders and -# pinning its terminal-outcome fence -- driven by a lane that retries one -# hash per pass. The bound is applied to *admission* instead: once the -# backlog holds ``_collapse_cleanup_retry_backlog_max()`` records -# (PRISM_BLOCK_CANDIDATE_CLEANUP_RETRY_BACKLOG_MAX, default -# DEFAULT_BLOCK_CANDIDATE_CLEANUP_RETRY_BACKLOG_MAX) the collapse hands no -# further rows to its fenced bulk terminalization, and every row it declines -# stays durable and pending for the ordinary per-row path. Nothing already -# terminal loses its record, its holders, or its fence. -# -# A persistently engaged backpressure would otherwise print one line per -# page or per dequeued sibling; the warning is rate limited to this interval -# while the ``backlog_deferred`` counter still moves per row. -BLOCK_CANDIDATE_CLEANUP_BACKPRESSURE_LOG_SECONDS = 60.0 -# The closed vocabulary the backpressure warning names its caller from, so -# the field can never carry a hash, a page cursor, or a free-form reason. -PRISM_BLOCK_CANDIDATE_CLEANUP_BACKPRESSURE_CALLERS = ( - # The replay-adoption page walk (#183/#196). - "replay-page", - # The dequeue-time stale sibling skip (#181 item 2). - "dequeue", -) -# The same-hash disposition guard remembers every terminal outcome so a late -# duplicate -- queued, replayed, retried, parked, or quarantined -- joins the -# decision instead of re-offering a block qbitd already answered for. That -# memory was process-lifetime historical state: one testnet4 storm left -# 312,000 entries behind, and every decided-height collapse poll copied the -# whole registry under the global lock (~124 ms at that size, growing with -# every block the process ever disposed) while unrelated share acks waited. -# -# The registry is bounded instead, and eviction is oldest-first over the -# entries that no in-memory copy still needs: every lane that can hold a -# candidate object (or its still-owed terminal work) pins its hash and is -# never evicted, so a fenced hash cannot become offerable again by being -# forgotten. That includes the instant between lanes -- a candidate the -# submitter has taken out of a holder or queue but not yet handed to its -# disposition flight is named by the dequeued-hash pin, moved in and out -# under the same lock holds that empty and refill the lanes around it. The -# bound sits far above the number of copies the process can -# hold at once -- the bounded admission queue -# (MAX_PENDING_BLOCK_CANDIDATES), one replay enumeration batch -# (MAX_BLOCK_REPLAY_ENUMERATION_ROWS), and the single-slot retry holders -- -# so ordinary operation never even reaches an eviction. -MAX_BLOCK_CANDIDATE_TERMINAL_OUTCOMES = 8192 -# One insert examines at most this many of the oldest entries beyond the -# overflow it has to clear, so trimming stays O(1) amortized under the global -# lock even when a run of pinned hashes sits at the front of the registry. -# A pinned entry the scan passes is moved behind the window rather than -# dropped, so every pass makes progress against the next one. -BLOCK_CANDIDATE_TERMINAL_OUTCOME_EVICTION_SCAN = 128 -BLOCK_SUBMITTER_WAIT_HEARTBEAT_SLICE_SECONDS = 0.25 -MAX_BLOCK_SUBMITTER_STUCK_CALL_WORKERS = 2 -BLOCK_CANDIDATE_RETRY_HEARTBEAT_SLICE_SECONDS = 0.25 -BLOCK_CANDIDATE_INTENT_SCHEMA = "qbit.prism.block-candidate-intent.v1" -PRISM_STALE_JOB_ABANDON_CLASSES = ( - "tip_moved", - "balance_stale", - "append_epoch_stale", -) -# A would-be terminal abandonment refused because the candidate's own block -# hash is (or was recently observed as) part of the active chain: qbitd -# accepted the block even though this process has not completed the accepted -# success tail (for example when the direct submitblock ack was lost in -# transport and acceptance was learned from a blockwait tip observation). -# The candidate defers and retries until the tail finalizes it as submitted. -PRISM_REJECTION_BLOCK_ACCEPT_PENDING = "accepted-pending-finalization" -# The ledger reported the candidate's row terminally disposed (reorg -# quarantine, rejection, or reversal) before the confirmation landed. That is -# a routine race around a chain reorganization: terminal for this candidate, -# benign for the pool, so it must never escalate to a coordinator shutdown -# the way an unexplained confirmation failure does. -PRISM_REJECTION_LEDGER_CONFIRMATION_SUPERSEDED = "ledger-confirmation-superseded" -# Used only by lightweight embedders that bypass dataclass/coordinator -# construction. Production instances adopt their eagerly installed state at -# service construction. Serializing the fallback prevents concurrent -# first-touch callers from ever publishing different containers for the same -# state. -_STATE_BACKFILL_LOCK = threading.Lock() - -# -- decided-height candidate collapse (#183) -------------------------- -# Bounded, fixed-label outcome keys for the collapse selector/apply. They -# are the whole metric label space: no block hash, parent hash, or job ID -# ever becomes a Prometheus label. -PRISM_BLOCK_CANDIDATE_COLLAPSE_OUTCOMES = ( - # Durable pending rows the selector examined. - "considered", - # Rows preserved because their height was never probed: the walk had - # spent MAX_BLOCK_CANDIDATE_COLLAPSE_HEIGHT_PROBES distinct heights. - "height_deferred", - # Rows that satisfied every predicate clause. - "selected", - # Selected rows whose disposition lease was held elsewhere. - "lease_skipped", - # Leased rows the immediate pre-write re-read disqualified. - "revalidation_dropped", - # Qualified rows the fenced batch write did not transition. - "write_lost", - # Rows the fenced batch write actually transitioned. - "abandoned", - # Rows preserved because a page-scope read or write failed closed. - "fail_closed", - # Won rows whose post-write cleanup raised. - "cleanup_failed", - # Deferred cleanups a later bounded retry pass finished in full. - "cleanup_recovered", - # Retry passes that ran but still left at least one step owed. - "cleanup_retry_failed", - # -- issue #198: cleanup-retry backlog backpressure ----------------------- - # Rows that satisfied (or were never asked) predicate S and were - # preserved -- left durable and pending for the per-row path -- because - # the cleanup-retry backlog was at its configured admission bound. Shared - # by the replay-page walk and the dequeue-time skip like the outcomes - # above; the warning names the caller, the counter counts rows. - "backlog_deferred", - # -- issue #181 item 2: the dequeue-time stale sibling skip ------------- - # These three name the *population* the skip judged, one dequeued - # candidate at a time, and partition it exactly: - # dequeue_considered = dequeue_skipped + dequeue_preserved. The outcomes - # above stay shared with the replay-adoption page path, because they - # describe the same machinery (one selection, one lease set, one fenced - # write, one cleanup) whichever caller drove it. - # - # Dequeued candidates predicate S was actually asked about: the ledger - # can answer a fenced batch abandonment and a pool-block probe, and the - # candidate carries a readable hash and parent. - "dequeue_considered", - # Dequeued candidates terminalized before any node offer. - "dequeue_skipped", - # Dequeued candidates handed to the ordinary offer path, for any reason - # at all -- current work, offer evidence, an unprobed height, a lighter - # occupant, a lost fenced write, or a read that failed closed. - "dequeue_preserved", -) -# Issue #181 item 3: the interval from definitive node acceptance (the -# submitblock RPC returning None on the submitter thread) to the accepted -# block's payout preview becoming visible to waiting child work. Both exits -# of _publish_accepted_block_payout_preview publish -- the atomic generation -# publication, and issue #188's fenced local-retention branch, which installs -# the compact preview and notifies waiters without installing a generation -- -# so the label set separates them rather than merging a degraded publication -# into the healthy one. -PRISM_ACCEPTED_BLOCK_PREVIEW_PUBLICATION_RESULTS = ( - "published", - "degraded", -) -# The acceptance criterion is a p95 below the 5 s child wait budget -# (DEFAULT_ACCEPTED_BLOCK_PAYOUT_PREVIEW_WAIT_SECONDS), so the boundary at -# exactly 5.0 has to exist and the approach to it has to be resolved. The -# shared PRISM_JOB_BUILD_SECONDS_BUCKETS tuple is tuned for millisecond job -# builds and steps 1.0 -> 2.5 -> 5.0 -> 10.0 across the whole decision range; -# this dedicated tuple keeps that one unwidened while still extending well -# past the budget so a saturated tail is bucketed instead of folded into -# +Inf. -PRISM_ACCEPTED_BLOCK_PREVIEW_PUBLICATION_SECONDS_BUCKETS = ( - 0.01, - 0.05, - 0.1, - 0.25, - 0.5, - 1.0, - 1.5, - 2.0, - 3.0, - 4.0, - 5.0, - 7.5, - 10.0, - 20.0, - 30.0, - 60.0, -) -# Acceptance stamps are keyed by block hash and consumed by the first -# publication observed for that hash. A hash that never reaches publication -# -- a terminal accounting failure, a withdrawn acceptance -- would otherwise -# retain its stamp forever, so the oldest entries are evicted past this cap. -# Max-block admission and the physical block rate keep the live population -# far below it; this is a leak bound, not a working set. -MAX_ACCEPTED_BLOCK_PREVIEW_PUBLICATION_STAMPS = 512 -# Issue #224: the event name of the one bounded structured line each closed -# acceptance-to-publication interval emits. It is the only place in this -# lane's output where a block hash and height appear alongside the timings; -# the phase metrics carry the fixed phase label and nothing else. -PRISM_ACCEPTED_LANDING_LOG_EVENT = "accepted_block_preview_landing" -# The terminal cleanup one won row owes, in the order the apply runs it. -# A deferred retry replays exactly the steps its own hash still owes, so -# the step names are a closed set: a retry can never repeat a step that -# already completed, and never invent one the apply does not run. -BLOCK_CANDIDATE_COLLAPSE_CLEANUP_PAYOUT_STEP = "payout-preview-withdrawal" -BLOCK_CANDIDATE_COLLAPSE_CLEANUP_FLOOR_STEP = "pending-share-floor" -BLOCK_CANDIDATE_COLLAPSE_CLEANUP_STEPS = ( - BLOCK_CANDIDATE_COLLAPSE_CLEANUP_PAYOUT_STEP, - "finalize-retry", - "retry-state", - "outstanding-and-tip-observation", - "terminal-outcome", - "abandonment-accounting", - BLOCK_CANDIDATE_COLLAPSE_CLEANUP_FLOOR_STEP, -) -# The per-row path abandons a candidate whose height was decided by another -# block as a tip-moved stale job; the bulk path must land in the same -# bounded reason/class buckets or the abandonment series splits in two. -# ``share_submission`` imports this module, so its PRISM_REJECTION_STALE_JOB -# is restated here rather than imported; a collapse test pins them equal. -PRISM_BLOCK_CANDIDATE_COLLAPSE_REASON = "stale-job" -PRISM_BLOCK_CANDIDATE_COLLAPSE_STALE_JOB_CLASS = "tip_moved" -# A storm collapses thousands of rows in one apply. Logs stay bounded: at -# most this many parent/job/reason groups are printed, each with at most -# this many sample hashes, plus one remainder line. -BLOCK_CANDIDATE_COLLAPSE_LOG_GROUPS = 8 -BLOCK_CANDIDATE_COLLAPSE_LOG_SAMPLE_HASHES = 3 -# A systemic cleanup fault would otherwise print one line and one -# traceback per row of a storm-sized page; only the first few are -# detailed and the rest are summarized by count. -BLOCK_CANDIDATE_COLLAPSE_LOG_FAILURES = 3 -# A ledger or node that cannot answer the page-scope reads fails closed on -# every poll. The counter still moves per row; the log line is rate limited -# so a persistently degraded read cannot flood the journal. -BLOCK_CANDIDATE_COLLAPSE_FAIL_CLOSED_LOG_SECONDS = 60.0 -# Chain-height probes one replay walk may spend on collapse, counted in -# *distinct heights* rather than rows. -# -# The storm this path exists for is one decided height behind thousands of -# rows, and it costs a single probe no matter how wide the page is. A page -# whose rows span many heights is the opposite shape: selection would read -# ``getblockhash`` (and then the occupant's ``getblockheader``) once per -# distinct height, sequentially, on the sole block-submitter thread. A page -# holds up to MAX_BLOCK_REPLAY_ENUMERATION_ROWS rows, so that is up to 1,024 -# heights and ~2,048 round trips; at a production RPC deadline the thread -# would sit in collapse for hours while a live solve waited behind it for -# submitblock. The bound makes the chain cost of a walk a constant -- -# at most this many ``getblockhash`` reads and at most one occupant header -# each -- independent of how many rows or heights the backlog carries. -# -# Rows at a height the walk did not probe are simply never selected. They -# stay durable, keep every piece of evidence they had, and are adopted by -# the ordinary per-row disposition path, which disposed of them exactly this -# way before collapse existed; a later walk starts with a fresh budget and -# may probe their heights. Collapse is an optimisation over that path, so -# spending less of it is always safe and never terminalizes anything. -MAX_BLOCK_CANDIDATE_COLLAPSE_HEIGHT_PROBES = 8 -# PRISM records every network difficulty in scaled integer units: the -# coordinator stamps a candidate's ``found_block.network_difficulty`` with -# ``template_artifacts.scaled_network_difficulty(template["bits"])``, which is -# ``pow_limit_target * 1_000_000 // template_target`` -- roughly the raw -# Bitcoin-style difficulty multiplied by a million. The collapse selector -# compares that stored value against the occupying block's work, so the -# occupant has to be converted into the same units from the same inputs. -# These restate the scale and the qbit powLimit rather than importing them -# from ``template_artifacts`` (or ``public_api``): both sit above this module -# in the import graph, and B1 must not grow an edge to the template/payout -# layer for two constants. ``_collapse_scaled_difficulty`` reapplies the -# formula, and a collapse test pins it equal to the production helper. -COLLAPSE_DIFFICULTY_SCALE = 1_000_000 -COLLAPSE_POW_LIMIT_BITS = "207fffff" -COLLAPSE_POW_LIMIT_TARGET = direct_stratum.target_from_compact_hex( - COLLAPSE_POW_LIMIT_BITS -) - - -def _pending_rows_accepts_cursor(pending_rows: Callable[..., Any]) -> bool: - """Report whether a ledger's pending-row reader can paginate. - - Signature introspection runs first because the alternative — probing - with the keyword and catching TypeError — spends a bounded ledger - worker and records a landing-class ledger-call sample on every startup - against a ledger that never supported cursors. The probe still runs - (and still falls back) whenever a callable cannot be introspected; this - only skips it where the answer is already knowable. - """ - try: - parameters = inspect.signature(pending_rows).parameters.values() - except (TypeError, ValueError): - return True - return any( - parameter.name == "after_cursor" - or parameter.kind is inspect.Parameter.VAR_KEYWORD - for parameter in parameters - ) - - -def _block_replay_cursor_key(after_cursor: object | None) -> str | None: - """Fold one opaque enumeration cursor into a hashable dedupe-key part. - - The cursor is a JSON-safe list, so it cannot go into a tuple key as-is. - Only identity matters here: two calls share an in-flight ledger worker - exactly when they are the same query. - """ - if after_cursor is None: - return None - return json.dumps(after_cursor, separators=(",", ":"), default=repr) - - -def _collapse_block_hash(value: object) -> str | None: - """Normalize one block hash, or report it unusable. - - Every collapse comparison is between hashes from three unrelated - sources -- a durable intent, a durable row key, and a qbitd RPC -- so - they are folded to one canonical spelling here. Anything that is not a - 64-character hex string is unusable rather than merely differently - spelled, and its row is never selected. - """ - if not isinstance(value, str): - return None - key = value.strip().lower() - if len(key) != 64: - return None - try: - int(key, 16) - except ValueError: - return None - return key - - -def _block_candidate_hash_of(candidate: object) -> str | None: - """The canonical block hash one candidate object carries, if any.""" - return _collapse_block_hash( - getattr( - getattr(candidate, "submission", None), - "block_hash_hex", - None, - ) - ) - - -def _collapse_height(value: object) -> int | None: - """Parse one block height, rejecting booleans and inexact numbers. - - ``bool`` is an ``int`` subclass, so a stray ``True`` would otherwise - parse as height 1 and compare against a real tip height. - """ - if isinstance(value, bool) or value is None: - return None - if isinstance(value, int): - return int(value) - if isinstance(value, float): - if not math.isfinite(value) or value != int(value): - return None - return int(value) - if isinstance(value, str): - try: - return int(value.strip(), 10) - except ValueError: - return None - return None - - -def _collapse_row_height(durable_row: object) -> int | None: - """The height one durable page row claims, or None when it states none. - - The single place that knows where a fetched row records its height, so - the cheap pre-selection peek in ``_collapse_superseded_block_candidates`` - and the decode in ``_superseded_candidate_row`` can never drift apart on - it. This is only a peek: it validates nothing beyond the parse, and every - fact it reads is re-read and cross-checked against the row's template by - the decode, which is what actually decides the row. - """ - if not isinstance(durable_row, dict): - return None - intent = durable_row.get("candidate") - if not isinstance(intent, Mapping): - return None - return _collapse_height(intent.get("expected_height")) - - -def _collapse_difficulty(value: object) -> int | float | None: - """Parse one stored work value, rejecting booleans and non-finite numbers. - - A NaN compares false against every threshold, so an unparsed NaN would - silently answer "not enough work" (safe) or, inverted, "enough work" - (not safe). Rejecting it outright keeps the clause decidable. - - An integral spelling is kept as an ``int`` rather than widened to - ``float``. PRISM stores ``found_block.network_difficulty`` as the scaled - *integer* ``scaled_network_difficulty`` returns, and above 2**53 a float - conversion rounds it -- often upwards, which would make a candidate look - strictly heavier than the identical work the chain reports and defeat - exactly the equal-work case clause 4b/5 has to accept. Python compares - ``int`` and ``float`` exactly, so a non-integral spelling stays usable - without dragging the integral ones through binary floating point. - """ - if isinstance(value, bool) or value is None: - return None - number: int | float - if isinstance(value, int): - number = value - elif isinstance(value, float): - number = value - elif isinstance(value, str): - text = value.strip() - try: - number = int(text, 10) - except ValueError: - try: - number = float(text) - except (OverflowError, ValueError): - return None - else: - return None - if isinstance(number, float) and not math.isfinite(number): - return None - if number <= 0: - return None - return number - - -def _collapse_scaled_difficulty(bits: object) -> int | None: - """Convert one header's compact ``bits`` to PRISM-scaled difficulty units. - - This is deliberately the same integer formula - ``template_artifacts.scaled_network_difficulty`` applies to the - template's bits when the coordinator stamps a candidate's - ``found_block.network_difficulty`` -- recomputed here from - ``direct_stratum`` rather than imported, because this module must not - grow an edge to the template/payout layer. Both sides of clause 4b/5 - therefore land on the identical integer for identical work, with no - float rounding at the equality boundary the clause turns on. - - Returns ``None`` for anything that is not an 8-character compact hex - string with a positive target, so a header missing ``bits``, or - carrying a malformed or zero one, fails its page closed instead of - being read as "no work". - """ - if not isinstance(bits, str): - return None - compact = bits.strip().lower() - if len(compact) != 8 or any(char not in "0123456789abcdef" for char in compact): - return None - try: - target = direct_stratum.target_from_compact_hex(compact) - except ValueError: - return None - if target <= 0: - return None - return max( - 1, - (COLLAPSE_POW_LIMIT_TARGET * COLLAPSE_DIFFICULTY_SCALE) // target, - ) - - -class _BlockCandidateCollapseFailedClosed(Exception): - """A page-scope chain read was unavailable or proved nothing. - - Raised, not returned, because every caller's answer is the same: this - page selects nothing and every one of its rows is preserved for the - ordinary per-row path. - """ - - -@dataclass -class _CollapsedCandidateCleanup: - """One terminal hash's outstanding cleanup, kept for a later retry. - - After the fenced batch write the durable row is terminal, so the row is - partitioned out of the replay page and no later enumeration will ever - return it: a cleanup step that failed has no durable replay source and - would otherwise leave its in-memory state -- a payout preview or its - tombstone, a pending-share floor holder, an outstanding-hash marker -- - installed for the process lifetime. This record is that replay source. - - It holds no candidate object, no durable row, and no node submission, - so a deferred cleanup can only finish tearing state down; there is - nothing here to re-adopt or re-offer with. ``shares`` carries the exact - pending-share floor holders the apply resolved, because the floor keys - holders by object identity and the queues they were read from are - drained long before a retry runs. ``shares_resolved`` is false only - when the apply aborted before it could index them, in which case the - retry re-runs the (bounded, in-memory) scan itself. - """ - - block_hash: str - steps: frozenset[str] - shares: tuple[PendingShare, ...] = () - shares_resolved: bool = True - attempts: int = 0 - delay_seconds: float = 0.0 - not_before_monotonic: float = 0.0 - # When this hash's cleanup was first deferred. Kept across merges and - # reschedules so the oldest-age gauge measures how long the row has - # owed cleanup, not how long since its last failed attempt. - deferred_monotonic: float = 0.0 - - -@dataclass(frozen=True) -class _SupersededCandidateRow: - """One durable pending row reduced to the facts predicate S reads.""" - - block_hash: str - parent_hash: str - height: int - # PRISM-scaled units (see COLLAPSE_DIFFICULTY_SCALE), as stamped by the - # coordinator. Comparable only against a chain-side work value converted - # into the same units by ``_collapse_scaled_difficulty``. - network_difficulty: int | float - job_id: str - - -@dataclass -class _CollapseHeightProbeBudget: - """Distinct chain heights a collapse walk may still read, and what it read. - - One budget is created per replay walk and shared by that walk's pages, - so the walk's chain cost is the bound whether the backlog arrives as one - page or fifty. A pass handed no budget (a direct call, or the legacy - single-page shape) gets a fresh full one, which is the same ceiling - applied to a walk of one. - - Only heights are charged. The occupant header a probed height leads to is - read at most once per probed height, so charging the height bounds both - round trips at once and keeps the accounting to the single number the - comment on MAX_BLOCK_CANDIDATE_COLLAPSE_HEIGHT_PROBES explains. - - The two caches live here, beside the counter, for the same reason the - counter does: a walk that has paid for a height must not pay for it - again on its next page. Each page builds a fresh - ``_BlockCandidateChainView`` over this one budget, and a view that - cached its reads only for itself would recharge the walk for the same - height on every page -- so the one decided height behind a storm would - exhaust the budget after eight pages and defer the entire remainder of - the backlog. ``difficulty`` is keyed by block hash, whose work is - immutable, so sharing it can only remove round trips. Sharing ``active`` - widens the window between a selection read and the write it feeds, which - is precisely the window the pre-write revalidation exists to close: that - pass builds its own budget, so its caches -- and therefore its chain - reads -- stay fresh and independent of this walk's. - """ - - remaining: int = MAX_BLOCK_CANDIDATE_COLLAPSE_HEIGHT_PROBES - # Height -> the active-chain block this walk read there. - active: dict[int, str] = field(default_factory=dict) - # Block hash -> its PRISM-scaled work, re-derived from its header bits. - difficulty: dict[str, int] = field(default_factory=dict) - - @property - def exhausted(self) -> bool: - return self.remaining <= 0 - - def spend(self) -> bool: - """Claim one height probe; False when the walk has none left.""" - if self.remaining <= 0: - return False - self.remaining -= 1 - return True - - -class _BlockCandidateChainView: - """Bounded chain reads for one collapse pass. - - The best tip is read once for the whole page and the active block at a - height once per distinct candidate height, so a 3,120-row storm at one - decided height costs a fixed handful of RPCs instead of one round trip - per row. Every miss or failure is a fail-closed page, never a guess. - - A height-diverse page cannot turn "once per distinct height" into a walk - of the node as long as the page: ``probe_budget`` caps the distinct - heights this view will ever read, and a height past the cap is reported - as unprobed rather than guessed at. - - "Once per distinct height" is a property of the *walk*, not of this - view. The caches are the budget's, so a page-two view over the same - budget already knows every height page one probed: a storm at one - decided height costs the walk one ``getblockhash`` and one - ``getblockheader`` however many pages it spans. Only the page-scope tip - reads are per view, which is what keeps each page's selection running - against a freshly read tip. - """ - - def __init__( - self, - service: "BlockCandidateService", - *, - probe_budget: _CollapseHeightProbeBudget | None = None, - ) -> None: - self._service = service - self._probe_budget = ( - probe_budget - if probe_budget is not None - else _CollapseHeightProbeBudget() - ) - self._active: dict[int, str] = self._probe_budget.active - self._difficulty: dict[str, int] = self._probe_budget.difficulty - # Per view, never on the budget: the tip is the one fact a view must - # not inherit from an earlier page, and keeping the memo here is what - # preserves "each page's selection runs against a freshly read tip" - # while letting one page read it once. Both reads were already made - # at most once per view by the page selector and the pre-write - # revalidation, so memoizing changes no existing caller's round - # trips; it exists so a caller that needs the tip *before* handing - # this view to selection -- the dequeue-time skip, which gates on - # clause 3 before it spends a durable read -- does not pay for a - # second one. - self._tip: str | None = None - self._tip_height: int | None = None - - def _call(self, method: str, params: list[object] | None = None) -> object: - coordinator = self._service._coordinator - coordinator._record_block_submitter_phase(f"replay-collapse-{method}") - try: - if params is None: - return coordinator.rpc.call(method) - return coordinator.rpc.call(method, params) - except Exception as exc: - raise _BlockCandidateCollapseFailedClosed( - f"{method} failed during candidate collapse: {exc}" - ) from exc - finally: - coordinator._record_block_submitter_phase( - f"replay-collapse-{method}:complete" - ) - - def best_tip(self) -> str: - """The chain's best tip, read once for the life of this view.""" - if self._tip is not None: - return self._tip - tip = _collapse_block_hash(self._call("getbestblockhash")) - if tip is None: - raise _BlockCandidateCollapseFailedClosed( - "best tip hash is not a block hash" - ) - self._tip = tip - return tip - - def tip_height(self) -> int: - """The best tip's height, read once for the life of this view.""" - if self._tip_height is not None: - return self._tip_height - height = _collapse_height(self._call("getblockcount")) - if height is None or height < 0: - raise _BlockCandidateCollapseFailedClosed( - "best tip height is not an integer" - ) - self._tip_height = height - return height - - def active_at(self, height: int) -> str | None: - """The active-chain block occupying ``height``, read once per height. - - ``None`` means this walk has spent its height-probe budget and the - height was never read. It is not "no occupant" and it is not a - fail-closed read: it says only that the chain was never asked, so - every caller must preserve the row rather than decide anything about - it. A height this walk has already read is free -- on this page and - on every later page of the same walk -- and stays answerable after - the budget is gone. - """ - cached = self._active.get(height) - if cached is not None: - return cached - if not self._probe_budget.spend(): - return None - active = _collapse_block_hash(self._call("getblockhash", [int(height)])) - if active is None: - raise _BlockCandidateCollapseFailedClosed( - f"active block at height {height} is not a block hash" - ) - self._active[height] = active - return active - - def cached_difficulty(self, block_hash: str) -> int | None: - """The work this walk already read for a hash, without a new call.""" - return self._difficulty.get(block_hash) - - def difficulty_of(self, block_hash: str) -> int: - """One occupying block's work, in PRISM-scaled units, once per hash. - - Read from the header's compact ``bits``, not its ``difficulty`` - field. The candidate row this value is compared against carries - ``scaled_network_difficulty(template["bits"])`` -- the raw - difficulty times COLLAPSE_DIFFICULTY_SCALE -- while - ``getblockheader.difficulty`` is the raw float. Comparing the two - directly made an equal-work occupant read as a millionth of the - candidate's work, so clause 4b/5 rejected it and a decided height - never collapsed. Recomputing from bits with the production formula - puts both sides on the same integer scale, and keeping it integral - means the equal-work boundary the clause turns on is decided - exactly rather than by float comparison. - """ - cached = self._difficulty.get(block_hash) - if cached is not None: - return cached - header = self._call("getblockheader", [block_hash]) - if not isinstance(header, dict): - raise _BlockCandidateCollapseFailedClosed( - f"header for {block_hash} is not an object" - ) - difficulty = _collapse_scaled_difficulty(header.get("bits")) - if difficulty is None: - raise _BlockCandidateCollapseFailedClosed( - f"header for {block_hash} carries no usable compact bits" - ) - self._difficulty[block_hash] = difficulty - return difficulty - - -@dataclass -class _AcceptedLandingAttribution: - """One landing's in-flight phase attribution, keyed by block hash. - - Issue #224. A landing's sub-phases are measured on the accounting and - block-work threads, while the interval they belong to is owned by the - acceptance stamp taken on the submitter thread; the block hash is the only - identity both sides share, exactly as it is for the stamp itself. - - ``phase_seconds`` accumulates only while that hash's interval is open, so - a stretch that ran before definitive acceptance -- the fallback landing - that submits inside the balance serializer reaches its reconcile and - chain-probe stretches before the offer resolves -- and a retry's stretch - after the interval already closed both land in the process-wide phase - family without ever being claimed by a landing's diagnostic. - """ - - block_height: int | None = None - phase_seconds: dict[str, float] = field(default_factory=dict) - lane_wait_recorded: bool = False - publish_started_monotonic: float | None = None - - -def _accepted_landing_log(event: str, **fields: object) -> None: - """Single-line JSON lifecycle log, the payout-artifact line's shape.""" - print( - "prism coordinator: " - + json.dumps({"event": event, **fields}, sort_keys=True), - flush=True, - ) - - -def _accepted_landing_candidate_identity( - candidate: Any, -) -> tuple[str | None, int | None]: - """This candidate's hash and height, or ``None`` when unreadable. - - Attribution is observability: a candidate shape this cannot read must - never turn the start of a landing into an exception. - """ - try: - block_hash = str(candidate.submission.block_hash_hex).lower() - except Exception: - return (None, None) - if not block_hash: - return (None, None) - try: - block_height: int | None = int(candidate.context.template["height"]) - except Exception: - block_height = None - return (block_hash, block_height) - - -@dataclass(frozen=True) -class PrismBlockCandidate: - """A block-worthy submission queued for the block-submitter thread. - - A share that met its target is acknowledged and credited on the client - thread, then queued here for the submitter to land the block off the hot - path. When the hash solved the block but missed the share target (floor - above network difficulty), credit_share_on_accept is set and the candidate - is instead submitted synchronously by handle_submit: that share is valid - only if the block lands, so its credit and the miner's accept/reject follow - the block outcome directly rather than being queued. - """ - - context: PrismJobContext - submission: direct_stratum.DirectQbitSubmission - extranonce1_hex: str - extranonce2_hex: str - pending_share: PendingShare - client: ClientState - credit_share_on_accept: bool = False - durable_replay: bool = False - # When this in-process attempt became runnable: live candidates stamp - # share-accept time, durable outbox replays stamp row-restore time. The - # block-submit histogram measures from here to submitblock's return -- - # the race-critical interval a lost block round is decided in. - landed_monotonic: float = field(default_factory=time.monotonic) - - -@dataclass -class _BlockCandidateDispositionFlight: - """One same-hash submission guard shared by its holder and waiters.""" - - # The node-offer thread acquires this guard and the accounting thread - # releases it after durable finalization. A plain Lock permits that - # deliberate ownership transfer; RLock does not. - lock: threading.Lock = field(default_factory=threading.Lock) - users: int = 0 - - -@dataclass(frozen=True) -class _BlockCandidateDispositionLease: - """A same-hash guard held across node offer and durable finalization.""" - - block_hash: str - flight: _BlockCandidateDispositionFlight - - -@dataclass(frozen=True) -class _BlockCandidateNodeSubmission: - """Result of the latency-critical qbitd fast-lane call.""" - - attempted: bool - result: object = None - error: BaseException | None = None - - -def _is_definitive_node_acceptance( - node_submission: _BlockCandidateNodeSubmission | None, -) -> bool: - """Report whether an offer is a *definitive* node acceptance. - - qbitd's ``submitblock`` returns JSON ``null`` for a block it accepted and - a rejection string ("duplicate", "inconclusive", a validation reason) for - anything else, so the only shape that names an accepted block is an - attempted offer that returned no result and raised nothing. - - All three clauses are load-bearing, and ``error is None`` is the one that - is easy to drop by accident: the transport failure path in - :meth:`_submit_block_candidate_to_node` builds - ``_BlockCandidateNodeSubmission(attempted=True, error=exc)``, which leaves - ``result`` at its ``None`` default. A two-clause test would therefore read - a *failed* offer as an acceptance -- the single worst misclassification - available here, because every consumer of this predicate treats a true - answer as "the node already has this block". - - Consumers must share this one definition rather than re-spelling it: - :meth:`_stash_retained_block_candidate_node_submission` decides whether an - offer may be replayed from memory, and the accounting handoff decides - which lane a task joins. Those two answers must never diverge. - """ - if node_submission is None: - return False - return ( - bool(node_submission.attempted) - and node_submission.error is None - and node_submission.result is None - ) - - -@dataclass -class _BlockSubmitterLedgerCall: - """One still-running direct outbox call, reused across paced retries.""" - - started_monotonic: float = field(default_factory=time.monotonic) - done: threading.Event = field(default_factory=threading.Event) - result: object = None - error: BaseException | None = None - - -@dataclass -class _BlockSubmitterRpcCall: - """One hard-deadline, single-flight submitblock transport call.""" - - started_monotonic: float = field(default_factory=time.monotonic) - done: threading.Event = field(default_factory=threading.Event) - result: object = None - error: BaseException | None = None - - -@dataclass(frozen=True) -class _BlockCandidateAccountingTask: - """A node-offered candidate awaiting serialized durable accounting.""" - - candidate: PrismBlockCandidate - node_submission: _BlockCandidateNodeSubmission - disposition_lease: _BlockCandidateDispositionLease - # Set by the accounting lane's own exception path once it has retained - # this task's candidate for retry while still holding the disposition. - # The loop's catch reads it so a candidate the lane retained is not - # retained (and counted) a second time, while a failure that fired - # before the lane could retain -- a delegate raising ahead of it -- is - # still retained there. - retained_for_retry: threading.Event = field( - default_factory=threading.Event, - compare=False, - repr=False, - ) - - -class BlockSubmitterDatabaseTimeout(TimeoutError): - """A submitter ledger phase exceeded its coordinator-side deadline.""" - - -@dataclass(frozen=True) -class BlockCandidateAttemptResult: - """Structured result of the landing callback used by retry/terminalization.""" - - accepted: bool - reason: str | None - error: str - - def retryable(self, retryable_reasons: frozenset[str]) -> bool: - return not self.accepted and ( - self.reason is None or self.reason in retryable_reasons - ) - - -@dataclass(frozen=True) -class BlockCandidateRunResult: - """Result returned after one in-memory wakeup or retry slot is consumed.""" - - ran: bool - refresh_client: Any | None = None - - -class BlockCandidateSubmitPort(Protocol): - """Land one candidate through the coordinator's submit entrypoint. - - The landing tail has two forms and only the coordinator can tell them - apart: embedders and tests replace the bound ``submit_block_candidate`` - to stand in for the node submission, and that replacement is installed on - the instance *after* this service is constructed. The port therefore - carries the whole landing call shape -- an already-created - ``node_submission`` when the caller made one, and whether the caller - already owns the same-hash disposition -- and lets the coordinator resolve - the entrypoint per call, instead of the service inspecting it through the - runtime seam. - """ - - def __call__( - self, - candidate: PrismBlockCandidate, - *, - node_submission: _BlockCandidateNodeSubmission | None = None, - disposition_held: bool = False, - ) -> bool: ... - - -@dataclass(frozen=True) -class BlockCandidatePorts: - """Callback seams back into the coordinator, resolved at use time. - - ``runtime`` is the owning coordinator; service bodies reach every - cross-owner concern (and every monkeypatch-sensitive sibling delegate) - through it at call time, per the in-tree Runtime seam convention. The - named callables preserve the historical narrow port surface for focused - embedders and the layer-original B1 ownership tests. - """ - - runtime: Any - ledger: Callable[[], Any] - stop_event: Callable[[], threading.Event] - writer_operation: Callable[[str], AbstractContextManager[object]] - submit_candidate: BlockCandidateSubmitPort - reject_terminal_prepared: Callable[[PrismBlockCandidate], None] - begin_preview: Callable[[str, int], None] - clear_preview: Callable[[str, bool], None] - share_writer: Callable[[], Any] - finish_pending_candidate: Callable[[PendingShare], None] - refresh_after_accept: Callable[[Any], None] - record_heartbeat: Callable[[str], None] - replay_entrypoint: Callable[[], int] - submit_next_entrypoint: Callable[[float | None], bool] - next_retry_delay: Callable[[str], float] - log: Callable[[str], None] - - -def _candidate_block_hex(candidate: PrismBlockCandidate) -> Any: - """Return the candidate's serialized block, never an unmaterialized one. - - ``assemble_submission`` materializes ``block_hex`` only for a hash that - solved the block, which every candidate route requires, so a real - submission always carries the bytes by the time a candidate exists. If one - ever does not, the cause is a defect in that coupling and the block is - already lost -- but an empty value written to the durable intent loses it - twice, resurrecting after every restart as a candidate that can never be - offered to the node. Fail on the client thread instead, before the intent - is written. Duck-typed embedders that retain no block bytes at all keep - the historical empty-string behaviour: the invariant is a property of the - real submission type, and the landing path already guards them separately. - - The guard deliberately does not apply to a durable replay. A replayed - candidate's intent already exists, and ``block_candidate_from_intent`` - rebuilds every one of them as a real ``DirectQbitSubmission`` -- so an - embedder's tolerated empty value comes back wearing the strict type. The - only caller that re-encodes a replayed candidate is the credit-on-accept - append, which runs *after* the block has landed; raising there would fail - a credit for a block already in the chain, which is strictly worse than - re-persisting the same empty value the row already holds. Re-persisting an - existing intent is not the write this guard exists to stop. - - A hydrated replay can carry the block as a streamed ``SpoolStringView`` - (a raw block of roughly 512 KiB or more). It is returned as is: - ``str()`` refuses it, and the codec writes its stored encoding verbatim, - byte-identical to the plain string (#255). - """ - submission = candidate.submission - block_hex = getattr(submission, "block_hex", "") - if callable(getattr(block_hex, "iter_encoded_chunks", None)): - return block_hex - block_hex = str(block_hex) - if ( - not block_hex - and not candidate.durable_replay - and isinstance(submission, direct_stratum.DirectQbitSubmission) - ): - raise ValueError("block candidate submission carries no serialized block") - return block_hex - - -def block_candidate_intent(candidate: PrismBlockCandidate) -> PreparedCandidateIntent: - """Return the immutable intent needed to resume a candidate after restart. - - Issue #255. The result is a :class:`PreparedCandidateIntent`: a mapping - with the historical keys, whose ``shares_json`` is the job's immutable - share sequence itself (a page-backed window, a daemon mirror or a plain - list) rather than a list copy, and whose body streams the v1 identity - JSON in bounded chunks. Preparation validates every field synchronously - on the calling thread -- the client thread, before any share credit -- - exactly where the historical whole-document ``json.dumps`` did, but in - bounded C calls, and it refuses what PostgreSQL's jsonb parser used to - refuse (NaN/Infinity, ``\\u0000``, unpaired surrogates). - """ - context = candidate.context - submission = candidate.submission - intent = { - "schema": BLOCK_CANDIDATE_INTENT_SCHEMA, - "block_hash_hex": str(submission.block_hash_hex).lower(), - "block_hex": _candidate_block_hex(candidate), - "coinbase_tx_hex": str(getattr(submission, "coinbase_tx_hex", "")), - "parent_hash": str(context.template["previousblockhash"]).lower(), - "expected_height": int(context.template["height"]), - "template": { - "previousblockhash": context.template["previousblockhash"], - "height": int(context.template["height"]), - "coinbasevalue": int(context.template["coinbasevalue"]), - }, - # The immutable source sequence, never ``list(...)``: the codec - # streams it (verbatim page bytes where the sequence carries them). - "shares_json": context.shares_json, - "prior_balances": context.prior_balances, - "found_block": context.found_block, - "prospective_prior_balances": getattr(context, "prospective_prior_balances", None), - "witness_merkle_leaves_hex": direct_stratum.witness_merkle_leaves_hex( - getattr(context.job, "transaction_hexes", ()) - ), - "extranonce1_hex": candidate.extranonce1_hex, - "extranonce2_hex": candidate.extranonce2_hex, - "username": context.worker.username, - "pending_share": dataclasses.asdict(candidate.pending_share), - "credit_share_on_accept": candidate.credit_share_on_accept, - "collection_only": bool(context.collection_only), - } - # Fail on the client thread before committing a share if a field carries - # a value that cannot survive the durable boundary. One bounded encode - # pass replaces the historical whole-document ``json.dumps``; the bytes - # are discarded and only the manifest (digest, chunk digests, share - # span) is kept until storage asks for the chunks. - return prepare_candidate_intent(intent) - - -def _replayable_share_sequence(shares: Any) -> Any: - """The share sequence a decoded intent adopts, without a list copy. - - A plain list or tuple is adopted as is (the durable decode owns it), and - any other sequence -- a page-backed window, a daemon mirror, a spool - sequence -- is adopted as the immutable, replayable object it already - is. Anything that is not a sequence at all fails exactly as ``list()`` - used to. - """ - if isinstance(shares, (str, bytes)) or not isinstance(shares, Sequence): - raise TypeError("block candidate intent shares_json is not a sequence") - return shares - - -def _collapse_row_from_header(row: Mapping[str, Any]) -> dict[str, Any]: - """Shape one metadata-first header row as the page row predicate S reads. - - The collapse selector (``_superseded_candidate_row``) validates a page - row's ``candidate`` facts strictly and fails its page closed on any - malformed one; the header carries exactly those facts as an explicit - projection, so they are restated here under the same keys and nothing - else. The header row itself is preserved on the result so descriptor - registration reads the same object the collapse decided on. - """ - header = row.get("header") - if not isinstance(header, Mapping): - header = {} - template = header.get("template") - found_block = header.get("found_block") - pending_share = header.get("pending_share") - return { - **row, - "candidate": { - "block_hash_hex": header.get("block_hash_hex"), - "parent_hash": header.get("parent_hash"), - "expected_height": header.get("expected_height"), - "template": ( - { - "previousblockhash": template.get("previousblockhash"), - "height": template.get("height"), - } - if isinstance(template, Mapping) - else template - ), - "found_block": ( - {"network_difficulty": found_block.get("network_difficulty")} - if isinstance(found_block, Mapping) - else found_block - ), - "pending_share": ( - {"job_id": pending_share.get("job_id", "")} - if isinstance(pending_share, Mapping) - else pending_share - ), - }, - } - - -def _dequeued_candidate_collapse_row( - candidate: PrismBlockCandidate, - *, - pool_block_exists: bool, -) -> dict[str, Any]: - """Shape one dequeued candidate as the durable page row predicate S reads. - - The dequeue-time skip (issue #181 item 2) judges an in-memory candidate, - not a fetched outbox row, so it has to present one. It deliberately does - *not* call :func:`block_candidate_intent` to do that: that function - re-serializes the whole block, recomputes every witness merkle leaf, and - round-trips the result through ``json.dumps`` to validate it -- work - proportional to the block, paid per dequeued sibling, to read four - scalars. This builds only the fields ``_superseded_candidate_row`` - actually decodes. - - Every value here is copied from the same place the durable intent copies - it from, so the row this returns and the row the outbox holds for the - same candidate carry identical facts; a regression pins that field by - field rather than trusting the restatement. No predicate input is - coerced or defaulted: a candidate whose context cannot answer one of - them raises, and the caller fails that candidate closed onto the - ordinary offer path. ``job_id`` is the one exception, because it is not - a predicate input at all -- it only groups the collapse log line. - """ - context = candidate.context - template = context.template - return { - "block_hash": str(candidate.submission.block_hash_hex).lower(), - "candidate": { - "block_hash_hex": str(candidate.submission.block_hash_hex).lower(), - "parent_hash": str(template["previousblockhash"]).lower(), - "expected_height": int(template["height"]), - "template": { - "previousblockhash": template["previousblockhash"], - "height": int(template["height"]), - }, - "found_block": context.found_block, - "pending_share": { - "job_id": getattr(candidate.pending_share, "job_id", ""), - }, - }, - # Read durably by the caller, never assumed: clause 2 is the fact - # that keeps an offered, landed candidate out of a terminal set. - "pool_block_exists": bool(pool_block_exists), - } - - -def _joined_intent_text(value: Any) -> str: - """A hydrated string fact the consumer needs whole, joined explicitly. - - Only the fixed-metadata fields are always decoded as ``str``; any other - string above the view threshold stays a streamed ``SpoolStringView``, - which refuses ``str()`` (#255). Everything else keeps ``str()``. - """ - if callable(getattr(value, "iter_text_chunks", None)): - return "".join(value.iter_text_chunks()) - return str(value) - - -def block_candidate_from_intent(intent: Mapping[str, Any]) -> PrismBlockCandidate: - """Decode and validate a durable candidate intent without side effects. - - Accepts the historical dict and the prepared/hydrated mapping alike. - ``shares_json`` is adopted as the sequence the intent carries -- a spool - sequence decodes page by page on demand -- and is never copied into a - second list (#255). - """ - if not isinstance(intent, Mapping): - raise TypeError("block candidate intent must be an object") - if intent.get("schema") != BLOCK_CANDIDATE_INTENT_SCHEMA: - raise ValueError("unsupported block candidate intent schema") - block_hash = str(intent["block_hash_hex"]).lower() - # Not fixed metadata: a username with a JSON encoding over 1 MiB is - # hydrated as a streamed view, and the worker identity needs it whole. - username = _joined_intent_text(intent["username"]) - template = dict(intent["template"]) - if str(template.get("previousblockhash", "")).lower() != str(intent["parent_hash"]).lower(): - raise ValueError("block candidate parent hash does not match template") - if int(template.get("height", -1)) != int(intent["expected_height"]): - raise ValueError("block candidate height does not match template") - submission = direct_stratum.DirectQbitSubmission( - coinbase_tx_hex=str(intent["coinbase_tx_hex"]), - coinbase_txid_preimage_hex="", - header_hex="", - block_hex=(intent["block_hex"] if callable(getattr(intent["block_hex"], "iter_encoded_chunks", None)) - else str(intent["block_hex"])), - block_hash_hex=block_hash, - block_hash_int=int(block_hash, 16), - share_pass=True, - block_pass=True, - applied_version_hex="", - ) - prospective = intent.get("prospective_prior_balances") - if isinstance(prospective, Sequence) and not isinstance(prospective, (str, bytes)): - def normalized_balance(row: Any) -> tuple[str, str, str, int]: - return str(row[0]), str(row[1]), str(row[2]), int(row[3]) - - # Preserve eager validation without retaining a second collection. - for row in prospective: - normalized_balance(row) - prospective = MappedSequence(prospective, normalized_balance) - else: - prospective = None - context = PrismJobContext( - job=SimpleNamespace( - transaction_hexes=(), - witness_merkle_leaves_hex=_replayable_share_sequence( - intent.get("witness_merkle_leaves_hex", []) - ), - ), - template=template, - shares_json=_replayable_share_sequence(intent["shares_json"]), - prior_balances=_replayable_share_sequence(intent["prior_balances"]), - found_block=dict(intent["found_block"]), - share_weight=0, - collection_only=bool(intent.get("collection_only", False)), - worker=WorkerIdentity( - username=username, - payout_address="", - worker_name=None, - script_pubkey_hex="", - p2mr_program_hex="", - ), - issued_at_ms=0, - prospective_prior_balances=prospective, - # Append-invalidation epochs are process-local counters, so a - # stamp from the process that built this candidate is meaningless - # after a restart. The negative sentinel tells the landing epoch - # fence to stand down and instead revalidate the recorded window - # against the durable ledger at its declared anchor (carry - # balances do not move on a share append, so the prior-balance - # fence alone cannot see an omitted late row). - payout_append_invalidation_epoch=-1, - ) - return PrismBlockCandidate( - context=context, - submission=submission, - extranonce1_hex=str(intent["extranonce1_hex"]), - extranonce2_hex=str(intent["extranonce2_hex"]), - pending_share=PendingShare(**dict(intent["pending_share"])), - client=SimpleNamespace(username=username), - credit_share_on_accept=bool(intent.get("credit_share_on_accept", False)), - ) - - -class BlockCandidateService: - """Own the durable replay queue, retry ordering, and submitter lifecycle.""" - - def __init__( - self, - ports: BlockCandidatePorts, - *, - candidate_queue: queue.Queue[PrismBlockCandidate] | None = None, - retry_initial_seconds: float = DEFAULT_BLOCK_CANDIDATE_RETRY_INITIAL_SECONDS, - retry_max_seconds: float = DEFAULT_BLOCK_CANDIDATE_RETRY_MAX_SECONDS, - retryable_reasons: frozenset[str] = frozenset(), - submit_seconds_buckets: tuple[float, ...] | None = None, - ) -> None: - self.ports = ports - self._coordinator = ports.runtime - self.candidate_queue = candidate_queue or queue.Queue( - maxsize=MAX_PENDING_BLOCK_CANDIDATES - ) - self.retry_initial_seconds = max(0.0, float(retry_initial_seconds)) - self.retry_max_seconds = max( - self.retry_initial_seconds, - float(retry_max_seconds), - ) - self.retryable_reasons = frozenset(retryable_reasons) - self.retry_delays: dict[str, float] = {} - self.finalize_retries: dict[str, tuple[bool, str]] = {} - self.retry_candidate: PrismBlockCandidate | None = None - self.wakeups_coalesced = 0 - self.retries = 0 - self.poisoned = 0 - self.dropped = 0 - self.abandoned_counts: dict[str, int] = {} - self.outcome = threading.local() - self._state_lock = threading.RLock() - self._backoff_started_monotonic: float | None = None - self._backoff_deadline_monotonic: float | None = None - self._backoff_delay_seconds = 0.0 - # Hashes of block candidates this process may still land (durable - # outbox pending, queued, retained for retry, or mid-disposition). - # Membership lets every tip-observation channel recognize the pool's - # own block the moment it becomes the chain tip. - self._outstanding_block_candidate_hashes: set[str] = set() - # Outstanding candidate hashes observed as the chain tip - # (hash -> monotonic stamp). qbitd only reports a candidate hash as - # its tip after accepting that block, so an entry here is acceptance - # evidence that outlives transient fork views and lost submitblock - # acks; disposition/abandon paths consult it before treating any - # instantaneous chain probe as terminal truth. - self._tip_observed_accepted_block_hashes: dict[str, float] = {} - # Durable cleanup can fail after a terminal decision and force the - # same hash through that decision again; abandonment metrics count - # candidates, not cleanup attempts. Retired by the terminal-outcome - # eviction pass, which is the one place that has already proved a - # hash unpinned by every live and cleanup-owing lane. - self._counted_block_candidate_abandonments: set[str] = set() - self._block_submit_metrics_lock = threading.Lock() - if submit_seconds_buckets is None: - submit_seconds_buckets = tuple(PRISM_JOB_BUILD_SECONDS_BUCKETS) - # Landed candidate -> submitblock-return interval. Post-submit - # bookkeeping (audit build, persistence, outbox finalize) is - # deliberately excluded; the outbox created_at/completed_at span - # already covers it. - self.block_submit_seconds_histogram: dict[str, Any] = { - "buckets": {bucket: 0 for bucket in submit_seconds_buckets}, - "sum": 0.0, - "count": 0, - } - # Issue #181 item 3. Definitive node acceptance is learned on the - # submitter thread and the preview is published on the accounting - # thread, so the start stamp has to outlive the offer: the hash is - # the only identity both sides share. A stamp of None is the - # already-observed tombstone -- it keeps a re-offer of the same hash - # from restarting an interval that has already been measured -- and - # both the stamps and the histograms live under one leaf lock that - # never nests, so an observation can be taken while the publication - # path still holds the payout-balance mutation lock. - self._accepted_block_preview_publication_lock = threading.Lock() - self.accepted_block_preview_publication_seconds_histogram: dict[ - str, dict[str, Any] - ] = { - result: { - "buckets": { - bucket: 0 - for bucket in ( - PRISM_ACCEPTED_BLOCK_PREVIEW_PUBLICATION_SECONDS_BUCKETS - ) - }, - "sum": 0.0, - "count": 0, - } - for result in PRISM_ACCEPTED_BLOCK_PREVIEW_PUBLICATION_RESULTS - } - self._accepted_block_preview_acceptance_monotonic: dict[ - str, float | None - ] = {} - # Issue #224: the landing phase attribution for the hashes whose - # intervals are open, under the same leaf lock as the stamps above and - # bounded by the same cap with the same oldest-first eviction. A record - # is popped by the publication that closes its interval and discarded - # on terminal abandonment, so like the stamps this cap is a leak bound - # rather than a working set. - self._accepted_landing_attributions: dict[ - str, _AcceptedLandingAttribution - ] = {} - self._block_replay_enumeration_owed_flag = False - # A forced enumeration that meets cleanup backpressure adopts at - # most that one bounded page, then lets the submitter drain it before - # another outbox walk. This flag prevents an eager re-entry from - # growing the otherwise-unbounded replay queue page by page. - self._block_replay_backpressure_drain_pending = False - # Targeted ancestor re-drive state (issue #190), all under - # coordinator.lock. A found block whose finalization keeps deferring - # on the same unresolved accepted-ancestor payout transition arms a - # forced durable-replay pass for that ancestor after a configured - # deferral streak; the counters feed the qbit_prism_accepted_parent_ - # redrive_* metrics. One record per ancestor plus one child-keyed - # blocking index, each insertion-ordered by recency for eviction. - self._ancestor_redrive_records: dict[str, _AncestorRedriveRecord] = {} - self._ancestor_redrive_last_blocking: dict[str, str] = {} - self.accepted_parent_redrive_attempt_count = 0 - self.accepted_parent_redrive_resolved_count = 0 - self.accepted_parent_redrive_exhausted_count = 0 - # Fixed-key selector/apply outcome counters for the decided-height - # collapse. The key set is closed so the rendered series carries no - # block hash, parent hash, or job ID in a label. - self._block_candidate_collapse_counts: dict[str, int] = { - outcome: 0 for outcome in PRISM_BLOCK_CANDIDATE_COLLAPSE_OUTCOMES - } - self._block_candidate_collapse_fail_closed_logged_monotonic: ( - float | None - ) = None - # Terminal hashes whose collapse cleanup did not complete, keyed by - # hash so the backlog is bounded by the affected rows themselves and - # a repeated failure re-registers rather than accumulating. - self._block_candidate_collapse_cleanup_retries: dict[ - str, - _CollapsedCandidateCleanup, - ] = {} - - # -- twelve historical field aliases ----------------------------------- - - @property - def _retry_block_candidate(self) -> PrismBlockCandidate | None: - """Historical attribute spelling used by the retry-merge plumbing.""" - return self.retry_candidate - - @_retry_block_candidate.setter - def _retry_block_candidate(self, value: PrismBlockCandidate | None) -> None: - self.retry_candidate = value - - # -- replayed-candidate credit floor ----------------------------------- - - def adopt_replayed_candidate(self, candidate: PrismBlockCandidate) -> None: - """Adopt a decoded credit-bearing candidate's snapshot-floor holder. - - A below-target candidate can credit its older accepted stamp after - durable replay. Registering the reconstructed PendingShare on the S3 - pending floor before the candidate becomes visible to job issuance - keeps startup prewarm anchored below that stamp. - """ - if candidate.credit_share_on_accept: - self.ports.share_writer().adopt_pending_share(candidate.pending_share) - - def _release_dropped_duplicate_candidate_floor( - self, - candidate: PrismBlockCandidate, - ) -> None: - """Release a duplicate-dropped credit candidate's floor holder. - - Same-hash duplicates are distinct objects (each durable-replay decode - adopts a freshly reconstructed PendingShare; each live stamp registers - its own), and the pending floor keys holders by object identity, so - the object whose disposition actually lands releases only its own - holder. Dropping a credit-bearing duplicate without this release - pins the job/payout snapshot anchor below its stamp until restart. - Non-credit candidates are exempt: their holder belongs to the - share-append commit path, which may still be in flight. - """ - if getattr(candidate, "credit_share_on_accept", False): - self._coordinator._finish_pending_share_candidate( - candidate.pending_share - ) - - # -- queue admission --------------------------------------------------- - - def enqueue(self, candidate: PrismBlockCandidate) -> bool: - # Outstanding from admission (even when the wakeup coalesces below: - # the durable outbox row keeps the candidate replayable), so a tip - # observation can register acceptance before the submitter drains it. - self._register_outstanding_block_candidate( - str(candidate.submission.block_hash_hex) - ) - queue_obj = self.candidate_queue - if queue_obj is None: - queue_obj = queue.Queue(maxsize=MAX_PENDING_BLOCK_CANDIDATES) - self.candidate_queue = queue_obj - try: - queue_obj.put_nowait(candidate) - return True - except queue.Full: - # The candidate is already durable. A full queue merely coalesces - # this wakeup; the submitter re-reads pending outbox rows whenever - # it drains the queue, so no candidate is discarded. - with self._coordinator.lock: - self.wakeups_coalesced = int(self.wakeups_coalesced) + 1 - print( - "prism coordinator: block candidate wakeup coalesced " - f"hash={candidate.submission.block_hash_hex} (submitter queue full)", - flush=True, - ) - return False - - # -- startup enumeration gate (#120) ----------------------------------- - - def _note_block_replay_enumeration_owed(self) -> None: - """Mark that pending durable candidates have not been enumerated yet.""" - with self._coordinator.lock: - self._block_replay_enumeration_owed_flag = True - - def _clear_block_replay_enumeration_owed(self) -> None: - with self._coordinator.lock: - self._block_replay_enumeration_owed_flag = False - - def _block_replay_enumeration_owed(self) -> bool: - with self._coordinator.lock: - return bool(self._block_replay_enumeration_owed_flag) - - def _run_startup_block_candidate_replay(self) -> bool: - """Run best-effort pre-accept replay without dying on a slow ledger.""" - # Until the durable outbox has been enumerated once, this process - # cannot know whether a pending accepted candidate exists. Child job - # builds fail closed on that uncertainty instead of snapshotting a - # payout base that may omit a pending parent's carry (issue #188); - # replay_pending_block_candidates clears the flag on success. - self._coordinator._note_block_replay_enumeration_owed() - try: - return self._coordinator._run_startup_writer_replay( - self._coordinator.replay_pending_block_candidates - ) - except TimeoutError: - print( - "prism coordinator: startup block candidate replay timed out " - "phase=replay-outbox-query " - f"timeout={self._coordinator._block_landing_db_timeout():g}s; " - "continuing startup; block submitter loop will retry and " - "job builds stay blocked until pending candidates are known", - flush=True, - ) - return True - - # -- durable replay and quarantine ------------------------------------- - - def _ensure_block_replay_state(self) -> None: - """Backfill replay/maintenance queues for lightweight coordinators.""" - with _STATE_BACKFILL_LOCK: - if not hasattr(self, "_block_replay_candidate_queue"): - self._block_replay_candidate_queue = queue.Queue() - if not hasattr(self, "_block_replay_inflight_hashes"): - self._block_replay_inflight_hashes: set[str] = set() - if not hasattr(self, "_block_quarantine_queue"): - self._block_quarantine_queue = queue.Queue() - if not hasattr(self, "_block_quarantine_hashes"): - self._block_quarantine_hashes: set[str] = set() - if not hasattr(self, "_block_replay_backpressure_drain_pending"): - self._block_replay_backpressure_drain_pending = False - - def _enqueue_replayed_block_candidate( - self, - candidate: PrismBlockCandidate, - ) -> bool: - """Queue one durable replay behind live solves, once per process.""" - self._ensure_block_replay_state() - block_hash = str(candidate.submission.block_hash_hex).lower() - with self._coordinator.lock: - duplicate = ( - block_hash in self._block_replay_inflight_hashes - or block_hash - in getattr(self, "_block_candidate_terminal_outcomes", {}) - # A hash a single-slot retry holder still carries has a live - # in-memory copy whose durable row is deliberately still - # pending. Before the #190 re-drive, the retry short-circuit - # kept enumeration from ever seeing such a row mid-retention; - # a forced pass reaches it, and adopting it would install a - # second in-process copy (and re-arm its payout barrier) - # under the holder's. Skipping is safe: the row stays durable - # and pending, and a later pass re-reads it once the holder - # lets go. Only the holders qualify -- the wider lane - # registries (the outstanding set above all) name every - # admitted hash, and skipping those would break the - # queue-overflow recovery this enumeration exists for. - or block_hash in self._held_block_candidate_retry_hashes() - ) - if not duplicate: - self._block_replay_inflight_hashes.add(block_hash) - if duplicate: - # The instance decode already adopted this object's credit floor - # holder; the earlier same-hash flight owns the disposition and - # releases only its own holder. - self._release_dropped_duplicate_candidate_floor(candidate) - return False - try: - # This is an in-memory condition update, not accounting. Install - # it before making the candidate visible to the raw lane so - # startup prewarm cannot build a child from the old payout base - # after qbitd accepts but before the RPC response returns. - self._coordinator._begin_accepted_block_payout_preview( - block_hash, - block_height=int(candidate.context.template["height"]), - ) - self._block_replay_candidate_queue.put_nowait(candidate) - except BaseException: - with self._coordinator.lock: - self._block_replay_inflight_hashes.discard(block_hash) - raise - return True - - def _queue_invalid_block_candidate_for_quarantine( - self, - block_hash: str, - error: str, - *, - pending_share: PendingShare | None = None, - ) -> None: - """Move malformed-row cleanup off the node-offer lane.""" - if not block_hash: - return - self._ensure_block_replay_state() - key = block_hash.lower() - with self._coordinator.lock: - if key in self._block_quarantine_hashes: - return - self._block_quarantine_hashes.add(key) - self._block_quarantine_queue.put_nowait((key, error, pending_share)) - - def _adopt_durable_block_candidate_rows( - self, - durable_rows: list[Any], - ) -> int: - """Decode and queue one fetched batch, quarantining malformed rows. - - Shared by both enumeration shapes (keyset pages and the legacy - widening window) so a fallback pass restores rows through exactly - the same decode, dedupe, and poison path. - """ - queued = 0 - self._coordinator._record_block_submitter_phase("replay-restore") - for durable_row in durable_rows: - durable_block_hash = "" - # Published to poison cleanup only after its durable credit - # holder was adopted successfully (the instance decode adopts - # the credit floor before the candidate is visible anywhere). - decoded_candidate: PrismBlockCandidate | None = None - try: - if not isinstance(durable_row, dict): - raise ValueError("durable block candidate row is not an object") - durable_block_hash = str(durable_row["block_hash"]).lower() - if "header" in durable_row: - # Metadata-first row (#255). A ledger whose bodies live - # in storage registers a small descriptor now (its - # accepted-parent and credit-floor obligations included) - # and hydrates the body only when the descriptor is - # dequeued; an in-memory ledger hands the intent back - # immediately and takes the classic decode below. - if self._replay_hydration_deferred(): - if self._register_durable_block_candidate_header_row( - durable_row - ): - queued += 1 - if self._block_replay_backpressure_drain_pending: - break - continue - intent = self._coordinator.ledger.hydrate_block_candidate_intent( - durable_row - ) - else: - intent = durable_row["candidate"] - if not isinstance(intent, Mapping): - raise ValueError("durable block candidate intent is not an object") - intent_block_hash = str(intent.get("block_hash_hex", "")).lower() - if not durable_block_hash or intent_block_hash != durable_block_hash: - raise ValueError("durable block candidate row key does not match intent") - decoded_candidate = dataclass_replace( - self._coordinator.block_candidate_from_intent(intent), - durable_replay=True, - ) - # Durable acceptance-state reads stay in accounting. The - # separate replay queue keeps these recovered rows behind any - # live solve while still exposing the whole batch to qbitd. - if self._coordinator._enqueue_replayed_block_candidate( - decoded_candidate - ): - queued += 1 - except Exception: - print("prism coordinator: invalid durable block candidate intent", flush=True) - traceback.print_exc() - self._coordinator._queue_invalid_block_candidate_for_quarantine( - durable_block_hash, - "invalid durable candidate intent", - pending_share=( - decoded_candidate.pending_share - if decoded_candidate is not None - and decoded_candidate.credit_share_on_accept - else None - ), - ) - return queued - - # -- metadata-first descriptors and bounded hydration (#255) ----------- - - def _replay_hydration_deferred(self) -> bool: - """Whether this ledger's bodies must be hydrated at dequeue time.""" - return bool( - getattr(self._coordinator.ledger, "candidate_hydration_deferred", False) - ) - - def _ensure_replay_descriptor_state(self) -> None: - with _STATE_BACKFILL_LOCK: - if not hasattr(self, "_block_replay_floor_holders"): - self._block_replay_floor_holders: dict[str, PendingShare] = {} - if not hasattr(self, "_block_replay_descriptor_overflow_count"): - self._block_replay_descriptor_overflow_count = 0 - - def _durable_candidate_descriptor( - self, - row: Mapping[str, Any], - ) -> DurableCandidateDescriptor: - """Strictly type one header row; any malformed fact raises. - - The same facts the collapse selector validates, plus the ones - registration and hydration need: a decodable ``PendingShare``, the - credit flag, and for a chunked body a sealed reference whose digest - agrees with the row. A row that fails here enters the existing - quarantine path exactly as a malformed intent did. - """ - block_hash = _collapse_block_hash(row.get("block_hash")) - header = row.get("header") - if block_hash is None or not isinstance(header, Mapping): - raise ValueError("durable block candidate header is not an object") - if _collapse_block_hash(header.get("block_hash_hex")) != block_hash: - raise ValueError("durable block candidate row key does not match its header") - parent_hash = _collapse_block_hash(header.get("parent_hash")) - height = _collapse_height(header.get("expected_height")) - if parent_hash is None or height is None or height < 0: - raise ValueError( - "durable block candidate header carries no usable parent or height" - ) - template = header.get("template") - if not isinstance(template, Mapping): - raise ValueError("durable block candidate header carries no template") - if _collapse_block_hash(template.get("previousblockhash")) != parent_hash: - raise ValueError("durable block candidate header template parent disagrees") - if _collapse_height(template.get("height")) != height: - raise ValueError("durable block candidate header template height disagrees") - pending_share = header.get("pending_share") - if not isinstance(pending_share, Mapping): - raise ValueError("durable block candidate header carries no pending share") - # Validates the field set exactly as the historical decode did. - PendingShare(**dict(pending_share)) - credit = header.get("credit_share_on_accept") - if not isinstance(credit, bool): - raise ValueError("durable block candidate header carries no credit flag") - # An oversized header replaces an over-bound fact with None and defers - # it to the body (``replay_header_from_fields``); a long worker name - # does this to ``username`` and ``pending_share.share_id``. That - # placeholder is not corruption: hydration reads the real value and - # the identity digest verifies it. - oversized = header.get("oversized") is True - username = header.get("username") - if not isinstance(username, str) and not (oversized and username is None): - raise ValueError("durable block candidate header carries no username") - storage_version = int(row.get("storage_version") or 1) - candidate_sha256 = str(row.get("candidate_sha256") or "") - body: CandidateBodyRef | None = None - if storage_version == 2: - body_row = row.get("body") - if not isinstance(body_row, Mapping): - raise ValueError("durable block candidate row carries no body reference") - if body_row.get("state") != "sealed": - raise ValueError("durable block candidate body is not sealed") - body = CandidateBodyRef.from_row(body_row) - if body.candidate_sha256 != candidate_sha256: - raise ValueError("durable block candidate body digest disagrees with its row") - elif storage_version != 1: - raise ValueError(f"unsupported candidate storage version {storage_version}") - return DurableCandidateDescriptor( - block_hash=block_hash, - block_height=int(height), - parent_hash=parent_hash, - candidate_sha256=candidate_sha256, - storage_version=storage_version, - credit_share_on_accept=credit, - collection_only=bool(header.get("collection_only", False)), - username=username, - pending_share=dict(pending_share), - accepted_at_present=bool(header.get("accepted_at_present")), - accepted_at_ms=pending_share.get("accepted_at_ms"), - body=body, - cursor=row.get("cursor"), - row=dict(row), - header_oversized=oversized, - ) - - def _register_durable_block_candidate_header_row( - self, - row: Mapping[str, Any], - ) -> bool: - return self._register_replayed_block_candidate_descriptor( - self._durable_candidate_descriptor(row) - ) - - def _adopt_replay_descriptor_floor( - self, - descriptor: DurableCandidateDescriptor, - ) -> PendingShare: - """Hold the credit-bearing stamp's snapshot floor for one hash. - - One holder per hash for as long as the row is pending and owned - here; a descriptor re-registered after an overflow or a paced retry - reuses it rather than stacking a second holder under the first. - """ - self._ensure_replay_descriptor_state() - with self._coordinator.lock: - existing = self._block_replay_floor_holders.get(descriptor.block_hash) - if existing is not None: - return existing - holder = PendingShare(**descriptor.pending_share) - self._block_replay_floor_holders[descriptor.block_hash] = holder - self.ports.share_writer().adopt_pending_share(holder) - return holder - - def _take_replay_floor_holder(self, block_hash: str) -> PendingShare | None: - """Hand the holder to the object that will release it; no release.""" - self._ensure_replay_descriptor_state() - with self._coordinator.lock: - return self._block_replay_floor_holders.pop(block_hash, None) - - def _release_replay_floor_holder(self, block_hash: str) -> None: - holder = self._take_replay_floor_holder(block_hash) - if holder is not None: - self._coordinator._finish_pending_share_candidate(holder) - - def _register_replayed_block_candidate_descriptor( - self, - descriptor: DurableCandidateDescriptor, - ) -> bool: - """Register one durable row's obligations and queue its descriptor. - - Mirrors ``_enqueue_replayed_block_candidate`` for a row whose body - has not been read: the same duplicate rules, the same payout-preview - registration before the row is visible to the submitter, and the - credit floor adopted from the header's pending share. When the - in-memory window is full enumeration pauses before adopting any - further obligations. The enumeration-owed gate protects job builds - until the queue drains and the remaining durable rows are adopted. - """ - self._ensure_block_replay_state() - self._ensure_replay_descriptor_state() - block_hash = descriptor.block_hash - overflow = False - with self._coordinator.lock: - duplicate = ( - block_hash in self._block_replay_inflight_hashes - or block_hash in getattr(self, "_block_candidate_terminal_outcomes", {}) - or block_hash in self._held_block_candidate_retry_hashes() - ) - if not duplicate: - overflow = ( - self._block_replay_candidate_queue.qsize() - >= MAX_BLOCK_REPLAY_DESCRIPTORS_IN_MEMORY - ) - if not overflow: - self._block_replay_inflight_hashes.add(block_hash) - if duplicate: - return False - if overflow: - with self._coordinator.lock: - self._block_replay_descriptor_overflow_count += 1 - self._block_replay_backpressure_drain_pending = True - self._coordinator._note_block_replay_enumeration_owed() - return False - try: - if descriptor.credit_share_on_accept: - self._adopt_replay_descriptor_floor(descriptor) - self._coordinator._begin_accepted_block_payout_preview( - block_hash, - block_height=descriptor.block_height, - ) - self._block_replay_candidate_queue.put_nowait(descriptor) - except BaseException: - with self._coordinator.lock: - self._block_replay_inflight_hashes.discard(block_hash) - raise - return True - - @staticmethod - def _replay_descriptor_ready(descriptor: DurableCandidateDescriptor) -> bool: - return time.monotonic() >= float(descriptor.not_before_monotonic) - - def _hydrate_replay_descriptor( - self, - descriptor: DurableCandidateDescriptor, - ) -> PrismBlockCandidate | None: - """Read one descriptor's body and decode it, off every lock. - - At most one hydration runs at a time (this is the submitter thread), - through the bounded ledger call so a wedged read stays registered - under a key that names the body rather than reusing another page's - call. Outcomes: a candidate; ``None`` after the descriptor was - dropped (its row is no longer pending), quarantined (malformed - durable content) or requeued for a paced retry (transport failure, - spool backpressure, deadline). - """ - coordinator = self._coordinator - block_hash = descriptor.block_hash - coordinator._record_block_submitter_phase("replay-hydrate") - hydrate = getattr(coordinator.ledger, "hydrate_block_candidate_intent", None) - intent = None - transferred = False - try: - if not callable(hydrate): - raise CandidateBodyUnavailable("ledger cannot hydrate durable candidates") - intent = coordinator._run_block_submitter_ledger_call( - ("replay-hydrate", block_hash, descriptor.body_key), - "replay-hydrate", - lambda: hydrate( - descriptor.row, - cancelled=coordinator.stop_event.is_set, - ), - timeout_seconds=coordinator._block_landing_db_timeout(), - call_class="landing", - ) - candidate = block_candidate_from_intent(intent) - if str(candidate.submission.block_hash_hex).lower() != block_hash: - raise ValueError("hydrated block candidate does not match its row") - if candidate.credit_share_on_accept: - holder = self._take_replay_floor_holder(block_hash) - if holder is None: - holder = self._adopt_replay_descriptor_floor(descriptor) - self._take_replay_floor_holder(block_hash) - body_share = candidate.pending_share - if body_share is not None and body_share != holder: - # The header's copy held the floor until the body was - # read, but an oversized header carries None - # placeholders. The digest-verified body is - # authoritative: adopt it before releasing the stand-in - # so the floor never lapses. - self.ports.share_writer().adopt_pending_share(body_share) - self._coordinator._finish_pending_share_candidate(holder) - holder = body_share - candidate = dataclass_replace(candidate, pending_share=holder) - candidate = dataclass_replace(candidate, durable_replay=True) - transferred = True - return candidate - except ShutdownInProgress: - self._requeue_replay_descriptor(descriptor) - raise - except CandidateBodyUnavailable as exc: - print( - "prism coordinator: durable block candidate is no longer pending; " - f"dropping its replay descriptor hash={block_hash} ({exc})", - flush=True, - ) - self._drop_replay_descriptor(descriptor) - return None - except ( - CandidateBodyIntegrityError, - CandidateCodecError, - ValueError, - TypeError, - KeyError, - ): - print("prism coordinator: invalid durable block candidate body", flush=True) - traceback.print_exc() - self._quarantine_replay_descriptor(descriptor) - return None - except Exception: - print( - "prism coordinator: durable block candidate hydration failed " - f"hash={block_hash}; the row stays pending and is retried", - flush=True, - ) - traceback.print_exc() - self._requeue_replay_descriptor(descriptor) - return None - - finally: - if not transferred and isinstance(intent, PreparedCandidateIntent): - intent.release() - - def _drop_replay_descriptor(self, descriptor: DurableCandidateDescriptor) -> None: - block_hash = descriptor.block_hash - with self._coordinator.lock: - self._block_replay_inflight_hashes.discard(block_hash) - self._release_replay_floor_holder(block_hash) - self._coordinator._clear_accepted_block_payout_preview(block_hash) - - def _quarantine_replay_descriptor( - self, - descriptor: DurableCandidateDescriptor, - ) -> None: - block_hash = descriptor.block_hash - with self._coordinator.lock: - self._block_replay_inflight_hashes.discard(block_hash) - # The quarantine step releases the holder after the row is terminal. - holder = self._take_replay_floor_holder(block_hash) - self._coordinator._queue_invalid_block_candidate_for_quarantine( - block_hash, - "invalid durable candidate intent", - pending_share=holder, - ) - - def _requeue_replay_descriptor(self, descriptor: DurableCandidateDescriptor) -> None: - """Park the descriptor at the back of the window with backoff.""" - attempts = int(descriptor.attempts) + 1 - delay = min( - float(self.retry_initial_seconds) * (2 ** min(attempts, 16)), - float(self.retry_max_seconds), - ) - self._block_replay_candidate_queue.put_nowait( - dataclass_replace( - descriptor, - attempts=attempts, - not_before_monotonic=time.monotonic() + delay, - ) - ) - - def replay_descriptor_snapshot(self) -> dict[str, int]: - """Bounded gauges for the metadata-first replay window.""" - self._ensure_block_replay_state() - self._ensure_replay_descriptor_state() - with self._coordinator.lock: - return { - "queued": int(self._block_replay_candidate_queue.qsize()), - "floor_holders": len(self._block_replay_floor_holders), - "overflow": int(self._block_replay_descriptor_overflow_count), - } - - def _run_block_candidate_body_janitor_step(self) -> None: - """One bounded chunk-page reclaim of a retired body, rate limited. - - Separate from every terminalization statement and never under - ``coordinator.lock``: the fenced terminal write only detaches and - retires a manifest, and this step drains its chunks a page at a - time. A failure is logged and the loop moves on. - """ - coordinator = self._coordinator - reap = getattr(coordinator.ledger, "reap_retired_candidate_bodies", None) - if not callable(reap): - return - now = time.monotonic() - last = float(getattr(self, "_block_candidate_body_janitor_last_monotonic", 0.0)) - if now - last < BLOCK_CANDIDATE_BODY_JANITOR_INTERVAL_SECONDS: - return - self._block_candidate_body_janitor_last_monotonic = now - try: - outcome = coordinator._run_block_submitter_ledger_call( - ("candidate-body-janitor",), - "candidate-body-janitor", - lambda: reap(), - ) - except ShutdownInProgress: - raise - except Exception: - print("prism coordinator: candidate body janitor step failed", flush=True) - traceback.print_exc() - return - if isinstance(outcome, dict) and ( - outcome.get("deleted_chunks") - or outcome.get("deleted_pages") - or outcome.get("deleted_spans") - or outcome.get("deleted_bodies") - or outcome.get("pending") - ): - # A retired body still has parts (or a manifest) to reclaim on - # a later step: let the next loop iteration step again. - self._block_candidate_body_janitor_last_monotonic = 0.0 - if any( - outcome.get(key) - for key in ("deleted_chunks", "deleted_pages", "deleted_spans", "deleted_bodies") - ): - print( - "prism coordinator: candidate body janitor reclaimed " - f"chunks={outcome.get('deleted_chunks', 0)} " - f"pages={outcome.get('deleted_pages', 0)} " - f"spans={outcome.get('deleted_spans', 0)} " - f"bodies={outcome.get('deleted_bodies', 0)} " - f"body_id={outcome.get('body_id')}", - flush=True, - ) - - # -- decided-height collapse (#183) ------------------------------------ - - def _ensure_block_candidate_collapse_state(self) -> None: - """Backfill the collapse counters and cleanup-retry registry.""" - if ( - hasattr(self, "_block_candidate_collapse_counts") - and hasattr(self, "_block_candidate_collapse_cleanup_retries") - and hasattr(self, "_block_candidate_collapse_cleanup_inflight") - and hasattr(self, "_block_candidate_cleanup_backpressure_engagements") - ): - return - with _STATE_BACKFILL_LOCK: - if not hasattr(self, "_block_candidate_collapse_counts"): - self._block_candidate_collapse_counts: dict[str, int] = { - outcome: 0 - for outcome in PRISM_BLOCK_CANDIDATE_COLLAPSE_OUTCOMES - } - if not hasattr(self, "_block_candidate_collapse_cleanup_retries"): - self._block_candidate_collapse_cleanup_retries: dict[ - str, - _CollapsedCandidateCleanup, - ] = {} - if not hasattr(self, "_block_candidate_collapse_cleanup_inflight"): - # A retry moves its record here for the duration of the - # attempt. It must remain part of both the admission depth - # and the terminal-outcome pin set while the accounting lane - # is running cleanup outside coordinator.lock. - self._block_candidate_collapse_cleanup_inflight: dict[ - str, - _CollapsedCandidateCleanup, - ] = {} - if not hasattr(self, "_block_candidate_cleanup_backpressure_engagements"): - # Occasions on which the admission bound withheld at least - # one row from bulk terminalization, and when the last - # bounded warning was printed. Both under coordinator.lock. - self._block_candidate_cleanup_backpressure_engagements = 0 - self._block_candidate_cleanup_backpressure_logged_monotonic: ( - float | None - ) = None - - def _record_block_candidate_collapse( - self, - outcome: str, - count: int = 1, - ) -> None: - if outcome not in PRISM_BLOCK_CANDIDATE_COLLAPSE_OUTCOMES: - raise ValueError( - f"unknown block candidate collapse outcome: {outcome}" - ) - if count <= 0: - return - self._ensure_block_candidate_collapse_state() - with self._coordinator.lock: - counts = self._block_candidate_collapse_counts - counts[outcome] = int(counts.get(outcome, 0)) + int(count) - - def block_candidate_collapse_snapshot(self) -> dict[str, int]: - """Copied fixed-key collapse counters for metrics rendering. - - The key set is closed and carries no hash, parent, or job ID, so the - renderer can label by outcome without any unbounded label space. - """ - self._ensure_block_candidate_collapse_state() - with self._coordinator.lock: - counts = dict(self._block_candidate_collapse_counts) - return { - outcome: int(counts.get(outcome, 0)) - for outcome in PRISM_BLOCK_CANDIDATE_COLLAPSE_OUTCOMES - } - - def _block_candidate_collapse_evidence( - self, - hashes: Iterable[object], - *, - ignore_leases: frozenset[str] = frozenset(), - ) -> frozenset[str]: - """Which of ``hashes`` some evidence says were offered to qbitd, or are being. - - Answers membership for the hashes the caller is actually deciding -- - one durable page during selection, the leased subset during - revalidation -- instead of materializing the union of every registry. - The registries this reads are not all page-sized: - ``_block_candidate_terminal_outcomes`` is historical state that grows - with every block the process ever disposed (312,000 entries after one - testnet4 storm), and copying it per poll cost ~124 ms with - ``coordinator.lock`` held against every unrelated share ack. Probing - the page's own hashes keeps the cost O(page * registries) and makes - it independent of how much disposition history has accumulated. - - Deliberately excludes ``_outstanding_block_candidate_hashes`` and - ``_block_replay_inflight_hashes``. Both mean "this process still owns - eventual disposition", not "a node offer happened": live admission - marks a candidate outstanding before it even reaches the bounded - queue, and replay adoption marks every restored row inflight, so - folding either one in excludes an entire storm and makes the whole - selector a safe no-op. ``attempt_count`` is excluded for the mirror - reason: it is stamped only *after* the node offer, so at the instant - that matters it proves nothing. - - ``ignore_leases`` lets the pre-write revalidation re-read this - evidence without the apply's own disposition leases disqualifying - the very rows they were claimed for. - - Every registry read here is keyed by the canonical lowercase hash the - rest of the submitter compares on -- the same spelling - ``_collapse_block_hash`` folds a page row to -- which is what lets a - membership probe replace the old lowercasing walk without changing - which hashes count as evidence. - """ - coordinator = self._coordinator - coordinator._ensure_job_cache_state() - self._ensure_block_candidate_disposition_state() - probes: set[str] = set() - for value in hashes: - key = _collapse_block_hash(value) - if key is not None: - probes.add(key) - if not probes: - return frozenset() - evidence: set[str] = set() - with self._block_submitter_lock( - self._block_candidate_disposition_registry_lock, - "candidate-disposition-registry", - ): - flights = self._block_candidate_disposition_flights - evidence.update( - key - for key in probes - if key in flights and key not in ignore_leases - ) - with coordinator.lock: - for holder in ( - self.retry_candidate, - getattr(self, "_block_accounting_deferred_retry_candidate", None), - ): - if holder is None: - continue - held = _collapse_block_hash( - getattr( - getattr(holder, "submission", None), - "block_hash_hex", - None, - ) - ) - if held is not None and held in probes: - evidence.add(held) - for registry in ( - self._block_disposition_waiting_retries, - self.finalize_retries, - getattr(self, "_block_candidate_retained_node_submissions", None), - self._tip_observed_accepted_block_hashes, - getattr(coordinator, "_accounted_accepted_block_hashes", None), - self._block_candidate_terminal_outcomes, - ): - if not registry: - continue - evidence.update(key for key in probes if key in registry) - return frozenset(evidence) - - def _collapse_pool_block_exists(self, durable_row: object) -> bool: - """Read one row's durable pool-block fact, or fail the page closed. - - A missing key is not "no pool block": it is a page that cannot - answer the one fact keeping an offered, landed candidate out of a - terminal set. Reading it as false would abandon exactly the rows - that must never be abandoned, so the whole page fails closed. The - fenced batch write re-asks under the writer fence regardless. - """ - if ( - not isinstance(durable_row, dict) - or "pool_block_exists" not in durable_row - ): - raise _BlockCandidateCollapseFailedClosed( - "durable page row carries no pool block existence fact" - ) - exists = durable_row["pool_block_exists"] - if not isinstance(exists, bool): - raise _BlockCandidateCollapseFailedClosed( - "durable page row pool block existence is not a boolean" - ) - return exists - - def _superseded_candidate_row( - self, - durable_row: object, - ) -> _SupersededCandidateRow: - """Reduce one page row to S's inputs, or fail its page closed. - - The preserved page still reaches ordinary decode/quarantine, but no - sibling may be terminalized from a snapshot containing a malformed - candidate fact. - """ - if not isinstance(durable_row, dict): - raise _BlockCandidateCollapseFailedClosed( - "durable candidate row is not an object" - ) - block_hash = _collapse_block_hash(durable_row.get("block_hash")) - intent = durable_row.get("candidate") - if block_hash is None or not isinstance(intent, Mapping): - raise _BlockCandidateCollapseFailedClosed( - "durable candidate row carries no usable hash or intent" - ) - if _collapse_block_hash(intent.get("block_hash_hex")) != block_hash: - raise _BlockCandidateCollapseFailedClosed( - "durable candidate row key does not match its intent" - ) - parent_hash = _collapse_block_hash(intent.get("parent_hash")) - height = _collapse_row_height(durable_row) - if parent_hash is None or height is None or height < 0: - raise _BlockCandidateCollapseFailedClosed( - "durable candidate intent carries no usable parent or height" - ) - template = intent.get("template") - if not isinstance(template, dict): - raise _BlockCandidateCollapseFailedClosed( - "durable candidate intent carries no template" - ) - # The decode path refuses an intent whose template disagrees with - # its own parent/height fields. Reading either half of a - # disagreeing pair here would compare the chain against a fact the - # candidate does not actually carry. - if _collapse_block_hash(template.get("previousblockhash")) != parent_hash: - raise _BlockCandidateCollapseFailedClosed( - "durable candidate template parent disagrees with its intent" - ) - if _collapse_height(template.get("height")) != height: - raise _BlockCandidateCollapseFailedClosed( - "durable candidate template height disagrees with its intent" - ) - found_block = intent.get("found_block") - if not isinstance(found_block, dict): - raise _BlockCandidateCollapseFailedClosed( - "durable candidate intent carries no found-block facts" - ) - # PRISM-scaled units; see COLLAPSE_DIFFICULTY_SCALE and - # _BlockCandidateChainView.difficulty_of for the chain-side half of - # the comparison this feeds. - network_difficulty = _collapse_difficulty( - found_block.get("network_difficulty") - ) - if network_difficulty is None: - raise _BlockCandidateCollapseFailedClosed( - "durable candidate intent carries no usable network difficulty" - ) - pending_share = intent.get("pending_share") - job_id = ( - str(pending_share.get("job_id", "")) - if isinstance(pending_share, dict) - else "" - ) - return _SupersededCandidateRow( - block_hash=block_hash, - parent_hash=parent_hash, - height=height, - network_difficulty=network_difficulty, - job_id=job_id, - ) - - def _select_superseded_block_candidates( - self, - durable_rows: list[Any], - chain: _BlockCandidateChainView, - *, - ignore_leases: frozenset[str] = frozenset(), - ) -> list[_SupersededCandidateRow]: - """Apply predicate S to one fetched page of durable pending rows. - - Every clause is conjunctive and every unknown is a rejection. The - chain view bounds the cost: one best-tip and one tip-height read for - the page, one active-block and one header read per distinct - candidate height, and at most - MAX_BLOCK_CANDIDATE_COLLAPSE_HEIGHT_PROBES distinct heights for the - whole walk. A row whose height the walk cannot afford to probe is - counted and skipped, which preserves it for the per-row path exactly - as any other rejected clause does. - - ``ignore_leases`` names disposition flights the *caller itself* - holds, exactly as it does for the pre-write revalidation: those - flights are not treated as offer evidence, and every other member of - the evidence set still rejects. The replay-adoption page path passes - nothing and is unchanged; the dequeue-time skip (issue #181 item 2) - passes the single hash whose lease ``submit_next`` claimed before it - called here, because that flight is this pass and this pass has - made no node offer. See - ``_skip_superseded_block_candidate_at_dequeue`` for why no other - flight can hide behind it. - """ - # Clause 2 is read first, for the whole page, before a single chain - # round trip: a page that cannot answer it selects nothing at all, - # and a ledger whose page reader never carries the fact (a - # compatibility intent-only reader) must not spend a tip read per - # poll to discover that again. The list is page-bounded. - pool_block_facts = [ - self._collapse_pool_block_exists(durable_row) - for durable_row in durable_rows - ] - # Evidence is asked about this page's hashes only. A malformed row - # contributes no probe and is failed closed by - # ``_superseded_candidate_row`` below regardless; every row that - # survives that decode carries the row key probed here, because the - # decode refuses an intent whose hash disagrees with it. - evidence = self._block_candidate_collapse_evidence( - ( - durable_row.get("block_hash") - for durable_row in durable_rows - if isinstance(durable_row, dict) - ), - ignore_leases=ignore_leases, - ) - tip = chain.best_tip() - tip_height = chain.tip_height() - selected: list[_SupersededCandidateRow] = [] - seen: set[str] = set() - height_deferred = 0 - for pool_block_exists, durable_row in zip(pool_block_facts, durable_rows): - row = self._superseded_candidate_row(durable_row) - if row.block_hash in seen: - continue - seen.add(row.block_hash) - if pool_block_exists: - continue - if row.block_hash in evidence: - continue - # Clause 3: a candidate building on the current best tip is the - # next block, not a superseded sibling. - if row.parent_hash == tip: - continue - # Clause 4: only a height the chain has already decided, decided - # by somebody else. - if row.height > tip_height: - continue - active = chain.active_at(row.height) - if active is None: - # The walk has spent its bounded height probes. Nothing is - # known about this height, so nothing may be concluded about - # this row: it stays durable, keeps its evidence, and reaches - # the per-row path this pass and the next walk's collapse - # after that. - height_deferred += 1 - continue - if active == row.block_hash: - continue - # Clause 4b/5: a lower-work occupant is not a decision. The - # per-row path can still reorg a heavier sibling into the chain, - # and a terminal batch write would destroy that block. Both - # sides are PRISM-scaled integers: the row as the coordinator - # stamped it, the occupant as re-derived from its header bits. - if chain.difficulty_of(active) < row.network_difficulty: - continue - selected.append(row) - self._record_block_candidate_collapse("height_deferred", height_deferred) - return selected - - def _revalidate_superseded_block_candidates( - self, - leased: list[_SupersededCandidateRow], - chain: _BlockCandidateChainView, - ) -> tuple[list[_SupersededCandidateRow], str]: - """Re-read the chain under the held leases, immediately before the write. - - Selection ran before any lease existed, so a selected candidate - could have been offered, accepted, and become the active block in - the gap. Re-reading the best tip and each retained height's occupant - with the leases held closes it: nothing else can drive this hash - through a node offer while the lease is ours, and the fenced write - is the very next thing that happens. - - The occupant's header is re-read only when the occupying hash - changed; an unchanged occupant keeps the work value this walk - already read for that hash, so the steady case adds no header round - trip. - """ - # Budgeted for exactly the heights selection already probed and no - # more. The leased set is a subset of one bounded selection, so this - # can never refuse a re-read the write depends on, and it cannot draw - # on the walk's budget either -- the walk's remaining probes are for - # pages this apply has not seen. A budget of its own is a cache of - # its own as well: the walk's active-height reads, which may have - # been made pages ago, are never reused here, so every leased height - # is genuinely re-read from the chain under the held leases. - fresh = _BlockCandidateChainView( - self, - probe_budget=_CollapseHeightProbeBudget( - remaining=len({row.height for row in leased}), - ), - ) - tip = fresh.best_tip() - leased_hashes = frozenset(row.block_hash for row in leased) - evidence = self._block_candidate_collapse_evidence( - leased_hashes, - ignore_leases=leased_hashes, - ) - qualified: list[_SupersededCandidateRow] = [] - for row in leased: - if row.block_hash in evidence: - continue - if row.parent_hash == tip or row.block_hash == tip: - continue - # An occupied height is by construction at or below the tip, so - # this read subsumes the height bound as well as clause 4b. - active = fresh.active_at(row.height) - if active is None or active == row.block_hash: - # Unreachable while the budget above matches the leased - # heights, and a drop either way: an unprobed height is a row - # this write must not carry. - continue - # Same clause, same units as selection: a cache hit reuses the - # scaled integer that pass derived, and a changed occupant is - # re-derived from its own header bits rather than compared as a - # raw float against the row's scaled value. - difficulty = chain.cached_difficulty(active) - if difficulty is None: - difficulty = fresh.difficulty_of(active) - if difficulty < row.network_difficulty: - continue - qualified.append(row) - return qualified, tip - - def _note_block_candidate_collapse_fail_closed( - self, - rows: int, - detail: object, - ) -> bool: - """Count a fail-closed page and log it at a bounded rate. - - Returns whether this occasion logged, so a caller holding an - exception can attach its traceback under the same rate limit - instead of flooding a persistently degraded read. - """ - self._record_block_candidate_collapse("fail_closed", rows) - now = time.monotonic() - with self._coordinator.lock: - last = getattr( - self, - "_block_candidate_collapse_fail_closed_logged_monotonic", - None, - ) - due = ( - last is None - or (now - float(last)) - >= BLOCK_CANDIDATE_COLLAPSE_FAIL_CLOSED_LOG_SECONDS - ) - if due: - self._block_candidate_collapse_fail_closed_logged_monotonic = now - if due: - print( - "prism coordinator: superseded block candidate collapse failed " - f"closed rows={rows}: {detail}; every row keeps the per-row " - "path", - flush=True, - ) - return due - - def _collapsed_candidate_floor_holders( - self, - abandoned: Iterable[str], - ) -> dict[str, list[PrismBlockCandidate]]: - """Index the in-memory candidate objects owning each collapsed hash. - - The pending-share floor keys holders by object identity, so only the - exact object a lane is holding may be released and never a same-hash - sibling's. A durable row that was never adopted owns no object and - no holder, and none is invented for it: decoding one just to release - it would adopt a fresh floor holder in order to drop it. - """ - wanted = frozenset(abandoned) - holders: dict[str, list[PrismBlockCandidate]] = {} - if not wanted: - return holders - self._ensure_block_replay_state() - for queue_obj in ( - self.candidate_queue, - self._block_replay_candidate_queue, - ): - if queue_obj is None: - continue - with queue_obj.mutex: - queued = list(queue_obj.queue) - for item in queued: - key = _collapse_block_hash( - getattr( - getattr(item, "submission", None), - "block_hash_hex", - None, - ) - ) - if key in wanted: - holders.setdefault(key, []).append(item) - return holders - - def _run_collapsed_candidate_cleanup_steps( - self, - abandoned: tuple[str, ...], - *, - owed: dict[str, frozenset[str]], - shares: dict[str, tuple[PendingShare, ...]] | None = None, - ) -> tuple[dict[str, frozenset[str]], dict[str, tuple[PendingShare, ...]]]: - """Run each hash's owed cleanup steps and report what it still owes. - - Every step is idempotent and independently guarded, so a hash whose - step raises cannot strand the rest of the page mid-way, and a step - the caller does not list is simply not run -- which is what lets a - deferred retry replay only the remainder. - - Returns the still-owed steps per hash (absent means fully clean) and - the floor holders this pass resolved, so a caller that has to defer - the remainder keeps the exact holder objects it would otherwise lose - when the queues drain. - """ - coordinator = self._coordinator - coordinator._record_block_submitter_phase("replay-collapse-cleanup") - remaining: dict[str, set[str]] = {} - detailed_failures = 0 - - def note_failure(block_hash: str, step: str) -> None: - nonlocal detailed_failures - detailed = detailed_failures < BLOCK_CANDIDATE_COLLAPSE_LOG_FAILURES - remaining.setdefault(block_hash, set()).add(step) - if not detailed: - return - detailed_failures += 1 - print( - "prism coordinator: collapsed block candidate cleanup failed " - f"step={step} hash={block_hash}", - flush=True, - ) - traceback.print_exc() - - # One outer acquisition for the whole page. Each withdrawal takes - # this lock anyway (it is re-entrant); acquiring it per hash would - # hand a storm-sized page thousands of chances to interleave with - # descendant payout work between two halves of one hash's cleanup. - with self._block_submitter_lock( - coordinator._payout_balance_mutation_lock, - "payout-balance-mutation", - ): - for block_hash in abandoned: - if ( - BLOCK_CANDIDATE_COLLAPSE_CLEANUP_PAYOUT_STEP - not in owed.get(block_hash, frozenset()) - ): - continue - try: - # Withdraw with published invalidation, then drop the - # tombstone: the durable row is already terminal, so no - # replay source is left for the tombstone to fence, and - # a retained one is exactly the preview wait storm this - # collapse exists to end. - coordinator._clear_accepted_block_payout_preview( - block_hash, - invalidate_published=True, - ) - coordinator._clear_accepted_block_payout_preview(block_hash) - self._discard_accepted_landing_attribution(block_hash) - except Exception: - note_failure( - block_hash, - BLOCK_CANDIDATE_COLLAPSE_CLEANUP_PAYOUT_STEP, - ) - outcome = SimpleNamespace( - reason=PRISM_BLOCK_CANDIDATE_COLLAPSE_REASON, - stale_job_class=PRISM_BLOCK_CANDIDATE_COLLAPSE_STALE_JOB_CLASS, - ) - if shares is None: - # Unguarded on purpose: a page-scope indexing fault proves - # nothing about any hash's remaining steps, so the caller must - # see it as an abort and defer the whole won set. - shares = { - block_hash: tuple( - candidate.pending_share for candidate in candidates - ) - for block_hash, candidates in ( - self._collapsed_candidate_floor_holders(abandoned).items() - ) - } - for block_hash in abandoned: - steps = owed.get(block_hash, frozenset()) - - def run_step(step: str, action: Callable[[], None]) -> None: - if step not in steps: - return - try: - action() - except Exception: - note_failure(block_hash, step) - - def clear_finalize_retry() -> None: - with coordinator.lock: - self.finalize_retries.pop(block_hash, None) - - run_step("finalize-retry", clear_finalize_retry) - run_step( - "retry-state", - lambda: coordinator._clear_block_candidate_retry_state(block_hash), - ) - # Stops matching tip observations and drops the tip observation - # stamp in the same critical section. - run_step( - "outstanding-and-tip-observation", - lambda: coordinator._discard_outstanding_block_candidate(block_hash), - ) - # Also drops the fast-lane reservation, replay-inflight marker, - # and any parked same-hash retry object's own floor holder. - run_step( - "terminal-outcome", - lambda: coordinator._record_block_candidate_terminal_outcome( - block_hash, - accepted=False, - ), - ) - run_step( - "abandonment-accounting", - lambda: coordinator._record_committed_block_candidate_abandonment( - block_hash, - outcome, - ), - ) - for pending_share in shares.get(block_hash, ()): - run_step( - BLOCK_CANDIDATE_COLLAPSE_CLEANUP_FLOOR_STEP, - lambda pending_share=pending_share: ( - coordinator._finish_pending_share_commit(pending_share) - ), - ) - return ( - { - block_hash: frozenset(steps) - for block_hash, steps in remaining.items() - }, - shares, - ) - - def _publish_collapsed_candidate_terminal_fence( - self, - abandoned: Iterable[str], - ) -> None: - """Fence every won hash from node submission before any cleanup runs. - - The fenced batch write is the moment a row becomes durably terminal, - and the ``terminal-outcome`` cleanup step is what normally publishes - that fact in memory. That step can fail, or a page-level abort can - stop the pass before it -- and the failure leaves the hash in the - cleanup-retry registry while a same-hash candidate may still be - sitting in the live, replay, retry, or waiting lane. Once the apply - releases its disposition lease that candidate is dequeued, finds no - terminal outcome, and offers a durably abandoned block to the node. - - Publishing the outcome here closes that window with the fence the - rest of the submitter already honours. The apply holds every won - hash's disposition lease from before the write until after this - call, so no same-hash lane can be inside its own guarded region - while this runs, and none can enter one afterwards without seeing - the fence. It is a lock-guarded assignment into a dict the - disposition state already owns: no durable read or write, no queue - admission, and nothing that could re-adopt or re-offer a row. - - The full ``terminal-outcome`` step still runs in cleanup for its - other side effects -- the fast-lane reservation, the replay-inflight - marker, and the parked same-hash retry's own floor holder -- and - remains owed, and retried, whenever it fails. - - Contained so it cannot raise: this sits between a won write and the - page partition, and an exception escaping here would fail the page - open and replay-adopt rows whose durable outbox entries are gone. - """ - try: - self._ensure_block_candidate_disposition_state() - published: set[str] = set() - with self._coordinator.lock: - outcomes = self._block_candidate_terminal_outcomes - for block_hash in abandoned: - self._stamp_block_candidate_terminal_outcome( - outcomes, - block_hash, - False, - ) - published.add(block_hash) - self._bound_block_candidate_terminal_outcomes( - frozenset(published) - ) - except Exception: - # Degrades to the pre-publication behaviour: the cleanup step - # still owes the outcome and the retry registry still carries it. - print( - "prism coordinator: collapsed block candidate terminal fence " - "could not be published", - flush=True, - ) - traceback.print_exc() - - def _clean_up_collapsed_block_candidates( - self, - abandoned: tuple[str, ...], - ) -> frozenset[str]: - """Mirror per-row terminal abandonment cleanup for the rows we won. - - Driven by the hashes the fenced write returned, never by the hashes - it was asked about: a requested-but-absent row was won by somebody - else (or already terminal), and running this cleanup for it would - discard state its real owner still needs. - - Returns the hashes whose cleanup did not complete. The apply owns - the ``cleanup_failed`` accounting for the whole page: counting a - contained failure here and a later page-level abort there would - report one affected hash twice. Each of those hashes keeps its - still-owed steps in the cleanup-retry registry, because its durable - row is terminal and no enumeration will ever hand it back. Their - terminal fence is already published by the apply, so a step this - pass leaves owed can never leave one of them offerable to the node. - """ - every_step = frozenset(BLOCK_CANDIDATE_COLLAPSE_CLEANUP_STEPS) - remaining, shares = self._run_collapsed_candidate_cleanup_steps( - abandoned, - owed={block_hash: every_step for block_hash in abandoned}, - ) - for block_hash in abandoned: - owed = remaining.get(block_hash) - if owed: - self._defer_collapsed_candidate_cleanup( - block_hash, - owed, - shares=shares.get(block_hash, ()), - ) - else: - # A hash the fenced write can only win once, but a direct - # caller may repeat the cleanup; a completed pass discharges - # whatever an earlier one left owed. - self._discharge_collapsed_candidate_cleanup(block_hash) - if remaining: - print( - "prism coordinator: collapsed block candidate cleanup failed " - f"rows={len(remaining)} of {len(abandoned)}; their durable rows " - "are already terminal and their cleanup is retried", - flush=True, - ) - return frozenset(remaining) - - # -- deferred cleanup retry -------------------------------------------- - - def _collapsed_candidate_cleanup_registry( - self, - ) -> dict[str, _CollapsedCandidateCleanup]: - self._ensure_block_candidate_collapse_state() - return self._block_candidate_collapse_cleanup_retries - - def _defer_collapsed_candidate_cleanup( - self, - block_hash: str, - steps: Iterable[str], - *, - shares: Iterable[PendingShare] = (), - shares_resolved: bool = True, - ) -> None: - """Keep one terminal hash's owed cleanup steps for a later retry. - - Contained so it cannot raise: the apply's abort path calls this on - its way to returning the won set, and an exception escaping there - would fail the whole page closed and replay-adopt rows whose durable - outbox entries are already gone. - """ - try: - owed = frozenset(steps).intersection( - BLOCK_CANDIDATE_COLLAPSE_CLEANUP_STEPS - ) - if not owed: - return - registry = self._collapsed_candidate_cleanup_registry() - now = time.monotonic() - with self._coordinator.lock: - record = registry.pop(block_hash, None) - if record is None: - record = _CollapsedCandidateCleanup( - block_hash=block_hash, - steps=owed, - shares=tuple(shares), - shares_resolved=bool(shares_resolved), - delay_seconds=max(0.0, float(self.retry_initial_seconds)), - # The first attempt is due immediately; only a failed - # attempt starts the backoff. - not_before_monotonic=now, - deferred_monotonic=now, - ) - else: - # The floor keys holders by object identity, so merge by - # identity rather than by value: two reconstructed shares - # for one hash are two distinct holders. - by_identity = {id(share): share for share in record.shares} - for share in shares: - by_identity.setdefault(id(share), share) - record.steps = record.steps | owed - record.shares = tuple(by_identity.values()) - record.shares_resolved = record.shares_resolved and bool( - shares_resolved - ) - record.not_before_monotonic = min( - record.not_before_monotonic, - now, - ) - if not record.deferred_monotonic: - record.deferred_monotonic = now - # Re-inserted at the tail so a storm-sized backlog is retried - # round-robin instead of starving behind its oldest entry. - registry[block_hash] = record - except Exception: - print( - "prism coordinator: collapsed block candidate cleanup could " - f"not be deferred hash={block_hash}", - flush=True, - ) - traceback.print_exc() - - def _discharge_collapsed_candidate_cleanup(self, block_hash: str) -> bool: - """Drop one hash's retry record; True when one was still owed.""" - registry = self._collapsed_candidate_cleanup_registry() - with self._coordinator.lock: - return registry.pop(block_hash, None) is not None - - def collapsed_candidate_cleanup_backlog(self) -> dict[str, frozenset[str]]: - """The cleanup steps each terminal hash still owes (diagnostics).""" - registry = self._collapsed_candidate_cleanup_registry() - with self._coordinator.lock: - combined: dict[str, frozenset[str]] = {} - for source in ( - registry, - self._block_candidate_collapse_cleanup_inflight, - ): - for block_hash, record in source.items(): - combined[block_hash] = ( - combined.get(block_hash, frozenset()) | record.steps - ) - return combined - - # -- cleanup-retry backlog bound (#198) -------------------------------- - - def _collapse_cleanup_retry_backlog_max(self) -> int: - """Retry records the backlog may hold before admission stops. - - Read from the coordinator attribute first, so an embedder or a test - can pin it the way the sibling accounting knobs are pinned, then - from the loaded ``BlockConfig``, then the shipped default. Clamp it - into the same safe range enforced by startup validation. A direct - runtime value below one degrades into "admit one row at a time while - the backlog is empty"; one above the fence registry's capacity can - never weaken the hard memory bound. - """ - coordinator = self._coordinator - value = getattr( - coordinator, - "block_candidate_cleanup_retry_backlog_max", - None, - ) - if value is None: - block_config = getattr(getattr(coordinator, "config", None), "block", None) - value = getattr( - block_config, - "candidate_cleanup_retry_backlog_max", - None, - ) - if value is None: - value = DEFAULT_BLOCK_CANDIDATE_CLEANUP_RETRY_BACKLOG_MAX - return min( - MAX_BLOCK_CANDIDATE_CLEANUP_RETRY_BACKLOG_MAX, - max(1, int(value)), - ) - - def _collapse_cleanup_admission_headroom(self) -> tuple[int, int, int]: - """(rows the collapse may still admit, backlog depth, bound). - - Only the collapse callers on the block-submitter thread add records. - The accounting lane moves one record into the in-flight registry - while retrying it, so the owed authority remains counted until the - attempt either discharges or re-registers it. - """ - registry = self._collapsed_candidate_cleanup_registry() - maximum = self._collapse_cleanup_retry_backlog_max() - with self._coordinator.lock: - depth = len(registry) + len( - self._block_candidate_collapse_cleanup_inflight - ) - return max(0, maximum - depth), depth, maximum - - def _collapse_cleanup_backpressure_engagement_count(self) -> int: - """Return the monotonic engagement counter without walking records.""" - self._ensure_block_candidate_collapse_state() - with self._coordinator.lock: - return int(self._block_candidate_cleanup_backpressure_engagements) - - def collapsed_candidate_cleanup_backlog_snapshot(self) -> dict[str, Any]: - """Fixed-key gauges over the cleanup-retry backlog, for metrics. - - The key set is closed and carries no hash or step name. The walk is - O(backlog) under ``coordinator.lock``, which the admission bound - keeps at a few thousand entries at most; every value is a copy, so - the renderer never holds a reference into the registry. - - ``terminal_outcome_pins`` counts the records whose hash currently - holds a published terminal-outcome fence -- the entries the - registry forbids the fence eviction from dropping. It equals the - depth whenever the apply's fence publication succeeded; a smaller - value names records whose fence is still owed to the - ``terminal-outcome`` step. - """ - self._ensure_block_candidate_collapse_state() - self._ensure_block_candidate_disposition_state() - maximum = self._collapse_cleanup_retry_backlog_max() - now = time.monotonic() - with self._coordinator.lock: - registry = self._block_candidate_collapse_cleanup_retries - inflight = self._block_candidate_collapse_cleanup_inflight - outcomes = self._block_candidate_terminal_outcomes - records = tuple(registry.values()) + tuple(inflight.values()) - depth = len(records) - holder_ids: set[int] = set() - pinned_hashes: set[str] = set() - oldest: float | None = None - for record in records: - holder_ids.update(id(share) for share in record.shares) - if record.block_hash in outcomes: - pinned_hashes.add(record.block_hash) - stamp = float(record.deferred_monotonic) - if oldest is None or stamp < oldest: - oldest = stamp - engagements = int( - self._block_candidate_cleanup_backpressure_engagements - ) - return { - "depth": depth, - "backlog_max": maximum, - "oldest_age_seconds": ( - -1.0 if oldest is None else max(0.0, now - oldest) - ), - "pending_share_holders": len(holder_ids), - "terminal_outcome_pins": len(pinned_hashes), - "backpressure_active": depth >= maximum, - "backpressure_engagements": engagements, - } - - def _note_block_candidate_cleanup_backpressure( - self, - *, - caller: str, - rows: int, - admitted: int, - depth: int, - maximum: int, - ) -> None: - """Count rows the admission bound preserved and warn at a bounded rate. - - Every field of the warning is drawn from a closed vocabulary or is a - count: the caller name, the rows preserved, the rows this same pass - still admitted, the backlog depth and bound at the decision, and the - oldest record's age. No hash, parent, page cursor, or step name ever - appears, so a storm-long engagement cannot grow the log line's shape - and a persistent one cannot flood the journal. - """ - if caller not in PRISM_BLOCK_CANDIDATE_CLEANUP_BACKPRESSURE_CALLERS: - raise ValueError(f"unknown cleanup backpressure caller: {caller}") - if rows <= 0: - return - self._record_block_candidate_collapse("backlog_deferred", rows) - now = time.monotonic() - with self._coordinator.lock: - self._block_candidate_cleanup_backpressure_engagements = ( - int(self._block_candidate_cleanup_backpressure_engagements) + 1 - ) - last = self._block_candidate_cleanup_backpressure_logged_monotonic - due = ( - last is None - or (now - float(last)) - >= BLOCK_CANDIDATE_CLEANUP_BACKPRESSURE_LOG_SECONDS - ) - if due: - self._block_candidate_cleanup_backpressure_logged_monotonic = now - if not due: - return - oldest_age = float( - self.collapsed_candidate_cleanup_backlog_snapshot()["oldest_age_seconds"] - ) - print( - "prism coordinator: collapsed block candidate cleanup backpressure " - f"engaged caller={caller} rows_preserved={rows} admitted={admitted} " - f"backlog={depth} backlog_max={maximum} " - f"oldest_seconds={oldest_age:.3f}; admitting them to bulk " - "terminalization would take the cleanup-retry backlog past its " - "bound, so they stay durable and pending for the per-row path " - "until it drains", - flush=True, - ) - - def _run_one_collapsed_block_candidate_cleanup_retry(self) -> bool: - """Retry one deferred collapse cleanup; True when one was attempted. - - Driven from the accounting lane, because a terminal hash's cleanup - is accounting work and the durable row behind it is already gone. - The lane offers it two ways: immediately whenever the lane is truly - idle, and otherwise on an explicit cadence after at most - ``_block_accounting_cleanup_retry_work_items()`` completed work - items, so neither sustained accounting traffic nor a continuously - replenished quarantine queue can starve a due record. The pass is - bounded three ways: at most one hash per call, only that hash's - still-owed steps, and only once its own backoff deadline has passed. - It never reads or writes the outbox and never enqueues a candidate, - so it can only finish tearing state down -- a terminal row can be - neither re-adopted nor re-offered from here. - """ - registry = self._collapsed_candidate_cleanup_registry() - inflight = self._block_candidate_collapse_cleanup_inflight - now = time.monotonic() - with self._coordinator.lock: - due: _CollapsedCandidateCleanup | None = None - for record in registry.values(): - if ( - record.block_hash not in inflight - and record.not_before_monotonic <= now - ): - due = record - break - if due is None: - return False - # Move, rather than drop, the authority record for the duration - # of the attempt. Admission, gauges, and terminal-outcome - # eviction all count the in-flight registry, so cleanup can run - # outside coordinator.lock without opening an undercount or an - # unfenced window. A second lane also skips the same hash. - registry.pop(due.block_hash, None) - inflight[due.block_hash] = due - due.attempts += 1 - attempt_returned = False - try: - result = self._attempt_collapsed_block_candidate_cleanup_retry(due) - attempt_returned = True - return result - finally: - with self._coordinator.lock: - retained = inflight.pop(due.block_hash, None) - if ( - not attempt_returned - and retained is not None - and due.block_hash not in registry - ): - # An unexpected exception outside the contained step - # runner must not discard the only cleanup authority. - registry[due.block_hash] = retained - - def _attempt_collapsed_block_candidate_cleanup_retry( - self, - due: _CollapsedCandidateCleanup, - ) -> bool: - """Run one claimed retry while its authority remains in-flight.""" - self._coordinator._record_block_submitter_phase( - "replay-collapse-cleanup-retry" - ) - block_hash = due.block_hash - owed = due.steps - shares = due.shares - shares_resolved = due.shares_resolved - if ( - BLOCK_CANDIDATE_COLLAPSE_CLEANUP_FLOOR_STEP in owed - and not shares_resolved - ): - # The apply aborted before it could index the floor holders. The - # scan reads only the in-memory queues, so repeating it is cheap - # and safe; a candidate already dropped from them released its - # own holder on the way out. - try: - holders = self._collapsed_candidate_floor_holders((block_hash,)) - except Exception: - print( - "prism coordinator: collapsed block candidate cleanup " - f"retry could not index floor holders hash={block_hash}", - flush=True, - ) - traceback.print_exc() - self._record_block_candidate_collapse("cleanup_retry_failed") - self._reschedule_collapsed_candidate_cleanup( - due, - owed, - shares=shares, - shares_resolved=False, - ) - return True - by_identity = {id(share): share for share in shares} - for candidate in holders.get(block_hash, ()): - by_identity.setdefault( - id(candidate.pending_share), - candidate.pending_share, - ) - shares = tuple(by_identity.values()) - shares_resolved = True - try: - remaining, _shares = self._run_collapsed_candidate_cleanup_steps( - (block_hash,), - owed={block_hash: owed}, - shares={block_hash: shares}, - ) - still_owed = remaining.get(block_hash, frozenset()) - except Exception: - # Same reasoning as the apply's abort: a pass that died proves - # nothing about the steps it skipped, so the hash owes them all - # again. - print( - "prism coordinator: collapsed block candidate cleanup retry " - f"aborted hash={block_hash}", - flush=True, - ) - traceback.print_exc() - still_owed = owed - if still_owed: - self._record_block_candidate_collapse("cleanup_retry_failed") - self._reschedule_collapsed_candidate_cleanup( - due, - still_owed, - shares=shares, - shares_resolved=shares_resolved, - ) - return True - # Counted once per hash: the record is gone, so no later pass can - # report the same recovery twice, and the series can never exceed - # the cleanup_failed set that created these records. - self._record_block_candidate_collapse("cleanup_recovered") - print( - "prism coordinator: collapsed block candidate cleanup recovered " - f"hash={block_hash} attempts={due.attempts}", - flush=True, - ) - return True - - def _reschedule_collapsed_candidate_cleanup( - self, - record: _CollapsedCandidateCleanup, - steps: Iterable[str], - *, - shares: Iterable[PendingShare], - shares_resolved: bool, - ) -> None: - """Re-register a retried hash behind its own doubled backoff. - - The pacing mirrors ``next_retry_delay``: the configured candidate - retry backoff, doubled per attempt and capped, so a systemically - broken cleanup cannot spin the accounting lane while a transient one - still recovers on its next tick. - """ - initial = max(0.0, float(self.retry_initial_seconds)) - maximum = max(initial, float(self.retry_max_seconds)) - delay = min(maximum, max(initial, float(record.delay_seconds) * 2)) - self._defer_collapsed_candidate_cleanup( - record.block_hash, - steps, - shares=shares, - shares_resolved=shares_resolved, - ) - registry = self._collapsed_candidate_cleanup_registry() - with self._coordinator.lock: - rescheduled = registry.get(record.block_hash) - if rescheduled is None: - return - rescheduled.attempts = max(rescheduled.attempts, record.attempts) - rescheduled.delay_seconds = delay - rescheduled.not_before_monotonic = time.monotonic() + delay - # The retry took the record out of the registry before the - # attempt, so the re-registration above minted a fresh stamp; - # the age gauge measures from the first deferral, not the - # latest failure. - if record.deferred_monotonic: - rescheduled.deferred_monotonic = min( - rescheduled.deferred_monotonic or record.deferred_monotonic, - record.deferred_monotonic, - ) - - def _log_collapsed_block_candidates( - self, - qualified: list[_SupersededCandidateRow], - abandoned: frozenset[str], - *, - considered: int, - selected: int, - lease_skipped: int, - revalidation_dropped: int, - ) -> None: - """Emit one bounded summary instead of one line per candidate.""" - groups: dict[tuple[str, str], list[str]] = {} - for row in qualified: - if row.block_hash not in abandoned: - continue - groups.setdefault((row.parent_hash, row.job_id[:64]), []).append( - row.block_hash - ) - print( - "prism coordinator: collapsed superseded block candidates " - f"considered={considered} selected={selected} " - f"lease_skipped={lease_skipped} " - f"revalidation_dropped={revalidation_dropped} " - f"abandoned={len(abandoned)} " - f"reason={PRISM_BLOCK_CANDIDATE_COLLAPSE_REASON} " - f"stale_job_class={PRISM_BLOCK_CANDIDATE_COLLAPSE_STALE_JOB_CLASS} " - f"groups={len(groups)}", - flush=True, - ) - ordered = sorted(groups.items(), key=lambda item: (-len(item[1]), item[0])) - for (parent_hash, job_id), hashes in ( - ordered[:BLOCK_CANDIDATE_COLLAPSE_LOG_GROUPS] - ): - sample = ",".join( - sorted(hashes)[:BLOCK_CANDIDATE_COLLAPSE_LOG_SAMPLE_HASHES] - ) - print( - "prism coordinator: collapsed candidate group " - f"parent={parent_hash} job={job_id} " - f"reason={PRISM_BLOCK_CANDIDATE_COLLAPSE_REASON} " - f"count={len(hashes)} sample={sample}", - flush=True, - ) - remainder = len(ordered) - BLOCK_CANDIDATE_COLLAPSE_LOG_GROUPS - if remainder > 0: - print( - "prism coordinator: collapsed candidate groups not shown " - f"count={remainder}", - flush=True, - ) - - def _apply_superseded_block_candidate_collapse( - self, - selected: list[_SupersededCandidateRow], - chain: _BlockCandidateChainView, - *, - page_rows: int, - timeout_seconds: float | None, - call_class: str, - held_leases: frozenset[str] = frozenset(), - ) -> frozenset[str]: - """Lease, revalidate, write once, then clean up exactly what we won. - - ``held_leases`` names hashes whose disposition lease the caller - already holds and will release itself. They are neither claimed nor - released here, and they count as leased for every step that follows - -- including the pre-write revalidation's own self-exemption, which - already ignores the leases this apply is operating under. The - replay-adoption page path passes nothing and behaves exactly as - before; the dequeue-time skip passes the one hash ``submit_next`` - leased before it dequeued anything. - """ - coordinator = self._coordinator - coordinator._record_block_submitter_phase("replay-collapse-lease") - leases: dict[str, _BlockCandidateDispositionLease] = {} - try: - for row in selected: - if row.block_hash in held_leases: - continue - # Never block: a synchronous submit that has persisted its - # intent but not yet claimed its own lease must not queue - # behind a page of maintenance work. - lease = coordinator._claim_block_candidate_disposition( - row.block_hash, - blocking=False, - ) - if lease is not None: - leases[row.block_hash] = lease - leased = [ - row - for row in selected - if row.block_hash in leases or row.block_hash in held_leases - ] - self._record_block_candidate_collapse( - "lease_skipped", - len(selected) - len(leased), - ) - if not leased: - return frozenset() - coordinator._record_block_submitter_phase("replay-collapse-revalidate") - try: - qualified, tip = self._revalidate_superseded_block_candidates( - leased, - chain, - ) - except Exception as exc: - logged = self._note_block_candidate_collapse_fail_closed( - page_rows, - exc, - ) - if logged and not isinstance( - exc, - _BlockCandidateCollapseFailedClosed, - ): - traceback.print_exc() - return frozenset() - self._record_block_candidate_collapse( - "revalidation_dropped", - len(leased) - len(qualified), - ) - if not qualified: - return frozenset() - hashes = tuple(row.block_hash for row in qualified) - error = f"tip moved before submit: {tip}" - coordinator._record_block_submitter_phase("replay-collapse-write") - try: - mark = coordinator.ledger.mark_block_candidates_abandoned - returned = coordinator._run_block_submitter_ledger_call( - ("collapse-superseded", hashes), - "collapse-superseded", - lambda: mark(block_hashes=hashes, error=error), - timeout_seconds=timeout_seconds, - call_class=call_class, - ) - except Exception as exc: - # The rows stay pending, so nothing is lost: the whole page - # is preserved and the per-row path disposes of it. A - # timed-out call may still land on its bounded worker; the - # rows it transitions simply replay as terminal, which is - # the same convergence every other outbox mutation has. - if self._note_block_candidate_collapse_fail_closed(page_rows, exc): - traceback.print_exc() - return frozenset() - if not isinstance(returned, (list, tuple, set, frozenset)): - # The contract is a hash set, not a count or a boolean. A - # ledger that answers with something else has told us - # nothing about which rows it won. - self._note_block_candidate_collapse_fail_closed( - page_rows, - "fenced batch abandonment did not return a hash set", - ) - return frozenset() - requested = frozenset(hashes) - abandoned = frozenset( - key - for key in ( - _collapse_block_hash(value) for value in (returned or ()) - ) - # A returned hash outside the request would mean the ledger - # transitioned a row this apply never leased; refuse to run - # cleanup for it rather than tear down somebody else's state. - if key is not None and key in requested - ) - self._record_block_candidate_collapse( - "write_lost", - len(requested - abandoned), - ) - if not abandoned: - return frozenset() - self._record_block_candidate_collapse("abandoned", len(abandoned)) - # Before any cleanup, and while every won hash's lease is still - # held: a cleanup that fails or aborts must not leave a durably - # terminal row offerable to the node. - self._publish_collapsed_candidate_terminal_fence(abandoned) - try: - cleanup_failed = self._clean_up_collapsed_block_candidates( - tuple( - row.block_hash - for row in qualified - if row.block_hash in abandoned - ) - ) - except Exception: - # The rows are durably terminal whatever happened here, so - # the caller must still partition them out of the page: a - # cleanup fault cannot be allowed to replay-adopt a row - # whose outbox entry is gone. An abort proves nothing about - # the steps it skipped, so the whole won set is affected -- - # which subsumes any hash a contained step already failed. - cleanup_failed = abandoned - # Every won hash owes every step again, and the floor - # holders were never indexed, so the retry re-runs that scan - # too. Deferral is contained, so this stays on the path that - # returns the won set. - for block_hash in abandoned: - self._defer_collapsed_candidate_cleanup( - block_hash, - BLOCK_CANDIDATE_COLLAPSE_CLEANUP_STEPS, - shares_resolved=False, - ) - print( - "prism coordinator: collapsed block candidate cleanup " - f"aborted rows={len(abandoned)}; their durable rows are " - "already terminal and their cleanup is retried", - flush=True, - ) - traceback.print_exc() - if cleanup_failed: - # Counted once per apply over distinct hashes. Contained - # per-step failures and a later abort describe the same - # rows, so this series can never exceed the won set. - self._record_block_candidate_collapse( - "cleanup_failed", - len(cleanup_failed), - ) - try: - self._log_collapsed_block_candidates( - qualified, - abandoned, - considered=page_rows, - selected=len(selected), - lease_skipped=len(selected) - len(leased), - revalidation_dropped=len(leased) - len(qualified), - ) - except Exception: - # Diagnostics are not cleanup state. Keep a formatter fault - # visible without claiming hashes whose terminal cleanup - # completed successfully. - print( - "prism coordinator: collapsed block candidate logging " - f"failed rows={len(abandoned)}", - flush=True, - ) - traceback.print_exc() - return abandoned - finally: - for lease in leases.values(): - coordinator._release_block_candidate_disposition(lease) - - def _collapse_superseded_block_candidates( - self, - durable_rows: list[Any], - *, - timeout_seconds: float | None, - call_class: str, - probe_budget: _CollapseHeightProbeBudget | None = None, - ) -> list[Any]: - """Terminalize this page's decided-height siblings before adoption. - - Returns the rows that still have to be adopted: exactly the page - minus the hashes the fenced write actually transitioned. A won row's - fetched payload is a stale copy of an intent whose durable row is - now terminal, so it must never be replay-adopted or re-arm a payout - barrier. Anything short of a won row -- a fail-closed read, an - unclaimable lease, a revalidation drop, an unprobed height, a partial - return, a failed write -- preserves the row and leaves it to the - per-row path. - - ``probe_budget`` is the enumeration walk's shared chain-height - budget, carrying both the walk's remaining probes and the reads it - has already made; passes given none get a fresh one of their own. A - spent budget ends this page only when the page has nothing at a - height the walk already read: the counter bounds new heights, and a - cached height costs nothing to answer from however late in the walk - it is asked. - """ - if not durable_rows: - return durable_rows - if not callable( - getattr( - self._coordinator.ledger, - "mark_block_candidates_abandoned", - None, - ) - ): - # A ledger with no fenced batch abandonment has no safe bulk - # form at all. That is a structural absence, not a fail-closed - # read, so it neither counts nor spends a chain round trip: the - # per-row path disposes of every row exactly as before. - return durable_rows - if ( - probe_budget is not None - and probe_budget.exhausted - and not any( - _collapse_row_height(durable_row) in probe_budget.active - for durable_row in durable_rows - ) - ): - # This walk has spent its chain-height budget on earlier pages - # *and* not one row here sits at a height those pages already - # read. A spent counter only forbids reading a new height; a - # height already in the walk's cache stays answerable for free, - # and a page holding such rows still qualifies them below. This - # page holds none, so selection could not qualify a single row - # and the two page-scope tip reads it opens with would be two - # more round trips for a pass that is already decided. Every row - # is preserved. - # - # The peek is deliberately lenient: a row it cannot read a height - # from contributes no cached height, which at worst preserves a - # page the decode would have failed closed on anyway -- exactly - # what the unconditional early return did for every such page. - self._record_block_candidate_collapse( - "height_deferred", - len(durable_rows), - ) - return durable_rows - headroom, backlog_depth, backlog_max = ( - self._collapse_cleanup_admission_headroom() - ) - if headroom <= 0: - # Issue #198. The cleanup-retry backlog is at its bound: every - # row the fenced write could win here would add one more record - # that only the accounting lane can ever drain. Nothing is - # selected and no chain read is spent; every row stays durable - # and pending for the per-row path, exactly as an unprobed - # height leaves it. Rows already terminal keep their records, - # holders, and fences untouched -- this bounds admission, never - # cleanup authority. - self._note_block_candidate_cleanup_backpressure( - caller="replay-page", - rows=len(durable_rows), - admitted=0, - depth=backlog_depth, - maximum=backlog_max, - ) - return durable_rows - self._coordinator._record_block_submitter_phase("replay-collapse-select") - self._record_block_candidate_collapse("considered", len(durable_rows)) - chain = _BlockCandidateChainView(self, probe_budget=probe_budget) - try: - selected = self._select_superseded_block_candidates( - durable_rows, - chain, - ) - except Exception as exc: - logged = self._note_block_candidate_collapse_fail_closed( - len(durable_rows), - exc, - ) - if logged and not isinstance(exc, _BlockCandidateCollapseFailedClosed): - traceback.print_exc() - return durable_rows - if not selected: - return durable_rows - self._record_block_candidate_collapse("selected", len(selected)) - if len(selected) > headroom: - # Predicate S is unchanged: every one of these rows satisfied - # it and is counted as selected. Admission is what is bounded: - # the fenced write may win at most ``headroom`` more rows before - # the backlog reaches its bound, so only that many, in page - # order, are handed on. The remainder is preserved, not - # disposed of. - self._note_block_candidate_cleanup_backpressure( - caller="replay-page", - rows=len(selected) - headroom, - admitted=headroom, - depth=backlog_depth, - maximum=backlog_max, - ) - selected = selected[:headroom] - try: - abandoned = self._apply_superseded_block_candidate_collapse( - selected, - chain, - page_rows=len(durable_rows), - timeout_seconds=timeout_seconds, - call_class=call_class, - ) - except Exception as exc: - if self._note_block_candidate_collapse_fail_closed( - len(durable_rows), - exc, - ): - traceback.print_exc() - return durable_rows - if not abandoned: - return durable_rows - return [ - durable_row - for durable_row in durable_rows - if _collapse_block_hash( - durable_row.get("block_hash") - if isinstance(durable_row, dict) - else None - ) - not in abandoned - ] - - # -- dequeue-time stale sibling skip (issue #181 item 2) ---------------- - - def _block_candidate_dequeue_chain(self) -> _BlockCandidateChainView: - """A chain view over the submitter's own tip-epoch height cache. - - The replay walk shares one ``_CollapseHeightProbeBudget`` across its - pages so a storm at one decided height costs the walk one - ``getblockhash`` and one ``getblockheader`` however many pages it - spans. A dequeue burst has exactly that shape -- hundreds of - siblings of one decided height, arriving one candidate at a time -- - so it shares a budget the same way, and for the same reason: without - it every sibling would re-read the occupant and its header. - - The cache is retired by :meth:`_retire_block_candidate_dequeue_chain` - the moment the best tip changes, so its lifetime is exactly one tip - epoch. Only the height caches are shared: the tip itself is memoized - per view, so every candidate's selection still runs against a - freshly read best tip, and the pre-write revalidation builds its own - budget and re-reads the occupant under the held lease regardless. - - Submitter-thread state, like the replay walk's budget: ``submit_next`` - is the only caller and the block-submitter thread is the only thread - that reaches it. - """ - budget = getattr(self, "_block_candidate_dequeue_probe_budget", None) - if budget is None: - budget = _CollapseHeightProbeBudget() - self._block_candidate_dequeue_probe_budget = budget - self._block_candidate_dequeue_probe_tip: str | None = None - return _BlockCandidateChainView(self, probe_budget=budget) - - def _retire_block_candidate_dequeue_chain(self, tip: str) -> None: - """Drop every height this cache read under an earlier best tip. - - The best tip names the whole active chain beneath it: a block at any - height can only change by changing every descendant, so while the - tip hash is unchanged every ``getblockhash(H)`` below it answers - identically. Retiring exactly on a tip change therefore makes the - cache valid for precisely as long as it is kept, and returns the - bounded probe allowance so a long-lived submitter never stops - probing new heights. - - That is a stronger statement than the walk's cross-page sharing - needs, and the pre-write revalidation is unaffected either way: it - builds its own budget and re-reads the occupant from the chain under - the held lease before anything terminal happens. - """ - if getattr(self, "_block_candidate_dequeue_probe_tip", None) == tip: - return - budget = self._block_candidate_dequeue_probe_budget - budget.active.clear() - budget.difficulty.clear() - budget.remaining = MAX_BLOCK_CANDIDATE_COLLAPSE_HEIGHT_PROBES - self._block_candidate_dequeue_probe_tip = tip - - def _skip_superseded_block_candidate_at_dequeue( - self, - candidate: PrismBlockCandidate, - *, - timeout_seconds: float | None = None, - call_class: str = "fast", - ) -> bool: - """Terminalize one provably-stale dequeued candidate before any offer. - - Returns True when the durable row was abandoned and every piece of - this candidate's in-memory state was torn down, so ``submit_next`` - must release its lease and consume the wakeup without offering - anything. False means "offer it", for any reason at all: this is an - optimisation over the per-row path and declining it is always safe. - - Why this exists (issue #181, the 2026-08-20 spike): a candidate - whose parent is no longer the best tip and that carries no offer - evidence is provably stale *before* the offer, and the per-row path - spends one ``submitblock``, ~6 chain reads and two ledger writes - discovering that -- plus an accounting task, a fast-lane - reservation, and an accepted-block payout-preview barrier armed and - withdrawn -- for each one. Those are the individual - ``block candidate abandoned reason=stale-job: tip moved before - submit`` bursts the 2026-08-21 validation still showed for - live/fast-path siblings after #196 removed the replay population. - - The predicate is #196's, unchanged: same evidence set, same - clauses, same fencing, same cleanup, same terminal write. This is a - second *caller*, not a second notion of staleness. The one - divergence is the self-exemption below. - - **The self-exemption, and why nothing else can hide behind it.** - ``submit_next`` claims this hash's disposition lease before it gets - here, so the hash is in ``_block_candidate_disposition_flights`` -- - a member of #196's evidence set E -- and #196's selector would - reject the row on clause 1 forever. Exactly one flight is exempted: - the one this pass holds the lock of, named by this candidate's own - hash and passed as ``ignore_leases``/``held_leases``. No other - flight can be exempted, because no other hash is ever in that set; - and no other *holder* of this flight can hide behind the exemption, - because a flight's registry entry is shared by its holder and its - waiters while its lock has exactly one owner -- this pass. A pass - merely waiting on the lock has offered nothing, and once the fenced - write lands, ``_publish_collapsed_candidate_terminal_fence`` stamps - the terminal outcome while the lease is still ours, so the waiter - wakes into the fence rather than into an offer. Every other member - of E still rejects this hash: retry holders, the deferred accounting - retry, ``_block_disposition_waiting_retries``, ``finalize_retries``, - ``_block_candidate_retained_node_submissions``, - ``_tip_observed_accepted_block_hashes``, - ``_accounted_accepted_block_hashes``, - ``_block_candidate_terminal_outcomes``, and a ``qbit_pool_blocks`` - row in any state. - - **Chain, not observation set.** Replay adoption does not register a - hash outstanding, so a blockwait observation of a replayed hash is - dropped until dequeue and the in-memory tip view says nothing about - it. Clause 3 is therefore decided against a freshly read - ``getbestblockhash``, never against the coordinator's published tip. - - **Fail closed.** Any unreadable fact -- a chain read, the pool-block - probe, an intent this candidate cannot answer, a fenced write that - did not return this hash -- preserves the candidate for the offer - path. Nothing is ever abandoned on an unknown. - - One diagnostic note: because this reuses #196's selection and apply - verbatim, the submitter phase stamps those emit still read - ``replay-collapse-*``. They name the machinery, which is shared, not - the caller. The ``dequeue_considered``/``dequeue_skipped``/ - ``dequeue_preserved`` counters are what separate this caller's - population from the replay walk's, and the one phase this method - stamps itself -- ``dequeue-collapse-pool-block`` -- names the only - durable read it adds. - """ - coordinator = self._coordinator - ledger = coordinator.ledger - if not callable(getattr(ledger, "mark_block_candidates_abandoned", None)): - # A ledger with no fenced batch abandonment has no safe bulk form - # at all; a structural absence, so it neither counts nor spends a - # round trip. Same reasoning as the page-level driver. - return False - pool_block_reader = getattr(ledger, "pool_block_state", None) - if not callable(pool_block_reader): - # Clause 2 has no durable answer here. Reading it as false would - # abandon exactly the rows that must never be abandoned. - return False - block_hash = _block_candidate_hash_of(candidate) - if block_hash is None: - return False - headroom, backlog_depth, backlog_max = ( - self._collapse_cleanup_admission_headroom() - ) - if headroom <= 0: - # Issue #198: same bound, same answer as the page walk. The - # candidate is not considered at all -- the dequeue partition - # counters stay exact -- and it takes the ordinary offer path, - # whose per-row disposition has its own cleanup contract. - self._note_block_candidate_cleanup_backpressure( - caller="dequeue", - rows=1, - admitted=0, - depth=backlog_depth, - maximum=backlog_max, - ) - return False - held = frozenset((block_hash,)) - self._record_block_candidate_collapse("dequeue_considered") - try: - if self._block_candidate_collapse_evidence(held, ignore_leases=held): - # Clause 1 first, because it is the only clause that costs - # nothing: the selector re-asks it below, but asking here - # keeps both the chain round trip and the durable pool-block - # probe off every candidate some evidence already excludes. - self._record_block_candidate_collapse("dequeue_preserved") - return False - parent_hash = _collapse_block_hash( - candidate.context.template["previousblockhash"] - ) - if parent_hash is None: - raise _BlockCandidateCollapseFailedClosed( - "dequeued candidate carries no usable parent hash" - ) - chain = self._block_candidate_dequeue_chain() - tip = chain.best_tip() - self._retire_block_candidate_dequeue_chain(tip) - if parent_hash == tip: - # Clause 3, short-circuited before any durable read: this is - # the block waiting to be offered, not a superseded sibling. - # Keeping it first is what holds the acceptance path's added - # cost to a single getbestblockhash. - # - # That read is not replaceable by the coordinator's published - # tip (``_current_published_tip_hash_locked``), tempting as a - # free in-memory gate is. Published work is what a *blocked* - # refresh stops updating, and a blocked refresh is #181's - # symptom: during the incident the published tip sits at the - # very parent every stale sibling names, so a gate on it - # would decline to skip exactly the population this exists - # for. The chain is asked instead, which is also what D5 - # requires for replay-adopted rows. - self._record_block_candidate_collapse("dequeue_preserved") - return False - pool_block_exists = ( - coordinator._run_block_submitter_ledger_call( - ("dequeue-collapse-pool-block", block_hash), - "dequeue-collapse-pool-block", - lambda: pool_block_reader(block_hash=block_hash), - timeout_seconds=timeout_seconds, - call_class=call_class, - ) - is not None - ) - selected = self._select_superseded_block_candidates( - [ - _dequeued_candidate_collapse_row( - candidate, - pool_block_exists=pool_block_exists, - ) - ], - chain, - ignore_leases=held, - ) - except Exception as exc: - logged = self._note_block_candidate_collapse_fail_closed(1, exc) - if logged and not isinstance(exc, _BlockCandidateCollapseFailedClosed): - traceback.print_exc() - self._record_block_candidate_collapse("dequeue_preserved") - return False - if not selected: - self._record_block_candidate_collapse("dequeue_preserved") - return False - self._record_block_candidate_collapse("selected", len(selected)) - try: - abandoned = self._apply_superseded_block_candidate_collapse( - selected, - chain, - page_rows=1, - timeout_seconds=timeout_seconds, - call_class=call_class, - held_leases=held, - ) - except Exception as exc: - if self._note_block_candidate_collapse_fail_closed(1, exc): - traceback.print_exc() - self._record_block_candidate_collapse("dequeue_preserved") - return False - if block_hash not in abandoned: - # Anything short of a won row -- an unclaimable lease, a - # revalidation drop, a lost fenced write -- is preserved. - self._record_block_candidate_collapse("dequeue_preserved") - return False - # The floor holder is bound to the *queued object's* identity, and - # this object has already left its lane, so the apply's own - # pending-share step -- which indexes holders by scanning the live - # and replay queues -- cannot see it. Release it here, through the - # same seam ``submit_next`` uses for a dropped duplicate. Correct - # whether the apply's cleanup completed or was deferred: a deferred - # retry has no queue left to find this object in either. - self._release_dropped_duplicate_candidate_floor(candidate) - self._record_block_candidate_collapse("dequeue_skipped") - return True - - def _block_replay_should_yield_to_live_candidates( - self, - probe_budget: _CollapseHeightProbeBudget, - ) -> bool: - """Whether this walk should hand the submitter back before another page. - - The block-submitter thread that walks these pages is the only thread - that reaches ``submitblock``, so a backlog it keeps walking is a - backlog that delays a live solve. Between pages -- never inside one - page's fenced collapse, its write, or its adoption -- the walk gives - that thread back as soon as two things hold at once: - - * a live candidate or a retry is waiting, so there is something - strictly more valuable than finishing the enumeration; and - * the walk has already spent its chain-height probes, so the pages it - would go on to fetch can no longer decide a height it has not - already read. - - Both halves matter. Yielding on a waiting candidate alone would cut - short the same-height storm this path exists for, which walks several - pages on a single probe. Yielding on a spent budget alone would leave - the enumeration owed -- and job builds blocked -- for a walk that had - nothing better to do than finish. The caller re-enumerates from the - outbox on its next pass with a fresh budget, and every row this walk - fetched was adopted before the yield, so the backlog still shrinks. - - A spent budget is not the same as nothing left to collapse: the pages - past the yield may hold siblings at heights this walk already read, - and those would have collapsed for free. What the yield gives up is - therefore bounded and recovered -- those rows stay durable and pending, - and the next walk re-probes their heights on a fresh budget -- while - what it buys is the only thread that reaches ``submitblock``. - """ - if not probe_budget.exhausted: - return False - with self._coordinator.lock: - if self.retry_candidate is not None: - return True - queue_obj = self.candidate_queue - return queue_obj is not None and not queue_obj.empty() - - # -- targeted ancestor re-drive (issue #190) --------------------------- - - def _ancestor_redrive_defer_threshold(self) -> int: - return max( - 1, - int( - getattr( - self._coordinator, - "accepted_parent_redrive_defer_threshold", - DEFAULT_ACCEPTED_PARENT_REDRIVE_DEFER_THRESHOLD, - ) - ), - ) - - def _ancestor_redrive_attempt_cap(self) -> int: - """Re-drives one ancestor may trigger; zero disables the mechanism.""" - return max( - 0, - int( - getattr( - self._coordinator, - "accepted_parent_redrive_attempt_max", - DEFAULT_ACCEPTED_PARENT_REDRIVE_ATTEMPT_MAX, - ) - ), - ) - - def _evict_stale_redrive_entries_locked(self) -> None: - """Bound per-hash re-drive bookkeeping. Caller holds the runtime lock. - - Entries are dropped on resolution, so eviction only engages under a - pathological stream of distinct never-resolving ancestors. The - deferral path re-inserts its ancestor's record and its child's - blocking entry on every deferral, so an actively wedged pair sits at - the back of each registry and is the last thing an oldest-first - eviction touches -- exhausted records included, whose survival is - what keeps the per-ancestor cap a lifetime bound. The two registries - are keyed by different hash spaces (ancestors vs children), so each - is trimmed strictly against itself: a hash that is both a child and - an ancestor must not lose its ancestor record to the eviction of a - stale child entry. - """ - while len(self._ancestor_redrive_records) > MAX_ANCESTOR_REDRIVE_TRACKED_HASHES: - self._ancestor_redrive_records.pop( - next(iter(self._ancestor_redrive_records)), None - ) - while ( - len(self._ancestor_redrive_last_blocking) - > MAX_ANCESTOR_REDRIVE_TRACKED_HASHES - ): - self._ancestor_redrive_last_blocking.pop( - next(iter(self._ancestor_redrive_last_blocking)), None - ) - - def note_pending_parent_transition_deferral( - self, - block_hash: str, - ancestor_hash: str, - ) -> None: - """Track one finalization deferral against its blocking ancestor. - - Called by the coordinator's pending-parent fence each time a - candidate's finalization defers because ``ancestor_hash``'s accepted - payout transition is unresolved. The deferral itself only re-checks - the transition, so a streak of them proves the retry loop cannot - resolve the ancestor on its own; on crossing the configured - threshold a targeted durable-replay pass is armed for the submitter - loop. Bounded per ancestor by the attempt cap, past which deferrals - fall back to exactly the pre-#190 behavior with the - publication-progress watchdog as the backstop. - """ - cap = self._ancestor_redrive_attempt_cap() - if cap <= 0: - # Zero disables the mechanism outright: no streaks, no armed - # passes, and no exhaustion accounting. A disabled install must - # not emit the alert-worthy exhausted signal a spent cap means. - return - child = str(block_hash).lower() - ancestor = str(ancestor_hash).lower() - threshold = self._ancestor_redrive_defer_threshold() - requested = False - exhausted = False - attempts = 0 - streak = 0 - self._ensure_block_replay_state() - with self._coordinator.lock: - # Re-inserted (not updated in place) so insertion order tracks - # recency and the eviction above stays oldest-first. - self._ancestor_redrive_last_blocking.pop(child, None) - self._ancestor_redrive_last_blocking[child] = ancestor - record = self._ancestor_redrive_records.pop(ancestor, None) - if record is None: - record = _AncestorRedriveRecord() - self._ancestor_redrive_records[ancestor] = record - # A replay-lane copy of the ancestor means in-process resolution - # is already underway -- an earlier re-drive's adoption, or - # ordinary replay. Deferrals during that window prove nothing a - # new pass could act on (it would no-op on the adopted check), - # so they must not bank toward attempts: burning the cap against - # a mechanism that is mid-fix would exhaust it before slow-but- - # working finalization completes, standing the re-drive down for - # an ancestor it had already reached. - ancestor_inflight = ancestor in self._block_replay_inflight_hashes - if not record.armed and not record.exhausted and not ancestor_inflight: - record.streak += 1 - streak = record.streak - if record.streak >= threshold: - if record.attempts < cap: - record.attempts += 1 - record.armed = True - record.streak = 0 - attempts = record.attempts - self.accepted_parent_redrive_attempt_count = ( - int(self.accepted_parent_redrive_attempt_count) + 1 - ) - requested = True - else: - record.exhausted = True - attempts = record.attempts - self.accepted_parent_redrive_exhausted_count = ( - int(self.accepted_parent_redrive_exhausted_count) + 1 - ) - exhausted = True - self._evict_stale_redrive_entries_locked() - if requested: - print( - "prism coordinator: arming in-process ancestor re-drive " - f"ancestor={ancestor} child={child} deferral_streak={streak} " - f"attempt={attempts}/{cap}", - flush=True, - ) - if exhausted: - print( - "prism coordinator: ancestor re-drive attempts exhausted " - f"ancestor={ancestor} child={child} attempts={attempts}; " - "deferrals continue and the publication-progress watchdog " - "remains the backstop", - flush=True, - ) - - def note_pending_parent_transition_resolved(self, block_hash: str) -> None: - """Drop re-drive bookkeeping once a child's ancestor fence passes.""" - child = str(block_hash).lower() - resolved_ancestor: str | None = None - with self._coordinator.lock: - ancestor = self._ancestor_redrive_last_blocking.pop(child, None) - if ancestor is None: - return - record = self._ancestor_redrive_records.pop(ancestor, None) - for key in [ - key - for key, value in self._ancestor_redrive_last_blocking.items() - if value == ancestor - ]: - self._ancestor_redrive_last_blocking.pop(key, None) - if record is not None and record.consumed > 0: - # Counted only when a forced pass actually ran: a transition - # that resolved through the ordinary landing tail while a - # request sat armed-but-unconsumed is the mechanism standing - # by, not succeeding, and must not inflate the resolved - # series operators compare against attempts. - self.accepted_parent_redrive_resolved_count = ( - int(self.accepted_parent_redrive_resolved_count) + 1 - ) - resolved_ancestor = ancestor - if resolved_ancestor is not None: - print( - "prism coordinator: pending ancestor payout transition " - "resolved after in-process re-drive " - f"ancestor={resolved_ancestor} child={child}", - flush=True, - ) - - def _ancestor_redrive_owed(self) -> bool: - """Whether a forced durable-replay pass is armed for any ancestor.""" - with self._coordinator.lock: - return any( - record.armed - for record in self._ancestor_redrive_records.values() - ) - - def _consume_ancestor_redrive_requests(self) -> tuple[str, ...]: - """Take (and clear) every armed re-drive; one forced pass serves all. - - Consumed up front deliberately: a pass whose enumeration fails burns - the attempt rather than re-running unbounded, a fresh deferral - streak re-arms the next attempt, and the per-ancestor cap bounds the - total. - """ - with self._coordinator.lock: - consumed = tuple( - ancestor - for ancestor, record in self._ancestor_redrive_records.items() - if record.armed - ) - for ancestor in consumed: - record = self._ancestor_redrive_records[ancestor] - record.armed = False - record.consumed = record.attempts - return consumed - - def _block_candidate_owned_in_process(self, block_hash: str) -> bool | None: - """Whether any live lane still names this hash, or None if unreadable. - - Reads exactly the pin set the terminal-outcome eviction trusts, via - the shared collector, so the two proofs cannot drift apart. None - means a leaf lane could not be read without waiting under the global - lock; callers treat that as owned, which is the safe direction. - """ - key = str(block_hash).lower() - self._ensure_block_candidate_disposition_state() - self._ensure_block_replay_state() - with self._coordinator.lock: - pins = self._collect_live_block_candidate_pins() - if pins is None: - return None - pinned, held = pins - if key in held: - return True - return any(key in registry for registry in pinned) - - def _resolve_unreplayable_ancestor_transition(self, block_hash: str) -> None: - """Converge a stuck transition with durable state when replay cannot. - - A forced enumeration that adopts the ancestor's pending outbox row is - the ordinary re-drive; this handles the remaining wedge shape, where - the armed transition has no pending durable row left -- the state a - process restart resolves simply by not rebuilding the transition. It - is cleared here only under the same proof a fresh startup replay - would compute: the completed enumeration found no pending row for - the hash, no in-process lane still owns a copy that could finish (or - withdraw) the transition itself, the durable pool-block row reports - the block confirmed and not reversed, and a fresh chain probe places - the block's own hash on the active chain at its height -- the exact - predicate the landing's already-confirmed branch clears the preview - under, which consults the durable row only for a block it has just - proven active. The probe matters because a reorg qbitd has seen but - the reconciler has not yet written back (reconciliation fails closed - while a landed transition exists) leaves the durable row - stale-confirmed, and clearing on the row alone would unfence - descendants onto the orphaned block's carry. Anything short of the - full proof leaves the transition alone and the watchdog remains the - backstop. - """ - coordinator = self._coordinator - coordinator._ensure_job_cache_state() - key = str(block_hash).lower() - with coordinator._accepted_block_payout_preview_condition: - transition = coordinator._accepted_block_payout_previews.get(key) - if transition is None: - return - if self._block_candidate_owned_in_process(key) is not False: - print( - "prism coordinator: ancestor re-drive left the transition to " - f"its in-process owner hash={key}", - flush=True, - ) - return - block_state_reader = getattr(coordinator.ledger, "pool_block_state", None) - if not callable(block_state_reader): - return - try: - block_state = coordinator._run_block_submitter_ledger_call( - ("redrive-pool-block-state", key), - "redrive-pool-block-state", - lambda: block_state_reader(block_hash=key), - ) - except Exception: - print( - "prism coordinator: ancestor re-drive pool-block state read " - f"failed hash={key}", - flush=True, - ) - traceback.print_exc() - return - confirmed = ( - isinstance(block_state, dict) - and str(block_state.get("chain_state", "")) == "confirmed" - and str(block_state.get("maturity_state", "")) != "reversed" - ) - if not confirmed: - print( - "prism coordinator: ancestor re-drive could not resolve the " - f"transition hash={key} (no pending outbox row, pool block " - "not durably confirmed); watchdog remains the backstop", - flush=True, - ) - return - # The durable row alone is not proof against a reorg the fenced - # reconciler has not written back yet; require the same node-side - # activity the landing proves before it trusts this row. Anything - # but a definite True (probe failure included) stands down. - chain_probe = self._block_candidate_chain_probe( - key, - expected_height=getattr(transition, "block_height", None), - ) - if chain_probe is not True: - print( - "prism coordinator: ancestor re-drive could not prove the " - f"durably confirmed block active on the node's chain " - f"hash={key} probe={chain_probe}; watchdog remains the " - "backstop", - flush=True, - ) - return - # Durable state already includes this block's payout carry, so - # removing the in-memory override changes no logical payout state -- - # the same clear the landing performs for an already-confirmed - # exact-idempotent replay. - coordinator._clear_accepted_block_payout_preview(key) - print( - "prism coordinator: ancestor re-drive cleared a stale transition " - f"for durably confirmed block hash={key}", - flush=True, - ) - - def _conclude_ancestor_redrive_pass( - self, - redrive_hashes: tuple[str, ...], - enumeration_truncated: bool, - ) -> None: - """Disposition each consumed re-drive after its enumeration ran. - - A truncated pass proved nothing about missing rows, so every - transition is left for the next attempt (a fresh deferral streak - re-arms it) or the watchdog. A complete pass either finds the - ancestor (re-)adopted -- ordinary replay finalization owns it from - there -- or runs the guarded stale-transition sweep. - """ - if not redrive_hashes: - return - if enumeration_truncated: - print( - "prism coordinator: ancestor re-drive enumeration was " - "truncated; leaving pending transitions for the next " - "attempt or the watchdog", - flush=True, - ) - return - for redrive_hash in redrive_hashes: - key = str(redrive_hash).lower() - with self._coordinator.lock: - adopted = key in self._block_replay_inflight_hashes - if adopted: - # The pending outbox row was (re-)adopted -- by this pass - # or an earlier one -- so ordinary replay finalization now - # owns resolving the transition. - print( - "prism coordinator: ancestor re-drive enumerated a " - f"pending candidate hash={key}", - flush=True, - ) - continue - self._resolve_unreplayable_ancestor_transition(key) - - def replay_pending(self) -> int: - """Queue durable candidate intents not completed by an earlier process.""" - self._coordinator._record_block_submitter_phase("replay-check-memory") - self._ensure_block_replay_state() - # The normal submitter loop drains replay work before asking the - # outbox again. Enforce that ordering here too for direct callers: a - # forced walk paused by cleanup backpressure must not append another - # page to the unbounded replay queue before its bounded page drains. - if self._block_replay_backpressure_drain_pending: - if not self._block_replay_candidate_queue.empty(): - return 0 - self._block_replay_backpressure_drain_pending = False - # While startup enumeration is still owed, correctness requires the - # outbox query even if live candidates are queued: job builds stay - # blocked until pending candidates are known, and only a successful - # enumeration can unblock them. - enumeration_owed = self._coordinator._block_replay_enumeration_owed() - # A targeted ancestor re-drive (issue #190) must reach the outbox - # query even while a retained retry or queued live work exists -- - # those short-circuits are exactly what starved this path during the - # observed wedge -- so an armed request bypasses each of them below. - redrive_hashes = self._consume_ancestor_redrive_requests() - redrive_owed = bool(redrive_hashes) - # One flag for "this pass must reach and complete the outbox query". - # The startup gate and a targeted re-drive need the same enumeration - # semantics -- the short-circuit bypasses, the query budget, the - # keyset pagination, and the widening-window exit below all key off - # it together -- so the re-drive genuinely runs the code startup - # replay runs rather than a poll-budget approximation of it. - forced_enumeration = enumeration_owed or redrive_owed - with self._coordinator.lock: - if not forced_enumeration and self.retry_candidate is not None: - return 0 - # A live wakeup is already the lowest-latency route to qbitd. Never - # park it behind the outbox query that exists only to recover missing - # wakeups after queue pressure or restart. - queue_obj = self.candidate_queue - if ( - not forced_enumeration - and queue_obj is not None - and not queue_obj.empty() - ): - return 0 - if ( - not forced_enumeration - and not self._block_replay_candidate_queue.empty() - ): - return 0 - # A forced enumeration gates wedge resolution (startup: job - # issuance; re-drive: a stuck payout transition), so it runs with - # the landing-class budget instead of the poll budget (issue #188 - # fix 4); the periodic steady-state poll keeps the tight budget. The - # metrics class follows the budget so a slow or timed-out forced - # enumeration surfaces on the landing series the alerts watch. - replay_query_timeout = ( - self._coordinator._block_landing_db_timeout() - if forced_enumeration - else None - ) - replay_query_call_class = "landing" if forced_enumeration else "fast" - pending_rows = getattr( - self._coordinator.ledger, - "pending_block_candidate_rows", - None, - ) - fetch_durable_page: Callable[..., list[Any]] | None = None - # Issue #255: a ledger that can answer metadata-first pages is - # enumerated through them -- typed headers only, a hard row and byte - # cap per page, and exhaustion answered by the outbox rather than - # inferred from a short page. Bodies are hydrated one at a time when - # a descriptor is dequeued, never during enumeration. - pending_headers = getattr( - self._coordinator.ledger, - "pending_block_candidate_headers", - None, - ) - fetch_header_page: Callable[..., Any] | None = None - # A ledger whose row query was replaced by a cursorless double keeps - # exactly the windowed semantics its replacement models. - if callable(pending_headers) and ( - not callable(pending_rows) or _pending_rows_accepts_cursor(pending_rows) - ): - - def fetch_header_page( - limit: int, - *, - page: int, - after_cursor: object | None, - ) -> Any: - return self._coordinator._run_block_submitter_ledger_call( - ( - "replay-outbox-query", - limit, - page, - _block_replay_cursor_key(after_cursor), - ), - "replay-outbox-query", - lambda: pending_headers( - limit=limit, - after_cursor=after_cursor, - max_bytes=HEADER_PAGE_MAX_BYTES, - ), - timeout_seconds=replay_query_timeout, - call_class=replay_query_call_class, - ) - - def fetch_durable_rows(limit: int) -> list[Any]: - header_limit = min(int(limit), HEADER_PAGE_MAX_ROWS) - header_page = self._coordinator._run_block_submitter_ledger_call( - ("replay-outbox-query", header_limit), - "replay-outbox-query", - lambda: pending_headers( - limit=header_limit, - after_cursor=None, - max_bytes=HEADER_PAGE_MAX_BYTES, - ), - timeout_seconds=replay_query_timeout, - call_class=replay_query_call_class, - ) - return [_collapse_row_from_header(row) for row in header_page.rows] - - elif callable(pending_rows): - - def fetch_durable_rows(limit: int) -> list[Any]: - return self._coordinator._run_block_submitter_ledger_call( - ("replay-outbox-query", limit), - "replay-outbox-query", - # Restore a batch with no per-row database work. In-flight - # dedupe lets later rows reach qbitd even while the oldest - # candidate is still accounting. - lambda: pending_rows(limit=limit), - timeout_seconds=replay_query_timeout, - call_class=replay_query_call_class, - ) - - if _pending_rows_accepts_cursor(pending_rows): - - def fetch_durable_page( - limit: int, - *, - page: int, - after_cursor: object | None, - ) -> list[Any]: - return self._coordinator._run_block_submitter_ledger_call( - # The page ordinal and its cursor belong in the - # dedupe key: a timed-out call stays registered for - # the next paced retry to reuse, and reusing page - # N's in-flight call to answer page N+1 would - # silently drop a whole page of pending candidates - # from the enumeration. - ( - "replay-outbox-query", - limit, - page, - _block_replay_cursor_key(after_cursor), - ), - "replay-outbox-query", - lambda: pending_rows( - limit=limit, - after_cursor=after_cursor, - ), - timeout_seconds=replay_query_timeout, - call_class=replay_query_call_class, - ) - - else: - pending = getattr( - self._coordinator.ledger, - "pending_block_candidates", - None, - ) - if not callable(pending): - self._coordinator._clear_block_replay_enumeration_owed() - return 0 - - def fetch_durable_rows(limit: int) -> list[Any]: - pending_intents = self._coordinator._run_block_submitter_ledger_call( - ("replay-outbox-query", limit), - "replay-outbox-query", - lambda: pending(limit=limit), - timeout_seconds=replay_query_timeout, - call_class=replay_query_call_class, - ) - return [ - { - "block_hash": ( - intent.get("block_hash_hex", "") - if isinstance(intent, Mapping) - else "" - ), - "candidate": intent, - } - for intent in pending_intents - ] - - queued = 0 - enumeration_truncated = False - enumeration_paginated = False - # One chain-height budget for the whole walk -- both its remaining - # probes and the reads they bought -- so a backlog split across fifty - # pages costs the node what a single page does. - collapse_probe_budget = _CollapseHeightProbeBudget() - if forced_enumeration and ( - fetch_header_page is not None or fetch_durable_page is not None - ): - # A row can embed an entire payout window. Smaller pages reduce - # the uninterrupted JSON decode without weakening the complete - # cursor walk, the legacy window cap, or cleanup admission bounds. - block_config = getattr( - getattr(self._coordinator, "config", None), "block", None - ) - configured_page_size = getattr( - block_config, "replay_page_size", MAX_BLOCK_REPLAY_ENUMERATION_ROWS - ) - page_size = min( - MAX_BLOCK_REPLAY_ENUMERATION_ROWS, - max(1, int(configured_page_size)), - ) - # Header pages honour the configured size and, on top of it, - # the hard #255 row cap; the byte cap is enforced by the ledger. - # Neither bound weakens the completeness proof: the page reports - # exhaustion explicitly, so a page cut short by either cap is - # never mistaken for the end of the backlog. - header_page_size = min(page_size, HEADER_PAGE_MAX_ROWS) - # Pagination, not a widening window: the doubling loop below - # fails closed once one page would have to hold the entire - # backlog, so a backlog larger than the cap kept enumeration - # owed (and every job build blocked) until it drained under the - # cap on its own. A keyset cursor walks a backlog of any size - # with the same bounded per-query cost, and only a page proven - # short ends the walk. - page = 0 - after_cursor: object | None = None - while True: - page += 1 - header_page: Any = None - if fetch_header_page is not None: - header_page = fetch_header_page( - header_page_size, - page=page, - after_cursor=after_cursor, - ) - durable_rows = [ - _collapse_row_from_header(row) for row in header_page.rows - ] - else: - try: - durable_rows = fetch_durable_page( - page_size, - page=page, - after_cursor=after_cursor, - ) - except TypeError: - if page > 1: - # Cursor support was already proven by an earlier - # page, so this is a real fault and not a legacy - # ledger; adopted rows must not be re-adopted by a - # fallback pass that starts over from the top. - raise - # A ledger without cursor support keeps exactly today's - # windowed semantics, fail-closed truncation included. - break - enumeration_paginated = True - # Collapse this page's decided-height siblings before any of - # it is adopted, so a row this apply terminalizes is never - # queued and never re-arms a payout barrier. The cursor and - # short-page proof stay bound to the *fetched* page: what the - # collapse removes was removed from the outbox too, so the - # enumeration is still complete. - backpressure_before = ( - self._collapse_cleanup_backpressure_engagement_count() - ) - retained_rows = self._collapse_superseded_block_candidates( - durable_rows, - timeout_seconds=replay_query_timeout, - call_class=replay_query_call_class, - probe_budget=collapse_probe_budget, - ) - cleanup_backpressured = ( - self._collapse_cleanup_backpressure_engagement_count() - != backpressure_before - ) - queued += self._adopt_durable_block_candidate_rows(retained_rows) - if self._block_replay_backpressure_drain_pending: - # A descriptor was withheld before its floor/preview was - # adopted. Even an exhausted SQL page is incomplete here. - enumeration_truncated = True - self._coordinator._note_block_replay_enumeration_owed() - break - print( - "prism coordinator: pending block candidate enumeration " - f"page={page} rows={len(durable_rows)} " - f"limit={header_page_size if header_page is not None else page_size}", - flush=True, - ) - if header_page is not None: - if header_page.exhausted: - # The outbox answered that no pending row followed - # this page, which is the completeness the job-build - # gate waits on. A short byte-capped page does not - # say that, and does not end the walk. - break - elif len(durable_rows) < page_size: - # A short page proves no pending row followed it at query - # time, which is the completeness the job-build gate waits - # on. - break - if cleanup_backpressured: - # A full preserved page may have more pending rows behind - # it. Stop here rather than materializing every later page - # in the unbounded replay queue; the enumeration stays - # owed and resumes only after this bounded page drains. - enumeration_truncated = True - self._block_replay_backpressure_drain_pending = True - print( - "prism coordinator: pending block candidate " - f"enumeration page={page} paused after cleanup " - "backpressure; replay queue drains before enumeration " - "resumes and job builds stay blocked until a complete " - "enumeration succeeds", - flush=True, - ) - break - if self._block_replay_should_yield_to_live_candidates( - collapse_probe_budget - ): - enumeration_truncated = True - print( - "prism coordinator: pending block candidate " - f"enumeration page={page} yielded to a waiting live " - "candidate; job builds stay blocked until a complete " - "enumeration succeeds", - flush=True, - ) - break - if header_page is not None: - next_cursor = header_page.next_cursor - else: - next_cursor = ( - durable_rows[-1].get("cursor") - if durable_rows and isinstance(durable_rows[-1], dict) - else None - ) - if next_cursor is None or next_cursor == after_cursor: - # Either the ledger accepted the keyword without keying - # its rows, or it accepted the cursor without honouring - # it (a **kwargs double, say) and re-served the same - # page. Both leave the walk unable to advance, so fail - # closed instead of looping forever or declaring an - # unproven enumeration complete. - enumeration_truncated = True - print( - "prism coordinator: pending block candidate " - f"enumeration page={page} did not advance its " - "cursor; job builds stay blocked until a complete " - "enumeration succeeds", - flush=True, - ) - break - after_cursor = next_cursor - if not enumeration_paginated: - enumeration_limit = MAX_PENDING_BLOCK_CANDIDATES - while True: - durable_rows = fetch_durable_rows(enumeration_limit) - # Same collapse-then-adopt order on the legacy widening - # window; the truncation proof below still measures the - # fetched window, not the surviving remainder. - backpressure_before = ( - self._collapse_cleanup_backpressure_engagement_count() - ) - retained_rows = self._collapse_superseded_block_candidates( - durable_rows, - timeout_seconds=replay_query_timeout, - call_class=replay_query_call_class, - probe_budget=collapse_probe_budget, - ) - cleanup_backpressured = ( - self._collapse_cleanup_backpressure_engagement_count() - != backpressure_before - ) - queued += self._adopt_durable_block_candidate_rows(retained_rows) - if self._block_replay_backpressure_drain_pending: - # A descriptor was withheld before its floor/preview was - # adopted. Even an exhausted SQL page is incomplete here. - enumeration_truncated = True - self._coordinator._note_block_replay_enumeration_owed() - break - if len(durable_rows) < enumeration_limit or not forced_enumeration: - # A short page proves no further pending row existed at - # query time -- the completeness the re-drive's - # stale-transition sweep below also relies on. - break - if cleanup_backpressured: - enumeration_truncated = True - self._block_replay_backpressure_drain_pending = True - print( - "prism coordinator: pending block candidate " - f"enumeration at {enumeration_limit} rows paused after " - "cleanup backpressure; replay queue drains before " - "enumeration resumes and job builds stay blocked until " - "a complete enumeration succeeds", - flush=True, - ) - break - if self._block_replay_should_yield_to_live_candidates( - collapse_probe_budget - ): - enumeration_truncated = True - print( - "prism coordinator: pending block candidate " - f"enumeration at {enumeration_limit} rows yielded to " - "a waiting live candidate; job builds stay blocked " - "until a complete enumeration succeeds", - flush=True, - ) - break - # A full batch may hide more pending rows, and a hidden row could - # be the active parent whose carry a child job must not omit. - # Re-query with a doubled window until the result is provably - # untruncated; in-flight dedupe makes re-seen rows free. - if enumeration_limit >= MAX_BLOCK_REPLAY_ENUMERATION_ROWS: - enumeration_truncated = True - print( - "prism coordinator: pending block candidate enumeration " - f"still truncated at {enumeration_limit} rows; job builds " - "stay blocked until a complete enumeration succeeds", - flush=True, - ) - break - enumeration_limit = min( - enumeration_limit * 2, - MAX_BLOCK_REPLAY_ENUMERATION_ROWS, - ) - if not enumeration_truncated: - # Every pending candidate is now known and its payout barrier armed - # (or quarantined), so child job builds may proceed. A truncated - # pass instead leaves enumeration owed: the queued batch drains, - # and the submitter loop re-enumerates the remainder. - self._coordinator._clear_block_replay_enumeration_owed() - self._coordinator._record_startup_phase_once("block_replay_enumerated") - self._conclude_ancestor_redrive_pass(redrive_hashes, enumeration_truncated) - if queued: - print( - f"prism coordinator: replayed {queued} pending block candidate(s)", - flush=True, - ) - return queued - - def _run_one_invalid_block_candidate_quarantine(self) -> bool: - self._ensure_block_replay_state() - try: - item = self._block_quarantine_queue.get_nowait() - except queue.Empty: - return False - block_hash, error = item[0], item[1] - pending_share = item[2] if len(item) > 2 else None - completed = False - try: - self._coordinator._record_block_submitter_phase("replay-quarantine") - quarantine = getattr( - self._coordinator.ledger, - "mark_block_candidate_abandoned", - None, - ) - if callable(quarantine): - quarantined = self._coordinator._run_block_submitter_ledger_call( - ("replay-quarantine", block_hash), - "replay-quarantine", - lambda: quarantine(block_hash=block_hash, error=error), - ) - self._coordinator._clear_accepted_block_payout_preview(block_hash) - if pending_share is not None: - # The row's durable credit holder was adopted before the - # failure. Its outbox row is now terminal, so a later - # successful replay re-creates a fresh holder; release - # this one instead of clamping snapshot anchors forever. - self._coordinator._finish_pending_share_candidate(pending_share) - if quarantined: - self._coordinator._clear_block_candidate_retry_state(block_hash) - self._coordinator._discard_outstanding_block_candidate(block_hash) - with self._coordinator.lock: - self.poisoned = int(self.poisoned) + 1 - completed = True - return True - except Exception: - print( - "prism coordinator: invalid candidate quarantine failed " - f"hash={block_hash}", - flush=True, - ) - traceback.print_exc() - return True - finally: - self._block_quarantine_queue.task_done() - if completed: - with self._coordinator.lock: - self._block_quarantine_hashes.discard(block_hash) - elif not self._coordinator.stop_event.is_set(): - self._block_quarantine_queue.put_nowait( - (block_hash, error, pending_share) - ) - - # -- retry state ------------------------------------------------------- - - def wait_for_retry(self, delay_seconds: float) -> bool: - """Wait for intentional backoff without impersonating stuck work. - - Retry waits heartbeat in bounded slices. Direct outbox calls and lock - admission use the same phase-aware pattern; work that is not covered - by an explicit deadline remains watchdog-eligible. - """ - delay_seconds = max(0.0, float(delay_seconds)) - if delay_seconds <= 0: - return self._coordinator.stop_event.is_set() - started = time.monotonic() - with self._state_lock: - self._backoff_started_monotonic = started - self._backoff_deadline_monotonic = started + delay_seconds - self._backoff_delay_seconds = delay_seconds - remaining = delay_seconds - try: - while remaining > 0: - self._coordinator._record_block_submitter_wait("retry-backoff") - wait_slice = min(remaining, self._block_work_wait_slice()) - if self._coordinator.stop_event.wait(wait_slice): - return True - remaining = max(0.0, remaining - wait_slice) - self._coordinator._record_block_submitter_wait("retry-backoff:complete") - return False - finally: - with self._state_lock: - self._backoff_started_monotonic = None - self._backoff_deadline_monotonic = None - self._backoff_delay_seconds = 0.0 - - def backoff_snapshot(self) -> tuple[bool, float, float]: - now = time.monotonic() - with self._state_lock: - deadline = self._backoff_deadline_monotonic - return ( - deadline is not None, - max(0.0, deadline - now) if deadline is not None else 0.0, - self._backoff_delay_seconds, - ) - - def block_submit_seconds_snapshot(self) -> tuple[dict[float, int], float, int]: - """Copied landed-to-RPC histogram state for metrics rendering.""" - with self._block_submit_metrics_lock: - histogram = self.block_submit_seconds_histogram - return ( - dict(histogram["buckets"]), - float(histogram["sum"]), - int(histogram["count"]), - ) - - def accepted_block_preview_publication_snapshot( - self, - ) -> dict[str, dict[str, Any]]: - """Copied acceptance-to-preview-publication histograms, by result.""" - with self._accepted_block_preview_publication_lock: - return { - result: { - "buckets": dict(histogram["buckets"]), - "sum": float(histogram["sum"]), - "count": int(histogram["count"]), - } - for result, histogram in ( - self.accepted_block_preview_publication_seconds_histogram - ).items() - } - - def next_retry_delay(self, block_hash: str) -> float: - initial = max(0.0, float(self.retry_initial_seconds)) - maximum = max(initial, float(self.retry_max_seconds)) - with self._coordinator.lock: - delays = self.retry_delays - if delays is None: - delays = {} - self.retry_delays = delays - delay = float(delays.get(block_hash, initial)) - delays[block_hash] = min(maximum, max(initial, delay * 2)) - return min(delay, maximum) - - def clear_retry_state(self, block_hash: str) -> None: - with self._coordinator.lock: - delays = self.retry_delays - if delays is not None: - delays.pop(block_hash, None) - landing_timeouts = getattr(self, "_block_landing_timeout_counts", None) - if landing_timeouts is not None: - landing_timeouts.pop(block_hash, None) - not_before = getattr( - self, - "_block_candidate_retry_not_before", - None, - ) - if not_before is not None: - not_before.pop(block_hash, None) - retained = getattr( - self, - "_block_candidate_retained_node_submissions", - None, - ) - if retained is not None: - retained.pop(str(block_hash).lower(), None) - stamped = getattr( - self, - "_block_candidate_retained_submission_monotonic", - None, - ) - if stamped is not None: - stamped.pop(str(block_hash).lower(), None) - - def mark_attempted(self, block_hash: str) -> None: - mark_attempted = getattr( - self._coordinator.ledger, - "mark_block_candidate_attempted", - None, - ) - if callable(mark_attempted): - self._coordinator._run_block_submitter_ledger_call( - ("mark-attempted", block_hash), - "mark-attempted", - lambda: mark_attempted(block_hash=block_hash), - ) - - def _merge_block_candidate_retry_locked( - self, - attribute: str, - candidate: PrismBlockCandidate, - ) -> None: - """Merge one retry by parent-first order. Caller holds the runtime lock.""" - candidate_height = int(candidate.context.template["height"]) - candidate_hash = str(candidate.submission.block_hash_hex).lower() - existing = getattr(self, attribute, None) - if existing is None: - setattr(self, attribute, candidate) - return - existing_height = int(existing.context.template["height"]) - existing_hash = str(existing.submission.block_hash_hex).lower() - if candidate_hash == existing_hash: - setattr(self, attribute, candidate) - if existing is not candidate: - # The newer same-hash object takes the slot; the displaced - # duplicate is dropped and carries its own floor holder. - self._release_dropped_duplicate_candidate_floor(existing) - return - if attribute != "_retry_block_candidate": - if candidate_height < existing_height: - setattr(self, attribute, candidate) - return - - # The raw lane has one parent-first head slot, but every displaced - # hash still needs an in-memory wakeup. Durable replay dedupe keeps a - # replayed descendant marked in-flight, so relying on a later outbox - # scan here could otherwise suppress it forever. - self._ensure_block_candidate_disposition_state() - waiting = self._block_disposition_waiting_retries - if candidate_height < existing_height: - waiting[existing_hash] = existing - setattr(self, attribute, candidate) - else: - waiting[candidate_hash] = candidate - - def retain_for_retry(self, candidate: PrismBlockCandidate) -> None: - """Keep the oldest unresolved candidate ahead of queued descendants.""" - candidate_hash = str(candidate.submission.block_hash_hex).lower() - # A retained candidate will be re-disposed, so the disposition seal - # (which stopped tip-observation matching at a terminal commit) no - # longer applies: the terminal work did not complete. Re-register - # immediately -- not at the next disposition -- so acceptance - # evidence arriving during the retry backoff is not lost. - self._coordinator._register_outstanding_block_candidate(candidate_hash) - with self._coordinator.lock: - self.retries = int(self.retries) + 1 - accounting_owner = ( - threading.get_ident() - == getattr(self, "_block_accounting_thread_ident", None) - and bool( - getattr( - self, - "_block_accounting_holds_disposition", - False, - ) - ) - ) - retry_attribute = ( - "_block_accounting_deferred_retry_candidate" - if accounting_owner - else "_retry_block_candidate" - ) - self._merge_block_candidate_retry_locked( - retry_attribute, - candidate, - ) - - def _pace_block_candidate_retry(self, block_hash: str) -> None: - """Apply per-candidate retry backoff without convoying accounting. - - On the block_accounting thread the disposition lease and writer - admission stay held until the accounting task's finally clause, so - sleeping here would stall every queued accounting task and keep an - armed payout barrier blocking balance mutation for the whole backoff - window. Record a not-before deadline instead; the dequeue path honors - it, and replay_pending_block_candidates already short-circuits while - the retained candidate occupies the retry slot. - """ - delay_seconds = self._coordinator._next_block_candidate_retry_delay(block_hash) - accounting_owner = ( - threading.get_ident() - == getattr(self, "_block_accounting_thread_ident", None) - and bool( - getattr( - self, - "_block_accounting_holds_disposition", - False, - ) - ) - ) - if not accounting_owner: - self._coordinator._wait_for_block_candidate_retry(delay_seconds) - return - with self._coordinator.lock: - not_before = getattr( - self, - "_block_candidate_retry_not_before", - None, - ) - if not_before is None: - not_before = {} - self._block_candidate_retry_not_before = not_before - not_before[str(block_hash).lower()] = ( - time.monotonic() + delay_seconds - ) - - def _block_candidate_retry_ready_locked( - self, - candidate: PrismBlockCandidate, - ) -> bool: - """Return whether a parked retry's backoff deadline has passed. - - Caller holds the runtime lock. A ready entry is dropped so a candidate - that later lands terminally leaves no stale pacing behind. - """ - not_before = getattr(self, "_block_candidate_retry_not_before", None) - if not not_before: - return True - block_hash = str(candidate.submission.block_hash_hex).lower() - deadline = not_before.get(block_hash) - if deadline is None: - return True - if time.monotonic() < deadline: - return False - not_before.pop(block_hash, None) - return True - - # -- deadline-classed ledger call workers ------------------------------ - - def _ensure_block_submitter_ledger_call_state(self) -> None: - if not hasattr(self, "_block_submitter_ledger_calls_lock"): - self._block_submitter_ledger_calls_lock = threading.Lock() - if not hasattr(self, "_block_submitter_ledger_calls"): - self._block_submitter_ledger_calls = {} - if not hasattr(self, "_block_submitter_ledger_worker_slots"): - self._block_submitter_ledger_worker_slots = threading.BoundedSemaphore( - MAX_BLOCK_SUBMITTER_STUCK_CALL_WORKERS - ) - - def _block_submitter_db_timeout(self) -> float: - return max( - 0.001, - float( - getattr( - self._coordinator, - "block_submit_db_timeout_seconds", - DEFAULT_BLOCK_SUBMIT_DB_TIMEOUT_SECONDS, - ) - ), - ) - - def _block_landing_watchdog_ceiling(self) -> float: - """Largest landing budget the configured watchdog can tolerate. - - Landing-class work runs on the block-work thread the watchdog - monitors, so the landing budget and the watchdog tolerance are one - system, not two independent settings. Deriving the ceiling from the - configured tolerance keeps them in step through every override - instead of pinning a second literal that silently goes stale. - - A deployment that turned the watchdog off has no hard-exit hazard to - stay under, and clamping it anyway would cost the operator budget for - no safety at all -- so that case gets no ceiling. The attribute is - read defensively and defaults to *enabled*: clamping is the safe - answer when the setting cannot be determined. - """ - if not bool(getattr(self._coordinator, "watchdog_enabled", True)): - return float("inf") - return max( - 0.001, - float( - getattr( - self._coordinator, - "watchdog_timeout_seconds", - DEFAULT_PRISM_WATCHDOG_TIMEOUT_SECONDS, - ) - ) - * BLOCK_LANDING_DB_TIMEOUT_WATCHDOG_FRACTION, - ) - - def _note_block_landing_budget_clamped( - self, - *, - configured_base: float, - configured_cap: float, - ceiling: float, - ) -> None: - """Say once that the watchdog is granting less than was configured. - - Without this the operator's configured landing budget is quietly - reduced -- at the 120s default tolerance the reviewed 120s cap - becomes 60s -- and the two states an operator most needs to tell - apart become indistinguishable in the logs: escalation exhausted at - the configured cap, versus escalation that never reached it because - the ceiling stopped it first. Emitted once per process because - _block_landing_db_timeout runs on every landing attempt and a - per-attempt line would bury the landing's own diagnostics; the flag - flips under the coordinator lock so concurrent first landings still - print exactly one line. The print stays outside that lock -- no I/O - under a coordinator-wide lock on the landing path. - """ - with self._coordinator.lock: - if getattr(self, "_block_landing_budget_clamp_logged", False): - return - self._block_landing_budget_clamp_logged = True - watchdog_seconds = float( - getattr( - self._coordinator, - "watchdog_timeout_seconds", - DEFAULT_PRISM_WATCHDOG_TIMEOUT_SECONDS, - ) - ) - print( - "prism coordinator: landing db budget clamped by watchdog " - f"configured_base={configured_base:g}s " - f"configured_max={configured_cap:g}s " - f"granted_base={min(configured_base, ceiling):g}s " - f"granted_max={min(configured_cap, ceiling):g}s " - f"ceiling={ceiling:g}s " - f"watchdog_timeout={watchdog_seconds:g}s " - f"fraction={BLOCK_LANDING_DB_TIMEOUT_WATCHDOG_FRACTION:g}", - flush=True, - ) - - def _block_landing_db_timeout(self, block_hash: str | None = None) -> float: - """Landing-class deadline, escalated after observed landing timeouts. - - The first attempt already receives the full landing budget; a known - landing-class operation never begins at the one-second poll budget. - Escalation doubles per timed-out landing attempt for the same block - hash up to the reviewed cap. - - The reviewed cap is an upper bound, not the granted budget: every - value here is clamped to the watchdog-derived ceiling, which can only - lower it. Landing steps are spent on the watchdog-monitored block-work - thread, so an escalated budget that outruns the watchdog tolerance is - not a longer attempt but a hard exit mid-landing -- and because the - escalation counts live only in memory, the restart drops back to the - base budget and repeats the same doomed cycle (issue #125). At the - 120s default tolerance the ceiling is 60s and escalation runs - 30s -> 60s; at the 300s production tolerance the ceiling is 150s and - the configured 120s cap is unchanged. A deployment running with the - watchdog disabled has no ceiling at all and keeps its configured - values in full. Any clamp is announced once (see - _note_block_landing_budget_clamped) so a reduced budget is never a - silent behavior change. - """ - ceiling = self._block_landing_watchdog_ceiling() - configured_base = max( - 0.001, - float( - getattr( - self._coordinator, - "block_landing_db_timeout_seconds", - DEFAULT_BLOCK_LANDING_DB_TIMEOUT_SECONDS, - ) - ), - ) - configured_cap = max( - configured_base, - float( - getattr( - self._coordinator, - "block_landing_db_timeout_max_seconds", - DEFAULT_BLOCK_LANDING_DB_TIMEOUT_MAX_SECONDS, - ) - ), - ) - base = min(configured_base, ceiling) - cap = min(configured_cap, ceiling) - if base < configured_base or cap < configured_cap: - # An infinite ceiling (watchdog disabled) can never compare below - # a finite configured value, so a deployment that opted out of the - # watchdog never reaches this line. - self._note_block_landing_budget_clamped( - configured_base=configured_base, - configured_cap=configured_cap, - ceiling=ceiling, - ) - timeouts = 0 - if block_hash is not None: - with self._coordinator.lock: - counts = getattr(self, "_block_landing_timeout_counts", None) - if counts is not None: - timeouts = int(counts.get(block_hash, 0)) - return min(cap, base * (2.0 ** min(timeouts, 8))) - - def _note_block_landing_timeout(self, block_hash: str | None) -> None: - if block_hash is None: - return - with self._coordinator.lock: - counts = getattr(self, "_block_landing_timeout_counts", None) - if counts is None: - counts = {} - self._block_landing_timeout_counts = counts - counts[block_hash] = int(counts.get(block_hash, 0)) + 1 - - def _ensure_block_ledger_call_metrics(self) -> None: - if not hasattr(self, "_block_ledger_call_metrics_lock"): - self._block_ledger_call_metrics_lock = threading.Lock() - if not hasattr(self, "_block_ledger_call_metrics"): - self._block_ledger_call_metrics = {} - - def _record_block_ledger_call( - self, - *, - call_class: str, - budget_seconds: float, - duration_seconds: float, - timed_out: bool, - ) -> None: - """Track per-call-class submitter ledger latency and timeout counts.""" - self._ensure_block_ledger_call_metrics() - with self._block_ledger_call_metrics_lock: - stats = self._block_ledger_call_metrics.setdefault( - call_class, - { - "calls_total": 0, - "timeouts_total": 0, - "last_budget_seconds": 0.0, - "last_duration_seconds": 0.0, - "max_duration_seconds": 0.0, - }, - ) - stats["calls_total"] = int(stats["calls_total"]) + 1 - if timed_out: - stats["timeouts_total"] = int(stats["timeouts_total"]) + 1 - stats["last_budget_seconds"] = float(budget_seconds) - stats["last_duration_seconds"] = float(duration_seconds) - stats["max_duration_seconds"] = max( - float(stats["max_duration_seconds"]), float(duration_seconds) - ) - - def block_ledger_call_class_metrics(self) -> dict[str, dict[str, float | int]]: - self._ensure_block_ledger_call_metrics() - with self._block_ledger_call_metrics_lock: - return { - call_class: dict(stats) - for call_class, stats in self._block_ledger_call_metrics.items() - } - - def _block_submitter_stuck_call_exit_timeout(self) -> float: - return max( - 0.001, - float( - getattr( - self._coordinator, - "block_submit_stuck_call_exit_seconds", - DEFAULT_BLOCK_SUBMIT_STUCK_CALL_EXIT_SECONDS, - ) - ), - ) - - def _maybe_restart_for_stuck_block_call( - self, - *, - kind: str, - started_monotonic: float, - ) -> None: - """Fail stop when a poisoned worker pool stays exhausted.""" - age_seconds = max(0.0, time.monotonic() - started_monotonic) - exit_seconds = self._block_submitter_stuck_call_exit_timeout() - if age_seconds < exit_seconds: - return - stop_event = getattr(self._coordinator, "stop_event", None) - if stop_event is not None and stop_event.is_set(): - return - print( - "prism coordinator: block work call remained stuck; requesting " - f"restart kind={kind} age={age_seconds:.3f}s " - f"budget={exit_seconds:g}s", - flush=True, - ) - self._coordinator._fatal_exit_requested = True - self._coordinator.request_shutdown() - - def _maybe_restart_for_exhausted_block_call_pool( - self, - *, - kind: str, - calls_lock: threading.Lock, - calls: dict[Any, Any], - ) -> None: - """Age an exhausted pool even when retries reuse existing calls.""" - with calls_lock: - active_starts = [ - pending.started_monotonic - for pending in calls.values() - if not pending.done.is_set() - ] - if len(active_starts) < MAX_BLOCK_SUBMITTER_STUCK_CALL_WORKERS: - return - self._maybe_restart_for_stuck_block_call( - kind=kind, - started_monotonic=min(active_starts), - ) - - def _retire_finished_block_submitter_ledger_call( - self, - key: tuple[object, ...], - call: _BlockSubmitterLedgerCall, - ) -> None: - """Take one finished call out of the registry, by identity. - - A finished call has no live need: it is registered only so that a - same-key caller arriving while the worker is *still out* joins that - one call instead of starting a second one, and a worker that has - published ``done`` is not out any more. Leaving it behind would wait - on an invocation that may never come -- the exact key can be a whole - enumeration page of block hashes, and the late write itself is what - empties that page out of the outbox -- so the registry would keep - the completed call, and its hash tuple, for the life of the process. - - Both the worker and a waiter that observed the completion call this, - whichever gets there first; the identity check is what keeps either - of them from dropping a *different* call that has since taken the - key, and the ``done`` check is what keeps them from ever dropping a - worker that is still out. Waiters hold the call directly, so a - removal never costs anybody the result or error it is owed: it only - settles that the next caller for that key replays the idempotent - operation on a fresh call. - """ - with self._block_submitter_ledger_calls_lock: - if not call.done.is_set(): - return - if self._block_submitter_ledger_calls.get(key) is call: - del self._block_submitter_ledger_calls[key] - - @contextmanager - def _block_submitter_ledger_timeout_scope( - self, - timeout_seconds: float | None = None, - ) -> Iterator[None]: - """Apply the submitter's PostgreSQL deadline when the ledger supports it.""" - operation_timeout = getattr( - self._coordinator.ledger, - "operation_timeout", - None, - ) - if not callable(operation_timeout): - yield - return - with operation_timeout( - self._block_submitter_db_timeout() - if timeout_seconds is None - else timeout_seconds - ): - yield - - @contextmanager - def _block_work_ledger_progress_scope(self, phase: str) -> Iterator[None]: - """Keep a ledger admission wait visible to the block-work watchdog. - - A statement deadline only bounds work the server can cancel. Before - any SQL is sent the ledger must first win its own writer lock or read - semaphore, and that wait is local: no statement exists to cancel and - nothing reports until admission succeeds. The scopes below run on the - block-work owner thread the watchdog monitors, so an admission wait - that stamps nothing is indistinguishable from a wedged thread and can - cost the coordinator a hard exit while it is merely queued behind - another writer. - - Installing the ledger's progress hook makes that wait heartbeat in - watchdog-sized slices, using the same phase-stamping helper the - bounded-call wait loop already uses (it no-ops safely off the owner - thread). Ledgers predating the hook are left exactly as they were. - """ - operation_progress = getattr( - self._coordinator.ledger, - "operation_progress", - None, - ) - if not callable(operation_progress): - yield - return - with operation_progress( - lambda: self._coordinator._record_block_submitter_wait(phase), - slice_seconds=self._block_work_wait_slice(), - ): - yield - - @contextmanager - def _block_submitter_ledger_statement_timeout_scope(self) -> Iterator[None]: - """Give each post-submit ledger step a fresh short deadline.""" - statement_timeout = getattr( - self._coordinator.ledger, - "statement_timeout", - None, - ) - if callable(statement_timeout): - with self._block_work_ledger_progress_scope( - "wait-ledger-admission:submit" - ): - with statement_timeout(self._block_submitter_db_timeout()): - yield - return - # Duck-typed ledgers predating per-statement scopes still receive a - # bounded operation, even though their budget spans the whole tail. - with self._block_submitter_ledger_timeout_scope(): - yield - - @contextmanager - def _block_landing_ledger_statement_timeout_scope( - self, - block_hash: str | None = None, - ) -> Iterator[None]: - """Give each landing-class ledger step the landing budget. - - The accounting tail that lands an accepted block (persist, prior - balances, confirm, and the prepared-state rejection of a terminal - candidate) runs under this scope instead of the poll-class one. - Timed-out steps are recorded so the next attempt for the same block - hash escalates its budget; the ledger backends already guarantee - server-side cancellation completes and the pooled session is rolled - back or replaced before the paced retry re-enters here. - - The guarded body also runs node RPCs and audit/build work. Only a - ledger-originated deadline may escalate the next landing budget or - fire the landing-timeout alert: a node RPC timeout is not a database - cancellation, and escalating on it would page and widen PostgreSQL - deadlines for a database that never missed one. - """ - timeout_seconds = self._coordinator._block_landing_db_timeout(block_hash) - scope = getattr(self._coordinator.ledger, "statement_timeout", None) - if not callable(scope): - scope = getattr(self._coordinator.ledger, "operation_timeout", None) - started = time.monotonic() - timed_out = False - try: - if callable(scope): - with self._block_work_ledger_progress_scope( - "wait-ledger-admission:landing" - ): - with scope(timeout_seconds): - yield - else: - yield - except (LedgerOperationTimeout, BlockSubmitterDatabaseTimeout): - timed_out = True - self._coordinator._note_block_landing_timeout(block_hash) - raise - finally: - self._coordinator._record_block_ledger_call( - call_class="landing", - budget_seconds=timeout_seconds, - duration_seconds=max(0.0, time.monotonic() - started), - timed_out=timed_out, - ) - - def _run_block_submitter_ledger_call( - self, - key: tuple[object, ...], - phase: str, - operation: Callable[[], Any], - *, - timeout_seconds: float | None = None, - call_class: str = "fast", - ) -> Any: - """Run one direct outbox call without letting its driver wedge us. - - A timed-out call stays registered for as long as its worker is - still out, and the next paced retry joins that same call. This - bounds the coordinator-side wait without spawning an unbounded pile - of threads when a fake/misbehaving driver ignores the real - PostgreSQL statement deadline. Candidate outbox mutations are - idempotent, so a late completion converges with replay. - - The registry therefore holds in-flight calls and nothing else: the - worker retires its own entry the moment it publishes ``done`` (see - _retire_finished_block_submitter_ledger_call), so a key whose retry - never comes back cannot pin a completed call and its page of block - hashes for the life of the process. Waiters already parked on the - call hold it directly and still get its result or error; a caller - arriving after the removal simply replays the operation. - - call_class labels the per-class latency/timeout metrics and must - match the budget in use: a call given the landing deadline records - as "landing" so the landing-timeout alert covers it, instead of - inflating the fast-call budget gauge. - """ - if timeout_seconds is None: - timeout_seconds = self._block_submitter_db_timeout() - else: - timeout_seconds = max(0.001, float(timeout_seconds)) - self._ensure_block_submitter_ledger_call_state() - with self._block_submitter_ledger_calls_lock: - call = self._block_submitter_ledger_calls.get(key) - if call is None: - if not self._block_submitter_ledger_worker_slots.acquire( - blocking=False - ): - oldest_started = min( - ( - pending.started_monotonic - for pending in self._block_submitter_ledger_calls.values() - if not pending.done.is_set() - ), - default=time.monotonic(), - ) - self._maybe_restart_for_stuck_block_call( - kind="ledger-worker-pool", - started_monotonic=oldest_started, - ) - raise BlockSubmitterDatabaseTimeout( - f"{phase} could not acquire a bounded ledger worker" - ) - call = _BlockSubmitterLedgerCall() - self._block_submitter_ledger_calls[key] = call - - def run() -> None: - try: - with self._block_submitter_ledger_timeout_scope( - timeout_seconds - ): - call.result = operation() - except BaseException as exc: - call.error = exc - finally: - call.done.set() - # The slot goes back before the removal: a caller - # that arrives to find this key already retired must - # not then be told the bounded pool is exhausted. - self._block_submitter_ledger_worker_slots.release() - self._retire_finished_block_submitter_ledger_call( - key, - call, - ) - - threading.Thread( - target=run, - name=f"prism-block-ledger-{phase}", - daemon=True, - ).start() - - deadline = time.monotonic() + timeout_seconds - while not call.done.is_set(): - self._coordinator._record_block_submitter_wait(phase) - remaining = deadline - time.monotonic() - if remaining <= 0: - self._maybe_restart_for_exhausted_block_call_pool( - kind="ledger-worker-pool", - calls_lock=self._block_submitter_ledger_calls_lock, - calls=self._block_submitter_ledger_calls, - ) - print( - "prism coordinator: block submitter ledger phase timed out " - f"phase={phase} timeout={timeout_seconds:g}s", - flush=True, - ) - self._coordinator._record_block_ledger_call( - call_class=call_class, - budget_seconds=timeout_seconds, - duration_seconds=timeout_seconds, - timed_out=True, - ) - raise BlockSubmitterDatabaseTimeout( - f"{phase} exceeded {timeout_seconds:g}s" - ) - call.done.wait( - min( - remaining, - self._block_work_wait_slice(), - ) - ) - self._coordinator._record_block_submitter_wait(f"{phase}:complete") - # A server-side deadline normally completes the worker with a ledger - # timeout error before the coordinator-side wait expires. Every - # operation behind this wrapper is a database call, so a completed - # call carrying a timeout error is still a timed-out call for the - # per-class alert series. - self._coordinator._record_block_ledger_call( - call_class=call_class, - budget_seconds=timeout_seconds, - duration_seconds=max(0.0, time.monotonic() - call.started_monotonic), - timed_out=isinstance(call.error, TimeoutError), - ) - self._retire_finished_block_submitter_ledger_call(key, call) - if call.error is not None: - raise call.error - return call.result - - # -- submitblock transport --------------------------------------------- - - def _rpc_call_with_timeout( - self, - method: str, - params: list[object], - *, - timeout_seconds: float, - ) -> Any: - """Pass an explicit timeout to production RPCs and capable test doubles.""" - call = self._coordinator.rpc.call - supports_timeout = isinstance(self._coordinator.rpc, JsonRpc) - if not supports_timeout: - try: - parameters = inspect.signature(call).parameters.values() - supports_timeout = any( - parameter.name == "timeout" - or parameter.kind is inspect.Parameter.VAR_KEYWORD - for parameter in parameters - ) - except (TypeError, ValueError): - supports_timeout = False - if supports_timeout: - return call(method, params, timeout=timeout_seconds) - return call(method, params) - - def _ensure_block_submitter_rpc_call_state(self) -> None: - if not hasattr(self, "_block_submitter_rpc_calls_lock"): - self._block_submitter_rpc_calls_lock = threading.Lock() - if not hasattr(self, "_block_submitter_rpc_calls"): - self._block_submitter_rpc_calls = {} - if not hasattr(self, "_block_submitter_rpc_worker_slots"): - self._block_submitter_rpc_worker_slots = threading.BoundedSemaphore( - MAX_BLOCK_SUBMITTER_STUCK_CALL_WORKERS - ) - - def _run_submitblock_rpc_with_hard_deadline( - self, - *, - block_hash: str, - block_hex: str, - timeout_seconds: float, - ) -> Any: - """Bound wall time even when an RPC adapter ignores its timeout.""" - self._ensure_block_submitter_rpc_call_state() - with self._block_submitter_rpc_calls_lock: - call = self._block_submitter_rpc_calls.get(block_hash) - if call is None: - if not self._block_submitter_rpc_worker_slots.acquire( - blocking=False - ): - oldest_started = min( - ( - pending.started_monotonic - for pending in self._block_submitter_rpc_calls.values() - if not pending.done.is_set() - ), - default=time.monotonic(), - ) - self._maybe_restart_for_stuck_block_call( - kind="rpc-worker-pool", - started_monotonic=oldest_started, - ) - raise TimeoutError( - "submitblock could not acquire a bounded RPC worker" - ) - call = _BlockSubmitterRpcCall() - self._block_submitter_rpc_calls[block_hash] = call - - def run() -> None: - try: - call.result = self._rpc_call_with_timeout( - "submitblock", - [block_hex], - timeout_seconds=timeout_seconds, - ) - except BaseException as exc: - call.error = exc - finally: - call.done.set() - self._block_submitter_rpc_worker_slots.release() - - threading.Thread( - target=run, - name=f"prism-block-rpc-{block_hash[:12]}", - daemon=True, - ).start() - - deadline = time.monotonic() + timeout_seconds - while not call.done.is_set(): - self._coordinator._record_block_submitter_wait("submitblock-rpc") - remaining = deadline - time.monotonic() - if remaining <= 0: - self._maybe_restart_for_exhausted_block_call_pool( - kind="rpc-worker-pool", - calls_lock=self._block_submitter_rpc_calls_lock, - calls=self._block_submitter_rpc_calls, - ) - raise TimeoutError( - f"submitblock exceeded {timeout_seconds:g}s" - ) - call.done.wait( - min(remaining, self._block_work_wait_slice()) - ) - with self._block_submitter_rpc_calls_lock: - if self._block_submitter_rpc_calls.get(block_hash) is call: - self._block_submitter_rpc_calls.pop(block_hash, None) - if call.error is not None: - raise call.error - return call.result - - # -- node offer -------------------------------------------------------- - - def _arm_block_candidate_after_node_offer( - self, - candidate: PrismBlockCandidate, - node_submission: _BlockCandidateNodeSubmission, - ) -> None: - """Fence child payout work as soon as node acceptance is possible.""" - self._coordinator._stash_retained_block_candidate_node_submission( - str(candidate.submission.block_hash_hex), - node_submission, - ) - ambiguous_or_landed = ( - node_submission.error is not None - or node_submission.result in (None, "duplicate") - ) - if not ambiguous_or_landed: - self._coordinator._release_block_fast_lane_slot( - str(candidate.submission.block_hash_hex) - ) - return - block_hash = str(candidate.submission.block_hash_hex).lower() - expected_height = int(candidate.context.template["height"]) - self._coordinator._begin_accepted_block_payout_preview( - block_hash, - block_height=expected_height, - ) - - def _submit_block_candidate_to_node( - self, - candidate: PrismBlockCandidate, - ) -> _BlockCandidateNodeSubmission: - """Offer the durable candidate to qbitd before any accounting work.""" - block_hash = str(candidate.submission.block_hash_hex).lower() - self._coordinator._begin_accepted_block_payout_preview( - block_hash, - block_height=int(candidate.context.template["height"]), - ) - self._coordinator._register_outstanding_block_candidate(block_hash) - self._coordinator._record_block_submitter_phase("submitblock-rpc") - timeout_seconds = max( - 0.001, - float( - getattr( - self._coordinator, - "block_submit_rpc_timeout_seconds", - DEFAULT_BLOCK_SUBMIT_RPC_TIMEOUT_SECONDS, - ) - ), - ) - try: - result = self._coordinator._run_submitblock_rpc_with_hard_deadline( - block_hash=block_hash, - block_hex=candidate.submission.block_hex, - timeout_seconds=timeout_seconds, - ) - except BaseException as exc: - self._coordinator._record_block_submitter_phase("submitblock-rpc:error") - node_submission = _BlockCandidateNodeSubmission( - attempted=True, - error=exc, - ) - self._coordinator._arm_block_candidate_after_node_offer( - candidate, - node_submission, - ) - return node_submission - self._coordinator._record_block_submitter_phase("submitblock-rpc:complete") - landed_monotonic = getattr(candidate, "landed_monotonic", None) - if landed_monotonic is not None: - self._coordinator._observe_block_submit_seconds( - time.monotonic() - float(landed_monotonic) - ) - node_submission = _BlockCandidateNodeSubmission( - attempted=True, - result=result, - ) - if _is_definitive_node_acceptance(node_submission): - # Stamp the start of issue #181's acceptance-to-publication - # interval from the same predicate the lane routing and the - # retained-offer stash read, so "the node has this block" has - # exactly one definition in this module. - self._note_accepted_block_preview_acceptance(block_hash) - self._coordinator._arm_block_candidate_after_node_offer( - candidate, - node_submission, - ) - return node_submission - - def _node_submission_for_candidate( - self, - candidate: PrismBlockCandidate, - ) -> _BlockCandidateNodeSubmission: - """Choose the node fast lane unless the pool was already closed.""" - block_hash = str(candidate.submission.block_hash_hex).lower() - self._coordinator._record_block_submitter_phase("fast-lane-admission") - with self._coordinator.lock: - accounted_hashes = getattr( - self._coordinator, - "_accounted_accepted_block_hashes", - set(), - ) - accepted_count = int(self._coordinator.accepted_block_count) - pool_closed = block_hash not in accounted_hashes and ( - accepted_count >= int(self._coordinator.max_blocks) - or ( - bool(self._coordinator.stop_after_block) - and accepted_count >= 1 - ) - ) - if pool_closed: - return _BlockCandidateNodeSubmission(attempted=False) - return self._coordinator._submit_block_candidate_to_node(candidate) - - def _node_submission_for_candidate_or_retained( - self, - candidate: PrismBlockCandidate, - ) -> _BlockCandidateNodeSubmission: - """Reuse a retained definitive acceptance instead of re-offering. - - An in-process retry of a candidate whose earlier offer already - returned success must not ask the node again: the re-offer answers - "duplicate", which downgrades the classification to the moved live - tip and leans on chain probes that may be unavailable under the - same saturation that caused the retry. The stashed result reruns - the landing tail as if the first pass had continued. - """ - retained = self._coordinator._retained_block_candidate_node_submission( - str(candidate.submission.block_hash_hex) - ) - if retained is not None: - return retained - return self._coordinator._node_submission_for_candidate(candidate) - - def _node_submission_for_direct_candidate( - self, - candidate: PrismBlockCandidate, - ) -> _BlockCandidateNodeSubmission: - """Preserve active-replay semantics for non-queue embedders. - - The dedicated submitter always uses the unconditional fast lane. A - direct caller can instead be resuming a durable active ancestor, for - which another submit is unnecessary and some integrations do not - retain block bytes. This compatibility probe is not on the incident - queue-to-node path. - """ - block_hash = str(candidate.submission.block_hash_hex).lower() - expected_height = int(candidate.context.template["height"]) - try: - if str(self._coordinator.rpc.call("getbestblockhash")).lower() == block_hash: - return _BlockCandidateNodeSubmission(attempted=False) - except Exception: - pass - try: - if ( - self._coordinator.active_block_candidate_height(block_hash) - == expected_height - ): - return _BlockCandidateNodeSubmission(attempted=False) - except Exception: - pass - if not hasattr(candidate.submission, "block_hex"): - return _BlockCandidateNodeSubmission(attempted=False) - return self._coordinator._node_submission_for_candidate(candidate) - - def _account_block_candidate_after_node_submit( - self, - candidate: PrismBlockCandidate, - node_submission: _BlockCandidateNodeSubmission, - ) -> bool: - """Pass fast-lane evidence while tolerating legacy test embedders.""" - submit = self._coordinator.submit_block_candidate - supports_node_submission = True - try: - parameters = inspect.signature(submit).parameters.values() - supports_node_submission = any( - parameter.name == "node_submission" - or parameter.kind is inspect.Parameter.VAR_KEYWORD - for parameter in parameters - ) - except (TypeError, ValueError): - pass - if supports_node_submission: - return bool(submit(candidate, node_submission=node_submission)) - return bool(submit(candidate)) - - def _submit_synchronous_block_candidate( - self, - candidate: PrismBlockCandidate, - ) -> bool: - """Run the rare miner-facing path under one same-hash disposition.""" - coordinator = self._coordinator - block_hash = str(candidate.submission.block_hash_hex).lower() - with coordinator._block_candidate_disposition(block_hash): - terminal_outcome = coordinator._block_candidate_terminal_outcome( - block_hash - ) - if terminal_outcome is not None: - coordinator._finish_pending_share_commit(candidate.pending_share) - return terminal_outcome - outcome = self.outcome - # The accepted/rejected accounting tail may already be complete - # while only its durable outbox terminal update is retrying. A - # synchronous same-hash waiter must join that finalize-only state; - # another node offer/accounting pass could invert the outcome. - with coordinator.lock: - pending_finalize = self.finalize_retries.get(block_hash) - if pending_finalize is not None: - accepted, error = pending_finalize - coordinator._finalize_block_candidate( - candidate, - block_hash=block_hash, - accepted=accepted, - error=error, - outcome=outcome, - ) - return accepted - try: - if not coordinator._reserve_block_fast_lane_slot(block_hash): - raise RuntimeError( - "block candidate is waiting for fast-lane capacity" - ) - # Everything from definitive acceptance to here was queue, - # disposition and admission time the landing itself did not - # control (issue #224). A candidate whose earlier pass already - # stamped acceptance closes its lane here, before the retained - # offer is even looked up, so none of this landing's own work - # can reach ``lane_wait``. - self._note_accepted_landing_lane_start(candidate) - node_submission = coordinator._node_submission_for_candidate_or_retained(candidate) - # A *fresh* candidate has no stamp until the call above makes - # the offer, so the close before it found no interval to - # measure. Close again now that one may exist: it is a no-op - # once this interval's lane was recorded, so the retained case - # keeps the earlier, tighter sample and the fresh case reports - # the zero queue time it genuinely had rather than no sample - # at all. - self._note_accepted_landing_lane_start(candidate) - # Both closes sit above the attempt marker on purpose. The - # marker is this landing's own PostgreSQL write: charging it - # to ``lane_wait`` would grow the one stretch that exonerates - # the landing exactly when the writer is degraded, and a - # marker that timed out would keep growing it across the - # retry backoff. - coordinator._mark_block_candidate_attempted(block_hash) - with coordinator._block_landing_ledger_statement_timeout_scope(block_hash): - # The same-hash disposition is already held here, so the - # coordinator may run its serialized inner tail rather - # than the public entrypoint that would take that guard - # again. Which of the two applies is its call to make. - block_landed = self.ports.submit_candidate( - candidate, - node_submission=node_submission, - disposition_held=True, - ) - except BaseException: - coordinator._retain_block_candidate_for_retry(candidate) - raise - - if not block_landed: - reason = getattr(outcome, "reason", None) - if reason in {None, *self.retryable_reasons}: - coordinator._retain_block_candidate_for_retry(candidate) - raise RuntimeError( - "block candidate outcome is pending durable retry" - ) - abandon_error = getattr(outcome, "error", None) - try: - coordinator._record_block_submitter_phase( - "reject-prepared-block" - ) - with coordinator._block_landing_ledger_statement_timeout_scope( - block_hash - ): - coordinator._reject_terminal_prepared_block_candidate(candidate) - coordinator._record_block_submitter_phase( - "reject-prepared-block:complete" - ) - except Exception as exc: - # A prior attempt may have persisted prepared payout rows - # before this synchronous resubmit reached a false terminal - # verdict. Keep the outbox pending until those rows can be - # rejected; otherwise restart replay is removed while its - # balance transition remains live. - coordinator._defer_block_candidate( - "backend-rpc-unavailable", - "could not reject prepared state for terminal candidate", - worker=candidate.client.username or None, - ) - coordinator._retain_block_candidate_for_retry(candidate) - raise RuntimeError( - "could not reject prepared state for terminal candidate" - ) from exc - coordinator._finalize_block_candidate( - candidate, - block_hash=block_hash, - accepted=False, - error=str(abandon_error or reason), - outcome=outcome, - ) - return False - - coordinator._finalize_block_candidate( - candidate, - block_hash=block_hash, - accepted=True, - error="", - outcome=outcome, - ) - return True - - # -- same-hash disposition guard --------------------------------------- - - def _ensure_block_candidate_disposition_state(self) -> None: - """Backfill same-hash submission guards for lightweight embedders.""" - if ( - hasattr(self, "_block_candidate_disposition_registry_lock") - and hasattr(self, "_block_candidate_disposition_flights") - and hasattr(self, "_block_candidate_terminal_outcomes") - and hasattr(self, "_block_fast_lane_reservations") - and hasattr(self, "_block_disposition_waiting_retries") - and hasattr(self, "_block_candidate_dequeued_hashes") - ): - return - with _STATE_BACKFILL_LOCK: - if not hasattr(self, "_block_candidate_disposition_registry_lock"): - self._block_candidate_disposition_registry_lock = threading.Lock() - if not hasattr(self, "_block_candidate_disposition_flights"): - self._block_candidate_disposition_flights: dict[ - str, _BlockCandidateDispositionFlight - ] = {} - if not hasattr(self, "_block_candidate_terminal_outcomes"): - self._block_candidate_terminal_outcomes: dict[str, bool] = {} - if not hasattr(self, "_block_fast_lane_reservations"): - self._block_fast_lane_reservations: set[str] = set() - if not hasattr(self, "_block_disposition_waiting_retries"): - self._block_disposition_waiting_retries: dict[ - str, PrismBlockCandidate - ] = {} - if not hasattr(self, "_block_candidate_dequeued_hashes"): - # hash -> number of dequeued same-hash candidate objects that - # are between a lane and their disposition flight (or the - # waiting registry). Read and written under coordinator.lock. - self._block_candidate_dequeued_hashes: dict[str, int] = {} - - def _claim_block_candidate_disposition( - self, - block_hash: str, - *, - blocking: bool, - ) -> _BlockCandidateDispositionLease | None: - """Claim one hash without making unrelated node offers wait.""" - key = block_hash.lower() - self._ensure_block_candidate_disposition_state() - with self._block_submitter_lock( - self._block_candidate_disposition_registry_lock, - "candidate-disposition-registry", - ): - flight = self._block_candidate_disposition_flights.get(key) - if flight is None: - flight = _BlockCandidateDispositionFlight() - self._block_candidate_disposition_flights[key] = flight - flight.users += 1 - if blocking: - self._acquire_block_submitter_lock( - flight.lock, - f"candidate-disposition:{key}", - ) - acquired = True - else: - acquired = flight.lock.acquire(blocking=False) - if acquired: - return _BlockCandidateDispositionLease(key, flight) - self._drop_block_candidate_disposition_user(key, flight) - return None - - def _drop_block_candidate_disposition_user( - self, - key: str, - flight: _BlockCandidateDispositionFlight, - ) -> None: - with self._block_submitter_lock( - self._block_candidate_disposition_registry_lock, - "candidate-disposition-registry", - ): - flight.users -= 1 - if ( - flight.users == 0 - and self._block_candidate_disposition_flights.get(key) is flight - ): - self._block_candidate_disposition_flights.pop(key, None) - - def _release_block_candidate_disposition( - self, - lease: _BlockCandidateDispositionLease, - ) -> None: - lease.flight.lock.release() - self._drop_block_candidate_disposition_user( - lease.block_hash, - lease.flight, - ) - - # -- dequeued-hash pin ------------------------------------------------- - - def _pin_dequeued_block_candidate_locked( - self, - candidate: PrismBlockCandidate, - ) -> str: - """Name a candidate's hash from the instant it leaves its lane. - - Caller holds ``coordinator.lock`` and, in the same critical section, - has just taken ``candidate`` out of the retry holder, a candidate - queue, or the waiting registry. Until ``submit_next`` hands the hash - to its disposition flight or back to the waiting registry -- again - under the lock -- the candidate object lives only in that method's - local variable, and this pin is the only thing that tells the - terminal-outcome eviction a same-hash copy is still in memory. The - lanes and the flight each pin the hash in their own right; without - this entry a terminal duplicate's fence is the oldest unpinned - outcome exactly while the pass that is about to read it holds the - candidate in hand. - - Counted rather than set-valued so two dequeued same-hash objects - (a second caller of ``submit_next``) cannot unpin each other. - Returns the lowercase key the rest of the pass compares on. - """ - block_hash = str(candidate.submission.block_hash_hex).lower() - pins = self._block_candidate_dequeued_hashes - pins[block_hash] = pins.get(block_hash, 0) + 1 - return block_hash - - def _unpin_dequeued_block_candidate_locked(self, block_hash: str) -> None: - """Drop one dequeue's pin. Caller holds ``coordinator.lock``. - - Called only in the critical section that hands the hash on -- to the - flight the claim installed, or to the waiting registry -- or where - nothing needs it any more because the candidate object left with an - exception and no in-memory copy remains to offer the block. - """ - pins = self._block_candidate_dequeued_hashes - remaining = pins.get(block_hash, 0) - 1 - if remaining > 0: - pins[block_hash] = remaining - else: - pins.pop(block_hash, None) - - def _live_block_candidate_hash_registries(self) -> tuple[Any, ...]: - """Registries whose named hash forbids evicting its terminal outcome. - - Each one either holds a candidate object that has not reached the - node yet or the still-owed terminal work of one, and every such copy - reads the terminal outcome before it may offer the block again. - Dropping the outcome under one of them is exactly the window - ``_publish_collapsed_candidate_terminal_fence`` exists to close. - The dequeued-hash pin is the lane between lanes: the candidate - ``submit_next`` has taken out of a holder, queue, or the waiting - registry and not yet handed to its disposition flight. - - Read with ``coordinator.lock`` held. Backfilled state is fetched by - ``getattr`` rather than by an ``_ensure_...`` call, which would take - the state-backfill lock underneath the global one. - """ - return tuple( - registry - for registry in ( - getattr(self, "_outstanding_block_candidate_hashes", None), - getattr(self, "_block_replay_inflight_hashes", None), - getattr(self, "_block_quarantine_hashes", None), - getattr(self, "_block_fast_lane_reservations", None), - getattr(self, "_block_disposition_waiting_retries", None), - getattr(self, "finalize_retries", None), - getattr(self, "_block_candidate_retained_node_submissions", None), - getattr(self, "_block_candidate_collapse_cleanup_retries", None), - getattr(self, "_block_candidate_collapse_cleanup_inflight", None), - getattr(self, "_block_candidate_dequeued_hashes", None), - ) - if registry - ) - - def _held_block_candidate_retry_hashes(self) -> frozenset[str]: - """The hashes the single-slot retry holders are still carrying.""" - held: set[str] = set() - for holder in ( - getattr(self, "retry_candidate", None), - getattr(self, "_block_accounting_deferred_retry_candidate", None), - ): - key = _block_candidate_hash_of(holder) - if key is not None: - held.add(key) - return frozenset(held) - - @staticmethod - def _queued_block_candidate_hashes(queue_obj: object) -> frozenset[str] | None: - """The hashes still sitting in one candidate queue, or None if unread. - - The registries above stop naming a hash the moment its terminal - outcome is recorded -- ``_record_block_candidate_terminal_outcome`` - and the collapse cleanup's ``outstanding-and-tip-observation`` step - clear the outstanding and replay-inflight markers by design -- so a - *duplicate* copy of that hash still queued behind the pass that - finished would be named by nothing. That copy is precisely the one - the fence protects, so the queues themselves are read. - - The underlying deque is read under the queue's own mutex, because - iterating it while a producer or consumer mutates it is not safe. - The acquire is non-blocking: this runs with ``coordinator.lock`` - held, and waiting there for a queue's leaf lock would convoy the - whole coordinator behind an unrelated enqueue. Failing to take it - returns None and the caller skips the eviction pass entirely, which - only leaves the registry over its bound until the next outcome. - """ - if queue_obj is None: - return frozenset() - mutex = getattr(queue_obj, "mutex", None) - entries = getattr(queue_obj, "queue", None) - if mutex is None or entries is None: - return frozenset() - if not mutex.acquire(blocking=False): - return None - try: - items = list(entries) - finally: - mutex.release() - queued: set[str] = set() - for item in items: - key = _block_candidate_hash_of(item) - if key is not None: - queued.add(key) - return frozenset(queued) - - def _in_flight_block_candidate_hashes(self) -> frozenset[str] | None: - """The hashes with a live same-hash disposition flight, or None. - - A claimed flight means a pass is between its node offer and its - durable finalization for that hash. Same non-blocking rule as the - queues: the disposition registry lock is taken under - ``coordinator.lock`` here, the reverse of the ordinary order, so it - is only ever tried, never waited on. - """ - lock = getattr(self, "_block_candidate_disposition_registry_lock", None) - flights = getattr(self, "_block_candidate_disposition_flights", None) - if lock is None or flights is None: - return frozenset() - if not lock.acquire(blocking=False): - return None - try: - return frozenset(flights) - finally: - lock.release() - - def _collect_live_block_candidate_pins( - self, - ) -> tuple[tuple[Any, ...], frozenset[str]] | None: - """The one live pin set every hash-liveness proof reads. - - Returns the lane registries (checked by membership) and the union of - the retry-holder, queued (live and replay), and claimed-flight - hashes, or None when a leaf lane could not be read without waiting - under the global lock. Read with ``coordinator.lock`` held. Both the - terminal-outcome eviction and the ancestor re-drive's ownership - check (issue #190) sit on this collector, deliberately: each one's - safety argument is that a hash named by none of these sources has no - in-memory copy left, and a lane added to one consumer but not the - other would silently void the proof of whichever fell behind. - """ - pinned = self._live_block_candidate_hash_registries() - held = set(self._held_block_candidate_retry_hashes()) - for live in ( - self._queued_block_candidate_hashes( - getattr(self, "candidate_queue", None) - ), - self._queued_block_candidate_hashes( - getattr(self, "_block_replay_candidate_queue", None) - ), - self._in_flight_block_candidate_hashes(), - ): - if live is None: - return None - held |= live - return pinned, frozenset(held) - - @staticmethod - def _stamp_block_candidate_terminal_outcome( - outcomes: dict[str, bool], - block_hash: str, - accepted: bool, - ) -> None: - """Record one outcome as the newest entry of the FIFO registry. - - Re-recording an existing hash would otherwise keep its original - insertion position, leaving a freshly reasserted outcome at the - front of the eviction scan. - """ - outcomes.pop(block_hash, None) - outcomes[block_hash] = accepted - - def _bound_block_candidate_terminal_outcomes( - self, - protect: frozenset[str] = frozenset(), - ) -> int: - """Trim the terminal-outcome registry to its bound; return evictions. - - Called by the two writers with ``coordinator.lock`` already held and - their own entries already stamped. ``protect`` names the hashes this - very call published: a bulk fence page larger than the whole bound - must not evict the outcomes it just wrote, so the registry is left - temporarily over its bound instead. - - Eviction is oldest-first and skips every pinned hash, so the bound - is a target rather than a hard ceiling -- a registry whose entries - are all still live simply stays large, which is the safe direction. - The pins are themselves bounded state, so the total stays bounded. - - The pin set is the union of the lane registries (including the - dequeued-hash pin), the single-slot retry holders, the candidate - objects still queued on the live and replay lanes, and the hashes - with a claimed disposition flight. A hash held by none of those has - no in-memory copy left that could reach a node offer; anything - arriving for it afterwards is a fresh admission, which re-reads the - durable outbox row -- terminal there either way -- before it can - offer anything. - - That same proof retires the hash's counted-abandonment dedup key. - The key exists so a re-run terminal disposition -- a collapse whose - cleanup failed, a finalize-only replay -- counts its candidate once - rather than once per attempt, so it may only be dropped when no - copy is left to re-run anything. Both writer orderings are covered - by the one rule: the direct finalize path counts before it publishes - its terminal outcome, so its key is retired by a later pass over the - outcome it goes on to publish, and an ambiguous false finalize - failure has no outcome to evict at all while ``finalize_retries`` - pins it; the collapse publishes the outcome first and counts during - cleanup, and holds the hash's disposition lease -- an in-flight pin - -- across both. Dropping the key here therefore never lets a lane - that is still owed accounting count its candidate twice. - """ - outcomes = self._block_candidate_terminal_outcomes - overflow = len(outcomes) - MAX_BLOCK_CANDIDATE_TERMINAL_OUTCOMES - if overflow <= 0: - return 0 - pins = self._collect_live_block_candidate_pins() - if pins is None: - # A lane could not be read without waiting under the global - # lock. Evicting against an incomplete pin set could unfence - # a live copy, so nothing is evicted this round. - return 0 - pinned, held = pins - counted = getattr(self, "_counted_block_candidate_abandonments", None) - window = overflow + BLOCK_CANDIDATE_TERMINAL_OUTCOME_EVICTION_SCAN - dropped = 0 - for key in list(itertools.islice(outcomes, window)): - if dropped >= overflow: - break - if ( - key in protect - or key in held - or any(key in registry for registry in pinned) - ): - # A live, replayed, retried, parked, quarantined, or - # cleanup-owing copy of this hash is still in memory and - # will read this outcome before it could offer the block. - # Move it behind the scan window so it stops blocking the - # next pass instead of dropping it. - outcomes[key] = outcomes.pop(key) - continue - del outcomes[key] - if counted is not None: - # Retired under the very proof that let the fence go: a - # dedup key outliving its fence is what let this set grow - # without bound under a collapsed storm. - counted.discard(key) - dropped += 1 - return dropped - - def _block_candidate_terminal_outcome(self, block_hash: str) -> bool | None: - self._ensure_block_candidate_disposition_state() - with self._coordinator.lock: - return self._block_candidate_terminal_outcomes.get(block_hash.lower()) - - def _record_block_candidate_terminal_outcome( - self, - block_hash: str, - *, - accepted: bool, - ) -> None: - self._ensure_block_candidate_disposition_state() - dropped_waiting: PrismBlockCandidate | None = None - with self._coordinator.lock: - key = block_hash.lower() - self._stamp_block_candidate_terminal_outcome( - self._block_candidate_terminal_outcomes, - key, - accepted, - ) - self._bound_block_candidate_terminal_outcomes(frozenset((key,))) - self._block_fast_lane_reservations.discard(key) - replay_hashes = getattr(self, "_block_replay_inflight_hashes", None) - if replay_hashes is not None: - replay_hashes.discard(key) - waiting = getattr(self, "_block_disposition_waiting_retries", None) - if waiting is not None: - dropped_waiting = waiting.pop(key, None) - if dropped_waiting is not None: - # The parked same-hash wakeup dies with the terminal outcome - # recorded; its floor holder must not outlive it. - self._release_dropped_duplicate_candidate_floor(dropped_waiting) - - def _record_committed_block_candidate_abandonment( - self, - block_hash: str, - outcome: threading.local, - ) -> None: - """Count an abandonment only after its terminal cleanup is fixed. - - ``_abandon_block_candidate`` seals a proposed rejection before any - prepared payout rows are removed. That cleanup can fail, in which - case the candidate is deliberately re-registered and can still prove - accepted on a later pass. Counting at the seal would then expose the - same hash as both abandoned and accepted. The writer calls this only - after cleanup succeeds or after a false finalize-only disposition is - installed; direct accounting callers invoke it only after the full - serialized rejection path returns. - - The dedup key that makes this once-per-candidate is retired by - ``_bound_block_candidate_terminal_outcomes`` when that pass evicts - the hash's terminal outcome, which is the only place that has proved - no lane can re-enter this method for the hash. - """ - reason = getattr(outcome, "reason", None) - if not isinstance(reason, str) or not reason: - return - if reason in self.retryable_reasons: - return - # Terminal for this hash: no later attempt can close its interval, so - # the phase state it accumulated has no consumer left (issue #224). - self._discard_accepted_landing_attribution(block_hash) - stale_job_class = getattr(outcome, "stale_job_class", None) - key = block_hash.lower() - with self._coordinator.lock: - counted_abandonments = self._counted_block_candidate_abandonments - if key in counted_abandonments: - return - counted_abandonments.add(key) - counts = self.abandoned_counts - if counts is None: - counts = {} - self.abandoned_counts = counts - counts[reason] = int(counts.get(reason, 0)) + 1 - if stale_job_class is not None: - stale_counts = getattr(self, "stale_job_abandon_counts", None) - if stale_counts is None: - stale_counts = { - abandon_class: 0 - for abandon_class in PRISM_STALE_JOB_ABANDON_CLASSES - } - self.stale_job_abandon_counts = stale_counts - stale_counts[stale_job_class] = ( - int(stale_counts.get(stale_job_class, 0)) + 1 - ) - - def _reserve_block_fast_lane_slot(self, block_hash: str) -> bool: - """Reserve pool capacity while a node offer awaits terminal accounting.""" - key = block_hash.lower() - self._ensure_block_candidate_disposition_state() - with self._coordinator.lock: - reservations = self._block_fast_lane_reservations - if key in reservations: - return True - accepted_count = int( - getattr(self._coordinator, "accepted_block_count", 0) - ) - capacity = int(getattr(self._coordinator, "max_blocks", 2**31 - 1)) - stop_after_one = bool( - getattr(self._coordinator, "stop_after_block", False) - ) - reserved_count = len(reservations) - if accepted_count + reserved_count >= capacity: - return False - if stop_after_one and accepted_count + reserved_count >= 1: - return False - reservations.add(key) - return True - - def _release_block_fast_lane_slot(self, block_hash: str) -> None: - self._ensure_block_candidate_disposition_state() - with self._coordinator.lock: - reservations = self._block_fast_lane_reservations - if reservations is not None: - reservations.discard(block_hash.lower()) - - @contextmanager - def _block_candidate_disposition( - self, - block_hash: str, - ) -> Iterator[_BlockCandidateDispositionLease]: - """Serialize the full accepted/abandoned decision for one hash. - - A below-share-target solve submits synchronously while the durable - outbox can concurrently replay that same candidate. Keep both attempts - ordered until the accepted success tail records its process-local - completion; otherwise the replay can terminally abandon the outbox - during the gap after durable confirmation but before audit/share - evidence is complete. - """ - lease = self._coordinator._claim_block_candidate_disposition( - block_hash, - blocking=True, - ) - assert lease is not None - try: - yield lease - finally: - self._coordinator._release_block_candidate_disposition(lease) - - # -- accounting actor -------------------------------------------------- - - def _ensure_block_accounting_state(self) -> None: - if not hasattr(self, "_block_accounting_state_lock"): - self._block_accounting_state_lock = threading.Lock() - if not hasattr(self, "_block_accounting_queue"): - depth = max( - 1, - int( - getattr( - self._coordinator, - "block_accounting_queue_depth", - DEFAULT_BLOCK_ACCOUNTING_QUEUE_DEPTH, - ) - ), - ) - self._block_accounting_queue = queue.PriorityQueue(maxsize=depth) - if not hasattr(self, "_block_accounting_overflow_queue"): - self._block_accounting_overflow_queue = queue.PriorityQueue() - if not hasattr(self, "_block_accounting_accepted_queue"): - # Deliberately unbounded, for the reason the overflow queue is: - # a node offer has already happened and must never be converted - # back into a raw-submit retry. A maxsize here would need a spill - # target, and any queue-state-dependent spill is exactly the - # inversion this lane removes. Max-block admission - # (_reserve_block_fast_lane_slot, max_blocks, stop_after_block) - # and the physical block-acceptance rate bound how many - # unresolved real offers can exist at once. - self._block_accounting_accepted_queue = queue.PriorityQueue() - if not hasattr(self, "_block_accounting_sequence"): - self._block_accounting_sequence = 0 - if not hasattr(self, "_block_accounting_thread"): - self._block_accounting_thread = None - - def _start_block_accounting_thread(self) -> threading.Thread: - self._ensure_block_accounting_state() - with self._block_accounting_state_lock: - thread = self._block_accounting_thread - if thread is not None and thread.is_alive(): - return thread - self._record_block_work_heartbeat("block_accounting", "starting") - thread = threading.Thread( - target=self._coordinator.block_accounting_loop, - name="prism-block-accounting", - daemon=True, - ) - self._block_accounting_thread = thread - thread.start() - return thread - - def _enqueue_block_accounting_task( - self, - task: _BlockCandidateAccountingTask, - ) -> bool: - self._ensure_block_accounting_state() - with self._block_accounting_state_lock: - sequence = self._block_accounting_sequence - self._block_accounting_sequence += 1 - priority = int(task.candidate.context.template["height"]) - item = (priority, sequence, task) - if _is_definitive_node_acceptance(task.node_submission): - # Definitive acceptance leaves the primary/overflow population - # entirely. It does not get a better key inside it -- a key there - # is inert exactly when it matters. Under a burst the spillover - # rule below has already engaged, so the accepted task would be - # put behind the spill while the stale entries still sitting in - # the bounded primary queue -- up to - # DEFAULT_BLOCK_ACCOUNTING_QUEUE_DEPTH of them -- are all served - # ahead of it, each costing ~6 RPCs, two ledger writes and a - # payout-balance mutation. Priority only ever orders one heap; the - # primary-then-overflow dequeue discipline is what inverts it. - # - # Membership is what changes here, not the spillover rule. That - # rule is unchanged below, and the property it exists to protect -- - # once spillover begins, later handoffs join it rather than - # repeatedly refilling the bounded primary, so older spill entries - # are not starved by newer arrivals -- still governs exactly the - # population it was written for: non-definitive accounting work. - # - # A separate lane makes this class structurally un-spillable: no - # queue-state-dependent rule can place it behind stale work, - # because it shares no queue with stale work. Fairness for the - # stale lanes stops being an emergent property of two heaps and - # becomes the explicit dispatch quota - # BLOCK_ACCOUNTING_ACCEPTED_DISPATCH_QUOTA, enforced in - # block_accounting_loop and bounded there rather than argued for. - self._block_accounting_accepted_queue.put_nowait(item) - return True - if not self._block_accounting_overflow_queue.empty(): - # Once spillover begins, keep later handoffs behind it instead of - # repeatedly refilling the primary queue and starving older spill - # entries. - self._block_accounting_overflow_queue.put_nowait(item) - return True - try: - self._block_accounting_queue.put_nowait(item) - return True - except queue.Full: - # A node offer has already happened and must never be converted - # back into a raw-submit retry. Preserve its result and lease in - # an unbounded, process-local overflow queue; max-block admission - # bounds the number of unresolved real offers. - self._block_accounting_overflow_queue.put_nowait(item) - print( - "prism coordinator: block accounting handoff spilled " - f"hash={task.candidate.submission.block_hash_hex}", - flush=True, - ) - return True - - def _call_block_candidate_writer( - self, - candidate: PrismBlockCandidate, - *, - node_submission: _BlockCandidateNodeSubmission, - disposition_held: bool, - ) -> bool: - """Invoke the writer while preserving duck-typed test integrations.""" - writer = self._coordinator._submit_next_block_candidate_writer - supports_disposition_held = True - try: - parameters = inspect.signature(writer).parameters.values() - supports_disposition_held = any( - parameter.name == "disposition_held" - or parameter.kind is inspect.Parameter.VAR_KEYWORD - for parameter in parameters - ) - except (TypeError, ValueError): - pass - if supports_disposition_held: - return bool( - writer( - candidate, - node_submission=node_submission, - disposition_held=disposition_held, - ) - ) - return bool(writer(candidate, node_submission=node_submission)) - - def _restore_replayed_candidate_acceptance_evidence( - self, - candidate: PrismBlockCandidate, - ) -> None: - if not candidate.durable_replay: - return - block_hash = str(candidate.submission.block_hash_hex).lower() - block_state = None - state_read_failed = False - state_reader = getattr(self._coordinator.ledger, "pool_block_state", None) - if callable(state_reader): - try: - block_state = self._coordinator._run_block_submitter_ledger_call( - ("replay-pool-block-state", block_hash), - "replay-pool-block-state", - lambda: state_reader(block_hash=block_hash), - ) - except Exception: - traceback.print_exc() - state_read_failed = True - durable_chain_state = ( - str(block_state.get("chain_state", "")) - if block_state is not None - else "" - ) - if state_read_failed or durable_chain_state in {"prepared", "confirmed"}: - self._coordinator._register_outstanding_block_candidate(block_hash) - with self._coordinator.lock: - self._tip_observed_accepted_block_hashes[block_hash] = ( - time.monotonic() - ) - print( - "prism coordinator: restored acceptance evidence for " - f"replayed block candidate hash={block_hash} " - + ( - "after a failed durable-state read" - if state_read_failed - else f"chain_state={durable_chain_state}" - ), - flush=True, - ) - - def _run_block_accounting_task( - self, - task: _BlockCandidateAccountingTask, - ) -> None: - candidate = task.candidate - with self._coordinator.lock: - self._block_accounting_holds_disposition = True - self._block_accounting_deferred_retry_candidate = None - outcome = self.outcome - outcome.refresh_client = None - try: - self._coordinator._restore_replayed_candidate_acceptance_evidence( - candidate - ) - with self._coordinator._writer_operation("accepted_block_handling"): - self._coordinator._call_block_candidate_writer( - candidate, - node_submission=task.node_submission, - disposition_held=True, - ) - refresh_client = getattr(outcome, "refresh_client", None) - outcome.refresh_client = None - except ShutdownInProgress: - return - except Exception: - # An unexpected failure leaves the candidate to the retry lane. - # Retain it here, while this thread still owns the disposition: - # an accounting-owner retain routes into the deferred holder, - # which names the hash until the finally below merges it into - # the retry slot under coordinator.lock -- after the lease is - # released, but with no instant in between where nothing names - # it. The loop's catch used to retain only after this finally - # had already released the lease, and that instant was exactly - # the dequeued-candidate gap in its accounting-lane form: a - # terminal-outcome eviction there could unfence the hash the - # retried candidate was about to be re-disposed against. - self._coordinator._retain_block_candidate_for_retry(candidate) - task.retained_for_retry.set() - raise - finally: - self._coordinator._release_block_candidate_disposition( - task.disposition_lease - ) - with self._coordinator.lock: - self._block_accounting_holds_disposition = False - deferred_retry = getattr( - self, - "_block_accounting_deferred_retry_candidate", - None, - ) - self._block_accounting_deferred_retry_candidate = None - if deferred_retry is not None: - self._merge_block_candidate_retry_locked( - "_retry_block_candidate", - deferred_retry, - ) - if refresh_client is not None and not self._coordinator.stop_event.is_set(): - self._coordinator._record_block_submitter_phase("refresh-jobs") - self._coordinator.refresh_jobs_after_pending_accepted_block( - refresh_client, - heartbeat_name="block_accounting", - ) - self._coordinator._record_block_submitter_phase("refresh-jobs:complete") - - def _block_accounting_cleanup_retry_work_items(self) -> int: - """Completed work items the lane may run between cleanup offers. - - One work item is one finished ordinary accounting task from any of - the handoff queues, or one finished invalid-candidate quarantine - item. Bounded below at one so a misconfigured value degrades into - offering the retry after every item rather than never offering it at - all. - """ - return max( - 1, - int( - getattr( - self._coordinator, - "block_accounting_cleanup_retry_work_items", - DEFAULT_BLOCK_ACCOUNTING_CLEANUP_RETRY_WORK_ITEMS, - ) - ), - ) - - def _block_accounting_accepted_dispatch_quota(self) -> int: - """Consecutive accepted dispatches allowed while stale work waits. - - Bounded below at one so a misconfigured value degrades into strict - alternation -- one accepted task per stale task -- rather than into - an accepted lane that can never dispatch while any stale work exists. - """ - return max( - 1, - int( - getattr( - self._coordinator, - "block_accounting_accepted_dispatch_quota", - BLOCK_ACCOUNTING_ACCEPTED_DISPATCH_QUOTA, - ) - ), - ) - - def block_accounting_loop(self) -> None: - self._block_accounting_thread_ident = threading.get_ident() - self._ensure_block_accounting_state() - # Work items completed since the last deferred-cleanup offer. The - # idle branch below cannot be the only driver: either lane running - # continuously would hold a due terminal cleanup off for as long as - # the traffic lasted, and its durable row is already gone. Counting - # items here bounds that wait at one offer per configured cadence - # whichever lane the traffic is on. - work_items = 0 - # Definitive-acceptance tasks dispatched back to back since the last - # non-accepted dispatch. Counted separately from work_items on - # purpose: work_items paces the deferred-cleanup offer against *all* - # completed work, while this counter measures one thing only -- how - # long the accepted lane has been holding the stale lanes off. Never - # fold the two together. - accepted_run = 0 - while not self._coordinator.stop_event.is_set(): - self._coordinator._record_block_submitter_phase("accounting-queue") - source_queue = None - task = None - # Selection order is the accepted lane's whole contract, and it - # is stated as a bound rather than as a preference: - # - # whenever non-accepted accounting work is available, at most - # `quota` accepted tasks are dispatched between two consecutive - # dispatches of non-accepted work. - # - # Equivalently, a non-accepted task at the head of its lane waits - # behind at most `quota` accepted services, never behind an - # unbounded stream of them, whatever the accepted arrival rate. - # Starving it is not an option: a stale accounting task that never - # runs leaves its outbox row pending, its offer-time payout-preview - # barrier armed and its disposition lease held. - if accepted_run < self._block_accounting_accepted_dispatch_quota(): - try: - _priority, _sequence, task = ( - self._block_accounting_accepted_queue.get_nowait() - ) - source_queue = self._block_accounting_accepted_queue - accepted_run += 1 - except queue.Empty: - pass - if source_queue is None: - # The stale population, dequeued with exactly the discipline - # it has always had: bounded primary first, then the - # unbounded spillover behind it. - try: - _priority, _sequence, task = ( - self._block_accounting_queue.get_nowait() - ) - source_queue = self._block_accounting_queue - except queue.Empty: - try: - _priority, _sequence, task = ( - self._block_accounting_overflow_queue.get_nowait() - ) - source_queue = self._block_accounting_overflow_queue - except queue.Empty: - pass - if source_queue is not None: - accepted_run = 0 - if source_queue is None: - # No stale work exists, so the quota has nothing to yield to - # and is honoured vacuously; anything the quota was holding - # back runs now, and the run counter restarts from this - # dispatch rather than carrying an exhausted count forward. - try: - _priority, _sequence, task = ( - self._block_accounting_accepted_queue.get_nowait() - ) - source_queue = self._block_accounting_accepted_queue - accepted_run = 0 - except queue.Empty: - pass - if source_queue is None: - if self._coordinator._run_one_invalid_block_candidate_quarantine(): - work_items += 1 - if ( - work_items - >= self._block_accounting_cleanup_retry_work_items() - ): - work_items = 0 - self._run_one_collapsed_block_candidate_cleanup_retry() - continue - # A collapsed row's failed cleanup has no durable replay - # source left, so the accounting lane is the only thing - # that will ever finish it. A truly idle lane offers it - # here immediately rather than waiting out a cadence; the - # work-item cadence exists for the busy lanes, which - # never reach this branch at all. - work_items = 0 - if self._run_one_collapsed_block_candidate_cleanup_retry(): - continue - self._coordinator.stop_event.wait(self._block_work_wait_slice()) - continue - try: - self._coordinator._run_block_accounting_task(task) - except Exception: - print( - "prism coordinator: block accounting iteration failed; " - "durable candidate remains pending", - flush=True, - ) - traceback.print_exc() - retained = getattr(task, "retained_for_retry", None) - if retained is None or not retained.is_set(): - # The failure fired before _run_block_accounting_task's - # own exception path could retain the candidate -- a - # delegate raising ahead of it -- so it is retained here - # as before. A candidate the lane already retained (and - # merged into the retry slot) is not retained again. - self._coordinator._retain_block_candidate_for_retry( - task.candidate - ) - finally: - assert source_queue is not None - source_queue.task_done() - work_items += 1 - if work_items >= self._block_accounting_cleanup_retry_work_items(): - # The counter resets on every offer, due record or not: a - # cleanup backlog may therefore spend at most one attempt per - # cadence, so it can never starve accounting or quarantine in - # return. Deliberately left uncontained and outside the task's - # try/finally -- the retry pops its record before attempting - # it, and only its own failure paths re-register it, so an - # outer catch here could swallow the last reference to a - # terminal hash's owed steps. - work_items = 0 - self._run_one_collapsed_block_candidate_cleanup_retry() - - # -- submitter loop ---------------------------------------------------- - - def run(self) -> None: - # Boundary stamps in _record_block_candidate_progress are gated to - # this thread so client-thread dispositions cannot refresh the - # submitter's liveness budget on its behalf. - self._block_submitter_thread_ident = threading.get_ident() - self._coordinator._start_block_accounting_thread() - while not self.ports.stop_event().is_set(): - self._coordinator._record_block_submitter_phase("loop") - try: - # The in-memory wakeup is already backed by the durable - # outbox. Drain it before any recovery query so a saturated - # database cannot delay the first node submission. - with self._coordinator.lock: - retry_ready = self.retry_candidate is not None - queue_obj = self.candidate_queue - replay_queue = getattr( - self, - "_block_replay_candidate_queue", - None, - ) - wakeup_ready = bool( - (queue_obj is not None and not queue_obj.empty()) - or (replay_queue is not None and not replay_queue.empty()) - ) - if ( - retry_ready or wakeup_ready - ) and self._coordinator.submit_next_block_candidate( - defer_accounting=True - ): - if not self._ancestor_redrive_owed(): - continue - # An armed ancestor re-drive (issue #190) falls through - # to the replay entrypoint even though immediate work - # succeeded: sustained live traffic taking this - # `continue` every pass is exactly how the wedge starved - # the one path that resolves a stuck ancestor. - self.ports.replay_entrypoint() - self._run_block_candidate_body_janitor_step() - self._coordinator.submit_next_block_candidate( - timeout=1.0, - defer_accounting=True, - ) - except ShutdownInProgress: - # Admission can close after the loop condition. Durable block - # candidates remain in the outbox for the replacement writer. - return - except Exception: - phase = getattr(self, "_block_submitter_phase", "unknown") - print( - "prism coordinator: block submitter iteration failed " - f"phase={phase}; durable candidates remain pending", - flush=True, - ) - traceback.print_exc() - retry_delay = float(self.retry_initial_seconds) - if self._coordinator._wait_for_block_candidate_retry(retry_delay): - return - - def submit_next( - self, - timeout: float | None = None, - *, - defer_accounting: bool = False, - ) -> BlockCandidateRunResult: - """Dequeue and land one block candidate; ``ran`` is True when one ran. - - The block-submitter loop calls this continuously; tests call it - directly (through the coordinator delegate) to drain the queue - deterministically. - """ - coordinator = self._coordinator - coordinator._record_block_submitter_phase("dequeue-retry") - # The disposition state owns the dequeued-hash pin, and its backfill - # takes the state-backfill lock, so it runs before any coordinator - # lock hold below rather than underneath one. - self._ensure_block_candidate_disposition_state() - # Every dequeue below leaves its lane and enters the dequeued-hash - # pin in one coordinator.lock critical section. The lanes pin a - # terminal duplicate's outcome against eviction while it sits in - # them and the disposition flight pins it from the claim onward; - # between the two the candidate object is only this method's local - # variable, and the pin is what keeps the fence it is about to read - # from being the oldest unpinned entry in the registry. - with coordinator.lock: - candidate = self.retry_candidate - if candidate is not None and not self._block_candidate_retry_ready_locked( - candidate - ): - # Parked by _pace_block_candidate_retry: honor the backoff - # deadline without sleeping on the submitter or accounting - # lane. - candidate = None - if candidate is not None: - self.retry_candidate = None - block_hash = self._pin_dequeued_block_candidate_locked(candidate) - if candidate is None: - queue_obj = self.candidate_queue - self._ensure_block_replay_state() - replay_queue = self._block_replay_candidate_queue - if queue_obj is None and replay_queue is None: - return BlockCandidateRunResult(False) - deadline = ( - None - if timeout is None - else time.monotonic() + max(0.0, timeout) - ) - while candidate is None: - coordinator._record_block_submitter_phase("dequeue-queue") - descriptor: DurableCandidateDescriptor | None = None - with coordinator.lock: - # Live discoveries always outrank durable restart replay. - # The non-blocking get takes the queue's own mutex under - # coordinator.lock -- the order the eviction's queue read - # already establishes -- so the entry leaves the queue - # and enters the pin without a gap between them. - for candidate_queue in (queue_obj, replay_queue): - if candidate_queue is None: - continue - try: - item = candidate_queue.get_nowait() - except queue.Empty: - continue - if isinstance(item, DurableCandidateDescriptor): - if not self._replay_descriptor_ready(item): - # Parked by a paced hydration retry: rotate - # it to the back and look at the next lane. - candidate_queue.put_nowait(item) - continue - descriptor = item - else: - candidate = item - break - if candidate is None and descriptor is None: - waiting = self._block_disposition_waiting_retries - ready_hashes = [ - key - for key in waiting - if self._block_candidate_retry_ready_locked( - waiting[key] - ) - ] - if ready_hashes: - waiting_hash = min( - ready_hashes, - key=lambda key: int( - waiting[key].context.template["height"] - ), - ) - candidate = waiting.pop(waiting_hash) - if candidate is not None: - block_hash = self._pin_dequeued_block_candidate_locked( - candidate - ) - if descriptor is not None: - # Body-sized work: hydrate outside coordinator.lock, one - # descriptor at a time. The hash stays named by the - # replay-inflight registry throughout, so the terminal - # outcome fence it will read cannot be evicted meanwhile. - candidate = self._hydrate_replay_descriptor(descriptor) - if candidate is None: - return BlockCandidateRunResult(True) - with coordinator.lock: - block_hash = self._pin_dequeued_block_candidate_locked( - candidate - ) - if candidate is not None: - break - if deadline is None: - return BlockCandidateRunResult(False) - remaining = deadline - time.monotonic() - if remaining <= 0: - return BlockCandidateRunResult(False) - if coordinator.stop_event.wait( - min(remaining, self._block_work_wait_slice()) - ): - return BlockCandidateRunResult(False) - - try: - lease = coordinator._claim_block_candidate_disposition( - block_hash, - blocking=not defer_accounting, - ) - except BaseException: - # The candidate object leaves with the exception (its durable - # outbox row still replays), so no in-memory copy remains that - # the pin would be protecting. - with coordinator.lock: - self._unpin_dequeued_block_candidate_locked(block_hash) - raise - if lease is None: - # Another same-hash pass already spans node offer through durable - # finalization. Keep this wakeup outside the global parent retry - # slot until that lease transfers/releases: consuming it can lose - # an accounting retry, while repeatedly prioritizing it can starve - # unrelated live blocks. The waiting registry takes over the pin - # in the same critical section. - with coordinator.lock: - self._block_disposition_waiting_retries[block_hash] = candidate - self._unpin_dequeued_block_candidate_locked(block_hash) - coordinator._wait_for_block_candidate_retry( - float(self.retry_initial_seconds) - ) - return BlockCandidateRunResult(True) - # The claim installed this pass's disposition flight, which the - # eviction reads as a pin of its own; hand the hash over to it. - with coordinator.lock: - self._unpin_dequeued_block_candidate_locked(block_hash) - transferred = False - if coordinator._block_candidate_terminal_outcome(block_hash) is not None: - coordinator._release_block_candidate_disposition(lease) - # Another same-hash object's disposition landed first; this - # duplicate is dropped here and nothing later can release the - # floor holder it carries. - self._release_dropped_duplicate_candidate_floor(candidate) - return BlockCandidateRunResult(True) - - outcome = self.outcome - outcome.refresh_client = None - coordinator._record_block_submitter_phase("finalize-registry") - with coordinator.lock: - pending_finalize = self.finalize_retries.get(block_hash) - if pending_finalize is None: - # Issue #181 item 2. A candidate whose parent is no longer the - # best tip, whose height the chain has decided in favour of a - # block of at least its work, and which carries no offer - # evidence, is provably stale before any offer: skip it here - # rather than pay a submitblock, ~6 chain reads, two ledger - # writes, an accounting task and a payout-preview barrier to - # learn the same thing per row. - # - # Placed after the lease claim and the terminal-outcome fence -- - # both of which this depends on -- and before the fast-lane - # reservation, so a sibling on its way to being abandoned never - # consumes max-block capacity. The skip terminalizes the durable - # row itself (it does not drop the wakeup): the lease it holds - # is already in #196's evidence set, so a dropped row would be - # rejected by that selector forever, and the dequeued object's - # pending-share floor holder can only be released by draining - # this object. - # - # Guarded exactly as the node offer below is: the skip contains - # its own failures and answers False for every one of them, but - # an escape here would leave this hash's lease held forever and - # drop the only object that can release its floor holder. - try: - stale = self._skip_superseded_block_candidate_at_dequeue( - candidate - ) - except BaseException: - try: - coordinator._retain_block_candidate_for_retry(candidate) - finally: - coordinator._release_block_candidate_disposition(lease) - raise - if stale: - coordinator._release_block_candidate_disposition(lease) - return BlockCandidateRunResult(True) - permanently_closed = False - already_accounted = False - if defer_accounting: - with coordinator.lock: - accounted_hashes = getattr( - coordinator, - "_accounted_accepted_block_hashes", - set(), - ) - already_accounted = block_hash in accounted_hashes - accepted_count = int( - getattr(coordinator, "accepted_block_count", 0) - ) - permanently_closed = not already_accounted and ( - accepted_count - >= int(getattr(coordinator, "max_blocks", 2**31 - 1)) - or ( - bool(getattr(coordinator, "stop_after_block", False)) - and accepted_count >= 1 - ) - ) - if permanently_closed or already_accounted: - # Accounting must terminalize a durable outbox row even - # after pool capacity closes. An already-accounted hash - # likewise needs only its exact-idempotent/finalize tail. - node_submission = _BlockCandidateNodeSubmission( - attempted=False - ) - elif not coordinator._reserve_block_fast_lane_slot(block_hash): - # Capacity is provisionally occupied by another unresolved - # node offer. Preserve strict max-block semantics until - # that offer either accounts or terminates. Retained - # before the lease goes, as on the exception paths, so - # the retry holder names the hash before the flight - # stops doing so. - coordinator._retain_block_candidate_for_retry(candidate) - coordinator._release_block_candidate_disposition(lease) - coordinator._wait_for_block_candidate_retry( - float(self.retry_initial_seconds) - ) - return BlockCandidateRunResult(True) - else: - try: - node_submission = coordinator._node_submission_for_candidate_or_retained(candidate) - except BaseException: - try: - coordinator._retain_block_candidate_for_retry(candidate) - finally: - coordinator._release_block_candidate_disposition(lease) - raise - else: - try: - node_submission = coordinator._node_submission_for_candidate_or_retained(candidate) - except BaseException: - try: - coordinator._retain_block_candidate_for_retry(candidate) - finally: - coordinator._release_block_candidate_disposition(lease) - raise - else: - node_submission = _BlockCandidateNodeSubmission(attempted=False) - - if defer_accounting: - task = _BlockCandidateAccountingTask( - candidate=candidate, - node_submission=node_submission, - disposition_lease=lease, - ) - try: - enqueued = coordinator._enqueue_block_accounting_task(task) - except BaseException: - try: - coordinator._retain_block_candidate_for_retry(candidate) - finally: - coordinator._release_block_candidate_disposition(lease) - raise - if enqueued: - transferred = True - return BlockCandidateRunResult(True) - coordinator._retain_block_candidate_for_retry(candidate) - coordinator._release_block_candidate_disposition(lease) - coordinator._wait_for_block_candidate_retry( - float(self.retry_initial_seconds) - ) - return BlockCandidateRunResult(True) - - try: - with coordinator._writer_operation("accepted_block_handling"): - ran = coordinator._call_block_candidate_writer( - candidate, - node_submission=node_submission, - disposition_held=True, - ) - refresh_client = getattr(outcome, "refresh_client", None) - outcome.refresh_client = None - except ShutdownInProgress: - # The durable outbox remains pending and the replacement process - # will replay it. Dequeuing the in-memory wakeup during the - # admission-close race cannot lose candidate work. - return BlockCandidateRunResult(False) - finally: - if not transferred: - coordinator._release_block_candidate_disposition(lease) - # Fresh-job fanout is deliberately outside the writer admission. Once - # the candidate outbox is finalized it cannot mutate the ledger, so a - # blocked client send must not hold the writer lease during shutdown. - if refresh_client is not None and not coordinator.stop_event.is_set(): - coordinator._record_block_submitter_phase("refresh-jobs") - coordinator.refresh_jobs_after_pending_accepted_block( - refresh_client, - heartbeat_name="block_submitter", - ) - coordinator._record_block_submitter_phase("refresh-jobs:complete") - return BlockCandidateRunResult(ran, refresh_client) - - def attempt(self, candidate: PrismBlockCandidate) -> BlockCandidateAttemptResult: - """Run one direct landing attempt and structure its outcome.""" - self.outcome.reason = None - error = "candidate became stale or submission failed" - self._note_accepted_landing_lane_start(candidate) - try: - accepted = self.ports.submit_candidate(candidate) - except Exception: - accepted = False - error = "candidate submission raised an exception" - self.ports.log( - "prism coordinator: block candidate submission failed " - f"hash={candidate.submission.block_hash_hex}" - ) - traceback.print_exc() - return BlockCandidateAttemptResult( - accepted=accepted, - reason=getattr(self.outcome, "reason", None), - error=error, - ) - - def submit_writer( - self, - candidate: PrismBlockCandidate, - *, - node_submission: _BlockCandidateNodeSubmission | None = None, - disposition_held: bool = False, - ) -> bool: - """Land one dequeued block candidate inside writer admission.""" - coordinator = self._coordinator - block_hash = str(candidate.submission.block_hash_hex).lower() - if not disposition_held: - # Preserve the historical direct-writer seam while keeping its - # node offer and terminal outbox decision inside the same-hash - # guard. Production queue/accounting calls transfer an existing - # lease and skip this wrapper. - with coordinator._block_candidate_disposition(block_hash): - terminal_outcome = coordinator._block_candidate_terminal_outcome( - block_hash - ) - if terminal_outcome is not None: - return terminal_outcome - if node_submission is None: - node_submission = ( - coordinator._node_submission_for_candidate_or_retained( - candidate - ) - ) - return coordinator._submit_next_block_candidate_writer( - candidate, - node_submission=node_submission, - disposition_held=True, - ) - outcome = self.outcome - outcome.reason = None - outcome.error = None - outcome.stale_job_class = None - coordinator._record_block_submitter_phase("finalize-registry") - with coordinator.lock: - pending_finalize = self.finalize_retries.get(block_hash) - if pending_finalize is not None: - # Finalize-only replay: node submission, terminal accounting, and - # payout persistence already completed on the pass that armed - # this entry. It bypasses both submitblock and attempt marking. - accepted, error = pending_finalize - return coordinator._finalize_block_candidate( - candidate, - block_hash=block_hash, - accepted=accepted, - error=error, - outcome=outcome, - ) - # The queue, disposition and admission time ends here: everything - # below is work this landing runs (issue #224). A candidate whose - # earlier pass already stamped acceptance closes its lane here, before - # any retained offer is resolved. - self._note_accepted_landing_lane_start(candidate) - if node_submission is None: - node_submission = coordinator._node_submission_for_candidate_or_retained(candidate) - # A fresh candidate is stamped by the offer that call makes, so the - # close above found no interval. Close again against the stamp that - # now exists; it is a no-op once this interval's lane was recorded. - self._note_accepted_landing_lane_start(candidate) - # Both closes sit above the attempt marker on purpose. The marker is - # this landing's own PostgreSQL write, and a degraded writer must not - # inflate the one stretch that exonerates the landing -- nor keep - # inflating it across the retry backoff a timed-out marker triggers. - try: - coordinator._mark_block_candidate_attempted(block_hash) - except Exception: - print( - "prism coordinator: could not record block candidate attempt " - f"hash={block_hash}", - flush=True, - ) - traceback.print_exc() - coordinator._retain_block_candidate_for_retry(candidate) - coordinator._pace_block_candidate_retry(block_hash) - return True - accepted = False - error = "candidate became stale or submission failed" - try: - coordinator._record_block_submitter_phase("accounting") - with coordinator._block_landing_ledger_statement_timeout_scope(block_hash): - # ``disposition_held`` states that the serialized inner tail - # is available; a ``node_submission`` of None still selects - # the historical bare entrypoint call. Both distinctions are - # carried to the port rather than decided from here. - accepted = self.ports.submit_candidate( - candidate, - node_submission=node_submission, - disposition_held=disposition_held, - ) - except Exception: - error = "candidate submission raised an exception" - print( - "prism coordinator: block candidate submission failed " - f"hash={candidate.submission.block_hash_hex}", - flush=True, - ) - traceback.print_exc() - abandon_reason = getattr(outcome, "reason", None) if outcome is not None else None - abandon_error = getattr(outcome, "error", None) if outcome is not None else None - if not accepted and abandon_error: - error = str(abandon_error) - retryable = not accepted and ( - abandon_reason is None - or abandon_reason in self.retryable_reasons - ) - if retryable: - # Leave the outbox row pending. It will replay after a short pause - # or on process restart. Keep this parent ahead of queued children: - # a child built from its prospective balances cannot be validated - # against the database until the parent confirmation catches up. - print( - "prism coordinator: retained block candidate for retry " - f"hash={block_hash} reason={abandon_reason or 'exception'}", - flush=True, - ) - coordinator._retain_block_candidate_for_retry(candidate) - coordinator._pace_block_candidate_retry(block_hash) - return True - if not accepted: - try: - coordinator._record_block_submitter_phase("reject-prepared-block") - with coordinator._block_landing_ledger_statement_timeout_scope(block_hash): - coordinator._reject_terminal_prepared_block_candidate(candidate) - coordinator._record_block_submitter_phase( - "reject-prepared-block:complete" - ) - except Exception: - # Persistence may have committed before a later RPC/transport - # failure. Do not terminally discard the outbox row until its - # prepared balance deltas have also reached a terminal state. - print( - "prism coordinator: prepared block cleanup failed " - f"hash={block_hash}", - flush=True, - ) - traceback.print_exc() - coordinator._defer_block_candidate( - "backend-rpc-unavailable", - "could not reject prepared state for terminal candidate", - worker=candidate.client.username or None, - ) - coordinator._retain_block_candidate_for_retry(candidate) - coordinator._wait_for_block_candidate_retry( - coordinator._next_block_candidate_retry_delay(block_hash) - ) - return True - return coordinator._finalize_block_candidate( - candidate, - block_hash=block_hash, - accepted=accepted, - error=error, - outcome=outcome, - ) - - def finalize( - self, - candidate: PrismBlockCandidate, - *, - block_hash: str, - accepted: bool, - error: str, - outcome: threading.local, - ) -> bool: - """Drive a terminal candidate's durable outbox update, with backoff. - - Failure retains the candidate as a finalize-only replay: the next - paced attempt re-enters here directly, never submit_block_candidate, - so terminal abandonment accounting stays once-per-candidate and an - accepted candidate's audit/persist work is not redone per retry. - """ - coordinator = self._coordinator - coordinator._record_block_submitter_phase("finalize-preview") - coordinator._clear_accepted_block_payout_preview( - block_hash, - invalidate_published=not accepted, - ) - finish_name = ( - "mark_block_candidate_submitted" - if accepted - else "mark_block_candidate_abandoned" - ) - finish = getattr(coordinator.ledger, finish_name, None) - if callable(finish): - try: - if accepted: - coordinator._run_block_submitter_ledger_call( - ("finalize", block_hash, "submitted"), - "finalize-outbox-submitted", - lambda: finish(block_hash=block_hash), - ) - else: - coordinator._run_block_submitter_ledger_call( - ("finalize", block_hash, "abandoned"), - "finalize-outbox-abandoned", - lambda: finish(block_hash=block_hash, error=error), - ) - coordinator._record_committed_block_candidate_abandonment( - block_hash, - outcome, - ) - # The invalidation tombstone is needed until the durable - # outbox becomes terminal. A normal return (including an - # already-terminal/missing row) means there is no pending - # replay source left for this process to guard. - coordinator._clear_accepted_block_payout_preview(block_hash) - except Exception: - # Keep the coordinator alive. The terminal-state update - # failed, so the durable row stays pending and its replay - # must pace like any other retained retry. - print( - "prism coordinator: could not finalize durable block candidate " - f"hash={block_hash}", - flush=True, - ) - traceback.print_exc() - with coordinator.lock: - registry = self.finalize_retries - first_failure = block_hash not in registry - registry[block_hash] = (accepted, error) - if not accepted: - # The durable update is ambiguous, but this process has - # now frozen a false finalize-only disposition. It cannot - # return to chain-state evaluation until restart, where - # these process-local counters start fresh. - coordinator._record_committed_block_candidate_abandonment( - block_hash, - outcome, - ) - # The share row already reached its terminal outcome in this - # process; only the outbox mark is pending. Release the - # snapshot anchor floor now (idempotent) -- holding it across - # paced retries would clamp job snapshot anchors and - # under-count already-durable shares in reward windows. - coordinator._finish_pending_share_commit(candidate.pending_share) - coordinator._retain_block_candidate_for_retry(candidate) - if accepted and first_failure: - # The block is active regardless of the outbox update; - # post-accept fleet refresh must wait for neither ledger - # recovery nor the first backoff. Return unpaced so the - # caller refreshes immediately; the paced ladder starts - # from the first finalize-only replay. - outcome.refresh_client = candidate.client - return True - coordinator._wait_for_block_candidate_retry( - coordinator._next_block_candidate_retry_delay(block_hash) - ) - return True - elif not accepted: - # Compatibility ledgers without a durable candidate outbox have - # no restart replay source that could require the tombstone. - coordinator._clear_accepted_block_payout_preview(block_hash) - coordinator._record_committed_block_candidate_abandonment( - block_hash, - outcome, - ) - with coordinator.lock: - self.finalize_retries.pop(block_hash, None) - coordinator._clear_block_candidate_retry_state(block_hash) - coordinator._discard_outstanding_block_candidate(block_hash) - coordinator._record_block_candidate_terminal_outcome( - block_hash, - accepted=accepted, - ) - # Terminal for this process either way: an accepted candidate credited - # its share during the success tail (a no-op release here), and an - # abandoned one can only be credited by restart replay, which stamps a - # fresh PendingShare. Stop holding the snapshot anchor floor. - coordinator._finish_pending_share_commit(candidate.pending_share) - if accepted: - outcome.refresh_client = candidate.client - return True - - # -- retained node acceptance ------------------------------------------ - - def _stash_retained_block_candidate_node_submission( - self, - block_hash: str, - node_submission: _BlockCandidateNodeSubmission | None, - ) -> None: - """Record a definitive node acceptance for in-process retries. - - Recorded at the offer itself (the universal post-offer hook) so - every retention path — writer failures, defer-accounting handoff - failures, the accounting loop — is covered without each site having - to remember. A retryable failure after a successful offer would - otherwise re-offer on retry and read "duplicate", which classifies - against the moved live tip and can only rescue the block through - chain probes that may be unavailable under the same saturation. - The entry is read without consuming and lives until the candidate - reaches a terminal outcome, so repeated retryable failures keep - reusing the same known acceptance. - """ - if not _is_definitive_node_acceptance(node_submission): - # Only a definitive success is safe to reuse: an ambiguous or - # rejected offer must be re-offered so the node can resolve it. - # The predicate is shared with the accounting handoff's lane - # routing so the two readings of "the node has this block" cannot - # drift apart. - return - with self._coordinator.lock: - retained = getattr( - self, - "_block_candidate_retained_node_submissions", - None, - ) - if retained is None: - retained = {} - self._block_candidate_retained_node_submissions = retained - retained[str(block_hash).lower()] = node_submission - stamped = getattr( - self, - "_block_candidate_retained_submission_monotonic", - None, - ) - if stamped is None: - stamped = {} - self._block_candidate_retained_submission_monotonic = stamped - stamped[str(block_hash).lower()] = time.monotonic() - - def _retained_block_candidate_node_submission( - self, - block_hash: str, - ) -> _BlockCandidateNodeSubmission | None: - with self._coordinator.lock: - retained = getattr( - self, - "_block_candidate_retained_node_submissions", - None, - ) - if not retained: - return None - return retained.get(str(block_hash).lower()) - - def _block_candidate_acceptance_retained(self, block_hash: str) -> bool: - """Whether this process holds fresh first-party acceptance evidence. - - The retained stash records only definitive submitblock successes, - so its presence proves qbitd accepted this candidate — evidence of - the same strength as a recent own-hash tip observation, and - available precisely when saturation makes the chain probes answer - "unknown" (the observation registry can be empty after a definitive - ack: blockwait only reports the newest of rapid connects). It ages - on the same window as tip observations: acceptance at offer time - does not prove the block stayed canonical, and an orphaned block - never probes False — it is merely absent — so a candidate whose - probes stay inconclusive past the window must regain - abandonability instead of deferring forever behind a stale ack. - """ - with self._coordinator.lock: - stamped = getattr( - self, - "_block_candidate_retained_submission_monotonic", - None, - ) - if not stamped: - return False - recorded = stamped.get(str(block_hash).lower()) - if recorded is None: - return False - window = float( - getattr( - self._coordinator, - "observed_tip_accept_window_seconds", - DEFAULT_PRISM_OBSERVED_TIP_ACCEPT_WINDOW_SECONDS, - ) - ) - if window <= 0: - return True - return (time.monotonic() - recorded) <= window - - # -- outcome recording ------------------------------------------------- - - def record_deferred( - self, - reason: str, - message: str, - *, - worker: str | None, - ) -> None: - """Record a retryable outcome without counting a terminal abandonment.""" - del worker - outcome = self.outcome - outcome.reason = reason - outcome.error = None - outcome.stale_job_class = None - print( - f"prism coordinator: block candidate deferred reason={reason}: {message}", - flush=True, - ) - - # -- acceptance evidence ----------------------------------------------- - - def _block_candidate_acceptance_recorded(self, block_hash: str) -> bool: - """Return whether this process completed the candidate success tail.""" - self._coordinator._ensure_job_cache_state() - with self._coordinator.lock: - return ( - block_hash.lower() - in self._coordinator._accounted_accepted_block_hashes - ) - - def _register_outstanding_block_candidate(self, block_hash: str) -> None: - """Track a candidate this process may still land, for tip matching.""" - self._coordinator._ensure_job_cache_state() - with self._coordinator.lock: - self._outstanding_block_candidate_hashes.add(block_hash.lower()) - - def _discard_outstanding_block_candidate(self, block_hash: str) -> None: - """Stop matching tip observations once a candidate is terminal.""" - self._coordinator._ensure_job_cache_state() - key = block_hash.lower() - with self._coordinator.lock: - self._outstanding_block_candidate_hashes.discard(key) - self._tip_observed_accepted_block_hashes.pop(key, None) - - def _note_tip_observation_for_candidates(self, tip_hash: str) -> None: - """Register a tip observation that matches an outstanding candidate. - - qbitd only ever reports the pool's own candidate hash as its chain - tip after accepting that block, so the observation itself is - acceptance evidence -- even when the direct submitblock ack was lost - in transport and the accepted success tail has not run (blockwait - typically learns of the tip before, or instead of, the ack). Every - tip-observation channel funnels through here so later disposition - and abandon checks can outlive transient fork views. - """ - key = tip_hash.lower() - self._coordinator._ensure_job_cache_state() - newly_observed = False - with self._coordinator.lock: - if key in self._outstanding_block_candidate_hashes: - newly_observed = ( - key not in self._tip_observed_accepted_block_hashes - ) - self._tip_observed_accepted_block_hashes[key] = time.monotonic() - if newly_observed: - print( - "prism coordinator: chain tip observation matches pool block " - f"candidate hash={key}; acceptance registered pending " - "finalization", - flush=True, - ) - - def _block_candidate_acceptance_observed(self, block_hash: str) -> bool: - """Whether a recent tip observation already proved this candidate landed.""" - self._coordinator._ensure_job_cache_state() - with self._coordinator.lock: - observed = self._tip_observed_accepted_block_hashes.get( - block_hash.lower() - ) - if observed is None: - return False - window = float( - getattr( - self._coordinator, - "observed_tip_accept_window_seconds", - DEFAULT_PRISM_OBSERVED_TIP_ACCEPT_WINDOW_SECONDS, - ) - ) - if window <= 0: - return True - return (time.monotonic() - observed) <= window - - def _block_candidate_chain_probe( - self, - block_hash: str, - *, - expected_height: int | None = None, - ) -> bool | None: - """Fresh chain verdict for a candidate: proven active, proven wrong, or unknown. - - Returns True when the candidate's own hash is the fresh best tip or - an active chain header at its expected height, False when it is - provably active at the wrong height (a corrupt intent, never a tip - race), and None when this instantaneous view proves nothing (the - hash absent during a tip race, or the probe itself failing). - """ - key = block_hash.lower() - # The two probes are independent: a best-tip lookup failure must not - # suppress the active-header check, which subsumes it (the tip block - # itself reports one confirmation) and can prove acceptance alone. - try: - if str(self._coordinator.rpc.call("getbestblockhash")).lower() == key: - self._coordinator._note_tip_observation_for_candidates(key) - return True - except Exception: - print( - "prism coordinator: acceptance re-check best-tip probe " - f"failed hash={key}; trying the active-header probe", - flush=True, - ) - traceback.print_exc() - height: int | None = None - try: - height = self._coordinator.active_block_candidate_height(key) - except Exception: - print( - "prism coordinator: acceptance re-check header probe failed " - f"hash={key}; falling back to tip-observation evidence", - flush=True, - ) - traceback.print_exc() - if height is None: - return None - if expected_height is None or int(height) == int(expected_height): - self._coordinator._note_tip_observation_for_candidates(key) - return True - return False - - def _block_candidate_acceptance_pending( - self, - block_hash: str, - *, - expected_height: int | None = None, - ) -> bool: - """Return whether abandoning this candidate would discard an accepted block. - - A fresh probe wins in both directions: a candidate proven active is - accepted even if its tip observation was missed, and a candidate - proven active at the wrong height stays abandonable. When the probe - cannot prove either way, a recent own-hash tip observation keeps the - candidate deferring instead of terminal: qbitd accepted it once, so - only a durably settled chain view may discard it. - """ - probe = self._coordinator._block_candidate_chain_probe( - block_hash, - expected_height=expected_height, - ) - if probe is not None: - return probe - return self._coordinator._block_candidate_acceptance_observed(block_hash) - - def _count_accept_pending_defer(self) -> None: - with self._coordinator.lock: - self.block_candidate_accept_pending_defer_count = int( - getattr(self, "block_candidate_accept_pending_defer_count", 0) - ) + 1 - - def record_abandoned( - self, - reason: str, - message: str, - *, - block_hash: str, - worker: str | None, - preserve_if_accepted: bool = False, - expected_height: int | None = None, - stale_job_class: str | None = None, - ) -> bool: - """Record a lost/failed block candidate as a BLOCK-path event. - - The share that produced the candidate was acknowledged and, when it met - the share target, credited at submit time; the block losing its race - afterwards does not un-earn it and is NOT a share rejection. It is - counted under a dedicated block-abandonment counter (by reason, so a - benign 'tip moved' race is distinguishable from a real - submitblock-rejected/ledger failure) rather than the share-reject - counters, which stay a true measure of shares refused to miners. - - Every terminal abandonment withdraws its payout-preview transition - before the outcome becomes final. ``preserve_if_accepted`` closes the - moved-tip race: if another attempt completed this hash's accepted - success tail while withdrawal was in flight, the accepted disposition - wins and the caller must finalize the outbox as submitted. Returns - whether that accepted disposition won. - - Independent of that completed-tail record, a candidate whose own - block hash is the fresh best tip, an active chain header at its - expected height, a recent own-hash tip observation, or a fresh - retained definitive submitblock success is an ACCEPTED block whose - finalization is still pending (for example after a lost submitblock - ack, or when saturation makes both probes answer "unknown" at this - instant). Terminal abandonment would discard its payout - accounting and withdraw its landed preview -- fencing payout - publication for work qbitd already accepted -- so such candidates - defer for retry instead; only hashes provably absent from the active - chain (past the observation and retained-acceptance windows) - abandon terminally. The terminal - seal re-reads observation evidence atomically, so callers can order - follow-up durable work (rejecting prepared payout rows) strictly - afterward. Abandonment metrics commit only once that cleanup succeeds - or a false finalize-only disposition is frozen. - """ - coordinator = self._coordinator - if reason in self.retryable_reasons: - coordinator._defer_block_candidate(reason, message, worker=worker) - return False - if ( - stale_job_class is not None - and stale_job_class not in PRISM_STALE_JOB_ABANDON_CLASSES - ): - raise ValueError( - f"unknown stale job abandon class: {stale_job_class}" - ) - outcome = self.outcome - if ( - preserve_if_accepted - and coordinator._block_candidate_acceptance_recorded(block_hash) - ): - # Durable accepted state already equals the prospective view, so - # any transition recreated by the losing attempt is a no-op - # override, not a withdrawal. - coordinator._clear_accepted_block_payout_preview(block_hash) - outcome.reason = None - outcome.error = None - return True - chain_probe = coordinator._block_candidate_chain_probe( - block_hash, - expected_height=expected_height, - ) - if chain_probe is True or ( - chain_probe is None - and ( - coordinator._block_candidate_acceptance_observed(block_hash) - or coordinator._block_candidate_acceptance_retained(block_hash) - ) - ): - self._count_accept_pending_defer() - coordinator._defer_block_candidate( - PRISM_REJECTION_BLOCK_ACCEPT_PENDING, - "candidate is on (or was recently observed or definitively " - "accepted as) the active chain; refusing terminal " - f"abandonment (was {reason}: {message})", - worker=worker, - ) - return False - - # Own the cleanup invariant here rather than relying on every caller to - # remember it. Invalidation can block behind another candidate pass; - # recheck the accepted record afterward before committing abandonment. - # Capture the transition first: if late acceptance evidence forces a - # defer below, its published preview must be restorable. - with coordinator._accepted_block_payout_preview_condition: - withdrawn_transition = coordinator._accepted_block_payout_previews.get( - block_hash.lower() - ) - coordinator._clear_accepted_block_payout_preview( - block_hash, - invalidate_published=True, - ) - # The invalidation above can block long enough for the chain view to - # heal (a buried accepted block is not always re-observed as the tip - # while blockwait only reports the newest of rapid connects), so an - # unknown pre-withdrawal verdict must re-probe before the terminal - # commit. A provably wrong-height verdict is immutable (headers - # cannot change height) and is never re-probed. - late_probe = ( - chain_probe - if chain_probe is False - else coordinator._block_candidate_chain_probe( - block_hash, - expected_height=expected_height, - ) - ) - with coordinator.lock: - accepted_race_won = bool( - preserve_if_accepted - and block_hash.lower() - in coordinator._accounted_accepted_block_hashes - ) - # A blockwait observation can also register during the blocking - # invalidation. The disposition seal must consult that evidence - # atomically or the same blind spot reopens inside this window; - # the probe still wins both directions. Metrics are committed - # later, after prepared-state cleanup can no longer reverse this - # decision. - late_acceptance_observed = bool( - not accepted_race_won - and ( - late_probe is True - or ( - late_probe is not False - and ( - coordinator._block_candidate_acceptance_observed( - block_hash - ) - or coordinator._block_candidate_acceptance_retained( - block_hash - ) - ) - ) - ) - ) - if not accepted_race_won and not late_acceptance_observed: - outcome.reason = reason - outcome.error = message - outcome.stale_job_class = stale_job_class - # Seal the disposition in the same critical section that - # commits it: stop matching tip observations for this hash so - # no acceptance evidence can register between this terminal - # decision and the caller's follow-up durable work (rejecting - # prepared payout rows). Observation registration takes this - # same lock, so exclusion across the gap is total. A crash - # before the durable outbox update replays the candidate, - # which re-registers and re-evaluates from live chain state. - self._outstanding_block_candidate_hashes.discard( - block_hash.lower() - ) - self._tip_observed_accepted_block_hashes.pop( - block_hash.lower(), - None, - ) - if accepted_race_won: - coordinator._clear_accepted_block_payout_preview(block_hash) - outcome.reason = None - outcome.error = None - return True - if late_acceptance_observed: - # Restore the landed barrier the withdrawal just removed (and pop - # its fail-closed tombstone): the candidate is still an accepted - # block pending finalization, so descendant builders must keep - # waiting on its preview -- not fail closed -- until the deferred - # retry's accepted tail republishes it. Without this, retries - # that keep deferring on observation evidence alone would leave - # the tombstone fencing template refreshes, recreating the - # coordination-blocked stall this path exists to prevent. - coordinator._begin_accepted_block_payout_preview( - block_hash, - block_height=expected_height, - ) - if expected_height is not None: - coordinator._mark_accepted_block_payout_landed( - block_hash, - block_height=expected_height, - ) - if ( - withdrawn_transition is not None - and withdrawn_transition.preview is not None - ): - # The withdrawal superseded payout publication and, on a - # lost publication race, left delivery fenced. Republish the - # withdrawn preview now so admission reopens with the defer - # rather than staying coordination-blocked across the - # deferral cycles until the accepted tail republishes it. - try: - with self._accepted_landing_phase( - block_hash, - LANDING_PHASE_PREVIEW_PREPARE, - block_height=expected_height, - ): - materialized = coordinator._materialize_prior_balance_preview( - withdrawn_transition.preview - ) - with self._accepted_landing_publish_span( - block_hash, - block_height=expected_height, - ): - coordinator._publish_accepted_block_payout_preview( - block_hash, - materialized, - ) - except Exception: - print( - "prism coordinator: could not republish withdrawn " - f"payout preview hash={block_hash}; the scheduled " - "refresh will retry publication", - flush=True, - ) - traceback.print_exc() - self._count_accept_pending_defer() - coordinator._defer_block_candidate( - PRISM_REJECTION_BLOCK_ACCEPT_PENDING, - "acceptance evidence arrived during payout-preview " - f"withdrawal; refusing terminal abandonment (was {reason}: " - f"{message})", - worker=worker, - ) - outcome.error = None - return False - - print( - f"prism coordinator: block candidate abandoned reason={reason}: {message}", - flush=True, - ) - return False - - # -- block-work liveness ----------------------------------------------- - - def _block_work_heartbeat_owner(self) -> tuple[str, str] | None: - """Return the independent heartbeat/phase slots owned by this thread.""" - current = threading.get_ident() - if current == getattr(self, "_block_submitter_thread_ident", None): - return "block_submitter", "_block_submitter_phase" - if current == getattr(self, "_block_accounting_thread_ident", None): - return "block_accounting", "_block_accounting_phase" - return None - - def _record_block_work_heartbeat(self, name: str, phase: str) -> None: - """Record a phase while preserving one-argument heartbeat embedders.""" - heartbeat = self._coordinator._record_heartbeat - try: - heartbeat(name, phase=phase) - except TypeError as exc: - # Preserve the historical one-argument heartbeat seam used by - # focused embedders. Do not hide TypeErrors raised by a heartbeat - # implementation that did accept the keyword. - if "unexpected keyword argument 'phase'" not in str(exc): - raise - heartbeat(name) - - def _record_block_submitter_phase(self, phase: str) -> None: - """Stamp a named phase only from a dedicated block-work owner.""" - owner = self._block_work_heartbeat_owner() - if owner is None: - return - heartbeat_name, phase_attribute = owner - setattr(self, phase_attribute, phase) - self._record_block_work_heartbeat(heartbeat_name, phase) - - def _record_block_submitter_wait(self, phase: str) -> None: - """Heartbeat owner waits while preserving lightweight test behavior.""" - owner = self._block_work_heartbeat_owner() - if owner is None and not hasattr(self, "_block_submitter_thread_ident"): - self._coordinator._record_heartbeat("block_submitter") - return - self._coordinator._record_block_submitter_phase(phase) - - def _block_work_wait_slice(self) -> float: - """Choose a polling slice that stays inside the configured watchdog.""" - watchdog_budget = max( - 0.001, - float(getattr(self._coordinator, "watchdog_timeout_seconds", 120.0)), - ) - return min( - BLOCK_SUBMITTER_WAIT_HEARTBEAT_SLICE_SECONDS, - max(0.001, watchdog_budget * 0.9), - ) - - def _observe_coordinator_lock_wait(self, elapsed_seconds: float) -> None: - """Keep a sliced coordinator-lock wait visible and watchdog-safe.""" - owner = self._block_work_heartbeat_owner() - if owner is None: - return - heartbeat_name, phase_attribute = owner - current_phase = getattr(self, phase_attribute, "unknown") - wait_phase = f"wait-lock:coordinator-state:{current_phase}" - self._record_block_work_heartbeat(heartbeat_name, wait_phase) - now = time.monotonic() - log_interval = float( - getattr( - self._coordinator, - "block_submit_lock_wait_log_seconds", - DEFAULT_BLOCK_SUBMIT_LOCK_WAIT_LOG_SECONDS, - ) - ) - last_log = float( - getattr(self, "_block_submitter_last_lock_wait_log_monotonic", 0.0) - ) - if last_log <= 0 or now - last_log >= log_interval: - self._block_submitter_last_lock_wait_log_monotonic = now - print( - "prism coordinator: block submitter waiting on lock " - f"lock=coordinator-state phase={current_phase} " - f"elapsed={elapsed_seconds:.3f}s", - flush=True, - ) - - def _acquire_block_submitter_lock(self, lock: Any, name: str) -> None: - """Acquire a submit-path lock in heartbeat/logging slices.""" - owner = self._block_work_heartbeat_owner() - if owner is None and ( - hasattr(self, "_block_submitter_thread_ident") - or hasattr(self, "_block_accounting_thread_ident") - ): - lock.acquire() - return - started = time.monotonic() - last_log = started - log_interval = float( - getattr( - self._coordinator, - "block_submit_lock_wait_log_seconds", - DEFAULT_BLOCK_SUBMIT_LOCK_WAIT_LOG_SECONDS, - ) - ) - while not lock.acquire(timeout=self._block_work_wait_slice()): - phase = f"wait-lock:{name}" - self._coordinator._record_block_submitter_wait(phase) - now = time.monotonic() - if now - last_log >= log_interval: - print( - "prism coordinator: block submitter waiting on lock " - f"lock={name} elapsed={now - started:.3f}s", - flush=True, - ) - last_log = now - - @contextmanager - def _block_submitter_lock(self, lock: Any, name: str) -> Iterator[None]: - self._coordinator._acquire_block_submitter_lock(lock, name) - try: - yield - finally: - lock.release() - - def _record_block_candidate_progress( - self, - phase: str = "accounting-progress", - ) -> None: - """Stamp the submitter heartbeat at a candidate-disposition boundary. - - Stamps come only from the dedicated submitter thread: dispositions - also run on client connection threads (synchronous below-target - solves), and a stamp from those threads would refresh the - ``block_submitter`` budget while the dedicated thread might be - wedged elsewhere. A disposition crosses several ledger and - filesystem writes that slow down together under database pressure; - stamping each completed phase on the owner thread keeps a - progressing disposition inside the liveness budget while a wedged - phase still leaves the watchdog able to recover the process -- the - same shape as the CTV broadcaster's per-row stamping, whose name - likewise maps to a single thread. - """ - self._coordinator._record_block_submitter_phase(phase) - - def _observe_block_submit_seconds(self, elapsed_seconds: float) -> None: - self._coordinator._ensure_job_cache_state() - with self._block_submit_metrics_lock: - histogram = self.block_submit_seconds_histogram - histogram["count"] = int(histogram["count"]) + 1 - histogram["sum"] = float(histogram["sum"]) + max( - 0.0, elapsed_seconds - ) - buckets = histogram["buckets"] - assert isinstance(buckets, dict) - for bucket in tuple(buckets): - if elapsed_seconds <= bucket: - buckets[bucket] = int(buckets.get(bucket, 0)) + 1 - - def _note_accepted_block_preview_acceptance(self, block_hash: str) -> None: - """Stamp the moment definitive node acceptance became known. - - Deliberately not ``AcceptedBlockPayoutTransition.landed_monotonic``: - that stamp is armed *before* the RPC (see - ``_unmark_accepted_block_payout_landed``, "the next attempt re-arms - landed before its own RPC"), so an interval measured from it would - include the offer itself and would restart on every retry. This one - is taken on the submitter thread at the instant the offer resolved - definitively, which is the earliest moment acceptance is knowable - without a further chain probe. - """ - key = str(block_hash).lower() - stamped = time.monotonic() - with self._accepted_block_preview_publication_lock: - stamps = self._accepted_block_preview_acceptance_monotonic - if key in stamps: - # Either an unpublished stamp from an earlier definitive - # offer of this hash, or its observed tombstone. Both mean - # the interval is already owned; a re-offer must not restart - # a measurement that is running or already recorded. - return - stamps[key] = stamped - overflow = len(stamps) - MAX_ACCEPTED_BLOCK_PREVIEW_PUBLICATION_STAMPS - if overflow > 0: - for stale in list(stamps)[:overflow]: - del stamps[stale] - - def _accepted_landing_attribution_locked( - self, - key: str, - *, - block_height: int | None = None, - ) -> _AcceptedLandingAttribution: - """Get or create one hash's record; the caller holds the leaf lock. - - Bounded exactly as the acceptance stamps are: past the shared cap the - oldest entries are evicted. The record returned here was just inserted - at the newest end, so the trim can never drop the one in use. - """ - records = self._accepted_landing_attributions - record = records.get(key) - if record is None: - record = _AcceptedLandingAttribution() - records[key] = record - overflow = ( - len(records) - MAX_ACCEPTED_BLOCK_PREVIEW_PUBLICATION_STAMPS - ) - if overflow > 0: - for stale in list(records)[:overflow]: - del records[stale] - if block_height is not None and record.block_height is None: - record.block_height = int(block_height) - return record - - def _accepted_landing_interval_open_locked(self, key: str) -> bool: - """Whether this hash's acceptance-to-publication interval is running.""" - stamps = self._accepted_block_preview_acceptance_monotonic - return stamps.get(key) is not None - - def _commit_accepted_landing_phase( - self, - telemetry: Any, - key: str, - phase: str, - seconds: float, - *, - block_height: int | None = None, - ) -> None: - """Record one measured sub-phase of a landing. - - The family observation is unconditional -- every stretch a landing ran - belongs in the phase histogram, and its only label is the fixed phase - name. Only the per-landing diagnostic is conditional, because a stretch - outside an open interval was not part of the latency that interval - measures. - """ - elapsed = max(0.0, float(seconds)) - with self._accepted_block_preview_publication_lock: - if self._accepted_landing_interval_open_locked(key): - record = self._accepted_landing_attribution_locked( - key, - block_height=block_height, - ) - record.phase_seconds[phase] = ( - float(record.phase_seconds.get(phase, 0.0)) + elapsed - ) - # Deliberately outside the leaf lock: the shared telemetry owner has a - # lock of its own, and this one never nests. - telemetry.observe_landing_phase(phase, elapsed) - - @contextmanager - def _accepted_landing_phase( - self, - block_hash: str, - phase: str, - *, - block_height: int | None = None, - ) -> Iterator[None]: - """Time one landing sub-phase; records on every exit, raising or not. - - The phase name is validated and the shared owner resolved before the - guarded body, so everything the ``finally`` below runs is arithmetic - under two leaf locks and can never replace the exception a failing - stretch is raising. - """ - if phase not in PRISM_ACCEPTED_LANDING_PHASES: - raise ValueError(f"unknown accepted landing phase: {phase!r}") - telemetry = ensure_accepted_preview_telemetry(self._coordinator) - key = str(block_hash).lower() - started = time.monotonic() - try: - yield - finally: - self._commit_accepted_landing_phase( - telemetry, - key, - phase, - time.monotonic() - started, - block_height=block_height, - ) - - def _close_accepted_landing_lane_wait( - self, - block_hash: str, - *, - block_height: int | None = None, - ) -> None: - """Close ``lane_wait`` where the lane starts this hash's landing task. - - Once per open interval. A retried landing re-enters the lane, and - measuring again from the same acceptance stamp would charge the retry's - queue time to a stretch the first attempt already owns -- and would - keep growing that stretch for as long as the candidate kept retrying. - """ - key = str(block_hash).lower() - now = time.monotonic() - with self._accepted_block_preview_publication_lock: - started = self._accepted_block_preview_acceptance_monotonic.get(key) - if started is None: - # No definitive acceptance was stamped for this hash in this - # process, or its interval already closed. There is no lane - # wait to measure either way. - return - record = self._accepted_landing_attribution_locked( - key, - block_height=block_height, - ) - if record.lane_wait_recorded: - return - record.lane_wait_recorded = True - elapsed = max(0.0, now - float(started)) - record.phase_seconds[LANDING_PHASE_LANE_WAIT] = elapsed - ensure_accepted_preview_telemetry(self._coordinator).observe_landing_phase( - LANDING_PHASE_LANE_WAIT, - elapsed, - ) - - def _note_accepted_landing_lane_start(self, candidate: Any) -> None: - """Close ``lane_wait`` for the candidate whose landing starts now.""" - block_hash, block_height = _accepted_landing_candidate_identity(candidate) - if block_hash is None: - return - self._close_accepted_landing_lane_wait( - block_hash, - block_height=block_height, - ) - - @contextmanager - def _accepted_landing_publish_span( - self, - block_hash: str, - *, - block_height: int | None = None, - ) -> Iterator[None]: - """Time ``preview_publish`` up to the preview's visibility instant. - - The publication path closes the interval from inside the publish call, - so a plain timer around that call would record its phase after the - diagnostic had already been built from an empty cell. The span is - opened here and closed by whichever comes first: the observer below, at - the visibility instant it already measures, or this context manager's - exit when the call published nothing this hash was waiting for. Never - both -- the start value is the span's token. - - The record is allocated only for an open interval, exactly as every - other stretch allocates one. A startup replay or a chain-proven - candidate publishes without an in-process acceptance stamp, and the - observer returns before it can pop anything; a record opened for one of - those would be reported as in flight forever and would spend a slot the - bound reserves for live landings. The family observation below stays - unconditional either way -- every stretch a landing ran belongs in the - phase histogram. - """ - telemetry = ensure_accepted_preview_telemetry(self._coordinator) - key = str(block_hash).lower() - started = time.monotonic() - with self._accepted_block_preview_publication_lock: - attributed = self._accepted_landing_interval_open_locked(key) - if attributed: - record = self._accepted_landing_attribution_locked( - key, - block_height=block_height, - ) - record.publish_started_monotonic = started - try: - yield - finally: - self._close_accepted_landing_publish_span( - telemetry, - key, - started, - attributed=attributed, - ) - - def _close_accepted_landing_publish_span( - self, - telemetry: Any, - key: str, - started: float, - *, - attributed: bool = True, - ) -> None: - """Close a publish span this caller still owns, if it still owns it. - - ``attributed`` is what the span found when it opened: False means no - interval was running and no record was allocated, so there is nothing - to own and nothing the observer could have taken -- only the family - histogram is owed a sample. - """ - ended = time.monotonic() - elapsed = max(0.0, ended - started) - with self._accepted_block_preview_publication_lock: - if attributed: - record = self._accepted_landing_attributions.get(key) - if record is None or record.publish_started_monotonic != started: - # The observer closed this span at the visibility instant - # and recorded it there, or terminal cleanup discarded the - # record while the publication was in flight. - return - record.publish_started_monotonic = None - if self._accepted_landing_interval_open_locked(key): - record.phase_seconds[LANDING_PHASE_PREVIEW_PUBLISH] = ( - float( - record.phase_seconds.get( - LANDING_PHASE_PREVIEW_PUBLISH, 0.0 - ) - ) - + elapsed - ) - else: - # The interval closed under this publication without the - # observer taking the span: the acceptance stamp aged out - # of the shared cap. Nothing can build a diagnostic from - # this record now, so drop it rather than leave it in - # flight for the bound to evict a live landing over. - self._accepted_landing_attributions.pop(key, None) - telemetry.observe_landing_phase(LANDING_PHASE_PREVIEW_PUBLISH, elapsed) - - def _discard_accepted_landing_attribution(self, block_hash: str) -> None: - """Drop one hash's in-flight phase state on terminal disposition. - - Nothing can close the interval of a hash that will never publish, so - its record has no consumer left. The cap above would evict it - eventually; dropping it here keeps a storm of terminal candidates from - pushing live landings out of the bound. - """ - key = str(block_hash).lower() - with self._accepted_block_preview_publication_lock: - self._accepted_landing_attributions.pop(key, None) - - def accepted_landing_attribution_snapshot(self) -> dict[str, dict[str, float]]: - """Copied in-flight per-hash phase state, oldest first.""" - with self._accepted_block_preview_publication_lock: - return { - key: dict(record.phase_seconds) - for key, record in self._accepted_landing_attributions.items() - } - - def _record_accepted_landing_diagnostic( - self, - key: str, - *, - result: str, - elapsed: float, - record: _AcceptedLandingAttribution | None, - publish_seconds: float | None, - ) -> None: - """Retain and log one closed landing, outside the leaf lock. - - The rescan and ledger totals stay at the contract's neutral values. - This lane can obtain no per-landing causal evidence for either without - reading a process-wide delta, and such a read here would claim whatever - else the process happened to be doing -- a concurrent job build's - rescan, another lane's ledger reads -- as this block's work. The - separately attributed rescan and ledger-read families answer those - questions without the race. - """ - telemetry = ensure_accepted_preview_telemetry(self._coordinator) - if publish_seconds is not None: - telemetry.observe_landing_phase( - LANDING_PHASE_PREVIEW_PUBLISH, - publish_seconds, - ) - try: - diagnostic = AcceptedPreviewPublicationDiagnostic( - block_hash=key, - block_height=None if record is None else record.block_height, - result=result, - acceptance_to_publication_seconds=elapsed, - phase_seconds=( - dict(record.phase_seconds) if record is not None else {} - ), - reconcile_caller=RECONCILE_CALLER_LANDING, - ) - telemetry.record_publication_diagnostic(diagnostic) - _accepted_landing_log( - PRISM_ACCEPTED_LANDING_LOG_EVENT, - **diagnostic.log_fields(), - ) - except Exception: - # Attribution is observability. A record the contract refuses, or - # a log line that cannot be written, must not fail a publication - # that has already made the preview visible to waiting children. - traceback.print_exc() - - def _observe_accepted_block_preview_publication( - self, - block_hash: str, - *, - result: str, - ) -> None: - """Close the acceptance-to-publication interval for one hash. - - Only the first publication of a hash is measured: it is the one that - made the preview visible to children already waiting on the - transition, which is the latency the 5 s child wait budget is spent - against. Later matching republications observe nothing. - - That same first publication owns the landing's attribution (issue - #224): it is the only moment at which the interval, the phases - measured under it, and the publication result are all known and - final, so the diagnostic is built here and nowhere else. - """ - key = str(block_hash).lower() - published = time.monotonic() - with self._accepted_block_preview_publication_lock: - stamps = self._accepted_block_preview_acceptance_monotonic - if key not in stamps: - # No definitive offer of this hash was stamped in this - # process: a replayed candidate confirmed by chain probe, or - # a preview republished for an acceptance this process never - # made. There is no interval to close, and inventing one from - # the publication alone would report a zero. - return - started = stamps[key] - if started is None: - return - histogram = self.accepted_block_preview_publication_seconds_histogram.get( - result - ) - if histogram is None: - # The label set is closed by - # PRISM_ACCEPTED_BLOCK_PREVIEW_PUBLICATION_RESULTS. Drop the - # observation rather than growing series cardinality, and - # leave the stamp for a labelled publication to close. - return - stamps[key] = None - elapsed = max(0.0, published - float(started)) - histogram["count"] = int(histogram["count"]) + 1 - histogram["sum"] = float(histogram["sum"]) + elapsed - buckets = histogram["buckets"] - assert isinstance(buckets, dict) - for bucket in tuple(buckets): - if elapsed <= bucket: - buckets[bucket] = int(buckets.get(bucket, 0)) + 1 - # Pop the landing's attribution under the same lock that - # tombstoned the stamp, so a retry or a later republication of - # this hash can never build a second diagnostic for it. An open - # publish span belongs to this call: the preview is visible to - # waiting children exactly now. - record = self._accepted_landing_attributions.pop(key, None) - publish_seconds: float | None = None - if record is not None and record.publish_started_monotonic is not None: - publish_seconds = max( - 0.0, - published - float(record.publish_started_monotonic), - ) - record.publish_started_monotonic = None - record.phase_seconds[LANDING_PHASE_PREVIEW_PUBLISH] = ( - float( - record.phase_seconds.get(LANDING_PHASE_PREVIEW_PUBLISH, 0.0) - ) - + publish_seconds - ) - # Outside the leaf lock: the telemetry owner locks for itself and the - # lifecycle line is I/O. - self._record_accepted_landing_diagnostic( - key, - result=result, - elapsed=elapsed, - record=record, - publish_seconds=publish_seconds, - ) - - -class BlockCandidateCompatibilityField: - """Route temporary coordinator fields to the B1 service owner.""" - - def __init__(self, name: str, default: Any) -> None: - self.name = name - self.default = default - - def __get__(self, instance: Any, owner: type[Any]) -> Any: - if instance is None: - return self - service = instance.__dict__.get("_block_candidate_service") - if service is None: - value = instance.__dict__.get(self.name, self.default) - if callable(value) and getattr(value, "__candidate_default_factory__", False): - value = value() - instance.__dict__[self.name] = value - return value - return _compat_get(service, self.name) - - def __set__(self, instance: Any, value: Any) -> None: - service = instance.__dict__.get("_block_candidate_service") - if service is None: - instance.__dict__[self.name] = value - return - _compat_set(service, self.name, value) - - -def compatibility_default(factory: Callable[[], Any]) -> Callable[[], Any]: - setattr(factory, "__candidate_default_factory__", True) - return factory - - -class BlockCandidateStateField: - """Route one #113-era B1 coordinator field to the service owner. - - The descriptor keeps the historical attribute name readable and writable - on the coordinator while :class:`BlockCandidateService` owns the single - mutable copy, mirroring the S1/S2/S3/J1/P1/G1 extraction pattern. Fields - the service creates lazily preserve the pre-extraction ``hasattr``/ - ``getattr``-with-default semantics exactly: reading an unset field raises - AttributeError until first write. - """ - - def __init__(self, name: str, attribute: str | None = None) -> None: - self.name = name - self.attribute = attribute or name - - def __get__(self, instance: Any, owner: type[Any] | None = None) -> Any: - if instance is None: - return self - return getattr( - instance._ensure_block_candidate_service(), - self.attribute, - ) - - def __set__(self, instance: Any, value: Any) -> None: - setattr( - instance._ensure_block_candidate_service(), - self.attribute, - value, - ) - - -_COMPATIBILITY_FIELD_MAP = { - "block_candidate_queue": "candidate_queue", - "block_candidates_dropped": "dropped", - "block_candidate_wakeups_coalesced": "wakeups_coalesced", - "block_candidate_retry_count": "retries", - "block_candidate_poisoned_count": "poisoned", - "block_candidate_retry_initial_seconds": "retry_initial_seconds", - "block_candidate_retry_max_seconds": "retry_max_seconds", - "block_candidate_retry_delays": "retry_delays", - "_block_candidate_finalize_retries": "finalize_retries", - "block_candidate_abandoned_counts": "abandoned_counts", - "_retry_block_candidate": "retry_candidate", - "_block_candidate_outcome": "outcome", -} - -# The #113-era coordinator fields routed one-to-one onto the service. The -# handful whose service spelling differs (the pre-#113 backoff quartet) map -# through their historical service names. -_STATE_FIELD_MAP = { - "_block_replay_candidate_queue": "_block_replay_candidate_queue", - "_block_replay_inflight_hashes": "_block_replay_inflight_hashes", - "_block_quarantine_queue": "_block_quarantine_queue", - "_block_quarantine_hashes": "_block_quarantine_hashes", - "_block_candidate_disposition_registry_lock": "_block_candidate_disposition_registry_lock", - "_block_candidate_disposition_flights": "_block_candidate_disposition_flights", - "_block_candidate_terminal_outcomes": "_block_candidate_terminal_outcomes", - "_block_fast_lane_reservations": "_block_fast_lane_reservations", - "_block_disposition_waiting_retries": "_block_disposition_waiting_retries", - "_block_candidate_dequeued_hashes": "_block_candidate_dequeued_hashes", - "_block_accounting_state_lock": "_block_accounting_state_lock", - "_block_accounting_queue": "_block_accounting_queue", - "_block_accounting_overflow_queue": "_block_accounting_overflow_queue", - "_block_accounting_accepted_queue": "_block_accounting_accepted_queue", - "_block_accounting_sequence": "_block_accounting_sequence", - "_block_accounting_thread": "_block_accounting_thread", - "_block_accounting_thread_ident": "_block_accounting_thread_ident", - "_block_accounting_holds_disposition": "_block_accounting_holds_disposition", - "_block_accounting_deferred_retry_candidate": "_block_accounting_deferred_retry_candidate", - "_block_accounting_phase": "_block_accounting_phase", - "_block_submitter_thread_ident": "_block_submitter_thread_ident", - "_block_submitter_phase": "_block_submitter_phase", - "_block_submitter_last_lock_wait_log_monotonic": "_block_submitter_last_lock_wait_log_monotonic", - "_block_submitter_retry_state_lock": "_state_lock", - "_block_submitter_backoff_started_monotonic": "_backoff_started_monotonic", - "_block_submitter_backoff_deadline_monotonic": "_backoff_deadline_monotonic", - "_block_submitter_backoff_delay_seconds": "_backoff_delay_seconds", - "_block_submitter_ledger_calls_lock": "_block_submitter_ledger_calls_lock", - "_block_submitter_ledger_calls": "_block_submitter_ledger_calls", - "_block_submitter_ledger_worker_slots": "_block_submitter_ledger_worker_slots", - "_block_submitter_rpc_calls_lock": "_block_submitter_rpc_calls_lock", - "_block_submitter_rpc_calls": "_block_submitter_rpc_calls", - "_block_submitter_rpc_worker_slots": "_block_submitter_rpc_worker_slots", - "_block_landing_timeout_counts": "_block_landing_timeout_counts", - "_block_ledger_call_metrics_lock": "_block_ledger_call_metrics_lock", - "_block_ledger_call_metrics": "_block_ledger_call_metrics", - "_block_candidate_retry_not_before": "_block_candidate_retry_not_before", - "_block_candidate_retained_node_submissions": "_block_candidate_retained_node_submissions", - "_block_candidate_retained_submission_monotonic": "_block_candidate_retained_submission_monotonic", - "_counted_block_candidate_abandonments": "_counted_block_candidate_abandonments", - "_outstanding_block_candidate_hashes": "_outstanding_block_candidate_hashes", - "_tip_observed_accepted_block_hashes": "_tip_observed_accepted_block_hashes", - "block_candidate_accept_pending_defer_count": "block_candidate_accept_pending_defer_count", - "accepted_parent_redrive_attempt_count": "accepted_parent_redrive_attempt_count", - "accepted_parent_redrive_resolved_count": "accepted_parent_redrive_resolved_count", - "accepted_parent_redrive_exhausted_count": "accepted_parent_redrive_exhausted_count", - "stale_job_abandon_counts": "stale_job_abandon_counts", - "_block_submit_metrics_lock": "_block_submit_metrics_lock", - "block_submit_seconds_histogram": "block_submit_seconds_histogram", - "_accepted_block_preview_publication_lock": ( - "_accepted_block_preview_publication_lock" - ), - "accepted_block_preview_publication_seconds_histogram": ( - "accepted_block_preview_publication_seconds_histogram" - ), - "_accepted_block_preview_acceptance_monotonic": ( - "_accepted_block_preview_acceptance_monotonic" - ), - "_block_replay_enumeration_owed_flag": "_block_replay_enumeration_owed_flag", -} - - -def _compat_get(service: BlockCandidateService, name: str) -> Any: - return getattr(service, _COMPATIBILITY_FIELD_MAP[name]) - - -def _compat_set(service: BlockCandidateService, name: str, value: Any) -> None: - setattr(service, _COMPATIBILITY_FIELD_MAP[name], value) diff --git a/lab/prism/block_finalization.py b/lab/prism/block_finalization.py deleted file mode 100644 index b3f46117a..000000000 --- a/lab/prism/block_finalization.py +++ /dev/null @@ -1,2365 +0,0 @@ -"""Measured, replay-safe PRISM accepted-block finalization. - -B3 owner module: correctness after node-offer evidence exists. It owns the -post-offer admission/finalization policy of ``submit_block_candidate`` and -``_submit_block_candidate_serialized``, the replay-window decision that -consumes P1's durable window proof, ``_land_and_confirm_block_candidate``, -the CTV/share-credit tail, ordered PR 75 audit evidence publication, and -accepted-block process accounting. The node offer, same-hash disposition, -accounting handoff, and durable outbox terminalization stay with B1 -(``lab/prism/block_candidates.py``); payout previews, balance serialization, -append/anchor fences, and window proofs stay with P1 -(``lab/prism/payout_state.py``). The service accepts B1's prepared -``node_submission`` and never creates a second transport path — its rare -fenced fallback calls B1's node-offer seam through the runtime. -""" - -from __future__ import annotations - -from contextlib import AbstractContextManager, ExitStack, contextmanager -from dataclasses import dataclass -import os -from pathlib import Path -import sys -import threading -import time -import traceback -from typing import Any, Callable, Iterator, Mapping, Protocol - -from lab.prism import direct_stratum, public_api -from lab.prism.accepted_preview_telemetry import ( - LANDING_PHASE_BALANCE_LOCK_WAIT, - LANDING_PHASE_CHAIN_PROBE, - LANDING_PHASE_PREVIEW_PREPARE, - LANDING_PHASE_PRIOR_BALANCES_CHECK, - LANDING_PHASE_RECONCILE, -) -from lab.prism.audit_artifacts import AuditArtifactStore, AuditPublicationIdentity -from lab.prism.audit_bundle_view import streamed_sha256_json_hex -from lab.prism.block_candidates import ( - PRISM_REJECTION_LEDGER_CONFIRMATION_SUPERSEDED, - PrismBlockCandidate, - _BlockCandidateNodeSubmission, -) -from lab.prism.coordinator_shutdown import ShutdownInProgress -from lab.prism.share_ledger import WriterLeaseRenewalDeferred -from lab.prism.share_submission import ( - PRISM_REJECTION_BACKEND_RPC_UNAVAILABLE, - PRISM_REJECTION_POOL_CLOSED, - PRISM_REJECTION_STALE_JOB, -) - - -PRISM_REJECTION_CANDIDATE_AUDIT_MISMATCH = "candidate-audit-mismatch" -PRISM_REJECTION_SUBMITBLOCK_REJECTED = "submitblock-rejected" -PRISM_REJECTION_BLOCK_STALE = "block-stale" -PRISM_REJECTION_LEDGER_CONFIRMATION_FAILED = "ledger-confirmation-failed" -FINALIZATION_PHASES = ( - "admission", - "land_confirm", - "ctv_credit", - "evidence", - "audit_publish", - "accounting", -) - - -def _declared_anchor_ms(context: Any) -> int | None: - """The payout window anchor this candidate's job declared, if any. - - ``found_block`` records the window the issued job was built from, and its - anchor is what scopes an append-invalidation question to this candidate - rather than to whatever else moved the global epoch. A candidate that - carries no anchor -- a reconstruction whose intent predates the field -- - cannot scope the question at all, and every fence that asks reads ``None`` - as invalidated. - """ - found_block = getattr(context, "found_block", None) - if not isinstance(found_block, dict): - return None - anchor_ms = found_block.get("anchor_job_issued_at_ms") - return None if anchor_ms is None else int(anchor_ms) - - -@dataclass(frozen=True) -class FinalizationAdmission: - """Immutable result of candidate admission and active-chain classification. - - ``current_tip`` records the observed post-probe tip that payout - reconciliation must follow; the fresh-attempt classification against the - stamped parent stays local to admission. ``node_submission`` carries B1's - fast-lane offer evidence into the landing phase unchanged. - """ - - candidate: PrismBlockCandidate - context: Any - submission: direct_stratum.DirectQbitSubmission - worker: str | None - expected_height: int - block_hash: str - parent_hash: str - current_tip: str - already_active: bool - node_submission: _BlockCandidateNodeSubmission - - -@dataclass(frozen=True) -class LandedCandidate: - """Durable landing outputs consumed by the remaining ordered phases. - - Combines the landing's four accounting mappings with PR 75's ordered - audit publication identity and the verifier's identity evidence. - """ - - # For a canonical build this is the bounded artifact view returned by - # the bundle compiler: its small members are parsed, its window-sized - # share arrays stay on disk. The later phases read only header members - # (coinbase manifest, CTV manifest set), and the view is closed once the - # ordered phases finish (issue #255). - final_bundle: Mapping[str, Any] - report: dict[str, Any] - persistence: dict[str, Any] - confirmation: dict[str, Any] - audit_publication_identity: AuditPublicationIdentity - audit_verification_identity: dict[str, Any] - - -@dataclass(frozen=True) -class FinalizationEvidence: - """Evidence body and its normalized publication persistence identity.""" - - evidence: dict[str, Any] - publication_persistence: dict[str, Any] - - -class BlockFinalizationPort(Protocol): - """Explicit coordinator capabilities required by finalization. - - Every runtime attribute and method the service touches is declared here, - with one intentional exception: the ``verify_bundle`` verifier override is - consumed only through ``runtime.__dict__`` as a per-instance escape hatch - and deliberately stays out of the port. - """ - - ledger: Any - lock: Any - rpc: Any - accepted_block_count: int - latest_coinbase_size_bytes: int | None - ledger_writer_public_key_hex: str | None - max_blocks: int - reorg_reconciler_enabled: bool - stop_after_block: bool - _accounted_accepted_block_hashes: set[str] - _block_candidate_outcome: threading.local | None - _block_fast_lane_reservations: set[str] - _job_cache_lock: Any - _payout_append_landing_fence_lock: Any - _payout_balance_mutation_lock: Any - _payout_ledger_append_invalidation_epoch: int - _payout_state_generation: int - _payout_state_source: tuple[int, str | None, str, float] - _template_artifacts: Any - - def _abandon_block_candidate( - self, - reason: str, - message: str, - *, - block_hash: str, - worker: str | None, - preserve_if_accepted: bool = False, - expected_height: int | None = None, - stale_job_class: str | None = None, - ) -> bool: ... - - def _accepted_block_payout_preview_from_bundle( - self, - final_bundle: Mapping[str, Any], - *, - prior_balances: list[dict[str, object]] | None = None, - ) -> list[dict[str, object]]: ... - - def _accepted_block_payout_transition_landed(self, block_hash: str) -> bool: ... - - def _append_epoch_invalidated_declared_anchor( - self, - *, - baseline_epoch: int, - live_epoch: int, - declared_anchor_ms: int | None, - ) -> bool: ... - - def _audit_publication_identity( - self, - *, - block_hash: str, - block_height: int, - confirmation: Mapping[str, Any], - ) -> AuditPublicationIdentity: ... - - def _await_unfenced_appends_predating_anchor(self, anchor_ms: int) -> None: ... - - def _begin_accepted_block_payout_preview( - self, - block_hash: str, - *, - block_height: int | None = None, - ) -> None: ... - - def _block_candidate_acceptance_recorded(self, block_hash: str) -> bool: ... - - def _block_candidate_chain_probe( - self, - block_hash: str, - *, - expected_height: int | None = None, - ) -> bool | None: ... - - def _block_candidate_disposition( - self, - block_hash: str, - ) -> AbstractContextManager[Any]: ... - - def _block_candidate_terminal_outcome(self, block_hash: str) -> bool | None: ... - - def _block_payout_state_publication( - self, - *, - force: bool = False, - supersede_with: tuple[int, str | None, str, float] | None = None, - ) -> None: ... - - def _block_submitter_lock( - self, - lock: Any, - name: str, - ) -> AbstractContextManager[None]: ... - - def _cancel_obsolete_job_builds( - self, - reason: str, - *, - keep_published_snapshot: bool = False, - ) -> None: ... - - def _capture_payout_state_source( - self, - ) -> tuple[int, int, str | None, str, float]: ... - - def _clear_accepted_block_payout_preview( - self, - block_hash: str, - *, - invalidate_published: bool = False, - ) -> None: ... - - def _defer_for_pending_parent_payout_transition( - self, - *, - block_hash: str, - parent_hash: str, - parent_height: int, - worker: str | None, - active_candidate_hash: str | None = None, - active_candidate_height: int | None = None, - ) -> bool: ... - - def _ensure_audit_artifact_store(self) -> AuditArtifactStore: ... - - def _ensure_block_candidate_service(self) -> Any: ... - - def _ensure_job_cache_state(self) -> None: ... - - def _expose_inflight_scan_anchor(self, anchor_ms: int) -> int: ... - - def _land_and_confirm_block_candidate( - self, - candidate: PrismBlockCandidate, - *, - current_tip: str, - already_active: bool, - worker: str | None, - node_submission: _BlockCandidateNodeSubmission, - revalidated_append_epoch: int | None = None, - ) -> tuple[ - dict[str, Any], - dict[str, Any], - dict[str, Any], - dict[str, Any], - AuditPublicationIdentity, - dict[str, Any], - ] | None: ... - - def _mark_accepted_block_payout_landed( - self, - block_hash: str, - *, - block_height: int, - ) -> None: ... - - def _mark_tip_refresh_pending(self, _observation: object) -> int: ... - - def _materialize_prior_balance_preview( - self, - preview: tuple[tuple[str, str, str, int], ...], - ) -> list[dict[str, object]]: ... - - def _node_submission_for_direct_candidate( - self, - candidate: PrismBlockCandidate, - ) -> _BlockCandidateNodeSubmission: ... - - def _note_tip_observation_for_candidates(self, tip_hash: str) -> None: ... - - def _observe_payout_state_seconds( - self, - name: str, - elapsed_seconds: float, - *, - relation: str | None = None, - ) -> None: ... - - def _payout_balance_serializer_released( - self, - ) -> AbstractContextManager[None]: ... - - def _payout_source_requires_publication(self, candidate: Any = None) -> bool: ... - - def _payout_state_publication_fenced(self) -> bool: ... - - def _publish_accepted_block_payout_preview( - self, - block_hash: str, - balances: list[dict[str, object]], - ) -> list[dict[str, object]]: ... - - def _publish_current_payout_state_with_retry_budget( - self, - *, - initial_attempted: bool = False, - ) -> int | None: ... - - def _record_block_candidate_progress( - self, - phase: str = "accounting-progress", - ) -> None: ... - - def _record_block_submitter_phase(self, phase: str) -> None: ... - - def _record_committed_block_candidate_abandonment( - self, - block_hash: str, - outcome: threading.local, - ) -> None: ... - - def _register_outstanding_block_candidate(self, block_hash: str) -> None: ... - - def _replayed_payout_window_reproducible(self, context: Any) -> bool: ... - - def _require_fresh_ledger_lease_for_external_side_effect( - self, - component: str, - ) -> None: ... - - def _retire_inflight_scan_anchor(self, token: int | None) -> None: ... - - def _schedule_current_payout_ledger_artifact_if_missing(self) -> None: ... - - def _schedule_payout_ledger_artifact_preparation( - self, - payout_state_generation: int, - network_difficulty: int, - *, - min_interval_seconds: float | None = None, - bypass_build_interval: bool = False, - ) -> None: ... - - def _schedule_tip_refresh_retry(self) -> None: ... - - def _submit_block_candidate_serialized( - self, - candidate: PrismBlockCandidate, - *, - node_submission: _BlockCandidateNodeSubmission, - ) -> bool: ... - - def _submit_block_candidate_to_node( - self, - candidate: PrismBlockCandidate, - ) -> _BlockCandidateNodeSubmission: ... - - def _unmark_accepted_block_payout_landed(self, block_hash: str) -> None: ... - - def accepted_share_stats(self) -> tuple[int, int]: ... - - def active_block_candidate_height(self, block_hash: str) -> int | None: ... - - def append_accepted_share( - self, - client: Any, - context: Any, - submission: direct_stratum.DirectQbitSubmission, - pending_share: Any, - *, - credit_policy: str | None = None, - candidate_intent: dict[str, Any] | None = None, - ) -> str | None: ... - - def block_candidate_intent( - self, - candidate: PrismBlockCandidate, - ) -> dict[str, Any]: ... - - def build_audit_bundle( - self, - *, - shares: list[dict[str, object]], - found_block: dict[str, object], - prior_balances: list[dict[str, object]], - coinbase_script_sig_suffix_hex: str, - witness_merkle_leaves_hex: list[str] | None = None, - ctv_fee_parent_hash: str | None = None, - canonical_output_path: Path | None = None, - canonical_output_parent_fd: int | None = None, - canonical_output_adopter: Callable[[Path, os.stat_result], None] | None = None, - ) -> Mapping[str, Any]: ... - - def coinbase_script_sig_suffix_hex( - self, - extranonce1_hex: str, - extranonce2_hex: str, - ) -> str: ... - - def ensure_reorg_reconciled_for_tip( - self, - tip_hash: str, - *, - _coalesce_same_tip: bool = True, - ) -> bool: ... - - def normalized_prior_balances( - self, - balances: list[dict[str, object]], - ) -> list[dict[str, object]]: ... - - def prior_balances_match_current( - self, - prior_balances: list[dict[str, object]], - ) -> bool: ... - - def reconcile_prism_pool_blocks_once( - self, - *, - tip_hash: str | None = None, - _force_publish: bool = False, - _source_reserved: bool = False, - _wait_for_same_tip_flight: bool = True, - ) -> dict[str, object]: ... - - def reject_prepared_block( - self, - *, - block_hash: str, - active_tip_height: int, - ) -> dict[str, int | str]: ... - - def request_shutdown(self, signum: int | None = None) -> None: ... - - def settlement_balances_by_program( - self, - balances: list[dict[str, object]], - ) -> dict[str, int]: ... - - def trusted_ledger_writer_public_key_hex( - self, - bundle: Mapping[str, Any], - ) -> str: ... - - -class BlockFinalizationService: - """Own accepted-block finalization over explicit infrastructure ports. - - The service deliberately receives the whole coordinator as its runtime: - the finalization tail touches many coordinator-owned seams that focused - tests monkeypatch per instance, so every runtime capability is reached - explicitly through ``self.runtime`` against the typed - ``BlockFinalizationPort`` while the bounded phase metrics stay - service-owned. - """ - - runtime: BlockFinalizationPort - - def __init__(self, runtime: BlockFinalizationPort) -> None: - self.runtime = runtime - self._metrics_lock = threading.Lock() - self._phase_metrics = { - phase: {"count": 0, "sum": 0.0, "max": 0.0} - for phase in FINALIZATION_PHASES - } - self._last_candidate_started: float | None = None - self._candidate_intervals: dict[str, int | float | None] = { - "count": 0, - "sum": 0.0, - "min": None, - } - - @contextmanager - def _phase(self, name: str) -> Iterator[None]: - started = time.monotonic() - try: - yield - finally: - elapsed = max(0.0, time.monotonic() - started) - with self._metrics_lock: - metric = self._phase_metrics[name] - metric["count"] = int(metric["count"]) + 1 - metric["sum"] = float(metric["sum"]) + elapsed - metric["max"] = max(float(metric["max"]), elapsed) - - def _record_block_candidate_progress( - self, - phase: str = "accounting-progress", - ) -> None: - # Service-level compatibility seam: progress marks stay dispatched - # through the service so focused tests can intercept them with a - # subclass override; the base implementation forwards to the port. - self.runtime._record_block_candidate_progress(phase) - - def _accepted_landing_owner(self) -> Any | None: - """The B1 owner of per-landing phase attribution (issue #224). - - The landing's sub-phases are measured here, but the interval they - belong to is opened by B1 at definitive node acceptance and closed by - B1 at the first publication, so the per-hash state lives beside B1's - acceptance stamps rather than in this service. A runtime without that - seam -- the focused fakes that drive this service directly -- records - nothing and behaves exactly as before. - """ - ensure = getattr(self.runtime, "_ensure_block_candidate_service", None) - if not callable(ensure): - return None - return ensure() - - @contextmanager - def _landing_phase( - self, - phase: str, - *, - block_hash: str | None, - block_height: int | None = None, - ) -> Iterator[None]: - """Time one accepted-landing sub-phase through the B1 owner. - - The owner is resolved before the guarded body, so the recording it - runs on the way out is arithmetic under leaf locks and can never - replace the exception a failing stretch is raising. A call with no - block hash -- the disposition-path reread that is not landing a - candidate -- times nothing. - """ - owner = None if block_hash is None else self._accepted_landing_owner() - if owner is None: - yield - return - with owner._accepted_landing_phase( - block_hash, - phase, - block_height=block_height, - ): - yield - - @contextmanager - def _landing_publish_span( - self, - *, - block_hash: str, - block_height: int | None = None, - ) -> Iterator[None]: - """Time ``preview_publish`` up to the preview's visibility instant. - - The publication closes the interval from inside the publish call, so - B1 owns both ends of this span: it is opened here and closed by - whichever comes first, the publication observer or this exit. - """ - owner = self._accepted_landing_owner() - if owner is None: - yield - return - with owner._accepted_landing_publish_span( - block_hash, - block_height=block_height, - ): - yield - - def _stamped_prior_balances_match_current( - self, - prior_balances: list[dict[str, object]], - *, - block_hash: str | None = None, - block_height: int | None = None, - ) -> bool: - """Compare the job's payout base against the ledger, bracketed. - - The comparison is a ledger read run inline on the landing's - block-work thread, between the reorg-reconcile stretch and the - audit build. Bracketing it keeps that statement attributable: an - owner parked in ``prior-balances-check`` is waiting on a known - database read rather than sitting in an unnamed silence the - watchdog cannot tell from a wedge. Both landing call sites route - through here so the pair of stamps cannot drift apart. The stampers - no-op off the block-work owner thread, so client-thread - dispositions are unaffected. - """ - self.runtime._record_block_submitter_phase("prior-balances-check") - with self._landing_phase( - LANDING_PHASE_PRIOR_BALANCES_CHECK, - block_hash=block_hash, - block_height=block_height, - ): - matched = bool(self.runtime.prior_balances_match_current(prior_balances)) - self.runtime._record_block_submitter_phase("prior-balances-check:complete") - return matched - - def _note_candidate_started(self) -> None: - now = time.monotonic() - with self._metrics_lock: - previous = self._last_candidate_started - self._last_candidate_started = now - if previous is None: - return - interval = max(0.0, now - previous) - metric = self._candidate_intervals - metric["count"] = int(metric["count"]) + 1 - metric["sum"] = float(metric["sum"]) + interval - current_min = metric["min"] - metric["min"] = ( - interval if current_min is None else min(float(current_min), interval) - ) - - def metrics_snapshot(self) -> dict[str, Any]: - with self._metrics_lock: - return { - "phases": { - name: dict(value) for name, value in self._phase_metrics.items() - }, - "candidate_intervals": dict(self._candidate_intervals), - } - - def metrics_lines(self) -> list[str]: - snapshot = self.metrics_snapshot() - phases = snapshot["phases"] - intervals = snapshot["candidate_intervals"] - lines = [ - "# HELP qbit_prism_block_finalization_phase_seconds Accepted-block finalization wall time by ordered phase.", - "# TYPE qbit_prism_block_finalization_phase_seconds summary", - ] - for phase in FINALIZATION_PHASES: - metric = phases[phase] - lines.extend( - [ - f'qbit_prism_block_finalization_phase_seconds_sum{{phase="{phase}"}} {float(metric["sum"]):.6f}', - f'qbit_prism_block_finalization_phase_seconds_count{{phase="{phase}"}} {int(metric["count"])}', - f'qbit_prism_block_finalization_phase_seconds_max{{phase="{phase}"}} {float(metric["max"]):.6f}', - ] - ) - interval_count = int(intervals["count"]) - interval_sum = float(intervals["sum"]) - interval_min = intervals["min"] - lines.extend( - [ - "# HELP qbit_prism_block_candidate_interarrival_seconds Time between finalization starts.", - "# TYPE qbit_prism_block_candidate_interarrival_seconds summary", - f"qbit_prism_block_candidate_interarrival_seconds_sum {interval_sum:.6f}", - f"qbit_prism_block_candidate_interarrival_seconds_count {interval_count}", - "qbit_prism_block_candidate_interarrival_seconds_min " - + ("0.000000" if interval_min is None else f"{float(interval_min):.6f}"), - ] - ) - return lines - - def _land_and_confirm_block_candidate( - self, - candidate: PrismBlockCandidate, - *, - current_tip: str, - already_active: bool, - worker: str | None, - node_submission: _BlockCandidateNodeSubmission, - revalidated_append_epoch: int | None = None, - ) -> tuple[ - Mapping[str, Any], - dict[str, Any], - dict[str, Any], - dict[str, Any], - AuditPublicationIdentity, - dict[str, Any], - ] | None: - """Land, verify, publish, persist, and confirm one candidate. - - The balance serializer spans the last prior-state check through durable - confirmation. Reconciliation therefore cannot change the base beneath - the accepted coinbase, while ordinary job delivery remains unblocked. - The caller has already run submitblock on the lock/DB-free fast lane. - The audit bundle build and verification execute with the serializer - temporarily released (the landed fence stays armed), so neither block - announcement nor job delivery waits on audit construction. - """ - context = candidate.context - submission = candidate.submission - expected_height = int(context.template["height"]) - block_hash = str(submission.block_hash_hex).lower() - parent_hash = str(context.template["previousblockhash"]) - self.runtime._ensure_job_cache_state() - durable_payout_state = bool( - getattr(self.runtime.ledger, "durable_payout_state", False) - ) - with ExitStack() as balance_serializer: - # The wait for the balance serializer is a landing stretch of its - # own (issue #224). The stack holds the lock for exactly the block - # the plain ``with`` held it for, so only the acquisition is timed - # and the release point is unchanged. - with self._landing_phase( - LANDING_PHASE_BALANCE_LOCK_WAIT, - block_hash=block_hash, - block_height=expected_height, - ): - balance_serializer.enter_context( - self.runtime._block_submitter_lock( - self.runtime._payout_balance_mutation_lock, - "payout-balance-mutation", - ) - ) - if self.runtime._defer_for_pending_parent_payout_transition( - block_hash=block_hash, - parent_hash=parent_hash, - parent_height=expected_height - 1, - worker=worker, - active_candidate_hash=block_hash if already_active else None, - active_candidate_height=expected_height if already_active else None, - ): - return None - block_state: dict[str, object] | None = None - block_state_reader = getattr(self.runtime.ledger, "pool_block_state", None) - transition_already_landed = self.runtime._accepted_block_payout_transition_landed( - block_hash - ) - reorg_reconciled: bool | None = None - if already_active and not transition_already_landed: - # A replayed active ancestor may coexist with balances from an - # orphaned pool block. Reconcile that global state before this - # transition becomes a landed barrier and before validating its - # payout base. - try: - # Reconciliation is the longest heartbeat-silent stretch - # in the landing scope: a chain walk of ledger statements - # and getblockhash calls, one pair per watched block. The - # pass stamps its own per-row progress internally; these - # boundary stamps mark the stretch itself so a landing - # that enters it is never mistaken for a wedged thread. - self.runtime._record_block_submitter_phase("reorg-reconcile") - with self._landing_phase( - LANDING_PHASE_RECONCILE, - block_hash=block_hash, - block_height=expected_height, - ): - reorg_reconciled = ( - self.runtime.ensure_reorg_reconciled_for_tip( - current_tip, - _coalesce_same_tip=False, - ) - ) - self.runtime._record_block_submitter_phase( - "reorg-reconcile:complete" - ) - except Exception: - traceback.print_exc() - self.runtime._abandon_block_candidate( - PRISM_REJECTION_BACKEND_RPC_UNAVAILABLE, - "reorg reconciliation failed before block replay", - block_hash=block_hash, - worker=worker, - ) - return None - if not reorg_reconciled: - self.runtime._abandon_block_candidate( - PRISM_REJECTION_BACKEND_RPC_UNAVAILABLE, - "reorg reconciliation reported an untrusted chain view", - block_hash=block_hash, - worker=worker, - ) - return None - if already_active and callable(block_state_reader): - self.runtime._record_block_submitter_phase("pool-block-state") - with self._landing_phase( - LANDING_PHASE_CHAIN_PROBE, - block_hash=block_hash, - block_height=expected_height, - ): - block_state = block_state_reader(block_hash=block_hash) - self.runtime._record_block_submitter_phase("pool-block-state:complete") - already_confirmed = bool( - block_state is not None - and str(block_state.get("chain_state", "")) == "confirmed" - and str(block_state.get("maturity_state", "")) != "reversed" - ) - if already_confirmed: - # The outbox terminal update can fail after a fully durable - # confirmation. Do not replace later global balances with an - # ancestor-only preview during exact-idempotent replay. - self.runtime._clear_accepted_block_payout_preview(block_hash) - reorg_reconciled = True - elif already_active: - self.runtime._begin_accepted_block_payout_preview( - block_hash, - block_height=expected_height, - ) - self.runtime._mark_accepted_block_payout_landed( - block_hash, - block_height=expected_height, - ) - reorg_reconciled = True - elif transition_already_landed: - # A prior attempt reached submitblock while holding this - # serializer. External reconciliation is barred until it - # confirms or is withdrawn, so retry its durable steps directly. - reorg_reconciled = True - else: - try: - # Same unbounded chain walk as the replay path above. - self.runtime._record_block_submitter_phase("reorg-reconcile") - with self._landing_phase( - LANDING_PHASE_RECONCILE, - block_hash=block_hash, - block_height=expected_height, - ): - reorg_reconciled = ( - self.runtime.ensure_reorg_reconciled_for_tip( - current_tip, - _coalesce_same_tip=False, - ) - ) - self.runtime._record_block_submitter_phase( - "reorg-reconcile:complete" - ) - except Exception: - traceback.print_exc() - self.runtime._abandon_block_candidate( - PRISM_REJECTION_BACKEND_RPC_UNAVAILABLE, - "reorg reconciliation failed before block submit", - block_hash=block_hash, - worker=worker, - ) - return None - if not reorg_reconciled: - self.runtime._abandon_block_candidate( - PRISM_REJECTION_BACKEND_RPC_UNAVAILABLE, - "reorg reconciliation reported an untrusted chain view", - block_hash=block_hash, - worker=worker, - ) - return None - if ( - already_active - and not already_confirmed - and self.runtime._defer_for_pending_parent_payout_transition( - block_hash=block_hash, - parent_hash=parent_hash, - parent_height=expected_height - 1, - worker=worker, - ) - ): - return None - # A late-visible append advances only the append-invalidation - # epoch: the tip, payout generation, and balance digest all - # survive, yet this candidate's coinbase pays from a window that - # omitted the late row. The refresh wave retires the job - # asynchronously; until it does, membership admission still lets - # the pre-append job submit, so landing fails closed here. - # Collection candidates settle solver-pays-all and are exempt, as - # at every other epoch fence. A negative stamp is a candidate - # reconstructed from durable intent: the caller revalidated its - # recorded window against the durable ledger before this landing - # and rebased it onto the live epoch sequence at that read, so - # from here both kinds of candidate share one epoch fence. A - # reconstructed candidate on a backend that cannot revalidate - # carries no epoch and the fence stands down. - # - # The question is asked per anchor, not per counter. The epoch is - # global, so a bare inequality also fires for an append that only - # invalidated some newer live window -- and on this lane, where - # nothing has been offered, that abandoned a block before qbitd - # ever saw it (issue #126). The predicate forgives a moved epoch - # only when every bump between this candidate's baseline and the - # live value was caused by a row that provably does not predate - # this candidate's own declared anchor, and fails closed - # otherwise -- including when the anchor is unknown, which is - # what the bare comparison did for every candidate. - context_append_epoch = int( - getattr(context, "payout_append_invalidation_epoch", 0) - ) - effective_append_epoch = ( - context_append_epoch - if context_append_epoch >= 0 - else revalidated_append_epoch - ) - declared_anchor_ms = _declared_anchor_ms(context) - with self.runtime._job_cache_lock: - live_append_epoch = int( - self.runtime._payout_ledger_append_invalidation_epoch - ) - if ( - not already_active - and not getattr(context, "collection_only", False) - and not node_submission.attempted - and effective_append_epoch is not None - and self.runtime._append_epoch_invalidated_declared_anchor( - baseline_epoch=effective_append_epoch, - live_epoch=live_append_epoch, - declared_anchor_ms=declared_anchor_ms, - ) - ): - # Fail closed only while nothing has been offered: a - # fast-lane candidate already reached qbitd, so a moved - # epoch can no longer withhold its coinbase — the offer's - # own classification below decides (an error retries - # duplicate-safe, a rejection stays terminal, and an - # accepted or duplicate result proceeds to the post-offer - # fence, which keeps the as-issued snapshot for - # accounting). - self.runtime._abandon_block_candidate( - PRISM_REJECTION_STALE_JOB, - "payout window was invalidated by a late-visible share append", - block_hash=block_hash, - worker=worker, - expected_height=expected_height, - stale_job_class="append_epoch_stale", - ) - return None - if ( - durable_payout_state - and not already_active - and not self._stamped_prior_balances_match_current( - context.prior_balances, - block_hash=block_hash, - block_height=expected_height, - ) - ): - self.runtime._abandon_block_candidate( - PRISM_REJECTION_STALE_JOB, - "prior balances changed since the job was issued", - block_hash=block_hash, - worker=worker, - expected_height=expected_height, - stale_job_class="balance_stale", - ) - return None - if not already_active and not node_submission.attempted: - # One RPC round trip, but it runs between two already-stamped - # stretches on the block-work thread; leaving it unnamed puts - # an unbounded node response inside somebody else's phase. - self.runtime._record_block_submitter_phase("tip-height-rpc") - with self._landing_phase( - LANDING_PHASE_CHAIN_PROBE, - block_hash=block_hash, - block_height=expected_height, - ): - before_height = int(self.runtime.rpc.call("getblockcount")) - self.runtime._record_block_submitter_phase("tip-height-rpc:complete") - if before_height + 1 != expected_height: - self.runtime._abandon_block_candidate( - PRISM_REJECTION_BLOCK_STALE, - f"stale block height: template={expected_height} tip={before_height}", - block_hash=block_hash, - worker=worker, - expected_height=expected_height, - ) - return None - if not already_active: - # Register before a fallback submitblock can expose this hash as the new - # tip. Child builders will wait for the verified preview rather - # than reading balances that omit their new parent. - self.runtime._begin_accepted_block_payout_preview( - block_hash, - block_height=expected_height, - ) - # Treat the submit outcome as uncertain before entering RPC. - # If transport fails after qbitd accepted the block, this - # conservative barrier preserves the coinbase's payout base. - self.runtime._mark_accepted_block_payout_landed( - block_hash, - block_height=expected_height, - ) - fallback_submit_under_fence = not node_submission.attempted - if not node_submission.attempted: - - def _verify_lease_before_submitblock() -> None: - # Runs at the RPC boundary — after any wait on the - # landing fence lock below — so the lease runway the - # verification proves is still intact when the RPC - # starts. A predating append_batch can hold that - # lock across its ledger commit and epoch bump far - # longer than any scheduling headroom, and a - # verification taken before the wait would measure - # runway the wait then consumes. - try: - self.runtime._require_fresh_ledger_lease_for_external_side_effect( - "submitblock" - ) - except WriterLeaseRenewalDeferred: - if not transition_already_landed: - # The refusal fired before submitblock, so - # this attempt's outcome is not uncertain: - # qbitd provably never saw the block (no - # fast-lane node offer either — attempted is - # False). Unwind the landed bar this attempt - # armed — leaving it would bar reconciliation - # and payout-state publication for as long as - # the writer's own fenced write keeps the - # renewal deferred, though nothing needs - # preserving. The begun preview stays and the - # retry re-arms both. A bar landed by an - # earlier attempt that did reach submitblock - # keeps standing for that attempt's uncertain - # outcome. - self.runtime._unmark_accepted_block_payout_landed( - block_hash - ) - raise - - # The epoch fence above is advisory: it releases the lock - # after one read, so an append-side bump could still commit - # between that read and the RPC below. This one is - # authoritative -- the bump acquires the same fence lock, so - # holding it across submitblock means no late-visible append - # can advance the epoch between this comparison and the - # block entering qbitd. The lock spans the lease - # verification and one RPC, and only on this boundary; - # ordinary share commits never touch it (the append side - # takes it only for rows that predate a live anchor, and - # this landing's own declared anchor stays exposed for - # the landing's duration). - append_epoch_raced = False - if ( - effective_append_epoch is None - or getattr(context, "collection_only", False) - ): - _verify_lease_before_submitblock() - node_submission = self.runtime._submit_block_candidate_to_node( - candidate - ) - else: - with self.runtime._payout_append_landing_fence_lock: - with self.runtime._job_cache_lock: - live_append_epoch = int( - self.runtime._payout_ledger_append_invalidation_epoch - ) - # Anchor-scoped, exactly as the advisory read - # above: a bump that commits inside this window - # is terminal only if the row that caused it - # predates this candidate's declared anchor. - # Two guarantees make reading recorded bumps - # sufficient. The fence lock makes the answer - # authoritative: the bump takes the same lock, - # so none can commit between this comparison - # and the RPC below. And every append that - # predates this candidate's window has a bump - # to read: the landing's own anchor is exposed - # for the landing's duration, and the anchor- - # set maximum never decreases while any job is - # landable (an armed artifact's anchor is - # folded into the never-retired published- - # window watermark before the slot can drop - # it), so a predating row always finds an - # anchor to predate and stamps the epoch it - # advances -- never a silent commit this read - # would mistake for harmless history. - if self.runtime._append_epoch_invalidated_declared_anchor( - baseline_epoch=effective_append_epoch, - live_epoch=live_append_epoch, - declared_anchor_ms=declared_anchor_ms, - ): - append_epoch_raced = True - else: - _verify_lease_before_submitblock() - node_submission = ( - self.runtime._submit_block_candidate_to_node(candidate) - ) - if append_epoch_raced: - self.runtime._abandon_block_candidate( - PRISM_REJECTION_STALE_JOB, - "payout window was invalidated by a late-visible share append", - block_hash=block_hash, - worker=worker, - expected_height=expected_height, - stale_job_class="append_epoch_stale", - ) - return None - if node_submission.error is not None: - raise node_submission.error - result = node_submission.result - if result not in (None, "duplicate"): - self.runtime._abandon_block_candidate( - PRISM_REJECTION_SUBMITBLOCK_REJECTED, - f"submitblock rejected candidate: {result}", - block_hash=block_hash, - worker=worker, - expected_height=expected_height, - ) - return None - if ( - not fallback_submit_under_fence - and effective_append_epoch is not None - and not getattr(context, "collection_only", False) - ): - # The fast lane offered this block to qbitd without any - # ledger synchronization, so the landing fence can no - # longer gate submitblock itself. It still orders the - # accounting decision: wait out any fenced predating - # append whose durable commit is in flight (the commit - # holds this lock across its epoch bump). A moved epoch - # is no longer terminal here, though: every candidate - # reaching this point was accepted or already known by - # the node (rejections returned above), so its coinbase - # paid the as-issued window the moment it landed and no - # rebuild can change it. Abandoning would permanently - # discard payout accounting for a live block — the - # ledger would then re-pay the same carried balances in - # a later block — so the as-issued snapshot is kept for - # accounting and the predating share simply rides the - # next window (its ledger credit is untouched). The - # active-height check below still owns the - # did-it-actually-land verdict, and the pre-offer fence - # above still fails closed while nothing has been - # submitted. - with self.runtime._payout_append_landing_fence_lock: - with self.runtime._job_cache_lock: - live_append_epoch = int( - self.runtime._payout_ledger_append_invalidation_epoch - ) - if live_append_epoch != effective_append_epoch: - print( - "prism coordinator: payout window was " - "invalidated by a late-visible share append " - f"after the node offer hash={block_hash}; " - "keeping the as-issued payout snapshot for " - "accounting", - flush=True, - ) - active_hash = str( - self.runtime.rpc.call("getblockhash", [expected_height]) - ).lower() - if active_hash != block_hash: - self.runtime._abandon_block_candidate( - PRISM_REJECTION_SUBMITBLOCK_REJECTED, - f"submitted block is not active at height {expected_height}", - block_hash=block_hash, - worker=worker, - expected_height=expected_height, - ) - return None - self.runtime._cancel_obsolete_job_builds("direct PRISM block accepted") - self.runtime._mark_tip_refresh_pending(block_hash) - self.runtime._schedule_tip_refresh_retry() - - preview: list[dict[str, object]] | None = None - issued_preview = getattr(context, "prospective_prior_balances", None) - if not already_confirmed and issued_preview is not None: - # The compact preview came from the immutable issued job - # summary. Publish it before rebuilding/canonicalizing the full - # audit bundle, without retaining that bundle's shares tree. - with self._landing_phase( - LANDING_PHASE_PREVIEW_PREPARE, - block_hash=block_hash, - block_height=expected_height, - ): - preview = self.runtime._materialize_prior_balance_preview( - issued_preview - ) - if durable_payout_state and not ( - self._stamped_prior_balances_match_current( - context.prior_balances, - block_hash=block_hash, - block_height=expected_height, - ) - ): - self.runtime.request_shutdown() - self.runtime._clear_accepted_block_payout_preview( - block_hash, - invalidate_published=True, - ) - self.runtime._abandon_block_candidate( - PRISM_REJECTION_LEDGER_CONFIRMATION_FAILED, - "accepted block payout base changed before preview publication", - block_hash=block_hash, - worker=worker, - ) - return None - with self._landing_publish_span( - block_hash=block_hash, - block_height=expected_height, - ): - self.runtime._publish_accepted_block_payout_preview( - block_hash, - preview, - ) - - self.runtime._record_block_submitter_phase("audit-build") - # The bundle derives only from inputs frozen on the candidate - # (share window, prior balances, extranonces, template fields), - # so the serializer is released around the builder/verifier - # subprocess work: submitblock above has already run for fresh - # candidates -- announcement is never delayed by audit - # construction -- and the landed fence keeps reconciliation out - # while job delivery proceeds. Child builds consume the compact - # preview published above until the verified preview lands. - with self.runtime._payout_balance_serializer_released(): - audit_store = self.runtime._ensure_audit_artifact_store() - candidate_artifact = audit_store.issue_candidate( - block_hash=submission.block_hash_hex - ) - candidate_bundle_path = candidate_artifact.path - compiler_transferred_candidate = False - - def adopt_compiler_output(path: Path, value: os.stat_result) -> None: - nonlocal compiler_transferred_candidate - audit_store.adopt_compiler_candidate( - candidate_artifact, - path=path, - value=value, - ) - compiler_transferred_candidate = True - - compiler_parent_fd = audit_store.duplicate_root_directory_fd() - try: - final_bundle = self.runtime.build_audit_bundle( - shares=context.shares_json, - found_block=context.found_block, - prior_balances=context.prior_balances, - coinbase_script_sig_suffix_hex=self.runtime.coinbase_script_sig_suffix_hex( - candidate.extranonce1_hex, - candidate.extranonce2_hex, - ), - witness_merkle_leaves_hex=list( - getattr(context.job, "witness_merkle_leaves_hex", ()) - ) - or direct_stratum.witness_merkle_leaves_hex( - getattr(context.job, "transaction_hexes", ()) - ), - ctv_fee_parent_hash=parent_hash, - canonical_output_path=candidate_bundle_path, - canonical_output_parent_fd=compiler_parent_fd, - canonical_output_adopter=adopt_compiler_output, - ) - except BaseException: - audit_store.discard_candidate(candidate_artifact) - raise - finally: - os.close(compiler_parent_fd) - # A real build returns the bundle compiler's bounded view of - # the canonical artifact it just wrote: header members are - # parsed, the window-sized share arrays remain on disk, and - # every consumer below (coinbase check, trusted key, payout - # preview, persistence) reads it as a mapping (#255). - # Compatibility builders used by tests and older integrations - # may ignore canonical_output_path. Persist their logical - # bundle via the normal canonicalization fallback without - # mislabeling bytes. - try: - if not candidate_bundle_path.exists(): - candidate_bundle_path = audit_store.write_compatibility_candidate( - candidate_artifact, - final_bundle, - ) - else: - if not compiler_transferred_candidate: - raise RuntimeError( - "audit builder created an output path without exact inode transfer" - ) - final_manifest = final_bundle["signed_coinbase_manifest"]["manifest"] - final_coinbase_tx_hex_raw = final_manifest["coinbase_tx_hex"] - if not isinstance(final_coinbase_tx_hex_raw, str): - raise ValueError( - "final audit bundle coinbase_tx_hex is not a string" - ) - final_coinbase_tx_hex = final_coinbase_tx_hex_raw.lower() - except BaseException: - audit_store.discard_candidate(candidate_artifact) - raise - if final_coinbase_tx_hex != submission.coinbase_tx_hex.lower(): - audit_store.discard_candidate(candidate_artifact) - self.runtime.request_shutdown() - self.runtime._clear_accepted_block_payout_preview( - block_hash, - invalidate_published=True, - ) - self.runtime._abandon_block_candidate( - PRISM_REJECTION_CANDIDATE_AUDIT_MISMATCH, - "final audit bundle coinbase does not match submitted coinbase", - block_hash=block_hash, - worker=worker, - ) - return None - payout_commit_started: float | None = None - payout_commit_source: int | None = None - try: - with self.runtime._payout_balance_serializer_released(): - self.runtime._record_block_submitter_phase("audit-verify") - # Intentional escape hatch outside the typed port: only a - # per-instance override placed in the runtime's __dict__ - # replaces the verifier, never a class-level attribute. - verifier_override = self.runtime.__dict__.get("verify_bundle") - configured_writer_key = getattr( - self.runtime, - "ledger_writer_public_key_hex", - None, - ) - verified_audit = audit_store.verify_candidate( - candidate_artifact, - coinbase_tx_hex=submission.coinbase_tx_hex, - expected_coinbase_value_sats=int( - context.template["coinbasevalue"] - ), - expected_block_height=expected_height, - trusted_writer_public_key_hex=( - self.runtime.trusted_ledger_writer_public_key_hex(final_bundle) - ), - trust_source=( - "configured" - if configured_writer_key is not None - else "embedded_test_only" - ), - verifier=( - verifier_override - if callable(verifier_override) - else None - ), - ) - audit_store.require_current_verified_candidate( - verified_audit, - candidate_artifact, - ) - report = dict(verified_audit.report) - persistence_canonical_bundle_path = ( - candidate_bundle_path - if verified_audit.canonical_copy_eligible - else None - ) - with self._landing_phase( - LANDING_PHASE_PREVIEW_PREPARE, - block_hash=block_hash, - block_height=expected_height, - ): - # The preview walks the bundle's accounts row by - # row; on a bounded view they stream from disk. - verified_preview = ( - self.runtime._accepted_block_payout_preview_from_bundle( - final_bundle, - prior_balances=context.prior_balances, - ) - ) - self.runtime._record_block_submitter_phase("audit-verify:complete") - if not already_confirmed: - if preview is None and durable_payout_state: - # A prior-balances reread, never a payout-window - # rescan: the ledger read below re-observes the base - # this candidate's coinbase pays from. - with self._landing_phase( - LANDING_PHASE_PRIOR_BALANCES_CHECK, - block_hash=block_hash, - block_height=expected_height, - ): - live_prior_balances = ( - self.runtime.settlement_balances_by_program( - self.runtime.ledger.current_prior_balances() - ) - ) - expected_prior_balances = ( - self.runtime.settlement_balances_by_program( - context.prior_balances - ) - ) - if live_prior_balances != expected_prior_balances: - self.runtime.request_shutdown() - self.runtime._clear_accepted_block_payout_preview( - block_hash, - invalidate_published=True, - ) - self.runtime._abandon_block_candidate( - PRISM_REJECTION_LEDGER_CONFIRMATION_FAILED, - "accepted block payout base changed before preview publication", - block_hash=block_hash, - worker=worker, - ) - return None - try: - with self._landing_publish_span( - block_hash=block_hash, - block_height=expected_height, - ): - self.runtime._publish_accepted_block_payout_preview( - block_hash, - verified_preview, - ) - except RuntimeError as exc: - self.runtime.request_shutdown() - self.runtime._clear_accepted_block_payout_preview( - block_hash, - invalidate_published=True, - ) - self.runtime._abandon_block_candidate( - PRISM_REJECTION_CANDIDATE_AUDIT_MISMATCH, - "verified final payout preview does not match the " - f"issued block job: {exc}", - block_hash=block_hash, - worker=worker, - ) - return None - preview = verified_preview - - # The verified preview is now the effective balance snapshot, - # so persistence can do canonicalization, body writes, copies, - # and bulk SQL without owning the delivery gate. - payout_commit_started = time.monotonic() - payout_commit_source = self.runtime._capture_payout_state_source()[1] - self.runtime._record_block_submitter_phase("persist-accepted-block") - persistence = self.runtime.ledger.persist_accepted_block( - block_hash=submission.block_hash_hex, - block_height=expected_height, - parent_hash=parent_hash, - final_bundle=final_bundle, - audit_report=report, - canonical_bundle_path=persistence_canonical_bundle_path, - ) - self.runtime._record_block_submitter_phase("persist-accepted-block:complete") - active_hash = str( - self.runtime.rpc.call("getblockhash", [expected_height]) - ).lower() - if active_hash != block_hash: - if already_confirmed: - self.runtime._abandon_block_candidate( - PRISM_REJECTION_BACKEND_RPC_UNAVAILABLE, - "accepted ancestor left the active chain during replay", - block_hash=block_hash, - worker=worker, - ) - return None - # Seal the disposition BEFORE touching the prepared - # payout rows: a rejected row can never be promoted by a - # later confirmation (and reactivation only covers - # inactive rows), so rejection must follow -- never - # precede -- a terminal decision. The abandon defers on - # live acceptance evidence; its terminal commit consults - # that evidence atomically with the commit AND seals the - # hash against further observation matching, so no - # evidence can register anywhere in the gap between the - # sealed decision and this rejection. A crash in between - # leaves the outbox row pending; restart replay - # re-registers the hash and re-evaluates from live chain - # state before rejecting. - self.runtime._abandon_block_candidate( - PRISM_REJECTION_BLOCK_STALE, - "accepted block left the active chain before ledger confirmation", - block_hash=block_hash, - worker=worker, - expected_height=expected_height, - ) - outcome = getattr( - self.runtime, "_block_candidate_outcome", None - ) - sealed_reason = ( - getattr(outcome, "reason", None) - if outcome is not None - else None - ) - if sealed_reason == PRISM_REJECTION_BLOCK_STALE: - active_tip_height = int(self.runtime.rpc.call("getblockcount")) - self.runtime.reject_prepared_block( - block_hash=block_hash, - active_tip_height=active_tip_height, - ) - return None - self.runtime._record_block_submitter_phase("confirm-accepted-block") - with audit_store.publication_order_guard(): - confirmation = self.runtime.ledger.confirm_accepted_block( - block_hash=block_hash, - # The ledger confirmation function matches this value - # against the candidate row's own height. An accepted - # ancestor can be finalized after newer blocks arrive. - active_tip_height=expected_height, - ) - confirmed_count = int(confirmation.get("confirmed_count", 0)) - # 1 is the fresh flip and 2 the idempotent replay of a row - # this writer already confirmed. Both leave a confirmed - # row addressed by the ordinal the flip allocated -- the - # replay allocates nothing -- so both carry a publication - # identity and both complete the candidate. - if confirmed_count in {1, 2}: - audit_publication_identity = ( - self.runtime._audit_publication_identity( - block_hash=block_hash, - block_height=expected_height, - confirmation=confirmation, - ) - ) - self.runtime._record_block_submitter_phase( - "confirm-accepted-block:complete" - ) - # 2 is the ledger's idempotent disposition: this writer had - # already confirmed the row, so the call changed nothing. It - # completes the candidate exactly like a fresh flip, and its - # payout publication falls through the covered-replay checks - # below -- a covered confirmation forces no publication of its - # own, because the state it would republish is the state its - # original flip already published (issue #61). Those checks are - # not weakened by it: a source left unpublished or a leaked - # delivery fence still publishes here, exactly as before. - if confirmed_count not in {1, 2}: - # -1 is the ledger's superseded disposition: the row was - # terminally disposed (reorg quarantine, rejection, or - # reversal) before this confirmation landed. That is - # terminal for the candidate but benign for the pool, so - # the coordinator keeps serving; the shutdown escalation - # is reserved for a genuinely unexplained failure. - superseded = confirmed_count == -1 - if not superseded: - self.runtime.request_shutdown() - self.runtime._clear_accepted_block_payout_preview( - block_hash, - invalidate_published=True, - ) - if superseded: - self.runtime._abandon_block_candidate( - PRISM_REJECTION_LEDGER_CONFIRMATION_SUPERSEDED, - "ledger row for accepted block " - f"{block_hash} was superseded before confirmation", - block_hash=block_hash, - worker=worker, - ) - else: - self.runtime._abandon_block_candidate( - PRISM_REJECTION_LEDGER_CONFIRMATION_FAILED, - f"ledger did not confirm accepted block {block_hash}", - block_hash=block_hash, - worker=worker, - ) - return None - - if durable_payout_state: - # A fresh confirmation just made this block the ledger's - # newest confirmed pool block: the landing holds the - # balance serializer and every younger landing defers on - # this block's unresolved payout transition (re-armed by - # startup replay enumeration after a restart), so the - # transactionally maintained current-balance summary IS - # the active-chain view as of this block. Read that - # O(recipients) summary instead of the O(history) carry - # scan, whose growth past the parent-preview wait stalled - # back-to-back landings (issue #209). Only an exact replay - # of an already-confirmed ancestor can run after a later - # pool block changed global balances; that replay still - # compares against its height-bounded active-chain view, - # which also preserves the invariant across restart after - # a prior post-confirm mismatch instead of silently - # accepting the already-confirmed row on the next attempt. - as_of_reader = ( - getattr( - self.runtime.ledger, - "prior_balances_after_pool_block", - None, - ) - if already_confirmed - else None - ) - confirmed_balances = self.runtime.normalized_prior_balances( - as_of_reader(block_hash=block_hash) - if callable(as_of_reader) - else self.runtime.ledger.current_prior_balances() - ) - if self.runtime.settlement_balances_by_program( - confirmed_balances - ) != self.runtime.settlement_balances_by_program(preview): - self.runtime.request_shutdown() - self.runtime._clear_accepted_block_payout_preview( - block_hash, - invalidate_published=True, - ) - self.runtime._abandon_block_candidate( - PRISM_REJECTION_LEDGER_CONFIRMATION_FAILED, - "confirmed payout balances do not match the published " - f"preview for accepted block {block_hash}", - block_hash=block_hash, - worker=worker, - ) - return None - # Durability caught up to the already-published logical state; - # clearing the parent override needs no second generation bump. - self.runtime._clear_accepted_block_payout_preview(block_hash) - self.runtime._schedule_current_payout_ledger_artifact_if_missing() - payout_publication_required = ( - self.runtime._payout_source_requires_publication() - ) - payout_publication_fenced = ( - self.runtime._payout_state_publication_fenced() - ) - # These two are the whole publication decision, for a fresh - # confirmation and a covered replay alike: nothing pending and - # nothing fenced means the published payout state already - # covers this candidate, so republishing it would bump the - # payout generation, wipe the job-bundle cache, and abort every - # in-flight refresh for identical state. Never make the - # disposition alone force a publication past them. - if payout_publication_required or payout_publication_fenced: - # A covered replay normally has no publication work. The - # exception is a leaked delivery fence whose source already - # published: force one republish so the replay heals it. - covered_replay_fence = ( - payout_publication_fenced - and not payout_publication_required - ) - with self.runtime.lock: - pending_cause = self.runtime._payout_state_source[2] - # A bounded preview-publication loss already left the gate - # fenced and its retry scheduled. Do not monopolize the - # submitter with a second retry budget. Uncertain commits, - # ordinary unfenced tip sources, and a covered replay's - # leaked fence still reconcile now. - publish_now = ( - covered_replay_fence - or pending_cause == "direct_block_uncertain" - or not payout_publication_fenced - ) - published: int | None = None - if publish_now and getattr( - self.runtime, - "reorg_reconciler_enabled", - True, - ): - with self.runtime.lock: - latest_tip = self.runtime._payout_state_source[1] - summary = self.runtime.reconcile_prism_pool_blocks_once( - tip_hash=latest_tip, - _force_publish=True, - _source_reserved=True, - ) - reconciled_generation = summary.get("published_generation") - if isinstance(reconciled_generation, int): - published = reconciled_generation - elif publish_now: - published = ( - self.runtime._publish_current_payout_state_with_retry_budget() - ) - if publish_now and published is None: - # The block is durably confirmed; only the payout - # publication lost its race. Aborting would keep the - # outbox row pending and replay persist/confirm churn - # for an already-final block. Keep delivery fenced and - # let the scheduled tip refresh publish the newest - # source; this candidate's durable work is complete. - self.runtime._block_payout_state_publication() - print( - "prism coordinator: accepted block confirmed " - "durably; payout publication deferred to the " - f"scheduled refresh hash={block_hash}", - flush=True, - ) - return ( - final_bundle, - report, - persistence, - confirmation, - audit_publication_identity, - dict(verified_audit.verification_identity), - ) - except Exception: - if payout_commit_started is not None and payout_commit_source is not None: - # Persistence/confirmation can report failure after a - # durable partial commit. Supersede every prepared source - # and keep all delivery fenced until replay/reconciliation - # proves the resulting ledger state. - self.runtime._block_payout_state_publication( - supersede_with=( - payout_commit_source, - block_hash, - "direct_block_uncertain", - payout_commit_started, - ) - ) - raise - finally: - if payout_commit_started is not None: - self.runtime._observe_payout_state_seconds( - "preparation", - max(0.0, time.monotonic() - payout_commit_started), - ) - audit_store.discard_candidate(candidate_artifact) - - def _admit_candidate( - self, - candidate: PrismBlockCandidate, - *, - node_submission: _BlockCandidateNodeSubmission, - ) -> FinalizationAdmission | bool | None: - """Classify one post-offer candidate against pool and chain state. - - Returns an admission for the ordered phases, a terminal bool for an - outcome the disposition can finalize directly (an already-recorded - acceptance or an accepted race win), or ``None`` for a terminal - abandonment already recorded through the outcome seam. - """ - outcome = getattr(self.runtime, "_block_candidate_outcome", None) - if outcome is None: - outcome = threading.local() - self.runtime._block_candidate_outcome = outcome - outcome.reason = None - outcome.error = None - outcome.stale_job_class = None - context = candidate.context - submission = candidate.submission - worker = candidate.client.username or None - expected_height = int(context.template["height"]) - block_hash = str(submission.block_hash_hex).lower() - parent_hash = str(context.template["previousblockhash"]) - self.runtime._ensure_job_cache_state() - # Every disposition (queue drain, synchronous below-target submit, - # outbox replay, retained retry) marks its hash outstanding so tip - # observations arriving on other threads can register acceptance. - self.runtime._register_outstanding_block_candidate(block_hash) - self._record_block_candidate_progress("disposition-start") - if ( - self.runtime._block_candidate_acceptance_recorded(block_hash) - and node_submission.error is not None - ): - # A concurrent same-hash pass completed the success tail while - # this duplicate-safe node offer waited for disposition. Do not - # recreate its payout transition or accounting work. - self.runtime._clear_accepted_block_payout_preview(block_hash) - return True - with self.runtime.lock: - accepted_count = int(self.runtime.accepted_block_count) - pool_closed = ( - block_hash not in self.runtime._accounted_accepted_block_hashes - and ( - accepted_count >= int(self.runtime.max_blocks) - or (bool(self.runtime.stop_after_block) and accepted_count >= 1) - ) - ) - if pool_closed: - with self._landing_phase( - LANDING_PHASE_CHAIN_PROBE, - block_hash=block_hash, - block_height=expected_height, - ): - closed_pool_chain_probe = self.runtime._block_candidate_chain_probe( - block_hash, - expected_height=expected_height, - ) - if closed_pool_chain_probe is True: - # The chain provably contains this block; its payout accounting - # must complete regardless of when the pool stopped accepting - # new work. Fall through to the normal disposition below, which - # resumes the accepted success tail. Observation evidence alone - # deliberately does not open this gate -- an unprovable view - # defers via the abandon path instead, so a closed pool can - # never fall through to submitblock on stale evidence. - pool_closed = False - if pool_closed: - self.runtime._abandon_block_candidate( - PRISM_REJECTION_POOL_CLOSED, - "pool is no longer accepting blocks", - block_hash=block_hash, - worker=worker, - expected_height=expected_height, - ) - return None - self._record_block_candidate_progress("current-tip-rpc") - observed_tip = str(self.runtime.rpc.call("getbestblockhash")) - self._record_block_candidate_progress("current-tip-rpc:complete") - # A successful or transport-ambiguous fast-lane call can change the - # tip before this post-submit probe. It is still a *fresh* attempt, - # not an active replay: run the normal validation/persistence tail - # against the candidate's stamped parent. A later getblockhash check - # proves a successful acknowledgement, while an ambiguous transport - # outcome stays pending for duplicate-safe replay. Duplicate replies - # are replay evidence and retain the live-tip classification; the - # abandonment tie-breaker separately consults this process's own - # recorded offer evidence so an unprovable chain view cannot - # terminally discard a block the node already told us it has. - fresh_or_uncertain_submit = bool( - node_submission.attempted - and ( - node_submission.error is not None - or node_submission.result is None - ) - ) - current_tip = parent_hash if fresh_or_uncertain_submit else observed_tip - landed_height: int | None = None - if current_tip.lower() == block_hash: - landed_height = expected_height - elif current_tip != parent_hash: - try: - landed_height = self.runtime.active_block_candidate_height(block_hash) - except Exception: - traceback.print_exc() - self.runtime._abandon_block_candidate( - PRISM_REJECTION_BACKEND_RPC_UNAVAILABLE, - "could not determine whether a prior candidate is active", - block_hash=block_hash, - worker=worker, - ) - return None - already_active = landed_height == expected_height - if landed_height is not None and not already_active: - self.runtime._abandon_block_candidate( - PRISM_REJECTION_BLOCK_STALE, - f"candidate active at unexpected height {landed_height}", - block_hash=block_hash, - worker=worker, - expected_height=expected_height, - ) - return None - if already_active: - # A disposition probe is a tip observation too: remember it so a - # later attempt cannot terminally abandon this hash on a racing - # chain snapshot after this attempt fails mid-tail. - self.runtime._note_tip_observation_for_candidates(block_hash) - print( - "prism coordinator: resuming finalization for active block candidate " - f"height={landed_height} hash={submission.block_hash_hex}", - flush=True, - ) - elif parent_hash != current_tip: - if self.runtime._block_candidate_acceptance_recorded(block_hash): - # A duplicate wakeup can reach this check after the accepted - # success tail but after a newer tip hides the candidate from - # the active-header probe. Its durable work is already done; - # let the caller finalize the outbox as submitted. - self.runtime._clear_accepted_block_payout_preview(block_hash) - return True - accepted_race_won = self.runtime._abandon_block_candidate( - PRISM_REJECTION_STALE_JOB, - f"tip moved before submit: {current_tip}", - block_hash=block_hash, - worker=worker, - preserve_if_accepted=True, - expected_height=expected_height, - stale_job_class="tip_moved", - ) - return bool(accepted_race_won) - if ( - node_submission.attempted - and node_submission.error is None - and node_submission.result not in (None, "duplicate") - and not already_active - ): - self.runtime._abandon_block_candidate( - PRISM_REJECTION_SUBMITBLOCK_REJECTED, - f"submitblock rejected candidate: {node_submission.result}", - block_hash=block_hash, - worker=worker, - expected_height=expected_height, - ) - return None - return FinalizationAdmission( - candidate=candidate, - context=context, - submission=submission, - worker=worker, - expected_height=expected_height, - block_hash=block_hash, - parent_hash=parent_hash, - # Fresh-attempt classification may use the stamped parent, but - # payout reconciliation must follow the observed post-submit tip. - current_tip=observed_tip, - already_active=already_active, - node_submission=node_submission, - ) - - def _land_candidate( - self, - admission: FinalizationAdmission, - ) -> LandedCandidate | None: - """Decide replay validity, fence the anchor, and land the candidate.""" - candidate = admission.candidate - context = admission.context - worker = admission.worker - expected_height = admission.expected_height - block_hash = admission.block_hash - node_submission = admission.node_submission - # A reconstructed candidate revalidates BEFORE the balance - # serializer: the audit share-window replay is the slow oracle walk - # and takes the ledger writer lock, so running it inside - # _payout_balance_mutation_lock would stall share persistence and - # every other landing for the walk's duration. The read is safe out - # here because the window is append-only content -- a pass can only - # be invalidated by a later append, and any such append after the - # baseline epoch captured below advances the live epoch (this - # landing's declared anchor stays exposed to the append-side - # predates() checks), which the landing's fences compare against. - collection_only = bool(getattr(context, "collection_only", False)) - context_append_epoch = int( - getattr(context, "payout_append_invalidation_epoch", 0) - ) - revalidated_append_epoch: int | None = None - landing_anchor_token: int | None = None - if not admission.already_active and not collection_only: - declared_anchor_ms = _declared_anchor_ms(context) - if declared_anchor_ms is not None: - # With no armed artifact and no in-flight walk (an outbox - # replay at startup, or a landing after the artifact was - # disarmed), a replay-shaped append would skip the epoch - # bump entirely; exposing the landing window's own anchor - # guarantees the bump the fences below check for. - landing_anchor_token = self.runtime._expose_inflight_scan_anchor( - int(declared_anchor_ms) - ) - # An append classified as unfenced before that exposure - # commits outside the landing fence, so its row could - # become durable mid-submitblock while its epoch bump - # queues behind the fence this landing holds across the - # RPC. Wait those commits (and their bump attempts - # against the now-exposed anchor) out before any window - # revalidation or epoch fence runs; no fence is held - # here, so the bump can always proceed. - self.runtime._await_unfenced_appends_predating_anchor( - int(declared_anchor_ms) - ) - try: - reconstructed_needs_revalidation = ( - not admission.already_active - and not collection_only - and context_append_epoch < 0 - and bool(getattr(self.runtime.ledger, "durable_payout_state", False)) - ) - if reconstructed_needs_revalidation and node_submission.attempted: - # Revalidation guards an offer the node has not yet seen: a - # reconstructed window that omits a durably appended share - # must not mint a coinbase. Once the fast lane has offered - # the durable bytes, the coinbase is the node's to judge — - # an accepted or duplicate result proceeds with the - # as-issued snapshot (the post-offer epoch fences log - # rather than abandon), a rejection stayed terminal above, - # and an ambiguous transport error re-offers duplicate- - # safely on retry. Abandoning an already-offered candidate - # here would permanently discard payout accounting for a - # block qbitd may have accepted, and the audit walk is the - # slow oracle whose deadline under saturation would - # otherwise defer-loop an accepted block forever. - print( - "prism coordinator: reconstructed candidate was already " - f"offered to the node hash={block_hash}; keeping the " - "as-issued payout snapshot (window revalidation " - "skipped)", - flush=True, - ) - if ( - reconstructed_needs_revalidation - and not node_submission.attempted - ): - with self.runtime._job_cache_lock: - revalidation_base_epoch = int( - self.runtime._payout_ledger_append_invalidation_epoch - ) - try: - window_reproducible = ( - self.runtime._replayed_payout_window_reproducible(context) - ) - except Exception: - traceback.print_exc() - self.runtime._abandon_block_candidate( - PRISM_REJECTION_BACKEND_RPC_UNAVAILABLE, - "durable share-window replay failed for the " - "reconstructed candidate", - block_hash=block_hash, - worker=worker, - ) - return None - if not window_reproducible: - self.runtime._abandon_block_candidate( - PRISM_REJECTION_STALE_JOB, - "replayed payout window omits a durably appended share", - block_hash=block_hash, - worker=worker, - expected_height=expected_height, - stale_job_class="append_epoch_stale", - ) - return None - revalidated_append_epoch = revalidation_base_epoch - # Land through the coordinator/runtime seam so per-instance - # monkeypatches on _land_and_confirm_block_candidate intercept. - landed = self.runtime._land_and_confirm_block_candidate( - candidate, - current_tip=admission.current_tip, - already_active=admission.already_active, - worker=worker, - node_submission=node_submission, - revalidated_append_epoch=revalidated_append_epoch, - ) - finally: - self.runtime._retire_inflight_scan_anchor(landing_anchor_token) - if landed is None: - return None - return LandedCandidate(*landed) - - def _candidate_already_accounted(self, block_hash: str) -> bool: - with self.runtime.lock: - return block_hash in self.runtime._accounted_accepted_block_hashes - - def _persist_ctv_and_credit( - self, - admission: FinalizationAdmission, - landed: LandedCandidate, - ) -> dict[str, Any] | None: - candidate = admission.candidate - block_hash = admission.block_hash - ctv_persistence = None - # On a bounded view the manifest set's fanout manifests stay lazy on - # disk; the digest streams the sorted canonical encoding and the - # ledger statement streams the set into its literal (#255). - ctv_manifest_set = landed.final_bundle.get("ctv_fanout_manifest_set") - if isinstance(ctv_manifest_set, Mapping): - self._record_block_candidate_progress("ctv-manifest-persist") - ctv_persistence = self.runtime.ledger.persist_ctv_fanout_manifest_set( - block_hash=block_hash, - manifest_set=ctv_manifest_set, - manifest_set_sha256=streamed_sha256_json_hex(ctv_manifest_set), - ) - self._record_block_candidate_progress("ctv-manifest-persist:complete") - if candidate.credit_share_on_accept: - self._record_block_candidate_progress("accepted-share-credit") - self.runtime.append_accepted_share( - candidate.client, - admission.context, - admission.submission, - candidate.pending_share, - candidate_intent=self.runtime.block_candidate_intent(candidate), - ) - # The preview window was intentionally clamped below this pending - # winning share. Once the append is durable, enqueue an urgent - # delta fold so a rapidly found child does not wait for the normal - # 60-second cadence to include its payout obligation. - with self.runtime._job_cache_lock: - payout_generation = self.runtime._payout_state_generation - template_artifacts = self.runtime._template_artifacts - if template_artifacts is not None: - self.runtime._schedule_payout_ledger_artifact_preparation( - payout_generation, - template_artifacts.network_difficulty, - bypass_build_interval=True, - ) - self._record_block_candidate_progress("accepted-share-credit:complete") - return ctv_persistence - - def _build_finalization_evidence( - self, - admission: FinalizationAdmission, - landed: LandedCandidate, - ctv_persistence: dict[str, Any] | None, - ) -> FinalizationEvidence: - # Aggregate counts only: materializing the whole share history - # (all_shares) here would scan the full ledger twice per block, - # and would grow without bound as the ledger grows. The counters are - # served from the ledger's maintained cache; a cold cache (first - # read after process start) runs the exact aggregate synchronously - # and stays watchdog-eligible on purpose, so a wedged read keeps the - # exit-and-replay recovery path instead of hanging the disposition - # invisibly. - self._record_block_candidate_progress("accepted-share-stats") - evidence_share_count, evidence_distinct_miners = self.runtime.accepted_share_stats() - self._record_block_candidate_progress("accepted-share-stats:complete") - evidence = { - "schema": "qbit.prism.live-stratum-evidence.v1", - "block_hash": admission.block_hash, - "block_height": admission.expected_height, - "coinbase_tx_hex": admission.submission.coinbase_tx_hex, - "audit_report": landed.report, - "ledger_backend": self.runtime.ledger.backend_name, - "persistence": landed.persistence, - "confirmation": landed.confirmation, - "audit_verification_identity": landed.audit_verification_identity, - "ctv_persistence": ctv_persistence, - "accepted_share_count": evidence_share_count, - "distinct_miner_count": evidence_distinct_miners, - "job_share_count": len(admission.context.shares_json), - } - publication_persistence = dict(landed.persistence) - publication_persistence.setdefault( - "audit_bundle_sha256", - landed.report.get("audit_bundle_sha256_hex"), - ) - publication_persistence.setdefault("body_uri", "") - evidence["persistence"] = publication_persistence - return FinalizationEvidence( - evidence=evidence, - publication_persistence=publication_persistence, - ) - - def _publish_finalization_evidence( - self, - landed: LandedCandidate, - prepared: FinalizationEvidence, - ) -> dict[str, Any]: - audit_store = self.runtime._ensure_audit_artifact_store() - self._record_block_candidate_progress("evidence-write") - # This is the live writer finalizing a block it confirmed itself in - # this landing. Two distinct found blocks can land inside one - # confirm->publish window -- the synchronous client-thread tail and - # the accounting actor do not serialise across hashes -- so this - # publication can arrive already superseded on the publication - # ordinal. It must then still write its own height+hash live - # envelope, which is the block's only published evidence pointer. - # - # Confirming with a valid lease does not prove the lease survived the - # confirm->publish gap: a writer deposed inside the gap would reach - # this call site with only stale reads behind it. Restoring a - # superseded envelope is a mutation of the shared audit root that the - # current evidence reference never re-validates, so restore authority - # requires a fresh bounded non-blocking proof of the exact guarded - # session (a no-op for ledgers without the writer lease guard). The - # proof gates only the restore: withholding authority degrades to the - # pre-restore behaviour of skipping the superseded envelope write, - # which is always safe, whereas failing the publication on a lease - # hiccup would be a live regression on the found-block path. The - # fence may arm a hard exit internally for a genuinely dead lease; - # that exit proceeds on its own and is not undone here. - restore_superseded_envelope = True - lease_fence_component = "superseded_audit_envelope_restore" - require_fresh_lease = getattr( - self.runtime, - "_require_fresh_ledger_lease_for_external_side_effect", - None, - ) - withheld_restore_authority: Exception | None = None - with self.runtime._payout_balance_mutation_lock: - with audit_store.publication_order_guard(): - # The proof runs with both locks held: the wait on either - # lock is unbounded, so a writer deposed while queued must - # not restore on authority it proved before the wait. - if callable(require_fresh_lease): - try: - require_fresh_lease(lease_fence_component) - except (ShutdownInProgress, WriterLeaseRenewalDeferred) as exc: - restore_superseded_envelope = False - withheld_restore_authority = exc - publication_floor_reader = getattr( - self.runtime.ledger, - "audit_publication_sequence_floor", - None, - ) - if callable(publication_floor_reader): - # This is deliberately a fresh durable-row read immediately - # before A1 publication. Confirmation-time state or a raw - # sequence value cannot fence rollback gaps and restart - # replays. P1's local serializer plus A1's process guard - # prevent another confirmation from allocating between this - # read and the durable publication decision. Reactivation - # preserves its block's already-published ordinal. - publication_floor_sequence = publication_floor_reader() - else: - # Compatibility-only ledgers used by legacy embeddings/tests - # do not own durable ordinal state. Production memory/Postgres - # backends implement the reader above. - publication_floor_sequence = ( - landed.audit_publication_identity.sequence - ) - publication = audit_store.publish_success( - identity=landed.audit_publication_identity, - publication_floor_sequence=publication_floor_sequence, - report=landed.report, - persistence=prepared.publication_persistence, - evidence=prepared.evidence, - verification_identity=landed.audit_verification_identity, - created_at=public_api.utc_now_iso(), - # Granted by the fresh-lease proof above; a deposed writer - # replaying stale state never receives restore authority. - restore_superseded_envelope=restore_superseded_envelope, - ) - if withheld_restore_authority is not None: - # Withheld authority silently reverts to the historical lossy - # behaviour for a superseded publication, so it must be - # visible. Diagnostics go to stderr because worker processes - # reserve stdout for a strict JSON-lines protocol, and are - # emitted only after the store locks are released (see below). - print( - "prism coordinator: withholding superseded-envelope " - f"restore authority component={lease_fence_component} " - f"error={type(withheld_restore_authority).__name__}: " - f"{withheld_restore_authority}", - file=sys.stderr, - flush=True, - ) - if publication.superseded_envelope_written: - # Emitted only after the balance lock and the publication order - # guard have been released: an undrained stderr must never block - # the found-block path while a store lock is held. - print( - "prism coordinator: wrote live envelope for superseded " - f"publication sequence={publication.identity.sequence} " - f"floor_sequence={publication_floor_sequence} " - f"height={publication.identity.block_height} " - f"hash={publication.identity.block_hash}", - file=sys.stderr, - flush=True, - ) - published_evidence = dict(publication.evidence) - self._record_block_candidate_progress("evidence-write:complete") - return published_evidence - - def _account_finalized_candidate( - self, - admission: FinalizationAdmission, - landed: LandedCandidate, - published_evidence: dict[str, Any], - ) -> bool: - # The copied publication is intentionally consumed before accounting; - # converting an invalid publication remains a finalization failure. - del published_evidence - block_hash = admission.block_hash - with self.runtime.lock: - newly_accounted = block_hash not in self.runtime._accounted_accepted_block_hashes - if newly_accounted: - self.runtime._accounted_accepted_block_hashes.add(block_hash) - self.runtime.accepted_block_count += 1 - # Replace this hash's provisional capacity reservation with - # its durable accounted slot atomically. Keeping both until - # the outbox terminal write would double-count the block and - # unnecessarily reject an unrelated next solve. - self.runtime._block_fast_lane_reservations.discard(block_hash) - self.runtime.latest_coinbase_size_bytes = len( - str( - landed.final_bundle["signed_coinbase_manifest"]["manifest"][ - "coinbase_tx_hex" - ] - ) - ) // 2 - should_stop = ( - newly_accounted - and (self.runtime.stop_after_block or self.runtime.accepted_block_count >= self.runtime.max_blocks) - ) - if not newly_accounted: - return True - print( - "prism coordinator: qbit accepted direct PRISM block " - f"height={admission.expected_height} hash={block_hash}", - flush=True, - ) - if should_stop: - self.runtime.request_shutdown() - else: - # The public submitter wrapper performs this fanout only after its - # writer scope (including outbox finalization) exits. The rare - # synchronous share path consumes the same marker after sending - # the Stratum result. - admission.candidate.client.post_accept_refresh_block = ( - admission.expected_height, - block_hash, - ) - return True - - def _submit_block_candidate_serialized( - self, - candidate: PrismBlockCandidate, - *, - node_submission: _BlockCandidateNodeSubmission, - ) -> bool: - """Process a candidate while its same-hash disposition guard is held. - - Runs the ordered, replay-safe post-offer finalization phases. B1's - accounting actor enters here directly with the disposition already - held; the decorated public wrapper reaches the same runner through - the coordinator seam. - """ - self._note_candidate_started() - with self._phase("admission"): - admission = self._admit_candidate( - candidate, - node_submission=node_submission, - ) - if admission is None: - return False - if isinstance(admission, bool): - return admission - with self._phase("land_confirm"): - landed = self._land_candidate(admission) - if landed is None: - return False - try: - self._record_block_candidate_progress("durable-accounting:complete") - if self._candidate_already_accounted(admission.block_hash): - # The previous attempt completed every success side effect - # but its durable outbox terminal update failed. submit_next - # will retry that update after this exact-idempotent - # confirmation without double counting the block or - # replacing newer evidence/work. - return True - with self._phase("ctv_credit"): - ctv_persistence = self._persist_ctv_and_credit(admission, landed) - with self._phase("evidence"): - prepared = self._build_finalization_evidence( - admission, - landed, - ctv_persistence, - ) - with self._phase("audit_publish"): - published_evidence = self._publish_finalization_evidence( - landed, - prepared, - ) - with self._phase("accounting"): - return self._account_finalized_candidate( - admission, - landed, - published_evidence, - ) - finally: - # The landed bundle is the last legitimate owner of the bounded - # artifact view; retire its descriptor on every exit so neither - # a failed phase nor a completed one leaves the candidate file - # pinned (#255). Compatibility dictionaries have nothing to - # close. - close_view = getattr( - getattr(landed, "final_bundle", None), - "close", - None, - ) - if callable(close_view): - close_view() - - def submit_block_candidate( - self, - candidate: PrismBlockCandidate, - *, - node_submission: _BlockCandidateNodeSubmission | None = None, - ) -> bool: - """Land one block candidate, then finalize its audit and payout state. - - Runs on the block-submitter thread (tests call it synchronously). It - never raises for a lost race and holds self.runtime.lock only for short - in-memory state mutation -- never across RPC, psql, subprocess, or - file I/O -- so share acks and job pushes stay fast while a block - lands. The durable candidate outbox is the pre-submit recovery boundary; - full audit and payout persistence happens after the latency-sensitive - ``submitblock`` call and is replayable after a crash. Returns True only - after that finalization completes. - """ - block_hash = str(candidate.submission.block_hash_hex).lower() - with self.runtime._block_candidate_disposition(block_hash): - terminal_outcome = self.runtime._block_candidate_terminal_outcome(block_hash) - if terminal_outcome is not None: - return terminal_outcome - if node_submission is None: - node_submission = self.runtime._node_submission_for_direct_candidate(candidate) - accepted = self.runtime._submit_block_candidate_serialized( - candidate, - node_submission=node_submission, - ) - if not accepted: - outcome = getattr(self.runtime, "_block_candidate_outcome", None) - if outcome is not None: - # Direct embedders do not use the outbox-finalization - # wrapper. A normal return means the serialized path also - # completed any prepared-state rejection it initiated. - self.runtime._record_committed_block_candidate_abandonment( - block_hash, - outcome, - ) - return accepted diff --git a/lab/prism/bounded_executor.py b/lab/prism/bounded_executor.py deleted file mode 100644 index 862f7719e..000000000 --- a/lab/prism/bounded_executor.py +++ /dev/null @@ -1,166 +0,0 @@ -"""Bounded priority executor used for PRISM job delivery.""" - -from __future__ import annotations - -from concurrent.futures import Future -import heapq -import math -import queue -import threading -from typing import Any, Callable - - -class _DeliveryQueueFull(RuntimeError): - """The bounded delivery executor cannot admit another task.""" - - -class _BoundedPriorityExecutor: - """Small Future-compatible executor with bounded, priority-ordered work.""" - - def __init__( - self, - *, - max_workers: int, - max_queue_size: int, - thread_name_prefix: str = "prism-job-delivery", - ) -> None: - self.max_workers = max_workers - self.max_queue_size = max_queue_size - self._queue: queue.PriorityQueue[tuple[object, ...]] = queue.PriorityQueue( - maxsize=max_queue_size - ) - self._lock = threading.Lock() - self._sequence = 0 - self._active_workers = 0 - self._shutdown = False - self._threads = [ - threading.Thread( - target=self._worker, - name=f"{thread_name_prefix}-{index + 1}", - daemon=True, - ) - for index in range(max_workers) - ] - for thread in self._threads: - thread.start() - - def submit( - self, - function: Callable[..., Any], - /, - *args: object, - # Same-tip refreshes are the lowest-priority delivery class; the - # numeric default stays owner-local so this executor never imports - # the delivery-priority registry. - priority: int = 2, - **kwargs: object, - ) -> Future[Any]: - future: Future[Any] = Future() - with self._lock: - if self._shutdown: - raise RuntimeError("delivery executor is shut down") - self._sequence += 1 - item = ( - int(priority), - self._sequence, - future, - function, - args, - kwargs, - ) - try: - self._queue.put_nowait(item) - except queue.Full as exc: - raise _DeliveryQueueFull("delivery executor queue is full") from exc - return future - - def _worker(self) -> None: - while True: - item = self._queue.get() - try: - _, _, future, function, args, kwargs = item - if function is None: - return - assert isinstance(future, Future) - if not future.set_running_or_notify_cancel(): - continue - with self._lock: - self._active_workers += 1 - try: - result = function(*args, **kwargs) - except BaseException as exc: - future.set_exception(exc) - else: - future.set_result(result) - finally: - with self._lock: - self._active_workers -= 1 - finally: - self._queue.task_done() - # An idle worker's frame stays alive in queue.get(). Release - # the previous task outside all locks before waiting: delivery - # arguments can own an entire superseded payout window. Cover - # failed and cancelled tasks too, and drop any prior result. - item = future = function = args = kwargs = result = None - - def cancel(self, future: Future[Any]) -> bool: - """Cancel ``future`` and immediately discard it when still queued. - - ``Future.cancel`` alone leaves the cancelled entry in PriorityQueue - until a worker dequeues it. Removing the exact entry while holding the - queue mutex makes bounded admission available to a replacement at the - cancellation boundary. A worker that already dequeued the entry owns - the normal ``task_done`` path, so the two paths cannot double-release. - """ - removed = False - with self._queue.mutex: - queued_items = self._queue.queue - for index, item in enumerate(queued_items): - if item[2] is not future: - continue - queued_items.pop(index) - heapq.heapify(queued_items) - self._queue.unfinished_tasks -= 1 - if self._queue.unfinished_tasks == 0: - self._queue.all_tasks_done.notify_all() - self._queue.not_full.notify() - removed = True - break - # Invoke callbacks only after releasing the queue mutex. Initial-job - # cancellation callbacks may submit a replacement to this executor. - future.cancel() - return removed - - def stats(self) -> tuple[int, int]: - with self._lock: - return self._queue.qsize(), self._active_workers - - def shutdown(self, *, wait: bool = True, cancel_futures: bool = False) -> None: - with self._lock: - if self._shutdown: - threads = list(self._threads) - already_shutdown = True - else: - self._shutdown = True - threads = list(self._threads) - already_shutdown = False - if already_shutdown: - if wait: - for thread in threads: - thread.join() - return - if cancel_futures: - while True: - try: - item = self._queue.get_nowait() - except queue.Empty: - break - future = item[2] - if isinstance(future, Future): - future.cancel() - self._queue.task_done() - for index in range(len(threads)): - self._queue.put((math.inf, index, None, None, (), {})) - if wait: - for thread in threads: - thread.join() diff --git a/lab/prism/bundle_compiler.py b/lab/prism/bundle_compiler.py deleted file mode 100644 index d3479a7e1..000000000 --- a/lab/prism/bundle_compiler.py +++ /dev/null @@ -1,2639 +0,0 @@ -#!/usr/bin/env python3 -"""Cancelable subprocess and daemon transport for PRISM audit-bundle builds. - -The compiler owns the complete builder wire/process domain: the compact -audit payload encoding, the share-window spool representation and its -leases, the one-shot subprocess build, and the persistent ``--serve`` daemon -protocol. It never imports ``prism_coordinator`` or ``job_bundle``: shared -scheduler state and refresh metrics are reached through the -:class:`BundleCompilerRuntime` port (resolved at call time so coordinator -monkeypatch seams keep working), and the job-bundle exception/control types -are injected at construction exactly like the historical layer's -``superseded_error`` port. -""" - -from __future__ import annotations - -from collections import OrderedDict -from contextlib import ExitStack, contextmanager -from dataclasses import dataclass, field -import json -import os -from pathlib import Path -import selectors -import stat -import subprocess -import tempfile -import threading -import time -import uuid -from collections.abc import Mapping, Sequence -from typing import Any, Callable, Iterable, Iterator, Protocol - -from lab.prism.audit_bundle_view import CanonicalAuditBundleView -from lab.prism.coordinator_config import ( - DEFAULT_PRISM_BUNDLE_BUILD_TIMEOUT_SECONDS, - DEFAULT_PRISM_JOB_BUILD_CANCEL_GRACE_SECONDS, - env_bool, -) -from lab.prism.prism_tools import prism_tool_command -from lab.prism.share_json_stream import ( - SHARE_JSON_BATCH_RECORDS, - SHARE_JSON_CHUNK_BYTES, - iter_json_array_text_chunks, - iter_json_object_text_chunks, -) -from lab.prism.share_ledger import DaemonWindowMirrorDivergence - - -PRISM_BUILDER_PHASE_METRICS_PREFIX = "qbit-prism-build-phase-metrics " -# Owner-local duplicate of the coordinator's admission poll cadence; the -# compiler must not import the coordinator for one pacing constant. -PRISM_TIP_REFRESH_ADMISSION_POLL_SECONDS = 0.05 -# Longest one pipe readiness wait may block before a transport loop -# re-checks cancellation, supersession and its absolute deadline. This is -# the cadence the retired fixed post-EAGAIN sleeps enforced; a readiness -# wait wakes the moment the pipe can progress and only sleeps this long -# when it cannot. -PRISM_BUILDER_PIPE_WAIT_SLICE_SECONDS = min( - 0.02, - PRISM_TIP_REFRESH_ADMISSION_POLL_SECONDS, -) -# Largest single os.read from a builder pipe. Matches the default Linux -# pipe capacity so one wakeup drains one full pipe. -PRISM_BUILDER_PIPE_READ_CHUNK_BYTES = 1 << 16 -# Upper bound for one kernel transfer from the share-window spool file into -# the audit builder's stdin pipe. The kernel clamps each call to the free -# pipe capacity anyway; the bound only paces cancellation checkpoints. -PRISM_SPOOL_SPLICE_CHUNK_BYTES = 1 << 20 -# JSONL protocol version this coordinator speaks with the --serve audit -# builder. The daemon announces its version in a startup handshake; any -# mismatch retires the daemon and every build falls back to one-shot mode. -# Version 2 adds the prepare_window request (window fold, canonical digest, -# and incremental advance daemon-side) and the append-invalidation epoch tag. -PRISM_SERVE_BUILDER_PROTOCOL_VERSION = 2 -# Parsed share windows the --serve daemon retains; the coordinator mirrors -# the bound to predict which uploads the daemon still holds. -PRISM_SERVE_BUILDER_WINDOW_CACHE_ENTRIES = 2 - - -def canonical_bundle_bytes(bundle: dict[str, Any]) -> bytes: - """Use J1's typed Rust adapter for canonical audit-bundle bytes. - - Generic compact JSON is not equivalent: serde restores typed field order, - emits UTF-8, and omits optional defaults. Keep those digest semantics in J1. - """ - - completed = subprocess.run( - prism_tool_command("qbit-prism-audit-canonicalize") + ["--input", "-"], - input=json.dumps(bundle).encode("utf-8"), - stdout=subprocess.PIPE, - stderr=subprocess.PIPE, - check=False, - ) - if completed.returncode != 0: - stderr = completed.stderr.decode(errors="replace").strip() - raise RuntimeError(f"qbit-prism-audit-canonicalize failed: {stderr}") - return completed.stdout - - -class CancellationPort(Protocol): - """Cooperative-cancellation surface consumed by the compiler.""" - - def is_set(self) -> bool: ... - - def raise_if_cancelled(self, phase: str) -> None: ... - - -class BundleBuildControlPort(Protocol): - """Registered build control consumed for supersession checks.""" - - cancel_event: Any - process: Any - - -class BundleCompilerRuntime(Protocol): - """Typed port over the coordinator, resolved at call time.""" - - signing_seed_hex: str - ledger_attestation_signing_seed_hex: str - - def _ensure_job_cache_state(self) -> None: ... - - def _ensure_tip_refresh_state(self) -> None: ... - - def _job_build_checkpoint(self, phase: str, cancellation: Any) -> None: ... - - def _job_build_phases(self) -> dict[str, float]: ... - - def prism_payout_policy(self) -> dict[str, object]: ... - - def prism_ctv_settlement_config( - self, - *, - block_height: int, - parent_hash: str | None, - ) -> dict[str, object] | None: ... - - def _observe_tip_refresh_build_phase(self, name: str, elapsed: float) -> None: ... - - def _record_tip_refresh_ipc_bytes(self, direction: str, byte_count: int) -> None: ... - - def _register_job_bundle_process(self, control: Any, process: Any) -> None: ... - - def _unregister_job_bundle_process(self, control: Any, process: Any) -> None: ... - - def _note_window_mirror_divergence(self) -> None: ... - - -def _compact_share_payload( - shares: list[dict[str, object]], -) -> tuple[list[tuple[str, str, str]], list[tuple[object, ...]]]: - """Deduplicate share identities into the audit-builder compact form.""" - - identity_indexes: dict[tuple[str, str, str], int] = {} - identities: list[tuple[str, str, str]] = [] - compact_shares: list[tuple[object, ...]] = [] - for share in shares: - identity = ( - str(share["miner_id"]), - str(share["order_key"]), - str(share["p2mr_program_hex"]), - ) - identity_index = identity_indexes.get(identity) - if identity_index is None: - identity_index = len(identities) - identity_indexes[identity] = identity_index - identities.append(identity) - compact_shares.append( - ( - share["share_seq"], - share["share_id"], - identity_index, - share["share_difficulty"], - share["job_issued_at_ms"], - share["accepted_at_ms"], - share.get("credit_policy"), - ) - ) - return identities, compact_shares - - -def _compact_share_tail_chunks( - shares: Iterable[dict[str, object]], - *, - batch_records: int = SHARE_JSON_BATCH_RECORDS, - chunk_chars: int = SHARE_JSON_CHUNK_BYTES, -) -> tuple[str, ...]: - """The audit-builder payload tail as bounded ASCII chunks. - - Concatenated, the chunks are byte-identical to the historical tail -- - ``',"compact_share_identities":' + json.dumps(identities) + - ',"compact_shares":' + json.dumps(compact_shares) + "}"`` with compact - separators over :func:`_compact_share_payload`'s two lists, the exact - bytes the spool held and the in-memory fallback wrote -- but no - ``json.dumps`` call covers more than ``batch_records`` records, so - encoding a 200k-share window no longer holds the GIL in one piece - (#236). Identities are indexed in first-seen order exactly as before; - the share rows are encoded as they are walked, and the identity table, - complete only once the walk ends, is placed ahead of them in the - returned order. - """ - identity_indexes: dict[tuple[str, str, str], int] = {} - identities: list[tuple[str, str, str]] = [] - - def compact_rows() -> Iterator[tuple[object, ...]]: - for share in shares: - identity = ( - str(share["miner_id"]), - str(share["order_key"]), - str(share["p2mr_program_hex"]), - ) - identity_index = identity_indexes.get(identity) - if identity_index is None: - identity_index = len(identities) - identity_indexes[identity] = identity_index - identities.append(identity) - yield ( - share["share_seq"], - share["share_id"], - identity_index, - share["share_difficulty"], - share["job_issued_at_ms"], - share["accepted_at_ms"], - share.get("credit_policy"), - ) - - share_chunks = tuple( - iter_json_array_text_chunks( - compact_rows(), - batch_records=batch_records, - chunk_chars=chunk_chars, - ) - ) - identity_chunks = tuple( - iter_json_array_text_chunks( - identities, - batch_records=batch_records, - chunk_chars=chunk_chars, - ) - ) - return ( - ',"compact_share_identities":[', - *identity_chunks, - '],"compact_shares":[', - *share_chunks, - "]}", - ) - - -PRISM_BUILDER_INPUT_ARRAY_KEYS = ( - "shares", - "prior_balances", - "witness_merkle_leaves_hex", - "compact_share_identities", - "compact_shares", -) - - -def _iter_share_window_items( - value: Any, - *, - batch_records: int, - chunk_chars: int, -) -> Iterator[str | bytes]: - """The ``shares`` array body from the cheapest exact source. - - A daemon-mirror sequence carries the window already encoded as - ``canonical_items`` (sorted-key records with numeric difficulties, the - bytes the builder daemon itself parses); a page-backed window carries - one such fragment per page. Both are spliced in bounded slices without - parsing a record -- in particular without touching the mirror's parsed - tuple cache. Any other replayable sequence is walked record by record. - The builder deserializes by field name, so record key order is - irrelevant to the canonical output. - """ - items = getattr(value, "canonical_items", None) - if isinstance(items, (bytes, bytearray, memoryview)): - view = memoryview(items) - for offset in range(0, len(view), chunk_chars): - yield bytes(view[offset : offset + chunk_chars]) - return - pages = getattr(value, "pages", None) - if isinstance(pages, (list, tuple)) and pages and all( - isinstance(getattr(page, "canonical_json_items", None), (bytes, bytearray, memoryview)) - for page in pages - ): - needs_separator = False - for page in pages: - page_items = memoryview(page.canonical_json_items) - if not len(page_items): - continue - if needs_separator: - yield b"," - for offset in range(0, len(page_items), chunk_chars): - yield bytes(page_items[offset : offset + chunk_chars]) - needs_separator = True - return - yield from iter_json_array_text_chunks( - value, - batch_records=batch_records, - chunk_chars=chunk_chars, - ) - - -def _iter_build_input_chunks( - payload: Mapping[str, object], - *, - array_keys: Iterable[str] = PRISM_BUILDER_INPUT_ARRAY_KEYS, - batch_records: int = SHARE_JSON_BATCH_RECORDS, - chunk_chars: int = SHARE_JSON_CHUNK_BYTES, -) -> Iterator[str | bytes]: - """The one-shot builder input in bounded chunks, every array streamed. - - Byte-identical to ``json.dumps(payload, separators=(",", ":"))`` for a - payload of plain lists. :func:`iter_json_object_text_chunks` streams - only ``list``/``tuple`` members; here every member named in - ``array_keys`` streams whatever replayable sequence it holds (each - ``json.dumps`` covers at most ``batch_records`` records), the ``shares`` - member additionally splices pre-encoded window bytes when the sequence - carries them, and the remaining members (found block, policy, script - suffix, settlement configuration) are fixed-size and take one - ``json.dumps`` each. Chunks are ``str`` except spliced window bytes, - which are yielded as ``bytes`` (#255). - """ - keys = list(payload) - if any(not isinstance(key, str) for key in keys): - yield json.dumps(dict(payload), separators=(",", ":")) - return - streamed = set(array_keys) - pending: list[str] = [] - pending_chars = 0 - - def flush() -> Iterator[str]: - nonlocal pending, pending_chars - if pending: - chunk = "".join(pending) - pending = [] - pending_chars = 0 - yield chunk - - def push(text: str) -> Iterator[str]: - nonlocal pending_chars - if not text: - return - pending.append(text) - pending_chars += len(text) - if pending_chars >= chunk_chars: - yield from flush() - - yield from push("{") - first = True - for key in keys: - value = payload[key] - prefix = ("" if first else ",") + json.dumps(key) + ":" - first = False - if ( - key in streamed - and not isinstance(value, (str, bytes, bytearray, Mapping)) - and isinstance(value, Iterable) - ): - yield from push(prefix + "[") - pieces: Iterable[str | bytes] - if key == "shares": - pieces = _iter_share_window_items( - value, - batch_records=batch_records, - chunk_chars=chunk_chars, - ) - else: - pieces = iter_json_array_text_chunks( - value, - batch_records=batch_records, - chunk_chars=chunk_chars, - ) - for piece in pieces: - if isinstance(piece, (bytes, bytearray)): - yield from flush() - yield bytes(piece) - else: - yield from push(piece) - yield from push("]") - else: - yield from push(prefix + json.dumps(value, separators=(",", ":"))) - yield from push("}") - yield from flush() - - -def _iter_prepare_window_request_chunks( - request_fields: dict[str, object], -) -> Iterator[bytes]: - """One prepare_window request line in bounded chunks, newline included. - - Byte-identical to ``json.dumps(request_fields, separators=(",", ":"))`` - plus the terminating newline: the envelope members are each one small - ``json.dumps`` and the ``records`` array is encoded batch by batch. - """ - for chunk in iter_json_object_text_chunks( - request_fields, - array_keys=("records",), - ): - yield chunk.encode("utf-8") - yield b"\n" - - -def _share_window_spool_file() -> Any: - """Anonymous spool file for the serialized share-window payload tail. - - Created in the same temporary filesystem the builder's captured output - and stderr already use, and unlinked from birth: a crashed coordinator - can never strand a multi-megabyte window on disk, and closing the last - descriptor is the entire cleanup story. - """ - return tempfile.TemporaryFile() - - -@dataclass -class _ShareWindowSerialization: - """Derived share-window forms cached per canonical window. - - For a fixed (canonical digest, count, window weight) the serialized share - window is immutable: its canonical digest and the - audit-builder compact payload fragments are pure functions of the - artifact's share snapshot. Recomputing them inside every template-bump - build serialized the whole window under the GIL each time. The compact - fragments are derived lazily by the first audit-builder invocation -- - embedders that replace the builder never require the compact schema. - - The fragments' invariant byte encoding is additionally spooled to an - anonymous temp file once per key, so every build after the first can feed - the audit builder's stdin through kernel transfers instead of re-writing - megabytes of unchanged JSON through Python. Spool access is leased: - rotation to a new generation retires the spool, and the descriptor closes - when the last in-flight transfer releases it, never underneath one. - """ - - key: tuple[str, int, int] - share_count: int - share_snapshot_sha256: str - _source_artifact: Any = field( - default=None, - repr=False, - compare=False, - ) - _compact_lock: threading.Lock = field( - default_factory=threading.Lock, - repr=False, - ) - # The audit-builder payload tail (the compact identities and shares - # members plus the closing brace) as bounded ASCII chunks, encoded once - # per generation; see compact_tail_chunks. - _compact_tail_chunks: tuple[str, ...] | None = field( - default=None, - repr=False, - ) - _compact_tail_chars: int = field(default=0, repr=False) - _spool_lock: threading.Lock = field( - default_factory=threading.Lock, - repr=False, - ) - _spool_file: Any = field(default=None, repr=False) - _spool_size: int = field(default=0, repr=False) - _spool_failed: bool = field(default=False, repr=False) - _spool_retired: bool = field(default=False, repr=False) - _spool_leases: int = field(default=0, repr=False) - # Call-time-resolved spool-file factory. The coordinator wires a lambda - # that resolves its own module global so the historical - # ``prism_coordinator._share_window_spool_file`` patch seam keeps working. - _spool_factory: Callable[[], Any] | None = field( - default=None, - repr=False, - compare=False, - ) - - def compact_tail_chunks( - self, - shares: list[dict[str, object]], - ) -> tuple[str, ...]: - """Encoded payload tail chunks, computed once per generation. - - Concurrent builders block here instead of duplicating the encode; the - window is immutable for this key, so first-writer-wins is exact. - Every chunk is ASCII (``json.dumps``' ``ensure_ascii`` default), so - its character count is its byte count on every transport, and no - chunk splits a record. - """ - with self._compact_lock: - if self._compact_tail_chunks is None: - chunks = _compact_share_tail_chunks(shares) - self._compact_tail_chars = sum(len(chunk) for chunk in chunks) - self._compact_tail_chunks = chunks - return self._compact_tail_chunks - - @property - def compact_json_bytes(self) -> int: - """Resident bytes of the encoded tail; 0 until its first use.""" - return int(self._compact_tail_chars) - - def acquire_spooled_tail( - self, - shares: list[dict[str, object]], - ) -> tuple[Any, int] | None: - """Lease the spooled payload tail, writing it on first use. - - Returns the open spool file and its byte size, or None when spooling - is unavailable (creation failed earlier, a transfer poisoned it, or - the generation was retired). The caller must pair a successful lease - with release_spooled_tail once its transfer is finished. - """ - with self._spool_lock: - if self._spool_failed or self._spool_retired: - return None - if self._spool_file is None: - chunks = self.compact_tail_chunks(shares) - spool = None - spool_factory = self._spool_factory or _share_window_spool_file - try: - spool = spool_factory() - for chunk in chunks: - spool.write(chunk.encode("utf-8")) - spool.flush() - size = spool.seek(0, os.SEEK_END) - except (OSError, ValueError): - # Spooling is an optimization; the in-memory pipe path - # remains authoritative. Failure is sticky so a broken - # temp filesystem is not retried on every build. - self._spool_failed = True - if spool is not None: - try: - spool.close() - except OSError: - pass - return None - self._spool_file = spool - self._spool_size = int(size) - self._spool_leases += 1 - return self._spool_file, self._spool_size - - def release_spooled_tail(self) -> None: - with self._spool_lock: - self._spool_leases -= 1 - self._close_spool_if_unused_locked() - - def mark_spool_failed(self) -> None: - """Poison the spool after a transfer-side failure (still leased).""" - with self._spool_lock: - self._spool_failed = True - - def retire_spool(self) -> None: - """Stop new leases and close the file once in-flight ones finish.""" - with self._spool_lock: - self._spool_retired = True - self._close_spool_if_unused_locked() - - def _close_spool_if_unused_locked(self) -> None: - if ( - self._spool_leases > 0 - or self._spool_file is None - or not (self._spool_retired or self._spool_failed) - ): - return - spool, self._spool_file = self._spool_file, None - self._spool_size = 0 - try: - spool.close() - except OSError: - pass - - -class _PipeReadinessWaiter: - """Readiness wait for one non-blocking pipe end of a builder process. - - The transport loops drive ``os.read``/``os.write``/``os.splice`` on - non-blocking descriptors so that cancellation, supersession and the - absolute deadline are checked between syscalls. They used to sleep a - fixed 20 ms after every EAGAIN, charging one sleep per drained pipe - capacity (64 KiB on Linux): a multi-megabyte window exchange then paid - orders of magnitude more in idle waits than in copying (#236). A waiter - blocks in the platform selector instead, bounded by BOTH the remaining - deadline and the cancellation-check cadence, so a waiting build wakes as - soon as bytes arrive or drain and still re-checks its control signals on - the old schedule when nothing happens. - - One instance serves one transport operation. Registration is lazy -- an - operation satisfied from buffered bytes or by syscalls that never block - costs no selector -- and ``close`` releases the registration on every - exit path. A descriptor the selector refuses (a regular file standing - in for the pipe, a closed or fake descriptor) degrades to the historical - bounded sleep so the retry loop keeps working; so does one the selector - registers but cannot poll (``SelectSelector`` over a Windows pipe - handle, a descriptor torn down under the registration): the selector is - released once, the descriptor stays unwatchable for the rest of the - operation, and the sleep takes only whatever budget the failed poll - left. Readiness reported by the selector may be spurious and EINTR is - never a wait: both are resolved by the caller retrying the syscall after - its control checks. - """ - - __slots__ = ("_events", "_file_descriptor", "_selector", "_unwatchable", "waits") - - def __init__(self, file_descriptor: int, events: int) -> None: - self._file_descriptor = file_descriptor - self._events = events - self._selector: selectors.BaseSelector | None = None - self._unwatchable = False - # Blocked waits taken by this operation; read by tests and the - # transport benchmark, never by production metrics. - self.waits = 0 - - def __enter__(self) -> _PipeReadinessWaiter: - return self - - def __exit__(self, *exc_info: object) -> None: - self.close() - - def wait(self, deadline: float) -> bool: - """Block until ready, the cadence slice ends, or the deadline passes. - - Returns True when readiness was reported (possibly spuriously) and - False when the slice or the deadline elapsed first. It never raises - for an exhausted deadline: the calling loop owns the ordering of its - cancellation, supersession and deadline checks and performs them - before the next syscall. - """ - self.waits += 1 - now = time.monotonic() - timeout = min(PRISM_BUILDER_PIPE_WAIT_SLICE_SECONDS, deadline - now) - if timeout <= 0: - return False - # This wait may consume no more than one slice, however it ends. - slice_deadline = min(now + timeout, deadline) - selector = self._selector - if selector is None and not self._unwatchable: - try: - selector = selectors.DefaultSelector() - try: - selector.register(self._file_descriptor, self._events) - except BaseException: - selector.close() - raise - except (OSError, ValueError, KeyError): - selector = None - self._unwatchable = True - else: - self._selector = selector - if selector is not None: - try: - # select() retries EINTR itself with the remaining timeout - # (PEP 475); a signal handler that raises propagates as is. - return bool(selector.select(timeout)) - except InterruptedError: - # Not a readiness verdict and not a fault: the caller - # re-checks its control signals and retries the syscall. - return False - except (OSError, ValueError): - # Registered, but the selector cannot poll this descriptor. - # Release it exactly once, remember that for the rest of the - # operation, and finish THIS wait with the bounded sleep - # over whatever budget the failed poll left -- never a fresh - # full slice, never past the absolute deadline. - self._detach_selector() - timeout = slice_deadline - time.monotonic() - if timeout <= 0: - return False - time.sleep(timeout) - return False - - def _detach_selector(self) -> None: - self._unwatchable = True - selector, self._selector = self._selector, None - if selector is not None: - try: - selector.close() - except OSError: - pass - - def close(self) -> None: - selector, self._selector = self._selector, None - if selector is not None: - selector.close() - - -class _ServeBuilderUnavailable(RuntimeError): - """Daemon anomaly; the build must transparently use one-shot mode.""" - - -@dataclass(frozen=True) -class PreparedWindowOutcome: - """One prepare_window round trip's typed result. - - ``status`` is one of: - - - ``"prepared"`` -- the daemon holds the folded window; the digest, - record count, advance stats, and the canonical-items byte payload - (full stream for ``full`` mode, prefix-drop plus appended suffix for - ``advance``) are populated. - - ``"needs_full"`` -- the daemon does not hold ``base_digest`` (respawn, - eviction, epoch change); the caller re-sends a full preparation. - Ordinary control flow, never a daemon anomaly. - - ``"fallback"`` -- an advance invariant failed daemon-side; the caller - takes exactly its IncrementalWindowFallback path (clear the cache, - full rescan). Ordinary control flow, never a daemon anomaly. - - ``"fold_invalid"`` -- the full snapshot itself violates the fold's - invariants; the in-process oracle raises the same rejection, so the - caller falls through to it for the authoritative error. - - ``"out_of_range"`` -- the request carries an integer outside the - daemon's declared widths (``share_seq`` u64, ``share_difficulty`` and - ``network_difficulty`` u128, ``template_height`` u64, ``ntime`` u32, - millisecond stamps i64, ``window_weight`` u128) or a difficulty total - that leaves u128. The daemon is healthy, keeps its state and stays - registered; this materialization degrades to the in-process pipeline, - whose unbounded integers fold the window exactly. Ordinary control - flow, never a daemon anomaly -- distinct from a malformed request, - which does retire the daemon. - - ``"busy"`` -- another build owns the daemon and it could not be had - within this preparation's budget. The daemon is healthy and stays - registered; this materialization degrades to the in-process pipeline. - Ordinary control flow, never a daemon anomaly. - - ``rejection`` is populated for ``fallback`` and ``fold_invalid``: the - daemon's stable, machine-readable category for the refused condition - (the parity oracle's vocabulary), never to be inferred from ``error``, - whose text is diagnostic and free to drift. - """ - - status: str - share_snapshot_sha256: str | None = None - record_count: int = 0 - added_rows: int = 0 - expired_rows: int = 0 - touched_pages: int = 0 - window_items: bytes | None = field(default=None, repr=False) - retained_drop_bytes: int = 0 - appended_items: bytes = field(default=b"", repr=False) - error: str | None = None - rejection: str | None = None - - -@dataclass -class _ServeBuilderClient: - """Line-oriented I/O state for one long-lived --serve audit builder.""" - - process: subprocess.Popen[bytes] - stdout_buffer: bytearray = field(default_factory=bytearray, repr=False) - # True from the first byte of a request until its terminating newline. - # A request that escapes half-written leaves the daemon's stdin holding - # an unterminated line, and the NEXT request would concatenate onto it, - # so the daemon must be retired rather than reused. - request_incomplete: bool = False - # Mirrors the daemon's bounded LRU so requests can predict whether the - # window must ride along. Divergence is repaired by the daemon's - # needs_window response, never trusted blindly. - uploaded_windows: OrderedDict[str, None] = field( - default_factory=OrderedDict, - repr=False, - ) - - def note_uploaded_window(self, share_snapshot_sha256: str) -> None: - self.uploaded_windows.pop(share_snapshot_sha256, None) - self.uploaded_windows[share_snapshot_sha256] = None - while len(self.uploaded_windows) > PRISM_SERVE_BUILDER_WINDOW_CACHE_ENTRIES: - self.uploaded_windows.popitem(last=False) - - def close(self) -> None: - # Tolerates lightweight process fakes used by embedders and tests, - # which do not necessarily expose the full Popen surface. - process = self.process - poll = getattr(process, "poll", None) - wait = getattr(process, "wait", None) - if callable(poll): - if poll() is None: - try: - process.kill() - except (ProcessLookupError, OSError, AttributeError): - pass - if callable(wait): - try: - wait(timeout=5.0) - except ( - subprocess.TimeoutExpired, - OSError, - TypeError, - ValueError, - AttributeError, - ): - pass - for stream in ( - getattr(process, "stdin", None), - getattr(process, "stdout", None), - ): - if stream is None or not hasattr(stream, "close"): - continue - try: - stream.close() - except OSError: - pass - - -class BundleCompiler: - """Compile summaries or canonical bundles with exact cancellation rules. - - Owns the persistent ``--serve`` daemon state and its metrics; every - scheduler counter, refresh metric, and payload input is reached through - the runtime port so the coordinator remains the single owner of that - state and its monkeypatch seams. - """ - - def __init__( - self, - runtime: BundleCompilerRuntime, - *, - superseded_error: Callable[[str], BaseException], - cancellation_error_types: tuple[type[BaseException], ...], - build_control_type: type, - tool_command: Callable[[str], list[str]] | None = None, - ) -> None: - self._runtime = runtime - self._superseded_error = superseded_error - self._cancellation_error_types = tuple(cancellation_error_types) - self._build_control_type = build_control_type - # Call-time-resolved builder-command factory: the coordinator wires a - # lambda over its own module global so the historical - # ``prism_coordinator.prism_tool_command`` patch seam keeps working. - self._tool_command = tool_command or prism_tool_command - # Serializes daemon ownership per request. Contended builds do - # not queue behind the daemon; they take the one-shot path. - self._serve_builder_lock = threading.Lock() - self._serve_builder: _ServeBuilderClient | None = None - self._serve_builder_shutdown = False - # Counters get their own short-lived lock so a metrics scrape - # never waits behind _serve_builder_lock, which one request can - # hold for a whole daemon round trip. - self._serve_builder_metrics_lock = threading.Lock() - # Guarded by _serve_builder_metrics_lock. - self.serve_builder_counts = { - "requests": 0, - "fallbacks": 0, - "spawns": 0, - "window_uploads": 0, - "window_prepares": 0, - "window_prepare_contended": 0, - "window_prepare_out_of_range": 0, - } - # Guarded by _serve_builder_metrics_lock. - self.serve_builder_window_cache_counts = {"hits": 0, "misses": 0} - - @contextmanager - def _routed_window_mirror_divergence(self) -> Iterator[None]: - """Report a refuted daemon window mirror, then let the error stand. - - A mirror reconciles its bytes, digest and record count when it is - built, so nothing here should ever fire; if it does, the build that - touched the mirror is not the place the problem started, and it must - not be the only place the problem is visible either. Invalidating - the cached window keeps the NEXT build off the same bytes, and the - counted event turns a lone traceback into something a scrape sees. - """ - try: - yield - except DaemonWindowMirrorDivergence: - note = getattr( - self._runtime, - "_note_window_mirror_divergence", - None, - ) - if callable(note): - note() - raise - - def shutdown_serve_builder(self) -> None: - """Retire the persistent audit builder; builds revert to one-shot.""" - self._runtime._ensure_job_cache_state() - with self._serve_builder_lock: - self._serve_builder_shutdown = True - client, self._serve_builder = self._serve_builder, None - if client is not None: - client.close() - - def _retire_serve_builder_locked(self) -> None: - client, self._serve_builder = self._serve_builder, None - if client is not None: - client.close() - - def _record_live_serve_builder_termination( - self, - client: _ServeBuilderClient | None, - ) -> None: - """Account for deliberately killing a still-running daemon. - - Crash paths count themselves when the EOF surfaces; a live worker - retired for a timeout, malformed response, protocol mismatch, or - spool failure must land in the termination counters so its - replacement reads as a restart, mirroring the cancellation path. - """ - runtime = self._runtime - if client is None: - return - poll = getattr(client.process, "poll", None) - if not callable(poll) or poll() is not None: - return - with runtime._job_build_scheduler_lock: - runtime.job_build_worker_counts["terminations"] += 1 - runtime._job_build_worker_restart_pending = True - - def _observe_builder_phase_metrics(self, metrics: dict[str, Any]) -> None: - """Apply one build's builder-side phase timings to refresh metrics. - - Shared by the one-shot stderr metrics line and the --serve response - payload so serialization_copy and the builder phase histograms stay - comparable regardless of the transport that ran the build. - """ - runtime = self._runtime - phase_seconds = metrics.get("phases_seconds", {}) - if isinstance(phase_seconds, dict): - for phase in ( - "payout_state_derivation", - "ctv_manifest_construction", - "coinbase_bundle_construction", - "signing_verification", - ): - elapsed = phase_seconds.get(phase) - if isinstance(elapsed, (int, float)): - runtime._observe_tip_refresh_build_phase( - phase, - float(elapsed), - ) - rust_serialization = sum( - float(metrics.get(name, 0.0)) - for name in ( - "input_deserialization_seconds", - "output_serialization_seconds", - ) - ) - runtime._observe_tip_refresh_build_phase( - "serialization_copy", - rust_serialization, - ) - - def _spawn_serve_builder_locked( - self, - deadline: float, - cancellation: CancellationPort | None, - ) -> _ServeBuilderClient: - runtime = self._runtime - command = self._tool_command("qbit-prism-build-audit-bundle") + [ - "--serve", - "--signing-key-seed-hex", - runtime.signing_seed_hex, - "--ledger-signing-key-seed-hex", - runtime.ledger_attestation_signing_seed_hex, - ] - try: - process = subprocess.Popen( - command, - stdin=subprocess.PIPE, - stdout=subprocess.PIPE, - # The daemon inherits the coordinator's stderr so a crash's - # diagnostics land in the journal instead of a per-request - # capture file that dies with the request. - stderr=None, - close_fds=True, - ) - except OSError as exc: - raise _ServeBuilderUnavailable( - f"audit-builder daemon spawn failed: {exc}" - ) from exc - # The start (and any pending restart it satisfies) is recorded as - # soon as the worker exists, exactly like the one-shot path: a - # handshake-phase death or protocol mismatch must not leave the - # lifecycle totals describing fewer workers than were launched. - restarted = False - with runtime._job_build_scheduler_lock: - if runtime._job_build_worker_restart_pending: - runtime.job_build_worker_counts["restarts"] += 1 - runtime._job_build_worker_restart_pending = False - restarted = True - runtime.job_build_worker_counts["starts"] += 1 - if restarted: - with runtime._tip_refresh_metrics_lock: - runtime.tip_refresh_worker_restarts += 1 - client = _ServeBuilderClient(process=process) - try: - assert process.stdin is not None and process.stdout is not None - os.set_blocking(process.stdin.fileno(), False) - os.set_blocking(process.stdout.fileno(), False) - handshake_line = self._serve_builder_read_line( - client, - deadline, - cancellation, - None, - ) - handshake = json.loads(handshake_line) - if ( - not isinstance(handshake, dict) - or handshake.get("event") != "handshake" - or handshake.get("protocol") - != PRISM_SERVE_BUILDER_PROTOCOL_VERSION - ): - raise _ServeBuilderUnavailable( - "audit-builder daemon announced an unsupported protocol" - ) - except _ServeBuilderUnavailable: - self._record_live_serve_builder_termination(client) - client.close() - raise - except (OSError, ValueError, AttributeError) as exc: - self._record_live_serve_builder_termination(client) - client.close() - raise _ServeBuilderUnavailable( - f"audit-builder daemon handshake failed: {exc}" - ) from exc - except BaseException: - self._record_live_serve_builder_termination(client) - client.close() - raise - return client - - def _serve_builder_read_line( - self, - client: _ServeBuilderClient, - deadline: float, - cancellation: CancellationPort | None, - build_control: BundleBuildControlPort | None, - ) -> bytes: - runtime = self._runtime - stdout = client.process.stdout - assert stdout is not None - file_descriptor = stdout.fileno() - # Bytes before this offset were already scanned for the newline; the - # buffer only grows until the line is cut, so a long response line - # is scanned once rather than once per received chunk. - scanned = 0 - with _PipeReadinessWaiter(file_descriptor, selectors.EVENT_READ) as waiter: - while True: - newline_index = client.stdout_buffer.find(b"\n", scanned) - if newline_index >= 0: - line = bytes(client.stdout_buffer[:newline_index]) - del client.stdout_buffer[: newline_index + 1] - return line - scanned = len(client.stdout_buffer) - if cancellation is not None: - cancellation.raise_if_cancelled("serve builder response") - if ( - build_control is not None - and build_control.cancel_event.is_set() - ): - raise self._superseded_error( - "audit-builder daemon request was canceled after supersession" - ) - if time.monotonic() >= deadline: - # Not counted as a worker failure here: the fallback - # one-shot runs against this same exhausted deadline and - # its own timeout path records the failure exactly once. - raise _ServeBuilderUnavailable( - "audit-builder daemon timed out" - ) - try: - chunk = os.read( - file_descriptor, - PRISM_BUILDER_PIPE_READ_CHUNK_BYTES, - ) - except InterruptedError: - # Not a wait: retried straight after the control checks. - continue - except BlockingIOError: - waiter.wait(deadline) - continue - except OSError as exc: - raise _ServeBuilderUnavailable( - f"audit-builder daemon read failed: {exc}" - ) from exc - if not chunk: - if ( - build_control is not None - and build_control.cancel_event.is_set() - ): - raise self._superseded_error( - "audit-builder daemon was terminated after supersession" - ) - with runtime._job_build_scheduler_lock: - runtime.job_build_worker_counts["crashes"] += 1 - runtime._job_build_worker_restart_pending = True - raise _ServeBuilderUnavailable( - "audit-builder daemon exited mid-request" - ) - client.stdout_buffer += chunk - - def _serve_builder_read_exact( - self, - client: _ServeBuilderClient, - byte_count: int, - deadline: float, - cancellation: CancellationPort | None, - build_control: BundleBuildControlPort | None, - ) -> bytes: - """Read exactly byte_count raw bytes following a response line. - - prepare_window responses append a raw canonical-items section after - the JSON envelope; the section length rides in the envelope, so this - drains the line reader's buffer first and then reads from the pipe - with the same deadline/cancellation discipline as line reads. - """ - runtime = self._runtime - stdout = client.process.stdout - assert stdout is not None - file_descriptor = stdout.fileno() - out = bytearray() - if client.stdout_buffer: - take = min(byte_count, len(client.stdout_buffer)) - out += client.stdout_buffer[:take] - del client.stdout_buffer[:take] - with _PipeReadinessWaiter(file_descriptor, selectors.EVENT_READ) as waiter: - while len(out) < byte_count: - if cancellation is not None: - cancellation.raise_if_cancelled("serve builder response") - if ( - build_control is not None - and build_control.cancel_event.is_set() - ): - raise self._superseded_error( - "audit-builder daemon request was canceled after supersession" - ) - if time.monotonic() >= deadline: - raise _ServeBuilderUnavailable( - "audit-builder daemon timed out" - ) - try: - chunk = os.read( - file_descriptor, - min( - PRISM_BUILDER_PIPE_READ_CHUNK_BYTES, - byte_count - len(out), - ), - ) - except InterruptedError: - continue - except BlockingIOError: - waiter.wait(deadline) - continue - except OSError as exc: - raise _ServeBuilderUnavailable( - f"audit-builder daemon read failed: {exc}" - ) from exc - if not chunk: - with runtime._job_build_scheduler_lock: - runtime.job_build_worker_counts["crashes"] += 1 - runtime._job_build_worker_restart_pending = True - raise _ServeBuilderUnavailable( - "audit-builder daemon exited mid-request" - ) - out += chunk - return bytes(out) - - def _serve_builder_write( - self, - client: _ServeBuilderClient, - data: bytes, - deadline: float, - cancellation: CancellationPort | None, - build_control: BundleBuildControlPort | None, - ) -> int: - stdin = client.process.stdin - assert stdin is not None - file_descriptor = stdin.fileno() - remaining = memoryview(data) - written_total = 0 - with _PipeReadinessWaiter(file_descriptor, selectors.EVENT_WRITE) as waiter: - while remaining: - if cancellation is not None: - cancellation.raise_if_cancelled("serve builder request") - if ( - build_control is not None - and build_control.cancel_event.is_set() - ): - raise self._superseded_error( - "audit-builder daemon request was canceled after supersession" - ) - if time.monotonic() >= deadline: - # Not counted as a worker failure here: the fallback - # one-shot runs against this same exhausted deadline and - # its own timeout path records the failure exactly once. - raise _ServeBuilderUnavailable( - "audit-builder daemon timed out" - ) - try: - written = os.write(file_descriptor, remaining) - except InterruptedError: - continue - except BlockingIOError: - waiter.wait(deadline) - continue - except BrokenPipeError as exc: - if ( - build_control is not None - and build_control.cancel_event.is_set() - ): - raise self._superseded_error( - "audit-builder daemon was terminated after supersession" - ) from exc - raise _ServeBuilderUnavailable( - "audit-builder daemon input pipe closed" - ) from exc - except OSError as exc: - raise _ServeBuilderUnavailable( - f"audit-builder daemon write failed: {exc}" - ) from exc - if written <= 0: - raise _ServeBuilderUnavailable( - "audit-builder daemon input pipe closed" - ) - written_total += written - remaining = remaining[written:] - return written_total - - def _serve_builder_splice_spool( - self, - client: _ServeBuilderClient, - share_serialization: _ShareWindowSerialization, - spool_file: Any, - spool_size: int, - deadline: float, - cancellation: CancellationPort | None, - build_control: BundleBuildControlPort | None, - ) -> int: - stdin = client.process.stdin - assert stdin is not None - stdin_fd = stdin.fileno() - spool_fd = spool_file.fileno() - offset = 0 - # EAGAIN here always means the pipe is full (the spool is a regular - # file), so the wait is for the daemon's stdin to drain. A splice - # that raised moved nothing -- offset_src is explicit -- so retrying - # from the same offset never repeats a byte, and a splice that moved - # part of its request advances the offset by exactly that much. - with _PipeReadinessWaiter(stdin_fd, selectors.EVENT_WRITE) as waiter: - while offset < spool_size: - if cancellation is not None: - cancellation.raise_if_cancelled("serve builder request") - if ( - build_control is not None - and build_control.cancel_event.is_set() - ): - raise self._superseded_error( - "audit-builder daemon request was canceled after supersession" - ) - if time.monotonic() >= deadline: - # Not counted as a worker failure here: the fallback - # one-shot runs against this same exhausted deadline and - # its own timeout path records the failure exactly once. - raise _ServeBuilderUnavailable( - "audit-builder daemon timed out" - ) - try: - moved = os.splice( - spool_fd, - stdin_fd, - min(PRISM_SPOOL_SPLICE_CHUNK_BYTES, spool_size - offset), - offset_src=offset, - ) - except InterruptedError: - continue - except BlockingIOError: - waiter.wait(deadline) - continue - except BrokenPipeError as exc: - # The daemon's stdin closed; the spool itself is fine. - if ( - build_control is not None - and build_control.cancel_event.is_set() - ): - raise self._superseded_error( - "audit-builder daemon was terminated after supersession" - ) from exc - raise _ServeBuilderUnavailable( - "audit-builder daemon input pipe closed" - ) from exc - except OSError as exc: - if ( - build_control is not None - and build_control.cancel_event.is_set() - ): - raise self._superseded_error( - "audit-builder daemon was terminated after supersession" - ) from exc - # The spool itself cannot stream. Poison it so the - # one-shot fallback -- and every later build -- writes - # the cached in-memory fragments instead of retrying the - # same transfer and failing mid-stream. Whatever prefix - # already reached the daemon is never re-sent: the - # anomaly retires this daemon with the request marked - # incomplete. - share_serialization.mark_spool_failed() - raise _ServeBuilderUnavailable( - f"audit-builder daemon spool transfer failed: {exc}" - ) from exc - if moved <= 0: - raise _ServeBuilderUnavailable( - "audit-builder daemon input pipe closed" - ) - offset += moved - return offset - - def _serve_builder_request_locked( - self, - client: _ServeBuilderClient, - *, - deadline: float, - payload: dict[str, object], - shares: list[dict[str, object]], - precomposed: Callable[[], tuple[str, ...]], - share_serialization: _ShareWindowSerialization, - cancellation: CancellationPort | None, - build_control: BundleBuildControlPort | None, - record_phase_metrics: bool, - append_invalidation_epoch: int | None = None, - ) -> dict[str, Any]: - """One JSONL round trip; raises _ServeBuilderUnavailable on anomaly.""" - runtime = self._runtime - share_snapshot_sha256 = share_serialization.share_snapshot_sha256 - request_fields = dict(payload) - request_fields["window_key"] = { - "share_snapshot_sha256": share_snapshot_sha256, - } - if append_invalidation_epoch is not None: - # Lets the daemon drop prepared window state from an older - # append-invalidation epoch; see the prepare_window protocol. - request_fields["append_invalidation_epoch"] = int( - append_invalidation_epoch - ) - prefix = json.dumps(request_fields, separators=(",", ":")).encode( - "utf-8" - ) - phases = runtime._job_build_phases() - input_bytes = 0 - input_serialization_elapsed = 0.0 - - def send_request(upload_window: bool) -> None: - nonlocal input_bytes, input_serialization_elapsed - serialization_started = time.monotonic() - client.request_incomplete = True - try: - if not upload_window: - input_bytes += self._serve_builder_write( - client, - prefix + b"\n", - deadline, - cancellation, - build_control, - ) - client.request_incomplete = False - return - input_bytes += self._serve_builder_write( - client, - prefix[:-1], - deadline, - cancellation, - build_control, - ) - with self._routed_window_mirror_divergence(): - lease = ( - share_serialization.acquire_spooled_tail(shares) - if hasattr(os, "splice") - else None - ) - if lease is not None: - try: - spool_file, spool_size = lease - input_bytes += self._serve_builder_splice_spool( - client, - share_serialization, - spool_file, - spool_size, - deadline, - cancellation, - build_control, - ) - finally: - share_serialization.release_spooled_tail() - else: - # Spooling unavailable: the cached tail streams from - # memory chunk by chunk, each one bounded, so this - # fallback never re-encodes the window whole either. - for chunk in precomposed(): - input_bytes += self._serve_builder_write( - client, - chunk.encode("utf-8"), - deadline, - cancellation, - build_control, - ) - input_bytes += self._serve_builder_write( - client, - b"\n", - deadline, - cancellation, - build_control, - ) - client.request_incomplete = False - finally: - elapsed = time.monotonic() - serialization_started - phases["input_serialization"] = ( - phases.get("input_serialization", 0.0) + elapsed - ) - # Accumulated across a possible needs_window re-send and - # observed once per build so serialization_copy stays - # comparable with the one-shot transport. - input_serialization_elapsed += elapsed - - def read_response() -> tuple[dict[str, Any], bytes]: - worker_started = time.monotonic() - line = self._serve_builder_read_line( - client, - deadline, - cancellation, - build_control, - ) - phases["worker"] = phases.get("worker", 0.0) + ( - time.monotonic() - worker_started - ) - output_started = time.monotonic() - try: - value = json.loads(line) - except ValueError as exc: - raise _ServeBuilderUnavailable( - f"audit-builder daemon response was malformed: {exc}" - ) from exc - finally: - phases["output_serialization"] = phases.get( - "output_serialization", - 0.0, - ) + (time.monotonic() - output_started) - if not isinstance(value, dict): - raise _ServeBuilderUnavailable( - "audit-builder daemon response was not an object" - ) - return value, line - - upload_window = share_snapshot_sha256 not in client.uploaded_windows - send_request(upload_window) - response, response_line = read_response() - if ( - response.get("ok") is not True - and bool(response.get("needs_window")) - and not upload_window - ): - # The daemon evicted this window (respawn or generation churn the - # coordinator's mirror missed); repeat the request with the - # window riding along. - upload_window = True - send_request(True) - response, response_line = read_response() - if response.get("ok") is not True: - raise _ServeBuilderUnavailable( - "audit-builder daemon error: " - f"{response.get('error', 'unknown')}" - ) - summary = response.get("summary") - if not isinstance(summary, dict): - raise _ServeBuilderUnavailable( - "audit-builder daemon returned a malformed summary" - ) - # Every successful response promotes the window in the mirror, hits - # included: the daemon moves a hit entry to most-recent position, and - # a mirror that only tracked uploads would evict a different key and - # either re-upload a still-cached window or bounce on needs_window. - client.note_uploaded_window(share_snapshot_sha256) - if upload_window: - with self._serve_builder_metrics_lock: - self.serve_builder_counts["window_uploads"] += 1 - window_cache = response.get("window_cache") - if isinstance(window_cache, dict): - outcome = "hits" if bool(window_cache.get("hit")) else "misses" - with self._serve_builder_metrics_lock: - self.serve_builder_window_cache_counts[outcome] += 1 - if record_phase_metrics: - runtime._observe_tip_refresh_build_phase( - "serialization_copy", - input_serialization_elapsed, - ) - metrics_value = response.get("metrics") - if isinstance(metrics_value, dict): - try: - self._observe_builder_phase_metrics(metrics_value) - except (TypeError, ValueError): - # Metrics are diagnostic only. A malformed timing payload - # must never invalidate an otherwise valid summary. - pass - runtime._record_tip_refresh_ipc_bytes("input", input_bytes) - runtime._record_tip_refresh_ipc_bytes("output", len(response_line)) - return summary - - def _build_audit_bundle_via_serve_builder( - self, - *, - payload: dict[str, object], - shares: list[dict[str, object]], - precomposed: Callable[[], tuple[str, ...]], - share_serialization: _ShareWindowSerialization, - cancellation: CancellationPort | None, - record_phase_metrics: bool, - deadline: float, - append_invalidation_epoch: int | None = None, - ) -> dict[str, Any] | None: - """Build through the persistent daemon; None means use one-shot. - - Any daemon anomaly -- spawn failure, handshake or protocol mismatch, - crash, timeout, malformed response -- retires the daemon and returns - None so the caller's one-shot path runs unchanged against the same - deadline (whatever budget the daemon consumed stays consumed). - Cancellation and supersession raise exactly like the one-shot path - instead of falling back. - """ - runtime = self._runtime - if not env_bool("PRISM_BUILDER_SERVE", "1"): - return None - runtime._ensure_job_cache_state() - if not getattr(runtime, "signing_seed_hex", None) or not getattr( - runtime, - "ledger_attestation_signing_seed_hex", - None, - ): - return None - if not self._serve_builder_lock.acquire(blocking=False): - # A concurrent build owns the daemon. One-shot is cheaper than - # queueing behind a multi-second build. - return None - try: - if self._serve_builder_shutdown: - return None - build_control = getattr( - runtime._job_build_phase_local, - "bundle_build_control", - None, - ) - if not isinstance(build_control, self._build_control_type): - build_control = None - client: _ServeBuilderClient | None = None - try: - client = self._serve_builder - if client is not None and client.process.poll() is not None: - # The daemon died between requests (its own crash or an - # external kill). Record the lifecycle transition before - # retiring it so the immediate respawn below reads as a - # crash-and-restart rather than an ordinary start. - with runtime._job_build_scheduler_lock: - runtime.job_build_worker_counts["crashes"] += 1 - runtime._job_build_worker_restart_pending = True - self._retire_serve_builder_locked() - client = None - if client is None: - client = self._spawn_serve_builder_locked( - deadline, - cancellation, - ) - self._serve_builder = client - with self._serve_builder_metrics_lock: - self.serve_builder_counts["spawns"] += 1 - if build_control is not None: - runtime._register_job_bundle_process( - build_control, - client.process, # type: ignore[arg-type] - ) - try: - summary = self._serve_builder_request_locked( - client, - deadline=deadline, - payload=payload, - shares=shares, - precomposed=precomposed, - share_serialization=share_serialization, - cancellation=cancellation, - build_control=build_control, - record_phase_metrics=record_phase_metrics, - append_invalidation_epoch=append_invalidation_epoch, - ) - finally: - if build_control is not None: - runtime._unregister_job_bundle_process( - build_control, - client.process, - ) - except self._cancellation_error_types: - # The daemon stream is indeterminate mid-request; retire it - # so the replacement build starts clean. - self._retire_serve_builder_locked() - with runtime._job_build_scheduler_lock: - runtime.job_build_worker_counts["terminations"] += 1 - runtime._job_build_worker_restart_pending = True - raise - except _ServeBuilderUnavailable: - self._record_live_serve_builder_termination( - self._serve_builder - ) - self._retire_serve_builder_locked() - with self._serve_builder_metrics_lock: - self.serve_builder_counts["fallbacks"] += 1 - return None - except (OSError, ValueError): - self._record_live_serve_builder_termination( - self._serve_builder - ) - self._retire_serve_builder_locked() - with self._serve_builder_metrics_lock: - self.serve_builder_counts["fallbacks"] += 1 - return None - except BaseException: - # Anything else -- a payload the serializer refused, a - # divergence raised while composing the window fragments -- - # is not a daemon anomaly, so it propagates. But if it - # escaped a half-written request, the daemon's stdin now - # holds an unterminated line and the next request would - # concatenate onto it, so the daemon goes with the error - # rather than answering "malformed serve request" to some - # innocent later build. - if client is not None and client.request_incomplete: - self._record_live_serve_builder_termination(client) - self._retire_serve_builder_locked() - with self._serve_builder_metrics_lock: - self.serve_builder_counts["fallbacks"] += 1 - raise - else: - with self._serve_builder_metrics_lock: - self.serve_builder_counts["requests"] += 1 - return summary - finally: - self._serve_builder_lock.release() - - def prepare_payout_window( - self, - *, - mode: str, - records_json: list[dict[str, object]], - anchor_job_issued_at_ms: int, - append_invalidation_epoch: int, - window_weight: int | None = None, - page_size: int | None = None, - base_digest: str | None = None, - wait_for_daemon: bool = True, - ) -> PreparedWindowOutcome | None: - """One prepare_window round trip; None means use in-process Python. - - Called once per window generation from artifact materialization, - never per build. Any daemon anomaly -- spawn failure, handshake or - protocol mismatch, crash, timeout, malformed response -- retires the - daemon and returns None so the caller's in-process pipeline runs - unchanged. The ``needs_full``/``fallback``/``fold_invalid``/ - ``out_of_range`` outcomes are ordinary control flow and never retire - the daemon; ``out_of_range`` in particular used to arrive as a - malformed-request error and retire a healthy daemon once per - materialization for as long as the window held the value. - - ``wait_for_daemon=False`` refuses to queue at all: urgent - materializations (the accepted-block preview) must never wait behind - another build's round trip, so a contended daemon answers ``busy`` - immediately. Either way, losing the race for the daemon is not a - daemon fault and is never recorded as one. - """ - runtime = self._runtime - if not env_bool("PRISM_BUILDER_SERVE", "1"): - return None - runtime._ensure_job_cache_state() - if not getattr(runtime, "signing_seed_hex", None) or not getattr( - runtime, - "ledger_attestation_signing_seed_hex", - None, - ): - return None - request_fields: dict[str, object] = { - "request": "prepare_window", - "mode": mode, - "append_invalidation_epoch": int(append_invalidation_epoch), - "anchor_job_issued_at_ms": int(anchor_job_issued_at_ms), - "records": records_json, - } - if window_weight is not None: - request_fields["window_weight"] = int(window_weight) - if page_size is not None: - request_fields["page_size"] = int(page_size) - if base_digest is not None: - request_fields["base_digest"] = base_digest - budget = float( - getattr( - runtime, - "bundle_build_timeout_seconds", - DEFAULT_PRISM_BUNDLE_BUILD_TIMEOUT_SECONDS, - ) - ) - # Unlike builds (which prefer one-shot over queueing), a routine - # window preparation has no cheaper transport to defect to, so it - # will wait for the daemon -- but only for its own budget, and the - # exchange's deadline is stamped AFTER the wait. Charging a build's - # runtime against this preparation's budget would make the first - # write inside the lock read as a daemon timeout, and the anomaly - # handler would SIGKILL a daemon that had done nothing wrong. - if wait_for_daemon: - acquired = self._serve_builder_lock.acquire(timeout=budget) - else: - acquired = self._serve_builder_lock.acquire(blocking=False) - if not acquired: - with self._serve_builder_metrics_lock: - self.serve_builder_counts["window_prepare_contended"] = ( - self.serve_builder_counts.get( - "window_prepare_contended", - 0, - ) - + 1 - ) - return PreparedWindowOutcome( - status="busy", - error="audit-builder daemon is owned by another build", - ) - deadline = time.monotonic() + budget - try: - if self._serve_builder_shutdown: - return None - try: - client = self._serve_builder - if client is not None and client.process.poll() is not None: - with runtime._job_build_scheduler_lock: - runtime.job_build_worker_counts["crashes"] += 1 - runtime._job_build_worker_restart_pending = True - self._retire_serve_builder_locked() - client = None - if client is None: - client = self._spawn_serve_builder_locked(deadline, None) - self._serve_builder = client - with self._serve_builder_metrics_lock: - self.serve_builder_counts["spawns"] += 1 - # The request streams in bounded chunks: each envelope - # member is one small json.dumps and the record array is - # encoded batch by batch, so a 200k-record preparation never - # holds the GIL for one whole-window encode (#236). Same - # bytes as one json.dumps of the fields. From the first byte - # until the terminating newline the daemon's stdin holds a - # partial line; anything that escapes in between retires - # the daemon below rather than letting the next request - # concatenate onto the fragment. - client.request_incomplete = True - for chunk in _iter_prepare_window_request_chunks( - request_fields - ): - self._serve_builder_write( - client, - chunk, - deadline, - None, - None, - ) - client.request_incomplete = False - envelope_line = self._serve_builder_read_line( - client, - deadline, - None, - None, - ) - envelope = json.loads(envelope_line) - if ( - not isinstance(envelope, dict) - or envelope.get("request") != "prepare_window" - ): - raise _ServeBuilderUnavailable( - "audit-builder daemon returned a malformed" - " prepare_window response" - ) - if envelope.get("ok") is not True: - rejection = envelope.get("rejection") - if not isinstance(rejection, str): - rejection = None - if bool(envelope.get("needs_full")): - outcome = PreparedWindowOutcome( - status="needs_full", - error=str(envelope.get("error", "")), - ) - elif bool(envelope.get("fallback")): - outcome = PreparedWindowOutcome( - status="fallback", - error=str(envelope.get("error", "")), - rejection=rejection, - ) - elif bool(envelope.get("fold_invalid")): - outcome = PreparedWindowOutcome( - status="fold_invalid", - error=str(envelope.get("error", "")), - rejection=rejection, - ) - elif bool(envelope.get("out_of_range")): - # The daemon declined a value outside its declared - # widths. It is healthy and keeps its state; count - # the decline so the degraded materializations are - # visible, and let the in-process fold take over. - with self._serve_builder_metrics_lock: - self.serve_builder_counts[ - "window_prepare_out_of_range" - ] = ( - self.serve_builder_counts.get( - "window_prepare_out_of_range", - 0, - ) - + 1 - ) - outcome = PreparedWindowOutcome( - status="out_of_range", - error=str(envelope.get("error", "")), - ) - else: - raise _ServeBuilderUnavailable( - "audit-builder daemon prepare_window error: " - f"{envelope.get('error', 'unknown')}" - ) - else: - digest = envelope.get("share_snapshot_sha256") - if not isinstance(digest, str) or not digest: - raise _ServeBuilderUnavailable( - "audit-builder daemon prepare_window response" - " carries no digest" - ) - window_items: bytes | None = None - retained_drop_bytes = 0 - appended_items = b"" - if "window_items_len" in envelope: - window_items = self._serve_builder_read_exact( - client, - int(envelope["window_items_len"]), - deadline, - None, - None, - ) - else: - retained_drop_bytes = int( - envelope["retained_drop_bytes"] - ) - appended_items = self._serve_builder_read_exact( - client, - int(envelope["appended_items_len"]), - deadline, - None, - None, - ) - # The raw section's terminating newline. - self._serve_builder_read_exact( - client, - 1, - deadline, - None, - None, - ) - # The prepared window doubles as the build cache entry; - # note it in the LRU mirror so the next build for this - # digest skips its window upload entirely. - client.note_uploaded_window(digest) - outcome = PreparedWindowOutcome( - status="prepared", - share_snapshot_sha256=digest, - record_count=int(envelope.get("record_count", 0)), - added_rows=int(envelope.get("added_rows", 0)), - expired_rows=int(envelope.get("expired_rows", 0)), - touched_pages=int(envelope.get("touched_pages", 0)), - window_items=window_items, - retained_drop_bytes=retained_drop_bytes, - appended_items=appended_items, - ) - except _ServeBuilderUnavailable: - self._record_live_serve_builder_termination( - self._serve_builder - ) - self._retire_serve_builder_locked() - with self._serve_builder_metrics_lock: - self.serve_builder_counts["fallbacks"] += 1 - return None - except (OSError, ValueError, KeyError, TypeError): - self._record_live_serve_builder_termination( - self._serve_builder - ) - self._retire_serve_builder_locked() - with self._serve_builder_metrics_lock: - self.serve_builder_counts["fallbacks"] += 1 - return None - except BaseException: - # Not a daemon anomaly, so it propagates -- but if it - # escaped a half-written request the daemon's stdin holds - # an unterminated line, and the daemon goes with the error - # rather than answering "malformed serve request" to some - # innocent later build. - client = self._serve_builder - if client is not None and client.request_incomplete: - self._record_live_serve_builder_termination(client) - self._retire_serve_builder_locked() - with self._serve_builder_metrics_lock: - self.serve_builder_counts["fallbacks"] += 1 - raise - else: - with self._serve_builder_metrics_lock: - self.serve_builder_counts["window_prepares"] = ( - self.serve_builder_counts.get("window_prepares", 0) + 1 - ) - return outcome - finally: - self._serve_builder_lock.release() - - def build_audit_bundle( - self, - *, - shares: list[dict[str, object]], - found_block: dict[str, object], - prior_balances: list[dict[str, object]], - coinbase_script_sig_suffix_hex: str, - witness_merkle_leaves_hex: list[str] | None = None, - ctv_fee_parent_hash: str | None = None, - canonical_output_path: Path | None = None, - canonical_output_parent_fd: int | None = None, - canonical_output_adopter: Callable[[Path, os.stat_result], None] | None = None, - summary_only: bool = False, - payout_policy: dict[str, object] | None = None, - ctv_settlement: dict[str, object] | None = None, - cancellation: CancellationPort | None = None, - share_serialization: _ShareWindowSerialization | None = None, - append_invalidation_epoch: int | None = None, - ) -> dict[str, Any]: - runtime = self._runtime - runtime._ensure_job_cache_state() - runtime._ensure_tip_refresh_state() - if cancellation is not None: - runtime._job_build_checkpoint("serialization", cancellation) - payload: dict[str, object] = { - "found_block": found_block, - "prior_balances": prior_balances, - "payout_policy": ( - runtime.prism_payout_policy() - if payout_policy is None - else payout_policy - ), - "coinbase_script_sig_suffix_hex": coinbase_script_sig_suffix_hex, - "witness_merkle_leaves_hex": witness_merkle_leaves_hex or [], - } - job_build_phase_local = getattr(runtime, "_job_build_phase_local", None) - record_phase_metrics = bool( - getattr(job_build_phase_local, "tip_refresh_metrics", False) - ) - precomposed: Callable[[], tuple[str, ...]] | None = None - if summary_only: - artifact_started = time.monotonic() - if ( - share_serialization is not None - and share_serialization.share_count == len(shares) - ): - # The compact conversion and its encoded fragments are pure - # per-generation functions of the share window, resolved - # lazily: a daemon build that hits prepared/uploaded window - # state never sends the window, so walking the shares (which - # may be a lazily-parsed daemon mirror) is deferred until a - # transport actually needs the compact bytes. - serialization = share_serialization - - def precompose() -> tuple[str, ...]: - with self._routed_window_mirror_divergence(): - return serialization.compact_tail_chunks(shares) - - precomposed = precompose - else: - with self._routed_window_mirror_divergence(): - identities, compact_shares = _compact_share_payload(shares) - payload["compact_share_identities"] = identities - payload["compact_shares"] = compact_shares - if record_phase_metrics: - runtime._observe_tip_refresh_build_phase( - "serialization_copy", - time.monotonic() - artifact_started, - ) - else: - # Canonical builds serialize the full share window inline. Any - # replayable sequence -- a list, the daemon mirror, a page-backed - # window -- is walked record by record while the builder input - # is written; it is never copied into a list first (#255). - if isinstance(shares, (str, bytes, bytearray)) or not isinstance( - shares, - (Sequence, Iterable), - ): - raise TypeError("shares must be a sequence of share records") - payload["shares"] = shares - if ctv_settlement is None and payout_policy is None: - ctv_settlement = runtime.prism_ctv_settlement_config( - block_height=int(found_block["block_height"]), - parent_hash=ctv_fee_parent_hash, - ) - if ctv_settlement is not None: - payload["ctv_settlement"] = ctv_settlement - if canonical_output_path is not None and summary_only: - raise ValueError("canonical output and job summary output are mutually exclusive") - # One deadline covers the whole build regardless of transport: a - # daemon anomaly falls back to the one-shot subprocess with only the - # remaining budget, so a hung daemon cannot double the configured - # limit while progress health already treats one limit as stuck. - build_deadline = time.monotonic() + float( - getattr( - runtime, - "bundle_build_timeout_seconds", - DEFAULT_PRISM_BUNDLE_BUILD_TIMEOUT_SECONDS, - ) - ) - if ( - summary_only - and canonical_output_path is None - and precomposed is not None - and share_serialization is not None - ): - # The persistent builder serves the artifact-backed summary path, - # whose window identity and cached fragments it can key on. Every - # anomaly falls back to the one-shot subprocess below. - served = self._build_audit_bundle_via_serve_builder( - payload=payload, - shares=shares, - precomposed=precomposed, - share_serialization=share_serialization, - cancellation=cancellation, - record_phase_metrics=record_phase_metrics, - deadline=build_deadline, - append_invalidation_epoch=append_invalidation_epoch, - ) - if served is not None: - return served - command = self._tool_command("qbit-prism-build-audit-bundle") + [ - "--input", - "-", - "--signing-key-seed-hex", - runtime.signing_seed_hex, - "--ledger-signing-key-seed-hex", - runtime.ledger_attestation_signing_seed_hex, - ] - command.append("--job-summary-output" if summary_only else "--canonical-output") - if record_phase_metrics: - command.append("--phase-metrics") - if ( - canonical_output_path is not None - and canonical_output_parent_fd is None - ): - canonical_output_path.parent.mkdir(parents=True, exist_ok=True) - succeeded = False - created_output = False - created_output_fd: int | None = None - try: - with ExitStack() as stack: - if canonical_output_path is None: - output = stack.enter_context( - tempfile.TemporaryFile(mode="w+", encoding="utf-8") - ) - else: - if canonical_output_parent_fd is None: - output = stack.enter_context( - canonical_output_path.open("x+", encoding="utf-8") - ) - else: - output_fd = os.open( - canonical_output_path.name, - os.O_RDWR - | os.O_CREAT - | os.O_EXCL - | getattr(os, "O_NOFOLLOW", 0), - 0o666, - dir_fd=canonical_output_parent_fd, - ) - output = stack.enter_context( - os.fdopen(output_fd, "w+", encoding="utf-8") - ) - created_output = True - # Keep the created inode allocated until cleanup compares - # it with the published name. This closes the Linux inode- - # reuse window after an adopter replaces the path. - created_output_fd = os.dup(output.fileno()) - stderr = stack.enter_context( - tempfile.TemporaryFile(mode="w+", encoding="utf-8") - ) - process = subprocess.Popen( - command, - stdin=subprocess.PIPE, - stdout=output, - stderr=stderr, - text=True, - encoding="utf-8", - close_fds=True, - ) - build_control = getattr( - job_build_phase_local, - "bundle_build_control", - None, - ) - if isinstance(build_control, self._build_control_type): - runtime._register_job_bundle_process(build_control, process) - with runtime._job_build_scheduler_lock: - if runtime._job_build_worker_restart_pending: - runtime.job_build_worker_counts["restarts"] += 1 - runtime._job_build_worker_restart_pending = False - runtime.job_build_worker_counts["starts"] += 1 - assert process.stdin is not None - input_byte_count = 0 - worker_deadline = build_deadline - killed_process_wait_seconds = max( - 0.001, - float( - getattr( - runtime, - "job_build_cancel_grace_seconds", - DEFAULT_PRISM_JOB_BUILD_CANCEL_GRACE_SECONDS, - ) - ), - ) - coordinator = runtime - build_control_type = self._build_control_type - superseded_error = self._superseded_error - - class _CancelableInput: - def __init__(self, stream: Any) -> None: - self.stream = stream - self.waiter: _PipeReadinessWaiter | None = None - try: - file_descriptor = int(stream.fileno()) - except (AttributeError, OSError, TypeError, ValueError): - # Lightweight process fakes used by embedders and - # tests do not necessarily expose an OS pipe. - self.file_descriptor: int | None = None - else: - os.set_blocking(file_descriptor, False) - self.file_descriptor = file_descriptor - self.waiter = _PipeReadinessWaiter( - file_descriptor, - selectors.EVENT_WRITE, - ) - - def wait_writable(self) -> None: - """Block until the child's stdin can drain. - - Bounded by the build deadline and the cancellation - cadence exactly like the daemon transport; the - caller re-runs check_cancelled before retrying. - """ - if self.waiter is not None: - self.waiter.wait(worker_deadline) - - def close(self) -> None: - waiter, self.waiter = self.waiter, None - if waiter is not None: - waiter.close() - - def check_cancelled(self) -> None: - if cancellation is not None: - cancellation.raise_if_cancelled( - "builder input serialization" - ) - if ( - isinstance(build_control, build_control_type) - and build_control.cancel_event.is_set() - ): - raise superseded_error( - "audit-builder input was canceled after supersession" - ) - if time.monotonic() >= worker_deadline: - with coordinator._tip_refresh_metrics_lock: - coordinator.tip_refresh_worker_failures += 1 - raise RuntimeError( - "qbit-prism-build-audit-bundle timed out" - ) - - def write(self, value: str | bytes) -> int: - nonlocal input_byte_count - self.check_cancelled() - if self.file_descriptor is None: - text = ( - value.decode("utf-8") - if isinstance(value, (bytes, bytearray)) - else value - ) - written = int(self.stream.write(text)) - input_byte_count += len(text[:written].encode("utf-8")) - return written - encoded = ( - bytes(value) - if isinstance(value, (bytes, bytearray)) - else value.encode("utf-8") - ) - remaining = memoryview(encoded) - while remaining: - self.check_cancelled() - try: - written = os.write( - self.file_descriptor, - remaining, - ) - except InterruptedError: - continue - except BlockingIOError: - self.wait_writable() - continue - if written <= 0: - raise BrokenPipeError( - "audit-builder input pipe closed" - ) - input_byte_count += written - remaining = remaining[written:] - return len(value) - - serialization_started = time.monotonic() - spool_lease: tuple[Any, int] | None = None - sink: _CancelableInput | None = None - try: - sink = _CancelableInput(process.stdin) - - def write_precomposed_tail() -> None: - assert precomposed is not None - for chunk in precomposed(): - sink.write(chunk) - - if ( - precomposed is not None - and share_serialization is not None - and sink.file_descriptor is not None - and hasattr(os, "splice") - ): - spool_lease = share_serialization.acquire_spooled_tail( - shares - ) - if precomposed is not None: - # Per-build fields are encoded fresh; the dominant - # share-window fragments stream from the spool file - # written once per generation (kernel moves the bytes - # into the pipe), falling back to the cached strings - # whenever no spool is available. - prefix = json.dumps(payload, separators=(",", ":")) - sink.write(prefix[:-1]) - spool_transferred = False - if spool_lease is not None: - spool_file, spool_size = spool_lease - spool_fd = spool_file.fileno() - offset = 0 - while offset < spool_size: - sink.check_cancelled() - try: - moved = os.splice( - spool_fd, - sink.file_descriptor, - min( - PRISM_SPOOL_SPLICE_CHUNK_BYTES, - spool_size - offset, - ), - offset_src=offset, - ) - except InterruptedError: - continue - except BlockingIOError: - sink.wait_writable() - continue - except BrokenPipeError: - # The child's stdin closed; the spool - # itself is fine and the surrounding - # pipe-error handling owns this. - raise - except OSError: - # The spool cannot stream (unsupported - # filesystem or an I/O error); the cached - # in-memory fragments stay authoritative - # for this and every later build. - share_serialization.mark_spool_failed() - if offset: - # Part of the tail already reached the - # pipe; this build cannot be repaired - # by re-writing it from memory. - raise - break - if moved <= 0: - raise BrokenPipeError( - "audit-builder input pipe closed" - ) - offset += moved - input_byte_count += moved - spool_transferred = ( - spool_size > 0 and offset >= spool_size - ) - if not spool_transferred: - write_precomposed_tail() - else: - # The share arrays (the canonical build's full - # window, or an artifact-less summary's compact - # form) stream in bounded batches and every other - # member is one small json.dumps: the same bytes as - # one json.dump of the payload, without the - # pure-Python encoder's per-token writes or a - # whole-window C call. A daemon-mirror window is - # walked here, so a refuted mirror is routed exactly - # as the retired list() copy was. - with self._routed_window_mirror_divergence(): - for chunk in _iter_build_input_chunks(payload): - sink.write(chunk) - except BrokenPipeError: - # Prefer the builder's diagnostic below. - pass - except BaseException as exc: - try: - process.kill() - except ProcessLookupError: - pass - try: - process.wait(timeout=killed_process_wait_seconds) - except subprocess.TimeoutExpired: - pass - if isinstance(exc, self._cancellation_error_types): - with runtime._job_build_scheduler_lock: - runtime.job_build_worker_counts["terminations"] += 1 - runtime._job_build_worker_restart_pending = True - raise - finally: - if sink is not None: - sink.close() - if spool_lease is not None and share_serialization is not None: - share_serialization.release_spooled_tail() - phases = runtime._job_build_phases() - phases["input_serialization"] = phases.get( - "input_serialization", - 0.0, - ) + (time.monotonic() - serialization_started) - try: - process.stdin.close() - except (BlockingIOError, BrokenPipeError): - pass - if record_phase_metrics: - runtime._observe_tip_refresh_build_phase( - "serialization_copy", - time.monotonic() - serialization_started, - ) - runtime._record_tip_refresh_ipc_bytes("input", input_byte_count) - worker_started = time.monotonic() - terminated = False - returncode: int | None = None - if cancellation is None and not hasattr(process, "poll"): - returncode = process.wait( - timeout=max(0.001, worker_deadline - time.monotonic()) - ) - else: - while returncode is None: - returncode = process.poll() - if returncode is not None: - break - cancelled_by_control = ( - isinstance(build_control, self._build_control_type) - and build_control.cancel_event.is_set() - ) - cancelled_by_request = ( - cancellation is not None and cancellation.is_set() - ) - if cancelled_by_control or cancelled_by_request: - terminated = True - process.terminate() - try: - returncode = process.wait( - timeout=max( - 0.0, - float( - getattr( - runtime, - "job_build_cancel_grace_seconds", - DEFAULT_PRISM_JOB_BUILD_CANCEL_GRACE_SECONDS, - ) - ), - ) - ) - except subprocess.TimeoutExpired: - process.kill() - try: - returncode = process.wait( - timeout=killed_process_wait_seconds - ) - except subprocess.TimeoutExpired: - returncode = process.poll() - with runtime._job_build_scheduler_lock: - runtime.job_build_worker_counts["terminations"] += 1 - runtime._job_build_worker_restart_pending = True - break - if time.monotonic() >= worker_deadline: - process.kill() - try: - returncode = process.wait( - timeout=killed_process_wait_seconds - ) - except subprocess.TimeoutExpired: - returncode = process.poll() - with runtime._tip_refresh_metrics_lock: - runtime.tip_refresh_worker_failures += 1 - raise RuntimeError( - "qbit-prism-build-audit-bundle timed out" - ) - time.sleep(min(0.02, PRISM_TIP_REFRESH_ADMISSION_POLL_SECONDS)) - phases["worker"] = phases.get("worker", 0.0) + ( - time.monotonic() - worker_started - ) - if terminated and cancellation is not None: - if cancellation.is_set(): - cancellation.raise_if_cancelled("builder worker") - if ( - terminated - and isinstance(build_control, self._build_control_type) - and build_control.cancel_event.is_set() - ): - raise self._superseded_error( - "audit-builder subprocess was canceled after supersession" - ) - stderr.seek(0) - error_text = stderr.read() - if returncode != 0: - with runtime._job_build_scheduler_lock: - runtime.job_build_worker_counts["crashes"] += 1 - runtime._job_build_worker_restart_pending = True - if record_phase_metrics: - with runtime._tip_refresh_metrics_lock: - runtime.tip_refresh_worker_failures += 1 - raise RuntimeError( - f"qbit-prism-build-audit-bundle failed: {error_text}" - ) - if ( - isinstance(build_control, self._build_control_type) - and build_control.cancel_event.is_set() - ): - raise self._superseded_error( - "audit-builder result completed after supersession" - ) - output.flush() - output_size = os.fstat(output.fileno()).st_size - if record_phase_metrics: - runtime._record_tip_refresh_ipc_bytes("output", output_size) - for line in error_text.splitlines(): - if not line.startswith(PRISM_BUILDER_PHASE_METRICS_PREFIX): - continue - raw_metrics = line.removeprefix( - PRISM_BUILDER_PHASE_METRICS_PREFIX - ) - try: - metrics = json.loads(raw_metrics) - if isinstance(metrics, dict): - self._observe_builder_phase_metrics(metrics) - except (TypeError, ValueError, json.JSONDecodeError): - # Metrics are diagnostic only. A malformed timing - # line must never invalidate an otherwise valid - # signed bundle. - pass - if canonical_output_path is not None: - os.fsync(output.fileno()) - output.seek(0) - output_started = time.monotonic() - if cancellation is not None: - cancellation.raise_if_cancelled("builder output serialization") - bundle: Any - if canonical_output_path is not None: - # The canonical artifact on disk is the authority. It is - # scanned once through bounded windows into a view whose - # window-sized arrays (``shares`` and the reward - # manifest's counted shares) stay on disk and replay on - # demand; nothing window-sized is decoded into Python - # objects in this process (#255). The descriptor is - # duplicated so the view outlives this stack; the file - # may be unlinked later without invalidating the view. - scan_checkpoint: Callable[[], None] | None = None - if cancellation is not None: - scan_checkpoint = lambda: cancellation.raise_if_cancelled( # noqa: E731 - "builder output scan" - ) - bundle = CanonicalAuditBundleView.scan( - os.dup(output.fileno()), - path=canonical_output_path, - cancellation=scan_checkpoint, - ) - scan_stats = bundle.scan_stats - if scan_stats.oversized_values: - # One JSON value is the decoder's unit, so a record - # above the soft limit was decoded whole. Valid work - # is never refused for its size; the event is made - # visible instead. - print( - "prism coordinator: canonical audit artifact " - f"scan decoded {scan_stats.oversized_values} " - "oversized JSON value(s) " - f"max_value_chars={scan_stats.max_value_chars} " - "window_high_water_chars=" - f"{scan_stats.window_high_water_chars} " - f"path={canonical_output_path.name}", - flush=True, - ) - else: - # Embedders without a candidate artifact receive the - # decoded document, as before; the finalization path - # always names an artifact. - bundle = json.load(output) - phases["output_serialization"] = phases.get( - "output_serialization", - 0.0, - ) + (time.monotonic() - output_started) - try: - if cancellation is not None: - cancellation.raise_if_cancelled("builder verification") - if ( - canonical_output_path is not None - and canonical_output_adopter is not None - ): - canonical_output_adopter( - canonical_output_path, - os.fstat(output.fileno()), - ) - except BaseException: - close_view = getattr(bundle, "close", None) - if callable(close_view): - close_view() - raise - succeeded = True - return bundle - finally: - try: - if ( - canonical_output_path is not None - and created_output - and not succeeded - ): - created_output_identity = ( - None - if created_output_fd is None - else os.fstat(created_output_fd) - ) - self._remove_created_output_if_same( - canonical_output_path, - created_output_identity, - parent_fd=canonical_output_parent_fd, - ) - finally: - if created_output_fd is not None: - os.close(created_output_fd) - - @staticmethod - def _remove_created_output_if_same( - path: Path, - identity: os.stat_result | None, - *, - parent_fd: int | None = None, - ) -> None: - if identity is None: - return - try: - current = ( - path.lstat() - if parent_fd is None - else os.stat(path.name, dir_fd=parent_fd, follow_symlinks=False) - ) - except FileNotFoundError: - return - if not stat.S_ISREG(current.st_mode): - # Cleanup has no portable no-replace restore primitive for a - # directory/symlink. Reject it before moving an unowned path. - return - if not BundleCompiler._output_identity_matches(identity, current): - # Preserve a replacement already visible at the canonical name. - return - quarantine = path.with_name(f".{path.name}.{uuid.uuid4().hex}.cleanup") - try: - if parent_fd is None: - os.replace(path, quarantine) - else: - os.replace( - path.name, - quarantine.name, - src_dir_fd=parent_fd, - dst_dir_fd=parent_fd, - ) - except FileNotFoundError: - return - moved = ( - quarantine.lstat() - if parent_fd is None - else os.stat( - quarantine.name, - dir_fd=parent_fd, - follow_symlinks=False, - ) - ) - if BundleCompiler._output_identity_matches(identity, moved): - if parent_fd is None: - quarantine.unlink() - else: - os.unlink(quarantine.name, dir_fd=parent_fd) - return - try: - if parent_fd is None: - os.link(quarantine, path, follow_symlinks=False) - else: - os.link( - quarantine.name, - path.name, - src_dir_fd=parent_fd, - dst_dir_fd=parent_fd, - follow_symlinks=False, - ) - except OSError: - # A second replacement won after quarantine. Keep both artifacts; - # an unowned non-regular replacement is likewise preserved there. - return - if parent_fd is None: - quarantine.unlink() - else: - os.unlink(quarantine.name, dir_fd=parent_fd) - - @staticmethod - def _output_identity_matches( - identity: os.stat_result, - value: os.stat_result, - ) -> bool: - return bool( - value.st_dev == identity.st_dev - and value.st_ino == identity.st_ino - and value.st_mode == identity.st_mode - and value.st_size == identity.st_size - and value.st_mtime_ns == identity.st_mtime_ns - and stat.S_ISREG(value.st_mode) - ) - - -__all__ = [ - "BundleBuildControlPort", - "BundleCompiler", - "BundleCompilerRuntime", - "CancellationPort", - "PreparedWindowOutcome", - "PRISM_BUILDER_INPUT_ARRAY_KEYS", - "PRISM_BUILDER_PHASE_METRICS_PREFIX", - "PRISM_SERVE_BUILDER_PROTOCOL_VERSION", - "PRISM_SERVE_BUILDER_WINDOW_CACHE_ENTRIES", - "PRISM_SPOOL_SPLICE_CHUNK_BYTES", - "PRISM_TIP_REFRESH_ADMISSION_POLL_SECONDS", - "_ServeBuilderClient", - "_ServeBuilderUnavailable", - "_ShareWindowSerialization", - "_iter_build_input_chunks", - "_iter_share_window_items", - "_compact_share_payload", - "_compact_share_tail_chunks", - "_iter_prepare_window_request_chunks", - "_share_window_spool_file", - "canonical_bundle_bytes", -] diff --git a/lab/prism/candidate_codec.py b/lab/prism/candidate_codec.py deleted file mode 100644 index a171c760a..000000000 --- a/lab/prism/candidate_codec.py +++ /dev/null @@ -1,2238 +0,0 @@ -#!/usr/bin/env python3 -"""Bounded codec and immutable body handles for durable block candidates. - -Issue #255. A block-candidate intent carries the whole payout window under -``shares_json`` (375,000 shares and ~232 MB of JSON at the stress size), and -the historical path materialized it three times in the lease-bearing -coordinator: ``list(context.shares_json)``, a whole ``json.dumps`` to -validate it, and a second whole ``json.dumps`` to inline it as a ``jsonb`` -literal. Each of those is one C call that holds the GIL for the duration -while the writer-lease monitor waits to run. - -This module replaces the whole-body calls with a streaming encoder whose -individual C calls are bounded (at most :data:`CODEC_BATCH_RECORDS` records -or roughly :data:`CODEC_BATCH_TARGET_BYTES` of output per ``json.dumps``; -strings over :data:`CODEC_STRING_SLICE_CHARS` code points are escaped in -slices) and hands the result to storage as fixed -:data:`CANDIDATE_BODY_CHUNK_BYTES` byte chunks with per-chunk digests. - -Nothing the encoder keeps scales with the body. It emits three kinds of -events -- chunks, per-field *spans* (one per large top-level field) and -*page* entries (one per bounded batch of a large array) -- and retains only -a scalar :class:`CandidateBodyManifest`. A reader gets the same facts back -from storage in bounded pages and keeps the page index on disk next to the -spool file, bisecting it with ``pread`` instead of holding a list. - -Byte identity is the contract. The body bytes are exactly the historical -v1 identity JSON -- ``json.dumps(block_candidate_identity(intent), -sort_keys=True, separators=(",", ":"))`` with the pending acknowledgment -stamp neutralized -- so ``candidate_sha256`` is unchanged for every existing -row and every existing oracle. Chunk boundaries, spans, pages and the -storage version never enter that digest. - -Three body shapes exist: - -* :class:`SourceCandidateBody` re-encodes on demand from the immutable - share sequence the live job pinned (a page-backed window, wave B's lazy - daemon sequence, or a plain list). It retains no encoded copy. -* :class:`SpoolCandidateBody` is a durable replay body hydrated to a local - spool file in bounded pages; :class:`SpoolJsonArraySequence` decodes one - bounded page per ``json.loads`` from it and digests the share array - without decoding it at all. -* :class:`PreparedCandidateIntent` wraps either one behind the historical - mapping interface (``intent["shares_json"]`` is the immutable sequence, - never a list copy) so dict-facing callers keep working. - -This module is a leaf: standard library only. -""" - -from __future__ import annotations - -import hashlib -import io -import json -import os -import re -import struct -import threading -import weakref -from collections.abc import Iterable, Iterator, Mapping, MutableMapping, Sequence -from dataclasses import dataclass, replace -from typing import Any, Callable - -# Storage format version of a chunked body. Distinct from the v1 mining -# intent schema string (``qbit.prism.block-candidate-intent.v1``), which is -# neither renamed nor re-hashed. -CANDIDATE_BODY_STORAGE_VERSION = 2 -LEGACY_CANDIDATE_STORAGE_VERSION = 1 -# Fixed byte size of every body chunk but the last. Storage, transport and -# the reader all enforce it independently. -CANDIDATE_BODY_CHUNK_BYTES = 256 * 1024 -# Records per ``json.dumps`` call on the fast path, and the output size the -# adaptive batch aims for. -CODEC_BATCH_RECORDS = 256 -CODEC_BATCH_TARGET_BYTES = 256 * 1024 -# Output of one batch above which its items are re-encoded one by one -# through the incremental path (a giant field inside a record). -CODEC_BATCH_OVERSIZED_BYTES = 4 * 1024 * 1024 -# Ordinary per-value ``json.dumps``/``json.loads`` input ceiling. Longer -# strings are escaped and decoded slice by slice; a top-level field whose -# encoding is larger than this becomes a *large field* with its own span. -# Nothing is rejected for size alone. -CODEC_STRING_SLICE_CHARS = 4096 -CODEC_FAST_PATH_BYTES = 64 * 1024 -# Historical reader ceilings, retained for callers that import them. The -# reader no longer decodes a page or a record whole above -# ``candidate_spool_view.SPOOL_VIEW_DECODE_BYTES``: an oversized record is a -# lazy view, and no record size is rejected. -CODEC_PAGE_DECODE_BYTES = 16 * 1024 * 1024 -CODEC_WALK_RECORD_MAX_BYTES = 256 * 1024 * 1024 -# Records decoded per page when a body carries no page index for a span. -SPOOL_WALK_BATCH_RECORDS = 256 -# Bytes read from a spool per refill while walking records. -SPOOL_READ_BYTES = 256 * 1024 -# Page entries written per index statement / read per index page. -INDEX_PAGE_ENTRIES = 256 -# Span rows read per statement. -INDEX_SPAN_ROWS = 64 - -CANDIDATE_SHARES_KEY = "shares_json" -CANDIDATE_INTENT_SCHEMA = "qbit.prism.block-candidate-intent.v1" - -_COMPACT_SEPARATORS = (",", ":") -# One on-disk page index record: body offset, record index (u64, u64). -_INDEX_RECORD = struct.Struct("<QQ") - -# Replay-header field bounds (bytes of the JSON text of each fact). The -# header is an explicit projection of a few typed facts; anything longer -# than this is not a fact the header needs and is deferred to the body. -HEADER_TEXT_FIELD_BYTES = 256 -HEADER_NUMERIC_FIELD_BYTES = 128 -HEADER_HASH_FIELD_BYTES = 64 - - -class CandidateCodecError(ValueError): - """The intent cannot be encoded as durable JSON (bad field or value).""" - - -class CandidateBodyIntegrityError(RuntimeError): - """A body's bytes do not match its manifest or its source changed.""" - - -# -------------------------------------------------------------------------- -# manifest, spans, pages, chunks -# -------------------------------------------------------------------------- - - -@dataclass(frozen=True) -class BodyChunk: - """One fixed-offset slice of a body with its own digest.""" - - ordinal: int - data: bytes - sha256: str - - @property - def length(self) -> int: - return len(self.data) - - -@dataclass(frozen=True) -class FieldSpan: - """Byte span of one large top-level field inside the body. - - ``kind`` is ``array`` (decoded page by page), ``string`` (decoded slice - by slice) or ``value`` (any other JSON value, decoded in one call bounded - by the field's own size). ``item_count`` is the array length for arrays. - ``pages_exact`` is False when the page entries are boundary *hints* - (verbatim daemon bytes scanned rather than encoded record by record); a - reader validates every hint by decoding and falls back to the walker. - """ - - field: str - kind: str - start: int - end: int - item_count: int - page_count: int - pages_exact: bool - - def to_json(self) -> dict[str, Any]: - return { - "field": self.field, - "kind": self.kind, - "start": self.start, - "end": self.end, - "item_count": self.item_count, - "page_count": self.page_count, - "pages_exact": self.pages_exact, - } - - @classmethod - def from_json(cls, payload: Mapping[str, Any]) -> FieldSpan: - span = cls( - field=str(payload["field"]), - kind=str(payload["kind"]), - start=int(payload["start"]), - end=int(payload["end"]), - item_count=int(payload.get("item_count") or 0), - page_count=int(payload.get("page_count") or 0), - pages_exact=bool(payload.get("pages_exact", True)), - ) - if span.kind not in {"array", "string", "value"}: - raise CandidateBodyIntegrityError("body span has an unknown kind") - if span.start < 0 or span.end < span.start or span.item_count < 0 or span.page_count < 0: - raise CandidateBodyIntegrityError("body span is out of range") - return span - - -@dataclass(frozen=True) -class PageEntry: - """Where one bounded page of a large array starts inside the body.""" - - field: str - ordinal: int - offset: int - record_index: int - - -@dataclass(frozen=True) -class CandidateBodyManifest: - """Scalar facts about a body; nothing here scales with its size.""" - - storage_version: int - candidate_sha256: str - byte_count: int - chunk_count: int - chunk_bytes: int - share_count: int - shares_offset: int - shares_end: int - span_count: int - page_count: int - - def to_json(self) -> dict[str, Any]: - return { - "storage_version": self.storage_version, - "candidate_sha256": self.candidate_sha256, - "byte_count": self.byte_count, - "chunk_count": self.chunk_count, - "chunk_bytes": self.chunk_bytes, - "share_count": self.share_count, - "shares_offset": self.shares_offset, - "shares_end": self.shares_end, - "span_count": self.span_count, - "page_count": self.page_count, - } - - @classmethod - def from_json(cls, payload: Mapping[str, Any]) -> CandidateBodyManifest: - try: - manifest = cls( - storage_version=int(payload["storage_version"]), - candidate_sha256=str(payload["candidate_sha256"]), - byte_count=int(payload["byte_count"]), - chunk_count=int(payload["chunk_count"]), - chunk_bytes=int(payload["chunk_bytes"]), - share_count=int(payload["share_count"]), - shares_offset=int(payload["shares_offset"]), - shares_end=int(payload["shares_end"]), - span_count=int(payload.get("span_count") or 0), - page_count=int(payload.get("page_count") or 0), - ) - except (KeyError, TypeError, ValueError) as exc: - raise CandidateBodyIntegrityError("manifest is malformed") from exc - manifest.validate() - return manifest - - def validate(self) -> None: - if self.storage_version != CANDIDATE_BODY_STORAGE_VERSION: - raise CandidateBodyIntegrityError( - f"unsupported candidate body storage version {self.storage_version}" - ) - if self.chunk_bytes <= 0 or self.chunk_bytes > CANDIDATE_BODY_CHUNK_BYTES: - raise CandidateBodyIntegrityError("manifest chunk size is out of range") - if self.byte_count < 0 or self.chunk_count < 0: - raise CandidateBodyIntegrityError("manifest counts are negative") - if self.chunk_count != chunk_ordinals_for(self.byte_count, self.chunk_bytes): - raise CandidateBodyIntegrityError( - "manifest chunk count disagrees with its byte count" - ) - # A body without a share array (a candidate-only intent, as some - # tools and tests persist) carries an empty span at offset 0. - if not (0 <= self.shares_offset <= self.shares_end <= self.byte_count): - raise CandidateBodyIntegrityError("manifest share span is out of range") - if self.shares_offset == self.shares_end and self.share_count: - raise CandidateBodyIntegrityError("manifest declares shares without a span") - if len(self.candidate_sha256) != 64: - raise CandidateBodyIntegrityError("manifest digest is malformed") - if self.span_count < 0 or self.page_count < 0: - raise CandidateBodyIntegrityError("manifest index counts are negative") - - -def chunk_ordinals_for(byte_count: int, chunk_bytes: int = CANDIDATE_BODY_CHUNK_BYTES) -> int: - return -(-int(byte_count) // int(chunk_bytes)) - - -# -------------------------------------------------------------------------- -# jsonb compatibility -# -------------------------------------------------------------------------- - -# PostgreSQL ``jsonb`` rejects two things CPython's encoder happily emits: -# the ``\\u0000`` escape (text cannot hold NUL) and an unpaired UTF-16 -# surrogate escape. The historical route inlined the intent as a jsonb -# literal, so both failed on the server before any credit; a chunked body -# never reaches a jsonb parser, so the same rejection happens here, at -# preparation. Escape-aware: a backslash run of even length before ``\u`` -# is escaped backslashes, not an escape start. -_ESCAPED_NUL_RE = re.compile(r"(?<!\\)(?:\\\\)*\\u0000") -_SURROGATE_ESCAPE_RE = re.compile(r"(?<!\\)(?:\\\\)*\\u(d[89ab][0-9a-f]{2}|d[c-f][0-9a-f]{2})") -_LOW_SURROGATE_ESCAPE_RE = re.compile(r"\\ud[c-f][0-9a-f]{2}") -_LONE_SURROGATE_RE = re.compile( - "[\ud800-\udbff](?![\udc00-\udfff])|(?<![\ud800-\udbff])[\udc00-\udfff]" -) - - -def _reject_jsonb_incompatible_text(text: str) -> None: - """Refuse encoded JSON text PostgreSQL's jsonb parser would refuse.""" - if "\\u0000" in text and _ESCAPED_NUL_RE.search(text): - raise CandidateCodecError( - "unsupported Unicode escape sequence: \\u0000 cannot be converted to text" - ) - if "\\ud" not in text: - return - expected_low_at = -1 - for match in _SURROGATE_ESCAPE_RE.finditer(text): - unit = match.group(1) - escape_start = match.end() - 6 - if unit[1] in "89ab": - if expected_low_at == escape_start: - raise CandidateCodecError("Unicode high surrogate must not follow a high surrogate") - following = text[match.end() : match.end() + 6] - if not _LOW_SURROGATE_ESCAPE_RE.fullmatch(following): - raise CandidateCodecError("Unicode high surrogate must not be alone") - expected_low_at = match.end() - elif expected_low_at == escape_start: - expected_low_at = -1 - else: - raise CandidateCodecError("Unicode low surrogate must follow a high surrogate") - - -def _reject_jsonb_incompatible_str(value: str) -> None: - """The Python-string form of :func:`_reject_jsonb_incompatible_text`.""" - if "\x00" in value: - raise CandidateCodecError( - "unsupported Unicode escape sequence: \\u0000 cannot be converted to text" - ) - if _LONE_SURROGATE_RE.search(value): - raise CandidateCodecError("unpaired UTF-16 surrogate cannot be stored as jsonb") - - -# -------------------------------------------------------------------------- -# streaming encoder -# -------------------------------------------------------------------------- - - -class _ChunkSink: - """Turns encoded text into fixed-size chunks and a running digest.""" - - __slots__ = ( - "_chunk_bytes", - "_on_chunk", - "_buffer", - "_buffered", - "offset", - "_digest", - "_ordinal", - ) - - def __init__( - self, - chunk_bytes: int, - on_chunk: Callable[[BodyChunk], None] | None, - ) -> None: - self._chunk_bytes = int(chunk_bytes) - self._on_chunk = on_chunk - self._buffer: list[bytes] = [] - self._buffered = 0 - self.offset = 0 - self._digest = hashlib.sha256() - self._ordinal = 0 - - @property - def chunk_count(self) -> int: - return self._ordinal - - def write_text(self, text: str) -> None: - if text: - self.write_bytes(text.encode("utf-8")) - - def write_bytes(self, data: bytes) -> None: - if not data: - return - self._digest.update(data) - self.offset += len(data) - # Fill the current chunk to exactly its size before emitting, and - # never re-join a remainder: a verbatim page of any size is consumed - # in one linear pass. - view = memoryview(data) - while len(view): - room = self._chunk_bytes - self._buffered - piece = view[:room] - self._buffer.append(bytes(piece)) - self._buffered += len(piece) - view = view[room:] - if self._buffered >= self._chunk_bytes: - self._emit() - - def _emit(self) -> None: - chunk_data = self._buffer[0] if len(self._buffer) == 1 else b"".join(self._buffer) - self._buffer = [] - self._buffered = 0 - chunk = BodyChunk( - ordinal=self._ordinal, - data=chunk_data, - sha256=hashlib.sha256(chunk_data).hexdigest(), - ) - self._ordinal += 1 - if self._on_chunk is not None: - self._on_chunk(chunk) - - def finish(self) -> str: - if self._buffered: - self._emit() - return self._digest.hexdigest() - - -class _IndexEvents: - """Collects span/page events for the field being encoded, bounded. - - Page entries are handed to ``on_page`` in batches of at most - :data:`INDEX_PAGE_ENTRIES`; the span is handed to ``on_span`` when the - field ends. Only the current batch is ever held. - """ - - __slots__ = ("on_span", "on_page", "_field", "_pages", "_page_ordinal", "_start", "span_count", "page_count", "_kind") - - def __init__( - self, - on_span: Callable[[FieldSpan], None] | None, - on_page: Callable[[list[PageEntry]], None] | None, - ) -> None: - self.on_span = on_span - self.on_page = on_page - self._field = "" - self._kind = "" - self._pages: list[PageEntry] = [] - self._page_ordinal = 0 - self._start = 0 - self.span_count = 0 - self.page_count = 0 - - def begin(self, field_name: str, kind: str, start: int) -> None: - self._field = field_name - self._kind = kind - self._start = start - self._pages = [] - self._page_ordinal = 0 - - def page(self, offset: int, record_index: int) -> None: - self._pages.append( - PageEntry( - field=self._field, - ordinal=self._page_ordinal, - offset=offset, - record_index=record_index, - ) - ) - self._page_ordinal += 1 - if len(self._pages) >= INDEX_PAGE_ENTRIES: - self._flush_pages() - - def _flush_pages(self) -> None: - if not self._pages: - return - if self.on_page is not None: - self.on_page(self._pages) - self.page_count += len(self._pages) - self._pages = [] - - def end(self, end: int, *, item_count: int, pages_exact: bool, emit: bool) -> None: - if not emit: - # A small field: no span, no pages -- decoded from the skeleton. - self._pages = [] - self._page_ordinal = 0 - return - self._flush_pages() - span = FieldSpan( - field=self._field, - kind=self._kind, - start=self._start, - end=end, - item_count=item_count, - page_count=self._page_ordinal, - pages_exact=pages_exact, - ) - self.span_count += 1 - if self.on_span is not None: - self.on_span(span) - - -def _dumps_scalar(value: Any) -> str: - # ``allow_nan=False`` is the deliberate difference from the historical - # ``json.dumps``: NaN/Infinity used to pass the client-side validation - # and then fail as a jsonb literal on the server, after the share was - # already inside the fenced statement. A chunked body never reaches a - # jsonb parser, so the rejection has to happen here, before credit. - text = json.dumps(value, allow_nan=False, sort_keys=True, separators=_COMPACT_SEPARATORS) - if isinstance(value, (dict, list, tuple)): - _reject_jsonb_incompatible_text(text) - return text - - -def _dumps_key(key: str) -> str: - _reject_jsonb_incompatible_str(key) - return json.dumps(key) - - -def _write_string(value: str, sink: _ChunkSink) -> None: - if len(value) <= CODEC_STRING_SLICE_CHARS: - _reject_jsonb_incompatible_str(value) - sink.write_text(json.dumps(value)) - return - # ``ensure_ascii`` escapes code point by code point (a non-BMP character - # is one code point and becomes one surrogate pair), so slicing by code - # point and escaping each slice reproduces the whole-string escape. - sink.write_text('"') - start = 0 - while start < len(value): - end = min(len(value), start + CODEC_STRING_SLICE_CHARS) - if end < len(value) and "\ud800" <= value[end - 1] <= "\udbff" and "\udc00" <= value[end] <= "\udfff": - end += 1 - piece = value[start:end] - _reject_jsonb_incompatible_str(piece) - sink.write_text(json.dumps(piece)[1:-1]) - start = end - sink.write_text('"') - - -def _write_value(value: Any, sink: _ChunkSink) -> None: - """Write ``value`` exactly as ``json.dumps(value, sort_keys=True)`` would.""" - encoded_chunks = getattr(value, "iter_byte_chunks", None) - if callable(encoded_chunks): - for chunk in encoded_chunks(): - sink.write_bytes(chunk) - return - if value is None or value is True or value is False: - sink.write_text(_dumps_scalar(value)) - return - if isinstance(value, str): - _write_string(value, sink) - return - if isinstance(value, (int, float)): - sink.write_text(_dumps_scalar(value)) - return - if isinstance(value, dict): - keys = list(value) - if not keys: - sink.write_text("{}") - return - # Sort before coercion, matching CPython's failure on incomparable - # key types. Even a numeric-key extension can contain a giant value; - # it must use the same streamed value encoder as ordinary fields. - keys.sort() - sink.write_text("{") - first = True - for key in keys: - if not first: - sink.write_text(",") - if isinstance(key, str): - encoded_key = key - elif key is None or isinstance(key, (bool, int, float)): - encoded_key = _dumps_scalar(key) - else: - raise TypeError(f"keys must be str, int, float, bool or None, not {type(key).__name__}") - _write_string(encoded_key, sink) - sink.write_text(":") - first = False - _write_value(value[key], sink) - sink.write_text("}") - return - if isinstance(value, (list, tuple)): - _write_array(value, sink, on_page=None) - return - raise TypeError(f"Object of type {type(value).__name__} is not JSON serializable") - - -def _batch_text(batch: list[Any]) -> str: - # ``_dumps_scalar`` already ran the jsonb-compatibility scan over the - # batch text (whole records, so escape runs never straddle a boundary). - return _dumps_scalar(batch)[1:-1] - - -def _encoded_size_bound(value: Any, budget: int) -> int | None: - """Conservatively bound an ordinary value *before* calling the C codec. - - Stop inspecting once the byte budget is spent. In particular, an oversized - string needs only its length checked; measuring it must not encode it first. - Twelve ASCII bytes per code point covers ensure_ascii's surrogate pairs. - """ - if budget < 2: - return None - if isinstance(value, str): - size = 2 + 12 * len(value) - elif value is None or isinstance(value, bool): - size = 5 - elif isinstance(value, int): - size = 2 + value.bit_length() // 3 - elif isinstance(value, float): - size = 32 - elif isinstance(value, (dict, list, tuple)): - size = 2 - if isinstance(value, dict): - for key, item in value.items(): - if not isinstance(key, str): - return None - key_size = 2 + 12 * len(key) - item_size = _encoded_size_bound(item, budget - size - key_size - 2) - if item_size is None: - return None - size += key_size + item_size + 2 - else: - for item in value: - item_size = _encoded_size_bound(item, budget - size - 1) - if item_size is None: - return None - size += item_size + 1 - else: - return None - return size if size <= budget else None - - -def _write_array( - items: Iterable[Any], - sink: _ChunkSink, - *, - on_page: Callable[[int, int], None] | None, -) -> int: - """Write one JSON array in bounded batches; returns the item count. - - ``on_page(offset, record_index)`` fires at the start of every batch. - """ - sink.write_text("[") - batch: list[Any] = [] - batch_bytes = 0 - first = True - record_index = 0 - - def flush(batch: list[Any]) -> None: - nonlocal first, record_index - if not first: - sink.write_text(",") - first = False - if on_page is not None: - on_page(sink.offset, record_index) - text = _batch_text(batch) - sink.write_text(text) - record_index += len(batch) - - for item in items: - size = _encoded_size_bound(item, CODEC_BATCH_TARGET_BYTES - 2) - if batch and (size is None or batch_bytes + size + 1 > CODEC_BATCH_TARGET_BYTES - 2): - flush(batch) - batch = [] - batch_bytes = 0 - if size is None: - if not first: - sink.write_text(",") - first = False - if on_page is not None: - on_page(sink.offset, record_index) - _write_value(item, sink) - record_index += 1 - continue - batch.append(item) - batch_bytes += size + 1 - if len(batch) >= CODEC_BATCH_RECORDS: - flush(batch) - batch = [] - batch_bytes = 0 - if batch: - flush(batch) - sink.write_text("]") - return record_index - - -def _canonical_share_pages(shares: Any) -> Iterator[tuple[bytes, int | None]] | None: - """Already-encoded share pages, when the sequence can supply them. - - Returns ``None`` for a sequence that has to be iterated and re-encoded. - Three shapes are consumed verbatim: - - * the ledger's page-backed window (``pages[*].canonical_json_items``), - whose bytes CPython's own encoder produced from the very records the - sequence iterates; - * a spool body written by this codec (``canonical_share_pages``); - * the daemon mirror (``canonical_items`` + ``record_count``), whose - items stream is the canonical encoding the mirror reconciled against - its record count and digest at construction. Copying it is the only - way to stage a daemon-owned window without the sequence's whole - parse (its ``__iter__`` on the current tree materializes every - record); wave B of #254 keeps the same bytes contract. Record - boundaries are not known for this shape, so its page entries are - scanned *hints* that a reader validates by decoding. - """ - pages_method = getattr(shares, "canonical_share_pages", None) - if callable(pages_method): - return pages_method() - canonical_items = getattr(shares, "canonical_items", None) - if isinstance(canonical_items, bytes) and isinstance( - getattr(shares, "record_count", None), int - ): - - def iter_daemon_items() -> Iterator[tuple[bytes, int | None]]: - # One raw slice: the sink consumes it linearly and the digest - # update releases the GIL for a buffer this size. No copy of - # the stream is made beyond the chunk pieces themselves. - yield canonical_items, None - - return iter_daemon_items() - pages = getattr(shares, "pages", None) - if isinstance(pages, tuple) and all( - isinstance(getattr(page, "canonical_json_items", None), bytes) - and isinstance(getattr(page, "prism_json_records", None), tuple) - for page in pages - ): - - def iter_pages() -> Iterator[tuple[bytes, int | None]]: - for page in pages: - if page.canonical_json_items: - yield page.canonical_json_items, len(page.prism_json_records) - - return iter_pages() - return None - - -_JSON_STRUCTURE = re.compile(rb'["\\{}\[\],]') - - -class _RecordPageScanner: - """Locate exact top-level separators using bounded byte scans. - - Quoted text and nested arrays/objects cannot become page boundaries. - State persists across source chunks, including a split escape pair. - """ - - def __init__(self, on_page: Callable[[int, int], None]) -> None: - self.on_page = on_page - self.started = False - self.in_string = False - self.depth = 0 - self.escape_at = -2 - self.record_index = 0 - self.page_index = 0 - self.page_start = 0 - - def feed(self, data: bytes, base_offset: int) -> None: - if not data: - return - if not self.started: - self.started = True - self.page_start = base_offset - self.on_page(base_offset, 0) - for offset in range(0, len(data), SPOOL_READ_BYTES): - piece = data[offset:offset + SPOOL_READ_BYTES] - for match in _JSON_STRUCTURE.finditer(piece): - position = base_offset + offset + match.start() - token = match[0] - if self.in_string: - if position == self.escape_at + 1: - self.escape_at = -2 - elif token == b"\\": - self.escape_at = position - elif token == b'"': - self.in_string = False - elif token == b'"': - self.in_string = True - elif token in (b"{", b"["): - self.depth += 1 - elif token in (b"}", b"]"): - self.depth -= 1 - if self.depth < 0: - raise CandidateBodyIntegrityError("array source has unmatched delimiters") - elif token == b"," and self.depth == 0: - self.record_index += 1 - if (self.record_index - self.page_index >= CODEC_BATCH_RECORDS - or position + 1 - self.page_start >= CODEC_BATCH_TARGET_BYTES): - self.page_start = position + 1 - self.page_index = self.record_index - self.on_page(self.page_start, self.page_index) - - def finish(self) -> int: - if self.in_string or self.depth: - raise CandidateBodyIntegrityError("array source ends inside a record") - return self.record_index + int(self.started) - - -def _write_sequence(sequence: Any, sink: _ChunkSink, events: _IndexEvents) -> tuple[int, bool]: - """Copy canonical items with exact page boundaries and no decoded mirror.""" - pages = _canonical_share_pages(sequence) - if pages is None: - return _write_array(iter(sequence), sink, on_page=events.page), True - sink.write_text("[") - scanner = _RecordPageScanner(events.page) - first = True - raw_mode = False - for data, record_count in pages: - if not data: - continue - if not first and (record_count is not None or not raw_mode): - scanner.feed(b",", sink.offset) - sink.write_text(",") - raw_mode = record_count is None - first = False - scanner.feed(data, sink.offset) - sink.write_bytes(data) - count = scanner.finish() - if count != len(sequence): - raise CandidateBodyIntegrityError("array pages disagree with the sequence length") - sink.write_text("]") - return count, True - - -def _write_shares(shares: Any, sink: _ChunkSink, events: _IndexEvents) -> tuple[int, int, int, bool]: - """Write the share array; returns (start, end, count, pages_exact).""" - start = sink.offset - events.begin(CANDIDATE_SHARES_KEY, "array", start) - count, pages_exact = _write_sequence(shares, sink, events) - end = sink.offset - events.end(end, item_count=count, pages_exact=pages_exact, emit=True) - return start, end, count, pages_exact - - -def _write_top_level_field(key: str, value: Any, sink: _ChunkSink, events: _IndexEvents) -> None: - """Write one non-share top-level field, spanning it when it is large. - - Any sequence that is not a string is an array here -- a list, a tuple, - or a spool-backed array adapter from a hydrated body -- so a hydrated - intent re-stages without materializing its large fields. - """ - start = sink.offset - if isinstance(value, Sequence) and not isinstance(value, (str, bytes, bytearray)): - events.begin(key, "array", start) - count, pages_exact = _write_sequence(value, sink, events) - end = sink.offset - events.end(end, item_count=count, pages_exact=pages_exact, emit=(end - start) > CODEC_FAST_PATH_BYTES) - return - if isinstance(value, str): - events.begin(key, "string", start) - _write_string(value, sink) - end = sink.offset - events.end(end, item_count=0, pages_exact=True, emit=(end - start) > CODEC_FAST_PATH_BYTES) - return - events.begin(key, "value", start) - _write_value(value, sink) - end = sink.offset - events.end(end, item_count=0, pages_exact=True, emit=(end - start) > CODEC_FAST_PATH_BYTES) - - -def encode_identity_body( - identity: Mapping[str, Any], - shares: Any, - *, - chunk_bytes: int = CANDIDATE_BODY_CHUNK_BYTES, - on_chunk: Callable[[BodyChunk], None] | None = None, - on_span: Callable[[FieldSpan], None] | None = None, - on_page: Callable[[list[PageEntry]], None] | None = None, -) -> CandidateBodyManifest: - """Encode ``{**identity, "shares_json": shares}`` in bounded pieces. - - ``identity`` is the small-field mapping with the acknowledgment stamp - already neutralized. The concatenated chunks handed to ``on_chunk`` are - byte-identical to ``json.dumps({**identity, "shares_json": - list(shares)}, sort_keys=True, separators=(",", ":"))``. ``on_span`` and - ``on_page`` receive the index events; the returned manifest is scalar. - """ - if shares is not None and ( - isinstance(shares, (str, bytes)) or not isinstance(shares, Sequence) - ): - raise CandidateCodecError("shares_json must be a sequence of records") - keys = list(identity) - if CANDIDATE_SHARES_KEY in keys: - raise CandidateCodecError("identity fields must not carry shares_json") - if any(not isinstance(key, str) for key in keys): - raise CandidateCodecError("candidate intent keys must be strings") - if shares is not None: - keys.append(CANDIDATE_SHARES_KEY) - keys.sort() - sink = _ChunkSink(chunk_bytes, on_chunk) - events = _IndexEvents(on_span, on_page) - sink.write_text("{") - first = True - shares_start = shares_end = 0 - share_count = 0 - for key in keys: - sink.write_text(("" if first else ",") + _dumps_key(key) + ":") - first = False - if key == CANDIDATE_SHARES_KEY: - shares_start, shares_end, share_count, _exact = _write_shares(shares, sink, events) - else: - _write_top_level_field(key, identity[key], sink, events) - sink.write_text("}") - digest = sink.finish() - manifest = CandidateBodyManifest( - storage_version=CANDIDATE_BODY_STORAGE_VERSION, - candidate_sha256=digest, - byte_count=sink.offset, - chunk_count=sink.chunk_count, - chunk_bytes=int(chunk_bytes), - share_count=share_count, - shares_offset=shares_start, - shares_end=shares_end, - span_count=events.span_count, - page_count=events.page_count, - ) - manifest.validate() - return manifest - - -# -------------------------------------------------------------------------- -# identity normalization and replay header -# -------------------------------------------------------------------------- - - -def split_candidate_fields( - fields: Mapping[str, Any], -) -> tuple[dict[str, Any], Any]: - """Return ``(small facts, shares sequence or None)`` without copying. - - ``None`` means the intent carries no ``shares_json`` key at all (a - candidate-only intent as some tools persist); the historical identity - JSON of such a document has no share member and is reproduced as is. - """ - if not isinstance(fields, Mapping): - raise TypeError("block candidate intent must be an object") - facts = { - str(key): value - for key, value in fields.items() - if key != CANDIDATE_SHARES_KEY - } - return facts, fields.get(CANDIDATE_SHARES_KEY) if CANDIDATE_SHARES_KEY in fields else None - - -def neutralized_identity_facts(facts: Mapping[str, Any]) -> tuple[dict[str, Any], bool, Any]: - """Apply ``block_candidate_identity``'s stamp rule to the small facts. - - Returns ``(identity facts, stamp present, original stamp)``. The rule - is exactly the historical one: only when ``pending_share`` is a mapping - that carries ``accepted_at_ms`` is that one value replaced by ``None``. - """ - pending_share = facts.get("pending_share") - if isinstance(pending_share, dict) and "accepted_at_ms" in pending_share: - identity = { - **facts, - "pending_share": {**pending_share, "accepted_at_ms": None}, - } - return identity, True, pending_share["accepted_at_ms"] - return dict(facts), False, None - - -def restore_pending_stamp( - facts: Mapping[str, Any], - *, - present: bool, - accepted_at_ms: Any, -) -> dict[str, Any]: - """Undo :func:`neutralized_identity_facts` for a body read back.""" - restored = dict(facts) - pending_share = restored.get("pending_share") - if present and isinstance(pending_share, dict): - restored["pending_share"] = {**pending_share, "accepted_at_ms": accepted_at_ms} - return restored - - -def _bounded_text(value: Any, limit: int) -> Any: - """A string fact within ``limit`` bytes of JSON text, else None.""" - if not isinstance(value, str) or len(value) > limit: - return None - if len(json.dumps(value)) > limit: - return None - return value - - -def _bounded_number(value: Any, limit: int) -> Any: - """An int/float fact within ``limit`` bytes of JSON text, else None.""" - if isinstance(value, bool) or not isinstance(value, (int, float)): - return None - if len(json.dumps(value)) > limit: - return None - return value - - -_PENDING_SHARE_TEXT_FIELDS = ("share_id", "miner_id", "order_key", "p2mr_program_hex", "job_id", "credit_policy") -_PENDING_SHARE_NUMBER_FIELDS = ( - "share_difficulty", - "network_difficulty", - "template_height", - "job_issued_at_ms", - "accepted_at_ms", - "ntime", -) - - -def replay_header_from_fields(fields: Mapping[str, Any]) -> dict[str, Any]: - """The small typed summary an outbox row carries beside its body. - - Explicit projection with per-fact byte bounds, never subtraction: - every key is named and every fact is either within its bound or - replaced by ``None`` with ``oversized`` set, so a page of headers has a - provable maximum size regardless of what the body holds. An oversized - fact is deferred to the body (hydration reads the real value); nothing - is rejected here. ``pending_share`` keeps the original acknowledgment - stamp. Mirrors ``qbit_prism_bounded_replay_header`` in SQL. - """ - template = fields.get("template") - found_block = fields.get("found_block") - pending_share = fields.get("pending_share") - oversized = False - - def text(value: Any, limit: int = HEADER_TEXT_FIELD_BYTES) -> Any: - nonlocal oversized - bounded = _bounded_text(value, limit) - if bounded is None and value is not None: - oversized = True - return bounded - - def number(value: Any, limit: int = HEADER_NUMERIC_FIELD_BYTES) -> Any: - nonlocal oversized - bounded = _bounded_number(value, limit) - if bounded is None and value is not None: - oversized = True - return bounded - - bounded_pending: dict[str, Any] | None = None - if isinstance(pending_share, Mapping): - bounded_pending = {} - for key, value in pending_share.items(): - if not isinstance(key, str) or len(key) > 64 or len(key.encode("utf-8")) > 64: - oversized = True - continue - if key in _PENDING_SHARE_NUMBER_FIELDS: - bounded_pending[str(key)] = number(value) - elif key in _PENDING_SHARE_TEXT_FIELDS: - bounded_pending[str(key)] = text(value) - elif isinstance(value, str): - bounded_pending[str(key)] = text(value) - elif isinstance(value, (int, float)) and not isinstance(value, bool): - bounded_pending[str(key)] = number(value) - elif value is None or isinstance(value, bool): - bounded_pending[str(key)] = value - else: - oversized = True - bounded_pending[str(key)] = None - if len(bounded_pending) > 32: - oversized = True - break - elif pending_share is not None: - oversized = True - header = { - "schema": text(fields.get("schema")), - "block_hash_hex": text(fields.get("block_hash_hex"), HEADER_HASH_FIELD_BYTES + 2), - "parent_hash": text(fields.get("parent_hash"), HEADER_HASH_FIELD_BYTES + 2), - "expected_height": number(fields.get("expected_height")), - "template": ( - { - "previousblockhash": text(template.get("previousblockhash"), HEADER_HASH_FIELD_BYTES + 2), - "height": number(template.get("height")), - "coinbasevalue": number(template.get("coinbasevalue")), - } - if isinstance(template, Mapping) - else None - ), - "found_block": ( - {"network_difficulty": number(found_block.get("network_difficulty"))} - if isinstance(found_block, Mapping) - else None - ), - "pending_share": bounded_pending, - "credit_share_on_accept": ( - fields.get("credit_share_on_accept") - if isinstance(fields.get("credit_share_on_accept"), bool) - else None - ), - "collection_only": ( - fields.get("collection_only") - if isinstance(fields.get("collection_only"), bool) - else None - ), - "username": text(fields.get("username")), - "accepted_at_present": bool( - isinstance(pending_share, Mapping) and "accepted_at_ms" in pending_share - ), - "oversized": oversized, - } - return header - - -# -------------------------------------------------------------------------- -# on-disk page index -# -------------------------------------------------------------------------- - - -class SpoolFieldIndex: - """Spans and page entries of one spool body, kept on disk. - - The index file holds every field's page entries as fixed 16-byte - records (``<QQ``: body offset, record index) in one contiguous region - per field. In memory there is one small :class:`FieldSpan` plus a region - start per large field -- a handful of entries -- never the page list. - """ - - __slots__ = ("path", "_spans", "_regions", "_lock") - - def __init__(self, path: str) -> None: - self.path = path - self._spans: dict[str, FieldSpan] = {} - self._regions: dict[str, int] = {} - self._lock = threading.Lock() - - @classmethod - def create(cls, path: str) -> SpoolFieldIndex: - with open(path, "wb"): - pass - return cls(path) - - @classmethod - def open_written(cls, path: str, spans_in_order: Sequence[FieldSpan]) -> SpoolFieldIndex: - """Attach to an index file another process wrote span by span. - - The writer appended each field's region in the order it began the - spans, so the regions are recovered from the page counts alone and - the file size is checked against them. - """ - index = cls(path) - position = 0 - for span in spans_in_order: - index._spans[span.field] = span - index._regions[span.field] = position - position += span.page_count * _INDEX_RECORD.size - if os.path.getsize(path) != position: - raise CandidateBodyIntegrityError("page index size disagrees with its spans") - return index - - def spans(self) -> dict[str, FieldSpan]: - with self._lock: - return dict(self._spans) - - def span(self, field_name: str) -> FieldSpan | None: - with self._lock: - return self._spans.get(field_name) - - def begin_field(self, span: FieldSpan) -> None: - """Declare a field's span. - - The encoder emits a field's page batches *before* its span (the - span is known only when the field ends), so the region start is - pinned by whichever comes first: the first page append, or this - declaration for a field with no pages. - """ - with self._lock: - if span.field in self._spans: - raise CandidateBodyIntegrityError("duplicate body span") - self._spans[span.field] = span - self._regions.setdefault(span.field, os.path.getsize(self.path)) - - def append_pages(self, field_name: str, entries: Iterable[tuple[int, int]]) -> None: - """Append ``(offset, record_index)`` records for the field being written.""" - with self._lock: - self._regions.setdefault(field_name, os.path.getsize(self.path)) - packed = b"".join(_INDEX_RECORD.pack(int(offset), int(index)) for offset, index in entries) - with open(self.path, "ab") as handle: - handle.write(packed) - - def page_count(self, field_name: str) -> int: - return self._spans[field_name].page_count - - def entry(self, field_name: str, ordinal: int) -> tuple[int, int]: - span = self._spans[field_name] - if not (0 <= ordinal < span.page_count): - raise IndexError(ordinal) - position = self._regions[field_name] + ordinal * _INDEX_RECORD.size - with open(self.path, "rb") as handle: - handle.seek(position) - raw = handle.read(_INDEX_RECORD.size) - if len(raw) != _INDEX_RECORD.size: - raise CandidateBodyIntegrityError("page index is truncated") - offset, record_index = _INDEX_RECORD.unpack(raw) - return int(offset), int(record_index) - - def set_record_index(self, field_name: str, ordinal: int, record_index: int) -> None: - """Correct one hint's record index after an exact decode.""" - offset, _ = self.entry(field_name, ordinal) - position = self._regions[field_name] + ordinal * _INDEX_RECORD.size - with self._lock, open(self.path, "r+b") as handle: - handle.seek(position) - handle.write(_INDEX_RECORD.pack(int(offset), int(record_index))) - - def bisect_record(self, field_name: str, record_index: int) -> int: - """The page ordinal holding ``record_index`` (exact pages only).""" - low, high = 0, self._spans[field_name].page_count - while high - low > 1: - middle = (low + high) // 2 - if self.entry(field_name, middle)[1] <= record_index: - low = middle - else: - high = middle - return low - - def mark_exact(self, field_name: str) -> None: - with self._lock: - span = self._spans[field_name] - self._spans[field_name] = replace(span, pages_exact=True) - - def close(self) -> None: - try: - os.unlink(self.path) - except OSError: - pass - - -# -------------------------------------------------------------------------- -# body handles -# -------------------------------------------------------------------------- - - -class CandidateBody: - """Replayable immutable body bytes plus the manifest describing them. - - Chunks are delivered push-style, one at a time, to a consumer callable: - every consumer this codebase has (validation, storage upload, spool - write, helper pipe) is push-shaped, and a push interface keeps at most - one chunk plus the encoder's bounded batch alive at any instant. Index - events (spans, page batches) ride the same pass. - """ - - manifest: CandidateBodyManifest - - def write_chunks( - self, - consumer: Callable[[BodyChunk], None], - *, - on_span: Callable[[FieldSpan], None] | None = None, - on_page: Callable[[list[PageEntry]], None] | None = None, - ) -> None: - raise NotImplementedError - - def close(self) -> None: - """Release what this handle owns; idempotent.""" - - -class SourceCandidateBody(CandidateBody): - """A live body: re-encoded from the pinned source sequence on demand. - - Nothing encoded is retained between passes. Every pass re-derives the - bytes, and the rolling digest of what was handed out must equal the - manifest's digest, so a source that changed underneath is detected - before the caller seals anything. - """ - - __slots__ = ("manifest", "_identity", "_shares") - - def __init__( - self, - identity: Mapping[str, Any], - shares: Any, - manifest: CandidateBodyManifest, - ) -> None: - self.manifest = manifest - self._identity = identity - self._shares = shares - - def write_chunks( - self, - consumer: Callable[[BodyChunk], None], - *, - on_span: Callable[[FieldSpan], None] | None = None, - on_page: Callable[[list[PageEntry]], None] | None = None, - ) -> None: - manifest = encode_identity_body( - self._identity, - self._shares, - chunk_bytes=self.manifest.chunk_bytes, - on_chunk=consumer, - on_span=on_span, - on_page=on_page, - ) - if manifest != self.manifest: - raise CandidateBodyIntegrityError( - "candidate body source changed after validation" - ) - - def close(self) -> None: - self._shares = () - self._identity = {} - - -class SpoolCandidateBody(CandidateBody): - """A durable body materialized to a local spool file with a page index. - - The spool holds exactly the body bytes and the index its spans and - pages. Chunk iteration re-reads and re-hashes the file, so a reader - never trusts a stale manifest over the bytes on disk. Both files are - unlinked when the last owner releases the handle (a ``weakref.finalize`` - on the handle, plus explicit ``close``), never while a sequence still - reads them. - """ - - __slots__ = ("manifest", "path", "index", "_finalizer") - - def __init__( - self, - path: str, - manifest: CandidateBodyManifest, - index: SpoolFieldIndex, - *, - release: Callable[[], None] | None = None, - ) -> None: - self.manifest = manifest - self.path = path - self.index = index - self._finalizer = weakref.finalize(self, _unlink_spool, path, index.path, release) - - def iter_chunks(self) -> Iterator[BodyChunk]: - chunk_bytes = self.manifest.chunk_bytes - digest = hashlib.sha256() - with open(self.path, "rb") as handle: - ordinal = 0 - while True: - data = handle.read(chunk_bytes) - if not data: - break - if ordinal >= self.manifest.chunk_count: - raise CandidateBodyIntegrityError("spool body is longer than its manifest") - digest.update(data) - yield BodyChunk(ordinal=ordinal, data=data, sha256=hashlib.sha256(data).hexdigest()) - ordinal += 1 - if ordinal != self.manifest.chunk_count or digest.hexdigest() != self.manifest.candidate_sha256: - raise CandidateBodyIntegrityError("spool body disagrees with its manifest") - - def write_chunks( - self, - consumer: Callable[[BodyChunk], None], - *, - on_span: Callable[[FieldSpan], None] | None = None, - on_page: Callable[[list[PageEntry]], None] | None = None, - ) -> None: - for chunk in self.iter_chunks(): - consumer(chunk) - if on_span is None and on_page is None: - return - for field_name, span in sorted(self.index.spans().items()): - if on_span is not None: - on_span(span) - if on_page is None: - continue - batch: list[PageEntry] = [] - for ordinal in range(span.page_count): - offset, record_index = self.index.entry(field_name, ordinal) - batch.append(PageEntry(field=field_name, ordinal=ordinal, offset=offset, record_index=record_index)) - if len(batch) >= INDEX_PAGE_ENTRIES: - on_page(batch) - batch = [] - if batch: - on_page(batch) - - def read_span(self, start: int, end: int) -> bytes: - if start < 0 or end < start or end > self.manifest.byte_count: - raise ValueError("spool span out of range") - with open(self.path, "rb") as handle: - handle.seek(start) - data = handle.read(end - start) - if len(data) != end - start: - raise CandidateBodyIntegrityError("spool body is shorter than its manifest") - return data - - def close(self) -> None: - self._finalizer() - - @property - def alive(self) -> bool: - return self._finalizer.alive - - -def _unlink_spool(path: str, index_path: str, release: Callable[[], None] | None) -> None: - for target in (path, index_path): - try: - os.unlink(target) - except OSError: - pass - if release is not None: - try: - release() - except Exception: # noqa: BLE001 - finalizers must not raise - pass - - -# -------------------------------------------------------------------------- -# spool decoding -# -------------------------------------------------------------------------- - - -def _views() -> Any: - """The lazy-view module (``candidate_spool_view``), imported on first use. - - That module builds on this one and on the audit adapters' retryable - resource-pressure class, so the import is deferred to keep this module - a standard-library leaf at import time. - """ - from lab.prism import candidate_spool_view - - return candidate_spool_view - - -def _walk_record_batches( - body: SpoolCandidateBody, - start: int, - end: int, - *, - batch_records: int = SPOOL_WALK_BATCH_RECORDS, -) -> Iterator[tuple[int, int, tuple[Any, ...]]]: - """``(batch_start, batch_end, records)`` over ``body[start:end]``. - - The range holds records joined by ``,``. Record boundaries come from the - structural byte scanner, so no record is decoded to find its end, and - each record follows the view module's decode policy: one bounded - ``json.loads`` when it fits :data:`~lab.prism.candidate_spool_view.SPOOL_VIEW_DECODE_BYTES`, - a lazy view when it does not. There is no per-record size ceiling. - """ - views = _views() - batch: list[Any] = [] - batch_start = start - batch_end = start - for item_start, item_end in views.iter_item_spans(body, start, end): - if not batch: - batch_start = item_start - batch.append(views.decode_body_span(body, item_start, item_end)) - batch_end = item_end - if len(batch) >= batch_records: - yield batch_start, batch_end, tuple(batch) - batch = [] - if batch: - yield batch_start, batch_end, tuple(batch) - - -def _walk_records( - body: SpoolCandidateBody, - start: int, - end: int, - *, - batch_records: int = SPOOL_WALK_BATCH_RECORDS, -) -> Iterator[tuple[Any, ...]]: - """Decode ``body[start:end]`` (records joined by ``,``) in bounded batches.""" - for _batch_start, _batch_end, records in _walk_record_batches( - body, start, end, batch_records=batch_records - ): - yield records - - -class SpoolJsonArraySequence(Sequence): - """One large JSON array field decoded page by page from a spool body. - - Iteration decodes one page per ``json.loads`` from the on-disk page - index while the page fits the view module's decode threshold; a larger - page (one holding an oversized record) is split by the structural - scanner and each record follows the decode policy, so an oversized - record is a lazy view rather than a whole decode or a rejection. Random - access bisects the index with ``pread``. A span whose page entries are - not exact (legacy scanned hints) is walked sequentially once, ignoring - the hints, and the exact page starts recovered by that walk serve random - access from then on. Only the page last decoded is cached. - """ - - __slots__ = ("_body", "_span", "_lock", "_cached_page", "_cached_records", "_walk_index") - - def __init__(self, body: SpoolCandidateBody, span: FieldSpan) -> None: - self._body = body - self._span = span - self._lock = threading.Lock() - self._cached_page = -1 - self._cached_records: tuple[Any, ...] = () - self._walk_index: Any = None - - @property - def body(self) -> SpoolCandidateBody: - return self._body - - @property - def span(self) -> FieldSpan: - return self._body.index.span(self._span.field) or self._span - - @property - def byte_range(self) -> tuple[int, int]: - return self._span.start, self._span.end - - @property - def byte_length(self) -> int: - return self._span.end - self._span.start - - def __len__(self) -> int: - return self._span.item_count - - # -- pages ------------------------------------------------------------------ - - def _indexed(self) -> bool: - """True when the on-disk page index can serve random access.""" - span = self.span - return span.page_count > 0 and span.pages_exact - - def _index_entry(self, ordinal: int) -> tuple[int, int]: - try: - return self._body.index.entry(self._span.field, ordinal) - except OSError as exc: - raise CandidateBodyIntegrityError(f"spool page index is unreadable: {exc}") from exc - - def _page_span(self, page: int) -> tuple[int, int, int]: - """(start, end, record_index) of one page; ``end`` excludes the separator.""" - start, record_index = self._index_entry(page) - if page + 1 < self._span.page_count: - end = self._index_entry(page + 1)[0] - 1 - else: - end = self._span.end - 1 - if not (self._span.start < start <= end < self._span.end): - raise CandidateBodyIntegrityError("spool page index points outside its span") - return start, end, record_index - - def _decode_records(self, start: int, end: int) -> tuple[Any, ...]: - """Records in ``body[start:end]``: one bounded call, or the walk.""" - views = _views() - if end <= start: - return () - if end - start <= views.SPOOL_VIEW_DECODE_BYTES: - data = views.read_body_span(self._body, start, end) - try: - records = json.loads(b"[" + data + b"]") - except ValueError as exc: - raise CandidateBodyIntegrityError("spool page is not valid JSON") from exc - return tuple(records) - return tuple( - record - for _batch_start, _batch_end, batch in _walk_record_batches(self._body, start, end) - for record in batch - ) - - def _decode_page(self, page: int) -> tuple[Any, ...]: - with self._lock: - if page == self._cached_page: - return self._cached_records - start, end, _ = self._page_span(page) - decoded = self._decode_records(start, end) - with self._lock: - self._cached_page = page - self._cached_records = decoded - return decoded - - def _walk_pages(self, span: FieldSpan) -> Iterator[tuple[Any, ...]]: - """Sequential walk of a span without a usable page index. - - Records the exact batch starts in a private bounded index so later - random access bisects instead of walking again. - """ - views = _views() - with self._lock: - index = None if self._walk_index is not None else views._OffsetIndex(2) - running = 0 - try: - for batch_start, _batch_end, records in _walk_record_batches( - self._body, span.start + 1, span.end - 1 - ): - if index is not None: - index.append(batch_start, running) - running += len(records) - yield records - if running != span.item_count: - raise CandidateBodyIntegrityError( - f"spool array holds {running} records where {span.item_count} were declared" - ) - except BaseException: - if index is not None: - index.close() - raise - if index is not None: - with self._lock: - if self._walk_index is None: - self._walk_index = index - index = None - if index is not None: - index.close() - - def iter_pages(self) -> Iterator[tuple[Any, ...]]: - """Decoded pages in order; each one bounded C call (or a walk).""" - span = self.span - if not self._indexed(): - yield from self._walk_pages(span) - return - running = 0 - for page in range(span.page_count): - records = self._decode_page(page) - running += len(records) - yield records - if running != span.item_count: - raise CandidateBodyIntegrityError( - f"spool array holds {running} records where {span.item_count} were declared" - ) - - # -- Sequence protocol ---------------------------------------------------------- - - def __iter__(self) -> Iterator[Any]: - for page in self.iter_pages(): - yield from page - - def __getitem__(self, index: int | slice) -> Any: - if isinstance(index, slice): - start, stop, step = index.indices(len(self)) - return tuple(self[position] for position in range(start, stop, step)) - if isinstance(index, bool) or not isinstance(index, int): - raise TypeError("spool array indices must be integers or slices") - resolved = index + len(self) if index < 0 else index - if not 0 <= resolved < len(self): - raise IndexError(index) - span = self.span - if self._indexed(): - try: - page = self._body.index.bisect_record(self._span.field, resolved) - except OSError as exc: - raise CandidateBodyIntegrityError(f"spool page index is unreadable: {exc}") from exc - records = self._decode_page(page) - first = self._index_entry(page)[1] - return records[resolved - first] - walk = self._walk_index - if walk is None: - for _page in self.iter_pages(): - pass - walk = self._walk_index - if walk is None: - raise IndexError(index) - low, high = 0, walk.count - while high - low > 1: - middle = (low + high) // 2 - if walk.get(middle)[1] <= resolved: - low = middle - else: - high = middle - batch_start, first = walk.get(low) - batch_end = walk.get(low + 1)[0] - 1 if low + 1 < walk.count else span.end - 1 - records = self._decode_records(batch_start, batch_end) - return records[resolved - first] - - def __eq__(self, other: object) -> bool: - if other is self: - return True - if isinstance(other, (str, bytes, bytearray)) or not isinstance(other, Sequence): - return NotImplemented - if len(self) != len(other): - return False - other_iter = iter(other) - for item in self: - try: - candidate = next(other_iter) - except StopIteration: - return False - if item != candidate: - return False - return True - - def __ne__(self, other: object) -> bool: - result = self.__eq__(other) - if result is NotImplemented: - return result # type: ignore[return-value] - return not result - - __hash__ = None # type: ignore[assignment] - - def __repr__(self) -> str: - return f"SpoolJsonArraySequence(field={self._span.field!r}, items={len(self)}, bytes={self.byte_length})" - - # -- verbatim bytes -------------------------------------------------------------- - - def iter_byte_chunks(self, *, chunk_bytes: int | None = None) -> Iterator[bytes]: - """The array's exact encoding as stored, brackets included.""" - return _views().iter_body_bytes(self._body, self._span.start, self._span.end, chunk_bytes=chunk_bytes) - - def iter_encoded_chunks(self) -> Iterator[str]: - return _views().iter_body_text(self._body, self._span.start, self._span.end) - - def canonical_json_sha256(self) -> str: - digest = hashlib.sha256() - for chunk in self.iter_byte_chunks(): - digest.update(chunk) - return digest.hexdigest() - - def canonical_share_pages(self) -> Iterator[tuple[bytes, int | None]]: - """The encoded pages, verbatim, for re-staging this body. - - A page within the read size is one complete page with its record - count; a larger page (an oversized record) and a span without a - usable index stream as raw continuation pieces, which the encoder's - structural scanner re-pages exactly. Nothing page-sized is held. - """ - views = _views() - span = self.span - if not self._indexed(): - for chunk in views.iter_body_bytes(self._body, span.start + 1, span.end - 1): - yield chunk, None - return - for page in range(span.page_count): - start, end, record_index = self._page_span(page) - if end - start > views.SPOOL_VIEW_READ_BYTES: - for chunk in views.iter_body_bytes(self._body, start, end): - yield chunk, None - continue - next_index = ( - self._index_entry(page + 1)[1] - if page + 1 < span.page_count - else span.item_count - ) - yield views.read_body_span(self._body, start, end), next_index - record_index - - def close(self) -> None: - """Release the private walk index, if one was built; the body is untouched.""" - with self._lock: - walk = self._walk_index - if walk is not None: - walk.close() - - -# Historical name kept for the share array. -SpoolShareJsonSequence = SpoolJsonArraySequence - - -def decode_json_string_span(body: SpoolCandidateBody, span: FieldSpan) -> str: - """The whole text of one large JSON string field, decoded in slices. - - Every C call is one slice; the join is bounded only by the field, which - is why hydration uses this for the consensus-bounded metadata fields - alone and hands any other oversized string out as a streamed view. - """ - return _views().decode_json_string(body, span.start, span.end) - - -def decode_spool_small_fields(body: SpoolCandidateBody) -> dict[str, Any]: - """The body's fields other than the share array, bounded per field. - - Builds a *skeleton* of the body in which every spanned (large) field is - replaced by an empty placeholder and decodes it with one ``json.loads`` - bounded by the view decode ceiling, falling back to per-member reads - if the aggregate metadata exceeds that ceiling. Each - large field then follows the view module's policy: an array becomes a - :class:`SpoolJsonArraySequence`; a string in - :data:`~lab.prism.candidate_spool_view.SPOOL_DECODED_METADATA_FIELDS` - is decoded whole in slices (its size is bounded by consensus and its - consumers need the text); any other value is a plain value within the - decode threshold and a lazy view above it. Nothing is rejected for size. - """ - views = _views() - manifest = body.manifest - spans = sorted(body.index.spans().values(), key=lambda span: span.start) - validated_end = 0 - for span in spans: - if span.start < validated_end or span.end > manifest.byte_count: - raise CandidateBodyIntegrityError("spool spans overlap or exceed the body") - validated_end = span.end - - class SkeletonFull(Exception): - pass - - class BoundedSkeleton(io.BytesIO): - def write(self, data: bytes) -> int: - if self.tell() + len(data) > views.SPOOL_VIEW_DECODE_BYTES: - raise SkeletonFull - return super().write(data) - - skeleton = BoundedSkeleton() - cursor = 0 - try: - for span in spans: - _copy_span(body, cursor, span.start, skeleton) - skeleton.write(b"[]" if span.kind == "array" else b'""' if span.kind == "string" else b"null") - cursor = span.end - _copy_span(body, cursor, manifest.byte_count, skeleton) - fields = json.loads(skeleton.getvalue()) - except SkeletonFull: - # Many individually small metadata fields can exceed the aggregate - # fast-path budget. Decode those members separately through the same - # view policy instead of growing one native JSON call. - root = views.SpoolObjectView(body, 0, manifest.byte_count) - try: - fields = dict(root.items()) - finally: - root.close() - except ValueError as exc: - raise CandidateBodyIntegrityError("spool body skeleton is not valid JSON") from exc - if not isinstance(fields, dict): - raise CandidateBodyIntegrityError("spool body does not decode to an intent") - for span in spans: - if span.field == CANDIDATE_SHARES_KEY: - continue - if span.kind == "array": - fields[span.field] = SpoolJsonArraySequence(body, span) - elif span.kind == "string" and span.field in views.SPOOL_DECODED_METADATA_FIELDS: - fields[span.field] = views.decode_json_string(body, span.start, span.end) - else: - fields[span.field] = views.decode_body_span(body, span.start, span.end) - fields.pop(CANDIDATE_SHARES_KEY, None) - return fields - - -def _copy_span(body: SpoolCandidateBody, start: int, end: int, sink: io.BytesIO) -> None: - for chunk in _views().iter_body_bytes(body, start, end): - sink.write(chunk) - - -# -------------------------------------------------------------------------- -# prepared intent -# -------------------------------------------------------------------------- - - -class PreparedCandidateIntent(MutableMapping): - """A validated candidate intent behind the historical mapping interface. - - ``facts`` holds every field but ``shares_json`` with the pending share's - original acknowledgment stamp; ``shares`` is the immutable sequence - (never a list copy); ``body`` streams the v1 identity bytes; the - remaining attributes are the small typed facts storage needs. Mapping - access (``intent["shares_json"]``, ``dict(intent)``, ``intent.get``) - behaves as the historical dict did. Mutation (``pop``, item - assignment) is supported for dict-facing callers and tools; it marks - the intent *dirty*, and the next preparation re-derives the manifest - from the mutated fields rather than trusting the stale one. - """ - - __slots__ = ( - "facts", - "shares", - "body", - "block_hash", - "candidate_sha256", - "accepted_at_present", - "accepted_at_ms", - "_key_order", - "dirty", - ) - - def __init__( - self, - *, - facts: dict[str, Any], - shares: Any, - body: CandidateBody, - accepted_at_present: bool, - accepted_at_ms: Any, - key_order: tuple[str, ...], - ) -> None: - self.facts = facts - self.shares = shares - self.body = body - self.block_hash = str(facts.get("block_hash_hex", "")).lower() - self.candidate_sha256 = body.manifest.candidate_sha256 - self.accepted_at_present = accepted_at_present - self.accepted_at_ms = accepted_at_ms - self._key_order = key_order - self.dirty = False - - def __setitem__(self, key: str, value: Any) -> None: - key = str(key) - if key == CANDIDATE_SHARES_KEY: - self.shares = value - else: - self.facts[key] = value - if key not in self._key_order: - self._key_order = (*self._key_order, key) - self.dirty = True - - def __delitem__(self, key: str) -> None: - if key == CANDIDATE_SHARES_KEY: - if self.shares is None: - raise KeyError(key) - self.shares = None - else: - del self.facts[key] - self._key_order = tuple(name for name in self._key_order if name != key) - self.dirty = True - - @property - def manifest(self) -> CandidateBodyManifest: - return self.body.manifest - - @property - def storage_version(self) -> int: - return self.body.manifest.storage_version - - @property - def has_shares(self) -> bool: - return self.shares is not None - - def __getitem__(self, key: str) -> Any: - if key == CANDIDATE_SHARES_KEY: - if self.shares is None: - raise KeyError(key) - return self.shares - return self.facts[key] - - def __iter__(self) -> Iterator[str]: - return iter(self._key_order) - - def __len__(self) -> int: - return len(self._key_order) - - def __contains__(self, key: object) -> bool: - if key == CANDIDATE_SHARES_KEY: - return self.shares is not None - return key in self.facts - - def __repr__(self) -> str: - return ( - f"PreparedCandidateIntent(block_hash={self.block_hash!r}, " - f"candidate_sha256={self.candidate_sha256!r}, " - f"shares={self.manifest.share_count}, bytes={self.manifest.byte_count})" - ) - - def replay_header(self) -> dict[str, Any]: - """The bounded typed summary the outbox row carries beside the body.""" - return replay_header_from_fields({**self.facts, "block_hash_hex": self.block_hash}) - - def release(self) -> None: - """Drop the body's hold on its source; the facts stay readable.""" - self.body.close() - - -def prepare_candidate_intent( - fields: Mapping[str, Any], - *, - chunk_bytes: int = CANDIDATE_BODY_CHUNK_BYTES, -) -> PreparedCandidateIntent: - """Validate and index an intent mapping without materializing its body. - - One bounded encode pass runs now -- the synchronous validation that - used to be a whole ``json.dumps`` -- producing the scalar manifest and - discarding the bytes and index events. The returned intent re-encodes - from the same immutable source whenever storage asks for chunks, and - refuses to publish if the source changed. - """ - if isinstance(fields, PreparedCandidateIntent): - if not fields.dirty: - return fields - # Mutated after preparation: re-derive from the current fields. - fields = dict(fields) - facts, shares = split_candidate_fields(fields) - block_hash = str(facts.get("block_hash_hex", "")).lower() - if not block_hash: - raise CandidateCodecError("block candidate is missing block_hash_hex") - facts["block_hash_hex"] = block_hash - identity, present, stamp = neutralized_identity_facts(facts) - try: - manifest = encode_identity_body(identity, shares, chunk_bytes=chunk_bytes) - except CandidateCodecError: - raise - except TypeError: - # Exactly the exception the historical whole-document json.dumps - # raised for an unsupported value type; callers key on it. - raise - except ValueError as exc: - raise CandidateCodecError(str(exc)) from exc - key_order = tuple(str(key) for key in fields.keys()) - return PreparedCandidateIntent( - facts=facts, - shares=shares, - body=SourceCandidateBody(identity, shares, manifest), - accepted_at_present=present, - accepted_at_ms=stamp, - key_order=key_order, - ) - - -def prepared_intent_from_spool( - body: SpoolCandidateBody, - *, - accepted_at_present: bool, - accepted_at_ms: Any, -) -> PreparedCandidateIntent: - """Rebuild the mapping view of a hydrated body. - - Facts stay a plain dict; each value follows the view module's decode - policy. ``pending_share`` alone is shallow-copied into a dict when it - hydrates as a lazy object, so the acknowledgment stamp is restored the - historical way (its members keep the policy: an oversized ``share_id`` - is a streamed string). - """ - identity = decode_spool_small_fields(body) - pending_share = identity.get("pending_share") - if ( - accepted_at_present - and isinstance(pending_share, Mapping) - and not isinstance(pending_share, dict) - ): - try: - identity["pending_share"] = dict(pending_share.items()) - except BaseException: - # The view's scratch index must not outlive this failure even - # while a retained traceback keeps the view itself reachable. - pending_share.close() - raise - facts = restore_pending_stamp( - identity, - present=accepted_at_present, - accepted_at_ms=accepted_at_ms, - ) - shares_span = body.index.span(CANDIDATE_SHARES_KEY) - if shares_span is None: - if body.manifest.share_count or body.manifest.shares_end > body.manifest.shares_offset: - raise CandidateBodyIntegrityError("spool body carries no share span") - shares = None - keys = sorted(facts.keys()) - else: - shares = SpoolJsonArraySequence(body, shares_span) - keys = sorted([*facts.keys(), CANDIDATE_SHARES_KEY]) - return PreparedCandidateIntent( - facts=facts, - shares=shares, - body=body, - accepted_at_present=accepted_at_present, - accepted_at_ms=accepted_at_ms, - key_order=tuple(keys), - ) - - -class SpoolWriter: - """Write verified chunks and index events to a spool + index pair.""" - - def __init__(self, path: str, index_path: str, manifest: CandidateBodyManifest) -> None: - self.manifest = manifest - self.path = path - self.index = SpoolFieldIndex.create(index_path) - self._handle = open(path, "wb") - self._digest = hashlib.sha256() - self._written = 0 - self._ordinal = 0 - self._span_count = 0 - self._page_count = 0 - self._open_field: str | None = None - - def chunk(self, chunk: BodyChunk) -> None: - manifest = self.manifest - if chunk.ordinal != self._ordinal: - raise CandidateBodyIntegrityError("spool chunks arrived out of order") - if self._ordinal >= manifest.chunk_count: - raise CandidateBodyIntegrityError("more chunks than the manifest declares") - expected_length = ( - manifest.chunk_bytes - if self._ordinal + 1 < manifest.chunk_count - else manifest.byte_count - manifest.chunk_bytes * (manifest.chunk_count - 1) - ) - if len(chunk.data) != expected_length: - raise CandidateBodyIntegrityError("spool chunk length disagrees with the manifest") - if hashlib.sha256(chunk.data).hexdigest() != chunk.sha256: - raise CandidateBodyIntegrityError("spool chunk digest disagrees with its bytes") - self._handle.write(chunk.data) - self._digest.update(chunk.data) - self._written += len(chunk.data) - self._ordinal += 1 - - def span(self, span: FieldSpan) -> None: - self.index.begin_field(span) - self._span_count += 1 - - def pages(self, entries: Iterable[PageEntry]) -> None: - batch = list(entries) - if not batch: - return - field_name = batch[0].field - if any(entry.field != field_name for entry in batch): - raise CandidateBodyIntegrityError("page batch spans several fields") - self.index.append_pages(field_name, ((entry.offset, entry.record_index) for entry in batch)) - self._page_count += len(batch) - - def finish(self, *, release: Callable[[], None] | None = None) -> SpoolCandidateBody: - """Verify and hand over the one body handle that owns the files. - - Exactly one :class:`SpoolCandidateBody` is created here: its - finalizer unlinks the spool pair, so a second handle on the same - path would delete the files as soon as the first was dropped. - """ - self._handle.close() - manifest = self.manifest - if self._ordinal != manifest.chunk_count or self._written != manifest.byte_count: - raise CandidateBodyIntegrityError("spool body is incomplete") - if self._digest.hexdigest() != manifest.candidate_sha256: - raise CandidateBodyIntegrityError("spool body digest disagrees with the manifest") - if self._span_count != manifest.span_count or self._page_count != manifest.page_count: - raise CandidateBodyIntegrityError("spool index disagrees with the manifest") - return SpoolCandidateBody(self.path, manifest, self.index, release=release) - - def abort(self) -> None: - try: - self._handle.close() - finally: - for target in (self.path, self.index.path): - try: - os.unlink(target) - except OSError: - pass - - -def write_spool_from_body( - path: str, - index_path: str, - body: CandidateBody, -) -> SpoolCandidateBody: - """Materialize any body (with its index events) to a spool pair.""" - writer = SpoolWriter(path, index_path, body.manifest) - try: - body.write_chunks(writer.chunk, on_span=writer.span, on_page=writer.pages) - return writer.finish() - except BaseException: - writer.abort() - raise - - -# -------------------------------------------------------------------------- -# legacy (v1 jsonb) normalization -- runs in the isolated helper only -# -------------------------------------------------------------------------- - - -def legacy_json_to_spool( - text: str, - path: str, - index_path: str, - *, - chunk_bytes: int = CANDIDATE_BODY_CHUNK_BYTES, - spool_limit_bytes: int | None = None, -) -> tuple[CandidateBodyManifest, list[FieldSpan], bool, Any]: - """Normalize one legacy ``candidate`` JSON text into a spool body pair. - - Whole-document work by design: this runs only inside the compatibility - helper process, never in the lease-bearing coordinator. Returns the - scalar manifest, the span rows (one per large field), and the pending - stamp's presence and original value; the page entries are in the index - file. - """ - document = json.loads(text) - if not isinstance(document, dict): - raise CandidateCodecError("legacy candidate is not an object") - facts, shares = split_candidate_fields(document) - identity, present, stamp = neutralized_identity_facts(facts) - index = SpoolFieldIndex.create(index_path) - spans: list[FieldSpan] = [] - written = 0 - - def admit(size: int) -> None: - nonlocal written - if spool_limit_bytes is not None and written + size > spool_limit_bytes: - raise OSError("legacy candidate spool reservation exhausted") - written += size - - def on_span(span: FieldSpan) -> None: - index.begin_field(span) - spans.append(span) - - with open(path, "wb") as handle: - def write_chunk(chunk: BodyChunk) -> None: - admit(len(chunk.data)) - handle.write(chunk.data) - - def write_pages(batch: list[PageEntry]) -> None: - admit(len(batch) * _INDEX_RECORD.size) - index.append_pages(batch[0].field, ((entry.offset, entry.record_index) for entry in batch)) - - manifest = encode_identity_body( - identity, - shares, - chunk_bytes=chunk_bytes, - on_chunk=write_chunk, - on_span=on_span, - on_page=write_pages, - ) - return manifest, spans, present, stamp - - -def jsonb_equivalent(left: Any, right: Any) -> bool: - """PostgreSQL ``jsonb`` equality over two decoded documents. - - Numbers compare as exact decimals (``1.0 = 1``, ``1e16 = 10^16``), the - boolean/null/number/string/array/object type classes never cross, and - objects compare as key sets regardless of order. Decode both sides with - ``parse_float=decimal.Decimal`` for exactness. - """ - from decimal import Decimal - - def kind(value: Any) -> str: - if value is None: - return "null" - if isinstance(value, bool): - return "bool" - if isinstance(value, (int, float, Decimal)): - return "number" - if isinstance(value, str): - return "string" - if isinstance(value, list): - return "array" - if isinstance(value, dict): - return "object" - raise TypeError(f"not a JSON value: {type(value).__name__}") - - left_kind = kind(left) - if left_kind != kind(right): - return False - if left_kind == "number": - return Decimal(str(left)) == Decimal(str(right)) - if left_kind in {"null", "bool", "string"}: - return left == right - if left_kind == "array": - return len(left) == len(right) and all( - jsonb_equivalent(a, b) for a, b in zip(left, right, strict=True) - ) - if set(left) != set(right): - return False - return all(jsonb_equivalent(left[key], right[key]) for key in left) - - -__all__ = [ - "CANDIDATE_BODY_CHUNK_BYTES", - "CANDIDATE_BODY_STORAGE_VERSION", - "CANDIDATE_INTENT_SCHEMA", - "CANDIDATE_SHARES_KEY", - "CODEC_BATCH_RECORDS", - "CODEC_BATCH_TARGET_BYTES", - "CODEC_FAST_PATH_BYTES", - "CODEC_STRING_SLICE_CHARS", - "HEADER_TEXT_FIELD_BYTES", - "INDEX_PAGE_ENTRIES", - "INDEX_SPAN_ROWS", - "LEGACY_CANDIDATE_STORAGE_VERSION", - "BodyChunk", - "CandidateBody", - "CandidateBodyIntegrityError", - "CandidateBodyManifest", - "CandidateCodecError", - "FieldSpan", - "PageEntry", - "PreparedCandidateIntent", - "SourceCandidateBody", - "SpoolCandidateBody", - "SpoolFieldIndex", - "SpoolJsonArraySequence", - "SpoolShareJsonSequence", - "SpoolWriter", - "chunk_ordinals_for", - "decode_json_string_span", - "decode_spool_small_fields", - "encode_identity_body", - "jsonb_equivalent", - "legacy_json_to_spool", - "neutralized_identity_facts", - "prepare_candidate_intent", - "prepared_intent_from_spool", - "replay_header_from_fields", - "restore_pending_stamp", - "split_candidate_fields", - "write_spool_from_body", -] diff --git a/lab/prism/candidate_spool_view.py b/lab/prism/candidate_spool_view.py deleted file mode 100644 index 7d9608d19..000000000 --- a/lab/prism/candidate_spool_view.py +++ /dev/null @@ -1,1029 +0,0 @@ -#!/usr/bin/env python3 -"""Bounded lazy views over a hydrated candidate spool body (issue #255). - -A hydrated block-candidate body is the v1 identity JSON on a local spool -file. The historical layout puts no bound on any single value: a share's -``share_id``, the miner ``username`` or a ``recipient_id`` may be of any -length, and ``found_block`` may carry nested arrays. The first reader -decoded every value whole, so one oversized field became one unbounded -``json.loads`` call in the lease-bearing coordinator, and the size ceilings -that guarded those calls rejected valid bodies as corruption. - -This module replaces the whole-value decode with one policy applied to -every JSON value in the body: - -* a value whose encoding is at most :data:`SPOOL_VIEW_DECODE_BYTES` is - decoded with one ``json.loads`` bounded by that size and returned as the - plain Python value it always was; -* a larger value becomes a lazy view over its own byte range: a - :class:`SpoolObjectView` (a ``Mapping``), a :class:`SpoolArrayView` (a - ``Sequence``), a :class:`SpoolStringView` (a streamed string) or a - :class:`SpoolRawValue` (an oversized scalar token). Members and items of - a lazy container are decoded on access under the same policy, so the - largest C call the reader ever makes is bounded by the threshold, at any - nesting depth, for a body of any size. - -Nothing is rejected for size. Every view exposes its original encoding -verbatim (``iter_byte_chunks``, ``iter_encoded_chunks``) so digests and -re-staging never need the decoded value, and a streamed string exposes its -text through ``iter_text_chunks`` in pieces that never split an escape or a -surrogate pair. The streaming protocol is the audit adapters' one -(``StreamedJsonString``, ``RawJsonDocument`` in ``audit_bundle_view``), so -the compact-body encoder embeds these views without materializing them. - -Structure is located with a stateful byte scanner (quoted text and nested -containers are skipped, escape pairs and multi-byte UTF-8 sequences are -carried across read boundaries), and per-container item indices live in -memory only up to a small cap, spilling to an unlinked scratch file beyond -it. A scratch file that cannot be created is -:class:`~lab.prism.audit_bundle_view.ArtifactResourcePressure` (retryable, -never corruption); a corrupt, truncated or closed body is -:class:`~lab.prism.candidate_codec.CandidateBodyIntegrityError`. - -Ownership: every view holds the :class:`SpoolCandidateBody` it reads and -nothing holds a view back, so there are no reference cycles and a dropped -view releases its scratch descriptor by reference count alone, with the -cyclic collector disabled. ``PreparedCandidateIntent.release()`` closes -the body; views read after that fail closed. -""" - -from __future__ import annotations - -import codecs -import hashlib -import json -import os -import re -import struct -import tempfile -import threading -import weakref -from collections.abc import ItemsView, Iterator, Mapping, Sequence, ValuesView -from typing import Any - -from lab.prism.audit_bundle_view import ArtifactResourcePressure -from lab.prism.candidate_codec import ( - SPOOL_READ_BYTES, - CandidateBodyIntegrityError, - SpoolCandidateBody, -) - -# Hard ceiling on the encoded size of any value decoded in-process with one -# ``json.loads``. A larger value is a lazy view, never a Python object. -SPOOL_VIEW_DECODE_BYTES = 1024 * 1024 -# Bytes of encoded string text decoded per ``json.loads`` while streaming. -SPOOL_VIEW_STRING_SLICE_BYTES = 64 * 1024 -# Bytes read from the spool per positional read while scanning or copying. -SPOOL_VIEW_READ_BYTES = SPOOL_READ_BYTES -# Item-index entries held in memory before the index spills to a scratch file. -SPOOL_VIEW_INDEX_MEMORY_ENTRIES = 256 -# Object member keys are cached in memory only while the object has at most -# this many members, each key at most this many encoded bytes. -SPOOL_VIEW_KEY_CACHE_ENTRIES = 1024 -SPOOL_VIEW_KEY_CACHE_BYTES = 4096 -# Top-level string fields hydrated as ``str`` whatever their size, each -# decoded in bounded slices and joined. Every one has a bound of its own -# that is not this reader's to impose -- the coinbase transaction and the -# hashes by consensus (the block), the extranonces by the Stratum session, -# the schema by the protocol -- and its consumers need the text whole (hex -# parsing, txid computation, identity checks). The raw block is absent on -# purpose: consensus bounds it too, but a submitter can stream it, so above -# the threshold it stays a streamed view. This is the explicit decoded -# fixed-metadata boundary; nothing outside it is ever joined implicitly. -SPOOL_DECODED_METADATA_FIELDS = frozenset( - { - "schema", - "block_hash_hex", - "parent_hash", - "coinbase_tx_hex", - "extranonce1_hex", - "extranonce2_hex", - } -) -# Pending index bytes buffered before they are written to the scratch file. -_INDEX_FLUSH_BYTES = 64 * 1024 - -_STRUCTURAL = re.compile(rb'["\\{}\[\],:]') -_HIGH_SURROGATE_TAIL = re.compile(r"\\u[dD][89abAB][0-9a-fA-F]{2}$") - -_QUOTE = 0x22 -_BACKSLASH = 0x5C -_COMMA = 0x2C -_COLON = 0x3A -_OPEN = (0x7B, 0x5B) -_CLOSE = (0x7D, 0x5D) - - -# -------------------------------------------------------------------------- -# bounded body access -# -------------------------------------------------------------------------- - - -def read_body_span(body: SpoolCandidateBody, start: int, end: int) -> bytes: - """Body bytes ``[start, end)``; a closed or unreadable body fails closed.""" - if end <= start: - return b"" - if not body.alive: - raise CandidateBodyIntegrityError("spool body is closed") - try: - return body.read_span(start, end) - except ValueError as exc: - raise CandidateBodyIntegrityError(f"spool span is out of range: {exc}") from exc - except OSError as exc: - raise CandidateBodyIntegrityError(f"spool body is unreadable: {exc}") from exc - - -def iter_body_bytes( - body: SpoolCandidateBody, - start: int, - end: int, - *, - chunk_bytes: int | None = None, -) -> Iterator[bytes]: - """Body bytes ``[start, end)`` in bounded pieces.""" - size = max(1, int(chunk_bytes if chunk_bytes is not None else SPOOL_VIEW_READ_BYTES)) - position = start - while position < end: - stop = min(end, position + size) - yield read_body_span(body, position, stop) - position = stop - - -def iter_body_text(body: SpoolCandidateBody, start: int, end: int) -> Iterator[str]: - """Body bytes ``[start, end)`` decoded as UTF-8 text, chunk by chunk.""" - decoder = codecs.getincrementaldecoder("utf-8")() - try: - for chunk in iter_body_bytes(body, start, end): - text = decoder.decode(chunk) - if text: - yield text - tail = decoder.decode(b"", final=True) - except UnicodeDecodeError as exc: - raise CandidateBodyIntegrityError(f"spool body is not valid UTF-8: {exc}") from exc - if tail: - yield tail - - -def _loads(data: bytes) -> Any: - try: - return json.loads(data) - except ValueError as exc: - raise CandidateBodyIntegrityError(f"spool value is not valid JSON: {exc}") from exc - - -# -------------------------------------------------------------------------- -# structural scanner -# -------------------------------------------------------------------------- - - -def _iter_top_level_tokens( - body: SpoolCandidateBody, - start: int, - end: int, -) -> Iterator[tuple[int, int]]: - """``(position, byte)`` of every ``,`` and ``:`` at depth 0 in ``[start, end)``. - - Quoted text (escape-aware, including an escape pair split by a read - boundary) and nested containers are skipped; multi-byte UTF-8 never - matches a structural byte, so raw UTF-8 text is safe. The range must - end outside any string or container. - """ - in_string = False - escape_at = -2 - depth = 0 - position = start - read_bytes = max(1, int(SPOOL_VIEW_READ_BYTES)) - while position < end: - stop = min(end, position + read_bytes) - piece = read_body_span(body, position, stop) - for match in _STRUCTURAL.finditer(piece): - at = position + match.start() - token = piece[match.start()] - if in_string: - if at == escape_at + 1: - escape_at = -2 - elif token == _BACKSLASH: - escape_at = at - elif token == _QUOTE: - in_string = False - elif token == _QUOTE: - in_string = True - elif token in _OPEN: - depth += 1 - elif token in _CLOSE: - depth -= 1 - if depth < 0: - raise CandidateBodyIntegrityError( - f"spool value has unmatched delimiters at byte {at}" - ) - elif depth == 0: - yield at, token - position = stop - if in_string or depth: - raise CandidateBodyIntegrityError("spool value ends inside a string or container") - - -def iter_item_spans( - body: SpoolCandidateBody, - start: int, - end: int, -) -> Iterator[tuple[int, int]]: - """``(item_start, item_end)`` of the comma-joined values in ``[start, end)``. - - The range is an array body without its brackets or a page of records - (the codec's page index points at exactly such ranges). An empty range - holds no items. - """ - if end <= start: - return - item_start = start - for at, token in _iter_top_level_tokens(body, start, end): - if token == _COMMA: - yield item_start, at - item_start = at + 1 - else: - raise CandidateBodyIntegrityError(f"spool array holds a stray colon at byte {at}") - yield item_start, end - - -def iter_member_spans( - body: SpoolCandidateBody, - start: int, - end: int, -) -> Iterator[tuple[int, int, int, int]]: - """``(key_start, key_end, value_start, value_end)`` of the members in ``[start, end)``. - - The range is an object body without its braces. An empty range holds - no members. - """ - if end <= start: - return - member_start = start - colon = -1 - for at, token in _iter_top_level_tokens(body, start, end): - if token == _COLON: - if colon >= 0: - raise CandidateBodyIntegrityError(f"spool object member has two colons at byte {at}") - colon = at - else: - if colon < 0: - raise CandidateBodyIntegrityError(f"spool object member has no colon at byte {at}") - yield member_start, colon, colon + 1, at - member_start = at + 1 - colon = -1 - if colon < 0: - raise CandidateBodyIntegrityError("spool object member has no colon") - yield member_start, colon, colon + 1, end - - -# -------------------------------------------------------------------------- -# disk-spilling offset index -# -------------------------------------------------------------------------- - - -class _OffsetIndex: - """Append-only fixed-width offset entries, bounded in memory. - - Entries stay in a bytearray up to :data:`SPOOL_VIEW_INDEX_MEMORY_ENTRIES` - and move to an unlinked scratch file beyond that, read back with - ``pread``. The scratch descriptor closes with the last owner and on - :meth:`close`; a scratch file that cannot be created or written is - resource pressure, not corruption. - """ - - __slots__ = ( - "__weakref__", - "_closed", - "_count", - "_file", - "_finalizer", - "_memory", - "_memory_entries", - "_pending", - "_struct", - "_written", - ) - - def __init__(self, fields: int, *, memory_entries: int | None = None) -> None: - self._struct = struct.Struct("<" + "Q" * int(fields)) - self._memory_entries = int( - SPOOL_VIEW_INDEX_MEMORY_ENTRIES if memory_entries is None else memory_entries - ) - self._memory = bytearray() - self._pending = bytearray() - self._count = 0 - self._written = 0 - self._file: Any = None - self._finalizer: weakref.finalize | None = None - self._closed = False - - @property - def count(self) -> int: - return self._count - - @property - def spilled(self) -> bool: - return self._file is not None - - def fileno(self) -> int | None: - return None if self._file is None else self._file.fileno() - - def append(self, *values: int) -> None: - if self._closed: - raise CandidateBodyIntegrityError("spool view index is closed") - packed = self._struct.pack(*values) - if self._file is None: - if self._count < self._memory_entries: - self._memory += packed - self._count += 1 - return - self._spill() - self._pending += packed - self._count += 1 - if len(self._pending) >= _INDEX_FLUSH_BYTES: - self._flush() - - def _spill(self) -> None: - try: - handle = tempfile.TemporaryFile() - except OSError as exc: - raise ArtifactResourcePressure( - f"cannot create the spool view index scratch file: {exc}" - ) from exc - self._file = handle - self._finalizer = weakref.finalize(self, handle.close) - self._pending = bytearray(self._memory) - self._memory = bytearray() - self._flush() - - def _flush(self) -> None: - if not self._pending: - return - if self._file is None or self._file.closed: - raise CandidateBodyIntegrityError("spool view index is closed") - data = memoryview(bytes(self._pending)) - offset = self._written * self._struct.size - fd = self._file.fileno() - while data: - try: - written = os.pwrite(fd, data, offset) - except InterruptedError: - continue - except OSError as exc: - raise ArtifactResourcePressure( - f"cannot write the spool view index scratch file: {exc}" - ) from exc - data = data[written:] - offset += written - self._written += len(self._pending) // self._struct.size - self._pending = bytearray() - - def get(self, index: int) -> tuple[int, ...]: - if not 0 <= index < self._count: - raise IndexError(index) - if self._closed: - raise CandidateBodyIntegrityError("spool view index is closed") - if self._file is None: - return self._struct.unpack_from(self._memory, index * self._struct.size) - self._flush() - if self._file.closed: - raise CandidateBodyIntegrityError("spool view index is closed") - position = index * self._struct.size - try: - raw = os.pread(self._file.fileno(), self._struct.size, position) - except OSError as exc: - raise ArtifactResourcePressure( - f"cannot read the spool view index scratch file: {exc}" - ) from exc - if len(raw) != self._struct.size: - raise CandidateBodyIntegrityError("spool view index is truncated") - return self._struct.unpack(raw) - - def close(self) -> None: - self._closed = True - self._memory = bytearray() - self._pending = bytearray() - if self._finalizer is not None: - self._finalizer() - - @property - def closed(self) -> bool: - return self._closed - - -# -------------------------------------------------------------------------- -# views -# -------------------------------------------------------------------------- - - -class _SpoolView: - """One byte range of a body, exposed verbatim.""" - - __slots__ = ("_body", "_end", "_start") - - def __init__(self, body: SpoolCandidateBody, start: int, end: int) -> None: - self._body = body - self._start = int(start) - self._end = int(end) - - @property - def body(self) -> SpoolCandidateBody: - return self._body - - @property - def byte_range(self) -> tuple[int, int]: - return self._start, self._end - - @property - def byte_length(self) -> int: - return self._end - self._start - - def iter_byte_chunks(self, *, chunk_bytes: int | None = None) -> Iterator[bytes]: - """The value's exact encoding as stored, in bounded pieces.""" - return iter_body_bytes(self._body, self._start, self._end, chunk_bytes=chunk_bytes) - - def iter_encoded_chunks(self) -> Iterator[str]: - """The value's JSON encoding as text, in bounded pieces.""" - return iter_body_text(self._body, self._start, self._end) - - def canonical_json_sha256(self) -> str: - digest = hashlib.sha256() - for chunk in self.iter_byte_chunks(): - digest.update(chunk) - return digest.hexdigest() - - def close(self) -> None: - """Release scratch resources this view owns; the body is untouched.""" - - __hash__ = None # type: ignore[assignment] - - -def _text_stream_equals_str(chunks: Iterator[str], other: str) -> bool: - offset = 0 - for chunk in chunks: - stop = offset + len(chunk) - if stop > len(other) or other[offset:stop] != chunk: - return False - offset = stop - return offset == len(other) - - -def _text_streams_equal(left: Iterator[str], right: Iterator[str]) -> bool: - left_buffer = "" - right_buffer = "" - left_done = right_done = False - while True: - while not left_buffer and not left_done: - try: - left_buffer = next(left) - except StopIteration: - left_done = True - while not right_buffer and not right_done: - try: - right_buffer = next(right) - except StopIteration: - right_done = True - if left_done or right_done: - return left_done and right_done and not left_buffer and not right_buffer - size = min(len(left_buffer), len(right_buffer)) - if left_buffer[:size] != right_buffer[:size]: - return False - left_buffer = left_buffer[size:] - right_buffer = right_buffer[size:] - - -class SpoolStringView(_SpoolView): - """A JSON string above the decode threshold, streamed from the body. - - ``iter_text_chunks`` yields the decoded value in pieces cut only at - points that split neither an escape sequence nor a UTF-16 surrogate - pair; ``iter_encoded_chunks`` and ``iter_byte_chunks`` yield the stored - encoding. Equality with a ``str`` streams the comparison. ``str()`` is - refused deliberately: a consumer that needs the text whole must say so - through :func:`materialize_spool_views` (or join the chunks itself) so - a repr can never be stored where the value belongs. - """ - - __slots__ = () - - def iter_text_chunks(self, *, slice_bytes: int | None = None) -> Iterator[str]: - return iter_json_string_text_chunks( - self._body, self._start, self._end, slice_bytes=slice_bytes - ) - - def text_length(self) -> int: - """Length in code points, computed by streaming.""" - return sum(len(chunk) for chunk in self.iter_text_chunks()) - - def __eq__(self, other: object) -> bool: - if other is self: - return True - if isinstance(other, str): - return _text_stream_equals_str(self.iter_text_chunks(), other) - if isinstance(other, SpoolStringView): - return _text_streams_equal(self.iter_text_chunks(), other.iter_text_chunks()) - return NotImplemented - - def __ne__(self, other: object) -> bool: - result = self.__eq__(other) - if result is NotImplemented: - return result # type: ignore[return-value] - return not result - - def __bool__(self) -> bool: - return True - - def __str__(self) -> str: - raise TypeError( - "SpoolStringView is a streamed string; iterate iter_text_chunks() " - "or call materialize_spool_views() to obtain the text" - ) - - def __repr__(self) -> str: - return f"SpoolStringView(bytes={self.byte_length})" - - __hash__ = None # type: ignore[assignment] - - -class SpoolRawValue(_SpoolView): - """An oversized non-string scalar token (a number or literal). - - This encoder never produces one (integers are bounded by the - interpreter's digit limit), so it exists only so an unexpected body is - reported faithfully rather than refused: the token is reachable through - its encoding and equality compares encodings. - """ - - __slots__ = () - - def __eq__(self, other: object) -> bool: - if other is self: - return True - if isinstance(other, SpoolRawValue): - return _text_streams_equal(self.iter_encoded_chunks(), other.iter_encoded_chunks()) - return NotImplemented - - def __ne__(self, other: object) -> bool: - result = self.__eq__(other) - if result is NotImplemented: - return result # type: ignore[return-value] - return not result - - def __repr__(self) -> str: - return f"SpoolRawValue(bytes={self.byte_length})" - - __hash__ = None # type: ignore[assignment] - - -class _ObjectItemsView(ItemsView): - def __iter__(self) -> Iterator[tuple[str, Any]]: - return self._mapping._iter_items() # type: ignore[attr-defined] - - -class _ObjectValuesView(ValuesView): - def __iter__(self) -> Iterator[Any]: - for _key, value in self._mapping._iter_items(): # type: ignore[attr-defined] - yield value - - -class SpoolObjectView(_SpoolView, Mapping): - """A JSON object above the decode threshold, decoded member by member. - - The member index (key and value byte spans) is built by one structural - scan on first access and kept bounded in memory; keys are cached only - for objects with few, short keys, and otherwise looked up by scanning. - Each member value follows the module's decode policy, so a member can - itself be a view. Equality with a mapping compares member by member. - """ - - __slots__ = ("_index", "_keys", "_lock") - - def __init__(self, body: SpoolCandidateBody, start: int, end: int) -> None: - super().__init__(body, start, end) - self._index: _OffsetIndex | None = None - self._keys: dict[str, int] | None = None - self._lock = threading.Lock() - - def _ensure_indexed(self) -> _OffsetIndex: - index = self._index - if index is not None: - return index - with self._lock: - if self._index is not None: - return self._index - index = _OffsetIndex(4) - keys: dict[str, int] | None = {} - try: - for ordinal, (key_start, key_end, value_start, value_end) in enumerate( - iter_member_spans(self._body, self._start + 1, self._end - 1) - ): - index.append(key_start, key_end, value_start, value_end) - if keys is not None: - if ( - ordinal >= SPOOL_VIEW_KEY_CACHE_ENTRIES - or key_end - key_start > SPOOL_VIEW_KEY_CACHE_BYTES - ): - keys = None - else: - keys[self._decode_key(key_start, key_end)] = ordinal - except BaseException: - index.close() - raise - self._index = index - self._keys = keys - return index - - def _decode_key(self, start: int, end: int) -> str: - return decode_json_string(self._body, start, end) - - def _value_at(self, index: _OffsetIndex, ordinal: int) -> Any: - _key_start, _key_end, value_start, value_end = index.get(ordinal) - return decode_body_span(self._body, value_start, value_end) - - def _ordinal_of(self, key: str) -> int | None: - index = self._ensure_indexed() - if self._keys is not None: - return self._keys.get(key) - found: int | None = None - for ordinal in range(index.count): - key_start, key_end, _value_start, _value_end = index.get(ordinal) - if SpoolStringView(self._body, key_start, key_end) == key: - found = ordinal - return found - - def _iter_items(self) -> Iterator[tuple[str, Any]]: - index = self._ensure_indexed() - for ordinal in range(index.count): - key_start, key_end, value_start, value_end = index.get(ordinal) - yield self._decode_key(key_start, key_end), decode_body_span( - self._body, value_start, value_end - ) - - def __getitem__(self, key: str) -> Any: - if not isinstance(key, str): - raise KeyError(key) - ordinal = self._ordinal_of(key) - if ordinal is None: - raise KeyError(key) - return self._value_at(self._ensure_indexed(), ordinal) - - def __iter__(self) -> Iterator[str]: - index = self._ensure_indexed() - for ordinal in range(index.count): - key_start, key_end, _value_start, _value_end = index.get(ordinal) - yield self._decode_key(key_start, key_end) - - def __len__(self) -> int: - return self._ensure_indexed().count - - def __contains__(self, key: object) -> bool: - return isinstance(key, str) and self._ordinal_of(key) is not None - - def items(self) -> ItemsView: - return _ObjectItemsView(self) - - def values(self) -> ValuesView: - return _ObjectValuesView(self) - - def __eq__(self, other: object) -> bool: - if other is self: - return True - if not isinstance(other, Mapping): - return NotImplemented - if len(self) != len(other): - return False - for key, value in self._iter_items(): - if key not in other: - return False - if value != other[key]: - return False - return True - - def __ne__(self, other: object) -> bool: - result = self.__eq__(other) - if result is NotImplemented: - return result # type: ignore[return-value] - return not result - - def __repr__(self) -> str: - members = "?" if self._index is None else str(self._index.count) - return f"SpoolObjectView(bytes={self.byte_length}, members={members})" - - def close(self) -> None: - with self._lock: - index = self._index - if index is not None: - index.close() - - __hash__ = None # type: ignore[assignment] - - -class SpoolArrayView(_SpoolView, Sequence): - """A JSON array above the decode threshold, decoded item by item. - - The item index is built by one structural scan on first access and - kept bounded in memory (spilling to a scratch file). Items follow the - module's decode policy. ``canonical_share_pages`` hands the stored item - bytes to the codec's verbatim re-staging path, and equality with a - sequence compares item by item. - """ - - __slots__ = ("_index", "_lock") - - def __init__(self, body: SpoolCandidateBody, start: int, end: int) -> None: - super().__init__(body, start, end) - self._index: _OffsetIndex | None = None - self._lock = threading.Lock() - - def _ensure_indexed(self) -> _OffsetIndex: - index = self._index - if index is not None: - return index - with self._lock: - if self._index is not None: - return self._index - index = _OffsetIndex(2) - try: - for item_start, item_end in iter_item_spans( - self._body, self._start + 1, self._end - 1 - ): - index.append(item_start, item_end) - except BaseException: - index.close() - raise - self._index = index - return index - - def __len__(self) -> int: - return self._ensure_indexed().count - - def __iter__(self) -> Iterator[Any]: - index = self._ensure_indexed() - for ordinal in range(index.count): - item_start, item_end = index.get(ordinal) - yield decode_body_span(self._body, item_start, item_end) - - def __getitem__(self, index: int | slice) -> Any: - count = len(self) - if isinstance(index, slice): - start, stop, step = index.indices(count) - return tuple(self[position] for position in range(start, stop, step)) - if isinstance(index, bool) or not isinstance(index, int): - raise TypeError("spool array indices must be integers or slices") - resolved = index + count if index < 0 else index - if not 0 <= resolved < count: - raise IndexError(index) - item_start, item_end = self._ensure_indexed().get(resolved) - return decode_body_span(self._body, item_start, item_end) - - def canonical_share_pages(self) -> Iterator[tuple[bytes, int | None]]: - """The stored item bytes, verbatim, for the codec's re-staging path.""" - for chunk in iter_body_bytes(self._body, self._start + 1, self._end - 1): - yield chunk, None - - def __eq__(self, other: object) -> bool: - if other is self: - return True - if isinstance(other, (str, bytes, bytearray)) or not isinstance(other, Sequence): - return NotImplemented - if len(self) != len(other): - return False - other_iter = iter(other) - for item in self: - try: - candidate = next(other_iter) - except StopIteration: - return False - if item != candidate: - return False - return True - - def __ne__(self, other: object) -> bool: - result = self.__eq__(other) - if result is NotImplemented: - return result # type: ignore[return-value] - return not result - - def __repr__(self) -> str: - items = "?" if self._index is None else str(self._index.count) - return f"SpoolArrayView(bytes={self.byte_length}, items={items})" - - def close(self) -> None: - with self._lock: - index = self._index - if index is not None: - index.close() - - __hash__ = None # type: ignore[assignment] - - -# -------------------------------------------------------------------------- -# decode policy -# -------------------------------------------------------------------------- - - -def decode_body_span(body: SpoolCandidateBody, start: int, end: int) -> Any: - """One JSON value at ``[start, end)``: plain when small, a view when not. - - A value within :data:`SPOOL_VIEW_DECODE_BYTES` is one bounded - ``json.loads``; a larger one is classified by its first and last byte - without reading anything in between. - """ - size = end - start - if size <= 0: - raise CandidateBodyIntegrityError("spool value is empty") - if size <= SPOOL_VIEW_DECODE_BYTES: - return _loads(read_body_span(body, start, end)) - first = read_body_span(body, start, start + 1)[0] - last = read_body_span(body, end - 1, end)[0] - if first == 0x7B and last == 0x7D: - return SpoolObjectView(body, start, end) - if first == 0x5B and last == 0x5D: - return SpoolArrayView(body, start, end) - if first == _QUOTE and last == _QUOTE: - return SpoolStringView(body, start, end) - if first in (0x7B, 0x5B, _QUOTE) or last in (0x7D, 0x5D, _QUOTE): - raise CandidateBodyIntegrityError("spool value delimiters do not match") - return SpoolRawValue(body, start, end) - - -def _escape_is_real(text: str, at: int) -> bool: - """The backslash at ``at`` starts an escape (an even run precedes it).""" - preceding = at - len(text[:at].rstrip("\\")) - return preceding % 2 == 0 - - -def _safe_cut(text: str) -> int: - """Largest prefix of encoded string text that ends no escape mid-way. - - Three cuts, applied in order: an odd trailing backslash run leaves its - last backslash for the next piece; an incomplete ``\\uXXXX`` in the last - five characters moves whole; and a complete high-surrogate escape at the - end moves whole so its low half is decoded in the same ``json.loads`` - (two lone surrogates concatenated are not the code point they encode). - """ - cut = len(text) - run = cut - len(text.rstrip("\\")) - if run % 2 == 1: - cut -= 1 - else: - at = text.rfind("\\u", max(0, cut - 5)) - if at >= 0 and _escape_is_real(text, at): - cut = at - head = text[:cut] - match = _HIGH_SURROGATE_TAIL.search(head) - if match is not None and _escape_is_real(head, match.start()): - cut = match.start() - return cut - - -def iter_json_string_text_chunks( - body: SpoolCandidateBody, - start: int, - end: int, - *, - slice_bytes: int | None = None, -) -> Iterator[str]: - """Decoded text of the JSON string at ``[start, end)`` (quotes included). - - Each piece is one ``json.loads`` over at most ``slice_bytes`` of encoded - text plus a short carry; raw UTF-8 is decoded incrementally so a read - boundary never splits a code point either. - """ - # Clamped to the decode threshold so no call exceeds it however the - # knobs are set; at least 32 so a carried escape always makes progress. - size = max( - 32, - min( - int(slice_bytes if slice_bytes is not None else SPOOL_VIEW_STRING_SLICE_BYTES), - int(SPOOL_VIEW_DECODE_BYTES), - ), - ) - if end - start < 2: - raise CandidateBodyIntegrityError("spool string span is too short") - if read_body_span(body, start, start + 1) != b'"' or read_body_span(body, end - 1, end) != b'"': - raise CandidateBodyIntegrityError("spool string span is not quoted") - decoder = codecs.getincrementaldecoder("utf-8")() - position = start + 1 - stop = end - 1 - carry = "" - while position < stop: - read_to = min(stop, position + size) - try: - text = carry + decoder.decode(read_body_span(body, position, read_to), final=read_to >= stop) - except UnicodeDecodeError as exc: - raise CandidateBodyIntegrityError(f"spool string is not valid UTF-8: {exc}") from exc - position = read_to - if position < stop: - cut = _safe_cut(text) - carry = text[cut:] - text = text[:cut] - else: - carry = "" - if text: - try: - piece = json.loads('"' + text + '"') - except ValueError as exc: - raise CandidateBodyIntegrityError(f"spool string is not valid JSON: {exc}") from exc - yield piece - if carry: - raise CandidateBodyIntegrityError("spool string ends inside an escape") - - -def decode_json_string(body: SpoolCandidateBody, start: int, end: int) -> str: - """The whole text of the JSON string at ``[start, end)``. - - The explicit materialization boundary for callers that need the value - whole (for example a bounded metadata field); every C call stays a slice. - """ - return "".join(iter_json_string_text_chunks(body, start, end)) - - -# -------------------------------------------------------------------------- -# consumer helpers -# -------------------------------------------------------------------------- - - -def is_spool_view(value: object) -> bool: - return isinstance(value, _SpoolView) - - -def iter_string_text_chunks(value: Any) -> Iterator[str]: - """Text of a ``str`` or a :class:`SpoolStringView`, chunk by chunk. - - The one call a consumer needs to accept either shape (a COPY writer - streaming ``share_id`` values, a hashing loop). - """ - if isinstance(value, str): - if value: - yield value - return - if isinstance(value, SpoolStringView): - yield from value.iter_text_chunks() - return - raise TypeError(f"expected a string or a SpoolStringView, not {type(value).__name__}") - - -def materialize_spool_views(value: Any) -> Any: - """A plain Python object graph with every view decoded whole. - - The explicit boundary for consumers that must hold a value entire. - Strings are joined from their chunks, containers rebuilt member by - member; nothing here is bounded by anything but the value's own size, - which is why the call is explicit. - """ - if isinstance(value, SpoolStringView): - return "".join(value.iter_text_chunks()) - if isinstance(value, SpoolRawValue): - return json.loads("".join(value.iter_encoded_chunks())) - if isinstance(value, Mapping): - return {str(key): materialize_spool_views(item) for key, item in value.items()} - if isinstance(value, (str, bytes, bytearray)) or not isinstance(value, Sequence): - return value - return [materialize_spool_views(item) for item in value] - - -def close_spool_views(value: Any) -> None: - """Release the scratch resources of every view reachable from ``value``. - - Walks mappings and non-string sequences shallowly enough to find the - views a hydrated intent's facts hold; a page-indexed share sequence is - left alone (it owns nothing but the body reference). - """ - if isinstance(value, _SpoolView): - value.close() - return - if isinstance(value, Mapping): - for item in list(value.values()) if isinstance(value, dict) else (): - close_spool_views(item) - return - if isinstance(value, (list, tuple)): - for item in value: - close_spool_views(item) - - -__all__ = [ - "SPOOL_DECODED_METADATA_FIELDS", - "SPOOL_VIEW_DECODE_BYTES", - "SPOOL_VIEW_INDEX_MEMORY_ENTRIES", - "SPOOL_VIEW_KEY_CACHE_BYTES", - "SPOOL_VIEW_KEY_CACHE_ENTRIES", - "SPOOL_VIEW_READ_BYTES", - "SPOOL_VIEW_STRING_SLICE_BYTES", - "SpoolArrayView", - "SpoolObjectView", - "SpoolRawValue", - "SpoolStringView", - "close_spool_views", - "decode_body_span", - "decode_json_string", - "is_spool_view", - "iter_body_bytes", - "iter_body_text", - "iter_item_spans", - "iter_json_string_text_chunks", - "iter_member_spans", - "iter_string_text_chunks", - "materialize_spool_views", - "read_body_span", -] diff --git a/lab/prism/candidate_store.py b/lab/prism/candidate_store.py deleted file mode 100644 index 82fbabc80..000000000 --- a/lab/prism/candidate_store.py +++ /dev/null @@ -1,1687 +0,0 @@ -#!/usr/bin/env python3 -"""Chunked candidate body storage: statements, hydration, legacy lane, janitor. - -Issue #255. The ledger's candidate rows used to carry the whole intent as -one ``jsonb`` value, so every durable write inlined a payout-window-sized -literal, every replay page ``json_agg``-ed complete candidates, and psycopg -decoded them in one GIL-held call. This module owns the storage side of the -bounded replacement: - -* **Staging.** A body is uploaded as fixed byte chunks into - ``qbit_block_candidate_body_chunk`` under a manifest row whose owner is - the exact producer session, with its page index in - ``qbit_block_candidate_body_span``/``_page`` written in bounded batches. - Staging grants nothing: no share credit, no replay eligibility, no - lease-row touch. The seal statement locks the manifest row, proves - completeness from the chunk and index rows themselves and re-hashes - every chunk on the server; from then on the database's own triggers - refuse any chunk insert, update or delete until authoritative - retirement. -* **Publication.** The ledger's fenced statements (share append, standalone - intent persist) reference a sealed manifest by id and compare-and-swap on - the outbox row's observed version; the exact-body precomparison runs - here, outside that critical section, as a paged server-side chunk digest - check. -* **Metadata-first replay.** ``header_page_sql`` projects only a bounded - typed summary per row (``qbit_prism_bounded_replay_header``), at most - :data:`HEADER_PAGE_MAX_ROWS` rows and :data:`HEADER_PAGE_MAX_BYTES` per - page, and answers exhaustion explicitly. -* **Hydration.** :class:`CandidateBodyHydrator` reads the scalar manifest, - the span rows and the page rows in bounded pages (the page index goes - straight to disk), then at most :data:`BODY_READ_MAX_CHUNKS` chunks per - statement, verifying state, digests and a rolling body digest as it - goes. A body that turns terminal or corrupt mid-read is discarded. -* **Legacy lane.** Pending v1 ``jsonb`` rows are converted by an isolated - helper process (``python -m lab.prism.candidate_store``) that owns its - own database connection, cannot take the writer lease, is memory- and - deadline-limited, and hands back a spool/index pair plus a scalar - manifest, never a Python object graph. -* **Janitor.** Two separate statements: a tiny fenced compare-and-swap that - retires orphaned staging/sealed bodies (``retire_orphan_bodies_sql``), - and a lease-free bounded deletion of one retired body's chunks a page at - a time (``reap_retired_chunks_sql``). ``retired`` is permanent, and a - publication can only reference a ``sealed`` body, so nothing revives a - body once it is reapable. -""" - -from __future__ import annotations - -import base64 -import binascii -import json -import os -import shlex -import subprocess -import sys -import tempfile -import threading -import time -import uuid -from collections.abc import Callable, Mapping -from dataclasses import dataclass, field -from pathlib import Path -from typing import Any - -from lab.prism.candidate_codec import ( - CANDIDATE_BODY_CHUNK_BYTES, - CANDIDATE_BODY_STORAGE_VERSION, - INDEX_PAGE_ENTRIES, - INDEX_SPAN_ROWS, - LEGACY_CANDIDATE_STORAGE_VERSION, - BodyChunk, - CandidateBody, - CandidateBodyIntegrityError, - CandidateBodyManifest, - FieldSpan, - PageEntry, - PreparedCandidateIntent, - SpoolWriter, - jsonb_equivalent, - legacy_json_to_spool, - prepared_intent_from_spool, -) - -# Chunks per body read statement; three 256 KiB chunks base64-encode to -# exactly 1 MiB of returned text, which is the transport ceiling. -BODY_READ_MAX_CHUNKS = 3 -BODY_READ_MAX_BYTES = 1024 * 1024 -# Base64 text for one chunk: 4/3 of the bytes plus one newline per 76 chars. -CHUNK_BASE64_MAX_CHARS = ((CANDIDATE_BODY_CHUNK_BYTES + 2) // 3) * 4 + CANDIDATE_BODY_CHUNK_BYTES // 57 + 2 -# Header enumeration page: rows and running header bytes. The byte cap is -# the bound that matters for the native decoder (every header is a bounded -# projection, so a 256 KiB page holds a few hundred of them at most); the -# row cap matches the configured replay page maximum rather than a smaller -# number, because every page costs the collapse walk two chain tip reads and -# a probe budget, and a 64-row cap multiplied a storm's chain reads by -# sixteen in the existing cost contracts. -HEADER_PAGE_MAX_ROWS = 1024 -HEADER_PAGE_MAX_BYTES = 256 * 1024 -# Per-header ceiling enforced on read; the bounded projection is far below. -HEADER_MAX_BYTES = 8 * 1024 -# Local spool admission for hydrated bodies (operator-reviewed reservation). -DEFAULT_SPOOL_RESERVATION_BYTES = 4 * 1024 * 1024 * 1024 -# Chunk deletions per janitor statement and the age after which an -# unreferenced staging/sealed body is an orphan. -JANITOR_CHUNKS_PER_STEP = 64 -STALE_STAGING_SECONDS = 15 * 60 -# Legacy helper process limits. -LEGACY_HELPER_MEMORY_BYTES = 4 * 1024 * 1024 * 1024 -LEGACY_HELPER_TIMEOUT_SECONDS = 600.0 -LEGACY_HELPER_POLL_SECONDS = 0.2 -# The schema capability this code understands; a database that declares a -# higher one is newer than this process and must not be written by it. -CANDIDATE_SCHEMA_CAPABILITY = "candidate_storage_version" - -CHUNK_LITERAL_TAG = "qbit_prism_chunk" - - -class CandidateStorageError(RuntimeError): - """A storage operation failed in a way that is not a transport retry.""" - - -class CandidateBodyUnavailable(CandidateStorageError): - """The body no longer exists as a sealed, referenced manifest.""" - - -class SpoolAdmissionExhausted(CandidateStorageError): - """Local spool reservation would be exceeded; backpressure, not loss.""" - - -class IncompatibleCandidateSchema(CandidateStorageError): - """The database's candidate schema is newer than this process, or unmigrated.""" - - -def new_body_id() -> str: - return uuid.uuid4().hex - - -# -------------------------------------------------------------------------- -# typed rows -# -------------------------------------------------------------------------- - - -@dataclass(frozen=True) -class CandidateBodyRef: - """Small reference to a stored body, carried by a header row.""" - - body_id: str - storage_version: int - candidate_sha256: str - byte_count: int - chunk_count: int - chunk_bytes: int - share_count: int - - @classmethod - def from_row(cls, row: Mapping[str, Any]) -> CandidateBodyRef: - body_id = row.get("body_id") - if not isinstance(body_id, str) or len(body_id) != 32: - raise ValueError("candidate body reference carries no body id") - return cls( - body_id=body_id, - storage_version=int(row.get("storage_version", CANDIDATE_BODY_STORAGE_VERSION)), - candidate_sha256=str(row["candidate_sha256"]), - byte_count=int(row["byte_count"]), - chunk_count=int(row["chunk_count"]), - chunk_bytes=int(row["chunk_bytes"]), - share_count=int(row["share_count"]), - ) - - def to_json(self) -> dict[str, Any]: - return { - "body_id": self.body_id, - "storage_version": self.storage_version, - "candidate_sha256": self.candidate_sha256, - "byte_count": self.byte_count, - "chunk_count": self.chunk_count, - "chunk_bytes": self.chunk_bytes, - "share_count": self.share_count, - } - - -@dataclass(frozen=True) -class CandidateHeaderPage: - """One metadata-first replay page with an explicit completeness fact.""" - - rows: tuple[dict[str, Any], ...] - next_cursor: Any - exhausted: bool - fetched: int - truncated_by_bytes: bool - - def __iter__(self): # noqa: ANN204 - Sequence-like convenience - return iter(self.rows) - - def __len__(self) -> int: - return len(self.rows) - - def __getitem__(self, index: int) -> dict[str, Any]: - return self.rows[index] - - -@dataclass(frozen=True) -class DurableCandidateDescriptor: - """Everything replay needs about a pending row before its body exists. - - Built from one header row after stale collapse and strict typing. The - descriptor is what the bounded replay window holds; the body is - hydrated only when the descriptor is dequeued. - """ - - block_hash: str - block_height: int - parent_hash: str - candidate_sha256: str - storage_version: int - credit_share_on_accept: bool - collection_only: bool - # None only when the header is oversized: the fact was deferred to the - # body, which hydration reads (``replay_header_from_fields``). - username: str | None - pending_share: dict[str, Any] - accepted_at_present: bool - accepted_at_ms: Any - body: CandidateBodyRef | None - cursor: Any - # The header row this descriptor was built from, handed back to the - # ledger at hydration time (it carries the body reference and the - # original pending stamp). - row: dict[str, Any] = field(default_factory=dict, compare=False, repr=False) - header_oversized: bool = False - attempts: int = 0 - not_before_monotonic: float = 0.0 - - @property - def submission(self) -> Any: - """Duck-typed hash carrier for registries that read ``.submission``.""" - from types import SimpleNamespace - - return SimpleNamespace(block_hash_hex=self.block_hash) - - @property - def body_key(self) -> str: - return self.body.body_id if self.body is not None else "legacy" - - -# -------------------------------------------------------------------------- -# statement builders -# -------------------------------------------------------------------------- - -Literal = Callable[[str], str] -JsonbLiteral = Callable[[object], str] - - -def chunk_hex_literal(data: bytes) -> str: - return f"decode(${CHUNK_LITERAL_TAG}${data.hex()}${CHUNK_LITERAL_TAG}$, 'hex')" - - -def schema_capability_sql() -> str: - """Read the candidate storage version the schema declares.""" - return f""" -SELECT json_build_object( - 'declared', ( - SELECT capability_value - FROM qbit_prism_schema_capabilities - WHERE capability = '{CANDIDATE_SCHEMA_CAPABILITY}' - ) -); -""" - - -def candidate_schema_refusal(declared: int | None, has_body_table: bool) -> str | None: - """Why this process must not use the database's candidate schema, or None. - - Every storage version needs the 002 migration: replay projects headers - through its function and the terminal outbox statements retire body - references in its columns, so ``PRISM_CANDIDATE_STORAGE_VERSION=1`` - stops version-2 writes but never makes an unmigrated database usable. - The capability row is declared last, so its absence also marks a - partially applied migration. A declared version above this process's - is the rollback floor. Shared by the coordinator boot check and the - offline recovery reader. - """ - if declared is not None and declared > CANDIDATE_BODY_STORAGE_VERSION: - return ( - f"database declares candidate storage version {declared}; this " - f"process understands up to {CANDIDATE_BODY_STORAGE_VERSION} and " - "must not write to it (rollback floor)" - ) - if not has_body_table or declared is None or declared < CANDIDATE_BODY_STORAGE_VERSION: - return ( - "the database lacks the 002_candidate_bodies.sql migration; apply it " - "(PRISM_POSTGRES_INIT_SCHEMA=1 does so at startup) before starting " - "this release. PRISM_CANDIDATE_STORAGE_VERSION=1 only stops " - "version-2 writes: replay and terminal outbox statements use the " - "migrated schema for every storage version" - ) - return None - - -def stage_body_sql( - payload: Mapping[str, Any], - *, - jsonb: JsonbLiteral, -) -> str: - """Create one staging manifest; idempotent on ``body_id``. - - No lease CTE: staging is private preparation and must not touch the - lease row. The producer identity is recorded so chunk uploads and the - seal can require the same session, and so an orphan can be identified - by the janitor. - """ - return f""" -WITH input AS ( - SELECT {jsonb(payload)} AS data -), -inserted AS ( - INSERT INTO qbit_block_candidate_body ( - body_id, storage_version, block_hash, candidate_sha256, - byte_count, chunk_count, chunk_bytes, share_count, - shares_offset, shares_end, span_count, page_count, - staging_writer_id, staging_writer_epoch, staging_session_token - ) - SELECT - data->>'body_id', (data->>'storage_version')::integer, - data->>'block_hash', data->>'candidate_sha256', - (data->>'byte_count')::bigint, (data->>'chunk_count')::integer, - (data->>'chunk_bytes')::integer, (data->>'share_count')::bigint, - (data->>'shares_offset')::bigint, (data->>'shares_end')::bigint, - (data->>'span_count')::integer, (data->>'page_count')::bigint, - data->>'writer_id', (data->>'writer_epoch')::bigint, - data->>'writer_session_token' - FROM input - ON CONFLICT (body_id) DO NOTHING - RETURNING body_id -) -SELECT json_build_object( - 'staged', (SELECT count(*) FROM inserted), - -- A row this statement inserted is invisible to its own final SELECT - -- (one snapshot per statement), so the fresh case is answered from the - -- RETURNING CTE. - 'state', COALESCE( - ( - SELECT state FROM qbit_block_candidate_body body, input - WHERE body.body_id = input.data->>'body_id' - ), - CASE WHEN EXISTS (SELECT 1 FROM inserted) THEN 'staging' END - ) -); -""" - - -def _owner_cte(alias: str = "owner") -> str: - return f""" -{alias} AS ( - SELECT body.body_id - FROM qbit_block_candidate_body body, input - WHERE body.body_id = input.data->>'body_id' - AND body.state = 'staging' - AND body.staging_writer_id = input.data->>'writer_id' - AND body.staging_writer_epoch = (input.data->>'writer_epoch')::bigint - AND body.staging_session_token = input.data->>'writer_session_token' -)""" - - -def body_chunk_sql( - payload: Mapping[str, Any], - data: bytes, - *, - jsonb: JsonbLiteral, -) -> str: - """Upload one chunk; idempotent on ``(body_id, ordinal)``. - - The insert is guarded by the manifest still being in ``staging`` under - the uploading session (and, independently, by the chunk table's - trigger), so a sealed or retired body can never gain a chunk and a - foreign session cannot fill someone else's staging. - """ - return f""" -WITH input AS ( - SELECT {jsonb(payload)} AS data -),{_owner_cte()}, -inserted AS ( - INSERT INTO qbit_block_candidate_body_chunk (body_id, ordinal, chunk, chunk_sha256) - SELECT - owner.body_id, (input.data->>'ordinal')::integer, - {chunk_hex_literal(data)}, input.data->>'chunk_sha256' - FROM owner, input - ON CONFLICT (body_id, ordinal) DO NOTHING - RETURNING ordinal -) -SELECT json_build_object( - 'owned', EXISTS (SELECT 1 FROM owner), - 'inserted', (SELECT count(*) FROM inserted) -); -""" - - -def body_index_rows_sql( - payload: Mapping[str, Any], - *, - jsonb: JsonbLiteral, -) -> str: - """Upload one bounded batch of span rows and/or page rows. - - ``payload['spans']`` holds at most :data:`INDEX_SPAN_ROWS` span objects - and ``payload['pages']`` at most :data:`INDEX_PAGE_ENTRIES` page arrays - ``[field, ordinal, offset, record_index]``. Idempotent on the primary - keys, and owner-guarded exactly like a chunk upload. - """ - return f""" -WITH input AS ( - SELECT {jsonb(payload)} AS data -),{_owner_cte()}, -inserted_spans AS ( - INSERT INTO qbit_block_candidate_body_span ( - body_id, field, kind, start_offset, end_offset, item_count, page_count, pages_exact - ) - SELECT - owner.body_id, span->>'field', span->>'kind', - (span->>'start')::bigint, (span->>'end')::bigint, - (span->>'item_count')::bigint, (span->>'page_count')::bigint, - (span->>'pages_exact')::boolean - FROM owner, input, jsonb_array_elements(COALESCE(input.data->'spans', '[]'::jsonb)) AS span - ON CONFLICT (body_id, field) DO NOTHING - RETURNING field -), -inserted_pages AS ( - INSERT INTO qbit_block_candidate_body_page ( - body_id, field, page_ordinal, body_offset, record_index - ) - SELECT - owner.body_id, page->>0, (page->>1)::bigint, (page->>2)::bigint, (page->>3)::bigint - FROM owner, input, jsonb_array_elements(COALESCE(input.data->'pages', '[]'::jsonb)) AS page - ON CONFLICT (body_id, field, page_ordinal) DO NOTHING - RETURNING page_ordinal -) -SELECT json_build_object( - 'owned', EXISTS (SELECT 1 FROM owner), - 'spans', (SELECT count(*) FROM inserted_spans), - 'pages', (SELECT count(*) FROM inserted_pages) -); -""" - - -def seal_body_sql( - payload: Mapping[str, Any], - *, - jsonb: JsonbLiteral, -) -> str: - """Seal a complete, verified staging body under its producer session. - - The manifest row is locked (``FOR UPDATE``) so a concurrent chunk or - index upload serializes behind the seal and then finds the body sealed - (its trigger refuses the insert). Completeness is proven from the rows - themselves (chunk count, byte sum, ordinal range, span and page counts) - and integrity by re-hashing every stored chunk on the server against - the digest the uploader recorded. Nothing here trusts a client-side - upload count. - """ - return f""" -WITH input AS ( - SELECT {jsonb(payload)} AS data -), -target AS ( - SELECT body.* - FROM qbit_block_candidate_body body, input - WHERE body.body_id = input.data->>'body_id' - FOR UPDATE -), -observed AS ( - SELECT - count(*) AS chunk_count, - COALESCE(sum(octet_length(chunk)), 0) AS byte_count, - COALESCE(min(ordinal), 0) AS min_ordinal, - COALESCE(max(ordinal), -1) AS max_ordinal, - COALESCE(bool_and(sha256(chunk) = decode(chunk_sha256, 'hex')), true) AS digests_ok, - COALESCE(bool_and( - octet_length(chunk) = (SELECT chunk_bytes FROM target) - OR ordinal = (SELECT chunk_count FROM target) - 1 - ), true) AS lengths_ok - FROM qbit_block_candidate_body_chunk chunk - WHERE chunk.body_id = (SELECT body_id FROM target) -), -observed_index AS ( - SELECT - (SELECT count(*) FROM qbit_block_candidate_body_span span - WHERE span.body_id = (SELECT body_id FROM target)) AS span_count, - (SELECT count(*) FROM qbit_block_candidate_body_page page - WHERE page.body_id = (SELECT body_id FROM target)) AS page_count -), -complete AS ( - SELECT target.body_id - FROM target, observed, observed_index, input - WHERE target.state = 'staging' - AND target.staging_writer_id = input.data->>'writer_id' - AND target.staging_writer_epoch = (input.data->>'writer_epoch')::bigint - AND target.staging_session_token = input.data->>'writer_session_token' - AND target.candidate_sha256 = input.data->>'candidate_sha256' - AND observed.chunk_count = target.chunk_count - AND observed.byte_count = target.byte_count - AND (target.chunk_count = 0 OR observed.min_ordinal = 0) - AND observed.max_ordinal = target.chunk_count - 1 - AND observed.digests_ok - AND observed.lengths_ok - AND observed_index.span_count = target.span_count - AND observed_index.page_count = target.page_count -), -sealed AS ( - UPDATE qbit_block_candidate_body - SET state = 'sealed', sealed_at = clock_timestamp() - FROM complete - WHERE qbit_block_candidate_body.body_id = complete.body_id - RETURNING qbit_block_candidate_body.body_id -) -SELECT json_build_object( - 'sealed', (SELECT count(*) FROM sealed), - 'state', (SELECT state FROM target), - 'observed_chunk_count', (SELECT chunk_count FROM observed), - 'observed_byte_count', (SELECT byte_count FROM observed), - 'observed_span_count', (SELECT span_count FROM observed_index), - 'observed_page_count', (SELECT page_count FROM observed_index), - 'digests_ok', (SELECT digests_ok FROM observed), - 'lengths_ok', (SELECT lengths_ok FROM observed) -); -""" - - -def retire_staging_body_sql(body_id: str, *, text: Literal) -> str: - """Hand a failed staging body to the janitor (no lease; own body only).""" - return f""" -WITH retired AS ( - UPDATE qbit_block_candidate_body - SET state = 'retired', retired_at = clock_timestamp() - WHERE body_id = {text(body_id)} - AND state = 'staging' - RETURNING body_id -) -SELECT json_build_object('retired', (SELECT count(*) FROM retired)); -""" - - -def body_manifest_sql(body_id: str, *, text: Literal) -> str: - """Read one scalar manifest with its live state and reference fact.""" - return f""" -SELECT json_build_object( - 'found', EXISTS (SELECT 1 FROM qbit_block_candidate_body WHERE body_id = {text(body_id)}), - 'manifest', ( - SELECT json_build_object( - 'body_id', body_id, - 'storage_version', storage_version, - 'block_hash', block_hash, - 'candidate_sha256', candidate_sha256, - 'byte_count', byte_count, - 'chunk_count', chunk_count, - 'chunk_bytes', chunk_bytes, - 'share_count', share_count, - 'shares_offset', shares_offset, - 'shares_end', shares_end, - 'span_count', span_count, - 'page_count', page_count, - 'state', state, - 'referenced', EXISTS ( - SELECT 1 FROM qbit_block_candidate_outbox outbox - WHERE outbox.body_id = body.body_id AND outbox.state = 'pending' - ) - ) - FROM qbit_block_candidate_body body - WHERE body_id = {text(body_id)} - ) -); -""" - - -def body_spans_sql(body_id: str, after_field: str | None, *, text: Literal) -> str: - """Read at most :data:`INDEX_SPAN_ROWS` span rows, keyset by field name.""" - after = "" if after_field is None else f"\n AND field > {text(after_field)}" - return f""" -SELECT json_build_object( - 'state', (SELECT state FROM qbit_block_candidate_body WHERE body_id = {text(body_id)}), - 'spans', ( - SELECT COALESCE(json_agg(json_build_object( - 'field', span.field, 'kind', span.kind, - 'start', span.start_offset, 'end', span.end_offset, - 'item_count', span.item_count, 'page_count', span.page_count, - 'pages_exact', span.pages_exact - ) ORDER BY span.field), '[]'::json) - FROM ( - SELECT * FROM qbit_block_candidate_body_span - WHERE body_id = {text(body_id)}{after} - ORDER BY field - LIMIT {INDEX_SPAN_ROWS} - ) span - ) -); -""" - - -def body_pages_sql(body_id: str, field_name: str, from_ordinal: int, *, text: Literal) -> str: - """Read at most :data:`INDEX_PAGE_ENTRIES` page rows of one field.""" - return f""" -SELECT json_build_object( - 'state', (SELECT state FROM qbit_block_candidate_body WHERE body_id = {text(body_id)}), - 'pages', ( - SELECT COALESCE(json_agg(json_build_array( - page.page_ordinal, page.body_offset, page.record_index - ) ORDER BY page.page_ordinal), '[]'::json) - FROM ( - SELECT * FROM qbit_block_candidate_body_page - WHERE body_id = {text(body_id)} - AND field = {text(field_name)} - AND page_ordinal >= {int(from_ordinal)} - ORDER BY page_ordinal - LIMIT {INDEX_PAGE_ENTRIES} - ) page - ) -); -""" - - -def body_page_sql( - body_id: str, - from_ordinal: int, - max_chunks: int, - *, - text: Literal, -) -> str: - """Read at most ``max_chunks`` consecutive chunks of one sealed body. - - Bounded by the SELECT itself (``LIMIT``), so the PGresult is bounded - too -- not merely the client's fetch size. Base64 keeps the transport - text at four thirds of the raw bytes. - """ - max_chunks = max(1, min(int(max_chunks), BODY_READ_MAX_CHUNKS)) - return f""" -SELECT json_build_object( - 'state', (SELECT state FROM qbit_block_candidate_body WHERE body_id = {text(body_id)}), - 'referenced', EXISTS ( - SELECT 1 FROM qbit_block_candidate_outbox outbox - WHERE outbox.body_id = {text(body_id)} AND outbox.state = 'pending' - ), - 'chunks', ( - SELECT COALESCE(json_agg(json_build_object( - 'ordinal', page.ordinal, - 'sha256', page.chunk_sha256, - 'base64', encode(page.chunk, 'base64') - ) ORDER BY page.ordinal), '[]'::json) - FROM ( - SELECT ordinal, chunk_sha256, chunk - FROM qbit_block_candidate_body_chunk - WHERE body_id = {text(body_id)} - AND ordinal >= {int(from_ordinal)} - ORDER BY ordinal - LIMIT {max_chunks} - ) page - ) -); -""" - - -def header_page_sql( - limit: int, - after_cursor: tuple[str, str] | None, - max_bytes: int, - *, - text: Literal, -) -> str: - """Metadata-first pending page with explicit exhaustion. - - Projects the bounded typed replay header for both storage versions - through ``qbit_prism_bounded_replay_header``: v2 rows carry the - writer's projection in ``replay_header``; v1 rows have the same named - keys projected out of the legacy jsonb (explicit field projection, - never ``- 'shares_json'`` over an arbitrary document). Every fact is - bounded on the server, so a header has a provable maximum size and - the byte cap is a real bound even for the page's first row. The inner - query fetches one row beyond the row cap so the page can prove - exhaustion, and the running byte total cuts the page at ``max_bytes`` - while reporting that it did. - """ - limit = max(1, min(int(limit), HEADER_PAGE_MAX_ROWS)) - max_bytes = max(1, min(int(max_bytes), HEADER_PAGE_MAX_BYTES)) - after_predicate = "" - if after_cursor is not None: - created_at_text, cursor_block_hash = after_cursor - after_predicate = ( - "\n AND (created_at, block_hash) > " - f"({text(created_at_text)}::timestamptz, {text(cursor_block_hash)})" - ) - return f""" -WITH fetched AS ( - SELECT - block_hash, - storage_version, - candidate_sha256, - body_id, - created_at, - to_char( - created_at AT TIME ZONE 'UTC', - 'YYYY-MM-DD"T"HH24:MI:SS.US"Z"' - ) AS cursor_created_at, - qbit_prism_bounded_replay_header( - CASE - WHEN storage_version = 2 THEN replay_header - ELSE jsonb_build_object( - 'schema', candidate->'schema', - 'block_hash_hex', candidate->'block_hash_hex', - 'parent_hash', candidate->'parent_hash', - 'expected_height', candidate->'expected_height', - 'template', candidate->'template', - 'found_block', jsonb_build_object( - 'network_difficulty', candidate->'found_block'->'network_difficulty' - ), - 'pending_share', candidate->'pending_share', - 'credit_share_on_accept', candidate->'credit_share_on_accept', - 'collection_only', candidate->'collection_only', - 'username', candidate->'username', - 'accepted_at_present', (candidate->'pending_share') ? 'accepted_at_ms' - ) - END - ) AS header, - EXISTS ( - SELECT 1 - FROM qbit_pool_blocks pool - WHERE pool.block_hash = qbit_block_candidate_outbox.block_hash - ) AS pool_block_exists - FROM qbit_block_candidate_outbox - WHERE state = 'pending'{after_predicate} - ORDER BY created_at, block_hash - LIMIT {limit + 1} -), -measured AS ( - SELECT - fetched.*, - row_number() OVER (ORDER BY created_at, block_hash) AS row_position, - sum(octet_length(header::text)) OVER ( - ORDER BY created_at, block_hash - ROWS BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW - ) AS running_bytes - FROM fetched -), -page AS ( - SELECT measured.* - FROM measured - WHERE row_position <= {limit} - AND (row_position = 1 OR running_bytes <= {max_bytes}) -) -SELECT json_build_object( - 'fetched', (SELECT count(*) FROM fetched), - 'returned', (SELECT count(*) FROM page), - 'rows', ( - SELECT COALESCE(json_agg(json_build_object( - 'block_hash', page.block_hash, - 'storage_version', page.storage_version, - 'candidate_sha256', page.candidate_sha256, - 'header', page.header, - 'header_bytes', octet_length(page.header::text), - 'body', ( - SELECT json_build_object( - 'body_id', body.body_id, - 'storage_version', body.storage_version, - 'candidate_sha256', body.candidate_sha256, - 'byte_count', body.byte_count, - 'chunk_count', body.chunk_count, - 'chunk_bytes', body.chunk_bytes, - 'share_count', body.share_count, - 'state', body.state - ) - FROM qbit_block_candidate_body body - WHERE body.body_id = page.body_id - ), - 'pool_block_exists', page.pool_block_exists, - 'cursor', json_build_array(page.cursor_created_at, page.block_hash) - ) ORDER BY page.created_at, page.block_hash), '[]'::json) - FROM page - ) -); -""" - - -def retire_orphan_bodies_sql(payload: Mapping[str, Any], *, jsonb: JsonbLiteral) -> str: - """Tiny fenced compare-and-swap: orphaned bodies become ``retired``. - - Under the writer fence (a deposed session retires nothing), but the - only row this touches besides the lease is one manifest row: a - ``staging`` body older than ``stale_staging_seconds`` owned by another - session, or a ``sealed`` body older than that age that no outbox row - references. ``retired`` is permanent -- the manifest trigger permits no - transition out of it -- and publication references only ``sealed`` - bodies, so a retired body can never be revived. - - The race with a publication in flight is settled by the database, not - by this statement's snapshot: publication locks the body row ``FOR - SHARE`` while it checks the body is sealed, this statement locks it - ``FOR UPDATE``, and the manifest trigger refuses to retire a body any - outbox row references (its check runs with a fresh snapshot after the - lock wait). Whichever side loses the lock re-evaluates and fails - closed; in-process, both statements also serialize behind the ledger's - writer gate. - """ - return f""" -WITH input AS ( - SELECT {jsonb(payload)} AS data -), -lease AS ( - UPDATE qbit_ledger_writer_lease - SET updated_at = clock_timestamp() - FROM input - WHERE singleton - AND writer_id = input.data->>'writer_id' - AND writer_epoch = (input.data->>'writer_epoch')::bigint - AND writer_session_token = input.data->>'writer_session_token' - RETURNING writer_id -), -target AS ( - SELECT body.body_id - FROM qbit_block_candidate_body body, input - WHERE EXISTS (SELECT 1 FROM lease) - AND ( - ( - body.state = 'staging' - AND body.created_at < clock_timestamp() - - (input.data->>'stale_staging_seconds')::double precision * interval '1 second' - AND NOT ( - body.staging_writer_id = input.data->>'writer_id' - AND body.staging_session_token = input.data->>'writer_session_token' - ) - ) - OR ( - body.state = 'sealed' - AND body.sealed_at < clock_timestamp() - - (input.data->>'stale_staging_seconds')::double precision * interval '1 second' - ) - ) - AND NOT EXISTS ( - SELECT 1 FROM qbit_block_candidate_outbox outbox - WHERE outbox.body_id = body.body_id - ) - ORDER BY body.created_at, body.body_id - LIMIT 1 - FOR UPDATE OF body -), -retired AS ( - UPDATE qbit_block_candidate_body - SET state = 'retired', retired_at = clock_timestamp() - FROM target - WHERE qbit_block_candidate_body.body_id = target.body_id - AND qbit_block_candidate_body.state IN ('staging', 'sealed') - AND NOT EXISTS ( - SELECT 1 FROM qbit_block_candidate_outbox outbox - WHERE outbox.body_id = qbit_block_candidate_body.body_id - ) - RETURNING qbit_block_candidate_body.body_id -) -SELECT CASE - WHEN NOT EXISTS (SELECT 1 FROM lease) THEN - json_build_object('error', 'writer lease is not active') - ELSE json_build_object('retired', COALESCE((SELECT json_agg(body_id) FROM retired), '[]'::json)) -END; -""" - - -def reap_retired_chunks_sql(payload: Mapping[str, Any], *, jsonb: JsonbLiteral) -> str: - """Delete one bounded page of one retired body's parts; no lease touch. - - Runs entirely outside the lease row: ``retired`` is a permanent state - that nothing publishes from, so reclaiming its bytes needs no fence - and a deposed session doing it cannot harm anyone. - - Every step is bounded and every probe is an existence test, never a - count: at most ``max_chunks`` chunk rows go, then (once the statement's - snapshot shows no chunk left) at most ``max_chunks`` page rows, then - span rows, and the manifest itself only on a later step whose snapshot - finds every child table empty. The returned ``remaining`` flags let the - caller keep stepping until the body is gone, so a body with zero chunks - but many page rows never stalls. - """ - return f""" -WITH input AS ( - SELECT {jsonb(payload)} AS data -), -target AS ( - SELECT body.body_id - FROM qbit_block_candidate_body body - WHERE body.state = 'retired' - AND NOT EXISTS ( - SELECT 1 FROM qbit_block_candidate_outbox outbox - WHERE outbox.body_id = body.body_id - ) - ORDER BY body.retired_at, body.body_id - LIMIT 1 -), -before AS ( - SELECT - EXISTS (SELECT 1 FROM qbit_block_candidate_body_chunk chunk, target - WHERE chunk.body_id = target.body_id) AS chunks, - EXISTS (SELECT 1 FROM qbit_block_candidate_body_page page, target - WHERE page.body_id = target.body_id) AS pages, - EXISTS (SELECT 1 FROM qbit_block_candidate_body_span span, target - WHERE span.body_id = target.body_id) AS spans -), -deleted_chunks AS ( - DELETE FROM qbit_block_candidate_body_chunk - WHERE ctid IN ( - SELECT chunk.ctid - FROM qbit_block_candidate_body_chunk chunk, target - WHERE chunk.body_id = target.body_id - ORDER BY chunk.ordinal - LIMIT (SELECT (data->>'max_chunks')::integer FROM input) - ) - RETURNING body_id -), -deleted_pages AS ( - DELETE FROM qbit_block_candidate_body_page - WHERE NOT (SELECT chunks FROM before) - AND ctid IN ( - SELECT page.ctid - FROM qbit_block_candidate_body_page page, target - WHERE page.body_id = target.body_id - ORDER BY page.field, page.page_ordinal - LIMIT (SELECT (data->>'max_chunks')::integer FROM input) - ) - RETURNING body_id -), -deleted_spans AS ( - DELETE FROM qbit_block_candidate_body_span - WHERE NOT (SELECT chunks FROM before) - AND NOT (SELECT pages FROM before) - AND ctid IN ( - SELECT span.ctid - FROM qbit_block_candidate_body_span span, target - WHERE span.body_id = target.body_id - ORDER BY span.field - LIMIT (SELECT (data->>'max_chunks')::integer FROM input) - ) - RETURNING body_id -), -deleted_bodies AS ( - DELETE FROM qbit_block_candidate_body - WHERE body_id IN (SELECT body_id FROM target) - AND state = 'retired' - AND NOT (SELECT chunks FROM before) - AND NOT (SELECT pages FROM before) - AND NOT (SELECT spans FROM before) - RETURNING body_id -) -SELECT json_build_object( - 'body_id', (SELECT body_id FROM target), - 'deleted_chunks', (SELECT count(*) FROM deleted_chunks), - 'deleted_pages', (SELECT count(*) FROM deleted_pages), - 'deleted_spans', (SELECT count(*) FROM deleted_spans), - 'deleted_bodies', (SELECT count(*) FROM deleted_bodies), - 'remaining', (SELECT json_build_object('chunks', chunks, 'pages', pages, 'spans', spans) FROM before) -); -""" - - -# -------------------------------------------------------------------------- -# spool admission -# -------------------------------------------------------------------------- - - -class SpoolAdmission: - """Global byte accounting for hydrated spool files.""" - - def __init__( - self, - directory: str | Path | None = None, - *, - limit_bytes: int = DEFAULT_SPOOL_RESERVATION_BYTES, - ) -> None: - self._lock = threading.Lock() - self._limit = int(limit_bytes) - self._reserved = 0 - self._directory = ( - Path(directory) - if directory is not None - else Path(tempfile.gettempdir()) / "qbit-prism-candidate-spool" - ) - - @property - def directory(self) -> Path: - return self._directory - - def snapshot(self) -> dict[str, int]: - with self._lock: - return {"reserved_bytes": self._reserved, "limit_bytes": self._limit} - - def reserve(self, byte_count: int) -> Callable[[], None]: - """Reserve ``byte_count``; returns the idempotent release callable.""" - byte_count = max(0, int(byte_count)) - with self._lock: - if self._reserved + byte_count > self._limit: - raise SpoolAdmissionExhausted( - f"candidate spool reservation exhausted: {self._reserved + byte_count} " - f"> {self._limit} bytes" - ) - self._reserved += byte_count - released = False - lock = self._lock - - def release() -> None: - nonlocal released - with lock: - if not released: - released = True - self._reserved -= byte_count - - return release - - def new_spool_paths(self, block_hash: str) -> tuple[str, str]: - """A fresh ``(body path, index path)`` pair in the spool directory.""" - self._directory.mkdir(parents=True, exist_ok=True) - handle, path = tempfile.mkstemp( - prefix=f"candidate-{block_hash[:16]}-", - suffix=".body", - dir=str(self._directory), - ) - os.close(handle) - index_path = path[: -len(".body")] + ".idx" - try: - with open(index_path, "wb"): - pass - except BaseException: - os.unlink(path) - raise - return path, index_path - - -# -------------------------------------------------------------------------- -# hydration -# -------------------------------------------------------------------------- - - -def parse_manifest_row(row: Mapping[str, Any]) -> tuple[CandidateBodyManifest, str, bool]: - """Typed manifest, live state and referenced flag from a manifest read.""" - if not isinstance(row, Mapping) or not row.get("found"): - raise CandidateBodyUnavailable("candidate body manifest does not exist") - manifest_json = row.get("manifest") - if not isinstance(manifest_json, Mapping): - raise CandidateBodyUnavailable("candidate body manifest is unreadable") - manifest = CandidateBodyManifest.from_json(manifest_json) - state = str(manifest_json.get("state")) - referenced = bool(manifest_json.get("referenced")) - return manifest, state, referenced - - -def _strict_base64(encoded: str, *, max_chars: int) -> bytes: - """Decode PostgreSQL ``encode(..., 'base64')`` output strictly. - - The size is checked before any decoding allocates. PostgreSQL wraps at - 76 characters with ``\\n``; only that whitespace is tolerated, and the - alphabet is validated rather than silently filtered. - """ - if not isinstance(encoded, str): - raise CandidateBodyIntegrityError("candidate body page chunk carries no bytes") - if len(encoded) > max_chars: - raise CandidateBodyIntegrityError("candidate body page chunk exceeds the transport bound") - compact = encoded.replace("\n", "") - if "\r" in compact or " " in compact: - raise CandidateBodyIntegrityError("candidate body page chunk carries unexpected whitespace") - try: - return base64.b64decode(compact.encode("ascii"), validate=True) - except (binascii.Error, ValueError, UnicodeEncodeError) as exc: - raise CandidateBodyIntegrityError("candidate body page chunk is not valid base64") from exc - - -def parse_body_page(row: Mapping[str, Any], *, max_chunks: int = BODY_READ_MAX_CHUNKS) -> tuple[str | None, bool, list[BodyChunk]]: - """Decode one page read: state, referenced flag and verified chunks. - - The chunk count and every chunk's encoded length are checked against - the page bound before any base64 allocation; the decoded total is - checked against the read bound as well. - """ - if not isinstance(row, Mapping): - raise CandidateBodyIntegrityError("candidate body page is not an object") - state = row.get("state") - chunks_json = row.get("chunks") - if not isinstance(chunks_json, list): - raise CandidateBodyIntegrityError("candidate body page carries no chunk list") - if len(chunks_json) > max(1, min(int(max_chunks), BODY_READ_MAX_CHUNKS)): - raise CandidateBodyIntegrityError("candidate body page exceeds the chunk bound") - chunks: list[BodyChunk] = [] - total = 0 - for entry in chunks_json: - if not isinstance(entry, Mapping): - raise CandidateBodyIntegrityError("candidate body page chunk is malformed") - data = _strict_base64(entry.get("base64"), max_chars=CHUNK_BASE64_MAX_CHARS) - total += len(data) - if len(data) > CANDIDATE_BODY_CHUNK_BYTES or total > BODY_READ_MAX_BYTES: - raise CandidateBodyIntegrityError("candidate body page exceeds the read bound") - chunks.append( - BodyChunk( - ordinal=int(entry.get("ordinal", -1)), - data=data, - sha256=str(entry.get("sha256", "")), - ) - ) - return (None if state is None else str(state)), bool(row.get("referenced")), chunks - - -def parse_span_rows(row: Mapping[str, Any]) -> tuple[str | None, list[FieldSpan]]: - if not isinstance(row, Mapping) or not isinstance(row.get("spans"), list): - raise CandidateBodyIntegrityError("candidate body span page is malformed") - spans = row["spans"] - if len(spans) > INDEX_SPAN_ROWS: - raise CandidateBodyIntegrityError("candidate body span page exceeds its bound") - state = row.get("state") - return (None if state is None else str(state)), [FieldSpan.from_json(span) for span in spans] - - -def parse_page_rows(row: Mapping[str, Any], field_name: str) -> tuple[str | None, list[PageEntry]]: - if not isinstance(row, Mapping) or not isinstance(row.get("pages"), list): - raise CandidateBodyIntegrityError("candidate body page-index page is malformed") - pages = row["pages"] - if len(pages) > INDEX_PAGE_ENTRIES: - raise CandidateBodyIntegrityError("candidate body page-index page exceeds its bound") - entries: list[PageEntry] = [] - for page in pages: - if not isinstance(page, list) or len(page) != 3: - raise CandidateBodyIntegrityError("candidate body page-index entry is malformed") - entries.append( - PageEntry(field=field_name, ordinal=int(page[0]), offset=int(page[1]), record_index=int(page[2])) - ) - state = row.get("state") - return (None if state is None else str(state)), entries - - -class CandidateBodyHydrator: - """Hydrate one sealed body to a spool/index pair in bounded pages.""" - - def __init__( - self, - *, - read_manifest: Callable[[str], Any], - read_spans: Callable[[str, str | None], Any], - read_pages: Callable[[str, str, int], Any], - read_page: Callable[[str, int, int], Any], - admission: SpoolAdmission, - cancelled: Callable[[], bool] | None = None, - max_chunks_per_page: int = BODY_READ_MAX_CHUNKS, - ) -> None: - self._read_manifest = read_manifest - self._read_spans = read_spans - self._read_pages = read_pages - self._read_page = read_page - self._admission = admission - self._cancelled = cancelled or (lambda: False) - self._max_chunks = max(1, min(int(max_chunks_per_page), BODY_READ_MAX_CHUNKS)) - - def hydrate( - self, - ref: CandidateBodyRef, - *, - block_hash: str, - accepted_at_present: bool, - accepted_at_ms: Any, - ) -> PreparedCandidateIntent: - manifest, state, referenced = parse_manifest_row(self._read_manifest(ref.body_id)) - if state != "sealed" or not referenced: - raise CandidateBodyUnavailable( - f"candidate body {ref.body_id} is {state} and " - f"{'referenced' if referenced else 'unreferenced'}" - ) - if manifest.candidate_sha256 != ref.candidate_sha256: - raise CandidateBodyIntegrityError("candidate body manifest digest differs from its outbox row") - if manifest.byte_count != ref.byte_count or manifest.chunk_count != ref.chunk_count: - raise CandidateBodyIntegrityError("candidate body manifest disagrees with its outbox row") - release = self._admission.reserve(manifest.byte_count + manifest.page_count * 16) - paths: tuple[str, str] = () - writer = None - body = None - try: - paths = self._admission.new_spool_paths(block_hash) - writer = SpoolWriter(*paths, manifest) - self._spool_index(ref.body_id, manifest, writer) - self._spool_chunks(ref.body_id, manifest, writer) - body = writer.finish(release=release) - return prepared_intent_from_spool( - body, - accepted_at_present=accepted_at_present, - accepted_at_ms=accepted_at_ms, - ) - except BaseException: - if body is not None: - body.close() - elif writer is not None: - writer.abort() - else: - for path in paths: - try: - os.unlink(path) - except FileNotFoundError: - pass - release() - raise - - def _check_state(self, state: str | None) -> None: - if state != "sealed": - raise CandidateBodyUnavailable("candidate body turned terminal during hydration") - - def _spool_index(self, body_id: str, manifest: CandidateBodyManifest, writer: SpoolWriter) -> None: - after: str | None = None - seen = 0 - while True: - if self._cancelled(): - raise CandidateStorageError("candidate body hydration cancelled") - state, spans = parse_span_rows(self._read_spans(body_id, after)) - self._check_state(state) - for span in spans: - writer.span(span) - self._spool_pages(body_id, span, writer) - seen += len(spans) - if len(spans) < INDEX_SPAN_ROWS: - break - after = spans[-1].field - if seen != manifest.span_count: - raise CandidateBodyIntegrityError("candidate body span rows disagree with the manifest") - - def _spool_pages(self, body_id: str, span: FieldSpan, writer: SpoolWriter) -> None: - ordinal = 0 - while ordinal < span.page_count: - if self._cancelled(): - raise CandidateStorageError("candidate body hydration cancelled") - state, entries = parse_page_rows(self._read_pages(body_id, span.field, ordinal), span.field) - self._check_state(state) - if not entries: - raise CandidateBodyIntegrityError("candidate body page index is incomplete") - for entry in entries: - if entry.ordinal != ordinal: - raise CandidateBodyIntegrityError("candidate body page index is not contiguous") - ordinal += 1 - writer.pages(entries) - if ordinal != span.page_count: - raise CandidateBodyIntegrityError("candidate body page index disagrees with its span") - - def _spool_chunks(self, body_id: str, manifest: CandidateBodyManifest, writer: SpoolWriter) -> None: - ordinal = 0 - while ordinal < manifest.chunk_count: - if self._cancelled(): - raise CandidateStorageError("candidate body hydration cancelled") - state, referenced, chunks = parse_body_page( - self._read_page(body_id, ordinal, self._max_chunks), - max_chunks=self._max_chunks, - ) - if state != "sealed" or not referenced: - raise CandidateBodyUnavailable("candidate body turned terminal during hydration") - if not chunks: - raise CandidateBodyIntegrityError("candidate body page returned no chunks") - for chunk in chunks: - if chunk.ordinal != ordinal: - raise CandidateBodyIntegrityError("candidate body chunks are not contiguous") - writer.chunk(chunk) - ordinal += 1 - - -# -------------------------------------------------------------------------- -# legacy helper (isolated process) -# -------------------------------------------------------------------------- - - -@dataclass(frozen=True) -class LegacyTransport: - """How the helper reaches PostgreSQL: a psql command or a DSN.""" - - psql_command: str | None = None - database_url: str | None = None - - def to_json(self) -> dict[str, Any]: - return {"psql_command": self.psql_command, "database_url": self.database_url} - - -@dataclass -class LegacyHelperResult: - manifest: CandidateBodyManifest - spans: list[FieldSpan] - accepted_at_present: bool - accepted_at_ms: Any - row_candidate_sha256: str | None - identity_matches_row: bool - - -class LegacyCandidateHelper: - """Run the isolated legacy conversion/comparison child process.""" - - def __init__( - self, - transport: LegacyTransport, - *, - python: str | None = None, - memory_limit_bytes: int = LEGACY_HELPER_MEMORY_BYTES, - timeout_seconds: float = LEGACY_HELPER_TIMEOUT_SECONDS, - cancelled: Callable[[], bool] | None = None, - ) -> None: - self._transport = transport - self._python = python or sys.executable - self._memory_limit = int(memory_limit_bytes) - self._timeout = float(timeout_seconds) - self._cancelled = cancelled or (lambda: False) - self._lock = threading.Lock() # admit at most one conversion - - def with_cancellation(self, cancelled: Callable[[], bool]) -> LegacyCandidateHelper: - helper = LegacyCandidateHelper( - self._transport, - python=self._python, - memory_limit_bytes=self._memory_limit, - timeout_seconds=self._timeout, - cancelled=cancelled, - ) - # Cancellation wrappers share the same admission slot. - helper._lock = self._lock - return helper - - def _run( - self, - request: Mapping[str, Any], - body: CandidateBody | None = None, - ) -> dict[str, Any]: - """Supervise input, output, cancellation and retirement together. - - A helper may wait on PostgreSQL before reading stdin. The deadline - therefore starts before sending body bytes, and a blocked pipe writer - runs alongside the supervisor. Diagnostic pipes are drained with fixed - retention limits so a failed child cannot deadlock or grow the parent. - """ - deadline = time.monotonic() + self._timeout - - def check_deadline() -> None: - if self._cancelled(): - raise CandidateStorageError("legacy candidate helper cancelled") - if time.monotonic() >= deadline: - raise CandidateStorageError("legacy candidate helper exceeded its deadline") - - while not self._lock.acquire(timeout=LEGACY_HELPER_POLL_SECONDS): - check_deadline() - process = None - threads: list[threading.Thread] = [] - output = bytearray() - errors = bytearray() - io_errors: list[BaseException] = [] - limit = 64 * 1024 - from lab.prism.audit_bundle_view import RECORD_HELPER_ADMISSION - - admitted = False - try: - RECORD_HELPER_ADMISSION.acquire(check_deadline) - admitted = True - check_deadline() - repo_root = Path(__file__).resolve().parents[2] - env = dict(os.environ) - env["PYTHONPATH"] = str(repo_root) + ( - os.pathsep + env["PYTHONPATH"] if env.get("PYTHONPATH") else "" - ) - # Apply limits in the freshly exec'd child. preexec_fn is unsafe - # in this multithreaded process, before Python locks are reset. - process = subprocess.Popen( - [self._python, "-m", "lab.prism.candidate_store"], - stdin=subprocess.PIPE, stdout=subprocess.PIPE, - stderr=subprocess.PIPE, cwd=str(repo_root), env=env, - ) - assert process.stdin and process.stdout and process.stderr - - def drain(pipe: Any, target: bytearray) -> None: - try: - while chunk := pipe.read(8192): - target.extend(chunk) - if len(target) > limit: - del target[:-limit] - except BaseException as exc: - io_errors.append(exc) - - def produce() -> None: - try: - assert process is not None and process.stdin is not None - process.stdin.write((json.dumps({ - **request, "memory_limit_bytes": self._memory_limit, - }) + "\n").encode("utf-8")) - process.stdin.flush() - if body is not None: - def write_chunk(chunk: BodyChunk) -> None: - check_deadline() - process.stdin.write(chunk.data) - body.write_chunks(write_chunk) - process.stdin.flush() - except BaseException as exc: - io_errors.append(exc) - finally: - try: - process.stdin.close() - except OSError: - pass - - for target, args, name in ( - (drain, (process.stdout, output), "stdout"), - (drain, (process.stderr, errors), "stderr"), - (produce, (), "input"), - ): - thread = threading.Thread(target=target, args=args, name=f"prism-legacy-helper-{name}") - threads.append(thread) - thread.start() - while process.poll() is None: - check_deadline() - if io_errors: - raise CandidateStorageError("legacy candidate helper transport failed") from io_errors[0] - time.sleep(LEGACY_HELPER_POLL_SECONDS) - for thread in threads: - thread.join() - if process.returncode != 0: - raise CandidateStorageError( - "legacy candidate helper failed: " + errors.decode("utf-8", "replace")[-2000:] - ) - if io_errors: - raise CandidateStorageError("legacy candidate helper transport failed") from io_errors[0] - try: - result = json.loads(output.decode("utf-8").strip().splitlines()[-1]) - except (ValueError, IndexError) as exc: - raise CandidateStorageError("legacy candidate helper returned no verdict") from exc - if not isinstance(result, dict): - raise CandidateStorageError("legacy candidate helper verdict is malformed") - if "error" in result: - raise CandidateStorageError(f"legacy candidate helper: {result['error']}") - return result - finally: - if process is not None: - if process.poll() is None: - process.kill() - process.wait() - for thread in threads: - thread.join() - for pipe in (process.stdin, process.stdout, process.stderr): - if pipe is not None: - pipe.close() - if admitted: - RECORD_HELPER_ADMISSION.release() - self._lock.release() - - def convert(self, block_hash: str, spool_path: str, index_path: str, *, spool_limit_bytes: int | None = None) -> LegacyHelperResult: - result = self._run({ - "mode": "convert", - "transport": self._transport.to_json(), - "block_hash": block_hash, - "spool_path": spool_path, - "index_path": index_path, - "chunk_bytes": CANDIDATE_BODY_CHUNK_BYTES, - "spool_limit_bytes": spool_limit_bytes, - }) - manifest = CandidateBodyManifest.from_json(result["manifest"]) - spans_json = result.get("spans") - if not isinstance(spans_json, list) or len(spans_json) > INDEX_SPAN_ROWS * 64: - raise CandidateStorageError("legacy candidate helper returned malformed spans") - return LegacyHelperResult( - manifest=manifest, - spans=[FieldSpan.from_json(span) for span in spans_json], - accepted_at_present=bool(result.get("accepted_at_present")), - accepted_at_ms=result.get("accepted_at_ms"), - row_candidate_sha256=result.get("row_candidate_sha256"), - identity_matches_row=bool(result.get("identity_matches_row")), - ) - - def compare(self, block_hash: str, body: CandidateBody) -> bool: - """jsonb-equivalence of the legacy row's identity and ``body``.""" - return self._compare("compare", block_hash, body, body_id=None) - - def compare_stored_body(self, body_id: str, block_hash: str, body: CandidateBody) -> bool: - """jsonb-equivalence of a stored chunked body and ``body``. - - The fallback behind the byte comparison in - ``PsqlShareLedger.compare_candidate_body``: the helper reads the - stored chunks itself (page by page, its own connection), decodes - both documents whole -- isolated, memory- and deadline-limited -- - and answers PostgreSQL's jsonb equality. - """ - return self._compare("compare_stored", block_hash, body, body_id=body_id) - - def _compare( - self, - mode: str, - block_hash: str, - body: CandidateBody, - *, - body_id: str | None, - ) -> bool: - result = self._run({ - "mode": mode, - "transport": self._transport.to_json(), - "block_hash": block_hash, - "body_id": body_id, - "body_bytes": body.manifest.byte_count, - }, body) - return bool(result.get("equal")) - - -def _helper_query_json(transport: Mapping[str, Any], sql: str) -> Any: - """Run one statement that yields a single JSON value, in the helper.""" - database_url = transport.get("database_url") - psql_command = transport.get("psql_command") - if database_url: - try: - import psycopg - except ImportError: - psycopg = None - if psycopg is not None: - with psycopg.connect(database_url, autocommit=True) as connection: - row = connection.execute(sql).fetchone() - if row is None: - return None - value = row[0] - return json.loads(value) if isinstance(value, str) else value - if not psql_command: - raise CandidateStorageError("legacy helper has no database transport") - completed = subprocess.run( - [*shlex.split(psql_command), "--tuples-only", "--no-align", "--quiet"], - input=sql, - text=True, - stdout=subprocess.PIPE, - stderr=subprocess.PIPE, - check=False, - ) - if completed.returncode != 0: - raise CandidateStorageError(f"helper psql read failed: {completed.stderr.strip()}") - output = completed.stdout.strip() - if not output: - return None - return json.loads(output.splitlines()[-1]) - - -def _helper_fetch_legacy_text(transport: Mapping[str, Any], block_hash: str) -> tuple[str | None, str | None]: - """Fetch ``candidate::text`` and ``candidate_sha256`` for one v1 row.""" - quoted = block_hash.lower().replace(chr(39), chr(39) * 2) - document = _helper_query_json( - transport, - "SELECT json_build_object('candidate', candidate::text, 'sha', candidate_sha256) " - "FROM qbit_block_candidate_outbox " - f"WHERE block_hash = '{quoted}' " - f"AND storage_version = {LEGACY_CANDIDATE_STORAGE_VERSION} AND candidate IS NOT NULL;", - ) - if not isinstance(document, dict): - return None, None - return document.get("candidate"), document.get("sha") - - -def _helper_fetch_stored_body(transport: Mapping[str, Any], body_id: str) -> bytes | None: - """Read one stored chunked body in the helper, three chunks per statement.""" - quoted = body_id.replace(chr(39), chr(39) * 2) - header = _helper_query_json( - transport, - "SELECT json_build_object('state', state, 'chunk_count', chunk_count) " - f"FROM qbit_block_candidate_body WHERE body_id = '{quoted}';", - ) - if not isinstance(header, dict) or header.get("state") != "sealed": - return None - chunk_count = int(header.get("chunk_count") or 0) - pieces: list[bytes] = [] - ordinal = 0 - while ordinal < chunk_count: - page = _helper_query_json( - transport, - "SELECT COALESCE(json_agg(json_build_object('ordinal', ordinal, 'base64', encode(chunk, 'base64')) " - "ORDER BY ordinal), '[]'::json) FROM (SELECT ordinal, chunk FROM qbit_block_candidate_body_chunk " - f"WHERE body_id = '{quoted}' AND ordinal >= {ordinal} ORDER BY ordinal LIMIT {BODY_READ_MAX_CHUNKS}) page;", - ) - if not isinstance(page, list) or not page: - return None - for entry in page: - if int(entry.get("ordinal", -1)) != ordinal: - return None - pieces.append(_strict_base64(entry.get("base64"), max_chars=CHUNK_BASE64_MAX_CHARS)) - ordinal += 1 - return b"".join(pieces) - - -def helper_main(stdin_stream: Any = None, stdout_stream: Any = None) -> int: - """Entry point of the isolated helper process.""" - stdin_buffer = (stdin_stream or sys.stdin.buffer) - stdout_text = stdout_stream or sys.stdout - header = stdin_buffer.readline() - try: - request = json.loads(header.decode("utf-8")) - if os.name == "posix": - import resource - memory_limit = int(request.get("memory_limit_bytes", LEGACY_HELPER_MEMORY_BYTES)) - resource.setrlimit(resource.RLIMIT_AS, (memory_limit, memory_limit)) - mode = request["mode"] - transport = request["transport"] - block_hash = str(request["block_hash"]).lower() - if mode == "compare_stored": - from decimal import Decimal - - stored = _helper_fetch_stored_body(transport, str(request["body_id"])) - if stored is None: - raise CandidateBodyUnavailable("stored candidate body is not sealed") - ours_bytes = stdin_buffer.read() - theirs = json.loads(stored.decode("utf-8"), parse_float=Decimal) - ours = json.loads(ours_bytes.decode("utf-8"), parse_float=Decimal) - result = {"equal": jsonb_equivalent(theirs, ours)} - stdout_text.write(json.dumps(result) + "\n") - stdout_text.flush() - return 0 - legacy_text, row_sha = _helper_fetch_legacy_text(transport, block_hash) - if legacy_text is None: - raise CandidateBodyUnavailable("legacy candidate row is not pending") - if mode == "convert": - manifest, spans, present, stamp = legacy_json_to_spool( - legacy_text, - str(request["spool_path"]), - str(request["index_path"]), - chunk_bytes=int(request.get("chunk_bytes", CANDIDATE_BODY_CHUNK_BYTES)), - spool_limit_bytes=request.get("spool_limit_bytes"), - ) - # The spans are a handful of rows (one per large field); the - # page entries stay in the index file. - result = { - "manifest": manifest.to_json(), - "spans": [span.to_json() for span in spans], - "accepted_at_present": present, - "accepted_at_ms": stamp, - "row_candidate_sha256": row_sha, - "identity_matches_row": manifest.candidate_sha256 == row_sha, - } - elif mode == "compare": - from decimal import Decimal - - ours_bytes = stdin_buffer.read() - legacy = json.loads(legacy_text, parse_float=Decimal) - ours = json.loads(ours_bytes.decode("utf-8"), parse_float=Decimal) - for document in (legacy, ours): - pending = document.get("pending_share") if isinstance(document, dict) else None - if isinstance(pending, dict) and "accepted_at_ms" in pending: - pending["accepted_at_ms"] = None - result = {"equal": jsonb_equivalent(legacy, ours)} - else: - raise CandidateStorageError(f"unknown helper mode {mode!r}") - except Exception as exc: # noqa: BLE001 - reported to the parent - stdout_text.write(json.dumps({"error": f"{type(exc).__name__}: {exc}"}) + "\n") - stdout_text.flush() - return 1 - stdout_text.write(json.dumps(result) + "\n") - stdout_text.flush() - return 0 - - -if __name__ == "__main__": # pragma: no cover - exercised as a subprocess - sys.exit(helper_main()) - - -__all__ = [ - "BODY_READ_MAX_BYTES", - "BODY_READ_MAX_CHUNKS", - "CANDIDATE_SCHEMA_CAPABILITY", - "HEADER_MAX_BYTES", - "HEADER_PAGE_MAX_BYTES", - "HEADER_PAGE_MAX_ROWS", - "JANITOR_CHUNKS_PER_STEP", - "STALE_STAGING_SECONDS", - "CandidateBodyHydrator", - "CandidateBodyRef", - "CandidateBodyUnavailable", - "CandidateHeaderPage", - "CandidateStorageError", - "DurableCandidateDescriptor", - "IncompatibleCandidateSchema", - "LegacyCandidateHelper", - "LegacyHelperResult", - "LegacyTransport", - "SpoolAdmission", - "SpoolAdmissionExhausted", - "body_chunk_sql", - "body_index_rows_sql", - "body_manifest_sql", - "body_page_sql", - "body_pages_sql", - "body_spans_sql", - "candidate_schema_refusal", - "chunk_hex_literal", - "header_page_sql", - "helper_main", - "new_body_id", - "parse_body_page", - "parse_manifest_row", - "parse_page_rows", - "parse_span_rows", - "reap_retired_chunks_sql", - "retire_orphan_bodies_sql", - "retire_staging_body_sql", - "schema_capability_sql", - "seal_body_sql", - "stage_body_sql", -] diff --git a/lab/prism/candidate_window.py b/lab/prism/candidate_window.py deleted file mode 100644 index f7a8a4911..000000000 --- a/lab/prism/candidate_window.py +++ /dev/null @@ -1,266 +0,0 @@ -"""Bounded, exact membership checks for a replayed candidate's share window. - -Only omissions matter: the durable window must be a subset of the recorded -window. PostgreSQL performs that comparison against a private temporary table -fed through bounded COPY writes, returning a boolean rather than a window-sized -JSON value. No ledger mutation or writer-lease lock is involved. The in-memory -ledger compatibility path uses an on-disk index instead of a Python set. -""" - -from __future__ import annotations - -from collections.abc import Callable, Iterable, Iterator, Mapping -import hashlib -import json -from pathlib import Path -import sqlite3 -import subprocess -import tempfile -from typing import Any - - -COPY_TEXT_CHARACTERS = 4096 -COPY_TEXT_MAX_BYTES = COPY_TEXT_CHARACTERS * 4 -COPY_TEXT_MAX_RECORDS = 256 -_CREATE = """ -CREATE TEMPORARY TABLE qbit_candidate_recorded_ids (share_id text COLLATE "C" NOT NULL) -ON COMMIT DROP; -SET LOCAL work_mem = '8MB'; -""" -_COPY = "COPY pg_temp.qbit_candidate_recorded_ids (share_id) FROM STDIN" - - -def recorded_share_ids(shares: Iterable[Any]) -> Iterator[Any]: - """Keep the historical dictionary/type/string conversion semantics.""" - for row in shares: - if isinstance(row, Mapping): - value = row.get("share_id") - yield value if callable(getattr(value, "iter_text_chunks", None)) else str(value) - - -def _text_pieces(value: Any) -> Iterator[str]: - streamed = getattr(value, "iter_text_chunks", None) - chunks = streamed() if callable(streamed) else (value,) - for chunk in chunks: - for offset in range(0, len(chunk), COPY_TEXT_CHARACTERS): - yield chunk[offset:offset + COPY_TEXT_CHARACTERS] - - -def copy_text_chunks( - values: Iterable[Any], *, check: Callable[[], Any] = lambda: None, -) -> Iterator[bytes]: - """Encode COPY text without encoding or escaping an entire field at once. - - A UTF-8 character is at most four bytes; every escaped ASCII character - uses two. Thus even an unusually long share ID obeys the same write bound. - Literal backslashes (including \\N and \\.) cannot become COPY commands. - """ - buffer = bytearray() - records = 0 - for value in values: - for part in _text_pieces(value): - if "\x00" in part: - # PostgreSQL text cannot contain NUL. In psql's COPY input, - # it can also truncate the stream without a useful diagnostic. - raise ValueError("candidate share ID contains a NUL character") - part = part.replace("\\", "\\\\").replace("\t", "\\t") - part = part.replace("\n", "\\n").replace("\r", "\\r") - encoded = part.encode("utf-8") - if buffer and len(buffer) + len(encoded) > COPY_TEXT_MAX_BYTES: - check() - yield bytes(buffer) - buffer.clear() - records = 0 - buffer.extend(encoded) - if len(buffer) == COPY_TEXT_MAX_BYTES: - check() - yield bytes(buffer) - buffer.clear() - records = 0 - buffer.append(10) - records += 1 - if records == COPY_TEXT_MAX_RECORDS: - check() - yield bytes(buffer) - buffer.clear() - records = 0 - if buffer: - check() - yield bytes(buffer) - - -def disk_window_covers(recorded: Iterable[Any], durable: Iterable[Any]) -> bool: - """Compatibility path for in-memory ledgers; never retain the ID set.""" - with tempfile.TemporaryDirectory(prefix="prism-candidate-membership-") as root, tempfile.TemporaryFile() as values: - connection = sqlite3.connect(Path(root) / "ids.sqlite") - try: - connection.execute("PRAGMA cache_size = -2048") - connection.execute("PRAGMA temp_store = FILE") - connection.execute( - "CREATE TABLE recorded (digest BLOB, start INTEGER, length INTEGER)" - ) - connection.execute("CREATE INDEX recorded_digest ON recorded (digest)") - for value in recorded: - digest = hashlib.sha256() - start = values.tell() - for piece in _text_pieces(value): - encoded = piece.encode("utf-8") - digest.update(encoded) - values.write(encoded) - connection.execute("INSERT INTO recorded VALUES (?, ?, ?)", - (digest.digest(), start, values.tell() - start)) - for value in recorded_share_ids(durable): - digest = hashlib.sha256() - for piece in _text_pieces(value): - digest.update(piece.encode("utf-8")) - matched = False - for start, length in connection.execute( - "SELECT start, length FROM recorded WHERE digest = ?", (digest.digest(),), - ): - # A digest only selects possible matches. Equality still - # compares every byte, including under a hash collision. - values.seek(start) - remaining = length - for piece in _text_pieces(value): - encoded = piece.encode("utf-8") - if len(encoded) > remaining or values.read(len(encoded)) != encoded: - break - remaining -= len(encoded) - else: - if remaining == 0: - matched = True - break - if not matched: - return False - return True - finally: - connection.close() - - -def _query(anchor_job_issued_at_ms: int, network_difficulty: int) -> str: - return f""" -SELECT json_build_object('reproducible', NOT EXISTS ( - SELECT 1 FROM qbit_audit_share_window( - to_timestamp(({int(anchor_job_issued_at_ms)}::double precision / 1000.0)), - {int(network_difficulty)}::numeric - ) AS durable - WHERE NOT EXISTS ( - SELECT 1 FROM pg_temp.qbit_candidate_recorded_ids AS recorded - WHERE recorded.share_id = durable.share_id - ) -)); -""" - - -def _flushing_copy_writer(cursor: Any) -> Any: - # Keep psycopg optional for the psql-only deployment. LibpqWriter's - # standard finish handles CopyDone/CopyFail and cancellation correctly. - from psycopg.copy import LibpqWriter - from psycopg.generators import copy_to - - class FlushingWriter(LibpqWriter): - def write(self, data: bytes) -> None: - # On Linux the default writer can grow libpq's output buffer with - # the entire window while PostgreSQL is not consuming the socket. - # Our producer caps each write at 16 KiB; flush it before advancing. - self.connection.wait(copy_to(self._pgconn, data, flush=True)) - - return FlushingWriter(cursor) - - -def postgres_window_covers( - ledger: Any, - shares: Iterable[Any], - *, - anchor_job_issued_at_ms: int, - network_difficulty: int, -) -> bool: - """Use one durable snapshot, bounded transport, and exact text equality. - - The COPY table is connection-private and disappears on commit or rollback. - Server join memory can spill under work_mem. It grants neither credit nor - replay eligibility. A lost connection fails the read; the caller's existing - candidate retry repeats it with a fresh table and its immutable share view. - """ - from lab.prism.share_ledger import LedgerOperationTimeout, parse_single_json_value - - query = _query(anchor_job_issued_at_ms, network_difficulty) - with ledger._operation_gate(ledger._read_semaphore, "read slot"): - native = getattr(ledger, "_native", None) - if native is not None and callable(getattr(native, "connection", None)): - timeout = ledger._remaining_operation_timeout() - kwargs = {} if timeout is None else {"timeout_seconds": timeout} - try: - with native.connection(**kwargs) as connection: - with connection.transaction(): - def arm_statement() -> None: - remaining = ledger._remaining_operation_timeout() - if remaining is None: - return - milliseconds = max(1, int(remaining * 1000)) - connection.execute( - f"SET LOCAL statement_timeout = '{milliseconds}ms'" - ) - connection.execute( - f"SET LOCAL lock_timeout = '{milliseconds}ms'" - ) - - arm_statement() - connection.execute(_CREATE) - ledger._note_operation_progress() - with connection.cursor() as cursor: - arm_statement() - with cursor.copy(_COPY, writer=_flushing_copy_writer(cursor)) as copy: - for chunk in copy_text_chunks( - recorded_share_ids(shares), - check=ledger._note_json_row_batch, - ): - copy.write(chunk) - ledger._note_json_row_batch() - # Refresh the statement budget after streaming, so - # each phase shares the caller's original deadline. - arm_statement() - connection.execute("ANALYZE pg_temp.qbit_candidate_recorded_ids") - ledger._note_json_row_batch() - arm_statement() - row = connection.execute(query).fetchone() - result = parse_single_json_value(row[0] if row else None) - ledger._remaining_operation_timeout() - except native._psycopg.OperationalError as exc: - from lab.prism.share_ledger import _is_postgres_deadline_error - - if _is_postgres_deadline_error(exc): - raise LedgerOperationTimeout("candidate window check timed out") from exc - raise RuntimeError(f"postgres candidate window check failed: {exc}") from exc - else: - # psql reads a boundedly-written file. communicate(input=...) would - # reconstitute the entire recorded window in a single bytes object. - with tempfile.TemporaryFile() as source, tempfile.TemporaryFile() as errors: - source.write(_CREATE.encode("ascii")) - source.write((_COPY + ";\n").encode("ascii")) - for chunk in copy_text_chunks( - recorded_share_ids(shares), check=ledger._note_json_row_batch, - ): - source.write(chunk) - source.write(b"\\.\n") - source.write(b"ANALYZE pg_temp.qbit_candidate_recorded_ids;\n") - source.write(query.encode("ascii")) - source.seek(0) - command, kwargs, timeout = ledger._psql_invocation() - try: - completed = subprocess.run( - command, stdin=source, stdout=subprocess.PIPE, stderr=errors, - check=False, **kwargs, - ) - except subprocess.TimeoutExpired as exc: - raise LedgerOperationTimeout("candidate window check timed out") from exc - errors.seek(0) - # COPY errors can echo a whole input line. Keep diagnostics - # bounded as well; the spool itself dies with this operation. - error_text = errors.read(64 * 1024).decode("utf-8", errors="replace") - ledger._check_psql_exit(completed.returncode, error_text, timeout) - ledger._remaining_operation_timeout() - result = json.loads(completed.stdout) - if not isinstance(result, dict) or type(result.get("reproducible")) is not bool: - raise RuntimeError("candidate window check returned no boolean result") - return result["reproducible"] diff --git a/lab/prism/capacity_evidence.py b/lab/prism/capacity_evidence.py deleted file mode 100644 index 9e77fb288..000000000 --- a/lab/prism/capacity_evidence.py +++ /dev/null @@ -1,563 +0,0 @@ -"""Strict validation for PRISM production-capacity evidence artifacts.""" - -from __future__ import annotations - -import json -import re -import uuid -from dataclasses import dataclass -from datetime import datetime, timedelta, timezone -from decimal import Decimal, InvalidOperation -from pathlib import Path -from typing import Any, Mapping - - -SCHEMA = "qbit-prism-capacity-evidence/v2" -TEST_PATH = "stratum-to-postgres" -QUALIFICATION_ARTIFACT = "qualification" -EXAMPLE_ARTIFACT = "example" -DEFAULT_MAX_AGE_SECONDS = 24 * 60 * 60 -DEFAULT_MAX_FUTURE_SKEW_SECONDS = 5 * 60 -MINIMUM_PHASE_DURATION_SECONDS = Decimal("60") -MINIMUM_RECONNECT_EVENTS = 10 -MINIMUM_DATABASE_DELAY_MILLISECONDS = Decimal("10") - -DIFFICULTY_CONFIGURATION_KEYS = ( - "PRISM_STRATUM_SHARE_DIFF", - "PRISM_STRATUM_VARDIFF_MIN_DIFF", - "PRISM_STRATUM_VARDIFF_START_DIFF", - "PRISM_STRATUM_VARDIFF_MAX_DIFF", -) -DECIMAL_CONFIGURATION_KEYS = ( - *DIFFICULTY_CONFIGURATION_KEYS, - "PRISM_STRATUM_VARDIFF_TARGET_SECONDS", - "PRISM_STRATUM_VARDIFF_RETARGET_SECONDS", - "PRISM_STRATUM_VARDIFF_MAX_STEP_UP", - "PRISM_STRATUM_VARDIFF_MAX_STEP_DOWN", - "PRISM_STRATUM_VARDIFF_EWMA_ALPHA", - "PRISM_STRATUM_VARDIFF_RETARGET_TOLERANCE", - "PRISM_STRATUM_VARDIFF_IDLE_SWEEP_SECONDS", - "PRISM_SHARE_COMMIT_TIMEOUT_SECONDS", - "PRISM_STRATUM_SEND_TIMEOUT_SECONDS", -) -INTEGER_CONFIGURATION_KEYS = ( - "PRISM_STRATUM_VARDIFF", - "PRISM_SHARE_COMMIT_BATCH_SIZE", - "PRISM_SHARE_COMMIT_LINGER_MILLISECONDS", -) -CONFIGURATION_KEYS = (*DECIMAL_CONFIGURATION_KEYS, *INTEGER_CONFIGURATION_KEYS) -SUBJECT_KEYS = ( - "coordinator_revision", - "coordinator_image_digest", - "postgres_server_version", - "database_profile_sha256", -) -REQUIRED_PHASES = ("steady_state", "reconnect", "slow_database") - -_RFC3339_PATTERN = re.compile( - r"^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}(?:\.\d+)?(?:Z|[+-]\d{2}:\d{2})$" -) -_HEX_40_PATTERN = re.compile(r"^[0-9a-f]{40}$") -_HEX_64_PATTERN = re.compile(r"^[0-9a-f]{64}$") -_IMAGE_DIGEST_PATTERN = re.compile(r"^sha256:[0-9a-f]{64}$") - -_TOP_LEVEL_KEYS = { - "schema", - "artifact_kind", - "test_path", - "generated_at", - "run_id", - "subject", - "durability", - "configuration", - "forecast_peak_shares_per_second", - "test_duration_seconds", - "offered_valid_shares", - "acknowledged_shares", - "postgres_unique_committed_shares", - "rejected_valid_shares", - "missing_acknowledged_share_ids", - "unexpected_committed_share_ids", - "acknowledged_share_ids_sha256", - "postgres_share_ids_sha256", - "ack_latency_milliseconds", - "ack_p99_limit_milliseconds", - "phases", -} -_DURABILITY_KEYS = {"fsync", "full_page_writes", "synchronous_commit"} -_LATENCY_KEYS = {"p50", "p99"} -_COMMON_PHASE_KEYS = { - "completed", - "duration_seconds", - "offered_valid_shares", - "acknowledged_shares", - "postgres_unique_committed_shares", - "rejected_valid_shares", - "missing_acknowledged_share_ids", - "unexpected_committed_share_ids", - "acknowledged_share_ids_sha256", - "postgres_share_ids_sha256", - "ack_latency_milliseconds", -} - - -class EvidenceError(ValueError): - """Raised when a capacity evidence document is incomplete or inconsistent.""" - - -@dataclass(frozen=True) -class CapacityEvidenceSummary: - generated_at: datetime - measured_shares_per_second: Decimal - capacity_multiple: Decimal - acknowledged_shares: int - ack_p50_milliseconds: Decimal - ack_p99_milliseconds: Decimal - ack_p99_limit_milliseconds: Decimal - - -@dataclass(frozen=True) -class _PhaseSummary: - duration_seconds: Decimal - offered_valid_shares: int - acknowledged_shares: int - committed_shares: int - p50_milliseconds: Decimal - p99_milliseconds: Decimal - - -def _mapping(value: object, field: str) -> Mapping[str, Any]: - if not isinstance(value, dict): - raise EvidenceError(f"{field} must be a JSON object") - return value - - -def _unique_json_object(pairs: list[tuple[str, object]]) -> dict[str, object]: - result: dict[str, object] = {} - for key, value in pairs: - if key in result: - raise EvidenceError(f"capacity evidence contains duplicate JSON key {key!r}") - result[key] = value - return result - - -def _exact_keys(value: Mapping[str, Any], expected: set[str], field: str) -> None: - missing = sorted(expected - set(value)) - unknown = sorted(set(value) - expected) - if missing: - raise EvidenceError(f"{field} is missing required fields: {', '.join(missing)}") - if unknown: - raise EvidenceError(f"{field} contains unknown fields: {', '.join(unknown)}") - - -def _decimal(value: object, field: str, *, allow_zero: bool = False) -> Decimal: - if isinstance(value, bool): - raise EvidenceError(f"{field} must be a finite decimal number") - try: - parsed = Decimal(str(value)) - except (InvalidOperation, ValueError) as exc: - raise EvidenceError(f"{field} must be a finite decimal number") from exc - if not parsed.is_finite() or parsed < 0 or (not allow_zero and parsed == 0): - qualifier = "non-negative" if allow_zero else "positive" - raise EvidenceError(f"{field} must be a finite {qualifier} decimal number") - return parsed - - -def _integer(value: object, field: str, *, allow_zero: bool = False) -> int: - if isinstance(value, bool): - raise EvidenceError(f"{field} must be an integer") - try: - parsed = int(str(value)) - except ValueError as exc: - raise EvidenceError(f"{field} must be an integer") from exc - if str(parsed) != str(value).strip(): - raise EvidenceError(f"{field} must be an integer") - if parsed < 0 or (not allow_zero and parsed == 0): - qualifier = "non-negative" if allow_zero else "positive" - raise EvidenceError(f"{field} must be a {qualifier} integer") - return parsed - - -def _configuration_decimal(value: object, field: str, *, allow_zero: bool = False) -> Decimal: - parsed = _decimal(value, field, allow_zero=allow_zero) - if parsed == Decimal("0.000000001"): - raise EvidenceError(f"{field} uses the lab-only 1e-9 difficulty") - return parsed - - -def _sha256(value: object, field: str) -> str: - if not isinstance(value, str) or _HEX_64_PATTERN.fullmatch(value) is None: - raise EvidenceError(f"{field} must be 64 lowercase hex characters") - return value - - -def _parse_timestamp(value: object) -> datetime: - if not isinstance(value, str) or _RFC3339_PATTERN.fullmatch(value) is None: - raise EvidenceError("generated_at must be an RFC 3339 timestamp") - try: - parsed = datetime.fromisoformat(value[:-1] + "+00:00" if value.endswith("Z") else value) - except ValueError as exc: - raise EvidenceError("generated_at must be an RFC 3339 timestamp") from exc - return parsed.astimezone(timezone.utc) - - -def _validate_timestamp_freshness( - generated_at: datetime, - *, - current_time: datetime | None, - max_age_seconds: int, - max_future_skew_seconds: int, -) -> None: - if max_age_seconds <= 0: - raise EvidenceError("capacity evidence maximum age must be positive") - if max_future_skew_seconds < 0: - raise EvidenceError("capacity evidence future skew must be non-negative") - now = current_time or datetime.now(timezone.utc) - if now.tzinfo is None: - raise EvidenceError("capacity evidence current time must include a timezone") - now = now.astimezone(timezone.utc) - if generated_at > now + timedelta(seconds=max_future_skew_seconds): - raise EvidenceError("capacity evidence generated_at is too far in the future") - if generated_at < now - timedelta(seconds=max_age_seconds): - raise EvidenceError( - f"capacity evidence is older than the allowed {max_age_seconds} seconds" - ) - - -def _validate_subject( - value: object, - expected: Mapping[str, str] | None, - *, - allow_example: bool, -) -> None: - subject = _mapping(value, "subject") - _exact_keys(subject, set(SUBJECT_KEYS), "subject") - revision = subject["coordinator_revision"] - if not isinstance(revision, str) or _HEX_40_PATTERN.fullmatch(revision) is None: - raise EvidenceError("subject.coordinator_revision must be 40 lowercase hex characters") - image_digest = subject["coordinator_image_digest"] - if not isinstance(image_digest, str) or _IMAGE_DIGEST_PATTERN.fullmatch(image_digest) is None: - raise EvidenceError( - "subject.coordinator_image_digest must be sha256 followed by 64 lowercase hex characters" - ) - postgres_version = subject["postgres_server_version"] - if not isinstance(postgres_version, str) or not postgres_version.strip(): - raise EvidenceError("subject.postgres_server_version must be a non-empty string") - _sha256(subject["database_profile_sha256"], "subject.database_profile_sha256") - - if expected is None: - if allow_example: - return - raise EvidenceError("expected deployment subject is required for qualification evidence") - for key in SUBJECT_KEYS: - if key not in expected or not expected[key]: - raise EvidenceError(f"expected deployment subject is missing {key}") - if subject[key] != expected[key]: - raise EvidenceError(f"subject.{key} does not match the deployment value") - - -def _validate_durability(value: object) -> None: - durability = _mapping(value, "durability") - _exact_keys(durability, _DURABILITY_KEYS, "durability") - for key in sorted(_DURABILITY_KEYS): - if durability[key] != "on": - raise EvidenceError(f"durability.{key} must be 'on'") - - -def _normalize_configuration(value: object, field: str) -> dict[str, Decimal | int]: - configuration = _mapping(value, field) - _exact_keys(configuration, set(CONFIGURATION_KEYS), field) - normalized: dict[str, Decimal | int] = {} - for key in DECIMAL_CONFIGURATION_KEYS: - allow_zero = key in { - "PRISM_STRATUM_VARDIFF_RETARGET_TOLERANCE", - "PRISM_STRATUM_VARDIFF_IDLE_SWEEP_SECONDS", - "PRISM_STRATUM_SEND_TIMEOUT_SECONDS", - } - parser = _configuration_decimal if key in DIFFICULTY_CONFIGURATION_KEYS else _decimal - normalized[key] = parser(configuration[key], f"{field}.{key}", allow_zero=allow_zero) - for key in INTEGER_CONFIGURATION_KEYS: - allow_zero = key in { - "PRISM_STRATUM_VARDIFF", - "PRISM_SHARE_COMMIT_LINGER_MILLISECONDS", - } - normalized[key] = _integer(configuration[key], f"{field}.{key}", allow_zero=allow_zero) - if normalized["PRISM_STRATUM_VARDIFF"] not in {0, 1}: - raise EvidenceError(f"{field}.PRISM_STRATUM_VARDIFF must be 0 or 1") - - min_diff = normalized["PRISM_STRATUM_VARDIFF_MIN_DIFF"] - start_diff = normalized["PRISM_STRATUM_VARDIFF_START_DIFF"] - max_diff = normalized["PRISM_STRATUM_VARDIFF_MAX_DIFF"] - assert isinstance(min_diff, Decimal) - assert isinstance(start_diff, Decimal) - assert isinstance(max_diff, Decimal) - if not min_diff <= start_diff <= max_diff: - raise EvidenceError( - f"{field} vardiff values must satisfy minimum <= start <= maximum" - ) - for key in ("PRISM_STRATUM_VARDIFF_MAX_STEP_UP", "PRISM_STRATUM_VARDIFF_MAX_STEP_DOWN"): - if normalized[key] < 1: - raise EvidenceError(f"{field}.{key} must be at least 1") - if normalized["PRISM_STRATUM_VARDIFF_EWMA_ALPHA"] > 1: - raise EvidenceError(f"{field}.PRISM_STRATUM_VARDIFF_EWMA_ALPHA must not exceed 1") - return normalized - - -def _validate_configuration( - value: object, - expected: Mapping[str, str] | None, - *, - allow_example: bool, -) -> dict[str, Decimal | int]: - normalized = _normalize_configuration(value, "configuration") - if expected is None: - if allow_example: - return normalized - raise EvidenceError("expected deployment configuration is required for qualification evidence") - expected_normalized = _normalize_configuration(expected, "expected configuration") - for key in CONFIGURATION_KEYS: - if normalized[key] != expected_normalized[key]: - raise EvidenceError(f"configuration.{key} does not match the deployment value") - return normalized - - -def _validate_latency(value: object, field: str, p99_limit: Decimal) -> tuple[Decimal, Decimal]: - latency = _mapping(value, field) - _exact_keys(latency, _LATENCY_KEYS, field) - p50 = _decimal(latency["p50"], f"{field}.p50", allow_zero=True) - p99 = _decimal(latency["p99"], f"{field}.p99", allow_zero=True) - if p50 > p99: - raise EvidenceError(f"{field} p50 latency cannot exceed p99 latency") - if p99 > p99_limit: - raise EvidenceError(f"{field} p99 latency {p99}ms exceeds the required {p99_limit}ms") - return p50, p99 - - -def _validate_share_counts(value: Mapping[str, Any], field: str) -> tuple[int, int, int]: - offered = _integer(value["offered_valid_shares"], f"{field}.offered_valid_shares") - acknowledged = _integer(value["acknowledged_shares"], f"{field}.acknowledged_shares") - committed = _integer( - value["postgres_unique_committed_shares"], - f"{field}.postgres_unique_committed_shares", - ) - rejected = _integer( - value["rejected_valid_shares"], f"{field}.rejected_valid_shares", allow_zero=True - ) - missing = _integer( - value["missing_acknowledged_share_ids"], - f"{field}.missing_acknowledged_share_ids", - allow_zero=True, - ) - unexpected = _integer( - value["unexpected_committed_share_ids"], - f"{field}.unexpected_committed_share_ids", - allow_zero=True, - ) - ack_digest = _sha256( - value["acknowledged_share_ids_sha256"], f"{field}.acknowledged_share_ids_sha256" - ) - committed_digest = _sha256( - value["postgres_share_ids_sha256"], f"{field}.postgres_share_ids_sha256" - ) - if rejected != 0 or offered != acknowledged: - raise EvidenceError( - f"{field} did not acknowledge every offered valid share: " - f"offered={offered} acknowledged={acknowledged} rejected={rejected}" - ) - if acknowledged != committed or missing != 0 or unexpected != 0: - raise EvidenceError( - f"{field} failed ACK-to-Postgres reconciliation: acknowledged={acknowledged} " - f"committed={committed} missing={missing} unexpected={unexpected}" - ) - if ack_digest != committed_digest: - raise EvidenceError(f"{field} ACK and Postgres share-identifier digests differ") - return offered, acknowledged, committed - - -def _validate_phase( - name: str, - value: object, - *, - forecast: Decimal, - p99_limit: Decimal, -) -> _PhaseSummary: - field = f"phases.{name}" - phase = _mapping(value, field) - expected_keys = set(_COMMON_PHASE_KEYS) - if name == "reconnect": - expected_keys.add("reconnect_events") - elif name == "slow_database": - expected_keys.add("database_delay_milliseconds") - _exact_keys(phase, expected_keys, field) - if phase["completed"] is not True: - raise EvidenceError(f"{field}.completed must be true") - duration = _decimal(phase["duration_seconds"], f"{field}.duration_seconds") - if duration < MINIMUM_PHASE_DURATION_SECONDS: - raise EvidenceError( - f"{field}.duration_seconds must be at least {MINIMUM_PHASE_DURATION_SECONDS}" - ) - offered, acknowledged, committed = _validate_share_counts(phase, field) - p50, p99 = _validate_latency(phase["ack_latency_milliseconds"], f"{field}.ack_latency_milliseconds", p99_limit) - measured_rate = Decimal(acknowledged) / duration - if measured_rate < forecast * 2: - raise EvidenceError( - f"{field} sustained rate must be at least 2x forecast peak: " - f"measured={measured_rate} forecast={forecast}" - ) - if name == "reconnect": - reconnect_events = _integer(phase["reconnect_events"], f"{field}.reconnect_events") - if reconnect_events < MINIMUM_RECONNECT_EVENTS: - raise EvidenceError( - f"{field}.reconnect_events must be at least {MINIMUM_RECONNECT_EVENTS}" - ) - elif name == "slow_database": - database_delay = _decimal( - phase["database_delay_milliseconds"], - f"{field}.database_delay_milliseconds", - ) - if database_delay < MINIMUM_DATABASE_DELAY_MILLISECONDS: - raise EvidenceError( - f"{field}.database_delay_milliseconds must be at least " - f"{MINIMUM_DATABASE_DELAY_MILLISECONDS}" - ) - return _PhaseSummary(duration, offered, acknowledged, committed, p50, p99) - - -def validate_capacity_evidence( - payload: object, - *, - expected_configuration: Mapping[str, str] | None = None, - expected_subject: Mapping[str, str] | None = None, - expected_forecast_peak_shares_per_second: object | None = None, - expected_ack_p99_limit_milliseconds: object | None = None, - current_time: datetime | None = None, - max_age_seconds: int = DEFAULT_MAX_AGE_SECONDS, - max_future_skew_seconds: int = DEFAULT_MAX_FUTURE_SKEW_SECONDS, - enforce_freshness: bool = True, - allow_example: bool = False, -) -> CapacityEvidenceSummary: - document = _mapping(payload, "document") - _exact_keys(document, _TOP_LEVEL_KEYS, "document") - if document["schema"] != SCHEMA: - raise EvidenceError(f"schema must be {SCHEMA!r}") - if document["test_path"] != TEST_PATH: - raise EvidenceError(f"test_path must be {TEST_PATH!r}") - - artifact_kind = document["artifact_kind"] - if artifact_kind not in {QUALIFICATION_ARTIFACT, EXAMPLE_ARTIFACT}: - raise EvidenceError("artifact_kind must be 'qualification' or 'example'") - if artifact_kind == EXAMPLE_ARTIFACT and not allow_example: - raise EvidenceError("example capacity evidence is rejected outside explicit test validation") - try: - run_id = uuid.UUID(str(document["run_id"])) - except (ValueError, AttributeError) as exc: - raise EvidenceError("run_id must be a UUID") from exc - if artifact_kind == QUALIFICATION_ARTIFACT and run_id.int == 0: - raise EvidenceError("qualification evidence requires a non-zero run_id") - - generated_at = _parse_timestamp(document["generated_at"]) - if artifact_kind == QUALIFICATION_ARTIFACT and enforce_freshness: - _validate_timestamp_freshness( - generated_at, - current_time=current_time, - max_age_seconds=max_age_seconds, - max_future_skew_seconds=max_future_skew_seconds, - ) - _validate_subject(document["subject"], expected_subject, allow_example=allow_example) - _validate_durability(document["durability"]) - configuration = _validate_configuration( - document["configuration"], expected_configuration, allow_example=allow_example - ) - - forecast = _decimal( - document["forecast_peak_shares_per_second"], - "forecast_peak_shares_per_second", - ) - if expected_forecast_peak_shares_per_second is None: - if not allow_example: - raise EvidenceError("externally configured forecast peak share rate is required") - else: - expected_forecast = _decimal( - expected_forecast_peak_shares_per_second, - "expected forecast peak shares per second", - ) - if forecast != expected_forecast: - raise EvidenceError( - "forecast_peak_shares_per_second does not match the deployment value" - ) - - p99_limit = _decimal( - document["ack_p99_limit_milliseconds"], - "ack_p99_limit_milliseconds", - ) - if expected_ack_p99_limit_milliseconds is None: - if not allow_example: - raise EvidenceError("externally configured ACK p99 limit is required") - else: - expected_p99_limit = _decimal( - expected_ack_p99_limit_milliseconds, - "expected ACK p99 limit milliseconds", - ) - if p99_limit != expected_p99_limit: - raise EvidenceError("ack_p99_limit_milliseconds does not match the deployment value") - commit_timeout_milliseconds = ( - configuration["PRISM_SHARE_COMMIT_TIMEOUT_SECONDS"] * Decimal(1000) - ) - if p99_limit > commit_timeout_milliseconds: - raise EvidenceError( - "ack_p99_limit_milliseconds cannot exceed PRISM_SHARE_COMMIT_TIMEOUT_SECONDS" - ) - - duration = _decimal(document["test_duration_seconds"], "test_duration_seconds") - offered, acknowledged, committed = _validate_share_counts(document, "capacity run") - measured_rate = Decimal(acknowledged) / duration - capacity_multiple = measured_rate / forecast - if capacity_multiple < Decimal(2): - raise EvidenceError( - "measured sustained rate must be at least 2x forecast peak: " - f"measured={measured_rate} forecast={forecast}" - ) - p50, p99 = _validate_latency( - document["ack_latency_milliseconds"], "ack_latency_milliseconds", p99_limit - ) - - phases = _mapping(document["phases"], "phases") - _exact_keys(phases, set(REQUIRED_PHASES), "phases") - phase_summaries = { - name: _validate_phase(name, phases[name], forecast=forecast, p99_limit=p99_limit) - for name in REQUIRED_PHASES - } - if sum((phase.duration_seconds for phase in phase_summaries.values()), Decimal(0)) != duration: - raise EvidenceError("phase durations must equal test_duration_seconds") - if sum(phase.offered_valid_shares for phase in phase_summaries.values()) != offered: - raise EvidenceError("phase offered-share totals must equal the capacity-run total") - if sum(phase.acknowledged_shares for phase in phase_summaries.values()) != acknowledged: - raise EvidenceError("phase acknowledged-share totals must equal the capacity-run total") - if sum(phase.committed_shares for phase in phase_summaries.values()) != committed: - raise EvidenceError("phase committed-share totals must equal the capacity-run total") - - return CapacityEvidenceSummary( - generated_at=generated_at, - measured_shares_per_second=measured_rate, - capacity_multiple=capacity_multiple, - acknowledged_shares=acknowledged, - ack_p50_milliseconds=p50, - ack_p99_milliseconds=p99, - ack_p99_limit_milliseconds=p99_limit, - ) - - -def load_capacity_evidence( - path: Path, - **validation_options: object, -) -> CapacityEvidenceSummary: - try: - payload = json.loads( - path.read_text(encoding="utf-8"), - object_pairs_hook=_unique_json_object, - ) - except OSError as exc: - raise EvidenceError(f"cannot read {path}: {exc.strerror or exc}") from exc - except json.JSONDecodeError as exc: - raise EvidenceError(f"{path} is not valid JSON: {exc.msg}") from exc - return validate_capacity_evidence(payload, **validation_options) diff --git a/lab/prism/coordinator_config.py b/lab/prism/coordinator_config.py deleted file mode 100644 index 45c03291e..000000000 --- a/lab/prism/coordinator_config.py +++ /dev/null @@ -1,2252 +0,0 @@ -"""Immutable PRISM coordinator configuration and environment loading.""" - -from __future__ import annotations - -import json -import math -import shlex -import sys -from os import environ as _PROCESS_ENVIRON -from dataclasses import dataclass, replace as dataclass_replace -from decimal import Decimal, InvalidOperation -from pathlib import Path -from typing import Mapping - -from lab.auxpow import vardiff -from lab.prism import direct_stratum -from lab.prism.ctv_broadcaster_daemon import MAX_CTV_FANOUT_BROADCASTER_CHUNK_SIZE -from lab.prism.process_telemetry import HeapCensusConfig -from lab.prism.share_ledger import ( - DEFAULT_AUDIT_SHARE_SEGMENT_SIZE, - DEFAULT_CTV_BROADCAST_ATTEMPT_DETAIL_LIMIT, - DEFAULT_CTV_BROADCAST_RETRY_BACKOFF_SECONDS, - DEFAULT_LEASE_ACQUIRE_ATTEMPTS, - DEFAULT_LEASE_ACQUIRE_LOCK_TIMEOUT_SECONDS, - DEFAULT_POSTGRES_IDLE_IN_TRANSACTION_TIMEOUT_SECONDS, - DEFAULT_POSTGRES_TCP_KEEPALIVES_COUNT, - DEFAULT_POSTGRES_TCP_KEEPALIVES_IDLE_SECONDS, - DEFAULT_POSTGRES_TCP_KEEPALIVES_INTERVAL_SECONDS, -) -from lab.prism.writer_lease_timing import DEFAULT_WRITER_LEASE_HEARTBEAT_POLICY - - -DEFAULT_P2MR_SPEND_INPUT_BYTES = 3_680 -DEFAULT_MIN_OUTPUT_FEERATE_SATS_PER_BYTE = 1 -DEFAULT_MIN_OUTPUT_SAFETY_MULTIPLIER = 4 -DEFAULT_TESTNET_USERNAME_FALLBACK_ADDRESS = ( - "tq1zlsq9dpxz8mennhdpr9nf9s0f2tjtq6gxs9m84k6xglhkfp92q2zszzu4m3" -) -DEFAULT_PRISM_COINBASE_TAG = "/PRISM/" -MAX_PRISM_COINBASE_TAG_BYTES = 40 -DEFAULT_DIRECT_COINBASE_PAYOUT_FLOOR_SATS = 10_485_760 -DEFAULT_MAX_COINBASE_SETTLEMENT_OUTPUTS = 16 -DEFAULT_MAX_DIRECT_COINBASE_OUTPUTS = 12 -DEFAULT_MAX_CTV_FANOUT_RECIPIENTS_PER_TRANSACTION = 1_000 -DEFAULT_CTV_FANOUT_FEE_PREMIUM_BPS = 12_000 -TESTNET_QBIT_CHAINS = {"testnet", "testnet3", "testnet4", "signet"} -DEFAULT_PRISM_BLOCKPOLL_SECONDS = 2.0 -DEFAULT_PRISM_BLOCKWAIT_TIMEOUT_SECONDS = 5.0 -DEFAULT_PRISM_TIP_REFRESH_FAILURE_HOLDOFF_SECONDS = 1.0 -DEFAULT_PRISM_JOB_BUNDLE_CACHE_SECONDS = 10.0 -DEFAULT_PRISM_BUNDLE_BUILD_TIMEOUT_SECONDS = 60.0 -# Minimum spacing between speculative payout-ledger-artifact rebuilds armed by -# a failed reuse fence. Every fence failure wants a fresh artifact; the floor -# prevents repeated delta/carry reads and also paces any cold/full fallback. -DEFAULT_PRISM_PAYOUT_ARTIFACT_REARM_MIN_SECONDS = 5.0 -# How long an armed artifact keeps serving NEW reuse decisions, measured -# from its install (prepared_monotonic), never from its anchor: the reward -# window walk itself takes multiple seconds at production share volume, so -# a bound measured from the wall-clock anchor let a slow build arm an -# artifact that was already expired on arrival -- the born-expired rebuild -# livelock behind the 2026-07-29 rollback of the first anchor-scoped -# deploy. Freshness starts when the window becomes reusable. -DEFAULT_PRISM_PAYOUT_ARTIFACT_REANCHOR_SECONDS = 60.0 -# Normal generations may reuse the last materialized window until this floor -# elapses. Found-block settlement bypasses the floor; the default follows the -# existing re-anchor cadence so one knob continues to describe the expected -# freshness rhythm when the new setting is absent. -DEFAULT_PRISM_PAYOUT_ARTIFACT_MIN_BUILD_INTERVAL_SECONDS = 60.0 -# A full oracle pass periodically checks the append-only delta assumption. -# Reorg/carry mutations and invariant failures force this path immediately. -DEFAULT_PRISM_PAYOUT_ARTIFACT_FULL_RESCAN_SECONDS = 3_600.0 -# Route the payout-window fold, canonical digest, and incremental advance -# through the persistent --serve audit builder instead of in-process Python. -# Defaults to Python: the Rust path is adopted deliberately, per #131's -# per-component-switch method, and any daemon anomaly still degrades to the -# in-process pipeline for that materialization without flipping this. -DEFAULT_PRISM_WINDOW_PIPELINE_RUST = False -# Issue #226: bind glibc mallinfo2 through ctypes on every /metrics scrape -# and export arena/in-use/free/mmapped bytes beside the interpreter's own -# allocation and GC counters. On by default: it is the one always-on reading -# that separates glibc arena fragmentation from Python retention on a -# running coordinator. The switch exists because mallinfo2 walks glibc's -# free lists under each arena lock -- negligible on a healthy heap, but the -# only reading here that is not a plain counter load -- so an operator can -# stop that call without a redeploy. Off renders the allocator gauges as -# unavailable (availability 0, bytes -1); the interpreter gauges stay on. -DEFAULT_PRISM_MALLOC_TELEMETRY = True -# Issue #226 part 2: the operator-triggered heap census. Off by default and -# armed only by this switch: the walk (gc.get_objects plus a pass over every -# tracked object) holds the GIL for seconds on a multi-gigabyte heap, so it -# must never be reachable by accident. On, SIGUSR1 writes one bounded JSON -# report per signal to PRISM_HEAP_CENSUS_DIR; nothing else changes. -DEFAULT_PRISM_HEAP_CENSUS = False -# Where reports land: a subdirectory of the audit directory, which is the -# one persistent volume the compose stack mounts into the coordinator. -DEFAULT_PRISM_HEAP_CENSUS_SUBDIR = "heap-census" -# The bounds. Top-N caps both type lists and both tracemalloc lists; the -# byte cap is enforced by halving top-N until the JSON fits, never by cutting -# the document; the walk cap stops the per-object pass and marks the report -# truncated. The ceilings below are hard: a value above them refuses startup -# rather than promising a "bounded" census that is not. -DEFAULT_PRISM_HEAP_CENSUS_TOP_N = 40 -MAX_PRISM_HEAP_CENSUS_TOP_N = 200 -DEFAULT_PRISM_HEAP_CENSUS_MAX_BYTES = 262_144 -MIN_PRISM_HEAP_CENSUS_MAX_BYTES = 4_096 -MAX_PRISM_HEAP_CENSUS_MAX_BYTES = 4 * 1024 * 1024 -DEFAULT_PRISM_HEAP_CENSUS_MAX_SECONDS = 10.0 -MAX_PRISM_HEAP_CENSUS_MAX_SECONDS = 120.0 -# Signals arriving closer together than this coalesce into one action, so a -# repeated kill cannot stack heap walks or arena-lock trims. -DEFAULT_PRISM_HEAP_CENSUS_MIN_INTERVAL_SECONDS = 60.0 -# tracemalloc has a standing cost on every allocation for as long as it -# traces, so it is a second, separate opt-in that only means anything with -# the census on; on without the census refuses startup. -DEFAULT_PRISM_HEAP_CENSUS_TRACEMALLOC = False -# malloc_trim on SIGRTMIN+1. It takes every glibc arena lock in turn while -# it walks that arena's free lists, so it is an operator action, not a -# default; the periodic variant is paced and floored so it cannot become a -# steady stall. 0 disables the periodic trim. -DEFAULT_PRISM_MALLOC_TRIM_SIGNAL = False -DEFAULT_PRISM_MALLOC_TRIM_INTERVAL_SECONDS = 0.0 -MIN_PRISM_MALLOC_TRIM_INTERVAL_SECONDS = 60.0 - - -def validate_payout_artifact_age_bounds( - max_anchor_age_seconds: float, - refresh_floor_seconds: float, -) -> None: - """Refuse an anchor ceiling the background re-anchor cannot beat. - - A ceiling at or below the re-anchor floor plus the debounce and the - multi-second window walk turns every reuse probe into a ceiling - rejection while the rebuild runs debounced in parallel -- mechanically - the 2026-07-30 walk-per-build brownout. The 2x floor requirement keeps - comfortable margin over that sum at any plausible walk duration. - """ - if max_anchor_age_seconds <= 2.0 * refresh_floor_seconds: - raise SystemExit( - "PRISM_PAYOUT_ARTIFACT_MAX_ANCHOR_AGE_SECONDS must exceed " - "2x the payout-artifact refresh floor" - ) - - -# Audit-facing ceiling on how far a served window's declared anchor may -# trail the live ledger, in wall-clock terms. Deliberately much looser than -# the reuse-staleness knob: shares stamped after an anchor deterministically -# belong to the next window (never lost; bundles declare the artifact's own -# anchor, so reused windows stay audit-reproducible), and the ceiling only -# caps worst-case payout-preview lag under pathological build latency or a -# wedged pending-commit floor. An artifact whose anchor is already past the -# ceiling at install time is born expired: discarded, and counted as a -# preparation failure for re-arm backoff pacing. -DEFAULT_PRISM_PAYOUT_ARTIFACT_MAX_ANCHOR_AGE_SECONDS = 300.0 -# Bounds one routine/initial admission wait behind publication-priority -# builds. A parked request past the deadline fails fast with a visible log -# line and counter instead of silently riding the client's own much longer -# timeout; the initial-job runner retries with its normal pacing. -DEFAULT_PRISM_ROUTINE_ADMISSION_DEADLINE_SECONDS = 15.0 -DEFAULT_PRISM_CTV_BROADCASTER_CHUNK_SIZE = 5 -DEFAULT_PRISM_REORG_RECONCILE_CACHE_SECONDS = 5.0 -DEFAULT_PRISM_HEALTH_REFRESH_SECONDS = 5.0 -DEFAULT_PRISM_METRICS_REFRESH_SECONDS = 5.0 -DEFAULT_PRISM_HEALTH_PENDING_REFRESH_MAX_AGE_SECONDS = 15.0 -DEFAULT_PRISM_HEALTH_TIP_POLL_MAX_AGE_SECONDS = 15.0 -DEFAULT_PRISM_STRATUM_SEND_TIMEOUT_SECONDS = 20.0 -DEFAULT_PRISM_STRATUM_MAX_CONNECTIONS = 384 -DEFAULT_PRISM_STRATUM_MAX_CONNECTIONS_PER_USERNAME = 0 -DEFAULT_PRISM_STRATUM_MAX_PENDING_INITIAL_JOBS = 128 -DEFAULT_PRISM_STRATUM_INITIAL_JOB_TIMEOUT_SECONDS = 30.0 -DEFAULT_PRISM_MINING_HEALTH_STARTUP_GRACE_SECONDS = 30.0 -DEFAULT_PRISM_STRATUM_ACCEPT_RESOURCE_EXHAUSTION_BACKOFF_SECONDS = 1.0 -# Kernel accept backlog per stratum listener. Sized so a whole fleet -# reconnecting inside a restart window parks in the backlog instead of being -# SYN-dropped; the kernel caps the effective value at net.core.somaxconn. -DEFAULT_PRISM_STRATUM_LISTEN_BACKLOG = 1024 -# How long bind() retries EADDRINUSE at startup, covering overlap with a -# predecessor process that is still draining its shutdown while holding the -# port. Zero fails fast (the historical behavior). -DEFAULT_PRISM_STRATUM_BIND_RETRY_SECONDS = 10.0 -DEFAULT_PRISM_PAYOUT_ADDRESS_CACHE_MAX_ENTRIES = 4_096 -DEFAULT_PRISM_PAYOUT_ADDRESS_CACHE_TTL_SECONDS = 3_600.0 -DEFAULT_PRISM_STALE_GRACE_SECONDS = 3.0 -# How old the refresh-published tip may be before mining.submit stops trusting -# it and falls back to a live getbestblockhash per share. Healthy coordinators -# republish/reconfirm it every blockpoll interval; a bounded divergence lease -# separately covers replacement construction without failing open forever. -DEFAULT_PRISM_SUBMIT_TIP_MAX_AGE_SECONDS = 10.0 -DEFAULT_PRISM_SAME_TIP_JOB_RETENTION_SECONDS = 30.0 -DEFAULT_PRISM_SAME_TIP_JOB_RETENTION_PER_CONNECTION = 64 -# How many same-tip retained contexts may outlive their connection so a -# reconnecting client (same authorized username) can submit work computed -# against the prior connection's jobs instead of eating unknown-job rejects -# across a proxy flap. Entries keep the normal same-tip TTL; 0 restores the -# historical purge-on-disconnect behavior. -DEFAULT_PRISM_DISCONNECTED_JOB_RETENTION = 16_384 -DEFAULT_PRISM_TIP_REFRESH_MAX_WORKERS = 16 -DEFAULT_PRISM_INITIAL_JOB_MAX_WORKERS = 4 -DEFAULT_PRISM_JOB_BUILD_TIMEOUT_SECONDS = 60.0 -DEFAULT_PRISM_JOB_BUILD_CANCEL_GRACE_SECONDS = 0.25 -DEFAULT_PRISM_JOB_BUILD_EXECUTOR_WORKERS = 2 -# Upper bound for PRISM_PYTHON_SWITCH_INTERVAL_SECONDS. The interpreter default -# is 5 ms, so anything approaching a second is a misplaced decimal point rather -# than a tuning intent -- and the failure it would cause is the one hardest to -# read back from a running pool: threads holding the GIL for whole seconds, -# presenting as ack-latency and lock-convoy symptoms with no obvious cause. -# Reject at startup instead. -MAX_PRISM_PYTHON_SWITCH_INTERVAL_SECONDS = 1.0 -# Lower bound, and it is the interpreter's own resolution rather than a policy -# choice: CPython stores this interval in whole microseconds, so anything below -# one truncates to *zero* -- sys.setswitchinterval(9e-07) leaves -# sys.getswitchinterval() reporting 0.0. A zero interval asks for a switch check -# at every opportunity, which is the opposite of what someone lowering the value -# is reaching for and is worst precisely here, in a thread-heavy coordinator. -# The same truncation applies harmlessly above the bound: a requested 1.5us -# applies as 1us, so a tuned value may read back slightly lower than it was set. -MIN_PRISM_PYTHON_SWITCH_INTERVAL_SECONDS = 1e-06 -DEFAULT_PRISM_VARDIFF_IDLE_SWEEP_SECONDS = 15.0 -# Reconnect difficulty retention: a reconnecting worker (same listener + -# exact username) resumes at its last converged difficulty instead of the -# lane start, so a mass reconnect does not re-flood the share path while -# every session re-climbs. The TTL bounds how stale a resumed value may be, -# the entry cap bounds retained state (2x the deployed 4096 connection cap), -# and the start factor caps a resume at a multiple of the lane's own start -# difficulty (1024 = five 4x retarget steps, the climb reconnects previously -# repeated). -DEFAULT_PRISM_VARDIFF_RESUME_TTL_SECONDS = 900.0 -DEFAULT_PRISM_VARDIFF_RESUME_MAX_ENTRIES = 8192 -DEFAULT_PRISM_VARDIFF_RESUME_MAX_START_FACTOR = Decimal("1024") -# Fast-arrival initial convergence defaults (#132). 64x in one move is what -# lets a standard-lane session started at 16384 cross the 500000 a rental -# router filters at without waiting out several ordinary 4x retargets; 8 -# accepted shares over at least 1 second, showing at least a 4x gap, is the -# evidence required before that one move is allowed. -DEFAULT_PRISM_VARDIFF_INITIAL_MAX_STEP_UP = Decimal("64") -DEFAULT_PRISM_VARDIFF_INITIAL_MIN_SHARES = 8 -DEFAULT_PRISM_VARDIFF_INITIAL_MIN_STEP_UP = Decimal("4") -DEFAULT_PRISM_VARDIFF_INITIAL_MIN_SECONDS = Decimal("1") -DEFAULT_PRISM_WORKER_METRICS_LIMIT = 100 -DEFAULT_SHARE_COMMIT_BATCH_SIZE = 64 -DEFAULT_SHARE_COMMIT_LINGER_MILLISECONDS = 5.0 -DEFAULT_SHARE_COMMIT_TIMEOUT_SECONDS = 15.0 -DEFAULT_PRISM_WRITER_QUIESCENCE_TIMEOUT_SECONDS = 15.0 -DEFAULT_PRISM_TEMPLATE_MAX_AGE_SECONDS = 120 -DEFAULT_PRISM_COORDINATION_BLOCKED_EXIT_SECONDS = 900.0 -# How long a monitored coordinator thread may stay heartbeat-silent before the -# watchdog hard-exits the process. Deadlines that are spent on a monitored -# thread derive their ceiling from this value rather than repeating the number, -# so retuning the watchdog cannot leave a budget stranded above its tolerance. -DEFAULT_PRISM_WATCHDOG_TIMEOUT_SECONDS = 120.0 -# A watchdog release must never turn a wedged DB path into a second outage. -# The release worker is daemonized and the watchdog hard-exits at this total -# wall-clock deadline whether quiescence or the fresh DB connection completes. -DEFAULT_PRISM_WATCHDOG_LEASE_RELEASE_TIMEOUT_SECONDS = 5.0 -# A fast-adoptable coordinator proves its guarded session live several times -# inside the silence proof required by PsqlShareLedger. This runs through a -# dedicated connection, not the ordinary ledger lock or shared native pool, -# and must never wait on the lease tuple's row lock: fenced writes hold it -# for whole transactions (persist_accepted_block can exceed the guard's -# statement timeout many times over). The lease TTL is renewed only when -# that tuple is uncontended (SKIP LOCKED), so an idle coordinator still -# keeps lease_expires_at ahead of different-identity expiry claims. -# -# The five values below are not independent knobs. They are the terms of one -# inequality — exit before adoption — owned by -# lab.prism.writer_lease_timing.WriterLeaseHeartbeatPolicy, which derives -# them from measurable phase budgets (the guard session's statement timeout, -# the heartbeat interval, and a process scheduler-slack allowance) and -# validates them at import. They are read from that policy rather than -# re-derived here, so retuning one term cannot silently break the argument -# the others depend on. See writer_lease_timing for the derivation and for -# why the shipped adoption silence is what it is (issue #212). -DEFAULT_PRISM_LEDGER_LEASE_HEARTBEAT_SECONDS = ( - DEFAULT_WRITER_LEASE_HEARTBEAT_POLICY.heartbeat_interval_seconds -) -# The monitor kills a coordinator before PostgreSQL can observe the full -# adoption silence if the heartbeat query itself wedges. Its short release -# budget keeps the old process gone before a successor is allowed to CAS. -DEFAULT_PRISM_LEDGER_LEASE_HEARTBEAT_FAILURE_SECONDS = ( - DEFAULT_WRITER_LEASE_HEARTBEAT_POLICY.failure_budget_seconds -) -DEFAULT_PRISM_LEDGER_LEASE_HEARTBEAT_MONITOR_SECONDS = ( - DEFAULT_WRITER_LEASE_HEARTBEAT_POLICY.monitor_interval_seconds -) -DEFAULT_PRISM_LEDGER_LEASE_HEARTBEAT_EXIT_TIMEOUT_SECONDS = ( - DEFAULT_WRITER_LEASE_HEARTBEAT_POLICY.exit_margin_seconds -) -DEFAULT_PRISM_LEDGER_LEASE_EXTERNAL_FENCE_TIMEOUT_SECONDS = 0.5 -# Headroom added on top of the longest guarded RPC deadline when sizing the -# ledger's own-write deferral margin. It absorbs what the deadline itself -# cannot: the scheduling gap between the fence verification returning and -# the RPC's first byte, coordinator-vs-DB clock drift (the lease runway is -# measured by clock_timestamp()), and the node's application tail — the -# client-side deadline severs the socket, which bounds transmission but -# cannot cancel a handler qbitd already received in full, and these RPC -# handlers apply within milliseconds of complete receipt. The client side -# of the RPC can never exceed its wall-clock deadline (mutating methods are -# in _QBIT_RPC_NO_TRANSPORT_RETRY_METHODS and never retry; retried read -# paths share one deadline across attempts), so a multiplicative factor -# here would only make the deferral needlessly eager as operators raise -# deadlines. A node that stalls longer than this headroom after fully -# receiving a mutating request is the documented residual of preflight -# fencing between independent systems (see the external-side-effect fence -# docstring); no client-side teardown can bound it. -LEASE_AUTHORITY_MARGIN_HEADROOM_SECONDS = 5.0 -# The node fast lane is intentionally shorter than the normal ten-second RPC -# budget: an ambiguous timeout leaves the durable outbox pending and replay -# safely submits the same hash again. -DEFAULT_BLOCK_SUBMIT_RPC_TIMEOUT_SECONDS = 1.0 -# Every PostgreSQL operation reached by the submitter inherits this deadline. -# Direct outbox calls also run behind a coordinator-side single-flight guard, -# so a driver that ignores the deadline cannot freeze the submitter thread. -DEFAULT_BLOCK_SUBMIT_DB_TIMEOUT_SECONDS = 1.0 -# Landing-class deadline. The post-acceptance landing/accounting tail -# (persist, prior balances, confirm, prepared-state rejection of a terminal -# candidate) must not share the one-second budget meant for cheap outbox -# polls and fast-lane-adjacent calls: a structurally slow landing under that -# ceiling is statement-canceled on every attempt and can never complete -# (issue #188 — mainnet block 42743's landing was canceled 13 times while -# Postgres was otherwise idle). The first landing attempt receives this full -# budget; retries escalate after observed landing timeouts. -DEFAULT_BLOCK_LANDING_DB_TIMEOUT_SECONDS = 30.0 -# Reviewed escalation cap for landing retries. Per-attempt wall clock stays -# bounded by the escalated statement budget, cycles are paced by the -# candidate retry backoff, and the stuck-call/coordination watchdogs remain -# the overall bound. An accepted block's landing is never abandoned outright; -# it converges through durable replay. This is an upper bound only: landing -# steps run on watchdog-monitored block-work threads, so the watchdog-derived -# ceiling below can lower the budget actually granted, and only lower it. -DEFAULT_BLOCK_LANDING_DB_TIMEOUT_MAX_SECONDS = 120.0 -# Fraction of the configured watchdog tolerance a landing budget may occupy. -# A landing step spends its budget on the block-work thread the watchdog -# monitors, so the reviewed cap and the watchdog are not independent knobs: -# a cap at or above the tolerance lets a legal, still-running call trip the -# watchdog, and the in-memory escalation state dies with the process, so the -# restart replays the same doomed attempt from the base budget forever -# (issue #125). The landing scope is bracketed to keep that span short: the -# ledger admission wait stamps in slices, gate bodies that hold their lock -# across two statements report between them, and the reorg walk, prior- -# balance checks, tip-height RPC and unfenced-append drain each stamp their -# own phases. What remains between two stamps is normally one server-side -# statement, so half the tolerance leaves roughly a statement of margin -# before the watchdog fires, and the two values can no longer drift apart -# when either is retuned. It is a margin, not a guarantee: a network stall -# mid-statement has no client-side socket-read deadline on the native -# psycopg path and can outlast any budget -- that case is a genuinely -# wedged process, and a watchdog exit is the correct outcome. -BLOCK_LANDING_DB_TIMEOUT_WATCHDOG_FRACTION = 0.5 -DEFAULT_BLOCK_SUBMIT_LOCK_WAIT_LOG_SECONDS = 5.0 -DEFAULT_BLOCK_SUBMIT_STUCK_CALL_EXIT_SECONDS = 30.0 -# How long an own-hash tip observation keeps protecting a block candidate -# from terminal abandonment while instantaneous chain probes disagree -# (transient sibling/fork views during quick-succession blocks, RPC blips). -# Within the window the candidate defers and retries until a fresh probe -# proves it active again (the accepted tail then finalizes it as submitted); -# once the window expires with the hash still absent from the active chain, -# the candidate is genuinely stale and abandons terminally. -DEFAULT_PRISM_OBSERVED_TIP_ACCEPT_WINDOW_SECONDS = 300.0 -# Fail-closed bound on unresolved accepted-parent transitions (landed -# candidates whose durable bookkeeping has not reached a terminal state). -# Child jobs may keep issuing against published previews while landings -# complete, but once this many transitions are unresolved at once, new job -# builds block instead of stacking further prospective balance chains. The -# durable outbox (MAX_PENDING_BLOCK_CANDIDATES) remains the wider bound. -DEFAULT_ACCEPTED_PARENT_UNRESOLVED_DEPTH_MAX = 8 -# In-process targeted ancestor re-drive (issue #190). A found block whose -# finalization keeps deferring on the same unresolved accepted-ancestor -# payout transition can only be resolved by re-driving that ancestor through -# the durable-replay path -- the deferral itself merely re-checks the -# transition. After this many consecutive deferrals on one ancestor the -# submitter forces a durable-outbox enumeration (the same code startup -# replay runs) so the stuck ancestor is re-adopted and finalized in-process -# instead of waiting out the publication-progress watchdog restart. -DEFAULT_ACCEPTED_PARENT_REDRIVE_DEFER_THRESHOLD = 3 -# How many re-drives one ancestor may trigger before the mechanism stands -# down for that hash. On exhaustion deferrals continue exactly as before -# the fix and the watchdog restart remains the backstop, so the re-drive -# itself can never livelock the submitter. -DEFAULT_ACCEPTED_PARENT_REDRIVE_ATTEMPT_MAX = 3 -# Issue #198: the decided-height collapse keeps one in-memory retry record -# per terminal row whose post-write cleanup failed (see -# ``_CollapsedCandidateCleanup`` in ``lab/prism/block_candidates.py``). Those -# records are the only replay source the row has left, so they are never -# dropped; what is bounded instead is *admission*: once the backlog holds -# this many records the collapse stops handing new rows to its fenced bulk -# terminalization and every unselected row stays durable and pending for -# the ordinary per-row path. -# -# The default is derived from the candidate-storm measurement recorded in -# ``docs/prism-overload-alerts.md``: one record costs on the order of a -# kilobyte plus the pending-share holders it retains, a drained backlog -# recovers at thousands of records per second on an idle accounting lane, -# and the registry pins one terminal-outcome fence per record. The bound is -# half the terminal-outcome registry (MAX_BLOCK_CANDIDATE_TERMINAL_OUTCOMES, -# 8,192) so a saturated backlog can never pin the whole fence registry, and -# it absorbs the observed 3,120-candidate storm (3,119 siblings) plus one -# full replay page (1,024 rows) without engaging. -DEFAULT_BLOCK_CANDIDATE_CLEANUP_RETRY_BACKLOG_MAX = 4096 -# The knob may not exceed the terminal-outcome registry bound: every retry -# record pins its hash's fence, and a backlog allowed to grow past the -# registry would leave the fence eviction with nothing it may drop. Restated -# here rather than imported because ``block_candidates`` imports this module; -# a block-candidate test pins the two equal. -MAX_BLOCK_CANDIDATE_CLEANUP_RETRY_BACKLOG_MAX = 8192 -# Maximum payload rows in a forced replay query. Operators may reduce this -# independently of the cleanup bounds when candidate JSON is very large. -MAX_BLOCK_REPLAY_PAGE_SIZE = 1024 -DEFAULT_HIGHDIFF_DIFFICULTY = "500000" -DEFAULT_HIGHDIFF_MAX_DIFFICULTY = "4294967296" -# Coinbase output ordering policies accepted by PRISM_COINBASE_OUTPUT_POLICY. -# "canonical" preserves the historical lexicographic output ordering; -# "pool-fee-first" reserves a direct settlement slot for a positive pool fee -# and emits it at coinbase vout 0. -VALID_COINBASE_OUTPUT_POLICIES = frozenset({"canonical", "pool-fee-first"}) - - -Env = Mapping[str, str] - - -def _current_environ(environ: Env | None) -> Env: - return _PROCESS_ENVIRON if environ is None else environ - - -def env(name: str, default: str | None = None, *, environ: Env | None = None) -> str: - value = _current_environ(environ).get(name, default) - if value is None or value == "": - raise SystemExit(f"{name} is required") - return value - - -def env_int(name: str, default: int, *, environ: Env | None = None) -> int: - return int(env(name, str(default), environ=environ)) - - -def env_positive_int(name: str, default: int, *, environ: Env | None = None) -> int: - try: - value = env_int(name, default, environ=environ) - except ValueError as exc: - raise SystemExit(f"{name} must be an integer") from exc - if value <= 0: - raise SystemExit(f"{name} must be positive") - return value - - -def env_positive_int_with_legacy( - primary_name: str, - legacy_name: str, - default: int, - *, - environ: Env | None = None, -) -> int: - if env_optional(primary_name, environ=environ) is not None: - return env_positive_int(primary_name, default, environ=environ) - return env_positive_int(legacy_name, default, environ=environ) - - -def env_nonnegative_int(name: str, default: int, *, environ: Env | None = None) -> int: - try: - value = env_int(name, default, environ=environ) - except ValueError as exc: - raise SystemExit(f"{name} must be an integer") from exc - if value < 0: - raise SystemExit(f"{name} must be non-negative") - return value - - -def env_nonnegative_int_with_legacy( - primary_name: str, - legacy_name: str, - default: int, - *, - environ: Env | None = None, -) -> int: - if env_optional(primary_name, environ=environ) is not None: - return env_nonnegative_int(primary_name, default, environ=environ) - return env_nonnegative_int(legacy_name, default, environ=environ) - - -def env_positive_float(name: str, default: float, *, environ: Env | None = None) -> float: - try: - value = float(env(name, str(default), environ=environ)) - except ValueError as exc: - raise SystemExit(f"{name} must be a number") from exc - if not math.isfinite(value): - raise SystemExit(f"{name} must be finite") - if value <= 0: - raise SystemExit(f"{name} must be positive") - return value - - -def env_nonnegative_float(name: str, default: float, *, environ: Env | None = None) -> float: - try: - value = float(env(name, str(default), environ=environ)) - except ValueError as exc: - raise SystemExit(f"{name} must be a number") from exc - if not math.isfinite(value): - raise SystemExit(f"{name} must be finite") - if value < 0: - raise SystemExit(f"{name} must be non-negative") - return value - - -def env_optional_positive_int(name: str, *, environ: Env | None = None) -> int | None: - raw = env_optional(name, environ=environ) - if raw is None: - return None - try: - value = int(raw) - except ValueError as exc: - raise SystemExit(f"{name} must be an integer") from exc - if value <= 0: - raise SystemExit(f"{name} must be positive") - return value - - -def env_optional_positive_float(name: str, *, environ: Env | None = None) -> float | None: - raw = env_optional(name, environ=environ) - if raw is None: - return None - try: - value = float(raw) - except ValueError as exc: - raise SystemExit(f"{name} must be a number") from exc - if not math.isfinite(value): - raise SystemExit(f"{name} must be finite") - if value <= 0: - raise SystemExit(f"{name} must be positive") - return value - - -def env_optional_positive_int_with_legacy( - primary_name: str, - legacy_name: str, - *, - environ: Env | None = None, -) -> int | None: - value = env_optional_positive_int(primary_name, environ=environ) - if value is not None: - return value - return env_optional_positive_int(legacy_name, environ=environ) - - -def env_decimal(name: str, default: str, *, environ: Env | None = None) -> Decimal: - try: - value = Decimal(env(name, default, environ=environ)) - except InvalidOperation as exc: - raise SystemExit(f"{name} must be a decimal number") from exc - if not value.is_finite(): - raise SystemExit(f"{name} must be finite") - if value <= 0: - raise SystemExit(f"{name} must be positive") - return value - - -def env_bool(name: str, default: str, *, environ: Env | None = None) -> bool: - return env(name, default, environ=environ).lower() in {"1", "true", "yes", "on"} - - -_STRICT_BOOL_TRUE = frozenset({"1", "true", "yes", "on"}) -_STRICT_BOOL_FALSE = frozenset({"0", "false", "no", "off"}) - - -def env_strict_bool(name: str, default: bool, *, environ: Env | None = None) -> bool: - """Boolean that fails closed: anything non-boolean refuses startup. - - ``env_bool`` silently reads unrecognized values as false, which is the - wrong failure mode for a pipeline-routing switch -- a typo must be a - visible startup error, never a silent flip to either side. - """ - raw = _current_environ(environ).get(name) - if raw is None or raw == "": - return default - normalized = raw.lower() - if normalized in _STRICT_BOOL_TRUE: - return True - if normalized in _STRICT_BOOL_FALSE: - return False - raise SystemExit(f"{name} must be a boolean (0/1/true/false/yes/no/on/off)") - - -def env_window_pipeline_rust(*, environ: Env | None = None) -> bool: - """Whether the payout-window pipeline routes through the Rust daemon. - - Independent of PRISM_BUILDER_SERVE: with the daemon transport disabled - this switch is inert (the materialization path checks the transport - itself), not an error. - """ - return env_strict_bool( - "PRISM_WINDOW_PIPELINE_RUST", - DEFAULT_PRISM_WINDOW_PIPELINE_RUST, - environ=environ, - ) - - -def env_malloc_telemetry(*, environ: Env | None = None) -> bool: - """Whether /metrics binds glibc mallinfo2 for the allocator gauges. - - Strict like the Rust switch: a non-boolean value refuses startup through - the coordinator config loader rather than silently landing on either - side. The metrics renderer reads the same strict switch when it is - constructed, so it can only ever see a value the loader already - validated in this process's environment. - """ - return env_strict_bool( - "PRISM_MALLOC_TELEMETRY", - DEFAULT_PRISM_MALLOC_TELEMETRY, - environ=environ, - ) - - -def env_heap_census_config( - *, - environ: Env | None = None, - audit_dir: Path | None = None, -) -> HeapCensusConfig: - """The issue #226 census, trim, and tracing switches, validated fail-closed. - - Every switch is strict like the Rust and malloc-telemetry switches, and - every bound is range-checked against the ceilings beside its default: a - census that is "bounded" by a top-N of a million or a walk cap of an hour - is not bounded, so those values refuse startup. ``audit_dir`` is the - resolved ``PRISM_AUDIT_DIR`` when the coordinator loader already has it; - standalone callers get the same derivation the loader uses. - """ - source = _current_environ(environ) - if audit_dir is None: - evidence_path = Path(env("PRISM_EVIDENCE_PATH", "prism-live-evidence.json", environ=source)) - audit_dir = Path(env("PRISM_AUDIT_DIR", str(evidence_path.parent), environ=source)) - enabled = env_strict_bool("PRISM_HEAP_CENSUS", DEFAULT_PRISM_HEAP_CENSUS, environ=source) - # Optional: the compose passthrough delivers an empty string when the - # deploy dotenv leaves it unset, and empty must mean the default, not a - # refused startup. - output_dir = env_optional("PRISM_HEAP_CENSUS_DIR", environ=source) or str( - audit_dir / DEFAULT_PRISM_HEAP_CENSUS_SUBDIR - ) - top_n = env_positive_int( - "PRISM_HEAP_CENSUS_TOP_N", DEFAULT_PRISM_HEAP_CENSUS_TOP_N, environ=source - ) - if top_n > MAX_PRISM_HEAP_CENSUS_TOP_N: - raise SystemExit( - f"PRISM_HEAP_CENSUS_TOP_N must be at most {MAX_PRISM_HEAP_CENSUS_TOP_N}" - ) - max_bytes = env_positive_int( - "PRISM_HEAP_CENSUS_MAX_BYTES", DEFAULT_PRISM_HEAP_CENSUS_MAX_BYTES, environ=source - ) - if not MIN_PRISM_HEAP_CENSUS_MAX_BYTES <= max_bytes <= MAX_PRISM_HEAP_CENSUS_MAX_BYTES: - raise SystemExit( - "PRISM_HEAP_CENSUS_MAX_BYTES must be between " - f"{MIN_PRISM_HEAP_CENSUS_MAX_BYTES} and {MAX_PRISM_HEAP_CENSUS_MAX_BYTES}" - ) - max_seconds = env_positive_float( - "PRISM_HEAP_CENSUS_MAX_SECONDS", DEFAULT_PRISM_HEAP_CENSUS_MAX_SECONDS, environ=source - ) - if max_seconds > MAX_PRISM_HEAP_CENSUS_MAX_SECONDS: - raise SystemExit( - f"PRISM_HEAP_CENSUS_MAX_SECONDS must be at most {MAX_PRISM_HEAP_CENSUS_MAX_SECONDS:g}" - ) - min_interval = env_nonnegative_float( - "PRISM_HEAP_CENSUS_MIN_INTERVAL_SECONDS", - DEFAULT_PRISM_HEAP_CENSUS_MIN_INTERVAL_SECONDS, - environ=source, - ) - tracemalloc_enabled = env_strict_bool( - "PRISM_HEAP_CENSUS_TRACEMALLOC", DEFAULT_PRISM_HEAP_CENSUS_TRACEMALLOC, environ=source - ) - if tracemalloc_enabled and not enabled: - raise SystemExit("PRISM_HEAP_CENSUS_TRACEMALLOC requires PRISM_HEAP_CENSUS=1") - trim_signal = env_strict_bool( - "PRISM_MALLOC_TRIM_SIGNAL", DEFAULT_PRISM_MALLOC_TRIM_SIGNAL, environ=source - ) - trim_interval = env_nonnegative_float( - "PRISM_MALLOC_TRIM_INTERVAL_SECONDS", - DEFAULT_PRISM_MALLOC_TRIM_INTERVAL_SECONDS, - environ=source, - ) - if 0 < trim_interval < MIN_PRISM_MALLOC_TRIM_INTERVAL_SECONDS: - raise SystemExit( - "PRISM_MALLOC_TRIM_INTERVAL_SECONDS must be 0 or at least " - f"{MIN_PRISM_MALLOC_TRIM_INTERVAL_SECONDS:g}" - ) - return HeapCensusConfig( - enabled=enabled, - output_dir=output_dir, - top_n=top_n, - max_bytes=max_bytes, - max_seconds=max_seconds, - min_interval_seconds=min_interval, - tracemalloc_enabled=tracemalloc_enabled, - malloc_trim_signal_enabled=trim_signal, - malloc_trim_interval_seconds=trim_interval, - ) - - -# The census as shipped: every switch off, every bound at its default. The -# lifecycle config carries this so a positional construction keeps working. -DEFAULT_PRISM_HEAP_CENSUS_CONFIG = HeapCensusConfig( - enabled=DEFAULT_PRISM_HEAP_CENSUS, - output_dir=str(Path("prism-live-evidence.json").parent / DEFAULT_PRISM_HEAP_CENSUS_SUBDIR), - top_n=DEFAULT_PRISM_HEAP_CENSUS_TOP_N, - max_bytes=DEFAULT_PRISM_HEAP_CENSUS_MAX_BYTES, - max_seconds=DEFAULT_PRISM_HEAP_CENSUS_MAX_SECONDS, - min_interval_seconds=DEFAULT_PRISM_HEAP_CENSUS_MIN_INTERVAL_SECONDS, - tracemalloc_enabled=DEFAULT_PRISM_HEAP_CENSUS_TRACEMALLOC, - malloc_trim_signal_enabled=DEFAULT_PRISM_MALLOC_TRIM_SIGNAL, - malloc_trim_interval_seconds=DEFAULT_PRISM_MALLOC_TRIM_INTERVAL_SECONDS, -) - - -def env_optional_bool(name: str, *, environ: Env | None = None) -> bool | None: - raw = env_optional(name, environ=environ) - if raw is None: - return None - return raw.lower() in {"1", "true", "yes", "on"} - - -def env_optional(name: str, *, environ: Env | None = None) -> str | None: - value = _current_environ(environ).get(name) - if value is None or value == "": - return None - return value - - -def production_mode(*, environ: Env | None = None) -> bool: - return ( - env_bool("QBIT_PRODUCTION", "0", environ=environ) - or env_bool("QBIT_TOOLS_PRODUCTION", "0", environ=environ) - or env("QBIT_CHAIN", "regtest", environ=environ).lower() in {"main", "mainnet"} - ) - - -def validate_same_tip_job_retention_limits( - *, - retention_seconds: float, - per_connection: int, - max_connections: int, - production: bool, -) -> None: - if retention_seconds <= 0: - return - if per_connection <= 0: - raise SystemExit( - "PRISM_STRATUM_SAME_TIP_JOB_RETENTION_PER_CONNECTION must be positive " - "when same-tip retention is enabled" - ) - if production and max_connections <= 0: - raise SystemExit( - "production mode requires a positive PRISM_STRATUM_MAX_CONNECTIONS " - "when same-tip retention is enabled" - ) - - -def validate_block_candidate_cleanup_retry_backlog_max(value: int) -> int: - """Refuse a cleanup-retry backlog bound the fence registry cannot carry. - - Positivity is checked by the env reader; this is the upper bound. The - error names both the limit and why it exists so an operator retuning - the knob learns the contract rather than a bare number. - """ - if int(value) > MAX_BLOCK_CANDIDATE_CLEANUP_RETRY_BACKLOG_MAX: - raise SystemExit( - "PRISM_BLOCK_CANDIDATE_CLEANUP_RETRY_BACKLOG_MAX cannot exceed " - f"{MAX_BLOCK_CANDIDATE_CLEANUP_RETRY_BACKLOG_MAX}: every retry " - "record pins a terminal-outcome fence and the fence registry is " - "bounded at that size" - ) - return int(value) - - -def require_production_env(name: str, *, environ: Env | None = None) -> str: - value = env_optional(name, environ=environ) - if value is None: - raise SystemExit(f"production mode requires {name}") - return value - - -def validate_prism_production_gate(*, environ: Env | None = None) -> None: - if not production_mode(environ=environ): - return - - for name in ( - "PRISM_ALLOW_MEMORY_LEDGER", - "PRISM_ALLOW_TEST_SIGNING_SEEDS", - "PRISM_ALLOW_BUNDLE_EMBEDDED_LEDGER_KEY", - "PRISM_ALLOW_FIXED_LEDGER_SESSION_TOKEN", - ): - if env_bool(name, "0", environ=environ): - raise SystemExit(f"production mode rejects {name}=1") - - # Stale grace is deliberately not pinned to zero on mainnet: with a zero - # window every in-flight prior-tip share is rejected (Stratum error 21) at - # each block, which miners read as pool failure rather than as stale - # work. The bounded default credits those shares instead (see - # docs/prism-rejections.md; bundles with credited rows use the v1.1 - # audit schema, which this release's verifier accepts). - - production_difficulties: dict[str, Decimal] = {} - for name in ( - "PRISM_STRATUM_SHARE_DIFF", - "PRISM_STRATUM_VARDIFF_MIN_DIFF", - "PRISM_STRATUM_VARDIFF_START_DIFF", - "PRISM_STRATUM_VARDIFF_MAX_DIFF", - ): - raw_value = require_production_env(name, environ=environ) - if not raw_value: - raise SystemExit(f"production mode requires an explicit {name}") - try: - value = Decimal(raw_value) - except InvalidOperation as exc: - raise SystemExit(f"{name} must be a decimal number") from exc - if not value.is_finite() or value <= 0: - raise SystemExit(f"{name} must be positive") - if value == Decimal("0.000000001"): - raise SystemExit(f"{name} cannot use the lab-only 1e-9 difficulty") - production_difficulties[name] = value - if ( - production_difficulties["PRISM_STRATUM_VARDIFF_MIN_DIFF"] - > production_difficulties["PRISM_STRATUM_VARDIFF_START_DIFF"] - ): - raise SystemExit("production vardiff minimum exceeds its start difficulty") - if ( - production_difficulties["PRISM_STRATUM_VARDIFF_START_DIFF"] - > production_difficulties["PRISM_STRATUM_VARDIFF_MAX_DIFF"] - ): - raise SystemExit("production vardiff start exceeds its maximum difficulty") - - prism_database_url = env_optional("PRISM_DATABASE_URL", environ=environ) - if prism_database_url is None and env_optional("PRISM_POSTGRES_PSQL_COMMAND", environ=environ) is None: - raise SystemExit("production mode requires PRISM_DATABASE_URL or PRISM_POSTGRES_PSQL_COMMAND") - if env_optional("PRISM_POSTGRES_PASSWORD", environ=environ) == "change-this": - raise SystemExit("production mode requires a non-default PRISM_POSTGRES_PASSWORD") - if prism_database_url is not None and "change-this" in prism_database_url: - raise SystemExit("production mode requires a non-default PRISM_DATABASE_URL") - - require_production_env("PRISM_MANIFEST_SIGNING_SEED_HEX", environ=environ) - require_production_env("PRISM_LEDGER_ATTESTATION_SIGNING_SEED_HEX", environ=environ) - require_production_env("PRISM_LEDGER_WRITER_PUBLIC_KEY_HEX", environ=environ) - require_production_env("PRISM_LEDGER_WRITER_ID", environ=environ) - require_production_env("PRISM_LEDGER_WRITER_EPOCH", environ=environ) - require_production_env("PRISM_AUDIT_DIR", environ=environ) - require_production_env("PRISM_EVIDENCE_PATH", environ=environ) - - if env_optional("PRISM_LEDGER_WRITER_SESSION_TOKEN", environ=environ) is not None: - raise SystemExit("production mode requires managed ledger session tokens; unset PRISM_LEDGER_WRITER_SESSION_TOKEN") - - if env_nonnegative_int( - "PRISM_STRATUM_MAX_CONNECTIONS", - DEFAULT_PRISM_STRATUM_MAX_CONNECTIONS, - environ=environ, - ) <= 0: - raise SystemExit( - "production mode requires a positive PRISM_STRATUM_MAX_CONNECTIONS" - ) - env_positive_int( - "PRISM_STRATUM_MAX_PENDING_INITIAL_JOBS", - DEFAULT_PRISM_STRATUM_MAX_PENDING_INITIAL_JOBS, - environ=environ, - ) - if env_nonnegative_float( - "PRISM_STRATUM_INITIAL_JOB_TIMEOUT_SECONDS", - DEFAULT_PRISM_STRATUM_INITIAL_JOB_TIMEOUT_SECONDS, - environ=environ, - ) <= 0: - raise SystemExit( - "production mode requires a positive " - "PRISM_STRATUM_INITIAL_JOB_TIMEOUT_SECONDS" - ) - - require_production_env("QBIT_RPC_USER", environ=environ) - qbit_rpc_password = require_production_env("QBIT_RPC_PASSWORD", environ=environ) - if qbit_rpc_password == "change-this": - raise SystemExit("production mode requires a non-default QBIT_RPC_PASSWORD") - - if env("QBIT_CHAIN", "regtest", environ=environ).lower() in {"main", "mainnet"} and env_bool( - "PRISM_CTV_SETTLEMENT_ENABLED", "0", environ=environ - ): - require_production_env( - "PRISM_CTV_FANOUT_FEE_MARKET_RATE_BITS_PER_1000_WEIGHT", environ=environ - ) - env_positive_int( - "PRISM_CTV_FANOUT_FEE_MARKET_RATE_BITS_PER_1000_WEIGHT", 0, environ=environ - ) - - validate_same_tip_job_retention_limits( - retention_seconds=env_nonnegative_float( - "PRISM_STRATUM_SAME_TIP_JOB_RETENTION_SECONDS", - DEFAULT_PRISM_SAME_TIP_JOB_RETENTION_SECONDS, - environ=environ, - ), - per_connection=env_nonnegative_int( - "PRISM_STRATUM_SAME_TIP_JOB_RETENTION_PER_CONNECTION", - DEFAULT_PRISM_SAME_TIP_JOB_RETENTION_PER_CONNECTION, - environ=environ, - ), - max_connections=env_nonnegative_int( - "PRISM_STRATUM_MAX_CONNECTIONS", - DEFAULT_PRISM_STRATUM_MAX_CONNECTIONS, - environ=environ, - ), - production=True, - ) - - -def validate_hex(value: str, *, name: str, expected_bytes: int | None = None) -> str: - try: - bytes.fromhex(value) - except ValueError as exc: - raise SystemExit(f"{name} must be hex") from exc - if expected_bytes is not None and len(value) != expected_bytes * 2: - raise SystemExit(f"{name} must be {expected_bytes * 2} hex chars") - return value.lower() - - -def env_seed_hex(name: str, *, test_default: str, environ: Env | None = None) -> str: - value = env_optional(name, environ=environ) - if value is None: - if env_bool("PRISM_ALLOW_TEST_SIGNING_SEEDS", "0", environ=environ): - value = test_default - else: - raise SystemExit(f"{name} is required") - return validate_hex(value, name=name, expected_bytes=32) - - -def validate_initial_job_max_workers(workers: int, max_pending: int) -> int: - """Reject first-job worker counts the pending-client bound cannot feed. - - The dedicated executor's queue is bounded by - PRISM_STRATUM_MAX_PENDING_INITIAL_JOBS, so workers beyond that count can - never receive a task; unbounded values would also risk partially failed - thread creation on constrained hosts. - """ - if workers <= 0: - raise SystemExit("PRISM_INITIAL_JOB_MAX_WORKERS must be positive") - if workers > max_pending: - raise SystemExit( - "PRISM_INITIAL_JOB_MAX_WORKERS cannot exceed " - "PRISM_STRATUM_MAX_PENDING_INITIAL_JOBS" - ) - return workers - - -def resolve_initial_job_max_workers( - explicit: int | None, - max_pending: int, -) -> int: - """Resolve the first-job worker count against the pending-client bound. - - Deployments that bound pending clients below the default worker count - were valid before the worker knob existed; the implicit default must cap - itself to the pending bound rather than fail their startup. An explicit - setting keeps strict validation. - """ - if explicit is None: - return min(DEFAULT_PRISM_INITIAL_JOB_MAX_WORKERS, int(max_pending)) - return validate_initial_job_max_workers(explicit, max_pending) - - -def validate_job_build_executor_workers(workers: int) -> int: - """Reject executor widths the build scheduler can never use. - - The scheduler admits at most two concurrent build flights (one active, - one retiring); extra executor threads would never receive work, so a - higher setting would be a silent no-op rather than a tuning gain. - """ - if workers <= 0: - raise SystemExit("PRISM_JOB_BUILD_EXECUTOR_WORKERS must be positive") - if workers > DEFAULT_PRISM_JOB_BUILD_EXECUTOR_WORKERS: - raise SystemExit( - "PRISM_JOB_BUILD_EXECUTOR_WORKERS cannot exceed " - f"{DEFAULT_PRISM_JOB_BUILD_EXECUTOR_WORKERS}" - ) - return workers - - -def resolve_python_switch_interval_seconds( - *, environ: Env | None = None -) -> float | None: - """Return the configured interpreter switch interval, or None to leave it. - - Unset means the interpreter keeps whatever CPython defaults to (5 ms on - every version this pool has run), which is the deployed behaviour today. - Absent is therefore not "use 5 ms" but "do not call - ``sys.setswitchinterval`` at all" -- the two differ if a future - interpreter changes its own default, and this knob should not pin a value - the tree never chose. - """ - interval = env_optional_positive_float( - "PRISM_PYTHON_SWITCH_INTERVAL_SECONDS", environ=environ - ) - if interval is None: - return None - if interval < MIN_PRISM_PYTHON_SWITCH_INTERVAL_SECONDS: - raise SystemExit( - "PRISM_PYTHON_SWITCH_INTERVAL_SECONDS cannot be below " - f"{MIN_PRISM_PYTHON_SWITCH_INTERVAL_SECONDS} " - "(the interpreter would truncate it to zero)" - ) - if interval > MAX_PRISM_PYTHON_SWITCH_INTERVAL_SECONDS: - raise SystemExit( - "PRISM_PYTHON_SWITCH_INTERVAL_SECONDS cannot exceed " - f"{MAX_PRISM_PYTHON_SWITCH_INTERVAL_SECONDS}" - ) - return interval - - -def apply_python_switch_interval(*, environ: Env | None = None) -> float | None: - """Apply the configured switch interval; return what was applied, if any. - - Must run before any thread starts. The interval decides how long a - CPU-bound thread may hold the GIL before the interpreter forces a switch, - so it is process-global state that existing threads observe unevenly if it - changes underneath them. - - #143's gate evidence measured this as a real lever rather than a - micro-optimisation: a sweep moved throughput 15% and CPU-per-share 13%, - and the 5 ms default had the worst ack tail of the three settings tested. - The right value is host-specific -- it trades share-ack tail latency - against interpreter overhead, and the balance shifts with core count and - per-core speed -- so this ships as a knob with no default rather than as a - value picked on one machine and inherited everywhere. - """ - interval = resolve_python_switch_interval_seconds(environ=environ) - if interval is None: - return None - sys.setswitchinterval(interval) - return interval - - -def load_prism_vardiff_config( - startup_difficulty: Decimal, *, environ: Env | None = None -) -> vardiff.VardiffConfig: - initial_convergence_enabled = env_bool( - "PRISM_STRATUM_VARDIFF_INITIAL_CONVERGENCE", - "1", - environ=environ, - ) - high_diff_arrival_threshold = env_decimal( - "PRISM_STRATUM_VARDIFF_HIGH_DIFF_ARRIVAL_DIFF", - DEFAULT_HIGHDIFF_DIFFICULTY, - environ=environ, - ) - max_step_factor = env_decimal( - "PRISM_STRATUM_VARDIFF_MAX_STEP_UP", "4", environ=environ - ) - if initial_convergence_enabled: - initial_min_shares = env_int( - "PRISM_STRATUM_VARDIFF_INITIAL_MIN_SHARES", - DEFAULT_PRISM_VARDIFF_INITIAL_MIN_SHARES, - environ=environ, - ) - if initial_min_shares < 1: - raise SystemExit( - "PRISM_STRATUM_VARDIFF_INITIAL_MIN_SHARES must be at least 1" - ) - initial_max_step_factor = env_decimal( - "PRISM_STRATUM_VARDIFF_INITIAL_MAX_STEP_UP", - str(DEFAULT_PRISM_VARDIFF_INITIAL_MAX_STEP_UP), - environ=environ, - ) - initial_min_step_factor = env_decimal( - "PRISM_STRATUM_VARDIFF_INITIAL_MIN_STEP_UP", - str(DEFAULT_PRISM_VARDIFF_INITIAL_MIN_STEP_UP), - environ=environ, - ) - if initial_min_step_factor < 1: - raise SystemExit( - "PRISM_STRATUM_VARDIFF_INITIAL_MIN_STEP_UP must be at least 1" - ) - initial_min_seconds = env_decimal( - "PRISM_STRATUM_VARDIFF_INITIAL_MIN_SECONDS", - str(DEFAULT_PRISM_VARDIFF_INITIAL_MIN_SECONDS), - environ=environ, - ) - if initial_max_step_factor < max_step_factor: - raise SystemExit( - "PRISM_STRATUM_VARDIFF_INITIAL_MAX_STEP_UP is below " - "PRISM_STRATUM_VARDIFF_MAX_STEP_UP" - ) - else: - # The relaxation is off, so do not parse its environment settings: - # initial_convergence_ready() refuses immediately on - # initial_convergence_enabled, and calculate_next_difficulty() keeps - # the ordinary max_step_factor. Defaults keep VardiffConfig's - # unconditional invariants satisfied without allowing dead settings - # (including malformed strings) to prevent startup. - initial_min_shares = DEFAULT_PRISM_VARDIFF_INITIAL_MIN_SHARES - initial_min_step_factor = DEFAULT_PRISM_VARDIFF_INITIAL_MIN_STEP_UP - initial_max_step_factor = max( - DEFAULT_PRISM_VARDIFF_INITIAL_MAX_STEP_UP, - max_step_factor, - ) - initial_min_seconds = DEFAULT_PRISM_VARDIFF_INITIAL_MIN_SECONDS - return vardiff.VardiffConfig( - enabled=env_bool("PRISM_STRATUM_VARDIFF", "1", environ=environ), - target_share_interval_seconds=env_decimal( - "PRISM_STRATUM_VARDIFF_TARGET_SECONDS", "15", environ=environ - ), - min_difficulty=env_decimal( - "PRISM_STRATUM_VARDIFF_MIN_DIFF", str(startup_difficulty), environ=environ - ), - max_difficulty=env_decimal("PRISM_STRATUM_VARDIFF_MAX_DIFF", "1024", environ=environ), - retarget_interval_seconds=env_decimal( - "PRISM_STRATUM_VARDIFF_RETARGET_SECONDS", "90", environ=environ - ), - max_step_factor=max_step_factor, - startup_difficulty=env_decimal( - "PRISM_STRATUM_VARDIFF_START_DIFF", str(startup_difficulty), environ=environ - ), - max_step_down_factor=env_decimal( - "PRISM_STRATUM_VARDIFF_MAX_STEP_DOWN", "4", environ=environ - ), - ewma_alpha=env_decimal("PRISM_STRATUM_VARDIFF_EWMA_ALPHA", "0.4", environ=environ), - retarget_tolerance=env_decimal( - "PRISM_STRATUM_VARDIFF_RETARGET_TOLERANCE", "0.25", environ=environ - ), - initial_convergence_enabled=initial_convergence_enabled, - initial_max_step_factor=initial_max_step_factor, - initial_min_accepted_shares=initial_min_shares, - initial_min_step_factor=initial_min_step_factor, - initial_min_elapsed_seconds=initial_min_seconds, - high_diff_arrival_threshold=high_diff_arrival_threshold, - ) - - -@dataclass(frozen=True) -class StratumListenerProfile: - """One stratum listener with its own difficulty policy. - - Every listener feeds the same coordinator, ledger, and settlement path; - the profile only decides where to listen and which difficulty bounds its - clients get. - """ - - name: str - bind: str - port: int - share_difficulty: Decimal - vardiff_config: vardiff.VardiffConfig - heartbeat_name: str - # Difficulty this listener never advertises below, even when the qbit - # network target is easier. Zero means no floor: stamped jobs keep the - # network cap (a share is never required to be harder than a block). The - # high-diff listener sets its configured minimum because marketplace - # verification (NiceHash-style) checks the first advertised difficulty, - # which must hold even while network difficulty sits below the floor. - minimum_advertised_difficulty: Decimal = Decimal("0") - - -def load_prism_highdiff_listener( - base_bind: str, - base_vardiff_config: vardiff.VardiffConfig, - *, - environ: Env | None = None, -) -> StratumListenerProfile | None: - """Optional high-difficulty listener for rental-scale miners. - - Disabled unless PRISM_STRATUM_HIGHDIFF_PORT is set. The 500k default floor - matches the NiceHash SHA-256 pool-verification minimum, which must hold - from the first mining.set_difficulty a client sees. - """ - - port_value = env_optional("PRISM_STRATUM_HIGHDIFF_PORT", environ=environ) - if port_value is None: - return None - try: - port = int(port_value) - except ValueError as exc: - raise SystemExit("PRISM_STRATUM_HIGHDIFF_PORT must be an integer") from exc - if not 0 < port < 65536: - raise SystemExit("PRISM_STRATUM_HIGHDIFF_PORT must be a valid TCP port") - min_difficulty = env_decimal( - "PRISM_STRATUM_HIGHDIFF_MIN_DIFF", DEFAULT_HIGHDIFF_DIFFICULTY, environ=environ - ) - start_difficulty = env_decimal( - "PRISM_STRATUM_HIGHDIFF_START_DIFF", DEFAULT_HIGHDIFF_DIFFICULTY, environ=environ - ) - max_difficulty = env_decimal( - "PRISM_STRATUM_HIGHDIFF_MAX_DIFF", DEFAULT_HIGHDIFF_MAX_DIFFICULTY, environ=environ - ) - if min_difficulty > start_difficulty: - raise SystemExit("PRISM_STRATUM_HIGHDIFF_MIN_DIFF exceeds PRISM_STRATUM_HIGHDIFF_START_DIFF") - if start_difficulty > max_difficulty: - raise SystemExit("PRISM_STRATUM_HIGHDIFF_START_DIFF exceeds PRISM_STRATUM_HIGHDIFF_MAX_DIFF") - # The fixed difficulty (used when vardiff is disabled) tracks the start - # difficulty unless explicitly set, and must respect the listener bounds: - # advertising below the floor would break the marketplace verification - # this listener exists for. - share_value = env_optional("PRISM_STRATUM_HIGHDIFF_SHARE_DIFF", environ=environ) - if share_value is None: - share_difficulty = start_difficulty - else: - try: - share_difficulty = Decimal(share_value) - except Exception as exc: - raise SystemExit("PRISM_STRATUM_HIGHDIFF_SHARE_DIFF must be a decimal") from exc - if not share_difficulty.is_finite() or share_difficulty <= 0: - raise SystemExit("PRISM_STRATUM_HIGHDIFF_SHARE_DIFF must be positive") - if share_difficulty < min_difficulty: - raise SystemExit("PRISM_STRATUM_HIGHDIFF_SHARE_DIFF is below PRISM_STRATUM_HIGHDIFF_MIN_DIFF") - if share_difficulty > max_difficulty: - raise SystemExit("PRISM_STRATUM_HIGHDIFF_SHARE_DIFF exceeds PRISM_STRATUM_HIGHDIFF_MAX_DIFF") - try: - config = dataclass_replace( - base_vardiff_config, - min_difficulty=min_difficulty, - max_difficulty=max_difficulty, - startup_difficulty=start_difficulty, - ) - except ValueError as exc: - raise SystemExit(f"invalid PRISM_STRATUM_HIGHDIFF_* difficulty bounds: {exc}") from exc - return StratumListenerProfile( - name="highdiff", - # env_optional so an empty value (compose default passthrough) means - # "inherit the default listener bind" instead of a startup failure. - bind=env_optional("PRISM_STRATUM_HIGHDIFF_BIND", environ=environ) or base_bind, - port=port, - share_difficulty=share_difficulty, - vardiff_config=config, - heartbeat_name="stratum_accept_highdiff", - minimum_advertised_difficulty=min_difficulty, - ) - - -def default_prism_payout_policy(*, environ: Env | None = None) -> dict[str, object]: - policy: dict[str, object] = { - "p2mr_spend_input_bytes": env_positive_int( - "PRISM_PAYOUT_P2MR_SPEND_INPUT_BYTES", - DEFAULT_P2MR_SPEND_INPUT_BYTES, - environ=environ, - ), - "target_feerate_sats_per_byte": env_positive_int_with_legacy( - "PRISM_PAYOUT_TARGET_FEERATE_BITS_PER_BYTE", - "PRISM_PAYOUT_TARGET_FEERATE_SATS_PER_BYTE", - DEFAULT_MIN_OUTPUT_FEERATE_SATS_PER_BYTE, - environ=environ, - ), - "safety_multiplier": env_positive_int( - "PRISM_PAYOUT_SAFETY_MULTIPLIER", - DEFAULT_MIN_OUTPUT_SAFETY_MULTIPLIER, - environ=environ, - ), - } - min_output_sats = env_optional_positive_int_with_legacy( - "PRISM_PAYOUT_MIN_OUTPUT_BITS", - "PRISM_PAYOUT_MIN_OUTPUT_SATS", - environ=environ, - ) - if min_output_sats is not None: - policy["min_output_sats"] = min_output_sats - return policy - - -def default_prism_coinbase_tag_hex(*, environ: Env | None = None) -> str: - tag = _current_environ(environ).get("PRISM_COINBASE_TAG", DEFAULT_PRISM_COINBASE_TAG) - try: - tag_bytes = tag.encode("ascii") - except UnicodeEncodeError as exc: - raise SystemExit("PRISM_COINBASE_TAG must be ASCII") from exc - if len(tag_bytes) > MAX_PRISM_COINBASE_TAG_BYTES: - raise SystemExit( - f"PRISM_COINBASE_TAG must be at most {MAX_PRISM_COINBASE_TAG_BYTES} bytes" - ) - if any(byte < 0x20 or byte > 0x7e for byte in tag_bytes): - raise SystemExit("PRISM_COINBASE_TAG must contain printable ASCII only") - return tag_bytes.hex() - - -def default_prism_username_fallback_address(*, environ: Env | None = None) -> str | None: - configured = env_optional("PRISM_USERNAME_FALLBACK_ADDRESS", environ=environ) - if configured is not None: - return configured - if (_current_environ(environ).get("QBIT_CHAIN") or "regtest").lower() in TESTNET_QBIT_CHAINS: - return DEFAULT_TESTNET_USERNAME_FALLBACK_ADDRESS - return None - - -def _parse_share_weights(raw: str) -> tuple[tuple[str, int], ...]: - if not raw: - return () - try: - parsed = json.loads(raw) - except json.JSONDecodeError as exc: - raise SystemExit(f"PRISM_STRATUM_SHARE_WEIGHTS_JSON is not valid JSON: {exc}") from exc - if not isinstance(parsed, dict): - raise SystemExit("PRISM_STRATUM_SHARE_WEIGHTS_JSON must be an object") - weights: list[tuple[str, int]] = [] - for username, weight in parsed.items(): - parsed_weight = int(weight) - if parsed_weight <= 0: - raise SystemExit(f"share weight for {username} must be positive") - weights.append((str(username), parsed_weight)) - return tuple(weights) - - -def load_share_weights(*, environ: Env | None = None) -> dict[str, int]: - """Load the legacy username-weight mapping for compatibility callers.""" - - source = _current_environ(environ) - return dict(_parse_share_weights(source.get("PRISM_STRATUM_SHARE_WEIGHTS_JSON", ""))) - - -@dataclass(frozen=True) -class RpcConfig: - host: str - port: int - user: str - password: str - chain: str - expected_genesis_hash: str | None - minimum_peers_raw: str | None - - -@dataclass(frozen=True) -class StratumConfig: - bind: str - port: int - extranonce2_size: int - stale_grace_seconds: float - send_timeout_seconds: float - max_connections: int - max_connections_per_username: int - max_pending_initial_jobs: int - initial_job_timeout_seconds: float - initial_job_max_workers: int - accept_resource_exhaustion_backoff_seconds: float - listen_backlog: int - bind_retry_seconds: float - share_difficulty: Decimal - vardiff_config: vardiff.VardiffConfig - vardiff_idle_sweep_seconds: float - listener_profiles: tuple[StratumListenerProfile, ...] - default_share_weight: int - share_weights_by_username: tuple[tuple[str, int], ...] - username_fallback_address: str | None - fallback_version_mask: int - same_tip_job_retention_seconds: float - same_tip_job_retention_per_connection: int - disconnected_job_retention: int - payout_address_cache_max_entries: int - payout_address_cache_ttl_seconds: float - # Reconnect difficulty retention knobs, appended with defaults so existing - # positional constructions keep working. - vardiff_resume_enabled: bool = True - vardiff_resume_ttl_seconds: float = DEFAULT_PRISM_VARDIFF_RESUME_TTL_SECONDS - vardiff_resume_max_entries: int = DEFAULT_PRISM_VARDIFF_RESUME_MAX_ENTRIES - vardiff_resume_max_start_factor: Decimal = ( - DEFAULT_PRISM_VARDIFF_RESUME_MAX_START_FACTOR - ) - - -@dataclass(frozen=True) -class JobPipelineConfig: - blockpoll_seconds: float - blockwait_enabled: bool - blockwait_timeout_seconds: float - tip_refresh_failure_holdoff_seconds: float - submit_tip_max_age_seconds: float - tip_refresh_max_workers: int - tip_refresh_epoch_fanout: bool - job_build_timeout_seconds: float - job_build_cancel_grace_seconds: float - job_build_executor_workers: int - routine_admission_deadline_seconds: float - worker_metrics_limit: int - reorg_reconciler_enabled: bool - job_bundle_cache_seconds: float - bundle_build_timeout_seconds: float - payout_artifact_rearm_min_seconds: float - payout_artifact_reuse_enabled: bool - payout_artifact_reanchor_seconds: float - payout_artifact_min_build_interval_seconds: float - payout_artifact_full_rescan_seconds: float - payout_artifact_max_anchor_age_seconds: float - template_cache_seconds: float - template_refresh_failure_exit_seconds: float - reorg_reconcile_cache_seconds: float - min_ready_miners: int - payout_environment: tuple[tuple[str, str], ...] - pool_fee_enabled_raw: str | None - pool_fee_bps_raw: str | None - pool_fee_address: str | None - pool_fee_program_hex: str | None - pool_fee_recipient_id: str | None - pool_fee_order_key: str | None - template_max_age_raw: str | None - # Appended with a default so existing positional constructions keep - # working. The loader passes the strict env read, so a non-boolean - # PRISM_WINDOW_PIPELINE_RUST refuses startup rather than starting on - # either side of the switch. - window_pipeline_rust_enabled: bool = DEFAULT_PRISM_WINDOW_PIPELINE_RUST - - -@dataclass(frozen=True) -class BlockConfig: - submit_rpc_timeout_seconds: float - submit_db_timeout_seconds: float - landing_db_timeout_seconds: float - landing_db_timeout_max_seconds: float - submit_lock_wait_log_seconds: float - submit_stuck_call_exit_seconds: float - observed_tip_accept_window_seconds: float - accepted_parent_unresolved_depth_max: int - # Defaulted so existing positional constructions keep working. - accepted_parent_redrive_defer_threshold: int = ( - DEFAULT_ACCEPTED_PARENT_REDRIVE_DEFER_THRESHOLD - ) - accepted_parent_redrive_attempt_max: int = ( - DEFAULT_ACCEPTED_PARENT_REDRIVE_ATTEMPT_MAX - ) - # Issue #198: retry records the collapse cleanup backlog may hold before - # bulk terminalization stops admitting new rows. - candidate_cleanup_retry_backlog_max: int = ( - DEFAULT_BLOCK_CANDIDATE_CLEANUP_RETRY_BACKLOG_MAX - ) - replay_page_size: int = MAX_BLOCK_REPLAY_PAGE_SIZE - - -@dataclass(frozen=True) -class LedgerConfig: - psql_command: str - database_url: str | None - allow_memory_ledger: bool - native_client_mode: str - writer_id: str - writer_epoch: int - writer_session_token: str | None - initialize_schema: bool - lease_ttl_seconds: float - lease_acquire_lock_timeout_seconds: float - lease_acquire_attempts: int - postgres_idle_in_transaction_timeout_seconds: float - postgres_tcp_keepalives_idle_seconds: int - postgres_tcp_keepalives_interval_seconds: int - postgres_tcp_keepalives_count: int - read_concurrency: int - accepted_stats_cache_seconds: float - reward_window_cache_seconds: float - signing_seed_hex: str - attestation_signing_seed_hex: str - writer_public_key_hex: str | None - share_commit_batch_size: int - share_commit_linger_seconds: float - share_commit_timeout_seconds: float - share_recovery_path: Path - # Issue #255: chunked candidate bodies. Version 2 writes bounded chunk - # bodies; version 1 keeps the legacy whole-jsonb write for the - # compatibility/rollback release. The reader handles both. - candidate_storage_version: int = 2 - candidate_spool_dir: str | None = None - candidate_spool_reservation_bytes: int = 4 * 1024 * 1024 * 1024 - - -@dataclass(frozen=True) -class AuditConfig: - evidence_path: Path - directory: Path - share_segment_size: int - live_bundle_retention: int - candidate_retention_seconds: int - bind: str | None - port: int - - -@dataclass(frozen=True) -class CtvConfig: - settlement_enabled_raw: str | None - broadcaster_enabled: bool - broadcaster_wallet: str | None - broadcaster_fee_sats: int - broadcaster_limit: int - broadcaster_chunk_size: int - broadcaster_interval_seconds: float - broadcast_attempt_detail_limit: int - broadcast_retry_backoff_seconds: int - settlement_environment: tuple[tuple[str, str], ...] - - -@dataclass(frozen=True) -class LifecycleConfig: - health_refresh_seconds: float - metrics_refresh_seconds: float - pending_refresh_health_deadline_seconds: float - coherent_tip_poll_health_deadline_seconds: float - mining_health_startup_grace_seconds: float - writer_quiescence_timeout_seconds: float - coordination_blocked_exit_seconds: float - watchdog_enabled: bool - watchdog_timeout_seconds: float - watchdog_interval_seconds: float - watchdog_lease_release_timeout_seconds: float - ledger_lease_heartbeat_seconds: float - ledger_lease_heartbeat_failure_seconds: float - ledger_lease_heartbeat_monitor_seconds: float - ledger_lease_heartbeat_exit_timeout_seconds: float - ledger_lease_external_fence_timeout_seconds: float - # Appended with a default so existing positional constructions keep - # working. The loader passes the strict env read, so a non-boolean - # PRISM_MALLOC_TELEMETRY refuses startup. The metrics renderer resolves - # the same switch from the environment itself; this field records the - # validated value in the config snapshot. - malloc_telemetry_enabled: bool = DEFAULT_PRISM_MALLOC_TELEMETRY - # Issue #226 part 2. Validated by the loader so a bad census bound - # refuses startup. prism_coordinator.main re-reads the same strict - # switches from the environment when it arms the census (the - # malloc-telemetry precedent above), so this field records the validated - # value in the config snapshot and can only ever agree with what main saw. - heap_census: HeapCensusConfig = DEFAULT_PRISM_HEAP_CENSUS_CONFIG - - -@dataclass(frozen=True) -class CoordinatorConfig: - rpc: RpcConfig - stratum: StratumConfig - jobs: JobPipelineConfig - block: BlockConfig - ledger: LedgerConfig - audit: AuditConfig - ctv: CtvConfig - lifecycle: LifecycleConfig - production: bool - hot_path_log_enabled: bool - coinbase_tag_hex: str - stop_after_block: bool - max_blocks: int - - -def _selected_environment(source: Env, names: tuple[str, ...]) -> tuple[tuple[str, str], ...]: - return tuple((name, source[name]) for name in names if name in source) - - -def load_coordinator_config(environ: Env | None = None) -> CoordinatorConfig: - """Load and validate one immutable configuration snapshot. - - Passing a mapping gives tests and embedding callers a no-global-environment - construction path. The zero-argument production path snapshots ``os.environ``. - """ - - source: Env = dict(_PROCESS_ENVIRON) if environ is None else dict(environ) - validate_prism_production_gate(environ=source) - production = production_mode(environ=source) - - rpc = RpcConfig( - host=env("QBIT_RPC_HOST", environ=source), - port=env_int("QBIT_RPC_PORT", 18452, environ=source), - user=env("QBIT_RPC_USER", environ=source), - password=env("QBIT_RPC_PASSWORD", environ=source), - chain=env("QBIT_CHAIN", "regtest", environ=source), - expected_genesis_hash=env_optional("QBIT_EXPECTED_GENESIS_HASH", environ=source), - minimum_peers_raw=source.get("PRISM_MIN_PEERS"), - ) - - blockpoll_seconds = env_positive_float( - "PRISM_BLOCKPOLL_SECONDS", DEFAULT_PRISM_BLOCKPOLL_SECONDS, environ=source - ) - same_tip_seconds = env_nonnegative_float( - "PRISM_STRATUM_SAME_TIP_JOB_RETENTION_SECONDS", - DEFAULT_PRISM_SAME_TIP_JOB_RETENTION_SECONDS, - environ=source, - ) - same_tip_per_connection = env_nonnegative_int( - "PRISM_STRATUM_SAME_TIP_JOB_RETENTION_PER_CONNECTION", - DEFAULT_PRISM_SAME_TIP_JOB_RETENTION_PER_CONNECTION, - environ=source, - ) - max_connections = env_nonnegative_int( - "PRISM_STRATUM_MAX_CONNECTIONS", DEFAULT_PRISM_STRATUM_MAX_CONNECTIONS, environ=source - ) - max_pending_initial_jobs = env_positive_int( - "PRISM_STRATUM_MAX_PENDING_INITIAL_JOBS", - DEFAULT_PRISM_STRATUM_MAX_PENDING_INITIAL_JOBS, - environ=source, - ) - if max_connections > 0 and max_pending_initial_jobs > max_connections: - raise SystemExit( - "PRISM_STRATUM_MAX_PENDING_INITIAL_JOBS cannot exceed " - "PRISM_STRATUM_MAX_CONNECTIONS" - ) - validate_same_tip_job_retention_limits( - retention_seconds=same_tip_seconds, - per_connection=same_tip_per_connection, - max_connections=max_connections, - production=production, - ) - initial_job_max_workers = resolve_initial_job_max_workers( - env_optional_positive_int("PRISM_INITIAL_JOB_MAX_WORKERS", environ=source), - max_pending_initial_jobs, - ) - job_build_executor_workers = validate_job_build_executor_workers( - env_positive_int( - "PRISM_JOB_BUILD_EXECUTOR_WORKERS", - DEFAULT_PRISM_JOB_BUILD_EXECUTOR_WORKERS, - environ=source, - ) - ) - tip_refresh_max_workers = env_positive_int( - "PRISM_TIP_REFRESH_MAX_WORKERS", DEFAULT_PRISM_TIP_REFRESH_MAX_WORKERS, environ=source - ) - if tip_refresh_max_workers > DEFAULT_PRISM_TIP_REFRESH_MAX_WORKERS: - raise SystemExit( - "PRISM_TIP_REFRESH_MAX_WORKERS cannot exceed " - f"{DEFAULT_PRISM_TIP_REFRESH_MAX_WORKERS}" - ) - template_refresh_failure_exit_seconds = env_nonnegative_float( - "PRISM_TEMPLATE_REFRESH_FAILURE_EXIT_SECONDS", - DEFAULT_PRISM_TEMPLATE_MAX_AGE_SECONDS, - environ=source, - ) - if production and template_refresh_failure_exit_seconds <= 0: - raise SystemExit( - "production mode requires a positive PRISM_TEMPLATE_REFRESH_FAILURE_EXIT_SECONDS" - ) - payout_artifact_rearm_min_seconds = env_positive_float( - "PRISM_PAYOUT_ARTIFACT_REARM_MIN_SECONDS", - DEFAULT_PRISM_PAYOUT_ARTIFACT_REARM_MIN_SECONDS, - environ=source, - ) - payout_artifact_reanchor_seconds = env_positive_float( - "PRISM_PAYOUT_ARTIFACT_REANCHOR_SECONDS", - DEFAULT_PRISM_PAYOUT_ARTIFACT_REANCHOR_SECONDS, - environ=source, - ) - payout_artifact_min_build_interval_seconds = env_positive_float( - "PRISM_PAYOUT_ARTIFACT_MIN_BUILD_INTERVAL_SECONDS", - payout_artifact_reanchor_seconds, - environ=source, - ) - payout_artifact_max_anchor_age_seconds = env_positive_float( - "PRISM_PAYOUT_ARTIFACT_MAX_ANCHOR_AGE_SECONDS", - DEFAULT_PRISM_PAYOUT_ARTIFACT_MAX_ANCHOR_AGE_SECONDS, - environ=source, - ) - validate_payout_artifact_age_bounds( - payout_artifact_max_anchor_age_seconds, - max( - payout_artifact_reanchor_seconds, - payout_artifact_min_build_interval_seconds, - ), - ) - - bind = env("PRISM_STRATUM_BIND", "127.0.0.1", environ=source) - port = env_int("PRISM_STRATUM_PORT", 3340, environ=source) - share_difficulty = env_decimal("PRISM_STRATUM_SHARE_DIFF", "0.000000001", environ=source) - vardiff_config = load_prism_vardiff_config(share_difficulty, environ=source) - listener_profiles = [ - StratumListenerProfile( - name="default", - bind=bind, - port=port, - share_difficulty=share_difficulty, - vardiff_config=vardiff_config, - heartbeat_name="stratum_accept", - ) - ] - highdiff_profile = load_prism_highdiff_listener(bind, vardiff_config, environ=source) - if highdiff_profile is not None: - if highdiff_profile.port == port and highdiff_profile.bind == bind: - raise SystemExit("PRISM_STRATUM_HIGHDIFF_PORT must differ from PRISM_STRATUM_PORT") - listener_profiles.append(highdiff_profile) - default_share_weight = env_int("PRISM_STRATUM_SHARE_WEIGHT", 1, environ=source) - if default_share_weight <= 0: - raise SystemExit("PRISM_STRATUM_SHARE_WEIGHT must be positive") - try: - fallback_version_mask = direct_stratum.normalize_version_rolling_mask( - env( - "PRISM_VERSION_ROLLING_MASK", - direct_stratum.QBIT_VERSION_ROLLING_MASK_HEX, - environ=source, - ), - field_name="PRISM_VERSION_ROLLING_MASK", - ) - except ValueError as exc: - raise SystemExit(str(exc)) from exc - - vardiff_resume_max_start_factor = env_decimal( - "PRISM_STRATUM_VARDIFF_RESUME_MAX_START_FACTOR", - str(DEFAULT_PRISM_VARDIFF_RESUME_MAX_START_FACTOR), - environ=source, - ) - if vardiff_resume_max_start_factor < 1: - raise SystemExit( - "PRISM_STRATUM_VARDIFF_RESUME_MAX_START_FACTOR must be at least 1" - ) - - stratum = StratumConfig( - bind=bind, - port=port, - extranonce2_size=env_int("PRISM_STRATUM_EXTRANONCE2_SIZE", 8, environ=source), - stale_grace_seconds=env_nonnegative_float( - "PRISM_STRATUM_STALE_GRACE_SECONDS", DEFAULT_PRISM_STALE_GRACE_SECONDS, environ=source - ), - send_timeout_seconds=env_nonnegative_float( - "PRISM_STRATUM_SEND_TIMEOUT_SECONDS", - DEFAULT_PRISM_STRATUM_SEND_TIMEOUT_SECONDS, - environ=source, - ), - max_connections=max_connections, - max_connections_per_username=env_nonnegative_int( - "PRISM_STRATUM_MAX_CONNECTIONS_PER_USERNAME", - DEFAULT_PRISM_STRATUM_MAX_CONNECTIONS_PER_USERNAME, - environ=source, - ), - max_pending_initial_jobs=max_pending_initial_jobs, - initial_job_timeout_seconds=env_nonnegative_float( - "PRISM_STRATUM_INITIAL_JOB_TIMEOUT_SECONDS", - DEFAULT_PRISM_STRATUM_INITIAL_JOB_TIMEOUT_SECONDS, - environ=source, - ), - initial_job_max_workers=initial_job_max_workers, - accept_resource_exhaustion_backoff_seconds=env_positive_float( - "PRISM_STRATUM_ACCEPT_RESOURCE_EXHAUSTION_BACKOFF_SECONDS", - DEFAULT_PRISM_STRATUM_ACCEPT_RESOURCE_EXHAUSTION_BACKOFF_SECONDS, - environ=source, - ), - listen_backlog=env_positive_int( - "PRISM_STRATUM_LISTEN_BACKLOG", DEFAULT_PRISM_STRATUM_LISTEN_BACKLOG, environ=source - ), - bind_retry_seconds=env_nonnegative_float( - "PRISM_STRATUM_BIND_RETRY_SECONDS", - DEFAULT_PRISM_STRATUM_BIND_RETRY_SECONDS, - environ=source, - ), - share_difficulty=share_difficulty, - vardiff_config=vardiff_config, - vardiff_idle_sweep_seconds=env_nonnegative_float( - "PRISM_STRATUM_VARDIFF_IDLE_SWEEP_SECONDS", - DEFAULT_PRISM_VARDIFF_IDLE_SWEEP_SECONDS, - environ=source, - ), - listener_profiles=tuple(listener_profiles), - default_share_weight=default_share_weight, - share_weights_by_username=_parse_share_weights( - source.get("PRISM_STRATUM_SHARE_WEIGHTS_JSON", "") - ), - username_fallback_address=default_prism_username_fallback_address(environ=source), - fallback_version_mask=fallback_version_mask, - same_tip_job_retention_seconds=same_tip_seconds, - same_tip_job_retention_per_connection=same_tip_per_connection, - disconnected_job_retention=env_nonnegative_int( - "PRISM_STRATUM_DISCONNECTED_JOB_RETENTION", - DEFAULT_PRISM_DISCONNECTED_JOB_RETENTION, - environ=source, - ), - payout_address_cache_max_entries=env_nonnegative_int( - "PRISM_PAYOUT_ADDRESS_CACHE_MAX_ENTRIES", - DEFAULT_PRISM_PAYOUT_ADDRESS_CACHE_MAX_ENTRIES, - environ=source, - ), - payout_address_cache_ttl_seconds=env_nonnegative_float( - "PRISM_PAYOUT_ADDRESS_CACHE_TTL_SECONDS", - DEFAULT_PRISM_PAYOUT_ADDRESS_CACHE_TTL_SECONDS, - environ=source, - ), - vardiff_resume_enabled=env_bool( - "PRISM_STRATUM_VARDIFF_RESUME", "1", environ=source - ), - vardiff_resume_ttl_seconds=env_nonnegative_float( - "PRISM_STRATUM_VARDIFF_RESUME_TTL_SECONDS", - DEFAULT_PRISM_VARDIFF_RESUME_TTL_SECONDS, - environ=source, - ), - vardiff_resume_max_entries=env_nonnegative_int( - "PRISM_STRATUM_VARDIFF_RESUME_MAX_ENTRIES", - DEFAULT_PRISM_VARDIFF_RESUME_MAX_ENTRIES, - environ=source, - ), - vardiff_resume_max_start_factor=vardiff_resume_max_start_factor, - ) - - jobs = JobPipelineConfig( - blockpoll_seconds=blockpoll_seconds, - blockwait_enabled=env_bool("PRISM_BLOCKWAIT_ENABLED", "1", environ=source), - blockwait_timeout_seconds=env_positive_float( - "PRISM_BLOCKWAIT_TIMEOUT_SECONDS", - DEFAULT_PRISM_BLOCKWAIT_TIMEOUT_SECONDS, - environ=source, - ), - tip_refresh_failure_holdoff_seconds=env_nonnegative_float( - "PRISM_TIP_REFRESH_FAILURE_HOLDOFF_SECONDS", - DEFAULT_PRISM_TIP_REFRESH_FAILURE_HOLDOFF_SECONDS, - environ=source, - ), - submit_tip_max_age_seconds=env_nonnegative_float( - "PRISM_SUBMIT_TIP_MAX_AGE_SECONDS", - DEFAULT_PRISM_SUBMIT_TIP_MAX_AGE_SECONDS, - environ=source, - ), - tip_refresh_max_workers=tip_refresh_max_workers, - tip_refresh_epoch_fanout=env_bool( - "PRISM_TIP_REFRESH_EPOCH_FANOUT", "0", environ=source - ), - job_build_timeout_seconds=env_positive_float( - "PRISM_JOB_BUILD_TIMEOUT_SECONDS", DEFAULT_PRISM_JOB_BUILD_TIMEOUT_SECONDS, environ=source - ), - job_build_cancel_grace_seconds=env_nonnegative_float( - "PRISM_JOB_BUILD_CANCEL_GRACE_SECONDS", - DEFAULT_PRISM_JOB_BUILD_CANCEL_GRACE_SECONDS, - environ=source, - ), - job_build_executor_workers=job_build_executor_workers, - routine_admission_deadline_seconds=env_positive_float( - "PRISM_ROUTINE_ADMISSION_DEADLINE_SECONDS", - DEFAULT_PRISM_ROUTINE_ADMISSION_DEADLINE_SECONDS, - environ=source, - ), - worker_metrics_limit=env_nonnegative_int( - "PRISM_WORKER_METRICS_LIMIT", DEFAULT_PRISM_WORKER_METRICS_LIMIT, environ=source - ), - reorg_reconciler_enabled=env_bool("PRISM_REORG_RECONCILER_ENABLED", "1", environ=source), - job_bundle_cache_seconds=env_nonnegative_float( - "PRISM_JOB_BUNDLE_CACHE_SECONDS", - DEFAULT_PRISM_JOB_BUNDLE_CACHE_SECONDS, - environ=source, - ), - bundle_build_timeout_seconds=env_positive_float( - "PRISM_BUNDLE_BUILD_TIMEOUT_SECONDS", - DEFAULT_PRISM_BUNDLE_BUILD_TIMEOUT_SECONDS, - environ=source, - ), - payout_artifact_rearm_min_seconds=payout_artifact_rearm_min_seconds, - payout_artifact_reuse_enabled=env_bool( - "PRISM_PAYOUT_ARTIFACT_REUSE", "1", environ=source - ), - window_pipeline_rust_enabled=env_window_pipeline_rust(environ=source), - payout_artifact_reanchor_seconds=payout_artifact_reanchor_seconds, - payout_artifact_min_build_interval_seconds=( - payout_artifact_min_build_interval_seconds - ), - payout_artifact_full_rescan_seconds=env_positive_float( - "PRISM_PAYOUT_ARTIFACT_FULL_RESCAN_SECONDS", - DEFAULT_PRISM_PAYOUT_ARTIFACT_FULL_RESCAN_SECONDS, - environ=source, - ), - payout_artifact_max_anchor_age_seconds=payout_artifact_max_anchor_age_seconds, - template_cache_seconds=env_nonnegative_float( - "PRISM_TEMPLATE_CACHE_SECONDS", blockpoll_seconds, environ=source - ), - template_refresh_failure_exit_seconds=template_refresh_failure_exit_seconds, - reorg_reconcile_cache_seconds=env_nonnegative_float( - "PRISM_REORG_RECONCILE_CACHE_SECONDS", - DEFAULT_PRISM_REORG_RECONCILE_CACHE_SECONDS, - environ=source, - ), - min_ready_miners=env_int("PRISM_MIN_READY_MINERS", 3, environ=source), - payout_environment=_selected_environment( - source, - ( - "PRISM_PAYOUT_P2MR_SPEND_INPUT_BYTES", - "PRISM_PAYOUT_TARGET_FEERATE_BITS_PER_BYTE", - "PRISM_PAYOUT_TARGET_FEERATE_SATS_PER_BYTE", - "PRISM_PAYOUT_SAFETY_MULTIPLIER", - "PRISM_PAYOUT_MIN_OUTPUT_BITS", - "PRISM_PAYOUT_MIN_OUTPUT_SATS", - ), - ), - pool_fee_enabled_raw=source.get("PRISM_POOL_FEE_ENABLED"), - pool_fee_bps_raw=env_optional("PRISM_POOL_FEE_BPS", environ=source), - pool_fee_address=env_optional("PRISM_POOL_FEE_ADDRESS", environ=source), - pool_fee_program_hex=env_optional("PRISM_POOL_FEE_P2MR_PROGRAM_HEX", environ=source), - pool_fee_recipient_id=env_optional("PRISM_POOL_FEE_RECIPIENT_ID", environ=source), - pool_fee_order_key=env_optional("PRISM_POOL_FEE_ORDER_KEY", environ=source), - template_max_age_raw=source.get("PRISM_TEMPLATE_MAX_AGE_SECONDS"), - ) - - block = BlockConfig( - replay_page_size=env_positive_int( - "PRISM_BLOCK_REPLAY_PAGE_SIZE", MAX_BLOCK_REPLAY_PAGE_SIZE, environ=source - ), - submit_rpc_timeout_seconds=env_positive_float( - "PRISM_BLOCK_SUBMIT_RPC_TIMEOUT_SECONDS", - DEFAULT_BLOCK_SUBMIT_RPC_TIMEOUT_SECONDS, - environ=source, - ), - submit_db_timeout_seconds=env_positive_float( - "PRISM_BLOCK_SUBMIT_DB_TIMEOUT_SECONDS", - DEFAULT_BLOCK_SUBMIT_DB_TIMEOUT_SECONDS, - environ=source, - ), - landing_db_timeout_seconds=env_positive_float( - "PRISM_BLOCK_LANDING_DB_TIMEOUT_SECONDS", - DEFAULT_BLOCK_LANDING_DB_TIMEOUT_SECONDS, - environ=source, - ), - landing_db_timeout_max_seconds=env_positive_float( - "PRISM_BLOCK_LANDING_DB_TIMEOUT_MAX_SECONDS", - DEFAULT_BLOCK_LANDING_DB_TIMEOUT_MAX_SECONDS, - environ=source, - ), - submit_lock_wait_log_seconds=env_positive_float( - "PRISM_BLOCK_SUBMIT_LOCK_WAIT_LOG_SECONDS", - DEFAULT_BLOCK_SUBMIT_LOCK_WAIT_LOG_SECONDS, - environ=source, - ), - submit_stuck_call_exit_seconds=env_positive_float( - "PRISM_BLOCK_SUBMIT_STUCK_CALL_EXIT_SECONDS", - DEFAULT_BLOCK_SUBMIT_STUCK_CALL_EXIT_SECONDS, - environ=source, - ), - observed_tip_accept_window_seconds=env_positive_float( - "PRISM_OBSERVED_TIP_ACCEPT_WINDOW_SECONDS", - DEFAULT_PRISM_OBSERVED_TIP_ACCEPT_WINDOW_SECONDS, - environ=source, - ), - accepted_parent_unresolved_depth_max=env_positive_int( - "PRISM_ACCEPTED_PARENT_UNRESOLVED_DEPTH_MAX", - DEFAULT_ACCEPTED_PARENT_UNRESOLVED_DEPTH_MAX, - environ=source, - ), - accepted_parent_redrive_defer_threshold=env_positive_int( - "PRISM_ACCEPTED_PARENT_REDRIVE_DEFER_THRESHOLD", - DEFAULT_ACCEPTED_PARENT_REDRIVE_DEFER_THRESHOLD, - environ=source, - ), - accepted_parent_redrive_attempt_max=env_nonnegative_int( - "PRISM_ACCEPTED_PARENT_REDRIVE_ATTEMPT_MAX", - DEFAULT_ACCEPTED_PARENT_REDRIVE_ATTEMPT_MAX, - environ=source, - ), - candidate_cleanup_retry_backlog_max=( - validate_block_candidate_cleanup_retry_backlog_max( - env_positive_int( - "PRISM_BLOCK_CANDIDATE_CLEANUP_RETRY_BACKLOG_MAX", - DEFAULT_BLOCK_CANDIDATE_CLEANUP_RETRY_BACKLOG_MAX, - environ=source, - ) - ) - ), - ) - - if block.replay_page_size > MAX_BLOCK_REPLAY_PAGE_SIZE: - raise SystemExit( - "PRISM_BLOCK_REPLAY_PAGE_SIZE cannot exceed " - f"{MAX_BLOCK_REPLAY_PAGE_SIZE}" - ) - - evidence_path = Path(env("PRISM_EVIDENCE_PATH", "prism-live-evidence.json", environ=source)) - audit_dir = Path(env("PRISM_AUDIT_DIR", str(evidence_path.parent), environ=source)) - audit = AuditConfig( - evidence_path=evidence_path, - directory=audit_dir, - share_segment_size=env_nonnegative_int( - "PRISM_AUDIT_SHARE_SEGMENT_SIZE", DEFAULT_AUDIT_SHARE_SEGMENT_SIZE, environ=source - ), - live_bundle_retention=env_nonnegative_int( - "PRISM_AUDIT_LIVE_BUNDLE_RETENTION", 5, environ=source - ), - candidate_retention_seconds=env_nonnegative_int( - "PRISM_AUDIT_CANDIDATE_RETENTION_SECONDS", 24 * 60 * 60, environ=source - ), - bind=source.get("PRISM_AUDIT_BIND"), - port=int(source.get("PRISM_AUDIT_PORT", "0") or "0"), - ) - - configured_writer_public_key = env_optional( - "PRISM_LEDGER_WRITER_PUBLIC_KEY_HEX", environ=source - ) - if configured_writer_public_key is not None: - writer_public_key = validate_hex( - configured_writer_public_key, - name="PRISM_LEDGER_WRITER_PUBLIC_KEY_HEX", - expected_bytes=32, - ) - elif env_bool("PRISM_ALLOW_BUNDLE_EMBEDDED_LEDGER_KEY", "0", environ=source): - writer_public_key = None - else: - raise SystemExit( - "PRISM_LEDGER_WRITER_PUBLIC_KEY_HEX is required; " - "set PRISM_ALLOW_BUNDLE_EMBEDDED_LEDGER_KEY=1 only for local tests" - ) - psql_command = source.get("PRISM_POSTGRES_PSQL_COMMAND", "") - database_url = source.get("PRISM_DATABASE_URL", "") - if not psql_command and database_url: - psql_command = f"psql {shlex.quote(database_url)}" - writer_session_token = env_optional("PRISM_LEDGER_WRITER_SESSION_TOKEN", environ=source) - if writer_session_token is not None and not env_bool( - "PRISM_ALLOW_FIXED_LEDGER_SESSION_TOKEN", "0", environ=source - ): - raise SystemExit( - "PRISM_LEDGER_WRITER_SESSION_TOKEN requires " - "PRISM_ALLOW_FIXED_LEDGER_SESSION_TOKEN=1 for local tests" - ) - lease_acquire_lock_timeout_seconds = env_positive_float( - "PRISM_LEDGER_LEASE_ACQUIRE_LOCK_TIMEOUT_SECONDS", - DEFAULT_LEASE_ACQUIRE_LOCK_TIMEOUT_SECONDS, - environ=source, - ) - lease_acquire_attempts = env_positive_int( - "PRISM_LEDGER_LEASE_ACQUIRE_ATTEMPTS", - DEFAULT_LEASE_ACQUIRE_ATTEMPTS, - environ=source, - ) - postgres_idle_in_transaction_timeout_seconds = env_positive_float( - "PRISM_POSTGRES_IDLE_IN_TRANSACTION_TIMEOUT_SECONDS", - DEFAULT_POSTGRES_IDLE_IN_TRANSACTION_TIMEOUT_SECONDS, - environ=source, - ) - lease_acquire_budget_seconds = ( - lease_acquire_attempts * lease_acquire_lock_timeout_seconds - ) - if postgres_idle_in_transaction_timeout_seconds > lease_acquire_budget_seconds: - print( - "prism coordinator: " - "PRISM_POSTGRES_IDLE_IN_TRANSACTION_TIMEOUT_SECONDS " - f"({postgres_idle_in_transaction_timeout_seconds:g}s) exceeds the " - "writer-lease acquire budget " - f"PRISM_LEDGER_LEASE_ACQUIRE_ATTEMPTS ({lease_acquire_attempts}) x " - "PRISM_LEDGER_LEASE_ACQUIRE_LOCK_TIMEOUT_SECONDS " - f"({lease_acquire_lock_timeout_seconds:g}s) = " - f"{lease_acquire_budget_seconds:g}s: an orphaned " - "idle-in-transaction backend holding the lease row lock is not " - "reaped mid-budget, so startup exhausts every acquire attempt at " - "the same locked row instead of self-healing; raise the acquire " - "lock deadline or attempts, or lower the idle-in-transaction " - "timeout", - file=sys.stderr, - flush=True, - ) - ledger = LedgerConfig( - psql_command=psql_command, - database_url=database_url or None, - allow_memory_ledger=env_bool("PRISM_ALLOW_MEMORY_LEDGER", "0", environ=source), - native_client_mode=env("PRISM_POSTGRES_NATIVE_CLIENT", "auto", environ=source), - writer_id=env("PRISM_LEDGER_WRITER_ID", "prism-coordinator", environ=source), - writer_epoch=env_int("PRISM_LEDGER_WRITER_EPOCH", 1, environ=source), - writer_session_token=writer_session_token, - initialize_schema=env("PRISM_POSTGRES_INIT_SCHEMA", "0", environ=source) - in {"1", "true", "yes"}, - lease_ttl_seconds=env_positive_float( - "PRISM_LEDGER_LEASE_TTL_SECONDS", 60.0, environ=source - ), - lease_acquire_lock_timeout_seconds=lease_acquire_lock_timeout_seconds, - lease_acquire_attempts=lease_acquire_attempts, - postgres_idle_in_transaction_timeout_seconds=( - postgres_idle_in_transaction_timeout_seconds - ), - postgres_tcp_keepalives_idle_seconds=env_positive_int( - "PRISM_POSTGRES_TCP_KEEPALIVES_IDLE_SECONDS", - DEFAULT_POSTGRES_TCP_KEEPALIVES_IDLE_SECONDS, - environ=source, - ), - postgres_tcp_keepalives_interval_seconds=env_positive_int( - "PRISM_POSTGRES_TCP_KEEPALIVES_INTERVAL_SECONDS", - DEFAULT_POSTGRES_TCP_KEEPALIVES_INTERVAL_SECONDS, - environ=source, - ), - postgres_tcp_keepalives_count=env_positive_int( - "PRISM_POSTGRES_TCP_KEEPALIVES_COUNT", - DEFAULT_POSTGRES_TCP_KEEPALIVES_COUNT, - environ=source, - ), - read_concurrency=env_positive_int("PRISM_POSTGRES_READ_CONCURRENCY", 4, environ=source), - accepted_stats_cache_seconds=env_nonnegative_float( - "PRISM_ACCEPTED_STATS_CACHE_SECONDS", 60.0, environ=source - ), - reward_window_cache_seconds=env_nonnegative_float( - "PRISM_PUBLIC_REWARD_WINDOW_CACHE_SECONDS", 30.0, environ=source - ), - signing_seed_hex=env_seed_hex( - "PRISM_MANIFEST_SIGNING_SEED_HEX", test_default="42" * 32, environ=source - ), - attestation_signing_seed_hex=env_seed_hex( - "PRISM_LEDGER_ATTESTATION_SIGNING_SEED_HEX", - test_default="43" * 32, - environ=source, - ), - writer_public_key_hex=writer_public_key, - share_commit_batch_size=env_positive_int( - "PRISM_SHARE_COMMIT_BATCH_SIZE", DEFAULT_SHARE_COMMIT_BATCH_SIZE, environ=source - ), - share_commit_linger_seconds=env_nonnegative_float( - "PRISM_SHARE_COMMIT_LINGER_MILLISECONDS", - DEFAULT_SHARE_COMMIT_LINGER_MILLISECONDS, - environ=source, - ) - / 1000.0, - share_commit_timeout_seconds=env_positive_float( - "PRISM_SHARE_COMMIT_TIMEOUT_SECONDS", DEFAULT_SHARE_COMMIT_TIMEOUT_SECONDS, environ=source - ), - candidate_storage_version=env_positive_int( - "PRISM_CANDIDATE_STORAGE_VERSION", 2, environ=source - ), - candidate_spool_dir=env_optional("PRISM_CANDIDATE_SPOOL_DIR", environ=source) or None, - candidate_spool_reservation_bytes=env_positive_int( - "PRISM_CANDIDATE_SPOOL_RESERVATION_BYTES", - 4 * 1024 * 1024 * 1024, - environ=source, - ), - share_recovery_path=Path( - env( - "PRISM_SHARE_RECOVERY_PATH", - str(audit_dir / "prism-unpersisted-shares.jsonl"), - environ=source, - ) - ), - ) - - configured_broadcaster_enabled = env_optional_bool( - "PRISM_CTV_BROADCASTER_ENABLED", environ=source - ) - broadcaster_enabled = ( - configured_broadcaster_enabled - if configured_broadcaster_enabled is not None - else env_bool("PRISM_CTV_SETTLEMENT_ENABLED", "0", environ=source) - ) - broadcaster_wallet = env_optional("PRISM_CTV_BROADCASTER_WALLET", environ=source) - broadcaster_fee_sats = env_nonnegative_int_with_legacy( - "PRISM_CTV_BROADCASTER_FEE_BITS", - "PRISM_CTV_BROADCASTER_FEE_SATS", - 0, - environ=source, - ) - if broadcaster_enabled and broadcaster_fee_sats > 0 and not broadcaster_wallet: - raise SystemExit( - "PRISM_CTV_BROADCASTER_WALLET is required when " - "PRISM_CTV_BROADCASTER_FEE_BITS is positive" - ) - broadcaster_chunk_size = env_positive_int( - "PRISM_CTV_BROADCASTER_CHUNK_SIZE", - DEFAULT_PRISM_CTV_BROADCASTER_CHUNK_SIZE, - environ=source, - ) - if broadcaster_chunk_size > MAX_CTV_FANOUT_BROADCASTER_CHUNK_SIZE: - raise SystemExit( - "PRISM_CTV_BROADCASTER_CHUNK_SIZE must be at most " - f"{MAX_CTV_FANOUT_BROADCASTER_CHUNK_SIZE}" - ) - ctv = CtvConfig( - settlement_enabled_raw=source.get("PRISM_CTV_SETTLEMENT_ENABLED"), - broadcaster_enabled=broadcaster_enabled, - broadcaster_wallet=broadcaster_wallet, - broadcaster_fee_sats=broadcaster_fee_sats, - broadcaster_limit=env_positive_int("PRISM_CTV_BROADCASTER_LIMIT", 100, environ=source), - broadcaster_chunk_size=broadcaster_chunk_size, - broadcaster_interval_seconds=env_positive_float( - "PRISM_CTV_BROADCASTER_INTERVAL_SECONDS", 30.0, environ=source - ), - broadcast_attempt_detail_limit=env_nonnegative_int( - "PRISM_CTV_BROADCAST_ATTEMPT_DETAIL_LIMIT", - DEFAULT_CTV_BROADCAST_ATTEMPT_DETAIL_LIMIT, - environ=source, - ), - broadcast_retry_backoff_seconds=env_nonnegative_int( - "PRISM_CTV_BROADCAST_RETRY_BACKOFF_SECONDS", - DEFAULT_CTV_BROADCAST_RETRY_BACKOFF_SECONDS, - environ=source, - ), - settlement_environment=_selected_environment( - source, - ( - "PRISM_DIRECT_COINBASE_PAYOUT_FLOOR_BITS", - "PRISM_DIRECT_COINBASE_PAYOUT_FLOOR_SATS", - "PRISM_RESERVED_COINBASE_OUTPUTS", - "PRISM_MAX_COINBASE_SETTLEMENT_OUTPUTS", - "PRISM_MAX_DIRECT_COINBASE_OUTPUTS", - "PRISM_MAX_CTV_FANOUT_RECIPIENTS_PER_TRANSACTION", - "PRISM_CTV_FANOUT_FEE_MARKET_RATE_BITS_PER_1000_WEIGHT", - "PRISM_CTV_FANOUT_FEE_MARKET_RATE_SATS_PER_1000_WEIGHT", - "PRISM_CTV_FANOUT_FEE_ESTIMATE_TARGET_BLOCKS", - "PRISM_CTV_FANOUT_FEE_PREMIUM_BPS", - ), - ), - ) - - watchdog_timeout_seconds = env_positive_float( - "PRISM_WATCHDOG_TIMEOUT_SECONDS", - DEFAULT_PRISM_WATCHDOG_TIMEOUT_SECONDS, - environ=source, - ) - if watchdog_timeout_seconds < 1.0: - raise SystemExit( - "PRISM_WATCHDOG_TIMEOUT_SECONDS must be at least 1 so the " - "escalated landing-budget ceiling derived from half of it stays " - "below the tolerance" - ) - - lifecycle = LifecycleConfig( - health_refresh_seconds=env_positive_float( - "PRISM_HEALTH_REFRESH_SECONDS", DEFAULT_PRISM_HEALTH_REFRESH_SECONDS, environ=source - ), - metrics_refresh_seconds=env_positive_float( - "PRISM_METRICS_REFRESH_SECONDS", - DEFAULT_PRISM_METRICS_REFRESH_SECONDS, - environ=source, - ), - pending_refresh_health_deadline_seconds=env_positive_float( - "PRISM_HEALTH_PENDING_REFRESH_MAX_AGE_SECONDS", - DEFAULT_PRISM_HEALTH_PENDING_REFRESH_MAX_AGE_SECONDS, - environ=source, - ), - coherent_tip_poll_health_deadline_seconds=env_positive_float( - "PRISM_HEALTH_TIP_POLL_MAX_AGE_SECONDS", - DEFAULT_PRISM_HEALTH_TIP_POLL_MAX_AGE_SECONDS, - environ=source, - ), - mining_health_startup_grace_seconds=env_nonnegative_float( - "PRISM_MINING_HEALTH_STARTUP_GRACE_SECONDS", - DEFAULT_PRISM_MINING_HEALTH_STARTUP_GRACE_SECONDS, - environ=source, - ), - writer_quiescence_timeout_seconds=env_positive_float( - "PRISM_WRITER_QUIESCENCE_TIMEOUT_SECONDS", - DEFAULT_PRISM_WRITER_QUIESCENCE_TIMEOUT_SECONDS, - environ=source, - ), - coordination_blocked_exit_seconds=env_positive_float( - "PRISM_COORDINATION_BLOCKED_EXIT_SECONDS", - DEFAULT_PRISM_COORDINATION_BLOCKED_EXIT_SECONDS, - environ=source, - ), - watchdog_enabled=env_bool("PRISM_WATCHDOG_ENABLED", "1", environ=source), - watchdog_timeout_seconds=watchdog_timeout_seconds, - watchdog_interval_seconds=env_positive_float( - "PRISM_WATCHDOG_INTERVAL_SECONDS", 15.0, environ=source - ), - # Lease/watchdog timing envelopes are derived, not environment-facing: - # the values enter the immutable snapshot so services consume typed - # configuration instead of rereading process state at run time. - watchdog_lease_release_timeout_seconds=( - DEFAULT_PRISM_WATCHDOG_LEASE_RELEASE_TIMEOUT_SECONDS - ), - ledger_lease_heartbeat_seconds=DEFAULT_PRISM_LEDGER_LEASE_HEARTBEAT_SECONDS, - ledger_lease_heartbeat_failure_seconds=( - DEFAULT_PRISM_LEDGER_LEASE_HEARTBEAT_FAILURE_SECONDS - ), - ledger_lease_heartbeat_monitor_seconds=( - DEFAULT_PRISM_LEDGER_LEASE_HEARTBEAT_MONITOR_SECONDS - ), - ledger_lease_heartbeat_exit_timeout_seconds=( - DEFAULT_PRISM_LEDGER_LEASE_HEARTBEAT_EXIT_TIMEOUT_SECONDS - ), - ledger_lease_external_fence_timeout_seconds=( - DEFAULT_PRISM_LEDGER_LEASE_EXTERNAL_FENCE_TIMEOUT_SECONDS - ), - malloc_telemetry_enabled=env_malloc_telemetry(environ=source), - heap_census=env_heap_census_config(environ=source, audit_dir=audit_dir), - ) - max_blocks = env_int("PRISM_MAX_BLOCKS", 1, environ=source) - if max_blocks <= 0: - raise SystemExit("PRISM_MAX_BLOCKS must be positive") - return CoordinatorConfig( - rpc=rpc, - stratum=stratum, - jobs=jobs, - block=block, - ledger=ledger, - audit=audit, - ctv=ctv, - lifecycle=lifecycle, - production=production, - hot_path_log_enabled=env_bool("PRISM_HOT_PATH_LOG", "0", environ=source), - coinbase_tag_hex=default_prism_coinbase_tag_hex(environ=source), - stop_after_block=env("PRISM_STOP_AFTER_BLOCK", "1", environ=source) - in {"1", "true", "yes"}, - max_blocks=max_blocks, - ) diff --git a/lab/prism/coordinator_shutdown.py b/lab/prism/coordinator_shutdown.py deleted file mode 100644 index 0d858e948..000000000 --- a/lab/prism/coordinator_shutdown.py +++ /dev/null @@ -1,300 +0,0 @@ -"""Writer admission and shutdown state for the PRISM coordinator. - -The controller is the sole owner of the writer barrier, the one-shot lease -release claim/outcome, and the drain metrics. It owns no heartbeat I/O, -thread, or process-exit state: lease heartbeat threads belong to -``lab.prism.background_services.LedgerLeaseHeartbeatService`` and the -coordinator keeps the orchestration adapters (``request_shutdown``, -``shutdown``, ``release_ledger_lease``, ``drain_non_writer_components``, -``shutdown_metrics_lines``) around controller decisions. -""" - -from __future__ import annotations - -from contextlib import AbstractContextManager -from functools import wraps -import signal -import threading -import time -from typing import Any, Callable, Protocol - -# Owner-local copy of the block submitter's admission-wait heartbeat slice. -# The submitter machinery itself remains coordinator-owned at this layer, so -# the coordinator keeps its own module constant; this leaf module must not -# import the coordinator to share one. -BLOCK_SUBMITTER_WAIT_HEARTBEAT_SLICE_SECONDS = 0.25 - - -class ShutdownInProgress(RuntimeError): - """Raised when work that could mutate the ledger arrives after shutdown.""" - - -class _WriterOperationToken: - """One transferable writer admission held until durable work completes.""" - - def __init__(self, controller: "CoordinatorShutdownController", component: str): - self.controller = controller - self.component = component - self.finished = False - - def finish(self) -> None: - self.controller.finish_token(self) - - -class CoordinatorShutdownController: - """Coordinates the writer barrier, one-shot lease release, and final drain. - - Writer operations enter through :meth:`enter_writer` before shutdown or - inherit an already-admitted operation on the same thread. Queue admissions - use transferable tokens so a share remains visible to the barrier while it - moves from a client thread to the group-commit writer. - """ - - def __init__(self, writer_quiescence_timeout_seconds: float): - self.writer_quiescence_timeout_seconds = writer_quiescence_timeout_seconds - self.condition = threading.Condition(threading.RLock()) - self.local = threading.local() - self.phase = "running" - self.reason: str | None = None - self.signal_number: int | None = None - self.sigterm_monotonic: float | None = None - self.shutdown_started_monotonic: float | None = None - self.active_writers: dict[str, int] = {} - self.shutdowns_total = 0 - self.writer_quiescence_outcomes = {"success": 0, "timeout": 0} - self.writer_quiescence_seconds = 0.0 - self.lease_release_attempts_total = 0 - self.lease_release_outcomes = { - "success": 0, - "not_held": 0, - "unsupported": 0, - "failure": 0, - } - self.lease_release_seconds = 0.0 - self.lease_release_attempted = False - self.lease_release_succeeded = False - self.lease_release_withheld = False - self.sigterm_to_lease_release_seconds = 0.0 - self.sigterm_release_observed = False - self.release_withheld_total = 0 - self.non_writer_drain_seconds = 0.0 - self.non_writer_drains_total = 0 - self._drain_claimed = False - - def request_shutdown(self, signum: int | None) -> None: - """Close admission atomically; the caller only needs to set its event.""" - now = time.monotonic() - with self.condition: - if signum == signal.SIGTERM and self.sigterm_monotonic is None: - self.sigterm_monotonic = now - if self.signal_number is None and signum is not None: - self.signal_number = signum - if self.phase == "running": - self.phase = "requested" - self.condition.notify_all() - - def begin_shutdown(self, reason: str) -> bool: - with self.condition: - if self.phase not in {"running", "requested"}: - return False - self.phase = "quiescing_writers" - self.reason = reason - self.shutdown_started_monotonic = time.monotonic() - self.shutdowns_total += 1 - self.condition.notify_all() - return True - - def wait_for_lease_handling(self) -> bool: - """Wait for the one shutdown owner to release or safely withhold.""" - in_progress = { - "requested", - "quiescing_writers", - "writers_quiesced", - "releasing_lease", - } - with self.condition: - while self.phase in in_progress: - self.condition.wait() - return self.lease_release_succeeded - - def _thread_writer_depth(self) -> int: - return int(getattr(self.local, "writer_depth", 0)) - - def _admit_writer_locked(self, component: str, *, inherited: bool) -> _WriterOperationToken: - if self.lease_release_attempted: - raise ShutdownInProgress("PRISM writer lease release has already started") - if self.phase != "running" and not inherited: - raise ShutdownInProgress("PRISM coordinator is shutting down") - self.active_writers[component] = self.active_writers.get(component, 0) + 1 - return _WriterOperationToken(self, component) - - def enter_writer( - self, - component: str, - *, - wait_callback: Callable[[], None] | None = None, - ) -> _WriterOperationToken: - depth = self._thread_writer_depth() - if wait_callback is None: - with self.condition: - token = self._admit_writer_locked(component, inherited=depth > 0) - else: - while not self.condition.acquire( - timeout=BLOCK_SUBMITTER_WAIT_HEARTBEAT_SLICE_SECONDS - ): - wait_callback() - try: - token = self._admit_writer_locked(component, inherited=depth > 0) - finally: - self.condition.release() - self.local.writer_depth = depth + 1 - return token - - def exit_writer(self, token: _WriterOperationToken) -> None: - depth = self._thread_writer_depth() - self.local.writer_depth = max(0, depth - 1) - token.finish() - - def reserve_writer(self, component: str) -> _WriterOperationToken: - """Reserve work that will finish on another thread.""" - with self.condition: - return self._admit_writer_locked( - component, - inherited=self._thread_writer_depth() > 0, - ) - - def finish_token(self, token: _WriterOperationToken) -> None: - with self.condition: - if token.finished: - return - token.finished = True - remaining = self.active_writers.get(token.component, 0) - 1 - if remaining > 0: - self.active_writers[token.component] = remaining - else: - self.active_writers.pop(token.component, None) - self.condition.notify_all() - - def has_active_writer(self, components: set[str]) -> bool: - with self.condition: - return any(self.active_writers.get(component, 0) for component in components) - - def writer_admission_closed(self) -> bool: - with self.condition: - return self.phase != "running" - - def wait_for_writer_quiescence( - self, - timeout_seconds: float | None = None, - ) -> tuple[bool, float, dict[str, int]]: - started = time.monotonic() - timeout = ( - self.writer_quiescence_timeout_seconds - if timeout_seconds is None - else max(0.0, float(timeout_seconds)) - ) - deadline = started + timeout - with self.condition: - while self.active_writers: - remaining = deadline - time.monotonic() - if remaining <= 0: - break - self.condition.wait(remaining) - elapsed = max(0.0, time.monotonic() - started) - quiesced = not self.active_writers - blockers = dict(sorted(self.active_writers.items())) - outcome = "success" if quiesced else "timeout" - self.writer_quiescence_outcomes[outcome] += 1 - self.writer_quiescence_seconds = elapsed - if quiesced: - self.phase = "writers_quiesced" - else: - self.phase = "release_withheld" - self.lease_release_withheld = True - self.release_withheld_total += 1 - self.condition.notify_all() - return quiesced, elapsed, blockers - - def claim_lease_release(self) -> tuple[bool, dict[str, int]]: - with self.condition: - if self.lease_release_attempted or self.lease_release_withheld: - return False, {} - if self.active_writers: - return False, dict(sorted(self.active_writers.items())) - self.lease_release_attempted = True - self.lease_release_attempts_total += 1 - self.phase = "releasing_lease" - self.condition.notify_all() - return True, {} - - def finish_lease_release(self, outcome: str, elapsed: float) -> None: - with self.condition: - self.lease_release_outcomes[outcome] += 1 - self.lease_release_seconds = elapsed - self.lease_release_succeeded = outcome != "failure" - self.phase = "lease_released" if outcome != "failure" else "lease_release_failed" - if outcome != "failure" and self.sigterm_monotonic is not None: - self.sigterm_to_lease_release_seconds = max( - 0.0, - time.monotonic() - self.sigterm_monotonic, - ) - self.sigterm_release_observed = True - self.condition.notify_all() - - def claim_non_writer_drain(self) -> bool: - with self.condition: - if self._drain_claimed: - return False - if self.phase not in { - "lease_released", - "lease_release_failed", - "release_withheld", - }: - return False - self._drain_claimed = True - self.phase = "draining_non_writers" - return True - - def finish_non_writer_drain(self, elapsed: float) -> None: - with self.condition: - self.non_writer_drain_seconds = elapsed - self.non_writer_drains_total += 1 - self.phase = "complete" - self.condition.notify_all() - - def snapshot(self) -> dict[str, Any]: - with self.condition: - return { - "phase": self.phase, - "active_writers": dict(self.active_writers), - "shutdowns_total": self.shutdowns_total, - "writer_quiescence_outcomes": dict(self.writer_quiescence_outcomes), - "writer_quiescence_seconds": self.writer_quiescence_seconds, - "lease_release_attempts_total": self.lease_release_attempts_total, - "lease_release_outcomes": dict(self.lease_release_outcomes), - "lease_release_seconds": self.lease_release_seconds, - "lease_release_withheld": self.lease_release_withheld, - "sigterm_to_lease_release_seconds": self.sigterm_to_lease_release_seconds, - "sigterm_release_observed": self.sigterm_release_observed, - "release_withheld_total": self.release_withheld_total, - "non_writer_drain_seconds": self.non_writer_drain_seconds, - "non_writer_drains_total": self.non_writer_drains_total, - } - - -class _WriterOperationOwner(Protocol): - def _writer_operation(self, component: str) -> AbstractContextManager[object]: ... - - -def ledger_writer_operation(component: str) -> Callable[[Callable[..., Any]], Callable[..., Any]]: - """Decorate an entry point that can mutate the PRISM ledger.""" - - def decorate(method: Callable[..., Any]) -> Callable[..., Any]: - @wraps(method) - def guarded(self: _WriterOperationOwner, *args: Any, **kwargs: Any) -> Any: - with self._writer_operation(component): - return method(self, *args, **kwargs) - - return guarded - - return decorate diff --git a/lab/prism/ctv_broadcaster.py b/lab/prism/ctv_broadcaster.py deleted file mode 100644 index 35408548a..000000000 --- a/lab/prism/ctv_broadcaster.py +++ /dev/null @@ -1,525 +0,0 @@ -#!/usr/bin/env python3 -"""Non-custodial CTV fanout broadcaster engine (live-node orchestration). - -This is the operational loop that gets a precommitted CTV fanout confirmed. It -is deliberately decoupled from persistence: given a fanout artifact, a node RPC -callable, and optionally a funding wallet, it - -- gates broadcasting on parent-coinbase maturity and active-chain state, -- submits a fee-bearing fanout parent directly when settlement reserved a - network fee in the fanout template, -- builds and signs a child-pays-for-parent (CPFP) child that spends the - fanout's keyless P2A anchor plus a funding input the wallet owns when extra - fee sponsorship is requested, -- submits the ``[parent]`` transaction or ``[parent, child]`` package, -- detects confirmation, and -- can fee-bump a stuck child by re-issuing it at a higher fee. - -It holds **no covenant key**: the fanout outputs are fixed by the covenant. When -the parent fee reserved at block-build time is no longer enough, the broadcaster -can only add fee from its own funding input, with change returning to it. -Because the anchor is keyless, any owed miner can run this same engine to rescue -their payout. - -``broadcast`` / ``bump`` return a :class:`BroadcastAttempt` for the caller to -journal (e.g. into ``qbit_ctv_fanout_broadcast_attempts``); this module does not -touch Postgres so the node-orchestration logic is testable in isolation and -against a regtest node without a database. -""" - -from __future__ import annotations - -from dataclasses import dataclass -from decimal import Decimal -from typing import Any, Callable, Optional - -from lab.prism.share_ledger import WriterLeaseRenewalDeferred - -# qbit coinbase maturity (consensus): a coinbase output is spendable once the -# active tip is at least this many blocks above the coinbase height. -COINBASE_MATURITY = 1000 -SATOSHIS_PER_QBIT = 100_000_000 -P2A_ANCHOR_SCRIPT_PUBKEY_HEX = "51024e73" -# RBF-signalling, non-final sequence so a stuck child can be re-issued higher. -CPFP_CHILD_SEQUENCE = 0xFFFF_FFFD -CPFP_CHILD_TX_VERSION = 3 # TRUC, so a 0-fee parent relays as a package -TEMPLATE_MISMATCH = "Transaction template hash does not match" - -# Settlement statuses, matching the Rust `FanoutSettlementStatus` plus a -# `broadcast` (in-mempool, unconfirmed) state the live loop needs. -AWAITING_MATURITY = "awaiting_maturity" -BROADCASTABLE = "broadcastable" -BROADCAST = "broadcast" -CONFIRMED = "confirmed" -REORGED = "reorged" - - -class BroadcasterError(RuntimeError): - pass - - -class RpcError(RuntimeError): - def __init__(self, method: str, error: Any): - self.method = method - self.error = error - super().__init__(f"qbit RPC {method} failed: {error}") - - -@dataclass(frozen=True) -class FanoutArtifact: - """The recoverable data the broadcaster needs, as persisted alongside a - fanout manifest set.""" - - fanout_txid: str - fanout_tx_hex: str - anchor_vout: int | None - coinbase_txid: str - coinbase_block_hash: str - coinbase_height: int - parent_coinbase_vout: int - - -@dataclass(frozen=True) -class BroadcastAttempt: - fanout_txid: str - status: str - submitted: bool - child_txid: Optional[str] = None - fee_sats: Optional[int] = None - package_msg: Optional[str] = None - detail: str = "" - - -def qbit_to_sats(amount: Any) -> int: - return int((Decimal(str(amount)) * SATOSHIS_PER_QBIT).to_integral_exact()) - - -def _compact_size(value: int) -> bytes: - if value < 0xFD: - return bytes([value]) - if value <= 0xFFFF: - return b"\xfd" + value.to_bytes(2, "little") - if value <= 0xFFFF_FFFF: - return b"\xfe" + value.to_bytes(4, "little") - return b"\xff" + value.to_bytes(8, "little") - - -def _serialize_input(display_txid_hex: str, vout: int, sequence: int) -> bytes: - txid = bytes.fromhex(display_txid_hex) - if len(txid) != 32: - raise BroadcasterError(f"txid must be 32 bytes: {display_txid_hex}") - # Transactions serialize the prevout txid in internal (reversed) byte order. - return txid[::-1] + vout.to_bytes(4, "little") + b"\x00" + sequence.to_bytes(4, "little") - - -def _serialize_output(value_sats: int, script_pubkey_hex: str) -> bytes: - spk = bytes.fromhex(script_pubkey_hex) - return value_sats.to_bytes(8, "little") + _compact_size(len(spk)) + spk - - -def build_unsigned_cpfp_child( - fanout_txid: str, - anchor_vout: int, - funding_txid: str, - funding_vout: int, - funding_value_sats: int, - fee_sats: int, - change_script_pubkey_hex: str, -) -> tuple[str, int]: - """Serialize the unsigned CPFP child (legacy form, ready for the wallet to - sign the funding input). Mirrors the Rust ``build_cpfp_child``: the anchor - input is keyless (empty scriptSig/witness) and the funding input is left - unsigned. Returns ``(unsigned_tx_hex, change_value_sats)``. - """ - if fee_sats <= 0: - raise BroadcasterError("CPFP fee must be positive") - if fee_sats >= funding_value_sats: - raise BroadcasterError( - "funding value must exceed the fee so the change output is positive" - ) - if fanout_txid == funding_txid and anchor_vout == funding_vout: - raise BroadcasterError("funding input must differ from the anchor outpoint") - change_value_sats = funding_value_sats - fee_sats # anchor contributes 0 - tx = CPFP_CHILD_TX_VERSION.to_bytes(4, "little") - tx += _compact_size(2) - tx += _serialize_input(fanout_txid, anchor_vout, CPFP_CHILD_SEQUENCE) - tx += _serialize_input(funding_txid, funding_vout, CPFP_CHILD_SEQUENCE) - tx += _compact_size(1) - tx += _serialize_output(change_value_sats, change_script_pubkey_hex) - tx += (0).to_bytes(4, "little") # lock_time - return tx.hex(), change_value_sats - - -class CtvFanoutBroadcaster: - def __init__( - self, - rpc_call: Callable[..., Any], - *, - funding_wallet: str | None = None, - maturity: int = COINBASE_MATURITY, - before_external_side_effect: Callable[[str], None] | None = None, - ) -> None: - self._rpc = rpc_call - self.funding_wallet = funding_wallet - self.maturity = maturity - self._before_external_side_effect = before_external_side_effect - self._parent_spender_by_outpoint: dict[tuple[str, int], str] = {} - self._parent_spender_scanned_min_height: int | None = None - self._parent_spender_scanned_max_height: int | None = None - - # -- chain queries ----------------------------------------------------- - - def _tx_info(self, txid: str) -> Optional[dict[str, Any]]: - try: - return self._rpc("getrawtransaction", [txid, True]) - except Exception: - return None - - def _block_info(self, block_hash: str) -> Optional[dict[str, Any]]: - try: - return self._rpc("getblock", [block_hash]) - except Exception: - return None - - def _tip_height(self) -> int: - return int(self._rpc("getblockchaininfo")["blocks"]) - - def tip_height(self) -> int: - """Active-chain tip height, for callers that gate work on coinbase - maturity without paying for a full per-artifact settlement probe.""" - return self._tip_height() - - def _in_mempool(self, txid: str) -> bool: - return txid in set(self._rpc("getrawmempool")) - - def _parent_coinbase_output_spent_on_chain(self, artifact: FanoutArtifact) -> bool: - try: - return ( - self._rpc( - "gettxout", - [artifact.coinbase_txid, artifact.parent_coinbase_vout, False], - ) - is None - ) - except Exception: - return False - - def _parent_coinbase_spender_txid_on_chain( - self, - artifact: FanoutArtifact, - *, - tip_height: int, - parent_spent_on_chain: bool | None = None, - ) -> str | None: - if parent_spent_on_chain is None: - parent_spent_on_chain = self._parent_coinbase_output_spent_on_chain(artifact) - if not parent_spent_on_chain: - return None - outpoint = (artifact.coinbase_txid.lower(), artifact.parent_coinbase_vout) - if outpoint in self._parent_spender_by_outpoint: - return self._parent_spender_by_outpoint[outpoint] - self._scan_parent_spenders_on_chain(artifact.coinbase_height + 1, tip_height) - return self._parent_spender_by_outpoint.get(outpoint) - - def _scan_parent_spenders_on_chain(self, start_height: int, tip_height: int) -> None: - """Index prevout spenders for ``[start_height, tip_height]``, reusing - any range already scanned. - - Only heights that were actually read extend the covered window, and the - window is kept contiguous. A transient RPC failure mid-scan therefore - leaves the unread tail *uncovered* so a later call retries it, instead - of permanently hiding a spender (which would misclassify a confirmed - fanout as reorged) until the process restarts. - """ - if start_height > tip_height: - return - min_height = self._parent_spender_scanned_min_height - max_height = self._parent_spender_scanned_max_height - - if min_height is None or max_height is None: - last_scanned = self._scan_parent_spenders_on_chain_range(start_height, tip_height) - if last_scanned is not None: - self._parent_spender_scanned_min_height = start_height - self._parent_spender_scanned_max_height = last_scanned - return - - # Extend upward. Starting at ``max_height + 1`` (rather than - # ``start_height``) keeps the covered window contiguous even when this - # artifact's coinbase sits above everything scanned so far. - if tip_height > max_height: - last_scanned = self._scan_parent_spenders_on_chain_range(max_height + 1, tip_height) - if last_scanned is not None: - self._parent_spender_scanned_max_height = last_scanned - - # Extend downward. Only lower the watermark if the whole gap below the - # window was scanned, so a partial scan never claims coverage over the - # blocks it skipped. - if start_height < min_height: - lower_end = min_height - 1 - last_scanned = self._scan_parent_spenders_on_chain_range(start_height, lower_end) - if last_scanned == lower_end: - self._parent_spender_scanned_min_height = start_height - - def _scan_parent_spenders_on_chain_range(self, start_height: int, end_height: int) -> int | None: - """Scan ``[start_height, end_height]`` ascending, recording every prevout - spender seen. - - Returns the highest height scanned contiguously from ``start_height``, - or ``None`` if not even the first block could be read, so the caller can - avoid extending its covered window past a failed scan. - """ - last_scanned: int | None = None - for height in range(start_height, end_height + 1): - try: - block_hash = str(self._rpc("getblockhash", [height])) - block = self._rpc("getblock", [block_hash, 2]) - except Exception: - return last_scanned - if not isinstance(block, dict): - return last_scanned - for tx in block.get("tx", []): - if not isinstance(tx, dict): - continue - txid = str(tx.get("txid", "")) - if not txid: - continue - for txin in tx.get("vin", []): - if not isinstance(txin, dict): - continue - parent_txid = str(txin.get("txid", "")).lower() - if not parent_txid: - continue - try: - vout = int(txin.get("vout", -1)) - except (TypeError, ValueError): - continue - self._parent_spender_by_outpoint[(parent_txid, vout)] = txid - last_scanned = height - return last_scanned - - def settlement_status(self, artifact: FanoutArtifact) -> str: - """Derive settlement status purely from chain facts. A confirmed fanout - is ``CONFIRMED`` regardless of the coinbase's reorg state.""" - info = self._tx_info(artifact.fanout_txid) - if info is not None and int(info.get("confirmations", 0)) >= 1: - return CONFIRMED - - tip_height = self._tip_height() - block = self._block_info(artifact.coinbase_block_hash) - if block is None or int(block.get("confirmations", -1)) < 0: - if tip_height < artifact.coinbase_height + self.maturity: - return AWAITING_MATURITY - # The funding coinbase is height-mature but no longer on the active - # chain and the fanout has not confirmed: the payout must be - # recomputed. - return REORGED - - if tip_height < artifact.coinbase_height + self.maturity: - return AWAITING_MATURITY - parent_spent_on_chain = self._parent_coinbase_output_spent_on_chain(artifact) - if parent_spent_on_chain: - spender_txid = self._parent_coinbase_spender_txid_on_chain( - artifact, - tip_height=tip_height, - parent_spent_on_chain=True, - ) - if spender_txid is not None and spender_txid.lower() == artifact.fanout_txid.lower(): - return CONFIRMED - return REORGED - if self._in_mempool(artifact.fanout_txid): - return BROADCAST - return BROADCASTABLE - - # -- broadcasting ------------------------------------------------------ - - def _fence_external_side_effect(self, operation: str) -> None: - if self._before_external_side_effect is not None: - self._before_external_side_effect(operation) - - def _select_funding_utxo(self, required_sats: int) -> dict[str, Any]: - if self.funding_wallet is None: - raise BroadcasterError("funding wallet is required for CPFP fee sponsorship") - utxos = self._rpc("listunspent", [1, 9999999, [], True], wallet=self.funding_wallet) - spendable = [ - utxo - for utxo in utxos - if bool(utxo.get("spendable", False)) and qbit_to_sats(utxo["amount"]) > required_sats - ] - if not spendable: - raise BroadcasterError( - f"{self.funding_wallet} has no spendable UTXO above {required_sats} sats" - ) - return max(spendable, key=lambda utxo: qbit_to_sats(utxo["amount"])) - - def _build_signed_child(self, artifact: FanoutArtifact, fee_sats: int) -> tuple[str, str]: - if self.funding_wallet is None: - raise BroadcasterError("funding wallet is required for CPFP fee sponsorship") - if artifact.anchor_vout is None: - raise BroadcasterError("fanout has no CPFP anchor") - funding = self._select_funding_utxo(fee_sats) - self._fence_external_side_effect("ctv_wallet_getnewaddress") - change_address = self._rpc("getnewaddress", ["", "p2mr"], wallet=self.funding_wallet) - change_spk = self._rpc("getaddressinfo", [change_address], wallet=self.funding_wallet)[ - "scriptPubKey" - ] - unsigned_hex, _change = build_unsigned_cpfp_child( - artifact.fanout_txid, - artifact.anchor_vout, - str(funding["txid"]), - int(funding["vout"]), - qbit_to_sats(funding["amount"]), - fee_sats, - change_spk, - ) - # The anchor input is keyless; declaring its prevout lets the wallet sign - # only the funding input and leave the anchor witness empty. - anchor_prevtx = [ - { - "txid": artifact.fanout_txid, - "vout": artifact.anchor_vout, - "scriptPubKey": P2A_ANCHOR_SCRIPT_PUBKEY_HEX, - "amount": 0, - } - ] - self._fence_external_side_effect("ctv_wallet_sign") - signed = self._rpc( - "signrawtransactionwithwallet", - [unsigned_hex, anchor_prevtx], - wallet=self.funding_wallet, - ) - if not bool(signed.get("complete", False)): - raise BroadcasterError(f"funding input did not sign cleanly: {signed.get('errors')}") - return str(signed["hex"]), unsigned_hex - - def _submit_package(self, artifact: FanoutArtifact, signed_child_hex: str) -> dict[str, Any]: - self._fence_external_side_effect("ctv_submitpackage") - return self._rpc( - "submitpackage", [[artifact.fanout_tx_hex, signed_child_hex], 0] - ) - - def _submit_parent(self, artifact: FanoutArtifact, *, detail: str) -> BroadcastAttempt: - self._fence_external_side_effect("ctv_sendrawtransaction") - txid = str(self._rpc("sendrawtransaction", [artifact.fanout_tx_hex])) - submitted = txid.lower() == artifact.fanout_txid.lower() - return BroadcastAttempt( - fanout_txid=artifact.fanout_txid, - status=BROADCAST if submitted else BROADCASTABLE, - submitted=submitted, - fee_sats=0, - package_msg="success" if submitted else "txid_mismatch", - detail=detail if submitted else f"{detail}: submitted txid {txid} did not match artifact", - ) - - def broadcast(self, artifact: FanoutArtifact, fee_sats: int) -> BroadcastAttempt: - """Idempotently get the fanout into the mempool. - - No-op (with the observed status) unless the fanout is ``BROADCASTABLE``; - already-broadcast or confirmed fanouts are not re-submitted. - """ - status = self.settlement_status(artifact) - if status in (CONFIRMED, BROADCAST): - return BroadcastAttempt( - fanout_txid=artifact.fanout_txid, - status=status, - submitted=False, - detail="already in mempool or confirmed; no re-broadcast", - ) - if status != BROADCASTABLE: - return BroadcastAttempt( - fanout_txid=artifact.fanout_txid, - status=status, - submitted=False, - detail=f"not broadcastable ({status})", - ) - if fee_sats < 0: - return BroadcastAttempt( - fanout_txid=artifact.fanout_txid, - status=BROADCASTABLE, - submitted=False, - fee_sats=fee_sats, - package_msg="invalid_fee", - detail="initial broadcast: fee_sats must be non-negative", - ) - return self._submit(artifact, fee_sats, detail="initial broadcast") - - def bump(self, artifact: FanoutArtifact, higher_fee_sats: int) -> BroadcastAttempt: - """Re-issue the CPFP child at a higher fee to chase the fee market. - - The TRUC child is replaceable, so a higher-fee child supersedes the - stuck one. Refuses to bump a fanout that has already confirmed.""" - status = self.settlement_status(artifact) - if status == CONFIRMED: - return BroadcastAttempt( - fanout_txid=artifact.fanout_txid, - status=CONFIRMED, - submitted=False, - detail="already confirmed; nothing to bump", - ) - if status == REORGED: - return BroadcastAttempt( - fanout_txid=artifact.fanout_txid, - status=REORGED, - submitted=False, - detail="coinbase reorged; recompute payout before bumping", - ) - if higher_fee_sats < 0: - return BroadcastAttempt( - fanout_txid=artifact.fanout_txid, - status=status, - submitted=False, - fee_sats=higher_fee_sats, - package_msg="invalid_fee", - detail="fee bump: higher_fee_sats must be non-negative", - ) - return self._submit(artifact, higher_fee_sats, detail="fee bump") - - def _submit(self, artifact: FanoutArtifact, fee_sats: int, *, detail: str) -> BroadcastAttempt: - try: - if artifact.anchor_vout is None: - return self._submit_parent( - artifact, - detail=f"{detail}: no CPFP anchor", - ) - if fee_sats == 0: - return BroadcastAttempt( - fanout_txid=artifact.fanout_txid, - status=BROADCASTABLE, - submitted=False, - fee_sats=fee_sats, - package_msg="invalid_fee", - detail=f"{detail}: CPFP fee must be positive for anchored fanout", - ) - signed_child_hex, _unsigned = self._build_signed_child(artifact, fee_sats) - result = self._submit_package(artifact, signed_child_hex) - except WriterLeaseRenewalDeferred: - # The lease fence withheld the RPC before it ran: not a - # broadcast outcome. Converting it into an error attempt would - # journal a failure and push the payout behind the persistent - # retry backoff; propagating lets the daemon's caller retry on - # its ordinary pass interval once the deferring write commits. - raise - except Exception as exc: - return BroadcastAttempt( - fanout_txid=artifact.fanout_txid, - status=BROADCASTABLE, - submitted=False, - fee_sats=fee_sats, - package_msg="error", - detail=f"{detail}: {exc}", - ) - package_msg = str(result.get("package_msg", "")) - submitted = package_msg.lower() == "success" - child_txid = None - for txid, tx_result in (result.get("tx-results", {}) or {}).items(): - wtxid = tx_result.get("txid") if isinstance(tx_result, dict) else None - if wtxid and wtxid != artifact.fanout_txid: - child_txid = wtxid - return BroadcastAttempt( - fanout_txid=artifact.fanout_txid, - status=BROADCAST if submitted else BROADCASTABLE, - submitted=submitted, - child_txid=child_txid, - fee_sats=fee_sats, - package_msg=package_msg, - detail=detail if submitted else f"{detail}: package rejected ({package_msg})", - ) diff --git a/lab/prism/ctv_broadcaster_daemon.py b/lab/prism/ctv_broadcaster_daemon.py deleted file mode 100644 index 42911ea27..000000000 --- a/lab/prism/ctv_broadcaster_daemon.py +++ /dev/null @@ -1,301 +0,0 @@ -#!/usr/bin/env python3 -"""Durable CTV fanout broadcaster loop glue. - -The lower-level :mod:`lab.prism.ctv_broadcaster` module knows how to talk to a -qbit node and funding wallet. This module connects that engine to the PRISM -ledger: read pending fanout artifacts, derive live status, submit broadcastable -packages, journal attempts, and update durable state. -""" - -from __future__ import annotations - -from dataclasses import dataclass -from datetime import datetime, timezone -from time import monotonic -from typing import Any, Callable, Protocol - -from lab.prism.ctv_broadcaster import ( - AWAITING_MATURITY, - BROADCAST, - BROADCASTABLE, - CONFIRMED, - REORGED, - BroadcastAttempt, - CtvFanoutBroadcaster, - FanoutArtifact, -) - - -LEDGER_STATUS_BY_BROADCASTER_STATUS = { - AWAITING_MATURITY: "awaiting_maturity", - BROADCASTABLE: "broadcastable", - BROADCAST: "broadcast_submitted", - CONFIRMED: "confirmed", - REORGED: "reorged", -} -MAX_CTV_FANOUT_BROADCASTER_CHUNK_SIZE = 10 - - -class CtvFanoutLedger(Protocol): - def pending_ctv_fanout_statuses(self, *, limit: int = 100) -> list[dict[str, object]]: ... - - def update_ctv_fanout_status(self, *, fanout_txid: str, settlement_status: str) -> dict[str, int | str]: ... - - def record_ctv_fanout_broadcast_attempt( - self, - *, - fanout_txid: str, - attempt_status: str, - package_tx_hexes: list[str] | None = None, - package_txids: list[str] | None = None, - submit_result: dict[str, Any] | None = None, - error: str | None = None, - ) -> dict[str, int | str]: ... - - -@dataclass(frozen=True) -class CtvFanoutDaemonResult: - scanned_count: int - submitted_count: int - updated_count: int - failed_count: int - yielded_to_tip_refresh: bool = False - - -@dataclass(frozen=True) -class CtvFanoutChunkResult: - processed_count: int - elapsed_seconds: float - - -def artifact_from_status_row(row: dict[str, object]) -> FanoutArtifact: - anchor_vout = row.get("anchor_vout") - return FanoutArtifact( - fanout_txid=str(row["fanout_txid"]), - fanout_tx_hex=str(row["fanout_tx_hex"]), - anchor_vout=None if anchor_vout is None else int(anchor_vout), - coinbase_txid=str(row["parent_coinbase_txid"]), - coinbase_block_hash=str(row["block_hash"]), - coinbase_height=int(row["block_height"]), - parent_coinbase_vout=int(row["parent_coinbase_vout"]), - ) - - -def _coinbase_height_from_row(row: dict[str, object]) -> int | None: - try: - return int(row["block_height"]) - except (KeyError, TypeError, ValueError): - return None - - -class CtvFanoutBroadcastDaemon: - def __init__(self, ledger: CtvFanoutLedger, broadcaster: CtvFanoutBroadcaster, *, fee_sats: int) -> None: - if fee_sats < 0: - raise ValueError("fee_sats must be non-negative") - self.ledger = ledger - self.broadcaster = broadcaster - self.fee_sats = fee_sats - - def run_once( - self, - *, - limit: int = 100, - chunk_size: int = 1, - progress_callback: Callable[[], None] | None = None, - tip_refresh_pending: Callable[[], bool] | None = None, - chunk_callback: Callable[[CtvFanoutChunkResult], None] | None = None, - ) -> CtvFanoutDaemonResult: - """Process one batch in committed chunks that can yield to a tip refresh. - - The ledger query returns materialized rows and each ledger mutation is a - complete operation. Consequently no ledger transaction or lock remains - held while the pending-refresh callback runs between chunks. - - Rows whose stored status already matches the derived status are not - rewritten, and rows that are provably still coinbase-immature are not - probed at all. Because fenced writes are what refresh the writer - lease, a probe that derived an unchanged status renews the lease in - place of the write it used to make, and a pass with no fenced refresh - at all renews once at the end. - """ - if chunk_size <= 0: - raise ValueError("chunk_size must be positive") - if chunk_size > MAX_CTV_FANOUT_BROADCASTER_CHUNK_SIZE: - raise ValueError( - "chunk_size must be at most " - f"{MAX_CTV_FANOUT_BROADCASTER_CHUNK_SIZE}" - ) - rows = self.ledger.pending_ctv_fanout_statuses(limit=limit) - scanned_count = 0 - submitted_count = 0 - updated_count = 0 - failed_count = 0 - yielded_to_tip_refresh = False - lease_refreshed = False - tip_height: int | None = None - - def record_progress() -> None: - if progress_callback is not None: - progress_callback() - - def active_tip_height() -> int: - # One chain-tip probe per pass stands in for the per-row - # settlement probes of rows that are provably still immature. - nonlocal tip_height - if tip_height is None: - tip_height = int(self.broadcaster.tip_height()) - return tip_height - - def renew_lease() -> None: - nonlocal lease_refreshed - lease_refreshed = True - renew_writer_lease = getattr(self.ledger, "renew_writer_lease", None) - if renew_writer_lease is not None: - renew_writer_lease() - - for chunk_start in range(0, len(rows), chunk_size): - if tip_refresh_pending is not None and tip_refresh_pending(): - yielded_to_tip_refresh = True - break - - chunk_started = monotonic() if chunk_callback is not None else None - chunk_processed_count = 0 - chunk = rows[chunk_start : chunk_start + chunk_size] - for row in chunk: - scanned_count += 1 - chunk_processed_count += 1 - stored_status = str(row.get("settlement_status") or row.get("status") or "") - if stored_status in { - "confirmed", - "reorged", - "failed", - }: - record_progress() - continue - if not broadcast_attempt_due(row.get("next_broadcast_attempt_at")): - record_progress() - continue - if stored_status == "awaiting_maturity": - coinbase_height = _coinbase_height_from_row(row) - if ( - coinbase_height is not None - and active_tip_height() < coinbase_height + self.broadcaster.maturity - ): - # While the funding coinbase is height-immature the - # settlement probe can only re-derive awaiting_maturity - # (a fanout spending an immature coinbase cannot be on - # chain, and a reorged coinbase stays awaiting_maturity - # until height-mature), so skip the per-row qbitd RPCs - # and the identical status rewrite. - record_progress() - continue - artifact = artifact_from_status_row(row) - attempt = self.broadcaster.broadcast(artifact, self.fee_sats) - if attempt.submitted: - submitted_count += 1 - lease_refreshed = True - self._journal_attempt(attempt, attempt_status="submitted") - # record_ctv_fanout_broadcast_attempt moves the durable row to - # broadcast_submitted, so do not double-update here. - record_progress() - continue - - if attempt.fee_sats is not None: - attempt_status = "planned" if attempt.package_msg == "error" else "rejected" - lease_refreshed = True - self._journal_attempt(attempt, attempt_status=attempt_status) - failed_count += 1 - record_progress() - continue - - next_status = LEDGER_STATUS_BY_BROADCASTER_STATUS.get(attempt.status) - if next_status is not None: - if next_status != stored_status: - lease_refreshed = True - self.ledger.update_ctv_fanout_status( - fanout_txid=attempt.fanout_txid, - settlement_status=next_status, - ) - updated_count += 1 - else: - # The probe was several qbitd RPCs but derived an - # unchanged status, so no fenced write refreshes the - # lease here anymore; renew in the write's place so a - # stretch of slow unchanged probes cannot outlast the - # lease TTL. This keeps the per-row refresh cadence - # the removed no-op rewrites used to provide. - renew_lease() - record_progress() - continue - - lease_refreshed = True - self._journal_attempt(attempt, attempt_status="failed") - failed_count += 1 - record_progress() - - if chunk_callback is not None: - assert chunk_started is not None - chunk_callback( - CtvFanoutChunkResult( - processed_count=chunk_processed_count, - elapsed_seconds=max(0.0, monotonic() - chunk_started), - ) - ) - - if not lease_refreshed: - # Fenced writes are the only thing that refreshes the writer - # lease, and the no-op status rewrites this pass used to make were - # an otherwise-idle daemon's only fenced writes. Renew explicitly - # so the lease does not sit expired while idle and a fenced-out - # daemon still fails fast. Yielded passes renew too: consecutive - # yields must not let the lease lapse, and the single lease-row - # update is negligible next to the per-row work a yield defers. - renew_lease() - - return CtvFanoutDaemonResult( - scanned_count=scanned_count, - submitted_count=submitted_count, - updated_count=updated_count, - failed_count=failed_count, - yielded_to_tip_refresh=yielded_to_tip_refresh, - ) - - def _journal_attempt(self, attempt: BroadcastAttempt, *, attempt_status: str) -> None: - package_txids = [attempt.fanout_txid] - if attempt.child_txid is not None: - package_txids.append(attempt.child_txid) - self.ledger.record_ctv_fanout_broadcast_attempt( - fanout_txid=attempt.fanout_txid, - attempt_status=attempt_status, - package_txids=package_txids, - submit_result={ - "package_msg": attempt.package_msg, - "submitted": attempt.submitted, - "fee_sats": attempt.fee_sats, - }, - error=None if attempt.submitted else attempt.detail, - ) - - -def broadcast_attempt_due(value: object) -> bool: - if value is None: - return True - if isinstance(value, datetime): - candidate = value - else: - text = str(value).strip() - if not text: - return True - if " " in text and "T" not in text: - text = text.replace(" ", "T", 1) - if text.endswith("Z"): - text = text[:-1] + "+00:00" - elif text.endswith("+00"): - text = text[:-3] + "+00:00" - try: - candidate = datetime.fromisoformat(text) - except ValueError: - return True - if candidate.tzinfo is None: - candidate = candidate.replace(tzinfo=timezone.utc) - return candidate <= datetime.now(timezone.utc) diff --git a/lab/prism/ctv_runtime.py b/lab/prism/ctv_runtime.py deleted file mode 100644 index 17e5973ca..000000000 --- a/lab/prism/ctv_runtime.py +++ /dev/null @@ -1,411 +0,0 @@ -"""Coordinator-facing lifecycle and metrics for the CTV broadcaster daemon. - -The CTV broadcast engine and durable daemon remain in their existing modules. -This service owns only the process runtime seam: daemon construction, writer -admission, the dedicated loop, and its bounded-cardinality metrics. -""" - -from __future__ import annotations - -from contextlib import AbstractContextManager -from dataclasses import dataclass, replace -import threading -import time -import traceback -from typing import Any, Callable, Protocol - -from lab.prism.coordinator_config import CtvConfig -from lab.prism.ctv_broadcaster import CtvFanoutBroadcaster -from lab.prism.ctv_broadcaster_daemon import ( - CtvFanoutBroadcastDaemon, - CtvFanoutChunkResult, - CtvFanoutDaemonResult, -) -from lab.prism.share_ledger import WriterLeaseRenewalDeferred - - -CTV_FANOUT_BROADCASTER_SERVICE_NAME = "ctv_fanout_broadcaster" -CTV_BROADCAST_STATE_COMPONENT = "ctv_broadcast_state" -PRISM_CTV_BROADCASTER_SECONDS_BUCKETS = ( - 1.0, - 5.0, - 10.0, - 30.0, - 60.0, - 120.0, - 300.0, - 600.0, -) -PRISM_CTV_BROADCASTER_CHUNK_SECONDS_BUCKETS = ( - 0.01, - 0.025, - 0.05, - 0.1, - 0.25, - 0.5, - 1.0, - 2.5, - 5.0, - 10.0, - 30.0, -) -PRISM_CTV_BROADCASTER_CHUNK_ROWS_BUCKETS = (1, 2, 5, 10, 25, 50, 100) - - -class StopSignal(Protocol): - def is_set(self) -> bool: ... - - def wait(self, timeout: float) -> bool: ... - - -@dataclass(frozen=True, slots=True) -class CtvRuntimeConfig: - enabled: bool - wallet: str | None - fee_sats: int - limit: int - chunk_size: int - interval_seconds: float - - @classmethod - def from_coordinator_config(cls, config: CtvConfig) -> "CtvRuntimeConfig": - return cls( - enabled=config.broadcaster_enabled, - wallet=config.broadcaster_wallet, - fee_sats=config.broadcaster_fee_sats, - limit=config.broadcaster_limit, - chunk_size=config.broadcaster_chunk_size, - interval_seconds=config.broadcaster_interval_seconds, - ) - - -@dataclass(frozen=True, slots=True) -class CtvRuntimeMetricsSnapshot: - pass_seconds_bucket_counts: dict[float, int] - pass_seconds_sum: float - pass_count: int - processed_rows_total: int - yielded_total: int - chunk_seconds_bucket_counts: dict[float, int] - chunk_rows_bucket_counts: dict[int, int] - chunk_seconds_sum: float - chunk_rows_sum: int - chunk_count: int - - -class CtvRuntimeService: - """Own the coordinator's CTV daemon instance, loop, and metric state.""" - - def __init__( - self, - *, - rpc_call: Callable[..., Any], - ledger_source: Callable[[], object], - writer_admission: Callable[[str], AbstractContextManager[object]], - tip_refresh_pending: Callable[[], bool], - heartbeat: Callable[[], None], - stop_signal: Callable[[], StopSignal], - config: CtvRuntimeConfig, - # Fenced external-effect verification for broadcast RPCs. The port - # resolves the coordinator seam at call time so lease-service tests - # keep their monkeypatch surface. - before_external_side_effect: Callable[..., Any] | None = None, - # The coordinator-owned shutdown-admission error type. Injected until - # the shutdown controller owns its own module; the loop treats it as a - # silent clean exit exactly like the historical inline loop. - shutdown_exception: type[BaseException] = RuntimeError, - daemon_type: Callable[..., CtvFanoutBroadcastDaemon] = CtvFanoutBroadcastDaemon, - broadcaster_type: Callable[..., CtvFanoutBroadcaster] = CtvFanoutBroadcaster, - monotonic: Callable[[], float] = time.monotonic, - print_exception: Callable[[], None] = traceback.print_exc, - ) -> None: - self._rpc_call = rpc_call - self._ledger_source = ledger_source - self._writer_admission = writer_admission - self._tip_refresh_pending = tip_refresh_pending - self._heartbeat = heartbeat - self._stop_signal = stop_signal - self._before_external_side_effect = before_external_side_effect - self._shutdown_exception = shutdown_exception - self._daemon_type = daemon_type - self._broadcaster_type = broadcaster_type - self._monotonic = monotonic - self._print_exception = print_exception - self._config_lock = threading.Lock() - self._config = config - - self._daemon_lock = threading.Lock() - self._daemon: CtvFanoutBroadcastDaemon | None = None - self._metrics_lock = threading.Lock() - self._pass_seconds_bucket_counts = { - bucket: 0 for bucket in PRISM_CTV_BROADCASTER_SECONDS_BUCKETS - } - self._pass_seconds_sum = 0.0 - self._pass_count = 0 - self._processed_rows_total = 0 - self._yielded_total = 0 - self._chunk_seconds_bucket_counts = { - bucket: 0 for bucket in PRISM_CTV_BROADCASTER_CHUNK_SECONDS_BUCKETS - } - self._chunk_rows_bucket_counts = { - bucket: 0 for bucket in PRISM_CTV_BROADCASTER_CHUNK_ROWS_BUCKETS - } - self._chunk_seconds_sum = 0.0 - self._chunk_rows_sum = 0 - self._chunk_count = 0 - - @property - def daemon(self) -> CtvFanoutBroadcastDaemon | None: - with self._daemon_lock: - return self._daemon - - @daemon.setter - def daemon(self, daemon: CtvFanoutBroadcastDaemon | None) -> None: - with self._daemon_lock: - self._daemon = daemon - - @property - def config(self) -> CtvRuntimeConfig: - """Return one immutable configuration snapshot under service ownership.""" - with self._config_lock: - return self._config - - def replace_config(self, **changes: object) -> None: - """Support temporary coordinator compatibility properties.""" - with self._daemon_lock: - with self._config_lock: - previous = self._config - replacement = replace(previous, **changes) - self._config = replacement - if ( - previous.wallet != replacement.wallet - or previous.fee_sats != replacement.fee_sats - ): - self._daemon = None - - def make_daemon( - self, - config: CtvRuntimeConfig | None = None, - ) -> CtvFanoutBroadcastDaemon: - config = self.config if config is None else config - if config.fee_sats > 0 and not config.wallet: - raise ValueError( - "ctv_broadcaster_wallet is required when ctv_broadcaster_fee_sats is positive" - ) - broadcaster_kwargs: dict[str, Any] = {"funding_wallet": config.wallet} - if self._before_external_side_effect is not None: - broadcaster_kwargs["before_external_side_effect"] = ( - self._before_external_side_effect - ) - broadcaster = self._broadcaster_type( - self._rpc_call, - **broadcaster_kwargs, - ) - return self._daemon_type( - self._ledger_source(), - broadcaster, - fee_sats=config.fee_sats, - ) - - def _configured_daemon( - self, - ) -> tuple[CtvRuntimeConfig, CtvFanoutBroadcastDaemon]: - with self._daemon_lock: - with self._config_lock: - config = self._config - if self._daemon is None: - self._daemon = self.make_daemon(config) - return config, self._daemon - - def run_once( - self, - *, - progress_callback: Callable[[], None] | None = None, - chunk_callback: Callable[[CtvFanoutChunkResult], None] | None = None, - ) -> CtvFanoutDaemonResult: - with self._writer_admission(CTV_BROADCAST_STATE_COMPONENT): - config, daemon = self._configured_daemon() - return daemon.run_once( - limit=config.limit, - progress_callback=progress_callback, - chunk_size=config.chunk_size, - tip_refresh_pending=self._tip_refresh_pending, - chunk_callback=( - self.observe_chunk if chunk_callback is None else chunk_callback - ), - ) - - def record_progress(self) -> None: - self._heartbeat() - with self._metrics_lock: - self._processed_rows_total += 1 - - def observe_pass(self, elapsed_seconds: float) -> None: - with self._metrics_lock: - self._pass_count += 1 - self._pass_seconds_sum += elapsed_seconds - for bucket in PRISM_CTV_BROADCASTER_SECONDS_BUCKETS: - if elapsed_seconds <= bucket: - self._pass_seconds_bucket_counts[bucket] += 1 - - def observe_chunk(self, result: CtvFanoutChunkResult) -> None: - self._heartbeat() - with self._metrics_lock: - self._chunk_count += 1 - self._chunk_seconds_sum += result.elapsed_seconds - self._chunk_rows_sum += result.processed_count - for bucket in PRISM_CTV_BROADCASTER_CHUNK_SECONDS_BUCKETS: - if result.elapsed_seconds <= bucket: - self._chunk_seconds_bucket_counts[bucket] += 1 - for bucket in PRISM_CTV_BROADCASTER_CHUNK_ROWS_BUCKETS: - if result.processed_count <= bucket: - self._chunk_rows_bucket_counts[bucket] += 1 - - def record_yield(self) -> None: - with self._metrics_lock: - self._yielded_total += 1 - - def loop( - self, - *, - run_once: Callable[..., CtvFanoutDaemonResult] | None = None, - progress_callback: Callable[[], None] | None = None, - observe_pass: Callable[[float], None] | None = None, - record_yield: Callable[[], None] | None = None, - ) -> None: - """Run the dedicated broadcaster loop until shutdown. - - The optional call ports retain the coordinator's temporary method-level - compatibility seam; normal registered service threads use the owned - methods directly. - """ - pass_runner = self.run_once if run_once is None else run_once - progress_recorder = ( - self.record_progress if progress_callback is None else progress_callback - ) - pass_observer = self.observe_pass if observe_pass is None else observe_pass - yield_recorder = self.record_yield if record_yield is None else record_yield - while not self._stop_signal().is_set(): - self._heartbeat() - started = self._monotonic() - shutdown_admission_closed = False - try: - try: - result = pass_runner(progress_callback=progress_recorder) - except self._shutdown_exception: - shutdown_admission_closed = True - return - finally: - # Stamp completion before logging or entering the interval - # wait. A blocked row never reaches this finally clause, so - # the watchdog remains able to recover a wedged operation. - self._heartbeat() - if not shutdown_admission_closed: - pass_observer(max(0.0, self._monotonic() - started)) - except WriterLeaseRenewalDeferred as exc: - # A preflight refusal, not a failure: the writer's own - # fenced write is holding the lease renewal, and the fence - # withheld the guarded RPC before it ran. The next pass on - # the ordinary interval retries once the write commits and - # a renewal lands. - print( - "prism coordinator: CTV fanout broadcaster pass " - f"withheld: {exc}", - flush=True, - ) - except Exception: - print("prism coordinator: CTV fanout broadcaster pass failed", flush=True) - self._print_exception() - else: - if result.yielded_to_tip_refresh: - yield_recorder() - if result.scanned_count or result.submitted_count or result.failed_count: - print( - "prism coordinator: CTV fanout broadcaster " - f"scanned={result.scanned_count} " - f"submitted={result.submitted_count} " - f"updated={result.updated_count} " - f"failed={result.failed_count}", - flush=True, - ) - if self._stop_signal().wait(self.config.interval_seconds): - break - - def startup_summary(self) -> str: - config = self.config - return ( - "prism coordinator: CTV fanout broadcaster enabled " - f"mode={'cpfp' if config.fee_sats > 0 else 'direct'} " - f"fee_bits={config.fee_sats} " - f"wallet={'configured' if config.wallet else 'none'} " - f"interval={config.interval_seconds:g}s " - f"limit={config.limit} " - f"chunk_size={config.chunk_size}" - ) - - def metrics_snapshot(self) -> CtvRuntimeMetricsSnapshot: - with self._metrics_lock: - return CtvRuntimeMetricsSnapshot( - pass_seconds_bucket_counts=dict(self._pass_seconds_bucket_counts), - pass_seconds_sum=self._pass_seconds_sum, - pass_count=self._pass_count, - processed_rows_total=self._processed_rows_total, - yielded_total=self._yielded_total, - chunk_seconds_bucket_counts=dict(self._chunk_seconds_bucket_counts), - chunk_rows_bucket_counts=dict(self._chunk_rows_bucket_counts), - chunk_seconds_sum=self._chunk_seconds_sum, - chunk_rows_sum=self._chunk_rows_sum, - chunk_count=self._chunk_count, - ) - - @property - def processed_rows_total(self) -> int: - return self.metrics_snapshot().processed_rows_total - - @property - def pass_count(self) -> int: - return self.metrics_snapshot().pass_count - - def metrics_lines(self) -> list[str]: - snapshot = self.metrics_snapshot() - metric_name = "qbit_prism_ctv_fanout_broadcaster_pass_seconds" - chunk_seconds_name = "qbit_prism_ctv_fanout_broadcaster_chunk_seconds" - chunk_rows_name = "qbit_prism_ctv_fanout_broadcaster_chunk_rows" - return [ - "# HELP qbit_prism_ctv_fanout_broadcaster_processed_rows_total CTV fanout rows completed by the broadcaster loop.", - "# TYPE qbit_prism_ctv_fanout_broadcaster_processed_rows_total counter", - f"qbit_prism_ctv_fanout_broadcaster_processed_rows_total {snapshot.processed_rows_total}", - "# HELP qbit_prism_ctv_fanout_broadcaster_pass_seconds CTV fanout broadcaster pass wall time.", - "# TYPE qbit_prism_ctv_fanout_broadcaster_pass_seconds histogram", - *[ - f'{metric_name}_bucket{{le="{bucket:g}"}} ' - f"{snapshot.pass_seconds_bucket_counts.get(bucket, 0)}" - for bucket in PRISM_CTV_BROADCASTER_SECONDS_BUCKETS - ], - f'{metric_name}_bucket{{le="+Inf"}} {snapshot.pass_count}', - f"{metric_name}_sum {snapshot.pass_seconds_sum:.6f}", - f"{metric_name}_count {snapshot.pass_count}", - "# HELP qbit_prism_ctv_fanout_broadcaster_tip_refresh_yields_total CTV broadcaster passes yielding between committed chunks for a pending tip refresh.", - "# TYPE qbit_prism_ctv_fanout_broadcaster_tip_refresh_yields_total counter", - f"qbit_prism_ctv_fanout_broadcaster_tip_refresh_yields_total {snapshot.yielded_total}", - "# HELP qbit_prism_ctv_fanout_broadcaster_chunk_seconds CTV broadcaster committed chunk wall time.", - "# TYPE qbit_prism_ctv_fanout_broadcaster_chunk_seconds histogram", - *[ - f'{chunk_seconds_name}_bucket{{le="{bucket:g}"}} ' - f"{snapshot.chunk_seconds_bucket_counts.get(bucket, 0)}" - for bucket in PRISM_CTV_BROADCASTER_CHUNK_SECONDS_BUCKETS - ], - f'{chunk_seconds_name}_bucket{{le="+Inf"}} {snapshot.chunk_count}', - f"{chunk_seconds_name}_sum {snapshot.chunk_seconds_sum:.6f}", - f"{chunk_seconds_name}_count {snapshot.chunk_count}", - "# HELP qbit_prism_ctv_fanout_broadcaster_chunk_rows Rows processed per committed CTV broadcaster chunk.", - "# TYPE qbit_prism_ctv_fanout_broadcaster_chunk_rows histogram", - *[ - f'{chunk_rows_name}_bucket{{le="{bucket}"}} ' - f"{snapshot.chunk_rows_bucket_counts.get(bucket, 0)}" - for bucket in PRISM_CTV_BROADCASTER_CHUNK_ROWS_BUCKETS - ], - f'{chunk_rows_name}_bucket{{le="+Inf"}} {snapshot.chunk_count}', - f"{chunk_rows_name}_sum {snapshot.chunk_rows_sum}", - f"{chunk_rows_name}_count {snapshot.chunk_count}", - ] diff --git a/lab/prism/direct_stratum.py b/lab/prism/direct_stratum.py deleted file mode 100644 index b882ff9c8..000000000 --- a/lab/prism/direct_stratum.py +++ /dev/null @@ -1,532 +0,0 @@ -#!/usr/bin/env python3 -"""Transport-independent direct qbit Stratum helpers for PRISM pool jobs.""" - -from __future__ import annotations - -import json -import re -from dataclasses import dataclass -from decimal import Decimal, InvalidOperation, getcontext -from typing import Any - -from lab.auxpow import stratum_codec - -getcontext().prec = 40 - -DIFF1_TARGET = int("00000000ffff0000000000000000000000000000000000000000000000000000", 16) -HEX_MASK_RE = re.compile(r"^(?:0x)?[0-9a-fA-F]{1,8}$") -QBIT_VERSION_ROLLING_MASK = 0x1FFFE000 -QBIT_VERSION_ROLLING_MASK_HEX = f"{QBIT_VERSION_ROLLING_MASK:08x}" - - -@dataclass(frozen=True) -class VersionRollingMaskSelection: - selected_mask: int - source: str - detail: str - advertised_mask: str | None = None - - -@dataclass(frozen=True) -class DirectQbitStratumJob: - job_id: str - previousblockhash_display: str - prevhash: str - coinb1: str - coinb2: str - full_coinbase_prefix: str - full_coinbase_suffix: str - merkle_branch: tuple[str, ...] - transaction_hexes: tuple[str, ...] - version: str - nbits: str - ntime: str - qbit_target: int - share_target: int - share_difficulty: Decimal - extranonce1_hex: str - extranonce2_size: int - clean_jobs: bool = True - # The seven mining.notify params every client of one job build shares - # (prevhash through ntime), serialized once at build time. Per-client - # stamping copies the reference, so a fleet-wide notify wave splices job - # id and clean_jobs around this fragment instead of re-serializing the - # coinbase parts once per connection. None on hand-built jobs; senders - # fall back to full serialization. - notify_shared_params_json: str | None = None - - -@dataclass(frozen=True) -class DirectQbitSubmission: - coinbase_tx_hex: str - coinbase_txid_preimage_hex: str - header_hex: str - block_hex: str - block_hash_hex: str - block_hash_int: int - share_pass: bool - block_pass: bool - applied_version_hex: str - - -def normalize_version_rolling_mask(value: Any, *, field_name: str) -> int: - if isinstance(value, int): - if value < 0 or value > 0xFFFFFFFF: - raise ValueError(f"{field_name} must fit uint32") - return value - - text = str(value).strip() - if not HEX_MASK_RE.fullmatch(text): - raise ValueError(f"{field_name} must be 1 to 8 hex chars") - if text.lower().startswith("0x"): - text = text[2:] - return int(text, 16) & 0xFFFFFFFF - - -def select_version_rolling_mask(template: dict[str, Any], fallback_mask: int) -> VersionRollingMaskSelection: - if fallback_mask < 0 or fallback_mask > 0xFFFFFFFF: - raise ValueError("fallback version rolling mask must fit uint32") - if "versionrollingmask" not in template: - return VersionRollingMaskSelection(fallback_mask, "fallback", "missing_versionrollingmask") - - advertised = template.get("versionrollingmask") - try: - selected = normalize_version_rolling_mask(advertised, field_name="versionrollingmask") - except ValueError as exc: - raise ValueError(f"invalid getblocktemplate.versionrollingmask: {exc}") from exc - - if selected == 0: - return VersionRollingMaskSelection(selected, "qbit_getblocktemplate", "disabled_by_zero_mask", str(advertised)) - return VersionRollingMaskSelection(selected, "qbit_getblocktemplate", "advertised", str(advertised)) - - -def compact_size(value: int) -> bytes: - if value < 0: - raise ValueError("compact size cannot be negative") - if value < 253: - return bytes([value]) - if value <= 0xFFFF: - return b"\xfd" + value.to_bytes(2, "little") - if value <= 0xFFFF_FFFF: - return b"\xfe" + value.to_bytes(4, "little") - if value <= 0xFFFF_FFFF_FFFF_FFFF: - return b"\xff" + value.to_bytes(8, "little") - raise ValueError("compact size is too large") - - -def read_compact_size(data: bytes, offset: int, *, field_name: str) -> tuple[int, int]: - if offset >= len(data): - raise ValueError(f"{field_name} truncated before compact size") - first = data[offset] - offset += 1 - if first < 253: - return first, offset - if first == 253: - width = 2 - elif first == 254: - width = 4 - else: - width = 8 - if offset + width > len(data): - raise ValueError(f"{field_name} truncated inside compact size") - return int.from_bytes(data[offset : offset + width], "little"), offset + width - - -def skip_bytes(data: bytes, offset: int, count: int, *, field_name: str) -> int: - if count < 0: - raise ValueError(f"{field_name} cannot skip a negative byte count") - if offset + count > len(data): - raise ValueError(f"{field_name} truncated") - return offset + count - - -def strip_witness_transaction(tx_hex: str) -> bytes: - tx = bytes.fromhex(stratum_codec.validate_hex(tx_hex, field_name="transaction")) - if len(tx) < 10: - raise ValueError("transaction is too short") - - version = tx[:4] - offset = 4 - has_witness = tx[offset] == 0 and tx[offset + 1] != 0 - if has_witness: - offset += 2 - body_start = offset - - input_count, offset = read_compact_size(tx, offset, field_name="transaction inputs") - for input_index in range(input_count): - offset = skip_bytes(tx, offset, 36, field_name=f"transaction input[{input_index}] prevout") - script_len, offset = read_compact_size(tx, offset, field_name=f"transaction input[{input_index}] script") - offset = skip_bytes(tx, offset, script_len, field_name=f"transaction input[{input_index}] script") - offset = skip_bytes(tx, offset, 4, field_name=f"transaction input[{input_index}] sequence") - - output_count, offset = read_compact_size(tx, offset, field_name="transaction outputs") - for output_index in range(output_count): - offset = skip_bytes(tx, offset, 8, field_name=f"transaction output[{output_index}] value") - script_len, offset = read_compact_size(tx, offset, field_name=f"transaction output[{output_index}] script") - offset = skip_bytes(tx, offset, script_len, field_name=f"transaction output[{output_index}] script") - - witness_start = offset - if has_witness: - for input_index in range(input_count): - item_count, offset = read_compact_size(tx, offset, field_name=f"transaction witness[{input_index}]") - for item_index in range(item_count): - item_len, offset = read_compact_size( - tx, - offset, - field_name=f"transaction witness[{input_index}][{item_index}]", - ) - offset = skip_bytes( - tx, - offset, - item_len, - field_name=f"transaction witness[{input_index}][{item_index}]", - ) - - locktime = tx[offset : offset + 4] - if len(locktime) != 4: - raise ValueError("transaction truncated before locktime") - offset += 4 - if offset != len(tx): - raise ValueError("transaction has trailing bytes after locktime") - - if not has_witness: - return tx - return version + tx[body_start:witness_start] + locktime - - -def transaction_txid_internal(tx_hex: str) -> bytes: - return stratum_codec.double_sha256(strip_witness_transaction(tx_hex)) - - -def transaction_wtxid_internal(tx_hex: str) -> bytes: - tx = bytes.fromhex(stratum_codec.validate_hex(tx_hex, field_name="transaction")) - return stratum_codec.double_sha256(tx) - - -def witness_merkle_leaves_hex(transaction_hexes: tuple[str, ...]) -> list[str]: - return [transaction_wtxid_internal(tx_hex).hex() for tx_hex in transaction_hexes] - - -def transaction_txid_display(tx_hex: str) -> str: - return transaction_txid_internal(tx_hex)[::-1].hex() - - -def target_from_compact_hex(nbits_hex: str) -> int: - nbits_hex = stratum_codec.validate_uint32_hex(nbits_hex, field_name="nbits") - compact = int(nbits_hex, 16) - size = compact >> 24 - mantissa = compact & 0x007FFFFF - if size <= 3: - return mantissa >> (8 * (3 - size)) - return mantissa << (8 * (size - 3)) - - -def difficulty_target(difficulty: Decimal | int | str) -> int: - parsed = parse_positive_decimal(difficulty, field_name="difficulty") - target = int(Decimal(DIFF1_TARGET) / parsed) - return max(1, min(target, (1 << 256) - 1)) - - -def target_difficulty(target: int) -> Decimal: - if target <= 0: - raise ValueError("target must be positive") - return Decimal(DIFF1_TARGET) / Decimal(target) - - -def effective_share_target( - desired_share_difficulty: Decimal | int | str, - qbit_target: int, - *, - minimum_advertised_difficulty: Decimal | int | str = Decimal("0"), -) -> int: - share_target = max(difficulty_target(desired_share_difficulty), qbit_target) - minimum = parse_nonnegative_decimal(minimum_advertised_difficulty, field_name="minimum_advertised_difficulty") - if minimum > 0: - share_target = min(share_target, difficulty_target(minimum)) - return share_target - - -def parse_positive_decimal(value: Decimal | int | str, *, field_name: str) -> Decimal: - try: - parsed = Decimal(str(value)) - except InvalidOperation as exc: - raise ValueError(f"{field_name} must be a decimal") from exc - if not parsed.is_finite() or parsed <= 0: - raise ValueError(f"{field_name} must be positive") - return parsed - - -def parse_nonnegative_decimal(value: Decimal | int | str, *, field_name: str) -> Decimal: - try: - parsed = Decimal(str(value)) - except InvalidOperation as exc: - raise ValueError(f"{field_name} must be a decimal") from exc - if not parsed.is_finite() or parsed < 0: - raise ValueError(f"{field_name} must be non-negative") - return parsed - - -def coinbase_scriptsig_span(tx: bytes, *, field_name: str) -> tuple[int, int]: - """Return the (start, length) byte span of the coinbase input's scriptSig. - - The transaction is parsed structurally (version, optional SegWit marker/flag, - then the coinbase input's outpoint and scriptSig length prefix) so the - extranonce can be located by byte offset instead of by substring search. This - is collision proof: bytes that merely *look* like the extranonce placeholder - elsewhere in the coinbase (a payout output script, the witness commitment, the - witness nonce, the all-zero null outpoint, or the scriptSig height push) are - never mistaken for the extranonce insertion point. - """ - if len(tx) < 10: - raise ValueError(f"{field_name} is too short to be a coinbase transaction") - offset = 4 # version - if tx[offset] == 0 and tx[offset + 1] != 0: # SegWit marker + flag - offset += 2 - input_count, offset = read_compact_size(tx, offset, field_name=f"{field_name} inputs") - if input_count < 1: - raise ValueError(f"{field_name} has no coinbase input") - offset = skip_bytes(tx, offset, 36, field_name=f"{field_name} coinbase outpoint") - script_len, offset = read_compact_size(tx, offset, field_name=f"{field_name} coinbase scriptSig") - script_start = offset - # Validate the scriptSig is fully present without advancing past it. - skip_bytes(tx, offset, script_len, field_name=f"{field_name} coinbase scriptSig") - return script_start, script_len - - -def split_coinbase_extranonce(tx_hex: str, placeholder_hex: str, *, field_name: str) -> tuple[str, str]: - """Split a coinbase around the extranonce placeholder at its known offset. - - The pool builder appends the extranonce placeholder (extranonce1 followed by - ``extranonce2_size`` zero bytes) to the end of the coinbase input's scriptSig. - We locate that scriptSig by parsing the transaction and split immediately - before the placeholder, returning the hex prefix/suffix around it. Unlike a - raw ``tx_hex.find(placeholder)`` this never trips over a coincidental copy of - the placeholder bytes elsewhere in the coinbase, so a single unlucky template - cannot corrupt the split or crash the caller. - """ - tx_hex = stratum_codec.validate_hex(tx_hex, field_name=field_name) - placeholder_hex = stratum_codec.validate_hex(placeholder_hex, field_name="extranonce placeholder") - if not placeholder_hex: - raise ValueError("extranonce placeholder cannot be empty") - tx = bytes.fromhex(tx_hex) - placeholder = bytes.fromhex(placeholder_hex) - script_start, script_len = coinbase_scriptsig_span(tx, field_name=field_name) - if len(placeholder) > script_len: - raise ValueError(f"{field_name} coinbase scriptSig is shorter than the extranonce placeholder") - script_end = script_start + script_len - placeholder_start = script_end - len(placeholder) - if tx[placeholder_start:script_end] != placeholder: - raise ValueError(f"{field_name} does not end its coinbase scriptSig with the extranonce placeholder") - return tx[:placeholder_start].hex(), tx[script_end:].hex() - - -def transaction_hexes_from_template(template: dict[str, Any]) -> tuple[str, ...]: - txs = template.get("transactions", []) - if not isinstance(txs, list): - raise ValueError("template transactions must be an array") - result = [] - for index, tx in enumerate(txs): - if not isinstance(tx, dict) or "data" not in tx: - raise ValueError(f"template transaction[{index}] must include data") - result.append(stratum_codec.validate_hex(str(tx["data"]), field_name=f"template transaction[{index}] data")) - return tuple(result) - - -def merkle_branch_for_coinbase(transaction_hexes: tuple[str, ...]) -> tuple[str, ...]: - hashes = [b"\x00" * 32] - hashes.extend(transaction_txid_internal(tx_hex) for tx_hex in transaction_hexes) - index = 0 - branch: list[str] = [] - while len(hashes) > 1: - sibling_index = index ^ 1 - if sibling_index >= len(hashes): - sibling_index = index - branch.append(hashes[sibling_index].hex()) - - next_hashes = [] - for offset in range(0, len(hashes), 2): - left = hashes[offset] - right = hashes[offset + 1] if offset + 1 < len(hashes) else hashes[offset] - next_hashes.append(stratum_codec.double_sha256(left + right)) - hashes = next_hashes - index //= 2 - return tuple(branch) - - -def make_job_from_builder_manifest( - *, - job_id: str, - template: dict[str, Any], - manifest: dict[str, Any], - extranonce1_hex: str, - extranonce2_size: int, - desired_share_difficulty: Decimal | int | str, - minimum_advertised_difficulty: Decimal | int | str = Decimal("0"), - clean_jobs: bool = True, - transaction_hexes: tuple[str, ...] | None = None, -) -> DirectQbitStratumJob: - if extranonce2_size <= 0: - raise ValueError("extranonce2_size must be positive") - extranonce1_hex = stratum_codec.validate_hex(extranonce1_hex, field_name="extranonce1") - coinbase_tx_hex = stratum_codec.validate_hex(str(manifest["coinbase_tx_hex"]), field_name="coinbase_tx_hex") - expected_txid = manifest.get("coinbase_txid") - actual_txid = transaction_txid_display(coinbase_tx_hex) - if expected_txid is not None and str(expected_txid).lower() != actual_txid: - raise ValueError("manifest coinbase_txid does not match coinbase_tx_hex without witness") - - placeholder_hex = extranonce1_hex + ("00" * extranonce2_size) - no_witness_coinbase_hex = strip_witness_transaction(coinbase_tx_hex).hex() - coinb1, coinb2 = split_coinbase_extranonce( - no_witness_coinbase_hex, - placeholder_hex, - field_name="coinbase txid preimage", - ) - full_prefix, full_suffix = split_coinbase_extranonce( - coinbase_tx_hex, - placeholder_hex, - field_name="full coinbase transaction", - ) - - if transaction_hexes is None: - transaction_hexes = transaction_hexes_from_template(template) - else: - transaction_hexes = tuple( - stratum_codec.validate_hex(tx_hex, field_name=f"transaction_hexes[{index}]") - for index, tx_hex in enumerate(transaction_hexes) - ) - - nbits = stratum_codec.validate_uint32_hex(str(template["bits"]), field_name="template bits") - qbit_target = target_from_compact_hex(nbits) - share_target = effective_share_target( - desired_share_difficulty, - qbit_target, - minimum_advertised_difficulty=minimum_advertised_difficulty, - ) - version = f"{int(template['version']) & 0xFFFFFFFF:08x}" - ntime = f"{int(template['curtime']) & 0xFFFFFFFF:08x}" - previousblockhash = stratum_codec.validate_hash_hex( - str(template["previousblockhash"]), - field_name="template previousblockhash", - ) - - prevhash = stratum_codec.stratum_prevhash_from_display_hash(previousblockhash) - merkle_branch = merkle_branch_for_coinbase(transaction_hexes) - return DirectQbitStratumJob( - job_id=str(job_id), - previousblockhash_display=previousblockhash, - prevhash=prevhash, - coinb1=coinb1, - coinb2=coinb2, - full_coinbase_prefix=full_prefix, - full_coinbase_suffix=full_suffix, - merkle_branch=merkle_branch, - transaction_hexes=transaction_hexes, - version=version, - nbits=nbits, - ntime=ntime, - qbit_target=qbit_target, - share_target=share_target, - share_difficulty=target_difficulty(share_target), - extranonce1_hex=extranonce1_hex, - extranonce2_size=extranonce2_size, - clean_jobs=clean_jobs, - notify_shared_params_json=notify_shared_params_json( - prevhash=prevhash, - coinb1=coinb1, - coinb2=coinb2, - merkle_branch=merkle_branch, - version=version, - nbits=nbits, - ntime=ntime, - ), - ) - - -def notify_shared_params_json( - *, - prevhash: str, - coinb1: str, - coinb2: str, - merkle_branch: tuple[str, ...], - version: str, - nbits: str, - ntime: str, -) -> str: - """Serialize the seven client-invariant mining.notify params once. - - The returned fragment is the JSON of the seven-element list with its - outer brackets stripped, so a sender can splice the per-client job id - before it and the per-client clean_jobs flag after it and produce bytes - identical to serializing the full params list. - """ - return json.dumps( - [prevhash, coinb1, coinb2, list(merkle_branch), version, nbits, ntime] - )[1:-1] - - -def assemble_submission( - job: DirectQbitStratumJob, - *, - extranonce2_hex: str, - ntime_hex: str, - nonce_hex: str, - version_bits_hex: str | None = None, - version_mask: int = 0, -) -> DirectQbitSubmission: - extranonce2_hex = stratum_codec.validate_hex(extranonce2_hex, field_name="extranonce2") - if len(extranonce2_hex) != job.extranonce2_size * 2: - raise ValueError("unexpected extranonce2 size") - version_hex = stratum_codec.apply_version_bits(job.version, version_bits_hex, version_mask) - coinbase_without_witness, header = stratum_codec.assemble_header_from_notify_submit( - coinb1_hex=job.coinb1, - extranonce1_hex=job.extranonce1_hex, - extranonce2_hex=extranonce2_hex, - coinb2_hex=job.coinb2, - merkle_branch_hex=list(job.merkle_branch), - version_hex=version_hex, - prevhash_hex=job.prevhash, - ntime_hex=ntime_hex, - nbits_hex=job.nbits, - nonce_hex=nonce_hex, - ) - full_coinbase_hex = job.full_coinbase_prefix + job.extranonce1_hex + extranonce2_hex + job.full_coinbase_suffix - if strip_witness_transaction(full_coinbase_hex) != coinbase_without_witness: - raise ValueError("full coinbase witness transaction does not match Stratum txid preimage") - header_hash_int = stratum_codec.header_hash_int(header) - block_pass = header_hash_int <= job.qbit_target - return DirectQbitSubmission( - coinbase_tx_hex=full_coinbase_hex, - coinbase_txid_preimage_hex=coinbase_without_witness.hex(), - header_hex=header.hex(), - # Serializing the block costs O(block bytes) of GIL-held decode, - # concatenate and re-encode, and only the block-landing path ever - # reads the result, so only a hash that actually solved the block - # pays for it. ``block_pass`` is the exact precondition rather than a - # heuristic: share_submission.classify_submit_work makes it necessary - # for every route that builds a PrismBlockCandidate, so the bytes are - # always materialized here -- eagerly, on the client thread, inside - # the call that already validates the coinbase -- before a candidate - # exists and long before its durable intent is written. An ordinary - # share can never need them. - block_hex=( - serialize_block(header, full_coinbase_hex, job.transaction_hexes).hex() - if block_pass - else "" - ), - block_hash_hex=stratum_codec.header_hash_hex(header), - block_hash_int=header_hash_int, - share_pass=header_hash_int <= job.share_target, - block_pass=block_pass, - applied_version_hex=version_hex, - ) - - -def serialize_block(header: bytes, coinbase_tx_hex: str, transaction_hexes: tuple[str, ...]) -> bytes: - if len(header) != 80: - raise ValueError("header must be 80 bytes") - txs = [bytes.fromhex(stratum_codec.validate_hex(coinbase_tx_hex, field_name="coinbase_tx_hex"))] - txs.extend( - bytes.fromhex(stratum_codec.validate_hex(tx_hex, field_name=f"transaction_hexes[{index}]")) - for index, tx_hex in enumerate(transaction_hexes) - ) - return header + compact_size(len(txs)) + b"".join(txs) diff --git a/lab/prism/endpoint_registry.py b/lab/prism/endpoint_registry.py deleted file mode 100644 index 9f4543d79..000000000 --- a/lab/prism/endpoint_registry.py +++ /dev/null @@ -1,805 +0,0 @@ -"""Declarative classification of every endpoint the PRISM HTTP surface serves. - -The coordinator process answers three unrelated kinds of request on one -listener (``lab/prism/audit_http.py``): a public read surface anyone on the -internet may poll, an operator surface describing this process instance, and a -set of payout-operations reads that contend with the writer lease the -block-landing path holds. Issue #145 extracts the first of those into its own -process; this module is the boundary made explicit so the split can be reviewed -one route at a time before anything moves. - -Nothing here changes request handling. ``AuditHttpFacade`` still owns dispatch, -and this registry is asserted against it in -``tests/test_prism_endpoint_registry.py`` so a new route cannot be added without -being classified. - -The boundary is not invented here. ``PRISM.md`` already states it: the -coordinator "exposes a private audit/ops listener and a dashboard-safe public -API from the same process", lists ``/audit/*``, ``/healthz``, ``/metrics`` and -``/owed-balances`` as private/internal, and concludes that "operators can expose -only ``/public/v1`` through a reverse proxy or dashboard frontend". -``docs/public-dashboard-api/README.md`` repeats it as a contract -- "the public -API must not expose ``/audit/*``, ``/metrics``, ``/healthz``, operator controls -..." -- and ``tests/test_public_dashboard_api_contract.py`` enforces it by -asserting the OpenAPI document contains no ``/audit/`` path. - -So published audit evidence reaches the public through the content-addressed -``/public/v1/artifacts/{sha256}`` route and the artifact links that reference -it, not through ``/audit/*``. The evidence content is public; the ``/audit/*`` -routes are internal tooling over the same data, with raw shapes -(``fanout_tx_hex``, ``manifest_set_json``, full broadcast-attempt histories), -raw ``sats`` units, no pagination, and in one case a caller-controlled unbounded -window. - -The three axes a classification answers: - -``Audience`` - ``PUBLIC_READ`` is the documented, versioned dashboard contract. - ``OPERATOR`` describes a process instance or the private audit/ops tooling. - ``PAYOUT_AFFECTING`` reports live payout obligations -- the state the - settlement machinery itself acts on. - -``LedgerAccess`` - How the response reaches durable state. ``WRITER_LOCK`` reads take - ``share_ledger._operation_gate(self._lock, "writer lock")`` and therefore - serialize against the lease-holding writer -- public volume on one of these - is share-ingest backpressure. ``READ_SLOT`` reads take the bounded read - semaphore (``PRISM_POSTGRES_READ_CONCURRENCY``) and are replica-eligible. - ``ARTIFACT_FILE`` reads come from the published audit artifacts, which are - already the durable public record. ``PROCESS_STATE`` reads in-memory - coordinator state and cannot leave the process at all. - -``Disposition`` - Where the route lands after issue #145. - -``max_staleness_seconds`` - How old an answer an extracted route may serve before it must refuse. - -Staleness is a contract, not a nicety: a silently stale dashboard is worse than -an honest one. ``lab/prism/observability.py`` set the precedent for this -process -- the cached ``/metrics`` endpoint returns 503 rather than serve an -unbounded-stale snapshot, using ``stale_after = max(3 * refresh_seconds, -MINIMUM_HEALTH_STALE_SECONDS)`` with a 15-second floor, and always publishes -the observed age. The extracted public tier follows the same discipline with -budgets derived from the caches that actually sit under each route rather than -hand-picked per endpoint: - - budget = max(3 * (cache_ttl_seconds + underlying_cache_seconds), - MINIMUM_PUBLIC_STALE_SECONDS) - -``cache_ttl_seconds`` is the route's own shared-response TTL from -``public_api.public_cache_policy`` at its documented default: 5s for the plain -row reads, 30s for the pool-wide aggregates (``/public/v1/pool-summary``, -``/public/v1/hashrate-series``, ``/public/v1/miners/{recipient_id}/workers``), -and 300s for ``/public/v1/mining-configuration``. - -``underlying_cache_seconds`` is any second cache stacked beneath that one. Only -one exists today: the ledger's pool reward-window aggregate -(``share_ledger._pool_reward_window_aggregate``, -``PRISM_PUBLIC_REWARD_WINDOW_CACHE_SECONDS``, default 30s), reached from -``dashboard_miner_reward_window`` and therefore counted only for the miner page -``/public/v1/miners/{recipient_id}``. Two caches in series compound, so the -budget must cover their sum, not the larger of the two. - -The factor of 3 and the 15-second floor are the precedent's, deliberately: a -budget of exactly one TTL would refuse a response that is merely due for -refresh, and a sub-15s budget on a 5s TTL would make ordinary scheduling jitter -look like an outage. - -``/public/v1/artifacts/{sha256}`` is the one route with no budget at all, and -says so through ``immutable_content`` rather than by omission. Its content is -immutable and content-addressed: a body that hashes to the requested sha256 is -correct no matter how old it is, so refusing it for age would be refusing a -correct answer. The only observable staleness is the 404 -> 200 transition for -an artifact published moments ago, which no budget would fix. - -Replica lag is the third staleness source named in -``docs/prism-public-operator-endpoint-split.md``. No replica exists yet, so it -contributes 0 here; when one lands it becomes another -``underlying_cache_seconds`` term rather than a new mechanism. -""" - -from __future__ import annotations - -from dataclasses import dataclass -from enum import Enum - - -class Audience(Enum): - PUBLIC_READ = "public_read" - OPERATOR = "operator" - PAYOUT_AFFECTING = "payout_affecting" - - -class LedgerAccess(Enum): - WRITER_LOCK = "writer_lock" - READ_SLOT = "read_slot" - ARTIFACT_FILE = "artifact_file" - PROCESS_STATE = "process_state" - QBIT_RPC = "qbit_rpc" - NONE = "none" - - -class Disposition(Enum): - EXTRACT = "extract" - RETAIN = "retain" - - -# The staleness floor, matching observability.MINIMUM_HEALTH_STALE_SECONDS. -MINIMUM_PUBLIC_STALE_SECONDS = 15 - -# Documented defaults of the caches a public response can sit behind. These are -# the defaults the budgets below are derived from; an operator who raises a TTL -# past its budget is told so by tests/test_prism_endpoint_registry.py. -PUBLIC_CACHE_TTL_DEFAULT_SECONDS = 5 -PUBLIC_AGGREGATE_CACHE_TTL_DEFAULT_SECONDS = 30 -PUBLIC_CONFIG_CACHE_TTL_DEFAULT_SECONDS = 300 -POOL_REWARD_WINDOW_CACHE_DEFAULT_SECONDS = 30 - - -def staleness_budget_seconds( - *, - cache_ttl_seconds: int, - underlying_cache_seconds: int = 0, -) -> int: - """Derive one route's staleness budget from the caches beneath it. - - See the module docstring: three refresh intervals of every cache in the - series, floored at MINIMUM_PUBLIC_STALE_SECONDS. - """ - - if cache_ttl_seconds < 0 or underlying_cache_seconds < 0: - raise ValueError("cache seconds must be nonnegative") - return max( - 3 * (cache_ttl_seconds + underlying_cache_seconds), - MINIMUM_PUBLIC_STALE_SECONDS, - ) - - -@dataclass(frozen=True) -class Endpoint: - """One classified route. - - ``paths`` lists every request path that reaches the same handler, so an - alias cannot be silently dropped by the extraction. - """ - - paths: tuple[str, ...] - audience: Audience - disposition: Disposition - access: tuple[LedgerAccess, ...] - rationale: str - # Ledger methods reached, as evidence for the access classification. - ledger_methods: tuple[str, ...] = () - # Set when a route's audience and its current implementation disagree -- - # the work the extraction must do before the route can move. - extraction_blocker: str | None = None - # How old a response this route may serve before it must refuse with 503. - # Derived through staleness_budget_seconds(), never hand-picked. None means - # no budget, which is only legitimate together with immutable_content. - max_staleness_seconds: float | None = None - # Content-addressed, immutable bodies: correct at any age, so this route - # never refuses for staleness. Stated rather than left implicit. - immutable_content: bool = False - - def __post_init__(self) -> None: - if not self.paths: - raise ValueError("endpoint must list at least one path") - for path in self.paths: - if not path.startswith("/"): - raise ValueError(f"endpoint path must be absolute: {path!r}") - if not self.access: - raise ValueError(f"endpoint {self.paths[0]} must declare ledger access") - if not self.rationale: - raise ValueError(f"endpoint {self.paths[0]} must carry a rationale") - if (self.audience is Audience.PUBLIC_READ) != ( - self.disposition is Disposition.EXTRACT - ): - raise ValueError( - f"endpoint {self.paths[0]}: exactly the public read surface is " - "extracted, so audience and disposition must agree" - ) - if ( - LedgerAccess.WRITER_LOCK in self.access - and self.disposition is Disposition.EXTRACT - and self.extraction_blocker is None - ): - raise ValueError( - f"endpoint {self.paths[0]} takes the writer lock and cannot be " - "extracted without recording the blocker that must be removed" - ) - if self.immutable_content and self.disposition is not Disposition.EXTRACT: - raise ValueError( - f"endpoint {self.paths[0]}: immutable_content describes the " - "extracted public contract only" - ) - if self.immutable_content and self.max_staleness_seconds is not None: - raise ValueError( - f"endpoint {self.paths[0]}: immutable content is correct at any " - "age, so it must not also carry a staleness budget" - ) - if self.disposition is Disposition.EXTRACT: - if self.max_staleness_seconds is None and not self.immutable_content: - raise ValueError( - f"endpoint {self.paths[0]}: every extracted route must state " - "the staleness it tolerates, or declare immutable_content" - ) - if ( - self.max_staleness_seconds is not None - and self.max_staleness_seconds < MINIMUM_PUBLIC_STALE_SECONDS - ): - raise ValueError( - f"endpoint {self.paths[0]}: staleness budget must not fall " - f"below {MINIMUM_PUBLIC_STALE_SECONDS}s" - ) - elif self.max_staleness_seconds is not None: - raise ValueError( - f"endpoint {self.paths[0]}: the staleness contract covers the " - "extracted public surface; retained routes answer for " - "themselves" - ) - - @property - def primary_path(self) -> str: - return self.paths[0] - - -# --- Operator surface ------------------------------------------------------- -# -# Process state, plus the private audit/ops listener. PRISM.md lists these as -# private/internal and instructs operators not to expose them to the internet. - -_OPERATOR: tuple[Endpoint, ...] = ( - Endpoint( - paths=("/healthz",), - audience=Audience.OPERATOR, - disposition=Disposition.RETAIN, - access=(LedgerAccess.PROCESS_STATE,), - rationale=( - "Liveness of this coordinator instance: job-build progress, tip " - "refresh, writer-lease state. Answered from an in-memory snapshot " - "refreshed by a background thread, and overlaid with live progress " - "state on every request so a cached ok=true cannot mask a failed " - "refresh. A different process cannot answer it. Wired to the " - "compose healthcheck." - ), - ), - Endpoint( - paths=("/metrics",), - audience=Audience.OPERATOR, - disposition=Disposition.RETAIN, - access=(LedgerAccess.PROCESS_STATE,), - rationale=( - "Prometheus exposition of this process's counters and gauges. " - "Serves only the complete cached snapshot and returns 503 once the " - "snapshot exceeds max(3 * refresh, 15s) -- the staleness precedent " - "the extracted service follows." - ), - ), - Endpoint( - paths=("/audit/latest",), - audience=Audience.OPERATOR, - disposition=Disposition.RETAIN, - access=(LedgerAccess.ARTIFACT_FILE,), - rationale=( - "The most recently published evidence envelope, read from the " - "audit artifact directory. Public consumers reach the same " - "evidence content-addressed via /public/v1/artifacts/{sha256}; " - "this route is the operator's view of the live envelope and is " - "listed as private/internal in PRISM.md." - ), - ), - Endpoint( - paths=("/audit/share-window",), - audience=Audience.OPERATOR, - disposition=Disposition.RETAIN, - access=(LedgerAccess.WRITER_LOCK,), - ledger_methods=("audit_share_window",), - rationale=( - "Raw per-share rows for a caller-supplied anchor. The window is " - "caller-controlled and unbounded, and the read is fenced behind " - "the writer lock, so this is an operator reconciliation tool -- " - "the public equivalent is the aggregated reward leaderboard." - ), - ), - Endpoint( - paths=("/audit/blocks/{block_hash}/payouts", "/audit/block/{block_hash}"), - audience=Audience.OPERATOR, - disposition=Disposition.RETAIN, - access=(LedgerAccess.WRITER_LOCK,), - ledger_methods=("audit_block_payouts",), - rationale=( - "Per-recipient payout rows for a settled block in raw sats, with " - "maturity_state and chain_state. The public per-miner slice is " - "/public/v1/miners/{recipient_id}/payouts. The short " - "/audit/block/{block_hash} form is a legacy alias for the same " - "handler." - ), - ), - Endpoint( - paths=("/audit/blocks/{block_hash}/bundle",), - audience=Audience.OPERATOR, - disposition=Disposition.RETAIN, - access=(LedgerAccess.WRITER_LOCK, LedgerAccess.ARTIFACT_FILE), - ledger_methods=("audit_bundle",), - rationale=( - "The full inline audit bundle plus coinbase_tx_hex for one block. " - "The bundle body itself is public, content-addressed at " - "/public/v1/artifacts/{sha256} and linked from the public " - "settlement-artifacts route; this is the internal by-block " - "lookup." - ), - ), - Endpoint( - paths=("/audit/commitments/{commitment_leaf_hex}/bundle",), - audience=Audience.OPERATOR, - disposition=Disposition.RETAIN, - access=(LedgerAccess.WRITER_LOCK, LedgerAccess.ARTIFACT_FILE), - ledger_methods=("audit_bundle_by_commitment",), - rationale=( - "The same bundle addressed by audit commitment leaf. No public " - "commitment-leaf lookup exists; a verifier holding a leaf resolves " - "the bundle sha256 from the block's public settlement artifacts " - "and fetches it content-addressed." - ), - ), - Endpoint( - paths=("/audit/blocks/{block_hash}/ctv-fanout-manifest-set",), - audience=Audience.OPERATOR, - disposition=Disposition.RETAIN, - access=(LedgerAccess.READ_SLOT,), - ledger_methods=("audit_ctv_fanout_manifest_set",), - rationale=( - "The full CTV recovery payload, including parent_coinbase_tx_hex " - "and manifest_set_json -- broadcast material, not a dashboard " - "read. /public/v1/blocks/{block_hash}/settlement-artifacts is the " - "public reshaping of the same ledger read." - ), - ), - Endpoint( - paths=("/audit/blocks/{block_hash}/ctv-fanouts",), - audience=Audience.OPERATOR, - disposition=Disposition.RETAIN, - access=(LedgerAccess.READ_SLOT,), - ledger_methods=("audit_ctv_fanouts", "audit_ctv_fanout_manifest_set"), - rationale=( - "Raw artifact dicts with fanout_tx_hex and sats fields, delegating " - "to the manifest-set read above." - ), - ), - Endpoint( - paths=("/audit/fanouts/pending",), - audience=Audience.OPERATOR, - disposition=Disposition.RETAIN, - access=(LedgerAccess.READ_SLOT,), - ledger_methods=("pending_ctv_fanout_statuses",), - rationale=( - "The broadcaster's work queue, filtered to fanouts due now, with " - "full broadcast-attempt histories. /public/v1/fanouts/pending is " - "the paginated public view built from a different read model." - ), - ), - Endpoint( - paths=("/audit/fanouts/{fanout_txid}/status",), - audience=Audience.OPERATOR, - disposition=Disposition.RETAIN, - access=(LedgerAccess.READ_SLOT,), - ledger_methods=("ctv_fanout_status",), - rationale=( - "Raw broadcast status including submit_result and attempt detail. " - "/public/v1/fanouts/{fanout_txid} wraps the same ledger read in " - "the public schema." - ), - ), -) - - -# --- Payout-affecting surface ---------------------------------------------- -# -# Live payout obligations: the state the settlement machinery acts on. Out of -# scope for extraction by the terms of issue #145. - -_PAYOUT_AFFECTING: tuple[Endpoint, ...] = ( - Endpoint( - paths=("/audit/carry-forward-integrity", "/audit/ledger-integrity"), - audience=Audience.PAYOUT_AFFECTING, - disposition=Disposition.RETAIN, - access=(LedgerAccess.WRITER_LOCK,), - ledger_methods=("carry_forward_integrity_report",), - rationale=( - "The drift count and mismatch list over active carry-forward rows, " - "plus the audit_head_sha256 operators are told to mirror after " - "payout-affecting blocks. It reports whether the payout ledger " - "disagrees with itself. It runs two statements under one held " - "writer gate, so it is also the worst route on the surface to " - "expose to uncontrolled volume." - ), - ), - Endpoint( - paths=("/owed", "/owed-balances"), - audience=Audience.PAYOUT_AFFECTING, - disposition=Disposition.RETAIN, - access=(LedgerAccess.WRITER_LOCK,), - ledger_methods=("current_owed_balances",), - rationale=( - "Every recipient's outstanding owed balance -- the obligations the " - "next settlement will pay. O(recipients) under the writer lock. A " - "miner's own balance is available per-recipient from the public " - "tier; there is no public all-recipients dump." - ), - ), - Endpoint( - paths=("/miners/{recipient_id}/status", "/payouts/{recipient_id}/status"), - audience=Audience.PAYOUT_AFFECTING, - disposition=Disposition.RETAIN, - access=(LedgerAccess.WRITER_LOCK,), - ledger_methods=("current_owed_balances", "recipient_payout_history"), - rationale=( - "The pre-dashboard payout-operations view: owed balances plus " - "payout history in raw sats. Both aliases reach one handler. " - "Superseded for public consumption by " - "/public/v1/miners/{recipient_id}, which reports bits." - ), - ), -) - - -# --- Public read surface ---------------------------------------------------- -# -# The documented, versioned dashboard contract. Exactly this moves. - -_PUBLIC_READ: tuple[Endpoint, ...] = ( - Endpoint( - paths=("/public/v1/pool-summary",), - audience=Audience.PUBLIC_READ, - disposition=Disposition.EXTRACT, - access=(LedgerAccess.READ_SLOT, LedgerAccess.QBIT_RPC), - ledger_methods=("dashboard_pool_snapshot",), - rationale=( - "Pool-wide headline figures. Needs a qbit RPC client for the " - "network summary, which the extracted service gets its own copy " - "of -- it is a node read, not coordinator state." - ), - # Pool-wide aggregate: the 30s aggregate TTL, nothing beneath it. - max_staleness_seconds=staleness_budget_seconds( - cache_ttl_seconds=PUBLIC_AGGREGATE_CACHE_TTL_DEFAULT_SECONDS, - ), - ), - Endpoint( - paths=("/public/v1/blocks",), - audience=Audience.PUBLIC_READ, - disposition=Disposition.EXTRACT, - access=(LedgerAccess.READ_SLOT,), - ledger_methods=("dashboard_blocks",), - rationale="Paginated found-block history.", - max_staleness_seconds=staleness_budget_seconds( - cache_ttl_seconds=PUBLIC_CACHE_TTL_DEFAULT_SECONDS, - ), - ), - Endpoint( - paths=("/public/v1/leaderboard",), - audience=Audience.PUBLIC_READ, - disposition=Disposition.EXTRACT, - access=(LedgerAccess.READ_SLOT, LedgerAccess.QBIT_RPC), - ledger_methods=("dashboard_leaderboard", "dashboard_reward_leaderboard"), - rationale=( - "Both windows. window=3h is a plain read-slot aggregate; " - "window=reward additionally needs the network difficulty from RPC " - "to size the reward window." - ), - # Both windows read the ledger directly. dashboard_reward_leaderboard - # runs its own qbit_prism_window scan rather than going through the - # cached pool aggregate, so nothing stacks under the response TTL. - max_staleness_seconds=staleness_budget_seconds( - cache_ttl_seconds=PUBLIC_CACHE_TTL_DEFAULT_SECONDS, - ), - ), - Endpoint( - paths=("/public/v1/hashrate-series",), - audience=Audience.PUBLIC_READ, - disposition=Disposition.EXTRACT, - access=(LedgerAccess.READ_SLOT,), - ledger_methods=("dashboard_hashrate_series",), - rationale=( - "Bucketed hashrate history for the pool or one miner. The widest " - "ledger scan on the public surface and the strongest single " - "argument for moving this traffic off the primary." - ), - max_staleness_seconds=staleness_budget_seconds( - cache_ttl_seconds=PUBLIC_AGGREGATE_CACHE_TTL_DEFAULT_SECONDS, - ), - ), - Endpoint( - paths=("/public/v1/block-markers",), - audience=Audience.PUBLIC_READ, - disposition=Disposition.EXTRACT, - access=(LedgerAccess.READ_SLOT,), - ledger_methods=("dashboard_block_markers",), - rationale=( - "Found-block markers pre-aggregated onto the hashrate chart's " - "bucket grid. The pool finds hundreds of blocks a day, so the " - "dashboard cannot place markers on long timeframes by paging " - "/public/v1/blocks; this groups the small qbit_pool_blocks table " - "per bucket instead -- a cheap read-slot scan, so it carries the " - "plain row-read TTL rather than the aggregate one." - ), - max_staleness_seconds=staleness_budget_seconds( - cache_ttl_seconds=PUBLIC_CACHE_TTL_DEFAULT_SECONDS, - ), - ), - Endpoint( - paths=("/public/v1/mining-configuration",), - audience=Audience.PUBLIC_READ, - disposition=Disposition.EXTRACT, - access=(LedgerAccess.NONE,), - rationale=( - "Stratum endpoints and pool fee. Touches no ledger at all -- it " - "is assembled from environment configuration." - ), - extraction_blocker=( - "Falls back to getattr(coordinator, 'port') and " - "getattr(coordinator, 'bind') when the public URL env vars are " - "unset. The extracted service has no coordinator, so " - "PRISM_PUBLIC_STRATUM_URL / PRISM_STRATUM_PORT must be supplied " - "explicitly rather than inferred." - ), - # Environment-derived and near-static, so it carries the longest - # response TTL on the surface and, from it, the longest budget. - max_staleness_seconds=staleness_budget_seconds( - cache_ttl_seconds=PUBLIC_CONFIG_CACHE_TTL_DEFAULT_SECONDS, - ), - ), - Endpoint( - paths=("/public/v1/miners/{recipient_id}",), - audience=Audience.PUBLIC_READ, - disposition=Disposition.EXTRACT, - access=( - LedgerAccess.WRITER_LOCK, - LedgerAccess.READ_SLOT, - LedgerAccess.QBIT_RPC, - ), - ledger_methods=( - "current_owed_balances", - "dashboard_miner_share_summary", - "dashboard_miner_reward_window", - "dashboard_miner_payout_rows", - "dashboard_miner_worker_rows", - "dashboard_miner_lifetime_earnings_bits", - "dashboard_miner_pending_maturity_bits", - "dashboard_blocks", - ), - rationale=( - "A miner's own dashboard page. The most-polled route on the " - "surface and the clearest public read." - ), - extraction_blocker=( - "owed_balance_for_recipient() calls current_owed_balances(), a " - "writer-lock read that returns every recipient's balance and then " - "filters in Python. So the single most-polled public route takes " - "the writer lock today. The extracted service needs a " - "per-recipient owed-balance read that does not fence." - ), - # The one route with a second cache beneath the response cache: its - # reward-window slice comes from the shared pool aggregate, so the two - # TTLs compound. - max_staleness_seconds=staleness_budget_seconds( - cache_ttl_seconds=PUBLIC_CACHE_TTL_DEFAULT_SECONDS, - underlying_cache_seconds=POOL_REWARD_WINDOW_CACHE_DEFAULT_SECONDS, - ), - ), - Endpoint( - paths=("/public/v1/miners/{recipient_id}/earnings",), - audience=Audience.PUBLIC_READ, - disposition=Disposition.EXTRACT, - access=(LedgerAccess.READ_SLOT,), - ledger_methods=("dashboard_miner_earning_rows",), - rationale="Per-block earnings history for one miner.", - max_staleness_seconds=staleness_budget_seconds( - cache_ttl_seconds=PUBLIC_CACHE_TTL_DEFAULT_SECONDS, - ), - ), - Endpoint( - paths=("/public/v1/miners/{recipient_id}/payouts",), - audience=Audience.PUBLIC_READ, - disposition=Disposition.EXTRACT, - access=(LedgerAccess.READ_SLOT,), - ledger_methods=("dashboard_miner_payout_rows",), - rationale="Per-payout history for one miner.", - max_staleness_seconds=staleness_budget_seconds( - cache_ttl_seconds=PUBLIC_CACHE_TTL_DEFAULT_SECONDS, - ), - ), - Endpoint( - paths=("/public/v1/miners/{recipient_id}/workers",), - audience=Audience.PUBLIC_READ, - disposition=Disposition.EXTRACT, - access=(LedgerAccess.READ_SLOT,), - ledger_methods=("dashboard_miner_worker_rows",), - rationale="Worker list for one miner.", - # Classified as a pool-wide aggregate by public_cache_policy: it - # aggregates a miner's whole recent share history. - max_staleness_seconds=staleness_budget_seconds( - cache_ttl_seconds=PUBLIC_AGGREGATE_CACHE_TTL_DEFAULT_SECONDS, - ), - ), - Endpoint( - paths=("/public/v1/blocks/{block_hash}/settlement-artifacts",), - audience=Audience.PUBLIC_READ, - disposition=Disposition.EXTRACT, - access=( - LedgerAccess.WRITER_LOCK, - LedgerAccess.READ_SLOT, - LedgerAccess.ARTIFACT_FILE, - ), - ledger_methods=( - "audit_ctv_fanout_manifest_set", - "audit_bundle", - "ctv_fanout_status", - "dashboard_public_artifact_exists", - ), - rationale=( - "The public settlement view of a block, linking to the artifacts " - "that prove it. This is how published evidence reaches the public " - "without exposing /audit/*." - ), - extraction_blocker=( - "The CTV path is read-slot, but a direct-coinbase block falls " - "through to direct_coinbase_settlement_payload(), which calls " - "audit_bundle() under the writer lock. The extracted service must " - "take the direct-coinbase branch from the artifact file." - ), - max_staleness_seconds=staleness_budget_seconds( - cache_ttl_seconds=PUBLIC_CACHE_TTL_DEFAULT_SECONDS, - ), - ), - Endpoint( - paths=("/public/v1/fanouts/pending",), - audience=Audience.PUBLIC_READ, - disposition=Disposition.EXTRACT, - access=(LedgerAccess.READ_SLOT,), - ledger_methods=("dashboard_pending_fanout_rows",), - rationale="Public paginated view of fanouts awaiting broadcast.", - max_staleness_seconds=staleness_budget_seconds( - cache_ttl_seconds=PUBLIC_CACHE_TTL_DEFAULT_SECONDS, - ), - ), - Endpoint( - paths=("/public/v1/fanouts/{fanout_txid}",), - audience=Audience.PUBLIC_READ, - disposition=Disposition.EXTRACT, - access=(LedgerAccess.READ_SLOT,), - ledger_methods=("ctv_fanout_status",), - rationale="Public view of one fanout's broadcast state.", - max_staleness_seconds=staleness_budget_seconds( - cache_ttl_seconds=PUBLIC_CACHE_TTL_DEFAULT_SECONDS, - ), - ), - Endpoint( - paths=("/public/v1/artifacts/{sha256}",), - audience=Audience.PUBLIC_READ, - disposition=Disposition.EXTRACT, - access=(LedgerAccess.READ_SLOT, LedgerAccess.ARTIFACT_FILE), - ledger_methods=("dashboard_public_artifact",), - rationale=( - "Content-addressed fetch of a published artifact -- the sanctioned " - "public path to audit evidence, and the reason the public API can " - "keep /audit/* internal. Already resolves the body from body_uri " - "on disk when the inline column is NULL, verified against the " - "requested sha256. Its PRISM_PUBLIC_ARTIFACT_CACHE_* knobs move " - "with the extracted service. Content-addressed and immutable: a " - "body that hashes to the requested sha256 is correct at any age, " - "so this route carries no staleness budget and must never refuse " - "for staleness." - ), - immutable_content=True, - ), -) - - -ENDPOINTS: tuple[Endpoint, ...] = _OPERATOR + _PAYOUT_AFFECTING + _PUBLIC_READ - - -def endpoints_by_audience(audience: Audience) -> tuple[Endpoint, ...]: - return tuple(item for item in ENDPOINTS if item.audience is audience) - - -def endpoints_by_disposition(disposition: Disposition) -> tuple[Endpoint, ...]: - return tuple(item for item in ENDPOINTS if item.disposition is disposition) - - -def extracted_paths() -> tuple[str, ...]: - """Every request path the extracted public service must answer.""" - - return tuple( - path - for item in endpoints_by_disposition(Disposition.EXTRACT) - for path in item.paths - ) - - -def retained_paths() -> tuple[str, ...]: - """Every request path the coordinator keeps.""" - - return tuple( - path - for item in endpoints_by_disposition(Disposition.RETAIN) - for path in item.paths - ) - - -def extraction_blockers() -> tuple[tuple[str, str], ...]: - """(path, blocker) for each route whose implementation must change first.""" - - return tuple( - (item.primary_path, item.extraction_blocker) - for item in ENDPOINTS - if item.extraction_blocker is not None - ) - - -def _template_matches(template: str, path: str) -> bool: - """True when a concrete request path fills one registry path template.""" - - template_parts = template.split("/") - path_parts = path.split("/") - if len(template_parts) != len(path_parts): - return False - for expected, actual in zip(template_parts, path_parts): - if expected.startswith("{") and expected.endswith("}"): - if not actual: - return False - continue - if expected != actual: - return False - return True - - -def endpoint_for_request_path(path: str) -> Endpoint | None: - """Classify one concrete request path, or None when it is not a route. - - Literal paths win over templates so /public/v1/fanouts/pending resolves to - the pending list rather than to /public/v1/fanouts/{fanout_txid}, which is - the same precedence dispatch() applies. - """ - - for endpoint in ENDPOINTS: - if path in endpoint.paths: - return endpoint - for endpoint in ENDPOINTS: - for template in endpoint.paths: - if "{" in template and _template_matches(template, path): - return endpoint - return None - - -def writer_lock_paths() -> tuple[str, ...]: - """Paths that currently serialize against the lease-holding writer.""" - - return tuple( - path - for item in ENDPOINTS - if LedgerAccess.WRITER_LOCK in item.access - for path in item.paths - ) - - -__all__ = [ - "Audience", - "Disposition", - "ENDPOINTS", - "Endpoint", - "LedgerAccess", - "MINIMUM_PUBLIC_STALE_SECONDS", - "POOL_REWARD_WINDOW_CACHE_DEFAULT_SECONDS", - "PUBLIC_AGGREGATE_CACHE_TTL_DEFAULT_SECONDS", - "PUBLIC_CACHE_TTL_DEFAULT_SECONDS", - "PUBLIC_CONFIG_CACHE_TTL_DEFAULT_SECONDS", - "endpoint_for_request_path", - "endpoints_by_audience", - "endpoints_by_disposition", - "extracted_paths", - "extraction_blockers", - "retained_paths", - "staleness_budget_seconds", - "writer_lock_paths", -] diff --git a/lab/prism/future_callbacks.py b/lab/prism/future_callbacks.py deleted file mode 100644 index 24ef6ff54..000000000 --- a/lab/prism/future_callbacks.py +++ /dev/null @@ -1,31 +0,0 @@ -"""Completion callbacks whose captured work is released after invocation.""" - -from __future__ import annotations - -from concurrent.futures import Future -from typing import Callable, TypeVar - - -_Result = TypeVar("_Result") - - -def add_releasing_done_callback( - future: Future[_Result], callback: Callable[[Future[_Result]], object] -) -> None: - """Keep a callback alive until completion, then release its captures. - - Future retains registered callbacks after invoking them. A callback that - captures a request holding that future therefore retains the request and - result until cyclic GC, even when all external owners release them. - Leave only this empty wrapper registered once the callback runs, including - when it raises. Future still owns callback ordering and error handling. - """ - pending: Callable[[Future[_Result]], object] | None = callback - - def finish(completed: Future[_Result]) -> None: - nonlocal pending - completing, pending = pending, None - if completing is not None: - completing(completed) - - future.add_done_callback(finish) diff --git a/lab/prism/job_build_benchmark.py b/lab/prism/job_build_benchmark.py deleted file mode 100644 index 880665f71..000000000 --- a/lab/prism/job_build_benchmark.py +++ /dev/null @@ -1,546 +0,0 @@ -#!/usr/bin/env python3 -"""PRISM coordinator job-build latency benchmark. - -Runs an in-process coordinator against a synthetic ledger/RPC whose latencies -are injected to match measurements from a live host, subscribes N clients, -and replays tip changes the way the blockpoll does. It reports the -distribution of the same per-client elapsed the coordinator logs as -"sent job ... elapsed=". - -Two modes on the same code: - --mode cached per-template job cache enabled (the fix) - --mode uncached all cache TTLs forced to 0, reproducing the legacy - build-everything-per-client-per-rebuild behavior - -The audit bundle builder can be the real Rust subprocess (--real-builder, -honoring PRISM_TOOL_BIN_DIR / PRISM_CARGO and using test signing seeds) fed a -share window of --shares synthetic shares, or a stand-in that sleeps for the -measured builder latency (--fake-builder-delay). - -The benchmark never touches a real ledger database and never mutates chain -state: the only external process it may run is the (pure computation) bundle -builder. -""" - -from __future__ import annotations - -import argparse -import contextlib -import hashlib -import json -import os -import socket -import sys -import threading -import time -from dataclasses import dataclass -from decimal import Decimal -from pathlib import Path - -if not __package__: - sys.path.insert(0, str(Path(__file__).resolve().parents[2])) - -from lab.auxpow import vardiff -from lab.prism import direct_stratum -from lab.prism.coordinator_config import default_prism_coinbase_tag_hex -from lab.prism.prism_coordinator import ( - ClientState, - PRISM_REJECTION_REASON_IDS, - PrismCoordinator, - WorkerIdentity, -) - -EXTRANONCE2_SIZE = 8 - - -def benchmark_miner_programs(count: int) -> tuple[str, ...]: - return tuple( - hashlib.sha256(f"bench-miner-{index}".encode()).hexdigest() - for index in range(count) - ) - - -def sanitize_environment() -> None: - """Make runs reproducible regardless of ambient container env: drop all - PRISM_*/QBIT_* configuration except the tool binary resolution knobs.""" - keep = {"PRISM_TOOL_BIN_DIR", "PRISM_CARGO", "CARGO"} - for name in list(os.environ): - if (name.startswith("PRISM_") or name.startswith("QBIT_")) and name not in keep: - del os.environ[name] - - -@dataclass -class InjectedLatencies: - gbt_seconds: float - snapshot_seconds: float - stats_seconds: float - balances_seconds: float - reorg_watch_seconds: float - mark_mature_seconds: float - rpc_small_seconds: float - - -class BenchLedger: - backend_name = "bench" - - def __init__( - self, - share_count: int, - miner_count: int, - latencies: InjectedLatencies, - ) -> None: - self.latencies = latencies - self.miner_programs = benchmark_miner_programs(miner_count) - now = int(time.time() * 1000) - self.shares_json = [ - { - "share_seq": index + 1, - "share_id": f"bench-share-{index + 1}", - "miner_id": f"bench-miner-{index % miner_count}", - "order_key": f"bench-miner-{index % miner_count:06d}", - "p2mr_program_hex": self.miner_programs[index % miner_count], - "share_difficulty": 16384, - "network_difficulty": 226646186, - "template_height": 9, - "job_id": f"bench-job-{index + 1}", - "job_issued_at_ms": now - (share_count - index) * 1_000, - "accepted_at_ms": now - (share_count - index) * 1_000, - "ntime": int(time.time()) - (share_count - index), - } - for index in range(share_count) - ] - - class _Record: - __slots__ = ("payload",) - - def __init__(self, payload: dict[str, object]) -> None: - self.payload = payload - - def to_prism_json(self) -> dict[str, object]: - return self.payload - - def accepted_share_stats(self) -> dict[str, int]: - time.sleep(self.latencies.stats_seconds) - return { - "accepted_share_count": len(self.shares_json), - "distinct_miner_count": len(self.miner_programs), - } - - def all_shares(self) -> list[object]: - raise AssertionError("benchmark expects accepted_share_stats to be used") - - def snapshot_at_job_issue(self, anchor_job_issued_at_ms: int, *, window_weight: int | None = None) -> list[object]: - time.sleep(self.latencies.snapshot_seconds) - return [self._Record(payload) for payload in self.shares_json] - - def current_prior_balances(self) -> list[dict[str, object]]: - time.sleep(self.latencies.balances_seconds) - return [] - - def reorg_watch_blocks(self, *, active_tip_height: int) -> list[dict[str, object]]: - time.sleep(self.latencies.reorg_watch_seconds) - return [] - - def mark_mature_pool_payouts(self, *, active_tip_height: int) -> dict[str, int]: - time.sleep(self.latencies.mark_mature_seconds) - return {"matured_count": 0} - - def metrics(self) -> dict[str, int]: - return {"blocks": 0, "owed_accounts": 0} - - -class BenchRpc: - def __init__(self, template: dict[str, object], latencies: InjectedLatencies) -> None: - self.template = template - self.latencies = latencies - self.calls: dict[str, int] = {} - self.lock = threading.Lock() - - def call(self, method: str, params: list[object] | None = None) -> object: - with self.lock: - self.calls[method] = self.calls.get(method, 0) + 1 - if method == "getblocktemplate": - time.sleep(self.latencies.gbt_seconds) - with self.lock: - return dict(self.template) - time.sleep(self.latencies.rpc_small_seconds) - if method == "getbestblockhash": - with self.lock: - return str(self.template["previousblockhash"]) - if method == "getblockchaininfo": - with self.lock: - height = int(self.template["height"]) - 1 - return {"initialblockdownload": False, "blocks": height, "headers": height} - if method == "getblockcount": - with self.lock: - return int(self.template["height"]) - 1 - if method == "getblockhash": - return "00" * 32 - raise AssertionError(f"unexpected RPC method {method}") - - def advance_tip(self, new_tip_hex: str) -> None: - with self.lock: - template = dict(self.template) - template["previousblockhash"] = new_tip_hex - template["height"] = int(template["height"]) + 1 - template["curtime"] = int(time.time()) - self.template = template - - -def synthetic_manifest_coinbase_hex(suffix_hex: str) -> str: - height_push = "03aabbcc" - script_sig = height_push + suffix_hex - output = (50_00000000).to_bytes(8, "little").hex() + "0151" - return ( - "01000000" - + "01" - + "00" * 32 - + "ffffffff" - + direct_stratum.compact_size(len(bytes.fromhex(script_sig))).hex() - + script_sig - + "ffffffff" - + "01" - + output - + "00000000" - ) - - -def base_template(height: int) -> dict[str, object]: - return { - "height": height, - "previousblockhash": "11" * 32, - "bits": "1b00ffff", - "version": 0x20000000, - "curtime": int(time.time()), - "coinbasevalue": 50_00000000, - "transactions": [], - } - - -def build_coordinator(args: argparse.Namespace, latencies: InjectedLatencies) -> tuple[PrismCoordinator, BenchRpc, BenchLedger]: - server = PrismCoordinator.__new__(PrismCoordinator) - template = base_template(height=1000) - rpc = BenchRpc(template, latencies) - ledger = BenchLedger(args.shares, args.miners, latencies) - server.rpc = rpc - server.ledger = ledger - server.qbit_chain = "testnet4" - server.lock = threading.RLock() - server.stop_event = threading.Event() - server.clients = set() - server.jobs = {} - server.job_counter = 0 - server.connection_counter = 0 - server.accepted_block_count = 0 - server.max_blocks = 1_000_000_000 - server.started_monotonic = time.monotonic() - server.submitted_share_count = 0 - server.stale_share_count = 0 - server.duplicate_share_count = 0 - server.low_difficulty_share_count = 0 - server.rejection_counts_by_reason = {reason: 0 for reason in PRISM_REJECTION_REASON_IDS} - server.job_build_failure_count = 0 - server.tip_refresh_job_count = 0 - server.post_accept_refresh_failure_count = 0 - server.reorg_reconciler_enabled = True - server.reorg_inactive_block_count = 0 - server.reorg_reactivated_block_count = 0 - server.reorg_reconcile_skip_count = 0 - server.reorg_reconcile_error_count = 0 - server.matured_payout_count = 0 - server.last_reorg_reconciled_tip_hash = None - server.last_reorg_reconciled_trusted = False - server.last_reorg_reconciled_monotonic = None - server.latest_evidence = None - server.latest_coinbase_size_bytes = None - server.tip_template_snapshot = None - server.extranonce2_size = EXTRANONCE2_SIZE - server.coinbase_tag_hex = default_prism_coinbase_tag_hex() - server.share_difficulty = Decimal("1") - server.vardiff_config = vardiff.VardiffConfig( - enabled=True, - target_share_interval_seconds=Decimal("15"), - min_difficulty=Decimal("0.000000001"), - max_difficulty=Decimal("1024"), - retarget_interval_seconds=Decimal("90"), - max_step_factor=Decimal("4"), - startup_difficulty=Decimal("1"), - max_step_down_factor=Decimal("4"), - ewma_alpha=Decimal("0.4"), - retarget_tolerance=Decimal("0.25"), - ) - server.default_share_weight = 1 - server.share_weights_by_username = {} - server.min_ready_miners = 1 - server.blockpoll_seconds = 2.0 - server.signing_seed_hex = "42" * 32 - server.ledger_attestation_signing_seed_hex = "43" * 32 - server._prism_payout_policy_cache = None - server._ctv_fanout_market_fee_rate_cache = {} - if args.mode == "cached": - server.job_bundle_cache_seconds = args.bundle_ttl - server.template_cache_seconds = args.template_ttl - server.reorg_reconcile_cache_seconds = args.reorg_ttl - else: - server.job_bundle_cache_seconds = 0.0 - server.template_cache_seconds = 0.0 - server.reorg_reconcile_cache_seconds = 0.0 - server.health_refresh_seconds = 5.0 - server.stratum_send_timeout_seconds = 20.0 - server._ensure_job_cache_state() - server._ensure_watchdog_state() - server.watchdog_timeout_seconds = 3600.0 - - if not args.real_builder: - delay = args.fake_builder_delay - - def fake_build_audit_bundle(**kwargs: object) -> dict[str, object]: - time.sleep(delay) - suffix_hex = str(kwargs["coinbase_script_sig_suffix_hex"]) - return { - "found_block": dict(kwargs["found_block"]), - "signed_coinbase_manifest": { - "manifest": {"coinbase_tx_hex": synthetic_manifest_coinbase_hex(suffix_hex)} - }, - } - - server.build_audit_bundle = fake_build_audit_bundle # type: ignore[method-assign] - return server, rpc, ledger - - -class BenchClientPool: - def __init__( - self, - server: PrismCoordinator, - count: int, - miner_programs: tuple[str, ...], - ) -> None: - self.server = server - self.clients: list[ClientState] = [] - self.drains: list[threading.Thread] = [] - self.peer_sockets: list[socket.socket] = [] - for index in range(count): - left, right = socket.socketpair() - drain = threading.Thread(target=self._drain, args=(right,), daemon=True) - drain.start() - miner_index = index % len(miner_programs) - program = miner_programs[miner_index] - worker = WorkerIdentity( - username=f"bench-miner-{miner_index}.rig{index}", - payout_address=f"bench-miner-{miner_index}", - worker_name=f"rig{index}", - script_pubkey_hex="5220" + program, - p2mr_program_hex=program, - ) - state = ClientState( - sock=left, - address=("127.0.0.1", 50_000 + index), - connection_id=index + 1, - extranonce1_hex=f"{index + 1:08x}", - ) - state.subscribed = True - state.authorized = True - state.username = worker.username - state.worker = worker - # Spread desired difficulties so stamping is exercised per client. - state.share_difficulty = Decimal(2 ** (index % 6)) - self.clients.append(state) - self.peer_sockets.append(right) - with server.lock: - server.clients.add(state) - - @staticmethod - def _drain(sock: socket.socket) -> None: - try: - while sock.recv(1 << 16): - pass - except OSError: - pass - - def close(self) -> None: - for state in self.clients: - try: - state.close() - except OSError: - pass - for sock in self.peer_sockets: - try: - sock.close() - except OSError: - pass - - -def percentile(sorted_values: list[float], fraction: float) -> float: - if not sorted_values: - return float("nan") - index = min(len(sorted_values) - 1, max(0, int(round(fraction * (len(sorted_values) - 1))))) - return sorted_values[index] - - -def summarize(samples: list[float]) -> dict[str, object]: - ordered = sorted(samples) - return { - "n": len(ordered), - "p50": round(percentile(ordered, 0.50), 4), - "p90": round(percentile(ordered, 0.90), 4), - "p99": round(percentile(ordered, 0.99), 4), - "max": round(ordered[-1], 4) if ordered else float("nan"), - } - - -def summarize_histogram(histogram: dict[str, object]) -> dict[str, object]: - count = int(histogram["count"]) - total = float(histogram["sum"]) - return { - "n": count, - "average": round(total / count, 4) if count else float("nan"), - } - - -def main() -> int: - parser = argparse.ArgumentParser(description=__doc__) - parser.add_argument("--mode", choices=("cached", "uncached"), default="cached") - parser.add_argument("--clients", type=int, default=50) - parser.add_argument("--tip-flips", type=int, default=10) - parser.add_argument("--shares", type=int, default=21_868, help="synthetic share window size") - parser.add_argument("--miners", type=int, default=2, help="distinct payout identities") - parser.add_argument( - "--ctv-settlement", - action="store_true", - help="route eligible payouts through a realistic CTV fanout manifest", - ) - parser.add_argument("--real-builder", action="store_true", help="run the real qbit-prism bundle builder subprocess") - parser.add_argument( - "--precompute-payout-artifact", - action="store_true", - help="prepare the immutable ledger artifact before timed tip refreshes", - ) - parser.add_argument("--fake-builder-delay", type=float, default=1.0, help="simulated bundle builder seconds when not using --real-builder") - parser.add_argument("--gbt-latency", type=float, default=0.017) - parser.add_argument("--snapshot-latency", type=float, default=0.44) - parser.add_argument("--stats-latency", type=float, default=0.02) - parser.add_argument("--balances-latency", type=float, default=0.065) - parser.add_argument("--reorg-watch-latency", type=float, default=0.057) - parser.add_argument("--mark-mature-latency", type=float, default=0.07) - parser.add_argument("--rpc-small-latency", type=float, default=0.002) - parser.add_argument("--bundle-ttl", type=float, default=10.0) - parser.add_argument("--template-ttl", type=float, default=2.0) - parser.add_argument("--reorg-ttl", type=float, default=5.0) - parser.add_argument("--output-json", type=str, default="") - args = parser.parse_args() - if args.miners <= 0: - parser.error("--miners must be positive") - - sanitize_environment() - if args.ctv_settlement: - os.environ["PRISM_CTV_SETTLEMENT_ENABLED"] = "1" - os.environ["PRISM_DIRECT_COINBASE_PAYOUT_FLOOR_BITS"] = str(50_00000000) - os.environ[ - "PRISM_CTV_FANOUT_FEE_MARKET_RATE_BITS_PER_1000_WEIGHT" - ] = "1" - latencies = InjectedLatencies( - gbt_seconds=args.gbt_latency, - snapshot_seconds=args.snapshot_latency, - stats_seconds=args.stats_latency, - balances_seconds=args.balances_latency, - reorg_watch_seconds=args.reorg_watch_latency, - mark_mature_seconds=args.mark_mature_latency, - rpc_small_seconds=args.rpc_small_latency, - ) - server, rpc, ledger = build_coordinator(args, latencies) - if args.precompute_payout_artifact: - artifacts = server.store_template_artifacts(dict(rpc.template)) - if artifacts is None: - raise RuntimeError("benchmark template artifact preparation failed") - # Model a running coordinator whose current tip and payout generation - # are already published before the timed refresh arrives. - server.observe_tip_first_seen( - str(rpc.template["previousblockhash"]), - observation_sequence=1, - ) - if not server.ensure_reorg_reconciled_for_tip(rpc.template["previousblockhash"]): - raise RuntimeError("benchmark payout reconciliation failed") - server._prepare_payout_ledger_artifact( - server._ensure_payout_state_service().snapshot().generation, - artifacts.network_difficulty, - ) - # The benchmark's timed path models an already-published artifact, not - # overlap with the speculative publisher worker that prepared it. - server.shutdown_payout_artifact_executor() - - samples: list[float] = [] - phase_totals: dict[str, float] = {} - original_observe = server.observe_job_build_elapsed - - def observing(elapsed_seconds: float, phases: dict[str, float]) -> None: - samples.append(elapsed_seconds) - for phase, duration in phases.items(): - phase_totals[phase] = phase_totals.get(phase, 0.0) + duration - original_observe(elapsed_seconds, phases) - - server.observe_job_build_elapsed = observing # type: ignore[method-assign] - - pool = BenchClientPool(server, args.clients, ledger.miner_programs) - flip_wall_times: list[float] = [] - try: - # Coordinator job logs go to stderr so stdout stays a clean JSON report. - with contextlib.redirect_stdout(sys.stderr): - # Initial poll acts as the join storm: every client needs a first job. - for flip in range(args.tip_flips + 1): - if flip > 0: - rpc.advance_tip(f"{flip:064x}") - started = time.monotonic() - refreshed = server.poll_qbit_tip_template_once() - flip_wall_times.append(time.monotonic() - started) - if refreshed != args.clients: - print( - f"benchmark: warning flip={flip} refreshed {refreshed} of {args.clients} clients", - file=sys.stderr, - ) - finally: - pool.close() - - # Read the delivery/cache metrics from their owner services instead of - # coordinator-private fields; the coordinator facade only routes here. - tip_refresh_service = server._ensure_tip_refresh_service() - job_bundle_service = server._ensure_job_bundle_service() - report = { - "schema": "qbit.prism.job-build-benchmark.v1", - "mode": args.mode, - "clients": args.clients, - "tip_flips": args.tip_flips, - "share_window": args.shares, - "miners": args.miners, - "ctv_settlement": bool(args.ctv_settlement), - "real_builder": bool(args.real_builder), - "precomputed_payout_artifact": bool(args.precompute_payout_artifact), - "injected_latencies": latencies.__dict__, - "sent_job_elapsed_seconds": summarize(samples), - "full_refresh_wall_seconds": summarize(flip_wall_times), - "tip_to_first_delivery_seconds": summarize_histogram( - tip_refresh_service.tip_refresh_histograms["first_delivery"] - ), - "tip_to_last_delivery_seconds": summarize_histogram( - tip_refresh_service.tip_refresh_histograms["last_delivery"] - ), - "phase_totals_seconds": {k: round(v, 3) for k, v in sorted(phase_totals.items())}, - "bundle_phase_totals_seconds": { - phase: round(float(histogram["sum"]), 4) - for phase, histogram in ( - tip_refresh_service.tip_refresh_build_phase_histograms.items() - ) - }, - "builder_ipc_bytes": dict(tip_refresh_service.tip_refresh_ipc_bytes), - "cache_hits": dict(job_bundle_service.job_cache_hit_counts), - "cache_misses": dict(job_bundle_service.job_cache_miss_counts), - "getblocktemplate_calls": rpc.calls.get("getblocktemplate", 0), - } - text = json.dumps(report, indent=2) - print(text) - if args.output_json: - Path(args.output_json).write_text(text + "\n", encoding="utf-8") - return 0 - - -if __name__ == "__main__": - raise SystemExit(main()) diff --git a/lab/prism/job_bundle.py b/lab/prism/job_bundle.py deleted file mode 100644 index 7471acf7a..000000000 --- a/lab/prism/job_bundle.py +++ /dev/null @@ -1,3628 +0,0 @@ -#!/usr/bin/env python3 -"""Immutable PRISM job bundles, their cache, and the bounded build scheduler. - -The service owns the shared job-bundle cache, the bounded latest-wins build -scheduler (active/retiring/pending flights, priority and routine admission, -promise cleanup, orphan sweep), the per-generation issued-at anchors, the -share-window serialization cache slot, and the template-artifact repository. - -It never imports ``prism_coordinator``. Every cross-domain fact -- payout -state, tip-refresh authority, ledger snapshots, refresh metrics, live -configuration attributes -- is reached through the :class:`JobBundleRuntime` -typed port, resolved at call time so the historical coordinator monkeypatch -seams (including the instance-level facade patches used by the current test -suite) keep intercepting exactly as before the extraction. -""" - -from __future__ import annotations - -from collections import OrderedDict -from concurrent.futures import ( - FIRST_COMPLETED, - CancelledError, - Future, - InvalidStateError, - ThreadPoolExecutor, - wait, -) -from dataclasses import dataclass, field, replace as dataclass_replace -from decimal import Context, getcontext, localcontext -import hashlib -import json -import subprocess -import threading -import time -from types import FrameType -import weakref -from typing import Any, Callable, Protocol - -from lab.prism import direct_stratum -from lab.prism.bundle_compiler import _ShareWindowSerialization -from lab.prism.future_callbacks import add_releasing_done_callback -from lab.prism.share_json_stream import share_array_json_view -from lab.prism.share_ledger import DaemonShareJsonSequence -from lab.prism.coordinator_config import ( - DEFAULT_PRISM_JOB_BUILD_CANCEL_GRACE_SECONDS, - DEFAULT_PRISM_JOB_BUILD_EXECUTOR_WORKERS, - DEFAULT_PRISM_JOB_BUILD_TIMEOUT_SECONDS, - DEFAULT_PRISM_JOB_BUNDLE_CACHE_SECONDS, - DEFAULT_PRISM_ROUTINE_ADMISSION_DEADLINE_SECONDS, -) -from lab.prism.template_artifacts import ( - CachedTemplateArtifacts, - PayoutStatePublicationBlocked, - QbitTipTemplateSnapshot, - TemplateArtifactRepository, - TemplateRefreshBlocked, - TemplateRefreshSuperseded, -) - - -# Owner-local copies of the still-coordinator-visible constants; these are -# compatibility duplicates by design, not missed removals. -MAX_PRISM_JOB_BUNDLE_CACHE_ENTRIES = 128 -PRISM_JOB_EXTRANONCE1_PLACEHOLDER_HEX = "00000000" -PRISM_REWARD_WINDOW_MULTIPLIER = 8 -PRISM_SNAPSHOT_WINDOW_MARGIN = 2 -PRISM_JOB_BUILD_SECONDS_BUCKETS = ( - 0.01, - 0.025, - 0.05, - 0.1, - 0.25, - 0.5, - 1.0, - 2.5, - 5.0, - 10.0, - 30.0, -) -PRISM_JOB_BUILD_PHASES = ( - "reorg", - "template", - "merkle", - "ledger", - "payout_artifact", - "payout", - "ctv", - "input_serialization", - "worker", - "output_serialization", - "assembly", - "bundle", - "preparation_wait", - "executor_queue", - "client_lock", - "payout_gate", - "stamp", - "socket_send", - "send", -) -PRISM_JOB_CACHE_KINDS = ("template", "bundle") -# A flight whose executor future is finished normally leaves its slot inside -# the future's done callback. The admission sweep only treats such a flight -# as orphaned after this grace, so a callback that is merely between future -# completion and slot cleanup is never mistaken for a dead one. -PRISM_JOB_BUILD_ORPHAN_SWEEP_GRACE_SECONDS = 1.0 -# Cancellation-check slice while an initial request rides a subscribed -# publication-priority build promise. Promise completion wakes the waiter -# immediately; this only bounds how stale a cancellation can go unnoticed. -PRISM_INITIAL_JOB_SUBSCRIBE_POLL_SECONDS = 0.25 - - -def now_ms() -> int: - return int(time.time() * 1000) - - -def canonical_json_text(value: object) -> str: - return json.dumps(value, sort_keys=True, separators=(",", ":"), default=str) - - -def canonical_json_sha256(value: object) -> str: - incremental_digest = getattr(value, "canonical_json_sha256", None) - if callable(incremental_digest): - return str(incremental_digest()) - return hashlib.sha256(canonical_json_text(value).encode()).hexdigest() - - -class WorkerIdentityPort(Protocol): - """The exact worker facts a collection-mode build binds into its work.""" - - payout_address: str - p2mr_program_hex: str - - -class JobBuildCancelled(TemplateRefreshBlocked): - """An immutable build was cancelled or timed out.""" - - -class JobBuildSuperseded(JobBuildCancelled, TemplateRefreshSuperseded): - """A coordination race cooperatively cancelled immutable construction.""" - - -class JobBundleBuildSuperseded(JobBuildSuperseded): - """A newer tip or payout generation canceled this deterministic build.""" - - -class CollectionIdentityUnavailable(TemplateRefreshBlocked): - """Current collection work is waiting for an authorized worker identity.""" - - -class JobBuildAdmissionDeadlineExceeded(TemplateRefreshBlocked): - """A routine request parked behind publication priority past its bound. - - Subclasses TemplateRefreshBlocked so callers retry with their normal - pacing; the raise site logs and counts the expiry so parked admission is - visible instead of silently riding the client's own longer timeout. - """ - - -class JobBuildWaiterCancelled(RuntimeError): - """A bundle waiter became obsolete before it acquired preparation.""" - - -class JobBuildCancellation: - def __init__(self, *, timeout_seconds: float) -> None: - self._event = threading.Event() - self._lock = threading.Lock() - self.started_monotonic = time.monotonic() - self.deadline_monotonic = self.started_monotonic + timeout_seconds - self.last_checkpoint_monotonic = self.started_monotonic - self.cancelled_monotonic: float | None = None - self.reason: str | None = None - - def cancel(self, reason: str) -> bool: - with self._lock: - if self._event.is_set(): - return False - self.reason = reason - self.cancelled_monotonic = time.monotonic() - self._event.set() - return True - - def is_set(self) -> bool: - if self._event.is_set(): - return True - if time.monotonic() >= self.deadline_monotonic: - self.cancel("timeout") - return True - return False - - def raise_if_cancelled(self, phase: str) -> None: - if self.is_set(): - reason = self.reason or "cancelled" - if reason == "timeout": - raise JobBuildCancelled( - f"job build timeout at {phase}; immediate retry scheduled" - ) - raise JobBuildSuperseded( - f"job build {reason} at {phase}; immediate retry scheduled" - ) - with self._lock: - self.last_checkpoint_monotonic = time.monotonic() - - -# The executor task a build flight submits. The coordinator facade and the -# service method share this name, so a traceback carrying it belongs to a -# build that ran on the job-build executor. -_JOB_BUILD_EXECUTOR_ENTRY = "_execute_job_build_request" - - -def _clear_finished_frame(frame: FrameType) -> None: - try: - frame.clear() - except RuntimeError: - # frame.clear() refuses an executing frame and leaves it intact. The - # frames selected below have all returned, so this only guards the - # promise resolution against a foreign frame spliced into a chained - # or re-raised exception by code outside the producer. - pass - - -def _release_finished_job_build_frames(error: BaseException) -> None: - """Drop the locals of the finished producer frames a build error retains. - - A build that observes its cancellation raises from deep inside the - executor task. The stored exception's traceback references every frame it - unwound through, from the executor's task plumbing and the entry point - down to the checkpoint, and those frames still own the request (as a - local and inside the task's argument tuple), its template artifacts, the - ledger snapshot, and the converted share rows. The request owns the - promise that stores the exception, so the whole graph is a cycle that - only cyclic GC reclaims. - - The traceback's first entry is the executor's work-item frame that caught - the error and stored it on the future. That frame is still executing - while done callbacks run and is never touched. Every entry below it has - returned: the error could only reach the work item by propagating out of - all of them, including the executor context frame that unpacked the task - arguments. Those finished frames are cleared, and so are the finished - producer frames recorded by the error's ``__cause__`` and ``__context__`` - chain (a compiler failure re-raised as supersession keeps the failure as - its cause, whose own traceback still reaches the request). A chained - entry is only cleared from the first frame the cancellation itself - unwound through, so a frame that belongs to some other execution is left - alone. Clearing a finished frame drops only its locals; formatted - tracebacks keep their files, line numbers, and source lines, and the - chain itself is preserved. - """ - - head = error.__traceback__ - entry = head - while ( - entry is not None - and entry.tb_frame.f_code.co_name != _JOB_BUILD_EXECUTOR_ENTRY - ): - entry = entry.tb_next - if head is None or entry is None: - return - finished: set[FrameType] = set() - node = head.tb_next - while node is not None: - finished.add(node.tb_frame) - _clear_finished_frame(node.tb_frame) - node = node.tb_next - seen = {id(error)} - pending = [error] - while pending: - current = pending.pop() - for chained in (current.__cause__, current.__context__): - if chained is None or id(chained) in seen: - continue - seen.add(id(chained)) - pending.append(chained) - node = chained.__traceback__ - while node is not None and node.tb_frame not in finished: - node = node.tb_next - while node is not None: - finished.add(node.tb_frame) - _clear_finished_frame(node.tb_frame) - node = node.tb_next - - -def _job_build_error_for_waiter(error: JobBuildCancelled) -> JobBuildCancelled: - """Return one waiter's private copy of a promise's cancellation error. - - The copy carries the same type, message, diagnostic attributes, notes, - and ``__cause__``/``__context__`` chain, and shares the producer - traceback entries, so a formatted traceback still shows where the build - observed its cancellation and why. Raising a private instance leaves the - stored exception's traceback untouched: no waiter frame is ever appended - to it, and concurrent waiters cannot interleave their frames in each - other's diagnostics. - """ - - private = type(error)(*error.args) - private.__dict__.update(error.__dict__) - notes = getattr(error, "__notes__", None) - if notes is not None: - private.__notes__ = list(notes) - private.__cause__ = error.__cause__ - private.__context__ = error.__context__ - private.__suppress_context__ = error.__suppress_context__ - return private.with_traceback(error.__traceback__) - - -def _await_job_build_promise( - promise: "Future[CachedJobBundle]", - timeout: float, -) -> "CachedJobBundle": - """Wait for a shared build promise without re-raising its stored error. - - ``Future.result`` re-raises the stored instance, which prepends the - waiter's frames to the exception's traceback. A waiter that owns the - request or promise while it unwinds -- the retrying bundle loop does -- - thereby rebuilds the request -> promise -> exception -> traceback -> - request cycle for every cancelled build it observes. Cancellation - outcomes are raised as waiter-private copies instead; successful builds - and unexpected errors keep ``Future.result`` semantics exactly. - """ - - error = promise.exception(timeout=timeout) - if isinstance(error, JobBuildCancelled): - raise _job_build_error_for_waiter(error) - return promise.result() - - -@dataclass(frozen=True) -class JobBuildKey: - """Every immutable input that can affect one constructed mining job.""" - - best_tip_hash: str - previous_block_hash: str - template_fingerprint: str - template_generation: int - payout_state_generation: int - payout_artifact_sha256: str - mode: str - collection_identity: tuple[str, str] | None - block_height: int - coinbase_value_sats: int - network_difficulty: int - issued_at_ms: int - payout_policy_sha256: str - ctv_settlement_sha256: str | None - witness_merkle_sha256: str - transaction_set_sha256: str - coinbase_suffix_hex: str - signing_key_sha256: str - ledger_signing_key_sha256: str - numeric_context_sha256: str - share_snapshot_sha256: str = "" - payout_append_invalidation_epoch: int = 0 - - -@dataclass(frozen=True) -class CachedJobBundle: - """One heavy job build (ledger snapshot + signed manifest + base job) - shared across every client on the same template. - - The base job is built with the extranonce1 placeholder; per-client jobs - are stamped from it by swapping job_id, extranonce1, difficulty, and the - clean_jobs flag. All other fields are byte-identical across clients - because the stratum coinbase split excludes the extranonce window. - """ - - key: tuple[object, ...] - template: dict[str, Any] - template_fingerprint: str - coinbase_manifest: dict[str, Any] - shares_json: list[dict[str, object]] - prior_balances: list[dict[str, object]] - found_block: dict[str, object] - collection_only: bool - issued_at_ms: int - base_job: direct_stratum.DirectQbitStratumJob - built_monotonic: float - template_generation: int = 0 - payout_state_generation: int = 0 - payout_artifact_generation: int = 0 - # Collection coinbases commit a synthetic share to this exact payout - # identity. Ready bundles have no worker-specific inputs and keep this - # unset, which makes accidental cross-worker stamping fail closed. - collection_identity: tuple[str, str] | None = None - # Compact carry-forward state derived from the immutable job summary. It - # lets an accepted block publish child payout state without retaining the - # full audit bundle (and its duplicate shares tree) in every cached job. - prospective_prior_balances: tuple[tuple[str, str, str, int], ...] | None = None - build_key: JobBuildKey | None = None - # A synchronous ready build publishes its fenced ledger window for reuse - # only once this bundle wins cache publication (or is served as pinned - # snapshot work). Installing mid-build would re-key concurrent lookups - # away from the in-flight single flight and fork redundant builds. - prepared_ledger_artifact: "PayoutLedgerArtifact | None" = field( - default=None, - compare=False, - repr=False, - ) - - -@dataclass -class JobBuildRequest: - key: JobBuildKey - cache_key: tuple[object, ...] - equivalence_key: tuple[object, ...] - artifacts: CachedTemplateArtifacts - template_json: str - transaction_hexes: tuple[str, ...] - witness_merkle_leaves_hex: tuple[str, ...] - worker: "WorkerIdentityPort | None" - mode: str - payout_artifact: "PayoutStateArtifact" - payout_ledger_artifact: "PayoutLedgerArtifact | None" - payout_policy_json: str - ctv_settlement_json: str | None - decimal_context: Context = field(repr=False) - cancellation: JobBuildCancellation - idle_retarget: bool = False - publication_critical: bool = False - request_source: str = "routine" - priority_admission_recorded: bool = False - promise: Future[CachedJobBundle] = field(default_factory=Future) - requested_monotonic: float = field(default_factory=time.monotonic) - superseded_monotonic: float | None = None - # Exposed in-flight scan anchor owned by this build (see - # _expose_inflight_scan_anchor). Set by the synchronous ledger read so - # the executor wrapper can retire the exposure when the build dies - # before handing the token to its seeded artifact. - inflight_scan_anchor_token: int | None = None - - -@dataclass(eq=False) -class JobBuildFlight: - request: JobBuildRequest - future: Future[CachedJobBundle] | None = None - # First time the admission sweep saw this flight finished but still - # occupying its slot; eviction waits out the sweep grace from here. - orphan_observed_monotonic: float | None = None - - -@dataclass -class JobBundleBuildControl: - key: tuple[object, ...] - previousblockhash: str - payout_state_generation: int - payout_artifact_generation: int - cancel_event: threading.Event = field(default_factory=threading.Event) - process: subprocess.Popen[str] | None = None - - -@dataclass -class _SharedBundlePreparationFlight: - event: threading.Event = field(default_factory=threading.Event) - result: CachedJobBundle | None = None - error: BaseException | None = None - waiters: int = 0 - - -class JobBundleRuntime(Protocol): - """Typed port over the coordinator, resolved at call time. - - Every member is looked up on the live coordinator object when used, so - instance monkeypatches (``server.build_audit_bundle = ...`` and friends) - and coordinator-owned live configuration attributes keep working exactly - as before the extraction. Owner facades route back into this service; - that round trip is deliberate: it is what preserves the current - white-box patch surface until later stack layers repoint those tests. - """ - - # Cross-domain objects and live configuration attributes. - lock: Any - rpc: Any - ledger: Any - stop_event: Any - extranonce2_size: int - share_difficulty: Any - min_ready_miners: int - job_build_timeout_seconds: float - job_build_cancel_grace_seconds: float - - # First-touch initializers. - def _ensure_job_cache_state(self) -> None: ... - - def _ensure_tip_refresh_state(self) -> None: ... - - # Payout-state facades (owner extracted by a later chunk). - def _current_payout_state_artifact(self, cancellation: Any = None) -> Any: ... - - def _usable_payout_ledger_artifact( - self, - payout_state_generation: int, - network_difficulty: int, - ) -> Any: ... - - def _await_pending_parent_payout_preview( - self, - parent_hash: str, - *, - parent_height: int, - ) -> Any: ... - - def _prior_balances_for_job_parent( - self, - parent_hash: str, - *, - parent_height: int, - fallback_balances: list[dict[str, object]], - ) -> list[dict[str, object]]: ... - - def _serialize_prior_balance_preview(self, balances: Any) -> Any: ... - - def _accepted_block_payout_preview_from_bundle( - self, - bundle: dict[str, Any], - *, - prior_balances: list[dict[str, object]], - ) -> Any: ... - - def _install_payout_ledger_artifact(self, artifact: Any) -> Any: ... - - def _payout_artifact_reuse_active(self) -> bool: ... - - def _payout_artifact_max_anchor_age_ms(self) -> float: ... - - def _payout_artifact_reanchor_seconds(self) -> float: ... - - def _expose_inflight_scan_anchor(self, anchor_ms: int) -> int: ... - - def _retire_inflight_scan_anchor(self, token: int | None) -> None: ... - - def _publish_seedless_job_window_anchor_locked(self, anchor_ms: int) -> None: ... - - # Tip-refresh facades (owner extracted by a later chunk). - def _newest_observed_tip_locked(self) -> str | None: ... - - def _artifacts_buildable_locked(self, artifacts: CachedTemplateArtifacts) -> bool: ... - - def _published_snapshot_artifacts_locked( - self, - artifacts: CachedTemplateArtifacts, - ) -> bool: ... - - def _published_tip_authoritative_locked(self, now: float) -> bool: ... - - def _schedule_tip_refresh_retry(self) -> None: ... - - def _observe_tip_refresh_build_phase(self, name: str, elapsed: float) -> None: ... - - def _vardiff_idle_tip_divergence_locked(self) -> bool: ... - - # Shared readiness/stats and builder facades. - def pool_readiness_latched(self) -> bool: ... - - def accepted_share_stats(self) -> tuple[int, int]: ... - - def prism_payout_policy(self) -> dict[str, object]: ... - - def prism_ctv_settlement_config( - self, - *, - block_height: int, - parent_hash: str | None, - ) -> dict[str, object] | None: ... - - def coinbase_script_sig_suffix_hex( - self, - extranonce1_hex: str, - extranonce2_hex: str, - ) -> str: ... - - def _job_snapshot_anchor_ms(self, issued_at_ms: int) -> int: ... - - def _job_build_checkpoint(self, phase: str, cancellation: Any) -> None: ... - - def build_audit_bundle(self, **kwargs: Any) -> dict[str, Any]: ... - - -class JobBundleService: - """Sole owner of the job-bundle cache, scheduler, and readiness seams.""" - - def __init__( - self, - runtime: JobBundleRuntime, - *, - spool_factory: Callable[[], Any] | None = None, - payout_ledger_artifact_type: type | None = None, - now_ms: Callable[[], int] | None = None, - canonical_json_sha256_override: Callable[[object], str] | None = None, - ) -> None: - self._runtime = runtime - # Call-time-resolved wall clock: the coordinator wires a lambda over - # its own module global so the historical ``prism_coordinator.now_ms`` - # patch seam keeps steering anchor selection. - self._now_ms = now_ms if now_ms is not None else globals()["now_ms"] - # Same seam for the canonical digest: white-box tests intercept it - # through the coordinator module global. - self._canonical_json_sha256 = ( - canonical_json_sha256 - if canonical_json_sha256_override is None - else canonical_json_sha256_override - ) - # Call-time-resolved spool-file factory (see bundle_compiler); the - # coordinator wires its own module global through here so the - # historical patch seam keeps steering spool creation. - self._spool_factory = spool_factory - # The payout-ledger artifact remains coordinator-owned until the - # payout-state owner is extracted; the class is injected so seeded - # artifacts stay type-identical with coordinator-built ones. - self._payout_ledger_artifact_type = payout_ledger_artifact_type - self._job_cache_lock = threading.Lock() - self._active_job_bundle_builds: dict[ - tuple[object, ...], JobBundleBuildControl - ] = {} - # Compatibility seam for embedders/tests. Expensive construction is - # coordinated by the bounded latest-wins scheduler below, not held - # under this lock. - self._job_build_lock = threading.Lock() - self._job_build_scheduler_lock = threading.RLock() - self._job_build_priority_preparations: dict[int, float] = {} - self._job_build_priority_preparation_sequence = 0 - self._job_build_routine_preparations: dict[ - int, - weakref.ReferenceType[JobBuildCancellation], - ] = {} - self._job_build_routine_preparation_sequence = 0 - self._job_build_priority_changed = threading.Event() - self._job_build_executor: ThreadPoolExecutor | None = None - self._job_build_executor_shutdown = False - self._job_build_active: JobBuildFlight | None = None - self._job_build_retiring: JobBuildFlight | None = None - self._job_build_pending: JobBuildRequest | None = None - self._job_build_issued_at_ms: OrderedDict[int, int] = OrderedDict() - self._job_bundle_cache: OrderedDict[ - tuple[object, ...], CachedJobBundle - ] = OrderedDict() - self._job_build_phase_local = threading.local() - self.job_cache_hit_counts = {kind: 0 for kind in PRISM_JOB_CACHE_KINDS} - self.job_cache_miss_counts = {kind: 0 for kind in PRISM_JOB_CACHE_KINDS} - self.job_build_seconds_bucket_counts = { - bucket: 0 for bucket in PRISM_JOB_BUILD_SECONDS_BUCKETS - } - self.job_build_seconds_sum = 0.0 - self.job_build_count = 0 - self.job_build_phase_seconds = {phase: 0.0 for phase in PRISM_JOB_BUILD_PHASES} - self.job_build_scheduler_counts = { - "requests": 0, - "starts": 0, - "completions": 0, - "supersessions": 0, - "obsolete_results": 0, - "orphan_evicted": 0, - } - self.job_build_priority_counts = { - result: 0 - for result in ( - "started", - "coalesced", - "queued", - "routine_deferred", - "routine_preempted", - ) - } - self.job_build_priority_admission_seconds = { - "sum": 0.0, - "count": 0, - } - self.initial_job_prepared_work_counts = { - result: 0 - for result in ( - "cache_hit", - "singleflight", - "deferred", - "subscribed", - "admission_deadline", - ) - } - self._admission_deadline_last_log_monotonic: float | None = None - self.job_build_cancellation_seconds = { - "sum": 0.0, - "count": 0, - } - self.job_build_replacement_start_seconds = { - "sum": 0.0, - "count": 0, - } - self.job_build_worker_counts = { - "starts": 0, - "terminations": 0, - "crashes": 0, - "restarts": 0, - } - self._job_build_worker_restart_pending = False - self._share_window_serialization_lock = threading.Lock() - self._share_window_serialization: _ShareWindowSerialization | None = None - # Keyed shared-preparation flights absorbed from the historical - # coordinator field pair; retained for the still-coordinator-side - # metrics reader until later chunks retire them. - self._bundle_preparation_lock = threading.Lock() - self._bundle_preparation_flights: dict[ - tuple[object, ...], _SharedBundlePreparationFlight - ] = {} - self.shared_bundle_build_counts = { - outcome: 0 - for outcome in ("started", "completed", "superseded", "failed") - } - self.shared_bundle_preparation_seconds_sum = 0.0 - self.shared_bundle_preparation_count = 0 - self.shared_bundle_preparation_waiters = 0 - self._prepared_ready_bundle: CachedJobBundle | None = None - self._prepared_ready_snapshot: QbitTipTemplateSnapshot | None = None - self.job_preparation_pending = False - # The template-artifact repository shares the job-cache lock: the - # historical layout guarded template currency and bundle admission - # with one lock and remaining coordinator domains still rely on it. - self.template_repository = TemplateArtifactRepository( - runtime, - lock=self._job_cache_lock, - ) - - # -- phase bookkeeping ------------------------------------------------- - - def _job_build_phases(self) -> dict[str, float]: - """Per-thread scratch dict of phase timings for the current build.""" - self._runtime._ensure_job_cache_state() - phases = getattr(self._job_build_phase_local, "phases", None) - if phases is None: - phases = {} - self._job_build_phase_local.phases = phases - return phases - - def observe_job_build_elapsed( - self, - elapsed_seconds: float, - phases: dict[str, float], - ) -> None: - self._runtime._ensure_job_cache_state() - with self._job_cache_lock: - self.job_build_count += 1 - self.job_build_seconds_sum += elapsed_seconds - for bucket in PRISM_JOB_BUILD_SECONDS_BUCKETS: - if elapsed_seconds <= bucket: - self.job_build_seconds_bucket_counts[bucket] += 1 - for phase, duration in phases.items(): - if phase in self.job_build_phase_seconds: - self.job_build_phase_seconds[phase] += duration - - def _flush_job_build_phases(self, phases: dict[str, float]) -> None: - """Fold a worker thread's phase accruals into the exported counters. - - Only the per-client delivery entry points clear and flush the - thread-local phase dict; the job-build executor and payout-artifact - preparation workers never passed through them, so the template/ - ledger/assembly/bundle seconds they accrued were exported as - exactly 0.0 while production paid multi-second walks -- the blind - spot that misdirected the 2026-07-31 incident review. Build counts - and the duration histogram stay per-delivery; this folds phase - seconds only, and clears the dict so a long-lived worker thread - cannot double-report an earlier request's accruals. - """ - if not phases: - return - self._runtime._ensure_job_cache_state() - with self._job_cache_lock: - for phase, duration in phases.items(): - if phase in self.job_build_phase_seconds: - self.job_build_phase_seconds[phase] += duration - phases.clear() - - def _record_job_cache_event(self, kind: str, *, hit: bool) -> None: - self._runtime._ensure_job_cache_state() - with self._job_cache_lock: - counts = self.job_cache_hit_counts if hit else self.job_cache_miss_counts - counts[kind] = int(counts.get(kind, 0)) + 1 - - # -- build-control registry -------------------------------------------- - - def _cancel_obsolete_job_bundle_builds( - self, - *, - current_tip: str | None = None, - payout_state_generation: int | None = None, - ) -> None: - """Cancel only builds proven obsolete by a newer exact generation.""" - self._runtime._ensure_job_cache_state() - processes: list[subprocess.Popen[str]] = [] - with self._job_cache_lock: - for control in self._active_job_bundle_builds.values(): - obsolete = ( - current_tip is not None - and control.previousblockhash != current_tip - ) or ( - payout_state_generation is not None - and control.payout_state_generation - != int(payout_state_generation) - ) - if not obsolete or control.cancel_event.is_set(): - continue - control.cancel_event.set() - if control.process is not None: - processes.append(control.process) - for process in processes: - if process.poll() is not None: - continue - try: - process.terminate() - except ProcessLookupError: - pass - - def _register_job_bundle_process( - self, - control: JobBundleBuildControl, - process: subprocess.Popen[str], - ) -> None: - terminate = False - with self._job_cache_lock: - if ( - self._active_job_bundle_builds.get(control.key) is not control - or control.cancel_event.is_set() - ): - terminate = True - else: - control.process = process - if terminate and process.poll() is None: - try: - process.terminate() - except ProcessLookupError: - pass - - def _unregister_job_bundle_process( - self, - control: JobBundleBuildControl, - process: Any, - ) -> None: - """Detach a shared daemon from a build control after its request. - - Registration makes supersession terminate the in-flight process. - Once this build's daemon request has completed, a later supersession - of the build must not kill the shared daemon out from under - whichever request owns it next. - """ - with self._job_cache_lock: - if control.process is process: - control.process = None - - # -- bounded latest-wins scheduler ------------------------------------- - - def _job_build_executor_locked(self) -> ThreadPoolExecutor: - runtime = self._runtime - if self._job_build_executor_shutdown: - raise RuntimeError("job build executor is shut down") - executor = self._job_build_executor - if executor is None: - executor = ThreadPoolExecutor( - max_workers=int( - getattr( - runtime, - "job_build_executor_workers", - DEFAULT_PRISM_JOB_BUILD_EXECUTOR_WORKERS, - ) - ), - thread_name_prefix="prism-job-build", - ) - self._job_build_executor = executor - return executor - - def _start_job_build_locked(self, request: JobBuildRequest) -> JobBuildFlight: - runtime = self._runtime - executor = runtime._job_build_executor_locked() - flight = JobBuildFlight(request=request) - self.job_build_scheduler_counts["starts"] += 1 - self.shared_bundle_build_counts["started"] += 1 - if request.superseded_monotonic is not None: - elapsed = max(0.0, time.monotonic() - request.superseded_monotonic) - self.job_build_replacement_start_seconds["sum"] += elapsed - self.job_build_replacement_start_seconds["count"] += 1 - future = executor.submit(runtime._execute_job_build_request, request) - flight.future = future - runtime._record_priority_admission_locked(request, "started") - return flight - - def _arm_job_build_locked(self, flight: JobBuildFlight) -> None: - runtime = self._runtime - future = flight.future - assert future is not None - add_releasing_done_callback( - future, - lambda completed: runtime._job_build_done(flight, completed), - ) - - def _execute_job_build_request( - self, - request: JobBuildRequest, - ) -> CachedJobBundle: - runtime = self._runtime - request.cancellation.raise_if_cancelled("start") - control = JobBundleBuildControl( - key=request.equivalence_key, - previousblockhash=request.key.previous_block_hash, - payout_state_generation=request.key.payout_state_generation, - payout_artifact_generation=( - request.payout_ledger_artifact.generation - if request.payout_ledger_artifact is not None - else 0 - ), - ) - with self._job_cache_lock: - self._active_job_bundle_builds[control.key] = control - previous_control = getattr( - self._job_build_phase_local, - "bundle_build_control", - None, - ) - self._job_build_phase_local.bundle_build_control = control - phases = runtime._job_build_phases() - phases.clear() - try: - with localcontext(request.decimal_context): - return runtime.build_shared_job_bundle( - request.artifacts, - request.worker, - mode=request.mode, - payout_state_generation=request.key.payout_state_generation, - payout_artifact=request.payout_ledger_artifact, - key=request.cache_key, - build_request=request, - ) - except BaseException: - # A build that died past its synchronous ledger read can no - # longer hand its exposed scan anchor to a seeded artifact; - # retire it here so a dead walk does not keep advancing the - # append-invalidation epoch. Successful builds either retired - # the token at the publication fence or handed it to the seed, - # whose install fence retires it. - runtime._retire_inflight_scan_anchor( - request.inflight_scan_anchor_token - ) - raise - finally: - runtime._flush_job_build_phases(phases) - self._job_build_phase_local.bundle_build_control = previous_control - with self._job_cache_lock: - if self._active_job_bundle_builds.get(control.key) is control: - self._active_job_bundle_builds.pop(control.key, None) - control.process = None - - @staticmethod - def _collection_job_builds_are_independent( - first: JobBuildRequest, - second: JobBuildRequest, - ) -> bool: - """Distinct workers in one immutable collection cohort are peers.""" - - return ( - first.mode == "collection" - and second.mode == "collection" - and first.key.collection_identity != second.key.collection_identity - and dataclass_replace(first.key, collection_identity=None) - == dataclass_replace(second.key, collection_identity=None) - ) - - @staticmethod - def _job_build_requests_can_share( - first: JobBuildRequest, - second: JobBuildRequest, - ) -> bool: - """Share exact builds plus ready work stable across clock-only refreshes.""" - - return first.equivalence_key == second.equivalence_key or ( - first.mode == "ready" - and second.mode == "ready" - and first.cache_key == second.cache_key - ) - - @staticmethod - def _ready_job_build_precedes_collection( - first: JobBuildRequest, - second: JobBuildRequest, - ) -> bool: - """Live ready work cannot be displaced by a collection-mode retry.""" - - return ( - first.mode == "ready" - and second.mode == "collection" - and not first.cancellation.is_set() - ) - - @staticmethod - def _defer_job_build_locked( - *blockers: Future[CachedJobBundle], - ) -> Future[CachedJobBundle]: - """Wake a bounded lower-priority waiter when occupied capacity exits.""" - - deferred: Future[CachedJobBundle] = Future() - wake_lock = threading.Lock() - - def wake_for_retry(_completed: Future[CachedJobBundle]) -> None: - with wake_lock: - if not deferred.done(): - deferred.set_exception( - JobBuildSuperseded( - "job build capacity became available; retrying" - ) - ) - - for blocker in blockers: - blocker.add_done_callback(wake_for_retry) - return deferred - - @staticmethod - def _job_build_is_publication_critical(request: object) -> bool: - return bool(getattr(request, "publication_critical", False)) - - @staticmethod - def _job_build_promise_done(request: object) -> bool: - """Whether a request's shared promise has already been resolved. - - Reads defensively like the other request predicates: embedders and - tests plant lightweight request doubles without a promise, which are - never resolved-and-abandoned flights. - """ - - promise = getattr(request, "promise", None) - return promise is not None and promise.done() - - def _record_priority_admission_locked( - self, - request: JobBuildRequest, - result: str, - ) -> None: - """Observe first builder admission from publication-priority reservation.""" - - runtime = self._runtime - if not runtime._job_build_is_publication_critical(request): - return - self.job_build_priority_counts[result] += 1 - if result not in {"started", "coalesced"}: - return - if request.priority_admission_recorded: - return - request.priority_admission_recorded = True - elapsed = max(0.0, time.monotonic() - request.requested_monotonic) - self.job_build_priority_admission_seconds["sum"] += elapsed - self.job_build_priority_admission_seconds["count"] += 1 - - def _record_initial_prepared_work_locked(self, result: str) -> None: - self.initial_job_prepared_work_counts[result] += 1 - - def _new_job_build_cancellation(self) -> JobBuildCancellation: - return JobBuildCancellation( - timeout_seconds=max( - 0.001, - float( - getattr( - self._runtime, - "job_build_timeout_seconds", - DEFAULT_PRISM_JOB_BUILD_TIMEOUT_SECONDS, - ) - ), - ) - ) - - def _begin_job_build_priority_preparation( - self, - requested_monotonic: float | None = None, - ) -> tuple[int, float]: - """Reserve publication priority before immutable request construction.""" - - started = ( - time.monotonic() - if requested_monotonic is None - else requested_monotonic - ) - with self._job_build_scheduler_lock: - self._job_build_priority_preparation_sequence += 1 - token = self._job_build_priority_preparation_sequence - self._job_build_priority_preparations[token] = started - for routine_cancellation_ref in tuple( - self._job_build_routine_preparations.values() - ): - routine_cancellation = routine_cancellation_ref() - if routine_cancellation is not None: - routine_cancellation.cancel("publication priority") - self._job_build_routine_preparations.clear() - self._job_build_priority_changed.set() - return token, started - - def _finish_job_build_priority_preparation(self, token: int) -> None: - with self._job_build_scheduler_lock: - self._job_build_priority_preparations.pop(token, None) - self._job_build_priority_changed.set() - - def _begin_routine_job_build_preparation( - self, - *, - request_source: str, - cancelled: Callable[[], bool] | None, - ) -> tuple[int, JobBuildCancellation, CachedJobBundle | None]: - """Atomically admit cancellable routine request construction. - - Initial requests do not poll out the publication-priority window: - they subscribe to the in-flight priority build's promise, wake when - it completes, and hand its bundle back to the caller. Riding the - result that displaced them replaces the old defer-then-rebuild cycle - (and its wakeup storm) with one shared build. - """ - - runtime = self._runtime - deferred_recorded = False - subscription_recorded = False - subscribed_bundle: CachedJobBundle | None = None - admission_started_monotonic = time.monotonic() - admission_deadline_seconds = float( - getattr( - runtime, - "routine_admission_deadline_seconds", - DEFAULT_PRISM_ROUTINE_ADMISSION_DEADLINE_SECONDS, - ) - ) - while True: - self._job_build_priority_changed.clear() - priority_promises: tuple[Future[CachedJobBundle], ...] = () - with self._job_build_scheduler_lock: - if not runtime._publication_priority_scheduled_locked(): - self._job_build_routine_preparation_sequence += 1 - token = self._job_build_routine_preparation_sequence - preparation_cancellation = ( - runtime._new_job_build_cancellation() - ) - coordinator_ref = weakref.ref(runtime) - - def remove_dead_preparation( - dead_ref: weakref.ReferenceType[ - JobBuildCancellation - ], - *, - preparation_token: int = token, - ) -> None: - coordinator = coordinator_ref() - if coordinator is None: - return - with coordinator._job_build_scheduler_lock: - if ( - coordinator._job_build_routine_preparations.get( - preparation_token - ) - is dead_ref - ): - coordinator._job_build_routine_preparations.pop( - preparation_token, - None, - ) - - self._job_build_routine_preparations[token] = weakref.ref( - preparation_cancellation, - remove_dead_preparation, - ) - return token, preparation_cancellation, subscribed_bundle - if not deferred_recorded: - self.job_build_priority_counts["routine_deferred"] += 1 - if request_source == "initial": - runtime._record_initial_prepared_work_locked("deferred") - deferred_recorded = True - if request_source == "initial": - priority_promises = ( - runtime._publication_priority_promises_locked() - ) - if priority_promises and not subscription_recorded: - runtime._record_initial_prepared_work_locked("subscribed") - subscription_recorded = True - if cancelled is not None and cancelled(): - raise JobBuildWaiterCancelled( - "job bundle request was cancelled behind publication priority" - ) - stop_event = getattr(runtime, "stop_event", None) - if stop_event is not None and stop_event.is_set(): - raise JobBuildWaiterCancelled( - "coordinator stopped behind publication priority" - ) - parked_seconds = time.monotonic() - admission_started_monotonic - if parked_seconds > admission_deadline_seconds: - # Unbounded parking here is how the 2026-07-29 incident hid: - # initial jobs rode the client's own 90s timeout with zero - # tracebacks. Fail fast and visibly instead; callers retry - # with their normal pacing. - with self._job_build_scheduler_lock: - if request_source == "initial": - runtime._record_initial_prepared_work_locked( - "admission_deadline" - ) - now_monotonic = time.monotonic() - should_log = ( - self._admission_deadline_last_log_monotonic is None - or now_monotonic - - self._admission_deadline_last_log_monotonic >= 5.0 - ) - if should_log: - self._admission_deadline_last_log_monotonic = ( - now_monotonic - ) - if should_log: - print( - "prism coordinator: job admission deadline exceeded " - f"source={request_source} " - f"parked_seconds={parked_seconds:.1f}", - flush=True, - ) - raise JobBuildAdmissionDeadlineExceeded( - "job bundle admission parked behind publication priority " - f"for {parked_seconds:.1f}s" - ) - if priority_promises: - for promise in priority_promises: - if ( - promise.done() - and not promise.cancelled() - and promise.exception() is None - ): - subscribed_bundle = promise.result() - pending_promises = tuple( - promise - for promise in priority_promises - if not promise.done() - ) - if pending_promises: - done, _pending = wait( - pending_promises, - timeout=PRISM_INITIAL_JOB_SUBSCRIBE_POLL_SECONDS, - return_when=FIRST_COMPLETED, - ) - for completed in done: - if ( - completed.cancelled() - or completed.exception() is not None - ): - continue - subscribed_bundle = completed.result() - else: - # Every live priority promise has resolved; the scheduler - # is about to sweep the finished flights. Waiting on the - # event instead of the done promises avoids a hot loop. - self._job_build_priority_changed.wait(0.05) - else: - # Reservation-only window: the priority request object (and - # its promise) does not exist yet. This window only spans - # immutable request construction, never the build itself. - self._job_build_priority_changed.wait(0.05) - - def _publication_priority_promises_locked( - self, - ) -> tuple[Future[CachedJobBundle], ...]: - """Promises of live publication-critical build requests.""" - - runtime = self._runtime - promises: list[Future[CachedJobBundle]] = [] - pending = self._job_build_pending - if ( - pending is not None - and not pending.cancellation.is_set() - and runtime._job_build_is_publication_critical(pending) - ): - promises.append(pending.promise) - for flight in (self._job_build_active, self._job_build_retiring): - if ( - flight is not None - and not flight.request.cancellation.is_set() - and runtime._job_build_is_publication_critical(flight.request) - ): - promises.append(flight.request.promise) - return tuple(promises) - - def _finish_routine_job_build_preparation(self, token: int) -> None: - with self._job_build_scheduler_lock: - self._job_build_routine_preparations.pop(token, None) - - def _publication_priority_scheduled_locked(self) -> bool: - runtime = self._runtime - if self._job_build_priority_preparations: - return True - pending = self._job_build_pending - if ( - pending is not None - and not pending.cancellation.is_set() - and not runtime._job_build_promise_done(pending) - and runtime._job_build_is_publication_critical(pending) - ): - return True - return any( - flight is not None - and not flight.request.cancellation.is_set() - and not runtime._job_build_promise_done(flight.request) - and runtime._job_build_is_publication_critical(flight.request) - for flight in ( - self._job_build_active, - self._job_build_retiring, - ) - ) - - def _job_build_can_inherit_publication_priority( - self, - existing: JobBuildRequest, - incoming: JobBuildRequest, - ) -> bool: - """Reject an almost-expired or stalled routine flight as a critical owner.""" - - runtime = self._runtime - if ( - not runtime._job_build_is_publication_critical(incoming) - or runtime._job_build_is_publication_critical(existing) - ): - return True - cancellation = existing.cancellation - total_budget = max( - 0.001, - cancellation.deadline_monotonic - cancellation.started_monotonic, - ) - remaining_budget = cancellation.deadline_monotonic - time.monotonic() - progress_budget = max( - 0.001, - float( - getattr( - runtime, - "job_build_cancel_grace_seconds", - DEFAULT_PRISM_JOB_BUILD_CANCEL_GRACE_SECONDS, - ) - ), - ) - progress_age = ( - time.monotonic() - - float( - getattr( - cancellation, - "last_checkpoint_monotonic", - cancellation.started_monotonic, - ) - ) - ) - return ( - remaining_budget >= total_budget / 2.0 - and progress_age <= progress_budget - ) - - @staticmethod - def _resolve_cancelled_job_build_promise( - request: JobBuildRequest, - reason: str, - ) -> None: - """Wake every waiter on a terminated flight's promise immediately. - - Safe against the build's own completion: the done callback checks - ``promise.done()`` before resolving, so the first resolution wins and - a late build result for a cancelled flight is dropped exactly as it - was when the callback produced the exception itself. - """ - - promise = getattr(request, "promise", None) - if promise is None or promise.done(): - return - if reason == "timeout": - promise.set_exception( - JobBuildCancelled("job build flight cancelled by timeout") - ) - return - promise.set_exception( - JobBuildSuperseded(f"job build flight cancelled: {reason}") - ) - - @staticmethod - def _job_build_flight_outcome( - request: JobBuildRequest, - future: Future[CachedJobBundle], - ) -> tuple[CachedJobBundle | None, BaseException | None]: - """Map a finished executor future onto the shared promise outcome.""" - - # Inspect the terminal error without re-raising it here: this frame - # owns both the request and future, so adding it to a stored error's - # traceback would retain the completed build until cyclic GC. - if future.cancelled(): - return None, CancelledError() - error = future.exception() - if error is not None: - if isinstance(error, JobBuildCancelled): - # The finished producer frames still own the request and its - # payout window through this traceback. Release them before - # the error is published to waiters; unexpected failures keep - # their full traceback for post-mortem inspection. - _release_finished_job_build_frames(error) - return None, error - result = future.result() - if request.cancellation.is_set(): - if request.cancellation.reason == "timeout": - error = JobBuildCancelled( - "job build completed after its timeout" - ) - else: - error = JobBuildSuperseded( - "obsolete job build completed after cancellation" - ) - return result, error - - def _evict_orphaned_job_build_flights_locked(self) -> list[str]: - """Evict finished flights whose completion never released their slot. - - A flight's done callback resolves its promise, vacates its slot, and - promotes pending work. Done callbacks swallow exceptions, so if one - dies mid-pass the flight wedges: admission keeps treating the slot as - occupied, deferred requesters chain onto a promise nobody will ever - resolve, and promotion never runs again even though the executor is - idle. Sweep such flights out at admission time: resolve the promise - from the finished future so parked waiters wake, free the slot so the - next requester becomes the new owner, and promote any parked pending - request. A flight whose future is still queued or executing is never - touched; its own completion performs the normal cleanup. - - Returns the eviction log lines instead of printing them: the caller - holds the scheduler lock, and a blocked stdout must not stall - admission (the admission-deadline log follows the same discipline). - """ - - runtime = self._runtime - eviction_logs: list[str] = [] - grace_seconds = float( - getattr( - runtime, - "job_build_orphan_sweep_grace_seconds", - PRISM_JOB_BUILD_ORPHAN_SWEEP_GRACE_SECONDS, - ) - ) - now = time.monotonic() - evicted = False - for slot_name in ("_job_build_active", "_job_build_retiring"): - flight: JobBuildFlight | None = getattr(self, slot_name) - if flight is None: - continue - # Embedders and tests plant lightweight flight doubles; read the - # lifecycle fields defensively like the rest of the scheduler. - future = getattr(flight, "future", None) - if future is not None and not future.done(): - continue - observed = getattr(flight, "orphan_observed_monotonic", None) - if observed is None: - try: - flight.orphan_observed_monotonic = now - except AttributeError: - pass - continue - if now - observed < grace_seconds: - continue - request = flight.request - promise = getattr(request, "promise", None) - if promise is not None and not promise.done(): - if future is None: - runtime._resolve_cancelled_job_build_promise( - request, - "superseded", - ) - else: - result, error = runtime._job_build_flight_outcome( - request, - future, - ) - try: - if error is not None: - promise.set_exception(error) - else: - assert result is not None - promise.set_result(result) - except InvalidStateError: - # The flight's own done callback won the resolution - # race between our done() check and this set. - pass - setattr(self, slot_name, None) - self.job_build_scheduler_counts["orphan_evicted"] += 1 - evicted = True - eviction_logs.append( - "prism coordinator: evicted orphaned job build flight " - f"slot={slot_name.removeprefix('_job_build_')} " - f"cancelled={flight.request.cancellation.is_set()} " - f"started={future is not None}" - ) - if evicted: - runtime._promote_pending_job_build_locked() - self._job_build_priority_changed.set() - return eviction_logs - - def _cancel_job_build_flight_locked( - self, - flight: JobBuildFlight, - reason: str, - *, - now: float | None = None, - ) -> bool: - runtime = self._runtime - if not flight.request.cancellation.cancel(reason): - # Already cancelled (possibly by its own deadline). A flight that - # never reached the executor still has no other resolver, so - # settle its waiters here instead of leaving them to the sweep. - if getattr(flight, "future", None) is None: - runtime._resolve_cancelled_job_build_promise( - flight.request, - flight.request.cancellation.reason or reason, - ) - return False - try: - flight.request.superseded_monotonic = ( - time.monotonic() if now is None else now - ) - self.job_build_scheduler_counts["supersessions"] += 1 - if reason == "publication priority": - self.job_build_priority_counts["routine_preempted"] += 1 - finally: - # A submitted future delivers the promise from its done callback - # when the build observes the cancellation and drains -- refresh - # drivers rely on that ordering to keep one heavy build in - # flight, and a completed-but-unswept future is the admission - # sweep's to resolve. A flight that never reached the executor - # has no callback at all: cancellation must wake its waiters - # itself or they burn their full wait deadline. - if getattr(flight, "future", None) is None: - runtime._resolve_cancelled_job_build_promise( - flight.request, - reason, - ) - self._job_build_priority_changed.set() - return True - - def _promote_pending_job_build_locked(self) -> None: - runtime = self._runtime - pending = self._job_build_pending - if pending is None: - return - if runtime._job_build_promise_done(pending): - # Every waiter has already been answered (eviction or a raced - # resolution); building it would spend a bounded executor slot on - # work nobody consumes. - self._job_build_pending = None - return - active = self._job_build_active - retiring = self._job_build_retiring - if ( - not runtime._job_build_is_publication_critical(pending) - and any( - flight is not None - and not flight.request.cancellation.is_set() - and not runtime._job_build_promise_done(flight.request) - and runtime._job_build_is_publication_critical(flight.request) - for flight in (active, retiring) - ) - ): - # The pending request may predate an exact-flight priority upgrade. - # Preserve the same admission invariant as _request_job_build: - # routine work cannot displace scheduled publication work. - return - if active is not None: - if retiring is not None: - return - if runtime._ready_job_build_precedes_collection( - active.request, - pending, - ) and not runtime._job_build_is_publication_critical(pending): - return - if not runtime._collection_job_builds_are_independent( - active.request, - pending, - ): - reason = ( - "publication priority" - if runtime._job_build_is_publication_critical(pending) - and not runtime._job_build_is_publication_critical(active.request) - else "superseded" - ) - runtime._cancel_job_build_flight_locked(active, reason) - self._job_build_retiring = active - self._job_build_active = None - elif retiring is not None: - if runtime._ready_job_build_precedes_collection( - retiring.request, - pending, - ) and not runtime._job_build_is_publication_critical(pending): - return - if not runtime._collection_job_builds_are_independent( - retiring.request, - pending, - ): - reason = ( - "publication priority" - if runtime._job_build_is_publication_critical(pending) - and not runtime._job_build_is_publication_critical(retiring.request) - else "superseded" - ) - runtime._cancel_job_build_flight_locked(retiring, reason) - self._job_build_pending = None - try: - flight = runtime._start_job_build_locked(pending) - except BaseException: - # The pending slot is already vacated: with no flight there is no - # done callback left that could ever resolve this promise for its - # waiters. - runtime._resolve_cancelled_job_build_promise(pending, "superseded") - raise - self._job_build_active = flight - runtime._arm_job_build_locked(flight) - - def _job_build_done( - self, - flight: JobBuildFlight, - future: Future[CachedJobBundle], - ) -> None: - runtime = self._runtime - request = flight.request - result, error = runtime._job_build_flight_outcome(request, future) - try: - with self._job_build_scheduler_lock: - self.job_build_scheduler_counts["completions"] += 1 - self.shared_bundle_preparation_count += 1 - self.shared_bundle_preparation_seconds_sum += max( - 0.0, - time.monotonic() - request.cancellation.started_monotonic, - ) - if request.cancellation.cancelled_monotonic is not None: - elapsed = max( - 0.0, - time.monotonic() - - request.cancellation.cancelled_monotonic, - ) - self.job_build_cancellation_seconds["sum"] += elapsed - self.job_build_cancellation_seconds["count"] += 1 - coordination_cancelled = isinstance( - error, JobBuildSuperseded - ) or ( - request.cancellation.is_set() - and request.cancellation.reason != "timeout" - ) - if error is not None and coordination_cancelled: - self.job_build_scheduler_counts["obsolete_results"] += 1 - self.shared_bundle_build_counts["superseded"] += 1 - with runtime._tip_refresh_metrics_lock: - runtime.tip_refresh_superseded_results += 1 - elif error is not None: - self.shared_bundle_build_counts["failed"] += 1 - else: - self.shared_bundle_build_counts["completed"] += 1 - if self._job_build_active is flight: - self._job_build_active = None - if self._job_build_retiring is flight: - self._job_build_retiring = None - runtime._promote_pending_job_build_locked() - finally: - # Done callbacks swallow exceptions, so a raise anywhere in the - # bookkeeping above must not skip promise resolution: an orphaned - # promise strands every joiner until its wait deadline and leaves - # a finished flight wedged in its slot with the executor idle. - try: - if not request.promise.done(): - if error is not None: - request.promise.set_exception(error) - else: - assert result is not None - request.promise.set_result(result) - except InvalidStateError: - # The admission sweep can resolve a wedged flight between the - # done() check and this set; either resolution wakes waiters. - pass - self._job_build_priority_changed.set() - - def _request_job_build(self, request: JobBuildRequest) -> Future[CachedJobBundle]: - runtime = self._runtime - runtime._ensure_job_cache_state() - with self._job_build_scheduler_lock: - eviction_logs = runtime._evict_orphaned_job_build_flights_locked() - for eviction_log in eviction_logs: - print(eviction_log, flush=True) - with self._job_build_scheduler_lock: - if getattr(request, "idle_retarget", False): - with runtime.lock: - defer_idle = runtime._vardiff_idle_tip_divergence_locked() - if defer_idle: - request.cancellation.cancel( - "idle retarget deferred during unpublished tip refresh" - ) - if not request.promise.done(): - request.promise.set_exception( - JobBuildSuperseded( - "idle retarget deferred during unpublished tip refresh" - ) - ) - return request.promise - self.job_build_scheduler_counts["requests"] += 1 - active = self._job_build_active - retiring = self._job_build_retiring - pending = self._job_build_pending - publication_critical = runtime._job_build_is_publication_critical( - request - ) - if ( - active is not None - and not active.request.cancellation.is_set() - and not runtime._job_build_promise_done(active.request) - and runtime._job_build_requests_can_share(active.request, request) - and runtime._job_build_can_inherit_publication_priority( - active.request, - request, - ) - ): - if publication_critical: - active.request.publication_critical = True - active.request.request_source = request.request_source - active.request.requested_monotonic = request.requested_monotonic - active.request.priority_admission_recorded = True - runtime._record_priority_admission_locked(request, "coalesced") - if request.request_source == "initial": - runtime._record_initial_prepared_work_locked("singleflight") - return active.request.promise - if ( - retiring is not None - and not retiring.request.cancellation.is_set() - and not runtime._job_build_promise_done(retiring.request) - and runtime._job_build_requests_can_share(retiring.request, request) - and runtime._job_build_can_inherit_publication_priority( - retiring.request, - request, - ) - ): - if publication_critical: - retiring.request.publication_critical = True - retiring.request.request_source = request.request_source - retiring.request.requested_monotonic = request.requested_monotonic - retiring.request.priority_admission_recorded = True - runtime._record_priority_admission_locked(request, "coalesced") - if request.request_source == "initial": - runtime._record_initial_prepared_work_locked("singleflight") - return retiring.request.promise - if ( - pending is not None - and not pending.cancellation.is_set() - and not runtime._job_build_promise_done(pending) - and runtime._job_build_requests_can_share(pending, request) - and runtime._job_build_can_inherit_publication_priority( - pending, - request, - ) - ): - if publication_critical: - pending.publication_critical = True - pending.request_source = request.request_source - pending.requested_monotonic = request.requested_monotonic - runtime._record_priority_admission_locked(request, "queued") - now = time.monotonic() - for occupied in (active, retiring): - if ( - occupied is not None - and not runtime._job_build_requests_can_share( - occupied.request, - pending, - ) - and not runtime._job_build_is_publication_critical( - occupied.request - ) - ): - runtime._cancel_job_build_flight_locked( - occupied, - "publication priority", - now=now, - ) - runtime._promote_pending_job_build_locked() - if request.request_source == "initial": - runtime._record_initial_prepared_work_locked("singleflight") - return pending.promise - - if not publication_critical: - priority_blockers = tuple( - blocker - for blocker in ( - active.request if active is not None else None, - retiring.request if retiring is not None else None, - pending, - ) - if blocker is not None - and not blocker.cancellation.is_set() - # A resolved promise is not live priority work: deferring - # on it wakes instantly and spins until the sweep evicts. - and not runtime._job_build_promise_done(blocker) - and runtime._job_build_is_publication_critical(blocker) - ) - if priority_blockers: - self.job_build_priority_counts["routine_deferred"] += 1 - if request.request_source == "initial": - runtime._record_initial_prepared_work_locked("deferred") - return runtime._defer_job_build_locked( - *(blocker.promise for blocker in priority_blockers) - ) - - if request.mode == "collection": - possible_blockers = ( - pending, - active.request if active is not None else None, - retiring.request if retiring is not None else None, - ) - for blocker in possible_blockers: - if ( - blocker is not None - and not blocker.cancellation.is_set() - and not runtime._job_build_promise_done(blocker) - and runtime._ready_job_build_precedes_collection( - blocker, - request, - ) - and not ( - publication_critical - and not runtime._job_build_is_publication_critical( - blocker - ) - ) - ): - return runtime._defer_job_build_locked(blocker.promise) - if active is None: - if pending is not None: - if runtime._collection_job_builds_are_independent( - pending, - request, - ): - self._job_build_pending = None - try: - flight = runtime._start_job_build_locked(pending) - except BaseException: - # The pending slot is already vacated: with no - # flight there is no done callback left that - # could resolve this promise for its waiters. - runtime._resolve_cancelled_job_build_promise( - pending, - "superseded", - ) - raise - if retiring is None: - try: - replacement = runtime._start_job_build_locked( - request - ) - except BaseException: - # The first start succeeded: slot and arm it - # so its completion still resolves and sweeps - # normally before this failure propagates. - self._job_build_active = flight - runtime._arm_job_build_locked(flight) - raise - self._job_build_retiring = flight - self._job_build_active = replacement - runtime._arm_job_build_locked(flight) - runtime._arm_job_build_locked(replacement) - return request.promise - self._job_build_active = flight - runtime._arm_job_build_locked(flight) - return runtime._defer_job_build_locked( - flight.request.promise, - retiring.request.promise, - ) - pending.cancellation.cancel("superseded while pending") - if not pending.promise.done(): - pending.promise.set_exception( - JobBuildSuperseded("pending job build was superseded") - ) - self._job_build_pending = None - self.job_build_scheduler_counts["supersessions"] += 1 - if ( - retiring is not None - and not runtime._collection_job_builds_are_independent( - retiring.request, - request, - ) - ): - now = time.monotonic() - reason = ( - "publication priority" - if publication_critical - and not runtime._job_build_is_publication_critical( - retiring.request - ) - else "superseded" - ) - if runtime._cancel_job_build_flight_locked( - retiring, - reason, - now=now, - ): - request.superseded_monotonic = now - flight = runtime._start_job_build_locked(request) - self._job_build_active = flight - runtime._arm_job_build_locked(flight) - return request.promise - - if runtime._collection_job_builds_are_independent( - active.request, - request, - ): - if self._job_build_retiring is None: - self._job_build_retiring = active - flight = runtime._start_job_build_locked(request) - self._job_build_active = flight - runtime._arm_job_build_locked(flight) - return request.promise - if pending is None: - self._job_build_pending = request - if publication_critical: - runtime._record_priority_admission_locked(request, "queued") - now = time.monotonic() - for occupied in (active, self._job_build_retiring): - if ( - occupied is not None - and not runtime._job_build_is_publication_critical( - occupied.request - ) - ): - runtime._cancel_job_build_flight_locked( - occupied, - "publication priority", - now=now, - ) - return request.promise - assert retiring is not None - if publication_critical: - pending.cancellation.cancel("superseded while pending") - if not pending.promise.done(): - pending.promise.set_exception( - JobBuildSuperseded( - "pending job build was superseded by publication priority" - ) - ) - self.job_build_scheduler_counts["supersessions"] += 1 - self._job_build_pending = request - runtime._record_priority_admission_locked(request, "queued") - now = time.monotonic() - for occupied in (active, retiring): - if not runtime._job_build_is_publication_critical( - occupied.request - ): - runtime._cancel_job_build_flight_locked( - occupied, - "publication priority", - now=now, - ) - return request.promise - return runtime._defer_job_build_locked( - active.request.promise, - retiring.request.promise, - ) - - now = time.monotonic() - for obsolete in (active, retiring): - if obsolete is not None: - reason = ( - "publication priority" - if publication_critical - and not runtime._job_build_is_publication_critical( - obsolete.request - ) - else "superseded" - ) - runtime._cancel_job_build_flight_locked( - obsolete, - reason, - now=now, - ) - request.superseded_monotonic = now - if retiring is None: - self._job_build_retiring = active - flight = runtime._start_job_build_locked(request) - self._job_build_active = flight - runtime._arm_job_build_locked(flight) - return request.promise - - previous_pending = self._job_build_pending - if previous_pending is not None: - previous_pending.cancellation.cancel("superseded while pending") - if not previous_pending.promise.done(): - previous_pending.promise.set_exception( - JobBuildSuperseded("pending job build was superseded") - ) - self.job_build_scheduler_counts["supersessions"] += 1 - self._job_build_pending = request - if publication_critical: - runtime._record_priority_admission_locked(request, "queued") - return request.promise - - def _cancel_obsolete_job_builds( - self, - reason: str, - *, - keep_published_snapshot: bool = False, - ) -> None: - runtime = self._runtime - runtime._ensure_job_cache_state() - published_parent: str | None = None - published_fingerprint: str | None = None - if keep_published_snapshot: - # A per-client build for exactly the published snapshot is still - # valid, creditable work when a same-tip template bump sweeps old - # fingerprints. Detection-time sweeps deliberately do not use this: - # once a replacement tip is detected, even published-tip builds - # must vacate the single-flight lane for the replacement's work. - # Snapshot the identity outside the scheduler lock so the keep - # check stays lock-free below. - with runtime.lock: - published = getattr(runtime, "current_tip_first_seen", None) - snapshot = getattr(runtime, "tip_template_snapshot", None) - snapshot_tip = getattr(snapshot, "bestblockhash", None) - snapshot_fingerprint = getattr( - snapshot, "template_fingerprint", None - ) - if ( - published is not None - and snapshot_tip == published[0] - and snapshot_fingerprint is not None - and runtime._published_tip_authoritative_locked(time.monotonic()) - ): - published_parent = published[0] - published_fingerprint = snapshot_fingerprint - - def keep(request: JobBuildRequest) -> bool: - return bool( - published_fingerprint is not None - and request.artifacts.previousblockhash == published_parent - and request.artifacts.fingerprint == published_fingerprint - ) - - with self._job_build_scheduler_lock: - for flight in (self._job_build_active, self._job_build_retiring): - if flight is not None and not keep(flight.request): - runtime._cancel_job_build_flight_locked(flight, reason) - pending = self._job_build_pending - if pending is not None and not keep(pending): - pending.cancellation.cancel(reason) - if not pending.promise.done(): - pending.promise.set_exception( - JobBuildSuperseded(f"pending job build {reason}") - ) - self._job_build_pending = None - self.job_build_scheduler_counts["supersessions"] += 1 - - def shutdown_job_build_executor(self) -> None: - runtime = self._runtime - runtime._ensure_job_cache_state() - with self._job_build_scheduler_lock: - for flight in (self._job_build_active, self._job_build_retiring): - if flight is None: - continue - flight.request.cancellation.cancel("shutdown") - future = getattr(flight, "future", None) - if future is None or future.done(): - # A terminal future's callback has already run (or died) - # and executor shutdown will not fire it again: this is - # the last chance to settle the flight's waiters. - runtime._resolve_cancelled_job_build_promise( - flight.request, - "shutdown", - ) - pending = self._job_build_pending - if pending is not None: - pending.cancellation.cancel("shutdown") - if not pending.promise.done(): - pending.promise.set_exception( - JobBuildSuperseded("pending job build cancelled by shutdown") - ) - self._job_build_pending = None - executor = self._job_build_executor - self._job_build_executor = None - self._job_build_executor_shutdown = True - if executor is not None: - executor.shutdown(wait=True, cancel_futures=True) - - # -- cache identity and admission -------------------------------------- - - def _job_bundle_payout_state_current(self, bundle: CachedJobBundle) -> bool: - runtime = self._runtime - runtime._ensure_job_cache_state() - with self._job_cache_lock: - artifact = runtime._published_payout_state.artifact - return bool( - bundle.payout_state_generation == runtime._payout_state_generation - and bundle.build_key is not None - and ( - bundle.collection_only - or int( - getattr( - bundle.build_key, - "payout_append_invalidation_epoch", - 0, - ) - ) - == runtime._payout_ledger_append_invalidation_epoch - ) - and artifact is not None - and bundle.build_key.payout_artifact_sha256 - == artifact.prior_balances_sha256 - ) - - @staticmethod - def _collection_bundle_identity(worker: WorkerIdentityPort) -> tuple[str, str]: - return worker.payout_address, worker.p2mr_program_hex - - def _job_bundle_key( - self, - artifacts: CachedTemplateArtifacts, - *, - mode: str, - payout_state_generation: int, - payout_artifact_generation: int = 0, - worker: WorkerIdentityPort | None, - ) -> tuple[object, ...]: - runtime = self._runtime - if mode == "ready": - return ( - artifacts.fingerprint, - artifacts.previousblockhash, - "ready", - payout_state_generation, - payout_artifact_generation, - ) - if mode != "collection": - raise ValueError(f"unknown PRISM job-bundle mode: {mode}") - if worker is None: - raise CollectionIdentityUnavailable( - "collection-mode worker identity is temporarily unavailable" - ) - return ( - artifacts.fingerprint, - artifacts.previousblockhash, - "collection", - artifacts.generation, - payout_state_generation, - payout_artifact_generation, - *runtime._collection_bundle_identity(worker), - ) - - def _job_bundle_mode(self, requested_mode: str | None) -> str: - runtime = self._runtime - if requested_mode is not None: - if requested_mode not in {"ready", "collection"}: - raise ValueError( - f"unknown PRISM job-bundle mode: {requested_mode}" - ) - return requested_mode - return "ready" if runtime.pool_readiness_latched() else "collection" - - def _lookup_job_bundle( - self, - key: tuple[object, ...], - ) -> CachedJobBundle | None: - runtime = self._runtime - ttl = getattr(runtime, "job_bundle_cache_seconds", DEFAULT_PRISM_JOB_BUNDLE_CACHE_SECONDS) - now = time.monotonic() - with self._job_cache_lock: - if ttl <= 0: - self._job_bundle_cache.clear() - return None - # The entry-count cap is not a memory bound: one production entry - # can reference more than 100k shares. Expired entries must release - # their snapshots instead of remaining resident until count eviction. - expired = [ - cache_key - for cache_key, entry in self._job_bundle_cache.items() - if now - entry.built_monotonic > ttl - ] - for cache_key in expired: - self._job_bundle_cache.pop(cache_key, None) - return self._job_bundle_cache.get(key) - return None - - def _no_artifact_job_bundle_key( - self, - artifacts: CachedTemplateArtifacts, - *, - mode: str, - payout_state_generation: int, - worker: WorkerIdentityPort | None, - ) -> tuple[object, ...]: - """Fallback cache identity for work built before an artifact re-key. - - A synchronous build caches its bundle under the no-artifact key and - then publishes its anchored window as the payout ledger artifact. The - next lookup is keyed to that artifact and would miss the still-fresh - bundle and rebuild identical work: within one payout generation and - template identity the no-artifact bundle binds the same balances, - window anchor, and policy inputs, so it remains a correct cache - identity to serve -- but only while its window digest matches the - armed artifact (see _no_artifact_bundle_matches_artifact); a fence - re-arm can install a fresher window that cached work must not shadow. - """ - return self._runtime._job_bundle_key( - artifacts, - mode=mode, - payout_state_generation=payout_state_generation, - payout_artifact_generation=0, - worker=worker, - ) - - @staticmethod - def _no_artifact_bundle_matches_artifact( - cached: CachedJobBundle, - payout_artifact: Any, - ) -> bool: - """Whether pre-re-key cached work carries exactly the artifact's window. - - A speculative rebuild can arm a fresher artifact while an older - no-artifact bundle is still inside its cache TTL. Serving that bundle - would shadow the fresher window behind cached work, so the fallback - identity is honored only when the window digests are byte-identical - -- which the bundle that published the artifact satisfies by - construction. - """ - return ( - payout_artifact is not None - and payout_artifact.share_snapshot_sha256 is not None - and cached.build_key is not None - and cached.build_key.share_snapshot_sha256 - == payout_artifact.share_snapshot_sha256 - ) - - def _job_bundle_entry_usable( - self, - cached: CachedJobBundle | None, - artifacts: CachedTemplateArtifacts, - ) -> bool: - """Re-validate freshness and readiness for cached bundles. - - Readiness is monotonic in practice (the distinct accepted-miner count - only grows), so submit-capable ready bundles are served as-is while - their declared window anchor stays inside the artifact staleness - bound. A cached collection bundle is re-checked against the cheap - aggregate stats: once the pool is ready it must stop being served, or - jobs would keep collecting winning shares without submitting blocks - for up to the cache TTL. - """ - runtime = self._runtime - if cached is None: - return False - # Usability follows the same split-authority rule as construction: a - # cached bundle for the newest detected tip stays reusable across - # refresh retries while the previous tip is still published, and the - # published snapshot's own bundle stays servable for pinned issuance. - with runtime.lock: - parent_usable = runtime._artifacts_buildable_locked(artifacts) - if not parent_usable: - return False - with self._job_cache_lock: - if runtime._payout_state_publication_blocked: - return False - if not runtime._job_bundle_payout_state_current(cached): - return False - if not cached.collection_only: - # Cached-ready-bundle validity follows the same ceiling/floor - # split as artifact reuse. The declared anchor is gated only by - # the audit CEILING: the anchor is frozen per template - # generation and predates the window walk, so a tight - # wall-clock gate here declared every rebuilt bundle dead on - # arrival once its template generation outlived the bound - # (2026-07-29 rollback). - found_block = getattr(cached, "found_block", None) - declared_anchor_ms = ( - found_block.get("anchor_job_issued_at_ms") - if isinstance(found_block, dict) - else None - ) - if declared_anchor_ms is not None and ( - self._now_ms() - int(declared_anchor_ms) - > runtime._payout_artifact_max_anchor_age_ms() - ): - return False - # A bundle keyed to the currently armed artifact generation - # carries exactly the armed window, so it serves for as long as - # reuse itself does (the armed artifact is event-invalidated by - # the payout fences and re-anchored in the background). Every - # other ready bundle (no-artifact builds and survivors of a - # re-key) carries a window nothing will re-anchor, so it keeps - # serving only inside the re-anchor floor -- its declared - # anchor is already ceiling-gated above, and a wall-clock - # budget tighter than the build cadence is exactly the - # 2026-07-30 brownout (every consecutive build fell back to the - # synchronous reward-window walk). - if int(getattr(cached, "payout_artifact_generation", 0)) > 0: - with self._job_cache_lock: - armed = runtime._payout_ledger_artifact - if ( - armed is not None - and int(armed.generation) - == int(cached.payout_artifact_generation) - ): - return True - return ( - time.monotonic() - float(cached.built_monotonic) - <= runtime._payout_artifact_reanchor_seconds() - ) - # Collection bundles sign a synthetic bootstrap share containing the - # exact template ntime. A clock-only observation keeps the stable work - # fingerprint, but it must rebuild this signed bundle instead of - # rebinding the old manifest to a new template generation. - if ( - cached.template is not artifacts.template - or cached.template_generation != artifacts.generation - ): - return False - try: - _, ready_miner_count = runtime.accepted_share_stats() - except Exception: - # If readiness cannot be proven, force the normal build path. That - # path will either build an up-to-date bundle or surface the ledger - # failure instead of continuing to issue no-submit collection jobs. - return False - return ready_miner_count < runtime.min_ready_miners - - def _bind_cached_bundle_to_artifacts( - self, - cached: CachedJobBundle, - artifacts: CachedTemplateArtifacts, - ) -> CachedJobBundle: - """Return the cached heavy bundle bound to this exact observation. - - Clock-only template changes intentionally keep the stable fingerprint. - Ready bundles may reuse their ledger snapshot and signed manifest, but - the Stratum base job must still carry the observing template's exact - ntime and generation. Collection bundles are filtered before this point - because their signed synthetic share contains the template ntime. - """ - runtime = self._runtime - if ( - cached.template is artifacts.template - and cached.template_generation == artifacts.generation - ): - return cached - manifest = cached.coinbase_manifest - base_job = direct_stratum.make_job_from_builder_manifest( - job_id="prism-template-base", - template=artifacts.template, - manifest=manifest, - extranonce1_hex=PRISM_JOB_EXTRANONCE1_PLACEHOLDER_HEX, - extranonce2_size=runtime.extranonce2_size, - desired_share_difficulty=runtime.share_difficulty, - clean_jobs=True, - transaction_hexes=artifacts.transaction_hexes, - ) - return dataclass_replace( - cached, - template=artifacts.template, - base_job=base_job, - template_generation=artifacts.generation, - build_key=( - dataclass_replace( - cached.build_key, - best_tip_hash=artifacts.previousblockhash, - previous_block_hash=artifacts.previousblockhash, - template_generation=artifacts.generation, - block_height=int(artifacts.template["height"]), - coinbase_value_sats=int(artifacts.template["coinbasevalue"]), - ) - if cached.build_key is not None - else None - ), - ) - - def _new_job_build_request( - self, - artifacts: CachedTemplateArtifacts, - worker: WorkerIdentityPort | None, - *, - mode: str, - payout_state_generation: int, - cache_key: tuple[object, ...], - payout_ledger_artifact: Any = None, - idle_retarget: bool = False, - publication_critical: bool = False, - request_source: str = "routine", - priority_requested_monotonic: float | None = None, - preparation_cancellation: JobBuildCancellation | None = None, - ) -> JobBuildRequest: - runtime = self._runtime - cancellation = ( - runtime._new_job_build_cancellation() - if preparation_cancellation is None - else preparation_cancellation - ) - cancellation.raise_if_cancelled("immutable snapshot") - # A pending accepted-parent transition owns the balances children must - # observe. Wait for its preview before acquiring any confirmed payout - # input so a child build cannot take -- or cache into the published - # artifact -- confirmed state that omits its new parent. The preview - # itself is re-resolved consistently during construction. - runtime._await_pending_parent_payout_preview( - artifacts.previousblockhash, - parent_height=int(artifacts.template["height"]) - 1, - ) - payout_artifact = runtime._current_payout_state_artifact(cancellation) - if payout_artifact.generation != payout_state_generation: - raise JobBuildSuperseded( - "payout artifact generation changed before build request" - ) - - phases = runtime._job_build_phases() - payout_started = time.monotonic() - payout_policy_json = canonical_json_text(runtime.prism_payout_policy()) - phases["payout"] = phases.get("payout", 0.0) + ( - time.monotonic() - payout_started - ) - cancellation.raise_if_cancelled("payout policy") - ctv_started = time.monotonic() - ctv_settlement = runtime.prism_ctv_settlement_config( - block_height=int(artifacts.template["height"]), - parent_hash=artifacts.previousblockhash, - ) - ctv_settlement_json = ( - canonical_json_text(ctv_settlement) - if ctv_settlement is not None - else None - ) - phases["ctv"] = phases.get("ctv", 0.0) + ( - time.monotonic() - ctv_started - ) - cancellation.raise_if_cancelled("CTV configuration") - - suffix_hex = runtime.coinbase_script_sig_suffix_hex( - PRISM_JOB_EXTRANONCE1_PLACEHOLDER_HEX, - "00" * runtime.extranonce2_size, - ) - collection_identity = ( - runtime._collection_bundle_identity(worker) - if mode == "collection" and worker is not None - else None - ) - decimal_context = getcontext().copy() - numeric_context_sha256 = self._canonical_json_sha256( - { - "precision": decimal_context.prec, - "rounding": decimal_context.rounding, - "minimum_exponent": decimal_context.Emin, - "maximum_exponent": decimal_context.Emax, - "capitals": decimal_context.capitals, - "clamp": decimal_context.clamp, - } - ) - with self._job_cache_lock: - issued_at_ms = self._job_build_issued_at_ms.get(artifacts.generation) - current_append_invalidation_epoch = int( - runtime._payout_ledger_append_invalidation_epoch - ) - if issued_at_ms is None: - # The issued time doubles as the audit window anchor, so it - # must not cover a stamped share whose commit is still in - # flight: the frozen anchor stays reproducible from the - # durable ledger for every rebuild of this generation. The - # pending-commit clamp is pure anchor selection -- it freezes - # the highest anchor whose covered shares are all durable, and - # shares stamped above it deterministically belong to the next - # window -- so nothing else needs to be captured with it. - candidate_anchor_ms = runtime._job_snapshot_anchor_ms(self._now_ms()) - with self._job_cache_lock: - issued_at_ms = self._job_build_issued_at_ms.get( - artifacts.generation - ) - if issued_at_ms is None: - issued_at_ms = candidate_anchor_ms - self._job_build_issued_at_ms[artifacts.generation] = ( - issued_at_ms - ) - while len(self._job_build_issued_at_ms) > 128: - self._job_build_issued_at_ms.popitem(last=False) - build_append_invalidation_epoch = ( - int(payout_ledger_artifact.append_invalidation_epoch) - if mode == "ready" and payout_ledger_artifact is not None - else current_append_invalidation_epoch - if mode == "ready" - else 0 - ) - build_key = JobBuildKey( - best_tip_hash=artifacts.previousblockhash, - previous_block_hash=artifacts.previousblockhash, - template_fingerprint=artifacts.fingerprint, - template_generation=artifacts.generation, - payout_state_generation=payout_state_generation, - payout_artifact_sha256=payout_artifact.prior_balances_sha256, - mode=mode, - collection_identity=collection_identity, - block_height=int(artifacts.template["height"]), - coinbase_value_sats=int(artifacts.template["coinbasevalue"]), - network_difficulty=int(artifacts.network_difficulty), - issued_at_ms=issued_at_ms, - payout_policy_sha256=hashlib.sha256( - payout_policy_json.encode() - ).hexdigest(), - ctv_settlement_sha256=( - hashlib.sha256(ctv_settlement_json.encode()).hexdigest() - if ctv_settlement_json is not None - else None - ), - witness_merkle_sha256=self._canonical_json_sha256( - artifacts.witness_merkle_leaves_hex - ), - transaction_set_sha256=self._canonical_json_sha256( - artifacts.transaction_hexes - ), - coinbase_suffix_hex=suffix_hex, - signing_key_sha256=hashlib.sha256( - str(getattr(runtime, "signing_seed_hex", "")).encode() - ).hexdigest(), - ledger_signing_key_sha256=hashlib.sha256( - str( - getattr( - runtime, - "ledger_attestation_signing_seed_hex", - "", - ) - ).encode() - ).hexdigest(), - numeric_context_sha256=numeric_context_sha256, - payout_append_invalidation_epoch=( - build_append_invalidation_epoch - ), - ) - immutable_identity: tuple[object, ...] = ( - cache_key, - artifacts.generation, - issued_at_ms, - payout_artifact.prior_balances_sha256, - build_key.payout_policy_sha256, - build_key.ctv_settlement_sha256, - build_key.witness_merkle_sha256, - build_key.transaction_set_sha256, - build_key.coinbase_suffix_hex, - build_key.signing_key_sha256, - build_key.ledger_signing_key_sha256, - build_key.numeric_context_sha256, - build_key.payout_append_invalidation_epoch, - ) - return JobBuildRequest( - key=build_key, - cache_key=cache_key, - equivalence_key=immutable_identity, - artifacts=artifacts, - template_json=canonical_json_text(artifacts.template), - transaction_hexes=artifacts.transaction_hexes, - witness_merkle_leaves_hex=artifacts.witness_merkle_leaves_hex, - worker=worker, - mode=mode, - payout_artifact=payout_artifact, - payout_ledger_artifact=payout_ledger_artifact, - payout_policy_json=payout_policy_json, - ctv_settlement_json=ctv_settlement_json, - decimal_context=decimal_context, - cancellation=cancellation, - idle_retarget=idle_retarget, - publication_critical=publication_critical, - request_source=request_source, - requested_monotonic=( - cancellation.started_monotonic - if priority_requested_monotonic is None - else priority_requested_monotonic - ), - ) - - def _cache_job_bundle_if_current( - self, - built: CachedJobBundle, - artifacts: CachedTemplateArtifacts, - ) -> bool: - """Cache only current state; report whether payout state stayed valid.""" - runtime = self._runtime - with self._job_cache_lock: - with runtime.lock: - buildable = runtime._artifacts_buildable_locked(artifacts) - snapshot_pinned = runtime._published_snapshot_artifacts_locked( - artifacts - ) - published_artifact = runtime._published_payout_state.artifact - if not buildable: - return False - if ( - built.payout_state_generation != runtime._payout_state_generation - or built.build_key is None - or ( - not built.collection_only - and built.build_key.payout_append_invalidation_epoch - != runtime._payout_ledger_append_invalidation_epoch - ) - or published_artifact is None - or built.build_key.payout_artifact_sha256 - != published_artifact.prior_balances_sha256 - ): - return False - current = runtime._template_artifacts - globally_current = ( - current is not None - and current.fingerprint == artifacts.fingerprint - and current.previousblockhash == artifacts.previousblockhash - and ( - not built.collection_only - or current.generation == artifacts.generation - ) - ) - if not globally_current and not snapshot_pinned: - # Only the current template observation may win the cache - # race; the sole exception is a pinned rebuild of exactly the - # published snapshot, which repeated direct issuance would - # otherwise rebuild for the rest of the unpublished window. - return False - self._job_bundle_cache[built.key] = built - self._job_bundle_cache.move_to_end(built.key) - while len(self._job_bundle_cache) > MAX_PRISM_JOB_BUNDLE_CACHE_ENTRIES: - oldest_key = next(iter(self._job_bundle_cache)) - self._job_bundle_cache.pop(oldest_key, None) - # Arm the build's fenced ledger window only after the bundle itself - # won cache publication: lookups re-keyed by the fresh artifact fall - # back to exactly this cached no-artifact entry instead of forking a - # redundant build. - if built.prepared_ledger_artifact is not None: - runtime._install_payout_ledger_artifact(built.prepared_ledger_artifact) - return True - - def _probe_initial_job_bundle( - self, - artifacts: CachedTemplateArtifacts, - worker: WorkerIdentityPort | None, - mode: str | None, - ) -> CachedJobBundle | None: - """Serve an initial request straight from the bundle cache. - - Routine admission exists to keep CPU-heavy request construction from - competing with a publication-critical build; a cache probe does none - of that work. Probing before admission lets a first job ship the - already-published bundle immediately instead of waiting out an entire - priority build it would never have contended with. - """ - runtime = self._runtime - resolved_mode = runtime._job_bundle_mode(mode) - if resolved_mode == "collection" and worker is None: - raise CollectionIdentityUnavailable( - "collection-mode worker identity is temporarily unavailable" - ) - with self._job_cache_lock: - payout_state_generation = runtime._payout_state_generation - payout_artifact = ( - runtime._usable_payout_ledger_artifact( - payout_state_generation, - artifacts.network_difficulty, - ) - if resolved_mode == "ready" - else None - ) - key = runtime._job_bundle_key( - artifacts, - mode=resolved_mode, - payout_state_generation=payout_state_generation, - payout_artifact_generation=( - payout_artifact.generation if payout_artifact is not None else 0 - ), - worker=worker, - ) - cached = runtime._lookup_job_bundle(key) - if cached is None and payout_artifact is not None: - fallback = runtime._lookup_job_bundle( - runtime._no_artifact_job_bundle_key( - artifacts, - mode=resolved_mode, - payout_state_generation=payout_state_generation, - worker=worker, - ) - ) - if fallback is not None and runtime._no_artifact_bundle_matches_artifact( - fallback, - payout_artifact, - ): - cached = fallback - if not runtime._job_bundle_entry_usable(cached, artifacts): - return None - runtime._record_job_cache_event("bundle", hit=True) - with self._job_build_scheduler_lock: - self.initial_job_prepared_work_counts["cache_hit"] += 1 - assert cached is not None - return runtime._bind_cached_bundle_to_artifacts(cached, artifacts) - - def shared_job_bundle( - self, - artifacts: CachedTemplateArtifacts, - worker: WorkerIdentityPort | None = None, - *, - mode: str | None = None, - cancelled: Callable[[], bool] | None = None, - retry_superseded: bool = True, - idle_retarget: bool = False, - publication_critical: bool = False, - request_source: str = "routine", - priority_requested_monotonic: float | None = None, - ) -> CachedJobBundle: - """Return one immutable heavy build through a work-identity flight. - - Equivalent callers share one future. Publication-critical refreshes - can cancel routine work and start in the bounded replacement slot; - reconnect, Vardiff, authorization, and same-tip work must defer while - that priority work is active. Only the latest pending request is - retained during repeated supersession. - """ - runtime = self._runtime - runtime._ensure_job_cache_state() - runtime._ensure_tip_refresh_state() - priority_token: int | None = None - if publication_critical: - ( - priority_token, - priority_requested_monotonic, - ) = runtime._begin_job_build_priority_preparation( - priority_requested_monotonic - ) - try: - return runtime._shared_job_bundle_after_priority_admission( - artifacts, - worker, - mode=mode, - cancelled=cancelled, - retry_superseded=retry_superseded, - idle_retarget=idle_retarget, - publication_critical=publication_critical, - request_source=request_source, - priority_requested_monotonic=priority_requested_monotonic, - ) - finally: - if priority_token is not None: - runtime._finish_job_build_priority_preparation(priority_token) - - def _shared_job_bundle_after_priority_admission( - self, - artifacts: CachedTemplateArtifacts, - worker: WorkerIdentityPort | None = None, - *, - mode: str | None = None, - cancelled: Callable[[], bool] | None = None, - retry_superseded: bool = True, - idle_retarget: bool = False, - publication_critical: bool = False, - request_source: str = "routine", - priority_requested_monotonic: float | None = None, - ) -> CachedJobBundle: - runtime = self._runtime - while True: - routine_preparation_token: int | None = None - preparation_cancellation: JobBuildCancellation | None = None - subscribed_bundle: CachedJobBundle | None = None - if not publication_critical: - if request_source == "initial": - probed = runtime._probe_initial_job_bundle( - artifacts, - worker, - mode, - ) - if probed is not None: - return probed - ( - routine_preparation_token, - preparation_cancellation, - subscribed_bundle, - ) = runtime._begin_routine_job_build_preparation( - request_source=request_source, - cancelled=cancelled, - ) - resolved_mode = runtime._job_bundle_mode(mode) - if preparation_cancellation is not None: - preparation_cancellation.raise_if_cancelled( - "request preparation admission" - ) - if resolved_mode == "collection" and worker is None: - raise CollectionIdentityUnavailable( - "collection-mode worker identity is temporarily unavailable" - ) - with self._job_cache_lock: - payout_state_generation = runtime._payout_state_generation - payout_artifact = ( - runtime._usable_payout_ledger_artifact( - payout_state_generation, - artifacts.network_difficulty, - ) - if resolved_mode == "ready" - else None - ) - if preparation_cancellation is not None: - preparation_cancellation.raise_if_cancelled( - "payout artifact lookup" - ) - payout_artifact_generation = ( - payout_artifact.generation if payout_artifact is not None else 0 - ) - key = runtime._job_bundle_key( - artifacts, - mode=resolved_mode, - payout_state_generation=payout_state_generation, - payout_artifact_generation=payout_artifact_generation, - worker=worker, - ) - no_artifact_key = ( - runtime._no_artifact_job_bundle_key( - artifacts, - mode=resolved_mode, - payout_state_generation=payout_state_generation, - worker=worker, - ) - if payout_artifact_generation != 0 - else None - ) - cached = runtime._lookup_job_bundle(key) - if ( - cached is None - and subscribed_bundle is not None - and ( - subscribed_bundle.key == key - or ( - no_artifact_key is not None - and subscribed_bundle.key == no_artifact_key - and runtime._no_artifact_bundle_matches_artifact( - subscribed_bundle, - payout_artifact, - ) - ) - ) - ): - # The subscribed priority result may not have won the global - # cache-publication race yet; consume it directly. Usability - # below applies exactly the checks a cache entry would face. - cached = subscribed_bundle - if cached is None and no_artifact_key is not None: - fallback = runtime._lookup_job_bundle(no_artifact_key) - if ( - fallback is not None - and runtime._no_artifact_bundle_matches_artifact( - fallback, - payout_artifact, - ) - ): - cached = fallback - if runtime._job_bundle_entry_usable(cached, artifacts): - if preparation_cancellation is not None: - preparation_cancellation.raise_if_cancelled( - "bundle cache lookup" - ) - if routine_preparation_token is not None: - runtime._finish_routine_job_build_preparation( - routine_preparation_token - ) - runtime._record_job_cache_event("bundle", hit=True) - if request_source == "initial": - with self._job_build_scheduler_lock: - self.initial_job_prepared_work_counts["cache_hit"] += 1 - assert cached is not None - return runtime._bind_cached_bundle_to_artifacts(cached, artifacts) - if runtime._job_bundle_mode(mode) != resolved_mode: - if routine_preparation_token is not None: - runtime._finish_routine_job_build_preparation( - routine_preparation_token - ) - continue - runtime._record_job_cache_event("bundle", hit=False) - try: - request = runtime._new_job_build_request( - artifacts, - worker, - mode=resolved_mode, - payout_state_generation=payout_state_generation, - cache_key=key, - payout_ledger_artifact=payout_artifact, - idle_retarget=idle_retarget, - publication_critical=publication_critical, - request_source=request_source, - priority_requested_monotonic=( - priority_requested_monotonic - ), - preparation_cancellation=preparation_cancellation, - ) - # Preserve the historical readiness handoff without holding a - # lock across construction: only admission and the final mode - # re-selection are serialized here. - with self._job_build_lock: - with self._job_build_scheduler_lock: - if routine_preparation_token is not None: - runtime._finish_routine_job_build_preparation( - routine_preparation_token - ) - routine_preparation_token = None - request.cancellation.raise_if_cancelled( - "scheduler admission" - ) - if runtime._job_bundle_mode(mode) != resolved_mode: - request.cancellation.cancel( - "worker mode superseded" - ) - continue - if cancelled is not None and cancelled(): - raise JobBuildWaiterCancelled( - "job bundle request was cancelled before preparation" - ) - promise = runtime._request_job_build(request) - wait_deadline = time.monotonic() + max( - 0.001, - float(runtime.job_build_timeout_seconds) - + float(runtime.job_build_cancel_grace_seconds) - + 1.0, - ) - while True: - if cancelled is not None and cancelled(): - raise JobBuildWaiterCancelled( - "job bundle waiter was cancelled during preparation" - ) - try: - built = _await_job_build_promise( - promise, - min( - 0.1, - max(0.001, wait_deadline - time.monotonic()), - ), - ) - break - except TimeoutError: - if time.monotonic() >= wait_deadline: - raise - except TimeoutError as exc: - request.cancellation.cancel("timeout") - runtime._schedule_tip_refresh_retry() - raise JobBuildCancelled( - "job build timed out; immediate retry scheduled" - ) from exc - except JobBuildCancelled: - if routine_preparation_token is not None: - runtime._finish_routine_job_build_preparation( - routine_preparation_token - ) - runtime._schedule_tip_refresh_retry() - if not retry_superseded: - raise - with runtime.lock: - buildable = runtime._artifacts_buildable_locked(artifacts) - if not buildable: - raise - with self._job_cache_lock: - current = runtime._template_artifacts - payout_current = ( - payout_state_generation == runtime._payout_state_generation - ) - if current is artifacts and payout_current: - continue - if current is artifacts: - continue - raise - built = runtime._bind_cached_bundle_to_artifacts(built, artifacts) - if not runtime._cache_job_bundle_if_current(built, artifacts): - with self._job_build_scheduler_lock: - self.job_build_scheduler_counts["obsolete_results"] += 1 - with runtime.lock: - buildable = runtime._artifacts_buildable_locked(artifacts) - if not buildable: - with runtime._tip_refresh_metrics_lock: - runtime.tip_refresh_superseded_results += 1 - raise JobBuildSuperseded( - "observed tip changed before cache publication" - ) - with self._job_cache_lock: - payout_current = ( - built.payout_state_generation - == runtime._payout_state_generation - and ( - built.collection_only - or ( - built.build_key is not None - and built.build_key.payout_append_invalidation_epoch - == runtime._payout_ledger_append_invalidation_epoch - ) - ) - ) - with runtime.lock: - published_snapshot = getattr( - runtime, - "tip_template_snapshot", - None, - ) - published_artifacts = ( - published_snapshot.template_artifacts - if published_snapshot is not None - else None - ) - if ( - payout_current - and built.template is artifacts.template - and built.template_generation == artifacts.generation - and ( - not retry_superseded - or published_artifacts is artifacts - ) - ): - # Snapshot-owned work may outlive an unrelated global - # cache fill. Return it only to its refresh validator or - # retained collection delivery; never retain it globally. - if built.prepared_ledger_artifact is not None: - runtime._install_payout_ledger_artifact( - built.prepared_ledger_artifact - ) - return built - if retry_superseded: - with self._job_cache_lock: - current = runtime._template_artifacts - if current is artifacts: - continue - raise JobBuildSuperseded( - "job build key changed before cache publication" - ) - return built - - def _share_window_serialization_for_artifact( - self, - payout_artifact: Any, - shares: list[dict[str, object]], - ) -> _ShareWindowSerialization: - """Cached digest and builder fragments for the artifact's share window. - - Single-slot cache keyed by the canonical content digest and share - count. A debounced payout generation can retag an - unchanged window, so generation identity would unnecessarily rebuild - and re-spool the same multi-megabyte compact payload. The compute - deliberately takes no cancellation checkpoints -- the result outlives - any one requester and is served to whichever build wins next. - """ - self._runtime._ensure_job_cache_state() - # The digest is computed under the slot lock on purpose: the two - # bounded build flights can request the same window concurrently, and - # letting both walk the share tree would duplicate exactly the - # GIL-heavy pass this cache removes. The loser waits briefly and - # reuses the winner's entry; a different-key waiter is serialized - # behind at most one O(window) pass. - with self._share_window_serialization_lock: - cached = self._share_window_serialization - if ( - payout_artifact.share_snapshot_sha256 is None - and cached is not None - and cached._source_artifact is payout_artifact - and cached.share_count == len(shares) - ): - return cached - share_snapshot_sha256 = ( - payout_artifact.share_snapshot_sha256 - or self._canonical_json_sha256(share_array_json_view(shares)) - ) - # The compact payload is a pure function of the shares (see the - # class docstring), so the digest and count fully identify it. - # The live-difficulty window weight was deliberately dropped - # from this key: ASERT retargets every block, so a weight - # component rotated the key -- and re-spooled an identical - # multi-megabyte payload -- per block. - key = (share_snapshot_sha256, len(shares)) - if ( - cached is not None - and cached.key == key - and cached.share_count == len(shares) - ): - cached._source_artifact = payout_artifact - return cached - serialization = _ShareWindowSerialization( - key=key, - share_count=len(shares), - share_snapshot_sha256=share_snapshot_sha256, - _source_artifact=payout_artifact, - _spool_factory=self._spool_factory, - ) - self._share_window_serialization = serialization - if cached is not None: - # Generation rotation retires the replaced spool; a build still - # holding a lease keeps the descriptor alive until it releases. - cached.retire_spool() - return serialization - - def retire_share_window_spool(self) -> None: - """Release the cached share-window spool during shutdown. - - Runs after the build executors quiesce, so any lease still held by a - draining transfer defers the close to its own release. - """ - self._runtime._ensure_job_cache_state() - with self._share_window_serialization_lock: - serialization = self._share_window_serialization - if serialization is not None: - serialization.retire_spool() - - def build_shared_job_bundle( - self, - artifacts: CachedTemplateArtifacts, - worker: WorkerIdentityPort | None = None, - *, - mode: str | None = None, - payout_state_generation: int | None = None, - payout_artifact: Any = None, - key: tuple[object, ...] | None = None, - build_request: JobBuildRequest | None = None, - ) -> CachedJobBundle: - runtime = self._runtime - phases = runtime._job_build_phases() - resolved_mode = runtime._job_bundle_mode(mode) - if resolved_mode == "collection" and worker is None: - raise CollectionIdentityUnavailable( - "collection-mode worker identity is temporarily unavailable" - ) - with self._job_cache_lock: - publication_blocked = runtime._payout_state_publication_blocked - if payout_state_generation is None: - payout_state_generation = runtime._payout_state_generation - if publication_blocked: - raise PayoutStatePublicationBlocked( - "payout state invalidation is pending publication" - ) - if key is None: - key = runtime._job_bundle_key( - artifacts, - mode=resolved_mode, - payout_state_generation=payout_state_generation, - payout_artifact_generation=( - payout_artifact.generation - if payout_artifact is not None - else 0 - ), - worker=worker, - ) - if build_request is None: - build_request = runtime._new_job_build_request( - artifacts, - worker, - mode=resolved_mode, - payout_state_generation=payout_state_generation, - cache_key=key, - payout_ledger_artifact=payout_artifact, - ) - else: - payout_artifact = build_request.payout_ledger_artifact - cancellation = build_request.cancellation - runtime._job_build_checkpoint("ledger_snapshot", cancellation) - template_value = json.loads(build_request.template_json) - if not isinstance(template_value, dict): - raise RuntimeError("immutable job template is not an object") - template: dict[str, Any] = template_value - issued_at_ms = build_request.key.issued_at_ms - started = time.monotonic() - snapshot_window_weight = ( - PRISM_REWARD_WINDOW_MULTIPLIER - * PRISM_SNAPSHOT_WINDOW_MARGIN - * int(build_request.key.network_difficulty) - ) - prepared_ledger_artifact: Any = None - snapshot_accepted_count: int | None = None - inflight_scan_anchor_token: int | None = None - if payout_artifact is not None: - # The reuse decision was made at request preparation. Nothing - # that happens to the ARMED slot afterwards -- a fresher window - # installing (generation re-key), a same-window anchor refresh - # swapping the instance, or the artifact simply aging out of new - # reuse decisions -- invalidates the copy this build already - # selected: its window stays audit-reproducible at its declared - # anchor. Scrapping admitted work on any of those (as the first - # anchor-scoped deploy did on the generation re-key) turns - # routine churn into a rebuild storm. Only two fences gate - # completion: the payout generation and the published balances - # this artifact binds. - with self._job_cache_lock: - current_payout_state_generation = runtime._payout_state_generation - published_artifact = runtime._published_payout_state.artifact - current_append_invalidation_epoch = ( - runtime._payout_ledger_append_invalidation_epoch - ) - if current_payout_state_generation != payout_state_generation: - raise JobBuildSuperseded( - "payout generation changed before construction" - ) - if ( - payout_artifact.append_invalidation_epoch - != current_append_invalidation_epoch - ): - raise JobBuildSuperseded( - "payout window invalidated before construction" - ) - if published_artifact is None: - try: - published_artifact = runtime._current_payout_state_artifact() - except Exception as exc: - raise JobBuildSuperseded( - "published payout state unavailable before construction" - ) from exc - artifact_balances_sha256 = ( - payout_artifact.prior_balances_sha256 - or self._canonical_json_sha256(payout_artifact.prior_balances) - ) - if ( - artifact_balances_sha256 - != published_artifact.prior_balances_sha256 - ): - raise JobBuildSuperseded( - "precomputed payout artifact changed before construction" - ) - prior_balances = list(payout_artifact.prior_balances) - if ( - artifact_balances_sha256 - != build_request.key.payout_artifact_sha256 - ): - raise JobBuildSuperseded( - "precomputed payout artifact does not match payout generation" - ) - # A daemon-mirror sequence is kept as-is rather than copied into - # a list: it parses its dicts lazily, and the routine build path - # must not materialize dicts it never consumes. The rare - # consumers that need real dicts (the durable candidate intent, - # the canonical audit build) coerce at their own call sites. - # Every other artifact sequence keeps the historical list copy. - artifact_shares_json = payout_artifact.shares_json - shares = ( - artifact_shares_json - if isinstance(artifact_shares_json, DaemonShareJsonSequence) - else list(artifact_shares_json) - ) - # The artifact binds the key above; the balances actually used may - # still be an accepted parent's prospective carry state. - prior_balances = runtime._prior_balances_for_job_parent( - str(template["previousblockhash"]), - parent_height=int(template["height"]) - 1, - fallback_balances=prior_balances, - ) - else: - with runtime._payout_state_prepare_lock: - with self._job_cache_lock: - published_artifact = runtime._published_payout_state.artifact - if runtime._payout_state_publication_blocked: - raise PayoutStatePublicationBlocked( - "payout state invalidation is pending publication" - ) - if ( - payout_state_generation != runtime._payout_state_generation - or build_request.key.payout_append_invalidation_epoch - != runtime._payout_ledger_append_invalidation_epoch - or published_artifact is None - or published_artifact.prior_balances_sha256 - != build_request.key.payout_artifact_sha256 - ): - raise JobBuildSuperseded( - "payout generation changed before ledger snapshot" - ) - if resolved_mode == "ready": - try: - accepted_now, _ = runtime.accepted_share_stats() - except Exception: - # Artifact seeding is speculative; snapshot errors - # stay owned by the bundle build itself. Without an - # observed count the seed below is skipped rather - # than armed with a fabricated one. - accepted_now = None - if accepted_now is not None: - # Informational only: the window is scoped by the - # frozen anchor (shares committing during the read - # land above it and belong to the next window), so - # no bracket around the read is needed. - snapshot_accepted_count = int(accepted_now) - if resolved_mode == "ready": - # Publish the read's frozen anchor: a late-visible - # append committing mid-read predates it without - # holding the pending floor, and this read's database - # snapshot may already exclude the row. The recorded - # invalidation epoch keeps the resulting bundle and - # its seeded artifact from arming or serving. The - # exposure must outlive this read: nothing else exposes - # this window's anchor until the seeded artifact passes - # its install fence, so an append committing during - # bundle construction or cache publication would - # otherwise advance no epoch and the seed would arm a - # window that delta reads can never rediscover the row - # from. The token therefore rides the seeded artifact - # (retired by its install fence); a build that raises, - # or seeds nothing, retires it on the spot. - inflight_scan_anchor_token = ( - runtime._expose_inflight_scan_anchor(issued_at_ms) - ) - # Mirrored onto the request so the executor wrapper can - # retire the exposure when this build dies anywhere past - # the read (checkpoint cancellation, manifest assembly, - # the publication fence). - build_request.inflight_scan_anchor_token = ( - inflight_scan_anchor_token - ) - try: - records = runtime.ledger.snapshot_at_job_issue( - issued_at_ms, - window_weight=snapshot_window_weight, - ) - except BaseException: - runtime._retire_inflight_scan_anchor( - inflight_scan_anchor_token - ) - raise - else: - records = [] - # An accepted parent's prospective carry state supersedes the - # published artifact for children built on that parent; the - # published balances remain the fallback for ordinary tips. - prior_balances = runtime._prior_balances_for_job_parent( - str(template["previousblockhash"]), - parent_height=int(template["height"]) - 1, - fallback_balances=build_request.payout_artifact.prior_balances(), - ) - runtime._job_build_checkpoint("ledger_snapshot_complete", cancellation) - shares = [] - for index, record in enumerate(records): - if index % 256 == 0: - runtime._job_build_checkpoint( - "ledger_snapshot_conversion", - cancellation, - ) - shares.append(record.to_prism_json()) - # A reused artifact carries shares snapshotted at its own earlier - # anchor. The bundle must declare that anchor: replaying the audit - # window at this job's fresher anchor could include a share that was - # already durable at artifact build time but stamped after the - # artifact's clamped anchor, and the artifact's share set excludes it - # by construction. - bundle_anchor_ms = ( - payout_artifact.snapshot_anchor_ms - if payout_artifact is not None - and payout_artifact.snapshot_anchor_ms is not None - else issued_at_ms - ) - ledger_elapsed = time.monotonic() - started - phases["ledger"] = phases.get("ledger", 0.0) + ledger_elapsed - if resolved_mode == "ready": - runtime._observe_tip_refresh_build_phase( - "ledger_snapshot", - ledger_elapsed, - ) - share_serialization: _ShareWindowSerialization | None = None - if resolved_mode == "ready" and payout_artifact is not None: - share_serialization = runtime._share_window_serialization_for_artifact( - payout_artifact, - shares, - ) - share_snapshot_sha256 = share_serialization.share_snapshot_sha256 - else: - # A plain list digests through the streaming view so the hook's - # fallback never serializes the whole window in one call. - share_snapshot_sha256 = self._canonical_json_sha256( - share_array_json_view(shares) - ) - if ( - payout_artifact is None - and snapshot_accepted_count is not None - and shares - and runtime._payout_artifact_reuse_active() - ): - # This build already paid the full ledger read; carry the window - # it produced so cache publication can arm it for builds arriving - # while its anchor stays inside the staleness bound. The artifact - # carries the published payout-state balances, not this bundle's - # possibly parent-adjusted view: reuse re-applies the parent - # override itself and the reuse fence hashes the artifact - # balances against the published payout artifact. Skipped - # entirely under the kill-switch: nothing may arm while reuse - # is disabled. - assert self._payout_ledger_artifact_type is not None, ( - "job bundle service requires the payout-ledger artifact type " - "to seed reusable windows" - ) - seed_prior_balances = tuple( - build_request.payout_artifact.prior_balances() - ) - prepared_ledger_artifact = self._payout_ledger_artifact_type( - generation=0, - payout_state_generation=payout_state_generation, - network_difficulty=int(build_request.key.network_difficulty), - accepted_share_count=snapshot_accepted_count, - shares_json=tuple(shares), - prior_balances=seed_prior_balances, - prepared_monotonic=time.monotonic(), - snapshot_anchor_ms=issued_at_ms, - share_snapshot_sha256=share_snapshot_sha256, - prior_balances_sha256=self._canonical_json_sha256( - seed_prior_balances - ), - append_invalidation_epoch=( - build_request.key.payout_append_invalidation_epoch - ), - inflight_scan_anchor_token=inflight_scan_anchor_token, - ) - final_build_key = dataclass_replace( - build_request.key, - share_snapshot_sha256=share_snapshot_sha256, - ) - runtime._job_build_checkpoint("payout_derivation", cancellation) - started = time.monotonic() - placeholder_suffix_hex = final_build_key.coinbase_suffix_hex - collection_identity: tuple[str, str] | None = None - previous_metrics_scope = bool( - getattr(self._job_build_phase_local, "tip_refresh_metrics", False) - ) - self._job_build_phase_local.tip_refresh_metrics = resolved_mode == "ready" - try: - if resolved_mode == "ready": - if not shares: - raise RuntimeError( - "ready-pool ledger snapshot contained no payout shares" - ) - runtime._job_build_checkpoint("ctv_manifest", cancellation) - runtime._job_build_checkpoint("signing_verification", cancellation) - bundle = runtime.build_audit_bundle( - shares=shares, - found_block={ - "block_height": int(template["height"]), - "coinbase_value_sats": int(template["coinbasevalue"]), - "network_difficulty": artifacts.network_difficulty, - "anchor_job_issued_at_ms": bundle_anchor_ms, - }, - prior_balances=prior_balances, - coinbase_script_sig_suffix_hex=placeholder_suffix_hex, - witness_merkle_leaves_hex=list( - build_request.witness_merkle_leaves_hex - ), - ctv_fee_parent_hash=str(template["previousblockhash"]), - summary_only=True, - payout_policy=json.loads(build_request.payout_policy_json), - ctv_settlement=( - json.loads(build_request.ctv_settlement_json) - if build_request.ctv_settlement_json is not None - else None - ), - cancellation=cancellation, - share_serialization=share_serialization, - append_invalidation_epoch=( - payout_artifact.append_invalidation_epoch - if payout_artifact is not None - else None - ), - ) - collection_only = False - else: - assert worker is not None - runtime._job_build_checkpoint("ctv_manifest", cancellation) - runtime._job_build_checkpoint("signing_verification", cancellation) - bundle = runtime.build_collection_bundle( - template=template, - transaction_hexes=build_request.transaction_hexes, - worker=worker, - network_difficulty=final_build_key.network_difficulty, - issued_at_ms=issued_at_ms, - suffix_hex=placeholder_suffix_hex, - summary_only=True, - payout_policy=json.loads(build_request.payout_policy_json), - ctv_settlement=( - json.loads(build_request.ctv_settlement_json) - if build_request.ctv_settlement_json is not None - else None - ), - cancellation=cancellation, - ) - shares = [] - collection_only = True - collection_identity = runtime._collection_bundle_identity(worker) - finally: - self._job_build_phase_local.tip_refresh_metrics = previous_metrics_scope - manifest = bundle["signed_coinbase_manifest"]["manifest"] - prospective_prior_balances: ( - tuple[tuple[str, str, str, int], ...] | None - ) = None - payout_policy_manifest = bundle.get("payout_policy_manifest") - if isinstance(payout_policy_manifest, dict) and isinstance( - payout_policy_manifest.get("accounts"), - list, - ): - prospective_prior_balances = runtime._serialize_prior_balance_preview( - runtime._accepted_block_payout_preview_from_bundle( - bundle, - prior_balances=prior_balances, - ) - ) - runtime._job_build_checkpoint("bundle_assembly", cancellation) - assembly_started = time.monotonic() - base_job = direct_stratum.make_job_from_builder_manifest( - job_id="prism-template-base", - template=template, - manifest=manifest, - extranonce1_hex=PRISM_JOB_EXTRANONCE1_PLACEHOLDER_HEX, - extranonce2_size=runtime.extranonce2_size, - desired_share_difficulty=runtime.share_difficulty, - clean_jobs=True, - transaction_hexes=build_request.transaction_hexes, - ) - phases["assembly"] = phases.get("assembly", 0.0) + ( - time.monotonic() - assembly_started - ) - runtime._job_build_checkpoint("serialization", cancellation) - runtime._job_build_checkpoint("bundle_publication", cancellation) - if resolved_mode == "ready": - with self._job_cache_lock: - if ( - final_build_key.payout_append_invalidation_epoch - != runtime._payout_ledger_append_invalidation_epoch - ): - raise JobBuildSuperseded( - "payout window invalidated before bundle publication" - ) - if ( - inflight_scan_anchor_token is not None - and prepared_ledger_artifact is None - ): - # The fence above just proved, under this same lock hold, - # that no append predated this build's window. Seeded - # builds keep their exposure until the artifact's - # install fence settles; with no seed, nothing else - # exposes this window's anchor once the walk exposure - # retires, yet jobs stamped from this bundle keep - # serving the window until they retire. Hand the anchor - # to the published-window watermark under this same lock - # hold, so a replay-shaped append committing after this - # return still predates a live anchor and advances the - # epoch those jobs' landing fences check. - runtime._publish_seedless_job_window_anchor_locked( - issued_at_ms - ) - runtime._payout_window_inflight_scan_anchors.pop( - inflight_scan_anchor_token, - None, - ) - inflight_scan_anchor_token = None - build_request.inflight_scan_anchor_token = None - phases["bundle"] = phases.get("bundle", 0.0) + (time.monotonic() - started) - return CachedJobBundle( - key=key, - # Construction consumed the detached canonical template above; - # publication retains the exact snapshot-owned object identity used - # by the existing validation token. - template=artifacts.template, - template_fingerprint=artifacts.fingerprint, - # Only this manifest is needed to bind later clock-only template - # observations. Retaining the returned logical bundle duplicated - # the entire shares tree already held in shares_json. - coinbase_manifest=manifest, - shares_json=shares, - prior_balances=prior_balances, - found_block=bundle["found_block"], - collection_only=collection_only, - issued_at_ms=issued_at_ms, - base_job=base_job, - built_monotonic=time.monotonic(), - template_generation=artifacts.generation, - payout_state_generation=payout_state_generation, - payout_artifact_generation=( - payout_artifact.generation if payout_artifact is not None else 0 - ), - collection_identity=collection_identity, - prospective_prior_balances=prospective_prior_balances, - build_key=final_build_key, - prepared_ledger_artifact=prepared_ledger_artifact, - ) - - def build_collection_bundle( - self, - *, - template: dict[str, Any], - transaction_hexes: tuple[str, ...], - worker: WorkerIdentityPort, - network_difficulty: int, - issued_at_ms: int, - suffix_hex: str, - summary_only: bool = False, - payout_policy: dict[str, object] | None = None, - ctv_settlement: dict[str, object] | None = None, - cancellation: JobBuildCancellation | None = None, - ) -> dict[str, Any]: - runtime = self._runtime - if cancellation is not None: - cancellation.raise_if_cancelled("collection payout derivation") - share = { - "share_seq": 1, - "share_id": "bootstrap-share", - "miner_id": worker.payout_address, - "order_key": worker.payout_address, - "p2mr_program_hex": worker.p2mr_program_hex, - "share_difficulty": network_difficulty, - "network_difficulty": network_difficulty, - "template_height": int(template["height"]) - 1, - "job_id": "bootstrap-job", - "job_issued_at_ms": issued_at_ms, - "accepted_at_ms": issued_at_ms, - "ntime": int(template["curtime"]), - } - return runtime.build_audit_bundle( - shares=[share], - found_block={ - "block_height": int(template["height"]), - "coinbase_value_sats": int(template["coinbasevalue"]), - "network_difficulty": network_difficulty, - "anchor_job_issued_at_ms": issued_at_ms, - }, - prior_balances=[], - coinbase_script_sig_suffix_hex=suffix_hex, - witness_merkle_leaves_hex=direct_stratum.witness_merkle_leaves_hex(transaction_hexes), - ctv_fee_parent_hash=str(template["previousblockhash"]), - summary_only=summary_only, - payout_policy=payout_policy, - ctv_settlement=ctv_settlement, - cancellation=cancellation, - ) - - -__all__ = [ - "CachedJobBundle", - "CollectionIdentityUnavailable", - "JobBuildAdmissionDeadlineExceeded", - "JobBuildCancellation", - "JobBuildCancelled", - "JobBuildFlight", - "JobBuildKey", - "JobBuildRequest", - "JobBuildSuperseded", - "JobBuildWaiterCancelled", - "JobBundleBuildControl", - "JobBundleBuildSuperseded", - "JobBundleRuntime", - "JobBundleService", - "MAX_PRISM_JOB_BUNDLE_CACHE_ENTRIES", - "PRISM_INITIAL_JOB_SUBSCRIBE_POLL_SECONDS", - "PRISM_JOB_BUILD_ORPHAN_SWEEP_GRACE_SECONDS", - "PRISM_JOB_BUILD_PHASES", - "PRISM_JOB_BUILD_SECONDS_BUCKETS", - "PRISM_JOB_CACHE_KINDS", - "PRISM_JOB_EXTRANONCE1_PLACEHOLDER_HEX", - "PRISM_REWARD_WINDOW_MULTIPLIER", - "PRISM_SNAPSHOT_WINDOW_MARGIN", - "WorkerIdentityPort", - "canonical_json_sha256", - "canonical_json_text", - "now_ms", -] diff --git a/lab/prism/job_delivery.py b/lab/prism/job_delivery.py deleted file mode 100644 index ce7861d98..000000000 --- a/lab/prism/job_delivery.py +++ /dev/null @@ -1,3042 +0,0 @@ -#!/usr/bin/env python3 -"""Per-session PRISM job delivery, initial jobs, and retained-job ownership. - -The service owns the S2 domain: job stamping/registration, initial-job -scheduling and its bounded executor, prepared-job delivery admission (epoch -admission floor and post-write proof), difficulty adjacency, and all -retained/evicted-job state including the #100 cross-connection disconnected -retention authority. - -It never imports ``prism_coordinator``. Session membership, Stratum socket -writes, payout delivery gates, job-bundle preparation, and live configuration -attributes are reached through the :class:`JobDeliveryRuntime` typed port, -resolved at call time so the historical coordinator monkeypatch seams -(including the instance-level facade patches used by the current test suite) -keep intercepting exactly as before the extraction. ``ClientState`` remains -coordinator-owned at this layer (it moves with the session owner); client -objects are used strictly through duck typing, exactly as the current code -does. -""" - -from __future__ import annotations - -from collections import OrderedDict -from collections.abc import Mapping -from concurrent.futures import Future -from contextlib import contextmanager, nullcontext -from dataclasses import dataclass, field, replace as dataclass_replace -from decimal import Decimal -import json -import threading -import time -import traceback -from typing import Any, Callable, Iterator, Protocol - -from lab.auxpow import vardiff -from lab.prism import direct_stratum -from lab.prism.bounded_executor import _BoundedPriorityExecutor, _DeliveryQueueFull -from lab.prism.future_callbacks import add_releasing_done_callback -from lab.prism.coordinator_config import ( - DEFAULT_PRISM_DISCONNECTED_JOB_RETENTION, - DEFAULT_PRISM_INITIAL_JOB_MAX_WORKERS, # noqa: F401 - compatibility re-export - DEFAULT_PRISM_SAME_TIP_JOB_RETENTION_PER_CONNECTION, - DEFAULT_PRISM_SAME_TIP_JOB_RETENTION_SECONDS, - DEFAULT_PRISM_STALE_GRACE_SECONDS, -) -from lab.prism.job_bundle import ( - CachedJobBundle, - JobBuildWaiterCancelled, - PRISM_JOB_BUILD_PHASES, # noqa: F401 - owner-shared phase registry -) -# Imported from its owning module rather than the template_artifacts -# compatibility surface: this type is new, so it has no historical import -# path to preserve there. -from lab.prism.payout_state import AcceptedParentPayoutPreviewPending -from lab.prism.template_artifacts import ( - CachedTemplateArtifacts, - QbitTipTemplateSnapshot, - TemplateRefreshBlocked, - TemplateRefreshSuperseded, - qbit_template_fingerprint, -) -from lab.prism.tip_refresh import ( - FanoutCancellation, - RefreshResult, - TipRefreshValidationToken, - _TipRefreshTrustBlocked, -) - - -DEFAULT_PRISM_EVICTED_JOB_PRUNE_INTERVAL_SECONDS = 1.0 -# Owner-local copy of the still-coordinator-visible constant; a compatibility -# duplicate by design. This module is the coordinator's compatibility import -# source for the constant. -PRISM_TIP_REFRESH_ADMISSION_POLL_SECONDS = 0.05 -MAX_ACTIVE_PRISM_JOBS_PER_CLIENT = 16 -# Never-served clients outrank the fleet's tip-refresh wave: a client with no -# active job is producing nothing until its first notify, while a client with -# stale-tip work keeps mining (stale-grace credits it) for the wave's duration. -PRISM_DELIVERY_PRIORITY_INITIAL = 0 -PRISM_DELIVERY_PRIORITY_NEW_TIP = 1 -PRISM_DELIVERY_PRIORITY_SAME_TIP = 2 -PRISM_EVICTED_JOB_CLASSES = ("same_tip", "stale_grace") -PRISM_EVICTED_JOB_SUBMIT_OUTCOMES = ( - "accepted_same_tip", - "credited_stale_grace", - "accepted_same_tip_cross_connection", -) -PRISM_EVICTED_JOB_CAPACITY_SCOPES = ("connection", "disconnected") -# Credit policies recorded on accepted ledger rows. Normal shares carry no -# policy; a policy marks a share that was credited by an explicit pool rule -# (documented in docs/prism-rejections.md) so audits can distinguish them. -PRISM_CREDIT_POLICY_STALE_GRACE = "stale-grace" - - -@dataclass(frozen=True) -class PrismJobContext: - job: direct_stratum.DirectQbitStratumJob - template: dict[str, Any] - shares_json: list[dict[str, object]] - prior_balances: list[dict[str, object]] - found_block: dict[str, object] - share_weight: int - collection_only: bool - worker: WorkerIdentity - issued_at_ms: int - template_fingerprint: str | None = None - template_generation: int = 0 - payout_state_generation: int = 0 - prospective_prior_balances: tuple[tuple[str, str, str, int], ...] | None = None - payout_artifact_generation: int = 0 - payout_append_invalidation_epoch: int = 0 - # Digest of the published prior-balance snapshot this job's payouts are - # keyed to. A periodic self-check repair replaces the published snapshot - # in place -- the payout generation and append epoch both survive the - # swap -- so this digest is the only fence that identifies an active job - # still carrying refuted balances. - payout_artifact_sha256: str | None = None - connection_id: int = 0 - authorization_generation: int = 0 - difficulty_generation: int = 0 - tip_refresh_epoch_sequence: int = 0 - # Version-rolling mask negotiated on the connection this job was - # delivered to. Cross-connection resumes validate in-flight version bits - # against this mask, not the replacement connection's (often still 0). - version_mask: int = 0 - - -@dataclass(frozen=True) -class EvictedJobEntry: - context: PrismJobContext - connection_id: int - evicted_monotonic: float - previousblockhash: str - client: ClientState | None = None - - -@dataclass(eq=False) -class PendingInitialJob: - client: ClientState - authorization_generation: int - worker: WorkerIdentity - requested_monotonic: float - deadline_monotonic: float | None - connection_id: int | None = None - difficulty_generation: int | None = None - cancelled: threading.Event = field(default_factory=threading.Event) - future: Future[bool] | None = None - predecessor: Future[bool] | None = None - - -@dataclass(frozen=True) -class InitialJobAdmissionSnapshot: - """One internally consistent copy of the first-job admission domain. - - Metrics and health readers take this instead of reaching into the live - admission state, so an observability refresh never has to hold the - coordinator lock and the admission lock at the same time (#159). - """ - - pending: dict[Any, PendingInitialJob] - queue_rejections: int - timeout_disconnects: int - cancelled: int - coalesced: int - queue_capacity_reclaimed: int - last_delivery_monotonic: float | None - - -class PendingInitialJobsView(Mapping): - """Read-only, lock-protected view of the first-job admission map. - - The mutable mapping is private to :class:`JobDeliveryService` and is only - ever read or written under the admission lock. Every compatibility reader - -- the session owner's reauthorization capacity pre-check, health and - metrics, focused tests -- reaches the queue through this view, so an - external ``len()``, ``in``, lookup, or iteration is itself taken under the - owning lock. Such a read can therefore never observe a replacement or a - cancellation half applied, and can never mutate the queue. - - ``len`` and ``in`` stay O(1) under the lock rather than copying the map: - the reauthorization pre-check runs once per re-authorize, and a snapshot - copy there would put O(N) work back inside a lock a reconnect herd - contends on -- the exact shape #159 exists to remove. Bulk materialization - (``copy``, ``items``, ``values``, ``keys``, iteration) takes one locked - copy, so it cannot tear against concurrent admission. - - These reads are exact at the instant they are taken, but they are not a - reservation: the authoritative capacity decision is always re-taken under - the same lock in :meth:`JobDeliveryService.schedule_initial_job`. - - The view is strictly immutable: it exposes no ``__setitem__``, - ``__delitem__``, ``pop``, ``clear``, ``update``, or ``setdefault``, so no - caller outside this service can reach admission state to write it. - Embedders and focused tests that need to install a request seed the whole - mapping through the ``pending_initial_jobs`` setter, which adopts its - contents under the admission lock. - """ - - __slots__ = ("_service",) - - def __init__(self, service: "JobDeliveryService") -> None: - self._service = service - - def copy(self) -> dict[Any, PendingInitialJob]: - """Return one locked, internally consistent copy of the queue.""" - service = self._service - with service._initial_job_admission_lock: - return dict(service._pending_initial_jobs) - - def __getitem__(self, key: Any) -> PendingInitialJob: - service = self._service - with service._initial_job_admission_lock: - return service._pending_initial_jobs[key] - - def __contains__(self, key: object) -> bool: - service = self._service - with service._initial_job_admission_lock: - return key in service._pending_initial_jobs - - def __len__(self) -> int: - service = self._service - with service._initial_job_admission_lock: - return len(service._pending_initial_jobs) - - def get( - self, - key: Any, - default: Any = None, - ) -> PendingInitialJob | None: - service = self._service - with service._initial_job_admission_lock: - return service._pending_initial_jobs.get(key, default) - - def __iter__(self) -> Iterator[Any]: - return iter(self.copy()) - - def keys(self) -> Any: - return self.copy().keys() - - def values(self) -> Any: - return self.copy().values() - - def items(self) -> Any: - return self.copy().items() - - def __eq__(self, other: object) -> bool: - return self.copy() == other - - def __ne__(self, other: object) -> bool: - return not self.__eq__(other) - - __hash__ = None # type: ignore[assignment] - - def __repr__(self) -> str: - return f"{type(self).__name__}({self.copy()!r})" - - -class JobBuildFailed(RuntimeError): - """Internal signal used to distinguish a skipped build from a no-op.""" - - -class JobDeliveryRuntime(Protocol): - """Typed port over the coordinator, resolved at call time. - - Every member is looked up on the live coordinator object when used, so - instance monkeypatches (``server.maybe_send_job = ...`` and friends) and - coordinator-owned live configuration attributes keep working exactly as - before the extraction. Owner facades route back into this service; that - round trip is deliberate: it is what preserves the current white-box - patch surface until later stack layers repoint those tests. The legacy - S2 field names also resolve here -- coordinator class descriptors route - them to this service's single mutable copy -- and the R1 published-tip - fields resolve through the tip-refresh owner's descriptors the same way. - """ - - # Cross-domain objects and live configuration attributes. - _job_cache_lock: Any - _payout_ledger_append_invalidation_epoch: Any - _payout_state_delivery_gate: Any - _payout_state_generation: Any - _payout_state_publication_blocked: Any - _pool_ready_latched: Any - _published_payout_state: Any - _template_artifacts: Any - _tip_refresh_epoch_payout_generation: Any - _tip_refresh_epoch_sequence: Any - _tip_refresh_epoch_tip_hash: Any - clients: Any - current_tip_first_seen: Any - current_tip_parent: Any - disconnected_job_retention: Any - hot_path_log_enabled: Any - initial_job_max_workers: Any - job_build_failure_count: Any - lock: Any - reorg_reconciler_enabled: Any - rpc: Any - same_tip_job_retention_per_connection: Any - same_tip_job_retention_seconds: Any - stale_grace_seconds: Any - stop_event: Any - stratum_initial_job_timeout_seconds: Any - stratum_max_pending_initial_jobs: Any - tip_refresh_epoch_fanout: Any - tip_template_snapshot: Any - vardiff_config: Any - - def _client_vardiff_lock(self, *args: Any, **kwargs: Any) -> Any: ... - - def _collection_bundle_identity(self, *args: Any, **kwargs: Any) -> Any: ... - - def _current_published_tip_hash_locked(self, *args: Any, **kwargs: Any) -> Any: ... - - def _ensure_evicted_job_state(self, *args: Any, **kwargs: Any) -> Any: ... - - def _ensure_initial_job_state(self, *args: Any, **kwargs: Any) -> Any: ... - - def _ensure_job_cache_state(self, *args: Any, **kwargs: Any) -> Any: ... - - def _idle_bundle_current_locked(self, *args: Any, **kwargs: Any) -> Any: ... - - def _issuance_artifacts_current(self, *args: Any, **kwargs: Any) -> Any: ... - - def _job_build_phases(self, *args: Any, **kwargs: Any) -> Any: ... - - def _observe_payout_gate_admission(self, *args: Any, **kwargs: Any) -> Any: ... - - def _payout_delivery(self, *args: Any, **kwargs: Any) -> Any: ... - - def _progress_bundle_build_finished(self, *args: Any, **kwargs: Any) -> Any: ... - - def _progress_bundle_build_started(self, *args: Any, **kwargs: Any) -> Any: ... - - def _record_first_payout_delivery(self, *args: Any, **kwargs: Any) -> Any: ... - - def _record_progress_delivery(self, *args: Any, **kwargs: Any) -> Any: ... - - def _record_startup_phase_once(self, *args: Any, **kwargs: Any) -> Any: ... - - def _submit_delivery_task(self, *args: Any, **kwargs: Any) -> Any: ... - - def bury_evicted_job(self, *args: Any, **kwargs: Any) -> Any: ... - - def client_minimum_advertised_difficulty(self, *args: Any, **kwargs: Any) -> Any: ... - - def desired_client_share_difficulty(self, *args: Any, **kwargs: Any) -> Any: ... - - def difficulty_payload(self, *args: Any, **kwargs: Any) -> Any: ... - - def disconnect_client(self, *args: Any, **kwargs: Any) -> Any: ... - - def ensure_reorg_reconciled_for_current_tip(self, *args: Any, **kwargs: Any) -> Any: ... - - def job_issuance_template_artifacts(self, *args: Any, **kwargs: Any) -> Any: ... - - def job_payload(self, *args: Any, **kwargs: Any) -> Any: ... - - def observe_job_build_elapsed(self, *args: Any, **kwargs: Any) -> Any: ... - - def qbit_chain_view_untrusted(self, *args: Any, **kwargs: Any) -> Any: ... - - def send_difficulty(self, *args: Any, **kwargs: Any) -> Any: ... - - def send_job(self, *args: Any, **kwargs: Any) -> Any: ... - - def share_weight_for_worker(self, *args: Any, **kwargs: Any) -> Any: ... - - def shared_job_bundle(self, *args: Any, **kwargs: Any) -> Any: ... - - -class JobDeliveryService: - """Sole owner of S2 job delivery, initial jobs, and retained jobs.""" - - def __init__( - self, - runtime: JobDeliveryRuntime, - *, - stratum_error: type[BaseException], - ) -> None: - self._runtime = runtime - # The Stratum error type remains coordinator-owned until the session - # owner is extracted; it is injected so this leaf module never - # imports prism_coordinator. - self._stratum_error = stratum_error - self.jobs: dict[str, PrismJobContext] = {} - self.job_counter = 0 - # Private: every read and write of this mapping happens under the - # admission lock below. Callers outside this service reach it only - # through the read-only ``pending_initial_jobs`` view. - self._pending_initial_jobs: dict[Any, PendingInitialJob] = {} - # Sole owner of the first-job admission domain: the admission map - # itself, its exact capacity decision, request replacement/coalescing/ - # cancellation, and every admission counter and timestamp below. - # - # It is the innermost lock in the process. Nothing acquires another - # lock while holding it, and it is never held across a socket write, a - # disconnect, a job build, an executor shutdown wait, or a future - # cancellation (``Future.cancel`` runs done-callbacks inline, and this - # module's callback re-enters admission). Cancellation callbacks are - # therefore always invoked after the admission state is committed and - # the lock is released. Splitting this domain out of ``runtime.lock`` - # is what stops a reconnect herd's queue bookkeeping from convoying - # behind fleet-wide coordinator work (#159). - self._initial_job_admission_lock = threading.Lock() - self._pending_initial_jobs_view = PendingInitialJobsView(self) - self._initial_job_executor_lock = threading.Lock() - self._initial_job_executor: _BoundedPriorityExecutor | None = None - self._initial_job_executor_shutdown = False - self.initial_job_queue_rejection_count = 0 - self.initial_job_timeout_count = 0 - self.initial_job_cancelled_count = 0 - self.initial_job_coalesced_count = 0 - self.initial_job_queue_capacity_reclaimed_count = 0 - self.initial_job_sent_count = 0 - self.initial_job_failed_count = 0 - self.initial_job_superseded_count = 0 - self.initial_job_delivery_latency_seconds_sum = 0.0 - self.initial_job_delivery_latency_count = 0 - self.last_initial_job_delivery_monotonic: float | None = None - # Globally insertion ordered, with per-connection indexes for the - # independent TTL and capacity limits. Prior-tip entries never consume - # the same-tip cap while stale-grace still protects them. - self.evicted_job_graveyard: OrderedDict[str, EvictedJobEntry] = OrderedDict() - self.evicted_jobs_by_connection: dict[int, OrderedDict[str, None]] = {} - self.evicted_same_tip_by_connection: dict[int, OrderedDict[str, None]] = {} - self.evicted_same_tip_job_ids: OrderedDict[str, None] = OrderedDict() - self.evicted_job_index_tip_hash: str | None = None - self.evicted_job_next_prune_monotonic = 0.0 - self.evicted_job_expiration_counts = { - job_class: 0 for job_class in PRISM_EVICTED_JOB_CLASSES - } - self.evicted_job_capacity_eviction_counts = { - scope: 0 for scope in PRISM_EVICTED_JOB_CAPACITY_SCOPES - } - self.evicted_job_submit_counts = { - outcome: 0 for outcome in PRISM_EVICTED_JOB_SUBMIT_OUTCOMES - } - # Insertion-ordered ids of graveyard entries whose connection is - # gone (entry.client is None); bounds cross-connection retention - # independently of live connections' per-connection caps. - self._disconnected_evicted_job_ids: OrderedDict[str, None] = OrderedDict() - - # -- state adoption ---------------------------------------------------- - - @property - def pending_initial_jobs(self) -> PendingInitialJobsView: - """Read-only compatibility surface over the admission map. - - The legacy attribute name is coordinator-visible (and reaches the - session owner's reauthorization capacity pre-check through the - coordinator descriptor), so it must not hand out the mutable mapping: - that is what left a capacity read outside the lock that owns it. - """ - return self._pending_initial_jobs_view - - @pending_initial_jobs.setter - def pending_initial_jobs(self, value: Any) -> None: - """Adopt a mapping assigned through the legacy name. - - Construction and focused tests assign through this name; adopt the - contents into the private map instead of retaining the caller's - mutable object, which would put admission state back outside the lock. - """ - with self._initial_job_admission_lock: - self._pending_initial_jobs.clear() - if value: - self._pending_initial_jobs.update(value) - - def ensure_evicted_job_state(self) -> None: - """Ensure/adopt the retained-job index, converting legacy seeded state.""" - runtime = self._runtime - graveyard = getattr(self, "evicted_job_graveyard", None) - rebuild_indexes = False - if not isinstance(graveyard, OrderedDict): - converted: OrderedDict[str, EvictedJobEntry] = OrderedDict() - for job_id, entry in (graveyard or {}).items(): - if isinstance(entry, EvictedJobEntry): - converted[job_id] = entry - continue - context, connection_id, evicted_monotonic = entry - client = next( - ( - candidate - for candidate in getattr(runtime, "clients", ()) - if candidate.connection_id == connection_id - ), - None, - ) - converted[job_id] = EvictedJobEntry( - context=context, - connection_id=connection_id, - evicted_monotonic=evicted_monotonic, - previousblockhash=str(context.template["previousblockhash"]), - client=client, - ) - self.evicted_job_graveyard = converted - rebuild_indexes = True - if not hasattr(runtime, "same_tip_job_retention_seconds"): - runtime.same_tip_job_retention_seconds = ( - DEFAULT_PRISM_SAME_TIP_JOB_RETENTION_SECONDS - ) - if not hasattr(runtime, "same_tip_job_retention_per_connection"): - runtime.same_tip_job_retention_per_connection = ( - DEFAULT_PRISM_SAME_TIP_JOB_RETENTION_PER_CONNECTION - ) - if not hasattr(runtime, "disconnected_job_retention"): - runtime.disconnected_job_retention = ( - DEFAULT_PRISM_DISCONNECTED_JOB_RETENTION - ) - current_tip = runtime._current_published_tip_hash_locked() - if self.evicted_job_index_tip_hash != current_tip: - rebuild_indexes = True - if rebuild_indexes: - self._rebuild_evicted_job_indexes_locked() - - # -- narrow adapters for other owners ---------------------------------- - - def next_job_id(self) -> str: - """Allocate the next delivery job id under the coordinator lock.""" - runtime = self._runtime - with runtime.lock: - self.job_counter += 1 - return f"prism-{self.job_counter}" - - def complete_delivery( - self, - client: Any, - context: PrismJobContext, - delivered_monotonic: float, - ) -> None: - """Commit S2 delivery bookkeeping after a successful socket write.""" - runtime = self._runtime - runtime.note_tip_work_delivered( - client, - str(context.template["previousblockhash"]), - ) - runtime._record_progress_delivery(client, context, delivered_monotonic) - - def stamp_job_for_client( - self, - client: ClientState, - cached: CachedJobBundle, - *, - clean_jobs: bool, - tip_refresh_epoch_sequence: int | None = None, - ) -> PrismJobContext: - runtime = self._runtime - if client.worker is None: - raise self._stratum_error(20, "client is not authorized") - if cached.collection_only and cached.collection_identity != ( - runtime._collection_bundle_identity(client.worker) - ): - raise self._stratum_error( - 20, - "collection bundle payout identity no longer matches client authorization", - ) - with runtime.lock: - self.job_counter += 1 - job_id = f"prism-{self.job_counter}" - if tip_refresh_epoch_sequence is None: - tip_refresh_epoch_sequence = ( - runtime._tip_refresh_epoch_for_bundle_locked(cached) - ) - share_target = direct_stratum.effective_share_target( - runtime.desired_client_share_difficulty(client), - cached.base_job.qbit_target, - minimum_advertised_difficulty=runtime.client_minimum_advertised_difficulty(client), - ) - job = dataclass_replace( - cached.base_job, - job_id=job_id, - extranonce1_hex=client.extranonce1_hex, - share_target=share_target, - share_difficulty=direct_stratum.target_difficulty(share_target), - clean_jobs=clean_jobs, - ) - return PrismJobContext( - job=job, - template=cached.template, - shares_json=cached.shares_json, - prior_balances=cached.prior_balances, - found_block=cached.found_block, - share_weight=runtime.share_weight_for_worker(client.worker), - collection_only=cached.collection_only, - worker=client.worker, - issued_at_ms=cached.issued_at_ms, - template_fingerprint=cached.template_fingerprint, - template_generation=cached.template_generation, - payout_state_generation=cached.payout_state_generation, - prospective_prior_balances=cached.prospective_prior_balances, - payout_artifact_generation=cached.payout_artifact_generation, - payout_append_invalidation_epoch=( - int( - getattr( - cached.build_key, - "payout_append_invalidation_epoch", - 0, - ) - ) - if cached.build_key is not None - else 0 - ), - payout_artifact_sha256=( - cached.build_key.payout_artifact_sha256 - if cached.build_key is not None - else None - ), - connection_id=client.connection_id, - authorization_generation=int( - getattr(client, "authorization_generation", 0) - ), - difficulty_generation=int( - getattr(client, "difficulty_generation", 0) - ), - tip_refresh_epoch_sequence=int(tip_refresh_epoch_sequence), - version_mask=int(getattr(client, "version_mask", 0)), - ) - - def initial_job_executor(self) -> _BoundedPriorityExecutor: - runtime = self._runtime - runtime._ensure_initial_job_state() - with self._initial_job_executor_lock: - if self._initial_job_executor_shutdown: - raise RuntimeError("initial job executor is shut down") - executor = self._initial_job_executor - if executor is None: - executor = _BoundedPriorityExecutor( - max_workers=runtime.initial_job_max_workers, - max_queue_size=runtime.stratum_max_pending_initial_jobs, - thread_name_prefix="prism-initial-job-delivery", - ) - self._initial_job_executor = executor - return executor - - def shutdown_initial_job_executor(self) -> None: - runtime = self._runtime - runtime._ensure_initial_job_state() - # Reclaim the whole admission domain in one constant-shape critical - # section, then cancel the futures outside it: every cancellation runs - # this module's done-callback inline, and that callback re-enters - # admission. - with self._initial_job_admission_lock: - reclaimed = tuple(self._pending_initial_jobs.values()) - self._pending_initial_jobs.clear() - futures = [] - for request in reclaimed: - request.cancelled.set() - if request.future is not None: - futures.append(request.future) - self.initial_job_cancelled_count += 1 - for future in futures: - runtime._cancel_initial_job_future(future) - with self._initial_job_executor_lock: - executor = self._initial_job_executor - self._initial_job_executor = None - self._initial_job_executor_shutdown = True - if executor is not None: - # Running workers observe request cancellation before shutdown - # returns; queued reconnect work is cancelled without starting. - executor.shutdown(wait=True, cancel_futures=True) - - def initial_job_admission_snapshot(self) -> InitialJobAdmissionSnapshot: - """Return one internally consistent copy of the admission domain. - - Health and metrics readers call this instead of walking the live - admission state under the coordinator lock, so a refresh never holds - two locks at once and never observes a half-committed replacement. - """ - with self._initial_job_admission_lock: - return InitialJobAdmissionSnapshot( - pending=dict(self._pending_initial_jobs), - queue_rejections=self.initial_job_queue_rejection_count, - timeout_disconnects=self.initial_job_timeout_count, - cancelled=self.initial_job_cancelled_count, - coalesced=self.initial_job_coalesced_count, - queue_capacity_reclaimed=( - self.initial_job_queue_capacity_reclaimed_count - ), - last_delivery_monotonic=self.last_initial_job_delivery_monotonic, - ) - - def _cancel_initial_job_future(self, future: Future[bool]) -> bool: - """Cancel one initial-job future and account physical queue removal.""" - executor = getattr(self, "_initial_job_executor", None) - reclaimed = bool( - executor is not None - and executor.cancel(future) - ) - if executor is None: - future.cancel() - if reclaimed: - # Admission accounting only; the cancellation callbacks that - # ``executor.cancel`` ran inline above have already completed, so - # taking the admission lock here cannot re-enter it. - with self._initial_job_admission_lock: - self.initial_job_queue_capacity_reclaimed_count += 1 - return reclaimed - - def _initial_request_current_locked(self, request: PendingInitialJob) -> bool: - """Coordinator-domain currency for one first-job request. - - Called with ``runtime.lock`` held (session membership and the - published-tip identity are read here). The pending-slot term is read - under the admission lock -- taken and released before the rest of the - predicate, and innermost as always -- so it cannot observe a - replacement or cancellation half applied. - """ - runtime = self._runtime - client = request.client - deadline = request.deadline_monotonic - with self._initial_job_admission_lock: - owns_pending_slot = self._pending_initial_jobs.get(client) is request - return ( - owns_pending_slot - and client in runtime.clients - and not getattr(client, "closing", False) - and ( - request.connection_id is None - or client.connection_id == request.connection_id - ) - and client.authorized - and client.subscribed - and client.worker == request.worker - and int(getattr(client, "authorization_generation", 0)) - == request.authorization_generation - and ( - request.difficulty_generation is None - or int(getattr(client, "difficulty_generation", 0)) - == request.difficulty_generation - ) - and (deadline is None or time.monotonic() < deadline) - and not request.cancelled.is_set() - ) - - def _initial_request_cancelled(self, request: PendingInitialJob) -> bool: - runtime = self._runtime - if request.cancelled.is_set() or runtime.stop_event.is_set(): - return True - with runtime.lock: - runtime._ensure_initial_job_state() - return not runtime._initial_request_current_locked(request) - - def _cancel_pending_initial_job_locked( - self, - client: ClientState, - *, - count: bool, - ) -> PendingInitialJob | None: - """Release this client's pending first-job slot, if it owns one. - - The historical ``_locked`` name records that the disconnect path calls - this with ``runtime.lock`` held; the admission state it mutates is now - owned by the admission lock, which is taken (and released) inside. The - future is cancelled after the slot is committed so the inline - done-callback observes the released slot and cannot deadlock. - """ - runtime = self._runtime - runtime._ensure_initial_job_state() - with self._initial_job_admission_lock: - request = self._pending_initial_jobs.pop(client, None) - if request is None: - return None - request.cancelled.set() - future = request.future - if count: - self.initial_job_cancelled_count += 1 - if future is not None: - runtime._cancel_initial_job_future(future) - return request - - def _client_has_current_tip_job_locked(self, client: ClientState) -> bool: - runtime = self._runtime - context = client.active_job - if context is None: - return False - payout_generation = int(getattr(runtime, "_payout_state_generation", 0)) - if int(getattr(context, "payout_state_generation", payout_generation)) != payout_generation: - return False - current_tip = runtime._current_published_tip_hash_locked() - if current_tip is None: - # An active job is not proof that tip observation is alive. Keep - # coverage fail-closed until blockpoll/blockwait has published the - # tip that makes the job current. - return False - snapshot = getattr(runtime, "tip_template_snapshot", None) - if snapshot is None: - # Focused embedders may only publish the observed tip. Production - # startup and blockpoll publish a full snapshot, in which case the - # exact template identity checks below are mandatory. - return str(context.template.get("previousblockhash", "")) == current_tip - if snapshot.bestblockhash != current_tip or snapshot.template_artifacts is None: - return False - return bool( - str(context.template.get("previousblockhash", "")) == current_tip - and getattr(context, "template_fingerprint", None) - == snapshot.template_fingerprint - and int(getattr(context, "template_generation", 0)) - == snapshot.template_generation - and context.template is snapshot.template_artifacts.template - and int(getattr(context, "connection_id", client.connection_id)) - == client.connection_id - and int(getattr(context, "authorization_generation", 0)) - == int(getattr(client, "authorization_generation", 0)) - and int(getattr(context, "difficulty_generation", 0)) - == int(getattr(client, "difficulty_generation", 0)) - ) - - def note_initial_job_delivered( - self, - client: ClientState, - *, - validated_current: bool = False, - ) -> None: - runtime = self._runtime - runtime._ensure_initial_job_state() - if not validated_current: - # Coordinator-domain question (published tip identity, payout - # generation): a short constant-time read, taken and released - # before any admission state is touched. - with runtime.lock: - if not runtime._client_has_current_tip_job_locked(client): - return - future: Future[bool] | None = None - completed = False - with self._initial_job_admission_lock: - request = self._pending_initial_jobs.pop(client, None) - if request is not None: - request.cancelled.set() - future = request.future - delivered = time.monotonic() - self.initial_job_sent_count += 1 - self.initial_job_delivery_latency_seconds_sum += max( - 0.0, delivered - request.requested_monotonic - ) - self.initial_job_delivery_latency_count += 1 - self.last_initial_job_delivery_monotonic = delivered - completed = True - if future is not None: - runtime._cancel_initial_job_future(future) - if completed: - runtime._record_startup_phase_once("first_job_delivered") - - def schedule_initial_job(self, client: ClientState) -> bool: - """Coalesce and enqueue one first-job request without blocking its handler.""" - runtime = self._runtime - # Focused tests and embedders replace maybe_send_job on the instance as - # a synchronous seam. Preserve it without affecting the production - # class path, which always uses the bounded executor below. - if "maybe_send_job" in runtime.__dict__: - return bool(runtime.maybe_send_job(client, clean_jobs=True)) - - now = time.monotonic() - runtime._ensure_initial_job_state() - if ( - not client.subscribed - or not client.authorized - or client.worker is None - or getattr(client, "closing", False) - ): - return True - generation = int(getattr(client, "authorization_generation", 0)) - difficulty_generation = int(getattr(client, "difficulty_generation", 0)) - worker = client.worker - connection_id = client.connection_id - - def represents_this_authorization(candidate: PendingInitialJob) -> bool: - return ( - candidate.connection_id == connection_id - and candidate.authorization_generation == generation - and candidate.difficulty_generation == difficulty_generation - and candidate.worker == worker - ) - - # Pure queue bookkeeping: a live request already represents exactly - # this authorization. A reconnect herd's repeat requests settle here - # without ever reaching the coordinator lock. - with self._initial_job_admission_lock: - existing = self._pending_initial_jobs.get(client) - if existing is not None and represents_this_authorization(existing): - self.initial_job_coalesced_count += 1 - return True - - # Coordinator-domain snapshot: whether this connection already holds - # current published-tip work is global tip identity, so it stays under - # runtime.lock -- taken alone, for a constant-time read, and released - # before the admission commit below. - with runtime.lock: - has_current_tip_job = runtime._client_has_current_tip_job_locked(client) - - # Configuration is read outside the admission lock so nothing but - # local state is touched inside it. - capacity = int(runtime.stratum_max_pending_initial_jobs) - timeout = float(runtime.stratum_initial_job_timeout_seconds) - - reject = False - deferred = False - superseded_future: Future[bool] | None = None - request: PendingInitialJob | None = None - with self._initial_job_admission_lock: - if getattr(client, "closing", False): - # Re-read under the admission lock: retirement and the timeout - # sweep both commit ``closing`` before releasing this client's - # slot, so an installation that observed a stale False here - # would strand a request behind a session already being torn - # down. - return True - existing = self._pending_initial_jobs.get(client) - if existing is not None and represents_this_authorization(existing): - self.initial_job_coalesced_count += 1 - return True - if existing is not None: - existing.cancelled.set() - superseded_future = existing.future - self.initial_job_cancelled_count += 1 - self.initial_job_superseded_count += 1 - if has_current_tip_job: - self._pending_initial_jobs.pop(client, None) - if not has_current_tip_job: - if ( - existing is None - and len(self._pending_initial_jobs) >= capacity - ): - self.initial_job_queue_rejection_count += 1 - reject = True - else: - predecessor = None - if existing is not None: - for candidate in (existing.future, existing.predecessor): - if candidate is not None and not candidate.done(): - predecessor = candidate - break - request = PendingInitialJob( - client=client, - connection_id=connection_id, - authorization_generation=generation, - difficulty_generation=difficulty_generation, - worker=worker, - requested_monotonic=now, - deadline_monotonic=now + timeout if timeout > 0 else None, - predecessor=predecessor, - ) - # Install the replacement before releasing the lock: the - # superseded future's cancellation callback runs inline - # below and must observe the replacement already holding - # the client slot. - self._pending_initial_jobs[client] = request - deferred = predecessor is not None - if superseded_future is not None: - # Cancellation callbacks run inline here, with the admission state - # already committed and the admission lock released; the - # predecessor callback then hands off exactly one client slot - # instead of mistaking the obsolete request for a terminal failure. - runtime._cancel_initial_job_future(superseded_future) - if has_current_tip_job: - return True - if reject or request is None: - runtime.disconnect_client(client) - return False - if deferred: - return True - - return runtime._submit_initial_job_request(request) - - def request_initial_job_delivery(self, client: ClientState) -> bool: - """Compatibility name for the single bounded initial-job pipeline.""" - runtime = self._runtime - return runtime.schedule_initial_job(client) - - def cancel_initial_job_delivery(self, client: ClientState) -> None: - runtime = self._runtime - runtime._ensure_initial_job_state() - runtime._cancel_pending_initial_job_locked(client, count=True) - - def _submit_initial_job_request(self, request: PendingInitialJob) -> bool: - runtime = self._runtime - client = request.client - try: - future = runtime._submit_delivery_task( - runtime.initial_job_executor(), - runtime._run_initial_job, - request, - priority=PRISM_DELIVERY_PRIORITY_INITIAL, - ) - except (_DeliveryQueueFull, RuntimeError): - disconnect = False - with self._initial_job_admission_lock: - if self._pending_initial_jobs.get(client) is request: - self._pending_initial_jobs.pop(client, None) - request.cancelled.set() - self.initial_job_queue_rejection_count += 1 - disconnect = True - if disconnect: - runtime.disconnect_client(client) - # An obsolete submit can race its already-installed replacement. - # It owns neither the client slot nor the right to retire the live - # session when admission fails. - return not disconnect - orphan: Future[bool] | None = None - with self._initial_job_admission_lock: - if self._pending_initial_jobs.get(client) is request: - request.future = future - else: - orphan = future - if orphan is not None: - # A replacement already owns the client slot. Reclaim the queue - # entry outside the admission lock; the done-callback installed - # just below then fires immediately and hands the slot on. - runtime._cancel_initial_job_future(orphan) - add_releasing_done_callback( - future, - lambda completed: runtime._initial_job_future_finished(request, completed) - ) - return True - - def _initial_job_future_finished( - self, - request: PendingInitialJob, - future: Future[bool], - ) -> None: - """Release failed first-job requests instead of stranding capacity.""" - runtime = self._runtime - delivered = False - if not future.cancelled(): - # Re-raising a stored task error here would add this request-owning - # frame to its traceback, retaining the client through the future. - error = future.exception() - if error is None: - try: - delivered = bool(future.result()) - except Exception as exc: - error = exc - if error is not None: - if not isinstance(error, Exception): - raise error - with runtime.lock: - runtime.job_build_failure_count = int( - getattr(runtime, "job_build_failure_count", 0) - ) + 1 - print( - "prism coordinator: initial job task failed " - f"connection={request.client.connection_id}", - flush=True, - ) - traceback.print_exception(error) - error = None - - # Coordinator-domain question, asked only on the delivered path and - # only for as long as the constant-time read takes. - proven_delivered = False - if delivered: - with runtime.lock: - runtime._ensure_initial_job_state() - proven_delivered = bool( - runtime._initial_request_current_locked(request) - and runtime._client_has_current_tip_job_locked(request.client) - ) - - disconnect = False - first_delivery = False - replacement: PendingInitialJob | None = None - with self._initial_job_admission_lock: - current = self._pending_initial_jobs.get(request.client) - if current is not request: - if ( - current is not None - and current.future is None - and current.predecessor is future - ): - current.predecessor = None - replacement = current - elif proven_delivered: - self._pending_initial_jobs.pop(request.client, None) - request.cancelled.set() - self.last_initial_job_delivery_monotonic = time.monotonic() - first_delivery = True - else: - self._pending_initial_jobs.pop(request.client, None) - request.cancelled.set() - if ( - request.deadline_monotonic is not None - and request.deadline_monotonic <= time.monotonic() - ): - # Commit teardown while this request still owns the pending - # slot: a concurrent reauthorization re-reads ``closing`` - # under this lock and cannot install a replacement between - # expiry and disconnect. - request.client.closing = True - self.initial_job_timeout_count += 1 - self.initial_job_cancelled_count += 1 - else: - self.initial_job_failed_count += 1 - disconnect = True - if first_delivery: - runtime._record_startup_phase_once("first_job_delivered") - if replacement is not None: - runtime._submit_initial_job_request(replacement) - if disconnect: - runtime.disconnect_client(request.client) - - def _log_accepted_parent_preview_pending( - self, - exc: AcceptedParentPayoutPreviewPending, - *, - connection_id: object, - ) -> None: - """Rate-limited retry note for accepted-parent preview backpressure. - - Kept separate from the generic build-failure log (and its own 5s - stamp) so benign backpressure never crowds out, or is mistaken for, - a counted job-build failure. - """ - now = time.monotonic() - last = getattr(self, "_last_preview_pending_log_monotonic", None) - if last is not None and now - last < 5.0: - return - self._last_preview_pending_log_monotonic = now - print( - "prism coordinator: job delivery waiting on accepted-parent payout " - f"preview reason={exc.retry_reason} connection={connection_id}; " - f"retrying: {exc}", - flush=True, - ) - - def _run_initial_job(self, request: PendingInitialJob) -> bool: - """Prepare outside client locks, then atomically stamp and send current work.""" - runtime = self._runtime - retry_delay = 0.05 - last_failure_log_monotonic: float | None = None - - def retry_later() -> bool: - nonlocal retry_delay - request.cancelled.wait(retry_delay) - retry_delay = min(1.0, retry_delay * 2) - return not runtime._initial_request_cancelled(request) - - try: - while not runtime._initial_request_cancelled(request): - try: - if not runtime.ensure_reorg_reconciled_for_current_tip(): - if not retry_later(): - return False - continue - artifacts = runtime.job_issuance_template_artifacts() - if runtime._initial_request_cancelled(request): - return False - bundle = runtime.shared_job_bundle( - artifacts, - request.worker, - request_source="initial", - cancelled=lambda: ( - runtime._initial_request_cancelled(request) - or not runtime._issuance_artifacts_current(artifacts) - ), - ) - live_tip = str(runtime.rpc.call("getbestblockhash")) - if artifacts.previousblockhash != live_tip: - # Feed the observation into detection like every other - # live-tip reader; the refresh path owns publication. - runtime.observe_tip_for_refresh(live_tip) - with runtime.lock: - published = getattr(runtime, "current_tip_first_seen", None) - pinned_authoritative = bool( - published is not None - and artifacts.previousblockhash == published[0] - and runtime._published_tip_authoritative_locked( - time.monotonic() - ) - ) - if not pinned_authoritative: - # Authority lapsed: submit classification is on the - # live RPC read now, so published-tip work would be - # rejected. Drop it and rebuild from live state. - with runtime._job_cache_lock: - if runtime._template_artifacts is artifacts: - runtime._template_artifacts = None - if not retry_later(): - return False - continue - except JobBuildWaiterCancelled: - if runtime._initial_request_cancelled(request): - return False - if not retry_later(): - return False - continue - except AcceptedParentPayoutPreviewPending as exc: - # Coordination backpressure, not a build failure: the - # accepted parent's preview has simply not published yet. - # Coalesce onto the same bounded retry as any other - # blocked refresh and stay out of - # ``job_build_failure_count``. The raise site already - # counted the timeout and scheduled the tip-refresh - # retry, so sustained pendency still escalates through - # the coordination-blocked watchdog instead of retrying - # silently forever. - if runtime._initial_request_cancelled(request): - return False - self._log_accepted_parent_preview_pending( - exc, - connection_id=request.client.connection_id, - ) - if not retry_later(): - return False - continue - except TemplateRefreshBlocked: - if runtime._initial_request_cancelled(request): - return False - if not retry_later(): - return False - continue - except Exception: - with runtime.lock: - runtime.job_build_failure_count = int( - getattr(runtime, "job_build_failure_count", 0) - ) + 1 - now = time.monotonic() - if ( - last_failure_log_monotonic is None - or now - last_failure_log_monotonic >= 5.0 - ): - last_failure_log_monotonic = now - print( - "prism coordinator: initial job preparation failed " - f"connection={request.client.connection_id}; retrying", - flush=True, - ) - traceback.print_exc() - if not retry_later(): - return False - continue - delivered = runtime._deliver_initial_bundle(request, artifacts, bundle) - if delivered is None: - if not retry_later(): - return False - continue - return delivered - return False - except OSError: - runtime.disconnect_client(request.client) - return False - - def _deliver_initial_bundle( - self, - request: PendingInitialJob, - artifacts: CachedTemplateArtifacts, - bundle: CachedJobBundle, - ) -> bool | None: - runtime = self._runtime - client = request.client - - def cancelled() -> bool: - return runtime._initial_request_cancelled(request) - - def bundle_payout_stale() -> bool: - # Unlocked at the gate (a cancellation hint, exactly like direct - # delivery's callback); authoritative only when re-read under - # _job_cache_lock at the commit boundary below. - published_artifact = runtime._published_payout_state.artifact - return ( - bundle.payout_state_generation != runtime._payout_state_generation - or bundle.build_key is None - # A periodic self-check repair replaces the published balance - # snapshot in place: the generation and append epoch both - # survive the swap, so only the digest identifies work keyed - # to the refuted balances. Every other serving decision - # (cache usability, idle issuance, cache publication) already - # applies this fence; the initial-delivery commit must too, - # or a bundle probed before the repair becomes the miner's - # active job with the refuted balances. - or published_artifact is None - or bundle.build_key.payout_artifact_sha256 - != published_artifact.prior_balances_sha256 - or ( - not bundle.collection_only - and int( - getattr( - bundle.build_key, - "payout_append_invalidation_epoch", - 0, - ) - ) - != runtime._payout_ledger_append_invalidation_epoch - ) - ) - - if not self._acquire_client_job_lock(client, cancelled): - return False - try: - if cancelled(): - return False - if not runtime._issuance_artifacts_current(artifacts): - return None - runtime._ensure_job_cache_state() - gate_started = time.monotonic() - with runtime._payout_delivery( - lambda: cancelled() or bundle_payout_stale(), - generation=bundle.payout_state_generation, - ) as admitted: - runtime._observe_payout_gate_admission( - admitted, - generation=bundle.payout_state_generation, - fallback_wait_seconds=time.monotonic() - gate_started, - ) - if cancelled(): - return False - if not admitted: - # Non-admission with a bundle in hand is transient gate - # state: a payout invalidation pending publication, or - # this bundle's generation going stale while waiting. - # Rebuild and retry within the request deadline; treating - # it as terminal turned every unlucky join during a payout - # publication into a disconnect-reconnect-rebuild loop. - return None - with runtime._job_cache_lock: - payout_current = not bundle_payout_stale() - if not payout_current or not runtime._issuance_artifacts_current(artifacts): - return None - # Difficulty state precedes coordinator admission everywhere. - # A share holding this client's Vardiff lock may delay only - # this delivery; it must never make initial-job delivery hold - # the global control-plane lock while waiting. - with runtime._client_vardiff_lock(client): - # The Vardiff wait happens after the payout check above; a - # late-visible append can invalidate the bundle during it. - # Mirror direct delivery: the live epoch re-read and the - # stamp are one commit boundary under _job_cache_lock. - with runtime._job_cache_lock: - if bundle_payout_stale(): - return None - with runtime.lock: - if not runtime._initial_request_current_locked(request): - return False - context = runtime.stamp_job_for_client( - client, - bundle, - clean_jobs=True, - ) - if not runtime._admit_client_tip_refresh_epoch_locked( - client, - context.tip_refresh_epoch_sequence, - ): - # The fence blocks only strictly older epochs, so - # this connection already registered newer refresh - # work whose delivery the epoch machinery owns. - # Retrying here can spin until the initial-job - # deadline disconnects an already-served client; - # complete a request that newer current-tip work - # satisfies instead. - runtime.note_initial_job_delivered(client) - with self._initial_job_admission_lock: - still_owns_slot = ( - self._pending_initial_jobs.get(client) - is request - ) - if not still_owns_slot: - return False - return None - client.active_job = context - for job_id in tuple(client.active_job_ids): - runtime.bury_evicted_job(client, job_id, prune=False) - self.jobs.pop(job_id, None) - client.active_job_ids.clear() - runtime.prune_evicted_job_graveyard(force=False) - self.jobs[context.job.job_id] = context - client.active_job_ids.add(context.job.job_id) - runtime.prune_client_active_jobs(client) - - runtime.send_job_update(client, context.job) - mark_delivered = getattr(admitted, "mark_delivered", None) - if callable(mark_delivered): - mark_delivered() - runtime.apply_job_difficulty(client, context.job) - runtime.note_tip_work_delivered( - client, - str(context.template["previousblockhash"]), - ) - delivered_monotonic = time.monotonic() - runtime._record_first_payout_delivery( - bundle.payout_state_generation, - delivered_monotonic, - ) - runtime._record_progress_delivery( - client, - context, - delivered_monotonic, - ) - runtime.note_initial_job_delivered(client, validated_current=True) - return True - finally: - client.job_update_lock.release() - - def sweep_initial_job_timeouts(self, *, now: float | None = None) -> int: - runtime = self._runtime - runtime._ensure_initial_job_state() - now = time.monotonic() if now is None else now - timed_out: list[PendingInitialJob] = [] - futures: list[Future[bool]] = [] - with self._initial_job_admission_lock: - expired = [ - request - for request in self._pending_initial_jobs.values() - if request.deadline_monotonic is not None - and request.deadline_monotonic <= now - ] - for request in expired: - self._pending_initial_jobs.pop(request.client, None) - request.cancelled.set() - if request.future is not None: - futures.append(request.future) - # Commit teardown while this request still owns the pending - # slot. A concurrent reauthorization re-reads closing under - # this lock and cannot install a replacement between expiry - # and disconnect. - request.client.closing = True - self.initial_job_timeout_count += 1 - self.initial_job_cancelled_count += 1 - timed_out.append(request) - # Queue reclamation runs cancellation callbacks inline; both it and - # the disconnects below stay outside the admission lock. - for future in futures: - runtime._cancel_initial_job_future(future) - for request in timed_out: - runtime.disconnect_client(request.client) - return len(timed_out) - - def initial_job_timeout_loop(self) -> None: - runtime = self._runtime - while not runtime.stop_event.wait(1.0): - runtime.sweep_initial_job_timeouts() - - def send_prepared_job( - self, - client: ClientState, - bundle: CachedJobBundle, - snapshot: QbitTipTemplateSnapshot, - validation_token: TipRefreshValidationToken, - expected_connection_id: int, - expected_active_job: PrismJobContext | None, - cancel_event: FanoutCancellation | None = None, - submitted_monotonic: float | None = None, - ) -> RefreshResult: - runtime = self._runtime - worker_started = time.monotonic() - started = worker_started if submitted_monotonic is None else submitted_monotonic - phases = runtime._job_build_phases() - phases.clear() - - def cancelled() -> bool: - return runtime._prepared_tip_refresh_obsolete( - validation_token, - bundle, - snapshot, - cancel_event, - ) or getattr(client, "closing", False) - - phases["executor_queue"] = max(0.0, worker_started - started) - client_lock_started = worker_started - client_lock_acquired = False - client_lock_attempted = False - try: - while True: - with runtime.lock: - if ( - client not in runtime.clients - or client.connection_id != expected_connection_id - or getattr(client, "closing", False) - ): - return RefreshResult("disconnected") - if cancelled(): - phases["client_lock"] = max( - 0.0, - time.monotonic() - client_lock_started, - ) - runtime._record_tip_refresh_cancellation( - "client_lock" if client_lock_attempted else "executor_queue" - ) - return RefreshResult("skipped") - client_lock_attempted = True - client_lock_acquired = client.job_update_lock.acquire( - timeout=PRISM_TIP_REFRESH_ADMISSION_POLL_SECONDS - ) - if client_lock_acquired: - break - phases["client_lock"] = max( - 0.0, - time.monotonic() - client_lock_started, - ) - if cancelled(): - runtime._record_tip_refresh_cancellation("client_lock") - return RefreshResult("skipped") - with runtime.lock: - if ( - client not in runtime.clients - or client.connection_id != expected_connection_id - ): - return RefreshResult("disconnected") - active_epoch = int( - getattr( - getattr(client, "active_job", None), - "tip_refresh_epoch_sequence", - 0, - ) - ) - epoch_fanout = bool( - getattr(runtime, "tip_refresh_epoch_fanout", False) - ) - if ( - not runtime.client_can_receive_jobs(client) - or ( - epoch_fanout - and runtime._client_tip_refresh_epoch_blocked_locked( - client, - validation_token.epoch_sequence, - ) - ) - or ( - ( - not epoch_fanout - or active_epoch >= validation_token.epoch_sequence - ) - and runtime.intervening_job_supersedes_snapshot( - client.active_job, - expected_active_job, - snapshot, - ) - ) - or not runtime.client_needs_tip_template_refresh(client, snapshot) - ): - return RefreshResult("skipped") - runtime._ensure_job_cache_state() - payout_gate_started = time.monotonic() - with runtime._payout_state_delivery_gate.delivery_cancelable( - cancelled, - generation=bundle.payout_state_generation, - priority=True, - ) as payout_admitted: - phases["payout_gate"] = max( - 0.0, - time.monotonic() - payout_gate_started, - ) - runtime._observe_payout_gate_admission( - payout_admitted, - generation=bundle.payout_state_generation, - fallback_wait_seconds=phases["payout_gate"], - ) - if not payout_admitted or cancelled(): - runtime._record_tip_refresh_cancellation("payout_gate") - return RefreshResult("skipped") - fanout_admitted = ( - cancel_event is None or cancel_event.begin_delivery() - ) - if not fanout_admitted: - runtime._record_tip_refresh_cancellation("payout_gate") - return RefreshResult("skipped") - try: - # The validation token binds this immutable bundle to the - # exact observed artifact object. Fanout tasks consult only - # in-memory publication/cancellation state, never RPC or - # the mutable cache. - # Preserve one coherent difficulty snapshot through stamp - # and commit, but acquire it before global control-plane - # admission so a concurrent share cannot form the inverse - # coordinator -> Vardiff edge. - with runtime._client_vardiff_lock(client): - with runtime.lock: - token_current = runtime._tip_refresh_token_current_locked( - validation_token, - bundle, - snapshot, - ) - if not token_current: - if cancel_event is not None: - cancel_event.cancel() - return RefreshResult("skipped") - if ( - client not in runtime.clients - or client.connection_id != expected_connection_id - ): - return RefreshResult("disconnected") - active_epoch = int( - getattr( - getattr(client, "active_job", None), - "tip_refresh_epoch_sequence", - 0, - ) - ) - epoch_fanout = bool( - getattr(runtime, "tip_refresh_epoch_fanout", False) - ) - if ( - not runtime.client_can_receive_jobs(client) - or ( - epoch_fanout - and runtime._client_tip_refresh_epoch_blocked_locked( - client, - validation_token.epoch_sequence, - ) - ) - or ( - ( - not epoch_fanout - or active_epoch - >= validation_token.epoch_sequence - ) - and runtime.intervening_job_supersedes_snapshot( - client.active_job, - expected_active_job, - snapshot, - ) - ) - or not runtime.client_needs_tip_template_refresh( - client, - snapshot, - ) - ): - return RefreshResult("skipped") - if not runtime._admit_client_tip_refresh_epoch_locked( - client, - validation_token.epoch_sequence, - ): - return RefreshResult("skipped") - clean_jobs = runtime.client_tip_changed_for_snapshot( - client, - snapshot, - ) - stamp_started = time.monotonic() - context = runtime.stamp_job_for_client( - client, - bundle, - clean_jobs=clean_jobs, - tip_refresh_epoch_sequence=( - validation_token.epoch_sequence - ), - ) - phases["stamp"] = time.monotonic() - stamp_started - client.active_job = context - if clean_jobs: - for job_id in tuple(client.active_job_ids): - runtime.bury_evicted_job(client, job_id, prune=False) - self.jobs.pop(job_id, None) - client.active_job_ids.clear() - runtime.prune_evicted_job_graveyard(force=False) - self.jobs[context.job.job_id] = context - client.active_job_ids.add(context.job.job_id) - runtime.prune_client_active_jobs(client) - - socket_send_started = time.monotonic() - try: - runtime.send_job_update(client, context.job) - payout_admitted.mark_delivered() - finally: - socket_send_finished = time.monotonic() - phases["socket_send"] = max( - 0.0, - socket_send_finished - socket_send_started, - ) - runtime.apply_job_difficulty(client, context.job) - runtime.note_tip_work_delivered( - client, - str(context.template["previousblockhash"]), - ) - runtime.note_initial_job_delivered(client, validated_current=True) - delivered_monotonic = time.monotonic() - runtime._record_first_payout_delivery( - context.payout_state_generation, - delivered_monotonic, - ) - runtime._record_progress_delivery( - client, - context, - delivered_monotonic, - ) - if getattr(runtime, "hot_path_log_enabled", False): - print( - "prism coordinator: sent prepared job " - f"connection={client.connection_id} username={client.username} " - f"job={context.job.job_id} " - f"elapsed={delivered_monotonic - started:.3f}s", - flush=True, - ) - return RefreshResult("sent", delivered_monotonic) - finally: - if cancel_event is not None: - cancel_event.end_delivery() - finally: - if client_lock_acquired: - client.job_update_lock.release() - runtime.observe_job_build_elapsed( - max(0.0, time.monotonic() - started), - phases, - ) - - def note_tip_work_delivered(self, client: ClientState, job_parent_hash: str) -> None: - """Record the first time this connection was sent work for a tip. - - First delivery wins per tip: same-tip template refreshes must not slide - the connection's stale-grace anchor forward. - """ - now = time.monotonic() - # Per-connection state only. The coordinator lock is deliberately - # absent: a delivery to one connection must never wait on fleet-wide - # control-plane work, and the O(N) current-tip coverage census that - # used to run here now runs once per observability refresh instead of - # once per delivery (#159). - # - # Anchoring is serialized by this connection's job-update lock. Every - # production delivery path already holds it across send-then-anchor, so - # re-entering it costs nothing and extends the same serialization to - # the direct compatibility callers. Lightweight embedders and focused - # doubles may supply a minimal lock exposing only the timed - # acquire/release pair the delivery path uses; those callers hold it - # already, so anchor without re-entering rather than failing on the - # missing protocol. - lock = getattr(client, "job_update_lock", None) - with lock if hasattr(lock, "__enter__") else nullcontext(): - delivered = client.tip_work_delivered - if delivered is None or delivered[0] != job_parent_hash: - client.tip_work_delivered = (job_parent_hash, now) - - def _evicted_job_class_locked(self, entry: EvictedJobEntry) -> str: - runtime = self._runtime - current_tip = runtime._current_published_tip_hash_locked() - if current_tip is None or entry.previousblockhash == current_tip: - return "same_tip" - return "stale_grace" - - def bury_evicted_job( - self, - client: ClientState, - job_id: str, - *, - now: float | None = None, - prune: bool = True, - ) -> None: - runtime = self._runtime - with runtime.lock: - runtime._ensure_evicted_job_state() - context = self.jobs.get(job_id) - if context is None: - return - self._remove_evicted_job_locked(job_id) - self.evicted_job_graveyard[job_id] = EvictedJobEntry( - context=context, - connection_id=client.connection_id, - evicted_monotonic=time.monotonic() if now is None else now, - previousblockhash=str(context.template["previousblockhash"]), - client=client, - ) - self._index_evicted_job_locked(job_id, self.evicted_job_graveyard[job_id]) - self._enforce_evicted_same_tip_capacity_locked(client.connection_id) - if prune: - runtime.prune_evicted_job_graveyard(now=now, force=False) - - def prune_evicted_job_graveyard( - self, - *, - now: float | None = None, - force: bool = True, - ) -> None: - runtime = self._runtime - with runtime.lock: - runtime._ensure_evicted_job_state() - if not self.evicted_job_graveyard: - return - now = time.monotonic() if now is None else now - if not force and now < self.evicted_job_next_prune_monotonic: - return - self.evicted_job_next_prune_monotonic = ( - now + DEFAULT_PRISM_EVICTED_JOB_PRUNE_INTERVAL_SECONDS - ) - for job_id, entry in tuple(self.evicted_job_graveyard.items()): - job_class, expired = self._evicted_job_expired_locked(entry, now=now) - if expired: - self._remove_evicted_job_locked(job_id) - self.evicted_job_expiration_counts[job_class] += 1 - - def evicted_job_entry( - self, - client: ClientState, - job_id: str, - ) -> EvictedJobEntry | None: - runtime = self._runtime - with runtime.lock: - runtime._ensure_evicted_job_state() - entry = getattr(self, "evicted_job_graveyard", {}).get(job_id) - if entry is None: - return None - if ( - entry.connection_id != client.connection_id - and not self._cross_connection_evicted_entry_allowed_locked( - entry, - client, - ) - ): - return None - job_class, expired = self._evicted_job_expired_locked( - entry, - now=time.monotonic(), - ) - if expired: - self._remove_evicted_job_locked(job_id) - self.evicted_job_expiration_counts[job_class] += 1 - return None - return entry - - def note_evicted_job_submit( - self, - credit_policy: str | None, - *, - cross_connection: bool = False, - ) -> None: - runtime = self._runtime - if credit_policy == PRISM_CREDIT_POLICY_STALE_GRACE: - outcome = "credited_stale_grace" - elif cross_connection: - outcome = "accepted_same_tip_cross_connection" - else: - outcome = "accepted_same_tip" - with runtime.lock: - runtime._ensure_evicted_job_state() - self.evicted_job_submit_counts[outcome] += 1 - - def client_can_receive_jobs(self, client: ClientState) -> bool: - return ( - not getattr(client, "closing", False) - and client.subscribed - and client.authorized - and client.worker is not None - ) - - def client_needs_tip_template_refresh( - self, - client: ClientState, - snapshot: QbitTipTemplateSnapshot, - ) -> bool: - runtime = self._runtime - context = client.active_job - if context is None: - return True - if getattr(runtime, "tip_refresh_epoch_fanout", False): - current_epoch = int( - getattr(runtime, "_tip_refresh_epoch_sequence", 0) - ) - context_epoch = int( - getattr(context, "tip_refresh_epoch_sequence", 0) - ) - if context_epoch > current_epoch: - return False - if context_epoch < current_epoch: - return True - current_epoch_tip = getattr( - runtime, - "_tip_refresh_epoch_tip_hash", - None, - ) - if ( - current_epoch_tip is not None - and str(context.template.get("previousblockhash", "")) - != current_epoch_tip - ): - return True - if getattr(context, "collection_only", False) and getattr(runtime, "_pool_ready_latched", False - ): - # The pool crossed min_ready_miners after this job was issued. - # A collection job keeps settling solved blocks solver-pays-all, - # so replace it with windowed work on the next poller pass. - return True - template = context.template - previousblockhash = str(template.get("previousblockhash", "")) - context_fingerprint = getattr(context, "template_fingerprint", None) - if context_fingerprint is None: - context_fingerprint = qbit_template_fingerprint(template) - context_payout_generation = int( - getattr(context, "payout_state_generation", 0) - ) - context_payout_artifact_sha256 = getattr( - context, - "payout_artifact_sha256", - None, - ) - published_payout_state = getattr(runtime, "_published_payout_state", None) - published_payout_artifact = ( - published_payout_state.artifact - if published_payout_state is not None - else None - ) - active_needs_refresh = ( - previousblockhash != snapshot.bestblockhash - or previousblockhash != snapshot.previousblockhash - or context_fingerprint != snapshot.template_fingerprint - or context_payout_generation - != int(getattr(runtime, "_payout_state_generation", 0)) - # A self-check repair swaps the published balance snapshot in - # place: the generation and append epoch both survive, so only - # the digest identifies an active job still mining the refuted - # balances. Jobs stamped before the digest existed (or while no - # artifact is published mid-publication) defer to the generation - # fence above. - or ( - context_payout_artifact_sha256 is not None - and published_payout_artifact is not None - and context_payout_artifact_sha256 - != published_payout_artifact.prior_balances_sha256 - ) - # A late-visible append advances only the epoch: the generation - # and digest survive, yet the job's window omitted the late row - # (possibly a replayed winning share). Collection jobs settle - # solver-pays-all and are exempt, as at every other epoch fence. - or ( - not getattr(context, "collection_only", False) - and int( - getattr(context, "payout_append_invalidation_epoch", 0) - ) - != int( - getattr( - runtime, - "_payout_ledger_append_invalidation_epoch", - 0, - ) - ) - ) - ) - if active_needs_refresh: - return True - if not getattr(runtime, "tip_refresh_epoch_fanout", False): - return False - delivered = getattr(client, "_progress_delivered_context", None) - if delivered is None: - return True - delivered_fingerprint = getattr( - delivered, - "template_fingerprint", - None, - ) - if delivered_fingerprint is None: - delivered_fingerprint = qbit_template_fingerprint( - delivered.template - ) - # Reselection must cover every condition the epoch fixpoint counts - # as undelivered, or the owning wave re-enters forever with no - # candidate to serve. A delivered collection job after the pool - # latches ready is one such condition even when its epoch, tip, - # payout generation, and fingerprint all remain current, because - # registration can advance past a delivery whose send failed. - # A client whose current-epoch delivery is registered but unproven - # is deliberately reselected here (the fence admits equal epochs, - # so the re-serve is idempotent on the wire). Deduplicating on the - # admitted epoch instead would be wrong: admission is monotonic - # and survives a failed send, so an admitted-equal skip would - # deselect the client while the fixpoint still counts it - # undelivered. - return ( - int(getattr(delivered, "connection_id", 0)) - != int(client.connection_id) - or int(getattr(delivered, "tip_refresh_epoch_sequence", 0)) - != current_epoch - or str(delivered.template.get("previousblockhash", "")) - != current_epoch_tip - or int(getattr(delivered, "payout_state_generation", 0)) - != int(getattr(runtime, "_tip_refresh_epoch_payout_generation", 0)) - or delivered_fingerprint != snapshot.template_fingerprint - or ( - bool(getattr(delivered, "collection_only", False)) - and bool(getattr(runtime, "_pool_ready_latched", False)) - ) - ) - - def intervening_job_supersedes_snapshot( - self, - active_job: PrismJobContext | None, - expected_active_job: PrismJobContext | None, - snapshot: QbitTipTemplateSnapshot, - ) -> bool: - runtime = self._runtime - if active_job is expected_active_job or active_job is None: - return False - active_payout_generation = int( - getattr(active_job, "payout_state_generation", 0) - ) - if active_payout_generation < int( - getattr(runtime, "_payout_state_generation", 0) - ): - # Template ordering cannot make a payout-stale intervening job - # authoritative. Let the reconciled refresh replace it. - return False - active_parent_hash = str( - getattr(active_job, "template", {}).get("previousblockhash", "") - ) - if ( - active_parent_hash != snapshot.bestblockhash - or active_parent_hash != snapshot.previousblockhash - ): - # Artifact generations order fetch starts, not chain tips. A fetch - # for the old tip can start after this exact new-tip observation - # and therefore carry a larger generation; it must not prevent - # the new-tip snapshot from replacing that stale work. - return False - active_generation = int(getattr(active_job, "template_generation", 0)) - snapshot_generation = int(getattr(snapshot, "template_generation", 0)) - if active_generation <= 0 or snapshot_generation <= 0: - # Legacy/test contexts without ordering metadata retain the safe - # behavior: never overwrite an unclassified intervening job. - return True - return active_generation >= snapshot_generation - - def client_tip_changed_for_snapshot( - self, - client: ClientState, - snapshot: QbitTipTemplateSnapshot, - ) -> bool: - runtime = self._runtime - context = client.active_job - if context is None: - return True - previousblockhash = str(context.template.get("previousblockhash", "")) - context_payout_artifact_sha256 = getattr( - context, - "payout_artifact_sha256", - None, - ) - published_payout_state = getattr(runtime, "_published_payout_state", None) - published_payout_artifact = ( - published_payout_state.artifact - if published_payout_state is not None - else None - ) - return ( - previousblockhash != snapshot.bestblockhash - or previousblockhash != snapshot.previousblockhash - or int(getattr(context, "payout_state_generation", 0)) - != int(getattr(runtime, "_payout_state_generation", 0)) - # A refresh selected because a self-check repair replaced the - # published balance digest must also retire the refuted job: a - # clean_jobs=False replacement leaves the old job ID admissible, - # so a late block submission could still commit the refuted - # allocation. Same None guards as the reselection predicate. - or ( - context_payout_artifact_sha256 is not None - and published_payout_artifact is not None - and context_payout_artifact_sha256 - != published_payout_artifact.prior_balances_sha256 - ) - # Likewise for append-epoch skew: the replacement window covers - # the late-visible row, so the job mining the pre-append window - # must not stay admissible beside it. - or ( - not getattr(context, "collection_only", False) - and int( - getattr(context, "payout_append_invalidation_epoch", 0) - ) - != int( - getattr( - runtime, - "_payout_ledger_append_invalidation_epoch", - 0, - ) - ) - ) - ) - - def maybe_send_job( - self, - client: ClientState, - *, - clean_jobs: bool, - raise_on_reorg_failure: bool = False, - raise_on_build_failure: bool = False, - tip_refresh_snapshot: QbitTipTemplateSnapshot | None = None, - tip_refresh_observation_sequence: int | None = None, - ) -> bool: - runtime = self._runtime - with client.job_update_lock: - return runtime._maybe_send_job_locked( - client, - clean_jobs=clean_jobs, - raise_on_reorg_failure=raise_on_reorg_failure, - raise_on_build_failure=raise_on_build_failure, - tip_refresh_snapshot=tip_refresh_snapshot, - tip_refresh_observation_sequence=tip_refresh_observation_sequence, - ) - - def _maybe_send_job_locked( - self, - client: ClientState, - *, - clean_jobs: bool, - raise_on_reorg_failure: bool = False, - raise_on_build_failure: bool = False, - tip_refresh_snapshot: QbitTipTemplateSnapshot | None = None, - tip_refresh_observation_sequence: int | None = None, - prepared_bundle: CachedJobBundle | None = None, - commit_guard: Callable[[], bool] | None = None, - commit_guard_lock: threading.RLock | None = None, - prepared_bundle_allow_uncached: bool = False, - ) -> bool: - runtime = self._runtime - if not client.subscribed or not client.authorized or client.worker is None: - return False - runtime._ensure_job_cache_state() - started = time.monotonic() - phases = runtime._job_build_phases() - phases.clear() - if getattr(runtime, "hot_path_log_enabled", False): - print( - f"prism coordinator: building job connection={client.connection_id} username={client.username}", - flush=True, - ) - phase_started = time.monotonic() - guarded_refresh = tip_refresh_snapshot is not None - if guarded_refresh != (tip_refresh_observation_sequence is not None): - raise ValueError("tip refresh snapshot and observation sequence must be paired") - if prepared_bundle is not None and guarded_refresh: - raise ValueError("prepared idle bundles cannot be combined with tip refresh guards") - if prepared_bundle_allow_uncached and prepared_bundle is None: - raise ValueError("uncached prepared delivery requires a prepared bundle") - if prepared_bundle is not None: - pass - elif guarded_refresh: - assert tip_refresh_snapshot is not None - assert tip_refresh_observation_sequence is not None - with runtime.lock: - refresh_current = runtime._tip_refresh_snapshot_current_locked( - tip_refresh_snapshot, - tip_refresh_observation_sequence, - ) - if not refresh_current: - runtime._schedule_tip_refresh_retry() - raise TemplateRefreshSuperseded( - "tip refresh snapshot was superseded before client job build" - ) - try: - chain_view_untrusted = bool( - getattr(runtime, "reorg_reconciler_enabled", True) - and runtime.qbit_chain_view_untrusted() - ) - except Exception as exc: - runtime._schedule_tip_refresh_retry() - raise _TipRefreshTrustBlocked( - "qbit chain trust check failed before sequential client job build" - ) from exc - if chain_view_untrusted: - runtime._schedule_tip_refresh_retry() - raise _TipRefreshTrustBlocked( - "qbit chain view became untrusted before sequential client job build" - ) - else: - try: - if not runtime.ensure_reorg_reconciled_for_current_tip(): - if raise_on_reorg_failure: - raise _TipRefreshTrustBlocked( - "qbit chain view became untrusted before client job build" - ) - return False - except AcceptedParentPayoutPreviewPending: - # Propagated verbatim: a late accepted-parent preview must - # never be relabelled into a reorg-reconciliation failure by - # the generic clause below, which would arm the ordinary - # template-refresh failure budget. - raise - except TemplateRefreshBlocked: - raise - except Exception as exc: - print( - f"prism coordinator: reorg reconciliation failed before job build " - f"connection={client.connection_id} username={client.username}; skipping this job", - flush=True, - ) - traceback.print_exc() - if raise_on_reorg_failure: - raise TemplateRefreshBlocked( - "reorg reconciliation failed before client job build" - ) from exc - return False - phases["reorg"] = time.monotonic() - phase_started - built_from_guarded_artifacts = bool( - guarded_refresh - and tip_refresh_snapshot.template_artifacts is not None - and "build_job_for_client" not in runtime.__dict__ - ) - try: - if prepared_bundle is not None: - context = runtime.stamp_job_for_client( - client, - prepared_bundle, - clean_jobs=clean_jobs, - ) - elif built_from_guarded_artifacts: - assert tip_refresh_snapshot is not None - assert tip_refresh_snapshot.template_artifacts is not None - context = runtime.build_job_for_client_from_artifacts( - client, - tip_refresh_snapshot.template_artifacts, - clean_jobs=clean_jobs, - publication_critical=True, - request_source="tip_refresh", - ) - else: - context = runtime.build_job_for_client(client, clean_jobs=clean_jobs) - except AcceptedParentPayoutPreviewPending as exc: - # Same coalescing retry as any other blocked build, but typed all - # the way up: the caller's coordination-blocked handling keeps it - # out of ``job_build_failure_count`` and the ordinary - # template-refresh failure budget, while its own longer streak - # deadline preserves the existing watchdog/health escalation. - runtime._schedule_tip_refresh_retry() - self._log_accepted_parent_preview_pending( - exc, - connection_id=client.connection_id, - ) - if guarded_refresh or raise_on_reorg_failure or raise_on_build_failure: - raise - return False - except TemplateRefreshBlocked: - runtime._schedule_tip_refresh_retry() - if guarded_refresh or raise_on_reorg_failure or raise_on_build_failure: - raise - return False - except Exception as exc: - # A single bad template (e.g. a coinbase whose bytes collide with the - # extranonce placeholder, or a transient getblocktemplate failure) must - # never tear down the miner's connection. Log it, count it, and skip - # this job; the next share/retarget or block change rebuilds a fresh one. - # Only the build is isolated: nothing has been registered or sent yet, so - # there is no stale job state. Downstream send failures still surface to - # handle_client, which disconnects the (now dead) socket and cleans up. - with runtime.lock: - runtime.job_build_failure_count += 1 - print( - f"prism coordinator: job build failed connection={client.connection_id} " - f"username={client.username}; keeping client connected and skipping this template", - flush=True, - ) - traceback.print_exc() - if raise_on_build_failure: - raise JobBuildFailed( - f"job build failed for connection {client.connection_id}" - ) from exc - return False - # Linearize direct delivery against the immutable publication pointer. - # Expensive build and ledger reads happened under the preparation lock, - # outside this admission boundary. - with runtime._job_cache_lock: - current_payout_generation = runtime._payout_state_generation - current_append_invalidation_epoch = ( - runtime._payout_ledger_append_invalidation_epoch - ) - published_tip = runtime._published_payout_state.source_tip_hash - publication_blocked = runtime._payout_state_publication_blocked - context_payout_generation = int( - getattr( - context, - "payout_state_generation", - current_payout_generation, - ) - ) - context_append_invalidation_epoch = int( - getattr(context, "payout_append_invalidation_epoch", 0) - ) - context_collection_only = bool( - getattr(context, "collection_only", False) - ) - context_payout_artifact_sha256 = getattr( - context, - "payout_artifact_sha256", - None, - ) - - def context_payout_digest_stale() -> bool: - # A periodic self-check repair swaps the published balance - # snapshot in place: the payout generation and append epoch both - # survive the swap, so only the digest identifies a context still - # keyed to the refuted balances. Contexts without a digest defer - # to the generation fence. Unlocked at the gate (a cancellation - # hint); authoritative when re-read under _job_cache_lock at the - # commit boundary below. - if context_payout_artifact_sha256 is None: - return False - published_artifact = runtime._published_payout_state.artifact - return ( - published_artifact is None - or context_payout_artifact_sha256 - != published_artifact.prior_balances_sha256 - ) - - context_template = getattr(context, "template", None) - context_parent = ( - str(context_template.get("previousblockhash", "")) - if isinstance(context_template, dict) - else "" - ) - with runtime.lock: - published_authority = getattr(runtime, "current_tip_first_seen", None) - published_authoritative = runtime._published_tip_authoritative_locked( - time.monotonic() - ) - pinned_published_delivery = bool( - context_parent - and published_authority is not None - and context_parent == published_authority[0] - and published_authoritative - ) - lapsed_live_validated = False - if ( - not guarded_refresh - and context_parent - and published_authority is not None - and not published_authoritative - ): - # The published authority lapsed, so per-share classification has - # fallen back to the live RPC read. Mirror the initial-job path: - # revalidate against the live tip (outside every lock), record - # the observation for the refresh machinery, and drop work that - # would be stale on arrival. - try: - lapsed_live_tip = str(runtime.rpc.call("getbestblockhash")) - except Exception: - lapsed_live_tip = None - if lapsed_live_tip is not None: - runtime.observe_tip_for_refresh(lapsed_live_tip) - if context_parent != lapsed_live_tip: - runtime._schedule_tip_refresh_retry() - return False - lapsed_live_validated = True - priority_delivery = ( - not publication_blocked - and context_payout_generation == current_payout_generation - and ( - context_collection_only - or context_append_invalidation_epoch - == current_append_invalidation_epoch - ) - and ( - published_tip is None - or context_parent == published_tip - # Reconciliation can source payout state at the detected tip - # before submit authority flips. Pinned published-tip work is - # still the currently creditable work, so it must keep the - # priority lane instead of tripping the gate's non-priority - # same-generation rejection for the whole unpublished window. - or pinned_published_delivery - ) - ) - payout_gate_started = time.monotonic() - with runtime._payout_state_delivery_gate.delivery_cancelable( - lambda: ( - context_payout_generation != runtime._payout_state_generation - or context_payout_digest_stale() - or ( - not context_collection_only - and context_append_invalidation_epoch - != runtime._payout_ledger_append_invalidation_epoch - ) - ), - generation=context_payout_generation, - priority=priority_delivery, - ) as payout_admitted: - payout_gate_wait = max(0.0, time.monotonic() - payout_gate_started) - phases["payout_gate"] = phases.get("payout_gate", 0.0) + payout_gate_wait - runtime._observe_payout_gate_admission( - payout_admitted, - generation=context_payout_generation, - fallback_wait_seconds=payout_gate_wait, - ) - if not payout_admitted: - runtime._schedule_tip_refresh_retry() - if guarded_refresh: - raise TemplateRefreshSuperseded( - "payout state changed during client job build" - ) - return False - def commit_context_locked() -> bool: - if getattr(client, "closing", False): - return False - if commit_guard is not None and not commit_guard(): - return False - if guarded_refresh: - assert tip_refresh_snapshot is not None - assert tip_refresh_observation_sequence is not None - if not runtime._tip_refresh_snapshot_current_locked( - tip_refresh_snapshot, - tip_refresh_observation_sequence, - ): - runtime._schedule_tip_refresh_retry() - raise TemplateRefreshSuperseded( - "tip refresh snapshot was superseded during client job build" - ) - artifacts = tip_refresh_snapshot.template_artifacts - if built_from_guarded_artifacts and artifacts is not None and ( - context.template is not artifacts.template - or context.template_fingerprint != artifacts.fingerprint - or context.template_generation != artifacts.generation - ): - raise TemplateRefreshBlocked( - "client job build did not use the guarded refresh artifacts" - ) - else: - published_now = getattr(runtime, "current_tip_first_seen", None) - if published_now is None: - # Bootstrap: nothing published yet, so there is no - # authority for this work to contradict. - pass - elif runtime._published_tip_authoritative_locked( - time.monotonic() - ): - if context_parent and context_parent != published_now[0]: - # Authority moved while this direct build waited - # on the payout gate or client lock. Sending now - # would advertise work that classifies stale-job - # on arrival; skip and let the refresh fanout (or - # the pending retry) deliver current work. - runtime._schedule_tip_refresh_retry() - return False - elif context_parent and not lapsed_live_validated: - # The lease lapsed during the wait and this context - # never passed a live-tip revalidation: cached - # observations may be blind to the tip mining.submit - # now classifies against. Defer to a fresh pass whose - # pre-wait live check settles it without holding the - # coordinator lock across an RPC. - runtime._schedule_tip_refresh_retry() - return False - if not runtime._admit_client_tip_refresh_epoch_locked( - client, - int(getattr(context, "tip_refresh_epoch_sequence", 0)), - ): - runtime._schedule_tip_refresh_retry() - return False - client.active_job = context - if clean_jobs: - for job_id in client.active_job_ids: - runtime.bury_evicted_job(client, job_id, prune=False) - self.jobs.pop(job_id, None) - client.active_job_ids.clear() - runtime.prune_evicted_job_graveyard(force=False) - self.jobs[context.job.job_id] = context - client.active_job_ids.add(context.job.job_id) - runtime.prune_client_active_jobs(client) - return True - - def commit_context() -> bool: - if prepared_bundle is not None: - # Exact cache identity and the client/window guard are one - # commit boundary. A tip or payout publication that wins - # this race makes the idle task a no-op instead of - # delivering stale work. - with runtime._job_cache_lock: - if not runtime._idle_bundle_current_locked( - client, - prepared_bundle, - allow_uncached=prepared_bundle_allow_uncached, - ): - return False - with runtime.lock: - return commit_context_locked() - with runtime._job_cache_lock: - if context_payout_digest_stale() or ( - not context_collection_only - and context_append_invalidation_epoch - != runtime._payout_ledger_append_invalidation_epoch - ): - return False - with runtime.lock: - return commit_context_locked() - - if commit_guard_lock is None: - committed = commit_context() - else: - # Per-client state is acquired before coordinator admission. - # A busy share can delay only this delivery; it cannot hold - # runtime.lock while the delivery waits for Vardiff state. - with commit_guard_lock: - committed = commit_context() - if not committed: - return False - phase_started = time.monotonic() - runtime.send_job_update(client, context.job) - payout_admitted.mark_delivered() - runtime.apply_job_difficulty(client, context.job) - runtime.note_tip_work_delivered(client, str(context.template["previousblockhash"])) - delivered_monotonic = time.monotonic() - runtime._record_first_payout_delivery( - context_payout_generation, - delivered_monotonic, - ) - runtime._record_progress_delivery( - client, - context, - delivered_monotonic, - ) - runtime._consume_retained_collection_refresh(context) - runtime.note_initial_job_delivered( - client, - validated_current=guarded_refresh, - ) - phases["send"] = delivered_monotonic - phase_started - elapsed = time.monotonic() - started - runtime.observe_job_build_elapsed(elapsed, phases) - if getattr(runtime, "hot_path_log_enabled", False): - phase_report = ",".join( - f"{phase}:{phases[phase]:.3f}" - for phase in PRISM_JOB_BUILD_PHASES - if phase in phases - ) - print( - f"prism coordinator: sent job connection={client.connection_id} username={client.username} " - f"job={context.job.job_id} collection={context.collection_only} elapsed={elapsed:.3f}s " - f"phases={phase_report}", - flush=True, - ) - return True - - def prune_client_active_jobs(self, client: ClientState) -> None: - runtime = self._runtime - for job_id in tuple(client.active_job_ids): - if job_id not in self.jobs: - client.active_job_ids.discard(job_id) - ordered_active_job_ids = [ - job_id for job_id in self.jobs if job_id in client.active_job_ids - ] - while len(ordered_active_job_ids) > MAX_ACTIVE_PRISM_JOBS_PER_CLIENT: - oldest_job_id = ordered_active_job_ids.pop(0) - client.active_job_ids.remove(oldest_job_id) - runtime.bury_evicted_job(client, oldest_job_id) - self.jobs.pop(oldest_job_id, None) - - def apply_job_difficulty(self, client: ClientState, job: direct_stratum.DirectQbitStratumJob) -> None: - runtime = self._runtime - with runtime._client_vardiff_lock(client): - config = ( - client.vardiff_config - or client.listener_vardiff_config - or runtime.vardiff_config - ) - if not config.enabled: - client.share_difficulty = job.share_difficulty - client.pending_share_difficulty = None - return - pending = client.pending_share_difficulty - client.share_difficulty = job.share_difficulty - if pending is not None and job.share_difficulty == pending: - client.pending_share_difficulty = None - - def apply_client_difficulty_requests(self, client: ClientState) -> Decimal | None: - """Specialize the client's difficulty policy from its recorded requests - (password ``d=``/``md=`` and ``mining.suggest_difficulty``), clamped to - the pristine listener bounds. The listener floor always wins: on a - high-diff listener no request can drop a client below the configured - minimum. Explicit ``d=`` outranks a suggestion. Returns the resolved - target difficulty, or None when the client requested nothing.""" - runtime = self._runtime - with runtime._client_vardiff_lock(client): - base = client.listener_vardiff_config or runtime.vardiff_config - requested = ( - client.requested_difficulty - if client.requested_difficulty is not None - else client.suggested_difficulty - ) - if requested is None and client.requested_min_difficulty is None: - # No live requests: drop any stale specialization so the client - # falls back to the pristine listener policy. - client.vardiff_config = None - return None - floor = base.min_difficulty - if client.requested_min_difficulty is not None: - floor = vardiff.clamp( - client.requested_min_difficulty, - base.min_difficulty, - base.max_difficulty, - ) - if requested is None: - requested = client.share_difficulty - target = vardiff.clamp(requested, floor, base.max_difficulty) - client.vardiff_config = dataclass_replace( - base, - min_difficulty=floor, - startup_difficulty=target, - ) - return target - - def advertise_client_difficulty(self, client: ClientState, target: Decimal) -> bool: - """Move a client to an explicitly requested difficulty. - - Before the client can receive jobs the value is applied directly (the - first set_difficulty/notify pair picks it up). Afterwards it uses the - same job-gated pending mechanism as vardiff retargets: the difficulty - is advertised together with the job it applies to, or not at all. - Returns True only when a fresh set_difficulty/notify pair went out, so - callers about to send their own job can skip a duplicate pair.""" - runtime = self._runtime - with client.job_update_lock: - return runtime._advertise_client_difficulty_locked(client, target) - - def _advertise_client_difficulty_locked( - self, - client: ClientState, - target: Decimal, - ) -> bool: - runtime = self._runtime - applied_directly = False - schedule_initial = False - with runtime._client_vardiff_lock(client): - current = client.pending_share_difficulty or client.share_difficulty - if target == current: - return False - if not (client.subscribed and client.authorized) or ( - client.active_job is None and "maybe_send_job" not in runtime.__dict__ - ): - client.share_difficulty = target - client.pending_share_difficulty = None - client.difficulty_generation = int( - getattr(client, "difficulty_generation", 0) - ) + 1 - applied_directly = True - schedule_initial = bool( - client.subscribed - and client.authorized - and client.worker is not None - ) - else: - prior_pending = client.pending_share_difficulty - prior_generation = int( - getattr(client, "difficulty_generation", 0) - ) - advertised_generation = prior_generation + 1 - client.pending_share_difficulty = target - client.difficulty_generation = advertised_generation - if applied_directly: - if schedule_initial: - # A pending first-job request captured the previous difficulty - # generation. Replace it atomically so its cancellation callback - # hands the client slot to current work instead of disconnecting. - runtime.request_initial_job_delivery(client) - return False - runtime._ensure_initial_job_state() - # Lock-protected membership read; the replacement decision itself is - # re-taken exactly under the same lock in schedule_initial_job. - with self._initial_job_admission_lock: - initial_pending = client in self._pending_initial_jobs - if initial_pending: - runtime.request_initial_job_delivery(client) - return False - if not runtime.stop_event.is_set() and runtime.maybe_send_job(client, clean_jobs=True): - return True - with runtime._client_vardiff_lock(client): - if ( - client.pending_share_difficulty == target - and int(getattr(client, "difficulty_generation", 0)) - == advertised_generation - ): - client.pending_share_difficulty = prior_pending - client.difficulty_generation = prior_generation - return False - - def send_job_update( - self, - client: ClientState, - job: direct_stratum.DirectQbitStratumJob, - ) -> None: - runtime = self._runtime - # Preserve instance-level send method replacements used by focused - # tests; normal coordinators use the atomic socket batch below. - if "send_difficulty" in runtime.__dict__ or "send_job" in runtime.__dict__: - runtime.send_difficulty(client, job) - runtime.send_job(client, job) - return - payloads = [ - runtime.difficulty_payload(job.share_difficulty), - runtime.job_payload(job), - ] - shared_params = getattr(job, "notify_shared_params_json", None) - if shared_params is None: - client.send_batch(payloads) - return - # Splice the per-client fields around the fragment serialized once - # per bundle build; at fleet scale the coinb1/coinb2/merkle parts - # dominate the notify and are byte-identical for every client. - notify_line = ( - '{"id": null, "method": "mining.notify", "params": [' - + json.dumps(job.job_id) - + ", " - + shared_params - + ", " - + ("true" if job.clean_jobs else "false") - + "]}" - ) - client.send_batch( - payloads, - preserialized=( - json.dumps(payloads[0]).encode() - + b"\n" - + notify_line.encode() - + b"\n" - ), - ) - - def build_job_for_client(self, client: ClientState, *, clean_jobs: bool) -> PrismJobContext: - runtime = self._runtime - if client.worker is None: - raise self._stratum_error(20, "client is not authorized") - runtime._ensure_job_cache_state() - artifacts = ( - runtime._retained_collection_artifacts() - or runtime.job_issuance_template_artifacts() - ) - return runtime.build_job_for_client_from_artifacts( - client, - artifacts, - clean_jobs=clean_jobs, - ) - - def build_job_for_client_from_artifacts( - self, - client: ClientState, - artifacts: CachedTemplateArtifacts, - *, - clean_jobs: bool, - publication_critical: bool = False, - request_source: str = "routine", - ) -> PrismJobContext: - runtime = self._runtime - runtime._ensure_job_cache_state() - phases = runtime._job_build_phases() - while True: - worker = client.worker - if worker is None: - raise self._stratum_error(20, "client is not authorized") - progress_build_token = runtime._progress_bundle_build_started() - try: - cached_bundle = runtime.shared_job_bundle( - artifacts, - worker, - publication_critical=publication_critical, - request_source=request_source, - ) - finally: - runtime._progress_bundle_build_finished(progress_build_token) - current_worker = client.worker - if not cached_bundle.collection_only or current_worker == worker: - break - # Reauthorization changed a genuine collection input while the - # worker-specific bundle was being built. Re-select the latest - # identity without refetching or discarding the exact artifacts. - stamp_started = time.monotonic() - context = runtime.stamp_job_for_client(client, cached_bundle, clean_jobs=clean_jobs) - phases["stamp"] = phases.get("stamp", 0.0) + (time.monotonic() - stamp_started) - return context - - def _acquire_client_job_lock( - self, - client: ClientState, - cancelled: Callable[[], bool], - ) -> bool: - while not cancelled(): - if client.job_update_lock.acquire(timeout=0.1): - return True - return False - - @contextmanager - def _cancellable_client_job_lock( - self, - client: ClientState, - cancelled: Callable[[], bool], - ) -> Iterator[bool]: - acquired = self._acquire_client_job_lock(client, cancelled) - try: - yield acquired - finally: - if acquired: - client.job_update_lock.release() - - @staticmethod - def _client_has_delivered_work_locked(client: ClientState) -> bool: - """Return whether a socket write completed for any usable job.""" - - return bool( - client.tip_work_delivered is not None - or client._progress_delivered_context is not None - ) - - def _enforce_evicted_same_tip_capacity_locked( - self, - connection_id: int | None = None, - ) -> None: - runtime = self._runtime - connection_ids = ( - (connection_id,) - if connection_id is not None - else tuple(self.evicted_same_tip_by_connection) - ) - per_connection_cap = int(runtime.same_tip_job_retention_per_connection) - for candidate_connection_id in connection_ids: - job_ids = self.evicted_same_tip_by_connection.get(candidate_connection_id) - while job_ids is not None and len(job_ids) > per_connection_cap: - oldest_job_id = next(iter(job_ids)) - self._remove_evicted_job_locked(oldest_job_id) - self.evicted_job_capacity_eviction_counts["connection"] += 1 - job_ids = self.evicted_same_tip_by_connection.get(candidate_connection_id) - - def _evicted_job_expired_locked( - self, - entry: EvictedJobEntry, - *, - now: float, - ) -> tuple[str, bool]: - runtime = self._runtime - job_class = runtime._evicted_job_class_locked(entry) - if job_class == "same_tip": - ttl = float(runtime.same_tip_job_retention_seconds) - return job_class, ttl <= 0 or now - entry.evicted_monotonic > ttl - return job_class, self._stale_grace_entry_expired_locked( - entry, - now=now, - ttl=float( - getattr( - runtime, - "stale_grace_seconds", - DEFAULT_PRISM_STALE_GRACE_SECONDS, - ) - ), - ) - - def _index_evicted_job_locked(self, job_id: str, entry: EvictedJobEntry) -> None: - runtime = self._runtime - self.evicted_jobs_by_connection.setdefault( - entry.connection_id, - OrderedDict(), - )[job_id] = None - if runtime._evicted_job_class_locked(entry) != "same_tip": - return - self.evicted_same_tip_by_connection.setdefault( - entry.connection_id, - OrderedDict(), - )[job_id] = None - self.evicted_same_tip_job_ids[job_id] = None - - def _rebuild_evicted_job_indexes_locked(self) -> None: - runtime = self._runtime - self.evicted_jobs_by_connection = {} - self.evicted_same_tip_by_connection = {} - self.evicted_same_tip_job_ids = OrderedDict() - for job_id, entry in self.evicted_job_graveyard.items(): - self._index_evicted_job_locked(job_id, entry) - self.evicted_job_index_tip_hash = runtime._current_published_tip_hash_locked() - self._enforce_evicted_same_tip_capacity_locked() - - def _remove_evicted_job_locked(self, job_id: str) -> EvictedJobEntry | None: - entry = self.evicted_job_graveyard.pop(job_id, None) - if entry is None: - return None - connection_jobs = self.evicted_jobs_by_connection.get(entry.connection_id) - if connection_jobs is not None: - connection_jobs.pop(job_id, None) - if not connection_jobs: - self.evicted_jobs_by_connection.pop(entry.connection_id, None) - connection_jobs = self.evicted_same_tip_by_connection.get(entry.connection_id) - if connection_jobs is not None: - connection_jobs.pop(job_id, None) - if not connection_jobs: - self.evicted_same_tip_by_connection.pop(entry.connection_id, None) - self.evicted_same_tip_job_ids.pop(job_id, None) - self._disconnected_evicted_job_ids.pop(job_id, None) - return entry - - def _stale_grace_entry_expired_locked( - self, - entry: EvictedJobEntry, - *, - now: float, - ttl: float, - ) -> bool: - runtime = self._runtime - current_tip = runtime._current_published_tip_hash_locked() - first_seen = getattr(runtime, "current_tip_first_seen", None) - if ( - ttl <= 0 - or current_tip is None - or first_seen is None - or str(first_seen[0]) != current_tip - or first_seen[1] is None - ): - return True - - # Submit eligibility is exactly one chain parent behind, so pruning - # must use that same relationship. The prior poll observation can lag - # (for example when authorize/vardiff issued work on an intermediate - # tip), and using it here would drop work submit would still credit. - # Until the parent RPC has populated the cache, retain conservatively; - # submit classification fetches it before granting stale grace. - cached_parent = getattr(runtime, "current_tip_parent", None) - if ( - cached_parent is not None - and cached_parent[0] == current_tip - and entry.previousblockhash != cached_parent[1] - ): - return True - - client = entry.client - if client is not None: - delivered = client.tip_work_delivered - if delivered is None or delivered[0] != current_tip: - # Match stale_grace_deadline_open: prior-tip shares stay in - # flight until this connection receives replacement work. - return False - anchor = delivered[1] - else: - # Disconnect normally removes these entries. Keep legacy/test - # orphan state bounded from the refresh path's tip-flip anchor. - anchor = float(first_seen[1]) - return now - anchor > ttl - - def _enforce_disconnected_evicted_capacity_locked(self) -> None: - runtime = self._runtime - cap = int( - getattr( - runtime, - "disconnected_job_retention", - DEFAULT_PRISM_DISCONNECTED_JOB_RETENTION, - ) - ) - while len(self._disconnected_evicted_job_ids) > max(0, cap): - oldest_job_id = next(iter(self._disconnected_evicted_job_ids)) - self._remove_evicted_job_locked(oldest_job_id) - self.evicted_job_capacity_eviction_counts["disconnected"] += 1 - - def cleanup_disconnected_client(self, client: ClientState) -> None: - """Retire a disconnected client's job/retention state (S1 port). - - The caller (the session owner's disconnect path) already holds - ``client.job_update_lock``; every mixed lock path uses - ``job_update_lock -> coordinator lock`` and this method takes the - coordinator lock itself. - """ - runtime = self._runtime - with runtime.lock: - runtime._ensure_evicted_job_state() - retention_cap = int( - getattr( - runtime, - "disconnected_job_retention", - DEFAULT_PRISM_DISCONNECTED_JOB_RETENTION, - ) - ) - if retention_cap > 0: - # Devices behind a flapping proxy keep mining the - # jobs this connection delivered; bury the active - # ones so their in-flight shares can resume against - # the retained context after the reconnect. - for job_id in tuple(client.active_job_ids): - runtime.bury_evicted_job(client, job_id, prune=False) - for job_id in tuple(client.active_job_ids): - self.jobs.pop(job_id, None) - client.active_job_ids.clear() - client.active_job = None - for job_id in tuple( - self.evicted_jobs_by_connection.get(client.connection_id, ()) - ): - entry = self.evicted_job_graveyard.get(job_id) - if entry is None: - continue - if ( - retention_cap <= 0 - or self._evicted_job_class_locked(entry) != "same_tip" - ): - # Stale-grace work stays tied to the connection - # that received it (its expiry anchors on that - # connection's delivered work); only fully valid - # same-tip contexts survive the disconnect. - self._remove_evicted_job_locked(job_id) - continue - # Detach from the dead connection object: expiry then - # anchors on the entry's own eviction time (same-tip - # TTL) or the published tip flip, never on delivery - # state this connection can no longer advance. - self.evicted_job_graveyard[job_id] = dataclass_replace( - entry, - client=None, - ) - self._disconnected_evicted_job_ids[job_id] = None - self._enforce_disconnected_evicted_capacity_locked() - client.authorized = False - client.worker = None - client.username = "" - - def _cross_connection_evicted_entry_allowed_locked( - self, - entry: EvictedJobEntry, - client: ClientState, - ) -> bool: - """Whether a retained context of a dead connection may serve this - submitter. - - Only same-tip contexts qualify (they are fully valid current work; - stale-grace expiry anchors on the original connection's delivery - state, which a dead connection can never advance), and only for an - authorized reconnect of the same username -- the credit goes to the - context's original worker identity either way, so a foreign - submitter could at most donate work, but refusing keeps job ids - unguessable-by-effect and the accounting per miner honest. Replays - stay rejected by the (username, header) dedup key. - """ - runtime = self._runtime - if entry.client is not None: - # The owning connection is still alive; job ids stay scoped to it. - return False - if ( - int( - getattr( - runtime, - "disconnected_job_retention", - DEFAULT_PRISM_DISCONNECTED_JOB_RETENTION, - ) - ) - <= 0 - ): - return False - if not client.authorized or not client.username: - return False - worker = getattr(entry.context, "worker", None) - username = getattr(worker, "username", None) - if not username or username != client.username: - return False - return runtime._evicted_job_class_locked(entry) == "same_tip" - - -__all__ = [ - "DEFAULT_PRISM_EVICTED_JOB_PRUNE_INTERVAL_SECONDS", - "DEFAULT_PRISM_INITIAL_JOB_MAX_WORKERS", - "EvictedJobEntry", - "JobBuildFailed", - "JobDeliveryRuntime", - "JobDeliveryService", - "MAX_ACTIVE_PRISM_JOBS_PER_CLIENT", - "PRISM_CREDIT_POLICY_STALE_GRACE", - "PRISM_DELIVERY_PRIORITY_INITIAL", - "PRISM_DELIVERY_PRIORITY_NEW_TIP", - "PRISM_DELIVERY_PRIORITY_SAME_TIP", - "PRISM_EVICTED_JOB_CAPACITY_SCOPES", - "PRISM_EVICTED_JOB_CLASSES", - "PRISM_EVICTED_JOB_SUBMIT_OUTCOMES", - "PRISM_TIP_REFRESH_ADMISSION_POLL_SECONDS", - "PendingInitialJob", - "PrismJobContext", -] diff --git a/lab/prism/metrics.py b/lab/prism/metrics.py deleted file mode 100644 index 9cb4bd21f..000000000 --- a/lab/prism/metrics.py +++ /dev/null @@ -1,2031 +0,0 @@ -"""Read-only Prometheus assembly for the PRISM coordinator. - -The renderer owns the complete live/base metrics document and the formatter -bodies PR 80 extracted from the coordinator. It holds no producer state: -counters, histograms, and label registries stay with their owner services -(J1, B1, share submission, P1, reorg, ledger, observability) and reach the -renderer through the :class:`MetricsPort` as copied snapshots or bounded -per-domain ``*_metrics_lines()`` formatters. The renderer never calls the -cached-metrics path, so the snapshot refresher can call it without recursion. -""" - -from __future__ import annotations - -import time -from typing import Any, Protocol - -from lab.prism.accepted_preview_telemetry import ( - PRISM_ACCEPTED_LANDING_PHASES, - PRISM_PAYOUT_WINDOW_FULL_RESCAN_PATHS, - PRISM_PAYOUT_WINDOW_FULL_RESCAN_REASONS, - PRISM_REORG_RECONCILE_CALLERS, - PRISM_REORG_RECONCILE_STEPS, - ensure_accepted_preview_telemetry, - fold_ledger_read_stats, -) -from lab.prism.block_candidates import ( - PRISM_ACCEPTED_BLOCK_PREVIEW_PUBLICATION_RESULTS, - PRISM_ACCEPTED_BLOCK_PREVIEW_PUBLICATION_SECONDS_BUCKETS, - PRISM_BLOCK_CANDIDATE_COLLAPSE_OUTCOMES, - PRISM_STALE_JOB_ABANDON_CLASSES, -) -from lab.prism.coordinator_config import ( - DEFAULT_PRISM_INITIAL_JOB_MAX_WORKERS, - DEFAULT_PRISM_JOB_BUILD_EXECUTOR_WORKERS, - DEFAULT_PRISM_MALLOC_TELEMETRY, - env_malloc_telemetry, -) -from lab.prism.job_bundle import ( - PRISM_JOB_BUILD_PHASES, - PRISM_JOB_BUILD_SECONDS_BUCKETS, - PRISM_JOB_CACHE_KINDS, -) -from lab.prism.job_delivery import ( - PRISM_EVICTED_JOB_CAPACITY_SCOPES, - PRISM_EVICTED_JOB_CLASSES, - PRISM_EVICTED_JOB_SUBMIT_OUTCOMES, -) -from lab.prism.background_services import PRISM_GC_PAUSE_SECONDS_BUCKETS -from lab.prism.process_telemetry import ( - PRISM_GC_GENERATIONS, - ProcessHeapTelemetry, -) -from lab.prism.reorg_reconciler import ( - PRISM_REORG_RECONCILE_LOOKUP_PATHS, - PRISM_REORG_RECONCILE_LOOKUP_SOURCES, -) -from lab.prism.share_ledger import DaemonShareWindowMirror -from lab.prism.share_submission import PRISM_SHARE_ACK_RESULTS -from lab.prism.vardiff_service import PRISM_VARDIFF_RESUME_OUTCOMES -from lab.prism.writer_lease_timing import ( - LEASE_HEARTBEAT_MODES, - LEASE_HEARTBEAT_OUTCOMES, - LEASE_HEARTBEAT_PHASES, - LEASE_HEARTBEAT_POLICY_TERMS, - LEASE_MONITOR_LATE_WAKE_SLACK_FRACTIONS, - LEASE_MONITOR_WAKE_DELAY_BUCKETS, -) - -# Issue #226 (and the gauge list issue #185 asked for): the closed component -# label set of qbit_prism_component_entries. Every entry is one len() or -# Queue.qsize() over a structure its owner constructs in __init__ (or -# backfills through its own _ensure_* method), so a scrape costs a fixed -# number of constant-time reads and the series set cannot grow with jobs, -# tips, candidates, connections, or generations. Extend this tuple -# deliberately: tests/test_prism_metrics.py pins it as the family's exact -# series set. -PRISM_COMPONENT_ENTRY_KINDS = ( - # The payout window: the in-process incremental window's pages and - # records, the armed ledger artifact's share sequence, and the - # payout-state maps that are keyed by hash, epoch, or token. - "payout_window_pages", - "payout_window_records", - "payout_ledger_artifact_shares", - "accepted_block_payout_previews", - "invalidated_accepted_block_payout_previews", - "payout_append_invalidation_stamps", - "payout_window_inflight_scan_anchors", - "payout_unfenced_append_inflight_stamps", - # The single-slot share-window serialization cache: shares it covers. - "share_window_serialization_shares", - # Job contexts and the bundle cache. - "job_contexts", - "job_bundle_cache", - "job_build_issued_stamps", - "bundle_preparation_flights", - "active_job_bundle_builds", - # The evicted-job graveyard. - "evicted_job_graveyard", - "evicted_same_tip_job_ids", - # Candidate and replay state: the live, replay, and quarantine lanes - # plus every hash-keyed registry the B1 owner retains. - "block_candidate_queue", - "block_replay_queue", - "block_replay_inflight_hashes", - "block_quarantine_queue", - "block_quarantine_hashes", - "outstanding_block_candidate_hashes", - "tip_observed_accepted_block_hashes", - "counted_block_candidate_abandonments", - "accepted_block_preview_stamps", - "ancestor_redrive_records", - "block_candidate_terminal_outcomes", - "block_candidate_disposition_flights", - "block_disposition_waiting_retries", - "block_candidate_dequeued_hashes", - "accounted_accepted_block_hashes", - # Reconcile flights and the trusted-tip memo. - "reconcile_flights", - "reconcile_trusted_memo", - # Pending-share holders on the snapshot-anchor floor. - "pending_share_commit_floor", - # The daemon mirror: records in a mirror-backed window, and the - # compiler's bounded mirror of the daemon's uploaded-window LRU. - "daemon_window_mirror_records", - "daemon_uploaded_windows", -) -# The closed component label set of qbit_prism_component_bytes: the retained -# byte-sized payloads whose size, not count, is what scales with the window. -PRISM_COMPONENT_BYTE_KINDS = ( - "payout_window_canonical_json", - "daemon_window_mirror_canonical_items", - "share_window_serialization_spool", - "share_window_serialization_compact_json", -) - - -class MetricsPort(Protocol): - """Explicit read capabilities used to assemble one metrics document. - - The renderer is observational but not pure: it still performs live - RPC/ledger reads and calls bounded retained-job pruning, and several - ``_ensure_*`` calls can initialize owner state on first touch. The - issue #224 attribution owner is likewise attached to the port's - instance ``__dict__`` on first render - (``ensure_accepted_preview_telemetry``), so the port needs an - instance dictionary but no dedicated attribute or method for it. - """ - - ledger: Any - rpc: Any - lock: Any - clients: Any - connection_limit_rejection_counts: Any - evicted_job_capacity_eviction_counts: Any - evicted_job_expiration_counts: Any - evicted_job_graveyard: Any - evicted_job_submit_counts: Any - evicted_same_tip_job_ids: Any - worker_metrics_lock: Any - worker_share_counts: Any - worker_rejection_counts: Any - accepted_block_count: Any - matured_payout_count: Any - post_accept_refresh_failure_count: Any - reorg_inactive_block_count: Any - reorg_reactivated_block_count: Any - reorg_reconcile_error_count: Any - reorg_reconcile_skip_count: Any - reorg_reconcile_lookup_counts: Any - started_monotonic: Any - tip_refresh_job_count: Any - vardiff_config: Any - listener_profiles: Any - accept_resource_exhaustion_count: Any - block_candidate_abandoned_counts: Any - block_candidate_accept_pending_defer_count: Any - block_candidate_poisoned_count: Any - block_candidate_retry_count: Any - block_candidate_wakeups_coalesced: Any - block_candidates_dropped: Any - block_solves_dropped_counts: Any - stale_job_abandon_counts: Any - connection_setup_failure_count: Any - idle_retarget_count: Any - jobs: Any - jobs_lock: Any - job_build_failure_count: Any - latest_coinbase_size_bytes: Any - share_append_queue: Any - share_append_failure_count: Any - shares_recovered_to_disk: Any - shares_replayed: Any - share_replay_conflicts: Any - initial_job_sent_count: Any - initial_job_cancelled_count: Any - initial_job_coalesced_count: Any - initial_job_failed_count: Any - initial_job_superseded_count: Any - initial_job_delivery_latency_seconds_sum: Any - initial_job_delivery_latency_count: Any - initial_job_queue_capacity_reclaimed_count: Any - initial_job_max_workers: Any - _initial_job_executor: Any - _bundle_preparation_lock: Any - shared_bundle_build_counts: Any - shared_bundle_preparation_seconds_sum: Any - shared_bundle_preparation_count: Any - shared_bundle_preparation_waiters: Any - _job_cache_lock: Any - job_build_seconds_bucket_counts: Any - job_build_seconds_sum: Any - job_build_count: Any - job_build_phase_seconds: Any - job_cache_hit_counts: Any - job_cache_miss_counts: Any - _job_build_scheduler_lock: Any - job_build_scheduler_counts: Any - job_build_priority_counts: Any - job_build_priority_admission_seconds: Any - initial_job_prepared_work_counts: Any - job_build_cancellation_seconds: Any - job_build_replacement_start_seconds: Any - job_build_worker_counts: Any - job_build_executor_workers: Any - _job_build_active: Any - _job_build_pending: Any - _job_build_retiring: Any - _job_build_priority_preparations: Any - _accepted_parent_preview_wait_timeouts: Any - accepted_parent_redrive_attempt_count: Any - accepted_parent_redrive_resolved_count: Any - accepted_parent_redrive_exhausted_count: Any - _accounted_accepted_block_hashes: Any - - def _accepted_parent_unresolved_depth(self) -> int: ... - def _accepted_parent_unresolved_depth_cap(self) -> int: ... - def _ensure_block_candidate_service(self) -> Any: ... - def _ensure_bundle_compiler(self) -> Any: ... - def _ensure_connection_capacity_state(self) -> Any: ... - def _ensure_evicted_job_state(self) -> Any: ... - def _ensure_initial_job_state(self) -> Any: ... - def _ensure_job_bundle_service(self) -> Any: ... - def _ensure_job_cache_state(self) -> Any: ... - def _ensure_observability_service(self) -> Any: ... - def _ensure_lease_heartbeat_service(self) -> Any: ... - def _ensure_payout_state_service(self) -> Any: ... - def _ensure_share_writer_service(self) -> Any: ... - def _ensure_shutdown_controller(self) -> Any: ... - def _ensure_worker_metrics_state(self) -> Any: ... - def _job_build_is_publication_critical(self, request: Any) -> bool: ... - def accepted_share_stats(self) -> Any: ... - def accepted_parent_unresolved_ages_seconds(self) -> list[float]: ... - def audit_artifact_metrics(self) -> Any: ... - def block_candidate_collapse_snapshot(self) -> dict[str, int]: ... - def block_finalization_metrics_lines(self) -> Any: ... - def block_ledger_call_class_metrics(self) -> Any: ... - def block_submitter_snapshot(self) -> dict[str, object]: ... - def coordinator_lock_contention_snapshot(self) -> tuple[int, int, float, float]: ... - def ctv_fanout_broadcaster_metrics_lines(self) -> Any: ... - def mining_delivery_snapshot(self) -> Any: ... - def payout_state_metrics_lines(self) -> Any: ... - def process_resource_metrics(self) -> Any: ... - def progress_health_metrics_lines(self) -> Any: ... - def prometheus_label_value(self, value: str) -> str: ... - def prune_evicted_job_graveyard(self, *, force: bool) -> Any: ... - def rejection_reason_ids(self) -> tuple[str, ...]: ... - def share_accounting_snapshot(self) -> dict[str, object]: ... - def share_ack_snapshot(self) -> dict[str, dict[str, Any]]: ... - def startup_phase_seconds(self) -> dict[str, float]: ... - def tip_refresh_metrics_lines(self) -> Any: ... - def vardiff_convergence_snapshot(self) -> Any: ... - def vardiff_idle_metrics_lines(self) -> Any: ... - - -class MetricsRenderer: - """Collect and format one complete metrics generation.""" - - def __init__( - self, - port: MetricsPort, - *, - process_telemetry: ProcessHeapTelemetry | None = None, - ) -> None: - self.port = port - if process_telemetry is None: - # Resolution order, most authoritative first. A port attribute - # lets an embedder or a test pin the switch outright. Otherwise - # the validated lifecycle snapshot the coordinator was built with - # wins: a caller that passed an explicit CoordinatorConfig with - # telemetry disabled must not have mallinfo2 run anyway because - # the ambient environment happens to say otherwise. Only with no - # snapshot at all do we read the environment. Resolved once here - # so a scrape never touches any of them. - enabled = getattr(port, "malloc_telemetry_enabled", None) - if enabled is None: - config = getattr(port, "config", None) - lifecycle = getattr(config, "lifecycle", None) - enabled = getattr(lifecycle, "malloc_telemetry_enabled", None) - if enabled is None: - try: - enabled = env_malloc_telemetry() - except BaseException: - # The config loader validated this switch at startup, so - # reaching here means the environment changed underneath a - # running process. env_malloc_telemetry is fail-closed and - # exits on a bad value, which is right at startup and wrong - # on a metrics thread: killing the coordinator over a - # telemetry switch would turn an observability knob into an - # outage. Fall back to the shipped default instead. - enabled = DEFAULT_PRISM_MALLOC_TELEMETRY - process_telemetry = ProcessHeapTelemetry(malloc_enabled=bool(enabled)) - self._process_telemetry = process_telemetry - - def render(self) -> str: - ledger_metrics = self.port.ledger.metrics() - audit_metrics = self.port.audit_artifact_metrics() - mining_metrics = self.port.mining_delivery_snapshot() - process_rss_bytes, process_open_fds = self.port.process_resource_metrics() - accepted_share_count = self.port.accepted_share_stats()[0] - share_accounting = self.port.share_accounting_snapshot() - submitted_share_count = int(share_accounting["submitted"]) - stale_share_count = int(share_accounting["stale"]) - duplicate_share_count = int(share_accounting["duplicate"]) - low_difficulty_share_count = int(share_accounting["low_difficulty"]) - collection_block_submission_count = int( - share_accounting["collection_block"] - ) - rejection_counts = share_accounting["rejections"] - assert isinstance(rejection_counts, dict) - grace_credited_share_count = int(share_accounting["grace_credited"]) - # The submission owner's copied snapshot (via the routing descriptor) - # is read outside the accounting lock so owner locks never nest. - block_solves_dropped_counts = dict( - getattr( - self.port, - "block_solves_dropped_counts", - {"stale_grace": 0}, - ) - ) - stale_percent = 0.0 - if submitted_share_count > 0: - stale_percent = (stale_share_count / submitted_share_count) * 100.0 - idle_retarget_count = int(getattr(self.port, "idle_retarget_count", 0)) - with self.port.lock: - self.port._ensure_connection_capacity_state() - active_connection_count = len(self.port.clients) - connection_limit_rejection_counts = dict( - self.port.connection_limit_rejection_counts - ) - accept_resource_exhaustion_count = int( - getattr(self.port, "accept_resource_exhaustion_count", 0) - ) - connection_setup_failure_count = int( - getattr(self.port, "connection_setup_failure_count", 0) - ) - self.port._ensure_evicted_job_state() - self.port.prune_evicted_job_graveyard(force=False) - same_tip_context_count = len(self.port.evicted_same_tip_job_ids) - evicted_job_context_counts = { - "same_tip": same_tip_context_count, - "stale_grace": len(self.port.evicted_job_graveyard) - same_tip_context_count, - } - evicted_job_submit_counts = dict(self.port.evicted_job_submit_counts) - evicted_job_expiration_counts = dict(self.port.evicted_job_expiration_counts) - evicted_job_capacity_eviction_counts = dict( - self.port.evicted_job_capacity_eviction_counts - ) - stale_job_abandon_counts = dict( - getattr( - self.port, - "stale_job_abandon_counts", - { - abandon_class: 0 - for abandon_class in PRISM_STALE_JOB_ABANDON_CLASSES - }, - ) - ) - self.port._ensure_worker_metrics_state() - with self.port.worker_metrics_lock: - worker_share_counts = { - label: dict(counts) - for label, counts in self.port.worker_share_counts.items() - } - worker_rejection_counts = dict(self.port.worker_rejection_counts) - coinbase_weight_headroom = 2_000_000 - latest_coinbase_size_bytes = getattr( - self.port, "latest_coinbase_size_bytes", None - ) - if latest_coinbase_size_bytes is not None: - coinbase_weight_headroom = 2_000_000 - int(latest_coinbase_size_bytes) - ctv_pending = 0 - ctv_broadcastable = 0 - ctv_failed = 0 - pending_ctv_fanouts = getattr( - self.port.ledger, "pending_ctv_fanout_statuses", None - ) - if callable(pending_ctv_fanouts): - try: - for fanout in pending_ctv_fanouts(limit=1_000): - ctv_pending += 1 - status = str(fanout.get("settlement_status", "")) - if status == "broadcastable": - ctv_broadcastable += 1 - elif status == "failed": - ctv_failed += 1 - except Exception: - ctv_pending = -1 - ctv_broadcastable = -1 - ctv_failed = -1 - if ctv_failed >= 0: - ctv_failed = int(ledger_metrics.get("ctv_fanouts_failed", ctv_failed)) - ibd = 0 - peers = 0 - try: - blockchain_info = self.port.rpc.call("getblockchaininfo") - if isinstance(blockchain_info, dict) and blockchain_info.get("initialblockdownload"): - ibd = 1 - except Exception: - ibd = -1 - try: - network_info = self.port.rpc.call("getnetworkinfo") - if isinstance(network_info, dict): - peers = int(network_info.get("connections", 0)) - except Exception: - peers = -1 - rejection_reason_ids = self.port.rejection_reason_ids() - lines = [ - "# HELP qbit_prism_accepted_shares_total Accepted shares recorded by the canonical PRISM ledger.", - "# TYPE qbit_prism_accepted_shares_total counter", - f"qbit_prism_accepted_shares_total {accepted_share_count}", - *self.accepted_stats_reconcile_metric_lines(), - *self.landing_observability_metrics_lines(), - "# HELP qbit_prism_submitted_shares_total Stratum share submissions seen by the PRISM coordinator.", - "# TYPE qbit_prism_submitted_shares_total counter", - f"qbit_prism_submitted_shares_total {submitted_share_count}", - "# HELP qbit_prism_stratum_active_connections Active admitted Stratum connections across all listeners.", - "# TYPE qbit_prism_stratum_active_connections gauge", - f"qbit_prism_stratum_active_connections {active_connection_count}", - "# HELP qbit_prism_stratum_connection_limit Configured global Stratum connection ceiling; zero means unlimited.", - "# TYPE qbit_prism_stratum_connection_limit gauge", - f"qbit_prism_stratum_connection_limit {mining_metrics['connection_capacity']}", - "# HELP qbit_prism_stratum_peak_active_connections Peak admitted Stratum connections since process start.", - "# TYPE qbit_prism_stratum_peak_active_connections gauge", - f"qbit_prism_stratum_peak_active_connections {mining_metrics['peak_active_connections']}", - "# HELP qbit_prism_stratum_subscribed_connections Active subscribed Stratum connections.", - "# TYPE qbit_prism_stratum_subscribed_connections gauge", - f"qbit_prism_stratum_subscribed_connections {mining_metrics['subscribed_connections']}", - "# HELP qbit_prism_stratum_authorized_connections Active subscribed and authorized Stratum connections.", - "# TYPE qbit_prism_stratum_authorized_connections gauge", - f"qbit_prism_stratum_authorized_connections {mining_metrics['authorized_connections']}", - "# HELP qbit_prism_stratum_pending_initial_jobs Authorized clients awaiting their first usable current-tip job.", - "# TYPE qbit_prism_stratum_pending_initial_jobs gauge", - f"qbit_prism_stratum_pending_initial_jobs {mining_metrics['pending_initial_jobs']}", - "# HELP qbit_prism_stratum_pending_initial_job_limit Configured bound for clients awaiting their first usable job.", - "# TYPE qbit_prism_stratum_pending_initial_job_limit gauge", - f"qbit_prism_stratum_pending_initial_job_limit {mining_metrics['pending_initial_job_capacity']}", - "# HELP qbit_prism_stratum_oldest_pending_initial_job_seconds Age of the oldest pending first-job request.", - "# TYPE qbit_prism_stratum_oldest_pending_initial_job_seconds gauge", - f"qbit_prism_stratum_oldest_pending_initial_job_seconds {mining_metrics['oldest_pending_initial_job_age_seconds']}", - "# HELP qbit_prism_stratum_oldest_genuinely_pending_initial_job_seconds Age of the oldest authorized client that has never received usable work.", - "# TYPE qbit_prism_stratum_oldest_genuinely_pending_initial_job_seconds gauge", - f"qbit_prism_stratum_oldest_genuinely_pending_initial_job_seconds {mining_metrics['oldest_genuinely_pending_initial_job_age_seconds']}", - "# HELP qbit_prism_stratum_current_tip_coverage_gap_seconds Continuous age of current-tip job coverage below 95 percent.", - "# TYPE qbit_prism_stratum_current_tip_coverage_gap_seconds gauge", - f"qbit_prism_stratum_current_tip_coverage_gap_seconds {mining_metrics['current_tip_coverage_gap_age_seconds']}", - "# HELP qbit_prism_stratum_initial_job_queue_rejections_total Sessions closed because bounded first-job delivery was full.", - "# TYPE qbit_prism_stratum_initial_job_queue_rejections_total counter", - f"qbit_prism_stratum_initial_job_queue_rejections_total {mining_metrics['initial_job_queue_rejections']}", - "# HELP qbit_prism_stratum_initial_job_timeouts_total Sessions disconnected after first-job delivery timed out.", - "# TYPE qbit_prism_stratum_initial_job_timeouts_total counter", - f"qbit_prism_stratum_initial_job_timeouts_total {mining_metrics['initial_job_timeout_disconnects']}", - "# HELP qbit_prism_stratum_initial_job_tasks_total First-job tasks canceled or coalesced before duplicate work.", - "# TYPE qbit_prism_stratum_initial_job_tasks_total counter", - f'qbit_prism_stratum_initial_job_tasks_total{{result="cancelled"}} {mining_metrics["initial_job_cancelled_tasks"]}', - f'qbit_prism_stratum_initial_job_tasks_total{{result="coalesced"}} {mining_metrics["initial_job_coalesced_tasks"]}', - "# HELP qbit_prism_stratum_initial_job_queue_capacity_reclaimed_total Queued first-job admission slots reclaimed immediately by cancellation.", - "# TYPE qbit_prism_stratum_initial_job_queue_capacity_reclaimed_total counter", - f'qbit_prism_stratum_initial_job_queue_capacity_reclaimed_total {mining_metrics["initial_job_queue_capacity_reclaimed"]}', - "# HELP qbit_prism_stratum_clients_with_current_tip_jobs Authorized clients holding usable current-tip work.", - "# TYPE qbit_prism_stratum_clients_with_current_tip_jobs gauge", - f"qbit_prism_stratum_clients_with_current_tip_jobs {mining_metrics['clients_with_current_tip_jobs']}", - "# HELP qbit_prism_stratum_current_tip_job_coverage Ratio of authorized clients holding current-tip work.", - "# TYPE qbit_prism_stratum_current_tip_job_coverage gauge", - f"qbit_prism_stratum_current_tip_job_coverage {mining_metrics['current_tip_job_coverage']}", - "# HELP qbit_prism_stratum_semantic_current_work_ratio Ratio of authorized clients whose work matches the current template fingerprint, payout generation, and client session.", - "# TYPE qbit_prism_stratum_semantic_current_work_ratio gauge", - f"qbit_prism_stratum_semantic_current_work_ratio {mining_metrics['semantic_current_work_ratio']}", - "# HELP qbit_prism_stratum_handler_threads Active per-connection Stratum handler threads.", - "# TYPE qbit_prism_stratum_handler_threads gauge", - f"qbit_prism_stratum_handler_threads {mining_metrics['handler_threads']}", - "# HELP qbit_prism_job_delivery_queue_depth Current bounded delivery executor queue depth.", - "# TYPE qbit_prism_job_delivery_queue_depth gauge", - f"qbit_prism_job_delivery_queue_depth {mining_metrics['delivery_executor_queue_depth']}", - "# HELP qbit_prism_job_delivery_active_workers Delivery executor workers currently running tasks.", - "# TYPE qbit_prism_job_delivery_active_workers gauge", - f"qbit_prism_job_delivery_active_workers {mining_metrics['delivery_executor_active_workers']}", - "# HELP qbit_prism_process_resident_memory_bytes Current process RSS bytes, or -1 when unavailable.", - "# TYPE qbit_prism_process_resident_memory_bytes gauge", - f"qbit_prism_process_resident_memory_bytes {process_rss_bytes}", - "# HELP qbit_prism_process_open_file_descriptors Current process open descriptor count, or -1 when unavailable.", - "# TYPE qbit_prism_process_open_file_descriptors gauge", - f"qbit_prism_process_open_file_descriptors {process_open_fds}", - "# HELP qbit_prism_stratum_connection_limit_rejections_total Stratum connections rejected by an explicitly configured admission limit.", - "# TYPE qbit_prism_stratum_connection_limit_rejections_total counter", - *[ - f'qbit_prism_stratum_connection_limit_rejections_total{{scope="{scope}"}} {int(connection_limit_rejection_counts.get(scope, 0))}' - for scope in ("global", "username") - ], - "# HELP qbit_prism_stratum_accept_resource_exhaustions_total Recoverable Stratum accept or client-setup failures caused by process or system descriptor exhaustion.", - "# TYPE qbit_prism_stratum_accept_resource_exhaustions_total counter", - f"qbit_prism_stratum_accept_resource_exhaustions_total {accept_resource_exhaustion_count}", - "# HELP qbit_prism_stratum_connection_setup_failures_total Admitted Stratum connections cleaned up after socket or handler-thread setup failure.", - "# TYPE qbit_prism_stratum_connection_setup_failures_total counter", - f"qbit_prism_stratum_connection_setup_failures_total {connection_setup_failure_count}", - "# HELP qbit_prism_stale_shares_total Stratum shares rejected or ignored as stale.", - "# TYPE qbit_prism_stale_shares_total counter", - f"qbit_prism_stale_shares_total {stale_share_count}", - "# HELP qbit_prism_duplicate_shares_total Duplicate Stratum shares rejected.", - "# TYPE qbit_prism_duplicate_shares_total counter", - f"qbit_prism_duplicate_shares_total {duplicate_share_count}", - "# HELP qbit_prism_low_difficulty_shares_total Low-difficulty Stratum shares rejected.", - "# TYPE qbit_prism_low_difficulty_shares_total counter", - f"qbit_prism_low_difficulty_shares_total {low_difficulty_share_count}", - "# HELP qbit_prism_collection_block_submissions_total Solver-pays-all block candidates submitted from collection-mode jobs.", - "# TYPE qbit_prism_collection_block_submissions_total counter", - f"qbit_prism_collection_block_submissions_total {collection_block_submission_count}", - "# HELP qbit_prism_grace_credited_shares_total Accepted shares credited by the stale-grace policy.", - "# TYPE qbit_prism_grace_credited_shares_total counter", - f"qbit_prism_grace_credited_shares_total {grace_credited_share_count}", - "# HELP qbit_prism_block_solves_dropped_total Block-passing submissions intentionally excluded from block submission by a bounded policy reason.", - "# TYPE qbit_prism_block_solves_dropped_total counter", - *[ - f'qbit_prism_block_solves_dropped_total{{reason="{reason}"}} {int(block_solves_dropped_counts.get(reason, 0))}' - for reason in ("stale_grace",) - ], - "# HELP qbit_prism_rejections_total PRISM share or block rejections by canonical reason ID.", - "# TYPE qbit_prism_rejections_total counter", - *[ - f'qbit_prism_rejections_total{{reason_id="{reason}"}} {int(rejection_counts.get(reason, 0))}' - for reason in rejection_reason_ids - ], - "# HELP qbit_prism_worker_submitted_shares_total Stratum share submissions by bounded worker label.", - "# TYPE qbit_prism_worker_submitted_shares_total counter", - *[ - f'qbit_prism_worker_submitted_shares_total{{worker="{self.port.prometheus_label_value(label)}"}} {int(counts.get("submitted", 0))}' - for label, counts in sorted(worker_share_counts.items()) - ], - "# HELP qbit_prism_worker_accepted_shares_total Accepted shares by bounded worker label.", - "# TYPE qbit_prism_worker_accepted_shares_total counter", - *[ - f'qbit_prism_worker_accepted_shares_total{{worker="{self.port.prometheus_label_value(label)}"}} {int(counts.get("accepted", 0))}' - for label, counts in sorted(worker_share_counts.items()) - ], - "# HELP qbit_prism_worker_grace_credited_shares_total Stale-grace credited shares by bounded worker label.", - "# TYPE qbit_prism_worker_grace_credited_shares_total counter", - *[ - f'qbit_prism_worker_grace_credited_shares_total{{worker="{self.port.prometheus_label_value(label)}"}} {int(counts.get("grace", 0))}' - for label, counts in sorted(worker_share_counts.items()) - ], - "# HELP qbit_prism_worker_rejections_total PRISM share or block rejections by bounded worker label and reason ID.", - "# TYPE qbit_prism_worker_rejections_total counter", - *[ - f'qbit_prism_worker_rejections_total{{worker="{self.port.prometheus_label_value(label)}",reason_id="{reason}"}} {int(count)}' - for (label, reason), count in sorted(worker_rejection_counts.items()) - ], - "# HELP qbit_prism_job_build_failures_total Job builds skipped after a template/coinbase error without dropping the client.", - "# TYPE qbit_prism_job_build_failures_total counter", - f"qbit_prism_job_build_failures_total {self.port.job_build_failure_count}", - "# HELP qbit_prism_block_candidates_dropped_total Legacy counter; durable candidate outbox rows are never dropped on queue overflow.", - "# TYPE qbit_prism_block_candidates_dropped_total counter", - f"qbit_prism_block_candidates_dropped_total {int(getattr(self.port, 'block_candidates_dropped', 0))}", - "# HELP qbit_prism_block_candidate_wakeups_coalesced_total Candidate queue wakeups coalesced while the durable outbox retained the work.", - "# TYPE qbit_prism_block_candidate_wakeups_coalesced_total counter", - f"qbit_prism_block_candidate_wakeups_coalesced_total {int(getattr(self.port, 'block_candidate_wakeups_coalesced', 0))}", - "# HELP qbit_prism_block_candidate_retries_total Transient candidate outcomes retained for durable retry.", - "# TYPE qbit_prism_block_candidate_retries_total counter", - f"qbit_prism_block_candidate_retries_total {int(getattr(self.port, 'block_candidate_retry_count', 0))}", - "# HELP qbit_prism_block_candidate_accept_pending_defers_total Terminal abandonments refused because the candidate is (or was recently observed as) an active chain block; the candidate retries until its accepted success tail finalizes it as submitted.", - "# TYPE qbit_prism_block_candidate_accept_pending_defers_total counter", - f"qbit_prism_block_candidate_accept_pending_defers_total {int(getattr(self.port, 'block_candidate_accept_pending_defer_count', 0))}", - "# HELP qbit_prism_block_candidate_poisoned_total Invalid durable candidate intents quarantined from replay.", - "# TYPE qbit_prism_block_candidate_poisoned_total counter", - f"qbit_prism_block_candidate_poisoned_total {int(getattr(self.port, 'block_candidate_poisoned_count', 0))}", - "# HELP qbit_prism_block_candidates_abandoned_total Block candidates that did not land (lost tip race or failed submit), by reason. Not share rejections: the underlying share was accepted.", - "# TYPE qbit_prism_block_candidates_abandoned_total counter", - *[ - f'qbit_prism_block_candidates_abandoned_total{{reason_id="{reason}"}} {int(count)}' - for reason, count in sorted(getattr(self.port, "block_candidate_abandoned_counts", {}).items()) - ], - "# HELP qbit_prism_stale_job_abandons_total Terminal stale-job block candidate abandonments by bounded cause.", - "# TYPE qbit_prism_stale_job_abandons_total counter", - *[ - f'qbit_prism_stale_job_abandons_total{{class="{abandon_class}"}} {int(stale_job_abandon_counts.get(abandon_class, 0))}' - for abandon_class in PRISM_STALE_JOB_ABANDON_CLASSES - ], - "# HELP qbit_prism_share_append_queue_depth Accepted shares waiting on the ledger writer thread.", - "# TYPE qbit_prism_share_append_queue_depth gauge", - f"qbit_prism_share_append_queue_depth {self.port.share_append_queue.qsize() if getattr(self.port, 'share_append_queue', None) is not None else 0}", - "# HELP qbit_prism_share_append_failures_total Shares in group commits that failed before acknowledgement.", - "# TYPE qbit_prism_share_append_failures_total counter", - f"qbit_prism_share_append_failures_total {int(getattr(self.port, 'share_append_failure_count', 0))}", - "# HELP qbit_prism_shares_recovered_to_disk_total Legacy pre-commit-ACK shares written to the upgrade recovery file.", - "# TYPE qbit_prism_shares_recovered_to_disk_total counter", - f"qbit_prism_shares_recovered_to_disk_total {int(getattr(self.port, 'shares_recovered_to_disk', 0))}", - "# HELP qbit_prism_shares_replayed_total Recovery-file shares replayed into the ledger at startup.", - "# TYPE qbit_prism_shares_replayed_total counter", - f"qbit_prism_shares_replayed_total {int(getattr(self.port, 'shares_replayed', 0))}", - "# HELP qbit_prism_share_replay_conflicts_total Recovery-file rows quarantined because the durable row disagrees with the journal payload.", - "# TYPE qbit_prism_share_replay_conflicts_total counter", - f"qbit_prism_share_replay_conflicts_total {int(getattr(self.port, 'share_replay_conflicts', 0))}", - "# HELP qbit_prism_tip_refresh_jobs_total Client jobs refreshed after qbit tip/template changes.", - "# TYPE qbit_prism_tip_refresh_jobs_total counter", - f"qbit_prism_tip_refresh_jobs_total {self.port.tip_refresh_job_count}", - "# HELP qbit_prism_active_job_contexts Current retained PRISM job contexts.", - "# TYPE qbit_prism_active_job_contexts gauge", - f"qbit_prism_active_job_contexts {len(getattr(self.port, 'jobs', {}))}", - "# HELP qbit_prism_evicted_job_contexts Evicted job contexts retained by safety class.", - "# TYPE qbit_prism_evicted_job_contexts gauge", - *[ - f'qbit_prism_evicted_job_contexts{{class="{job_class}"}} {evicted_job_context_counts[job_class]}' - for job_class in PRISM_EVICTED_JOB_CLASSES - ], - "# HELP qbit_prism_evicted_job_submits_total Accepted submits validated against an evicted job context.", - "# TYPE qbit_prism_evicted_job_submits_total counter", - *[ - f'qbit_prism_evicted_job_submits_total{{outcome="{outcome}"}} {int(evicted_job_submit_counts.get(outcome, 0))}' - for outcome in PRISM_EVICTED_JOB_SUBMIT_OUTCOMES - ], - "# HELP qbit_prism_evicted_job_expirations_total Retained job contexts removed after their class TTL.", - "# TYPE qbit_prism_evicted_job_expirations_total counter", - *[ - f'qbit_prism_evicted_job_expirations_total{{class="{job_class}"}} {int(evicted_job_expiration_counts.get(job_class, 0))}' - for job_class in PRISM_EVICTED_JOB_CLASSES - ], - "# HELP qbit_prism_evicted_job_capacity_evictions_total Same-tip retained contexts removed by a configured count limit.", - "# TYPE qbit_prism_evicted_job_capacity_evictions_total counter", - *[ - f'qbit_prism_evicted_job_capacity_evictions_total{{scope="{scope}"}} {int(evicted_job_capacity_eviction_counts.get(scope, 0))}' - for scope in PRISM_EVICTED_JOB_CAPACITY_SCOPES - ], - "# HELP qbit_prism_post_accept_refresh_failures_total Immediate clean-job refreshes that failed after direct block acceptance.", - "# TYPE qbit_prism_post_accept_refresh_failures_total counter", - f"qbit_prism_post_accept_refresh_failures_total {self.port.post_accept_refresh_failure_count}", - "# HELP qbit_prism_reorg_inactive_blocks_total PRISM pool blocks quarantined after leaving the active chain.", - "# TYPE qbit_prism_reorg_inactive_blocks_total counter", - f"qbit_prism_reorg_inactive_blocks_total {self.port.reorg_inactive_block_count}", - "# HELP qbit_prism_reorg_reactivated_blocks_total Quarantined PRISM pool blocks restored after returning to the active chain.", - "# TYPE qbit_prism_reorg_reactivated_blocks_total counter", - f"qbit_prism_reorg_reactivated_blocks_total {self.port.reorg_reactivated_block_count}", - "# HELP qbit_prism_reorg_reconcile_skips_total Reorg reconciliation passes skipped because qbitd chain view was not trusted.", - "# TYPE qbit_prism_reorg_reconcile_skips_total counter", - f"qbit_prism_reorg_reconcile_skips_total {self.port.reorg_reconcile_skip_count}", - "# HELP qbit_prism_reorg_reconcile_errors_total Reorg reconciliation errors that prevented ordered job issuance.", - "# TYPE qbit_prism_reorg_reconcile_errors_total counter", - f"qbit_prism_reorg_reconcile_errors_total {self.port.reorg_reconcile_error_count}", - "# HELP qbit_prism_reorg_reconcile_lookups_total Reconcile demand by caller path and the source that satisfied it.", - "# TYPE qbit_prism_reorg_reconcile_lookups_total counter", - *[ - f'qbit_prism_reorg_reconcile_lookups_total{{path="{path}",source="{source}"}} ' - f"{int(getattr(self.port, 'reorg_reconcile_lookup_counts', {}).get((path, source), 0))}" - for path in PRISM_REORG_RECONCILE_LOOKUP_PATHS - for source in PRISM_REORG_RECONCILE_LOOKUP_SOURCES - ], - "# HELP qbit_prism_matured_payouts_total Payout entries marked mature by the coordinator tip reconciliation path.", - "# TYPE qbit_prism_matured_payouts_total counter", - f"qbit_prism_matured_payouts_total {self.port.matured_payout_count}", - "# HELP qbit_prism_vardiff_idle_retargets_total Vardiff retargets triggered by the idle zero-accepted-share sweep.", - "# TYPE qbit_prism_vardiff_idle_retargets_total counter", - f"qbit_prism_vardiff_idle_retargets_total {idle_retarget_count}", - # qbit_prism_shares_per_second is deliberately absent. It divided - # the ledger's lifetime accepted count by this process's uptime -- - # different domains, so the value tracked 1/uptime rather than any - # share rate: enormous right after a restart, decaying toward zero - # as the process aged, and never once the pool's throughput. Use - # rate(qbit_prism_accepted_shares_total[5m]), which is what the - # counter above exists for, or sum the per-lane gauge for this - # process's own since-start commit rate. - "# HELP qbit_prism_stale_share_percent Percent of submitted shares classified stale.", - "# TYPE qbit_prism_stale_share_percent gauge", - f"qbit_prism_stale_share_percent {stale_percent:.12g}", - "# HELP qbit_prism_blocks_accepted_total Blocks accepted through the PRISM coordinator.", - "# TYPE qbit_prism_blocks_accepted_total counter", - f"qbit_prism_blocks_accepted_total {self.port.accepted_block_count}", - "# HELP qbit_prism_persisted_blocks Persisted PRISM pool block rows.", - "# TYPE qbit_prism_persisted_blocks gauge", - f"qbit_prism_persisted_blocks {ledger_metrics['blocks']}", - "# HELP qbit_prism_inactive_pool_blocks PRISM pool block rows currently quarantined as inactive.", - "# TYPE qbit_prism_inactive_pool_blocks gauge", - f"qbit_prism_inactive_pool_blocks {ledger_metrics.get('inactive_blocks', 0)}", - "# HELP qbit_prism_reversed_pool_blocks PRISM pool block rows terminally reversed.", - "# TYPE qbit_prism_reversed_pool_blocks gauge", - f"qbit_prism_reversed_pool_blocks {ledger_metrics.get('reversed_blocks', 0)}", - "# HELP qbit_prism_rejected_pool_blocks PRISM pool block rows rejected before confirmation.", - "# TYPE qbit_prism_rejected_pool_blocks gauge", - f"qbit_prism_rejected_pool_blocks {ledger_metrics.get('rejected_blocks', 0)}", - "# HELP qbit_prism_owed_accounts Current accounts with positive carried owed balances.", - "# TYPE qbit_prism_owed_accounts gauge", - f"qbit_prism_owed_accounts {ledger_metrics['owed_accounts']}", - "# HELP qbit_prism_coinbase_weight_headroom_bytes Remaining qbit block weight bytes after the latest pool coinbase.", - "# TYPE qbit_prism_coinbase_weight_headroom_bytes gauge", - f"qbit_prism_coinbase_weight_headroom_bytes {coinbase_weight_headroom}", - "# HELP qbit_prism_ctv_fanouts_pending Pending non-terminal CTV fanouts known to the ledger, or -1 if unavailable.", - "# TYPE qbit_prism_ctv_fanouts_pending gauge", - f"qbit_prism_ctv_fanouts_pending {ctv_pending}", - "# HELP qbit_prism_ctv_fanouts_broadcastable CTV fanouts that are mature enough to broadcast, or -1 if unavailable.", - "# TYPE qbit_prism_ctv_fanouts_broadcastable gauge", - f"qbit_prism_ctv_fanouts_broadcastable {ctv_broadcastable}", - "# HELP qbit_prism_ctv_fanouts_failed CTV fanouts with failed or rejected broadcast state, or -1 if unavailable.", - "# TYPE qbit_prism_ctv_fanouts_failed gauge", - f"qbit_prism_ctv_fanouts_failed {ctv_failed}", - "# HELP qbit_prism_vardiff_enabled Whether PRISM Stratum vardiff is enabled.", - "# TYPE qbit_prism_vardiff_enabled gauge", - f"qbit_prism_vardiff_enabled {1 if self.port.vardiff_config.enabled else 0}", - "# HELP qbit_prism_qbitd_initial_block_download qbitd initialblockdownload status, or -1 if unavailable.", - "# TYPE qbit_prism_qbitd_initial_block_download gauge", - f"qbit_prism_qbitd_initial_block_download {ibd}", - "# HELP qbit_prism_qbitd_peers qbitd peer count, or -1 if unavailable.", - "# TYPE qbit_prism_qbitd_peers gauge", - f"qbit_prism_qbitd_peers {peers}", - "# HELP qbit_prism_audit_artifact_bytes Bytes used by PRISM audit artifacts in PRISM_AUDIT_DIR by artifact kind.", - "# TYPE qbit_prism_audit_artifact_bytes gauge", - *[ - f'qbit_prism_audit_artifact_bytes{{kind="{kind}"}} {audit_metrics[kind]["bytes"]}' - for kind in ("body", "share_segment", "live_bundle", "candidate", "other") - ], - "# HELP qbit_prism_audit_artifact_files PRISM audit artifact file count in PRISM_AUDIT_DIR by artifact kind.", - "# TYPE qbit_prism_audit_artifact_files gauge", - *[ - f'qbit_prism_audit_artifact_files{{kind="{kind}"}} {audit_metrics[kind]["files"]}' - for kind in ("body", "share_segment", "live_bundle", "candidate", "other") - ], - "# HELP qbit_prism_audit_artifact_scan_error Whether the latest PRISM_AUDIT_DIR metric scan failed.", - "# TYPE qbit_prism_audit_artifact_scan_error gauge", - f"qbit_prism_audit_artifact_scan_error {audit_metrics['scan_error']}", - ] - lines.extend(self.lease_heartbeat_metrics_lines()) - lines.extend(self.shutdown_metrics_lines()) - lines.extend(self.coordinator_lock_metrics_lines()) - lines.extend(self.block_submitter_metrics_lines()) - lines.extend(self.block_candidate_cleanup_backlog_metrics_lines()) - lines.extend(self.share_ack_metrics_lines()) - lines.extend(self.port.ctv_fanout_broadcaster_metrics_lines()) - lines.extend(self.port.vardiff_idle_metrics_lines()) - lines.extend(self.vardiff_convergence_metrics_lines()) - lines.extend(self.port.block_finalization_metrics_lines()) - lines.extend(self.job_build_metrics_lines()) - lines.extend(self.port.tip_refresh_metrics_lines()) - lines.extend(self.port.payout_state_metrics_lines()) - lines.extend(self.initial_delivery_metrics_lines()) - lines.extend(self.port.progress_health_metrics_lines()) - lines.extend(self.accepted_preview_attribution_metrics_lines()) - lines.extend(self.gc_pause_metrics_lines()) - lines.extend(self.process_heap_metrics_lines()) - lines.extend(self.component_cardinality_metrics_lines()) - return "\n".join(lines) + "\n" - - def gc_pause_metrics_lines(self) -> list[str]: - """Issue #227: cyclic-collector pause durations, per generation. - - Three families, each labelled only by the closed - PRISM_GC_GENERATIONS set. They sit immediately before the issue - #226 heap block whose ``qbit_prism_process_gc_*`` families they - complement: those export ``gc.get_stats()`` -- how many passes ran - and what they freed -- and cannot say how long any pass held the - interpreter; these are the durations, from ``gc.callbacks``. The - recorder is armed with the lease heartbeat (it is the stall - attribution's instrument), so every series is zero on a - coordinator that never armed the heartbeat. - """ - pauses = self.port._ensure_lease_heartbeat_service().snapshot()["gc_pauses"] - assert isinstance(pauses, dict) - lines = [ - "# HELP qbit_prism_process_gc_pause_seconds Cyclic collector pause duration per generation, from gc.callbacks; the qbit_prism_process_gc_* count families say how many passes ran, this says how long each held the interpreter.", - "# TYPE qbit_prism_process_gc_pause_seconds histogram", - ] - for generation in PRISM_GC_GENERATIONS: - entry = pauses[generation] - buckets = entry["buckets"] - lines.extend( - f'qbit_prism_process_gc_pause_seconds_bucket{{generation="{generation}",le="{bucket:g}"}} {int(buckets.get(bucket, 0))}' - for bucket in PRISM_GC_PAUSE_SECONDS_BUCKETS - ) - lines.append( - f'qbit_prism_process_gc_pause_seconds_bucket{{generation="{generation}",le="+Inf"}} {int(entry["count"])}' - ) - lines.append( - f'qbit_prism_process_gc_pause_seconds_sum{{generation="{generation}"}} {float(entry["sum"]):.6f}' - ) - lines.append( - f'qbit_prism_process_gc_pause_seconds_count{{generation="{generation}"}} {int(entry["count"])}' - ) - lines.extend( - [ - "# HELP qbit_prism_process_gc_last_pause_seconds Duration of the most recent cyclic collector pass per generation.", - "# TYPE qbit_prism_process_gc_last_pause_seconds gauge", - *[ - f'qbit_prism_process_gc_last_pause_seconds{{generation="{generation}"}} {float(pauses[generation]["last_seconds"]):.6f}' - for generation in PRISM_GC_GENERATIONS - ], - "# HELP qbit_prism_process_gc_max_pause_seconds Longest cyclic collector pass observed per generation since the lease heartbeat armed.", - "# TYPE qbit_prism_process_gc_max_pause_seconds gauge", - *[ - f'qbit_prism_process_gc_max_pause_seconds{{generation="{generation}"}} {float(pauses[generation]["max_seconds"]):.6f}' - for generation in PRISM_GC_GENERATIONS - ], - ] - ) - return lines - - def share_ack_metrics_lines(self) -> list[str]: - histograms = self.port.share_ack_snapshot() - lines = [ - "# HELP qbit_prism_share_ack_seconds mining.submit line arrival to Stratum response, by outcome.", - "# TYPE qbit_prism_share_ack_seconds histogram", - ] - for result in PRISM_SHARE_ACK_RESULTS: - histogram = histograms[result] - buckets = histogram["buckets"] - lines.extend( - f'qbit_prism_share_ack_seconds_bucket{{result="{result}",le="{bucket:g}"}} {int(buckets.get(bucket, 0))}' - for bucket in PRISM_JOB_BUILD_SECONDS_BUCKETS - ) - lines.append( - f'qbit_prism_share_ack_seconds_bucket{{result="{result}",le="+Inf"}} {histogram["count"]}' - ) - lines.append( - f'qbit_prism_share_ack_seconds_sum{{result="{result}"}} {histogram["sum"]:.6f}' - ) - lines.append( - f'qbit_prism_share_ack_seconds_count{{result="{result}"}} {histogram["count"]}' - ) - return lines - - def vardiff_convergence_metrics_lines(self) -> list[str]: - """Vardiff convergence and reconnect-resume observability. - - The lane label set is deterministic: configured listener profiles in - order, then any additional lane names observed in the counters, - sorted. The per-second gauge divides a process-local count by this - process's uptime, so numerator and denominator share a domain -- but - that numerator is narrower than "accepted". VardiffService.note_accepted - runs only where the share writer sees a newly inserted ledger row, and - the startup recovery-journal replay reaches the ledger without passing - through it at all, so an exact replay and a recovered share are both - acked accepted while never being lane-counted. The lanes therefore sum - to this coordinator's own commit rate, not to its accepted-ack count. - """ - snapshot = self.port.vardiff_convergence_snapshot() - lane_accepted = snapshot["lane_accepted_shares"] - assert isinstance(lane_accepted, dict) - resume_outcomes = snapshot["resume_outcomes"] - assert isinstance(resume_outcomes, dict) - lanes = [ - str(profile.name) - for profile in getattr(self.port, "listener_profiles", ()) or () - ] - lanes.extend(sorted(lane for lane in lane_accepted if lane not in lanes)) - elapsed = max(0.001, time.monotonic() - self.port.started_monotonic) - return [ - "# HELP qbit_prism_vardiff_sessions_at_max_difficulty Sessions whose current difficulty sits at their effective vardiff ceiling.", - "# TYPE qbit_prism_vardiff_sessions_at_max_difficulty gauge", - f"qbit_prism_vardiff_sessions_at_max_difficulty {int(snapshot['sessions_at_max_difficulty'])}", - "# HELP qbit_prism_vardiff_lane_accepted_shares_total Accepted shares by Stratum lane.", - "# TYPE qbit_prism_vardiff_lane_accepted_shares_total counter", - *[ - f'qbit_prism_vardiff_lane_accepted_shares_total{{lane="{self.port.prometheus_label_value(lane)}"}} {int(lane_accepted.get(lane, 0))}' - for lane in lanes - ], - "# HELP qbit_prism_vardiff_lane_accepted_shares_per_second Accepted shares per second by Stratum lane, averaged since coordinator start; a long-run average cannot show a transient reconnect storm -- use rate(qbit_prism_vardiff_lane_accepted_shares_total[5m]) for that. The numerator counts shares this process newly committed on the live submission path, not the ledger lifetime total published by qbit_prism_accepted_shares_total: an exact replay of an already-durable share and a startup recovery-journal replay are both acked accepted without being lane-counted, so the lanes sum to this coordinator's own commit rate and need not reconcile against an accepted-ack count.", - "# TYPE qbit_prism_vardiff_lane_accepted_shares_per_second gauge", - *[ - f'qbit_prism_vardiff_lane_accepted_shares_per_second{{lane="{self.port.prometheus_label_value(lane)}"}} {int(lane_accepted.get(lane, 0)) / elapsed:.12g}' - for lane in lanes - ], - "# HELP qbit_prism_vardiff_resume_total Reconnect difficulty resume attempts by outcome; clamped means the retained value was pulled into the lane's plausibility bounds, overridden means an adopted value was superseded by an explicit difficulty request in the same authorize, so resumed + clamped - overridden is the number that stuck.", - "# TYPE qbit_prism_vardiff_resume_total counter", - *[ - f'qbit_prism_vardiff_resume_total{{outcome="{outcome}"}} {int(resume_outcomes.get(outcome, 0))}' - for outcome in PRISM_VARDIFF_RESUME_OUTCOMES - ], - "# HELP qbit_prism_vardiff_resume_retained_sessions Worker sessions holding a retained difficulty eligible for resume.", - "# TYPE qbit_prism_vardiff_resume_retained_sessions gauge", - f"qbit_prism_vardiff_resume_retained_sessions {int(snapshot['retained_sessions'])}", - ] - - def coordinator_lock_metrics_lines(self) -> list[str]: - acquisition_count, contention_count, wait_sum, wait_max = ( - self.port.coordinator_lock_contention_snapshot() - ) - return [ - "# HELP qbit_prism_coordinator_lock_acquisitions_total Coordinator control-plane lock acquisitions granted, contended or not; divide qbit_prism_coordinator_lock_contentions_total by this for the contended fraction. A timed-out wait is counted as contention without an acquisition, so that fraction is an upper bound where a caller passes a timeout.", - "# TYPE qbit_prism_coordinator_lock_acquisitions_total counter", - f"qbit_prism_coordinator_lock_acquisitions_total {int(acquisition_count)}", - "# HELP qbit_prism_coordinator_lock_contentions_total Coordinator control-plane lock acquisitions that had to wait.", - "# TYPE qbit_prism_coordinator_lock_contentions_total counter", - f"qbit_prism_coordinator_lock_contentions_total {int(contention_count)}", - "# HELP qbit_prism_coordinator_lock_wait_seconds Coordinator control-plane lock wait duration for contended acquisitions.", - "# TYPE qbit_prism_coordinator_lock_wait_seconds summary", - f"qbit_prism_coordinator_lock_wait_seconds_sum {float(wait_sum):.6f}", - f"qbit_prism_coordinator_lock_wait_seconds_count {int(contention_count)}", - "# HELP qbit_prism_coordinator_lock_wait_seconds_max Longest observed coordinator control-plane lock wait.", - "# TYPE qbit_prism_coordinator_lock_wait_seconds_max gauge", - f"qbit_prism_coordinator_lock_wait_seconds_max {float(wait_max):.6f}", - ] - - @staticmethod - def _accepted_block_preview_publication_lines( - histograms: dict[str, Any], - ) -> list[str]: - """Per-result bucket/sum/count lines for the publication histogram.""" - lines: list[str] = [] - for result in PRISM_ACCEPTED_BLOCK_PREVIEW_PUBLICATION_RESULTS: - histogram = histograms[result] - buckets = histogram["buckets"] - count = int(histogram["count"]) - lines.extend( - f"qbit_prism_accepted_block_preview_publication_seconds_bucket" - f'{{result="{result}",le="{bucket:g}"}} ' - f"{int(buckets.get(bucket, 0))}" - for bucket in ( - PRISM_ACCEPTED_BLOCK_PREVIEW_PUBLICATION_SECONDS_BUCKETS - ) - ) - lines.append( - f"qbit_prism_accepted_block_preview_publication_seconds_bucket" - f'{{result="{result}",le="+Inf"}} {count}' - ) - lines.append( - f"qbit_prism_accepted_block_preview_publication_seconds_sum" - f'{{result="{result}"}} {float(histogram["sum"]):.6f}' - ) - lines.append( - f"qbit_prism_accepted_block_preview_publication_seconds_count" - f'{{result="{result}"}} {count}' - ) - return lines - - def block_submitter_metrics_lines(self) -> list[str]: - snapshot = self.port.block_submitter_snapshot() - submit_buckets = snapshot["submit_seconds_buckets"] - assert isinstance(submit_buckets, dict) - submit_sum = float(snapshot["submit_seconds_sum"]) - submit_count = int(snapshot["submit_seconds_count"]) - backoff_active = bool(snapshot["backoff_active"]) - backoff_remaining = float(snapshot["backoff_remaining_seconds"]) - backoff_delay = float(snapshot["backoff_delay_seconds"]) - # The B1 owner copies these under the coordinator lock; the key set - # is fixed by PRISM_BLOCK_CANDIDATE_COLLAPSE_OUTCOMES so the series - # cardinality cannot grow with the candidate population. - collapse_counts = self.port.block_candidate_collapse_snapshot() - # Issue #181 item 3: the label set is closed by - # PRISM_ACCEPTED_BLOCK_PREVIEW_PUBLICATION_RESULTS, so this series - # carries no block hash or height and cannot grow with the candidate - # population. - preview_publication = snapshot["accepted_preview_publication"] - assert isinstance(preview_publication, dict) - return [ - "# HELP qbit_prism_block_submit_seconds Seconds from a block candidate landing in this process to its submitblock RPC returning.", - "# TYPE qbit_prism_block_submit_seconds histogram", - *[ - f'qbit_prism_block_submit_seconds_bucket{{le="{bucket:g}"}} {int(submit_buckets.get(bucket, 0))}' - for bucket in PRISM_JOB_BUILD_SECONDS_BUCKETS - ], - f'qbit_prism_block_submit_seconds_bucket{{le="+Inf"}} {submit_count}', - f"qbit_prism_block_submit_seconds_sum {submit_sum:.6f}", - f"qbit_prism_block_submit_seconds_count {submit_count}", - "# HELP qbit_prism_accepted_block_preview_publication_seconds Seconds from definitive qbitd acceptance of a block candidate to its payout preview becoming visible to waiting child work, by publication result.", - "# TYPE qbit_prism_accepted_block_preview_publication_seconds histogram", - *self._accepted_block_preview_publication_lines(preview_publication), - "# HELP qbit_prism_block_candidates_pending Durable block candidates awaiting a terminal outcome, or -1 if unavailable.", - "# TYPE qbit_prism_block_candidates_pending gauge", - f"qbit_prism_block_candidates_pending {int(snapshot['pending_count'])}", - "# HELP qbit_prism_block_candidate_oldest_pending_seconds Age of the oldest durable pending block candidate, or -1 if unavailable.", - "# TYPE qbit_prism_block_candidate_oldest_pending_seconds gauge", - f"qbit_prism_block_candidate_oldest_pending_seconds {float(snapshot['oldest_pending_age_seconds']):.6f}", - "# HELP qbit_prism_block_candidate_oldest_unattempted_seconds Age of the oldest durable candidate that has never entered processing, or -1 if unavailable.", - "# TYPE qbit_prism_block_candidate_oldest_unattempted_seconds gauge", - f"qbit_prism_block_candidate_oldest_unattempted_seconds {float(snapshot['oldest_unattempted_age_seconds']):.6f}", - "# HELP qbit_prism_block_submitter_retry_backoff_active Whether the submitter is in an intentional interruptible retry wait.", - "# TYPE qbit_prism_block_submitter_retry_backoff_active gauge", - f"qbit_prism_block_submitter_retry_backoff_active {1 if backoff_active else 0}", - "# HELP qbit_prism_block_submitter_retry_backoff_remaining_seconds Remaining intentional submitter retry wait.", - "# TYPE qbit_prism_block_submitter_retry_backoff_remaining_seconds gauge", - f"qbit_prism_block_submitter_retry_backoff_remaining_seconds {backoff_remaining:.6f}", - "# HELP qbit_prism_block_submitter_retry_backoff_seconds Current intentional submitter retry delay.", - "# TYPE qbit_prism_block_submitter_retry_backoff_seconds gauge", - f"qbit_prism_block_submitter_retry_backoff_seconds {backoff_delay:.6f}", - "# HELP qbit_prism_block_candidate_collapse_total Durable pending block candidates handled by the decided-height collapse, by bounded outcome. The outcome label set is closed: no block hash, parent hash, or job ID is ever a label.", - "# TYPE qbit_prism_block_candidate_collapse_total counter", - *[ - f'qbit_prism_block_candidate_collapse_total{{outcome="{outcome}"}} {int(collapse_counts.get(outcome, 0))}' - for outcome in PRISM_BLOCK_CANDIDATE_COLLAPSE_OUTCOMES - ], - ] - - def block_candidate_cleanup_backlog_metrics_lines(self) -> list[str]: - """Issue #198: the collapse cleanup-retry backlog and its bound. - - Seven single-series families, none labelled: the B1 owner copies - the values under the coordinator lock and the key set of its - snapshot is closed, so the export cannot grow with the candidate - population or carry a hash. The bound is exported beside the depth - so an alert can be a ratio against the configured contract instead - of a deployment-specific literal (the same shape as the - accepted-parent depth cap). - """ - snapshot = ( - self.port._ensure_block_candidate_service() - .collapsed_candidate_cleanup_backlog_snapshot() - ) - return [ - "# HELP qbit_prism_block_candidate_cleanup_retry_backlog Durably terminal collapsed block candidates whose in-memory cleanup is still owed and retried by the accounting lane.", - "# TYPE qbit_prism_block_candidate_cleanup_retry_backlog gauge", - f"qbit_prism_block_candidate_cleanup_retry_backlog {int(snapshot['depth'])}", - "# HELP qbit_prism_block_candidate_cleanup_retry_backlog_max Configured cleanup-retry backlog depth at which the decided-height collapse stops admitting rows to bulk terminalization.", - "# TYPE qbit_prism_block_candidate_cleanup_retry_backlog_max gauge", - f"qbit_prism_block_candidate_cleanup_retry_backlog_max {int(snapshot['backlog_max'])}", - "# HELP qbit_prism_block_candidate_cleanup_retry_oldest_seconds Age of the oldest owed collapse cleanup since it was first deferred, or -1 when none is owed.", - "# TYPE qbit_prism_block_candidate_cleanup_retry_oldest_seconds gauge", - f"qbit_prism_block_candidate_cleanup_retry_oldest_seconds {float(snapshot['oldest_age_seconds']):.6f}", - "# HELP qbit_prism_block_candidate_cleanup_retry_pending_share_holders Pending-share floor holders retained by owed collapse cleanups, by exact object identity.", - "# TYPE qbit_prism_block_candidate_cleanup_retry_pending_share_holders gauge", - f"qbit_prism_block_candidate_cleanup_retry_pending_share_holders {int(snapshot['pending_share_holders'])}", - "# HELP qbit_prism_block_candidate_cleanup_retry_terminal_outcome_pins Terminal-outcome fences the cleanup-retry backlog pins against eviction.", - "# TYPE qbit_prism_block_candidate_cleanup_retry_terminal_outcome_pins gauge", - f"qbit_prism_block_candidate_cleanup_retry_terminal_outcome_pins {int(snapshot['terminal_outcome_pins'])}", - "# HELP qbit_prism_block_candidate_cleanup_backpressure_active Whether the cleanup-retry backlog is at its bound and bulk terminalization is refusing new rows.", - "# TYPE qbit_prism_block_candidate_cleanup_backpressure_active gauge", - f"qbit_prism_block_candidate_cleanup_backpressure_active {1 if snapshot['backpressure_active'] else 0}", - "# HELP qbit_prism_block_candidate_cleanup_backpressure_total Occasions on which the cleanup-retry backlog bound preserved at least one row from bulk terminalization.", - "# TYPE qbit_prism_block_candidate_cleanup_backpressure_total counter", - f"qbit_prism_block_candidate_cleanup_backpressure_total {int(snapshot['backpressure_engagements'])}", - ] - - def process_heap_metrics_lines(self) -> list[str]: - """Issue #226: always-on interpreter and glibc allocator readings. - - Eleven families. Only the three GC families carry a label, and its - value set is the closed PRISM_GC_GENERATIONS; nothing here can grow - with traffic. The collector is a snapshot read -- allocation and - GC counters, the thread count, and one mallinfo2 call -- and never - walks the heap. Where mallinfo2 is unavailable (musl, macOS, glibc - before 2.33, or PRISM_MALLOC_TELEMETRY=0) the availability gauge - renders 0 and the four byte gauges render -1, never 0. - """ - sample = self._process_telemetry.sample() - return [ - "# HELP qbit_prism_process_allocated_blocks Memory blocks currently allocated by the CPython allocator (sys.getallocatedblocks); a live-object proxy that rises with retention and stays flat under pure allocator fragmentation.", - "# TYPE qbit_prism_process_allocated_blocks gauge", - f"qbit_prism_process_allocated_blocks {int(sample.allocated_blocks)}", - "# HELP qbit_prism_process_gc_trigger_count gc.get_count(): the cyclic collector's per-generation trigger counters, compared against gc.get_threshold() to decide when to collect. Generation 0 is allocations minus deallocations, NOT a count of retained objects. CPython 3.13 made the collector incremental, so the third entry is unused and reads 0; the series is kept for a fixed family set.", - "# TYPE qbit_prism_process_gc_trigger_count gauge", - *[ - f'qbit_prism_process_gc_trigger_count{{generation="{generation}"}} {int(count)}' - for generation, count in zip(PRISM_GC_GENERATIONS, sample.gc_trigger_count) - ], - "# HELP qbit_prism_process_gc_collections_total Cyclic collector passes completed per generation since process start (gc.get_stats).", - "# TYPE qbit_prism_process_gc_collections_total counter", - *[ - f'qbit_prism_process_gc_collections_total{{generation="{generation}"}} {int(count)}' - for generation, count in zip(PRISM_GC_GENERATIONS, sample.gc_collections) - ], - "# HELP qbit_prism_process_gc_collected_objects_total Objects the cyclic collector freed per generation since process start.", - "# TYPE qbit_prism_process_gc_collected_objects_total counter", - *[ - f'qbit_prism_process_gc_collected_objects_total{{generation="{generation}"}} {int(count)}' - for generation, count in zip(PRISM_GC_GENERATIONS, sample.gc_collected) - ], - "# HELP qbit_prism_process_gc_uncollectable_objects_total Objects the cyclic collector found unreachable but could not free per generation since process start; growth is a leak the collector already knows about.", - "# TYPE qbit_prism_process_gc_uncollectable_objects_total counter", - *[ - f'qbit_prism_process_gc_uncollectable_objects_total{{generation="{generation}"}} {int(count)}' - for generation, count in zip(PRISM_GC_GENERATIONS, sample.gc_uncollectable) - ], - "# HELP qbit_prism_process_threads Live Python threads (threading.active_count); each one is a candidate glibc malloc arena.", - "# TYPE qbit_prism_process_threads gauge", - f"qbit_prism_process_threads {int(sample.threads)}", - "# HELP qbit_prism_process_malloc_info_available Whether glibc mallinfo2 is bound and PRISM_MALLOC_TELEMETRY is on (1), or the allocator byte gauges are unavailable (0): the symbol is glibc 2.33+ only and absent on musl and macOS. While 0 the byte gauges render -1, never 0.", - "# TYPE qbit_prism_process_malloc_info_available gauge", - f"qbit_prism_process_malloc_info_available {1 if sample.malloc_info_available else 0}", - "# HELP qbit_prism_process_malloc_arena_bytes Heap space glibc malloc holds for its arenas across every thread arena, excluding mmapped chunks (mallinfo2.arena), or -1 when unavailable.", - "# TYPE qbit_prism_process_malloc_arena_bytes gauge", - f"qbit_prism_process_malloc_arena_bytes {int(sample.malloc_arena_bytes)}", - "# HELP qbit_prism_process_malloc_in_use_bytes Bytes in allocated chunks across every glibc malloc arena (mallinfo2.uordblks), or -1 when unavailable; arena minus this is the fragmentation glibc has not returned to the kernel.", - "# TYPE qbit_prism_process_malloc_in_use_bytes gauge", - f"qbit_prism_process_malloc_in_use_bytes {int(sample.malloc_in_use_bytes)}", - "# HELP qbit_prism_process_malloc_free_bytes Bytes in free chunks glibc malloc retains across every arena (mallinfo2.fordblks), or -1 when unavailable.", - "# TYPE qbit_prism_process_malloc_free_bytes gauge", - f"qbit_prism_process_malloc_free_bytes {int(sample.malloc_free_bytes)}", - "# HELP qbit_prism_process_malloc_mmapped_bytes Bytes in chunks glibc malloc served directly through mmap (mallinfo2.hblkhd), or -1 when unavailable.", - "# TYPE qbit_prism_process_malloc_mmapped_bytes gauge", - f"qbit_prism_process_malloc_mmapped_bytes {int(sample.malloc_mmapped_bytes)}", - ] - - def component_cardinality_metrics_lines(self) -> list[str]: - """Issue #226 (and #185's gauge list): retained entries per component. - - Two families, each a closed ``component`` label set: entry counts and - retained bytes over the state that plausibly scales with the payout - window, the job population, or a candidate storm. The reads are - deliberately direct attribute loads on the owning services rather - than tolerant ``getattr`` defaults: a renamed owner field must fail - the shipped-owner test loudly, never report a silent zero. The owners - are the ones the renderer already constructs through - ``_ensure_job_cache_state``; the reorg reconciler is read only when - it exists, because the coordinator builds it lazily and it is the - sole holder of its flights, so an absent service is a true zero. - - Every read is one ``len()``, ``qsize()``, or attribute load, atomic - under the GIL, so no owner lock is taken beyond the coordinator lock - the renderer already holds for the graveyard: a gauge that - tolerates one entry of skew must never queue the scrape behind a - daemon round trip or a payout-window walk. - - The one exception is the page-backed payout window, where records and - canonical bytes are summed over the window's pages: O(pages), about - 120 iterations of two ``len()`` calls for a 63k-share window at the - default page size, not O(shares). A mirror-backed window costs neither - sum. This is a bounded per-scrape cost, not a heap walk. - """ - port = self.port - port._ensure_job_cache_state() - bundles = port._ensure_job_bundle_service() - payout = port._ensure_payout_state_service() - writer = port._ensure_share_writer_service() - compiler = port._ensure_bundle_compiler() - candidates = port._ensure_block_candidate_service() - candidates._ensure_block_replay_state() - candidates._ensure_block_candidate_disposition_state() - reconciler = getattr(port, "__dict__", {}).get("_reorg_reconciler_service") - serialization = bundles._share_window_serialization - cached_window = payout._incremental_payout_artifact_window - artifact = payout._payout_ledger_artifact - daemon_client = compiler._serve_builder - window = cached_window.window if cached_window is not None else None - window_pages = window_records = window_canonical_bytes = 0 - mirror_records = mirror_bytes = 0 - # Exactly one component accounts for the cached window, because these - # gauges exist to attribute resident bytes and an operator summing the - # family must not count the same object twice. The two backings are - # mutually exclusive -- the mirror *is* the cached window on the Rust - # path -- so a mirror-backed window reports zero pages, records, and - # canonical bytes under payout_window_*, and its size appears only - # under daemon_window_mirror_*. Which pair is non-zero is also how an - # operator reads the backing off /metrics. - if isinstance(window, DaemonShareWindowMirror): - mirror_records = int(window.record_count) - mirror_bytes = len(window.canonical_items) - elif window is not None: - pages = window.pages - window_pages = len(pages) - window_records = sum(len(page.records) for page in pages) - window_canonical_bytes = sum( - len(page.canonical_json_items) for page in pages - ) - with port.lock: - port._ensure_evicted_job_state() - graveyard_entries = len(port.evicted_job_graveyard) - same_tip_entries = len(port.evicted_same_tip_job_ids) - outstanding_hashes = len(candidates._outstanding_block_candidate_hashes) - tip_observed_hashes = len(candidates._tip_observed_accepted_block_hashes) - counted_abandonments = len(candidates._counted_block_candidate_abandonments) - ancestor_redrives = len(candidates._ancestor_redrive_records) - terminal_outcomes = len(candidates._block_candidate_terminal_outcomes) - waiting_retries = len(candidates._block_disposition_waiting_retries) - dequeued_hashes = len(candidates._block_candidate_dequeued_hashes) - accounted_hashes = len(port._accounted_accepted_block_hashes) - trusted_memo = ( - len(reconciler._reorg_reconcile_trusted_memo) - if reconciler is not None - else 0 - ) - entries = { - "payout_window_pages": window_pages, - "payout_window_records": window_records, - "payout_ledger_artifact_shares": ( - len(artifact.shares_json) if artifact is not None else 0 - ), - "accepted_block_payout_previews": len(payout._accepted_block_payout_previews), - "invalidated_accepted_block_payout_previews": len( - payout._invalidated_accepted_block_payout_previews - ), - "payout_append_invalidation_stamps": len( - payout._payout_append_invalidation_stamps - ), - "payout_window_inflight_scan_anchors": len( - payout._payout_window_inflight_scan_anchors - ), - "payout_unfenced_append_inflight_stamps": len( - payout._payout_unfenced_append_inflight_stamps - ), - "share_window_serialization_shares": ( - int(serialization.share_count) if serialization is not None else 0 - ), - "job_contexts": len(port.jobs), - "job_bundle_cache": len(bundles._job_bundle_cache), - "job_build_issued_stamps": len(bundles._job_build_issued_at_ms), - "bundle_preparation_flights": len(bundles._bundle_preparation_flights), - "active_job_bundle_builds": len(bundles._active_job_bundle_builds), - "evicted_job_graveyard": graveyard_entries, - "evicted_same_tip_job_ids": same_tip_entries, - "block_candidate_queue": candidates.candidate_queue.qsize(), - "block_replay_queue": candidates._block_replay_candidate_queue.qsize(), - "block_replay_inflight_hashes": len(candidates._block_replay_inflight_hashes), - "block_quarantine_queue": candidates._block_quarantine_queue.qsize(), - "block_quarantine_hashes": len(candidates._block_quarantine_hashes), - "outstanding_block_candidate_hashes": outstanding_hashes, - "tip_observed_accepted_block_hashes": tip_observed_hashes, - "counted_block_candidate_abandonments": counted_abandonments, - "accepted_block_preview_stamps": len( - candidates._accepted_block_preview_acceptance_monotonic - ), - "ancestor_redrive_records": ancestor_redrives, - "block_candidate_terminal_outcomes": terminal_outcomes, - "block_candidate_disposition_flights": len( - candidates._block_candidate_disposition_flights - ), - "block_disposition_waiting_retries": waiting_retries, - "block_candidate_dequeued_hashes": dequeued_hashes, - "accounted_accepted_block_hashes": accounted_hashes, - "reconcile_flights": ( - len(reconciler._reconcile_flights) if reconciler is not None else 0 - ), - "reconcile_trusted_memo": trusted_memo, - "pending_share_commit_floor": len(writer._pending_share_commit_floor), - "daemon_window_mirror_records": mirror_records, - "daemon_uploaded_windows": ( - len(daemon_client.uploaded_windows) if daemon_client is not None else 0 - ), - } - byte_sizes = { - "payout_window_canonical_json": window_canonical_bytes, - "daemon_window_mirror_canonical_items": mirror_bytes, - "share_window_serialization_spool": ( - int(serialization._spool_size) if serialization is not None else 0 - ), - # json.dumps with its ASCII default, so characters are bytes. - "share_window_serialization_compact_json": ( - int(serialization.compact_json_bytes) - if serialization is not None - else 0 - ), - } - return [ - "# HELP qbit_prism_component_entries Entries retained by each in-process coordinator component; the component label set is closed and carries no job id, tip hash, generation, or worker name.", - "# TYPE qbit_prism_component_entries gauge", - *[ - f'qbit_prism_component_entries{{component="{kind}"}} {int(entries[kind])}' - for kind in PRISM_COMPONENT_ENTRY_KINDS - ], - "# HELP qbit_prism_component_bytes Bytes retained by each byte-sized coordinator payload (the cached payout window's canonical JSON, the daemon window mirror, and the share-window serialization compact strings); the component label set is closed. All are resident bytes except share_window_serialization_spool, which is an on-disk temporary file and is not part of RSS.", - "# TYPE qbit_prism_component_bytes gauge", - *[ - f'qbit_prism_component_bytes{{component="{kind}"}} {int(byte_sizes[kind])}' - for kind in PRISM_COMPONENT_BYTE_KINDS - ], - ] - - def landing_observability_metrics_lines(self) -> list[str]: - """Landing-path metrics for issue #188's pre-deadline alerting. - - The prior-balances read crossed the one-second submitter deadline - silently over several weeks; these series make that growth, landing - timeouts, and unresolved accepted-parent age visible before they - become an outage. - """ - lines: list[str] = [ - "# HELP qbit_prism_block_ledger_calls_total Submitter ledger calls by deadline class.", - "# TYPE qbit_prism_block_ledger_calls_total counter", - "# HELP qbit_prism_block_ledger_call_timeouts_total Submitter ledger deadline expiries by deadline class.", - "# TYPE qbit_prism_block_ledger_call_timeouts_total counter", - "# HELP qbit_prism_block_ledger_call_budget_seconds Most recent statement budget applied per deadline class (escalates for retried landings).", - "# TYPE qbit_prism_block_ledger_call_budget_seconds gauge", - "# HELP qbit_prism_block_ledger_call_last_duration_seconds Duration of the most recent call per deadline class.", - "# TYPE qbit_prism_block_ledger_call_last_duration_seconds gauge", - "# HELP qbit_prism_block_ledger_call_max_duration_seconds Longest observed call per deadline class since process start.", - "# TYPE qbit_prism_block_ledger_call_max_duration_seconds gauge", - ] - for call_class, stats in sorted( - self.port.block_ledger_call_class_metrics().items() - ): - label = self.port.prometheus_label_value(call_class) - lines.extend( - [ - f'qbit_prism_block_ledger_calls_total{{call_class="{label}"}} {int(stats["calls_total"])}', - f'qbit_prism_block_ledger_call_timeouts_total{{call_class="{label}"}} {int(stats["timeouts_total"])}', - f'qbit_prism_block_ledger_call_budget_seconds{{call_class="{label}"}} {float(stats["last_budget_seconds"]):.6f}', - f'qbit_prism_block_ledger_call_last_duration_seconds{{call_class="{label}"}} {float(stats["last_duration_seconds"]):.6f}', - f'qbit_prism_block_ledger_call_max_duration_seconds{{call_class="{label}"}} {float(stats["max_duration_seconds"]):.6f}', - ] - ) - # The depth gauge must equal what the admission fence counts. The - # fence counts every landed transition; the age list can only report - # the ones carrying a monotonic landing stamp. A degraded preview - # publication and a replayed durable candidate both arm ``landed`` - # without that stamp, so deriving depth from the ages would read zero - # while the fence already refuses admission at depth one. - unresolved_depth = self.port._accepted_parent_unresolved_depth() - unresolved_depth_cap = self.port._accepted_parent_unresolved_depth_cap() - unresolved_ages = self.port.accepted_parent_unresolved_ages_seconds() - oldest_unresolved = max(unresolved_ages) if unresolved_ages else -1.0 - with self.port.lock: - preview_wait_timeouts = int( - getattr(self.port, "_accepted_parent_preview_wait_timeouts", 0) - ) - lines.extend( - [ - "# HELP qbit_prism_accepted_parent_unresolved_transitions Landed accepted-block transitions whose durable bookkeeping is unresolved.", - "# TYPE qbit_prism_accepted_parent_unresolved_transitions gauge", - f"qbit_prism_accepted_parent_unresolved_transitions {unresolved_depth}", - "# HELP qbit_prism_accepted_parent_unresolved_depth_max Configured unresolved accepted-parent depth at which job admission fails closed.", - "# TYPE qbit_prism_accepted_parent_unresolved_depth_max gauge", - f"qbit_prism_accepted_parent_unresolved_depth_max {unresolved_depth_cap}", - "# HELP qbit_prism_accepted_parent_unresolved_oldest_seconds Age of the oldest unresolved accepted-parent transition, or -1 when none.", - "# TYPE qbit_prism_accepted_parent_unresolved_oldest_seconds gauge", - f"qbit_prism_accepted_parent_unresolved_oldest_seconds {oldest_unresolved:.6f}", - "# HELP qbit_prism_accepted_parent_preview_wait_timeouts_total Child job builds that timed out waiting for an accepted-parent payout preview.", - "# TYPE qbit_prism_accepted_parent_preview_wait_timeouts_total counter", - f"qbit_prism_accepted_parent_preview_wait_timeouts_total {preview_wait_timeouts}", - "# HELP qbit_prism_accepted_parent_redrive_attempts_total In-process ancestor re-drives armed after repeated finalization deferrals on one pending accepted-parent transition.", - "# TYPE qbit_prism_accepted_parent_redrive_attempts_total counter", - "qbit_prism_accepted_parent_redrive_attempts_total " - f"{int(getattr(self.port, 'accepted_parent_redrive_attempt_count', 0))}", - "# HELP qbit_prism_accepted_parent_redrive_resolved_total Pending accepted-parent transitions that resolved after at least one in-process re-drive attempt.", - "# TYPE qbit_prism_accepted_parent_redrive_resolved_total counter", - "qbit_prism_accepted_parent_redrive_resolved_total " - f"{int(getattr(self.port, 'accepted_parent_redrive_resolved_count', 0))}", - "# HELP qbit_prism_accepted_parent_redrive_exhausted_total Ancestors whose re-drive attempt cap was exhausted with the transition still pending; the publication-progress watchdog remains the backstop.", - "# TYPE qbit_prism_accepted_parent_redrive_exhausted_total counter", - "qbit_prism_accepted_parent_redrive_exhausted_total " - f"{int(getattr(self.port, 'accepted_parent_redrive_exhausted_count', 0))}", - ] - ) - prior_stats_fn = getattr(self.port.ledger, "prior_balances_read_stats", None) - if callable(prior_stats_fn): - prior_stats = prior_stats_fn() - lines.extend( - [ - "# HELP qbit_prism_prior_balances_reads_total Prior-balances reads served by the ledger.", - "# TYPE qbit_prism_prior_balances_reads_total counter", - f"qbit_prism_prior_balances_reads_total {int(prior_stats['reads_total'])}", - "# HELP qbit_prism_prior_balances_read_last_seconds Duration of the most recent prior-balances read.", - "# TYPE qbit_prism_prior_balances_read_last_seconds gauge", - f"qbit_prism_prior_balances_read_last_seconds {float(prior_stats['last_seconds']):.6f}", - "# HELP qbit_prism_prior_balances_read_max_seconds Longest prior-balances read since process start.", - "# TYPE qbit_prism_prior_balances_read_max_seconds gauge", - f"qbit_prism_prior_balances_read_max_seconds {float(prior_stats['max_seconds']):.6f}", - ] - ) - lines.extend(self._ledger_read_gate_metric_lines()) - startup_phases = self.port.startup_phase_seconds() - if startup_phases: - lines.extend( - [ - "# HELP qbit_prism_startup_phase_seconds Seconds from serve() start to each startup phase, recorded once.", - "# TYPE qbit_prism_startup_phase_seconds gauge", - ] - ) - for phase, seconds in sorted(startup_phases.items()): - label = self.port.prometheus_label_value(phase) - lines.append( - f'qbit_prism_startup_phase_seconds{{phase="{label}"}} {float(seconds):.6f}' - ) - return lines - - def _ledger_read_gate_metric_lines(self) -> list[str]: - """Split read-slot admission wait from SQL execution, per operation. - - A single duration per ledger call cannot say whether a budget went to - coordinator-local admission or to PostgreSQL, and issue #211 turned on - exactly that distinction: the replay enumeration reported - ``exceeded 5s`` while its statement deadline was barely touched and - the database showed no blocked backends. These two families answer it - from a scrape -- ``..._gate_wait_seconds_*`` is contention inside this - process, ``..._execute_seconds_*`` is the server (a cancelled - statement's return tail included). The timeout counters are split the - same way, so an admission expiry and a statement expiry never land on - one series. - """ - stats_fn = getattr(self.port.ledger, "ledger_read_gate_stats", None) - if not callable(stats_fn): - return [] - # Issue #224 Wave 0: the operation label is bounded by - # PRISM_LEDGER_READ_OPERATIONS. Names outside the contract fold - # into ``other`` (summed) instead of opening a series; members - # pass through untouched, so the shipped operation renders - # byte-for-byte as before. - stats_by_operation = fold_ledger_read_stats(stats_fn() or {}) - if not stats_by_operation: - return [] - # Samples stay grouped under their own family: the exposition format - # wants every line of a metric emitted as one block, so the loop is - # over families and the operations are the inner dimension. - families: tuple[tuple[str, str, str, str, bool], ...] = ( - ( - "qbit_prism_ledger_read_calls_total", - "counter", - "Ledger read-slot operations completed or failed, by operation.", - "calls_total", - True, - ), - ( - "qbit_prism_ledger_read_gate_wait_seconds_total", - "counter", - "Cumulative coordinator-local read-slot admission wait, by operation.", - "gate_wait_seconds_total", - False, - ), - ( - "qbit_prism_ledger_read_gate_wait_seconds_max", - "gauge", - "Longest coordinator-local read-slot admission wait since process start, by operation.", - "gate_wait_seconds_max", - False, - ), - ( - "qbit_prism_ledger_read_gate_timeouts_total", - "counter", - "Read-slot operations whose deadline expired before admission, so no statement was ever sent.", - "gate_timeouts_total", - True, - ), - ( - "qbit_prism_ledger_read_execute_seconds_total", - "counter", - "Cumulative PostgreSQL execution time for admitted read-slot statements, by operation.", - "execute_seconds_total", - False, - ), - ( - "qbit_prism_ledger_read_execute_seconds_max", - "gauge", - "Longest PostgreSQL execution time for an admitted read-slot statement since process start, by operation.", - "execute_seconds_max", - False, - ), - ( - "qbit_prism_ledger_read_execute_timeouts_total", - "counter", - "Admitted read-slot statements whose deadline expired inside PostgreSQL, cancel lag included.", - "execute_timeouts_total", - True, - ), - ) - operations = [ - (self.port.prometheus_label_value(str(operation)), stats) - for operation, stats in sorted(stats_by_operation.items()) - ] - lines: list[str] = [] - for name, metric_type, help_text, field, integral in families: - lines.append(f"# HELP {name} {help_text}") - lines.append(f"# TYPE {name} {metric_type}") - for label, stats in operations: - value = stats.get(field, 0) - rendered = ( - f"{int(value)}" if integral else f"{float(value):.6f}" - ) - lines.append(f'{name}{{operation="{label}"}} {rendered}') - return lines - - @staticmethod - def _duration_summary_lines( - name: str, - labels: str, - stats: dict[str, Any], - ) -> list[str]: - """``_sum`` / ``_count`` / ``_max`` for one labelled cell.""" - return [ - f'{name}_sum{{{labels}}} {float(stats["sum"]):.6f}', - f'{name}_count{{{labels}}} {int(stats["count"])}', - f'{name}_max{{{labels}}} {float(stats["max"]):.6f}', - ] - - def accepted_preview_attribution_metrics_lines(self) -> list[str]: - """Issue #224 Wave 0: the shared acceptance-to-preview attribution. - - Four summary families, one owner snapshot per render, every label - drawn from a closed vocabulary in - ``lab.prism.accepted_preview_telemetry`` and every cell of every - product rendered from zero. The export is therefore constant-size - regardless of block cadence and carries no hash or height; the - per-publication record that may carry those is the owner's - diagnostics ring, which is deliberately not a metric. - """ - snapshot = ensure_accepted_preview_telemetry(self.port).snapshot() - landing = snapshot["landing_phases"] - passes = snapshot["reconcile_passes"] - steps = snapshot["reconcile_steps"] - rescans = snapshot["full_rescans"] - landing_name = "qbit_prism_accepted_block_landing_phase_seconds" - pass_name = "qbit_prism_reorg_reconcile_pass_seconds" - step_name = "qbit_prism_reorg_reconcile_step_seconds" - rescan_name = "qbit_prism_payout_window_full_rescan_seconds" - lines = [ - f"# HELP {landing_name} Accepted-block landing wall time between definitive node acceptance and payout preview publication, by bounded sub-phase; prior_balances_check is a prior-balances reread, never a payout-window rescan.", - f"# TYPE {landing_name} summary", - ] - for phase in PRISM_ACCEPTED_LANDING_PHASES: - lines.extend( - self._duration_summary_lines( - landing_name, f'phase="{phase}"', landing[phase] - ) - ) - lines.extend( - [ - f"# HELP {pass_name} Reorg reconciliation passes by bounded caller; _count is the pass count, and a pass minus the sum of its steps is the unattributed remainder.", - f"# TYPE {pass_name} summary", - ] - ) - for caller in PRISM_REORG_RECONCILE_CALLERS: - lines.extend( - self._duration_summary_lines( - pass_name, f'caller="{caller}"', passes[caller] - ) - ) - lines.extend( - [ - f"# HELP {step_name} Reorg reconciliation wall time by bounded caller and step; admission_wait is writer admission plus the payout-balance mutation lock, and candidate_prepare is where a reconcile_invalidation full rescan executes.", - f"# TYPE {step_name} summary", - ] - ) - for caller in PRISM_REORG_RECONCILE_CALLERS: - for step in PRISM_REORG_RECONCILE_STEPS: - lines.extend( - self._duration_summary_lines( - step_name, - f'caller="{caller}",step="{step}"', - steps[(caller, step)], - ) - ) - lines.extend( - [ - f"# HELP {rescan_name} Full payout-window oracle rescans by bounded reason and window pipeline path; a prior-balances reread is a ledger read operation, not an observation here.", - f"# TYPE {rescan_name} summary", - ] - ) - for reason in PRISM_PAYOUT_WINDOW_FULL_RESCAN_REASONS: - for path in PRISM_PAYOUT_WINDOW_FULL_RESCAN_PATHS: - lines.extend( - self._duration_summary_lines( - rescan_name, - f'reason="{reason}",path="{path}"', - rescans[(reason, path)], - ) - ) - return lines - - def accepted_stats_reconcile_metric_lines(self) -> list[str]: - """Surface reconcile liveness now that failures no longer raise. - - Serving maintained counters means a failing or wedged reconcile is - invisible to callers, so its age and failure count must be visible - to scrapes for alerting instead. - """ - status_fn = getattr(self.port.ledger, "accepted_stats_reconcile_status", None) - if not callable(status_fn): - return [] - status = status_fn() - lines = [ - "# HELP qbit_prism_accepted_stats_reconcile_failures_total Failed background accepted-stats reconcile passes.", - "# TYPE qbit_prism_accepted_stats_reconcile_failures_total counter", - f"qbit_prism_accepted_stats_reconcile_failures_total {int(status.get('failures') or 0)}", - ] - age = status.get("age_seconds") - if age is not None: - lines.extend( - [ - "# HELP qbit_prism_accepted_stats_reconcile_age_seconds Seconds since the accepted-share counters were last reconciled against the ledger.", - "# TYPE qbit_prism_accepted_stats_reconcile_age_seconds gauge", - f"qbit_prism_accepted_stats_reconcile_age_seconds {float(age):.6f}", - ] - ) - return lines - - def lease_heartbeat_metrics_lines(self) -> list[str]: - """Writer-lease heartbeat phase attribution and its timing policy. - - Issue #212 was diagnosable only from exit messages after the fact: - nothing exported where a heartbeat's wall-clock went, or how much - envelope the shipped policy actually leaves. Every series here has - a fixed label vocabulary (modes, outcomes, phases and policy terms - are closed sets in ``lab.prism.writer_lease_timing``), so scrape - cardinality is constant regardless of traffic or restarts. - """ - snapshot = self.port._ensure_lease_heartbeat_service().snapshot() - attempts = snapshot["attempts"] - outcomes = snapshot["outcomes"] - last_phases = snapshot["last_phase_seconds"] - worst_phases = snapshot["worst_phase_seconds"] - policy_seconds = snapshot["policy_seconds"] - wakes = snapshot["monitor_wakes"] - probe = snapshot["stall_probe"] - assert isinstance(attempts, dict) - assert isinstance(outcomes, dict) - assert isinstance(last_phases, dict) - assert isinstance(worst_phases, dict) - assert isinstance(policy_seconds, dict) - assert isinstance(wakes, dict) - assert isinstance(probe, dict) - wake_buckets = wakes["buckets"] - late_wakes = wakes["late_wakes"] - assert isinstance(wake_buckets, dict) - assert isinstance(late_wakes, dict) - return [ - "# HELP qbit_prism_lease_heartbeat_running Whether the writer-lease heartbeat thread is alive.", - "# TYPE qbit_prism_lease_heartbeat_running gauge", - f"qbit_prism_lease_heartbeat_running {int(bool(snapshot['running']))}", - "# HELP qbit_prism_lease_heartbeat_attempts_total Guard verifications by mode.", - "# TYPE qbit_prism_lease_heartbeat_attempts_total counter", - *[ - f'qbit_prism_lease_heartbeat_attempts_total{{mode="{mode}"}} {int(attempts.get(mode, 0))}' - for mode in LEASE_HEARTBEAT_MODES - ], - "# HELP qbit_prism_lease_heartbeat_outcomes_total Guard verification outcomes.", - "# TYPE qbit_prism_lease_heartbeat_outcomes_total counter", - *[ - f'qbit_prism_lease_heartbeat_outcomes_total{{outcome="{outcome}"}} {int(outcomes.get(outcome, 0))}' - for outcome in LEASE_HEARTBEAT_OUTCOMES - ], - "# HELP qbit_prism_lease_heartbeat_phase_seconds Phase breakdown of the latest guard verification.", - "# TYPE qbit_prism_lease_heartbeat_phase_seconds gauge", - *[ - f'qbit_prism_lease_heartbeat_phase_seconds{{phase="{phase}"}} {float(last_phases.get(phase, 0.0)):.6f}' - for phase in LEASE_HEARTBEAT_PHASES - ], - "# HELP qbit_prism_lease_heartbeat_worst_phase_seconds Phase breakdown of the slowest guard verification observed.", - "# TYPE qbit_prism_lease_heartbeat_worst_phase_seconds gauge", - *[ - f'qbit_prism_lease_heartbeat_worst_phase_seconds{{phase="{phase}"}} {float(worst_phases.get(phase, 0.0)):.6f}' - for phase in LEASE_HEARTBEAT_PHASES - ], - "# HELP qbit_prism_lease_heartbeat_activity_age_seconds Age of the newest monitor-visible heartbeat activity mark.", - "# TYPE qbit_prism_lease_heartbeat_activity_age_seconds gauge", - f"qbit_prism_lease_heartbeat_activity_age_seconds {float(snapshot['activity_age_seconds']):.6f}", - "# HELP qbit_prism_lease_heartbeat_server_proven_age_seconds Age of the newest completed guard round trip.", - "# TYPE qbit_prism_lease_heartbeat_server_proven_age_seconds gauge", - f"qbit_prism_lease_heartbeat_server_proven_age_seconds {float(snapshot['server_proven_age_seconds']):.6f}", - "# HELP qbit_prism_lease_heartbeat_monitor_wake_delay_seconds Worst observed lateness of the heartbeat monitor's own poll.", - "# TYPE qbit_prism_lease_heartbeat_monitor_wake_delay_seconds gauge", - f"qbit_prism_lease_heartbeat_monitor_wake_delay_seconds {float(snapshot['monitor_wake_delay_seconds']):.6f}", - "# HELP qbit_prism_lease_heartbeat_policy_seconds Terms of the validated writer-lease heartbeat timing policy.", - "# TYPE qbit_prism_lease_heartbeat_policy_seconds gauge", - *[ - f'qbit_prism_lease_heartbeat_policy_seconds{{term="{term}"}} {float(policy_seconds.get(term, 0.0)):.6f}' - for term in LEASE_HEARTBEAT_POLICY_TERMS - ], - # Issue #227: the monitor's lateness as signals that re-arm. The - # lifetime gauge above never resets; these do the alerting. - "# HELP qbit_prism_lease_heartbeat_monitor_wake_lateness_seconds Lateness of every heartbeat-monitor poll wake (issue #227); express the wake-delay alert on this or on the late-wake counters, not on the lifetime gauge.", - "# TYPE qbit_prism_lease_heartbeat_monitor_wake_lateness_seconds histogram", - *[ - f'qbit_prism_lease_heartbeat_monitor_wake_lateness_seconds_bucket{{le="{bucket:g}"}} {int(wake_buckets.get(bucket, 0))}' - for bucket in LEASE_MONITOR_WAKE_DELAY_BUCKETS - ], - f'qbit_prism_lease_heartbeat_monitor_wake_lateness_seconds_bucket{{le="+Inf"}} {int(wakes["count"])}', - f'qbit_prism_lease_heartbeat_monitor_wake_lateness_seconds_sum {float(wakes["sum"]):.6f}', - f'qbit_prism_lease_heartbeat_monitor_wake_lateness_seconds_count {int(wakes["count"])}', - "# HELP qbit_prism_lease_heartbeat_monitor_late_wakes_total Monitor wakes at least the labelled fraction of the policy's scheduler slack late.", - "# TYPE qbit_prism_lease_heartbeat_monitor_late_wakes_total counter", - *[ - f'qbit_prism_lease_heartbeat_monitor_late_wakes_total{{slack_fraction="{fraction}"}} {int(late_wakes.get(fraction, 0))}' - for fraction in LEASE_MONITOR_LATE_WAKE_SLACK_FRACTIONS - ], - "# HELP qbit_prism_lease_heartbeat_monitor_wake_delay_window_max_seconds Worst monitor wake lateness inside the trailing 300s window; falls back once a stall ages out, unlike the lifetime gauge.", - "# TYPE qbit_prism_lease_heartbeat_monitor_wake_delay_window_max_seconds gauge", - f"qbit_prism_lease_heartbeat_monitor_wake_delay_window_max_seconds {float(wakes['window_max_seconds']):.6f}", - "# HELP qbit_prism_lease_heartbeat_monitor_wake_delay_record_age_seconds Seconds since the lifetime monitor wake-delay record was last raised.", - "# TYPE qbit_prism_lease_heartbeat_monitor_wake_delay_record_age_seconds gauge", - f"qbit_prism_lease_heartbeat_monitor_wake_delay_record_age_seconds {float(wakes['record_age_seconds']):.6f}", - "# HELP qbit_prism_lease_heartbeat_monitor_exit_guarantee_breaches_total Monitor wakes later than the policy's max_guaranteed_monitor_lateness: beats on which exit-before-adoption was not guaranteed (issue #227).", - "# TYPE qbit_prism_lease_heartbeat_monitor_exit_guarantee_breaches_total counter", - f"qbit_prism_lease_heartbeat_monitor_exit_guarantee_breaches_total {int(snapshot['exit_guarantee_breaches'])}", - "# HELP qbit_prism_lease_heartbeat_monitor_worst_exit_guarantee_overrun_seconds Worst amount by which a late monitor wake could have placed the hard exit after the successor's adoption edge.", - "# TYPE qbit_prism_lease_heartbeat_monitor_worst_exit_guarantee_overrun_seconds gauge", - f"qbit_prism_lease_heartbeat_monitor_worst_exit_guarantee_overrun_seconds {float(snapshot['worst_exit_guarantee_overrun_seconds']):.6f}", - "# HELP qbit_prism_lease_heartbeat_stall_probe_samples_total Stack samples the stall probe took on monitor wakes at least half a scheduler slack late.", - "# TYPE qbit_prism_lease_heartbeat_stall_probe_samples_total counter", - f"qbit_prism_lease_heartbeat_stall_probe_samples_total {int(probe['samples'])}", - "# HELP qbit_prism_lease_heartbeat_stall_probe_suppressed_total Stall-probe triggers refused by its rate limit (at most 3 samples per 60s).", - "# TYPE qbit_prism_lease_heartbeat_stall_probe_suppressed_total counter", - f"qbit_prism_lease_heartbeat_stall_probe_suppressed_total {int(probe['suppressed'])}", - "# HELP qbit_prism_lease_heartbeat_monitor_breach_warnings_suppressed_total Lateness-beyond-slack warnings the monitor refused under its rate limit (at most 3 per 60s); the breaches themselves are still counted.", - "# TYPE qbit_prism_lease_heartbeat_monitor_breach_warnings_suppressed_total counter", - f"qbit_prism_lease_heartbeat_monitor_breach_warnings_suppressed_total {int(snapshot['slack_breach_warnings_suppressed'])}", - ] - - def shutdown_metrics_lines(self) -> list[str]: - snapshot = self.port._ensure_shutdown_controller().snapshot() - quiescence = snapshot["writer_quiescence_outcomes"] - release = snapshot["lease_release_outcomes"] - active = snapshot["active_writers"] - assert isinstance(quiescence, dict) - assert isinstance(release, dict) - assert isinstance(active, dict) - return [ - "# HELP qbit_prism_shutdowns_total Controlled coordinator shutdown sequences started.", - "# TYPE qbit_prism_shutdowns_total counter", - f"qbit_prism_shutdowns_total {int(snapshot['shutdowns_total'])}", - "# HELP qbit_prism_shutdown_writer_operations Active admitted ledger-mutating operations by component.", - "# TYPE qbit_prism_shutdown_writer_operations gauge", - *[ - f'qbit_prism_shutdown_writer_operations{{component="{self.port.prometheus_label_value(str(component))}"}} {int(count)}' - for component, count in sorted(active.items()) - ], - "# HELP qbit_prism_shutdown_writer_quiescence_total Writer-quiescence outcomes.", - "# TYPE qbit_prism_shutdown_writer_quiescence_total counter", - *[ - f'qbit_prism_shutdown_writer_quiescence_total{{outcome="{outcome}"}} {int(quiescence.get(outcome, 0))}' - for outcome in ("success", "timeout") - ], - "# HELP qbit_prism_shutdown_writer_quiescence_seconds Duration of the latest writer-quiescence barrier.", - "# TYPE qbit_prism_shutdown_writer_quiescence_seconds gauge", - f"qbit_prism_shutdown_writer_quiescence_seconds {float(snapshot['writer_quiescence_seconds']):.6f}", - "# HELP qbit_prism_shutdown_lease_release_attempts_total Writer-lease release attempts.", - "# TYPE qbit_prism_shutdown_lease_release_attempts_total counter", - f"qbit_prism_shutdown_lease_release_attempts_total {int(snapshot['lease_release_attempts_total'])}", - "# HELP qbit_prism_shutdown_lease_release_total Writer-lease release outcomes.", - "# TYPE qbit_prism_shutdown_lease_release_total counter", - *[ - f'qbit_prism_shutdown_lease_release_total{{outcome="{outcome}"}} {int(release.get(outcome, 0))}' - for outcome in ("success", "not_held", "unsupported", "failure") - ], - "# HELP qbit_prism_shutdown_lease_release_seconds Duration of the latest writer-lease release attempt.", - "# TYPE qbit_prism_shutdown_lease_release_seconds gauge", - f"qbit_prism_shutdown_lease_release_seconds {float(snapshot['lease_release_seconds']):.6f}", - "# HELP qbit_prism_shutdown_sigterm_to_lease_release_seconds Time from SIGTERM admission close to safe lease release, or -1 if unobserved.", - "# TYPE qbit_prism_shutdown_sigterm_to_lease_release_seconds gauge", - "qbit_prism_shutdown_sigterm_to_lease_release_seconds " - + ( - f"{float(snapshot['sigterm_to_lease_release_seconds']):.6f}" - if snapshot["sigterm_release_observed"] - else "-1" - ), - "# HELP qbit_prism_shutdown_release_withheld_total Shutdowns that withheld lease release because a writer did not quiesce.", - "# TYPE qbit_prism_shutdown_release_withheld_total counter", - f"qbit_prism_shutdown_release_withheld_total {int(snapshot['release_withheld_total'])}", - "# HELP qbit_prism_shutdown_non_writer_drain_seconds Duration of cleanup after writer lease handling.", - "# TYPE qbit_prism_shutdown_non_writer_drain_seconds gauge", - f"qbit_prism_shutdown_non_writer_drain_seconds {float(snapshot['non_writer_drain_seconds']):.6f}", - ] - - def initial_delivery_metrics_lines(self) -> list[str]: - self.port._ensure_initial_job_state() - mining = self.port.mining_delivery_snapshot() - with self.port.lock: - counts = { - "sent": self.port.initial_job_sent_count, - "cancelled": self.port.initial_job_cancelled_count, - "coalesced": self.port.initial_job_coalesced_count, - "failed": self.port.initial_job_failed_count, - "superseded": self.port.initial_job_superseded_count, - } - latency_sum = self.port.initial_job_delivery_latency_seconds_sum - latency_count = self.port.initial_job_delivery_latency_count - queue_capacity_reclaimed = ( - self.port.initial_job_queue_capacity_reclaimed_count - ) - executor = getattr(self.port, "_initial_job_executor", None) - queued, slots = executor.stats() if executor is not None else (0, 0) - configured_workers = int( - getattr( - self.port, - "initial_job_max_workers", - DEFAULT_PRISM_INITIAL_JOB_MAX_WORKERS, - ) - ) - with self.port._bundle_preparation_lock: - build_counts = dict(self.port.shared_bundle_build_counts) - preparation_sum = self.port.shared_bundle_preparation_seconds_sum - preparation_count = self.port.shared_bundle_preparation_count - waiters = self.port.shared_bundle_preparation_waiters - return [ - "# HELP qbit_prism_stratum_subscribed_clients Subscribed Stratum clients.", - "# TYPE qbit_prism_stratum_subscribed_clients gauge", - f'qbit_prism_stratum_subscribed_clients {mining["subscribed_clients"]}', - "# HELP qbit_prism_stratum_authorized_clients Subscribed and authorized Stratum clients.", - "# TYPE qbit_prism_stratum_authorized_clients gauge", - f'qbit_prism_stratum_authorized_clients {mining["authorized_clients"]}', - "# HELP qbit_prism_clients_without_current_tip_job Authorized clients without usable current-tip work.", - "# TYPE qbit_prism_clients_without_current_tip_job gauge", - f'qbit_prism_clients_without_current_tip_job {mining["clients_without_current_tip_job"]}', - "# HELP qbit_prism_clients_with_no_active_job Authorized clients with no active job at all.", - "# TYPE qbit_prism_clients_with_no_active_job gauge", - f'qbit_prism_clients_with_no_active_job {mining["clients_with_no_active_job"]}', - "# HELP qbit_prism_clients_with_current_tip_job Authorized clients with usable current-tip work.", - "# TYPE qbit_prism_clients_with_current_tip_job gauge", - f'qbit_prism_clients_with_current_tip_job {mining["clients_with_current_tip_job"]}', - "# HELP qbit_prism_current_tip_job_coverage_ratio Fraction of authorized clients with current-tip work.", - "# TYPE qbit_prism_current_tip_job_coverage_ratio gauge", - f'qbit_prism_current_tip_job_coverage_ratio {float(mining["current_tip_job_coverage_ratio"]):.12g}', - "# HELP qbit_prism_initial_job_deliveries_pending Coalesced initial deliveries queued or running.", - "# TYPE qbit_prism_initial_job_deliveries_pending gauge", - f'qbit_prism_initial_job_deliveries_pending {mining["clients_pending_initial_job"]}', - "# HELP qbit_prism_initial_job_delivery_tasks_inflight Bounded shared delivery slots currently occupied.", - "# TYPE qbit_prism_initial_job_delivery_tasks_inflight gauge", - f"qbit_prism_initial_job_delivery_tasks_inflight {slots}", - "# HELP qbit_prism_initial_job_delivery_queue_depth Initial-job tasks waiting for a dedicated worker.", - "# TYPE qbit_prism_initial_job_delivery_queue_depth gauge", - f"qbit_prism_initial_job_delivery_queue_depth {queued}", - "# HELP qbit_prism_initial_job_delivery_active_workers Dedicated initial-job workers currently running tasks.", - "# TYPE qbit_prism_initial_job_delivery_active_workers gauge", - f"qbit_prism_initial_job_delivery_active_workers {slots}", - "# HELP qbit_prism_initial_job_delivery_configured_workers Configured dedicated initial-job worker count.", - "# TYPE qbit_prism_initial_job_delivery_configured_workers gauge", - f"qbit_prism_initial_job_delivery_configured_workers {configured_workers}", - "# HELP qbit_prism_initial_job_delivery_seconds Authorization-to-current-job latency.", - "# TYPE qbit_prism_initial_job_delivery_seconds summary", - f"qbit_prism_initial_job_delivery_seconds_sum {latency_sum:.6f}", - f"qbit_prism_initial_job_delivery_seconds_count {latency_count}", - "# HELP qbit_prism_initial_job_requests_total Initial delivery outcomes.", - "# TYPE qbit_prism_initial_job_requests_total counter", - *[ - f'qbit_prism_initial_job_requests_total{{result="{result}"}} {count}' - for result, count in sorted(counts.items()) - ], - "# HELP qbit_prism_initial_job_queue_capacity_reclaimed_total Queued initial-job slots reclaimed immediately by cancellation.", - "# TYPE qbit_prism_initial_job_queue_capacity_reclaimed_total counter", - f"qbit_prism_initial_job_queue_capacity_reclaimed_total {queue_capacity_reclaimed}", - "# HELP qbit_prism_shared_bundle_preparation_seconds Heavy shared bundle preparation wall time.", - "# TYPE qbit_prism_shared_bundle_preparation_seconds summary", - f"qbit_prism_shared_bundle_preparation_seconds_sum {preparation_sum:.6f}", - f"qbit_prism_shared_bundle_preparation_seconds_count {preparation_count}", - "# HELP qbit_prism_shared_bundle_preparation_waiters Callers waiting on the keyed shared preparation flight.", - "# TYPE qbit_prism_shared_bundle_preparation_waiters gauge", - f"qbit_prism_shared_bundle_preparation_waiters {waiters}", - "# HELP qbit_prism_shared_bundle_builds_total Shared bundle builds by terminal outcome.", - "# TYPE qbit_prism_shared_bundle_builds_total counter", - *[ - f'qbit_prism_shared_bundle_builds_total{{result="{result}"}} {count}' - for result, count in sorted(build_counts.items()) - ], - ] - - def job_build_metrics_lines(self) -> list[str]: - self.port._ensure_job_cache_state() - with self.port._job_cache_lock: - bucket_counts = dict(self.port.job_build_seconds_bucket_counts) - build_sum = self.port.job_build_seconds_sum - build_count = self.port.job_build_count - phase_seconds = dict(self.port.job_build_phase_seconds) - hit_counts = dict(self.port.job_cache_hit_counts) - miss_counts = dict(self.port.job_cache_miss_counts) - # Health-refresh failure counts are owner state; read them from - # the observability service directly (no coordinator property). - health_refresh_failures = ( - self.port._ensure_observability_service() - .state() - .health_snapshot_refresh_failure_count - ) - with self.port._job_build_scheduler_lock: - scheduler_counts = dict(self.port.job_build_scheduler_counts) - priority_counts = dict(self.port.job_build_priority_counts) - priority_admission_seconds = dict( - self.port.job_build_priority_admission_seconds - ) - initial_prepared_counts = dict( - self.port.initial_job_prepared_work_counts - ) - cancellation_seconds = dict(self.port.job_build_cancellation_seconds) - replacement_seconds = dict( - self.port.job_build_replacement_start_seconds - ) - worker_counts = dict(self.port.job_build_worker_counts) - active_builds = int(self.port._job_build_active is not None) - pending_builds = int(self.port._job_build_pending is not None) - priority_requests = tuple( - request - for request in ( - ( - self.port._job_build_active.request - if self.port._job_build_active is not None - else None - ), - ( - self.port._job_build_retiring.request - if self.port._job_build_retiring is not None - else None - ), - self.port._job_build_pending, - ) - if request is not None - and not request.cancellation.is_set() - and self.port._job_build_is_publication_critical(request) - ) - priority_preparations = tuple( - self.port._job_build_priority_preparations.values() - ) - priority_active = int( - bool(priority_requests or priority_preparations) - ) - priority_age_seconds = max( - ( - time.monotonic() - request.requested_monotonic - for request in priority_requests - ), - default=0.0, - ) - priority_age_seconds = max( - priority_age_seconds, - max( - ( - time.monotonic() - started - for started in priority_preparations - ), - default=0.0, - ), - ) - lock = getattr(self.port, "lock", None) - if lock is not None: - with lock: - connected_clients = len(getattr(self.port, "clients", ())) - else: - connected_clients = len(getattr(self.port, "clients", ())) - lines = [ - "# HELP qbit_prism_job_build_seconds Wall time from client job build or prepared submission to completion, including skipped prepared tasks.", - "# TYPE qbit_prism_job_build_seconds histogram", - ] - for bucket in PRISM_JOB_BUILD_SECONDS_BUCKETS: - lines.append( - f'qbit_prism_job_build_seconds_bucket{{le="{bucket:g}"}} {bucket_counts.get(bucket, 0)}' - ) - lines.extend( - [ - f'qbit_prism_job_build_seconds_bucket{{le="+Inf"}} {build_count}', - f"qbit_prism_job_build_seconds_sum {build_sum:.6f}", - f"qbit_prism_job_build_seconds_count {build_count}", - "# HELP qbit_prism_job_build_phase_seconds_total Cumulative job build wall time by phase.", - "# TYPE qbit_prism_job_build_phase_seconds_total counter", - *[ - f'qbit_prism_job_build_phase_seconds_total{{phase="{phase}"}} {phase_seconds.get(phase, 0.0):.6f}' - for phase in PRISM_JOB_BUILD_PHASES - ], - "# HELP qbit_prism_job_cache_hits_total Job build cache hits by cache kind.", - "# TYPE qbit_prism_job_cache_hits_total counter", - *[ - f'qbit_prism_job_cache_hits_total{{cache="{kind}"}} {int(hit_counts.get(kind, 0))}' - for kind in PRISM_JOB_CACHE_KINDS - ], - "# HELP qbit_prism_job_cache_misses_total Job build cache misses by cache kind.", - "# TYPE qbit_prism_job_cache_misses_total counter", - *[ - f'qbit_prism_job_cache_misses_total{{cache="{kind}"}} {int(miss_counts.get(kind, 0))}' - for kind in PRISM_JOB_CACHE_KINDS - ], - "# HELP qbit_prism_health_snapshot_refresh_failures_total Background health snapshot refreshes that raised.", - "# TYPE qbit_prism_health_snapshot_refresh_failures_total counter", - f"qbit_prism_health_snapshot_refresh_failures_total {health_refresh_failures}", - "# HELP qbit_prism_connected_clients Currently connected Stratum clients.", - "# TYPE qbit_prism_connected_clients gauge", - f"qbit_prism_connected_clients {connected_clients}", - "# HELP qbit_prism_job_build_requests_total Immutable job build requests admitted to the latest-wins scheduler.", - "# TYPE qbit_prism_job_build_requests_total counter", - f'qbit_prism_job_build_requests_total {int(scheduler_counts.get("requests", 0))}', - "# HELP qbit_prism_job_build_starts_total Immutable job builds started by the bounded executor.", - "# TYPE qbit_prism_job_build_starts_total counter", - f'qbit_prism_job_build_starts_total {int(scheduler_counts.get("starts", 0))}', - "# HELP qbit_prism_job_build_completions_total Immutable job build executions completed.", - "# TYPE qbit_prism_job_build_completions_total counter", - f'qbit_prism_job_build_completions_total {int(scheduler_counts.get("completions", 0))}', - "# HELP qbit_prism_job_build_supersessions_total Active or pending builds replaced by a newer immutable key.", - "# TYPE qbit_prism_job_build_supersessions_total counter", - f'qbit_prism_job_build_supersessions_total {int(scheduler_counts.get("supersessions", 0))}', - "# HELP qbit_prism_job_build_obsolete_results_total Obsolete build results discarded before cache or delivery.", - "# TYPE qbit_prism_job_build_obsolete_results_total counter", - f'qbit_prism_job_build_obsolete_results_total {int(scheduler_counts.get("obsolete_results", 0))}', - "# HELP qbit_prism_job_build_orphan_evicted_total Finished flights evicted from scheduler slots after their completion never released them.", - "# TYPE qbit_prism_job_build_orphan_evicted_total counter", - f'qbit_prism_job_build_orphan_evicted_total {int(scheduler_counts.get("orphan_evicted", 0))}', - "# HELP qbit_prism_job_build_active Current latest-generation build executions.", - "# TYPE qbit_prism_job_build_active gauge", - f"qbit_prism_job_build_active {active_builds}", - "# HELP qbit_prism_job_build_pending Newest build request waiting for a bounded executor slot.", - "# TYPE qbit_prism_job_build_pending gauge", - f"qbit_prism_job_build_pending {pending_builds}", - "# HELP qbit_prism_job_build_cancellation_seconds Cancellation signal to obsolete execution completion.", - "# TYPE qbit_prism_job_build_cancellation_seconds summary", - f'qbit_prism_job_build_cancellation_seconds_sum {float(cancellation_seconds.get("sum", 0.0)):.6f}', - f'qbit_prism_job_build_cancellation_seconds_count {int(cancellation_seconds.get("count", 0))}', - "# HELP qbit_prism_job_build_replacement_start_seconds Supersession signal to replacement build start.", - "# TYPE qbit_prism_job_build_replacement_start_seconds summary", - f'qbit_prism_job_build_replacement_start_seconds_sum {float(replacement_seconds.get("sum", 0.0)):.6f}', - f'qbit_prism_job_build_replacement_start_seconds_count {int(replacement_seconds.get("count", 0))}', - "# HELP qbit_prism_job_build_priority_events_total Publication-critical scheduler admissions and routine-work displacement.", - "# TYPE qbit_prism_job_build_priority_events_total counter", - *[ - f'qbit_prism_job_build_priority_events_total{{result="{result}"}} {int(priority_counts.get(result, 0))}' - for result in ( - "started", - "coalesced", - "queued", - "routine_deferred", - "routine_preempted", - ) - ], - "# HELP qbit_prism_job_build_priority_admission_seconds Publication-priority reservation to builder start or exact-flight coalescing.", - "# TYPE qbit_prism_job_build_priority_admission_seconds summary", - f'qbit_prism_job_build_priority_admission_seconds_sum {float(priority_admission_seconds.get("sum", 0.0)):.6f}', - f'qbit_prism_job_build_priority_admission_seconds_count {int(priority_admission_seconds.get("count", 0))}', - "# HELP qbit_prism_job_build_priority_active Whether publication-critical build work is preparing, running, retiring, or pending.", - "# TYPE qbit_prism_job_build_priority_active gauge", - f"qbit_prism_job_build_priority_active {priority_active}", - "# HELP qbit_prism_job_build_priority_age_seconds Age of the oldest admitted publication-critical build request.", - "# TYPE qbit_prism_job_build_priority_age_seconds gauge", - f"qbit_prism_job_build_priority_age_seconds {priority_age_seconds:.6f}", - "# HELP qbit_prism_initial_job_prepared_work_total Initial jobs that reused, coalesced behind, or deferred to prepared shared work.", - "# TYPE qbit_prism_initial_job_prepared_work_total counter", - *[ - f'qbit_prism_initial_job_prepared_work_total{{result="{result}"}} {int(initial_prepared_counts.get(result, 0))}' - for result in ( - "cache_hit", - "singleflight", - "deferred", - "subscribed", - "admission_deadline", - ) - ], - "# HELP qbit_prism_job_build_worker_events_total Pure builder subprocess lifecycle events.", - "# TYPE qbit_prism_job_build_worker_events_total counter", - *[ - f'qbit_prism_job_build_worker_events_total{{event="{event}"}} {int(worker_counts.get(event, 0))}' - for event in ("starts", "terminations", "crashes", "restarts") - ], - "# HELP qbit_prism_job_build_configured_workers Configured bounded job-build executor worker count.", - "# TYPE qbit_prism_job_build_configured_workers gauge", - ( - "qbit_prism_job_build_configured_workers " - f"{int(getattr(self.port, 'job_build_executor_workers', DEFAULT_PRISM_JOB_BUILD_EXECUTOR_WORKERS))}" - ), - ] - ) - return lines diff --git a/lab/prism/observability.py b/lab/prism/observability.py deleted file mode 100644 index b5a9c37bc..000000000 --- a/lab/prism/observability.py +++ /dev/null @@ -1,780 +0,0 @@ -"""Cached PRISM base health with a fresh monotonic progress overlay.""" - -from __future__ import annotations - -from dataclasses import dataclass -import threading -from typing import Callable, Iterator, Mapping, Protocol - -from lab.prism.progress_health import overlay_progress_health - - -HEALTH_SCHEMA = "qbit.prism.audit-health.v1" -MINIMUM_HEALTH_STALE_SECONDS = 15.0 -METRICS_FAILURE_CLASSES = ("exception", "invalid_payload") - -# What /metrics says about the payload it just returned, so a consumer can tell -# fresh from stale from unavailable without parsing the Prometheus body (issue -# #184). Same shape as #164's database-state header: one bounded value an -# operator greps and a dashboard frontend switches a banner on, rather than a -# warn-code line to parse or a gauge to scrape out of the body it qualifies. -METRICS_STATE_HEADER = "X-Prism-Metrics-State" -METRICS_STATE_FRESH = "fresh" -METRICS_STATE_STALE = "stale" -METRICS_STATE_UNAVAILABLE = "unavailable" -METRICS_STATES = ( - METRICS_STATE_FRESH, - METRICS_STATE_STALE, - METRICS_STATE_UNAVAILABLE, -) - -# Warning 110 is the registered "Response is Stale" warn-code (RFC 7234 5.5.1), -# with this service named as the agent that added it, exactly as #164 says it -# on a cached public response. Said only on a stale payload this endpoint -# actually served: the warm-up refusal has no cached body to qualify. -METRICS_STALE_WARNING_HEADER = "Warning" -METRICS_STALE_WARNING = ( - '110 qbit-prism "metrics snapshot is stale; serving last complete payload"' -) - - -@dataclass(frozen=True) -class MiningDeliveryInputs: - """One immutable collection of coordinator facts used for health policy.""" - - active_connections: int - connection_capacity: int - peak_active_connections: int - subscribed_connections: int - authorized_connections: int - pending_initial_jobs: int - pending_initial_job_capacity: int - oldest_pending_initial_job_age_seconds: float - oldest_genuinely_pending_initial_job_age_seconds: float - clients_with_current_tip_jobs: int - clients_with_semantically_current_work: int - clients_with_no_active_job: int - last_initial_job_delivery_monotonic: float | None - initial_job_timeout_seconds: float - initial_job_queue_rejections: int - initial_job_timeout_disconnects: int - initial_job_cancelled_tasks: int - initial_job_coalesced_tasks: int - initial_job_queue_capacity_reclaimed: int - handler_threads: int - delivery_executor_queue_depth: int - delivery_executor_active_workers: int - started_monotonic: float - startup_grace_seconds: float - stale_unknown_rejections: int - submitted_shares: int - job_preparation_pending: bool - current_observed_tip: str | None - prepared_bundle_current: bool - prepared_bundle_tip: str | None - prepared_bundle_template_generation: int | None - prepared_bundle_payout_generation: int | None - - -class ObservabilityPort(Protocol): - """Narrow coordinator adapter required by health collection.""" - - def monotonic(self) -> float: ... - - def mining_delivery_inputs(self, now: float) -> MiningDeliveryInputs: ... - - def accepted_share_stats(self) -> tuple[int, int]: ... - - def ledger_backend(self) -> str: ... - - def block_counts(self) -> tuple[int, int]: ... - - def progress_health(self) -> Mapping[str, object]: ... - - def health_refresh_seconds(self) -> float: ... - - def render_metrics_payload(self) -> str: ... - - def metrics_refresh_seconds(self) -> float: ... - - def record_startup_phase(self, phase: str) -> None: ... - - def stop_requested(self) -> bool: ... - - def wait_for_stop(self, timeout: float) -> bool: ... - - def log(self, message: str) -> None: ... - - def log_exception(self) -> None: ... - - -@dataclass(frozen=True) -class ObservabilityState: - health_snapshot: dict[str, object] | None - health_snapshot_monotonic: float | None - health_refresh_loop_running: bool - health_snapshot_refresh_failure_count: int - mining_overload_started_monotonic: float | None - mining_delivery_failure_started_monotonic: float | None - mining_semantic_work_gap_started_monotonic: float | None - - -@dataclass(frozen=True) -class MetricsObservabilityState: - metrics_snapshot: str | None - metrics_snapshot_monotonic: float | None - metrics_refresh_loop_running: bool - metrics_collection_success_count: int - metrics_collection_failure_count: int - metrics_collection_generation: int - metrics_failure_exception_count: int - metrics_failure_invalid_payload_count: int - metrics_last_failure_class: str | None - - -@dataclass(frozen=True) -class MetricsSnapshotResponse: - """One /metrics answer: the bytes to send, and what they are (issue #184). - - Iterable as the ``(status, body)`` pair this read used to return, so the - callers that only ever needed those two keep working unchanged while the - HTTP layer gets the freshness facts it must publish out of band. - """ - - status: int - body: str - state: str - age_seconds: int | None - - def __iter__(self) -> Iterator[object]: - yield self.status - yield self.body - - def response_headers(self) -> dict[str, str]: - """State this answer's freshness where no body parsing is needed.""" - - headers = { - # A scrape answer is never re-servable: an intermediary holding a - # stale body would republish it under this response's own Age. - "Cache-Control": "no-store", - METRICS_STATE_HEADER: self.state, - } - if self.age_seconds is not None: - headers["Age"] = str(self.age_seconds) - if self.state == METRICS_STATE_STALE: - headers[METRICS_STALE_WARNING_HEADER] = METRICS_STALE_WARNING - return headers - - -class _InvalidMetricsPayload(ValueError): - pass - - -class ObservabilityService: - """Own health policy, the cached base snapshot, and refresher lifecycle.""" - - def __init__( - self, - port: ObservabilityPort, - *, - lock_factory: Callable[[], threading.RLock] = threading.RLock, - ) -> None: - self.port = port - self._lock: threading.RLock = lock_factory() - self._health_snapshot: dict[str, object] | None = None - self._health_snapshot_monotonic: float | None = None - self._health_refresh_loop_running = False - self._health_snapshot_refresh_failure_count = 0 - self._mining_overload_started_monotonic: float | None = None - self._mining_delivery_failure_started_monotonic: float | None = None - self._mining_semantic_work_gap_started_monotonic: float | None = None - self._mining_delivery_lock = threading.Lock() - self._metrics_lock = threading.RLock() - self._metrics_collection_lock = threading.Lock() - self._metrics_snapshot: str | None = None - self._metrics_snapshot_monotonic: float | None = None - self._metrics_refresh_loop_running = False - self._metrics_collection_success_count = 0 - self._metrics_collection_failure_count = 0 - self._metrics_collection_generation = 0 - self._metrics_failure_counts = { - failure_class: 0 for failure_class in METRICS_FAILURE_CLASSES - } - self._metrics_last_failure_class: str | None = None - - def state(self) -> ObservabilityState: - with self._lock: - return ObservabilityState( - health_snapshot=( - None - if self._health_snapshot is None - else dict(self._health_snapshot) - ), - health_snapshot_monotonic=self._health_snapshot_monotonic, - health_refresh_loop_running=self._health_refresh_loop_running, - health_snapshot_refresh_failure_count=( - self._health_snapshot_refresh_failure_count - ), - mining_overload_started_monotonic=( - self._mining_overload_started_monotonic - ), - mining_delivery_failure_started_monotonic=( - self._mining_delivery_failure_started_monotonic - ), - mining_semantic_work_gap_started_monotonic=( - self._mining_semantic_work_gap_started_monotonic - ), - ) - - def metrics_state(self) -> MetricsObservabilityState: - with self._metrics_lock: - return MetricsObservabilityState( - metrics_snapshot=self._metrics_snapshot, - metrics_snapshot_monotonic=self._metrics_snapshot_monotonic, - metrics_refresh_loop_running=self._metrics_refresh_loop_running, - metrics_collection_success_count=( - self._metrics_collection_success_count - ), - metrics_collection_failure_count=( - self._metrics_collection_failure_count - ), - metrics_collection_generation=self._metrics_collection_generation, - metrics_failure_exception_count=self._metrics_failure_counts[ - "exception" - ], - metrics_failure_invalid_payload_count=self._metrics_failure_counts[ - "invalid_payload" - ], - metrics_last_failure_class=self._metrics_last_failure_class, - ) - - def replace_lock_for_test(self, lock: threading.RLock) -> None: - self._lock = lock - - def clear_health_snapshot_for_test(self) -> None: - with self._lock: - self._health_snapshot = None - - def set_health_snapshot_monotonic_for_test( - self, - value: float | None, - ) -> None: - with self._lock: - self._health_snapshot_monotonic = value - - def set_loop_running_for_test(self, value: bool) -> None: - with self._lock: - self._health_refresh_loop_running = bool(value) - - def set_delivery_failure_started_monotonic_for_test( - self, - value: float | None, - ) -> None: - # A test that starts past the deadline means the whole delivery gap: - # seed the semantic timer alongside the strict tip-hash one. - with self._lock: - self._mining_delivery_failure_started_monotonic = value - self._mining_semantic_work_gap_started_monotonic = value - - def reset_delivery_failure(self) -> None: - """Start the next current-tip coverage gap with a fresh grace window.""" - - with self._lock: - self._mining_delivery_failure_started_monotonic = None - - def mining_delivery_snapshot( - self, - *, - now: float | None = None, - ) -> dict[str, object]: - with self._mining_delivery_lock: - return self._mining_delivery_snapshot_serialized(now=now) - - def _mining_delivery_snapshot_serialized( - self, - *, - now: float | None, - ) -> dict[str, object]: - now = self.port.monotonic() if now is None else now - inputs = self.port.mining_delivery_inputs(now) - authorized = inputs.authorized_connections - current = inputs.clients_with_current_tip_jobs - semantic_current = inputs.clients_with_semantically_current_work - coverage = current / authorized if authorized else 1.0 - semantic_coverage = ( - semantic_current / authorized if authorized else 1.0 - ) - cap_saturated = ( - inputs.connection_capacity > 0 - and inputs.active_connections >= inputs.connection_capacity - ) - pending_saturated = ( - inputs.pending_initial_jobs >= inputs.pending_initial_job_capacity - ) - # A reconnect incident is operationally significant well before - # nearly every miner is missing work. Treat any sustained loss of - # at least five percent of current-job coverage as degraded. - poor_coverage = authorized > 0 and coverage < 0.95 - # Exact-tip churn alone must not fail health (#216): a fleet holding - # work for the current template content is being served even while - # the strict tip-hash gauge lags the observed tip, so the delivery - # stall is timed on semantic coverage. Boundary: overload_now, - # reject_storm, and the cap-saturated reason deliberately stay on - # exact-tip coverage. Each already needs independent corroboration - # (queue/cap saturation or an observed rejection storm) and - # diagnoses strict-tip operational pressure; semantic coverage - # changes only the delivery-stalled predicate. - poor_semantic_coverage = authorized > 0 and semantic_coverage < 0.95 - overload_now = pending_saturated or (cap_saturated and poor_coverage) - with self._lock: - if poor_coverage: - if self._mining_delivery_failure_started_monotonic is None: - self._mining_delivery_failure_started_monotonic = now - else: - self._mining_delivery_failure_started_monotonic = None - delivery_failure_started = ( - self._mining_delivery_failure_started_monotonic - ) - if poor_semantic_coverage: - if self._mining_semantic_work_gap_started_monotonic is None: - self._mining_semantic_work_gap_started_monotonic = now - else: - self._mining_semantic_work_gap_started_monotonic = None - semantic_gap_started = ( - self._mining_semantic_work_gap_started_monotonic - ) - if overload_now: - if self._mining_overload_started_monotonic is None: - self._mining_overload_started_monotonic = now - else: - self._mining_overload_started_monotonic = None - overload_started = self._mining_overload_started_monotonic - delivery_failure_age = ( - max(0.0, now - delivery_failure_started) - if delivery_failure_started is not None - else 0.0 - ) - semantic_gap_age = ( - max(0.0, now - semantic_gap_started) - if semantic_gap_started is not None - else 0.0 - ) - overload_age = ( - max(0.0, now - overload_started) - if overload_started is not None - else 0.0 - ) - deadline = ( - inputs.initial_job_timeout_seconds - if inputs.initial_job_timeout_seconds > 0 - else None - ) - startup_age = max(0.0, now - inputs.started_monotonic) - in_startup_grace = startup_age < inputs.startup_grace_seconds - # A nonzero genuine pending age implies at least one genuinely - # pending authorized client existed at capture time, so the age - # threshold alone is the starvation predicate. - initial_job_starved = bool( - deadline is not None - and inputs.oldest_genuinely_pending_initial_job_age_seconds >= deadline - ) - # The semantic gap must be sustained for a full deadline, and the - # strict gap alone can never fire it. Exact current-tip work is - # checked against the same template fingerprint, payout generation, - # and client-session generations whenever a template snapshot is - # published, so in production it is a subset of the semantic count - # and this reduces to the semantic gap; an embedder that publishes - # only the observed tip has no semantic evidence, and its exact-tip - # work must still count as delivered. - current_work_coverage_stalled = bool( - deadline is not None - and poor_semantic_coverage - and semantic_gap_age >= deadline - and poor_coverage - and delivery_failure_age >= deadline - ) - no_delivery_progress = bool( - initial_job_starved or current_work_coverage_stalled - ) - reject_storm = ( - poor_coverage - and inputs.submitted_shares > 0 - and inputs.stale_unknown_rejections / inputs.submitted_shares >= 0.95 - ) - persistent_overload = deadline is not None and overload_age >= deadline - unhealthy_reasons: list[str] = [] - if not in_startup_grace: - if no_delivery_progress: - unhealthy_reasons.append("initial-delivery-stalled") - if pending_saturated and persistent_overload: - unhealthy_reasons.append("pending-initial-jobs-saturated") - if cap_saturated and poor_coverage and persistent_overload: - unhealthy_reasons.append("connection-capacity-saturated") - if reject_storm: - unhealthy_reasons.append("stale-unknown-rejection-storm") - mining_ready = not unhealthy_reasons - return { - "mining_ready": mining_ready, - "mining_delivery_healthy": mining_ready, - "mining_health_startup_grace": in_startup_grace, - "active_connections": inputs.active_connections, - "connection_capacity": inputs.connection_capacity, - "peak_active_connections": inputs.peak_active_connections, - "subscribed_connections": inputs.subscribed_connections, - "authorized_connections": authorized, - "pending_initial_jobs": inputs.pending_initial_jobs, - "pending_initial_job_capacity": inputs.pending_initial_job_capacity, - "oldest_pending_initial_job_age_seconds": round( - inputs.oldest_pending_initial_job_age_seconds, - 3, - ), - "oldest_genuinely_pending_initial_job_age_seconds": round( - inputs.oldest_genuinely_pending_initial_job_age_seconds, - 3, - ), - "clients_with_current_tip_jobs": current, - "current_tip_job_coverage": round(coverage, 6), - "clients_with_semantically_current_work": semantic_current, - "semantic_current_work_ratio": round(semantic_coverage, 6), - "current_tip_coverage_gap_age_seconds": round( - delivery_failure_age, - 3, - ), - "semantic_current_work_gap_age_seconds": round( - semantic_gap_age, - 3, - ), - "connection_capacity_saturated": cap_saturated, - "pending_initial_jobs_saturated": pending_saturated, - "initial_delivery_stalled": no_delivery_progress, - "overload": bool(overload_now or reject_storm), - "overload_age_seconds": round(overload_age, 3), - "unhealthy_reasons": unhealthy_reasons, - "initial_job_queue_rejections": inputs.initial_job_queue_rejections, - "initial_job_timeout_disconnects": ( - inputs.initial_job_timeout_disconnects - ), - "initial_job_cancelled_tasks": inputs.initial_job_cancelled_tasks, - "initial_job_coalesced_tasks": inputs.initial_job_coalesced_tasks, - "initial_job_queue_capacity_reclaimed": ( - inputs.initial_job_queue_capacity_reclaimed - ), - "handler_threads": inputs.handler_threads, - "delivery_executor_queue_depth": ( - inputs.delivery_executor_queue_depth - ), - "delivery_executor_active_workers": ( - inputs.delivery_executor_active_workers - ), - # Compatibility aliases and preparation visibility introduced by - # the prewarm work. These retain the original bounded-pipeline - # names above for existing dashboards. - "subscribed_clients": inputs.subscribed_connections, - "authorized_clients": authorized, - "clients_with_no_active_job": inputs.clients_with_no_active_job, - "clients_without_current_tip_job": authorized - current, - "clients_with_current_tip_job": current, - "clients_pending_initial_job": inputs.pending_initial_jobs, - "current_tip_job_coverage_ratio": coverage, - # Compatibility alias: this now reports only genuine first-job - # starvation. Current-tip fanout lag has its own age above. - "oldest_initial_job_pending_seconds": round( - inputs.oldest_genuinely_pending_initial_job_age_seconds, - 3, - ), - "job_preparation_pending": inputs.job_preparation_pending, - "current_observed_tip": inputs.current_observed_tip, - "prepared_bundle_current": inputs.prepared_bundle_current, - "prepared_bundle_tip": inputs.prepared_bundle_tip, - "prepared_bundle_template_generation": ( - inputs.prepared_bundle_template_generation - ), - "prepared_bundle_payout_generation": ( - inputs.prepared_bundle_payout_generation - ), - } - - def base_health_payload(self) -> dict[str, object]: - accepted_share_count, ready_miner_count = self.port.accepted_share_stats() - mining = self.mining_delivery_snapshot() - accepted_block_count, max_blocks = self.port.block_counts() - return { - "ok": bool(mining["mining_ready"]), - "schema": HEALTH_SCHEMA, - "ledger_backend": self.port.ledger_backend(), - "accepted_share_count": accepted_share_count, - "ready_miner_count": ready_miner_count, - "accepted_block": accepted_block_count > 0, - "accepted_block_count": accepted_block_count, - "max_blocks": max_blocks, - **mining, - } - - def _with_current_progress( - self, - base_health: Mapping[str, object], - ) -> dict[str, object]: - return self.apply_progress_health( - base_health, - self.port.progress_health(), - ) - - @staticmethod - def apply_progress_health( - base_health: Mapping[str, object], - progress: Mapping[str, object], - ) -> dict[str, object]: - """Retain the existing overlay contract behind the new owner.""" - - return dict(overlay_progress_health(base_health, progress)) - - def health_payload(self) -> dict[str, object]: - return self._with_current_progress(self.base_health_payload()) - - def refresh_health_snapshot(self) -> dict[str, object]: - # Cache only ledger/session-backed base health. Progress is monotonic, - # in-memory state and is deliberately re-read for every response. - base_health = self.base_health_payload() - with self._lock: - self._health_snapshot = base_health - self._health_snapshot_monotonic = self.port.monotonic() - self.port.record_startup_phase("health_snapshot_warm") - return self._with_current_progress(base_health) - - def cached_health_payload(self) -> tuple[int, dict[str, object]]: - with self._lock: - snapshot = ( - None - if self._health_snapshot is None - else dict(self._health_snapshot) - ) - snapshot_monotonic = self._health_snapshot_monotonic - loop_running = self._health_refresh_loop_running - if snapshot is None or snapshot_monotonic is None: - if not loop_running: - # No refresher (tests, or audit HTTP without serve()): compute - # inline like the legacy endpoint did. - payload = self.refresh_health_snapshot() - return (200 if payload.get("ok") else 503), payload - # The refresher owns the cold accepted-share warm-up; until its - # first snapshot lands, report an explicit starting state instead - # of running ledger aggregates on the handler thread (issue #188 - # fix 4). - payload = self._with_current_progress( - { - "ok": False, - "schema": HEALTH_SCHEMA, - "state": "starting", - "error": "health snapshot warm-up has not completed yet", - } - ) - payload["ok"] = False - return 503, payload - age_seconds = self.port.monotonic() - snapshot_monotonic - stale_after = max( - 3 * self.port.health_refresh_seconds(), - MINIMUM_HEALTH_STALE_SECONDS, - ) - if age_seconds > stale_after: - payload = self._with_current_progress( - { - "ok": False, - "schema": HEALTH_SCHEMA, - "error": "health snapshot is stale", - "snapshot_age_seconds": round(age_seconds, 3), - } - ) - payload["ok"] = False - return 503, payload - # Ledger-backed fields stay cached, but progress state is an in-memory - # monotonic snapshot and must be overlaid on every request. Otherwise a - # cached ok=true response can mask a known failed refresh for another - # full cache cycle (the production incident this endpoint must expose). - payload = self._with_current_progress(snapshot) - payload["snapshot_age_seconds"] = round(age_seconds, 3) - return (200 if payload.get("ok") else 503), payload - - def begin_refresh_loop(self) -> bool: - with self._lock: - if self._health_refresh_loop_running: - return False - self._health_refresh_loop_running = True - return True - - def health_snapshot_loop(self) -> None: - # The running flag is a one-shot latch (upstream #120): it is armed - # before the refresher thread can be dispatched and deliberately - # never cleared, so a request racing loop exit (or arriving during - # shutdown drain) fails closed through the cached path instead of - # re-opening cached_health_payload's legacy inline aggregate. - while not self.port.stop_requested(): - try: - self.refresh_health_snapshot() - except Exception: - with self._lock: - self._health_snapshot_refresh_failure_count += 1 - self.port.log("prism coordinator: health snapshot refresh failed") - self.port.log_exception() - if self.port.wait_for_stop(self.port.health_refresh_seconds()): - break - - @staticmethod - def _metrics_failure_class(exc: Exception) -> str: - if isinstance(exc, _InvalidMetricsPayload): - return "invalid_payload" - return "exception" - - def refresh_metrics_snapshot(self) -> str: - """Collect and atomically publish one complete Prometheus document.""" - - try: - with self._metrics_collection_lock: - payload = self.port.render_metrics_payload() - if not isinstance(payload, str) or not payload or not payload.endswith( - "\n" - ): - raise _InvalidMetricsPayload( - "metrics renderer must return non-empty newline-terminated text" - ) - collected_monotonic = self.port.monotonic() - with self._metrics_lock: - self._metrics_snapshot = payload - self._metrics_snapshot_monotonic = collected_monotonic - self._metrics_collection_success_count += 1 - self._metrics_collection_generation += 1 - self._metrics_last_failure_class = None - return payload - except Exception as exc: - failure_class = self._metrics_failure_class(exc) - with self._metrics_lock: - self._metrics_collection_failure_count += 1 - self._metrics_failure_counts[failure_class] += 1 - self._metrics_last_failure_class = failure_class - raise - - def _metrics_snapshot_response( - self, - *, - now: float, - refresh_seconds: float, - ) -> MetricsSnapshotResponse: - with self._metrics_lock: - snapshot = self._metrics_snapshot - snapshot_monotonic = self._metrics_snapshot_monotonic - running = self._metrics_refresh_loop_running - successes = self._metrics_collection_success_count - generation = self._metrics_collection_generation - failure_counts = dict(self._metrics_failure_counts) - age_seconds = ( - -1.0 - if snapshot_monotonic is None - else max(0.0, now - snapshot_monotonic) - ) - stale_after = max(3 * refresh_seconds, MINIMUM_HEALTH_STALE_SECONDS) - stale = snapshot is None or age_seconds > stale_after - diagnostic_lines = [ - "# HELP qbit_prism_metrics_snapshot_available Whether a complete metrics snapshot is available.", - "# TYPE qbit_prism_metrics_snapshot_available gauge", - f"qbit_prism_metrics_snapshot_available {1 if snapshot is not None else 0}", - "# HELP qbit_prism_metrics_snapshot_stale Whether the complete metrics snapshot exceeded its staleness budget.", - "# TYPE qbit_prism_metrics_snapshot_stale gauge", - f"qbit_prism_metrics_snapshot_stale {1 if stale else 0}", - "# HELP qbit_prism_metrics_snapshot_age_seconds Age of the complete metrics snapshot, or -1 before first success.", - "# TYPE qbit_prism_metrics_snapshot_age_seconds gauge", - f"qbit_prism_metrics_snapshot_age_seconds {age_seconds:.3f}", - "# HELP qbit_prism_metrics_collector_running Whether the background metrics collector is running.", - "# TYPE qbit_prism_metrics_collector_running gauge", - f"qbit_prism_metrics_collector_running {1 if running else 0}", - "# HELP qbit_prism_metrics_collection_successes_total Complete metrics snapshots published.", - "# TYPE qbit_prism_metrics_collection_successes_total counter", - f"qbit_prism_metrics_collection_successes_total {successes}", - "# HELP qbit_prism_metrics_collection_failures_total Metrics collection failures by bounded class.", - "# TYPE qbit_prism_metrics_collection_failures_total counter", - *( - f'qbit_prism_metrics_collection_failures_total{{class="{failure_class}"}} {failure_counts[failure_class]}' - for failure_class in METRICS_FAILURE_CLASSES - ), - "# HELP qbit_prism_metrics_snapshot_generation Complete metrics snapshot generation.", - "# TYPE qbit_prism_metrics_snapshot_generation gauge", - f"qbit_prism_metrics_snapshot_generation {generation}", - ] - prefix = "" if snapshot is None else snapshot - body = prefix + "\n".join(diagnostic_lines) + "\n" - if snapshot is None: - # Nothing complete has ever been published, so there is no payload - # to serve and no truthful age to publish beside it. Refuse, and - # say so out of band too: this body is diagnostics only, and a - # scraper that stored it would be storing a metrics document that - # is missing every metric the process actually reports. - return MetricsSnapshotResponse( - status=503, - body=body, - state=METRICS_STATE_UNAVAILABLE, - age_seconds=None, - ) - # A complete payload exists, so it is served -- stale or not (#184). - # Prometheus discards the body of a non-200, so answering 503 here threw - # the last known-good document away at exactly the moment it was the - # only thing left to look at, and took the diagnostic gauges above with - # it. The staleness is not hidden: it travels in the response metadata - # and in snapshot_stale/snapshot_age_seconds. Nothing is rendered or - # collected on this thread either way -- the published generation is - # returned as it stands, so a collector wedged on the GIL or on the - # renderer lock cannot delay or re-enter this path. - return MetricsSnapshotResponse( - status=200, - body=body, - state=METRICS_STATE_STALE if stale else METRICS_STATE_FRESH, - age_seconds=int(age_seconds), - ) - - def cached_metrics_payload(self) -> MetricsSnapshotResponse: - """Return the cached answer using only monotonic time and metrics state.""" - - now = self.port.monotonic() - refresh_seconds = self.port.metrics_refresh_seconds() - return self._metrics_snapshot_response( - now=now, - refresh_seconds=refresh_seconds, - ) - - def begin_metrics_refresh_loop(self) -> bool: - with self._metrics_lock: - if self._metrics_refresh_loop_running: - return False - self._metrics_refresh_loop_running = True - return True - - def metrics_snapshot_loop(self) -> None: - try: - while not self.port.stop_requested(): - try: - self.refresh_metrics_snapshot() - except Exception: - self.port.log("prism coordinator: metrics snapshot refresh failed") - self.port.log_exception() - if self.port.wait_for_stop(self.port.metrics_refresh_seconds()): - break - finally: - with self._metrics_lock: - self._metrics_refresh_loop_running = False - - -__all__ = [ - "HEALTH_SCHEMA", - "MINIMUM_HEALTH_STALE_SECONDS", - "METRICS_FAILURE_CLASSES", - "METRICS_STALE_WARNING", - "METRICS_STALE_WARNING_HEADER", - "METRICS_STATES", - "METRICS_STATE_FRESH", - "METRICS_STATE_HEADER", - "METRICS_STATE_STALE", - "METRICS_STATE_UNAVAILABLE", - "MetricsObservabilityState", - "MetricsSnapshotResponse", - "MiningDeliveryInputs", - "ObservabilityPort", - "ObservabilityService", - "ObservabilityState", -] diff --git a/lab/prism/payout_state.py b/lab/prism/payout_state.py deleted file mode 100644 index e6f2fdb17..000000000 --- a/lab/prism/payout_state.py +++ /dev/null @@ -1,5944 +0,0 @@ -#!/usr/bin/env python3 -"""PRISM payout-state ownership: generations, artifacts, previews, fencing. - -This module owns the P1 domain: the payout-state generation and its -publication/delivery gate, the payout ledger artifact (incremental windows, -reuse/debounce/re-anchor policy and its background preparation executor), -accepted-block payout previews and their landing transitions, and the #112 -append/anchor fencing (inflight scan anchors, the published seedless job -window anchor, the append invalidation epoch, the rare append-vs-submit -landing fence, and the unfenced-append registry/drain). - -It never imports ``prism_coordinator``. The ledger handle, J1 job cache and -scheduler, R1 refresh scheduling, G1 progress recording, S3's pending-commit -floor, the shutdown controller, and live configuration attributes are reached -through the :class:`PayoutStateRuntime` typed port, resolved at call time so -the historical coordinator monkeypatch seams (including the instance-level -facade patches used by the current test suite) keep intercepting exactly as -before the extraction. Owner facades route back into this service; that -round trip is deliberate. - -Cross-owner lock order is unchanged by the extraction: payout publication and -delivery use this owner's delivery gate and balance-mutation lock; append -classification briefly takes and releases the state lock, then takes the -append-landing fence around the ledger commit; landing drains unfenced -appends without the fence, then holds fence -> state for the final epoch -check; no socket or RPC work runs while registry/retained locks are held. -""" - -from __future__ import annotations - -from concurrent.futures import Future, ThreadPoolExecutor -from contextlib import contextmanager -from dataclasses import dataclass, field, replace as dataclass_replace -import hashlib -import json -import threading -import time -from typing import Any, Callable, Iterator, Mapping, Protocol, Sequence - -from lab.prism.accepted_preview_telemetry import ( - FULL_RESCAN_PATH_DAEMON, - FULL_RESCAN_PATH_IN_PROCESS, - ensure_accepted_preview_telemetry, -) -from lab.prism.candidate_window import disk_window_covers, recorded_share_ids -from lab.prism.coordinator_config import ( - DEFAULT_ACCEPTED_PARENT_UNRESOLVED_DEPTH_MAX, - DEFAULT_PRISM_PAYOUT_ARTIFACT_FULL_RESCAN_SECONDS, - DEFAULT_PRISM_PAYOUT_ARTIFACT_MAX_ANCHOR_AGE_SECONDS, - DEFAULT_PRISM_PAYOUT_ARTIFACT_REANCHOR_SECONDS, - DEFAULT_PRISM_PAYOUT_ARTIFACT_REARM_MIN_SECONDS, - env_bool, - env_window_pipeline_rust, -) -from lab.prism.coordinator_shutdown import ( - BLOCK_SUBMITTER_WAIT_HEARTBEAT_SLICE_SECONDS, -) -from lab.prism.share_json_stream import ( - release_share_list_incrementally, - share_array_json_view, -) -from lab.prism.share_ledger import ( - DEFAULT_INCREMENTAL_SHARE_WINDOW_PAGE_SIZE, - DaemonShareWindowMirror, - DaemonWindowMirrorDivergence, - IncrementalShareJsonSequence, # noqa: F401 - annotation/compatibility export - IncrementalShareWindow, - IncrementalWindowAdvanceStats, - IncrementalWindowFallback, - PendingShare, # noqa: F401 - annotation/compatibility export -) - - -DEFAULT_ACCEPTED_BLOCK_PAYOUT_PREVIEW_WAIT_SECONDS = 5.0 -DEFAULT_PRISM_PAYOUT_RECONCILE_SUPERSESSION_RETRIES = 8 -PRISM_PAYOUT_DELIVERY_GENERATIONS = ("current", "stale", "future") -# Consecutive aborted speculative rebuilds double the re-arm interval up to -# this multiplier (80s at the default floor). Rebuilds abort on generation -# supersession, snapshot errors, empty windows, or a pathologically old -# pending-commit floor; backing off stops database preparation from cycling -- -# and from holding the payout preparation lock tip builds also need -- -# while the underlying condition persists. Any successful install or -# event-driven preparation resets the backoff. -PRISM_PAYOUT_ARTIFACT_REARM_BACKOFF_CAP = 16 -# How many append-invalidation bumps keep the stamp of the row that caused -# them. The stamps are what let a landing ask whether a bump between its own -# baseline epoch and the live one invalidated ITS window, instead of reading -# the bare counter and abandoning for an append that only invalidated a newer -# window (issue #126). Only a replay-shaped append -- a durable row that -# predates a live anchor -- ever adds an entry, and a landing's baseline is -# stamped when its job was issued and stops mattering the moment the block -# lands, so a few hundred spans far more such appends than any candidate can -# outlive. Falling off the end costs nothing but the relaxation: a baseline -# older than the retained history fails closed, exactly as the bare counter -# did. -PRISM_PAYOUT_APPEND_INVALIDATION_STAMP_HISTORY = 256 -# Owner-local copies of the still-coordinator-visible reward-window constants -# and the admission poll cadence; compatibility duplicates by design so this -# leaf module never imports upward. -PRISM_REWARD_WINDOW_MULTIPLIER = 8 -PRISM_SNAPSHOT_WINDOW_MARGIN = 2 - -# qbit retargets every block (ASERT), so the snapshot weight -- margin times -# the 8 x difficulty reward window -- drifts a fraction of a percent with -# every tip change. Treating any weight delta as a cache invalidation -# therefore forced the full oracle rescan (a multi-second ledger read under -# the postgres writer lock) on effectively every block, which is exactly the -# hold that starves the block submitter's landing-window reads and delays -# accepted-block preview publication (mainnet alert #236, 2026-08-25). -# -# The snapshot deliberately retains PRISM_SNAPSHOT_WINDOW_MARGIN times the -# reward window, and every consumer re-derives the exact 8 x difficulty -# cutoff from its own live network_difficulty (_prism_window_shares and the -# fold's requested_window_weight); the cached superset therefore stays -# payout-correct while it still covers the live reward window. Hold the -# cached weight inside a band instead of demanding equality: rescan only -# when the cached weight falls below 5/8 of the live snapshot weight (the -# 8 x reward window plus 25% headroom out of the 2 x margin) or exceeds -# twice it (bounding how oversized a stale superset may grow). Forced -# rescans -- landing reconcile invalidation, cold start, invariant -# failures, daemon rebuilds -- still re-center the weight at the live -# difficulty, and the periodic runtime-check validates the pipeline at the -# cached weight so its match verdict stays like-for-like. -PRISM_SNAPSHOT_WINDOW_WEIGHT_BAND_FLOOR_NUM = 5 -PRISM_SNAPSHOT_WINDOW_WEIGHT_BAND_FLOOR_DEN = 8 -PRISM_SNAPSHOT_WINDOW_WEIGHT_BAND_CEILING = 2 - -# Payout coverage depends on the floor retaining at least the whole reward -# window out of the margined snapshot: FLOOR_NUM/FLOOR_DEN >= 1/MARGIN. -# The fold has no error path for an under-covering superset -- it would -# silently truncate the window tail and misdistribute the coinbase -- so a -# margin retune that breaks the relationship must fail at import, not at -# payout time. Not an `assert`: python -O would strip it. -if ( - PRISM_SNAPSHOT_WINDOW_WEIGHT_BAND_FLOOR_NUM * PRISM_SNAPSHOT_WINDOW_MARGIN - < PRISM_SNAPSHOT_WINDOW_WEIGHT_BAND_FLOOR_DEN -): - raise AssertionError( - "snapshot window weight band floor no longer covers the reward " - "window at the configured snapshot margin" - ) - - -def _snapshot_window_weight_within_band( - cached_weight: int, live_weight: int -) -> bool: - """Whether a cached snapshot weight still serves the live reward window.""" - if cached_weight <= 0 or live_weight <= 0: - return False - if cached_weight == live_weight: - return True - if ( - cached_weight * PRISM_SNAPSHOT_WINDOW_WEIGHT_BAND_FLOOR_DEN - < live_weight * PRISM_SNAPSHOT_WINDOW_WEIGHT_BAND_FLOOR_NUM - ): - return False - return cached_weight <= live_weight * PRISM_SNAPSHOT_WINDOW_WEIGHT_BAND_CEILING - - -PRISM_TIP_REFRESH_ADMISSION_POLL_SECONDS = 0.05 -# Payout preparation/publication/first-delivery latency buckets; value-equal -# to the tip-refresh owner's seconds buckets so exposition is unchanged. -PRISM_PAYOUT_SECONDS_BUCKETS = ( - 0.01, - 0.025, - 0.05, - 0.1, - 0.25, - 0.5, - 1.0, - 2.5, - 5.0, - 10.0, - 30.0, -) -# Where one payout-window build spends its time, as a closed phase set -# (issue #228). ``ledger_read`` is every PostgreSQL read the build issues -# (full oracle, delta, periodic self-check, carry balances); -# ``record_conversion`` is folding records into pages, canonical JSON and -# digests in-process; ``daemon_prepare`` is the Rust daemon round trip -# (prepare or upload); ``lock_wait`` is time queued for the preparation -# lock. Fixed cardinality: no per-tip, per-generation or per-reason label. -PRISM_PAYOUT_WINDOW_BUILD_PHASES = ( - "ledger_read", - "record_conversion", - "daemon_prepare", - "lock_wait", -) -# Staleness the coordinator accepts on the tip-refresh path (issue #228). -# A routine candidate preparation that finds the preparation lock busy -# reuses the armed share window instead of queueing behind the lock -# holder, provided the window's anchor is no older than this; and a -# candidate build on that path defers an overdue periodic self-check -# while its cached window is inside the same bound, for at most this long -# past the check's due time (measured from the last self-check attempt, -# so build frequency cannot extend it). The number is one background -# re-anchor interval (DEFAULT_PRISM_PAYOUT_ARTIFACT_REANCHOR_SECONDS): in -# steady state the re-anchor keeps the armed window younger than this, -# so a window inside the bound is one the delivery path is already -# serving to new jobs as fresh, and a fanout that reuses it declares an -# anchor no older than reuse itself declares. Past the bound the wave -# pays the (incremental) build instead: the re-anchor has already flagged -# that window for replacement, and carrying it into a new payout -# generation would let a stalled background walker pin a stale anchor -# for every following generation. It sits well inside the 300 s audit -# ceiling that bounds how far any served window may trail the live -# ledger. -PRISM_TIP_REFRESH_WINDOW_STALENESS_SECONDS = 60.0 -# How one payout-window build ended, as a closed set (issue #228). The -# phase family is observed under this label on every exit, so a build that -# lost its generation race, hit the audit ceiling, found an empty window or -# died with the ledger still contributes the time it measured; those are -# exactly the slow builds the split exists to attribute. -PRISM_PAYOUT_WINDOW_BUILD_OUTCOMES = ( - "completed", - "debounced", - "aborted", - "failed", -) - - -def canonical_json_text(value: object) -> str: - """Owner-local canonical serialization (no reverse imports by design).""" - return json.dumps(value, sort_keys=True, separators=(",", ":"), default=str) - - -def canonical_json_sha256(value: object) -> str: - """Owner-local canonical digest (no reverse imports by design).""" - incremental_digest = getattr(value, "canonical_json_sha256", None) - if callable(incremental_digest): - return str(incremental_digest()) - return hashlib.sha256(canonical_json_text(value).encode()).hexdigest() - - -class TemplateRefreshBlocked(RuntimeError): - """A live template was fetched, but safe work could not be issued.""" - - -class TemplateRefreshSuperseded(TemplateRefreshBlocked): - """Concurrent tip/payout progress invalidated this refresh attempt. - - Raised only for coordination races that a scheduled retry resolves on its - own: the tip advanced mid-refresh, the payout-state generation moved, or a - newer observation superseded the prepared work. Unlike its parent, this - subclass never arms the ordinary template-refresh failure budget. Repeated - coordination blocks are tracked by their own longer deadline; a genuine - RPC/build/trust failure must raise plain TemplateRefreshBlocked so it still - takes the ordinary budgeted restart path. - """ - - -class PayoutStatePublicationBlocked(TemplateRefreshBlocked): - """Job construction is waiting for a prepared payout publication.""" - - -class AcceptedParentPayoutPreviewPending(PayoutStatePublicationBlocked): - """A child build's bounded wait expired with its parent's preview unpublished. - - Raised only for the one benign backpressure condition: an armed, - un-withdrawn accepted-parent transition whose prospective balances have - not published within ``accepted_block_payout_preview_wait_seconds``. The - parent is still landing normally, so nothing has failed -- the child is - simply early. - - Subclassing PayoutStatePublicationBlocked keeps this inside the - coordination-blocked family: the retry coalesces with the scheduled tip - refresh, it never arms ``job_build_failure_count`` or the ordinary - template-refresh failure budget, and a sustained continuous streak still - exits through the coordination-blocked watchdog rather than retrying - silently forever. Callers that catch either parent keep working - unchanged. - - Deliberately *not* used for any other block: RPC/build/trust failures, - withdrawn previews, unknown replay-enumeration state, unresolved-depth - caps, and invariant violations keep their existing types so their - persistence still takes the ordinary budgeted restart path. - """ - - #: Stable machine-readable reason for retry logs and delivery telemetry. - retry_reason = "accepted_parent_preview_pending" - - def __init__( - self, - message: str = "accepted parent payout preview is not ready yet", - *, - parent_hash: str | None = None, - waited_seconds: float | None = None, - timeout_count: int | None = None, - ) -> None: - super().__init__(message) - self.parent_hash = parent_hash - self.waited_seconds = waited_seconds - self.timeout_count = timeout_count - - -@dataclass(frozen=True) -class PayoutStateArtifact: - """Immutable ledger-backed inputs published with one payout generation.""" - - generation: int - source_generation: int - prior_balances_json: str = field(repr=False) - prior_balances_sha256: str - prepared_monotonic: float - - def prior_balances(self) -> list[dict[str, object]]: - value = json.loads(self.prior_balances_json) - if not isinstance(value, list): - raise RuntimeError("published payout artifact is not a balance list") - return value - - -@dataclass(frozen=True) -class AcceptedBlockPayoutTransition: - """Prospective balances for one durable candidate across its landing seam.""" - - block_height: int | None = None - landed: bool = False - preview: tuple[tuple[str, str, str, int], ...] | None = None - published_generation: int | None = None - landed_monotonic: float | None = None - - -@dataclass(frozen=True) -class PayoutStateCandidate: - """Immutable result of payout work prepared outside delivery admission.""" - - base_generation: int - source_generation: int - source_tip_hash: str | None - cause: str - invalidated_monotonic: float - prepared_monotonic: float - accepted_block_hash: str | None = None - accepted_block_preview: tuple[tuple[str, str, str, int], ...] | None = None - accepted_block_withdrawal: bool = False - accepted_block_height: int | None = None - ledger_artifact: PayoutLedgerArtifact | None = field( - default=None, - compare=False, - repr=False, - ) - - -@dataclass(frozen=True) -class PublishedPayoutState: - """The payout snapshot identity to which cached jobs are stamped.""" - - generation: int - source_generation: int - source_tip_hash: str | None - published_monotonic: float - artifact: PayoutStateArtifact | None = field(default=None, repr=False) - - -@dataclass(frozen=True) -class PayoutLedgerArtifact: - """Immutable ledger input prepared independently of a qbit template.""" - - generation: int - payout_state_generation: int - network_difficulty: int - # Durable accepted-share total observed near the snapshot. Informational - # (diagnostics only): validity is scoped to snapshot_anchor_ms, and the - # count cannot be scoped to a clamped anchor because share stamping and - # writer enqueue are not atomic. - accepted_share_count: int - shares_json: Sequence[dict[str, object]] = field(repr=False) - prior_balances: tuple[dict[str, object], ...] = field(repr=False) - prepared_monotonic: float - # The anchor the share snapshot was actually taken at. Bundles built from - # this artifact must declare it as anchor_job_issued_at_ms: an auditor - # replaying qbit_audit_share_window at the declared anchor must reproduce - # exactly these shares, which only holds at the snapshot's own anchor. - # That declaration is also what makes reuse valid while shares keep - # landing: a share stamped after this anchor deterministically belongs to - # the next window (it is never lost), so the artifact stays - # audit-reproducible for as long as it serves. Validity is event-driven - # (the payout-generation and balances fences); wall-clock time only - # enters through the loose anchor ceiling that backstops how far a - # served window may trail the live ledger, never through install age. - snapshot_anchor_ms: int | None = None - # Canonical digest of shares_json. Cached bundles built before this - # artifact was armed may only keep serving re-keyed lookups when their - # own window digest matches; anything else must rebuild from the - # artifact so a fresher window is never shadowed by older cached work. - share_snapshot_sha256: str | None = None - # Canonical digest of prior_balances, memoized at construction. The - # reuse probe hashes the balances on every serving decision and the - # reused-build fences hash them twice more; the tuple is immutable, so - # the O(accounts) canonicalization is paid once here instead. - prior_balances_sha256: str | None = None - # Internal coordination token. A late-visible append can invalidate a - # previously exact anchored window without changing payout generation. - # Artifacts prepared before that append may never be armed again. - append_invalidation_epoch: int = field( - default=0, - compare=False, - repr=False, - ) - # Exposed in-flight scan anchor carried from the synchronous bundle - # build that seeded this artifact (see _expose_inflight_scan_anchor). - # Nothing else exposes this window's anchor between the ledger read and - # the install fence, so the exposure rides here and the install retires - # it once the fence settles either way. - inflight_scan_anchor_token: int | None = field( - default=None, - compare=False, - repr=False, - ) - # Build-path diagnostics only. They do not participate in artifact - # equality or any signed/wire representation. - window_build_mode: str | None = field(default=None, compare=False, repr=False) - window_delta_rows: int = field(default=0, compare=False, repr=False) - window_expired_rows: int = field(default=0, compare=False, repr=False) - window_touched_pages: int = field(default=0, compare=False, repr=False) - window_build_seconds: float = field(default=0.0, compare=False, repr=False) - window_full_rescan_reason: str | None = field( - default=None, - compare=False, - repr=False, - ) - window_self_check_recentered: bool = field( - default=False, - compare=False, - repr=False, - ) - - -@dataclass(frozen=True) -class _IncrementalPayoutArtifactWindow: - """Coordinator-owned exact window plus its canonical materialization.""" - - window: IncrementalShareWindow - shares_json: IncrementalShareJsonSequence = field(repr=False) - share_snapshot_sha256: str - refreshed_monotonic: float - full_rescan_monotonic: float - full_rescan_attempt_monotonic: float - append_invalidation_epoch: int = 0 - # Set when a daemon mirror was adopted from the in-process oracle - # without the daemon being prepared for its digest (issue #207): a - # periodic self-check re-center or repair whose daemon preparation - # was declined. The next advance then answers ``needs_full`` for a - # deliberate coordinator decision, and this names it in the rebuild - # reason instead of reporting a daemon eviction. - daemon_unprepared_reason: str | None = None - - -class _DaemonWindowRebuildRequired(Exception): - """The daemon path cannot advance; rebuild from the full oracle. - - Raised only inside the payout-window materialization: ``reason`` becomes - the full-rescan reason routed through the existing invalidation - machinery, and ``clear_cache`` distinguishes a refuted local state (the - mirror diverged, or the daemon transport vanished under a mirror-typed - cache) from a daemon that merely lost its own copy while the coordinator - mirror stays valid. - """ - - def __init__(self, reason: str, *, clear_cache: bool) -> None: - super().__init__(reason) - self.reason = reason - self.clear_cache = clear_cache - - -@dataclass(frozen=True) -class _PayoutWindowMaterialization: - """One exact share-window materialization and bounded-work metadata.""" - - shares_json: Sequence[dict[str, object]] = field(repr=False) - share_snapshot_sha256: str - snapshot_anchor_ms: int - mode: str - record_count: int - stats: IncrementalWindowAdvanceStats - full_rescan_reason: str | None = None - balance_check_prior_balances: tuple[dict[str, object], ...] | None = field( - default=None, - repr=False, - ) - balance_check_mismatch: bool = False - # An overdue periodic self-check was left for the next build off the - # critical path because this build's cached window was inside - # PRISM_TIP_REFRESH_WINDOW_STALENESS_SECONDS (issue #228). - self_check_deferred: bool = False - # The periodic self-check re-centered an oversized window at the live - # weight (issue #207); a deliberate coordinator decision, flagged apart - # from any daemon reason. - self_check_recentered: bool = False - - -class _ReservedPayoutSource(tuple): - """The source tuple handed back to the caller that reserved it. - - Unpacks, indexes and compares exactly like the plain five-tuple a - capture returns; the difference is the one-shot entitlement it carries - (issue #228). The reserving caller's own preparation acts on the cause - it reserved by presenting this object, whatever other preparations - have done at any generation in between, so no shared bookkeeping and - no pruning rule can take that turn away. The entitlement is consumed - the first time it is honoured; a second presentation inherits the - tuple like any capture. - """ - - def __new__( - cls, - base_generation: int, - source_generation: int, - tip_hash: str | None, - cause: str, - invalidated_monotonic: float, - ) -> "_ReservedPayoutSource": - self = super().__new__( - cls, - ( - base_generation, - source_generation, - tip_hash, - cause, - invalidated_monotonic, - ), - ) - self._entitled = True - return self - - def consume_entitlement(self) -> bool: - """Take the reserver's one honoured turn; caller holds runtime.lock.""" - if self._entitled: - self._entitled = False - return True - return False - - -@dataclass(frozen=True) -class _PayoutWindowBuildAttribution: - """Why one candidate build was requested, for its log line (issue #228). - - Set on the building thread by ``_prepared_payout_state_candidate`` and - read by ``_build_payout_ledger_artifact``, which the coordinator - delegate reaches with a fixed positional signature; the thread-local - carries what that signature cannot. - """ - - # The cause string in the captured source tuple, verbatim. - cause: str - # True when the capture found a cause another preparation had already - # acted on: the tuple was inherited for supersession identity only. - cause_inherited: bool - # "caller_flag", "reserved_cause", "caller_flag+reserved_cause" or - # "none": what, if anything, marked this build as a payout mutation. - force_origin: str - # A routine candidate on the tip-refresh path: an overdue periodic - # self-check is deferred while the cached window is inside the - # accepted staleness bound. - critical_path: bool - - -@dataclass -class PayoutDeliveryAdmission: - admitted: bool - wait_seconds: float - generation: int - published_generation: int - relation: str - delivered: bool = False - - def __bool__(self) -> bool: - return self.admitted - - def mark_delivered(self) -> None: - if not self.admitted: - raise RuntimeError("payout delivery completed without admission") - self.delivered = True - - -class PayoutStateDeliveryGate: - """Order delivery admission around a very short payout publication. - - A publisher first closes admission and drains sends that already crossed - the boundary. It does not own the atomic publication section while that - drain is in progress. Once drained, publication ownership is transferred - to the caller for the generation/cache pointer swap only. - """ - - def __init__(self) -> None: - self._condition = threading.Condition() - self._active_deliveries = 0 - self._publisher_waiting = False - self._mutation_owner: int | None = None - self._mutation_depth = 0 - self._published_generation = 0 - self._priority_generation: int | None = None - self._delivery_blocked = False - - @staticmethod - def _generation_relation(generation: int, published_generation: int) -> str: - if generation < published_generation: - return "stale" - if generation > published_generation: - return "future" - return "current" - - @contextmanager - def delivery(self) -> Iterator[None]: - with self.delivery_cancelable(lambda: False, priority=True) as admission: - if not admission: - raise RuntimeError("uncancelled payout delivery was not admitted") - yield - admission.mark_delivered() - - @contextmanager - def delivery_cancelable( - self, - cancelled: Callable[[], bool], - *, - generation: int | None = None, - priority: bool = False, - poll_seconds: float = PRISM_TIP_REFRESH_ADMISSION_POLL_SECONDS, - ) -> Iterator[PayoutDeliveryAdmission]: - """Admit a delivery unless cancellation wins while mutation owns the gate.""" - - started = time.monotonic() - admitted = False - with self._condition: - if generation is None: - generation = self._published_generation - while True: - if cancelled(): - break - if self._delivery_blocked: - break - if generation < self._published_generation: - break - publication_blocked = ( - self._publisher_waiting or self._mutation_owner is not None - ) - priority_blocked = ( - self._priority_generation is not None - and generation == self._priority_generation - and not priority - ) - if priority_blocked: - # A same-generation job that is not for the published tip - # must not occupy a waiter slot indefinitely. Reject it so - # its caller can rebuild current-tip work; the reserved - # first-delivery lane remains available to priority work. - break - future_blocked = generation > self._published_generation - if ( - not publication_blocked - and not future_blocked - ): - self._active_deliveries += 1 - admitted = True - break - self._condition.wait(timeout=poll_seconds) - published_generation = self._published_generation - relation = self._generation_relation(generation, published_generation) - admission = PayoutDeliveryAdmission( - admitted=admitted, - wait_seconds=max(0.0, time.monotonic() - started), - generation=generation, - published_generation=published_generation, - relation=relation, - ) - try: - yield admission - finally: - if admitted: - with self._condition: - if self._active_deliveries <= 0: - raise RuntimeError("payout delivery gate released without admission") - self._active_deliveries -= 1 - if ( - priority - and admission.delivered - and generation == self._priority_generation - ): - # Keep routine same-generation sends queued until the - # first prioritized current-tip socket delivery exits. - # Privileged synchronization admissions do not consume - # this reservation merely by leaving the gate. - self._priority_generation = None - if self._active_deliveries == 0: - self._condition.notify_all() - elif self._priority_generation is None: - self._condition.notify_all() - - @contextmanager - def publication(self) -> Iterator[None]: - owner = threading.get_ident() - with self._condition: - if self._mutation_owner == owner: - self._mutation_depth += 1 - else: - while self._mutation_owner is not None or self._publisher_waiting: - self._condition.wait() - self._publisher_waiting = True - while self._active_deliveries: - self._condition.wait() - self._mutation_owner = owner - self._mutation_depth = 1 - self._publisher_waiting = False - try: - yield - finally: - with self._condition: - if self._mutation_owner != owner or self._mutation_depth <= 0: - raise RuntimeError("payout mutation gate released by non-owner") - self._mutation_depth -= 1 - if self._mutation_depth == 0: - self._mutation_owner = None - self._condition.notify_all() - - def publish_generation(self, generation: int, *, prioritize_delivery: bool) -> None: - owner = threading.get_ident() - with self._condition: - if self._mutation_owner != owner: - raise RuntimeError("payout generation published outside atomic section") - if generation <= self._published_generation: - raise RuntimeError("payout generation did not advance") - self._published_generation = generation - self._priority_generation = generation if prioritize_delivery else None - self._delivery_blocked = False - - def block_delivery( - self, - mark_blocked: Callable[[], bool] | None = None, - ) -> bool: - """Reject admission until publication, atomically with caller state.""" - - with self._condition: - # A publisher drops the condition while swapping its immutable - # pointer. Wait only for that short section, never for admitted - # socket sends or fanout cancellation. - while self._mutation_owner is not None: - self._condition.wait() - if mark_blocked is not None and not mark_blocked(): - return False - self._delivery_blocked = True - self._condition.notify_all() - return True - - @contextmanager - def mutation(self) -> Iterator[None]: - """Compatibility alias for tests and callers that only need exclusion.""" - - with self.publication(): - yield - - -@dataclass(frozen=True) -class PayoutStateSnapshot: - """Copied identity facts for cross-owner delivery decisions.""" - - generation: int - published: PublishedPayoutState - publication_blocked: bool - - -class PayoutStateRuntime(Protocol): - """Typed port over the coordinator, resolved at call time. - - Every member is looked up on the live coordinator object when used, so - instance monkeypatches (``server._prepared_payout_state_candidate = ...`` - and friends) and coordinator-owned live configuration attributes keep - working exactly as before the extraction. Owner facades route back into - this service; that round trip is deliberate. The legacy P1 field names - (``_payout_state_generation`` through ``payout_artifact_event_counts``) - also resolve here -- coordinator class descriptors route them to this - service's single mutable copy. - """ - - # Cross-domain objects and live configuration attributes. - _job_cache_lock: Any - ledger: Any - lock: Any - stop_event: Any - - def _ensure_job_cache_state(self, *args: Any, **kwargs: Any) -> Any: ... - - def _ensure_shutdown_controller(self, *args: Any, **kwargs: Any) -> Any: ... - - def _record_heartbeat(self, *args: Any, **kwargs: Any) -> Any: ... - - def _record_block_submitter_phase(self, *args: Any, **kwargs: Any) -> Any: ... - - def _block_work_wait_slice(self, *args: Any, **kwargs: Any) -> Any: ... - - def _record_progress_payout_generation(self, *args: Any, **kwargs: Any) -> Any: ... - - def _observe_accepted_block_preview_publication( - self, - *args: Any, - **kwargs: Any, - ) -> Any: ... - - def _writer_operation(self, *args: Any, **kwargs: Any) -> Any: ... - - -class PayoutStateService: - """Sole owner of P1 payout state, artifacts, previews, and fencing.""" - - def __init__( - self, - runtime: PayoutStateRuntime, - *, - shutdown_error: type[BaseException], - now_ms: Callable[[], int], - canonical_json_text_override: Callable[[object], str] = canonical_json_text, - canonical_json_sha256_override: Callable[[object], str] = canonical_json_sha256, - ) -> None: - self._runtime = runtime - # The shutdown exception type stays coordinator-owned until the - # shutdown controller is extracted, the wall-clock stamp and the - # canonical serialization resolve the coordinator module's globals at - # call time (tests patch them there); all are injected so this leaf - # module never imports prism_coordinator. - self._shutdown_error = shutdown_error - self._now_ms = now_ms - self._canonical_json_text = canonical_json_text_override - self._canonical_json_sha256 = canonical_json_sha256_override - self._payout_state_generation = 0 - # Ledger mutations and the ledger reads used to build signed jobs - # share this lock. They may be expensive, but they never block - # delivery of an already-published immutable generation. - self._payout_state_prepare_lock = threading.RLock() - self._payout_state_source: tuple[int, str | None, str, float] = ( - 0, - None, - "startup", - time.monotonic(), - ) - self._published_payout_state = PublishedPayoutState( - generation=self._payout_state_generation, - source_generation=0, - source_tip_hash=None, - published_monotonic=time.monotonic(), - ) - self._payout_ledger_artifact: PayoutLedgerArtifact | None = None - self._payout_ledger_artifact_generation = 0 - # Guarded by _job_cache_lock. Unlike payout generation, this - # advances only when a newly visible row predates an anchored - # share window and makes pre-append work unsafe to publish. - self._payout_ledger_append_invalidation_epoch = 0 - # Guarded by _job_cache_lock. Per advanced epoch, the stamp of the - # row whose durability caused that bump: max(job_issued_at_ms, - # accepted_at_ms), which is exactly the quantity - # _pending_share_predates_anchor tests against an anchor -- a row - # predates an anchor iff both its stamps do, i.e. iff this max does. - # Bounded to the most recent - # PRISM_PAYOUT_APPEND_INVALIDATION_STAMP_HISTORY bumps; - # _payout_append_invalidation_oldest_epoch names the oldest epoch - # still retained (the epoch the next bump would take, while there - # has been none), so a landing whose baseline fell off the end is - # told the question cannot be answered rather than answered from a - # gap. - self._payout_append_invalidation_stamps: dict[int, int] = {} - self._payout_append_invalidation_oldest_epoch = 1 - # Serializes the append-side epoch bump against the landing's - # final epoch-check-and-submitblock boundary. Ordinary share - # commits never touch it: the append side acquires it only for - # rows that predate a live anchor (the rare replay-shaped - # append), and the landing holds it across exactly one RPC, so - # an epoch advance can never slip between the landing's last - # epoch read and the block entering qbitd. Ordered strictly - # outside _job_cache_lock. - self._payout_append_landing_fence_lock = threading.Lock() - # Guarded by _job_cache_lock. The anchors of ledger window walks - # whose results are not yet visible anywhere else, published - # before each read begins. A late-visible append can predate such - # an anchor while no completed window or armed artifact exposes - # one, and the walk's database snapshot may already exclude the - # row; the append-side invalidation must still advance the epoch - # so the walk's result cannot arm or serve. A synchronous bundle - # build keeps its entry exposed past its own return -- until the - # seeded artifact's install fence settles -- because nothing else - # exposes that window's anchor in the meantime. - self._payout_window_inflight_scan_anchors: dict[int, int] = {} - self._payout_window_inflight_scan_anchor_token = 0 - # Guarded by _job_cache_lock. One entry per durable append - # currently committing OUTSIDE the landing fence, holding the - # batch's most predating row stamp (min over rows of - # max(job_issued_at_ms, accepted_at_ms)). The unfenced - # classification is a one-time predicate: an anchor exposed - # after it cannot retroactively fence the in-flight commit, so - # a landing that just exposed its declared anchor drains - # matching entries before its epoch fences arm. The condition - # shares _job_cache_lock and signals every deregistration. - self._payout_unfenced_append_inflight_stamps: dict[int, int] = {} - self._payout_unfenced_append_inflight_token = 0 - self._payout_unfenced_append_drained = threading.Condition( - runtime._job_cache_lock - ) - # Guarded by _job_cache_lock. The highest declared anchor among - # published job windows that nothing else keeps exposed: a - # seedless publication hands its anchor over at the publication - # fence (a build under the reuse kill-switch, or one with nothing - # to seed), and the armed artifact's anchor is folded in before - # the slot can drop or replace it - # (_replace_payout_ledger_artifact_locked). Jobs stamped from such - # a window keep serving it until they retire, and nothing else - # exposes its anchor between publication and the landing's own - # exposure -- so the anchor must stay visible to the append-side - # predates() checks: a replay-shaped append committing in that gap - # has to advance the epoch those jobs' landing fences compare - # against. Monotonic and never retired; ordinary share commits can - # never match it because every job anchor is clamped below the - # pending-commit floor. - self._payout_published_job_window_anchor_ms: int | None = None - # Guarded by _payout_state_prepare_lock. It is independent of the - # published artifact generation: normal generation bumps retag - # the same immutable share window without another ledger walk. - self._incremental_payout_artifact_window: ( - _IncrementalPayoutArtifactWindow | None - ) = None - # Guarded by _payout_state_prepare_lock. Explicit append-side - # invalidations leave a one-shot diagnostic for the oracle build - # that replaces the discarded incremental window. - self._incremental_payout_artifact_window_invalidation_reason: ( - str | None - ) = None - # Guarded by _payout_state_prepare_lock. A periodic balance oracle - # mismatch keeps the stale published digest off the reuse path - # until a later publication installs a different digest. - self._payout_prior_balances_reuse_invalidated_sha256: str | None = None - self._payout_artifact_executor_lock = threading.Lock() - self._payout_artifact_executor: ThreadPoolExecutor | None = None - self._payout_artifact_future: Future[None] | None = None - self._payout_artifact_requested: tuple[int, int] | None = None - self._payout_artifact_requested_bypass = False - self._payout_artifact_executor_shutdown = False - # Guarded by _payout_artifact_executor_lock. Stamped by every - # accepted preparation request so fence-failure re-arms measure - # their debounce from the newest scheduled rebuild, whichever - # path requested it. - self._payout_artifact_last_schedule_monotonic: float | None = None - # Guarded by _payout_artifact_executor_lock. - self._payout_artifact_rearm_backoff = 1 - # Orders reconciliation mutations against final job-delivery - # admission while preserving parallel sends to different miners. - self._payout_state_delivery_gate = PayoutStateDeliveryGate() - # Keep durable payout-balance transitions serialized even when their - # preparation intentionally runs outside the delivery gate. The - # accepted-block path can hold this lock across expensive writes - # without preventing replacement jobs from reaching miners. - self._payout_balance_mutation_lock = threading.RLock() - self._accepted_block_payout_preview_condition = threading.Condition() - # Durable replay registers an unlanded transition. Once its block - # is active, reconciliation is barred and child/descendant builders - # wait for or consume its verified prospective balance snapshot. - self._accepted_block_payout_previews: dict[ - str, AcceptedBlockPayoutTransition - ] = {} - # A landed transition can be withdrawn before durability catches - # up. Keep a height-bearing tombstone so a new exact/descendant - # build cannot fall through to database balances that omit it in - # the gap before durable retry re-registers the candidate. - self._invalidated_accepted_block_payout_previews: dict[ - str, int | None - ] = {} - self._payout_state_metrics_lock = threading.Lock() - self.payout_state_histograms = { - name: { - "buckets": { - bucket: 0 for bucket in PRISM_PAYOUT_SECONDS_BUCKETS - }, - "sum": 0.0, - "count": 0, - } - for name in ("preparation", "publish", "first_delivery") - } - self.payout_gate_wait_histograms = { - relation: { - "buckets": { - bucket: 0 for bucket in PRISM_PAYOUT_SECONDS_BUCKETS - }, - "sum": 0.0, - "count": 0, - } - for relation in PRISM_PAYOUT_DELIVERY_GENERATIONS - } - self.payout_state_candidates_discarded = 0 - # Guarded by _payout_state_metrics_lock. Where builds spend their - # time, by the closed phase set and the closed outcome set (issue - # #228). - self.payout_window_build_phase_histograms = { - (phase, outcome): { - "buckets": { - bucket: 0 for bucket in PRISM_PAYOUT_SECONDS_BUCKETS - }, - "sum": 0.0, - "count": 0, - } - for phase in PRISM_PAYOUT_WINDOW_BUILD_PHASES - for outcome in PRISM_PAYOUT_WINDOW_BUILD_OUTCOMES - } - # Per-thread build state: the attribution a candidate preparation - # hands its build, and the phase accumulator of the build in - # progress on this thread. - self._payout_window_build_context = threading.local() - # Guarded by runtime.lock. The highest source generation any - # candidate preparation has acted on; it never moves backwards. A - # capture at that generation or below inherits the tuple for - # supersession identity only, so the cause a withdrawal reserved is - # acted on once (issue #228). The reserver's own turn does not live - # here: it travels in the _ReservedPayoutSource its reservation - # returned. The tuple itself is never cleared, because publication - # supersession and the finalizer's pending-cause read depend on it - # staying readable. - self._payout_source_cause_high_water: int | None = None - # Guarded by runtime.lock. A critical-path build deferred an overdue - # periodic self-check; the next publication requests a background - # preparation so the check runs off the critical path promptly. - self._payout_self_check_owed = False - self._payout_first_delivery_pending: tuple[int, float] | None = None - self._payout_state_publication_blocked = False - # Artifact lifecycle observability (the 2026-07-29 incident was - # invisible precisely here). Guarded by - # _payout_artifact_executor_lock. - self.payout_artifact_event_counts = { - event: 0 - for event in ( - "built", - "build_aborted", - "debounced", - "incremental", - "full_rescan", - "self_check_match", - "self_check_mismatch", - "self_check_failed", - "balance_check_mismatch", - "found_block_cached", - "tip_refresh_cached", - "installed", - "refreshed", - "already_current", - "discarded", - "born_expired", - "rearm_scheduled", - ) - } - - def snapshot(self) -> PayoutStateSnapshot: - """Copied cross-owner identity facts (memo rule: no shared mutables).""" - with self._runtime.lock: - return PayoutStateSnapshot( - generation=int(self._payout_state_generation), - published=self._published_payout_state, - publication_blocked=bool(self._payout_state_publication_blocked), - ) - - @staticmethod - def _incremental_window_records_supported(records: Sequence[object]) -> bool: - """Whether a ledger snapshot exposes the append-only window contract.""" - - required = ( - "share_seq", - "share_id", - "share_difficulty", - "job_issued_at_ms", - "accepted_at_ms", - "to_prism_json", - ) - return bool(records) and all( - all(hasattr(record, attribute) for attribute in required) - for record in records - ) - - def _observe_payout_window_full_rescan( - self, - reason: str, - path: str, - seconds: float, - ) -> None: - """Record one executed full oracle rescan in the #224 owner. - - Only the two places that actually run the whole-window oracle call - this: ``_full_payout_window_materialization`` and the periodic - self-check inside the incremental materialization. A prior-balances - reread (``force_prior_balances_read``) never reaches here. - """ - ensure_accepted_preview_telemetry( - self._runtime - ).observe_payout_window_full_rescan(reason, path, seconds) - - def _full_payout_window_materialization( - self, - *, - snapshot_anchor_ms: int, - snapshot_window_weight: int, - reason: str, - observed_monotonic: float, - append_invalidation_epoch: int, - bypass_build_interval: bool = False, - ) -> _PayoutWindowMaterialization: - """Run the exact ledger oracle and atomically replace cached pages.""" - rescan_started = time.monotonic() - # The path is chosen by which pipeline folded the window; the ledger - # read and any daemon decline that degrades to the in-process fold - # belong to the rescan either way. Recorded on every exit so a - # rescan that dies with the ledger still owns its wall-clock. - rescan_path = FULL_RESCAN_PATH_IN_PROCESS - try: - materialized, rescan_path = self._full_payout_window_oracle( - snapshot_anchor_ms=snapshot_anchor_ms, - snapshot_window_weight=snapshot_window_weight, - reason=reason, - observed_monotonic=observed_monotonic, - append_invalidation_epoch=append_invalidation_epoch, - bypass_build_interval=bypass_build_interval, - ) - finally: - self._observe_payout_window_full_rescan( - reason, - rescan_path, - time.monotonic() - rescan_started, - ) - return materialized - - def _full_payout_window_oracle( - self, - *, - snapshot_anchor_ms: int, - snapshot_window_weight: int, - reason: str, - observed_monotonic: float, - append_invalidation_epoch: int, - bypass_build_interval: bool = False, - ) -> tuple[_PayoutWindowMaterialization, str]: - """The oracle read plus fold; returns the window and the fold path.""" - runtime = self._runtime - - # Timed in a finally: a read that dies with the ledger is exactly - # the slow read the phase family exists to attribute. - read_started = time.monotonic() - try: - records = list( - runtime.ledger.snapshot_at_job_issue( - snapshot_anchor_ms, - window_weight=snapshot_window_weight, - ) - ) - finally: - self._note_window_build_phase( - "ledger_read", - time.monotonic() - read_started, - ) - zero_stats = IncrementalWindowAdvanceStats( - added_rows=0, - expired_rows=0, - touched_pages=0, - ) - if not runtime._incremental_window_records_supported(records): - # Small test/custom ledgers predating the delta contract retain the - # legacy full-read behavior. Production ledger records always - # expose the complete AcceptedShareRecord shape. - runtime._incremental_payout_artifact_window = None - conversion_started = time.monotonic() - shares_json = tuple(record.to_prism_json() for record in records) - # The digest hook prefers a sequence's own canonical digest; - # the view streams the freshly converted tuple record-batch by - # record-batch instead of one whole-window json.dumps. - digest = self._canonical_json_sha256( - share_array_json_view(shares_json) - ) - self._note_window_build_phase( - "record_conversion", - time.monotonic() - conversion_started, - ) - return ( - _PayoutWindowMaterialization( - shares_json=shares_json, - share_snapshot_sha256=digest, - snapshot_anchor_ms=int(snapshot_anchor_ms), - mode="full_rescan", - record_count=len(records), - stats=zero_stats, - full_rescan_reason=reason, - ), - FULL_RESCAN_PATH_IN_PROCESS, - ) - - if self._window_pipeline_rust_enabled(): - materialized = self._daemon_full_window_materialization( - records=records, - snapshot_anchor_ms=snapshot_anchor_ms, - snapshot_window_weight=snapshot_window_weight, - reason=reason, - observed_monotonic=observed_monotonic, - append_invalidation_epoch=append_invalidation_epoch, - bypass_build_interval=bypass_build_interval, - ) - if materialized is not None: - # The daemon holds the window as bytes now; the ledger rows - # are finished with. Drop them in bounded slices so their - # release is not one refcount cascade under the GIL (#236). - release_share_list_incrementally(records) - return materialized, FULL_RESCAN_PATH_DAEMON - # Any daemon outcome that cannot produce a window here -- an - # anomaly (already retired), the transport disabled, a daemon - # another build owns, a snapshot the fold must reject, or a - # value outside the daemon's declared integer widths (the - # daemon declines it and stays healthy) -- degrades to the - # in-process pipeline below for this materialization, which is - # also where an invalid snapshot raises its authoritative error - # and where unbounded integers fold the out-of-range window. - - conversion_started = time.monotonic() - window = IncrementalShareWindow.from_full_snapshot( - records, - anchor_job_issued_at_ms=snapshot_anchor_ms, - window_weight=snapshot_window_weight, - ) - shares_json = window.json_records() - digest = self._canonical_json_sha256(shares_json) - self._note_window_build_phase( - "record_conversion", - time.monotonic() - conversion_started, - ) - runtime._incremental_payout_artifact_window = ( - _IncrementalPayoutArtifactWindow( - window=window, - shares_json=shares_json, - share_snapshot_sha256=digest, - refreshed_monotonic=observed_monotonic, - full_rescan_monotonic=observed_monotonic, - full_rescan_attempt_monotonic=observed_monotonic, - append_invalidation_epoch=append_invalidation_epoch, - ) - ) - return ( - _PayoutWindowMaterialization( - shares_json=shares_json, - share_snapshot_sha256=digest, - snapshot_anchor_ms=int(snapshot_anchor_ms), - mode="full_rescan", - record_count=len(shares_json), - stats=zero_stats, - full_rescan_reason=reason, - ), - FULL_RESCAN_PATH_IN_PROCESS, - ) - - def _window_pipeline_rust_enabled(self) -> bool: - """Whether window materialization routes through the --serve daemon. - - Fail-closed at the env layer (a non-boolean PRISM_WINDOW_PIPELINE_RUST - refuses startup via SystemExit); inert -- not an error -- when the - daemon transport itself is disabled or when the runtime exposes no - prepare port (embedders and slim test coordinators). - """ - if not env_window_pipeline_rust(): - return False - if not env_bool("PRISM_BUILDER_SERVE", "1"): - return False - return callable(getattr(self._runtime, "prepare_payout_window", None)) - - def _daemon_full_window_materialization( - self, - *, - records: Sequence[Any], - snapshot_anchor_ms: int, - snapshot_window_weight: int, - reason: str, - observed_monotonic: float, - append_invalidation_epoch: int, - bypass_build_interval: bool = False, - ) -> _PayoutWindowMaterialization | None: - """Fold one full snapshot through the daemon; None means fold here. - - The daemon returns the canonical digest plus the full canonical - items stream; the mirror construction reconciles those bytes with - both the digest and the record count before anything downstream may - consume them. Every non-prepared outcome -- a daemon anomaly (already - retired by the transport), a daemon another build owns, the fold - rejecting the snapshot (the in-process oracle must raise the - authoritative error), an integer outside the daemon's declared - widths (``out_of_range``: the daemon is kept, the window is folded - here), or a refuted byte mirror -- degrades to the in-process - pipeline for this materialization. - """ - runtime = self._runtime - prepare = getattr(runtime, "prepare_payout_window", None) - if not callable(prepare): - return None - conversion_started = time.monotonic() - records_json = [record.to_prism_json() for record in records] - self._note_window_build_phase( - "record_conversion", - time.monotonic() - conversion_started, - ) - daemon_started = time.monotonic() - try: - outcome = prepare( - mode="full", - records_json=records_json, - anchor_job_issued_at_ms=int(snapshot_anchor_ms), - append_invalidation_epoch=int(append_invalidation_epoch), - window_weight=int(snapshot_window_weight), - page_size=DEFAULT_INCREMENTAL_SHARE_WINDOW_PAGE_SIZE, - # An accepted-block preview publishes on the critical path; - # it folds in-process rather than queueing behind another - # build. - wait_for_daemon=not bypass_build_interval, - ) - finally: - self._note_window_build_phase( - "daemon_prepare", - time.monotonic() - daemon_started, - ) - # The converted dicts were consumed by the request; drop them in - # bounded slices rather than in one refcount cascade (#236). - release_share_list_incrementally(records_json) - if outcome is None or outcome.status != "prepared": - return None - conversion_started = time.monotonic() - try: - mirror = DaemonShareWindowMirror.from_full_items( - anchor_job_issued_at_ms=int(snapshot_anchor_ms), - window_weight=int(snapshot_window_weight), - page_size=DEFAULT_INCREMENTAL_SHARE_WINDOW_PAGE_SIZE, - record_count=int(outcome.record_count), - canonical_items=outcome.window_items or b"", - share_snapshot_sha256=outcome.share_snapshot_sha256 or "", - ) - except DaemonWindowMirrorDivergence: - runtime._note_window_mirror_divergence() - return None - shares_json = mirror.json_records() - self._note_window_build_phase( - "record_conversion", - time.monotonic() - conversion_started, - ) - runtime._incremental_payout_artifact_window = ( - _IncrementalPayoutArtifactWindow( - window=mirror, - shares_json=shares_json, - share_snapshot_sha256=mirror.share_snapshot_sha256, - refreshed_monotonic=observed_monotonic, - full_rescan_monotonic=observed_monotonic, - full_rescan_attempt_monotonic=observed_monotonic, - append_invalidation_epoch=append_invalidation_epoch, - ) - ) - return _PayoutWindowMaterialization( - shares_json=shares_json, - share_snapshot_sha256=mirror.share_snapshot_sha256, - snapshot_anchor_ms=int(snapshot_anchor_ms), - mode="full_rescan", - record_count=mirror.record_count, - stats=IncrementalWindowAdvanceStats( - added_rows=0, - expired_rows=0, - touched_pages=0, - ), - full_rescan_reason=reason, - ) - - def _daemon_advance_window( - self, - mirror: DaemonShareWindowMirror, - delta_records: Sequence[Any], - *, - snapshot_anchor_ms: int, - append_invalidation_epoch: int, - bypass_build_interval: bool = False, - unprepared_reason: str | None = None, - ) -> tuple[DaemonShareWindowMirror, IncrementalWindowAdvanceStats]: - """Advance the daemon-prepared window by one append-only delta. - - Outcome mapping, mirroring the in-process contract exactly: - ``fallback`` raises IncrementalWindowFallback (same handler, same - ``incremental_invariant_failed`` full rescan, daemon not retired); - ``needs_full`` and every anomaly raise _DaemonWindowRebuildRequired - so the caller rebuilds through the full oracle path. ``busy`` is a - healthy daemon another build owns: it rebuilds too, because opaque - mirror bytes cannot advance in-process, but it keeps its own reason - so a contended daemon is never mistaken for an unavailable one. - ``out_of_range`` is a healthy daemon declining a delta outside its - declared integer widths: the rebuild goes through the full path, - where the daemon declines the snapshot the same way and the - in-process fold takes the window -- reasoned - ``window_value_out_of_range``, the daemon untouched. - - ``unprepared_reason`` is the cached entry's own account of why the - daemon may not hold this base: a periodic self-check adopted the - mirror from the in-process oracle and its daemon preparation was - declined (issue #207). A ``needs_full`` for such a mirror is that - deliberate decision surfacing, and is reasoned with it rather than - reported as daemon state loss. - """ - runtime = self._runtime - prepare = getattr(runtime, "prepare_payout_window", None) - if not callable(prepare) or not self._window_pipeline_rust_enabled(): - # The switch or transport flipped between the cache-mode check - # and this advance; opaque bytes cannot advance in-process. - raise _DaemonWindowRebuildRequired( - "window_pipeline_mode_changed", - clear_cache=True, - ) - conversion_started = time.monotonic() - records_json = [record.to_prism_json() for record in delta_records] - self._note_window_build_phase( - "record_conversion", - time.monotonic() - conversion_started, - ) - daemon_started = time.monotonic() - try: - outcome = prepare( - mode="advance", - records_json=records_json, - anchor_job_issued_at_ms=int(snapshot_anchor_ms), - append_invalidation_epoch=int(append_invalidation_epoch), - base_digest=mirror.share_snapshot_sha256, - wait_for_daemon=not bypass_build_interval, - ) - finally: - self._note_window_build_phase( - "daemon_prepare", - time.monotonic() - daemon_started, - ) - if outcome is None: - raise _DaemonWindowRebuildRequired( - "window_daemon_unavailable", - clear_cache=True, - ) - if outcome.status == "busy": - # Contention, not sickness: the daemon is untouched and still - # holds this base, so the oracle rebuild below re-prepares it - # without the cache being thrown away on a healthy daemon's - # behalf. - raise _DaemonWindowRebuildRequired( - "window_daemon_busy", - clear_cache=False, - ) - if outcome.status == "needs_full": - # The daemon no longer holds the base window. A mirror adopted - # without a daemon preparation names its own reason; anything - # else is a respawn or eviction. Either way the coordinator - # mirror is still digest-verified, so only the daemon needs - # re-seeding via the full path. - raise _DaemonWindowRebuildRequired( - unprepared_reason or "window_daemon_state_lost", - clear_cache=False, - ) - if outcome.status == "fallback": - raise IncrementalWindowFallback( - outcome.error or "daemon advance invariant failed" - ) - if outcome.status == "out_of_range": - # The mirror is opaque daemon bytes and the daemon has just - # declared it cannot hold this window, so the cache is useless - # here; the full path re-asks once (declined again) and folds - # in-process. Not an anomaly: the daemon stays registered. - raise _DaemonWindowRebuildRequired( - "window_value_out_of_range", - clear_cache=True, - ) - if outcome.status != "prepared": - raise _DaemonWindowRebuildRequired( - "window_daemon_unavailable", - clear_cache=True, - ) - conversion_started = time.monotonic() - try: - advanced = mirror.advanced( - anchor_job_issued_at_ms=int(snapshot_anchor_ms), - record_count=int(outcome.record_count), - retained_drop_bytes=int(outcome.retained_drop_bytes), - appended_items=outcome.appended_items, - share_snapshot_sha256=outcome.share_snapshot_sha256 or "", - ) - except DaemonWindowMirrorDivergence: - runtime._note_window_mirror_divergence() - raise _DaemonWindowRebuildRequired( - "window_mirror_divergence", - clear_cache=True, - ) from None - finally: - self._note_window_build_phase( - "record_conversion", - time.monotonic() - conversion_started, - ) - return advanced, IncrementalWindowAdvanceStats( - added_rows=int(outcome.added_rows), - expired_rows=int(outcome.expired_rows), - touched_pages=int(outcome.touched_pages), - ) - - def _daemon_adopted_window( - self, - *, - full_window: IncrementalShareWindow, - shares_json: IncrementalShareJsonSequence, - digest: str, - advanced_window: IncrementalShareWindow | DaemonShareWindowMirror, - snapshot_anchor_ms: int, - append_invalidation_epoch: int, - ) -> tuple[DaemonShareWindowMirror, str | None]: - """Adopt the self-check oracle's window as a mirror the daemon holds. - - Issue #207: the periodic self-check used to install a mirror built - from the oracle's own pages without telling the daemon, so after an - oversized re-center (or a repair after a mismatch or an outage) the - next advance answered ``needs_full`` and paid a second full - ``snapshot_at_job_issue`` under the writer lock, recorded as - ``window_daemon_state_lost``. When the daemon already holds the - adopted digest (its own advance produced it) the mirror is taken - from the oracle pages as before. Otherwise the daemon is prepared - for the adopted window here, once, so the next build advances - incrementally. If that preparation is declined -- busy, unavailable, - out of range, or a digest the in-process oracle refutes -- the - oracle mirror is installed anyway and tagged with the daemon's own - outcome reason (``window_daemon_busy``, ``window_daemon_unavailable``, - ``window_value_out_of_range`` or ``window_mirror_divergence``, all - members of the closed rescan-reason vocabulary), so the - ``needs_full`` it will draw is reported as what declined the - adoption and never as daemon state loss. - """ - runtime = self._runtime - - def mirror_from_oracle_pages() -> DaemonShareWindowMirror: - # The bytes come from the oracle's own pages (already encoded); - # the digest is a function of exactly the retained records. - return DaemonShareWindowMirror( - anchor_job_issued_at_ms=full_window.anchor_job_issued_at_ms, - window_weight=full_window.window_weight, - page_size=full_window.page_size, - record_count=len(shares_json), - canonical_items=b",".join( - page.canonical_json_items - for page in full_window.pages - if page.canonical_json_items - ), - share_snapshot_sha256=digest, - ) - - if ( - isinstance(advanced_window, DaemonShareWindowMirror) - and advanced_window.share_snapshot_sha256 == digest - ): - return mirror_from_oracle_pages(), None - prepare = getattr(runtime, "prepare_payout_window", None) - if not callable(prepare): - return mirror_from_oracle_pages(), "window_daemon_unavailable" - conversion_started = time.monotonic() - records_json = [record.to_prism_json() for record in full_window.records()] - self._note_window_build_phase( - "record_conversion", - time.monotonic() - conversion_started, - ) - daemon_started = time.monotonic() - try: - outcome = prepare( - mode="full", - records_json=records_json, - anchor_job_issued_at_ms=int(snapshot_anchor_ms), - append_invalidation_epoch=int(append_invalidation_epoch), - window_weight=int(full_window.window_weight), - page_size=int(full_window.page_size), - # The self-check only runs off the found-block critical path. - wait_for_daemon=True, - ) - finally: - self._note_window_build_phase( - "daemon_prepare", - time.monotonic() - daemon_started, - ) - release_share_list_incrementally(records_json) - if outcome is None: - return mirror_from_oracle_pages(), "window_daemon_unavailable" - if outcome.status == "busy": - return mirror_from_oracle_pages(), "window_daemon_busy" - if outcome.status == "out_of_range": - return mirror_from_oracle_pages(), "window_value_out_of_range" - if outcome.status != "prepared": - return mirror_from_oracle_pages(), "window_daemon_unavailable" - if outcome.share_snapshot_sha256 != digest: - # The daemon folded the same records to a different digest. - # The in-process oracle is authoritative here; surface the - # disagreement and keep its window. - runtime._record_payout_artifact_event("window_mirror_divergence") - return mirror_from_oracle_pages(), "window_mirror_divergence" - conversion_started = time.monotonic() - try: - mirror = DaemonShareWindowMirror.from_full_items( - anchor_job_issued_at_ms=int(snapshot_anchor_ms), - window_weight=int(full_window.window_weight), - page_size=int(full_window.page_size), - record_count=int(outcome.record_count), - canonical_items=outcome.window_items or b"", - share_snapshot_sha256=outcome.share_snapshot_sha256 or "", - ) - except DaemonWindowMirrorDivergence: - runtime._record_payout_artifact_event("window_mirror_divergence") - return mirror_from_oracle_pages(), "window_mirror_divergence" - finally: - self._note_window_build_phase( - "record_conversion", - time.monotonic() - conversion_started, - ) - return mirror, None - - def _incremental_payout_window_materialization( - self, - *, - snapshot_anchor_ms: int, - snapshot_window_weight: int, - force_full_rescan: bool, - bypass_build_interval: bool, - append_invalidation_epoch: int, - reused_prior_balances_sha256: str | None = None, - ) -> _PayoutWindowMaterialization: - """Return an exact window using debounce, delta folding, or the oracle. - - Must run under ``_payout_state_prepare_lock``. The cache replacement - happens only after a complete materialization, except an explicit - forced invalidation clears it before the oracle so a failed reorg - check cannot later resume from unverified append-only state. The - append-epoch check below is what keeps a late-visible append from - being advanced over; it does not depend on the request flag. - """ - runtime = self._runtime - - observed = time.monotonic() - cached = runtime._incremental_payout_artifact_window - full_reason: str | None = None - if force_full_rescan: - runtime._incremental_payout_artifact_window = None - runtime._incremental_payout_artifact_window_invalidation_reason = None - cached = None - full_reason = "reconcile_invalidation" - elif cached is None: - full_reason = ( - runtime._incremental_payout_artifact_window_invalidation_reason - or "cold_start" - ) - elif cached.append_invalidation_epoch != append_invalidation_epoch: - # Append invalidation is published without waiting for this - # preparation lock. A build that acquires the lock first must not - # advance the pre-append cache under the new epoch. - runtime._incremental_payout_artifact_window = None - cached = None - full_reason = "late_visible_append" - elif isinstance( - cached.window, DaemonShareWindowMirror - ) and not self._window_pipeline_rust_enabled(): - # The Rust switch went off while a daemon mirror was cached; the - # in-process pipeline cannot advance opaque bytes, so rebuild - # from the oracle. The converse (a Python window cached while - # the switch is on) deliberately keeps advancing in Python -- - # that is the degraded state while the daemon is unavailable -- - # and the periodic runtime-check converts it back. - runtime._incremental_payout_artifact_window = None - runtime._incremental_payout_artifact_window_invalidation_reason = None - cached = None - full_reason = "window_pipeline_mode_changed" - elif not _snapshot_window_weight_within_band( - int(cached.window.window_weight), int(snapshot_window_weight) - ): - runtime._incremental_payout_artifact_window = None - runtime._incremental_payout_artifact_window_invalidation_reason = None - cached = None - full_reason = "snapshot_window_weight_out_of_band" - elif snapshot_anchor_ms < cached.window.anchor_job_issued_at_ms: - runtime._incremental_payout_artifact_window = None - runtime._incremental_payout_artifact_window_invalidation_reason = None - cached = None - full_reason = "anchor_regression" - - if cached is None: - materialized = runtime._full_payout_window_materialization( - snapshot_anchor_ms=snapshot_anchor_ms, - snapshot_window_weight=snapshot_window_weight, - reason=full_reason or "cache_invalidated", - observed_monotonic=observed, - append_invalidation_epoch=append_invalidation_epoch, - bypass_build_interval=bypass_build_interval, - ) - runtime._incremental_payout_artifact_window_invalidation_reason = None - return materialized - - min_interval = runtime._payout_artifact_min_build_interval_seconds() - full_rescan_seconds = float( - getattr( - runtime, - "payout_artifact_full_rescan_seconds", - DEFAULT_PRISM_PAYOUT_ARTIFACT_FULL_RESCAN_SECONDS, - ) - ) - # Bypass builds (accepted-block previews) refresh - # refreshed_monotonic on every block yet never run the periodic - # oracle themselves, so a sustained sub-interval block cadence - # would otherwise send every routine build back from this - # debounce and postpone the configured runtime-check indefinitely. - # An overdue runtime-check disarms the debounce for routine builds; - # the urgent preview path stays debounce-free and oracle-free - # either way. - self_check_overdue = ( - observed - cached.full_rescan_attempt_monotonic - >= full_rescan_seconds - ) - # Issue #228: a routine candidate on the tip-refresh path must not - # pay the oracle while a valid recent window exists. If its cached - # window is inside the accepted staleness bound, an overdue - # self-check is left for a build off that path. The deferral is - # bounded in wall-clock terms against the last self-check attempt, - # not against the last build: once the check is a full staleness - # bound past due it runs here whatever the build frequency, so - # candidates arriving faster than the bound cannot defer it - # forever. A deferral also leaves the check owed, and the next - # publication requests a background preparation that normally runs - # it off the critical path before that grace expires. - attribution = getattr( - self._payout_window_build_context, "attribution", None - ) - critical_path = attribution is not None and attribution.critical_path - cached_window_fresh = ( - int(snapshot_anchor_ms) - int(cached.window.anchor_job_issued_at_ms) - <= PRISM_TIP_REFRESH_WINDOW_STALENESS_SECONDS * 1000.0 - ) - self_check_grace_exhausted = ( - observed - cached.full_rescan_attempt_monotonic - >= full_rescan_seconds + PRISM_TIP_REFRESH_WINDOW_STALENESS_SECONDS - ) - self_check_deferred = bool( - self_check_overdue - and not bypass_build_interval - and critical_path - and cached_window_fresh - and not self_check_grace_exhausted - ) - run_self_check = ( - not bypass_build_interval - and self_check_overdue - and not self_check_deferred - ) - if ( - not bypass_build_interval - and not self_check_overdue - and observed - cached.refreshed_monotonic < min_interval - ): - return _PayoutWindowMaterialization( - shares_json=cached.shares_json, - share_snapshot_sha256=cached.share_snapshot_sha256, - snapshot_anchor_ms=cached.window.anchor_job_issued_at_ms, - mode="debounced", - record_count=cached.window.record_count, - stats=IncrementalWindowAdvanceStats(0, 0, 0), - ) - - delta_reader = getattr(runtime.ledger, "snapshot_between_job_issues", None) - if not callable(delta_reader): - return runtime._full_payout_window_materialization( - snapshot_anchor_ms=snapshot_anchor_ms, - snapshot_window_weight=snapshot_window_weight, - reason="delta_api_unavailable", - observed_monotonic=observed, - append_invalidation_epoch=append_invalidation_epoch, - bypass_build_interval=bypass_build_interval, - ) - - try: - delta_started = time.monotonic() - try: - delta_records = list( - delta_reader( - cached.window.anchor_job_issued_at_ms, - snapshot_anchor_ms, - ) - ) - finally: - self._note_window_build_phase( - "ledger_read", - time.monotonic() - delta_started, - ) - if isinstance(cached.window, DaemonShareWindowMirror): - advanced_window, stats = self._daemon_advance_window( - cached.window, - delta_records, - snapshot_anchor_ms=snapshot_anchor_ms, - append_invalidation_epoch=append_invalidation_epoch, - bypass_build_interval=bypass_build_interval, - unprepared_reason=cached.daemon_unprepared_reason, - ) - else: - advance_started = time.monotonic() - try: - advanced_window, stats = cached.window.advance( - delta_records, - anchor_job_issued_at_ms=snapshot_anchor_ms, - ) - finally: - self._note_window_build_phase( - "record_conversion", - time.monotonic() - advance_started, - ) - except _DaemonWindowRebuildRequired as rebuild: - # Daemon window loss (respawn/eviction), a daemon anomaly, a - # daemon another build owns, or a refuted byte mirror. The cached - # mirror stays valid when the daemon merely lost state or was - # busy; the oracle rebuild below re-prepares it. - if rebuild.clear_cache: - runtime._incremental_payout_artifact_window = None - return runtime._full_payout_window_materialization( - snapshot_anchor_ms=snapshot_anchor_ms, - snapshot_window_weight=snapshot_window_weight, - reason=rebuild.reason, - observed_monotonic=observed, - append_invalidation_epoch=append_invalidation_epoch, - bypass_build_interval=bypass_build_interval, - ) - except (IncrementalWindowFallback, ValueError): - runtime._incremental_payout_artifact_window = None - return runtime._full_payout_window_materialization( - snapshot_anchor_ms=snapshot_anchor_ms, - snapshot_window_weight=snapshot_window_weight, - reason="incremental_invariant_failed", - observed_monotonic=observed, - append_invalidation_epoch=append_invalidation_epoch, - bypass_build_interval=bypass_build_interval, - ) - - if delta_records or stats.expired_rows: - # Each window page owns its immutable JSON representation. Page - # identities survive normal advancement, so this view and its - # digest convert only append/head boundary pages; retained share - # records are never flattened or re-encoded here. - conversion_started = time.monotonic() - shares_json = advanced_window.json_records() - digest = self._canonical_json_sha256(shares_json) - self._note_window_build_phase( - "record_conversion", - time.monotonic() - conversion_started, - ) - else: - shares_json = cached.shares_json - digest = cached.share_snapshot_sha256 - - advanced = _IncrementalPayoutArtifactWindow( - window=advanced_window, - shares_json=shares_json, - share_snapshot_sha256=digest, - refreshed_monotonic=observed, - full_rescan_monotonic=cached.full_rescan_monotonic, - full_rescan_attempt_monotonic=( - cached.full_rescan_attempt_monotonic - ), - append_invalidation_epoch=append_invalidation_epoch, - ) - mode = "incremental" - check_reason: str | None = None - self_check_recentered = False - balance_check_prior_balances: tuple[dict[str, object], ...] | None = None - balance_check_mismatch = False - if run_self_check: - # The self-check is a whole-window oracle read at the cached - # weight; it is recorded as a full rescan under its check reason - # (the periodic family) and always folds in-process. - self_check_started = time.monotonic() - try: - # Within the tolerance band the cached and live snapshot - # weights legitimately differ. The match verdict must - # compare like-for-like at the cached weight (a live-weight - # comparison would turn ordinary difficulty drift into - # systematic false self_check_mismatch), and on ordinary - # drift the adopted window must ALSO keep the cached weight: - # adopting a digest the daemon never prepared would answer - # needs_full on the next advance and pay a second full - # oracle read under the writer lock at every runtime-check. - # Re-centering is therefore one-sided and thresholded -- - # only when the cached weight sits more than 25% above the - # live snapshot weight, the oversize direction nothing else - # bounds (a falling difficulty could otherwise pin an - # up-to-2x oversized oracle scan); the undersize direction - # is safety-bounded by the band floor and re-centered by - # every forced rescan. The oversize case makes the cached - # weight the larger one, so the single oracle read at the - # cached weight covers both exact cutoffs. - cached_window_weight = int(advanced_window.window_weight) - live_window_weight = int(snapshot_window_weight) - recenter_oversized = ( - cached_window_weight * 4 > live_window_weight * 5 - ) - oracle_started = time.monotonic() - try: - full_records = list( - runtime.ledger.snapshot_at_job_issue( - snapshot_anchor_ms, - window_weight=cached_window_weight, - ) - ) - finally: - self._note_window_build_phase( - "ledger_read", - time.monotonic() - oracle_started, - ) - conversion_started = time.monotonic() - comparison_window = IncrementalShareWindow.from_full_snapshot( - full_records, - anchor_job_issued_at_ms=snapshot_anchor_ms, - window_weight=cached_window_weight, - ) - if recenter_oversized: - full_window = IncrementalShareWindow.from_full_snapshot( - full_records, - anchor_job_issued_at_ms=snapshot_anchor_ms, - window_weight=live_window_weight, - ) - else: - full_window = comparison_window - shares_json = full_window.json_records() - digest = self._canonical_json_sha256(shares_json) - if isinstance(advanced_window, DaemonShareWindowMirror): - # The mirror holds canonical bytes, not records; digest - # equality is the same comparison, since the canonical - # digest is a function of exactly the retained records. - comparison_digest = ( - digest - if full_window is comparison_window - else self._canonical_json_sha256( - comparison_window.json_records() - ) - ) - matched = ( - comparison_digest - == advanced_window.share_snapshot_sha256 - ) - else: - matched = ( - comparison_window.records() - == advanced_window.records() - ) - self._note_window_build_phase( - "record_conversion", - time.monotonic() - conversion_started, - ) - except Exception: - # The already-validated delta remains usable. Space failed - # oracle attempts by the configured runtime-check interval so a - # saturated/unavailable database cannot turn a periodic - # safety check into one full CTE retry every minute. - advanced = dataclass_replace( - advanced, - full_rescan_attempt_monotonic=observed, - ) - mode = "incremental_self_check_failed" - check_reason = "periodic_self_check_failed" - else: - oracle_window: ( - IncrementalShareWindow | DaemonShareWindowMirror - ) = full_window - daemon_unprepared_reason: str | None = None - if self._window_pipeline_rust_enabled(): - # Keep the cache in mirror form so the next delta build - # advances through the daemon rather than mode-cycling; - # this is also how a Python-advancing cache (the degraded - # state after a daemon outage) converts back. When the - # daemon does not already hold the adopted digest (an - # oversized re-center, a mismatch repair, or the - # conversion after an outage) it is prepared for it here - # (issue #207), so the next advance never answers - # needs_full for a decision the coordinator itself made. - oracle_window, daemon_unprepared_reason = ( - self._daemon_adopted_window( - full_window=full_window, - shares_json=shares_json, - digest=digest, - advanced_window=advanced_window, - snapshot_anchor_ms=snapshot_anchor_ms, - append_invalidation_epoch=append_invalidation_epoch, - ) - ) - advanced = _IncrementalPayoutArtifactWindow( - window=oracle_window, - shares_json=shares_json, - share_snapshot_sha256=digest, - refreshed_monotonic=observed, - full_rescan_monotonic=observed, - full_rescan_attempt_monotonic=observed, - append_invalidation_epoch=append_invalidation_epoch, - daemon_unprepared_reason=daemon_unprepared_reason, - ) - mode = "self_check_match" if matched else "self_check_mismatch" - check_reason = "periodic_self_check" - # A deliberate re-center is flagged on the build (issue - # #207) rather than given a reason of its own: the rescan - # reason vocabulary is closed in the #224 telemetry owner, - # and with the daemon prepared for the adopted digest the - # re-center draws no rebuild for a later build to name. - self_check_recentered = bool(recenter_oversized) - if reused_prior_balances_sha256 is not None: - try: - balances_started = time.monotonic() - try: - checked_balances = tuple( - runtime.ledger.current_prior_balances() - ) - finally: - self._note_window_build_phase( - "ledger_read", - time.monotonic() - balances_started, - ) - checked_balances_sha256 = self._canonical_json_sha256( - checked_balances - ) - if ( - checked_balances_sha256 - != reused_prior_balances_sha256 - ): - balance_check_prior_balances = checked_balances - balance_check_mismatch = True - runtime._payout_prior_balances_reuse_invalidated_sha256 = ( - reused_prior_balances_sha256 - ) - # Equal-window refreshes intentionally retain - # their armed balances. Drop the stale artifact so - # the fresh balance read cannot be discarded as a - # window-only no-op when this build is installed. - with runtime._job_cache_lock: - armed = runtime._payout_ledger_artifact - if armed is not None: - armed_balances_sha256 = ( - armed.prior_balances_sha256 - or self._canonical_json_sha256( - armed.prior_balances - ) - ) - if ( - armed_balances_sha256 - == reused_prior_balances_sha256 - ): - self._disarm_payout_ledger_artifact_locked() - except Exception: - # The carry backstop is independent of the window - # oracle that just succeeded. Reverting here would - # serve a window the oracle may have refuted for a - # whole rescan interval, so keep the full window and - # leave the reused balances unverified until the next - # runtime-check. - check_reason = ( - "periodic_self_check_balance_check_failed" - ) - if check_reason is not None: - self._observe_payout_window_full_rescan( - check_reason, - FULL_RESCAN_PATH_IN_PROCESS, - time.monotonic() - self_check_started, - ) - - runtime._incremental_payout_artifact_window = advanced - return _PayoutWindowMaterialization( - shares_json=advanced.shares_json, - share_snapshot_sha256=advanced.share_snapshot_sha256, - snapshot_anchor_ms=advanced.window.anchor_job_issued_at_ms, - mode=mode, - record_count=advanced.window.record_count, - stats=stats, - full_rescan_reason=check_reason, - balance_check_prior_balances=balance_check_prior_balances, - balance_check_mismatch=balance_check_mismatch, - self_check_deferred=self_check_deferred, - self_check_recentered=self_check_recentered, - ) - - def _build_payout_ledger_artifact( - self, - expected_payout_state_generation: int, - artifact_payout_state_generation: int, - network_difficulty: int, - force_full_rescan: bool = False, - bypass_build_interval: bool = False, - during_publication: bool = False, - *, - force_prior_balances_read: bool = False, - ) -> PayoutLedgerArtifact | None: - """Build a stable ledger snapshot without publishing it. - - The build proper is ``_build_payout_ledger_artifact_scoped``; this - seam owns the per-build phase accumulator and observes it on every - exit under a closed outcome (issue #228), so a build that aborts (a - lost generation race, an anchor past the audit ceiling, an empty - window) or dies with the ledger still contributes the time it - measured. The slow builds are exactly the ones that lose those - races, and a family that only counted finished builds would read - as healthy during the incident it exists for. - """ - phases = self._begin_window_build_phases() - outcome = "failed" - try: - artifact, outcome = self._build_payout_ledger_artifact_scoped( - expected_payout_state_generation, - artifact_payout_state_generation, - network_difficulty, - force_full_rescan, - bypass_build_interval, - during_publication, - force_prior_balances_read=force_prior_balances_read, - phases=phases, - ) - return artifact - finally: - self._finish_window_build_phases() - self._observe_window_build_phases(phases, outcome) - - def _build_payout_ledger_artifact_scoped( - self, - expected_payout_state_generation: int, - artifact_payout_state_generation: int, - network_difficulty: int, - force_full_rescan: bool = False, - bypass_build_interval: bool = False, - during_publication: bool = False, - *, - force_prior_balances_read: bool = False, - phases: dict[str, float], - ) -> tuple[PayoutLedgerArtifact | None, str]: - """The build proper; returns the artifact and its closed outcome. - - Validity is anchor-scoped, not count-scoped. The pending-commit clamp - selects the highest clean anchor: every share stamped at or below it - is already durable, so the window read at that anchor is exact and - reproducible no matter how many shares commit while the walk runs. - Shares stamped above the anchor deterministically belong to the next - window; concurrent writers therefore never invalidate this attempt. - - ``force_full_rescan`` discards the incremental window and runs the - oracle (``reconcile_invalidation``), which also rereads the prior - balances. ``force_prior_balances_read`` is the narrower #224 intent: - the share window is still exact (reconciliation never writes - ``qbit_share_ledger``) but the published prior balances may not be, - so the window is reused through the normal debounce/delta path and - only the carry aggregate is read live. It is never a full rescan. - - Issue #228 adds attribution: a candidate preparation on the calling - thread says which cause and flag asked for the build (read from the - thread-local context, since the coordinator delegate reaches this - seam with a fixed signature), and every build is timed by phase. - """ - runtime = self._runtime - runtime._ensure_job_cache_state() - # A candidate preparation on this thread says why it asked for - # this build; background preparations carry no attribution. - attribution = getattr( - self._payout_window_build_context, "attribution", None - ) - payout_mutated = bool(force_full_rescan or force_prior_balances_read) - build_started = time.monotonic() - ledger_started = time.monotonic() - materialized: _PayoutWindowMaterialization - try: - lock_wait_started = time.monotonic() - with runtime._payout_state_prepare_lock: - self._note_window_build_phase( - "lock_wait", - time.monotonic() - lock_wait_started, - ) - with runtime._job_cache_lock: - if ( - expected_payout_state_generation - != runtime._payout_state_generation - ): - return None, "aborted" - append_invalidation_epoch = int( - runtime._payout_ledger_append_invalidation_epoch - ) - published_payout_artifact = ( - runtime._published_payout_state.artifact - ) - invalidated_balances_sha256 = ( - runtime._payout_prior_balances_reuse_invalidated_sha256 - ) - if ( - invalidated_balances_sha256 is not None - and published_payout_artifact is not None - and published_payout_artifact.prior_balances_sha256 - != invalidated_balances_sha256 - ): - # A later publication installed a different balance - # snapshot, so the digest that failed its oracle check can - # no longer enter this generation's reuse path. - runtime._payout_prior_balances_reuse_invalidated_sha256 = None - invalidated_balances_sha256 = None - reuse_published_balances = bool( - not payout_mutated - and published_payout_artifact is not None - and published_payout_artifact.generation - == expected_payout_state_generation - and published_payout_artifact.prior_balances_sha256 - != invalidated_balances_sha256 - ) - snapshot_window_weight = ( - PRISM_REWARD_WINDOW_MULTIPLIER - * PRISM_SNAPSHOT_WINDOW_MARGIN - * int(network_difficulty) - ) - clamp_now_ms = self._now_ms() - snapshot_anchor_ms = runtime._job_snapshot_anchor_ms(clamp_now_ms) - if ( - clamp_now_ms - snapshot_anchor_ms - > runtime._payout_artifact_max_anchor_age_ms() - ): - # The floor is held this far below now only by a wedged - # writer or a leaked release; an artifact anchored there - # would already be past the audit ceiling -- born - # expired -- on arrival. Abort before paying the window - # walk: the re-arm backoff paces retries and the - # share-commit liveness watchdog owns recovery. - return None, "aborted" - window_started = time.monotonic() - # Publish the walk's anchor before the ledger read: a - # late-visible append committing mid-walk can predate it - # while a cold or forcibly cleared cache exposes no other - # anchor, and the walk's database snapshot may already - # exclude the row. Retiring in the finally is safe here - # because the materialization installs its incremental - # pointer -- the walk's durable visibility -- before this - # exposure is withdrawn. - inflight_anchor_token = runtime._expose_inflight_scan_anchor( - snapshot_anchor_ms - ) - try: - materialized = runtime._incremental_payout_window_materialization( - snapshot_anchor_ms=snapshot_anchor_ms, - snapshot_window_weight=snapshot_window_weight, - force_full_rescan=force_full_rescan, - bypass_build_interval=bypass_build_interval, - append_invalidation_epoch=append_invalidation_epoch, - reused_prior_balances_sha256=( - published_payout_artifact.prior_balances_sha256 - if reuse_published_balances - and published_payout_artifact is not None - else None - ), - ) - finally: - runtime._retire_inflight_scan_anchor(inflight_anchor_token) - if materialized.balance_check_mismatch: - # Emit at detection time: a later, unrelated artifact - # serialization or stats failure must not hide the oracle - # alarm that already invalidated reuse. - runtime._record_payout_artifact_event( - "balance_check_mismatch" - ) - runtime._payout_artifact_log( - "payout_artifact_balance_check_mismatch", - payout_state_generation=int( - artifact_payout_state_generation - ), - balance_check_mismatch=True, - ) - if ( - materialized.balance_check_mismatch - and materialized.balance_check_prior_balances is not None - and published_payout_artifact is not None - ): - # The oracle refuted the published balance snapshot, so - # invalidating reuse alone is not enough: consumers keyed - # to the published digest (the armed-artifact fence, new - # build keys, and the synchronous fallback) would keep - # serving the refuted bytes until an unrelated generation - # change. Publish the repaired balances for the same - # generation so this build's artifact can arm and serve. - runtime._publish_self_check_repaired_balances( - expected_payout_state_generation, - stale_prior_balances_sha256=( - published_payout_artifact.prior_balances_sha256 - ), - balances=materialized.balance_check_prior_balances, - ) - window_build_seconds = time.monotonic() - window_started - if materialized.balance_check_prior_balances is not None: - # The periodic oracle already paid for the live aggregate. - # Use that exact result rather than issuing a duplicate - # carry query in the mismatch build. - prior_balances = ( - materialized.balance_check_prior_balances - ) - prior_balances_source = "ledger" - elif reuse_published_balances: - assert published_payout_artifact is not None - # Prior balances are generation-owned immutable state. - # Normal share-window refreshes cannot change them, so - # reuse the exact published bytes instead of aggregating - # the 1.3M-row carry tables on every debounced generation. - # Reconcile/withdrawal mutations force a ledger read. - prior_balances = ( - published_payout_artifact.prior_balances() - ) - prior_balances_source = "published" - else: - balances_started = time.monotonic() - try: - prior_balances = runtime.ledger.current_prior_balances() - finally: - self._note_window_build_phase( - "ledger_read", - time.monotonic() - balances_started, - ) - prior_balances_source = "ledger" - accepted_share_count, _ = runtime.accepted_share_stats() - except Exception as exc: - # Artifact preparation is speculative. The synchronous bundle path - # still owns errors when current work actually requires a snapshot. - if during_publication: - runtime._record_payout_artifact_event("build_aborted") - runtime._payout_artifact_log( - "payout_artifact_build_aborted", - payout_state_generation=int( - artifact_payout_state_generation - ), - duration_seconds=round( - time.monotonic() - build_started, - 3, - ), - during_publication=True, - build_interval_bypassed=bool(bypass_build_interval), - abort_reason=type(exc).__name__, - phase_seconds={ - phase: round(float(phases.get(phase, 0.0)), 6) - for phase in PRISM_PAYOUT_WINDOW_BUILD_PHASES - }, - ) - return None, "failed" - finally: - runtime._observe_tip_refresh_build_phase( - "ledger_snapshot", - time.monotonic() - ledger_started, - ) - if materialized.self_check_deferred: - # Owed until a publication hands it to the background walker. - with runtime.lock: - self._payout_self_check_owed = True - if not materialized.shares_json: - return None, "aborted" - copy_started = time.monotonic() - shares_json = materialized.shares_json - frozen_balances = tuple(prior_balances) - runtime._observe_tip_refresh_build_phase( - "serialization_copy", - time.monotonic() - copy_started, - ) - artifact = PayoutLedgerArtifact( - generation=0, - payout_state_generation=artifact_payout_state_generation, - network_difficulty=int(network_difficulty), - accepted_share_count=accepted_share_count, - shares_json=shares_json, - prior_balances=frozen_balances, - prepared_monotonic=time.monotonic(), - snapshot_anchor_ms=materialized.snapshot_anchor_ms, - share_snapshot_sha256=materialized.share_snapshot_sha256, - prior_balances_sha256=self._canonical_json_sha256(frozen_balances), - append_invalidation_epoch=append_invalidation_epoch, - window_build_mode=materialized.mode, - window_delta_rows=materialized.stats.added_rows, - window_expired_rows=materialized.stats.expired_rows, - window_touched_pages=materialized.stats.touched_pages, - window_build_seconds=window_build_seconds, - window_full_rescan_reason=materialized.full_rescan_reason, - window_self_check_recentered=materialized.self_check_recentered, - ) - build_seconds = time.monotonic() - build_started - log_fields: dict[str, object] = { - "payout_state_generation": int(artifact_payout_state_generation), - "duration_seconds": round(build_seconds, 3), - "window_build_seconds": round(window_build_seconds, 3), - "window_build_mode": materialized.mode, - "window_shares": materialized.record_count, - "delta_rows": materialized.stats.added_rows, - "expired_rows": materialized.stats.expired_rows, - "touched_pages": materialized.stats.touched_pages, - "anchor_age_ms": self._now_ms() - materialized.snapshot_anchor_ms, - "during_publication": bool(during_publication), - "build_interval_bypassed": bool(bypass_build_interval), - "prior_balances_source": prior_balances_source, - "prior_balances_read_forced": bool(force_prior_balances_read), - "balance_check_mismatch": materialized.balance_check_mismatch, - # Issue #228: which caller forced this build and why, readable - # from the line alone. ``invalidation_cause`` is the captured - # source cause verbatim; ``force_origin`` says whether a payout - # mutation came from the caller's explicit flag, from the - # reserved cause, both, or neither; ``cause_inherited`` marks a - # capture that found a cause already acted on (the sticky-tuple - # shape that used to force full rescans); ``window_oracle_forced`` - # is the explicit exact-oracle request. - "invalidation_cause": ( - attribution.cause if attribution is not None else None - ), - "force_origin": ( - attribution.force_origin if attribution is not None else "none" - ), - "cause_inherited": bool( - attribution is not None and attribution.cause_inherited - ), - "payout_mutated": payout_mutated, - "window_oracle_forced": bool(force_full_rescan), - "self_check_deferred": bool(materialized.self_check_deferred), - "self_check_recentered": bool(materialized.self_check_recentered), - "phase_seconds": { - phase: round(float(phases.get(phase, 0.0)), 6) - for phase in PRISM_PAYOUT_WINDOW_BUILD_PHASES - }, - } - if materialized.full_rescan_reason is not None: - log_fields["full_rescan_reason"] = materialized.full_rescan_reason - if materialized.mode == "debounced": - runtime._record_payout_artifact_event("debounced") - runtime._payout_artifact_log( - "payout_artifact_build_debounced", - **log_fields, - ) - else: - runtime._record_payout_artifact_event("built") - if materialized.mode == "full_rescan": - runtime._record_payout_artifact_event("full_rescan") - elif materialized.mode == "self_check_match": - runtime._record_payout_artifact_event("self_check_match") - elif materialized.mode == "self_check_mismatch": - runtime._record_payout_artifact_event("self_check_mismatch") - elif materialized.mode == "incremental_self_check_failed": - runtime._record_payout_artifact_event("incremental") - runtime._record_payout_artifact_event("self_check_failed") - else: - runtime._record_payout_artifact_event("incremental") - runtime._payout_artifact_log("payout_artifact_built", **log_fields) - return artifact, ( - "debounced" if materialized.mode == "debounced" else "completed" - ) - - def _prepare_payout_ledger_artifact( - self, - payout_state_generation: int, - network_difficulty: int, - *, - bypass_build_interval: bool = False, - ) -> None: - """Prepare and atomically publish an artifact for a current generation.""" - runtime = self._runtime - build_started = time.monotonic() - artifact = runtime._build_payout_ledger_artifact( - payout_state_generation, - payout_state_generation, - network_difficulty, - False, - bypass_build_interval, - ) - build_seconds = time.monotonic() - build_started - if artifact is None: - # A superseding generation, a snapshot error, an empty window, or - # a pathologically old pending-commit floor discarded this - # attempt; that condition can persist, and each attempt performs - # ledger preparation under the shared lock. Back the - # fence-failure re-arm off until something arms. - runtime._record_payout_artifact_event("build_aborted") - runtime._payout_artifact_log( - "payout_artifact_build_aborted", - payout_state_generation=int(payout_state_generation), - duration_seconds=round(build_seconds, 3), - ) - with runtime._payout_artifact_executor_lock: - runtime._payout_artifact_rearm_backoff = min( - runtime._payout_artifact_rearm_backoff * 2, - PRISM_PAYOUT_ARTIFACT_REARM_BACKOFF_CAP, - ) - return - if not runtime._install_payout_ledger_artifact(artifact): - # A discarded install -- born expired after a slow walk, an - # ordering loss to a fresher window, or a generation change -- - # armed nothing. It must not collapse the re-arm interval: an - # unconditional reset here is exactly the reset-then-rewalk - # livelock that rolled back the first anchor-scoped deploy. - with runtime._payout_artifact_executor_lock: - runtime._payout_artifact_rearm_backoff = min( - runtime._payout_artifact_rearm_backoff * 2, - PRISM_PAYOUT_ARTIFACT_REARM_BACKOFF_CAP, - ) - - def _install_payout_ledger_artifact( - self, - artifact: PayoutLedgerArtifact, - ) -> bool: - """Atomically publish a prepared artifact for its own generation. - - Snapshot-freshness-ordered and idempotent: every snapshot is taken at - a clean anchor (strictly below all pending commits), and the durable - window at or below an anchor is immutable, so the anchor orders - snapshots even when a build delayed in window conversion finishes - after a later snapshot installed (completion time cannot order - snapshots). Equal-anchor installs with an identical window -- every - flight waiter re-runs cache publication with the same prepared - artifact, and a same-window speculative rebuild re-reads unchanged - state under an equal anchor -- keep the installed generation instead - of re-keying bundle lookups for nothing. - - Returns True when the window is armed (installed fresh, refreshed in - place, or confirmed already current); only that outcome resets the - re-arm backoff. A discarded install returns False and leaves the - backoff alone -- in particular a BORN-EXPIRED artifact, whose anchor - aged past the audit ceiling during its own window walk. Arming (or - crediting) such an install would collapse the re-arm interval and - rewalk the reward window continuously: the livelock behind the - 2026-07-29 rollback. - """ - runtime = self._runtime - try: - return runtime._install_payout_ledger_artifact_outcome(artifact) - finally: - # A seeded artifact carries the exposed scan anchor of the - # synchronous read that produced it. The install decision above - # is atomic under _job_cache_lock: either the window armed (its - # own anchor is now visible to append invalidation) or it was - # discarded and can never serve, so the exposure has done its - # job either way. - runtime._retire_inflight_scan_anchor( - artifact.inflight_scan_anchor_token - ) - - def _install_payout_ledger_artifact_outcome( - self, - artifact: PayoutLedgerArtifact, - ) -> bool: - runtime = self._runtime - if not runtime._payout_artifact_reuse_active(): - # Kill-switch: never arm. An armed artifact would churn install - # events while disabled and re-key the idle-bundle fast path to - # a generation the reuse probe refuses anyway. - return False - anchor_age_ms = ( - None - if artifact.snapshot_anchor_ms is None - else self._now_ms() - int(artifact.snapshot_anchor_ms) - ) - if ( - anchor_age_ms is None - or anchor_age_ms > runtime._payout_artifact_max_anchor_age_ms() - ): - runtime._record_payout_artifact_event("born_expired") - runtime._payout_artifact_log( - "payout_artifact_born_expired", - payout_state_generation=int(artifact.payout_state_generation), - anchor_age_ms=anchor_age_ms, - window_shares=len(artifact.shares_json), - ) - return False - with runtime._job_cache_lock: - outcome, armed_generation = ( - runtime._install_payout_ledger_artifact_locked(artifact) - ) - runtime._record_payout_artifact_event(outcome) - if outcome == "discarded": - runtime._payout_artifact_log( - "payout_artifact_discarded", - payout_state_generation=int(artifact.payout_state_generation), - anchor_age_ms=anchor_age_ms, - window_shares=len(artifact.shares_json), - ) - return False - if outcome in ("installed", "refreshed"): - runtime._payout_artifact_log( - "payout_artifact_" + outcome, - generation=armed_generation, - payout_state_generation=int(artifact.payout_state_generation), - anchor_age_ms=anchor_age_ms, - window_shares=len(artifact.shares_json), - ) - # An armed artifact -- installed fresh or confirmed already current - # -- proves preparation can succeed again; let the next fence-failure - # re-arm run at the configured floor. - with runtime._payout_artifact_executor_lock: - runtime._payout_artifact_rearm_backoff = 1 - return True - - def _install_payout_ledger_artifact_locked( - self, - artifact: PayoutLedgerArtifact, - ) -> tuple[str, int | None]: - """Ordering core of the install; returns (outcome, armed generation).""" - runtime = self._runtime - if ( - artifact.payout_state_generation != runtime._payout_state_generation - or artifact.append_invalidation_epoch - != runtime._payout_ledger_append_invalidation_epoch - ): - return "discarded", None - current = runtime._payout_ledger_artifact - if ( - current is not None - and current.payout_state_generation - == artifact.payout_state_generation - ): - current_anchor_ms = ( - -1 - if current.snapshot_anchor_ms is None - else int(current.snapshot_anchor_ms) - ) - artifact_anchor_ms = ( - -1 - if artifact.snapshot_anchor_ms is None - else int(artifact.snapshot_anchor_ms) - ) - if current_anchor_ms > artifact_anchor_ms: - return "discarded", None - if ( - current.network_difficulty == artifact.network_difficulty - and current.share_snapshot_sha256 is not None - and current.share_snapshot_sha256 - == artifact.share_snapshot_sha256 - ): - # The armed artifact already carries exactly this - # window; keep its generation (no lookup re-key) but - # still treat the preparation as a success, and advance - # the freshness clock in place either way -- otherwise a - # quiet share stream, or a pending-commit floor pinning - # the anchor so every re-prove lands at the SAME anchor, - # would cycle identical rebuilds forever without ever - # un-staling the armed artifact. No fresher window is - # constructible while the window bytes are unchanged, so - # the re-walk that just re-proved them earns the credit. - # The armed balances are kept: they are what the reuse - # fence hashes against the published payout state - # (including an accepted parent's preview patch). - anchor_advanced = artifact_anchor_ms > current_anchor_ms - self._replace_payout_ledger_artifact_locked( - dataclass_replace( - current, - accepted_share_count=artifact.accepted_share_count, - prepared_monotonic=time.monotonic(), - snapshot_anchor_ms=( - artifact.snapshot_anchor_ms - if anchor_advanced - else current.snapshot_anchor_ms - ), - ) - ) - if anchor_advanced: - return "refreshed", int(current.generation) - return "already_current", int(current.generation) - template_artifacts = getattr( - runtime, - "_template_artifacts", - None, - ) - if ( - current.network_difficulty != artifact.network_difficulty - and template_artifacts is not None - and current.network_difficulty - == int(template_artifacts.network_difficulty) - and artifact.network_difficulty - != int(template_artifacts.network_difficulty) - ): - # Whether a delayed pre-retarget build's anchor ties or - # leads, keep the artifact the live template difficulty - # can actually reuse: a wrong-difficulty install would - # fail every reuse probe on the difficulty check, which - # never re-arms, until the next synchronous build - # re-seeded the live window. - return "discarded", None - runtime._payout_ledger_artifact_generation += 1 - # Freshness runs from the moment the window becomes reusable, not - # from build completion: a sync-seeded artifact is constructed - # mid-bundle-build and reaches cache publication only after the - # audit builder, so stamping here keeps the budget honest at every - # install site. - self._replace_payout_ledger_artifact_locked( - dataclass_replace( - artifact, - generation=runtime._payout_ledger_artifact_generation, - prepared_monotonic=time.monotonic(), - ) - ) - return "installed", int(runtime._payout_ledger_artifact_generation) - - def _payout_artifact_preparation_loop(self) -> None: - runtime = self._runtime - while True: - with runtime._payout_artifact_executor_lock: - request = runtime._payout_artifact_requested - bypass_build_interval = ( - runtime._payout_artifact_requested_bypass - ) - runtime._payout_artifact_requested = None - runtime._payout_artifact_requested_bypass = False - if request is None: - runtime._payout_artifact_future = None - return - phases = runtime._job_build_phases() - phases.clear() - try: - runtime._prepare_payout_ledger_artifact( - *request, - bypass_build_interval=bypass_build_interval, - ) - finally: - runtime._flush_job_build_phases(phases) - - def _schedule_payout_ledger_artifact_preparation( - self, - payout_state_generation: int, - network_difficulty: int, - *, - min_interval_seconds: float | None = None, - bypass_build_interval: bool = False, - ) -> None: - """Latest-generation-wins scheduling with one worker and one slot. - - With min_interval_seconds, the debounce check, timestamp update, and - request enqueue are one atomic step under the executor lock: a - concurrent dequeue or a racing probe cannot slip a duplicate request - into the emptied slot and run the reward-window snapshot twice. - """ - runtime = self._runtime - runtime._ensure_job_cache_state() - if not runtime._payout_artifact_reuse_active(): - # Kill-switch: no background reward-window walks either. Off - # disables the reuse line -- probe refusals, no walks, no - # arming -- restoring pre-reuse delivery economics; the - # synchronous build path keeps the re-landed anchor-selection - # semantics rather than the pre-#102 count fences. - return - with runtime._payout_artifact_executor_lock: - if runtime._payout_artifact_executor_shutdown: - return - if min_interval_seconds is not None: - # Consecutive aborted rebuilds stretch the re-arm interval; - # event-driven preparations below reset the backoff because - # they mark a real state change worth retrying promptly for. - effective_interval = ( - min_interval_seconds * runtime._payout_artifact_rearm_backoff - ) - if ( - runtime._payout_artifact_last_schedule_monotonic is not None - and time.monotonic() - - runtime._payout_artifact_last_schedule_monotonic - < effective_interval - ): - return - rearm_backoff = runtime._payout_artifact_rearm_backoff - else: - runtime._payout_artifact_rearm_backoff = 1 - rearm_backoff = None - runtime._payout_artifact_requested = ( - int(payout_state_generation), - int(network_difficulty), - ) - runtime._payout_artifact_requested_bypass = ( - runtime._payout_artifact_requested_bypass - or bool(bypass_build_interval) - ) - runtime._payout_artifact_last_schedule_monotonic = time.monotonic() - if runtime._payout_artifact_future is None: - executor = runtime._payout_artifact_executor - if executor is None: - executor = ThreadPoolExecutor( - max_workers=1, - thread_name_prefix="prism-payout-artifact", - ) - runtime._payout_artifact_executor = executor - runtime._payout_artifact_future = executor.submit( - runtime._payout_artifact_preparation_loop - ) - if rearm_backoff is not None: - # Only debounced fence-failure re-arms log here; event-driven - # preparations ride payout publications, which are already - # visible. Recorded outside the executor lock the counter - # shares. - runtime._record_payout_artifact_event("rearm_scheduled") - runtime._payout_artifact_log( - "payout_artifact_rearm_scheduled", - payout_state_generation=int(payout_state_generation), - backoff=int(rearm_backoff), - ) - - def _payout_artifact_max_anchor_age_ms(self) -> float: - """Audit ceiling on a served window's wall-clock anchor age.""" - runtime = self._runtime - return ( - float( - getattr( - runtime, - "payout_artifact_max_anchor_age_seconds", - DEFAULT_PRISM_PAYOUT_ARTIFACT_MAX_ANCHOR_AGE_SECONDS, - ) - ) - * 1000.0 - ) - - def _payout_artifact_reanchor_seconds(self) -> float: - """Anchor age that triggers a background re-anchor while reuse keeps serving.""" - runtime = self._runtime - return float( - getattr( - runtime, - "payout_artifact_reanchor_seconds", - DEFAULT_PRISM_PAYOUT_ARTIFACT_REANCHOR_SECONDS, - ) - ) - - def _payout_artifact_min_build_interval_seconds(self) -> float: - """Minimum normal cadence; found-block settlement bypasses it.""" - runtime = self._runtime - - return float( - getattr( - runtime, - "payout_artifact_min_build_interval_seconds", - runtime._payout_artifact_reanchor_seconds(), - ) - ) - - def _payout_artifact_reuse_active(self) -> bool: - """Master kill-switch (PRISM_PAYOUT_ARTIFACT_REUSE) for the reuse line.""" - runtime = self._runtime - return bool(getattr(runtime, "payout_artifact_reuse_enabled", True)) - - @staticmethod - def _payout_artifact_log(event: str, **fields: object) -> None: - """Single-line JSON lifecycle log; the incident was invisible here.""" - print( - "prism coordinator: " - + json.dumps({"event": event, **fields}, sort_keys=True), - flush=True, - ) - - def _record_payout_artifact_event(self, event: str) -> None: - runtime = self._runtime - runtime._ensure_job_cache_state() - with runtime._payout_artifact_executor_lock: - counts = runtime.payout_artifact_event_counts - counts[event] = int(counts.get(event, 0)) + 1 - - def _note_window_mirror_divergence(self) -> None: - """Record a refuted mirror and make sure it cannot be advanced again. - - Mirror construction reconciles bytes, digest and record count - eagerly, so a divergence reaching a consumer means an assumption - this code makes about the daemon's output is wrong. Whatever raised - it is already failing its own operation; the job here is to make - sure the artifact window that produced the doubt is not carried into - the NEXT build, and that the event is visible in metrics rather than - only in whatever traceback the caller prints. - - Deliberately does not take _payout_state_prepare_lock: callers reach - this from build and landing paths that must not queue behind an - oracle rebuild, and the cost of racing a concurrent materialization - is one extra full rescan, never an unsound window. - """ - runtime = self._runtime - runtime._record_payout_artifact_event("window_mirror_divergence") - runtime._incremental_payout_artifact_window = None - runtime._incremental_payout_artifact_window_invalidation_reason = ( - "window_mirror_divergence" - ) - - def _usable_payout_ledger_artifact( - self, - payout_state_generation: int, - network_difficulty: int, - *, - rearm_on_fence_failure: bool = True, - ) -> PayoutLedgerArtifact | None: - """Return the armed artifact when reuse is valid for NEW work. - - Validity is anchor-scoped and EVENT-DRIVEN: the artifact's window is - exact at its own snapshot anchor (which reused bundles declare), so - shares landing after the anchor never invalidate it -- they belong - to the next window by construction -- and the artifact stays valid - until a payout event actually changes the state it snapshots (the - generation and balances fences below). Wall-clock time is NOT a - validity input below the audit ceiling: the 2026-07-29 rollback was - a 10s anchor-age budget rejecting every slow build on arrival, and - the 2026-07-30 brownout was the same budget moved to install time -- - at production cadence (a shared build every 10-14s, a 4-8s window - walk) a 10s budget expires between consecutive builds, so every - build fell back to the synchronous reward-window walk while the - re-arm worker walked the same window in parallel. Past the - REANCHOR floor the probe schedules a debounced background re-anchor - but KEEPS SERVING the armed window; only past the audit ceiling - (the bound on how far a paid window may trail the live ledger -- - post-anchor shares settle in the next window, none are lost) does - reuse fail closed. The ceiling governs NEW reuse and issuance - decisions; an in-flight build keeps the window it already selected, - so a declared anchor slightly past the ceiling can still publish. - """ - runtime = self._runtime - runtime._ensure_job_cache_state() - if not runtime._payout_artifact_reuse_active(): - return None - with runtime._job_cache_lock: - artifact = runtime._payout_ledger_artifact - published_artifact = runtime._published_payout_state.artifact - append_invalidation_epoch = ( - runtime._payout_ledger_append_invalidation_epoch - ) - if ( - artifact is None - or artifact.payout_state_generation != payout_state_generation - or artifact.network_difficulty != int(network_difficulty) - or artifact.append_invalidation_epoch - != append_invalidation_epoch - ): - return None - if artifact.snapshot_anchor_ms is None: - # Without a recorded anchor the artifact cannot declare the - # anchor reused bundles must stamp; fail closed. - return None - anchor_age_ms = self._now_ms() - int(artifact.snapshot_anchor_ms) - if anchor_age_ms > runtime._payout_artifact_max_anchor_age_ms(): - # The declared anchor crossed the audit ceiling; the window may - # no longer be served to new work. Queue a bounded speculative - # rebuild that re-anchors it; in-flight builds keep the copy - # they selected. - if rearm_on_fence_failure: - runtime._rearm_payout_ledger_artifact_after_fence_failure( - payout_state_generation, - network_difficulty, - ) - runtime._record_payout_artifact_event("probe_rejected_ceiling") - return None - if anchor_age_ms > runtime._payout_artifact_reanchor_seconds() * 1000.0: - # Aging but valid: schedule the debounced background re-anchor - # and keep serving. The delivery path must never pay the - # synchronous reward-window walk while a correct armed window - # exists; the re-anchor swaps in a fresher window off the - # critical path. Counted unconditionally so the canary can - # distinguish "serving past the floor, re-anchor pending" from - # "never crossing the floor" -- rearm_scheduled alone - # under-counts through the debounce and suppression paths. - runtime._record_payout_artifact_event("probe_past_floor") - if rearm_on_fence_failure: - runtime._rearm_payout_ledger_artifact_after_fence_failure( - payout_state_generation, - network_difficulty, - ) - if published_artifact is None: - try: - published_artifact = runtime._current_payout_state_artifact() - except Exception: - return None - balances_sha256 = artifact.prior_balances_sha256 or self._canonical_json_sha256( - artifact.prior_balances - ) - with runtime._job_cache_lock: - if ( - runtime._payout_state_generation != payout_state_generation - or runtime._published_payout_state.artifact is not published_artifact - or runtime._payout_ledger_append_invalidation_epoch - != artifact.append_invalidation_epoch - ): - return None - latest = runtime._payout_ledger_artifact - if latest is not artifact: - # An equal-window freshness restamp (already_current / - # refreshed) replaces the armed object while this probe was - # hashing balances. The restamp keeps the generation, the - # window bytes, and the balances object, and only moves the - # freshness stamp (and possibly the anchor) forward, so the - # validity established above still holds for the restamped - # copy; failing closed here would turn the intentional - # pinned-floor recovery path into a spurious synchronous - # reward-window walk. A real re-key changes the generation - # or the window sha and still fails closed. - if ( - latest is None - or latest.generation != artifact.generation - or latest.payout_state_generation - != artifact.payout_state_generation - or latest.network_difficulty != artifact.network_difficulty - or latest.share_snapshot_sha256 is None - or latest.share_snapshot_sha256 - != artifact.share_snapshot_sha256 - or latest.snapshot_anchor_ms is None - or latest.prior_balances is not artifact.prior_balances - or latest.append_invalidation_epoch - != artifact.append_invalidation_epoch - ): - return None - artifact = latest - if balances_sha256 != published_artifact.prior_balances_sha256: - # A candidate can carry a ledger snapshot prepared before its - # payout state is published. Never keep retrying that stale - # shortcut; the synchronous path will take a fresh snapshot. - self._disarm_payout_ledger_artifact_locked() - return None - served = artifact - # Recorded outside the cache lock the event counter's own lock - # ordering forbids nesting under. - runtime._record_payout_artifact_event("served_reuse") - return served - - def _rearm_payout_ledger_artifact_after_fence_failure( - self, - payout_state_generation: int, - network_difficulty: int, - ) -> None: - """Debounced rebuild scheduling for an artifact past the re-anchor floor or ceiling. - - The interval floor keeps delta/carry reads (and any full fallback) - from running continuously when artifacts age out faster than rebuilds - complete. - Landed accepted-block previews suppress the re-arm entirely: a - speculative rebuild in that window would read database balances the - published prospective state has already superseded, and the - durable-confirmation path resumes preparation itself once the gap - closes. - """ - runtime = self._runtime - runtime._ensure_job_cache_state() - with runtime._job_cache_lock: - if ( - runtime._payout_state_publication_blocked - or payout_state_generation != runtime._payout_state_generation - ): - return - with runtime._accepted_block_payout_preview_condition: - if any( - transition.landed - for transition in runtime._accepted_block_payout_previews.values() - ): - return - cached_window = getattr( - runtime, - "_incremental_payout_artifact_window", - None, - ) - if ( - cached_window is not None - and time.monotonic() - cached_window.refreshed_monotonic - < runtime._payout_artifact_min_build_interval_seconds() - ): - # A pinned/old declared anchor can cross the re-anchor floor while - # the window itself was just re-proved. Do not enqueue a worker - # every five seconds merely to retag the same pages and re-read - # carry balances; the next probe after the real cadence expires - # will schedule it. - return - runtime._schedule_payout_ledger_artifact_preparation( - payout_state_generation, - network_difficulty, - min_interval_seconds=float( - getattr( - runtime, - "payout_artifact_rearm_min_seconds", - DEFAULT_PRISM_PAYOUT_ARTIFACT_REARM_MIN_SECONDS, - ) - ), - ) - - def _schedule_current_payout_ledger_artifact_if_missing(self) -> None: - """Resume speculative preparation after a durable preview catches up.""" - runtime = self._runtime - runtime._ensure_job_cache_state() - with runtime._job_cache_lock: - payout_state_generation = runtime._payout_state_generation - template_artifacts = runtime._template_artifacts - if template_artifacts is None: - return - if runtime._usable_payout_ledger_artifact( - payout_state_generation, - template_artifacts.network_difficulty, - # This call site enqueues unconditionally below; letting the - # probe also re-arm would race the one-slot worker's dequeue and - # run the reward-window snapshot twice for one resumption. - rearm_on_fence_failure=False, - ) is not None: - return - runtime._schedule_payout_ledger_artifact_preparation( - payout_state_generation, - template_artifacts.network_difficulty, - ) - - def shutdown_payout_artifact_executor(self) -> None: - runtime = self._runtime - runtime._ensure_job_cache_state() - with runtime._payout_artifact_executor_lock: - executor = runtime._payout_artifact_executor - runtime._payout_artifact_executor = None - runtime._payout_artifact_executor_shutdown = True - runtime._payout_artifact_requested = None - if executor is not None: - executor.shutdown(wait=True, cancel_futures=True) - - def _prepare_payout_state_artifact( - self, - *, - generation: int, - source_generation: int, - cancellation: _JobBuildCancellation | None = None, - ) -> PayoutStateArtifact: - """Snapshot carry-forward balances once for a payout generation.""" - runtime = self._runtime - - started = time.monotonic() - if cancellation is not None: - cancellation.raise_if_cancelled("payout artifact read") - with runtime._payout_state_prepare_lock: - balances = runtime.ledger.current_prior_balances() - if cancellation is not None: - cancellation.raise_if_cancelled("payout artifact serialization") - artifact = runtime._payout_state_artifact_from_balances( - generation=generation, - source_generation=source_generation, - balances=balances, - ) - phases = runtime._job_build_phases() - phases["payout_artifact"] = phases.get("payout_artifact", 0.0) + ( - time.monotonic() - started - ) - return artifact - - def _payout_state_artifact_from_balances( - self, - *, - generation: int, - source_generation: int, - balances: list[dict[str, object]], - ) -> PayoutStateArtifact: - balances_json = self._canonical_json_text(balances) - return PayoutStateArtifact( - generation=generation, - source_generation=source_generation, - prior_balances_json=balances_json, - prior_balances_sha256=hashlib.sha256(balances_json.encode()).hexdigest(), - prepared_monotonic=time.monotonic(), - ) - - def _publish_self_check_repaired_balances( - self, - expected_payout_state_generation: int, - *, - stale_prior_balances_sha256: str, - balances: Sequence[dict[str, object]], - ) -> bool: - """Replace the published balance snapshot the periodic oracle refuted. - - The published artifact is otherwise immutable for its generation, but - a confirmed drift means its bytes no longer match the durable ledger - (a payout mutation whose response was lost). Swapping in the repaired - balances re-keys new work to the corrected digest, and every serving - decision for NOT-yet-stamped work re-checks that digest. Jobs already - stamped for miners have no such fence -- their admission compares - only the payout generation and append epoch, which both survive this - swap -- so the repair also schedules the same refresh wave a payout - publication schedules; digest-aware reselection then replaces exactly - the active jobs keyed to the refuted snapshot. Only the exact refuted - snapshot is replaced: any concurrent publication wins the guarded - swap (and drives its own wave). - """ - runtime = self._runtime - runtime._ensure_job_cache_state() - with runtime._job_cache_lock: - source_generation = runtime._published_payout_state.source_generation - repaired = runtime._payout_state_artifact_from_balances( - generation=int(expected_payout_state_generation), - source_generation=int(source_generation), - balances=list(balances), - ) - with runtime._job_cache_lock: - published = runtime._published_payout_state - current = published.artifact - if ( - runtime._payout_state_publication_blocked - or runtime._payout_state_generation - != expected_payout_state_generation - or published.generation != expected_payout_state_generation - or published.source_generation != source_generation - or current is None - or current.generation != expected_payout_state_generation - or current.prior_balances_sha256 - != stale_prior_balances_sha256 - ): - return False - runtime._published_payout_state = dataclass_replace( - published, - artifact=repaired, - ) - runtime._record_payout_artifact_event("balance_repair_published") - runtime._payout_artifact_log( - "payout_artifact_balance_repair_published", - payout_state_generation=int(expected_payout_state_generation), - stale_prior_balances_sha256=stale_prior_balances_sha256, - repaired_prior_balances_sha256=repaired.prior_balances_sha256, - ) - # Active jobs keyed to the refuted digest remain mineable until a - # wave replaces them -- a block solved from one would commit the - # stale allocation. The pending mark survives a superseded wave and - # the digest reselection is idempotent, so a wave that raced this - # repair converges on the next pass. - runtime._mark_tip_refresh_pending(int(expected_payout_state_generation)) - runtime._schedule_tip_refresh_retry() - return True - - def _current_payout_state_artifact( - self, - cancellation: _JobBuildCancellation | None = None, - ) -> PayoutStateArtifact: - """Return the immutable artifact published for the current generation.""" - runtime = self._runtime - - runtime._ensure_job_cache_state() - while True: - with runtime._job_cache_lock: - if runtime._payout_state_publication_blocked: - raise PayoutStatePublicationBlocked( - "payout state invalidation is pending publication" - ) - published = runtime._published_payout_state - if ( - published.generation == runtime._payout_state_generation - and published.artifact is not None - and published.artifact.generation == published.generation - ): - return published.artifact - generation = runtime._payout_state_generation - source_generation = published.source_generation - artifact = runtime._prepare_payout_state_artifact( - generation=generation, - source_generation=source_generation, - cancellation=cancellation, - ) - with runtime._job_cache_lock: - published = runtime._published_payout_state - if ( - runtime._payout_state_publication_blocked - or runtime._payout_state_generation != generation - or published.source_generation != source_generation - ): - if cancellation is not None: - cancellation.raise_if_cancelled( - "payout artifact publication race" - ) - continue - runtime._published_payout_state = dataclass_replace( - published, - artifact=artifact, - ) - return artifact - - @contextmanager - def _payout_balance_mutation(self) -> Iterator[None]: - """Serialize durable balance changes without excluding delivery.""" - runtime = self._runtime - runtime._ensure_job_cache_state() - with runtime._block_submitter_lock( - runtime._payout_balance_mutation_lock, - "payout-balance-mutation", - ): - with runtime._accepted_block_payout_preview_condition: - landed_transition = any( - transition.landed - for transition in runtime._accepted_block_payout_previews.values() - ) - if landed_transition: - raise PayoutStatePublicationBlocked( - "accepted block payout confirmation is still pending" - ) - yield - - def _begin_accepted_block_payout_preview( - self, - block_hash: str, - *, - block_height: int | None = None, - ) -> None: - """Prevent child work from snapshotting pre-accept balances.""" - runtime = self._runtime - runtime._ensure_job_cache_state() - key = block_hash.lower() - with runtime._accepted_block_payout_preview_condition: - runtime._invalidated_accepted_block_payout_previews.pop(key, None) - existing = runtime._accepted_block_payout_previews.get(key) - if existing is None: - runtime._accepted_block_payout_previews[key] = ( - AcceptedBlockPayoutTransition(block_height=block_height) - ) - elif ( - block_height is not None - and existing.block_height is not None - and existing.block_height != block_height - ): - raise RuntimeError("accepted block payout transition height changed") - elif existing.block_height is None and block_height is not None: - runtime._accepted_block_payout_previews[key] = dataclass_replace( - existing, - block_height=block_height, - ) - - def _mark_accepted_block_payout_landed( - self, - block_hash: str, - *, - block_height: int, - ) -> None: - """Bar reconciliation after submitblock makes a candidate active.""" - runtime = self._runtime - runtime._ensure_job_cache_state() - key = block_hash.lower() - with runtime._accepted_block_payout_preview_condition: - existing = runtime._accepted_block_payout_previews.get( - key, - AcceptedBlockPayoutTransition(block_height=block_height), - ) - if existing.block_height not in {None, block_height}: - raise RuntimeError("accepted block payout transition height changed") - runtime._accepted_block_payout_previews[key] = dataclass_replace( - existing, - block_height=block_height, - landed=True, - landed_monotonic=( - existing.landed_monotonic - if existing.landed_monotonic is not None - else time.monotonic() - ), - ) - runtime._accepted_block_payout_preview_condition.notify_all() - - def _unmark_accepted_block_payout_landed(self, block_hash: str) -> None: - """Withdraw a landed bar armed for an attempt that never reached RPC. - - Only sound while the submit outcome is NOT uncertain: the caller - must know submitblock was never invoked under this arming (a - preflight refusal such as ``WriterLeaseRenewalDeferred``), so there - is no possibly-active coinbase for the barrier to preserve and no - reason to keep reconciliation and payout-state publication barred - while the candidate waits for its retry. The pending preview entry - survives — child work keeps waiting instead of snapshotting - pre-accept balances, matching the startup-replay state a durable - candidate restores — and the next attempt re-arms ``landed`` before - its own RPC. - """ - runtime = self._runtime - runtime._ensure_job_cache_state() - key = block_hash.lower() - with runtime._accepted_block_payout_preview_condition: - existing = runtime._accepted_block_payout_previews.get(key) - if existing is None or not existing.landed: - return - runtime._accepted_block_payout_previews[key] = dataclass_replace( - existing, - landed=False, - landed_monotonic=None, - ) - runtime._accepted_block_payout_preview_condition.notify_all() - - def _publish_accepted_block_payout_preview( - self, - block_hash: str, - balances: list[dict[str, object]], - ) -> list[dict[str, object]]: - """Publish the balances child work must observe after confirmation. - - Canonicalization happens before the delivery boundary. The gate only - installs an immutable pointer and advances the generation, so no job - bound to the pre-accept balances can land after publication. - """ - runtime = self._runtime - normalized = runtime.normalized_prior_balances(balances) - serialized = runtime._serialize_prior_balance_preview(normalized) - key = block_hash.lower() - with runtime._block_submitter_lock( - runtime._payout_balance_mutation_lock, - "payout-balance-mutation", - ): - with runtime._accepted_block_payout_preview_condition: - existing = runtime._accepted_block_payout_previews.get(key) - existing_preview = existing.preview if existing is not None else None - if existing_preview is not None: - if existing_preview != serialized: - raise RuntimeError( - "accepted block payout preview changed during retry" - ) - if ( - existing is not None - and existing.published_generation is not None - ): - # A matching preview that already carries a published - # generation is visible to waiting children, so this - # call is the publication for issue #181's interval - # when it is the first one seen for the hash; the - # observer drops it otherwise. - runtime._observe_accepted_block_preview_publication( - key, - result="published", - ) - return runtime._materialize_prior_balance_preview( - existing_preview - ) - # A bounded publication loss retains the compact preview - # locally while delivery remains fenced. Matching retries - # must still cross the atomic publication boundary so they - # install a generation and reopen admission. - - published = None - try: - captured = runtime._capture_payout_state_source() - reserved = runtime._reserve_payout_state_source_if_current( - captured[1], - "accepted_block_preview", - tip_hash=key, - invalidated_monotonic=time.monotonic(), - ) - if reserved is None: - candidate = runtime._prepared_payout_state_candidate( - runtime._capture_payout_state_source(), - bypass_build_interval=True, - ) - else: - candidate = runtime._prepared_payout_state_candidate(reserved) - candidate = runtime._accepted_block_preview_candidate( - candidate, - block_hash=key, - preview=serialized, - ) - # Close delivery admission before exposing the preview. The - # generation publication below performs the only atomic pointer - # swap and reopens admission; no preparation lock is involved. - runtime._block_payout_state_publication(force=True) - published = runtime._publish_payout_state_candidate(candidate) - if published is None: - max_retries = max( - 0, - int( - getattr( - runtime, - "payout_reconcile_supersession_retries", - DEFAULT_PRISM_PAYOUT_RECONCILE_SUPERSESSION_RETRIES, - ) - ), - ) - for _attempt in range(max_retries): - candidate = runtime._accepted_block_preview_candidate( - runtime._prepared_payout_state_candidate( - runtime._capture_payout_state_source(), - bypass_build_interval=True, - ), - block_hash=key, - preview=serialized, - ) - published = runtime._publish_payout_state_candidate(candidate) - if published is not None: - break - except Exception as exc: - # Last-resort degraded retention (issue #188): candidate - # preparation already degrades a ledger-timeout window build - # to the cached armed window, so reaching here means an - # in-memory publication step itself failed. Fall through to - # the fenced local-retention branch below exactly like a - # lost atomic publication: admission is force-blocked, the - # immutable compact preview remains visible to children - # already waiting on the transition, and the tip-refresh - # retry loop must publish a fresh source before new job - # admission resumes. - published = None - print( - "prism coordinator: accepted block payout preview " - f"publication degraded hash={key}: {exc}", - flush=True, - ) - if published is None: - runtime._block_payout_state_publication(force=True) - # Admission is now globally fenced, so retaining the compact - # preview locally is safe even though it did not win an atomic - # generation publication. Finalization may finish; the normal - # retry loop will publish the newest source before delivery - # resumes. - with runtime._accepted_block_payout_preview_condition: - transition = runtime._accepted_block_payout_previews.get( - key, - AcceptedBlockPayoutTransition(landed=True), - ) - runtime._accepted_block_payout_previews[key] = dataclass_replace( - transition, - landed=True, - preview=serialized, - published_generation=None, - ) - runtime._accepted_block_payout_preview_condition.notify_all() - # Both exits above make the preview observable to children already - # waiting on the transition -- the generation publication through the - # atomic pointer swap, the fenced retention branch through the - # compact preview plus notify_all -- so both close issue #181's - # acceptance-to-publication interval. They are labelled apart because - # only one of them installed a generation. - runtime._observe_accepted_block_preview_publication( - key, - result="published" if published is not None else "degraded", - ) - return runtime._materialize_prior_balance_preview(serialized) - - def _accepted_block_preview_candidate( - self, - candidate: PayoutStateCandidate, - *, - block_hash: str, - preview: tuple[tuple[str, str, str, int], ...], - ) -> PayoutStateCandidate: - """Bind a compact preview to its prepared artifact before gating.""" - runtime = self._runtime - ledger_artifact = candidate.ledger_artifact - if ledger_artifact is not None: - # The artifact was prepared before accepted-block persistence, so - # replace only its balance view with the verified prospective - # state. This allocation must stay outside delivery publication. - # The memoized balances digest must be re-derived with the patch: - # a stale digest would fail the reuse probe's balances fence - # against the very state this patch installs. - patched_balances = tuple( - runtime._materialize_prior_balance_preview(preview) - ) - ledger_artifact = dataclass_replace( - ledger_artifact, - prior_balances=patched_balances, - prior_balances_sha256=self._canonical_json_sha256(patched_balances), - ) - return dataclass_replace( - candidate, - accepted_block_hash=block_hash, - accepted_block_preview=preview, - ledger_artifact=ledger_artifact, - ) - - @staticmethod - def _serialize_prior_balance_preview( - balances: list[dict[str, object]], - ) -> tuple[tuple[str, str, str, int], ...]: - return tuple( - ( - str(balance["recipient_id"]), - str(balance["order_key"]), - str(balance["p2mr_program_hex"]), - int(balance["balance_sats"]), - ) - for balance in balances - ) - - def _accepted_block_payout_preview_from_bundle( - self, - final_bundle: dict[str, Any], - *, - prior_balances: list[dict[str, object]] | None = None, - ) -> list[dict[str, object]]: - """Derive the confirmed carry-forward view from a verified bundle. - - The bundle may be a dictionary or a bounded artifact view whose - ``accounts`` member is a lazy on-disk sequence (issue #255); the - accounts are walked once, row by row, in either case. - """ - runtime = self._runtime - manifest = final_bundle.get("payout_policy_manifest") - accounts = manifest.get("accounts") if isinstance(manifest, Mapping) else None - if ( - accounts is None - or isinstance(accounts, (str, bytes, bytearray)) - or not isinstance(accounts, Sequence) - ): - raise RuntimeError("accepted block payout manifest is missing accounts") - prior_identities: dict[str, tuple[str, str]] = {} - for balance in prior_balances or []: - program = str(balance.get("p2mr_program_hex", "")).lower() - identity = ( - str(balance.get("order_key", "")), - str(balance.get("recipient_id", "")), - ) - prior_identities[program] = min( - identity, - prior_identities.get(program, identity), - ) - balances: list[dict[str, object]] = [] - for account in accounts: - if not isinstance(account, Mapping): - continue - if str(account.get("account_type", "miner")) == "pool_fee": - continue - balance_sats = int(account.get("carry_forward_balance_sats", 0)) - if balance_sats == 0: - continue - program = str(account.get("p2mr_program_hex", "")).lower() - account_identity = ( - str(account.get("order_key", "")), - str(account.get("recipient_id", "")), - ) - order_key, recipient_id = min( - account_identity, - prior_identities.get(program, account_identity), - ) - balances.append( - { - "recipient_id": recipient_id, - "order_key": order_key, - "p2mr_program_hex": program, - "balance_sats": balance_sats, - } - ) - return runtime.normalized_prior_balances(balances) - - @staticmethod - def _materialize_prior_balance_preview( - preview: tuple[tuple[str, str, str, int], ...], - ) -> list[dict[str, object]]: - return [ - { - "recipient_id": recipient_id, - "order_key": order_key, - "p2mr_program_hex": p2mr_program_hex, - "balance_sats": balance_sats, - } - for recipient_id, order_key, p2mr_program_hex, balance_sats in preview - ] - - def _clear_accepted_block_payout_preview( - self, - block_hash: str, - *, - invalidate_published: bool = False, - ) -> None: - runtime = self._runtime - runtime._ensure_job_cache_state() - key = block_hash.lower() - with runtime._block_submitter_lock( - runtime._payout_balance_mutation_lock, - "payout-balance-mutation", - ): - with runtime._accepted_block_payout_preview_condition: - existing = runtime._accepted_block_payout_previews.get(key) - if existing is None: - if not invalidate_published: - runtime._invalidated_accepted_block_payout_previews.pop( - key, - None, - ) - runtime._accepted_block_payout_preview_condition.notify_all() - return - if not invalidate_published: - # Durable state now equals the published prospective view; - # removing the override changes no logical payout state. - runtime._accepted_block_payout_previews.pop(key, None) - runtime._invalidated_accepted_block_payout_previews.pop(key, None) - runtime._accepted_block_payout_preview_condition.notify_all() - return - if existing.preview is None: - # Nothing crossed a generation boundary. A landed - # transition still becomes a tombstone so descendants - # cannot fall through to an uncertain database snapshot. - runtime._accepted_block_payout_previews.pop(key, None) - if existing.landed: - runtime._invalidated_accepted_block_payout_previews[key] = ( - existing.block_height - ) - runtime._accepted_block_payout_preview_condition.notify_all() - return - - captured = runtime._capture_payout_state_source() - reserved = runtime._reserve_payout_state_source_if_current( - captured[1], - "accepted_block_preview_withdrawn", - tip_hash=captured[2], - invalidated_monotonic=time.monotonic(), - ) - candidate = runtime._prepared_payout_state_candidate( - reserved if reserved is not None else runtime._capture_payout_state_source(), - force_full_window_rescan=True, - bypass_build_interval=True, - ) - candidate = dataclass_replace( - candidate, - accepted_block_hash=key, - accepted_block_withdrawal=True, - accepted_block_height=existing.block_height, - ) - runtime._block_payout_state_publication(force=True) - published = runtime._publish_payout_state_candidate(candidate) - if published is None: - max_retries = max( - 0, - int( - getattr( - runtime, - "payout_reconcile_supersession_retries", - DEFAULT_PRISM_PAYOUT_RECONCILE_SUPERSESSION_RETRIES, - ) - ), - ) - for _attempt in range(max_retries): - candidate = dataclass_replace( - runtime._prepared_payout_state_candidate( - runtime._capture_payout_state_source(), - force_full_window_rescan=True, - bypass_build_interval=True, - ), - accepted_block_hash=key, - accepted_block_withdrawal=True, - accepted_block_height=existing.block_height, - ) - published = runtime._publish_payout_state_candidate(candidate) - if published is not None: - break - if published is None: - # Delivery remains fenced. Install the tombstone immediately - # so even local builders cannot fall through while the newest - # source waits for a retry publication. - with runtime._accepted_block_payout_preview_condition: - runtime._accepted_block_payout_previews.pop(key, None) - runtime._invalidated_accepted_block_payout_previews[key] = ( - existing.block_height - ) - runtime._accepted_block_payout_preview_condition.notify_all() - - def _accepted_block_payout_transition_landed(self, block_hash: str) -> bool: - runtime = self._runtime - runtime._ensure_job_cache_state() - with runtime._accepted_block_payout_preview_condition: - transition = runtime._accepted_block_payout_previews.get(block_hash.lower()) - return transition is not None and transition.landed - - def _accepted_block_payout_transition_for_parent( - self, - parent_hash: str, - *, - parent_height: int | None = None, - ) -> tuple[str, bool] | None: - """Select the highest active exact/ancestor payout transition. - - The boolean result is true for an invalidated landed transition. The - caller decides whether to wait for a live preview or fail closed on its - tombstone. - """ - runtime = self._runtime - runtime._ensure_job_cache_state() - key = parent_hash.lower() - with runtime._accepted_block_payout_preview_condition: - exact_transition = runtime._accepted_block_payout_previews.get(key) - exact_invalidated = ( - key in runtime._invalidated_accepted_block_payout_previews - ) - fail_closed_candidate_hashes = { - candidate_hash - for candidate_hash, transition in ( - runtime._accepted_block_payout_previews.items() - ) - if transition.landed - } - fail_closed_candidate_hashes.update( - runtime._invalidated_accepted_block_payout_previews - ) - ancestor_candidates = [ - (candidate_hash, transition.block_height, False) - for candidate_hash, transition in runtime._accepted_block_payout_previews.items() - if exact_transition is None - and not exact_invalidated - and transition.block_height is not None - and parent_height is not None - and transition.block_height <= parent_height - ] - ancestor_candidates.extend( - ( - candidate_hash, - candidate_height, - True, - ) - for candidate_hash, candidate_height in ( - runtime._invalidated_accepted_block_payout_previews.items() - ) - if exact_transition is None - and not exact_invalidated - and candidate_height is not None - and parent_height is not None - and candidate_height <= parent_height - ) - if exact_transition is not None or exact_invalidated: - return key, exact_invalidated - if not ancestor_candidates: - return None - - active_ancestors: list[tuple[int, str, bool]] = [] - # One probe per distinct height, not per candidate. Every offered or - # replay-adopted block candidate arms a preview barrier, and siblings - # of one solved block all share a height -- so a candidate storm makes - # this loop issue thousands of identical getblockhash calls for a - # single answer, twice per built bundle, on the path every child build - # takes before it can wait for a preview. - # - # Scoped to this scan deliberately, and not memoized across calls: the - # active block at a height changes under reorgs, and a longer-lived - # cache would answer with a chain that no longer exists. Within one - # scan a single view is what the comparison below already assumes. - active_hash_by_height: dict[int, str] = {} - try: - for ( - candidate_hash, - candidate_height, - candidate_invalidated, - ) in ancestor_candidates: - assert candidate_height is not None - active_hash = active_hash_by_height.get(candidate_height) - if active_hash is None: - active_hash = str( - runtime.rpc.call("getblockhash", [candidate_height]) - ).lower() - active_hash_by_height[candidate_height] = active_hash - if active_hash == candidate_hash: - active_ancestors.append( - ( - candidate_height, - candidate_hash, - candidate_invalidated, - ) - ) - except Exception as exc: - runtime._schedule_tip_refresh_retry() - raise TemplateRefreshBlocked( - "could not validate an accepted payout preview on the active chain" - ) from exc - if not active_ancestors: - if any( - candidate_hash in fail_closed_candidate_hashes - for candidate_hash, _candidate_height, _invalidated in ( - ancestor_candidates - ) - ): - # A prepared artifact published with an accepted preview carries - # that prospective balance view. If its block left the active - # chain between reconciliation and this build, falling through - # to the artifact would stamp unrelated work with those balances. - # Wait for withdrawal/reconciliation to publish a new payout - # generation instead of using either the artifact or live DB. - runtime._schedule_tip_refresh_retry() - raise PayoutStatePublicationBlocked( - "accepted payout transition is no longer active" - ) - return None - _, selected_key, selected_invalidated = max(active_ancestors) - return selected_key, selected_invalidated - - def _accepted_parent_unresolved_depth_cap(self) -> int: - runtime = self._runtime - return max( - 1, - int( - getattr( - runtime, - "accepted_parent_unresolved_depth_max", - DEFAULT_ACCEPTED_PARENT_UNRESOLVED_DEPTH_MAX, - ) - ), - ) - - def accepted_parent_unresolved_ages_seconds(self) -> list[float]: - """Ages of landed transitions whose bookkeeping is still unresolved.""" - runtime = self._runtime - runtime._ensure_job_cache_state() - now = time.monotonic() - with runtime._accepted_block_payout_preview_condition: - return [ - max(0.0, now - transition.landed_monotonic) - for transition in runtime._accepted_block_payout_previews.values() - if transition.landed and transition.landed_monotonic is not None - ] - - def _accepted_parent_unresolved_depth(self) -> int: - runtime = self._runtime - runtime._ensure_job_cache_state() - with runtime._accepted_block_payout_preview_condition: - return sum( - 1 - for transition in runtime._accepted_block_payout_previews.values() - if transition.landed - ) - - def _await_pending_parent_payout_preview( - self, - parent_hash: str, - *, - parent_height: int | None = None, - ) -> list[dict[str, object]] | None: - """Wait out a pending accepted-parent transition, returning its preview. - - Returns None when no landed transition governs this parent, without - touching any confirmed payout input. Ordering this wait before every - confirmed read lets children of a pending parent block here instead of - taking (or caching) balances that omit their new parent. - """ - runtime = self._runtime - # Startup-replay window (issue #188 fix 4): before the durable outbox - # has been enumerated, an unarmed pending parent may exist. A child - # built now could pay balances that omit that parent's carry, so job - # builds fail closed until enumeration succeeds. - if runtime._block_replay_enumeration_owed(): - runtime._schedule_tip_refresh_retry() - raise TemplateRefreshBlocked( - "durable block candidate replay has not enumerated pending " - "candidates yet" - ) - selected = runtime._accepted_block_payout_transition_for_parent( - parent_hash, - parent_height=parent_height, - ) - if selected is None: - return None - # Fail closed once too many landed transitions remain unresolved: - # every published preview a child consumes stacks another prospective - # balance chain on unfinished bookkeeping, so the depth of that stack - # must stay bounded even while degraded previews keep jobs flowing. - # Issuance stops at the cap, not past it: a child issued while - # exactly depth_cap transitions are unresolved could land and create - # a (cap + 1)th, so the configured maximum would not actually bound - # the chain. - unresolved_depth = runtime._accepted_parent_unresolved_depth() - depth_cap = runtime._accepted_parent_unresolved_depth_cap() - if unresolved_depth >= depth_cap: - runtime._schedule_tip_refresh_retry() - raise TemplateRefreshBlocked( - "unresolved accepted-parent depth " - f"{unresolved_depth} meets or exceeds cap {depth_cap}" - ) - selected_key, selected_invalidated = selected - if selected_invalidated: - runtime._schedule_tip_refresh_retry() - raise TemplateRefreshBlocked( - "accepted parent payout preview was withdrawn" - ) - - wait_seconds = max( - 0.0, - float( - getattr( - runtime, - "accepted_block_payout_preview_wait_seconds", - DEFAULT_ACCEPTED_BLOCK_PAYOUT_PREVIEW_WAIT_SECONDS, - ) - ), - ) - deadline = time.monotonic() + wait_seconds - timed_out = False - invalidated = False - with runtime._accepted_block_payout_preview_condition: - while selected_key in runtime._accepted_block_payout_previews: - transition = runtime._accepted_block_payout_previews[selected_key] - if transition.preview is not None: - return runtime._materialize_prior_balance_preview( - transition.preview - ) - if runtime.stop_event.is_set(): - raise RuntimeError( - "coordinator stopped while accepted payout preview was pending" - ) - remaining = deadline - time.monotonic() - if remaining <= 0: - timed_out = True - break - runtime._accepted_block_payout_preview_condition.wait( - timeout=min(0.25, remaining) - ) - invalidated = ( - selected_key in runtime._invalidated_accepted_block_payout_previews - ) - if invalidated: - runtime._schedule_tip_refresh_retry() - raise TemplateRefreshBlocked( - "accepted parent payout preview was withdrawn" - ) - if timed_out: - with runtime.lock: - timeout_count = ( - int(getattr(runtime, "_accepted_parent_preview_wait_timeouts", 0)) - + 1 - ) - runtime._accepted_parent_preview_wait_timeouts = timeout_count - runtime._schedule_tip_refresh_retry() - # Typed backpressure, not a failure: the transition is armed and - # un-withdrawn, so the parent's preview is merely late. The typed - # raise keeps this out of the ordinary template-refresh failure - # budget while the counter above (and the coordination-blocked - # streak the delivery boundaries feed) keeps sustained pendency - # visible and still escalating. - raise AcceptedParentPayoutPreviewPending( - "accepted parent payout preview is not ready yet" - f" (parent={selected_key} waited={wait_seconds:.3f}s" - f" timeouts={timeout_count})", - parent_hash=selected_key, - waited_seconds=wait_seconds, - timeout_count=timeout_count, - ) - # The transition reached a terminal durable state while waiting; the - # caller's confirmed fallback now includes the parent. - return None - - def _prior_balances_for_job_parent( - self, - parent_hash: str, - *, - parent_height: int | None = None, - fallback_balances: Sequence[dict[str, object]] | None = None, - ) -> list[dict[str, object]]: - """Return prospective balances, otherwise a prepared/ledger fallback.""" - runtime = self._runtime - preview = runtime._await_pending_parent_payout_preview( - parent_hash, - parent_height=parent_height, - ) - if preview is not None: - return preview - return ( - list(fallback_balances) - if fallback_balances is not None - else runtime.ledger.current_prior_balances() - ) - - def _observe_payout_state_seconds( - self, - name: str, - elapsed_seconds: float, - *, - relation: str | None = None, - ) -> None: - runtime = self._runtime - runtime._ensure_job_cache_state() - with runtime._payout_state_metrics_lock: - if name == "gate_wait": - if relation not in PRISM_PAYOUT_DELIVERY_GENERATIONS: - raise ValueError(f"unknown payout delivery generation: {relation}") - histogram = runtime.payout_gate_wait_histograms[str(relation)] - else: - histogram = runtime.payout_state_histograms[name] - histogram["count"] = int(histogram["count"]) + 1 - histogram["sum"] = float(histogram["sum"]) + elapsed_seconds - buckets = histogram["buckets"] - assert isinstance(buckets, dict) - for bucket in PRISM_PAYOUT_SECONDS_BUCKETS: - if elapsed_seconds <= bucket: - buckets[bucket] = int(buckets.get(bucket, 0)) + 1 - - def _observe_payout_gate_admission( - self, - admission: object, - *, - generation: int, - fallback_wait_seconds: float, - ) -> None: - runtime = self._runtime - runtime._ensure_job_cache_state() - with runtime._job_cache_lock: - published_generation = runtime._payout_state_generation - relation = getattr(admission, "relation", None) - if relation not in PRISM_PAYOUT_DELIVERY_GENERATIONS: - relation = PayoutStateDeliveryGate._generation_relation( - generation, - published_generation, - ) - wait_seconds = float( - getattr(admission, "wait_seconds", fallback_wait_seconds) - ) - runtime._observe_payout_state_seconds( - "gate_wait", - max(0.0, wait_seconds), - relation=relation, - ) - - def _reserve_payout_state_source( - self, - cause: str, - *, - tip_hash: str | None = None, - invalidated_monotonic: float | None = None, - ) -> int: - runtime = self._runtime - runtime._ensure_job_cache_state() - invalidated = ( - time.monotonic() - if invalidated_monotonic is None - else invalidated_monotonic - ) - with runtime.lock: - generation = runtime._payout_state_source[0] + 1 - runtime._payout_state_source = ( - generation, - tip_hash, - cause, - invalidated, - ) - return generation - - def _reserve_payout_state_source_if_current( - self, - expected_source_generation: int, - cause: str, - *, - tip_hash: str | None = None, - invalidated_monotonic: float | None = None, - ) -> tuple[int, int, str | None, str, float] | None: - """Reserve and capture a source only if preparation was not superseded.""" - runtime = self._runtime - - runtime._ensure_job_cache_state() - invalidated = ( - time.monotonic() - if invalidated_monotonic is None - else invalidated_monotonic - ) - # Match publication's lock order so the returned base generation and - # newly reserved source form one atomic candidate identity. - with runtime._job_cache_lock: - with runtime.lock: - if runtime._payout_state_source[0] != expected_source_generation: - return None - source_generation = expected_source_generation + 1 - runtime._payout_state_source = ( - source_generation, - tip_hash, - cause, - invalidated, - ) - # The reserver's entitlement to act on this cause travels - # with the tuple it gets back (issue #228). - return _ReservedPayoutSource( - runtime._payout_state_generation, - source_generation, - tip_hash, - cause, - invalidated, - ) - - def _capture_payout_state_source( - self, - ) -> tuple[int, int, str | None, str, float]: - runtime = self._runtime - runtime._ensure_job_cache_state() - with runtime.lock: - source_generation, source_tip, cause, invalidated = ( - runtime._payout_state_source - ) - with runtime._job_cache_lock: - base_generation = runtime._payout_state_generation - return ( - base_generation, - source_generation, - source_tip, - cause, - invalidated, - ) - - def _consume_payout_source_cause( - self, - captured: tuple[int, int, str | None, str, float], - ) -> bool: - """Whether this preparation may act on the source's reserved cause. - - Two rules, and nothing shared between them that a pruning policy - could reach: - - - The reserver's own preparation presents the ``_ReservedPayoutSource`` - its reservation returned, and acts on the cause exactly once, - whatever other preparations have done at any generation since. - The entitlement lives in that object, so a foreign preparation - at a later generation cannot defeat it; its candidate may be - doomed at the publication check, but its build keeps the - found-block immediacy and degrade of issue #188. - - A plain capture acts only if its source generation is above the - high-water mark of everything acted on so far, and moves the mark - there. At the mark, someone already acted (the mark got there by - acting); below it, the capture is stale. Neither ever rewinds - the mark, so a stale preparation cannot re-enable a newer - generation's cause. - - The reserver's turn also lifts the mark to its generation when that - is newer, so a later capture at the same generation inherits. The - tuple itself is never cleared: candidate supersession and the - finalizer's pending-cause read depend on it staying readable. - """ - runtime = self._runtime - source_generation = int(captured[1]) - with runtime.lock: - high_water = self._payout_source_cause_high_water - newer = high_water is None or source_generation > high_water - if isinstance(captured, _ReservedPayoutSource): - if not captured.consume_entitlement(): - return False - if newer: - self._payout_source_cause_high_water = source_generation - return True - if newer: - self._payout_source_cause_high_water = source_generation - return True - return False - - def _begin_window_build_phases(self) -> dict[str, float]: - phases = {phase: 0.0 for phase in PRISM_PAYOUT_WINDOW_BUILD_PHASES} - self._payout_window_build_context.phases = phases - return phases - - def _finish_window_build_phases(self) -> None: - self._payout_window_build_context.phases = None - - def _note_window_build_phase(self, phase: str, elapsed_seconds: float) -> None: - """Attribute one timed slice of the build in progress on this thread.""" - if phase not in PRISM_PAYOUT_WINDOW_BUILD_PHASES: - raise ValueError(f"unknown payout window build phase: {phase}") - phases = getattr(self._payout_window_build_context, "phases", None) - if phases is not None: - phases[phase] = float(phases.get(phase, 0.0)) + max( - 0.0, float(elapsed_seconds) - ) - - def _observe_window_build_phases( - self, - phases: dict[str, float], - outcome: str, - ) -> None: - if outcome not in PRISM_PAYOUT_WINDOW_BUILD_OUTCOMES: - raise ValueError(f"unknown payout window build outcome: {outcome}") - runtime = self._runtime - with runtime._payout_state_metrics_lock: - for phase in PRISM_PAYOUT_WINDOW_BUILD_PHASES: - elapsed = max(0.0, float(phases.get(phase, 0.0))) - histogram = self.payout_window_build_phase_histograms[ - (phase, outcome) - ] - histogram["count"] = int(histogram["count"]) + 1 - histogram["sum"] = float(histogram["sum"]) + elapsed - buckets = histogram["buckets"] - assert isinstance(buckets, dict) - for bucket in PRISM_PAYOUT_SECONDS_BUCKETS: - if elapsed <= bucket: - buckets[bucket] = int(buckets.get(bucket, 0)) + 1 - - def _cached_tip_refresh_payout_artifact( - self, - *, - base_generation: int, - artifact_payout_state_generation: int, - network_difficulty: int, - ) -> PayoutLedgerArtifact | None: - """Reuse the armed window for a routine candidate when the lock is busy. - - The tip-refresh path's counterpart of the found-block degrade - (issue #188): a routine candidate must not queue behind another - build's oracle while an exact, still-valid armed window exists. - Reuse holds exactly where the delivery path's own reuse holds -- - current payout generation, current append epoch, balances equal - to the published bytes -- narrowed to an anchor no older than - PRISM_TIP_REFRESH_WINDOW_STALENESS_SECONDS and a live difficulty - inside the snapshot-weight band the incremental window already - serves across (in-band ASERT drift keeps the cached weight; the - artifact is tagged with the template difficulty either way). - Balances are the published bytes, which is what a routine build - would reuse itself. Anything else returns None and the caller - queues for a fresh build as before. - """ - runtime = self._runtime - if not runtime._payout_artifact_reuse_active(): - return None - with runtime._job_cache_lock: - artifact = runtime._payout_ledger_artifact - published_artifact = runtime._published_payout_state.artifact - generation_current = base_generation == runtime._payout_state_generation - append_epoch_current = bool( - artifact is not None - and artifact.append_invalidation_epoch - == runtime._payout_ledger_append_invalidation_epoch - ) - if ( - artifact is None - or published_artifact is None - or not generation_current - or not append_epoch_current - or artifact.payout_state_generation != base_generation - or artifact.snapshot_anchor_ms is None - ): - return None - anchor_age_ms = self._now_ms() - int(artifact.snapshot_anchor_ms) - if anchor_age_ms > PRISM_TIP_REFRESH_WINDOW_STALENESS_SECONDS * 1000.0: - return None - weight_per_difficulty = ( - PRISM_REWARD_WINDOW_MULTIPLIER * PRISM_SNAPSHOT_WINDOW_MARGIN - ) - if not _snapshot_window_weight_within_band( - weight_per_difficulty * int(artifact.network_difficulty), - weight_per_difficulty * int(network_difficulty), - ): - return None - balances_sha256 = artifact.prior_balances_sha256 or self._canonical_json_sha256( - artifact.prior_balances - ) - if balances_sha256 != published_artifact.prior_balances_sha256: - return None - reused = dataclass_replace( - artifact, - generation=0, - payout_state_generation=int(artifact_payout_state_generation), - network_difficulty=int(network_difficulty), - prepared_monotonic=time.monotonic(), - window_build_mode="tip_refresh_cached", - window_delta_rows=0, - window_expired_rows=0, - window_touched_pages=0, - window_build_seconds=0.0, - window_full_rescan_reason="prepare_lock_busy", - ) - runtime._record_payout_artifact_event("tip_refresh_cached") - runtime._payout_artifact_log( - "payout_artifact_tip_refresh_cached", - payout_state_generation=int(artifact_payout_state_generation), - duration_seconds=0.0, - window_build_mode="tip_refresh_cached", - window_shares=len(reused.shares_json), - anchor_age_ms=anchor_age_ms, - during_publication=True, - build_interval_bypassed=False, - fallback_reason="prepare_lock_busy", - ) - return reused - - def _prepared_payout_state_candidate( - self, - captured: tuple[int, int, str | None, str, float], - *, - force_full_window_rescan: bool = False, - force_prior_balances_read: bool = False, - bypass_build_interval: bool = False, - ) -> PayoutStateCandidate: - """Prepare a candidate, optionally forcing the window or balance read. - - ``force_full_window_rescan`` keeps its full fail-closed meaning (the - withdrawal path and every explicit caller). ``force_prior_balances_read`` - is the reconciler's intent after a confirmed payout mutation: reuse - the exact armed share window and reread the carry aggregate (#224). - """ - runtime = self._runtime - base_generation, source_generation, source_tip, cause, invalidated = captured - ledger_artifact: PayoutLedgerArtifact | None = None - with runtime._job_cache_lock: - template_artifacts = runtime._template_artifacts - if ( - template_artifacts is not None - and getattr(runtime, "_pool_ready_latched", False) - ): - # A reserved cause is acted on once, by the preparation that - # consumed it. Before this, a withdrawal's cause stayed readable - # in the sticky source tuple until the next tip displaced it, - # and every routine capture in between forced the full oracle - # for a window nothing had changed (issue #228). - cause_honored = self._consume_payout_source_cause(captured) - decisive_cause = cause if cause_honored else None - cause_forces_rescan = ( - decisive_cause == "accepted_block_preview_withdrawn" - ) - force_full = force_full_window_rescan or cause_forces_rescan - # Either flag invalidates the published balances; only the - # fail-closed full rescan touches the share window. The #224 - # intent (``force_prior_balances_read``) is what reconciliation - # asks for after a confirmed mutation, because it moves - # pool-block, payout-entry and carry rows and never the - # append-only share ledger. - payout_mutated = force_full or force_prior_balances_read - caller_flag = force_full_window_rescan or force_prior_balances_read - if caller_flag and cause_forces_rescan: - force_origin = "caller_flag+reserved_cause" - elif caller_flag: - force_origin = "caller_flag" - elif cause_forces_rescan: - force_origin = "reserved_cause" - else: - force_origin = "none" - bypass_interval = ( - bypass_build_interval - or decisive_cause - in { - "accepted_block_preview", - "accepted_block_preview_withdrawn", - } - ) - immediate_preview = ( - decisive_cause == "accepted_block_preview" - or ( - bypass_build_interval - and not force_full_window_rescan - and decisive_cause != "accepted_block_preview_withdrawn" - ) - ) - routine = not payout_mutated and not immediate_preview - self._payout_window_build_context.attribution = ( - _PayoutWindowBuildAttribution( - cause=cause, - cause_inherited=not cause_honored, - force_origin=force_origin, - critical_path=routine, - ) - ) - # Forwarded only when set: the build seam predates the intent - # and per-instance patches of it keep their exact signature. - build_intent: dict[str, bool] = ( - {"force_prior_balances_read": True} - if force_prior_balances_read - else {} - ) - try: - prepare_lock_acquired = True - lock_acquired_here = False - reused_armed_window = False - if immediate_preview or routine: - # A periodic oracle may already own this lock and its - # Postgres query cannot be cancelled safely. Found-block - # publication must not queue behind that 11--36 second - # scan (issue #188): use the last exact, still-valid - # armed window and patch its balances with the verified - # preview below. A routine tip-refresh candidate follows - # the same rule while an armed window inside the - # accepted staleness bound exists (issue #228); without - # one it queues for a fresh build as before. - prepare_lock_acquired = runtime._payout_state_prepare_lock.acquire( - blocking=False - ) - lock_acquired_here = prepare_lock_acquired - if routine and not prepare_lock_acquired: - ledger_artifact = self._cached_tip_refresh_payout_artifact( - base_generation=base_generation, - artifact_payout_state_generation=base_generation + 1, - network_difficulty=template_artifacts.network_difficulty, - ) - reused_armed_window = ledger_artifact is not None - prepare_lock_acquired = not reused_armed_window - if prepare_lock_acquired and not reused_armed_window: - try: - ledger_artifact = runtime._build_payout_ledger_artifact( - base_generation, - base_generation + 1, - template_artifacts.network_difficulty, - force_full, - bypass_interval, - True, - **build_intent, - ) - except self._shutdown_error: - raise - except Exception as exc: - if not immediate_preview: - raise - # Found-block publication exists to decouple child - # job issuance from slow landing bookkeeping (issue - # #188). A window build that dies with the ledger - # must not abort the atomic publication that reopens - # admission: degrade to the cached armed window - # below, exactly like a busy prepare lock. The - # preview itself never depends on this build; it is - # the verified balance state either way. - print( - "prism coordinator: found-block payout window build " - f"failed; degrading to cached armed window: {exc}", - flush=True, - ) - ledger_artifact = None - finally: - if lock_acquired_here: - runtime._payout_state_prepare_lock.release() - elif not reused_armed_window: - ledger_artifact = None - finally: - self._payout_window_build_context.attribution = None - if immediate_preview and ledger_artifact is None: - ledger_artifact = runtime._cached_found_block_payout_artifact( - base_generation=base_generation, - artifact_payout_state_generation=base_generation + 1, - network_difficulty=template_artifacts.network_difficulty, - fallback_reason=( - "build_failed" - if prepare_lock_acquired - else "prepare_lock_busy" - ), - ) - return PayoutStateCandidate( - base_generation=base_generation, - source_generation=source_generation, - source_tip_hash=source_tip, - cause=cause, - invalidated_monotonic=invalidated, - prepared_monotonic=time.monotonic(), - ledger_artifact=ledger_artifact, - ) - - def _cached_found_block_payout_artifact( - self, - *, - base_generation: int, - artifact_payout_state_generation: int, - network_difficulty: int, - fallback_reason: str, - ) -> PayoutLedgerArtifact | None: - """Retag an exact armed window when immediate preparation is unavailable. - - The caller always replaces its balance view with the accepted block's - verified prospective balances before publication. Only the share - window and its declared anchor are reused here. - """ - runtime = self._runtime - - with runtime._job_cache_lock: - artifact = runtime._payout_ledger_artifact - generation_current = base_generation == runtime._payout_state_generation - append_epoch_current = bool( - artifact is not None - and artifact.append_invalidation_epoch - == runtime._payout_ledger_append_invalidation_epoch - ) - anchor_age_ms = ( - None - if artifact is None or artifact.snapshot_anchor_ms is None - else self._now_ms() - int(artifact.snapshot_anchor_ms) - ) - if ( - not generation_current - or not append_epoch_current - or artifact is None - or artifact.payout_state_generation != base_generation - or artifact.network_difficulty != int(network_difficulty) - or anchor_age_ms is None - or anchor_age_ms > runtime._payout_artifact_max_anchor_age_ms() - ): - runtime._record_payout_artifact_event("build_aborted") - runtime._payout_artifact_log( - "payout_artifact_build_aborted", - payout_state_generation=int(artifact_payout_state_generation), - duration_seconds=0.0, - during_publication=True, - build_interval_bypassed=True, - abort_reason=f"{fallback_reason}_without_usable_window", - ) - return None - reused = dataclass_replace( - artifact, - generation=0, - payout_state_generation=int(artifact_payout_state_generation), - prepared_monotonic=time.monotonic(), - window_build_mode="found_block_cached", - window_delta_rows=0, - window_expired_rows=0, - window_touched_pages=0, - window_build_seconds=0.0, - window_full_rescan_reason=fallback_reason, - ) - runtime._record_payout_artifact_event("found_block_cached") - runtime._payout_artifact_log( - "payout_artifact_found_block_cached", - payout_state_generation=int(artifact_payout_state_generation), - duration_seconds=0.0, - window_build_mode="found_block_cached", - window_shares=len(reused.shares_json), - anchor_age_ms=anchor_age_ms, - during_publication=True, - build_interval_bypassed=True, - fallback_reason=fallback_reason, - ) - return reused - - def _current_payout_state_candidate(self) -> PayoutStateCandidate: - runtime = self._runtime - return runtime._prepared_payout_state_candidate( - runtime._capture_payout_state_source() - ) - - def _record_discarded_payout_candidate(self) -> None: - runtime = self._runtime - runtime._ensure_job_cache_state() - with runtime._payout_state_metrics_lock: - runtime.payout_state_candidates_discarded += 1 - - def _block_payout_state_publication( - self, - *, - force: bool = False, - supersede_with: tuple[int, str | None, str, float] | None = None, - ) -> None: - """Atomically close delivery, optionally reserving a newer source.""" - runtime = self._runtime - - runtime._ensure_job_cache_state() - - pending_source: int | None = None - - def mark_blocked() -> bool: - nonlocal pending_source - with runtime._job_cache_lock: - with runtime.lock: - if supersede_with is not None: - ( - expected_source, - fallback_tip, - cause, - invalidated, - ) = supersede_with - current_source, current_tip, _, _ = ( - runtime._payout_state_source - ) - # A newer tip/source wins its identity, but it must be - # superseded so no candidate prepared before an - # uncertain durable commit can publish afterward. - source_tip = ( - fallback_tip - if current_source == expected_source - else current_tip - ) - pending_source = current_source + 1 - runtime._payout_state_source = ( - pending_source, - source_tip, - cause, - invalidated, - ) - else: - pending_source = runtime._payout_state_source[0] - if ( - not force - and supersede_with is None - and pending_source - == runtime._published_payout_state.source_generation - ): - return False - runtime._payout_state_publication_blocked = True - runtime._job_bundle_cache.clear() - return True - - # Close fleet admission atomically with the cache fence. Escaped - # immutable bundles remain stamped with the old generation, but cannot - # cross the boundary while the ledger has newer unpublished state. - if not runtime._payout_state_delivery_gate.block_delivery(mark_blocked): - return - # Construction cancellation is independent of fanout activation. An - # obsolete builder may still be in its ledger or subprocess phase and - # must be preempted before a replacement request is submitted. - runtime._cancel_obsolete_job_builds("payout generation superseded") - with runtime._job_cache_lock: - next_payout_generation = runtime._payout_state_generation + 1 - with runtime.lock: - payout_invalidated_monotonic = runtime._payout_state_source[3] - runtime._record_progress_payout_generation( - next_payout_generation, - payout_invalidated_monotonic, - ) - with runtime.lock: - active = getattr(runtime, "_active_tip_refresh", None) - if ( - active is not None - and active[0].payout_state_generation < next_payout_generation - ): - active[1].cancel() - elif active is not None: - return - assert pending_source is not None - runtime._mark_tip_refresh_pending(next_payout_generation) - runtime._schedule_tip_refresh_retry() - - def _payout_state_publication_fenced(self) -> bool: - """Report whether delivery is still blocked awaiting a publication.""" - runtime = self._runtime - - runtime._ensure_job_cache_state() - with runtime._job_cache_lock: - return runtime._payout_state_publication_blocked - - def _payout_source_requires_publication( - self, - candidate: PayoutStateCandidate | None = None, - ) -> bool: - """Report whether an invalidation source still lacks a publication.""" - runtime = self._runtime - - runtime._ensure_job_cache_state() - with runtime._job_cache_lock: - published_source = runtime._published_payout_state.source_generation - if candidate is not None: - return candidate.source_generation != published_source - with runtime.lock: - return runtime._payout_state_source[0] != published_source - - def _captured_payout_source_requires_publication( - self, - captured: tuple[int, int, str | None, str, float], - ) -> bool: - """Publication check for a captured source, without a candidate. - - Answers exactly what _payout_source_requires_publication would answer - for a candidate prepared from ``captured``. Candidate preparation can - include a multi-second ledger snapshot, so decision sites must be able - to ask this first and prepare only when publication will follow. - """ - runtime = self._runtime - - runtime._ensure_job_cache_state() - with runtime._job_cache_lock: - return captured[1] != runtime._published_payout_state.source_generation - - def _publish_payout_state_candidate( - self, - candidate: PayoutStateCandidate, - ) -> int | None: - """Publish a prepared candidate, or reject it if its source moved.""" - runtime = self._runtime - - runtime._ensure_job_cache_state() - runtime._ensure_tip_refresh_state() - published_generation: int | None = None - schedule_retry = False - active_to_cancel: _FanoutCancellation | None = None - publication_born_expired: tuple[int | None, int] | None = None - publication_installed: tuple[int, int, int] | None = None - publish_started = 0.0 - with runtime._job_cache_lock: - with runtime.lock: - if ( - candidate.source_generation != runtime._payout_state_source[0] - or candidate.base_generation != runtime._payout_state_generation - ): - runtime._record_discarded_payout_candidate() - return None - try: - if ( - candidate.accepted_block_preview is not None - and not candidate.accepted_block_withdrawal - ): - # The preview IS the balance state this generation publishes. - # The durable ledger only catches up after persistence, so a - # ledger read here would cache pre-parent balances against the - # post-parent generation and outlive the transition override. - artifact = runtime._payout_state_artifact_from_balances( - generation=candidate.base_generation + 1, - source_generation=candidate.source_generation, - balances=runtime._materialize_prior_balance_preview( - candidate.accepted_block_preview - ), - ) - elif candidate.ledger_artifact is not None: - # Candidate preparation captured balances under the same - # ledger fence as its share window. Reuse that immutable copy - # instead of issuing the carry-forward aggregate twice during - # every publication. - artifact = runtime._payout_state_artifact_from_balances( - generation=candidate.base_generation + 1, - source_generation=candidate.source_generation, - balances=list(candidate.ledger_artifact.prior_balances), - ) - else: - artifact = runtime._prepare_payout_state_artifact( - generation=candidate.base_generation + 1, - source_generation=candidate.source_generation, - ) - except Exception: - # The delivery/cache fence remains closed. A later reconciliation - # retry can prepare and publish the artifact; no generation lacking - # immutable payout inputs is ever exposed to builders. - runtime._schedule_tip_refresh_retry() - raise - with runtime._job_cache_lock: - with runtime.lock: - if ( - candidate.source_generation != runtime._payout_state_source[0] - or candidate.base_generation != runtime._payout_state_generation - ): - runtime._record_discarded_payout_candidate() - return None - with runtime._payout_state_delivery_gate.publication(): - # publication() has already drained admitted old sends. Start the - # critical-section timer only now; drain latency is delivery wait, - # not time spent holding the atomic payout mutation section. - publish_started = time.monotonic() - with runtime._job_cache_lock: - with runtime.lock: - source_generation = runtime._payout_state_source[0] - if ( - candidate.source_generation == source_generation - and candidate.base_generation - == runtime._payout_state_generation - ): - published_generation = runtime._payout_state_generation + 1 - if candidate.accepted_block_hash is not None: - key = candidate.accepted_block_hash - with runtime._accepted_block_payout_preview_condition: - transition = runtime._accepted_block_payout_previews.get( - key, - AcceptedBlockPayoutTransition( - block_height=candidate.accepted_block_height, - landed=True, - ), - ) - if candidate.accepted_block_withdrawal: - runtime._accepted_block_payout_previews.pop(key, None) - runtime._invalidated_accepted_block_payout_previews[ - key - ] = ( - transition.block_height - if transition.block_height is not None - else candidate.accepted_block_height - ) - else: - existing_preview = transition.preview - if ( - existing_preview is not None - and existing_preview - != candidate.accepted_block_preview - ): - raise RuntimeError( - "accepted block payout preview changed " - "during atomic publication" - ) - runtime._invalidated_accepted_block_payout_previews.pop( - key, - None, - ) - runtime._accepted_block_payout_previews[key] = ( - dataclass_replace( - transition, - landed=True, - preview=candidate.accepted_block_preview, - published_generation=published_generation, - ) - ) - runtime._accepted_block_payout_preview_condition.notify_all() - runtime._payout_state_generation = published_generation - prepared_artifact = candidate.ledger_artifact - candidate_anchor_age_ms = ( - None - if prepared_artifact is None - or prepared_artifact.snapshot_anchor_ms is None - else self._now_ms() - - int(prepared_artifact.snapshot_anchor_ms) - ) - if ( - prepared_artifact is not None - and runtime._payout_artifact_reuse_active() - and prepared_artifact.payout_state_generation - == published_generation - and prepared_artifact.append_invalidation_epoch - == runtime._payout_ledger_append_invalidation_epoch - and candidate_anchor_age_ms is not None - and candidate_anchor_age_ms - <= runtime._payout_artifact_max_anchor_age_ms() - ): - runtime._payout_ledger_artifact_generation += 1 - # Restamp freshness at the install: the candidate - # artifact was built before publication, and the - # delivery-gate drain between the two can outlive - # the reuse budget -- a freshly published payout - # generation must never arm an already-stale - # artifact and force the next builds back through - # the synchronous reward-window walk. - self._replace_payout_ledger_artifact_locked( - dataclass_replace( - prepared_artifact, - generation=runtime._payout_ledger_artifact_generation, - prepared_monotonic=time.monotonic(), - ) - ) - publication_installed = ( - runtime._payout_ledger_artifact_generation, - candidate_anchor_age_ms, - len(prepared_artifact.shares_json), - ) - else: - if ( - prepared_artifact is not None - and runtime._payout_artifact_reuse_active() - and prepared_artifact.payout_state_generation - == published_generation - and prepared_artifact.append_invalidation_epoch - == runtime._payout_ledger_append_invalidation_epoch - ): - # Same admission rule as - # _install_payout_ledger_artifact: candidate - # construction plus the delivery-gate drain - # can push the declared anchor past the - # audit ceiling, and arming such a - # BORN-EXPIRED artifact would fail every - # reuse probe on anchor age -- with the - # re-arm suppressed while an accepted - # preview awaits durability. Discard it; the - # post-publication probe schedules - # preparation (or the durable-confirmation - # path resumes it once the preview lands). - publication_born_expired = ( - candidate_anchor_age_ms, - len(prepared_artifact.shares_json), - ) - self._disarm_payout_ledger_artifact_locked() - runtime._published_payout_state = PublishedPayoutState( - generation=published_generation, - source_generation=candidate.source_generation, - source_tip_hash=candidate.source_tip_hash, - published_monotonic=publish_started, - artifact=artifact, - ) - epoch_tip_hash = runtime._payout_epoch_tip_hash_locked() - if epoch_tip_hash is not None: - runtime._mint_tip_refresh_epoch_locked( - tip_hash=epoch_tip_hash, - payout_state_generation=published_generation, - started_monotonic=candidate.invalidated_monotonic, - ) - # Re-anchor the stamping identity to the minted - # epoch in the same lock hold. A stale identity - # stamps rebuilt bundles at epoch 0, which the - # fence blocks for every previously admitted - # connection until the next wave republishes. - # The publish helper no-ops when the epoch tip - # is not the published snapshot tip; the owning - # wave handles that transition. - published_snapshot = getattr( - runtime, - "tip_template_snapshot", - None, - ) - if published_snapshot is not None: - runtime._publish_tip_refresh_epoch_identity_locked( - published_snapshot - ) - runtime._payout_state_publication_blocked = False - runtime._job_bundle_cache.clear() - runtime._retained_collection_refresh = None - active = getattr(runtime, "_active_tip_refresh", None) - if active is None: - schedule_retry = True - elif active[0].payout_state_generation < published_generation: - # The payout gate itself rejects this old generation. - # Signal its fanout only after atomic publication. - active_to_cancel = active[1] - schedule_retry = True - with runtime._payout_state_metrics_lock: - runtime._payout_first_delivery_pending = ( - published_generation, - candidate.invalidated_monotonic, - ) - if published_generation is not None: - # The mutation owner still blocks every delivery admission, - # so the pointer swap and gate generation remain one atomic - # publication boundary. Do not acquire the gate condition - # while holding coordinator locks: cancellation callbacks take - # those locks after entering the gate wait loop. - runtime._payout_state_delivery_gate.publish_generation( - published_generation, - prioritize_delivery=True, - ) - runtime._observe_payout_state_seconds( - "publish", - max(0.0, time.monotonic() - publish_started), - ) - if published_generation is None: - runtime._record_discarded_payout_candidate() - return None - if publication_born_expired is not None: - born_expired_age_ms, born_expired_shares = publication_born_expired - runtime._record_payout_artifact_event("born_expired") - runtime._payout_artifact_log( - "payout_artifact_born_expired", - payout_state_generation=int(published_generation), - anchor_age_ms=born_expired_age_ms, - window_shares=born_expired_shares, - during_publication=True, - ) - if publication_installed is not None: - # The pointer swap above is an install site that bypasses - # _install_payout_ledger_artifact; record it in the same event - # family or the publication path stays invisible to the - # lifecycle observability this counter exists for. - installed_generation, installed_age_ms, installed_shares = ( - publication_installed - ) - runtime._record_payout_artifact_event("installed") - runtime._payout_artifact_log( - "payout_artifact_installed", - generation=int(installed_generation), - payout_state_generation=int(published_generation), - anchor_age_ms=installed_age_ms, - window_shares=installed_shares, - during_publication=True, - ) - # A publication is a fresh start for speculative preparation: the - # candidate artifact installs through the atomic pointer swap above - # (not _install_payout_ledger_artifact), so whatever re-arm backoff - # pre-publication share traffic accumulated must be released here or - # a share landing right after publication would stay suppressed for - # the whole scaled interval. - with runtime._payout_artifact_executor_lock: - runtime._payout_artifact_rearm_backoff = 1 - runtime._cancel_obsolete_job_bundle_builds( - payout_state_generation=published_generation - ) - runtime._record_progress_payout_generation( - published_generation, - candidate.invalidated_monotonic, - ) - if active_to_cancel is not None: - active_to_cancel.cancel() - runtime._cancel_obsolete_job_builds("payout generation published") - if schedule_retry: - runtime._mark_tip_refresh_pending(published_generation) - runtime._schedule_tip_refresh_retry() - with runtime._job_cache_lock: - current_artifacts = runtime._template_artifacts - published_artifact_usable = ( - runtime._usable_payout_ledger_artifact( - published_generation, - current_artifacts.network_difficulty, - ) - if current_artifacts is not None - else None - ) - accepted_preview_pending_durability = ( - candidate.accepted_block_hash is not None - and not candidate.accepted_block_withdrawal - ) - with runtime.lock: - self_check_owed = self._payout_self_check_owed - if ( - current_artifacts is not None - and (published_artifact_usable is None or self_check_owed) - and not accepted_preview_pending_durability - ): - # A background artifact built before accepted-block confirmation - # would carry the old database balances under the new payout - # generation. Child builders use the compact preview until the - # durable state catches up; artifact preparation resumes then. - # A self-check deferred by this generation's candidate build is - # handed to the same background walker here (issue #228), so - # it runs off the critical path instead of waiting for a - # re-anchor that fresh publications keep postponing. - with runtime.lock: - self._payout_self_check_owed = False - runtime._schedule_payout_ledger_artifact_preparation( - published_generation, - current_artifacts.network_difficulty, - ) - return published_generation - - def _record_first_payout_delivery( - self, - generation: int, - delivered_monotonic: float, - ) -> None: - runtime = self._runtime - runtime._ensure_job_cache_state() - elapsed: float | None = None - with runtime._payout_state_metrics_lock: - pending = runtime._payout_first_delivery_pending - if pending is not None and pending[0] == generation: - elapsed = max(0.0, delivered_monotonic - pending[1]) - runtime._payout_first_delivery_pending = None - if elapsed is not None: - runtime._observe_payout_state_seconds("first_delivery", elapsed) - - def _advance_payout_state_generation(self) -> int: - """Publish a payout-only invalidation with no expensive gate work.""" - runtime = self._runtime - # No production caller remains: direct-block finalization publishes - # through submit_block_candidate's reserve/publish path instead. Tests - # keep using this as the smallest complete reserve -> block -> publish - # invalidation cycle. - runtime._ensure_job_cache_state() - runtime._reserve_payout_state_source("payout_only") - prepared_started = time.monotonic() - with runtime._payout_state_prepare_lock: - # Close build/delivery admission before releasing snapshot readers. - # Publication may then drain already-admitted sends without holding - # the preparation lock needed by later ledger work. - runtime._block_payout_state_publication(force=True) - runtime._observe_payout_state_seconds( - "preparation", - max(0.0, time.monotonic() - prepared_started), - ) - generation = runtime._publish_current_payout_state_with_retry_budget() - if generation is None: - raise TemplateRefreshSuperseded( - "payout-only invalidation was superseded; immediate retry scheduled" - ) - return generation - - def _publish_current_payout_state_with_retry_budget( - self, - *, - initial_attempted: bool = False, - ) -> int | None: - """Publish the current source with a bounded supersession budget.""" - runtime = self._runtime - - max_retries = max( - 0, - int( - getattr( - runtime, - "payout_reconcile_supersession_retries", - DEFAULT_PRISM_PAYOUT_RECONCILE_SUPERSESSION_RETRIES, - ) - ), - ) - attempts = max_retries + (0 if initial_attempted else 1) - for _attempt in range(attempts): - candidate = runtime._current_payout_state_candidate() - published = runtime._publish_payout_state_candidate(candidate) - if published is not None: - return published - runtime._block_payout_state_publication() - return None - - def _payout_delivery( - self, - cancelled: Callable[[], bool], - *, - generation: int, - ) -> Any: - """Use cancellable admission while retaining focused gate test seams.""" - runtime = self._runtime - gate = runtime._payout_state_delivery_gate - delivery_cancelable = getattr(gate, "delivery_cancelable", None) - if callable(delivery_cancelable): - return delivery_cancelable( - cancelled, - generation=generation, - priority=True, - ) - delivery = gate.delivery - try: - return delivery(cancelled) - except TypeError: - return delivery() - - def _expose_inflight_scan_anchor(self, anchor_ms: int) -> int: - """Publish a ledger walk's anchor to the append-side invalidation. - - The entry must stay exposed for as long as the walk's result is - invisible to ``_invalidate_incremental_payout_window_for_append``: - from before the database read (whose snapshot may already exclude a - row committing mid-read) until either the result is discarded or its - window becomes visible through the incremental cache or the armed - artifact's own anchor. Entries older than the audit ceiling are - pruned lazily here: ``_install_payout_ledger_artifact`` rejects any - artifact whose anchor aged past that same ceiling as born-expired, - so a leaked exposure (a built bundle every waiter abandoned) can - never outlive the last install that could have used it. - """ - runtime = self._runtime - runtime._ensure_job_cache_state() - max_age_ms = runtime._payout_artifact_max_anchor_age_ms() - stale_before_ms = self._now_ms() - max_age_ms - with runtime._job_cache_lock: - for token, exposed_anchor_ms in tuple( - runtime._payout_window_inflight_scan_anchors.items() - ): - if exposed_anchor_ms < stale_before_ms: - runtime._payout_window_inflight_scan_anchors.pop(token, None) - runtime._payout_window_inflight_scan_anchor_token += 1 - token = runtime._payout_window_inflight_scan_anchor_token - runtime._payout_window_inflight_scan_anchors[token] = int(anchor_ms) - return token - - def _retire_inflight_scan_anchor(self, token: int | None) -> None: - """Withdraw an exposed walk anchor once its result became visible - (or provably never will). Idempotent; ``None`` is a no-op.""" - runtime = self._runtime - if token is None: - return - runtime._ensure_job_cache_state() - with runtime._job_cache_lock: - runtime._payout_window_inflight_scan_anchors.pop(int(token), None) - - def _fold_published_job_window_anchor_locked( - self, - anchor_ms: int | None, - ) -> None: - """Raise the published-window anchor watermark to ``anchor_ms``. - - Caller holds _job_cache_lock -- the lock every predates() read - takes -- so the raise is atomic with whatever exposure handed the - anchor over: no append can be classified against an anchor set - that has already lost the anchor but not yet gained the watermark. - Raising it early or redundantly is always safe, because a higher - watermark can only classify MORE rows as predating, and that is - the fail-closed direction: an extra bump is forgiven per anchor by - _append_epoch_invalidated_declared_anchor, while a missing one - leaves no stamp for a landing's fences to refuse. ``None`` is a - no-op. - """ - if anchor_ms is None: - return - runtime = self._runtime - runtime._payout_published_job_window_anchor_ms = max( - int(anchor_ms), - runtime._payout_published_job_window_anchor_ms or 0, - ) - - def _publish_seedless_job_window_anchor_locked(self, anchor_ms: int) -> None: - """Keep a seedless published window's anchor visible to appends. - - Caller holds _job_cache_lock, at the bundle publication fence of a - build whose walk exposure retires with no seeded artifact to carry - it (the reuse kill-switch, or nothing to seed). Jobs stamped from - that bundle keep serving the window until they retire, so the - declared anchor must stay in the predates() anchor set past the - exposure: a replay-shaped append that starts and finishes entirely - between publication and a landing's own anchor exposure would - otherwise see no live anchor, skip its epoch bump, and leave no - registry entry for the landing's drain to find. Monotonic and never - retired; the pending-commit floor keeps every ordinary share commit - above it. - """ - self._fold_published_job_window_anchor_locked(anchor_ms) - - def _replace_payout_ledger_artifact_locked( - self, - artifact: PayoutLedgerArtifact | None, - ) -> None: - """Every write to the armed-artifact slot goes through here. - - Caller holds _job_cache_lock. A seeded window's anchor may be - exposed to the append-side predates() checks by nothing but the - armed artifact, while jobs stamped from that window stay landable - after the slot changes: a disarm or replacement retires the cache, - never the jobs. Folding the outgoing artifact's anchor into the - never-retired published-window watermark BEFORE the slot moves -- - under the same lock every predates() read takes, so no append can - observe the anchor gone with the watermark not yet raised -- - preserves the invariant the landing fences assume: the anchor-set - maximum never decreases while any job is landable, so every append - that predates a live or seeded anchor bumps the epoch and leaves a - stamp for the fences to read. Skipping the fold is the disarm gap: - the bump that consults the seeded anchor also disarms the artifact, - emptying the anchor set, and a later row genuinely predating a - still-landable candidate's window then commits with no bump at all - -- nothing for the anchor-scoped predicate to refuse, so the - landing submits an invalidated window. The permanence is what the - per-anchor predicate makes affordable: a retired anchor forces - extra bumps only from replay-shaped rows (the pending-commit floor - keeps ordinary commits above every job anchor), and each is - forgiven per anchor instead of over-abandoning (issue #126). - """ - runtime = self._runtime - outgoing = runtime._payout_ledger_artifact - if outgoing is not None: - self._fold_published_job_window_anchor_locked( - outgoing.snapshot_anchor_ms - ) - runtime._payout_ledger_artifact = artifact - - def _disarm_payout_ledger_artifact_locked(self) -> None: - """Drop the armed artifact, keeping its anchor in the watermark.""" - self._replace_payout_ledger_artifact_locked(None) - - @staticmethod - def _pending_share_predates_anchor( - pending_share: PendingShare, - anchor_ms: int | None, - ) -> bool: - return bool( - anchor_ms is not None - and int(pending_share.job_issued_at_ms) <= anchor_ms - and int(pending_share.accepted_at_ms) <= anchor_ms - ) - - def _pending_share_predates_live_anchor_locked( - self, - pending_share: PendingShare, - ) -> bool: - # Caller holds _job_cache_lock. The incremental pointer is - # immutable and replaced atomically; its owning lock is - # deliberately not acquired here, because an urgent append must - # disarm the separately guarded artifact even while a long - # oracle build owns _payout_state_prepare_lock. Reading it under - # _job_cache_lock orders the read against the in-flight scan - # anchors: a window walk makes its result visible (the - # incremental pointer, or the armed artifact's install fence) - # before retiring its exposed anchor under this lock, so a walk - # that could have missed this row always exposes an anchor to - # the predates checks below. - runtime = self._runtime - predates = runtime._pending_share_predates_anchor - cached = getattr(runtime, "_incremental_payout_artifact_window", None) - artifact = runtime._payout_ledger_artifact - inflight_anchors_ms = tuple( - runtime._payout_window_inflight_scan_anchors.values() - ) - cached_anchor_ms = ( - None - if cached is None - else int(cached.window.anchor_job_issued_at_ms) - ) - artifact_anchor_ms = ( - None - if artifact is None or artifact.snapshot_anchor_ms is None - else int(artifact.snapshot_anchor_ms) - ) - published_job_anchor_ms = getattr( - runtime, "_payout_published_job_window_anchor_ms", None - ) - return bool( - predates(pending_share, cached_anchor_ms) - or predates(pending_share, artifact_anchor_ms) - or predates(pending_share, published_job_anchor_ms) - or any( - predates(pending_share, anchor_ms) - for anchor_ms in inflight_anchors_ms - ) - ) - - @contextmanager - def _landing_fence_for_predating_append( - self, - pending_shares: list[PendingShare], - ) -> Iterator[bool]: - """Hold the landing fence across a durable append of predating rows. - - An epoch bump that only starts after its row is durable leaves a - gap: a landing can acquire the fence, verify the pre-bump epoch, - and enter submitblock after the row committed but before the bump - starts, still landing a coinbase whose window omits the durable - share. Any append whose rows predate a live anchor therefore takes - the fence BEFORE the ledger commit and holds it through the epoch - bump, so the durable append and its invalidation land on the same - side of a landing's fence-guarded epoch-check-and-submit boundary. - Ordinary share commits stay off the lock entirely: the unfenced - peek here fires only for the rare replay-shaped append. Yields - whether the fence is held so the caller can thread it into - _record_late_visible_payout_append. - - The unfenced classification is a one-time predicate, so an anchor - exposed after it (a landing publishing its declared anchor) cannot - retroactively fence this commit. An unfenced batch therefore - registers itself, atomically with the classification, in - _payout_unfenced_append_inflight_stamps and deregisters only after - the caller's post-commit epoch-bump attempt ran inside this block; - _await_unfenced_appends_predating_anchor lets a landing wait those - commits out before its epoch fences arm. - """ - runtime = self._runtime - runtime._ensure_job_cache_state() - unfenced_token: int | None = None - with runtime._job_cache_lock: - fence_needed = any( - runtime._pending_share_predates_live_anchor_locked(pending) - for pending in pending_shares - ) - if not fence_needed and pending_shares: - runtime._payout_unfenced_append_inflight_token += 1 - unfenced_token = runtime._payout_unfenced_append_inflight_token - runtime._payout_unfenced_append_inflight_stamps[ - unfenced_token - ] = min( - max( - int(pending.job_issued_at_ms), - int(pending.accepted_at_ms), - ) - for pending in pending_shares - ) - if not fence_needed: - try: - yield False - finally: - if unfenced_token is not None: - with runtime._payout_unfenced_append_drained: - runtime._payout_unfenced_append_inflight_stamps.pop( - unfenced_token, None - ) - runtime._payout_unfenced_append_drained.notify_all() - return - with runtime._payout_append_landing_fence_lock: - yield True - - def _block_work_wait_slice(self) -> float: - """Slice length for a condition wait taken on the block-work thread. - - The block-candidate owner derives this from the configured watchdog - tolerance, so reusing it keeps every sliced wait on that thread in - step with the watchdog through operator overrides instead of pinning - a second literal that goes stale. Duck-typed runtimes in focused - tests do not carry the helper; they fall back to the same - admission-wait slice the shutdown controller keeps its own copy of. - """ - wait_slice = getattr(self._runtime, "_block_work_wait_slice", None) - if callable(wait_slice): - return max(0.001, float(wait_slice())) - return BLOCK_SUBMITTER_WAIT_HEARTBEAT_SLICE_SECONDS - - def _record_block_work_phase(self, phase: str) -> None: - """Stamp a block-work liveness phase when the runtime records them. - - The recorder itself decides whether the calling thread is a - registered block-work owner and records nothing otherwise, which is - what keeps a share-writer or client thread from refreshing the - budget of an owner that is genuinely wedged. Duck-typed runtimes - without the recorder simply do not stamp. - """ - recorder = getattr(self._runtime, "_record_block_submitter_phase", None) - if callable(recorder): - recorder(phase) - - def _await_unfenced_appends_predating_anchor(self, anchor_ms: int) -> None: - """Wait until no unfenced in-flight append can predate ``anchor_ms``. - - Called by a landing right after exposing its declared anchor. An - append classified as unfenced before that exposure commits outside - the landing fence, so holding the fence across submitblock would - not exclude its durable commit -- and its post-commit epoch bump - would queue behind the very fence the landing holds, arriving only - after the block entered qbitd. Draining here restores the fence - contract: each matching append finishes its commit and its bump - attempt against the now-exposed anchor before this returns, so a - predating row advances the epoch the landing's fences check, and - every append classified after the exposure sees the anchor and - takes the fence itself. Appends whose rows cannot predate the - anchor never block this wait, and the registry holds entries only - for the duration of a ledger commit, so the common path is one - locked emptiness check. Commit-scoped retention is sound because an - append that completed before this drain cannot have predated - ``anchor_ms`` silently: every landable declared anchor is exposed - from publication onward (the armed artifact's anchor -- which the - published-window watermark absorbs before the slot can drop it -- - or that same watermark a seedless build hands its anchor to at - the publication fence), so such an append classified as fenced and - advanced the epoch this landing's fences compare against. - """ - runtime = self._runtime - runtime._ensure_job_cache_state() - anchor = int(anchor_ms) - # The drain is unbounded by design and stays that way: it enforces - # the fencing contract above, so a deadline here could let a landing - # proceed past appends whose epoch bumps its own fences must see. - # What an untimed wait may not do is run heartbeat-silent -- this - # call happens inline on the watchdog-monitored block-work thread, - # where an unbroken silence longer than the tolerance is a hard exit - # mid-landing (issue #125). Waiting in watchdog-sized slices and - # stamping each one that still finds the predicate true keeps the - # wait exactly as long as it was while making it visible; the - # ``while`` re-check is what makes a timed wait semantically - # transparent. The common path -- nothing in flight predates the - # anchor -- never enters the loop and so stamps nothing at all. - slice_seconds = self._block_work_wait_slice() - with runtime._payout_unfenced_append_drained: - while any( - stamp_ms <= anchor - for stamp_ms in ( - runtime._payout_unfenced_append_inflight_stamps.values() - ) - ): - self._record_block_work_phase("wait-unfenced-append-drain") - runtime._payout_unfenced_append_drained.wait( - timeout=slice_seconds - ) - - def _record_late_visible_payout_append( - self, - pending_share: PendingShare, - *, - landing_fence_owned: bool = False, - ) -> int | None: - """Advance the append-invalidation epoch for a newly durable row. - - Two-phase bump. The unfenced peek keeps ordinary share commits off - the landing fence lock entirely; only a row that predates a live - anchor re-checks and bumps under it. Holding that lock for the bump - is what makes the landing's final epoch fence authoritative: a bump - cannot commit while a landing sits between its last epoch read and - submitblock, so the invalidation lands either before that read - (candidate abandoned) or after the RPC returns (the refresh wave - retires the job). The predicate re-runs under the lock because the - anchor set may have moved during the unfenced gap. A caller that - already holds the fence around the durable commit itself - (landing_fence_owned) bumps without re-acquiring. - - Returns the advanced epoch, or None when no live anchor predates - the row. The caller must follow a non-None return with - _retire_payout_windows_for_late_append OUTSIDE the fence: that step - waits on _payout_state_prepare_lock, which a long oracle build can - hold, and the wait must not block landings. - """ - runtime = self._runtime - runtime._ensure_job_cache_state() - - def bump_if_predating_locked() -> int | None: - with runtime._job_cache_lock: - if not runtime._pending_share_predates_live_anchor_locked( - pending_share - ): - return None - runtime._payout_ledger_append_invalidation_epoch += 1 - epoch = int(runtime._payout_ledger_append_invalidation_epoch) - self._record_append_invalidation_stamp_locked( - epoch, pending_share - ) - # The fold inside the disarm is load-bearing here: this bump - # may be consulting the armed artifact's anchor as the last - # live exposure of a seeded window, and dropping it bare - # would let the NEXT predating row commit against an empty - # anchor set -- silently, with no stamp for a still-landable - # candidate's fences to refuse. - self._disarm_payout_ledger_artifact_locked() - return epoch - - if landing_fence_owned: - return bump_if_predating_locked() - with runtime._job_cache_lock: - if not runtime._pending_share_predates_live_anchor_locked( - pending_share - ): - return None - with runtime._payout_append_landing_fence_lock: - return bump_if_predating_locked() - - def _record_append_invalidation_stamp_locked( - self, - epoch: int, - pending_share: PendingShare, - ) -> None: - """Remember which row caused ``epoch``, and prune the oldest. - - Caller holds _job_cache_lock and has just advanced the epoch to - ``epoch``. The stamp recorded is the row's own predating quantity, - so a landing can compare it against its declared anchor with the - same test the append side used against the live anchor set. - - Pruning drops entries from the old end only, and advances the - retained floor by exactly what it dropped. Acceptance therefore - never widens: a baseline that can no longer be answered is refused - by _append_epoch_invalidated_declared_anchor rather than treated as - a bump-free stretch. - """ - stamps = self._payout_append_invalidation_stamps - stamps[epoch] = max( - int(pending_share.job_issued_at_ms), - int(pending_share.accepted_at_ms), - ) - oldest_retained = epoch - PRISM_PAYOUT_APPEND_INVALIDATION_STAMP_HISTORY + 1 - if oldest_retained > self._payout_append_invalidation_oldest_epoch: - for dropped in range( - self._payout_append_invalidation_oldest_epoch, oldest_retained - ): - stamps.pop(dropped, None) - self._payout_append_invalidation_oldest_epoch = oldest_retained - - def _append_epoch_invalidated_declared_anchor( - self, - *, - baseline_epoch: int, - live_epoch: int, - declared_anchor_ms: int | None, - ) -> bool: - """Did an append between two epochs invalidate THIS window? - - The invalidation epoch is one global counter, so a landing that - compares its baseline against the live value learns only that some - window somewhere was invalidated. An append stamped after this - candidate's declared anchor but before a newer live window's anchor - bumps that counter without touching this candidate's payout window, - and on the fallback submit path the bare comparison abandoned the - block before it ever reached qbitd (issue #126). Scoping the - question to the anchor answers what the fence actually needs. - - Fails closed on anything it cannot prove: an unknown declared - anchor is as terminal as it was before, and so is a baseline older - than the retained stamp history or an epoch that somehow moved - backwards. Only a bump whose recorded row provably does not predate - this anchor is forgiven. - - Must not be called while holding _job_cache_lock; it takes that - lock itself. The caller that needs the answer to be authoritative - holds _payout_append_landing_fence_lock across its live-epoch read - and this call, which is what stops a bump from landing in between. - """ - baseline = int(baseline_epoch) - live = int(live_epoch) - if baseline == live: - return False - if live < baseline: - return True - if declared_anchor_ms is None: - return True - anchor = int(declared_anchor_ms) - runtime = self._runtime - runtime._ensure_job_cache_state() - with runtime._job_cache_lock: - if baseline + 1 < self._payout_append_invalidation_oldest_epoch: - return True - stamps = self._payout_append_invalidation_stamps - # Bounded by the retention above: the floor check just proved - # every epoch in this range is retained, so the walk is at most - # PRISM_PAYOUT_APPEND_INVALIDATION_STAMP_HISTORY entries long. - for epoch in range(baseline + 1, live + 1): - stamp = stamps.get(epoch) - if stamp is None or stamp <= anchor: - return True - return False - - def _invalidate_incremental_payout_window_for_append( - self, - pending_share: PendingShare, - ) -> None: - """Disarm payout work that predates a newly visible eligible row. - - Normal submissions hold the pending-share commit floor, so their - accepted timestamp remains above the cached anchor until the row is - durable. Durable replay reconstructs the original timestamps without - that process-local floor. If both stamps are already covered by an - incremental or armed-artifact anchor, the snapshot cannot discover - the late-visible row. Publish the invalidation without waiting for an - in-flight oracle walk, then clear the incremental cache under its own - lock; the epoch prevents pre-append work from re-arming meanwhile. - - This wrapper serves post-commit callers that do not hold the landing - fence; the durable append paths take the fence around the commit via - _landing_fence_for_predating_append and call the bump and retire - steps directly. - """ - runtime = self._runtime - invalidation_epoch = runtime._record_late_visible_payout_append( - pending_share - ) - if invalidation_epoch is None: - return - runtime._retire_payout_windows_for_late_append( - pending_share, invalidation_epoch - ) - - def _retire_payout_windows_for_late_append( - self, - pending_share: PendingShare, - invalidation_epoch: int, - ) -> None: - """Disarm cached payout windows after a late-append epoch bump. - - Runs outside the landing fence. Active jobs stamped against the - pre-append window still pass the generation and digest fences; only - epoch-aware reselection replaces them. Schedule the same refresh wave - a repair publication schedules, and do it before the prepare-lock - wait below so a long oracle build holding that lock cannot delay - superseding the stale jobs. - """ - runtime = self._runtime - - def predates(anchor_ms: int | None) -> bool: - return runtime._pending_share_predates_anchor( - pending_share, anchor_ms - ) - - runtime._mark_tip_refresh_pending(invalidation_epoch) - runtime._schedule_tip_refresh_retry() - - with runtime._payout_state_prepare_lock: - cached = runtime._incremental_payout_artifact_window - if cached is None: - runtime._incremental_payout_artifact_window_invalidation_reason = ( - "late_visible_append" - ) - return - anchor_ms = int(cached.window.anchor_job_issued_at_ms) - if cached.append_invalidation_epoch >= invalidation_epoch: - # A post-invalidation build already replaced the cache. - return - if predates(anchor_ms): - runtime._incremental_payout_artifact_window = None - runtime._incremental_payout_artifact_window_invalidation_reason = ( - "late_visible_append" - ) - else: - # The armed artifact was affected but this older incremental - # anchor was not. Retag it so a later delta build can preserve - # the exact append-only window instead of forcing an oracle. - runtime._incremental_payout_artifact_window = dataclass_replace( - cached, - append_invalidation_epoch=invalidation_epoch, - ) - - @contextmanager - def _payout_balance_serializer_released(self) -> Iterator[None]: - """Temporarily release the balance serializer around candidate audit - construction. - - The caller is the landing, which holds the serializer exactly once - (its single outer acquisition; no caller of - _land_and_confirm_block_candidate holds it). The landed-transition - fence armed before the release keeps reconciliation from mutating - balances while the serializer is free, so job delivery proceeds - during the expensive builder/verifier subprocess work instead of - queueing for its whole duration. - """ - runtime = self._runtime - runtime._payout_balance_mutation_lock.release() - try: - yield - finally: - runtime._acquire_block_submitter_lock( - runtime._payout_balance_mutation_lock, - "payout-balance-mutation", - ) - - def _replayed_payout_window_reproducible( - self, - context: PrismJobContext, - ) -> bool: - """Whether a reconstructed candidate's payout window replays intact. - - Append-invalidation epochs are process-local, so the landing epoch - fence stands down for a candidate rebuilt from durable intent. The - durable ledger is the surviving authority instead: the reward-window - contract keeps the share window replayable at the artifact's declared - anchor, so a share row that became durably visible only after the - recorded window walk -- on either side of a restart -- appears in the - replayed window while the recorded coinbase omits it. Only omissions - fail the candidate: an appended row leaves the tip, height, and carry - balances untouched, which is exactly why no other landing fence can - see it, while a recorded row absent from the replay is not this - hazard (share rows are append-only, and drift from settled payout - state is governed by the reorg and prior-balance fences). - """ - runtime = self._runtime - found_block = getattr(context, "found_block", None) - anchor_ms = ( - found_block.get("anchor_job_issued_at_ms") - if isinstance(found_block, dict) - else None - ) - network_difficulty = ( - found_block.get("network_difficulty") - if isinstance(found_block, dict) - else None - ) - audit_share_window = getattr(runtime.ledger, "audit_share_window", None) - window_covers = getattr(runtime.ledger, "candidate_window_covers", None) - if ( - anchor_ms is None - or network_difficulty is None - or not (callable(window_covers) or callable(audit_share_window)) - ): - # Fail closed: a candidate whose window cannot be replayed at a - # declared anchor cannot prove its coinbase pays the window the - # durable ledger requires. - return False - try: - if callable(window_covers): - return window_covers( - context.shares_json, - anchor_job_issued_at_ms=int(anchor_ms), - network_difficulty=int(network_difficulty), - ) - # Compatibility for in-memory ledgers and older test adapters. - # The PostgreSQL path returns only a boolean, never durable rows. - durable_rows = audit_share_window( - anchor_job_issued_at_ms=int(anchor_ms), - network_difficulty=int(network_difficulty), - ) - return disk_window_covers(recorded_share_ids(context.shares_json), durable_rows) - except DaemonWindowMirrorDivergence: - # Fail closed, exactly like a candidate whose window cannot be - # replayed at all: a recorded window the coordinator can no - # longer read cannot prove the coinbase pays it. - self._note_window_mirror_divergence() - return False - - def normalized_prior_balances(self, balances: list[dict[str, object]]) -> list[dict[str, object]]: - rows = [ - { - "recipient_id": str(balance.get("recipient_id", "")), - "order_key": str(balance.get("order_key", "")), - "p2mr_program_hex": str(balance.get("p2mr_program_hex", "")), - "balance_sats": int(balance.get("balance_sats", 0)), - } - for balance in balances - ] - rows.sort( - key=lambda row: ( - row["order_key"], - row["recipient_id"], - row["p2mr_program_hex"], - row["balance_sats"], - ) - ) - return rows - - @staticmethod - def settlement_balances_by_program( - balances: list[dict[str, object]], - ) -> dict[str, int]: - """Aggregate owed balances by P2MR program for settlement equality. - - A miner may re-authorize the same payout address under any valid - spelling (bech32 permits an all-uppercase form), which changes the - recipient/order-key identity strings while committing to the same - payout program. Ledger and preview views can then label one program - with different identity strings while agreeing on every settled - amount, so settlement equality must be judged on the program and - amount alone. Identity strings stay display metadata. Zero totals - are dropped to match both the ledger reader (nonzero HAVING filter) - and the preview builder (zero balances skipped). - """ - aggregated: dict[str, int] = {} - for balance in balances: - program = str(balance.get("p2mr_program_hex", "")).lower() - aggregated[program] = aggregated.get(program, 0) + int( - balance.get("balance_sats", 0) - ) - return {program: total for program, total in aggregated.items() if total != 0} - - def prior_balances_match_current(self, prior_balances: list[dict[str, object]]) -> bool: - runtime = self._runtime - return runtime.settlement_balances_by_program( - prior_balances - ) == runtime.settlement_balances_by_program(runtime.ledger.current_prior_balances()) - - def payout_state_metrics_lines(self) -> list[str]: - runtime = self._runtime - runtime._ensure_job_cache_state() - with runtime._payout_state_metrics_lock: - state_histograms = { - name: { - "buckets": dict(histogram["buckets"]), - "sum": float(histogram["sum"]), - "count": int(histogram["count"]), - } - for name, histogram in runtime.payout_state_histograms.items() - } - gate_histograms = { - relation: { - "buckets": dict(histogram["buckets"]), - "sum": float(histogram["sum"]), - "count": int(histogram["count"]), - } - for relation, histogram in runtime.payout_gate_wait_histograms.items() - } - phase_histograms = { - key: { - "buckets": dict(histogram["buckets"]), - "sum": float(histogram["sum"]), - "count": int(histogram["count"]), - } - for key, histogram in self.payout_window_build_phase_histograms.items() - } - discarded = runtime.payout_state_candidates_discarded - - metric_names = { - "preparation": "qbit_prism_payout_preparation_seconds", - "publish": "qbit_prism_payout_publish_seconds", - "first_delivery": "qbit_prism_payout_invalidation_first_delivery_seconds", - } - descriptions = { - "preparation": "Payout reconciliation and candidate preparation outside delivery publication.", - "publish": "Atomic payout generation/cache publication gate-hold time.", - "first_delivery": "Payout invalidation to first delivery of the published generation.", - } - lines: list[str] = [] - for name, metric_name in metric_names.items(): - histogram = state_histograms[name] - buckets = histogram["buckets"] - assert isinstance(buckets, dict) - lines.extend( - [ - f"# HELP {metric_name} {descriptions[name]}", - f"# TYPE {metric_name} histogram", - *[ - f'{metric_name}_bucket{{le="{bucket:g}"}} {int(buckets.get(bucket, 0))}' - for bucket in PRISM_PAYOUT_SECONDS_BUCKETS - ], - f'{metric_name}_bucket{{le="+Inf"}} {histogram["count"]}', - f'{metric_name}_sum {float(histogram["sum"]):.6f}', - f'{metric_name}_count {histogram["count"]}', - ] - ) - - gate_name = "qbit_prism_payout_gate_wait_seconds" - lines.extend( - [ - "# HELP qbit_prism_payout_gate_wait_seconds Delivery admission wait by generation relationship to the published payout state.", - "# TYPE qbit_prism_payout_gate_wait_seconds histogram", - ] - ) - for relation in PRISM_PAYOUT_DELIVERY_GENERATIONS: - histogram = gate_histograms[relation] - buckets = histogram["buckets"] - assert isinstance(buckets, dict) - lines.extend( - [ - *[ - f'{gate_name}_bucket{{generation="{relation}",le="{bucket:g}"}} {int(buckets.get(bucket, 0))}' - for bucket in PRISM_PAYOUT_SECONDS_BUCKETS - ], - f'{gate_name}_bucket{{generation="{relation}",le="+Inf"}} {histogram["count"]}', - f'{gate_name}_sum{{generation="{relation}"}} {float(histogram["sum"]):.6f}', - f'{gate_name}_count{{generation="{relation}"}} {histogram["count"]}', - ] - ) - phase_name = "qbit_prism_payout_window_build_phase_seconds" - lines.extend( - [ - f"# HELP {phase_name} Payout-window build time by phase (PostgreSQL reads, in-process record conversion, Rust daemon prepare or upload, preparation-lock waits) and by how the build ended, observed on every exit so aborted and failed builds keep the time they measured. Both label sets are closed.", - f"# TYPE {phase_name} histogram", - ] - ) - for phase in PRISM_PAYOUT_WINDOW_BUILD_PHASES: - for outcome in PRISM_PAYOUT_WINDOW_BUILD_OUTCOMES: - histogram = phase_histograms[(phase, outcome)] - buckets = histogram["buckets"] - assert isinstance(buckets, dict) - labels = f'phase="{phase}",outcome="{outcome}"' - lines.extend( - [ - *[ - f'{phase_name}_bucket{{{labels},le="{bucket:g}"}} {int(buckets.get(bucket, 0))}' - for bucket in PRISM_PAYOUT_SECONDS_BUCKETS - ], - f'{phase_name}_bucket{{{labels},le="+Inf"}} {histogram["count"]}', - f'{phase_name}_sum{{{labels}}} {float(histogram["sum"]):.6f}', - f'{phase_name}_count{{{labels}}} {histogram["count"]}', - ] - ) - lines.extend( - [ - "# HELP qbit_prism_payout_candidates_discarded_total Prepared payout candidates discarded after source supersession.", - "# TYPE qbit_prism_payout_candidates_discarded_total counter", - f"qbit_prism_payout_candidates_discarded_total {discarded}", - ] - ) - return lines - - -__all__ = [ - "AcceptedBlockPayoutTransition", - "AcceptedParentPayoutPreviewPending", - "DEFAULT_ACCEPTED_BLOCK_PAYOUT_PREVIEW_WAIT_SECONDS", - "DEFAULT_PRISM_PAYOUT_RECONCILE_SUPERSESSION_RETRIES", - "PRISM_PAYOUT_ARTIFACT_REARM_BACKOFF_CAP", - "PRISM_PAYOUT_DELIVERY_GENERATIONS", - "PRISM_PAYOUT_SECONDS_BUCKETS", - "PRISM_PAYOUT_WINDOW_BUILD_OUTCOMES", - "PRISM_PAYOUT_WINDOW_BUILD_PHASES", - "PRISM_TIP_REFRESH_WINDOW_STALENESS_SECONDS", - "PayoutDeliveryAdmission", - "PayoutLedgerArtifact", - "PayoutStateArtifact", - "PayoutStateCandidate", - "PayoutStateDeliveryGate", - "PayoutStatePublicationBlocked", - "PayoutStateRuntime", - "PayoutStateService", - "PayoutStateSnapshot", - "PublishedPayoutState", - "TemplateRefreshBlocked", - "TemplateRefreshSuperseded", - "_IncrementalPayoutArtifactWindow", - "_PayoutWindowMaterialization", - "canonical_json_sha256", - "canonical_json_text", -] diff --git a/lab/prism/prism_coordinator.py b/lab/prism/prism_coordinator.py deleted file mode 100644 index fee9b8fa7..000000000 --- a/lab/prism/prism_coordinator.py +++ /dev/null @@ -1,10216 +0,0 @@ -#!/usr/bin/env python3 -"""Minimal live direct qbit Stratum coordinator for PRISM regtest proof.""" - -from __future__ import annotations - -import copy -from collections import OrderedDict -from concurrent.futures import ( - Future, - ThreadPoolExecutor, -) -from contextlib import ExitStack, contextmanager -import faulthandler -import hashlib -import json -import os -import queue -import random # noqa: F401 - patch seam for tip-refresh holdoff jitter tests -import shlex -import signal -import socket -import subprocess -import tempfile -import threading -import time -import traceback -import uuid -from dataclasses import replace as dataclass_replace -from decimal import Decimal, ROUND_CEILING -from http.server import BaseHTTPRequestHandler -from pathlib import Path -from typing import Any, Callable, Iterator, Mapping, Sequence - -import sys - -if not __package__: - sys.path.insert(0, str(Path(__file__).resolve().parents[2])) - -from lab.auxpow import stratum_codec, vardiff -from lab.prism import direct_stratum -# Compatibility re-exports; new callers should import lab.prism.bounded_executor. -from lab.prism.bounded_executor import ( - _BoundedPriorityExecutor, # noqa: F401 - compatibility re-export - _DeliveryQueueFull, # noqa: F401 - compatibility re-export -) -from lab.prism.prism_tools import prism_tool_command -# Compatibility re-exports; new callers should import lab.prism.background_services. -from lab.prism.writer_lease_timing import ( - WRITER_LEASE_HEARTBEAT_SCHEDULER_SLACK_SECONDS, -) -from lab.prism.background_services import ( - BackgroundServiceRegistry, # noqa: F401 - compatibility re-export - BackgroundServiceSpec, # noqa: F401 - compatibility re-export - LedgerLeaseHeartbeatPorts, - LedgerLeaseHeartbeatService, - WatchdogPorts, - WatchdogService, - _LeaseHeartbeatStateField, - guard_session_prover as _lease_guard_session_prover, - guard_session_verifier as _lease_guard_session_verifier, -) -# Compatibility re-exports; new callers should import lab.prism.coordinator_shutdown. -from lab.prism.coordinator_shutdown import ( - CoordinatorShutdownController, # noqa: F401 - compatibility re-export - ShutdownInProgress, # noqa: F401 - compatibility re-export - ledger_writer_operation, # noqa: F401 - compatibility re-export -) -from lab.prism.coordinator_config import ( - BLOCK_LANDING_DB_TIMEOUT_WATCHDOG_FRACTION, # noqa: F401 - CoordinatorConfig, - apply_python_switch_interval, - DEFAULT_ACCEPTED_PARENT_UNRESOLVED_DEPTH_MAX, # noqa: F401 - DEFAULT_BLOCK_LANDING_DB_TIMEOUT_MAX_SECONDS, # noqa: F401 - DEFAULT_BLOCK_LANDING_DB_TIMEOUT_SECONDS, # noqa: F401 - DEFAULT_BLOCK_SUBMIT_LOCK_WAIT_LOG_SECONDS, # noqa: F401 - DEFAULT_BLOCK_SUBMIT_RPC_TIMEOUT_SECONDS, - DEFAULT_BLOCK_SUBMIT_STUCK_CALL_EXIT_SECONDS, # noqa: F401 - DEFAULT_CTV_FANOUT_FEE_PREMIUM_BPS, - DEFAULT_DIRECT_COINBASE_PAYOUT_FLOOR_SATS, - DEFAULT_MAX_COINBASE_SETTLEMENT_OUTPUTS, - DEFAULT_MAX_CTV_FANOUT_RECIPIENTS_PER_TRANSACTION, - DEFAULT_MAX_DIRECT_COINBASE_OUTPUTS, - DEFAULT_PRISM_BUNDLE_BUILD_TIMEOUT_SECONDS, - DEFAULT_PRISM_COORDINATION_BLOCKED_EXIT_SECONDS, - DEFAULT_PRISM_CTV_BROADCASTER_CHUNK_SIZE, - DEFAULT_PRISM_DISCONNECTED_JOB_RETENTION, # noqa: F401 - DEFAULT_PRISM_HEALTH_PENDING_REFRESH_MAX_AGE_SECONDS, - DEFAULT_PRISM_HEALTH_REFRESH_SECONDS, # noqa: F401 - compatibility re-export - DEFAULT_PRISM_HEALTH_TIP_POLL_MAX_AGE_SECONDS, - DEFAULT_PRISM_INITIAL_JOB_MAX_WORKERS, - DEFAULT_PRISM_JOB_BUILD_CANCEL_GRACE_SECONDS, - DEFAULT_PRISM_JOB_BUILD_EXECUTOR_WORKERS, - DEFAULT_PRISM_JOB_BUILD_TIMEOUT_SECONDS, - DEFAULT_PRISM_JOB_BUNDLE_CACHE_SECONDS, - DEFAULT_PRISM_LEDGER_LEASE_EXTERNAL_FENCE_TIMEOUT_SECONDS, - DEFAULT_PRISM_LEDGER_LEASE_HEARTBEAT_EXIT_TIMEOUT_SECONDS, - DEFAULT_PRISM_LEDGER_LEASE_HEARTBEAT_FAILURE_SECONDS, - DEFAULT_PRISM_LEDGER_LEASE_HEARTBEAT_MONITOR_SECONDS, - DEFAULT_PRISM_LEDGER_LEASE_HEARTBEAT_SECONDS, - DEFAULT_PRISM_METRICS_REFRESH_SECONDS, # noqa: F401 - compatibility re-export - DEFAULT_PRISM_MINING_HEALTH_STARTUP_GRACE_SECONDS, # noqa: F401 - DEFAULT_PRISM_OBSERVED_TIP_ACCEPT_WINDOW_SECONDS, # noqa: F401 - DEFAULT_PRISM_PAYOUT_ARTIFACT_FULL_RESCAN_SECONDS, # noqa: F401 - DEFAULT_PRISM_PAYOUT_ARTIFACT_MAX_ANCHOR_AGE_SECONDS, # noqa: F401 - DEFAULT_PRISM_PAYOUT_ARTIFACT_MIN_BUILD_INTERVAL_SECONDS, # noqa: F401 - DEFAULT_PRISM_PAYOUT_ARTIFACT_REANCHOR_SECONDS, # noqa: F401 - DEFAULT_PRISM_PAYOUT_ARTIFACT_REARM_MIN_SECONDS, # noqa: F401 - DEFAULT_PRISM_REORG_RECONCILE_CACHE_SECONDS, - DEFAULT_PRISM_ROUTINE_ADMISSION_DEADLINE_SECONDS, # noqa: F401 - DEFAULT_PRISM_STALE_GRACE_SECONDS, - DEFAULT_PRISM_STRATUM_ACCEPT_RESOURCE_EXHAUSTION_BACKOFF_SECONDS, # noqa: F401 - DEFAULT_PRISM_STRATUM_BIND_RETRY_SECONDS, # noqa: F401 - DEFAULT_PRISM_STRATUM_INITIAL_JOB_TIMEOUT_SECONDS, - DEFAULT_PRISM_STRATUM_LISTEN_BACKLOG, # noqa: F401 - DEFAULT_PRISM_STRATUM_MAX_CONNECTIONS, - DEFAULT_PRISM_STRATUM_MAX_CONNECTIONS_PER_USERNAME, # noqa: F401 - DEFAULT_PRISM_STRATUM_MAX_PENDING_INITIAL_JOBS, - DEFAULT_PRISM_STRATUM_SEND_TIMEOUT_SECONDS, - DEFAULT_PRISM_TEMPLATE_MAX_AGE_SECONDS, - DEFAULT_PRISM_TIP_REFRESH_FAILURE_HOLDOFF_SECONDS, # noqa: F401 - DEFAULT_PRISM_TIP_REFRESH_MAX_WORKERS, - DEFAULT_PRISM_WATCHDOG_LEASE_RELEASE_TIMEOUT_SECONDS, - DEFAULT_PRISM_WATCHDOG_TIMEOUT_SECONDS, # noqa: F401 - DEFAULT_PRISM_WORKER_METRICS_LIMIT, - DEFAULT_PRISM_WRITER_QUIESCENCE_TIMEOUT_SECONDS, - LEASE_AUTHORITY_MARGIN_HEADROOM_SECONDS, - DEFAULT_SHARE_COMMIT_BATCH_SIZE, # noqa: F401 - compatibility re-export - DEFAULT_SHARE_COMMIT_LINGER_MILLISECONDS, # noqa: F401 - compatibility re-export - DEFAULT_SHARE_COMMIT_TIMEOUT_SECONDS, # noqa: F401 - compatibility re-export - StratumListenerProfile, # noqa: F401 - compatibility re-export - TESTNET_QBIT_CHAINS, # noqa: F401 - compatibility re-export - VALID_COINBASE_OUTPUT_POLICIES, - default_prism_payout_policy, - default_prism_username_fallback_address, # noqa: F401 - compatibility re-export - env, - env_bool, - env_heap_census_config, - env_int, - env_nonnegative_float, - env_nonnegative_int, - env_optional, - env_optional_positive_int, - env_optional_positive_int_with_legacy, - env_positive_float, - env_positive_int, - env_positive_int_with_legacy, - load_coordinator_config, - load_share_weights, - resolve_initial_job_max_workers, # noqa: F401 - compatibility re-export - validate_hex, - validate_initial_job_max_workers, # noqa: F401 - compatibility re-export - validate_job_build_executor_workers, # noqa: F401 - compatibility re-export - validate_payout_artifact_age_bounds, # noqa: F401 - compatibility re-export -) -from lab.prism.ctv_broadcaster import CtvFanoutBroadcaster -from lab.prism.ctv_broadcaster_daemon import ( - CtvFanoutBroadcastDaemon, - CtvFanoutChunkResult, - CtvFanoutDaemonResult, -) -# Compatibility re-exports; new callers should import lab.prism.ctv_runtime. -from lab.prism.ctv_runtime import ( - CtvRuntimeConfig, - CtvRuntimeService, -) -# Compatibility re-exports; new callers should import the owning J1 modules. -from lab.prism.audit_artifacts import ( - AuditArtifactConfig, - AuditArtifactStore, - AuditPublicationIdentity, -) -from lab.prism.audit_http import ( - AuditHttpConfig, - AuditHttpFacade, - AuditHttpPort, -) -from lab.prism.observability import ( - METRICS_STATE_FRESH, - METRICS_STATE_UNAVAILABLE, - MetricsSnapshotResponse, - MiningDeliveryInputs, - ObservabilityPort, - ObservabilityService, -) -from lab.prism.block_candidates import ( - BLOCK_SUBMITTER_WAIT_HEARTBEAT_SLICE_SECONDS, - BlockCandidateAttemptResult, # noqa: F401 - compatibility re-export - BlockCandidateCompatibilityField, - BlockCandidatePorts, - BlockCandidateRunResult, # noqa: F401 - compatibility re-export - BlockCandidateService, - BlockCandidateStateField, - BlockSubmitterDatabaseTimeout, # noqa: F401 - compatibility re-export - DEFAULT_BLOCK_ACCOUNTING_QUEUE_DEPTH, # noqa: F401 - compatibility re-export - DEFAULT_BLOCK_CANDIDATE_RETRY_INITIAL_SECONDS, - DEFAULT_BLOCK_CANDIDATE_RETRY_MAX_SECONDS, - MAX_BLOCK_SUBMITTER_STUCK_CALL_WORKERS, # noqa: F401 - compatibility re-export - MAX_PENDING_BLOCK_CANDIDATES, - PRISM_REJECTION_BLOCK_ACCEPT_PENDING, - PRISM_REJECTION_LEDGER_CONFIRMATION_SUPERSEDED, - PRISM_STALE_JOB_ABANDON_CLASSES, - PrismBlockCandidate, - _BlockCandidateAccountingTask, # noqa: F401 - compatibility re-export - _BlockCandidateDispositionFlight, # noqa: F401 - compatibility re-export - _BlockCandidateDispositionLease, # noqa: F401 - compatibility re-export - _BlockCandidateNodeSubmission, - _BlockSubmitterLedgerCall, # noqa: F401 - compatibility re-export - _BlockSubmitterRpcCall, # noqa: F401 - compatibility re-export - _STATE_FIELD_MAP as _BLOCK_CANDIDATE_STATE_FIELD_MAP, - block_candidate_from_intent as decode_block_candidate_intent, - block_candidate_intent as encode_block_candidate_intent, - compatibility_default as candidate_compatibility_default, -) -from lab.prism.block_finalization import ( - PRISM_REJECTION_BLOCK_STALE, - PRISM_REJECTION_CANDIDATE_AUDIT_MISMATCH, - PRISM_REJECTION_LEDGER_CONFIRMATION_FAILED, - PRISM_REJECTION_SUBMITBLOCK_REJECTED, - BlockFinalizationService, -) -from lab.prism.accepted_preview_telemetry import ( - ensure_accepted_preview_telemetry, -) -from lab.prism.metrics import MetricsRenderer -from lab.prism.reorg_reconciler import ( - DEFAULT_PRISM_RECONCILE_FLIGHT_WAIT_SECONDS, - PRISM_RECONCILE_PREFETCH_JOIN_TIMEOUT_SECONDS, - ReorgCompatibilityField, - ReorgPorts, - ReorgReconcilerService, - qbit_chain_view_untrusted as reorg_chain_view_untrusted, -) -from lab.prism.vardiff_service import ( - PRISM_VARDIFF_IDLE_SECONDS_BUCKETS, - PRISM_VARDIFF_IDLE_SKIP_REASONS, - VARDIFF_COMPATIBILITY_FIELDS, - IdleRetargetRequest as _IdleRetargetRequest, - VardiffCompatibilityField, - VardiffService, -) -from lab.prism.share_submission import ( - PRISM_REJECTION_BACKEND_RPC_UNAVAILABLE, - PRISM_REJECTION_DUPLICATE_SHARE, - PRISM_REJECTION_INVALID_EXTRANONCE, - PRISM_REJECTION_INVALID_NTIME_OR_NONCE, - PRISM_REJECTION_LOW_DIFFICULTY, - PRISM_REJECTION_MALFORMED_SUBMIT, - PRISM_REJECTION_POOL_CLOSED, - PRISM_REJECTION_STALE_JOB, - PRISM_REJECTION_UNAUTHORIZED_WORKER, - PRISM_REJECTION_UNKNOWN_JOB, - BlockSolvesDroppedCompatibilityField, - RecentShareCompatibilityField, - RecentShareIndex, - ShareSubmissionPorts, - ShareSubmissionService, - SubmitControlSnapshot, - empty_share_ack_histograms, -) -from lab.prism.bundle_compiler import ( - BundleCompiler, - PRISM_BUILDER_PHASE_METRICS_PREFIX, # noqa: F401 - compatibility re-export - PRISM_SERVE_BUILDER_PROTOCOL_VERSION, # noqa: F401 - compatibility re-export - PRISM_SERVE_BUILDER_WINDOW_CACHE_ENTRIES, # noqa: F401 - compatibility re-export - PRISM_SPOOL_SPLICE_CHUNK_BYTES, # noqa: F401 - compatibility re-export - _ServeBuilderClient, # noqa: F401 - compatibility re-export - _ServeBuilderUnavailable, # noqa: F401 - compatibility re-export - _ShareWindowSerialization, # noqa: F401 - compatibility re-export - _compact_share_payload, # noqa: F401 - compatibility re-export - _share_window_spool_file, # noqa: F401 - compatibility re-export - canonical_bundle_bytes, -) -from lab.prism.job_bundle import ( - CachedJobBundle, - JobBuildAdmissionDeadlineExceeded, # noqa: F401 - compatibility re-export - PRISM_JOB_BUILD_SECONDS_BUCKETS, - JobBuildCancellation as _JobBuildCancellation, - JobBuildCancelled, - JobBuildFlight as _JobBuildFlight, - JobBuildRequest as _JobBuildRequest, - JobBuildSuperseded, - JobBundleBuildControl as _JobBundleBuildControl, - JobBundleBuildSuperseded as _JobBundleBuildSuperseded, - JobBundleService, -) -from lab.prism.job_delivery import ( - EvictedJobEntry, - JobBuildFailed as _JobBuildFailed, - JobDeliveryService, - PRISM_CREDIT_POLICY_STALE_GRACE, - PRISM_DELIVERY_PRIORITY_INITIAL, - PRISM_DELIVERY_PRIORITY_NEW_TIP, - PRISM_DELIVERY_PRIORITY_SAME_TIP, - PRISM_EVICTED_JOB_CAPACITY_SCOPES, - PRISM_EVICTED_JOB_CLASSES, - PRISM_EVICTED_JOB_SUBMIT_OUTCOMES, - PendingInitialJob, - PrismJobContext, -) -from lab.prism.payout_state import ( - PayoutStatePublicationBlocked, - TemplateRefreshBlocked, - TemplateRefreshSuperseded, -) -from lab.prism.template_artifacts import ( - CachedTemplateArtifacts, - QbitTipTemplateSnapshot, - TemplateArtifactRepository, # noqa: F401 - compatibility re-export -) -from lab.prism.tip_refresh import ( - FanoutCancellation as _FanoutCancellation, - PRISM_TIP_REFRESH_BUILD_PHASES, # noqa: F401 - compatibility re-export - PRISM_TIP_REFRESH_COVERAGE_TARGETS, # noqa: F401 - compatibility re-export - PRISM_TIP_REFRESH_FAILURE_HOLDOFF_JITTER_FRACTION, # noqa: F401 - compatibility re-export - PRISM_TIP_REFRESH_REENTRY_BACKOFF_SECONDS, # noqa: F401 - compatibility re-export - PRISM_TIP_REFRESH_SECONDS_BUCKETS, # noqa: F401 - compatibility re-export - PRISM_TIP_REFRESH_WAVE_OUTCOMES, # noqa: F401 - compatibility re-export - PRISM_TIP_REFRESH_WAVE_PASS_BUDGET, # noqa: F401 - compatibility re-export - RefreshResult, - RetainedCollectionRefresh, - TipRefreshService, - TipRefreshValidationToken, - _TipRefreshEpochCoverage, # noqa: F401 - compatibility re-export - _TipRefreshFanoutSuperseded, # noqa: F401 - compatibility re-export - _TipRefreshTrustBlocked, # noqa: F401 - compatibility re-export -) -# Compatibility re-exports; new callers should import lab.prism.payout_state. -from lab.prism.payout_state import ( - DEFAULT_ACCEPTED_BLOCK_PAYOUT_PREVIEW_WAIT_SECONDS, # noqa: F401 - compatibility re-export - DEFAULT_PRISM_PAYOUT_RECONCILE_SUPERSESSION_RETRIES, - PRISM_PAYOUT_ARTIFACT_REARM_BACKOFF_CAP, # noqa: F401 - compatibility re-export - PRISM_PAYOUT_DELIVERY_GENERATIONS, # noqa: F401 - compatibility re-export - PayoutLedgerArtifact, - PayoutStateArtifact, - PayoutStateCandidate, - PayoutStateService, - PayoutStateSnapshot, # noqa: F401 - compatibility re-export - _IncrementalPayoutArtifactWindow, # noqa: F401 - compatibility re-export - _PayoutWindowMaterialization, -) -from lab.prism.process_telemetry import install_heap_census -# Compatibility re-exports; new callers should import lab.prism.share_writer. -from lab.prism.share_writer import ( - MAX_PENDING_SHARE_APPENDS, - PENDING_SHARE_COMMIT_WARN_SECONDS as PRISM_PENDING_SHARE_COMMIT_WARN_SECONDS, # noqa: F401 - PendingShareAppend, - ShareWriter, - ShareWriterCompatibilityField, - ShareWriterError, # noqa: F401 - compatibility re-export - ShareWriterQueueFull, # noqa: F401 - compatibility re-export -) -# Compatibility re-exports; new callers should import lab.prism.stratum_session. -from lab.prism.stratum_session import ( - ClientState, - P2mrAddressValidator, - SessionRegistry, # noqa: F401 - compatibility re-export - StratumError, - StratumSessionService, - WorkerIdentity, - _P2mrAddressValidationFlight, # noqa: F401 - compatibility re-export - apply_stratum_send_timeout as apply_socket_send_timeout, - client_vardiff_lock, - difficulty_payload as stratum_difficulty_payload, - error_payload as stratum_error_payload, - job_payload as stratum_job_payload, - result_payload as stratum_result_payload, - stratum_accept_heartbeat_names as configured_accept_heartbeat_names, -) -# Compatibility re-exports; new callers should import lab.prism.progress_health. -from lab.prism.progress_health import ( - EligibilitySnapshot, # noqa: F401 - compatibility re-export - ProgressHealthConfig, - ProgressHealthService, - ProgressHealthSnapshot, - WorkGeneration, -) -from lab.prism.rpc import ( - DEFAULT_QBIT_RPC_CALL_TIMEOUT_SECONDS, - JsonRpc, - _QBIT_RPC_NO_TRANSPORT_RETRY_METHODS, # noqa: F401 - compatibility re-export -) -from lab.prism.candidate_store import IncompatibleCandidateSchema -from lab.prism.share_ledger import ( - DEFAULT_AUDIT_SHARE_SEGMENT_SIZE, - DEFAULT_CTV_BROADCAST_ATTEMPT_DETAIL_LIMIT, - DEFAULT_CTV_BROADCAST_RETRY_BACKOFF_SECONDS, - DEFAULT_LEASE_ACQUIRE_ATTEMPTS, - DEFAULT_LEASE_ACQUIRE_LOCK_TIMEOUT_SECONDS, - DEFAULT_POSTGRES_IDLE_IN_TRANSACTION_TIMEOUT_SECONDS, - DEFAULT_POSTGRES_TCP_KEEPALIVES_COUNT, - DEFAULT_POSTGRES_TCP_KEEPALIVES_IDLE_SECONDS, - DEFAULT_POSTGRES_TCP_KEEPALIVES_INTERVAL_SECONDS, - DEFAULT_WRITER_LEASE_ADOPTION_SILENCE_SECONDS, # noqa: F401 - compatibility re-export - DaemonWindowMirrorDivergence, - PendingShare, - PsqlShareLedger, - SingleWriterShareLedger, - WRITER_LEASE_HEARTBEAT_SESSION_PREFIX, - WRITER_LEASE_VERIFICATION_MAX_STATEMENTS, # noqa: F401 - compatibility re-export - WriterLeaseRenewalDeferred, # noqa: F401 - compatibility re-export -) - -MAX_PRISM_JOB_BUNDLE_CACHE_ENTRIES = 128 -# The reorg reconcile memo/flight/prefetch constants and machinery moved to -# lab.prism.reorg_reconciler; the two runtime-override defaults the port -# wiring reads are re-imported above. -# Read-to-ack latency labels for mining.submit now live with the -# share-submission owner (PRISM_SHARE_ACK_RESULTS is re-exported above). -# Cancellation-check slice while an initial request rides a subscribed -# publication-priority build promise. Promise completion wakes the waiter -# immediately; this only bounds how stale a cancellation can go unnoticed. -PRISM_INITIAL_JOB_SUBSCRIBE_POLL_SECONDS = 0.25 -# A flight whose executor future is finished normally leaves its slot inside -# the future's done callback. The admission sweep only treats such a flight -# as orphaned after this grace, so a callback that is merely between future -# completion and slot cleanup is never mistaken for a dead one. -PRISM_JOB_BUILD_ORPHAN_SWEEP_GRACE_SECONDS = 1.0 -# The vardiff idle worker/pending limits moved to lab.prism.vardiff_service; -# the bucket/skip registries are re-imported above for metric consumers. -# Accepted shares use a small, bounded group-commit queue. Every submitter -# waits for its batch's Postgres commit before receiving Stratum success, so -# this is a latency-smoothing bound rather than a durable backlog. -# The reward window is 8x network difficulty (must match PRISM_WINDOW_MULTIPLIER -# in crates/qbit-prism/src/lib.rs and the SQL). The job-build snapshot only needs -# the shares that window can cover; requesting a margin above it returns a -# guaranteed superset (the audit bundle re-selects the exact 8x window, so the -# digest is unchanged) while keeping the query O(window), not O(ledger history). -PRISM_REWARD_WINDOW_MULTIPLIER = 8 -PRISM_SNAPSHOT_WINDOW_MARGIN = 2 -# Evicted jobs remain tied to their immutable validation context. Current-tip -# entries use an independent bounded TTL; once their tip is replaced, only the -# existing stale-grace lifetime and eligibility rules can retain/credit them. -# Extranonce1 placeholder used for the shared per-template job build. The -# stratum coinbase split cuts the whole extranonce window (extranonce1 + -# zeroed extranonce2) out of coinb1/coinb2, so the placeholder value never -# reaches miners; real connections stamp their own extranonce1 into the job. -# Client extranonce1 values start at 1, so the placeholder never collides. -PRISM_JOB_EXTRANONCE1_PLACEHOLDER_HEX = "00000000" -# The job-build seconds buckets, phase registry, and cache-kind registry are -# canonical in lab.prism.job_bundle; the metrics renderer imports them there. -# The ten submit-path rejection reasons moved to lab.prism.share_submission -# and the four accepted-block finalization reasons moved to -# lab.prism.block_finalization; both groups are re-exported above. Only the -# process-internal reason stays here. -PRISM_REJECTION_INTERNAL_ERROR = "internal-error" -PRISM_RETRYABLE_BLOCK_CANDIDATE_REASONS = frozenset( - { - PRISM_REJECTION_BACKEND_RPC_UNAVAILABLE, - PRISM_REJECTION_INTERNAL_ERROR, - PRISM_REJECTION_LEDGER_CONFIRMATION_FAILED, - PRISM_REJECTION_CANDIDATE_AUDIT_MISMATCH, - PRISM_REJECTION_BLOCK_ACCEPT_PENDING, - } -) -# Used only by lightweight embedders that bypass dataclass/coordinator -# construction. Production instances install these locks eagerly in __init__. -# Serializing the fallback prevents concurrent first-touch callers from ever -# publishing different lock objects for the same state. -_HOT_PATH_LOCK_INITIALIZATION_LOCK = threading.Lock() -# Aggregation bucket for per-worker share metrics once the distinct-worker -# label budget is exhausted. -PRISM_WORKER_METRICS_OVERFLOW_LABEL = "_other" -PRISM_REJECTION_REASON_IDS = ( - PRISM_REJECTION_STALE_JOB, - PRISM_REJECTION_DUPLICATE_SHARE, - PRISM_REJECTION_LOW_DIFFICULTY, - PRISM_REJECTION_MALFORMED_SUBMIT, - PRISM_REJECTION_UNAUTHORIZED_WORKER, - PRISM_REJECTION_UNKNOWN_JOB, - PRISM_REJECTION_INVALID_EXTRANONCE, - PRISM_REJECTION_INVALID_NTIME_OR_NONCE, - PRISM_REJECTION_CANDIDATE_AUDIT_MISMATCH, - PRISM_REJECTION_SUBMITBLOCK_REJECTED, - PRISM_REJECTION_BACKEND_RPC_UNAVAILABLE, - PRISM_REJECTION_INTERNAL_ERROR, - PRISM_REJECTION_POOL_CLOSED, - PRISM_REJECTION_BLOCK_STALE, - PRISM_REJECTION_LEDGER_CONFIRMATION_FAILED, - PRISM_REJECTION_LEDGER_CONFIRMATION_SUPERSEDED, -) -PRISM_TEMPLATE_FINGERPRINT_VOLATILE_KEYS = frozenset( - { - # qbit can legitimately advance these without making already issued - # jobs stale. Rebuilding every miner job for clock-only changes would - # turn the poller into continuous audit-bundle churn. - "curtime", - "longpollid", - "mintime", - } -) - - -class _ObservedRLock: - """RLock with no mutex acquisition on uncontended acquisitions. - - The coordinator lock protects control-plane publication state. Its wait - counters intentionally record only acquisitions that fail an immediate - probe, and the shared mutex guarding them is therefore never taken on an - uncontended acquisition, so observing this lock cannot recreate the - share-path convoy the metrics are meant to diagnose. - - A contended count on its own is an absolute number with no denominator, so - a contended percentage cannot be derived from a live process. The - acquisition counter that supplies that denominator does run on every - acquisition, contended or not, and preserves the property above by where - it runs rather than by how often: it is incremented only while the - underlying RLock is already held, which puts it inside a critical section - the lock has already serialised. It therefore adds no mutex acquisition - and no cross-thread ordering that the lock did not already impose, and it - cannot convoy the fast path. - - That placement is also why this counter does not rest on an increment - being atomic under the GIL -- the lock, not the interpreter, is what makes - it safe, so it stays correct on a free-threaded build (see #150, which - audits that class of assumption in this tree). The one unsynchronised - access is the read in contention_snapshot, which holds only - _metrics_lock: it assumes an attribute read yields a whole value rather - than a torn one, and may observe the counter one increment stale. Both are - acceptable for a monotonic counter read by a metrics scrape, and neither - is load-bearing for correctness anywhere else. - - Only granted acquisitions are counted. A wait that times out records - contention without an acquisition, so where a caller passes a timeout the - derived contended percentage is an upper bound. Hold *duration* is - deliberately not instrumented: a re-entrant lock would need per-thread - depth tracking to avoid double counting nested holds, and a clock read on - every acquire and release is exactly the fast-path work this class exists - to avoid. - """ - - def __init__( - self, - *, - wait_observer: Callable[[float], None] | None = None, - ) -> None: - self._lock = threading.RLock() - self._metrics_lock = threading.Lock() - # Guarded by _lock itself, never by _metrics_lock. See the class - # docstring: every write below happens with _lock held. - self._acquisition_count = 0 - self._contention_count = 0 - self._wait_seconds_sum = 0.0 - self._wait_seconds_max = 0.0 - self._wait_observer = wait_observer - - def acquire(self, blocking: bool = True, timeout: float = -1) -> bool: - if not blocking: - if not self._lock.acquire(blocking=False): - return False - self._acquisition_count += 1 - return True - if self._lock.acquire(blocking=False): - self._acquisition_count += 1 - return True - started = time.monotonic() - observer = self._wait_observer - if observer is None: - acquired = self._lock.acquire(blocking=True, timeout=timeout) - else: - deadline = None if timeout < 0 else started + max(0.0, timeout) - acquired = False - while not acquired: - wait_slice = BLOCK_SUBMITTER_WAIT_HEARTBEAT_SLICE_SECONDS - if deadline is not None: - remaining = deadline - time.monotonic() - if remaining <= 0: - break - wait_slice = min(wait_slice, remaining) - acquired = self._lock.acquire( - blocking=True, - timeout=wait_slice, - ) - if not acquired: - observer(max(0.0, time.monotonic() - started)) - waited = max(0.0, time.monotonic() - started) - if acquired: - # Still under _lock, as on the uncontended path above. - self._acquisition_count += 1 - with self._metrics_lock: - self._contention_count += 1 - self._wait_seconds_sum += waited - self._wait_seconds_max = max(self._wait_seconds_max, waited) - return acquired - - def release(self) -> None: - self._lock.release() - - def __enter__(self) -> _ObservedRLock: - self.acquire() - return self - - def __exit__(self, *_args: object) -> None: - self.release() - - def _is_owned(self) -> bool: - return self._lock._is_owned() # type: ignore[attr-defined] - - def contention_snapshot(self) -> tuple[int, int, float, float]: - """Return (acquisitions, contentions, wait_sum, wait_max). - - The acquisition count is read without _lock on purpose -- taking the - coordinator lock to publish a metric would be the observation cost - this class refuses to pay. - """ - with self._metrics_lock: - return ( - self._acquisition_count, - self._contention_count, - self._wait_seconds_sum, - self._wait_seconds_max, - ) - - -def now_ms() -> int: - return int(time.time() * 1000) - - -def qbit_gbt_rules(chain: str) -> list[str]: - rules = ["segwit"] - if chain.strip().lower() == "signet": - rules.append("signet") - return rules - - -def qbit_template_fingerprint(template: dict[str, Any]) -> str: - stable_template = { - key: value - for key, value in template.items() - if key not in PRISM_TEMPLATE_FINGERPRINT_VOLATILE_KEYS - } - encoded = json.dumps( - stable_template, - sort_keys=True, - separators=(",", ":"), - default=str, - ).encode() - return hashlib.sha256(encoded).hexdigest() - - -def canonical_json_text(value: object) -> str: - return json.dumps(value, sort_keys=True, separators=(",", ":"), default=str) - - -def canonical_json_sha256(value: object) -> str: - incremental_digest = getattr(value, "canonical_json_sha256", None) - if callable(incremental_digest): - return str(incremental_digest()) - return hashlib.sha256(canonical_json_text(value).encode()).hexdigest() - - -class _BundlePreparationSuperseded(TemplateRefreshSuperseded): - """The exact work identity lost to a newer tip/template observation. - - Subclasses TemplateRefreshSuperseded: losing the shared-bundle build race - to a newer tip/template observation is coordination churn, so it escapes - the poll without arming the template-refresh failure budget. - """ - - -class _ProgressHealthStateField: - """Route a legacy raw ``_progress_*`` coordinator attribute to its owner. - - G1 owns the aggregate progress state. Reads and writes deliberately take - no lock: the historical white-box protocol holds - ``coordinator._progress_health_lock`` (the service lock) around direct - field access, and the service's own methods lock internally. - """ - - def __init__(self, attribute: str) -> None: - self._attribute = attribute - - def __get__( - self, - instance: "PrismCoordinator | None", - owner: type | None = None, - ) -> object: - if instance is None: - return self - return getattr( - instance._ensure_progress_health_service(), - self._attribute, - ) - - def __set__(self, instance: "PrismCoordinator", value: object) -> None: - setattr( - instance._ensure_progress_health_service(), - self._attribute, - value, - ) - - -class _JobBundleStateField: - """Route one legacy J1 coordinator field to its owner service. - - The descriptor keeps the historical attribute name readable and writable - on the coordinator while :class:`JobBundleService` owns the single - mutable copy, mirroring the progress-health extraction pattern. Direct - test assignment through the legacy name is thereby adopted by the owner. - """ - - def __init__(self, attribute: str) -> None: - self._attribute = attribute - - def __get__( - self, - instance: "PrismCoordinator | None", - owner: type | None = None, - ) -> object: - if instance is None: - return self - return getattr( - instance._ensure_job_bundle_service(), - self._attribute, - ) - - def __set__(self, instance: "PrismCoordinator", value: object) -> None: - setattr( - instance._ensure_job_bundle_service(), - self._attribute, - value, - ) - - -class _TemplateArtifactStateField: - """Route the legacy template-artifact fields to their repository owner.""" - - def __init__(self, attribute: str) -> None: - self._attribute = attribute - - def __get__( - self, - instance: "PrismCoordinator | None", - owner: type | None = None, - ) -> object: - if instance is None: - return self - repository = instance._ensure_job_bundle_service().template_repository - return getattr(repository, self._attribute) - - def __set__(self, instance: "PrismCoordinator", value: object) -> None: - repository = instance._ensure_job_bundle_service().template_repository - if self._attribute == "_template_artifacts": - repository.adopt_template_artifacts(value) - return - repository.adopt_template_artifact_generation(value) - - -class _BundleCompilerStateField: - """Route the legacy serve-builder daemon fields to the compiler owner.""" - - def __init__(self, attribute: str) -> None: - self._attribute = attribute - - def __get__( - self, - instance: "PrismCoordinator | None", - owner: type | None = None, - ) -> object: - if instance is None: - return self - return getattr(instance._ensure_bundle_compiler(), self._attribute) - - def __set__(self, instance: "PrismCoordinator", value: object) -> None: - setattr(instance._ensure_bundle_compiler(), self._attribute, value) - - -class _TipRefreshStateField: - """Route one legacy R1 coordinator field to its owner service. - - The descriptor keeps the historical attribute name readable and writable - on the coordinator while :class:`TipRefreshService` owns the single - mutable copy. The twelve historical plain fields (and the tip-detection - epoch) are lazily created on the service, so ``hasattr``/``getattr`` - -with-default semantics of the pre-extraction coordinator are preserved - exactly: reading an unset field raises AttributeError until first write. - """ - - def __init__(self, attribute: str) -> None: - self._attribute = attribute - - def __get__( - self, - instance: "PrismCoordinator | None", - owner: type | None = None, - ) -> object: - if instance is None: - return self - return getattr( - instance._ensure_tip_refresh_service(), - self._attribute, - ) - - def __set__(self, instance: "PrismCoordinator", value: object) -> None: - setattr( - instance._ensure_tip_refresh_service(), - self._attribute, - value, - ) - - -class _JobDeliveryStateField: - """Route one legacy S2 coordinator field to its owner service. - - The descriptor keeps the historical attribute name readable and writable - on the coordinator while :class:`JobDeliveryService` owns the single - mutable copy, mirroring the J1/G1 extraction pattern. Direct test - assignment through the legacy name is thereby adopted by the owner. - """ - - def __init__(self, attribute: str) -> None: - self._attribute = attribute - - def __get__( - self, - instance: "PrismCoordinator | None", - owner: type | None = None, - ) -> object: - if instance is None: - return self - return getattr( - instance._ensure_job_delivery_service(), - self._attribute, - ) - - def __set__(self, instance: "PrismCoordinator", value: object) -> None: - setattr( - instance._ensure_job_delivery_service(), - self._attribute, - value, - ) - - -class _PayoutStateStateField: - """Route one legacy P1 coordinator field to its owner service. - - The descriptor keeps the historical attribute name readable and writable - on the coordinator while :class:`PayoutStateService` owns the single - mutable copy, mirroring the S1/S2/S3/J1/G1 extraction pattern. Direct - test assignment through the legacy name is thereby adopted by the owner. - """ - - def __init__(self, attribute: str) -> None: - self._attribute = attribute - - def __get__( - self, - instance: "PrismCoordinator | None", - owner: type | None = None, - ) -> object: - if instance is None: - return self - return getattr( - instance._ensure_payout_state_service(), - self._attribute, - ) - - def __set__(self, instance: "PrismCoordinator", value: object) -> None: - setattr( - instance._ensure_payout_state_service(), - self._attribute, - value, - ) - - -class _StratumSessionStateField: - """Route one legacy S1 coordinator field to its owner service. - - The descriptor keeps the historical attribute name readable and writable - on the coordinator while :class:`StratumSessionService` (its - ``SessionRegistry`` and ``P2mrAddressValidator``) owns the single mutable - copy, mirroring the S2/J1/G1 extraction pattern. Direct test assignment - through the legacy name is thereby adopted by the owner. - """ - - def __init__(self, attribute: str) -> None: - self._attribute = attribute - - def __get__( - self, - instance: "PrismCoordinator | None", - owner: type | None = None, - ) -> object: - if instance is None: - return self - return getattr( - instance._ensure_stratum_session_service(), - self._attribute, - ) - - def __set__(self, instance: "PrismCoordinator", value: object) -> None: - setattr( - instance._ensure_stratum_session_service(), - self._attribute, - value, - ) - - -class _CoordinatorObservability(ObservabilityPort): - """Collect immutable health inputs without exporting coordinator state.""" - - def __init__(self, coordinator: "PrismCoordinator") -> None: - self.coordinator = coordinator - - def monotonic(self) -> float: - return time.monotonic() - - def mining_delivery_inputs(self, now: float) -> MiningDeliveryInputs: - coordinator = self.coordinator - coordinator._ensure_initial_job_state() - # The first-job admission domain is owned by its own lock in the S2 - # service. Copy it once, before the coordinator lock is taken, so this - # refresh never holds both locks and never reads a half-committed - # replacement (#159). - admission = ( - coordinator._ensure_job_delivery_service() - .initial_job_admission_snapshot() - ) - pending_jobs = admission.pending - with coordinator.lock: - active = len(coordinator.clients) - current_tip = coordinator._current_published_tip_hash_locked() - published_snapshot = getattr(coordinator, "tip_template_snapshot", None) - subscribed = sum( - 1 for client in coordinator.clients if client.subscribed - ) - authorized_clients = [ - client - for client in coordinator.clients - if client.subscribed - and client.authorized - and client.worker is not None - ] - authorized = len(authorized_clients) - clients_with_current_work = [ - client - for client in authorized_clients - if coordinator._client_has_current_tip_job_locked(client) - ] - current = len(clients_with_current_work) - # Semantic currency alongside the strict gauge above: template - # fingerprint and payout generation, plus the connection, - # authorization, and difficulty generations that make the job - # usable by this client. Template-generation and object-identity - # churn remain intentionally excluded. The strict predicate reads - # 0 whenever a template generation was minted since the last - # fanout; this gauge instead answers whether miners actually hold - # usable work for the current template content. - payout_generation_now = int( - getattr(coordinator, "_payout_state_generation", 0) - ) - semantic_current = sum( - 1 - for client in authorized_clients - if client.active_job is not None - and published_snapshot is not None - and getattr(client.active_job, "template_fingerprint", None) - == published_snapshot.template_fingerprint - and int( - getattr(client.active_job, "payout_state_generation", -1) - ) - == payout_generation_now - and int( - getattr( - client.active_job, - "connection_id", - client.connection_id, - ) - ) - == client.connection_id - and int( - getattr(client.active_job, "authorization_generation", 0) - ) - == int(getattr(client, "authorization_generation", 0)) - and int( - getattr(client.active_job, "difficulty_generation", 0) - ) - == int(getattr(client, "difficulty_generation", 0)) - ) - clients_with_no_active_job = sum( - 1 for client in authorized_clients if client.active_job is None - ) - pending_requests = list(pending_jobs.values()) - pending = len(pending_requests) - oldest_age = max( - ( - max(0.0, now - request.requested_monotonic) - for request in pending_requests - ), - default=0.0, - ) - genuinely_pending_initial_clients = [ - client - for client in authorized_clients - if not JobDeliveryService._client_has_delivered_work_locked(client) - ] - genuine_initial_started = [ - started - for client in genuinely_pending_initial_clients - for started in ( - client.authorized_monotonic, - ( - pending_jobs[client].requested_monotonic - if client in pending_jobs - else None - ), - ) - if started is not None - ] - oldest_genuine_initial_age = max( - (max(0.0, now - started) for started in genuine_initial_started), - default=0.0, - ) - connection_limit = int( - getattr( - coordinator, - "stratum_max_connections", - DEFAULT_PRISM_STRATUM_MAX_CONNECTIONS, - ) - ) - pending_limit = int(coordinator.stratum_max_pending_initial_jobs) - timeout = float(coordinator.stratum_initial_job_timeout_seconds) - timeout_disconnects = admission.timeout_disconnects - queue_rejections = admission.queue_rejections - cancelled = admission.cancelled - coalesced = admission.coalesced - queue_capacity_reclaimed = admission.queue_capacity_reclaimed - peak = coordinator.peak_active_connection_count - handlers = coordinator.handler_thread_count - last_delivery = admission.last_delivery_monotonic - - # This census is the fleet's only current-tip coverage scan: job - # delivery used to repeat it per delivered job under the coordinator - # lock, which made every reconnect herd O(N^2) inside the lock the herd - # was already queued on (#159). Restored coverage resets the delivery - # failure here, after the snapshot and outside the lock. - if authorized == 0 or current / authorized >= 0.95: - coordinator._ensure_observability_service().reset_delivery_failure() - - coordinator._ensure_job_cache_state() - with coordinator._job_cache_lock: - prepared_bundle = coordinator._prepared_ready_bundle - prepared_snapshot = coordinator._prepared_ready_snapshot - preparation_pending = bool(coordinator.job_preparation_pending) - payout_generation = int(coordinator._payout_state_generation) - prepared_current = bool( - prepared_bundle is not None - and prepared_snapshot is published_snapshot - and published_snapshot is not None - and current_tip is not None - and published_snapshot.bestblockhash == current_tip - and published_snapshot.template_artifacts is not None - and not prepared_bundle.collection_only - and prepared_bundle.template - is published_snapshot.template_artifacts.template - and prepared_bundle.template_fingerprint - == published_snapshot.template_fingerprint - and prepared_bundle.template_generation - == published_snapshot.template_generation - and prepared_bundle.payout_state_generation == payout_generation - ) - rejection_counts = getattr(coordinator, "rejection_counts_by_reason", {}) - stale_unknown = int( - rejection_counts.get(PRISM_REJECTION_STALE_JOB, 0) - ) + int(rejection_counts.get(PRISM_REJECTION_UNKNOWN_JOB, 0)) - executor = getattr(coordinator, "_tip_refresh_executor", None) - queue_depth, active_workers = ( - executor.stats() if executor is not None else (0, 0) - ) - return MiningDeliveryInputs( - active_connections=active, - connection_capacity=connection_limit, - peak_active_connections=peak, - subscribed_connections=subscribed, - authorized_connections=authorized, - pending_initial_jobs=pending, - pending_initial_job_capacity=pending_limit, - oldest_pending_initial_job_age_seconds=oldest_age, - oldest_genuinely_pending_initial_job_age_seconds=( - oldest_genuine_initial_age - ), - clients_with_current_tip_jobs=current, - clients_with_semantically_current_work=semantic_current, - clients_with_no_active_job=clients_with_no_active_job, - last_initial_job_delivery_monotonic=last_delivery, - initial_job_timeout_seconds=timeout, - initial_job_queue_rejections=queue_rejections, - initial_job_timeout_disconnects=timeout_disconnects, - initial_job_cancelled_tasks=cancelled, - initial_job_coalesced_tasks=coalesced, - initial_job_queue_capacity_reclaimed=queue_capacity_reclaimed, - handler_threads=handlers, - delivery_executor_queue_depth=queue_depth, - delivery_executor_active_workers=active_workers, - started_monotonic=float(getattr(coordinator, "started_monotonic", now)), - startup_grace_seconds=float( - getattr( - coordinator, - "mining_health_startup_grace_seconds", - DEFAULT_PRISM_MINING_HEALTH_STARTUP_GRACE_SECONDS, - ) - ), - stale_unknown_rejections=stale_unknown, - submitted_shares=int(getattr(coordinator, "submitted_share_count", 0)), - job_preparation_pending=preparation_pending, - current_observed_tip=current_tip, - prepared_bundle_current=prepared_current, - prepared_bundle_tip=( - prepared_snapshot.bestblockhash - if prepared_snapshot is not None - else None - ), - prepared_bundle_template_generation=( - prepared_bundle.template_generation - if prepared_bundle is not None - else None - ), - prepared_bundle_payout_generation=( - prepared_bundle.payout_state_generation - if prepared_bundle is not None - else None - ), - ) - - def accepted_share_stats(self) -> tuple[int, int]: - return self.coordinator.accepted_share_stats() - - def ledger_backend(self) -> str: - return str(self.coordinator.ledger.backend_name) - - def block_counts(self) -> tuple[int, int]: - coordinator = self.coordinator - lock = getattr(coordinator, "lock", None) - if lock is None: - return int(coordinator.accepted_block_count), int(coordinator.max_blocks) - with lock: - return int(coordinator.accepted_block_count), int(coordinator.max_blocks) - - def progress_health(self) -> Mapping[str, object]: - return self.coordinator.progress_health_snapshot() - - def health_refresh_seconds(self) -> float: - return float( - getattr( - self.coordinator, - "health_refresh_seconds", - DEFAULT_PRISM_HEALTH_REFRESH_SECONDS, - ) - ) - - def render_metrics_payload(self) -> str: - return self.coordinator._render_metrics_payload() - - def metrics_refresh_seconds(self) -> float: - return float( - getattr( - self.coordinator, - "metrics_refresh_seconds", - DEFAULT_PRISM_METRICS_REFRESH_SECONDS, - ) - ) - - def record_startup_phase(self, phase: str) -> None: - # The startup-phase recorder stays coordinator runtime state; owners - # stamp through this dynamic seam (upstream #120, issue #188). - self.coordinator._record_startup_phase_once(phase) - - def stop_requested(self) -> bool: - return self.coordinator.stop_event.is_set() - - def wait_for_stop(self, timeout: float) -> bool: - return self.coordinator.stop_event.wait(timeout) - - def log(self, message: str) -> None: - print(message, flush=True) - - def log_exception(self) -> None: - traceback.print_exc() - - -class _CoordinatorAuditHttp(AuditHttpPort): - """Expose purpose-specific dynamic reads to the HTTP facade.""" - - def __init__( - self, - coordinator: "PrismCoordinator", - *, - allow_uncached_compatibility: bool = False, - ) -> None: - self.coordinator = coordinator - self.allow_uncached_compatibility = allow_uncached_compatibility - - def cached_health_payload(self) -> tuple[int, Mapping[str, object]]: - cached = getattr(self.coordinator, "cached_health_payload", None) - if callable(cached): - return cached() - if self.allow_uncached_compatibility: - return 200, self.coordinator.health_payload() - raise RuntimeError("cached health is unavailable") - - def cached_metrics_payload(self) -> MetricsSnapshotResponse: - cached = getattr(self.coordinator, "cached_metrics_payload", None) - if callable(cached): - response = self._as_metrics_response(cached()) - if response.status != 503 or not self.allow_uncached_compatibility: - # Carried through whole rather than unpacked: the freshness - # state and age this read decided are what /metrics publishes - # out of band (issue #184), and rebuilding them here from the - # status alone would lose the fresh/stale distinction that - # both now answer 200. - return response - if self.allow_uncached_compatibility: - # Legacy make_audit_handler path only: no refresher owns the - # snapshot, so this renders inline and the result is by definition - # freshly collected. - return MetricsSnapshotResponse( - status=200, - body=self.coordinator.metrics_payload(), - state=METRICS_STATE_FRESH, - age_seconds=0, - ) - raise RuntimeError("cached metrics are unavailable") - - @staticmethod - def _as_metrics_response(cached: object) -> MetricsSnapshotResponse: - """Accept the response type, or adapt a legacy ``(status, body)`` pair. - - Coordinator doubles predating the #184 response type still answer with - the bare pair. A pair carries no freshness beyond its status, so the - state is derived from exactly that and nothing is invented. - """ - - if isinstance(cached, MetricsSnapshotResponse): - return cached - status, body = cached # type: ignore[misc] - return MetricsSnapshotResponse( - status=int(status), - body=str(body), - state=( - METRICS_STATE_UNAVAILABLE - if int(status) == 503 - else METRICS_STATE_FRESH - ), - age_seconds=None if int(status) == 503 else 0, - ) - - def latest_evidence_payload(self) -> Mapping[str, object] | None: - return self.coordinator.latest_evidence_payload() - - def owed_balances_payload(self) -> Mapping[str, object]: - return self.coordinator.owed_balances_payload() - - def carry_forward_integrity_payload(self) -> Mapping[str, object]: - return self.coordinator.carry_forward_integrity_payload() - - def miner_status_payload(self, recipient_id: str) -> Mapping[str, object]: - return self.coordinator.miner_status_payload(recipient_id) - - def ledger_backend(self) -> str: - return str(self.coordinator.ledger.backend_name) - - def audit_share_window( - self, - *, - anchor_job_issued_at_ms: int, - network_difficulty: int, - ) -> list[dict[str, object]]: - return self.coordinator.ledger.audit_share_window( - anchor_job_issued_at_ms=anchor_job_issued_at_ms, - network_difficulty=network_difficulty, - ) - - def audit_block_payouts( - self, - *, - block_hash: str, - ) -> list[dict[str, object]]: - return self.coordinator.ledger.audit_block_payouts(block_hash=block_hash) - - def audit_ctv_fanouts( - self, - *, - block_hash: str, - ) -> list[dict[str, object]]: - return self.coordinator.ledger.audit_ctv_fanouts(block_hash=block_hash) - - def audit_ctv_fanout_manifest_set( - self, - *, - block_hash: str, - ) -> Mapping[str, object] | None: - return self.coordinator.ledger.audit_ctv_fanout_manifest_set( - block_hash=block_hash - ) - - def ctv_fanout_status( - self, - *, - fanout_txid: str, - ) -> Mapping[str, object] | None: - return self.coordinator.ledger.ctv_fanout_status(fanout_txid=fanout_txid) - - def pending_ctv_fanout_statuses( - self, - *, - limit: int, - ) -> list[dict[str, object]]: - return self.coordinator.ledger.pending_ctv_fanout_statuses(limit=limit) - - def audit_bundle( - self, - *, - block_hash: str, - ) -> Mapping[str, object] | None: - return self.coordinator.ledger.audit_bundle(block_hash=block_hash) - - def audit_bundle_by_commitment( - self, - *, - commitment_leaf_hex: str, - ) -> Mapping[str, object] | None: - return self.coordinator.ledger.audit_bundle_by_commitment( - commitment_leaf_hex=commitment_leaf_hex - ) - - -class PrismCoordinator: - # Share-submission owner state routed through descriptors; the - # ShareSubmissionService holds the single mutable copy once it exists - # (see lab/prism/share_submission.py). Pre-service writes land in the - # instance dict and are adopted at service construction. - recent_share_keys = RecentShareCompatibilityField() - block_solves_dropped_counts = BlockSolvesDroppedCompatibilityField() - - # Reorg reconciler owner state routed through descriptors; the - # ReorgReconcilerService holds the single mutable copy once it exists - # (see lab/prism/reorg_reconciler.py). Pre-service writes land in - # backing slots and are adopted at service construction. - reorg_reconciler_enabled = ReorgCompatibilityField("enabled", True) - reorg_reconcile_cache_seconds = ReorgCompatibilityField( - "cache_seconds", - DEFAULT_PRISM_REORG_RECONCILE_CACHE_SECONDS, - ) - reorg_inactive_block_count = ReorgCompatibilityField( - "inactive_block_count", - 0, - ) - reorg_reactivated_block_count = ReorgCompatibilityField( - "reactivated_block_count", - 0, - ) - reorg_reconcile_skip_count = ReorgCompatibilityField( - "reconcile_skip_count", - 0, - ) - reorg_reconcile_error_count = ReorgCompatibilityField( - "reconcile_error_count", - 0, - ) - matured_payout_count = ReorgCompatibilityField("matured_payout_count", 0) - last_reorg_reconciled_tip_hash = ReorgCompatibilityField( - "last_tip_hash", - None, - ) - last_reorg_reconciled_trusted = ReorgCompatibilityField( - "last_trusted", - False, - ) - last_reorg_reconciled_monotonic = ReorgCompatibilityField( - "last_monotonic", - None, - ) - - # J1/template/compiler owner state routed through descriptors; the owner - # services hold the single mutable copy (see lab/prism/job_bundle.py, - # lab/prism/template_artifacts.py, and lab/prism/bundle_compiler.py). - _job_cache_lock = _JobBundleStateField("_job_cache_lock") - _active_job_bundle_builds = _JobBundleStateField("_active_job_bundle_builds") - _job_build_lock = _JobBundleStateField("_job_build_lock") - _job_build_scheduler_lock = _JobBundleStateField("_job_build_scheduler_lock") - _job_build_priority_preparations = _JobBundleStateField( - "_job_build_priority_preparations" - ) - _job_build_priority_preparation_sequence = _JobBundleStateField( - "_job_build_priority_preparation_sequence" - ) - _job_build_routine_preparations = _JobBundleStateField( - "_job_build_routine_preparations" - ) - _job_build_routine_preparation_sequence = _JobBundleStateField( - "_job_build_routine_preparation_sequence" - ) - _job_build_priority_changed = _JobBundleStateField("_job_build_priority_changed") - _job_build_executor = _JobBundleStateField("_job_build_executor") - _job_build_executor_shutdown = _JobBundleStateField("_job_build_executor_shutdown") - _job_build_active = _JobBundleStateField("_job_build_active") - _job_build_retiring = _JobBundleStateField("_job_build_retiring") - _job_build_pending = _JobBundleStateField("_job_build_pending") - _job_build_issued_at_ms = _JobBundleStateField("_job_build_issued_at_ms") - _job_bundle_cache = _JobBundleStateField("_job_bundle_cache") - _job_build_phase_local = _JobBundleStateField("_job_build_phase_local") - job_cache_hit_counts = _JobBundleStateField("job_cache_hit_counts") - job_cache_miss_counts = _JobBundleStateField("job_cache_miss_counts") - job_build_seconds_bucket_counts = _JobBundleStateField( - "job_build_seconds_bucket_counts" - ) - job_build_seconds_sum = _JobBundleStateField("job_build_seconds_sum") - job_build_count = _JobBundleStateField("job_build_count") - job_build_phase_seconds = _JobBundleStateField("job_build_phase_seconds") - job_build_scheduler_counts = _JobBundleStateField("job_build_scheduler_counts") - job_build_priority_counts = _JobBundleStateField("job_build_priority_counts") - job_build_priority_admission_seconds = _JobBundleStateField( - "job_build_priority_admission_seconds" - ) - initial_job_prepared_work_counts = _JobBundleStateField( - "initial_job_prepared_work_counts" - ) - _admission_deadline_last_log_monotonic = _JobBundleStateField( - "_admission_deadline_last_log_monotonic" - ) - job_build_cancellation_seconds = _JobBundleStateField( - "job_build_cancellation_seconds" - ) - job_build_replacement_start_seconds = _JobBundleStateField( - "job_build_replacement_start_seconds" - ) - job_build_worker_counts = _JobBundleStateField("job_build_worker_counts") - _job_build_worker_restart_pending = _JobBundleStateField( - "_job_build_worker_restart_pending" - ) - _share_window_serialization_lock = _JobBundleStateField( - "_share_window_serialization_lock" - ) - _share_window_serialization = _JobBundleStateField("_share_window_serialization") - _bundle_preparation_lock = _JobBundleStateField("_bundle_preparation_lock") - _bundle_preparation_flights = _JobBundleStateField("_bundle_preparation_flights") - shared_bundle_build_counts = _JobBundleStateField("shared_bundle_build_counts") - shared_bundle_preparation_seconds_sum = _JobBundleStateField( - "shared_bundle_preparation_seconds_sum" - ) - shared_bundle_preparation_count = _JobBundleStateField( - "shared_bundle_preparation_count" - ) - shared_bundle_preparation_waiters = _JobBundleStateField( - "shared_bundle_preparation_waiters" - ) - _prepared_ready_bundle = _JobBundleStateField("_prepared_ready_bundle") - _prepared_ready_snapshot = _JobBundleStateField("_prepared_ready_snapshot") - job_preparation_pending = _JobBundleStateField("job_preparation_pending") - _template_artifacts = _TemplateArtifactStateField("_template_artifacts") - _template_artifact_generation = _TemplateArtifactStateField( - "_template_artifact_generation" - ) - _serve_builder_lock = _BundleCompilerStateField("_serve_builder_lock") - _serve_builder = _BundleCompilerStateField("_serve_builder") - _serve_builder_shutdown = _BundleCompilerStateField("_serve_builder_shutdown") - _serve_builder_metrics_lock = _BundleCompilerStateField( - "_serve_builder_metrics_lock" - ) - serve_builder_counts = _BundleCompilerStateField("serve_builder_counts") - serve_builder_window_cache_counts = _BundleCompilerStateField( - "serve_builder_window_cache_counts" - ) - - # R1 owner state routed through descriptors; TipRefreshService holds - # the single mutable copy (see lab/prism/tip_refresh.py). The twelve - # historical plain fields and the tip-detection epoch stay lazily - # created on the owner, preserving legacy hasattr semantics. - _tip_refresh_lock = _TipRefreshStateField("_tip_refresh_lock") - _tip_refresh_singleflight_lock = _TipRefreshStateField( - "_tip_refresh_singleflight_lock" - ) - _tip_refresh_executor_lock = _TipRefreshStateField("_tip_refresh_executor_lock") - _tip_refresh_executor = _TipRefreshStateField("_tip_refresh_executor") - _tip_refresh_executor_shutdown = _TipRefreshStateField( - "_tip_refresh_executor_shutdown" - ) - _tip_refresh_metrics_lock = _TipRefreshStateField("_tip_refresh_metrics_lock") - tip_refresh_histograms = _TipRefreshStateField("tip_refresh_histograms") - tip_refresh_build_phase_histograms = _TipRefreshStateField( - "tip_refresh_build_phase_histograms" - ) - tip_refresh_client_counts = _TipRefreshStateField("tip_refresh_client_counts") - tip_refresh_cancellation_counts = _TipRefreshStateField( - "tip_refresh_cancellation_counts" - ) - tip_refresh_wave_outcome_counts = _TipRefreshStateField( - "tip_refresh_wave_outcome_counts" - ) - tip_refresh_coverage_histograms = _TipRefreshStateField( - "tip_refresh_coverage_histograms" - ) - tip_refresh_inflight = _TipRefreshStateField("tip_refresh_inflight") - tip_refresh_build_inflight = _TipRefreshStateField("tip_refresh_build_inflight") - tip_refresh_build_queue_depth = _TipRefreshStateField( - "tip_refresh_build_queue_depth" - ) - tip_refresh_singleflight_hits = _TipRefreshStateField( - "tip_refresh_singleflight_hits" - ) - tip_refresh_superseded_results = _TipRefreshStateField( - "tip_refresh_superseded_results" - ) - tip_refresh_worker_failures = _TipRefreshStateField("tip_refresh_worker_failures") - tip_refresh_worker_restarts = _TipRefreshStateField("tip_refresh_worker_restarts") - tip_refresh_ipc_bytes = _TipRefreshStateField("tip_refresh_ipc_bytes") - _tip_refresh_pending_event = _TipRefreshStateField("_tip_refresh_pending_event") - _tip_refresh_pending_counter = _TipRefreshStateField("_tip_refresh_pending_counter") - _tip_refresh_pending_token = _TipRefreshStateField("_tip_refresh_pending_token") - _tip_refresh_retry = _TipRefreshStateField("_tip_refresh_retry") - _tip_refresh_retry_counter = _TipRefreshStateField("_tip_refresh_retry_counter") - _tip_refresh_retry_consumed = _TipRefreshStateField("_tip_refresh_retry_consumed") - _tip_refresh_failure_holdoff_until = _TipRefreshStateField( - "_tip_refresh_failure_holdoff_until" - ) - _tip_refresh_failure_tip = _TipRefreshStateField("_tip_refresh_failure_tip") - _active_tip_refresh = _TipRefreshStateField("_active_tip_refresh") - _tip_refresh_epoch_sequence = _TipRefreshStateField("_tip_refresh_epoch_sequence") - _tip_refresh_epoch_tip_hash = _TipRefreshStateField("_tip_refresh_epoch_tip_hash") - _tip_refresh_epoch_payout_generation = _TipRefreshStateField( - "_tip_refresh_epoch_payout_generation" - ) - _tip_refresh_epoch_coverage = _TipRefreshStateField("_tip_refresh_epoch_coverage") - _published_tip_refresh_epoch_identity = _TipRefreshStateField( - "_published_tip_refresh_epoch_identity" - ) - _retained_collection_refresh = _TipRefreshStateField("_retained_collection_refresh") - current_tip_first_seen = _TipRefreshStateField("current_tip_first_seen") - current_tip_parent = _TipRefreshStateField("current_tip_parent") - current_tip_observation_sequence = _TipRefreshStateField( - "current_tip_observation_sequence" - ) - current_tip_observed_monotonic = _TipRefreshStateField( - "current_tip_observed_monotonic" - ) - tip_template_snapshot = _TipRefreshStateField("tip_template_snapshot") - latest_detected_tip = _TipRefreshStateField("latest_detected_tip") - tip_refresh_divergence_started_monotonic = _TipRefreshStateField( - "tip_refresh_divergence_started_monotonic" - ) - tip_observation_sequence = _TipRefreshStateField("tip_observation_sequence") - last_successful_template_refresh_monotonic = _TipRefreshStateField( - "last_successful_template_refresh_monotonic" - ) - template_refresh_failure_started_monotonic = _TipRefreshStateField( - "template_refresh_failure_started_monotonic" - ) - tip_refresh_job_count = _TipRefreshStateField("tip_refresh_job_count") - post_accept_refresh_failure_count = _TipRefreshStateField( - "post_accept_refresh_failure_count" - ) - tip_detection_epoch = _TipRefreshStateField("tip_detection_epoch") - - # S2 owner state routed through descriptors; JobDeliveryService holds - # the single mutable copy (see lab/prism/job_delivery.py). - pending_initial_jobs = _JobDeliveryStateField("pending_initial_jobs") - initial_job_queue_rejection_count = _JobDeliveryStateField( - "initial_job_queue_rejection_count" - ) - initial_job_timeout_count = _JobDeliveryStateField("initial_job_timeout_count") - initial_job_cancelled_count = _JobDeliveryStateField("initial_job_cancelled_count") - initial_job_coalesced_count = _JobDeliveryStateField("initial_job_coalesced_count") - initial_job_queue_capacity_reclaimed_count = _JobDeliveryStateField( - "initial_job_queue_capacity_reclaimed_count" - ) - initial_job_sent_count = _JobDeliveryStateField("initial_job_sent_count") - initial_job_failed_count = _JobDeliveryStateField("initial_job_failed_count") - initial_job_superseded_count = _JobDeliveryStateField( - "initial_job_superseded_count" - ) - initial_job_delivery_latency_seconds_sum = _JobDeliveryStateField( - "initial_job_delivery_latency_seconds_sum" - ) - initial_job_delivery_latency_count = _JobDeliveryStateField( - "initial_job_delivery_latency_count" - ) - last_initial_job_delivery_monotonic = _JobDeliveryStateField( - "last_initial_job_delivery_monotonic" - ) - _initial_job_executor_lock = _JobDeliveryStateField("_initial_job_executor_lock") - _initial_job_executor = _JobDeliveryStateField("_initial_job_executor") - _initial_job_executor_shutdown = _JobDeliveryStateField( - "_initial_job_executor_shutdown" - ) - jobs = _JobDeliveryStateField("jobs") - job_counter = _JobDeliveryStateField("job_counter") - evicted_job_graveyard = _JobDeliveryStateField("evicted_job_graveyard") - evicted_jobs_by_connection = _JobDeliveryStateField("evicted_jobs_by_connection") - evicted_same_tip_by_connection = _JobDeliveryStateField( - "evicted_same_tip_by_connection" - ) - evicted_same_tip_job_ids = _JobDeliveryStateField("evicted_same_tip_job_ids") - evicted_job_index_tip_hash = _JobDeliveryStateField("evicted_job_index_tip_hash") - evicted_job_next_prune_monotonic = _JobDeliveryStateField( - "evicted_job_next_prune_monotonic" - ) - evicted_job_expiration_counts = _JobDeliveryStateField( - "evicted_job_expiration_counts" - ) - evicted_job_capacity_eviction_counts = _JobDeliveryStateField( - "evicted_job_capacity_eviction_counts" - ) - evicted_job_submit_counts = _JobDeliveryStateField("evicted_job_submit_counts") - _disconnected_evicted_job_ids = _JobDeliveryStateField( - "_disconnected_evicted_job_ids" - ) - - # S1 owner state routed through descriptors; StratumSessionService (its - # SessionRegistry and P2mrAddressValidator) holds the single mutable copy - # (see lab/prism/stratum_session.py). - clients = _StratumSessionStateField("clients") - connection_counter = _StratumSessionStateField("connection_counter") - connection_limit_rejection_counts = _StratumSessionStateField( - "connection_limit_rejection_counts" - ) - peak_active_connection_count = _StratumSessionStateField( - "peak_active_connection_count" - ) - handler_thread_count = _StratumSessionStateField("handler_thread_count") - accept_resource_exhaustion_count = _StratumSessionStateField( - "accept_resource_exhaustion_count" - ) - connection_setup_failure_count = _StratumSessionStateField( - "connection_setup_failure_count" - ) - _p2mr_address_cache_lock = _StratumSessionStateField("_p2mr_address_cache_lock") - _p2mr_address_cache = _StratumSessionStateField("_p2mr_address_cache") - _p2mr_address_validation_inflight = _StratumSessionStateField( - "_p2mr_address_validation_inflight" - ) - - # S3 owner state routed through descriptors; ShareWriter holds the single - # mutable copy (see lab/prism/share_writer.py). - share_append_queue = ShareWriterCompatibilityField("share_append_queue") - share_commit_batch_size = ShareWriterCompatibilityField("share_commit_batch_size") - share_commit_linger_seconds = ShareWriterCompatibilityField( - "share_commit_linger_seconds" - ) - share_commit_timeout_seconds = ShareWriterCompatibilityField( - "share_commit_timeout_seconds" - ) - share_writer_active = ShareWriterCompatibilityField("share_writer_active") - share_append_failure_count = ShareWriterCompatibilityField( - "share_append_failure_count" - ) - share_recovery_path = ShareWriterCompatibilityField("share_recovery_path") - share_recovery_lock = ShareWriterCompatibilityField("share_recovery_lock") - shares_recovered_to_disk = ShareWriterCompatibilityField( - "shares_recovered_to_disk" - ) - shares_replayed = ShareWriterCompatibilityField("shares_replayed") - share_replay_conflicts = ShareWriterCompatibilityField( - "share_replay_conflicts" - ) - _pending_share_commit_lock = ShareWriterCompatibilityField( - "_pending_share_commit_lock" - ) - _pending_share_commit_floor = ShareWriterCompatibilityField( - "_pending_share_commit_floor" - ) - - # P1 owner state routed through descriptors; PayoutStateService holds - # the single mutable copy (see lab/prism/payout_state.py). - _payout_state_generation = _PayoutStateStateField( - "_payout_state_generation" - ) - _payout_state_prepare_lock = _PayoutStateStateField( - "_payout_state_prepare_lock" - ) - _payout_state_source = _PayoutStateStateField("_payout_state_source") - _published_payout_state = _PayoutStateStateField("_published_payout_state") - _payout_ledger_artifact = _PayoutStateStateField("_payout_ledger_artifact") - _payout_ledger_artifact_generation = _PayoutStateStateField( - "_payout_ledger_artifact_generation" - ) - _payout_ledger_append_invalidation_epoch = _PayoutStateStateField( - "_payout_ledger_append_invalidation_epoch" - ) - _payout_append_landing_fence_lock = _PayoutStateStateField( - "_payout_append_landing_fence_lock" - ) - _payout_window_inflight_scan_anchors = _PayoutStateStateField( - "_payout_window_inflight_scan_anchors" - ) - _payout_window_inflight_scan_anchor_token = _PayoutStateStateField( - "_payout_window_inflight_scan_anchor_token" - ) - _payout_unfenced_append_inflight_stamps = _PayoutStateStateField( - "_payout_unfenced_append_inflight_stamps" - ) - _payout_unfenced_append_inflight_token = _PayoutStateStateField( - "_payout_unfenced_append_inflight_token" - ) - _payout_unfenced_append_drained = _PayoutStateStateField( - "_payout_unfenced_append_drained" - ) - _payout_published_job_window_anchor_ms = _PayoutStateStateField( - "_payout_published_job_window_anchor_ms" - ) - _incremental_payout_artifact_window = _PayoutStateStateField( - "_incremental_payout_artifact_window" - ) - _incremental_payout_artifact_window_invalidation_reason = _PayoutStateStateField( - "_incremental_payout_artifact_window_invalidation_reason" - ) - _payout_prior_balances_reuse_invalidated_sha256 = _PayoutStateStateField( - "_payout_prior_balances_reuse_invalidated_sha256" - ) - _payout_artifact_executor_lock = _PayoutStateStateField( - "_payout_artifact_executor_lock" - ) - _payout_artifact_executor = _PayoutStateStateField( - "_payout_artifact_executor" - ) - _payout_artifact_future = _PayoutStateStateField("_payout_artifact_future") - _payout_artifact_requested = _PayoutStateStateField( - "_payout_artifact_requested" - ) - _payout_artifact_requested_bypass = _PayoutStateStateField( - "_payout_artifact_requested_bypass" - ) - _payout_artifact_executor_shutdown = _PayoutStateStateField( - "_payout_artifact_executor_shutdown" - ) - _payout_artifact_last_schedule_monotonic = _PayoutStateStateField( - "_payout_artifact_last_schedule_monotonic" - ) - _payout_artifact_rearm_backoff = _PayoutStateStateField( - "_payout_artifact_rearm_backoff" - ) - _payout_state_delivery_gate = _PayoutStateStateField( - "_payout_state_delivery_gate" - ) - _payout_balance_mutation_lock = _PayoutStateStateField( - "_payout_balance_mutation_lock" - ) - _accepted_block_payout_preview_condition = _PayoutStateStateField( - "_accepted_block_payout_preview_condition" - ) - _accepted_block_payout_previews = _PayoutStateStateField( - "_accepted_block_payout_previews" - ) - _invalidated_accepted_block_payout_previews = _PayoutStateStateField( - "_invalidated_accepted_block_payout_previews" - ) - _payout_state_metrics_lock = _PayoutStateStateField( - "_payout_state_metrics_lock" - ) - payout_state_histograms = _PayoutStateStateField("payout_state_histograms") - payout_gate_wait_histograms = _PayoutStateStateField( - "payout_gate_wait_histograms" - ) - payout_state_candidates_discarded = _PayoutStateStateField( - "payout_state_candidates_discarded" - ) - _payout_first_delivery_pending = _PayoutStateStateField( - "_payout_first_delivery_pending" - ) - _payout_state_publication_blocked = _PayoutStateStateField( - "_payout_state_publication_blocked" - ) - payout_artifact_event_counts = _PayoutStateStateField( - "payout_artifact_event_counts" - ) - - _progress_current_template_generation = _ProgressHealthStateField( - "_current_template_generation" - ) - _progress_current_template_fingerprint = _ProgressHealthStateField( - "_current_template_fingerprint" - ) - _progress_current_payout_generation = _ProgressHealthStateField( - "_current_payout_generation" - ) - _progress_published_template_generation = _ProgressHealthStateField( - "_published_template_generation" - ) - _progress_published_template_fingerprint = _ProgressHealthStateField( - "_published_template_fingerprint" - ) - _progress_published_payout_generation = _ProgressHealthStateField( - "_published_payout_generation" - ) - _progress_has_published_work = _ProgressHealthStateField("_has_published_work") - _progress_last_tip_poll_monotonic = _ProgressHealthStateField( - "_last_tip_poll_monotonic" - ) - _progress_last_delivery_template_generation = _ProgressHealthStateField( - "_last_delivery_template_generation" - ) - _progress_last_delivery_template_fingerprint = _ProgressHealthStateField( - "_last_delivery_template_fingerprint" - ) - _progress_last_delivery_payout_generation = _ProgressHealthStateField( - "_last_delivery_payout_generation" - ) - _progress_last_delivery_monotonic = _ProgressHealthStateField( - "_last_delivery_monotonic" - ) - _progress_pending_since_monotonic = _ProgressHealthStateField( - "_pending_since_monotonic" - ) - _progress_publication_divergence_since_monotonic = _ProgressHealthStateField( - "_publication_divergence_since_monotonic" - ) - _progress_refresh_signal_pending = _ProgressHealthStateField( - "_refresh_signal_pending" - ) - _progress_active_refresh_count = _ProgressHealthStateField( - "_active_refresh_count" - ) - _progress_last_refresh_activity_monotonic = _ProgressHealthStateField( - "_last_refresh_activity_monotonic" - ) - _progress_bundle_build_counter = _ProgressHealthStateField( - "_bundle_build_counter" - ) - - @property - def _progress_health_lock(self) -> threading.Lock: - return self._ensure_progress_health_service()._lock - - @property - def _progress_bundle_builds(self) -> dict[int, float]: - return self._ensure_progress_health_service()._bundle_builds - - block_candidate_queue = BlockCandidateCompatibilityField( - "block_candidate_queue", - candidate_compatibility_default( - lambda: queue.Queue(maxsize=MAX_PENDING_BLOCK_CANDIDATES) - ), - ) - block_candidates_dropped = BlockCandidateCompatibilityField( - "block_candidates_dropped", 0 - ) - block_candidate_wakeups_coalesced = BlockCandidateCompatibilityField( - "block_candidate_wakeups_coalesced", 0 - ) - block_candidate_retry_count = BlockCandidateCompatibilityField( - "block_candidate_retry_count", 0 - ) - block_candidate_poisoned_count = BlockCandidateCompatibilityField( - "block_candidate_poisoned_count", 0 - ) - block_candidate_retry_initial_seconds = BlockCandidateCompatibilityField( - "block_candidate_retry_initial_seconds", - DEFAULT_BLOCK_CANDIDATE_RETRY_INITIAL_SECONDS, - ) - block_candidate_retry_max_seconds = BlockCandidateCompatibilityField( - "block_candidate_retry_max_seconds", - DEFAULT_BLOCK_CANDIDATE_RETRY_MAX_SECONDS, - ) - block_candidate_retry_delays = BlockCandidateCompatibilityField( - "block_candidate_retry_delays", - candidate_compatibility_default(lambda: {}), - ) - block_candidate_abandoned_counts = BlockCandidateCompatibilityField( - "block_candidate_abandoned_counts", - candidate_compatibility_default(lambda: {}), - ) - _retry_block_candidate = BlockCandidateCompatibilityField( - "_retry_block_candidate", None - ) - _block_candidate_outcome = BlockCandidateCompatibilityField( - "_block_candidate_outcome", - candidate_compatibility_default(lambda: threading.local()), - ) - _block_candidate_finalize_retries = BlockCandidateCompatibilityField( - "_block_candidate_finalize_retries", - candidate_compatibility_default(lambda: {}), - ) - # The #113-era block-candidate fields route to the B1 owner one-to-one - # (the pre-#113 backoff quartet keeps its historical service spelling). - _block_replay_candidate_queue = BlockCandidateStateField( - "_block_replay_candidate_queue" - ) - _block_replay_inflight_hashes = BlockCandidateStateField( - "_block_replay_inflight_hashes" - ) - _block_quarantine_queue = BlockCandidateStateField("_block_quarantine_queue") - _block_quarantine_hashes = BlockCandidateStateField("_block_quarantine_hashes") - _block_candidate_disposition_registry_lock = BlockCandidateStateField( - "_block_candidate_disposition_registry_lock" - ) - _block_candidate_disposition_flights = BlockCandidateStateField( - "_block_candidate_disposition_flights" - ) - _block_candidate_terminal_outcomes = BlockCandidateStateField( - "_block_candidate_terminal_outcomes" - ) - _block_fast_lane_reservations = BlockCandidateStateField( - "_block_fast_lane_reservations" - ) - _block_disposition_waiting_retries = BlockCandidateStateField( - "_block_disposition_waiting_retries" - ) - _block_candidate_dequeued_hashes = BlockCandidateStateField( - "_block_candidate_dequeued_hashes" - ) - _block_accounting_state_lock = BlockCandidateStateField( - "_block_accounting_state_lock" - ) - _block_accounting_queue = BlockCandidateStateField("_block_accounting_queue") - _block_accounting_overflow_queue = BlockCandidateStateField( - "_block_accounting_overflow_queue" - ) - _block_accounting_accepted_queue = BlockCandidateStateField( - "_block_accounting_accepted_queue" - ) - _block_accounting_sequence = BlockCandidateStateField( - "_block_accounting_sequence" - ) - _block_accounting_thread = BlockCandidateStateField("_block_accounting_thread") - _block_accounting_thread_ident = BlockCandidateStateField( - "_block_accounting_thread_ident" - ) - _block_accounting_holds_disposition = BlockCandidateStateField( - "_block_accounting_holds_disposition" - ) - _block_accounting_deferred_retry_candidate = BlockCandidateStateField( - "_block_accounting_deferred_retry_candidate" - ) - _block_accounting_phase = BlockCandidateStateField("_block_accounting_phase") - _block_submitter_thread_ident = BlockCandidateStateField( - "_block_submitter_thread_ident" - ) - _block_submitter_phase = BlockCandidateStateField("_block_submitter_phase") - _block_submitter_last_lock_wait_log_monotonic = BlockCandidateStateField( - "_block_submitter_last_lock_wait_log_monotonic" - ) - _block_submitter_retry_state_lock = BlockCandidateStateField( - "_block_submitter_retry_state_lock", "_state_lock" - ) - _block_submitter_backoff_started_monotonic = BlockCandidateStateField( - "_block_submitter_backoff_started_monotonic", "_backoff_started_monotonic" - ) - _block_submitter_backoff_deadline_monotonic = BlockCandidateStateField( - "_block_submitter_backoff_deadline_monotonic", "_backoff_deadline_monotonic" - ) - _block_submitter_backoff_delay_seconds = BlockCandidateStateField( - "_block_submitter_backoff_delay_seconds", "_backoff_delay_seconds" - ) - _block_submitter_ledger_calls_lock = BlockCandidateStateField( - "_block_submitter_ledger_calls_lock" - ) - _block_submitter_ledger_calls = BlockCandidateStateField( - "_block_submitter_ledger_calls" - ) - _block_submitter_ledger_worker_slots = BlockCandidateStateField( - "_block_submitter_ledger_worker_slots" - ) - _block_submitter_rpc_calls_lock = BlockCandidateStateField( - "_block_submitter_rpc_calls_lock" - ) - _block_submitter_rpc_calls = BlockCandidateStateField( - "_block_submitter_rpc_calls" - ) - _block_submitter_rpc_worker_slots = BlockCandidateStateField( - "_block_submitter_rpc_worker_slots" - ) - _block_landing_timeout_counts = BlockCandidateStateField( - "_block_landing_timeout_counts" - ) - _block_ledger_call_metrics_lock = BlockCandidateStateField( - "_block_ledger_call_metrics_lock" - ) - _block_ledger_call_metrics = BlockCandidateStateField( - "_block_ledger_call_metrics" - ) - _block_candidate_retry_not_before = BlockCandidateStateField( - "_block_candidate_retry_not_before" - ) - _block_candidate_retained_node_submissions = BlockCandidateStateField( - "_block_candidate_retained_node_submissions" - ) - _block_candidate_retained_submission_monotonic = BlockCandidateStateField( - "_block_candidate_retained_submission_monotonic" - ) - _counted_block_candidate_abandonments = BlockCandidateStateField( - "_counted_block_candidate_abandonments" - ) - _outstanding_block_candidate_hashes = BlockCandidateStateField( - "_outstanding_block_candidate_hashes" - ) - _tip_observed_accepted_block_hashes = BlockCandidateStateField( - "_tip_observed_accepted_block_hashes" - ) - block_candidate_accept_pending_defer_count = BlockCandidateStateField( - "block_candidate_accept_pending_defer_count" - ) - accepted_parent_redrive_attempt_count = BlockCandidateStateField( - "accepted_parent_redrive_attempt_count" - ) - accepted_parent_redrive_resolved_count = BlockCandidateStateField( - "accepted_parent_redrive_resolved_count" - ) - accepted_parent_redrive_exhausted_count = BlockCandidateStateField( - "accepted_parent_redrive_exhausted_count" - ) - stale_job_abandon_counts = BlockCandidateStateField("stale_job_abandon_counts") - _block_submit_metrics_lock = BlockCandidateStateField( - "_block_submit_metrics_lock" - ) - block_submit_seconds_histogram = BlockCandidateStateField( - "block_submit_seconds_histogram" - ) - _accepted_block_preview_publication_lock = BlockCandidateStateField( - "_accepted_block_preview_publication_lock" - ) - accepted_block_preview_publication_seconds_histogram = ( - BlockCandidateStateField( - "accepted_block_preview_publication_seconds_histogram" - ) - ) - _accepted_block_preview_acceptance_monotonic = BlockCandidateStateField( - "_accepted_block_preview_acceptance_monotonic" - ) - _block_replay_enumeration_owed_flag = BlockCandidateStateField( - "_block_replay_enumeration_owed_flag" - ) - - # Vardiff owner state routed through descriptors; VardiffService holds - # the single mutable copy (see lab/prism/vardiff_service.py). Before the - # lazy service exists, reads/writes land in the instance dict and are - # adopted at single-flight construction, preserving embedders, bare - # __new__ test instances, and concurrent first use. - idle_retarget_count = VardiffCompatibilityField( - "idle_retarget_count", - lambda: 0, - ) - _vardiff_idle_lock = VardiffCompatibilityField( - "_vardiff_idle_lock", - threading.Lock, - ) - _vardiff_idle_executor = VardiffCompatibilityField( - "_vardiff_idle_executor", - lambda: None, - ) - _vardiff_idle_executor_shutdown = VardiffCompatibilityField( - "_vardiff_idle_executor_shutdown", - lambda: False, - ) - _vardiff_idle_pending = VardiffCompatibilityField( - "_vardiff_idle_pending", - set, - ) - vardiff_idle_queue_depth = VardiffCompatibilityField( - "vardiff_idle_queue_depth", - lambda: 0, - ) - vardiff_idle_inflight = VardiffCompatibilityField( - "vardiff_idle_inflight", - lambda: 0, - ) - vardiff_idle_clients_inspected = VardiffCompatibilityField( - "vardiff_idle_clients_inspected", - lambda: 0, - ) - vardiff_idle_skip_counts = VardiffCompatibilityField( - "vardiff_idle_skip_counts", - lambda: {reason: 0 for reason in PRISM_VARDIFF_IDLE_SKIP_REASONS}, - ) - vardiff_idle_task_failures = VardiffCompatibilityField( - "vardiff_idle_task_failures", - lambda: 0, - ) - vardiff_idle_sweep_histogram = VardiffCompatibilityField( - "vardiff_idle_sweep_histogram", - lambda: { - "buckets": { - bucket: 0 for bucket in PRISM_VARDIFF_IDLE_SECONDS_BUCKETS - }, - "sum": 0.0, - "count": 0, - }, - ) - vardiff_idle_task_histogram = VardiffCompatibilityField( - "vardiff_idle_task_histogram", - lambda: { - "buckets": { - bucket: 0 for bucket in PRISM_VARDIFF_IDLE_SECONDS_BUCKETS - }, - "sum": 0.0, - "count": 0, - }, - ) - - def _ensure_block_candidate_service(self) -> BlockCandidateService: - service = self.__dict__.get("_block_candidate_service") - if service is not None: - return service - init_lock = self.__dict__.setdefault( - "_block_candidate_service_init_lock", - threading.Lock(), - ) - with init_lock: - service = self.__dict__.get("_block_candidate_service") - if service is not None: - return service - candidate_queue = self.__dict__.get("block_candidate_queue") - if candidate_queue is None: - candidate_queue = queue.Queue(maxsize=MAX_PENDING_BLOCK_CANDIDATES) - stop_event = self.__dict__.get("stop_event") - if stop_event is None: - stop_event = threading.Event() - self.stop_event = stop_event - service = BlockCandidateService( - BlockCandidatePorts( - runtime=self, - ledger=lambda: self.ledger, - stop_event=lambda: self.stop_event, - writer_operation=lambda component: self._writer_operation(component), - submit_candidate=( - lambda candidate, *, node_submission=None, disposition_held=False: ( - self._land_block_candidate_submission( - candidate, - node_submission=node_submission, - disposition_held=disposition_held, - ) - ) - ), - reject_terminal_prepared=( - lambda candidate: self._reject_terminal_prepared_block_candidate( - candidate - ) - ), - begin_preview=lambda block_hash, block_height: ( - self._begin_accepted_block_payout_preview( - block_hash, - block_height=block_height, - ) - ), - clear_preview=lambda block_hash, invalidate: ( - self._clear_accepted_block_payout_preview( - block_hash, - invalidate_published=invalidate, - ) - ), - share_writer=lambda: self._ensure_share_writer_service(), - finish_pending_candidate=( - lambda pending: self._finish_pending_share_candidate(pending) - ), - refresh_after_accept=lambda client: ( - self.refresh_jobs_after_pending_accepted_block( - client, - heartbeat_name="block_submitter", - ) - ), - record_heartbeat=lambda name: self._record_heartbeat(name), - replay_entrypoint=lambda: self.replay_pending_block_candidates(), - submit_next_entrypoint=( - lambda timeout: self.submit_next_block_candidate(timeout=timeout) - ), - next_retry_delay=lambda block_hash: ( - self._next_block_candidate_retry_delay(block_hash) - ), - log=lambda message: print(message, flush=True), - ), - candidate_queue=candidate_queue, - retry_initial_seconds=float( - self.__dict__.get( - "block_candidate_retry_initial_seconds", - DEFAULT_BLOCK_CANDIDATE_RETRY_INITIAL_SECONDS, - ) - ), - retry_max_seconds=float( - self.__dict__.get( - "block_candidate_retry_max_seconds", - DEFAULT_BLOCK_CANDIDATE_RETRY_MAX_SECONDS, - ) - ), - retryable_reasons=PRISM_RETRYABLE_BLOCK_CANDIDATE_REASONS, - submit_seconds_buckets=PRISM_JOB_BUILD_SECONDS_BUCKETS, - ) - service.dropped = int( - self.__dict__.get("block_candidates_dropped", 0) - ) - service.wakeups_coalesced = int( - self.__dict__.get("block_candidate_wakeups_coalesced", 0) - ) - service.retries = int( - self.__dict__.get("block_candidate_retry_count", 0) - ) - service.poisoned = int( - self.__dict__.get("block_candidate_poisoned_count", 0) - ) - service.retry_delays = self.__dict__.get( - "block_candidate_retry_delays", - service.retry_delays, - ) - service.finalize_retries = self.__dict__.get( - "_block_candidate_finalize_retries", - service.finalize_retries, - ) - service.abandoned_counts = self.__dict__.get( - "block_candidate_abandoned_counts", - service.abandoned_counts, - ) - service.retry_candidate = self.__dict__.get( - "_retry_block_candidate" - ) - service.outcome = self.__dict__.get( - "_block_candidate_outcome", - service.outcome, - ) - # Adopt any remaining pre-service legacy values (direct __dict__ - # installs by focused embedders) before removing them; ordinary - # attribute writes already route through the class descriptors. - for legacy_name, service_attribute in ( - _BLOCK_CANDIDATE_STATE_FIELD_MAP.items() - ): - if legacy_name in self.__dict__: - setattr(service, service_attribute, self.__dict__[legacy_name]) - self.__dict__["_block_candidate_service"] = service - for name in ( - "block_candidate_queue", - "block_candidates_dropped", - "block_candidate_wakeups_coalesced", - "block_candidate_retry_count", - "block_candidate_poisoned_count", - "block_candidate_retry_initial_seconds", - "block_candidate_retry_max_seconds", - "block_candidate_retry_delays", - "_block_candidate_finalize_retries", - "block_candidate_abandoned_counts", - "_retry_block_candidate", - "_block_candidate_outcome", - *_BLOCK_CANDIDATE_STATE_FIELD_MAP, - ): - self.__dict__.pop(name, None) - return service - - def _ensure_vardiff_service(self) -> VardiffService: - service = self.__dict__.get("_vardiff_service") - if service is not None: - return service - init_lock = self.__dict__.setdefault( - "_vardiff_service_init_lock", - threading.RLock(), - ) - with init_lock: - service = self.__dict__.get("_vardiff_service") - if service is not None: - return service - # Adopt pre-service legacy fields (production __init__ writes and - # direct __dict__ installs by focused embedders) before removing - # them; ordinary attribute access already routes through the - # class descriptors under this same lock. - initial = { - name: self.__dict__.pop(name) - for name in VARDIFF_COMPATIBILITY_FIELDS - if name in self.__dict__ - } - service = VardiffService(self) - for name, value in initial.items(): - setattr(service, name, value) - self.__dict__["_vardiff_service"] = service - return service - - def _ensure_block_finalization_service(self) -> BlockFinalizationService: - service = self.__dict__.get("_block_finalization_service") - if service is not None: - return service - init_lock = self.__dict__.setdefault( - "_block_finalization_service_init_lock", - threading.Lock(), - ) - with init_lock: - service = self.__dict__.get("_block_finalization_service") - if service is None: - service = BlockFinalizationService(self) - self.__dict__["_block_finalization_service"] = service - return service - - # The five PR 79 health compatibility properties were removed by this - # layer; health snapshot state is read and poked only through the - # observability service's state() and narrow *_for_test accessors. - - def __init__(self, config: CoordinatorConfig | None = None) -> None: - self.config = load_coordinator_config() if config is None else config - rpc_config = self.config.rpc - stratum_config = self.config.stratum - job_config = self.config.jobs - block_config = self.config.block - ledger_config = self.config.ledger - audit_config = self.config.audit - ctv_config = self.config.ctv - lifecycle_config = self.config.lifecycle - - self.rpc = JsonRpc( - host=rpc_config.host, - port=rpc_config.port, - user=rpc_config.user, - password=rpc_config.password, - ) - self.qbit_chain = rpc_config.chain - self.bind = stratum_config.bind - self.port = stratum_config.port - self.extranonce2_size = stratum_config.extranonce2_size - self.blockpoll_seconds = job_config.blockpoll_seconds - # Push-style tip detection rides waitfornewblock; the poll loop above - # stays as the fallback and still covers same-tip template refreshes. - self.blockwait_enabled = job_config.blockwait_enabled - self.blockwait_timeout_seconds = job_config.blockwait_timeout_seconds - self.block_submit_rpc_timeout_seconds = block_config.submit_rpc_timeout_seconds - self.block_submit_db_timeout_seconds = block_config.submit_db_timeout_seconds - self.block_landing_db_timeout_seconds = block_config.landing_db_timeout_seconds - self.block_landing_db_timeout_max_seconds = ( - block_config.landing_db_timeout_max_seconds - ) - self.accepted_parent_unresolved_depth_max = ( - block_config.accepted_parent_unresolved_depth_max - ) - self.accepted_parent_redrive_defer_threshold = ( - block_config.accepted_parent_redrive_defer_threshold - ) - self.accepted_parent_redrive_attempt_max = ( - block_config.accepted_parent_redrive_attempt_max - ) - self.block_submit_lock_wait_log_seconds = ( - block_config.submit_lock_wait_log_seconds - ) - self.block_submit_stuck_call_exit_seconds = ( - block_config.submit_stuck_call_exit_seconds - ) - # An own-hash tip observation is acceptance evidence even when the - # direct submitblock ack was lost; this window bounds how long that - # evidence blocks a terminal abandonment while fresh chain probes - # cannot prove the candidate active. - self.observed_tip_accept_window_seconds = ( - block_config.observed_tip_accept_window_seconds - ) - # After a FAILED refresh pass, the fallback poller re-attempts no - # sooner than this many seconds (plus jitter) while the tip the failed - # pass worked against is still current, so a persistent blockage or - # sustained payout churn costs ~1 attempt/second instead of one per - # 0.25s trigger slice. A successful pass or a newly observed tip - # re-arms immediately. Zero restores unspaced retries. - self.tip_refresh_failure_holdoff_seconds = ( - job_config.tip_refresh_failure_holdoff_seconds - ) - # Zero disables stale-grace crediting (every prior-tip share rejects, - # the pre-grace behavior). - self.stale_grace_seconds = stratum_config.stale_grace_seconds - # Per-share/per-job stdout logging is debug-only: at production share - # rates each print is a journald flush on the Stratum hot path. - self.hot_path_log_enabled = self.config.hot_path_log_enabled - # Zero disables the observed-tip reuse (every submit re-reads the tip - # over RPC, the legacy behavior). - self.submit_tip_max_age_seconds = job_config.submit_tip_max_age_seconds - self.same_tip_job_retention_seconds = ( - stratum_config.same_tip_job_retention_seconds - ) - self.same_tip_job_retention_per_connection = ( - stratum_config.same_tip_job_retention_per_connection - ) - self.disconnected_job_retention = stratum_config.disconnected_job_retention - self.tip_refresh_max_workers = job_config.tip_refresh_max_workers - self.tip_refresh_epoch_fanout = job_config.tip_refresh_epoch_fanout - self.job_build_timeout_seconds = job_config.job_build_timeout_seconds - self.job_build_cancel_grace_seconds = job_config.job_build_cancel_grace_seconds - self.vardiff_idle_sweep_seconds = stratum_config.vardiff_idle_sweep_seconds - # Reconnect difficulty retention: a reconnecting worker resumes at - # its last converged difficulty instead of the lane start. Off - # restores lane-start behavior exactly. - self.vardiff_resume_enabled = stratum_config.vardiff_resume_enabled - self.vardiff_resume_ttl_seconds = stratum_config.vardiff_resume_ttl_seconds - self.vardiff_resume_max_entries = stratum_config.vardiff_resume_max_entries - self.vardiff_resume_max_start_factor = ( - stratum_config.vardiff_resume_max_start_factor - ) - # Zero collapses every worker into the overflow label (per-worker - # metrics effectively off) without touching the aggregate counters. - self.worker_metrics_limit = job_config.worker_metrics_limit - self.reorg_reconciler_enabled = job_config.reorg_reconciler_enabled - # Per-template job caching. A zero disables the corresponding cache - # (every build redoes that stage), which is also the legacy behavior. - self.job_bundle_cache_seconds = job_config.job_bundle_cache_seconds - self.bundle_build_timeout_seconds = job_config.bundle_build_timeout_seconds - self.payout_artifact_rearm_min_seconds = ( - job_config.payout_artifact_rearm_min_seconds - ) - # Master kill-switch for the payout-artifact reuse line. Off - # restores the pre-reuse behavior (every ready build pays the - # synchronous reward-window walk and no background walks run), so a - # production regression is an env flip instead of a rollback. - self.payout_artifact_reuse_enabled = job_config.payout_artifact_reuse_enabled - self.payout_artifact_reanchor_seconds = ( - job_config.payout_artifact_reanchor_seconds - ) - self.payout_artifact_min_build_interval_seconds = ( - job_config.payout_artifact_min_build_interval_seconds - ) - self.payout_artifact_full_rescan_seconds = ( - job_config.payout_artifact_full_rescan_seconds - ) - self.payout_artifact_max_anchor_age_seconds = ( - job_config.payout_artifact_max_anchor_age_seconds - ) - self.routine_admission_deadline_seconds = ( - job_config.routine_admission_deadline_seconds - ) - self.template_cache_seconds = job_config.template_cache_seconds - self.template_refresh_failure_exit_seconds = ( - job_config.template_refresh_failure_exit_seconds - ) - self.coordination_blocked_exit_seconds = ( - lifecycle_config.coordination_blocked_exit_seconds - ) - self.last_successful_template_refresh_monotonic: float | None = None - self.template_refresh_failure_started_monotonic: float | None = None - # Coordination-blocked streak state moved to the WatchdogService - # owner (see lab/prism/background_services.py). - self.reorg_reconcile_cache_seconds = job_config.reorg_reconcile_cache_seconds - self.health_refresh_seconds = lifecycle_config.health_refresh_seconds - self.metrics_refresh_seconds = lifecycle_config.metrics_refresh_seconds - self.health_pending_refresh_max_age_seconds = ( - lifecycle_config.pending_refresh_health_deadline_seconds - ) - self.health_tip_poll_max_age_seconds = ( - lifecycle_config.coherent_tip_poll_health_deadline_seconds - ) - self.stratum_send_timeout_seconds = stratum_config.send_timeout_seconds - # Zero remains an explicit unlimited option for focused local/regtest - # use. Deployments default to a conservative ceiling above PRISM's - # normal 200-250 connection population. - self.stratum_max_connections = stratum_config.max_connections - self.stratum_max_connections_per_username = ( - stratum_config.max_connections_per_username - ) - self.stratum_max_pending_initial_jobs = stratum_config.max_pending_initial_jobs - if config is None: - # The worker knobs stay visibly env-wired in this module for the - # zero-argument production path (an upstream lost-fix detector - # asserts the source); the values are identical to what the - # snapshot loader just validated. A supplied snapshot never - # rereads the environment. - self.initial_job_max_workers = resolve_initial_job_max_workers( - env_optional_positive_int("PRISM_INITIAL_JOB_MAX_WORKERS"), - self.stratum_max_pending_initial_jobs, - ) - self.job_build_executor_workers = validate_job_build_executor_workers( - env_positive_int( - "PRISM_JOB_BUILD_EXECUTOR_WORKERS", - DEFAULT_PRISM_JOB_BUILD_EXECUTOR_WORKERS, - ) - ) - else: - self.initial_job_max_workers = stratum_config.initial_job_max_workers - self.job_build_executor_workers = job_config.job_build_executor_workers - self.stratum_initial_job_timeout_seconds = ( - stratum_config.initial_job_timeout_seconds - ) - self.mining_health_startup_grace_seconds = ( - lifecycle_config.mining_health_startup_grace_seconds - ) - self.stratum_accept_resource_exhaustion_backoff_seconds = ( - stratum_config.accept_resource_exhaustion_backoff_seconds - ) - self.stratum_listen_backlog = stratum_config.listen_backlog - self.stratum_bind_retry_seconds = stratum_config.bind_retry_seconds - self.payout_address_cache_max_entries = ( - stratum_config.payout_address_cache_max_entries - ) - self.payout_address_cache_ttl_seconds = ( - stratum_config.payout_address_cache_ttl_seconds - ) - self.coinbase_tag_hex = self.config.coinbase_tag_hex - self.share_difficulty = stratum_config.share_difficulty - self.vardiff_config = stratum_config.vardiff_config - self.listener_profiles = list(stratum_config.listener_profiles) - self.default_share_weight = stratum_config.default_share_weight - self.share_weights_by_username = dict(stratum_config.share_weights_by_username) - self.username_fallback_address = stratum_config.username_fallback_address - self.min_ready_miners = job_config.min_ready_miners - self.signing_seed_hex = ledger_config.signing_seed_hex - self.ledger_attestation_signing_seed_hex = ( - ledger_config.attestation_signing_seed_hex - ) - self.ledger_writer_public_key_hex = ledger_config.writer_public_key_hex - self.evidence_path = audit_config.evidence_path - self.audit_dir = audit_config.directory - self.audit_share_segment_size = audit_config.share_segment_size - self.audit_live_bundle_retention = audit_config.live_bundle_retention - self.audit_candidate_retention_seconds = ( - audit_config.candidate_retention_seconds - ) - self.ctv_broadcast_attempt_detail_limit = ( - ctv_config.broadcast_attempt_detail_limit - ) - self.ctv_broadcast_retry_backoff_seconds = ( - ctv_config.broadcast_retry_backoff_seconds - ) - self.audit_bind = audit_config.bind - self.audit_port = audit_config.port - self.stop_after_block = self.config.stop_after_block - self.max_blocks = self.config.max_blocks - self.version_mask_selection = self.resolve_version_rolling_mask( - stratum_config.fallback_version_mask - ) - self.version_mask = self.version_mask_selection.selected_mask - self.writer_quiescence_timeout_seconds = ( - lifecycle_config.writer_quiescence_timeout_seconds - ) - self.ledger = self.make_ledger() - # Incremental hashrate rollup maintenance rides the writer because - # only the lease holder may mutate ledger-derived state; the public - # read tier never runs it. The flag and cadence stay plain - # attributes so tests and embedders can retune a live coordinator. - self.hashrate_rollup_enabled = env_bool("PRISM_HASHRATE_ROLLUP_ENABLED", "1") - self.hashrate_rollup_interval_seconds = env_positive_float( - "PRISM_HASHRATE_ROLLUP_INTERVAL_SECONDS", 15.0 - ) - self.hashrate_rollup_batch_shares = env_positive_int( - "PRISM_HASHRATE_ROLLUP_BATCH_SHARES", 50000 - ) - self._upgrade_legacy_audit_evidence() - self._ctv_fanout_market_fee_rate_cache: dict[tuple[int | None, str | None], int] = {} - self.lock = _ObservedRLock( - wait_observer=self._observe_coordinator_lock_wait, - ) - self.clients: set[ClientState] = set() - self.connection_limit_rejection_counts = {"global": 0, "username": 0} - self.peak_active_connection_count = 0 - self.handler_thread_count = 0 - self.accept_resource_exhaustion_count = 0 - self.connection_setup_failure_count = 0 - self.pending_initial_jobs: dict[ClientState, PendingInitialJob] = {} - self.initial_job_queue_rejection_count = 0 - self.initial_job_timeout_count = 0 - self.initial_job_cancelled_count = 0 - self.initial_job_coalesced_count = 0 - self.initial_job_queue_capacity_reclaimed_count = 0 - self.last_initial_job_delivery_monotonic: float | None = None - self._p2mr_address_cache_lock = threading.Lock() - self._p2mr_address_cache: OrderedDict[ - str, tuple[float, tuple[str, str]] - ] = OrderedDict() - self._p2mr_address_validation_inflight: dict[ - str, _P2mrAddressValidationFlight - ] = {} - self.jobs: dict[str, PrismJobContext] = {} - # Share-path ownership is deliberately disjoint from the coordinator - # control-plane lock. Deduplication remains process-wide so an exact - # header replay across reauthorization/connections is still rejected; - # the submission owner adopts this seed on first hot-path touch. - self.recent_share_keys: set[tuple[object, ...]] = set() - self._share_accounting_lock = threading.Lock() - self.connection_counter = 0 - self.job_counter = 0 - self.accepted_block_count = 0 - # A durable outbox terminal update can fail after the accepted-block - # success tail has completed. Same-process replay must not count or - # announce that hash twice; a fresh process intentionally starts with - # an empty set and reconstructs its process-local count from replay. - self._accounted_accepted_block_hashes: set[str] = set() - self.started_monotonic = time.monotonic() - self.submitted_share_count = 0 - self.stale_share_count = 0 - self.duplicate_share_count = 0 - self.low_difficulty_share_count = 0 - self.collection_block_submission_count = 0 - self.block_solves_dropped_counts = {"stale_grace": 0} - self._pool_ready_latched = False - self.grace_credited_share_count = 0 - self.idle_retarget_count = 0 - self._ensure_vardiff_idle_state() - self.rejection_counts_by_reason = {reason: 0 for reason in PRISM_REJECTION_REASON_IDS} - # Per-worker share accounting with a bounded label set; see - # worker_metric_label for the admission rule. - self.worker_metrics_lock = threading.Lock() - self.worker_share_counts: dict[str, dict[str, int]] = {} - self.worker_rejection_counts: dict[tuple[str, str], int] = {} - # Globally insertion ordered, with per-connection indexes for the - # independent TTL and capacity limits. Prior-tip entries never consume - # the same-tip cap while stale-grace still protects them. - self.evicted_job_graveyard: OrderedDict[str, EvictedJobEntry] = OrderedDict() - self.evicted_jobs_by_connection: dict[int, OrderedDict[str, None]] = {} - self.evicted_same_tip_by_connection: dict[int, OrderedDict[str, None]] = {} - self.evicted_same_tip_job_ids: OrderedDict[str, None] = OrderedDict() - self.evicted_job_index_tip_hash: str | None = None - self.evicted_job_next_prune_monotonic = 0.0 - self.evicted_job_expiration_counts = { - job_class: 0 for job_class in PRISM_EVICTED_JOB_CLASSES - } - self.evicted_job_capacity_eviction_counts = { - scope: 0 for scope in PRISM_EVICTED_JOB_CAPACITY_SCOPES - } - self.evicted_job_submit_counts = { - outcome: 0 for outcome in PRISM_EVICTED_JOB_SUBMIT_OUTCOMES - } - # Published share-validation authority. Detection is kept separately - # in latest_detected_tip so a waiter can cancel obsolete refresh work - # without invalidating jobs before their replacement is ready. - # The flip stamp is None for the startup baseline, which never opens - # the stale-grace window. - self.current_tip_first_seen: tuple[str, float | None] | None = None - self.current_tip_parent: tuple[str, str] | None = None - self.latest_detected_tip: tuple[str, int] | None = None - # Start of the current detected-vs-published divergence epoch. Unlike - # latest_detected_tip, this stamp is not renewed by B -> C -> D while - # published work is still on A; submit authority therefore has a - # bounded lease even during repeated refresh failures. - self.tip_refresh_divergence_started_monotonic: float | None = None - # When the refresh path last published or reconfirmed the authoritative - # tip. Bounds normal submit reuse when no replacement is actively - # pending (see submit_stale_check_tip). - self.current_tip_observed_monotonic: float | None = None - # Block candidates are landed by a dedicated submitter thread so a - # winning share's ack (and every other client's) never waits on - # audit/persist/submitblock; see enqueue_block_candidate. - self.block_candidate_queue: queue.Queue[PrismBlockCandidate] = queue.Queue( - maxsize=MAX_PENDING_BLOCK_CANDIDATES - ) - # Durable recovery work is separate and lower priority: a newly found - # live block must never sit behind a restart batch. The in-flight set - # lets batch replay expose later rows while an older row is still in - # accounting without repeatedly queueing the older hash. - self._block_replay_candidate_queue: queue.Queue[ - PrismBlockCandidate - ] = queue.Queue() - self._block_replay_inflight_hashes: set[str] = set() - self._block_quarantine_queue: queue.Queue[tuple[str, str]] = queue.Queue() - self._block_quarantine_hashes: set[str] = set() - self._block_candidate_disposition_registry_lock = threading.Lock() - self._block_candidate_disposition_flights: dict[ - str, _BlockCandidateDispositionFlight - ] = {} - self._block_candidate_terminal_outcomes: dict[str, bool] = {} - self._block_fast_lane_reservations: set[str] = set() - self._block_disposition_waiting_retries: dict[ - str, PrismBlockCandidate - ] = {} - self._block_accounting_queue: queue.PriorityQueue[ - tuple[int, int, _BlockCandidateAccountingTask] - ] = queue.PriorityQueue() - self._block_accounting_overflow_queue: queue.PriorityQueue[ - tuple[int, int, _BlockCandidateAccountingTask] - ] = queue.PriorityQueue() - # Definitive node acceptances only, and unbounded for the same reason - # the overflow queue is: the offer already happened, so it may never - # be converted back into a raw-submit retry. - self._block_accounting_accepted_queue: queue.PriorityQueue[ - tuple[int, int, _BlockCandidateAccountingTask] - ] = queue.PriorityQueue() - self._block_accounting_sequence = 0 - self._block_accounting_state_lock = threading.Lock() - self._block_accounting_thread: threading.Thread | None = None - self.block_candidates_dropped = 0 - self.block_candidate_wakeups_coalesced = 0 - self.block_candidate_retry_count = 0 - self.block_candidate_poisoned_count = 0 - self.block_candidate_accept_pending_defer_count = 0 - # Hashes of block candidates this process may still land (durable - # outbox pending, queued, retained for retry, or mid-disposition). - # Membership lets every tip-observation channel recognize the pool's - # own block the moment it becomes the chain tip. - self._outstanding_block_candidate_hashes: set[str] = set() - # Outstanding candidate hashes observed as the chain tip - # (hash -> monotonic stamp). qbitd only reports a candidate hash as - # its tip after accepting that block, so an entry here is acceptance - # evidence that outlives transient fork views and lost submitblock - # acks; disposition/abandon paths consult it before treating any - # instantaneous chain probe as terminal truth. - self._tip_observed_accepted_block_hashes: dict[str, float] = {} - self.block_candidate_retry_initial_seconds = ( - DEFAULT_BLOCK_CANDIDATE_RETRY_INITIAL_SECONDS - ) - self.block_candidate_retry_max_seconds = DEFAULT_BLOCK_CANDIDATE_RETRY_MAX_SECONDS - self.block_candidate_retry_delays: dict[str, float] = {} - self._block_submitter_retry_state_lock = threading.Lock() - self._block_submitter_backoff_started_monotonic: float | None = None - self._block_submitter_backoff_deadline_monotonic: float | None = None - self._block_submitter_backoff_delay_seconds = 0.0 - self._block_submitter_ledger_calls_lock = threading.Lock() - self._block_submitter_ledger_calls: dict[ - tuple[object, ...], _BlockSubmitterLedgerCall - ] = {} - self._block_submitter_ledger_worker_slots = threading.BoundedSemaphore( - MAX_BLOCK_SUBMITTER_STUCK_CALL_WORKERS - ) - self._block_submitter_rpc_calls_lock = threading.Lock() - self._block_submitter_rpc_calls: dict[str, _BlockSubmitterRpcCall] = {} - self._block_submitter_rpc_worker_slots = threading.BoundedSemaphore( - MAX_BLOCK_SUBMITTER_STUCK_CALL_WORKERS - ) - self._fatal_exit_requested = False - self._block_submitter_last_lock_wait_log_monotonic = 0.0 - # Terminal candidates whose durable outbox update failed; replays for - # these run finalize-only (see _finalize_block_candidate). - self._block_candidate_finalize_retries: dict[str, tuple[bool, str]] = {} - # A block candidate that loses its tip race (or fails to submit) is a - # BLOCK-path event, not a share rejection: under the async model the - # share was already accepted and credited, so it must not touch the - # share-reject counters (that would inflate stale_share_percent with - # block-race losses). Tracked here by reason instead. - self.block_candidate_abandoned_counts: dict[str, int] = {} - # Durable cleanup can fail after a terminal decision and force the - # same hash through that decision again; abandonment metrics count - # candidates, not cleanup attempts. - self._counted_block_candidate_abandonments: set[str] = set() - self.stale_job_abandon_counts = { - abandon_class: 0 - for abandon_class in PRISM_STALE_JOB_ABANDON_CLASSES - } - # Accepted shares drain through a bounded group-commit writer. A - # submitting client waits on its entry's completion event, making the - # database commit the acknowledgement boundary without paying one - # process/transaction round trip per share during bursts. - self.share_append_queue: queue.Queue[PendingShareAppend] = queue.Queue( - maxsize=MAX_PENDING_SHARE_APPENDS - ) - self.share_commit_batch_size = ledger_config.share_commit_batch_size - self.share_commit_linger_seconds = ledger_config.share_commit_linger_seconds - self.share_commit_timeout_seconds = ledger_config.share_commit_timeout_seconds - self.share_writer_active = False - self.share_append_failure_count = 0 - # Retain the historical recovery-file reader for clean upgrades from a - # release that could acknowledge before Postgres commit. New shares - # are never written here: an unavailable ledger produces no success - # acknowledgement and an exact retry is idempotent. - self.share_recovery_path = ledger_config.share_recovery_path - self.share_recovery_lock = threading.Lock() - self.shares_recovered_to_disk = 0 - self.shares_replayed = 0 - self.share_replay_conflicts = 0 - self.job_build_failure_count = 0 - self.tip_refresh_job_count = 0 - self.post_accept_refresh_failure_count = 0 - self.reorg_inactive_block_count = 0 - self.reorg_reactivated_block_count = 0 - self.reorg_reconcile_skip_count = 0 - self.reorg_reconcile_error_count = 0 - self.matured_payout_count = 0 - # The full accepted-block bundle is durable in the audit store. Keeping - # it here only to derive one metric pinned the complete share window for - # the lifetime of the coordinator. - self.latest_coinbase_size_bytes: int | None = None - self.tip_template_snapshot: QbitTipTemplateSnapshot | None = None - self._tip_refresh_lock = threading.Lock() - self._tip_refresh_singleflight_lock = threading.Lock() - self._tip_refresh_executor_lock = threading.Lock() - self._tip_refresh_executor: _BoundedPriorityExecutor | None = None - self._tip_refresh_executor_shutdown = False - self._initial_job_executor_lock = threading.Lock() - self._initial_job_executor: _BoundedPriorityExecutor | None = None - self._initial_job_executor_shutdown = False - self._tip_refresh_pending_event = threading.Event() - self._tip_refresh_pending_counter = 0 - self._tip_refresh_pending_token: int | None = None - self._tip_refresh_retry = threading.Event() - self._tip_refresh_retry_counter = 0 - self._tip_refresh_retry_consumed = 0 - self._tip_refresh_failure_holdoff_until: float | None = None - self._tip_refresh_failure_tip: str | None = None - self._active_tip_refresh: tuple[ - TipRefreshValidationToken, - _FanoutCancellation, - ] | None = None - self._retained_collection_refresh: RetainedCollectionRefresh | None = None - self.last_reorg_reconciled_tip_hash: str | None = None - self.last_reorg_reconciled_trusted = False - self.last_reorg_reconciled_monotonic: float | None = None - self._prism_payout_policy_cache: dict[str, object] | None = None - self._ensure_job_cache_state() - # Eager reorg owner root: the reconciler service adopts the compat - # seed values assigned above and owns every flight/memo/lookup/ - # prefetch container from here on. - self._ensure_reorg_reconciler_service() - self.stop_event = threading.Event() - self._shutdown_controller = CoordinatorShutdownController( - self.writer_quiescence_timeout_seconds - ) - # Liveness watchdog: each monitored loop stamps a monotonic heartbeat; - # if any goes stale past the timeout the process exits non-zero so the - # container/systemd restart policy recovers a *hung* coordinator (a - # healthcheck alone does not restart it under plain compose). - self._heartbeats: dict[str, float] = {} - self._heartbeat_phases: dict[str, str] = {} - self._watchdog_pauses: dict[str, int] = {} - self._heartbeats_lock = threading.Lock() - self.watchdog_enabled = lifecycle_config.watchdog_enabled - self.watchdog_timeout_seconds = lifecycle_config.watchdog_timeout_seconds - self.watchdog_interval_seconds = lifecycle_config.watchdog_interval_seconds - # Lease/watchdog timing envelopes are projected from the immutable - # snapshot; services read the live attributes only because focused - # timing tests override them per instance after construction. - self.watchdog_lease_release_timeout_seconds = ( - lifecycle_config.watchdog_lease_release_timeout_seconds - ) - self.ledger_lease_heartbeat_seconds = ( - lifecycle_config.ledger_lease_heartbeat_seconds - ) - self.ledger_lease_heartbeat_failure_seconds = ( - lifecycle_config.ledger_lease_heartbeat_failure_seconds - ) - self.ledger_lease_heartbeat_monitor_seconds = ( - lifecycle_config.ledger_lease_heartbeat_monitor_seconds - ) - self.ledger_lease_heartbeat_exit_timeout_seconds = ( - lifecycle_config.ledger_lease_heartbeat_exit_timeout_seconds - ) - self.ledger_lease_external_fence_timeout_seconds = ( - lifecycle_config.ledger_lease_external_fence_timeout_seconds - ) - self._ctv_runtime_init_lock = threading.Lock() - self._ctv_runtime = self._make_ctv_runtime_service( - CtvRuntimeConfig.from_coordinator_config(ctv_config) - ) - # Eager B1 owner root: the block-candidate service adopted the legacy - # fields installed above (the first routed write constructs it) and - # owns every candidate queue/retry/disposition container from here on. - self._ensure_block_candidate_service() - # Eager B3 owner root: post-offer accepted-block finalization and its - # bounded phase/interarrival metrics live on the finalization service; - # B1's accounting actor reaches it through the coordinator seams. - self._ensure_block_finalization_service() - # Eager background-service root: process loop specifications, their - # start-once state, and the exact drain handles live in the registry; - # serve() starts named services at the existing recovery boundaries. - self._background_services = self._make_background_service_registry() - - def _ensure_share_hot_path_state(self) -> None: - """Backfill dedicated accounting state for lightweight embedders. - - Reduced first-touch shim: the recent-share duplicate window and the - share-ACK histograms are owned by the share-submission service, so - only the legacy accounting-counter lock is ensured here. - """ - if hasattr(self, "_share_accounting_lock"): - return - with _HOT_PATH_LOCK_INITIALIZATION_LOCK: - if not hasattr(self, "_share_accounting_lock"): - self._share_accounting_lock = threading.Lock() - - def _observe_share_ack_seconds( - self, - result: str, - elapsed_seconds: float, - ) -> None: - # Coordinator/session routing seam: the session owner stamps request - # receipt and reports the accepted/rejected response boundary here; - # the share-submission owner holds the histogram state. - self._ensure_share_submission_service().observe_share_ack_seconds( - result, - elapsed_seconds, - ) - - def share_ack_snapshot(self) -> dict[str, dict[str, Any]]: - """Copied share-ACK histograms for the metrics renderer. - - The data is the submission owner's copied snapshot. A coordinator - whose submission service was never touched has observed nothing, so - it reports zeroed histograms (or a legacy embedder's seeded state) - without forcing service construction. - """ - service = self.__dict__.get("_share_submission_service") - if service is not None: - return service.share_ack_snapshot() - legacy = self.__dict__.get("share_ack_histograms") - return ( - { - result: { - "buckets": dict(histogram["buckets"]), - "sum": float(histogram["sum"]), - "count": int(histogram["count"]), - } - for result, histogram in legacy.items() - } - if legacy is not None - else empty_share_ack_histograms() - ) - - def share_ack_metrics_lines(self) -> list[str]: - # Compatibility wrapper for the extracted metrics owner. - return self._ensure_metrics_renderer().share_ack_metrics_lines() - - @staticmethod - def _client_vardiff_lock(client: ClientState) -> threading.RLock: - return client_vardiff_lock(client) - - def _reserve_recent_share_key(self, share_key: tuple[object, ...]) -> bool: - self._ensure_share_hot_path_state() - return self._ensure_share_submission_service().recent_shares.reserve( - share_key # type: ignore[arg-type] - ) - - def _forget_recent_share_key(self, share_key: tuple[object, ...]) -> None: - self._ensure_share_submission_service().recent_shares.release( - share_key # type: ignore[arg-type] - ) - - def _share_submit_control_snapshot( - self, - client: ClientState, - job_id: str, - ) -> SubmitControlSnapshot: - pool_closed, context, published_tip = self._submit_control_snapshot( - client, - job_id, - ) - return SubmitControlSnapshot( - pool_open=not pool_closed, - active_context=context, - published_tip=published_tip, - ) - - def _release_submit_share_key(self, share_key: tuple[str, str]) -> None: - self._ensure_share_submission_service().recent_shares.release(share_key) - - def _note_collection_block_candidate( - self, - context: PrismJobContext, - submission: Any, - ) -> None: - self._ensure_share_hot_path_state() - with self._share_accounting_lock: - self.collection_block_submission_count = ( - getattr(self, "collection_block_submission_count", 0) + 1 - ) - print( - f"prism coordinator: collection-mode block candidate settles " - f"solver-pays-all miner={context.worker.payout_address} " - f"hash={submission.block_hash_hex}", - flush=True, - ) - - def _note_submit_accounting( - self, - worker_name: str, - client: ClientState, - ) -> None: - self.note_worker_submitted_share(worker_name) - self.note_vardiff_submitted_share(client) - - def _ensure_share_submission_service(self) -> ShareSubmissionService: - service = self.__dict__.get("_share_submission_service") - if service is not None: - return service - init_lock = self.__dict__.setdefault( - "_share_submission_service_init_lock", - threading.Lock(), - ) - with init_lock: - service = self.__dict__.get("_share_submission_service") - if service is not None: - return service - initial_recent_shares = self.__dict__.pop("recent_share_keys", set()) - initial_dropped_solves = self.__dict__.pop( - "block_solves_dropped_counts", None - ) - initial_share_ack = self.__dict__.pop("share_ack_histograms", None) - service = ShareSubmissionService( - ShareSubmissionPorts( - reject=lambda rejected, worker: self.reject_stratum( - rejected.code, - rejected.reason, - rejected.message, - worker=worker, - ), - control_snapshot=self._share_submit_control_snapshot, - note_submitted=self._note_submit_accounting, - retained_entry=lambda client, job_id: self.evicted_job_entry( - client, - job_id, - ), - live_tip=lambda: str(self.rpc.call("getbestblockhash")), - stale_grace_eligible=( - lambda client, context, current_tip: ( - self.context_eligible_for_stale_grace( - client, - context, - current_tip, - ) - ) - ), - assemble=lambda client, context, request, version_mask: ( - direct_stratum.assemble_submission( - context.job, - extranonce2_hex=request.extranonce2_hex, - ntime_hex=request.ntime_hex, - nonce_hex=request.nonce_hex, - version_bits_hex=request.version_bits_hex, - version_mask=version_mask, - ) - ), - pending_share=lambda context, submission, ntime_hex, credit_policy: ( - self.pending_share_from_submission( - context=context, - submission=submission, - ntime_hex=ntime_hex, - credit_policy=credit_policy, - ) - ), - append_share=( - lambda client, context, submission, pending, policy, intent: ( - self.append_accepted_share( - client, - context, - submission, - pending, - credit_policy=policy, - candidate_intent=intent, - ) - ) - ), - note_retained_submit=( - lambda policy, cross_connection: self.note_evicted_job_submit( - policy, - cross_connection=cross_connection, - ) - ), - note_collection_candidate=( - lambda context, submission: self._note_collection_block_candidate( - context, - submission, - ) - ), - candidate_intent=lambda candidate: self.block_candidate_intent( - candidate - ), - finish_pending_commit=lambda pending: ( - self._finish_pending_share_commit(pending) - ), - record_terminal_outcome=lambda block_hash, accepted: ( - self._record_block_candidate_terminal_outcome( - block_hash, - accepted=accepted, - ) - ), - submit_synchronous_candidate=( - lambda candidate, share_key, worker, retained, policy: ( - self._submit_synchronous_credit_candidate( - candidate, - share_key=share_key, - worker_name=worker, - evicted_entry=retained, - credit_policy=policy, - ) - ) - ), - enqueue_candidate=lambda candidate: self.enqueue_block_candidate( - candidate - ), - log=lambda message: print(message, flush=True), - log_exception=traceback.print_exc, - ), - extranonce2_size=int(self.extranonce2_size), - recent_shares=RecentShareIndex(initial=initial_recent_shares), - ) - if initial_dropped_solves is not None: - service.replace_dropped_solves(initial_dropped_solves) - if initial_share_ack is not None: - service.replace_share_ack_histograms(initial_share_ack) - self.__dict__["_share_submission_service"] = service - return service - - def record_rejection(self, reason: str, *, worker: str | None = None) -> None: - if reason not in PRISM_REJECTION_REASON_IDS: - raise ValueError(f"unknown PRISM rejection reason: {reason}") - self._ensure_share_hot_path_state() - with self._share_accounting_lock: - counts = getattr(self, "rejection_counts_by_reason", None) - if counts is None: - counts = {reason_id: 0 for reason_id in PRISM_REJECTION_REASON_IDS} - self.rejection_counts_by_reason = counts - counts[reason] = int(counts.get(reason, 0)) + 1 - if reason in {PRISM_REJECTION_STALE_JOB, PRISM_REJECTION_UNKNOWN_JOB, PRISM_REJECTION_BLOCK_STALE}: - self.stale_share_count += 1 - elif reason == PRISM_REJECTION_DUPLICATE_SHARE: - self.duplicate_share_count += 1 - elif reason == PRISM_REJECTION_LOW_DIFFICULTY: - self.low_difficulty_share_count += 1 - if worker is not None: - self._ensure_worker_metrics_state() - with self.worker_metrics_lock: - label = self._worker_metric_label_locked(worker) - key = (label, reason) - self.worker_rejection_counts[key] = ( - int(self.worker_rejection_counts.get(key, 0)) + 1 - ) - - def reject_stratum(self, code: int, reason: str, message: str, *, worker: str | None = None) -> None: - self.record_rejection(reason, worker=worker) - raise StratumError(code, message, reason=reason) - - def worker_metric_label(self, worker: str) -> str: - """Metric label for one worker, from a bounded label set. - - The label is the stratum username as authorized (payout address plus - optional worker suffix). Usernames are miner-supplied, so the set of - distinct labels is capped: new workers past the cap aggregate into the - overflow label instead of growing metric cardinality without bound. - """ - self._ensure_worker_metrics_state() - with self.worker_metrics_lock: - return self._worker_metric_label_locked(worker) - - def _worker_metric_label_locked(self, worker: str) -> str: - label = worker or "_unauthenticated" - if len(label) > 128: - label = label[:128] - share_counts = self.worker_share_counts - if label in share_counts: - return label - limit = getattr(self, "worker_metrics_limit", DEFAULT_PRISM_WORKER_METRICS_LIMIT) - if len(share_counts) >= max(0, int(limit)): - label = PRISM_WORKER_METRICS_OVERFLOW_LABEL - share_counts.setdefault(label, {"submitted": 0, "accepted": 0, "grace": 0}) - return label - - def note_worker_submitted_share(self, worker: str) -> None: - self._ensure_worker_metrics_state() - with self.worker_metrics_lock: - label = self._worker_metric_label_locked(worker) - self.worker_share_counts[label]["submitted"] += 1 - - def note_worker_accepted_share(self, worker: str, credit_policy: str | None) -> None: - self._ensure_worker_metrics_state() - with self.worker_metrics_lock: - label = self._worker_metric_label_locked(worker) - counts = self.worker_share_counts[label] - counts["accepted"] += 1 - if credit_policy == PRISM_CREDIT_POLICY_STALE_GRACE: - counts["grace"] += 1 - if credit_policy == PRISM_CREDIT_POLICY_STALE_GRACE: - self._ensure_share_hot_path_state() - with self._share_accounting_lock: - self.grace_credited_share_count = ( - int(getattr(self, "grace_credited_share_count", 0)) + 1 - ) - - @staticmethod - def prometheus_label_value(value: str) -> str: - return ( - value.replace("\\", "\\\\") - .replace("\n", "\\n") - .replace("\r", "\\r") - .replace('"', '\\"') - ) - - def resolve_version_rolling_mask(self, fallback_mask: int) -> direct_stratum.VersionRollingMaskSelection: - try: - template = self.rpc.call("getblocktemplate", [{"rules": qbit_gbt_rules(self.qbit_chain)}]) - if not isinstance(template, dict): - raise RuntimeError("getblocktemplate returned non-object") - except Exception as exc: - return direct_stratum.VersionRollingMaskSelection( - fallback_mask, - "fallback", - f"probe_error:{exc}", - ) - try: - return direct_stratum.select_version_rolling_mask(template, fallback_mask) - except ValueError as exc: - raise SystemExit(str(exc)) from exc - - def parse_share_weights(self) -> dict[str, int]: - config = getattr(self, "config", None) - if config is not None: - return dict(config.stratum.share_weights_by_username) - return load_share_weights() - - def share_weight_for_worker(self, worker: WorkerIdentity) -> int: - return self.share_weights_by_username.get( - worker.username, - self.share_weights_by_username.get(worker.payout_address, self.default_share_weight), - ) - - def _ensure_audit_artifact_store(self) -> AuditArtifactStore: - init_lock = self.__dict__.setdefault( - "_audit_artifact_store_init_lock", - threading.Lock(), - ) - assert isinstance(init_lock, type(threading.Lock())) - with init_lock: - configured_audit_dir = self.__dict__.get("audit_dir") - if configured_audit_dir is None: - # Normal construction always sets audit_dir before any store - # use; only bare test instances reach this fallback. Root it - # under a per-instance temporary directory so a focused test - # never litters the process working directory. - fallback_root = self.__dict__.get("_audit_artifact_fallback_root") - if fallback_root is None: - fallback_root = Path( - tempfile.mkdtemp(prefix="prism-audit-fallback-") - ) - self.__dict__["_audit_artifact_fallback_root"] = fallback_root - configured_audit_dir = Path(fallback_root) / "prism-audit" - audit_dir = Path(configured_audit_dir) - evidence_path = Path( - self.__dict__.get( - "evidence_path", - audit_dir / "prism-live-stratum-evidence.json", - ) - ) - live_retention = int( - self.__dict__.get("audit_live_bundle_retention", 5) - ) - candidate_retention = int( - self.__dict__.get( - "audit_candidate_retention_seconds", - 24 * 60 * 60, - ) - ) - share_segment_size = int( - self.__dict__.get( - "audit_share_segment_size", - DEFAULT_AUDIT_SHARE_SEGMENT_SIZE, - ) - ) - # The verifier subprocess honors the coordinator's configured - # bundle-build budget rather than a store-local fixed default. - verifier_timeout_seconds = max( - 0.001, - float( - self.__dict__.get( - "bundle_build_timeout_seconds", - DEFAULT_PRISM_BUNDLE_BUILD_TIMEOUT_SECONDS, - ) - ), - ) - store = self.__dict__.get("_audit_artifact_store") - if not isinstance(store, AuditArtifactStore): - store = AuditArtifactStore( - AuditArtifactConfig( - root=audit_dir, - evidence_path=evidence_path, - live_bundle_retention=live_retention, - candidate_retention_seconds=candidate_retention, - share_segment_size=share_segment_size, - verifier_timeout_seconds=verifier_timeout_seconds, - ), - canonicalizer=canonical_bundle_bytes, - ) - self.__dict__["_audit_artifact_store"] = store - if "_audit_latest_evidence_seed" in self.__dict__: - store.set_latest_evidence_for_compatibility( - self.__dict__.pop("_audit_latest_evidence_seed") - ) - else: - updates: dict[str, Any] = {} - if store.root != audit_dir.expanduser().absolute().resolve(): - updates["root"] = audit_dir - expected_evidence = ( - evidence_path.expanduser().absolute().parent.resolve() - / evidence_path.name - ) - if store.evidence_path != expected_evidence: - updates["evidence_path"] = evidence_path - if store.live_bundle_retention != live_retention: - updates["live_bundle_retention"] = live_retention - if store.candidate_retention_seconds != candidate_retention: - updates["candidate_retention_seconds"] = candidate_retention - if store.share_segment_size != share_segment_size: - updates["share_segment_size"] = share_segment_size - if store.verifier_timeout_seconds != verifier_timeout_seconds: - updates["verifier_timeout_seconds"] = verifier_timeout_seconds - if updates: - store.reconfigure(**updates) - return store - - def _upgrade_legacy_audit_evidence(self) -> None: - store = self._ensure_audit_artifact_store() - with self._payout_balance_mutation_lock: - with store.publication_order_guard(): - legacy = store.legacy_evidence_identity() - if legacy is None: - return - reader = getattr(self.ledger, "pool_block_state", None) - floor_reader = getattr( - self.ledger, - "audit_publication_sequence_floor", - None, - ) - if not callable(reader) or not callable(floor_reader): - store.invalidate_unprovable_legacy_evidence() - return - state = reader(block_hash=legacy.block_hash) - if not isinstance(state, dict): - store.invalidate_unprovable_legacy_evidence() - return - sequence = state.get("audit_publication_sequence") - state_block_hash = state.get("block_hash") - state_block_height = state.get("block_height") - if ( - sequence is None - or isinstance(sequence, bool) - or not isinstance(sequence, int) - or sequence <= 0 - or not isinstance(state_block_hash, str) - or state_block_hash != legacy.block_hash - or isinstance(state_block_height, bool) - or not isinstance(state_block_height, int) - or state_block_height != legacy.block_height - or str(state.get("chain_state") or "") != "confirmed" - or str(state.get("maturity_state") or "") - not in {"immature", "mature"} - ): - store.invalidate_unprovable_legacy_evidence() - return - publication_floor_sequence = floor_reader() - store.adopt_legacy_publication_identity( - AuditPublicationIdentity( - int(sequence), - legacy.block_height, - legacy.block_hash, - ), - publication_floor_sequence=publication_floor_sequence, - ) - - def _audit_publication_identity( - self, - *, - block_hash: str, - block_height: int, - confirmation: Mapping[str, Any], - ) -> AuditPublicationIdentity: - sequence = confirmation.get("audit_publication_sequence") - if sequence is None: - # Compatibility-only fake ledgers in unit tests predate the durable - # ordinal. Production Postgres and the memory ledger always return - # it; never synthesize for an identified durable backend. - if str(confirmation.get("backend") or "") not in {"", "fake"}: - raise RuntimeError( - "ledger confirmation omitted audit publication sequence" - ) - sequences = self.__dict__.setdefault( - "_compat_audit_publication_sequences", - {}, - ) - assert isinstance(sequences, dict) - sequence = sequences.get(block_hash) - if sequence is None: - sequence = max( - [ - self._ensure_audit_artifact_store().publication_sequence_floor(), - *(int(value) for value in sequences.values()), - ] - ) + 1 - sequences[block_hash] = sequence - if isinstance(sequence, bool) or not isinstance(sequence, int): - raise RuntimeError("ledger confirmation returned invalid publication sequence") - if isinstance(block_height, bool) or not isinstance(block_height, int): - raise RuntimeError("ledger confirmation returned invalid block height") - canonical_block_hash = str(block_hash).lower() - if canonical_block_hash != block_hash: - raise RuntimeError("ledger confirmation returned non-canonical block hash") - return AuditPublicationIdentity( - sequence, - block_height, - canonical_block_hash, - ) - - def make_ledger( - self, - *, - lease_retry_sleep: Callable[[float], None] | None = None, - ) -> SingleWriterShareLedger | PsqlShareLedger: - config = getattr(self, "config", None) - ledger_config = config.ledger if config is not None else None - psql_command = ( - ledger_config.psql_command - if ledger_config is not None - else os.environ.get("PRISM_POSTGRES_PSQL_COMMAND", "") - ) - database_url = ( - ledger_config.database_url or "" - if ledger_config is not None - else os.environ.get("PRISM_DATABASE_URL", "") - ) - if not psql_command and database_url: - psql_command = f"psql {shlex.quote(database_url)}" - if not psql_command: - allow_memory_ledger = ( - ledger_config.allow_memory_ledger - if ledger_config is not None - else env_bool("PRISM_ALLOW_MEMORY_LEDGER", "0") - ) - if not allow_memory_ledger: - raise SystemExit( - "PRISM_DATABASE_URL or PRISM_POSTGRES_PSQL_COMMAND is required; " - "set PRISM_ALLOW_MEMORY_LEDGER=1 only for local tests" - ) - return SingleWriterShareLedger( - ctv_broadcast_attempt_detail_limit=getattr( - self, - "ctv_broadcast_attempt_detail_limit", - DEFAULT_CTV_BROADCAST_ATTEMPT_DETAIL_LIMIT, - ), - ctv_broadcast_retry_backoff_seconds=getattr( - self, - "ctv_broadcast_retry_backoff_seconds", - DEFAULT_CTV_BROADCAST_RETRY_BACKOFF_SECONDS, - ), - ) - writer_session_token = ( - ledger_config.writer_session_token - if ledger_config is not None - else env_optional("PRISM_LEDGER_WRITER_SESSION_TOKEN") - ) - if ( - ledger_config is None - and writer_session_token is not None - and not env_bool("PRISM_ALLOW_FIXED_LEDGER_SESSION_TOKEN", "0") - ): - raise SystemExit( - "PRISM_LEDGER_WRITER_SESSION_TOKEN requires " - "PRISM_ALLOW_FIXED_LEDGER_SESSION_TOKEN=1 for local tests" - ) - if writer_session_token is None: - writer_session_token = ( - f"{WRITER_LEASE_HEARTBEAT_SESSION_PREFIX}{uuid.uuid4().hex}" - ) - audit_store = self._ensure_audit_artifact_store() - ledger = self._construct_psql_share_ledger( - psql_command, - database_url, - ledger_config, - writer_session_token, - audit_store, - lease_retry_sleep=lease_retry_sleep, - ) - try: - # Issue #255 rollback floor: refuse a database whose candidate - # storage is newer than this process, or one without the 002 - # migration (every storage version needs it), before any share - # or candidate is written. - ledger.verify_candidate_schema() - except IncompatibleCandidateSchema as exc: - ledger.close() - raise SystemExit(f"prism coordinator: {exc}") from exc - except Exception as exc: # noqa: BLE001 - a transport failure is not a verdict - print( - "prism coordinator: candidate schema capability check could not " - f"run at boot ({type(exc).__name__}: {exc}); a version-2 candidate " - "write against an unmigrated database fails closed on its own", - flush=True, - ) - return ledger - - def _construct_psql_share_ledger( - self, - psql_command: str, - database_url: str, - ledger_config: Any, - writer_session_token: str | None, - audit_store: Any, - *, - lease_retry_sleep: Callable[[float], None] | None = None, - ) -> PsqlShareLedger: - config = getattr(self, "config", None) - return PsqlShareLedger( - psql_command=psql_command, - lease_retry_sleep=lease_retry_sleep, - database_url=database_url or None, - native_client_mode=( - ledger_config.native_client_mode - if ledger_config is not None - else env("PRISM_POSTGRES_NATIVE_CLIENT", "auto") - ), - writer_id=( - ledger_config.writer_id - if ledger_config is not None - else env("PRISM_LEDGER_WRITER_ID", "prism-coordinator") - ), - writer_epoch=( - ledger_config.writer_epoch - if ledger_config is not None - else env_int("PRISM_LEDGER_WRITER_EPOCH", 1) - ), - writer_session_token=writer_session_token, - initialize_schema=( - ledger_config.initialize_schema - if ledger_config is not None - else env("PRISM_POSTGRES_INIT_SCHEMA", "0") in {"1", "true", "yes"} - ), - lease_ttl_seconds=( - ledger_config.lease_ttl_seconds - if ledger_config is not None - else env_positive_float("PRISM_LEDGER_LEASE_TTL_SECONDS", 60.0) - ), - lease_acquire_lock_timeout_seconds=( - ledger_config.lease_acquire_lock_timeout_seconds - if ledger_config is not None - else env_positive_float( - "PRISM_LEDGER_LEASE_ACQUIRE_LOCK_TIMEOUT_SECONDS", - DEFAULT_LEASE_ACQUIRE_LOCK_TIMEOUT_SECONDS, - ) - ), - lease_acquire_attempts=( - ledger_config.lease_acquire_attempts - if ledger_config is not None - else env_positive_int( - "PRISM_LEDGER_LEASE_ACQUIRE_ATTEMPTS", - DEFAULT_LEASE_ACQUIRE_ATTEMPTS, - ) - ), - postgres_idle_in_transaction_timeout_seconds=( - ledger_config.postgres_idle_in_transaction_timeout_seconds - if ledger_config is not None - else env_positive_float( - "PRISM_POSTGRES_IDLE_IN_TRANSACTION_TIMEOUT_SECONDS", - DEFAULT_POSTGRES_IDLE_IN_TRANSACTION_TIMEOUT_SECONDS, - ) - ), - postgres_tcp_keepalives_idle_seconds=( - ledger_config.postgres_tcp_keepalives_idle_seconds - if ledger_config is not None - else env_positive_int( - "PRISM_POSTGRES_TCP_KEEPALIVES_IDLE_SECONDS", - DEFAULT_POSTGRES_TCP_KEEPALIVES_IDLE_SECONDS, - ) - ), - postgres_tcp_keepalives_interval_seconds=( - ledger_config.postgres_tcp_keepalives_interval_seconds - if ledger_config is not None - else env_positive_int( - "PRISM_POSTGRES_TCP_KEEPALIVES_INTERVAL_SECONDS", - DEFAULT_POSTGRES_TCP_KEEPALIVES_INTERVAL_SECONDS, - ) - ), - postgres_tcp_keepalives_count=( - ledger_config.postgres_tcp_keepalives_count - if ledger_config is not None - else env_positive_int( - "PRISM_POSTGRES_TCP_KEEPALIVES_COUNT", - DEFAULT_POSTGRES_TCP_KEEPALIVES_COUNT, - ) - ), - # The own-write deferral margin must cover the longest RPC the - # lease fence can authorize — submitblock's dedicated deadline - # and the CTV broadcast RPCs riding JsonRpc.call's default — - # plus fixed headroom for fence-to-RPC scheduling and clock - # drift (see LEASE_AUTHORITY_MARGIN_HEADROOM_SECONDS; the RPCs - # themselves cannot outlive their wall-clock deadlines). The - # ledger floors this at half the TTL, so short deadlines keep - # today's behavior and only an operator raising a guarded - # deadline widens the deferral window with it. - lease_authority_margin_seconds=( - LEASE_AUTHORITY_MARGIN_HEADROOM_SECONDS - + max( - float( - config.block.submit_rpc_timeout_seconds - if config is not None - else getattr( - self, - "block_submit_rpc_timeout_seconds", - DEFAULT_BLOCK_SUBMIT_RPC_TIMEOUT_SECONDS, - ) - ), - DEFAULT_QBIT_RPC_CALL_TIMEOUT_SECONDS, - ) - ), - read_concurrency=( - ledger_config.read_concurrency - if ledger_config is not None - else env_positive_int("PRISM_POSTGRES_READ_CONCURRENCY", 4) - ), - candidate_storage_version=( - ledger_config.candidate_storage_version - if ledger_config is not None - else env_positive_int("PRISM_CANDIDATE_STORAGE_VERSION", 2) - ), - candidate_spool_dir=( - ledger_config.candidate_spool_dir - if ledger_config is not None - else (env_optional("PRISM_CANDIDATE_SPOOL_DIR") or None) - ), - candidate_spool_reservation_bytes=( - ledger_config.candidate_spool_reservation_bytes - if ledger_config is not None - else env_positive_int( - "PRISM_CANDIDATE_SPOOL_RESERVATION_BYTES", - 4 * 1024 * 1024 * 1024, - ) - ), - accepted_stats_cache_seconds=( - ledger_config.accepted_stats_cache_seconds - if ledger_config is not None - else env_nonnegative_float("PRISM_ACCEPTED_STATS_CACHE_SECONDS", 60.0) - ), - reward_window_cache_seconds=( - ledger_config.reward_window_cache_seconds - if ledger_config is not None - else env_nonnegative_float( - "PRISM_PUBLIC_REWARD_WINDOW_CACHE_SECONDS", 30.0 - ) - ), - audit_artifact_store=audit_store, - ctv_broadcast_attempt_detail_limit=getattr( - self, - "ctv_broadcast_attempt_detail_limit", - DEFAULT_CTV_BROADCAST_ATTEMPT_DETAIL_LIMIT, - ), - ctv_broadcast_retry_backoff_seconds=getattr( - self, - "ctv_broadcast_retry_backoff_seconds", - DEFAULT_CTV_BROADCAST_RETRY_BACKOFF_SECONDS, - ), - ) - - def load_trusted_ledger_writer_public_key(self) -> str | None: - config = getattr(self, "config", None) - if config is not None: - return config.ledger.writer_public_key_hex - configured = env_optional("PRISM_LEDGER_WRITER_PUBLIC_KEY_HEX") - if configured is not None: - return validate_hex(configured, name="PRISM_LEDGER_WRITER_PUBLIC_KEY_HEX", expected_bytes=32) - if env_bool("PRISM_ALLOW_BUNDLE_EMBEDDED_LEDGER_KEY", "0"): - return None - raise SystemExit( - "PRISM_LEDGER_WRITER_PUBLIC_KEY_HEX is required; " - "set PRISM_ALLOW_BUNDLE_EMBEDDED_LEDGER_KEY=1 only for local tests" - ) - - def prism_payout_policy(self) -> dict[str, object]: - cached = getattr(self, "_prism_payout_policy_cache", None) - if cached is not None: - return cached - - policy = default_prism_payout_policy() - output_policy = env_optional("PRISM_COINBASE_OUTPUT_POLICY") or "canonical" - if output_policy not in VALID_COINBASE_OUTPUT_POLICIES: - raise SystemExit( - "PRISM_COINBASE_OUTPUT_POLICY must be one of: " - + ", ".join(sorted(VALID_COINBASE_OUTPUT_POLICIES)) - ) - fee_bps_raw = env_optional("PRISM_POOL_FEE_BPS") - fee_enabled = env_bool("PRISM_POOL_FEE_ENABLED", "0") - fee_address = env_optional("PRISM_POOL_FEE_ADDRESS") - fee_program_hex = env_optional("PRISM_POOL_FEE_P2MR_PROGRAM_HEX") - fee_recipient_id = env_optional("PRISM_POOL_FEE_RECIPIENT_ID") - fee_order_key = env_optional("PRISM_POOL_FEE_ORDER_KEY") - has_fee_config = any( - value is not None - for value in (fee_bps_raw, fee_address, fee_program_hex, fee_recipient_id, fee_order_key) - ) - if not fee_enabled: - if has_fee_config: - raise SystemExit("set PRISM_POOL_FEE_ENABLED=1 when configuring pool fees") - if output_policy == "pool-fee-first": - raise SystemExit( - "PRISM_COINBASE_OUTPUT_POLICY=pool-fee-first requires PRISM_POOL_FEE_ENABLED=1" - " and a configured pool fee" - ) - self._prism_payout_policy_cache = policy - return policy - if fee_bps_raw is None: - raise SystemExit("PRISM_POOL_FEE_BPS is required when pool fees are enabled") - - try: - fee_bps = int(fee_bps_raw) - except ValueError as exc: - raise SystemExit("PRISM_POOL_FEE_BPS must be an integer") from exc - if fee_bps < 0 or fee_bps > 10_000: - raise SystemExit("PRISM_POOL_FEE_BPS must be between 0 and 10000") - if (fee_address is None) == (fee_program_hex is None): - raise SystemExit("set exactly one of PRISM_POOL_FEE_ADDRESS or PRISM_POOL_FEE_P2MR_PROGRAM_HEX") - - if fee_address is not None: - validation = self.rpc.call("validateaddress", [fee_address]) - if not isinstance(validation, dict) or not validation.get("isvalid"): - raise SystemExit(f"PRISM_POOL_FEE_ADDRESS is not a valid qbit address: {fee_address}") - script = str(validation.get("scriptPubKey") or "") - if not script.startswith("5220") or len(script) != 68: - raise SystemExit("PRISM_POOL_FEE_ADDRESS must resolve to a P2MR script") - fee_policy = { - "fee_bps": fee_bps, - "recipient_id": fee_address, - "order_key": fee_address, - "p2mr_program_hex": script[4:], - } - else: - program_hex = validate_hex( - fee_program_hex or "", - name="PRISM_POOL_FEE_P2MR_PROGRAM_HEX", - expected_bytes=32, - ) - if fee_recipient_id is None: - raise SystemExit("PRISM_POOL_FEE_RECIPIENT_ID is required with PRISM_POOL_FEE_P2MR_PROGRAM_HEX") - fee_policy = { - "fee_bps": fee_bps, - "recipient_id": fee_recipient_id, - "order_key": fee_order_key or fee_recipient_id, - "p2mr_program_hex": program_hex, - } - - policy["pool_fee_policy"] = fee_policy - if output_policy != "canonical": - # Omitted for canonical so canonical payloads and signed audit - # artifacts keep their historical bytes. - policy["coinbase_output_policy"] = output_policy - self._prism_payout_policy_cache = policy - return policy - - def prism_ctv_settlement_config( - self, - *, - block_height: int | None = None, - parent_hash: str | None = None, - ) -> dict[str, object] | None: - if not env_bool("PRISM_CTV_SETTLEMENT_ENABLED", "0"): - return None - direct_floor_sats = env_positive_int_with_legacy( - "PRISM_DIRECT_COINBASE_PAYOUT_FLOOR_BITS", - "PRISM_DIRECT_COINBASE_PAYOUT_FLOOR_SATS", - DEFAULT_DIRECT_COINBASE_PAYOUT_FLOOR_SATS, - ) - reserved_coinbase_outputs = env_int("PRISM_RESERVED_COINBASE_OUTPUTS", 0) - if reserved_coinbase_outputs < 0: - raise SystemExit("PRISM_RESERVED_COINBASE_OUTPUTS must be non-negative") - config: dict[str, object] = { - "direct_floor_sats": direct_floor_sats, - "config": { - "max_coinbase_settlement_outputs": env_positive_int( - "PRISM_MAX_COINBASE_SETTLEMENT_OUTPUTS", - DEFAULT_MAX_COINBASE_SETTLEMENT_OUTPUTS, - ), - "max_direct_coinbase_outputs": env_positive_int( - "PRISM_MAX_DIRECT_COINBASE_OUTPUTS", - DEFAULT_MAX_DIRECT_COINBASE_OUTPUTS, - ), - "max_fanout_recipients_per_transaction": env_positive_int( - "PRISM_MAX_CTV_FANOUT_RECIPIENTS_PER_TRANSACTION", - DEFAULT_MAX_CTV_FANOUT_RECIPIENTS_PER_TRANSACTION, - ), - "reserved_coinbase_outputs": reserved_coinbase_outputs, - }, - } - config["fanout_fee_rate_policy"] = { - "market_fee_rate_sats_per_1000_weight": self.ctv_fanout_market_fee_rate_bits_per_1000_weight( - block_height=block_height, - parent_hash=parent_hash, - ), - "premium_bps": env_positive_int( - "PRISM_CTV_FANOUT_FEE_PREMIUM_BPS", - DEFAULT_CTV_FANOUT_FEE_PREMIUM_BPS, - ), - } - return config - - def ctv_fanout_market_fee_rate_bits_per_1000_weight( - self, - *, - block_height: int | None = None, - parent_hash: str | None = None, - ) -> int: - configured_rate = env_optional_positive_int_with_legacy( - "PRISM_CTV_FANOUT_FEE_MARKET_RATE_BITS_PER_1000_WEIGHT", - "PRISM_CTV_FANOUT_FEE_MARKET_RATE_SATS_PER_1000_WEIGHT", - ) - if configured_rate is not None: - return configured_rate - fee_rate_cache = getattr(self, "_ctv_fanout_market_fee_rate_cache", None) - if fee_rate_cache is None: - fee_rate_cache = {} - self._ctv_fanout_market_fee_rate_cache = fee_rate_cache - cache_key = (block_height, parent_hash) - if cache_key in fee_rate_cache: - return fee_rate_cache[cache_key] - try: - estimate = self.rpc.call( - "estimatesmartfee", - [env_positive_int("PRISM_CTV_FANOUT_FEE_ESTIMATE_TARGET_BLOCKS", 2)], - ) - if not isinstance(estimate, dict): - raise RuntimeError("estimatesmartfee returned non-object") - errors = estimate.get("errors") - if errors: - raise RuntimeError(f"estimatesmartfee returned errors: {errors}") - feerate = Decimal(str(estimate.get("feerate", ""))) - if not feerate.is_finite() or feerate <= 0: - raise RuntimeError(f"estimatesmartfee returned invalid feerate: {estimate.get('feerate')!r}") - rate = int((feerate * Decimal(100_000_000)).to_integral_value(rounding=ROUND_CEILING)) - if rate <= 0: - raise RuntimeError("estimatesmartfee rounded to a non-positive rate") - except Exception as exc: - raise RuntimeError( - "unable to compute PRISM CTV fanout fee rate; set " - "PRISM_CTV_FANOUT_FEE_MARKET_RATE_BITS_PER_1000_WEIGHT or fix " - f"estimatesmartfee: {exc}" - ) from exc - fee_rate_cache[cache_key] = rate - return rate - - def _ensure_job_bundle_service(self) -> JobBundleService: - service = self.__dict__.get("job_bundle_service") - if service is not None: - return service - init_lock = self.__dict__.get("_job_bundle_service_init_lock") - if init_lock is None: - # CPython's setdefault is atomic under the GIL. Focused tests may - # construct through __new__, while normal instances build the - # service on the first legacy-field touch during __init__. - init_lock = self.__dict__.setdefault( - "_job_bundle_service_init_lock", - threading.Lock(), - ) - with init_lock: - service = self.__dict__.get("job_bundle_service") - if service is not None: - return service - service = JobBundleService( - self, - # Resolve the spool-file factory through this module's global - # at call time so the historical - # ``prism_coordinator._share_window_spool_file`` patch seam - # keeps steering spool creation. - spool_factory=lambda: _share_window_spool_file(), - payout_ledger_artifact_type=PayoutLedgerArtifact, - now_ms=lambda: now_ms(), - canonical_json_sha256_override=( - lambda value: canonical_json_sha256(value) - ), - ) - self.__dict__["job_bundle_service"] = service - return service - - def _ensure_bundle_compiler(self) -> BundleCompiler: - compiler = self.__dict__.get("bundle_compiler") - if compiler is not None: - return compiler - init_lock = self.__dict__.get("_bundle_compiler_init_lock") - if init_lock is None: - init_lock = self.__dict__.setdefault( - "_bundle_compiler_init_lock", - threading.Lock(), - ) - with init_lock: - compiler = self.__dict__.get("bundle_compiler") - if compiler is not None: - return compiler - compiler = BundleCompiler( - self, - superseded_error=_JobBundleBuildSuperseded, - cancellation_error_types=( - JobBuildCancelled, - _JobBundleBuildSuperseded, - ), - build_control_type=_JobBundleBuildControl, - # Resolve the builder command through this module's global at - # call time so the historical patch seam keeps working. - tool_command=lambda name: prism_tool_command(name), - ) - self.__dict__["bundle_compiler"] = compiler - return compiler - - def _ensure_tip_refresh_service(self) -> TipRefreshService: - service = self.__dict__.get("tip_refresh_service") - if service is not None: - return service - init_lock = self.__dict__.get("_tip_refresh_service_init_lock") - if init_lock is None: - # CPython's setdefault is atomic under the GIL. Focused tests may - # construct through __new__, while normal instances build the - # service on the first legacy-field touch during __init__. - init_lock = self.__dict__.setdefault( - "_tip_refresh_service_init_lock", - threading.Lock(), - ) - with init_lock: - service = self.__dict__.get("tip_refresh_service") - if service is not None: - return service - service = TipRefreshService( - self, - # Coordinator-owned exception types and the S2 delivery - # priorities are injected so the R1 owner never imports this - # module (or job_delivery, which sits above it). - shutdown_error=ShutdownInProgress, - job_build_failed_error=_JobBuildFailed, - delivery_priority_initial=PRISM_DELIVERY_PRIORITY_INITIAL, - delivery_priority_new_tip=PRISM_DELIVERY_PRIORITY_NEW_TIP, - delivery_priority_same_tip=PRISM_DELIVERY_PRIORITY_SAME_TIP, - ) - self.__dict__["tip_refresh_service"] = service - return service - - def _ensure_job_delivery_service(self) -> JobDeliveryService: - service = self.__dict__.get("job_delivery_service") - if service is not None: - return service - init_lock = self.__dict__.get("_job_delivery_service_init_lock") - if init_lock is None: - init_lock = self.__dict__.setdefault( - "_job_delivery_service_init_lock", - threading.Lock(), - ) - with init_lock: - service = self.__dict__.get("job_delivery_service") - if service is not None: - return service - service = JobDeliveryService( - self, - # The Stratum error type is owned by the session module; it - # is injected so the S2 owner never imports the session owner - # (or this module). - stratum_error=StratumError, - ) - self.__dict__["job_delivery_service"] = service - return service - - def _ensure_payout_state_service(self) -> PayoutStateService: - service = self.__dict__.get("payout_state_service") - if service is not None: - return service - init_lock = self.__dict__.get("_payout_state_service_init_lock") - if init_lock is None: - init_lock = self.__dict__.setdefault( - "_payout_state_service_init_lock", - threading.Lock(), - ) - with init_lock: - service = self.__dict__.get("payout_state_service") - if service is not None: - return service - service = PayoutStateService( - self, - # The shutdown exception type stays coordinator-owned until - # the shutdown controller is extracted; the wall-clock stamp - # and canonical serialization resolve this module's globals - # at call time so existing monkeypatches keep intercepting. - shutdown_error=ShutdownInProgress, - now_ms=lambda: now_ms(), - canonical_json_text_override=( - lambda value: canonical_json_text(value) - ), - canonical_json_sha256_override=( - lambda value: canonical_json_sha256(value) - ), - ) - self.__dict__["payout_state_service"] = service - return service - - def _ensure_share_writer_service(self) -> ShareWriter: - service = self.__dict__.get("share_writer_service") - if service is not None: - return service - init_lock = self.__dict__.get("_share_writer_service_init_lock") - if init_lock is None: - init_lock = self.__dict__.setdefault( - "_share_writer_service_init_lock", - threading.Lock(), - ) - with init_lock: - service = self.__dict__.get("share_writer_service") - if service is not None: - return service - service = ShareWriter( - self, - # The rejection-reason registry stays coordinator-owned; the - # wall-clock stamp resolves this module's now_ms global at - # call time so existing monkeypatches keep intercepting. - internal_error_reason=PRISM_REJECTION_INTERNAL_ERROR, - now_ms=lambda: now_ms(), - ) - self.__dict__["share_writer_service"] = service - return service - - def _ensure_stratum_session_service(self) -> StratumSessionService: - service = self.__dict__.get("stratum_session_service") - if service is not None: - return service - init_lock = self.__dict__.get("_stratum_session_service_init_lock") - if init_lock is None: - init_lock = self.__dict__.setdefault( - "_stratum_session_service_init_lock", - threading.Lock(), - ) - with init_lock: - service = self.__dict__.get("stratum_session_service") - if service is not None: - return service - service = StratumSessionService( - self, - # The shutdown exception type stays coordinator-owned until - # the shutdown controller is extracted; inject it so the S1 - # owner never imports this module. - shutdown_error=ShutdownInProgress, - pool_closed_reason=PRISM_REJECTION_POOL_CLOSED, - ) - self.__dict__["stratum_session_service"] = service - return service - - def _ensure_reorg_reconciler_service(self) -> ReorgReconcilerService: - service = self.__dict__.get("_reorg_reconciler_service") - if service is not None: - return service - init_lock = self.__dict__.setdefault( - "_reorg_reconciler_service_init_lock", - threading.Lock(), - ) - with init_lock: - service = self.__dict__.get("_reorg_reconciler_service") - if service is not None: - return service - service = ReorgReconcilerService( - ReorgPorts( - rpc_call=lambda *args, **kwargs: self.rpc.call( - *args, - **kwargs, - ), - ledger=lambda: self.ledger, - ensure_job_cache_state=self._ensure_job_cache_state, - # The memo/counter state lock is the coordinator's - # control-plane lock (resolved live: focused tests - # replace it), preserving the atomicity tip observation - # relies on when it evicts memo entries under self.lock. - state_lock=lambda: self.lock, - source_tip=self._reorg_payout_source_tip, - reserve_external_tip=lambda tip: self._reserve_payout_state_source( - "external_tip", - tip_hash=tip, - ), - max_supersession_retries=lambda: getattr( - self, - "payout_reconcile_supersession_retries", - DEFAULT_PRISM_PAYOUT_RECONCILE_SUPERSESSION_RETRIES, - ), - # #113's heartbeat-aware acquisition of the payout-state - # preparation lock. - prepare_lock=lambda: self._block_submitter_lock( - self._payout_state_prepare_lock, - "payout-state-prepare", - ), - capture_source=lambda: self._capture_payout_state_source(), - prepared_candidate=lambda captured, **kwargs: ( - self._prepared_payout_state_candidate( - captured, - **kwargs, - ) - ), - captured_publication_required=lambda captured: ( - self._captured_payout_source_requires_publication( - captured - ) - ), - block_publication=lambda **kwargs: ( - self._block_payout_state_publication(**kwargs) - ), - publication_guard=lambda: ( - self._ensure_audit_artifact_store().publication_order_guard() - ), - publish_candidate=lambda candidate: ( - self._publish_payout_state_candidate(candidate) - ), - observe_preparation=lambda elapsed: ( - self._observe_payout_state_seconds( - "preparation", - elapsed, - ) - ), - chain_view_untrusted=lambda: self.qbit_chain_view_untrusted(), - reorg_proof_snapshot=lambda: ( - self._ensure_tip_refresh_service().reorg_proof_snapshot() - ), - flight_wait_seconds=lambda: float( - getattr( - self, - "reconcile_flight_wait_seconds", - DEFAULT_PRISM_RECONCILE_FLIGHT_WAIT_SECONDS, - ) - ), - prefetch_join_timeout_seconds=lambda: getattr( - self, - "reconcile_prefetch_join_timeout_seconds", - PRISM_RECONCILE_PREFETCH_JOIN_TIMEOUT_SECONDS, - ), - reconcile_with_admission=lambda **kwargs: ( - self.reconcile_prism_pool_blocks_once(**kwargs) - ), - reconcile_serialized=lambda **kwargs: ( - self._reconcile_prism_pool_blocks_serialized(**kwargs) - ), - ensure_tip=lambda tip: self.ensure_reorg_reconciled_for_tip( - tip - ), - # Accepted-block landings run a reconcile pass inline on - # the watchdog-monitored block-work thread. The recorder - # stamps only from that thread's registered owner, so - # background reconciliation passes and per-client callers - # keep recording nothing at all. - record_progress=lambda phase: ( - self._record_block_submitter_phase(phase) - ), - # Issue #224: the one attribution owner every lane and - # the renderer share; resolved live so it is the same - # object the metrics port reads. - accepted_preview_telemetry=lambda: ( - ensure_accepted_preview_telemetry(self) - ), - ), - enabled=bool(self.reorg_reconciler_enabled), - cache_seconds=self.reorg_reconcile_cache_seconds, - inactive_block_count=int(self.reorg_inactive_block_count), - reactivated_block_count=int(self.reorg_reactivated_block_count), - reconcile_skip_count=int(self.reorg_reconcile_skip_count), - reconcile_error_count=int(self.reorg_reconcile_error_count), - matured_payout_count=int(self.matured_payout_count), - last_tip_hash=self.last_reorg_reconciled_tip_hash, - last_trusted=bool(self.last_reorg_reconciled_trusted), - last_monotonic=self.last_reorg_reconciled_monotonic, - ) - self.__dict__["_reorg_reconciler_service"] = service - for name in ( - "enabled", - "cache_seconds", - "inactive_block_count", - "reactivated_block_count", - "reconcile_skip_count", - "reconcile_error_count", - "matured_payout_count", - "last_tip_hash", - "last_trusted", - "last_monotonic", - ): - self.__dict__.pop(f"_reorg_compat_{name}", None) - return service - - def _reorg_payout_source_tip(self) -> str | None: - """Payout-state snapshot source tip under the control-plane lock.""" - with self.lock: - return self._payout_state_source[1] - - def _ensure_job_cache_state(self) -> None: - # J1 scheduler/cache/template state lives in its owner service; the - # legacy raw fields route there through class descriptors and the - # service adopts direct test assignments through those descriptors. - self._ensure_job_bundle_service() - if not hasattr(self, "job_build_timeout_seconds"): - self.job_build_timeout_seconds = DEFAULT_PRISM_JOB_BUILD_TIMEOUT_SECONDS - if not hasattr(self, "job_build_cancel_grace_seconds"): - self.job_build_cancel_grace_seconds = ( - DEFAULT_PRISM_JOB_BUILD_CANCEL_GRACE_SECONDS - ) - # P1 payout state lives in its owner service; the legacy raw - # fields route there through class descriptors and the service - # adopts direct test assignments through those descriptors (see - # lab/prism/payout_state.py). - self._ensure_payout_state_service() - # The S3 owner holds the pending-commit floor lock and dictionary - # (see lab/prism/share_writer.py); ensure it exists for anchor reads. - self._ensure_share_writer_service() - # The serve-builder daemon state lives in the compiler owner; the - # legacy raw fields route there through class descriptors. - self._ensure_bundle_compiler() - # Reorg flight/memo/lookup/prefetch state lives in its owner service - # (see lab/prism/reorg_reconciler.py); the service constructor is the - # single initializer for those containers. - if not hasattr(self, "_accounted_accepted_block_hashes"): - self._accounted_accepted_block_hashes: set[str] = set() - # The B1 owner (created on first touch) holds the block-submit - # histogram, acceptance-evidence containers, and abandonment dedupe - # set; the legacy raw fields route there through class descriptors. - self._ensure_block_candidate_service() - # Health snapshot and mining-delivery timer state lives in the - # observability owner; the legacy raw fields route there through the - # class compatibility properties. - self._ensure_observability_service() - # The aggregate progress-health state lives in its owner service; the - # legacy raw ``_progress_*`` fields route there through descriptors. - self._ensure_progress_health_service() - - def _ensure_observability_service(self) -> ObservabilityService: - service = self.__dict__.get("_observability_service") - if service is not None: - return service - init_lock = self.__dict__.setdefault( - "_observability_service_init_lock", - threading.Lock(), - ) - with init_lock: - service = self.__dict__.get("_observability_service") - if service is None: - service = ObservabilityService(_CoordinatorObservability(self)) - self.__dict__["_observability_service"] = service - return service - - def _ensure_audit_http_facade(self) -> AuditHttpFacade: - facade = self.__dict__.get("_audit_http_facade") - if facade is not None: - return facade - init_lock = self.__dict__.setdefault( - "_audit_http_facade_init_lock", - threading.Lock(), - ) - with init_lock: - facade = self.__dict__.get("_audit_http_facade") - if facade is None: - facade = AuditHttpFacade( - _CoordinatorAuditHttp(self), - AuditHttpConfig( - bind=str(getattr(self, "audit_bind", None) or "127.0.0.1"), - port=int(getattr(self, "audit_port", 0)), - ), - ) - self.__dict__["_audit_http_facade"] = facade - return facade - - def _job_build_phases(self) -> dict[str, float]: - """Per-thread scratch dict of phase timings for the current build.""" - return self._ensure_job_bundle_service()._job_build_phases() - - def _cancel_obsolete_job_bundle_builds( - self, - *, - current_tip: str | None = None, - payout_state_generation: int | None = None, - ) -> None: - """Cancel only builds proven obsolete by a newer exact generation.""" - self._ensure_job_bundle_service()._cancel_obsolete_job_bundle_builds( - current_tip=current_tip, - payout_state_generation=payout_state_generation, - ) - - def _register_job_bundle_process( - self, - control: _JobBundleBuildControl, - process: subprocess.Popen[str], - ) -> None: - self._ensure_job_bundle_service()._register_job_bundle_process(control, process) - - def _unregister_job_bundle_process( - self, - control: _JobBundleBuildControl, - process: Any, - ) -> None: - """Detach a shared daemon from a build control after its request.""" - self._ensure_job_bundle_service()._unregister_job_bundle_process(control, process) - - @staticmethod - def _incremental_window_records_supported(records: Sequence[object]) -> bool: - """Whether a ledger snapshot exposes the append-only window contract.""" - return PayoutStateService._incremental_window_records_supported(records) - - def _full_payout_window_materialization(self, *, snapshot_anchor_ms: int, snapshot_window_weight: int, reason: str, observed_monotonic: float, append_invalidation_epoch: int, bypass_build_interval: bool=False) -> _PayoutWindowMaterialization: - """Run the exact ledger oracle and atomically replace cached pages.""" - return self._ensure_payout_state_service()._full_payout_window_materialization(snapshot_anchor_ms=snapshot_anchor_ms, snapshot_window_weight=snapshot_window_weight, reason=reason, observed_monotonic=observed_monotonic, append_invalidation_epoch=append_invalidation_epoch, bypass_build_interval=bypass_build_interval) - - def _incremental_payout_window_materialization(self, *, snapshot_anchor_ms: int, snapshot_window_weight: int, force_full_rescan: bool, bypass_build_interval: bool, append_invalidation_epoch: int, reused_prior_balances_sha256: str | None=None) -> _PayoutWindowMaterialization: - """Return an exact window using debounce, delta folding, or the oracle.""" - return self._ensure_payout_state_service()._incremental_payout_window_materialization(snapshot_anchor_ms=snapshot_anchor_ms, snapshot_window_weight=snapshot_window_weight, force_full_rescan=force_full_rescan, bypass_build_interval=bypass_build_interval, append_invalidation_epoch=append_invalidation_epoch, reused_prior_balances_sha256=reused_prior_balances_sha256) - - def _build_payout_ledger_artifact(self, expected_payout_state_generation: int, artifact_payout_state_generation: int, network_difficulty: int, force_full_rescan: bool=False, bypass_build_interval: bool=False, during_publication: bool=False, *, force_prior_balances_read: bool=False) -> PayoutLedgerArtifact | None: - """Build a stable ledger snapshot without publishing it.""" - return self._ensure_payout_state_service()._build_payout_ledger_artifact(expected_payout_state_generation, artifact_payout_state_generation, network_difficulty, force_full_rescan, bypass_build_interval, during_publication, force_prior_balances_read=force_prior_balances_read) - - def _prepare_payout_ledger_artifact(self, payout_state_generation: int, network_difficulty: int, *, bypass_build_interval: bool=False) -> None: - """Prepare and atomically publish an artifact for a current generation.""" - return self._ensure_payout_state_service()._prepare_payout_ledger_artifact(payout_state_generation, network_difficulty, bypass_build_interval=bypass_build_interval) - - def _install_payout_ledger_artifact(self, artifact: PayoutLedgerArtifact) -> bool: - """Atomically publish a prepared artifact for its own generation.""" - return self._ensure_payout_state_service()._install_payout_ledger_artifact(artifact) - - def _install_payout_ledger_artifact_outcome(self, artifact: PayoutLedgerArtifact) -> bool: - return self._ensure_payout_state_service()._install_payout_ledger_artifact_outcome(artifact) - - def _install_payout_ledger_artifact_locked(self, artifact: PayoutLedgerArtifact) -> tuple[str, int | None]: - """Ordering core of the install; returns (outcome, armed generation).""" - return self._ensure_payout_state_service()._install_payout_ledger_artifact_locked(artifact) - - def _payout_artifact_preparation_loop(self) -> None: - return self._ensure_payout_state_service()._payout_artifact_preparation_loop() - - def _schedule_payout_ledger_artifact_preparation(self, payout_state_generation: int, network_difficulty: int, *, min_interval_seconds: float | None=None, bypass_build_interval: bool=False) -> None: - """Latest-generation-wins scheduling with one worker and one slot.""" - return self._ensure_payout_state_service()._schedule_payout_ledger_artifact_preparation(payout_state_generation, network_difficulty, min_interval_seconds=min_interval_seconds, bypass_build_interval=bypass_build_interval) - - def _payout_artifact_max_anchor_age_ms(self) -> float: - """Audit ceiling on a served window's wall-clock anchor age.""" - return self._ensure_payout_state_service()._payout_artifact_max_anchor_age_ms() - - def _payout_artifact_reanchor_seconds(self) -> float: - """Anchor age that triggers a background re-anchor while reuse keeps serving.""" - return self._ensure_payout_state_service()._payout_artifact_reanchor_seconds() - - def _payout_artifact_min_build_interval_seconds(self) -> float: - """Minimum normal cadence; found-block settlement bypasses it.""" - return self._ensure_payout_state_service()._payout_artifact_min_build_interval_seconds() - - def _payout_artifact_reuse_active(self) -> bool: - """Master kill-switch (PRISM_PAYOUT_ARTIFACT_REUSE) for the reuse line.""" - return self._ensure_payout_state_service()._payout_artifact_reuse_active() - - @staticmethod - def _payout_artifact_log(event: str, **fields: object) -> None: - """Single-line JSON lifecycle log; the incident was invisible here.""" - return PayoutStateService._payout_artifact_log(event, **fields) - - def _record_payout_artifact_event(self, event: str) -> None: - return self._ensure_payout_state_service()._record_payout_artifact_event(event) - - def _note_window_mirror_divergence(self) -> None: - """Invalidate a refuted daemon window mirror and count the event.""" - return self._ensure_payout_state_service()._note_window_mirror_divergence() - - def _usable_payout_ledger_artifact(self, payout_state_generation: int, network_difficulty: int, *, rearm_on_fence_failure: bool=True) -> PayoutLedgerArtifact | None: - """Return the armed artifact when reuse is valid for NEW work.""" - return self._ensure_payout_state_service()._usable_payout_ledger_artifact(payout_state_generation, network_difficulty, rearm_on_fence_failure=rearm_on_fence_failure) - - def _rearm_payout_ledger_artifact_after_fence_failure(self, payout_state_generation: int, network_difficulty: int) -> None: - """Debounced rebuild scheduling for an artifact past the re-anchor floor or ceiling.""" - return self._ensure_payout_state_service()._rearm_payout_ledger_artifact_after_fence_failure(payout_state_generation, network_difficulty) - - def _schedule_current_payout_ledger_artifact_if_missing(self) -> None: - """Resume speculative preparation after a durable preview catches up.""" - return self._ensure_payout_state_service()._schedule_current_payout_ledger_artifact_if_missing() - - def shutdown_payout_artifact_executor(self) -> None: - return self._ensure_payout_state_service().shutdown_payout_artifact_executor() - - def _job_build_checkpoint( - self, - phase: str, - cancellation: _JobBuildCancellation, - ) -> None: - """Cooperative boundary around coordinator and isolated-worker phases.""" - - cancellation.raise_if_cancelled(phase) - - def _record_job_cache_event(self, kind: str, *, hit: bool) -> None: - self._ensure_job_bundle_service()._record_job_cache_event(kind, hit=hit) - - def _prepare_payout_state_artifact(self, *, generation: int, source_generation: int, cancellation: _JobBuildCancellation | None=None) -> PayoutStateArtifact: - """Snapshot carry-forward balances once for a payout generation.""" - return self._ensure_payout_state_service()._prepare_payout_state_artifact(generation=generation, source_generation=source_generation, cancellation=cancellation) - - def _payout_state_artifact_from_balances(self, *, generation: int, source_generation: int, balances: list[dict[str, object]]) -> PayoutStateArtifact: - return self._ensure_payout_state_service()._payout_state_artifact_from_balances(generation=generation, source_generation=source_generation, balances=balances) - - def _publish_self_check_repaired_balances(self, expected_payout_state_generation: int, *, stale_prior_balances_sha256: str, balances: Sequence[dict[str, object]]) -> bool: - """Replace the published balance snapshot the periodic oracle refuted.""" - return self._ensure_payout_state_service()._publish_self_check_repaired_balances(expected_payout_state_generation, stale_prior_balances_sha256=stale_prior_balances_sha256, balances=balances) - - def _current_payout_state_artifact(self, cancellation: _JobBuildCancellation | None=None) -> PayoutStateArtifact: - """Return the immutable artifact published for the current generation.""" - return self._ensure_payout_state_service()._current_payout_state_artifact(cancellation) - - def _job_build_executor_locked(self) -> ThreadPoolExecutor: - return self._ensure_job_bundle_service()._job_build_executor_locked() - - def _start_job_build_locked(self, request: _JobBuildRequest) -> _JobBuildFlight: - return self._ensure_job_bundle_service()._start_job_build_locked(request) - - def _arm_job_build_locked(self, flight: _JobBuildFlight) -> None: - self._ensure_job_bundle_service()._arm_job_build_locked(flight) - - def _execute_job_build_request( - self, - request: _JobBuildRequest, - ) -> CachedJobBundle: - return self._ensure_job_bundle_service()._execute_job_build_request(request) - - _collection_job_builds_are_independent = staticmethod( - JobBundleService._collection_job_builds_are_independent - ) - - _job_build_requests_can_share = staticmethod( - JobBundleService._job_build_requests_can_share - ) - - _ready_job_build_precedes_collection = staticmethod( - JobBundleService._ready_job_build_precedes_collection - ) - - _defer_job_build_locked = staticmethod( - JobBundleService._defer_job_build_locked - ) - - _job_build_is_publication_critical = staticmethod( - JobBundleService._job_build_is_publication_critical - ) - - _job_build_promise_done = staticmethod( - JobBundleService._job_build_promise_done - ) - - def _record_priority_admission_locked( - self, - request: _JobBuildRequest, - result: str, - ) -> None: - """Observe first builder admission from publication-priority reservation.""" - self._ensure_job_bundle_service()._record_priority_admission_locked(request, result) - - def _record_initial_prepared_work_locked(self, result: str) -> None: - self._ensure_job_bundle_service()._record_initial_prepared_work_locked(result) - - def _new_job_build_cancellation(self) -> _JobBuildCancellation: - return self._ensure_job_bundle_service()._new_job_build_cancellation() - - def _begin_job_build_priority_preparation( - self, - requested_monotonic: float | None = None, - ) -> tuple[int, float]: - """Reserve publication priority before immutable request construction.""" - return self._ensure_job_bundle_service()._begin_job_build_priority_preparation(requested_monotonic) - - def _finish_job_build_priority_preparation(self, token: int) -> None: - self._ensure_job_bundle_service()._finish_job_build_priority_preparation(token) - - def _begin_routine_job_build_preparation( - self, - *, - request_source: str, - cancelled: Callable[[], bool] | None, - ) -> tuple[int, _JobBuildCancellation, CachedJobBundle | None]: - """Atomically admit cancellable routine request construction.""" - return self._ensure_job_bundle_service()._begin_routine_job_build_preparation( - request_source=request_source, - cancelled=cancelled, - ) - - def _publication_priority_promises_locked( - self, - ) -> tuple[Future[CachedJobBundle], ...]: - """Promises of live publication-critical build requests.""" - return self._ensure_job_bundle_service()._publication_priority_promises_locked() - - def _finish_routine_job_build_preparation(self, token: int) -> None: - self._ensure_job_bundle_service()._finish_routine_job_build_preparation(token) - - def _publication_priority_scheduled_locked(self) -> bool: - return self._ensure_job_bundle_service()._publication_priority_scheduled_locked() - - def _job_build_can_inherit_publication_priority( - self, - existing: _JobBuildRequest, - incoming: _JobBuildRequest, - ) -> bool: - """Reject an almost-expired or stalled routine flight as a critical owner.""" - return self._ensure_job_bundle_service()._job_build_can_inherit_publication_priority( - existing, - incoming, - ) - - _resolve_cancelled_job_build_promise = staticmethod( - JobBundleService._resolve_cancelled_job_build_promise - ) - - _job_build_flight_outcome = staticmethod( - JobBundleService._job_build_flight_outcome - ) - - def _evict_orphaned_job_build_flights_locked(self) -> list[str]: - """Evict finished flights whose completion never released their slot.""" - return self._ensure_job_bundle_service()._evict_orphaned_job_build_flights_locked() - - def _cancel_job_build_flight_locked( - self, - flight: _JobBuildFlight, - reason: str, - *, - now: float | None = None, - ) -> bool: - return self._ensure_job_bundle_service()._cancel_job_build_flight_locked(flight, reason, now=now) - - def _promote_pending_job_build_locked(self) -> None: - self._ensure_job_bundle_service()._promote_pending_job_build_locked() - - def _job_build_done( - self, - flight: _JobBuildFlight, - future: Future[CachedJobBundle], - ) -> None: - self._ensure_job_bundle_service()._job_build_done(flight, future) - - def _request_job_build(self, request: _JobBuildRequest) -> Future[CachedJobBundle]: - return self._ensure_job_bundle_service()._request_job_build(request) - - def _cancel_obsolete_job_builds( - self, - reason: str, - *, - keep_published_snapshot: bool = False, - ) -> None: - self._ensure_job_bundle_service()._cancel_obsolete_job_builds( - reason, - keep_published_snapshot=keep_published_snapshot, - ) - - def shutdown_job_build_executor(self) -> None: - self._ensure_job_bundle_service().shutdown_job_build_executor() - - def _job_bundle_payout_state_current(self, bundle: CachedJobBundle) -> bool: - return self._ensure_job_bundle_service()._job_bundle_payout_state_current(bundle) - - def _payout_balance_mutation(self) -> Iterator[None]: - """Serialize durable balance changes without excluding delivery.""" - return self._ensure_payout_state_service()._payout_balance_mutation() - - def _begin_accepted_block_payout_preview(self, block_hash: str, *, block_height: int | None=None) -> None: - """Prevent child work from snapshotting pre-accept balances.""" - return self._ensure_payout_state_service()._begin_accepted_block_payout_preview(block_hash, block_height=block_height) - - def _mark_accepted_block_payout_landed(self, block_hash: str, *, block_height: int) -> None: - """Bar reconciliation after submitblock makes a candidate active.""" - return self._ensure_payout_state_service()._mark_accepted_block_payout_landed(block_hash, block_height=block_height) - - def _unmark_accepted_block_payout_landed(self, block_hash: str) -> None: - """Withdraw a landed bar armed for an attempt that never reached RPC.""" - return self._ensure_payout_state_service()._unmark_accepted_block_payout_landed(block_hash) - - def _publish_accepted_block_payout_preview(self, block_hash: str, balances: list[dict[str, object]]) -> list[dict[str, object]]: - """Publish the balances child work must observe after confirmation.""" - return self._ensure_payout_state_service()._publish_accepted_block_payout_preview(block_hash, balances) - - def _accepted_block_preview_candidate(self, candidate: PayoutStateCandidate, *, block_hash: str, preview: tuple[tuple[str, str, str, int], ...]) -> PayoutStateCandidate: - """Bind a compact preview to its prepared artifact before gating.""" - return self._ensure_payout_state_service()._accepted_block_preview_candidate(candidate, block_hash=block_hash, preview=preview) - - @staticmethod - def _serialize_prior_balance_preview(balances: list[dict[str, object]]) -> tuple[tuple[str, str, str, int], ...]: - return PayoutStateService._serialize_prior_balance_preview(balances) - - def _accepted_block_payout_preview_from_bundle(self, final_bundle: dict[str, Any], *, prior_balances: list[dict[str, object]] | None=None) -> list[dict[str, object]]: - """Derive the confirmed carry-forward view from a verified bundle.""" - return self._ensure_payout_state_service()._accepted_block_payout_preview_from_bundle(final_bundle, prior_balances=prior_balances) - - @staticmethod - def _materialize_prior_balance_preview(preview: tuple[tuple[str, str, str, int], ...]) -> list[dict[str, object]]: - return PayoutStateService._materialize_prior_balance_preview(preview) - - def _clear_accepted_block_payout_preview(self, block_hash: str, *, invalidate_published: bool=False) -> None: - return self._ensure_payout_state_service()._clear_accepted_block_payout_preview(block_hash, invalidate_published=invalidate_published) - - def _accepted_block_payout_transition_landed(self, block_hash: str) -> bool: - return self._ensure_payout_state_service()._accepted_block_payout_transition_landed(block_hash) - - def _accepted_block_payout_transition_for_parent(self, parent_hash: str, *, parent_height: int | None=None) -> tuple[str, bool] | None: - """Select the highest active exact/ancestor payout transition.""" - return self._ensure_payout_state_service()._accepted_block_payout_transition_for_parent(parent_hash, parent_height=parent_height) - - def _accepted_parent_unresolved_depth_cap(self) -> int: - return self._ensure_payout_state_service()._accepted_parent_unresolved_depth_cap() - - def accepted_parent_unresolved_ages_seconds(self) -> list[float]: - """Ages of landed transitions whose bookkeeping is still unresolved.""" - return self._ensure_payout_state_service().accepted_parent_unresolved_ages_seconds() - - def _accepted_parent_unresolved_depth(self) -> int: - return self._ensure_payout_state_service()._accepted_parent_unresolved_depth() - - def _await_pending_parent_payout_preview(self, parent_hash: str, *, parent_height: int | None=None) -> list[dict[str, object]] | None: - """Wait out a pending accepted-parent transition, returning its preview.""" - return self._ensure_payout_state_service()._await_pending_parent_payout_preview(parent_hash, parent_height=parent_height) - - def _prior_balances_for_job_parent(self, parent_hash: str, *, parent_height: int | None=None, fallback_balances: Sequence[dict[str, object]] | None=None) -> list[dict[str, object]]: - """Return prospective balances, otherwise a prepared/ledger fallback.""" - return self._ensure_payout_state_service()._prior_balances_for_job_parent(parent_hash, parent_height=parent_height, fallback_balances=fallback_balances) - - def _observe_payout_state_seconds(self, name: str, elapsed_seconds: float, *, relation: str | None=None) -> None: - return self._ensure_payout_state_service()._observe_payout_state_seconds(name, elapsed_seconds, relation=relation) - - def _observe_payout_gate_admission(self, admission: object, *, generation: int, fallback_wait_seconds: float) -> None: - return self._ensure_payout_state_service()._observe_payout_gate_admission(admission, generation=generation, fallback_wait_seconds=fallback_wait_seconds) - - def _reserve_payout_state_source(self, cause: str, *, tip_hash: str | None=None, invalidated_monotonic: float | None=None) -> int: - return self._ensure_payout_state_service()._reserve_payout_state_source(cause, tip_hash=tip_hash, invalidated_monotonic=invalidated_monotonic) - - def _reserve_payout_state_source_if_current(self, expected_source_generation: int, cause: str, *, tip_hash: str | None=None, invalidated_monotonic: float | None=None) -> tuple[int, int, str | None, str, float] | None: - """Reserve and capture a source only if preparation was not superseded.""" - return self._ensure_payout_state_service()._reserve_payout_state_source_if_current(expected_source_generation, cause, tip_hash=tip_hash, invalidated_monotonic=invalidated_monotonic) - - def _capture_payout_state_source(self) -> tuple[int, int, str | None, str, float]: - return self._ensure_payout_state_service()._capture_payout_state_source() - - def _prepared_payout_state_candidate(self, captured: tuple[int, int, str | None, str, float], *, force_full_window_rescan: bool=False, force_prior_balances_read: bool=False, bypass_build_interval: bool=False) -> PayoutStateCandidate: - return self._ensure_payout_state_service()._prepared_payout_state_candidate(captured, force_full_window_rescan=force_full_window_rescan, force_prior_balances_read=force_prior_balances_read, bypass_build_interval=bypass_build_interval) - - def _cached_found_block_payout_artifact(self, *, base_generation: int, artifact_payout_state_generation: int, network_difficulty: int, fallback_reason: str) -> PayoutLedgerArtifact | None: - """Retag an exact armed window when immediate preparation is unavailable.""" - return self._ensure_payout_state_service()._cached_found_block_payout_artifact(base_generation=base_generation, artifact_payout_state_generation=artifact_payout_state_generation, network_difficulty=network_difficulty, fallback_reason=fallback_reason) - - def _current_payout_state_candidate(self) -> PayoutStateCandidate: - return self._ensure_payout_state_service()._current_payout_state_candidate() - - def _record_discarded_payout_candidate(self) -> None: - return self._ensure_payout_state_service()._record_discarded_payout_candidate() - - def _block_payout_state_publication(self, *, force: bool=False, supersede_with: tuple[int, str | None, str, float] | None=None) -> None: - """Atomically close delivery, optionally reserving a newer source.""" - return self._ensure_payout_state_service()._block_payout_state_publication(force=force, supersede_with=supersede_with) - - def _payout_state_publication_fenced(self) -> bool: - """Report whether delivery is still blocked awaiting a publication.""" - return self._ensure_payout_state_service()._payout_state_publication_fenced() - - def _payout_source_requires_publication(self, candidate: PayoutStateCandidate | None=None) -> bool: - """Report whether an invalidation source still lacks a publication.""" - return self._ensure_payout_state_service()._payout_source_requires_publication(candidate) - - def _captured_payout_source_requires_publication(self, captured: tuple[int, int, str | None, str, float]) -> bool: - """Publication check for a captured source, without a candidate.""" - return self._ensure_payout_state_service()._captured_payout_source_requires_publication(captured) - - def _publish_payout_state_candidate(self, candidate: PayoutStateCandidate) -> int | None: - """Publish a prepared candidate, or reject it if its source moved.""" - return self._ensure_payout_state_service()._publish_payout_state_candidate(candidate) - - def _record_first_payout_delivery(self, generation: int, delivered_monotonic: float) -> None: - return self._ensure_payout_state_service()._record_first_payout_delivery(generation, delivered_monotonic) - - def _advance_payout_state_generation(self) -> int: - """Publish a payout-only invalidation with no expensive gate work.""" - return self._ensure_payout_state_service()._advance_payout_state_generation() - - def _publish_current_payout_state_with_retry_budget(self, *, initial_attempted: bool=False) -> int | None: - """Publish the current source with a bounded supersession budget.""" - return self._ensure_payout_state_service()._publish_current_payout_state_with_retry_budget(initial_attempted=initial_attempted) - - def observe_job_build_elapsed(self, elapsed_seconds: float, phases: dict[str, float]) -> None: - self._ensure_job_bundle_service().observe_job_build_elapsed(elapsed_seconds, phases) - - def _flush_job_build_phases(self, phases: dict[str, float]) -> None: - """Fold a worker thread's phase accruals into the exported counters.""" - self._ensure_job_bundle_service()._flush_job_build_phases(phases) - - def _reserve_template_artifact_generation(self) -> int: - """Reserve template ordering when a fetch starts, not when it finishes.""" - return self._ensure_job_bundle_service().template_repository.reserve_generation() - - def _derive_template_artifacts( - self, - template: dict[str, Any], - *, - generation: int, - ) -> CachedTemplateArtifacts: - return self._ensure_job_bundle_service().template_repository.derive(template, generation=generation) - - def _store_template_artifacts( - self, - artifacts: CachedTemplateArtifacts, - ) -> bool: - return self._ensure_job_bundle_service().template_repository.store_artifacts(artifacts) - - def store_template_artifacts( - self, - template: dict[str, Any], - *, - generation: int | None = None, - ) -> CachedTemplateArtifacts | None: - """Best-effort cache fill from an already-fetched template (blockpoll).""" - return self._ensure_job_bundle_service().template_repository.store(template, generation=generation) - - def job_issuance_template_artifacts(self) -> CachedTemplateArtifacts: - """Template artifacts for direct (non-refresh) job issuance.""" - return self._ensure_job_bundle_service().template_repository.issuance() - - def current_template_artifacts(self) -> CachedTemplateArtifacts: - """Return fresh template artifacts, fetching a template on cache miss.""" - return self._ensure_job_bundle_service().template_repository.current() - - _collection_bundle_identity = staticmethod( - JobBundleService._collection_bundle_identity - ) - - def _job_bundle_key( - self, - artifacts: CachedTemplateArtifacts, - *, - mode: str, - payout_state_generation: int, - payout_artifact_generation: int = 0, - worker: WorkerIdentity | None, - ) -> tuple[object, ...]: - return self._ensure_job_bundle_service()._job_bundle_key( - artifacts, - mode=mode, - payout_state_generation=payout_state_generation, - payout_artifact_generation=payout_artifact_generation, - worker=worker, - ) - - def _job_bundle_mode(self, requested_mode: str | None) -> str: - return self._ensure_job_bundle_service()._job_bundle_mode(requested_mode) - - def _lookup_job_bundle( - self, - key: tuple[object, ...], - ) -> CachedJobBundle | None: - return self._ensure_job_bundle_service()._lookup_job_bundle(key) - - def _no_artifact_job_bundle_key( - self, - artifacts: CachedTemplateArtifacts, - *, - mode: str, - payout_state_generation: int, - worker: WorkerIdentity | None, - ) -> tuple[object, ...]: - """Fallback cache identity for work built before an artifact re-key.""" - return self._ensure_job_bundle_service()._no_artifact_job_bundle_key( - artifacts, - mode=mode, - payout_state_generation=payout_state_generation, - worker=worker, - ) - - _no_artifact_bundle_matches_artifact = staticmethod( - JobBundleService._no_artifact_bundle_matches_artifact - ) - - def _job_bundle_entry_usable( - self, - cached: CachedJobBundle | None, - artifacts: CachedTemplateArtifacts, - ) -> bool: - """Re-validate freshness and readiness for cached bundles.""" - return self._ensure_job_bundle_service()._job_bundle_entry_usable(cached, artifacts) - - def _bind_cached_bundle_to_artifacts( - self, - cached: CachedJobBundle, - artifacts: CachedTemplateArtifacts, - ) -> CachedJobBundle: - """Return the cached heavy bundle bound to this exact observation.""" - return self._ensure_job_bundle_service()._bind_cached_bundle_to_artifacts(cached, artifacts) - - def _new_job_build_request( - self, - artifacts: CachedTemplateArtifacts, - worker: WorkerIdentity | None, - *, - mode: str, - payout_state_generation: int, - cache_key: tuple[object, ...], - payout_ledger_artifact: PayoutLedgerArtifact | None = None, - idle_retarget: bool = False, - publication_critical: bool = False, - request_source: str = "routine", - priority_requested_monotonic: float | None = None, - preparation_cancellation: _JobBuildCancellation | None = None, - ) -> _JobBuildRequest: - return self._ensure_job_bundle_service()._new_job_build_request( - artifacts, - worker, - mode=mode, - payout_state_generation=payout_state_generation, - cache_key=cache_key, - payout_ledger_artifact=payout_ledger_artifact, - idle_retarget=idle_retarget, - publication_critical=publication_critical, - request_source=request_source, - priority_requested_monotonic=priority_requested_monotonic, - preparation_cancellation=preparation_cancellation, - ) - - def _newest_observed_tip_locked(self) -> str | None: - """Newest live-tip observation, ahead of published submit authority. - - Detection and publication are split: a winning refresh builds for a - detected tip while the previous tip remains published. Build-pipeline - supersession checks must compare against this detection view, or a - replacement build for a freshly detected tip would classify itself as - obsolete before it could ever be published. - """ - detected = getattr(self, "latest_detected_tip", None) - if detected is not None: - return detected[0] - published = getattr(self, "current_tip_first_seen", None) - return published[0] if published is not None else None - - def _artifacts_buildable_locked( - self, - artifacts: CachedTemplateArtifacts, - ) -> bool: - """Whether work for these artifacts may still be built and cached. - - The newest detected tip covers replacement construction. Exactly the - published snapshot additionally stays buildable while the published - tip retains share-classification authority, so pinned direct issuance - can rebuild published work (for example after a payout-generation - prune) instead of classifying itself superseded for the whole - unpublished window. Anything else -- including other templates for - the published parent -- is superseded construction and must stop. - """ - newest = self._newest_observed_tip_locked() - if newest is None or artifacts.previousblockhash == newest: - return True - return self._published_snapshot_artifacts_locked(artifacts) - - def _published_snapshot_artifacts_locked( - self, - artifacts: CachedTemplateArtifacts, - ) -> bool: - """Whether these artifacts are exactly the still-authoritative published snapshot.""" - published = getattr(self, "current_tip_first_seen", None) - published_snapshot = getattr(self, "tip_template_snapshot", None) - return bool( - published is not None - and published_snapshot is not None - and published_snapshot.bestblockhash == published[0] - and artifacts.previousblockhash == published[0] - and published_snapshot.template_fingerprint == artifacts.fingerprint - and self._published_tip_authoritative_locked(time.monotonic()) - ) - - def _cache_job_bundle_if_current( - self, - built: CachedJobBundle, - artifacts: CachedTemplateArtifacts, - ) -> bool: - """Cache only current state; report whether payout state stayed valid.""" - return self._ensure_job_bundle_service()._cache_job_bundle_if_current(built, artifacts) - - def _probe_initial_job_bundle( - self, - artifacts: CachedTemplateArtifacts, - worker: WorkerIdentity | None, - mode: str | None, - ) -> CachedJobBundle | None: - """Serve an initial request straight from the bundle cache.""" - return self._ensure_job_bundle_service()._probe_initial_job_bundle(artifacts, worker, mode) - - def shared_job_bundle( - self, - artifacts: CachedTemplateArtifacts, - worker: WorkerIdentity | None = None, - *, - mode: str | None = None, - cancelled: Callable[[], bool] | None = None, - retry_superseded: bool = True, - idle_retarget: bool = False, - publication_critical: bool = False, - request_source: str = "routine", - priority_requested_monotonic: float | None = None, - ) -> CachedJobBundle: - """Return one immutable heavy build through a work-identity flight.""" - return self._ensure_job_bundle_service().shared_job_bundle( - artifacts, - worker, - mode=mode, - cancelled=cancelled, - retry_superseded=retry_superseded, - idle_retarget=idle_retarget, - publication_critical=publication_critical, - request_source=request_source, - priority_requested_monotonic=priority_requested_monotonic, - ) - - def _shared_job_bundle_after_priority_admission( - self, - artifacts: CachedTemplateArtifacts, - worker: WorkerIdentity | None = None, - *, - mode: str | None = None, - cancelled: Callable[[], bool] | None = None, - retry_superseded: bool = True, - idle_retarget: bool = False, - publication_critical: bool = False, - request_source: str = "routine", - priority_requested_monotonic: float | None = None, - ) -> CachedJobBundle: - return self._ensure_job_bundle_service()._shared_job_bundle_after_priority_admission( - artifacts, - worker, - mode=mode, - cancelled=cancelled, - retry_superseded=retry_superseded, - idle_retarget=idle_retarget, - publication_critical=publication_critical, - request_source=request_source, - priority_requested_monotonic=priority_requested_monotonic, - ) - - def _share_window_serialization_for_artifact( - self, - payout_artifact: PayoutLedgerArtifact, - shares: list[dict[str, object]], - ) -> _ShareWindowSerialization: - """Cached digest and builder fragments for the artifact's share window.""" - return self._ensure_job_bundle_service()._share_window_serialization_for_artifact( - payout_artifact, - shares, - ) - - def build_shared_job_bundle( - self, - artifacts: CachedTemplateArtifacts, - worker: WorkerIdentity | None = None, - *, - mode: str | None = None, - payout_state_generation: int | None = None, - payout_artifact: PayoutLedgerArtifact | None = None, - key: tuple[object, ...] | None = None, - build_request: _JobBuildRequest | None = None, - ) -> CachedJobBundle: - return self._ensure_job_bundle_service().build_shared_job_bundle( - artifacts, - worker, - mode=mode, - payout_state_generation=payout_state_generation, - payout_artifact=payout_artifact, - key=key, - build_request=build_request, - ) - - def stamp_job_for_client( - self, - client: ClientState, - cached: CachedJobBundle, - *, - clean_jobs: bool, - tip_refresh_epoch_sequence: int | None = None, - ) -> PrismJobContext: - return self._ensure_job_delivery_service().stamp_job_for_client( - client, - cached, - clean_jobs=clean_jobs, - tip_refresh_epoch_sequence=tip_refresh_epoch_sequence, - ) - - def _current_payout_generation(self) -> int: - """Narrow P1 adapter: the current payout-state generation.""" - return int(self._ensure_payout_state_service()._payout_state_generation) - - def _payout_delivery_snapshot(self) -> PayoutStateSnapshot: - """Narrow P1 adapter: copied identity facts for delivery decisions.""" - return self._ensure_payout_state_service().snapshot() - - def _payout_delivery_cancelable( - self, - cancelled: Callable[[], bool], - *, - generation: int, - priority: bool, - ) -> Any: - """Narrow P1 adapter: cancellable delivery-gate admission.""" - gate = self._ensure_payout_state_service()._payout_state_delivery_gate - return gate.delivery_cancelable( - cancelled, - generation=generation, - priority=priority, - ) - - def _job_delivery_observe_payout_admission( - self, - admission: Any, - *, - generation: int, - fallback_wait_seconds: float, - ) -> None: - """Narrow P1 adapter: record a delivery-gate admission observation.""" - self._observe_payout_gate_admission( - admission, - generation=generation, - fallback_wait_seconds=fallback_wait_seconds, - ) - - def _payout_template_network_difficulty(self) -> int | None: - """Narrow J1 adapter for P1: current template network difficulty.""" - service = self.__dict__.get("job_bundle_service") - if service is None: - return None - artifacts = service.template_repository.current_artifacts() - return None if artifacts is None else artifacts.network_difficulty - - def _invalidate_payout_job_cache(self) -> None: - """Narrow J1 adapter for P1: drop cached bundles after invalidation.""" - self._ensure_job_cache_state() - with self._job_cache_lock: - self._job_bundle_cache.clear() - - def _on_payout_state_invalidated( - self, - generation: int, - invalidated_monotonic: float, - ) -> None: - """Narrow G1/R1 adapter for P1: a payout generation was invalidated. - - The verbatim publication bodies keep their inline calls; this seam - exists for later stack layers that consume P1 through ports. - """ - self._record_progress_payout_generation( - generation, - invalidated_monotonic, - ) - - def _on_payout_state_published( - self, - generation: int, - invalidated_monotonic: float, - ) -> None: - """Narrow G1/R1 adapter for P1: a payout generation was published. - - The verbatim publication bodies keep their inline calls; this seam - exists for later stack layers that consume P1 through ports. - """ - self._record_progress_payout_generation( - generation, - invalidated_monotonic, - ) - - def _next_job_delivery_id(self) -> str: - """Allocate the next delivery job id through the S2 owner.""" - return self._ensure_job_delivery_service().next_job_id() - - def _complete_job_delivery( - self, - client: ClientState, - context: PrismJobContext, - delivered_monotonic: float, - ) -> None: - """Commit S2 delivery bookkeeping after a successful socket write.""" - self._ensure_job_delivery_service().complete_delivery( - client, - context, - delivered_monotonic, - ) - - def accepted_share_stats(self) -> tuple[int, int]: - """Return (accepted share count, distinct miner count) cheaply. - - Prefers the ledger's aggregate query; falls back to materializing - all_shares for ledgers that do not implement it. - """ - stats = getattr(self.ledger, "accepted_share_stats", None) - if callable(stats): - payload = stats() - return ( - int(payload["accepted_share_count"]), - int(payload["distinct_miner_count"]), - ) - shares = self.ledger.all_shares() - miner_ids = {getattr(share, "miner_id", None) for share in shares} - miner_ids.discard(None) - return len(shares), len(miner_ids) - - def _ensure_watchdog_state(self) -> None: - if not hasattr(self, "_heartbeats_lock"): - self._heartbeats_lock = threading.Lock() - if not hasattr(self, "_heartbeats"): - self._heartbeats = {} - if not hasattr(self, "_heartbeat_phases"): - self._heartbeat_phases = {} - if not hasattr(self, "_watchdog_pauses"): - self._watchdog_pauses = {} - - def _legacy_ctv_runtime_config(self) -> CtvRuntimeConfig: - coordinator_config = getattr(self, "config", None) - ctv_config = getattr(coordinator_config, "ctv", None) - if ctv_config is not None: - config = CtvRuntimeConfig.from_coordinator_config(ctv_config) - else: - config = CtvRuntimeConfig( - enabled=False, - wallet=None, - fee_sats=0, - limit=100, - chunk_size=DEFAULT_PRISM_CTV_BROADCASTER_CHUNK_SIZE, - interval_seconds=30.0, - ) - overrides = self.__dict__.get("_ctv_runtime_compat_config", {}) - if overrides: - config = dataclass_replace(config, **overrides) - return config - - def _make_ctv_runtime_service( - self, - config: CtvRuntimeConfig | None = None, - ) -> CtvRuntimeService: - runtime = CtvRuntimeService( - rpc_call=lambda *args, **kwargs: self.rpc.call(*args, **kwargs), - ledger_source=lambda: getattr(self, "ledger", None), - writer_admission=lambda component: self._writer_operation(component), - tip_refresh_pending=lambda: self.tip_refresh_is_pending(), - heartbeat=lambda: self._record_heartbeat("ctv_fanout_broadcaster"), - stop_signal=lambda: self.stop_event, - config=self._legacy_ctv_runtime_config() if config is None else config, - before_external_side_effect=( - lambda *args, **kwargs: ( - self._require_fresh_ledger_lease_for_external_side_effect( - *args, **kwargs - ) - ) - ), - shutdown_exception=ShutdownInProgress, - # Resolve the process types through this module's globals at call - # time so existing patch("lab.prism.prism_coordinator....") test - # seams keep intercepting daemon/broadcaster construction. - daemon_type=lambda *args, **kwargs: CtvFanoutBroadcastDaemon( - *args, **kwargs - ), - broadcaster_type=lambda *args, **kwargs: CtvFanoutBroadcaster( - *args, **kwargs - ), - # Preserve temporary coordinator patch points for focused tests. - monotonic=lambda: time.monotonic(), - print_exception=lambda: traceback.print_exc(), - ) - compat_daemon = self.__dict__.pop("_ctv_runtime_compat_daemon", None) - if compat_daemon is not None: - runtime.daemon = compat_daemon - return runtime - - def _ensure_ctv_runtime(self) -> CtvRuntimeService: - runtime = self.__dict__.get("_ctv_runtime") - if runtime is not None: - return runtime - init_lock = self.__dict__.get("_ctv_runtime_init_lock") - if init_lock is None: - # CPython's setdefault is atomic under the GIL. Focused tests may - # construct through __new__, while normal instances install this - # lock in __init__ before any process thread can start. - init_lock = self.__dict__.setdefault( - "_ctv_runtime_init_lock", - threading.Lock(), - ) - with init_lock: - runtime = self.__dict__.get("_ctv_runtime") - if runtime is not None: - return runtime - config = self._legacy_ctv_runtime_config() - runtime = self._make_ctv_runtime_service(config) - self.__dict__["_ctv_runtime"] = runtime - # The service is now the sole configuration owner. Removing the - # pre-init store prevents an old override from being replayed if a - # later compatibility property updates the live service. - self.__dict__.pop("_ctv_runtime_compat_config", None) - return runtime - - def _ctv_runtime_config_value(self, name: str) -> object: - runtime = self.__dict__.get("_ctv_runtime") - if runtime is not None: - return getattr(runtime.config, name) - return getattr(self._legacy_ctv_runtime_config(), name) - - def _set_ctv_runtime_config_value(self, name: str, value: object) -> None: - runtime = self.__dict__.get("_ctv_runtime") - if runtime is not None: - runtime.replace_config(**{name: value}) - return - init_lock = self.__dict__.get("_ctv_runtime_init_lock") - if init_lock is None: - init_lock = self.__dict__.setdefault( - "_ctv_runtime_init_lock", - threading.Lock(), - ) - with init_lock: - runtime = self.__dict__.get("_ctv_runtime") - if runtime is not None: - runtime.replace_config(**{name: value}) - return - overrides = self.__dict__.setdefault("_ctv_runtime_compat_config", {}) - overrides[name] = value - - @property - def ctv_broadcaster_enabled(self) -> bool: - return bool(self._ctv_runtime_config_value("enabled")) - - @ctv_broadcaster_enabled.setter - def ctv_broadcaster_enabled(self, value: bool) -> None: - self._set_ctv_runtime_config_value("enabled", bool(value)) - - @property - def ctv_broadcaster_wallet(self) -> str | None: - value = self._ctv_runtime_config_value("wallet") - return None if value is None else str(value) - - @ctv_broadcaster_wallet.setter - def ctv_broadcaster_wallet(self, value: str | None) -> None: - self._set_ctv_runtime_config_value("wallet", value) - - @property - def ctv_broadcaster_fee_sats(self) -> int: - return int(self._ctv_runtime_config_value("fee_sats")) - - @ctv_broadcaster_fee_sats.setter - def ctv_broadcaster_fee_sats(self, value: int) -> None: - self._set_ctv_runtime_config_value("fee_sats", int(value)) - - @property - def ctv_broadcaster_limit(self) -> int: - return int(self._ctv_runtime_config_value("limit")) - - @ctv_broadcaster_limit.setter - def ctv_broadcaster_limit(self, value: int) -> None: - self._set_ctv_runtime_config_value("limit", int(value)) - - @property - def ctv_broadcaster_chunk_size(self) -> int: - return int(self._ctv_runtime_config_value("chunk_size")) - - @ctv_broadcaster_chunk_size.setter - def ctv_broadcaster_chunk_size(self, value: int) -> None: - self._set_ctv_runtime_config_value("chunk_size", int(value)) - - @property - def ctv_broadcaster_interval_seconds(self) -> float: - return float(self._ctv_runtime_config_value("interval_seconds")) - - @ctv_broadcaster_interval_seconds.setter - def ctv_broadcaster_interval_seconds(self, value: float) -> None: - self._set_ctv_runtime_config_value("interval_seconds", float(value)) - - @property - def ctv_fanout_broadcast_daemon(self) -> CtvFanoutBroadcastDaemon | None: - runtime = self.__dict__.get("_ctv_runtime") - if runtime is not None: - return runtime.daemon - init_lock = self.__dict__.get("_ctv_runtime_init_lock") - if init_lock is None: - init_lock = self.__dict__.setdefault( - "_ctv_runtime_init_lock", - threading.Lock(), - ) - with init_lock: - runtime = self.__dict__.get("_ctv_runtime") - if runtime is None: - return self.__dict__.get("_ctv_runtime_compat_daemon") - return runtime.daemon - - @ctv_fanout_broadcast_daemon.setter - def ctv_fanout_broadcast_daemon( - self, - daemon: CtvFanoutBroadcastDaemon | None, - ) -> None: - runtime = self.__dict__.get("_ctv_runtime") - if runtime is not None: - runtime.daemon = daemon - return - init_lock = self.__dict__.get("_ctv_runtime_init_lock") - if init_lock is None: - init_lock = self.__dict__.setdefault( - "_ctv_runtime_init_lock", - threading.Lock(), - ) - with init_lock: - runtime = self.__dict__.get("_ctv_runtime") - if runtime is None: - self.__dict__["_ctv_runtime_compat_daemon"] = daemon - else: - runtime.daemon = daemon - - @property - def ctv_broadcaster_processed_rows_total(self) -> int: - return self._ensure_ctv_runtime().processed_rows_total - - @property - def ctv_broadcaster_pass_count(self) -> int: - return self._ensure_ctv_runtime().pass_count - - def _ensure_ctv_broadcaster_metrics_state(self) -> None: - self._ensure_ctv_runtime() - - def _record_ctv_fanout_broadcaster_progress(self) -> None: - self._ensure_ctv_runtime().record_progress() - - def observe_ctv_fanout_broadcaster_pass(self, elapsed_seconds: float) -> None: - self._ensure_ctv_runtime().observe_pass(elapsed_seconds) - - def observe_ctv_fanout_broadcaster_chunk( - self, - result: CtvFanoutChunkResult, - ) -> None: - self._ensure_ctv_runtime().observe_chunk(result) - - def _record_ctv_fanout_broadcaster_yield(self) -> None: - self._ensure_ctv_runtime().record_yield() - - def _ensure_worker_metrics_state(self) -> None: - if not hasattr(self, "worker_metrics_lock"): - self.worker_metrics_lock = threading.Lock() - if not hasattr(self, "worker_share_counts"): - self.worker_share_counts = {} - if not hasattr(self, "worker_rejection_counts"): - self.worker_rejection_counts = {} - - def _ensure_initial_job_state(self) -> None: - # S2 initial-job state lives in its owner service; the legacy raw - # fields route there through class descriptors and the service adopts - # direct test assignments through those descriptors. Only the live - # configuration attributes and S1/health fields remain seeded here. - self._ensure_job_delivery_service() - if not hasattr(self, "stratum_max_pending_initial_jobs"): - self.stratum_max_pending_initial_jobs = ( - DEFAULT_PRISM_STRATUM_MAX_PENDING_INITIAL_JOBS - ) - if not hasattr(self, "stratum_initial_job_timeout_seconds"): - self.stratum_initial_job_timeout_seconds = ( - DEFAULT_PRISM_STRATUM_INITIAL_JOB_TIMEOUT_SECONDS - ) - if not hasattr(self, "initial_job_max_workers"): - self.initial_job_max_workers = DEFAULT_PRISM_INITIAL_JOB_MAX_WORKERS - if not hasattr(self, "handler_thread_count"): - self.handler_thread_count = 0 - if not hasattr(self, "peak_active_connection_count"): - self.peak_active_connection_count = len(getattr(self, "clients", ())) - - def delivery_queue_limit(self) -> int: - pending_limit = int( - getattr( - self, - "stratum_max_pending_initial_jobs", - DEFAULT_PRISM_STRATUM_MAX_PENDING_INITIAL_JOBS, - ) - ) - connection_limit = int( - getattr( - self, - "stratum_max_connections", - DEFAULT_PRISM_STRATUM_MAX_CONNECTIONS, - ) - ) - return max( - int(getattr(self, "tip_refresh_max_workers", DEFAULT_PRISM_TIP_REFRESH_MAX_WORKERS)), - pending_limit, - connection_limit if connection_limit > 0 else pending_limit, - ) - - def _ensure_tip_refresh_state(self) -> None: - # R1 owner state lives in TipRefreshService; the legacy raw fields - # route there through class descriptors and the service adopts direct - # test assignments through those descriptors. Only the live - # configuration attributes and the two historically seeded lazy - # fields remain here. - self._ensure_tip_refresh_service() - if not hasattr(self, "tip_refresh_max_workers"): - self.tip_refresh_max_workers = DEFAULT_PRISM_TIP_REFRESH_MAX_WORKERS - if not hasattr(self, "tip_refresh_epoch_fanout"): - self.tip_refresh_epoch_fanout = False - if not hasattr(self, "latest_detected_tip"): - self.latest_detected_tip = None - if not hasattr(self, "tip_refresh_divergence_started_monotonic"): - self.tip_refresh_divergence_started_monotonic = None - - def _tip_refresh_prune_evicted_jobs(self, now: float, force: bool) -> None: - """Resolve the retained-prune compatibility seam at call time.""" - override = self.__dict__.get("prune_evicted_job_graveyard") - if callable(override): - override(now=now, force=force) - return - self._ensure_job_delivery_service().prune_evicted_job_graveyard( - now=now, - force=force, - ) - - _tip_refresh_hashrate_proxy = staticmethod( - TipRefreshService._tip_refresh_hashrate_proxy - ) - - def _payout_epoch_tip_hash_locked(self) -> str | None: - return self._ensure_tip_refresh_service()._payout_epoch_tip_hash_locked() - - def _mint_tip_refresh_epoch_locked( - self, - *, - tip_hash: str, - payout_state_generation: int, - started_monotonic: float, - ) -> int: - return self._ensure_tip_refresh_service()._mint_tip_refresh_epoch_locked( - tip_hash=tip_hash, - payout_state_generation=payout_state_generation, - started_monotonic=started_monotonic, - ) - - def _tip_refresh_epoch_for_bundle_locked( - self, - cached: CachedJobBundle, - ) -> int: - return self._ensure_tip_refresh_service()._tip_refresh_epoch_for_bundle_locked( - cached, - ) - - def _publish_tip_refresh_epoch_identity_locked( - self, - snapshot: QbitTipTemplateSnapshot, - ) -> None: - return self._ensure_tip_refresh_service()._publish_tip_refresh_epoch_identity_locked( - snapshot, - ) - - def _admit_client_tip_refresh_epoch_locked( - self, - client: ClientState, - epoch_sequence: int, - ) -> bool: - return self._ensure_tip_refresh_service()._admit_client_tip_refresh_epoch_locked( - client, - epoch_sequence, - ) - - _client_tip_refresh_epoch_blocked_locked = staticmethod( - TipRefreshService._client_tip_refresh_epoch_blocked_locked - ) - - def _tip_refresh_epoch_coverage_reached_locked( - self, - client: ClientState, - context: PrismJobContext, - delivered_monotonic: float, - ) -> list[tuple[str, float]]: - return self._ensure_tip_refresh_service()._tip_refresh_epoch_coverage_reached_locked( - client, - context, - delivered_monotonic, - ) - - def _record_tip_refresh_epoch_coverage( - self, - reached: list[tuple[str, float]], - ) -> None: - return self._ensure_tip_refresh_service()._record_tip_refresh_epoch_coverage( - reached, - ) - - def _record_tip_refresh_wave_outcome(self, outcome: str) -> None: - return self._ensure_tip_refresh_service()._record_tip_refresh_wave_outcome( - outcome, - ) - - def _tip_refresh_epoch_fixpoint_reached(self) -> bool: - return self._ensure_tip_refresh_service()._tip_refresh_epoch_fixpoint_reached() - - def _retain_collection_refresh( - self, - snapshot: QbitTipTemplateSnapshot, - observation_sequence: int, - payout_state_generation: int, - ) -> None: - return self._ensure_tip_refresh_service()._retain_collection_refresh( - snapshot, - observation_sequence, - payout_state_generation, - ) - - def _retained_collection_artifacts(self) -> CachedTemplateArtifacts | None: - return self._ensure_tip_refresh_service()._retained_collection_artifacts() - - def _retain_current_collection_refresh_if_unrepresented(self) -> None: - return self._ensure_tip_refresh_service()._retain_current_collection_refresh_if_unrepresented( - ) - - def _note_collection_identity_available(self, client: ClientState) -> None: - return self._ensure_tip_refresh_service()._note_collection_identity_available( - client, - ) - - def _consume_retained_collection_refresh( - self, - context: PrismJobContext, - ) -> None: - return self._ensure_tip_refresh_service()._consume_retained_collection_refresh( - context, - ) - - def tip_refresh_is_pending(self) -> bool: - return self._tip_refresh_pending() - - def _tip_refresh_pending(self) -> bool: - return self._ensure_tip_refresh_service()._tip_refresh_pending() - - def _mark_tip_refresh_pending(self, _observation: object) -> int: - return self._ensure_tip_refresh_service()._mark_tip_refresh_pending( - _observation, - ) - - def _claim_tip_refresh_pending(self) -> int | None: - return self._ensure_tip_refresh_service()._claim_tip_refresh_pending() - - def _mark_tip_refresh_pending_for_poll( - self, - owned_token: int | None, - _observation: object, - ) -> int | None: - return self._ensure_tip_refresh_service()._mark_tip_refresh_pending_for_poll( - owned_token, - _observation, - ) - - def _clear_tip_refresh_pending(self, token: int) -> None: - return self._ensure_tip_refresh_service()._clear_tip_refresh_pending(token) - - def _clear_tip_refresh_pending_for_completed_refresh( - self, - snapshot: QbitTipTemplateSnapshot, - observation_sequence: int, - payout_state_generation: int, - pending_signal_token: int | None = None, - ) -> bool: - return self._ensure_tip_refresh_service()._clear_tip_refresh_pending_for_completed_refresh( - snapshot, - observation_sequence, - payout_state_generation, - pending_signal_token, - ) - - def _schedule_tip_refresh_retry(self) -> None: - return self._ensure_tip_refresh_service()._schedule_tip_refresh_retry() - - def _consume_tip_refresh_retry(self) -> bool: - return self._ensure_tip_refresh_service()._consume_tip_refresh_retry() - - def _note_tip_refresh_attempt_failed( - self, - observed_tip: str | None = None, - ) -> None: - return self._ensure_tip_refresh_service()._note_tip_refresh_attempt_failed( - observed_tip, - ) - - def _tip_refresh_failure_holdoff_remaining(self) -> float: - return self._ensure_tip_refresh_service()._tip_refresh_failure_holdoff_remaining( - ) - - def _observe_tip_refresh_seconds(self, name: str, elapsed_seconds: float) -> None: - return self._ensure_tip_refresh_service()._observe_tip_refresh_seconds( - name, - elapsed_seconds, - ) - - def _observe_tip_refresh_build_phase( - self, - phase: str, - elapsed_seconds: float, - ) -> None: - return self._ensure_tip_refresh_service()._observe_tip_refresh_build_phase( - phase, - elapsed_seconds, - ) - - def _record_tip_refresh_ipc_bytes(self, direction: str, byte_count: int) -> None: - return self._ensure_tip_refresh_service()._record_tip_refresh_ipc_bytes( - direction, - byte_count, - ) - - def _record_tip_refresh_client_result(self, result: str) -> None: - return self._ensure_tip_refresh_service()._record_tip_refresh_client_result( - result, - ) - - def _record_tip_refresh_cancellation(self, stage: str) -> None: - return self._ensure_tip_refresh_service()._record_tip_refresh_cancellation( - stage, - ) - - def _tip_refresh_future_started(self) -> None: - return self._ensure_tip_refresh_service()._tip_refresh_future_started() - - def _tip_refresh_future_finished(self, _future: Future[RefreshResult]) -> None: - return self._ensure_tip_refresh_service()._tip_refresh_future_finished(_future) - - def tip_refresh_executor(self) -> _BoundedPriorityExecutor: - return self._ensure_tip_refresh_service().tip_refresh_executor() - - def initial_job_executor(self) -> _BoundedPriorityExecutor: - return self._ensure_job_delivery_service().initial_job_executor() - - def shutdown_initial_job_executor(self) -> None: - return self._ensure_job_delivery_service().shutdown_initial_job_executor() - - def shutdown_tip_refresh_executor(self) -> None: - return self._ensure_tip_refresh_service().shutdown_tip_refresh_executor() - - def retire_share_window_spool(self) -> None: - """Release the cached share-window spool during shutdown.""" - self._ensure_job_bundle_service().retire_share_window_spool() - - def _cancel_initial_job_future(self, future: Future[bool]) -> bool: - return self._ensure_job_delivery_service()._cancel_initial_job_future(future) - - def _initial_request_current_locked(self, request: PendingInitialJob) -> bool: - return self._ensure_job_delivery_service()._initial_request_current_locked( - request, - ) - - def _initial_request_cancelled(self, request: PendingInitialJob) -> bool: - return self._ensure_job_delivery_service()._initial_request_cancelled(request) - - def _cancel_pending_initial_job_locked( - self, - client: ClientState, - *, - count: bool, - ) -> PendingInitialJob | None: - return self._ensure_job_delivery_service()._cancel_pending_initial_job_locked( - client, - count=count, - ) - - def _client_has_current_tip_job_locked(self, client: ClientState) -> bool: - return self._ensure_job_delivery_service()._client_has_current_tip_job_locked( - client, - ) - - def note_initial_job_delivered( - self, - client: ClientState, - *, - validated_current: bool = False, - ) -> None: - return self._ensure_job_delivery_service().note_initial_job_delivered( - client, - validated_current=validated_current, - ) - - def schedule_initial_job(self, client: ClientState) -> bool: - return self._ensure_job_delivery_service().schedule_initial_job(client) - - def request_initial_job_delivery(self, client: ClientState) -> bool: - return self._ensure_job_delivery_service().request_initial_job_delivery(client) - - def cancel_initial_job_delivery(self, client: ClientState) -> None: - return self._ensure_job_delivery_service().cancel_initial_job_delivery(client) - - def _submit_initial_job_request(self, request: PendingInitialJob) -> bool: - return self._ensure_job_delivery_service()._submit_initial_job_request(request) - - def _initial_job_future_finished( - self, - request: PendingInitialJob, - future: Future[bool], - ) -> None: - return self._ensure_job_delivery_service()._initial_job_future_finished( - request, - future, - ) - - def _run_initial_job(self, request: PendingInitialJob) -> bool: - return self._ensure_job_delivery_service()._run_initial_job(request) - - def _template_artifacts_are_current(self, artifacts: CachedTemplateArtifacts) -> bool: - return self._ensure_job_bundle_service().template_repository.artifacts_are_current(artifacts) - - def _issuance_artifacts_current(self, artifacts: CachedTemplateArtifacts) -> bool: - """Issuance-side currency for direct job delivery. - - Current means either the live template view (the newest stored - artifacts) or exactly the published snapshot while the published tip - still owns share classification. During a detected-but-unpublished - refresh, pinned published-snapshot work must stay deliverable; judging - it against the detected-tip globals would defer every direct issuance - for the entire construction window that publication is deliberately - decoupled from. - """ - if self._template_artifacts_are_current(artifacts): - return True - with self.lock: - published = getattr(self, "current_tip_first_seen", None) - published_snapshot = getattr(self, "tip_template_snapshot", None) - return bool( - published is not None - and published_snapshot is not None - and published_snapshot.bestblockhash == published[0] - and artifacts.previousblockhash == published[0] - and published_snapshot.template_fingerprint == artifacts.fingerprint - and self._published_tip_authoritative_locked(time.monotonic()) - ) - - def _payout_delivery(self, cancelled: Callable[[], bool], *, generation: int) -> Any: - """Use cancellable admission while retaining focused gate test seams.""" - return self._ensure_payout_state_service()._payout_delivery(cancelled, generation=generation) - - @staticmethod - def _submit_delivery_task( - executor: object, - function: Callable[..., Any], - *args: object, - priority: int, - ) -> Future[Any]: - submit = getattr(executor, "submit") - if isinstance(executor, _BoundedPriorityExecutor): - return submit(function, *args, priority=priority) - return submit(function, *args) - - def _deliver_initial_bundle( - self, - request: PendingInitialJob, - artifacts: CachedTemplateArtifacts, - bundle: CachedJobBundle, - ) -> bool | None: - return self._ensure_job_delivery_service()._deliver_initial_bundle( - request, - artifacts, - bundle, - ) - - def sweep_initial_job_timeouts(self, *, now: float | None = None) -> int: - return self._ensure_job_delivery_service().sweep_initial_job_timeouts(now=now) - - def initial_job_timeout_loop(self) -> None: - return self._ensure_job_delivery_service().initial_job_timeout_loop() - - def _record_heartbeat(self, name: str, *, phase: str | None = None) -> None: - self._ensure_watchdog_state() - with self._heartbeats_lock: - self._heartbeats[name] = time.monotonic() - if phase is not None: - self._heartbeat_phases[name] = phase - - def _block_work_heartbeat_owner(self) -> tuple[str, str] | None: - """Return the independent heartbeat/phase slots owned by this thread.""" - return self._ensure_block_candidate_service()._block_work_heartbeat_owner() - - def _record_block_work_heartbeat(self, name: str, phase: str) -> None: - """Record a phase while preserving one-argument heartbeat embedders.""" - self._ensure_block_candidate_service()._record_block_work_heartbeat( - name, - phase, - ) - - def _record_block_submitter_phase(self, phase: str) -> None: - """Stamp a named phase only from a dedicated block-work owner.""" - self._ensure_block_candidate_service()._record_block_submitter_phase(phase) - - def _record_block_submitter_wait(self, phase: str) -> None: - """Heartbeat owner waits while preserving lightweight test behavior.""" - self._ensure_block_candidate_service()._record_block_submitter_wait(phase) - - def _block_work_wait_slice(self) -> float: - """Choose a polling slice that stays inside the configured watchdog.""" - return self._ensure_block_candidate_service()._block_work_wait_slice() - - def _observe_coordinator_lock_wait(self, elapsed_seconds: float) -> None: - """Keep a sliced coordinator-lock wait visible and watchdog-safe.""" - self._ensure_block_candidate_service()._observe_coordinator_lock_wait( - elapsed_seconds - ) - - def _acquire_block_submitter_lock(self, lock: Any, name: str) -> None: - """Acquire a submit-path lock in heartbeat/logging slices.""" - self._ensure_block_candidate_service()._acquire_block_submitter_lock( - lock, - name, - ) - - def _block_submitter_lock(self, lock: Any, name: str) -> Iterator[None]: - return self._ensure_block_candidate_service()._block_submitter_lock( - lock, - name, - ) - - def _overdue_heartbeats(self, now: float) -> list[str]: - self._ensure_watchdog_state() - with self._heartbeats_lock: - paused = set(self._watchdog_pauses) - overdue: list[str] = [] - for name, last in self._heartbeats.items(): - if name in paused or now - last <= self.watchdog_timeout_seconds: - continue - phase = self._heartbeat_phases.get(name) - overdue.append(f"{name}:{phase}" if phase else name) - return sorted(overdue) - - def _pause_watchdog_heartbeat(self, name: str) -> None: - self._ensure_watchdog_state() - with self._heartbeats_lock: - self._watchdog_pauses[name] = self._watchdog_pauses.get(name, 0) + 1 - self._heartbeats[name] = time.monotonic() - - def _resume_watchdog_heartbeat(self, name: str) -> None: - self._ensure_watchdog_state() - with self._heartbeats_lock: - depth = self._watchdog_pauses.get(name, 0) - if depth <= 1: - self._watchdog_pauses.pop(name, None) - else: - self._watchdog_pauses[name] = depth - 1 - self._heartbeats[name] = time.monotonic() - - def _remove_watchdog_heartbeat(self, name: str) -> None: - self._ensure_watchdog_state() - with self._heartbeats_lock: - self._heartbeats.pop(name, None) - self._heartbeat_phases.pop(name, None) - self._watchdog_pauses.pop(name, None) - - def _registered_watchdog_heartbeat_names(self, *names: str) -> tuple[str, ...]: - self._ensure_watchdog_state() - with self._heartbeats_lock: - return tuple(name for name in names if name in self._heartbeats) - - def stratum_accept_heartbeat_names(self) -> tuple[str, ...]: - return configured_accept_heartbeat_names( - getattr(self, "listener_profiles", None) - ) - - def _make_background_service_registry(self) -> BackgroundServiceRegistry: - """Describe process loops in their historical shutdown-join order. - - Every specification is registered unconditionally with a - call-time-resolved target so post-construction test monkeypatches of - the loop methods and of the enable/interval attributes keep working; - serve() applies the live conditional at each named start. The - watchdog specification is deliberately absent here: serve() registers - it inline at its start boundary so the mandatory - publication-progress watchdog thread is created before any - synchronous startup prewarm/recovery work. The one capability - conditional is hashrate rollup maintenance: it exists only for - ledgers that expose ``advance_hashrate_rollups`` (the Postgres - backend), because a memory-backed ledger never grows that method at - runtime -- unlike an enable flag, the capability cannot flip after - construction, so registering an unstartable service would only - mislead the registry snapshot. - """ - specifications = [ - BackgroundServiceSpec( - name="qbit_blockpoll", - thread_name="prism-qbit-block-poll", - target=lambda: self.blockpoll_loop(), - daemon=True, - join_timeout=1.0, - watchdog_monitored=True, - registration_identity=("qbit_blockpoll",), - ), - BackgroundServiceSpec( - name="block_submitter", - thread_name="prism-block-submitter", - target=lambda: self.block_submit_loop(), - daemon=True, - join_timeout=1.0, - watchdog_monitored=True, - registration_identity=("block_submitter",), - ), - BackgroundServiceSpec( - name="qbit_blockwait", - thread_name="prism-qbit-block-wait", - target=lambda: self.blockwait_loop(), - daemon=True, - join_timeout=1.0, - watchdog_monitored=True, - registration_identity=("qbit_blockwait",), - ), - BackgroundServiceSpec( - name="vardiff_idle_sweep", - thread_name="prism-vardiff-idle-sweep", - target=lambda: self.vardiff_idle_sweep_loop(), - daemon=True, - join_timeout=1.0, - watchdog_monitored=True, - registration_identity=("vardiff_idle_sweep",), - ), - BackgroundServiceSpec( - name="initial_job_timeout_sweep", - thread_name="prism-initial-job-timeouts", - target=lambda: self.initial_job_timeout_loop(), - daemon=True, - join_timeout=1.0, - watchdog_monitored=False, - registration_identity=("initial_job_timeout_sweep",), - ), - BackgroundServiceSpec( - name="share_writer", - thread_name="prism-share-writer", - target=lambda: self.share_append_loop(), - daemon=True, - join_timeout=5.0, - watchdog_monitored=True, - registration_identity=("share_writer",), - ), - BackgroundServiceSpec( - name="ctv_fanout_broadcaster", - thread_name="prism-ctv-fanout-broadcaster", - target=lambda: self.ctv_fanout_broadcaster_loop(), - daemon=True, - join_timeout=1.0, - watchdog_monitored=True, - registration_identity=("ctv_fanout_broadcaster",), - ), - self._health_snapshot_service_spec(), - self._metrics_snapshot_service_spec(), - ] - if callable( - getattr(getattr(self, "ledger", None), "advance_hashrate_rollups", None) - ): - specifications.append( - BackgroundServiceSpec( - name="hashrate-rollup-maintenance", - thread_name="prism-hashrate-rollup-maintenance", - target=lambda: self.hashrate_rollup_maintenance_loop(), - daemon=True, - join_timeout=1.0, - watchdog_monitored=False, - registration_identity=("hashrate-rollup-maintenance",), - ) - ) - return BackgroundServiceRegistry(specifications) - - def _health_snapshot_service_spec(self) -> BackgroundServiceSpec: - return BackgroundServiceSpec( - name="health_snapshot_refresher", - thread_name="prism-health-snapshot-refresher", - target=lambda: self.health_snapshot_loop(), - daemon=True, - join_timeout=1.0, - watchdog_monitored=False, - registration_identity=("health_snapshot_refresher",), - ) - - def _metrics_snapshot_service_spec(self) -> BackgroundServiceSpec: - return BackgroundServiceSpec( - name="metrics_snapshot_refresher", - thread_name="prism-metrics-snapshot-refresher", - target=lambda: self.metrics_snapshot_loop(), - daemon=True, - join_timeout=1.0, - watchdog_monitored=False, - registration_identity=("metrics_snapshot_refresher",), - ) - - def _ensure_background_services(self) -> BackgroundServiceRegistry: - registry = getattr(self, "_background_services", None) - if registry is None: - registry = self._make_background_service_registry() - self._background_services = registry - return registry - - def _start_background_service(self, name: str) -> threading.Thread: - registry = self._ensure_background_services() - return registry.start( - name, - on_started=lambda specification: ( - self._record_heartbeat(specification.name) - if specification.watchdog_monitored - else None - ), - ) - - def _start_secondary_accept_service( - self, - server: socket.socket, - profile: StratumListenerProfile, - ) -> threading.Thread: - registry = self._ensure_background_services() - service_name = profile.heartbeat_name - registry.register_if_absent( - BackgroundServiceSpec( - name=service_name, - thread_name=f"prism-stratum-accept-{profile.name}", - target=lambda: self.accept_loop(server, profile), - daemon=True, - join_timeout=1.0, - watchdog_monitored=True, - registration_identity=( - "secondary_stratum_accept", - id(server), - id(profile), - ), - ) - ) - return self._start_background_service(service_name) - - # Lease heartbeat state is owned by LedgerLeaseHeartbeatService; these - # descriptors keep the historical private coordinator attribute names - # readable/writable for tests and embedders with exactly one owner copy. - _ledger_lease_heartbeat_freshness_lock = _LeaseHeartbeatStateField( - "freshness_lock" - ) - _ledger_lease_heartbeat_failure_lock = _LeaseHeartbeatStateField( - "failure_lock" - ) - _ledger_lease_heartbeat_last_success_monotonic = _LeaseHeartbeatStateField( - "last_success_monotonic" - ) - _ledger_lease_heartbeat_last_progress_monotonic = _LeaseHeartbeatStateField( - "last_progress_monotonic" - ) - _ledger_lease_heartbeat_last_server_proven_monotonic = ( - _LeaseHeartbeatStateField("last_server_proven_monotonic") - ) - _ledger_lease_heartbeat_failed = _LeaseHeartbeatStateField("failed") - _ledger_lease_heartbeat_ready = _LeaseHeartbeatStateField("ready") - _ledger_lease_heartbeat_stop_event = _LeaseHeartbeatStateField("stop_event") - _ledger_lease_heartbeat_thread = _LeaseHeartbeatStateField("thread") - _ledger_lease_heartbeat_monitor_thread = _LeaseHeartbeatStateField( - "monitor_thread" - ) - _ledger_lease_heartbeat_exit_thread = _LeaseHeartbeatStateField( - "exit_thread" - ) - _ledger_lease_heartbeat_failure_reason = _LeaseHeartbeatStateField( - "failure_reason" - ) - _ledger_lease_heartbeat_failure_has_traceback = _LeaseHeartbeatStateField( - "failure_has_traceback" - ) - - def _ensure_lease_heartbeat_service(self) -> LedgerLeaseHeartbeatService: - """Single-flight lazy root for the lease heartbeat owner. - - Every port resolves live coordinator state at call time so the - documented monkeypatch seams (test timing attributes, patched - ``_ledger_lease_heartbeat_hard_exit`` / ``_watchdog_hard_exit`` - methods, replaced loop methods, swapped ledgers) keep working. - """ - service = self.__dict__.get("_lease_heartbeat_service") - if service is not None: - return service - # CPython's setdefault is atomic under the GIL. This lazy path exists - # for focused tests that construct a coordinator with __new__; normal - # instances reach it before any process thread starts. - init_lock = self.__dict__.setdefault( - "_lease_heartbeat_service_init_lock", - threading.Lock(), - ) - with init_lock: - service = self.__dict__.get("_lease_heartbeat_service") - if service is not None: - return service - service = LedgerLeaseHeartbeatService( - LedgerLeaseHeartbeatPorts( - ledger=lambda: getattr(self, "ledger", None), - heartbeat_seconds=lambda: float( - getattr( - self, - "ledger_lease_heartbeat_seconds", - DEFAULT_PRISM_LEDGER_LEASE_HEARTBEAT_SECONDS, - ) - ), - failure_seconds=lambda: float( - getattr( - self, - "ledger_lease_heartbeat_failure_seconds", - DEFAULT_PRISM_LEDGER_LEASE_HEARTBEAT_FAILURE_SECONDS, - ) - ), - monitor_seconds=lambda: float( - getattr( - self, - "ledger_lease_heartbeat_monitor_seconds", - DEFAULT_PRISM_LEDGER_LEASE_HEARTBEAT_MONITOR_SECONDS, - ) - ), - exit_timeout_seconds=lambda: float( - getattr( - self, - "ledger_lease_heartbeat_exit_timeout_seconds", - DEFAULT_PRISM_LEDGER_LEASE_HEARTBEAT_EXIT_TIMEOUT_SECONDS, - ) - ), - external_fence_timeout_seconds=lambda: float( - getattr( - self, - "ledger_lease_external_fence_timeout_seconds", - DEFAULT_PRISM_LEDGER_LEASE_EXTERNAL_FENCE_TIMEOUT_SECONDS, - ) - ), - lease_hard_exit=( - lambda message, *, include_traceback: ( - self._ledger_lease_heartbeat_hard_exit( - message, - include_traceback=include_traceback, - ) - ) - ), - watchdog_hard_exit=( - lambda reason, *, timeout_seconds: ( - self._watchdog_hard_exit( - reason, - timeout_seconds=timeout_seconds, - ) - ) - ), - scheduler_slack_seconds=lambda: float( - getattr( - self, - "ledger_lease_heartbeat_scheduler_slack_seconds", - WRITER_LEASE_HEARTBEAT_SCHEDULER_SLACK_SECONDS, - ) - ), - heartbeat_loop=lambda: self.ledger_lease_heartbeat_loop(), - monitor_loop=lambda: ( - self.ledger_lease_heartbeat_monitor_loop() - ), - ) - ) - # Adopt pre-service fixture fields once and remove the legacy - # keys; the descriptors above route all later access. - for legacy_key, attribute in ( - ("_ledger_lease_heartbeat_freshness_lock", "freshness_lock"), - ("_ledger_lease_heartbeat_failure_lock", "failure_lock"), - ( - "_ledger_lease_heartbeat_last_success_monotonic", - "last_success_monotonic", - ), - ( - "_ledger_lease_heartbeat_last_progress_monotonic", - "last_progress_monotonic", - ), - ( - "_ledger_lease_heartbeat_last_server_proven_monotonic", - "last_server_proven_monotonic", - ), - ("_ledger_lease_heartbeat_failed", "failed"), - ("_ledger_lease_heartbeat_ready", "ready"), - ("_ledger_lease_heartbeat_stop_event", "stop_event"), - ("_ledger_lease_heartbeat_thread", "thread"), - ("_ledger_lease_heartbeat_monitor_thread", "monitor_thread"), - ("_ledger_lease_heartbeat_exit_thread", "exit_thread"), - ("_ledger_lease_heartbeat_failure_reason", "failure_reason"), - ( - "_ledger_lease_heartbeat_failure_has_traceback", - "failure_has_traceback", - ), - ): - if legacy_key in self.__dict__: - setattr(service, attribute, self.__dict__.pop(legacy_key)) - self.__dict__["_lease_heartbeat_service"] = service - return service - - def _record_ledger_lease_heartbeat_success( - self, - renewal_started_monotonic: float, - ) -> None: - """Advance the conservative freshness edge without regression.""" - self._ensure_lease_heartbeat_service().record_success( - renewal_started_monotonic - ) - - def _record_ledger_lease_heartbeat_progress(self) -> None: - """Stamp mid-verification progress for the staleness monitor only.""" - self._ensure_lease_heartbeat_service().record_progress() - - def _record_ledger_lease_heartbeat_server_proven(self) -> None: - """Stamp a completed server round trip: progress plus envelope edge.""" - self._ensure_lease_heartbeat_service().record_server_proven() - - def _ledger_lease_adoption_silence_seconds(self) -> float: - """Resolve the ledger's adoption-silence window for timing checks.""" - return self._ensure_lease_heartbeat_service().adoption_silence_seconds() - - def _ledger_lease_heartbeat_activity_age_seconds(self) -> float: - """Age of the newest monitor-visible heartbeat activity mark.""" - return self._ensure_lease_heartbeat_service().activity_age_seconds() - - @staticmethod - def _ledger_lease_guard_session_verifier( - ledger: object, - ) -> Callable[..., object] | None: - """Return the guarded-session liveness check for ``ledger``.""" - return _lease_guard_session_verifier(ledger) - - @staticmethod - def _ledger_lease_guard_session_prover( - ledger: object, - ) -> Callable[..., object] | None: - """Return the cheap read-only ownership proof for ``ledger``.""" - return _lease_guard_session_prover(ledger) - - def _ledger_lease_heartbeat_snapshot(self) -> dict[str, object]: - """Phase attribution and policy terms for the metrics surface.""" - return self._ensure_lease_heartbeat_service().snapshot() - - def _start_ledger_lease_heartbeat(self) -> threading.Thread | None: - return self._ensure_lease_heartbeat_service().start() - - def _ledger_lease_heartbeat_hard_exit( - self, - message: str, - *, - include_traceback: bool, - ) -> None: - self._ensure_lease_heartbeat_service().hard_exit( - message, - include_traceback=include_traceback, - ) - - def ledger_lease_heartbeat_loop(self) -> None: - self._ensure_lease_heartbeat_service().heartbeat_loop() - - def ledger_lease_heartbeat_monitor_loop(self) -> None: - self._ensure_lease_heartbeat_service().monitor_loop() - - def _stop_ledger_lease_heartbeat( - self, - *, - deadline: float | None = None, - ) -> bool: - return self._ensure_lease_heartbeat_service().stop(deadline=deadline) - - def _require_fresh_ledger_lease_for_external_side_effect( - self, - component: str, - ) -> None: - """Synchronously verify the exact guarded session before an RPC effect.""" - self._ensure_lease_heartbeat_service( - ).require_fresh_lease_for_external_side_effect(component) - - @contextmanager - def _watchdog_paused(self, *names: str) -> Iterator[None]: - for name in names: - self._pause_watchdog_heartbeat(name) - try: - yield - finally: - for name in reversed(names): - self._resume_watchdog_heartbeat(name) - - def watchdog_loop(self) -> None: - self._ensure_watchdog_service().run() - - def _watchdog_hard_exit( - self, - reason: str, - *, - timeout_seconds: float | None = None, - ) -> None: - """Dynamic hard-exit seam; the bounded-exit body is service-owned. - - Both WatchdogService.run and the ledger-lease heartbeat service - trigger this coordinator method at call time so per-instance - monkeypatches intercept every exit path. - """ - self._ensure_watchdog_service().hard_exit( - reason, - timeout_seconds=timeout_seconds, - ) - - def _ensure_watchdog_service(self) -> WatchdogService: - service = self.__dict__.get("_watchdog_service") - if service is None: - # Idempotent construction; no init lock is needed. - service = WatchdogService( - WatchdogPorts( - wait_for_stop=lambda timeout: self.stop_event.wait(timeout), - interval_seconds=lambda: self.watchdog_interval_seconds, - fatal_exit_requested=lambda: bool( - getattr(self, "_fatal_exit_requested", False) - ), - publication_state=lambda now: ( - self._publication_watchdog_state(now) - ), - hard_exit=lambda reason: self._watchdog_hard_exit(reason), - liveness_enabled=lambda: bool( - getattr(self, "watchdog_enabled", True) - ), - overdue_heartbeats=lambda now: self._overdue_heartbeats(now), - liveness_timeout_seconds=lambda: ( - self.watchdog_timeout_seconds - ), - coordination_budget_seconds=lambda: float( - getattr( - self, - "coordination_blocked_exit_seconds", - DEFAULT_PRISM_COORDINATION_BLOCKED_EXIT_SECONDS, - ) - ), - publication_budget_seconds=lambda: float( - getattr( - self, - "template_refresh_failure_exit_seconds", - DEFAULT_PRISM_TEMPLATE_MAX_AGE_SECONDS, - ) - ), - ensure_job_cache_state=self._ensure_job_cache_state, - publication_failure_expired=lambda now: ( - self.publication_progress_failure_expired(now) - ), - publication_divergence_since=( - self._publication_divergence_since_locked - ), - lease_release_timeout_seconds=lambda: float( - getattr( - self, - "watchdog_lease_release_timeout_seconds", - DEFAULT_PRISM_WATCHDOG_LEASE_RELEASE_TIMEOUT_SECONDS, - ) - ), - shutdown_controller=lambda: self._ensure_shutdown_controller(), - request_shutdown=lambda: self.request_shutdown(None), - release_ledger_lease=lambda deadline: ( - self.release_ledger_lease( - fresh_connection=True, - deadline=deadline, - emit_logs=False, - ) - ), - lease_failure_reason=lambda: getattr( - self, - "_ledger_lease_heartbeat_failure_reason", - None, - ), - exit_process=lambda code: os._exit(code), - log=lambda message: print(message, flush=True), - ) - ) - self.__dict__["_watchdog_service"] = service - return service - - def _publication_divergence_since_locked(self) -> float | None: - """Locked snapshot of the progress-health divergence timestamp. - - Progress-health keeps ownership of the divergence state; the - watchdog reads it through this port while holding its own - coordination lock (watchdog lock before progress-health lock, - never the reverse). - """ - with self._progress_health_lock: - return self._progress_publication_divergence_since_monotonic - - def _ensure_shutdown_controller(self) -> CoordinatorShutdownController: - controller = getattr(self, "_shutdown_controller", None) - if controller is not None: - return controller - candidate = CoordinatorShutdownController( - float( - getattr( - self, - "writer_quiescence_timeout_seconds", - DEFAULT_PRISM_WRITER_QUIESCENCE_TIMEOUT_SECONDS, - ) - ) - ) - # CPython's setdefault is atomic under the GIL. This lazy path exists - # for focused tests that construct a coordinator with __new__; normal - # instances create the controller in __init__ before threads start. - return self.__dict__.setdefault("_shutdown_controller", candidate) - - @contextmanager - def _writer_operation(self, component: str) -> Iterator[None]: - controller = self._ensure_shutdown_controller() - block_work_owner = self._block_work_heartbeat_owner() is not None - phase = f"writer-admission:{component}" - if block_work_owner: - self._record_block_submitter_phase(phase) - if block_work_owner: - token = controller.enter_writer( - component, - wait_callback=lambda: self._record_block_submitter_phase(phase), - ) - else: - # Keep the historical one-argument seam for focused embedders and - # test controllers; only dedicated block-work owners need sliced - # admission heartbeats. - token = controller.enter_writer(component) - try: - yield - finally: - controller.exit_writer(token) - - def request_shutdown(self, signum: int | None = None) -> None: - """Signal-safe-sized shutdown request; the ordered work runs elsewhere.""" - self._ensure_shutdown_controller().request_shutdown(signum) - self.stop_event.set() - - @staticmethod - def _shutdown_log(event: str, **fields: object) -> None: - print( - "prism coordinator: " - + json.dumps({"event": event, **fields}, sort_keys=True), - flush=True, - ) - - def _cancel_active_tip_refresh_for_shutdown(self) -> None: - return self._ensure_tip_refresh_service()._cancel_active_tip_refresh_for_shutdown( - ) - - def shutdown(self, *, reason: str = "graceful") -> bool: - """Quiesce every ledger writer and release its lease exactly once. - - Returns true when release completed safely (including a ledger without - lease support or an already-absent exact session lease). A timeout - deliberately withholds release while a tracked writer may still run. - """ - controller = self._ensure_shutdown_controller() - if not controller.begin_shutdown(reason): - return controller.wait_for_lease_handling() - - self.stop_event.set() - self._cancel_active_tip_refresh_for_shutdown() - self._shutdown_log( - "shutdown_start", - reason=reason, - signal=controller.signal_number, - writer_quiescence_timeout_seconds=controller.writer_quiescence_timeout_seconds, - ) - - quiesced, elapsed, blockers = controller.wait_for_writer_quiescence() - self._shutdown_log( - "writer_quiescence", - duration_seconds=round(elapsed, 6), - outcome="success" if quiesced else "timeout", - blockers=blockers, - ) - if not quiesced: - for component, active_count in blockers.items(): - self._shutdown_log( - "lease_release_withheld", - component=component, - active_operations=active_count, - reason="writer_quiescence_timeout", - ) - return False - return self.release_ledger_lease() - - def release_ledger_lease( - self, - *, - fresh_connection: bool = False, - deadline: float | None = None, - emit_logs: bool = True, - ) -> bool: - """Release a quiesced writer lease at most once. - - The exact-session database fence makes an already-absent lease safe. - Exceptions remain best-effort: they are observable, never retried from - a duplicate finally block, and leave TTL fencing intact. - """ - shutdown_log: Callable[..., None] = ( - self._shutdown_log if emit_logs else lambda *_args, **_kwargs: None - ) - controller = self._ensure_shutdown_controller() - claimed, blockers = controller.claim_lease_release() - if not claimed: - if blockers: - shutdown_log( - "lease_release_withheld", - reason="active_writer_operations", - blockers=blockers, - ) - return controller.lease_release_succeeded - - if not self._stop_ledger_lease_heartbeat(deadline=deadline): - controller.finish_lease_release("failure", 0.0) - shutdown_log( - "lease_release", - duration_seconds=0.0, - outcome="heartbeat_stop_timeout", - released=False, - ) - return False - - ledger = getattr(self, "ledger", None) - release = ( - getattr(ledger, "release_writer_lease_fresh_connection", None) - if fresh_connection - else None - ) - if release is None: - release = getattr(ledger, "release_writer_lease", None) - shutdown_log( - "lease_release_attempt", - supported=release is not None, - fresh_connection=fresh_connection, - ) - if release is None: - controller.finish_lease_release("unsupported", 0.0) - shutdown_log( - "lease_release", - duration_seconds=0.0, - outcome="unsupported", - released=False, - ) - return True - started = time.monotonic() - try: - released = release() - except Exception: - elapsed = max(0.0, time.monotonic() - started) - controller.finish_lease_release("failure", elapsed) - shutdown_log( - "lease_release", - duration_seconds=round(elapsed, 6), - outcome="failure", - released=False, - ) - if emit_logs: - traceback.print_exc() - return False - elapsed = max(0.0, time.monotonic() - started) - outcome = "success" if released else "not_held" - controller.finish_lease_release(outcome, elapsed) - snapshot = controller.snapshot() - shutdown_log( - "lease_release", - duration_seconds=round(elapsed, 6), - outcome=outcome, - released=bool(released), - sigterm_to_release_seconds=( - round(float(snapshot["sigterm_to_lease_release_seconds"]), 6) - if snapshot["sigterm_release_observed"] - else None - ), - ) - return True - - def drain_non_writer_components( - self, - threads: list[tuple[threading.Thread, float]] | None = None, - ) -> None: - """Drain threads, fanout sends, and executors only after lease handling.""" - controller = self._ensure_shutdown_controller() - if not controller.claim_non_writer_drain(): - return - started = time.monotonic() - drain_threads: Sequence[tuple[threading.Thread, float]] - if threads is None: - # Registry-owned drain: only started registered services, in - # registration order. serve() passes an explicit list combining - # this with its non-registry threads; focused shutdown callers - # keep their historical explicit-thread seam. - drain_threads = self._ensure_background_services().threads_to_drain() - else: - drain_threads = threads - # The audit HTTP facade is demand-created; stop it only if it ever - # existed. Its bounded stop runs after lease release and before the - # non-writer thread joins so the listener port frees promptly. - audit_http = self.__dict__.get("_audit_http_facade") - if audit_http is not None and not audit_http.stop(): - self._shutdown_log( - "audit_http_stop", - outcome="timeout", - ) - for thread, timeout in drain_threads: - thread.join(timeout=timeout) - self.shutdown_vardiff_idle_executor() - self.shutdown_tip_refresh_executor() - elapsed = max(0.0, time.monotonic() - started) - controller.finish_non_writer_drain(elapsed) - self._shutdown_log( - "non_writer_drain", - duration_seconds=round(elapsed, 6), - lease_release_succeeded=controller.lease_release_succeeded, - outcome="complete", - ) - - def open_stratum_listeners( - self, listener_stack: ExitStack - ) -> list[tuple[socket.socket, StratumListenerProfile]] | None: - """Bind and listen on every stratum listener profile. - - Called before the slow parts of startup (qbit readiness, policy - validation, block-work recovery) so miners reconnecting through a - restart park in the kernel accept backlog instead of getting - connection refused, which sends firmware into reconnect backoff or - failover and costs hashrate. bind() retries EADDRINUSE for a bounded - window because a predecessor process may still hold the port while - draining its shutdown. Returns None when shutdown is requested during - the retry, so startup can abort gracefully. - """ - return StratumSessionService.open_stratum_listeners( - listener_stack, - self.listener_profiles, - backlog=int( - getattr( - self, - "stratum_listen_backlog", - DEFAULT_PRISM_STRATUM_LISTEN_BACKLOG, - ) - ), - retry_seconds=float( - getattr( - self, - "stratum_bind_retry_seconds", - DEFAULT_PRISM_STRATUM_BIND_RETRY_SECONDS, - ) - ), - stop_event=getattr(self, "stop_event", None), - ) - - def serve(self) -> None: - with ExitStack() as listener_stack: - self._serve_with_listener_stack(listener_stack) - - def _serve_with_listener_stack(self, listener_stack: ExitStack) -> None: - self._startup_phase_origin_monotonic = time.monotonic() - lease_heartbeat_thread = self._start_ledger_lease_heartbeat() - self._record_startup_phase_once("lease_heartbeat_started") - # Listeners come up first: connections complete their TCP handshake in - # the kernel backlog while the rest of startup runs, so a fast restart - # never bounces miners with connection refused. accept() still starts - # only after block-work recovery below. - listeners = self.open_stratum_listeners(listener_stack) - if listeners is None: - return - deadline = time.time() + 60 - while time.time() < deadline: - try: - self.rpc.call("getblockcount") - break - except Exception: - # A shutdown signal during the readiness wait must release the - # bound ports promptly, or a successor's bind retry window can - # expire against this process. - if self.stop_event.wait(1): - return - if self.stop_event.is_set(): - return - self._record_startup_phase_once("node_rpc_ready") - self.validate_live_chain_identity() - self.validate_live_template_and_fee_policy() - self.prism_payout_policy() - print( - f"prism coordinator: listening on {self.bind}:{self.port} " - f"share_diff={self.share_difficulty} ready_miners={self.min_ready_miners} " - f"vardiff={'on' if self.vardiff_config.enabled else 'off'} " - f"max_blocks={self.max_blocks} " - f"blockpoll={self.blockpoll_seconds:g}s " - f"version_mask={stratum_codec.format_mask_hex(self.version_mask)} " - f"version_mask_source={self.version_mask_selection.source}:{self.version_mask_selection.detail} " - f"ledger={self.ledger.backend_name} " - f"ledger_execution={getattr(self.ledger, 'execution_backend', self.ledger.backend_name)} " - f"hot_path_log={'on' if self.hot_path_log_enabled else 'off'}", - flush=True, - ) - for profile in self.listener_profiles[1:]: - print( - f"prism coordinator: {profile.name} listener on {profile.bind}:{profile.port} " - f"start_diff={profile.vardiff_config.startup_difficulty} " - f"min_diff={profile.vardiff_config.min_difficulty} " - f"max_diff={profile.vardiff_config.max_difficulty} " - f"share_diff={profile.share_difficulty}", - flush=True, - ) - if self.audit_bind and self.audit_port: - self.start_audit_server() - # Make a best-effort attempt to recover one block candidate before - # accepting Stratum connections. If a slow ledger exhausts the short - # database budget, finish startup and let the dedicated submitter loop - # retry every durable outbox row with its ordinary paced backoff. - if not self._run_startup_block_candidate_replay(): - return - if self.stop_event.is_set(): - return - self._record_block_work_heartbeat("block_submitter", "starting") - block_accounting_thread = self._start_block_accounting_thread() - self._start_background_service("block_submitter") - # Threads owned outside the background-service registry (the block - # accounting worker and the lease heartbeat pair below) keep their - # historical bounded joins alongside the registry-owned drains. - extra_drain_threads: list[tuple[threading.Thread, float]] = [ - (block_accounting_thread, 1.0), - ] - # Publication progress is mandatory even when the operator disables - # ordinary heartbeat checks. Start the watchdog before any synchronous - # startup prewarm/recovery work: a timeout-ignoring block call can ask - # for fail-stop while the main thread is itself wedged in that work. - # The specification is registered at this exact boundary (not in - # _make_background_service_registry) so serve() keeps owning the - # watchdog thread's creation order. - self._ensure_background_services().register_if_absent( - BackgroundServiceSpec( - name="watchdog", - thread_name="prism-watchdog", - target=self.watchdog_loop, - daemon=True, - join_timeout=1.0, - watchdog_monitored=False, - registration_identity=("watchdog_loop",), - ) - ) - self._start_background_service("watchdog") - prepared = self.prewarm_startup_jobs() - print( - "prism coordinator: startup job preparation " - f"status={'complete' if prepared is not None else 'deferred'} " - f"mode={'ready' if prepared is not None else 'collection'} " - f"tip={self.tip_template_snapshot.bestblockhash if self.tip_template_snapshot else 'unknown'}", - flush=True, - ) - # Seed listener liveness before accepting so the watchdog never fires - # during startup. Background-loop heartbeats derive from their service - # specifications at each named start below. - for _, profile in listeners: - self._record_heartbeat(profile.heartbeat_name) - self._start_background_service("qbit_blockpoll") - if self.blockwait_enabled: - self._start_background_service("qbit_blockwait") - if self.vardiff_idle_sweep_seconds > 0: - self._start_background_service("vardiff_idle_sweep") - if self.stratum_initial_job_timeout_seconds > 0: - self._start_background_service("initial_job_timeout_sweep") - if lease_heartbeat_thread is not None: - extra_drain_threads.append((lease_heartbeat_thread, 1.0)) - lease_heartbeat_monitor_thread = getattr( - self, - "_ledger_lease_heartbeat_monitor_thread", - None, - ) - if lease_heartbeat_monitor_thread is not None: - extra_drain_threads.append((lease_heartbeat_monitor_thread, 1.0)) - # Replay any shares stranded on disk by a prior ledger-outage - # shutdown before serving, so no acked share is lost across restart. - if not self._run_startup_writer_replay( - self.replay_recovered_shares, - drain_threads=[ - *self._ensure_background_services().threads_to_drain(), - *extra_drain_threads, - ], - ): - return - self.share_writer_active = True - self._start_background_service("share_writer") - if self.ctv_broadcaster_enabled: - self._start_background_service("ctv_fanout_broadcaster") - print( - "prism coordinator: CTV fanout broadcaster enabled " - f"mode={'cpfp' if self.ctv_broadcaster_fee_sats > 0 else 'direct'} " - f"fee_bits={self.ctv_broadcaster_fee_sats} " - f"wallet={'configured' if self.ctv_broadcaster_wallet else 'none'} " - f"interval={self.ctv_broadcaster_interval_seconds:g}s " - f"limit={self.ctv_broadcaster_limit} " - f"chunk_size={self.ctv_broadcaster_chunk_size}", - flush=True, - ) - self._start_hashrate_rollup_maintenance_if_enabled() - if self.watchdog_enabled: - print( - "prism coordinator: liveness and publication-progress watchdog enabled " - f"timeout={self.watchdog_timeout_seconds:g}s " - f"publication_budget={self.template_refresh_failure_exit_seconds:g}s " - f"coordination_blocked_budget={self.coordination_blocked_exit_seconds:g}s " - f"interval={self.watchdog_interval_seconds:g}s", - flush=True, - ) - else: - print( - "prism coordinator: publication-progress watchdog enabled " - f"budget={self.template_refresh_failure_exit_seconds:g}s " - f"coordination_blocked_budget={self.coordination_blocked_exit_seconds:g}s " - f"interval={self.watchdog_interval_seconds:g}s", - flush=True, - ) - # Load durable worker difficulty evidence before either listener starts - # accepting. The service performs one bounded startup read and then - # serves authorizations from its in-memory cache, so a miner's first - # request never waits on PostgreSQL. - vardiff_service = self._ensure_vardiff_service() - vardiff_service.start_durable_prune_scheduler() - for extra_server, extra_profile in listeners[1:]: - self._start_secondary_accept_service(extra_server, extra_profile) - try: - self.accept_loop(*listeners[0]) - finally: - # Free the listen ports the moment accepting stops so a successor - # process can bind while the shutdown drain below runs. - for server, _ in listeners: - try: - server.close() - except OSError: - pass - # The writer barrier and lease release intentionally precede - # joins and the tip-refresh executor drain: those may be stuck - # in unrelated client delivery or obsolete fanout work. - self.shutdown(reason="serve_exit") - self.drain_non_writer_components( - [ - *self._ensure_background_services().threads_to_drain(), - *extra_drain_threads, - ] - ) - - def _record_startup_phase_once(self, phase: str) -> None: - """Record seconds from serve() start to a named startup phase, once. - - Lease adoption, accepted-stat warm-up, audit binding, replay - enumeration, first tip template, and first delivered job are timed - separately so a slow phase is attributable instead of folding into - one opaque time-to-ready figure (issue #188). - """ - origin = getattr(self, "_startup_phase_origin_monotonic", None) - if origin is None: - return - elapsed = max(0.0, time.monotonic() - float(origin)) - # Lightweight embedders (signal-handling startup tests) can reach - # this before dataclass construction installs self.lock. - with getattr(self, "lock", None) or _HOT_PATH_LOCK_INITIALIZATION_LOCK: - phases = getattr(self, "_startup_phase_seconds", None) - if phases is None: - phases = {} - self._startup_phase_seconds = phases - if phase not in phases: - phases[phase] = elapsed - - def startup_phase_seconds(self) -> dict[str, float]: - with getattr(self, "lock", None) or _HOT_PATH_LOCK_INITIALIZATION_LOCK: - phases = getattr(self, "_startup_phase_seconds", None) - return dict(phases) if phases is not None else {} - - def _note_block_replay_enumeration_owed(self) -> None: - """Mark that pending durable candidates have not been enumerated yet.""" - self._ensure_block_candidate_service()._note_block_replay_enumeration_owed() - - def _clear_block_replay_enumeration_owed(self) -> None: - self._ensure_block_candidate_service()._clear_block_replay_enumeration_owed() - - def _block_replay_enumeration_owed(self) -> bool: - return self._ensure_block_candidate_service()._block_replay_enumeration_owed() - - def _run_startup_writer_replay( - self, - replay: Callable[[], int], - *, - drain_threads: list[tuple[threading.Thread, float]] | None = None, - ) -> bool: - """Run startup ledger replay, stopping cleanly if shutdown wins.""" - try: - replay() - except ShutdownInProgress: - if drain_threads is not None: - self.shutdown(reason="serve_startup_exit") - self.drain_non_writer_components(drain_threads) - return False - return True - - def _run_startup_block_candidate_replay(self) -> bool: - """Run best-effort pre-accept replay without dying on a slow ledger.""" - return self._ensure_block_candidate_service()._run_startup_block_candidate_replay() - - def accept_loop(self, server: socket.socket, profile: StratumListenerProfile) -> None: - self._ensure_stratum_session_service().accept_loop(server, profile) - - def _record_stratum_resource_exhaustion( - self, - *, - listener_name: str, - location: str, - error_number: int | None, - ) -> int: - return self._ensure_stratum_session_service().record_stratum_resource_exhaustion( - listener_name=listener_name, - location=location, - error_number=error_number, - ) - - def _wait_after_stratum_resource_failure(self, heartbeat_name: str) -> None: - self._ensure_stratum_session_service().wait_after_stratum_resource_failure( - heartbeat_name - ) - - def _ensure_connection_capacity_state(self) -> None: - self._ensure_stratum_session_service() - - def _note_connection_limit_rejection_locked(self, scope: str) -> int: - return self._ensure_stratum_session_service().registry.note_rejection_locked( - scope - ) - - def reserve_client_username(self, client: ClientState, worker: WorkerIdentity) -> bool: - """Atomically reserve an exact Stratum username for one connection.""" - return self._ensure_stratum_session_service().reserve_client_username( - client, worker - ) - - def start_audit_server(self) -> None: - # Mark the nonblocking refreshers running before the listener can - # dispatch a request: a probe retrying connection-refused queues in - # the accept backlog at listen() time, and a handler serving it - # before the flag is set would take cached_health_payload's legacy - # inline path -- running the minutes-long accepted-share aggregate - # on the handler thread and racing the refresher with a duplicate - # query instead of returning the documented starting state. - self.start_health_snapshot_refresher() - self.start_metrics_snapshot_refresher() - # Bind before the health snapshot warm-up completes: the cold - # accepted-share aggregate can take minutes on a grown ledger, and - # for that whole window the listener must answer with an explicit - # starting state instead of connection refused (issue #188 fix 4). - self._ensure_audit_http_facade().start() - self._record_startup_phase_once("audit_listener_bound") - print( - f"prism coordinator: audit HTTP listening on {self.audit_bind}:{self.audit_port}", - flush=True, - ) - - def apply_stratum_send_timeout(self, sock: socket.socket) -> None: - """Bound blocking sends to miners without touching receive semantics. - - Job refreshes use a bounded executor, but an unresponsive peer whose - TCP buffer is full must still release its worker eventually. - SO_SNDTIMEO turns that into an OSError, which the refresh path treats - as a dead client without failing delivery to other miners. - A plain socket timeout is not usable here: it would also apply to - recv, disconnecting idle-but-healthy miners. - """ - apply_socket_send_timeout( - sock, - getattr( - self, - "stratum_send_timeout_seconds", - DEFAULT_PRISM_STRATUM_SEND_TIMEOUT_SECONDS, - ), - ) - - def _wait_for_blockpoll_trigger(self) -> bool: - return self._ensure_tip_refresh_service()._wait_for_blockpoll_trigger() - - def blockpoll_loop(self) -> None: - return self._ensure_tip_refresh_service().blockpoll_loop() - - def template_refresh_failure_expired(self, now: float) -> bool: - return self._ensure_tip_refresh_service().template_refresh_failure_expired(now) - - def publication_progress_failure_expired(self, now: float) -> bool: - return self._ensure_tip_refresh_service().publication_progress_failure_expired( - now, - ) - - def _record_coordination_blocked_refresh(self, now: float) -> None: - self._ensure_watchdog_service().record_coordination_blocked_refresh(now) - - def _clear_coordination_blocked_streak(self) -> None: - self._ensure_watchdog_service().clear_coordination_blocked_streak() - - def coordination_blocked_streak_age_seconds( - self, - now: float | None = None, - ) -> float: - return ( - self._ensure_watchdog_service().coordination_blocked_streak_age_seconds( - now - ) - ) - - def coordination_blocked_streak_expired(self, now: float) -> bool: - return ( - self._ensure_watchdog_service().coordination_blocked_streak_expired( - now - ) - ) - - def _publication_watchdog_state( - self, - now: float, - ) -> tuple[str | None, float, float, float]: - return self._ensure_watchdog_service().publication_watchdog_state(now) - - def _record_template_refresh_failure(self, now: float) -> None: - return self._ensure_tip_refresh_service()._record_template_refresh_failure(now) - - def blockwait_once(self, known_tip: str) -> str: - return self._ensure_tip_refresh_service().blockwait_once(known_tip) - - def blockwait_loop(self) -> None: - return self._ensure_tip_refresh_service().blockwait_loop() - - @staticmethod - def _blockwait_unsupported(exc: Exception) -> bool: - detail = str(exc).lower() - return ( - "-32601" in detail - or "-32602" in detail - or "method not found" in detail - or "unknown method" in detail - or "invalid params" in detail - or "invalid parameter" in detail - or "wrong number of" in detail - or "too many parameters" in detail - or "incorrect number of" in detail - ) - - def make_ctv_fanout_broadcast_daemon(self) -> CtvFanoutBroadcastDaemon: - return self._ensure_ctv_runtime().make_daemon() - - def run_ctv_fanout_broadcaster_once( - self, - *, - progress_callback: Callable[[], None] | None = None, - ) -> CtvFanoutDaemonResult: - return self._ensure_ctv_runtime().run_once( - progress_callback=progress_callback, - chunk_callback=self.observe_ctv_fanout_broadcaster_chunk, - ) - - def ctv_fanout_broadcaster_loop(self) -> None: - self._ensure_ctv_runtime().loop( - run_once=self.run_ctv_fanout_broadcaster_once, - progress_callback=self._record_ctv_fanout_broadcaster_progress, - observe_pass=self.observe_ctv_fanout_broadcaster_pass, - record_yield=self._record_ctv_fanout_broadcaster_yield, - ) - - def _start_hashrate_rollup_maintenance_if_enabled(self) -> None: - """Start rollup maintenance when configured on and the ledger can. - - The capability check mirrors the registration conditional: a - memory-backed ledger has no ``advance_hashrate_rollups`` and no - registered service to start, so asking the registry for the name - would fail rather than no-op. Lightweight test/embedder - coordinators built without the full constructor carry neither the - flag nor a capable ledger, so an absent flag reads as disabled. - """ - if not getattr(self, "hashrate_rollup_enabled", False): - return - if not callable( - getattr(getattr(self, "ledger", None), "advance_hashrate_rollups", None) - ): - return - self._start_background_service("hashrate-rollup-maintenance") - print( - "prism coordinator: hashrate rollup maintenance enabled " - f"interval={self.hashrate_rollup_interval_seconds:g}s " - f"batch={self.hashrate_rollup_batch_shares}", - flush=True, - ) - - def hashrate_rollup_maintenance_loop(self) -> None: - """Advance the incremental hashrate rollups until shutdown. - - Each pass folds one watermarked batch into the rollup tables and - keeps going while the ledger reports more history behind the - watermark -- which is exactly how a cold database backfills itself: - the first passes replay all existing shares through the same - statement live shares take, so there is no separate migration to - operate. A failed pass is logged and retried on the next interval; - the watermark guarantees a retry can never double-count what a - partial pass already folded in. - - Every pass runs inside the writer-operation admission, like every - other mutation of ledger-derived state: shutdown quiescence must - count an in-flight pass, and admission must refuse a new one, or a - pass slipping past the stop check could renew the just-released - lease -- the maintenance statement renews on exact identity, which - an orderly release keeps -- and mutate rollups after shutdown - reported the writer quiesced. A refused admission is the orderly - stop, not a failure. - """ - while not self.stop_event.is_set(): - try: - while not self.stop_event.is_set(): - with self._writer_operation("hashrate_rollup_maintenance"): - result = self.ledger.advance_hashrate_rollups( - batch_limit=self.hashrate_rollup_batch_shares, - ) - print( - "prism coordinator: hashrate rollup advance " - f"scanned={result['scanned']} " - f"last_share_seq={result['last_share_seq']} " - f"caught_up={'true' if result['caught_up'] else 'false'}", - flush=True, - ) - if result["caught_up"]: - break - except ShutdownInProgress: - return - except Exception: - print( - "prism coordinator: hashrate rollup maintenance pass failed", - flush=True, - ) - traceback.print_exc() - if self.stop_event.wait(self.hashrate_rollup_interval_seconds): - break - - def _tip_refresh_artifacts( - self, - snapshot: QbitTipTemplateSnapshot, - ) -> CachedTemplateArtifacts: - return self._ensure_tip_refresh_service()._tip_refresh_artifacts(snapshot) - - def prepare_tip_refresh_bundle( - self, - snapshot: QbitTipTemplateSnapshot, - *, - priority_requested_monotonic: float | None = None, - ) -> CachedJobBundle: - return self._ensure_tip_refresh_service().prepare_tip_refresh_bundle( - snapshot, - priority_requested_monotonic=priority_requested_monotonic, - ) - - def prewarm_current_tip_ready_bundle(self) -> CachedJobBundle | None: - return self._ensure_tip_refresh_service().prewarm_current_tip_ready_bundle() - - def prewarm_startup_jobs(self) -> CachedJobBundle | None: - return self._ensure_tip_refresh_service().prewarm_startup_jobs() - - def _tip_refresh_token_current_locked( - self, - token: TipRefreshValidationToken, - bundle: CachedJobBundle, - snapshot: QbitTipTemplateSnapshot, - ) -> bool: - return self._ensure_tip_refresh_service()._tip_refresh_token_current_locked( - token, - bundle, - snapshot, - ) - - def _tip_refresh_token_prepublication_current_locked( - self, - token: TipRefreshValidationToken, - bundle: CachedJobBundle, - snapshot: QbitTipTemplateSnapshot, - ) -> bool: - return self._ensure_tip_refresh_service()._tip_refresh_token_prepublication_current_locked( - token, - bundle, - snapshot, - ) - - def _tip_refresh_snapshot_current_locked( - self, - snapshot: QbitTipTemplateSnapshot, - observation_sequence: int, - ) -> bool: - return self._ensure_tip_refresh_service()._tip_refresh_snapshot_current_locked( - snapshot, - observation_sequence, - ) - - def _validate_prepared_tip_refresh( - self, - bundle: CachedJobBundle, - snapshot: QbitTipTemplateSnapshot, - observation_sequence: int, - ) -> TipRefreshValidationToken: - return self._ensure_tip_refresh_service()._validate_prepared_tip_refresh( - bundle, - snapshot, - observation_sequence, - ) - - def _activate_tip_refresh( - self, - token: TipRefreshValidationToken, - bundle: CachedJobBundle, - snapshot: QbitTipTemplateSnapshot, - cancel_event: _FanoutCancellation, - ) -> None: - return self._ensure_tip_refresh_service()._activate_tip_refresh( - token, - bundle, - snapshot, - cancel_event, - ) - - def _publish_prepared_tip_refresh( - self, - token: TipRefreshValidationToken, - bundle: CachedJobBundle, - snapshot: QbitTipTemplateSnapshot, - *, - parent_hash: str | None, - ) -> _FanoutCancellation: - return self._ensure_tip_refresh_service()._publish_prepared_tip_refresh( - token, - bundle, - snapshot, - parent_hash=parent_hash, - ) - - def _clear_active_tip_refresh( - self, - token: TipRefreshValidationToken, - cancel_event: _FanoutCancellation, - ) -> None: - return self._ensure_tip_refresh_service()._clear_active_tip_refresh( - token, - cancel_event, - ) - - def _prepared_tip_refresh_obsolete( - self, - validation_token: TipRefreshValidationToken, - bundle: CachedJobBundle, - snapshot: QbitTipTemplateSnapshot, - cancel_event: _FanoutCancellation | None, - ) -> bool: - return self._ensure_tip_refresh_service()._prepared_tip_refresh_obsolete( - validation_token, - bundle, - snapshot, - cancel_event, - ) - - def send_prepared_job( - self, - client: ClientState, - bundle: CachedJobBundle, - snapshot: QbitTipTemplateSnapshot, - validation_token: TipRefreshValidationToken, - expected_connection_id: int, - expected_active_job: PrismJobContext | None, - cancel_event: _FanoutCancellation | None = None, - submitted_monotonic: float | None = None, - ) -> RefreshResult: - return self._ensure_job_delivery_service().send_prepared_job( - client, - bundle, - snapshot, - validation_token, - expected_connection_id, - expected_active_job, - cancel_event, - submitted_monotonic, - ) - - def _fanout_prepared_tip_refresh( - self, - clients: list[ClientState], - bundle: CachedJobBundle, - snapshot: QbitTipTemplateSnapshot, - *, - observation_sequence: int | None = None, - validation_token: TipRefreshValidationToken | None = None, - preactivated_cancel_event: _FanoutCancellation | None = None, - executor: ThreadPoolExecutor | None = None, - expected_active_jobs: dict[ClientState, PrismJobContext | None] | None = None, - heartbeat_name: str, - ) -> tuple[int, float | None, float | None, int]: - return self._ensure_tip_refresh_service()._fanout_prepared_tip_refresh( - clients, - bundle, - snapshot, - observation_sequence=observation_sequence, - validation_token=validation_token, - preactivated_cancel_event=preactivated_cancel_event, - executor=executor, - expected_active_jobs=expected_active_jobs, - heartbeat_name=heartbeat_name, - ) - - def _interrupted_wave_outcome(self, superseded_outcome: str) -> str: - return self._ensure_tip_refresh_service()._interrupted_wave_outcome( - superseded_outcome, - ) - - def _tip_refresh_wave_reenters(self, completed_passes: int) -> bool: - return self._ensure_tip_refresh_service()._tip_refresh_wave_reenters( - completed_passes, - ) - - def poll_qbit_tip_template_once( - self, - *, - heartbeat_name: str = "qbit_blockpoll", - ) -> int: - return self._ensure_tip_refresh_service().poll_qbit_tip_template_once( - heartbeat_name=heartbeat_name, - ) - - def _poll_qbit_tip_template_pass_once( - self, - *, - heartbeat_name: str, - refresh_started: float, - observation_sequence: int | None = None, - observed_best_tip: str | None = None, - ) -> int: - return self._ensure_tip_refresh_service()._poll_qbit_tip_template_pass_once( - heartbeat_name=heartbeat_name, - refresh_started=refresh_started, - observation_sequence=observation_sequence, - observed_best_tip=observed_best_tip, - ) - - def _probe_tip_while_refresh_waiting(self) -> None: - return self._ensure_tip_refresh_service()._probe_tip_while_refresh_waiting() - - def _detected_tip_supersedes_locked( - self, - tip_hash: str, - observation_sequence: int, - ) -> bool: - return self._ensure_tip_refresh_service()._detected_tip_supersedes_locked( - tip_hash, - observation_sequence, - ) - - def _raise_if_tip_refresh_superseded( - self, - snapshot: QbitTipTemplateSnapshot, - observation_sequence: int, - ) -> None: - return self._ensure_tip_refresh_service()._raise_if_tip_refresh_superseded( - snapshot, - observation_sequence, - ) - - def _reserve_tip_observation_sequence(self) -> int: - return self._ensure_tip_refresh_service()._reserve_tip_observation_sequence() - - def observe_tip_for_refresh( - self, - tip_hash: str, - *, - observation_sequence: int | None = None, - mark_pending: bool = True, - ) -> bool: - return self._ensure_tip_refresh_service().observe_tip_for_refresh( - tip_hash, - observation_sequence=observation_sequence, - mark_pending=mark_pending, - ) - - def observe_tip_first_seen( - self, - tip_hash: str, - *, - observation_sequence: int | None = None, - publish_refresh_observation: bool = False, - published_snapshot: QbitTipTemplateSnapshot | None = None, - ) -> bool: - return self._ensure_tip_refresh_service().observe_tip_first_seen( - tip_hash, - observation_sequence=observation_sequence, - publish_refresh_observation=publish_refresh_observation, - published_snapshot=published_snapshot, - ) - - def _fetch_tip_parent_hash(self, tip_hash: str) -> str | None: - return self._ensure_tip_refresh_service()._fetch_tip_parent_hash(tip_hash) - - def current_tip_parent_hash(self, tip_hash: str) -> str | None: - return self._ensure_tip_refresh_service().current_tip_parent_hash(tip_hash) - - def submit_stale_check_tip(self) -> str: - return self._ensure_tip_refresh_service().submit_stale_check_tip() - - def _submit_stale_check_tip_locked(self, now: float) -> str | None: - """Return the authoritative published submit tip while holding self.lock.""" - if not self._published_tip_authoritative_locked(now): - return None - observed = getattr(self, "current_tip_first_seen", None) - assert observed is not None - return str(observed[0]) - - def _submit_control_snapshot( - self, - client: ClientState, - job_id: str, - ) -> tuple[bool, PrismJobContext | None, str | None]: - """Snapshot normal-submit control state in one bounded lock hold. - - Pool closure, active-job membership, and published-tip authority must - each be point-in-time consistent with their control-plane writers, but - they do not need three separate admissions through the same lock. The - caller performs live-tip RPC fallback, stale-grace classification, - hashing, persistence, and accounting only after this lock is released. - """ - with self.lock: - pool_closed = self.accepted_block_count >= self.max_blocks - context = self.jobs.get(job_id) - if context is not None and job_id not in client.active_job_ids: - context = None - published_tip = self._submit_stale_check_tip_locked(time.monotonic()) - return pool_closed, context, published_tip - - def _published_tip_authoritative_locked(self, now: float) -> bool: - return self._ensure_tip_refresh_service()._published_tip_authoritative_locked( - now, - ) - - def stale_grace_deadline_open( - self, - client: ClientState, - current_tip: str, - now: float | None = None, - ) -> bool: - grace_seconds = float(getattr(self, "stale_grace_seconds", DEFAULT_PRISM_STALE_GRACE_SECONDS)) - if grace_seconds <= 0: - return False - now = time.monotonic() if now is None else now - with self.lock: - first_seen = getattr(self, "current_tip_first_seen", None) - delivered = client.tip_work_delivered - # Only successful refresh publication anchors current_tip_first_seen. - # If this tip is merely detected, the window is not open: self-healing - # from a lagging submit's RPC read would extend grace arbitrarily past - # the real publication boundary. Fall through to stale-job instead. - if first_seen is None or first_seen[0] != current_tip: - return False - # A None stamp is the startup baseline (see observe_tip_first_seen): the - # tip did not just flip, so there is no in-flight prior-tip work to - # rescue and the window stays closed. - if first_seen[1] is None: - return False - if delivered is not None and delivered[0] == current_tip: - # This connection already received current-tip work: its window runs - # from that delivery, so a slow refresh pass cannot strand shares - # that were in flight when replacement work finally arrived. - return now - delivered[1] <= grace_seconds - # The refresh path saw the flip but has not delivered current-tip work - # to this connection yet (slow pass, aborted reorg reconcile, transient - # build failure). Its prior-tip shares are still in flight; keep the - # window open. Bounded by the exactly-one-tip-back parent rule at the - # next flip, by delivery (which starts the grace clock above), and by - # disconnect when sends to the client fail. - return True - - def context_eligible_for_stale_grace( - self, - client: ClientState, - context: PrismJobContext, - current_tip: str, - ) -> bool: - if not self.stale_grace_deadline_open(client, current_tip): - return False - parent_hash = self.current_tip_parent_hash(current_tip) - return bool(parent_hash) and str(context.template["previousblockhash"]) == parent_hash - - def note_tip_work_delivered(self, client: ClientState, job_parent_hash: str) -> None: - return self._ensure_job_delivery_service().note_tip_work_delivered( - client, - job_parent_hash, - ) - - def _ensure_evicted_job_state(self) -> None: - # Compatibility initializer: the retained-job index is owned by - # JobDeliveryService. Ensuring it adopts legacy/test-seeded graveyard - # state (including pre-owner tuple entries) and rebuilds the - # per-connection/same-tip/disconnected indexes for the current tip. - self._ensure_job_delivery_service().ensure_evicted_job_state() - - def _current_published_tip_hash_locked(self) -> str | None: - first_seen = getattr(self, "current_tip_first_seen", None) - if first_seen is not None: - return str(first_seen[0]) - snapshot = getattr(self, "tip_template_snapshot", None) - if snapshot is not None: - return str(snapshot.bestblockhash) - return None - - def _evicted_job_class_locked(self, entry: EvictedJobEntry) -> str: - return self._ensure_job_delivery_service()._evicted_job_class_locked(entry) - - def bury_evicted_job( - self, - client: ClientState, - job_id: str, - *, - now: float | None = None, - prune: bool = True, - ) -> None: - return self._ensure_job_delivery_service().bury_evicted_job( - client, - job_id, - now=now, - prune=prune, - ) - - def prune_evicted_job_graveyard( - self, - *, - now: float | None = None, - force: bool = True, - ) -> None: - return self._ensure_job_delivery_service().prune_evicted_job_graveyard( - now=now, - force=force, - ) - - def evicted_job_entry( - self, - client: ClientState, - job_id: str, - ) -> EvictedJobEntry | None: - return self._ensure_job_delivery_service().evicted_job_entry(client, job_id) - - def evicted_submit_context( - self, - client: ClientState, - entry: EvictedJobEntry, - current_tip: str, - ) -> tuple[PrismJobContext, str | None] | None: - context = entry.context - if str(context.template["previousblockhash"]) == current_tip: - return context, None - if entry.connection_id != client.connection_id: - # Cross-connection resumes are same-tip only. A tip that moves - # between the graveyard lookup and this classification must not - # fall through to stale grace: that window anchors on the - # submitting connection's delivery state, which says nothing - # about work another (dead) connection delivered. - return None - if not self.context_eligible_for_stale_grace(client, context, current_tip): - return None - return context, PRISM_CREDIT_POLICY_STALE_GRACE - - def note_evicted_job_submit( - self, - credit_policy: str | None, - *, - cross_connection: bool = False, - ) -> None: - return self._ensure_job_delivery_service().note_evicted_job_submit( - credit_policy, - cross_connection=cross_connection, - ) - - def refresh_jobs_after_pending_accepted_block( - self, - client: ClientState, - *, - heartbeat_name: str = "qbit_blockpoll", - ) -> int: - return self._ensure_tip_refresh_service().refresh_jobs_after_pending_accepted_block( - client, - heartbeat_name=heartbeat_name, - ) - - def refresh_jobs_after_accepted_block( - self, *, block_height: int, block_hash: str, heartbeat_name: str = "qbit_blockpoll" - ) -> int: - return self._ensure_tip_refresh_service().refresh_jobs_after_accepted_block( - block_height=block_height, - block_hash=block_hash, - heartbeat_name=heartbeat_name, - ) - - def fetch_qbit_tip_template_snapshot(self) -> QbitTipTemplateSnapshot: - return self._ensure_job_bundle_service().template_repository.fetch_coherent_snapshot() - - def _note_reorg_reconcile_outcome( - self, - tip_hash: str | None, - *, - trusted: bool, - clear_memo: bool = False, - evict_others: bool = False, - proof_epoch: int | None = None, - ) -> None: - """Compatibility seam for the extracted reorg reconciler owner.""" - self._ensure_reorg_reconciler_service().note_outcome( - tip_hash, - trusted=trusted, - clear_memo=clear_memo, - evict_others=evict_others, - proof_epoch=proof_epoch, - ) - - def _evict_reorg_reconcile_memo_for_new_tip_locked( - self, - tip_hash: str, - ) -> None: - self._ensure_reorg_reconciler_service().evict_memo_for_new_tip_locked( - tip_hash - ) - - def ensure_reorg_reconciled_for_current_tip( - self, - *, - expected_tip_hash: str | None = None, - ) -> bool: - return self._ensure_reorg_reconciler_service().ensure_current( - expected_tip_hash=expected_tip_hash - ) - - def _reorg_reconcile_memo_fresh(self, tip_hash: str) -> bool: - return self._ensure_reorg_reconciler_service().memo_fresh(tip_hash) - - def _record_reorg_reconcile_lookup(self, path: str, source: str) -> None: - self._ensure_reorg_reconciler_service().record_lookup(path, source) - - @property - def reorg_reconcile_lookup_counts(self) -> dict[tuple[str, str], int]: - """Copied lookup counters from the reorg owner (metrics consumer).""" - return ( - self._ensure_reorg_reconciler_service() - .reorg_reconcile_lookup_snapshot() - ) - - def _submit_reconcile_prefetch( - self, - tip_hash: str, - *, - prove: bool = False, - ) -> Future[bool] | None: - return self._ensure_reorg_reconciler_service().submit_prefetch( - tip_hash, - prove=prove, - ) - - @staticmethod - def _discard_stale_reconcile_prefetch(future: Future[bool]) -> None: - ReorgReconcilerService.discard_stale_prefetch(future) - - def _join_reconcile_prefetch_bounded(self, prefetch: Future[bool]) -> bool: - return self._ensure_reorg_reconciler_service().join_prefetch_bounded( - prefetch - ) - - def _reconcile_snapshot_tip_bounded(self, tip_hash: str) -> bool: - return self._ensure_reorg_reconciler_service().snapshot_tip_bounded( - tip_hash - ) - - def shutdown_reconcile_prefetch_executor(self) -> None: - self._ensure_reorg_reconciler_service().shutdown_prefetch_executor() - - def ensure_reorg_reconciled_for_tip( - self, - tip_hash: str, - *, - _coalesce_same_tip: bool = True, - _caller: str | None = None, - ) -> bool: - """Reconcile one tip, optionally bypassing same-tip flight reuse. - - Lock-owning accepted-block callers disable waiting for an existing - leader because it may itself be waiting for the payout-balance - mutation lock. Their own pass remains visible to ordinary followers. - ``_caller`` optionally names the #224 attribution caller; the - service infers ``landing`` from the flight bypass when omitted. - """ - return self._ensure_reorg_reconciler_service().ensure_tip( - tip_hash, - _coalesce_same_tip=_coalesce_same_tip, - _caller=_caller, - ) - - def qbit_chain_view_untrusted(self) -> bool: - return reorg_chain_view_untrusted( - lambda *args, **kwargs: self.rpc.call(*args, **kwargs), - str(getattr(self, "qbit_chain", "regtest")), - ) - - def validate_live_chain_identity(self) -> None: - """Fail closed when a public-chain node is wrong, isolated, or behind.""" - configured = str(getattr(self, "qbit_chain", "regtest")).strip().lower() - info = self.rpc.call("getblockchaininfo") - if not isinstance(info, dict): - raise RuntimeError("getblockchaininfo returned non-object") - reported = str(info.get("chain", "")).strip().lower() - aliases = { - "main": {"main", "mainnet"}, - "mainnet": {"main", "mainnet"}, - } - allowed = aliases.get(configured, {configured}) - if reported not in allowed: - raise RuntimeError( - f"configured qbit chain {configured!r} does not match RPC chain {reported!r}" - ) - - expected_genesis = env_optional("QBIT_EXPECTED_GENESIS_HASH") - if configured in {"main", "mainnet"} and expected_genesis is None: - raise RuntimeError("QBIT_EXPECTED_GENESIS_HASH is required on mainnet") - if expected_genesis is not None: - expected_genesis = validate_hex( - expected_genesis, - name="QBIT_EXPECTED_GENESIS_HASH", - expected_bytes=32, - ) - live_genesis = str(self.rpc.call("getblockhash", [0])).lower() - if live_genesis != expected_genesis: - raise RuntimeError( - "QBIT_EXPECTED_GENESIS_HASH does not match the connected qbit node" - ) - - if configured not in {"main", "mainnet", *TESTNET_QBIT_CHAINS}: - return - if info.get("initialblockdownload") is not False: - raise RuntimeError("public-chain qbitd is still in initial block download") - try: - blocks = int(info["blocks"]) - headers = int(info["headers"]) - except (KeyError, TypeError, ValueError) as exc: - raise RuntimeError("public-chain qbitd did not report numeric blocks and headers") from exc - if blocks < 0 or headers < 0: - raise RuntimeError("public-chain qbitd reported negative blocks or headers") - if blocks != headers: - raise RuntimeError(f"public-chain qbitd is not caught up: blocks={blocks}, headers={headers}") - network_info = self.rpc.call("getnetworkinfo") - if not isinstance(network_info, dict): - raise RuntimeError("getnetworkinfo returned non-object") - try: - connections = int(network_info["connections"]) - except (KeyError, TypeError, ValueError) as exc: - raise RuntimeError("public-chain qbitd did not report a numeric peer count") from exc - minimum_peers = env_positive_int("PRISM_MIN_PEERS", 1) - if connections < minimum_peers: - raise RuntimeError( - f"public-chain qbitd has {connections} peers, requires at least {minimum_peers}" - ) - - @staticmethod - def rpc_fee_rate_bits_per_1000_weight(value: object, *, field: str) -> int: - try: - fee_rate = Decimal(str(value)) - except Exception as exc: - raise RuntimeError(f"{field} is not a decimal fee rate") from exc - if not fee_rate.is_finite() or fee_rate <= 0: - raise RuntimeError(f"{field} is not a positive fee rate") - return int( - (fee_rate * Decimal(100_000_000)).to_integral_value(rounding=ROUND_CEILING) - ) - - def validate_live_template_and_fee_policy(self) -> None: - artifacts = self.current_template_artifacts() - template = artifacts.template - if not artifacts.previousblockhash: - raise RuntimeError("getblocktemplate.previousblockhash was missing") - try: - template_time = int(template["curtime"]) - except (KeyError, TypeError, ValueError) as exc: - raise RuntimeError("getblocktemplate.curtime was missing or not numeric") from exc - max_age = env_nonnegative_int( - "PRISM_TEMPLATE_MAX_AGE_SECONDS", - DEFAULT_PRISM_TEMPLATE_MAX_AGE_SECONDS, - ) - template_age = int(time.time()) - template_time - if template_age > max_age: - raise RuntimeError( - f"qbit block template is stale: age={template_age}s exceeds {max_age}s" - ) - self.last_successful_template_refresh_monotonic = time.monotonic() - - settlement = self.prism_ctv_settlement_config( - block_height=int(template["height"]) if "height" in template else None, - parent_hash=artifacts.previousblockhash, - ) - if settlement is None: - return - policy = settlement["fanout_fee_rate_policy"] - assert isinstance(policy, dict) - configured_rate = int(policy["market_fee_rate_sats_per_1000_weight"]) - mempool_info = self.rpc.call("getmempoolinfo") - if not isinstance(mempool_info, dict): - raise RuntimeError("getmempoolinfo returned non-object") - relay_floors = [ - self.rpc_fee_rate_bits_per_1000_weight(mempool_info[name], field=name) - for name in ("minrelaytxfee", "mempoolminfee") - if mempool_info.get(name) is not None - ] - if not relay_floors: - raise RuntimeError("getmempoolinfo did not report a relay fee floor") - required_rate = max(relay_floors) - if configured_rate < required_rate: - raise RuntimeError( - "PRISM CTV fanout fee rate is below the connected node relay floor: " - f"configured={configured_rate} required={required_rate} bits/1000 weight" - ) - - def reconcile_prism_pool_blocks_once( - self, - *, - tip_hash: str | None = None, - _force_publish: bool = False, - _source_reserved: bool = False, - _wait_for_same_tip_flight: bool = True, - _caller: str | None = None, - ) -> dict[str, object]: - """Public reconcile entrypoint: service flight admission runs first. - - Same-tip flight coalescing happens *before* the serialized adapter's - writer admission so followers never queue behind it; see the service - docstring for the full contract. ``_caller`` optionally names the - #224 attribution caller; a forced publication without one is the - post-confirm path. - """ - return self._ensure_reorg_reconciler_service().reconcile_with_flights( - tip_hash=tip_hash, - _force_publish=_force_publish, - _source_reserved=_source_reserved, - _wait_for_same_tip_flight=_wait_for_same_tip_flight, - _caller=_caller, - ) - - @ledger_writer_operation("payout_reconciliation") - def _reconcile_prism_pool_blocks_serialized( - self, - *, - tip_hash: str | None = None, - _force_publish: bool = False, - _source_reserved: bool = False, - ) -> dict[str, object]: - """Serialize reconciliation against accepted-block finalization. - - Cross-owner adapter retained by the coordinator: writer-admission - decorator, then the direct payout-balance mutation lock, then the - landed-preview fail-closed check, then the service core. Never - replace this sequence with the _payout_balance_mutation() wrapper. - """ - self._ensure_job_cache_state() - with self._payout_balance_mutation_lock: - with self._accepted_block_payout_preview_condition: - if any( - transition.landed - for transition in self._accepted_block_payout_previews.values() - ): - raise PayoutStatePublicationBlocked( - "accepted block payout confirmation is still pending" - ) - return self._ensure_reorg_reconciler_service().reconcile( - tip_hash=tip_hash, - force_publish=_force_publish, - source_reserved=_source_reserved, - ) - - def client_can_receive_jobs(self, client: ClientState) -> bool: - return self._ensure_job_delivery_service().client_can_receive_jobs(client) - - def pool_readiness_latched(self) -> bool: - """Latch, once, the transition past min_ready_miners. - - Readiness is monotonic (a lifetime distinct-accepted-miner count), so - a single observation is permanent and later checks stay ledger-free. - The poll loop refreshes the latch outside the coordinator lock. - """ - if getattr(self, "_pool_ready_latched", False): - return True - try: - _, ready_miner_count = self.accepted_share_stats() - except Exception: - return False - if ready_miner_count >= getattr(self, "min_ready_miners", 3): - became_ready = False - with self.lock: - if not getattr(self, "_pool_ready_latched", False): - self._pool_ready_latched = True - self._retained_collection_refresh = None - became_ready = True - if became_ready: - # Collection work becomes obsolete without changing the qbit - # template fingerprint or payout generation. Start the health - # deadline at the readiness transition itself. - self._progress_note_refresh_pending() - return True - return False - - def client_needs_tip_template_refresh( - self, - client: ClientState, - snapshot: QbitTipTemplateSnapshot, - ) -> bool: - return self._ensure_job_delivery_service().client_needs_tip_template_refresh( - client, - snapshot, - ) - - def intervening_job_supersedes_snapshot( - self, - active_job: PrismJobContext | None, - expected_active_job: PrismJobContext | None, - snapshot: QbitTipTemplateSnapshot, - ) -> bool: - return self._ensure_job_delivery_service().intervening_job_supersedes_snapshot( - active_job, - expected_active_job, - snapshot, - ) - - def client_tip_changed_for_snapshot( - self, - client: ClientState, - snapshot: QbitTipTemplateSnapshot, - ) -> bool: - return self._ensure_job_delivery_service().client_tip_changed_for_snapshot( - client, - snapshot, - ) - - def handle_client(self, client: ClientState) -> None: - self._ensure_stratum_session_service().handle_client(client) - - def disconnect_client(self, client: ClientState) -> None: - # Retire admission and fanout eligibility without waiting behind job - # delivery. Only the first caller owns socket close and final cleanup. - self._ensure_stratum_session_service().disconnect_client(client) - - def handle_request(self, client: ClientState, request: dict[str, object]) -> None: - """Dispatch one request, translating shutdown races to Stratum errors.""" - self._ensure_stratum_session_service().handle_request(client, request) - - def _handle_request(self, client: ClientState, request: dict[str, object]) -> None: - self._ensure_stratum_session_service()._handle_request(client, request) - - def handle_suggest_difficulty(self, client: ClientState, request_id: object, params: list[object]) -> None: - self._ensure_stratum_session_service().handle_suggest_difficulty( - client, request_id, params - ) - - def handle_configure(self, client: ClientState, request_id: object, params: list[object]) -> None: - self._ensure_stratum_session_service().handle_configure( - client, request_id, params - ) - - def send_result(self, client: ClientState, request_id: object, result: object) -> None: - client.send(stratum_result_payload(request_id, result)) - - def send_error(self, client: ClientState, request_id: object, code: int, message: str, *, reason: str | None = None) -> None: - client.send(stratum_error_payload(request_id, code, message, reason=reason)) - - def resolve_worker(self, username: str) -> WorkerIdentity: - return self._ensure_stratum_session_service().resolve_worker(username) - - def validate_p2mr_address(self, address: str, *, label: str) -> tuple[str, str]: - return self._ensure_stratum_session_service().address_validator.validate( - address, label=label - ) - - @staticmethod - def _raise_shared_p2mr_address_validation_error(error: BaseException) -> None: - P2mrAddressValidator._raise_shared_error(error) - - def _ensure_p2mr_address_cache_state(self) -> None: - # The validator (created on first touch) owns the cache trio; legacy - # assignments through the descriptor-routed names are adopted by it. - self._ensure_stratum_session_service() - - def maybe_send_job( - self, - client: ClientState, - *, - clean_jobs: bool, - raise_on_reorg_failure: bool = False, - raise_on_build_failure: bool = False, - tip_refresh_snapshot: QbitTipTemplateSnapshot | None = None, - tip_refresh_observation_sequence: int | None = None, - ) -> bool: - return self._ensure_job_delivery_service().maybe_send_job( - client, - clean_jobs=clean_jobs, - raise_on_reorg_failure=raise_on_reorg_failure, - raise_on_build_failure=raise_on_build_failure, - tip_refresh_snapshot=tip_refresh_snapshot, - tip_refresh_observation_sequence=tip_refresh_observation_sequence, - ) - - def _maybe_send_job_locked( - self, - client: ClientState, - *, - clean_jobs: bool, - raise_on_reorg_failure: bool = False, - raise_on_build_failure: bool = False, - tip_refresh_snapshot: QbitTipTemplateSnapshot | None = None, - tip_refresh_observation_sequence: int | None = None, - prepared_bundle: CachedJobBundle | None = None, - commit_guard: Callable[[], bool] | None = None, - commit_guard_lock: threading.RLock | None = None, - prepared_bundle_allow_uncached: bool = False, - ) -> bool: - return self._ensure_job_delivery_service()._maybe_send_job_locked( - client, - clean_jobs=clean_jobs, - raise_on_reorg_failure=raise_on_reorg_failure, - raise_on_build_failure=raise_on_build_failure, - tip_refresh_snapshot=tip_refresh_snapshot, - tip_refresh_observation_sequence=tip_refresh_observation_sequence, - prepared_bundle=prepared_bundle, - commit_guard=commit_guard, - commit_guard_lock=commit_guard_lock, - prepared_bundle_allow_uncached=prepared_bundle_allow_uncached, - ) - - def prune_client_active_jobs(self, client: ClientState) -> None: - return self._ensure_job_delivery_service().prune_client_active_jobs(client) - - def send_difficulty(self, client: ClientState, job: direct_stratum.DirectQbitStratumJob) -> None: - self.send_difficulty_value(client, job.share_difficulty) - - def send_difficulty_value(self, client: ClientState, difficulty: Decimal) -> None: - client.send(self.difficulty_payload(difficulty)) - - @staticmethod - def difficulty_payload(difficulty: Decimal) -> dict[str, object]: - return stratum_difficulty_payload(difficulty) - - def client_vardiff_config(self, client: ClientState) -> vardiff.VardiffConfig: - return self._ensure_vardiff_service().client_config(client) - - def client_startup_difficulty(self, profile: StratumListenerProfile | None = None) -> Decimal: - return self._ensure_vardiff_service().startup_difficulty(profile) - - def desired_client_share_difficulty(self, client: ClientState) -> Decimal: - return self._ensure_vardiff_service().desired_difficulty(client) - - def client_minimum_advertised_difficulty(self, client: ClientState) -> Decimal: - return self._ensure_vardiff_service().minimum_advertised_difficulty(client) - - def resume_client_difficulty(self, client: ClientState) -> Decimal | None: - return self._ensure_vardiff_service().apply_resumed_difficulty(client) - - def note_vardiff_resume_overridden(self) -> None: - self._ensure_vardiff_service().note_resume_overridden() - - def record_session_difficulty(self, client: ClientState) -> None: - # Only a connection that produced an accepted share may refresh the - # retained value's TTL. Without that evidence a reconnect loop of - # silent sessions would re-stamp the entry forever and defeat the TTL. - self._ensure_vardiff_service().record_session_difficulty( - client, - share_backed=bool(getattr(client, "vardiff_accepted_any", False)), - ) - - def apply_job_difficulty(self, client: ClientState, job: direct_stratum.DirectQbitStratumJob) -> None: - return self._ensure_job_delivery_service().apply_job_difficulty(client, job) - - def apply_client_difficulty_requests(self, client: ClientState) -> Decimal | None: - return self._ensure_job_delivery_service().apply_client_difficulty_requests( - client, - ) - - def advertise_client_difficulty(self, client: ClientState, target: Decimal) -> bool: - return self._ensure_job_delivery_service().advertise_client_difficulty( - client, - target, - ) - - def _advertise_client_difficulty_locked( - self, - client: ClientState, - target: Decimal, - ) -> bool: - return self._ensure_job_delivery_service()._advertise_client_difficulty_locked( - client, - target, - ) - - def normalized_prior_balances(self, balances: list[dict[str, object]]) -> list[dict[str, object]]: - return self._ensure_payout_state_service().normalized_prior_balances(balances) - - @staticmethod - def settlement_balances_by_program(balances: list[dict[str, object]]) -> dict[str, int]: - """Aggregate owed balances by P2MR program for settlement equality.""" - return PayoutStateService.settlement_balances_by_program(balances) - - def prior_balances_match_current(self, prior_balances: list[dict[str, object]]) -> bool: - ensure = getattr(self, "_ensure_payout_state_service", None) - if ensure is not None: - return ensure().prior_balances_match_current(prior_balances) - # Unbound duck-typed callers (the settlement comparison tests) supply - # only settlement_balances_by_program and the ledger; keep the exact - # #87 settlement-equality comparison for them. - return self.settlement_balances_by_program( - prior_balances - ) == self.settlement_balances_by_program( - self.ledger.current_prior_balances() - ) - - def send_job(self, client: ClientState, job: direct_stratum.DirectQbitStratumJob) -> None: - client.send(self.job_payload(job)) - - @staticmethod - def job_payload(job: direct_stratum.DirectQbitStratumJob) -> dict[str, object]: - return stratum_job_payload(job) - - def send_job_update( - self, - client: ClientState, - job: direct_stratum.DirectQbitStratumJob, - ) -> None: - return self._ensure_job_delivery_service().send_job_update(client, job) - - def build_job_for_client(self, client: ClientState, *, clean_jobs: bool) -> PrismJobContext: - return self._ensure_job_delivery_service().build_job_for_client( - client, - clean_jobs=clean_jobs, - ) - - def build_job_for_client_from_artifacts( - self, - client: ClientState, - artifacts: CachedTemplateArtifacts, - *, - clean_jobs: bool, - publication_critical: bool = False, - request_source: str = "routine", - ) -> PrismJobContext: - return self._ensure_job_delivery_service().build_job_for_client_from_artifacts( - client, - artifacts, - clean_jobs=clean_jobs, - publication_critical=publication_critical, - request_source=request_source, - ) - - def build_collection_bundle( - self, - *, - template: dict[str, Any], - transaction_hexes: tuple[str, ...], - worker: WorkerIdentity, - network_difficulty: int, - issued_at_ms: int, - suffix_hex: str, - summary_only: bool = False, - payout_policy: dict[str, object] | None = None, - ctv_settlement: dict[str, object] | None = None, - cancellation: _JobBuildCancellation | None = None, - ) -> dict[str, Any]: - return self._ensure_job_bundle_service().build_collection_bundle( - template=template, - transaction_hexes=transaction_hexes, - worker=worker, - network_difficulty=network_difficulty, - issued_at_ms=issued_at_ms, - suffix_hex=suffix_hex, - summary_only=summary_only, - payout_policy=payout_policy, - ctv_settlement=ctv_settlement, - cancellation=cancellation, - ) - - def shutdown_serve_builder(self) -> None: - """Retire the persistent audit builder; builds revert to one-shot.""" - self._ensure_bundle_compiler().shutdown_serve_builder() - - def build_audit_bundle( - self, - *, - shares: list[dict[str, object]], - found_block: dict[str, object], - prior_balances: list[dict[str, object]], - coinbase_script_sig_suffix_hex: str, - witness_merkle_leaves_hex: list[str] | None = None, - ctv_fee_parent_hash: str | None = None, - canonical_output_path: Path | None = None, - canonical_output_parent_fd: int | None = None, - canonical_output_adopter: Callable[[Path, os.stat_result], None] | None = None, - summary_only: bool = False, - payout_policy: dict[str, object] | None = None, - ctv_settlement: dict[str, object] | None = None, - cancellation: _JobBuildCancellation | None = None, - share_serialization: _ShareWindowSerialization | None = None, - append_invalidation_epoch: int | None = None, - ) -> dict[str, Any]: - return self._ensure_bundle_compiler().build_audit_bundle( - shares=shares, - found_block=found_block, - prior_balances=prior_balances, - coinbase_script_sig_suffix_hex=coinbase_script_sig_suffix_hex, - witness_merkle_leaves_hex=witness_merkle_leaves_hex, - ctv_fee_parent_hash=ctv_fee_parent_hash, - canonical_output_path=canonical_output_path, - canonical_output_parent_fd=canonical_output_parent_fd, - canonical_output_adopter=canonical_output_adopter, - summary_only=summary_only, - payout_policy=payout_policy, - ctv_settlement=ctv_settlement, - cancellation=cancellation, - share_serialization=share_serialization, - append_invalidation_epoch=append_invalidation_epoch, - ) - - def prepare_payout_window( - self, - *, - mode: str, - records_json: list[dict[str, object]], - anchor_job_issued_at_ms: int, - append_invalidation_epoch: int, - window_weight: int | None = None, - page_size: int | None = None, - base_digest: str | None = None, - wait_for_daemon: bool = True, - ) -> Any: - """Fold/advance one payout window through the persistent builder.""" - return self._ensure_bundle_compiler().prepare_payout_window( - mode=mode, - records_json=records_json, - anchor_job_issued_at_ms=anchor_job_issued_at_ms, - append_invalidation_epoch=append_invalidation_epoch, - window_weight=window_weight, - page_size=page_size, - base_digest=base_digest, - wait_for_daemon=wait_for_daemon, - ) - - def coinbase_script_sig_suffix_hex(self, extranonce1_hex: str, extranonce2_hex: str) -> str: - extranonce1_hex = validate_hex(extranonce1_hex, name="extranonce1") - extranonce2_hex = validate_hex(extranonce2_hex, name="extranonce2") - return self.coinbase_tag_hex + extranonce1_hex + extranonce2_hex - - @ledger_writer_operation("share_submission") - def handle_submit(self, client: ClientState, params: list[object]) -> bool: - return self._ensure_share_submission_service().handle(client, params) - - def _submit_synchronous_credit_candidate( - self, - candidate: PrismBlockCandidate, - *, - share_key: tuple[str, str], - worker_name: str, - evicted_entry: EvictedJobEntry | None, - credit_policy: str | None, - ) -> bool: - """Land one below-target block synchronously with its share credit. - - The hash solved a block but missed the assigned share target - (possible only while the listener floor sits above network - difficulty). It is a valid share ONLY if the block lands, so land - it synchronously: the miner's accept/reject and the ledger credit - then both reflect the real outcome -- never an "accepted" ack with - no ledger row. This path is rare (an honest miner does not submit - below its assigned target), so it does not affect the async - common-path latency. On failure the submitter already recorded the - specific block-failure reason in block_candidate_abandoned_counts; - reject_stratum additionally counts the miner-facing rejection - (globally and per worker) so this rare synchronous path is not - missing from the rejection metrics. - """ - submission = candidate.submission - pending_share = candidate.pending_share - persist_intent = getattr(self.ledger, "persist_block_candidate_intent", None) - durable_candidate_state: str | None = None - try: - candidate_intent = self.block_candidate_intent(candidate) - if callable(persist_intent): - persist_result = self._run_block_submitter_ledger_call( - ( - "persist-candidate-intent", - str(submission.block_hash_hex).lower(), - ), - "persist-candidate-intent", - lambda: persist_intent(candidate_intent), - ) - result_state = getattr(persist_result, "state", None) - if result_state is not None: - durable_candidate_state = str(result_state) - except BaseException: - # No retry slot is safe until the pre-submit outbox boundary is - # durable. Let the miner retry this submission instead. Without - # a durable intent nothing can commit this stamped share, so - # stop holding the snapshot anchor floor under it. - self._finish_pending_share_commit(pending_share) - self._release_submit_share_key(share_key) - raise - if durable_candidate_state in {"submitted", "abandoned"}: - # A process restart clears the in-memory disposition cache, - # but the existing outbox row remains authoritative. Join its - # terminal result before any new raw node offer. - block_landed = durable_candidate_state == "submitted" - self._record_block_candidate_terminal_outcome( - submission.block_hash_hex, - accepted=block_landed, - ) - self._finish_pending_share_commit(pending_share) - else: - try: - block_landed = self._submit_synchronous_block_candidate(candidate) - except BaseException: - self._release_submit_share_key(share_key) - raise - if not block_landed: - self._release_submit_share_key(share_key) - self.reject_stratum( - 23, - PRISM_REJECTION_LOW_DIFFICULTY, - "low difficulty share", - worker=worker_name, - ) - elif evicted_entry is not None: - self.note_evicted_job_submit( - credit_policy, - cross_connection=( - evicted_entry.connection_id - != candidate.client.connection_id - ), - ) - return False - - def block_candidate_intent(self, candidate: PrismBlockCandidate) -> Any: - """Return the immutable intent needed to resume a candidate after restart. - - The durable boundary is where a daemon-mirror share sequence used to - be forced to real dicts, so it is also where a refuted mirror would - first be seen on the landing path. The prepared intent (#255) now - streams the mirror's canonical bytes without parsing them, but the - route stays in place: a divergence raised while preparing must not - propagate, because the candidate cannot be persisted from a window - the coordinator no longer trusts, and the next build must not resume - from that window either. - """ - try: - return encode_block_candidate_intent(candidate) - except DaemonWindowMirrorDivergence: - self._note_window_mirror_divergence() - raise - - def block_candidate_from_intent( - self, - intent: Mapping[str, Any] | None = None, - ) -> PrismBlockCandidate: - # This helper was historically a static method. Preserve class-level - # decode calls while instance calls additionally adopt S3's durable - # credit-candidate holder before the reconstructed value is published. - coordinator: PrismCoordinator | None - if intent is None: - if not isinstance(self, Mapping): - raise TypeError("block candidate intent must be an object") - intent = self - coordinator = None - else: - coordinator = self - candidate = decode_block_candidate_intent(intent) - if candidate.credit_share_on_accept and coordinator is not None: - # A below-target candidate can credit this older accepted stamp - # after durable replay. Adopt its stable logical floor before - # startup prewarm/job issuance. Ordinary asynchronous candidates - # already committed their share and need no floor. - coordinator._ensure_block_candidate_service().adopt_replayed_candidate( - candidate - ) - return candidate - - def _ensure_pending_share_commit_state(self) -> None: - # The S3 owner (created on first touch) holds the floor lock and the - # pending-commit floor; legacy assignments through the descriptor - # routed names are adopted by it. - self._ensure_share_writer_service() - - def _finish_pending_share_commit(self, pending_share: PendingShare) -> None: - """Drop a share from the snapshot anchor floor.""" - self._ensure_share_writer_service().finish_pending_share(pending_share) - - def _finish_pending_share_attempt(self, pending_share: PendingShare) -> None: - """Release one stamped submission attempt's floor authority (seam).""" - self._ensure_share_writer_service().finish_pending_attempt(pending_share) - - def _finish_pending_share_candidate(self, pending_share: PendingShare) -> None: - """Release the durable candidate actor's floor authority (seam).""" - self._ensure_share_writer_service().finish_pending_candidate(pending_share) - - def _job_snapshot_anchor_ms(self, issued_at_ms: int) -> int: - """Clamp a share-snapshot anchor below every coverable share stamp.""" - return self._ensure_share_writer_service().snapshot_anchor_ms(issued_at_ms) - - def pending_share_from_submission( - self, - *, - context: PrismJobContext, - submission: direct_stratum.DirectQbitSubmission, - ntime_hex: str, - credit_policy: str | None = None, - ) -> PendingShare: - return self._ensure_share_writer_service().pending_share_from_submission( - context=context, - submission=submission, - ntime_hex=ntime_hex, - credit_policy=credit_policy, - ) - - def _expose_inflight_scan_anchor(self, anchor_ms: int) -> int: - """Publish a ledger walk's anchor to the append-side invalidation.""" - return self._ensure_payout_state_service()._expose_inflight_scan_anchor(anchor_ms) - - def _retire_inflight_scan_anchor(self, token: int | None) -> None: - """Withdraw an exposed walk anchor once its result became visible""" - return self._ensure_payout_state_service()._retire_inflight_scan_anchor(token) - - def _publish_seedless_job_window_anchor_locked(self, anchor_ms: int) -> None: - """Keep a seedless published window's anchor visible to appends.""" - return self._ensure_payout_state_service()._publish_seedless_job_window_anchor_locked(anchor_ms) - - @staticmethod - def _pending_share_predates_anchor(pending_share: PendingShare, anchor_ms: int | None) -> bool: - return PayoutStateService._pending_share_predates_anchor(pending_share, anchor_ms) - - def _pending_share_predates_live_anchor_locked(self, pending_share: PendingShare) -> bool: - return self._ensure_payout_state_service()._pending_share_predates_live_anchor_locked(pending_share) - - def _landing_fence_for_predating_append(self, pending_shares: list[PendingShare]) -> Iterator[bool]: - """Hold the landing fence across a durable append of predating rows.""" - return self._ensure_payout_state_service()._landing_fence_for_predating_append(pending_shares) - - def _await_unfenced_appends_predating_anchor(self, anchor_ms: int) -> None: - """Wait until no unfenced in-flight append can predate ``anchor_ms``.""" - return self._ensure_payout_state_service()._await_unfenced_appends_predating_anchor(anchor_ms) - - def _record_late_visible_payout_append(self, pending_share: PendingShare, *, landing_fence_owned: bool=False) -> int | None: - """Advance the append-invalidation epoch for a newly durable row.""" - return self._ensure_payout_state_service()._record_late_visible_payout_append(pending_share, landing_fence_owned=landing_fence_owned) - - def _append_epoch_invalidated_declared_anchor(self, *, baseline_epoch: int, live_epoch: int, declared_anchor_ms: int | None) -> bool: - """Whether an append between two epochs invalidated this declared anchor.""" - return self._ensure_payout_state_service()._append_epoch_invalidated_declared_anchor( - baseline_epoch=baseline_epoch, - live_epoch=live_epoch, - declared_anchor_ms=declared_anchor_ms, - ) - - def _invalidate_incremental_payout_window_for_append(self, pending_share: PendingShare) -> None: - """Disarm payout work that predates a newly visible eligible row.""" - return self._ensure_payout_state_service()._invalidate_incremental_payout_window_for_append(pending_share) - - def _retire_payout_windows_for_late_append(self, pending_share: PendingShare, invalidation_epoch: int) -> None: - """Disarm cached payout windows after a late-append epoch bump.""" - return self._ensure_payout_state_service()._retire_payout_windows_for_late_append(pending_share, invalidation_epoch) - - @ledger_writer_operation("share_persistence") - def append_accepted_share( - self, - client: ClientState, - context: PrismJobContext, - submission: direct_stratum.DirectQbitSubmission, - pending_share: PendingShare, - *, - credit_policy: str | None = None, - candidate_intent: dict[str, Any] | None = None, - ) -> str | None: - return self._ensure_share_writer_service().append_accepted_share( - client, - context, - submission, - pending_share, - credit_policy=credit_policy, - candidate_intent=candidate_intent, - ) - - def enqueue_share_append(self, entry: PendingShareAppend, *, wait: bool = False) -> None: - self._ensure_share_writer_service().enqueue_share_append(entry, wait=wait) - - def share_append_loop(self) -> None: - self._ensure_share_writer_service().share_append_loop() - - def _append_share_batch(self, batch: list[PendingShareAppend]) -> bool: - """Commit a writer batch, then release every waiting submitter.""" - return self._ensure_share_writer_service().append_share_batch(batch) - - def _recover_share_to_disk(self, entry: PendingShareAppend, reason: str) -> None: - self._ensure_share_writer_service().recover_share_to_disk(entry, reason) - - @ledger_writer_operation("share_recovery_replay") - def replay_recovered_shares(self) -> int: - """Replay any recovery-file shares into the ledger at startup.""" - return self._ensure_share_writer_service().replay_recovered_shares() - - def _append_share_entry(self, entry: PendingShareAppend, *, retry_until_stopped: bool = False) -> bool: - return self._ensure_share_writer_service().append_share_entry( - entry, retry_until_stopped=retry_until_stopped - ) - - def accepted_share_difficulty(self, context: PrismJobContext) -> int: - override = self.share_weights_by_username.get( - context.worker.username, - self.share_weights_by_username.get(context.worker.payout_address), - ) - if override is not None: - return max(1, int(override)) - return scaled_target_difficulty(context.job.share_target) - - def note_vardiff_submitted_share(self, client: ClientState) -> None: - self._ensure_share_hot_path_state() - with self._share_accounting_lock: - self.submitted_share_count += 1 - self._ensure_vardiff_service().note_submitted(client) - - def note_vardiff_accepted_share(self, client: ClientState, job: direct_stratum.DirectQbitStratumJob) -> None: - self._ensure_vardiff_service().note_accepted(client, job.share_difficulty) - - def _ensure_vardiff_idle_state(self) -> None: - self._ensure_vardiff_service() - - def _record_vardiff_idle_skip(self, reason: str) -> None: - self._ensure_vardiff_service().record_idle_skip(reason) - - def _observe_vardiff_idle_seconds(self, name: str, elapsed_seconds: float) -> None: - self._ensure_vardiff_service().observe_idle_seconds(name, elapsed_seconds) - - def _idle_bundle_current_locked( - self, - client: ClientState, - bundle: CachedJobBundle, - *, - allow_uncached: bool = False, - ) -> bool: - """Check one exact issuance observation. Caller holds ``_job_cache_lock``. - - Ready bundles can reuse an older same-tip heavy cache entry, but the - prepared copy must be rebound to the current template artifacts before - delivery. During a detected-but-unpublished refresh, issuance stays - pinned to the published snapshot. Accept that copy only when it still - shares the immutable heavy payload with the cache entry from which it - was derived. - """ - artifacts = self._idle_job_issuance_artifacts_locked() - if artifacts is None or self._payout_state_publication_blocked: - return False - payout_artifact = self._published_payout_state.artifact - if ( - bundle.build_key is None - or payout_artifact is None - or ( - not bundle.collection_only - and int( - getattr( - bundle.build_key, - "payout_append_invalidation_epoch", - 0, - ) - ) - != self._payout_ledger_append_invalidation_epoch - ) - or bundle.build_key.payout_artifact_sha256 - != payout_artifact.prior_balances_sha256 - ): - return False - observed_tip = getattr(self, "current_tip_first_seen", None) - if observed_tip is not None and observed_tip[0] != artifacts.previousblockhash: - return False - if ( - bundle.template_fingerprint != artifacts.fingerprint - or bundle.payout_state_generation != self._payout_state_generation - or str(bundle.template.get("previousblockhash", "")) - != artifacts.previousblockhash - ): - return False - if not bundle.collection_only: - # Idle issuance is a NEW serving decision, so the audit ceiling - # applies here exactly as it does at the reuse probe and the - # cached-bundle lookup -- without this gate the idle fast path - # was the one issuance route that could hand out a window past - # the ceiling. - found_block = getattr(bundle, "found_block", None) - declared_anchor_ms = ( - found_block.get("anchor_job_issued_at_ms") - if isinstance(found_block, dict) - else None - ) - if declared_anchor_ms is not None and ( - now_ms() - int(declared_anchor_ms) - > self._payout_artifact_max_anchor_age_ms() - ): - return False - published_tip = self._published_payout_state.source_tip_hash - if published_tip is not None and published_tip != artifacts.previousblockhash: - return False - worker = client.worker - if worker is None: - return False - mode = "ready" if getattr(self, "_pool_ready_latched", False) else "collection" - if bundle.collection_only != (mode == "collection"): - return False - if ( - bundle.template is not artifacts.template - or bundle.template_generation != artifacts.generation - ): - # Collection manifests sign the template ntime, while ready - # bundles can cheaply rebind their base job. Either way, the - # object stamped for delivery must represent this observation. - return False - key = self._job_bundle_key( - artifacts, - mode=mode, - payout_state_generation=self._payout_state_generation, - payout_artifact_generation=bundle.payout_artifact_generation, - worker=worker, - ) - ttl = float( - getattr( - self, - "job_bundle_cache_seconds", - DEFAULT_PRISM_JOB_BUNDLE_CACHE_SECONDS, - ) - ) - if ttl <= 0: - # A zero TTL deliberately disables global bundle retention. The - # dedicated worker may still deliver the exact bundle it just - # built after every live template/payout/client guard above has - # passed; the cache-only sweep never opts into this exception. - return allow_uncached and bundle.key == key - cached = self._job_bundle_cache.get(key) - if cached is None: - return False - if cached is not bundle and not ( - bundle.key == cached.key - and bundle.coinbase_manifest is cached.coinbase_manifest - and bundle.shares_json is cached.shares_json - and bundle.prior_balances is cached.prior_balances - and bundle.found_block is cached.found_block - and bundle.collection_only == cached.collection_only - and bundle.issued_at_ms == cached.issued_at_ms - and bundle.built_monotonic == cached.built_monotonic - and bundle.payout_state_generation - == cached.payout_state_generation - and bundle.payout_artifact_generation - == cached.payout_artifact_generation - and bundle.collection_identity == cached.collection_identity - ): - return False - return time.monotonic() - cached.built_monotonic <= ttl - - def _idle_job_issuance_artifacts_locked( - self, - ) -> CachedTemplateArtifacts | None: - """Cache-only counterpart of ``job_issuance_template_artifacts``. - - The idle sweep and its final delivery guard cannot fetch a template. - They must still honor the published-snapshot pin used by every other - direct issuance path while a replacement tip is detected but has not - yet been published. - """ - artifacts = self._template_artifacts - with self.lock: - published = getattr(self, "current_tip_first_seen", None) - latest_detected = getattr(self, "latest_detected_tip", None) - published_snapshot = getattr(self, "tip_template_snapshot", None) - pinned = bool( - published is not None - and published_snapshot is not None - and published_snapshot.bestblockhash == published[0] - and published_snapshot.template_artifacts is not None - and self._published_tip_authoritative_locked(time.monotonic()) - and ( - ( - latest_detected is not None - and latest_detected[0] != published[0] - ) - or ( - artifacts is not None - and artifacts.previousblockhash != published[0] - ) - ) - ) - if pinned: - assert published_snapshot is not None - assert published_snapshot.template_artifacts is not None - return published_snapshot.template_artifacts - return artifacts - - def _cached_idle_job_bundle(self, client: ClientState) -> CachedJobBundle | None: - """Return only an exact issuance bundle; never build or query.""" - self._ensure_job_cache_state() - with self._job_cache_lock: - artifacts = self._idle_job_issuance_artifacts_locked() - worker = client.worker - if artifacts is None or worker is None: - return None - mode = "ready" if getattr(self, "_pool_ready_latched", False) else "collection" - payout_artifact = getattr(self, "_payout_ledger_artifact", None) - payout_artifact_generation = ( - payout_artifact.generation - if mode == "ready" - and payout_artifact is not None - and payout_artifact.payout_state_generation - == self._payout_state_generation - and payout_artifact.network_difficulty - == artifacts.network_difficulty - else 0 - ) - key = self._job_bundle_key( - artifacts, - mode=mode, - payout_state_generation=self._payout_state_generation, - payout_artifact_generation=payout_artifact_generation, - worker=worker, - ) - bundle = self._job_bundle_cache.get(key) - if bundle is None or not self._idle_bundle_current_locked(client, bundle): - return None - return bundle - - def _build_idle_job_bundle( - self, - request: _IdleRetargetRequest, - ) -> CachedJobBundle: - """Build on the dedicated idle executor without holding a client lock.""" - with self.lock: - if self._vardiff_idle_tip_divergence_locked(): - raise JobBuildSuperseded( - "idle retarget deferred during unpublished tip refresh" - ) - artifacts = ( - self._retained_collection_artifacts() - or self.job_issuance_template_artifacts() - ) - return self.shared_job_bundle( - artifacts, - request.worker, - retry_superseded=False, - idle_retarget=True, - ) - - def _vardiff_idle_tip_divergence_locked(self) -> bool: - """Whether detected tip work still lacks published submit authority.""" - return self._ensure_vardiff_service().idle_tip_diverged_locked() - - def _idle_request_skip_reason( - self, - request: _IdleRetargetRequest, - ) -> str | None: - return self._ensure_vardiff_service().request_skip_reason(request) - - def shutdown_vardiff_idle_executor(self) -> None: - self._ensure_vardiff_service().shutdown_idle_executor() - - def vardiff_idle_sweep_loop(self) -> None: - self._ensure_vardiff_service().idle_sweep_loop() - - def vardiff_idle_sweep_once(self) -> int: - return self._ensure_vardiff_service().idle_sweep_once() - - def retarget_client( - self, - client: ClientState, - *, - current_difficulty: Decimal, - accepted_shares: int, - submitted_shares: int, - accepted_difficulty: Decimal, - elapsed_seconds: Decimal, - require_idle: bool = False, - prepared_bundle: CachedJobBundle | None = None, - expected_connection_id: int | None = None, - expected_worker: WorkerIdentity | None = None, - expected_active_job: PrismJobContext | None = None, - expected_window_started: float | None = None, - initial_convergence: bool = False, - ) -> bool: - return self._ensure_vardiff_service().retarget( - client, - current_difficulty=current_difficulty, - accepted_shares=accepted_shares, - submitted_shares=submitted_shares, - accepted_difficulty=accepted_difficulty, - elapsed_seconds=elapsed_seconds, - require_idle=require_idle, - prepared_bundle=prepared_bundle, - expected_connection_id=expected_connection_id, - expected_worker=expected_worker, - expected_active_job=expected_active_job, - expected_window_started=expected_window_started, - initial_convergence=initial_convergence, - ) - - def enqueue_block_candidate(self, candidate: PrismBlockCandidate) -> bool: - return self._ensure_block_candidate_service().enqueue(candidate) - - def _ensure_block_replay_state(self) -> None: - """Backfill replay/maintenance queues for lightweight coordinators.""" - self._ensure_block_candidate_service()._ensure_block_replay_state() - - def _enqueue_replayed_block_candidate( - self, - candidate: PrismBlockCandidate, - ) -> bool: - """Queue one durable replay behind live solves, once per process.""" - return self._ensure_block_candidate_service()._enqueue_replayed_block_candidate( - candidate - ) - - def _queue_invalid_block_candidate_for_quarantine( - self, - block_hash: str, - error: str, - *, - pending_share: PendingShare | None = None, - ) -> None: - """Move malformed-row cleanup off the node-offer lane.""" - self._ensure_block_candidate_service()._queue_invalid_block_candidate_for_quarantine( - block_hash, - error, - pending_share=pending_share, - ) - - @ledger_writer_operation("accepted_block_handling") - def replay_pending_block_candidates(self) -> int: - """Queue durable candidate intents not completed by an earlier process.""" - return self._ensure_block_candidate_service().replay_pending() - - def _ensure_block_submitter_retry_state(self) -> None: - self._ensure_block_candidate_service() - - def _ensure_block_submitter_ledger_call_state(self) -> None: - self._ensure_block_candidate_service()._ensure_block_submitter_ledger_call_state() - - def _block_submitter_db_timeout(self) -> float: - return self._ensure_block_candidate_service()._block_submitter_db_timeout() - - def _block_landing_db_timeout(self, block_hash: str | None = None) -> float: - """Landing-class deadline, escalated after observed landing timeouts.""" - return self._ensure_block_candidate_service()._block_landing_db_timeout( - block_hash - ) - - def _note_block_landing_timeout(self, block_hash: str | None) -> None: - self._ensure_block_candidate_service()._note_block_landing_timeout(block_hash) - - def _ensure_block_ledger_call_metrics(self) -> None: - self._ensure_block_candidate_service()._ensure_block_ledger_call_metrics() - - def _record_block_ledger_call( - self, - *, - call_class: str, - budget_seconds: float, - duration_seconds: float, - timed_out: bool, - ) -> None: - """Track per-call-class submitter ledger latency and timeout counts.""" - self._ensure_block_candidate_service()._record_block_ledger_call( - call_class=call_class, - budget_seconds=budget_seconds, - duration_seconds=duration_seconds, - timed_out=timed_out, - ) - - def block_ledger_call_class_metrics(self) -> dict[str, dict[str, float | int]]: - return self._ensure_block_candidate_service().block_ledger_call_class_metrics() - - def block_candidate_collapse_snapshot(self) -> dict[str, int]: - return self._ensure_block_candidate_service().block_candidate_collapse_snapshot() - - def _block_submitter_stuck_call_exit_timeout(self) -> float: - return self._ensure_block_candidate_service()._block_submitter_stuck_call_exit_timeout() - - def _maybe_restart_for_stuck_block_call( - self, - *, - kind: str, - started_monotonic: float, - ) -> None: - """Fail stop when a poisoned worker pool stays exhausted.""" - self._ensure_block_candidate_service()._maybe_restart_for_stuck_block_call( - kind=kind, - started_monotonic=started_monotonic, - ) - - def _maybe_restart_for_exhausted_block_call_pool( - self, - *, - kind: str, - calls_lock: threading.Lock, - calls: dict[Any, Any], - ) -> None: - """Age an exhausted pool even when retries reuse existing calls.""" - self._ensure_block_candidate_service()._maybe_restart_for_exhausted_block_call_pool( - kind=kind, - calls_lock=calls_lock, - calls=calls, - ) - - def _block_submitter_ledger_timeout_scope( - self, - timeout_seconds: float | None = None, - ) -> Iterator[None]: - """Apply the submitter's PostgreSQL deadline when the ledger supports it.""" - return self._ensure_block_candidate_service()._block_submitter_ledger_timeout_scope( - timeout_seconds - ) - - def _block_submitter_ledger_statement_timeout_scope(self) -> Iterator[None]: - """Give each post-submit ledger step a fresh short deadline.""" - return self._ensure_block_candidate_service()._block_submitter_ledger_statement_timeout_scope() - - def _block_landing_ledger_statement_timeout_scope( - self, - block_hash: str | None = None, - ) -> Iterator[None]: - """Give each landing-class ledger step the landing budget.""" - return self._ensure_block_candidate_service()._block_landing_ledger_statement_timeout_scope( - block_hash - ) - - def _run_block_submitter_ledger_call( - self, - key: tuple[object, ...], - phase: str, - operation: Callable[[], Any], - *, - timeout_seconds: float | None = None, - call_class: str = "fast", - ) -> Any: - """Run one direct outbox call without letting its driver wedge us.""" - return self._ensure_block_candidate_service()._run_block_submitter_ledger_call( - key, - phase, - operation, - timeout_seconds=timeout_seconds, - call_class=call_class, - ) - - def _wait_for_block_candidate_retry(self, delay_seconds: float) -> bool: - """Wait for intentional backoff without impersonating stuck work. - - Only this bounded retry wait refreshes the submitter heartbeat by - itself; SQL, RPC and lock phases go through their own phase-aware - helpers, so a genuinely blocked candidate phase remains - watchdog-eligible. - """ - return self._ensure_block_candidate_service().wait_for_retry(delay_seconds) - - def _mark_block_candidate_attempted(self, block_hash: str) -> None: - self._ensure_block_candidate_service().mark_attempted(block_hash) - - def _rpc_call_with_timeout( - self, - method: str, - params: list[object], - *, - timeout_seconds: float, - ) -> Any: - """Pass an explicit timeout to production RPCs and capable test doubles.""" - return self._ensure_block_candidate_service()._rpc_call_with_timeout( - method, - params, - timeout_seconds=timeout_seconds, - ) - - def _ensure_block_submitter_rpc_call_state(self) -> None: - self._ensure_block_candidate_service()._ensure_block_submitter_rpc_call_state() - - def _run_submitblock_rpc_with_hard_deadline( - self, - *, - block_hash: str, - block_hex: str, - timeout_seconds: float, - ) -> Any: - """Bound wall time even when an RPC adapter ignores its timeout.""" - return self._ensure_block_candidate_service()._run_submitblock_rpc_with_hard_deadline( - block_hash=block_hash, - block_hex=block_hex, - timeout_seconds=timeout_seconds, - ) - - def _arm_block_candidate_after_node_offer( - self, - candidate: PrismBlockCandidate, - node_submission: _BlockCandidateNodeSubmission, - ) -> None: - """Fence child payout work as soon as node acceptance is possible.""" - self._ensure_block_candidate_service()._arm_block_candidate_after_node_offer( - candidate, - node_submission, - ) - - def _submit_block_candidate_to_node( - self, - candidate: PrismBlockCandidate, - ) -> _BlockCandidateNodeSubmission: - """Offer the durable candidate to qbitd before any accounting work.""" - return self._ensure_block_candidate_service()._submit_block_candidate_to_node( - candidate - ) - - def _node_submission_for_candidate( - self, - candidate: PrismBlockCandidate, - ) -> _BlockCandidateNodeSubmission: - """Choose the node fast lane unless the pool was already closed.""" - return self._ensure_block_candidate_service()._node_submission_for_candidate( - candidate - ) - - def _node_submission_for_candidate_or_retained( - self, - candidate: PrismBlockCandidate, - ) -> _BlockCandidateNodeSubmission: - """Reuse a retained definitive acceptance instead of re-offering.""" - return self._ensure_block_candidate_service()._node_submission_for_candidate_or_retained( - candidate - ) - - def _node_submission_for_direct_candidate( - self, - candidate: PrismBlockCandidate, - ) -> _BlockCandidateNodeSubmission: - """Preserve active-replay semantics for non-queue embedders.""" - return self._ensure_block_candidate_service()._node_submission_for_direct_candidate( - candidate - ) - - def _account_block_candidate_after_node_submit( - self, - candidate: PrismBlockCandidate, - node_submission: _BlockCandidateNodeSubmission, - ) -> bool: - """Pass fast-lane evidence while tolerating legacy test embedders.""" - return self._ensure_block_candidate_service()._account_block_candidate_after_node_submit( - candidate, - node_submission, - ) - - def _submit_synchronous_block_candidate( - self, - candidate: PrismBlockCandidate, - ) -> bool: - """Run the rare miner-facing path under one same-hash disposition.""" - return self._ensure_block_candidate_service()._submit_synchronous_block_candidate( - candidate - ) - - def _ensure_block_candidate_disposition_state(self) -> None: - """Backfill same-hash submission guards for lightweight embedders.""" - self._ensure_block_candidate_service()._ensure_block_candidate_disposition_state() - - def _claim_block_candidate_disposition( - self, - block_hash: str, - *, - blocking: bool, - ) -> _BlockCandidateDispositionLease | None: - """Claim one hash without making unrelated node offers wait.""" - return self._ensure_block_candidate_service()._claim_block_candidate_disposition( - block_hash, - blocking=blocking, - ) - - def _drop_block_candidate_disposition_user( - self, - key: str, - flight: _BlockCandidateDispositionFlight, - ) -> None: - self._ensure_block_candidate_service()._drop_block_candidate_disposition_user( - key, - flight, - ) - - def _release_block_candidate_disposition( - self, - lease: _BlockCandidateDispositionLease, - ) -> None: - self._ensure_block_candidate_service()._release_block_candidate_disposition( - lease - ) - - def _block_candidate_terminal_outcome(self, block_hash: str) -> bool | None: - return self._ensure_block_candidate_service()._block_candidate_terminal_outcome( - block_hash - ) - - def _record_block_candidate_terminal_outcome( - self, - block_hash: str, - *, - accepted: bool, - ) -> None: - self._ensure_block_candidate_service()._record_block_candidate_terminal_outcome( - block_hash, - accepted=accepted, - ) - - def _record_committed_block_candidate_abandonment( - self, - block_hash: str, - outcome: threading.local, - ) -> None: - """Count an abandonment only after its terminal cleanup is fixed.""" - self._ensure_block_candidate_service()._record_committed_block_candidate_abandonment( - block_hash, - outcome, - ) - - def _reserve_block_fast_lane_slot(self, block_hash: str) -> bool: - """Reserve pool capacity while a node offer awaits terminal accounting.""" - return self._ensure_block_candidate_service()._reserve_block_fast_lane_slot( - block_hash - ) - - def _release_block_fast_lane_slot(self, block_hash: str) -> None: - self._ensure_block_candidate_service()._release_block_fast_lane_slot( - block_hash - ) - - def _block_candidate_disposition( - self, - block_hash: str, - ) -> Iterator[_BlockCandidateDispositionLease]: - """Serialize the full accepted/abandoned decision for one hash.""" - return self._ensure_block_candidate_service()._block_candidate_disposition( - block_hash - ) - - def _ensure_block_accounting_state(self) -> None: - self._ensure_block_candidate_service()._ensure_block_accounting_state() - - def _start_block_accounting_thread(self) -> threading.Thread: - return self._ensure_block_candidate_service()._start_block_accounting_thread() - - def _enqueue_block_accounting_task( - self, - task: _BlockCandidateAccountingTask, - ) -> bool: - return self._ensure_block_candidate_service()._enqueue_block_accounting_task( - task - ) - - def _run_one_invalid_block_candidate_quarantine(self) -> bool: - return self._ensure_block_candidate_service()._run_one_invalid_block_candidate_quarantine() - - def _call_block_candidate_writer( - self, - candidate: PrismBlockCandidate, - *, - node_submission: _BlockCandidateNodeSubmission, - disposition_held: bool, - ) -> bool: - """Invoke the writer while preserving duck-typed test integrations.""" - return self._ensure_block_candidate_service()._call_block_candidate_writer( - candidate, - node_submission=node_submission, - disposition_held=disposition_held, - ) - - def _restore_replayed_candidate_acceptance_evidence( - self, - candidate: PrismBlockCandidate, - ) -> None: - self._ensure_block_candidate_service()._restore_replayed_candidate_acceptance_evidence( - candidate - ) - - def _run_block_accounting_task( - self, - task: _BlockCandidateAccountingTask, - ) -> None: - self._ensure_block_candidate_service()._run_block_accounting_task(task) - - def block_accounting_loop(self) -> None: - self._ensure_block_candidate_service().block_accounting_loop() - - def block_submit_loop(self) -> None: - self._ensure_block_candidate_service().run() - - def submit_next_block_candidate( - self, - timeout: float | None = None, - *, - defer_accounting: bool = False, - ) -> bool: - """Dequeue and land one block candidate; returns True when one ran. - - The block-submitter loop calls this continuously; tests call it - directly to drain the queue deterministically. - """ - return self._ensure_block_candidate_service().submit_next( - timeout, - defer_accounting=defer_accounting, - ).ran - - def _submit_next_block_candidate_writer( - self, - candidate: PrismBlockCandidate, - *, - node_submission: _BlockCandidateNodeSubmission | None = None, - disposition_held: bool = False, - ) -> bool: - """Land one dequeued block candidate inside writer admission.""" - return self._ensure_block_candidate_service().submit_writer( - candidate, - node_submission=node_submission, - disposition_held=disposition_held, - ) - - def _finalize_block_candidate( - self, - candidate: PrismBlockCandidate, - *, - block_hash: str, - accepted: bool, - error: str, - outcome: threading.local, - ) -> bool: - """Drive a terminal candidate's durable outbox update, with backoff.""" - return self._ensure_block_candidate_service().finalize( - candidate, - block_hash=block_hash, - accepted=accepted, - error=error, - outcome=outcome, - ) - - def _merge_block_candidate_retry_locked( - self, - attribute: str, - candidate: PrismBlockCandidate, - ) -> None: - """Merge one retry by parent-first order. Caller holds self.lock.""" - self._ensure_block_candidate_service()._merge_block_candidate_retry_locked( - attribute, - candidate, - ) - - def _retain_block_candidate_for_retry(self, candidate: PrismBlockCandidate) -> None: - """Keep the oldest unresolved candidate ahead of queued descendants.""" - self._ensure_block_candidate_service().retain_for_retry(candidate) - - def _reject_terminal_prepared_block_candidate( - self, - candidate: PrismBlockCandidate, - ) -> None: - """Reject durable prepared deltas before abandoning a stale candidate.""" - state_reader = getattr(self.ledger, "pool_block_state", None) - if not callable(state_reader): - return - block_hash = str(candidate.submission.block_hash_hex).lower() - state = state_reader(block_hash=block_hash) - if state is None or str(state.get("chain_state", "")) != "prepared": - return - active_tip_height = int(self.rpc.call("getblockcount")) - result = self.reject_prepared_block( - block_hash=block_hash, - active_tip_height=active_tip_height, - ) - if int(result.get("rejected_count", 0)) == 1: - return - state = state_reader(block_hash=block_hash) - if state is not None and str(state.get("chain_state", "")) == "prepared": - raise RuntimeError( - f"ledger did not reject prepared block candidate {block_hash}" - ) - - def _next_block_candidate_retry_delay(self, block_hash: str) -> float: - return self._ensure_block_candidate_service().next_retry_delay(block_hash) - - def _pace_block_candidate_retry(self, block_hash: str) -> None: - """Apply per-candidate retry backoff without convoying accounting.""" - self._ensure_block_candidate_service()._pace_block_candidate_retry(block_hash) - - def _block_candidate_retry_ready_locked( - self, - candidate: PrismBlockCandidate, - ) -> bool: - """Return whether a parked retry's backoff deadline has passed.""" - return self._ensure_block_candidate_service()._block_candidate_retry_ready_locked( - candidate - ) - - def _stash_retained_block_candidate_node_submission( - self, - block_hash: str, - node_submission: _BlockCandidateNodeSubmission | None, - ) -> None: - """Record a definitive node acceptance for in-process retries.""" - self._ensure_block_candidate_service()._stash_retained_block_candidate_node_submission( - block_hash, - node_submission, - ) - - def _retained_block_candidate_node_submission( - self, - block_hash: str, - ) -> _BlockCandidateNodeSubmission | None: - return self._ensure_block_candidate_service()._retained_block_candidate_node_submission( - block_hash - ) - - def _block_candidate_acceptance_retained(self, block_hash: str) -> bool: - """Whether this process holds fresh first-party acceptance evidence.""" - return self._ensure_block_candidate_service()._block_candidate_acceptance_retained( - block_hash - ) - - def _clear_block_candidate_retry_state(self, block_hash: str) -> None: - self._ensure_block_candidate_service().clear_retry_state(block_hash) - - def _defer_block_candidate(self, reason: str, message: str, *, worker: str | None) -> None: - """Record a retryable outcome without counting a terminal abandonment.""" - self._ensure_block_candidate_service().record_deferred( - reason, - message, - worker=worker, - ) - - def _block_candidate_acceptance_recorded(self, block_hash: str) -> bool: - """Return whether this process completed the candidate success tail.""" - return self._ensure_block_candidate_service()._block_candidate_acceptance_recorded( - block_hash - ) - - def _register_outstanding_block_candidate(self, block_hash: str) -> None: - """Track a candidate this process may still land, for tip matching.""" - self._ensure_block_candidate_service()._register_outstanding_block_candidate( - block_hash - ) - - def _discard_outstanding_block_candidate(self, block_hash: str) -> None: - """Stop matching tip observations once a candidate is terminal.""" - self._ensure_block_candidate_service()._discard_outstanding_block_candidate( - block_hash - ) - - def _note_tip_observation_for_candidates(self, tip_hash: str) -> None: - """Register a tip observation that matches an outstanding candidate.""" - self._ensure_block_candidate_service()._note_tip_observation_for_candidates( - tip_hash - ) - - def _block_candidate_acceptance_observed(self, block_hash: str) -> bool: - """Whether a recent tip observation already proved this candidate landed.""" - return self._ensure_block_candidate_service()._block_candidate_acceptance_observed( - block_hash - ) - - def _block_candidate_chain_probe( - self, - block_hash: str, - *, - expected_height: int | None = None, - ) -> bool | None: - """Fresh chain verdict: proven active, proven wrong, or unknown.""" - return self._ensure_block_candidate_service()._block_candidate_chain_probe( - block_hash, - expected_height=expected_height, - ) - - def _block_candidate_acceptance_pending( - self, - block_hash: str, - *, - expected_height: int | None = None, - ) -> bool: - """Return whether abandoning this candidate would discard an accepted block.""" - return self._ensure_block_candidate_service()._block_candidate_acceptance_pending( - block_hash, - expected_height=expected_height, - ) - - def _count_accept_pending_defer(self) -> None: - self._ensure_block_candidate_service()._count_accept_pending_defer() - - def _note_pending_parent_payout_transition_deferred( - self, - block_hash: str, - ancestor_hash: str, - ) -> None: - """Track a finalization deferral against its blocking ancestor (#190).""" - self._ensure_block_candidate_service().note_pending_parent_transition_deferral( - block_hash, - ancestor_hash, - ) - - def _note_pending_parent_payout_transition_resolved( - self, - block_hash: str, - ) -> None: - """Drop ancestor re-drive bookkeeping once the fence passes (#190).""" - self._ensure_block_candidate_service().note_pending_parent_transition_resolved( - block_hash - ) - - def _abandon_block_candidate( - self, - reason: str, - message: str, - *, - block_hash: str, - worker: str | None, - preserve_if_accepted: bool = False, - expected_height: int | None = None, - stale_job_class: str | None = None, - ) -> bool: - """Record a lost/failed block candidate as a BLOCK-path event.""" - return self._ensure_block_candidate_service().record_abandoned( - reason, - message, - block_hash=block_hash, - worker=worker, - preserve_if_accepted=preserve_if_accepted, - expected_height=expected_height, - stale_job_class=stale_job_class, - ) - - def active_block_candidate_height(self, block_hash: str) -> int | None: - """Return the active-chain height for a previously submitted candidate.""" - try: - header = self.rpc.call("getblockheader", [block_hash]) - except Exception as exc: - detail = str(exc).lower() - if "block not found" in detail or "not found" in detail or "-5" in detail: - return None - raise - if not isinstance(header, dict): - return None - try: - confirmations = int(header.get("confirmations", 0)) - height = int(header["height"]) - except (KeyError, TypeError, ValueError): - return None - return height if confirmations > 0 else None - - def _defer_for_pending_parent_payout_transition( - self, - *, - block_hash: str, - parent_hash: str, - parent_height: int, - worker: str | None, - active_candidate_hash: str | None = None, - active_candidate_height: int | None = None, - ) -> bool: - """Defer finalization while an active payout ancestor is not durable.""" - if (active_candidate_hash is None) != (active_candidate_height is None): - raise ValueError("active candidate hash and height must be provided together") - - def preserve_active_candidate_barrier() -> None: - if active_candidate_hash is None or active_candidate_height is None: - return - self._begin_accepted_block_payout_preview( - active_candidate_hash, - block_height=active_candidate_height, - ) - self._mark_accepted_block_payout_landed( - active_candidate_hash, - block_height=active_candidate_height, - ) - - try: - pending_parent_transition = ( - self._accepted_block_payout_transition_for_parent( - parent_hash, - parent_height=parent_height, - ) - ) - except TemplateRefreshBlocked as exc: - preserve_active_candidate_barrier() - self._abandon_block_candidate( - PRISM_REJECTION_BACKEND_RPC_UNAVAILABLE, - f"could not validate pending ancestor payout state: {exc}", - block_hash=block_hash, - worker=worker, - ) - return True - if pending_parent_transition is None: - self._note_pending_parent_payout_transition_resolved(block_hash) - return False - pending_ancestor_hash, pending_ancestor_invalidated = ( - pending_parent_transition - ) - if not pending_ancestor_invalidated: - # Issue #190: the deferral below only re-checks the transition; - # it can never resolve the stuck ancestor. Count the deferral so - # a streak against one ancestor arms a targeted in-process - # re-drive through the durable-replay path. A tombstoned - # (invalidated) transition is a withdrawal in progress that the - # withdrawal retry publication owns, not a replay-resolvable - # pendency, so it stays outside the streak. - self._note_pending_parent_payout_transition_deferred( - block_hash, - pending_ancestor_hash, - ) - preserve_active_candidate_barrier() - self._abandon_block_candidate( - PRISM_REJECTION_LEDGER_CONFIRMATION_FAILED, - "parent or ancestor payout confirmation is still pending", - block_hash=block_hash, - worker=worker, - ) - return True - - def _payout_balance_serializer_released(self) -> Iterator[None]: - """Temporarily release the balance serializer around candidate audit""" - return self._ensure_payout_state_service()._payout_balance_serializer_released() - - def _replayed_payout_window_reproducible(self, context: PrismJobContext) -> bool: - """Whether a reconstructed candidate's payout window replays intact.""" - return self._ensure_payout_state_service()._replayed_payout_window_reproducible(context) - - def _land_and_confirm_block_candidate( - self, - candidate: PrismBlockCandidate, - *, - current_tip: str, - already_active: bool, - worker: str | None, - node_submission: _BlockCandidateNodeSubmission, - revalidated_append_epoch: int | None = None, - ) -> tuple[ - dict[str, Any], - dict[str, Any], - dict[str, Any], - dict[str, Any], - AuditPublicationIdentity, - dict[str, Any], - ] | None: - """Land, verify, publish, persist, and confirm one candidate (B3).""" - return self._ensure_block_finalization_service()._land_and_confirm_block_candidate( - candidate, - current_tip=current_tip, - already_active=already_active, - worker=worker, - node_submission=node_submission, - revalidated_append_epoch=revalidated_append_epoch, - ) - - def _land_block_candidate_submission( - self, - candidate: PrismBlockCandidate, - *, - node_submission: _BlockCandidateNodeSubmission | None = None, - disposition_held: bool = False, - ) -> bool: - """Run one candidate's landing tail behind the named submit port. - - Embeddings and tests replace the bound ``submit_block_candidate`` to - stand in for the node submission, so which of the two landing tails - applies is a fact about this coordinator rather than about the - block-candidate owner. Deciding it here keeps that owner from reaching - back through the runtime seam to inspect the entrypoint, and keeps the - decision per call: the replacement is installed on the instance after - the block-candidate service is constructed. - - ``node_submission`` is an already-created offer result when the caller - made one, and None when none was created -- which keeps the historical - bare entrypoint call. ``disposition_held`` states that the caller - already owns the same-hash disposition, which is what makes the - serialized inner tail, rather than the public entrypoint that would - take that guard again, the correct production call. - """ - submit = self.submit_block_candidate - production_submit = ( - getattr(submit, "__func__", None) - is PrismCoordinator.submit_block_candidate - ) - if disposition_held and production_submit: - assert node_submission is not None - return self._submit_block_candidate_serialized( - candidate, - node_submission=node_submission, - ) - if node_submission is None: - return bool(submit(candidate)) - return self._account_block_candidate_after_node_submit( - candidate, - node_submission, - ) - - @ledger_writer_operation("accepted_block_handling") - def submit_block_candidate( - self, - candidate: PrismBlockCandidate, - *, - node_submission: _BlockCandidateNodeSubmission | None = None, - ) -> bool: - """Land one block candidate, then finalize its audit and payout state.""" - return self._ensure_block_finalization_service().submit_block_candidate( - candidate, - node_submission=node_submission, - ) - - def _record_block_candidate_progress( - self, - phase: str = "accounting-progress", - ) -> None: - """Stamp the submitter heartbeat at a candidate-disposition boundary.""" - self._ensure_block_candidate_service()._record_block_candidate_progress(phase) - - def _submit_block_candidate_serialized( - self, - candidate: PrismBlockCandidate, - *, - node_submission: _BlockCandidateNodeSubmission, - ) -> bool: - """Process a candidate while its same-hash disposition guard is held.""" - return self._ensure_block_finalization_service()._submit_block_candidate_serialized( - candidate, - node_submission=node_submission, - ) - - @ledger_writer_operation("accepted_block_handling") - def reject_prepared_block(self, *, block_hash: str, active_tip_height: int) -> dict[str, int | str]: - reject = getattr(self.ledger, "reject_prepared_block", None) - if callable(reject): - return reject(block_hash=block_hash, active_tip_height=active_tip_height) - return self.ledger.reverse_immature_block( - block_hash=block_hash, - active_tip_height=active_tip_height, - ) - - @staticmethod - def verified_canonical_bundle_path( - candidate_bundle_path: Path, - report: dict[str, Any], - ) -> Path | None: - return AuditArtifactStore.verified_canonical_bundle_path( - candidate_bundle_path, - report, - ) - - def prune_audit_artifacts(self, *, keep_live_path: Path | None = None) -> None: - self._ensure_audit_artifact_store().prune_best_effort( - keep_live_path=keep_live_path - ) - - def verify_bundle( - self, - bundle_path: Path, - coinbase_tx_hex: str, - ledger_writer_public_key_hex: str, - *, - expected_coinbase_value_sats: int, - expected_block_height: int | None = None, - ) -> dict[str, Any]: - return self._ensure_audit_artifact_store().verify_bundle( - bundle_path, - coinbase_tx_hex, - ledger_writer_public_key_hex, - expected_coinbase_value_sats=expected_coinbase_value_sats, - expected_block_height=expected_block_height, - ) - - def trusted_ledger_writer_public_key_hex(self, bundle: dict[str, Any]) -> str: - return AuditArtifactStore.trusted_writer_key( - getattr(self, "ledger_writer_public_key_hex", None), - bundle, - allow_embedded_test_key=( - getattr(self, "ledger_writer_public_key_hex", None) is None - ), - ) - - def _progress_now(self) -> float: - clock = getattr(self, "_progress_monotonic", None) - return float(clock() if callable(clock) else time.monotonic()) - - def _ensure_progress_health_service(self) -> ProgressHealthService: - service = self.__dict__.get("progress_health_service") - if service is not None: - return service - init_lock = self.__dict__.get("_progress_health_init_lock") - if init_lock is None: - # CPython's setdefault is atomic under the GIL. Focused tests may - # construct through __new__, while normal instances build the - # service during __init__ before any process thread can start. - init_lock = self.__dict__.setdefault( - "_progress_health_init_lock", - threading.Lock(), - ) - with init_lock: - service = self.__dict__.get("progress_health_service") - if service is not None: - return service - service = ProgressHealthService( - started_monotonic=float( - getattr(self, "started_monotonic", time.monotonic()) - ), - initial_payout_generation=int( - getattr(self, "_payout_state_generation", 0) - ), - # Resolve the clock through the coordinator seam at call time - # so the historical ``_progress_monotonic`` test override - # keeps steering the service after construction. - monotonic=self._progress_now, - ) - self.__dict__["progress_health_service"] = service - return service - - def _progress_health_config(self) -> ProgressHealthConfig: - return ProgressHealthConfig( - pending_refresh_deadline_seconds=float( - getattr( - self, - "health_pending_refresh_max_age_seconds", - DEFAULT_PRISM_HEALTH_PENDING_REFRESH_MAX_AGE_SECONDS, - ) - ), - tip_poll_deadline_seconds=float( - getattr( - self, - "health_tip_poll_max_age_seconds", - DEFAULT_PRISM_HEALTH_TIP_POLL_MAX_AGE_SECONDS, - ) - ), - bundle_build_deadline_seconds=float( - getattr( - self, - "bundle_build_timeout_seconds", - DEFAULT_PRISM_BUNDLE_BUILD_TIMEOUT_SECONDS, - ) - ), - ) - - def _progress_work_generation( - self, - snapshot: QbitTipTemplateSnapshot, - payout_generation: int, - ) -> WorkGeneration: - return WorkGeneration( - template_generation=int(snapshot.template_generation), - template_fingerprint=snapshot.template_fingerprint, - payout_generation=int(payout_generation), - ) - - def _progress_note_refresh_pending(self, started_monotonic: float | None = None) -> None: - self._ensure_job_cache_state() - self._ensure_progress_health_service().mark_refresh_pending( - started_monotonic - ) - - def _record_progress_tip_poll( - self, - snapshot: QbitTipTemplateSnapshot, - observed_monotonic: float | None = None, - ) -> None: - """Publish a coherent qbit tip/template observation to health state.""" - self._ensure_job_cache_state() - payout_generation = int(getattr(self, "_payout_state_generation", 0)) - self._ensure_progress_health_service().observe_tip( - self._progress_work_generation(snapshot, payout_generation), - observed_monotonic, - ) - - def _record_progress_payout_generation( - self, - generation: int, - invalidated_monotonic: float | None = None, - ) -> None: - self._ensure_job_cache_state() - self._ensure_progress_health_service().observe_payout_generation( - generation, - invalidated_monotonic, - ) - - def _record_progress_publication( - self, - snapshot: QbitTipTemplateSnapshot, - payout_generation: int, - ) -> None: - """Record that current in-memory work is available for delivery.""" - self._ensure_job_cache_state() - service = self._ensure_progress_health_service() - with self.lock: - latest_detected = getattr(self, "latest_detected_tip", None) - published = service.publish_work( - self._progress_work_generation(snapshot, payout_generation), - matches_latest_tip=bool( - latest_detected is None - or latest_detected[0] == snapshot.bestblockhash - ), - ) - if published: - self._progress_note_refresh_activity() - self._progress_reconcile_pending() - - def _record_progress_delivery( - self, - client: ClientState, - context: PrismJobContext, - delivered_monotonic: float, - ) -> None: - """Record a completed current-generation socket delivery. - - Cross-owner adapter by design: the session owner commits exact S1 - registry proof first, then reports the delivery to G1. - """ - self._ensure_stratum_session_service().record_successful_delivery( - client, context, delivered_monotonic - ) - - def _progress_refresh_started(self) -> None: - """Track a coherent poll pass while it continues making progress.""" - self._ensure_job_cache_state() - self._ensure_progress_health_service().refresh_started() - - def _progress_note_refresh_activity( - self, - observed_monotonic: float | None = None, - ) -> None: - self._ensure_job_cache_state() - self._ensure_progress_health_service().note_refresh_activity( - observed_monotonic - ) - - def _progress_refresh_finished(self) -> None: - self._ensure_job_cache_state() - self._ensure_progress_health_service().refresh_finished() - - def _progress_bundle_build_started(self) -> int: - self._ensure_job_cache_state() - return self._ensure_progress_health_service().bundle_build_started() - - def _progress_bundle_build_finished(self, token: int) -> None: - self._ensure_job_cache_state() - self._ensure_progress_health_service().bundle_build_finished(token) - - def _progress_eligible_client_counts( - self, - template_fingerprint: str | None, - payout_generation: int, - ) -> tuple[int, int]: - with self.lock: - eligible = [ - client - for client in self.clients - if self.client_can_receive_jobs(client) - ] - delivered_work = [ - client._progress_delivered_context - for client in eligible - ] - ready_mode_required = bool( - getattr(self, "_pool_ready_latched", False) - ) - requiring_refresh = 0 - for delivered_context in delivered_work: - delivered_fingerprint = None - delivered_payout_generation = -1 - if delivered_context is not None: - delivered_fingerprint = getattr( - delivered_context, - "template_fingerprint", - None, - ) - if delivered_fingerprint is None: - delivered_fingerprint = qbit_template_fingerprint( - delivered_context.template - ) - delivered_payout_generation = int( - getattr(delivered_context, "payout_state_generation", 0) - ) - if ( - delivered_fingerprint != template_fingerprint - or delivered_payout_generation != payout_generation - or ( - ready_mode_required - and bool( - getattr(delivered_context, "collection_only", False) - ) - ) - ): - requiring_refresh += 1 - return len(eligible), requiring_refresh - - def _progress_reconcile_pending(self, *, now: float | None = None) -> None: - """Clear pending state exactly when publication/delivery is sufficient.""" - self._ensure_job_cache_state() - self._ensure_progress_health_service().reconcile_pending( - self._progress_eligible_client_counts, - now=now, - ) - - def _progress_health_value( - self, - *, - now: float | None = None, - ) -> ProgressHealthSnapshot: - self._ensure_job_cache_state() - started_monotonic = getattr(self, "started_monotonic", None) - return self._ensure_progress_health_service().snapshot( - self._progress_eligible_client_counts, - int(getattr(self, "_payout_state_generation", 0)), - self._progress_health_config(), - None if started_monotonic is None else float(started_monotonic), - now=now, - ) - - def progress_health_snapshot(self, *, now: float | None = None) -> dict[str, object]: - """Return a bounded, monotonic-only mining progress health snapshot.""" - return self._progress_health_value(now=now).as_mapping() - - @staticmethod - def _apply_progress_health( - payload: dict[str, object], - progress: dict[str, object], - ) -> dict[str, object]: - return ObservabilityService.apply_progress_health(payload, progress) - - def progress_health_metrics_lines(self) -> list[str]: - return list( - ProgressHealthService.metrics_lines( - self._progress_health_value(), - coordination_blocked_age_seconds=( - self.coordination_blocked_streak_age_seconds() - ), - ) - ) - - def ready_miner_count(self) -> int: - return self.accepted_share_stats()[1] - - def mining_delivery_snapshot( - self, - *, - now: float | None = None, - ) -> dict[str, object]: - return self._ensure_observability_service().mining_delivery_snapshot(now=now) - - def health_payload(self) -> dict[str, object]: - return self._ensure_observability_service().health_payload() - - def refresh_health_snapshot(self) -> dict[str, object]: - return self._ensure_observability_service().refresh_health_snapshot() - - def cached_health_payload(self) -> tuple[int, dict[str, object]]: - return self._ensure_observability_service().cached_health_payload() - - def health_snapshot_loop(self) -> None: - self._ensure_observability_service().health_snapshot_loop() - - def start_health_snapshot_refresher(self) -> None: - service = self._ensure_observability_service() - if not service.begin_refresh_loop(): - return - # The first refresh seeds the exact accepted-share aggregate, which - # can take minutes on a grown ledger. It runs inside the background - # loop so the audit listener bind path never blocks on it; until it - # completes, cached_health_payload reports an explicit starting - # state (issue #188 fix 4). The running flag above is set before the - # registry can start the thread, preserving the #120 - # mark-before-listener-dispatch contract for start_audit_server. - self._start_background_service("health_snapshot_refresher") - - def refresh_metrics_snapshot(self) -> str: - return self._ensure_observability_service().refresh_metrics_snapshot() - - def cached_metrics_payload(self) -> MetricsSnapshotResponse: - return self._ensure_observability_service().cached_metrics_payload() - - def metrics_snapshot_loop(self) -> None: - self._ensure_observability_service().metrics_snapshot_loop() - - def start_metrics_snapshot_refresher(self) -> None: - service = self._ensure_observability_service() - if not service.begin_metrics_refresh_loop(): - return - # Like the health refresher above, the collector is marked running - # before its thread can be dispatched and never collects on the - # scrape path: /metrics serves only the cached complete snapshot. - self._start_background_service("metrics_snapshot_refresher") - - @property - def latest_evidence(self) -> dict[str, Any] | None: - if ( - "_audit_artifact_store" not in self.__dict__ - and "audit_dir" not in self.__dict__ - and "evidence_path" not in self.__dict__ - ): - value = self.__dict__.get("_audit_latest_evidence_seed") - return copy.deepcopy(value) if isinstance(value, dict) else None - return self._ensure_audit_artifact_store().latest_evidence() - - @latest_evidence.setter - def latest_evidence(self, payload: Mapping[str, Any] | None) -> None: - if ( - "_audit_artifact_store" not in self.__dict__ - and "audit_dir" not in self.__dict__ - and "evidence_path" not in self.__dict__ - ): - self.__dict__["_audit_latest_evidence_seed"] = ( - copy.deepcopy(dict(payload)) if payload is not None else None - ) - return - self._ensure_audit_artifact_store().set_latest_evidence_for_compatibility( - payload - ) - - def latest_evidence_payload(self) -> dict[str, object] | None: - return self._ensure_audit_artifact_store().latest_evidence() - - def owed_balances_payload(self) -> dict[str, object]: - return { - "schema": "qbit.prism.owed-balances.v1", - "ledger_backend": self.ledger.backend_name, - "balances": self.ledger.current_owed_balances(), - } - - def carry_forward_integrity_payload(self) -> dict[str, object]: - report = self.ledger.carry_forward_integrity_report() - report["ledger_backend"] = self.ledger.backend_name - return report - - def miner_status_payload(self, recipient_id: str) -> dict[str, object]: - recipient_id = recipient_id.strip() - if not recipient_id: - raise ValueError("recipient_id is required") - balances = [ - balance - for balance in self.ledger.current_owed_balances() - if str(balance.get("recipient_id", "")) == recipient_id - ] - owed_balance_sats = sum(int(balance.get("balance_sats", 0)) for balance in balances) - return { - "schema": "qbit.prism.miner-status.v1", - "ledger_backend": self.ledger.backend_name, - "recipient_id": recipient_id, - "owed_balance_sats": owed_balance_sats, - "owed_balances": balances, - "recent_payouts": self.ledger.recipient_payout_history(recipient_id=recipient_id), - } - - @staticmethod - def process_resource_metrics() -> tuple[int, int]: - """Return cheap Linux RSS/descriptor gauges without extra processes.""" - rss_bytes = -1 - open_descriptors = -1 - try: - statm = Path("/proc/self/statm").read_text(encoding="ascii").split() - rss_bytes = int(statm[1]) * int(os.sysconf("SC_PAGE_SIZE")) - except (OSError, ValueError, IndexError): - pass - try: - open_descriptors = len(tuple(Path("/proc/self/fd").iterdir())) - except OSError: - pass - return rss_bytes, open_descriptors - - def coordinator_lock_contention_snapshot(self) -> tuple[int, int, float, float]: - """Raw (acquisitions, contentions, wait_sum, wait_max) from the lock.""" - snapshot = getattr(self.lock, "contention_snapshot", None) - if callable(snapshot): - acquisition_count, contention_count, wait_sum, wait_max = snapshot() - return ( - int(acquisition_count), - int(contention_count), - float(wait_sum), - float(wait_max), - ) - return 0, 0, 0.0, 0.0 - - def coordinator_lock_metrics_lines(self) -> list[str]: - # Compatibility wrapper for the extracted metrics owner. - return self._ensure_metrics_renderer().coordinator_lock_metrics_lines() - - def _observe_block_submit_seconds(self, elapsed_seconds: float) -> None: - self._ensure_block_candidate_service()._observe_block_submit_seconds( - elapsed_seconds - ) - - def _observe_accepted_block_preview_publication( - self, - block_hash: str, - *, - result: str, - ) -> None: - # Called from the P1 publication boundary; the B1 owner holds the - # acceptance stamp taken on the submitter thread. - service = self._ensure_block_candidate_service() - service._observe_accepted_block_preview_publication( - block_hash, - result=result, - ) - - def block_submitter_snapshot(self) -> dict[str, object]: - """Merged durable pending-candidate and B1 owner snapshot. - - Combines the ledger's durable pending aggregates with the block- - candidate service's copied backoff, submit-histogram, and - acceptance-to-preview-publication snapshots; each owner copies under - its own lock, so no locks nest here. - """ - pending_metrics: dict[str, object] = { - "pending_count": -1, - "oldest_pending_age_seconds": -1.0, - "oldest_unattempted_age_seconds": -1.0, - } - pending_snapshot = getattr(self.ledger, "block_candidate_pending_metrics", None) - if callable(pending_snapshot): - try: - pending_metrics.update(pending_snapshot()) - except Exception: - # Metrics collection is diagnostic. Candidate processing and - # its watchdog remain authoritative when this read is down. - pass - service = self._ensure_block_candidate_service() - backoff_active, backoff_remaining, backoff_delay = ( - service.backoff_snapshot() - ) - submit_buckets, submit_sum, submit_count = ( - service.block_submit_seconds_snapshot() - ) - accepted_preview_publication = ( - service.accepted_block_preview_publication_snapshot() - ) - pending_metrics.update( - { - "backoff_active": backoff_active, - "backoff_remaining_seconds": backoff_remaining, - "backoff_delay_seconds": backoff_delay, - "submit_seconds_buckets": submit_buckets, - "submit_seconds_sum": submit_sum, - "submit_seconds_count": submit_count, - "accepted_preview_publication": accepted_preview_publication, - } - ) - return pending_metrics - - def block_submitter_metrics_lines(self) -> list[str]: - # Compatibility wrapper for the extracted metrics owner. - return self._ensure_metrics_renderer().block_submitter_metrics_lines() - - def landing_observability_metrics_lines(self) -> list[str]: - # Compatibility wrapper for the extracted metrics owner. - return ( - self._ensure_metrics_renderer().landing_observability_metrics_lines() - ) - - def _accepted_stats_reconcile_metric_lines(self) -> list[str]: - # Compatibility wrapper for the extracted metrics owner. - return ( - self._ensure_metrics_renderer().accepted_stats_reconcile_metric_lines() - ) - - def share_accounting_snapshot(self) -> dict[str, object]: - """Copy the hot-path share counters under the accounting lock.""" - self._ensure_share_hot_path_state() - with self._share_accounting_lock: - return { - "submitted": int(getattr(self, "submitted_share_count", 0)), - "stale": int(getattr(self, "stale_share_count", 0)), - "duplicate": int(getattr(self, "duplicate_share_count", 0)), - "low_difficulty": int( - getattr(self, "low_difficulty_share_count", 0) - ), - "collection_block": int( - getattr(self, "collection_block_submission_count", 0) - ), - "rejections": dict( - getattr(self, "rejection_counts_by_reason", {}) - ), - "grace_credited": int( - getattr(self, "grace_credited_share_count", 0) - ), - } - - @staticmethod - def rejection_reason_ids() -> tuple[str, ...]: - # The renderer must not import coordinator globals. - return PRISM_REJECTION_REASON_IDS - - def _ensure_metrics_renderer(self) -> MetricsRenderer: - renderer = self.__dict__.get("_metrics_renderer") - if renderer is None: - # Idempotent construction; no init lock is needed. - renderer = MetricsRenderer(self) - self.__dict__["_metrics_renderer"] = renderer - return renderer - - def _render_metrics_payload(self) -> str: - return self._ensure_metrics_renderer().render() - - def metrics_payload(self) -> str: - """Compatibility renderer; HTTP uses the complete cached snapshot.""" - - return self._render_metrics_payload() - - def shutdown_metrics_lines(self) -> list[str]: - # Compatibility wrapper for the extracted metrics owner. - return self._ensure_metrics_renderer().shutdown_metrics_lines() - - def audit_artifact_metrics(self) -> dict[str, dict[str, int] | int]: - return self._ensure_audit_artifact_store().metrics_snapshot() - - @staticmethod - def audit_artifact_kind(name: str) -> str: - return AuditArtifactStore.artifact_kind(name) - - def ctv_fanout_broadcaster_metrics_lines(self) -> list[str]: - return self._ensure_ctv_runtime().metrics_lines() - - def initial_delivery_metrics_lines(self) -> list[str]: - # Compatibility wrapper for the extracted metrics owner. - return self._ensure_metrics_renderer().initial_delivery_metrics_lines() - - def vardiff_idle_metrics_lines(self) -> list[str]: - return self._ensure_vardiff_service().metrics_lines() - - def vardiff_convergence_snapshot(self) -> dict[str, object]: - return self._ensure_vardiff_service().convergence_snapshot() - - def block_finalization_metrics_lines(self) -> list[str]: - return self._ensure_block_finalization_service().metrics_lines() - - def tip_refresh_metrics_lines(self) -> list[str]: - return self._ensure_tip_refresh_service().tip_refresh_metrics_lines() - - def job_build_metrics_lines(self) -> list[str]: - # Compatibility wrapper for the extracted metrics owner. - return self._ensure_metrics_renderer().job_build_metrics_lines() - - def payout_state_metrics_lines(self) -> list[str]: - return self._ensure_payout_state_service().payout_state_metrics_lines() - - -def make_audit_handler(coordinator: PrismCoordinator) -> type[BaseHTTPRequestHandler]: - """Compatibility factory backed by the coordinator-free H1 facade.""" - - return AuditHttpFacade( - _CoordinatorAuditHttp( - coordinator, - allow_uncached_compatibility=True, - ) - ).handler_type() - - -def target_from_compact(bits_hex: str) -> int: - return direct_stratum.target_from_compact_hex(bits_hex) - - -def scaled_network_difficulty(bits_hex: str) -> int: - template_target = target_from_compact(bits_hex) - return scaled_target_difficulty(template_target) - - -def scaled_target_difficulty(target: int) -> int: - if target <= 0: - raise ValueError("target must be positive") - pow_limit_target = target_from_compact("207fffff") - return max(1, (pow_limit_target * 1_000_000) // target) - - -def main() -> int: - # A native fault (SIGSEGV/SIGABRT/SIGBUS/SIGFPE) otherwise kills the - # process with zero Python-side output; enable unconditionally so the - # tracebacks of every thread reach stderr even when PYTHONFAULTHANDLER - # is unset in the deployment environment. SIGUSR2 gives an on-demand - # all-thread dump for live triage without disturbing the process. - faulthandler.enable(all_threads=True) - faulthandler.register(signal.SIGUSR2, all_threads=True) - # Before the coordinator, because constructing it starts threads and the - # switch interval is process-global: applied afterwards, already-running - # threads would observe the change at an arbitrary point in their own - # scheduling. Silent when unset -- an unconfigured pool must not log a - # value it did not choose. - switch_interval = apply_python_switch_interval() - if switch_interval is not None: - print( - "prism interpreter switch interval set to " - f"{switch_interval:g}s (default is 5ms)", - flush=True, - ) - coordinator = PrismCoordinator() - - def _request_shutdown(signum: int, _frame: Any) -> None: - # Keep the handler to an atomic admission close plus wakeup. Writer - # quiescence, lease I/O, logging, and thread drainage run in normal - # control flow after serve observes the event. - coordinator.request_shutdown(signum) - - signal.signal(signal.SIGTERM, _request_shutdown) - signal.signal(signal.SIGINT, _request_shutdown) - # Issue #226 part 2: the operator-triggered heap census (SIGUSR1) and - # malloc_trim (SIGRTMIN+1). Nothing is registered unless the switches are - # on, and each handler only writes one byte to the census thread's pipe -- - # the SIGUSR2 shape above with the work moved off the handler. The strict - # reader here sees the same environment the config loader validated when - # the coordinator loaded its configuration (the malloc-telemetry - # precedent), so the hook needs nothing from the instance. The census - # itself lives in process_telemetry. - install_heap_census(env_heap_census_config()) - try: - coordinator.serve() - finally: - coordinator.shutdown(reason="main_finally") - coordinator.drain_non_writer_components() - return 1 if getattr(coordinator, "_fatal_exit_requested", False) else 0 - - -if __name__ == "__main__": - raise SystemExit(main()) diff --git a/lab/prism/prism_tools.py b/lab/prism/prism_tools.py deleted file mode 100644 index ac673b85b..000000000 --- a/lab/prism/prism_tools.py +++ /dev/null @@ -1,32 +0,0 @@ -"""Resolution of qbit-prism CLI tool invocations. - -The coordinator and ledger shell out to Rust helpers (audit bundle builder, -verifier, canonicalizer). Invoking them through ``cargo run`` re-checks the -build graph on every call, which is measurable overhead on a hot path and -much worse under load. When ``PRISM_TOOL_BIN_DIR`` points at a directory with -prebuilt binaries (the container image sets it to the release target dir), -run the binary directly; otherwise fall back to ``cargo run`` so developer -flows keep working without a prebuilt target dir. -""" - -from __future__ import annotations - -import os -import shlex -from pathlib import Path - - -def prism_tool_command(bin_name: str) -> list[str]: - """Return the argv prefix for a qbit-prism CLI tool. - - Tool arguments should be appended to the returned list; the ``cargo run`` - fallback already ends with ``--`` so positional arguments are never - swallowed by cargo itself. - """ - bin_dir = os.environ.get("PRISM_TOOL_BIN_DIR") - if bin_dir: - candidate = Path(bin_dir) / bin_name - if candidate.is_file() and os.access(candidate, os.X_OK): - return [str(candidate)] - cargo = shlex.split(os.environ.get("PRISM_CARGO", os.environ.get("CARGO", "cargo"))) - return cargo + ["run", "--quiet", "-p", "qbit-prism", "--bin", bin_name, "--"] diff --git a/lab/prism/process_telemetry.py b/lab/prism/process_telemetry.py deleted file mode 100644 index e1d7a0caa..000000000 --- a/lab/prism/process_telemetry.py +++ /dev/null @@ -1,1622 +0,0 @@ -"""Cheap, always-on interpreter and allocator telemetry for ``/metrics``. - -Issue #226: the production coordinator's resident set grows by roughly -390 MB per hour and nothing in the tree can say whether that growth is live -Python objects, pymalloc fragmentation, or glibc per-thread arena -fragmentation across the process's 64 threads. This module collects the -fixed-cardinality, constant-cost signals that discriminate between those -hypotheses on a running coordinator: - -- ``sys.getallocatedblocks()`` -- the live allocation count. Rising with RSS - means retention; flat while RSS rises means fragmentation. -- ``gc.get_count()`` and ``gc.get_stats()`` -- collector pressure and the - uncollectable count, per generation. ``get_count()`` returns the collector's - *trigger* counters (generation 0 is allocations minus deallocations), which - it compares against ``gc.get_threshold()``; they are not a count of retained - objects and they reset on every collection. Only ``get_stats()``'s - ``uncollectable`` counts objects. -- ``threading.active_count()`` -- every thread is a candidate glibc arena. -- glibc ``mallinfo2`` -- arena, in-use, free, and mmapped bytes across every - arena, so arena-minus-in-use is the fragmentation glibc has not returned. - -Everything the scrape collector reads is a snapshot. It never walks the -heap: no ``gc.get_objects()``, no ``tracemalloc``. It runs on every scrape. - -``mallinfo2`` exists only in glibc 2.33 and later. musl and macOS have no -such symbol, and the older int-typed ``mallinfo`` is deliberately not used -as a fallback because its 32-bit fields wrap on a multi-gigabyte heap. The -binding is attempted once per collector, and an absent symbol is remembered -so a scrape never repeats the lookup and never logs. Absence renders as a -separate availability gauge at 0 with every byte gauge at -1; it never -renders as zero bytes, which an operator would read as an empty heap. - -The second half of this module (issue #226 part 2) is the part that *does* -walk the heap, and it is built so that it can only ever do so on purpose: - -- :class:`HeapCensus` takes a bounded type histogram over - ``gc.get_objects()`` and, when ``tracemalloc`` is tracing, a top-N of - allocation sites, and writes the result to a file. It is off unless - ``PRISM_HEAP_CENSUS=1``, it is triggered only by an operator signal - (``SIGUSR1``), it runs on its own thread (the signal handler only writes - one byte to a pipe), and it is capped on top-N, on output bytes, on walk - seconds, and on files kept. No HTTP route, Stratum message, or metrics - scrape can reach it. -- :class:`MallocTrimmer` calls glibc ``malloc_trim(0)`` and reports the - allocator's before/after readings through :class:`ProcessHeapTelemetry`. - ``malloc_trim`` takes every arena's lock in turn while it walks that - arena's free lists, so a thread allocating in that arena at that moment - waits; on a fragmented multi-gigabyte heap that wait is a stall of tens - to hundreds of milliseconds, not a wedge. It is exposed on ``SIGRTMIN+1`` - when ``PRISM_MALLOC_TRIM_SIGNAL=1`` and as a paced periodic action when - ``PRISM_MALLOC_TRIM_INTERVAL_SECONDS`` is set, never below the floor. -- :func:`evaluate_rss_bound` is the automated soak check: a resident-set - series must stay within a stated multiple of its post-warm-up baseline. - ``python3 -m lab.prism.process_telemetry rss-bound`` runs it. -""" - -from __future__ import annotations - -import argparse -import ctypes -import datetime as _datetime -import gc -import json -import os -import re -import select -import sys -import tempfile -import threading -import time -import tracemalloc -from dataclasses import dataclass, field -from pathlib import Path -from typing import Any, Callable, Iterable, Sequence - -# Closed label set. CPython reports three entries from both gc.get_count() -# and gc.get_stats(); an interpreter reporting fewer renders -1 for the -# missing generation rather than growing or shrinking the series set. Since -# CPython 3.13 the collector is incremental and the third threshold is -# unused, so get_count()'s third entry reads 0 on the shipped interpreter. -# The series is kept regardless: a fixed family set across interpreter -# versions is worth more than dropping a known-zero gauge. -PRISM_GC_GENERATIONS = ("0", "1", "2") - -# Field order of ``struct mallinfo2`` (glibc >= 2.33); every field is size_t. -_MALLINFO2_FIELDS = ( - "arena", - "ordblks", - "smblks", - "hblks", - "hblkhd", - "usmblks", - "fsmblks", - "uordblks", - "fordblks", - "keepcost", -) - - -class MallInfo2(ctypes.Structure): - """``struct mallinfo2`` as returned by glibc's ``mallinfo2()``.""" - - _fields_ = [(name, ctypes.c_size_t) for name in _MALLINFO2_FIELDS] - - -def resolve_mallinfo2() -> Callable[[], MallInfo2]: - """Bind glibc's ``mallinfo2`` through ctypes, or raise. - - Looks the symbol up in the running program's global namespace, which - is where the libc Python itself links resolves. Raises ``AttributeError`` - where the symbol is absent (musl, macOS, glibc before 2.33) and whatever - ``ctypes`` raises where the global handle cannot be opened at all. The - caller treats every exception as "unavailable". - """ - libc = ctypes.CDLL(None) - function = libc.mallinfo2 - function.restype = MallInfo2 - function.argtypes = () - return function - - -@dataclass(frozen=True) -class ProcessHeapSample: - """One scrape's worth of interpreter and allocator readings.""" - - allocated_blocks: int - gc_trigger_count: tuple[int, int, int] - gc_collections: tuple[int, int, int] - gc_collected: tuple[int, int, int] - gc_uncollectable: tuple[int, int, int] - threads: int - malloc_info_available: bool - # The four byte gauges are -1, never 0, while malloc_info_available is - # False: a zero arena on a 17 GB process is a lie an operator would act on. - malloc_arena_bytes: int - malloc_in_use_bytes: int - malloc_free_bytes: int - malloc_mmapped_bytes: int - - -def _per_generation(values: tuple[int, ...] | list[int]) -> tuple[int, int, int]: - """Pin a per-generation reading to the closed three-generation set.""" - padded = [int(value) for value in values[: len(PRISM_GC_GENERATIONS)]] - while len(padded) < len(PRISM_GC_GENERATIONS): - padded.append(-1) - return (padded[0], padded[1], padded[2]) - - -_UNRESOLVED = object() - - -class ProcessHeapTelemetry: - """Collector for :class:`ProcessHeapSample`; one per renderer. - - ``malloc_enabled`` is the ``PRISM_MALLOC_TELEMETRY`` switch: off skips the - ``mallinfo2`` call entirely (it walks glibc's free lists under each arena - lock, which is cheap on a healthy heap but is the one reading here that - is not a plain interpreter counter) and renders the allocator gauges as - unavailable. The interpreter readings have no switch; they are pure - counter loads and are always on. - """ - - def __init__( - self, - *, - malloc_enabled: bool = True, - mallinfo_resolver: Callable[[], Callable[[], MallInfo2]] = resolve_mallinfo2, - ) -> None: - self._malloc_enabled = bool(malloc_enabled) - self._mallinfo_resolver = mallinfo_resolver - # Resolved exactly once: the bound function, or None once the symbol - # has proven absent or broken. Never re-probed and never logged, so - # a platform without mallinfo2 costs nothing per scrape. - self._mallinfo2: object = _UNRESOLVED - self._resolve_lock = threading.Lock() - - @property - def malloc_enabled(self) -> bool: - return self._malloc_enabled - - def _resolved_mallinfo2(self) -> Callable[[], MallInfo2] | None: - if not self._malloc_enabled: - return None - resolved = self._mallinfo2 - if resolved is _UNRESOLVED: - with self._resolve_lock: - resolved = self._mallinfo2 - if resolved is _UNRESOLVED: - try: - resolved = self._mallinfo_resolver() - except Exception: - # Absent symbol, unsupported loader, or anything - # else the C boundary raises: unavailable, for good. - resolved = None - self._mallinfo2 = resolved - return resolved # type: ignore[return-value] - - def sample(self) -> ProcessHeapSample: - stats = gc.get_stats() - arena = in_use = free = mmapped = -1 - available = False - mallinfo2 = self._resolved_mallinfo2() - if mallinfo2 is not None: - try: - info = mallinfo2() - except Exception: - # A symbol that binds but fails at call time is treated the - # same as an absent one from now on: no per-scrape retry. - self._mallinfo2 = None - else: - available = True - arena = int(info.arena) - in_use = int(info.uordblks) - free = int(info.fordblks) - mmapped = int(info.hblkhd) - return ProcessHeapSample( - allocated_blocks=int(sys.getallocatedblocks()), - gc_trigger_count=_per_generation(gc.get_count()), - gc_collections=_per_generation( - [int(entry.get("collections", -1)) for entry in stats] - ), - gc_collected=_per_generation( - [int(entry.get("collected", -1)) for entry in stats] - ), - gc_uncollectable=_per_generation( - [int(entry.get("uncollectable", -1)) for entry in stats] - ), - threads=int(threading.active_count()), - malloc_info_available=available, - malloc_arena_bytes=arena, - malloc_in_use_bytes=in_use, - malloc_free_bytes=free, - malloc_mmapped_bytes=mmapped, - ) - - -# --------------------------------------------------------------------------- -# Resident set and address-space readings (census, trim, and the storm -# instrument; never the scrape, which has its own reader on the coordinator). -# --------------------------------------------------------------------------- - - -def read_resident_memory_bytes(pid: int | str = "self") -> int: - """RSS bytes from ``/proc/<pid>/statm``, or -1 where that is unavailable.""" - try: - fields = Path(f"/proc/{pid}/statm").read_text(encoding="ascii").split() - return int(fields[1]) * int(os.sysconf("SC_PAGE_SIZE")) - except (OSError, ValueError, IndexError, AttributeError): - return -1 - - -_MIB = 1024 * 1024 -_GIB = 1024 * _MIB - -# The size bands the #226 capture was reported in: 658 of the 943 anonymous -# regions on union-mainnet fell in the 4-64 MiB band, which is the glibc -# per-thread heap (HEAP_MAX_SIZE = 64 MiB) signature. -ANONYMOUS_REGION_BANDS: tuple[tuple[str, int, int | None], ...] = ( - ("under_4mib", 0, 4 * _MIB), - ("4mib_to_64mib", 4 * _MIB, 64 * _MIB), - ("64mib_to_1gib", 64 * _MIB, _GIB), - ("over_1gib", _GIB, None), -) - - -@dataclass(frozen=True) -class AnonymousMapShape: - """Private writable anonymous regions of a process, by size band. - - ``heap_segment_bytes`` is the size of the ``[heap]`` (brk) segment, which - is glibc's main arena; the per-thread arenas are the bare anonymous - regions. Guard pages (``---p``), file mappings, and the stack are not - counted. ``available`` is False where ``/proc/<pid>/maps`` cannot be read - and every count is then -1. - """ - - available: bool - regions: int - total_bytes: int - heap_segment_bytes: int - band_regions: dict[str, int] = field(default_factory=dict) - band_bytes: dict[str, int] = field(default_factory=dict) - - def as_dict(self) -> dict[str, Any]: - return { - "available": self.available, - "regions": self.regions, - "total_bytes": self.total_bytes, - "heap_segment_bytes": self.heap_segment_bytes, - "band_regions": dict(self.band_regions), - "band_bytes": dict(self.band_bytes), - } - - -def read_anonymous_map_shape(pid: int | str = "self") -> AnonymousMapShape: - """Histogram the private writable anonymous regions of ``/proc/<pid>/maps``.""" - try: - text = Path(f"/proc/{pid}/maps").read_text(encoding="ascii", errors="replace") - except OSError: - return AnonymousMapShape( - available=False, - regions=-1, - total_bytes=-1, - heap_segment_bytes=-1, - ) - band_regions = {name: 0 for name, _low, _high in ANONYMOUS_REGION_BANDS} - band_bytes = {name: 0 for name, _low, _high in ANONYMOUS_REGION_BANDS} - regions = 0 - total = 0 - heap_segment = 0 - for line in text.splitlines(): - parts = line.split(None, 5) - if len(parts) < 5: - continue - span, perms = parts[0], parts[1] - pathname = parts[5].strip() if len(parts) > 5 else "" - if not perms.startswith("rw") or "p" not in perms: - continue - if pathname and pathname != "[heap]": - continue - try: - start_text, end_text = span.split("-", 1) - size = int(end_text, 16) - int(start_text, 16) - except ValueError: - continue - if pathname == "[heap]": - heap_segment += size - regions += 1 - total += size - for name, low, high in ANONYMOUS_REGION_BANDS: - if size >= low and (high is None or size < high): - band_regions[name] += 1 - band_bytes[name] += size - break - return AnonymousMapShape( - available=True, - regions=regions, - total_bytes=total, - heap_segment_bytes=heap_segment, - band_regions=band_regions, - band_bytes=band_bytes, - ) - - -@dataclass(frozen=True) -class MallocInfoSummary: - """glibc ``malloc_info`` folded across every arena. - - ``arenas`` is the arena count (main plus every per-thread arena ever - created; arenas outlive their threads). The free bytes are split by - where they sit, because that decides what ``malloc_trim`` can do: - ``bin_bytes`` are interior free chunks, which ``malloc_trim`` returns to - the kernel with ``madvise`` in every arena; ``top_bytes`` are the free - space at the end of each arena's heaps, which ``malloc_trim`` returns - only for the main arena -- a per-thread heap's top shrinks only when the - thread that owns the arena frees a chunk of 64 KiB or more while the top - exceeds ``M_TRIM_THRESHOLD``. All -1 when ``available`` is False. - """ - - available: bool - arenas: int - top_bytes: int - bin_bytes: int - fast_bytes: int - system_bytes: int - - def as_dict(self) -> dict[str, Any]: - return { - "available": self.available, - "arenas": self.arenas, - "top_bytes": self.top_bytes, - "bin_bytes": self.bin_bytes, - "fast_bytes": self.fast_bytes, - "system_bytes": self.system_bytes, - } - - -_MALLOC_INFO_UNAVAILABLE = MallocInfoSummary( - available=False, - arenas=-1, - top_bytes=-1, - bin_bytes=-1, - fast_bytes=-1, - system_bytes=-1, -) -_HEAP_ELEMENT = re.compile(r'<heap nr="\d+">(.*?)</heap>', re.S) -_REST_TOTAL = re.compile(r'<total type="rest" count="\d+" size="(\d+)"/>') -_FAST_TOTAL = re.compile(r'<total type="fast" count="\d+" size="(\d+)"/>') -_SYSTEM_CURRENT = re.compile(r'<system type="current" size="(\d+)"/>') -_SIZE_ROW = re.compile(r'<size from="\d+" to="\d+" total="(\d+)" count="\d+"/>') -_UNSORTED_ROW = re.compile(r'<unsorted from="\d+" to="\d+" total="(\d+)" count="\d+"/>') - - -def _first_int(pattern: re.Pattern[str], body: str) -> int: - match = pattern.search(body) - return int(match.group(1)) if match else 0 - - -def parse_malloc_info_xml(text: str) -> MallocInfoSummary: - """Fold one ``malloc_info`` document into :class:`MallocInfoSummary`. - - Per ``<heap>``: glibc prints one ``<size>`` row per non-empty size class - -- the fastbin classes first, then the small and large bins -- and the - unsorted list as its own ``<unsorted>`` row. ``<total type="fast">`` is - exactly the sum of the fastbin rows, and ``<total type="rest">`` is the - bins (small, large, unsorted) plus the top chunk; fastbins are not part - of ``rest``. So the bin bytes are all rows plus unsorted minus the fast - total, and top is ``rest`` minus that. Summing the fastbin rows into the - bins would understate top by the fast total, and top is the number an - operator reads to decide whether ``malloc_trim`` can help. - """ - arenas = top = bins = fast = system = 0 - for heap in _HEAP_ELEMENT.finditer(text): - body = heap.group(1) - arenas += 1 - rest = _first_int(_REST_TOTAL, body) - fast_here = _first_int(_FAST_TOTAL, body) - rows = sum(int(total) for total in _SIZE_ROW.findall(body)) + sum( - int(total) for total in _UNSORTED_ROW.findall(body) - ) - in_bins = max(0, rows - fast_here) - bins += in_bins - top += max(0, rest - in_bins) - fast += fast_here - system += _first_int(_SYSTEM_CURRENT, body) - return MallocInfoSummary( - available=True, - arenas=arenas, - top_bytes=top, - bin_bytes=bins, - fast_bytes=fast, - system_bytes=system, - ) - - -def read_malloc_info_summary() -> MallocInfoSummary: - """Fold glibc ``malloc_info`` across arenas, or the unavailable sentinel. - - ``malloc_info`` writes one ``<heap nr="...">`` element per arena. Like - ``mallinfo2`` it walks every arena under its lock -- and unlike it, it - prints a row per size class per arena -- so it is a census and - storm-instrument reading, never a scrape reading. The XML goes through a - temporary file and the C ``FILE*`` API because that is the only - interface glibc offers for it; :func:`parse_malloc_info_xml` does the - folding. - """ - try: - libc = ctypes.CDLL(None) - malloc_info = libc.malloc_info - fopen = libc.fopen - fclose = libc.fclose - except (OSError, AttributeError): - return _MALLOC_INFO_UNAVAILABLE - malloc_info.restype = ctypes.c_int - malloc_info.argtypes = (ctypes.c_int, ctypes.c_void_p) - fopen.restype = ctypes.c_void_p - fopen.argtypes = (ctypes.c_char_p, ctypes.c_char_p) - fclose.restype = ctypes.c_int - fclose.argtypes = (ctypes.c_void_p,) - try: - with tempfile.NamedTemporaryFile(prefix="prism-malloc-info-", suffix=".xml") as spool: - handle = fopen(spool.name.encode(), b"w") - if not handle: - return _MALLOC_INFO_UNAVAILABLE - try: - if malloc_info(0, handle) != 0: - return _MALLOC_INFO_UNAVAILABLE - finally: - fclose(handle) - text = Path(spool.name).read_text(encoding="ascii", errors="replace") - except (OSError, ValueError): - return _MALLOC_INFO_UNAVAILABLE - return parse_malloc_info_xml(text) - - -def read_malloc_arena_count() -> int: - """Count glibc malloc arenas through ``malloc_info``, or -1 when unavailable.""" - return read_malloc_info_summary().arenas - - -# --------------------------------------------------------------------------- -# Issue #226 part 2: the operator-triggered bounded heap census. -# --------------------------------------------------------------------------- - -HEAP_CENSUS_FORMAT = "qbit-prism-heap-census/v1" -# Files kept per output directory. Each is capped at the configured byte -# ceiling, so the disk footprint is bounded without a further knob. -HEAP_CENSUS_RETAINED_FILES = 32 -HEAP_CENSUS_FILE_PREFIX = "heap-census-" -HEAP_CENSUS_FILE_SUFFIX = ".json" -# Width of the per-process sequence in a report's file name; a million -# censuses at the 60 s floor is almost two years of one process. -HEAP_CENSUS_SEQUENCE_WIDTH = 6 -# The walk checks the clock every this many objects. 1024 objects is a few -# hundred microseconds, so the walk cap is honoured to well under a -# millisecond without paying a clock read per object. -_HEAP_CENSUS_CLOCK_STRIDE = 1024 -_HEAP_CENSUS_TYPE_NAME_LIMIT = 120 -_HEAP_CENSUS_SITE_LIMIT = 240 -# tracemalloc frame depth. One frame groups by file:line, which is what a -# top-N by site needs; deeper traces multiply the standing cost of tracing. -HEAP_CENSUS_TRACEMALLOC_FRAMES = 1 -# The signals. SIGUSR2 belongs to faulthandler (prism_coordinator.main); -# SIGUSR1 is free. The trim action takes the first real-time signal above -# SIGRTMIN so it cannot be confused with the census. -HEAP_CENSUS_SIGNAL_NAME = "SIGUSR1" -MALLOC_TRIM_SIGNAL_NAME = "SIGRTMIN+1" - - -@dataclass(frozen=True) -class HeapCensusConfig: - """The validated census, trim, and tracing switches and bounds. - - Built by ``lab.prism.coordinator_config.env_heap_census_config`` from the - ``PRISM_HEAP_CENSUS*`` and ``PRISM_MALLOC_TRIM*`` environment, which is - also where the defaults and their ceilings live. Every field is - deliberate: there is no default here, so a caller cannot construct an - enabled census without choosing its bounds. - """ - - enabled: bool - output_dir: str - top_n: int - max_bytes: int - max_seconds: float - min_interval_seconds: float - tracemalloc_enabled: bool - malloc_trim_signal_enabled: bool - malloc_trim_interval_seconds: float - - @property - def anything_armed(self) -> bool: - return bool( - self.enabled - or self.malloc_trim_signal_enabled - or self.malloc_trim_interval_seconds > 0 - ) - - -def _truncate(text: str, limit: int) -> str: - return text if len(text) <= limit else text[: limit - 3] + "..." - - -def _type_name(kind: type) -> str: - module = getattr(kind, "__module__", None) or "?" - qualname = getattr(kind, "__qualname__", None) or getattr(kind, "__name__", "?") - return _truncate(f"{module}.{qualname}", _HEAP_CENSUS_TYPE_NAME_LIMIT) - - -def _utc_stamp(now: float) -> str: - return ( - _datetime.datetime.fromtimestamp(now, tz=_datetime.timezone.utc) - .strftime("%Y%m%dT%H%M%SZ") - ) - - -def _log_line(message: str) -> None: - print(message, flush=True) - - -@dataclass(frozen=True) -class HeapCensusOutcome: - """What one census produced: where it went and the bounds it met.""" - - path: str - bytes_written: int - tracked_objects: int - objects_walked: int - walk_truncated: bool - top_n_written: int - seconds: float - tracemalloc_traced: bool - - -class HeapCensus: - """The bounded heap walk, and the file it writes. - - :meth:`take` is synchronous and is the only method that touches the heap. - It is what the census thread calls, what a test calls directly, and it - refuses to do anything at all when the configuration is not enabled -- - the entry point is inert, not merely unregistered, when the switch is - off. - - The walk is ``gc.get_objects()`` followed by one pass over the returned - list taking ``type()`` and ``sys.getsizeof()`` of each object. The - ``gc.get_objects()`` call itself is a single C step that allocates one - pointer per tracked object and holds the GIL for its duration; nothing - can interrupt it, and on a heap of tens of millions of tracked objects - it is a stall of seconds on every thread. The walk cap applies to the - pass that follows: the clock is checked every - ``_HEAP_CENSUS_CLOCK_STRIDE`` objects and the pass stops, marking the - report truncated, when ``max_seconds`` is used up. That is why the - census runs only on an operator's signal, on its own thread, and never - on a request path. - """ - - def __init__( - self, - config: HeapCensusConfig, - *, - telemetry: ProcessHeapTelemetry | None = None, - clock: Callable[[], float] = time.monotonic, - wall_clock: Callable[[], float] = time.time, - log: Callable[[str], None] = _log_line, - malloc_info_reader: Callable[[], MallocInfoSummary] = read_malloc_info_summary, - ) -> None: - self._config = config - self._telemetry = telemetry if telemetry is not None else ProcessHeapTelemetry() - self._clock = clock - self._wall_clock = wall_clock - self._log = log - self._malloc_info_reader = malloc_info_reader - self._lock = threading.Lock() - self._sequence = 0 - self.last_outcome: HeapCensusOutcome | None = None - - @property - def config(self) -> HeapCensusConfig: - return self._config - - @property - def enabled(self) -> bool: - return self._config.enabled - - def take(self, reason: str) -> HeapCensusOutcome | None: - """Walk the heap once, bounded, and write the report; None when disabled. - - Serialized: a second caller while one census is running waits for it - rather than starting another walk. Never raises past this frame -- - a census that fails is logged and returns None, because the census - thread must survive its own diagnostics. - """ - if not self._config.enabled: - return None - with self._lock: - try: - outcome = self._take_locked(reason) - except Exception as exc: # pragma: no cover - defensive; logged - self._log(f"prism heap census failed reason={reason} error={exc!r}") - return None - self.last_outcome = outcome - return outcome - - def _take_locked(self, reason: str) -> HeapCensusOutcome: - config = self._config - started = self._clock() - taken_at = self._wall_clock() - deadline = started + config.max_seconds - before = self._telemetry.sample() - resident_before = read_resident_memory_bytes() - malloc_info = self._malloc_info_reader() - map_shape = read_anonymous_map_shape() - get_objects_started = self._clock() - objects = gc.get_objects() - get_objects_seconds = self._clock() - get_objects_started - tracked = len(objects) - histogram, walked, truncated, distinct_type_objects = self._walk_types( - objects, deadline - ) - del objects - walk_seconds = self._clock() - started - traced = self._tracemalloc_section(config.top_n) - by_count = sorted(histogram.items(), key=lambda item: (-item[1][0], item[0])) - by_bytes = sorted(histogram.items(), key=lambda item: (-item[1][1], item[0])) - report: dict[str, Any] = { - "format": HEAP_CENSUS_FORMAT, - "reason": _truncate(str(reason), 80), - "taken_at": _datetime.datetime.fromtimestamp( - taken_at, tz=_datetime.timezone.utc - ).isoformat(), - "pid": os.getpid(), - "python": sys.version.split()[0], - "platform": sys.platform, - "bounds": { - "top_n": config.top_n, - "max_bytes": config.max_bytes, - "max_seconds": config.max_seconds, - }, - "walk": { - "tracked_objects": tracked, - "objects_walked": walked, - # Type objects seen, and the printed names they merged into - # (fewer when two types share a name). - "distinct_types": distinct_type_objects, - "distinct_type_names": len(histogram), - "truncated": truncated, - "get_objects_seconds": round(get_objects_seconds, 6), - "seconds": round(walk_seconds, 6), - }, - "process": { - "resident_memory_bytes": resident_before, - "allocated_blocks": before.allocated_blocks, - "threads": before.threads, - "gc_trigger_count": list(before.gc_trigger_count), - "gc_collections": list(before.gc_collections), - "gc_collected": list(before.gc_collected), - "gc_uncollectable": list(before.gc_uncollectable), - "malloc_info_available": before.malloc_info_available, - "malloc_arena_bytes": before.malloc_arena_bytes, - "malloc_in_use_bytes": before.malloc_in_use_bytes, - "malloc_free_bytes": before.malloc_free_bytes, - "malloc_mmapped_bytes": before.malloc_mmapped_bytes, - "malloc_arena_count": malloc_info.arenas, - # Where the free bytes sit decides what malloc_trim can do: - # bins it returns everywhere, per-thread heap tops it cannot. - "malloc_free_top_bytes": malloc_info.top_bytes, - "malloc_free_bin_bytes": malloc_info.bin_bytes, - "malloc_free_fast_bytes": malloc_info.fast_bytes, - "anonymous_map": map_shape.as_dict(), - }, - "tracemalloc": traced, - } - payload, top_n_written = self._fit_payload(report, by_count, by_bytes, config) - path = self._write(payload, taken_at) - self._retain_files(Path(config.output_dir)) - outcome = HeapCensusOutcome( - path=str(path), - bytes_written=len(payload), - tracked_objects=tracked, - objects_walked=walked, - walk_truncated=truncated, - top_n_written=top_n_written, - seconds=self._clock() - started, - tracemalloc_traced=bool(traced.get("tracing")), - ) - self._log( - "prism heap census written " - f"reason={report['reason']} path={outcome.path} bytes={outcome.bytes_written} " - f"tracked_objects={tracked} walked={walked} truncated={truncated} " - f"top_n={top_n_written} seconds={outcome.seconds:.3f} " - f"tracemalloc={'on' if outcome.tracemalloc_traced else 'off'}" - ) - return outcome - - def _walk_types( - self, - objects: list[Any], - deadline: float, - ) -> tuple[dict[str, list[int]], int, bool, int]: - histogram: dict[type, list[int]] = {} - getsizeof = sys.getsizeof - clock = self._clock - walked = 0 - truncated = False - for index, obj in enumerate(objects): - if index % _HEAP_CENSUS_CLOCK_STRIDE == 0 and index and clock() > deadline: - truncated = True - break - kind = type(obj) - entry = histogram.get(kind) - if entry is None: - entry = histogram[kind] = [0, 0] - entry[0] += 1 - try: - entry[1] += getsizeof(obj) - except Exception: - # A __sizeof__ that raises is not the census's problem; the - # count still stands. - pass - walked += 1 - # Two distinct type objects can share a printed name (a class defined - # twice, or two long names that truncate alike). Merge them rather - # than let one overwrite the other: a census that drops a retained - # type is worse than no census, and this is the instrument for - # finding the leak. - named: dict[str, list[int]] = {} - for kind, entry in histogram.items(): - name = _type_name(kind) - merged = named.get(name) - if merged is None: - named[name] = [entry[0], entry[1]] - else: - merged[0] += entry[0] - merged[1] += entry[1] - return named, walked, truncated, len(histogram) - - def _tracemalloc_section(self, top_n: int) -> dict[str, Any]: - if not tracemalloc.is_tracing(): - return {"tracing": False} - traced_now, traced_peak = tracemalloc.get_traced_memory() - snapshot = tracemalloc.take_snapshot().filter_traces( - (tracemalloc.Filter(False, tracemalloc.__file__),) - ) - by_line = [ - { - "site": _truncate(str(stat.traceback), _HEAP_CENSUS_SITE_LIMIT), - "bytes": int(stat.size), - "count": int(stat.count), - } - for stat in snapshot.statistics("lineno")[:top_n] - ] - by_file = [ - { - "site": _truncate(str(stat.traceback), _HEAP_CENSUS_SITE_LIMIT), - "bytes": int(stat.size), - "count": int(stat.count), - } - for stat in snapshot.statistics("filename")[:top_n] - ] - del snapshot - return { - "tracing": True, - "frames": tracemalloc.get_traceback_limit(), - "traced_bytes": int(traced_now), - "traced_peak_bytes": int(traced_peak), - "by_line": by_line, - "by_file": by_file, - } - - @staticmethod - def _fit_payload( - report: dict[str, Any], - by_count: list[tuple[str, list[int]]], - by_bytes: list[tuple[str, list[int]]], - config: HeapCensusConfig, - ) -> tuple[bytes, int]: - """Serialize under ``max_bytes`` by shrinking top-N, never by cutting JSON.""" - top_n = config.top_n - while True: - report["types_by_count"] = [ - {"type": name, "count": entry[0], "shallow_bytes": entry[1]} - for name, entry in by_count[:top_n] - ] - report["types_by_bytes"] = [ - {"type": name, "count": entry[0], "shallow_bytes": entry[1]} - for name, entry in by_bytes[:top_n] - ] - traced = report["tracemalloc"] - if traced.get("tracing"): - traced["by_line"] = traced["by_line"][:top_n] - traced["by_file"] = traced["by_file"][:top_n] - report["top_n_written"] = top_n - report["fit_to_max_bytes"] = top_n < config.top_n - payload = json.dumps(report, indent=1, sort_keys=True).encode("utf-8") - if len(payload) <= config.max_bytes or top_n == 0: - return payload, top_n - top_n //= 2 - - def _write(self, payload: bytes, taken_at: float) -> Path: - directory = Path(self._config.output_dir) - directory.mkdir(parents=True, exist_ok=True) - # heap-census-<UTC stamp>-<pid>-<sequence>.json. The sequence is this - # process's census count, zero-padded, so plain name order is the - # order the reports were written: newest-first retention below never - # consults a filesystem timestamp, whose granularity can tie. - stem = f"{HEAP_CENSUS_FILE_PREFIX}{_utc_stamp(taken_at)}-{os.getpid()}" - while True: - self._sequence += 1 - path = directory / f"{stem}-{self._sequence:0{HEAP_CENSUS_SEQUENCE_WIDTH}d}{HEAP_CENSUS_FILE_SUFFIX}" - if not path.exists(): - break - temporary = path.with_suffix(".tmp") - temporary.write_bytes(payload) - os.replace(temporary, path) - return path - - @staticmethod - def _retain_files(directory: Path) -> None: - """Keep the newest ``HEAP_CENSUS_RETAINED_FILES`` census files. - - Newest by name: the stamp, then the pid, then the zero-padded - per-process sequence, so the order does not depend on the - filesystem's timestamp granularity. - """ - try: - names = sorted( - entry.name - for entry in directory.iterdir() - if entry.name.startswith(HEAP_CENSUS_FILE_PREFIX) - and entry.name.endswith(HEAP_CENSUS_FILE_SUFFIX) - ) - except OSError: - return - for name in names[: max(0, len(names) - HEAP_CENSUS_RETAINED_FILES)]: - try: - (directory / name).unlink() - except OSError: - pass - - -# --------------------------------------------------------------------------- -# malloc_trim, bounded and reported through the always-on telemetry. -# --------------------------------------------------------------------------- - - -def resolve_malloc_trim() -> Callable[[int], int]: - """Bind glibc's ``malloc_trim`` through ctypes, or raise where absent.""" - libc = ctypes.CDLL(None) - function = libc.malloc_trim - function.restype = ctypes.c_int - function.argtypes = (ctypes.c_size_t,) - return function - - -@dataclass(frozen=True) -class MallocTrimResult: - """One ``malloc_trim(0)`` call and the allocator readings around it. - - The deltas are the mechanism the trim is judged by: ``free`` bytes and - RSS should fall while ``in_use`` bytes are unchanged (a trim frees - nothing the program still holds). ``released`` is glibc's own return - value, 1 when any memory went back to the kernel. - """ - - reason: str - released: bool - seconds: float - resident_before: int - resident_after: int - before: ProcessHeapSample - after: ProcessHeapSample - - @property - def resident_delta(self) -> int: - if self.resident_before < 0 or self.resident_after < 0: - return 0 - return self.resident_after - self.resident_before - - @property - def arena_delta(self) -> int: - return self.after.malloc_arena_bytes - self.before.malloc_arena_bytes - - @property - def in_use_delta(self) -> int: - return self.after.malloc_in_use_bytes - self.before.malloc_in_use_bytes - - @property - def free_delta(self) -> int: - return self.after.malloc_free_bytes - self.before.malloc_free_bytes - - def as_dict(self) -> dict[str, Any]: - return { - "reason": self.reason, - "released": self.released, - "seconds": round(self.seconds, 6), - "resident_before": self.resident_before, - "resident_after": self.resident_after, - "resident_delta": self.resident_delta, - "malloc_info_available": self.before.malloc_info_available - and self.after.malloc_info_available, - "arena_before": self.before.malloc_arena_bytes, - "arena_after": self.after.malloc_arena_bytes, - "arena_delta": self.arena_delta, - "in_use_before": self.before.malloc_in_use_bytes, - "in_use_after": self.after.malloc_in_use_bytes, - "in_use_delta": self.in_use_delta, - "free_before": self.before.malloc_free_bytes, - "free_after": self.after.malloc_free_bytes, - "free_delta": self.free_delta, - } - - -class MallocTrimmer: - """Call ``malloc_trim(0)`` once per request and report its effect. - - Serialized by a lock so two trims never overlap. The symbol is resolved - once and its absence remembered, exactly like ``mallinfo2`` in the - collector: on musl or macOS :meth:`trim_once` returns None every time - and never logs more than once. - """ - - def __init__( - self, - telemetry: ProcessHeapTelemetry | None = None, - *, - resolver: Callable[[], Callable[[int], int]] = resolve_malloc_trim, - clock: Callable[[], float] = time.monotonic, - rss_reader: Callable[[], int] = read_resident_memory_bytes, - log: Callable[[str], None] = _log_line, - ) -> None: - self._telemetry = telemetry if telemetry is not None else ProcessHeapTelemetry() - self._resolver = resolver - self._clock = clock - self._rss_reader = rss_reader - self._log = log - self._lock = threading.Lock() - self._malloc_trim: object = _UNRESOLVED - self.last_result: MallocTrimResult | None = None - self.calls = 0 - - def _resolved(self) -> Callable[[int], int] | None: - resolved = self._malloc_trim - if resolved is _UNRESOLVED: - try: - resolved = self._resolver() - except Exception: - resolved = None - self._log("prism malloc_trim unavailable: glibc malloc_trim symbol not bound") - self._malloc_trim = resolved - return resolved # type: ignore[return-value] - - def trim_once(self, reason: str) -> MallocTrimResult | None: - with self._lock: - malloc_trim = self._resolved() - if malloc_trim is None: - return None - before = self._telemetry.sample() - resident_before = self._rss_reader() - started = self._clock() - try: - released = int(malloc_trim(0)) - except Exception as exc: - self._malloc_trim = None - self._log(f"prism malloc_trim failed reason={reason} error={exc!r}") - return None - seconds = self._clock() - started - after = self._telemetry.sample() - resident_after = self._rss_reader() - result = MallocTrimResult( - reason=_truncate(str(reason), 80), - released=bool(released), - seconds=seconds, - resident_before=resident_before, - resident_after=resident_after, - before=before, - after=after, - ) - self.calls += 1 - self.last_result = result - self._log( - "prism malloc_trim " - + " ".join(f"{key}={value}" for key, value in result.as_dict().items()) - ) - return result - - -# --------------------------------------------------------------------------- -# The service: one thread, one pipe, two signals, one periodic action. -# --------------------------------------------------------------------------- - -_CENSUS_BYTE = b"c" -_TRIM_BYTE = b"t" -_STOP_BYTE = b"q" - - -class HeapCensusService: - """Own the census thread and dispatch its requests with rate limits. - - A signal handler runs as Python code on the main thread between two - bytecodes, and a second signal can interrupt it at any bytecode, so the - handler must not take a lock it might already hold: ``Event.set`` would - do exactly that. The handlers therefore only store the reason string - (one attribute assignment) and write one byte to a non-blocking pipe, - which is the same self-pipe shape ``signal.set_wakeup_fd`` uses. A full - pipe means a request is already queued, and the byte is dropped: the - requests coalesce rather than pile up. - - The worker reads the pipe, applies ``min_interval_seconds`` per action - kind, runs the census or the trim, and runs the periodic trim when its - interval is set. Nothing here is reachable from the metrics renderer, - the audit HTTP handler, the public read service, or the Stratum - dispatcher; the coordinator only calls :func:`install_heap_census` from - ``main``. - """ - - THREAD_NAME = "prism-heap-census" - - def __init__( - self, - config: HeapCensusConfig, - *, - census: HeapCensus | None = None, - trimmer: MallocTrimmer | None = None, - telemetry: ProcessHeapTelemetry | None = None, - clock: Callable[[], float] = time.monotonic, - log: Callable[[str], None] = _log_line, - ) -> None: - self._config = config - telemetry = telemetry if telemetry is not None else ProcessHeapTelemetry() - self.census = census if census is not None else HeapCensus( - config, telemetry=telemetry, clock=clock, log=log - ) - self.trimmer = trimmer if trimmer is not None else MallocTrimmer( - telemetry, clock=clock, log=log - ) - self._clock = clock - self._log = log - self._read_fd, self._write_fd = os.pipe() - os.set_blocking(self._write_fd, False) - os.set_blocking(self._read_fd, False) - self._census_reason = "unset" - self._trim_reason = "unset" - self._last_census_monotonic: float | None = None - self._last_trim_monotonic: float | None = None - self._next_periodic_trim: float | None = ( - clock() + config.malloc_trim_interval_seconds - if config.malloc_trim_interval_seconds > 0 - else None - ) - self._thread: threading.Thread | None = None - self._closed = False - # (signal module, signum, previous handler) for every handler this - # service armed, so close() can disarm them before the pipe goes. - self._armed: list[tuple[Any, int, Any]] = [] - self.suppressed = {"census": 0, "trim": 0} - - @property - def config(self) -> HeapCensusConfig: - return self._config - - # -- signal registration ------------------------------------------------ - - def arm_signal(self, signal_module: Any, signum: int, handler: Any) -> None: - """Register ``handler`` and remember what it replaced.""" - getsignal = getattr(signal_module, "getsignal", None) - previous = getsignal(signum) if callable(getsignal) else None - signal_module.signal(signum, handler) - self._armed.append((signal_module, signum, previous)) - - def disarm_signals(self) -> None: - """Restore every handler :meth:`arm_signal` replaced.""" - while self._armed: - signal_module, signum, previous = self._armed.pop() - if previous is None: - previous = getattr(signal_module, "SIG_DFL", None) - if previous is None: - continue - try: - signal_module.signal(signum, previous) - except (OSError, ValueError, TypeError): - pass - - # -- signal-safe entry points ------------------------------------------- - - def _wake(self, byte: bytes) -> bool: - # After close() the descriptor numbers may belong to something else; - # a late signal must not write into a stranger's pipe. - if self._closed: - return False - try: - os.write(self._write_fd, byte) - except (BlockingIOError, InterruptedError, OSError): - return False - return True - - def request_census(self, reason: str) -> bool: - """Queue a census; safe to call from a signal handler.""" - if not self._config.enabled: - return False - self._census_reason = reason - return self._wake(_CENSUS_BYTE) - - def request_trim(self, reason: str) -> bool: - """Queue a trim; safe to call from a signal handler.""" - self._trim_reason = reason - return self._wake(_TRIM_BYTE) - - # -- dispatch with rate limits ------------------------------------------ - - def dispatch(self, kind: str, reason: str) -> object | None: - """Run one action now unless its rate limit suppresses it. - - Returns the census outcome, the trim result, or None when suppressed - or unavailable. The periodic trim goes through the same gate. - """ - now = self._clock() - floor = self._config.min_interval_seconds - if kind == "census": - if not self._config.enabled: - return None - last = self._last_census_monotonic - if last is not None and now - last < floor: - self.suppressed["census"] += 1 - self._log( - f"prism heap census suppressed reason={reason} " - f"min_interval_seconds={floor:g} since_last={now - last:.1f}" - ) - return None - outcome = self.census.take(reason) - self._last_census_monotonic = self._clock() - return outcome - if kind == "trim": - last = self._last_trim_monotonic - if last is not None and now - last < floor: - self.suppressed["trim"] += 1 - self._log( - f"prism malloc_trim suppressed reason={reason} " - f"min_interval_seconds={floor:g} since_last={now - last:.1f}" - ) - return None - result = self.trimmer.trim_once(reason) - self._last_trim_monotonic = self._clock() - return result - raise ValueError(f"unknown census action {kind!r}") - - def periodic_trim_due(self) -> bool: - return ( - self._next_periodic_trim is not None - and self._clock() >= self._next_periodic_trim - ) - - def _timeout(self) -> float | None: - if self._next_periodic_trim is None: - return None - return max(0.0, self._next_periodic_trim - self._clock()) - - def run_once(self, timeout: float | None) -> bool: - """One worker iteration: wait, then serve what arrived. False on stop.""" - try: - ready, _w, _x = select.select([self._read_fd], [], [], timeout) - except (OSError, ValueError): - return False - if ready: - try: - pending = os.read(self._read_fd, 64) - except (BlockingIOError, InterruptedError): - pending = b"" - except OSError: - return False - if _STOP_BYTE in pending: - return False - if _CENSUS_BYTE in pending: - self.dispatch("census", self._census_reason) - if _TRIM_BYTE in pending: - self.dispatch("trim", self._trim_reason) - if self.periodic_trim_due(): - self.dispatch("trim", "periodic") - self._next_periodic_trim = ( - self._clock() + self._config.malloc_trim_interval_seconds - ) - return True - - def run_forever(self) -> None: - while not self._closed: - try: - if not self.run_once(self._timeout()): - return - except Exception as exc: # pragma: no cover - defensive; logged - self._log(f"prism heap census thread error={exc!r}") - - def start(self) -> threading.Thread: - if self._thread is None: - self._thread = threading.Thread( - target=self.run_forever, - name=self.THREAD_NAME, - daemon=True, - ) - self._thread.start() - return self._thread - - def close(self) -> None: - """Disarm the signals, stop the worker, release the pipe. - - Order matters: handlers first, so no signal can reach ``_wake`` once - the descriptors are gone; then the stop byte while the pipe is still - open; then ``_closed`` so any handler already past the disarm is a - no-op; then the descriptors. Tests call this; in the coordinator the - daemon thread dies with the process. - """ - if self._closed: - return - self.disarm_signals() - self._wake(_STOP_BYTE) - self._closed = True - thread = self._thread - if thread is not None and thread is not threading.current_thread(): - thread.join(timeout=5.0) - for descriptor in (self._read_fd, self._write_fd): - try: - os.close(descriptor) - except OSError: - pass - - -def heap_census_signal(signal_module: Any) -> int | None: - return getattr(signal_module, "SIGUSR1", None) - - -def malloc_trim_signal(signal_module: Any) -> int | None: - base = getattr(signal_module, "SIGRTMIN", None) - if base is None: - return None - return int(base) + 1 - - -def install_heap_census( - config: HeapCensusConfig, - *, - signal_module: Any = None, - log: Callable[[str], None] = _log_line, - telemetry: ProcessHeapTelemetry | None = None, - start_thread: bool = True, -) -> HeapCensusService | None: - """Register the census and trim signals and start the worker; None when off. - - The one call ``prism_coordinator.main`` makes. With every switch off - nothing is registered, no thread starts, tracemalloc is not started, - and the return value is None: the walk is unreachable, not merely idle. - ``signal_module`` is injectable so a test can prove which handlers were - registered without sending real signals to itself. - """ - if not config.anything_armed: - return None - if signal_module is None: - import signal as signal_module # noqa: PLC0415 - registration only - if config.enabled and config.tracemalloc_enabled and not tracemalloc.is_tracing(): - tracemalloc.start(HEAP_CENSUS_TRACEMALLOC_FRAMES) - service = HeapCensusService(config, telemetry=telemetry, log=log) - armed: list[str] = [] - if config.enabled: - census_signal = heap_census_signal(signal_module) - if census_signal is None: - log("prism heap census: SIGUSR1 unavailable on this platform; census not armed") - else: - - def _on_census_signal(signum: int, _frame: Any) -> None: - service.request_census(f"signal:{signum}") - - service.arm_signal(signal_module, census_signal, _on_census_signal) - armed.append(f"census={HEAP_CENSUS_SIGNAL_NAME}") - if config.malloc_trim_signal_enabled: - trim_signal = malloc_trim_signal(signal_module) - if trim_signal is None: - log("prism malloc_trim: SIGRTMIN unavailable on this platform; trim signal not armed") - else: - - def _on_trim_signal(signum: int, _frame: Any) -> None: - service.request_trim(f"signal:{signum}") - - service.arm_signal(signal_module, trim_signal, _on_trim_signal) - armed.append(f"trim={MALLOC_TRIM_SIGNAL_NAME}") - if config.malloc_trim_interval_seconds > 0: - armed.append(f"periodic_trim={config.malloc_trim_interval_seconds:g}s") - if start_thread: - service.start() - log( - "prism heap census armed " - + " ".join(armed) - + f" dir={config.output_dir} top_n={config.top_n} max_bytes={config.max_bytes}" - f" max_seconds={config.max_seconds:g} min_interval_seconds={config.min_interval_seconds:g}" - f" tracemalloc={'on' if config.enabled and config.tracemalloc_enabled else 'off'}" - ) - return service - - -# --------------------------------------------------------------------------- -# The RSS bound: the automated soak check. -# --------------------------------------------------------------------------- - -# The stated bound. The baseline is the largest resident set seen during the -# warm-up (one hour: long enough for the payout window to be materialized -# and the first full rescan to run), and every later sample must stay under -# twice it. The #226 slope (390 MB/h from a 125-145 MiB start) breaches this -# inside the second hour; #185's post-storm drain (125 -> 613 MiB) would -# breach it only if the storm fell outside the warm-up and did not drain. -RSS_BOUND_DEFAULT_MULTIPLE = 2.0 -RSS_BOUND_DEFAULT_WARMUP_SECONDS = 3600.0 -# A 24 h soak with an hour of tolerance for a late first sample. -RSS_BOUND_DEFAULT_MIN_SPAN_SECONDS = 23 * 3600.0 - - -@dataclass(frozen=True) -class RssBoundVerdict: - passed: bool - reasons: tuple[str, ...] - multiple: float - warmup_seconds: float - samples: int - span_seconds: float - baseline_bytes: int - bound_bytes: int - peak_bytes: int - peak_at_seconds: float - peak_ratio: float - first_breach_at_seconds: float | None - slope_bytes_per_hour: float | None - - def as_dict(self) -> dict[str, Any]: - return { - "passed": self.passed, - "reasons": list(self.reasons), - "multiple": self.multiple, - "warmup_seconds": self.warmup_seconds, - "samples": self.samples, - "span_seconds": self.span_seconds, - "baseline_bytes": self.baseline_bytes, - "bound_bytes": self.bound_bytes, - "peak_bytes": self.peak_bytes, - "peak_at_seconds": self.peak_at_seconds, - "peak_ratio": self.peak_ratio, - "first_breach_at_seconds": self.first_breach_at_seconds, - "slope_bytes_per_hour": self.slope_bytes_per_hour, - } - - -def _least_squares_slope(points: Sequence[tuple[float, float]]) -> float | None: - if len(points) < 2: - return None - n = float(len(points)) - mean_x = sum(x for x, _y in points) / n - mean_y = sum(y for _x, y in points) / n - denominator = sum((x - mean_x) ** 2 for x, _y in points) - if denominator == 0: - return None - return sum((x - mean_x) * (y - mean_y) for x, y in points) / denominator - - -def evaluate_rss_bound( - samples: Iterable[tuple[float, int]], - *, - warmup_seconds: float = RSS_BOUND_DEFAULT_WARMUP_SECONDS, - multiple: float = RSS_BOUND_DEFAULT_MULTIPLE, - min_span_seconds: float = 0.0, - max_slope_bytes_per_hour: float | None = None, -) -> RssBoundVerdict: - """Judge a ``(seconds, rss_bytes)`` series against the stated multiple. - - Times may be absolute or relative; they are re-based on the first - sample. Samples with a negative RSS (the -1 "unavailable" sentinel) are - ignored. The verdict fails when any post-warm-up sample exceeds - ``multiple`` times the warm-up peak, when the series is too short to - contain both a warm-up and a post-warm-up sample, when it spans less - than ``min_span_seconds``, or when ``max_slope_bytes_per_hour`` is given - and the least-squares slope over the post-warm-up samples exceeds it. - """ - if multiple <= 1.0: - raise ValueError("multiple must be greater than 1.0") - if warmup_seconds < 0: - raise ValueError("warmup_seconds must be non-negative") - ordered = sorted( - (float(seconds), int(rss)) for seconds, rss in samples if int(rss) >= 0 - ) - reasons: list[str] = [] - if not ordered: - return RssBoundVerdict( - passed=False, - reasons=("no usable samples",), - multiple=multiple, - warmup_seconds=warmup_seconds, - samples=0, - span_seconds=0.0, - baseline_bytes=-1, - bound_bytes=-1, - peak_bytes=-1, - peak_at_seconds=0.0, - peak_ratio=0.0, - first_breach_at_seconds=None, - slope_bytes_per_hour=None, - ) - origin = ordered[0][0] - rebased = [(seconds - origin, rss) for seconds, rss in ordered] - span = rebased[-1][0] - warmup = [(t, r) for t, r in rebased if t <= warmup_seconds] - steady = [(t, r) for t, r in rebased if t > warmup_seconds] - if not warmup: - reasons.append("no sample inside the warm-up window") - if not steady: - reasons.append("no sample after the warm-up window") - if span < min_span_seconds: - reasons.append( - f"series spans {span:.0f}s, below the required {min_span_seconds:.0f}s" - ) - baseline = max((r for _t, r in warmup), default=-1) - bound = int(baseline * multiple) if baseline >= 0 else -1 - peak_at, peak = max(rebased, key=lambda item: (item[1], -item[0])) - peak_ratio = (peak / baseline) if baseline > 0 else 0.0 - first_breach: float | None = None - if baseline >= 0: - for t, r in steady: - if r > bound: - first_breach = t - break - if first_breach is not None: - reasons.append( - f"resident set exceeded {multiple:g}x the warm-up baseline " - f"({baseline} bytes) at {first_breach:.0f}s" - ) - slope_per_second = _least_squares_slope([(t, float(r)) for t, r in steady]) - slope_per_hour = slope_per_second * 3600.0 if slope_per_second is not None else None - if ( - max_slope_bytes_per_hour is not None - and slope_per_hour is not None - and slope_per_hour > max_slope_bytes_per_hour - ): - reasons.append( - f"post-warm-up slope {slope_per_hour:.0f} bytes/h exceeds " - f"{max_slope_bytes_per_hour:.0f} bytes/h" - ) - return RssBoundVerdict( - passed=not reasons, - reasons=tuple(reasons), - multiple=multiple, - warmup_seconds=warmup_seconds, - samples=len(rebased), - span_seconds=span, - baseline_bytes=baseline, - bound_bytes=bound, - peak_bytes=peak, - peak_at_seconds=peak_at, - peak_ratio=peak_ratio, - first_breach_at_seconds=first_breach, - slope_bytes_per_hour=slope_per_hour, - ) - - -def read_rss_samples(path: Path) -> list[tuple[float, int]]: - """Parse ``seconds,rss_bytes`` lines; blank lines and ``#`` comments are skipped.""" - samples: list[tuple[float, int]] = [] - for number, raw in enumerate(path.read_text(encoding="utf-8").splitlines(), 1): - line = raw.split("#", 1)[0].strip() - if not line: - continue - parts = [part.strip() for part in line.replace("\t", ",").split(",")] - if len(parts) < 2: - raise ValueError(f"{path}:{number}: expected 'seconds,rss_bytes'") - try: - samples.append((float(parts[0]), int(float(parts[1])))) - except ValueError as exc: - raise ValueError(f"{path}:{number}: {exc}") from exc - return samples - - -def _build_cli() -> argparse.ArgumentParser: - parser = argparse.ArgumentParser( - prog="python3 -m lab.prism.process_telemetry", - description="Issue #226 soak tooling: sample a resident set, judge a series.", - ) - commands = parser.add_subparsers(dest="command", required=True) - sample = commands.add_parser( - "rss-sample", - help="print one 'epoch_seconds,rss_bytes' line for a pid", - ) - sample.add_argument("--pid", default="self", help="process id (default: this process)") - bound = commands.add_parser( - "rss-bound", - help="judge a 'seconds,rss_bytes' series against the stated multiple", - ) - bound.add_argument("--samples", type=Path, required=True, help="CSV of seconds,rss_bytes") - bound.add_argument( - "--warmup-seconds", - type=float, - default=RSS_BOUND_DEFAULT_WARMUP_SECONDS, - help=f"baseline window (default {RSS_BOUND_DEFAULT_WARMUP_SECONDS:g})", - ) - bound.add_argument( - "--multiple", - type=float, - default=RSS_BOUND_DEFAULT_MULTIPLE, - help=f"allowed multiple of the warm-up peak (default {RSS_BOUND_DEFAULT_MULTIPLE:g})", - ) - bound.add_argument( - "--min-span-seconds", - type=float, - default=RSS_BOUND_DEFAULT_MIN_SPAN_SECONDS, - help=f"refuse a series shorter than this (default {RSS_BOUND_DEFAULT_MIN_SPAN_SECONDS:g})", - ) - bound.add_argument( - "--max-slope-bytes-per-hour", - type=float, - default=None, - help="also fail when the post-warm-up least-squares slope exceeds this", - ) - return parser - - -def main(argv: Sequence[str] | None = None) -> int: - args = _build_cli().parse_args(argv) - if args.command == "rss-sample": - rss = read_resident_memory_bytes(args.pid) - print(f"{time.time():.0f},{rss}") - return 0 if rss >= 0 else 1 - try: - samples = read_rss_samples(args.samples) - except (OSError, ValueError) as exc: - print(f"rss-bound: {exc}", file=sys.stderr) - return 2 - try: - verdict = evaluate_rss_bound( - samples, - warmup_seconds=args.warmup_seconds, - multiple=args.multiple, - min_span_seconds=args.min_span_seconds, - max_slope_bytes_per_hour=args.max_slope_bytes_per_hour, - ) - except ValueError as exc: - print(f"rss-bound: {exc}", file=sys.stderr) - return 2 - print(json.dumps(verdict.as_dict(), indent=2, sort_keys=True)) - return 0 if verdict.passed else 1 - - -__all__ = [ - "ANONYMOUS_REGION_BANDS", - "AnonymousMapShape", - "HEAP_CENSUS_FORMAT", - "HEAP_CENSUS_RETAINED_FILES", - "HEAP_CENSUS_SIGNAL_NAME", - "HEAP_CENSUS_TRACEMALLOC_FRAMES", - "HeapCensus", - "HeapCensusConfig", - "HeapCensusOutcome", - "HeapCensusService", - "MALLOC_TRIM_SIGNAL_NAME", - "MallInfo2", - "MallocInfoSummary", - "MallocTrimResult", - "MallocTrimmer", - "PRISM_GC_GENERATIONS", - "ProcessHeapSample", - "ProcessHeapTelemetry", - "RSS_BOUND_DEFAULT_MIN_SPAN_SECONDS", - "RSS_BOUND_DEFAULT_MULTIPLE", - "RSS_BOUND_DEFAULT_WARMUP_SECONDS", - "RssBoundVerdict", - "evaluate_rss_bound", - "heap_census_signal", - "install_heap_census", - "main", - "malloc_trim_signal", - "parse_malloc_info_xml", - "read_anonymous_map_shape", - "read_malloc_arena_count", - "read_malloc_info_summary", - "read_resident_memory_bytes", - "read_rss_samples", - "resolve_malloc_trim", - "resolve_mallinfo2", -] - - -if __name__ == "__main__": - raise SystemExit(main()) diff --git a/lab/prism/progress_health.py b/lab/prism/progress_health.py deleted file mode 100644 index a20ab4a70..000000000 --- a/lab/prism/progress_health.py +++ /dev/null @@ -1,698 +0,0 @@ -#!/usr/bin/env python3 -"""Monotonic mining-progress readiness state for the PRISM coordinator.""" - -from __future__ import annotations - -from dataclasses import dataclass -from types import MappingProxyType -import threading -import time -from typing import Callable, Mapping - - -PROGRESS_HEALTH_REASONS = ( - "tip_poll_stale", - "refresh_pending_too_long", - "current_generation_not_published", - "current_generation_not_delivered", - "bundle_build_stuck", -) - - -@dataclass(frozen=True) -class WorkGeneration: - template_generation: int - template_fingerprint: str | None - payout_generation: int - - -@dataclass(frozen=True) -class DeliveryProof: - connection_id: int - delivered_work: WorkGeneration - collection_only: bool - delivered_monotonic: float - - -@dataclass(frozen=True) -class EligibilitySnapshot: - eligible_connection_ids: tuple[int, ...] - delivery_proofs: tuple[DeliveryProof, ...] - ready_mode_required: bool - - -# Adapter signature used by the coordinator to supply live client facts: -# ``(template_fingerprint, payout_generation) -> (eligible, requiring_refresh)``. -EligibleClientCounts = Callable[[str | None, int], tuple[int, int]] - - -@dataclass(frozen=True) -class ProgressHealthConfig: - pending_refresh_deadline_seconds: float - tip_poll_deadline_seconds: float - bundle_build_deadline_seconds: float - - -@dataclass(frozen=True) -class ProgressHealthSnapshot: - ok: bool - reason: str | None - reasons: tuple[str, ...] - pending_refresh: bool - pending_refresh_age_seconds: float | None - tip_poll_age_seconds: float - tip_refresh_in_progress: bool - tip_refresh_progress_age_seconds: float | None - current_template_generation: int - published_template_generation: int - current_payout_generation: int - published_payout_generation: int - last_valid_delivery_age_seconds: float | None - eligible_client_count: int - eligible_clients_requiring_refresh: int - bundle_build_oldest_age_seconds: float - - def as_mapping(self) -> dict[str, object]: - """Return the existing mutable HTTP/test compatibility representation.""" - - return { - "ok": self.ok, - "reason": self.reason, - "reasons": list(self.reasons), - "pending_refresh": self.pending_refresh, - "pending_refresh_age_seconds": self.pending_refresh_age_seconds, - "tip_poll_age_seconds": self.tip_poll_age_seconds, - "tip_refresh_in_progress": self.tip_refresh_in_progress, - "tip_refresh_progress_age_seconds": self.tip_refresh_progress_age_seconds, - "current_template_generation": self.current_template_generation, - "published_template_generation": self.published_template_generation, - "current_payout_generation": self.current_payout_generation, - "published_payout_generation": self.published_payout_generation, - "last_valid_delivery_age_seconds": self.last_valid_delivery_age_seconds, - "eligible_client_count": self.eligible_client_count, - "eligible_clients_requiring_refresh": ( - self.eligible_clients_requiring_refresh - ), - "bundle_build_oldest_age_seconds": self.bundle_build_oldest_age_seconds, - } - - -@dataclass(frozen=True) -class _ProgressStateCopy: - current_template_generation: int - current_template_fingerprint: str | None - current_payout_generation: int - published_template_generation: int - published_template_fingerprint: str | None - published_payout_generation: int - has_published_work: bool - last_tip_poll_monotonic: float | None - last_delivery_template_fingerprint: str | None - last_delivery_payout_generation: int - last_delivery_monotonic: float | None - pending_since_monotonic: float | None - publication_divergence_since_monotonic: float | None - refresh_signal_pending: bool - active_refresh_count: int - last_refresh_activity_monotonic: float | None - bundle_build_starts: tuple[float, ...] - - -class RefreshActivityToken: - """One idempotent, context-managed refresh activity lifetime.""" - - def __init__(self, service: ProgressHealthService) -> None: - self._service = service - self._state_lock = threading.Lock() - self._finished = False - - def note_activity(self, observed_monotonic: float | None = None) -> None: - with self._state_lock: - if self._finished: - return - self._service.note_refresh_activity(observed_monotonic) - - def finish(self) -> None: - with self._state_lock: - if self._finished: - return - self._finished = True - self._service.refresh_finished() - - cancel = finish - - def __enter__(self) -> RefreshActivityToken: - return self - - def __exit__(self, *_args: object) -> None: - self.finish() - - -class BundleBuildToken: - """One idempotent, context-managed bundle construction lifetime.""" - - def __init__(self, service: ProgressHealthService, token_id: int) -> None: - self._service = service - self._token_id = token_id - self._state_lock = threading.Lock() - self._finished = False - - def finish(self) -> None: - with self._state_lock: - if self._finished: - return - self._finished = True - self._service.bundle_build_finished(self._token_id) - - cancel = finish - - def __enter__(self) -> BundleBuildToken: - return self - - def __exit__(self, *_args: object) -> None: - self.finish() - - -class ProgressHealthService: - """Own aggregate mining-progress state and evaluate bounded readiness. - - State fields are deliberately plain attributes matching the historical - coordinator layout one-for-one: the coordinator routes its legacy raw - ``_progress_*`` names here through class descriptors, and the historical - white-box protocol mutates them directly while holding ``_lock``. Service - methods take the lock themselves. - """ - - def __init__( - self, - *, - started_monotonic: float, - initial_payout_generation: int = 0, - monotonic: Callable[[], float] = time.monotonic, - ) -> None: - self._monotonic = monotonic - self._lock = threading.Lock() - self._current_template_generation = 0 - self._current_template_fingerprint: str | None = None - self._current_payout_generation = int(initial_payout_generation) - self._published_template_generation = 0 - self._published_template_fingerprint: str | None = None - self._published_payout_generation = 0 - self._has_published_work = False - self._last_tip_poll_monotonic: float | None = None - self._last_delivery_template_generation = 0 - self._last_delivery_template_fingerprint: str | None = None - self._last_delivery_payout_generation = 0 - self._last_delivery_monotonic: float | None = None - self._pending_since_monotonic: float | None = float(started_monotonic) - # Start of the current detected-vs-published divergence epoch. Cleared - # only by current-work publication/delivery that matches the latest - # detected tip, so old delivery backlog cannot renew or expire a later - # legitimate divergence (the publication watchdog reads it). - self._publication_divergence_since_monotonic: float | None = float( - started_monotonic - ) - self._refresh_signal_pending = False - self._active_refresh_count = 0 - self._last_refresh_activity_monotonic: float | None = None - self._bundle_build_counter = 0 - self._bundle_builds: dict[int, float] = {} - - def now(self) -> float: - return float(self._monotonic()) - - def mark_refresh_pending(self, started_monotonic: float | None = None) -> None: - started = self.now() if started_monotonic is None else started_monotonic - with self._lock: - pending_since = self._pending_since_monotonic - if pending_since is None or started < pending_since: - self._pending_since_monotonic = started - divergence_since = self._publication_divergence_since_monotonic - if divergence_since is None or started < divergence_since: - self._publication_divergence_since_monotonic = started - self._refresh_signal_pending = True - - def observe_tip( - self, - work: WorkGeneration, - observed_monotonic: float | None = None, - ) -> None: - """Publish a coherent qbit tip/template observation to health state.""" - observed = self.now() if observed_monotonic is None else observed_monotonic - with self._lock: - if work.template_generation < self._current_template_generation: - # A slower concurrent poll can finish after a newer coherent - # observation. It proves only that the obsolete generation was - # coherent, so it must not renew freshness for current work. - return - self._current_template_generation = work.template_generation - self._current_template_fingerprint = work.template_fingerprint - self._current_payout_generation = max( - self._current_payout_generation, - work.payout_generation, - ) - self._last_tip_poll_monotonic = observed - same_published_work = bool( - self._has_published_work - and self._published_template_fingerprint - == work.template_fingerprint - and self._published_payout_generation - == self._current_payout_generation - ) - if same_published_work: - # Observation generations order RPC races. When the semantic - # template fingerprint and payout generation are unchanged, - # already-issued work remains current and can be reconciled to - # the latest observation without a needless socket delivery. - self._published_template_generation = work.template_generation - else: - pending_since = self._pending_since_monotonic - if pending_since is None or observed < pending_since: - self._pending_since_monotonic = observed - divergence_since = self._publication_divergence_since_monotonic - if divergence_since is None or observed < divergence_since: - self._publication_divergence_since_monotonic = observed - - def observe_payout_generation( - self, - generation: int, - invalidated_monotonic: float | None = None, - ) -> None: - invalidated = ( - self.now() - if invalidated_monotonic is None - else invalidated_monotonic - ) - with self._lock: - if generation < self._current_payout_generation: - return - self._current_payout_generation = generation - if generation != self._published_payout_generation: - pending_since = self._pending_since_monotonic - if pending_since is None or invalidated < pending_since: - self._pending_since_monotonic = invalidated - divergence_since = self._publication_divergence_since_monotonic - if divergence_since is None or invalidated < divergence_since: - self._publication_divergence_since_monotonic = invalidated - self._refresh_signal_pending = True - - def publish_work( - self, - work: WorkGeneration, - *, - matches_latest_tip: bool = True, - ) -> bool: - """Record that current in-memory work is available for delivery.""" - with self._lock: - if ( - work.template_generation < self._published_template_generation - or work.payout_generation < self._published_payout_generation - ): - return False - self._published_template_generation = max( - self._published_template_generation, - work.template_generation, - ) - self._published_template_fingerprint = work.template_fingerprint - self._published_payout_generation = max( - self._published_payout_generation, - work.payout_generation, - ) - self._has_published_work = True - if ( - matches_latest_tip - and self._current_template_fingerprint == work.template_fingerprint - and self._current_payout_generation == work.payout_generation - ): - self._refresh_signal_pending = False - self._publication_divergence_since_monotonic = None - return True - - def record_delivery( - self, - proof: DeliveryProof, - ready_mode_required: bool, - *, - matches_latest_tip: bool = True, - ) -> bool: - """Record a completed current-generation socket delivery.""" - work = proof.delivered_work - with self._lock: - if ( - work.template_fingerprint == self._current_template_fingerprint - and work.payout_generation == self._current_payout_generation - and not (ready_mode_required and proof.collection_only) - ): - self._last_delivery_template_generation = work.template_generation - self._last_delivery_template_fingerprint = work.template_fingerprint - self._last_delivery_payout_generation = work.payout_generation - self._last_delivery_monotonic = proof.delivered_monotonic - # A successful delivery proves publication for its coherent - # generation. Only the latest observed tip can close the - # outstanding divergence below. - self._published_template_generation = max( - self._published_template_generation, - self._current_template_generation, - ) - self._published_template_fingerprint = work.template_fingerprint - self._published_payout_generation = max( - self._published_payout_generation, - work.payout_generation, - ) - self._has_published_work = True - self._refresh_signal_pending = False - if matches_latest_tip: - self._publication_divergence_since_monotonic = None - return True - return False - - def refresh_started(self) -> None: - """Track a coherent poll pass while it continues making progress.""" - started = self.now() - with self._lock: - self._active_refresh_count += 1 - self._last_refresh_activity_monotonic = started - - def note_refresh_activity( - self, - observed_monotonic: float | None = None, - ) -> None: - observed = ( - self.now() - if observed_monotonic is None - else observed_monotonic - ) - with self._lock: - if self._active_refresh_count > 0: - last_activity = self._last_refresh_activity_monotonic - if last_activity is None or observed > last_activity: - self._last_refresh_activity_monotonic = observed - - def refresh_finished(self) -> None: - with self._lock: - self._active_refresh_count = max(0, self._active_refresh_count - 1) - - def start_refresh(self) -> RefreshActivityToken: - self.refresh_started() - return RefreshActivityToken(self) - - def bundle_build_started(self) -> int: - started = self.now() - with self._lock: - self._bundle_build_counter += 1 - token = self._bundle_build_counter - self._bundle_builds[token] = started - return token - - def bundle_build_finished(self, token: int) -> None: - with self._lock: - self._bundle_builds.pop(token, None) - - def start_bundle_build(self) -> BundleBuildToken: - return BundleBuildToken(self, self.bundle_build_started()) - - @staticmethod - def requiring_refresh( - eligibility: EligibilitySnapshot, - work: WorkGeneration, - ) -> int: - proofs = { - proof.connection_id: proof - for proof in eligibility.delivery_proofs - } - requiring_refresh = 0 - for connection_id in eligibility.eligible_connection_ids: - proof = proofs.get(connection_id) - if ( - proof is None - or proof.delivered_work.template_fingerprint - != work.template_fingerprint - or proof.delivered_work.payout_generation - != work.payout_generation - or ( - eligibility.ready_mode_required - and proof.collection_only - ) - ): - requiring_refresh += 1 - return requiring_refresh - - def _copy_state_locked(self) -> _ProgressStateCopy: - return _ProgressStateCopy( - current_template_generation=self._current_template_generation, - current_template_fingerprint=self._current_template_fingerprint, - current_payout_generation=self._current_payout_generation, - published_template_generation=self._published_template_generation, - published_template_fingerprint=self._published_template_fingerprint, - published_payout_generation=self._published_payout_generation, - has_published_work=self._has_published_work, - last_tip_poll_monotonic=self._last_tip_poll_monotonic, - last_delivery_template_fingerprint=( - self._last_delivery_template_fingerprint - ), - last_delivery_payout_generation=self._last_delivery_payout_generation, - last_delivery_monotonic=self._last_delivery_monotonic, - pending_since_monotonic=self._pending_since_monotonic, - publication_divergence_since_monotonic=( - self._publication_divergence_since_monotonic - ), - refresh_signal_pending=self._refresh_signal_pending, - active_refresh_count=self._active_refresh_count, - last_refresh_activity_monotonic=( - self._last_refresh_activity_monotonic - ), - bundle_build_starts=tuple(self._bundle_builds.values()), - ) - - def publication_divergence_since(self) -> float | None: - """Locked read of the divergence stamp for the publication watchdog.""" - with self._lock: - return self._publication_divergence_since_monotonic - - def reconcile_pending( - self, - eligible_counts: EligibleClientCounts, - *, - now: float | None = None, - ) -> None: - """Clear pending state exactly when publication/delivery is sufficient.""" - current = self.now() if now is None else now - with self._lock: - state_key = ( - self._current_template_fingerprint, - self._current_payout_generation, - ) - _, requiring_refresh = eligible_counts(*state_key) - with self._lock: - if state_key != ( - self._current_template_fingerprint, - self._current_payout_generation, - ): - return - published_current = bool( - self._has_published_work - and self._published_template_fingerprint == state_key[0] - and self._published_payout_generation == state_key[1] - ) - refresh_required = bool( - self._refresh_signal_pending - or not published_current - or requiring_refresh > 0 - ) - if refresh_required: - if self._pending_since_monotonic is None: - self._pending_since_monotonic = current - else: - self._pending_since_monotonic = None - - def snapshot( - self, - eligible_counts: EligibleClientCounts, - current_payout_generation: int, - config: ProgressHealthConfig, - started_monotonic: float | None = None, - *, - now: float | None = None, - ) -> ProgressHealthSnapshot: - """Return a bounded, monotonic-only mining progress health snapshot.""" - current = self.now() if now is None else now - with self._lock: - tracked_payout_generation = self._current_payout_generation - if current_payout_generation > tracked_payout_generation: - self.observe_payout_generation(current_payout_generation, current) - self.reconcile_pending(eligible_counts, now=current) - - with self._lock: - state = self._copy_state_locked() - - eligible_count, requiring_refresh = eligible_counts( - state.current_template_fingerprint, - state.current_payout_generation, - ) - started = ( - current if started_monotonic is None else float(started_monotonic) - ) - tip_poll_reference = ( - started - if state.last_tip_poll_monotonic is None - else state.last_tip_poll_monotonic - ) - tip_poll_age = max(0.0, current - tip_poll_reference) - refresh_activity_age = ( - None - if ( - state.active_refresh_count <= 0 - or state.last_refresh_activity_monotonic is None - ) - else max(0.0, current - state.last_refresh_activity_monotonic) - ) - pending_age = ( - None - if state.pending_since_monotonic is None - else max(0.0, current - state.pending_since_monotonic) - ) - oldest_bundle_age = ( - 0.0 - if not state.bundle_build_starts - else max(0.0, current - min(state.bundle_build_starts)) - ) - published_current = bool( - state.has_published_work - and state.published_template_fingerprint - == state.current_template_fingerprint - and state.published_payout_generation - == state.current_payout_generation - ) - delivered_current = bool( - state.current_template_fingerprint is not None - and state.last_delivery_template_fingerprint - == state.current_template_fingerprint - and state.last_delivery_payout_generation - == state.current_payout_generation - ) - delivery_age = ( - max(0.0, current - state.last_delivery_monotonic) - if delivered_current and state.last_delivery_monotonic is not None - else None - ) - reasons: list[str] = [] - refresh_is_progressing = bool( - state.active_refresh_count > 0 - and refresh_activity_age is not None - and refresh_activity_age <= config.tip_poll_deadline_seconds - ) - if ( - tip_poll_age > config.tip_poll_deadline_seconds - and not refresh_is_progressing - ): - reasons.append("tip_poll_stale") - if oldest_bundle_age > config.bundle_build_deadline_seconds: - reasons.append("bundle_build_stuck") - if not state.has_published_work: - reasons.append("current_generation_not_published") - elif ( - pending_age is not None - and pending_age > config.pending_refresh_deadline_seconds - ): - reasons.append("refresh_pending_too_long") - if state.refresh_signal_pending or not published_current: - reasons.append("current_generation_not_published") - elif requiring_refresh > 0: - reasons.append("current_generation_not_delivered") - reasons = list(dict.fromkeys(reasons)) - - def rounded(value: float | None) -> float | None: - return None if value is None else round(value, 3) - - return ProgressHealthSnapshot( - ok=not reasons, - reason=reasons[0] if reasons else None, - reasons=tuple(reasons), - pending_refresh=state.pending_since_monotonic is not None, - pending_refresh_age_seconds=rounded(pending_age), - tip_poll_age_seconds=rounded(tip_poll_age), - tip_refresh_in_progress=state.active_refresh_count > 0, - tip_refresh_progress_age_seconds=rounded(refresh_activity_age), - current_template_generation=state.current_template_generation, - published_template_generation=state.published_template_generation, - current_payout_generation=state.current_payout_generation, - published_payout_generation=state.published_payout_generation, - last_valid_delivery_age_seconds=rounded(delivery_age), - eligible_client_count=eligible_count, - eligible_clients_requiring_refresh=requiring_refresh, - bundle_build_oldest_age_seconds=rounded(oldest_bundle_age), - ) - - @staticmethod - def overlay( - base_health: Mapping[str, object], - snapshot: ProgressHealthSnapshot, - ) -> Mapping[str, object]: - return overlay_progress_health(base_health, snapshot.as_mapping()) - - @staticmethod - def metrics_lines( - snapshot: ProgressHealthSnapshot, - *, - coordination_blocked_age_seconds: float, - ) -> tuple[str, ...]: - pending_age = snapshot.pending_refresh_age_seconds - delivery_age = snapshot.last_valid_delivery_age_seconds - active_reasons = set(snapshot.reasons) - return ( - "# HELP qbit_prism_refresh_pending Whether current template or payout work still requires publication or delivery.", - "# TYPE qbit_prism_refresh_pending gauge", - f"qbit_prism_refresh_pending {1 if snapshot.pending_refresh else 0}", - "# HELP qbit_prism_refresh_pending_age_seconds Monotonic age of the oldest unresolved current-work refresh.", - "# TYPE qbit_prism_refresh_pending_age_seconds gauge", - f"qbit_prism_refresh_pending_age_seconds {float(pending_age or 0.0):.6f}", - "# HELP qbit_prism_tip_poll_age_seconds Monotonic age of the last coherent qbit tip/template poll.", - "# TYPE qbit_prism_tip_poll_age_seconds gauge", - f"qbit_prism_tip_poll_age_seconds {float(snapshot.tip_poll_age_seconds):.6f}", - "# HELP qbit_prism_current_generation_delivery_age_seconds Monotonic age of the last valid current-generation delivery, or -1 when none exists.", - "# TYPE qbit_prism_current_generation_delivery_age_seconds gauge", - f"qbit_prism_current_generation_delivery_age_seconds {float(delivery_age) if delivery_age is not None else -1.0:.6f}", - "# HELP qbit_prism_bundle_build_oldest_age_seconds Monotonic age of the oldest active bundle build.", - "# TYPE qbit_prism_bundle_build_oldest_age_seconds gauge", - f"qbit_prism_bundle_build_oldest_age_seconds {float(snapshot.bundle_build_oldest_age_seconds):.6f}", - "# HELP qbit_prism_template_refresh_coordination_blocked_age_seconds Monotonic age of the continuous coordination-blocked template-refresh streak, or 0 when clear.", - "# TYPE qbit_prism_template_refresh_coordination_blocked_age_seconds gauge", - f"qbit_prism_template_refresh_coordination_blocked_age_seconds {coordination_blocked_age_seconds:.6f}", - "# HELP qbit_prism_health_state Current progress-health state by bounded reason.", - "# TYPE qbit_prism_health_state gauge", - f'qbit_prism_health_state{{reason="healthy"}} {1 if snapshot.ok else 0}', - *( - f'qbit_prism_health_state{{reason="{reason}"}} {1 if reason in active_reasons else 0}' - for reason in PROGRESS_HEALTH_REASONS - ), - ) - - -def overlay_progress_health( - base_health: Mapping[str, object], - progress: Mapping[str, object], -) -> Mapping[str, object]: - """Overlay current progress without masking an independent base failure.""" - - base_ok = bool(base_health.get("mining_ready", base_health.get("ok"))) - result = dict(base_health) - result.update(progress) - result["ok"] = base_ok and bool(progress["ok"]) - if progress["ok"]: - result.pop("reason", None) - result.pop("reasons", None) - return MappingProxyType(result) - - -__all__ = [ - "BundleBuildToken", - "DeliveryProof", - "EligibilitySnapshot", - "PROGRESS_HEALTH_REASONS", - "ProgressHealthConfig", - "ProgressHealthService", - "ProgressHealthSnapshot", - "RefreshActivityToken", - "WorkGeneration", - "overlay_progress_health", -] diff --git a/lab/prism/public_api.py b/lab/prism/public_api.py deleted file mode 100644 index 5e263b290..000000000 --- a/lab/prism/public_api.py +++ /dev/null @@ -1,2262 +0,0 @@ -#!/usr/bin/env python3 -"""Public dashboard read-model API wrappers for PRISM.""" - -from __future__ import annotations - -import hashlib -import json -import math -import os -import threading -import time -import urllib.parse -from collections import OrderedDict, deque -from dataclasses import dataclass -from datetime import datetime, timedelta, timezone -from decimal import Decimal, InvalidOperation -from typing import Any, Callable, Iterator - -from lab.prism.ctv_broadcaster import COINBASE_MATURITY -from lab.prism import direct_stratum - -TERAHASH = Decimal(1_000_000_000_000) -QBIT_DIFFICULTY_SCALE = 1_000_000 -QBIT_POW_LIMIT_BITS = "207fffff" -QBIT_POW_LIMIT_TARGET = direct_stratum.target_from_compact_hex(QBIT_POW_LIMIT_BITS) -HASHES_PER_QBIT_SCALED_DIFFICULTY = ( - Decimal(2**256) / Decimal(QBIT_POW_LIMIT_TARGET) / Decimal(QBIT_DIFFICULTY_SCALE) -) -MAX_SEARCH_LENGTH = 128 -MAX_RECIPIENT_ID_LENGTH = 256 -BLOCKS_CHAIN_STATE_FILTERS = ("active", "all", "reversed") -# getnetworkhashps window and lane. The pool's shares are credited against the -# permissionless lane, so the ratio pool hashrate_ths / network hashrate_ths is -# only meaningful against that lane's estimate, not the all-lanes aggregate. -# 120 blocks and height -1 are the RPC's own documented defaults, passed -# explicitly because the lane argument is positional after them. -NETWORK_HASHRATE_BLOCK_WINDOW = 120 -NETWORK_HASHRATE_LANE = "permissionless" -HASHRATE_SERIES_BUCKET_SECONDS = {"5m": 300, "1h": 3600, "1d": 86400} -HASHRATE_SERIES_RANGE_SECONDS = { - "1w": 7 * 86400, - "1m": 30 * 86400, - "6m": 180 * 86400, - "all": None, -} -# The bucket vocabulary each range may request. Chart resolution is a product -# choice, but request cost is not: the ledger scan and the response both grow -# with range/bucket, and the widest combinations (a 5m bucket over months of -# history) exceed PRISM_PUBLIC_CACHE_MAX_RESPONSE_BYTES, so every such request -# bypasses the response cache and re-scans -- an unbounded-cost endpoint any -# stranger can hit. The static vocabulary closes that: coarse ranges admit only -# buckets whose point counts stay dashboard-sized, and auto_bucket() always -# resolves inside these sets, so `bucket=auto` is never rejected. Shared by -# every bucketed public series endpoint, not just the hashrate series. -HASHRATE_SERIES_ALLOWED_BUCKETS: dict[str, frozenset[str]] = { - "1w": frozenset({"5m", "1h", "1d"}), - "1m": frozenset({"1h", "1d"}), - "6m": frozenset({"1d"}), - "all": frozenset({"1d"}), -} -DEFAULT_HASHRATE_SMOOTHING_SECONDS = 30 * 60 -BLOCK_MARKERS_MAX_BLOCKS_PER_BUCKET = 3 - -PUBLIC_ERROR_CODES = { - "not_found": "not_found", - "rate_limited": "rate_limited", - "internal_error": "internal_error", - "qbit_rpc_unavailable": "upstream_unavailable", - # A cached response outlived its staleness budget and the public read - # service refused to serve it. Reported as upstream_unavailable, which is - # already in the documented error enum, because the condition is exactly - # that: the data behind this route is too old to answer with. - "stale_read_model": "upstream_unavailable", - # The public read tier's replica is missing, writable, or its replication - # stream has gone silent past the configured bound, so this route has no - # data it is willing to answer with. Same documented wire code as - # stale_read_model for the same reason: from a client's side the two are - # one condition. - "replica_unavailable": "upstream_unavailable", - # A public origin computation hit its per-request statement deadline - # (PRISM_PUBLIC_READ_STATEMENT_TIMEOUT_SECONDS) before the ledger - # answered. Its own wire code rather than upstream_unavailable: the - # database is reachable, this particular read was too expensive, and a - # retry a moment later can succeed off the cache -- which is not what - # upstream_unavailable tells a client. - "read_timeout": "read_timeout", -} - - -@dataclass(frozen=True) -class PublicCachePolicy: - ttl_seconds: int - stale_while_revalidate_seconds: int - immutable: bool = False - - -@dataclass(frozen=True) -class RawJsonBody: - """Pre-serialized JSON response body served byte-for-byte. - - Content-addressed artifact responses must satisfy - sha256(response bytes) == the advertised artifact sha256, so the exact - canonical bytes bypass write_json's generic sorted-key re-serialization - and its trailing newline. - """ - - body: bytes - - -@dataclass(frozen=True) -class UncacheableJsonBody: - """JSON payload whose degraded provenance must not enter shared caches.""" - - payload: object - reason: str - - -@dataclass -class _PublicCacheEntry: - status: int - payload: object - stored_at: float - expires_at: float - # When the entry stops being servable even as a stale-while-revalidate - # answer. Recorded at store time so capacity reaping can honor the window - # this entry was stored under without re-deriving the route's policy. - stale_until: float - - -@dataclass -class _PublicInflight: - event: threading.Event - result: tuple[int, object, str, int] | None = None - exception: BaseException | None = None - - -# Shared by every public response cache in this process. Admission is -# nonblocking: excess stale hits keep their bounded-age response without -# creating workers or queued refresh tasks behind the database read slots. -_PUBLIC_REFRESH_SLOTS = threading.BoundedSemaphore(4) - - -class PublicResponseCache: - """Small in-process cache for public read-model responses.""" - - def __init__(self) -> None: - self._lock = threading.Lock() - self._entries: OrderedDict[ - tuple[str, tuple[tuple[str, tuple[str, ...]], ...]], - _PublicCacheEntry, - ] = OrderedDict() - self._inflight: dict[ - tuple[str, tuple[tuple[str, tuple[str, ...]], ...]], - _PublicInflight, - ] = {} - - def get_or_compute( - self, - *, - key: tuple[str, tuple[tuple[str, tuple[str, ...]], ...]], - ttl_seconds: int, - compute: Callable[[], tuple[int, object]], - stale_while_revalidate_seconds: int = 0, - ) -> tuple[int, object, str, int]: - if ttl_seconds <= 0: - status, payload = compute() - return status, payload, "BYPASS", 0 - - now = time.monotonic() - stale_result: tuple[int, object, str, int] | None = None - refresh: _PublicInflight | None = None - with self._lock: - entry = self._entries.get(key) - if entry is not None: - if entry.expires_at > now: - self._entries.move_to_end(key) - return entry.status, entry.payload, "HIT", max(0, int(now - entry.stored_at)) - if ( - stale_while_revalidate_seconds > 0 - and now <= entry.expires_at + stale_while_revalidate_seconds - ): - # Inside the stale-while-revalidate window the expired - # entry is served immediately -- with its honest age -- - # instead of blocking this requester on a recompute that - # can take seconds. The entry stays in place: it remains - # the answer for the rest of the window, including when - # the refresh below fails. At most one background refresh - # runs per key, deduped through the same inflight map that - # coalesces blocking misses, so a stampede of stale hits - # costs one origin call. - self._entries.move_to_end(key) - stale_result = ( - entry.status, - entry.payload, - "STALE", - max(0, int(now - entry.stored_at)), - ) - if key not in self._inflight and _PUBLIC_REFRESH_SLOTS.acquire(blocking=False): - refresh = _PublicInflight(event=threading.Event()) - self._inflight[key] = refresh - else: - # Expired past any revalidation window: drop the stale - # slot so it cannot count toward the bound. - del self._entries[key] - if stale_result is None: - inflight = self._inflight.get(key) - if inflight is None: - inflight = _PublicInflight(event=threading.Event()) - self._inflight[key] = inflight - owner = True - else: - owner = False - - if stale_result is not None: - if refresh is not None: - try: - threading.Thread( - target=self._refresh_entry_admitted, - args=(key, ttl_seconds, stale_while_revalidate_seconds, compute, refresh), - name="prism-public-cache-refresh", - daemon=True, - ).start() - except Exception: - # Thread creation can fail under resource pressure, after - # the refresh slot was already registered. Free the slot - # and wake any coalesced waiter, or a burst of stale keys - # would pin refreshes off forever and, past the window, - # leave blocking requests waiting on an event nobody can - # set. The stale answer below is still the right response; - # the next stale hit simply retries the refresh. - with self._lock: - if self._inflight.get(key) is refresh: - del self._inflight[key] - refresh.event.set() - _PUBLIC_REFRESH_SLOTS.release() - return stale_result - - if not owner: - # Coalesce onto the owner's single origin call. Reuse its result even - # when it was not cacheable (BYPASS), and re-raise its exception, so - # waiters never re-run an expensive or failing compute(). The owner - # may be a background refresh whose stale entry has since aged past - # the revalidation window; waiting for it is the same bargain. - inflight.event.wait() - exception = inflight.exception - result = inflight.result - # Drop this frame's name for the flight before re-raising (#251). - # Raising prepends this frame to the shared exception's traceback, - # and a frame still naming the flight would close the cycle - # flight -> exception -> traceback -> frame -> flight, keeping the - # owner's failed compute frame and its locals alive until cyclic - # GC. The flight itself keeps the exception for later observers. - del inflight - if exception is not None: - try: - raise exception - finally: - # Unbind the local for the same reason: the traceback's - # frame must not name the exception it belongs to. - del exception - if result is not None: - return result - # Owner finished without recording a result; recompute defensively. - return self.get_or_compute(key=key, ttl_seconds=ttl_seconds, compute=compute) - - try: - status, payload = compute() - if 200 <= status < 300 and cacheable_payload_size(payload): - self._store_entry( - key, - status=status, - payload=payload, - ttl_seconds=ttl_seconds, - stale_while_revalidate_seconds=stale_while_revalidate_seconds, - ) - inflight.result = (status, payload, "MISS", 0) - else: - inflight.result = (status, payload, "BYPASS", 0) - return inflight.result - except BaseException as exc: - inflight.exception = exc - raise - finally: - with self._lock: - self._inflight.pop(key, None) - inflight.event.set() - # Release the owner frame's reference last, after every waiter has - # been woken (#251). On failure the exception's traceback already - # holds this frame, so a frame that kept naming the flight would - # form flight -> exception -> traceback -> frame -> flight and pin - # the failed compute frame, with the origin call's locals, until - # cyclic GC. Waiters still hold the flight through their own - # frames until each has observed the recorded result or error. - del inflight - - def _store_entry( - self, - key: tuple[str, tuple[tuple[str, tuple[str, ...]], ...]], - *, - status: int, - payload: object, - ttl_seconds: int, - stale_while_revalidate_seconds: int = 0, - ) -> None: - now = time.monotonic() - with self._lock: - self._entries[key] = _PublicCacheEntry( - status=status, - payload=payload, - stored_at=now, - expires_at=now + ttl_seconds, - stale_until=now + ttl_seconds + max(0, stale_while_revalidate_seconds), - ) - self._entries.move_to_end(key) - max_entries = public_cache_max_entries() - if len(self._entries) > max_entries: - # Reap dead entries before LRU eviction so dead slots never - # push out still-fresh keys. Dead means past the entry's own - # recorded revalidation window, not merely past its TTL: an - # expired entry inside that window is still the promised - # instant answer for its next visitor, and discarding it here - # would turn that visitor's stale hit into a blocking miss. - for stale_key in [ - stored_key - for stored_key, stored in self._entries.items() - if stored_key != key and stored.stale_until <= now - ]: - del self._entries[stale_key] - while len(self._entries) > max_entries: - self._entries.popitem(last=False) - - def _refresh_entry_admitted(self, *args) -> None: - try: - self._refresh_entry(*args) - finally: - # A failed compute retains caller frames through f_back too. Drop - # the flight-bearing arguments before returning the admission slot. - del args - _PUBLIC_REFRESH_SLOTS.release() - - def _refresh_entry( - self, - key: tuple[str, tuple[tuple[str, tuple[str, ...]], ...]], - ttl_seconds: int, - stale_while_revalidate_seconds: int, - compute: Callable[[], tuple[int, object]], - inflight: _PublicInflight, - ) -> None: - """Recompute one stale-served key off the request thread. - - The same compute through the same status/size gates as a blocking - miss. Success replaces the entry; a failure or a non-cacheable result - leaves the stale entry exactly where it was -- still servable until - its revalidation window ends -- and the finally clause frees the - inflight slot either way, so a later stale hit can retry. A blocking - request that arrives beyond the window while this runs coalesces onto - it through the inflight map, exactly as it would onto a foreground - owner; the recorded result and exception exist for those waiters. - """ - try: - status, payload = compute() - if 200 <= status < 300 and cacheable_payload_size(payload): - self._store_entry( - key, - status=status, - payload=payload, - ttl_seconds=ttl_seconds, - stale_while_revalidate_seconds=stale_while_revalidate_seconds, - ) - inflight.result = (status, payload, "MISS", 0) - else: - inflight.result = (status, payload, "BYPASS", 0) - except BaseException as exc: - inflight.exception = exc - finally: - with self._lock: - self._inflight.pop(key, None) - inflight.event.set() - # Same ownership rule as the foreground owner (#251): the swallowed - # exception's traceback captured this frame on arrival, so the - # frame must not keep naming the flight that stores it, or the - # completed refresh pins its failed compute locals until cyclic GC. - del inflight - - -def _is_hex64(value: str) -> bool: - return len(value) == 64 and all(char in "0123456789abcdefABCDEF" for char in value) - - -def _canonical_cache_path(path: str) -> str: - """Lowercase 64-hex hash path segments so casing variants share one cache entry. - - dispatch() resolves block, fanout, and artifact hashes through clean_hash(), - which lowercases hex. The cache key applies the same canonicalization so the - same resource requested with different hash casing shares one entry. Only - genuine 64-hex segments are folded, so non-hash routes (the literal - /fanouts/pending list, or invalid ids that dispatch rejects) keep distinct - keys and never collide with a canonicalized one. - """ - if path.startswith("/public/v1/blocks/") and path.endswith("/settlement-artifacts"): - inner = path[len("/public/v1/blocks/"):-len("/settlement-artifacts")].strip() - if _is_hex64(inner): - return f"/public/v1/blocks/{inner.lower()}/settlement-artifacts" - return path - if path.startswith("/public/v1/artifacts/"): - inner = path[len("/public/v1/artifacts/"):].strip() - if _is_hex64(inner): - return f"/public/v1/artifacts/{inner.lower()}" - return path - if path.startswith("/public/v1/fanouts/"): - inner = path[len("/public/v1/fanouts/"):].strip() - if _is_hex64(inner): - return f"/public/v1/fanouts/{inner.lower()}" - return path - return path - - -def public_cache_key(path: str, query: dict[str, list[str]]) -> tuple[str, tuple[tuple[str, tuple[str, ...]], ...]]: - return _canonical_cache_path(path), tuple(sorted((key, tuple(values)) for key, values in query.items())) - - -def _is_public_aggregate_path(path: str) -> bool: - """Pool-wide aggregate read models that are expensive to recompute. - - Pool summary and the hashrate series scan large share-ledger ranges, and - the worker list aggregates a miner's recent share history, so these - endpoints carry a longer default shared-cache TTL than the cheap row - reads. - """ - if path in {"/public/v1/pool-summary", "/public/v1/hashrate-series"}: - return True - if path.startswith("/public/v1/miners/"): - parts = path.removeprefix("/public/v1/miners/").split("/") - return len(parts) == 2 and parts[1] == "workers" - return False - - -def public_cache_policy(path: str) -> PublicCachePolicy: - if not env_bool("PRISM_PUBLIC_CACHE_ENABLED", default=True): - return PublicCachePolicy(ttl_seconds=0, stale_while_revalidate_seconds=0) - if path == "/public/v1/mining-configuration": - return PublicCachePolicy( - ttl_seconds=env_nonnegative_int("PRISM_PUBLIC_CONFIG_CACHE_TTL_SECONDS", 300), - stale_while_revalidate_seconds=env_nonnegative_int( - "PRISM_PUBLIC_CONFIG_CACHE_STALE_WHILE_REVALIDATE_SECONDS", - 3600, - ), - ) - if path.startswith("/public/v1/artifacts/"): - return PublicCachePolicy( - ttl_seconds=env_nonnegative_int("PRISM_PUBLIC_ARTIFACT_CACHE_TTL_SECONDS", 86400), - stale_while_revalidate_seconds=env_nonnegative_int( - "PRISM_PUBLIC_ARTIFACT_CACHE_STALE_WHILE_REVALIDATE_SECONDS", - 86400, - ), - immutable=True, - ) - if _is_public_aggregate_path(path): - return PublicCachePolicy( - ttl_seconds=env_nonnegative_int("PRISM_PUBLIC_AGGREGATE_CACHE_TTL_SECONDS", 30), - stale_while_revalidate_seconds=env_nonnegative_int( - "PRISM_PUBLIC_AGGREGATE_CACHE_STALE_WHILE_REVALIDATE_SECONDS", - 30, - ), - ) - return PublicCachePolicy( - ttl_seconds=env_nonnegative_int("PRISM_PUBLIC_CACHE_TTL_SECONDS", 5), - stale_while_revalidate_seconds=env_nonnegative_int( - "PRISM_PUBLIC_CACHE_STALE_WHILE_REVALIDATE_SECONDS", - 30, - ), - ) - - -def public_cache_headers(policy: PublicCachePolicy, *, cache_state: str, age_seconds: int) -> dict[str, str]: - headers = { - "Cache-Control": "public, max-age=0, must-revalidate", - "Age": str(max(0, age_seconds)), - } - if env_bool("PRISM_PUBLIC_CACHE_DEBUG_HEADERS", default=False): - headers["X-Prism-Public-Cache"] = cache_state.lower() - if policy.ttl_seconds > 0: - shared_policy = f"public, max-age={policy.ttl_seconds}" - if policy.stale_while_revalidate_seconds > 0: - shared_policy += f", stale-while-revalidate={policy.stale_while_revalidate_seconds}" - if policy.immutable: - shared_policy += ", immutable" - headers["CDN-Cache-Control"] = shared_policy - headers["Vercel-CDN-Cache-Control"] = shared_policy - return headers - - -def public_error_headers() -> dict[str, str]: - return {"Cache-Control": "no-store"} - - -def public_cache_max_entries() -> int: - return max(1, env_nonnegative_int("PRISM_PUBLIC_CACHE_MAX_ENTRIES", 512)) - - -def cacheable_payload_size(payload: object) -> bool: - max_bytes = env_nonnegative_int("PRISM_PUBLIC_CACHE_MAX_RESPONSE_BYTES", 1_048_576) - if max_bytes <= 0: - return False - if isinstance(payload, UncacheableJsonBody): - return False - if isinstance(payload, RawJsonBody): - return len(payload.body) <= max_bytes - body_size = len(json.dumps(payload, sort_keys=True).encode()) + 1 - return body_size <= max_bytes - - -def env_bool(name: str, *, default: bool) -> bool: - raw = os.environ.get(name) - if raw is None or raw == "": - return default - return raw.strip().lower() in {"1", "true", "yes", "on"} - - -def env_nonnegative_int(name: str, default: int) -> int: - raw = os.environ.get(name) - if raw is None or raw == "": - return default - try: - parsed = int(raw) - except ValueError: - return default - return max(0, parsed) - - -class PublicApiError(Exception): - def __init__(self, status: int, code: str, message: str): - super().__init__(message) - self.status = status - self.code = code - self.message = message - - -def dispatch(coordinator: Any, path: str, query: dict[str, list[str]]) -> tuple[int, object]: - if path == "/public/v1/pool-summary": - return 200, pool_summary(coordinator) - if path == "/public/v1/blocks": - page, limit = pagination_params(query) - chain_state = first_query_value(query, "chain_state") or "active" - if chain_state not in BLOCKS_CHAIN_STATE_FILTERS: - raise PublicApiError( - 400, - "invalid_chain_state", - "chain_state must be one of active, all, reversed", - ) - return 200, blocks(coordinator, page=page, limit=limit, chain_state=chain_state) - if path == "/public/v1/leaderboard": - page, limit = pagination_params(query) - search = search_param(query) - window = first_query_value(query, "window") or "3h" - raw_recipient_id = first_query_value(query, "recipient_id") - if window == "3h": - if raw_recipient_id is not None: - raise PublicApiError( - 400, - "invalid_reward_filter", - "recipient_id requires window=reward", - ) - return 200, leaderboard(coordinator, page=page, limit=limit, search=search) - if window == "reward": - recipient_id = clean_recipient_id(raw_recipient_id) if raw_recipient_id is not None else None - if search is not None and recipient_id is not None: - raise PublicApiError( - 400, - "invalid_reward_filter", - "search and recipient_id are mutually exclusive", - ) - return 200, reward_leaderboard( - coordinator, - page=page, - limit=limit, - search=search, - recipient_id=recipient_id, - ) - raise PublicApiError(400, "invalid_window", "window must be one of 3h, reward") - if path == "/public/v1/hashrate-series": - subject = first_query_value(query, "subject") or "pool" - range_id = first_query_value(query, "range") or "1m" - bucket = first_query_value(query, "bucket") or "auto" - view = first_query_value(query, "view") - return 200, hashrate_series(coordinator, subject=subject, range_id=range_id, bucket=bucket, view=view) - if path == "/public/v1/block-markers": - range_id = first_query_value(query, "range") or "1m" - bucket = first_query_value(query, "bucket") or "auto" - return 200, block_markers(coordinator, range_id=range_id, bucket=bucket) - if path == "/public/v1/mining-configuration": - return 200, mining_configuration(coordinator) - if path.startswith("/public/v1/miners/"): - suffix = path.removeprefix("/public/v1/miners/") - parts = suffix.split("/") - if len(parts) == 2 and parts[1] == "earnings": - recipient_id = urllib.parse.unquote(parts[0]) - page, limit = pagination_params(query) - return 200, miner_earnings(coordinator, recipient_id=recipient_id, page=page, limit=limit) - if len(parts) == 2 and parts[1] == "payouts": - recipient_id = urllib.parse.unquote(parts[0]) - page, limit = pagination_params(query) - return 200, miner_payouts(coordinator, recipient_id=recipient_id, page=page, limit=limit) - if len(parts) == 2 and parts[1] == "workers": - recipient_id = urllib.parse.unquote(parts[0]) - page, limit = pagination_params(query) - search = search_param(query) - hide_inactive = (first_query_value(query, "hide_inactive") or "true").lower() in {"1", "true", "yes", "on"} - return 200, miner_workers( - coordinator, - recipient_id=recipient_id, - page=page, - limit=limit, - search=search, - hide_inactive=hide_inactive, - ) - if len(parts) == 1: - return 200, miner(coordinator, recipient_id=urllib.parse.unquote(suffix)) - raise PublicApiError(404, "not_found", "unknown public dashboard endpoint") - if path.startswith("/public/v1/blocks/") and path.endswith("/settlement-artifacts"): - block_hash = clean_hash(path.removeprefix("/public/v1/blocks/").removesuffix("/settlement-artifacts")) - return 200, settlement_artifacts(coordinator, block_hash=block_hash) - if path == "/public/v1/fanouts/pending": - page, limit = pagination_params(query) - return 200, pending_fanouts(coordinator, page=page, limit=limit) - if path.startswith("/public/v1/fanouts/"): - fanout_txid = clean_hash(path.removeprefix("/public/v1/fanouts/"), name="fanout txid") - return 200, fanout(coordinator, fanout_txid=fanout_txid) - if path.startswith("/public/v1/artifacts/"): - sha256 = clean_hash(path.removeprefix("/public/v1/artifacts/"), name="artifact sha256") - return 200, artifact(coordinator, sha256=sha256) - raise PublicApiError(404, "not_found", "unknown public dashboard endpoint") - - -def error_payload(code: str, message: str) -> dict[str, object]: - error_code = PUBLIC_ERROR_CODES.get(code) - if error_code is None: - error_code = "bad_request" if code.startswith("invalid_") else "internal_error" - return { - "schema": "prism.dashboard.error.v1", - "error": { - "code": error_code, - "message": message, - "request_id": None, - }, - } - - -def pool_summary(coordinator: Any) -> dict[str, object]: - generated_at = utc_now_iso() - network = network_summary(coordinator) - # Looked up after network_summary so a missing-compact-bits 503 still - # short-circuits first; a failed hashrate lookup alone must never fail - # this endpoint (the field is nullable by contract). - network["hashrate_ths"] = network_hashrate_ths(coordinator) - ledger_snapshot = coordinator.ledger.dashboard_pool_snapshot( - current_network_difficulty=network["network_difficulty"], - generated_at=generated_at, - ) - return { - "schema": "prism.dashboard.pool-summary.v1", - "generated_at": generated_at, - "network": network, - "pool": { - "name": os.environ.get("PRISM_PUBLIC_POOL_NAME", "PRISM"), - "hashrate_ths": ledger_snapshot["hashrate_ths"], - "participants_3h": ledger_snapshot["participants_3h"], - "blocks_found_total": ledger_snapshot["blocks_found_total"], - "prism_blocks_total": ledger_snapshot["prism_blocks_total"], - "blocks_reversed_total": ledger_snapshot["blocks_reversed_total"], - "blocks_inactive_total": ledger_snapshot["blocks_inactive_total"], - "total_mined_bits": ledger_snapshot["total_mined_bits"], - "expected_time_to_block_seconds": expected_time_to_block_seconds( - hashrate_ths=str(ledger_snapshot["hashrate_ths"]["h3"]), - network_difficulty=str(network["network_difficulty"]), - ), - "latest_block": ledger_snapshot["latest_block"], - "reward_window": ledger_snapshot["reward_window"], - }, - } - - -def blocks(coordinator: Any, *, page: int, limit: int, chain_state: str = "active") -> dict[str, object]: - generated_at = utc_now_iso() - if chain_state == "active": - # The default keeps today's read-model call and today's schema tag - # exactly: no chain_state argument reaches the ledger, so the default - # response stays byte-compatible with pre-filter consumers. - payload = coordinator.ledger.dashboard_blocks(page=page, limit=limit) - schema = "prism.dashboard.blocks.v1" - else: - payload = coordinator.ledger.dashboard_blocks(page=page, limit=limit, chain_state=chain_state) - schema = "prism.dashboard.blocks.v2" - for row in payload["rows"]: - row.setdefault("bits", "00000000") - if row.get("bits") is None: - row["bits"] = "00000000" - row.setdefault("network_difficulty", "0") - if row.get("network_difficulty") is None: - row["network_difficulty"] = "0" - return { - "schema": schema, - "generated_at": generated_at, - "pagination": payload["pagination"], - "rows": payload["rows"], - } - - -def leaderboard(coordinator: Any, *, page: int, limit: int, search: str | None) -> dict[str, object]: - generated_at = utc_now_iso() - payload = coordinator.ledger.dashboard_leaderboard(page=page, limit=limit, search=search) - return { - "schema": "prism.dashboard.leaderboard.v1", - "generated_at": generated_at, - "window": { - "id": "3h", - "started_at": payload["started_at"], - "ended_at": payload["ended_at"], - }, - "totals": payload["totals"], - "pagination": payload["pagination"], - "rows": payload["rows"], - } - - -def reward_leaderboard( - coordinator: Any, - *, - page: int, - limit: int, - search: str | None, - recipient_id: str | None, -) -> dict[str, object]: - network = network_summary(coordinator) - payload = coordinator.ledger.dashboard_reward_leaderboard( - page=page, - limit=limit, - search=search, - recipient_id=recipient_id, - current_network_difficulty=network["network_difficulty"], - ) - if Decimal(str(payload["window"]["counted_window_weight"])) != Decimal( - str(payload["totals"]["pool_counted_share_difficulty"]) - ): - raise RuntimeError("reward leaderboard counted window weight does not match pool total") - return { - "schema": "prism.dashboard.leaderboard.v2", - "generated_at": payload["window"]["ended_at"], - "window": payload["window"], - "totals": payload["totals"], - "pagination": payload["pagination"], - "rows": payload["rows"], - } - - -def hashrate_series( - coordinator: Any, - *, - subject: str, - range_id: str, - bucket: str, - view: str | None = None, -) -> dict[str, object]: - if range_id not in {"1w", "1m", "6m", "all"}: - raise PublicApiError(400, "invalid_range", "range must be one of 1w, 1m, 6m, all") - if bucket == "auto": - bucket = auto_bucket(range_id) - if bucket not in {"5m", "1h", "1d"}: - raise PublicApiError(400, "invalid_bucket", "bucket must be one of auto, 5m, 1h, 1d") - allowed_hashrate_bucket(range_id, bucket) - if subject == "pool": - subject_type = "pool" - subject_id = None - elif subject.startswith("miner:") and subject.removeprefix("miner:"): - subject_type = "miner" - subject_id = subject.removeprefix("miner:") - else: - raise PublicApiError(400, "invalid_subject", "subject must be pool or miner:{recipient_id}") - # An absent view keeps the v1 response; only `both` opts into the dual-rate - # v2 contract. Anything else is rejected rather than silently - # served as v1, so a client that mistypes the view cannot mistake the - # legacy smoothed rate for the raw one it asked for. - if view is not None and view != "both": - raise PublicApiError(400, "invalid_view", "view must be both or omitted") - dual_rate = view == "both" - now = datetime.now(timezone.utc) - generated_at = iso_datetime(now) - generated_at_epoch = int(now.timestamp()) - bucket_seconds = HASHRATE_SERIES_BUCKET_SECONDS[bucket] - window_seconds = public_hashrate_smoothing_seconds() - range_seconds = HASHRATE_SERIES_RANGE_SECONDS[range_id] - lookback_seconds = 0 - min_epoch: int | None = None - range_anchor_epoch: int | None = None - if range_seconds is not None and ( - dual_rate or window_seconds // bucket_seconds >= 2 - ): - # V2 raw rates always divide by a full bucket, so trim a leading bucket - # that straddles the range boundary even when this coarse bucket has no - # trailing smoothing. V1 keeps its frozen no-smoothing behavior. - range_anchor_epoch = generated_at_epoch - min_epoch = hashrate_series_min_epoch( - range_anchor_epoch, - range_seconds, - bucket_seconds, - ) - if window_seconds // bucket_seconds >= 2 and range_seconds is not None: - # Fetch one full smoothing window of pre-range history so the first - # in-range points average over real data instead of artificial zeros, - # then trim the context-only points after smoothing. The ledger anchors - # its range lower bound on the same epoch min_epoch derives from, so - # the trim cannot disagree with the fetch under clock skew. - lookback_seconds = (window_seconds // bucket_seconds) * bucket_seconds - range_anchor_epoch = generated_at_epoch - min_epoch = hashrate_series_min_epoch( - range_anchor_epoch, - range_seconds, - bucket_seconds, - ) - points = coordinator.ledger.dashboard_hashrate_series( - subject_type=subject_type, - subject_id=subject_id, - range_id=range_id, - bucket=bucket, - lookback_seconds=lookback_seconds, - range_anchor_epoch=range_anchor_epoch, - ) - subject_payload = {"type": subject_type, "id": subject_id} - if dual_rate: - smoothing = hashrate_series_smoothing(bucket_seconds=bucket_seconds, window_seconds=window_seconds) - return { - "schema": "prism.dashboard.hashrate-series.v2", - "generated_at": generated_at, - "subject": subject_payload, - "range": range_id, - "bucket": bucket, - "bucket_seconds": bucket_seconds, - "unit": "ths", - "rate_basis": "accepted_share_difficulty", - "smoothing": smoothing, - "points": dual_rate_hashrate_series_points( - points, - bucket_seconds=bucket_seconds, - window_seconds=window_seconds, - min_epoch=min_epoch, - generated_at_epoch=generated_at_epoch, - ), - } - points = smooth_hashrate_series_points( - points, - bucket_seconds=bucket_seconds, - window_seconds=window_seconds, - min_epoch=min_epoch, - ) - return { - "schema": "prism.dashboard.hashrate-series.v1", - "generated_at": generated_at, - "subject": subject_payload, - "range": range_id, - "bucket": bucket, - "unit": "ths", - "points": points, - } - - -def block_markers(coordinator: Any, *, range_id: str, bucket: str) -> dict[str, object]: - """Found-block markers pre-bucketed onto the hashrate chart's grid. - - The dashboard draws markers over the hashrate series, so every bucket - epoch here is floor(epoch(found_at) / bucket_seconds) * bucket_seconds -- - the identical arithmetic the hashrate buckets use -- and the bounded - ranges keep only fully covered buckets via the same - hashrate_series_min_epoch anchor the chart trims against. Paging - /public/v1/blocks cannot serve this: at hundreds of found blocks a day the - long timeframes would need hundreds of pages per chart render. - """ - if range_id not in {"1w", "1m", "6m", "all"}: - raise PublicApiError(400, "invalid_range", "range must be one of 1w, 1m, 6m, all") - if bucket == "auto": - bucket = auto_bucket(range_id) - if bucket not in {"5m", "1h", "1d"}: - raise PublicApiError(400, "invalid_bucket", "bucket must be one of auto, 5m, 1h, 1d") - # Markers land on top of the hashrate chart, so the hashrate series' - # static bucket-per-range vocabulary is the markers' vocabulary too. - allowed_hashrate_bucket(range_id, bucket) - now = datetime.now(timezone.utc) - generated_at = iso_datetime(now) - bucket_seconds = HASHRATE_SERIES_BUCKET_SECONDS[bucket] - range_seconds = HASHRATE_SERIES_RANGE_SECONDS[range_id] - # Anchor the range lower bound on this process's clock, exactly as - # hashrate_series does, so marker buckets and hashrate buckets share one - # grid even when the database clock disagrees. `all` has no lower bound. - range_anchor_epoch = int(now.timestamp()) if range_seconds is not None else None - payload = coordinator.ledger.dashboard_block_markers( - range_id=range_id, - bucket=bucket, - range_anchor_epoch=range_anchor_epoch, - ) - points: list[dict[str, object]] = [] - for row in payload["points"]: - block_count = int(row["block_count"]) - points.append( - { - "timestamp": row["timestamp"], - "block_count": block_count, - "blocks": [ - { - "height": int(block["height"]), - "hash": str(block["hash"]), - "found_at": str(block["found_at"]), - } - for block in row["blocks"] - ], - "truncated": block_count > BLOCK_MARKERS_MAX_BLOCKS_PER_BUCKET, - } - ) - return { - "schema": "prism.dashboard.block-markers.v1", - "generated_at": generated_at, - "range": range_id, - "bucket": bucket, - "bucket_seconds": bucket_seconds, - "total_blocks": int(payload["total_blocks"]), - "points": points, - } - - -def mining_configuration(coordinator: Any) -> dict[str, object]: - port = int(getattr(coordinator, "port", os.environ.get("PRISM_STRATUM_PORT", "3340"))) - host = os.environ.get("PRISM_PUBLIC_STRATUM_HOST") or str(getattr(coordinator, "bind", "127.0.0.1")) - url = os.environ.get("PRISM_PUBLIC_STRATUM_URL") or f"stratum+tcp://{host}:{port}" - stratum_endpoints = [mining_endpoint("Primary", url, fallback_port=port)] - highdiff_endpoint = public_highdiff_stratum_endpoint(host=host) - if highdiff_endpoint is not None: - stratum_endpoints.append(highdiff_endpoint) - # Read the fee through the same clamped/robust helper the reward estimate uses so - # the displayed fee and estimated_reward_bits never diverge (and a bad env value - # can't 500 this endpoint). - pool_fee_bps = public_pool_fee_bps() - return { - "schema": "prism.dashboard.mining-configuration.v1", - "generated_at": utc_now_iso(), - "active_configuration_id": "default", - "configurations": [ - { - "id": "default", - "label": os.environ.get("PRISM_PUBLIC_CONFIGURATION_LABEL", "PRISM default"), - "description": os.environ.get( - "PRISM_PUBLIC_CONFIGURATION_DESCRIPTION", - "Default PRISM Stratum endpoint using the pool's current block template and payout policy.", - ), - "pool_fee_bps": pool_fee_bps, - "block_template_policy": os.environ.get( - "PRISM_PUBLIC_BLOCK_TEMPLATE_POLICY", - "pool-selected qbit block template with PRISM payout settlement", - ), - "stratum_endpoints": stratum_endpoints, - } - ], - } - - -def mining_endpoint(label: str, url: str, *, fallback_port: int) -> dict[str, object]: - return { - "label": label, - "url": url, - "protocol": "stratum_v1", - "default_port": stratum_url_default_port(url, fallback_port=fallback_port), - } - - -def public_highdiff_stratum_endpoint(*, host: str) -> dict[str, object] | None: - highdiff_port = optional_tcp_port(os.environ.get("PRISM_STRATUM_HIGHDIFF_PORT")) - if highdiff_port is None: - return None - url = ( - os.environ.get("PRISM_PUBLIC_STRATUM_HIGHDIFF_URL") - or public_stratum_url_with_port(os.environ.get("PRISM_PUBLIC_STRATUM_URL"), highdiff_port) - or f"stratum+tcp://{host}:{highdiff_port}" - ) - return mining_endpoint("High-diff", url, fallback_port=highdiff_port) - - -def public_stratum_url_with_port(url: str | None, port: int) -> str | None: - if url is None or url.strip() == "": - return None - try: - parsed = urllib.parse.urlparse(url) - except ValueError: - return None - if not parsed.scheme or parsed.hostname is None: - return None - host = parsed.hostname - if ":" in host and not host.startswith("["): - host = f"[{host}]" - return f"{parsed.scheme}://{host}:{port}" - - -def optional_tcp_port(value: str | None) -> int | None: - if value is None or value.strip() == "": - return None - try: - port = int(value) - except ValueError: - return None - return port if 0 < port < 65536 else None - - -def stratum_url_default_port(url: str, *, fallback_port: int) -> int: - try: - port = urllib.parse.urlparse(url).port - except ValueError: - return fallback_port - return port if port is not None else fallback_port - - -def public_minimum_payout_bits() -> int: - value = first_optional_nonnegative_int( - os.environ.get("PRISM_PUBLIC_MINIMUM_PAYOUT_BITS"), - os.environ.get("PRISM_PAYOUT_MIN_OUTPUT_BITS"), - os.environ.get("PRISM_PAYOUT_MIN_OUTPUT_SATS"), - ) - return value if value is not None else 0 - - -def public_pool_fee_bps() -> int: - """Pool fee (basis points) surfaced to the dashboard, clamped to [0, 10000].""" - try: - bps = int(os.environ.get("PRISM_PUBLIC_POOL_FEE_BPS") or "0") - except (TypeError, ValueError): - return 0 - return max(0, min(10_000, bps)) - - -def public_hashrate_smoothing_seconds() -> int: - """Trailing window used to steady hashrate-series points, clamped to [0, 86400]. - - 0 disables smoothing and returns raw per-bucket rates. - """ - raw = os.environ.get("PRISM_PUBLIC_HASHRATE_SMOOTHING_SECONDS") - if raw is None or raw.strip() == "": - return DEFAULT_HASHRATE_SMOOTHING_SECONDS - try: - seconds = int(raw) - except (TypeError, ValueError): - return DEFAULT_HASHRATE_SMOOTHING_SECONDS - return max(0, min(86_400, seconds)) - - -def latest_block_coinbase_value_bits(coordinator: Any) -> int | None: - """Best-available estimate of the coinbase reward for the next block. - - Uses the most recently found block's coinbase value as the reference for an - upcoming block. Returns None when no block is available yet (fresh pool) so - callers can omit the estimate rather than showing a fabricated amount. - """ - try: - payload = coordinator.ledger.dashboard_blocks(page=1, limit=1) - except Exception: - return None - rows = payload.get("rows") if isinstance(payload, dict) else None - if not rows: - return None - try: - value = int(rows[0].get("coinbase_value_bits")) - except (TypeError, ValueError): - return None - return value if value > 0 else None - - -def estimated_next_block_reward_bits( - *, share_percent: str | None, expected_coinbase_bits: object, pool_fee_bps: int -) -> int | None: - """Miner's expected reward if the pool solves the next block. - - Their reward-window share (share_percent) applied to the miner-distributable - portion of the expected coinbase (coinbase minus the pool fee). Returns None - when the share or a coinbase reference is unavailable, so the dashboard can - fall back to showing only the share percentage. - """ - if share_percent is None or expected_coinbase_bits is None: - return None - try: - coinbase = Decimal(str(expected_coinbase_bits)) - percent = Decimal(str(share_percent)) - except (ArithmeticError, ValueError): - return None - if not coinbase.is_finite() or not percent.is_finite() or coinbase <= 0 or percent < 0: - return None - fee_bps = max(0, min(10_000, pool_fee_bps)) - distributable = coinbase * Decimal(10_000 - fee_bps) / Decimal(10_000) - reward = distributable * percent / Decimal(100) - return max(0, int(reward)) - - -def miner(coordinator: Any, *, recipient_id: str) -> dict[str, object]: - recipient_id = clean_recipient_id(recipient_id) - network = network_summary(coordinator) - share_summary = miner_share_summary(coordinator.ledger, recipient_id) - reward_window = miner_reward_window( - coordinator.ledger, - recipient_id, - current_network_difficulty=network["network_difficulty"], - ) - owed_balance_bits = owed_balance_for_recipient(coordinator.ledger, recipient_id) - payouts = payout_rows(coordinator, recipient_id=recipient_id, page=1, limit=5) - recent_payouts = payouts["rows"] - workers_payload = miner_worker_rows( - coordinator.ledger, - recipient_id=recipient_id, - page=1, - limit=5, - search=None, - hide_inactive=False, - ) - workers = workers_payload["rows"] - reward_window_percent = reward_window["share_percent"] - minimum_payout_bits = public_minimum_payout_bits() - lifetime_earnings_bits = lifetime_earnings_for_recipient(coordinator.ledger, recipient_id) - pending_maturity_bits = pending_maturity_bits_for_recipient(coordinator.ledger, recipient_id) - estimated_reward_bits = estimated_next_block_reward_bits( - share_percent=reward_window_percent, - expected_coinbase_bits=latest_block_coinbase_value_bits(coordinator), - pool_fee_bps=public_pool_fee_bps(), - ) - return { - "schema": "prism.dashboard.miner.v1", - "generated_at": utc_now_iso(), - "recipient_id": recipient_id, - "display_name": None, - "owed_balance_bits": owed_balance_bits, - "lifetime_earnings_bits": lifetime_earnings_bits, - "pending_maturity_bits": pending_maturity_bits, - "unpaid_earnings_bits": owed_balance_bits, - "minimum_payout_bits": minimum_payout_bits, - "hashrate_ths": share_summary["hashrate_ths"], - "shares": { - "accepted_3h": share_summary["accepted_3h"], - "accepted_difficulty_3h": share_summary["accepted_difficulty_3h"], - "last_share_at": share_summary["last_share_at"], - }, - "estimated_next_block": { - "share_percent": reward_window_percent, - "estimated_reward_bits": estimated_reward_bits, - }, - "estimated_time_to_minimum_payout_seconds": 0 if owed_balance_bits >= minimum_payout_bits else None, - "reward_window_percent": reward_window_percent, - "workers_currently_hashing": int(workers_payload.get("active_count", sum(1 for row in workers if row["status"] == "active"))), - "workers": workers[:5], - "recent_payouts": recent_payouts, - } - - -def miner_earnings(coordinator: Any, *, recipient_id: str, page: int, limit: int) -> dict[str, object]: - recipient_id = clean_recipient_id(recipient_id) - return { - "schema": "prism.dashboard.miner-earnings.v1", - "generated_at": utc_now_iso(), - "recipient_id": recipient_id, - **earnings_rows(coordinator, recipient_id=recipient_id, page=page, limit=limit), - } - - -def miner_payouts(coordinator: Any, *, recipient_id: str, page: int, limit: int) -> dict[str, object]: - recipient_id = clean_recipient_id(recipient_id) - return { - "schema": "prism.dashboard.miner-payouts.v1", - "generated_at": utc_now_iso(), - "recipient_id": recipient_id, - **payout_rows(coordinator, recipient_id=recipient_id, page=page, limit=limit), - } - - -def miner_workers( - coordinator: Any, - *, - recipient_id: str, - page: int, - limit: int, - search: str | None, - hide_inactive: bool, -) -> dict[str, object]: - recipient_id = clean_recipient_id(recipient_id) - payload = miner_worker_rows( - coordinator.ledger, - recipient_id=recipient_id, - page=page, - limit=limit, - search=search, - hide_inactive=hide_inactive, - ) - return { - "schema": "prism.dashboard.miner-workers.v1", - "generated_at": utc_now_iso(), - "recipient_id": recipient_id, - "pagination": payload["pagination"], - "rows": payload["rows"], - } - - -def settlement_artifacts(coordinator: Any, *, block_hash: str) -> dict[str, object]: - generated_at = utc_now_iso() - payload = coordinator.ledger.audit_ctv_fanout_manifest_set(block_hash=block_hash) - if payload is None: - payload = direct_coinbase_settlement_payload(coordinator.ledger, block_hash=block_hash) - if payload is None: - raise PublicApiError(404, "not_found", "unknown PRISM settlement artifact block") - artifacts = payload.get("artifacts", []) - if not isinstance(artifacts, list): - artifacts = [] - artifact_links: list[dict[str, object]] = [] - audit_bundle_sha256 = nullable_str(payload.get("audit_bundle_sha256")) - if audit_bundle_sha256 and public_artifact_available(coordinator.ledger, sha256=audit_bundle_sha256): - artifact_links.append(artifact_link("audit_bundle", audit_bundle_sha256, None)) - manifest_set_sha256 = nullable_str(payload.get("manifest_set_sha256")) - if manifest_set_sha256: - manifest_set_json = nullable_str(payload.get("manifest_set_json")) - artifact_links.append( - artifact_link( - "ctv_manifest_set", - manifest_set_sha256, - len(manifest_set_json.encode()) if manifest_set_json is not None else None, - ) - ) - fanout_rows = settlement_fanout_rows(coordinator.ledger, payload, artifacts) - return { - "schema": "prism.dashboard.settlement-artifacts.v1", - "generated_at": generated_at, - "block_hash": block_hash, - "block_height": first_int(payload.get("block_height"), *(row.get("block_height") for row in fanout_rows), default=0), - "settlement_mode": str(payload.get("settlement_mode") or "hybrid_coinbase_ctv_fanout"), - "audit_bundle_sha256": audit_bundle_sha256, - "payout_manifest_sha256": nullable_str(payload.get("payout_manifest_sha256")), - "artifact_links": artifact_links, - "fanouts": [fanout_public_row(row) for row in fanout_rows], - } - - -def direct_coinbase_settlement_payload(ledger: Any, *, block_hash: str) -> dict[str, object] | None: - read_model = getattr(ledger, "dashboard_direct_coinbase_settlement", None) - if callable(read_model): - try: - payload = read_model(block_hash=block_hash) - except RuntimeError as exc: - if audit_bundle_body_read_failed(exc): - return None - raise - return payload if isinstance(payload, dict) else None - # In-memory ledger only: audit_bundle() is a writer-lock read on the - # Postgres ledger, so a public settlement lookup must not reach it. - getter = getattr(ledger, "audit_bundle", None) - if not callable(getter): - return None - try: - row = getter(block_hash=block_hash) - except RuntimeError as exc: - if audit_bundle_body_read_failed(exc): - return None - raise - if not isinstance(row, dict): - return None - bundle = row.get("audit_bundle") - if not isinstance(bundle, dict): - return None - if audit_bundle_settlement_mode(bundle) != "direct_coinbase": - return None - return { - "block_hash": optional_hex_hash(row.get("block_hash")) or block_hash, - "block_height": first_int( - row.get("block_height"), - audit_bundle_section_value(bundle, "found_block", "block_height"), - audit_bundle_section_value(bundle, "reward_manifest", "block_height"), - audit_bundle_section_value(bundle, "ledger_window_attestation", "block_height"), - default=0, - ), - "settlement_mode": "direct_coinbase", - "audit_bundle_sha256": nullable_str(row.get("audit_bundle_sha256")), - "payout_manifest_sha256": nullable_str(row.get("payout_manifest_sha256")), - "artifacts": [], - } - - -def audit_bundle_body_read_failed(exc: RuntimeError) -> bool: - message = str(exc) - return ( - message.startswith("audit bundle body is not retrievable") - or message.startswith("audit bundle body hash mismatch") - or message.startswith("audit bundle body is not valid JSON") - ) - - -def audit_bundle_settlement_mode(bundle: dict[str, object]) -> str | None: - decision = bundle.get("settlement_mode_decision") - if not isinstance(decision, dict): - return None - return nullable_str(decision.get("mode")) - - -def audit_bundle_section_value(bundle: dict[str, object], section_name: str, key: str) -> object: - section = bundle.get(section_name) - if not isinstance(section, dict): - return None - return section.get(key) - - -def pending_fanouts(coordinator: Any, *, page: int, limit: int) -> dict[str, object]: - generated_at = utc_now_iso() - read_model = getattr(coordinator.ledger, "dashboard_pending_fanout_rows", None) - if callable(read_model): - payload = read_model(page=page, limit=limit) - rows = payload["rows"] - pagination_payload = payload["pagination"] - else: - raise PublicApiError(500, "internal_error", "internal server error") - return { - "schema": "prism.dashboard.pending-fanouts.v1", - "generated_at": generated_at, - "pagination": pagination_payload, - "rows": [fanout_public_row(row) for row in rows], - } - - -def fanout(coordinator: Any, *, fanout_txid: str) -> dict[str, object]: - payload = coordinator.ledger.ctv_fanout_status(fanout_txid=fanout_txid) - if payload is None: - raise PublicApiError(404, "not_found", "unknown PRISM CTV fanout") - return { - "schema": "prism.dashboard.fanout.v1", - "generated_at": utc_now_iso(), - "fanout": fanout_public_row(payload), - } - - -def artifact(coordinator: Any, *, sha256: str) -> object: - document = getattr(coordinator.ledger, "dashboard_public_artifact_document", None) - if callable(document): - row = document(sha256=sha256) - if not isinstance(row, dict): - raise PublicApiError(404, "not_found", "unknown public PRISM artifact") - canonical_json = row.get("canonical_json") - if isinstance(canonical_json, str): - body = canonical_json.encode() - if hashlib.sha256(body).hexdigest() == sha256: - return RawJsonBody(body=body) - # Stored canonical text that does not hash to its content address - # is a store-integrity fault; fall through to the re-serialized - # response rather than serve bytes that contradict the address. - payload = row.get("payload") - if payload is None: - raise PublicApiError(404, "not_found", "unknown public PRISM artifact") - fallback_reason = row.get("canonical_fallback_reason") - if isinstance(fallback_reason, str) and fallback_reason: - return UncacheableJsonBody(payload=payload, reason=fallback_reason) - return payload - getter = getattr(coordinator.ledger, "dashboard_public_artifact", None) - if not callable(getter): - raise PublicApiError(404, "not_found", "unknown public PRISM artifact") - payload = getter(sha256=sha256) - if payload is None: - raise PublicApiError(404, "not_found", "unknown public PRISM artifact") - return payload - - -def public_artifact_available(ledger: Any, *, sha256: str) -> bool: - exists = getattr(ledger, "dashboard_public_artifact_exists", None) - if callable(exists): - return bool(exists(sha256=sha256)) - getter = getattr(ledger, "dashboard_public_artifact", None) - return bool(callable(getter) and getter(sha256=sha256) is not None) - - -def settlement_fanout_rows(ledger: Any, payload: dict[str, object], artifacts: list[object]) -> list[dict[str, object]]: - parent_fields = _fanout_parent_fields(payload) - rows: list[dict[str, object]] = [] - for artifact in artifacts: - if not isinstance(artifact, dict): - continue - row = {**artifact, **parent_fields} - fanout_txid = optional_hex_hash(row.get("fanout_txid")) - if fanout_txid: - live_status = ledger.ctv_fanout_status(fanout_txid=fanout_txid) - if isinstance(live_status, dict): - row = {**row, **live_status} - rows.append(row) - return rows - - -def fanout_public_row(payload: dict[str, object]) -> dict[str, object]: - fanout_tx_hex = str(payload.get("fanout_tx_hex") or "") - attempts = payload.get("broadcast_attempts") - last_attempt = None - if isinstance(attempts, list) and attempts: - last_attempt = attempts[-1] - last_broadcast_attempt_at = payload.get("last_broadcast_attempt_at") - last_broadcast_error = payload.get("last_broadcast_error") - if last_broadcast_attempt_at is None and isinstance(last_attempt, dict): - last_broadcast_attempt_at = last_attempt.get("attempted_at") - if last_broadcast_error is None and isinstance(last_attempt, dict): - last_broadcast_error = last_attempt.get("error") - status = str(payload.get("settlement_status") or payload.get("status") or "awaiting_maturity") - manifest_sha256 = str(payload.get("manifest_sha256") or "") - explorer_prefix = os.environ.get("PRISM_PUBLIC_EXPLORER_TX_URL_PREFIX") - fanout_txid = str(payload.get("fanout_txid") or "") - block_height = first_int(payload.get("block_height"), default=0) - anchor_vout = nullable_int(payload.get("anchor_vout")) - fee_bits = fanout_fee_bits(payload) - return { - "fanout_txid": fanout_txid, - "block_hash": str(payload.get("block_hash") or ""), - "block_height": block_height, - "status": status, - "broadcastable_at_height": fanout_broadcastable_at_height(payload, block_height=block_height), - "manifest_set_sha256": str(payload.get("manifest_set_sha256") or ""), - "manifest_sha256": manifest_sha256, - "manifest_url": artifact_url(manifest_sha256) if manifest_sha256 else None, - "audit_bundle_sha256": nullable_str(payload.get("audit_bundle_sha256")), - "parent_coinbase_txid": str(payload.get("parent_coinbase_txid") or ""), - "parent_coinbase_vout": int(payload.get("parent_coinbase_vout", 0)), - "anchor_vout": anchor_vout, - "covenant_output_value_bits": int(payload.get("covenant_output_value_sats", 0)), - "fanout_output_sum_bits": int(payload.get("fanout_output_sum_sats", 0)), - "fanout_fee_bits": fee_bits, - "fanout_tx_hex": fanout_tx_hex, - "fanout_tx_sha256": fanout_transaction_hash(payload, fanout_tx_hex), - "cpfp_anchor_spendable": status == "broadcastable" and anchor_vout is not None and fee_bits == 0, - "last_broadcast_attempt_at": public_timestamp(last_broadcast_attempt_at), - "last_broadcast_error": nullable_str(last_broadcast_error), - "explorer_url": explorer_prefix.rstrip("/") + "/" + fanout_txid if explorer_prefix and fanout_txid else None, - } - - -def fanout_fee_bits(payload: dict[str, object]) -> int: - explicit_fee = payload.get("fanout_fee_sats") - if explicit_fee is not None: - try: - return max(0, int(explicit_fee)) - except (TypeError, ValueError): - return 0 - try: - covenant_value = int(payload.get("covenant_output_value_sats", 0)) - output_sum = int(payload.get("fanout_output_sum_sats", 0)) - except (TypeError, ValueError): - return 0 - return max(0, covenant_value - output_sum) - - -def nullable_int(value: object) -> int | None: - if value is None: - return None - try: - return int(value) - except (TypeError, ValueError): - return None - - -def fanout_broadcastable_at_height(payload: dict[str, object], *, block_height: int) -> int | None: - explicit_height = payload.get("broadcastable_at_height") - if explicit_height is not None: - try: - return int(explicit_height) - except (TypeError, ValueError): - return None - if block_height <= 0: - return None - return block_height + COINBASE_MATURITY - - -def artifact_link(kind: str, sha256: str, byte_length: int | None) -> dict[str, object]: - return { - "kind": kind, - "sha256": sha256, - "url": artifact_url(sha256), - "content_type": "application/json", - "byte_length": byte_length, - } - - -def artifact_url(sha256: str) -> str: - return f"/public/v1/artifacts/{sha256}" - - -def _fanout_parent_fields(payload: dict[str, object]) -> dict[str, object]: - fields: dict[str, object] = {} - for key in ("block_hash", "block_height", "audit_bundle_sha256", "manifest_set_sha256"): - value = payload.get(key) - if value is not None: - fields[key] = value - return fields - - -def network_summary(coordinator: Any) -> dict[str, object]: - try: - blockchain_info = coordinator.rpc.call("getblockchaininfo") - except Exception as exc: - raise PublicApiError(503, "qbit_rpc_unavailable", "qbit RPC getblockchaininfo failed") from exc - if not isinstance(blockchain_info, dict): - raise PublicApiError(503, "qbit_rpc_unavailable", "qbit RPC getblockchaininfo returned an invalid payload") - chain = str(blockchain_info.get("chain") or os.environ.get("QBIT_CHAIN") or "qbit") - template: dict[str, object] = {} - try: - raw_template = coordinator.rpc.call("getblocktemplate", [{"rules": qbit_gbt_rules(chain)}]) - if isinstance(raw_template, dict): - template = raw_template - except Exception: - template = {} - try: - network_info = coordinator.rpc.call("getnetworkinfo") - except Exception: - network_info = {} - # network_difficulty is reported in PRISM's scaled difficulty units - # (QBIT_DIFFICULTY_SCALE, see scaled_network_difficulty) so it lines up with the - # scaled per-share difficulties recorded in the ledger. Every downstream consumer - # assumes those units: the reward-window weight is network_difficulty * 8 compared - # against summed share_difficulty, and expected_time_to_block_seconds multiplies by - # HASHES_PER_QBIT_SCALED_DIFFICULTY. Derive it from the compact bits -- the same - # source the coordinator's share pipeline uses (build_job_for_client) -- rather than - # the raw getblockchaininfo.difficulty / getblocktemplate difficulty float, which is - # ~QBIT_DIFFICULTY_SCALE (1e6) times smaller. Feeding the raw value here made the ETA - # round to 0 and collapsed the reward window to a single share. - bits = None - difficulty = None - for raw_bits in (template.get("bits"), blockchain_info.get("bits")): - if raw_bits is None: - continue - candidate = str(raw_bits).lower() - if len(candidate) != 8 or any(char not in "0123456789abcdef" for char in candidate): - continue - try: - candidate_difficulty = scaled_network_difficulty(candidate) - except ValueError: - continue - bits = candidate - difficulty = candidate_difficulty - break - if bits is None or difficulty is None: - raise PublicApiError(503, "qbit_rpc_unavailable", "qbit RPC did not provide valid compact bits") - try: - difficulty_string = decimal_string(difficulty) - except Exception as exc: - raise PublicApiError(503, "qbit_rpc_unavailable", "qbit RPC returned invalid network difficulty") from exc - peers = first_nonnegative_int(network_info.get("connections") if isinstance(network_info, dict) else None) - return { - "name": chain, - "height": first_nonnegative_int(blockchain_info.get("blocks"), blockchain_info.get("headers")), - "tip_hash": str(blockchain_info.get("bestblockhash") or "0" * 64).lower(), - "bits": bits, - "network_difficulty": difficulty_string, - "initial_block_download": bool(blockchain_info.get("initialblockdownload", False)), - "peers": peers, - } - - -def network_hashrate_ths(coordinator: Any) -> str | None: - """Chain-derived estimate of the permissionless-lane network hashrate, TH/s. - - qbitd's getnetworkhashps answers in H/s over a trailing block window; the - permissionless lane is requested because that is the lane pool shares are - credited against, so consumers can divide a pool hashrate_ths window by - this value for an approximate share-of-network. Nullable and non-fatal by - contract: any RPC failure or non-numeric answer yields None, never an - error response -- pool-summary may only 503 over missing compact bits. - """ - try: - raw = coordinator.rpc.call( - "getnetworkhashps", - [NETWORK_HASHRATE_BLOCK_WINDOW, -1, NETWORK_HASHRATE_LANE], - ) - except Exception: - return None - if raw is None or isinstance(raw, bool): - return None - try: - hashes_per_second = Decimal(str(raw)) - except (ArithmeticError, ValueError, TypeError): - return None - if not hashes_per_second.is_finite() or hashes_per_second < 0: - return None - return decimal_string(hashes_per_second / TERAHASH) - - -def first_present(*values: object) -> object | None: - for value in values: - if value is not None: - return value - return None - - -def first_nonnegative_int(*values: object, default: int = 0) -> int: - for value in values: - if value is None: - continue - try: - return max(0, int(value)) - except (TypeError, ValueError): - continue - return default - - -def expected_time_to_block_seconds(*, hashrate_ths: str, network_difficulty: str) -> int | None: - hashrate = Decimal(hashrate_ths) - if hashrate <= 0: - return None - difficulty = Decimal(network_difficulty) - seconds = difficulty * HASHES_PER_QBIT_SCALED_DIFFICULTY / (hashrate * TERAHASH) - if seconds < 0: - return None - return max(0, int(seconds.to_integral_value())) - - -def hashrate_ths_from_difficulty(total_difficulty: int | str | Decimal, seconds: int) -> str: - if seconds <= 0: - return "0" - difficulty = Decimal(str(total_difficulty)) - if difficulty <= 0: - return "0" - return decimal_string(difficulty * HASHES_PER_QBIT_SCALED_DIFFICULTY / Decimal(seconds) / TERAHASH) - - -def hashrate_series_min_epoch(now_epoch: int, range_seconds: int, bucket_seconds: int) -> int: - """First bucket epoch whose full span lies inside the bounded range. - - A bucket straddling the range start is trimmed from the response: the - lookback-widened ledger query would aggregate pre-range shares into its raw - fields (and the unwidened query clipped it short), so neither raw reading - describes a fully covered bucket. Its shares still feed the trailing - windows of the kept points. - """ - range_start_epoch = now_epoch - range_seconds - return -(-range_start_epoch // bucket_seconds) * bucket_seconds - - -def smooth_hashrate_series_points( - points: list[dict[str, object]], - *, - bucket_seconds: int, - window_seconds: int, - min_epoch: int | None = None, -) -> list[dict[str, object]]: - """Recompute each point's hashrate over a trailing window of buckets. - - Vardiff keeps accepted-share counts per bucket small, so raw 5m bucket rates - swing tens of percent point-to-point and the dashboard charts come out - choppy. Each point keeps its bucket cadence but its rate is averaged over - the trailing window, with buckets missing from the series counting as zero - difficulty. accepted_share_count and accepted_share_difficulty stay - per-bucket. Windows smaller than two buckets leave the rates unchanged. - - Points before min_epoch are pre-range lookback context: they feed the - trailing windows of the first in-range points and are dropped from the - result. A point whose timestamp or integer difficulty does not parse is - dropped as well -- it can be neither windowed nor range-checked, and - dropping it beats leaking pre-range buckets or failing the endpoint over a - display refinement. Points are returned ascending by timestamp. - - The trailing total is a rolling int sum (add the entering bucket, evict - expired ones), keeping the pass linear in the number of points regardless - of the window size and exact regardless of the thread-local Decimal - precision. - """ - bucket_count = window_seconds // bucket_seconds if bucket_seconds > 0 else 0 - if (bucket_count < 2 and min_epoch is None) or not points: - return points - effective_window_seconds = bucket_count * bucket_seconds - parsed_points = _parse_hashrate_series_points(points) - if bucket_count < 2: - return [point for epoch, _, point in parsed_points if min_epoch is None or epoch >= min_epoch] - smoothed: list[dict[str, object]] = [] - for epoch, _, point, window_total in _trailing_window_totals(parsed_points, effective_window_seconds): - if min_epoch is not None and epoch < min_epoch: - continue - smoothed.append( - { - **point, - "hashrate_ths": hashrate_ths_from_difficulty(window_total, effective_window_seconds), - } - ) - return smoothed - - -def hashrate_series_smoothing(*, bucket_seconds: int, window_seconds: int) -> dict[str, object]: - """Describe the smoothing a dual-rate series was produced with. - - Mirrors smooth_hashrate_series_points: a window shorter than two buckets - (including the smoothing kill switch at 0) leaves rates untouched, so the - method is reported as `none` and the effective window is the bucket - itself. Otherwise the trailing window is the whole number of buckets that - fit in the configured seconds. - """ - bucket_count = window_seconds // bucket_seconds if bucket_seconds > 0 else 0 - if bucket_count < 2: - return {"method": "none", "window_seconds": bucket_seconds} - return {"method": "trailing", "window_seconds": bucket_count * bucket_seconds} - - -def dual_rate_hashrate_series_points( - points: list[dict[str, object]], - *, - bucket_seconds: int, - window_seconds: int, - min_epoch: int | None = None, - generated_at_epoch: int, -) -> list[dict[str, object]]: - """Build v2 points carrying both the raw and the trailing-window rate. - - raw_hashrate_ths is the bucket's credited difficulty over the full bucket - duration, recomputed here rather than trusting the ledger's per-bucket - rate so the documented definition holds for every backend. - smoothed_hashrate_ths is the same trailing-window estimate the v1 series - serves as hashrate_ths; when smoothing is off or shorter than two buckets - it equals the raw rate. A bucket is complete once generated_at has reached - its end, so the newest bucket is normally still accumulating credit and - its raw rate under-reads the true rate. - - Pre-range lookback context and malformed points are handled exactly as in - smooth_hashrate_series_points. Points are returned ascending by timestamp. - """ - smoothing = hashrate_series_smoothing(bucket_seconds=bucket_seconds, window_seconds=window_seconds) - trailing = smoothing["method"] == "trailing" - effective_window_seconds = int(smoothing["window_seconds"]) - parsed_points = _parse_hashrate_series_points(points) - dual_rate: list[dict[str, object]] = [] - for epoch, difficulty, point, window_total in _trailing_window_totals(parsed_points, effective_window_seconds): - if min_epoch is not None and epoch < min_epoch: - continue - try: - accepted_share_count = int( - Decimal(str(point.get("accepted_share_count", 0))) - ) - except (ValueError, InvalidOperation, OverflowError): - accepted_share_count = 0 - raw_hashrate_ths = hashrate_ths_from_difficulty(difficulty, bucket_seconds) - dual_rate.append( - { - "timestamp": point["timestamp"], - "raw_hashrate_ths": raw_hashrate_ths, - "smoothed_hashrate_ths": ( - hashrate_ths_from_difficulty(window_total, effective_window_seconds) - if trailing - else raw_hashrate_ths - ), - "accepted_share_count": accepted_share_count, - "accepted_share_difficulty": str(difficulty), - "complete": generated_at_epoch >= epoch + bucket_seconds, - } - ) - return dual_rate - - -def _parse_hashrate_series_points( - points: list[dict[str, object]], -) -> list[tuple[int, int, dict[str, object]]]: - """Return (epoch, difficulty, point) ascending by epoch, dropping malformed points.""" - parsed_points: list[tuple[int, int, dict[str, object]]] = [] - for point in points: - try: - parsed = datetime.strptime(str(point["timestamp"]), "%Y-%m-%dT%H:%M:%SZ") - epoch = int(parsed.replace(tzinfo=timezone.utc).timestamp()) - difficulty = int(str(point["accepted_share_difficulty"])) - except (KeyError, ValueError): - continue - parsed_points.append((epoch, difficulty, point)) - parsed_points.sort(key=lambda entry: entry[0]) - return parsed_points - - -def _trailing_window_totals( - parsed_points: list[tuple[int, int, dict[str, object]]], - window_seconds: int, -) -> Iterator[tuple[int, int, dict[str, object], int]]: - """Yield each parsed point with the difficulty total of its trailing window. - - The window covers buckets with epoch in (epoch - window_seconds, epoch]; - buckets missing from the series contribute nothing, so gaps count as zero - difficulty. - """ - window: deque[tuple[int, int]] = deque() - window_total = 0 - for epoch, difficulty, point in parsed_points: - window.append((epoch, difficulty)) - window_total += difficulty - while window and window[0][0] <= epoch - window_seconds: - _, expired_difficulty = window.popleft() - window_total -= expired_difficulty - yield epoch, difficulty, point, window_total - - -def pagination(page: int, limit: int, total_count: int) -> dict[str, int]: - total_pages = math.ceil(total_count / limit) if total_count else 0 - return { - "page": page, - "limit": limit, - "total_count": total_count, - "total_pages": total_pages, - } - - -def pagination_params(query: dict[str, list[str]]) -> tuple[int, int]: - try: - page = int(first_query_value(query, "page") or "1") - limit = int(first_query_value(query, "limit") or "15") - except ValueError as exc: - raise PublicApiError(400, "invalid_pagination", "page and limit must be integers") from exc - if page < 1: - raise PublicApiError(400, "invalid_page", "page must be >= 1") - if limit < 1 or limit > 100: - raise PublicApiError(400, "invalid_limit", "limit must be between 1 and 100") - return page, limit - - -def clean_hash(value: str, *, name: str = "hash") -> str: - value = value.strip() - if len(value) != 64 or any(char not in "0123456789abcdefABCDEF" for char in value): - raise PublicApiError(400, "invalid_hash", f"{name} must be 64 hex characters") - return value.lower() - - -def clean_recipient_id(value: str) -> str: - value = value.strip() - if not value: - raise PublicApiError(400, "invalid_recipient_id", "recipient_id is required") - if len(value) > MAX_RECIPIENT_ID_LENGTH: - raise PublicApiError(400, "invalid_recipient_id", "recipient_id must be 256 characters or fewer") - return value - - -def first_query_value(query: dict[str, list[str]], key: str) -> str | None: - values = query.get(key) - if values: - return values[0] - return None - - -def search_param(query: dict[str, list[str]]) -> str | None: - search = first_query_value(query, "search") - if search is not None and len(search) > MAX_SEARCH_LENGTH: - raise PublicApiError(400, "invalid_search", "search must be 128 characters or fewer") - return search - - -def auto_bucket(range_id: str) -> str: - if range_id == "1w": - return "1h" - if range_id == "1m": - return "1h" - return "1d" - - -def allowed_hashrate_bucket(range_id: str, bucket: str) -> None: - """Reject a bucket outside the range's static vocabulary. - - Called after auto resolution, so ``bucket`` is always concrete; auto - resolves inside every allowed set and can never be rejected here. A range - absent from HASHRATE_SERIES_ALLOWED_BUCKETS passes -- range validity is - the caller's own check, and this helper must stay reusable by every - bucketed public series endpoint, so it speaks only of buckets and ranges. - """ - allowed = HASHRATE_SERIES_ALLOWED_BUCKETS.get(range_id) - if allowed is None or bucket in allowed: - return - ordered = sorted(allowed, key=HASHRATE_SERIES_BUCKET_SECONDS.__getitem__) - raise PublicApiError( - 400, - "invalid_bucket", - f"bucket {bucket} is not allowed for range {range_id}; " - f"allowed: {', '.join(ordered)}", - ) - - -def qbit_gbt_rules(chain: str) -> list[str]: - rules = ["segwit"] - if "signet" in chain.strip().lower(): - rules.append("signet") - return rules - - -def target_from_compact(bits_hex: str) -> int: - return direct_stratum.target_from_compact_hex(bits_hex) - - -def scaled_network_difficulty(bits_hex: str) -> int: - template_target = target_from_compact(bits_hex) - if template_target <= 0: - raise ValueError("compact bits target must be positive") - if QBIT_POW_LIMIT_TARGET <= 0: - raise ValueError("pow limit target must be positive") - return max(1, (QBIT_POW_LIMIT_TARGET * QBIT_DIFFICULTY_SCALE) // template_target) - - -def utc_now_iso() -> str: - return iso_datetime(datetime.now(timezone.utc)) - - -def iso_datetime(value: datetime) -> str: - if value.tzinfo is None: - value = value.replace(tzinfo=timezone.utc) - return value.astimezone(timezone.utc).replace(microsecond=0).isoformat().replace("+00:00", "Z") - - -def decimal_string(value: int | str | Decimal) -> str: - decimal = Decimal(str(value)) - if decimal == decimal.to_integral_value(): - return str(decimal.quantize(Decimal(1))) - return format(decimal.normalize(), "f") - - -def nullable_str(value: object) -> str | None: - if value is None: - return None - text = str(value) - return text if text else None - - -def optional_hex_hash(value: object) -> str | None: - text = nullable_str(value) - if text is None: - return None - text = text.lower() - if len(text) == 64 and all(char in "0123456789abcdef" for char in text): - return text - return None - - -def fanout_transaction_hash(payload: dict[str, object], fanout_tx_hex: str) -> str: - fanout_txid = optional_hex_hash(payload.get("fanout_txid")) - if fanout_txid is not None: - return fanout_txid - derived_txid = txid_from_tx_hex(fanout_tx_hex) - if derived_txid is not None: - return derived_txid - raise PublicApiError(500, "internal_error", "fanout transaction hash unavailable") - - -def txid_from_tx_hex(value: str) -> str | None: - if not value: - return None - try: - return direct_stratum.transaction_txid_display(value) - except ValueError: - return None - - -def first_int(*values: object, default: int) -> int: - value = first_optional_int(*values) - return value if value is not None else default - - -def first_optional_int(*values: object) -> int | None: - for value in values: - if value is None: - continue - try: - return int(value) - except (TypeError, ValueError): - continue - return None - - -def first_optional_nonnegative_int(*values: object) -> int | None: - for value in values: - if value is None: - continue - text = str(value).strip() - if not text: - continue - try: - parsed = int(text) - except (TypeError, ValueError): - continue - if parsed >= 0: - return parsed - return None - - -def public_timestamp(value: object) -> str | None: - if value is None: - return None - text = str(value) - if " " in text and "T" not in text: - text = text.replace(" ", "T", 1) - if text.endswith("+00"): - return text[:-3] + "Z" - if text.endswith("+00:00"): - return text[:-6] + "Z" - return text - - -def shares_for_recipient(ledger: Any, recipient_id: str) -> list[object]: - return [share for share in ledger.all_shares() if str(getattr(share, "miner_id", "")) == recipient_id] - - -def miner_share_summary(ledger: Any, recipient_id: str) -> dict[str, object]: - read_model = getattr(ledger, "dashboard_miner_share_summary", None) - if callable(read_model): - return read_model(recipient_id=recipient_id) - shares = shares_for_recipient(ledger, recipient_id) - pool_3h_difficulty = difficulty_since(ledger.all_shares(), hours=3) - miner_3h_difficulty = difficulty_since(shares, hours=3) - return { - "hashrate_ths": miner_hashrate_rollups(shares), - "accepted_3h": share_count_since(shares, hours=3), - "accepted_difficulty_3h": str(miner_3h_difficulty), - "last_share_at": last_share_at(shares), - "share_percent": percent_string(miner_3h_difficulty, pool_3h_difficulty), - } - - -def miner_reward_window(ledger: Any, recipient_id: str, *, current_network_difficulty: object) -> dict[str, object]: - read_model = getattr(ledger, "dashboard_miner_reward_window", None) - if callable(read_model): - return read_model( - recipient_id=recipient_id, - current_network_difficulty=current_network_difficulty, - ) - return { - "accepted_difficulty": "0", - "pool_accepted_difficulty": "0", - "share_percent": None, - } - - -def _share_time(share: object) -> datetime: - return datetime.fromtimestamp(int(getattr(share, "accepted_at_ms")) / 1000, timezone.utc) - - -def difficulty_since(shares: list[object], *, hours: int = 0, minutes: int = 0) -> int: - now = datetime.now(timezone.utc) - cutoff = now - timedelta(hours=hours, minutes=minutes) - return sum(int(getattr(share, "share_difficulty")) for share in shares if cutoff <= _share_time(share) <= now) - - -def share_count_since(shares: list[object], *, hours: int = 0, minutes: int = 0) -> int: - now = datetime.now(timezone.utc) - cutoff = now - timedelta(hours=hours, minutes=minutes) - return sum(1 for share in shares if cutoff <= _share_time(share) <= now) - - -def last_share_at(shares: list[object]) -> str | None: - if not shares: - return None - return iso_datetime(max(_share_time(share) for share in shares)) - - -def miner_hashrate_rollups(shares: list[object]) -> dict[str, str]: - return { - "m1": hashrate_ths_from_difficulty(difficulty_since(shares, minutes=1), 60), - "m5": hashrate_ths_from_difficulty(difficulty_since(shares, minutes=5), 5 * 60), - "m10": hashrate_ths_from_difficulty(difficulty_since(shares, minutes=10), 10 * 60), - "h3": hashrate_ths_from_difficulty(difficulty_since(shares, hours=3), 3 * 60 * 60), - "h24": hashrate_ths_from_difficulty(difficulty_since(shares, hours=24), 24 * 60 * 60), - } - - -def percent_string(part: int | str | Decimal, total: int | str | Decimal) -> str | None: - part_decimal = Decimal(str(part)) - total_decimal = Decimal(str(total)) - if total_decimal <= 0: - return None - return decimal_string(part_decimal * Decimal(100) / total_decimal) - - -def owed_balance_for_recipient(ledger: Any, recipient_id: str) -> int: - read_model = getattr(ledger, "dashboard_miner_owed_balance_bits", None) - if callable(read_model): - return int(read_model(recipient_id=recipient_id)) - # In-memory ledger only: current_owed_balances() is a writer-lock read on - # the Postgres ledger, and returns every recipient's balance to filter one. - return sum( - int(balance.get("balance_sats", 0)) - for balance in ledger.current_owed_balances() - if str(balance.get("recipient_id")) == recipient_id - ) - - -def payout_rows(coordinator: Any, *, recipient_id: str, page: int, limit: int) -> dict[str, object]: - read_model = getattr(coordinator.ledger, "dashboard_miner_payout_rows", None) - if callable(read_model): - return read_model(recipient_id=recipient_id, page=page, limit=limit) - history = coordinator.ledger.recipient_payout_history(recipient_id=recipient_id, limit=1_000) - rows = [miner_payout_row(row) for row in history] - offset = (page - 1) * limit - return {"pagination": pagination(page, limit, len(rows)), "rows": rows[offset : offset + limit]} - - -def earnings_rows(coordinator: Any, *, recipient_id: str, page: int, limit: int) -> dict[str, object]: - read_model = getattr(coordinator.ledger, "dashboard_miner_earning_rows", None) - if callable(read_model): - return read_model(recipient_id=recipient_id, page=page, limit=limit) - history = coordinator.ledger.recipient_payout_history(recipient_id=recipient_id, limit=1_000) - rows = [miner_earning_row(row) for row in history] - offset = (page - 1) * limit - return {"pagination": pagination(page, limit, len(rows)), "rows": rows[offset : offset + limit]} - - -def miner_worker_rows( - ledger: Any, - *, - recipient_id: str, - page: int, - limit: int, - search: str | None, - hide_inactive: bool, -) -> dict[str, object]: - read_model = getattr(ledger, "dashboard_miner_worker_rows", None) - if callable(read_model): - return read_model( - recipient_id=recipient_id, - page=page, - limit=limit, - search=search, - hide_inactive=hide_inactive, - ) - rows = worker_rows(ledger, recipient_id=recipient_id) - active_count = sum(1 for row in rows if row["status"] == "active") - if search: - rows = [row for row in rows if search.lower() in str(row["worker_name"]).lower()] - if hide_inactive: - rows = [row for row in rows if row["status"] == "active"] - offset = (page - 1) * limit - return { - "pagination": pagination(page, limit, len(rows)), - "rows": rows[offset : offset + limit], - "active_count": active_count, - } - - -def miner_payout_row(row: dict[str, object]) -> dict[str, object]: - onchain = str(row.get("action")) == "onchain" - fanout_txid = nullable_str(row.get("fanout_txid")) if onchain else None - coinbase_txid = nullable_str(row.get("coinbase_txid")) if onchain else None - if fanout_txid: - # The recipient's output is committed inside a CTV fanout rather than - # paid directly from the coinbase: report the fanout transaction and - # the committed output value (gross entitlement minus the fanout fee - # share), which is what actually lands on-chain. - transaction_kind = "ctv_fanout" - txid = fanout_txid - onchain_amount_sats = first_int( - row.get("fanout_amount_sats"), - row.get("onchain_amount_sats"), - default=0, - ) - elif coinbase_txid: - transaction_kind = "coinbase" - txid = coinbase_txid - onchain_amount_sats = int(row.get("onchain_amount_sats", 0)) - else: - transaction_kind = "carry_forward" - txid = None - onchain_amount_sats = int(row.get("onchain_amount_sats", 0)) - explorer_prefix = os.environ.get("PRISM_PUBLIC_EXPLORER_TX_URL_PREFIX") - return { - "block_height": int(row.get("block_height", 0)), - "block_hash": str(row.get("block_hash") or ""), - "created_at": required_public_timestamp(row.get("created_at")), - "transaction_id": txid, - "transaction_kind": transaction_kind, - "onchain_amount_bits": onchain_amount_sats, - "carry_forward_balance_bits": int(row.get("carry_forward_balance_sats", 0)), - "action": str(row.get("action") or "accrued"), - "maturity_state": str(row.get("maturity_state") or "immature"), - "explorer_url": explorer_prefix.rstrip("/") + "/" + txid if explorer_prefix and txid else None, - } - - -def lifetime_earnings_for_recipient(ledger: Any, recipient_id: str) -> int: - read_model = getattr(ledger, "dashboard_miner_lifetime_earnings_bits", None) - if callable(read_model): - return int(read_model(recipient_id=recipient_id)) - history = ledger.recipient_payout_history(recipient_id=recipient_id, limit=1_000) - return sum(gross_earning_bits_from_row(row) for row in history) - - -def pending_maturity_bits_for_recipient(ledger: Any, recipient_id: str) -> int: - read_model = getattr(ledger, "dashboard_miner_pending_maturity_bits", None) - if callable(read_model): - return int(read_model(recipient_id=recipient_id)) - history = ledger.recipient_payout_history(recipient_id=recipient_id, limit=1_000) - return sum( - max(0, int(row.get("onchain_amount_sats", 0)) - int(row.get("settlement_fee_sats", 0))) - for row in history - if row.get("action") == "onchain" and row.get("maturity_state") == "immature" - ) - - -def miner_earning_row(row: dict[str, object]) -> dict[str, object]: - gross = gross_earning_bits_from_row(row) - settlement_fee = int(row.get("settlement_fee_sats", 0)) - net = max(0, gross - settlement_fee) - block_hash = str(row.get("block_hash") or "") - explorer_prefix = os.environ.get("PRISM_PUBLIC_EXPLORER_BLOCK_URL_PREFIX") - return { - "block_height": int(row.get("block_height", 0)), - "block_hash": block_hash, - "found_at": required_public_timestamp(row.get("found_at", row.get("created_at"))), - "reward_share_percent": reward_share_percent_from_row(row, gross), - "gross_earning_bits": gross, - "settlement_fee_bits": settlement_fee, - "net_earning_bits": net, - "maturity_state": str(row.get("maturity_state") or "immature"), - "settlement_artifacts_url": f"/public/v1/blocks/{block_hash}/settlement-artifacts" if block_hash else None, - "explorer_url": explorer_prefix.rstrip("/") + "/" + block_hash if explorer_prefix and block_hash else None, - } - - -def worker_rows(ledger: Any, *, recipient_id: str) -> list[dict[str, object]]: - shares = shares_for_recipient(ledger, recipient_id) - by_worker: dict[str, list[object]] = {} - for share in shares: - by_worker.setdefault(worker_name_from_share(share, recipient_id), []).append(share) - rows = [ - { - "worker_name": worker_name, - "status": "active" if share_count_since(worker_shares, minutes=10) > 0 else "inactive", - "last_share_at": last_share_at(worker_shares), - "hashrate_ths_60s": hashrate_ths_from_difficulty(difficulty_since(worker_shares, minutes=1), 60), - "hashrate_ths_3h": hashrate_ths_from_difficulty(difficulty_since(worker_shares, hours=3), 3 * 60 * 60), - } - for worker_name, worker_shares in by_worker.items() - ] - rows.sort(key=lambda row: (row["status"] != "active", str(row["worker_name"]))) - return rows - - -def worker_name_from_share(share: object, recipient_id: str) -> str: - username = str(getattr(share, "share_id", "")).rsplit(":", 1)[0] - if not username: - return "default" - if username == recipient_id: - return "default" - if username.startswith(recipient_id + "."): - worker_name = username[len(recipient_id) + 1 :] - return worker_name or "default" - if "." in username: - worker_name = username.split(".", 1)[1] - return worker_name or "default" - return "default" - - -def gross_earning_bits_from_row(row: dict[str, object]) -> int: - fallback_gross = int(row.get("onchain_amount_sats", 0)) + int(row.get("carry_forward_balance_sats", 0)) - return int(row.get("gross_amount_sats", fallback_gross)) - - -def reward_share_percent_from_row(row: dict[str, object], gross: int) -> str: - raw_percent = nullable_str(row.get("reward_share_percent")) - if raw_percent is not None: - return decimal_string(raw_percent) - block_gross = first_optional_int( - row.get("block_gross_amount_sats"), - row.get("total_gross_amount_sats"), - row.get("block_total_gross_amount_sats"), - ) - if block_gross is None: - raise PublicApiError(500, "internal_error", "internal server error") - return percent_string(gross, block_gross) or "0" - - -def required_public_timestamp(value: object) -> str: - timestamp = public_timestamp(value) - if timestamp is None: - raise PublicApiError(500, "internal_error", "internal server error") - return timestamp diff --git a/lab/prism/public_read_service.py b/lab/prism/public_read_service.py deleted file mode 100644 index d9a02eba1..000000000 --- a/lab/prism/public_read_service.py +++ /dev/null @@ -1,1535 +0,0 @@ -#!/usr/bin/env python3 -"""Standalone process serving the PRISM public ``/public/v1`` read surface. - -The coordinator used to serve this surface from its own audit/ops listener, -under the GIL that acknowledges shares and lands blocks, against the primary -Postgres the lease-holding writer commits through. Public read traffic scales -with public interest rather than with hashrate, so it is the one workload whose -volume the pool does not control -- and it sat on the money path. This process -takes it off that path. - -What this is *not*: a rewrite. ``lab.prism.public_api`` reaches its host through -a deliberately small surface -- ``.ledger``, ``.rpc``, and an environment-backed -``getattr`` for the advertised Stratum endpoint -- so the extraction is a new -host object plus a small HTTP shell. Routing, payloads, cache policy and error -shapes stay in ``public_api``; ``tests/test_prism_public_read_service.py`` pins -this process's responses against ``public_api.dispatch`` directly. - -Three properties this process guarantees that the coordinator's listener did -not need to: - -Read-only in the strong sense - The ledger is constructed without any ``PRISM_LEDGER_WRITER_*`` input and - never acquires a writer lease: every public read model runs through - ``PsqlShareLedger._run_read_json`` (a bounded read slot), never through the - writer gate. The in-memory ledger is refused outright -- it lacks the - ``dashboard_*`` read models, so ``public_api``'s duck-typed fallbacks would - quietly reintroduce writer-lock reads (``current_owed_balances``, - ``recipient_payout_history``) on the public surface, which is precisely the - contention this extraction exists to remove. - -Fail-closed configuration - ``public_api.mining_configuration`` falls back to - ``PRISM_STRATUM_PORT``/``127.0.0.1`` when its host exposes no ``port`` or - ``bind``. This process deliberately exposes neither, so that fallback is - live -- and left alone it would advertise ``127.0.0.1:3340`` to the world as - the pool's Stratum endpoint. ``PRISM_PUBLIC_STRATUM_URL`` is therefore - required at startup rather than defaulted. - -An honest staleness contract - Every extracted response states the age it is willing to serve - (``X-Prism-Staleness-Budget-Seconds``) alongside the age it actually has - (``Age``), and refuses with 503 rather than serve past it. The budgets live - in ``lab.prism.endpoint_registry`` and are derived from the caches beneath - each route; see that module's docstring. This follows - ``lab/prism/observability.py``, where the cached ``/metrics`` endpoint - already returns 503 rather than serve an unbounded-stale snapshot. - - The same contract is what a database outage is answered with (#164). While - the readiness probe says Postgres is unreachable, a database-backed route - serves its warm cache entry if it has one inside that budget -- labelled as - degraded, so the client is told the difference between "30s old because - that is the poll interval" and "30s old because this is the last answer - that exists" -- and refuses with 503 otherwise. It never calls the database - to refill and never extends the entry it serves, so the budget stays the - real bound: ordinary traffic cannot keep a stale dashboard alive past it. - -Entry point: ``python3 -m lab.prism.public_read_service``. -""" - -from __future__ import annotations - -import ipaddress -import json -import math -import os -import re -import shlex -import signal -import socket -import sys -import threading -import time -import urllib.parse -from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer -from pathlib import Path -from typing import Any, Callable - -if not __package__: - sys.path.insert(0, str(Path(__file__).resolve().parents[2])) - -from lab.prism import endpoint_registry, public_api -from lab.prism.audit_artifacts import AuditArtifactConfig, AuditArtifactStore -from lab.prism.coordinator_config import ( - DEFAULT_POSTGRES_IDLE_IN_TRANSACTION_TIMEOUT_SECONDS, - DEFAULT_POSTGRES_TCP_KEEPALIVES_COUNT, - DEFAULT_POSTGRES_TCP_KEEPALIVES_IDLE_SECONDS, - DEFAULT_POSTGRES_TCP_KEEPALIVES_INTERVAL_SECONDS, - env, - env_int, - env_nonnegative_float, - env_positive_float, - env_positive_int, -) -from lab.prism.rpc import JsonRpc -from lab.prism.share_ledger import LedgerOperationTimeout, PsqlShareLedger - - -DEFAULT_PUBLIC_API_BIND = "0.0.0.0" -DEFAULT_PUBLIC_API_PORT = 3342 - -# Deadline for one origin computation's database work. Sized above the ~15s -# nginx proxy timeout in front of this tier: a statement that outlives the -# client's patience by a little may still land a cache entry the retry can -# hit, but one that runs for minutes after the client hung up only holds a -# read slot against every other public read. -DEFAULT_PUBLIC_READ_STATEMENT_TIMEOUT_SECONDS = 20 - -# Emitted on every extracted response: the age this route is willing to serve. -# The observed age travels beside it in the standard Age header. -STALENESS_BUDGET_HEADER = "X-Prism-Staleness-Budget-Seconds" - -# What that header says for /public/v1/artifacts/{sha256}, the one route with no -# budget. A number would be a lie in either direction; this names the contract. -UNBOUNDED_STALENESS_BUDGET = "unbounded" - -# Said on every response served, or refused, while the readiness probe reports -# the database unreachable. Age and STALENESS_BUDGET_HEADER already carry the -# numbers; these two carry the fact, which the numbers alone cannot. Without -# them an ordinary 30-second-old poll response and the last answer that still -# exists are the same response. -# -# Warning 110 is the registered "Response is Stale" warn-code (RFC 7234 5.5.1, -# retained by clients that still parse the header), with this service named as -# the agent that added it. The X- header is what an operator greps and what a -# dashboard frontend switches an outage banner on, because it is a single value -# rather than a warn-code line to parse. -DEGRADED_WARNING_HEADER = "Warning" -DEGRADED_WARNING = '110 qbit-prism "database unavailable; serving cached response"' -DATABASE_STATE_HEADER = "X-Prism-Database-State" -DATABASE_STATE_UNAVAILABLE = "unavailable" -ARTIFACT_CANONICAL_STATE_HEADER = "X-Prism-Artifact-Canonical-State" - -# How often the background prober re-checks Postgres, and the age at which its -# answer stops counting. Same shape as observability.py's health staleness: -# three refresh intervals, floored, so ordinary jitter is not an outage. -DEFAULT_READINESS_PROBE_INTERVAL_SECONDS = 5.0 -MINIMUM_READINESS_STALE_SECONDS = 15.0 - -# How long the standby's replication stream may be silent before its answers -# stop counting as current. Bounds the *stream*, not replay lag -- see -# ReplicaProbe. -DEFAULT_REPLICA_MAX_LAG_SECONDS = 60.0 - -# Observed replay lag, published on every replica-backed response. Advisory: -# the enforced bound is the heartbeat age, for the reason ReplicaProbe explains. -REPLICA_LAG_HEADER = "X-Prism-Replica-Lag-Seconds" - -# require: refuse replica-backed routes unless the backing server is a standby -# whose replication stream is live within the bound. off: serve whatever the -# DSN names, which is the behaviour of the extraction before a standby existed. -# off is the default so that merging this does not 503 a deployment that has -# not provisioned one yet; the shipped compose sets require. -REPLICA_MODE_REQUIRE = "require" -REPLICA_MODE_OFF = "off" -REPLICA_MODES = (REPLICA_MODE_REQUIRE, REPLICA_MODE_OFF) - -HEALTH_SCHEMA = "qbit.prism.public-read-health.v1" - - -class PublicReadConfigurationError(RuntimeError): - """Startup refused because the service would otherwise serve wrong facts.""" - - -class PublicReadCoordinator: - """The coordinator stand-in ``lab.prism.public_api`` reads through. - - ``public_api`` needs exactly ``.ledger`` (24 references) and ``.rpc`` - (getblockchaininfo / getblocktemplate / getnetworkinfo). It also probes - ``getattr(coordinator, "port")`` and ``getattr(coordinator, "bind")`` in - ``mining_configuration``, both of which already fall back to the - environment. - - Those two attributes are deliberately absent here. On the coordinator they - named the Stratum listener that process was itself running; this process - runs no Stratum listener, so any value it invented would be a guess - published to miners as connection instructions. Leaving them undefined - routes the lookup to the environment, and ``require_public_stratum_url()`` - makes the operator state the answer rather than inherit ``127.0.0.1``. - """ - - def __init__(self, *, ledger: Any, rpc: Any) -> None: - self.ledger = ledger - self.rpc = rpc - - -class ServiceMetrics: - """Counters for this process only, rendered as Prometheus text. - - Intentionally small: the coordinator keeps its own /metrics with the pool's - real operational signal. What is worth knowing here is whether the tier is - serving, whether its response cache is earning its keep, and whether the - staleness contract is actually refusing anything. - """ - - def __init__(self) -> None: - self._lock = threading.Lock() - self._requests_total = 0 - self._responses_by_status: dict[int, int] = {} - # "stale" is a served response: an expired entry inside its - # stale-while-revalidate window, answered immediately while one - # background refresh recomputes it. - self._cache_by_state: dict[str, int] = { - "hit": 0, - "miss": 0, - "bypass": 0, - "stale": 0, - } - self._staleness_refusals = 0 - self._replica_refusals = 0 - self._degraded_responses = 0 - self._database_outage_refusals = 0 - - def record_request(self) -> None: - with self._lock: - self._requests_total += 1 - - def record_response(self, status: int) -> None: - with self._lock: - self._responses_by_status[status] = ( - self._responses_by_status.get(status, 0) + 1 - ) - - def record_cache_state(self, cache_state: str) -> None: - key = cache_state.lower() - with self._lock: - if key in self._cache_by_state: - self._cache_by_state[key] += 1 - - def record_staleness_refusal(self) -> None: - with self._lock: - self._staleness_refusals += 1 - - def record_replica_refusal(self) -> None: - with self._lock: - self._replica_refusals += 1 - - def record_degraded_response(self) -> None: - with self._lock: - self._degraded_responses += 1 - - def record_database_outage_refusal(self) -> None: - with self._lock: - self._database_outage_refusals += 1 - - def render( - self, - *, - ready: bool, - readiness_age_seconds: float, - replica: dict[str, object] | None = None, - ) -> str: - with self._lock: - requests_total = self._requests_total - responses = dict(self._responses_by_status) - cache_states = dict(self._cache_by_state) - refusals = self._staleness_refusals - replica_refusals = self._replica_refusals - degraded_responses = self._degraded_responses - database_outage_refusals = self._database_outage_refusals - lines = [ - "# HELP qbit_prism_public_requests_total Public read requests accepted.", - "# TYPE qbit_prism_public_requests_total counter", - f"qbit_prism_public_requests_total {requests_total}", - "# HELP qbit_prism_public_responses_total Public read responses by status code.", - "# TYPE qbit_prism_public_responses_total counter", - ] - lines.extend( - f'qbit_prism_public_responses_total{{status="{status}"}} {count}' - for status, count in sorted(responses.items()) - ) - lines.extend( - [ - "# HELP qbit_prism_public_cache_total Response-cache outcomes by state.", - "# TYPE qbit_prism_public_cache_total counter", - ] - ) - lines.extend( - f'qbit_prism_public_cache_total{{state="{state}"}} {count}' - for state, count in sorted(cache_states.items()) - ) - lines.extend( - [ - "# HELP qbit_prism_public_staleness_refusals_total Responses refused for exceeding their staleness budget.", - "# TYPE qbit_prism_public_staleness_refusals_total counter", - f"qbit_prism_public_staleness_refusals_total {refusals}", - "# HELP qbit_prism_public_ledger_ready Whether the last Postgres readiness probe succeeded.", - "# TYPE qbit_prism_public_ledger_ready gauge", - f"qbit_prism_public_ledger_ready {1 if ready else 0}", - "# HELP qbit_prism_public_ledger_probe_age_seconds Age of the last readiness probe, or -1 before the first.", - "# TYPE qbit_prism_public_ledger_probe_age_seconds gauge", - f"qbit_prism_public_ledger_probe_age_seconds {readiness_age_seconds:.3f}", - "# HELP qbit_prism_public_replica_refusals_total Responses refused for failing the replica freshness contract.", - "# TYPE qbit_prism_public_replica_refusals_total counter", - f"qbit_prism_public_replica_refusals_total {replica_refusals}", - "# HELP qbit_prism_public_degraded_responses_total Cached responses served while the database was unreachable.", - "# TYPE qbit_prism_public_degraded_responses_total counter", - f"qbit_prism_public_degraded_responses_total {degraded_responses}", - "# HELP qbit_prism_public_database_outage_refusals_total Responses refused because the database was unreachable and no warm entry existed.", - "# TYPE qbit_prism_public_database_outage_refusals_total counter", - f"qbit_prism_public_database_outage_refusals_total {database_outage_refusals}", - ] - ) - if replica is not None: - # -1 rather than omission for the unknowns: a missing series and a - # series reading zero are the same alert, and neither is true. - lines.extend( - [ - "# HELP qbit_prism_public_replica_in_recovery Whether the backing server is a standby.", - "# TYPE qbit_prism_public_replica_in_recovery gauge", - f"qbit_prism_public_replica_in_recovery {1 if replica.get('in_recovery') else 0}", - "# HELP qbit_prism_public_replica_heartbeat_age_seconds Age of the newest walreceiver message, or -1 when disconnected. This is the enforced bound.", - "# TYPE qbit_prism_public_replica_heartbeat_age_seconds gauge", - f"qbit_prism_public_replica_heartbeat_age_seconds {_metric_number(replica.get('receiver_heartbeat_age_seconds'))}", - "# HELP qbit_prism_public_replica_replay_lag_seconds Wall clock behind the newest replayed commit, or -1. Advisory: grows on an idle primary.", - "# TYPE qbit_prism_public_replica_replay_lag_seconds gauge", - f"qbit_prism_public_replica_replay_lag_seconds {_metric_number(replica.get('replay_lag_seconds'))}", - "# HELP qbit_prism_public_replica_apply_backlog_bytes WAL received but not yet replayed, or -1.", - "# TYPE qbit_prism_public_replica_apply_backlog_bytes gauge", - f"qbit_prism_public_replica_apply_backlog_bytes {_metric_number(replica.get('apply_backlog_bytes'))}", - "# HELP qbit_prism_public_replica_max_lag_seconds The configured replication-stream silence bound.", - "# TYPE qbit_prism_public_replica_max_lag_seconds gauge", - f"qbit_prism_public_replica_max_lag_seconds {_metric_number(replica.get('max_lag_seconds'))}", - ] - ) - return "\n".join(lines) + "\n" - - -class ReplicaProbe: - """Is the standby this tier reads through still a standby, and still fed? - - Two refusals and one number, from one query per interval: - - * **Not in recovery.** The public tier exists to keep public read volume - off the primary the coordinator lands blocks through. A DSN that - resolves to a writable server silently undoes that, and the failure is - invisible in the responses -- they are correct, just served from the - wrong place. So it is refused rather than served. - * **A silent replication stream.** The liveness proof is the walreceiver - heartbeat age, *not* the transactional replay lag. Heartbeats flow every - ``wal_receiver_status_interval`` (10s by default) whether or not the - primary has anything to send, whereas replay lag grows with wall clock - on an idle primary -- so bounding replay lag would 503 a perfectly - healthy replica of a quiet pool every time the pool went quiet, which is - exactly when the dashboard is least likely to be wrong. - - Replay lag and apply backlog are still measured and published (header, - /healthz, /metrics); they are the number an operator wants, just not the - number a gate can be built on. - - Freshness ages locally, which is what makes this fail *closed*. The bound - is checked against the heartbeat age the last successful probe reported - **plus how long ago that probe ran**, so a probe that starts failing -- - or a standby that stops answering entirely -- ages its own last good - answer out of the bound instead of pinning it there forever. - - Runs as ReadinessProbe's probe callable rather than owning a thread: - readiness and replica freshness are the same question, and one probe means - one round trip per interval instead of two. - """ - - def __init__( - self, - status_probe: Callable[[], dict[str, object]], - *, - max_lag_seconds: float = DEFAULT_REPLICA_MAX_LAG_SECONDS, - monotonic: Callable[[], float] = time.monotonic, - ) -> None: - self._status_probe = status_probe - self._max_lag_seconds = float(max_lag_seconds) - self._monotonic = monotonic - self._lock = threading.Lock() - self._status: dict[str, object] | None = None - self._probed_at: float | None = None - self._last_error: str | None = None - - @property - def max_lag_seconds(self) -> float: - return self._max_lag_seconds - - def check(self) -> bool: - """One probe. True when the backing server is a live standby. - - Failure is recorded and re-raised so ReadinessProbe reports the real - error text rather than a bare "returned false"; the recorded snapshot - is deliberately left in place, to age out through freshness_error() - rather than vanish and read as warm-up. - """ - try: - status = self._status_probe() - except Exception as exc: # noqa: BLE001 - recorded, then re-raised - with self._lock: - self._last_error = f"{type(exc).__name__}: {exc}" - raise - with self._lock: - self._status = dict(status) - self._probed_at = self._monotonic() - self._last_error = None - return bool(status.get("in_recovery")) - - def view(self) -> dict[str, object]: - """What /healthz and /metrics report, including the aged numbers.""" - - with self._lock: - status = dict(self._status) if self._status is not None else None - probed_at = self._probed_at - last_error = self._last_error - payload: dict[str, object] = { - "probed": status is not None, - "max_lag_seconds": self._max_lag_seconds, - } - if last_error is not None: - payload["last_error"] = last_error - if status is None or probed_at is None: - return payload - snapshot_age = max(0.0, self._monotonic() - probed_at) - payload["probe_age_seconds"] = round(snapshot_age, 3) - payload["in_recovery"] = bool(status.get("in_recovery")) - heartbeat = _coerce_float(status.get("receiver_heartbeat_age_seconds")) - payload["receiver_heartbeat_age_seconds"] = ( - None if heartbeat is None else round(heartbeat + snapshot_age, 3) - ) - replay_lag = _coerce_float(status.get("replay_lag_seconds")) - payload["replay_lag_seconds"] = ( - None if replay_lag is None else round(replay_lag + snapshot_age, 3) - ) - backlog = _coerce_float(status.get("apply_backlog_bytes")) - payload["apply_backlog_bytes"] = None if backlog is None else int(backlog) - return payload - - def replay_lag_seconds(self) -> float | None: - """Observed replay lag for the response header, aged locally.""" - - value = self.view().get("replay_lag_seconds") - return None if value is None else float(value) - - def freshness_error(self) -> tuple[str, str] | None: - """(code, message) for the 503 a replica-backed route owes, or None.""" - - with self._lock: - status = dict(self._status) if self._status is not None else None - probed_at = self._probed_at - last_error = self._last_error - if status is None or probed_at is None: - message = "public read service is warming up" - if last_error is not None: - message = f"{message}: replica probe failed: {last_error}" - return "replica_unavailable", message - if not status.get("in_recovery"): - return ( - "replica_unavailable", - "public read service refuses to serve from a server that is " - "not in recovery; point PRISM_PUBLIC_DATABASE_URL at the read " - "replica, or set PRISM_PUBLIC_REPLICA_MODE=off to serve from " - "the primary deliberately", - ) - heartbeat = _coerce_float(status.get("receiver_heartbeat_age_seconds")) - if heartbeat is None: - return ( - "replica_unavailable", - "read replica replication stream is not connected; the " - "walreceiver heartbeat is absent", - ) - silent_for = heartbeat + max(0.0, self._monotonic() - probed_at) - if silent_for > self._max_lag_seconds: - detail = "" - if last_error is not None: - detail = f" (last probe failed: {last_error})" - return ( - "replica_unavailable", - f"read replica replication stream has been silent for " - f"{silent_for:.1f}s, beyond the configured bound " - f"{self._max_lag_seconds:.1f}s{detail}", - ) - return None - - -def _coerce_float(value: object) -> float | None: - """Postgres json numbers arrive as int/float/str depending on the driver.""" - - if value is None: - return None - try: - return float(value) # type: ignore[arg-type] - except (TypeError, ValueError): - return None - - -class ReadinessProbe: - """Background Postgres liveness, so /healthz never queries on the request thread. - - The compose healthcheck polls this forever. Probing inline would put a - database round trip -- and a read slot -- on every poll, and a stalled - database would hang the health request rather than answer it. Instead one - background thread runs the constant-select probe on an interval and - publishes the result; the request thread only reads two fields. - """ - - def __init__( - self, - probe: Callable[[], bool], - *, - interval_seconds: float = DEFAULT_READINESS_PROBE_INTERVAL_SECONDS, - monotonic: Callable[[], float] = time.monotonic, - ) -> None: - self._probe = probe - self._interval_seconds = max(0.1, float(interval_seconds)) - self._monotonic = monotonic - self._lock = threading.Lock() - self._ok = False - self._error: str | None = None - self._checked_at: float | None = None - self._stop = threading.Event() - self._thread: threading.Thread | None = None - - @property - def stale_after_seconds(self) -> float: - return max(3 * self._interval_seconds, MINIMUM_READINESS_STALE_SECONDS) - - def check_once(self) -> None: - try: - ok = bool(self._probe()) - error = None if ok else "readiness probe returned false" - except Exception as exc: # noqa: BLE001 - any failure means "not ready" - ok = False - error = f"{type(exc).__name__}: {exc}" - with self._lock: - self._ok = ok - self._error = error - self._checked_at = self._monotonic() - - def start(self) -> None: - # One synchronous probe first so a freshly started service reports its - # real state immediately instead of a warm-up 503 the healthcheck would - # have to wait out. - self.check_once() - self._thread = threading.Thread( - target=self._run, - name="prism-public-readiness", - daemon=True, - ) - self._thread.start() - - def _run(self) -> None: - while not self._stop.wait(self._interval_seconds): - self.check_once() - - def stop(self) -> None: - self._stop.set() - thread = self._thread - if thread is not None: - thread.join(timeout=5) - - def age_seconds(self) -> float: - with self._lock: - checked_at = self._checked_at - if checked_at is None: - return -1.0 - return max(0.0, self._monotonic() - checked_at) - - def snapshot(self) -> tuple[int, dict[str, object]]: - with self._lock: - ok = self._ok - error = self._error - checked_at = self._checked_at - if checked_at is None: - return 503, { - "ok": False, - "schema": HEALTH_SCHEMA, - "state": "starting", - "error": "readiness probe has not completed yet", - "probe_age_seconds": -1.0, - } - age_seconds = max(0.0, self._monotonic() - checked_at) - stale = age_seconds > self.stale_after_seconds - payload: dict[str, object] = { - "ok": bool(ok and not stale), - "schema": HEALTH_SCHEMA, - "state": "ready" if (ok and not stale) else "unready", - "probe_age_seconds": round(age_seconds, 3), - "probe_stale_after_seconds": self.stale_after_seconds, - } - if stale: - payload["error"] = "readiness probe is stale" - elif error is not None: - payload["error"] = error - return (200 if payload["ok"] else 503), payload - - -def path_reads_database(path: str) -> bool: - """Does this route's answer come out of the database? - - One predicate, two gates. The replica contract asks it to decide which - routes a silent standby may not answer; the outage gate asks it to decide - which routes a dead database may not answer. The exempt set is the same - both times, and for the same reasons, so it is derived once here rather - than stated twice. - - Derived from the endpoint registry rather than a hand-kept path list, so a - route added there is classified once. Two kinds of route are exempt: - - * those that touch no read slot at all -- today only - /public/v1/mining-configuration, assembled from environment. It has no - database answer to lose; - * content-addressed ones, which declare ``immutable_content``. A body that - hashes to the requested sha256 is correct at any age, which is why the - registry gives them no staleness budget either; refusing one for - replication lag, or for an outage, would contradict the unbounded budget - already published on it. - - An unclassified path is not database-backed here because dispatch() 404s it - before it can read anything. - """ - endpoint = endpoint_registry.endpoint_for_request_path(path) - if endpoint is None or endpoint.immutable_content: - return False - return endpoint_registry.LedgerAccess.READ_SLOT in endpoint.access - - -def path_occupies_read_slot(path: str) -> bool: - """Does serving this route's origin miss take a ledger read slot? - - Deliberately not ``path_reads_database``: that predicate exempts - content-addressed routes because an immutable body is correct at any age, - which is an answer about *staleness* -- replica lag and outages. The - statement deadline asks a different question: does a cold request occupy - one of the bounded Postgres read slots while it computes? A cold - ``/public/v1/artifacts/{sha256}`` does exactly that, for as long as the - lookup takes, so the deadline must cover it even though the staleness - gates never should. - """ - endpoint = endpoint_registry.endpoint_for_request_path(path) - if endpoint is None: - return False - return endpoint_registry.LedgerAccess.READ_SLOT in endpoint.access - - -def public_read_statement_timeout_seconds() -> int: - """The per-request database deadline, in seconds. 0 disables the bound. - - Read per request rather than captured at startup, so the knob behaves - like the cache TTL knobs beside it and a background cache refresh sees - the same value a foreground request would. - """ - return public_api.env_nonnegative_int( - "PRISM_PUBLIC_READ_STATEMENT_TIMEOUT_SECONDS", - DEFAULT_PUBLIC_READ_STATEMENT_TIMEOUT_SECONDS, - ) - - -def bounded_public_dispatch( - coordinator: Any, - path: str, - query: dict[str, list[str]], - *, - occupies_read_slot: bool, -) -> tuple[int, object]: - """One origin computation, deadline-bounded when it takes a read slot. - - Without a bound, a request the fronting proxy has already abandoned at - its ~15s timeout keeps its Postgres statement running to completion, - holding one of the bounded read slots (PRISM_POSTGRES_READ_CONCURRENCY) - against every other public read and competing with the share writer's - I/O for as long as the scan takes. The ledger's ``statement_timeout`` - scope arms a server-side ``SET LOCAL statement_timeout`` per statement - plus a matching local admission bound, so the whole request stops costing - anything shortly after its caller stops listening. - - The scope is the ledger's ``operation_timeout``, not ``statement_timeout``: - the latter hands every admission and SQL statement a fresh budget, so a - route that performs several sequential ledger reads (the miner detail - page does) could run many multiples of the configured limit after the - proxy hung up. ``operation_timeout`` is one deadline counting down across - the entire dispatch -- every statement still gets server-side - cancellation, armed with whatever remains of the request's budget. - - Duck-typed exactly like the coordinator's block-submitter scope: a ledger - without the scope runs unbounded, as it always has. On the ledger's - psql-subprocess fallback the server-side statement deadline is not armed; - admission waits still observe the bound, and no deadline at all is an - acceptable degradation there. Routes are wrapped by - ``path_occupies_read_slot`` -- registry READ_SLOT access, immutable or - not -- rather than by the staleness gates' ``path_reads_database``: a - cold content-addressed artifact lookup holds a read slot exactly like any - other cold read, even though its body, once found, is correct at any - age. - - A ledger-reported deadline becomes the public ``read_timeout`` 503. Its - error response is no-store and never cached, so one expensive miss - cannot poison the response cache with a refusal. - """ - if not occupies_read_slot: - return public_api.dispatch(coordinator, path, query) - timeout_seconds = public_read_statement_timeout_seconds() - operation_timeout = getattr(coordinator.ledger, "operation_timeout", None) - if timeout_seconds <= 0 or not callable(operation_timeout): - return public_api.dispatch(coordinator, path, query) - try: - with operation_timeout(float(timeout_seconds)): - return public_api.dispatch(coordinator, path, query) - except LedgerOperationTimeout as exc: - raise public_api.PublicApiError( - 503, - "read_timeout", - "the read timed out; try again shortly", - ) from exc - - -def staleness_budget_for_path(path: str) -> float | None: - """The budget for one concrete request path, or None when it has none. - - None means "never refuse this for age", which today is only - /public/v1/artifacts/{sha256} -- content-addressed and immutable, so a body - that hashes to the requested sha256 is correct however old it is. An - unclassified path also lands here, but dispatch() 404s it before any - staleness question arises. - """ - endpoint = endpoint_registry.endpoint_for_request_path(path) - if endpoint is None or endpoint.immutable_content: - return None - return endpoint.max_staleness_seconds - - -def staleness_budget_header_value(path: str) -> str | None: - """What X-Prism-Staleness-Budget-Seconds says for one path, or None to omit. - - Every classified route publishes a budget, including the immutable one: - silence there would be indistinguishable from a route that simply forgot, - and "unbounded" is the actual contract -- a content-addressed body is - correct at any age. A path that is not a route at all publishes nothing, - because there is no route whose budget it could be. - """ - endpoint = endpoint_registry.endpoint_for_request_path(path) - if endpoint is None: - return None - if endpoint.immutable_content: - return UNBOUNDED_STALENESS_BUDGET - if endpoint.max_staleness_seconds is None: - return None - return _format_budget(endpoint.max_staleness_seconds) - - -# The refusal body, built once so the handler can recognise it by identity. -# -# Reported under ``stale_read_model`` because ``public_api`` owns the public -# error enum and that is the entry whose meaning this is: no answer for this -# route exists inside the age it publishes. It is the same documented wire code -# (``upstream_unavailable``) the replica refusal already resolves to -- from a -# client's side the conditions are one condition -- and inventing an -# undocumented code instead would silently ship a 503 that says "internal -# error", because error_payload() rewrites codes it does not know. -DATABASE_UNAVAILABLE_PAYLOAD = public_api.error_payload( - "stale_read_model", - "public read service cannot reach the database and has no cached response " - "for this request inside its staleness budget", -) - - -def database_unavailable_response() -> tuple[int, dict[str, object]]: - """The 503 a database-backed route owes while the database is unreachable. - - Shaped as a ``compute`` for ``PublicResponseCache.get_or_compute`` because - that is how the warm/cold distinction gets made without a second cache API: - the cache answers a warm key from its entry and never calls this, and on a - cold, bypassed or expired key it calls this and stores nothing, because the - cache only stores 2xx. So the same call expresses "serve what you have" and - "refuse, and do not go and ask". - """ - - return 503, DATABASE_UNAVAILABLE_PAYLOAD - - -def staleness_error_payload(*, budget_seconds: float, age_seconds: int) -> dict[str, object]: - """A refusal in the existing public error schema, naming both numbers. - - Uses public_api.error_payload so the body is prism.dashboard.error.v1 like - every other public error; a bespoke schema here would make the one response - a client most needs to parse the one it cannot. - """ - return public_api.error_payload( - "stale_read_model", - ( - "cached response age " - f"{age_seconds}s exceeds the staleness budget of " - f"{budget_seconds:g}s for this endpoint" - ), - ) - - -class PublicReadService: - """One public read tier instance: what to read through, and what to remember. - - Holds the collaborators a handler needs -- the ``public_api`` host object, - the shared response cache, this process's counters, and the background - readiness probe -- so that binding a socket is a separate, trivial step. - Tests construct one of these around a fake coordinator and get the real - request path without a database, an RPC node, or a signal handler. - """ - - def __init__( - self, - coordinator: Any, - *, - response_cache: public_api.PublicResponseCache | None = None, - metrics: ServiceMetrics | None = None, - readiness: ReadinessProbe | None = None, - replica: ReplicaProbe | None = None, - ) -> None: - self.coordinator = coordinator - self.cache = ( - public_api.PublicResponseCache() if response_cache is None else response_cache - ) - self.metrics = ServiceMetrics() if metrics is None else metrics - self.readiness = readiness - # None when PRISM_PUBLIC_REPLICA_MODE=off: no replica contract is - # enforced and no replica facts are published, because there are none. - self.replica = replica - - -class BoundedPublicHTTPServer(ThreadingHTTPServer): - """Admit a fixed number of connections without a waiting worker queue.""" - - daemon_threads = True - - def __init__(self, address, handler, *, max_connections=64, timeout_seconds=10.0): - if not 1 <= max_connections <= 1024: - raise PublicReadConfigurationError("PRISM_PUBLIC_HTTP_MAX_CONNECTIONS must be 1-1024") - if not math.isfinite(timeout_seconds) or timeout_seconds <= 0: - raise PublicReadConfigurationError("PRISM_PUBLIC_HTTP_TIMEOUT_SECONDS must be finite and positive") - self.connection_timeout = timeout_seconds - self._connection_slots = threading.BoundedSemaphore(max_connections) - super().__init__(address, handler) - - def process_request(self, request, client_address): - if not self._connection_slots.acquire(blocking=False): - # Do not block the accept loop writing a refusal to an idle peer. - self.shutdown_request(request) - return - try: - super().process_request(request, client_address) - except BaseException: - self._connection_slots.release() - raise - - def process_request_thread(self, request, client_address): - try: - super().process_request_thread(request, client_address) - finally: - self._connection_slots.release() - - -def make_handler(service: PublicReadService) -> type[BaseHTTPRequestHandler]: - """Build the request handler for one public read service instance. - - Deliberately not ``AuditHttpFacade``: that class's cancel-startup and - bounded-serve-exit machinery encodes shutdown races specific to starting and - stopping a listener inside the coordinator's long-lived process. This one - binds, serves, and exits on a signal, so it needs none of it. - """ - - coordinator = service.coordinator - cache = service.cache - service_metrics = service.metrics - readiness = service.readiness - replica = service.replica - - class PublicReadHandler(BaseHTTPRequestHandler): - server_version = "PrismPublicRead/1" - protocol_version = "HTTP/1.1" - - def setup(self) -> None: - self.connection_timeout = getattr(self.server, "connection_timeout", 10.0) - self.request.settimeout(self.connection_timeout) - super().setup() - - def handle_one_request(self) -> None: - # An idle socket timeout alone allows a slow drip to retain a slot. - # One timer per admitted worker bounds the entire request line and - # header phase, then is retired before application dispatch. - self._headers_expired = threading.Event() - - def expire_headers() -> None: - self._headers_expired.set() - try: - self.connection.shutdown(socket.SHUT_RDWR) - except OSError: - pass - - self._header_timer = threading.Timer(self.connection_timeout, expire_headers) - self._header_timer.daemon = True - self._header_timer.start() - try: - super().handle_one_request() - finally: - self._finish_headers() - self.close_connection = True - - def _finish_headers(self) -> None: - timer = self._header_timer - if timer is not None: - timer.cancel() - timer.join() - # The callback captures this handler. Break that ownership - # cycle so completed connections retire without cyclic GC. - self._header_timer = None - - def parse_request(self) -> bool: - try: - parsed = super().parse_request() - finally: - self._finish_headers() - # The stdlib parser can accept EOF as the end of headers. A timer - # shutdown must not turn an incomplete request into application work. - return parsed and not self._headers_expired.is_set() - - def end_headers(self) -> None: - # One response per connection prevents idle keep-alive clients - # from reserving the bounded worker pool between requests. - self.send_header("Connection", "close") - super().end_headers() - - def do_GET(self) -> None: # noqa: N802 - BaseHTTPRequestHandler API - parsed = urllib.parse.urlparse(self.path) - # Same normalization the coordinator listener applied, so a - # trailing slash resolves to the same route it always did. - path = parsed.path.rstrip("/") or "/" - query = urllib.parse.parse_qs(parsed.query) - service_metrics.record_request() - try: - if path == "/healthz": - self.handle_healthz() - return - if path == "/metrics": - self.handle_metrics() - return - if path == "/public/v1" or path.startswith("/public/v1/"): - self.handle_public(path, query) - return - self.write_json(404, {"error": "unknown endpoint"}) - except public_api.PublicApiError as exc: - self.write_json( - exc.status, - public_api.error_payload(exc.code, exc.message), - headers=self.with_budget(path, public_api.public_error_headers()), - ) - except Exception: # noqa: BLE001 - never leak internals to the public - # Matches the coordinator listener's old behaviour for - # /public/v1: a generic 500 in the public error schema, with no - # exception text. ValueError was folded in there too -- it took - # the same branch -- so it needs no separate arm here. - self.write_json( - 500, - public_api.error_payload("internal_error", "internal server error"), - headers=self.with_budget(path, public_api.public_error_headers()), - ) - - def handle_public(self, path: str, query: dict[str, list[str]]) -> None: - reads_database = path_reads_database(path) - # Before the cache, not after. A replica outage does not make - # cached entries expire, so a post-cache check would keep answering - # 200 from the last good snapshot for as long as traffic kept the - # entry warm -- the precise failure this gate exists to prevent. - if replica is not None and reads_database: - refusal = replica.freshness_error() - if refusal is not None: - code, message = refusal - service_metrics.record_replica_refusal() - headers = self.with_replica_lag( - public_api.public_error_headers() - ) - self.write_json( - 503, - public_api.error_payload(code, message), - headers=self.with_budget(path, headers), - ) - return - # Also before the cache, and before any origin computation: while - # readiness says the database is unreachable, a database-backed - # route may still answer from what it already has, but it may not - # go and ask. Handing the cache a non-cacheable 503 instead of - # dispatch() is exactly that instruction -- a warm key returns its - # entry and never reaches this, a cold, bypassed or expired one - # gets the refusal and stores nothing. - degraded = ( - readiness is not None - and reads_database - and readiness.snapshot()[0] != 200 - ) - cache_policy = public_api.public_cache_policy(path) - budget_seconds = staleness_budget_for_path(path) - # The stale-while-revalidate window the policy advertises to CDNs - # also lets the in-process cache serve an expired entry while one - # background refresh recomputes it -- but the budget check below - # is unconditional, so the window is clamped to what the route's - # staleness budget leaves after the TTL. Without the clamp, a - # route whose ttl+swr exceeds its budget (the 5s routes: 5+30 - # against a 15s budget) would turn ages the blocking path used to - # recompute into 503 refusals. While the database is unreachable - # the window is withdrawn entirely: the outage contract serves - # only unexpired entries, and its refusal compute must not be - # deferred to a background thread nobody answers. - ttl_seconds = cache_policy.ttl_seconds - swr_seconds = cache_policy.stale_while_revalidate_seconds - if budget_seconds is not None: - ttl_seconds = max(0, min(ttl_seconds, int(budget_seconds))) - swr_seconds = max(0, min(swr_seconds, int(budget_seconds) - ttl_seconds)) - if degraded: - swr_seconds = 0 - # Use the same effective policy for origin storage and CDN headers. - # Otherwise downstream stale serving can outlive this route's budget. - cache_policy = public_api.PublicCachePolicy( - ttl_seconds=ttl_seconds, - stale_while_revalidate_seconds=swr_seconds, - immutable=cache_policy.immutable, - ) - status, payload, cache_state, age_seconds = cache.get_or_compute( - key=public_api.public_cache_key(path, query), - ttl_seconds=cache_policy.ttl_seconds, - stale_while_revalidate_seconds=swr_seconds, - compute=( - database_unavailable_response - if degraded - else lambda: bounded_public_dispatch( - coordinator, - path, - query, - occupies_read_slot=path_occupies_read_slot(path), - ) - ), - ) - service_metrics.record_cache_state(cache_state) - if payload is DATABASE_UNAVAILABLE_PAYLOAD: - # Nothing warm to serve. Recognised by identity rather than - # from this request's own `degraded`, because get_or_compute - # coalesces: a request that arrived while another was computing - # gets that one's result whatever its own view of readiness - # was. Keying on the request would ship this 503 with ordinary - # shared-cache headers to a waiter whose probe had recovered in - # between, and a CDN would then hold the refusal. - service_metrics.record_database_outage_refusal() - headers = self.with_replica_lag(public_api.public_error_headers()) - headers[DATABASE_STATE_HEADER] = DATABASE_STATE_UNAVAILABLE - self.write_json( - status, payload, headers=self.with_budget(path, headers) - ) - return - if isinstance(payload, public_api.UncacheableJsonBody): - # Legacy audit rows can predate canonical artifact storage. - # Their reconstructed JSON remains available for continuity, - # but it must never be cached under an immutable content URL. - headers = self.with_replica_lag(public_api.public_error_headers()) - headers[ARTIFACT_CANONICAL_STATE_HEADER] = payload.reason - self.write_json( - status, - payload.payload, - headers=self.with_budget(path, headers), - ) - return - if budget_seconds is not None and age_seconds > budget_seconds: - # Refuse rather than serve past the budget. A silently stale - # dashboard is worse than an honest one. - service_metrics.record_staleness_refusal() - headers = public_api.public_error_headers() - headers[STALENESS_BUDGET_HEADER] = _format_budget(budget_seconds) - headers["Age"] = str(max(0, age_seconds)) - if degraded: - # Stated, but deliberately not counted as an outage - # refusal: the budget check is unconditional, so this entry - # would have been refused with a healthy database too. The - # outage is why no fresher one replaced it, which is worth - # saying and is not the same claim. - headers[DATABASE_STATE_HEADER] = DATABASE_STATE_UNAVAILABLE - self.write_json( - 503, - staleness_error_payload( - budget_seconds=budget_seconds, - age_seconds=age_seconds, - ), - headers=self.with_replica_lag(headers), - ) - return - headers = public_api.public_cache_headers( - cache_policy, - cache_state=cache_state, - age_seconds=age_seconds, - ) - headers = self.with_replica_lag(headers) - if degraded and cache_state == "HIT": - # A cached body, inside the budget the same response publishes, - # served with the outage stated rather than implied. The - # shared-cache headers are deliberately left as they were: the - # CDN TTL is the route's own, which the registry derives at a - # third of the budget or less, so a downstream cache cannot - # carry this body past the bound stated on it. - # - # Conditioned on the hit, not on `degraded` alone: a coalescing - # waiter can receive an origin result computed by a request - # that still saw a healthy database, and labelling that fresh - # body "serving cached response" would be untrue. - service_metrics.record_degraded_response() - headers[DEGRADED_WARNING_HEADER] = DEGRADED_WARNING - headers[DATABASE_STATE_HEADER] = DATABASE_STATE_UNAVAILABLE - self.write_json(status, payload, headers=self.with_budget(path, headers)) - - def with_replica_lag(self, headers: dict[str, str]) -> dict[str, str]: - """Publish observed replay lag beside the enforced heartbeat bound. - - Advisory, and deliberately so: this is the number that answers "how - far behind is the data I just got", which the enforced bound does - not. Omitted rather than guessed when nothing has replayed yet. - """ - - if replica is None: - return headers - lag_seconds = replica.replay_lag_seconds() - if lag_seconds is not None: - headers[REPLICA_LAG_HEADER] = f"{lag_seconds:.3f}" - return headers - - def with_budget(self, path: str, headers: dict[str, str]) -> dict[str, str]: - """Attach the route's budget so callers can see it on every response.""" - - budget = staleness_budget_header_value(path) - if budget is not None: - headers[STALENESS_BUDGET_HEADER] = budget - return headers - - def handle_healthz(self) -> None: - if readiness is None: - payload: dict[str, object] = {"ok": True, "schema": HEALTH_SCHEMA} - status = 200 - else: - status, payload = readiness.snapshot() - if replica is not None: - # The replication facts are why a 503 here says what it says, - # so they travel with it rather than only in /metrics. - payload["replica"] = replica.view() - refusal = replica.freshness_error() - if refusal is not None: - payload["ok"] = False - payload["state"] = "unready" - # Overwrite rather than defer to the readiness probe's - # text: in require mode the replica check *is* the - # readiness probe, so a failed contract shows up there as - # the useless "readiness probe returned false". The - # probe's own exception, when there was one, is in the - # replica block beside this. - payload["error"] = refusal[1] - status = 503 - self.write_json(status, payload) - - def handle_metrics(self) -> None: - ready = True - age_seconds = -1.0 - if readiness is not None: - status, _ = readiness.snapshot() - ready = status == 200 - age_seconds = readiness.age_seconds() - body = service_metrics.render( - ready=ready, - readiness_age_seconds=age_seconds, - replica=None if replica is None else replica.view(), - ).encode() - service_metrics.record_response(200) - try: - self.send_response(200) - self.send_header("Content-Type", "text/plain; version=0.0.4") - self.send_header("Content-Length", str(len(body))) - self.send_header("Cache-Control", "no-store") - self.end_headers() - self.wfile.write(body) - except (BrokenPipeError, ConnectionResetError, TimeoutError): - return - - def write_json( - self, - status: int, - payload: object, - headers: dict[str, str] | None = None, - ) -> None: - # Byte-identical framing to the coordinator listener's write_json: - # sorted keys and a trailing newline, so the extracted bodies match - # the pre-extraction ones exactly. - if isinstance(payload, public_api.RawJsonBody): - # Content-addressed bytes (#154): any re-serialization (key - # order, separators, trailing newline) would break - # sha256(response body) == the advertised artifact hash. - # /public/v1/artifacts/{sha256} moved here, so this branch - # moved with it. - body = payload.body - else: - body = json.dumps(payload, sort_keys=True).encode() + b"\n" - service_metrics.record_response(status) - try: - self.send_response(status) - self.send_header("Content-Type", "application/json") - self.send_header("Content-Length", str(len(body))) - for key, value in (headers or {}).items(): - self.send_header(key, value) - self.end_headers() - self.wfile.write(body) - except (BrokenPipeError, ConnectionResetError, TimeoutError): - # A client with a short timeout hung up before the response was - # written; nothing to salvage. - return - - def log_message(self, format: str, *args: object) -> None: # noqa: A002 - # BaseHTTPRequestHandler logs every request to stderr by default. - # This tier's request volume is the whole reason it exists, so that - # default would be the loudest thing in the container's logs. - return - - return PublicReadHandler - - -def _metric_number(value: object) -> str: - """Render a possibly-absent replica number as a gauge value. - - -1 stands in for "not known", matching how the readiness probe age already - reports its own pre-first-probe state. - """ - - number = _coerce_float(value) - return "-1" if number is None else f"{number:.3f}" - - -def _format_budget(budget_seconds: float) -> str: - """Render a budget without a trailing ``.0`` on whole-second values.""" - - if float(budget_seconds).is_integer(): - return str(int(budget_seconds)) - return f"{budget_seconds:g}" - - -def require_public_stratum_url(environ: dict[str, str] | None = None) -> str: - """Refuse to start unless the advertised Stratum endpoint is stated. - - public_api.mining_configuration composes the advertised endpoint from - getattr(coordinator, "bind"/"port") with an environment fallback of - 127.0.0.1:3340. This process exposes neither attribute, on purpose, so - without PRISM_PUBLIC_STRATUM_URL the public mining-configuration endpoint - would confidently tell every miner to point its hashrate at the loopback - address of whatever machine read the page. - """ - source = os.environ if environ is None else environ - value = (source.get("PRISM_PUBLIC_STRATUM_URL") or "").strip() - if not value: - raise PublicReadConfigurationError( - "PRISM_PUBLIC_STRATUM_URL is required: the public read service runs " - "no Stratum listener, so it cannot infer the pool's endpoint, and " - "the fallback would advertise 127.0.0.1 to miners" - ) - validate_advertised_stratum_url(source["PRISM_PUBLIC_STRATUM_URL"], "PRISM_PUBLIC_STRATUM_URL") - highdiff_url = source.get("PRISM_PUBLIC_STRATUM_HIGHDIFF_URL") - if public_api.optional_tcp_port(source.get("PRISM_STRATUM_HIGHDIFF_PORT")) is not None and highdiff_url: - validate_advertised_stratum_url(highdiff_url, "PRISM_PUBLIC_STRATUM_HIGHDIFF_URL") - return value - - -def validate_advertised_stratum_url(value: str, setting: str) -> None: - try: - parsed = urllib.parse.urlsplit(value) - hostname = parsed.hostname or "" - if ":" in hostname: - ipaddress.IPv6Address(hostname) - valid_host = True - else: - ascii_host = hostname.encode("idna").decode("ascii").rstrip(".") - valid_host = bool(ascii_host) and len(ascii_host) <= 253 and all( - re.fullmatch(r"[A-Za-z0-9](?:[A-Za-z0-9-]{0,61}[A-Za-z0-9])?", label) - for label in ascii_host.split(".") - ) - valid = ( - valid_host and parsed.scheme in {"stratum+tcp", "stratum+ssl"} - and parsed.hostname and parsed.port is not None and 1 <= parsed.port <= 65535 - and parsed.username is None and parsed.password is None - and not parsed.path and not parsed.query and not parsed.fragment - and not any(c.isspace() or ord(c) < 32 for c in value) - ) - except (ValueError, UnicodeError): - valid = False - if not valid: - raise PublicReadConfigurationError( - f"{setting} must be a stratum+tcp or stratum+ssl URL " - "with a hostname and port (1-65535), without credentials, path, query or fragment" - ) - - -def build_ledger_from_env(environ: dict[str, str] | None = None) -> PsqlShareLedger: - """Construct the read-only Postgres ledger this service reads through. - - Two refusals rather than defaults: - - * The in-memory ledger is rejected. It implements none of the ``dashboard_*`` - read models, so public_api's duck-typed fallbacks would silently take the - writer-lock paths (current_owed_balances, recipient_payout_history, - ledger.all_shares) that this extraction exists to get off the public - surface -- a memory-ledger public tier would look fine and reintroduce the - exact contention it removed. - * No PRISM_LEDGER_WRITER_* input is read at all. This process must never - acquire a writer lease; leaving the writer identity at its default and - touching only read models keeps that structural rather than conventional. - """ - source = os.environ if environ is None else environ - if _env_flag(source.get("PRISM_ALLOW_MEMORY_LEDGER")): - raise PublicReadConfigurationError( - "PRISM_ALLOW_MEMORY_LEDGER is not supported by the public read " - "service: the in-memory ledger has no dashboard_* read models, so " - "public_api would fall back to writer-lock reads " - "(current_owed_balances, recipient_payout_history) on public routes" - ) - psql_command = source.get("PRISM_POSTGRES_PSQL_COMMAND", "") - database_url = source.get("PRISM_DATABASE_URL", "") - if not database_url and source.get("PRISM_PUBLIC_DATABASE_HOST"): - # Compose supplies separate literal components; an explicit URL wins. - quote = lambda value: urllib.parse.quote(value, safe="") - user = quote(source.get("PRISM_POSTGRES_USER", "qbit")) - password = quote(source.get("PRISM_POSTGRES_PASSWORD", "change-this")) - database = quote(source.get("PRISM_POSTGRES_DB", "qbit")) - host = source["PRISM_PUBLIC_DATABASE_HOST"] - database_url = f"postgresql://{user}:{password}@{host}:5432/{database}" - if not psql_command and database_url: - psql_command = f"psql {shlex.quote(database_url)}" - if not psql_command: - raise PublicReadConfigurationError( - "PRISM_DATABASE_URL or PRISM_POSTGRES_PSQL_COMMAND is required: the " - "public read service serves every route from Postgres" - ) - return PsqlShareLedger( - psql_command=psql_command, - database_url=database_url or None, - native_client_mode=env("PRISM_POSTGRES_NATIVE_CLIENT", "auto"), - # Constructing an ordinary ledger claims the single-writer lease and - # gives the object a real writer lock. read_only does neither: no lease - # is acquired, and the writer gate refuses to be taken at all, so a - # public route that reached a fenced read would fail loudly here rather - # than quietly serialize dashboard polls against block landing. - read_only=True, - # Never initialize or repair schema from the read tier. - initialize_schema=False, - postgres_idle_in_transaction_timeout_seconds=env_positive_float( - "PRISM_POSTGRES_IDLE_IN_TRANSACTION_TIMEOUT_SECONDS", - DEFAULT_POSTGRES_IDLE_IN_TRANSACTION_TIMEOUT_SECONDS, - ), - postgres_tcp_keepalives_idle_seconds=env_positive_int( - "PRISM_POSTGRES_TCP_KEEPALIVES_IDLE_SECONDS", - DEFAULT_POSTGRES_TCP_KEEPALIVES_IDLE_SECONDS, - ), - postgres_tcp_keepalives_interval_seconds=env_positive_int( - "PRISM_POSTGRES_TCP_KEEPALIVES_INTERVAL_SECONDS", - DEFAULT_POSTGRES_TCP_KEEPALIVES_INTERVAL_SECONDS, - ), - postgres_tcp_keepalives_count=env_positive_int( - "PRISM_POSTGRES_TCP_KEEPALIVES_COUNT", - DEFAULT_POSTGRES_TCP_KEEPALIVES_COUNT, - ), - read_concurrency=env_positive_int("PRISM_POSTGRES_READ_CONCURRENCY", 4), - # Read by the ledger itself, and the second cache the miner page's - # staleness budget accounts for; both processes construct one. - reward_window_cache_seconds=env_nonnegative_float( - "PRISM_PUBLIC_REWARD_WINDOW_CACHE_SECONDS", - 30.0, - ), - audit_artifact_store=build_audit_artifact_store(source), - pool_application_name="prism-public-read", - ) - - -def build_audit_artifact_store( - environ: dict[str, str] | None = None, -) -> AuditArtifactStore | None: - """Open the audit artifact root read-only, for artifact and settlement reads. - - /public/v1/artifacts/{sha256} reads immutable compressed canonical bundles - first. A clean miss can use a labelled, non-cacheable legacy body; damage - fails closed. The compose service mounts that volume :ro precisely so this - process cannot write to it, which also means it cannot create the publication - lock file a normal store opens O_RDWR at construction -- hence read_only. - """ - source = os.environ if environ is None else environ - audit_dir = (source.get("PRISM_AUDIT_DIR") or "").strip() - if not audit_dir: - return None - root = Path(audit_dir) - return AuditArtifactStore( - AuditArtifactConfig( - root=root, - evidence_path=root / "prism-live-stratum-evidence.json", - ), - read_only=True, - ) - - -def build_rpc_from_env(environ: dict[str, str] | None = None) -> JsonRpc: - """This service's own qbit RPC client, built the way the coordinator builds its own. - - Read-only calls only: public_api reaches RPC for getblockchaininfo, - getblocktemplate and getnetworkinfo, all of which describe the node rather - than change it. - """ - del environ # env() reads os.environ directly, as every other PRISM entry point does. - return JsonRpc( - host=env("QBIT_RPC_HOST"), - port=env_int("QBIT_RPC_PORT", 18452), - user=env("QBIT_RPC_USER"), - password=env("QBIT_RPC_PASSWORD"), - ) - - -def _env_flag(raw: str | None) -> bool: - if raw is None: - return False - return raw.strip().lower() in {"1", "true", "yes", "on"} - - -def resolve_replica_mode(environ: dict[str, str] | None = None) -> str: - """Read PRISM_PUBLIC_REPLICA_MODE, refusing anything it does not mean. - - An unrecognised value is a configuration error, not a reason to pick a - default: the two modes differ in whether this process will serve from the - coordinator's primary, so guessing gets that wrong silently in one - direction or the other. - """ - source = os.environ if environ is None else environ - raw = (source.get("PRISM_PUBLIC_REPLICA_MODE") or "").strip().lower() - if not raw: - return REPLICA_MODE_OFF - if raw not in REPLICA_MODES: - raise PublicReadConfigurationError( - "PRISM_PUBLIC_REPLICA_MODE must be one of " - f"{', '.join(REPLICA_MODES)} (got {raw!r})" - ) - return raw - - -def build_replica_probe( - ledger: Any, - environ: dict[str, str] | None = None, -) -> ReplicaProbe | None: - """The replica contract, or None when this deployment has no standby yet.""" - - if resolve_replica_mode(environ) != REPLICA_MODE_REQUIRE: - return None - return ReplicaProbe( - ledger.read_replica_status, - max_lag_seconds=env_positive_float( - "PRISM_PUBLIC_REPLICA_MAX_LAG_SECONDS", - DEFAULT_REPLICA_MAX_LAG_SECONDS, - ), - ) - - -def build_service(environ: dict[str, str] | None = None) -> tuple[ - PublicReadCoordinator, - ReadinessProbe, - ServiceMetrics, - ReplicaProbe | None, -]: - """Validate configuration and assemble the process's collaborators. - - Every refusal happens here, before a socket is bound, so a misconfigured - service fails to start rather than starting and serving wrong facts. - """ - require_public_stratum_url(environ) - ledger = build_ledger_from_env(environ) - rpc = build_rpc_from_env(environ) - coordinator = PublicReadCoordinator(ledger=ledger, rpc=rpc) - replica = build_replica_probe(ledger, environ) - readiness = ReadinessProbe( - # In require mode the replication probe *is* the readiness probe: it - # round-trips to the same database and additionally proves the server - # is the standby this tier is supposed to be reading. Running both - # would be two round trips to answer one question. - ledger.dashboard_readiness_probe if replica is None else replica.check, - interval_seconds=env_positive_float( - "PRISM_PUBLIC_READINESS_PROBE_INTERVAL_SECONDS", - DEFAULT_READINESS_PROBE_INTERVAL_SECONDS, - ), - ) - return coordinator, readiness, ServiceMetrics(), replica - - -def main(argv: list[str] | None = None) -> int: - del argv - bind = os.environ.get("PRISM_PUBLIC_API_BIND") or DEFAULT_PUBLIC_API_BIND - port = env_int("PRISM_PUBLIC_API_PORT", DEFAULT_PUBLIC_API_PORT) - try: - max_connections = env_positive_int("PRISM_PUBLIC_HTTP_MAX_CONNECTIONS", 64) - connection_timeout = env_positive_float("PRISM_PUBLIC_HTTP_TIMEOUT_SECONDS", 10.0) - coordinator, readiness, metrics, replica = build_service() - handler = make_handler(PublicReadService( - coordinator, metrics=metrics, readiness=readiness, replica=replica, - )) - server = BoundedPublicHTTPServer( - (bind, port), handler, max_connections=max_connections, - timeout_seconds=connection_timeout, - ) - except (PublicReadConfigurationError, ValueError) as exc: - print(f"prism-public-read: {exc}", file=sys.stderr, flush=True) - return 2 - - readiness.start() - - def request_shutdown(signum: int, _frame: object) -> None: - # shutdown() blocks until serve_forever() returns and must not be called - # from the serving thread, so hand it to a helper thread. - del signum - threading.Thread(target=server.shutdown, daemon=True).start() - - signal.signal(signal.SIGTERM, request_shutdown) - signal.signal(signal.SIGINT, request_shutdown) - - backing = ( - "any server the DSN names (replica mode off)" - if replica is None - else ( - "a standby with a replication stream no more than " - f"{replica.max_lag_seconds:g}s silent" - ) - ) - print( - f"prism-public-read: serving /public/v1 on {bind}:{port}, reading from " - f"{backing}", - file=sys.stderr, - flush=True, - ) - try: - server.serve_forever() - finally: - readiness.stop() - server.server_close() - return 0 - - -if __name__ == "__main__": - raise SystemExit(main()) diff --git a/lab/prism/recover_pending_blocks.py b/lab/prism/recover_pending_blocks.py deleted file mode 100644 index f9a214ab7..000000000 --- a/lab/prism/recover_pending_blocks.py +++ /dev/null @@ -1,476 +0,0 @@ -"""Recover allowlisted active-chain candidates using normal PRISM accounting. - -Invoke in a fresh managed coordinator container with the normal coordinator -stopped. Without --apply this only reads metadata and qbitd chain identity. -Apply holds the existing writer lease and heartbeat, loads one payload at a -time, and never starts serve(), Stratum, ordinary replay, or job prewarming. -""" - -from __future__ import annotations - -import argparse -import hmac -import json -import signal -import threading -from collections.abc import Mapping -from dataclasses import dataclass, replace -from typing import Any, Callable - -from lab.auxpow.stratum_codec import header_hash_hex -from lab.prism.block_candidates import _BlockCandidateNodeSubmission -from lab.prism.candidate_spool_view import iter_string_text_chunks -from lab.prism.candidate_store import CANDIDATE_SCHEMA_CAPABILITY, candidate_schema_refusal -from lab.prism.coordinator_config import CoordinatorConfig, load_coordinator_config -from lab.prism.prism_coordinator import PrismCoordinator -from lab.prism.recovery_json import cooperative_json -from lab.prism.rpc import JsonRpc -from lab.prism.share_ledger import ( - PsqlShareLedger, - block_candidate_identity_sha256, - canonical_hex, - database_url_from_psql_command, -) - -MAX_RECOVERY_BLOCKS = 32 -SUMMARY_SCHEMA = "qbit.prism.pending-block-recovery.v1" - - -class RecoveryError(RuntimeError): - pass - - -@dataclass(frozen=True) -class RecoveryBlock: - block_hash: str - height: int - parent_hash: str - state: str - - -def refuse_lease_wait(_seconds: float) -> None: - raise RecoveryError( - "exclusive writer lease unavailable; stop the coordinator and wait " - "for its lease release before retrying" - ) - - -class RecoveryCoordinator(PrismCoordinator): - """Use the production assembly without unrelated startup/publication work.""" - - def make_ledger(self, **_kwargs: Any) -> PsqlShareLedger: - ledger = super().make_ledger(lease_retry_sleep=refuse_lease_wait) - if not isinstance(ledger, PsqlShareLedger): - raise RecoveryError("recovery requires the native PostgreSQL ledger") - self.ledger = ledger # Available for cleanup even if __init__ later fails. - return ledger - - def _upgrade_legacy_audit_evidence(self) -> None: - # The normal coordinator owns this unrelated startup migration. - pass - - def _submit_block_candidate_to_node(self, *_args: Any, **_kwargs: Any) -> Any: - # Active-chain accounting must not create a new node offer if a chain - # observation changes during finalization. The normal verifier and - # accounting fences still run; stop and preserve the pending intent. - raise RecoveryError("recovery refuses to submit a block to the node") - - -class RecoveryReader: - """Metadata and single-payload reads on a server-enforced read-only session.""" - - def __init__(self, database_url: str) -> None: - import psycopg - from psycopg.rows import dict_row - - self.connection = psycopg.connect( - database_url, - autocommit=True, - row_factory=dict_row, - connect_timeout=5, - ) - try: - self.connection.execute("SET default_transaction_read_only = on") - self.connection.execute("SET statement_timeout = '30s'") - self.connection.execute("SET lock_timeout = '5s'") - except BaseException: - self.connection.close() - raise - - def close(self) -> None: - self.connection.close() - - def require_candidate_schema(self) -> None: - """Refuse a database this release cannot finalize against (#255). - - The coordinator's ledger applies the same rule when --apply builds - it; checking here lets the read-only plan report an unmigrated - database before any writer lease is requested. - """ - presence = self.connection.execute( - """SELECT to_regclass('qbit_block_candidate_body') IS NOT NULL AS has_body_table, - to_regclass('qbit_prism_schema_capabilities') IS NOT NULL AS has_capabilities""" - ).fetchone() - declared = None - if presence["has_capabilities"]: - capability = self.connection.execute( - """SELECT capability_value FROM qbit_prism_schema_capabilities - WHERE capability = %s""", - (CANDIDATE_SCHEMA_CAPABILITY,), - ).fetchone() - if capability is not None: - declared = int(capability["capability_value"]) - refusal = candidate_schema_refusal(declared, bool(presence["has_body_table"])) - if refusal is not None: - raise RecoveryError(refusal) - - def metadata(self, block_hash: str) -> dict[str, Any] | None: - return self.connection.execute( - """SELECT outbox.block_hash, outbox.state, outbox.candidate_sha256, - pool.block_height, pool.parent_hash, pool.chain_state, - EXISTS (SELECT 1 FROM qbit_pool_audit_bundles audit - WHERE audit.block_hash = outbox.block_hash) AS has_audit - FROM qbit_block_candidate_outbox outbox - LEFT JOIN qbit_pool_blocks pool USING (block_hash) - WHERE outbox.block_hash = %s""", - (block_hash,), - ).fetchone() - - def candidate(self, block_hash: str) -> dict[str, Any]: - """One pending row's payload facts, for either storage version. - - Issue #255: a version-2 row carries no ``candidate`` jsonb. Its - body lives in ``qbit_block_candidate_body`` and is read back in - bounded pages by the ledger (``hydrate_block_candidate_intent``), - never through this read-only session. The columns below name the - version explicitly so a chunked candidate is decoded through that - route rather than mistaken for a missing or corrupt payload. - """ - row = self.connection.execute( - """SELECT outbox.candidate, outbox.candidate_sha256, - outbox.storage_version, outbox.body_id, outbox.replay_header, - body.byte_count, body.chunk_count, body.chunk_bytes, - body.share_count, body.state AS body_state - FROM qbit_block_candidate_outbox outbox - LEFT JOIN qbit_block_candidate_body body ON body.body_id = outbox.body_id - WHERE outbox.block_hash = %s AND outbox.state = 'pending'""", - (block_hash,), - ).fetchone() - if row is None: - raise RecoveryError(f"pending candidate disappeared: {block_hash}") - return row - - -def require_active_block(rpc: Any, block_hash: str) -> tuple[int, str]: - header = rpc.call("getblockheader", [block_hash]) - height = header.get("height") if isinstance(header, dict) else None - if isinstance(height, bool) or not isinstance(height, int) or height < 1: - raise RecoveryError(f"invalid chain header for {block_hash}") - if rpc.call("getblockhash", [height]) != block_hash: - raise RecoveryError(f"candidate is not on the active chain: {block_hash}") - parent = canonical_hex( - str(header.get("previousblockhash", "")), name="parent_hash", expected_bytes=32 - ) - return height, parent - - -def require_completed(row: dict[str, Any] | None, block: RecoveryBlock) -> None: - if not row or not ( - row["state"] == "submitted" - and row["chain_state"] == "confirmed" - and row["block_height"] == block.height - and row["parent_hash"] == block.parent_hash - and row["has_audit"] is True - ): - raise RecoveryError(f"accounting/outbox completion is not proven: {block.block_hash}") - - -def plan_recovery(reader: RecoveryReader, rpc: Any, hashes: list[str]) -> list[RecoveryBlock]: - """Validate the schema and entire allowlist before the first writer lease.""" - hashes = [canonical_hex(value, name="block_hash", expected_bytes=32) for value in hashes] - if not hashes or len(hashes) > MAX_RECOVERY_BLOCKS or len(set(hashes)) != len(hashes): - raise RecoveryError(f"specify 1–{MAX_RECOVERY_BLOCKS} distinct block hashes") - reader.require_candidate_schema() - blocks = [] - for block_hash in hashes: - row = reader.metadata(block_hash) - if not row or row["state"] not in {"pending", "submitted"}: - raise RecoveryError(f"candidate is missing or abandoned: {block_hash}") - height, parent = require_active_block(rpc, block_hash) - block = RecoveryBlock(block_hash, height, parent, row["state"]) - if block.state == "submitted": - require_completed(row, block) - parent_row = reader.metadata(parent) - if parent_row and parent_row["state"] == "pending" and parent not in hashes: - raise RecoveryError(f"pending parent must be included in the allowlist: {parent}") - blocks.append(block) - return sorted(blocks, key=lambda block: block.height) - - -def load_intent( - coordinator: PrismCoordinator, block: RecoveryBlock, row: dict[str, Any] -) -> Any: - """The intent mapping for one pending row, by storage version (#255). - - Version 1 is the legacy whole-jsonb payload the read-only session - already decoded (cooperatively, in this standalone process). Version 2 - is hydrated by the coordinator's ledger in bounded chunk pages into a - spool file, and its decoded facts are exposed through the same mapping - interface. Any other version is refused rather than skipped. - """ - storage_version = int(row.get("storage_version") or 1) - if storage_version == 1: - return row["candidate"] - if storage_version != 2: - raise RecoveryError( - f"unsupported candidate storage version {storage_version}: {block.block_hash}" - ) - if row.get("body_id") is None or row.get("body_state") != "sealed": - raise RecoveryError(f"chunked candidate body is not sealed: {block.block_hash}") - hydrate = getattr(coordinator.ledger, "hydrate_block_candidate_intent", None) - if not callable(hydrate): - raise RecoveryError("the ledger cannot hydrate chunked candidate bodies") - header_row = { - "block_hash": block.block_hash, - "storage_version": 2, - "candidate_sha256": str(row["candidate_sha256"]), - "header": row.get("replay_header") or {}, - "body": { - "body_id": row["body_id"], - "storage_version": 2, - "candidate_sha256": str(row["candidate_sha256"]), - "byte_count": row["byte_count"], - "chunk_count": row["chunk_count"], - "chunk_bytes": row["chunk_bytes"], - "share_count": row["share_count"], - "state": row["body_state"], - }, - } - return hydrate(header_row, cancelled=coordinator.stop_event.is_set) - - -def block_hex_prefix(block_hex: Any, length: int) -> str: - """The first ``length`` characters of a plain or streamed block hex. - - A hydrated version-2 body keeps a large ``block_hex`` as a streamed - view that refuses ``str()``; only the header prefix is needed here. - """ - prefix = "" - for chunk in iter_string_text_chunks(block_hex): - prefix += chunk - if len(prefix) >= length: - break - return prefix[:length] - - -def decode_candidate( - coordinator: PrismCoordinator, block: RecoveryBlock, row: dict[str, Any] -) -> Any: - intent = load_intent(coordinator, block, row) - if not isinstance(intent, Mapping) or intent.get("block_hash_hex") != block.block_hash: - raise RecoveryError("candidate payload does not match its outbox key") - digest = block_candidate_identity_sha256(intent) - if not hmac.compare_digest(digest, str(row["candidate_sha256"])): - raise RecoveryError(f"candidate identity digest mismatch: {block.block_hash}") - if ( - intent.get("expected_height") != block.height - or intent.get("parent_hash") != block.parent_hash - ): - raise RecoveryError(f"candidate does not match its active-chain header: {block.block_hash}") - header = bytes.fromhex(block_hex_prefix(intent["block_hex"], 160)) - if len(header) != 80 or header_hash_hex(header) != block.block_hash: - raise RecoveryError(f"candidate block bytes do not match its hash: {block.block_hash}") - return replace(coordinator.block_candidate_from_intent(intent), durable_replay=True) - - -def recover_block( - coordinator: PrismCoordinator, reader: RecoveryReader, block: RecoveryBlock -) -> None: - # Re-read after acquisition; the earlier plan is advisory. - row = reader.metadata(block.block_hash) - if row and row["state"] == "submitted": - require_completed(row, block) - require_active_block(coordinator.rpc, block.block_hash) - return - candidate = decode_candidate(coordinator, block, reader.candidate(block.block_hash)) - with coordinator._writer_operation("accepted_block_handling"): - if coordinator.stop_event.is_set(): - raise RecoveryError("recovery interrupted") - if require_active_block(coordinator.rpc, block.block_hash) != ( - block.height, - block.parent_hash, - ): - raise RecoveryError("chain identity changed since the recovery plan") - coordinator._require_fresh_ledger_lease_for_external_side_effect("block-recovery") - coordinator._mark_block_candidate_attempted(block.block_hash) - with coordinator._block_landing_ledger_statement_timeout_scope(block.block_hash): - accepted = coordinator.submit_block_candidate( - candidate, - node_submission=_BlockCandidateNodeSubmission(attempted=False), - ) - if not accepted: - raise RecoveryError( - f"normal finalization deferred or rejected {block.block_hash}; outbox retained" - ) - coordinator._finalize_block_candidate( - candidate, - block_hash=block.block_hash, - accepted=True, - error="", - outcome=coordinator._block_candidate_outcome, - ) - require_completed(reader.metadata(block.block_hash), block) - require_active_block(coordinator.rpc, block.block_hash) - - -def recovery_config(config: CoordinatorConfig) -> CoordinatorConfig: - return replace( - config, - ledger=replace( - config.ledger, - initialize_schema=False, - native_client_mode="on", - writer_session_token=None, - ), - stop_after_block=False, - max_blocks=2**31 - 1, - ) - - -def apply_recovery( - config: CoordinatorConfig, - reader: RecoveryReader, - blocks: list[RecoveryBlock], - *, - timeout_seconds: int, - report: Callable[[dict[str, Any]], None], -) -> None: - coordinator = RecoveryCoordinator.__new__(RecoveryCoordinator) - initialized = False - deadline = None - old_handlers = {} - try: - coordinator.__init__(recovery_config(config)) - initialized = True - if coordinator._start_ledger_lease_heartbeat() is None: - raise RecoveryError("the guarded lease heartbeat did not start") - - def interrupt(signum: int, _frame: Any) -> None: - coordinator.request_shutdown(signum) - raise RecoveryError("recovery interrupted; pending work remains replayable") - - for signum in (signal.SIGTERM, signal.SIGINT): - old_handlers[signum] = signal.signal(signum, interrupt) - deadline = threading.Timer( - timeout_seconds, - coordinator._watchdog_hard_exit, - args=("pending-block recovery deadline exceeded",), - ) - deadline.daemon = True - deadline.start() - coordinator.validate_live_chain_identity() - coordinator.validate_live_template_and_fee_policy() - coordinator.prism_payout_policy() - for block in blocks: - report({"event": "recovering", "block_hash": block.block_hash, "height": block.height}) - recover_block(coordinator, reader, block) - report({"event": "completed", "block_hash": block.block_hash, "height": block.height}) - finally: - # Keep the deadline armed through writer drainage. An uncertain write - # must not turn a hung cleanup into an indefinitely held lease. - try: - if initialized: - if not coordinator.shutdown(reason="block_recovery_finished"): - raise RecoveryError("writer drainage/lease release failed") - coordinator.drain_non_writer_components() - else: - ledger = getattr(coordinator, "ledger", None) - if ledger is not None: - ledger.release_writer_lease() - finally: - try: - ledger = getattr(coordinator, "ledger", None) - if ledger is not None: - ledger.close() - finally: - try: - store = coordinator.__dict__.get("_audit_artifact_store") - if store is not None: - store.close() - finally: - if deadline is not None: - deadline.cancel() - deadline.join(timeout=1) - for signum, handler in old_handlers.items(): - signal.signal(signum, handler) - - -def main(argv: list[str] | None = None) -> int: - parser = argparse.ArgumentParser(description=__doc__) - parser.add_argument("--block-hash", action="append", required=True, dest="hashes") - parser.add_argument( - "--apply", - action="store_true", - help="complete accounting; normal coordinator must be stopped", - ) - parser.add_argument( - "--timeout-seconds", - type=int, - default=600, - help="processing and cleanup deadline after construction (default: 600)", - ) - args = parser.parse_args(argv) - if not 1 <= args.timeout_seconds <= 3600: - parser.error("--timeout-seconds must be between 1 and 3600") - reader = None - - def report(event: dict[str, Any]) -> None: - print(json.dumps({"schema": SUMMARY_SCHEMA, **event}, sort_keys=True), flush=True) - - try: - with cooperative_json(): - config = load_coordinator_config() - database_url = config.ledger.database_url or database_url_from_psql_command( - config.ledger.psql_command - ) - if not database_url or config.ledger.allow_memory_ledger: - raise RecoveryError( - "recovery requires PRISM_DATABASE_URL and the native PostgreSQL client" - ) - reader = RecoveryReader(database_url) - rpc = JsonRpc( - host=config.rpc.host, - port=config.rpc.port, - user=config.rpc.user, - password=config.rpc.password, - ) - blocks = plan_recovery(reader, rpc, args.hashes) - report( - { - "event": "plan", - "apply": args.apply, - "blocks": [block.__dict__ for block in blocks], - } - ) - if args.apply: - apply_recovery( - config, reader, blocks, timeout_seconds=args.timeout_seconds, report=report - ) - report({"event": "success", "apply": args.apply}) - return 0 - except Exception as exc: - # Driver errors can embed a credential-bearing DSN or SQL payload. - report( - { - "event": "failed", - "error": str(exc) if isinstance(exc, RecoveryError) else type(exc).__name__, - } - ) - return 1 - finally: - if reader is not None: - reader.close() - - -if __name__ == "__main__": - raise SystemExit(main()) diff --git a/lab/prism/recovery_json.py b/lab/prism/recovery_json.py deleted file mode 100644 index f5886d3f5..000000000 --- a/lab/prism/recovery_json.py +++ /dev/null @@ -1,38 +0,0 @@ -"""Cooperative stdlib JSON for the standalone recovery process only. - -Python's fallback codecs yield the interpreter between records. They trade -throughput for heartbeat responsiveness; they are not a byte/memory bound. -Never install these process-global settings in an existing coordinator. -""" - -from __future__ import annotations - -import json -import json.decoder -import json.encoder -import json.scanner -from contextlib import contextmanager -from typing import Iterator - - -@contextmanager -def cooperative_json() -> Iterator[None]: - previous = ( - json.scanner.make_scanner, - json.decoder.scanstring, - json.encoder.c_make_encoder, - json._default_decoder, - ) - try: - json.scanner.make_scanner = json.scanner.py_make_scanner - json.decoder.scanstring = json.decoder.py_scanstring - json.encoder.c_make_encoder = None - json._default_decoder = json.JSONDecoder() - yield - finally: - ( - json.scanner.make_scanner, - json.decoder.scanstring, - json.encoder.c_make_encoder, - json._default_decoder, - ) = previous diff --git a/lab/prism/reorg_reconciler.py b/lab/prism/reorg_reconciler.py deleted file mode 100644 index f521776d7..000000000 --- a/lab/prism/reorg_reconciler.py +++ /dev/null @@ -1,1366 +0,0 @@ -"""PRISM active-chain reconciliation and payout-publication ownership. - -The service owns same-tip flight admission, the per-tip trusted memo with its -lookup counters, the bounded reconcile-prefetch executor, and the core -reconcile state machine. The coordinator retains its serialized cross-owner -adapter (writer admission, payout-balance mutation lock, landed-preview -fail-closed check) and thin public/private delegates; flight coalescing runs -*before* that adapter so same-tip followers never queue behind writer -admission. Memo, lookup, and outcome state is guarded by the coordinator's -control-plane lock supplied through ``ReorgPorts.state_lock`` so tip -observation can evict memo entries atomically with its own state updates. -""" - -from __future__ import annotations - -from collections import OrderedDict -from concurrent.futures import Future, ThreadPoolExecutor -from contextlib import AbstractContextManager, contextmanager -from dataclasses import dataclass -import threading -import time -from typing import Any, Callable, Iterator, Mapping - -from lab.prism.accepted_preview_telemetry import ( - AcceptedPreviewTelemetry, - PRISM_REORG_RECONCILE_CALLERS, - RECONCILE_CALLER_JOB_BUILD, - RECONCILE_CALLER_LANDING, - RECONCILE_CALLER_OTHER, - RECONCILE_CALLER_POST_CONFIRM, - RECONCILE_CALLER_TIP_REFRESH, - RECONCILE_STEP_ADMISSION_WAIT, - RECONCILE_STEP_CANDIDATE_PREPARE, - RECONCILE_STEP_CHAIN_PROBE, - RECONCILE_STEP_MUTATIONS, - RECONCILE_STEP_PUBLISH, - RECONCILE_STEP_WATCH_QUERY, -) -from lab.prism.coordinator_config import ( - DEFAULT_PRISM_REORG_RECONCILE_CACHE_SECONDS, - TESTNET_QBIT_CHAINS, -) -from lab.prism.payout_state import PayoutStateCandidate, TemplateRefreshSuperseded - - -# Bounded number of tips whose trusted reconcile outcome is memoized. Reorg -# churn moves between a handful of competing tips; eight entries cover deep -# flip-flop churn while keeping eviction scans trivial. -PRISM_REORG_RECONCILE_MEMO_MAX_TIPS = 8 -# How long an unflagged same-tip caller waits for an in-flight reconcile pass -# before falling back to its own serialized pass. -DEFAULT_PRISM_RECONCILE_FLIGHT_WAIT_SECONDS = 30.0 -# Reconcile demand accounting: which caller asked, and what satisfied it. -# "overlap" is a tip-refresh join that reused the pass prefetched alongside -# its template fetch; "serial" is a full serialized pass of the caller's own. -PRISM_REORG_RECONCILE_LOOKUP_PATHS = ("tip_refresh", "job_build") -PRISM_REORG_RECONCILE_LOOKUP_SOURCES = ("memo_hit", "overlap", "serial") -# Bounded wait when joining the prefetched reconcile pass. Long enough for a -# healthy pass (DB round trips plus a few RPCs), far shorter than the -# publication-progress failure budget, so a stuck pass surfaces as the normal -# blocked-retry path instead of parking the poll loop's liveness heartbeat. -PRISM_RECONCILE_PREFETCH_JOIN_TIMEOUT_SECONDS = 20.0 -# Hard ceiling for the operator override: the join must stay comfortably -# below the template-refresh failure budget or the bound stops bounding. -PRISM_RECONCILE_PREFETCH_JOIN_TIMEOUT_CEILING_SECONDS = 60.0 -# How deeply a pool block must be buried before the stranded-prepared sweep -# will reject it. -# -# A prepared row is owned by the live submit/replay path while its outbox -# entry exists; the sweep exists only for rows that lost that owner, and it -# cannot tell the two apart from the ledger alone. Depth is the proxy. 100 -# blocks is far past any legitimate in-flight finalization — a landing that -# is still retrying, an ancestor replay working its way forward, a -# confirmation racing its own audit publication all resolve within a handful -# of blocks — while still healing a genuinely stranded row within roughly -# two hours of block time rather than leaving its payout entries, carry -# forward, and CTV fanout artifacts pinned immature indefinitely. -STRANDED_PREPARED_REJECT_MIN_DEPTH = 100 -# One pass's bound on the sweep. Each returned row costs a getblockhash and -# possibly a fenced ledger mutation, and the pass already walks the reorg -# watch set; the remainder is picked up by the next pass. -STRANDED_PREPARED_SWEEP_LIMIT = 64 - - -def _no_reconcile_progress(phase: str) -> None: - """Discard reconcile progress stamps for ports built without a recorder.""" - return None - - -class _ReconcileFlight: - """One in-flight reconcile pass shared by concurrent same-tip callers.""" - - __slots__ = ("event", "summary", "exception") - - def __init__(self) -> None: - self.event = threading.Event() - self.summary: dict[str, object] | None = None - self.exception: BaseException | None = None - - -@dataclass(frozen=True) -class ReorgPorts: - """Dynamic infrastructure required by active-chain reconciliation. - - Callables that reach documented coordinator monkeypatch seams - (``reconcile_with_admission``, ``reconcile_serialized``, ``ensure_tip``, - ``chain_view_untrusted``) resolve the coordinator method at call time so - per-instance patches intercept every internal path. - """ - - rpc_call: Callable[..., object] - ledger: Callable[[], Any] - ensure_job_cache_state: Callable[[], None] - state_lock: Callable[[], AbstractContextManager[object]] - source_tip: Callable[[], str | None] - reserve_external_tip: Callable[[str], None] - max_supersession_retries: Callable[[], int] - prepare_lock: Callable[[], AbstractContextManager[object]] - capture_source: Callable[[], tuple[int, int, str | None, str, float]] - prepared_candidate: Callable[..., PayoutStateCandidate] - captured_publication_required: Callable[ - [tuple[int, int, str | None, str, float]], - bool, - ] - block_publication: Callable[..., None] - publication_guard: Callable[[], AbstractContextManager[object]] - publish_candidate: Callable[[PayoutStateCandidate], int | None] - observe_preparation: Callable[[float], None] - chain_view_untrusted: Callable[[], bool] - reorg_proof_snapshot: Callable[[], tuple[str | None, int]] - flight_wait_seconds: Callable[[], float] - prefetch_join_timeout_seconds: Callable[[], float] - reconcile_with_admission: Callable[..., Mapping[str, object]] - reconcile_serialized: Callable[..., dict[str, object]] - ensure_tip: Callable[[str], bool] - # Liveness stamp for a reconcile pass running inline on a - # watchdog-monitored thread. An accepted-block landing calls - # ``ensure_tip`` on the block-work thread, and one pass is an unbounded - # chain walk -- a getblockhash plus up to two ledger statements per - # watched block -- so bracketing the call from outside would still leave - # a silence proportional to the number of watched blocks. The coordinator - # supplies its block-work phase recorder, which no-ops on every other - # thread; ports built without one get a sink so background reconciliation - # and focused tests are unchanged. - record_progress: Callable[[str], None] = _no_reconcile_progress - # Issue #224 attribution owner: the coordinator supplies - # ``ensure_accepted_preview_telemetry(self)`` so the reconcile pass, - # step and caller families land in the one owner the renderer reads. - # Ports built without one get a private owner so focused tests observe - # the same families; nothing recorded here changes flight, memo, - # admission, lock or publication behavior. - accepted_preview_telemetry: ( - Callable[[], AcceptedPreviewTelemetry] | None - ) = None - - -@dataclass(frozen=True) -class ReorgState: - """Copied bookkeeping/last-outcome snapshot for observers.""" - - inactive_block_count: int - reactivated_block_count: int - reconcile_skip_count: int - reconcile_error_count: int - matured_payout_count: int - last_tip_hash: str | None - last_trusted: bool - last_monotonic: float | None - - -def qbit_chain_view_untrusted( - rpc_call: Callable[..., object], - chain: str, -) -> bool: - """Return whether the node cannot provide a coherent validated tip.""" - - blockchain_info = rpc_call("getblockchaininfo") - if not isinstance(blockchain_info, dict): - raise RuntimeError("getblockchaininfo returned non-object") - public_chain = chain.lower() in { - "main", - "mainnet", - *TESTNET_QBIT_CHAINS, - } - if ( - blockchain_info.get("initialblockdownload") is not False - if public_chain - else bool(blockchain_info.get("initialblockdownload")) - ): - return True - blocks_raw = blockchain_info.get("blocks") - headers_raw = blockchain_info.get("headers") - if public_chain and (blocks_raw is None or headers_raw is None): - return True - if blocks_raw is not None and headers_raw is not None: - try: - blocks = int(blocks_raw) - headers = int(headers_raw) - if blocks < 0 or headers < 0 or headers != blocks: - return True - except (TypeError, ValueError) as exc: - raise RuntimeError("getblockchaininfo blocks/headers are not integers") from exc - return False - - -class ReorgReconcilerService: - """Own flights, trusted memo/lookup state, prefetch, and the core pass.""" - - def __init__( - self, - ports: ReorgPorts, - *, - enabled: bool = True, - cache_seconds: float = DEFAULT_PRISM_REORG_RECONCILE_CACHE_SECONDS, - inactive_block_count: int = 0, - reactivated_block_count: int = 0, - reconcile_skip_count: int = 0, - reconcile_error_count: int = 0, - matured_payout_count: int = 0, - last_tip_hash: str | None = None, - last_trusted: bool = False, - last_monotonic: float | None = None, - ) -> None: - self._ports = ports - self.enabled = bool(enabled) - self.cache_seconds = cache_seconds - self.inactive_block_count = int(inactive_block_count) - self.reactivated_block_count = int(reactivated_block_count) - self.reconcile_skip_count = int(reconcile_skip_count) - self.reconcile_error_count = int(reconcile_error_count) - self.matured_payout_count = int(matured_payout_count) - self.last_tip_hash = last_tip_hash - self.last_trusted = bool(last_trusted) - self.last_monotonic = last_monotonic - self._reconcile_flight_lock = threading.Lock() - # In-flight reconcile passes keyed by tip hash. Unflagged callers - # for a tip already being reconciled await that pass's summary - # instead of queueing a redundant serialized pass of their own. - self._reconcile_flights: dict[str, _ReconcileFlight] = {} - # Monotonic completion time of the last trusted reconcile pass, - # per tip hash, guarded by the coordinator state lock. Entries are - # unarmed only by an untrusted outcome for their own tip (or a - # reconcile error, which clears the whole map); see note_outcome. - self._reorg_reconcile_trusted_memo: OrderedDict[str, float] = ( - OrderedDict() - ) - self._reconcile_prefetch_executor_lock = threading.Lock() - # Single-worker lane that overlaps the tip-refresh reconcile pass - # with the template fetch. Only the refresh singleflight owner - # submits, so at most one prefetched pass is in flight; same-tip - # followers still coalesce through _reconcile_flights. - self._reconcile_prefetch_executor: ThreadPoolExecutor | None = None - self._reconcile_prefetch_executor_shutdown = False - # At most one outstanding prefetch, keyed by tip, guarded by - # _reconcile_prefetch_executor_lock. Failed refresh attempts - # (for example a template-RPC outage) reuse it instead of - # queueing another serialized pass per retry. - self._reconcile_prefetch_pending: ( - tuple[str, Future[bool], bool] | None - ) = None - # Reconcile demand by (caller path, satisfying source), guarded by - # the coordinator state lock. - self.reorg_reconcile_lookup_counts = { - (path, source): 0 - for path in PRISM_REORG_RECONCILE_LOOKUP_PATHS - for source in PRISM_REORG_RECONCILE_LOOKUP_SOURCES - } - # Issue #224 caller attribution. Entry points stamp the bounded - # caller on the calling thread (the landing's in-lock ensure_tip, - # the job-build ensure_current, the tip-refresh prefetch lane) and - # the core pass reads it there, so the coordinator's monkeypatch - # seams (reconcile_with_admission / reconcile_serialized / - # ensure_tip) keep their exact signatures. The same slot carries - # the stamp ``admission_wait`` is measured from. - self._reconcile_attribution = threading.local() - self._fallback_telemetry_lock = threading.Lock() - self._fallback_telemetry: AcceptedPreviewTelemetry | None = None - - @property - def accepted_preview_telemetry(self) -> AcceptedPreviewTelemetry: - """The shared #224 owner, or a private one for ports without it.""" - port = self._ports.accepted_preview_telemetry - if port is not None: - return port() - telemetry = self._fallback_telemetry - if telemetry is None: - with self._fallback_telemetry_lock: - telemetry = self._fallback_telemetry - if telemetry is None: - telemetry = AcceptedPreviewTelemetry() - self._fallback_telemetry = telemetry - return telemetry - - def _attributed_caller(self) -> str | None: - return getattr(self._reconcile_attribution, "caller", None) - - @contextmanager - def _attributed(self, caller: str | None) -> Iterator[None]: - """Stamp ``caller`` on this thread for the passes it runs. - - The outermost entry point wins: a tip-refresh prefetch that reaches - ``ensure_tip`` through the coordinator keeps ``tip_refresh``, and a - forced publication inside an attributed stretch keeps that stretch's - label. ``None`` stamps nothing. - """ - local = self._reconcile_attribution - previous = getattr(local, "caller", None) - if caller is None or previous is not None: - yield - return - if caller not in PRISM_REORG_RECONCILE_CALLERS: - raise ValueError(f"unknown reorg reconcile caller: {caller!r}") - local.caller = caller - try: - yield - finally: - local.caller = previous - - def _serialized_pass( - self, - caller: str | None, - **kwargs: object, - ) -> dict[str, object]: - """Run one pass through the serialized adapter, attributed and stamped. - - The stamp is what ``reconcile`` measures ``admission_wait`` from: the - writer admission and payout-balance mutation lock the coordinator - adapter takes before the core pass starts. It is cleared on every - exit so a pass refused before the core (a landed preview) cannot - leave a stale stamp for a later direct call on this thread. - """ - local = self._reconcile_attribution - with self._attributed(caller): - local.admission_started = time.monotonic() - try: - return self._ports.reconcile_serialized(**kwargs) - finally: - local.admission_started = None - - @staticmethod - def _candidate_intent(payout_changed: bool) -> dict[str, bool]: - """Preparation intent for a pass whose mutations may have moved balances. - - Reconciliation mutates pool-block, payout-entry and carry state but - never ``qbit_share_ledger`` (the verified invariant recorded in - ``lab.prism.accepted_preview_telemetry``), so the accepted-share - window at a fixed anchor stays exact after a maturation, an inactive - marking or a reactivation. A confirmed mutation therefore asks for a - live prior-balances reread, not the O(window) oracle rescan that - ``force_full_window_rescan`` still means everywhere else (#224). The - reread intent is forwarded only when it applies so preparation seams - that predate it keep their exact signature. - """ - intent: dict[str, bool] = {"force_full_window_rescan": False} - if payout_changed: - intent["force_prior_balances_read"] = True - return intent - - def snapshot(self) -> ReorgState: - with self._ports.state_lock(): - return ReorgState( - inactive_block_count=self.inactive_block_count, - reactivated_block_count=self.reactivated_block_count, - reconcile_skip_count=self.reconcile_skip_count, - reconcile_error_count=self.reconcile_error_count, - matured_payout_count=self.matured_payout_count, - last_tip_hash=self.last_tip_hash, - last_trusted=self.last_trusted, - last_monotonic=self.last_monotonic, - ) - - def reorg_reconcile_lookup_snapshot(self) -> dict[tuple[str, str], int]: - """Copied lookup counters for the metrics renderer.""" - with self._ports.state_lock(): - return dict(self.reorg_reconcile_lookup_counts) - - def note_outcome( - self, - tip_hash: str | None, - *, - trusted: bool, - clear_memo: bool = False, - evict_others: bool = False, - proof_epoch: int | None = None, - ) -> None: - """Record a reconcile outcome in the per-tip trusted memo. - - A trusted pass arms the memo for its own tip. An untrusted outcome - (superseded publication, untrusted chain view) unarms only its own - tip: it proves nothing about reconciliations already completed for - other tips, and unarming them globally forces every job build into a - redundant full pass. A pass that applied orphan/maturity row - mutations passes evict_others=True: cached proofs for other tips - were taken against pre-mutation rows and no longer hold, even if the - chain later flips back before any tip observation lands. A reconcile - error passes clear_memo=True; a partially applied ledger mutation - invalidates every cached outcome. ``proof_epoch`` carries the - tip-detection epoch the pass started its reads in: arming is refused - when the epoch moved during the pass, so a flip away and back can - never re-arm an entry with a proof from the closed epoch (the - latest-detected-hash guard alone cannot see the round trip). - """ - - self._ports.ensure_job_cache_state() - now = time.monotonic() - with self._ports.state_lock(): - self.last_tip_hash = tip_hash - self.last_trusted = trusted - self.last_monotonic = now - memo = self._reorg_reconcile_trusted_memo - if clear_memo: - memo.clear() - return - if evict_others: - for cached_tip in list(memo): - if cached_tip != tip_hash: - del memo[cached_tip] - if tip_hash is None: - return - if trusted: - latest_tip, detection_epoch = ( - self._ports.reorg_proof_snapshot() - ) - if latest_tip is not None and latest_tip != tip_hash: - # This pass finished for a tip that is no longer the - # newest detected one; its epoch is over. Arming would - # re-add an entry the newer observation already evicted - # and let a flip-back reuse a pre-flip outcome. - return - if proof_epoch is not None and proof_epoch != detection_epoch: - # The pass spanned a detection cycle; every memo - # consumer (refresh joins, initial-job and vardiff-idle - # builds) must see a full re-proof instead. - return - memo[tip_hash] = now - memo.move_to_end(tip_hash) - while len(memo) > PRISM_REORG_RECONCILE_MEMO_MAX_TIPS: - memo.popitem(last=False) - else: - memo.pop(tip_hash, None) - - def evict_memo_for_new_tip_locked(self, tip_hash: str) -> None: - """Drop trusted-reconcile entries for every tip except ``tip_hash``. - - Called under the coordinator state lock when a newer tip is - detected. A detected flip ends the epoch of every previously cached - outcome: if the chain later flips back to an earlier hash within the - cache TTL, pool-block chain state must be re-proven by a fresh pass, - not assumed from a pre-flip reconciliation. Detection is - observation-sequenced, so only genuinely newer observations evict. - """ - - self._ports.ensure_job_cache_state() - memo = self._reorg_reconcile_trusted_memo - for cached_tip in list(memo): - if cached_tip != tip_hash: - del memo[cached_tip] - - def ensure_current( - self, - *, - expected_tip_hash: str | None = None, - ) -> bool: - if not self.enabled and expected_tip_hash is None: - return True - current_tip = str(self._ports.rpc_call("getbestblockhash")) - if expected_tip_hash is not None and current_tip != expected_tip_hash: - raise TemplateRefreshSuperseded( - "qbit tip changed while prepared work was queued " - f"expected={expected_tip_hash} current={current_tip}" - ) - if not self.enabled: - return True - if self.memo_fresh(current_tip): - self.record_lookup("job_build", "memo_hit") - return True - self.record_lookup("job_build", "serial") - with self._attributed(RECONCILE_CALLER_JOB_BUILD): - return self._ports.ensure_tip(current_tip) - - def memo_fresh(self, tip_hash: str) -> bool: - """True when a trusted pass for ``tip_hash`` is inside the cache TTL - and the live chain view is still trusted. - - A fresh memo entry lets a caller reuse the completed pass instead of - queueing a redundant serialized one. The memo is per tip: an - untrusted outcome recorded for another tip never unarms this one. - The chain-view trust check is NOT cached: headers can run ahead of - the validated tip without the best block hash changing (an arriving - reorg), and job issuance must pause immediately, not a TTL later. - """ - ttl = self.cache_seconds - if ttl <= 0: - return False - self._ports.ensure_job_cache_state() - with self._ports.state_lock(): - reconciled_monotonic = self._reorg_reconcile_trusted_memo.get( - tip_hash - ) - return bool( - reconciled_monotonic is not None - and time.monotonic() - reconciled_monotonic <= ttl - and not self._ports.chain_view_untrusted() - ) - - def record_lookup(self, path: str, source: str) -> None: - if path not in PRISM_REORG_RECONCILE_LOOKUP_PATHS: - raise ValueError(f"unknown reorg reconcile lookup path: {path}") - if source not in PRISM_REORG_RECONCILE_LOOKUP_SOURCES: - raise ValueError(f"unknown reorg reconcile lookup source: {source}") - self._ports.ensure_job_cache_state() - with self._ports.state_lock(): - self.reorg_reconcile_lookup_counts[(path, source)] += 1 - - def prefetch_pass( - self, - tip_hash: str, - prove: bool = False, - ) -> bool: - """One prefetched reconcile, honoring the memo like the join does. - - A prefetch that queued behind a completed same-tip pass (abandoned - refresh attempts reuse the slot, but a replaced tip can leave one - queued) would otherwise re-run the full serialized pass for nothing. - A proving pass (the serial re-prove branches) bypasses the memo: - those branches exist precisely because the entry cannot be trusted. - """ - if not prove and self.memo_fresh(tip_hash): - return True - # Only the tip-refresh owner submits prefetches and re-proves. - with self._attributed(RECONCILE_CALLER_TIP_REFRESH): - return self._ports.ensure_tip(tip_hash) - - def submit_prefetch( - self, - tip_hash: str, - *, - prove: bool = False, - ) -> Future[bool] | None: - """Run one reconcile pass on the prefetch worker so it overlaps the - caller's template fetch. - - Returns ``None`` once shutdown has retired the executor; the caller - falls back to its serial pass. At most one prefetch is outstanding: - a refresh attempt that failed before its join (for example a - template-RPC outage) leaves its future in the slot, and the retry - reuses it for the same tip instead of queueing another serialized - pass behind the first. - """ - self._ports.ensure_job_cache_state() - stale_future: Future[bool] | None = None - future: Future[bool] | None = None - with self._reconcile_prefetch_executor_lock: - if self._reconcile_prefetch_executor_shutdown: - return None - pending = self._reconcile_prefetch_pending - if pending is not None: - pending_tip, pending_future, pending_proves = pending - if ( - not pending_future.done() - and pending_tip == tip_hash - and (pending_proves or not prove) - ): - # A proving pass satisfies both kinds of caller; a - # memo-honoring pass cannot satisfy a prove request and - # is replaced below like a tip change. - return pending_future - # Replaced tip or completed future: hand the old future off - # for disposal outside this lock -- cancellation runs done - # callbacks inline on this thread, and _clear_slot below - # re-takes the lock. - self._reconcile_prefetch_pending = None - if not pending_future.done(): - stale_future = pending_future - executor = self._reconcile_prefetch_executor - if executor is None: - executor = ThreadPoolExecutor( - max_workers=1, - thread_name_prefix="prism-reconcile-prefetch", - ) - self._reconcile_prefetch_executor = executor - try: - future = executor.submit( - self.prefetch_pass, tip_hash, prove - ) - self._reconcile_prefetch_pending = (tip_hash, future, prove) - except RuntimeError: - # Executor shutdown raced this submit; the serial path - # covers it (after the stale future is disposed below). - future = None - if stale_future is not None and not stale_future.cancel(): - # Already running for a replaced tip; let it finish detached. - # The slot holds the new tip, so at most one task ever waits - # behind the running one. - self.discard_stale_prefetch(stale_future) - if future is None: - return None - - def _clear_slot(done: Future[bool]) -> None: - with self._reconcile_prefetch_executor_lock: - pending_now = self._reconcile_prefetch_pending - if pending_now is not None and pending_now[1] is done: - self._reconcile_prefetch_pending = None - - # Registered outside the slot lock: a completed future runs the - # callback inline on this thread. - future.add_done_callback(_clear_slot) - return future - - @staticmethod - def discard_stale_prefetch(future: Future[bool]) -> None: - """Detach a prefetched pass whose tip was superseded during the - template fetch. - - The pass cannot be cancelled mid-ledger-mutation and already records - its own outcome/error accounting; the serial pass for the current - tip re-surfaces any condition that still applies. Observe its terminal - error without re-raising it into a callback frame that holds the future. - """ - - def _consume(done: Future[bool]) -> None: - if not done.cancelled(): - done.exception() - - future.add_done_callback(_consume) - - def join_prefetch_bounded(self, prefetch: Future[bool]) -> bool: - """Join a reconcile pass under the poll loop's bounded budget. - - On a genuine expiry the pass keeps running in its single prefetch - slot -- the paced retry re-joins the same future -- and the timeout - surfaces the normal blocked-retry path, keeping the poll loop's - liveness heartbeat fed while the pass catches up. The budget is - clamped below the loop's failure budget so a misconfigured override - cannot reinstate the park this bound exists to prevent. - """ - - join_timeout = min( - PRISM_RECONCILE_PREFETCH_JOIN_TIMEOUT_CEILING_SECONDS, - max( - 0.001, - float(self._ports.prefetch_join_timeout_seconds()), - ), - ) - try: - return prefetch.result(timeout=join_timeout) - except TimeoutError: - if prefetch.done(): - # The pass itself raised TimeoutError (socket.timeout is - # TimeoutError here): not a join expiry. Propagate silently - # so diagnosis points at the pass, not the join. - raise - print( - "prism coordinator: reconcile prefetch join exceeded " - f"{join_timeout:g}s; retrying refresh pass while it " - "completes", - flush=True, - ) - raise - - def snapshot_tip_bounded(self, tip_hash: str) -> bool: - """Run a snapshot-tip re-prove off-thread with the bounded join. - - The serial re-prove branches run the same crawling pass as the - overlapped prefetch; routing them through the prefetch slot gives - the poll loop one uniform bounded wait. Falls back to the direct - pass only when the prefetch executor has already been retired at - shutdown, whose exceptions the caller already maps to a clean exit. - """ - - prefetch = self.submit_prefetch(tip_hash, prove=True) - if prefetch is None: - with self._attributed(RECONCILE_CALLER_TIP_REFRESH): - return self._ports.ensure_tip(tip_hash) - return self.join_prefetch_bounded(prefetch) - - def shutdown_prefetch_executor(self) -> None: - self._ports.ensure_job_cache_state() - with self._reconcile_prefetch_executor_lock: - executor = self._reconcile_prefetch_executor - self._reconcile_prefetch_executor = None - self._reconcile_prefetch_executor_shutdown = True - self._reconcile_prefetch_pending = None - if executor is not None: - # A pass blocked on writer admission aborts via - # ShutdownInProgress on its own; never hold shutdown for it. - executor.shutdown(wait=False, cancel_futures=True) - - def ensure_tip( - self, - tip_hash: str, - *, - _coalesce_same_tip: bool = True, - _caller: str | None = None, - ) -> bool: - """Reconcile one tip, optionally bypassing same-tip flight reuse. - - Lock-owning accepted-block callers disable waiting for an existing - leader because it may itself be waiting for the payout-balance - mutation lock. Their own pass remains visible to ordinary followers. - ``_caller`` is the optional #224 attribution label; without one, the - flight bypass identifies the accepted-block landing (its only user) - and every other caller keeps whatever label its entry point stamped. - """ - if not self.enabled: - return True - caller = _caller - if caller is None and not _coalesce_same_tip: - caller = RECONCILE_CALLER_LANDING - with self._attributed(caller): - if _coalesce_same_tip: - summary = self._ports.reconcile_with_admission( - tip_hash=tip_hash - ) - else: - summary = self._ports.reconcile_with_admission( - tip_hash=tip_hash, - _wait_for_same_tip_flight=False, - ) - return not bool(summary.get("untrusted") or summary.get("superseded")) - - def reconcile_with_flights( - self, - *, - tip_hash: str | None = None, - _force_publish: bool = False, - _source_reserved: bool = False, - _wait_for_same_tip_flight: bool = True, - _caller: str | None = None, - ) -> dict[str, object]: - """Reconcile pool blocks, coalescing same-tip concurrent callers. - - Unflagged callers asking about a tip whose pass is already in flight - await that pass and share its summary instead of queueing another - full serialized pass. Callers carrying side-effect obligations (a - forced publication or an already-reserved source) and callers without - a tip always run their own pass. Lock-owning callers may disable - waiting for an existing flight while still registering as the visible - leader when no same-tip flight exists. ``_caller`` is the optional - #224 attribution label; a forced publication without one is the - post-confirm path (its only user). - """ - self._ports.ensure_job_cache_state() - caller = _caller - if caller is None and _force_publish: - caller = RECONCILE_CALLER_POST_CONFIRM - if tip_hash is None or _force_publish or _source_reserved: - return self._serialized_pass( - caller, - tip_hash=tip_hash, - _force_publish=_force_publish, - _source_reserved=_source_reserved, - ) - with self._reconcile_flight_lock: - flight = self._reconcile_flights.get(tip_hash) - leading = flight is None - if leading: - flight = _ReconcileFlight() - self._reconcile_flights[tip_hash] = flight - if not leading: - if not _wait_for_same_tip_flight: - return self._serialized_pass(caller, tip_hash=tip_hash) - wait_seconds = float(self._ports.flight_wait_seconds()) - if flight.event.wait(timeout=wait_seconds): - exception = flight.exception - if exception is not None: - raise exception - summary = flight.summary - assert summary is not None - # Followers get a copy: summaries are mutable dicts and the - # leader's caller already holds the original. - return dict(summary) - # Liveness backstop: the leader outlived the wait. Run our own - # pass; the writer lock still serializes the actual work. - return self._serialized_pass(caller, tip_hash=tip_hash) - try: - summary = self._serialized_pass(caller, tip_hash=tip_hash) - flight.summary = summary - return summary - except BaseException as exc: - flight.exception = exc - raise - finally: - with self._reconcile_flight_lock: - self._reconcile_flights.pop(tip_hash, None) - flight.event.set() - - def reconcile( - self, - *, - tip_hash: str | None = None, - force_publish: bool = False, - source_reserved: bool = False, - caller: str | None = None, - ) -> dict[str, object]: - """The core reconcile state machine (behind the serialized adapter). - - ``caller`` is the bounded #224 attribution label. When omitted it is - the label the entry point stamped on this thread, ``post_confirm`` - for a forced publication without one, and ``other`` for tools, - benchmarks and periodic passes. The pass family is recorded on every - exit -- success, untrusted skip, superseded budget, and the - error-after-possible-commit path -- and never changes the outcome. - """ - summary: dict[str, object] = { - "enabled": bool(self.enabled), - "untrusted": False, - "superseded": False, - "published_generation": None, - "watched_blocks": 0, - "inactive_blocks": 0, - "reactivated_blocks": 0, - "matured_payouts": 0, - "stranded_prepared_rejected": 0, - "stranded_prepared_canonical": 0, - } - if not self.enabled: - return summary - self._ports.ensure_job_cache_state() - if caller is None: - caller = self._attributed_caller() - if caller is None: - caller = ( - RECONCILE_CALLER_POST_CONFIRM - if force_publish - else RECONCILE_CALLER_OTHER - ) - if caller not in PRISM_REORG_RECONCILE_CALLERS: - raise ValueError(f"unknown reorg reconcile caller: {caller!r}") - telemetry = self.accepted_preview_telemetry - pass_started = time.monotonic() - local = self._reconcile_attribution - admission_started = getattr(local, "admission_started", None) - local.admission_started = None - if admission_started is not None: - # Stamped by _serialized_pass ahead of the coordinator adapter's - # writer admission and payout-balance mutation lock: the one - # stretch a tip-refresh caller spends behind a landing. - telemetry.observe_reconcile_step( - caller, - RECONCILE_STEP_ADMISSION_WAIT, - pass_started - admission_started, - ) - - def step(name: str) -> AbstractContextManager[None]: - return telemetry.reconcile_step(caller, name) - - try: - return self._reconcile_pass( - summary, - tip_hash=tip_hash, - force_publish=force_publish, - source_reserved=source_reserved, - step=step, - ) - finally: - telemetry.observe_reconcile_pass( - caller, - time.monotonic() - pass_started, - ) - - def _reconcile_pass( - self, - summary: dict[str, object], - *, - tip_hash: str | None, - force_publish: bool, - source_reserved: bool, - step: Callable[[str], AbstractContextManager[None]], - ) -> dict[str, object]: - """One attributed pass; ``step`` times each bounded reconcile step.""" - if not source_reserved and tip_hash is not None: - # Tip observation normally reserves this source before queueing - # reconciliation. Direct callers only need a new source when they - # are asking about a different tip; repeated reconciliation of the - # same tip must not supersede otherwise valid prepared work. - current_source_tip = self._ports.source_tip() - if current_source_tip != tip_hash: - self._ports.reserve_external_tip(tip_hash) - - inactive_blocks_total = 0 - reactivated_blocks_total = 0 - matured_payouts_total = 0 - stranded_prepared_rejected_total = 0 - stranded_prepared_canonical_total = 0 - supersession_retries = 0 - skip_recorded = False - max_supersession_retries = max( - 0, - int(self._ports.max_supersession_retries()), - ) - - proof_epoch = self._ports.reorg_proof_snapshot()[1] - - def finish(*, trusted: bool) -> dict[str, object]: - with self._ports.state_lock(): - self.inactive_block_count += inactive_blocks_total - self.reactivated_block_count += reactivated_blocks_total - self.matured_payout_count += matured_payouts_total - self.note_outcome( - tip_hash, - trusted=trusted, - # Row mutations invalidate proofs cached for other tips even - # when the mutating pass's tip was never observed (per-client - # callers reconcile straight off getbestblockhash). - evict_others=bool( - inactive_blocks_total - or reactivated_blocks_total - or matured_payouts_total - or stranded_prepared_rejected_total - ), - proof_epoch=proof_epoch, - ) - summary["inactive_blocks"] = inactive_blocks_total - summary["reactivated_blocks"] = reactivated_blocks_total - summary["matured_payouts"] = matured_payouts_total - summary["stranded_prepared_rejected"] = ( - stranded_prepared_rejected_total - ) - summary["stranded_prepared_canonical"] = ( - stranded_prepared_canonical_total - ) - return summary - - def retry_superseded_candidate() -> bool: - nonlocal supersession_retries, tip_hash - supersession_retries += 1 - if supersession_retries > max_supersession_retries: - summary["superseded"] = True - self._ports.block_publication() - return False - latest_tip = self._ports.source_tip() - tip_hash = latest_tip or tip_hash - return True - - while True: - candidate_to_publish: PayoutStateCandidate | None = None - error_candidate: PayoutStateCandidate | None = None - attempt_trusted = True - # The memo entry this attempt may arm must prove state for the - # epoch its reads happen in; a detection cycle during the pass - # (away, or away and back) refuses the arm in note_outcome. - proof_epoch = self._ports.reorg_proof_snapshot()[1] - try: - with self._ports.prepare_lock(): - prepared_started = time.monotonic() - captured_source = self._ports.capture_source() - payout_changed = False - payout_mutation_attempted = False - inactive_blocks = 0 - reactivated_blocks = 0 - matured_payouts = 0 - stranded_prepared_rejected = 0 - stranded_prepared_canonical = 0 - summary["untrusted"] = False - summary["watched_blocks"] = 0 - try: - if self._ports.chain_view_untrusted(): - if not skip_recorded: - with self._ports.state_lock(): - self.reconcile_skip_count += 1 - skip_recorded = True - summary["untrusted"] = True - attempt_trusted = False - if force_publish: - with step(RECONCILE_STEP_CANDIDATE_PREPARE): - candidate_to_publish = ( - self._ports.prepared_candidate( - captured_source - ) - ) - else: - # Every stamp below marks one completed round - # trip. A landing thread's watchdog budget is - # sized for a single statement, and this pass - # crosses an unbounded number of them, so the - # stamps have to follow the work itself rather - # than bracket the pass as a whole. - self._ports.record_progress( - "reorg-reconcile:tip-height" - ) - with step(RECONCILE_STEP_CHAIN_PROBE): - active_tip_height = int( - self._ports.rpc_call("getblockcount") - ) - ledger = self._ports.ledger() - watch_blocks = getattr( - ledger, - "reorg_watch_blocks", - None, - ) - if not callable(watch_blocks): - if ( - force_publish - or self._ports.captured_publication_required( - captured_source - ) - ): - with step(RECONCILE_STEP_CANDIDATE_PREPARE): - candidate_to_publish = ( - self._ports.prepared_candidate( - captured_source, - **self._candidate_intent( - payout_changed - ), - ) - ) - else: - self._ports.record_progress( - "reorg-reconcile:watch-blocks" - ) - with step(RECONCILE_STEP_WATCH_QUERY): - rows = watch_blocks( - active_tip_height=active_tip_height - ) - summary["watched_blocks"] = len(rows) - - for row in rows: - # Per row, not per pass: the row count is - # bounded only by how many pool blocks the - # reorg window holds, and each iteration - # can spend a getblockhash plus a ledger - # mutation. - self._ports.record_progress( - "reorg-reconcile:watch-block" - ) - block_height = int(row["block_height"]) - block_hash = str(row["block_hash"]).lower() - chain_state = str(row.get("chain_state", "")) - if block_height > active_tip_height: - if chain_state == "confirmed": - payout_mutation_attempted = True - with step(RECONCILE_STEP_MUTATIONS): - inactive = ( - ledger.mark_pool_block_inactive( - block_hash=block_hash, - active_tip_height=active_tip_height, - ) - ) - inactive_count = int( - inactive.get("inactive_count", 0) - ) - inactive_blocks += inactive_count - payout_changed = ( - payout_changed - or bool(inactive_count) - ) - continue - with step(RECONCILE_STEP_CHAIN_PROBE): - active_hash = str( - self._ports.rpc_call( - "getblockhash", - [block_height], - ) - ).lower() - on_active_chain = active_hash == block_hash - if ( - on_active_chain - and chain_state == "inactive" - ): - payout_mutation_attempted = True - with step(RECONCILE_STEP_MUTATIONS): - with self._ports.publication_guard(): - reactivated = ledger.reactivate_pool_block( - block_hash=block_hash, - active_tip_height=active_tip_height, - ) - reactivated_count = int( - reactivated.get( - "reactivated_count", - 0, - ) - ) - reactivated_blocks += reactivated_count - payout_changed = ( - payout_changed - or bool(reactivated_count) - ) - elif ( - not on_active_chain - and chain_state == "confirmed" - ): - payout_mutation_attempted = True - with step(RECONCILE_STEP_MUTATIONS): - inactive = ( - ledger.mark_pool_block_inactive( - block_hash=block_hash, - active_tip_height=active_tip_height, - ) - ) - inactive_count = int( - inactive.get("inactive_count", 0) - ) - inactive_blocks += inactive_count - payout_changed = ( - payout_changed - or bool(inactive_count) - ) - - # Rows the watch loop above structurally - # cannot see: it selects confirmed/inactive - # only, because a prepared row belongs to the - # live submit/replay path that holds its - # outbox entry. When that entry is gone the - # row has no owner left, and an orphaned one - # pins its payout entries, carry forward, and - # CTV fanout artifacts immature forever. - stranded_prepared = getattr( - ledger, - "stranded_prepared_blocks", - None, - ) - if callable(stranded_prepared): - self._ports.record_progress( - "reorg-reconcile:stranded-prepared-blocks" - ) - # The sweep's row read is the same kind - # of watch-set query as the watch loop's. - with step(RECONCILE_STEP_WATCH_QUERY): - stranded_rows = stranded_prepared( - active_tip_height=active_tip_height, - min_depth=( - STRANDED_PREPARED_REJECT_MIN_DEPTH - ), - limit=STRANDED_PREPARED_SWEEP_LIMIT, - ) - for row in stranded_rows: - # Per row for the same reason as the - # watch loop: a getblockhash plus a - # fenced ledger mutation each. - self._ports.record_progress( - "reorg-reconcile:stranded-prepared" - ) - block_height = int(row["block_height"]) - block_hash = str( - row["block_hash"] - ).lower() - with step(RECONCILE_STEP_CHAIN_PROBE): - active_hash = str( - self._ports.rpc_call( - "getblockhash", - [block_height], - ) - ).lower() - if active_hash == block_hash: - # Canonical but never confirmed: - # the confirm path owns audit - # publication sequencing, so this - # sweep must never take the row - # from under it. Report loudly and - # leave it alone. - stranded_prepared_canonical += 1 - print( - "prism coordinator: stranded " - "prepared pool block is " - "canonical and needs operator " - "review " - f"hash={block_hash} " - f"height={block_height} " - "depth=" - f"{active_tip_height - block_height}", - flush=True, - ) - continue - payout_mutation_attempted = True - with step(RECONCILE_STEP_MUTATIONS): - rejected = ledger.reject_prepared_block( - block_hash=block_hash, - active_tip_height=active_tip_height, - ) - # The fenced ledger function cascades - # payout entries, carry forward, and - # fanout artifacts, and returns 0 when - # the row already moved under us. - rejected_count = int( - rejected.get("rejected_count", 0) - ) - stranded_prepared_rejected += ( - rejected_count - ) - payout_changed = ( - payout_changed - or bool(rejected_count) - ) - if rejected_count: - print( - "prism coordinator: rejected " - "orphaned stranded prepared " - "pool block " - f"hash={block_hash} " - f"height={block_height} " - "depth=" - f"{active_tip_height - block_height}", - flush=True, - ) - - mark_mature = getattr( - ledger, - "mark_mature_pool_payouts", - None, - ) - if callable(mark_mature): - payout_mutation_attempted = True - self._ports.record_progress( - "reorg-reconcile:mature-payouts" - ) - with step(RECONCILE_STEP_MUTATIONS): - matured = mark_mature( - active_tip_height=active_tip_height - ) - matured_payouts = int( - matured.get("matured_count", 0) - ) - payout_changed = ( - payout_changed - or bool(matured_payouts) - ) - - inactive_blocks_total += inactive_blocks - reactivated_blocks_total += reactivated_blocks - matured_payouts_total += matured_payouts - stranded_prepared_rejected_total += ( - stranded_prepared_rejected - ) - stranded_prepared_canonical_total += ( - stranded_prepared_canonical - ) - if ( - payout_changed - or force_publish - or self._ports.captured_publication_required( - captured_source - ) - ): - # Candidate preparation embeds the ledger - # snapshot artifact; a pass that will not - # publish must not pay for one only to - # discard it. A confirmed payout mutation - # asks for a live prior-balances reread - # over the still-exact share window, not - # a full rescan (#224; see - # _candidate_intent). - self._ports.record_progress( - "reorg-reconcile:prepare-candidate" - ) - with step(RECONCILE_STEP_CANDIDATE_PREPARE): - candidate_to_publish = ( - self._ports.prepared_candidate( - captured_source, - **self._candidate_intent( - payout_changed - ), - ) - ) - except Exception: - inactive_blocks_total += inactive_blocks - reactivated_blocks_total += reactivated_blocks - matured_payouts_total += matured_payouts - stranded_prepared_rejected_total += ( - stranded_prepared_rejected - ) - stranded_prepared_canonical_total += ( - stranded_prepared_canonical - ) - # Durable partial mutations close admission before the - # preparation lock is released. Publication drains old - # socket sends afterward without blocking new ledger - # preparation or snapshot acquisition. - if payout_mutation_attempted: - # A mutator can commit server-side and still raise - # locally when its response is lost. The observed - # row counts are then unavailable, so conservatively - # force the same ledger re-read as a confirmed - # mutation. Read-only failures never reach this flag. - payout_changed = True - if payout_changed: - # Fail closed on the uncertain path: the forced - # full rescan is unchanged here and necessarily - # rereads the current prior balances, so a lost - # mutator response can never republish stale - # balances. - with step(RECONCILE_STEP_CANDIDATE_PREPARE): - error_candidate = ( - self._ports.prepared_candidate( - captured_source, - force_full_window_rescan=True, - ) - ) - self._ports.block_publication(force=True) - with self._ports.state_lock(): - self.inactive_block_count += ( - inactive_blocks_total - ) - self.reactivated_block_count += ( - reactivated_blocks_total - ) - self.matured_payout_count += matured_payouts_total - self.reconcile_error_count += 1 - # A pass that errored mid-mutation invalidates every - # cached outcome, not just its own tip's. - self.note_outcome( - tip_hash, - trusted=False, - clear_memo=True, - ) - raise - finally: - self._ports.observe_preparation( - max(0.0, time.monotonic() - prepared_started) - ) - - if candidate_to_publish is not None: - # Atomically fence cache/build/delivery admission before - # releasing the ledger snapshot lock. The potentially - # slow drain then happens in publication() below. - self._ports.block_publication(force=True) - except Exception: - if error_candidate is not None: - with step(RECONCILE_STEP_PUBLISH): - published_after_error = self._ports.publish_candidate( - error_candidate - ) - if published_after_error is None: - self._ports.block_publication() - raise - - if candidate_to_publish is not None: - self._ports.record_progress("reorg-reconcile:publish") - with step(RECONCILE_STEP_PUBLISH): - published = self._ports.publish_candidate( - candidate_to_publish - ) - if published is None: - # Preserve durable counts and retry iteratively against the - # newest source. The explicit budget prevents tip churn - # from monopolizing preparation indefinitely; the fence - # stays closed between attempts. - if retry_superseded_candidate(): - continue - return finish(trusted=False) - summary["published_generation"] = published - return finish(trusted=attempt_trusted) - - -class ReorgCompatibilityField: - """Route retained coordinator fields to their single service owner.""" - - def __init__(self, name: str, default: object) -> None: - self.name = name - self.default = default - self.backing = f"_reorg_compat_{name}" - - def __get__(self, instance: Any, owner: type[Any]) -> object: - if instance is None: - return self - service = instance.__dict__.get("_reorg_reconciler_service") - if service is not None: - return getattr(service, self.name) - return instance.__dict__.get(self.backing, self.default) - - def __set__(self, instance: Any, value: object) -> None: - service = instance.__dict__.get("_reorg_reconciler_service") - if service is not None: - setattr(service, self.name, value) - return - instance.__dict__[self.backing] = value diff --git a/lab/prism/rpc.py b/lab/prism/rpc.py deleted file mode 100644 index 31455a557..000000000 --- a/lab/prism/rpc.py +++ /dev/null @@ -1,241 +0,0 @@ -"""Thread-local JSON-RPC client used by PRISM processes.""" - -from __future__ import annotations - -import base64 -import http.client -import json -import socket -import threading -import time -import urllib.parse -from typing import Any - - -# The default deadline for every JsonRpc.call without an explicit timeout, -# including the fence-guarded CTV broadcast RPCs. The ledger's own-write -# deferral margin is derived from the guarded deadlines, so route changes -# through this constant rather than the call signature's literal. -DEFAULT_QBIT_RPC_CALL_TIMEOUT_SECONDS = 10.0 - -_QBIT_RPC_NO_TRANSPORT_RETRY_METHODS = frozenset( - { - "getnewaddress", - "sendrawtransaction", - "signrawtransactionwithwallet", - "submitblock", - "submitpackage", - } -) - - -class JsonRpc: - def __init__(self, *, host: str, port: int, user: str, password: str): - self.host = host - self.port = port - self.url = f"http://{host}:{port}" - credentials = f"{user}:{password}".encode() - self.auth = f"Basic {base64.b64encode(credentials).decode()}" - # Keep-alive connections, one per calling thread. qbitd is called on - # the hot share/block paths (a fresh getaddrinfo + TCP connect per call - # was ~seconds of overhead under load); reusing the connection removes - # that. threading.local keeps each thread's HTTPConnection private, so - # concurrent callers never share a non-thread-safe connection. - self._connections = threading.local() - - def _acquire_connection(self, timeout: float) -> http.client.HTTPConnection: - conn = getattr(self._connections, "conn", None) - if conn is None: - conn = http.client.HTTPConnection(self.host, self.port, timeout=timeout) - # Never let http.client resurrect a severed connection: with - # auto_open, request() silently reconnects when the deadline - # watchdog's close() lands between the pre-send check and the - # send, and that implicit fresh socket lives until the next - # watchdog sweep — long enough for a short mutating POST to - # reach qbitd after a caller released its ordering locks. With - # auto_open off a cleared socket makes send() raise - # NotConnected instead; call() connects explicitly under the - # watchdog and rechecks the deadline before any byte goes out. - conn.auto_open = 0 - self._connections.conn = conn - else: - # Reuse: refresh the deadline for this call on the live socket. - conn.timeout = timeout - if conn.sock is not None: - conn.sock.settimeout(timeout) - return conn - - def _drop_connection(self) -> None: - conn = getattr(self._connections, "conn", None) - if conn is not None: - try: - conn.close() - except Exception: - pass - self._connections.conn = None - - def call( - self, - method: str, - params: list[object] | None = None, - *, - wallet: str | None = None, - timeout: float = DEFAULT_QBIT_RPC_CALL_TIMEOUT_SECONDS, - ) -> Any: - streamed_block = ( - params[0] if method == "submitblock" and params and len(params) == 1 - and callable(getattr(params[0], "iter_byte_chunks", None)) else None - ) - body = None if streamed_block is not None else json.dumps( - { - "jsonrpc": "1.0", - "id": method, - "method": method, - "params": params or [], - } - ).encode() - path = "/" - if wallet is not None: - path = f"/wallet/{urllib.parse.quote(wallet, safe='')}" - headers = { - "Authorization": self.auth, - "Content-Type": "application/json", - "User-Agent": "qbit-prism-coordinator/0.1", - } - if streamed_block is not None: - prefix = b'{"jsonrpc":"1.0","id":"submitblock","method":"submitblock","params":[' - suffix = b']}' - headers["Content-Length"] = str(len(prefix) + streamed_block.byte_length + len(suffix)) - # Read-only calls retry once with a fresh connection after a transport - # error, normally an idle keep-alive that qbitd closed. Mutating calls - # never retry inside this method: their writer-lease fence applies to - # the outer call, so an invisible second POST could otherwise run after - # that lease was lost. Durable block/CTV workflows retry later as a new - # fully fenced operation and reconcile an uncertain first result. - last_exc: Exception | None = None - attempt_count = ( - 1 if method in _QBIT_RPC_NO_TRANSPORT_RETRY_METHODS else 2 - ) - deadline = time.monotonic() + max(0.001, float(timeout)) - for attempt in range(attempt_count): - remaining = deadline - time.monotonic() - if remaining <= 0: - if last_exc is not None: - raise last_exc - raise TimeoutError(f"qbit RPC {method} timed out") - conn = self._acquire_connection(remaining) - # The socket timeout bounds each individual socket operation, - # not the call: a response body arriving one packet per - # interval extends the call arbitrarily past its nominal - # deadline. Lease authority margins are sized from this - # timeout as a wall-clock bound on fence-guarded effects, so - # enforce it end-to-end: past the deadline a watchdog severs - # the connection's socket — repeatedly, because a stalled name - # resolution has no socket yet and the one connect() creates - # afterwards must not carry the RPC onward — until the attempt - # observes the abort as a transport error. For mutating calls - # that is the same uncertain outcome a socket timeout already - # produces, reconciled by durable replay as a new fully fenced - # operation. - watchdog_fired = threading.Event() - attempt_finished = threading.Event() - - # The allowance is the attempt's exact remaining wall clock, no - # grace: callers like the submitblock hard-deadline wrapper - # abandon the call and release ordering locks (the payout - # landing fence) the instant their deadline passes, so a byte - # transmitted in any grace window would race work those callers - # already unblocked. - def _enforce_deadline( - doomed: http.client.HTTPConnection = conn, - fired: threading.Event = watchdog_fired, - finished: threading.Event = attempt_finished, - allowance: float = remaining, - ) -> None: - if finished.wait(allowance): - return - fired.set() - while True: - sock = getattr(doomed, "sock", None) - if sock is not None: - try: - sock.shutdown(socket.SHUT_RDWR) - except OSError: - pass - try: - doomed.close() - except Exception: - pass - if finished.wait(0.1): - return - - watchdog = threading.Thread( - target=_enforce_deadline, - name="qbit-rpc-deadline", - daemon=True, - ) - watchdog.start() - try: - # Establish the connection (name resolution + TCP) under - # the watchdog and re-check the deadline before any request - # byte is sent. Left to conn.request(), a name resolution - # returning after the deadline resumes straight into - # sendall(), and a short mutating POST can reach qbitd - # between the watchdog's periodic sweeps; the explicit - # check makes the late-connect case lose deterministically. - # The check is against the wall clock, not just the - # watchdog flag: the flag lags the deadline by thread - # scheduling, and a caller that abandoned this call at its - # deadline may already have released ordering locks that - # a post-deadline send would race. - if getattr(conn, "sock", None) is None: - conn_connect = getattr(conn, "connect", None) - if conn_connect is not None: - conn_connect() - if watchdog_fired.is_set() or time.monotonic() >= deadline: - raise TimeoutError(f"qbit RPC {method} timed out") - def request_chunks(): - yield prefix - for chunk in streamed_block.iter_byte_chunks(): - if watchdog_fired.is_set() or time.monotonic() >= deadline: - raise TimeoutError(f"qbit RPC {method} timed out") - yield chunk - yield suffix - - conn.request("POST", path, body=request_chunks() if streamed_block is not None else body, headers=headers) - response = conn.getresponse() - data = response.read() # drain so the connection can be reused - except (http.client.HTTPException, OSError) as exc: - self._drop_connection() - if watchdog_fired.is_set(): - raise TimeoutError(f"qbit RPC {method} timed out") from exc - last_exc = exc - if attempt + 1 < attempt_count: - continue - raise - except BaseException: - # A streamed source can fail after sending a request prefix. - # That connection cannot carry a later RPC safely. - self._drop_connection() - raise - finally: - attempt_finished.set() - if response.status != 200: - # Non-200 bodies may hold a JSON-RPC error (qbitd returns the - # error object with a 500 for some methods); surface it as the - # same RuntimeError text callers already match on (e.g. the - # "-32601 / Method not found" blockwait-unsupported probe). - self._drop_connection() - detail = data.decode("utf-8", "replace") - try: - error = json.loads(detail).get("error") - except Exception: - error = None - if error is not None: - raise RuntimeError(f"qbit RPC {method} failed: {error}") - raise RuntimeError(f"qbit RPC {method} HTTP {response.status}: {detail[:200]}") - payload = json.loads(data) - if payload["error"] is not None: - raise RuntimeError(f"qbit RPC {method} failed: {payload['error']}") - return payload["result"] - raise last_exc if last_exc is not None else RuntimeError("qbit RPC call failed") diff --git a/lab/prism/run_ctv_broadcaster_daemon.py b/lab/prism/run_ctv_broadcaster_daemon.py deleted file mode 100644 index 642f930f1..000000000 --- a/lab/prism/run_ctv_broadcaster_daemon.py +++ /dev/null @@ -1,179 +0,0 @@ -#!/usr/bin/env python3 -"""Run the PRISM CTV fanout broadcaster daemon.""" - -from __future__ import annotations - -import os -import shlex -import time -from pathlib import Path - -import sys - -if not __package__: - sys.path.insert(0, str(Path(__file__).resolve().parents[2])) - -from lab.prism.ctv_broadcaster import CtvFanoutBroadcaster -from lab.prism.ctv_broadcaster_daemon import ( - CtvFanoutBroadcastDaemon, - CtvFanoutDaemonResult, - MAX_CTV_FANOUT_BROADCASTER_CHUNK_SIZE, -) -from lab.prism.coordinator_config import ( - DEFAULT_LEASE_ACQUIRE_ATTEMPTS, - DEFAULT_LEASE_ACQUIRE_LOCK_TIMEOUT_SECONDS, - DEFAULT_POSTGRES_IDLE_IN_TRANSACTION_TIMEOUT_SECONDS, - DEFAULT_POSTGRES_TCP_KEEPALIVES_COUNT, - DEFAULT_POSTGRES_TCP_KEEPALIVES_IDLE_SECONDS, - DEFAULT_POSTGRES_TCP_KEEPALIVES_INTERVAL_SECONDS, - env, - env_bool, - env_int, - env_positive_float, -) - -# The module-level env_positive_int below keeps its required-missing -# semantics for the daemon's own knobs; the coordinator_config variant is -# aliased for the ledger knobs so both scripts read them identically. -from lab.prism.coordinator_config import env_positive_int as config_env_positive_int -from lab.prism.rpc import JsonRpc -from lab.prism.share_ledger import PsqlShareLedger, SingleWriterShareLedger - - -def env_positive_int(name: str, default: int | None = None) -> int: - raw = os.environ.get(name) - if raw is None: - if default is None: - raise SystemExit(f"{name} is required") - value = default - else: - try: - value = int(raw) - except ValueError as exc: - raise SystemExit(f"{name} must be an integer") from exc - if value <= 0: - raise SystemExit(f"{name} must be positive") - return value - - -def env_nonnegative_int(name: str, default: int | None = None) -> int: - raw = os.environ.get(name) - if raw is None: - if default is None: - raise SystemExit(f"{name} is required") - value = default - else: - try: - value = int(raw) - except ValueError as exc: - raise SystemExit(f"{name} must be an integer") from exc - if value < 0: - raise SystemExit(f"{name} must be non-negative") - return value - - -def env_nonnegative_int_with_legacy(primary_name: str, legacy_name: str, default: int) -> int: - if os.environ.get(primary_name, "") != "": - return env_nonnegative_int(primary_name, default) - return env_nonnegative_int(legacy_name, default) - - -def make_ledger_from_env() -> SingleWriterShareLedger | PsqlShareLedger: - psql_command = os.environ.get("PRISM_POSTGRES_PSQL_COMMAND", "") - database_url = os.environ.get("PRISM_DATABASE_URL", "") - if not psql_command and database_url: - psql_command = f"psql {shlex.quote(database_url)}" - if not psql_command: - if not env_bool("PRISM_ALLOW_MEMORY_LEDGER", "0"): - raise SystemExit( - "PRISM_DATABASE_URL or PRISM_POSTGRES_PSQL_COMMAND is required; " - "set PRISM_ALLOW_MEMORY_LEDGER=1 only for local tests" - ) - return SingleWriterShareLedger() - return PsqlShareLedger( - psql_command=psql_command, - writer_id=env("PRISM_LEDGER_WRITER_ID", "prism-ctv-broadcaster"), - writer_epoch=env_int("PRISM_LEDGER_WRITER_EPOCH", 1), - writer_session_token=os.environ.get("PRISM_LEDGER_WRITER_SESSION_TOKEN"), - initialize_schema=env("PRISM_POSTGRES_INIT_SCHEMA", "0") in {"1", "true", "yes"}, - lease_ttl_seconds=env_positive_float("PRISM_LEDGER_LEASE_TTL_SECONDS", 60.0), - lease_acquire_lock_timeout_seconds=env_positive_float( - "PRISM_LEDGER_LEASE_ACQUIRE_LOCK_TIMEOUT_SECONDS", - DEFAULT_LEASE_ACQUIRE_LOCK_TIMEOUT_SECONDS, - ), - lease_acquire_attempts=config_env_positive_int( - "PRISM_LEDGER_LEASE_ACQUIRE_ATTEMPTS", - DEFAULT_LEASE_ACQUIRE_ATTEMPTS, - ), - postgres_idle_in_transaction_timeout_seconds=env_positive_float( - "PRISM_POSTGRES_IDLE_IN_TRANSACTION_TIMEOUT_SECONDS", - DEFAULT_POSTGRES_IDLE_IN_TRANSACTION_TIMEOUT_SECONDS, - ), - postgres_tcp_keepalives_idle_seconds=config_env_positive_int( - "PRISM_POSTGRES_TCP_KEEPALIVES_IDLE_SECONDS", - DEFAULT_POSTGRES_TCP_KEEPALIVES_IDLE_SECONDS, - ), - postgres_tcp_keepalives_interval_seconds=config_env_positive_int( - "PRISM_POSTGRES_TCP_KEEPALIVES_INTERVAL_SECONDS", - DEFAULT_POSTGRES_TCP_KEEPALIVES_INTERVAL_SECONDS, - ), - postgres_tcp_keepalives_count=config_env_positive_int( - "PRISM_POSTGRES_TCP_KEEPALIVES_COUNT", - DEFAULT_POSTGRES_TCP_KEEPALIVES_COUNT, - ), - ) - - -def make_daemon_from_env() -> CtvFanoutBroadcastDaemon: - rpc = JsonRpc( - host=env("QBIT_RPC_HOST"), - port=env_int("QBIT_RPC_PORT", 18452), - user=env("QBIT_RPC_USER"), - password=env("QBIT_RPC_PASSWORD"), - ) - wallet = os.environ.get("PRISM_CTV_BROADCASTER_WALLET") or None - fee_sats = env_nonnegative_int_with_legacy( - "PRISM_CTV_BROADCASTER_FEE_BITS", - "PRISM_CTV_BROADCASTER_FEE_SATS", - 0, - ) - if fee_sats > 0 and wallet is None: - raise SystemExit( - "PRISM_CTV_BROADCASTER_WALLET is required when " - "PRISM_CTV_BROADCASTER_FEE_BITS is positive" - ) - broadcaster = CtvFanoutBroadcaster(rpc.call, funding_wallet=wallet) - return CtvFanoutBroadcastDaemon(make_ledger_from_env(), broadcaster, fee_sats=fee_sats) - - -def print_result(result: CtvFanoutDaemonResult) -> None: - print( - "ctv broadcaster: " - f"scanned={result.scanned_count} " - f"submitted={result.submitted_count} " - f"updated={result.updated_count} " - f"failed={result.failed_count}", - flush=True, - ) - - -def main() -> int: - daemon = make_daemon_from_env() - limit = env_positive_int("PRISM_CTV_BROADCASTER_LIMIT", 100) - chunk_size = env_positive_int("PRISM_CTV_BROADCASTER_CHUNK_SIZE", 5) - if chunk_size > MAX_CTV_FANOUT_BROADCASTER_CHUNK_SIZE: - raise SystemExit( - "PRISM_CTV_BROADCASTER_CHUNK_SIZE must be at most " - f"{MAX_CTV_FANOUT_BROADCASTER_CHUNK_SIZE}" - ) - once = env_bool("PRISM_CTV_BROADCASTER_ONCE", "0") - interval_seconds = env_positive_int("PRISM_CTV_BROADCASTER_INTERVAL_SECONDS", 30) - while True: - print_result(daemon.run_once(limit=limit, chunk_size=chunk_size)) - if once: - return 0 - time.sleep(interval_seconds) - - -if __name__ == "__main__": - raise SystemExit(main()) diff --git a/lab/prism/share_json_stream.py b/lab/prism/share_json_stream.py deleted file mode 100644 index b90664e09..000000000 --- a/lab/prism/share_json_stream.py +++ /dev/null @@ -1,363 +0,0 @@ -#!/usr/bin/env python3 -"""Bounded JSON encoding and digesting for PRISM share arrays. - -Why this exists: a payout window at union-mainnet size (roughly 210k -accepted shares; 400k at the stress size) used to pass through single -``json.dumps``/``json.loads`` calls -- the daemon prepare request, the -audit-builder compact payload, the digest of a freshly read snapshot, the -lazy parse of a daemon-prepared window -- each one a C call that holds the -GIL for hundreds of milliseconds while the writer-lease monitor thread waits -to run (#236). Every helper here produces or consumes exactly the bytes the -historical whole-array call did, while bounding any one C call to a batch -of records, so the interpreter can switch threads between batches. - -Byte identity is the contract, not a nicety: canonical share bytes are -hashed into ``share_snapshot_sha256`` and mirrored by the Rust builder, so -the streamed encoding must reproduce CPython's ``json.dumps`` output exactly -(``ensure_ascii`` escapes, non-BMP surrogate pairs, sorted keys where -canonical, arbitrary-precision integers). The batch form relies on one fact -about CPython's encoder that the tests pin: ``json.dumps(items)[1:-1]`` is -the items' individual encodings joined by ``,``, so a batch of any size -encodes to the same bytes as the whole array minus its brackets. - -This module is a leaf: standard library only, imported by the ledger, the -bundle compiler and the payout-state service alike. -""" - -from __future__ import annotations - -import hashlib -import json -from collections.abc import Collection, Iterable, Iterator, Mapping, Sequence -from typing import Any, Callable - -# Records encoded per ``json.dumps`` call. 512 matches the payout window's -# page size, so a page encodes in exactly one bounded call. -SHARE_JSON_BATCH_RECORDS = 512 -# Target size of one yielded chunk. Batches are coalesced up to this many -# characters (bytes: every chunk is ASCII) before being handed to a -# transport, so a pipe write or spool write moves tens of KiB at a time -# instead of one record. -SHARE_JSON_CHUNK_BYTES = 64 * 1024 - -_COMPACT_SEPARATORS = (",", ":") -# Entries dropped per ``del`` slice by release_share_list_incrementally. -RELEASE_BATCH_RECORDS = 2048 - - -def _batch_encoder( - *, - sort_keys: bool, - default: Callable[[Any], Any] | None, -) -> Callable[[list[Any]], str]: - """One ``json.dumps`` over a batch, minus the array brackets.""" - - def encode(batch: list[Any]) -> str: - return json.dumps( - batch, - sort_keys=sort_keys, - separators=_COMPACT_SEPARATORS, - default=default, - )[1:-1] - - return encode - - -class _ChunkCoalescer: - """Accumulates text pieces and releases them in bounded chunks.""" - - __slots__ = ("_chunk_chars", "_pending", "_pending_chars") - - def __init__(self, chunk_chars: int) -> None: - self._chunk_chars = int(chunk_chars) - self._pending: list[str] = [] - self._pending_chars = 0 - - def push(self, text: str) -> str | None: - """Take one piece; return a chunk once the target size is reached.""" - if not text: - return None - self._pending.append(text) - self._pending_chars += len(text) - if self._pending_chars < self._chunk_chars: - return None - return self.flush() - - def flush(self) -> str | None: - if not self._pending: - return None - chunk = ( - self._pending[0] - if len(self._pending) == 1 - else "".join(self._pending) - ) - self._pending = [] - self._pending_chars = 0 - return chunk - - -def iter_json_array_text_chunks( - items: Iterable[Any], - *, - sort_keys: bool = False, - default: Callable[[Any], Any] | None = None, - batch_records: int = SHARE_JSON_BATCH_RECORDS, - chunk_chars: int = SHARE_JSON_CHUNK_BYTES, -) -> Iterator[str]: - """The items' encodings joined by ``,`` -- no brackets -- in bounded chunks. - - Concatenating every yielded chunk gives exactly - ``json.dumps(list(items), separators=(",", ":"), sort_keys=sort_keys, - default=default)[1:-1]``. No chunk ever splits an item, so a consumer - may frame chunk boundaries however it likes (a spool file, a pipe, a - digest). Each ``json.dumps`` call covers at most ``batch_records`` items. - """ - if batch_records <= 0: - raise ValueError("batch_records must be positive") - encode = _batch_encoder(sort_keys=sort_keys, default=default) - coalescer = _ChunkCoalescer(chunk_chars) - batch: list[Any] = [] - first_batch = True - for item in items: - batch.append(item) - if len(batch) < batch_records: - continue - text = encode(batch) - batch = [] - if not first_batch: - text = "," + text - first_batch = False - chunk = coalescer.push(text) - if chunk is not None: - yield chunk - if batch: - text = encode(batch) - if not first_batch: - text = "," + text - chunk = coalescer.push(text) - if chunk is not None: - yield chunk - chunk = coalescer.flush() - if chunk is not None: - yield chunk - - -def iter_json_array_byte_chunks( - items: Iterable[Any], - *, - sort_keys: bool = False, - default: Callable[[Any], Any] | None = None, - batch_records: int = SHARE_JSON_BATCH_RECORDS, - chunk_chars: int = SHARE_JSON_CHUNK_BYTES, -) -> Iterator[bytes]: - """:func:`iter_json_array_text_chunks`, UTF-8 encoded chunk by chunk.""" - for chunk in iter_json_array_text_chunks( - items, - sort_keys=sort_keys, - default=default, - batch_records=batch_records, - chunk_chars=chunk_chars, - ): - yield chunk.encode("utf-8") - - -def iter_json_object_text_chunks( - fields: Mapping[str, Any], - *, - array_keys: Collection[str], - sort_keys: bool = False, - default: Callable[[Any], Any] | None = None, - batch_records: int = SHARE_JSON_BATCH_RECORDS, - chunk_chars: int = SHARE_JSON_CHUNK_BYTES, -) -> Iterator[str]: - """One JSON object in bounded chunks, streaming its large array members. - - Concatenating every yielded chunk gives exactly ``json.dumps(fields, - separators=(",", ":"), sort_keys=sort_keys, default=default)``. Members - named in ``array_keys`` whose values are lists or tuples are encoded - item-batch by item-batch through :func:`iter_json_array_text_chunks`; - every other member is one ``json.dumps`` call, which is bounded by that - member's own size (found-block facts, balances, policy). A mapping with a - non-string key is encoded whole so CPython's own key coercion rules keep - applying; no share payload has such keys. - """ - keys = list(fields) - if any(not isinstance(key, str) for key in keys): - yield json.dumps( - fields, - sort_keys=sort_keys, - separators=_COMPACT_SEPARATORS, - default=default, - ) - return - if sort_keys: - keys.sort() - coalescer = _ChunkCoalescer(chunk_chars) - chunk = coalescer.push("{") - if chunk is not None: - yield chunk - first = True - for key in keys: - value = fields[key] - prefix = ("" if first else ",") + json.dumps(key) + ":" - first = False - if key in array_keys and isinstance(value, (list, tuple)): - chunk = coalescer.push(prefix + "[") - if chunk is not None: - yield chunk - for piece in iter_json_array_text_chunks( - value, - sort_keys=sort_keys, - default=default, - batch_records=batch_records, - chunk_chars=chunk_chars, - ): - chunk = coalescer.push(piece) - if chunk is not None: - yield chunk - chunk = coalescer.push("]") - else: - chunk = coalescer.push( - prefix - + json.dumps( - value, - sort_keys=sort_keys, - separators=_COMPACT_SEPARATORS, - default=default, - ) - ) - if chunk is not None: - yield chunk - chunk = coalescer.push("}") - if chunk is not None: - yield chunk - chunk = coalescer.flush() - if chunk is not None: - yield chunk - - -def iter_canonical_share_item_chunks( - shares: Iterable[Any], - *, - batch_records: int = SHARE_JSON_BATCH_RECORDS, - chunk_chars: int = SHARE_JSON_CHUNK_BYTES, -) -> Iterator[bytes]: - """Canonical share items (sorted keys, ``default=str``) in bounded chunks. - - Joined, the chunks are byte-identical to the historical - ``b",".join(json.dumps(share, sort_keys=True, separators=(",", ":"), - default=str).encode() for share in shares)`` -- the payout window page - encoding, the digest input framed by ``[``/``]``, and the daemon's - ``canonical_items`` stream. - """ - return iter_json_array_byte_chunks( - shares, - sort_keys=True, - default=str, - batch_records=batch_records, - chunk_chars=chunk_chars, - ) - - -def canonical_share_items_bytes(shares: Iterable[Any]) -> bytes: - """The complete canonical items stream for one bounded batch of shares. - - For a window page (at most the batch size) this is one ``json.dumps`` - call; callers holding a whole window must stream the chunks instead. - """ - return b"".join(iter_canonical_share_item_chunks(shares)) - - -def canonical_share_array_sha256(shares: Iterable[Any]) -> str: - """``sha256(json.dumps(list(shares), sort_keys=True, ...))`` streamed. - - Prefers a ``canonical_json_sha256`` method when the sequence carries one - (the paged window view, the daemon mirror), so an existing digest is - reused rather than recomputed; otherwise the array framing and every - record batch feed one running SHA-256 without allocating a whole - serialized array. - """ - existing = getattr(shares, "canonical_json_sha256", None) - if callable(existing): - return str(existing()) - digest = hashlib.sha256() - digest.update(b"[") - for chunk in iter_canonical_share_item_chunks(shares): - digest.update(chunk) - digest.update(b"]") - return digest.hexdigest() - - -class ShareArrayJsonSequence(Sequence): - """A plain share list with the paged view's ``canonical_json_sha256``. - - The coordinator's digest hooks (``canonical_json_sha256`` in the job - bundle, payout-state and coordinator modules, and the test-patchable - overrides wired through them) already prefer a sequence's own digest - method over serializing the value whole. Wrapping a freshly converted - list in this view routes it through that same preference, so the one - O(window) ``json.dumps`` those hooks would otherwise make becomes the - streamed digest -- without changing the hooks or what they are called - with elsewhere. - """ - - __slots__ = ("_shares",) - - def __init__(self, shares: Sequence[Any]) -> None: - self._shares = shares - - def __len__(self) -> int: - return len(self._shares) - - def __iter__(self) -> Iterator[Any]: - return iter(self._shares) - - def __getitem__(self, index: int | slice) -> Any: - return self._shares[index] - - def canonical_json_sha256(self) -> str: - return canonical_share_array_sha256(self._shares) - - -def share_array_json_view(shares: Sequence[Any]) -> Any: - """``shares`` itself when it can digest itself, else a streaming view.""" - if callable(getattr(shares, "canonical_json_sha256", None)): - return shares - return ShareArrayJsonSequence(shares) - - -def release_share_list_incrementally( - values: Any, - *, - batch_records: int = RELEASE_BATCH_RECORDS, -) -> None: - """Empty a finished per-share list in bounded slices. - - Dropping a list of a few hundred thousand records or dicts frees every - element in one refcount cascade, which CPython runs as a single C call - under the GIL -- measured at ~90-215 ms for 210k-400k parsed shares in - the #236 probe, on the same order as the JSON calls this module bounds. - Deleting from the tail in slices lets the interpreter switch threads - between batches. Only a list can be emptied in place; anything else is - left to ordinary garbage collection. - """ - if not isinstance(values, list): - return - batch_records = max(1, int(batch_records)) - while values: - del values[-batch_records:] - - -__all__ = [ - "SHARE_JSON_BATCH_RECORDS", - "SHARE_JSON_CHUNK_BYTES", - "ShareArrayJsonSequence", - "canonical_share_array_sha256", - "canonical_share_items_bytes", - "iter_canonical_share_item_chunks", - "iter_json_array_byte_chunks", - "iter_json_array_text_chunks", - "iter_json_object_text_chunks", - "release_share_list_incrementally", - "share_array_json_view", -] diff --git a/lab/prism/share_ledger.py b/lab/prism/share_ledger.py deleted file mode 100644 index 648fc42d9..000000000 --- a/lab/prism/share_ledger.py +++ /dev/null @@ -1,13106 +0,0 @@ -#!/usr/bin/env python3 -"""Single-writer accepted-share ledger helpers for the direct PRISM coordinator.""" - -from __future__ import annotations - -import json -import codecs -import copy -import hashlib -import logging -import os -import math -import shlex -import subprocess -import tempfile -import time -import traceback -import uuid -from collections.abc import Iterable, Mapping, Sequence -from contextlib import contextmanager -from dataclasses import dataclass, field, replace -from datetime import datetime, timedelta, timezone -from decimal import Decimal -from pathlib import Path -from threading import BoundedSemaphore, Lock, Thread, local -from typing import Any, Callable, ClassVar, Iterator, Protocol, runtime_checkable - -from lab.prism.candidate_codec import ( - CANDIDATE_BODY_STORAGE_VERSION, - INDEX_SPAN_ROWS, - LEGACY_CANDIDATE_STORAGE_VERSION, - CandidateBodyIntegrityError, - CandidateCodecError, - PreparedCandidateIntent, - SpoolCandidateBody, - SpoolFieldIndex, - prepare_candidate_intent, - prepared_intent_from_spool, - replay_header_from_fields, -) -from lab.prism.candidate_spool_view import is_spool_view, materialize_spool_views -from lab.prism.candidate_store import ( - BODY_READ_MAX_CHUNKS, - DEFAULT_SPOOL_RESERVATION_BYTES, - HEADER_PAGE_MAX_BYTES, - HEADER_PAGE_MAX_ROWS, - JANITOR_CHUNKS_PER_STEP, - LEGACY_HELPER_MEMORY_BYTES, - LEGACY_HELPER_TIMEOUT_SECONDS, - STALE_STAGING_SECONDS, - CandidateBodyHydrator, - CandidateBodyRef, - CandidateBodyUnavailable, - CandidateHeaderPage, - CandidateStorageError, - IncompatibleCandidateSchema, - LegacyCandidateHelper, - LegacyTransport, - SpoolAdmission, - body_chunk_sql, - body_index_rows_sql, - body_manifest_sql, - body_page_sql, - body_pages_sql, - body_spans_sql, - candidate_schema_refusal, - header_page_sql, - parse_body_page, - parse_manifest_row, - new_body_id, - reap_retired_chunks_sql, - retire_orphan_bodies_sql, - retire_staging_body_sql, - schema_capability_sql, - seal_body_sql, - stage_body_sql, -) -from lab.prism.audit_artifacts import ( - AuditArtifactConfig, - AuditArtifactStore, - CanonicalAuditBundleCorrupt, - canonical_audit_bundle_bytes, -) -from lab.prism.audit_bundle_view import ( - CanonicalAuditBundleView, - MappedSequence, - RawJsonDocument, - RawJsonRecord, - StreamedJsonString, - iter_json_chunks, - materialize_json, - streamed_sha256_json_hex, -) -from lab.prism.share_json_stream import ( - canonical_share_items_bytes, - iter_json_object_text_chunks, -) -from lab.prism.writer_lease_timing import ( # noqa: F401 - compatibility re-export - DEFAULT_WRITER_LEASE_ADOPTION_SILENCE_SECONDS, - WRITER_LEASE_GUARD_STATEMENT_TIMEOUT_SECONDS, - WRITER_LEASE_VERIFICATION_MAX_STATEMENTS, -) - - -LOGGER = logging.getLogger(__name__) - - -def _text_stream_contains(chunks: Iterable[str], marker: str) -> bool: - """``marker in "".join(chunks)`` without joining the chunks.""" - keep = max(0, len(marker) - 1) - tail = "" - for chunk in chunks: - window = tail + chunk - if marker in window: - return True - tail = window[-keep:] if keep else "" - return False - - -def _default_bundle_canonicalizer() -> Callable[[dict[str, Any]], bytes]: - # Resolved at call time: bundle_compiler pulls in coordinator_config, - # whose CTV imports reach back into this module during initialization. - from lab.prism.bundle_compiler import canonical_bundle_bytes - - return canonical_bundle_bytes - -AUDIT_BODY_REF_SCHEMA = "qbit.prism.audit-body-ref.v1" -AUDIT_BUNDLE_V2_SCHEMA = "qbit.prism.audit-bundle.v2" -AUDIT_SHARE_SEGMENT_SCHEMA = "qbit.prism.audit-share-segment.v1" -AUDIT_WINDOW_COMPLETENESS_PROOF_SCHEMA = "qbit.prism.window-completeness-proof.v1" -DEFAULT_AUDIT_SHARE_SEGMENT_SIZE = 10_000 -DEFAULT_CTV_BROADCAST_ATTEMPT_DETAIL_LIMIT = 20 -DEFAULT_CTV_BROADCAST_RETRY_BACKOFF_SECONDS = 300 -DEFAULT_INCREMENTAL_SHARE_WINDOW_PAGE_SIZE = 512 -# Startup writer-lease acquisition runs each statement under this per-attempt -# lock/statement deadline and gives up after this many timed-out attempts (see -# _run_lease_acquisition_json). 5 attempts x 5s per attempt (~25s) is sized to -# outlast a typical orphan reap rather than to guarantee one: the 15s -# idle_in_transaction_session_timeout below runs on the blocking backend's own -# clock, started when its transaction went idle and not when the successor -# began retrying, so the two are not directly comparable. In the common case -# the reap lands inside the budget and a later attempt acquires the lease with -# no operator action; a budget that runs out reaches the fatal error, which is -# still the point — a bounded visible failure instead of an unbounded wait. -DEFAULT_LEASE_ACQUIRE_ATTEMPTS = 5 -DEFAULT_LEASE_ACQUIRE_LOCK_TIMEOUT_SECONDS = 5.0 -# Session guards every coordinator Postgres session carries (see -# PostgresSessionGuards). idle_in_transaction_session_timeout makes the server -# abort a transaction whose client vanished between statements — the failure -# that otherwise leaves the qbit_ledger_writer_lease row lock held until TCP -# keepalive teardown (hours at OS defaults). The server-side tcp_keepalives_* -# GUCs are the backstop for backends idle *outside* a transaction, where the -# idle-in-transaction timer does not apply: on a TCP connection whose platform -# implements them, the defaults bound the server's teardown of a socket toward -# a vanished client at 30 + 3x10 = 60 seconds. Unix-socket connections ignore -# them, leaving the idle-in-transaction timeout as the guard there. -DEFAULT_POSTGRES_IDLE_IN_TRANSACTION_TIMEOUT_SECONDS = 15.0 -# Records decoded per batch by the row-result read the payout-window snapshot -# and delta run through (issue #236). Both production backends turn one -# SELECT into one JSON value per row and convert those rows this many at a -# time, rechecking the caller's deadline and stamping liveness between -# batches, so no single decode call grows with the window. The figure bounds -# the size of each C-level decode call and the gap between deadline checks, -# not the result: the complete snapshot is still returned as one list, and -# the driver still buffers the raw rows. -PAYOUT_WINDOW_ROW_BATCH_SIZE = 512 -# read_replica_status() runs on the public read service's background probe -# thread on a 5s cadence by default; a probe that outlives its own interval -# tells the freshness gate nothing it does not already know from the previous -# answer's age, so bound it well inside one interval. -DEFAULT_READ_REPLICA_PROBE_TIMEOUT_SECONDS = 3.0 -DEFAULT_POSTGRES_TCP_KEEPALIVES_COUNT = 3 -DEFAULT_POSTGRES_TCP_KEEPALIVES_IDLE_SECONDS = 30 -DEFAULT_POSTGRES_TCP_KEEPALIVES_INTERVAL_SECONDS = 10 -# Only the coordinator assigns this prefix, and PsqlShareLedger preserves it -# only after acquiring the writer/epoch advisory guard. Other ledger users and -# psql-only deployments retain ordinary TTL fencing and are never treated as -# fast-adoptable merely because they share an identity with a replacement. -WRITER_LEASE_HEARTBEAT_SESSION_PREFIX = "heartbeat-v1:" -# The third arm of _try_acquire_writer_lease's COALESCE. Both of the first two -# arms are empty at once in exactly one reachable case: the first-ever -# concurrent acquisition against an empty lease table, where the loser's -# ON CONFLICT DO UPDATE affects no rows and the holder SELECT still reads the -# statement snapshot it took before the winner committed. That is a retry -# signal local to this one statement -- a fresh statement snapshot sees the -# committed row -- so the statement names it instead of evaluating to SQL NULL -# and reaching the generic no-JSON parser error. -WRITER_LEASE_ACQUIRE_RETRY_KEY = "lease_snapshot_retry" -WRITER_LEASE_ACQUIRE_RETRY_SUBJECT = "qbit ledger writer lease" -# Attempts spent taking a fresh statement snapshot before the race is treated -# as something other than the transient it is. Two attempts suffice under -# READ COMMITTED (the second statement's snapshot postdates the winner's -# commit); the third covers a re-raced row and bounds a caller that pinned an -# older snapshot for the whole transaction, which no retry can advance. -WRITER_LEASE_ACQUIRE_RETRY_ATTEMPTS = 3 - - -class WriterLeaseRenewalDeferred(RuntimeError): - """An external side effect was withheld while the lease renewal is deferred. - - The guarded session is live, but its lease TTL renewal is blocked behind - this coordinator's own fenced write that has outlasted the TTL. Liveness - there assumes the write commits; a rollback would instead hand the - expired row to a queued different-identity claimant, so the fence - refuses the RPC without fencing the process. Callers retry on their own - cadence (broadcast pass interval, block-candidate outbox replay) and - succeed once a verification lands a renewal. - - Defined here, with the lease machinery, so side-effect executors like - the CTV broadcaster can pass the refusal through to their retrying - caller instead of misclassifying it as a failed attempt. - """ - - -# WRITER_LEASE_VERIFICATION_MAX_STATEMENTS and -# DEFAULT_WRITER_LEASE_ADOPTION_SILENCE_SECONDS now live in -# lab.prism.writer_lease_timing, next to the rest of the heartbeat timing -# policy they are terms of, and are re-exported above so every existing -# import site keeps working. -VALID_CREDIT_POLICIES = frozenset({"stale-grace"}) - - -class LedgerOperationTimeout(TimeoutError): - """A caller-scoped PostgreSQL deadline expired before work completed.""" - - -class ReadOnlyLedgerError(RuntimeError): - """A read-only ledger was asked for the writer lock.""" - - -class _RefusingWriterGate: - """Stands in for the writer lock on a read-only ledger. - - Every fenced statement and every writer-lock read in PsqlShareLedger -- - appends, block landing, settlement, and the O(recipients) reads like - current_owed_balances() and audit_bundle() -- reaches the lock through - ``_operation_gate(self._lock, "writer lock")``. Substituting a gate that - refuses to be acquired makes "this process never fences" a property of the - object rather than of which methods its callers happen to use, so a future - public route that reached a fenced read would fail loudly here instead of - quietly serializing dashboard traffic against the block-landing path. - """ - - __slots__ = () - - _MESSAGE = "ledger is read-only: the writer lock is not available" - - def acquire(self, timeout: float = -1.0) -> bool: - raise ReadOnlyLedgerError(self._MESSAGE) - - def release(self) -> None: - raise ReadOnlyLedgerError(self._MESSAGE) - - def __enter__(self) -> None: - raise ReadOnlyLedgerError(self._MESSAGE) - - def __exit__(self, *exc_info: object) -> None: - raise ReadOnlyLedgerError(self._MESSAGE) - - -def _is_postgres_deadline_error(error: BaseException | str) -> bool: - """Recognize backend cancellations caused by an armed caller deadline.""" - message = str(error).casefold() - sqlstate = getattr(error, "sqlstate", None) - if sqlstate is None: - sqlstate = getattr(getattr(error, "diag", None), "sqlstate", None) - normalized_sqlstate = str(sqlstate or "").upper() - if normalized_sqlstate in {"57014", "55P03"}: - return True - # psql's verbose mode exposes SQLSTATE even when lc_messages localizes the - # text. This helper is called only while our statement/lock deadlines are - # armed, and ledger SQL does not use NOWAIT, so scoped 55P03 is a timeout. - if "57014:" in message or "55p03:" in message: - return True - return any( - marker in message - for marker in ( - "canceling statement due to statement timeout", - "canceling statement due to lock timeout", - "connection timeout expired", - "timeout expired", - "connection timed out", - "operation timed out", - ) - ) - - -def validate_credit_policy(credit_policy: str | None) -> str | None: - if credit_policy is None: - return None - if credit_policy not in VALID_CREDIT_POLICIES: - raise ValueError(f"unsupported credit_policy: {credit_policy!r}") - return credit_policy - - -@dataclass(frozen=True) -class AcceptedShareRecord: - share_seq: int - share_id: str - miner_id: str - order_key: str - p2mr_program_hex: str - share_difficulty: int - network_difficulty: int - template_height: int - job_id: str - job_issued_at_ms: int - accepted_at_ms: int - ntime: int - credit_policy: str | None = None - # Append-result metadata is deliberately excluded from the durable/public - # share identity. It lets the coordinator make process-local accounting - # idempotent and observe the candidate state from the same transaction - # that established the pre-submit outbox boundary. - newly_inserted: bool = field(default=True, compare=False, repr=False) - candidate_outbox_state: str | None = field( - default=None, - compare=False, - repr=False, - ) - - def to_prism_json(self) -> dict[str, object]: - payload: dict[str, object] = { - "share_seq": self.share_seq, - "share_id": self.share_id, - "miner_id": self.miner_id, - "order_key": self.order_key, - "p2mr_program_hex": self.p2mr_program_hex, - "share_difficulty": self.share_difficulty, - "network_difficulty": self.network_difficulty, - "template_height": self.template_height, - "job_id": self.job_id, - "job_issued_at_ms": self.job_issued_at_ms, - "accepted_at_ms": self.accepted_at_ms, - "ntime": self.ntime, - } - if self.credit_policy is not None: - payload["credit_policy"] = self.credit_policy - return payload - - -@dataclass(frozen=True) -class PrismWindowShare: - share: AcceptedShareRecord - counted_difficulty: Decimal - - -@dataclass(frozen=True) -class PendingShare: - share_id: str - miner_id: str - order_key: str - p2mr_program_hex: str - share_difficulty: int - network_difficulty: int - template_height: int - job_id: str - job_issued_at_ms: int - accepted_at_ms: int - ntime: int - credit_policy: str | None = None - - -class IncrementalWindowFallback(RuntimeError): - """The cached payout window cannot be advanced from an append-only delta.""" - - -@dataclass(frozen=True) -class IncrementalWindowAdvanceStats: - """Bounded work performed while advancing one cached payout window.""" - - added_rows: int - expired_rows: int - touched_pages: int - - -@dataclass(frozen=True) -class _IncrementalShareWindowPage: - records: tuple[AcceptedShareRecord, ...] - total_difficulty: int - prism_json_records: tuple[dict[str, object], ...] - canonical_json_items: bytes - - @classmethod - def from_records( - cls, - records: tuple[AcceptedShareRecord, ...], - ) -> _IncrementalShareWindowPage: - prism_json_records = tuple(record.to_prism_json() for record in records) - # One bounded json.dumps per page (the page is at most one batch), - # byte-identical to encoding each record alone and joining with ",". - return cls( - records=records, - total_difficulty=sum(int(record.share_difficulty) for record in records), - prism_json_records=prism_json_records, - canonical_json_items=canonical_share_items_bytes(prism_json_records), - ) - - -@dataclass(frozen=True) -class IncrementalShareJsonSequence(Sequence[dict[str, object]]): - """Immutable JSON view backed by the payout window's persistent pages. - - Advancing a window allocates JSON only for append/head boundary pages. - Iteration remains wire-identical to the historical flat tuple, while the - canonical digest streams already-encoded page fragments without calling - ``to_prism_json`` on retained shares. - """ - - pages: tuple[_IncrementalShareWindowPage, ...] - record_count: int - - def __len__(self) -> int: - return self.record_count - - def __iter__(self) -> Iterator[dict[str, object]]: - for page in self.pages: - yield from page.prism_json_records - - def __getitem__( - self, - index: int | slice, - ) -> dict[str, object] | tuple[dict[str, object], ...]: - if isinstance(index, slice): - return tuple(self)[index] - resolved = index - if resolved < 0: - resolved += self.record_count - if resolved < 0 or resolved >= self.record_count: - raise IndexError(index) - for page in self.pages: - if resolved < len(page.prism_json_records): - return page.prism_json_records[resolved] - resolved -= len(page.prism_json_records) - raise IndexError(index) - - def canonical_json_sha256(self) -> str: - digest = hashlib.sha256() - digest.update(b"[") - needs_separator = False - for page in self.pages: - if not page.canonical_json_items: - continue - if needs_separator: - digest.update(b",") - digest.update(page.canonical_json_items) - needs_separator = True - digest.update(b"]") - return digest.hexdigest() - - -@dataclass(frozen=True) -class IncrementalShareWindow: - """Immutable paged cache of the exact whole-share payout-window superset. - - Pages are stable in ascending ``share_seq`` order. Normal advancement only - extends the newest page and expires weight from the oldest pages; retained - interior pages are never revisited. The final whole share crossing - ``window_weight`` is deliberately retained, matching the Postgres oracle. - """ - - anchor_job_issued_at_ms: int - window_weight: int - page_size: int - pages: tuple[_IncrementalShareWindowPage, ...] - total_difficulty: int - - @classmethod - def from_full_snapshot( - cls, - records: list[AcceptedShareRecord] | tuple[AcceptedShareRecord, ...], - *, - anchor_job_issued_at_ms: int, - window_weight: int, - page_size: int = DEFAULT_INCREMENTAL_SHARE_WINDOW_PAGE_SIZE, - ) -> IncrementalShareWindow: - """Build cache state from the full-rescan oracle. - - Applying the exact crossing-row cutoff here normalizes full-oracle - implementations and also provides the authoritative reset after an - incremental invariant failure. - """ - - anchor_job_issued_at_ms = int(anchor_job_issued_at_ms) - window_weight = int(window_weight) - page_size = int(page_size) - if window_weight <= 0: - raise ValueError("window_weight must be positive") - if page_size <= 0: - raise ValueError("page_size must be positive") - - eligible = sorted( - ( - record - for record in records - if int(record.job_issued_at_ms) <= anchor_job_issued_at_ms - and int(record.accepted_at_ms) <= anchor_job_issued_at_ms - ), - key=lambda record: int(record.share_seq), - ) - prior_seq: int | None = None - share_ids: set[str] = set() - for record in eligible: - share_seq = int(record.share_seq) - if prior_seq is not None and share_seq <= prior_seq: - raise ValueError("full payout window contains duplicate share_seq") - if record.share_id in share_ids: - raise ValueError("full payout window contains duplicate share_id") - if int(record.share_difficulty) <= 0: - raise ValueError("full payout window contains non-positive difficulty") - prior_seq = share_seq - share_ids.add(record.share_id) - - start = len(eligible) - retained_weight = 0 - for index in range(len(eligible) - 1, -1, -1): - if retained_weight >= window_weight: - break - retained_weight += int(eligible[index].share_difficulty) - start = index - retained = tuple(eligible[start:]) - pages = tuple( - _IncrementalShareWindowPage.from_records( - retained[offset : offset + page_size] - ) - for offset in range(0, len(retained), page_size) - ) - return cls( - anchor_job_issued_at_ms=anchor_job_issued_at_ms, - window_weight=window_weight, - page_size=page_size, - pages=pages, - total_difficulty=retained_weight, - ) - - def records(self) -> tuple[AcceptedShareRecord, ...]: - return tuple(record for page in self.pages for record in page.records) - - @property - def record_count(self) -> int: - """Retained record total; shared surface with DaemonShareWindowMirror.""" - return sum(len(page.records) for page in self.pages) - - def json_records(self) -> IncrementalShareJsonSequence: - return IncrementalShareJsonSequence( - pages=self.pages, - record_count=sum(len(page.records) for page in self.pages), - ) - - def advance( - self, - delta_records: list[AcceptedShareRecord] - | tuple[AcceptedShareRecord, ...], - *, - anchor_job_issued_at_ms: int, - ) -> tuple[IncrementalShareWindow, IncrementalWindowAdvanceStats]: - """Fold one newly eligible append delta and expire only the old edge.""" - - anchor_job_issued_at_ms = int(anchor_job_issued_at_ms) - if anchor_job_issued_at_ms < self.anchor_job_issued_at_ms: - raise IncrementalWindowFallback("snapshot anchor moved backwards") - - delta = tuple(delta_records) - prior_seq = ( - int(self.pages[-1].records[-1].share_seq) if self.pages else None - ) - prior_delta_seq: int | None = None - for record in delta: - share_seq = int(record.share_seq) - if int(record.share_difficulty) <= 0: - raise IncrementalWindowFallback( - "delta contains non-positive share difficulty" - ) - if ( - int(record.job_issued_at_ms) > anchor_job_issued_at_ms - or int(record.accepted_at_ms) > anchor_job_issued_at_ms - ): - raise IncrementalWindowFallback( - "delta contains a share ineligible at the new anchor" - ) - if ( - int(record.job_issued_at_ms) <= self.anchor_job_issued_at_ms - and int(record.accepted_at_ms) <= self.anchor_job_issued_at_ms - ): - raise IncrementalWindowFallback( - "delta repeats a share eligible at the previous anchor" - ) - if prior_seq is not None and share_seq <= prior_seq: - raise IncrementalWindowFallback( - "newly eligible share is not an append" - ) - if prior_delta_seq is not None and share_seq <= prior_delta_seq: - raise IncrementalWindowFallback("delta share_seq order is not increasing") - prior_delta_seq = share_seq - - # Copy page references, never their retained contents. ``touched`` is - # intentionally defined as pre-existing retained boundary pages whose - # records must be inspected or rewritten; newly allocated append pages - # and pages expired wholesale are not part of the retained interior. - pages = list(self.pages) - touched_existing_pages: set[int] = set() - delta_offset = 0 - if delta and pages and len(pages[-1].records) < self.page_size: - available = self.page_size - len(pages[-1].records) - appended = delta[:available] - if appended: - pages[-1] = _IncrementalShareWindowPage.from_records( - pages[-1].records + appended - ) - delta_offset = len(appended) - touched_existing_pages.add(len(self.pages) - 1) - while delta_offset < len(delta): - page_records = delta[delta_offset : delta_offset + self.page_size] - pages.append(_IncrementalShareWindowPage.from_records(page_records)) - delta_offset += len(page_records) - - total_difficulty = self.total_difficulty + sum( - int(record.share_difficulty) for record in delta - ) - expired_rows = 0 - first_retained_page = 0 - while ( - first_retained_page < len(pages) - and total_difficulty - - pages[first_retained_page].total_difficulty - >= self.window_weight - ): - page = pages[first_retained_page] - total_difficulty -= page.total_difficulty - expired_rows += len(page.records) - first_retained_page += 1 - original_page_offset = first_retained_page - if first_retained_page: - pages = pages[first_retained_page:] - - if pages: - first_page = pages[0] - partial_expired = 0 - while ( - partial_expired < len(first_page.records) - and total_difficulty - - int(first_page.records[partial_expired].share_difficulty) - >= self.window_weight - ): - total_difficulty -= int( - first_page.records[partial_expired].share_difficulty - ) - partial_expired += 1 - if partial_expired: - pages[0] = _IncrementalShareWindowPage.from_records( - first_page.records[partial_expired:] - ) - expired_rows += partial_expired - if original_page_offset < len(self.pages): - touched_existing_pages.add(original_page_offset) - - advanced = IncrementalShareWindow( - anchor_job_issued_at_ms=anchor_job_issued_at_ms, - window_weight=self.window_weight, - page_size=self.page_size, - pages=tuple(pages), - total_difficulty=total_difficulty, - ) - return advanced, IncrementalWindowAdvanceStats( - added_rows=len(delta), - expired_rows=expired_rows, - touched_pages=len(touched_existing_pages), - ) - - -class DaemonWindowMirrorDivergence(RuntimeError): - """The coordinator's byte mirror disagrees with what the daemon reported. - - Every mirror construction re-hashes the canonical items it holds against - the digest the daemon reported AND counts the records those bytes - actually contain against the count it reported, so a divergence between - the two implementations -- or a bug in the byte surgery itself -- becomes - a detected full-rescan instead of a silently wrong payout artifact. - Both checks are eager: nothing downstream may hold a mirror whose bytes - and metadata have not already been reconciled. - """ - - -# Bytes of one daemon canonical-items stream handed to the incremental UTF-8 -# decoder per call while the stream is walked. Read at call time so a test -# can shrink it and drive record and multi-byte boundaries across chunks. -CANONICAL_ITEMS_DECODE_CHUNK_BYTES = 64 * 1024 - - -def _walk_canonical_items( - fragment: bytes, - *, - chunk_bytes: int | None = None, - share_keys: bool = True, -) -> Iterator[dict[str, object]]: - """Yield each record of one canonical items span; return the trailing flag. - - The span is decoded in bounded chunks through an incremental UTF-8 - decoder and parsed one record at a time with ``raw_decode``, which - reports where each record's JSON ends without re-encoding anything, so - no single C call covers more than one chunk or one record whatever the - window size (#236): the count comes from the bytes themselves rather - than from a number the daemon declared, and the interpreter can switch - threads between records. A span that is not a run of complete records - separated by ``,`` -- a truncated record, a doubled or leading - separator, junk between records, invalid UTF-8 -- raises - DaemonWindowMirrorDivergence where it is reached, never a parse the - caller may retry. - - The generator's return value (``StopIteration.value``) is the trailing - flag, which distinguishes ``a,b`` from ``a,b,``: a dropped prefix - legitimately ends on a separator when records remain behind it, and a - complete stream never does. An empty span yields nothing and returns - False. - - ``share_keys`` makes every yielded record reuse one string object per - distinct key, the way a single whole-array ``json.loads`` shares keys - through its scanner memo; ``raw_decode`` clears that memo per call, and - without the hook a 400k-record parse would carry 13 private key strings - per record (about 60% more resident memory, and a proportionally longer - release). A walk that only counts records passes False and skips the - rebuild. - """ - total = len(fragment) - if not total: - return False - if chunk_bytes is None: - chunk_bytes = CANONICAL_ITEMS_DECODE_CHUNK_BYTES - chunk_bytes = max(1, int(chunk_bytes)) - view = memoryview(fragment) - utf8 = codecs.getincrementaldecoder("utf-8")() - if share_keys: - shared_keys: dict[str, str] = {} - - def rekey(record: dict[str, object]) -> dict[str, object]: - return { - shared_keys.setdefault(key, key): value - for key, value in record.items() - } - - decoder = json.JSONDecoder(object_hook=rekey) - else: - decoder = json.JSONDecoder() - offset = 0 - text = "" - position = 0 - - def refill(at_least: int = 0) -> bool: - """Decode the next chunk onto the unconsumed text; False once spent.""" - nonlocal offset, text, position - if offset >= total: - return False - end = min(offset + max(chunk_bytes, at_least), total) - try: - decoded = utf8.decode(view[offset:end], end >= total) - except UnicodeDecodeError as exc: - raise DaemonWindowMirrorDivergence( - "daemon window mirror items are not valid UTF-8" - ) from exc - offset = end - text = text[position:] + decoded - position = 0 - return True - - after_separator = False - while True: - while position >= len(text): - if not refill(): - # Spent where a record was required: only reachable right - # after a separator, so the span ended on one. (A non-empty - # span always decodes to at least one character, so the - # start never lands here.) - return after_separator - if text[position] != "{": - raise DaemonWindowMirrorDivergence( - "daemon window mirror items hold " - f"{text[position]!r} where a record was expected" - ) - while True: - try: - record, end = decoder.raw_decode(text, position) - except ValueError as exc: - # The record may merely be cut by the chunk boundary: pull - # more bytes -- at least as many as are already pending, so - # a genuinely malformed record is found in logarithmically - # many retries rather than one per chunk -- and retry from - # the same position. - if refill(len(text) - position): - continue - raise DaemonWindowMirrorDivergence( - f"daemon window mirror items hold a malformed record: {exc}" - ) from exc - break - after_separator = False - yield record - position = end - while position >= len(text): - if not refill(): - return False - if text[position] != ",": - raise DaemonWindowMirrorDivergence( - "daemon window mirror items hold " - f"{text[position]!r} where a record separator was expected" - ) - position += 1 - after_separator = True - - -def _canonical_items_layout(fragment: bytes) -> tuple[int, bool]: - """Record count and trailing-separator flag for one canonical items span. - - Walks the span record by record (see :func:`_walk_canonical_items`), - counting what the bytes actually hold and discarding each parsed record. - """ - walker = _walk_canonical_items(fragment, share_keys=False) - count = 0 - while True: - try: - next(walker) - except StopIteration as stop: - return count, bool(stop.value) - count += 1 - - -def _canonical_items_record_count(fragment: bytes) -> int: - """Record count of one complete canonical items stream.""" - count, trailing = _canonical_items_layout(fragment) - if trailing: - raise DaemonWindowMirrorDivergence( - "daemon window mirror items end on a record separator" - ) - return count - - -class DaemonShareJsonSequence(Sequence): - """Lazy, byte-backed twin of :class:`IncrementalShareJsonSequence`. - - When the daemon owns the payout-window fold, the coordinator holds only - the canonical items stream (every record's canonical JSON encoding joined - with ``,``) rather than materialized dicts. The routine build path needs - only the length, the digest, and this object's identity; the rare - consumers that genuinely need dicts (the found-block audit build, the - durable candidate intent, the one-shot builder fallback) force one - ``json.loads`` here, paying the parse exactly where the bytes are used. - Iteration order and the canonical digest are byte-identical to the paged - sequence by construction: both stream the same fragments in the same - order, and the digest framing is invariant to page layout. - - The count check below is a floor, not the contract: a sequence handed - out by :class:`DaemonShareWindowMirror` had its count reconciled with - its bytes at construction, so no consumer of a mirror can be the first - to learn of a divergence. - """ - - __slots__ = ("canonical_items", "record_count", "_parse_lock", "_parsed") - - def __init__(self, canonical_items: bytes, record_count: int) -> None: - self.canonical_items = bytes(canonical_items) - self.record_count = int(record_count) - self._parse_lock = Lock() - self._parsed: tuple[dict[str, object], ...] | None = None - - def __len__(self) -> int: - return self.record_count - - def _records(self) -> tuple[dict[str, object], ...]: - with self._parse_lock: - if self._parsed is None: - # Record at a time through the same strict walker that - # reconciled the mirror's count at construction: one - # ``raw_decode`` per record and one bounded UTF-8 decode per - # chunk, never a whole-window ``json.loads`` (#236). The - # parsed tuple is published only after every record parsed - # and the count matched, so a failure leaves nothing cached. - walker = _walk_canonical_items(self.canonical_items) - parsed: list[dict[str, object]] = [] - while True: - try: - parsed.append(next(walker)) - except StopIteration as stop: - trailing = bool(stop.value) - break - if trailing: - raise DaemonWindowMirrorDivergence( - "daemon window mirror items end on a record separator" - ) - if len(parsed) != self.record_count: - raise DaemonWindowMirrorDivergence( - "daemon window mirror parsed " - f"{len(parsed)} records where {self.record_count}" - " were declared" - ) - self._parsed = tuple(parsed) - return self._parsed - - def __iter__(self) -> Iterator[dict[str, object]]: - return iter(self._records()) - - def __getitem__( - self, - index: int | slice, - ) -> dict[str, object] | tuple[dict[str, object], ...]: - return self._records()[index] - - def canonical_json_sha256(self) -> str: - digest = hashlib.sha256() - digest.update(b"[") - digest.update(self.canonical_items) - digest.update(b"]") - return digest.hexdigest() - - -@dataclass(frozen=True) -class DaemonShareWindowMirror: - """Coordinator-held opaque mirror of one daemon-prepared payout window. - - Exposes the same identity surface as :class:`IncrementalShareWindow` - (anchor, weight, page size, ``record_count``) so the payout-state cache - policy code reads either interchangeably, while the window contents stay - pre-encoded bytes the daemon produced. Advancing applies the daemon's - reported byte surgery -- drop a prefix, append a suffix -- and every - construction verifies the resulting stream hashes to the daemon's digest - before anything downstream may consume it. - """ - - anchor_job_issued_at_ms: int - window_weight: int - page_size: int - record_count: int - canonical_items: bytes = field(repr=False) - share_snapshot_sha256: str - - @staticmethod - def _verified_items_digest(canonical_items: bytes, declared_digest: str) -> None: - digest = hashlib.sha256() - digest.update(b"[") - digest.update(canonical_items) - digest.update(b"]") - if digest.hexdigest() != declared_digest: - raise DaemonWindowMirrorDivergence( - "daemon window mirror bytes do not hash to the daemon's digest" - ) - - @staticmethod - def _verified_record_count(counted: int, declared_count: int) -> None: - """Pin the declared count to one counted from the mirror's own bytes. - - The digest above pins the bytes but says nothing about the count the - daemon reported alongside them, and the count is the one divergence - that would otherwise surface only when some consumer first parsed the - stream -- arbitrarily far from the materialization that produced it, - and after a partial request may already be on the daemon's stdin. - Counting eagerly here puts it beside the digest check, where every - construction pays it once and no consumer can be surprised. - """ - if counted != int(declared_count): - raise DaemonWindowMirrorDivergence( - f"daemon window mirror holds {counted} records where " - f"{int(declared_count)} were declared" - ) - - @classmethod - def from_full_items( - cls, - *, - anchor_job_issued_at_ms: int, - window_weight: int, - page_size: int, - record_count: int, - canonical_items: bytes, - share_snapshot_sha256: str, - ) -> DaemonShareWindowMirror: - cls._verified_items_digest(canonical_items, share_snapshot_sha256) - cls._verified_record_count( - _canonical_items_record_count(canonical_items), - record_count, - ) - return cls( - anchor_job_issued_at_ms=int(anchor_job_issued_at_ms), - window_weight=int(window_weight), - page_size=int(page_size), - record_count=int(record_count), - canonical_items=bytes(canonical_items), - share_snapshot_sha256=share_snapshot_sha256, - ) - - def _advanced_record_count( - self, - retained_drop_bytes: int, - appended_items: bytes, - ) -> int: - """Count an advance's records from the surgery, not the whole stream. - - Only the dropped prefix and the appended suffix are walked, so this - costs what the byte surgery itself costs rather than what a rescan of - the retained window would. The already-verified count of this mirror - supplies the retained middle, and the separator placement at both - seams is checked: a drop or an append landing inside a record would - otherwise let a miscount masquerade as an aligned edit. - """ - dropped, dropped_trailing = _canonical_items_layout( - self.canonical_items[:retained_drop_bytes] - ) - retained_items = self.canonical_items[retained_drop_bytes:] - if retained_items: - if dropped and not dropped_trailing: - raise DaemonWindowMirrorDivergence( - "daemon window advance dropped a partial record" - ) - elif dropped_trailing: - raise DaemonWindowMirrorDivergence( - "daemon window advance dropped every record but kept a" - " separator" - ) - retained = int(self.record_count) - dropped - if retained < 0: - raise DaemonWindowMirrorDivergence( - "daemon window advance dropped more records than the mirror" - " holds" - ) - if not appended_items: - return retained - if retained_items: - # The retained span already ends on a record, so the suffix must - # carry the separator that rejoins the two. - if appended_items[:1] != b",": - raise DaemonWindowMirrorDivergence( - "daemon window advance appended items without a record" - " separator" - ) - appended = _canonical_items_record_count(appended_items[1:]) - else: - appended = _canonical_items_record_count(appended_items) - if not appended: - # A suffix that carried only a separator would leave the stream - # ending on one, which no complete window ever does. - raise DaemonWindowMirrorDivergence( - "daemon window advance appended a separator with no record" - ) - return retained + appended - - def advanced( - self, - *, - anchor_job_issued_at_ms: int, - record_count: int, - retained_drop_bytes: int, - appended_items: bytes, - share_snapshot_sha256: str, - ) -> DaemonShareWindowMirror: - retained_drop_bytes = int(retained_drop_bytes) - if retained_drop_bytes < 0 or retained_drop_bytes > len(self.canonical_items): - raise DaemonWindowMirrorDivergence( - "daemon window advance dropped more bytes than the mirror holds" - ) - if retained_drop_bytes == 0 and not appended_items: - # Anchor-only advance: the stream is byte-identical, so a digest - # string comparison replaces the copy and the re-hash, and the - # count must be unchanged for the same reason. - canonical_items = self.canonical_items - if share_snapshot_sha256 != self.share_snapshot_sha256: - raise DaemonWindowMirrorDivergence( - "daemon window advance changed the digest without bytes" - ) - self._verified_record_count(self.record_count, record_count) - else: - canonical_items = ( - self.canonical_items[retained_drop_bytes:] + bytes(appended_items) - ) - self._verified_items_digest(canonical_items, share_snapshot_sha256) - self._verified_record_count( - self._advanced_record_count( - retained_drop_bytes, - bytes(appended_items), - ), - record_count, - ) - return DaemonShareWindowMirror( - anchor_job_issued_at_ms=int(anchor_job_issued_at_ms), - window_weight=self.window_weight, - page_size=self.page_size, - record_count=int(record_count), - canonical_items=canonical_items, - share_snapshot_sha256=share_snapshot_sha256, - ) - - def json_records(self) -> DaemonShareJsonSequence: - return DaemonShareJsonSequence( - canonical_items=self.canonical_items, - record_count=self.record_count, - ) - - -@dataclass(frozen=True) -class BlockCandidateIntentPersistResult: - inserted: bool - state: str - - def __bool__(self) -> bool: - return self.inserted - - -def _block_candidate_cursor_parts(cursor: object) -> tuple[object, str]: - """Split one opaque pending-candidate cursor into its ordering parts. - - The cursor is whatever ``pending_block_candidate_rows`` handed back on a - prior row and travels through the caller (and, for the Postgres backend, - through JSON) untouched, so it is validated on the way back in rather - than trusted. The shape is deliberately the two ordering columns and - nothing else: ordering by the creation stamp alone cannot resume, because - equal stamps would either re-emit or skip their peers. - """ - if isinstance(cursor, str) or not isinstance(cursor, Sequence): - raise ValueError("pending block candidate cursor is not a two-element list") - parts = list(cursor) - if len(parts) != 2: - raise ValueError("pending block candidate cursor is not a two-element list") - created_at, block_hash = parts - if not isinstance(block_hash, str) or not block_hash: - raise ValueError("pending block candidate cursor has no block hash") - return created_at, block_hash - - -def _normalized_block_candidate_hash_set( - block_hashes: Sequence[str], -) -> tuple[str, ...]: - """Normalize a caller-supplied block-hash set for a batch outbox write. - - Both backends key the outbox on the lowercase hash, so the same - lowercasing the single-hash terminal updates apply per call is applied - once here to the whole set. Duplicates collapse and the result is sorted - so a given input set always produces one identical target set -- - including the generated SQL text, which a caller must be able to reason - about without knowing the caller's iteration order. - """ - return tuple(sorted({str(block_hash).lower() for block_hash in block_hashes})) - - -def _memory_block_candidate_row_key(row: dict[str, Any]) -> tuple[float, str]: - """Total-order key for one in-memory pending outbox row.""" - return (float(row.get("created_monotonic", 0.0)), str(row["block_hash"])) - - -def _memory_block_candidate_cursor_key(cursor: object) -> tuple[float, str]: - """Rebuild the in-memory ordering key from a returned cursor.""" - created_monotonic, block_hash = _block_candidate_cursor_parts(cursor) - if isinstance(created_monotonic, bool) or not isinstance( - created_monotonic, (int, float) - ): - raise ValueError("pending block candidate cursor has no creation stamp") - return (float(created_monotonic), block_hash) - - -class ShareReplayConflict(RuntimeError): - """A recovery row reused a share ID with a different durable payload.""" - - -@dataclass(frozen=True) -class ShareReplayResult: - """Typed result for one legacy recovery-journal append.""" - - disposition: str - record: AcceptedShareRecord - - -class SingleWriterShareLedger: - """Assigns canonical share_seq values and returns immutable snapshots. - - The direct Stratum coordinator should append accepted shares through one - instance of this class. Later Postgres integration can keep this API shape - while moving storage to `qbit_share_ledger`. - """ - - # Bodies live in memory here, so a replay header row is hydrated at - # registration; the PostgreSQL ledger defers hydration to dequeue (#255). - candidate_hydration_deferred: ClassVar[bool] = False - - def __init__( - self, - *, - first_share_seq: int = 1, - ctv_broadcast_attempt_detail_limit: int = DEFAULT_CTV_BROADCAST_ATTEMPT_DETAIL_LIMIT, - ctv_broadcast_retry_backoff_seconds: int = DEFAULT_CTV_BROADCAST_RETRY_BACKOFF_SECONDS, - ): - if first_share_seq < 1: - raise ValueError("first_share_seq must be >= 1") - ctv_broadcast_attempt_detail_limit = int(ctv_broadcast_attempt_detail_limit) - if ctv_broadcast_attempt_detail_limit < 0: - raise ValueError("ctv_broadcast_attempt_detail_limit must be non-negative") - ctv_broadcast_retry_backoff_seconds = int(ctv_broadcast_retry_backoff_seconds) - if ctv_broadcast_retry_backoff_seconds < 0: - raise ValueError("ctv_broadcast_retry_backoff_seconds must be non-negative") - self._ctv_broadcast_attempt_detail_limit = ctv_broadcast_attempt_detail_limit - self._ctv_broadcast_retry_backoff_seconds = ctv_broadcast_retry_backoff_seconds - self._next_share_seq = first_share_seq - self._shares: list[AcceptedShareRecord] = [] - self._share_ids: set[str] = set() - self._shares_by_id: dict[str, AcceptedShareRecord] = {} - self._block_candidate_outbox: dict[str, dict[str, Any]] = {} - self._ctv_fanout_sets: dict[str, dict[str, Any]] = {} - self._ctv_fanout_statuses: dict[str, dict[str, Any]] = {} - self._ctv_fanout_attempts: dict[str, list[dict[str, Any]]] = {} - self._audit_publication_sequences: dict[str, int | None] = {} - self._next_audit_publication_sequence = 1 - self._inactive_audit_publications: set[str] = set() - self._memory_pool_blocks: dict[str, tuple[int, str, str]] = {} - self._lock = Lock() - - def append(self, pending: PendingShare) -> AcceptedShareRecord: - if pending.share_difficulty <= 0: - raise ValueError("share_difficulty must be positive") - if pending.network_difficulty <= 0: - raise ValueError("network_difficulty must be positive") - credit_policy = validate_credit_policy(pending.credit_policy) - with self._lock: - if pending.share_id in self._share_ids: - existing = self._shares_by_id[pending.share_id] - if self._pending_matches_record(pending, existing, credit_policy=credit_policy): - return replace(existing, newly_inserted=False) - raise ValueError("duplicate share_id payload mismatch") - record = AcceptedShareRecord( - share_seq=self._next_share_seq, - share_id=pending.share_id, - miner_id=pending.miner_id, - order_key=pending.order_key, - p2mr_program_hex=pending.p2mr_program_hex, - share_difficulty=pending.share_difficulty, - network_difficulty=pending.network_difficulty, - template_height=pending.template_height, - job_id=pending.job_id, - job_issued_at_ms=pending.job_issued_at_ms, - accepted_at_ms=pending.accepted_at_ms, - ntime=pending.ntime, - credit_policy=credit_policy, - ) - self._shares.append(record) - self._share_ids.add(pending.share_id) - self._shares_by_id[pending.share_id] = record - self._next_share_seq += 1 - return record - - def append_recovered_share(self, pending: PendingShare) -> ShareReplayResult: - """Append one recovery row with an explicit exact/conflict outcome.""" - if pending.share_difficulty <= 0: - raise ValueError("share_difficulty must be positive") - if pending.network_difficulty <= 0: - raise ValueError("network_difficulty must be positive") - credit_policy = validate_credit_policy(pending.credit_policy) - with self._lock: - existing = self._shares_by_id.get(pending.share_id) - if existing is not None: - if not self._pending_matches_record( - pending, - existing, - credit_policy=credit_policy, - ): - raise ShareReplayConflict( - f"recovered share payload conflicts with {pending.share_id}" - ) - return ShareReplayResult( - "exact_existing", - replace(existing, newly_inserted=False), - ) - record = AcceptedShareRecord( - share_seq=self._next_share_seq, - share_id=pending.share_id, - miner_id=pending.miner_id, - order_key=pending.order_key, - p2mr_program_hex=pending.p2mr_program_hex, - share_difficulty=pending.share_difficulty, - network_difficulty=pending.network_difficulty, - template_height=pending.template_height, - job_id=pending.job_id, - job_issued_at_ms=pending.job_issued_at_ms, - accepted_at_ms=pending.accepted_at_ms, - ntime=pending.ntime, - credit_policy=credit_policy, - ) - self._shares.append(record) - self._share_ids.add(pending.share_id) - self._shares_by_id[pending.share_id] = record - self._next_share_seq += 1 - return ShareReplayResult("inserted", replace(record)) - - @staticmethod - def _pending_matches_record( - pending: PendingShare, - record: AcceptedShareRecord, - *, - credit_policy: str | None, - ) -> bool: - return ( - pending.share_id == record.share_id - and pending.miner_id == record.miner_id - and pending.order_key == record.order_key - and pending.p2mr_program_hex.lower() == record.p2mr_program_hex.lower() - and int(pending.share_difficulty) == int(record.share_difficulty) - and int(pending.network_difficulty) == int(record.network_difficulty) - and int(pending.template_height) == int(record.template_height) - and pending.job_id == record.job_id - and int(pending.job_issued_at_ms) == int(record.job_issued_at_ms) - # accepted_at_ms is assigned when the coordinator receives an - # attempt. An exact header replay after restart gets a fresh stamp; - # the original durable row remains authoritative. - and int(pending.ntime) == int(record.ntime) - and credit_policy == record.credit_policy - ) - - def append_batch( - self, - entries: list[tuple[PendingShare, dict[str, Any] | None]], - ) -> list[AcceptedShareRecord]: - """Atomically append a small coordinator group-commit batch. - - The in-memory backend is used by tests and local demonstrations. Its - lock provides the same all-at-once visibility expected from the - Postgres implementation. - """ - records: list[AcceptedShareRecord] = [] - with self._lock: - # Validate the complete batch before mutating either collection. - seen_ids: set[str] = set() - seen_blocks: set[str] = set() - for pending, candidate in entries: - if pending.share_id in seen_ids: - raise ValueError("duplicate share_id in append batch") - seen_ids.add(pending.share_id) - credit_policy = validate_credit_policy(pending.credit_policy) - existing = self._shares_by_id.get(pending.share_id) - if existing is not None and not self._pending_matches_record( - pending, existing, credit_policy=credit_policy - ): - raise ValueError("duplicate share_id payload mismatch") - if candidate is not None: - block_hash = str(candidate.get("block_hash_hex", "")).lower() - if not block_hash: - raise ValueError("block candidate is missing block_hash_hex") - if block_hash in seen_blocks: - raise ValueError("duplicate block candidate in append batch") - seen_blocks.add(block_hash) - outbox = self._block_candidate_outbox.get(block_hash) - candidate_sha256 = block_candidate_identity_sha256(candidate) - if outbox is not None and ( - outbox["share_id"] not in {None, pending.share_id} - or - outbox["candidate_sha256"] != candidate_sha256 - or ( - outbox["candidate"] is not None - and block_candidate_identity(outbox["candidate"]) - != block_candidate_identity(candidate) - ) - ): - raise ValueError("block candidate payload mismatch") - - for pending, candidate in entries: - existing = self._shares_by_id.get(pending.share_id) - newly_inserted = existing is None - if existing is None: - credit_policy = validate_credit_policy(pending.credit_policy) - existing = AcceptedShareRecord( - share_seq=self._next_share_seq, - share_id=pending.share_id, - miner_id=pending.miner_id, - order_key=pending.order_key, - p2mr_program_hex=pending.p2mr_program_hex, - share_difficulty=pending.share_difficulty, - network_difficulty=pending.network_difficulty, - template_height=pending.template_height, - job_id=pending.job_id, - job_issued_at_ms=pending.job_issued_at_ms, - accepted_at_ms=pending.accepted_at_ms, - ntime=pending.ntime, - credit_policy=credit_policy, - ) - self._shares.append(existing) - self._share_ids.add(pending.share_id) - self._shares_by_id[pending.share_id] = existing - self._next_share_seq += 1 - candidate_state: str | None = None - if candidate is not None: - block_hash = str(candidate["block_hash_hex"]).lower() - self._block_candidate_outbox.setdefault( - block_hash, - { - "block_hash": block_hash, - "share_id": pending.share_id, - "candidate": candidate, - "candidate_sha256": block_candidate_identity_sha256(candidate), - "state": "pending", - "attempt_count": 0, - "last_error": None, - "created_monotonic": time.monotonic(), - }, - ) - self._block_candidate_outbox[block_hash]["share_id"] = pending.share_id - candidate_state = str( - self._block_candidate_outbox[block_hash]["state"] - ) - records.append( - replace( - existing, - newly_inserted=newly_inserted, - candidate_outbox_state=candidate_state, - ) - ) - return records - - def persist_block_candidate_intent( - self, - candidate: dict[str, Any], - ) -> BlockCandidateIntentPersistResult: - """Persist candidate work before a below-share-target synchronous submit.""" - block_hash = str(candidate.get("block_hash_hex", "")).lower() - if not block_hash: - raise ValueError("block candidate is missing block_hash_hex") - candidate_sha256 = block_candidate_identity_sha256(candidate) - with self._lock: - existing = self._block_candidate_outbox.get(block_hash) - if existing is not None: - if existing["candidate_sha256"] != candidate_sha256: - raise ValueError("block candidate payload mismatch") - return BlockCandidateIntentPersistResult( - inserted=False, - state=str(existing["state"]), - ) - self._block_candidate_outbox[block_hash] = { - "block_hash": block_hash, - "share_id": None, - # Stored as handed in: a PreparedCandidateIntent keeps its - # immutable share sequence, never a list copy (#255). - "candidate": candidate, - "candidate_sha256": candidate_sha256, - "state": "pending", - "attempt_count": 0, - "last_error": None, - "created_monotonic": time.monotonic(), - } - return BlockCandidateIntentPersistResult( - inserted=True, - state="pending", - ) - - def pending_block_candidates(self, *, limit: int = 32) -> list[dict[str, Any]]: - return [ - row["candidate"] - for row in self.pending_block_candidate_rows(limit=limit) - ] - - def pending_block_candidate_headers( - self, - *, - limit: int = HEADER_PAGE_MAX_ROWS, - after_cursor: object | None = None, - max_bytes: int = HEADER_PAGE_MAX_BYTES, - ) -> CandidateHeaderPage: - """Metadata-first page over the in-memory outbox (#255). - - Same total order, cursor and configured row limit as - ``pending_block_candidate_rows`` (so the query shape and the - short-page exhaustion proof match what a caller configured), but - each row carries only the bounded typed replay header. There is no - transport here, so the byte cap that bounds the PostgreSQL page's - native decode does not apply: an in-memory page is row-bounded only - and never reports a byte truncation. - """ - limit = max(1, min(int(limit), HEADER_PAGE_MAX_ROWS)) - if after_cursor is None: - fetched = self.pending_block_candidate_rows(limit=limit) - else: - fetched = self.pending_block_candidate_rows( - limit=limit, - after_cursor=after_cursor, - ) - rows: list[dict[str, Any]] = [] - truncated_by_bytes = False - for row in fetched[:limit]: - candidate = row["candidate"] - header = replay_header_from_fields(candidate) - header_bytes = len(json.dumps(header, separators=(",", ":"), default=str)) - with self._lock: - stored = self._block_candidate_outbox.get(str(row["block_hash"])) - rows.append( - { - "block_hash": str(row["block_hash"]), - "storage_version": ( - CANDIDATE_BODY_STORAGE_VERSION - if isinstance(candidate, PreparedCandidateIntent) - else LEGACY_CANDIDATE_STORAGE_VERSION - ), - "candidate_sha256": ( - str(stored["candidate_sha256"]) if stored is not None else "" - ), - "header": header, - "header_bytes": header_bytes, - "body": None, - "pool_block_exists": bool(row["pool_block_exists"]), - "cursor": row["cursor"], - } - ) - return CandidateHeaderPage( - rows=tuple(rows), - next_cursor=rows[-1]["cursor"] if rows else after_cursor, - exhausted=len(fetched) < limit and not truncated_by_bytes, - fetched=len(fetched), - truncated_by_bytes=truncated_by_bytes, - ) - - def hydrate_block_candidate_intent( - self, - row: dict[str, Any], - *, - cancelled: Callable[[], bool] | None = None, - ) -> Any: - """Return the stored intent mapping for one still-pending header row.""" - block_hash = str(row["block_hash"]).lower() - with self._lock: - stored = self._block_candidate_outbox.get(block_hash) - if stored is None or stored["state"] != "pending" or stored["candidate"] is None: - raise CandidateBodyUnavailable( - f"block candidate {block_hash} is no longer pending" - ) - return stored["candidate"] - - def pending_block_candidate_rows( - self, - *, - limit: int = 32, - after_cursor: object | None = None, - ) -> list[dict[str, Any]]: - """Return pending payloads together with their authoritative row keys. - - Rows carry an opaque ``cursor`` the caller passes back verbatim as - ``after_cursor`` to resume strictly after that row. The order is the - total order ``(created, block_hash)``, so a page shorter than - ``limit`` proves no further pending row existed at query time and a - backlog of any size enumerates completely in bounded pages. The - block-hash tiebreak is what makes it a *total* order: creation - stamps collide (``time.monotonic`` here, one transaction's - ``clock_timestamp`` in Postgres), and a cursor on a colliding stamp - alone would either replay or skip its peers. - - Each row also carries ``pool_block_exists``: whether a durable - ``qbit_pool_blocks`` row exists for that hash, i.e. whether the - candidate ever reached ``persist_accepted_block``. It is answered - inside this page read -- one bounded existence probe per returned row - -- because the alternative is one round trip per row, and a page is - read precisely when the backlog is large. The fact is advisory by the - time the caller holds it; the terminal batch update re-checks it under - the writer fence. - """ - after = ( - None - if after_cursor is None - else _memory_block_candidate_cursor_key(after_cursor) - ) - with self._lock: - ordered = sorted( - ( - (_memory_block_candidate_row_key(row), row) - for row in self._block_candidate_outbox.values() - if row["state"] == "pending" - ), - key=lambda entry: entry[0], - ) - return [ - { - "block_hash": str(row["block_hash"]), - "candidate": ( - dict(row["candidate"]) - if isinstance(row["candidate"], dict) - else row["candidate"] - ), - "pool_block_exists": ( - str(row["block_hash"]) in self._memory_pool_blocks - ), - "cursor": list(key), - } - for key, row in ordered - if after is None or key > after - ][:limit] - - def block_candidate_pending_metrics(self) -> dict[str, int | float]: - """Return bounded pending-candidate age gauges without exposing hashes.""" - now = time.monotonic() - with self._lock: - pending = [ - row - for row in self._block_candidate_outbox.values() - if row["state"] == "pending" - ] - unattempted = [ - row for row in pending if int(row["attempt_count"]) == 0 - ] - - def oldest_age(rows: list[dict[str, Any]]) -> float: - return max( - ( - max(0.0, now - float(row.get("created_monotonic", now))) - for row in rows - ), - default=0.0, - ) - - return { - "pending_count": len(pending), - "oldest_pending_age_seconds": oldest_age(pending), - "oldest_unattempted_age_seconds": oldest_age(unattempted), - } - - def mark_block_candidate_attempted(self, *, block_hash: str) -> bool: - """Record admission to a real processing phase for one durable row.""" - with self._lock: - row = self._block_candidate_outbox.get(block_hash.lower()) - if row is None or row["state"] != "pending": - return False - row["attempt_count"] = int(row["attempt_count"]) + 1 - return True - - def mark_block_candidate_submitted(self, *, block_hash: str) -> bool: - return self._finish_block_candidate(block_hash=block_hash, state="submitted", error=None) - - def mark_block_candidate_abandoned(self, *, block_hash: str, error: str) -> bool: - return self._finish_block_candidate(block_hash=block_hash, state="abandoned", error=error) - - def _finish_block_candidate(self, *, block_hash: str, state: str, error: str | None) -> bool: - with self._lock: - row = self._block_candidate_outbox.get(block_hash.lower()) - if row is None or row["state"] != "pending": - return False - row["state"] = state - row["last_error"] = error - row["candidate"] = None - return True - - def mark_block_candidates_abandoned( - self, - *, - block_hashes: Sequence[str], - error: str, - ) -> tuple[str, ...]: - """Terminally abandon a caller-supplied page of pending rows at once. - - Mirrors ``mark_block_candidate_abandoned`` for a set: only rows whose - current state is exactly ``pending`` transition, and the return value - is the normalized hashes this call actually transitioned rather than a - count, so the caller can restrict any follow-up cleanup to the rows it - won. Already-terminal and missing hashes are neither returned nor - mutated, and an empty set is a no-op. - - A hash that owns a durable pool-block row is also left alone: that row - only exists for a candidate that was offered and landed, so it is not - a superseded sibling regardless of what the caller observed earlier. - - The whole page runs under one lock acquisition so the returned set is - the outcome of a single atomic decision, matching the Postgres backend - where the same page is one fenced statement. - """ - targets = _normalized_block_candidate_hash_set(block_hashes) - if not targets: - return () - abandoned: list[str] = [] - with self._lock: - for block_hash in targets: - row = self._block_candidate_outbox.get(block_hash) - if row is None or row["state"] != "pending": - continue - if block_hash in self._memory_pool_blocks: - continue - row["state"] = "abandoned" - row["last_error"] = error - row["candidate"] = None - abandoned.append(block_hash) - return tuple(abandoned) - - def snapshot_at_job_issue( - self, - anchor_job_issued_at_ms: int, - *, - window_weight: int | None = None, - ) -> list[AcceptedShareRecord]: - with self._lock: - eligible = [ - replace(share) - for share in self._shares - if share.job_issued_at_ms <= anchor_job_issued_at_ms - and share.accepted_at_ms <= anchor_job_issued_at_ms - ] - if window_weight is None: - return eligible - # Match the Postgres oracle's exact whole-share crossing rule. This - # keeps synchronous and incremental artifacts byte-identical in local - # and embedded deployments instead of treating the bound as a hint. - return list( - IncrementalShareWindow.from_full_snapshot( - eligible, - anchor_job_issued_at_ms=anchor_job_issued_at_ms, - window_weight=int(window_weight), - ).records() - ) - - def snapshot_between_job_issues( - self, - previous_anchor_job_issued_at_ms: int, - anchor_job_issued_at_ms: int, - ) -> list[AcceptedShareRecord]: - """Return shares becoming eligible between two inclusive anchors.""" - - previous_anchor = int(previous_anchor_job_issued_at_ms) - anchor = int(anchor_job_issued_at_ms) - if anchor < previous_anchor: - raise ValueError("snapshot anchor moved backwards") - if anchor == previous_anchor: - return [] - with self._lock: - return [ - replace(share) - for share in self._shares - if share.job_issued_at_ms <= anchor - and share.accepted_at_ms <= anchor - and ( - share.job_issued_at_ms > previous_anchor - or share.accepted_at_ms > previous_anchor - ) - ] - - def all_shares(self) -> list[AcceptedShareRecord]: - with self._lock: - return [replace(share) for share in self._shares] - - def accepted_share_stats(self) -> dict[str, int]: - with self._lock: - return { - "accepted_share_count": len(self._shares), - "distinct_miner_count": len({share.miner_id for share in self._shares}), - } - - def current_owed_balances(self) -> list[dict[str, object]]: - return [] - - def current_prior_balances(self) -> list[dict[str, object]]: - return [] - - def carry_forward_integrity_report(self) -> dict[str, object]: - return { - "schema": "qbit.prism.carry-forward-integrity.v1", - "backend": "memory", - "checked_active_rows": 0, - "audit_chain_version": "qbit.prism.carry-forward-active-delta-chain.v1", - "audit_row_count": 0, - "audit_head_sha256": "00" * 32, - "mismatch_count": 0, - "current_drift_count": 0, - "current_drift": [], - "mismatches": [], - } - - def audit_share_window( - self, - *, - anchor_job_issued_at_ms: int, - network_difficulty: int, - ) -> list[dict[str, object]]: - return [] - - def audit_block_payouts(self, *, block_hash: str) -> list[dict[str, object]]: - return [] - - def recipient_payout_history(self, *, recipient_id: str, limit: int = 50) -> list[dict[str, object]]: - return [] - - def audit_bundle(self, *, block_hash: str) -> dict[str, object] | None: - return None - - def audit_bundle_by_commitment(self, *, commitment_leaf_hex: str) -> dict[str, object] | None: - return None - - def persist_ctv_fanout_manifest_set( - self, - *, - block_hash: str, - manifest_set: Mapping[str, Any], - manifest_set_sha256: str, - ) -> dict[str, int | str]: - # The in-memory ledger holds its state as plain objects by design; - # a streamed payload (lazy manifests from a bounded artifact view) - # is materialized here, in this test-and-parity backend only. - payload = materialize_json( - ctv_fanout_recovery_payload( - block_hash=block_hash, - manifest_set=manifest_set, - manifest_set_sha256=manifest_set_sha256, - ) - ) - with self._lock: - existing = self._ctv_fanout_sets.get(block_hash) - if existing is not None and existing != payload: - raise RuntimeError("existing CTV fanout manifest set does not match payload") - self._ctv_fanout_sets[block_hash] = copy.deepcopy(payload) - for artifact in payload["artifacts"]: - fanout_txid = str(artifact["fanout_txid"]) - existing_status = self._ctv_fanout_statuses.get(fanout_txid) - status_payload = { - **copy.deepcopy(artifact), - "schema": "qbit.prism.ctv-fanout-status.v1", - "block_hash": block_hash, - "manifest_set_sha256": manifest_set_sha256, - "settlement_status": existing_status.get("settlement_status", "awaiting_maturity") - if existing_status - else "awaiting_maturity", - "broadcast_attempts": self._ctv_fanout_attempts.get(fanout_txid, []), - } - if existing_status: - for key in _ctv_broadcast_summary_fields(): - if key in existing_status: - status_payload[key] = copy.deepcopy(existing_status[key]) - else: - status_payload.update(_empty_ctv_broadcast_summary()) - audit_bundle_sha256 = payload.get("audit_bundle_sha256") - if audit_bundle_sha256 is not None: - status_payload["audit_bundle_sha256"] = audit_bundle_sha256 - status_payload["broadcast_attempt_summary"] = _ctv_broadcast_attempt_summary(status_payload) - self._ctv_fanout_statuses[fanout_txid] = status_payload - return { - "backend": "memory", - "fanout_set_count": 1, - "fanout_artifact_count": len(payload["artifacts"]), - } - - def audit_ctv_fanout_manifest_set(self, *, block_hash: str) -> dict[str, object] | None: - with self._lock: - payload = self._ctv_fanout_sets.get(block_hash) - return copy.deepcopy(payload) if payload is not None else None - - def audit_ctv_fanouts(self, *, block_hash: str) -> list[dict[str, object]]: - with self._lock: - payload = self._ctv_fanout_sets.get(block_hash) - if payload is None: - return [] - return copy.deepcopy(payload["artifacts"]) - - def ctv_fanout_status(self, *, fanout_txid: str) -> dict[str, object] | None: - with self._lock: - payload = self._ctv_fanout_statuses.get(fanout_txid) - return copy.deepcopy(payload) if payload is not None else None - - def pending_ctv_fanout_statuses(self, *, limit: int = 100) -> list[dict[str, object]]: - limit = max(1, min(int(limit), 1_000)) - now = datetime.now(timezone.utc) - with self._lock: - rows = [ - copy.deepcopy(payload) - for payload in self._ctv_fanout_statuses.values() - if payload.get("settlement_status") not in {"confirmed", "reorged", "failed"} - and _ctv_broadcast_attempt_due(payload.get("next_broadcast_attempt_at"), now) - ] - rows.sort(key=lambda row: (str(row.get("block_hash", "")), int(row.get("chunk_index", 0)))) - return rows[:limit] - - def dashboard_pending_fanout_rows(self, *, page: int, limit: int) -> dict[str, object]: - from lab.prism import public_api - - now = datetime.now(timezone.utc) - with self._lock: - rows = [ - copy.deepcopy(payload) - for payload in self._ctv_fanout_statuses.values() - if payload.get("settlement_status") not in {"confirmed", "reorged", "failed"} - and _ctv_broadcast_attempt_due(payload.get("next_broadcast_attempt_at"), now) - ] - rows.sort(key=lambda row: (str(row.get("block_hash", "")), int(row.get("chunk_index", 0)))) - offset = (page - 1) * limit - return { - "pagination": public_api.pagination(page, limit, len(rows)), - "rows": rows[offset : offset + limit], - } - - def dashboard_public_artifact(self, *, sha256: str) -> dict[str, object] | None: - document = self.dashboard_public_artifact_document(sha256=sha256) - if document is None: - return None - return document.get("payload") - - def dashboard_public_artifact_document(self, *, sha256: str) -> dict[str, object] | None: - """Artifact payload plus, for manifest kinds, its canonical text. - - canonical_json is the exact serialized text the artifact's sha256 was - computed over, persisted at record time. Audit bundles are hashed - over Rust struct-order bytes that are not stored, so their - canonical_json is None and they keep the re-serialized response. - """ - with self._lock: - for payload in self._ctv_fanout_sets.values(): - if payload.get("audit_bundle_sha256") == sha256: - audit_bundle = payload.get("audit_bundle") - if not isinstance(audit_bundle, dict): - return None - return {"payload": copy.deepcopy(audit_bundle), "canonical_json": None} - if payload.get("manifest_set_sha256") == sha256: - manifest_set = payload.get("manifest_set") - if not isinstance(manifest_set, dict): - return None - manifest_set_json = payload.get("manifest_set_json") - return { - "payload": copy.deepcopy(manifest_set), - "canonical_json": manifest_set_json if isinstance(manifest_set_json, str) else None, - } - for artifact in payload.get("artifacts", []): - if not isinstance(artifact, dict): - continue - if artifact.get("manifest_sha256") == sha256: - manifest = artifact.get("manifest") - if not isinstance(manifest, dict): - return None - manifest_json = artifact.get("manifest_json") - return { - "payload": copy.deepcopy(manifest), - "canonical_json": manifest_json if isinstance(manifest_json, str) else None, - } - return None - - def update_ctv_fanout_status(self, *, fanout_txid: str, settlement_status: str) -> dict[str, int | str]: - validate_ctv_fanout_status(settlement_status) - with self._lock: - if fanout_txid not in self._ctv_fanout_statuses: - raise RuntimeError("unknown CTV fanout txid") - self._ctv_fanout_statuses[fanout_txid]["settlement_status"] = settlement_status - return {"backend": "memory", "updated_count": 1} - - def record_ctv_fanout_broadcast_attempt( - self, - *, - fanout_txid: str, - attempt_status: str, - package_tx_hexes: list[str] | None = None, - package_txids: list[str] | None = None, - submit_result: dict[str, Any] | None = None, - error: str | None = None, - ) -> dict[str, int | str]: - validate_ctv_fanout_attempt_status(attempt_status) - with self._lock: - if fanout_txid not in self._ctv_fanout_statuses: - raise RuntimeError("unknown CTV fanout txid") - attempts = self._ctv_fanout_attempts.setdefault(fanout_txid, []) - attempted_at = datetime.now(timezone.utc) - status_payload = self._ctv_fanout_statuses[fanout_txid] - total_attempts = int(status_payload.get("broadcast_attempt_count") or 0) + 1 - attempt = { - "attempt_seq": total_attempts, - "attempted_at": attempted_at, - "attempt_status": attempt_status, - "package_tx_hexes": package_tx_hexes or [], - "package_txids": package_txids or [], - "submit_result": submit_result, - "error": error, - } - if self._ctv_broadcast_attempt_detail_limit > 0: - if len(attempts) >= self._ctv_broadcast_attempt_detail_limit: - del attempts[0 : len(attempts) - self._ctv_broadcast_attempt_detail_limit + 1] - attempts.append(attempt) - counts = copy.deepcopy(status_payload.get("broadcast_attempt_status_counts") or {}) - if not isinstance(counts, dict): - counts = {} - counts[attempt_status] = int(counts.get(attempt_status) or 0) + 1 - next_attempt_at = None - retry_backoff_seconds = 0 - if attempt_status == "planned" and self._ctv_broadcast_retry_backoff_seconds > 0: - retry_backoff_seconds = self._ctv_broadcast_retry_backoff_seconds - next_attempt_at = attempted_at + timedelta(seconds=retry_backoff_seconds) - status_payload.update( - { - "broadcast_attempt_count": total_attempts, - "broadcast_attempt_detail_count": len(attempts), - "first_broadcast_attempt_at": status_payload.get("first_broadcast_attempt_at") or attempted_at, - "last_broadcast_attempt_at": attempted_at, - "last_broadcast_attempt_status": attempt_status, - "last_broadcast_package_tx_hexes": package_tx_hexes or [], - "last_broadcast_package_txids": package_txids or [], - "last_broadcast_submit_result": submit_result, - "last_broadcast_error": error, - "broadcast_attempt_status_counts": counts, - "next_broadcast_attempt_at": next_attempt_at, - "broadcast_retry_backoff_seconds": retry_backoff_seconds, - } - ) - self._ctv_fanout_statuses[fanout_txid]["broadcast_attempts"] = copy.deepcopy(attempts) - if attempt_status in {"submitted", "accepted"}: - self._ctv_fanout_statuses[fanout_txid]["settlement_status"] = "broadcast_submitted" - elif attempt_status in {"rejected", "failed"}: - self._ctv_fanout_statuses[fanout_txid]["settlement_status"] = "failed" - self._ctv_fanout_statuses[fanout_txid]["broadcast_attempt_summary"] = _ctv_broadcast_attempt_summary( - self._ctv_fanout_statuses[fanout_txid] - ) - return {"backend": "memory", "attempt_count": 1, "updated_count": 1 if attempt_status in {"submitted", "accepted", "rejected", "failed"} else 0} - - def metrics(self) -> dict[str, int]: - return { - "shares": len(self), - "blocks": 0, - "confirmed_blocks": 0, - "inactive_blocks": 0, - "rejected_blocks": 0, - "reversed_blocks": 0, - "payout_entries": 0, - "owed_accounts": 0, - "ctv_fanouts_failed": len( - [ - payload - for payload in self._ctv_fanout_statuses.values() - if payload.get("settlement_status") == "failed" - ] - ), - } - - def dashboard_pool_snapshot( - self, - *, - current_network_difficulty: int | str | Decimal, - generated_at: str, - ) -> dict[str, object]: - from lab.prism import public_api - - shares = self.all_shares() - now = datetime.now(timezone.utc) - window_weight = _reward_window_weight(current_network_difficulty) - window_shares = _prism_window_shares( - shares, - anchor_job_issued_at_ms=int(now.timestamp() * 1000), - requested_window_weight=window_weight, - ) - newest = max((row.share.accepted_at_ms for row in window_shares), default=None) - oldest = min((row.share.accepted_at_ms for row in window_shares), default=None) - return { - "hashrate_ths": { - "h1": public_api.hashrate_ths_from_difficulty( - _share_difficulty_between(shares, now - timedelta(hours=1), now), - 60 * 60, - ), - "h3": public_api.hashrate_ths_from_difficulty( - _share_difficulty_between(shares, now - timedelta(hours=3), now), - 3 * 60 * 60, - ), - "h24": public_api.hashrate_ths_from_difficulty( - _share_difficulty_between(shares, now - timedelta(hours=24), now), - 24 * 60 * 60, - ), - }, - "participants_3h": len( - { - share.miner_id - for share in shares - if now - timedelta(hours=3) <= _datetime_from_ms(share.accepted_at_ms) <= now - } - ), - "blocks_found_total": 0, - "prism_blocks_total": 0, - "blocks_reversed_total": 0, - "blocks_inactive_total": 0, - "total_mined_bits": 0, - "latest_block": None, - "reward_window": { - "window_multiplier": 8, - "requested_window_weight": public_api.decimal_string(window_weight), - "oldest_share_accepted_at": _iso_from_ms(oldest), - "newest_share_accepted_at": _iso_from_ms(newest), - "included_share_count": len(window_shares), - }, - } - - def dashboard_miner_reward_window( - self, - *, - recipient_id: str, - current_network_difficulty: int | str | Decimal, - ) -> dict[str, object]: - from lab.prism import public_api - - window_weight = _reward_window_weight(current_network_difficulty) - window_shares = _prism_window_shares( - self.all_shares(), - anchor_job_issued_at_ms=int(datetime.now(timezone.utc).timestamp() * 1000), - requested_window_weight=window_weight, - ) - miner_difficulty = sum( - row.counted_difficulty - for row in window_shares - if row.share.miner_id == recipient_id - ) - pool_difficulty = sum(row.counted_difficulty for row in window_shares) - share_percent = None - if pool_difficulty > 0: - share_percent = public_api.decimal_string(miner_difficulty * Decimal(100) / pool_difficulty) - return { - "accepted_difficulty": public_api.decimal_string(miner_difficulty), - "pool_accepted_difficulty": public_api.decimal_string(pool_difficulty), - "share_percent": share_percent, - } - - def dashboard_blocks(self, *, page: int, limit: int, chain_state: str = "active") -> dict[str, object]: - from lab.prism import public_api - - if chain_state not in public_api.BLOCKS_CHAIN_STATE_FILTERS: - raise ValueError("chain_state must be one of active, all, reversed") - return {"pagination": public_api.pagination(page, limit, 0), "rows": []} - - def dashboard_miner_lifetime_earnings_bits(self, *, recipient_id: str) -> int: - return 0 - - def dashboard_miner_pending_maturity_bits(self, *, recipient_id: str) -> int: - return 0 - - def dashboard_miner_payout_rows(self, *, recipient_id: str, page: int, limit: int) -> dict[str, object]: - from lab.prism import public_api - - return {"pagination": public_api.pagination(page, limit, 0), "rows": []} - - def dashboard_miner_earning_rows(self, *, recipient_id: str, page: int, limit: int) -> dict[str, object]: - from lab.prism import public_api - - return {"pagination": public_api.pagination(page, limit, 0), "rows": []} - - def dashboard_leaderboard(self, *, page: int, limit: int, search: str | None = None) -> dict[str, object]: - from lab.prism import public_api - - now = datetime.now(timezone.utc) - started = now - timedelta(hours=3) - shares = [ - share - for share in self.all_shares() - if started <= _datetime_from_ms(share.accepted_at_ms) <= now - and (not search or search.lower() in share.miner_id.lower()) - ] - by_miner: dict[str, dict[str, object]] = {} - for share in shares: - row = by_miner.setdefault( - share.miner_id, - { - "recipient_id": share.miner_id, - "difficulty": 0, - "last_share_at_ms": share.accepted_at_ms, - }, - ) - row["difficulty"] = int(row["difficulty"]) + int(share.share_difficulty) - row["last_share_at_ms"] = max(int(row["last_share_at_ms"]), share.accepted_at_ms) - total_difficulty = sum(int(row["difficulty"]) for row in by_miner.values()) - pool_hashrate_ths = public_api.hashrate_ths_from_difficulty(total_difficulty, 3 * 60 * 60) - ranked = sorted( - by_miner.values(), - key=lambda row: (-int(row["difficulty"]), str(row["recipient_id"])), - ) - rows: list[dict[str, object]] = [] - for index, row in enumerate(ranked, start=1): - percent = None - if total_difficulty: - percent = public_api.decimal_string(Decimal(int(row["difficulty"])) * Decimal(100) / Decimal(total_difficulty)) - hashrate_ths = public_api.hashrate_ths_from_difficulty(int(row["difficulty"]), 3 * 60 * 60) - rows.append( - { - "rank": index, - "recipient_id": row["recipient_id"], - "display_name": None, - "hashrate_ths_3h": hashrate_ths, - "share_percent": percent, - "hash_percent": _hash_percent(hashrate_ths, pool_hashrate_ths), - "blocks_found": 0, - "last_share_at": _iso_from_ms(int(row["last_share_at_ms"])), - } - ) - offset = (page - 1) * limit - return { - "started_at": public_api.iso_datetime(started), - "ended_at": public_api.iso_datetime(now), - "totals": { - "pool_hashrate_ths": pool_hashrate_ths, - "pool_accepted_share_difficulty": str(total_difficulty), - "participant_count": len(ranked), - }, - "pagination": public_api.pagination(page, limit, len(ranked)), - "rows": rows[offset : offset + limit], - } - - def dashboard_reward_leaderboard( - self, - *, - page: int, - limit: int, - current_network_difficulty: int | str | Decimal, - search: str | None = None, - recipient_id: str | None = None, - ) -> dict[str, object]: - from lab.prism import public_api - - if search is not None and recipient_id is not None: - raise ValueError("search and recipient_id are mutually exclusive") - now = datetime.now(timezone.utc) - anchor_ms = int(now.timestamp() * 1000) - requested_window_weight = _reward_window_weight(current_network_difficulty) - window_shares = _prism_window_shares( - self.all_shares(), - anchor_job_issued_at_ms=anchor_ms, - requested_window_weight=requested_window_weight, - ) - counted_window_weight = sum( - (row.counted_difficulty for row in window_shares), - Decimal(0), - ) - oldest_ms = min((row.share.accepted_at_ms for row in window_shares), default=None) - observed_span_seconds = None - if oldest_ms is not None: - observed_span_seconds = max(0, (anchor_ms - oldest_ms) // 1000) - - by_miner: dict[str, dict[str, object]] = {} - for window_share in window_shares: - share = window_share.share - row = by_miner.setdefault( - share.miner_id, - { - "recipient_id": share.miner_id, - "included_share_count": 0, - "counted_share_difficulty": Decimal(0), - "last_share_at_ms": share.accepted_at_ms, - }, - ) - row["included_share_count"] = int(row["included_share_count"]) + 1 - row["counted_share_difficulty"] = Decimal(row["counted_share_difficulty"]) + window_share.counted_difficulty - row["last_share_at_ms"] = max(int(row["last_share_at_ms"]), share.accepted_at_ms) - - ranked = sorted( - by_miner.values(), - key=lambda row: (-Decimal(row["counted_share_difficulty"]), str(row["recipient_id"])), - ) - rows: list[dict[str, object]] = [] - for index, row in enumerate(ranked, start=1): - counted_share_difficulty = Decimal(row["counted_share_difficulty"]) - share_percent = None - if counted_window_weight > 0: - share_percent = public_api.decimal_string( - counted_share_difficulty * Decimal(100) / counted_window_weight - ) - hashrate_ths = None - if observed_span_seconds is not None and observed_span_seconds > 0: - hashrate_ths = public_api.hashrate_ths_from_difficulty( - counted_share_difficulty, - observed_span_seconds, - ) - rows.append( - { - "rank": index, - "recipient_id": row["recipient_id"], - "display_name": None, - "hashrate_ths": hashrate_ths, - "included_share_count": int(row["included_share_count"]), - "counted_share_difficulty": public_api.decimal_string(counted_share_difficulty), - "share_percent": share_percent, - "blocks_found_total": 0, - "last_share_at": _iso_from_ms(int(row["last_share_at_ms"])), - } - ) - - filtered_rows = rows - if recipient_id is not None: - filtered_rows = [ - row - for row in rows - if row["recipient_id"] == recipient_id - ] - elif search: - normalized_search = search.lower() - filtered_rows = [ - row - for row in rows - if normalized_search in str(row["recipient_id"]).lower() - ] - offset = (page - 1) * limit - - pool_hashrate_ths = None - expected_time_to_block = None - if observed_span_seconds is not None and observed_span_seconds > 0 and counted_window_weight > 0: - pool_hashrate_ths = public_api.hashrate_ths_from_difficulty( - counted_window_weight, - observed_span_seconds, - ) - expected_time_to_block = public_api.expected_time_to_block_seconds( - hashrate_ths=pool_hashrate_ths, - network_difficulty=public_api.decimal_string(current_network_difficulty), - ) - - return { - "window": { - "id": "reward", - "started_at": _iso_from_ms(oldest_ms), - "ended_at": public_api.iso_datetime(now), - "observed_span_seconds": observed_span_seconds, - "network_difficulty": public_api.decimal_string(current_network_difficulty), - "window_multiplier": 8, - "requested_window_weight": public_api.decimal_string(requested_window_weight), - "counted_window_weight": public_api.decimal_string(counted_window_weight), - "included_share_count": len(window_shares), - "is_complete": requested_window_weight > 0 and counted_window_weight >= requested_window_weight, - }, - "totals": { - "pool_hashrate_ths": pool_hashrate_ths, - "pool_counted_share_difficulty": public_api.decimal_string(counted_window_weight), - "participant_count": len(ranked), - "expected_time_to_block_seconds": expected_time_to_block, - }, - "pagination": public_api.pagination(page, limit, len(filtered_rows)), - "rows": filtered_rows[offset : offset + limit], - } - - def dashboard_hashrate_series( - self, - *, - subject_type: str, - subject_id: str | None, - range_id: str, - bucket: str, - lookback_seconds: int = 0, - range_anchor_epoch: int | None = None, - ) -> list[dict[str, object]]: - from lab.prism import public_api - - now = datetime.now(timezone.utc) - # Anchor the range lower bound on the caller's clock when provided so - # it agrees with the caller's min_epoch trim regardless of clock skew. - anchor = ( - datetime.fromtimestamp(range_anchor_epoch, timezone.utc) - if range_anchor_epoch is not None - else now - ) - started = _series_start(anchor, range_id) - if lookback_seconds > 0 and range_id != "all": - # Pre-range context requested by the smoother; the caller trims - # these buckets from the response after windowing. - started -= timedelta(seconds=int(lookback_seconds)) - bucket_seconds = {"5m": 300, "1h": 3600, "1d": 86400}[bucket] - buckets: dict[int, dict[str, int]] = {} - for share in self.all_shares(): - accepted_at = _datetime_from_ms(share.accepted_at_ms) - if accepted_at < started or accepted_at > now: - continue - if subject_type == "miner" and share.miner_id != subject_id: - continue - bucket_epoch = int(accepted_at.timestamp()) // bucket_seconds * bucket_seconds - entry = buckets.setdefault(bucket_epoch, {"count": 0, "difficulty": 0}) - entry["count"] += 1 - entry["difficulty"] += int(share.share_difficulty) - return [ - { - "timestamp": public_api.iso_datetime(datetime.fromtimestamp(bucket_epoch, timezone.utc)), - "hashrate_ths": public_api.hashrate_ths_from_difficulty(entry["difficulty"], bucket_seconds), - "accepted_share_count": entry["count"], - "accepted_share_difficulty": str(entry["difficulty"]), - } - for bucket_epoch, entry in sorted(buckets.items()) - ] - - def dashboard_block_markers( - self, - *, - range_id: str, - bucket: str, - range_anchor_epoch: int | None = None, - ) -> dict[str, object]: - # The in-memory backend records no found_at or solver detail for its - # pool blocks -- dashboard_blocks serves an empty page for the same - # reason -- so the marker series degrades to empty the way the other - # in-memory dashboard read models do. - return {"total_blocks": 0, "points": []} - - def persist_accepted_block( - self, - *, - block_hash: str, - block_height: int, - parent_hash: str, - final_bundle: dict[str, Any], - audit_report: dict[str, Any], - canonical_bundle_path: Path | None = None, - ) -> dict[str, int | str]: - block_hash = canonical_hex(block_hash, name="block_hash", expected_bytes=32) - with self._lock: - previous = self._memory_pool_blocks.get(block_hash) - if previous is not None and previous[0] != int(block_height): - raise RuntimeError("memory pool block height conflicts") - if previous is None: - self._memory_pool_blocks[block_hash] = ( - int(block_height), - "prepared", - str(parent_hash), - ) - self._audit_publication_sequences.setdefault(block_hash, None) - share_count = len(self._shares) - return { - "backend": "memory", - "share_count": share_count, - "block_count": 0, - "payout_entry_count": 0, - "carry_forward_count": 0, - } - - def reverse_immature_block(self, *, block_hash: str, active_tip_height: int) -> dict[str, int | str]: - block_hash = canonical_hex(block_hash, name="block_hash", expected_bytes=32) - with self._lock: - block = self._memory_pool_blocks.get(block_hash) - if ( - block is not None - and block[1] in {"confirmed", "inactive"} - and int(active_tip_height) >= block[0] + 1000 - ): - raise RuntimeError( - f"refusing to reverse mature pool block {block_hash}" - ) - if block is None or block[1] not in { - "prepared", - "confirmed", - "inactive", - }: - count = 0 - else: - self._memory_pool_blocks[block_hash] = ( - block[0], - "reversed", - block[2], - ) - self._inactive_audit_publications.discard(block_hash) - count = 1 - return { - "backend": "memory", - "reversed_count": count, - } - - def reject_prepared_block(self, *, block_hash: str, active_tip_height: int) -> dict[str, int | str]: - block_hash = canonical_hex(block_hash, name="block_hash", expected_bytes=32) - with self._lock: - block = self._memory_pool_blocks.get(block_hash) - if block is None or block[1] != "prepared": - count = 0 - else: - self._memory_pool_blocks[block_hash] = ( - block[0], - "rejected", - block[2], - ) - count = 1 - return { - "backend": "memory", - "rejected_count": count, - } - - def confirm_accepted_block(self, *, block_hash: str, active_tip_height: int) -> dict[str, int | str]: - block_hash = canonical_hex(block_hash, name="block_hash", expected_bytes=32) - with self._lock: - block = self._memory_pool_blocks.get(block_hash) - if ( - block is None - or block[0] != int(active_tip_height) - or block[1] not in {"prepared", "confirmed"} - ): - # Mirror the Postgres disposition split: a row already - # terminally disposed (reorg quarantine, rejection, or - # reversal) reports superseded (-1); 0 keeps meaning no row - # or a live row this confirmation does not match. - if block is not None and block[1] in { - "inactive", - "rejected", - "reversed", - }: - return {"backend": "memory", "confirmed_count": -1} - return {"backend": "memory", "confirmed_count": 0} - # Mirror the Postgres disposition split again: only a flip out of - # 'prepared' is a fresh confirmation (1). A row already confirmed - # at this height is an idempotent replay (2) that keeps the - # ordinal its flip allocated and burns none. - already_confirmed = block[1] == "confirmed" - publication_sequence = self._audit_publication_sequences.get(block_hash) - if publication_sequence is None: - publication_sequence = self._next_audit_publication_sequence - self._next_audit_publication_sequence += 1 - self._audit_publication_sequences[block_hash] = publication_sequence - self._memory_pool_blocks[block_hash] = ( - block[0], - "confirmed", - block[2], - ) - return { - "backend": "memory", - "confirmed_count": 2 if already_confirmed else 1, - "audit_publication_sequence": publication_sequence, - } - - def reorg_watch_blocks(self, *, active_tip_height: int) -> list[dict[str, object]]: - return [] - - def stranded_prepared_blocks( - self, - *, - active_tip_height: int, - min_depth: int, - limit: int = 64, - ) -> list[dict[str, object]]: - """Return deeply buried rows still parked in the prepared state. - - ``reorg_watch_blocks`` deliberately watches only confirmed/inactive - rows, and a prepared row is normally resolved by the live - submit/replay path that owns it. A row whose outbox entry is gone - (quarantined, or completed by a process that died before confirming) - therefore has nothing left to re-examine it, and stays prepared - forever. This read finds those rows; the caller decides, against the - active chain, which ones are provably orphaned. - """ - with self._lock: - rows = [ - { - "block_hash": block_hash, - "block_height": int(block[0]), - "parent_hash": str(block[2]), - } - for block_hash, block in self._memory_pool_blocks.items() - # The memory backend derives maturity_state from chain_state - # (see pool_block_state): 'prepared' is always 'immature'. - if block[1] == "prepared" - and int(block[0]) <= int(active_tip_height) - int(min_depth) - ] - rows.sort(key=lambda row: (int(row["block_height"]), str(row["block_hash"]))) - return rows[: max(0, int(limit))] - - def mark_pool_block_inactive(self, *, block_hash: str, active_tip_height: int) -> dict[str, int | str]: - block_hash = canonical_hex(block_hash, name="block_hash", expected_bytes=32) - with self._lock: - block = self._memory_pool_blocks.get(block_hash) - if block is None or block[1] != "confirmed": - count = 0 - else: - self._inactive_audit_publications.add(block_hash) - self._memory_pool_blocks[block_hash] = ( - block[0], - "inactive", - block[2], - ) - count = 1 - return { - "backend": "memory", - "inactive_count": count, - } - - def reactivate_pool_block(self, *, block_hash: str, active_tip_height: int) -> dict[str, int | str]: - block_hash = canonical_hex(block_hash, name="block_hash", expected_bytes=32) - with self._lock: - block = self._memory_pool_blocks.get(block_hash) - if ( - block is None - or block[0] > int(active_tip_height) - or block[1] != "inactive" - or block_hash not in self._inactive_audit_publications - ): - count = 0 - sequence = None - else: - sequence = self._audit_publication_sequences.get(block_hash) - if sequence is None: - raise RuntimeError( - "inactive pool block has no audit publication sequence" - ) - self._inactive_audit_publications.remove(block_hash) - self._memory_pool_blocks[block_hash] = ( - block[0], - "confirmed", - block[2], - ) - count = 1 - return { - "backend": "memory", - "reactivated_count": count, - **( - {"audit_publication_sequence": int(sequence)} - if sequence is not None - else {} - ), - } - - def pool_block_state(self, *, block_hash: str) -> dict[str, object] | None: - block_hash = canonical_hex(block_hash, name="block_hash", expected_bytes=32) - with self._lock: - block = self._memory_pool_blocks.get(block_hash) - if block is None: - return None - publication_sequence = self._audit_publication_sequences.get(block_hash) - return { - "block_hash": block_hash, - "block_height": block[0], - "parent_hash": block[2], - "chain_state": block[1], - "maturity_state": ( - "reversed" - if block[1] in {"rejected", "reversed"} - else "immature" - ), - "audit_publication_sequence": publication_sequence, - } - - def audit_publication_sequence_floor(self) -> int: - """Return the newest ordinal attached to any durable pool-block row.""" - - with self._lock: - return max( - ( - sequence - for sequence in self._audit_publication_sequences.values() - if sequence is not None - ), - default=0, - ) - - def mark_mature_pool_payouts(self, *, active_tip_height: int) -> dict[str, int | str]: - return { - "backend": "memory", - "matured_count": 0, - } - - @property - def backend_name(self) -> str: - return "memory" - - def __len__(self) -> int: - with self._lock: - return len(self._shares) - - -def _datetime_from_ms(timestamp_ms: int) -> datetime: - return datetime.fromtimestamp(timestamp_ms / 1000, timezone.utc) - - -def _iso_from_ms(timestamp_ms: int | None) -> str | None: - if timestamp_ms is None: - return None - from lab.prism import public_api - - return public_api.iso_datetime(_datetime_from_ms(timestamp_ms)) - - -def _share_difficulty_between(shares: list[AcceptedShareRecord], started_at: datetime, ended_at: datetime) -> int: - return sum( - int(share.share_difficulty) - for share in shares - if started_at <= _datetime_from_ms(share.accepted_at_ms) <= ended_at - ) - - -def _reward_window_weight(current_network_difficulty: int | str | Decimal) -> Decimal: - difficulty = Decimal(str(current_network_difficulty)) - if difficulty < 0: - difficulty = Decimal(0) - return difficulty * Decimal(8) - - -def _hash_percent(hashrate_ths: str, pool_hashrate_ths: str) -> str | None: - from lab.prism import public_api - - pool_hashrate = Decimal(str(pool_hashrate_ths)) - if pool_hashrate <= 0: - return None - return public_api.decimal_string(Decimal(str(hashrate_ths)) * Decimal(100) / pool_hashrate) - - -def _solver_worker_name_sql(share_id_column: str) -> str: - """SQL expression deriving a worker name from a solving share's share_id. - - A solving share's share_id is "<stratum_username>:<block_hash>" (see - pending_share_from_submission), and the stratum username is - "<payout_address>[.<worker_name>]". Strip the trailing ":<block_hash>" segment - to recover the username, then take the substring after the first '.'. Returns - NULL when the share is absent (unattributed block) or carried no worker suffix, - matching the nullable-string schema and mirroring the worker-name derivation in - dashboard_miner_worker_rows. - """ - username = f"regexp_replace({share_id_column}, ':[^:]*$', '')" - dot = f"position('.' IN {username})" - return ( - f"CASE WHEN {share_id_column} IS NULL THEN null " - f"WHEN {dot} > 0 THEN NULLIF(substring({username} FROM {dot} + 1), '') " - f"ELSE null END" - ) - - -def _prism_window_shares( - shares: list[AcceptedShareRecord], - *, - anchor_job_issued_at_ms: int, - requested_window_weight: int | Decimal, -) -> list[PrismWindowShare]: - if requested_window_weight <= 0: - return [] - requested = Decimal(str(requested_window_weight)) - total = Decimal(0) - window_shares: list[PrismWindowShare] = [] - eligible = [ - share - for share in shares - if share.job_issued_at_ms <= anchor_job_issued_at_ms and share.accepted_at_ms <= anchor_job_issued_at_ms - ] - for share in sorted(eligible, key=lambda item: item.share_seq, reverse=True): - if total >= requested: - break - share_difficulty = Decimal(int(share.share_difficulty)) - counted_difficulty = min(share_difficulty, requested - total) - total += counted_difficulty - window_shares.append(PrismWindowShare(share=share, counted_difficulty=counted_difficulty)) - return window_shares - - -def _series_start(now: datetime, range_id: str) -> datetime: - if range_id == "1w": - return now - timedelta(days=7) - if range_id == "1m": - return now - timedelta(days=30) - if range_id == "6m": - return now - timedelta(days=180) - return datetime.fromtimestamp(0, timezone.utc) - - -def database_url_from_psql_command(command: list[str]) -> str | None: - """Best-effort DSN extraction from a psql invocation. - - Handles the common shapes the coordinator and deploy tooling produce: - ``psql postgres://...``, ``psql -d postgres://...`` and - ``psql --dbname=postgres://...``. Anything else (host/user flags, service - files) stays on the subprocess backend rather than risking a mistranslated - connection. - """ - expect_dbname = False - for arg in command[1:]: - if expect_dbname: - candidate = arg - expect_dbname = False - elif arg in {"-d", "--dbname"}: - expect_dbname = True - continue - elif arg.startswith("--dbname="): - candidate = arg.split("=", 1)[1] - else: - candidate = arg - if candidate.startswith("postgres://") or candidate.startswith("postgresql://"): - return candidate - return None - - -def _writer_lease_advisory_lock_key(writer_id: str, writer_epoch: int) -> int: - """Return a stable signed bigint key namespaced to the PRISM writer lease.""" - digest = hashlib.sha256( - b"qbit-prism-writer-lease\0" - + writer_id.encode("utf-8") - + b"\0" - + str(writer_epoch).encode("ascii") - ).digest() - return int.from_bytes(digest[:8], "big", signed=True) - - -@runtime_checkable -class LedgerSqlPort(Protocol): - """The pooled statement-execution seam `PsqlShareLedger` writes through. - - `_NativePostgresClient` is the production implementation. Naming the - contract lets a test substitute a whole PostgreSQL model at construction - time (see `sql_backend_factory`) instead of reassigning bound methods on - a live ledger, which is what the lease and landing concurrency tests - need: statement timing, tuple-lock waits and transaction lifetime are - properties of this seam, not of any single method. - - Two statement shapes cross it. ``run_json`` runs a statement that - evaluates to one JSON value (every mutation, lease statement and - aggregate read). ``run_json_rows`` runs a read-only statement that - yields one JSON value per row and hands the rows back in bounded - batches (issue #236): the payout-window snapshot and delta, whose - single ``json_agg`` value used to be decoded in one GIL-held call the - size of the whole window. - """ - - def run_json( - self, - sql: str, - *, - retry_safe: bool = False, - timeout_seconds: float | None = None, - on_statement_start: Callable[[], None] | None = None, - ) -> Any: ... - - def run_json_rows( - self, - sql: str, - *, - retry_safe: bool = False, - timeout_seconds: float | None = None, - on_statement_start: Callable[[], None] | None = None, - row_converter: Callable[[Any], Any] | None = None, - batch_size: int = PAYOUT_WINDOW_ROW_BATCH_SIZE, - on_batch: Callable[[], None] | None = None, - ) -> list[Any]: ... - - def run_script(self, sql: str) -> None: ... - - def close(self) -> None: ... - - -@runtime_checkable -class LeaseGuardPort(Protocol): - """The dedicated writer-lease guard session seam. - - `_NativePostgresLeaseGuard` is the production implementation. Unlike - `LedgerSqlPort` this session is never transparently replaced: losing it - loses the advisory lock and must fence the owning coordinator, so the - heartbeat's liveness proof depends on the session's identity surviving. - A substitute must honour the same contract, including the serialized - query slot that `on_query_start` marks, the per-statement round trip - `on_statement_end` marks, the per-statement parsed result - `on_statement_result` hands back (issue #227: how a statement's own - server-side execution time reaches the caller), and the in-slot - `followup` statement the attribution recheck relies on. - """ - - def try_acquire(self) -> bool: ... - - @property - def held(self) -> bool: ... - - def run_json( - self, - sql: str, - *, - on_query_start: Callable[[], None] | None = None, - on_statement_end: Callable[[], None] | None = None, - on_statement_result: Callable[[Any], None] | None = None, - followup: Callable[[Any], str | None] | None = None, - ) -> Any: ... - - def close(self) -> None: ... - - -@dataclass(frozen=True) -class PostgresSessionGuards: - """Session-level GUCs this ledger sets on every connection it opens. - - Two guarantees ride the one libpq ``options`` carrier, because a setting - delivered at connect time covers every statement the session can run — - including the autocommit ones — rather than being re-asserted per - transaction by code that a future caller might bypass. - - The first is orphan reaping. A coordinator client that disappears without - an RST — network partition, SIGSTOP, VM pause — can leave its backend idle - in transaction, still holding every row lock the transaction took. The - writer-lease landing CTE row-locks the qbit_ledger_writer_lease singleton, - so one such orphan blocks a successor's startup lease upsert until the - kernel's TCP keepalive teardown (hours at OS defaults). These GUCs make the - *server* resolve that on its own: idle_in_transaction_session_timeout - aborts the orphaned transaction and releases its locks, and the - server-side tcp_keepalives_* settings (all PGC_USERSET, settable per - session) bound how long the server keeps a dead socket alive for backends - the idle-in-transaction timer cannot cover. - - The second is read-only enforcement. ``read_only`` adds - default_transaction_read_only=on, so PostgreSQL refuses the write itself - for a ledger built with ``read_only=True``. The refusing writer gate is - the in-process half of that promise and remains defense in depth, but it - only covers paths that take the gate; the GUC covers every statement the - session can issue, wherever the ledger points. Without it the guarantee - came from the deployment topology — a hot standby refusing writes — which - is absent whenever the read tier is aimed at a writable primary. - - Constructed once in PsqlShareLedger.__init__ and shared by all three - connection paths — the pooled native client, the dedicated lease-guard - session, and the psql subprocess backend. The orphan coverage is real but - bounded: the tcp_keepalives_* settings are silently ignored on - Unix-socket connections and on platforms that do not implement them, so - idle_in_transaction_session_timeout is the guard that always applies; - and these are session settings, so they reach only the sessions *this* - coordinator opens. A foreign session already holding the lease row — - another deployment's coordinator, an operator's psql — is bounded only - by the caller-side acquisition deadline in _run_lease_acquisition_json. - """ - - idle_in_transaction_timeout_seconds: float - tcp_keepalives_idle_seconds: int - tcp_keepalives_interval_seconds: int - tcp_keepalives_count: int - read_only: bool = False - - def options_fragment(self) -> str: - """Render the guards as libpq ``options`` / PGOPTIONS ``-c`` flags. - - The read-only setting goes last so that it wins over any earlier - duplicate, the same way the whole fragment is placed after an - operator's DSN-level options by _merged_session_options. - """ - idle_ms = max(1, int(self.idle_in_transaction_timeout_seconds * 1000)) - fragment = ( - f"-c idle_in_transaction_session_timeout={idle_ms}ms " - f"-c tcp_keepalives_idle={self.tcp_keepalives_idle_seconds} " - f"-c tcp_keepalives_interval={self.tcp_keepalives_interval_seconds} " - f"-c tcp_keepalives_count={self.tcp_keepalives_count}" - ) - if self.read_only: - fragment += " -c default_transaction_read_only=on" - return fragment - - -def _merged_session_options( - psycopg_module: Any, - conninfo: str, - *coordinator_fragments: str, -) -> str: - """Merge the coordinator's session options with the conninfo's own. - - Passing ``options`` as a connect kwarg replaces any ``options`` value the - operator embedded in the DSN outright, so that value must be read back - out of the conninfo and kept. The coordinator's fragments go last: libpq - applies ``-c`` settings left to right with the last duplicate winning, so - a DSN-level default can never silently disable the session guards. - """ - existing = psycopg_module.conninfo.conninfo_to_dict(conninfo).get("options") - fragments = [str(existing).strip()] if existing else [] - fragments.extend(coordinator_fragments) - return " ".join(fragment for fragment in fragments if fragment) - - -class _NativePostgresClient: - """Persistent pooled psycopg client for the share ledger. - - Executes the exact same self-contained SQL text the psql subprocess - backend runs (values are inlined by the callers), but over long-lived - connections instead of one fork+connect per statement. Every statement - returns a single JSON value, mirroring the psql `--tuples-only` contract. - Connections are created lazily, run in autocommit (each statement is its - own synchronous commit, exactly like a psql invocation — including the - group-commit ``append_batch`` statement, whose durability comes from its - own ``set_config('synchronous_commit', 'on', true)``), and a connection - that raises is discarded so the next acquisition reconnects. - - ``application_name`` (overriding any value in the conninfo) marks every - pooled backend in ``pg_stat_activity`` as belonging to this process, so - the lease guard can attribute an in-flight lease-tuple lock to the - writer's own fenced transaction rather than a competing expiry claim - (see ``verify_writer_lease_guard_session``). - """ - - # The clock ``run_json_rows`` budgets its local decoding against. A class - # attribute so a test built through ``__new__`` can drive the - # between-batch deadline with a virtual clock; ``run_json``'s - # single-value path is unchanged and keeps reading time.monotonic. - _monotonic: Callable[[], float] = staticmethod(time.monotonic) - - def __init__( - self, - conninfo: str, - *, - pool_size: int, - application_name: str | None = None, - session_guards: PostgresSessionGuards | None = None, - ): - import psycopg # deferred: the subprocess backend must work without it - - self._psycopg = psycopg - self._conninfo = conninfo - self._application_name = application_name - self._session_guards = session_guards - self._pool_size = max(1, int(pool_size)) - self._slots = BoundedSemaphore(self._pool_size) - self._idle: list[Any] = [] - self._idle_lock = Lock() - self._closed = False - - @property - def pool_size(self) -> int: - return self._pool_size - - def _connect(self, timeout_seconds: float | None = None) -> Any: - kwargs: dict[str, Any] = {"autocommit": True} - if timeout_seconds is not None: - # libpq accepts integral connect_timeout seconds. Rounding up keeps - # sub-second statement budgets valid without silently disabling - # the connection deadline. - kwargs["connect_timeout"] = max(1, math.ceil(timeout_seconds)) - if self._application_name is not None: - kwargs["application_name"] = self._application_name - session_guards = getattr(self, "_session_guards", None) - if session_guards is not None: - # Every pooled session carries the orphan-reaping guards, merged - # so an operator's DSN-level options value survives with the - # guard fragment last (last -c duplicate wins). - kwargs["options"] = _merged_session_options( - self._psycopg, - self._conninfo, - session_guards.options_fragment(), - ) - return self._psycopg.connect(self._conninfo, **kwargs) - - @contextmanager - def connection(self, *, timeout_seconds: float | None = None) -> Iterator[Any]: - """Borrow a pooled connection; discard it if the caller raises.""" - started = time.monotonic() - if timeout_seconds is None: - acquired = self._slots.acquire() - else: - acquired = self._slots.acquire(timeout=max(0.0, timeout_seconds)) - if not acquired: - raise LedgerOperationTimeout("timed out waiting for a postgres pool slot") - conn = None - try: - with self._idle_lock: - if self._closed: - raise RuntimeError("postgres client is closed") - if self._idle: - conn = self._idle.pop() - if conn is None or conn.closed: - connect_timeout = timeout_seconds - if connect_timeout is not None: - connect_timeout = max( - 0.001, - connect_timeout - (time.monotonic() - started), - ) - conn = self._connect(connect_timeout) - yield conn - except BaseException: - if conn is not None: - try: - conn.close() - except Exception: - pass - raise - else: - with self._idle_lock: - if self._closed: - keep = False - else: - self._idle.append(conn) - keep = True - if not keep: - try: - conn.close() - except Exception: - pass - finally: - self._slots.release() - - def run_json( - self, - sql: str, - *, - retry_safe: bool = False, - timeout_seconds: float | None = None, - on_statement_start: Callable[[], None] | None = None, - ) -> Any: - """Run one JSON-returning statement. - - An ``OperationalError`` does not reveal whether PostgreSQL committed - before the response was lost. Retry once only when the caller has - explicitly classified the statement as safe to execute again; every - mutation fails after the first ambiguous execution. - """ - attempts = 2 if retry_safe else 1 - deadline = ( - None - if timeout_seconds is None - else time.monotonic() + max(0.0, timeout_seconds) - ) - for attempt in range(attempts): - try: - remaining = ( - None - if deadline is None - else max(0.0, deadline - time.monotonic()) - ) - if remaining is not None and remaining <= 0: - raise LedgerOperationTimeout("postgres statement deadline expired") - connection = ( - self.connection() - if remaining is None - else self.connection(timeout_seconds=remaining) - ) - with connection as conn: - if deadline is None: - if on_statement_start is not None: - on_statement_start() - row = conn.execute(sql).fetchone() - else: - remaining = deadline - time.monotonic() - if remaining <= 0: - raise LedgerOperationTimeout( - "postgres statement deadline expired" - ) - if on_statement_start is not None: - on_statement_start() - timeout_ms = max(1, int(remaining * 1000)) - # SET LOCAL confines both guards to this explicit - # transaction, so pooled connections cannot leak a - # submitter-specific deadline into unrelated work. - with conn.transaction(): - conn.execute( - f"SET LOCAL statement_timeout = '{timeout_ms}ms'" - ) - conn.execute( - f"SET LOCAL lock_timeout = '{timeout_ms}ms'" - ) - row = conn.execute(sql).fetchone() - return parse_single_json_value(row[0] if row else None) - except self._psycopg.OperationalError as exc: - if timeout_seconds is not None and _is_postgres_deadline_error(exc): - raise LedgerOperationTimeout( - f"postgres operation exceeded {timeout_seconds:g}s" - ) from exc - if attempt + 1 >= attempts: - raise RuntimeError(f"postgres query failed: {exc}") from exc - raise AssertionError("unreachable") - - def run_json_rows( - self, - sql: str, - *, - retry_safe: bool = False, - timeout_seconds: float | None = None, - on_statement_start: Callable[[], None] | None = None, - row_converter: Callable[[Any], Any] | None = None, - batch_size: int = PAYOUT_WINDOW_ROW_BATCH_SIZE, - on_batch: Callable[[], None] | None = None, - ) -> list[Any]: - """Run one read-only statement that yields one JSON value per row. - - The row-result counterpart of ``run_json`` (issue #236). ``run_json`` - returns a single JSON value, which for the payout-window snapshot - meant one ``json_agg`` over the whole window: the driver decoded - every record in one C call that never released the GIL, and the - lease monitor thread went unscheduled for the duration. Here the - statement projects one JSON object per row and the result is - consumed with ``fetchmany`` in ``batch_size`` slices, so the driver - decodes at most one slice per call and the interpreter can switch - threads between slices. - - Same connection borrowing, same deadline transaction shape (``SET - LOCAL statement_timeout`` / ``lock_timeout`` when a deadline is - armed), same ``OperationalError`` translation and the same - once-only retry for ``retry_safe`` statements as ``run_json``. Two - properties are specific to this path: - - * One SELECT, one snapshot. The complete result is received from - the server before the first row is decoded, and the deadline - transaction commits before local decoding starts, so a long - decode never holds a server transaction open against - ``idle_in_transaction_session_timeout``. A concurrent append - cannot appear in part of a snapshot, and a retry after a - connection loss discards every partially converted row and - re-executes the complete statement inside the original deadline. - * The deadline covers decoding. It is rechecked before every - batch, so a budget that expires while rows are still being - converted raises ``LedgerOperationTimeout`` and publishes - nothing, rather than returning late with a complete list. - - ``row_converter`` runs on each row's JSON value inside the batch - loop (the ledger passes its record constructor), so one batch's - intermediate dicts are released before the next is fetched. - ``on_batch`` fires after each converted batch; the ledger uses it to - recheck its own operation deadline and stamp liveness. The batch - size bounds each decode call and the interval between checks, not - client memory: psycopg's client-side cursor still buffers the raw - result, exactly as it did under ``run_json``. - """ - attempts = 2 if retry_safe else 1 - batch_size = max(1, int(batch_size)) - deadline = ( - None - if timeout_seconds is None - else self._monotonic() + max(0.0, timeout_seconds) - ) - for attempt in range(attempts): - try: - remaining = ( - None - if deadline is None - else max(0.0, deadline - self._monotonic()) - ) - if remaining is not None and remaining <= 0: - raise LedgerOperationTimeout("postgres statement deadline expired") - connection = ( - self.connection() - if remaining is None - else self.connection(timeout_seconds=remaining) - ) - with connection as conn: - if deadline is None: - if on_statement_start is not None: - on_statement_start() - cursor = conn.execute(sql) - else: - remaining = deadline - self._monotonic() - if remaining <= 0: - raise LedgerOperationTimeout( - "postgres statement deadline expired" - ) - if on_statement_start is not None: - on_statement_start() - timeout_ms = max(1, int(remaining * 1000)) - # Same SET LOCAL scoping as run_json. The block ends, - # and the server transaction commits, before any - # row is decoded: the client-side cursor already - # holds the complete result, and a long decode must - # not keep a server transaction open. - with conn.transaction(): - conn.execute( - f"SET LOCAL statement_timeout = '{timeout_ms}ms'" - ) - conn.execute( - f"SET LOCAL lock_timeout = '{timeout_ms}ms'" - ) - cursor = conn.execute(sql) - try: - return self._consume_json_rows( - cursor, - deadline=deadline, - row_converter=row_converter, - batch_size=batch_size, - on_batch=on_batch, - ) - finally: - cursor.close() - except self._psycopg.OperationalError as exc: - if timeout_seconds is not None and _is_postgres_deadline_error(exc): - raise LedgerOperationTimeout( - f"postgres operation exceeded {timeout_seconds:g}s" - ) from exc - if attempt + 1 >= attempts: - raise RuntimeError(f"postgres query failed: {exc}") from exc - raise AssertionError("unreachable") - - def _consume_json_rows( - self, - cursor: Any, - *, - deadline: float | None, - row_converter: Callable[[Any], Any] | None, - batch_size: int, - on_batch: Callable[[], None] | None, - ) -> list[Any]: - """Convert a buffered row result in bounded batches. - - A NULL row is rejected the way ``run_json`` rejects a NULL value; - a text row (a ``::text``-cast projection) is decoded here so both - column types produce the same Python objects. - """ - results: list[Any] = [] - while True: - if deadline is not None and self._monotonic() >= deadline: - raise LedgerOperationTimeout( - "postgres statement deadline expired while decoding rows" - ) - batch = cursor.fetchmany(batch_size) - if not batch: - return results - for row in batch: - value = parse_single_json_value(row[0] if row else None) - results.append( - value if row_converter is None else row_converter(value) - ) - if on_batch is not None: - on_batch() - - def run_script(self, sql: str) -> None: - """Run a multi-statement script (schema initialization).""" - with self.connection() as conn: - conn.execute(sql) - - def close(self) -> None: - with self._idle_lock: - self._closed = True - idle, self._idle = self._idle, [] - for conn in idle: - try: - conn.close() - except Exception: - pass - - -class _NativePostgresLeaseGuard: - """Hold one writer identity's PostgreSQL session advisory lock. - - Unlike the ordinary native pool, this connection is never transparently - replaced: losing it also loses the advisory lock and must fence the owning - coordinator. The lease heartbeat runs on this same isolated session. - """ - - def __init__( - self, - conninfo: str, - advisory_lock_key: int, - *, - session_guards: PostgresSessionGuards | None = None, - ): - import psycopg # deferred: psql-only users retain TTL fencing - - # The short statement_timeout stays: the guard session must never - # queue behind a fenced write (see verify_writer_lease_guard_session). - # It is also the dominant term in the heartbeat's staleness envelope - # (WriterLeaseHeartbeatPolicy.max_healthy_server_gap_seconds), so the - # value is taken from the timing policy rather than written twice. - # The session guards ride the same options string so this dedicated - # connection carries the same orphan bounds as every pooled one. - # Merged with any operator DSN-level options, coordinator fragments - # last so they win. - statement_timeout_ms = int( - round(WRITER_LEASE_GUARD_STATEMENT_TIMEOUT_SECONDS * 1000) - ) - options = f"-c statement_timeout={statement_timeout_ms}" - if session_guards is not None: - options = _merged_session_options( - psycopg, - conninfo, - options, - session_guards.options_fragment(), - ) - self._connection = psycopg.connect( - conninfo, - autocommit=True, - connect_timeout=2, - options=options, - ) - self._advisory_lock_key = advisory_lock_key - self._query_lock = Lock() - self._closed = False - self._held = False - - def try_acquire(self) -> bool: - row = self._connection.execute( - f"SELECT pg_try_advisory_lock({self._advisory_lock_key})" - ).fetchone() - self._held = bool(row and row[0]) - return self._held - - @property - def held(self) -> bool: - return self._held and not self._closed and not self._connection.closed - - def run_json( - self, - sql: str, - *, - on_query_start: Callable[[], None] | None = None, - on_statement_end: Callable[[], None] | None = None, - on_statement_result: Callable[[Any], None] | None = None, - followup: Callable[[Any], str | None] | None = None, - ) -> Any: - with self._query_lock: - # Queries on this session serialize behind the periodic heartbeat - # and any concurrent caller. The callback marks the moment the - # serialized slot is acquired, so callers can budget queue wait - # and statement execution separately. - if on_query_start is not None: - on_query_start() - if not self.held: - raise RuntimeError("postgres writer lease guard is not held") - row = self._connection.execute(sql).fetchone() - result = parse_single_json_value(row[0] if row else None) - # Every completed round trip is server-proven liveness, and the - # boundary the caller's phase attribution charges guard SQL - # against. It fires after the result is parsed so a malformed - # response is not counted as a healthy round trip. - # ``on_statement_result`` then hands the same round trip's parsed - # result to the caller, which is how a statement that reports - # its own server-side execution time (issue #227) gets that - # figure back per statement without a second round trip. The - # guard stays agnostic about the result's shape. - if on_statement_end is not None: - on_statement_end() - if on_statement_result is not None: - on_statement_result(result) - # A followup runs inside this same serialized slot: no second - # queue wait behind other guard callers can be charged to the - # caller's execution budget. Each execute on this autocommit - # session is still its own statement with a fresh snapshot, - # which is exactly what verification rechecks need. The - # callback owns termination by returning None. - while followup is not None: - next_sql = followup(result) - if next_sql is None: - break - row = self._connection.execute(next_sql).fetchone() - result = parse_single_json_value(row[0] if row else None) - if on_statement_end is not None: - on_statement_end() - if on_statement_result is not None: - on_statement_result(result) - return result - - def close(self) -> None: - """Close the session, releasing its advisory lock server-side.""" - if self._closed: - return - self._closed = True - self._held = False - try: - self._connection.close() - except Exception: - pass - - -# The JSON key both guard statements use to report their own server-side -# execution time (issue #227): ``clock_timestamp() - statement_timestamp()`` -# evaluated by the statement itself, so it costs no extra round trip and -# takes no lock. It is timing metadata only; nothing about liveness, -# identity, or the server-proven edge is read from it. -GUARD_SERVER_EXECUTION_SECONDS_KEY = "server_execution_seconds" -GUARD_SERVER_EXECUTION_SECONDS_SQL = ( - "extract(epoch FROM clock_timestamp() - statement_timestamp())" -) - - -def guard_server_execution_seconds(result: object) -> float | None: - """Server-reported execution seconds of one guard round trip, if any. - - ``None`` for a result that does not carry the key (an older statement - shape, a test fake) or carries something that is not a finite - non-negative number; callers then keep their conservative client-side - attribution rather than trusting a malformed figure. - """ - if not isinstance(result, dict): - return None - value = result.get(GUARD_SERVER_EXECUTION_SECONDS_KEY) - if value is None or isinstance(value, bool): - return None - try: - seconds = float(value) - except (TypeError, ValueError): - return None - if not math.isfinite(seconds) or seconds < 0.0: - return None - return seconds - - -def parse_single_json_value(value: object) -> Any: - """Normalize a one-row/one-column JSON query result. - - psycopg already decodes json/jsonb columns to Python objects; the psql - subprocess path yields text. A NULL result matches the subprocess - behavior of raising on empty output. - """ - if value is None: - raise RuntimeError("postgres query returned no JSON") - if isinstance(value, (str, bytes, bytearray)): - return json.loads(value) - return value - - -# The per-row JSON projection both payout-window reads return (issue #236): -# one object per accepted share, decoded row by row by either backend, rather -# than aggregated server-side with json_agg into a single value the client -# then had to decode in one call the size of the window. Same keys and value -# expressions the aggregate carried, so a record parses exactly as before. -_ACCEPTED_SHARE_JSON_ROW_SQL = """json_build_object( - 'share_seq', share_seq, - 'share_id', share_id, - 'miner_id', miner_id, - 'order_key', payout_order_key, - 'p2mr_program_hex', encode(p2mr_program, 'hex'), - 'share_difficulty', share_difficulty::text, - 'network_difficulty', network_difficulty::text, - 'template_height', template_height, - 'job_id', job_id, - 'job_issued_at_ms', round(extract(epoch FROM job_issued_at) * 1000)::bigint, - 'accepted_at_ms', round(extract(epoch FROM accepted_at) * 1000)::bigint, - 'ntime', ntime, - 'credit_policy', credit_policy -)""" - - -class PsqlShareLedger: - """Postgres-backed implementation of the coordinator share-ledger API. - - The process that owns this object is the single logical writer. It delegates - sequence assignment to `qbit_share_ledger.share_seq` and uses the canonical - SQL schema under `crates/qbit-prism/sql`. - """ - - durable_payout_state = True - - # The lease lifecycle's clock, as a class attribute so an instance built - # without __init__ (several tests exercise one statement that way) still - # reads a working clock. __init__ overrides it per instance with whatever - # was injected. Declaring it here rather than resolving it through getattr - # at each call site means a rename fails loudly at the assignment instead - # of silently reverting every scenario to wall-clock time. - _monotonic: Callable[[], float] = staticmethod(time.monotonic) - - # Guards the one-time creation of the per-instance read-timing state for - # ledgers built through __new__ (several focused tests exercise a single - # statement that way). A class attribute, so the check-then-set inside - # _ensure_ledger_read_timings cannot let two threads each publish their - # own dict and lose one of them; __init__ builds the state directly and - # never reaches it. - _ledger_read_timings_bootstrap: ClassVar[Lock] = Lock() - - @staticmethod - def _resolve_lease_authority_margin_seconds( - lease_ttl_seconds: float, - lease_authority_margin_seconds: float | None, - ) -> float: - """Resolve the own-write deferral margin for external side effects. - - Never below half the lease TTL: that floor keeps the deferral - engaged through the eroded tail of a long own fenced write even - when the configured guarded-RPC deadlines are short. A caller - supplies a larger margin when its longest fence-guarded RPC could - outlast the floor — the margin must cover the longest effect the - fence can authorize, or the effect outlives its runway and - degenerates into rollback-dependent authority. - - A margin that reaches the TTL is rejected outright rather than - accepted as defer-every-skip: the deferral only gates renewal - *skips*, while a verification over an uncontended row renews and - authorizes unconditionally — and a landed renewal's runway is - exactly one TTL. When the guarded effect's deadline can reach the - TTL, even a freshly renewed lease cannot outlast the effect, so - no deferral policy closes the authorize-then-expire window and - the configuration itself is unsafe. Failing construction turns a - silent split-brain hazard into a startup error: raise the lease - TTL or lower the guarded RPC deadlines. - """ - floor = lease_ttl_seconds / 2.0 - if lease_authority_margin_seconds is None: - return floor - margin = float(lease_authority_margin_seconds) - if not math.isfinite(margin) or margin < 0: - raise ValueError( - "lease_authority_margin_seconds must be finite and non-negative" - ) - if margin >= lease_ttl_seconds: - raise ValueError( - "lease_authority_margin_seconds " - f"({margin}) must stay below lease_ttl_seconds " - f"({lease_ttl_seconds}): a fence-guarded RPC whose deadline " - "can reach the lease TTL can outlive even a freshly renewed " - "lease, and renewals bypass the own-write deferral entirely; " - "raise the lease TTL or lower the guarded RPC deadlines" - ) - return max(floor, margin) - - def __init__( - self, - *, - psql_command: str, - database_url: str | None = None, - native_client_mode: str = "auto", - writer_id: str = "prism-coordinator", - writer_epoch: int = 1, - writer_session_token: str | None = None, - initialize_schema: bool = False, - schema_path: Path | None = None, - lease_retry_sleep: Callable[[float], None] | None = None, - monotonic: Callable[[], float] | None = None, - pool_application_name: str | None = None, - sql_backend_factory: Callable[..., LedgerSqlPort | None] | None = None, - lease_guard_factory: Callable[..., LeaseGuardPort | None] | None = None, - lease_retry_max_sleep_seconds: float = 15.0, - lease_ttl_seconds: float = 60.0, - lease_authority_margin_seconds: float | None = None, - lease_adoption_silence_seconds: float = DEFAULT_WRITER_LEASE_ADOPTION_SILENCE_SECONDS, - lease_acquire_lock_timeout_seconds: float = DEFAULT_LEASE_ACQUIRE_LOCK_TIMEOUT_SECONDS, - lease_acquire_attempts: int = DEFAULT_LEASE_ACQUIRE_ATTEMPTS, - postgres_idle_in_transaction_timeout_seconds: float = DEFAULT_POSTGRES_IDLE_IN_TRANSACTION_TIMEOUT_SECONDS, - postgres_tcp_keepalives_idle_seconds: int = DEFAULT_POSTGRES_TCP_KEEPALIVES_IDLE_SECONDS, - postgres_tcp_keepalives_interval_seconds: int = DEFAULT_POSTGRES_TCP_KEEPALIVES_INTERVAL_SECONDS, - postgres_tcp_keepalives_count: int = DEFAULT_POSTGRES_TCP_KEEPALIVES_COUNT, - read_only: bool = False, - read_concurrency: int = 4, - accepted_stats_cache_seconds: float = 60.0, - reward_window_cache_seconds: float = 30.0, - audit_body_dir: str | Path | None = None, - audit_bundle_canonicalizer: Callable[[dict[str, Any]], bytes] | None = None, - audit_share_segment_size: int = 0, - audit_artifact_store: AuditArtifactStore | None = None, - ctv_broadcast_attempt_detail_limit: int = DEFAULT_CTV_BROADCAST_ATTEMPT_DETAIL_LIMIT, - ctv_broadcast_retry_backoff_seconds: int = DEFAULT_CTV_BROADCAST_RETRY_BACKOFF_SECONDS, - candidate_storage_version: int = CANDIDATE_BODY_STORAGE_VERSION, - candidate_spool_dir: str | Path | None = None, - candidate_spool_reservation_bytes: int = DEFAULT_SPOOL_RESERVATION_BYTES, - candidate_schema_path: Path | None = None, - legacy_candidate_helper_timeout_seconds: float = LEGACY_HELPER_TIMEOUT_SECONDS, - legacy_candidate_helper_memory_bytes: int = LEGACY_HELPER_MEMORY_BYTES, - ): - if writer_epoch < 0: - raise ValueError("writer_epoch must be >= 0") - candidate_storage_version = int(candidate_storage_version) - if candidate_storage_version not in ( - LEGACY_CANDIDATE_STORAGE_VERSION, - CANDIDATE_BODY_STORAGE_VERSION, - ): - raise ValueError( - f"unsupported candidate storage version {candidate_storage_version}" - ) - read_only = bool(read_only) - if read_only and initialize_schema: - raise ValueError("a read-only ledger cannot initialize the schema") - accepted_stats_cache_seconds = float(accepted_stats_cache_seconds) - if not math.isfinite(accepted_stats_cache_seconds) or accepted_stats_cache_seconds < 0: - raise ValueError("accepted_stats_cache_seconds must be finite and non-negative") - reward_window_cache_seconds = float(reward_window_cache_seconds) - if not math.isfinite(reward_window_cache_seconds) or reward_window_cache_seconds < 0: - raise ValueError("reward_window_cache_seconds must be finite and non-negative") - lease_retry_max_sleep_seconds = float(lease_retry_max_sleep_seconds) - if lease_retry_max_sleep_seconds <= 0: - raise ValueError("lease_retry_max_sleep_seconds must be positive") - lease_ttl_seconds = float(lease_ttl_seconds) - if not math.isfinite(lease_ttl_seconds) or lease_ttl_seconds <= 0: - raise ValueError("lease_ttl_seconds must be finite and positive") - lease_authority_margin_seconds = ( - self._resolve_lease_authority_margin_seconds( - lease_ttl_seconds, - lease_authority_margin_seconds, - ) - ) - lease_adoption_silence_seconds = float(lease_adoption_silence_seconds) - if ( - not math.isfinite(lease_adoption_silence_seconds) - or lease_adoption_silence_seconds <= 0 - ): - raise ValueError("lease_adoption_silence_seconds must be finite and positive") - # Validated here, not only in load_config: run_ctv_broadcaster_daemon - # and backfill_ctv_fanouts construct this class directly and must not - # be able to disarm the lease-acquisition bound or the session guards - # with a zero or negative value. - lease_acquire_lock_timeout_seconds = float(lease_acquire_lock_timeout_seconds) - if ( - not math.isfinite(lease_acquire_lock_timeout_seconds) - or lease_acquire_lock_timeout_seconds <= 0 - ): - raise ValueError("lease_acquire_lock_timeout_seconds must be finite and positive") - lease_acquire_attempts = int(lease_acquire_attempts) - if lease_acquire_attempts <= 0: - raise ValueError("lease_acquire_attempts must be positive") - postgres_idle_in_transaction_timeout_seconds = float( - postgres_idle_in_transaction_timeout_seconds - ) - if ( - not math.isfinite(postgres_idle_in_transaction_timeout_seconds) - or postgres_idle_in_transaction_timeout_seconds <= 0 - ): - raise ValueError( - "postgres_idle_in_transaction_timeout_seconds must be finite and positive" - ) - postgres_tcp_keepalives_idle_seconds = int(postgres_tcp_keepalives_idle_seconds) - if postgres_tcp_keepalives_idle_seconds <= 0: - raise ValueError("postgres_tcp_keepalives_idle_seconds must be positive") - postgres_tcp_keepalives_interval_seconds = int( - postgres_tcp_keepalives_interval_seconds - ) - if postgres_tcp_keepalives_interval_seconds <= 0: - raise ValueError("postgres_tcp_keepalives_interval_seconds must be positive") - postgres_tcp_keepalives_count = int(postgres_tcp_keepalives_count) - if postgres_tcp_keepalives_count <= 0: - raise ValueError("postgres_tcp_keepalives_count must be positive") - read_concurrency = int(read_concurrency) - if read_concurrency <= 0: - raise ValueError("read_concurrency must be positive") - self._command = shlex.split(psql_command) - if not self._command: - raise ValueError("psql_command must not be empty") - self._writer_id = writer_id - self._writer_epoch = writer_epoch - self._writer_session_token = writer_session_token or uuid.uuid4().hex - # Advertised as application_name by every pooled connection so the - # lease guard can recognize the lease tuple's locker as one of this - # process's own backends (a fenced write outlasting the TTL) rather - # than a competing expiry claim. Unique per ledger instance: a - # predecessor or replacement process never shares it, so their - # in-flight writes still fence this session out. - self._pool_application_name = ( - pool_application_name or f"qbit-prism-writer-{uuid.uuid4().hex}" - ) - self._lease_ttl_seconds = lease_ttl_seconds - # SQL fragment for the writer-lease expiry. The lease is refreshed on - # every append (the dominant liveness signal during active mining), so a - # short TTL bounds how long a same-identity replacement writer waits - # after an *ungraceful* crash. Graceful shutdown releases the lease - # outright (see release_writer_lease), making restarts near-instant. - self._lease_interval_sql = f"make_interval(secs => {lease_ttl_seconds})" - self._lease_authority_margin_seconds = lease_authority_margin_seconds - # SQL fragment for the own-write deferral margin (see - # verify_writer_lease_guard_session): an own-write renewal skip over - # a committed row with less than this much TTL remaining defers - # external side effects instead of authorizing them. - self._lease_authority_margin_sql = ( - f"make_interval(secs => {lease_authority_margin_seconds})" - ) - self._lease_retry_sleep = lease_retry_sleep or time.sleep - # The lease lifecycle's only clock. Every interval this process - # measures itself — adoption silence from guard acquisition, caller - # deadlines, lease refresh age — reads through here, so a test can - # supply a virtual clock and drive an interleaving by advancing it - # rather than by sleeping and hoping. Production passes None and gets - # time.monotonic. - # Resolved at construction, not at each call: an instance built the - # ordinary way binds whichever clock is in force now, while the class - # attribute above still covers instances built without __init__. - self._monotonic = monotonic or time.monotonic - self._sql_backend_factory = sql_backend_factory - self._lease_guard_factory = lease_guard_factory - self._lease_retry_max_sleep_seconds = lease_retry_max_sleep_seconds - self._lease_retry_min_sleep_seconds = min(0.25, self._lease_retry_max_sleep_seconds) - self._lease_adoption_silence_seconds = lease_adoption_silence_seconds - self._lease_acquire_lock_timeout_seconds = lease_acquire_lock_timeout_seconds - self._lease_acquire_attempts = lease_acquire_attempts - # One immutable guard set shared by all three connection paths (the - # pooled native client, the dedicated lease-guard session, and the - # psql subprocess backend), so a session this coordinator opens is - # disowned by the server instead of holding the lease row on after - # its client vanishes. See PostgresSessionGuards for what the guards - # do and do not reach. - self._session_guards = PostgresSessionGuards( - idle_in_transaction_timeout_seconds=postgres_idle_in_transaction_timeout_seconds, - tcp_keepalives_idle_seconds=postgres_tcp_keepalives_idle_seconds, - tcp_keepalives_interval_seconds=postgres_tcp_keepalives_interval_seconds, - tcp_keepalives_count=postgres_tcp_keepalives_count, - # Fail closed at connect time rather than transaction by - # transaction: a read-only ledger tells the server it is read-only - # once, and every connection it can create -- pool slots, read - # slots, the psql subprocess, autocommit statements included -- - # inherits the refusal. - read_only=read_only, - ) - self._operation_timeout_local = local() - self._statement_timeout_local = local() - # Created here, not lazily in operation_progress: that scope's - # check-then-set is not atomic, so two block-work threads entering - # their first-ever scope on a fresh ledger can each build a local() - # and have one assignment win. The loser's hook would then live on an - # orphaned object and its admission wait would silently fall back to - # the heartbeat-silent path -- the exact failure the hook exists to - # remove. The scope keeps its lazy fallback for ledgers built through - # __new__ in focused tests; the production path must not race. - self._operation_progress_local = local() - self._read_only = read_only - # A read-only ledger never holds the writer lock, so it is not given - # one: see _RefusingWriterGate. Read-slot traffic is unaffected. - self._lock = _RefusingWriterGate() if read_only else Lock() - self._read_semaphore = BoundedSemaphore(read_concurrency) - # Issue #255: chunked candidate bodies. Staging and hydration take - # this one-slot gate rather than the writer lock, because neither - # touches the lease row and both move body-sized data. - self._candidate_storage_version = candidate_storage_version - self._candidate_body_gate = BoundedSemaphore(1) - self._candidate_spool = SpoolAdmission( - candidate_spool_dir, - limit_bytes=int(candidate_spool_reservation_bytes), - ) - self._candidate_schema_path = candidate_schema_path - self._legacy_candidate_helper = LegacyCandidateHelper( - LegacyTransport( - psql_command=psql_command, - database_url=database_url or database_url_from_psql_command(self._command), - ), - memory_limit_bytes=int(legacy_candidate_helper_memory_bytes), - timeout_seconds=float(legacy_candidate_helper_timeout_seconds), - ) - audit_share_segment_size = int(audit_share_segment_size) - if audit_share_segment_size < 0: - raise ValueError("audit_share_segment_size must be non-negative") - if audit_artifact_store is None and audit_body_dir is not None: - body_root = Path(audit_body_dir) - audit_artifact_store = AuditArtifactStore( - AuditArtifactConfig( - root=body_root, - evidence_path=body_root / "prism-live-stratum-evidence.json", - share_segment_size=audit_share_segment_size, - ), - # Legacy direct-ledger construction is an explicit adapter. - # Coordinator production wiring injects the shared A1 store. - canonicalizer=( - audit_bundle_canonicalizer or _default_bundle_canonicalizer() - ), - ) - self._audit_artifact_store = audit_artifact_store - self._audit_bundle_canonicalizer = ( - audit_bundle_canonicalizer or _default_bundle_canonicalizer() - ) - ctv_broadcast_attempt_detail_limit = int(ctv_broadcast_attempt_detail_limit) - if ctv_broadcast_attempt_detail_limit < 0: - raise ValueError("ctv_broadcast_attempt_detail_limit must be non-negative") - self._ctv_broadcast_attempt_detail_limit = ctv_broadcast_attempt_detail_limit - ctv_broadcast_retry_backoff_seconds = int(ctv_broadcast_retry_backoff_seconds) - if ctv_broadcast_retry_backoff_seconds < 0: - raise ValueError("ctv_broadcast_retry_backoff_seconds must be non-negative") - self._ctv_broadcast_retry_backoff_seconds = ctv_broadcast_retry_backoff_seconds - self._accepted_stats_cache_seconds = accepted_stats_cache_seconds - self._reward_window_cache_seconds = reward_window_cache_seconds - self._reward_window_cache_lock = Lock() - self._reward_window_cache: tuple[str, float, dict[str, Any]] | None = None - self._stats_lock = Lock() - self._stats_refresh_lock = Lock() - self._stats_counts: dict[str, int] | None = None - self._stats_max_share_seq = 0 - self._stats_note_buffer: dict[int, bool] | None = None - self._stats_refreshed_monotonic: float | None = None - self._stats_background_refresh_thread: Thread | None = None - self._stats_reconcile_failures = 0 - self._prior_balances_read_stats_lock = Lock() - self._prior_balances_reads_total = 0 - self._prior_balances_read_last_seconds = 0.0 - self._prior_balances_read_max_seconds = 0.0 - # Attribution for read-slot operations: local admission and server - # execution are recorded separately (see _run_attributed_read_json). - # Built here so the production path never reaches the __new__ - # bootstrap in _ensure_ledger_read_timings; the dict is published - # before the lock that guards it, so a reader that observes the lock - # observes the dict too. - self._ledger_read_timings: dict[str, dict[str, float | int]] = {} - self._ledger_read_timings_lock = Lock() - self._native = self._make_native_client( - native_client_mode, - database_url, - read_concurrency=read_concurrency, - ) - self._writer_lease_guard: LeaseGuardPort | None = None - try: - if not read_only: - # Constructing an ordinary ledger claims the single-writer - # lease. A read-only ledger must not, or a second process - # opening one would contend with -- and could adopt -- the - # lease the coordinator lands blocks under. - self._initialize_writer_lease_guard(database_url) - if initialize_schema: - path = schema_path or Path(__file__).resolve().parents[2] / "crates/qbit-prism/sql/001_share_ledger.sql" - self._run_script(path.read_text(encoding="utf-8")) - # Additive #255 migration, applied after the base schema - # on every start; every statement in it is idempotent. - candidate_path = ( - candidate_schema_path - or Path(__file__).resolve().parents[2] - / "crates/qbit-prism/sql/002_candidate_bodies.sql" - ) - self._run_script(candidate_path.read_text(encoding="utf-8")) - self._ensure_writer_lease() - except BaseException: - self.close() - raise - - def _make_native_client( - self, - native_client_mode: str, - database_url: str | None, - *, - read_concurrency: int, - ) -> LedgerSqlPort | None: - mode = (native_client_mode or "auto").strip().lower() - if mode in {"0", "false", "no", "off", "psql"}: - return None - if mode not in {"auto", "1", "true", "yes", "on", "native"}: - raise ValueError(f"unsupported native client mode: {native_client_mode!r}") - required = mode != "auto" - conninfo = database_url or database_url_from_psql_command(self._command) - if conninfo is None: - if required: - raise ValueError( - "PRISM_POSTGRES_NATIVE_CLIENT=1 requires PRISM_DATABASE_URL or a " - "postgres:// DSN inside PRISM_POSTGRES_PSQL_COMMAND" - ) - return None - # One pooled connection per concurrent reader plus one for the - # serialized write path (the coordinator's share writer thread). - pool_size = read_concurrency + 1 - if self._sql_backend_factory is not None: - # An injected backend is authoritative: it stands in for the - # whole server, so psycopg's availability is irrelevant and a - # None return means the same thing it does below (fall back to - # the psql subprocess path). - return self._sql_backend_factory( - conninfo, - pool_size=pool_size, - application_name=self._pool_application_name, - ) - try: - return _NativePostgresClient( - conninfo, - pool_size=pool_size, - application_name=self._pool_application_name, - session_guards=getattr(self, "_session_guards", None), - ) - except ImportError: - if required: - raise ValueError( - "PRISM_POSTGRES_NATIVE_CLIENT=1 requires the psycopg package" - ) from None - # The daemon's startup line reports the active execution backend, - # but this silent capability loss is worth its own line too: the - # psql fallback applies the schema through a subprocess whose - # atomicity contract every operator relies on. - print( - "prism ledger native PostgreSQL client unavailable: " - "psycopg import failed; falling back to the psql subprocess " - "backend", - flush=True, - ) - return None - - def _make_writer_lease_guard( - self, - database_url: str | None, - ) -> LeaseGuardPort | None: - if self._native is None: - return None - conninfo = database_url or database_url_from_psql_command(self._command) - if conninfo is None: - return None - advisory_lock_key = _writer_lease_advisory_lock_key( - self._writer_id, - self._writer_epoch, - ) - if self._lease_guard_factory is not None: - return self._lease_guard_factory( - conninfo, - advisory_lock_key=advisory_lock_key, - ) - return _NativePostgresLeaseGuard( - conninfo, - advisory_lock_key, - session_guards=getattr(self, "_session_guards", None), - ) - - def _initialize_writer_lease_guard(self, database_url: str | None) -> None: - if not self._writer_session_token.startswith( - WRITER_LEASE_HEARTBEAT_SESSION_PREFIX - ): - return - warned = False - while True: - guard = self._make_writer_lease_guard(database_url) - if guard is None: - # A psql subprocess cannot retain a session advisory lock. - # Downgrade before publishing the token so other processes - # conservatively retain TTL fencing for this owner. - self._writer_session_token = uuid.uuid4().hex - print( - "prism ledger writer fast adoption disabled: persistent " - "native PostgreSQL connection unavailable; using TTL fencing", - flush=True, - ) - return - try: - acquired = guard.try_acquire() - except Exception: - guard.close() - raise - if acquired: - self._writer_lease_guard = guard - # Adoption silence is measured from this moment as well as - # from the lease row's updated_at. The predecessor that just - # lost this advisory lock self-fences within its heartbeat - # failure budget; counting from acquisition guarantees it - # that time even when its lease row is already stale because - # a long fenced transaction withheld updated_at refreshes. - self._writer_lease_guard_acquired_monotonic = self._monotonic() - return - guard.close() - if not warned: - print( - "prism ledger writer guard held by a live same-identity " - "coordinator; waiting before lease acquisition", - flush=True, - ) - warned = True - self._lease_retry_sleep(self._lease_retry_min_sleep_seconds) - - @property - def execution_backend(self) -> str: - if getattr(self, "_native", None) is not None: - return "psycopg-pool" - return "psql-subprocess" - - def close(self) -> None: - """Release pooled native connections. Safe to call multiple times.""" - native = getattr(self, "_native", None) - if native is not None: - native.close() - self._close_writer_lease_guard() - - def _close_writer_lease_guard(self) -> None: - guard = getattr(self, "_writer_lease_guard", None) - if guard is None: - return - self._writer_lease_guard = None - guard.close() - - @property - def writer_lease_fast_adoption_capable(self) -> bool: - guard = getattr(self, "_writer_lease_guard", None) - return bool( - self._writer_session_token.startswith( - WRITER_LEASE_HEARTBEAT_SESSION_PREFIX - ) - and guard is not None - and guard.held - ) - - @property - def writer_lease_guard_required(self) -> bool: - return self._writer_session_token.startswith( - WRITER_LEASE_HEARTBEAT_SESSION_PREFIX - ) - - @property - def writer_lease_last_refresh_monotonic(self) -> float | None: - return getattr(self, "_writer_lease_last_refresh_monotonic", None) - - @property - def backend_name(self) -> str: - return "postgres-psql" - - def read_replica_status( - self, - *, - timeout_seconds: float = DEFAULT_READ_REPLICA_PROBE_TIMEOUT_SECONDS, - ) -> dict[str, object]: - """Replication-state probe backing the public read staleness contract. - - Returns: - - - ``in_recovery`` -- true only on a hot standby; - - ``replay_lag_seconds`` -- wall clock minus the newest replayed - transaction's commit time (None before any replay). This is an - informational staleness *indicator*, not a freshness proof: on an - idle primary it grows with wall time even though the replica is - fully caught up; - - ``receiver_heartbeat_age_seconds`` -- wall clock minus the newest - message from the primary's WAL sender (None when the walreceiver - is not connected). Heartbeats flow every - ``wal_receiver_status_interval`` (default 10s) even when the - primary is idle, so this is the replica-side liveness proof the - public read service enforces; - - ``apply_backlog_bytes`` -- WAL bytes received but not yet replayed. - - The extracted public read service (issue #145) polls this to enforce - its bounded-staleness contract and to refuse serving from a writable - primary. Reads run through the ordinary read pool, so the probe never - touches the writer lease. - - Bounded by default: this runs on the service's background probe - thread, and an unbounded probe against a wedged standby would leave - the freshness gate holding its last answer indefinitely rather than - ageing out into a refusal. - """ - sql = """ -SELECT json_build_object( - 'in_recovery', pg_is_in_recovery(), - 'replay_lag_seconds', CASE - WHEN pg_last_xact_replay_timestamp() IS NULL THEN NULL - ELSE extract(epoch FROM (clock_timestamp() - pg_last_xact_replay_timestamp())) - END, - 'receiver_heartbeat_age_seconds', CASE - WHEN (SELECT last_msg_receipt_time FROM pg_stat_wal_receiver) IS NULL THEN NULL - ELSE extract(epoch FROM (clock_timestamp() - (SELECT last_msg_receipt_time FROM pg_stat_wal_receiver))) - END, - 'apply_backlog_bytes', CASE - WHEN pg_last_wal_receive_lsn() IS NULL OR pg_last_wal_replay_lsn() IS NULL THEN NULL - ELSE pg_wal_lsn_diff(pg_last_wal_receive_lsn(), pg_last_wal_replay_lsn()) - END -); -""" - with self.operation_timeout(timeout_seconds): - row = self._run_read_json(sql) - if not isinstance(row, dict): - raise RuntimeError("read replica status probe did not return an object") - return row - - @contextmanager - def operation_timeout(self, timeout_seconds: float) -> Iterator[None]: - """Bound PostgreSQL and local admission for the current thread. - - The block submitter uses this scope for direct outbox operations. - Nested scopes keep the earliest deadline, so helper calls cannot - accidentally widen the caller's liveness budget. - """ - timeout_seconds = float(timeout_seconds) - if not math.isfinite(timeout_seconds) or timeout_seconds <= 0: - raise ValueError("operation timeout must be finite and positive") - timeout_local = getattr(self, "_operation_timeout_local", None) - if timeout_local is None: - timeout_local = local() - self._operation_timeout_local = timeout_local - previous = getattr(timeout_local, "deadline", None) - deadline = self._monotonic() + timeout_seconds - timeout_local.deadline = ( - deadline if previous is None else min(float(previous), deadline) - ) - try: - yield - finally: - if previous is None: - try: - del timeout_local.deadline - except AttributeError: - pass - else: - timeout_local.deadline = previous - - @contextmanager - def statement_timeout(self, timeout_seconds: float) -> Iterator[None]: - """Apply a fresh bound to each lock admission and SQL statement. - - Unlike ``operation_timeout``, this budget does not start counting down - across non-database work between calls. The block accounting tail can - therefore build and verify an audit bundle before giving each later - Postgres step its own short deadline. - """ - timeout_seconds = float(timeout_seconds) - if not math.isfinite(timeout_seconds) or timeout_seconds <= 0: - raise ValueError("statement timeout must be finite and positive") - timeout_local = getattr(self, "_statement_timeout_local", None) - if timeout_local is None: - timeout_local = local() - self._statement_timeout_local = timeout_local - previous = getattr(timeout_local, "timeout_seconds", None) - timeout_local.timeout_seconds = ( - timeout_seconds - if previous is None - else min(float(previous), timeout_seconds) - ) - try: - yield - finally: - if previous is None: - try: - del timeout_local.timeout_seconds - except AttributeError: - pass - else: - timeout_local.timeout_seconds = previous - - @contextmanager - def operation_progress( - self, - on_progress: Callable[[], None], - *, - slice_seconds: float, - ) -> Iterator[None]: - """Stamp caller liveness while a ledger admission wait is blocked. - - A landing-class caller runs its ledger step directly on a - watchdog-monitored block-work thread. Waiting for the writer lock or - the read semaphore is not database work: no statement has been sent, - so neither ``statement_timeout`` nor any server-side cancellation - bounds it, and the ledger has nothing to report until admission - succeeds. Without this hook the whole admission budget is - heartbeat-silent, and a coordinator that is merely queued behind - another writer is hard-exited by its own watchdog mid-landing -- - which loses the escalation state the retry depended on and restarts - the same doomed cycle. - - The hook fires between acquire slices, so it never runs while this - thread holds the gate, and it never replaces the caller's deadline: - ``_operation_gate`` still raises at the deadline - ``_remaining_operation_timeout`` reports. An exception from - ``on_progress`` propagates to the caller; a liveness stamp that - cannot be taken is a real failure, not something to swallow inside a - lock wait. - - Nesting merges the way ``operation_timeout``/``statement_timeout`` - merge: the effective slice is the minimum of the enclosing slices, so - an inner scope can only tighten the stamp cadence, never widen it. A - nested scope carrying a larger slice would otherwise lengthen the - heartbeat gaps its enclosing caller had already sized against its own - monitor. The callback itself does replace -- the innermost caller is - the one whose liveness is at stake -- and the outer pair is restored - on exit. - """ - slice_seconds = float(slice_seconds) - if not math.isfinite(slice_seconds) or slice_seconds <= 0: - raise ValueError("operation progress slice must be finite and positive") - progress_local = getattr(self, "_operation_progress_local", None) - if progress_local is None: - progress_local = local() - self._operation_progress_local = progress_local - previous = getattr(progress_local, "hook", None) - progress_local.hook = ( - (on_progress, slice_seconds) - if previous is None - else (on_progress, min(float(previous[1]), slice_seconds)) - ) - try: - yield - finally: - if previous is None: - try: - del progress_local.hook - except AttributeError: - pass - else: - progress_local.hook = previous - - def _operation_progress_hook(self) -> tuple[Callable[[], None], float] | None: - progress_local = getattr(self, "_operation_progress_local", None) - if progress_local is None: - return None - return getattr(progress_local, "hook", None) - - def _note_operation_progress(self) -> None: - """Report liveness between the statements of one gate-holding step. - - ``_acquire_operation_gate`` stamps only while a caller is *waiting* - for a gate, which is all a single-statement operation needs: once the - gate opens the caller is inside one server-side statement, and its - liveness monitor is sized for exactly that. An operation that issues - a second statement without releasing the gate breaks that sizing. - No admission slice runs between the two -- the gate is already held -- - so the monitor sees one unbroken silence of two statement budgets - where it was promised one. That is indistinguishable from a wedged - operation, and a watchdog with any tolerance below twice the budget - hard-exits a coordinator that is in fact making normal progress - (issue #125). Reporting here restores the contract: the first - statement's full round trip has completed, which is precisely the - evidence a liveness monitor watches for, while a genuinely stuck - statement still produces no report at all. - - With no hook installed this does nothing; an ordinary ledger caller - has no monitor to satisfy. An exception from the hook propagates, - matching ``_acquire_operation_gate``: a liveness stamp that cannot be - taken is a real failure, not something to swallow mid-operation. - """ - hook = self._operation_progress_hook() - if hook is None: - return - on_progress, _slice_seconds = hook - on_progress() - - def _remaining_operation_timeout(self) -> float | None: - timeout_local = getattr(self, "_operation_timeout_local", None) - deadline = ( - getattr(timeout_local, "deadline", None) - if timeout_local is not None - else None - ) - statement_timeout_local = getattr( - self, - "_statement_timeout_local", - None, - ) - statement_timeout_seconds = ( - getattr(statement_timeout_local, "timeout_seconds", None) - if statement_timeout_local is not None - else None - ) - if deadline is None: - return ( - None - if statement_timeout_seconds is None - else float(statement_timeout_seconds) - ) - remaining = float(deadline) - self._monotonic() - if remaining <= 0: - raise LedgerOperationTimeout("postgres operation deadline expired") - if statement_timeout_seconds is not None: - remaining = min(remaining, float(statement_timeout_seconds)) - return remaining - - def _acquire_operation_gate(self, gate: Any, name: str) -> None: - """Wait for one ledger gate inside the caller's remaining deadline. - - With no progress hook installed this is a single blocking acquire, as - it has always been: an ordinary caller has no liveness monitor to - satisfy and gains nothing from waking up. With a hook installed the - same total wait is served in slices so the caller can stamp its - heartbeat between them (see ``operation_progress`` for why an - admission wait would otherwise be silent). Slicing never widens the - wait: the deadline derived from ``_remaining_operation_timeout`` stays - authoritative and still produces the same timeout error. - """ - remaining = self._remaining_operation_timeout() - hook = self._operation_progress_hook() - if hook is None: - acquired = ( - gate.acquire() - if remaining is None - else gate.acquire(timeout=max(0.0, remaining)) - ) - if not acquired: - raise LedgerOperationTimeout(f"timed out waiting for postgres {name}") - return - on_progress, slice_seconds = hook - deadline = ( - None if remaining is None else self._monotonic() + max(0.0, remaining) - ) - while True: - wait_seconds = slice_seconds - if deadline is not None: - wait_seconds = min(wait_seconds, deadline - self._monotonic()) - # An expired budget still gets one non-blocking attempt, so a - # zero or negative remaining fails exactly where it always did. - if gate.acquire(timeout=max(0.0, wait_seconds)): - return - if deadline is not None and self._monotonic() >= deadline: - raise LedgerOperationTimeout(f"timed out waiting for postgres {name}") - on_progress() - - @contextmanager - def _operation_gate(self, gate: Any, name: str) -> Iterator[None]: - """Acquire a ledger lock/semaphore within the caller's deadline.""" - self._acquire_operation_gate(gate, name) - try: - yield - finally: - gate.release() - - def append(self, pending: PendingShare) -> AcceptedShareRecord: - if pending.share_difficulty <= 0: - raise ValueError("share_difficulty must be positive") - if pending.network_difficulty <= 0: - raise ValueError("network_difficulty must be positive") - credit_policy = validate_credit_policy(pending.credit_policy) - payload = { - **pending.__dict__, - "credit_policy": credit_policy, - "writer_id": self._writer_id, - "writer_epoch": self._writer_epoch, - "writer_session_token": self._writer_session_token, - } - sql = f""" -WITH payload AS ( - SELECT {self._jsonb_literal(payload)} AS data -), -existing_share AS ( - SELECT share_seq - FROM qbit_share_ledger - WHERE share_id = (SELECT data->>'share_id' FROM payload) -), -existing_miner AS ( - SELECT 1 - FROM qbit_share_ledger - WHERE accepted - AND miner_id = (SELECT data->>'miner_id' FROM payload) - LIMIT 1 -), -lease AS ( - UPDATE qbit_ledger_writer_lease - SET lease_expires_at = clock_timestamp() + {self._lease_interval_sql}, - updated_at = clock_timestamp() - FROM payload - WHERE qbit_ledger_writer_lease.singleton - AND qbit_ledger_writer_lease.writer_id = data->>'writer_id' - AND qbit_ledger_writer_lease.writer_epoch = (data->>'writer_epoch')::bigint - AND qbit_ledger_writer_lease.writer_session_token = data->>'writer_session_token' - RETURNING qbit_ledger_writer_lease.writer_id -), -inserted AS ( - INSERT INTO qbit_share_ledger ( - share_id, - miner_id, - payout_order_key, - p2mr_program, - share_difficulty, - network_difficulty, - template_height, - job_id, - job_issued_at, - ntime, - accepted_at, - credit_policy, - accepted, - writer_id, - writer_epoch - ) - SELECT - data->>'share_id', - data->>'miner_id', - data->>'order_key', - decode(data->>'p2mr_program_hex', 'hex'), - (data->>'share_difficulty')::numeric, - (data->>'network_difficulty')::numeric, - (data->>'template_height')::bigint, - data->>'job_id', - to_timestamp(((data->>'job_issued_at_ms')::double precision / 1000.0)), - (data->>'ntime')::bigint, - to_timestamp(((data->>'accepted_at_ms')::double precision / 1000.0)), - data->>'credit_policy', - true, - data->>'writer_id', - (data->>'writer_epoch')::bigint - FROM payload, lease - WHERE NOT EXISTS (SELECT 1 FROM existing_share) - RETURNING * -) -SELECT CASE - WHEN (SELECT count(*) FROM lease) = 0 THEN - json_build_object('error', 'writer lease is not active') - WHEN EXISTS (SELECT 1 FROM existing_share) THEN - json_build_object('error', 'duplicate share_id') - ELSE - (SELECT json_build_object( - 'share_seq', share_seq, - 'share_id', share_id, - 'miner_id', miner_id, - 'order_key', payout_order_key, - 'p2mr_program_hex', encode(p2mr_program, 'hex'), - 'share_difficulty', share_difficulty::text, - 'network_difficulty', network_difficulty::text, - 'template_height', template_height, - 'job_id', job_id, - 'job_issued_at_ms', round(extract(epoch FROM job_issued_at) * 1000)::bigint, - 'accepted_at_ms', round(extract(epoch FROM accepted_at) * 1000)::bigint, - 'ntime', ntime, - 'credit_policy', credit_policy, - 'new_miner', NOT EXISTS (SELECT 1 FROM existing_miner) - ) FROM inserted) -END; -""" - # Serialize the single writer through its durable commit and cache note. - # Stats reconciliation uses a separate read connection plus a share-seq - # watermark, so it never acquires this writer lock. - with self._operation_gate(self._lock, "writer lock"): - result = self._run_json(sql) - if "error" in result: - raise RuntimeError(str(result["error"])) - record = self._record_from_json(result) - self._note_appended_share( - record, - new_miner=bool(result.get("new_miner", False)), - ) - return record - - def _append_batch_with_replay_outcomes( - self, - entries: list[tuple[PendingShare, Any]], - ) -> list[ShareReplayResult]: - """Commit accepted shares and optional block intents in one transaction. - - Replaying the exact same payload is idempotent. Reusing a share ID or - block hash with different content fails the whole batch — a share-ID - payload mismatch raises the typed :class:`ShareReplayConflict`. - Postgres assigns the share sequence and makes every row visible before - this method returns, which is the coordinator's Stratum ACK boundary. - Each entry's outcome reports whether its row was ``inserted`` by this - statement or was an ``exact_existing`` durable duplicate. - - A batch with no candidate keeps the historical statement. A batch - carrying candidates takes the chunked-body route (#255) unless the - ledger was constructed in legacy storage mode, in which case the - candidate is materialized and written as the historical jsonb. - """ - if not entries: - return [] - if any(candidate is not None for _, candidate in entries): - if self._candidate_storage_version_value() == CANDIDATE_BODY_STORAGE_VERSION: - return self._append_batch_with_replay_outcomes_v2(entries) - entries = [ - (pending, self._legacy_candidate_document(candidate)) - for pending, candidate in entries - ] - return self._append_batch_with_replay_outcomes_v1(entries) - - def _append_batch_with_replay_outcomes_v1( - self, - entries: list[tuple[PendingShare, dict[str, Any] | None]], - ) -> list[ShareReplayResult]: - """The historical whole-jsonb batch statement (storage version 1).""" - payloads: list[dict[str, Any]] = [] - share_ids: set[str] = set() - block_hashes: set[str] = set() - for pending, candidate in entries: - if pending.share_difficulty <= 0: - raise ValueError("share_difficulty must be positive") - if pending.network_difficulty <= 0: - raise ValueError("network_difficulty must be positive") - if pending.share_id in share_ids: - raise ValueError("duplicate share_id in append batch") - share_ids.add(pending.share_id) - candidate_payload = candidate - if candidate_payload is not None: - block_hash = str(candidate_payload.get("block_hash_hex", "")).lower() - if not block_hash: - raise ValueError("block candidate is missing block_hash_hex") - if block_hash in block_hashes: - raise ValueError("duplicate block candidate in append batch") - block_hashes.add(block_hash) - candidate_payload = {**candidate_payload, "block_hash_hex": block_hash} - payloads.append( - { - "share": { - **pending.__dict__, - "credit_policy": validate_credit_policy(pending.credit_policy), - }, - "candidate": candidate_payload, - "candidate_sha256": ( - block_candidate_identity_sha256(candidate_payload) - if candidate_payload is not None - else None - ), - } - ) - payload = { - "entries": payloads, - "writer_id": self._writer_id, - "writer_epoch": self._writer_epoch, - "writer_session_token": self._writer_session_token, - } - sql = f""" -WITH input AS ( - SELECT - {self._jsonb_literal(payload)} AS root, - set_config('synchronous_commit', 'on', true) AS durability -), -payload AS ( - SELECT - item->'share' AS data, - NULLIF(item->'candidate', 'null'::jsonb) AS candidate, - item->>'candidate_sha256' AS candidate_sha256, - ordinality - FROM input, - jsonb_array_elements(root->'entries') WITH ORDINALITY AS rows(item, ordinality) -), -lease AS ( - UPDATE qbit_ledger_writer_lease - SET lease_expires_at = clock_timestamp() + {self._lease_interval_sql}, - updated_at = clock_timestamp() - FROM input - WHERE qbit_ledger_writer_lease.singleton - AND qbit_ledger_writer_lease.writer_id = root->>'writer_id' - AND qbit_ledger_writer_lease.writer_epoch = (root->>'writer_epoch')::bigint - AND qbit_ledger_writer_lease.writer_session_token = root->>'writer_session_token' - RETURNING qbit_ledger_writer_lease.writer_id -), -share_mismatch AS ( - SELECT data->>'share_id' AS share_id - FROM payload - JOIN qbit_share_ledger ledger ON ledger.share_id = data->>'share_id' - WHERE ledger.miner_id IS DISTINCT FROM data->>'miner_id' - OR ledger.payout_order_key IS DISTINCT FROM data->>'order_key' - OR ledger.p2mr_program IS DISTINCT FROM decode(data->>'p2mr_program_hex', 'hex') - OR ledger.share_difficulty IS DISTINCT FROM (data->>'share_difficulty')::numeric - OR ledger.network_difficulty IS DISTINCT FROM (data->>'network_difficulty')::numeric - OR ledger.template_height IS DISTINCT FROM (data->>'template_height')::bigint - OR ledger.job_id IS DISTINCT FROM data->>'job_id' - OR ledger.job_issued_at IS DISTINCT FROM to_timestamp((data->>'job_issued_at_ms')::double precision / 1000.0) - OR ledger.ntime IS DISTINCT FROM (data->>'ntime')::bigint - OR ledger.credit_policy IS DISTINCT FROM data->>'credit_policy' -), -candidate_mismatch AS ( - SELECT payload.candidate->>'block_hash_hex' AS block_hash - FROM payload - JOIN qbit_block_candidate_outbox outbox - ON outbox.block_hash = payload.candidate->>'block_hash_hex' - WHERE payload.candidate IS NOT NULL - AND ((outbox.share_id IS NOT NULL AND outbox.share_id IS DISTINCT FROM payload.data->>'share_id') - OR outbox.candidate_sha256 IS DISTINCT FROM payload.candidate_sha256 - OR (outbox.candidate IS NOT NULL - AND (outbox.candidate #- '{{pending_share,accepted_at_ms}}') - IS DISTINCT FROM (payload.candidate #- '{{pending_share,accepted_at_ms}}'))) -), -candidate_states AS ( - SELECT - payload.ordinality, - CASE - WHEN payload.candidate IS NULL THEN NULL - ELSE COALESCE(outbox.state, 'pending') - END AS candidate_outbox_state - FROM payload - LEFT JOIN qbit_block_candidate_outbox outbox - ON outbox.block_hash = payload.candidate->>'block_hash_hex' -), -batch_ok AS ( - SELECT 1 AS ok - WHERE EXISTS (SELECT 1 FROM lease) - AND NOT EXISTS (SELECT 1 FROM share_mismatch) - AND NOT EXISTS (SELECT 1 FROM candidate_mismatch) -), -inserted_shares AS ( - INSERT INTO qbit_share_ledger ( - share_id, miner_id, payout_order_key, p2mr_program, - share_difficulty, network_difficulty, template_height, job_id, - job_issued_at, ntime, accepted_at, credit_policy, accepted, - writer_id, writer_epoch - ) - SELECT - data->>'share_id', data->>'miner_id', data->>'order_key', - decode(data->>'p2mr_program_hex', 'hex'), - (data->>'share_difficulty')::numeric, - (data->>'network_difficulty')::numeric, - (data->>'template_height')::bigint, data->>'job_id', - to_timestamp((data->>'job_issued_at_ms')::double precision / 1000.0), - (data->>'ntime')::bigint, - to_timestamp((data->>'accepted_at_ms')::double precision / 1000.0), - data->>'credit_policy', true, root->>'writer_id', - (root->>'writer_epoch')::bigint - FROM payload, input, batch_ok - WHERE NOT EXISTS ( - SELECT 1 FROM qbit_share_ledger existing - WHERE existing.share_id = payload.data->>'share_id' - ) - ORDER BY payload.ordinality - ON CONFLICT (share_id) DO NOTHING - RETURNING qbit_share_ledger.* -), -inserted_candidates AS ( - INSERT INTO qbit_block_candidate_outbox ( - block_hash, share_id, candidate, candidate_sha256 - ) - SELECT - payload.candidate->>'block_hash_hex', payload.data->>'share_id', - payload.candidate, payload.candidate_sha256 - FROM payload, batch_ok - WHERE payload.candidate IS NOT NULL - ON CONFLICT (block_hash) DO UPDATE - SET share_id = EXCLUDED.share_id, - updated_at = clock_timestamp() - WHERE qbit_block_candidate_outbox.share_id IS NULL - AND qbit_block_candidate_outbox.candidate_sha256 = EXCLUDED.candidate_sha256 - RETURNING block_hash -), -records AS ( - SELECT - ledger.*, payload.ordinality, false AS newly_inserted, - false AS new_miner, candidate_states.candidate_outbox_state - FROM payload - JOIN qbit_share_ledger ledger ON ledger.share_id = payload.data->>'share_id' - JOIN candidate_states ON candidate_states.ordinality = payload.ordinality - UNION ALL - SELECT - inserted_shares.*, payload.ordinality, true AS newly_inserted, - -- Data-modifying CTEs and this main query share one pre-statement - -- MVCC snapshot. This base-table probe cannot see inserted_shares; - -- only the RETURNING CTE exposes those new rows to this statement. - NOT EXISTS ( - SELECT 1 - FROM qbit_share_ledger existing_miner - WHERE existing_miner.accepted - AND existing_miner.miner_id = inserted_shares.miner_id - ) - AND NOT EXISTS ( - SELECT 1 - FROM inserted_shares earlier_insert - WHERE earlier_insert.miner_id = inserted_shares.miner_id - AND earlier_insert.share_seq < inserted_shares.share_seq - ) AS new_miner, - candidate_states.candidate_outbox_state - FROM inserted_shares - JOIN payload ON payload.data->>'share_id' = inserted_shares.share_id - JOIN candidate_states ON candidate_states.ordinality = payload.ordinality -) -SELECT CASE - WHEN NOT EXISTS (SELECT 1 FROM lease) THEN - json_build_object('error', 'writer lease is not active') - WHEN EXISTS (SELECT 1 FROM share_mismatch) THEN - json_build_object( - 'error', 'duplicate share_id payload mismatch', - 'error_kind', 'share_replay_conflict', - 'share_ids', (SELECT json_agg(share_id ORDER BY share_id) FROM share_mismatch) - ) - WHEN EXISTS (SELECT 1 FROM candidate_mismatch) THEN - json_build_object( - 'error', 'block candidate payload mismatch', - 'block_hashes', (SELECT json_agg(block_hash ORDER BY block_hash) FROM candidate_mismatch) - ) - ELSE json_build_object( - 'records', ( - SELECT json_agg(json_build_object( - 'share_seq', records.share_seq, - 'share_id', records.share_id, - 'miner_id', records.miner_id, - 'order_key', records.payout_order_key, - 'p2mr_program_hex', encode(records.p2mr_program, 'hex'), - 'share_difficulty', records.share_difficulty::text, - 'network_difficulty', records.network_difficulty::text, - 'template_height', records.template_height, - 'job_id', records.job_id, - 'job_issued_at_ms', round(extract(epoch FROM records.job_issued_at) * 1000)::bigint, - 'accepted_at_ms', round(extract(epoch FROM records.accepted_at) * 1000)::bigint, - 'ntime', records.ntime, - 'credit_policy', records.credit_policy, - 'newly_inserted', records.newly_inserted, - 'new_miner', records.new_miner, - 'candidate_outbox_state', records.candidate_outbox_state - ) ORDER BY records.ordinality) - FROM records - ) - ) -END; -""" - with self._operation_gate(self._lock, "writer lock"): - result = self._run_json(sql) - if "error" in result: - if result.get("error_kind") == "share_replay_conflict": - raise ShareReplayConflict(str(result["error"])) - raise RuntimeError(str(result["error"])) - records = result.get("records") - if not isinstance(records, list) or len(records) != len(entries): - raise RuntimeError("Postgres share batch returned an incomplete result") - parsed = [self._record_from_json(record) for record in records] - committed = sorted( - zip(records, parsed, strict=True), - key=lambda item: item[1].share_seq, - ) - for payload, record in committed: - if bool(payload.get("newly_inserted", True)): - self._note_appended_share( - record, - new_miner=bool(payload.get("new_miner", False)), - ) - return [ - ShareReplayResult( - ( - "inserted" - if bool(payload.get("newly_inserted", True)) - else "exact_existing" - ), - record, - ) - for payload, record in zip(records, parsed, strict=True) - ] - - # -- chunked candidate bodies (#255) ----------------------------------- - - def _candidate_storage_version_value(self) -> int: - return int( - getattr(self, "_candidate_storage_version", CANDIDATE_BODY_STORAGE_VERSION) - ) - - def _ensure_candidate_store_state(self) -> None: - """Backfill #255 state for ledgers built without ``__init__``.""" - if not hasattr(self, "_candidate_body_gate"): - self._candidate_body_gate = BoundedSemaphore(1) - if not hasattr(self, "_candidate_spool"): - self._candidate_spool = SpoolAdmission() - if not hasattr(self, "_legacy_candidate_helper"): - command = getattr(self, "_command", None) or [] - self._legacy_candidate_helper = LegacyCandidateHelper( - LegacyTransport( - psql_command=" ".join(shlex.quote(part) for part in command) or None, - database_url=database_url_from_psql_command(list(command)), - ) - ) - - @staticmethod - def _legacy_candidate_document(candidate: Any) -> Any: - """Materialize an intent for the legacy whole-jsonb route only. - - Storage version 1 is the compatibility/rollback mode; its statement - inlines the whole document, so the share sequence is copied here - and nowhere else. A candidate-only intent keeps no share member, - and a hydrated replay (the rollback floor re-persisting a version-2 - row) has its streamed views decoded, because this route writes - plain JSON. - """ - if candidate is None or not isinstance(candidate, PreparedCandidateIntent): - return candidate - document = materialize_spool_views(candidate.facts) - if candidate.has_shares: - document["shares_json"] = [ - materialize_spool_views(share) if is_spool_view(share) else share - for share in candidate.shares - ] - return document - - def _writer_identity_payload(self) -> dict[str, Any]: - return { - "writer_id": self._writer_id, - "writer_epoch": int(self._writer_epoch), - "writer_session_token": self._writer_session_token, - } - - def _observe_candidate_outbox_row(self, block_hash: str) -> dict[str, Any] | None: - """One read-slot snapshot of an outbox row's publication facts. - - This is the version the precomparison binds to: ``state``, - ``share_id``, ``candidate_sha256``, ``storage_version`` and - ``body_id`` are re-asserted by the fenced statement, so any change - between this read and the commit fails that statement closed and - the caller redoes the comparison. - """ - sql = f""" -SELECT json_build_object( - 'found', EXISTS ( - SELECT 1 FROM qbit_block_candidate_outbox WHERE block_hash = {self._text_literal(block_hash)} - ), - 'state', (SELECT state FROM qbit_block_candidate_outbox WHERE block_hash = {self._text_literal(block_hash)}), - 'share_id', (SELECT share_id FROM qbit_block_candidate_outbox WHERE block_hash = {self._text_literal(block_hash)}), - 'candidate_sha256', (SELECT candidate_sha256 FROM qbit_block_candidate_outbox WHERE block_hash = {self._text_literal(block_hash)}), - 'storage_version', (SELECT storage_version FROM qbit_block_candidate_outbox WHERE block_hash = {self._text_literal(block_hash)}), - 'body_id', (SELECT body_id FROM qbit_block_candidate_outbox WHERE block_hash = {self._text_literal(block_hash)}), - 'body_state', ( - SELECT body.state - FROM qbit_block_candidate_body body - JOIN qbit_block_candidate_outbox outbox ON outbox.body_id = body.body_id - WHERE outbox.block_hash = {self._text_literal(block_hash)} - ) -); -""" - observed = self._run_attributed_read_json( - sql, - operation="observe_block_candidate_outbox_row", - gate=self._read_semaphore, - gate_name="read slot", - ) - if not isinstance(observed, dict) or not observed.get("found"): - return None - return { - "state": str(observed.get("state")), - "share_id": ( - None if observed.get("share_id") is None else str(observed["share_id"]) - ), - "candidate_sha256": str(observed.get("candidate_sha256")), - "storage_version": int(observed.get("storage_version") or 1), - "body_id": ( - None if observed.get("body_id") is None else str(observed["body_id"]) - ), - "body_state": ( - None if observed.get("body_state") is None else str(observed["body_state"]) - ), - } - - def verify_candidate_schema(self) -> dict[str, Any]: - """Refuse a database whose candidate storage is newer than this code. - - Reads what the schema declares (``qbit_prism_schema_capabilities``) - and whether the chunked-body tables exist. A declared storage - version above the one this process understands raises - :class:`IncompatibleCandidateSchema`; so does a database without - the 002 migration, whichever storage version is configured (replay - and the terminal outbox statements use its columns), before any - candidate is staged. Cached after the first successful check; the - coordinator also calls it at boot. - """ - cached = getattr(self, "_candidate_schema_verified", None) - if cached is not None: - return cached - presence = self._run_json( - """ -SELECT json_build_object( - 'has_body_table', to_regclass('qbit_block_candidate_body') IS NOT NULL, - 'has_capabilities', to_regclass('qbit_prism_schema_capabilities') IS NOT NULL -); -""" - ) - has_body_table = bool(isinstance(presence, dict) and presence.get("has_body_table")) - declared: int | None = None - if isinstance(presence, dict) and presence.get("has_capabilities"): - capability = self._run_json(schema_capability_sql()) - if isinstance(capability, dict) and capability.get("declared") is not None: - declared = int(capability["declared"]) - refusal = candidate_schema_refusal(declared, has_body_table) - if refusal is not None: - raise IncompatibleCandidateSchema(refusal) - verdict = {"declared": declared, "has_body_table": has_body_table} - self._candidate_schema_verified = verdict - return verdict - - def stage_candidate_body(self, prepared: PreparedCandidateIntent) -> str: - """Upload and seal one immutable body; returns its ``body_id``. - - Runs under the one-slot body gate, never the writer lock: staging - touches no lease row and grants no credit. One statement per chunk - (each bounded by the chunk size), index rows in batches of at most - 256 pages / 64 spans, then one seal statement that locks the - manifest, proves completeness from the rows and re-hashes every - chunk on the server. A failure retires the staging manifest so the - janitor reclaims it. The schema-capability refusal runs at - coordinator boot (``verify_candidate_schema``), not here. - """ - self._ensure_candidate_store_state() - manifest = prepared.manifest - body_id = new_body_id() - identity = self._writer_identity_payload() - with self._operation_gate(self._candidate_body_gate, "candidate body slot"): - staged = self._run_json( - stage_body_sql( - { - **manifest.to_json(), - "body_id": body_id, - "block_hash": prepared.block_hash, - **identity, - }, - jsonb=self._jsonb_literal, - ) - ) - if not isinstance(staged, dict) or staged.get("state") != "staging": - raise CandidateStorageError( - f"candidate body staging was not created: {staged!r}" - ) - try: - pending_spans: list[dict[str, Any]] = [] - - def flush_index(spans: list[dict[str, Any]], pages: list[list[Any]]) -> None: - if not spans and not pages: - return - outcome = self._run_json( - body_index_rows_sql( - {"body_id": body_id, **identity, "spans": spans, "pages": pages}, - jsonb=self._jsonb_literal, - ) - ) - if not isinstance(outcome, dict) or not outcome.get("owned"): - raise CandidateStorageError( - "candidate body staging lost its producer session" - ) - - def upload(chunk: Any) -> None: - outcome = self._run_json( - body_chunk_sql( - { - "body_id": body_id, - "ordinal": chunk.ordinal, - "chunk_sha256": chunk.sha256, - "length": len(chunk.data), - **identity, - }, - chunk.data, - jsonb=self._jsonb_literal, - ) - ) - if not isinstance(outcome, dict) or not outcome.get("owned"): - raise CandidateStorageError( - "candidate body staging lost its producer session" - ) - - def on_span(span: Any) -> None: - pending_spans.append(span.to_json()) - if len(pending_spans) >= INDEX_SPAN_ROWS: - flush_index(list(pending_spans), []) - pending_spans.clear() - - def on_page(batch: Any) -> None: - flush_index( - [], - [ - [entry.field, entry.ordinal, entry.offset, entry.record_index] - for entry in batch - ], - ) - - prepared.body.write_chunks(upload, on_span=on_span, on_page=on_page) - flush_index(list(pending_spans), []) - pending_spans.clear() - sealed = self._run_json( - seal_body_sql( - { - "body_id": body_id, - "candidate_sha256": manifest.candidate_sha256, - **identity, - }, - jsonb=self._jsonb_literal, - ) - ) - if not isinstance(sealed, dict) or int(sealed.get("sealed", 0)) != 1: - raise CandidateStorageError( - f"candidate body could not be sealed: {sealed!r}" - ) - except BaseException: - self._retire_staging_body(body_id) - raise - return body_id - - def _retire_staging_body(self, body_id: str) -> None: - """Best-effort: hand a failed staging body to the janitor.""" - try: - self._run_json(retire_staging_body_sql(body_id, text=self._text_literal)) - except Exception: # noqa: BLE001 - the janitor retires stale staging anyway - pass - - def compare_candidate_body( - self, - body_id: str, - prepared: PreparedCandidateIntent, - ) -> bool: - """The historical content comparator for a stored chunked body. - - Historically an existing row was accepted only when its identity - digest matched *and* its normalized jsonb equalled the new intent's. - The caller has already matched the digest; this is the second - check, kept as a comparison of content rather than of any digest: - - 1. the actual stored bytes are read back one bounded page at a time - (at most three chunks per statement) and compared with our - re-encoded bytes chunk by chunk, holding one page at a time; - 2. if the bytes differ, the normalized jsonb-equivalence fallback - runs in the isolated helper (numeric and key-order differences - that jsonb treats as equal), exactly the equality PostgreSQL - applied to the two jsonb documents before. - - A forced digest collision between distinct documents is therefore - rejected here, and a jsonb-equivalent retry with a different - encoding still passes. The result is bound to the body version the - fenced statement re-asserts (compare-and-swap). - """ - self._ensure_candidate_store_state() - manifest_row = self.read_candidate_body_manifest(body_id) - try: - manifest, state, _referenced = parse_manifest_row(manifest_row) - except (CandidateBodyUnavailable, CandidateBodyIntegrityError): - return False - if state != "sealed": - return False - bytes_equal = ( - manifest.byte_count == prepared.manifest.byte_count - and manifest.chunk_count == prepared.manifest.chunk_count - ) - if bytes_equal: - - class _Mismatch(Exception): - pass - - page: dict[int, bytes] = {} - - def consumer(chunk: Any) -> None: - if chunk.ordinal not in page: - page.clear() - page_state, _referenced, stored = parse_body_page( - self.read_candidate_body_page(body_id, chunk.ordinal, BODY_READ_MAX_CHUNKS), - max_chunks=BODY_READ_MAX_CHUNKS, - ) - if page_state != "sealed": - raise _Mismatch() - for entry in stored: - page[entry.ordinal] = entry.data - stored_bytes = page.get(chunk.ordinal) - if stored_bytes is None or stored_bytes != chunk.data: - raise _Mismatch() - - try: - prepared.body.write_chunks(consumer) - except _Mismatch: - bytes_equal = False - if bytes_equal: - return True - return bool( - self._legacy_candidate_helper.compare_stored_body( - body_id, prepared.block_hash, prepared.body - ) - ) - - def read_candidate_body_manifest(self, body_id: str) -> Any: - return self._run_attributed_read_json( - body_manifest_sql(body_id, text=self._text_literal), - operation="read_block_candidate_body_manifest", - gate=self._read_semaphore, - gate_name="read slot", - ) - - def read_candidate_body_spans(self, body_id: str, after_field: str | None) -> Any: - return self._run_attributed_read_json( - body_spans_sql(body_id, after_field, text=self._text_literal), - operation="read_block_candidate_body_spans", - gate=self._read_semaphore, - gate_name="read slot", - ) - - def read_candidate_body_pages(self, body_id: str, field_name: str, from_ordinal: int) -> Any: - return self._run_attributed_read_json( - body_pages_sql(body_id, field_name, from_ordinal, text=self._text_literal), - operation="read_block_candidate_body_pages", - gate=self._read_semaphore, - gate_name="read slot", - ) - - def read_candidate_body_page(self, body_id: str, from_ordinal: int, max_chunks: int) -> Any: - return self._run_attributed_read_json( - body_page_sql(body_id, from_ordinal, max_chunks, text=self._text_literal), - operation="read_block_candidate_body_page", - gate=self._read_semaphore, - gate_name="read slot", - ) - - def _bind_candidate_publication( - self, - prepared: PreparedCandidateIntent, - share_id: str, - ) -> dict[str, Any]: - """Precompare and stage outside the lease-row critical section. - - Returns the observed version the fenced statement must find - unchanged. A missing row means the body is staged and sealed now; an - existing row is compared exactly -- digest, share linkage, and for a - pending row the full body (server chunk re-hash for a chunked body, - the isolated compatibility helper for a legacy jsonb body). Terminal - rows compare on digest and linkage only, exactly as the historical - statement did once their payload had been deleted. - """ - observed = self._observe_candidate_outbox_row(prepared.block_hash) - if observed is None: - return { - "found": False, - "state": None, - "share_id": None, - "storage_version": None, - "body_id": self.stage_candidate_body(prepared), - } - if observed["candidate_sha256"] != prepared.candidate_sha256: - raise RuntimeError("block candidate payload mismatch") - if observed["share_id"] is not None and observed["share_id"] != share_id: - raise RuntimeError("block candidate payload mismatch") - if observed["state"] == "pending": - if observed["storage_version"] == CANDIDATE_BODY_STORAGE_VERSION: - if observed["body_id"] is None or not self.compare_candidate_body( - observed["body_id"], prepared - ): - raise RuntimeError("block candidate payload mismatch") - else: - self._ensure_candidate_store_state() - if not self._legacy_candidate_helper.compare( - prepared.block_hash, prepared.body - ): - raise RuntimeError("block candidate payload mismatch") - return { - "found": True, - "state": observed["state"], - "share_id": observed["share_id"], - "storage_version": observed["storage_version"], - "body_id": observed["body_id"], - } - - def _append_batch_with_replay_outcomes_v2( - self, - entries: list[tuple[PendingShare, Any]], - ) -> list[ShareReplayResult]: - """Chunked-body batch: stage outside the fence, publish inside it. - - Same lease fence, durability setting, share comparator, ordering - and ACK boundary as the historical statement. The candidate half - differs only in *where* body-sized work happens: precomparison and - staging run before the writer gate is taken, and the fenced - statement re-asserts the observed row version (compare-and-swap) - and requires a sealed manifest carrying the same digest before it - publishes a reference. If the version moved, nothing is committed - and the whole bind/publish is redone under the same call. - """ - prepared_entries: list[tuple[PendingShare, PreparedCandidateIntent | None]] = [] - share_ids: set[str] = set() - block_hashes: set[str] = set() - for pending, candidate in entries: - if pending.share_difficulty <= 0: - raise ValueError("share_difficulty must be positive") - if pending.network_difficulty <= 0: - raise ValueError("network_difficulty must be positive") - if pending.share_id in share_ids: - raise ValueError("duplicate share_id in append batch") - share_ids.add(pending.share_id) - prepared: PreparedCandidateIntent | None = None - if candidate is not None: - prepared = prepare_candidate_intent(candidate) - if prepared.block_hash in block_hashes: - raise ValueError("duplicate block candidate in append batch") - block_hashes.add(prepared.block_hash) - prepared_entries.append((pending, prepared)) - attempts = 0 - while True: - attempts += 1 - payloads: list[dict[str, Any]] = [] - for pending, prepared in prepared_entries: - candidate_payload: dict[str, Any] | None = None - if prepared is not None: - candidate_payload = { - "block_hash_hex": prepared.block_hash, - "candidate_sha256": prepared.candidate_sha256, - "header": prepared.replay_header(), - "expected": self._bind_candidate_publication( - prepared, pending.share_id - ), - } - candidate_payload["body_id"] = candidate_payload["expected"]["body_id"] - payloads.append( - { - "share": { - **pending.__dict__, - "credit_policy": validate_credit_policy(pending.credit_policy), - }, - "candidate": candidate_payload, - } - ) - outcome = self._publish_candidate_batch(payloads, len(prepared_entries)) - if outcome is not None: - return outcome - if attempts >= 3: - raise RuntimeError( - "block candidate outbox changed repeatedly during publication" - ) - - def _publish_candidate_batch( - self, - payloads: list[dict[str, Any]], - expected_count: int, - ) -> list[ShareReplayResult] | None: - payload = {"entries": payloads, **self._writer_identity_payload()} - sql = f""" -WITH input AS ( - SELECT - {self._jsonb_literal(payload)} AS root, - set_config('synchronous_commit', 'on', true) AS durability -), -payload AS ( - SELECT - item->'share' AS data, - NULLIF(item->'candidate', 'null'::jsonb) AS candidate, - ordinality - FROM input, - jsonb_array_elements(root->'entries') WITH ORDINALITY AS rows(item, ordinality) -), -lease AS ( - UPDATE qbit_ledger_writer_lease - SET lease_expires_at = clock_timestamp() + {self._lease_interval_sql}, - updated_at = clock_timestamp() - FROM input - WHERE qbit_ledger_writer_lease.singleton - AND qbit_ledger_writer_lease.writer_id = root->>'writer_id' - AND qbit_ledger_writer_lease.writer_epoch = (root->>'writer_epoch')::bigint - AND qbit_ledger_writer_lease.writer_session_token = root->>'writer_session_token' - RETURNING qbit_ledger_writer_lease.writer_id -), -share_mismatch AS ( - SELECT data->>'share_id' AS share_id - FROM payload - JOIN qbit_share_ledger ledger ON ledger.share_id = data->>'share_id' - WHERE ledger.miner_id IS DISTINCT FROM data->>'miner_id' - OR ledger.payout_order_key IS DISTINCT FROM data->>'order_key' - OR ledger.p2mr_program IS DISTINCT FROM decode(data->>'p2mr_program_hex', 'hex') - OR ledger.share_difficulty IS DISTINCT FROM (data->>'share_difficulty')::numeric - OR ledger.network_difficulty IS DISTINCT FROM (data->>'network_difficulty')::numeric - OR ledger.template_height IS DISTINCT FROM (data->>'template_height')::bigint - OR ledger.job_id IS DISTINCT FROM data->>'job_id' - OR ledger.job_issued_at IS DISTINCT FROM to_timestamp((data->>'job_issued_at_ms')::double precision / 1000.0) - OR ledger.ntime IS DISTINCT FROM (data->>'ntime')::bigint - OR ledger.credit_policy IS DISTINCT FROM data->>'credit_policy' -), -candidate_mismatch AS ( - -- Compare-and-swap against the version the precomparison observed. - SELECT payload.candidate->>'block_hash_hex' AS block_hash - FROM payload - LEFT JOIN qbit_block_candidate_outbox outbox - ON outbox.block_hash = payload.candidate->>'block_hash_hex' - WHERE payload.candidate IS NOT NULL - AND ( - (outbox.block_hash IS NOT NULL) IS DISTINCT FROM (payload.candidate->'expected'->>'found')::boolean - OR ( - outbox.block_hash IS NOT NULL - AND ( - outbox.state IS DISTINCT FROM payload.candidate->'expected'->>'state' - OR outbox.share_id IS DISTINCT FROM payload.candidate->'expected'->>'share_id' - OR outbox.candidate_sha256 IS DISTINCT FROM payload.candidate->>'candidate_sha256' - OR outbox.storage_version IS DISTINCT FROM (payload.candidate->'expected'->>'storage_version')::integer - OR outbox.body_id IS DISTINCT FROM payload.candidate->'expected'->>'body_id' - OR (outbox.share_id IS NOT NULL AND outbox.share_id IS DISTINCT FROM payload.data->>'share_id') - ) - ) - ) -), -sealed_bodies AS ( - -- The bodies this statement may reference, locked FOR SHARE so an - -- orphan retirement (FOR UPDATE) serializes against this publication - -- and whichever side loses the lock re-evaluates and fails closed. - SELECT body.body_id - FROM qbit_block_candidate_body body - WHERE body.body_id IN ( - SELECT payload.candidate->>'body_id' - FROM payload - WHERE payload.candidate IS NOT NULL - AND NOT (payload.candidate->'expected'->>'found')::boolean - ) - AND body.state = 'sealed' - AND EXISTS ( - SELECT 1 FROM payload - WHERE payload.candidate->>'body_id' = body.body_id - AND body.candidate_sha256 = payload.candidate->>'candidate_sha256' - AND body.block_hash = payload.candidate->>'block_hash_hex' - ) - FOR SHARE -), -candidate_unsealed AS ( - -- A new reference may point only at a complete sealed body with the - -- same digest and block hash. - SELECT payload.candidate->>'block_hash_hex' AS block_hash - FROM payload - WHERE payload.candidate IS NOT NULL - AND NOT (payload.candidate->'expected'->>'found')::boolean - AND NOT EXISTS ( - SELECT 1 FROM sealed_bodies - WHERE sealed_bodies.body_id = payload.candidate->>'body_id' - ) -), -candidate_states AS ( - SELECT - payload.ordinality, - CASE - WHEN payload.candidate IS NULL THEN NULL - ELSE COALESCE(outbox.state, 'pending') - END AS candidate_outbox_state - FROM payload - LEFT JOIN qbit_block_candidate_outbox outbox - ON outbox.block_hash = payload.candidate->>'block_hash_hex' -), -batch_ok AS ( - SELECT 1 AS ok - WHERE EXISTS (SELECT 1 FROM lease) - AND NOT EXISTS (SELECT 1 FROM share_mismatch) - AND NOT EXISTS (SELECT 1 FROM candidate_mismatch) - AND NOT EXISTS (SELECT 1 FROM candidate_unsealed) -), -inserted_shares AS ( - INSERT INTO qbit_share_ledger ( - share_id, miner_id, payout_order_key, p2mr_program, - share_difficulty, network_difficulty, template_height, job_id, - job_issued_at, ntime, accepted_at, credit_policy, accepted, - writer_id, writer_epoch - ) - SELECT - data->>'share_id', data->>'miner_id', data->>'order_key', - decode(data->>'p2mr_program_hex', 'hex'), - (data->>'share_difficulty')::numeric, - (data->>'network_difficulty')::numeric, - (data->>'template_height')::bigint, data->>'job_id', - to_timestamp((data->>'job_issued_at_ms')::double precision / 1000.0), - (data->>'ntime')::bigint, - to_timestamp((data->>'accepted_at_ms')::double precision / 1000.0), - data->>'credit_policy', true, root->>'writer_id', - (root->>'writer_epoch')::bigint - FROM payload, input, batch_ok - WHERE NOT EXISTS ( - SELECT 1 FROM qbit_share_ledger existing - WHERE existing.share_id = payload.data->>'share_id' - ) - ORDER BY payload.ordinality - ON CONFLICT (share_id) DO NOTHING - RETURNING qbit_share_ledger.* -), -inserted_candidates AS ( - INSERT INTO qbit_block_candidate_outbox ( - block_hash, share_id, candidate, candidate_sha256, - storage_version, body_id, replay_header, parent_hash, expected_height - ) - SELECT - payload.candidate->>'block_hash_hex', payload.data->>'share_id', - NULL, payload.candidate->>'candidate_sha256', - {CANDIDATE_BODY_STORAGE_VERSION}, payload.candidate->>'body_id', - payload.candidate->'header', - payload.candidate->'header'->>'parent_hash', - NULLIF(payload.candidate->'header'->>'expected_height', '')::bigint - FROM payload, batch_ok - WHERE payload.candidate IS NOT NULL - AND NOT (payload.candidate->'expected'->>'found')::boolean - ON CONFLICT (block_hash) DO UPDATE - SET share_id = EXCLUDED.share_id, - updated_at = clock_timestamp() - WHERE qbit_block_candidate_outbox.share_id IS NULL - AND qbit_block_candidate_outbox.state = 'pending' - AND qbit_block_candidate_outbox.candidate_sha256 = EXCLUDED.candidate_sha256 - RETURNING block_hash -), -linked_candidates AS ( - -- Credit-on-accept: the standalone intent already published this body; - -- the share joins the same identity without re-uploading anything. - UPDATE qbit_block_candidate_outbox - SET share_id = payload.data->>'share_id', - updated_at = clock_timestamp() - FROM payload, batch_ok - WHERE payload.candidate IS NOT NULL - AND (payload.candidate->'expected'->>'found')::boolean - AND qbit_block_candidate_outbox.block_hash = payload.candidate->>'block_hash_hex' - AND qbit_block_candidate_outbox.share_id IS NULL - AND qbit_block_candidate_outbox.state = 'pending' - AND qbit_block_candidate_outbox.candidate_sha256 = payload.candidate->>'candidate_sha256' - RETURNING qbit_block_candidate_outbox.block_hash -), -records AS ( - SELECT - ledger.*, payload.ordinality, false AS newly_inserted, - false AS new_miner, candidate_states.candidate_outbox_state - FROM payload - JOIN qbit_share_ledger ledger ON ledger.share_id = payload.data->>'share_id' - JOIN candidate_states ON candidate_states.ordinality = payload.ordinality - UNION ALL - SELECT - inserted_shares.*, payload.ordinality, true AS newly_inserted, - NOT EXISTS ( - SELECT 1 - FROM qbit_share_ledger existing_miner - WHERE existing_miner.accepted - AND existing_miner.miner_id = inserted_shares.miner_id - ) - AND NOT EXISTS ( - SELECT 1 - FROM inserted_shares earlier_insert - WHERE earlier_insert.miner_id = inserted_shares.miner_id - AND earlier_insert.share_seq < inserted_shares.share_seq - ) AS new_miner, - candidate_states.candidate_outbox_state - FROM inserted_shares - JOIN payload ON payload.data->>'share_id' = inserted_shares.share_id - JOIN candidate_states ON candidate_states.ordinality = payload.ordinality -) -SELECT CASE - WHEN NOT EXISTS (SELECT 1 FROM lease) THEN - json_build_object('error', 'writer lease is not active') - WHEN EXISTS (SELECT 1 FROM share_mismatch) THEN - json_build_object( - 'error', 'duplicate share_id payload mismatch', - 'error_kind', 'share_replay_conflict', - 'share_ids', (SELECT json_agg(share_id ORDER BY share_id) FROM share_mismatch) - ) - WHEN EXISTS (SELECT 1 FROM candidate_mismatch) THEN - json_build_object( - 'error', 'block candidate outbox changed during publication', - 'error_kind', 'candidate_version_changed', - 'block_hashes', (SELECT json_agg(block_hash ORDER BY block_hash) FROM candidate_mismatch) - ) - WHEN EXISTS (SELECT 1 FROM candidate_unsealed) THEN - json_build_object( - 'error', 'block candidate body is not sealed', - 'error_kind', 'candidate_body_unsealed', - 'block_hashes', (SELECT json_agg(block_hash ORDER BY block_hash) FROM candidate_unsealed) - ) - ELSE json_build_object( - 'records', ( - SELECT json_agg(json_build_object( - 'share_seq', records.share_seq, - 'share_id', records.share_id, - 'miner_id', records.miner_id, - 'order_key', records.payout_order_key, - 'p2mr_program_hex', encode(records.p2mr_program, 'hex'), - 'share_difficulty', records.share_difficulty::text, - 'network_difficulty', records.network_difficulty::text, - 'template_height', records.template_height, - 'job_id', records.job_id, - 'job_issued_at_ms', round(extract(epoch FROM records.job_issued_at) * 1000)::bigint, - 'accepted_at_ms', round(extract(epoch FROM records.accepted_at) * 1000)::bigint, - 'ntime', records.ntime, - 'credit_policy', records.credit_policy, - 'newly_inserted', records.newly_inserted, - 'new_miner', records.new_miner, - 'candidate_outbox_state', records.candidate_outbox_state - ) ORDER BY records.ordinality) - FROM records - ) - ) -END; -""" - with self._operation_gate(self._lock, "writer lock"): - result = self._run_json(sql) - if "error" in result: - if result.get("error_kind") == "share_replay_conflict": - raise ShareReplayConflict(str(result["error"])) - if result.get("error_kind") in {"candidate_version_changed", "candidate_body_unsealed"}: - return None - raise RuntimeError(str(result["error"])) - records = result.get("records") - if not isinstance(records, list) or len(records) != expected_count: - raise RuntimeError("Postgres share batch returned an incomplete result") - parsed = [self._record_from_json(record) for record in records] - committed = sorted( - zip(records, parsed, strict=True), - key=lambda item: item[1].share_seq, - ) - for record_payload, record in committed: - if bool(record_payload.get("newly_inserted", True)): - self._note_appended_share( - record, - new_miner=bool(record_payload.get("new_miner", False)), - ) - return [ - ShareReplayResult( - ( - "inserted" - if bool(record_payload.get("newly_inserted", True)) - else "exact_existing" - ), - record, - ) - for record_payload, record in zip(records, parsed, strict=True) - ] - - def append_batch( - self, - entries: list[tuple[PendingShare, Any]], - ) -> list[AcceptedShareRecord]: - return [ - outcome.record - for outcome in self._append_batch_with_replay_outcomes(entries) - ] - - def append_recovered_share(self, pending: PendingShare) -> ShareReplayResult: - """Use the exact batch comparator for one typed recovery outcome.""" - outcomes = self._append_batch_with_replay_outcomes([(pending, None)]) - if len(outcomes) != 1: - raise RuntimeError("Postgres recovery append returned an incomplete result") - return outcomes[0] - - def persist_block_candidate_intent( - self, - candidate: Any, - ) -> BlockCandidateIntentPersistResult: - """Persist candidate work that is not yet eligible for share credit. - - Historically only the identity digest is compared against an - existing row, and that is preserved. The chunked route stages and - seals the body outside the writer gate first; the fenced insert then - publishes a reference only when a sealed manifest with the same - digest and block hash exists, and leaves a colliding row alone. - """ - if not isinstance(candidate, PreparedCandidateIntent): - if not isinstance(candidate, dict): - raise TypeError("block candidate intent must be an object") - if not str(candidate.get("block_hash_hex", "")).lower(): - raise ValueError("block candidate is missing block_hash_hex") - if self._candidate_storage_version_value() != CANDIDATE_BODY_STORAGE_VERSION: - return self._persist_block_candidate_intent_v1( - self._legacy_candidate_document(candidate) - ) - try: - prepared = prepare_candidate_intent(candidate) - except CandidateCodecError as exc: - raise ValueError(str(exc)) from exc - block_hash = prepared.block_hash - candidate_sha256 = prepared.candidate_sha256 - observed = self._observe_candidate_outbox_row(block_hash) - if observed is not None: - if observed["candidate_sha256"] != candidate_sha256: - raise RuntimeError("block candidate payload mismatch") - return BlockCandidateIntentPersistResult( - inserted=False, - state=observed["state"], - ) - body_id = self.stage_candidate_body(prepared) - header = prepared.replay_header() - sql = f""" -WITH durability AS ( - SELECT set_config('synchronous_commit', 'on', true) -), -lease AS ( - UPDATE qbit_ledger_writer_lease - SET lease_expires_at = clock_timestamp() + {self._lease_interval_sql}, - updated_at = clock_timestamp() - FROM durability - WHERE singleton - AND writer_id = {self._text_literal(self._writer_id)} - AND writer_epoch = {int(self._writer_epoch)} - AND writer_session_token = {self._text_literal(self._writer_session_token)} - RETURNING writer_id -), -existing AS ( - SELECT candidate_sha256, state - FROM qbit_block_candidate_outbox - WHERE block_hash = {self._text_literal(block_hash)} -), -sealed_body AS ( - -- Locked FOR SHARE: an orphan retirement's FOR UPDATE serializes - -- against this publication, and the loser re-evaluates and fails closed. - SELECT body_id - FROM qbit_block_candidate_body - WHERE body_id = {self._text_literal(body_id)} - AND state = 'sealed' - AND candidate_sha256 = {self._text_literal(candidate_sha256)} - AND block_hash = {self._text_literal(block_hash)} - FOR SHARE -), -inserted AS ( - INSERT INTO qbit_block_candidate_outbox ( - block_hash, share_id, candidate, candidate_sha256, - storage_version, body_id, replay_header, parent_hash, expected_height - ) - SELECT - {self._text_literal(block_hash)}, NULL, NULL, - {self._text_literal(candidate_sha256)}, - {CANDIDATE_BODY_STORAGE_VERSION}, sealed_body.body_id, - {self._jsonb_literal(header)}, - {self._jsonb_literal(header)}->>'parent_hash', - NULLIF({self._jsonb_literal(header)}->>'expected_height', '')::bigint - FROM lease, sealed_body - WHERE NOT EXISTS (SELECT 1 FROM existing) - ON CONFLICT (block_hash) DO NOTHING - RETURNING block_hash -) -SELECT CASE - WHEN NOT EXISTS (SELECT 1 FROM lease) THEN - json_build_object('error', 'writer lease is not active') - WHEN EXISTS ( - SELECT 1 FROM existing - WHERE candidate_sha256 <> {self._text_literal(candidate_sha256)} - ) THEN - json_build_object('error', 'block candidate payload mismatch') - WHEN NOT EXISTS (SELECT 1 FROM existing) AND NOT EXISTS (SELECT 1 FROM sealed_body) THEN - json_build_object('error', 'block candidate body is not sealed') - ELSE - json_build_object( - 'inserted', (SELECT count(*) FROM inserted), - 'state', COALESCE((SELECT state FROM existing), 'pending') - ) -END; -""" - result = self._run_fenced_json(sql) - if "error" in result: - raise RuntimeError(str(result["error"])) - return BlockCandidateIntentPersistResult( - inserted=int(result.get("inserted", 0)) > 0, - state=str(result.get("state", "pending")), - ) - - def pending_block_candidate_headers( - self, - *, - limit: int = HEADER_PAGE_MAX_ROWS, - after_cursor: object | None = None, - max_bytes: int = HEADER_PAGE_MAX_BYTES, - ) -> CandidateHeaderPage: - """Metadata-first pending page (#255): typed headers, explicit exhaustion. - - Same keyset, order, cursor precision and read-slot gating as - ``pending_block_candidate_rows``; what differs is what one row - carries (a fixed projection of the replay header plus the body - reference, never the candidate) and what the page proves: the - statement fetches one row beyond the row cap, so ``exhausted`` is - answered by the outbox, and a page cut short by the byte cap is - reported as such rather than mistaken for the end. - """ - limit = max(1, min(int(limit), HEADER_PAGE_MAX_ROWS)) - cursor_parts: tuple[str, str] | None = None - if after_cursor is not None: - created_at_text, cursor_block_hash = _block_candidate_cursor_parts(after_cursor) - if not isinstance(created_at_text, str): - raise ValueError("pending block candidate cursor has no creation stamp") - cursor_parts = (created_at_text, cursor_block_hash) - sql = header_page_sql(limit, cursor_parts, max_bytes, text=self._text_literal) - read_gate = getattr(self, "_read_semaphore", None) - if read_gate is None: - # A ledger built without __init__ (focused tests) has no read - # slot; the statement itself is what those tests observe. - result = self._run_retry_safe_read_json(sql) - else: - result = self._run_attributed_read_json( - sql, - operation="pending_block_candidate_headers", - gate=read_gate, - gate_name="read slot", - ) - if not isinstance(result, dict) or not isinstance(result.get("rows"), list): - raise RuntimeError("pending block candidate header page is malformed") - rows: list[dict[str, Any]] = [] - for row in result["rows"]: - if not isinstance(row, dict) or "pool_block_exists" not in row: - raise RuntimeError( - "pending block candidate header row is missing pool block existence" - ) - row["pool_block_exists"] = bool(row["pool_block_exists"]) - row["storage_version"] = int(row.get("storage_version") or 1) - rows.append(row) - fetched = int(result.get("fetched", len(rows))) - returned = int(result.get("returned", len(rows))) - truncated_by_bytes = returned < min(fetched, limit) - return CandidateHeaderPage( - rows=tuple(rows), - next_cursor=rows[-1]["cursor"] if rows else after_cursor, - exhausted=fetched <= limit and not truncated_by_bytes, - fetched=fetched, - truncated_by_bytes=truncated_by_bytes, - ) - - def hydrate_block_candidate_intent( - self, - row: dict[str, Any], - *, - cancelled: Callable[[], bool] | None = None, - ) -> PreparedCandidateIntent: - """Materialize one header row's body to a spool and return the intent. - - Storage version 2 reads bounded chunk pages through the read slot, - verifying manifest state, referencing outbox row and every chunk - digest as it goes. Storage version 1 delegates the unavoidable - whole-document decode to the isolated compatibility helper, which - writes the spool itself. Neither path holds the writer gate. - """ - self._ensure_candidate_store_state() - block_hash = str(row["block_hash"]).lower() - header = row.get("header") if isinstance(row.get("header"), dict) else {} - pending_share = header.get("pending_share") if isinstance(header, dict) else None - accepted_at_present = bool(header.get("accepted_at_present")) if isinstance(header, dict) else False - accepted_at_ms = ( - pending_share.get("accepted_at_ms") - if isinstance(pending_share, dict) - else None - ) - storage_version = int(row.get("storage_version") or 1) - with self._operation_gate(self._candidate_body_gate, "candidate body slot"): - if storage_version == CANDIDATE_BODY_STORAGE_VERSION: - body_row = row.get("body") - if not isinstance(body_row, dict): - raise CandidateBodyUnavailable( - f"block candidate {block_hash} carries no body reference" - ) - ref = CandidateBodyRef.from_row(body_row) - hydrator = CandidateBodyHydrator( - read_manifest=self.read_candidate_body_manifest, - read_spans=self.read_candidate_body_spans, - read_pages=self.read_candidate_body_pages, - read_page=self.read_candidate_body_page, - admission=self._candidate_spool, - cancelled=cancelled, - ) - return hydrator.hydrate( - ref, - block_hash=block_hash, - accepted_at_present=accepted_at_present, - accepted_at_ms=accepted_at_ms, - ) - if storage_version != LEGACY_CANDIDATE_STORAGE_VERSION: - raise CandidateStorageError( - f"unsupported candidate storage version {storage_version}" - ) - helper = self._legacy_candidate_helper - if cancelled is not None: - helper = helper.with_cancellation(cancelled) - admission = self._candidate_spool.snapshot() - available = admission["limit_bytes"] - admission["reserved_bytes"] - if available <= 0: - raise CandidateStorageError("legacy candidate spool reservation exhausted") - release = self._candidate_spool.reserve(available) - paths: tuple[str, ...] = () - body = None - try: - paths = self._candidate_spool.new_spool_paths(block_hash) - path, index_path = paths - converted = helper.convert(block_hash, path, index_path, spool_limit_bytes=available) - actual_bytes = converted.manifest.byte_count + converted.manifest.page_count * 16 - release() - release = self._candidate_spool.reserve(actual_bytes) - index = SpoolFieldIndex.open_written(index_path, converted.spans) - if not converted.identity_matches_row: - print( - "prism ledger: legacy block candidate identity re-encoded with a " - f"different digest hash={block_hash} row={converted.row_candidate_sha256} " - f"body={converted.manifest.candidate_sha256}", - flush=True, - ) - body = SpoolCandidateBody(path, converted.manifest, index, release=release) - return prepared_intent_from_spool( - body, - accepted_at_present=converted.accepted_at_present, - accepted_at_ms=converted.accepted_at_ms, - ) - except BaseException: - if body is not None: - body.close() - else: - for path in paths: - try: - os.unlink(path) - except FileNotFoundError: - pass - release() - raise - - def retire_orphan_candidate_bodies( - self, - *, - stale_staging_seconds: float = STALE_STAGING_SECONDS, - ) -> tuple[str, ...]: - """Tiny fenced compare-and-swap: one orphaned body becomes ``retired``. - - Under the writer fence so a deposed session retires nothing, but it - touches one manifest row and no chunk. ``retired`` is permanent and - publication references only sealed bodies, so nothing revives it. - """ - result = self._run_fenced_json( - retire_orphan_bodies_sql( - { - **self._writer_identity_payload(), - "stale_staging_seconds": float(stale_staging_seconds), - }, - jsonb=self._jsonb_literal, - ) - ) - if not isinstance(result, dict): - raise RuntimeError("candidate body orphan retirement returned no result") - if "error" in result: - raise RuntimeError(str(result["error"])) - return tuple(str(value) for value in result.get("retired", ())) - - def reap_retired_candidate_chunks( - self, - *, - max_chunks: int = JANITOR_CHUNKS_PER_STEP, - ) -> dict[str, Any]: - """One lease-free bounded deletion of a retired body's chunks. - - No lease row is read or written: reclaiming a permanently retired - body's bytes needs no fence, and a deposed session doing it harms - nobody. Takes the body gate so it never competes with a staging or - hydration for the same connection budget. - """ - self._ensure_candidate_store_state() - with self._operation_gate(self._candidate_body_gate, "candidate body slot"): - result = self._run_json( - reap_retired_chunks_sql( - {"max_chunks": max(1, int(max_chunks))}, - jsonb=self._jsonb_literal, - ) - ) - if not isinstance(result, dict): - raise RuntimeError("candidate body janitor returned no result") - remaining = result.get("remaining") if isinstance(result.get("remaining"), dict) else {} - return { - "body_id": result.get("body_id"), - "deleted_chunks": int(result.get("deleted_chunks", 0)), - "deleted_pages": int(result.get("deleted_pages", 0)), - "deleted_spans": int(result.get("deleted_spans", 0)), - "deleted_bodies": int(result.get("deleted_bodies", 0)), - # True while this body still has work for a later step. - "pending": result.get("body_id") is not None - and int(result.get("deleted_bodies", 0)) == 0, - "remaining": { - "chunks": bool(remaining.get("chunks")), - "pages": bool(remaining.get("pages")), - "spans": bool(remaining.get("spans")), - }, - } - - def reap_retired_candidate_bodies( - self, - *, - max_chunks: int = JANITOR_CHUNKS_PER_STEP, - stale_staging_seconds: float = STALE_STAGING_SECONDS, - ) -> dict[str, Any]: - """One janitor step: retire one orphan (fenced), then reap one page. - - Two statements by design (#255 review): the fenced transition is a - single-row compare-and-swap, and the body-sized deletion that - follows never holds the lease row. - """ - retired = self.retire_orphan_candidate_bodies( - stale_staging_seconds=stale_staging_seconds - ) - outcome = self.reap_retired_candidate_chunks(max_chunks=max_chunks) - outcome["retired"] = retired - return outcome - - def candidate_spool_snapshot(self) -> dict[str, int]: - self._ensure_candidate_store_state() - return self._candidate_spool.snapshot() - - def _persist_block_candidate_intent_v1( - self, - candidate: dict[str, Any], - ) -> BlockCandidateIntentPersistResult: - """The historical whole-jsonb standalone persist (storage version 1).""" - block_hash = str(candidate.get("block_hash_hex", "")).lower() - if not block_hash: - raise ValueError("block candidate is missing block_hash_hex") - candidate = {**candidate, "block_hash_hex": block_hash} - candidate_sha256 = block_candidate_identity_sha256(candidate) - sql = f""" -WITH durability AS ( - SELECT set_config('synchronous_commit', 'on', true) -), -lease AS ( - UPDATE qbit_ledger_writer_lease - SET lease_expires_at = clock_timestamp() + {self._lease_interval_sql}, - updated_at = clock_timestamp() - FROM durability - WHERE singleton - AND writer_id = {self._text_literal(self._writer_id)} - AND writer_epoch = {int(self._writer_epoch)} - AND writer_session_token = {self._text_literal(self._writer_session_token)} - RETURNING writer_id -), -existing AS ( - SELECT candidate_sha256, state - FROM qbit_block_candidate_outbox - WHERE block_hash = {self._text_literal(block_hash)} -), -inserted AS ( - INSERT INTO qbit_block_candidate_outbox ( - block_hash, share_id, candidate, candidate_sha256 - ) - SELECT - {self._text_literal(block_hash)}, NULL, - {self._jsonb_literal(candidate)}, {self._text_literal(candidate_sha256)} - FROM lease - WHERE NOT EXISTS (SELECT 1 FROM existing) - ON CONFLICT (block_hash) DO NOTHING - RETURNING block_hash -) -SELECT CASE - WHEN NOT EXISTS (SELECT 1 FROM lease) THEN - json_build_object('error', 'writer lease is not active') - WHEN EXISTS ( - SELECT 1 FROM existing - WHERE candidate_sha256 <> {self._text_literal(candidate_sha256)} - ) THEN - json_build_object('error', 'block candidate payload mismatch') - ELSE - json_build_object( - 'inserted', (SELECT count(*) FROM inserted), - 'state', COALESCE((SELECT state FROM existing), 'pending') - ) -END; -""" - result = self._run_fenced_json(sql) - if "error" in result: - raise RuntimeError(str(result["error"])) - return BlockCandidateIntentPersistResult( - inserted=int(result.get("inserted", 0)) > 0, - state=str(result.get("state", "pending")), - ) - - def pending_block_candidates(self, *, limit: int = 32) -> list[dict[str, Any]]: - return [ - row["candidate"] - for row in self.pending_block_candidate_rows(limit=limit) - ] - - def pending_block_candidate_rows( - self, - *, - limit: int = 32, - after_cursor: object | None = None, - ) -> list[dict[str, Any]]: - """Compatibility view with owned lazy payloads, never aggregate JSON. - - New replay uses header descriptors directly. Callers of this older - interface still receive its four fields; each payload is hydrated - separately through the same bounded reader. Fill byte-truncated header - pages until the requested row limit or explicit exhaustion. - """ - if (getattr(self._run_json, "__func__", None) is not PsqlShareLedger._run_json - or getattr(self._run_sql, "__func__", None) is not PsqlShareLedger._run_sql): - return self._legacy_pending_block_candidate_rows(limit=limit, after_cursor=after_cursor) - limit = max(1, min(int(limit), HEADER_PAGE_MAX_ROWS)) - rows: list[dict[str, Any]] = [] - cursor = after_cursor - while len(rows) < limit: - page = self.pending_block_candidate_headers(limit=limit - len(rows), after_cursor=cursor) - for row in page.rows: - rows.append({ - "block_hash": row["block_hash"], - "candidate": self.hydrate_block_candidate_intent(row), - "pool_block_exists": row["pool_block_exists"], - "cursor": row["cursor"], - }) - if page.exhausted: - break - if not page.rows or page.next_cursor == cursor: - raise CandidateStorageError("candidate header enumeration did not advance") - cursor = page.next_cursor - return rows - - def _legacy_pending_block_candidate_rows( - self, - *, - limit: int = 32, - after_cursor: object | None = None, - ) -> list[dict[str, Any]]: - """Return pending payloads together with their authoritative row keys. - - Rows carry an opaque ``cursor`` the caller passes back verbatim as - ``after_cursor`` to resume strictly after that row, so a backlog - larger than one window enumerates completely in bounded pages - instead of forcing an ever-wider single query. The keyset predicate - and the ordering both stay on ``(created_at, block_hash)``, which is - exactly the partial index - ``qbit_block_candidate_outbox_pending_idx``: every page is one - bounded index range scan regardless of how far in the backlog it - starts. - - The cursor stamp is rendered at microsecond precision with an - explicit UTC marker because ``created_at`` is a ``timestamptz`` whose - stored resolution is microseconds: a second-precision stamp (the - format the public API endpoints use) would truncate, and the - resulting predicate would re-emit or skip whole sub-second groups. - - Each row also carries ``pool_block_exists``: whether a durable - ``qbit_pool_blocks`` row exists for that hash, i.e. whether the - candidate ever reached ``persist_accepted_block``. It is answered - inside this page read -- one bounded existence probe per returned row - -- because the alternative is one round trip per row, and a page is - read precisely when the backlog is large. The fact is advisory by the - time the caller holds it; the terminal batch update re-checks it under - the writer fence. - - **Gate.** This is one read-only statement, and it takes the bounded - read slot rather than the global writer lock (issue #211). Waiting for - writer admission bought this page nothing: the query already crosses - no in-process state, and every fact it returns is advisory the instant - the statement commits -- a candidate can land, or be terminalized by - another path, between the snapshot and anything the caller does with - it. What holding the lock did buy was a convoy. During accepted-block - accounting the enumeration spent most or all of its bounded budget - queued behind an unrelated long write, and on ``union-mainnet`` that - cost the fast-call budget outright while PostgreSQL was idle: the - outer call reported ``exceeded 5s`` with only ~0.4-1.3s of the inner - statement deadline consumed, and accepted candidates converged in - 79-186s with ``qbit_prism_accepted_parent_unresolved_oldest_seconds`` - reaching ~101s. - - Nothing downstream weakens as a result, because nothing downstream - ever trusted this snapshot: - - * ``mark_block_candidates_abandoned`` re-asks ``qbit_pool_blocks`` - *inside* the fenced ``UPDATE``, under the writer-id/epoch/ - session-token lease predicate, and returns the exact hash set it - transitioned -- so a row that acquired a pool block after this read - is silently absent instead of abandoned. - * ``mark_block_candidate_attempted`` and ``_finish_block_candidate`` - carry the same lease fence and additionally require - ``state = 'pending'``, so a row another path terminalized between - the snapshot and the write transitions nobody. - * The node-offer path re-reads ``pool_block_state`` and the chain at - dequeue rather than reusing a page fact - (``_skip_superseded_block_candidate_at_dequeue``). - - Capacity is unchanged too: the read semaphore admits - ``read_concurrency`` callers and the pooled client holds - ``read_concurrency + 1`` connections, so moving this statement from - the writer-lock class to the read-slot class still cannot demand more - connections than the pool has, and it opens no connection and starts - no thread of its own. - """ - if limit <= 0: - return [] - after_predicate = "" - if after_cursor is not None: - created_at_text, cursor_block_hash = _block_candidate_cursor_parts( - after_cursor - ) - if not isinstance(created_at_text, str): - raise ValueError( - "pending block candidate cursor has no creation stamp" - ) - after_predicate = ( - "\n AND (created_at, block_hash) > " - f"({self._text_literal(created_at_text)}::timestamptz, " - f"{self._text_literal(cursor_block_hash)})" - ) - sql = f""" -SELECT COALESCE( - json_agg( - json_build_object( - 'block_hash', pending.block_hash, - 'candidate', pending.candidate, - 'storage_version', pending.storage_version, - 'candidate_sha256', pending.candidate_sha256, - 'pool_block_exists', EXISTS ( - SELECT 1 - FROM qbit_pool_blocks pool - WHERE pool.block_hash = pending.block_hash - ), - 'cursor', json_build_array( - pending.cursor_created_at, - pending.block_hash - ) - ) - ORDER BY pending.created_at, pending.block_hash - ), - '[]'::json -) -FROM ( - SELECT - candidate, - storage_version, - candidate_sha256, - created_at, - to_char( - created_at AT TIME ZONE 'UTC', - 'YYYY-MM-DD"T"HH24:MI:SS.US"Z"' - ) AS cursor_created_at, - block_hash - FROM qbit_block_candidate_outbox - WHERE state = 'pending'{after_predicate} - ORDER BY created_at, block_hash - LIMIT {int(limit)} -) pending; -""" - rows = list( - self._run_attributed_read_json( - sql, - operation="pending_block_candidate_rows", - gate=self._read_semaphore, - gate_name="read slot", - ) - ) - # A row whose existence fact did not arrive is not a row with no pool - # block: reading a missing key as false would hand a caller a false - # negative on the one fact that keeps an offered, landed candidate out - # of a terminal set. Fail the page instead. - for row in rows: - if not isinstance(row, dict) or "pool_block_exists" not in row: - raise RuntimeError( - "pending block candidate row is missing pool block existence" - ) - row["pool_block_exists"] = bool(row["pool_block_exists"]) - return rows - - def block_candidate_pending_metrics(self) -> dict[str, int | float]: - """Return aggregate pending ages using the existing outbox index.""" - sql = """ -SELECT json_build_object( - 'pending_count', count(*), - 'oldest_pending_age_seconds', COALESCE( - EXTRACT(EPOCH FROM clock_timestamp() - min(created_at)), - 0 - ), - 'oldest_unattempted_age_seconds', COALESCE( - EXTRACT( - EPOCH FROM clock_timestamp() - - min(created_at) FILTER (WHERE attempt_count = 0) - ), - 0 - ) -) -FROM qbit_block_candidate_outbox -WHERE state = 'pending'; -""" - with self._operation_gate(self._lock, "writer lock"): - metrics = self._run_retry_safe_read_json(sql) - return { - "pending_count": int(metrics.get("pending_count", 0)), - "oldest_pending_age_seconds": float( - metrics.get("oldest_pending_age_seconds", 0.0) - ), - "oldest_unattempted_age_seconds": float( - metrics.get("oldest_unattempted_age_seconds", 0.0) - ), - } - - def mark_block_candidate_attempted(self, *, block_hash: str) -> bool: - """Fence and count entry into a candidate processing attempt.""" - sql = f""" -WITH lease AS ( - UPDATE qbit_ledger_writer_lease - SET lease_expires_at = clock_timestamp() + {self._lease_interval_sql}, - updated_at = clock_timestamp() - WHERE singleton - AND writer_id = {self._text_literal(self._writer_id)} - AND writer_epoch = {int(self._writer_epoch)} - AND writer_session_token = {self._text_literal(self._writer_session_token)} - RETURNING writer_id -), -updated AS ( - UPDATE qbit_block_candidate_outbox - SET attempt_count = attempt_count + 1, - updated_at = clock_timestamp() - FROM lease - WHERE block_hash = {self._text_literal(block_hash.lower())} - AND state = 'pending' - RETURNING block_hash -) -SELECT CASE - WHEN NOT EXISTS (SELECT 1 FROM lease) THEN - json_build_object('error', 'writer lease is not active') - ELSE - json_build_object('updated', (SELECT count(*) FROM updated)) -END; -""" - result = self._run_fenced_json(sql) - if "error" in result: - raise RuntimeError(str(result["error"])) - return int(result.get("updated", 0)) > 0 - - def mark_block_candidate_submitted(self, *, block_hash: str) -> bool: - return self._finish_block_candidate(block_hash=block_hash, state="submitted", error=None) - - def mark_block_candidate_abandoned(self, *, block_hash: str, error: str) -> bool: - return self._finish_block_candidate(block_hash=block_hash, state="abandoned", error=error) - - def mark_block_candidates_abandoned( - self, - *, - block_hashes: Sequence[str], - error: str, - ) -> tuple[str, ...]: - """Abandon a caller-supplied page of pending rows in one fenced write. - - Same writer-id/epoch/session-token fence as the single-hash terminal - updates, and the same terminal column set, but the row predicate is - set-oriented: one ``block_hash = ANY(...)`` statement for the whole - page rather than one statement per hash. At the storm cardinalities - this exists for, the per-row form is the cost. - - ``RETURNING`` feeds the result, so the value is the exact set of - hashes this fenced statement transitioned -- not a count, and not the - requested set. Rows that were already terminal, that another writer - won, or that do not exist are silently absent from it, which is what - lets the caller confine follow-up cleanup to rows it actually won. - An empty request performs no query at all, so no degenerate - ``ANY(ARRAY[])`` statement is ever generated. - - The predicate re-checks ``qbit_pool_blocks`` rather than trusting the - ``pool_block_exists`` the caller read on a prior page: a candidate can - land between that read and this write, and a pool-block row is the - durable evidence that it did. Re-asking under the writer fence closes - that window inside the statement that does the transition, so a row - that acquired one is silently absent from the returned set instead of - being abandoned after it was won. - """ - targets = _normalized_block_candidate_hash_set(block_hashes) - if not targets: - return () - sql = f""" -WITH lease AS ( - UPDATE qbit_ledger_writer_lease - SET lease_expires_at = clock_timestamp() + {self._lease_interval_sql}, - updated_at = clock_timestamp() - WHERE singleton - AND writer_id = {self._text_literal(self._writer_id)} - AND writer_epoch = {int(self._writer_epoch)} - AND writer_session_token = {self._text_literal(self._writer_session_token)} - RETURNING writer_id -), -updated AS ( - UPDATE qbit_block_candidate_outbox - SET state = 'abandoned', - last_error = {self._text_literal(error)}, - updated_at = clock_timestamp(), - completed_at = clock_timestamp(), - candidate = NULL, - body_id = NULL, - retired_body_id = COALESCE(qbit_block_candidate_outbox.body_id, retired_body_id) - FROM lease - WHERE block_hash = ANY({self._text_array_literal(targets)}) - AND state = 'pending' - AND NOT EXISTS ( - SELECT 1 - FROM qbit_pool_blocks pool - WHERE pool.block_hash = qbit_block_candidate_outbox.block_hash - ) - RETURNING block_hash, retired_body_id -), -retired_bodies AS ( - -- Detach only: the chunks are reclaimed later by the bounded janitor. - UPDATE qbit_block_candidate_body - SET state = 'retired', retired_at = clock_timestamp() - WHERE body_id IN (SELECT retired_body_id FROM updated WHERE retired_body_id IS NOT NULL) - AND state <> 'retired' - RETURNING body_id -) -SELECT CASE - WHEN NOT EXISTS (SELECT 1 FROM lease) THEN - json_build_object('error', 'writer lease is not active') - ELSE - json_build_object( - 'abandoned', - COALESCE( - (SELECT json_agg(block_hash ORDER BY block_hash) FROM updated), - '[]'::json - ) - ) -END; -""" - result = self._run_fenced_json(sql) - if "error" in result: - raise RuntimeError(str(result["error"])) - return tuple(str(value) for value in result.get("abandoned", ())) - - def _finish_block_candidate(self, *, block_hash: str, state: str, error: str | None) -> bool: - if state not in {"submitted", "abandoned"}: - raise ValueError("invalid block candidate terminal state") - sql = f""" -WITH lease AS ( - UPDATE qbit_ledger_writer_lease - SET lease_expires_at = clock_timestamp() + {self._lease_interval_sql}, - updated_at = clock_timestamp() - WHERE singleton - AND writer_id = {self._text_literal(self._writer_id)} - AND writer_epoch = {int(self._writer_epoch)} - AND writer_session_token = {self._text_literal(self._writer_session_token)} - RETURNING writer_id -), -updated AS ( - UPDATE qbit_block_candidate_outbox - SET state = {self._text_literal(state)}, - last_error = {self._text_literal(error) if error is not None else 'NULL'}, - updated_at = clock_timestamp(), - completed_at = clock_timestamp(), - candidate = NULL, - body_id = NULL, - retired_body_id = COALESCE(qbit_block_candidate_outbox.body_id, retired_body_id) - FROM lease - WHERE block_hash = {self._text_literal(block_hash.lower())} - AND state = 'pending' - RETURNING block_hash, retired_body_id -), -retired_bodies AS ( - -- Detach only: the chunks are reclaimed later by the bounded janitor. - UPDATE qbit_block_candidate_body - SET state = 'retired', retired_at = clock_timestamp() - WHERE body_id IN (SELECT retired_body_id FROM updated WHERE retired_body_id IS NOT NULL) - AND state <> 'retired' - RETURNING body_id -) -SELECT CASE - WHEN NOT EXISTS (SELECT 1 FROM lease) THEN - json_build_object('error', 'writer lease is not active') - ELSE - json_build_object('updated', (SELECT count(*) FROM updated)) -END; -""" - result = self._run_fenced_json(sql) - if "error" in result: - raise RuntimeError(str(result["error"])) - return int(result.get("updated", 0)) > 0 - - def snapshot_at_job_issue( - self, - anchor_job_issued_at_ms: int, - *, - window_weight: int | None = None, - ) -> list[AcceptedShareRecord]: - anchor = ( - f"to_timestamp(({int(anchor_job_issued_at_ms)}::double precision / 1000.0))" - ) - if window_weight is None: - # Whole accepted history up to the anchor. Kept for callers that - # want the full ledger (tools/tests); the coordinator passes a - # window_weight so the hot job-build path stays bounded. - rows_cte = f""" -WITH rows AS ( - SELECT * - FROM qbit_share_ledger - WHERE accepted - AND job_issued_at <= {anchor} - AND accepted_at <= {anchor} -)""" - else: - # Find the bounded reward window in indexed pages, then apply one - # exact cumulative cutoff over only those pages. The previous - # recursive query fetched one row per recursive step; at production - # window sizes that meant 100k+ lateral index probes. Paging keeps - # the scan O(window), not O(history), while returning the identical - # final crossing row and therefore byte-identical audit input. - # - # The final pass must consume the cutoff as a scalar subquery, not - # a joined relation: a join makes the planner apply the cutoff as - # a filter after walking the whole pkey backwards, while a scalar - # subquery becomes an InitPlan the index scan can use as its start - # bound, keeping this pass O(window) too. - # - # The page-granular stop relies on share_difficulty being NOT NULL - # and strictly positive (schema CHECK), which keeps the cumulative - # weight strictly increasing so the crossing row always lies in - # the last fetched page. The same holds for qbit_prism_window() - # in the schema file; relax the constraint and both walks need - # revisiting. - rows_cte = f""" -WITH RECURSIVE pages AS ( - SELECT page.min_share_seq, - page.page_weight, - page.page_weight AS cumulative_weight - FROM LATERAL ( - SELECT min(page_rows.share_seq) AS min_share_seq, - COALESCE(sum(page_rows.share_difficulty), 0)::numeric AS page_weight - FROM ( - SELECT ledger.share_seq, ledger.share_difficulty - FROM qbit_share_ledger ledger - WHERE ledger.accepted - AND ledger.job_issued_at <= {anchor} - AND ledger.accepted_at <= {anchor} - ORDER BY ledger.share_seq DESC - LIMIT 4096 - ) page_rows - ) page - UNION ALL - SELECT page.min_share_seq, - page.page_weight, - pages.cumulative_weight + page.page_weight - FROM pages - CROSS JOIN LATERAL ( - SELECT min(page_rows.share_seq) AS min_share_seq, - COALESCE(sum(page_rows.share_difficulty), 0)::numeric AS page_weight - FROM ( - SELECT ledger.share_seq, ledger.share_difficulty - FROM qbit_share_ledger ledger - WHERE ledger.accepted - AND ledger.job_issued_at <= {anchor} - AND ledger.accepted_at <= {anchor} - AND ledger.share_seq < pages.min_share_seq - ORDER BY ledger.share_seq DESC - LIMIT 4096 - ) page_rows - ) page - WHERE pages.cumulative_weight < {int(window_weight)}::numeric - AND pages.min_share_seq IS NOT NULL -), -page_cutoff AS ( - SELECT min(min_share_seq) AS min_share_seq - FROM pages - WHERE min_share_seq IS NOT NULL -), -ranked AS ( - SELECT ledger.*, - sum(ledger.share_difficulty) OVER ( - ORDER BY ledger.share_seq DESC - ROWS BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW - )::numeric AS cumulative_difficulty - FROM qbit_share_ledger ledger - WHERE ledger.accepted - AND ledger.job_issued_at <= {anchor} - AND ledger.accepted_at <= {anchor} - AND ledger.share_seq >= (SELECT min_share_seq FROM page_cutoff) -), -rows AS ( - SELECT * - FROM ranked - WHERE cumulative_difficulty - share_difficulty < {int(window_weight)}::numeric -)""" - # One JSON object per row, in the same final order the json_agg - # projection imposed (#236). The selection above is untouched: only - # the outer projection changed, so the window's membership, the - # weighted crossing row and the ascending share_seq order are the - # same rows, decoded in bounded batches instead of one call. - sql = rows_cte + f""" -SELECT {_ACCEPTED_SHARE_JSON_ROW_SQL} -FROM rows -ORDER BY share_seq ASC; -""" - # Job construction is a retry-safe MVCC read. Use the independent read - # pool so an accepted block's fenced bulk write cannot stall replacement - # work behind the single-writer connection lock. - # - # Attributed (#224) as ``payout_window_snapshot``: this fold is the - # payout-window read an accepted preview waits behind, and the alert - # could not say whether a slow one was queued for a read slot or slow - # inside PostgreSQL. The slot taken is the same slot ``_run_read_json`` - # takes, in the same order, under the same deadline; only the - # bookkeeping is new. - return self._run_attributed_read_json_rows( - sql, - operation="payout_window_snapshot", - gate=self._read_semaphore, - gate_name="read slot", - row_converter=self._record_from_json, - ) - - def snapshot_between_job_issues( - self, - previous_anchor_job_issued_at_ms: int, - anchor_job_issued_at_ms: int, - ) -> list[AcceptedShareRecord]: - """Return only shares that became anchor-eligible since a snapshot. - - The disjoint UNION branches let Postgres use the accepted-at and - job-issued-at indexes independently. A share belongs to exactly one - branch according to which timestamp crossed the previous anchor; - rows already eligible at that anchor cannot reappear. - """ - - previous_anchor_ms = int(previous_anchor_job_issued_at_ms) - anchor_ms = int(anchor_job_issued_at_ms) - if anchor_ms < previous_anchor_ms: - raise ValueError("snapshot anchor moved backwards") - if anchor_ms == previous_anchor_ms: - return [] - previous_anchor = ( - f"to_timestamp(({previous_anchor_ms}::double precision / 1000.0))" - ) - anchor = f"to_timestamp(({anchor_ms}::double precision / 1000.0))" - sql = f""" -WITH rows AS ( - SELECT ledger.* - FROM qbit_share_ledger ledger - WHERE ledger.accepted - AND ledger.accepted_at > {previous_anchor} - AND ledger.accepted_at <= {anchor} - AND ledger.job_issued_at <= {anchor} - UNION ALL - SELECT ledger.* - FROM qbit_share_ledger ledger - WHERE ledger.accepted - AND ledger.job_issued_at > {previous_anchor} - AND ledger.job_issued_at <= {anchor} - AND ledger.accepted_at <= {previous_anchor} -) -SELECT {_ACCEPTED_SHARE_JSON_ROW_SQL} -FROM rows -ORDER BY share_seq ASC; -""" - # Attributed (#224) as ``payout_window_delta``, apart from the whole - # snapshot above: an incremental advance and a full fold are different - # amounts of work, and folding them into one series would hide which - # of the two a landing actually paid for. Same read slot as before, - # and the same per-row projection as the snapshot (#236). - return self._run_attributed_read_json_rows( - sql, - operation="payout_window_delta", - gate=self._read_semaphore, - gate_name="read slot", - row_converter=self._record_from_json, - ) - - def all_shares(self) -> list[AcceptedShareRecord]: - sql = """ -SELECT COALESCE(json_agg(json_build_object( - 'share_seq', share_seq, - 'share_id', share_id, - 'miner_id', miner_id, - 'order_key', payout_order_key, - 'p2mr_program_hex', encode(p2mr_program, 'hex'), - 'share_difficulty', share_difficulty::text, - 'network_difficulty', network_difficulty::text, - 'template_height', template_height, - 'job_id', job_id, - 'job_issued_at_ms', round(extract(epoch FROM job_issued_at) * 1000)::bigint, - 'accepted_at_ms', round(extract(epoch FROM accepted_at) * 1000)::bigint, - 'ntime', ntime, - 'credit_policy', credit_policy -) ORDER BY share_seq ASC), '[]'::json) -FROM qbit_share_ledger -WHERE accepted; -""" - with self._operation_gate(self._lock, "writer lock"): - return [ - self._record_from_json(item) - for item in self._run_retry_safe_read_json(sql) - ] - - def accepted_share_stats(self) -> dict[str, int]: - """Aggregate counts without materializing the full share history. - - Health checks and readiness gates only need these two numbers, but - they ask for them every few seconds (bundle readiness re-checks, the - health refresher, metrics scrapes). Running the aggregate per call - parallel-seq-scans the whole ledger each time, so the counts are kept - incrementally instead: this process is the single lease-holding - writer, every accepted share passes through ``append``/``append_batch``, - and the counters are reconciled against the database once per - ``accepted_stats_cache_seconds`` in case anything mutated rows out of - band (e.g. reorg reversals flipping ``accepted``). - - Reconciliation never blocks a reader that already has counters: an - expired read returns the maintained counters immediately and arms a - single background refresh. The full-history aggregate takes minutes - on a grown ledger, and parking the first expired caller handed that - wait to whichever subsystem lost the race -- metrics scrapes, - readiness gates, and block-candidate evidence all read these - counters from latency-sensitive paths. Only the cold seed (no - counters yet) still waits for the exact aggregate. - """ - ttl = getattr(self, "_accepted_stats_cache_seconds", 0.0) - stats_lock = getattr(self, "_stats_lock", None) - if stats_lock is not None and ttl > 0: - now = time.monotonic() - with stats_lock: - if ( - self._stats_counts is not None - and self._stats_refreshed_monotonic is not None - ): - if now - self._stats_refreshed_monotonic <= ttl: - return dict(self._stats_counts) - counters = dict(self._stats_counts) - self._start_background_stats_reconcile_locked() - return counters - return self._refresh_accepted_share_stats() - - def _refresh_accepted_share_stats(self) -> dict[str, int]: - # Single-flight: concurrent TTL expiries (health refresher plus a - # metrics scrape) must not both run the aggregate and then race their - # cache writes, or an older snapshot could overwrite a newer one along - # with any increments noted in between. The second caller waits, sees - # the fresh cache, and returns it without another query. - refresh_lock = getattr(self, "_stats_refresh_lock", None) - stats_lock = getattr(self, "_stats_lock", None) - if refresh_lock is None or stats_lock is None: - return self._query_accepted_share_stats()[0] - with refresh_lock: - ttl = getattr(self, "_accepted_stats_cache_seconds", 0.0) - if ttl > 0: - now = time.monotonic() - with stats_lock: - if ( - self._stats_counts is not None - and self._stats_refreshed_monotonic is not None - and now - self._stats_refreshed_monotonic <= ttl - ): - return dict(self._stats_counts) - # Open the note buffer before taking the database snapshot. The - # aggregate runs on the concurrent read pool, never under the - # writer lock, and returns the highest share sequence visible in - # the same MVCC snapshot as its scalar counts. - note_buffer: dict[int, bool] = {} - with stats_lock: - self._stats_note_buffer = note_buffer - try: - counts, snapshot_max_share_seq = self._query_accepted_share_stats() - except BaseException: - with stats_lock: - if self._stats_note_buffer is note_buffer: - self._stats_note_buffer = None - raise - with stats_lock: - for share_seq, new_miner in note_buffer.items(): - if share_seq <= snapshot_max_share_seq: - continue - counts["accepted_share_count"] += 1 - counts["distinct_miner_count"] += int(new_miner) - self._stats_counts = dict(counts) - self._stats_max_share_seq = max( - [snapshot_max_share_seq, *note_buffer.keys()] - ) - self._stats_note_buffer = None - self._stats_refreshed_monotonic = time.monotonic() - return dict(self._stats_counts) - - def _start_background_stats_reconcile_locked(self) -> None: - """Arm the reconcile aggregate unless one is already running. - - Callers hold ``_stats_lock``. The thread slot, not - ``_stats_refresh_lock``, is the single-flight guard here: waiting on - the refresh lock would queue the caller behind the running - aggregate, which is exactly the wait the stale-serving read removes. - """ - thread = getattr(self, "_stats_background_refresh_thread", None) - if thread is not None and thread.is_alive(): - return - thread = Thread( - target=self._run_background_stats_reconcile, - name="prism-share-ledger-stats-reconcile", - daemon=True, - ) - self._stats_background_refresh_thread = thread - thread.start() - - def _run_background_stats_reconcile(self) -> None: - try: - self._refresh_accepted_share_stats() - except BaseException: - # Readers keep the last reconciled counters, which every append - # still advances; the next expired read arms another attempt. - # Failures no longer surface as caller errors, so they are - # counted for accepted_stats_reconcile_status instead. - stats_lock = getattr(self, "_stats_lock", None) - if stats_lock is not None: - with stats_lock: - self._stats_reconcile_failures = ( - getattr(self, "_stats_reconcile_failures", 0) + 1 - ) - traceback.print_exc() - - def accepted_stats_reconcile_status(self) -> dict[str, float | int | None]: - """Expose reconcile liveness for metrics and alerting. - - Serving maintained counters means a failing aggregate is no longer - visible as caller errors, and a wedged reconcile thread occupies the - single-flight slot until the process restarts. Both conditions - surface here: ``failures`` counts failed background passes, and - ``age_seconds`` grows while no reconcile publishes -- including - while one is wedged -- so alerting can page on either. - """ - stats_lock = getattr(self, "_stats_lock", None) - if stats_lock is None: - return {"age_seconds": None, "failures": 0} - with stats_lock: - refreshed = self._stats_refreshed_monotonic - failures = int(getattr(self, "_stats_reconcile_failures", 0)) - age = None if refreshed is None else max(0.0, time.monotonic() - refreshed) - return {"age_seconds": age, "failures": failures} - - def _query_accepted_share_stats(self) -> tuple[dict[str, int], int]: - sql = """ -SELECT json_build_object( - 'accepted_share_count', count(*), - 'distinct_miner_count', count(DISTINCT miner_id), - 'max_share_seq', COALESCE(max(share_seq), 0) -) -FROM qbit_share_ledger -WHERE accepted; -""" - payload = self._run_read_json(sql) - counts = { - "accepted_share_count": int(payload["accepted_share_count"]), - "distinct_miner_count": int(payload["distinct_miner_count"]), - } - return counts, int(payload["max_share_seq"]) - - def _note_appended_share( - self, - record: AcceptedShareRecord, - *, - new_miner: bool, - ) -> None: - """Advance the cached stats for a share this writer just committed. - - A refresh buffers notes while its aggregate runs, then replays only - records newer than the aggregate's share-sequence watermark. The - published watermark also suppresses a delayed note for a commit that - was already visible in the snapshot. Idempotent ``append_batch`` - replays are not noted because the batch result identifies rows it - actually inserted. - """ - stats_lock = getattr(self, "_stats_lock", None) - if stats_lock is None: - return - with stats_lock: - note_buffer = getattr(self, "_stats_note_buffer", None) - if note_buffer is not None: - note_buffer[record.share_seq] = new_miner - if self._stats_counts is None: - return - if record.share_seq <= self._stats_max_share_seq: - return - self._stats_counts["accepted_share_count"] += 1 - self._stats_counts["distinct_miner_count"] += int(new_miner) - self._stats_max_share_seq = record.share_seq - - def current_owed_balances(self) -> list[dict[str, object]]: - sql = """ -SELECT COALESCE(json_agg(json_build_object( - 'recipient_id', miner_id, - 'order_key', payout_order_key, - 'p2mr_program_hex', encode(p2mr_program, 'hex'), - 'balance_sats', owed_balance_sats::text -) ORDER BY payout_order_key, miner_id, encode(p2mr_program, 'hex')), '[]'::json) -FROM qbit_current_owed_balances() -WHERE owed_balance_sats > 0; -""" - with self._operation_gate(self._lock, "writer lock"): - balances = self._run_retry_safe_read_json(sql) - for balance in balances: - balance["balance_sats"] = int(balance["balance_sats"]) - return balances - - def dashboard_readiness_probe(self) -> bool: - """Cheapest possible proof that Postgres is reachable through a read slot. - - The extracted public read tier's /healthz needs to answer "can I still - reach the database" without running any aggregate: a health check that - scans the share ledger turns a liveness probe into load, and the - compose healthcheck runs it on an interval forever. This is a constant - select -- it touches no PRISM table -- taken through the same bounded - read slot as every other public read, so a saturated read pool shows up - as an unhealthy service rather than as a probe that jumps the queue. - """ - payload = self._run_read_json("SELECT json_build_object('ok', true);") - return bool(isinstance(payload, dict) and payload.get("ok")) - - def dashboard_miner_owed_balance_bits(self, *, recipient_id: str) -> int: - """One recipient's owed balance, read without the writer lock. - - The same total as summing current_owed_balances() for this recipient, - but filtered in SQL and taken through the bounded read slot instead of - the writer gate. current_owed_balances() returns every recipient's - balance under the writer lock, so serving the most-polled public route - (/public/v1/miners/{recipient_id}) from it made a dashboard poll - serialize against the lease-holding writer. The predicates match that - method exactly -- rows from qbit_current_owed_balances() with - owed_balance_sats > 0, restricted to this miner_id -- so the integer - returned here is the integer the Python sum produced. - """ - if not recipient_id: - raise ValueError("recipient_id is required") - sql = f""" -SELECT json_build_object( - 'owed_balance_bits', - COALESCE(( - SELECT sum(owed_balance_sats) - FROM qbit_current_owed_balances() - WHERE owed_balance_sats > 0 - AND miner_id = {self._text_literal(recipient_id)} - ), 0)::text -); -""" - payload = self._run_read_json(sql) - return int(payload["owed_balance_bits"]) - - def prior_balances_after_pool_block( - self, - *, - block_hash: str, - ) -> list[dict[str, object]]: - """Return active carry balances as of one confirmed pool block.""" - block_hash = canonical_hex(block_hash, name="block_hash", expected_bytes=32) - sql = f""" -WITH target AS ( - SELECT block_height - FROM qbit_pool_blocks - WHERE block_hash = {self._text_literal(block_hash)} - AND chain_state = 'confirmed' - AND maturity_state <> 'reversed' -), -balances AS ( - SELECT - (array_agg(carry.miner_id ORDER BY carry.payout_order_key, carry.miner_id))[1] AS miner_id, - (array_agg(carry.payout_order_key ORDER BY carry.payout_order_key, carry.miner_id))[1] AS payout_order_key, - carry.p2mr_program, - SUM(carry.gross_amount_sats::numeric - carry.onchain_amount_sats::numeric) AS balance_sats - FROM qbit_payout_carry_forward carry - JOIN qbit_pool_blocks block - ON block.block_hash = carry.block_hash - CROSS JOIN target - WHERE carry.maturity_state <> 'reversed' - AND block.chain_state = 'confirmed' - AND block.maturity_state <> 'reversed' - AND block.block_height <= target.block_height - GROUP BY carry.p2mr_program - HAVING SUM(carry.gross_amount_sats::numeric - carry.onchain_amount_sats::numeric) <> 0 -) -SELECT COALESCE(json_agg(json_build_object( - 'recipient_id', miner_id, - 'order_key', payout_order_key, - 'p2mr_program_hex', encode(p2mr_program, 'hex'), - 'balance_sats', balance_sats::text -) ORDER BY payout_order_key, miner_id, encode(p2mr_program, 'hex')), '[]'::json) -FROM balances; -""" - # Attributed (#224). Still the read slot: this is a retry-safe MVCC - # read of already-confirmed rows, and the audit path that asks for it - # never needed writer serialization. - balances = list( - self._run_attributed_read_json( - sql, - operation="prior_balances_after_pool_block", - gate=self._read_semaphore, - gate_name="read slot", - ) - ) - for balance in balances: - balance["balance_sats"] = int(balance["balance_sats"]) - return balances - - def current_prior_balances(self) -> list[dict[str, object]]: - sql = """ -SELECT COALESCE(json_agg(json_build_object( - 'recipient_id', miner_id, - 'order_key', payout_order_key, - 'p2mr_program_hex', encode(p2mr_program, 'hex'), - 'balance_sats', balance_sats::text -) ORDER BY payout_order_key, miner_id, encode(p2mr_program, 'hex')), '[]'::json) -FROM qbit_current_carry_forward_balances(); -""" - # Attributed (#224) without changing the primitive. This reread stays - # on the *writer lock*, which is where the landing's prior-balances - # check has always taken it: it must observe the fenced state a payout - # mutation leaves behind, so it serializes against that mutation - # rather than running beside it on a read slot. The attribution is the - # only thing added, and it is exactly the split the alert lacked -- - # whether a slow reread was queued behind a landing or slow inside - # PostgreSQL. - # - # The legacy aggregate below is left as it was: one duration covering - # admission and execution together, recorded only on success. - # ``qbit_prism_prior_balances_*`` is an existing operator-facing - # series and a reader comparing it across this change must not find - # its meaning quietly redefined, so it duplicates part of the new - # per-operation sample on purpose. - started = time.monotonic() - balances = self._run_attributed_read_json( - sql, - operation="current_prior_balances", - gate=self._lock, - gate_name="writer lock", - ) - self._note_prior_balances_read(max(0.0, time.monotonic() - started)) - for balance in balances: - balance["balance_sats"] = int(balance["balance_sats"]) - return balances - - def _note_prior_balances_read(self, duration_seconds: float) -> None: - with self._prior_balances_read_stats_lock: - self._prior_balances_reads_total += 1 - self._prior_balances_read_last_seconds = duration_seconds - self._prior_balances_read_max_seconds = max( - self._prior_balances_read_max_seconds, duration_seconds - ) - - def prior_balances_read_stats(self) -> dict[str, float | int]: - """Latency of the prior-balances read, the query whose silent growth - past the submitter deadline caused the #188 landing livelock.""" - with self._prior_balances_read_stats_lock: - return { - "reads_total": self._prior_balances_reads_total, - "last_seconds": self._prior_balances_read_last_seconds, - "max_seconds": self._prior_balances_read_max_seconds, - } - - def carry_forward_integrity_report(self) -> dict[str, object]: - sql = "SELECT qbit_carry_forward_integrity_report();" - with self._operation_gate(self._lock, "writer lock"): - report = self._run_retry_safe_read_json(sql) - # The audit head is a second statement under the same held lock: - # the two reads must observe one another's rows, so the gate - # cannot be released between them. Report the first statement's - # completed round trip so the pair costs a monitor one budget of - # silence at a time (see _note_operation_progress). - self._note_operation_progress() - audit_head = self._carry_forward_audit_head_locked() - report["backend"] = "postgres-psql" - report.update(audit_head) - report["checked_active_rows"] = int(report["checked_active_rows"]) - report["mismatch_count"] = int(report["mismatch_count"]) - report["current_drift_count"] = int(report.get("current_drift_count", 0)) - for row in report.get("mismatches", []): - for key in ( - "prior_balance_sats", - "expected_prior_balance_sats", - "candidate_balance_sats", - "expected_candidate_balance_sats", - "carry_forward_balance_sats", - "expected_carry_forward_balance_sats", - ): - row[key] = int(row[key]) - return report - - def _carry_forward_audit_head_locked(self) -> dict[str, object]: - sql = """ -SELECT COALESCE(json_agg(json_build_object( - 'carry_forward_seq', carry_forward_seq, - 'block_hash', block_hash, - 'block_height', block_height, - 'recipient_id', miner_id, - 'order_key', payout_order_key, - 'p2mr_program_hex', encode(p2mr_program, 'hex'), - 'gross_amount_sats', gross_amount_sats, - 'prior_balance_sats', prior_balance_sats::text, - 'candidate_balance_sats', candidate_balance_sats::text, - 'onchain_amount_sats', onchain_amount_sats, - 'settlement_fee_sats', settlement_fee_sats, - 'carry_forward_balance_sats', carry_forward_balance_sats::text, - 'action', action, - 'maturity_state', maturity_state -) ORDER BY block_height ASC, carry_forward_seq ASC), '[]'::json) -FROM ( - SELECT ledger.* - FROM qbit_payout_carry_forward ledger - JOIN qbit_pool_blocks block - ON block.block_hash = ledger.block_hash - WHERE ledger.maturity_state <> 'reversed' - AND block.chain_state = 'confirmed' - AND block.maturity_state <> 'reversed' -) active; -""" - rows = self._run_retry_safe_read_json(sql) - previous = bytes.fromhex("00" * 32) - version = "qbit.prism.carry-forward-active-delta-chain.v1" - for row in rows: - row_json = json.dumps(row, sort_keys=True, separators=(",", ":")).encode("utf-8") - previous = hashlib.sha256(previous + row_json).digest() - return { - "audit_chain_version": version, - "audit_row_count": len(rows), - "audit_head_sha256": previous.hex() if rows else "00" * 32, - } - - def candidate_window_covers( - self, - shares: Iterable[Any], - *, - anchor_job_issued_at_ms: int, - network_difficulty: int, - ) -> bool: - """Check replay omissions without returning or retaining a full window. - - This coordinator-only read uses connection-private temporary storage; - it is deliberately unavailable to the enforced read-only API ledger. - """ - from lab.prism.candidate_window import postgres_window_covers - - if self._read_only: - raise ReadOnlyLedgerError("candidate window checks require temporary storage") - return postgres_window_covers( - self, shares, - anchor_job_issued_at_ms=anchor_job_issued_at_ms, - network_difficulty=network_difficulty, - ) - - def audit_share_window( - self, - *, - anchor_job_issued_at_ms: int, - network_difficulty: int, - ) -> list[dict[str, object]]: - sql = f""" -SELECT COALESCE(json_agg(json_build_object( - 'window_multiplier', window_multiplier::text, - 'requested_window_weight', requested_window_weight::text, - 'share_seq', share_seq, - 'share_id', share_id, - 'miner_id', miner_id, - 'order_key', payout_order_key, - 'p2mr_program_hex', encode(p2mr_program, 'hex'), - 'share_difficulty', share_difficulty::text, - 'counted_difficulty', counted_difficulty::text, - 'job_issued_at_ms', round(extract(epoch FROM job_issued_at) * 1000)::bigint, - 'accepted_at_ms', round(extract(epoch FROM accepted_at) * 1000)::bigint, - 'credit_policy', credit_policy -) ORDER BY share_seq DESC), '[]'::json) -FROM qbit_audit_share_window( - to_timestamp(({int(anchor_job_issued_at_ms)}::double precision / 1000.0)), - {int(network_difficulty)}::numeric -); -""" - with self._operation_gate(self._lock, "writer lock"): - rows = self._run_retry_safe_read_json(sql) - for row in rows: - for key in ( - "window_multiplier", - "requested_window_weight", - "share_difficulty", - "counted_difficulty", - ): - row[key] = int(row[key]) - return rows - - def audit_block_payouts(self, *, block_hash: str) -> list[dict[str, object]]: - sql = f""" -SELECT COALESCE(json_agg(json_build_object( - 'block_hash', block_hash, - 'block_height', block_height, - 'coinbase_txid', coinbase_txid, - 'payout_manifest_sha256', payout_manifest_sha256, - 'chain_state', chain_state, - 'miner_id', miner_id, - 'order_key', payout_order_key, - 'p2mr_program_hex', encode(p2mr_program, 'hex'), - 'onchain_amount_sats', onchain_amount_sats, - 'carry_forward_balance_sats', carry_forward_balance_sats::text, - 'action', action, - 'maturity_state', maturity_state -) ORDER BY payout_order_key, miner_id, encode(p2mr_program, 'hex')), '[]'::json) -FROM qbit_audit_block_payouts({self._text_literal(block_hash)}); -""" - with self._operation_gate(self._lock, "writer lock"): - rows = self._run_retry_safe_read_json(sql) - for row in rows: - row["carry_forward_balance_sats"] = int(row["carry_forward_balance_sats"]) - return rows - - def recipient_payout_history(self, *, recipient_id: str, limit: int = 50) -> list[dict[str, object]]: - if not recipient_id: - raise ValueError("recipient_id is required") - limit = max(1, min(int(limit), 250)) - sql = f""" -SELECT COALESCE(json_agg(json_build_object( - 'block_hash', block.block_hash, - 'block_height', block.block_height, - 'coinbase_txid', block.coinbase_txid, - 'payout_manifest_sha256', block.payout_manifest_sha256, - 'recipient_id', payout.miner_id, - 'order_key', payout.payout_order_key, - 'p2mr_program_hex', encode(payout.p2mr_program, 'hex'), - 'onchain_amount_sats', payout.onchain_amount_sats, - 'carry_forward_balance_sats', payout.carry_forward_balance_sats::text, - 'action', payout.action, - 'maturity_state', payout.maturity_state, - 'created_at', payout.created_at::text -) ORDER BY payout.block_height DESC, payout.payout_entry_seq DESC), '[]'::json) -FROM ( - SELECT * - FROM qbit_pool_payout_entries - WHERE miner_id = {self._text_literal(recipient_id)} - ORDER BY block_height DESC, payout_entry_seq DESC - LIMIT {limit} -) payout -JOIN qbit_pool_blocks block - ON block.block_hash = payout.block_hash; -""" - with self._operation_gate(self._lock, "writer lock"): - rows = self._run_retry_safe_read_json(sql) - for row in rows: - row["carry_forward_balance_sats"] = int(row["carry_forward_balance_sats"]) - return rows - - def dashboard_miner_lifetime_earnings_bits(self, *, recipient_id: str) -> int: - if not recipient_id: - raise ValueError("recipient_id is required") - sql = f""" -SELECT json_build_object( - 'lifetime_earnings_bits', - COALESCE(( - SELECT sum(carry.gross_amount_sats) - FROM qbit_payout_carry_forward carry - JOIN qbit_pool_blocks block - ON block.block_hash = carry.block_hash - WHERE carry.miner_id = {self._text_literal(recipient_id)} - AND carry.maturity_state <> 'reversed' - AND block.chain_state <> 'reversed' - AND block.maturity_state <> 'reversed' - ), 0) -); -""" - payload = self._run_read_json(sql) - return int(payload["lifetime_earnings_bits"]) - - def dashboard_miner_pending_maturity_bits(self, *, recipient_id: str) -> int: - if not recipient_id: - raise ValueError("recipient_id is required") - sql = f""" -SELECT json_build_object( - 'pending_maturity_bits', - COALESCE(sum(GREATEST(carry.onchain_amount_sats - carry.settlement_fee_sats, 0)), 0) -) -FROM qbit_payout_carry_forward carry -JOIN qbit_pool_blocks block - ON block.block_hash = carry.block_hash -WHERE carry.miner_id = {self._text_literal(recipient_id)} - AND carry.action = 'onchain' - AND carry.maturity_state = 'immature' - AND block.chain_state = 'confirmed' - AND block.maturity_state = 'immature'; -""" - payload = self._run_read_json(sql) - return int(payload["pending_maturity_bits"]) - - def dashboard_miner_share_summary(self, *, recipient_id: str) -> dict[str, object]: - from lab.prism import public_api - - if not recipient_id: - raise ValueError("recipient_id is required") - sql = f""" -WITH bounds AS ( - SELECT clock_timestamp() AS now_at -), -pool AS ( - SELECT COALESCE(sum(share_difficulty), 0)::text AS h3_difficulty - FROM qbit_share_ledger, bounds - WHERE accepted - AND accepted_at >= bounds.now_at - interval '3 hours' - AND accepted_at <= bounds.now_at -), -miner_rollups AS ( - SELECT - count(*) FILTER (WHERE accepted_at >= bounds.now_at - interval '3 hours') AS accepted_3h, - COALESCE(sum(share_difficulty) FILTER (WHERE accepted_at >= bounds.now_at - interval '1 minute'), 0)::text AS m1_difficulty, - COALESCE(sum(share_difficulty) FILTER (WHERE accepted_at >= bounds.now_at - interval '5 minutes'), 0)::text AS m5_difficulty, - COALESCE(sum(share_difficulty) FILTER (WHERE accepted_at >= bounds.now_at - interval '10 minutes'), 0)::text AS m10_difficulty, - COALESCE(sum(share_difficulty) FILTER (WHERE accepted_at >= bounds.now_at - interval '3 hours'), 0)::text AS h3_difficulty, - COALESCE(sum(share_difficulty), 0)::text AS h24_difficulty - FROM qbit_share_ledger, bounds - WHERE accepted - AND miner_id = {self._text_literal(recipient_id)} - AND accepted_at >= bounds.now_at - interval '24 hours' - AND accepted_at <= bounds.now_at -), -miner_last AS ( - SELECT - to_char(max(accepted_at) AT TIME ZONE 'UTC', 'YYYY-MM-DD"T"HH24:MI:SS"Z"') AS last_share_at - FROM qbit_share_ledger, bounds - WHERE accepted - AND miner_id = {self._text_literal(recipient_id)} - AND accepted_at <= bounds.now_at -) -SELECT json_build_object( - 'accepted_3h', (SELECT accepted_3h FROM miner_rollups), - 'm1_difficulty', (SELECT m1_difficulty FROM miner_rollups), - 'm5_difficulty', (SELECT m5_difficulty FROM miner_rollups), - 'm10_difficulty', (SELECT m10_difficulty FROM miner_rollups), - 'h3_difficulty', (SELECT h3_difficulty FROM miner_rollups), - 'h24_difficulty', (SELECT h24_difficulty FROM miner_rollups), - 'pool_h3_difficulty', (SELECT h3_difficulty FROM pool), - 'last_share_at', (SELECT last_share_at FROM miner_last) -); -""" - payload = self._run_read_json(sql) - miner_3h_difficulty = str(payload.get("h3_difficulty") or "0") - pool_3h_difficulty = str(payload.get("pool_h3_difficulty") or "0") - m1_difficulty = str(payload.get("m1_difficulty") or "0") - m5_difficulty = str(payload.get("m5_difficulty") or "0") - m10_difficulty = str(payload.get("m10_difficulty") or "0") - h24_difficulty = str(payload.get("h24_difficulty") or "0") - return { - "hashrate_ths": { - "m1": public_api.hashrate_ths_from_difficulty(m1_difficulty, 60), - "m5": public_api.hashrate_ths_from_difficulty(m5_difficulty, 5 * 60), - "m10": public_api.hashrate_ths_from_difficulty(m10_difficulty, 10 * 60), - "h3": public_api.hashrate_ths_from_difficulty(miner_3h_difficulty, 3 * 60 * 60), - "h24": public_api.hashrate_ths_from_difficulty(h24_difficulty, 24 * 60 * 60), - }, - "accepted_3h": int(payload.get("accepted_3h") or 0), - "accepted_difficulty_3h": miner_3h_difficulty, - "last_share_at": payload.get("last_share_at"), - "share_percent": public_api.percent_string(miner_3h_difficulty, pool_3h_difficulty), - } - - def _pool_reward_window_aggregate(self, window_weight: Decimal) -> dict[str, Any]: - """Pool-wide PRISM reward-window totals grouped by miner, briefly cached. - - Every public miner page needs the same pool-wide recursive - qbit_prism_window scan and only differs in which miner's slice it - reads, so the aggregate is computed once and every miner-page request - within the cache window is served from the shared copy. Concurrent - misses wait for the in-flight computation instead of re-running it. - The cache is keyed by the requested window weight, so a - network-difficulty change recomputes immediately; a TTL of 0 disables - caching. - """ - from lab.prism import public_api - - window_weight_sql = public_api.decimal_string(window_weight) - cache_seconds = self._reward_window_cache_seconds - with self._reward_window_cache_lock: - cached = self._reward_window_cache - if ( - cache_seconds > 0 - and cached is not None - and cached[0] == window_weight_sql - and time.monotonic() - cached[1] < cache_seconds - ): - return cached[2] - sql = f""" -WITH bounds AS ( - SELECT clock_timestamp() AS ended_at -), -window_rows AS ( - SELECT window_row.* - FROM bounds - CROSS JOIN LATERAL qbit_prism_window(bounds.ended_at, {window_weight_sql}::numeric) AS window_row -) -SELECT json_build_object( - 'pool_counted_difficulty', COALESCE((SELECT sum(counted_difficulty) FROM window_rows), 0)::text, - 'miner_counted_difficulty', COALESCE(( - SELECT json_object_agg(miner_id, counted_difficulty) - FROM ( - SELECT miner_id, sum(counted_difficulty)::text AS counted_difficulty - FROM window_rows - GROUP BY miner_id - ) grouped - ), '{{}}'::json) -); -""" - payload = self._run_read_json(sql) - self._reward_window_cache = (window_weight_sql, time.monotonic(), payload) - return payload - - def dashboard_miner_reward_window( - self, - *, - recipient_id: str, - current_network_difficulty: int | str | Decimal, - ) -> dict[str, object]: - from lab.prism import public_api - - if not recipient_id: - raise ValueError("recipient_id is required") - window_weight = _reward_window_weight(current_network_difficulty) - aggregate = self._pool_reward_window_aggregate(window_weight) - pool_difficulty = Decimal(str(aggregate["pool_counted_difficulty"])) - by_miner = aggregate["miner_counted_difficulty"] - miner_difficulty = Decimal(str(by_miner.get(recipient_id, "0"))) - share_percent = None - if pool_difficulty > 0: - share_percent = public_api.decimal_string(miner_difficulty * Decimal(100) / pool_difficulty) - return { - "accepted_difficulty": public_api.decimal_string(miner_difficulty), - "pool_accepted_difficulty": public_api.decimal_string(pool_difficulty), - "share_percent": share_percent, - } - - def dashboard_miner_payout_rows(self, *, recipient_id: str, page: int, limit: int) -> dict[str, object]: - from lab.prism import public_api - - if not recipient_id: - raise ValueError("recipient_id is required") - offset = (page - 1) * limit - sql = f""" -WITH filtered AS ( - SELECT - payout.payout_entry_seq, - payout.block_hash, - payout.block_height, - payout.miner_id, - payout.payout_order_key, - payout.p2mr_program, - payout.onchain_amount_sats, - payout.carry_forward_balance_sats, - payout.action, - payout.maturity_state, - payout.created_at, - block.coinbase_txid, - block.payout_manifest_sha256 - FROM qbit_pool_payout_entries payout - JOIN qbit_pool_blocks block - ON block.block_hash = payout.block_hash - WHERE payout.miner_id = {self._text_literal(recipient_id)} - AND payout.maturity_state <> 'reversed' - AND block.chain_state <> 'reversed' - AND block.maturity_state <> 'reversed' -), -page_rows AS ( - SELECT * - FROM filtered - ORDER BY block_height DESC, payout_entry_seq DESC - LIMIT {int(limit)} OFFSET {int(offset)} -), -resolved AS ( - SELECT - page_rows.*, - fanout.fanout_txid, - fanout.fanout_vout, - fanout.fanout_amount_sats, - fanout.fanout_fee_sats, - fanout.fanout_gross_amount_sats, - fanout.fanout_status - FROM page_rows - LEFT JOIN LATERAL ( - SELECT - artifact.fanout_txid, - artifact.settlement_status AS fanout_status, - (output.value->>'vout')::integer AS fanout_vout, - (output.value->>'amount_sats')::bigint AS fanout_amount_sats, - COALESCE((output.value->>'fee_sats')::bigint, 0) AS fanout_fee_sats, - COALESCE( - (output.value->>'gross_amount_sats')::bigint, - (output.value->>'amount_sats')::bigint - ) AS fanout_gross_amount_sats - FROM qbit_ctv_fanout_artifacts artifact - CROSS JOIN LATERAL jsonb_array_elements(artifact.manifest->'precommitment'->'outputs') AS output(value) - WHERE artifact.block_hash = page_rows.block_hash - AND page_rows.action = 'onchain' - AND output.value->>'recipient_id' = page_rows.miner_id - AND output.value->>'order_key' = page_rows.payout_order_key - AND output.value->>'p2mr_program_hex' = encode(page_rows.p2mr_program, 'hex') - ORDER BY artifact.chunk_index ASC, (output.value->>'vout')::integer ASC - LIMIT 1 - ) fanout ON TRUE -) -SELECT json_build_object( - 'total_count', (SELECT count(*) FROM filtered), - 'rows', COALESCE(( - SELECT json_agg(json_build_object( - 'block_hash', block_hash, - 'block_height', block_height, - 'coinbase_txid', coinbase_txid, - 'payout_manifest_sha256', payout_manifest_sha256, - 'recipient_id', miner_id, - 'order_key', payout_order_key, - 'p2mr_program_hex', encode(p2mr_program, 'hex'), - 'onchain_amount_sats', onchain_amount_sats, - 'carry_forward_balance_sats', carry_forward_balance_sats::text, - 'action', action, - 'maturity_state', maturity_state, - 'created_at', created_at::text, - 'fanout_txid', fanout_txid, - 'fanout_vout', fanout_vout, - 'fanout_amount_sats', fanout_amount_sats, - 'fanout_fee_sats', fanout_fee_sats, - 'fanout_gross_amount_sats', fanout_gross_amount_sats, - 'fanout_status', fanout_status - ) ORDER BY block_height DESC, payout_entry_seq DESC) - FROM resolved - ), '[]'::json) -); -""" - payload = self._run_read_json(sql) - rows = payload["rows"] - for row in rows: - row["carry_forward_balance_sats"] = int(row["carry_forward_balance_sats"]) - return { - "pagination": public_api.pagination(page, limit, int(payload["total_count"])), - "rows": [public_api.miner_payout_row(row) for row in rows], - } - - def dashboard_miner_earning_rows(self, *, recipient_id: str, page: int, limit: int) -> dict[str, object]: - from lab.prism import public_api - - if not recipient_id: - raise ValueError("recipient_id is required") - offset = (page - 1) * limit - sql = f""" -WITH filtered AS ( - SELECT - carry.carry_forward_seq, - carry.block_hash, - carry.block_height, - carry.miner_id, - carry.payout_order_key, - carry.p2mr_program, - carry.gross_amount_sats, - carry.onchain_amount_sats, - carry.settlement_fee_sats, - carry.carry_forward_balance_sats, - carry.action, - carry.maturity_state, - carry.created_at, - block.found_at, - block.coinbase_txid, - block.payout_manifest_sha256 - FROM qbit_payout_carry_forward carry - JOIN qbit_pool_blocks block - ON block.block_hash = carry.block_hash - WHERE carry.miner_id = {self._text_literal(recipient_id)} - AND carry.maturity_state <> 'reversed' - AND block.chain_state <> 'reversed' - AND block.maturity_state <> 'reversed' -), -page_base AS ( - SELECT * - FROM filtered - ORDER BY block_height DESC, carry_forward_seq DESC - LIMIT {int(limit)} OFFSET {int(offset)} -), -block_totals AS ( - SELECT block_hash, sum(gross_amount_sats) AS block_gross_amount_sats - FROM qbit_payout_carry_forward - WHERE block_hash IN (SELECT block_hash FROM page_base) - GROUP BY block_hash -), -page_rows AS ( - SELECT - page_base.*, - block_totals.block_gross_amount_sats, - CASE - WHEN block_totals.block_gross_amount_sats > 0 THEN - (page_base.gross_amount_sats::numeric * 100::numeric / block_totals.block_gross_amount_sats::numeric)::text - ELSE '0' - END AS reward_share_percent - FROM page_base - JOIN block_totals - ON block_totals.block_hash = page_base.block_hash -) -SELECT json_build_object( - 'total_count', (SELECT count(*) FROM filtered), - 'rows', COALESCE(( - SELECT json_agg(json_build_object( - 'block_hash', block_hash, - 'block_height', block_height, - 'coinbase_txid', coinbase_txid, - 'payout_manifest_sha256', payout_manifest_sha256, - 'recipient_id', miner_id, - 'order_key', payout_order_key, - 'p2mr_program_hex', encode(p2mr_program, 'hex'), - 'gross_amount_sats', gross_amount_sats, - 'onchain_amount_sats', onchain_amount_sats, - 'settlement_fee_sats', settlement_fee_sats, - 'carry_forward_balance_sats', carry_forward_balance_sats::text, - 'action', action, - 'maturity_state', maturity_state, - 'created_at', created_at::text, - 'found_at', found_at::text, - 'block_gross_amount_sats', block_gross_amount_sats, - 'reward_share_percent', reward_share_percent - ) ORDER BY block_height DESC, carry_forward_seq DESC) - FROM page_rows - ), '[]'::json) -); -""" - payload = self._run_read_json(sql) - rows = payload["rows"] - for row in rows: - row["carry_forward_balance_sats"] = int(row["carry_forward_balance_sats"]) - return { - "pagination": public_api.pagination(page, limit, int(payload["total_count"])), - "rows": [public_api.miner_earning_row(row) for row in rows], - } - - def dashboard_miner_worker_rows( - self, - *, - recipient_id: str, - page: int, - limit: int, - search: str | None, - hide_inactive: bool, - ) -> dict[str, object]: - from lab.prism import public_api - - if not recipient_id: - raise ValueError("recipient_id is required") - offset = (page - 1) * limit - filters = ["true"] - if search: - filters.append(f"strpos(lower(worker_name), {self._text_literal(search.lower())}) > 0") - if hide_inactive: - filters.append("active") - where_filter = " AND ".join(filters) - sql = f""" -WITH bounds AS ( - SELECT clock_timestamp() AS now_at -), -named AS ( - SELECT - CASE - WHEN username = {self._text_literal(recipient_id)} THEN 'default' - WHEN left(username, {len(recipient_id) + 1}) = {self._text_literal(recipient_id + ".")} THEN COALESCE(NULLIF(substr(username, {len(recipient_id) + 2}), ''), 'default') - WHEN position('.' IN username) > 0 THEN COALESCE(NULLIF(substring(username FROM position('.' IN username) + 1), ''), 'default') - ELSE 'default' - END AS worker_name, - share_difficulty, - accepted_at - FROM ( - SELECT - regexp_replace(share_id, ':[^:]*$', '') AS username, - share_difficulty, - accepted_at - FROM qbit_share_ledger - WHERE accepted - AND miner_id = {self._text_literal(recipient_id)} - -- 3 hours is the largest rollup window this endpoint reports - -- (h3_difficulty), so the scan never needs the miner's full history. - AND accepted_at >= (SELECT now_at FROM bounds) - interval '3 hours' - ) shares -), -grouped AS ( - SELECT - worker_name, - max(accepted_at) AS last_share_at, - COALESCE(sum(share_difficulty) FILTER (WHERE accepted_at >= (SELECT now_at FROM bounds) - interval '1 minute'), 0)::text AS m1_difficulty, - COALESCE(sum(share_difficulty) FILTER (WHERE accepted_at >= (SELECT now_at FROM bounds) - interval '3 hours'), 0)::text AS h3_difficulty, - max(accepted_at) >= (SELECT now_at FROM bounds) - interval '10 minutes' AS active - FROM named - GROUP BY worker_name -), -filtered AS ( - SELECT * - FROM grouped - WHERE {where_filter} -), -page_rows AS ( - SELECT * - FROM filtered - ORDER BY active DESC, worker_name ASC - LIMIT {int(limit)} OFFSET {int(offset)} -) -SELECT json_build_object( - 'total_count', (SELECT count(*) FROM filtered), - 'active_count', (SELECT count(*) FROM grouped WHERE active), - 'rows', COALESCE(( - SELECT json_agg(json_build_object( - 'worker_name', worker_name, - 'status', CASE WHEN active THEN 'active' ELSE 'inactive' END, - 'last_share_at', to_char(last_share_at AT TIME ZONE 'UTC', 'YYYY-MM-DD"T"HH24:MI:SS"Z"'), - 'm1_difficulty', m1_difficulty, - 'h3_difficulty', h3_difficulty - ) ORDER BY active DESC, worker_name ASC) - FROM page_rows - ), '[]'::json) -); -""" - payload = self._run_read_json(sql) - rows = [ - { - "worker_name": row["worker_name"], - "status": row["status"], - "last_share_at": row["last_share_at"], - "hashrate_ths_60s": public_api.hashrate_ths_from_difficulty(row["m1_difficulty"], 60), - "hashrate_ths_3h": public_api.hashrate_ths_from_difficulty(row["h3_difficulty"], 3 * 60 * 60), - } - for row in payload["rows"] - ] - return { - "pagination": public_api.pagination(page, limit, int(payload["total_count"])), - "rows": rows, - "active_count": int(payload["active_count"]), - } - - def audit_bundle(self, *, block_hash: str) -> dict[str, object] | None: - sql = f""" -SELECT COALESCE( - ( - SELECT json_build_object( - 'block_hash', bundle.block_hash, - 'block_height', block.block_height, - 'payout_manifest_sha256', block.payout_manifest_sha256, - 'audit_bundle_sha256', bundle.audit_bundle_sha256, - 'coinbase_tx_hex', bundle.coinbase_tx_hex, - 'audit_bundle', bundle.audit_bundle, - 'body_uri', bundle.body_uri - ) - FROM qbit_pool_audit_bundles bundle - JOIN qbit_pool_blocks block - ON block.block_hash = bundle.block_hash - WHERE bundle.block_hash = {self._text_literal(block_hash)} - ), - 'null'::json -); -""" - with self._operation_gate(self._lock, "writer lock"): - row = self._run_retry_safe_read_json(sql) - return self._resolve_audit_bundle_row(row) - - def dashboard_direct_coinbase_settlement( - self, - *, - block_hash: str, - ) -> dict[str, object] | None: - """Public settlement facts for a direct-coinbase block, without fencing. - - /public/v1/blocks/{block_hash}/settlement-artifacts serves CTV blocks - from audit_ctv_fanout_manifest_set() (a read slot), but a - direct-coinbase block fell through to audit_bundle(), which reads the - same row under the writer lock. This is that read taken through the - bounded read slot instead, returning exactly the dict - public_api.direct_coinbase_settlement_payload() builds today. - - The row is resolved through _resolve_audit_bundle_row, so an - externalized body is read from body_uri on disk and sha256-verified - against audit_bundle_sha256 by the same helper the fenced path uses. - A body that is unretrievable, hash-mismatched, or not valid JSON is - reported as None rather than raised, matching what - public_api.audit_bundle_body_read_failed() already tolerates. - - Returns None when the block has no audit bundle, or when the bundle - settled in any mode other than direct_coinbase. - """ - from lab.prism import public_api - - sql = f""" -SELECT COALESCE( - ( - SELECT json_build_object( - 'block_hash', bundle.block_hash, - 'block_height', block.block_height, - 'payout_manifest_sha256', block.payout_manifest_sha256, - 'audit_bundle_sha256', bundle.audit_bundle_sha256, - 'audit_bundle', bundle.audit_bundle, - 'body_uri', bundle.body_uri - ) - FROM qbit_pool_audit_bundles bundle - JOIN qbit_pool_blocks block - ON block.block_hash = bundle.block_hash - WHERE bundle.block_hash = {self._text_literal(block_hash)} - ), - 'null'::json -); -""" - row = self._run_read_json(sql) - try: - resolved = self._resolve_audit_bundle_row(row) - except RuntimeError as exc: - if public_api.audit_bundle_body_read_failed(exc): - return None - raise - if not isinstance(resolved, dict): - return None - bundle = resolved.get("audit_bundle") - if not isinstance(bundle, dict): - return None - if public_api.audit_bundle_settlement_mode(bundle) != "direct_coinbase": - return None - return { - "block_hash": public_api.optional_hex_hash(resolved.get("block_hash")) - or block_hash, - "block_height": public_api.first_int( - resolved.get("block_height"), - public_api.audit_bundle_section_value( - bundle, "found_block", "block_height" - ), - public_api.audit_bundle_section_value( - bundle, "reward_manifest", "block_height" - ), - public_api.audit_bundle_section_value( - bundle, "ledger_window_attestation", "block_height" - ), - default=0, - ), - "settlement_mode": "direct_coinbase", - "audit_bundle_sha256": public_api.nullable_str( - resolved.get("audit_bundle_sha256") - ), - "payout_manifest_sha256": public_api.nullable_str( - resolved.get("payout_manifest_sha256") - ), - "artifacts": [], - } - - def audit_bundle_by_commitment(self, *, commitment_leaf_hex: str) -> dict[str, object] | None: - leaf = self._text_literal(commitment_leaf_hex) - sql = f""" -SELECT COALESCE( - ( - SELECT json_build_object( - 'block_hash', bundle.block_hash, - 'block_height', block.block_height, - 'payout_manifest_sha256', block.payout_manifest_sha256, - 'audit_commitment_leaf_hex', {leaf}, - 'audit_bundle_sha256', bundle.audit_bundle_sha256, - 'coinbase_tx_hex', bundle.coinbase_tx_hex, - 'audit_bundle', bundle.audit_bundle, - 'body_uri', bundle.body_uri - ) - FROM qbit_pool_audit_bundles bundle - JOIN qbit_pool_blocks block ON block.block_hash = bundle.block_hash - WHERE bundle.audit_commitment_leaves_hex ? {leaf} - OR bundle.witness_merkle_leaves_hex ? {leaf} - OR bundle.audit_bundle->'audit_commitment_leaves_hex' ? {leaf} - OR bundle.audit_bundle->'witness_merkle_leaves_hex' ? {leaf} - ORDER BY block.block_height DESC, bundle.created_at DESC, bundle.block_hash - LIMIT 1 - ), - 'null'::json -); -""" - with self._operation_gate(self._lock, "writer lock"): - row = self._run_retry_safe_read_json(sql) - return self._resolve_audit_bundle_row(row) - - def persist_ctv_fanout_manifest_set( - self, - *, - block_hash: str, - manifest_set: Mapping[str, Any], - manifest_set_sha256: str, - ) -> dict[str, int | str]: - payload = { - **ctv_fanout_recovery_payload( - block_hash=block_hash, - manifest_set=manifest_set, - manifest_set_sha256=manifest_set_sha256, - ), - "writer_id": self._writer_id, - "writer_epoch": self._writer_epoch, - "writer_session_token": self._writer_session_token, - } - sql_body = f"""lease AS ( - UPDATE qbit_ledger_writer_lease - SET lease_expires_at = clock_timestamp() + {self._lease_interval_sql}, - updated_at = clock_timestamp() - FROM payload - WHERE qbit_ledger_writer_lease.singleton - AND qbit_ledger_writer_lease.writer_id = data->>'writer_id' - AND qbit_ledger_writer_lease.writer_epoch = (data->>'writer_epoch')::bigint - AND qbit_ledger_writer_lease.writer_session_token = data->>'writer_session_token' - RETURNING qbit_ledger_writer_lease.writer_id -), -block_row AS ( - SELECT block_hash - FROM qbit_pool_blocks - WHERE block_hash = (SELECT data->>'block_hash' FROM payload) -), -existing_set AS ( - SELECT * - FROM qbit_ctv_fanout_sets - WHERE block_hash = (SELECT data->>'block_hash' FROM payload) -), -inserted_set AS ( - INSERT INTO qbit_ctv_fanout_sets ( - block_hash, - manifest_set_json, - manifest_set, - manifest_set_sha256, - settlement_mode, - parent_coinbase_txid, - parent_coinbase_tx_hex, - fanout_count, - fanout_output_sum_sats, - covenant_output_value_sats - ) - SELECT - data->>'block_hash', - data->>'manifest_set_json', - data->'manifest_set', - data->>'manifest_set_sha256', - data->>'settlement_mode', - data->>'parent_coinbase_txid', - data->>'parent_coinbase_tx_hex', - (data->>'fanout_count')::integer, - (data->>'fanout_output_sum_sats')::bigint, - (data->>'covenant_output_value_sats')::bigint - FROM payload, lease, block_row - WHERE NOT EXISTS (SELECT 1 FROM existing_set) - RETURNING block_hash -), -artifacts AS ( - SELECT data, artifact - FROM payload, - jsonb_array_elements(data->'artifacts') AS artifact -), -matching_existing_set AS ( - SELECT existing_set.block_hash - FROM existing_set, payload - WHERE existing_set.manifest_set = data->'manifest_set' - AND existing_set.manifest_set_json = data->>'manifest_set_json' - AND existing_set.manifest_set_sha256 = data->>'manifest_set_sha256' - AND existing_set.settlement_mode = data->>'settlement_mode' - AND existing_set.parent_coinbase_txid = data->>'parent_coinbase_txid' - AND existing_set.parent_coinbase_tx_hex = data->>'parent_coinbase_tx_hex' - AND existing_set.fanout_count = (data->>'fanout_count')::integer - AND existing_set.fanout_output_sum_sats = (data->>'fanout_output_sum_sats')::bigint - AND existing_set.covenant_output_value_sats = (data->>'covenant_output_value_sats')::bigint -), -expected_artifact_rows AS ( - SELECT - artifact->>'fanout_txid' AS fanout_txid, - data->>'block_hash' AS block_hash, - data->>'manifest_set_sha256' AS manifest_set_sha256, - artifact->>'manifest_json' AS manifest_json, - artifact->'manifest' AS manifest, - artifact->>'manifest_sha256' AS manifest_sha256, - artifact->>'precommitment_sha256' AS precommitment_sha256, - artifact->>'ctv_hash' AS ctv_hash, - artifact->>'commitment_witness_leaf_hex' AS commitment_witness_leaf_hex, - (artifact->>'chunk_index')::integer AS chunk_index, - (artifact->>'chunk_count')::integer AS chunk_count, - artifact->>'parent_coinbase_txid' AS parent_coinbase_txid, - (artifact->>'parent_coinbase_vout')::integer AS parent_coinbase_vout, - artifact->>'fanout_tx_template_hex' AS fanout_tx_template_hex, - artifact->>'fanout_tx_hex' AS fanout_tx_hex, - (artifact->>'anchor_vout')::integer AS anchor_vout, - (artifact->>'covenant_output_value_sats')::bigint AS covenant_output_value_sats, - (artifact->>'fanout_output_sum_sats')::bigint AS fanout_output_sum_sats - FROM artifacts -), -existing_artifact_rows AS ( - SELECT - fanout_txid, - block_hash, - manifest_set_sha256, - manifest_json, - manifest, - manifest_sha256, - precommitment_sha256, - ctv_hash, - commitment_witness_leaf_hex, - chunk_index, - chunk_count, - parent_coinbase_txid, - parent_coinbase_vout, - fanout_tx_template_hex, - fanout_tx_hex, - anchor_vout, - covenant_output_value_sats, - fanout_output_sum_sats - FROM qbit_ctv_fanout_artifacts - WHERE block_hash = (SELECT data->>'block_hash' FROM payload) -), -artifact_extra AS ( - SELECT * FROM existing_artifact_rows - EXCEPT ALL - SELECT * FROM expected_artifact_rows -), -inserted_artifacts AS ( - INSERT INTO qbit_ctv_fanout_artifacts ( - fanout_txid, - block_hash, - manifest_set_sha256, - manifest_json, - manifest, - manifest_sha256, - precommitment_sha256, - ctv_hash, - commitment_witness_leaf_hex, - chunk_index, - chunk_count, - parent_coinbase_txid, - parent_coinbase_vout, - fanout_tx_template_hex, - fanout_tx_hex, - anchor_vout, - covenant_output_value_sats, - fanout_output_sum_sats - ) - SELECT - expected.fanout_txid, - expected.block_hash, - expected.manifest_set_sha256, - expected.manifest_json, - expected.manifest, - expected.manifest_sha256, - expected.precommitment_sha256, - expected.ctv_hash, - expected.commitment_witness_leaf_hex, - expected.chunk_index, - expected.chunk_count, - expected.parent_coinbase_txid, - expected.parent_coinbase_vout, - expected.fanout_tx_template_hex, - expected.fanout_tx_hex, - expected.anchor_vout, - expected.covenant_output_value_sats, - expected.fanout_output_sum_sats - FROM expected_artifact_rows expected - WHERE EXISTS (SELECT 1 FROM lease) - AND ( - EXISTS (SELECT 1 FROM inserted_set) - OR ( - EXISTS (SELECT 1 FROM matching_existing_set) - AND NOT EXISTS (SELECT 1 FROM artifact_extra) - ) - ) - AND NOT EXISTS ( - SELECT 1 - FROM existing_artifact_rows existing - WHERE existing.fanout_txid = expected.fanout_txid - ) - RETURNING fanout_txid -) -SELECT CASE - WHEN (SELECT count(*) FROM lease) = 0 THEN - json_build_object('error', 'writer lease is not active') - WHEN (SELECT count(*) FROM block_row) = 0 THEN - json_build_object('error', 'unknown PRISM block') - WHEN (SELECT count(*) FROM existing_set) > 0 - AND (SELECT count(*) FROM matching_existing_set) = 0 THEN - json_build_object('error', 'existing CTV fanout manifest set does not match payload') - WHEN (SELECT count(*) FROM existing_set) > 0 - AND EXISTS (SELECT 1 FROM artifact_extra) THEN - json_build_object('error', 'existing CTV fanout artifacts do not match payload') - ELSE - json_build_object( - 'backend', 'postgres-psql', - 'fanout_set_count', CASE - WHEN (SELECT count(*) FROM inserted_set) > 0 THEN (SELECT count(*) FROM inserted_set) - ELSE (SELECT count(*) FROM existing_set) - END, - 'fanout_artifact_count', - (SELECT count(*) FROM existing_artifact_rows) - + (SELECT count(*) FROM inserted_artifacts) - ) -END; -""" - result = self._run_candidate_payload_json(payload, sql_body) - if "error" in result: - raise RuntimeError(str(result["error"])) - return { - "backend": str(result["backend"]), - "fanout_set_count": int(result["fanout_set_count"]), - "fanout_artifact_count": int(result["fanout_artifact_count"]), - } - - def audit_ctv_fanout_manifest_set(self, *, block_hash: str) -> dict[str, object] | None: - sql = f"SELECT qbit_audit_block_fanouts({self._text_literal(block_hash)});" - return self._run_read_json(sql) - - def audit_ctv_fanouts(self, *, block_hash: str) -> list[dict[str, object]]: - payload = self.audit_ctv_fanout_manifest_set(block_hash=block_hash) - if payload is None: - return [] - artifacts = payload.get("artifacts", []) - if not isinstance(artifacts, list): - return [] - return artifacts - - def ctv_fanout_status(self, *, fanout_txid: str) -> dict[str, object] | None: - sql = f"SELECT qbit_fanout_status({self._text_literal(fanout_txid)});" - return self._run_read_json(sql) - - def pending_ctv_fanout_statuses(self, *, limit: int = 100) -> list[dict[str, object]]: - limit = max(1, min(int(limit), 1_000)) - sql = f""" -SELECT COALESCE(json_agg(row_payload ORDER BY block_height ASC, chunk_index ASC), '[]'::json) -FROM ( - SELECT json_build_object( - 'schema', 'qbit.prism.ctv-fanout-status.v1', - 'fanout_txid', artifact.fanout_txid, - 'block_hash', artifact.block_hash, - 'block_height', block.block_height, - 'parent_hash', block.parent_hash, - 'chain_state', block.chain_state, - 'maturity_state', block.maturity_state, - 'coinbase_txid', block.coinbase_txid, - 'payout_manifest_sha256', block.payout_manifest_sha256, - 'audit_bundle_sha256', bundle.audit_bundle_sha256, - 'manifest_set_sha256', artifact.manifest_set_sha256, - 'manifest_sha256', artifact.manifest_sha256, - 'precommitment_sha256', artifact.precommitment_sha256, - 'ctv_hash', artifact.ctv_hash, - 'commitment_witness_leaf_hex', artifact.commitment_witness_leaf_hex, - 'chunk_index', artifact.chunk_index, - 'chunk_count', artifact.chunk_count, - 'parent_coinbase_txid', artifact.parent_coinbase_txid, - 'parent_coinbase_vout', artifact.parent_coinbase_vout, - 'fanout_tx_hex', artifact.fanout_tx_hex, - 'anchor_vout', artifact.anchor_vout, - 'covenant_output_value_sats', artifact.covenant_output_value_sats, - 'fanout_output_sum_sats', artifact.fanout_output_sum_sats, - 'settlement_status', artifact.settlement_status, - 'updated_at', artifact.updated_at::text, - 'broadcast_attempt_count', artifact.broadcast_attempt_count, - 'broadcast_attempt_detail_count', artifact.broadcast_attempt_detail_count, - 'first_broadcast_attempt_at', artifact.first_broadcast_attempt_at::text, - 'last_broadcast_attempt_at', artifact.last_broadcast_attempt_at::text, - 'last_broadcast_attempt_status', artifact.last_broadcast_attempt_status, - 'last_broadcast_package_tx_hexes', artifact.last_broadcast_package_tx_hexes, - 'last_broadcast_package_txids', artifact.last_broadcast_package_txids, - 'last_broadcast_submit_result', artifact.last_broadcast_submit_result, - 'last_broadcast_error', artifact.last_broadcast_error, - 'broadcast_attempt_status_counts', artifact.broadcast_attempt_status_counts, - 'next_broadcast_attempt_at', artifact.next_broadcast_attempt_at::text, - 'broadcast_retry_backoff_seconds', artifact.broadcast_retry_backoff_seconds, - 'broadcast_attempt_summary', json_build_object( - 'attempt_count', artifact.broadcast_attempt_count, - 'detail_count', artifact.broadcast_attempt_detail_count, - 'first_attempt_at', artifact.first_broadcast_attempt_at::text, - 'last_attempt_at', artifact.last_broadcast_attempt_at::text, - 'last_attempt_status', artifact.last_broadcast_attempt_status, - 'last_package_tx_hexes', artifact.last_broadcast_package_tx_hexes, - 'last_package_txids', artifact.last_broadcast_package_txids, - 'last_submit_result', artifact.last_broadcast_submit_result, - 'last_error', artifact.last_broadcast_error, - 'status_counts', artifact.broadcast_attempt_status_counts, - 'next_attempt_at', artifact.next_broadcast_attempt_at::text, - 'retry_backoff_seconds', artifact.broadcast_retry_backoff_seconds - ), - 'broadcast_attempts', COALESCE( - ( - SELECT json_agg(json_build_object( - 'attempt_seq', attempt.attempt_seq, - 'attempted_at', attempt.attempted_at::text, - 'attempt_status', attempt.attempt_status, - 'package_tx_hexes', attempt.package_tx_hexes, - 'package_txids', attempt.package_txids, - 'submit_result', attempt.submit_result, - 'error', attempt.error - ) ORDER BY attempt.attempt_seq ASC) - FROM qbit_ctv_fanout_broadcast_attempts attempt - WHERE attempt.fanout_txid = artifact.fanout_txid - ), - '[]'::json - ) - ) AS row_payload, - block.block_height, - artifact.chunk_index - FROM qbit_ctv_fanout_artifacts artifact - JOIN qbit_pool_blocks block - ON block.block_hash = artifact.block_hash - LEFT JOIN qbit_pool_audit_bundles bundle - ON bundle.block_hash = artifact.block_hash - WHERE artifact.settlement_status NOT IN ('confirmed', 'reorged', 'failed') - AND ( - artifact.next_broadcast_attempt_at IS NULL - OR artifact.next_broadcast_attempt_at <= clock_timestamp() - ) - ORDER BY block.block_height ASC, artifact.chunk_index ASC - LIMIT {limit} -) pending; -""" - return self._run_read_json(sql) - - def dashboard_pending_fanout_rows(self, *, page: int, limit: int) -> dict[str, object]: - from lab.prism import public_api - - offset = (page - 1) * limit - sql = f""" -WITH filtered AS ( - SELECT - artifact.fanout_txid, - artifact.block_hash, - block.block_height, - block.parent_hash, - block.chain_state, - block.maturity_state, - block.coinbase_txid, - block.payout_manifest_sha256, - bundle.audit_bundle_sha256, - artifact.manifest_set_sha256, - artifact.manifest_sha256, - artifact.precommitment_sha256, - artifact.ctv_hash, - artifact.commitment_witness_leaf_hex, - artifact.chunk_index, - artifact.chunk_count, - artifact.parent_coinbase_txid, - artifact.parent_coinbase_vout, - artifact.fanout_tx_hex, - artifact.anchor_vout, - artifact.covenant_output_value_sats, - artifact.fanout_output_sum_sats, - artifact.settlement_status, - artifact.updated_at, - artifact.broadcast_attempt_count, - artifact.broadcast_attempt_detail_count, - artifact.first_broadcast_attempt_at, - artifact.last_broadcast_attempt_at, - artifact.last_broadcast_attempt_status, - artifact.last_broadcast_package_tx_hexes, - artifact.last_broadcast_package_txids, - artifact.last_broadcast_submit_result, - artifact.last_broadcast_error, - artifact.broadcast_attempt_status_counts, - artifact.next_broadcast_attempt_at, - artifact.broadcast_retry_backoff_seconds - FROM qbit_ctv_fanout_artifacts artifact - JOIN qbit_pool_blocks block - ON block.block_hash = artifact.block_hash - LEFT JOIN qbit_pool_audit_bundles bundle - ON bundle.block_hash = artifact.block_hash - WHERE artifact.settlement_status NOT IN ('confirmed', 'reorged', 'failed') - AND ( - artifact.next_broadcast_attempt_at IS NULL - OR artifact.next_broadcast_attempt_at <= clock_timestamp() - ) -), -page_rows AS ( - SELECT * - FROM filtered - ORDER BY block_height ASC, chunk_index ASC - LIMIT {int(limit)} OFFSET {int(offset)} -) -SELECT json_build_object( - 'total_count', (SELECT count(*) FROM filtered), - 'rows', COALESCE(( - SELECT json_agg(json_build_object( - 'schema', 'qbit.prism.ctv-fanout-status.v1', - 'fanout_txid', fanout_txid, - 'block_hash', block_hash, - 'block_height', block_height, - 'parent_hash', parent_hash, - 'chain_state', chain_state, - 'maturity_state', maturity_state, - 'coinbase_txid', coinbase_txid, - 'payout_manifest_sha256', payout_manifest_sha256, - 'audit_bundle_sha256', audit_bundle_sha256, - 'manifest_set_sha256', manifest_set_sha256, - 'manifest_sha256', manifest_sha256, - 'precommitment_sha256', precommitment_sha256, - 'ctv_hash', ctv_hash, - 'commitment_witness_leaf_hex', commitment_witness_leaf_hex, - 'chunk_index', chunk_index, - 'chunk_count', chunk_count, - 'parent_coinbase_txid', parent_coinbase_txid, - 'parent_coinbase_vout', parent_coinbase_vout, - 'fanout_tx_hex', fanout_tx_hex, - 'anchor_vout', anchor_vout, - 'covenant_output_value_sats', covenant_output_value_sats, - 'fanout_output_sum_sats', fanout_output_sum_sats, - 'settlement_status', settlement_status, - 'updated_at', updated_at::text, - 'broadcast_attempt_count', broadcast_attempt_count, - 'broadcast_attempt_detail_count', broadcast_attempt_detail_count, - 'first_broadcast_attempt_at', first_broadcast_attempt_at::text, - 'last_broadcast_attempt_at', last_broadcast_attempt_at::text, - 'last_broadcast_attempt_status', last_broadcast_attempt_status, - 'last_broadcast_package_tx_hexes', last_broadcast_package_tx_hexes, - 'last_broadcast_package_txids', last_broadcast_package_txids, - 'last_broadcast_submit_result', last_broadcast_submit_result, - 'last_broadcast_error', last_broadcast_error, - 'broadcast_attempt_status_counts', broadcast_attempt_status_counts, - 'next_broadcast_attempt_at', next_broadcast_attempt_at::text, - 'broadcast_retry_backoff_seconds', broadcast_retry_backoff_seconds, - 'broadcast_attempt_summary', json_build_object( - 'attempt_count', broadcast_attempt_count, - 'detail_count', broadcast_attempt_detail_count, - 'first_attempt_at', first_broadcast_attempt_at::text, - 'last_attempt_at', last_broadcast_attempt_at::text, - 'last_attempt_status', last_broadcast_attempt_status, - 'last_package_tx_hexes', last_broadcast_package_tx_hexes, - 'last_package_txids', last_broadcast_package_txids, - 'last_submit_result', last_broadcast_submit_result, - 'last_error', last_broadcast_error, - 'status_counts', broadcast_attempt_status_counts, - 'next_attempt_at', next_broadcast_attempt_at::text, - 'retry_backoff_seconds', broadcast_retry_backoff_seconds - ), - 'broadcast_attempts', COALESCE( - ( - SELECT json_agg(json_build_object( - 'attempt_seq', attempt.attempt_seq, - 'attempted_at', attempt.attempted_at::text, - 'attempt_status', attempt.attempt_status, - 'package_tx_hexes', attempt.package_tx_hexes, - 'package_txids', attempt.package_txids, - 'submit_result', attempt.submit_result, - 'error', attempt.error - ) ORDER BY attempt.attempt_seq ASC) - FROM qbit_ctv_fanout_broadcast_attempts attempt - WHERE attempt.fanout_txid = page_rows.fanout_txid - ), - '[]'::json - ) - ) ORDER BY block_height ASC, chunk_index ASC) - FROM page_rows - ), '[]'::json) -); -""" - payload = self._run_read_json(sql) - return { - "pagination": public_api.pagination(page, limit, int(payload["total_count"])), - "rows": payload["rows"], - } - - def dashboard_public_artifact(self, *, sha256: str) -> dict[str, object] | None: - document = self.dashboard_public_artifact_document(sha256=sha256) - if document is None: - return None - return document.get("payload") - - def dashboard_public_artifact_document(self, *, sha256: str) -> dict[str, object] | None: - """Artifact payload plus its stored canonical text when available. - - canonical_json is the manifest_set_json/manifest_json text persisted - at record time next to the JSONB copies — the exact byte sequence the - artifact's sha256 was computed over. Audit bundles use the immutable - compressed canonical artifact beside the external body. One statement - serves both database reads so a request touches each artifact table at - most once. - - Replica ordering is intentionally simple. A visible row plus a missing - canonical file is legacy history and falls back to the JSONB/external - body. A file that reached shared storage before this replica replays its - row remains invisible and returns no match until replay catches up. - """ - sha256 = str(sha256).lower() - lit = self._text_literal(sha256) - sql = f""" -WITH audit AS ( - SELECT block_hash, audit_bundle, audit_bundle_sha256, body_uri - FROM qbit_pool_audit_bundles - WHERE audit_bundle_sha256 = {lit} - ORDER BY created_at DESC - LIMIT 1 -), -set_row AS ( - SELECT manifest_set, manifest_set_json - FROM qbit_ctv_fanout_sets - WHERE manifest_set_sha256 = {lit} - ORDER BY created_at DESC - LIMIT 1 -), -artifact_row AS ( - SELECT manifest, manifest_json - FROM qbit_ctv_fanout_artifacts - WHERE manifest_sha256 = {lit} - ORDER BY updated_at DESC - LIMIT 1 -) -SELECT json_build_object( - 'block_hash', (SELECT block_hash FROM audit), - 'audit_bundle', (SELECT audit_bundle FROM audit), - 'audit_bundle_sha256', (SELECT audit_bundle_sha256 FROM audit), - 'body_uri', (SELECT body_uri FROM audit), - 'has_audit_row', (SELECT count(*) FROM audit) > 0, - 'fallback', COALESCE( - (SELECT manifest_set FROM set_row), - (SELECT manifest FROM artifact_row) - ), - 'fallback_canonical', COALESCE( - (SELECT manifest_set_json FROM set_row), - (SELECT manifest_json FROM artifact_row) - ) -); -""" - row = self._run_read_json(sql) - if not isinstance(row, dict): - return None - if row.get("has_audit_row"): - block_hash = row.get("block_hash") - if isinstance(block_hash, str): - canonical_bytes = self._stored_canonical_audit_bundle_bytes( - block_hash, - sha256, - ) - if canonical_bytes is not None: - try: - canonical_payload = json.loads(canonical_bytes) - canonical_json = canonical_bytes.decode("utf-8") - except (UnicodeDecodeError, json.JSONDecodeError, TypeError) as exc: - LOGGER.error( - "canonical audit bundle unavailable reason=corrupt " - "block_hash=%s audit_bundle_sha256=%s", - block_hash, - sha256, - ) - raise CanonicalAuditBundleCorrupt( - "canonical audit bundle is not valid UTF-8 JSON" - ) from exc - if not isinstance(canonical_payload, dict): - LOGGER.error( - "canonical audit bundle unavailable reason=corrupt " - "block_hash=%s audit_bundle_sha256=%s", - block_hash, - sha256, - ) - raise CanonicalAuditBundleCorrupt( - "canonical audit bundle JSON is not an object" - ) - return { - "payload": canonical_payload, - "canonical_json": canonical_json, - } - body = row.get("audit_bundle") - if body is None: - body = self._read_external_body(row.get("body_uri"), expected_sha256=sha256) - if body is not None: - return { - "payload": body, - "canonical_json": None, - "canonical_fallback_reason": "missing", - } - fallback = row.get("fallback") - if not isinstance(fallback, dict): - return None - fallback_canonical = row.get("fallback_canonical") - return { - "payload": fallback, - "canonical_json": fallback_canonical if isinstance(fallback_canonical, str) else None, - } - - def dashboard_public_artifact_exists(self, *, sha256: str) -> bool: - sha256 = str(sha256).lower() - lit = self._text_literal(sha256) - sql = f""" -WITH audit AS ( - SELECT block_hash, audit_bundle, audit_bundle_sha256, body_uri - FROM qbit_pool_audit_bundles - WHERE audit_bundle_sha256 = {lit} - ORDER BY created_at DESC - LIMIT 1 -) -SELECT json_build_object( - 'has_audit_row', (SELECT count(*) FROM audit) > 0, - 'block_hash', (SELECT block_hash FROM audit), - 'audit_bundle_sha256', (SELECT audit_bundle_sha256 FROM audit), - 'audit_bundle_inline', (SELECT audit_bundle IS NOT NULL FROM audit), - 'body_uri', (SELECT body_uri FROM audit), - 'fallback_exists', - EXISTS ( - SELECT 1 - FROM qbit_ctv_fanout_sets - WHERE manifest_set_sha256 = {lit} - ) - OR EXISTS ( - SELECT 1 - FROM qbit_ctv_fanout_artifacts - WHERE manifest_sha256 = {lit} - ) -); -""" - row = self._run_read_json(sql) - if not isinstance(row, dict): - return False - if row.get("has_audit_row"): - block_hash = row.get("block_hash") - if ( - isinstance(block_hash, str) - and self._stored_canonical_audit_bundle_bytes(block_hash, sha256) - is not None - ): - return True - if row.get("audit_bundle_inline"): - return True - body_uri = row.get("body_uri") - if not body_uri: - return False - return self._external_body_available_for_sha(body_uri, sha256) - return bool(row.get("fallback_exists")) - - def update_ctv_fanout_status(self, *, fanout_txid: str, settlement_status: str) -> dict[str, int | str]: - validate_ctv_fanout_status(settlement_status) - payload = { - "fanout_txid": fanout_txid, - "settlement_status": settlement_status, - "writer_id": self._writer_id, - "writer_epoch": self._writer_epoch, - "writer_session_token": self._writer_session_token, - } - sql = f""" -WITH payload AS ( - SELECT {self._jsonb_literal(payload)} AS data -), -lease AS ( - UPDATE qbit_ledger_writer_lease - SET lease_expires_at = clock_timestamp() + {self._lease_interval_sql}, - updated_at = clock_timestamp() - FROM payload - WHERE qbit_ledger_writer_lease.singleton - AND qbit_ledger_writer_lease.writer_id = data->>'writer_id' - AND qbit_ledger_writer_lease.writer_epoch = (data->>'writer_epoch')::bigint - AND qbit_ledger_writer_lease.writer_session_token = data->>'writer_session_token' - RETURNING qbit_ledger_writer_lease.writer_id -), -updated AS ( - UPDATE qbit_ctv_fanout_artifacts - SET settlement_status = (SELECT data->>'settlement_status' FROM payload), - updated_at = clock_timestamp() - FROM lease - WHERE fanout_txid = (SELECT data->>'fanout_txid' FROM payload) - RETURNING fanout_txid -) -SELECT CASE - WHEN (SELECT count(*) FROM lease) = 0 THEN - json_build_object('error', 'writer lease is not active') - ELSE - json_build_object( - 'backend', 'postgres-psql', - 'updated_count', (SELECT count(*) FROM updated) - ) -END; -""" - result = self._run_fenced_json(sql) - if "error" in result: - raise RuntimeError(str(result["error"])) - return {"backend": str(result["backend"]), "updated_count": int(result["updated_count"])} - - def record_ctv_fanout_broadcast_attempt( - self, - *, - fanout_txid: str, - attempt_status: str, - package_tx_hexes: list[str] | None = None, - package_txids: list[str] | None = None, - submit_result: dict[str, Any] | None = None, - error: str | None = None, - ) -> dict[str, int | str]: - validate_ctv_fanout_attempt_status(attempt_status) - next_status = None - if attempt_status in {"submitted", "accepted"}: - next_status = "broadcast_submitted" - elif attempt_status in {"rejected", "failed"}: - next_status = "failed" - payload = { - "fanout_txid": fanout_txid, - "attempt_status": attempt_status, - "package_tx_hexes": package_tx_hexes or [], - "package_txids": package_txids or [], - "submit_result": submit_result, - "error": error, - "next_status": next_status, - "attempt_detail_limit": self._ctv_broadcast_attempt_detail_limit, - "retry_backoff_seconds": self._ctv_broadcast_retry_backoff_seconds, - "writer_id": self._writer_id, - "writer_epoch": self._writer_epoch, - "writer_session_token": self._writer_session_token, - } - sql = f""" -WITH payload AS ( - SELECT {self._jsonb_literal(payload)} AS data -), -lease AS ( - UPDATE qbit_ledger_writer_lease - SET lease_expires_at = clock_timestamp() + {self._lease_interval_sql}, - updated_at = clock_timestamp() - FROM payload - WHERE qbit_ledger_writer_lease.singleton - AND qbit_ledger_writer_lease.writer_id = data->>'writer_id' - AND qbit_ledger_writer_lease.writer_epoch = (data->>'writer_epoch')::bigint - AND qbit_ledger_writer_lease.writer_session_token = data->>'writer_session_token' - RETURNING qbit_ledger_writer_lease.writer_id -), -artifact_row AS ( - SELECT fanout_txid - FROM qbit_ctv_fanout_artifacts - WHERE fanout_txid = (SELECT data->>'fanout_txid' FROM payload) -), -existing_detail_count AS ( - SELECT count(*)::bigint AS detail_count - FROM qbit_ctv_fanout_broadcast_attempts - WHERE fanout_txid = (SELECT data->>'fanout_txid' FROM payload) -), -pruned AS ( - DELETE FROM qbit_ctv_fanout_broadcast_attempts old_attempt - USING payload, artifact_row - WHERE old_attempt.fanout_txid = artifact_row.fanout_txid - AND old_attempt.attempt_seq IN ( - SELECT retained.attempt_seq - FROM qbit_ctv_fanout_broadcast_attempts retained - WHERE retained.fanout_txid = artifact_row.fanout_txid - ORDER BY retained.attempt_seq DESC - OFFSET GREATEST((data->>'attempt_detail_limit')::integer - 1, 0) - ) - RETURNING old_attempt.attempt_seq -), -pruned_count AS ( - SELECT count(*)::bigint AS pruned_count FROM pruned -), -inserted AS ( - INSERT INTO qbit_ctv_fanout_broadcast_attempts ( - fanout_txid, - attempt_status, - package_tx_hexes, - package_txids, - submit_result, - error - ) - SELECT - data->>'fanout_txid', - data->>'attempt_status', - data->'package_tx_hexes', - data->'package_txids', - data->'submit_result', - data->>'error' - FROM payload, lease, artifact_row, pruned_count - WHERE (data->>'attempt_detail_limit')::integer > 0 - RETURNING attempt_seq -), -inserted_count AS ( - SELECT count(*)::bigint AS inserted_count FROM inserted -), -updated AS ( - UPDATE qbit_ctv_fanout_artifacts artifact - SET settlement_status = COALESCE(data->>'next_status', artifact.settlement_status), - updated_at = clock_timestamp(), - broadcast_attempt_count = artifact.broadcast_attempt_count + 1, - broadcast_attempt_detail_count = CASE - WHEN (data->>'attempt_detail_limit')::integer <= 0 THEN 0 - ELSE LEAST( - (data->>'attempt_detail_limit')::bigint, - GREATEST( - 0, - existing_detail_count.detail_count - - pruned_count.pruned_count - + inserted_count.inserted_count - ) - ) - END, - first_broadcast_attempt_at = COALESCE(artifact.first_broadcast_attempt_at, clock_timestamp()), - last_broadcast_attempt_at = clock_timestamp(), - last_broadcast_attempt_status = data->>'attempt_status', - last_broadcast_package_tx_hexes = data->'package_tx_hexes', - last_broadcast_package_txids = data->'package_txids', - last_broadcast_submit_result = data->'submit_result', - last_broadcast_error = data->>'error', - broadcast_attempt_status_counts = jsonb_set( - COALESCE(artifact.broadcast_attempt_status_counts, '{{}}'::jsonb), - ARRAY[data->>'attempt_status'], - to_jsonb( - COALESCE((artifact.broadcast_attempt_status_counts->>(data->>'attempt_status'))::bigint, 0) - + 1 - ), - true - ), - next_broadcast_attempt_at = CASE - WHEN data->>'attempt_status' = 'planned' - AND (data->>'retry_backoff_seconds')::bigint > 0 THEN - clock_timestamp() + make_interval(secs => (data->>'retry_backoff_seconds')::double precision) - ELSE NULL - END, - broadcast_retry_backoff_seconds = CASE - WHEN data->>'attempt_status' = 'planned' THEN (data->>'retry_backoff_seconds')::bigint - ELSE 0 - END - FROM payload, lease, artifact_row, existing_detail_count, pruned_count, inserted_count - WHERE artifact.fanout_txid = artifact_row.fanout_txid - RETURNING artifact.fanout_txid, artifact.broadcast_attempt_count, artifact.broadcast_attempt_detail_count -) -SELECT CASE - WHEN (SELECT count(*) FROM lease) = 0 THEN - json_build_object('error', 'writer lease is not active') - WHEN (SELECT count(*) FROM artifact_row) = 0 THEN - json_build_object('error', 'unknown CTV fanout txid') - ELSE - json_build_object( - 'backend', 'postgres-psql', - 'attempt_count', (SELECT count(*) FROM inserted), - 'updated_count', (SELECT count(*) FROM updated), - 'broadcast_attempt_count', (SELECT broadcast_attempt_count FROM updated LIMIT 1), - 'broadcast_attempt_detail_count', (SELECT broadcast_attempt_detail_count FROM updated LIMIT 1) - ) -END; -""" - result = self._run_fenced_json(sql) - if "error" in result: - raise RuntimeError(str(result["error"])) - return { - "backend": str(result["backend"]), - "attempt_count": int(result["attempt_count"]), - "updated_count": int(result["updated_count"]), - } - - def metrics(self) -> dict[str, int]: - # The accepted-share count comes from the incrementally maintained - # stats (see accepted_share_stats) so metrics scrapes never seq-scan - # the share ledger; the block/payout tables stay small. - accepted_share_count = int(self.accepted_share_stats()["accepted_share_count"]) - sql = """ -SELECT json_build_object( - 'blocks', (SELECT count(*) FROM qbit_pool_blocks), - 'confirmed_blocks', (SELECT count(*) FROM qbit_pool_blocks WHERE chain_state = 'confirmed'), - 'inactive_blocks', (SELECT count(*) FROM qbit_pool_blocks WHERE chain_state = 'inactive'), - 'rejected_blocks', (SELECT count(*) FROM qbit_pool_blocks WHERE chain_state = 'rejected'), - 'reversed_blocks', (SELECT count(*) FROM qbit_pool_blocks WHERE chain_state = 'reversed'), - 'payout_entries', (SELECT count(*) FROM qbit_pool_payout_entries), - 'owed_accounts', (SELECT count(*) FROM qbit_current_owed_balances() WHERE owed_balance_sats > 0), - 'ctv_fanouts_failed', (SELECT count(*) FROM qbit_ctv_fanout_artifacts WHERE settlement_status = 'failed') -); -""" - with self._operation_gate(self._lock, "writer lock"): - metrics = self._run_retry_safe_read_json(sql) - report = {str(key): int(value) for key, value in metrics.items()} - report["shares"] = accepted_share_count - return report - - def dashboard_pool_snapshot( - self, - *, - current_network_difficulty: int | str | Decimal, - generated_at: str, - ) -> dict[str, object]: - from lab.prism import public_api - - window_weight = _reward_window_weight(current_network_difficulty) - window_weight_sql = public_api.decimal_string(window_weight) - sql = f""" -WITH bounds AS ( - SELECT clock_timestamp() AS ended_at -), -latest_block_row AS ( - SELECT - block.block_hash, - block.block_height, - block.found_at, - block.payout_manifest_sha256 - FROM qbit_pool_blocks block - WHERE block.chain_state <> 'reversed' - ORDER BY block.block_height DESC, block.found_at DESC - LIMIT 1 -), -latest_block AS ( - SELECT - block.block_hash, - block.block_height, - block.found_at, - block.payout_manifest_sha256, - bundle.audit_bundle_sha256, - solver.miner_id AS solver_recipient_id, - solver.share_id AS solver_share_id - FROM latest_block_row block - LEFT JOIN qbit_pool_audit_bundles bundle - ON bundle.block_hash = block.block_hash - LEFT JOIN LATERAL ( - SELECT share.miner_id, share.share_id - FROM qbit_share_ledger share - WHERE share.accepted - AND length(share.share_id) >= 65 - AND lower(right(share.share_id, 64)) = block.block_hash - ORDER BY share.accepted_at DESC, share.share_seq DESC - LIMIT 1 - ) solver ON true -), -window_rows AS ( - SELECT window_row.* - FROM bounds - CROSS JOIN LATERAL qbit_prism_window(bounds.ended_at, {window_weight_sql}::numeric) AS window_row -), -window_summary AS ( - SELECT - count(*) AS included_share_count, - to_char(min(accepted_at) AT TIME ZONE 'UTC', 'YYYY-MM-DD"T"HH24:MI:SS"Z"') AS oldest_share_accepted_at, - to_char(max(accepted_at) AT TIME ZONE 'UTC', 'YYYY-MM-DD"T"HH24:MI:SS"Z"') AS newest_share_accepted_at - FROM window_rows -), -rollups AS ( - SELECT - COALESCE(sum(share_difficulty) FILTER (WHERE accepted_at >= bounds.ended_at - interval '1 hour'), 0)::text AS h1_difficulty, - COALESCE(sum(share_difficulty) FILTER (WHERE accepted_at >= bounds.ended_at - interval '3 hours'), 0)::text AS h3_difficulty, - COALESCE(sum(share_difficulty) FILTER (WHERE accepted_at >= bounds.ended_at - interval '24 hours'), 0)::text AS h24_difficulty, - count(DISTINCT miner_id) FILTER (WHERE accepted_at >= bounds.ended_at - interval '3 hours') AS participants_3h - FROM qbit_share_ledger, bounds - WHERE accepted - AND accepted_at >= bounds.ended_at - interval '24 hours' - AND accepted_at <= bounds.ended_at -) -SELECT json_build_object( - 'h1_difficulty', (SELECT h1_difficulty FROM rollups), - 'h3_difficulty', (SELECT h3_difficulty FROM rollups), - 'h24_difficulty', (SELECT h24_difficulty FROM rollups), - 'participants_3h', (SELECT participants_3h FROM rollups), - 'blocks_found_total', (SELECT count(*) FROM qbit_pool_blocks WHERE chain_state <> 'reversed'), - 'prism_blocks_total', (SELECT count(*) FROM qbit_pool_blocks WHERE chain_state <> 'reversed'), - 'blocks_reversed_total', (SELECT count(*) FROM qbit_pool_blocks WHERE chain_state = 'reversed'), - 'blocks_inactive_total', (SELECT count(*) FROM qbit_pool_blocks WHERE chain_state = 'inactive'), - 'total_mined_bits', COALESCE(( - SELECT sum(carry.gross_amount_sats) - FROM qbit_payout_carry_forward carry - JOIN qbit_pool_blocks block - ON block.block_hash = carry.block_hash - WHERE block.chain_state = 'confirmed' - AND block.maturity_state <> 'reversed' - AND carry.maturity_state <> 'reversed' - ), 0), - 'latest_block', COALESCE(( - SELECT json_build_object( - 'height', latest_block.block_height, - 'hash', latest_block.block_hash, - 'found_at', to_char(latest_block.found_at AT TIME ZONE 'UTC', 'YYYY-MM-DD"T"HH24:MI:SS"Z"'), - 'age_seconds', GREATEST(0, floor(extract(epoch FROM (clock_timestamp() - latest_block.found_at)))::bigint), - 'solver_recipient_id', COALESCE(latest_block.solver_recipient_id, ''), - 'solver_worker_name', {_solver_worker_name_sql("latest_block.solver_share_id")} - ) - FROM latest_block - ), 'null'::json), - 'oldest_share_accepted_at', (SELECT oldest_share_accepted_at FROM window_summary), - 'newest_share_accepted_at', (SELECT newest_share_accepted_at FROM window_summary), - 'included_share_count', (SELECT included_share_count FROM window_summary) -); -""" - row = self._run_read_json(sql) - return { - "hashrate_ths": { - "h1": public_api.hashrate_ths_from_difficulty(row["h1_difficulty"], 60 * 60), - "h3": public_api.hashrate_ths_from_difficulty(row["h3_difficulty"], 3 * 60 * 60), - "h24": public_api.hashrate_ths_from_difficulty(row["h24_difficulty"], 24 * 60 * 60), - }, - "participants_3h": int(row["participants_3h"]), - "blocks_found_total": int(row["blocks_found_total"]), - "prism_blocks_total": int(row["prism_blocks_total"]), - "blocks_reversed_total": int(row["blocks_reversed_total"]), - "blocks_inactive_total": int(row["blocks_inactive_total"]), - "total_mined_bits": int(row["total_mined_bits"]), - "latest_block": row["latest_block"], - "reward_window": { - "window_multiplier": 8, - "requested_window_weight": public_api.decimal_string(window_weight), - "oldest_share_accepted_at": row["oldest_share_accepted_at"], - "newest_share_accepted_at": row["newest_share_accepted_at"], - "included_share_count": int(row["included_share_count"]), - }, - } - - def dashboard_blocks(self, *, page: int, limit: int, chain_state: str = "active") -> dict[str, object]: - from lab.prism import public_api - - # The default filter is exactly the pre-filter read (hide reversed - # rows), and the default emits exactly the pre-filter SQL and row - # shape so the v1 response stays byte-compatible. The non-default - # filters additionally surface chain_state and disconnected_at, which - # the public v2 block row reports. - state_predicates = { - "active": "{column} <> 'reversed'", - "all": "true", - "reversed": "{column} = 'reversed'", - } - if chain_state not in state_predicates: - raise ValueError("chain_state must be one of active, all, reversed") - total_predicate = state_predicates[chain_state].format(column="chain_state") - page_predicate = state_predicates[chain_state].format(column="block.chain_state") - include_state = chain_state != "active" - state_columns = "\n block.chain_state,\n block.disconnected_at," if include_state else "" - # disconnected_at is a reorg disconnect time in the public contract, - # null for every non-reversed row. The ledger also stamps the column - # on rejected rows (their maturity_state 'reversed' requires it), so - # it is masked to reversed chain states rather than serialized as - # recorded. - state_json_fields = ( - "\n 'chain_state', rows.chain_state," - "\n 'disconnected_at', CASE WHEN rows.chain_state = 'reversed'" - " THEN to_char(rows.disconnected_at AT TIME ZONE 'UTC', 'YYYY-MM-DD\"T\"HH24:MI:SS\"Z\"') END," - if include_state - else "" - ) - offset = (page - 1) * limit - explorer_prefix = os.environ.get("PRISM_PUBLIC_EXPLORER_BLOCK_URL_PREFIX") - sql = f""" -WITH total AS ( - SELECT count(*) AS total_count - FROM qbit_pool_blocks - WHERE {total_predicate} -), -page_blocks AS ( - SELECT - block.block_hash, - block.block_height, - block.found_at,{state_columns} - block.payout_manifest_sha256 - FROM qbit_pool_blocks block - WHERE {page_predicate} - ORDER BY block.block_height DESC, block.found_at DESC - LIMIT {int(limit)} OFFSET {int(offset)} -), -rows AS ( - SELECT - block.block_hash, - block.block_height, - block.found_at,{state_columns} - block.payout_manifest_sha256, - COALESCE(bundle.found_block_network_difficulty::text, bundle.audit_bundle#>>'{{found_block,network_difficulty}}') AS audit_network_difficulty, - COALESCE(bundle.found_block_bits, bundle.audit_bundle#>>'{{found_block,bits}}') AS audit_bits, - COALESCE(bundle.found_block_coinbase_value_sats::text, bundle.audit_bundle#>>'{{found_block,coinbase_value_sats}}') AS audit_coinbase_value_sats, - bundle.audit_bundle_sha256, - solver.miner_id AS solver_recipient_id, - solver.share_difficulty::text AS solver_share_difficulty, - solver.network_difficulty::text AS solver_network_difficulty, - solver.share_id AS solver_share_id - FROM page_blocks block - LEFT JOIN qbit_pool_audit_bundles bundle - ON bundle.block_hash = block.block_hash - LEFT JOIN LATERAL ( - SELECT share.miner_id, share.share_difficulty, share.network_difficulty, share.share_id - FROM qbit_share_ledger share - WHERE share.accepted - AND length(share.share_id) >= 65 - AND lower(right(share.share_id, 64)) = block.block_hash - ORDER BY share.accepted_at DESC, share.share_seq DESC - LIMIT 1 - ) solver ON true -) -SELECT json_build_object( - 'total_count', (SELECT total_count FROM total), - 'rows', COALESCE(( - SELECT json_agg(json_build_object( - 'height', rows.block_height, - 'hash', rows.block_hash, - 'found_at', to_char(rows.found_at AT TIME ZONE 'UTC', 'YYYY-MM-DD"T"HH24:MI:SS"Z"'), - 'network_difficulty', COALESCE(rows.audit_network_difficulty, rows.solver_network_difficulty, '0'), - 'bits', COALESCE(rows.audit_bits, '00000000'), - 'solver_recipient_id', COALESCE(rows.solver_recipient_id, ''), - 'solver_worker_name', {_solver_worker_name_sql("rows.solver_share_id")}, - 'solver_share_difficulty', rows.solver_share_difficulty, - 'reward_window_weight', CASE - WHEN rows.audit_network_difficulty IS NULL THEN null - ELSE (rows.audit_network_difficulty::numeric * 8::numeric)::text - END, - 'coinbase_value_bits', COALESCE(rows.audit_coinbase_value_sats::bigint, 0), - 'audit_bundle_sha256', rows.audit_bundle_sha256, - 'payout_manifest_sha256', rows.payout_manifest_sha256,{state_json_fields} - 'explorer_url', null - ) ORDER BY rows.block_height DESC, rows.found_at DESC) - FROM rows - ), '[]'::json) -); -""" - payload = self._run_read_json(sql) - rows = payload["rows"] - if explorer_prefix: - for row in rows: - row["explorer_url"] = explorer_prefix.rstrip("/") + "/" + str(row["hash"]) - return { - "pagination": public_api.pagination(page, limit, int(payload["total_count"])), - "rows": rows, - } - - def dashboard_leaderboard(self, *, page: int, limit: int, search: str | None = None) -> dict[str, object]: - from lab.prism import public_api - - offset = (page - 1) * limit - search_filter = "" - if search: - search_filter = f"WHERE strpos(lower(miner_id), {self._text_literal(search.lower())}) > 0" - sql = f""" - WITH snapshot_clock AS ( - SELECT clock_timestamp() AS ended_at - ), - bounds AS ( - SELECT ended_at, ended_at - interval '3 hours' AS started_at - FROM snapshot_clock - ), - windowed AS ( - SELECT ledger.* - FROM qbit_share_ledger ledger, bounds - WHERE ledger.accepted - AND ledger.accepted_at >= bounds.started_at - AND ledger.accepted_at <= bounds.ended_at - ), -grouped AS ( - SELECT - miner_id, - sum(share_difficulty) AS accepted_share_difficulty, - max(accepted_at) AS last_share_at - FROM windowed - GROUP BY miner_id -), -filtered AS ( - SELECT * - FROM grouped - {search_filter} -), -blocks AS ( - SELECT solver.miner_id, count(*) AS blocks_found - FROM qbit_pool_blocks block - JOIN LATERAL ( - SELECT share.miner_id - FROM qbit_share_ledger share - WHERE share.accepted - AND length(share.share_id) >= 65 - AND lower(right(share.share_id, 64)) = block.block_hash - ORDER BY share.accepted_at DESC, share.share_seq DESC - LIMIT 1 - ) solver ON true - WHERE block.chain_state <> 'reversed' - GROUP BY solver.miner_id -), -totals AS ( - SELECT - COALESCE(sum(accepted_share_difficulty), 0) AS total_difficulty, - count(*) AS participant_count - FROM filtered -), -ranked AS ( - SELECT - row_number() OVER (ORDER BY accepted_share_difficulty DESC, filtered.miner_id ASC) AS rank, - filtered.miner_id, - filtered.accepted_share_difficulty, - filtered.last_share_at, - COALESCE(blocks.blocks_found, 0) AS blocks_found - FROM filtered - LEFT JOIN blocks - ON blocks.miner_id = filtered.miner_id -), -page_rows AS ( - SELECT * - FROM ranked - ORDER BY rank ASC - LIMIT {int(limit)} OFFSET {int(offset)} -) -SELECT json_build_object( - 'started_at', (SELECT to_char(started_at AT TIME ZONE 'UTC', 'YYYY-MM-DD"T"HH24:MI:SS"Z"') FROM bounds), - 'ended_at', (SELECT to_char(ended_at AT TIME ZONE 'UTC', 'YYYY-MM-DD"T"HH24:MI:SS"Z"') FROM bounds), - 'total_difficulty', (SELECT total_difficulty::text FROM totals), - 'participant_count', (SELECT participant_count FROM totals), - 'rows', COALESCE(( - SELECT json_agg(json_build_object( - 'rank', page_rows.rank, - 'recipient_id', page_rows.miner_id, - 'display_name', null, - 'accepted_share_difficulty', page_rows.accepted_share_difficulty::text, - 'share_percent', CASE - WHEN (SELECT total_difficulty FROM totals) > 0 THEN - (page_rows.accepted_share_difficulty * 100::numeric / (SELECT total_difficulty FROM totals))::text - ELSE null - END, - 'blocks_found', page_rows.blocks_found, - 'last_share_at', to_char(page_rows.last_share_at AT TIME ZONE 'UTC', 'YYYY-MM-DD"T"HH24:MI:SS"Z"') - ) ORDER BY page_rows.rank ASC) - FROM page_rows - ), '[]'::json) -); -""" - payload = self._run_read_json(sql) - rows: list[dict[str, object]] = [] - total_difficulty = str(payload["total_difficulty"]) - pool_hashrate_ths = public_api.hashrate_ths_from_difficulty(total_difficulty, 3 * 60 * 60) - for row in payload["rows"]: - share_percent = row["share_percent"] - hashrate_ths = public_api.hashrate_ths_from_difficulty( - row["accepted_share_difficulty"], - 3 * 60 * 60, - ) - rows.append( - { - "rank": int(row["rank"]), - "recipient_id": row["recipient_id"], - "display_name": row["display_name"], - "hashrate_ths_3h": hashrate_ths, - "share_percent": public_api.decimal_string(share_percent) if share_percent is not None else None, - "hash_percent": _hash_percent(hashrate_ths, pool_hashrate_ths), - "blocks_found": int(row["blocks_found"]), - "last_share_at": row["last_share_at"], - } - ) - participant_count = int(payload["participant_count"]) - return { - "started_at": payload["started_at"], - "ended_at": payload["ended_at"], - "totals": { - "pool_hashrate_ths": pool_hashrate_ths, - "pool_accepted_share_difficulty": total_difficulty, - "participant_count": participant_count, - }, - "pagination": public_api.pagination(page, limit, participant_count), - "rows": rows, - } - - def dashboard_reward_leaderboard( - self, - *, - page: int, - limit: int, - current_network_difficulty: int | str | Decimal, - search: str | None = None, - recipient_id: str | None = None, - ) -> dict[str, object]: - from lab.prism import public_api - - if search is not None and recipient_id is not None: - raise ValueError("search and recipient_id are mutually exclusive") - offset = (page - 1) * limit - network_difficulty = public_api.decimal_string(current_network_difficulty) - requested_window_weight = _reward_window_weight(current_network_difficulty) - requested_window_weight_sql = public_api.decimal_string(requested_window_weight) - row_filter = "" - if recipient_id is not None: - row_filter = f"WHERE ranked.miner_id = {self._text_literal(recipient_id)}" - elif search: - row_filter = ( - "WHERE strpos(lower(ranked.miner_id), " - f"{self._text_literal(search.lower())}) > 0" - ) - sql = f""" -WITH snapshot_clock AS ( - SELECT clock_timestamp() AS ended_at -), -window_rows AS MATERIALIZED ( - SELECT window_row.* - FROM snapshot_clock - CROSS JOIN LATERAL qbit_prism_window( - snapshot_clock.ended_at, - {requested_window_weight_sql}::numeric - ) AS window_row -), -window_summary AS ( - SELECT - count(*) AS included_share_count, - COALESCE(sum(counted_difficulty), 0) AS counted_window_weight, - min(accepted_at) AS oldest_share_accepted_at - FROM window_rows -), -grouped AS ( - SELECT - miner_id, - count(*) AS included_share_count, - sum(counted_difficulty) AS counted_share_difficulty, - max(accepted_at) AS last_share_at - FROM window_rows - GROUP BY miner_id -), -blocks AS ( - SELECT solver.miner_id, count(*) AS blocks_found_total - FROM qbit_pool_blocks block - JOIN LATERAL ( - SELECT share.miner_id - FROM qbit_share_ledger share - WHERE share.accepted - AND length(share.share_id) >= 65 - AND lower(right(share.share_id, 64)) = block.block_hash - ORDER BY share.accepted_at DESC, share.share_seq DESC - LIMIT 1 - ) solver ON true - WHERE block.chain_state <> 'reversed' - GROUP BY solver.miner_id -), -ranked AS ( - SELECT - row_number() OVER (ORDER BY grouped.counted_share_difficulty DESC, grouped.miner_id ASC) AS rank, - grouped.miner_id, - grouped.included_share_count, - grouped.counted_share_difficulty, - grouped.last_share_at, - COALESCE(blocks.blocks_found_total, 0) AS blocks_found_total - FROM grouped - LEFT JOIN blocks - ON blocks.miner_id = grouped.miner_id -), -filtered AS ( - SELECT * - FROM ranked - {row_filter} -), -page_rows AS ( - SELECT * - FROM filtered - ORDER BY rank ASC - LIMIT {int(limit)} OFFSET {int(offset)} -) -SELECT json_build_object( - 'ended_at', to_char(snapshot_clock.ended_at AT TIME ZONE 'UTC', 'YYYY-MM-DD"T"HH24:MI:SS"Z"'), - 'oldest_share_accepted_at', to_char(window_summary.oldest_share_accepted_at AT TIME ZONE 'UTC', 'YYYY-MM-DD"T"HH24:MI:SS"Z"'), - 'observed_span_seconds', CASE - WHEN window_summary.oldest_share_accepted_at IS NULL THEN null - ELSE GREATEST( - 0, - floor(extract(epoch FROM (snapshot_clock.ended_at - window_summary.oldest_share_accepted_at)))::bigint - ) - END, - 'counted_window_weight', window_summary.counted_window_weight::text, - 'included_share_count', window_summary.included_share_count, - 'participant_count', (SELECT count(*) FROM ranked), - 'total_count', (SELECT count(*) FROM filtered), - 'rows', COALESCE(( - SELECT json_agg(json_build_object( - 'rank', page_rows.rank, - 'recipient_id', page_rows.miner_id, - 'display_name', null, - 'included_share_count', page_rows.included_share_count, - 'counted_share_difficulty', page_rows.counted_share_difficulty::text, - 'share_percent', CASE - WHEN window_summary.counted_window_weight > 0 THEN - (page_rows.counted_share_difficulty * 100::numeric / window_summary.counted_window_weight)::text - ELSE null - END, - 'blocks_found_total', page_rows.blocks_found_total, - 'last_share_at', to_char(page_rows.last_share_at AT TIME ZONE 'UTC', 'YYYY-MM-DD"T"HH24:MI:SS"Z"') - ) ORDER BY page_rows.rank ASC) - FROM page_rows - ), '[]'::json) -) -FROM snapshot_clock -CROSS JOIN window_summary; -""" - payload = self._run_read_json(sql) - counted_window_weight = Decimal(str(payload["counted_window_weight"])) - observed_span_seconds = ( - int(payload["observed_span_seconds"]) - if payload.get("observed_span_seconds") is not None - else None - ) - pool_hashrate_ths = None - expected_time_to_block = None - if observed_span_seconds is not None and observed_span_seconds > 0 and counted_window_weight > 0: - pool_hashrate_ths = public_api.hashrate_ths_from_difficulty( - counted_window_weight, - observed_span_seconds, - ) - expected_time_to_block = public_api.expected_time_to_block_seconds( - hashrate_ths=pool_hashrate_ths, - network_difficulty=network_difficulty, - ) - - rows: list[dict[str, object]] = [] - for row in payload["rows"]: - counted_share_difficulty = public_api.decimal_string(row["counted_share_difficulty"]) - hashrate_ths = None - if observed_span_seconds is not None and observed_span_seconds > 0: - hashrate_ths = public_api.hashrate_ths_from_difficulty( - counted_share_difficulty, - observed_span_seconds, - ) - share_percent = row["share_percent"] - rows.append( - { - "rank": int(row["rank"]), - "recipient_id": row["recipient_id"], - "display_name": row["display_name"], - "hashrate_ths": hashrate_ths, - "included_share_count": int(row["included_share_count"]), - "counted_share_difficulty": counted_share_difficulty, - "share_percent": public_api.decimal_string(share_percent) if share_percent is not None else None, - "blocks_found_total": int(row["blocks_found_total"]), - "last_share_at": row["last_share_at"], - } - ) - - participant_count = int(payload["participant_count"]) - return { - "window": { - "id": "reward", - "started_at": payload["oldest_share_accepted_at"], - "ended_at": payload["ended_at"], - "observed_span_seconds": observed_span_seconds, - "network_difficulty": network_difficulty, - "window_multiplier": 8, - "requested_window_weight": requested_window_weight_sql, - "counted_window_weight": public_api.decimal_string(counted_window_weight), - "included_share_count": int(payload["included_share_count"]), - "is_complete": requested_window_weight > 0 and counted_window_weight >= requested_window_weight, - }, - "totals": { - "pool_hashrate_ths": pool_hashrate_ths, - "pool_counted_share_difficulty": public_api.decimal_string(counted_window_weight), - "participant_count": participant_count, - "expected_time_to_block_seconds": expected_time_to_block, - }, - "pagination": public_api.pagination(page, limit, int(payload["total_count"])), - "rows": rows, - } - - def dashboard_hashrate_series( - self, - *, - subject_type: str, - subject_id: str | None, - range_id: str, - bucket: str, - lookback_seconds: int = 0, - range_anchor_epoch: int | None = None, - ) -> list[dict[str, object]]: - """Serve the per-bucket credited hashrate series in O(buckets). - - The series is answered from the incremental rollup tables plus a - live tail over the shares the maintenance watermark has not folded - in yet, so its cost tracks the number of buckets in range rather - than the number of shares. The rollup path must stay byte-identical - to the historical raw ``qbit_share_ledger`` aggregation for the same - underlying shares -- same buckets, counts, difficulty strings, and - ordering -- which pins three invariants: - - - The watermark, the rollup rows, and the tail scan are read inside - one statement, so they share one snapshot. Reading the watermark - in a separate statement would let a concurrent maintenance pass - advance it in between and double-count the shares it folded in. - - A range lower bound that is not bucket-aligned makes the raw scan - emit a partial leading bucket (only the shares at or after the - bound), and the bucket containing the statement's upper bound can - hold a rollup-folded share whose coordinator-assigned accepted_at - runs ahead of the database clock -- a share the raw scan's - ``accepted_at <= ended_at`` excludes until the clocks catch up. - Neither partial end can be served from full rollup buckets, so - both are re-aggregated from the ledger over the already-rolled-up - sequence range; each scan is bounded by one bucket span of shares - regardless of the requested range. - - An absent progress row gates the rollup and boundary branches off - and widens the tail to the whole ledger (``share_seq > -1``), so - the merged output degrades to exactly the raw scan; a database - missing the rollup tables entirely runs the raw statement - unchanged. Both keep pre-migration and mid-backfill states - correct without operator action. - """ - from lab.prism import public_api - - bucket_seconds = {"5m": 300, "1h": 3600, "1d": 86400}[bucket] - range_interval = { - "1w": "7 days", - "1m": "30 days", - "6m": "180 days", - "window": "3 hours", - "all": None, - }[range_id] - range_filter = "" - range_start_sql: str | None = None - if range_interval is not None: - # Anchor the range lower bound on the caller's clock when provided - # so it agrees with the caller's min_epoch trim even when the - # database clock disagrees with the application clock. - range_anchor_sql = ( - f"to_timestamp({int(range_anchor_epoch)})" - if range_anchor_epoch is not None - else "bounds.ended_at" - ) - range_start_sql = f"{range_anchor_sql} - interval '{range_interval}'" - if lookback_seconds > 0: - # Pre-range context requested by the smoother; the caller trims - # these buckets from the response after windowing. - range_start_sql += f" - interval '{int(lookback_seconds)} seconds'" - range_filter = f"AND ledger.accepted_at >= {range_start_sql}" - subject_filter = "" - if subject_type == "miner": - subject_filter = f"AND ledger.miner_id = {self._text_literal(str(subject_id))}" - if not self._hashrate_rollup_schema_present(): - sql = f""" - WITH bounds AS ( - SELECT clock_timestamp() AS ended_at - ), - bucketed AS ( - SELECT - floor(extract(epoch FROM ledger.accepted_at) / {int(bucket_seconds)})::bigint * {int(bucket_seconds)} AS bucket_epoch, - count(*) AS accepted_share_count, - sum(ledger.share_difficulty) AS accepted_share_difficulty - FROM qbit_share_ledger ledger, bounds - WHERE ledger.accepted - AND ledger.accepted_at <= bounds.ended_at - {range_filter} - {subject_filter} - GROUP BY bucket_epoch - ) -SELECT COALESCE(json_agg(json_build_object( - 'timestamp', to_char(to_timestamp(bucket_epoch) AT TIME ZONE 'UTC', 'YYYY-MM-DD"T"HH24:MI:SS"Z"'), - 'accepted_share_count', accepted_share_count, - 'accepted_share_difficulty', accepted_share_difficulty::text -) ORDER BY bucket_epoch ASC), '[]'::json) -FROM bucketed; -""" - rows = self._run_read_json(sql) - return [ - { - "timestamp": row["timestamp"], - "hashrate_ths": public_api.hashrate_ths_from_difficulty(row["accepted_share_difficulty"], bucket_seconds), - "accepted_share_count": int(row["accepted_share_count"]), - "accepted_share_difficulty": str(row["accepted_share_difficulty"]), - } - for row in rows - ] - if subject_type == "miner": - rollup_table = "qbit_hashrate_rollup_miner" - rollup_subject_filter = ( - f"\n AND rollup.miner_id = {self._text_literal(str(subject_id))}" - ) - else: - rollup_table = "qbit_hashrate_rollup_pool" - rollup_subject_filter = "" - # The bucket containing bounds.ended_at is never served from its - # rollup row: maintenance folds by sequence, so a coordinator clock - # running ahead of the database clock can put a share into that - # bucket's rollup that the raw scan's accepted_at <= ended_at would - # exclude. Like the partial lower-bound bucket, it is re-aggregated - # from the already-rolled-up sequence range -- one bucket span of - # shares, whatever the requested range. - if range_start_sql is not None: - range_buckets_cte = f"""range_buckets AS ( - SELECT - range_bounds.range_started_at, - (ceil(extract(epoch FROM range_bounds.range_started_at) / {int(bucket_seconds)}))::bigint * {int(bucket_seconds)} AS first_full_bucket_epoch - FROM (SELECT {range_start_sql} AS range_started_at FROM bounds) range_bounds -), -""" - rollup_lower_bound = ( - "\n AND rollup.bucket_epoch >= (SELECT first_full_bucket_epoch FROM range_buckets)" - ) - boundary_window = """( - ledger.accepted_at < to_timestamp((SELECT first_full_bucket_epoch FROM range_buckets)) - OR ledger.accepted_at >= to_timestamp((SELECT bucket_epoch FROM current_bucket)) - )""" - boundary_range_filter = ( - "\n AND ledger.accepted_at >= (SELECT range_started_at FROM range_buckets)" - ) - tail_range_filter = ( - "\n AND ledger.accepted_at >= (SELECT range_started_at FROM range_buckets)" - ) - else: - range_buckets_cte = "" - rollup_lower_bound = "" - boundary_window = ( - "ledger.accepted_at >= to_timestamp((SELECT bucket_epoch FROM current_bucket))" - ) - boundary_range_filter = "" - tail_range_filter = "" - boundary_cte = f"""boundary AS ( - SELECT - floor(extract(epoch FROM ledger.accepted_at) / {int(bucket_seconds)})::bigint * {int(bucket_seconds)} AS bucket_epoch, - count(*) AS accepted_share_count, - sum(ledger.share_difficulty) AS accepted_share_difficulty - FROM qbit_share_ledger ledger, bounds - WHERE (SELECT rollups_ready FROM watermark) - AND ledger.accepted - AND ledger.share_seq <= (SELECT last_share_seq FROM watermark) - AND ledger.accepted_at <= bounds.ended_at - AND {boundary_window}{boundary_range_filter} - {subject_filter} - GROUP BY bucket_epoch -), -""" - boundary_union = """ UNION ALL - SELECT bucket_epoch, accepted_share_count, accepted_share_difficulty FROM boundary -""" - sql = f""" -WITH bounds AS ( - SELECT clock_timestamp() AS ended_at -), -progress AS ( - SELECT last_share_seq - FROM qbit_hashrate_rollup_progress - WHERE singleton -), -watermark AS ( - SELECT - COALESCE((SELECT last_share_seq FROM progress), -1) AS last_share_seq, - EXISTS (SELECT 1 FROM progress) AS rollups_ready -), -current_bucket AS ( - SELECT floor(extract(epoch FROM bounds.ended_at) / {int(bucket_seconds)})::bigint * {int(bucket_seconds)} AS bucket_epoch - FROM bounds -), -{range_buckets_cte}rolled AS ( - SELECT - rollup.bucket_epoch, - rollup.accepted_share_count, - rollup.accepted_share_difficulty - FROM {rollup_table} rollup, bounds - WHERE (SELECT rollups_ready FROM watermark) - AND rollup.grain_seconds = {int(bucket_seconds)} - AND rollup.bucket_epoch < (SELECT bucket_epoch FROM current_bucket){rollup_lower_bound}{rollup_subject_filter} -), -{boundary_cte}tail AS ( - SELECT - floor(extract(epoch FROM ledger.accepted_at) / {int(bucket_seconds)})::bigint * {int(bucket_seconds)} AS bucket_epoch, - count(*) AS accepted_share_count, - sum(ledger.share_difficulty) AS accepted_share_difficulty - FROM qbit_share_ledger ledger, bounds - WHERE ledger.accepted - AND ledger.share_seq > (SELECT last_share_seq FROM watermark) - AND ledger.accepted_at <= bounds.ended_at{tail_range_filter} - {subject_filter} - GROUP BY bucket_epoch -), -merged AS ( - SELECT - parts.bucket_epoch, - sum(parts.accepted_share_count)::bigint AS accepted_share_count, - sum(parts.accepted_share_difficulty) AS accepted_share_difficulty - FROM ( - SELECT bucket_epoch, accepted_share_count, accepted_share_difficulty FROM rolled -{boundary_union} UNION ALL - SELECT bucket_epoch, accepted_share_count, accepted_share_difficulty FROM tail - ) parts - GROUP BY parts.bucket_epoch -) -SELECT COALESCE(json_agg(json_build_object( - 'timestamp', to_char(to_timestamp(bucket_epoch) AT TIME ZONE 'UTC', 'YYYY-MM-DD"T"HH24:MI:SS"Z"'), - 'accepted_share_count', accepted_share_count, - 'accepted_share_difficulty', accepted_share_difficulty::text -) ORDER BY bucket_epoch ASC), '[]'::json) -FROM merged; -""" - rows = self._run_read_json(sql) - return [ - { - "timestamp": row["timestamp"], - "hashrate_ths": public_api.hashrate_ths_from_difficulty(row["accepted_share_difficulty"], bucket_seconds), - "accepted_share_count": int(row["accepted_share_count"]), - "accepted_share_difficulty": str(row["accepted_share_difficulty"]), - } - for row in rows - ] - - def _hashrate_rollup_schema_present(self) -> bool: - """Report whether the incremental hashrate rollup tables exist. - - The serving statement cannot even parse against a database that - predates the rollup DDL, so table existence has to be settled before - choosing a statement. A positive answer is cached for the life of - this instance: the rollup tables are only ever created (by the - writer's idempotent schema apply), never dropped. A negative answer - is deliberately not cached, so a read tier pointed at a database - whose writer applies the DDL mid-flight picks up the rollup path - without a restart. - """ - if getattr(self, "_hashrate_rollup_schema_ready", False): - return True - row = self._run_read_json( - """ -SELECT json_build_object( - 'rollup_schema_ready', - to_regclass('qbit_hashrate_rollup_progress') IS NOT NULL - AND to_regclass('qbit_hashrate_rollup_pool') IS NOT NULL - AND to_regclass('qbit_hashrate_rollup_miner') IS NOT NULL -); -""" - ) - ready = bool(row["rollup_schema_ready"]) - if ready: - self._hashrate_rollup_schema_ready = True - return ready - - def advance_hashrate_rollups(self, *, batch_limit: int) -> dict[str, object]: - """Fold the next watermarked share batch into the hashrate rollups. - - One statement, one transaction. ``qbit_share_ledger`` rows are - immutable and ``share_seq`` is append-only, so scanning strictly - above the stored watermark in sequence order folds every share into - its (grain, bucket) rows exactly once -- a late-clocked share still - lands in its correct ``accepted_at`` bucket, and no re-aggregation - window is needed. Rejected rows are read only to advance the - watermark. The first pass seeds the progress row itself and starts - from sequence 0, which is also how a grown ledger backfills. - - The watermark advance is a guarded upsert: it only applies while the - progress row still holds the value this statement read, and the - rollup upserts are gated on that advance having won. A concurrent - advance therefore leaves this pass writing nothing at all -- the - guarded upsert matches no row and both rollup inserts see an empty - gate -- and the caller raises instead of double-counting, because - two live maintenance passes mean the single-writer invariant is - already broken. - - The whole pass is additionally fenced on the writer lease, exactly - like every other mutation of ledger-derived state: the statement - renews the exact ``(writer_id, writer_epoch, writer_session_token)`` - lease tuple and the advance applies only when that renewal matched, - so a coordinator whose lease expired or was taken over cannot keep - mutating the rollups on the strength of its process-local lock - alone. A fenced-out pass writes nothing and raises. - """ - batch_limit = int(batch_limit) - if batch_limit <= 0: - raise ValueError("hashrate rollup batch limit must be positive") - sql = f""" -WITH lease AS ( - UPDATE qbit_ledger_writer_lease - SET lease_expires_at = clock_timestamp() + {self._lease_interval_sql}, - updated_at = clock_timestamp() - WHERE singleton - AND writer_id = {self._text_literal(self._writer_id)} - AND writer_epoch = {int(self._writer_epoch)} - AND writer_session_token = {self._text_literal(self._writer_session_token)} - RETURNING writer_id -), -progress AS ( - SELECT COALESCE(( - SELECT last_share_seq - FROM qbit_hashrate_rollup_progress - WHERE singleton - ), 0) AS last_share_seq -), -batch AS ( - SELECT - ledger.share_seq, - ledger.accepted, - ledger.accepted_at, - ledger.miner_id, - ledger.share_difficulty - FROM qbit_share_ledger ledger - WHERE ledger.share_seq > (SELECT last_share_seq FROM progress) - ORDER BY ledger.share_seq ASC - LIMIT {batch_limit} -), -batch_stats AS ( - SELECT - count(*) AS scanned, - COALESCE(max(batch.share_seq), (SELECT last_share_seq FROM progress)) AS next_share_seq - FROM batch -), -advance AS ( - INSERT INTO qbit_hashrate_rollup_progress (singleton, last_share_seq) - SELECT true, (SELECT next_share_seq FROM batch_stats) - WHERE EXISTS (SELECT 1 FROM lease) - ON CONFLICT (singleton) DO UPDATE - SET last_share_seq = EXCLUDED.last_share_seq, - updated_at = clock_timestamp() - WHERE qbit_hashrate_rollup_progress.last_share_seq = (SELECT last_share_seq FROM progress) - RETURNING last_share_seq -), -grains AS ( - SELECT grain_seconds - FROM (VALUES (300), (3600), (86400)) AS grain(grain_seconds) -), -pool_rollup AS ( - INSERT INTO qbit_hashrate_rollup_pool ( - grain_seconds, - bucket_epoch, - accepted_share_count, - accepted_share_difficulty - ) - SELECT - grains.grain_seconds, - floor(extract(epoch FROM batch.accepted_at) / grains.grain_seconds)::bigint * grains.grain_seconds AS bucket_epoch, - count(*) AS accepted_share_count, - sum(batch.share_difficulty) AS accepted_share_difficulty - FROM batch, grains - WHERE batch.accepted - AND EXISTS (SELECT 1 FROM advance) - GROUP BY grains.grain_seconds, bucket_epoch - ON CONFLICT (grain_seconds, bucket_epoch) DO UPDATE - SET accepted_share_count = qbit_hashrate_rollup_pool.accepted_share_count - + EXCLUDED.accepted_share_count, - accepted_share_difficulty = qbit_hashrate_rollup_pool.accepted_share_difficulty - + EXCLUDED.accepted_share_difficulty - RETURNING 1 -), -miner_rollup AS ( - INSERT INTO qbit_hashrate_rollup_miner ( - grain_seconds, - bucket_epoch, - miner_id, - accepted_share_count, - accepted_share_difficulty - ) - SELECT - grains.grain_seconds, - floor(extract(epoch FROM batch.accepted_at) / grains.grain_seconds)::bigint * grains.grain_seconds AS bucket_epoch, - batch.miner_id, - count(*) AS accepted_share_count, - sum(batch.share_difficulty) AS accepted_share_difficulty - FROM batch, grains - WHERE batch.accepted - AND EXISTS (SELECT 1 FROM advance) - GROUP BY grains.grain_seconds, bucket_epoch, batch.miner_id - ON CONFLICT (grain_seconds, bucket_epoch, miner_id) DO UPDATE - SET accepted_share_count = qbit_hashrate_rollup_miner.accepted_share_count - + EXCLUDED.accepted_share_count, - accepted_share_difficulty = qbit_hashrate_rollup_miner.accepted_share_difficulty - + EXCLUDED.accepted_share_difficulty - RETURNING 1 -) -SELECT CASE - WHEN NOT EXISTS (SELECT 1 FROM lease) THEN - json_build_object('error', 'writer lease is not active') - ELSE - json_build_object( - 'scanned', (SELECT scanned FROM batch_stats), - 'last_share_seq', (SELECT next_share_seq FROM batch_stats), - 'advanced', (SELECT count(*) FROM advance), - 'caught_up', (SELECT scanned FROM batch_stats) < {batch_limit} - ) -END; -""" - result = self._run_fenced_json(sql) - if "error" in result: - raise RuntimeError(str(result["error"])) - if int(result["advanced"]) != 1: - raise RuntimeError( - "hashrate rollup watermark advanced concurrently; " - "this pass wrote nothing (writer-fencing bug)" - ) - return { - "scanned": int(result["scanned"]), - "last_share_seq": int(result["last_share_seq"]), - "caught_up": bool(result["caught_up"]), - } - - def dashboard_block_markers( - self, - *, - range_id: str, - bucket: str, - range_anchor_epoch: int | None = None, - ) -> dict[str, object]: - from lab.prism import public_api - - bucket_seconds = {"5m": 300, "1h": 3600, "1d": 86400}[bucket] - range_seconds = { - "1w": 7 * 86400, - "1m": 30 * 86400, - "6m": 180 * 86400, - "all": None, - }[range_id] - range_filter = "" - if range_seconds is not None: - # Anchor the range lower bound on the caller's clock when provided - # so marker buckets sit on exactly the grid the hashrate series is - # trimmed against, even when the database clock disagrees with the - # application clock. Only fully covered buckets are kept: a bucket - # straddling the range start is dropped whole, the same boundary - # hashrate_series_min_epoch draws for the chart's points. `all` - # carries no lower bound. - anchor_epoch = ( - int(range_anchor_epoch) - if range_anchor_epoch is not None - else int(datetime.now(timezone.utc).timestamp()) - ) - min_epoch = public_api.hashrate_series_min_epoch( - anchor_epoch, - range_seconds, - bucket_seconds, - ) - range_filter = f"AND block.found_at >= to_timestamp({int(min_epoch)})" - # qbit_pool_blocks holds ~10^4 rows, so one grouped scan is cheap and - # no index beyond the primary key is needed. Reversed blocks are - # excluded by the same predicate dashboard_blocks applies. - sql = f""" -WITH in_range AS ( - SELECT - floor(extract(epoch FROM block.found_at) / {int(bucket_seconds)})::bigint * {int(bucket_seconds)} AS bucket_epoch, - block.block_hash, - block.block_height, - block.found_at - FROM qbit_pool_blocks block - WHERE block.chain_state <> 'reversed' - AND block.found_at <= clock_timestamp() - {range_filter} -), -ranked AS ( - SELECT - in_range.*, - row_number() OVER ( - PARTITION BY bucket_epoch - ORDER BY found_at DESC, block_height DESC - ) AS bucket_rank - FROM in_range -), -points AS ( - SELECT - bucket_epoch, - count(*) AS block_count, - json_agg(json_build_object( - 'height', block_height, - 'hash', block_hash, - 'found_at', to_char(found_at AT TIME ZONE 'UTC', 'YYYY-MM-DD"T"HH24:MI:SS"Z"') - ) ORDER BY found_at DESC, block_height DESC) - FILTER (WHERE bucket_rank <= {int(public_api.BLOCK_MARKERS_MAX_BLOCKS_PER_BUCKET)}) AS blocks - FROM ranked - GROUP BY bucket_epoch -) -SELECT json_build_object( - 'total_blocks', COALESCE((SELECT sum(block_count)::bigint FROM points), 0), - 'points', COALESCE(( - SELECT json_agg(json_build_object( - 'timestamp', to_char(to_timestamp(bucket_epoch) AT TIME ZONE 'UTC', 'YYYY-MM-DD"T"HH24:MI:SS"Z"'), - 'block_count', block_count, - 'blocks', blocks - ) ORDER BY bucket_epoch ASC) - FROM points - ), '[]'::json) -); -""" - payload = self._run_read_json(sql) - return { - "total_blocks": int(payload["total_blocks"]), - "points": payload["points"], - } - - def _audit_store(self) -> AuditArtifactStore: - store = getattr(self, "_audit_artifact_store", None) - if store is None: - raise RuntimeError("audit body store is not configured") - return store - - def _stored_canonical_audit_bundle_bytes( - self, - block_hash: str, - audit_bundle_sha256: str, - ) -> bytes | None: - """Return a verified stored canonical bundle, or permit missing fallback. - - Missing artifacts are expected for history predating issue #158. - Present-but-corrupt or unreadable artifacts fail closed instead of - silently serving a non-byte-exact legacy reconstruction. - """ - store = getattr(self, "_audit_artifact_store", None) - reader = getattr(store, "read_canonical_audit_bundle", None) - if not callable(reader): - LOGGER.warning( - "canonical audit bundle unavailable reason=missing " - "block_hash=%s audit_bundle_sha256=%s", - block_hash, - audit_bundle_sha256, - ) - return None - try: - canonical_bytes = reader(block_hash, audit_bundle_sha256) - except CanonicalAuditBundleCorrupt: - LOGGER.error( - "canonical audit bundle unavailable reason=corrupt " - "block_hash=%s audit_bundle_sha256=%s", - block_hash, - audit_bundle_sha256, - ) - raise - except (OSError, RuntimeError, TypeError, ValueError): - LOGGER.exception( - "canonical audit bundle unavailable reason=read_error " - "block_hash=%s audit_bundle_sha256=%s", - block_hash, - audit_bundle_sha256, - ) - raise - if canonical_bytes is None: - LOGGER.warning( - "canonical audit bundle unavailable reason=missing " - "block_hash=%s audit_bundle_sha256=%s", - block_hash, - audit_bundle_sha256, - ) - return None - if not isinstance(canonical_bytes, bytes): - LOGGER.error( - "canonical audit bundle unavailable reason=read_error " - "block_hash=%s audit_bundle_sha256=%s", - block_hash, - audit_bundle_sha256, - ) - raise TypeError("canonical audit bundle reader returned non-bytes") - return canonical_bytes - - def _audit_reader(self, body_uri: object) -> AuditArtifactStore: - del body_uri - store = getattr(self, "_audit_artifact_store", None) - if store is not None: - return store - # Compatibility-only dashboard resolvers historically initialized a - # read-only PsqlShareLedger subclass with just `_audit_body_dir`. Keep - # that explicit adapter working while routing every read through A1. - legacy_root = getattr(self, "_audit_body_dir", None) - if legacy_root is not None: - root = Path(legacy_root) - store = AuditArtifactStore( - AuditArtifactConfig( - root=root, - evidence_path=root / "prism-live-stratum-evidence.json", - ), - canonicalizer=( - getattr(self, "_audit_bundle_canonicalizer", None) - or _default_bundle_canonicalizer() - ), - ) - self._audit_artifact_store = store - return store - raise RuntimeError( - "audit bundle body is not retrievable: audit body store is not configured" - ) - - def _externalize_audit_body(self, block_hash: str, audit_bundle_sha256: str, final_bundle: Mapping[str, Any]) -> str | None: - if self._audit_artifact_store is None: - return None - return self._audit_store().externalize_audit_body(block_hash, audit_bundle_sha256, final_bundle) - - def _canonical_audit_body_bytes_for_sha(self, final_bundle: Mapping[str, Any], audit_bundle_sha256: str) -> bytes: - return self._audit_store().canonical_audit_body_bytes_for_sha(final_bundle, audit_bundle_sha256) - - def _audit_body_ref(self, **kwargs: Any) -> dict[str, Any] | None: - return self._audit_store().audit_body_ref(**kwargs) - - def _audit_bundle_v2(self, **kwargs: Any) -> dict[str, Any] | None: - kwargs.setdefault("load_missing_range", self._load_audit_share_ledger_range) - return self._audit_store().audit_bundle_v2(**kwargs) - - def _audit_share_parts(self, shares: list[Any]) -> list[dict[str, Any]] | None: - return self._audit_store().audit_share_parts(shares) - - def _audit_share_range_parts(self, shares: list[Any]) -> list[dict[str, Any]] | None: - return self._audit_store().audit_share_range_parts( - shares, - load_missing_range=self._load_audit_share_ledger_range, - ) - - def _audit_share_segment_payload(self, **kwargs: Any) -> dict[str, Any]: - return self._audit_store().audit_share_segment_payload(**kwargs) - - def _write_audit_share_segment(self, **kwargs: Any) -> tuple[str, str]: - return self._audit_store().write_audit_share_segment(**kwargs) - - def _write_audit_share_segment_range(self, **kwargs: Any) -> tuple[str, str]: - kwargs.setdefault("load_missing_range", self._load_audit_share_ledger_range) - return self._audit_store().write_audit_share_segment_range(**kwargs) - - def _merge_audit_share_ranges(self, existing_shares: list[Any], incoming_shares: list[Any], *, segment_path: Path) -> list[Any] | None: - return self._audit_store().merge_audit_share_ranges( - existing_shares, - incoming_shares, - segment_path=segment_path, - load_missing_range=self._load_audit_share_ledger_range, - ) - - def _load_audit_share_ledger_range( - self, - *, - first_share_seq: int, - last_share_seq: int, - ) -> list[dict[str, object]]: - sql = f""" -SELECT COALESCE(json_agg(json_build_object( - 'share_seq', share_seq, - 'share_id', share_id, - 'miner_id', miner_id, - 'order_key', payout_order_key, - 'p2mr_program_hex', encode(p2mr_program, 'hex'), - 'share_difficulty', share_difficulty::text, - 'network_difficulty', network_difficulty::text, - 'template_height', template_height, - 'job_id', job_id, - 'job_issued_at_ms', round(extract(epoch FROM job_issued_at) * 1000)::bigint, - 'accepted_at_ms', round(extract(epoch FROM accepted_at) * 1000)::bigint, - 'ntime', ntime, - 'credit_policy', credit_policy -) ORDER BY share_seq ASC), '[]'::json) -FROM qbit_share_ledger -WHERE accepted - AND share_seq BETWEEN {int(first_share_seq)} AND {int(last_share_seq)}; -""" - rows = self._run_read_json(sql) - if not isinstance(rows, list): - raise RuntimeError("audit share ledger backfill did not return a list") - return [self._record_from_json(row).to_prism_json() for row in rows] - - def _audit_shares_by_seq(self, shares: list[Any], *, segment_path: Path, require_contiguous: bool = True) -> dict[int, Any]: - return self._audit_store().audit_shares_by_seq( - shares, - segment_path=segment_path, - require_contiguous=require_contiguous, - ) - - def _storage_json_bytes(self, payload: dict[str, Any]) -> bytes: - return self._audit_store().storage_json_bytes(payload) - - @staticmethod - def _file_sha256_hex(path: Path) -> str: - return AuditArtifactStore.file_sha256_hex(path) - - def _canonical_audit_bundle_bytes(self, final_bundle: dict[str, Any]) -> bytes: - if self._audit_artifact_store is None: - return canonical_audit_bundle_bytes( - final_bundle, - self._audit_bundle_canonicalizer, - ) - return self._audit_store().canonical_audit_bundle_bytes(final_bundle) - - def _audit_body_path(self, block_hash: str, audit_bundle_sha256: str) -> Path: - return self._audit_store().body_path(block_hash, audit_bundle_sha256) - - def _resolve_audit_body_path(self, body_uri: object) -> Path: - return self._audit_store().resolve_owned_path(body_uri) - - def _external_audit_body_write_plan(self, payload: dict[str, Any]) -> str | None: - """Refresh the writer lease and decide whether this persist may write a body. - - The audit body store lives outside Postgres, so file writes cannot be in - the same transaction as the ledger insert. This preflight keeps stale - writers from creating artifacts by requiring the DB lease to be current - before any filesystem side effect. - """ - if self._audit_artifact_store is None: - return None - sql = f""" -WITH payload AS ( - SELECT {self._jsonb_literal(payload)} AS data -), -lease AS ( - UPDATE qbit_ledger_writer_lease - SET lease_expires_at = clock_timestamp() + {self._lease_interval_sql}, - updated_at = clock_timestamp() - FROM payload - WHERE qbit_ledger_writer_lease.singleton - AND qbit_ledger_writer_lease.writer_id = data->>'writer_id' - AND qbit_ledger_writer_lease.writer_epoch = (data->>'writer_epoch')::bigint - AND qbit_ledger_writer_lease.writer_session_token = data->>'writer_session_token' - RETURNING qbit_ledger_writer_lease.writer_id -), -existing_block AS ( - SELECT - block_hash, - block_height, - parent_hash, - coinbase_txid, - payout_manifest_sha256 - FROM qbit_pool_blocks - WHERE block_hash = (SELECT data->>'block_hash' FROM payload) -), -matching_existing_block AS ( - SELECT existing_block.block_hash - FROM existing_block, payload - WHERE existing_block.block_height = (data->>'block_height')::bigint - AND existing_block.parent_hash = data->>'parent_hash' - AND existing_block.coinbase_txid = data->>'coinbase_txid' - AND existing_block.payout_manifest_sha256 = data->>'payout_manifest_sha256' -), -existing_bundle AS ( - SELECT block_hash, audit_bundle_sha256, coinbase_tx_hex, body_uri - FROM qbit_pool_audit_bundles - WHERE block_hash = (SELECT data->>'block_hash' FROM payload) -), -matching_existing_bundle AS ( - SELECT existing_bundle.block_hash - FROM existing_bundle, payload - WHERE existing_bundle.audit_bundle_sha256 = data->>'audit_bundle_sha256' - AND existing_bundle.coinbase_tx_hex = data->>'coinbase_tx_hex' -) -SELECT CASE - WHEN (SELECT count(*) FROM lease) = 0 THEN - json_build_object('error', 'writer lease is not active') - WHEN (SELECT count(*) FROM existing_block) > 0 - AND (SELECT count(*) FROM matching_existing_block) = 0 THEN - json_build_object('error', 'existing block metadata does not match payload') - WHEN (SELECT count(*) FROM existing_block) > 0 - AND (SELECT count(*) FROM matching_existing_bundle) = 0 THEN - json_build_object('error', 'existing audit bundle does not match payload') - ELSE - json_build_object( - 'existing_block', (SELECT count(*) FROM existing_block) > 0, - 'existing_body_uri', (SELECT body_uri FROM existing_bundle LIMIT 1) - ) -END; -""" - result = self._run_fenced_json(sql) - if "error" in result: - raise RuntimeError(str(result["error"])) - existing_body_uri = result.get("existing_body_uri") - if existing_body_uri: - return str(existing_body_uri) - if result.get("existing_block"): - return None - return str(self._audit_body_path(payload["block_hash"], payload["audit_bundle_sha256"])) - - def preflight_canonical_bundle_publication( - self, - *, - block_hash: str, - audit_bundle_sha256: str, - ) -> dict[str, object]: - """Refresh the writer lease and confirm one block/digest row identity. - - The standalone canonical-bundle backfill calls this immediately before - each publication. It is the same fence `_external_audit_body_write_plan` - applies to the live path: a writer whose lease has lapsed must not keep - creating artifacts under the audit root, and the digest it is about to - publish must be the one the ledger row already advertises. Reads the - audit tables only; the sole mutation is the writer's own lease - heartbeat, so no schema changes. - - Returns the row's `body_uri` (None for legacy inline rows) and whether - an inline body is still present, which is what the backfill needs to - choose its byte source. Raises when the lease is not active or the - block/digest identity does not match. - """ - - block_hash = canonical_hex(str(block_hash), name="block_hash", expected_bytes=32) - digest = canonical_hex( - str(audit_bundle_sha256), - name="audit_bundle_sha256", - expected_bytes=32, - ) - payload = { - "block_hash": block_hash, - "audit_bundle_sha256": digest, - "writer_id": self._writer_id, - "writer_epoch": self._writer_epoch, - "writer_session_token": self._writer_session_token, - } - sql = f""" -WITH payload AS ( - SELECT {self._jsonb_literal(payload)} AS data -), -lease AS ( - UPDATE qbit_ledger_writer_lease - SET lease_expires_at = clock_timestamp() + {self._lease_interval_sql}, - updated_at = clock_timestamp() - FROM payload - WHERE qbit_ledger_writer_lease.singleton - AND qbit_ledger_writer_lease.writer_id = data->>'writer_id' - AND qbit_ledger_writer_lease.writer_epoch = (data->>'writer_epoch')::bigint - AND qbit_ledger_writer_lease.writer_session_token = data->>'writer_session_token' - RETURNING qbit_ledger_writer_lease.writer_id -), -bundle AS ( - SELECT - block_hash, - audit_bundle_sha256, - body_uri, - audit_bundle IS NOT NULL AS has_inline_bundle - FROM qbit_pool_audit_bundles - WHERE block_hash = (SELECT data->>'block_hash' FROM payload) -), -matching_bundle AS ( - SELECT bundle.* - FROM bundle, payload - WHERE bundle.audit_bundle_sha256 = data->>'audit_bundle_sha256' -) -SELECT CASE - WHEN (SELECT count(*) FROM lease) = 0 THEN - json_build_object('error', 'writer lease is not active') - WHEN (SELECT count(*) FROM bundle) = 0 THEN - json_build_object('error', 'audit bundle row does not exist') - WHEN (SELECT count(*) FROM matching_bundle) = 0 THEN - json_build_object('error', 'audit bundle digest does not match the stored row') - ELSE - json_build_object( - 'block_hash', (SELECT block_hash FROM matching_bundle LIMIT 1), - 'audit_bundle_sha256', (SELECT audit_bundle_sha256 FROM matching_bundle LIMIT 1), - 'body_uri', (SELECT body_uri FROM matching_bundle LIMIT 1), - 'has_inline_bundle', (SELECT has_inline_bundle FROM matching_bundle LIMIT 1) - ) -END; -""" - result = self._run_fenced_json(sql) - if not isinstance(result, dict): - raise RuntimeError("canonical bundle preflight returned no row") - if "error" in result: - raise RuntimeError(str(result["error"])) - body_uri = result.get("body_uri") - return { - "block_hash": block_hash, - "audit_bundle_sha256": digest, - "body_uri": str(body_uri) if body_uri else None, - "has_inline_bundle": bool(result.get("has_inline_bundle")), - } - - def canonical_bundle_bytes_for_backfill( - self, - *, - block_hash: str, - audit_bundle_sha256: str, - body_uri: object, - ) -> bytes: - """Recover verified canonical bytes for an already published body. - - Binds the ledger's append-only share loader to the store helper so the - standalone backfill can repair share segments that have left the disk - without reaching into ledger internals. Every recovered range is checked - against the digest the body already commits to, and the assembled bundle - against the advertised bundle digest. - """ - - return self._audit_store().canonical_bundle_bytes_from_external_body( - block_hash=block_hash, - audit_bundle_sha256=audit_bundle_sha256, - body_uri=body_uri, - load_missing_range=self._load_audit_share_ledger_range, - ) - - def publish_canonical_bundle_bytes( - self, - *, - block_hash: str, - audit_bundle_sha256: str, - canonical_bytes: bytes, - ) -> str: - """Publish canonical bytes under the store's ownership checks. - - Callers must run `preflight_canonical_bundle_publication` immediately - before this so the writer lease is current when the artifact lands. - """ - - return str( - self._audit_store().write_canonical_audit_bundle( - block_hash, - audit_bundle_sha256, - canonical_bytes, - ) - ) - - def _audit_body_byte_len(self, body_uri: object | None, final_bundle: Mapping[str, Any], canonical_bundle_path: Path | None = None) -> int: - if self._audit_artifact_store is None: - if isinstance(final_bundle, CanonicalAuditBundleView): - # The artifact's own length: no canonicalizer round trip and - # no whole-body copy for the inline compatibility lane. - final_bundle.verify_identity() - return final_bundle.byte_length - return len(self._canonical_audit_bundle_bytes(final_bundle)) - return self._audit_store().audit_body_byte_len(body_uri, final_bundle, canonical_bundle_path) - - def _prepare_external_audit_body(self, payload: dict[str, Any], final_bundle: Mapping[str, Any], *, canonical_bundle_path: Path | None = None) -> str | None: - if self._audit_artifact_store is None: - return None - normalized = { - **payload, - "block_hash": canonical_hex(str(payload["block_hash"]), name="block_hash", expected_bytes=32), - "audit_bundle_sha256": canonical_hex(str(payload["audit_bundle_sha256"]), name="audit_bundle_sha256", expected_bytes=32), - } - if canonical_bundle_path is not None: - self._audit_store().validate_canonical_source( - canonical_bundle_path, - str(normalized["audit_bundle_sha256"]), - final_bundle, - ) - body_uri = self._external_audit_body_write_plan(normalized) - return self._audit_store().prepare_external_audit_body( - normalized, - final_bundle, - body_uri=body_uri, - canonical_bundle_path=canonical_bundle_path, - load_missing_range=self._load_audit_share_ledger_range, - ) - - def _read_external_body(self, body_uri: object, *, expected_sha256: object | None = None) -> dict[str, object] | None: - return self._audit_reader(body_uri).read_external_body(body_uri, expected_sha256=expected_sha256) - - def _external_body_matches_sha(self, body_path: Path, expected_sha256: str) -> bool: - return self._audit_reader(body_path).external_body_matches_sha(body_path, expected_sha256) - - def _external_body_available_for_sha(self, body_uri: object, expected_sha256: str) -> bool: - try: - reader = self._audit_reader(body_uri) - except RuntimeError: - return False - return reader.external_body_available_for_sha(body_uri, expected_sha256) - - def _audit_share_segment_available(self, part: dict[str, Any], *, parent_body_uri: object) -> bool: - try: - self._audit_store().read_audit_share_segment(part, parent_body_uri=parent_body_uri) - except (OSError, RuntimeError, TypeError, ValueError): - return False - return True - - def _resolve_audit_body_ref(self, body_ref: dict[str, Any], *, expected_sha256: object | None, body_uri: object) -> dict[str, object]: - return self._audit_store().resolve_audit_body_ref(body_ref, expected_sha256=expected_sha256, body_uri=body_uri) - - def _resolve_audit_bundle_v2(self, body: dict[str, Any], *, expected_sha256: object | None, body_uri: object) -> dict[str, object]: - return self._audit_store().resolve_audit_bundle_v2(body, expected_sha256=expected_sha256, body_uri=body_uri) - - def _read_audit_share_segment(self, part: dict[str, Any], *, parent_body_uri: object) -> list[Any]: - return self._audit_store().read_audit_share_segment(part, parent_body_uri=parent_body_uri) - - def _select_audit_share_segment_range(self, shares: list[Any], *, first_share_seq: int, last_share_seq: int, parent_body_uri: object, body_uri: object) -> list[Any]: - return self._audit_store().select_audit_share_segment_range( - shares, - first_share_seq=first_share_seq, - last_share_seq=last_share_seq, - parent_body_uri=parent_body_uri, - body_uri=body_uri, - ) - - def _resolve_audit_bundle_row(self, row: object) -> dict[str, object] | None: - """Return an audit-bundle row with its body resolved inline. - - Reads the externalized body from body_uri when the inline JSONB is NULL, - so legacy inline rows and externalized rows present an identical shape to - callers. - """ - if not isinstance(row, dict): - return None - result = dict(row) - body = result.get("audit_bundle") - if body is None: - body = self._read_external_body( - result.get("body_uri"), - expected_sha256=result.get("audit_bundle_sha256"), - ) - result.pop("body_uri", None) - if body is None: - return None - result["audit_bundle"] = body - return result - - @staticmethod - def _accepted_block_payload_literal_pieces(payload: Mapping[str, Any]) -> Iterator[str]: - """The accepted-block payload as a dollar-quoted JSONB literal, streamed. - - Equivalent to ``_jsonb_literal(payload)`` for a payload of plain - values, but built from bounded chunks: recipient-scaled members - (the payout accounts) and transaction-scaled members (the leaf - arrays) stream from their lazy views, and on the legacy inline lane - (no audit body store configured) the verified canonical artifact is - spliced verbatim from its descriptor rather than decoded and - re-encoded. The dollar-quote tag is chosen with the same collision - rule as ``_jsonb_literal``, checked by streaming. The statement text - itself still scales with those members: this method bounds the - Python object graph and the number of copies, not the size of the - one statement the schema requires (#255). - """ - - def chunks() -> Iterator[str]: - return iter_json_chunks(payload) - - tag = "qbit_prism_json" - for _attempt in range(64): - if not _text_stream_contains(chunks(), f"${tag}$"): - break - tag += "_x" - else: - raise RuntimeError("accepted-block payload cannot be dollar-quoted") - yield f"${tag}$" - yield from chunks() - yield f"${tag}$::jsonb" - - def persist_accepted_block( - self, - *, - block_hash: str, - block_height: int, - parent_hash: str, - final_bundle: Mapping[str, Any], - audit_report: dict[str, Any], - canonical_bundle_path: Path | None = None, - ) -> dict[str, int | str]: - """Persist one accepted block, its audit bundle and payout rows. - - ``final_bundle`` is either a dictionary (compatibility builders and - the memory ledger's callers) or the bundle compiler's bounded view - of the canonical artifact. Only header members are read here; the - external body publication streams the artifact and the row carries - the body pointer, so no window-sized structure enters the statement - on the production (externalized) path. What the statement still - carries per accepted block, bounded by recipient and transaction - cardinality rather than by the share window: the payout accounts - array (one row per recipient), the audit-commitment leaves and the - witness merkle leaves (one per block transaction). - """ - manifest = final_bundle["signed_coinbase_manifest"]["manifest"] - found_block = final_bundle.get("found_block") or {} - audit_bundle_sha256 = canonical_hex( - str(audit_report["audit_bundle_sha256_hex"]), - name="audit_bundle_sha256", - expected_bytes=32, - ) - block_hash = canonical_hex(str(block_hash), name="block_hash", expected_bytes=32) - parent_hash = canonical_hex(str(parent_hash), name="parent_hash", expected_bytes=32) - payload = { - "block_hash": block_hash, - "block_height": block_height, - "parent_hash": parent_hash, - "coinbase_txid": audit_report["coinbase_txid"], - "payout_manifest_sha256": audit_report["coinbase_manifest_sha256_hex"], - "audit_bundle_sha256": audit_bundle_sha256, - "coinbase_tx_hex": audit_report["coinbase_tx_hex"], - "writer_id": self._writer_id, - "writer_epoch": self._writer_epoch, - "writer_session_token": self._writer_session_token, - } - body_uri = self._prepare_external_audit_body( - payload, - final_bundle, - canonical_bundle_path=canonical_bundle_path, - ) - audit_body_byte_len = self._audit_body_byte_len( - body_uri, - final_bundle, - canonical_bundle_path, - ) - # A bounded artifact view (the canonical build's result) has no - # dictionary form to embed; the legacy inline lane splices its - # canonical bytes into the statement literal instead (#255). - inline_bundle: object - if body_uri is not None: - inline_bundle = None - elif isinstance(final_bundle, CanonicalAuditBundleView): - final_bundle.verify_identity() - inline_bundle = RawJsonDocument(final_bundle.source) - else: - inline_bundle = final_bundle - payload = { - **payload, - # Externalized rows store the body in body_uri and NULL here; legacy - # rows (no body store configured) keep the inline body. - "audit_bundle": inline_bundle, - "body_uri": body_uri, - "audit_body_byte_len": audit_body_byte_len, - "schema_version": str(final_bundle.get("schema") or "qbit.prism.audit-bundle.v1"), - "found_block_network_difficulty": found_block.get("network_difficulty"), - "found_block_bits": found_block.get("bits"), - "found_block_coinbase_value_sats": found_block.get("coinbase_value_sats"), - # The leaf arrays and the payout accounts stay lazy on a bounded - # artifact view; the statement literal streams them below. - "audit_commitment_leaves_hex": final_bundle.get("audit_commitment_leaves_hex"), - "witness_merkle_leaves_hex": final_bundle.get("witness_merkle_leaves_hex"), - "accounts": final_bundle["payout_policy_manifest"]["accounts"], - } - sql_body = f"""lease AS ( - UPDATE qbit_ledger_writer_lease - SET lease_expires_at = clock_timestamp() + {self._lease_interval_sql}, - updated_at = clock_timestamp() - FROM payload - WHERE qbit_ledger_writer_lease.singleton - AND qbit_ledger_writer_lease.writer_id = data->>'writer_id' - AND qbit_ledger_writer_lease.writer_epoch = (data->>'writer_epoch')::bigint - AND qbit_ledger_writer_lease.writer_session_token = data->>'writer_session_token' - RETURNING qbit_ledger_writer_lease.writer_id -), -existing_block AS ( - SELECT - block_hash, - block_height, - parent_hash, - coinbase_txid, - payout_manifest_sha256 - FROM qbit_pool_blocks - WHERE block_hash = (SELECT data->>'block_hash' FROM payload) -), -inserted_block AS ( - INSERT INTO qbit_pool_blocks ( - block_hash, - block_height, - parent_hash, - coinbase_txid, - payout_manifest_sha256 - ) - SELECT - data->>'block_hash', - (data->>'block_height')::bigint, - data->>'parent_hash', - data->>'coinbase_txid', - data->>'payout_manifest_sha256' - FROM payload, lease - WHERE NOT EXISTS (SELECT 1 FROM existing_block) - RETURNING block_hash -), -accounts AS ( - SELECT - data, - account->>'recipient_id' AS miner_id, - account->>'order_key' AS payout_order_key, - decode(account->>'p2mr_program_hex', 'hex') AS p2mr_program, - (account->>'gross_amount_sats')::bigint AS gross_amount_sats, - (account->>'prior_balance_sats')::numeric AS prior_balance_sats, - (account->>'candidate_balance_sats')::numeric AS candidate_balance_sats, - (account->>'onchain_amount_sats')::bigint AS onchain_amount_sats, - COALESCE((account->>'settlement_fee_sats')::bigint, 0) AS settlement_fee_sats, - (account->>'carry_forward_balance_sats')::numeric AS carry_forward_balance_sats, - account->>'action' AS action - FROM payload, - jsonb_array_elements(data->'accounts') AS account -), -bundle_insert AS ( - INSERT INTO qbit_pool_audit_bundles ( - block_hash, - audit_bundle, - audit_bundle_sha256, - coinbase_tx_hex, - body_uri, - audit_body_byte_len, - schema_version, - found_block_network_difficulty, - found_block_bits, - found_block_coinbase_value_sats, - audit_commitment_leaves_hex, - witness_merkle_leaves_hex - ) - SELECT - data->>'block_hash', - CASE WHEN jsonb_typeof(data->'audit_bundle') = 'object' THEN data->'audit_bundle' ELSE NULL END, - data->>'audit_bundle_sha256', - data->>'coinbase_tx_hex', - data->>'body_uri', - (data->>'audit_body_byte_len')::bigint, - data->>'schema_version', - (data->>'found_block_network_difficulty')::numeric, - data->>'found_block_bits', - (data->>'found_block_coinbase_value_sats')::bigint, - CASE WHEN jsonb_typeof(data->'audit_commitment_leaves_hex') = 'array' THEN data->'audit_commitment_leaves_hex' ELSE NULL END, - CASE WHEN jsonb_typeof(data->'witness_merkle_leaves_hex') = 'array' THEN data->'witness_merkle_leaves_hex' ELSE NULL END - FROM payload, inserted_block - RETURNING block_hash -), -payout_insert AS ( - INSERT INTO qbit_pool_payout_entries ( - block_hash, - block_height, - miner_id, - payout_order_key, - p2mr_program, - onchain_amount_sats, - carry_forward_balance_sats, - action - ) - SELECT - data->>'block_hash', - (data->>'block_height')::bigint, - miner_id, - payout_order_key, - p2mr_program, - onchain_amount_sats, - carry_forward_balance_sats, - action - FROM accounts, inserted_block - RETURNING payout_entry_seq -), -carry_insert AS ( - INSERT INTO qbit_payout_carry_forward ( - block_height, - block_hash, - miner_id, - payout_order_key, - p2mr_program, - gross_amount_sats, - prior_balance_sats, - candidate_balance_sats, - onchain_amount_sats, - settlement_fee_sats, - carry_forward_balance_sats, - action - ) - SELECT - (data->>'block_height')::bigint, - data->>'block_hash', - miner_id, - payout_order_key, - p2mr_program, - gross_amount_sats, - prior_balance_sats, - candidate_balance_sats, - onchain_amount_sats, - settlement_fee_sats, - carry_forward_balance_sats, - action - FROM accounts, inserted_block - RETURNING carry_forward_seq -), -matching_existing_block AS ( - SELECT existing_block.block_hash - FROM existing_block, payload - WHERE existing_block.block_height = (data->>'block_height')::bigint - AND existing_block.parent_hash = data->>'parent_hash' - AND existing_block.coinbase_txid = data->>'coinbase_txid' - AND existing_block.payout_manifest_sha256 = data->>'payout_manifest_sha256' -), -existing_bundle AS ( - SELECT block_hash, audit_bundle_sha256, coinbase_tx_hex - FROM qbit_pool_audit_bundles - WHERE block_hash = (SELECT data->>'block_hash' FROM payload) -), -matching_existing_bundle AS ( - -- audit_bundle_sha256 is computed over the full bundle content, so matching - -- it (plus the coinbase tx) proves an identical body without comparing the - -- JSONB directly, which is NULL for externalized rows. - SELECT existing_bundle.block_hash - FROM existing_bundle, payload - WHERE existing_bundle.audit_bundle_sha256 = data->>'audit_bundle_sha256' - AND existing_bundle.coinbase_tx_hex = data->>'coinbase_tx_hex' -), -expected_payout_rows AS ( - SELECT - data->>'block_hash' AS block_hash, - (data->>'block_height')::bigint AS block_height, - miner_id, - payout_order_key, - p2mr_program, - onchain_amount_sats, - carry_forward_balance_sats, - action - FROM accounts -), -existing_payout_rows AS ( - SELECT - block_hash, - block_height, - miner_id, - payout_order_key, - p2mr_program, - onchain_amount_sats, - carry_forward_balance_sats, - action - FROM qbit_pool_payout_entries - WHERE block_hash = (SELECT data->>'block_hash' FROM payload) -), -payout_missing AS ( - SELECT * FROM expected_payout_rows - EXCEPT ALL - SELECT * FROM existing_payout_rows -), -payout_extra AS ( - SELECT * FROM existing_payout_rows - EXCEPT ALL - SELECT * FROM expected_payout_rows -), -expected_carry_rows AS ( - SELECT - (data->>'block_height')::bigint AS block_height, - data->>'block_hash' AS block_hash, - miner_id, - payout_order_key, - p2mr_program, - gross_amount_sats, - prior_balance_sats, - candidate_balance_sats, - onchain_amount_sats, - settlement_fee_sats, - carry_forward_balance_sats, - action - FROM accounts -), -existing_carry_rows AS ( - SELECT - block_height, - block_hash, - miner_id, - payout_order_key, - p2mr_program, - gross_amount_sats, - prior_balance_sats, - candidate_balance_sats, - onchain_amount_sats, - settlement_fee_sats, - carry_forward_balance_sats, - action - FROM qbit_payout_carry_forward - WHERE block_hash = (SELECT data->>'block_hash' FROM payload) -), -carry_missing AS ( - SELECT * FROM expected_carry_rows - EXCEPT ALL - SELECT * FROM existing_carry_rows -), -carry_extra AS ( - SELECT * FROM existing_carry_rows - EXCEPT ALL - SELECT * FROM expected_carry_rows -) -SELECT CASE - WHEN (SELECT count(*) FROM lease) = 0 THEN - json_build_object('error', 'writer lease is not active') - WHEN (SELECT count(*) FROM existing_block) > 0 - AND (SELECT count(*) FROM matching_existing_block) = 0 THEN - json_build_object('error', 'existing block metadata does not match payload') - WHEN (SELECT count(*) FROM existing_block) > 0 - AND (SELECT count(*) FROM matching_existing_bundle) = 0 THEN - json_build_object('error', 'existing audit bundle does not match payload') - WHEN (SELECT count(*) FROM existing_block) > 0 - AND ( - EXISTS (SELECT 1 FROM payout_missing) - OR EXISTS (SELECT 1 FROM payout_extra) - ) THEN - json_build_object('error', 'existing payout entries do not match payload') - WHEN (SELECT count(*) FROM existing_block) > 0 - AND ( - EXISTS (SELECT 1 FROM carry_missing) - OR EXISTS (SELECT 1 FROM carry_extra) - ) THEN - json_build_object('error', 'existing carry-forward rows do not match payload') - ELSE - json_build_object( - 'backend', 'postgres-psql', - 'share_count', (SELECT count(*) FROM qbit_share_ledger WHERE accepted), - 'block_count', CASE - WHEN (SELECT count(*) FROM inserted_block) > 0 THEN (SELECT count(*) FROM inserted_block) - ELSE (SELECT count(*) FROM existing_block) - END, - 'bundle_count', CASE - WHEN (SELECT count(*) FROM inserted_block) > 0 THEN (SELECT count(*) FROM bundle_insert) - ELSE (SELECT count(*) FROM existing_bundle) - END, - 'payout_entry_count', CASE - WHEN (SELECT count(*) FROM inserted_block) > 0 THEN (SELECT count(*) FROM payout_insert) - ELSE (SELECT count(*) FROM existing_payout_rows) - END, - 'carry_forward_count', CASE - WHEN (SELECT count(*) FROM inserted_block) > 0 THEN (SELECT count(*) FROM carry_insert) - ELSE (SELECT count(*) FROM existing_carry_rows) - END, - 'onchain_output_count', {int(manifest["payout_count"])} - ) -END; -""" - result = self._run_candidate_payload_json(payload, sql_body) - if "error" in result: - raise RuntimeError(str(result["error"])) - return { - "backend": str(result["backend"]), - "share_count": int(result["share_count"]), - "block_count": int(result["block_count"]), - "bundle_count": int(result["bundle_count"]), - "payout_entry_count": int(result["payout_entry_count"]), - "carry_forward_count": int(result["carry_forward_count"]), - "onchain_output_count": int(result["onchain_output_count"]), - "audit_bundle_sha256": audit_bundle_sha256, - "body_uri": str(body_uri) if body_uri is not None else "", - "audit_body_byte_len": audit_body_byte_len, - } - - def reverse_immature_block(self, *, block_hash: str, active_tip_height: int) -> dict[str, int | str]: - sql = f""" -SELECT json_build_object( - 'backend', 'postgres-psql', - 'reversed_count', qbit_reverse_immature_pool_block( - {self._text_literal(block_hash)}, - {int(active_tip_height)}, - {self._text_literal(self._writer_id)}, - {int(self._writer_epoch)}, - {self._text_literal(self._writer_session_token)}, - {self._lease_interval_sql} - ) -); -""" - result = self._run_fenced_json(sql) - if "error" in result: - raise RuntimeError(str(result["error"])) - return { - "backend": str(result["backend"]), - "reversed_count": int(result["reversed_count"]), - } - - def reject_prepared_block(self, *, block_hash: str, active_tip_height: int) -> dict[str, int | str]: - sql = f""" -SELECT json_build_object( - 'backend', 'postgres-psql', - 'rejected_count', qbit_reject_prepared_pool_block( - {self._text_literal(block_hash)}, - {int(active_tip_height)}, - {self._text_literal(self._writer_id)}, - {int(self._writer_epoch)}, - {self._text_literal(self._writer_session_token)}, - {self._lease_interval_sql} - ) -); -""" - result = self._run_fenced_json(sql) - if "error" in result: - raise RuntimeError(str(result["error"])) - return { - "backend": str(result["backend"]), - "rejected_count": int(result["rejected_count"]), - } - - def confirm_accepted_block(self, *, block_hash: str, active_tip_height: int) -> dict[str, int | str]: - sql = f""" -SELECT json_build_object( - 'backend', 'postgres-psql', - 'confirmed_count', qbit_confirm_pool_block( - {self._text_literal(block_hash)}, - {int(active_tip_height)}, - {self._text_literal(self._writer_id)}, - {int(self._writer_epoch)}, - {self._text_literal(self._writer_session_token)}, - {self._lease_interval_sql} - ) -); -""" - with self._operation_gate(self._lock, "writer lock"): - result = self._run_json(sql) - if "error" in result: - raise RuntimeError(str(result["error"])) - confirmed_count = int(result["confirmed_count"]) - publication_sequence: object | None = None - if confirmed_count in {1, 2}: - # Both confirming dispositions leave a confirmed row carrying - # an ordinal: 1 allocated one in the statement above, and 2 -- - # the idempotent replay -- returns the one its original flip - # allocated. The caller needs the ordinal either way to - # address this block's audit publication, so read it for - # both. Non-confirming dispositions (0, -1) have no row to - # read and stay a single statement. - # - # A data-modifying PL/pgSQL function runs under the statement's - # command snapshot, so a join in that same statement cannot see - # the freshly assigned ordinal. Read it in the next statement - # while retaining the ledger writer lock. - # - # Holding the lock across both statements is what makes this - # step's heartbeat-silent span two statement budgets rather - # than one: no admission slice runs in between, so a landing - # caller's liveness monitor gets nothing until the second - # statement returns. Report the completed first round trip - # before starting the second (see _note_operation_progress). - self._note_operation_progress() - state = self._run_retry_safe_read_json( - f""" -SELECT json_build_object( - 'audit_publication_sequence', ( - SELECT audit_publication_sequence - FROM qbit_pool_blocks - WHERE block_hash = {self._text_literal(block_hash)} - AND block_height = {int(active_tip_height)} - AND chain_state = 'confirmed' - AND maturity_state <> 'reversed' - ) -); -""" - ) - publication_sequence = state.get("audit_publication_sequence") - if publication_sequence is None: - raise RuntimeError( - "confirmed pool block has no audit publication sequence" - ) - response: dict[str, int | str] = { - "backend": str(result["backend"]), - "confirmed_count": confirmed_count, - } - if publication_sequence is not None: - response["audit_publication_sequence"] = int(publication_sequence) - return response - - def pool_block_state(self, *, block_hash: str) -> dict[str, object] | None: - block_hash = canonical_hex(block_hash, name="block_hash", expected_bytes=32) - sql = f""" -SELECT json_build_object( - 'state', ( - SELECT json_build_object( - 'block_hash', block_hash, - 'block_height', block_height, - 'parent_hash', parent_hash, - 'chain_state', chain_state, - 'maturity_state', maturity_state, - 'audit_publication_sequence', audit_publication_sequence - ) - FROM qbit_pool_blocks - WHERE block_hash = {self._text_literal(block_hash)} - ) -); -""" - with self._operation_gate(self._lock, "writer lock"): - result = self._run_retry_safe_read_json(sql) - if not isinstance(result, dict): - raise RuntimeError("pool block state query returned non-object JSON") - state = result.get("state") - if state is None: - return None - if not isinstance(state, dict): - raise RuntimeError("pool block state query returned non-object JSON") - state["block_height"] = int(state["block_height"]) - return state - - def audit_publication_sequence_floor(self) -> int: - """Return MAX durable pool-block ordinal, excluding sequence gaps.""" - - sql = """ -SELECT json_build_object( - 'audit_publication_sequence_floor', - COALESCE(MAX(audit_publication_sequence), 0) -) -FROM qbit_pool_blocks; -""" - with self._operation_gate(self._lock, "writer lock"): - result = self._run_retry_safe_read_json(sql) - if not isinstance(result, dict): - raise RuntimeError( - "audit publication sequence floor query returned non-object JSON" - ) - value = result.get("audit_publication_sequence_floor") - if isinstance(value, bool) or not isinstance(value, int) or value < 0: - raise RuntimeError("audit publication sequence floor is invalid") - return value - - def reorg_watch_blocks(self, *, active_tip_height: int) -> list[dict[str, object]]: - sql = """ -SELECT COALESCE(json_agg(json_build_object( - 'block_hash', block_hash, - 'block_height', block_height, - 'parent_hash', parent_hash, - 'chain_state', chain_state, - 'maturity_state', maturity_state -) ORDER BY block_height ASC, block_hash ASC), '[]'::json) -FROM qbit_pool_blocks -WHERE chain_state IN ('confirmed', 'inactive') - AND maturity_state = 'immature' -; -""" - with self._operation_gate(self._lock, "writer lock"): - rows = self._run_retry_safe_read_json(sql) - for row in rows: - row["block_height"] = int(row["block_height"]) - return rows - - def stranded_prepared_blocks( - self, - *, - active_tip_height: int, - min_depth: int, - limit: int = 64, - ) -> list[dict[str, object]]: - """Return deeply buried rows still parked in the prepared state. - - ``reorg_watch_blocks`` deliberately watches only confirmed/inactive - rows, and a prepared row is normally resolved by the live - submit/replay path that owns it through its outbox entry. A row - whose outbox entry is gone (quarantined, or completed by a process - that died before confirming) therefore has nothing left to - re-examine it, and stays prepared forever — holding immature payout - entries, carry-forward, and CTV fanout artifacts open with it. This - read finds those rows; the caller decides, against the active chain, - which ones are provably orphaned. - - The predicate leads with ``maturity_state`` and ``block_height`` so - it rides ``qbit_pool_blocks_maturity_idx``, and the depth floor - keeps the scan to rows the caller could actually act on. - """ - if limit <= 0: - return [] - depth_ceiling = int(active_tip_height) - int(min_depth) - sql = f""" -SELECT COALESCE(json_agg(json_build_object( - 'block_hash', block_hash, - 'block_height', block_height, - 'parent_hash', parent_hash -) ORDER BY block_height ASC, block_hash ASC), '[]'::json) -FROM ( - SELECT block_hash, block_height, parent_hash - FROM qbit_pool_blocks - WHERE maturity_state = 'immature' - AND block_height <= {depth_ceiling} - AND chain_state = 'prepared' - ORDER BY block_height ASC, block_hash ASC - LIMIT {int(limit)} -) stranded; -""" - with self._operation_gate(self._lock, "writer lock"): - rows = self._run_retry_safe_read_json(sql) - for row in rows: - row["block_height"] = int(row["block_height"]) - return rows - - def mark_pool_block_inactive(self, *, block_hash: str, active_tip_height: int) -> dict[str, int | str]: - sql = f""" -SELECT json_build_object( - 'backend', 'postgres-psql', - 'inactive_count', qbit_mark_pool_block_inactive( - {self._text_literal(block_hash)}, - {int(active_tip_height)}, - {self._text_literal(self._writer_id)}, - {int(self._writer_epoch)}, - {self._text_literal(self._writer_session_token)}, - {self._lease_interval_sql} - ) -); -""" - result = self._run_fenced_json(sql) - if "error" in result: - raise RuntimeError(str(result["error"])) - return { - "backend": str(result["backend"]), - "inactive_count": int(result["inactive_count"]), - } - - def reactivate_pool_block(self, *, block_hash: str, active_tip_height: int) -> dict[str, int | str]: - sql = f""" -SELECT json_build_object( - 'backend', 'postgres-psql', - 'reactivated_count', qbit_reactivate_pool_block( - {self._text_literal(block_hash)}, - {int(active_tip_height)}, - {self._text_literal(self._writer_id)}, - {int(self._writer_epoch)}, - {self._text_literal(self._writer_session_token)}, - {self._lease_interval_sql} - ) -); -""" - with self._operation_gate(self._lock, "writer lock"): - result = self._run_json(sql) - if "error" in result: - raise RuntimeError(str(result["error"])) - reactivated_count = int(result["reactivated_count"]) - publication_sequence: object | None = None - if reactivated_count == 1: - # Same two-statements-under-one-gate shape as - # confirm_accepted_block: the mutating statement's command - # snapshot cannot see the ordinal it just assigned, so the - # read runs next while the writer lock is still held. Report - # the completed first statement so a landing caller's monitor - # is not asked to sit through both budgets in silence. - self._note_operation_progress() - state = self._run_retry_safe_read_json( - f""" -SELECT json_build_object( - 'audit_publication_sequence', ( - SELECT audit_publication_sequence - FROM qbit_pool_blocks - WHERE block_hash = {self._text_literal(block_hash)} - AND block_height <= {int(active_tip_height)} - AND chain_state = 'confirmed' - AND maturity_state = 'immature' - ) -); -""" - ) - publication_sequence = state.get("audit_publication_sequence") - if publication_sequence is None: - raise RuntimeError( - "reactivated pool block has no audit publication sequence" - ) - response: dict[str, int | str] = { - "backend": str(result["backend"]), - "reactivated_count": reactivated_count, - } - if publication_sequence is not None: - response["audit_publication_sequence"] = int(publication_sequence) - return response - - def mark_mature_pool_payouts(self, *, active_tip_height: int) -> dict[str, int | str]: - sql = f""" -WITH lease AS ( - UPDATE qbit_ledger_writer_lease - SET lease_expires_at = clock_timestamp() + {self._lease_interval_sql}, - updated_at = clock_timestamp() - WHERE singleton - AND writer_id = {self._text_literal(self._writer_id)} - AND writer_epoch = {int(self._writer_epoch)} - AND writer_session_token = {self._text_literal(self._writer_session_token)} - RETURNING writer_id -) -SELECT CASE - WHEN (SELECT count(*) FROM lease) = 0 THEN - json_build_object('error', 'writer lease is not active') - ELSE - json_build_object( - 'backend', 'postgres-psql', - 'matured_count', qbit_mark_mature_pool_payouts({int(active_tip_height)}) - ) -END; -""" - result = self._run_fenced_json(sql) - if "error" in result: - raise RuntimeError(str(result["error"])) - return { - "backend": str(result["backend"]), - "matured_count": int(result["matured_count"]), - } - - def __len__(self) -> int: - sql = "SELECT json_build_object('count', count(*)) FROM qbit_share_ledger WHERE accepted;" - with self._operation_gate(self._lock, "writer lock"): - return int(self._run_retry_safe_read_json(sql)["count"]) - - def _run_candidate_payload_json(self, payload: Mapping[str, Any], sql_body: str) -> Any: - def pieces() -> Iterator[str]: - yield "\nWITH payload AS (\n SELECT " - yield from self._accepted_block_payload_literal_pieces(payload) - yield " AS data\n),\n" - yield sql_body - - # Existing in-memory SQL test/embedding adapters explicitly replace - # execution; preserve their string-based seam. Real backends spool. - if (getattr(self._run_fenced_json, "__func__", None) is not PsqlShareLedger._run_fenced_json - or getattr(self._run_json, "__func__", None) is not PsqlShareLedger._run_json - or getattr(self._run_sql, "__func__", None) is not PsqlShareLedger._run_sql - or (getattr(self, "_native", None) is not None - and not isinstance(self._native, _NativePostgresClient))): - return self._run_fenced_json("".join(pieces())) - from lab.prism.statement_spool import run_fenced_statement - return run_fenced_statement(self, pieces()) - - def _run_fenced_json(self, sql: str) -> Any: - with self._operation_gate(self._lock, "writer lock"): - return self._run_json(sql) - - def _run_read_json(self, sql: str) -> Any: - with self._operation_gate(self._read_semaphore, "read slot"): - return self._run_retry_safe_read_json(sql) - - def _run_attributed_read_json( - self, - sql: str, - *, - operation: str, - gate: Any, - gate_name: str, - ) -> Any: - """Run one gated single-value read, timing admission apart from execution. - - See ``_run_attributed_read`` for the bookkeeping; this is that helper - with ``_run_retry_safe_read_json`` as the statement. - """ - return self._run_attributed_read( - sql, - operation=operation, - gate=gate, - gate_name=gate_name, - execute=self._run_retry_safe_read_json, - ) - - def _run_attributed_read_json_rows( - self, - sql: str, - *, - operation: str, - gate: Any, - gate_name: str, - row_converter: Callable[[Any], Any] | None = None, - ) -> list[Any]: - """Run one gated row-result read, timing admission apart from execution. - - The row-result twin of ``_run_attributed_read_json`` (issue #236): - the same gate, taken and released the same way, the same admission - and execution samples under the same ``operation`` name, with - ``_run_retry_safe_read_json_rows`` as the statement. Execution time - still covers the whole statement including local decoding, exactly - as the single-value read's did when its decoding was one call. - """ - - def execute( - statement: str, - *, - on_statement_start: Callable[[], None] | None = None, - ) -> list[Any]: - return self._run_retry_safe_read_json_rows( - statement, - row_converter=row_converter, - on_statement_start=on_statement_start, - ) - - return self._run_attributed_read( - sql, - operation=operation, - gate=gate, - gate_name=gate_name, - execute=execute, - ) - - def _run_attributed_read( - self, - sql: str, - *, - operation: str, - gate: Any, - gate_name: str, - execute: Callable[..., Any], - ) -> Any: - """Run one gated read query, timing admission apart from execution. - - Identical to wrapping ``_operation_gate(gate, gate_name)`` around - ``execute`` (``_run_retry_safe_read_json`` or its row-result twin) - in what it acquires and what it executes -- the caller's own - admission primitive, the same retry-safe statement, no extra - connection and no extra thread -- and different only in what it - records. - - The gate is the caller's and stays the caller's. A read that takes the - bounded read semaphore keeps taking it; a read that takes the writer - lock keeps taking it. Attribution is a measurement, and a measurement - that quietly moved a statement between admission classes would change - which writes it serializes against -- the one property the rest of the - ledger reasons about. ``gate_name`` is the same string - ``_operation_gate`` would have been given, so an admission that - expires still names the primitive it was waiting for. - - ``operation`` is written at each call site as a bare string literal - taken from ``PRISM_LEDGER_READ_OPERATIONS`` in - ``lab.prism.accepted_preview_telemetry``, and deliberately not as an - imported constant: ``tests.test_prism_accepted_preview_telemetry`` - harvests these literals out of this file's source to fail the moment - a call site attributes a read under a name the closed vocabulary does - not carry. Importing the names instead would leave that check with - nothing to read. A name that escaped anyway is still bounded -- - ``fold_ledger_read_stats`` folds it into ``other`` before rendering, - so a drifted call site costs an attribution, never a new series. - - That record is the point. Issue #211 was diagnosed against an outer - call reporting ``replay-outbox-query exceeded 5s`` while the inner - PostgreSQL deadline still had seconds left and the server showed zero - blocked backends: the budget had gone to coordinator-local admission, - and nothing on ``/metrics`` said so. One duration covering both halves - cannot answer "database or convoy?", so the halves are counted - separately here and exported per operation. - - Execution time is measured across the whole statement including the - tail a server-cancelled statement spends returning, so a deadline that - expires inside PostgreSQL is attributed to PostgreSQL (cancel lag - included) rather than to the gate. - - Every exit path records, so a timed-out read is counted rather than - lost, and the gate is released in ``finally`` exactly as - ``_operation_gate`` releases it. - """ - gate_started = self._monotonic() - try: - self._acquire_operation_gate(gate, gate_name) - except BaseException as exc: - # Admission itself expired: no statement was ever sent, so there - # is no execution sample to record and the call counts as a gate - # timeout rather than a database one. - self._note_ledger_read_timing( - operation, - gate_wait_seconds=max(0.0, self._monotonic() - gate_started), - execute_seconds=None, - timed_out=isinstance(exc, TimeoutError), - ) - raise - gate_wait_seconds = max(0.0, self._monotonic() - gate_started) - execute_started: float | None = None - - def on_statement_start() -> None: - nonlocal execute_started - # A retry-safe native read may dispatch more than once after an - # ambiguous connection loss. Execution attribution begins at the - # first dispatch and includes the retry tail rather than resetting - # the clock for each attempt. - if execute_started is None: - execute_started = self._monotonic() - - timed_out = False - try: - return execute( - sql, - on_statement_start=on_statement_start, - ) - except BaseException as exc: - timed_out = isinstance(exc, TimeoutError) - raise - finally: - execute_seconds = ( - None - if execute_started is None - else max(0.0, self._monotonic() - execute_started) - ) - # Released before the record is taken: a bookkeeping failure must - # never leak the gate it was measuring. - gate.release() - self._note_ledger_read_timing( - operation, - gate_wait_seconds=gate_wait_seconds, - execute_seconds=execute_seconds, - timed_out=timed_out, - ) - - def _ensure_ledger_read_timings(self) -> Lock: - """Return the lock guarding this instance's read-timing record.""" - stats_lock = getattr(self, "_ledger_read_timings_lock", None) - if stats_lock is not None: - return stats_lock - with PsqlShareLedger._ledger_read_timings_bootstrap: - stats_lock = getattr(self, "_ledger_read_timings_lock", None) - if stats_lock is None: - self._ledger_read_timings = {} - stats_lock = Lock() - # Published last, so a caller that observes the lock also - # observes the dict the lock guards. - self._ledger_read_timings_lock = stats_lock - return stats_lock - - def _note_ledger_read_timing( - self, - operation: str, - *, - gate_wait_seconds: float, - execute_seconds: float | None, - timed_out: bool, - ) -> None: - stats_lock = self._ensure_ledger_read_timings() - gate_wait_seconds = max(0.0, float(gate_wait_seconds)) - with stats_lock: - stats = self._ledger_read_timings.setdefault( - operation, - { - "calls_total": 0, - "gate_wait_seconds_total": 0.0, - "gate_wait_seconds_max": 0.0, - "gate_timeouts_total": 0, - "execute_seconds_total": 0.0, - "execute_seconds_max": 0.0, - "execute_timeouts_total": 0, - }, - ) - stats["calls_total"] = int(stats["calls_total"]) + 1 - stats["gate_wait_seconds_total"] = ( - float(stats["gate_wait_seconds_total"]) + gate_wait_seconds - ) - stats["gate_wait_seconds_max"] = max( - float(stats["gate_wait_seconds_max"]), gate_wait_seconds - ) - if execute_seconds is None: - if timed_out: - stats["gate_timeouts_total"] = ( - int(stats["gate_timeouts_total"]) + 1 - ) - return - execute_seconds = max(0.0, float(execute_seconds)) - stats["execute_seconds_total"] = ( - float(stats["execute_seconds_total"]) + execute_seconds - ) - stats["execute_seconds_max"] = max( - float(stats["execute_seconds_max"]), execute_seconds - ) - if timed_out: - stats["execute_timeouts_total"] = ( - int(stats["execute_timeouts_total"]) + 1 - ) - - def ledger_read_gate_stats(self) -> dict[str, dict[str, float | int]]: - """Read-slot admission wait against SQL execution, by operation. - - The series this feeds exist so the *next* budget exhaustion is - attributable without a live debugging session: a rising - ``gate_wait_seconds_max`` is coordinator-local contention, a rising - ``execute_seconds_max`` is PostgreSQL. See - ``_run_attributed_read_json``. - """ - stats_lock = self._ensure_ledger_read_timings() - with stats_lock: - return { - operation: dict(stats) - for operation, stats in self._ledger_read_timings.items() - } - - def _run_retry_safe_read_json( - self, - sql: str, - *, - on_statement_start: Callable[[], None] | None = None, - ) -> Any: - native = getattr(self, "_native", None) - if native is not None: - timeout_seconds = self._remaining_operation_timeout() - run_kwargs: dict[str, Any] = {"retry_safe": True} - if on_statement_start is not None: - # The native client borrows or creates its connection before - # firing this signal, and rechecks the deadline afterward. - # Connection setup expiry is therefore local admission, not - # statement execution that never happened. - run_kwargs["on_statement_start"] = on_statement_start - if timeout_seconds is None: - return native.run_json(sql, **run_kwargs) - run_kwargs["timeout_seconds"] = timeout_seconds - return native.run_json(sql, **run_kwargs) - run_json = self._run_json - if getattr(run_json, "__func__", None) is PsqlShareLedger._run_json: - return run_json(sql, on_statement_start=on_statement_start) - # Test and embedding subclasses have historically overridden this - # private seam with the one-argument signature. Preserve that - # compatibility while treating entry into their replacement as the - # only observable statement-start boundary they expose. - if on_statement_start is not None: - on_statement_start() - return run_json(sql) - - # Batch size for the row-result read path; an instance attribute so a - # test can shrink it to exercise batch boundaries with a handful of rows. - _json_row_batch_size: int = PAYOUT_WINDOW_ROW_BATCH_SIZE - - # Replay header rows describe bodies that live in storage; the block - # submitter hydrates them one at a time at dequeue, never during the - # startup enumeration (#255). - candidate_hydration_deferred: ClassVar[bool] = True - - def _run_retry_safe_read_json_rows( - self, - sql: str, - *, - row_converter: Callable[[Any], Any] | None = None, - on_statement_start: Callable[[], None] | None = None, - ) -> list[Any]: - """Run one read-only statement that yields one JSON value per row. - - The row-result counterpart of ``_run_retry_safe_read_json`` (issue - #236), and the only way the payout-window snapshot and delta reach a - backend. The statement is one SELECT; each backend decodes its rows - in ``_json_row_batch_size`` batches and, between batches, rechecks - the caller's absolute operation deadline and stamps liveness - (``_note_json_row_batch``), so a window of any size is decoded in - bounded calls and a deadline that expires mid-decode fails the read - rather than returning late. - - Backend selection mirrors ``_run_retry_safe_read_json`` seam for - seam, so no backend keeps a whole-window aggregate by accident: - - * the native client runs ``run_json_rows`` (retry-safe, under the - same remaining deadline, with the same statement-start signal); - * a subclass that overrides the private ``_run_json`` seam stands in - for the whole server and answers a row-result statement with the - list of row values -- the same Python shape ``json_agg`` decoded - to, which is what the existing fakes already return -- converted - here in batches; - * a subclass that overrides the private ``_run_sql`` seam returns - psql's text, one JSON value per line, parsed line by line; - * the shipped subprocess backend spools psql's output and parses it - line by line (``_run_psql_json_rows``). - """ - native = getattr(self, "_native", None) - if native is not None: - timeout_seconds = self._remaining_operation_timeout() - run_kwargs: dict[str, Any] = { - "retry_safe": True, - "row_converter": row_converter, - "batch_size": self._json_row_batch_size, - "on_batch": self._note_json_row_batch, - } - if on_statement_start is not None: - run_kwargs["on_statement_start"] = on_statement_start - if timeout_seconds is not None: - run_kwargs["timeout_seconds"] = timeout_seconds - return native.run_json_rows(sql, **run_kwargs) - run_json = self._run_json - if getattr(run_json, "__func__", None) is not PsqlShareLedger._run_json: - timeout_seconds = self._remaining_operation_timeout() - deadline = ( - None - if timeout_seconds is None - else self._monotonic() + timeout_seconds - ) - if on_statement_start is not None: - on_statement_start() - rows = run_json(sql) - if not isinstance(rows, list): - raise RuntimeError( - "row-result statement seam returned a non-list value: " - f"{type(rows).__name__}" - ) - return self._convert_json_rows( - rows, - row_converter=row_converter, - deadline=deadline, - ) - run_sql = self._run_sql - if getattr(run_sql, "__func__", None) is not PsqlShareLedger._run_sql: - timeout_seconds = self._remaining_operation_timeout() - deadline = ( - None - if timeout_seconds is None - else self._monotonic() + timeout_seconds - ) - if on_statement_start is not None: - on_statement_start() - output = run_sql(sql) - return self._decode_json_lines( - output.splitlines(), - row_converter=row_converter, - deadline=deadline, - ) - return self._run_psql_json_rows( - sql, - row_converter=row_converter, - on_statement_start=on_statement_start, - ) - - def _note_json_row_batch(self) -> None: - """Between two decode batches: recheck the deadline, stamp liveness. - - The native client's ``on_batch`` hook. ``_remaining_operation_timeout`` - raises once the caller's absolute operation deadline has passed, so - a decode that outlives its budget fails at the next batch boundary - instead of completing late; the progress stamp is the same one a - multi-statement gate body makes between statements, and tells a - liveness monitor the read is advancing rather than wedged. - """ - self._remaining_operation_timeout() - self._note_operation_progress() - - def _check_json_row_deadline(self, deadline: float | None) -> None: - """Raise once the statement deadline or the operation deadline has passed. - - Called before local decoding starts, between batches, and before a - result is published -- including after a trailing partial batch and - for a result smaller than one batch -- so a read never returns - successfully after its budget merely because the last batch was - short. ``deadline`` is the per-statement absolute deadline computed - at dispatch from the same remaining budget the statement ran under; - ``_remaining_operation_timeout`` covers the caller's absolute - operation deadline, which is the same instant in production and can - differ only under a test's virtual clock. - """ - if deadline is not None and self._monotonic() >= deadline: - raise LedgerOperationTimeout( - "statement deadline expired while decoding rows" - ) - self._remaining_operation_timeout() - - def _convert_json_rows( - self, - rows: Sequence[Any], - *, - row_converter: Callable[[Any], Any] | None, - deadline: float | None, - ) -> list[Any]: - """Convert already-decoded row values in bounded batches. - - Deadline-checked before the first batch, between batches (where - liveness is also stamped) and before the result is returned. - """ - batch_size = max(1, int(self._json_row_batch_size)) - results: list[Any] = [] - self._check_json_row_deadline(deadline) - for start in range(0, len(rows), batch_size): - if start: - self._check_json_row_deadline(deadline) - self._note_operation_progress() - batch = rows[start : start + batch_size] - if row_converter is None: - results.extend(batch) - else: - results.extend(row_converter(value) for value in batch) - self._check_json_row_deadline(deadline) - return results - - def _decode_json_lines( - self, - lines: Iterable[bytes | str], - *, - row_converter: Callable[[Any], Any] | None, - deadline: float | None, - ) -> list[Any]: - """Parse one JSON value per line, in bounded batches. - - Blank lines are skipped: psql prints nothing at all for an empty - result and a trailing newline otherwise. Every other line must be - one complete JSON value -- with ``--tuples-only --no-align`` a row is - one line, and PostgreSQL's JSON output escapes control characters, - so no row spans lines -- and the values are never reassembled into - an array for a single ``json.loads``. The statement deadline computed - at dispatch and the caller's operation deadline are both checked - before decoding starts, between batches (where liveness is stamped) - and before the result is returned, so a trailing partial batch or a - result shorter than one batch cannot publish after the budget; the - native client keeps the same cadence. - """ - batch_size = max(1, int(self._json_row_batch_size)) - results: list[Any] = [] - in_batch = 0 - line_number = 0 - self._check_json_row_deadline(deadline) - for raw_line in lines: - line_number += 1 - line = raw_line.strip() - if not line: - continue - try: - if isinstance(line, bytes): - line = line.decode("utf-8") - value = json.loads(line) - except ValueError as exc: - raise RuntimeError( - "psql row-result statement returned malformed JSON on " - f"output line {line_number}: {exc}" - ) from exc - results.append(value if row_converter is None else row_converter(value)) - in_batch += 1 - if in_batch >= batch_size: - in_batch = 0 - self._check_json_row_deadline(deadline) - self._note_operation_progress() - self._check_json_row_deadline(deadline) - return results - - def _ensure_writer_lease(self) -> None: - while True: - acquire_started_monotonic = self._monotonic() - result = self._try_acquire_writer_lease() - if result.get("acquired"): - self._writer_lease_last_refresh_monotonic = ( - acquire_started_monotonic - ) - return - if self._can_adopt_writer_lease(result): - observed_session = str(result["writer_session_token"]) - adoption_started_monotonic = self._monotonic() - adoption = self._try_adopt_writer_lease(result) - if adoption.get("acquired"): - self._writer_lease_last_refresh_monotonic = ( - adoption_started_monotonic - ) - print( - "prism ledger writer lease adopted from same-identity " - f"predecessor session={observed_session}", - flush=True, - ) - return - # A CAS loss may mean the predecessor renewed concurrently or - # another replacement won. Re-observe the returned owner and - # require a fresh full silence interval before another CAS; - # permanently refusing this token would recreate the TTL - # outage if that renewal were its final act before dying. - result = adoption - if not self._can_wait_for_writer_lease(result): - raise RuntimeError( - "qbit ledger writer lease is held by " - f"{result.get('writer_id')} epoch={result.get('writer_epoch')} " - f"session={result.get('writer_session_token')} " - f"until {result.get('lease_expires_at')}" - ) - wait_seconds = max(0.0, float(result.get("lease_wait_seconds") or 0.0)) - adoption_wait_seconds = self._writer_lease_adoption_wait_seconds(result) - if adoption_wait_seconds is not None: - wait_seconds = min(wait_seconds, adoption_wait_seconds) - sleep_seconds = min( - self._lease_retry_max_sleep_seconds, - max(self._lease_retry_min_sleep_seconds, wait_seconds), - ) - print( - "prism ledger writer lease held until " - f"{result.get('lease_expires_at')}; waiting {sleep_seconds:.3g}s before retry " - f"(holder writer={result.get('writer_id')} epoch={result.get('writer_epoch')} " - f"session={result.get('writer_session_token')})", - flush=True, - ) - self._lease_retry_sleep(sleep_seconds) - - def _run_lease_acquisition_json(self, sql: str, description: str) -> Any: - """Run one lease-acquisition statement under a bounded lock deadline. - - The startup lease upsert and the adoption CAS both row-lock the - qbit_ledger_writer_lease singleton. Unbounded, either statement - queues behind whatever transaction already holds that row — in the - worst case an orphaned idle-in-transaction backend of a vanished - predecessor — inside PsqlShareLedger.__init__, before the - coordinator's watchdog arms, for as long as the kernel keeps the - dead peer's socket alive. Each attempt therefore runs under the - configured lock/statement deadline (statement_timeout arms both - SET LOCAL statement_timeout and lock_timeout on the native backend, - and the PGOPTIONS equivalents plus a subprocess timeout on psql). - - The retry budget (5 attempts x 5s by default) is sized to outlast a - typical orphan reap rather than to guarantee one: the session guards' - idle_in_transaction_session_timeout runs from the moment the blocking - transaction went idle, not from the moment this process started - retrying, so an orphan frequently clears partway through the budget - and a later attempt lands the lease with no operator action. A - statement still hitting its deadline on every attempt becomes the - fatal RuntimeError, which __init__'s close-and-reraise turns into a - visible process exit the supervisor can restart, instead of a silent - multi-hour hang. - - A deadline expiry names the lock conflict first because it is by far - the most common cause, but it is not proof of one: a connect timeout, - an exhausted pool slot, and a healthy-but-overloaded server all - surface as the same LedgerOperationTimeout. Both the per-attempt line - and the fatal error therefore quote the underlying exception and the - fatal error chains it, so the operator diagnoses from the cause rather - than from this layer's guess. - """ - attempts = self._lease_acquire_attempts - lock_timeout_seconds = self._lease_acquire_lock_timeout_seconds - cause: LedgerOperationTimeout | None = None - for attempt in range(1, attempts + 1): - try: - with self.statement_timeout(lock_timeout_seconds): - return self._run_json(sql) - except LedgerOperationTimeout as exc: - cause = exc - print( - f"prism ledger {description}: attempt {attempt}/{attempts} " - f"did not complete within its {lock_timeout_seconds:g}s " - "deadline (commonly the qbit_ledger_writer_lease row is " - "lock-blocked by another transaction, but an unreachable " - f"or overloaded server produces the same signal): {exc}", - flush=True, - ) - if attempt < attempts: - self._lease_retry_sleep(self._lease_retry_min_sleep_seconds) - raise RuntimeError( - f"qbit ledger {description} did not complete within its " - f"{lock_timeout_seconds:g}s deadline on any of {attempts} attempts " - "(commonly the qbit_ledger_writer_lease row is lock-blocked by " - "another transaction, but an unreachable or overloaded server " - f"produces the same signal): {cause}; the coordinator is exiting " - "rather than blocking startup indefinitely" - ) from cause - - def _try_acquire_writer_lease(self) -> dict[str, Any]: - """Claim, renew, or observe the writer lease in one statement. - - The statement is total: every outcome is a JSON object, so nothing - here can reach ``parse_single_json_value``'s NULL branch and surface a - raw driver error out of ``PsqlShareLedger.__init__``. Two arms are the - ordinary ones -- this identity took the lease, or someone else holds - it -- and the third exists because both can be empty at once during - the first-ever concurrent acquisition (see - WRITER_LEASE_ACQUIRE_RETRY_KEY). - - That third arm is handled here rather than by the caller because the - remedy is local to this statement: it carries no holder to wait on or - adopt, and only re-running gets the fresh READ COMMITTED snapshot in - which the winner's committed row is visible. The retry therefore - converges to one of the two ordinary arms, and ``_ensure_writer_lease`` - never sees the sentinel. - """ - payload = { - "writer_id": self._writer_id, - "writer_epoch": self._writer_epoch, - "writer_session_token": self._writer_session_token, - } - sql = f""" -WITH payload AS ( - SELECT {self._jsonb_literal(payload)} AS data -), -upsert AS ( -INSERT INTO qbit_ledger_writer_lease ( - singleton, - writer_id, - writer_epoch, - writer_session_token, - lease_expires_at -) -SELECT - true, - data->>'writer_id', - (data->>'writer_epoch')::bigint, - data->>'writer_session_token', - clock_timestamp() + {self._lease_interval_sql} -FROM payload -ON CONFLICT (singleton) DO UPDATE -SET writer_id = EXCLUDED.writer_id, - writer_epoch = EXCLUDED.writer_epoch, - writer_session_token = EXCLUDED.writer_session_token, - lease_expires_at = EXCLUDED.lease_expires_at, - updated_at = clock_timestamp() -WHERE ( - qbit_ledger_writer_lease.writer_id = EXCLUDED.writer_id - AND qbit_ledger_writer_lease.writer_epoch = EXCLUDED.writer_epoch - AND qbit_ledger_writer_lease.writer_session_token = EXCLUDED.writer_session_token - ) - OR qbit_ledger_writer_lease.lease_expires_at <= clock_timestamp() -RETURNING writer_id, writer_epoch, writer_session_token -) -SELECT COALESCE( - ( - SELECT json_build_object( - 'acquired', true, - 'writer_id', writer_id, - 'writer_epoch', writer_epoch, - 'writer_session_token', writer_session_token - ) - FROM upsert - ), - ( - SELECT json_build_object( - 'acquired', false, - 'writer_id', writer_id, - 'writer_epoch', writer_epoch, - 'writer_session_token', writer_session_token, - 'lease_expires_at', lease_expires_at::text, - 'lease_updated_at', updated_at::text, - 'lease_age_seconds', GREATEST( - 0, - EXTRACT(EPOCH FROM (clock_timestamp() - updated_at)) - ), - 'lease_wait_seconds', GREATEST( - 0, - EXTRACT(EPOCH FROM (lease_expires_at - clock_timestamp())) - ) - ) - FROM qbit_ledger_writer_lease - WHERE singleton - ), - json_build_object( - 'acquired', false, - '{WRITER_LEASE_ACQUIRE_RETRY_KEY}', true, - 'lease', '{WRITER_LEASE_ACQUIRE_RETRY_SUBJECT}', - 'retry_reason', - 'the {WRITER_LEASE_ACQUIRE_RETRY_SUBJECT} row was committed by a concurrent first acquisition after this statement snapshot' - ) -); -""" - for attempt in range(1, WRITER_LEASE_ACQUIRE_RETRY_ATTEMPTS + 1): - result = self._run_lease_acquisition_json( - sql, - "writer lease acquisition", - ) - if not isinstance(result, dict): - raise RuntimeError("psql writer lease query returned non-object JSON") - if not result.get(WRITER_LEASE_ACQUIRE_RETRY_KEY): - return result - print( - "prism ledger writer lease acquisition raced a concurrent first " - f"acquisition (attempt {attempt}/{WRITER_LEASE_ACQUIRE_RETRY_ATTEMPTS}); " - "retrying on a fresh statement snapshot", - flush=True, - ) - raise RuntimeError( - f"{WRITER_LEASE_ACQUIRE_RETRY_SUBJECT} acquisition still could not see " - "the lease row committed by a concurrent first acquisition after " - f"{WRITER_LEASE_ACQUIRE_RETRY_ATTEMPTS} fresh statement snapshots; " - "the coordinator is exiting rather than starting without the lease" - ) - - def _can_adopt_writer_lease(self, result: dict[str, Any]) -> bool: - wait_seconds = self._writer_lease_adoption_wait_seconds(result) - return wait_seconds is not None and wait_seconds <= 0 - - def _writer_lease_adoption_wait_seconds( - self, - result: dict[str, Any], - ) -> float | None: - """Return time until one heartbeat-capable predecessor is adoptable. - - A timestamp CAS alone only orders database mutations; it does not prove - that a live, idle predecessor cannot still perform an external wallet - or node RPC before its next fenced write. Fast adoption is therefore - restricted to coordinator sessions guarded by the same writer/epoch - PostgreSQL advisory lock. This process cannot reach this method until - that predecessor's guard session is gone. PostgreSQL must additionally - report the exact session unchanged for a full silence interval before - the CAS, giving a holder whose guard connection failed time to self-exit. - A renewing live twin retains the guard and cannot reach lease polling. - - The silence interval is measured from two independent edges and the - later one wins. The row edge (``updated_at`` age) alone is unsafe: a - long fenced transaction such as ``persist_accepted_block`` withholds - the predecessor's ``updated_at`` refresh until commit, so the row can - already look minutes silent the instant the predecessor dies. The - guard edge therefore requires a full interval to elapse after *this* - process acquired the advisory guard, guaranteeing the predecessor its - whole heartbeat failure budget to self-fence after losing the guard, - no matter how stale the row already is. - """ - observed_session = result.get("writer_session_token") - if not ( - self.writer_lease_fast_adoption_capable - and self._can_wait_for_writer_lease(result) - and isinstance(observed_session, str) - and observed_session.startswith(WRITER_LEASE_HEARTBEAT_SESSION_PREFIX) - and observed_session != self._writer_session_token - and result.get("lease_updated_at") is not None - ): - return None - guard_acquired_monotonic = getattr( - self, - "_writer_lease_guard_acquired_monotonic", - None, - ) - if guard_acquired_monotonic is None: - return None - try: - age_seconds = float(result.get("lease_age_seconds")) - except (TypeError, ValueError): - return None - if not math.isfinite(age_seconds) or age_seconds < 0: - return None - row_wait_seconds = max( - 0.0, - self._lease_adoption_silence_seconds - age_seconds, - ) - guard_held_seconds = max( - 0.0, - self._monotonic() - guard_acquired_monotonic, - ) - guard_wait_seconds = max( - 0.0, - self._lease_adoption_silence_seconds - guard_held_seconds, - ) - return max(row_wait_seconds, guard_wait_seconds) - - def _try_adopt_writer_lease(self, observed: dict[str, Any]) -> dict[str, Any]: - """Fence and replace one observed same-identity predecessor session. - - The caller first proves the heartbeat-capable predecessor has been - silent for the configured interval. PostgreSQL row-lock ordering then - makes this compare-and-swap the final database fence: a predecessor - renewal that wins first changes ``updated_at`` and makes this update - affect zero rows; if this update wins first, every later predecessor - mutation sees a different session and is fenced out. - """ - payload = { - "writer_id": self._writer_id, - "writer_epoch": self._writer_epoch, - "writer_session_token": self._writer_session_token, - "observed_writer_session_token": observed.get("writer_session_token"), - "observed_lease_updated_at": observed.get("lease_updated_at"), - } - sql = f""" -WITH payload AS ( - SELECT {self._jsonb_literal(payload)} AS data -), -adopted AS ( - UPDATE qbit_ledger_writer_lease - SET writer_session_token = data->>'writer_session_token', - lease_expires_at = clock_timestamp() + {self._lease_interval_sql}, - updated_at = clock_timestamp() - FROM payload - WHERE qbit_ledger_writer_lease.singleton - AND qbit_ledger_writer_lease.writer_id = data->>'writer_id' - AND qbit_ledger_writer_lease.writer_epoch = (data->>'writer_epoch')::bigint - AND qbit_ledger_writer_lease.writer_session_token = data->>'observed_writer_session_token' - AND qbit_ledger_writer_lease.updated_at = (data->>'observed_lease_updated_at')::timestamptz - AND qbit_ledger_writer_lease.lease_expires_at > clock_timestamp() - RETURNING writer_id, writer_epoch, writer_session_token -) -SELECT COALESCE( - ( - SELECT json_build_object( - 'acquired', true, - 'adopted', true, - 'writer_id', writer_id, - 'writer_epoch', writer_epoch, - 'writer_session_token', writer_session_token - ) - FROM adopted - ), - ( - SELECT json_build_object( - 'acquired', false, - 'adopted', false, - 'writer_id', writer_id, - 'writer_epoch', writer_epoch, - 'writer_session_token', writer_session_token, - 'lease_expires_at', lease_expires_at::text, - 'lease_updated_at', updated_at::text, - 'lease_age_seconds', GREATEST( - 0, - EXTRACT(EPOCH FROM (clock_timestamp() - updated_at)) - ), - 'lease_wait_seconds', GREATEST( - 0, - EXTRACT(EPOCH FROM (lease_expires_at - clock_timestamp())) - ) - ) - FROM qbit_ledger_writer_lease - WHERE singleton - ) -); -""" - result = self._run_lease_acquisition_json(sql, "writer lease adoption") - if not isinstance(result, dict): - raise RuntimeError("psql writer lease adoption query returned non-object JSON") - return result - - def _can_wait_for_writer_lease(self, result: dict[str, Any]) -> bool: - try: - holder_epoch = int(result.get("writer_epoch")) - except (TypeError, ValueError): - return False - return ( - result.get("writer_id") == self._writer_id - and holder_epoch == self._writer_epoch - and result.get("lease_expires_at") is not None - ) - - def renew_writer_lease(self) -> dict[str, int | str]: - """Refresh this writer's lease without touching any ledger rows. - - The lease is normally refreshed as a side effect of every fenced - write. A daemon pass that produced no writes calls this instead, so an - otherwise-idle writer's lease does not sit expired. Raises when the - exact ``(writer_id, writer_epoch, writer_session_token)`` no longer - holds the lease, matching the fenced-write failure mode so a fenced-out - writer still fails fast. - """ - return self._renew_writer_lease_with(self._run_fenced_json) - - def prove_writer_lease_guard_session( - self, - *, - on_query_start: Callable[[], None] | None = None, - on_statement_end: Callable[[], None] | None = None, - on_statement_server_seconds: Callable[[float | None], None] | None = None, - ) -> dict[str, Any]: - """Prove ownership on one cheap read-only statement; never renew. - - The frequent half of the split issue #212 asked for. Ownership and - TTL renewal are two different questions asked at two very different - rates: the heartbeat must prove *ownership* several times inside one - adoption-silence window, but the lease TTL is 60s and only needs a - writer-side refresh well before it lapses (every fenced write - refreshes it too). Paying for the renewal question on every beat is - what made the frequent statement expensive: ``FOR NO KEY UPDATE SKIP - LOCKED`` on the hot lease tuple, a ``pg_stat_activity`` scan to - attribute the tuple's locker, and — in the ambiguous case — a second - statement. Under a rapid-block burst that statement approached the - guard's 500ms statement timeout, which is exactly how a healthy - coordinator ran out of server-proven envelope. - - This statement asks only the ownership question, and asks it with - two ``EXISTS`` reads: - - * PostgreSQL still shows *this* backend holding the writer/epoch - advisory lock, and - * the last committed lease row still names this exact - ``(writer_id, writer_epoch, writer_session_token)``. - - Those are the same two conditions - :meth:`verify_writer_lease_guard_session` raises on, evaluated the - same way, so the exact-session guarantee is unchanged: a guard - session that died, or an identity that was fenced out, still raises - here and still hard-exits the coordinator. What is *not* asked is - anything that can block or that needs live backend state, so the - statement takes no row lock, never queues behind a fenced write, and - never needs an attribution recheck — it is one round trip, always. - - The lease-expiry guarantee is preserved by escalation rather than by - renewal. The result reports ``lease_renewal_due`` whenever the - committed row's remaining validity has fallen to the own-write - authority margin (which ``_resolve_lease_authority_margin_seconds`` - keeps at or above half the TTL, and strictly below it), and - ``lease_expired`` when it has lapsed outright — an expired row is - always also renewal-due. The heartbeat answers either flag by - running the full renewing verification immediately, on the same - beat. So the cheap proof short-circuits only while the lease is - comfortably valid, which is precisely the window in which the full - verification had nothing to decide; the moment renewal actually - matters — including for the whole of a long own fenced write that - keeps skipping renewals — every beat runs the same fail-closed - verification it ran before this split existed. - - Never call this in place of the verification before an external - side effect. A proof is liveness and identity, not authority: it - deliberately does not renew, so it cannot distinguish a row this - writer's own in-flight write will refresh from one it will roll - back. :meth:`require_fresh_lease_for_external_side_effect` keeps - using the full verification for that reason. - - ``on_query_start`` fires once the guarded session's serialized query - slot is acquired and ``on_statement_end`` once the round trip - returns, so a caller can attribute queue wait and server time - separately. - - ``on_statement_server_seconds`` receives, for the same round trip, - the execution time the statement measured on the server - (``clock_timestamp() - statement_timestamp()``, returned as a - column of the statement itself, so it costs no extra round trip - and takes no lock), or ``None`` when the answer did not carry one. - Issue #227: ``on_statement_end`` is a client-side instant, so a GIL - stall between PostgreSQL answering and this thread resuming is - indistinguishable from database time without it. It is timing - metadata only — the server-proven edge is still the send edge. - """ - if not self.writer_lease_fast_adoption_capable: - raise RuntimeError("writer session is not heartbeat-capable") - guard = self._writer_lease_guard - if guard is None: - raise RuntimeError("postgres writer lease guard is not held") - payload = { - "writer_id": self._writer_id, - "writer_epoch": self._writer_epoch, - "writer_session_token": self._writer_session_token, - } - lock_key = _writer_lease_advisory_lock_key( - self._writer_id, - self._writer_epoch, - ) - lock_classid = (lock_key >> 32) & 0xFFFFFFFF - lock_objid = lock_key & 0xFFFFFFFF - sql = f""" -WITH payload AS ( - SELECT {self._jsonb_literal(payload)} AS data -) -SELECT json_build_object( - 'backend', 'postgres-psql', - 'guard_advisory_lock_held', EXISTS ( - SELECT 1 - FROM pg_locks - WHERE locktype = 'advisory' - AND granted - AND pid = pg_backend_pid() - AND classid = {lock_classid}::oid - AND objid = {lock_objid}::oid - AND objsubid = 1 - ), - 'writer_session_token_current', EXISTS ( - SELECT 1 - FROM qbit_ledger_writer_lease, payload - WHERE qbit_ledger_writer_lease.singleton - AND qbit_ledger_writer_lease.writer_id = data->>'writer_id' - AND qbit_ledger_writer_lease.writer_epoch = (data->>'writer_epoch')::bigint - AND qbit_ledger_writer_lease.writer_session_token = data->>'writer_session_token' - ), - 'lease_expired', EXISTS ( - SELECT 1 - FROM qbit_ledger_writer_lease, payload - WHERE qbit_ledger_writer_lease.singleton - AND qbit_ledger_writer_lease.writer_id = data->>'writer_id' - AND qbit_ledger_writer_lease.writer_epoch = (data->>'writer_epoch')::bigint - AND qbit_ledger_writer_lease.writer_session_token = data->>'writer_session_token' - AND qbit_ledger_writer_lease.lease_expires_at <= clock_timestamp() - ), - 'lease_renewal_due', EXISTS ( - SELECT 1 - FROM qbit_ledger_writer_lease, payload - WHERE qbit_ledger_writer_lease.singleton - AND qbit_ledger_writer_lease.writer_id = data->>'writer_id' - AND qbit_ledger_writer_lease.writer_epoch = (data->>'writer_epoch')::bigint - AND qbit_ledger_writer_lease.writer_session_token = data->>'writer_session_token' - AND qbit_ledger_writer_lease.lease_expires_at - <= clock_timestamp() + {self._lease_authority_margin_sql} - ), - '{GUARD_SERVER_EXECUTION_SECONDS_KEY}', {GUARD_SERVER_EXECUTION_SECONDS_SQL} -); -""" - run_kwargs: dict[str, Callable[..., None]] = {} - if on_query_start is not None: - run_kwargs["on_query_start"] = on_query_start - if on_statement_end is not None: - run_kwargs["on_statement_end"] = on_statement_end - if on_statement_server_seconds is not None: - run_kwargs["on_statement_result"] = ( - lambda statement_result: on_statement_server_seconds( - guard_server_execution_seconds(statement_result) - ) - ) - result = guard.run_json(sql, **run_kwargs) - if not isinstance(result, dict): - raise RuntimeError( - "psql writer lease guard proof returned non-object JSON" - ) - if not result.get("guard_advisory_lock_held"): - raise RuntimeError( - "postgres writer lease guard advisory lock is no longer held" - ) - if not result.get("writer_session_token_current"): - raise RuntimeError("writer lease is not active") - # An expired row is renewal-due by construction, but state it - # explicitly: a caller must never be able to read a stale committed - # row as "proved and nothing to do". - renewal_due = bool( - result.get("lease_renewal_due") or result.get("lease_expired") - ) - return { - "backend": str(result["backend"]), - "verified_count": 1, - "renewed_count": 0, - "proof_only": True, - "lease_expired": bool(result.get("lease_expired")), - "lease_renewal_due": renewal_due, - } - - def verify_writer_lease_guard_session( - self, - *, - on_query_start: Callable[[], None] | None = None, - on_statement_progress: Callable[[], None] | None = None, - on_statement_end: Callable[[], None] | None = None, - on_statement_server_seconds: Callable[[float | None], None] | None = None, - ) -> dict[str, int | str]: - """Prove the guard session live; renew the TTL only without waiting. - - The coordinator's periodic heartbeat and its external-side-effect - fence both run this on the dedicated guard connection. It must never - wait on the ``qbit_ledger_writer_lease`` tuple lock: every fenced - write (share appends, ``persist_accepted_block``) row-locks that - tuple for its whole transaction, and a guarded statement queued - behind it hits the guard's statement timeout and hard-exits a - healthy coordinator. Liveness is therefore proven by non-blocking - reads: the session answers, PostgreSQL still shows this backend - holding the writer advisory lock, and the last committed lease row - still names this exact session. Guard-connection loss or a - fenced-out session raises, which callers treat as loss of the - guarded session. - - The lease TTL still needs a writer-side refresh, or an idle - coordinator (no fenced writes, CTV broadcaster disabled) would let - ``lease_expires_at`` lapse and any different-identity claimant could - seize the singleton row through its expiry CAS — that identity uses - a different advisory-lock key, so this process's guard would not - block it. The same statement therefore renews the exact-identity - lease row with ``FOR NO KEY UPDATE SKIP LOCKED``: renewal happens on - every heartbeat while the tuple is uncontended and is skipped - without queueing while a fenced transaction holds it (that - transaction refreshes the TTL itself when it commits). - - A skipped renewal is only trustworthy while the committed row is - unexpired. Once ``lease_expires_at`` has lapsed, a lock we skipped - may be a different-identity ``_try_acquire_writer_lease`` taking the - row through its expiry CAS, and the stale committed snapshot would - still name this session until that claim commits. The locking - renewal used to catch exactly this by queueing and re-evaluating - after the claimant committed; the non-blocking spelling recovers it - by failing closed whenever the renewal was lock-blocked and the - committed row was already expired. - - The one lock-blocked expired case that must not fail closed is this - coordinator's own fenced write outlasting the TTL (a slow - ``persist_accepted_block``): it holds the tuple's exclusive lock, so - no expiry claim can be in flight behind it, and its commit refreshes - ``lease_expires_at`` before any queued claimant re-evaluates its - expiry CAS — raising would roll back a valid write and restart-loop - on every similarly slow block. The statement therefore attributes - the committed row's locker through ``pg_stat_activity``: the row's - ``xmax`` is the locker's transaction id, and a backend running that - transaction under this process's unique pool ``application_name`` - is this writer's own fenced write. Only an expired row locked by - anyone else raises. - - That attribution can misfire in one direction: the statement's - lease-row reads all share one MVCC snapshot, while - ``pg_stat_activity`` reports live backend state. An own fenced - write committing between the snapshot and the probe clears its - backend's ``backend_xid`` while the snapshot still shows the - expired row that write locked, so the probe reports no own locker - for a renewal that in fact just landed. A would-be fail-closed is - therefore re-verified once on a fresh statement: its new snapshot - sees the committed refresh (the renewal lands normally), a - completed takeover (the exact-identity match fails closed), or the - same contended expiry (still fails closed). One recheck is enough - — each further miss needs another independent commit to land - inside the next statement's snapshot-to-probe window, and every - residual outcome remains fail-closed. The recheck executes inside - the guard's already-held serialized query slot, so it cannot queue - behind other guard callers; it adds at most one more non-blocking - statement to this verification's execution. - - That tolerance proves liveness only, not authority to act outside - the database: the survival argument assumes the fenced write - commits, and a rollback instead releases the expired row unchanged, - letting a queued different-identity claimant take it immediately. - The result therefore reports ``renewal_deferred_to_own_write`` for - this state, and external-side-effect fences must withhold the - guarded RPC while it is set, retrying on their own cadence until a - verification lands a renewal (the write committed, refreshing the - TTL) or fails closed. The heartbeat may keep treating it as live. - - The deferral engages before expiry as well: an own-write skip over - a committed row with less than the authority margin remaining also - defers. An RPC authorized on a nearly-lapsed row can outlive it, - and from expiry onward its authority degenerates into the same - rollback-dependent argument — the TTL erodes exactly while a long - own write withholds renewals, so the runway shrinks precisely when - the write's fate is least certain. The margin is never below half - the lease TTL and is raised by the coordinator to cover its - longest fence-guarded RPC deadline plus fixed headroom (see - ``_resolve_lease_authority_margin_seconds``), so every authorized - effect's transmission deadline — and, within the headroom, the - node's application of a fully received request — lands inside the - committed row's remaining validity. A node that stalls longer - than the headroom after complete receipt is the documented - residual of preflight fencing between independent systems; no - client-side deadline can cancel a handler the node already - received. - Skips over a row with at least the margin remaining keep - authorizing on the committed row's own standalone validity: every - fenced commit refreshes the TTL, so steady write traffic never - erodes the runway, and deferring every own-write skip would - withhold submitblock and broadcasts behind saturated append - traffic — the submitter liveness the fast-lane submit path exists - to protect. - - ``on_query_start`` fires once the guarded session's serialized query - slot is acquired, letting callers budget queue wait separately from - statement execution. The attribution recheck happens within that - one slot acquisition, so it neither re-fires the callback nor adds - a queue wait to the caller's execution budget. - - ``on_statement_progress`` fires when the recheck decides to run a - second statement: the first statement's full round trip has - completed, which is exactly the session-answers evidence a - liveness monitor watches for. Callers whose staleness budgets are - sized for one statement (the heartbeat monitor's failure window) - stamp progress here so a lawful two-statement verification is not - mistaken for a wedged heartbeat, while a genuinely stuck statement - still produces no progress at all. - - ``on_statement_end`` is the general form of the same signal: it - fires after *every* completed round trip, including the last one, - which is what a caller attributing guard SQL time per phase needs. - A caller that supplies it does not also need - ``on_statement_progress``. - - ``on_statement_server_seconds`` fires alongside ``on_statement_end`` - for every completed round trip with the execution time that - statement measured on the server (``clock_timestamp() - - statement_timestamp()``, one more column of the same statement — - no extra round trip, no lock), or ``None`` when the answer did not - carry one. Issue #227: the end mark is a client-side instant, so - without the server's own figure a GIL stall between the answer - leaving PostgreSQL and this thread resuming is booked as database - time. Timing metadata only: the server-proven edge is still the - conservative send edge, never a server clock reading. - """ - if not self.writer_lease_fast_adoption_capable: - raise RuntimeError("writer session is not heartbeat-capable") - guard = self._writer_lease_guard - if guard is None: - raise RuntimeError("postgres writer lease guard is not held") - payload = { - "writer_id": self._writer_id, - "writer_epoch": self._writer_epoch, - "writer_session_token": self._writer_session_token, - "pool_application_name": self._pool_application_name, - } - lock_key = _writer_lease_advisory_lock_key( - self._writer_id, - self._writer_epoch, - ) - # pg_locks splits a 64-bit advisory key into classid (high word) and - # objid (low word) with objsubid = 1. - lock_classid = (lock_key >> 32) & 0xFFFFFFFF - lock_objid = lock_key & 0xFFFFFFFF - sql = f""" -WITH payload AS ( - SELECT {self._jsonb_literal(payload)} AS data -), -renewable AS ( - SELECT qbit_ledger_writer_lease.singleton - FROM qbit_ledger_writer_lease, payload - WHERE qbit_ledger_writer_lease.singleton - AND qbit_ledger_writer_lease.writer_id = data->>'writer_id' - AND qbit_ledger_writer_lease.writer_epoch = (data->>'writer_epoch')::bigint - AND qbit_ledger_writer_lease.writer_session_token = data->>'writer_session_token' - FOR NO KEY UPDATE SKIP LOCKED -), -renewed AS ( - UPDATE qbit_ledger_writer_lease - SET lease_expires_at = clock_timestamp() + {self._lease_interval_sql}, - updated_at = clock_timestamp() - FROM renewable - WHERE qbit_ledger_writer_lease.singleton = renewable.singleton - RETURNING qbit_ledger_writer_lease.writer_id -) -SELECT json_build_object( - 'backend', 'postgres-psql', - 'guard_advisory_lock_held', EXISTS ( - SELECT 1 - FROM pg_locks - WHERE locktype = 'advisory' - AND granted - AND pid = pg_backend_pid() - AND classid = {lock_classid}::oid - AND objid = {lock_objid}::oid - AND objsubid = 1 - ), - 'writer_session_token_current', EXISTS ( - SELECT 1 - FROM qbit_ledger_writer_lease, payload - WHERE qbit_ledger_writer_lease.singleton - AND qbit_ledger_writer_lease.writer_id = data->>'writer_id' - AND qbit_ledger_writer_lease.writer_epoch = (data->>'writer_epoch')::bigint - AND qbit_ledger_writer_lease.writer_session_token = data->>'writer_session_token' - ), - 'lease_renewed_count', (SELECT count(*) FROM renewed), - 'lease_expired', EXISTS ( - SELECT 1 - FROM qbit_ledger_writer_lease, payload - WHERE qbit_ledger_writer_lease.singleton - AND qbit_ledger_writer_lease.writer_id = data->>'writer_id' - AND qbit_ledger_writer_lease.writer_epoch = (data->>'writer_epoch')::bigint - AND qbit_ledger_writer_lease.writer_session_token = data->>'writer_session_token' - AND qbit_ledger_writer_lease.lease_expires_at <= clock_timestamp() - ), - 'lease_expiring_within_authority_margin', EXISTS ( - SELECT 1 - FROM qbit_ledger_writer_lease, payload - WHERE qbit_ledger_writer_lease.singleton - AND qbit_ledger_writer_lease.writer_id = data->>'writer_id' - AND qbit_ledger_writer_lease.writer_epoch = (data->>'writer_epoch')::bigint - AND qbit_ledger_writer_lease.writer_session_token = data->>'writer_session_token' - AND qbit_ledger_writer_lease.lease_expires_at - <= clock_timestamp() + {self._lease_authority_margin_sql} - ), - 'lease_locked_by_this_process', EXISTS ( - SELECT 1 - FROM qbit_ledger_writer_lease, pg_stat_activity, payload - WHERE qbit_ledger_writer_lease.singleton - AND pg_stat_activity.application_name = data->>'pool_application_name' - AND pg_stat_activity.backend_xid IS NOT NULL - AND pg_stat_activity.backend_xid = qbit_ledger_writer_lease.xmax - ), - '{GUARD_SERVER_EXECUTION_SECONDS_KEY}', {GUARD_SERVER_EXECUTION_SECONDS_SQL} -); -""" - attribution_rechecks_left = WRITER_LEASE_VERIFICATION_MAX_STATEMENTS - 1 - - def attribution_recheck(result: Any) -> str | None: - # Runs inside the guard's held query slot, between statements. - # The locker probe reads live pg_stat_activity state against - # its statement's older row snapshot, so an own fenced write - # committing mid-statement leaves an expired locked row with - # no attributable locker — the same shape as a competing - # claim. The next statement's fresh snapshot resolves which - # one it was; only the ambiguous shape earns the recheck, and - # every recheck outcome that is not a landed renewal still - # fails closed below. - nonlocal attribution_rechecks_left - if attribution_rechecks_left and ( - self._lease_renewal_ambiguously_lock_blocked(result) - ): - attribution_rechecks_left -= 1 - if on_statement_progress is not None: - # The first statement's round trip just completed; - # liveness monitors must count it before the second - # statement's execution starts consuming their budget. - on_statement_progress() - return sql - return None - - run_kwargs: dict[str, Any] = {"followup": attribution_recheck} - if on_query_start is not None: - run_kwargs["on_query_start"] = on_query_start - if on_statement_end is not None: - run_kwargs["on_statement_end"] = on_statement_end - if on_statement_server_seconds is not None: - run_kwargs["on_statement_result"] = ( - lambda statement_result: on_statement_server_seconds( - guard_server_execution_seconds(statement_result) - ) - ) - result = guard.run_json(sql, **run_kwargs) - if not isinstance(result, dict): - raise RuntimeError( - "psql writer lease guard verification returned non-object JSON" - ) - if not result.get("guard_advisory_lock_held"): - raise RuntimeError( - "postgres writer lease guard advisory lock is no longer held" - ) - if not result.get("writer_session_token_current"): - raise RuntimeError("writer lease is not active") - try: - renewed_count = int(result.get("lease_renewed_count", 0)) - except (TypeError, ValueError): - renewed_count = 0 - renewal_deferred_to_own_write = bool( - renewed_count == 0 - and result.get("lease_locked_by_this_process") - and ( - result.get("lease_expired") - or result.get("lease_expiring_within_authority_margin") - ) - ) - if ( - renewed_count == 0 - and result.get("lease_expired") - and not renewal_deferred_to_own_write - ): - # The committed row still names this session but its TTL has - # lapsed and the tuple lock we declined to wait on may belong to - # a different-identity expiry claim whose commit would land right - # after this snapshot. A stale token read is not proof of - # liveness here; fail closed like the queueing renewal used to. - # The exemption is the writer's own fenced write outlasting the - # TTL: its exclusive tuple lock means no claim is in flight, and - # its commit refreshes the TTL before any queued claimant - # re-evaluates its expiry CAS, so the session provably survives. - # It survives as a *process*; the returned deferral flag tells - # external-side-effect fences the same state is not authority - # for a guarded RPC, because a rollback would hand the expired - # row to a queued claimant instead. - raise RuntimeError( - "writer lease is expired and its renewal was lock-blocked; " - "a competing expiry claim may be in flight" - ) - return { - "backend": str(result["backend"]), - "verified_count": 1, - "renewed_count": renewed_count, - "renewal_deferred_to_own_write": renewal_deferred_to_own_write, - } - - @staticmethod - def _lease_renewal_ambiguously_lock_blocked(result: Any) -> bool: - """The expired, lock-blocked, unattributable-locker result shape. - - This is the only shape whose meaning a single statement cannot - decide: a competing expiry claim and an own fenced write that - committed mid-statement both present it. Identity failures are - excluded — a lost advisory lock or session token is conclusive and - must surface through the ordinary raises, never a recheck. - """ - if not isinstance(result, dict): - return False - if not result.get("guard_advisory_lock_held"): - return False - if not result.get("writer_session_token_current"): - return False - try: - renewed_count = int(result.get("lease_renewed_count", 0)) - except (TypeError, ValueError): - renewed_count = 0 - return bool( - renewed_count == 0 - and result.get("lease_expired") - and not result.get("lease_locked_by_this_process") - ) - - def _renew_writer_lease_with( - self, - run_json: Callable[[str], Any], - ) -> dict[str, int | str]: - payload = { - "writer_id": self._writer_id, - "writer_epoch": self._writer_epoch, - "writer_session_token": self._writer_session_token, - } - sql = f""" -WITH payload AS ( - SELECT {self._jsonb_literal(payload)} AS data -), -lease AS ( - UPDATE qbit_ledger_writer_lease - SET lease_expires_at = clock_timestamp() + {self._lease_interval_sql}, - updated_at = clock_timestamp() - FROM payload - WHERE qbit_ledger_writer_lease.singleton - AND qbit_ledger_writer_lease.writer_id = data->>'writer_id' - AND qbit_ledger_writer_lease.writer_epoch = (data->>'writer_epoch')::bigint - AND qbit_ledger_writer_lease.writer_session_token = data->>'writer_session_token' - RETURNING qbit_ledger_writer_lease.writer_id -) -SELECT CASE - WHEN (SELECT count(*) FROM lease) = 0 THEN - json_build_object('error', 'writer lease is not active') - ELSE - json_build_object( - 'backend', 'postgres-psql', - 'renewed_count', (SELECT count(*) FROM lease) - ) -END; -""" - result = run_json(sql) - if not isinstance(result, dict): - raise RuntimeError("psql writer lease renewal returned non-object JSON") - if "error" in result: - raise RuntimeError(str(result["error"])) - return {"backend": str(result["backend"]), "renewed_count": int(result["renewed_count"])} - - def release_writer_lease(self) -> bool: - """Expire this writer's lease through the normal fenced DB path.""" - return self._release_writer_lease_with(self._run_fenced_json) - - def release_writer_lease_fresh_connection(self) -> bool: - """Expire this writer's lease through a one-shot psql connection. - - The watchdog uses this path because its shared ledger lock or native - connection pool may be held by the subsystem that stopped making - progress. ``_run_sql`` forks psql without touching either resource. - """ - return self._release_writer_lease_with(self._run_fresh_connection_json) - - def _release_writer_lease_with(self, run_json: Callable[[str], Any]) -> bool: - """Run the one exact-session release implementation with ``run_json``. - - Best-effort, intended for graceful shutdown. Only the exact - ``(writer_id, writer_epoch, writer_session_token)`` this process holds is - expired, so a lease already reassigned to another writer is left - untouched. Returns True if a held lease row was expired. - """ - payload = { - "writer_id": self._writer_id, - "writer_epoch": self._writer_epoch, - "writer_session_token": self._writer_session_token, - } - sql = f""" -WITH payload AS ( - SELECT {self._jsonb_literal(payload)} AS data -), -released AS ( - UPDATE qbit_ledger_writer_lease - SET lease_expires_at = clock_timestamp() - interval '1 second', - updated_at = clock_timestamp() - FROM payload - WHERE qbit_ledger_writer_lease.singleton - AND qbit_ledger_writer_lease.writer_id = data->>'writer_id' - AND qbit_ledger_writer_lease.writer_epoch = (data->>'writer_epoch')::bigint - AND qbit_ledger_writer_lease.writer_session_token = data->>'writer_session_token' - RETURNING qbit_ledger_writer_lease.writer_id -) -SELECT json_build_object('released', (SELECT count(*) FROM released)); -""" - try: - result = run_json(sql) - if not isinstance(result, dict): - raise RuntimeError("psql writer lease release returned non-object JSON") - return int(result.get("released", 0)) > 0 - finally: - # The owning coordinator calls release only after writer admission - # and the lease heartbeat are stopped. Releasing this session lock - # is the final handoff that lets a successor enter fast adoption. - self._close_writer_lease_guard() - - def _run_fresh_connection_json(self, sql: str) -> Any: - output = self._run_sql(sql).strip() - if not output: - raise RuntimeError("psql query returned no JSON") - return json.loads(output.splitlines()[-1]) - - def _run_json( - self, - sql: str, - *, - on_statement_start: Callable[[], None] | None = None, - ) -> Any: - native = getattr(self, "_native", None) - if native is not None: - timeout_seconds = self._remaining_operation_timeout() - run_kwargs: dict[str, Any] = {} - if on_statement_start is not None: - run_kwargs["on_statement_start"] = on_statement_start - if timeout_seconds is None: - return native.run_json(sql, **run_kwargs) - run_kwargs["timeout_seconds"] = timeout_seconds - return native.run_json(sql, **run_kwargs) - run_sql = self._run_sql - if getattr(run_sql, "__func__", None) is PsqlShareLedger._run_sql: - output = run_sql( - sql, - on_statement_start=on_statement_start, - ).strip() - else: - # The A1 gate and embedders historically override this private - # seam with the one-argument signature. Their replacement owns - # the full execution boundary, so entry is the only statement- - # start signal the base class can expose without breaking them. - if on_statement_start is not None: - on_statement_start() - output = run_sql(sql).strip() - if not output: - raise RuntimeError("psql query returned no JSON") - return json.loads(output.splitlines()[-1]) - - def _run_script(self, sql: str) -> None: - native = getattr(self, "_native", None) - if native is not None: - native.run_script(sql) - return - self._run_sql(sql) - - def _run_sql( - self, - sql: str, - *, - on_statement_start: Callable[[], None] | None = None, - ) -> str: - cmd, run_kwargs, timeout_seconds = self._psql_invocation() - # All local deadline validation is complete. Only now does this - # invocation count as execution: an expiry raised above never starts - # psql and must remain attributed to coordinator-local admission. - if on_statement_start is not None: - on_statement_start() - try: - completed = subprocess.run( - cmd, - input=sql, - text=True, - stdout=subprocess.PIPE, - stderr=subprocess.PIPE, - check=False, - **run_kwargs, - ) - except subprocess.TimeoutExpired as exc: - raise LedgerOperationTimeout( - f"psql operation exceeded {timeout_seconds:g}s" - ) from exc - self._check_psql_exit(completed.returncode, completed.stderr, timeout_seconds) - return completed.stdout - - def _run_psql_json_rows( - self, - sql: str, - *, - row_converter: Callable[[Any], Any] | None, - on_statement_start: Callable[[], None] | None, - ) -> list[Any]: - """Row-result counterpart of ``_run_sql`` for the subprocess backend. - - The identical psql invocation (``_psql_invocation``: same flags, - session guards, statement/lock deadline, connect timeout and - subprocess timeout), with two differences (issue #236). psql's - stdout goes to an unnamed temporary file instead of being captured - into one string, so no whole-window text ever becomes a single - Python object; and the rows are read back from that spool one line - at a time and decoded in bounded batches (``_decode_json_lines``). - - The spool is validated before a single row is read from it. The exit - status and stderr are translated exactly as ``_run_sql`` translates - them, a subprocess timeout kills and reaps the child before raising - ``LedgerOperationTimeout``, and the temporary file closes on every - exit path, so no record is ever published from a psql run that did - not complete successfully. - """ - cmd, run_kwargs, timeout_seconds = self._psql_invocation() - decode_deadline = ( - None - if timeout_seconds is None - else self._monotonic() + timeout_seconds - ) - # Same boundary as _run_sql: deadline validation is done, execution - # starts here. - if on_statement_start is not None: - on_statement_start() - with tempfile.TemporaryFile(mode="w+b") as spool: - process = subprocess.Popen( - cmd, - stdin=subprocess.PIPE, - stdout=spool, - stderr=subprocess.PIPE, - env=run_kwargs.get("env"), - ) - try: - _, stderr_bytes = process.communicate( - input=sql.encode("utf-8"), - timeout=run_kwargs.get("timeout"), - ) - except subprocess.TimeoutExpired as exc: - process.kill() - process.communicate() - raise LedgerOperationTimeout( - f"psql operation exceeded {timeout_seconds:g}s" - ) from exc - except BaseException: - process.kill() - process.wait() - raise - self._check_psql_exit( - process.returncode, - stderr_bytes.decode("utf-8", errors="replace"), - timeout_seconds, - ) - spool.seek(0) - return self._decode_json_lines( - spool, - row_converter=row_converter, - deadline=decode_deadline, - ) - - @staticmethod - def _check_psql_exit( - returncode: int, - stderr: str, - timeout_seconds: float | None, - ) -> None: - """Translate a psql exit status the way every subprocess call does.""" - if returncode == 0: - return - stderr = stderr.strip() - if timeout_seconds is not None and _is_postgres_deadline_error(stderr): - raise LedgerOperationTimeout( - f"psql operation exceeded {timeout_seconds:g}s" - ) - raise RuntimeError( - "psql command failed " - f"(exit {returncode}): {stderr}" - ) - - def _psql_invocation(self) -> tuple[list[str], dict[str, Any], float | None]: - """Build the psql argv, subprocess keywords and deadline for one statement. - - Shared by the single-value ``_run_sql`` and the row-result - ``_run_psql_json_rows`` so the two cannot drift: the same - ``--single-transaction`` / ``ON_ERROR_STOP`` invocation, the same - session guards on PGOPTIONS, and the same per-statement deadline - (statement_timeout / lock_timeout, PGCONNECT_TIMEOUT and the - subprocess timeout) derived from ``_remaining_operation_timeout``. - A deadline that has already expired raises here, before any process - is spawned, so it stays attributed to coordinator-local admission. - """ - cmd = [ - *self._command, - "--no-psqlrc", - # One transaction per invocation. For the multi-statement schema - # script this is belt-and-braces alongside the BEGIN/COMMIT - # wrapper inside the script itself (psql tolerates the nested - # BEGIN with a warning): without atomicity, a failure or - # interruption mid-script can commit trigger definitions while - # later statements -- including the carry-forward summary seed -- - # never run, and a live writer mutating carry state in that gap - # leaves a permanently partial summary. For single-statement - # queries this is semantically identical to autocommit. The SQL - # sent here contains no \connect and no commands that cannot run - # inside a transaction block; --no-psqlrc means no ON_ERROR_ - # ROLLBACK can be injected from a psqlrc either. - "--single-transaction", - "--set", - "ON_ERROR_STOP=1", - "--set", - "VERBOSITY=verbose", - "--tuples-only", - "--no-align", - "--quiet", - ] - timeout_seconds = self._remaining_operation_timeout() - run_kwargs: dict[str, Any] = {} - # The session guards ride every psql invocation, deadline or not: an - # orphaned idle-in-transaction backend is exactly the failure that - # occurs when no deadline-scoped work is running, so building - # PGOPTIONS only for deadline-bearing statements would leave the - # unbounded sessions — the dangerous ones — unguarded. A deadline, - # when armed, appends its per-statement bounds after the guards. - session_guards = getattr(self, "_session_guards", None) - option_fragments: list[str] = [] - if session_guards is not None: - option_fragments.append(session_guards.options_fragment()) - if timeout_seconds is not None: - timeout_ms = max(1, int(timeout_seconds * 1000)) - option_fragments.append( - f"-c statement_timeout={timeout_ms}ms " - f"-c lock_timeout={timeout_ms}ms" - ) - if option_fragments: - subprocess_env = dict(os.environ) - # Operator-supplied PGOPTIONS stay, ahead of the coordinator's - # fragments so a later -c duplicate resolves in our favor. - existing_options = subprocess_env.get("PGOPTIONS", "").strip() - subprocess_env["PGOPTIONS"] = " ".join( - option - for option in (existing_options, *option_fragments) - if option - ) - run_kwargs["env"] = subprocess_env - if timeout_seconds is not None: - run_kwargs["env"]["PGCONNECT_TIMEOUT"] = str( - max(1, math.ceil(timeout_seconds)) - ) - run_kwargs["timeout"] = timeout_seconds - return cmd, run_kwargs, timeout_seconds - - @staticmethod - def _record_from_json(payload: dict[str, Any]) -> AcceptedShareRecord: - return AcceptedShareRecord( - share_seq=int(payload["share_seq"]), - share_id=str(payload["share_id"]), - miner_id=str(payload["miner_id"]), - order_key=str(payload["order_key"]), - p2mr_program_hex=str(payload["p2mr_program_hex"]), - share_difficulty=int(payload["share_difficulty"]), - network_difficulty=int(payload["network_difficulty"]), - template_height=int(payload["template_height"]), - job_id=str(payload["job_id"]), - job_issued_at_ms=int(payload["job_issued_at_ms"]), - accepted_at_ms=int(payload["accepted_at_ms"]), - ntime=int(payload["ntime"]), - credit_policy=( - str(payload["credit_policy"]) - if payload.get("credit_policy") is not None - else None - ), - newly_inserted=bool(payload.get("newly_inserted", True)), - candidate_outbox_state=( - str(payload["candidate_outbox_state"]) - if payload.get("candidate_outbox_state") is not None - else None - ), - ) - - @staticmethod - def _jsonb_literal(payload: object) -> str: - raw = json.dumps(payload, separators=(",", ":")) - tag = "qbit_prism_json" - while f"${tag}$" in raw: - tag += "_x" - return f"${tag}${raw}${tag}$::jsonb" - - @staticmethod - def _text_literal(value: str) -> str: - return "'" + value.replace("'", "''") + "'" - - @classmethod - def _text_array_literal(cls, values: Sequence[str]) -> str: - """Render a non-empty text set as one array literal for ``= ANY``. - - Each element goes through the same quoting the scalar literals use. - The cast is explicit because an empty ``ARRAY[]`` has no inferable - element type; callers never build one, and the cast keeps that a - parse-time guarantee rather than a convention. - """ - elements = ", ".join(cls._text_literal(value) for value in values) - return f"ARRAY[{elements}]::text[]" - - -CTV_FANOUT_STATUSES = { - "awaiting_maturity", - "broadcastable", - "broadcast_submitted", - "confirmed", - "reorged", - "failed", -} - -CTV_FANOUT_ATTEMPT_STATUSES = {"planned", "submitted", "accepted", "rejected", "failed"} - - -def _ctv_broadcast_summary_fields() -> tuple[str, ...]: - return ( - "broadcast_attempt_count", - "broadcast_attempt_detail_count", - "first_broadcast_attempt_at", - "last_broadcast_attempt_at", - "last_broadcast_attempt_status", - "last_broadcast_package_tx_hexes", - "last_broadcast_package_txids", - "last_broadcast_submit_result", - "last_broadcast_error", - "broadcast_attempt_status_counts", - "next_broadcast_attempt_at", - "broadcast_retry_backoff_seconds", - ) - - -def _empty_ctv_broadcast_summary() -> dict[str, Any]: - return { - "broadcast_attempt_count": 0, - "broadcast_attempt_detail_count": 0, - "first_broadcast_attempt_at": None, - "last_broadcast_attempt_at": None, - "last_broadcast_attempt_status": None, - "last_broadcast_package_tx_hexes": [], - "last_broadcast_package_txids": [], - "last_broadcast_submit_result": None, - "last_broadcast_error": None, - "broadcast_attempt_status_counts": {}, - "next_broadcast_attempt_at": None, - "broadcast_retry_backoff_seconds": 0, - } - - -def _ctv_broadcast_attempt_summary(payload: dict[str, Any]) -> dict[str, Any]: - return { - "attempt_count": int(payload.get("broadcast_attempt_count") or 0), - "detail_count": int(payload.get("broadcast_attempt_detail_count") or 0), - "first_attempt_at": payload.get("first_broadcast_attempt_at"), - "last_attempt_at": payload.get("last_broadcast_attempt_at"), - "last_attempt_status": payload.get("last_broadcast_attempt_status"), - "last_package_tx_hexes": copy.deepcopy(payload.get("last_broadcast_package_tx_hexes") or []), - "last_package_txids": copy.deepcopy(payload.get("last_broadcast_package_txids") or []), - "last_submit_result": copy.deepcopy(payload.get("last_broadcast_submit_result")), - "last_error": payload.get("last_broadcast_error"), - "status_counts": copy.deepcopy(payload.get("broadcast_attempt_status_counts") or {}), - "next_attempt_at": payload.get("next_broadcast_attempt_at"), - "retry_backoff_seconds": int(payload.get("broadcast_retry_backoff_seconds") or 0), - } - - -def _ctv_broadcast_attempt_due(value: object, now: datetime) -> bool: - if value is None: - return True - if isinstance(value, datetime): - candidate = value - else: - text = str(value).strip() - if not text: - return True - if " " in text and "T" not in text: - text = text.replace(" ", "T", 1) - if text.endswith("Z"): - text = text[:-1] + "+00:00" - elif text.endswith("+00"): - text = text[:-3] + "+00:00" - try: - candidate = datetime.fromisoformat(text) - except ValueError: - return True - if candidate.tzinfo is None: - candidate = candidate.replace(tzinfo=timezone.utc) - return candidate <= now - - -def validate_ctv_fanout_status(status: str) -> None: - if status not in CTV_FANOUT_STATUSES: - raise ValueError(f"unsupported CTV fanout status: {status}") - - -def validate_ctv_fanout_attempt_status(status: str) -> None: - if status not in CTV_FANOUT_ATTEMPT_STATUSES: - raise ValueError(f"unsupported CTV fanout attempt status: {status}") - - -def ctv_fanout_recovery_payload( - *, - block_hash: str, - manifest_set: dict[str, Any], - manifest_set_sha256: str, -) -> dict[str, Any]: - block_hash = canonical_hex(block_hash, name="block_hash", expected_bytes=32) - manifest_set_sha256 = canonical_hex( - manifest_set_sha256, - name="manifest_set_sha256", - expected_bytes=32, - ) - manifests_raw = manifest_set.get("manifests") - streamed = _is_lazy_manifest_sequence(manifests_raw) - if not streamed and (not isinstance(manifests_raw, list) or not manifests_raw): - raise ValueError("manifest_set.manifests must be a non-empty array") - if streamed and not len(manifests_raw): - raise ValueError("manifest_set.manifests must be a non-empty array") - - if streamed: - # A bounded artifact view keeps the manifests on disk. They are - # walked once here in their canonical (chunk-ordered) sequence for - # validation and again, on demand, when the statement literal is - # streamed; the whole set is never sorted or copied (#255). - manifests: Sequence[Any] = MappedSequence(manifests_raw, _manifest_record) - first_manifest = manifests[0] - else: - manifests = sorted( - (require_mapping(manifest, "manifest") for manifest in manifests_raw), - key=lambda item: int(require_mapping(item.get("precommitment"), "precommitment")["chunk_index"]), - ) - first_manifest = manifests[0] - first_precommitment = require_mapping(first_manifest.get("precommitment"), "precommitment") - block_height_value = manifest_set.get("block_height", first_precommitment.get("block_height")) - block_height = int(block_height_value) if block_height_value is not None else None - fanout_count = int(manifest_set.get("fanout_count", len(manifests))) - if fanout_count != len(manifests): - raise ValueError("manifest_set.fanout_count must equal the number of manifests") - settlement_mode = str(manifest_set.get("settlement_mode", first_precommitment.get("settlement_mode", ""))) - if settlement_mode not in {"hybrid_coinbase_ctv_fanout", "ctv_fanout"}: - raise ValueError("manifest_set.settlement_mode must be a CTV settlement mode") - parent_coinbase_txid = canonical_hex( - str(manifest_set.get("parent_coinbase_txid", first_manifest.get("parent_coinbase_txid", ""))), - name="parent_coinbase_txid", - expected_bytes=32, - ) - parent_coinbase_tx_hex = canonical_hex( - str(first_manifest.get("parent_coinbase_tx_hex", "")), - name="parent_coinbase_tx_hex", - ) - fanout_output_sum_sats = int(manifest_set.get("fanout_output_sum_sats", 0)) - covenant_output_value_sats = int(manifest_set.get("covenant_output_value_sats", 0)) - - def artifact_for(expected_index: int, manifest: Mapping[str, Any]) -> dict[str, Any]: - precommitment = require_mapping(manifest.get("precommitment"), "precommitment") - precommitment_block_height = precommitment.get("block_height") - if block_height is not None and precommitment_block_height is not None and int(precommitment_block_height) != block_height: - raise ValueError("CTV fanout block height mismatch") - chunk_index = int(precommitment["chunk_index"]) - chunk_count = int(precommitment["chunk_count"]) - if chunk_index != expected_index: - raise ValueError("CTV fanout chunks must be contiguous from zero") - if chunk_count != fanout_count: - raise ValueError("CTV fanout chunk_count must equal fanout_count") - artifact_parent_txid = canonical_hex( - str(manifest.get("parent_coinbase_txid", "")), - name="manifest.parent_coinbase_txid", - expected_bytes=32, - ) - if artifact_parent_txid != parent_coinbase_txid: - raise ValueError("CTV fanout parent coinbase txid mismatch") - fanout_fee_sats = int(precommitment.get("fanout_fee_sats", 0)) - raw_anchor_vout = precommitment.get("anchor_vout") - if fanout_fee_sats > 0 and raw_anchor_vout is not None: - raise ValueError("built-in-fee CTV fanout must not include a CPFP anchor") - if fanout_fee_sats == 0 and raw_anchor_vout is None: - raise ValueError("zero-fee CTV fanout must include a CPFP anchor") - artifact = { - "fanout_txid": canonical_hex( - str(manifest["fanout_txid"]), - name="fanout_txid", - expected_bytes=32, - ), - # One manifest is bounded by one fanout transaction (its - # recipients are capped per transaction by the settlement - # configuration), so its canonical text and digest are - # per-manifest allocations; a decoded record is used directly - # instead of being deep-copied. - "manifest_json": ( - canonical_json_text(manifest) - if isinstance(manifest, dict) and not streamed - else "".join(iter_json_chunks(manifest, sort_keys=True)) - ), - "manifest": copy.deepcopy(manifest) if not streamed else manifest, - "manifest_sha256": ( - sha256_json_hex(manifest) - if isinstance(manifest, dict) and not streamed - else streamed_sha256_json_hex(manifest) - ), - "precommitment_sha256": canonical_hex( - str(manifest["precommitment_sha256_hex"]), - name="precommitment_sha256_hex", - expected_bytes=32, - ), - "ctv_hash": canonical_hex( - str(precommitment["ctv_hash_hex"]), - name="ctv_hash_hex", - expected_bytes=32, - ), - "commitment_witness_leaf_hex": canonical_hex( - str(manifest["commitment_witness_leaf_hex"]), - name="commitment_witness_leaf_hex", - ), - "chunk_index": chunk_index, - "chunk_count": chunk_count, - "parent_coinbase_txid": artifact_parent_txid, - "parent_coinbase_vout": int(manifest["parent_coinbase_vout"]), - "fanout_tx_template_hex": canonical_hex( - str(precommitment["fanout_tx_template_hex"]), - name="fanout_tx_template_hex", - ), - "fanout_tx_hex": canonical_hex(str(manifest["fanout_tx_hex"]), name="fanout_tx_hex"), - "anchor_vout": None if raw_anchor_vout is None else int(raw_anchor_vout), - "covenant_output_value_sats": int(manifest["covenant_output_value_sats"]), - "fanout_output_sum_sats": int(precommitment["fanout_output_sum_sats"]), - "settlement_status": "awaiting_maturity", - } - if block_height is not None: - artifact["block_height"] = block_height - return artifact - - # One validating walk; the sums never hold the artifacts. - output_sum = 0 - covenant_sum = 0 - for expected_index, manifest in enumerate(manifests): - artifact = artifact_for(expected_index, manifest) - output_sum += int(artifact["fanout_output_sum_sats"]) - covenant_sum += int(artifact["covenant_output_value_sats"]) - if output_sum != fanout_output_sum_sats: - raise ValueError("CTV fanout output sum mismatch") - if covenant_sum != covenant_output_value_sats: - raise ValueError("CTV covenant output value sum mismatch") - - artifacts: Sequence[Any] - if streamed: - artifacts = MappedSequence( - _EnumeratedSequence(manifests), - lambda item: artifact_for(item[0], item[1]), - ) - manifest_set_json: object = StreamedJsonString( - lambda: iter_json_chunks(manifest_set, sort_keys=True) - ) - manifest_set_member: object = manifest_set - else: - artifacts = [artifact_for(index, manifest) for index, manifest in enumerate(manifests)] - manifest_set_json = canonical_json_text(manifest_set) - manifest_set_member = copy.deepcopy(manifest_set) - - payload = { - "schema": "qbit.prism.ctv-fanout-recovery.v1", - "block_hash": block_hash, - "manifest_set_sha256": manifest_set_sha256, - "manifest_set_json": manifest_set_json, - "settlement_mode": settlement_mode, - "parent_coinbase_txid": parent_coinbase_txid, - "parent_coinbase_tx_hex": parent_coinbase_tx_hex, - "fanout_count": fanout_count, - "fanout_output_sum_sats": fanout_output_sum_sats, - "covenant_output_value_sats": covenant_output_value_sats, - "manifest_set": manifest_set_member, - "artifacts": artifacts, - } - if block_height is not None: - payload["block_height"] = block_height - audit_bundle_sha256 = manifest_set.get("audit_bundle_sha256") - if audit_bundle_sha256 is not None: - payload["audit_bundle_sha256"] = canonical_hex( - str(audit_bundle_sha256), - name="audit_bundle_sha256", - expected_bytes=32, - ) - audit_bundle = manifest_set.get("audit_bundle") - if isinstance(audit_bundle, dict): - payload["audit_bundle"] = copy.deepcopy(audit_bundle) - return payload - - -def _is_lazy_manifest_sequence(value: object) -> bool: - """A replayable, non-list manifest sequence (a bounded view member).""" - return isinstance(value, Sequence) and not isinstance( - value, - (list, tuple, str, bytes, bytearray), - ) - - -def _manifest_record(manifest: object) -> Mapping[str, Any]: - """One manifest as a mapping; an isolated record is decoded in-process. - - A manifest is bounded by one fanout transaction, whose recipients are - capped per transaction by the settlement configuration, so it never - reaches the isolated-decode limit in practice; the fallback keeps an - unexpected one usable rather than refusing it. - """ - if isinstance(manifest, RawJsonRecord): - manifest = materialize_json(manifest) - if not isinstance(manifest, Mapping): - raise ValueError("manifest must be an object") - return manifest - - -class _EnumeratedSequence(Sequence): - """``enumerate`` as a replayable sequence of ``(index, item)`` pairs.""" - - __slots__ = ("_base",) - - def __init__(self, base: Sequence[Any]) -> None: - self._base = base - - def __len__(self) -> int: - return len(self._base) - - def __iter__(self) -> Iterator[tuple[int, Any]]: - for index, item in enumerate(self._base): - yield index, item - - def __getitem__(self, index: int | slice) -> Any: - if isinstance(index, slice): - start, stop, step = index.indices(len(self._base)) - return [(position, self._base[position]) for position in range(start, stop, step)] - if index < 0: - index += len(self._base) - return index, self._base[index] - - -def require_mapping(value: object, name: str) -> dict[str, Any]: - if not isinstance(value, Mapping): - raise ValueError(f"{name} must be an object") - return value # type: ignore[return-value] - - -def sha256_json_hex(payload: object) -> str: - return hashlib.sha256(canonical_json_text(payload).encode()).hexdigest() - - -def block_candidate_identity(candidate: dict[str, Any]) -> dict[str, Any]: - """Candidate payload with its volatile acknowledgment stamp removed. - - A miner can resubmit the same solved block after a transient submit - outcome, and the rebuilt intent differs from the persisted one only in - pending_share.accepted_at_ms. That drift must stay idempotent against the - durable outbox while any other divergence remains a hard payload - mismatch. The stored payload keeps its original stamp; only comparisons - use this identity form. - """ - pending_share = candidate.get("pending_share") - if isinstance(pending_share, dict) and "accepted_at_ms" in pending_share: - candidate = { - **candidate, - "pending_share": {**pending_share, "accepted_at_ms": None}, - } - return candidate - - -def block_candidate_identity_sha256(candidate: Any) -> str: - """``sha256_json_hex`` of the candidate identity, streamed. - - The candidate carries the whole payout window under ``shares_json``, so - one ``json.dumps`` over it is exactly the whole-window C call the - writer-lease monitor must never wait behind (#236). The digest is - byte-identical to ``sha256_json_hex(block_candidate_identity(candidate))``; - the share array feeds it batch by batch. - - A :class:`PreparedCandidateIntent` already carries this digest (its - body *is* the identity JSON), and a mapping whose share sequence is not - a plain list (a page-backed window, a daemon mirror, a spool sequence) - is digested through the same bounded codec rather than materialized. - """ - if isinstance(candidate, PreparedCandidateIntent): - return prepare_candidate_intent(candidate).candidate_sha256 - if isinstance(candidate, dict) and not isinstance( - candidate.get("shares_json"), (list, tuple) - ) and "shares_json" in candidate: - return prepare_candidate_intent(candidate).candidate_sha256 - identity = block_candidate_identity(candidate) - if not isinstance(identity, dict): - return sha256_json_hex(identity) - digest = hashlib.sha256() - for chunk in iter_json_object_text_chunks( - identity, - array_keys=("shares_json",), - sort_keys=True, - ): - digest.update(chunk.encode("utf-8")) - return digest.hexdigest() - - -def sha256_bytes_hex(payload: bytes) -> str: - return hashlib.sha256(payload).hexdigest() - - -def canonical_json_text(payload: object) -> str: - return json.dumps(payload, sort_keys=True, separators=(",", ":")) - - -def canonical_hex(value: str, *, name: str, expected_bytes: int | None = None) -> str: - lowered = value.lower() - if not lowered: - raise ValueError(f"{name} must not be empty") - try: - bytes.fromhex(lowered) - except ValueError as exc: - raise ValueError(f"{name} must be hex") from exc - if expected_bytes is not None and len(lowered) != expected_bytes * 2: - raise ValueError(f"{name} must be {expected_bytes * 2} hex characters") - if len(lowered) % 2 != 0: - raise ValueError(f"{name} must have an even number of hex characters") - return lowered diff --git a/lab/prism/share_submission.py b/lab/prism/share_submission.py deleted file mode 100644 index 57911d6b6..000000000 --- a/lab/prism/share_submission.py +++ /dev/null @@ -1,799 +0,0 @@ -"""Pure submit classification and the narrow PRISM submission orchestrator.""" - -from __future__ import annotations - -from collections import OrderedDict -from dataclasses import dataclass -import threading -from typing import Any, Callable, Iterable, Mapping, NoReturn - -from lab.prism.block_candidates import PrismBlockCandidate -from lab.prism.job_bundle import PRISM_JOB_BUILD_SECONDS_BUCKETS -from lab.prism.job_delivery import ( - EvictedJobEntry, - PRISM_CREDIT_POLICY_STALE_GRACE, - PrismJobContext, -) -from lab.prism.share_ledger import PendingShare -from lab.prism.stratum_session import ClientState - - -PRISM_REJECTION_STALE_JOB = "stale-job" -PRISM_REJECTION_DUPLICATE_SHARE = "duplicate-share" -PRISM_REJECTION_LOW_DIFFICULTY = "low-difficulty" -PRISM_REJECTION_MALFORMED_SUBMIT = "malformed-submit" -PRISM_REJECTION_UNAUTHORIZED_WORKER = "unauthorized-worker" -PRISM_REJECTION_UNKNOWN_JOB = "unknown-job" -PRISM_REJECTION_INVALID_EXTRANONCE = "invalid-extranonce" -PRISM_REJECTION_INVALID_NTIME_OR_NONCE = "invalid-ntime-or-nonce" -PRISM_REJECTION_BACKEND_RPC_UNAVAILABLE = "backend-rpc-unavailable" -PRISM_REJECTION_POOL_CLOSED = "pool-closed" -DEFAULT_RECENT_SHARE_CAPACITY = 50_000 -# Read-to-ack latency labels for mining.submit. Accepted shares wait for the -# group commit before their ack; rejected shares (measured when the reject -# decision is made) skip it, so the pair separates commit pressure from -# thread-scheduling/GIL pressure as connection count grows. -PRISM_SHARE_ACK_RESULTS = ("accepted", "rejected") - - -def empty_share_ack_histograms() -> dict[str, dict[str, Any]]: - """Zeroed per-result share-ack histograms in the rendered shape.""" - - return { - result: { - "buckets": {bucket: 0 for bucket in PRISM_JOB_BUILD_SECONDS_BUCKETS}, - "sum": 0.0, - "count": 0, - } - for result in PRISM_SHARE_ACK_RESULTS - } - - -class RecentShareIndex: - """Thread-safe insertion-ordered duplicate window with bounded memory.""" - - def __init__( - self, - *, - capacity: int = DEFAULT_RECENT_SHARE_CAPACITY, - initial: Iterable[tuple[str, str]] = (), - ) -> None: - if capacity < 1: - raise ValueError("recent share capacity must be positive") - self.capacity = int(capacity) - self._lock = threading.Lock() - self._entries: OrderedDict[tuple[str, str], None] = OrderedDict() - self.replace(initial) - - def reserve(self, share_key: tuple[str, str]) -> bool: - with self._lock: - if share_key in self._entries: - return False - self._entries[share_key] = None - while len(self._entries) > self.capacity: - self._entries.popitem(last=False) - return True - - def release(self, share_key: tuple[str, str]) -> None: - with self._lock: - self._entries.pop(share_key, None) - - def replace(self, entries: Iterable[tuple[str, str]]) -> None: - with self._lock: - self._entries.clear() - for share_key in entries: - self._entries[share_key] = None - while len(self._entries) > self.capacity: - self._entries.popitem(last=False) - - def snapshot(self) -> tuple[tuple[str, str], ...]: - with self._lock: - return tuple(self._entries) - - -class RecentShareCompatibilityField: - """Temporary coordinator view over submission-owned duplicate state.""" - - def __get__(self, instance: Any, owner: type[Any]) -> Any: - if instance is None: - return self - service = instance.__dict__.get("_share_submission_service") - if service is not None: - return set(service.recent_shares.snapshot()) - return instance.__dict__.get("recent_share_keys", set()) - - def __set__(self, instance: Any, value: Iterable[tuple[str, str]]) -> None: - service = instance.__dict__.get("_share_submission_service") - if service is not None: - service.recent_shares.replace(value) - else: - instance.__dict__["recent_share_keys"] = set(value) - - -class BlockSolvesDroppedCompatibilityField: - """Owner-routing coordinator view over submission-owned drop accounting. - - Current callers read ``coordinator.block_solves_dropped_counts`` directly, - so the historical attribute stays readable and writable while the - submission owner keeps the single mutable copy. Reads return a copied - snapshot once the owner exists; pre-owner values live in the instance - dict until service construction adopts them, and an unset field keeps the - pre-extraction ``getattr``-with-default semantics by raising - AttributeError. - """ - - def __get__(self, instance: Any, owner: type[Any]) -> Any: - if instance is None: - return self - service = instance.__dict__.get("_share_submission_service") - if service is not None: - return service.dropped_solves_snapshot() - try: - return instance.__dict__["block_solves_dropped_counts"] - except KeyError: - raise AttributeError("block_solves_dropped_counts") from None - - def __set__(self, instance: Any, value: Mapping[str, int]) -> None: - service = instance.__dict__.get("_share_submission_service") - if service is not None: - service.replace_dropped_solves(value) - else: - instance.__dict__["block_solves_dropped_counts"] = dict(value) - - -@dataclass(frozen=True) -class SubmitRequest: - worker_name: str - job_id: str - extranonce2_hex: str - ntime_hex: str - nonce_hex: str - version_bits_hex: str | None - - -@dataclass(frozen=True) -class SubmitRejected(ValueError): - code: int - reason: str - message: str - - def __str__(self) -> str: - return self.message - - -@dataclass(frozen=True) -class SubmitContextInput: - active_context: PrismJobContext | None - retained_entry: EvictedJobEntry | None - current_tip: str - stale_grace_eligible: bool - # A retained entry resumed by a different connection than the one the job - # was delivered to. Cross-connection resumes are same-tip only; the - # stale-grace window anchors on the submitting connection's delivery - # state, which says nothing about work another (dead) connection - # delivered. - cross_connection: bool = False - - -@dataclass(frozen=True) -class SubmitContextDecision: - context: PrismJobContext - current_tip: str - source: str - credit_policy: str | None - retained_entry: EvictedJobEntry | None - cross_connection: bool = False - - -@dataclass(frozen=True) -class SubmitControlSnapshot: - """One bounded capture of coordinator-owned submit admission state.""" - - pool_open: bool - active_context: PrismJobContext | None - published_tip: str | None - - -@dataclass(frozen=True) -class SubmitWorkDecision: - share_key: tuple[str, str] - block_worthy: bool - credit_share_on_accept: bool - route: str - - -@dataclass(frozen=True) -class ShareSubmissionPorts: - reject: Callable[[SubmitRejected, str | None], NoReturn] - control_snapshot: Callable[ - [ClientState, str], - SubmitControlSnapshot, - ] - note_submitted: Callable[[str, ClientState], None] - retained_entry: Callable[[ClientState, str], EvictedJobEntry | None] - live_tip: Callable[[], str] - stale_grace_eligible: Callable[[ClientState, PrismJobContext, str], bool] - assemble: Callable[[ClientState, PrismJobContext, SubmitRequest, int], Any] - pending_share: Callable[ - [PrismJobContext, Any, str, str | None], - PendingShare, - ] - # Returns the durable candidate_outbox_state when the accepted share - # carried a block-candidate intent whose outbox row already exists - # (submitted/abandoned short-circuits the enqueue), else None. - append_share: Callable[ - [ - ClientState, - PrismJobContext, - Any, - PendingShare, - str | None, - dict[str, Any] | None, - ], - str | None, - ] - note_retained_submit: Callable[[str | None, bool], None] - note_collection_candidate: Callable[[PrismJobContext, Any], None] - # Returns the prepared intent (#255): a mapping over the historical - # keys whose share sequence is the job's immutable window, not a copy. - candidate_intent: Callable[[PrismBlockCandidate], Any] - finish_pending_commit: Callable[[PendingShare], None] - record_terminal_outcome: Callable[[str, bool], None] - submit_synchronous_candidate: Callable[ - [ - PrismBlockCandidate, - tuple[str, str], - str, - EvictedJobEntry | None, - str | None, - ], - bool, - ] - enqueue_candidate: Callable[[PrismBlockCandidate], bool] - log: Callable[[str], None] - log_exception: Callable[[], None] - - -def parse_submit_request(params: list[object]) -> SubmitRequest: - """Parse immutable wire input without touching coordinator state.""" - - if len(params) < 5: - raise SubmitRejected( - 20, - PRISM_REJECTION_MALFORMED_SUBMIT, - "submit params are incomplete", - ) - worker_name, job_id, extranonce2_hex, ntime_hex, nonce_hex = ( - str(item) for item in params[:5] - ) - return SubmitRequest( - worker_name=worker_name, - job_id=job_id, - extranonce2_hex=extranonce2_hex, - ntime_hex=ntime_hex, - nonce_hex=nonce_hex, - version_bits_hex=str(params[5]) if len(params) > 5 else None, - ) - - -def validate_submit_request( - request: SubmitRequest, - *, - authorized_username: str, - pool_open: bool, - extranonce2_size: int, -) -> None: - """Validate request identity and fixed-width fields in protocol order. - - A closed pool rejects before any share accounting: post-close submits - must not inflate global/per-worker submitted totals (the stale-percent - denominator) or vardiff windows they can never contribute to. - """ - - if request.worker_name != authorized_username: - raise SubmitRejected( - 20, - PRISM_REJECTION_UNAUTHORIZED_WORKER, - "submit username does not match authorized username", - ) - if not pool_open: - raise SubmitRejected( - 21, - PRISM_REJECTION_POOL_CLOSED, - "pool is no longer accepting shares", - ) - if len(request.extranonce2_hex) != extranonce2_size * 2: - raise SubmitRejected( - 20, - PRISM_REJECTION_INVALID_EXTRANONCE, - "unexpected extranonce2 size", - ) - if len(request.ntime_hex) != 8 or len(request.nonce_hex) != 8: - raise SubmitRejected( - 20, - PRISM_REJECTION_INVALID_NTIME_OR_NONCE, - "ntime and nonce must be 4-byte hex strings", - ) - - -def classify_submit_context(value: SubmitContextInput) -> SubmitContextDecision: - """Choose current, retained, or stale-grace work from captured facts.""" - - context = value.active_context - source = "active" - retained_entry: EvictedJobEntry | None = None - if context is None: - retained_entry = value.retained_entry - if retained_entry is None: - raise SubmitRejected(21, PRISM_REJECTION_UNKNOWN_JOB, "stale job") - context = retained_entry.context - source = "retained" - parent_hash = str(context.template["previousblockhash"]) - if parent_hash == value.current_tip: - return SubmitContextDecision( - context=context, - current_tip=value.current_tip, - source=source, - credit_policy=None, - retained_entry=retained_entry, - cross_connection=value.cross_connection, - ) - if value.cross_connection: - # Cross-connection resumes are same-tip only. A tip that moves - # between the graveyard lookup and this classification must not - # fall through to stale grace: that window anchors on the - # submitting connection's delivery state, which says nothing - # about work another (dead) connection delivered. - raise SubmitRejected(21, PRISM_REJECTION_STALE_JOB, "stale job") - if not value.stale_grace_eligible: - raise SubmitRejected(21, PRISM_REJECTION_STALE_JOB, "stale job") - return SubmitContextDecision( - context=context, - current_tip=value.current_tip, - source=source, - credit_policy=PRISM_CREDIT_POLICY_STALE_GRACE, - retained_entry=retained_entry, - cross_connection=value.cross_connection, - ) - - -def classify_submit_work( - context: PrismJobContext, - submission: Any, - *, - credit_policy: str | None, -) -> SubmitWorkDecision: - """Classify a proof as an ordinary share or one of two block routes. - - A floor-bearing listener holds the advertised share target above the - qbit network target while network difficulty sits below the floor, so a - submission can solve a block yet miss the share target. Never discard a - block over share bookkeeping: reject as low-difficulty only when the - hash is not block-worthy. Stale-grace work is never block-worthy: its - parent is no longer the chain tip. - """ - - block_worthy = bool(submission.block_pass) and ( - credit_policy != PRISM_CREDIT_POLICY_STALE_GRACE - ) - if not submission.share_pass and not block_worthy: - raise SubmitRejected( - 23, - PRISM_REJECTION_LOW_DIFFICULTY, - "low difficulty share", - ) - if not block_worthy: - route = "share" - elif submission.share_pass: - route = "async_block" - else: - route = "synchronous_block" - return SubmitWorkDecision( - share_key=(context.worker.username, submission.header_hex), - block_worthy=block_worthy, - credit_share_on_accept=route == "synchronous_block", - route=route, - ) - - -class ShareSubmissionService: - """Apply one pure submit decision through the coordinator's narrow ports. - - Beyond the hot path, the service owns the share-ACK latency histograms - (observed by the session owner through the coordinator's - ``_observe_share_ack_seconds`` routing seam) and the stale-grace - ``block_solves_dropped_counts`` accounting. Prometheus rendering stays in - the coordinator through PR 79 and reads this owner's copied snapshots. - """ - - def __init__( - self, - ports: ShareSubmissionPorts, - *, - extranonce2_size: int, - recent_shares: RecentShareIndex | None = None, - ) -> None: - self.ports = ports - self.extranonce2_size = int(extranonce2_size) - self.recent_shares = recent_shares or RecentShareIndex() - self._metrics_lock = threading.Lock() - self.share_ack_histograms = empty_share_ack_histograms() - self.block_solves_dropped_counts: dict[str, int] = {"stale_grace": 0} - - def observe_share_ack_seconds( - self, - result: str, - elapsed_seconds: float, - ) -> None: - if result not in PRISM_SHARE_ACK_RESULTS: - raise ValueError(f"unknown share ack result: {result}") - with self._metrics_lock: - histogram = self.share_ack_histograms[result] - histogram["count"] = int(histogram["count"]) + 1 - histogram["sum"] = float(histogram["sum"]) + max( - 0.0, elapsed_seconds - ) - buckets = histogram["buckets"] - assert isinstance(buckets, dict) - for bucket in PRISM_JOB_BUILD_SECONDS_BUCKETS: - if elapsed_seconds <= bucket: - buckets[bucket] = int(buckets.get(bucket, 0)) + 1 - - def share_ack_snapshot(self) -> dict[str, dict[str, Any]]: - """Copy the per-result share-ack histograms under the owner lock.""" - - with self._metrics_lock: - return { - result: { - "buckets": dict(histogram["buckets"]), - "sum": float(histogram["sum"]), - "count": int(histogram["count"]), - } - for result, histogram in self.share_ack_histograms.items() - } - - def replace_share_ack_histograms( - self, - histograms: Mapping[str, Mapping[str, Any]], - ) -> None: - """Adopt legacy pre-owner histogram state at service construction.""" - - with self._metrics_lock: - self.share_ack_histograms = { - result: { - "buckets": dict(histogram["buckets"]), - "sum": float(histogram["sum"]), - "count": int(histogram["count"]), - } - for result, histogram in histograms.items() - } - - def dropped_solves_snapshot(self) -> dict[str, int]: - """Copy the bounded-policy dropped block-solve counters.""" - - with self._metrics_lock: - return dict(self.block_solves_dropped_counts) - - def replace_dropped_solves(self, counts: Mapping[str, int]) -> None: - with self._metrics_lock: - self.block_solves_dropped_counts = { - str(reason): int(count) for reason, count in counts.items() - } - - def _count_dropped_block_solve(self, reason: str) -> None: - with self._metrics_lock: - self.block_solves_dropped_counts[reason] = ( - int(self.block_solves_dropped_counts.get(reason, 0)) + 1 - ) - - def _reject(self, rejected: SubmitRejected, *, worker: str | None) -> NoReturn: - self.ports.reject(rejected, worker) - - def _context_decision( - self, - client: ClientState, - request: SubmitRequest, - control: SubmitControlSnapshot, - ) -> SubmitContextDecision: - active = control.active_context - retained = ( - self.ports.retained_entry(client, request.job_id) - if active is None - else None - ) - if active is None and retained is None: - # Reject before any live-tip fallback: an unknown job needs no - # RPC to classify. - try: - return classify_submit_context( - SubmitContextInput( - active_context=None, - retained_entry=None, - current_tip="", - stale_grace_eligible=False, - ) - ) - except SubmitRejected as rejected: - self._reject(rejected, worker=request.worker_name) - current_tip = ( - control.published_tip - if control.published_tip is not None - else self.ports.live_tip() - ) - # Share classification (normal and stale-grace alike) is deliberately - # point-in-time against this single tip read: a tip that advances - # between here and the ledger append does not retroactively invalidate - # the share, exactly as a normal current-tip share stays credited when - # the tip moves during processing. Re-checking would add an RPC per - # share during post-block bursts only to reject valid work over - # processing latency. Block submission is different (chain state): - # submit_block_candidate re-checks the tip under lock before - # submitblock, and stale-grace shares never reach it. - context = active if active is not None else retained.context - cross_connection = ( - retained is not None - and retained.connection_id != client.connection_id - ) - stale_eligible = False - if ( - not cross_connection - and str(context.template["previousblockhash"]) != current_tip - ): - # A cross-connection retained entry never consults the window: it - # is same-tip only, so the mismatch rejects as stale below - # without any parent-tip RPC (matching evicted_submit_context). - try: - stale_eligible = self.ports.stale_grace_eligible( - client, - context, - current_tip, - ) - except Exception: - self.ports.log( - "prism coordinator: failed to classify evicted submit context" - if retained is not None - else "prism coordinator: failed to classify stale-grace parent tip" - ) - self.ports.log_exception() - self._reject( - SubmitRejected( - 20, - PRISM_REJECTION_BACKEND_RPC_UNAVAILABLE, - "failed to classify stale-grace parent tip", - ), - worker=request.worker_name, - ) - try: - return classify_submit_context( - SubmitContextInput( - active_context=active, - retained_entry=retained, - current_tip=current_tip, - stale_grace_eligible=stale_eligible, - cross_connection=cross_connection, - ) - ) - except SubmitRejected as rejected: - self._reject(rejected, worker=request.worker_name) - - def handle(self, client: ClientState, params: list[object]) -> bool: - try: - request = parse_submit_request(params) - except SubmitRejected as rejected: - self._reject(rejected, worker=client.username or None) - # Capture pool admission, active-job membership, and published-tip - # authority together. All accounting, RPC fallback, hashing, and - # persistence remain outside the coordinator lock behind this port. - control = self.ports.control_snapshot(client, request.job_id) - try: - validate_submit_request( - request, - authorized_username=client.username, - pool_open=control.pool_open, - extranonce2_size=self.extranonce2_size, - ) - except SubmitRejected as rejected: - worker = ( - client.username or None - if rejected.reason == PRISM_REJECTION_UNAUTHORIZED_WORKER - else request.worker_name - ) - self._reject(rejected, worker=worker) - - # Count submitted shares once, after the format checks, so the - # per-worker counter and the aggregate submitted total cover the same - # population; malformed extranonce/ntime submits are recorded only as - # rejections, not submits. - self.ports.note_submitted(request.worker_name, client) - decision = self._context_decision(client, request, control) - submit_version_mask = client.version_mask - if decision.cross_connection: - # In-flight work from the dead connection was rolled under the - # mask negotiated there; the replacement's own mask (often still - # 0 before mining.configure) must not judge those version bits. - submit_version_mask = int( - getattr(decision.context, "version_mask", 0) - ) - try: - submission = self.ports.assemble( - client, - decision.context, - request, - submit_version_mask, - ) - except ValueError as error: - self._reject( - SubmitRejected( - 20, - PRISM_REJECTION_MALFORMED_SUBMIT, - f"malformed submit: {error}", - ), - worker=request.worker_name, - ) - # A retained job keeps its original worker even if the connection is - # later re-authorized. Deduplication must use that immutable identity: - # otherwise the same header can be replayed under each new username. - share_key = (decision.context.worker.username, submission.header_hex) - if not self.recent_shares.reserve(share_key): - self._reject( - SubmitRejected( - 22, - PRISM_REJECTION_DUPLICATE_SHARE, - "duplicate share", - ), - worker=request.worker_name, - ) - if ( - submission.block_pass - and decision.credit_policy == PRISM_CREDIT_POLICY_STALE_GRACE - ): - # Count the dropped solve before low-difficulty classification: - # the drop is real whether the stale-grace share passes its share - # target (credited below) or misses it (rejected below). - self._count_dropped_block_solve("stale_grace") - self.ports.log( - "prism coordinator: stale-grace block solve dropped " - f"hash={submission.block_hash_hex} " - f"parent={decision.context.template['previousblockhash']}" - ) - try: - work = classify_submit_work( - decision.context, - submission, - credit_policy=decision.credit_policy, - ) - except SubmitRejected as rejected: - self._reject(rejected, worker=request.worker_name) - - if work.block_worthy and decision.context.collection_only: - # Collection-mode jobs are block-worthy too: their signed - # bootstrap manifest already commits the whole coinbase to the - # submitting worker, so the solve settles solver-pays-all. - self.ports.note_collection_candidate(decision.context, submission) - pending_share = self.ports.pending_share( - decision.context, - submission, - request.ntime_hex, - decision.credit_policy, - ) - if work.route == "share": - try: - self.ports.append_share( - client, - decision.context, - submission, - pending_share, - decision.credit_policy, - None, - ) - if decision.retained_entry is not None: - self.ports.note_retained_submit( - decision.credit_policy, - decision.cross_connection, - ) - except BaseException: - self.recent_shares.release(work.share_key) - raise - return False - - candidate = PrismBlockCandidate( - context=decision.context, - submission=submission, - # The mined coinbase embeds the extranonce1 the job was stamped - # with. A cross-connection resume submits through a client whose - # own extranonce1 differs from the retained job's, and the audit - # bundle suffix must match the coinbase actually in the block, - # so the job's value is authoritative whenever it carries one. - extranonce1_hex=str( - getattr(decision.context.job, "extranonce1_hex", None) - or client.extranonce1_hex - ), - extranonce2_hex=request.extranonce2_hex, - pending_share=pending_share, - client=client, - credit_share_on_accept=work.credit_share_on_accept, - ) - if work.route == "synchronous_block": - return self._submit_synchronous( - candidate, - work=work, - request=request, - decision=decision, - ) - return self._submit_asynchronous( - candidate, - work=work, - client=client, - decision=decision, - submission=submission, - ) - - def _submit_synchronous( - self, - candidate: PrismBlockCandidate, - *, - work: SubmitWorkDecision, - request: SubmitRequest, - decision: SubmitContextDecision, - ) -> bool: - # The hash solved a block but missed the assigned share target - # (possible only while the listener floor sits above network - # difficulty). It is a valid share ONLY if the block lands, so it - # lands synchronously behind this port: the durable-outbox boundary, - # the terminal-state short circuit, and the low-difficulty rejection - # for a candidate that does not land all run coordinator-side where - # the block-submitter machinery lives. - return self.ports.submit_synchronous_candidate( - candidate, - work.share_key, - request.worker_name, - decision.retained_entry, - decision.credit_policy, - ) - - def _submit_asynchronous( - self, - candidate: PrismBlockCandidate, - *, - work: SubmitWorkDecision, - client: ClientState, - decision: SubmitContextDecision, - submission: Any, - ) -> bool: - # A block-worthy submission that met the share target is a valid - # share regardless of the block's fate: credit it now, acknowledge - # the miner immediately, and land the block from the dedicated - # submitter thread (ckpool/btcpool/StratumV2 semantics). An orphaned - # candidate keeps its share credit. - try: - candidate_intent = self.ports.candidate_intent(candidate) - durable_candidate_state = self.ports.append_share( - client, - decision.context, - submission, - candidate.pending_share, - decision.credit_policy, - candidate_intent, - ) - if decision.retained_entry is not None: - self.ports.note_retained_submit( - decision.credit_policy, - decision.cross_connection, - ) - except BaseException: - # Idempotent with the append's own release; also covers an intent - # serialization failure before the append started. - self.ports.finish_pending_commit(candidate.pending_share) - self.recent_shares.release(work.share_key) - raise - if durable_candidate_state in {"submitted", "abandoned"}: - # A process restart clears the in-memory disposition cache, but - # the existing outbox row remains authoritative. Join its - # terminal result instead of enqueueing a new node offer. - self.ports.record_terminal_outcome( - submission.block_hash_hex, - durable_candidate_state == "submitted", - ) - return False - self.ports.enqueue_candidate(candidate) - return False diff --git a/lab/prism/share_writer.py b/lab/prism/share_writer.py deleted file mode 100644 index 964f9c37e..000000000 --- a/lab/prism/share_writer.py +++ /dev/null @@ -1,831 +0,0 @@ -#!/usr/bin/env python3 -"""Durable accepted-share writer, pending-commit floor, and legacy recovery. - -This module owns the S3 domain: the bounded group-commit queue and writer -loop, the pending-share snapshot-anchor floor, disk recovery of acked shares -the ledger could not persist, and startup replay of that recovery journal. - -It never imports ``prism_coordinator``. The ledger handle, shutdown -controller, watchdog heartbeats, the P1 append/anchor fencing entrypoints, -and live configuration attributes are reached through the -:class:`ShareWriterRuntime` typed port, resolved at call time so the -historical coordinator monkeypatch seams (including the instance-level facade -patches used by the current test suite) keep intercepting exactly as before -the extraction. Replay semantics deliberately keep the current tree's -behavior; the typed exact/conflict recovery consumption from the share -ledger arrives with a later chunk through the ``replay_recovered_shares`` -seam below. -""" - -from __future__ import annotations - -import dataclasses -from dataclasses import dataclass, field -import json -import os -import queue -import threading -import time -import traceback -from typing import Any, Protocol - -from lab.prism.share_ledger import PendingShare, ShareReplayConflict -from lab.prism.stratum_session import StratumError - - -MAX_PENDING_SHARE_APPENDS = 4_096 -# After this long on the pending-commit floor a share is assumed wedged (not -# normal group-commit latency) and the anchor clamp logs it once. -PENDING_SHARE_COMMIT_WARN_SECONDS = 30.0 -_WRITER_EXIT_COMPONENTS = frozenset( - {"share_submission", "share_persistence", "accepted_block_handling"} -) - - -class ShareWriterError(RuntimeError): - """A persistence operation failed before it could be acknowledged.""" - - -class ShareWriterQueueFull(ShareWriterError): - """The bounded share group-commit queue rejected an entry.""" - - -@dataclass -class PendingShareAppend: - """A share waiting for the ledger group-commit writer. - - The client thread does not count or acknowledge this share until - ``committed`` is set successfully. A block candidate intent, when - present, is inserted in the same transaction as the share. - """ - - pending_share: PendingShare - username: str - job_id: str - block_hash_hex: str - collection_only: bool - credit_policy: str | None - # The prepared candidate intent (#255), a mapping whose share sequence - # is the job's immutable window. Released by the writer the moment the - # entry reaches its terminal outcome so an idle writer local or a - # finished batch never pins a payout window. - candidate_intent: Any | None = None - committed: threading.Event = field(default_factory=threading.Event) - record: Any | None = None - error: BaseException | None = None - writer_token: "_WriterOperationToken | None" = None - - -class ShareWriterRuntime(Protocol): - """Typed port over the coordinator, resolved at call time. - - Every member is looked up on the live coordinator object when used, so - instance monkeypatches (``server._finish_pending_share_commit = ...``, - ``server._append_share_batch = ...`` and friends) and coordinator-owned - live state (the ledger handle, the P1 append/anchor fencing) keep working - exactly as before the extraction. Owner facades route back into this - service; that round trip is deliberate. The legacy S3 field names - (``share_append_queue`` through ``_pending_share_commit_floor``) also - resolve here -- coordinator class descriptors route them to this - service's single mutable copy. - """ - - # Cross-domain objects and live configuration attributes. - hot_path_log_enabled: Any - ledger: Any - stop_event: Any - - def _append_share_batch(self, *args: Any, **kwargs: Any) -> Any: ... - - def _append_share_entry(self, *args: Any, **kwargs: Any) -> Any: ... - - def _ensure_share_hot_path_state(self, *args: Any, **kwargs: Any) -> Any: ... - - def _ensure_shutdown_controller(self, *args: Any, **kwargs: Any) -> Any: ... - - def _finish_pending_share_commit(self, *args: Any, **kwargs: Any) -> Any: ... - - def _landing_fence_for_predating_append(self, *args: Any, **kwargs: Any) -> Any: ... - - def _record_heartbeat(self, *args: Any, **kwargs: Any) -> Any: ... - - def _record_late_visible_payout_append(self, *args: Any, **kwargs: Any) -> Any: ... - - def _recover_share_to_disk(self, *args: Any, **kwargs: Any) -> Any: ... - - def _retire_payout_windows_for_late_append(self, *args: Any, **kwargs: Any) -> Any: ... - - def accepted_share_difficulty(self, *args: Any, **kwargs: Any) -> Any: ... - - def enqueue_share_append(self, *args: Any, **kwargs: Any) -> Any: ... - - def note_vardiff_accepted_share(self, *args: Any, **kwargs: Any) -> Any: ... - - def note_worker_accepted_share(self, *args: Any, **kwargs: Any) -> Any: ... - - -class ShareWriter: - """Sole owner of S3 group commit, the pending floor, and recovery.""" - - def __init__( - self, - runtime: ShareWriterRuntime, - *, - internal_error_reason: str, - now_ms: Any, - ) -> None: - self._runtime = runtime - # The rejection-reason id remains a coordinator registry constant and - # the wall-clock stamp resolves the coordinator module's ``now_ms`` - # global at call time (tests patch it there); both are injected so - # this leaf module never imports prism_coordinator. - self._internal_error_reason = internal_error_reason - self._now_ms = now_ms - # Accepted shares drain through a bounded group-commit writer. A - # submitting client waits on its entry's completion event, making the - # database commit the acknowledgement boundary without paying one - # process/transaction round trip per share during bursts. - self.share_append_queue: queue.Queue[PendingShareAppend] = queue.Queue( - maxsize=MAX_PENDING_SHARE_APPENDS - ) - self.share_commit_batch_size = 64 - self.share_commit_linger_seconds = 0.005 - self.share_commit_timeout_seconds = 15.0 - self.share_writer_active = False - self.share_append_failure_count = 0 - # Retain the historical recovery-file reader for clean upgrades from a - # release that could acknowledge before Postgres commit. New shares - # are never written here: an unavailable ledger produces no success - # acknowledgement and an exact retry is idempotent. - self.share_recovery_path = None - self.share_recovery_lock = threading.Lock() - self.shares_recovered_to_disk = 0 - self.shares_replayed = 0 - self.share_replay_conflicts = 0 - self._pending_share_commit_lock = threading.Lock() - # Shares whose accepted_at_ms has been assigned but whose ledger - # row has not reached a terminal outcome in this process. Job and - # payout-artifact snapshot anchors clamp below every entry so a - # bundle's eligible-share set stays exactly reproducible from the - # durable ledger after those rows commit. - self._pending_share_commit_floor: dict[int, list[object]] = {} - - # -- pending-commit floor ---------------------------------------------- - - def finish_pending_share(self, pending_share: PendingShare) -> None: - """Drop a share from the snapshot anchor floor. - - Called once the share's ledger row reached a terminal outcome in this - process: durably committed, rejected back to the miner, recovered to - the on-disk replay file, or its block candidate terminally abandoned. - Idempotent, and a no-op for shares this process never registered - (intent/journal replays re-create PendingShare objects from JSON). - """ - with self._pending_share_commit_lock: - self._pending_share_commit_floor.pop(id(pending_share), None) - - def finish_pending_attempt(self, pending_share: PendingShare) -> None: - """Release one stamped submission attempt's floor authority. - - Layer-original seam: at this layer the floor keeps the current - tree's single object-identity holder per stamped share, so attempt - release and candidate release act on the same entry. A later chunk - introduces independent attempt/durable-candidate/actor holders behind - these two adapters without touching their callers. - """ - self.finish_pending_share(pending_share) - - def finish_pending_candidate(self, pending_share: PendingShare) -> None: - """Release the durable block-candidate actor's floor authority. - - See :meth:`finish_pending_attempt` for the staged holder split. - """ - self.finish_pending_share(pending_share) - - def adopt_pending_share(self, pending_share: PendingShare) -> None: - """Register an externally reconstructed share on the pending floor. - - Durable block-candidate replay decodes a credit-bearing PendingShare - whose accepted stamp predates this process. Adopting it here clamps - job/payout snapshot anchors below that stamp before the replayed - candidate becomes visible to issuance, exactly as if the share had - been stamped live. Released through the ordinary - :meth:`finish_pending_candidate` seam once the candidate reaches a - terminal outcome. - """ - with self._pending_share_commit_lock: - self._pending_share_commit_floor[id(pending_share)] = [ - pending_share, - time.monotonic(), - False, - ] - - def snapshot_anchor_ms(self, issued_at_ms: int) -> int: - """Clamp a share-snapshot anchor below every coverable share stamp. - - The reward-window contract lets an auditor replay - qbit_audit_share_window(anchor) against the durable ledger and expect - exactly the shares the published bundle counted. Two hazards would - violate that. A share whose accepted_at_ms is already assigned but - whose row has not committed yet (group-commit queue, in-flight batch, - or a block-candidate credit linked after landing) is invisible to the - MVCC snapshot now but joins later replays at any anchor at or above - its accepted_at_ms. And a share stamped right after this clamp - returns can land in the same millisecond as the clamp instant -- - never protected by the pending floor -- so an anchor equal to that - instant would cover it too (the window predicate is - anchor-inclusive). Anchoring strictly below every pending share and - strictly below the clamp-time millisecond keeps the issued snapshot - reproducible without making job builds wait behind the writer - connection: any stamp assigned after this call is at or above the - clamp instant, hence above the anchor. - """ - stale_share_ids: list[str] = [] - floor_ms: int | None = None - now_monotonic = time.monotonic() - with self._pending_share_commit_lock: - for entry in self._pending_share_commit_floor.values(): - share = entry[0] - accepted_at_ms = int(share.accepted_at_ms) - if floor_ms is None or accepted_at_ms < floor_ms: - floor_ms = accepted_at_ms - if ( - not entry[2] - and now_monotonic - float(entry[1]) - > PENDING_SHARE_COMMIT_WARN_SECONDS - ): - entry[2] = True - stale_share_ids.append(str(share.share_id)) - for share_id in stale_share_ids: - # A long-held floor entry is a wedged writer or a leaked release - # path, not normal group-commit latency. The share-commit liveness - # watchdog owns recovery; this log makes the anchor clamp visible. - print( - "prism coordinator: pending share commit is holding the job " - f"snapshot anchor floor share_id={share_id}", - flush=True, - ) - if floor_ms is None: - return issued_at_ms - 1 - return min(issued_at_ms - 1, floor_ms - 1) - - def pending_share_from_submission( - self, - *, - context: Any, - submission: Any, - ntime_hex: str, - credit_policy: str | None = None, - ) -> PendingShare: - runtime = self._runtime - share_difficulty = runtime.accepted_share_difficulty(context) - # Assign accepted_at_ms and register the commit-floor entry under one - # lock hold: a snapshot anchored between assignment and registration - # could otherwise anchor at or above this share and miss its later - # commit. Released via _finish_pending_share_commit. - with self._pending_share_commit_lock: - pending = PendingShare( - share_id=f"{context.worker.username}:{submission.block_hash_hex}", - miner_id=context.worker.payout_address, - order_key=context.worker.payout_address, - p2mr_program_hex=context.worker.p2mr_program_hex, - share_difficulty=share_difficulty, - network_difficulty=max(1, int(context.found_block["network_difficulty"])), - template_height=int(context.template["height"]) - 1, - job_id=context.job.job_id, - job_issued_at_ms=context.issued_at_ms, - accepted_at_ms=self._now_ms(), - ntime=int(ntime_hex, 16), - credit_policy=credit_policy, - ) - self._pending_share_commit_floor[id(pending)] = [ - pending, - time.monotonic(), - False, - ] - return pending - - # -- append paths ------------------------------------------------------ - - def append_accepted_share( - self, - client: Any, - context: Any, - submission: Any, - pending_share: PendingShare, - *, - credit_policy: str | None = None, - candidate_intent: dict[str, Any] | None = None, - ) -> str | None: - runtime = self._runtime - entry = PendingShareAppend( - pending_share=pending_share, - username=context.worker.username, - job_id=context.job.job_id, - block_hash_hex=submission.block_hash_hex, - collection_only=bool(context.collection_only), - credit_policy=credit_policy, - candidate_intent=candidate_intent, - ) - try: - if getattr(self, "share_writer_active", False): - runtime.enqueue_share_append(entry, wait=True) - else: - runtime._append_share_entry(entry) - finally: - # The append reached a terminal outcome for this process: durably - # committed, or surfaced an error the miner will retry with a fresh - # share. Either way the stamped share no longer holds the snapshot - # anchor floor. Idempotent with the group-commit writer's release. - runtime._finish_pending_share_commit(pending_share) - record = entry.record - # Exact ledger replays return the original record. Their durable share - # must not increment process-local worker/vardiff counters again. - if record is None or bool(getattr(record, "newly_inserted", True)): - runtime.note_worker_accepted_share(context.worker.username, credit_policy) - runtime.note_vardiff_accepted_share(client, context.job) - if candidate_intent is None or record is None: - return None - candidate_state = getattr(record, "candidate_outbox_state", None) - return str(candidate_state) if candidate_state is not None else None - - def enqueue_share_append( - self, entry: PendingShareAppend, *, wait: bool = False - ) -> None: - runtime = self._runtime - queue_obj = getattr(self, "share_append_queue", None) - if queue_obj is None: - queue_obj = queue.Queue(maxsize=MAX_PENDING_SHARE_APPENDS) - self.share_append_queue = queue_obj - if entry.writer_token is None: - entry.writer_token = runtime._ensure_shutdown_controller().reserve_writer( - "share_persistence" - ) - try: - if wait: - queue_obj.put( - entry, - timeout=getattr(self, "share_commit_timeout_seconds", 15.0), - ) - else: - queue_obj.put_nowait(entry) - except queue.Full: - entry.writer_token.finish() - entry.writer_token = None - raise StratumError( - 20, - "share ledger commit queue is full", - reason=self._internal_error_reason, - ) - if not wait: - return - # Once admitted, wait for a definite transaction outcome. A local - # timeout is ambiguous because Postgres may commit immediately after - # it; the liveness watchdog owns recovery from a wedged writer. - entry.committed.wait() - if entry.error is not None: - raise StratumError( - 20, - f"share ledger commit failed: {entry.error}", - reason=self._internal_error_reason, - ) - - def share_append_loop(self) -> None: - runtime = self._runtime - while True: - runtime._record_heartbeat("share_writer") - queue_obj = getattr(self, "share_append_queue", None) - if queue_obj is None: - queue_obj = queue.Queue(maxsize=MAX_PENDING_SHARE_APPENDS) - self.share_append_queue = queue_obj - stopping = runtime.stop_event.is_set() - try: - entry = queue_obj.get(timeout=0.2 if stopping else 1.0) - except queue.Empty: - controller = runtime._ensure_shutdown_controller() - if ( - stopping - and controller.writer_admission_closed() - and not controller.has_active_writer(set(_WRITER_EXIT_COMPONENTS)) - ): - return - continue - batch = [entry] - batch_size = max(1, int(getattr(self, "share_commit_batch_size", 64))) - deadline = time.monotonic() + max( - 0.0, float(getattr(self, "share_commit_linger_seconds", 0.005)) - ) - if entry.candidate_intent is not None: - deadline = time.monotonic() - while len(batch) < batch_size: - remaining = deadline - time.monotonic() - if remaining <= 0: - break - try: - next_entry = queue_obj.get(timeout=remaining) - batch.append(next_entry) - if next_entry.candidate_intent is not None: - break - except queue.Empty: - break - runtime._append_share_batch(batch) - - def append_share_batch(self, batch: list[PendingShareAppend]) -> bool: - """Commit a writer batch, then release every waiting submitter.""" - runtime = self._runtime - try: - invalidations: list[tuple[PendingShare, int]] = [] - # A batch holding a replay-shaped row commits under the landing - # fence: the durable append and its epoch bump must land on the - # same side of a landing's fence-guarded submit, or the landing - # can verify the pre-bump epoch after the row is already durable - # (see _landing_fence_for_predating_append). - with runtime._landing_fence_for_predating_append( - [entry.pending_share for entry in batch] - ) as fence_owned: - append_batch = getattr(runtime.ledger, "append_batch", None) - if callable(append_batch): - records = append_batch( - [(entry.pending_share, entry.candidate_intent) for entry in batch] - ) - else: - # Compatibility for lightweight test/tool ledgers. Production's - # Postgres ledger always supplies the atomic batch method. - records = [runtime.ledger.append(entry.pending_share) for entry in batch] - if len(records) != len(batch): - raise RuntimeError("share ledger returned an incomplete commit batch") - for entry, record in zip(batch, records, strict=True): - entry.record = record - invalidation_epoch = runtime._record_late_visible_payout_append( - entry.pending_share, - landing_fence_owned=fence_owned, - ) - if invalidation_epoch is not None: - invalidations.append( - (entry.pending_share, invalidation_epoch) - ) - for pending_share, invalidation_epoch in invalidations: - runtime._retire_payout_windows_for_late_append( - pending_share, invalidation_epoch - ) - if getattr(runtime, "hot_path_log_enabled", False): - for entry in batch: - record = entry.record - print( - "prism coordinator: accepted share " - f"seq={record.share_seq} miner={entry.username} job={entry.job_id} " - f"hash={entry.block_hash_hex} collection={entry.collection_only} " - f"credit_policy={entry.credit_policy or 'normal'}", - flush=True, - ) - return True - except Exception as exc: - runtime._ensure_share_hot_path_state() - with runtime._share_accounting_lock: - self.share_append_failure_count = ( - int(getattr(self, "share_append_failure_count", 0)) + len(batch) - ) - for entry in batch: - entry.error = exc - print( - f"prism coordinator: share ledger group commit failed count={len(batch)}", - flush=True, - ) - traceback.print_exc() - return False - finally: - for entry in batch: - runtime._finish_pending_share_commit(entry.pending_share) - # Terminal for this entry either way (committed, or failed - # and reported to the miner): drop the intent's hold on its - # window before waking the submitter. - entry.candidate_intent = None - entry.committed.set() - if entry.writer_token is not None: - entry.writer_token.finish() - entry.writer_token = None - - def recover_share_to_disk(self, entry: PendingShareAppend, reason: str) -> None: - """Durably capture an acked share the writer could not persist. - - Appends the canonical pending-share JSON to the recovery file (fsynced) - so a ledger outage or shutdown never silently loses a share the miner - was told was accepted; replayed on the next start. Best-effort: if even - the recovery write fails, log loudly rather than raise on the writer. - """ - path = getattr(self, "share_recovery_path", None) - if path is None: - print( - "prism coordinator: WOULD LOSE acked share (no recovery path) " - f"share_id={entry.pending_share.share_id} reason={reason}", - flush=True, - ) - return - try: - payload = json.dumps(dataclasses.asdict(entry.pending_share), separators=(",", ":")) - except Exception: - payload = None - with getattr(self, "share_recovery_lock", threading.Lock()): - try: - path.parent.mkdir(parents=True, exist_ok=True) - if payload is None: - raise ValueError("pending share is not serializable") - with open(path, "a", encoding="utf-8") as handle: - handle.write(payload + "\n") - handle.flush() - os.fsync(handle.fileno()) - self.shares_recovered_to_disk = ( - int(getattr(self, "shares_recovered_to_disk", 0)) + 1 - ) - print( - "prism coordinator: recovered unpersisted acked share to disk " - f"share_id={entry.pending_share.share_id} reason={reason}", - flush=True, - ) - except Exception: - print( - "prism coordinator: FAILED to recover acked share to disk; " - f"share may be lost share_id={entry.pending_share.share_id} reason={reason}", - flush=True, - ) - traceback.print_exc() - - def replay_recovered_shares(self) -> int: - """Replay any recovery-file shares into the ledger at startup. - - Ledgers with typed recovery support report each row's disposition - through ``share_ledger.append_recovered_share``: an ``inserted`` row - counts as replayed, an ``exact_existing`` row is an already-committed - duplicate from an earlier partial replay (skipped, not - double-counted). A :class:`ShareReplayConflict` is keyed to one - share_id, so the conflicting row is quarantined -- counted, logged, - and preserved in the retained journal -- while the remaining rows - keep replaying instead of stranding behind it; an unknown disposition - conservatively stops and keeps the journal file for inspection. Only - a clean pass may remove the journal, so a transient failure here - never drops shares. Embedder ledgers without ``append_recovered_share`` keep the - historical ``append`` path whose duplicate-share_id error is treated - as an exact replay. Writer admission stays with the coordinator - facade's ``ledger_writer_operation`` decorator. - """ - runtime = self._runtime - path = getattr(self, "share_recovery_path", None) - if path is None or not path.exists(): - return 0 - try: - lines = [line for line in path.read_text(encoding="utf-8").splitlines() if line.strip()] - except Exception: - print("prism coordinator: could not read share recovery file", flush=True) - traceback.print_exc() - return 0 - # Parse line-by-line and skip any single unparseable line rather than - # aborting the whole replay: a crash mid-append can leave the last line - # torn, and one torn line must not block the intact shares before it. - pendings: list[PendingShare] = [] - parse_failed = False - for line in lines: - try: - pendings.append(PendingShare(**json.loads(line))) - except Exception: - parse_failed = True - print("prism coordinator: skipping an unparseable recovered share line", flush=True) - traceback.print_exc() - # Replay in acceptance order. A share recovered out of FIFO order (a - # ledger flap during the shutdown drain, or an overflow-recovered newest - # share) otherwise sorts by file order; ordering by accepted_at_ms lands - # each share with a share_seq consistent with when it was accepted, so - # the reward window stays correctly ordered. - pendings.sort(key=lambda pending: pending.accepted_at_ms) - replayed = 0 - skipped_duplicates = 0 - replay_conflicts = 0 - for pending in pendings: - append_recovered = getattr( - runtime.ledger, "append_recovered_share", None - ) - try: - # A recovered row reconstructs its pre-crash timestamps, so - # it routinely predates a live anchor; each row commits under - # the landing fence with its epoch bump so a concurrent - # landing can never verify a pre-bump epoch against an - # already-durable replayed share. The window retirement runs - # after the fence releases, per row, so replay never holds - # the fence across the prepare-lock wait. - invalidation_epoch: int | None = None - with runtime._landing_fence_for_predating_append( - [pending] - ) as fence_owned: - if callable(append_recovered): - result = append_recovered(pending) - disposition = getattr(result, "disposition", None) - if disposition == "exact_existing": - # Already committed by an earlier (partial) - # replay; treat it as done so a retry never - # stops on the first committed row and strands - # every share after it. The durable row already - # bumped visibility on its original commit, so - # no new epoch bump happens here. - skipped_duplicates += 1 - continue - if disposition != "inserted": - print( - "prism coordinator: recovery ledger returned an " - f"unsupported replay disposition {disposition!r}; " - "keeping the file", - flush=True, - ) - self.shares_replayed = ( - int(getattr(self, "shares_replayed", 0)) + replayed - ) - return replayed - else: - runtime.ledger.append(pending) - invalidation_epoch = runtime._record_late_visible_payout_append( - pending, - landing_fence_owned=fence_owned, - ) - if invalidation_epoch is not None: - runtime._retire_payout_windows_for_late_append( - pending, invalidation_epoch - ) - replayed += 1 - except ShareReplayConflict: - # The durable row disagrees with the journal payload. The - # conflict is keyed to this one share_id and says nothing - # about the rows after it, so quarantine the row -- count it - # and keep the file for inspection -- and continue replaying - # the rest of the journal instead of stranding valid shares - # behind it. Exact duplicates replay idempotently, so a - # re-run of the retained file never double-credits. - replay_conflicts += 1 - self.share_replay_conflicts = ( - int(getattr(self, "share_replay_conflicts", 0)) + 1 - ) - print( - "prism coordinator: recovered share conflicts with durable " - "payload; quarantining the row and keeping the file " - f"share_id={pending.share_id}", - flush=True, - ) - continue - except Exception as exc: - if not callable(append_recovered) and "duplicate share_id" in str(exc): - # Historical string-matched idempotence for embedder - # ledgers without typed replay support. - skipped_duplicates += 1 - continue - print("prism coordinator: failed to replay a recovered share; keeping the file", flush=True) - traceback.print_exc() - self.shares_replayed = int(getattr(self, "shares_replayed", 0)) + replayed - return replayed - if skipped_duplicates: - print( - f"prism coordinator: skipped {skipped_duplicates} already-committed " - "recovered share(s) during replay", - flush=True, - ) - if replay_conflicts: - print( - f"prism coordinator: quarantined {replay_conflicts} conflicting " - "recovered share(s) during replay; keeping the file", - flush=True, - ) - if parse_failed or replay_conflicts: - # Keep the file (with its intact-but-already-replayed lines, which - # the ledger dedups on a re-run) so the torn or conflicting line is - # preserved for inspection rather than silently discarded. - self.shares_replayed = int(getattr(self, "shares_replayed", 0)) + replayed - if replayed: - print(f"prism coordinator: replayed {replayed} recovered share(s) into the ledger", flush=True) - return replayed - try: - path.unlink() - except FileNotFoundError: - pass - self.shares_replayed = int(getattr(self, "shares_replayed", 0)) + replayed - if replayed: - print(f"prism coordinator: replayed {replayed} recovered share(s) into the ledger", flush=True) - return replayed - - def append_share_entry( - self, entry: PendingShareAppend, *, retry_until_stopped: bool = False - ) -> bool: - """Synchronously append one accepted share. - - On the writer thread a transient ledger failure retries with capped - backoff so ordering is preserved and nothing is silently lost; the - synchronous path (no writer) propagates the exception exactly as the - pre-async code did. - - Returns True when the share was persisted to the ledger, or False when - it was recovered to disk instead (ledger still down at shutdown). The - caller uses that to keep the shutdown drain in order. - """ - runtime = self._runtime - backoff_seconds = 0.5 - invalidation_epoch: int | None = None - while True: - try: - # A replay-shaped row commits under the landing fence with - # its epoch bump (see _landing_fence_for_predating_append). - # The fence wraps one attempt only: the backoff sleep below - # runs outside it, so a retrying ledger outage never holds - # up a landing. - with runtime._landing_fence_for_predating_append( - [entry.pending_share] - ) as fence_owned: - append_batch = getattr(runtime.ledger, "append_batch", None) - if callable(append_batch): - record = append_batch( - [(entry.pending_share, entry.candidate_intent)] - )[0] - else: - record = runtime.ledger.append(entry.pending_share) - entry.record = record - invalidation_epoch = runtime._record_late_visible_payout_append( - entry.pending_share, - landing_fence_owned=fence_owned, - ) - break - except Exception: - if not retry_until_stopped: - raise - runtime._ensure_share_hot_path_state() - with runtime._share_accounting_lock: - self.share_append_failure_count = ( - int(getattr(self, "share_append_failure_count", 0)) + 1 - ) - print( - "prism coordinator: ledger share append failed; retrying " - f"share_id={entry.pending_share.share_id}", - flush=True, - ) - traceback.print_exc() - if runtime.stop_event.wait(backoff_seconds): - # Shutting down mid-outage: do not silently drop this - # already-acked, already-counted share -- recover it to - # disk for replay on the next start. - runtime._recover_share_to_disk(entry, "ledger unavailable at shutdown") - entry.candidate_intent = None - return False - backoff_seconds = min(backoff_seconds * 2, 5.0) - runtime._record_heartbeat("share_writer") - entry.candidate_intent = None - if invalidation_epoch is not None: - runtime._retire_payout_windows_for_late_append( - entry.pending_share, invalidation_epoch - ) - if getattr(runtime, "hot_path_log_enabled", False): - print( - "prism coordinator: accepted share " - f"seq={record.share_seq} miner={entry.username} job={entry.job_id} " - f"hash={entry.block_hash_hex} collection={entry.collection_only} " - f"credit_policy={entry.credit_policy or 'normal'}", - flush=True, - ) - entry.committed.set() - return True - - -class ShareWriterCompatibilityField: - """Route one legacy S3 coordinator field to its owner service. - - The descriptor keeps the historical attribute name readable and writable - on the coordinator while :class:`ShareWriter` owns the single mutable - copy, mirroring the S1/S2/J1/G1 extraction pattern. Direct test - assignment through the legacy name is thereby adopted by the owner. - """ - - def __init__(self, attribute: str) -> None: - self._attribute = attribute - - def __get__( - self, - instance: Any, - owner: type | None = None, - ) -> Any: - if instance is None: - return self - return getattr( - instance._ensure_share_writer_service(), - self._attribute, - ) - - def __set__(self, instance: Any, value: Any) -> None: - setattr( - instance._ensure_share_writer_service(), - self._attribute, - value, - ) - - -__all__ = [ - "MAX_PENDING_SHARE_APPENDS", - "PENDING_SHARE_COMMIT_WARN_SECONDS", - "PendingShareAppend", - "ShareWriter", - "ShareWriterCompatibilityField", - "ShareWriterError", - "ShareWriterQueueFull", - "ShareWriterRuntime", -] diff --git a/lab/prism/statement_spool.py b/lab/prism/statement_spool.py deleted file mode 100644 index b66cf9062..000000000 --- a/lab/prism/statement_spool.py +++ /dev/null @@ -1,105 +0,0 @@ -"""Execute streamed finalization statements with bounded coordinator memory. - -The existing atomic fenced SQL is unchanged. psql owns its unavoidable full -statement buffer in an isolated process; the coordinator passes an open file, -never the complete SQL string, and reads only a bounded result after success. -""" - -from __future__ import annotations - -import os -import subprocess -import sys -import tempfile -import time -from collections.abc import Iterable -from typing import Any - - -STATEMENT_CHUNK_CHARS = 4096 -STATEMENT_SPOOL_BYTES = 4 * 1024 * 1024 * 1024 -STATEMENT_HELPER_MEMORY_BYTES = 4 * 1024 * 1024 * 1024 -STATEMENT_RESULT_BYTES = 64 * 1024 -STATEMENT_HELPER_TIMEOUT_SECONDS = 600.0 -_EXEC_LIMITED = """ -import os, resource, sys -resource.setrlimit(resource.RLIMIT_AS, (int(sys.argv[1]), int(sys.argv[1]))) -resource.setrlimit(resource.RLIMIT_FSIZE, (int(sys.argv[2]), int(sys.argv[2]))) -os.execvp(sys.argv[3], sys.argv[3:]) -""" - - -def run_fenced_statement(ledger: Any, pieces: Iterable[str]) -> Any: - from lab.prism.audit_bundle_view import ArtifactResourcePressure, RECORD_HELPER_ADMISSION - from lab.prism.share_ledger import LedgerOperationTimeout, parse_single_json_value - - with tempfile.TemporaryFile() as statement, tempfile.TemporaryFile() as result, tempfile.TemporaryFile() as errors: - size = 0 - try: - for piece in pieces: - for offset in range(0, len(piece), STATEMENT_CHUNK_CHARS): - ledger._remaining_operation_timeout() - chunk = piece[offset:offset + STATEMENT_CHUNK_CHARS].encode("utf-8") - size += len(chunk) - if size > STATEMENT_SPOOL_BYTES: - raise ArtifactResourcePressure("finalization statement spool reservation exhausted") - statement.write(chunk) - except OSError as exc: - raise ArtifactResourcePressure("finalization statement spool unavailable") from exc - statement.seek(0) - deadline = time.monotonic() + STATEMENT_HELPER_TIMEOUT_SECONDS - - def check() -> None: - ledger._remaining_operation_timeout() - if time.monotonic() >= deadline: - raise LedgerOperationTimeout("finalization statement helper exceeded its deadline") - - with ledger._operation_gate(ledger._lock, "writer lock"), RECORD_HELPER_ADMISSION.hold(check): - command, kwargs, timeout = ledger._psql_invocation() - environment = dict(kwargs.get("env", os.environ)) - native = getattr(ledger, "_native", None) - if native is not None: - # Preserve the exact native DSN, including private schemas, - # then append the same guards and deadlines as ordinary psql. - from psycopg.conninfo import conninfo_to_dict, make_conninfo - - connection = conninfo_to_dict(native._conninfo) - guards = getattr(ledger, "_session_guards", None) - fragments = [connection.get("options", "")] - if guards is not None: - fragments.append(guards.options_fragment()) - if timeout is not None: - timeout_ms = max(1, int(timeout * 1000)) - fragments.append(f"-c statement_timeout={timeout_ms}ms -c lock_timeout={timeout_ms}ms") - connection["options"] = " ".join(filter(None, fragments)) - connection["application_name"] = ledger._pool_application_name - password = connection.pop("password", None) - if password is not None: - environment["PGPASSWORD"] = password - command = ["psql", "--dbname", make_conninfo( - **connection, - ), - *command[len(ledger._command):]] - if timeout is not None: - deadline = min(deadline, time.monotonic() + timeout) - process = subprocess.Popen( - [sys.executable, "-c", _EXEC_LIMITED, - str(STATEMENT_HELPER_MEMORY_BYTES), str(STATEMENT_RESULT_BYTES), *command], - stdin=statement, stdout=result, stderr=errors, env=environment, - ) - try: - while process.poll() is None: - check() - time.sleep(0.05) - finally: - if process.poll() is None: - process.kill() - process.wait() - errors.seek(0) - diagnostic = errors.read(STATEMENT_RESULT_BYTES).decode("utf-8", "replace") - ledger._check_psql_exit(process.returncode, diagnostic, timeout) - result.seek(0) - output = result.read(STATEMENT_RESULT_BYTES + 1) - if len(output) > STATEMENT_RESULT_BYTES: - raise RuntimeError("finalization statement result exceeds its byte limit") - return parse_single_json_value(output.decode("utf-8")) diff --git a/lab/prism/stratum_session.py b/lab/prism/stratum_session.py deleted file mode 100644 index 8b71c6f04..000000000 --- a/lab/prism/stratum_session.py +++ /dev/null @@ -1,1583 +0,0 @@ -#!/usr/bin/env python3 -"""PRISM Stratum listener and session lifecycle. - -This module owns the S1 domain: miner-facing connection admission, protocol -dispatch, worker resolution, payout-address validation, and the live-session -registry (membership, connection generations, handler-thread accounting, and -exact per-connection delivery proof). - -It never imports ``prism_coordinator``. Job delivery, share submission, -progress health, shutdown state, and live configuration attributes are -reached through the :class:`SessionRuntime` typed port, resolved at call -time so the historical coordinator monkeypatch seams (including the -instance-level facade patches used by the current test suite) keep -intercepting exactly as before the extraction. Share-ACK timing state -remains coordinator-owned at this layer (it moves with the PR 77 submission -owner); the session code stamps ``request_received_monotonic`` and reports -the response boundary through the runtime seam. -""" - -from __future__ import annotations - -from collections import OrderedDict -from contextlib import ExitStack -from dataclasses import dataclass, field -from decimal import Decimal -import errno -import json -import socket -import struct -import threading -import time -import traceback -from types import MappingProxyType -from typing import Any, Callable, Mapping, Protocol - -from lab.auxpow import stratum_codec, vardiff -from lab.prism import direct_stratum -from lab.prism.coordinator_config import ( - DEFAULT_PRISM_PAYOUT_ADDRESS_CACHE_MAX_ENTRIES, - DEFAULT_PRISM_PAYOUT_ADDRESS_CACHE_TTL_SECONDS, - DEFAULT_PRISM_STRATUM_ACCEPT_RESOURCE_EXHAUSTION_BACKOFF_SECONDS, - DEFAULT_PRISM_STRATUM_MAX_CONNECTIONS, - DEFAULT_PRISM_STRATUM_MAX_CONNECTIONS_PER_USERNAME, - StratumListenerProfile, - default_prism_username_fallback_address, -) -from lab.prism.progress_health import DeliveryProof, WorkGeneration -from lab.prism.template_artifacts import qbit_template_fingerprint - - -_VARDIFF_LOCK_INITIALIZATION_LOCK = threading.Lock() - - -@dataclass(frozen=True) -class WorkerIdentity: - username: str - payout_address: str - worker_name: str | None - script_pubkey_hex: str - p2mr_program_hex: str - - -@dataclass(eq=False) -class ClientState: - sock: socket.socket - address: tuple[str, int] - connection_id: int - extranonce1_hex: str - subscribed: bool = False - authorized: bool = False - authorization_generation: int = 0 - difficulty_generation: int = 0 - authorized_monotonic: float | None = None - username: str = "" - worker: WorkerIdentity | None = None - version_mask: int = 0 - active_job: object | None = None - # active_job is registered before the potentially blocking socket write. - # These delivery-proof fields advance only after send_job_update succeeds. - _progress_delivered_context: object | None = None - _progress_delivered_template_fingerprint: str | None = None - _progress_delivered_template_generation: int = 0 - _progress_delivered_payout_generation: int = -1 - _progress_delivered_monotonic: float | None = None - # Admission ordering only. Delivery proof remains in the progress fields - # above and advances exclusively after the socket write succeeds. - _tip_refresh_admitted_epoch_sequence: int = 0 - listener_name: str = "default" - # Pristine difficulty policy of the accepting listener; never mutated. - listener_vardiff_config: vardiff.VardiffConfig | None = None - # Floor below which stamped jobs never advertise, copied from the - # accepting listener profile. Zero (default listener) keeps the network - # cap authoritative. - minimum_advertised_difficulty: Decimal = Decimal("0") - # Per-client specialization of the listener policy (password d=/md= or - # mining.suggest_difficulty); recomputed from the pristine base on every - # request so repeat applications cannot compound. - vardiff_config: vardiff.VardiffConfig | None = None - requested_difficulty: Decimal | None = None - requested_min_difficulty: Decimal | None = None - suggested_difficulty: Decimal | None = None - share_difficulty: Decimal = Decimal("1") - pending_share_difficulty: Decimal | None = None - vardiff_window_started_monotonic: float = field(default_factory=time.monotonic) - vardiff_window_accepted: int = 0 - vardiff_window_submitted: int = 0 - vardiff_window_work: Decimal = Decimal("0") - vardiff_difficulty_estimate: Decimal | None = None - # Per-connection evidence that this session's difficulty is real: set on - # the first accepted share of the connection and never reset within it. - # Reconnect retention reads it to decide whether re-recording an - # unchanged difficulty may refresh its TTL; a session that resumed a - # retained value and submitted nothing must let that value keep ageing. - vardiff_accepted_any: bool = False - # Most recent accepted-share evidence. Durable reconnect retention uses - # the exact stamped difficulty plus its wall-clock evidence time so a - # later explicit difficulty request cannot accidentally be persisted as - # safe merely because this connection accepted an older, easier share. - vardiff_last_accepted_difficulty: Decimal | None = None - vardiff_last_accepted_wall_ms: int | None = None - # Fast-arrival initial convergence. True until this connection's first - # share-driven retarget actually commits; while set, an accepted share - # may trigger a retarget before the configured interval and that - # retarget may take the larger initial step bound. Cleared exactly once, - # at the paired-send commit point, so a build/send failure cannot spend - # the relaxation without the miner ever seeing the new difficulty, and - # so every later retarget is back on the ordinary configured bound. - vardiff_initial_convergence_pending: bool = True - # The statistical gate is evaluated once at the first sample that meets - # both its share-count and elapsed-time floors. A failed delivery may - # re-arm it, but an ordinary on-cadence window is never tested after every - # subsequent share (which would compound the false-positive probability). - vardiff_initial_convergence_evaluated: bool = False - # Connection-scoped arrival evidence for the bounded high-difficulty - # arrival histograms: when this session's vardiff accounting began, how - # many accepted shares it has produced since, and whether its one - # crossing of the high-diff threshold has already been observed. - vardiff_session_started_monotonic: float = field(default_factory=time.monotonic) - vardiff_session_accepted: int = 0 - vardiff_high_diff_arrival_recorded: bool = False - # Owns all per-client difficulty policy, estimates, and Vardiff windows. - # Share handlers release it before acquiring job_update_lock or the - # coordinator lock; job delivery may acquire it while job_update_lock is - # held. Keeping this lock per connection prevents one miner's accounting - # or slow socket from convoying tip publication for the whole coordinator. - vardiff_lock: threading.RLock = field(default_factory=threading.RLock) - active_job_ids: set[str] = field(default_factory=set) - post_accept_refresh_block: tuple[int, str] | None = None - # (job previousblockhash, monotonic) of the FIRST job this connection was - # sent for that tip. Anchors the per-connection stale-grace window: a - # prior-tip share is in flight until shortly after this connection - # received replacement work, however long the refresh pass took to reach - # it. See stale_grace_deadline_open. - tip_work_delivered: tuple[str, float] | None = None - # Protected by the coordinator lock. Disconnect retirement sets this before - # waiting for any per-client job update so queued work can reject the client. - closing: bool = False - # Stamped by the handler thread when a request line arrives, before JSON - # parsing. Only that thread reads it; mining.submit uses it to observe - # read-to-ack latency (the share-ingest saturation instrument). - request_received_monotonic: float | None = None - # Serializes every job build/register/send transition for this connection. - # The coordinator lock may be acquired while this lock is held, never in - # the reverse order. RLock permits authorize/retarget helpers to call the - # common maybe_send_job path while retaining the same serialization scope. - job_update_lock: threading.RLock = field(default_factory=threading.RLock) - send_lock: threading.Lock = field(default_factory=threading.Lock) - handler_thread_registered: bool = False - - def send(self, payload: dict[str, object]) -> None: - data = json.dumps(payload).encode() + b"\n" - with self.send_lock: - self.sock.sendall(data) - - def send_batch( - self, - payloads: list[dict[str, object]], - *, - preserialized: bytes | None = None, - ) -> None: - # Tests and embedders may replace ``send`` with an in-memory recorder; - # retain that seam while production sockets write the whole difficulty - # + notify pair under one send lock with no response interleaving. - # ``preserialized`` must be the exact serialization of ``payloads``; - # callers with a shared precomposed fragment pass it so a fleet-wide - # wave does not re-serialize identical coinbase parts per client. - if "send" in self.__dict__: - for payload in payloads: - self.send(payload) - return - data = ( - preserialized - if preserialized is not None - else b"".join( - json.dumps(payload).encode() + b"\n" for payload in payloads - ) - ) - with self.send_lock: - self.sock.sendall(data) - - def close(self) -> None: - try: - self.sock.shutdown(socket.SHUT_RDWR) - except OSError: - pass - try: - self.sock.close() - except OSError: - pass - - -def client_vardiff_lock(client: ClientState) -> threading.RLock: - """Return the per-client vardiff lock, including lightweight embedders. - - Production ClientState instances always have this field. The lazy - fallback preserves focused tests and embedders that construct a - ClientState with __new__ and populate only the attributes they use. - """ - lock = getattr(client, "vardiff_lock", None) - if lock is not None: - return lock - with _VARDIFF_LOCK_INITIALIZATION_LOCK: - lock = getattr(client, "vardiff_lock", None) - if lock is None: - lock = threading.RLock() - client.vardiff_lock = lock - return lock - - -class StratumError(RuntimeError): - def __init__( - self, - code: int, - message: str, - *, - reason: str | None = None, - disconnect: bool = False, - ): - super().__init__(message) - self.code = code - self.message = message - self.reason = reason - self.disconnect = disconnect - - -def parse_stratum_password_options(password: str) -> tuple[Decimal | None, Decimal | None]: - """Extract the pool-side d=N / md=N difficulty convention from a password. - - Unknown tokens and malformed values are ignored: miners routinely send - junk passwords ("x") and rejecting them would break every such rig. - Returns (requested_difficulty, requested_min_difficulty). - """ - - requested: Decimal | None = None - requested_min: Decimal | None = None - for token in password.split(","): - key, separator, raw_value = token.strip().partition("=") - if not separator: - continue - key = key.strip().lower() - if key not in {"d", "md"}: - continue - try: - value = Decimal(raw_value.strip()) - except Exception: - continue - if not value.is_finite() or value <= 0: - continue - if key == "d": - requested = value - else: - requested_min = value - return requested, requested_min - - -def parse_worker_username(username: str) -> tuple[str, str | None]: - payout_address, worker_name = split_worker_username(username) - if not payout_address: - raise StratumError(20, "username base is empty") - return payout_address, worker_name - - -def split_worker_username(username: str) -> tuple[str, str | None]: - payout_address, separator, worker_name = username.partition(".") - return payout_address, worker_name if separator else None - - -def result_payload(request_id: object, result: object) -> dict[str, object]: - return {"id": request_id, "result": result, "error": None} - - -def error_payload( - request_id: object, - code: int, - message: str, - *, - reason: str | None = None, -) -> dict[str, object]: - data = {"reason_id": reason} if reason is not None else None - return {"id": request_id, "result": None, "error": [code, message, data]} - - -def difficulty_payload(difficulty: Decimal) -> dict[str, object]: - return { - "id": None, - "method": "mining.set_difficulty", - "params": [float(difficulty)], - } - - -def job_payload(job: direct_stratum.DirectQbitStratumJob) -> dict[str, object]: - return { - "id": None, - "method": "mining.notify", - "params": [ - job.job_id, - job.prevhash, - job.coinb1, - job.coinb2, - list(job.merkle_branch), - job.version, - job.nbits, - job.ntime, - job.clean_jobs, - ], - } - - -def client_can_receive_jobs(client: ClientState) -> bool: - return ( - not getattr(client, "closing", False) - and client.subscribed - and client.authorized - and client.worker is not None - ) - - -def stratum_accept_heartbeat_names( - profiles: list[StratumListenerProfile] | tuple[StratumListenerProfile, ...] | None, -) -> tuple[str, ...]: - if not profiles: - return ("stratum_accept",) - return tuple(profile.heartbeat_name for profile in profiles) - - -def apply_stratum_send_timeout(sock: socket.socket, timeout_seconds: float) -> None: - """Bound blocking sends to miners without touching receive semantics. - - Job refreshes use a bounded executor, but an unresponsive peer whose - TCP buffer is full must still release its worker eventually. - SO_SNDTIMEO turns that into an OSError, which the refresh path treats - as a dead client without failing delivery to other miners. - A plain socket timeout is not usable here: it would also apply to - recv, disconnecting idle-but-healthy miners. - """ - if timeout_seconds <= 0: - return - seconds = int(timeout_seconds) - microseconds = int((timeout_seconds - seconds) * 1_000_000) - try: - sock.setsockopt( - socket.SOL_SOCKET, - socket.SO_SNDTIMEO, - struct.pack("ll", seconds, microseconds), - ) - except (AttributeError, OSError, struct.error): - # Platform without SO_SNDTIMEO support: keep legacy blocking sends. - return - - -@dataclass(frozen=True) -class DeliveredSessionContext: - context: object - delivered_monotonic: float - - -@dataclass(frozen=True) -class EligibleSession: - connection_id: int - delivered: DeliveredSessionContext | None - - -class SessionRegistry: - """Atomic owner of live membership and connection-scoped session facts. - - Callers hold the shared coordinator control-plane lock around the - ``*_locked`` helpers exactly as the pre-extraction coordinator did; the - registry itself adds no second lock. Compatibility containers seeded by - ``__new__`` embedders are adopted, never copied, so test-held references - remain live. - """ - - def __init__( - self, - *, - clients: object | None = None, - connection_generation: int = 0, - rejection_counts: dict[str, int] | None = None, - ) -> None: - self.clients = clients if clients is not None else set() - self.connection_generation = max( - int(connection_generation), - max( - (int(client.connection_id) for client in self.clients), # type: ignore[union-attr] - default=0, - ), - ) - self.rejection_counts = ( - rejection_counts - if rejection_counts is not None - else {"global": 0, "username": 0} - ) - self.peak_active_connections = len(self.clients) # type: ignore[arg-type] - self.handler_thread_count = sum( - int(getattr(client, "handler_thread_registered", False)) - for client in self.clients - ) - self._delivered_by_connection: dict[int, DeliveredSessionContext] = {} - - def adopt_clients(self, clients: object) -> None: - """Adopt a compatibility replacement without changing its order/type.""" - self.clients = clients - live_ids = { - int(getattr(client, "connection_id", 0)) for client in clients # type: ignore[union-attr] - } - self._delivered_by_connection = { - connection_id: delivered - for connection_id, delivered in self._delivered_by_connection.items() - if connection_id in live_ids - } - self.peak_active_connections = max( - self.peak_active_connections, - len(clients), # type: ignore[arg-type] - ) - self.connection_generation = max( - self.connection_generation, - max( - ( - int(getattr(client, "connection_id", 0)) - for client in clients # type: ignore[union-attr] - ), - default=0, - ), - ) - self.handler_thread_count = sum( - int(getattr(client, "handler_thread_registered", False)) - for client in clients # type: ignore[union-attr] - ) - - def note_rejection_locked(self, scope: str) -> int: - count = int(self.rejection_counts.get(scope, 0)) + 1 - self.rejection_counts[scope] = count - return count - - def begin_retirement_locked(self, client: ClientState) -> bool: - """Claim retirement ownership; only the first claimer cleans up.""" - if getattr(client, "closing", False) and client not in self.clients: - return False - client.closing = True - self.clients.discard(client) - self._delivered_by_connection.pop(client.connection_id, None) - return True - - def record_delivery_locked( - self, - client: ClientState, - context: object, - delivered_monotonic: float, - ) -> None: - """Commit exact delivery proof while the shared lock is held.""" - self._delivered_by_connection[client.connection_id] = ( - DeliveredSessionContext(context, delivered_monotonic) - ) - - def eligible_snapshot(self) -> Mapping[int, EligibleSession]: - """Return an immutable exact client_can_receive_jobs population. - - Caller holds the shared lock; the compatibility mirrors on the - client remain the staged-split fallback for embedder-seeded state. - """ - captured: dict[int, EligibleSession] = {} - for client in self.clients: - if not client_can_receive_jobs(client): - continue - delivered = self._delivered_by_connection.get(client.connection_id) - if ( - delivered is None - and getattr(client, "_progress_delivered_context", None) is not None - ): - delivered = DeliveredSessionContext( - client._progress_delivered_context, - float(client._progress_delivered_monotonic or 0.0), - ) - captured[client.connection_id] = EligibleSession( - connection_id=client.connection_id, - delivered=delivered, - ) - return MappingProxyType(captured) - - -@dataclass -class _P2mrAddressValidationFlight: - event: threading.Event = field(default_factory=threading.Event) - result: tuple[str, str] | None = None - error: BaseException | None = None - waiters: int = 0 - - -class P2mrAddressValidator: - """Bounded LRU and singleflight wrapper around validateaddress RPC.""" - - def __init__( - self, - *, - rpc_call: Callable[[str, list[object]], object], - max_entries: Callable[[], int], - ttl_seconds: Callable[[], float], - cache_lock: threading.Lock | None = None, - cache: OrderedDict[str, tuple[float, tuple[str, str]]] | None = None, - inflight: dict[str, _P2mrAddressValidationFlight] | None = None, - ) -> None: - self.rpc_call = rpc_call - self.max_entries = max_entries - self.ttl_seconds = ttl_seconds - self.cache_lock = cache_lock if cache_lock is not None else threading.Lock() - self.cache = cache if cache is not None else OrderedDict() - self.inflight = inflight if inflight is not None else {} - - def validate(self, address: str, *, label: str) -> tuple[str, str]: - with self.cache_lock: - cached = self.cache.get(address) - if cached is not None: - expires_monotonic, cached_result = cached - if expires_monotonic > time.monotonic(): - self.cache.move_to_end(address) - return cached_result - self.cache.pop(address, None) - pending = self.inflight.get(address) - is_leader = pending is None - if pending is None: - pending = _P2mrAddressValidationFlight() - self.inflight[address] = pending - else: - pending.waiters += 1 - - if not is_leader: - pending.event.wait() - if pending.result is not None: - return pending.result - if pending.error is not None: - self._raise_shared_error(pending.error) - raise RuntimeError("payout address validation completed without a result") - - try: - validation = self.rpc_call("validateaddress", [address]) - if not isinstance(validation, dict) or not validation.get("isvalid"): - raise StratumError(20, f"{label} is not a valid qbit address: {address}") - script = str(validation.get("scriptPubKey") or "") - if not script.startswith("5220") or len(script) != 68: - raise StratumError(20, f"{label} does not resolve to a P2MR script: {address}") - result = (script, script[4:]) - with self.cache_lock: - max_entries = int(self.max_entries()) - ttl_seconds = float(self.ttl_seconds()) - if max_entries > 0 and ttl_seconds > 0: - self.cache[address] = ( - time.monotonic() + ttl_seconds, - result, - ) - self.cache.move_to_end(address) - while len(self.cache) > max_entries: - self.cache.popitem(last=False) - pending.result = result - return result - except BaseException as exc: - with self.cache_lock: - pending.error = exc - raise - finally: - with self.cache_lock: - if self.inflight.get(address) is pending: - self.inflight.pop(address, None) - pending.event.set() - # Release the leader frame's reference after the waiters are woken - # (#251). On failure the stored error's traceback already holds - # this frame; keeping the flight named here would close - # flight -> error -> traceback -> frame -> flight and pin the - # failed RPC frame and its locals until cyclic GC. Waiters keep - # the flight alive through their own frames until each has read - # the recorded result or error. - del pending - - @staticmethod - def _raise_shared_error(error: BaseException) -> None: - if isinstance(error, StratumError): - raise StratumError( - error.code, - error.message, - reason=error.reason, - disconnect=error.disconnect, - ) from error - raise RuntimeError(str(error)) from error - - -class SessionRuntime(Protocol): - """Typed port over the coordinator, resolved at call time. - - Every member is looked up on the live coordinator object when used, so - instance monkeypatches (``server.send_result = ...``, - ``server.validate_p2mr_address = ...`` and friends) and coordinator-owned - live configuration attributes keep working exactly as before the - extraction. Owner facades route back into this service; that round trip - is deliberate: it preserves the current white-box patch surface until - later stack layers repoint those tests. The legacy S1 field names - (``clients``, ``connection_counter``, the connection counters, and the - payout-address cache trio) also resolve here -- coordinator class - descriptors route them to this service's single mutable copy. - """ - - # Cross-domain objects and live configuration attributes. - _pool_ready_latched: Any - clients: Any - extranonce2_size: Any - handler_thread_count: Any - latest_detected_tip: Any - listener_profiles: Any - lock: Any - pending_initial_jobs: Any - rpc: Any - stop_event: Any - stratum_max_pending_initial_jobs: Any - version_mask: Any - - def _cancel_pending_initial_job_locked(self, *args: Any, **kwargs: Any) -> Any: ... - - def _client_has_current_tip_job_locked(self, *args: Any, **kwargs: Any) -> Any: ... - - def _client_vardiff_lock(self, *args: Any, **kwargs: Any) -> Any: ... - - def _ensure_initial_job_state(self, *args: Any, **kwargs: Any) -> Any: ... - - def _ensure_job_cache_state(self, *args: Any, **kwargs: Any) -> Any: ... - - def _ensure_job_delivery_service(self, *args: Any, **kwargs: Any) -> Any: ... - - def _ensure_progress_health_service(self, *args: Any, **kwargs: Any) -> Any: ... - - def _ensure_shutdown_controller(self, *args: Any, **kwargs: Any) -> Any: ... - - def _handle_request(self, *args: Any, **kwargs: Any) -> Any: ... - - def _note_collection_identity_available(self, *args: Any, **kwargs: Any) -> Any: ... - - def _observe_share_ack_seconds(self, *args: Any, **kwargs: Any) -> Any: ... - - def _progress_note_refresh_activity(self, *args: Any, **kwargs: Any) -> Any: ... - - def _progress_reconcile_pending(self, *args: Any, **kwargs: Any) -> Any: ... - - def _record_heartbeat(self, *args: Any, **kwargs: Any) -> Any: ... - - def _record_stratum_resource_exhaustion(self, *args: Any, **kwargs: Any) -> Any: ... - - def _record_tip_refresh_epoch_coverage(self, *args: Any, **kwargs: Any) -> Any: ... - - def _retain_current_collection_refresh_if_unrepresented( - self, *args: Any, **kwargs: Any - ) -> Any: ... - - def _tip_refresh_epoch_coverage_reached_locked( - self, *args: Any, **kwargs: Any - ) -> Any: ... - - def _wait_after_stratum_resource_failure(self, *args: Any, **kwargs: Any) -> Any: ... - - def apply_client_difficulty_requests(self, *args: Any, **kwargs: Any) -> Any: ... - - def apply_stratum_send_timeout(self, *args: Any, **kwargs: Any) -> Any: ... - - def advertise_client_difficulty(self, *args: Any, **kwargs: Any) -> Any: ... - - def client_startup_difficulty(self, *args: Any, **kwargs: Any) -> Any: ... - - def disconnect_client(self, *args: Any, **kwargs: Any) -> Any: ... - - def handle_client(self, *args: Any, **kwargs: Any) -> Any: ... - - def handle_configure(self, *args: Any, **kwargs: Any) -> Any: ... - - def handle_submit(self, *args: Any, **kwargs: Any) -> Any: ... - - def handle_suggest_difficulty(self, *args: Any, **kwargs: Any) -> Any: ... - - def note_vardiff_resume_overridden(self, *args: Any, **kwargs: Any) -> Any: ... - - def record_session_difficulty(self, *args: Any, **kwargs: Any) -> Any: ... - - def refresh_jobs_after_pending_accepted_block( - self, *args: Any, **kwargs: Any - ) -> Any: ... - - def request_initial_job_delivery(self, *args: Any, **kwargs: Any) -> Any: ... - - def reserve_client_username(self, *args: Any, **kwargs: Any) -> Any: ... - - def resume_client_difficulty(self, *args: Any, **kwargs: Any) -> Any: ... - - def send_error(self, *args: Any, **kwargs: Any) -> Any: ... - - def send_result(self, *args: Any, **kwargs: Any) -> Any: ... - - def validate_p2mr_address(self, *args: Any, **kwargs: Any) -> Any: ... - - -class StratumSessionService: - """Sole owner of S1 session admission, dispatch, and registry proof.""" - - def __init__( - self, - runtime: SessionRuntime, - *, - shutdown_error: type[BaseException], - pool_closed_reason: str, - ) -> None: - self._runtime = runtime - # The shutdown exception type remains coordinator-owned until the - # shutdown controller is extracted; inject it so this leaf module - # never imports prism_coordinator. - self._shutdown_error = shutdown_error - self.pool_closed_reason = pool_closed_reason - self.registry = SessionRegistry() - self.accept_resource_exhaustion_count = 0 - self.connection_setup_failure_count = 0 - self.address_validator = P2mrAddressValidator( - rpc_call=lambda method, params: runtime.rpc.call(method, params), - max_entries=lambda: int( - getattr( - runtime, - "payout_address_cache_max_entries", - DEFAULT_PRISM_PAYOUT_ADDRESS_CACHE_MAX_ENTRIES, - ) - ), - ttl_seconds=lambda: float( - getattr( - runtime, - "payout_address_cache_ttl_seconds", - DEFAULT_PRISM_PAYOUT_ADDRESS_CACHE_TTL_SECONDS, - ) - ), - ) - - # -- legacy coordinator field names routed by class descriptors -------- - - @property - def clients(self) -> object: - return self.registry.clients - - @clients.setter - def clients(self, value: object) -> None: - self.registry.adopt_clients(value) - - @property - def connection_counter(self) -> int: - return self.registry.connection_generation - - @connection_counter.setter - def connection_counter(self, value: int) -> None: - self.registry.connection_generation = int(value) - - @property - def connection_limit_rejection_counts(self) -> dict[str, int]: - return self.registry.rejection_counts - - @connection_limit_rejection_counts.setter - def connection_limit_rejection_counts(self, value: dict[str, int]) -> None: - self.registry.rejection_counts = value - - @property - def peak_active_connection_count(self) -> int: - return self.registry.peak_active_connections - - @peak_active_connection_count.setter - def peak_active_connection_count(self, value: int) -> None: - self.registry.peak_active_connections = int(value) - - @property - def handler_thread_count(self) -> int: - return self.registry.handler_thread_count - - @handler_thread_count.setter - def handler_thread_count(self, value: int) -> None: - self.registry.handler_thread_count = int(value) - - @property - def _p2mr_address_cache_lock(self) -> threading.Lock: - return self.address_validator.cache_lock - - @_p2mr_address_cache_lock.setter - def _p2mr_address_cache_lock(self, value: threading.Lock) -> None: - self.address_validator.cache_lock = value - - @property - def _p2mr_address_cache(self) -> OrderedDict[str, tuple[float, tuple[str, str]]]: - return self.address_validator.cache - - @_p2mr_address_cache.setter - def _p2mr_address_cache( - self, value: OrderedDict[str, tuple[float, tuple[str, str]]] - ) -> None: - self.address_validator.cache = value - - @property - def _p2mr_address_validation_inflight( - self, - ) -> dict[str, _P2mrAddressValidationFlight]: - return self.address_validator.inflight - - @_p2mr_address_validation_inflight.setter - def _p2mr_address_validation_inflight( - self, value: dict[str, _P2mrAddressValidationFlight] - ) -> None: - self.address_validator.inflight = value - - # -- listener factory -------------------------------------------------- - - @staticmethod - def open_stratum_listeners( - listener_stack: ExitStack, - profiles: list[StratumListenerProfile] | tuple[StratumListenerProfile, ...], - *, - backlog: int, - retry_seconds: float, - stop_event: threading.Event | None, - ) -> list[tuple[socket.socket, StratumListenerProfile]] | None: - """Bind and listen on every stratum listener profile. - - Called before the slow parts of startup (qbit readiness, policy - validation, block-work recovery) so miners reconnecting through a - restart park in the kernel accept backlog instead of getting - connection refused, which sends firmware into reconnect backoff or - failover and costs hashrate. bind() retries EADDRINUSE for a bounded - window because a predecessor process may still hold the port while - draining its shutdown. Returns None when shutdown is requested during - the retry, so startup can abort gracefully. - """ - listeners: list[tuple[socket.socket, StratumListenerProfile]] = [] - for profile in profiles: - server = listener_stack.enter_context( - socket.socket(socket.AF_INET, socket.SOCK_STREAM) - ) - server.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) - bind_deadline = time.monotonic() + retry_seconds - warned = False - while True: - try: - server.bind((profile.bind, profile.port)) - break - except OSError as exc: - if exc.errno != errno.EADDRINUSE or time.monotonic() >= bind_deadline: - raise - if stop_event is not None and stop_event.is_set(): - # Shutting down mid-startup: stop contending for a - # port this process will never serve. - print( - f"prism coordinator: shutdown requested while waiting " - f"to bind {profile.bind}:{profile.port}; aborting startup", - flush=True, - ) - return None - if not warned: - print( - f"prism coordinator: {profile.name} listener port " - f"{profile.bind}:{profile.port} is busy; retrying bind " - f"for up to {retry_seconds:g}s", - flush=True, - ) - warned = True - time.sleep(0.1) - server.listen(backlog) - server.settimeout(1) - listeners.append((server, profile)) - return listeners - - # -- accept/admission -------------------------------------------------- - - def accept_loop( - self, server: socket.socket, profile: StratumListenerProfile - ) -> None: - runtime = self._runtime - registry = self.registry - while not runtime.stop_event.is_set(): - runtime._record_heartbeat(profile.heartbeat_name) - try: - sock, address = server.accept() - except socket.timeout: - continue - except OSError as exc: - # The listener socket is torn down by serve()'s ExitStack on - # shutdown while secondary accept threads may still be blocked - # in accept(). Descriptor exhaustion is recoverable: keep the - # accept loop alive and refresh its watchdog heartbeat while - # waiting for client/RPC descriptors to drain. - if runtime.stop_event.is_set(): - return - if exc.errno in {errno.EMFILE, errno.ENFILE}: - runtime._record_stratum_resource_exhaustion( - listener_name=profile.name, - location="accept", - error_number=exc.errno, - ) - runtime._wait_after_stratum_resource_failure(profile.heartbeat_name) - continue - raise - - if ( - runtime.stop_event.is_set() - or runtime._ensure_shutdown_controller().phase != "running" - ): - try: - sock.close() - except OSError: - pass - return - - with runtime.lock: - if ( - runtime.stop_event.is_set() - or runtime._ensure_shutdown_controller().phase != "running" - ): - try: - sock.close() - except OSError: - pass - return - max_connections = int( - getattr( - runtime, - "stratum_max_connections", - DEFAULT_PRISM_STRATUM_MAX_CONNECTIONS, - ) - ) - if max_connections > 0 and len(registry.clients) >= max_connections: - rejection_count = registry.note_rejection_locked("global") - client = None - else: - registry.connection_generation += 1 - connection_id = registry.connection_generation - client = ClientState( - sock=sock, - address=address, - connection_id=connection_id, - extranonce1_hex=f"{connection_id & 0xFFFFFFFF:08x}", - listener_name=profile.name, - listener_vardiff_config=profile.vardiff_config, - minimum_advertised_difficulty=profile.minimum_advertised_difficulty, - share_difficulty=runtime.client_startup_difficulty(profile), - ) - registry.clients.add(client) - runtime._ensure_initial_job_state() - registry.peak_active_connections = max( - registry.peak_active_connections, - len(registry.clients), - ) - if client is None: - try: - sock.close() - except OSError: - pass - if rejection_count == 1 or rejection_count % 100 == 0: - print( - "prism coordinator: rejected stratum connection at global limit " - f"limit={max_connections} count={rejection_count}", - flush=True, - ) - continue - try: - sock.settimeout(None) - runtime.apply_stratum_send_timeout(sock) - thread = threading.Thread( - target=runtime.handle_client, args=(client,), daemon=True - ) - with runtime.lock: - client.handler_thread_registered = True - registry.handler_thread_count += 1 - thread.start() - except (OSError, RuntimeError) as exc: - # Admission is atomic with the global count. Undo it if socket - # setup or thread creation fails before a handler owns cleanup, - # then keep this listener alive for the next connection. - try: - with runtime.lock: - if client.handler_thread_registered: - client.handler_thread_registered = False - registry.handler_thread_count = max( - 0, - registry.handler_thread_count - 1, - ) - runtime.disconnect_client(client) - except Exception: - print( - "prism coordinator: failed to fully close rejected stratum client " - f"address={address}", - flush=True, - ) - traceback.print_exc() - with runtime.lock: - self.connection_setup_failure_count = int( - getattr(self, "connection_setup_failure_count", 0) - ) + 1 - setup_failure_count = self.connection_setup_failure_count - if isinstance(exc, OSError) and exc.errno in {errno.EMFILE, errno.ENFILE}: - runtime._record_stratum_resource_exhaustion( - listener_name=profile.name, - location="connection-setup", - error_number=exc.errno, - ) - if setup_failure_count == 1 or setup_failure_count % 100 == 0: - print( - "prism coordinator: stratum connection setup failed; backing off " - f"listener={profile.name} address={address} " - f"error={exc!r} count={setup_failure_count}", - flush=True, - ) - runtime._wait_after_stratum_resource_failure(profile.heartbeat_name) - continue - - def record_stratum_resource_exhaustion( - self, - *, - listener_name: str, - location: str, - error_number: int | None, - ) -> int: - runtime = self._runtime - with runtime.lock: - self.accept_resource_exhaustion_count = int( - getattr(self, "accept_resource_exhaustion_count", 0) - ) + 1 - exhaustion_count = self.accept_resource_exhaustion_count - if exhaustion_count == 1 or exhaustion_count % 100 == 0: - print( - "prism coordinator: stratum resource exhaustion " - f"listener={listener_name} location={location} errno={error_number} " - f"count={exhaustion_count}", - flush=True, - ) - return exhaustion_count - - def wait_after_stratum_resource_failure(self, heartbeat_name: str) -> None: - runtime = self._runtime - backoff_seconds = getattr( - runtime, - "stratum_accept_resource_exhaustion_backoff_seconds", - DEFAULT_PRISM_STRATUM_ACCEPT_RESOURCE_EXHAUSTION_BACKOFF_SECONDS, - ) - remaining_seconds = max(0.0, float(backoff_seconds)) - watchdog_timeout_seconds = max( - 0.001, - float(getattr(runtime, "watchdog_timeout_seconds", 120.0)), - ) - heartbeat_interval_seconds = max( - 0.001, - min(1.0, watchdog_timeout_seconds / 2.0), - ) - deadline = time.monotonic() + remaining_seconds - while not runtime.stop_event.is_set(): - runtime._record_heartbeat(heartbeat_name) - remaining_seconds = deadline - time.monotonic() - if remaining_seconds <= 0: - return - if runtime.stop_event.wait( - min(remaining_seconds, heartbeat_interval_seconds) - ): - return - - def reserve_client_username( - self, client: ClientState, worker: WorkerIdentity - ) -> bool: - """Atomically reserve an exact Stratum username for one connection.""" - runtime = self._runtime - registry = self.registry - with runtime.lock: - limit = int( - getattr( - runtime, - "stratum_max_connections_per_username", - DEFAULT_PRISM_STRATUM_MAX_CONNECTIONS_PER_USERNAME, - ) - ) - active_for_username = sum( - 1 - for other in registry.clients - if ( - other is not client - and other.worker is not None - and other.username == worker.username - ) - ) - if limit > 0 and active_for_username >= limit: - rejection_count = registry.note_rejection_locked("username") - if rejection_count == 1 or rejection_count % 100 == 0: - print( - "prism coordinator: rejected stratum authorization at username limit " - f"username={worker.username!r} limit={limit} count={rejection_count}", - flush=True, - ) - return False - # Commit the replacement only after capacity validation. A failed - # reauthorization therefore leaves the prior live identity intact. - client.worker = worker - client.username = worker.username - return True - - # -- session lifecycle ------------------------------------------------- - - def handle_client(self, client: ClientState) -> None: - runtime = self._runtime - registry = self.registry - reader = None - try: - reader = client.sock.makefile("r", encoding="utf-8", newline="\n") - for line in reader: - if runtime.stop_event.is_set(): - break - line = line.strip() - if not line: - continue - client.request_received_monotonic = time.monotonic() - request_id: object = None - request: object = None - try: - request = json.loads(line) - if not isinstance(request, dict): - raise StratumError(20, "request must be an object") - request_id = request.get("id") - self.handle_request(client, request) - except json.JSONDecodeError as exc: - runtime.send_error(client, request_id, 20, f"invalid JSON: {exc.msg}") - except StratumError as exc: - runtime.send_error( - client, request_id, exc.code, exc.message, reason=exc.reason - ) - received_monotonic = client.request_received_monotonic - if ( - received_monotonic is not None - and isinstance(request, dict) - and request.get("method") == "mining.submit" - ): - # Symmetric with the accepted-path observation after - # send_result: both sides include the response write, - # so the accepted-vs-rejected comparison separates - # commit pressure from transport/thread saturation. - runtime._observe_share_ack_seconds( - "rejected", - time.monotonic() - received_monotonic, - ) - if exc.disconnect: - break - except Exception: - print( - f"prism coordinator: client thread failed address={client.address}", - flush=True, - ) - traceback.print_exc() - break - except (OSError, ValueError) as exc: - if isinstance(exc, OSError) and exc.errno in {errno.EMFILE, errno.ENFILE}: - runtime._record_stratum_resource_exhaustion( - listener_name=client.listener_name, - location="client-reader", - error_number=exc.errno, - ) - print( - "prism coordinator: stratum client socket failed " - f"address={client.address} error={exc!r}", - flush=True, - ) - finally: - try: - if reader is not None: - reader.close() - except (OSError, ValueError): - pass - finally: - runtime.disconnect_client(client) - with runtime.lock: - runtime._ensure_initial_job_state() - if getattr(client, "handler_thread_registered", False): - client.handler_thread_registered = False - registry.handler_thread_count = max( - 0, - registry.handler_thread_count - 1, - ) - - def disconnect_client(self, client: ClientState) -> None: - # Retire admission and fanout eligibility without waiting behind job - # delivery. Only the first caller owns socket close and final cleanup. - runtime = self._runtime - registry = self.registry - with runtime.lock: - # The timeout sweeper marks closing while leaving membership in - # place as its atomic handoff token. Whichever disconnect caller - # first removes that membership owns socket close and cleanup. - if not registry.begin_retirement_locked(client): - return - runtime._cancel_pending_initial_job_locked(client, count=True) - - # Retirement was just claimed, so this runs once per connection and - # outside the coordinator lock: retain the session's converged - # difficulty for a reconnect resume. Reached through the runtime seam - # so monkeypatched coordinators keep intercepting; a retention - # failure must never break disconnect cleanup. - record_session_difficulty = getattr( - runtime, "record_session_difficulty", None - ) - if record_session_difficulty is not None: - try: - record_session_difficulty(client) - except Exception: - print( - "prism coordinator: session difficulty retention failed " - f"address={client.address}", - flush=True, - ) - traceback.print_exc() - - # Do not take send_lock here: shutdown must interrupt an in-flight - # sendall as well as the handler's blocking reader. - try: - client.close() - finally: - # Every mixed lock path uses job_update_lock -> coordinator lock. - # Retirement above holds neither while this potentially waits. - with client.job_update_lock: - runtime._ensure_job_delivery_service().cleanup_disconnected_client( - client - ) - runtime._retain_current_collection_refresh_if_unrepresented() - - # -- protocol dispatch ------------------------------------------------- - - def handle_request(self, client: ClientState, request: dict[str, object]) -> None: - """Dispatch one request, translating shutdown races to Stratum errors.""" - try: - self._runtime._handle_request(client, request) - except self._shutdown_error as exc: - # A request can pass the initial shutdown check immediately before - # writer admission closes. Preserve the normal protocol response - # instead of surfacing a generic client-thread failure. - raise StratumError( - 20, - "coordinator is shutting down", - reason=self.pool_closed_reason, - disconnect=True, - ) from exc - - def _handle_request(self, client: ClientState, request: dict[str, object]) -> None: - runtime = self._runtime - if ( - runtime.stop_event.is_set() - or runtime._ensure_shutdown_controller().phase != "running" - ): - raise StratumError( - 20, - "coordinator is shutting down", - reason=self.pool_closed_reason, - disconnect=True, - ) - method = request.get("method") - params = request.get("params", []) - request_id = request.get("id") - if not isinstance(method, str): - raise StratumError(20, "missing method") - if not isinstance(params, list): - raise StratumError(20, "params must be an array") - - if method == "mining.configure": - runtime.handle_configure(client, request_id, params) - return - if method == "mining.subscribe": - with client.job_update_lock: - client.subscribed = True - runtime.send_result( - client, - request_id, - [[], client.extranonce1_hex, runtime.extranonce2_size], - ) - runtime._note_collection_identity_available(client) - needs_initial_job = client.authorized - if needs_initial_job: - runtime.request_initial_job_delivery(client) - return - if method == "mining.authorize": - username = str(params[0]) if params else "" - password = str(params[1]) if len(params) > 1 and params[1] is not None else "" - # Address validation may use RPC; it is unrelated to client job - # state and therefore stays outside the job-update lock. - worker = runtime.resolve_worker(username) - with client.job_update_lock: - was_authorized = client.authorized - if was_authorized: - with runtime.lock: - runtime._ensure_initial_job_state() - if ( - client not in runtime.pending_initial_jobs - and len(runtime.pending_initial_jobs) - >= runtime.stratum_max_pending_initial_jobs - and runtime._client_has_current_tip_job_locked(client) - ): - # Reject before mutating identity or difficulty. A - # working session keeps its current authorization - # when there is no live first-job slot for the - # superseding generation. - raise StratumError( - 20, - "initial job delivery capacity unavailable", - disconnect=False, - ) - if not runtime.reserve_client_username(client, worker): - raise StratumError( - 20, - "too many connections for username", - # A new connection has no useful session to preserve. A - # live miner re-authorizing to a full username does: keep - # its prior worker/session active after returning the - # capacity error. - disconnect=not client.authorized, - ) - # The password is authoritative for password-derived options: a - # re-authorize without d=/md= clears any prior override (a stored - # suggest_difficulty still applies via the request resolution). - with runtime._client_vardiff_lock(client): - # Only the FIRST authorize of a connection may resume the - # worker's retained difficulty (a live session already - # holds a converged value), and it must land before the - # password options: an explicit d= computes its target - # from requested_difficulty and outranks the resume, - # while an md=-only password clamps the just-resumed - # value through the share_difficulty fallback. - resumed: Decimal | None = None - if not was_authorized: - resumed = runtime.resume_client_difficulty(client) - client.requested_difficulty, client.requested_min_difficulty = ( - parse_stratum_password_options(password) - ) - target = runtime.apply_client_difficulty_requests(client) - if target is not None: - current = client.pending_share_difficulty or client.share_difficulty - if resumed is not None and target != resumed: - # The resume applied a value and this authorize - # immediately replaced it, so it never stuck. - # resumed/clamped stay attempt counters; this - # subtracts the ones an explicit difficulty - # request (d=/md=, or a suggestion sent before - # authorize) overrode. - runtime.note_vardiff_resume_overridden() - if target != current: - if not was_authorized: - client.share_difficulty = target - client.pending_share_difficulty = None - else: - client.pending_share_difficulty = target - client.difficulty_generation = int( - getattr(client, "difficulty_generation", 0) - ) + 1 - client.authorization_generation = int( - getattr(client, "authorization_generation", 0) - ) + 1 - client.authorized = True - client.authorized_monotonic = time.monotonic() - runtime.send_result(client, request_id, True) - runtime._note_collection_identity_available(client) - # Exactly one coalesced delivery represents this authorization, - # including a password-derived difficulty change. - runtime.request_initial_job_delivery(client) - return - if method == "mining.extranonce.subscribe": - runtime.send_result(client, request_id, True) - return - if method == "mining.suggest_difficulty": - runtime.handle_suggest_difficulty(client, request_id, params) - return - if method == "mining.submit": - received_monotonic = getattr( - client, - "request_received_monotonic", - None, - ) - accepted_and_closed = runtime.handle_submit(client, params) - try: - runtime.send_result(client, request_id, True) - if received_monotonic is not None: - # Rejects are observed symmetrically in handle_client - # after their error response write. - runtime._observe_share_ack_seconds( - "accepted", - time.monotonic() - received_monotonic, - ) - finally: - runtime.refresh_jobs_after_pending_accepted_block(client) - if accepted_and_closed: - client.close() - return - raise StratumError(20, f"unsupported method {method}") - - def handle_suggest_difficulty( - self, client: ClientState, request_id: object, params: list[object] - ) -> None: - runtime = self._runtime - with client.job_update_lock: - suggested: Decimal | None = None - if params: - try: - suggested = Decimal(str(params[0])) - except Exception: - suggested = None - if suggested is not None and (not suggested.is_finite() or suggested <= 0): - suggested = None - if suggested is not None: - with runtime._client_vardiff_lock(client): - client.suggested_difficulty = suggested - target = runtime.apply_client_difficulty_requests(client) - if target is not None: - runtime.advertise_client_difficulty(client, target) - runtime.send_result(client, request_id, True) - - def handle_configure( - self, client: ClientState, request_id: object, params: list[object] - ) -> None: - runtime = self._runtime - extensions = params[0] if params else [] - extension_params = params[1] if len(params) > 1 and isinstance(params[1], dict) else {} - result: dict[str, object] = {} - if isinstance(extensions, list): - for extension in extensions: - if extension == "version-rolling": - miner_mask = 0xFFFFFFFF - if "version-rolling.mask" in extension_params: - miner_mask = stratum_codec.parse_mask_hex( - extension_params["version-rolling.mask"], - field_name="version-rolling.mask", - ) - client.version_mask = runtime.version_mask & miner_mask - result["version-rolling"] = client.version_mask != 0 - result["version-rolling.mask"] = stratum_codec.format_mask_hex(client.version_mask) - else: - result[str(extension)] = False - runtime.send_result(client, request_id, result) - - # -- worker resolution ------------------------------------------------- - - def resolve_worker(self, username: str) -> WorkerIdentity: - runtime = self._runtime - payout_address, worker_name = split_worker_username(username) - try: - if not payout_address: - raise StratumError(20, "username base is empty") - script, p2mr_program_hex = runtime.validate_p2mr_address( - payout_address, label="username base" - ) - except StratumError as username_error: - fallback_address = getattr( - runtime, - "username_fallback_address", - default_prism_username_fallback_address(), - ) - if fallback_address is None: - raise username_error - print( - f"prism coordinator: username {username!r} cannot be used as a payout " - f"({username_error.message}); using fallback payout {fallback_address}", - flush=True, - ) - payout_address = fallback_address - script, p2mr_program_hex = runtime.validate_p2mr_address( - fallback_address, - label="PRISM_USERNAME_FALLBACK_ADDRESS", - ) - return WorkerIdentity( - username=username, - payout_address=payout_address, - worker_name=worker_name, - script_pubkey_hex=script, - p2mr_program_hex=p2mr_program_hex, - ) - - # -- registry proof ---------------------------------------------------- - - def record_successful_delivery( - self, - client: ClientState, - context: object, - delivered_monotonic: float, - ) -> None: - """Commit exact registry delivery proof, then report to G1. - - Proof-before-health ordering is a lost-fix detector: within the one - shared critical section the S1 registry proof (and its staged-split - compatibility mirrors on the client object) commits first, and only - then does the progress-health owner observe the delivery. - """ - runtime = self._runtime - runtime._ensure_job_cache_state() - service = runtime._ensure_progress_health_service() - fingerprint = getattr(context, "template_fingerprint", None) - if fingerprint is None: - fingerprint = qbit_template_fingerprint(context.template) - payout_generation = int(getattr(context, "payout_state_generation", 0)) - delivered_tip = str(context.template.get("previousblockhash", "")) - recorded = False - coverage_reached: list[tuple[str, float]] = [] - with runtime.lock: - self.registry.record_delivery_locked( - client, context, delivered_monotonic - ) - client._progress_delivered_context = context - client._progress_delivered_template_fingerprint = fingerprint - client._progress_delivered_template_generation = int( - getattr(context, "template_generation", 0) - ) - client._progress_delivered_payout_generation = payout_generation - client._progress_delivered_monotonic = delivered_monotonic - ready_mode_required = bool( - getattr(runtime, "_pool_ready_latched", False) - ) - latest_detected = getattr(runtime, "latest_detected_tip", None) - recorded = service.record_delivery( - DeliveryProof( - connection_id=int(getattr(client, "connection_id", 0)), - delivered_work=WorkGeneration( - template_generation=int( - getattr(context, "template_generation", 0) - ), - template_fingerprint=fingerprint, - payout_generation=payout_generation, - ), - collection_only=bool( - getattr(context, "collection_only", False) - ), - delivered_monotonic=delivered_monotonic, - ), - ready_mode_required, - matches_latest_tip=bool( - latest_detected is None - or latest_detected[0] == delivered_tip - ), - ) - coverage_reached = ( - runtime._tip_refresh_epoch_coverage_reached_locked( - client, - context, - delivered_monotonic, - ) - ) - if recorded: - runtime._progress_note_refresh_activity(delivered_monotonic) - runtime._progress_reconcile_pending(now=delivered_monotonic) - runtime._record_tip_refresh_epoch_coverage(coverage_reached) - - -__all__ = [ - "ClientState", - "DeliveredSessionContext", - "EligibleSession", - "P2mrAddressValidator", - "SessionRegistry", - "SessionRuntime", - "StratumError", - "StratumSessionService", - "WorkerIdentity", - "_P2mrAddressValidationFlight", - "apply_stratum_send_timeout", - "client_can_receive_jobs", - "client_vardiff_lock", - "difficulty_payload", - "error_payload", - "job_payload", - "parse_stratum_password_options", - "parse_worker_username", - "result_payload", - "split_worker_username", - "stratum_accept_heartbeat_names", -] diff --git a/lab/prism/template_artifacts.py b/lab/prism/template_artifacts.py deleted file mode 100644 index 26c5dc04d..000000000 --- a/lab/prism/template_artifacts.py +++ /dev/null @@ -1,605 +0,0 @@ -#!/usr/bin/env python3 -"""Immutable qbit template observations and their derived artifacts. - -The repository owns the coordinator's template-artifact cache and observation -generations. It does not import ``prism_coordinator``: every cross-domain -fact (tip publication, refresh scheduling, RPC access, cache-event metrics) -is reached through the :class:`TemplateArtifactRuntime` port, resolved at -call time so the historical coordinator monkeypatch seams keep working. -""" - -from __future__ import annotations - -import copy -from collections import OrderedDict -from dataclasses import dataclass, field -import hashlib -import json -import threading -import time -from typing import Any, Callable, Mapping, Protocol, Sequence - -from lab.prism import direct_stratum -from lab.prism.coordinator_config import DEFAULT_PRISM_BLOCKPOLL_SECONDS -# The refresh/publication exception family is owned by the payout-state -# module (its final owner); re-imported here so every existing import path -# keeps working. -from lab.prism.payout_state import ( - PayoutStatePublicationBlocked, # noqa: F401 - compatibility re-export - TemplateRefreshBlocked, # noqa: F401 - compatibility re-export - TemplateRefreshSuperseded, -) - - -PRISM_TEMPLATE_FINGERPRINT_VOLATILE_KEYS = frozenset( - { - # qbit can legitimately advance these without making already issued - # jobs stale. Rebuilding every miner job for clock-only changes would - # turn the poller into continuous audit-bundle churn. - "curtime", - "longpollid", - "mintime", - } -) - - -def qbit_template_fingerprint(template: dict[str, Any]) -> str: - stable_template = { - key: value - for key, value in template.items() - if key not in PRISM_TEMPLATE_FINGERPRINT_VOLATILE_KEYS - } - encoded = json.dumps( - stable_template, - sort_keys=True, - separators=(",", ":"), - default=str, - ).encode() - return hashlib.sha256(encoded).hexdigest() - - -def qbit_gbt_rules(chain: str) -> list[str]: - rules = ["segwit"] - if chain.strip().lower() == "signet": - rules.append("signet") - return rules - - -def target_from_compact(bits_hex: str) -> int: - return direct_stratum.target_from_compact_hex(bits_hex) - - -def scaled_target_difficulty(target: int) -> int: - if target <= 0: - raise ValueError("target must be positive") - pow_limit_target = target_from_compact("207fffff") - return max(1, (pow_limit_target * 1_000_000) // target) - - -def scaled_network_difficulty(bits_hex: str) -> int: - template_target = target_from_compact(bits_hex) - return scaled_target_difficulty(template_target) - - -class FrozenJsonDict(dict[str, Any]): - """JSON-compatible mapping that fails closed on mutation.""" - - __slots__ = () - - @staticmethod - def _immutable(*_args: object, **_kwargs: object) -> None: - raise TypeError("template artifact JSON is immutable") - - __setitem__ = _immutable - __delitem__ = _immutable - clear = _immutable - pop = _immutable - popitem = _immutable - setdefault = _immutable - update = _immutable - __ior__ = _immutable - - def __copy__(self) -> "FrozenJsonDict": - return self - - def __deepcopy__(self, _memo: dict[int, object]) -> "FrozenJsonDict": - return self - - -class FrozenJsonList(list[Any]): - """JSON-compatible sequence that fails closed on mutation.""" - - __slots__ = () - - @staticmethod - def _immutable(*_args: object, **_kwargs: object) -> None: - raise TypeError("template artifact JSON is immutable") - - __setitem__ = _immutable - __delitem__ = _immutable - __iadd__ = _immutable - __imul__ = _immutable - append = _immutable - clear = _immutable - extend = _immutable - insert = _immutable - pop = _immutable - remove = _immutable - reverse = _immutable - sort = _immutable - - def __copy__(self) -> "FrozenJsonList": - return self - - def __deepcopy__(self, _memo: dict[int, object]) -> "FrozenJsonList": - return self - - -def freeze_json(value: Any) -> Any: - """Detach and recursively freeze a JSON-like value without changing shape.""" - if isinstance(value, (FrozenJsonDict, FrozenJsonList)): - return value - if isinstance(value, Mapping): - frozen_dict = FrozenJsonDict() - dict.update( - frozen_dict, - ((str(key), freeze_json(item)) for key, item in value.items()), - ) - return frozen_dict - if isinstance(value, (list, tuple)): - frozen_list = FrozenJsonList() - list.extend(frozen_list, (freeze_json(item) for item in value)) - return frozen_list - return value - - -def freeze_json_rows(rows: Sequence[Mapping[str, Any]]) -> FrozenJsonList: - frozen = freeze_json(rows) - if not isinstance(frozen, FrozenJsonList): - raise TypeError("JSON rows must be a sequence") - return frozen - - -@dataclass(frozen=True) -class CachedTemplateArtifacts: - """Template plus everything derivable from it alone, shared by all clients. - - Derived fields are keyed by the template fingerprint: a refetch whose - fingerprint matches (only clock fields moved) reuses the previously - computed transaction hexes and witness merkle leaves instead of re-hashing - the full template. Generation records observation-start order so a slow, - older fetch cannot supersede a newer observation merely by finishing last. - """ - - template: dict[str, Any] - fingerprint: str - previousblockhash: str - transaction_hexes: tuple[str, ...] - witness_merkle_leaves_hex: tuple[str, ...] - network_difficulty: int - fetched_monotonic: float - generation: int = 0 - - -@dataclass(frozen=True) -class QbitTipTemplateSnapshot: - bestblockhash: str - previousblockhash: str - template_fingerprint: str - template_generation: int = 0 - # The observation owns this exact artifact object. It deliberately does - # not participate in snapshot equality: callers compare the stable - # identity fields above, while refresh preparation consumes the exact - # template and derivations that were observed even if the mutable cache's - # current pointer is replaced concurrently. - template_artifacts: CachedTemplateArtifacts | None = field( - default=None, - compare=False, - repr=False, - ) - - -class TemplateArtifactRuntime(Protocol): - """Typed port over the coordinator, resolved at call time. - - The repository reaches every cross-domain fact through this object so - instance-level monkeypatches on the coordinator (the historical test - seams) keep intercepting exactly as before the extraction. - """ - - rpc: Any - lock: Any - - def _ensure_job_cache_state(self) -> None: ... - - def _ensure_tip_refresh_state(self) -> None: ... - - def _job_build_phases(self) -> dict[str, float]: ... - - def _record_job_cache_event(self, kind: str, *, hit: bool) -> None: ... - - def _newest_observed_tip_locked(self) -> str | None: ... - - def _published_tip_authoritative_locked(self, now: float) -> bool: ... - - def _cancel_obsolete_job_builds( - self, - reason: str, - *, - keep_published_snapshot: bool = False, - ) -> None: ... - - def observe_tip_for_refresh(self, tip_hash: str) -> object: ... - - def _schedule_tip_refresh_retry(self) -> None: ... - - def store_template_artifacts( - self, - template: dict[str, Any], - *, - generation: int | None = None, - ) -> CachedTemplateArtifacts | None: ... - - def current_template_artifacts(self) -> CachedTemplateArtifacts: ... - - -class TemplateArtifactRepository: - """Sole owner of the template-artifact cache and observation generations. - - The repository deliberately shares the job-bundle cache lock: template - currency and job-bundle admission were historically guarded by one - ``_job_cache_lock`` and remaining coordinator domains still take that - lock around combined reads. - """ - - def __init__( - self, - runtime: TemplateArtifactRuntime, - *, - lock: threading.Lock, - ) -> None: - self._runtime = runtime - self._job_cache_lock = lock - self._template_artifacts: CachedTemplateArtifacts | None = None - self._template_artifact_generation = 0 - # Whether the generation counter was ever set explicitly (test - # embedders seed it through the coordinator compatibility field). - self._generation_explicit = False - - # -- compatibility adoption ------------------------------------------- - - def adopt_template_artifacts( - self, - artifacts: CachedTemplateArtifacts | None, - ) -> None: - """Adopt a directly assigned artifact object (legacy test seeding). - - Mirrors the historical first-touch initializer: a test that plants - ``_template_artifacts`` before touching the generation counter also - seeded the counter from the planted artifacts. - """ - self._template_artifacts = artifacts - if artifacts is not None and not self._generation_explicit: - self._template_artifact_generation = max( - self._template_artifact_generation, - int(getattr(artifacts, "generation", 0)), - ) - - def adopt_template_artifact_generation(self, generation: int) -> None: - self._template_artifact_generation = generation - self._generation_explicit = True - - # -- owner methods (bodies moved verbatim from the coordinator) -------- - - def reserve_generation(self) -> int: - """Reserve template ordering when a fetch starts, not when it finishes.""" - runtime = self._runtime - runtime._ensure_job_cache_state() - with self._job_cache_lock: - self._template_artifact_generation += 1 - return self._template_artifact_generation - - def derive( - self, - template: dict[str, Any], - *, - generation: int, - ) -> CachedTemplateArtifacts: - runtime = self._runtime - # Detach the observation from mutable RPC/test caller state. The - # dataclass then owns this exact tree for the lifetime of its snapshot. - template = copy.deepcopy(template) - fingerprint = qbit_template_fingerprint(template) - with self._job_cache_lock: - previous = self._template_artifacts - if previous is not None and previous.fingerprint == fingerprint: - return CachedTemplateArtifacts( - template=template, - fingerprint=fingerprint, - previousblockhash=str(template.get("previousblockhash", "")), - transaction_hexes=previous.transaction_hexes, - witness_merkle_leaves_hex=previous.witness_merkle_leaves_hex, - network_difficulty=previous.network_difficulty, - fetched_monotonic=time.monotonic(), - generation=generation, - ) - phases = runtime._job_build_phases() - started = time.monotonic() - transaction_hexes = direct_stratum.transaction_hexes_from_template(template) - witness_leaves = tuple(direct_stratum.witness_merkle_leaves_hex(transaction_hexes)) - network_difficulty = scaled_network_difficulty(str(template["bits"])) - phases["merkle"] = phases.get("merkle", 0.0) + (time.monotonic() - started) - return CachedTemplateArtifacts( - template=template, - fingerprint=fingerprint, - previousblockhash=str(template.get("previousblockhash", "")), - transaction_hexes=transaction_hexes, - witness_merkle_leaves_hex=witness_leaves, - network_difficulty=network_difficulty, - fetched_monotonic=time.monotonic(), - generation=generation, - ) - - def store_artifacts(self, artifacts: CachedTemplateArtifacts) -> bool: - runtime = self._runtime - changed = False - with self._job_cache_lock: - previous = self._template_artifacts - if previous is not None and artifacts.generation < previous.generation: - return False - self._template_artifacts = artifacts - if previous is not None and previous.fingerprint != artifacts.fingerprint: - changed = True - # Retain the published snapshot's bundles alongside the new - # fingerprint: until the replacement tip is published, direct - # issuance still serves published-snapshot work, and pruning - # it here would force those paths to defer for the whole - # detected-but-unpublished window. Publication moves the - # snapshot, so the next fingerprint change drops the old - # entries. - keep_fingerprints = {artifacts.fingerprint} - with runtime.lock: - published_snapshot = getattr(runtime, "tip_template_snapshot", None) - if published_snapshot is not None: - keep_fingerprints.add(published_snapshot.template_fingerprint) - runtime._job_bundle_cache = OrderedDict( - (key, entry) - for key, entry in runtime._job_bundle_cache.items() - if entry.template_fingerprint in keep_fingerprints - ) - if changed: - runtime._cancel_obsolete_job_builds( - "template fingerprint superseded", - keep_published_snapshot=True, - ) - return True - - def store( - self, - template: dict[str, Any], - *, - generation: int | None = None, - ) -> CachedTemplateArtifacts | None: - """Best-effort cache fill from an already-fetched template (blockpoll). - - Returns None instead of raising so a template the derivation cannot - digest degrades to the legacy per-build fetch path rather than failing - the poll. The returned artifacts describe this exact observation even - if a newer observation already won the cache-write race; blockpoll then - detects the mismatch before fanout. - """ - runtime = self._runtime - runtime._ensure_job_cache_state() - if generation is None: - generation = runtime._reserve_template_artifact_generation() - try: - artifacts = runtime._derive_template_artifacts( - template, - generation=generation, - ) - except Exception: - return None - runtime._store_template_artifacts(artifacts) - return artifacts - - def issuance(self) -> CachedTemplateArtifacts: - """Template artifacts for direct (non-refresh) job issuance. - - While a detected replacement tip is still unpublished, share - classification remains anchored to the published tip. Direct issuance - paths (initial delivery retries, Vardiff retargets, reauthorization) - must therefore keep handing out published-snapshot work; issuing - detected-tip work early would have every one of its shares rejected - as stale until the refresh publishes. Once the published authority - lapses (divergence lease expired), issuance falls through to the live - template exactly like submit classification falls back to the live - RPC read. - """ - runtime = self._runtime - runtime._ensure_tip_refresh_state() - with runtime.lock: - published = getattr(runtime, "current_tip_first_seen", None) - latest_detected = getattr(runtime, "latest_detected_tip", None) - published_snapshot = getattr(runtime, "tip_template_snapshot", None) - pinned = bool( - published is not None - and latest_detected is not None - and latest_detected[0] != published[0] - and published_snapshot is not None - and published_snapshot.bestblockhash == published[0] - and published_snapshot.template_artifacts is not None - and runtime._published_tip_authoritative_locked(time.monotonic()) - ) - if pinned: - assert published_snapshot is not None - assert published_snapshot.template_artifacts is not None - return published_snapshot.template_artifacts - artifacts = runtime.current_template_artifacts() - # The fetch above may itself be the first observation of a newer tip - # (a template-cache miss racing ahead of blockpoll/blockwait). The - # published tip still owns share classification, so serve its - # snapshot; the recorded detection has already armed the refresh. - with runtime.lock: - published = getattr(runtime, "current_tip_first_seen", None) - published_snapshot = getattr(runtime, "tip_template_snapshot", None) - repinned = bool( - published is not None - and artifacts.previousblockhash != published[0] - and published_snapshot is not None - and published_snapshot.bestblockhash == published[0] - and published_snapshot.template_artifacts is not None - and runtime._published_tip_authoritative_locked(time.monotonic()) - ) - if repinned: - assert published_snapshot is not None - assert published_snapshot.template_artifacts is not None - return published_snapshot.template_artifacts - return artifacts - - def current(self) -> CachedTemplateArtifacts: - """Return fresh template artifacts, fetching a template on cache miss.""" - runtime = self._runtime - runtime._ensure_job_cache_state() - ttl = getattr(runtime, "template_cache_seconds", DEFAULT_PRISM_BLOCKPOLL_SECONDS) - now = time.monotonic() - with self._job_cache_lock: - cached = self._template_artifacts - with runtime.lock: - observed_tip = runtime._newest_observed_tip_locked() - cached_tip_current = ( - observed_tip is None - or cached is None - or cached.previousblockhash == observed_tip - ) - if ( - cached is not None - and cached_tip_current - and ttl > 0 - and now - cached.fetched_monotonic <= ttl - ): - runtime._record_job_cache_event("template", hit=True) - return cached - runtime._record_job_cache_event("template", hit=False) - generation = runtime._reserve_template_artifact_generation() - phases = runtime._job_build_phases() - started = time.monotonic() - template = runtime.rpc.call( - "getblocktemplate", - [{"rules": qbit_gbt_rules(getattr(runtime, "qbit_chain", "regtest"))}], - ) - if not isinstance(template, dict): - raise RuntimeError("getblocktemplate returned non-object") - phases["template"] = phases.get("template", 0.0) + (time.monotonic() - started) - artifacts = runtime._derive_template_artifacts( - template, - generation=generation, - ) - if runtime._store_template_artifacts(artifacts): - # A direct fetch can be the first reader to see qbit advance. - # Record it as a detection like every other live-tip observation, - # or the buildability gates would treat the newer parent as - # unknown while pinned issuance still waits on blockpoll. - runtime.observe_tip_for_refresh(artifacts.previousblockhash) - return artifacts - # A later fetch completed first. Build from that current observation, - # never from the stale response that lost the cache-write race. - with self._job_cache_lock: - current = self._template_artifacts - if current is None: - raise RuntimeError("newer template artifacts disappeared after cache race") - runtime.observe_tip_for_refresh(current.previousblockhash) - return current - - def artifacts_are_current(self, artifacts: CachedTemplateArtifacts) -> bool: - runtime = self._runtime - runtime._ensure_job_cache_state() - with self._job_cache_lock: - current = self._template_artifacts - return ( - current is artifacts - or ( - current is not None - and current.fingerprint == artifacts.fingerprint - and current.generation == artifacts.generation - ) - ) - - def fetch_coherent_snapshot(self) -> QbitTipTemplateSnapshot: - runtime = self._runtime - # Reserve ordering before either RPC: a fetch that started on an older - # view must not become "newer" merely because its template arrived last. - generation = runtime._reserve_template_artifact_generation() - template = runtime.rpc.call( - "getblocktemplate", - [{"rules": qbit_gbt_rules(getattr(runtime, "qbit_chain", "regtest"))}], - ) - if not isinstance(template, dict): - raise RuntimeError("getblocktemplate returned non-object") - previousblockhash = str(template.get("previousblockhash", "") or "") - if not previousblockhash: - raise RuntimeError("getblocktemplate omitted previousblockhash") - # The template parent is the tip this work actually extends. Validate - # it after fetching the template so a tip transition between these RPCs - # cannot produce an old bestblockhash paired with newer work. Reject a - # template that was superseded before it can enter the shared cache or - # drive tip observation/graveyard pruning. - bestblockhash = str(runtime.rpc.call("getbestblockhash")) - if bestblockhash != previousblockhash: - # Record the discovery before failing: the retry-spacing gate - # releases on a newer DETECTED tip, and without blockwait this - # mismatch is the only place the new tip becomes known. An - # unrecorded discovery would hold the immediate new-tip retry - # for the full failure holdoff. - runtime.observe_tip_for_refresh(bestblockhash) - runtime._schedule_tip_refresh_retry() - raise TemplateRefreshSuperseded( - "qbit tip changed while fetching block template " - f"template_parent={previousblockhash} current={bestblockhash}" - ) - # The poll already paid for this template; seed the job-build cache so - # client job builds triggered by the refresh below reuse it instead of - # refetching one template per client. - artifacts = runtime.store_template_artifacts( - template, - generation=generation, - ) - if artifacts is not None: - return QbitTipTemplateSnapshot( - bestblockhash=bestblockhash, - previousblockhash=artifacts.previousblockhash, - template_fingerprint=artifacts.fingerprint, - template_generation=artifacts.generation, - template_artifacts=artifacts, - ) - raise TemplateRefreshBlocked( - "unable to derive exact artifacts for observed qbit template" - ) - - def current_artifacts(self) -> CachedTemplateArtifacts | None: - with self._job_cache_lock: - return self._template_artifacts - - -# Callable alias used by embedders that need a repository-compatible signature. -TemplateArtifactFetch = Callable[[], QbitTipTemplateSnapshot] - - -__all__ = [ - "CachedTemplateArtifacts", - "FrozenJsonDict", - "FrozenJsonList", - "PRISM_TEMPLATE_FINGERPRINT_VOLATILE_KEYS", - "PayoutStatePublicationBlocked", - "QbitTipTemplateSnapshot", - "TemplateArtifactRepository", - "TemplateArtifactRuntime", - "TemplateRefreshBlocked", - "TemplateRefreshSuperseded", - "freeze_json", - "freeze_json_rows", - "qbit_gbt_rules", - "qbit_template_fingerprint", - "scaled_network_difficulty", - "scaled_target_difficulty", - "target_from_compact", -] diff --git a/lab/prism/tip_refresh.py b/lab/prism/tip_refresh.py deleted file mode 100644 index 575532926..000000000 --- a/lab/prism/tip_refresh.py +++ /dev/null @@ -1,4092 +0,0 @@ -#!/usr/bin/env python3 -"""PRISM tip observation, publication, and bounded refresh ownership. - -The service owns the R1 domain: tip observation/publication state, the -latest-wins refresh waves and their epoch convergence authority (#101), -refresh pending/retry signaling, failure holdoff pacing, fanout delivery -coordination, blockpoll/blockwait loops, retained collection refreshes, and -the tip-refresh metric state. - -It never imports ``prism_coordinator``. Every cross-domain fact -- payout -state, job-bundle preparation, session membership, delivery submission, -reorg reconciliation, progress health, watchdog heartbeats, live -configuration attributes -- is reached through the :class:`TipRefreshRuntime` -typed port, resolved at call time so the historical coordinator monkeypatch -seams (including the instance-level facade patches used by the current test -suite) keep intercepting exactly as before the extraction. Reconcile -flights/memo/prefetch and share-ack timing remain coordinator-owned at this -layer (they belong to later stack owners); R1 reaches them only through -call-time runtime ports. -""" - -from __future__ import annotations - -from concurrent.futures import ( - FIRST_COMPLETED, - CancelledError as FuturesCancelledError, - Future, - ThreadPoolExecutor, - wait, -) -from dataclasses import dataclass, field -from decimal import Decimal, InvalidOperation -import os -import random -import threading -import time -import traceback -from typing import Any, Protocol - -from lab.prism.bounded_executor import _BoundedPriorityExecutor -from lab.prism.coordinator_config import ( - DEFAULT_PRISM_BLOCKPOLL_SECONDS, - DEFAULT_PRISM_BLOCKWAIT_TIMEOUT_SECONDS, - DEFAULT_PRISM_REORG_RECONCILE_CACHE_SECONDS, - DEFAULT_PRISM_SUBMIT_TIP_MAX_AGE_SECONDS, - DEFAULT_PRISM_TEMPLATE_MAX_AGE_SECONDS, - DEFAULT_PRISM_TIP_REFRESH_FAILURE_HOLDOFF_SECONDS, -) -from lab.prism.job_bundle import CachedJobBundle, JobBuildKey, JobBuildSuperseded -from lab.prism.template_artifacts import ( - CachedTemplateArtifacts, - PayoutStatePublicationBlocked, - QbitTipTemplateSnapshot, - TemplateRefreshBlocked, - TemplateRefreshSuperseded, - qbit_template_fingerprint, -) - - -# Owner-local copies of still-coordinator-visible constants; compatibility -# duplicates by design (the coordinator's import source for the admission -# poll interval is job_delivery.py, and the payout supersession-retry -# default remains a coordinator/P1 definition at this layer). -PRISM_TIP_REFRESH_ADMISSION_POLL_SECONDS = 0.05 -DEFAULT_PRISM_PAYOUT_RECONCILE_SUPERSESSION_RETRIES = 8 -# Fraction of the holdoff added as random jitter so coordinators sharing one -# qbitd do not phase-lock their blocked-refresh re-attempts. -PRISM_TIP_REFRESH_FAILURE_HOLDOFF_JITTER_FRACTION = 0.25 -PRISM_TIP_REFRESH_REENTRY_BACKOFF_SECONDS = 0.05 -PRISM_TIP_REFRESH_WAVE_PASS_BUDGET = 64 -PRISM_TIP_REFRESH_SECONDS_BUCKETS = ( - 0.01, - 0.025, - 0.05, - 0.1, - 0.25, - 0.5, - 1.0, - 2.5, - 5.0, - 10.0, - 30.0, -) -PRISM_TIP_REFRESH_BUILD_PHASES = ( - "reorg_reconcile", - "ledger_snapshot", - "payout_state_derivation", - "ctv_manifest_construction", - "coinbase_bundle_construction", - "signing_verification", - "serialization_copy", - "singleflight_wait", -) -PRISM_TIP_REFRESH_RESULTS = ("sent", "skipped", "disconnected", "failed") -PRISM_TIP_REFRESH_CANCELLATION_STAGES = ( - "executor_queue", - "client_lock", - "payout_gate", -) -PRISM_TIP_REFRESH_WAVE_OUTCOMES = ( - "completed", - "fanout_superseded", - "build_superseded", - "payout_blocked", - "trust_blocked", - "shutdown", - "error", -) -PRISM_TIP_REFRESH_COVERAGE_TARGETS = ( - ("50", Decimal("0.50")), - ("95", Decimal("0.95")), - ("99", Decimal("0.99")), -) - - -@dataclass(frozen=True) -class RetainedCollectionRefresh: - """Current immutable preparation waiting for a collection identity.""" - - snapshot: QbitTipTemplateSnapshot - observation_sequence: int - payout_state_generation: int - - -@dataclass(frozen=True) -class RefreshResult: - result: str - delivered_monotonic: float | None = None - - -@dataclass(frozen=True, eq=False) -class TipRefreshValidationToken: - """Immutable proof that one prepared refresh passed its expensive guard.""" - - tip_hash: str - template_fingerprint: str - template_generation: int - payout_state_generation: int - observation_sequence: int - build_key: JobBuildKey - snapshot: QbitTipTemplateSnapshot = field(repr=False) - epoch_sequence: int = 0 - - -@dataclass -class _TipRefreshEpochCoverage: - """Client cohort snapshot used to observe fixed-cardinality coverage latency.""" - - sequence: int - tip_hash: str - payout_state_generation: int - started_monotonic: float - client_weights: dict[int, Decimal] - total_weight: Decimal - delivered_weight: Decimal = Decimal(0) - delivered_clients: set[int] = field(default_factory=set) - recorded_targets: set[str] = field(default_factory=set) - - -class FanoutCancellation: - """Cancel a fanout without racing already-admitted deliveries. - - ``cancel`` closes admission without waiting, so workers can call it while - holding a client lock. The fanout coordinator calls ``set`` outside client - locks to wait for deliveries that already passed the final gate. - """ - - def __init__(self) -> None: - self._condition = threading.Condition() - self._cancelling = False - self._active_deliveries = 0 - - def is_set(self) -> bool: - with self._condition: - return self._cancelling - - def begin_delivery(self) -> bool: - with self._condition: - if self._cancelling: - return False - self._active_deliveries += 1 - return True - - def end_delivery(self) -> None: - with self._condition: - if self._active_deliveries <= 0: - raise RuntimeError("fanout delivery gate released without admission") - self._active_deliveries -= 1 - if self._active_deliveries == 0: - self._condition.notify_all() - - def cancel(self) -> None: - with self._condition: - self._cancelling = True - - def set(self) -> None: - self.cancel() - with self._condition: - while self._active_deliveries: - self._condition.wait() - - -class _TipRefreshFanoutSuperseded(TemplateRefreshSuperseded): - """A published fanout lost to a newer tip-refresh epoch.""" - - -class _TipRefreshTrustBlocked(TemplateRefreshBlocked): - """Refresh work stopped because the live chain view was not trusted.""" - - -class TipRefreshRuntime(Protocol): - """Typed port over the coordinator, resolved at call time. - - Every member is looked up on the live coordinator object when used, so - instance monkeypatches (``server.maybe_send_job = ...`` and friends) and - coordinator-owned live configuration attributes keep working exactly as - before the extraction. Owner facades route back into this service; that - round trip is deliberate: it is what preserves the current white-box - patch surface until later stack layers repoint those tests. The legacy - R1 field names also resolve here -- coordinator class descriptors route - them to this service's single mutable copy. - """ - - # Cross-domain objects and live configuration attributes. - _job_cache_lock: Any - _payout_artifact_executor_lock: Any - _payout_ledger_append_invalidation_epoch: Any - _payout_state_delivery_gate: Any - _payout_state_generation: Any - _payout_state_publication_blocked: Any - _payout_state_source: Any - _pool_ready_latched: Any - _prepared_ready_bundle: Any - _prepared_ready_snapshot: Any - _progress_has_published_work: Any - _progress_health_lock: Any - _progress_publication_divergence_since_monotonic: Any - _progress_published_payout_generation: Any - _progress_published_template_fingerprint: Any - _progress_refresh_signal_pending: Any - _published_payout_state: Any - _serve_builder_metrics_lock: Any - _template_artifacts: Any - blockpoll_seconds: Any - blockwait_timeout_seconds: Any - clients: Any - job_build_failure_count: Any - job_preparation_pending: Any - lock: Any - payout_artifact_event_counts: Any - payout_reconcile_supersession_retries: Any - reorg_reconcile_cache_seconds: Any - reorg_reconciler_enabled: Any - rpc: Any - serve_builder_counts: Any - serve_builder_window_cache_counts: Any - stop_event: Any - submit_tip_max_age_seconds: Any - template_cache_seconds: Any - template_refresh_failure_exit_seconds: Any - tip_refresh_epoch_fanout: Any - tip_refresh_failure_holdoff_seconds: Any - tip_refresh_max_workers: Any - watchdog_timeout_seconds: Any - - def _artifacts_buildable_locked(self, *args: Any, **kwargs: Any) -> Any: ... - - def _begin_job_build_priority_preparation(self, *args: Any, **kwargs: Any) -> Any: ... - - def _blockwait_unsupported(self, *args: Any, **kwargs: Any) -> Any: ... - - def _cancel_obsolete_job_builds(self, *args: Any, **kwargs: Any) -> Any: ... - - def _cancel_obsolete_job_bundle_builds(self, *args: Any, **kwargs: Any) -> Any: ... - - def _clear_coordination_blocked_streak(self, *args: Any, **kwargs: Any) -> Any: ... - - def _discard_stale_reconcile_prefetch(self, *args: Any, **kwargs: Any) -> Any: ... - - def _ensure_job_cache_state(self, *args: Any, **kwargs: Any) -> Any: ... - - def _ensure_tip_refresh_state(self, *args: Any, **kwargs: Any) -> Any: ... - - def _evict_reorg_reconcile_memo_for_new_tip_locked(self, *args: Any, **kwargs: Any) -> Any: ... - - def _finish_job_build_priority_preparation(self, *args: Any, **kwargs: Any) -> Any: ... - - def _join_reconcile_prefetch_bounded(self, *args: Any, **kwargs: Any) -> Any: ... - - def _newest_observed_tip_locked(self, *args: Any, **kwargs: Any) -> Any: ... - - def _note_tip_observation_for_candidates(self, *args: Any, **kwargs: Any) -> Any: ... - - def _progress_bundle_build_finished(self, *args: Any, **kwargs: Any) -> Any: ... - - def _progress_bundle_build_started(self, *args: Any, **kwargs: Any) -> Any: ... - - def _progress_note_refresh_pending(self, *args: Any, **kwargs: Any) -> Any: ... - - def _progress_refresh_finished(self, *args: Any, **kwargs: Any) -> Any: ... - - def _progress_refresh_started(self, *args: Any, **kwargs: Any) -> Any: ... - - def _reconcile_snapshot_tip_bounded(self, *args: Any, **kwargs: Any) -> Any: ... - - def _record_coordination_blocked_refresh(self, *args: Any, **kwargs: Any) -> Any: ... - - def _record_heartbeat(self, *args: Any, **kwargs: Any) -> Any: ... - - def _record_job_cache_event(self, *args: Any, **kwargs: Any) -> Any: ... - - def _record_progress_publication(self, *args: Any, **kwargs: Any) -> Any: ... - - def _record_progress_tip_poll(self, *args: Any, **kwargs: Any) -> Any: ... - - def _record_reorg_reconcile_lookup(self, *args: Any, **kwargs: Any) -> Any: ... - - def _record_startup_phase_once(self, *args: Any, **kwargs: Any) -> Any: ... - - def _remove_watchdog_heartbeat(self, *args: Any, **kwargs: Any) -> Any: ... - - def _reorg_reconcile_memo_fresh(self, *args: Any, **kwargs: Any) -> Any: ... - - def _submit_delivery_task(self, *args: Any, **kwargs: Any) -> Any: ... - - def _submit_reconcile_prefetch(self, *args: Any, **kwargs: Any) -> Any: ... - - def _submit_stale_check_tip_locked(self, *args: Any, **kwargs: Any) -> Any: ... - - def delivery_queue_limit(self, *args: Any, **kwargs: Any) -> Any: ... - - def disconnect_client(self, *args: Any, **kwargs: Any) -> Any: ... - - def ensure_reorg_reconciled_for_current_tip(self, *args: Any, **kwargs: Any) -> Any: ... - - def ensure_reorg_reconciled_for_tip(self, *args: Any, **kwargs: Any) -> Any: ... - - def fetch_qbit_tip_template_snapshot(self, *args: Any, **kwargs: Any) -> Any: ... - - def observe_job_build_elapsed(self, *args: Any, **kwargs: Any) -> Any: ... - - def pool_readiness_latched(self, *args: Any, **kwargs: Any) -> Any: ... - - def qbit_chain_view_untrusted(self, *args: Any, **kwargs: Any) -> Any: ... - - def retire_share_window_spool(self, *args: Any, **kwargs: Any) -> Any: ... - - def shutdown_job_build_executor(self, *args: Any, **kwargs: Any) -> Any: ... - - def shutdown_payout_artifact_executor(self, *args: Any, **kwargs: Any) -> Any: ... - - def shutdown_reconcile_prefetch_executor(self, *args: Any, **kwargs: Any) -> Any: ... - - def shutdown_serve_builder(self, *args: Any, **kwargs: Any) -> Any: ... - - def send_prepared_job(self, *args: Any, **kwargs: Any) -> Any: ... - - def shared_job_bundle(self, *args: Any, **kwargs: Any) -> Any: ... - - -class TipRefreshService: - """Sole owner of R1 tip observation, refresh scheduling, and metrics.""" - - def __init__( - self, - runtime: TipRefreshRuntime, - *, - shutdown_error: type[BaseException], - job_build_failed_error: type[BaseException], - delivery_priority_initial: int, - delivery_priority_new_tip: int, - delivery_priority_same_tip: int, - ) -> None: - self._runtime = runtime - # Coordinator-owned exception types and S2 delivery priorities are - # injected so this leaf module never imports prism_coordinator (or - # job_delivery, which sits above it in the owner layering). - self._shutdown_error = shutdown_error - self._job_build_failed_error = job_build_failed_error - self._delivery_priority_initial = delivery_priority_initial - self._delivery_priority_new_tip = delivery_priority_new_tip - self._delivery_priority_same_tip = delivery_priority_same_tip - self._tip_refresh_lock = threading.Lock() - self._tip_refresh_singleflight_lock = threading.Lock() - self._tip_refresh_executor_lock = threading.Lock() - self._tip_refresh_executor: _BoundedPriorityExecutor | None = None - self._tip_refresh_executor_shutdown = False - self._tip_refresh_metrics_lock = threading.Lock() - self.tip_refresh_histograms = { - name: { - "buckets": {bucket: 0 for bucket in PRISM_TIP_REFRESH_SECONDS_BUCKETS}, - "sum": 0.0, - "count": 0, - } - for name in ( - "refresh", - "bundle_build", - "first_delivery", - "last_delivery", - "fanout_wave", - ) - } - self.tip_refresh_build_phase_histograms = { - phase: { - "buckets": { - bucket: 0 for bucket in PRISM_TIP_REFRESH_SECONDS_BUCKETS - }, - "sum": 0.0, - "count": 0, - } - for phase in PRISM_TIP_REFRESH_BUILD_PHASES - } - self.tip_refresh_client_counts = { - result: 0 for result in PRISM_TIP_REFRESH_RESULTS - } - self.tip_refresh_cancellation_counts = { - stage: 0 for stage in PRISM_TIP_REFRESH_CANCELLATION_STAGES - } - self.tip_refresh_wave_outcome_counts = { - outcome: 0 for outcome in PRISM_TIP_REFRESH_WAVE_OUTCOMES - } - self.tip_refresh_coverage_histograms = { - target: { - "buckets": { - bucket: 0 for bucket in PRISM_TIP_REFRESH_SECONDS_BUCKETS - }, - "sum": 0.0, - "count": 0, - } - for target, _ratio in PRISM_TIP_REFRESH_COVERAGE_TARGETS - } - self.tip_refresh_inflight = 0 - self.tip_refresh_build_inflight = 0 - self.tip_refresh_build_queue_depth = 0 - self.tip_refresh_singleflight_hits = 0 - self.tip_refresh_superseded_results = 0 - self.tip_refresh_worker_failures = 0 - self.tip_refresh_worker_restarts = 0 - self.tip_refresh_ipc_bytes = {"input": 0, "output": 0} - self._tip_refresh_pending_event = threading.Event() - self._tip_refresh_pending_counter = 0 - self._tip_refresh_pending_token: int | None = None - self._tip_refresh_retry = threading.Event() - self._tip_refresh_retry_counter = 0 - self._tip_refresh_retry_consumed = 0 - self._tip_refresh_failure_holdoff_until: float | None = None - self._tip_refresh_failure_tip: str | None = None - self._active_tip_refresh: tuple[ - TipRefreshValidationToken, - FanoutCancellation, - ] | None = None - self._tip_refresh_epoch_sequence = 0 - self._tip_refresh_epoch_tip_hash: str | None = None - self._tip_refresh_epoch_payout_generation = 0 - self._tip_refresh_epoch_coverage: _TipRefreshEpochCoverage | None = None - self._published_tip_refresh_epoch_identity: tuple[ - int, - str, - int, - str, - int, - ] | None = None - self._retained_collection_refresh: RetainedCollectionRefresh | None = None - # The twelve historical plain coordinator fields (plus the - # tip-detection epoch) are deliberately NOT initialized here: they - # come into existence on first write exactly as the pre-extraction - # lazy fields did, so ``hasattr``/``getattr``-with-default legacy - # semantics (and the coordinator descriptor shims that route the old - # names to this service) stay byte-for-byte compatible: - # current_tip_first_seen, current_tip_parent, - # current_tip_observation_sequence, current_tip_observed_monotonic, - # tip_template_snapshot, latest_detected_tip, - # tip_refresh_divergence_started_monotonic, tip_observation_sequence, - # last_successful_template_refresh_monotonic, - # template_refresh_failure_started_monotonic, tip_refresh_job_count, - # post_accept_refresh_failure_count, tip_detection_epoch. - - def _retain_collection_refresh( - self, - snapshot: QbitTipTemplateSnapshot, - observation_sequence: int, - payout_state_generation: int, - ) -> None: - """Retain reusable current work until an eligible identity appears.""" - runtime = self._runtime - retained = RetainedCollectionRefresh( - snapshot=snapshot, - observation_sequence=observation_sequence, - payout_state_generation=payout_state_generation, - ) - should_log = False - with runtime.lock: - if not runtime._tip_refresh_snapshot_current_locked( - snapshot, - observation_sequence, - ): - return - if any( - runtime.client_can_receive_jobs(client) - for client in runtime.clients - ): - return - previous = self._retained_collection_refresh - self._retained_collection_refresh = retained - should_log = previous is None or ( - previous.snapshot.bestblockhash != snapshot.bestblockhash - or previous.payout_state_generation != payout_state_generation - ) - if should_log: - print( - "prism coordinator: collection refresh retained while no " - "authorized worker identity is available", - flush=True, - ) - - def _retained_collection_artifacts(self) -> CachedTemplateArtifacts | None: - """Return retained artifacts while their published work stays current. - - A same-tip poll advances its observation sequence before atomically - replacing ``tip_template_snapshot``. The published snapshot remains - reusable on both sides of that handoff even if the retained marker has - not yet been updated; a new tip or payout generation still invalidates - it immediately. - """ - runtime = self._runtime - runtime._ensure_job_cache_state() - runtime._ensure_tip_refresh_state() - with runtime._job_cache_lock: - payout_state_generation = runtime._payout_state_generation - with runtime.lock: - retained = self._retained_collection_refresh - if retained is None: - return None - if retained.payout_state_generation != payout_state_generation: - return None - current_tip = getattr(self, "current_tip_first_seen", None) - published_snapshot = self.tip_template_snapshot - if ( - published_snapshot is None - or current_tip is None - or current_tip[0] != published_snapshot.bestblockhash - ): - return None - return runtime._tip_refresh_artifacts(published_snapshot) - - def _retain_current_collection_refresh_if_unrepresented(self) -> None: - """Keep the last published collection work when the fleet empties.""" - runtime = self._runtime - runtime._ensure_tip_refresh_state() - if getattr(runtime, "_pool_ready_latched", False): - return - with runtime.lock: - if any( - runtime.client_can_receive_jobs(client) - for client in runtime.clients - ): - return - snapshot = self.tip_template_snapshot - observation_sequence = int( - getattr(self, "current_tip_observation_sequence", 0) - ) - if snapshot is None: - return - runtime._ensure_job_cache_state() - with runtime._job_cache_lock: - payout_state_generation = runtime._payout_state_generation - runtime._retain_collection_refresh( - snapshot, - observation_sequence, - payout_state_generation, - ) - - def _note_collection_identity_available(self, client: ClientState) -> None: - """Wake a retained collection refresh as soon as a client is eligible.""" - runtime = self._runtime - if not runtime.client_can_receive_jobs(client): - return - if runtime._retained_collection_artifacts() is None: - return - runtime._mark_tip_refresh_pending(client.connection_id) - runtime._schedule_tip_refresh_retry() - - def _consume_retained_collection_refresh( - self, - context: PrismJobContext, - ) -> None: - """Consume retention only after its collection work was delivered.""" - runtime = self._runtime - if not context.collection_only: - return - with runtime.lock: - retained = self._retained_collection_refresh - published_snapshot = self.tip_template_snapshot - artifacts = ( - published_snapshot.template_artifacts - if published_snapshot is not None - else None - ) - if ( - retained is not None - and retained.payout_state_generation - == context.payout_state_generation - and artifacts is not None - and context.template is artifacts.template - and context.template_fingerprint == artifacts.fingerprint - and context.template_generation == artifacts.generation - ): - self._retained_collection_refresh = None - - def _tip_refresh_pending(self) -> bool: - runtime = self._runtime - runtime._ensure_tip_refresh_state() - return self._tip_refresh_pending_event.is_set() - - def _mark_tip_refresh_pending(self, _observation: object) -> int: - runtime = self._runtime - runtime._ensure_tip_refresh_state() - with runtime.lock: - self._tip_refresh_pending_counter += 1 - token = self._tip_refresh_pending_counter - self._tip_refresh_pending_token = token - self._tip_refresh_pending_event.set() - return token - - def _claim_tip_refresh_pending(self) -> int | None: - """Snapshot pending work without replacing a newer producer's token.""" - runtime = self._runtime - runtime._ensure_tip_refresh_state() - with runtime.lock: - if not self._tip_refresh_pending_event.is_set(): - return None - return self._tip_refresh_pending_token - - def _mark_tip_refresh_pending_for_poll( - self, - owned_token: int | None, - _observation: object, - ) -> int | None: - """Mark poll-owned work only while no newer producer has superseded it.""" - runtime = self._runtime - runtime._ensure_tip_refresh_state() - with runtime.lock: - if self._tip_refresh_pending_token != owned_token: - return owned_token - if owned_token is not None: - self._tip_refresh_pending_event.set() - return owned_token - self._tip_refresh_pending_counter += 1 - token = self._tip_refresh_pending_counter - self._tip_refresh_pending_token = token - self._tip_refresh_pending_event.set() - return token - - def _clear_tip_refresh_pending(self, token: int) -> None: - runtime = self._runtime - runtime._ensure_tip_refresh_state() - with runtime.lock: - if self._tip_refresh_pending_token == token: - self._tip_refresh_pending_token = None - self._tip_refresh_pending_event.clear() - - def _clear_tip_refresh_pending_for_completed_refresh( - self, - snapshot: QbitTipTemplateSnapshot, - observation_sequence: int, - payout_state_generation: int, - pending_signal_token: int | None = None, - ) -> bool: - """Atomically acknowledge pending work handled by a completed poll.""" - runtime = self._runtime - runtime._ensure_job_cache_state() - with runtime._payout_state_delivery_gate.delivery_cancelable( - lambda: runtime._payout_state_generation != payout_state_generation, - generation=payout_state_generation, - priority=True, - ) as admission: - if not admission: - return False - with runtime._job_cache_lock: - payout_state_current = ( - runtime._payout_state_generation == payout_state_generation - ) - with runtime.lock: - refresh_current = runtime._tip_refresh_snapshot_current_locked( - snapshot, - observation_sequence, - ) - pending_owned = ( - self._tip_refresh_pending_token == pending_signal_token - ) - if ( - not payout_state_current - or not refresh_current - or not pending_owned - ): - return False - self._tip_refresh_pending_token = None - self._tip_refresh_pending_event.clear() - with runtime._progress_health_lock: - published_current = bool( - runtime._progress_has_published_work - and runtime._progress_published_template_fingerprint - == snapshot.template_fingerprint - and runtime._progress_published_payout_generation - == payout_state_generation - ) - if published_current: - # The completion guard above proves this coherent - # snapshot still represents the latest detected and - # published tip. A transient A -> B -> A observation - # therefore closes its publication-divergence epoch - # even when existing A work needed no new delivery. - runtime._progress_refresh_signal_pending = False - runtime._progress_publication_divergence_since_monotonic = None - return True - - def _schedule_tip_refresh_retry(self) -> None: - runtime = self._runtime - runtime._ensure_tip_refresh_state() - # Pair the Event with a monotonic generation so a producer cannot set - # it between a waiter's wake and clear and lose the newest retry. The - # event remains the blocking primitive; the generation is the durable - # coalesced work marker. - with runtime.lock: - self._tip_refresh_retry_counter += 1 - self._tip_refresh_retry.set() - - def _consume_tip_refresh_retry(self) -> bool: - """Consume all retry signals visible at one atomic wake boundary.""" - runtime = self._runtime - runtime._ensure_tip_refresh_state() - with runtime.lock: - generation = self._tip_refresh_retry_counter - if generation == self._tip_refresh_retry_consumed: - return False - self._tip_refresh_retry_consumed = generation - self._tip_refresh_retry.clear() - return True - - def _note_tip_refresh_attempt_failed( - self, - observed_tip: str | None = None, - ) -> None: - """Stamp a failed refresh pass so the poller spaces its re-attempt. - - ``observed_tip`` is the tip the failed pass worked against; a pass - that failed before learning one stamps the current observation so an - RPC outage with a static tip still gets spaced. - """ - runtime = self._runtime - runtime._ensure_tip_refresh_state() - holdoff = float( - getattr( - runtime, - "tip_refresh_failure_holdoff_seconds", - DEFAULT_PRISM_TIP_REFRESH_FAILURE_HOLDOFF_SECONDS, - ) - ) - if holdoff <= 0: - return - holdoff += random.uniform( - 0.0, - holdoff * PRISM_TIP_REFRESH_FAILURE_HOLDOFF_JITTER_FRACTION, - ) - with runtime.lock: - if observed_tip is None: - observed_tip = runtime._newest_observed_tip_locked() - self._tip_refresh_failure_tip = observed_tip - self._tip_refresh_failure_holdoff_until = time.monotonic() + holdoff - - def _tip_refresh_failure_holdoff_remaining(self) -> float: - """Seconds the poller must still wait before re-running a failed pass. - - Zero as soon as the newest known tip differs from the one the failed - pass worked against: spacing throttles re-attempts against an - unchanged, blocked or churning view, never the reaction to a - genuinely new tip (including one detected but not yet published). - """ - runtime = self._runtime - runtime._ensure_tip_refresh_state() - with runtime.lock: - deadline = self._tip_refresh_failure_holdoff_until - failed_tip = self._tip_refresh_failure_tip - current_hash = runtime._newest_observed_tip_locked() - if deadline is None: - return 0.0 - if current_hash != failed_tip: - return 0.0 - return max(0.0, deadline - time.monotonic()) - - def _observe_tip_refresh_seconds(self, name: str, elapsed_seconds: float) -> None: - runtime = self._runtime - runtime._ensure_tip_refresh_state() - with self._tip_refresh_metrics_lock: - histogram = self.tip_refresh_histograms[name] - histogram["count"] = int(histogram["count"]) + 1 - histogram["sum"] = float(histogram["sum"]) + elapsed_seconds - buckets = histogram["buckets"] - assert isinstance(buckets, dict) - for bucket in PRISM_TIP_REFRESH_SECONDS_BUCKETS: - if elapsed_seconds <= bucket: - buckets[bucket] = int(buckets.get(bucket, 0)) + 1 - - def _observe_tip_refresh_build_phase( - self, - phase: str, - elapsed_seconds: float, - ) -> None: - runtime = self._runtime - if phase not in PRISM_TIP_REFRESH_BUILD_PHASES: - raise ValueError(f"unknown tip refresh build phase: {phase}") - runtime._ensure_tip_refresh_state() - with self._tip_refresh_metrics_lock: - histogram = self.tip_refresh_build_phase_histograms[phase] - histogram["count"] = int(histogram["count"]) + 1 - histogram["sum"] = float(histogram["sum"]) + max( - 0.0, elapsed_seconds - ) - buckets = histogram["buckets"] - assert isinstance(buckets, dict) - for bucket in PRISM_TIP_REFRESH_SECONDS_BUCKETS: - if elapsed_seconds <= bucket: - buckets[bucket] = int(buckets.get(bucket, 0)) + 1 - - def _record_tip_refresh_ipc_bytes(self, direction: str, byte_count: int) -> None: - runtime = self._runtime - if direction not in {"input", "output"}: - raise ValueError(f"unknown tip refresh IPC direction: {direction}") - runtime._ensure_tip_refresh_state() - with self._tip_refresh_metrics_lock: - self.tip_refresh_ipc_bytes[direction] += max(0, int(byte_count)) - - def _record_tip_refresh_client_result(self, result: str) -> None: - runtime = self._runtime - if result not in PRISM_TIP_REFRESH_RESULTS: - raise ValueError(f"unknown tip refresh result: {result}") - runtime._ensure_tip_refresh_state() - with self._tip_refresh_metrics_lock: - self.tip_refresh_client_counts[result] += 1 - - def _record_tip_refresh_cancellation(self, stage: str) -> None: - runtime = self._runtime - if stage not in PRISM_TIP_REFRESH_CANCELLATION_STAGES: - raise ValueError(f"unknown tip refresh cancellation stage: {stage}") - runtime._ensure_tip_refresh_state() - with self._tip_refresh_metrics_lock: - self.tip_refresh_cancellation_counts[stage] += 1 - - def _record_tip_refresh_wave_outcome(self, outcome: str) -> None: - runtime = self._runtime - runtime._ensure_tip_refresh_state() - if outcome not in PRISM_TIP_REFRESH_WAVE_OUTCOMES: - raise ValueError(f"unknown tip refresh wave outcome: {outcome}") - with self._tip_refresh_metrics_lock: - self.tip_refresh_wave_outcome_counts[outcome] = ( - int(self.tip_refresh_wave_outcome_counts.get(outcome, 0)) + 1 - ) - - def _tip_refresh_future_started(self) -> None: - runtime = self._runtime - runtime._ensure_tip_refresh_state() - with self._tip_refresh_metrics_lock: - self.tip_refresh_inflight += 1 - - def _tip_refresh_future_finished(self, _future: Future[RefreshResult]) -> None: - runtime = self._runtime - runtime._ensure_tip_refresh_state() - with self._tip_refresh_metrics_lock: - self.tip_refresh_inflight = max(0, self.tip_refresh_inflight - 1) - - def tip_refresh_executor(self) -> _BoundedPriorityExecutor: - runtime = self._runtime - runtime._ensure_tip_refresh_state() - with self._tip_refresh_executor_lock: - if self._tip_refresh_executor_shutdown: - raise RuntimeError("tip refresh executor is shut down") - executor = self._tip_refresh_executor - if executor is None: - executor = _BoundedPriorityExecutor( - max_workers=runtime.tip_refresh_max_workers, - max_queue_size=runtime.delivery_queue_limit(), - thread_name_prefix="prism-tip-refresh-delivery", - ) - self._tip_refresh_executor = executor - return executor - - def shutdown_tip_refresh_executor(self) -> None: - runtime = self._runtime - runtime._ensure_tip_refresh_state() - with self._tip_refresh_executor_lock: - executor = self._tip_refresh_executor - self._tip_refresh_executor = None - self._tip_refresh_executor_shutdown = True - if executor is not None: - # Stop queued publication work before waiting on reconnect workers. - executor.shutdown(wait=False, cancel_futures=True) - runtime.shutdown_initial_job_executor() - if executor is not None: - # Running workers may already hold client/job state or be inside a - # socket send. Drain them before serve returns and the writer lease - # is released; queued workers are cancelled without starting. - executor.shutdown(wait=True) - runtime.shutdown_job_build_executor() - runtime.shutdown_payout_artifact_executor() - runtime.shutdown_reconcile_prefetch_executor() - runtime.retire_share_window_spool() - runtime.shutdown_serve_builder() - - def publication_progress_failure_expired(self, now: float) -> bool: - """Bound sustained detected/current publication divergence. - - Retry-loop heartbeats prove only that the driver is scheduled. This - dedicated deadline is independent of client-delivery health and is - cleared only by current work publication/delivery, so old delivery - backlog cannot make a later legitimate divergence expire immediately. - """ - runtime = self._runtime - budget = float( - getattr( - runtime, - "template_refresh_failure_exit_seconds", - DEFAULT_PRISM_TEMPLATE_MAX_AGE_SECONDS, - ) - ) - if budget <= 0: - return False - runtime._ensure_job_cache_state() - with runtime._progress_health_lock: - divergence_since = ( - runtime._progress_publication_divergence_since_monotonic - ) - return bool( - divergence_since is not None - and now - divergence_since >= budget - ) - - def _cancel_active_tip_refresh_for_shutdown(self) -> None: - runtime = self._runtime - runtime._ensure_tip_refresh_state() - with runtime.lock: - active = self._active_tip_refresh - if active is not None: - active[1].cancel() - self._tip_refresh_retry.clear() - - def _wait_for_blockpoll_trigger(self) -> bool: - """Wait for the normal interval or an immediate coalesced retry.""" - runtime = self._runtime - remaining = float(runtime.blockpoll_seconds) - while remaining > 0: - if runtime.stop_event.is_set(): - return False - # After a failed pass, leave retry signals unconsumed until the - # spacing window closes. Re-checked per slice: a newly observed - # tip zeroes the holdoff immediately, and signals arriving during - # the hold stay coalesced for the attempt that eventually runs. - holdoff = runtime._tip_refresh_failure_holdoff_remaining() - if holdoff <= 0 and runtime._consume_tip_refresh_retry(): - return not runtime.stop_event.is_set() - wait_seconds = min(remaining, 0.25) - if holdoff > 0: - # The retry event may already be set; pace on the stop event - # so the spacing window cannot be spun through instantly. - # Short slices keep a newly detected tip's release prompt. - # Beat per slice: a deliberately held poller is idle, not - # hung, and must not trip the watchdog when the configured - # holdoff exceeds its timeout. - runtime._record_heartbeat("qbit_blockpoll") - wait_seconds = min(wait_seconds, holdoff, 0.05) - runtime.stop_event.wait(wait_seconds) - else: - self._tip_refresh_retry.wait(wait_seconds) - remaining -= wait_seconds - # Interval-driven attempts respect the same spacing so a sub-holdoff - # blockpoll interval cannot re-run a failed pass early. - while not runtime.stop_event.is_set(): - holdoff = runtime._tip_refresh_failure_holdoff_remaining() - if holdoff <= 0: - break - runtime._record_heartbeat("qbit_blockpoll") - runtime.stop_event.wait(min(holdoff, 0.05)) - runtime._consume_tip_refresh_retry() - return not runtime.stop_event.is_set() - - def blockpoll_loop(self) -> None: - runtime = self._runtime - runtime._ensure_tip_refresh_state() - while runtime._wait_for_blockpoll_trigger(): - # A superseding observation or post-fanout tip change wakes this - # fallback loop immediately. The event coalesces repeated signals; - # ordinary same-tip polling retains its configured interval. - # Heartbeat at the top of each iteration: reaching here proves the - # loop is alive. A transient qbit RPC error still loops and beats; a - # hung RPC call never returns, so the beat goes stale and the - # watchdog restarts the process. - runtime._record_heartbeat("qbit_blockpoll") - try: - refreshed = runtime.poll_qbit_tip_template_once() - if refreshed: - print( - f"prism coordinator: refreshed {refreshed} client job(s) after qbit tip/template change", - flush=True, - ) - except self._shutdown_error: - # Admission can close after the loop condition but before a - # nested reconciliation enters the writer gate. That is an - # intentional shutdown stop, not a template-health failure. - return - except (TemplateRefreshSuperseded, PayoutStatePublicationBlocked) as exc: - # Coordination-blocked attempts remain outside the ordinary - # template failure budget. Their separate continuous-streak - # budget is recorded by poll_qbit_tip_template_once and - # enforced by the mandatory publication-progress watchdog. - # The retry is already scheduled by the raise site. - print( - f"prism coordinator: tip/template refresh superseded; retrying: {exc}", - flush=True, - ) - except Exception: - print("prism coordinator: qbit tip/template poll failed", flush=True) - traceback.print_exc() - if runtime.template_refresh_failure_expired(time.monotonic()): - print( - "prism coordinator: template refresh failure budget exhausted; " - "exiting non-zero so the restart policy recovers the process", - flush=True, - ) - os._exit(1) - - def template_refresh_failure_expired(self, now: float) -> bool: - runtime = self._runtime - budget = float( - getattr( - runtime, - "template_refresh_failure_exit_seconds", - DEFAULT_PRISM_TEMPLATE_MAX_AGE_SECONDS, - ) - ) - if budget <= 0: - return False - failure_started = getattr(self, "template_refresh_failure_started_monotonic", None) - return failure_started is not None and now - failure_started >= budget - - def _record_template_refresh_failure(self, now: float) -> None: - runtime = self._runtime - budget = float( - getattr( - runtime, - "template_refresh_failure_exit_seconds", - DEFAULT_PRISM_TEMPLATE_MAX_AGE_SECONDS, - ) - ) - if ( - budget > 0 - and getattr(self, "template_refresh_failure_started_monotonic", None) is None - ): - self.template_refresh_failure_started_monotonic = now - - def blockwait_once(self, known_tip: str) -> str: - """One waitfornewblock round: returns the tip after the wait. - - qbitd returns as soon as its tip differs from ``known_tip`` (or after - the server-side timeout, echoing the current tip), so a tip observed - between our last poll and this call is reported immediately rather - than being missed for a cycle. - """ - runtime = self._runtime - timeout_seconds = getattr( - runtime, - "blockwait_timeout_seconds", - DEFAULT_PRISM_BLOCKWAIT_TIMEOUT_SECONDS, - ) - watchdog_timeout = float(getattr(runtime, "watchdog_timeout_seconds", 120.0)) - max_rpc_timeout = max(1.0, watchdog_timeout * 0.8) - timeout_seconds = min(float(timeout_seconds), max(1.0, max_rpc_timeout - 1.0)) - result = runtime.rpc.call( - "waitfornewblock", - [max(1, int(timeout_seconds * 1000)), known_tip], - timeout=timeout_seconds + 10.0, - ) - if isinstance(result, dict): - new_tip = str(result.get("hash", "") or "") - if new_tip: - return new_tip - return known_tip - - def blockwait_loop(self) -> None: - """Push-style tip detection alongside the interval poller. - - Stale rejects are dominated by the window between a block connecting - and miners receiving fresh work; the poller alone leaves up to a full - PRISM_BLOCKPOLL_SECONDS of that window. This loop parks inside - waitfornewblock and triggers the same refresh path within milliseconds - of a new tip. The poller stays on as the fallback and still owns - same-tip template refreshes, which waitfornewblock does not signal. - Disabled cleanly when qbitd does not support the RPC. - """ - runtime = self._runtime - known_tip: str | None = None - while not runtime.stop_event.is_set(): - runtime._record_heartbeat("qbit_blockwait") - try: - if known_tip is None: - known_tip = str(runtime.rpc.call("getbestblockhash")) - runtime.observe_tip_for_refresh(known_tip) - new_tip = runtime.blockwait_once(known_tip) - if new_tip == known_tip: - if runtime.stop_event.wait(0.25): - return - continue - # Advance the wait cursor before observation/logging. If either - # operation raises, the next wait must not rediscover the same - # already-seen transition and create a notification storm. - known_tip = new_tip - # Detection can supersede/cancel obsolete heavy work, but only - # the blockpoll single-flight owner may fetch, build, and - # publish replacement work. - try: - runtime.observe_tip_for_refresh(new_tip) - finally: - # Preserve the wake even if observation-side cancellation - # or bookkeeping raises after known_tip advanced. - runtime._schedule_tip_refresh_retry() - print( - f"prism coordinator: blockwait saw new tip {new_tip}; " - "single-flight refresh scheduled", - flush=True, - ) - except Exception as exc: - if known_tip is not None and runtime._blockwait_unsupported(exc): - print( - "prism coordinator: waitfornewblock unavailable on this qbitd; " - "tip detection falls back to blockpoll only", - flush=True, - ) - runtime._remove_watchdog_heartbeat("qbit_blockwait") - return - print("prism coordinator: blockwait pass failed", flush=True) - traceback.print_exc() - if runtime.stop_event.wait(min(5.0, runtime.blockpoll_seconds)): - return - - def tip_refresh_metrics_lines(self) -> list[str]: - runtime = self._runtime - runtime._ensure_tip_refresh_state() - with self._tip_refresh_executor_lock: - executor_workers = ( - runtime.tip_refresh_max_workers - if self._tip_refresh_executor is not None - else 0 - ) - with self._tip_refresh_metrics_lock: - histograms = { - name: { - "buckets": dict(histogram["buckets"]), - "sum": float(histogram["sum"]), - "count": int(histogram["count"]), - } - for name, histogram in self.tip_refresh_histograms.items() - } - phase_histograms = { - phase: { - "buckets": dict(histogram["buckets"]), - "sum": float(histogram["sum"]), - "count": int(histogram["count"]), - } - for phase, histogram in self.tip_refresh_build_phase_histograms.items() - } - client_counts = dict(self.tip_refresh_client_counts) - cancellation_counts = dict(self.tip_refresh_cancellation_counts) - wave_outcome_counts = dict(self.tip_refresh_wave_outcome_counts) - coverage_histograms = { - target: { - "buckets": dict(histogram["buckets"]), - "sum": float(histogram["sum"]), - "count": int(histogram["count"]), - } - for target, histogram in ( - self.tip_refresh_coverage_histograms.items() - ) - } - inflight = self.tip_refresh_inflight - build_inflight = self.tip_refresh_build_inflight - build_queue_depth = self.tip_refresh_build_queue_depth - singleflight_hits = self.tip_refresh_singleflight_hits - superseded_results = self.tip_refresh_superseded_results - worker_failures = self.tip_refresh_worker_failures - worker_restarts = self.tip_refresh_worker_restarts - ipc_bytes = dict(self.tip_refresh_ipc_bytes) - - metric_names = { - "refresh": "qbit_prism_tip_refresh_seconds", - "bundle_build": "qbit_prism_tip_refresh_bundle_build_seconds", - "first_delivery": "qbit_prism_tip_refresh_first_delivery_seconds", - "last_delivery": "qbit_prism_tip_refresh_last_delivery_seconds", - "fanout_wave": "qbit_prism_tip_refresh_fanout_wave_seconds", - } - descriptions = { - "refresh": "Full qbit tip/template refresh pass wall time.", - "bundle_build": "Shared ready-pool refresh bundle preparation wall time.", - "first_delivery": "Tip observation to first successful client delivery.", - "last_delivery": "Tip observation to last successful client delivery.", - "fanout_wave": "First task submission to last successful delivery of one completed prepared fanout wave.", - } - lines: list[str] = [] - for name, metric_name in metric_names.items(): - histogram = histograms[name] - buckets = histogram["buckets"] - assert isinstance(buckets, dict) - lines.extend( - [ - f"# HELP {metric_name} {descriptions[name]}", - f"# TYPE {metric_name} histogram", - *[ - f'{metric_name}_bucket{{le="{bucket:g}"}} {int(buckets.get(bucket, 0))}' - for bucket in PRISM_TIP_REFRESH_SECONDS_BUCKETS - ], - f'{metric_name}_bucket{{le="+Inf"}} {histogram["count"]}', - f'{metric_name}_sum {float(histogram["sum"]):.6f}', - f'{metric_name}_count {histogram["count"]}', - ] - ) - phase_metric_name = "qbit_prism_tip_refresh_bundle_phase_seconds" - lines.extend( - [ - "# HELP qbit_prism_tip_refresh_bundle_phase_seconds Shared bundle-build phase wall time.", - "# TYPE qbit_prism_tip_refresh_bundle_phase_seconds histogram", - ] - ) - for phase in PRISM_TIP_REFRESH_BUILD_PHASES: - histogram = phase_histograms[phase] - buckets = histogram["buckets"] - assert isinstance(buckets, dict) - lines.extend( - [ - *[ - f'{phase_metric_name}_bucket{{phase="{phase}",le="{bucket:g}"}} {int(buckets.get(bucket, 0))}' - for bucket in PRISM_TIP_REFRESH_SECONDS_BUCKETS - ], - f'{phase_metric_name}_bucket{{phase="{phase}",le="+Inf"}} {histogram["count"]}', - f'{phase_metric_name}_sum{{phase="{phase}"}} {float(histogram["sum"]):.6f}', - f'{phase_metric_name}_count{{phase="{phase}"}} {histogram["count"]}', - ] - ) - lines.extend( - [ - "# HELP qbit_prism_tip_refresh_wave_outcomes_total Singleflight refresh waves by terminal or recovered supersession outcome.", - "# TYPE qbit_prism_tip_refresh_wave_outcomes_total counter", - *[ - f'qbit_prism_tip_refresh_wave_outcomes_total{{outcome="{outcome}"}} {int(wave_outcome_counts.get(outcome, 0))}' - for outcome in PRISM_TIP_REFRESH_WAVE_OUTCOMES - ], - "# HELP qbit_prism_tip_refresh_clients_total Client outcomes from tip/template refresh passes.", - "# TYPE qbit_prism_tip_refresh_clients_total counter", - *[ - f'qbit_prism_tip_refresh_clients_total{{result="{result}"}} {int(client_counts.get(result, 0))}' - for result in PRISM_TIP_REFRESH_RESULTS - ], - "# HELP qbit_prism_tip_refresh_cancellations_total Obsolete prepared refresh tasks canceled before delivery admission.", - "# TYPE qbit_prism_tip_refresh_cancellations_total counter", - *[ - f'qbit_prism_tip_refresh_cancellations_total{{stage="{stage}"}} {int(cancellation_counts.get(stage, 0))}' - for stage in PRISM_TIP_REFRESH_CANCELLATION_STAGES - ], - "# HELP qbit_prism_tip_refresh_inflight Prepared refresh client tasks currently queued or running.", - "# TYPE qbit_prism_tip_refresh_inflight gauge", - f"qbit_prism_tip_refresh_inflight {inflight}", - "# HELP qbit_prism_tip_refresh_executor_workers Configured persistent refresh executor workers, or zero before creation.", - "# TYPE qbit_prism_tip_refresh_executor_workers gauge", - f"qbit_prism_tip_refresh_executor_workers {executor_workers}", - "# HELP qbit_prism_tip_refresh_bundle_inflight Shared bundle builds currently running.", - "# TYPE qbit_prism_tip_refresh_bundle_inflight gauge", - f"qbit_prism_tip_refresh_bundle_inflight {build_inflight}", - "# HELP qbit_prism_tip_refresh_bundle_queue_depth Shared bundle callers waiting on bounded build admission or an identical single-flight.", - "# TYPE qbit_prism_tip_refresh_bundle_queue_depth gauge", - f"qbit_prism_tip_refresh_bundle_queue_depth {build_queue_depth}", - "# HELP qbit_prism_tip_refresh_bundle_singleflight_hits_total Shared bundle callers coalesced behind an identical build.", - "# TYPE qbit_prism_tip_refresh_bundle_singleflight_hits_total counter", - f"qbit_prism_tip_refresh_bundle_singleflight_hits_total {singleflight_hits}", - "# HELP qbit_prism_tip_refresh_bundle_superseded_results_total Completed or canceled shared bundles discarded after supersession.", - "# TYPE qbit_prism_tip_refresh_bundle_superseded_results_total counter", - f"qbit_prism_tip_refresh_bundle_superseded_results_total {superseded_results}", - "# HELP qbit_prism_tip_refresh_builder_worker_failures_total Audit-builder subprocess failures.", - "# TYPE qbit_prism_tip_refresh_builder_worker_failures_total counter", - f"qbit_prism_tip_refresh_builder_worker_failures_total {worker_failures}", - "# HELP qbit_prism_tip_refresh_builder_worker_restarts_total Long-lived builder worker restarts (persistent --serve daemon respawns after a crash or supersession).", - "# TYPE qbit_prism_tip_refresh_builder_worker_restarts_total counter", - f"qbit_prism_tip_refresh_builder_worker_restarts_total {worker_restarts}", - "# HELP qbit_prism_tip_refresh_builder_ipc_bytes_total Bytes copied across audit-builder subprocess IPC.", - "# TYPE qbit_prism_tip_refresh_builder_ipc_bytes_total counter", - *[ - f'qbit_prism_tip_refresh_builder_ipc_bytes_total{{direction="{direction}"}} {int(ipc_bytes.get(direction, 0))}' - for direction in ("input", "output") - ], - ] - ) - coverage_metric_name = "qbit_prism_tip_refresh_hashrate_coverage_seconds" - lines.extend( - [ - "# HELP qbit_prism_tip_refresh_hashrate_coverage_seconds Time from epoch detection until a fixed share of the snapshotted hashrate proxy received current work.", - "# TYPE qbit_prism_tip_refresh_hashrate_coverage_seconds histogram", - ] - ) - for target, _ratio in PRISM_TIP_REFRESH_COVERAGE_TARGETS: - histogram = coverage_histograms[target] - buckets = histogram["buckets"] - assert isinstance(buckets, dict) - lines.extend( - [ - *[ - f'{coverage_metric_name}_bucket{{coverage="{target}",le="{bucket:g}"}} {int(buckets.get(bucket, 0))}' - for bucket in PRISM_TIP_REFRESH_SECONDS_BUCKETS - ], - f'{coverage_metric_name}_bucket{{coverage="{target}",le="+Inf"}} {histogram["count"]}', - f'{coverage_metric_name}_sum{{coverage="{target}"}} {float(histogram["sum"]):.6f}', - f'{coverage_metric_name}_count{{coverage="{target}"}} {histogram["count"]}', - ] - ) - runtime._ensure_job_cache_state() - with runtime._serve_builder_metrics_lock: - serve_counts = dict(runtime.serve_builder_counts) - serve_window_counts = dict(runtime.serve_builder_window_cache_counts) - with runtime._payout_artifact_executor_lock: - payout_artifact_events = dict(runtime.payout_artifact_event_counts) - lines.extend( - [ - "# HELP qbit_prism_serve_builder_events_total Persistent audit-builder daemon lifecycle and request outcomes.", - "# TYPE qbit_prism_serve_builder_events_total counter", - *[ - f'qbit_prism_serve_builder_events_total{{event="{event}"}} {int(serve_counts.get(event, 0))}' - for event in ( - "requests", - "fallbacks", - "spawns", - "window_uploads", - "window_prepares", - "window_prepare_contended", - "window_prepare_out_of_range", - ) - ], - "# HELP qbit_prism_payout_artifact_events_total Payout-ledger-artifact lifecycle outcomes (build, install, reuse pacing).", - "# TYPE qbit_prism_payout_artifact_events_total counter", - *[ - f'qbit_prism_payout_artifact_events_total{{event="{event}"}} {int(payout_artifact_events.get(event, 0))}' - for event in ( - "built", - "build_aborted", - "debounced", - "incremental", - "full_rescan", - "self_check_match", - "self_check_mismatch", - "self_check_failed", - "balance_check_mismatch", - "found_block_cached", - "installed", - "refreshed", - "already_current", - "discarded", - "born_expired", - "rearm_scheduled", - "served_reuse", - "probe_rejected_ceiling", - "probe_past_floor", - "window_mirror_divergence", - ) - ], - "# HELP qbit_prism_serve_builder_window_cache_total Daemon parsed share-window cache outcomes.", - "# TYPE qbit_prism_serve_builder_window_cache_total counter", - *[ - f'qbit_prism_serve_builder_window_cache_total{{result="{result}"}} {int(serve_window_counts.get(counter_key, 0))}' - for result, counter_key in ( - ("hit", "hits"), - ("miss", "misses"), - ) - ], - ] - ) - return lines - - def _tip_refresh_artifacts( - self, - snapshot: QbitTipTemplateSnapshot, - ) -> CachedTemplateArtifacts: - artifacts = snapshot.template_artifacts - if ( - artifacts is None - or artifacts.fingerprint != snapshot.template_fingerprint - or artifacts.previousblockhash != snapshot.previousblockhash - or artifacts.generation != snapshot.template_generation - or snapshot.bestblockhash != snapshot.previousblockhash - or qbit_template_fingerprint(artifacts.template) != artifacts.fingerprint - or str(artifacts.template.get("previousblockhash", "")) - != artifacts.previousblockhash - ): - raise TemplateRefreshBlocked( - "tip/template snapshot does not own matching exact artifacts" - ) - return artifacts - - def prepare_tip_refresh_bundle( - self, - snapshot: QbitTipTemplateSnapshot, - *, - priority_requested_monotonic: float | None = None, - ) -> CachedJobBundle: - """Build ready-pool work from immutable shared inputs only. - - Client selection belongs exclusively to fanout. In particular, a - connection disappearing before or during this build cannot affect the - signed payout bundle or its cache lifetime. - """ - runtime = self._runtime - runtime._ensure_job_cache_state() - runtime._ensure_tip_refresh_state() - artifacts = runtime._tip_refresh_artifacts(snapshot) - build_started = time.monotonic() - progress_build_token = runtime._progress_bundle_build_started() - priority_token, priority_requested_monotonic = ( - runtime._begin_job_build_priority_preparation( - priority_requested_monotonic - ) - ) - try: - max_payout_retries = max( - 0, - int( - getattr( - runtime, - "payout_reconcile_supersession_retries", - DEFAULT_PRISM_PAYOUT_RECONCILE_SUPERSESSION_RETRIES, - ) - ), - ) - for attempt in range(max_payout_retries + 1): - with runtime._job_cache_lock: - payout_generation_before_build = ( - runtime._payout_state_generation - ) - try: - bundle = runtime.shared_job_bundle( - artifacts, - mode="ready", - retry_superseded=False, - publication_critical=True, - request_source="tip_refresh", - priority_requested_monotonic=( - priority_requested_monotonic - ), - ) - break - except JobBuildSuperseded: - if getattr(runtime, "tip_refresh_epoch_fanout", False): - raise - with runtime._job_cache_lock: - payout_generation_after_build = ( - runtime._payout_state_generation - ) - payout_publication_blocked = ( - runtime._payout_state_publication_blocked - ) - with runtime.lock: - artifacts_buildable = runtime._artifacts_buildable_locked( - artifacts - ) - if ( - attempt >= max_payout_retries - or payout_publication_blocked - or not artifacts_buildable - or payout_generation_after_build - == payout_generation_before_build - ): - raise - # Coalesce a completed payout generation into this same - # tip owner. Every abandoned attempt remains fenced by its - # immutable generation; only the latest coherent retry can - # reach final publication. - continue - else: # pragma: no cover - range always runs at least once - raise TemplateRefreshBlocked( - "payout generation did not stabilize during preparation" - ) - except TemplateRefreshBlocked: - raise - except Exception as exc: - with runtime.lock: - runtime.job_build_failure_count += 1 - raise TemplateRefreshBlocked("prepared refresh bundle build failed") from exc - finally: - runtime._finish_job_build_priority_preparation(priority_token) - runtime._progress_bundle_build_finished(progress_build_token) - runtime._observe_tip_refresh_seconds( - "bundle_build", - time.monotonic() - build_started, - ) - # Revalidate only the snapshot-owned object. A concurrent cache fill is - # unrelated to this refresh and cannot replace its exact artifacts. - artifacts = runtime._tip_refresh_artifacts(snapshot) - if ( - bundle.template is not artifacts.template - or bundle.template_fingerprint != artifacts.fingerprint - or bundle.template_generation != artifacts.generation - or str(bundle.template.get("previousblockhash", "")) - != artifacts.previousblockhash - ): - raise TemplateRefreshBlocked( - "prepared refresh bundle does not match exact template artifacts" - ) - if bundle.collection_only: - raise TemplateRefreshBlocked( - "ready-pool prepared refresh unexpectedly produced a collection bundle" - ) - return bundle - - def prewarm_current_tip_ready_bundle(self) -> CachedJobBundle | None: - """Publish one exact current-tip ready bundle before Stratum accepts.""" - runtime = self._runtime - runtime._ensure_job_cache_state() - with runtime._job_cache_lock: - runtime.job_preparation_pending = True - try: - observation_sequence = runtime._reserve_tip_observation_sequence() - snapshot = runtime.fetch_qbit_tip_template_snapshot() - runtime._record_progress_tip_poll(snapshot) - try: - reconciled = runtime.ensure_reorg_reconciled_for_tip( - snapshot.bestblockhash - ) - except Exception as exc: - raise TemplateRefreshBlocked( - "startup reorg reconciliation failed before job preparation" - ) from exc - if not reconciled: - raise TemplateRefreshBlocked( - "startup chain view remained untrusted during job preparation" - ) - - ready = runtime.pool_readiness_latched() - bundle: CachedJobBundle | None = None - if ready: - progress_build_token = runtime._progress_bundle_build_started() - try: - bundle = runtime.shared_job_bundle( - runtime._tip_refresh_artifacts(snapshot), - None, - publication_critical=True, - request_source="tip_refresh", - ) - finally: - runtime._progress_bundle_build_finished(progress_build_token) - if bundle.collection_only: - raise TemplateRefreshBlocked( - "startup ready preparation produced collection work" - ) - if bundle.payout_state_generation != int( - getattr(runtime, "_payout_state_generation", 0) - ): - raise TemplateRefreshSuperseded( - "payout state changed during startup job preparation" - ) - - if str(runtime.rpc.call("getbestblockhash")) != snapshot.bestblockhash: - raise TemplateRefreshSuperseded( - "qbit tip changed during startup job preparation" - ) - if not runtime.observe_tip_first_seen( - snapshot.bestblockhash, - observation_sequence=observation_sequence, - publish_refresh_observation=True, - published_snapshot=snapshot, - ): - raise TemplateRefreshSuperseded( - "startup job preparation was superseded before publication" - ) - with runtime._job_cache_lock: - runtime._prepared_ready_snapshot = snapshot if bundle is not None else None - runtime._prepared_ready_bundle = bundle - runtime._record_progress_publication( - snapshot, - ( - bundle.payout_state_generation - if bundle is not None - else int(getattr(runtime, "_payout_state_generation", 0)) - ), - ) - self.last_successful_template_refresh_monotonic = time.monotonic() - runtime._record_progress_tip_poll(snapshot) - return bundle - finally: - with runtime._job_cache_lock: - runtime.job_preparation_pending = False - - def prewarm_startup_jobs(self) -> CachedJobBundle | None: - """Best-effort startup prewarm; transient blocking defers to blockpoll.""" - runtime = self._runtime - try: - return runtime.prewarm_current_tip_ready_bundle() - except TemplateRefreshBlocked as exc: - # Startup prewarming is an optimization. A transient reconciliation, - # payout-generation, or tip race must not prevent Stratum listeners - # from opening; blockpoll and the bounded initial-job queue retry it. - runtime._schedule_tip_refresh_retry() - print( - "prism coordinator: startup job preparation deferred " - f"reason={exc}", - flush=True, - ) - return None - - def _tip_refresh_token_current_locked( - self, - token: TipRefreshValidationToken, - bundle: CachedJobBundle, - snapshot: QbitTipTemplateSnapshot, - ) -> bool: - runtime = self._runtime - return bool( - runtime._tip_refresh_token_prepublication_current_locked( - token, - bundle, - snapshot, - ) - and runtime._tip_refresh_snapshot_current_locked( - snapshot, - token.observation_sequence, - ) - ) - - def _tip_refresh_token_prepublication_current_locked( - self, - token: TipRefreshValidationToken, - bundle: CachedJobBundle, - snapshot: QbitTipTemplateSnapshot, - ) -> bool: - runtime = self._runtime - published = getattr(runtime, "_published_payout_state", None) - return bool( - token.snapshot is snapshot - and token.tip_hash == snapshot.bestblockhash - and token.template_fingerprint == snapshot.template_fingerprint - and token.template_generation == snapshot.template_generation - and bundle.template_fingerprint == token.template_fingerprint - and bundle.template_generation == token.template_generation - and bundle.payout_state_generation == token.payout_state_generation - and bundle.build_key is token.build_key - and token.build_key.best_tip_hash == snapshot.bestblockhash - and token.build_key.previous_block_hash == snapshot.previousblockhash - and token.build_key.template_fingerprint - == snapshot.template_fingerprint - and token.build_key.template_generation == snapshot.template_generation - and token.build_key.payout_state_generation - == token.payout_state_generation - and token.build_key.payout_append_invalidation_epoch - == int( - getattr( - runtime, - "_payout_ledger_append_invalidation_epoch", - 0, - ) - ) - and token.payout_state_generation - == int(getattr(runtime, "_payout_state_generation", 0)) - and published is not None - and published.artifact is not None - and token.build_key.payout_artifact_sha256 - == published.artifact.prior_balances_sha256 - and snapshot.template_artifacts is not None - and bundle.template is snapshot.template_artifacts.template - and not runtime._detected_tip_supersedes_locked( - snapshot.bestblockhash, - token.observation_sequence, - ) - and ( - not getattr(runtime, "tip_refresh_epoch_fanout", False) - or ( - token.epoch_sequence - == int(getattr(self, "_tip_refresh_epoch_sequence", 0)) - and token.tip_hash - == getattr(self, "_tip_refresh_epoch_tip_hash", None) - and token.payout_state_generation - == int( - getattr( - self, - "_tip_refresh_epoch_payout_generation", - 0, - ) - ) - ) - ) - ) - - def _tip_refresh_snapshot_current_locked( - self, - snapshot: QbitTipTemplateSnapshot, - observation_sequence: int, - ) -> bool: - runtime = self._runtime - current_tip = getattr(self, "current_tip_first_seen", None) - return bool( - self.tip_template_snapshot is snapshot - and current_tip is not None - and current_tip[0] == snapshot.bestblockhash - and int(getattr(self, "current_tip_observation_sequence", 0)) - == observation_sequence - and not runtime._detected_tip_supersedes_locked( - snapshot.bestblockhash, - observation_sequence, - ) - ) - - def _validate_prepared_tip_refresh( - self, - bundle: CachedJobBundle, - snapshot: QbitTipTemplateSnapshot, - observation_sequence: int, - ) -> TipRefreshValidationToken: - """Validate prepared work before publishing submit authority.""" - runtime = self._runtime - artifacts = runtime._tip_refresh_artifacts(snapshot) - if ( - bundle.template is not artifacts.template - or bundle.template_fingerprint != artifacts.fingerprint - or bundle.template_generation != artifacts.generation - or bundle.build_key is None - or bundle.build_key.best_tip_hash != snapshot.bestblockhash - or bundle.build_key.previous_block_hash != snapshot.previousblockhash - or bundle.build_key.template_fingerprint != artifacts.fingerprint - or bundle.build_key.template_generation != artifacts.generation - or bundle.build_key.mode != "ready" - ): - raise TemplateRefreshBlocked( - "prepared refresh bundle changed before final validation" - ) - try: - current_tip = str(runtime.rpc.call("getbestblockhash")) - except Exception as exc: - runtime._schedule_tip_refresh_retry() - raise TemplateRefreshBlocked( - "qbit tip validation failed before prepared fanout" - ) from exc - if current_tip != snapshot.bestblockhash: - runtime._schedule_tip_refresh_retry() - raise TemplateRefreshSuperseded( - "qbit tip changed before prepared fanout " - f"expected={snapshot.bestblockhash} current={current_tip}" - ) - try: - chain_view_untrusted = bool( - getattr(runtime, "reorg_reconciler_enabled", True) - and runtime.qbit_chain_view_untrusted() - ) - except Exception as exc: - runtime._schedule_tip_refresh_retry() - raise _TipRefreshTrustBlocked( - "qbit chain trust check failed before prepared fanout" - ) from exc - if chain_view_untrusted: - runtime._schedule_tip_refresh_retry() - raise _TipRefreshTrustBlocked( - "qbit chain view became untrusted before prepared fanout" - ) - with runtime.lock: - epoch_sequence = ( - int(getattr(self, "_tip_refresh_epoch_sequence", 0)) - if getattr(runtime, "tip_refresh_epoch_fanout", False) - else 0 - ) - token = TipRefreshValidationToken( - tip_hash=snapshot.bestblockhash, - template_fingerprint=artifacts.fingerprint, - template_generation=artifacts.generation, - payout_state_generation=bundle.payout_state_generation, - observation_sequence=observation_sequence, - epoch_sequence=epoch_sequence, - build_key=bundle.build_key, - snapshot=snapshot, - ) - if not runtime._tip_refresh_token_prepublication_current_locked( - token, - bundle, - snapshot, - ): - runtime._schedule_tip_refresh_retry() - raise TemplateRefreshSuperseded( - "prepared refresh was superseded before tip publication" - ) - return token - - def _activate_tip_refresh( - self, - token: TipRefreshValidationToken, - bundle: CachedJobBundle, - snapshot: QbitTipTemplateSnapshot, - cancel_event: FanoutCancellation, - ) -> None: - runtime = self._runtime - with runtime.lock: - if not runtime._tip_refresh_token_current_locked(token, bundle, snapshot): - runtime._schedule_tip_refresh_retry() - raise TemplateRefreshSuperseded( - "prepared refresh was superseded before cancellation registration" - ) - active = self._active_tip_refresh - if active is not None: - active[1].cancel() - self._active_tip_refresh = (token, cancel_event) - - def _publish_prepared_tip_refresh( - self, - token: TipRefreshValidationToken, - bundle: CachedJobBundle, - snapshot: QbitTipTemplateSnapshot, - *, - parent_hash: str | None, - ) -> FanoutCancellation: - """Atomically publish prepared work and register its cancellation token.""" - runtime = self._runtime - now = time.monotonic() - cancel_event = FanoutCancellation() - # Admit a synchronization-only reader of this payout generation. Do - # not mark it delivered: the gate's priority reservation belongs to - # the first current-tip socket delivery, not this publication fence. - with runtime._payout_state_delivery_gate.delivery_cancelable( - lambda: False, - generation=token.payout_state_generation, - priority=True, - ) as payout_admitted: - if not payout_admitted: - runtime._schedule_tip_refresh_retry() - raise TemplateRefreshSuperseded( - "prepared refresh was superseded before atomic publication" - ) - # Match payout publication's cache -> coordinator lock order. A - # payout fence can close admission after this reader entered, so - # recheck its blocked marker inside the same atomic section. - with runtime._job_cache_lock: - with runtime.lock: - current_sequence = int( - getattr(self, "current_tip_observation_sequence", 0) - ) - if ( - runtime._payout_state_publication_blocked - or current_sequence > token.observation_sequence - or not runtime._tip_refresh_token_prepublication_current_locked( - token, - bundle, - snapshot, - ) - ): - runtime._schedule_tip_refresh_retry() - raise TemplateRefreshSuperseded( - "prepared refresh was superseded before atomic publication" - ) - - first_seen = getattr(self, "current_tip_first_seen", None) - tip_changed = ( - first_seen is not None and first_seen[0] != token.tip_hash - ) - flip_stamp = ( - now - if tip_changed - else first_seen[1] - if first_seen is not None - else None - ) - self.current_tip_first_seen = ( - token.tip_hash, - flip_stamp, - ) - self.current_tip_observation_sequence = token.observation_sequence - self.current_tip_observed_monotonic = now - if parent_hash is not None: - self.current_tip_parent = (token.tip_hash, parent_hash) - else: - # The parent lookup is best-effort cleanup metadata. A - # transient RPC failure during a same-tip republication - # must not wipe the still-valid cached parent; only a - # parent belonging to a different tip is stale here. - prior_parent = getattr(self, "current_tip_parent", None) - if prior_parent is None or prior_parent[0] != token.tip_hash: - self.current_tip_parent = None - self.tip_template_snapshot = snapshot - runtime._record_startup_phase_once("first_tip_template_published") - self.tip_refresh_divergence_started_monotonic = None - if tip_changed: - # Retained collection work, the retained ready bundle, - # and graveyard classification all belong to the - # previously published tip. - self._retained_collection_refresh = None - runtime._prepared_ready_bundle = None - runtime._prepared_ready_snapshot = None - runtime.prune_evicted_job_graveyard(now=now, force=True) - - if not runtime._tip_refresh_token_current_locked( - token, - bundle, - snapshot, - ): - # The payout admission and both coordinator locks - # stabilize every field used by this predicate. - raise TemplateRefreshBlocked( - "prepared refresh publication did not produce a current token" - ) - runtime._publish_tip_refresh_epoch_identity_locked(snapshot) - active = self._active_tip_refresh - if active is not None: - active[1].cancel() - self._active_tip_refresh = (token, cancel_event) - return cancel_event - - def _clear_active_tip_refresh( - self, - token: TipRefreshValidationToken, - cancel_event: FanoutCancellation, - ) -> None: - runtime = self._runtime - with runtime.lock: - active = self._active_tip_refresh - if active is not None and active[0] is token and active[1] is cancel_event: - self._active_tip_refresh = None - - def _prepared_tip_refresh_obsolete( - self, - validation_token: TipRefreshValidationToken, - bundle: CachedJobBundle, - snapshot: QbitTipTemplateSnapshot, - cancel_event: FanoutCancellation | None, - ) -> bool: - runtime = self._runtime - if runtime.stop_event.is_set() or ( - cancel_event is not None and cancel_event.is_set() - ): - return True - with runtime.lock: - current = runtime._tip_refresh_token_current_locked( - validation_token, - bundle, - snapshot, - ) - if not current and cancel_event is not None: - cancel_event.cancel() - return not current - - def _fanout_prepared_tip_refresh( - self, - clients: list[ClientState], - bundle: CachedJobBundle, - snapshot: QbitTipTemplateSnapshot, - *, - observation_sequence: int | None = None, - validation_token: TipRefreshValidationToken | None = None, - preactivated_cancel_event: FanoutCancellation | None = None, - executor: ThreadPoolExecutor | None = None, - expected_active_jobs: dict[ClientState, PrismJobContext | None] | None = None, - heartbeat_name: str, - ) -> tuple[int, float | None, float | None, int]: - runtime = self._runtime - executor = executor or runtime.tip_refresh_executor() - cancel_event = preactivated_cancel_event or FanoutCancellation() - if observation_sequence is None: - with runtime.lock: - observation_sequence = int( - getattr(self, "current_tip_observation_sequence", 0) - ) - if validation_token is None: - validation_token = runtime._validate_prepared_tip_refresh( - bundle, - snapshot, - observation_sequence, - ) - if preactivated_cancel_event is None: - runtime._activate_tip_refresh( - validation_token, - bundle, - snapshot, - cancel_event, - ) - futures: dict[Future[RefreshResult], ClientState] = {} - submitted_at: dict[Future[RefreshResult], float] = {} - queued_cancellations: set[Future[RefreshResult]] = set() - with runtime.lock: - if expected_active_jobs is None: - expected_active_jobs = { - client: client.active_job - for client in clients - } - # Vardiff drives every client toward the same share interval, so - # the difficulty a client sustains is proportional to its hashrate. - # Snapshot the proxies in the same locked pass as the job snapshot. - hashrate_proxies = { - client: ( - client.vardiff_difficulty_estimate or client.share_difficulty - ) - for client in clients - } - # Bounded submission admits at most max_inflight tasks at a time, so - # queue priority alone cannot lift one client over a fleet wave that - # has not been submitted yet. Order admission itself: stale-job - # clients by descending hashrate, job-less clients last. A stale fast - # client burns its full rate on old-tip work every second of refresh - # lag, while a job-less client burns nothing while it waits regardless - # of its configured difficulty (it also keeps its initial-delivery - # queue priority once admitted). - clients = sorted( - clients, - key=lambda ordered: ( - expected_active_jobs.get(ordered) is not None, - hashrate_proxies.get(ordered, Decimal(0)), - ), - reverse=True, - ) - clients_iter = iter(clients) - max_inflight = max(1, int(runtime.tip_refresh_max_workers)) - - def record_queued_cancellation(future: Future[RefreshResult]) -> None: - if future in queued_cancellations: - return - queued_cancellations.add(future) - elapsed = max(0.0, time.monotonic() - submitted_at[future]) - runtime.observe_job_build_elapsed(elapsed, {"executor_queue": elapsed}) - runtime._record_tip_refresh_cancellation("executor_queue") - - def cancel_pending_futures(pending: set[Future[RefreshResult]]) -> None: - cancel_event.cancel() - for future in tuple(pending): - if future.cancel(): - record_queued_cancellation(future) - runtime._record_tip_refresh_client_result("skipped") - pending.discard(future) - - def submit_available(pending: set[Future[RefreshResult]]) -> None: - while ( - len(pending) < max_inflight - and not runtime.stop_event.is_set() - and not cancel_event.is_set() - ): - with runtime.lock: - token_current = runtime._tip_refresh_token_current_locked( - validation_token, - bundle, - snapshot, - ) - if not token_current: - cancel_event.cancel() - return - try: - client = next(clients_iter) - except StopIteration: - return - submitted = time.monotonic() - active_job = expected_active_jobs.get(client) - if active_job is None: - priority = self._delivery_priority_initial - elif runtime.client_tip_changed_for_snapshot(client, snapshot): - priority = self._delivery_priority_new_tip - else: - priority = self._delivery_priority_same_tip - future = runtime._submit_delivery_task( - executor, - runtime.send_prepared_job, - client, - bundle, - snapshot, - validation_token, - client.connection_id, - expected_active_jobs.get(client), - cancel_event, - submitted, - priority=priority, - ) - runtime._tip_refresh_future_started() - future.add_done_callback(runtime._tip_refresh_future_finished) - futures[future] = client - submitted_at[future] = submitted - pending.add(future) - - pending: set[Future[RefreshResult]] = set() - try: - sent = 0 - failed = 0 - first_delivery: float | None = None - last_delivery: float | None = None - invalidation: TemplateRefreshBlocked | None = None - last_live_trust_check = time.monotonic() - try: - submit_available(pending) - except RuntimeError: - cancel_pending_futures(pending) - cancel_event.set() - if pending: - wait(pending) - if not runtime.stop_event.is_set(): - runtime._schedule_tip_refresh_retry() - raise - while pending: - runtime._record_heartbeat(heartbeat_name) - if runtime.stop_event.is_set() or cancel_event.is_set(): - cancel_pending_futures(pending) - done, pending = wait( - pending, - timeout=PRISM_TIP_REFRESH_ADMISSION_POLL_SECONDS, - return_when=FIRST_COMPLETED, - ) - for future in done: - client = futures[future] - if future.cancelled(): - if future not in queued_cancellations: - record_queued_cancellation(future) - runtime._record_tip_refresh_client_result("skipped") - continue - try: - result = future.result() - except OSError: - runtime._record_tip_refresh_client_result("disconnected") - runtime.disconnect_client(client) - continue - except TemplateRefreshBlocked as exc: - runtime._record_tip_refresh_client_result("skipped") - invalidation = exc - cancel_pending_futures(pending) - continue - except Exception: - failed += 1 - runtime._record_tip_refresh_client_result("failed") - with runtime.lock: - runtime.job_build_failure_count += 1 - print( - "prism coordinator: prepared job fanout failed " - f"connection={client.connection_id} username={client.username}", - flush=True, - ) - traceback.print_exc() - continue - runtime._record_tip_refresh_client_result(result.result) - if result.result == "sent": - sent += 1 - delivered = result.delivered_monotonic - if delivered is not None: - first_delivery = ( - delivered - if first_delivery is None - else min(first_delivery, delivered) - ) - last_delivery = ( - delivered - if last_delivery is None - else max(last_delivery, delivered) - ) - if ( - pending - and invalidation is None - and not runtime.stop_event.is_set() - and time.monotonic() - last_live_trust_check >= 1.0 - ): - # Validation tokens keep queued per-client deliveries - # RPC-free, but they cannot observe headers advancing - # ahead of blocks while the best-block hash stays fixed. - # Recheck the live chain view from the fanout driver about - # once per second and cancel every delivery still queued - # if the view becomes untrusted. - try: - trusted = runtime.ensure_reorg_reconciled_for_current_tip( - expected_tip_hash=snapshot.bestblockhash, - ) - if not trusted: - raise _TipRefreshTrustBlocked( - "qbit chain view became untrusted during prepared fanout" - ) - last_live_trust_check = time.monotonic() - except self._shutdown_error: - # Admission can close after the stop-event check above - # but before reconciliation enters its writer scope. - # Preserve the intentional shutdown signal so the - # poller cannot consume its template-failure budget. - cancel_pending_futures(pending) - raise - except TemplateRefreshBlocked as exc: - invalidation = exc - except Exception as exc: - invalidation = _TipRefreshTrustBlocked( - "qbit chain trust check failed during prepared fanout" - ) - invalidation.__cause__ = exc - if invalidation is not None: - cancel_pending_futures(pending) - if invalidation is None: - try: - submit_available(pending) - except RuntimeError: - cancel_pending_futures(pending) - cancel_event.set() - if pending: - wait(pending) - if not runtime.stop_event.is_set(): - runtime._schedule_tip_refresh_retry() - raise - if invalidation is not None: - cancel_event.set() - runtime._schedule_tip_refresh_retry() - raise invalidation - with runtime.lock: - token_current = runtime._tip_refresh_token_current_locked( - validation_token, - bundle, - snapshot, - ) - if not token_current: - runtime._schedule_tip_refresh_retry() - raise _TipRefreshFanoutSuperseded( - "prepared refresh was superseded during fanout; immediate retry scheduled" - ) - try: - post_fanout_tip = str(runtime.rpc.call("getbestblockhash")) - except Exception as exc: - runtime._schedule_tip_refresh_retry() - raise TemplateRefreshBlocked( - "qbit tip validation failed after prepared fanout; " - "immediate retry scheduled" - ) from exc - if post_fanout_tip != snapshot.bestblockhash: - cancel_event.set() - runtime._schedule_tip_refresh_retry() - raise _TipRefreshFanoutSuperseded( - "qbit tip changed during prepared fanout; immediate retry scheduled " - f"expected={snapshot.bestblockhash} current={post_fanout_tip}" - ) - try: - post_fanout_untrusted = bool( - getattr(runtime, "reorg_reconciler_enabled", True) - and runtime.qbit_chain_view_untrusted() - ) - except Exception as exc: - cancel_event.set() - runtime._schedule_tip_refresh_retry() - raise _TipRefreshTrustBlocked( - "qbit chain trust check failed after prepared fanout; " - "immediate retry scheduled" - ) from exc - if post_fanout_untrusted: - cancel_event.set() - runtime._schedule_tip_refresh_retry() - raise _TipRefreshTrustBlocked( - "qbit chain view became untrusted during prepared fanout; " - "immediate retry scheduled" - ) - with runtime.lock: - token_current = runtime._tip_refresh_token_current_locked( - validation_token, - bundle, - snapshot, - ) - if not token_current: - cancel_event.set() - runtime._schedule_tip_refresh_retry() - raise _TipRefreshFanoutSuperseded( - "prepared refresh payout state changed during post-fanout " - "validation; immediate retry scheduled" - ) - if last_delivery is not None and submitted_at: - # Wall-clock span of the wave itself (first task submission - # to last successful delivery), independent of the reconcile - # and bundle-build stages the first/last_delivery histograms - # include. At fleet scale this is the dominant staleness term - # bounded by the delivery worker pool. - runtime._observe_tip_refresh_seconds( - "fanout_wave", - max(0.0, last_delivery - min(submitted_at.values())), - ) - return sent, first_delivery, last_delivery, failed - finally: - runtime._clear_active_tip_refresh(validation_token, cancel_event) - - def poll_qbit_tip_template_once( - self, - *, - heartbeat_name: str = "qbit_blockpoll", - ) -> int: - """Drive one latest-wins refresh wave as its singleflight owner.""" - runtime = self._runtime - runtime._ensure_tip_refresh_state() - runtime._ensure_job_cache_state() - refresh_started = time.monotonic() - observed_best_tip: str | None = None - try: - observation_sequence = runtime._reserve_tip_observation_sequence() - observed_best_tip = str(runtime.rpc.call("getbestblockhash")) - if not runtime.observe_tip_for_refresh( - observed_best_tip, - observation_sequence=observation_sequence, - mark_pending=False, - ): - runtime._schedule_tip_refresh_retry() - raise TemplateRefreshSuperseded( - "tip/template poll was superseded before template fetch" - ) - except TemplateRefreshSuperseded: - if not getattr(runtime, "tip_refresh_epoch_fanout", False): - runtime._record_coordination_blocked_refresh(time.monotonic()) - runtime._note_tip_refresh_attempt_failed(observed_best_tip) - raise - except Exception: - runtime._clear_coordination_blocked_streak() - runtime._record_template_refresh_failure(time.monotonic()) - runtime._note_tip_refresh_attempt_failed(observed_best_tip) - raise - assert observed_best_tip is not None - if not self._tip_refresh_singleflight_lock.acquire(blocking=False): - # A losing observer still publishes newer detection to cancellation - # state, but it does not own (or count) a refresh wave. - runtime.observe_tip_for_refresh( - observed_best_tip, - observation_sequence=observation_sequence, - mark_pending=True, - ) - runtime._schedule_tip_refresh_retry() - return 0 - - total_refreshed = 0 - saw_build_supersession = False - saw_fanout_supersession = False - outcome = "error" - first_pass = True - completed_passes = 0 - try: - while True: - try: - total_refreshed += runtime._poll_qbit_tip_template_pass_once( - heartbeat_name=heartbeat_name, - refresh_started=refresh_started, - observation_sequence=( - observation_sequence if first_pass else None - ), - observed_best_tip=( - observed_best_tip if first_pass else None - ), - ) - first_pass = False - completed_passes += 1 - except PayoutStatePublicationBlocked: - outcome = "payout_blocked" - raise - except _TipRefreshTrustBlocked: - outcome = "trust_blocked" - raise - except JobBuildSuperseded: - saw_build_supersession = True - first_pass = False - completed_passes += 1 - if runtime._tip_refresh_wave_reenters(completed_passes): - continue - outcome = runtime._interrupted_wave_outcome( - "build_superseded" - ) - raise - except TemplateRefreshSuperseded: - saw_fanout_supersession = True - first_pass = False - completed_passes += 1 - if runtime._tip_refresh_wave_reenters(completed_passes): - continue - outcome = runtime._interrupted_wave_outcome( - "fanout_superseded" - ) - raise - except Exception: - outcome = "error" - raise - - if ( - getattr(runtime, "tip_refresh_epoch_fanout", False) - and not runtime._tip_refresh_epoch_fixpoint_reached() - ): - saw_fanout_supersession = True - if runtime._tip_refresh_wave_reenters(completed_passes): - continue - outcome = runtime._interrupted_wave_outcome( - "fanout_superseded" - ) - return total_refreshed - outcome = ( - "build_superseded" - if saw_build_supersession - else "fanout_superseded" - if saw_fanout_supersession - else "completed" - ) - return total_refreshed - finally: - try: - runtime._record_tip_refresh_wave_outcome(outcome) - wave_elapsed = time.monotonic() - refresh_started - runtime._observe_tip_refresh_seconds( - "refresh", - wave_elapsed, - ) - if wave_elapsed > 1.0: - # One line per slow wave (~1/min at current cadence). - # Reconstructing wave latency by pairing blockwait lines - # with "refreshed N client job(s)" lines cannot - # attribute time or see superseded passes; this can. - print( - "prism coordinator: tip refresh wave " - f"outcome={outcome} refreshed={total_refreshed} " - f"passes={completed_passes} " - f"elapsed={wave_elapsed:.3f}s", - flush=True, - ) - finally: - self._tip_refresh_singleflight_lock.release() - - def _interrupted_wave_outcome(self, superseded_outcome: str) -> str: - """Classify a wave that declined re-entry. - - With epoch fanout active, a wave that would have re-entered but for - shutdown was terminated by shutdown, not by the supersession it - recovered; reporting it superseded would pollute the terminal - supersession counts that rollout evaluation watches. Legacy waves - never re-enter, so their supersession outcome stands. - """ - runtime = self._runtime - if ( - getattr(runtime, "tip_refresh_epoch_fanout", False) - and runtime.stop_event.is_set() - ): - return "shutdown" - return superseded_outcome - - def _tip_refresh_wave_reenters(self, completed_passes: int) -> bool: - """Gate owner-wave re-entry so tip churn cannot spin at RPC speed. - - Every re-entry pass costs template and reconcile RPC work against - qbitd exactly when block cadence is fastest. The inter-pass backoff - bounds that poll rate, and a wave that cannot converge within its - pass budget hands the remainder to the scheduled retry, whose fresh - wave restarts from the newest observed state. - """ - runtime = self._runtime - if not getattr(runtime, "tip_refresh_epoch_fanout", False): - return False - if runtime.stop_event.is_set(): - return False - if completed_passes >= PRISM_TIP_REFRESH_WAVE_PASS_BUDGET: - runtime._schedule_tip_refresh_retry() - return False - return not runtime.stop_event.wait( - PRISM_TIP_REFRESH_REENTRY_BACKOFF_SECONDS - ) - - def _poll_qbit_tip_template_pass_once( - self, - *, - heartbeat_name: str, - refresh_started: float, - observation_sequence: int | None = None, - observed_best_tip: str | None = None, - ) -> int: - runtime = self._runtime - runtime._ensure_tip_refresh_state() - runtime._ensure_job_cache_state() - publication_lock_acquired = False - progress_refresh_active = False - pending_signal_token: int | None = None - try: - observation_prevalidated = ( - observation_sequence is not None - and observed_best_tip is not None - ) - if observation_sequence is None: - observation_sequence = runtime._reserve_tip_observation_sequence() - # The interval poller has no push notification to mark priority for - # it. Probe the cheap best-tip RPC before fetching and deriving the - # template so CTV maintenance can yield as soon as a changed tip is - # observed, rather than after reconciliation or bundle preparation. - if observed_best_tip is None: - observed_best_tip = str(runtime.rpc.call("getbestblockhash")) - if ( - not observation_prevalidated - and not runtime.observe_tip_for_refresh( - observed_best_tip, - observation_sequence=observation_sequence, - mark_pending=False, - ) - ): - runtime._schedule_tip_refresh_retry() - raise TemplateRefreshSuperseded( - "tip/template poll was superseded before template fetch" - ) - pending_signal_token = runtime._claim_tip_refresh_pending() - with runtime.lock: - poll_start_clients = tuple( - client - for client in runtime.clients - if runtime.client_can_receive_jobs(client) - ) - with runtime.lock: - current_tip = getattr(self, "current_tip_first_seen", None) - if current_tip is not None and current_tip[0] != observed_best_tip: - pending_signal_token = runtime._mark_tip_refresh_pending_for_poll( - pending_signal_token, - observation_sequence, - ) - # Captured before reconciliation can begin: the pass may run on - # the prefetch worker while this thread fetches the template, so - # the mutation bracket must open before either starts. - payout_generation_before_reconciliation = int( - getattr(runtime, "_payout_state_generation", 0) - ) - # Reconciliation is keyed by tip hash and the probe above already - # observed the best hash, so a memo miss can run its full pass on - # the prefetch worker while this thread fetches and derives the - # template. Whether the pass can be skipped is decided at the - # join below against the memo's state AT THAT TIME -- a tip that - # flips away and back during the fetch evicts the entry so - # post-flip pool-block state is re-proven -- and publication - # still re-proves chain trust in _validate_prepared_tip_refresh - # before any fanout. With the memo disabled (TTL 0) there is no - # way to validate an overlapped pass across the fetch window, so - # the poll keeps its historical serial pass. - reconciler_enabled = bool( - getattr(runtime, "reorg_reconciler_enabled", True) - ) - reconcile_memo_enabled = reconciler_enabled and ( - float( - getattr( - runtime, - "reorg_reconcile_cache_seconds", - DEFAULT_PRISM_REORG_RECONCILE_CACHE_SECONDS, - ) - ) - > 0 - ) - reconcile_prefetch: Future[bool] | None = None - reconcile_probe_started = time.monotonic() - reconcile_detection_epoch = int( - getattr(self, "tip_detection_epoch", 0) - ) - if reconcile_memo_enabled and not runtime._reorg_reconcile_memo_fresh( - observed_best_tip - ): - reconcile_prefetch = runtime._submit_reconcile_prefetch( - observed_best_tip - ) - reconcile_probe_seconds = time.monotonic() - reconcile_probe_started - snapshot = self._reuse_current_tip_template_snapshot(observed_best_tip) - if snapshot is None: - runtime._record_job_cache_event("template", hit=False) - snapshot = runtime.fetch_qbit_tip_template_snapshot() - if not runtime.observe_tip_for_refresh( - snapshot.bestblockhash, - observation_sequence=observation_sequence, - mark_pending=False, - ): - runtime._schedule_tip_refresh_retry() - raise TemplateRefreshSuperseded( - "tip/template poll was superseded during template fetch" - ) - with runtime.lock: - published_after_fetch = getattr( - self, - "current_tip_first_seen", - None, - ) - if ( - published_after_fetch is not None - and published_after_fetch[0] != snapshot.bestblockhash - ): - pending_signal_token = runtime._mark_tip_refresh_pending_for_poll( - pending_signal_token, - observation_sequence, - ) - runtime._record_progress_tip_poll(snapshot) - runtime._progress_refresh_started() - progress_refresh_active = True - runtime.pool_readiness_latched() - with runtime.lock: - previous_snapshot = self.tip_template_snapshot - # Generation orders concurrent observations but is not itself - # a template change. Repeated observations of identical work - # must not trigger a clean fanout on every poll. - snapshot_changed = previous_snapshot is not None and ( - previous_snapshot.bestblockhash != snapshot.bestblockhash - or previous_snapshot.previousblockhash != snapshot.previousblockhash - or previous_snapshot.template_fingerprint - != snapshot.template_fingerprint - ) - if snapshot_changed: - clients = [ - client - for client in runtime.clients - if runtime.client_can_receive_jobs(client) - ] - else: - clients = [ - client - for client in runtime.clients - if runtime.client_can_receive_jobs(client) - and runtime.client_needs_tip_template_refresh(client, snapshot) - ] - # Capture the exact job each client had when this refresh pass - # selected it. A Vardiff/authorize path may install intervening - # work while the shared bundle is prepared or while its task - # waits in the executor queue. Artifact generations let the - # task replace stale intervening work while preserving work - # produced from a template stored after this snapshot. - expected_active_jobs = { - client: client.active_job - for client in clients - } - - if clients and snapshot_changed: - pending_signal_token = runtime._mark_tip_refresh_pending_for_poll( - pending_signal_token, - observation_sequence, - ) - - refreshed = 0 - build_failures = 0 - first_delivery: float | None = None - last_delivery: float | None = None - runtime._raise_if_tip_refresh_superseded( - snapshot, - observation_sequence, - ) - reconcile_source = "serial" - reconcile_join_started = time.monotonic() - try: - if ( - reconcile_memo_enabled - and int(getattr(self, "tip_detection_epoch", 0)) - == reconcile_detection_epoch - and runtime._reorg_reconcile_memo_fresh( - snapshot.bestblockhash - ) - ): - # Fresh at join time AND no detection interleaved the - # fetch. Both are required: a flip away and back evicts - # the entry, but a pass whose execution straddled the - # flip can re-arm it afterwards (the latest detected - # hash matches again), and its proof belongs to the - # closed epoch. Any epoch movement lands in the serial - # re-prove branches below. - reconcile_source = ( - "overlap" if reconcile_prefetch is not None else "memo_hit" - ) - if reconcile_prefetch is not None: - runtime._discard_stale_reconcile_prefetch( - reconcile_prefetch - ) - reconcile_prefetch = None - reorg_reconciled = True - elif ( - reconcile_prefetch is not None - and snapshot.bestblockhash == observed_best_tip - ): - reconcile_source = "overlap" - reorg_reconciled = runtime._join_reconcile_prefetch_bounded( - reconcile_prefetch - ) - if reorg_reconciled and ( - int(getattr(self, "tip_detection_epoch", 0)) - != reconcile_detection_epoch - ): - # A detection interleaved the fetch (a flip away, or - # away and back to this same hash): cached proofs - # were evicted and the overlapped pass may have run - # in the closed epoch. Re-prove off-thread with the - # same bounded join -- the crawl that slows the - # overlap join slows this re-prove identically. - reconcile_source = "serial" - reorg_reconciled = runtime._reconcile_snapshot_tip_bounded( - snapshot.bestblockhash - ) - # A trust flip after the pass completed (headers running - # ahead with no detection) is deliberately NOT re-checked - # here: prepared fanout re-proves the live chain view in - # _validate_prepared_tip_refresh, and sequential issuance - # re-proves it per client in - # ensure_reorg_reconciled_for_current_tip, whose trust - # check is never cached. A second live check here would - # break the one-trust-validation-per-refresh economy. - else: - if reconcile_prefetch is not None: - # The tip moved between the probe and the template - # fetch; the prefetched pass proved a superseded - # hash. Reconcile the snapshot tip off-thread with - # the same bounded join: the crawl that slows the - # overlap join reaches this branch through exactly - # the churn that moves tips mid-fetch, and the poll - # loop must never park on it here either. - runtime._discard_stale_reconcile_prefetch( - reconcile_prefetch - ) - reconcile_prefetch = None - reorg_reconciled = runtime._reconcile_snapshot_tip_bounded( - snapshot.bestblockhash - ) - except (self._shutdown_error, FuturesCancelledError): - # Shutdown may close writer admission (or cancel the queued - # prefetch) after this refresh has fetched a snapshot. Leave - # the refresh incomplete and let the controlled shutdown - # proceed without consuming the template failure budget or - # taking the hard-exit path. - return 0 - except (TemplateRefreshSuperseded, PayoutStatePublicationBlocked): - raise - except Exception as exc: - raise _TipRefreshTrustBlocked( - "qbit reorg reconciliation failed before refresh preparation" - ) from exc - finally: - # The effective serial reconcile cost of this refresh: the - # memo probe plus whatever wait the template fetch did not - # absorb. Observe failures too so a slow reconcile that - # blocks the refresh still shows up in the histogram. - runtime._observe_tip_refresh_build_phase( - "reorg_reconcile", - reconcile_probe_seconds - + (time.monotonic() - reconcile_join_started), - ) - if reconciler_enabled: - runtime._record_reorg_reconcile_lookup( - "tip_refresh", - reconcile_source, - ) - if not reorg_reconciled: - raise _TipRefreshTrustBlocked( - "qbit chain view remained untrusted after reorg reconciliation" - ) - payout_generation_after_reconciliation = int( - getattr(runtime, "_payout_state_generation", 0) - ) - if ( - payout_generation_after_reconciliation - != payout_generation_before_reconciliation - ): - # A same-tip reconciliation can invalidate signed payout state - # even when no client needed template work at initial - # selection. Reselect after the ledger mutation so every old- - # generation job is replaced from the post-reorg snapshot. - with runtime.lock: - clients = [ - client - for client in runtime.clients - if runtime.client_can_receive_jobs(client) - and runtime.client_needs_tip_template_refresh(client, snapshot) - ] - expected_active_jobs = { - client: client.active_job - for client in clients - } - # Reconciliation itself minted the payout-pending token and - # this pass has deliberately reselected clients from that new - # generation. Adopt exactly that token; completion still checks - # both token ownership and payout/detected-tip currentness, so - # a later producer cannot be cleared accidentally. - pending_signal_token = runtime._claim_tip_refresh_pending() - with runtime.lock: - selected_clients_list = list(clients) - selected_client_set = set(clients) - for client in poll_start_clients: - if client in selected_client_set: - continue - if snapshot_changed or runtime.client_needs_tip_template_refresh( - client, - snapshot, - ): - selected_clients_list.append(client) - selected_client_set.add(client) - expected_active_jobs[client] = client.active_job - selected_clients = tuple(selected_clients_list) - # Prepared-mode selection must cover every client this pass can - # end up serving. Revalidation only filters selected_clients, so - # deriving from the narrower initial list could publish authority - # through the sequential path with no shared bundle while - # poll-start targets still need prepared ready work. - use_prepared_fanout = bool( - selected_clients - and getattr(runtime, "_pool_ready_latched", False) - ) - ready_mode = bool(getattr(runtime, "_pool_ready_latched", False)) - bundle: CachedJobBundle | None = None - validation_token: TipRefreshValidationToken | None = None - preactivated_cancel_event: FanoutCancellation | None = None - prepared_executor: ThreadPoolExecutor | None = None - if use_prepared_fanout: - runtime._raise_if_tip_refresh_superseded( - snapshot, - observation_sequence, - ) - try: - bundle = runtime.prepare_tip_refresh_bundle( - snapshot, - priority_requested_monotonic=refresh_started, - ) - except PayoutStatePublicationBlocked: - for _client in selected_clients: - runtime._record_tip_refresh_client_result("skipped") - runtime._schedule_tip_refresh_retry() - raise - except TemplateRefreshBlocked: - for _client in selected_clients: - runtime._record_tip_refresh_client_result("failed") - raise - if ( - bundle.payout_state_generation - != payout_generation_after_reconciliation - ): - # Request construction can observe a payout generation - # published after reconciliation and still return a fully - # coherent newest-generation bundle. Adopt it only while - # it remains the current immutable payout pointer, then - # reselect the complete fleet so no old-generation client - # is omitted merely because selection preceded the build. - with runtime._job_cache_lock: - current_payout_generation = ( - runtime._payout_state_generation - ) - current_payout_artifact = ( - runtime._published_payout_state.artifact - ) - if ( - bundle.payout_state_generation - != current_payout_generation - or bundle.build_key is None - or current_payout_artifact is None - or bundle.build_key.payout_artifact_sha256 - != current_payout_artifact.prior_balances_sha256 - ): - runtime._schedule_tip_refresh_retry() - raise TemplateRefreshSuperseded( - "payout state changed after refresh preparation; " - "immediate retry scheduled" - ) - payout_generation_after_reconciliation = ( - current_payout_generation - ) - with runtime.lock: - selected_clients = tuple( - client - for client in runtime.clients - if runtime.client_can_receive_jobs(client) - and runtime.client_needs_tip_template_refresh( - client, - snapshot, - ) - ) - expected_active_jobs = { - client: client.active_job - for client in selected_clients - } - pending_signal_token = runtime._claim_tip_refresh_pending() - - runtime._raise_if_tip_refresh_superseded( - snapshot, - observation_sequence, - ) - # Construction is complete. Acquire the short publication lane - # only for final validation, snapshot publication, and current - # client selection. Other observations and replacement builds can - # progress while an obsolete builder is still unwinding. - while not self._tip_refresh_lock.acquire( - timeout=PRISM_TIP_REFRESH_ADMISSION_POLL_SECONDS - ): - runtime._record_heartbeat(heartbeat_name) - if runtime.stop_event.is_set(): - return 0 - runtime._probe_tip_while_refresh_waiting() - runtime._raise_if_tip_refresh_superseded( - snapshot, - observation_sequence, - ) - publication_lock_acquired = True - with runtime._job_cache_lock: - current_payout_generation = runtime._payout_state_generation - current_payout_artifact = runtime._published_payout_state.artifact - if ( - current_payout_generation - != payout_generation_after_reconciliation - ): - runtime._schedule_tip_refresh_retry() - raise TemplateRefreshSuperseded( - "payout state changed before refresh publication; " - "immediate retry scheduled" - ) - if ( - bundle is not None - and ( - current_payout_artifact is None - or bundle.build_key is None - or bundle.build_key.payout_artifact_sha256 - != current_payout_artifact.prior_balances_sha256 - ) - ): - runtime._schedule_tip_refresh_retry() - raise TemplateRefreshBlocked( - "complete build key changed before refresh publication" - ) - - # A ready-pool pass must build and validate its immutable shared - # bundle before publishing submit authority. Otherwise a cache, - # derivation, or final chain-validation failure can invalidate - # retained work without a deliverable replacement. Sequential / - # collection work has no shared preparation stage, so it commits - # here immediately before its worker-specific builds. - if use_prepared_fanout: - assert bundle is not None - # Acquire infrastructure before publishing authority. Once the - # active token is installed, every subsequent fanout exit is - # covered by its cancellation/cleanup finally block. - prepared_executor = runtime.tip_refresh_executor() - # Parent metadata is cleanup-only, but fetch it before the - # final live/trust guard so validation remains the last RPC - # boundary before atomic publication. - try: - prepared_parent_hash = runtime._fetch_tip_parent_hash( - snapshot.bestblockhash - ) - except Exception: - prepared_parent_hash = None - validation_token = runtime._validate_prepared_tip_refresh( - bundle, - snapshot, - observation_sequence, - ) - preactivated_cancel_event = runtime._publish_prepared_tip_refresh( - validation_token, - bundle, - snapshot, - parent_hash=prepared_parent_hash, - ) - else: - if not runtime.observe_tip_first_seen( - snapshot.bestblockhash, - observation_sequence=observation_sequence, - publish_refresh_observation=True, - published_snapshot=snapshot, - ): - raise TemplateRefreshSuperseded( - "tip/template poll was superseded by a newer tip observation" - ) - runtime.prune_evicted_job_graveyard(force=False) - with runtime.lock: - current_tip = getattr(self, "current_tip_first_seen", None) - if ( - current_tip is None - or current_tip[0] != snapshot.bestblockhash - or int(getattr(self, "current_tip_observation_sequence", 0)) - != observation_sequence - ): - raise TemplateRefreshSuperseded( - "tip/template poll was superseded before snapshot publication" - ) - if self.tip_template_snapshot is not snapshot: - raise TemplateRefreshBlocked( - "tip/template snapshot was not atomically published" - ) - - dropped_client_results: list[str] = [] - with runtime.lock: - # Revalidate the originally selected targets at publication. - # New connections keep their own pending/retained wake; this - # pass must not consume work that appeared after construction. - # Preserve the pre-build expected job pointer so an intervening - # authorize/Vardiff delivery is never overwritten. - current_clients: list[ClientState] = [] - for client in selected_clients: - if client not in runtime.clients: - dropped_client_results.append("disconnected") - elif not runtime.client_can_receive_jobs(client): - dropped_client_results.append("skipped") - elif not runtime.client_needs_tip_template_refresh(client, snapshot): - dropped_client_results.append("skipped") - else: - current_clients.append(client) - clients = current_clients - - for result in dropped_client_results: - runtime._record_tip_refresh_client_result(result) - - if bundle is not None and not bundle.collection_only: - with runtime._job_cache_lock: - if ( - bundle.payout_state_generation - == runtime._payout_state_generation - ): - runtime._prepared_ready_snapshot = snapshot - runtime._prepared_ready_bundle = bundle - - if ( - not use_prepared_fanout - and clients - and getattr(runtime, "_pool_ready_latched", False) - ): - # Publication already committed through the sequential path, - # so a ready-pool target that appeared after build selection - # has no shared bundle to receive. Leave the pending marker - # armed and let the immediate retry build for it. - runtime._mark_tip_refresh_pending(observation_sequence) - runtime._schedule_tip_refresh_retry() - raise TemplateRefreshBlocked( - "current clients appeared after build selection; retry scheduled" - ) - self._tip_refresh_lock.release() - publication_lock_acquired = False - - with runtime.lock: - progress_eligible_client = any( - runtime.client_can_receive_jobs(client) - for client in runtime.clients - ) - if use_prepared_fanout or not progress_eligible_client: - # Ready-mode work was prepared above. With no eligible clients, - # publishing the reconciled snapshot is sufficient; collection - # work with a live identity is not published until its worker- - # specific bundle is actually delivered below. - runtime._record_progress_publication( - snapshot, - payout_generation_after_reconciliation, - ) - - if not ready_mode: - with runtime.lock: - eligible_collection_client = any( - runtime.client_can_receive_jobs(client) - for client in runtime.clients - ) - if not eligible_collection_client: - runtime._retain_collection_refresh( - snapshot, - observation_sequence, - payout_generation_after_reconciliation, - ) - - if use_prepared_fanout: - assert bundle is not None - ( - refreshed, - first_delivery, - last_delivery, - build_failures, - ) = runtime._fanout_prepared_tip_refresh( - clients, - bundle, - snapshot, - observation_sequence=observation_sequence, - validation_token=validation_token, - preactivated_cancel_event=preactivated_cancel_event, - executor=prepared_executor, - expected_active_jobs=expected_active_jobs, - heartbeat_name=heartbeat_name, - ) - else: - for client in clients: - if runtime.stop_event.is_set(): - break - # Collection bundles are worker-specific, so build and - # validate each selected target independently. - runtime._record_heartbeat(heartbeat_name) - with runtime.lock: - target_connected = client in runtime.clients - target_eligible = ( - target_connected - and runtime.client_can_receive_jobs(client) - ) - if not target_connected: - runtime._record_tip_refresh_client_result("disconnected") - continue - if not target_eligible: - runtime._record_tip_refresh_client_result("skipped") - continue - try: - if runtime.maybe_send_job( - client, - clean_jobs=runtime.client_tip_changed_for_snapshot(client, snapshot), - raise_on_reorg_failure=True, - raise_on_build_failure=True, - tip_refresh_snapshot=snapshot, - tip_refresh_observation_sequence=observation_sequence, - ): - delivered = time.monotonic() - refreshed += 1 - first_delivery = ( - delivered - if first_delivery is None - else min(first_delivery, delivered) - ) - last_delivery = ( - delivered - if last_delivery is None - else max(last_delivery, delivered) - ) - runtime._record_tip_refresh_client_result("sent") - else: - runtime._record_tip_refresh_client_result("skipped") - except self._job_build_failed_error: - build_failures += 1 - runtime._record_tip_refresh_client_result("failed") - except OSError: - runtime._record_tip_refresh_client_result("disconnected") - runtime.disconnect_client(client) - - if not ready_mode: - with runtime.lock: - eligible_collection_client = any( - runtime.client_can_receive_jobs(client) - for client in runtime.clients - ) - if not eligible_collection_client: - runtime._retain_collection_refresh( - snapshot, - observation_sequence, - payout_generation_after_reconciliation, - ) - runtime._record_progress_publication( - snapshot, - payout_generation_after_reconciliation, - ) - - if clients: - try: - post_fanout_tip = str(runtime.rpc.call("getbestblockhash")) - except Exception as exc: - runtime._schedule_tip_refresh_retry() - raise TemplateRefreshBlocked( - "qbit tip validation failed after sequential refresh; " - "immediate retry scheduled" - ) from exc - if post_fanout_tip != snapshot.bestblockhash: - runtime._schedule_tip_refresh_retry() - raise TemplateRefreshSuperseded( - "qbit tip changed during sequential refresh; " - "immediate retry scheduled " - f"expected={snapshot.bestblockhash} current={post_fanout_tip}" - ) - if int(getattr(runtime, "_payout_state_generation", 0)) != ( - payout_generation_after_reconciliation - ): - runtime._schedule_tip_refresh_retry() - raise TemplateRefreshSuperseded( - "payout state changed during sequential refresh; " - "immediate retry scheduled" - ) - - if refreshed == 0 and build_failures: - raise TemplateRefreshBlocked( - f"job builds failed for {build_failures} client(s); no refreshed work was issued" - ) - if refreshed: - with runtime.lock: - self.tip_refresh_job_count += refreshed - assert first_delivery is not None and last_delivery is not None - runtime._observe_tip_refresh_seconds( - "first_delivery", - first_delivery - refresh_started, - ) - runtime._observe_tip_refresh_seconds( - "last_delivery", - last_delivery - refresh_started, - ) - if not runtime._clear_tip_refresh_pending_for_completed_refresh( - snapshot, - observation_sequence, - payout_generation_after_reconciliation, - pending_signal_token, - ): - # A newer tip or payout mutation won after the last delivery - # guard. Preserve its pending token and retry immediately. - pending_signal_token = None - runtime._schedule_tip_refresh_retry() - raise TemplateRefreshSuperseded( - "tip or payout state changed before refresh completion; " - "immediate retry scheduled" - ) - self.last_successful_template_refresh_monotonic = time.monotonic() - self.template_refresh_failure_started_monotonic = None - runtime._clear_coordination_blocked_streak() - self._clear_tip_refresh_failure_holdoff() - # A completed pass reconfirms that the coherent snapshot remained - # current through publication and fanout. Refresh the liveness - # stamp so a legitimately long, progressing pass does not become - # stale the instant its active marker is cleared. - runtime._record_progress_tip_poll(snapshot) - return refreshed - except TemplateRefreshSuperseded: - # Coordination-blocked refreshes -- a superseded tip, a pending - # payout publication fence, a refresh raced by payout mutation -- - # are churn between healthy components, not qbitd unhealthiness. - # They do not arm the ordinary failure budget, but their oldest - # continuous streak has its own longer restart deadline. Re-raise - # so callers still schedule their immediate retry. Plain - # TemplateRefreshBlocked stays budgeted below: it also wraps - # genuine failures (job builds failing, malformed template - # artifacts, untrusted chain views) whose persistence must still - # take the ordinary restart path. - # The retry-spacing stamp still applies: churn against an - # unchanged tip re-arms the poller no faster than the holdoff, - # while a genuinely newer tip zeroes it immediately. - if not getattr(runtime, "tip_refresh_epoch_fanout", False): - runtime._record_coordination_blocked_refresh(time.monotonic()) - runtime._note_tip_refresh_attempt_failed(observed_best_tip) - raise - except PayoutStatePublicationBlocked: - runtime._record_coordination_blocked_refresh(time.monotonic()) - runtime._note_tip_refresh_attempt_failed(observed_best_tip) - raise - except Exception: - runtime._clear_coordination_blocked_streak() - runtime._record_template_refresh_failure(time.monotonic()) - runtime._note_tip_refresh_attempt_failed(observed_best_tip) - raise - finally: - if publication_lock_acquired: - self._tip_refresh_lock.release() - if progress_refresh_active: - runtime._progress_refresh_finished() - - def _probe_tip_while_refresh_waiting(self) -> None: - """Detect a changed live tip without entering the heavy refresh lane.""" - runtime = self._runtime - observation_sequence = runtime._reserve_tip_observation_sequence() - try: - observed_tip = str(runtime.rpc.call("getbestblockhash")) - except Exception: - # The owning refresh still has to unwind or complete. Preserve its - # pending state and let the next bounded lock wait probe again. - return - runtime.observe_tip_for_refresh( - observed_tip, - observation_sequence=observation_sequence, - ) - - def _detected_tip_supersedes_locked( - self, - tip_hash: str, - observation_sequence: int, - ) -> bool: - latest = getattr(self, "latest_detected_tip", None) - return bool( - latest is not None - and latest[0] != tip_hash - and latest[1] > observation_sequence - ) - - def _raise_if_tip_refresh_superseded( - self, - snapshot: QbitTipTemplateSnapshot, - observation_sequence: int, - ) -> None: - """Stop obsolete work before entering another expensive phase.""" - runtime = self._runtime - with runtime.lock: - superseded = runtime._detected_tip_supersedes_locked( - snapshot.bestblockhash, - observation_sequence, - ) - if superseded: - runtime._schedule_tip_refresh_retry() - raise TemplateRefreshSuperseded( - "tip/template poll was superseded by a newer tip observation " - "before refresh preparation" - ) - - def _reserve_tip_observation_sequence(self) -> int: - runtime = self._runtime - with runtime.lock: - sequence = int(getattr(self, "tip_observation_sequence", 0)) + 1 - self.tip_observation_sequence = sequence - return sequence - - def observe_tip_for_refresh( - self, - tip_hash: str, - *, - observation_sequence: int | None = None, - mark_pending: bool = True, - ) -> bool: - """Record tip detection without publishing share-validation authority. - - A waiter outside ``_tip_refresh_lock`` must be able to cancel obsolete - bundle construction and fanout promptly. It must not update - ``current_tip_first_seen``: that value invalidates old jobs, so the - winning refresh publishes it only after replacement work is prepared - and validated. - """ - runtime = self._runtime - if observation_sequence is None: - observation_sequence = runtime._reserve_tip_observation_sequence() - runtime._ensure_job_cache_state() - runtime._ensure_tip_refresh_state() - # Before any sequencing decision: an own-candidate hash arriving as - # the tip is acceptance evidence (blockwait can see it before -- or - # instead of -- the direct submitblock ack) and must be registered - # even when this observation loses the sequence race below. - runtime._note_tip_observation_for_candidates(tip_hash) - now = time.monotonic() - active_to_cancel: FanoutCancellation | None = None - should_mark_pending = False - with runtime.lock: - latest = getattr(self, "latest_detected_tip", None) - if latest is not None and observation_sequence < latest[1]: - return latest[0] == tip_hash - published = getattr(self, "current_tip_first_seen", None) - prior_detected_hash = ( - latest[0] - if latest is not None - else published[0] - if published is not None - else None - ) - detection_changed = ( - prior_detected_hash is not None - and prior_detected_hash != tip_hash - ) - if detection_changed: - runtime._evict_reorg_reconcile_memo_for_new_tip_locked(tip_hash) - # Bumped exactly when cached reconcile proofs are dropped: - # a refresh's join compares this epoch to tell whether any - # flip (away, or away and back) interleaved its template - # fetch and invalidated a pass that ran concurrently. - self.tip_detection_epoch = ( - int(getattr(self, "tip_detection_epoch", 0)) + 1 - ) - if ( - detection_changed - and runtime._payout_state_source[1] != tip_hash - ): - # Supersede an in-progress immutable payout candidate as soon - # as the newer tip is detected, without publishing that tip as - # submit authority before replacement work is ready. - source_generation = runtime._payout_state_source[0] + 1 - runtime._payout_state_source = ( - source_generation, - tip_hash, - "external_tip", - now, - ) - if latest is None or detection_changed: - runtime._mint_tip_refresh_epoch_locked( - tip_hash=tip_hash, - payout_state_generation=int( - getattr(runtime, "_payout_state_generation", 0) - ), - started_monotonic=now, - ) - self.latest_detected_tip = (tip_hash, observation_sequence) - replacement_needed = published is None or published[0] != tip_hash - if published is not None and published[0] == tip_hash: - # A live observation has returned to (or reconfirmed) the - # published generation. This closes any unpublished divergence - # epoch without changing the published snapshot itself. - self.current_tip_observed_monotonic = now - self.tip_refresh_divergence_started_monotonic = None - elif ( - published is not None - and getattr( - self, - "tip_refresh_divergence_started_monotonic", - None, - ) - is None - ): - # Anchor the lease to the first departure from the published - # tip. Further B -> C observations deliberately do not renew it. - self.tip_refresh_divergence_started_monotonic = now - pending_already = bool( - getattr(self, "_tip_refresh_pending_event", None) - and self._tip_refresh_pending_event.is_set() - ) - active = getattr(self, "_active_tip_refresh", None) - if ( - active is not None - and active[0].tip_hash != tip_hash - and active[0].observation_sequence < observation_sequence - ): - active_to_cancel = active[1] - should_mark_pending = bool( - mark_pending - and ( - detection_changed - or (replacement_needed and not pending_already) - ) - ) - - if active_to_cancel is not None: - active_to_cancel.cancel() - if detection_changed: - runtime._progress_note_refresh_pending(now) - # In-flight constructions for older parents can no longer win; - # stop them at detection so replacement preparation starts - # immediately, well before publication moves submit authority. - runtime._cancel_obsolete_job_bundle_builds(current_tip=tip_hash) - runtime._cancel_obsolete_job_builds("chain tip superseded") - if should_mark_pending: - runtime._mark_tip_refresh_pending(observation_sequence) - runtime._schedule_tip_refresh_retry() - return True - - def observe_tip_first_seen( - self, - tip_hash: str, - *, - observation_sequence: int | None = None, - publish_refresh_observation: bool = False, - published_snapshot: QbitTipTemplateSnapshot | None = None, - ) -> bool: - """Publish a tip (and, when supplied, its coherent template snapshot). - - Detection and publication are deliberately separate. Callers doing a - prepared refresh must finish bundle construction and final validation - before invoking this method. - """ - runtime = self._runtime - if ( - published_snapshot is not None - and published_snapshot.bestblockhash != tip_hash - ): - raise ValueError("published snapshot does not match tip hash") - if observation_sequence is None: - observation_sequence = runtime._reserve_tip_observation_sequence() - if not runtime.observe_tip_for_refresh( - tip_hash, - observation_sequence=observation_sequence, - mark_pending=False, - ): - return False - now = time.monotonic() - with runtime.lock: - current_sequence = int( - getattr(self, "current_tip_observation_sequence", 0) - ) - if ( - observation_sequence < current_sequence - or runtime._detected_tip_supersedes_locked( - tip_hash, - observation_sequence, - ) - ): - return False - first_seen = getattr(self, "current_tip_first_seen", None) - if first_seen is not None and first_seen[0] == tip_hash: - # A same-tip re-observation proves the tip view is live; the - # freshness stamp bounds submit_stale_check_tip reuse. - self.current_tip_observed_monotonic = now - active = getattr(self, "_active_tip_refresh", None) - # A routine blockwait/poll observation of the same hash carries - # no newer template. While that hash is actively fanning out, - # do not invalidate its token merely by advancing the global - # observation sequence. The next real refresh observation can - # advance it after the active fanout clears. - if publish_refresh_observation and ( - active is None or active[0].tip_hash != tip_hash - ): - self.current_tip_observation_sequence = observation_sequence - if published_snapshot is not None: - self.tip_template_snapshot = published_snapshot - runtime._publish_tip_refresh_epoch_identity_locked( - published_snapshot - ) - self.tip_refresh_divergence_started_monotonic = None - return True - - # Fetch optional cleanup metadata before the atomic publication. A slow - # parent RPC must not create a window where submits see the new tip but - # the winning refresh has not yet published its coherent snapshot. - try: - parent_hash = runtime._fetch_tip_parent_hash(tip_hash) - except Exception: - parent_hash = None - - with runtime.lock: - current_sequence = int( - getattr(self, "current_tip_observation_sequence", 0) - ) - if ( - observation_sequence < current_sequence - or runtime._detected_tip_supersedes_locked( - tip_hash, - observation_sequence, - ) - ): - return False - first_seen = getattr(self, "current_tip_first_seen", None) - if first_seen is not None and first_seen[0] == tip_hash: - self.current_tip_observed_monotonic = now - active = getattr(self, "_active_tip_refresh", None) - if publish_refresh_observation and ( - active is None or active[0].tip_hash != tip_hash - ): - self.current_tip_observation_sequence = observation_sequence - if published_snapshot is not None: - self.tip_template_snapshot = published_snapshot - runtime._publish_tip_refresh_epoch_identity_locked( - published_snapshot - ) - self.tip_refresh_divergence_started_monotonic = None - return True - - tip_changed = first_seen is not None - # The first tip this process publishes is a startup baseline, not - # a tip flip: a None stamp keeps stale grace closed. - self.current_tip_first_seen = ( - tip_hash, - now if tip_changed else None, - ) - self.current_tip_observation_sequence = observation_sequence - self.current_tip_observed_monotonic = now - if published_snapshot is not None: - self.tip_template_snapshot = published_snapshot - runtime._publish_tip_refresh_epoch_identity_locked( - published_snapshot - ) - self.tip_refresh_divergence_started_monotonic = None - # Retained collection work is reusable throughout detection and - # preparation, but never after authority moves to a different tip. - self._retained_collection_refresh = None - if parent_hash is None: - self.current_tip_parent = None - else: - self.current_tip_parent = (tip_hash, parent_hash) - # Reclassify formerly same-tip entries immediately. On mainnet the - # zero stale-grace TTL removes them in this pass; on other chains - # the actual chain parent removes multi-tip-behind entries while - # the independently configured grace lifetime protects one-back. - runtime.prune_evicted_job_graveyard(now=now, force=True) - if tip_changed: - # A retained ready bundle belongs to the previously published tip - # and can never satisfy the consumer's published-snapshot identity - # check again; release it as soon as authority moves. - with runtime._job_cache_lock: - runtime._prepared_ready_bundle = None - runtime._prepared_ready_snapshot = None - return True - - def reorg_proof_snapshot(self) -> tuple[str | None, int]: - """Atomic (latest detected tip hash, detection epoch) reorg proof. - - The reorg reconciler remains coordinator-owned at this layer; its - later owner consumes this snapshot instead of reading the two fields - separately, so a detection between the reads can never pair a new tip - with a stale epoch. - """ - runtime = self._runtime - with runtime.lock: - latest = getattr(self, "latest_detected_tip", None) - return ( - latest[0] if latest is not None else None, - int(getattr(self, "tip_detection_epoch", 0)), - ) - - def _fetch_tip_parent_hash(self, tip_hash: str) -> str | None: - runtime = self._runtime - block = runtime.rpc.call("getblock", [tip_hash]) - if not isinstance(block, dict): - return None - parent = str(block.get("previousblockhash", "") or "") - if not parent: - return None - return parent - - def current_tip_parent_hash(self, tip_hash: str) -> str | None: - runtime = self._runtime - with runtime.lock: - cached = getattr(self, "current_tip_parent", None) - if cached is not None and cached[0] == tip_hash: - return cached[1] - first_seen = getattr(self, "current_tip_first_seen", None) - observed_sequence = ( - int(getattr(self, "current_tip_observation_sequence", 0)) - if first_seen is not None and first_seen[0] == tip_hash - else None - ) - parent = runtime._fetch_tip_parent_hash(tip_hash) - if parent is None: - return None - with runtime.lock: - current = getattr(self, "current_tip_first_seen", None) - if ( - observed_sequence is not None - and current is not None - and current[0] == tip_hash - and int(getattr(self, "current_tip_observation_sequence", 0)) - == observed_sequence - ): - self.current_tip_parent = (tip_hash, parent) - return parent - - def submit_stale_check_tip(self) -> str: - """Best-known chain tip for per-share submit classification. - - Prefers the tip for which the refresh path already published coherent - work (reconfirmed at least every PRISM_BLOCKPOLL_SECONDS while healthy) - so mining.submit never blocks on a getbestblockhash RPC per share. This - also removes the submit-races-ahead-of-the-refresh failure mode: a - submit-path RPC can observe a new tip seconds before jobs refresh, and - with PRISM_STRATUM_STALE_GRACE_SECONDS=0 (then pinned on mainnet) that - rejected every in-flight share on the old tip. Classifying against the - published tip keeps shares valid until the coordinator has prepared, - validated, and published the flip, and it is the same tip source the - stale-grace window and evicted-job classification are anchored to. - - During a detected-but-unpublished replacement, the published tip stays - authoritative beyond the ordinary freshness age so a large healthy - build cannot recreate the reject outage. That extension is bounded by - PRISM_TEMPLATE_REFRESH_FAILURE_EXIT_SECONDS and is anchored to the - first unpublished divergence; failed refreshes therefore still fall - back to the live RPC instead of accepting frozen work indefinitely. - """ - runtime = self._runtime - with runtime.lock: - published_tip = runtime._submit_stale_check_tip_locked(time.monotonic()) - if published_tip is not None: - return published_tip - return str(runtime.rpc.call("getbestblockhash")) - - def _published_tip_authoritative_locked(self, now: float) -> bool: - """True while the published tip still owns share classification. - - Either the ordinary freshness window (reconfirmed within - PRISM_SUBMIT_TIP_MAX_AGE_SECONDS) or the bounded detected-but- - unpublished replacement lease is open. Job issuance uses the same - predicate so work handed to miners is never classified against a - different tip than the one it was issued for. - """ - runtime = self._runtime - max_age = float( - getattr( - runtime, - "submit_tip_max_age_seconds", - DEFAULT_PRISM_SUBMIT_TIP_MAX_AGE_SECONDS, - ) - ) - if max_age <= 0: - return False - observed = getattr(self, "current_tip_first_seen", None) - if observed is None: - return False - observed_at = getattr(self, "current_tip_observed_monotonic", None) - if observed_at is not None and now - observed_at <= max_age: - return True - latest_detected = getattr(self, "latest_detected_tip", None) - divergence_started = getattr( - self, - "tip_refresh_divergence_started_monotonic", - None, - ) - divergence_budget = float( - getattr( - runtime, - "template_refresh_failure_exit_seconds", - DEFAULT_PRISM_TEMPLATE_MAX_AGE_SECONDS, - ) - ) - return bool( - latest_detected is not None - and latest_detected[0] != observed[0] - and divergence_started is not None - and divergence_budget > 0 - and now - divergence_started <= divergence_budget - ) - - def refresh_jobs_after_pending_accepted_block( - self, - client: ClientState, - *, - heartbeat_name: str = "qbit_blockpoll", - ) -> int: - runtime = self._runtime - with runtime.lock: - block = client.post_accept_refresh_block - client.post_accept_refresh_block = None - if block is None: - return 0 - block_height, block_hash = block - return runtime.refresh_jobs_after_accepted_block( - block_height=block_height, - block_hash=block_hash, - heartbeat_name=heartbeat_name, - ) - - def refresh_jobs_after_accepted_block( - self, *, block_height: int, block_hash: str, heartbeat_name: str = "qbit_blockpoll" - ) -> int: - runtime = self._runtime - try: - runtime._record_heartbeat(heartbeat_name) - observation_sequence = runtime._reserve_tip_observation_sequence() - observed_tip = str(runtime.rpc.call("getbestblockhash")) - if not runtime.observe_tip_for_refresh( - observed_tip, - observation_sequence=observation_sequence, - ): - raise TemplateRefreshSuperseded( - "post-accept tip observation was superseded" - ) - except TemplateRefreshSuperseded: - # A newer observation won before this notification could publish - # its tip. The shared driver is woken below; this is coordination - # churn, not a post-accept refresh failure. - return 0 - except Exception: - with runtime.lock: - self.post_accept_refresh_failure_count += 1 - print( - "prism coordinator: post-accept clean job refresh failed after direct PRISM block " - f"height={block_height} hash={block_hash}", - flush=True, - ) - traceback.print_exc() - return 0 - finally: - # Accepted-block payout/template changes can require a same-tip - # rebuild. Wake the driver even when the immediate best-tip RPC or - # observation fails; the owner will refetch a coherent snapshot. - runtime._schedule_tip_refresh_retry() - print( - "prism coordinator: scheduled single-flight job refresh after " - f"direct PRISM block height={block_height} hash={block_hash} " - f"observed_tip={observed_tip}", - flush=True, - ) - return 0 - - @staticmethod - def _tip_refresh_hashrate_proxy(client: ClientState) -> Decimal: - proxy = ( - getattr(client, "vardiff_difficulty_estimate", None) - or getattr(client, "share_difficulty", Decimal(0)) - ) - try: - resolved = Decimal(proxy) - except (InvalidOperation, TypeError, ValueError): - return Decimal(0) - return resolved if resolved.is_finite() and resolved > 0 else Decimal(0) - - def _payout_epoch_tip_hash_locked(self) -> str | None: - """Label a payout-minted epoch with the current refresh target tip. - - A payout publication advances the payout generation of the target - the fleet is already converging on; it must not move or invent the - tip. Candidate source hashes are unusable here: an accepted-block - preview stores the accepted block's own hash, which is not a - template parent and can trail an already-observed newer tip, so - labeling the epoch with it strands token currency, identity - publication, and the fixpoint until a later tip observation - remints. - """ - current_epoch_tip = getattr(self, "_tip_refresh_epoch_tip_hash", None) - if current_epoch_tip is not None: - return str(current_epoch_tip) - latest_detected = getattr(self, "latest_detected_tip", None) - if latest_detected is not None: - return str(latest_detected[0]) - published_tip = getattr(self, "current_tip_first_seen", None) - if published_tip is not None: - return str(published_tip[0]) - return None - - def _mint_tip_refresh_epoch_locked( - self, - *, - tip_hash: str, - payout_state_generation: int, - started_monotonic: float, - ) -> int: - """Mint the next target epoch while the coordinator lock is held.""" - runtime = self._runtime - if not getattr(runtime, "tip_refresh_epoch_fanout", False): - return 0 - sequence = int(self._tip_refresh_epoch_sequence) + 1 - self._tip_refresh_epoch_sequence = sequence - self._tip_refresh_epoch_tip_hash = tip_hash - self._tip_refresh_epoch_payout_generation = int(payout_state_generation) - client_weights = { - int(client.connection_id): runtime._tip_refresh_hashrate_proxy(client) - for client in runtime.clients - if runtime.client_can_receive_jobs(client) - } - total_weight = sum( - ( - weight - for weight in client_weights.values() - if weight > 0 - ), - Decimal(0), - ) - self._tip_refresh_epoch_coverage = _TipRefreshEpochCoverage( - sequence=sequence, - tip_hash=tip_hash, - payout_state_generation=int(payout_state_generation), - started_monotonic=float(started_monotonic), - client_weights=client_weights, - total_weight=total_weight, - ) - if total_weight <= 0: - tracker = self._tip_refresh_epoch_coverage - with self._tip_refresh_metrics_lock: - for target, _ratio in PRISM_TIP_REFRESH_COVERAGE_TARGETS: - tracker.recorded_targets.add(target) - histogram = self.tip_refresh_coverage_histograms[target] - histogram["count"] = int(histogram["count"]) + 1 - buckets = histogram["buckets"] - assert isinstance(buckets, dict) - for bucket in PRISM_TIP_REFRESH_SECONDS_BUCKETS: - buckets[bucket] = int(buckets.get(bucket, 0)) + 1 - return sequence - - def _tip_refresh_epoch_for_bundle_locked( - self, - cached: CachedJobBundle, - ) -> int: - runtime = self._runtime - if not getattr(runtime, "tip_refresh_epoch_fanout", False): - return 0 - identity = getattr( - self, - "_published_tip_refresh_epoch_identity", - None, - ) - if identity is None: - return 0 - ( - sequence, - tip_hash, - payout_generation, - template_fingerprint, - template_generation, - ) = identity - parent_hash = str(cached.template.get("previousblockhash", "")) - if ( - parent_hash != tip_hash - or int(cached.payout_state_generation) - != payout_generation - or cached.template_fingerprint != template_fingerprint - or int(cached.template_generation) - != template_generation - ): - return 0 - return sequence - - def _publish_tip_refresh_epoch_identity_locked( - self, - snapshot: QbitTipTemplateSnapshot, - ) -> None: - runtime = self._runtime - if not getattr(runtime, "tip_refresh_epoch_fanout", False): - return - sequence = int(getattr(self, "_tip_refresh_epoch_sequence", 0)) - tip_hash = getattr(self, "_tip_refresh_epoch_tip_hash", None) - payout_generation = int( - getattr(self, "_tip_refresh_epoch_payout_generation", 0) - ) - if ( - sequence <= 0 - or tip_hash != snapshot.bestblockhash - or payout_generation - != int(getattr(runtime, "_payout_state_generation", 0)) - ): - return - self._published_tip_refresh_epoch_identity = ( - sequence, - tip_hash, - payout_generation, - snapshot.template_fingerprint, - snapshot.template_generation, - ) - - def _admit_client_tip_refresh_epoch_locked( - self, - client: ClientState, - epoch_sequence: int, - ) -> bool: - """Fence per-client registration so epochs cannot regress on the wire.""" - runtime = self._runtime - if not getattr(runtime, "tip_refresh_epoch_fanout", False): - return True - if runtime._client_tip_refresh_epoch_blocked_locked(client, epoch_sequence): - return False - admitted_epoch = int( - getattr(client, "_tip_refresh_admitted_epoch_sequence", 0) - ) - client._tip_refresh_admitted_epoch_sequence = max( - int(epoch_sequence), - admitted_epoch, - ) - return True - - @staticmethod - def _client_tip_refresh_epoch_blocked_locked( - client: ClientState, - epoch_sequence: int, - ) -> bool: - active_epoch = int( - getattr( - getattr(client, "active_job", None), - "tip_refresh_epoch_sequence", - 0, - ) - ) - admitted_epoch = int( - getattr(client, "_tip_refresh_admitted_epoch_sequence", 0) - ) - return int(epoch_sequence) < max(active_epoch, admitted_epoch) - - def _tip_refresh_epoch_coverage_reached_locked( - self, - client: ClientState, - context: PrismJobContext, - delivered_monotonic: float, - ) -> list[tuple[str, float]]: - """Advance the active cohort while the coordinator lock is held.""" - runtime = self._runtime - if not getattr(runtime, "tip_refresh_epoch_fanout", False): - return [] - reached: list[tuple[str, float]] = [] - tracker = self._tip_refresh_epoch_coverage - connection_id = int(getattr(client, "connection_id", 0)) - if ( - tracker is None - or tracker.sequence != int(self._tip_refresh_epoch_sequence) - or tracker.total_weight <= 0 - or connection_id in tracker.delivered_clients - ): - return reached - weight = tracker.client_weights.get(connection_id) - if weight is None: - return reached - if ( - int(getattr(context, "connection_id", 0)) != connection_id - or int(getattr(context, "tip_refresh_epoch_sequence", 0)) - != tracker.sequence - or str(context.template.get("previousblockhash", "")) - != tracker.tip_hash - or int(getattr(context, "payout_state_generation", 0)) - != tracker.payout_state_generation - ): - return reached - tracker.delivered_clients.add(connection_id) - if weight <= 0: - return reached - tracker.delivered_weight += weight - coverage = tracker.delivered_weight / tracker.total_weight - elapsed = max( - 0.0, - float(delivered_monotonic) - tracker.started_monotonic, - ) - for target, ratio in PRISM_TIP_REFRESH_COVERAGE_TARGETS: - if target not in tracker.recorded_targets and coverage >= ratio: - tracker.recorded_targets.add(target) - reached.append((target, elapsed)) - return reached - - def _record_tip_refresh_epoch_coverage( - self, - reached: list[tuple[str, float]], - ) -> None: - """Observe fixed cohort thresholds without per-connection series.""" - if not reached: - return - with self._tip_refresh_metrics_lock: - for target, elapsed in reached: - histogram = self.tip_refresh_coverage_histograms[target] - histogram["count"] = int(histogram["count"]) + 1 - histogram["sum"] = float(histogram["sum"]) + elapsed - buckets = histogram["buckets"] - assert isinstance(buckets, dict) - for bucket in PRISM_TIP_REFRESH_SECONDS_BUCKETS: - if elapsed <= bucket: - buckets[bucket] = int(buckets.get(bucket, 0)) + 1 - - def _tip_refresh_epoch_fixpoint_reached(self) -> bool: - """Return whether every eligible connection has newest delivered work.""" - runtime = self._runtime - if not getattr(runtime, "tip_refresh_epoch_fanout", False): - return True - with runtime.lock: - sequence = int(getattr(self, "_tip_refresh_epoch_sequence", 0)) - tip_hash = getattr(self, "_tip_refresh_epoch_tip_hash", None) - payout_generation = int( - getattr(self, "_tip_refresh_epoch_payout_generation", 0) - ) - snapshot = getattr(self, "tip_template_snapshot", None) - ready_mode = bool(getattr(runtime, "_pool_ready_latched", False)) - for client in runtime.clients: - if not runtime.client_can_receive_jobs(client): - continue - context = getattr(client, "_progress_delivered_context", None) - if ( - context is None - or int(getattr(context, "connection_id", 0)) - != int(client.connection_id) - ): - return False - context_epoch = int( - getattr(context, "tip_refresh_epoch_sequence", 0) - ) - if context_epoch > sequence: - continue - if context_epoch < sequence: - return False - if str(context.template.get("previousblockhash", "")) != tip_hash: - return False - if ( - int(getattr(context, "payout_state_generation", 0)) - != payout_generation - ): - return False - if ready_mode and bool(getattr(context, "collection_only", False)): - return False - if ( - snapshot is not None - and snapshot.bestblockhash == tip_hash - and ( - getattr(context, "template_fingerprint", None) - != snapshot.template_fingerprint - ) - ): - return False - # Every eligible connection holds newest delivered work, so any - # refresh retry armed before this verification is satisfied. - # Consume it inside the same lock hold that proved convergence: - # signals armed afterwards stay armed, and blockpoll no longer - # re-runs a full wave against work a recovered supersession - # already finished. - self._tip_refresh_retry_consumed = self._tip_refresh_retry_counter - self._tip_refresh_retry.clear() - return True - - def _clear_tip_refresh_failure_holdoff(self) -> None: - runtime = self._runtime - runtime._ensure_tip_refresh_state() - with runtime.lock: - self._tip_refresh_failure_holdoff_until = None - self._tip_refresh_failure_tip = None - - def _reuse_current_tip_template_snapshot( - self, - observed_best_tip: str, - ) -> QbitTipTemplateSnapshot | None: - """Rebuild a snapshot from cached artifacts while their tip holds. - - Honors the PRISM_TEMPLATE_CACHE_SECONDS window that per-client job - builds already use: within it, a poll pass whose observed best tip - still equals the cached template's parent issues no getblocktemplate. - Rapid failed-refresh re-attempts then cost one cheap best-hash probe - each, while a changed tip or an expired window falls through to the - full fetch, so same-tip template rotation still lands on the normal - cadence. - """ - runtime = self._runtime - runtime._ensure_job_cache_state() - ttl = float( - getattr(runtime, "template_cache_seconds", DEFAULT_PRISM_BLOCKPOLL_SECONDS) - ) - if ttl <= 0: - return None - with runtime._job_cache_lock: - cached = runtime._template_artifacts - if cached is None: - return None - if time.monotonic() - cached.fetched_monotonic > ttl: - return None - if str(observed_best_tip) != cached.previousblockhash: - return None - runtime._record_job_cache_event("template", hit=True) - return QbitTipTemplateSnapshot( - bestblockhash=cached.previousblockhash, - previousblockhash=cached.previousblockhash, - template_fingerprint=cached.fingerprint, - template_generation=cached.generation, - template_artifacts=cached, - ) - - -__all__ = [ - "DEFAULT_PRISM_PAYOUT_RECONCILE_SUPERSESSION_RETRIES", - "FanoutCancellation", - "PRISM_TIP_REFRESH_ADMISSION_POLL_SECONDS", - "PRISM_TIP_REFRESH_BUILD_PHASES", - "PRISM_TIP_REFRESH_CANCELLATION_STAGES", - "PRISM_TIP_REFRESH_COVERAGE_TARGETS", - "PRISM_TIP_REFRESH_FAILURE_HOLDOFF_JITTER_FRACTION", - "PRISM_TIP_REFRESH_REENTRY_BACKOFF_SECONDS", - "PRISM_TIP_REFRESH_RESULTS", - "PRISM_TIP_REFRESH_SECONDS_BUCKETS", - "PRISM_TIP_REFRESH_WAVE_OUTCOMES", - "PRISM_TIP_REFRESH_WAVE_PASS_BUDGET", - "RefreshResult", - "RetainedCollectionRefresh", - "TipRefreshRuntime", - "TipRefreshService", - "TipRefreshValidationToken", -] diff --git a/lab/prism/vardiff_service.py b/lab/prism/vardiff_service.py deleted file mode 100644 index 627d9d6ce..000000000 --- a/lab/prism/vardiff_service.py +++ /dev/null @@ -1,2538 +0,0 @@ -"""PRISM vardiff windows, retarget delivery, and bounded idle work.""" - -from __future__ import annotations - -from collections import OrderedDict -from concurrent.futures import Future, ThreadPoolExecutor -from dataclasses import dataclass -from decimal import Decimal -import threading -import time -import traceback -from typing import Any, Callable, Iterable, Protocol - -from lab.auxpow import vardiff -from lab.prism.coordinator_config import ( - DEFAULT_PRISM_VARDIFF_RESUME_MAX_ENTRIES, - DEFAULT_PRISM_VARDIFF_RESUME_MAX_START_FACTOR, - DEFAULT_PRISM_VARDIFF_RESUME_TTL_SECONDS, - StratumListenerProfile, -) -from lab.prism.job_bundle import CachedJobBundle, JobBuildSuperseded -from lab.prism.job_delivery import PrismJobContext -from lab.prism.stratum_session import ( - ClientState, - WorkerIdentity, - client_vardiff_lock, -) -from lab.prism.worker_difficulty_store import ( - MemoryWorkerDifficultyStore, - PostgresWorkerDifficultyStore, - WorkerDifficultyStorePort, -) - - -PRISM_VARDIFF_IDLE_RETARGET_MAX_WORKERS = 2 -MAX_PENDING_VARDIFF_IDLE_RETARGETS = 8 -# Hard bound on the durable persistence lane. Entries coalesce by worker -# identity, so this bounds DISTINCT pending workers, not the event rate. -MAX_PENDING_VARDIFF_DURABLE_WRITES = 256 -PRISM_VARDIFF_DURABLE_PRUNE_INTERVAL_SECONDS = 300.0 -# TTL pruning deletes in bounded batches rather than one unbounded statement, -# so a large inherited backlog cannot produce a single delete that outruns the -# ledger's operation deadline and then never converges. Several batches run per -# pass; whatever is left waits for the next interval. -PRISM_VARDIFF_DURABLE_PRUNE_BATCH = 1024 -PRISM_VARDIFF_DURABLE_PRUNE_MAX_BATCHES = 8 -# How long the lane's worker parks between wake-ups. Work arrives by event; -# this only bounds how long a stop request waits for the loop to notice. -PRISM_VARDIFF_DURABLE_IDLE_WAKE_SECONDS = 1.0 -# Mirrors the coordinator's PRISM_JOB_BUILD_SECONDS_BUCKETS values; owned -# here so metric formatting does not import the coordinator module. -PRISM_VARDIFF_IDLE_SECONDS_BUCKETS = ( - 0.01, - 0.025, - 0.05, - 0.1, - 0.25, - 0.5, - 1.0, - 2.5, - 5.0, - 10.0, - 30.0, -) -PRISM_VARDIFF_IDLE_SKIP_REASONS = ( - "busy", - "disconnected", - "not_idle", - "cache_miss", - "queue_full", - "superseded", -) -# Bounded outcomes for one retarget taken under the fast-arrival initial -# convergence policy; also the metric label order for -# qbit_prism_vardiff_initial_retargets_total. Their sum is the attempt count -# published alongside them. -PRISM_VARDIFF_INITIAL_RETARGET_OUTCOMES = ( - "applied", # committed together with its paired job - "suppressed", # the computed step landed inside the retarget tolerance - "superseded", # difficulty or job state moved before the paired send - "failed", # the build or send raised; speculative state was restored -) -# Seconds and accepted-share buckets for the high-difficulty arrival -# histograms. Fixed bucket sets, no per-worker labels: the whole point is to -# see how long a rental-scale session spends below the threshold it should -# be credited at, fleet-wide. -PRISM_VARDIFF_HIGH_DIFF_ARRIVAL_SECONDS_BUCKETS = ( - 1.0, - 2.5, - 5.0, - 10.0, - 30.0, - 60.0, - 120.0, - 300.0, - 600.0, -) -PRISM_VARDIFF_HIGH_DIFF_ARRIVAL_SHARES_BUCKETS = ( - 1, - 2, - 4, - 8, - 16, - 32, - 64, - 128, - 256, - 512, -) -# Bounded outcomes for one reconnect difficulty resume attempt; also the -# metric label order for qbit_prism_vardiff_resume_total. -PRISM_VARDIFF_RESUME_OUTCOMES = ( - "resumed", # retained value adopted unchanged - "clamped", # retained value adopted after being pulled into bounds - "overridden", # adopted, then superseded by an explicit difficulty request - "expired", # entry existed, TTL had passed - "miss", # no entry for this key - "rejected", # entry present but unusable (non-finite / non-positive) - "disabled", # retention off, or vardiff disabled for this client -) -# Bounded outcomes for one durable worker-difficulty write. The first two are -# evidence-carrying upserts; the next two are the lower-only correction, whose -# "unchanged" means there was no higher stored row left to correct. -PRISM_VARDIFF_DURABLE_WRITE_OUTCOMES = ( - "applied", - "stale", - "lowered", - "unchanged", - "failed", - "dropped", -) - - -@dataclass(frozen=True) -class _PendingDurableWrite: - """One queued durable worker-difficulty write, keyed by worker identity. - - ``downward_only`` selects the atomic lower-only correction, which carries - no evidence timestamp: it reduces an existing row without touching - ``evidence_at``, so it can neither refresh the TTL of a value no share - proved nor refuse a later share-backed write whose evidence predates it. - """ - - key: tuple[str, str] - downward_only: bool - difficulty: Decimal - evidence_at_ms: int | None - now_ms: int - # A coalesced evidence write may also carry the minimum correction that - # must run if the store rejects that evidence as stale. Keeping this - # branch separate is load-bearing: folding it into ``difficulty`` loses - # the correction when a newer row already stands in the store. - stale_downward_difficulty: Decimal | None = None - stale_downward_now_ms: int | None = None - - @property - def has_downward_correction(self) -> bool: - return self.downward_only or self.stale_downward_difficulty is not None - - def _stale_downward(self) -> tuple[Decimal, int] | None: - if self.downward_only: - return (self.difficulty, self.now_ms) - if self.stale_downward_difficulty is None: - return None - return ( - self.stale_downward_difficulty, - int( - self.now_ms - if self.stale_downward_now_ms is None - else self.stale_downward_now_ms - ), - ) - - @staticmethod - def _minimum_downward( - earlier: tuple[Decimal, int] | None, - later: tuple[Decimal, int] | None, - ) -> tuple[Decimal, int] | None: - """Collapse lower-only writes without inventing an update timestamp. - - The first occurrence of the minimum wins: a later equal or higher - correction is a no-op after the earlier minimum has landed. - """ - if earlier is None: - return later - if later is None or earlier[0] <= later[0]: - return earlier - return later - - @classmethod - def _evidence_write( - cls, - source: "_PendingDurableWrite", - *, - difficulty: Decimal | None = None, - now_ms: int | None = None, - stale_downward: tuple[Decimal, int] | None = None, - ) -> "_PendingDurableWrite": - return cls( - key=source.key, - downward_only=False, - difficulty=source.difficulty if difficulty is None else difficulty, - evidence_at_ms=source.evidence_at_ms, - now_ms=source.now_ms if now_ms is None else now_ms, - stale_downward_difficulty=( - None if stale_downward is None else stale_downward[0] - ), - stale_downward_now_ms=( - None if stale_downward is None else stale_downward[1] - ), - ) - - def merged_with(self, later: "_PendingDurableWrite") -> "_PendingDurableWrite": - """Collapse this entry and a later one for the same worker. - - Evidence order, not queue order, selects the upsert that can stand. - Lower-only corrections are retained as a stale-upsert fallback when - an evidence write follows them, because a newer row already in the - store can reject every queued upsert while still needing the lower. - This is a compact representation of both possible store branches: - evidence accepted, or all queued evidence rejected as stale. - """ - if self.key != later.key: - raise ValueError("durable writes for different workers cannot merge") - - if later.downward_only: - if self.downward_only: - minimum = self._minimum_downward( - self._stale_downward(), - later._stale_downward(), - ) - assert minimum is not None - return _PendingDurableWrite( - key=self.key, - downward_only=True, - difficulty=minimum[0], - evidence_at_ms=None, - now_ms=minimum[1], - ) - stale_downward = self._minimum_downward( - self._stale_downward(), - later._stale_downward(), - ) - difficulty = self.difficulty - now_ms = self.now_ms - if later.difficulty < difficulty: - difficulty = later.difficulty - now_ms = later.now_ms - return self._evidence_write( - self, - difficulty=difficulty, - now_ms=now_ms, - stale_downward=stale_downward, - ) - - if self.downward_only: - stale_downward = self._minimum_downward( - self._stale_downward(), - later._stale_downward(), - ) - return self._evidence_write( - later, - stale_downward=stale_downward, - ) - - self_evidence = int(self.evidence_at_ms or 0) - later_evidence = int(later.evidence_at_ms or 0) - if later_evidence >= self_evidence: - stale_downward = self._minimum_downward( - self._stale_downward(), - later._stale_downward(), - ) - return self._evidence_write( - later, - stale_downward=stale_downward, - ) - - # ``later`` carries older evidence and is rejected whenever ``self`` - # applies. If ``later`` already represents a coalesced sequence, only - # its stale-upsert lower branch remains observable after ``self``. - later_stale_downward = later._stale_downward() - stale_downward = self._minimum_downward( - self._stale_downward(), - later_stale_downward, - ) - difficulty = self.difficulty - now_ms = self.now_ms - if ( - later_stale_downward is not None - and later_stale_downward[0] < difficulty - ): - difficulty, now_ms = later_stale_downward - return self._evidence_write( - self, - difficulty=difficulty, - now_ms=now_ms, - stale_downward=stale_downward, - ) - - -@dataclass(frozen=True) -class RetainedSessionDifficulty: - difficulty: Decimal - recorded_monotonic: float - - -def session_difficulty_key(client: ClientState) -> tuple[str, str] | None: - """The retention identity for one session: (listener name, exact username). - - Scoping by lane keeps the two lanes' resume policies independent: a - reconnect to the same port (the reconnect-storm case) always hits, while - a lane switch is simply a miss that behaves exactly like today. ``None`` - means the connection has no reserved worker yet, so nothing is retained. - """ - worker = getattr(client, "worker", None) - if worker is None: - return None - return (str(getattr(client, "listener_name", "default")), worker.username) - - -class SessionDifficultyStore: - """Bounded, TTL'd, in-process map of last-converged share difficulty. - - Keyed by :func:`session_difficulty_key`. The internal lock is a leaf - lock: nothing here calls back into the coordinator or client locks while - holding it. Retention is disabled entirely (``record`` never stores, - ``lookup`` always answers ``(None, "disabled")``) when ``max_entries`` or - ``ttl_seconds`` is non-positive. - - In-memory only: the store does not survive a coordinator restart, - including the liveness watchdog's terminal ``exit_process(1)`` in - ``background_services.py``. The mass-reconnect wave right after such a - restart therefore finds an empty store and every session starts at lane - start, unsmoothed by retention. - """ - - def __init__(self, *, max_entries: int, ttl_seconds: float) -> None: - self.max_entries = int(max_entries) - self.ttl_seconds = float(ttl_seconds) - self._lock = threading.Lock() - self._entries: OrderedDict[tuple[str, str], RetainedSessionDifficulty] = ( - OrderedDict() - ) - self.record_count = 0 - self.hit_count = 0 - self.expired_count = 0 - self.miss_count = 0 - self.evicted_count = 0 - - @property - def enabled(self) -> bool: - return self.max_entries > 0 and self.ttl_seconds > 0 - - def record( - self, - key: tuple[str, str], - difficulty: Decimal, - *, - now: float, - share_backed: bool, - ) -> None: - """Retain ``difficulty`` for ``key``, ageing the entry honestly. - - The stored difficulty is always updated, but ``recorded_monotonic`` - (what the TTL measures) is refreshed only when the value was - genuinely re-validated: a first record, a value that actually moved, - or an unchanged value backed by accepted shares. Re-recording an - unchanged value with no share evidence -- a session that resumed a - retained difficulty, submitted nothing and disconnected -- keeps the - original timestamp, so a reconnect loop shorter than the TTL can no - longer keep a stale value alive forever. LRU recency is independent - of the TTL and still moves on every record, exactly as on ``lookup``. - """ - if not self.enabled: - return - if ( - not isinstance(difficulty, Decimal) - or not difficulty.is_finite() - or difficulty <= 0 - ): - return - with self._lock: - existing = self._entries.get(key) - if ( - existing is None - or difficulty != existing.difficulty - or share_backed - ): - recorded_monotonic = float(now) - else: - recorded_monotonic = existing.recorded_monotonic - self._entries[key] = RetainedSessionDifficulty( - difficulty=difficulty, - recorded_monotonic=recorded_monotonic, - ) - self._entries.move_to_end(key) - self.record_count += 1 - while len(self._entries) > self.max_entries: - self._entries.popitem(last=False) - self.evicted_count += 1 - - def lookup( - self, - key: tuple[str, str], - *, - now: float, - ) -> tuple[Decimal | None, str]: - if not self.enabled: - return None, "disabled" - with self._lock: - retained = self._entries.get(key) - if retained is None: - self.miss_count += 1 - return None, "miss" - if now - retained.recorded_monotonic > self.ttl_seconds: - self._entries.pop(key, None) - self.expired_count += 1 - return None, "expired" - # A hit refreshes LRU recency only. The TTL measures the age of - # the retained value, not of the last access, so - # recorded_monotonic stays untouched; and the entry survives the - # hit because a miner may re-authorize. - self._entries.move_to_end(key) - self.hit_count += 1 - return retained.difficulty, "hit" - - def preload( - self, - entries: Iterable[ - tuple[tuple[str, str], Decimal, float] - ], - ) -> int: - """Seed bounded retained values without counting them as live writes. - - ``recorded_monotonic`` is reconstructed by the durable integration - layer from each wall-clock evidence timestamp. Callers provide oldest - first so the resulting LRU order still has the newest evidence at the - end. Invalid rows are ignored defensively; the durable stores already - validate them on write and parse. - """ - if not self.enabled: - return 0 - loaded = 0 - with self._lock: - for key, difficulty, recorded_monotonic in entries: - if ( - not isinstance(difficulty, Decimal) - or not difficulty.is_finite() - or difficulty <= 0 - ): - continue - self._entries[key] = RetainedSessionDifficulty( - difficulty=difficulty, - recorded_monotonic=float(recorded_monotonic), - ) - self._entries.move_to_end(key) - loaded += 1 - while len(self._entries) > self.max_entries: - self._entries.popitem(last=False) - self.evicted_count += 1 - return loaded - - def prune(self, *, now: float) -> int: - if not self.enabled: - return 0 - with self._lock: - expired_keys = [ - key - for key, retained in self._entries.items() - if now - retained.recorded_monotonic > self.ttl_seconds - ] - for key in expired_keys: - del self._entries[key] - self.expired_count += len(expired_keys) - return len(expired_keys) - - def snapshot(self) -> dict[str, int]: - with self._lock: - return { - "entries": len(self._entries), - "records": self.record_count, - "hits": self.hit_count, - "expired": self.expired_count, - "misses": self.miss_count, - "evicted": self.evicted_count, - } - - def __len__(self) -> int: - with self._lock: - return len(self._entries) - - -@dataclass(frozen=True) -class IdleRetargetRequest: - """Immutable idle-window identity captured by one bounded sweep.""" - - client: ClientState - connection_id: int - worker: WorkerIdentity - active_job: PrismJobContext - window_started_monotonic: float - current_difficulty: Decimal - elapsed_seconds: Decimal - - -class VardiffRuntime(Protocol): - """Coordinator operations needed at vardiff's ownership boundary. - - Adjacent reconcile, retained-job, tip-epoch, payout-fence, and heartbeat - state stays with its owning modules; the service reaches all of it - through these call-time coordinator seams so monkeypatched instances - keep intercepting. - """ - - lock: Any - clients: set[ClientState] - stop_event: threading.Event - vardiff_config: vardiff.VardiffConfig - share_difficulty: Decimal - vardiff_idle_sweep_seconds: float - vardiff_resume_enabled: bool - vardiff_resume_ttl_seconds: float - vardiff_resume_max_entries: int - vardiff_resume_max_start_factor: Decimal - - def client_can_receive_jobs(self, client: ClientState) -> bool: ... - - def _record_heartbeat(self, name: str) -> None: ... - - def _record_vardiff_idle_skip(self, reason: str) -> None: ... - - def _cached_idle_job_bundle(self, client: ClientState) -> CachedJobBundle | None: ... - - def _build_idle_job_bundle( - self, - request: IdleRetargetRequest, - ) -> CachedJobBundle: ... - - def _idle_bundle_current_locked( - self, - client: ClientState, - bundle: CachedJobBundle, - *, - allow_uncached: bool = False, - ) -> bool: ... - - def pool_readiness_latched(self) -> bool: ... - - def ensure_reorg_reconciled_for_current_tip(self) -> bool: ... - - def disconnect_client(self, client: ClientState) -> None: ... - - def _maybe_send_job_locked(self, client: ClientState, **kwargs: object) -> bool: ... - - def maybe_send_job(self, client: ClientState, *, clean_jobs: bool) -> bool: ... - - def retarget_client(self, client: ClientState, **kwargs: object) -> bool: ... - - -class VardiffCompatibilityField: - """Temporary coordinator view over state owned by ``VardiffService``.""" - - def __init__(self, name: str, default: Callable[[], Any]) -> None: - self.name = name - self.default = default - - def __get__(self, instance: Any, owner: type[Any]) -> Any: - if instance is None: - return self - init_lock = instance.__dict__.setdefault( - "_vardiff_service_init_lock", - threading.RLock(), - ) - with init_lock: - service = instance.__dict__.get("_vardiff_service") - if service is not None: - return getattr(service, self.name) - if self.name not in instance.__dict__: - instance.__dict__[self.name] = self.default() - return instance.__dict__[self.name] - - def __set__(self, instance: Any, value: Any) -> None: - init_lock = instance.__dict__.setdefault( - "_vardiff_service_init_lock", - threading.RLock(), - ) - with init_lock: - service = instance.__dict__.get("_vardiff_service") - if service is not None: - setattr(service, self.name, value) - else: - instance.__dict__[self.name] = value - - -VARDIFF_COMPATIBILITY_FIELDS = ( - "idle_retarget_count", - "_vardiff_idle_lock", - "_vardiff_idle_executor", - "_vardiff_idle_executor_shutdown", - "_vardiff_idle_pending", - "vardiff_idle_queue_depth", - "vardiff_idle_inflight", - "vardiff_idle_clients_inspected", - "vardiff_idle_skip_counts", - "vardiff_idle_task_failures", - "vardiff_idle_sweep_histogram", - "vardiff_idle_task_histogram", -) - - -def _new_bucket_histogram(buckets: tuple[float, ...]) -> dict[str, Any]: - return { - "buckets": {bucket: 0 for bucket in buckets}, - "sum": 0.0, - "count": 0, - } - - -def _new_histogram() -> dict[str, Any]: - return _new_bucket_histogram(PRISM_VARDIFF_IDLE_SECONDS_BUCKETS) - - -def _observe_bucket_histogram( - histogram: dict[str, Any], - buckets: tuple[float, ...], - value: float, -) -> None: - """Record one observation. Caller holds the histogram's owning lock.""" - histogram["count"] = int(histogram["count"]) + 1 - histogram["sum"] = float(histogram["sum"]) + float(value) - counts = histogram["buckets"] - for bucket in buckets: - if value <= bucket: - counts[bucket] = int(counts.get(bucket, 0)) + 1 - - -def _histogram_lines( - metric_name: str, - description: str, - histogram: dict[str, Any], - buckets: tuple[float, ...], -) -> list[str]: - return [ - f"# HELP {metric_name} {description}", - f"# TYPE {metric_name} histogram", - *[ - f'{metric_name}_bucket{{le="{bucket:g}"}} {histogram["buckets"].get(bucket, 0)}' - for bucket in buckets - ], - f'{metric_name}_bucket{{le="+Inf"}} {histogram["count"]}', - f'{metric_name}_sum {float(histogram["sum"]):.6f}', - f'{metric_name}_count {histogram["count"]}', - ] - - -def _worker_difficulty_store(runtime: VardiffRuntime) -> WorkerDifficultyStorePort: - """Resolve one process-wide durable-store adapter for the runtime. - - Focused embedders may inject ``worker_difficulty_store`` directly. In - production the adapter is attached to the ledger so recreating the - vardiff service in-process does not discard the memory reference store, - while PostgreSQL remains the cross-process source of truth. - """ - injected = getattr(runtime, "worker_difficulty_store", None) - if injected is not None: - return injected - ledger = getattr(runtime, "ledger", None) - owner = ledger if ledger is not None else runtime - existing = getattr(owner, "_worker_difficulty_store", None) - if existing is not None: - return existing - if ( - ledger is not None - and getattr(ledger, "backend_name", "") == "postgres-psql" - ): - store: WorkerDifficultyStorePort = ( - PostgresWorkerDifficultyStore.from_share_ledger( - ledger, - timeout_seconds=float( - getattr(runtime, "share_commit_timeout_seconds", 15.0) - ), - ) - ) - else: - store = MemoryWorkerDifficultyStore() - setattr(owner, "_worker_difficulty_store", store) - return store - - -class VardiffService: - """Own vardiff client windows and its bounded idle-retarget machinery.""" - - def __init__(self, runtime: VardiffRuntime) -> None: - self.runtime = runtime - self.idle_retarget_count = 0 - self._vardiff_idle_lock = threading.Lock() - self._vardiff_idle_executor: ThreadPoolExecutor | None = None - self._vardiff_idle_executor_shutdown = False - self._vardiff_idle_pending: set[tuple[ClientState, int]] = set() - self.vardiff_idle_queue_depth = 0 - self.vardiff_idle_inflight = 0 - self.vardiff_idle_clients_inspected = 0 - self.vardiff_idle_skip_counts = { - reason: 0 for reason in PRISM_VARDIFF_IDLE_SKIP_REASONS - } - self.vardiff_idle_task_failures = 0 - self.vardiff_idle_sweep_histogram = _new_histogram() - self.vardiff_idle_task_histogram = _new_histogram() - # Reconnect difficulty retention. Sized once from the runtime's - # loaded configuration (getattr keeps lightweight embedder runtimes - # working); vardiff_resume_enabled=False behaves exactly as - # max_entries=0, so the store never records and always answers - # "disabled". - resume_enabled = bool(getattr(runtime, "vardiff_resume_enabled", True)) - self.session_difficulty_store = SessionDifficultyStore( - max_entries=( - int( - getattr( - runtime, - "vardiff_resume_max_entries", - DEFAULT_PRISM_VARDIFF_RESUME_MAX_ENTRIES, - ) - ) - if resume_enabled - else 0 - ), - ttl_seconds=float( - getattr( - runtime, - "vardiff_resume_ttl_seconds", - DEFAULT_PRISM_VARDIFF_RESUME_TTL_SECONDS, - ) - ), - ) - # Leaf lock for the convergence counters below (resume outcomes and - # per-lane accepted shares); never call out of this module while - # holding it, and never take it under the store's lock. - self._vardiff_convergence_lock = threading.Lock() - self.vardiff_resume_outcome_counts = { - outcome: 0 for outcome in PRISM_VARDIFF_RESUME_OUTCOMES - } - self.vardiff_lane_accepted_counts: dict[str, int] = {} - self.vardiff_initial_retarget_attempts = 0 - self.vardiff_initial_retarget_outcome_counts = { - outcome: 0 for outcome in PRISM_VARDIFF_INITIAL_RETARGET_OUTCOMES - } - self.vardiff_high_diff_arrival_seconds_histogram = _new_bucket_histogram( - PRISM_VARDIFF_HIGH_DIFF_ARRIVAL_SECONDS_BUCKETS - ) - self.vardiff_high_diff_arrival_shares_histogram = _new_bucket_histogram( - PRISM_VARDIFF_HIGH_DIFF_ARRIVAL_SHARES_BUCKETS - ) - self.worker_difficulty_store = _worker_difficulty_store(runtime) - self.vardiff_durable_preload_records = 0 - self.vardiff_durable_preload_failures = 0 - self.vardiff_durable_prune_failures = 0 - self.vardiff_durable_pruned_records = 0 - self.vardiff_durable_coalesced = 0 - self.vardiff_durable_downward_dropped = 0 - self.vardiff_durable_write_outcome_counts = { - outcome: 0 for outcome in PRISM_VARDIFF_DURABLE_WRITE_OUTCOMES - } - # Durable writes leave the Stratum/client lock path through this - # bounded, coalescing, single-worker lane. Nothing here reads client - # state or waits on a client lock: every payload is snapshotted by the - # caller, so a slow database can never reach back into share - # accounting or job delivery. The queue is capped by DISTINCT worker - # identity, and safety-critical downward corrections outrank optional - # resume hints for both eviction and shutdown drain order. - self._vardiff_durable_lock = threading.Lock() - self._vardiff_durable_pending: OrderedDict[ - tuple[str, str], _PendingDurableWrite - ] = OrderedDict() - self._vardiff_durable_thread: threading.Thread | None = None - self._vardiff_durable_wake = threading.Event() - self._vardiff_durable_stopping = False - self._vardiff_durable_prune_requested = False - self._vardiff_durable_inflight = 0 - self._vardiff_durable_operation_timeout_seconds = float( - getattr(runtime, "share_commit_timeout_seconds", 15.0) - ) - self._vardiff_durable_next_prune_monotonic = ( - time.monotonic() + PRISM_VARDIFF_DURABLE_PRUNE_INTERVAL_SECONDS - ) - self._preload_worker_difficulties() - - def _preload_worker_difficulties(self) -> None: - """Hydrate the authorization-time map with one bounded durable read.""" - store = self.session_difficulty_store - if not store.enabled: - return - now_ms = max(0, int(time.time() * 1000)) - ttl_ms = max(0, int(store.ttl_seconds * 1000)) - cutoff_ms = max(0, now_ms - ttl_ms) - try: - recent = self.worker_difficulty_store.load_recent( - evidence_after_ms=cutoff_ms, - limit=store.max_entries, - ) - except Exception: - self.vardiff_durable_preload_failures += 1 - return - now_monotonic = time.monotonic() - # load_recent is newest-first; preload oldest-first so LRU recency - # ends in the same order as evidence freshness. - entries = [ - ( - (record.listener, record.worker_username), - record.difficulty, - now_monotonic - - max(0.0, (now_ms - record.evidence_at_ms) / 1000.0), - ) - for record in reversed(recent) - ] - self.vardiff_durable_preload_records = store.preload(entries) - # Retention is an optimization over the safe cold-start path, so a - # prune failure must not prevent mining or discard the records already - # loaded into the process-local map; the durable lane's own scheduler - # retries it on the ordinary interval. - self.vardiff_durable_pruned_records = self._prune_expired_rows(cutoff_ms) - - def client_config(self, client: ClientState) -> vardiff.VardiffConfig: - """The difficulty policy for one client: its per-client specialization - if any, else its listener profile, else the default listener's config - (clients created without one: tests, legacy callers).""" - with client_vardiff_lock(client): - return ( - client.vardiff_config - or client.listener_vardiff_config - or self.runtime.vardiff_config - ) - - def startup_difficulty( - self, - profile: StratumListenerProfile | None = None, - ) -> Decimal: - config = ( - profile.vardiff_config - if profile is not None - else self.runtime.vardiff_config - ) - fixed_difficulty = ( - profile.share_difficulty - if profile is not None - else self.runtime.share_difficulty - ) - if not config.enabled: - return fixed_difficulty - return vardiff.clamp( - config.startup_difficulty, - config.min_difficulty, - config.max_difficulty, - ) - - @staticmethod - def desired_difficulty(client: ClientState) -> Decimal: - # pending_share_difficulty is set by vardiff retargets and by explicit - # difficulty requests (d=/suggest_difficulty); either way it applies to - # the next stamped job regardless of whether vardiff is enabled. - with client_vardiff_lock(client): - return client.pending_share_difficulty or client.share_difficulty - - def minimum_advertised_difficulty(self, client: ClientState) -> Decimal: - """The difficulty stamped jobs never advertise below for this client. - - Zero everywhere except floor-bearing listeners (the high-diff port), - where the effective policy floor governs: the listener minimum, raised - by any md= specialization. The floor overrides the network-difficulty - cap because the listener's marketplace contract is checked against the - first advertised difficulty, even while qbit network difficulty sits - below the floor. - """ - with client_vardiff_lock(client): - if client.minimum_advertised_difficulty <= 0: - return Decimal("0") - config = ( - client.vardiff_config - or client.listener_vardiff_config - or self.runtime.vardiff_config - ) - return max(client.minimum_advertised_difficulty, config.min_difficulty) - - def note_submitted(self, client: ClientState) -> None: - with client_vardiff_lock(client): - config = ( - client.vardiff_config - or client.listener_vardiff_config - or self.runtime.vardiff_config - ) - if not config.enabled: - return - client.vardiff_window_submitted += 1 - - def note_accepted(self, client: ClientState, share_difficulty: Decimal) -> None: - now = time.monotonic() - # Lane share-rate accounting must stay observable even where vardiff - # is off, so count before the enabled early-out below. - lane = str(getattr(client, "listener_name", "default")) - with self._vardiff_convergence_lock: - self.vardiff_lane_accepted_counts[lane] = ( - self.vardiff_lane_accepted_counts.get(lane, 0) + 1 - ) - with client_vardiff_lock(client): - # Per-connection evidence for reconnect retention, set before the - # enabled early-out so it holds where vardiff is disabled too. - # Never reset within a connection: one accepted share is enough to - # prove the difficulty this session is running at is real, which - # is what lets disconnect refresh the retained value's TTL. - client.vardiff_accepted_any = True - client.vardiff_last_accepted_difficulty = share_difficulty - client.vardiff_last_accepted_wall_ms = max( - 0, - int(time.time() * 1000), - ) - config = ( - client.vardiff_config - or client.listener_vardiff_config - or self.runtime.vardiff_config - ) - if not config.enabled: - return - client.vardiff_window_accepted += 1 - client.vardiff_window_work += share_difficulty - client.vardiff_session_accepted = int( - getattr(client, "vardiff_session_accepted", 0) - ) + 1 - elapsed_seconds = Decimal( - str(max(0.001, now - client.vardiff_window_started_monotonic)) - ) - current_difficulty = ( - client.pending_share_difficulty or client.share_difficulty - ) - # Fast arrival: a session that has not yet had a retarget commit - # may cross the ordinary interval early, but only on decisive - # evidence. Both the early trigger and the larger step bound come - # from the same predicate, so a session can never take the wider - # step on evidence that was not good enough to trigger it. - initial_evaluation = bool( - getattr(client, "vardiff_initial_convergence_pending", True) - and not getattr( - client, - "vardiff_initial_convergence_evaluated", - False, - ) - and client.vardiff_window_accepted - >= config.initial_min_accepted_shares - and elapsed_seconds >= config.initial_min_elapsed_seconds - ) - if initial_evaluation: - client.vardiff_initial_convergence_evaluated = True - initial_convergence = initial_evaluation and vardiff.initial_convergence_ready( - current_difficulty=current_difficulty, - accepted_shares=client.vardiff_window_accepted, - accepted_difficulty=client.vardiff_window_work, - elapsed_seconds=elapsed_seconds, - config=config, - ) - if ( - elapsed_seconds < config.retarget_interval_seconds - and not initial_convergence - ): - return - accepted_shares = client.vardiff_window_accepted - submitted_shares = client.vardiff_window_submitted - accepted_difficulty = client.vardiff_window_work - client.vardiff_window_started_monotonic = now - client.vardiff_window_accepted = 0 - client.vardiff_window_submitted = 0 - client.vardiff_window_work = Decimal("0") - applied = False - try: - applied = bool( - self.runtime.retarget_client( - client, - current_difficulty=current_difficulty, - accepted_shares=accepted_shares, - submitted_shares=submitted_shares, - accepted_difficulty=accepted_difficulty, - elapsed_seconds=elapsed_seconds, - initial_convergence=initial_convergence, - ) - ) - finally: - if initial_convergence and not applied: - with client_vardiff_lock(client): - if getattr( - client, - "vardiff_initial_convergence_pending", - True, - ): - client.vardiff_initial_convergence_evaluated = False - - def _count_resume_outcome(self, outcome: str) -> None: - if outcome not in PRISM_VARDIFF_RESUME_OUTCOMES: - raise ValueError(f"unknown vardiff resume outcome: {outcome}") - with self._vardiff_convergence_lock: - self.vardiff_resume_outcome_counts[outcome] += 1 - - def _count_initial_retarget_attempt(self) -> None: - with self._vardiff_convergence_lock: - self.vardiff_initial_retarget_attempts += 1 - - def _count_initial_retarget_outcome(self, outcome: str) -> None: - if outcome not in PRISM_VARDIFF_INITIAL_RETARGET_OUTCOMES: - raise ValueError(f"unknown vardiff initial retarget outcome: {outcome}") - with self._vardiff_convergence_lock: - self.vardiff_initial_retarget_outcome_counts[outcome] += 1 - - def observe_high_diff_arrival( - self, - client: ClientState, - *, - config: vardiff.VardiffConfig, - previous_difficulty: Decimal, - next_difficulty: Decimal, - ) -> bool: - """Observe this connection's first vardiff crossing of the high-diff - threshold, in seconds and in accepted shares. - - Observation only -- the threshold never gates or clamps a retarget. - Recorded at most once per connection, and only for a retarget that - carried the session from below the threshold to at or above it, so a - lane that already starts above it (the high-diff listener) never - contributes and a step-down followed by a re-crossing does not - double count. Returns whether an observation was recorded. - - The client's vardiff lock is released before the counter lock is - taken: this module never nests the two in either direction. - """ - threshold = getattr( - config, - "high_diff_arrival_threshold", - Decimal("0"), - ) - if ( - not threshold.is_finite() - or threshold <= 0 - or previous_difficulty >= threshold - or next_difficulty < threshold - ): - return False - with client_vardiff_lock(client): - if getattr(client, "vardiff_high_diff_arrival_recorded", False): - return False - client.vardiff_high_diff_arrival_recorded = True - started = getattr(client, "vardiff_session_started_monotonic", None) - accepted = int(getattr(client, "vardiff_session_accepted", 0)) - with self._vardiff_convergence_lock: - if started is not None: - _observe_bucket_histogram( - self.vardiff_high_diff_arrival_seconds_histogram, - PRISM_VARDIFF_HIGH_DIFF_ARRIVAL_SECONDS_BUCKETS, - max(0.0, time.monotonic() - float(started)), - ) - _observe_bucket_histogram( - self.vardiff_high_diff_arrival_shares_histogram, - PRISM_VARDIFF_HIGH_DIFF_ARRIVAL_SHARES_BUCKETS, - float(accepted), - ) - return True - - def record_session_difficulty( - self, - client: ClientState, - difficulty: Decimal | None = None, - *, - share_backed: bool, - previous_difficulty: Decimal | None = None, - initial_convergence: bool = False, - ) -> None: - """Retain a session's converged difficulty for reconnect resume. - - Callers pass the just-committed retarget difficulty explicitly; the - disconnect seam omits it and the client's DELIVERED difficulty is - read under its vardiff lock instead. pending_share_difficulty is - deliberately not consulted here: it is advertised with a future job, - so a disconnect racing an in-flight retarget would otherwise retain a - value the miner was never given. Nothing is lost -- a retarget whose - paired send completed records its own value at that commit point. - - ``share_backed`` says whether this difficulty is backed by accepted - shares; the process-local store uses it to decide whether re-recording - an unchanged value may refresh its TTL. Durable writes are stricter: - a committed retarget may carry forward the share evidence that drove - its estimate, while a disconnect is durable only when the last - accepted share was stamped at the exact delivered difficulty. This - prevents an unproven explicit step-up from surviving a restart. - - A committed retarget that LOWERS the difficulty is durable even with - no accepted shares behind it, but it goes out as an atomic lower-only - correction rather than a write of new evidence: leaving a stale higher - row standing would let a restart resurrect a difficulty the live - session had already abandoned, while re-stamping evidence_at would let - the correction suppress later genuine share-backed evidence. - """ - key = session_difficulty_key(client) - if key is None: - return - if not self.session_difficulty_store.enabled: - return - committed_retarget = difficulty is not None - with client_vardiff_lock(client): - if difficulty is None: - difficulty = client.share_difficulty - evidence_difficulty = getattr( - client, - "vardiff_last_accepted_difficulty", - None, - ) - evidence_at_ms = getattr( - client, - "vardiff_last_accepted_wall_ms", - None, - ) - self.session_difficulty_store.record( - key, - difficulty, - now=time.monotonic(), - share_backed=share_backed, - ) - now_ms = max(0, int(time.time() * 1000)) - downward_commit = bool( - committed_retarget - and previous_difficulty is not None - and difficulty < previous_difficulty - ) - durably_share_backed = bool( - share_backed - and evidence_at_ms is not None - and ( - (committed_retarget and not initial_convergence) - or evidence_difficulty == difficulty - ) - ) - # A share-backed write already carries this exact difficulty plus the - # evidence that proves it, so it subsumes the downward correction; the - # lower-only path is for the moves no accepted share backs, above all - # the idle step-down. - if durably_share_backed: - self._enqueue_durable_write( - _PendingDurableWrite( - key=key, - downward_only=False, - difficulty=difficulty, - evidence_at_ms=int(evidence_at_ms), - now_ms=now_ms, - ) - ) - elif downward_commit: - self._enqueue_durable_write( - _PendingDurableWrite( - key=key, - downward_only=True, - difficulty=difficulty, - evidence_at_ms=None, - now_ms=now_ms, - ) - ) - - def _count_durable_write(self, outcome: str) -> None: - if outcome not in PRISM_VARDIFF_DURABLE_WRITE_OUTCOMES: - raise ValueError(f"unknown vardiff durable write outcome: {outcome}") - with self._vardiff_convergence_lock: - self.vardiff_durable_write_outcome_counts[outcome] += 1 - - # -- bounded coalescing persistence lane ------------------------------- - - def _enqueue_durable_write(self, pending: _PendingDurableWrite) -> None: - """Hand one durable write to the lane. Never touches a client lock. - - The payload is fully snapshotted by the caller, so nothing here reads - client state and nothing waits on ``job_update_lock``: a slow database - can never reach back into share accounting or job delivery. - - Entries coalesce by worker identity, which is what keeps the queue - bounded by DISTINCT workers rather than by event rate. The merge is - order-equivalent to applying both writes in sequence -- see - :meth:`_PendingDurableWrite.merged_with`. - - After the lane is closed a step-down is applied on the calling thread - rather than queued, because the shutdown drain has already run and a - queued entry would simply be stranded. - """ - lost: _PendingDurableWrite | None = None - apply_inline = False - with self._vardiff_durable_lock: - if self._vardiff_durable_stopping: - # The lane is closed: its drain has already run, so queueing - # here would strand the entry. An optional resume hint is - # dropped, but a step-down is safety critical -- losing it - # leaves a restart resuming an abandoned difficulty -- so it - # is applied on this thread instead, bounded by the store's - # own per-statement operation deadline. - if pending.has_downward_correction: - apply_inline = True - else: - lost = pending - else: - existing = self._vardiff_durable_pending.get(pending.key) - if existing is not None: - self._vardiff_durable_pending[pending.key] = ( - existing.merged_with(pending) - ) - self.vardiff_durable_coalesced += 1 - else: - lost = self._evict_for_locked(pending) - # _evict_for_locked returns the incoming entry itself when - # it is the one refused, in which case it must not also be - # queued -- that would push the lane past its bound. - if lost is not pending: - self._vardiff_durable_pending[pending.key] = pending - self._ensure_durable_worker_locked() - if apply_inline: - # Outside the lane lock: _apply_durable_write reaches the store. - self._apply_durable_write(pending) - return - if lost is not None: - if lost.has_downward_correction: - self._count_downward_drop(lost) - else: - self._count_durable_write("dropped") - self._vardiff_durable_wake.set() - - def _evict_for_locked( - self, - incoming: _PendingDurableWrite, - ) -> _PendingDurableWrite | None: - """Make room for ``incoming`` under the hard queue bound. - - Caller holds ``_vardiff_durable_lock``. Optional resume hints are - evicted before safety-critical downward corrections, and the incoming - entry itself is refused only when every queued entry is already a - downward correction -- which needs MAX_PENDING distinct workers - stepping down while the database is unavailable. - """ - if len(self._vardiff_durable_pending) < MAX_PENDING_VARDIFF_DURABLE_WRITES: - return None - for key, queued in self._vardiff_durable_pending.items(): - if not queued.has_downward_correction: - del self._vardiff_durable_pending[key] - return queued - if not incoming.has_downward_correction: - return incoming - oldest_key = next(iter(self._vardiff_durable_pending)) - return self._vardiff_durable_pending.pop(oldest_key) - - def _count_downward_drop(self, pending: _PendingDurableWrite) -> None: - """Record -- loudly -- a downward correction the lane could not keep. - - This is the one durable-write loss that can leave a restart resuming a - difficulty the live session abandoned, so it gets its own counter and - a log line instead of folding into the generic drop count. - """ - self._count_durable_write("dropped") - with self._vardiff_convergence_lock: - self.vardiff_durable_downward_dropped += 1 - downward = pending._stale_downward() - assert downward is not None - print( - "prism coordinator: durable vardiff step-down dropped " - f"listener={pending.key[0]} worker={pending.key[1]} " - f"difficulty={downward[0]} " - "(reconnect may resume a higher difficulty until its TTL expires)", - flush=True, - ) - - def _ensure_durable_worker_locked(self) -> None: - """Start the lane's single worker thread. Caller holds the lane lock.""" - if self._vardiff_durable_thread is not None: - return - if self._vardiff_durable_stopping: - return - thread = threading.Thread( - target=self._durable_worker_loop, - name="prism-vardiff-durable", - daemon=True, - ) - self._vardiff_durable_thread = thread - thread.start() - - def start_durable_prune_scheduler(self) -> None: - """Keep TTL pruning alive even when idle vardiff retargets are off. - - Production calls this once before accepting Stratum connections. The - existing durable lane then owns both write traffic and the periodic - prune clock, without adding database work to an accept/client thread. - Disabled retention still creates no worker. - """ - if not self.session_difficulty_store.enabled: - return - with self._vardiff_durable_lock: - self._ensure_durable_worker_locked() - - def _durable_worker_loop(self) -> None: - while True: - self._vardiff_durable_wake.wait( - PRISM_VARDIFF_DURABLE_IDLE_WAKE_SECONDS - ) - self._vardiff_durable_wake.clear() - with self._vardiff_durable_lock: - stopping = self._vardiff_durable_stopping - if not stopping: - # The lane owns its own clock. In particular, this must not - # depend on vardiff_idle_sweep_seconds: operators may disable - # idle retargets without disabling durable-row retention. - self.request_durable_prune_if_due() - self._drain_durable_pending( - deadline=None if not stopping else ( - time.monotonic() + self._vardiff_durable_operation_timeout_seconds - ), - final=stopping, - ) - self._prune_durable_worker_difficulties_if_requested() - if stopping: - return - - def _next_pending_locked(self) -> _PendingDurableWrite | None: - """Pop the next entry, safety-critical corrections first. - - Caller holds ``_vardiff_durable_lock``. Downward-first ordering is - what lets a deadline-bounded shutdown drain finish the writes that - matter even when optional resume hints are still queued behind them. - """ - chosen_key: tuple[str, str] | None = None - for key, queued in self._vardiff_durable_pending.items(): - if queued.has_downward_correction: - chosen_key = key - break - if chosen_key is None: - chosen_key = next(iter(self._vardiff_durable_pending), None) - if chosen_key is None: - return None - return self._vardiff_durable_pending.pop(chosen_key) - - def _drain_durable_pending( - self, - *, - deadline: float | None, - final: bool, - ) -> None: - """Apply queued writes until the queue empties or ``deadline`` passes. - - ``deadline`` bounds the whole drain with the same budget the ledger - gives one operation; each individual statement is separately bounded - by the store's own operation timeout. On a final drain anything still - queued when the budget runs out is accounted for explicitly. - """ - while True: - if deadline is not None and time.monotonic() >= deadline: - break - with self._vardiff_durable_lock: - pending = self._next_pending_locked() - # Counted as in-flight while the lane lock is held, so a - # concurrent flush cannot observe an empty queue between the - # pop and the write landing. - self._vardiff_durable_inflight += 1 if pending is not None else 0 - if pending is None: - return - try: - self._apply_durable_write(pending) - finally: - with self._vardiff_durable_lock: - self._vardiff_durable_inflight -= 1 - if not final: - return - with self._vardiff_durable_lock: - abandoned = list(self._vardiff_durable_pending.values()) - self._vardiff_durable_pending.clear() - for pending in abandoned: - if pending.has_downward_correction: - self._count_downward_drop(pending) - else: - self._count_durable_write("dropped") - - def _apply_durable_write(self, pending: _PendingDurableWrite) -> None: - if pending.downward_only: - try: - lowered = self.worker_difficulty_store.apply_downward( - listener=pending.key[0], - worker_username=pending.key[1], - difficulty=pending.difficulty, - now_ms=pending.now_ms, - ) - outcome = "lowered" if lowered.applied else "unchanged" - except Exception: - outcome = "failed" - self._count_durable_write(outcome) - return - - try: - result = self.worker_difficulty_store.upsert( - listener=pending.key[0], - worker_username=pending.key[1], - difficulty=pending.difficulty, - evidence_at_ms=int(pending.evidence_at_ms or 0), - now_ms=pending.now_ms, - ) - except Exception: - self._count_durable_write("failed") - return - self._count_durable_write("applied" if result.applied else "stale") - stale_downward = pending._stale_downward() - if result.applied or stale_downward is None: - return - try: - lowered = self.worker_difficulty_store.apply_downward( - listener=pending.key[0], - worker_username=pending.key[1], - difficulty=stale_downward[0], - now_ms=stale_downward[1], - ) - outcome = "lowered" if lowered.applied else "unchanged" - except Exception: - outcome = "failed" - self._count_durable_write(outcome) - - def request_durable_prune_if_due(self) -> bool: - """Arm a TTL prune when one is due, without doing database work here. - - Called from the durable worker's own scheduler, so pruning is driven - by the clock rather than by successful write traffic or the optional - idle-retarget sweep. A lane whose writes all fail, or a process that - inherits rows and writes none of its own, still prunes. Returns - whether a prune was armed. - """ - if not self.session_difficulty_store.enabled: - return False - now_monotonic = time.monotonic() - with self._vardiff_durable_lock: - if now_monotonic < self._vardiff_durable_next_prune_monotonic: - return False - self._vardiff_durable_next_prune_monotonic = ( - now_monotonic + PRISM_VARDIFF_DURABLE_PRUNE_INTERVAL_SECONDS - ) - if self._vardiff_durable_stopping: - return False - self._vardiff_durable_prune_requested = True - self._ensure_durable_worker_locked() - self._vardiff_durable_wake.set() - return True - - def _prune_expired_rows(self, cutoff_ms: int) -> int: - """Delete expired rows in bounded batches; never one open-ended DELETE. - - Every statement carries a finite ``limit`` so it stays inside the - ledger's operation deadline, and batching continues only while a full - batch comes back, so an inherited backlog drains over successive - intervals instead of stalling on a delete it can never finish. - """ - pruned = 0 - for _ in range(PRISM_VARDIFF_DURABLE_PRUNE_MAX_BATCHES): - try: - deleted = self.worker_difficulty_store.prune( - evidence_cutoff_ms=cutoff_ms, - limit=PRISM_VARDIFF_DURABLE_PRUNE_BATCH, - ) - except Exception: - with self._vardiff_convergence_lock: - self.vardiff_durable_prune_failures += 1 - break - pruned += int(deleted) - if int(deleted) < PRISM_VARDIFF_DURABLE_PRUNE_BATCH: - break - return pruned - - def _prune_durable_worker_difficulties_if_requested(self) -> None: - with self._vardiff_durable_lock: - if not self._vardiff_durable_prune_requested: - return - self._vardiff_durable_prune_requested = False - now_ms = max(0, int(time.time() * 1000)) - cutoff_ms = max( - 0, - now_ms - int(self.session_difficulty_store.ttl_seconds * 1000), - ) - pruned = self._prune_expired_rows(cutoff_ms) - if not pruned: - return - with self._vardiff_convergence_lock: - self.vardiff_durable_pruned_records += pruned - - def durable_pending_depth(self) -> int: - with self._vardiff_durable_lock: - return len(self._vardiff_durable_pending) - - def durable_coalesced_count(self) -> int: - """Coalesced-entry count, read under the lane lock that maintains it.""" - with self._vardiff_durable_lock: - return int(self.vardiff_durable_coalesced) - - def flush_durable_writes(self, *, timeout: float | None = None) -> bool: - """Wait until the lane has nothing queued or in flight. - - No production caller needs this -- durable retention is deliberately - fire-and-forget so mining never waits on it. It exists so tests and - the shutdown drain can observe the lane deterministically. Returns - whether the lane went quiet within the budget. - """ - budget = ( - self._vardiff_durable_operation_timeout_seconds - if timeout is None - else float(timeout) - ) - deadline = time.monotonic() + max(0.0, budget) - self._vardiff_durable_wake.set() - while True: - with self._vardiff_durable_lock: - quiet = ( - not self._vardiff_durable_pending - and self._vardiff_durable_inflight == 0 - ) - worker_running = self._vardiff_durable_thread is not None - if quiet: - return True - if not worker_running: - # Nothing will drain this; do it on the caller's thread rather - # than spin until the budget expires. - self._drain_durable_pending(deadline=deadline, final=False) - continue - if time.monotonic() >= deadline: - return False - time.sleep(0.001) - - def shutdown_durable_executor(self) -> None: - """Close the lane, draining safety-critical corrections first. - - Queued work is drained, never cancelled: the whole point of the - downward correction is that losing it leaves a restart resuming an - abandoned difficulty. The drain is bounded by the ledger's operation - timeout so shutdown cannot stall on an unreachable database, and - anything the budget cannot cover is counted and logged. - """ - with self._vardiff_durable_lock: - already_stopped = self._vardiff_durable_stopping - self._vardiff_durable_stopping = True - thread = self._vardiff_durable_thread - self._vardiff_durable_thread = None - if already_stopped and thread is None: - return - deadline = ( - time.monotonic() + self._vardiff_durable_operation_timeout_seconds - ) - self._vardiff_durable_wake.set() - if thread is not None: - thread.join(timeout=max(0.0, deadline - time.monotonic())) - # The worker owns the drain while it lives; this covers a lane whose - # worker never started, already exited, or ran out of budget. - self._drain_durable_pending(deadline=deadline, final=True) - - def note_resume_overridden(self) -> None: - """Count a resumed difficulty that an explicit request superseded. - - resumed/clamped stay attempt counters, so - ``resumed + clamped - overridden`` is the number of resumes that - actually stuck. The caller is the authorize path: it is the only - place that knows both the value the resume applied and the target a - password ``d=``/``md=`` or a pending suggestion resolved to. - """ - self._count_resume_outcome("overridden") - - def resume_client_difficulty(self, client: ClientState) -> Decimal | None: - """The difficulty a reconnecting worker should resume at, or None. - - A retained value is only adopted inside the lane's plausibility - bounds: the ceiling is min(lane max, lane start * resume factor), so - a stale or absurd entry can never be adopted as-is. A retained value - BELOW the lane start is adopted unchanged on purpose -- vardiff - targets a share rate, so a session at its converged difficulty is on - target by definition, and cold-starting it higher would give the pool - fewer shares, not more. - """ - with client_vardiff_lock(client): - config = ( - client.vardiff_config - or client.listener_vardiff_config - or self.runtime.vardiff_config - ) - if not config.enabled: - self._count_resume_outcome("disabled") - return None - key = session_difficulty_key(client) - if key is None: - return None - retained, outcome = self.session_difficulty_store.lookup( - key, - now=time.monotonic(), - ) - if retained is None: - self._count_resume_outcome(outcome) - return None - if not retained.is_finite() or retained <= 0: - self._count_resume_outcome("rejected") - return None - factor = getattr( - self.runtime, - "vardiff_resume_max_start_factor", - DEFAULT_PRISM_VARDIFF_RESUME_MAX_START_FACTOR, - ) - ceiling = min(config.max_difficulty, config.startup_difficulty * factor) - floor = config.min_difficulty - resumed = vardiff.clamp(retained, floor, max(floor, ceiling)) - self._count_resume_outcome( - "resumed" if resumed == retained else "clamped" - ) - return resumed - - def apply_resumed_difficulty(self, client: ClientState) -> Decimal | None: - """Adopt the retained difficulty onto a freshly authorizing client.""" - with client_vardiff_lock(client): - resumed = self.resume_client_difficulty(client) - if resumed is None: - return None - current = client.pending_share_difficulty or client.share_difficulty - if resumed != current: - client.share_difficulty = resumed - client.pending_share_difficulty = None - client.difficulty_generation = int( - getattr(client, "difficulty_generation", 0) - ) + 1 - return resumed - - def convergence_snapshot(self) -> dict[str, object]: - """Copied convergence/retention counters plus the at-ceiling census. - - Lock order in this codebase is client vardiff lock -> coordinator - lock, never the reverse; membership is therefore snapshotted under - the coordinator lock alone, released, and only then is each client's - vardiff lock taken to read its effective policy and difficulty. - - The at-ceiling census is therefore O(connections) lock acquisitions - per call, bounded by the deployed 4096-connection cap. That cost is - acceptable because this runs on the background metrics refresher and - /metrics serves its cached payload, and because each vardiff lock is - held for a few field reads only -- no client vardiff lock is ever - held across a socket write, so a slow miner cannot convoy the census. - """ - with self.runtime.lock: - clients = tuple(self.runtime.clients) - sessions_at_max = 0 - for client in clients: - with client_vardiff_lock(client): - config = ( - client.vardiff_config - or client.listener_vardiff_config - or self.runtime.vardiff_config - ) - if not config.enabled: - continue - current = ( - client.pending_share_difficulty or client.share_difficulty - ) - if current >= config.max_difficulty: - sessions_at_max += 1 - with self._vardiff_convergence_lock: - lane_accepted = dict(self.vardiff_lane_accepted_counts) - resume_outcomes = dict(self.vardiff_resume_outcome_counts) - durable_writes = dict(self.vardiff_durable_write_outcome_counts) - durable_pruned = int(self.vardiff_durable_pruned_records) - durable_prune_failures = int(self.vardiff_durable_prune_failures) - durable_downward_dropped = int(self.vardiff_durable_downward_dropped) - # Prune first so the retained-sessions gauge counts only entries a - # reconnect could still adopt. - self.session_difficulty_store.prune(now=time.monotonic()) - store = self.session_difficulty_store.snapshot() - return { - "sessions_at_max_difficulty": sessions_at_max, - "lane_accepted_shares": lane_accepted, - "resume_outcomes": resume_outcomes, - "retained_sessions": int(store["entries"]), - "store": store, - "durable_store": { - "preloaded": self.vardiff_durable_preload_records, - "preload_failures": self.vardiff_durable_preload_failures, - "pruned": durable_pruned, - "prune_failures": durable_prune_failures, - "pending": self.durable_pending_depth(), - "coalesced": self.durable_coalesced_count(), - "downward_dropped": durable_downward_dropped, - "writes": durable_writes, - }, - } - - def record_idle_skip(self, reason: str) -> None: - if reason not in PRISM_VARDIFF_IDLE_SKIP_REASONS: - raise ValueError(f"unknown vardiff idle skip reason: {reason}") - with self._vardiff_idle_lock: - self.vardiff_idle_skip_counts[reason] += 1 - - def observe_idle_seconds(self, name: str, elapsed_seconds: float) -> None: - if name not in {"sweep", "task"}: - raise ValueError(f"unknown vardiff idle histogram: {name}") - with self._vardiff_idle_lock: - histogram = getattr(self, f"vardiff_idle_{name}_histogram") - histogram["count"] = int(histogram["count"]) + 1 - histogram["sum"] = float(histogram["sum"]) + elapsed_seconds - buckets = histogram["buckets"] - for bucket in PRISM_VARDIFF_IDLE_SECONDS_BUCKETS: - if elapsed_seconds <= bucket: - buckets[bucket] = int(buckets.get(bucket, 0)) + 1 - - def idle_tip_diverged_locked(self) -> bool: - """Whether detected tip work still lacks published submit authority.""" - published = getattr(self.runtime, "current_tip_first_seen", None) - latest_detected = getattr(self.runtime, "latest_detected_tip", None) - return bool( - latest_detected is not None - and (published is None or latest_detected[0] != published[0]) - ) - - def request_skip_reason( - self, - request: IdleRetargetRequest, - ) -> str | None: - client = request.client - # Take the per-client lock before coordinator admission. A share can - # delay this client's idle retarget, but it can never make the retarget - # hold the coordinator lock while waiting and convoy tip publication. - with client_vardiff_lock(client): - with self.runtime.lock: - if self.idle_tip_diverged_locked(): - return "superseded" - if ( - client not in self.runtime.clients - or getattr(client, "closing", False) - or not self.runtime.client_can_receive_jobs(client) - ): - return "disconnected" - if ( - client.connection_id != request.connection_id - or client.worker != request.worker - or client.active_job is not request.active_job - or (client.pending_share_difficulty or client.share_difficulty) - != request.current_difficulty - ): - return "superseded" - if ( - client.vardiff_window_started_monotonic - != request.window_started_monotonic - or client.vardiff_window_accepted != 0 - or client.vardiff_window_submitted != 0 - ): - return "not_idle" - return None - - def _request_pending(self, request: IdleRetargetRequest) -> bool: - with self._vardiff_idle_lock: - return ( - request.client, - request.connection_id, - ) in self._vardiff_idle_pending - - def _finish_idle_task( - self, - key: tuple[ClientState, int], - queued_monotonic: float, - *, - started: bool, - ) -> None: - with self._vardiff_idle_lock: - if key not in self._vardiff_idle_pending: - return - self._vardiff_idle_pending.discard(key) - if started: - self.vardiff_idle_inflight = max(0, self.vardiff_idle_inflight - 1) - else: - self.vardiff_idle_queue_depth = max( - 0, - self.vardiff_idle_queue_depth - 1, - ) - self.observe_idle_seconds( - "task", - max(0.0, time.monotonic() - queued_monotonic), - ) - - def _run_idle_task( - self, - request: IdleRetargetRequest, - bundle: CachedJobBundle | None, - queued_monotonic: float, - ) -> None: - key = (request.client, request.connection_id) - with self._vardiff_idle_lock: - if key not in self._vardiff_idle_pending: - return - self.vardiff_idle_queue_depth = max( - 0, - self.vardiff_idle_queue_depth - 1, - ) - self.vardiff_idle_inflight += 1 - client = request.client - delivery_attempted = False - try: - reason = self.request_skip_reason(request) - if reason is not None: - self.runtime._record_vardiff_idle_skip(reason) - return - # Readiness may have crossed in the ledger after the sweep's - # cache-only snapshot. Refresh it on this bounded worker so a - # cached collection bundle cannot be delivered after the pool is - # ready for normal payout work. - self.runtime.pool_readiness_latched() - # Canonicalize the sweep's cache-only snapshot on the dedicated - # worker. shared_job_bundle() selects the current payout-artifact - # key and rebinds a ready heavy bundle to the latest same-tip - # template observation; a miss may build here, never on the sweep. - bundle = self.runtime._build_idle_job_bundle(request) - reason = self.request_skip_reason(request) - if reason is not None: - self.runtime._record_vardiff_idle_skip(reason) - return - # Prepared bundles bypass _maybe_send_job_locked's normal build - # admission, so preserve its live reorg/headers/IBD trust guard on - # the dedicated worker before taking the client lock or sending. - if not self.runtime.ensure_reorg_reconciled_for_current_tip(): - self.runtime._record_vardiff_idle_skip("superseded") - return - if not client.job_update_lock.acquire(blocking=False): - self.runtime._record_vardiff_idle_skip("busy") - return - try: - reason = self.request_skip_reason(request) - if reason is not None: - self.runtime._record_vardiff_idle_skip(reason) - return - with self.runtime._job_cache_lock: - bundle_current = self.runtime._idle_bundle_current_locked( - client, - bundle, - allow_uncached=True, - ) - if not bundle_current: - self.runtime._record_vardiff_idle_skip("superseded") - return - # Everything above this point is coordinator preparation. An - # OSError there belongs to qbit RPC/ledger I/O, not the miner - # socket. Only retire the connection after entering the paired - # client delivery path below. - delivery_attempted = True - retargeted = self.retarget_locked( - client, - current_difficulty=request.current_difficulty, - accepted_shares=0, - submitted_shares=0, - accepted_difficulty=Decimal("0"), - elapsed_seconds=request.elapsed_seconds, - require_idle=True, - prepared_bundle=bundle, - expected_connection_id=request.connection_id, - expected_worker=request.worker, - expected_active_job=request.active_job, - expected_window_started=request.window_started_monotonic, - prepared_bundle_allow_uncached=True, - ) - finally: - client.job_update_lock.release() - if retargeted: - with self.runtime.lock: - self.idle_retarget_count = int( - getattr(self, "idle_retarget_count", 0) - ) + 1 - return - reason = self.request_skip_reason(request) - if reason is not None: - self.runtime._record_vardiff_idle_skip(reason) - return - with self.runtime._job_cache_lock: - bundle_current = self.runtime._idle_bundle_current_locked( - client, - bundle, - allow_uncached=True, - ) - if not bundle_current: - self.runtime._record_vardiff_idle_skip("superseded") - except JobBuildSuperseded: - self.runtime._record_vardiff_idle_skip("superseded") - except OSError: - with self._vardiff_idle_lock: - self.vardiff_idle_task_failures += 1 - if delivery_attempted: - self.runtime.disconnect_client(client) - return - print( - "prism coordinator: idle vardiff retarget preparation failed; " - "keeping client connected", - flush=True, - ) - traceback.print_exc() - except Exception: - with self._vardiff_idle_lock: - self.vardiff_idle_task_failures += 1 - print("prism coordinator: idle vardiff retarget task failed", flush=True) - traceback.print_exc() - - def _enqueue_idle( - self, - request: IdleRetargetRequest, - bundle: CachedJobBundle | None, - ) -> str | None: - key = (request.client, request.connection_id) - queued_monotonic = time.monotonic() - with self._vardiff_idle_lock: - if ( - self._vardiff_idle_executor_shutdown - or key in self._vardiff_idle_pending - ): - return "superseded" - if len(self._vardiff_idle_pending) >= MAX_PENDING_VARDIFF_IDLE_RETARGETS: - return "queue_full" - executor = self._vardiff_idle_executor - if executor is None: - executor = ThreadPoolExecutor( - max_workers=PRISM_VARDIFF_IDLE_RETARGET_MAX_WORKERS, - thread_name_prefix="prism-vardiff-idle", - ) - self._vardiff_idle_executor = executor - self._vardiff_idle_pending.add(key) - self.vardiff_idle_queue_depth += 1 - try: - future = executor.submit( - self._run_idle_task, - request, - bundle, - queued_monotonic, - ) - except RuntimeError: - self._vardiff_idle_pending.discard(key) - self.vardiff_idle_queue_depth = max( - 0, - self.vardiff_idle_queue_depth - 1, - ) - return "queue_full" - - def finish_task(completed: Future[None]) -> None: - self._finish_idle_task( - key, - queued_monotonic, - started=not completed.cancelled(), - ) - - future.add_done_callback(finish_task) - return None - - def shutdown_idle_executor(self) -> None: - with self._vardiff_idle_lock: - executor = self._vardiff_idle_executor - self._vardiff_idle_executor = None - self._vardiff_idle_executor_shutdown = True - if executor is not None: - executor.shutdown(wait=True, cancel_futures=True) - self.shutdown_durable_executor() - - def idle_sweep_loop(self) -> None: - while not self.runtime.stop_event.wait( - self.runtime.vardiff_idle_sweep_seconds - ): - self.runtime._record_heartbeat("vardiff_idle_sweep") - try: - queued = self.idle_sweep_once() - if queued: - print( - "prism coordinator: idle vardiff sweep queued " - f"{queued} client(s)", - flush=True, - ) - except Exception: - print("prism coordinator: idle vardiff sweep failed", flush=True) - traceback.print_exc() - - def idle_sweep_once(self) -> int: - sweep_started = time.monotonic() - now = time.monotonic() - queued = 0 - try: - with self.runtime.lock: - clients = tuple(self.runtime.clients) - with self._vardiff_idle_lock: - self.vardiff_idle_clients_inspected += len(clients) - for client in clients: - self.runtime._record_heartbeat("vardiff_idle_sweep") - with client_vardiff_lock(client), self.runtime.lock: - if self.idle_tip_diverged_locked(): - reason = "superseded" - request = None - elif ( - client not in self.runtime.clients - or not self.runtime.client_can_receive_jobs(client) - ): - reason = "disconnected" - request = None - else: - active_job = client.active_job - worker = client.worker - config = ( - client.vardiff_config - or client.listener_vardiff_config - or self.runtime.vardiff_config - ) - if not config.enabled: - continue - if active_job is None or worker is None: - reason = "superseded" - request = None - else: - elapsed = Decimal( - str( - max( - 0.001, - now - - client.vardiff_window_started_monotonic, - ) - ) - ) - if ( - elapsed < config.retarget_interval_seconds - or client.vardiff_window_accepted != 0 - or client.vardiff_window_submitted != 0 - ): - reason = "not_idle" - request = None - else: - reason = None - request = IdleRetargetRequest( - client=client, - connection_id=client.connection_id, - worker=worker, - active_job=active_job, - window_started_monotonic=( - client.vardiff_window_started_monotonic - ), - current_difficulty=( - client.pending_share_difficulty - or client.share_difficulty - ), - elapsed_seconds=elapsed, - ) - if reason is not None: - self.runtime._record_vardiff_idle_skip(reason) - continue - assert request is not None - if self._request_pending(request): - self.runtime._record_vardiff_idle_skip("superseded") - continue - if not client.job_update_lock.acquire(blocking=False): - self.runtime._record_vardiff_idle_skip("busy") - continue - try: - reason = self.request_skip_reason(request) - finally: - client.job_update_lock.release() - if reason is not None: - self.runtime._record_vardiff_idle_skip(reason) - continue - bundle = self.runtime._cached_idle_job_bundle(client) - if bundle is None: - # The sweep itself stays cache-only. A missing/expired - # bundle is rebuilt only by the dedicated bounded worker, - # so the client still makes eventual vardiff progress. - self.runtime._record_vardiff_idle_skip("cache_miss") - reason = self._enqueue_idle(request, bundle) - if reason is not None: - self.runtime._record_vardiff_idle_skip(reason) - continue - queued += 1 - return queued - finally: - self.runtime._record_heartbeat("vardiff_idle_sweep") - self.observe_idle_seconds( - "sweep", - max(0.0, time.monotonic() - sweep_started), - ) - - def retarget( - self, - client: ClientState, - *, - current_difficulty: Decimal, - accepted_shares: int, - submitted_shares: int, - accepted_difficulty: Decimal, - elapsed_seconds: Decimal, - require_idle: bool = False, - prepared_bundle: CachedJobBundle | None = None, - expected_connection_id: int | None = None, - expected_worker: WorkerIdentity | None = None, - expected_active_job: PrismJobContext | None = None, - expected_window_started: float | None = None, - initial_convergence: bool = False, - ) -> bool: - acquired = client.job_update_lock.acquire(blocking=not require_idle) - if not acquired: - return False - try: - if require_idle and prepared_bundle is None: - prepared_bundle = self.runtime._cached_idle_job_bundle(client) - if prepared_bundle is None: - return False - return self.retarget_locked( - client, - current_difficulty=current_difficulty, - accepted_shares=accepted_shares, - submitted_shares=submitted_shares, - accepted_difficulty=accepted_difficulty, - elapsed_seconds=elapsed_seconds, - require_idle=require_idle, - prepared_bundle=prepared_bundle, - expected_connection_id=expected_connection_id, - expected_worker=expected_worker, - expected_active_job=expected_active_job, - expected_window_started=expected_window_started, - initial_convergence=initial_convergence, - ) - finally: - client.job_update_lock.release() - - def retarget_locked( - self, - client: ClientState, - *, - current_difficulty: Decimal, - accepted_shares: int, - submitted_shares: int, - accepted_difficulty: Decimal, - elapsed_seconds: Decimal, - require_idle: bool = False, - prepared_bundle: CachedJobBundle | None = None, - expected_connection_id: int | None = None, - expected_worker: WorkerIdentity | None = None, - expected_active_job: PrismJobContext | None = None, - expected_window_started: float | None = None, - prepared_bundle_allow_uncached: bool = False, - initial_convergence: bool = False, - ) -> bool: - config = self.client_config(client) - if not config.enabled: - return False - # Idle step-downs carry no accepted shares, so they can never be the - # decisive evidence the initial relaxation is granted for. - with client_vardiff_lock(client): - initial_convergence = bool( - initial_convergence - and not require_idle - and getattr( - client, - "vardiff_initial_convergence_pending", - True, - ) - ) - if initial_convergence: - self._count_initial_retarget_attempt() - if require_idle: - if prepared_bundle is None: - return False - with client_vardiff_lock(client), self.runtime.lock: - if expected_connection_id is None: - expected_connection_id = client.connection_id - if expected_worker is None: - expected_worker = client.worker - if expected_active_job is None: - expected_active_job = client.active_job - if expected_window_started is None: - expected_window_started = client.vardiff_window_started_monotonic - if ( - client not in self.runtime.clients - or getattr(client, "closing", False) - or not self.runtime.client_can_receive_jobs(client) - or client.connection_id != expected_connection_id - or client.worker != expected_worker - or client.active_job is not expected_active_job - or client.vardiff_window_started_monotonic - != expected_window_started - or client.vardiff_window_accepted != 0 - or client.vardiff_window_submitted != 0 - ): - return False - observed_difficulty = vardiff.observed_difficulty( - accepted_difficulty=accepted_difficulty, - elapsed_seconds=elapsed_seconds, - target_share_interval_seconds=config.target_share_interval_seconds, - ) - with client_vardiff_lock(client): - previous_estimate = client.vardiff_difficulty_estimate - if observed_difficulty is None: - difficulty_estimate = None - with client_vardiff_lock(client): - client.vardiff_difficulty_estimate = None - else: - difficulty_estimate = vardiff.smooth_difficulty_estimate( - observed=observed_difficulty, - previous=previous_estimate, - config=config, - ) - with client_vardiff_lock(client): - client.vardiff_difficulty_estimate = difficulty_estimate - next_difficulty = vardiff.calculate_next_difficulty( - current_difficulty=current_difficulty, - accepted_shares=accepted_shares, - elapsed_seconds=elapsed_seconds, - config=config, - accepted_difficulty=accepted_difficulty, - difficulty_estimate=difficulty_estimate, - initial_convergence=initial_convergence, - ) - if not vardiff.should_retarget( - current_difficulty, - next_difficulty, - config.retarget_tolerance, - ): - if initial_convergence: - self._count_initial_retarget_outcome("suppressed") - return False - idle_window_state: tuple[float, int, int, Decimal] | None = None - idle_window_reset_at: float | None = None - difficulty_superseded = False - prior_pending: Decimal | None = None - with client_vardiff_lock(client), self.runtime.lock: - previous_difficulty = ( - client.pending_share_difficulty or client.share_difficulty - ) - if previous_difficulty != current_difficulty: - # An explicit d=/md= request (or another retarget) moved this - # client while the step was being computed. It keeps its - # precedence: this retarget yields rather than overriding it. - difficulty_superseded = True - else: - if require_idle and ( - client not in self.runtime.clients - or getattr(client, "closing", False) - or not self.runtime.client_can_receive_jobs(client) - or client.connection_id != expected_connection_id - or client.worker != expected_worker - or client.active_job is not expected_active_job - or client.vardiff_window_started_monotonic - != expected_window_started - or client.vardiff_window_accepted != 0 - or client.vardiff_window_submitted != 0 - ): - # A share landed since the idle snapshot; the accept path - # owns this window. Abort the speculative step-down rather - # than overriding a client that just resumed submitting. - return False - if require_idle: - idle_window_state = ( - client.vardiff_window_started_monotonic, - client.vardiff_window_accepted, - client.vardiff_window_submitted, - client.vardiff_window_work, - ) - prior_pending = client.pending_share_difficulty - client.pending_share_difficulty = next_difficulty - if difficulty_superseded: - if initial_convergence: - self._count_initial_retarget_outcome("superseded") - return False - # Advertise the new difficulty only with its corresponding job. Idle - # retargets stamp an already-cached bundle; normal share-driven - # retargets retain the existing build path. Either path sends the pair - # together or restores the prior pending difficulty/window state. - def idle_commit_guard() -> bool: - nonlocal idle_window_reset_at - if not require_idle: - return True - # _maybe_send_job_locked holds vardiff_lock before entering the - # coordinator commit section, so this guard never waits under - # the coordinator lock. - if ( - self.runtime.stop_event.is_set() - or client not in self.runtime.clients - or getattr(client, "closing", False) - or not self.runtime.client_can_receive_jobs(client) - or client.connection_id != expected_connection_id - or client.worker != expected_worker - or client.active_job is not expected_active_job - or client.vardiff_window_started_monotonic - != expected_window_started - or client.vardiff_window_accepted != 0 - or client.vardiff_window_submitted != 0 - or client.pending_share_difficulty != next_difficulty - ): - return False - idle_window_reset_at = time.monotonic() - client.vardiff_window_started_monotonic = idle_window_reset_at - client.vardiff_window_accepted = 0 - client.vardiff_window_submitted = 0 - client.vardiff_window_work = Decimal("0") - return True - - def restore_speculative_retarget() -> None: - with client_vardiff_lock(client), self.runtime.lock: - if client.pending_share_difficulty == next_difficulty: - client.pending_share_difficulty = prior_pending - self.restore_idle_window_state( - client, - idle_window_state, - idle_window_reset_at, - ) - - # A same-tip retarget only needs to flush in-flight work on a - # step-down: firmware that applies mining.set_difficulty retroactively - # would otherwise submit sub-target shares against the old job. On a - # step-up the old job stays valid at its own stamped share_target, so - # keeping it avoids discarding the miner's in-flight work. - clean_jobs = next_difficulty < current_difficulty - try: - if require_idle: - sent = self.runtime._maybe_send_job_locked( - client, - clean_jobs=clean_jobs, - raise_on_build_failure=True, - prepared_bundle=prepared_bundle, - commit_guard=idle_commit_guard, - commit_guard_lock=client_vardiff_lock(client), - prepared_bundle_allow_uncached=( - prepared_bundle_allow_uncached - ), - ) - else: - sent = bool( - client.authorized - and client.subscribed - and not self.runtime.stop_event.is_set() - and self.runtime.maybe_send_job(client, clean_jobs=clean_jobs) - ) - # A completed paired send is the commit point. Shutdown may race - # immediately afterward, but it cannot make already-delivered work - # speculative again. - if sent: - if not require_idle: - # Commit point for the fast-arrival relaxation: it is - # spent only once a share-driven retarget has actually - # reached the miner, so a skipped build or a failed send - # cannot consume it silently. Every later retarget on - # this connection is back on the ordinary step bound. - with client_vardiff_lock(client): - client.vardiff_initial_convergence_pending = False - if initial_convergence: - self._count_initial_retarget_outcome("applied") - self.observe_high_diff_arrival( - client, - config=config, - previous_difficulty=current_difficulty, - next_difficulty=next_difficulty, - ) - # Retain the committed difficulty here, not only on - # disconnect, so sessions that die without a clean - # disconnect still resume at their converged value. An idle - # step-down carries no accepted shares, but it also moves the - # value, so the store refreshes its TTL on the changed-value - # branch instead. - self.record_session_difficulty( - client, - next_difficulty, - share_backed=accepted_shares > 0, - previous_difficulty=current_difficulty, - initial_convergence=initial_convergence, - ) - return True - except Exception: - # Cached stamping can surface _JobBuildFailed before delivery, and - # socket errors can surface during the paired send. Both must undo - # every speculative client mutation before the task reports failure. - restore_speculative_retarget() - if initial_convergence: - self._count_initial_retarget_outcome("failed") - raise - restore_speculative_retarget() - if initial_convergence: - self._count_initial_retarget_outcome("superseded") - return False - - @staticmethod - def restore_idle_window_state( - client: ClientState, - idle_window_state: tuple[float, int, int, Decimal] | None, - idle_window_reset_at: float | None, - ) -> None: - """Un-restart the idle vardiff window after a step-down that never - reached the miner (skipped build/send), so the next sweep can retry - immediately instead of waiting out another full retarget interval. - Caller must hold the client's vardiff_lock. No-op unless this retarget did the reset - and nothing else has restarted the window since.""" - if idle_window_reset_at is None or idle_window_state is None: - return - if ( - client.vardiff_window_started_monotonic == idle_window_reset_at - and client.vardiff_window_accepted == 0 - and client.vardiff_window_submitted == 0 - and client.vardiff_window_work == 0 - ): - ( - client.vardiff_window_started_monotonic, - client.vardiff_window_accepted, - client.vardiff_window_submitted, - client.vardiff_window_work, - ) = idle_window_state - - def metrics_lines(self) -> list[str]: - with self._vardiff_idle_lock: - sweep = { - "buckets": dict(self.vardiff_idle_sweep_histogram["buckets"]), - "sum": float(self.vardiff_idle_sweep_histogram["sum"]), - "count": int(self.vardiff_idle_sweep_histogram["count"]), - } - task = { - "buckets": dict(self.vardiff_idle_task_histogram["buckets"]), - "sum": float(self.vardiff_idle_task_histogram["sum"]), - "count": int(self.vardiff_idle_task_histogram["count"]), - } - inspected = self.vardiff_idle_clients_inspected - skip_counts = dict(self.vardiff_idle_skip_counts) - queue_depth = self.vardiff_idle_queue_depth - inflight = self.vardiff_idle_inflight - failures = self.vardiff_idle_task_failures - with self._vardiff_convergence_lock: - initial_attempts = self.vardiff_initial_retarget_attempts - initial_outcomes = dict(self.vardiff_initial_retarget_outcome_counts) - durable_writes = dict(self.vardiff_durable_write_outcome_counts) - durable_pruned = int(self.vardiff_durable_pruned_records) - durable_prune_failures = int(self.vardiff_durable_prune_failures) - durable_downward_dropped = int(self.vardiff_durable_downward_dropped) - durable_backend = ( - "postgres" - if isinstance( - self.worker_difficulty_store, - PostgresWorkerDifficultyStore, - ) - else "memory" - ) - arrival_seconds = { - "buckets": dict( - self.vardiff_high_diff_arrival_seconds_histogram["buckets"] - ), - "sum": float(self.vardiff_high_diff_arrival_seconds_histogram["sum"]), - "count": int(self.vardiff_high_diff_arrival_seconds_histogram["count"]), - } - arrival_shares = { - "buckets": dict( - self.vardiff_high_diff_arrival_shares_histogram["buckets"] - ), - "sum": float(self.vardiff_high_diff_arrival_shares_histogram["sum"]), - "count": int(self.vardiff_high_diff_arrival_shares_histogram["count"]), - } - - lines = [ - "# HELP qbit_prism_vardiff_idle_clients_inspected_total Clients inspected by bounded vardiff idle sweeps.", - "# TYPE qbit_prism_vardiff_idle_clients_inspected_total counter", - f"qbit_prism_vardiff_idle_clients_inspected_total {inspected}", - "# HELP qbit_prism_vardiff_idle_skips_total Idle retargets skipped by a bounded reason.", - "# TYPE qbit_prism_vardiff_idle_skips_total counter", - *[ - f'qbit_prism_vardiff_idle_skips_total{{reason="{reason}"}} {int(skip_counts.get(reason, 0))}' - for reason in PRISM_VARDIFF_IDLE_SKIP_REASONS - ], - "# HELP qbit_prism_vardiff_idle_queue_depth Cache-only idle retarget tasks waiting for a dedicated worker.", - "# TYPE qbit_prism_vardiff_idle_queue_depth gauge", - f"qbit_prism_vardiff_idle_queue_depth {queue_depth}", - "# HELP qbit_prism_vardiff_idle_inflight Cache-only idle retarget tasks currently running.", - "# TYPE qbit_prism_vardiff_idle_inflight gauge", - f"qbit_prism_vardiff_idle_inflight {inflight}", - "# HELP qbit_prism_vardiff_idle_task_failures_total Idle retarget tasks that failed during cached delivery.", - "# TYPE qbit_prism_vardiff_idle_task_failures_total counter", - f"qbit_prism_vardiff_idle_task_failures_total {failures}", - "# HELP qbit_prism_vardiff_initial_retarget_attempts_total Retargets attempted under the fast-arrival initial convergence policy: a connection whose first retarget has not yet committed produced enough accepted shares, over enough elapsed time, to show an observed difficulty far above the one it is stamped at. Equals the sum of qbit_prism_vardiff_initial_retargets_total.", - "# TYPE qbit_prism_vardiff_initial_retarget_attempts_total counter", - f"qbit_prism_vardiff_initial_retarget_attempts_total {int(initial_attempts)}", - "# HELP qbit_prism_vardiff_initial_retargets_total Fast-arrival initial retargets by outcome; only applied reached the miner as a paired difficulty and job, and only applied spends the connection's one initial relaxation.", - "# TYPE qbit_prism_vardiff_initial_retargets_total counter", - *[ - f'qbit_prism_vardiff_initial_retargets_total{{outcome="{outcome}"}} {int(initial_outcomes.get(outcome, 0))}' - for outcome in PRISM_VARDIFF_INITIAL_RETARGET_OUTCOMES - ], - "# HELP qbit_prism_vardiff_durable_preloaded Worker difficulties restored into the bounded authorization-time cache during service initialization.", - "# TYPE qbit_prism_vardiff_durable_preloaded gauge", - f"qbit_prism_vardiff_durable_preloaded {self.vardiff_durable_preload_records}", - "# HELP qbit_prism_vardiff_durable_preload_failures_total Durable startup preload failures; mining safely falls back to lane startup difficulty.", - "# TYPE qbit_prism_vardiff_durable_preload_failures_total counter", - f"qbit_prism_vardiff_durable_preload_failures_total {self.vardiff_durable_preload_failures}", - "# HELP qbit_prism_vardiff_durable_pruned_total Worker-difficulty rows deleted as expired, across the startup prune and every periodic TTL prune since.", - "# TYPE qbit_prism_vardiff_durable_pruned_total counter", - f"qbit_prism_vardiff_durable_pruned_total {durable_pruned}", - "# HELP qbit_prism_vardiff_durable_prune_failures_total TTL prune attempts that raised; rows stay until the next interval retries.", - "# TYPE qbit_prism_vardiff_durable_prune_failures_total counter", - f"qbit_prism_vardiff_durable_prune_failures_total {durable_prune_failures}", - "# HELP qbit_prism_vardiff_durable_pending Worker-difficulty writes queued on the bounded persistence lane right now.", - "# TYPE qbit_prism_vardiff_durable_pending gauge", - f"qbit_prism_vardiff_durable_pending {self.durable_pending_depth()}", - "# HELP qbit_prism_vardiff_durable_coalesced_total Queued writes absorbed into an existing entry for the same worker; the lane is bounded by distinct workers, not by event rate.", - "# TYPE qbit_prism_vardiff_durable_coalesced_total counter", - f"qbit_prism_vardiff_durable_coalesced_total {self.durable_coalesced_count()}", - "# HELP qbit_prism_vardiff_durable_downward_dropped_total Safe step-down corrections the lane could not persist. Non-zero means a reconnect may resume a difficulty the live session had abandoned, until its TTL expires; alert on it.", - "# TYPE qbit_prism_vardiff_durable_downward_dropped_total counter", - f"qbit_prism_vardiff_durable_downward_dropped_total {durable_downward_dropped}", - "# HELP qbit_prism_vardiff_durable_backend Active worker-difficulty retention backend.", - "# TYPE qbit_prism_vardiff_durable_backend gauge", - f'qbit_prism_vardiff_durable_backend{{backend="{durable_backend}"}} 1', - "# HELP qbit_prism_vardiff_durable_writes_total Share-backed worker-difficulty persistence attempts by bounded outcome.", - "# TYPE qbit_prism_vardiff_durable_writes_total counter", - *[ - f'qbit_prism_vardiff_durable_writes_total{{outcome="{outcome}"}} {int(durable_writes.get(outcome, 0))}' - for outcome in PRISM_VARDIFF_DURABLE_WRITE_OUTCOMES - ], - ] - lines.extend( - _histogram_lines( - "qbit_prism_vardiff_high_diff_arrival_seconds", - "Seconds from the start of a session's vardiff accounting until a retarget first carried it to the configured high-difficulty arrival threshold. Observed once per connection, and only for sessions that started below the threshold, so a lane already starting above it never contributes.", - arrival_seconds, - PRISM_VARDIFF_HIGH_DIFF_ARRIVAL_SECONDS_BUCKETS, - ) - ) - lines.extend( - _histogram_lines( - "qbit_prism_vardiff_high_diff_arrival_shares", - "Accepted shares a session produced before a retarget first carried it to the configured high-difficulty arrival threshold. Shares credited at the pre-arrival stamped target are exactly the under-credited ones, so this is the share-denominated cost of the climb.", - arrival_shares, - PRISM_VARDIFF_HIGH_DIFF_ARRIVAL_SHARES_BUCKETS, - ) - ) - for metric_name, description, histogram in ( - ( - "qbit_prism_vardiff_idle_sweep_seconds", - "Wall time of one bounded vardiff idle sweep.", - sweep, - ), - ( - "qbit_prism_vardiff_idle_retarget_task_seconds", - "Queue plus execution latency for cache-only idle retarget tasks.", - task, - ), - ): - lines.extend( - _histogram_lines( - metric_name, - description, - histogram, - PRISM_VARDIFF_IDLE_SECONDS_BUCKETS, - ) - ) - return lines diff --git a/lab/prism/worker_difficulty_store.py b/lab/prism/worker_difficulty_store.py deleted file mode 100644 index d01f5b315..000000000 --- a/lab/prism/worker_difficulty_store.py +++ /dev/null @@ -1,664 +0,0 @@ -"""Durable retention of delivered, safe per-worker vardiff difficulties. - -Storage seam only. The vardiff service keeps its in-process -``SessionDifficultyStore`` as the authorization-time source; this module gives -a later integration layer matching in-memory and PostgreSQL back ends for -retaining the last delivered, safe wire difficulty across coordinator -restarts. The caller — not this module — enforces share-backed upward moves: -the API carries the delivered decimal value and the evidence timestamp -explicitly so safety is never inferred from the username alone. Exact -usernames are public identities, not authentication secrets. - -Contract shared by both implementations: - -- One row per ``(listener, worker_username)``: the listener lane name plus the - exact Stratum username, matched byte-for-byte with no normalization. -- ``difficulty`` is the delivered decimal wire difficulty, retained exactly - (no integer or float conversion anywhere on the round trip). -- ``evidence_at_ms`` is when share-backed evidence last validated the value; - ``updated_at_ms`` is the last write of any kind. Both are caller-supplied - wall-clock milliseconds, so the two back ends stay deterministic and - comparable. -- ``upsert`` is atomic with deterministic last-write/evidence semantics: a - write whose evidence timestamp is older than the stored one is refused - outright, equal-or-newer evidence replaces the row (last write wins on - ties). Older evidence can therefore never displace newer evidence. -- ``apply_downward`` is the atomic lower-only correction: it reduces an - existing row to a smaller difficulty and leaves ``evidence_at`` untouched, - so a step-down the live session already made can never suppress later - genuine share-backed upward evidence, and can never refresh the TTL of a - value no share proved. A missing row, or one already at or below the - requested difficulty, is a no-op — never an insert, because there is no - stored value for a restart to resurrect. -- ``load_recent`` returns a bounded newest-first set whose evidence is - strictly newer than the cutoff, deterministically ordered for preload: - evidence timestamp descending, ties broken by (listener, worker_username) - in byte order (the key columns collate as "C", so both back ends agree). -- ``prune`` deletes rows whose evidence is at or before the cutoff — exactly - the complement of ``load_recent`` — entirely inside the store (an indexed - range scan in PostgreSQL), never via an unbounded application-side sweep. - -Deliberately absent: any per-key point read. Reconnect authorization must -never wait on a synchronous database round trip, so the only read is the -bounded batch preload an integration layer feeds into its in-process store. -""" - -from __future__ import annotations - -from dataclasses import dataclass -from decimal import Decimal -from threading import Lock -from typing import Any, Callable, Protocol, runtime_checkable - - -# Byte-identical to the qbit_worker_difficulty DDL block in -# crates/qbit-prism/sql/001_share_ledger.sql (which is the canonical, -# transactional apply path); a contract test pins the match. Both statements -# are idempotent, so ensure_schema() may be re-run freely and the monolithic -# file may be reapplied over an existing table. -WORKER_DIFFICULTY_SCHEMA_SQL = """\ -CREATE TABLE IF NOT EXISTS qbit_worker_difficulty ( - listener text COLLATE "C" NOT NULL CHECK (listener <> ''), - worker_username text COLLATE "C" NOT NULL CHECK (worker_username <> ''), - difficulty numeric NOT NULL CHECK ( - difficulty > 0 AND difficulty < 'Infinity'::numeric - ), - evidence_at timestamptz NOT NULL, - updated_at timestamptz NOT NULL DEFAULT clock_timestamp(), - PRIMARY KEY (listener, worker_username) -); - -CREATE INDEX IF NOT EXISTS qbit_worker_difficulty_evidence_idx - ON qbit_worker_difficulty (evidence_at DESC, listener, worker_username); -""" - -# Wall-clock bounds for every millisecond timestamp and cutoff this module -# accepts: non-negative (evidence predating the epoch is meaningless) and at -# most 9999-12-31T23:59:59.999Z, comfortably inside both PostgreSQL -# timestamptz range and exact float64 millisecond arithmetic. -MAX_TIMESTAMP_MS = 253_402_300_799_999 - -# Difficulty magnitude bounds enforced identically by both back ends, so the -# in-memory store cannot accept a value the PostgreSQL numeric apply would -# reject (or vice versa). Far beyond any real Stratum wire difficulty, far -# inside PostgreSQL numeric limits. -MAX_DIFFICULTY_DIGITS = 1_000 -MAX_DIFFICULTY_ADJUSTED_EXPONENT = 999 - - -@dataclass(frozen=True) -class WorkerDifficultyRecord: - """One retained per-worker difficulty row, identical in both back ends.""" - - listener: str - worker_username: str - difficulty: Decimal - evidence_at_ms: int - updated_at_ms: int - - -@dataclass(frozen=True) -class WorkerDifficultyUpsertResult: - """Outcome of one upsert. - - ``applied`` is False when the write carried evidence strictly older than - the stored row and was refused. ``stored`` is the row now standing for the - key; it can be None only on the PostgreSQL back end, in the narrow race - where a refused write's follow-up read finds no snapshot-visible row. - """ - - applied: bool - stored: WorkerDifficultyRecord | None - - -@dataclass(frozen=True) -class WorkerDifficultyLowerResult: - """Outcome of one lower-only correction. - - ``applied`` is True only when a stored row was actually reduced. A key with - no stored row, or one already at or below the requested difficulty, is a - no-op: there is nothing a restart could resurrect. ``stored`` is the row now - standing for the key, or None when no row exists (or, on the PostgreSQL - back end, in the narrow race where a no-op's follow-up read finds no - snapshot-visible row). - """ - - applied: bool - stored: WorkerDifficultyRecord | None - - -@runtime_checkable -class WorkerDifficultyStorePort(Protocol): - """The exact surface an integration layer may depend on. - - Four operations, none of which is a per-key synchronous read. - """ - - def upsert( - self, - *, - listener: str, - worker_username: str, - difficulty: Decimal, - evidence_at_ms: int, - now_ms: int, - ) -> WorkerDifficultyUpsertResult: ... - - def apply_downward( - self, - *, - listener: str, - worker_username: str, - difficulty: Decimal, - now_ms: int, - ) -> WorkerDifficultyLowerResult: ... - - def load_recent( - self, - *, - evidence_after_ms: int, - limit: int, - ) -> list[WorkerDifficultyRecord]: ... - - def prune( - self, - *, - evidence_cutoff_ms: int, - limit: int | None = None, - ) -> int: ... - - -def _validated_identity(value: str, name: str) -> str: - if not isinstance(value, str): - raise ValueError(f"{name} must be a string") - if value == "": - raise ValueError(f"{name} must not be empty") - if "\x00" in value: - # PostgreSQL text cannot hold NUL; refusing it here keeps the - # in-memory back end from accepting a key the durable one rejects. - raise ValueError(f"{name} must not contain NUL") - return value - - -def _validated_difficulty(difficulty: Decimal) -> Decimal: - if isinstance(difficulty, bool) or not isinstance(difficulty, Decimal): - raise ValueError("difficulty must be a Decimal (no float or int coercion)") - if not difficulty.is_finite() or difficulty <= 0: - raise ValueError("difficulty must be a positive finite Decimal") - parts = difficulty.as_tuple() - if len(parts.digits) > MAX_DIFFICULTY_DIGITS: - raise ValueError( - f"difficulty exceeds {MAX_DIFFICULTY_DIGITS} significant digits" - ) - if abs(difficulty.adjusted()) > MAX_DIFFICULTY_ADJUSTED_EXPONENT: - raise ValueError( - "difficulty magnitude exceeds " - f"1E±{MAX_DIFFICULTY_ADJUSTED_EXPONENT}" - ) - return difficulty - - -def _validated_timestamp_ms(value: int, name: str) -> int: - if isinstance(value, bool) or not isinstance(value, int): - raise ValueError(f"{name} must be an integer millisecond timestamp") - if value < 0 or value > MAX_TIMESTAMP_MS: - raise ValueError(f"{name} must be within [0, {MAX_TIMESTAMP_MS}]") - return value - - -def _validated_limit(limit: int) -> int: - if isinstance(limit, bool) or not isinstance(limit, int): - raise ValueError("limit must be an integer") - if limit <= 0: - raise ValueError("limit must be positive") - return limit - - -class MemoryWorkerDifficultyStore: - """In-memory reference implementation of the storage contract. - - Semantically identical to :class:`PostgresWorkerDifficultyStore` (the - parity gate drives both in lockstep). The lock is a leaf lock: nothing - here calls out of this module while holding it. - """ - - def __init__(self) -> None: - self._lock = Lock() - self._entries: dict[tuple[str, str], WorkerDifficultyRecord] = {} - - def upsert( - self, - *, - listener: str, - worker_username: str, - difficulty: Decimal, - evidence_at_ms: int, - now_ms: int, - ) -> WorkerDifficultyUpsertResult: - listener = _validated_identity(listener, "listener") - worker_username = _validated_identity(worker_username, "worker_username") - difficulty = _validated_difficulty(difficulty) - evidence_at_ms = _validated_timestamp_ms(evidence_at_ms, "evidence_at_ms") - now_ms = _validated_timestamp_ms(now_ms, "now_ms") - key = (listener, worker_username) - with self._lock: - existing = self._entries.get(key) - if existing is not None and evidence_at_ms < existing.evidence_at_ms: - return WorkerDifficultyUpsertResult(applied=False, stored=existing) - record = WorkerDifficultyRecord( - listener=listener, - worker_username=worker_username, - difficulty=difficulty, - evidence_at_ms=evidence_at_ms, - updated_at_ms=now_ms, - ) - self._entries[key] = record - return WorkerDifficultyUpsertResult(applied=True, stored=record) - - def apply_downward( - self, - *, - listener: str, - worker_username: str, - difficulty: Decimal, - now_ms: int, - ) -> WorkerDifficultyLowerResult: - listener = _validated_identity(listener, "listener") - worker_username = _validated_identity(worker_username, "worker_username") - difficulty = _validated_difficulty(difficulty) - now_ms = _validated_timestamp_ms(now_ms, "now_ms") - key = (listener, worker_username) - with self._lock: - existing = self._entries.get(key) - if existing is None or existing.difficulty <= difficulty: - return WorkerDifficultyLowerResult(applied=False, stored=existing) - # evidence_at is deliberately carried forward unchanged: this - # correction is not new evidence, and re-stamping it would both - # extend the TTL of an unproven value and let it refuse a later - # share-backed write whose evidence predates the step-down. - record = WorkerDifficultyRecord( - listener=existing.listener, - worker_username=existing.worker_username, - difficulty=difficulty, - evidence_at_ms=existing.evidence_at_ms, - updated_at_ms=now_ms, - ) - self._entries[key] = record - return WorkerDifficultyLowerResult(applied=True, stored=record) - - def load_recent( - self, - *, - evidence_after_ms: int, - limit: int, - ) -> list[WorkerDifficultyRecord]: - evidence_after_ms = _validated_timestamp_ms( - evidence_after_ms, "evidence_after_ms" - ) - limit = _validated_limit(limit) - with self._lock: - recent = [ - record - for record in self._entries.values() - if record.evidence_at_ms > evidence_after_ms - ] - recent.sort( - key=lambda record: ( - -record.evidence_at_ms, - record.listener, - record.worker_username, - ) - ) - return recent[:limit] - - def prune( - self, - *, - evidence_cutoff_ms: int, - limit: int | None = None, - ) -> int: - evidence_cutoff_ms = _validated_timestamp_ms( - evidence_cutoff_ms, "evidence_cutoff_ms" - ) - if limit is not None: - limit = _validated_limit(limit) - with self._lock: - doomed = [ - key - for key, record in self._entries.items() - if record.evidence_at_ms <= evidence_cutoff_ms - ] - if limit is not None: - doomed.sort( - key=lambda key: ( - self._entries[key].evidence_at_ms, - key[0], - key[1], - ) - ) - doomed = doomed[:limit] - for key in doomed: - del self._entries[key] - return len(doomed) - - def __len__(self) -> int: - with self._lock: - return len(self._entries) - - -def _text_literal(value: str) -> str: - return "'" + value.replace("'", "''") + "'" - - -def _difficulty_literal(difficulty: Decimal) -> str: - # str() of a validated finite Decimal contains only [0-9.E+-]; the quoted - # cast keeps the rendering injection-proof regardless. - return f"'{difficulty}'::numeric" - - -def _timestamp_literal(timestamp_ms: int) -> str: - # Same ms-to-timestamptz idiom as share_ledger.py. Validated bounds keep - # the double-precision division exact to well below the microsecond - # resolution timestamptz stores, so ms values round-trip exactly. - return f"to_timestamp(({timestamp_ms}::double precision / 1000.0))" - - -_RECORD_JSON_FIELDS = """\ -'listener', {alias}.listener, - 'worker_username', {alias}.worker_username, - 'difficulty', {alias}.difficulty::text, - 'evidence_at_ms', round(extract(epoch FROM {alias}.evidence_at) * 1000)::bigint, - 'updated_at_ms', round(extract(epoch FROM {alias}.updated_at) * 1000)::bigint""" - - -def _record_from_json(payload: dict[str, Any]) -> WorkerDifficultyRecord: - return WorkerDifficultyRecord( - listener=str(payload["listener"]), - worker_username=str(payload["worker_username"]), - # difficulty travels as text inside the JSON payload precisely so it - # never passes through a float. - difficulty=Decimal(str(payload["difficulty"])), - evidence_at_ms=int(payload["evidence_at_ms"]), - updated_at_ms=int(payload["updated_at_ms"]), - ) - - -class PostgresWorkerDifficultyStore: - """PostgreSQL implementation over an injected statement runner. - - ``run_json`` executes one self-contained SQL statement and returns its - single JSON value — the contract ``PsqlShareLedger._run_json`` and - ``LedgerSqlPort.run_json`` already honour, so the store rides the - coordinator's existing pooled connections without opening its own. - ``run_script`` (optional) executes a multi-statement script and is only - needed for :meth:`ensure_schema`; the canonical schema path remains the - monolithic ``001_share_ledger.sql`` apply, which creates the same table. - """ - - def __init__( - self, - run_json: Callable[[str], Any], - *, - run_script: Callable[[str], None] | None = None, - ) -> None: - self._run_json = run_json - self._run_script = run_script - - @classmethod - def from_share_ledger( - cls, - ledger: Any, - *, - timeout_seconds: float, - ) -> "PostgresWorkerDifficultyStore": - """Build over a bounded, admission-gated ``PsqlShareLedger`` seam. - - Worker-difficulty traffic is not writer-lease state, so it uses one - of the ledger's bounded read/concurrent statement slots. The statement - is still a mutation and is therefore never retried after an ambiguous - connection failure. - """ - - def run_json(sql: str) -> Any: - with ledger.operation_timeout(timeout_seconds): - with ledger._operation_gate( - ledger._read_semaphore, - "worker-difficulty slot", - ): - return ledger._run_json(sql) - - return cls(run_json, run_script=ledger._run_script) - - def ensure_schema(self) -> None: - """Idempotently apply this store's DDL (subset of the monolithic file).""" - if self._run_script is None: - raise RuntimeError( - "ensure_schema requires a run_script callable; the monolithic " - "001_share_ledger.sql apply also creates this table" - ) - self._run_script(WORKER_DIFFICULTY_SCHEMA_SQL) - - def upsert( - self, - *, - listener: str, - worker_username: str, - difficulty: Decimal, - evidence_at_ms: int, - now_ms: int, - ) -> WorkerDifficultyUpsertResult: - listener = _validated_identity(listener, "listener") - worker_username = _validated_identity(worker_username, "worker_username") - difficulty = _validated_difficulty(difficulty) - evidence_at_ms = _validated_timestamp_ms(evidence_at_ms, "evidence_at_ms") - now_ms = _validated_timestamp_ms(now_ms, "now_ms") - listener_literal = _text_literal(listener) - username_literal = _text_literal(worker_username) - applied_fields = _RECORD_JSON_FIELDS.format(alias="applied") - current_fields = _RECORD_JSON_FIELDS.format(alias="current") - # One atomic statement: the ON CONFLICT WHERE guard is what makes - # older evidence unable to displace newer evidence, and equal-or-newer - # evidence a deterministic last-write-wins replacement. The refused - # branch reads the standing row from the statement snapshot instead - # (a data-modifying CTE's effects are invisible to the outer query, - # so the applied branch must come from RETURNING). - payload = self._run_json( - f""" -WITH applied AS ( - INSERT INTO qbit_worker_difficulty AS existing ( - listener, - worker_username, - difficulty, - evidence_at, - updated_at - ) - VALUES ( - {listener_literal}, - {username_literal}, - {_difficulty_literal(difficulty)}, - {_timestamp_literal(evidence_at_ms)}, - {_timestamp_literal(now_ms)} - ) - ON CONFLICT (listener, worker_username) DO UPDATE - SET difficulty = EXCLUDED.difficulty, - evidence_at = EXCLUDED.evidence_at, - updated_at = EXCLUDED.updated_at - WHERE EXCLUDED.evidence_at >= existing.evidence_at - RETURNING listener, worker_username, difficulty, evidence_at, updated_at -) -SELECT json_build_object( - 'applied', EXISTS (SELECT 1 FROM applied), - 'stored', COALESCE( - (SELECT json_build_object( - {applied_fields} - ) FROM applied), - (SELECT json_build_object( - {current_fields} - ) - FROM qbit_worker_difficulty current - WHERE current.listener = {listener_literal} - AND current.worker_username = {username_literal}) - ) -); -""" - ) - stored = payload.get("stored") - return WorkerDifficultyUpsertResult( - applied=bool(payload["applied"]), - stored=None if stored is None else _record_from_json(stored), - ) - - def apply_downward( - self, - *, - listener: str, - worker_username: str, - difficulty: Decimal, - now_ms: int, - ) -> WorkerDifficultyLowerResult: - listener = _validated_identity(listener, "listener") - worker_username = _validated_identity(worker_username, "worker_username") - difficulty = _validated_difficulty(difficulty) - now_ms = _validated_timestamp_ms(now_ms, "now_ms") - listener_literal = _text_literal(listener) - username_literal = _text_literal(worker_username) - difficulty_literal = _difficulty_literal(difficulty) - lowered_fields = _RECORD_JSON_FIELDS.format(alias="lowered") - current_fields = _RECORD_JSON_FIELDS.format(alias="current") - # One atomic statement. The row-level UPDATE takes its own lock, so a - # concurrent upsert either precedes it (and is then compared against) - # or follows it (and wins on evidence); evidence_at is never written - # here, which is what keeps a step-down from suppressing later - # share-backed evidence. The no-op branch reads the standing row from - # the statement snapshot for the same reason upsert does. - payload = self._run_json( - f""" -WITH lowered AS ( - UPDATE qbit_worker_difficulty - SET difficulty = {difficulty_literal}, - updated_at = {_timestamp_literal(now_ms)} - WHERE listener = {listener_literal} - AND worker_username = {username_literal} - AND difficulty > {difficulty_literal} - RETURNING listener, worker_username, difficulty, evidence_at, updated_at -) -SELECT json_build_object( - 'applied', EXISTS (SELECT 1 FROM lowered), - 'stored', COALESCE( - (SELECT json_build_object( - {lowered_fields} - ) FROM lowered), - (SELECT json_build_object( - {current_fields} - ) - FROM qbit_worker_difficulty current - WHERE current.listener = {listener_literal} - AND current.worker_username = {username_literal}) - ) -); -""" - ) - stored = payload.get("stored") - return WorkerDifficultyLowerResult( - applied=bool(payload["applied"]), - stored=None if stored is None else _record_from_json(stored), - ) - - def load_recent( - self, - *, - evidence_after_ms: int, - limit: int, - ) -> list[WorkerDifficultyRecord]: - evidence_after_ms = _validated_timestamp_ms( - evidence_after_ms, "evidence_after_ms" - ) - limit = _validated_limit(limit) - recent_fields = _RECORD_JSON_FIELDS.format(alias="recent") - # The inner ORDER BY + LIMIT walk qbit_worker_difficulty_evidence_idx - # directly; the aggregate re-states the same ordering because - # json_agg offers no ordering guarantee of its own. - payload = self._run_json( - f""" -SELECT COALESCE( - ( - SELECT json_agg( - json_build_object( - {recent_fields} - ) - ORDER BY recent.evidence_at DESC, recent.listener, recent.worker_username - ) - FROM ( - SELECT listener, worker_username, difficulty, evidence_at, updated_at - FROM qbit_worker_difficulty - WHERE evidence_at > {_timestamp_literal(evidence_after_ms)} - ORDER BY evidence_at DESC, listener, worker_username - LIMIT {limit} - ) recent - ), - '[]'::json -); -""" - ) - return [_record_from_json(entry) for entry in payload] - - def prune( - self, - *, - evidence_cutoff_ms: int, - limit: int | None = None, - ) -> int: - evidence_cutoff_ms = _validated_timestamp_ms( - evidence_cutoff_ms, "evidence_cutoff_ms" - ) - cutoff_literal = _timestamp_literal(evidence_cutoff_ms) - if limit is None: - payload = self._run_json( - f""" -WITH deleted AS ( - DELETE FROM qbit_worker_difficulty - WHERE evidence_at <= {cutoff_literal} - RETURNING 1 -) -SELECT json_build_object('deleted', (SELECT count(*) FROM deleted)); -""" - ) - return int(payload["deleted"]) - limit = _validated_limit(limit) - # Oldest evidence first with the same tie-break the memory store - # applies, so a bounded prune removes a deterministic set. - payload = self._run_json( - f""" -WITH doomed AS ( - SELECT listener, worker_username - FROM qbit_worker_difficulty - WHERE evidence_at <= {cutoff_literal} - ORDER BY evidence_at ASC, listener, worker_username - LIMIT {limit} -), -deleted AS ( - DELETE FROM qbit_worker_difficulty target - USING doomed - WHERE target.listener = doomed.listener - AND target.worker_username = doomed.worker_username - AND target.evidence_at <= {cutoff_literal} - RETURNING 1 -) -SELECT json_build_object('deleted', (SELECT count(*) FROM deleted)); -""" - ) - return int(payload["deleted"]) - - -__all__ = [ - "MAX_DIFFICULTY_ADJUSTED_EXPONENT", - "MAX_DIFFICULTY_DIGITS", - "MAX_TIMESTAMP_MS", - "MemoryWorkerDifficultyStore", - "PostgresWorkerDifficultyStore", - "WORKER_DIFFICULTY_SCHEMA_SQL", - "WorkerDifficultyLowerResult", - "WorkerDifficultyRecord", - "WorkerDifficultyStorePort", - "WorkerDifficultyUpsertResult", -] diff --git a/lab/prism/writer_lease_timing.py b/lab/prism/writer_lease_timing.py deleted file mode 100644 index 3a69e9db3..000000000 --- a/lab/prism/writer_lease_timing.py +++ /dev/null @@ -1,859 +0,0 @@ -"""One validated timing policy for the PRISM writer-lease heartbeat. - -Why this module exists ----------------------- -The heartbeat's five timing knobs — adoption silence, heartbeat interval, -failure budget, monitor interval, exit margin — are not five independent -tunables. They encode one safety argument: - - a coordinator that has lost its guarded session must be *gone* before - any replacement becomes eligible to compare-and-swap the lease row. - -Historically the numbers were derived from the adoption silence alone -(``silence / 4`` and ``silence * 0.75``) and the coupling was re-derived by -hand at each call site, with only a printed warning when an override broke -it. Issue #212 is the failure mode that produces: the derived envelope was -sized for a heartbeat's *idle interval* but not for the *tail* of a lawful -verification, so ordinary rapid-block verification latency exhausted the -server-proven envelope and hard-exited healthy coordinators (69 restarts on -``union-mainnet``, activity ages 0.54-0.78s, server-proven ages 0.76-0.91s -against a 0.75s budget and a 0.80s cap). - -The fix is to state the whole inequality once, in terms of phases that are -actually measurable in production, and to reject a configuration that -breaks it instead of printing and continuing. - -The phase budgets ------------------ -Three inputs are physical properties of the deployment rather than free -choices: - -``guard_statement_timeout_seconds`` - The server-side ``statement_timeout`` on the dedicated guard session - (:class:`~lab.prism.share_ledger._NativePostgresLeaseGuard`). It is a - hard bound: no single guarded round trip can take longer, because - PostgreSQL cancels it. - -``heartbeat_interval_seconds`` - The idle gap the heartbeat waits between proofs. - -``scheduler_slack_seconds`` - The process-side scheduling allowance: thread wake-up and GIL - contention behind a rapid-block burst. It is applied in *both* - inequalities below, because it describes the process rather than one - thread — once to the heartbeat's stamping path, and once to the - monitor's own poll. Budgeting a stall for the heartbeat while - assuming the monitor wakes punctually is the mistake that made the - first cut of this policy unsafe. - -From those, the largest gap a *healthy* coordinator can leave between two -completed server round trips is - - max_healthy_server_gap = interval + guard_statement_timeout + slack - -and nothing shorter is a safe staleness bound. The argument is that at -every instant either a guarded statement is in flight — and it must answer -or be cancelled within the statement timeout — or none is, in which case -the heartbeat starts one within the remaining interval. Queueing behind a -concurrent external-effect fence does not extend the gap: the fence runs on -the same guard session and stamps at each of its own round trips. - -The safety inequality ---------------------- -The monitor measures staleness from completed round trips (client-side -marks can postdate a silent session death, so they cannot carry the -adoption guarantee). Write ``p`` for the newest server-proven edge and -``t_d`` for the instant the guarded session died; the conservative send-edge -stamping below guarantees ``p <= t_d``. The monitor polls every -``monitor_interval`` and may itself be ``scheduler_slack`` late, so it -observes ``age >= cap`` no later than - - p + cap + monitor_interval + scheduler_slack - -and the hard exit is then budgeted ``exit_margin``. A successor cannot -begin its silence window before the guard session is released, so it is -eligible no earlier than ``t_d + adoption_silence >= p + adoption_silence``. -Exit-before-adoption is therefore - - cap + exit_envelope <= adoption_silence (1) - where exit_envelope = exit_margin + 2 * monitor_interval - + scheduler_slack - heartbeat_interval < failure_budget (2) - max_healthy_server_gap <= cap (3) - -The second ``monitor_interval`` in the envelope is what makes (1) strict: -only one of the two is spent on poll granularity, so the worst-case exit -completes one whole poll interval *before* the adoption edge rather than -on it. - -(1) and (2) are *safety*: violating them means a lost coordinator may still -be alive when a replacement adopts, or that one idle wait alone exhausts -the liveness budget. They are rejected at startup. (3) is *stability*: -violating it does not permit two writers, it just guarantees that ordinary -tail latency will eventually hard-exit a healthy coordinator — exactly -issue #212. It is reported as an advisory, because a deliberately tiny -test or lab policy is allowed to trade stability away, and refusing to -start would be worse than restarting. - -What (1) actually guarantees ----------------------------- -(1) is a statement about a *budget*, not about the process. Write ``L`` -for the monitor's real lateness on the poll that first observes the stale -edge. That poll happens no later than ``p + cap + monitor_interval + L`` -and the exit completes ``exit_margin`` after it, so exit-before-adoption -holds for that beat if and only if - - L < adoption_silence - cap - monitor_interval - exit_margin - = scheduler_slack + monitor_interval (4) - -(``max_guaranteed_monitor_lateness_seconds`` below; the identity on the -right holds whenever the cap is the silence minus the exit envelope, which -it is for every policy that is not floored at the failure budget). Three -regimes follow: - -``L <= scheduler_slack`` - the case (1) budgets: the exit completes one whole monitor poll before - the adoption edge. -``scheduler_slack < L < scheduler_slack + monitor_interval`` - the strictness reserve is consumed: the exit still completes before - the adoption edge, but by less than one poll. -``L >= scheduler_slack + monitor_interval`` - exit-before-adoption is **not guaranteed** for that beat; the process - may still be alive up to ``L - (scheduler_slack + monitor_interval)`` - after the adoption edge. - -Production exceeded the assumption (issue #227) ------------------------------------------------ -Three days after this policy shipped, ``union-mainnet`` (bootstrap pin -``9c41894``, coordinator up 45h, zero restarts, no heartbeat exit) recorded -a monitor wake delay of **0.648s** — above the 0.50s slack and above the -0.55s bound (4). For the duration of that stall the sum on the left of -(1), evaluated with the real lateness, was ``1.30 + 0.05 + 0.648 + 0.10 = -2.098 > 2.00``. No guard loss coincided, so nothing happened; but the -assumption the proof rests on was violated in production. - -The stall was in-process, not on the host (32 cores 93% idle, PSI near -zero, no cgroup throttling). The window held a 5.1s -``reconcile_invalidation`` full rescan of a 52k-share payout window, and -the phase attribution of the same period showed why the old telemetry -could not see it: the worst ``guard_sql`` was 0.714s against a -server-enforced 0.50s statement timeout while ``scheduler_delay`` never -exceeded 22 microseconds in 645k attempts. ``guard_sql`` was measured -client-side, so a GIL stall between PostgreSQL answering and the heartbeat -thread resuming was booked as database time. - -Attribution that survives GIL stalls ------------------------------------- -Each guard statement now returns its own server-side execution time -(``clock_timestamp() - statement_timestamp()``, computed and returned by -the statement itself, so no extra round trip and no extra lock). The -attempt accumulator splits the client-measured statement span into -``guard_sql`` (server execution, when the statement reports it) and -``guard_client_resume`` (the remainder of the round trip: network transit -plus the time this process took to resume after the answer arrived). The -residual ``scheduler_delay = total - slot_wait - guard_sql`` therefore -absorbs every process-side delay, inside or outside a round trip; -``guard_client_resume`` is the part of it that fell inside one. A -statement that reports no server time keeps the old, deliberately -conservative attribution (the whole span is ``guard_sql``). The -server-proven edge is untouched: it is still the conservative send edge, -never a server-reported timestamp, because the ``p <= t_d`` step above -depends on it. - -The response to lateness beyond slack (the decision) ----------------------------------------------------- -Two responses were on the table. (a) Treat lateness beyond slack as -envelope consumed for that beat and tighten the cap for that beat. (b) -Accept the residual and rely on the fences. - -(a) is refused, on the arithmetic rather than on taste. On the late wake -``w`` the monitor can only exit *now*, completing at ``w + exit_margin``; -that bound does not move whether the threshold it compares the age -against is ``cap`` or ``cap - (L - slack)``. Tightening changes behaviour -only for server-proven ages in ``[cap - (L - slack), cap)`` — a band that a -healthy coordinator occupies whenever a slow statement coincides with the -stall (with the observed 0.648s the tightened threshold is 1.152s, below -the 1.25s a healthy gap may reach). So (a) hard-exits healthy -coordinators — issue #212's failure mode — and buys nothing against the -breach it is reacting to. - -(b) is safe because the un-covered interval permits no mutation. Three -observations: - -* A successor cannot begin its silence until the predecessor's *guard - session* is gone (``PsqlShareLedger._writer_lease_adoption_wait_seconds`` - cannot even be reached while the advisory guard is held). A stalled but - live process is therefore never adopted over; the hazard is the compound - event of a guard-session death *and* a monitor lateness beyond (4) in the - same window. -* During a GIL stall no Python thread runs, so nothing escapes *during* the - stall. What could escape is work done in the interval between the - adoption edge and the (late) exit, at most ``L - (slack + - monitor_interval)`` long. -* Every ledger write is fenced at the row: share appends and every landing - statement open with the exact-session ``lease AS (UPDATE - qbit_ledger_writer_lease ... WHERE writer_session_token = ...)`` CTE and - join it, and adoption rewrites ``writer_session_token``, so a stale - writer's statement matches zero lease rows and reports "writer lease is - not active". Every external effect (submitblock, CTV fanout, wallet) - runs behind ``require_fresh_lease_for_external_side_effect``, a - synchronous exact-session verification on the guard session that raises - — and hard-exits — the moment the committed row names another session. - The fence-to-RPC residual is the documented preflight residual between - independent systems and is not widened by the monitor's lateness. The - heartbeat's own next proof fails the same way. - -So the shipped response is (b), with the breach made loud instead of -invisible: the monitor's lateness feeds a fixed-bucket histogram, three -threshold counters (0.5x, 0.8x and 1.0x slack), a rolling-window maximum -and the age of the lifetime record; a wake beyond (4) increments an -exit-guarantee breach counter, records the worst overrun, and emits one -structured warning naming the breach; a wake beyond half the slack takes a -rate-limited stack sample of the running threads. The operator action is -in ``docs/prism-ledger-ops.md``. The durable fix — an exit path that does -not depend on the interpreter being scheduled — is an out-of-process -watchdog fed by the server-proven edge, scoped in issue #130. - -The shipped defaults --------------------- - guard_statement_timeout = 0.50 (server-enforced, unchanged) - heartbeat_interval = 0.25 - scheduler_slack = 0.50 (~3x the worst delay observed in the - union-mainnet burst behind #212; - exceeded once since, by a 0.648s - in-process stall, see #227 above) - max_healthy_server_gap = 0.25 + 0.50 + 0.50 = 1.25 - failure_budget = 1.25 (the same bound; one number, not two) - monitor_interval = 0.05 - exit_margin = 0.10 - exit_envelope = 0.10 + 2 x 0.05 + 0.50 = 0.70 - adoption_silence = 2.00 (> 1.25 + 0.70 = 1.95) - => server_proven_cap = 2.00 - 0.70 = 1.30 - => stability surplus = 1.30 - 1.25 = 0.05 - => max_guaranteed_monitor_lateness = 0.50 + 0.05 = 0.55 - -Checking (1) at the shipped numbers, with a monitor exactly as late as the -policy budgets: - - 1.30 + 0.05 + 0.50 + 0.10 = 1.95 < 2.00 - -so the old coordinator is gone 0.05s — one monitor poll — before the -successor may compare-and-swap, *provided* the monitor's real lateness is -within the 0.55s bound (4). That proviso is an assumption about the -process, not a property of the policy; production has exceeded it once. - -Genuine writer failover therefore costs 2.0s of adoption silence rather -than 1.0s. That is the price of the guard session's own 0.5s statement -timeout: the monitor can only trust completed round trips, so the envelope -must contain one whole statement plus one idle interval plus process -scheduling — and then, separately, room for the monitor itself to be that -late in noticing. The stability surplus is deliberately thin (0.05s): -inequality (1) is the safety bound and comes first, while (3) is already a -worst-case-on-worst-case (a statement running its whole server-side -timeout *and* the process stalling for its whole slack allowance in the -same beat). Production's worst observed server-proven age during the -issue #212 burst was 0.91s, well under the 1.30s cap, and the ownership -proof that now runs on most beats is a single non-blocking read. It is half of the 4.0s downstream mitigation this replaces, -and unlike that mitigation it is derived rather than dialled in. - -Lowering it further requires shrinking a phase, not the envelope: a shorter -statement timeout for the frequent ownership proof, or a shorter heartbeat -interval. The proof/renewal split (see -``PsqlShareLedger.prove_writer_lease_guard_session``) already removed the -*typical* cost — the frequent statement no longer scans -``pg_stat_activity`` or touches the lease tuple's row lock — but the -*bound* is still the guard session's statement timeout, and the envelope -must be sized for the bound. -""" - -from __future__ import annotations - -from dataclasses import dataclass -import math -import time -from typing import Callable - -# The server-side statement_timeout carried by the dedicated writer-lease -# guard session. It is the hard ceiling on one guarded round trip, so it is -# also the dominant term in the heartbeat's staleness envelope. Declared here -# rather than inline in the connection options so the policy below and the -# connection that enforces it cannot drift apart. -WRITER_LEASE_GUARD_STATEMENT_TIMEOUT_SECONDS = 0.5 - -# Statements verify_writer_lease_guard_session may lawfully run inside one -# guarded slot: the verification statement plus its single attribution -# recheck. Callers that budget the verification's execution wall-clock must -# cover this many server-side statement timeouts, or a lawful recheck under -# moderate database latency is killed by the caller's deadline instead of -# rescuing the coordinator. -WRITER_LEASE_VERIFICATION_MAX_STATEMENTS = 2 - -# Process-side delay the envelope must absorb between PostgreSQL answering -# and the monitor observing that answer: heartbeat thread wake-up, GIL -# contention behind a rapid-block burst, result handling, and the monitor's -# own late wake. Sized at roughly three times the worst delay observed in the -# union-mainnet burst that produced issue #212. Issue #227 recorded a 0.648s -# monitor wake delay on union-mainnet, above this allowance; the module -# docstring states what is and is not guaranteed when that happens. -WRITER_LEASE_HEARTBEAT_SCHEDULER_SLACK_SECONDS = 0.5 - -WRITER_LEASE_HEARTBEAT_INTERVAL_SECONDS = 0.25 -WRITER_LEASE_HEARTBEAT_MONITOR_SECONDS = 0.05 -WRITER_LEASE_HEARTBEAT_EXIT_MARGIN_SECONDS = 0.1 - -# Derived, not chosen: the largest gap between completed guard round trips a -# healthy coordinator can produce (see the module docstring). -WRITER_LEASE_HEARTBEAT_FAILURE_SECONDS = ( - WRITER_LEASE_HEARTBEAT_INTERVAL_SECONDS - + WRITER_LEASE_GUARD_STATEMENT_TIMEOUT_SECONDS - + WRITER_LEASE_HEARTBEAT_SCHEDULER_SLACK_SECONDS -) - -# The smallest adoption silence that satisfies inequality (1) is just above -# 1.95s with the budgets above. 2.0s is the next round value and leaves 0.05s -# of stability surplus above the healthy-gap bound. -DEFAULT_WRITER_LEASE_ADOPTION_SILENCE_SECONDS = 2.0 - -# The monitor wake-delay lateness observed on union-mainnet at 2026-09-04 -# (issue #227), the production evidence that the slack allowance is an -# assumption rather than a bound. Kept as a named constant so the tests that -# judge the policy against it cannot drift from the docstring. -OBSERVED_MONITOR_WAKE_DELAY_SECONDS_227 = 0.648 - -# Heartbeat modes, and the phase names the attribution and metrics use. -# Fixed vocabularies: metric label cardinality must not grow with traffic. -LEASE_HEARTBEAT_MODE_PROOF = "proof" -LEASE_HEARTBEAT_MODE_RENEW = "renew" -LEASE_HEARTBEAT_MODE_FENCE = "fence" -LEASE_HEARTBEAT_MODES = ( - LEASE_HEARTBEAT_MODE_PROOF, - LEASE_HEARTBEAT_MODE_RENEW, - LEASE_HEARTBEAT_MODE_FENCE, -) - -# guard_slot_wait + guard_sql + scheduler_delay = total. guard_client_resume -# is the part of scheduler_delay that fell inside a round trip (the answer -# had left PostgreSQL but this process had not resumed); it is reported -# separately because it is exactly the delay issue #227's client-side -# attribution could not see. -LEASE_HEARTBEAT_PHASES = ( - "guard_slot_wait", - "guard_sql", - "guard_client_resume", - "scheduler_delay", - "total", -) - -# The policy terms the metrics surface exports, in derivation order. Closed -# set: adding a term is a deliberate schema change, not a runtime accident. -LEASE_HEARTBEAT_POLICY_TERMS = ( - "adoption_silence", - "heartbeat_interval", - "failure_budget", - "monitor_interval", - "exit_margin", - "guard_statement_timeout", - "scheduler_slack", - "exit_envelope", - "server_proven_cap", - "max_healthy_server_gap", - "stability_surplus", - "max_guaranteed_monitor_lateness", -) - -LEASE_HEARTBEAT_OUTCOME_PROVEN = "proven" -LEASE_HEARTBEAT_OUTCOME_RENEWED = "renewed" -LEASE_HEARTBEAT_OUTCOME_RENEWAL_DUE = "renewal_due" -LEASE_HEARTBEAT_OUTCOME_DEFERRED = "deferred" -LEASE_HEARTBEAT_OUTCOME_FAILED = "failed" -LEASE_HEARTBEAT_OUTCOMES = ( - LEASE_HEARTBEAT_OUTCOME_PROVEN, - LEASE_HEARTBEAT_OUTCOME_RENEWED, - LEASE_HEARTBEAT_OUTCOME_RENEWAL_DUE, - LEASE_HEARTBEAT_OUTCOME_DEFERRED, - LEASE_HEARTBEAT_OUTCOME_FAILED, -) - -# Monitor wake-delay telemetry (issue #227). Every set below is closed, so -# the rendered cardinality is fixed: the histogram has these buckets and -# +Inf, the late-wake counters carry exactly these slack fractions, and the -# rolling window is a fixed number of slices. -# -# The buckets straddle the shipped policy's decision points: 0.25 (half the -# slack, the stall-probe trigger), 0.4 (0.8x slack), 0.5 (the slack itself), -# and 0.75 / 1.0 (the region the 0.648s production record fell in). -LEASE_MONITOR_WAKE_DELAY_BUCKETS = ( - 0.001, - 0.005, - 0.01, - 0.025, - 0.05, - 0.1, - 0.25, - 0.4, - 0.5, - 0.75, - 1.0, - 2.0, -) -# Label values are the fraction text, so a PromQL author can read -# `slack_fraction="1.0"` as "wakes at least a whole scheduler_slack late". -LEASE_MONITOR_LATE_WAKE_SLACK_FRACTIONS = ("0.5", "0.8", "1.0") -# The rolling-window maximum falls back once a stall ages out of the window, -# unlike the lifetime gauge. Thirty 10s slices give a 5-minute window, the -# same horizon the qbit-tools alert's changes[5m] clause reasons over. -LEASE_MONITOR_WAKE_DELAY_WINDOW_SECONDS = 300.0 -LEASE_MONITOR_WAKE_DELAY_WINDOW_SLICES = 30 -# The stall probe samples the running threads' stacks when a wake is at -# least this fraction of the slack late. Rate-limited so it can never turn a -# stall into a stall storm: at most MAX_SAMPLES per WINDOW, the rest are -# counted as suppressed. -LEASE_MONITOR_STALL_PROBE_TRIGGER_SLACK_FRACTION = 0.5 -LEASE_MONITOR_STALL_PROBE_MAX_SAMPLES_PER_WINDOW = 3 -LEASE_MONITOR_STALL_PROBE_WINDOW_SECONDS = 60.0 - - -class WriterLeaseHeartbeatPolicyError(ValueError): - """An unsafe heartbeat timing policy, refused at startup.""" - - -@dataclass(frozen=True, slots=True) -class WriterLeaseHeartbeatPolicy: - """The five coupled heartbeat timings plus the phases that justify them. - - Constructed from live configuration at startup and from the ledger's - own adoption-silence value, so an operator override reaches the same - inequality the compiled defaults satisfy. Never mutated: a policy is - resolved once per heartbeat start and once per monitor loop. - """ - - adoption_silence_seconds: float - heartbeat_interval_seconds: float - failure_budget_seconds: float - monitor_interval_seconds: float - exit_margin_seconds: float - guard_statement_timeout_seconds: float = ( - WRITER_LEASE_GUARD_STATEMENT_TIMEOUT_SECONDS - ) - scheduler_slack_seconds: float = ( - WRITER_LEASE_HEARTBEAT_SCHEDULER_SLACK_SECONDS - ) - - @property - def exit_envelope_seconds(self) -> float: - """Wall-clock the monitor needs to notice and finish a hard exit. - - Four terms, and leaving any of them out is a split-brain window: - - ``exit_margin`` - the hard exit's own budget once the decision is taken. - one ``monitor_interval`` - poll granularity. The monitor can be a full period past the - instant the cap actually elapsed before it looks. - ``scheduler_slack`` - the monitor thread's own budgeted lateness. The monitor is a - Python thread in the same process, behind the same GIL, as the - heartbeat whose stalls this policy explicitly budgets for; it - cannot be assumed to wake on time when the heartbeat is not. - Omitting this term is what made the first cut of this policy - unsafe: with the shipped budgets a maximally late monitor - observed the stale edge 0.45s *after* the successor's adoption - edge, so a stall the policy said was acceptable produced two - live writers. - one further ``monitor_interval`` - the strictness reserve. Without it the worst-case exit lands - exactly on the adoption edge rather than before it, and the - guarantee is "not after" instead of "before". - - Inequality (1) requires all of this to fit inside the adoption - silence alongside the staleness cap. - """ - return ( - self.exit_margin_seconds - + 2.0 * self.monitor_interval_seconds - + self.scheduler_slack_seconds - ) - - @property - def server_proven_cap_seconds(self) -> float: - """Staleness cap the monitor enforces on completed round trips. - - The whole remaining silence budget after the exit envelope, floored - at the failure budget so a deliberately tiny (advised) silence - degrades to the plain liveness bound rather than killing lawful - single-statement verifications. - """ - return max( - self.failure_budget_seconds, - self.adoption_silence_seconds - self.exit_envelope_seconds, - ) - - @property - def max_healthy_server_gap_seconds(self) -> float: - """Largest gap between round trips a healthy coordinator produces.""" - return ( - self.heartbeat_interval_seconds - + self.guard_statement_timeout_seconds - + self.scheduler_slack_seconds - ) - - @property - def stability_surplus_seconds(self) -> float: - """Headroom the staleness cap keeps above the healthy-gap bound.""" - return ( - self.server_proven_cap_seconds - - self.max_healthy_server_gap_seconds - ) - - @property - def max_guaranteed_monitor_lateness_seconds(self) -> float: - """Largest real monitor lateness for which exit precedes adoption. - - Inequality (4) of the module docstring. The monitor that first - observes the stale edge does so no later than ``p + cap + - monitor_interval + L`` and exits ``exit_margin`` later; the - successor is eligible no earlier than ``p + adoption_silence``. So - the exit is guaranteed to precede adoption exactly when ``L`` is - below this value, which equals ``scheduler_slack + - monitor_interval`` whenever the cap is not floored at the failure - budget. A wake later than this is an exit-guarantee breach for - that beat, whether or not a guard loss coincided. - """ - return max( - 0.0, - self.adoption_silence_seconds - - self.server_proven_cap_seconds - - self.monitor_interval_seconds - - self.exit_margin_seconds, - ) - - def exit_guarantee_overrun_seconds(self, monitor_lateness_seconds: float) -> float: - """How far one monitor wake pushed the worst-case exit past adoption. - - Zero while the wake is within - :attr:`max_guaranteed_monitor_lateness_seconds`; otherwise the time - by which the process could still have been alive after the - successor's adoption edge on that beat, had its guard session died. - """ - return max( - 0.0, - float(monitor_lateness_seconds) - - self.max_guaranteed_monitor_lateness_seconds, - ) - - def violations(self) -> tuple[str, ...]: - """Reasons this policy cannot guarantee exit before adoption.""" - reasons: list[str] = [] - for name in ( - "adoption_silence_seconds", - "heartbeat_interval_seconds", - "failure_budget_seconds", - "monitor_interval_seconds", - "exit_margin_seconds", - "guard_statement_timeout_seconds", - "scheduler_slack_seconds", - ): - value = float(getattr(self, name)) - if not math.isfinite(value) or value < 0.0: - reasons.append(f"{name} must be finite and non-negative, got {value!r}") - if reasons: - return tuple(reasons) - if self.heartbeat_interval_seconds <= 0.0: - reasons.append("heartbeat_interval_seconds must be positive") - if self.monitor_interval_seconds <= 0.0: - reasons.append("monitor_interval_seconds must be positive") - if self.adoption_silence_seconds <= 0.0: - reasons.append("adoption_silence_seconds must be positive") - if self.failure_budget_seconds <= self.heartbeat_interval_seconds: - reasons.append( - "failure budget " - f"{self.failure_budget_seconds:g}s must exceed the heartbeat " - f"interval {self.heartbeat_interval_seconds:g}s, or one idle " - "wait alone exhausts the liveness budget" - ) - if ( - self.adoption_silence_seconds - <= self.failure_budget_seconds + self.exit_envelope_seconds - ): - reasons.append( - "adoption silence " - f"{self.adoption_silence_seconds:g}s must exceed the failure " - f"budget {self.failure_budget_seconds:g}s plus the exit " - f"envelope {self.exit_envelope_seconds:g}s (exit margin " - f"{self.exit_margin_seconds:g}s + 2 x monitor interval " - f"{self.monitor_interval_seconds:g}s + scheduler slack " - f"{self.scheduler_slack_seconds:g}s), or a coordinator that " - "lost its guarded session may still be live when a " - "replacement becomes adoption-eligible" - ) - return tuple(reasons) - - def advisories(self) -> tuple[str, ...]: - """Reasons this policy is safe but will false-exit under tail latency.""" - if self.violations(): - return () - if self.stability_surplus_seconds >= 0.0: - return () - return ( - "server-proven staleness cap " - f"{self.server_proven_cap_seconds:g}s is below the largest gap a " - "healthy coordinator can leave between completed guard round " - f"trips ({self.max_healthy_server_gap_seconds:g}s = heartbeat " - f"interval {self.heartbeat_interval_seconds:g}s + guard statement " - f"timeout {self.guard_statement_timeout_seconds:g}s + scheduler " - f"slack {self.scheduler_slack_seconds:g}s); ordinary verification " - "tail latency will hard-exit a healthy coordinator (issue #212). " - "Raise the adoption silence or lower the heartbeat interval", - ) - - def validate(self) -> None: - """Raise :class:`WriterLeaseHeartbeatPolicyError` when unsafe.""" - reasons = self.violations() - if reasons: - raise WriterLeaseHeartbeatPolicyError( - "writer lease heartbeat timing policy is unsafe: " - + "; ".join(reasons) - ) - - def describe(self) -> str: - """One line naming every term of the inequality, for logs and exits.""" - return ( - f"adoption_silence={self.adoption_silence_seconds:g}s " - f"interval={self.heartbeat_interval_seconds:g}s " - f"failure_budget={self.failure_budget_seconds:g}s " - f"monitor={self.monitor_interval_seconds:g}s " - f"exit_margin={self.exit_margin_seconds:g}s " - f"guard_statement_timeout={self.guard_statement_timeout_seconds:g}s " - f"scheduler_slack={self.scheduler_slack_seconds:g}s " - f"exit_envelope={self.exit_envelope_seconds:g}s " - f"=> server_proven_cap={self.server_proven_cap_seconds:g}s " - f"healthy_gap={self.max_healthy_server_gap_seconds:g}s " - f"surplus={self.stability_surplus_seconds:g}s " - f"max_monitor_lateness={self.max_guaranteed_monitor_lateness_seconds:g}s" - ) - - -DEFAULT_WRITER_LEASE_HEARTBEAT_POLICY = WriterLeaseHeartbeatPolicy( - adoption_silence_seconds=DEFAULT_WRITER_LEASE_ADOPTION_SILENCE_SECONDS, - heartbeat_interval_seconds=WRITER_LEASE_HEARTBEAT_INTERVAL_SECONDS, - failure_budget_seconds=WRITER_LEASE_HEARTBEAT_FAILURE_SECONDS, - monitor_interval_seconds=WRITER_LEASE_HEARTBEAT_MONITOR_SECONDS, - exit_margin_seconds=WRITER_LEASE_HEARTBEAT_EXIT_MARGIN_SECONDS, -) -# The compiled defaults must satisfy the inequality they document. A future -# edit that breaks it fails at import rather than in production. -DEFAULT_WRITER_LEASE_HEARTBEAT_POLICY.validate() -assert not DEFAULT_WRITER_LEASE_HEARTBEAT_POLICY.advisories() - - -@dataclass(frozen=True, slots=True) -class WriterLeaseVerificationPhases: - """Where one guard verification's wall-clock actually went. - - The operator question issue #212 could not answer was "which phase - consumed the safety envelope?". Every heartbeat attempt now finishes - with this breakdown, the monitor's hard exit quotes the last one, and - the metrics surface exposes it with fixed cardinality. - - ``slot_wait + guard_sql + scheduler_delay == total``. - ``client_resume`` is the part of ``scheduler_delay`` that fell inside a - statement's round trip (issue #227): PostgreSQL had answered, or the - answer was in transit, and this process had not yet resumed. It is - only non-zero when the statement reported its own server-side - execution time (``server_reported``); otherwise ``guard_sql`` is the - whole client-measured span, as it was before #227, which is the - conservative reading. - """ - - mode: str - outcome: str - slot_wait_seconds: float - guard_sql_seconds: float - statement_count: int - scheduler_delay_seconds: float - total_seconds: float - client_resume_seconds: float = 0.0 - server_reported: bool = False - - def summary(self) -> str: - return ( - f"mode={self.mode} outcome={self.outcome} " - f"slot_wait={self.slot_wait_seconds:.3f}s " - f"guard_sql={self.guard_sql_seconds:.3f}s" - f"({self.statement_count} stmt" - f"{', server-timed' if self.server_reported else ''}) " - f"client_resume={self.client_resume_seconds:.3f}s " - f"scheduler={self.scheduler_delay_seconds:.3f}s " - f"total={self.total_seconds:.3f}s" - ) - - def phase_seconds(self) -> dict[str, float]: - """Fixed-key phase map; keys are :data:`LEASE_HEARTBEAT_PHASES`.""" - return { - "guard_slot_wait": self.slot_wait_seconds, - "guard_sql": self.guard_sql_seconds, - "guard_client_resume": self.client_resume_seconds, - "scheduler_delay": self.scheduler_delay_seconds, - "total": self.total_seconds, - } - - -UNATTRIBUTED_PHASES = WriterLeaseVerificationPhases( - mode="none", - outcome="none", - slot_wait_seconds=0.0, - guard_sql_seconds=0.0, - statement_count=0, - scheduler_delay_seconds=0.0, - total_seconds=0.0, -) - - -class WriterLeaseVerificationAttempt: - """Monotonic phase accumulator for one guard verification attempt. - - Written only by the thread running the attempt and read only after - :meth:`finish` publishes an immutable snapshot, so it needs no lock. - Marks are best effort: a verifier that reports no slot acquisition and - no statement completion still yields a truthful total, with the whole - duration landing in ``scheduler_delay`` — which is itself the signal - that the attempt never reached PostgreSQL. - """ - - __slots__ = ( - "mode", - "_monotonic", - "_started", - "_slot_acquired", - "_last_statement_end", - "_statement_count", - "_pending_send", - "_proven_edge", - "_server_seconds", - "_server_reported", - ) - - def __init__( - self, - mode: str, - *, - monotonic: Callable[[], float] | None = None, - ) -> None: - self.mode = mode - # Resolved at construction, not bound as a default argument: tests - # that drive the lease lifecycle patch ``time.monotonic`` on the - # module object, and a default captured at import time would keep - # reading the real clock behind their backs. - self._monotonic = monotonic or time.monotonic - self._started = self._monotonic() - self._slot_acquired: float | None = None - self._last_statement_end: float | None = None - self._statement_count = 0 - # The send edge of the statement currently in flight, and the - # conservative "PostgreSQL was alive no earlier than this" edge of - # the newest completed round trip. Both default to the attempt's - # start so a verifier that reports no marks still yields an edge - # that cannot overstate freshness. - self._pending_send = self._started - self._proven_edge = self._started - # Server-reported execution time, summed over the statements that - # reported one (issue #227). Kept apart from the client marks: it - # refines the attribution and never touches the proven edge. - self._server_seconds = 0.0 - self._server_reported = False - - @property - def started_monotonic(self) -> float: - return self._started - - @property - def proven_edge_monotonic(self) -> float: - """Conservative server-proven edge of the newest completed round trip.""" - return self._proven_edge - - def slot_acquired(self) -> None: - """The guard session's serialized query slot was handed over. - - This is also the send edge of the attempt's first statement: the - guard issues it immediately after handing over the slot. - """ - if self._slot_acquired is None: - self._slot_acquired = self._monotonic() - self._pending_send = self._slot_acquired - - def statement_completed(self) -> float: - """One guarded statement's round trip returned a result. - - Returns the conservative server-proven edge for that round trip — - when the statement was sent, not when the answer was handled — and - arms the next statement's send edge, since a followup leaves for - PostgreSQL as soon as this result is in hand. - """ - now = self._monotonic() - self._proven_edge = self._pending_send - self._pending_send = now - self._last_statement_end = now - self._statement_count += 1 - return self._proven_edge - - def statement_server_seconds(self, seconds: float | None) -> None: - """One guarded statement reported its own server-side execution time. - - ``seconds`` is ``clock_timestamp() - statement_timestamp()`` as - evaluated by the statement itself, so it excludes the answer's - transit and everything this process did after receipt. ``None`` - (a statement that does not report, or a malformed value) leaves - the attribution on its conservative client-side reading. Never - consulted for the server-proven edge: a server clock reading is - not a monotonic instant in this process, and the exit-ordering - proof needs the send edge. - """ - if seconds is None: - return - try: - value = float(seconds) - except (TypeError, ValueError): - return - if not math.isfinite(value) or value < 0.0: - return - self._server_seconds += value - self._server_reported = True - - def finish(self, outcome: str) -> WriterLeaseVerificationPhases: - finished = self._monotonic() - total = max(0.0, finished - self._started) - slot_wait = ( - max(0.0, self._slot_acquired - self._started) - if self._slot_acquired is not None - else 0.0 - ) - # Statement 1 spans [slot acquisition, first response]; statement k - # spans [response k-1, response k]. Without server-reported timing - # the sub-millisecond client gap between them is charged to SQL, - # which keeps the attribution conservative rather than flattering - # the database. With it, guard_sql is the server's own execution - # time (clamped to the span it must physically fit inside) and the - # rest of the span is client resume: transit plus this process's - # delay in handling an answer PostgreSQL had already given, which - # is the GIL stall issue #227 could not see. - if self._last_statement_end is not None and self._slot_acquired is not None: - statement_span = max(0.0, self._last_statement_end - self._slot_acquired) - else: - statement_span = 0.0 - if self._server_reported: - guard_sql = min(statement_span, self._server_seconds) - client_resume = statement_span - guard_sql - else: - guard_sql = statement_span - client_resume = 0.0 - return WriterLeaseVerificationPhases( - mode=self.mode, - outcome=outcome, - slot_wait_seconds=slot_wait, - guard_sql_seconds=guard_sql, - statement_count=self._statement_count, - scheduler_delay_seconds=max(0.0, total - slot_wait - guard_sql), - total_seconds=total, - client_resume_seconds=client_resume, - server_reported=self._server_reported, - ) diff --git a/rust-toolchain.toml b/rust-toolchain.toml new file mode 100644 index 000000000..a06e41d30 --- /dev/null +++ b/rust-toolchain.toml @@ -0,0 +1,9 @@ +# The one source of the Rust toolchain pin. rustup reads this file, so every +# cargo invocation in the checkout, and every CI job, uses exactly this +# version. scripts/check_toolchain_pin.py holds [workspace.package].rust-version +# in Cargo.toml and the builder image in lab/prism/Dockerfile to the same +# version; bump all three together. +[toolchain] +channel = "1.98.1" +profile = "minimal" +components = ["rustfmt", "clippy"] diff --git a/scripts/check-env.sh b/scripts/check-env.sh index 8de1c3f95..edfd5fe03 100644 --- a/scripts/check-env.sh +++ b/scripts/check-env.sh @@ -3,6 +3,25 @@ set -euo pipefail ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +# Compose must see the caller's environment, not the shell-only example defaults +# loaded below. Preserve even explicit empty overrides without putting values in +# external command arguments, files, or diagnostics. +COMPOSE_INPUT_NAMES=() +COMPOSE_INPUT_VALUES=() +COMPOSE_INPUT_COUNT=0 +COMPOSE_INPUT_KEYS='|' +while IFS= read -r name; do + COMPOSE_INPUT_KEYS+="${name}|" + # Readonly shell internals cannot be changed by sourced configuration and + # must not be assigned during restoration (some also vary in subshells). + declaration="$(declare -p "${name}")" + declaration="${declaration#declare -}" + [[ "${declaration%% *}" != *r* ]] || continue + COMPOSE_INPUT_NAMES+=("${name}") + COMPOSE_INPUT_VALUES+=("${!name}") + COMPOSE_INPUT_COUNT=$((COMPOSE_INPUT_COUNT + 1)) +done < <(compgen -e) +PUBLIC_COMPOSE_FILES=() ENV_DEPLOY_ENV_FILE="${DEPLOY_ENV_FILE:-}" ENV_QBIT_PROVIDER="${QBIT_PROVIDER:-}" ENV_MINING_LANES="${MINING_LANES:-}" @@ -27,8 +46,8 @@ ENV_QBIT_DATA_SOURCE="${QBIT_DATA_SOURCE:-}" ENV_BITCOIN_DATA_SOURCE="${BITCOIN_DATA_SOURCE:-}" ENV_PRISM_POSTGRES_DATA_SOURCE="${PRISM_POSTGRES_DATA_SOURCE:-}" ENV_PRISM_POSTGRES_WAL_SOURCE="${PRISM_POSTGRES_WAL_SOURCE:-}" -ENV_PRISM_POSTGRES_REPLICA_DATA_SOURCE="${PRISM_POSTGRES_REPLICA_DATA_SOURCE:-}" ENV_PRISM_AUDIT_DATA_SOURCE="${PRISM_AUDIT_DATA_SOURCE:-}" +ENV_PRISM_SECRETS_SOURCE="${PRISM_SECRETS_SOURCE:-}" ENV_QBIT_CHAIN="${QBIT_CHAIN:-}" ENV_QBIT_CHAIN_FLAG="${QBIT_CHAIN_FLAG:-}" ENV_QBIT_EXPECTED_GENESIS_HASH="${QBIT_EXPECTED_GENESIS_HASH:-}" @@ -65,20 +84,19 @@ ENV_BITCOIN_RPC_PASSWORD="${BITCOIN_RPC_PASSWORD:-}" ENV_BITCOIN_RPC_PORT_HOST="${BITCOIN_RPC_PORT_HOST:-}" ENV_PRISM_DATABASE_URL="${PRISM_DATABASE_URL:-}" ENV_PRISM_POSTGRES_PASSWORD="${PRISM_POSTGRES_PASSWORD:-}" -ENV_PRISM_POSTGRES_PSQL_COMMAND="${PRISM_POSTGRES_PSQL_COMMAND:-}" -ENV_PRISM_LEDGER_WRITER_ID="${PRISM_LEDGER_WRITER_ID:-}" -ENV_PRISM_LEDGER_WRITER_EPOCH="${PRISM_LEDGER_WRITER_EPOCH:-}" ENV_PRISM_LEDGER_WRITER_SESSION_TOKEN="${PRISM_LEDGER_WRITER_SESSION_TOKEN:-}" ENV_PRISM_LEDGER_WRITER_PUBLIC_KEY_HEX="${PRISM_LEDGER_WRITER_PUBLIC_KEY_HEX:-}" ENV_PRISM_MANIFEST_SIGNING_SEED_HEX="${PRISM_MANIFEST_SIGNING_SEED_HEX:-}" ENV_PRISM_LEDGER_ATTESTATION_SIGNING_SEED_HEX="${PRISM_LEDGER_ATTESTATION_SIGNING_SEED_HEX:-}" ENV_PRISM_AUDIT_DIR="${PRISM_AUDIT_DIR:-}" -ENV_PRISM_EVIDENCE_PATH="${PRISM_EVIDENCE_PATH:-}" ENV_PRISM_ALLOW_MEMORY_LEDGER="${PRISM_ALLOW_MEMORY_LEDGER:-}" ENV_PRISM_ALLOW_TEST_SIGNING_SEEDS="${PRISM_ALLOW_TEST_SIGNING_SEEDS:-}" ENV_PRISM_ALLOW_BUNDLE_EMBEDDED_LEDGER_KEY="${PRISM_ALLOW_BUNDLE_EMBEDDED_LEDGER_KEY:-}" ENV_PRISM_ALLOW_FIXED_LEDGER_SESSION_TOKEN="${PRISM_ALLOW_FIXED_LEDGER_SESSION_TOKEN:-}" ENV_PRISM_CTV_SETTLEMENT_ENABLED="${PRISM_CTV_SETTLEMENT_ENABLED:-}" +ENV_PRISM_POOL_FEE_ENABLED="${PRISM_POOL_FEE_ENABLED:-}" +ENV_PRISM_POOL_FEE_ADDRESS="${PRISM_POOL_FEE_ADDRESS:-}" +ENV_PRISM_POOL_FEE_P2MR_PROGRAM_HEX="${PRISM_POOL_FEE_P2MR_PROGRAM_HEX:-}" ENV_PRISM_CTV_FANOUT_FEE_MARKET_RATE_BITS_PER_1000_WEIGHT="${PRISM_CTV_FANOUT_FEE_MARKET_RATE_BITS_PER_1000_WEIGHT:-}" ENV_PRISM_CTV_FANOUT_FEE_PREMIUM_BPS="${PRISM_CTV_FANOUT_FEE_PREMIUM_BPS:-}" ENV_PRISM_STRATUM_SHARE_DIFF="${PRISM_STRATUM_SHARE_DIFF:-}" @@ -92,9 +110,6 @@ ENV_PRISM_STRATUM_VARDIFF_MAX_STEP_UP="${PRISM_STRATUM_VARDIFF_MAX_STEP_UP:-}" ENV_PRISM_STRATUM_VARDIFF_MAX_STEP_DOWN="${PRISM_STRATUM_VARDIFF_MAX_STEP_DOWN:-}" ENV_PRISM_STRATUM_VARDIFF_EWMA_ALPHA="${PRISM_STRATUM_VARDIFF_EWMA_ALPHA:-}" ENV_PRISM_STRATUM_VARDIFF_RETARGET_TOLERANCE="${PRISM_STRATUM_VARDIFF_RETARGET_TOLERANCE:-}" -ENV_PRISM_STRATUM_VARDIFF_IDLE_SWEEP_SECONDS="${PRISM_STRATUM_VARDIFF_IDLE_SWEEP_SECONDS:-}" -ENV_PRISM_SHARE_COMMIT_BATCH_SIZE="${PRISM_SHARE_COMMIT_BATCH_SIZE:-}" -ENV_PRISM_SHARE_COMMIT_LINGER_MILLISECONDS="${PRISM_SHARE_COMMIT_LINGER_MILLISECONDS:-}" ENV_PRISM_SHARE_COMMIT_TIMEOUT_SECONDS="${PRISM_SHARE_COMMIT_TIMEOUT_SECONDS:-}" ENV_PRISM_STRATUM_SEND_TIMEOUT_SECONDS="${PRISM_STRATUM_SEND_TIMEOUT_SECONDS:-}" ENV_PRISM_STRATUM_STALE_GRACE_SECONDS="${PRISM_STRATUM_STALE_GRACE_SECONDS:-}" @@ -166,8 +181,8 @@ for name in \ BITCOIN_DATA_SOURCE \ PRISM_POSTGRES_DATA_SOURCE \ PRISM_POSTGRES_WAL_SOURCE \ - PRISM_POSTGRES_REPLICA_DATA_SOURCE \ - PRISM_AUDIT_DATA_SOURCE; do + PRISM_AUDIT_DATA_SOURCE \ + PRISM_SECRETS_SOURCE; do override_name="ENV_${name}" if [[ -n "${!override_name}" ]]; then printf -v "${name}" '%s' "${!override_name}" @@ -278,18 +293,9 @@ fi if [[ -n "${ENV_PRISM_DATABASE_URL}" ]]; then PRISM_DATABASE_URL="${ENV_PRISM_DATABASE_URL}" fi -if [[ -n "${ENV_PRISM_POSTGRES_PSQL_COMMAND}" ]]; then - PRISM_POSTGRES_PSQL_COMMAND="${ENV_PRISM_POSTGRES_PSQL_COMMAND}" -fi if [[ -n "${ENV_PRISM_POSTGRES_PASSWORD}" ]]; then PRISM_POSTGRES_PASSWORD="${ENV_PRISM_POSTGRES_PASSWORD}" fi -if [[ -n "${ENV_PRISM_LEDGER_WRITER_ID}" ]]; then - PRISM_LEDGER_WRITER_ID="${ENV_PRISM_LEDGER_WRITER_ID}" -fi -if [[ -n "${ENV_PRISM_LEDGER_WRITER_EPOCH}" ]]; then - PRISM_LEDGER_WRITER_EPOCH="${ENV_PRISM_LEDGER_WRITER_EPOCH}" -fi if [[ -n "${ENV_PRISM_LEDGER_WRITER_SESSION_TOKEN}" ]]; then PRISM_LEDGER_WRITER_SESSION_TOKEN="${ENV_PRISM_LEDGER_WRITER_SESSION_TOKEN}" fi @@ -305,9 +311,6 @@ fi if [[ -n "${ENV_PRISM_AUDIT_DIR}" ]]; then PRISM_AUDIT_DIR="${ENV_PRISM_AUDIT_DIR}" fi -if [[ -n "${ENV_PRISM_EVIDENCE_PATH}" ]]; then - PRISM_EVIDENCE_PATH="${ENV_PRISM_EVIDENCE_PATH}" -fi if [[ -n "${ENV_PRISM_ALLOW_MEMORY_LEDGER}" ]]; then PRISM_ALLOW_MEMORY_LEDGER="${ENV_PRISM_ALLOW_MEMORY_LEDGER}" fi @@ -323,6 +326,15 @@ fi if [[ -n "${ENV_PRISM_CTV_SETTLEMENT_ENABLED}" ]]; then PRISM_CTV_SETTLEMENT_ENABLED="${ENV_PRISM_CTV_SETTLEMENT_ENABLED}" fi +if [[ -n "${ENV_PRISM_POOL_FEE_ENABLED}" ]]; then + PRISM_POOL_FEE_ENABLED="${ENV_PRISM_POOL_FEE_ENABLED}" +fi +if [[ -n "${ENV_PRISM_POOL_FEE_ADDRESS}" ]]; then + PRISM_POOL_FEE_ADDRESS="${ENV_PRISM_POOL_FEE_ADDRESS}" +fi +if [[ -n "${ENV_PRISM_POOL_FEE_P2MR_PROGRAM_HEX}" ]]; then + PRISM_POOL_FEE_P2MR_PROGRAM_HEX="${ENV_PRISM_POOL_FEE_P2MR_PROGRAM_HEX}" +fi if [[ -n "${ENV_PRISM_CTV_FANOUT_FEE_MARKET_RATE_BITS_PER_1000_WEIGHT}" ]]; then PRISM_CTV_FANOUT_FEE_MARKET_RATE_BITS_PER_1000_WEIGHT="${ENV_PRISM_CTV_FANOUT_FEE_MARKET_RATE_BITS_PER_1000_WEIGHT}" fi @@ -338,7 +350,6 @@ for name in \ PRISM_STRATUM_VARDIFF_MAX_STEP_DOWN \ PRISM_STRATUM_VARDIFF_EWMA_ALPHA \ PRISM_STRATUM_VARDIFF_RETARGET_TOLERANCE \ - PRISM_STRATUM_VARDIFF_IDLE_SWEEP_SECONDS \ PRISM_SHARE_COMMIT_TIMEOUT_SECONDS \ PRISM_STRATUM_SEND_TIMEOUT_SECONDS; do override_name="ENV_${name}" @@ -355,12 +366,6 @@ fi if [[ -n "${ENV_PRISM_STRATUM_VARDIFF_START_DIFF}" ]]; then PRISM_STRATUM_VARDIFF_START_DIFF="${ENV_PRISM_STRATUM_VARDIFF_START_DIFF}" fi -if [[ -n "${ENV_PRISM_SHARE_COMMIT_BATCH_SIZE}" ]]; then - PRISM_SHARE_COMMIT_BATCH_SIZE="${ENV_PRISM_SHARE_COMMIT_BATCH_SIZE}" -fi -if [[ -n "${ENV_PRISM_SHARE_COMMIT_LINGER_MILLISECONDS}" ]]; then - PRISM_SHARE_COMMIT_LINGER_MILLISECONDS="${ENV_PRISM_SHARE_COMMIT_LINGER_MILLISECONDS}" -fi if [[ -n "${ENV_PRISM_STRATUM_STALE_GRACE_SECONDS}" ]]; then PRISM_STRATUM_STALE_GRACE_SECONDS="${ENV_PRISM_STRATUM_STALE_GRACE_SECONDS}" fi @@ -597,40 +602,22 @@ check_absolute_storage_source() { check_distinct_storage_sources() { local names=("$@") - local args=() name output - command -v python3 >/dev/null 2>&1 || fail "python3 is required to validate storage paths" - for name in "${names[@]}"; do - args+=("${name}" "${!name}") + local left right + local left_name right_name + local left_value right_value + + for ((left = 0; left < ${#names[@]}; left++)); do + left_name="${names[left]}" + left_value="${!left_name}" + left_value="${left_value%/}" + for ((right = left + 1; right < ${#names[@]}; right++)); do + right_name="${names[right]}" + right_value="${!right_name}" + right_value="${right_value%/}" + [[ "${left_value}" != "${right_value}" ]] || \ + fail "release provenance storage sources ${left_name} and ${right_name} must be distinct" + done done - if ! output="$(python3 - "${args[@]}" <<'PYTHON' -from pathlib import Path -import sys - -sources = [] -try: - for name, raw in zip(sys.argv[1::2], sys.argv[2::2]): - path = Path(raw).resolve() - identities = [] - for parent in (path, *path.parents): - try: - stat = parent.stat() - identities.append((stat.st_dev, stat.st_ino)) - except FileNotFoundError: - identities.append(None) - for other_name, other_path, other_ids in sources: - if (path == other_path or path in other_path.parents - or other_path in path.parents - or identities[0] is not None and identities[0] in other_ids - or other_ids[0] is not None and other_ids[0] in identities): - sys.exit(f"release provenance storage sources {other_name} and {name} " - "must be distinct and non-overlapping (including filesystem aliases)") - sources.append((name, path, identities)) -except (OSError, RuntimeError): - sys.exit(f"cannot safely resolve release provenance storage source {name}") -PYTHON - )"; then - fail "${output}" - fi } check_production_deployment_inputs() { @@ -652,13 +639,13 @@ check_production_deployment_inputs() { check_digest_image PRISM_POSTGRES_IMAGE check_absolute_storage_source PRISM_POSTGRES_DATA_SOURCE check_absolute_storage_source PRISM_POSTGRES_WAL_SOURCE - check_absolute_storage_source PRISM_POSTGRES_REPLICA_DATA_SOURCE check_absolute_storage_source PRISM_AUDIT_DATA_SOURCE + check_absolute_storage_source PRISM_SECRETS_SOURCE storage_names+=( PRISM_POSTGRES_DATA_SOURCE PRISM_POSTGRES_WAL_SOURCE - PRISM_POSTGRES_REPLICA_DATA_SOURCE PRISM_AUDIT_DATA_SOURCE + PRISM_SECRETS_SOURCE ) fi check_distinct_storage_sources "${storage_names[@]}" @@ -683,40 +670,32 @@ require_lab_mode() { check_prism_production_difficulty() { local output - command -v python3 >/dev/null 2>&1 || fail "python3 is required to validate production PRISM difficulty" - if ! output="$(python3 - \ - "${PRISM_STRATUM_SHARE_DIFF:-}" \ - "${PRISM_STRATUM_VARDIFF_MIN_DIFF:-}" \ - "${PRISM_STRATUM_VARDIFF_START_DIFF:-}" \ - "${PRISM_STRATUM_VARDIFF_MAX_DIFF:-}" 2>&1 <<'PY' -from decimal import Decimal, InvalidOperation -import sys - -names = ( - "PRISM_STRATUM_SHARE_DIFF", - "PRISM_STRATUM_VARDIFF_MIN_DIFF", - "PRISM_STRATUM_VARDIFF_START_DIFF", - "PRISM_STRATUM_VARDIFF_MAX_DIFF", -) -values = {} -for name, raw_value in zip(names, sys.argv[1:]): - if not raw_value: - raise SystemExit(f"production mode requires an explicit {name}") - try: - value = Decimal(raw_value) - except InvalidOperation: - raise SystemExit(f"{name} must be a decimal number") - if not value.is_finite() or value <= 0: - raise SystemExit(f"{name} must be positive") - if value == Decimal("0.000000001"): - raise SystemExit(f"{name} cannot use the lab-only 1e-9 difficulty") - values[name] = value -if values["PRISM_STRATUM_VARDIFF_MIN_DIFF"] > values["PRISM_STRATUM_VARDIFF_START_DIFF"]: - raise SystemExit("production vardiff minimum exceeds its start difficulty") -if values["PRISM_STRATUM_VARDIFF_START_DIFF"] > values["PRISM_STRATUM_VARDIFF_MAX_DIFF"]: - raise SystemExit("production vardiff start exceeds its maximum difficulty") -PY - )"; then + # The native server validates the same bounds on startup. Keep the host-side + # Compose preflight independent of a Python interpreter or a host Rust build. + # ENVIRON preserves literal backslashes that awk -v would interpret. + if ! output="$(PRISM_CHECK_SHARE_DIFF="${PRISM_STRATUM_SHARE_DIFF:-}" \ + PRISM_CHECK_MIN_DIFF="${PRISM_STRATUM_VARDIFF_MIN_DIFF:-}" \ + PRISM_CHECK_START_DIFF="${PRISM_STRATUM_VARDIFF_START_DIFF:-}" \ + PRISM_CHECK_MAX_DIFF="${PRISM_STRATUM_VARDIFF_MAX_DIFF:-}" LC_ALL=C awk 'BEGIN { + share=ENVIRON["PRISM_CHECK_SHARE_DIFF"]; + minimum=ENVIRON["PRISM_CHECK_MIN_DIFF"]; + start=ENVIRON["PRISM_CHECK_START_DIFF"]; + maximum=ENVIRON["PRISM_CHECK_MAX_DIFF"]; + names[1]="PRISM_STRATUM_SHARE_DIFF"; values[1]=share; + names[2]="PRISM_STRATUM_VARDIFF_MIN_DIFF"; values[2]=minimum; + names[3]="PRISM_STRATUM_VARDIFF_START_DIFF"; values[3]=start; + names[4]="PRISM_STRATUM_VARDIFF_MAX_DIFF"; values[4]=maximum; + for (i=1;i<=4;i++) { + if (values[i]=="") {print "production mode requires an explicit " names[i];exit 1} + if (values[i] ~ /^-/ || tolower(values[i]) ~ /^(nan|inf|infinity)$/) {print names[i] " must be positive";exit 1} + if (values[i] !~ /^[+]?([0-9]+([.][0-9]*)?|[.][0-9]+)([eE][+-]?[0-9]+)?$/) {print names[i] " must be a decimal number";exit 1} + n=values[i]+0; + if (n<=0 || n>1.7976931348623157e308) {print names[i] " must be positive and finite";exit 1} + if (n==1e-9) {print names[i] " cannot use the lab-only 1e-9 difficulty";exit 1} + } + if (minimum+0>start+0) {print "production vardiff minimum exceeds its start difficulty";exit 1} + if (start+0>maximum+0) {print "production vardiff start exceeds its maximum difficulty";exit 1} + }')"; then fail "${output}" fi } @@ -727,6 +706,7 @@ check_production_gate() { local qbit_tools_production=0 local readiness_gate local launch_readiness_checks="unset" + local name value if production_mode_enabled; then production_enabled=1 @@ -766,17 +746,21 @@ check_production_gate() { fail "production mode rejects ${name}=1" fi done - [[ -n "${PRISM_DATABASE_URL:-}" || -n "${PRISM_POSTGRES_PSQL_COMMAND:-}" ]] || fail "production mode requires PRISM_DATABASE_URL or PRISM_POSTGRES_PSQL_COMMAND" + [[ -n "${PRISM_DATABASE_URL:-}" ]] || fail "production mode requires PRISM_DATABASE_URL" [[ "${PRISM_POSTGRES_PASSWORD:-}" != "change-this" ]] || fail "production mode requires a non-default PRISM_POSTGRES_PASSWORD" [[ "${PRISM_DATABASE_URL:-}" != *"change-this"* ]] || fail "production mode requires a non-default PRISM_DATABASE_URL" - [[ -n "${PRISM_MANIFEST_SIGNING_SEED_HEX:-}" ]] || fail "production mode requires PRISM_MANIFEST_SIGNING_SEED_HEX" - [[ -n "${PRISM_LEDGER_ATTESTATION_SIGNING_SEED_HEX:-}" ]] || fail "production mode requires PRISM_LEDGER_ATTESTATION_SIGNING_SEED_HEX" + # Production frontends read signing seeds only from mounted files; a + # direct value (anything but whitespace, as the native reader trims it) + # would put the seed in the process environment. + for name in \ + PRISM_MANIFEST_SIGNING_SEED_HEX \ + PRISM_LEDGER_ATTESTATION_SIGNING_SEED_HEX; do + value="${!name:-}" + [[ -z "${value//[[:space:]]/}" ]] || \ + fail "production mode requires mounted ${name}_FILE; unset ${name}" + done [[ -n "${PRISM_LEDGER_WRITER_PUBLIC_KEY_HEX:-}" ]] || fail "production mode requires PRISM_LEDGER_WRITER_PUBLIC_KEY_HEX" - [[ -n "${PRISM_LEDGER_WRITER_ID:-}" ]] || fail "production mode requires PRISM_LEDGER_WRITER_ID" - [[ -n "${PRISM_LEDGER_WRITER_EPOCH:-}" ]] || fail "production mode requires PRISM_LEDGER_WRITER_EPOCH" - [[ -z "${PRISM_LEDGER_WRITER_SESSION_TOKEN:-}" ]] || fail "production mode requires managed ledger session tokens; unset PRISM_LEDGER_WRITER_SESSION_TOKEN" - [[ -n "${PRISM_AUDIT_DIR:-}" ]] || fail "production mode requires PRISM_AUDIT_DIR" - [[ -n "${PRISM_EVIDENCE_PATH:-}" ]] || fail "production mode requires PRISM_EVIDENCE_PATH" + [[ -z "${PRISM_LEDGER_WRITER_SESSION_TOKEN:-}" ]] || fail "Rust PRISM manages database claims; unset PRISM_LEDGER_WRITER_SESSION_TOKEN" fi if mining_lane_enabled ckpool; then @@ -922,29 +906,18 @@ check_bitcoin_chain_selection() { check_prism_stale_grace() { mining_lane_enabled prism || return 0 - if ! command -v python3 >/dev/null 2>&1; then - # Production already requires python3 (check_prism_production_difficulty); - # lab bring-up does not, so skip rather than add a prerequisite there. - if production_mode_enabled; then - fail "python3 is required to validate PRISM_STRATUM_STALE_GRACE_SECONDS" - fi - printf 'doctor: python3 not found; skipping PRISM_STRATUM_STALE_GRACE_SECONDS validation\n' - return 0 - fi - # Mirror the coordinator's env_nonnegative_float(): the runtime's float - # syntax and range, finite and non-negative. - if ! python3 - "${PRISM_STRATUM_STALE_GRACE_SECONDS:-3}" >/dev/null 2>&1 <<'PY' -import math -import sys - -try: - value = float(sys.argv[1]) -except ValueError: - raise SystemExit(1) -raise SystemExit(0 if math.isfinite(value) and value >= 0 else 1) -PY + # Match Rust's decimal f64 parser and Duration::try_from_secs_f64: finite, + # non-negative and strictly below 2^64 seconds after float conversion. + # ENVIRON preserves literal backslashes that awk -v would interpret. + if ! PRISM_CHECK_STALE_GRACE="${PRISM_STRATUM_STALE_GRACE_SECONDS:-3}" LC_ALL=C awk 'BEGIN { + value=ENVIRON["PRISM_CHECK_STALE_GRACE"]; + if (value ~ /^[[:space:]]*$/) exit 0; + if (value !~ /^[+-]?([0-9]+([.][0-9]*)?|[.][0-9]+)([eE][+-]?[0-9]+)?$/) exit 1; + n=value+0; + exit !(n>=0 && n<18446744073709551616); + }' then - fail "PRISM_STRATUM_STALE_GRACE_SECONDS must be a finite non-negative number" + fail "PRISM_STRATUM_STALE_GRACE_SECONDS must be a finite non-negative number below 18446744073709551616 seconds" fi } @@ -966,6 +939,32 @@ check_ctv_fee_config() { fi } +# .env.example's lab pool fee recipient, which the server refuses in production. +DEVELOPMENT_POOL_FEE_P2MR_PROGRAM_HEX=dfeedfeedfeedfeedfeedfeedfeedfeedfeedfeedfeedfeedfeedfeedfeedfee + +# #525, #535: the server refuses a fee-off pool in every settlement mode at +# startup, in check-config and in self-check. +check_pool_fee_config() { + mining_lane_enabled prism || return 0 + local mode="direct settlement (PRISM_CTV_SETTLEMENT_ENABLED=0)" + if is_true_env "$(ascii_lower "${PRISM_CTV_SETTLEMENT_ENABLED:-0}")"; then + mode="PRISM_CTV_SETTLEMENT_ENABLED=1" + fi + is_true_env "$(ascii_lower "${PRISM_POOL_FEE_ENABLED:-0}")" \ + || fail "${mode} requires PRISM_POOL_FEE_ENABLED=1 (PRISM_POOL_FEE_BPS=0 is allowed) with a pool fee recipient: without a pool fee, sub-floor dust cannot be settled and the first balance below the payout floor stops mining" + local address="${PRISM_POOL_FEE_ADDRESS:-}" + local program + program="$(ascii_lower "${PRISM_POOL_FEE_P2MR_PROGRAM_HEX:-}")" + [[ -n "${address}" || -n "${program}" ]] \ + || fail "PRISM_POOL_FEE_ENABLED=1 needs a pool fee recipient: PRISM_POOL_FEE_ADDRESS, or PRISM_POOL_FEE_P2MR_PROGRAM_HEX with PRISM_POOL_FEE_RECIPIENT_ID" + # The lab program is sourced from .env.example beneath every env file here, + # so it is refused only where no address replaces it; an address beside it + # is left to the server, which sees only what Compose passes. + if production_mode_enabled && [[ -z "${address}" && "${program}" == "${DEVELOPMENT_POOL_FEE_P2MR_PROGRAM_HEX}" ]]; then + fail "production rejects the development pool fee recipient from .env.example; set PRISM_POOL_FEE_ADDRESS (and clear PRISM_POOL_FEE_P2MR_PROGRAM_HEX) or your own PRISM_POOL_FEE_P2MR_PROGRAM_HEX" + fi +} + check_bitcoin_peer_bootstrap() { mining_lane_enabled auxpow || return 0 local chain="${BITCOIN_CHAIN:-regtest}" @@ -984,15 +983,102 @@ check_bitcoin_peer_bootstrap() { fi } -case "${1:-}" in - "") ;; +check_public_reader_credentials() ( + local name index output production=0 prepare_lab_image=0 + local -a compose_args + if [[ "${make_deployment}" == 1 ]]; then + if production_mode_enabled || [[ "${QBIT_CHAIN:-regtest}" == main || "${QBIT_CHAIN:-regtest}" == mainnet ]]; then + production=1 + else + # Reader image preparation follows the qbit production selector. The + # full preflight validates other chains only for their enabled lanes. + check_qbit_chain_selection + # Pinned artifacts are prepared explicitly, including in a lab. Tagged + # lab images are rebuilt so an older image cannot omit the validator. + [[ "${PRISM_COORDINATOR_IMAGE:-}" == *@sha256:* ]] || prepare_lab_image=1 + fi + fi + # Restore precisely the environment Compose would receive from the caller. + while IFS= read -r name; do + [[ "${COMPOSE_INPUT_KEYS}" == *"|${name}|"* ]] || export -n "${name?}" + done < <(compgen -e) + for ((index=0; index<COMPOSE_INPUT_COUNT; index++)); do + export "${COMPOSE_INPUT_NAMES[index]}=${COMPOSE_INPUT_VALUES[index]}" + done + compose_args=(compose --env-file "${ROOT_DIR}/config/upstream.env.example") + if [[ -f "${ROOT_DIR}/config/upstream.env" ]]; then + compose_args=(compose --env-file "${ROOT_DIR}/config/upstream.env") + fi + if [[ -n "${ENV_DEPLOY_ENV_FILE}" ]]; then + local deploy_file="${ENV_DEPLOY_ENV_FILE}" + [[ "${deploy_file}" == /* ]] || deploy_file="${ROOT_DIR}/${deploy_file}" + compose_args+=(--env-file "${deploy_file}") + elif [[ -f "${ROOT_DIR}/.env" ]]; then + compose_args+=(--env-file "${ROOT_DIR}/.env") + fi + compose_args+=(-f "${ROOT_DIR}/compose.yaml") + if [[ "${production}" == 1 ]]; then + compose_args+=(-f "${ROOT_DIR}/compose.production.yaml") + fi + for name in "${PUBLIC_COMPOSE_FILES[@]-}"; do + [[ -n "${name}" ]] || continue + compose_args+=(-f "${name}") + done + if [[ "${prepare_lab_image}" == 1 ]]; then + printf 'doctor: preparing the selected PRISM lab image for reader validation\n' + if ! output="$(docker "${compose_args[@]}" --profile prism build --quiet prism-public-api 2>&1)"; then + fail "public reader image preparation failed; check the selected lab build configuration" + fi + fi + # Run the same public image/environment as deployment, without starting its + # database dependencies or publishing ports. The validator does no network IO. + # Capture Docker/Compose output: even a launcher error may contain env values. + # Compose run otherwise builds an absent image implicitly, even with --pull + # never. Remove only the build recipe in this final, credential-free overlay. + if output="$(printf 'services:\n prism-public-api:\n build: !reset null\n' | \ + docker "${compose_args[@]}" -f - --profile prism run --rm --no-deps --pull never -T \ + prism-public-api qbit-prism-server check-public-database-config 2>&1)"; then + [[ "${output}" == *"PRISM public database configuration valid; authentication is checked by readiness"* ]] || \ + fail "public reader validation returned no result; use an image supporting check-public-database-config" + printf 'doctor: public reader configuration valid; database authentication remains a readiness check\n' + else + # Emit only known, value-free configuration errors, never raw launcher logs. + for name in \ + 'PRISM_DATABASE_URL is required by the public service' \ + 'public PRISM_DATABASE_URL must use postgres or postgresql' \ + 'invalid public PRISM_DATABASE_URL connection options' \ + 'invalid public PRISM_DATABASE_URL' \ + 'production requires non-default database credentials'; do + [[ "${output}" != *"${name}"* ]] || fail "public reader: ${name}" + done + fail "public reader validation failed; prepare the selected PRISM image and ensure Docker Compose and the image support check-public-database-config" + fi +) + +public_reader_only=0 +make_deployment=0 +while [[ $# -gt 0 ]]; do +case "$1" in --require-lab) validate_mining_lanes require_lab_mode exit 0 ;; + --public-reader-only) public_reader_only=1 ;; + --make-deployment) make_deployment=1 ;; + --compose-file) + [[ $# -ge 2 && -n "$2" ]] || fail "--compose-file requires a file" + PUBLIC_COMPOSE_FILES+=("$2") + shift + ;; *) fail "unknown argument: ${1}" ;; esac +shift +done +if [[ "${public_reader_only}" == 1 ]]; then + check_public_reader_credentials + exit 0 +fi export QBIT_PRODUCTION QBIT_TOOLS_PRODUCTION QBIT_CHAIN CKPOOL_NON_TEST_READINESS_GATE export QBIT_MAINNET_LAUNCH_READINESS_CHECKS_ENABLED @@ -1015,6 +1101,7 @@ if mining_lane_enabled auxpow; then fi check_prism_stale_grace check_ctv_fee_config +check_pool_fee_config check_production_gate check_release_provenance_gate @@ -1024,6 +1111,9 @@ fi command -v docker >/dev/null 2>&1 || fail "docker is required" docker info >/dev/null 2>&1 || fail "docker daemon is not reachable" +if mining_lane_enabled prism; then + check_public_reader_credentials +fi case "${QBIT_PROVIDER}" in source) diff --git a/scripts/check_deleted_test_map.py b/scripts/check_deleted_test_map.py new file mode 100644 index 000000000..c200cc3b4 --- /dev/null +++ b/scripts/check_deleted_test_map.py @@ -0,0 +1,817 @@ +#!/usr/bin/env python3 +"""Fail when the deleted-test map stops being checkable evidence. + +`docs/prism-deleted-test-map.md` maps each Python test file the native cutover +deleted to the 3.x.x test that replaces it, or to the open issue that will. It +is parity evidence for the cutover go/no-go, and nothing used to read it: the +issues it linked closed one by one without the map changing. + +Usage: python3 scripts/check_deleted_test_map.py [--root DIR] [--check-issues] [--summary] + +Without flags the check is offline and is what required CI runs: + +- every table is one the map defines and every row parses (a line this script + cannot read is an error, never a skipped row); +- file-row names exactly match the pinned deletion set recorded in + `docs/prism-deleted-test-files.txt`, without needing Git history; +- case-row names exactly match the pinned test-method inventory recorded in + `docs/prism-deleted-test-cases.txt`; +- every `path::name` reference names a file in the repository and a test + function in it (`#[test]`-style attribute in Rust source, ignoring comments + and literal contents, or a Python method included by unittest discovery with + no effective skip-decorator flag); + Rust `#[ignore]` tests must be explicitly selected by the CI shard runner; + Rust functions must also be reachable through enabled modules from a Cargo + workspace target (unsupported conditional/macro inclusion fails closed); +- a status is one of the five the legend defines, the row's text leads with + it, a full or partial row cites a test and an open gap row links its owner issue; +- the summary table and the per-section counts equal the rows; +- the needs triage index lists exactly the needs triage rows; +- every issue link points at this repository's issue of the same number, and + an owner arrow introducing an issue number is followed by a complete link. + +`--check-issues` additionally asks GitHub for the state of every issue the map +names as the owner of uncovered behaviour and fails if one is closed. An owner +is any issue link inside a table row, or an arrow link (`→ [#N](...)`) anywhere +in the map. A closed issue is history: a row mentions it as plain `#N`, and +prose may link it without an arrow. Reads `GITHUB_TOKEN` or `GH_TOKEN` when set. + +`--summary` prints the row counts and the rows that are still open, including +partial rows with a "not covered" remainder but no owner issue, for the sign-off +comment on the qualification issue. + +Exits 0 when the map holds, 1 when it does not, and 2 when `--check-issues` +could not learn an issue's state (network, rate limit, unexpected reply): an +unknown state is reported as unknown, never as open. + +Python citation validation imports test modules and invokes their discovery +hooks, just as CI's Python runner does; it does not run test bodies or fixtures. +The required Python shard runner separately checks actual successful execution +of cited Python tests, so fixture/body skips cannot satisfy parity evidence. +""" + +from __future__ import annotations + +import argparse +from collections import Counter +from dataclasses import dataclass, field +import http.client +import inspect +import json +import math +import os +from pathlib import Path, PurePosixPath +import re +import ssl +import sys +import time +import unittest +import urllib.error +import urllib.parse +import urllib.request + +if __package__: + from .run_rust_test_shard import IGNORED + from .rust_test_reachability import RustReachability +else: + from run_rust_test_shard import IGNORED + from rust_test_reachability import RustReachability + + +ROOT = Path(__file__).resolve().parents[1] +PREFIX = "check-deleted-test-map" +MAP = Path("docs") / "prism-deleted-test-map.md" +FILE_MANIFEST = Path("docs") / "prism-deleted-test-files.txt" +CASE_MANIFEST = Path("docs") / "prism-deleted-test-cases.txt" +REPOSITORY = "Qbit-Org/qbit-mining-bootstrap" +DEFAULT_API_URL = "https://api.github.com" + +FULL = "full" +PARTIAL = "partial" +OPEN_GAP = "open gap" +NEEDS_TRIAGE = "needs triage" +RETIRED = "retired" +STATUSES = (FULL, PARTIAL, OPEN_GAP, NEEDS_TRIAGE, RETIRED) + +FILE_HEADER = ("2.x.x file", "Covered", "3.x.x replacement or status") +CASE_HEADER = ("2.x.x case", "Covered", "3.x.x replacement or status") +TRIAGE_HEADER = ("2.x.x file", "Area", "Reason") +SUMMARY_HEADER = ("Rows", "Full", "Partial", "Open gap", "Needs triage", "Retired") +SUMMARY_STATUSES = (FULL, PARTIAL, OPEN_GAP, NEEDS_TRIAGE, RETIRED) +SECTION_LIST_LEAD = "Sections and row counts:" + +UNESCAPED_PIPE = re.compile(r"(?<!\\)\|") +# Rows may carry `<!-- retired-setting: NAME -->` markers for check_prism_settings.py. +HTML_COMMENT = re.compile(r"<!--.*?-->") +SEPARATOR_CELL = re.compile(r"^:?-{3,}:?$") +NAME_CELL = re.compile(r"^`(?P<name>[^`]+)`$") +CODE_SPAN = re.compile(r"`([^`\n]+)`") +REFERENCE = re.compile( + r"^(?P<path>[A-Za-z0-9_][A-Za-z0-9_./-]*\.(?P<ext>rs|py))::(?P<name>[A-Za-z_][A-Za-z0-9_]*)$" +) +ISSUE_LINK = re.compile(r"(?P<arrow>(?:→|->)\s*)?\[#(?P<label>[0-9]+)\]\((?P<url>[^)\s]*)\)") +# Numeric owner claims, including broken brackets; leave the `→ [#N](…)` +# syntax example and standalone arrows alone. +OWNER_ARROW = re.compile(r"(?:→|->)\s*(?=[\[\s]*#[0-9]+)") +SECTION_LIST_ITEM = re.compile(r"^- (?P<name>.+) \((?P<count>[0-9]+)\)$") +RUST_FN = re.compile( + r"^\s*(?:pub(?:\([^)]*\))?\s+)?(?:async\s+)?fn\s+(?P<name>[A-Za-z_][A-Za-z0-9_]*)\s*[(<]" +) +RUST_TEST_ATTRIBUTE = re.compile(r"#\[\s*(?:[A-Za-z_][A-Za-z0-9_]*::)*test\b") +RUST_IGNORE_ATTRIBUTE = re.compile(r"#\s*\[\s*ignore\s*(?:\]|=)") +# Match character literals as whole tokens so a quote character cannot start +# a string. A lifetime or label has no closing apostrophe and remains code. +RUST_NON_CODE = re.compile( + r'//[^\n]*|/\*|\b(?:br|cr|r)(?P<hashes>\#*)"|"|' + r"'(?:\\(?:u\{[0-9A-Fa-f_]+\}|x[0-9A-Fa-f]{2}|[^\r\n])|[^'\\\r\n\t])'" +) +RUST_BLOCK_DELIMITER = re.compile(r"/\*|\*/") +PYTHON_DEF = re.compile(r"^\s*(?:async\s+)?def\s+(?P<name>[A-Za-z_][A-Za-z0-9_]*)\s*\(") + +REQUEST_TIMEOUT_SECONDS = 10.0 +ATTEMPTS = 3 + + +@dataclass(frozen=True) +class Row: + line: int + name: str + status: str + text: str + section: str + + +@dataclass(frozen=True) +class TriageRow: + line: int + name: str + area: str + text: str + + +@dataclass(frozen=True) +class IssueLink: + line: int + number: int + url: str + arrow: bool + + +@dataclass +class ParsedMap: + file_rows: list[Row] = field(default_factory=list) + case_rows: list[Row] = field(default_factory=list) + triage_rows: list[TriageRow] = field(default_factory=list) + summaries: list[tuple[int, list[str]]] = field(default_factory=list) + section_list: list[tuple[int, str, int]] = field(default_factory=list) + links: list[IssueLink] = field(default_factory=list) + errors: list[str] = field(default_factory=list) + + +def at(line: int, message: str) -> str: + return f"{MAP.as_posix()}:{line}: {message}" + + +def split_cells(line: str) -> list[str] | None: + stripped = HTML_COMMENT.sub("", line).strip() + if not stripped.endswith("|") or stripped.endswith("\\|") or len(stripped) < 2: + return None + return [cell.strip() for cell in UNESCAPED_PIPE.split(stripped)[1:-1]] + + +def parse_row(parsed: ParsedMap, number: int, cells: list[str], section: str, *, case: bool) -> None: + if len(cells) != 3: + parsed.errors.append( + at(number, f"row has {len(cells)} cells, expected 3 (write a literal pipe inside a cell as \\|)") + ) + return + name = NAME_CELL.match(cells[0]) + if name is None: + parsed.errors.append(at(number, f"first cell must be one `code` name, found {cells[0]!r}")) + return + status = cells[1] + if status not in STATUSES: + parsed.errors.append(at(number, f"unknown status {status!r}; the legend defines: {', '.join(STATUSES)}")) + return + row = Row(number, name.group("name"), status, cells[2], section) + (parsed.case_rows if case else parsed.file_rows).append(row) + + +def parse_table(parsed: ParsedMap, table: list[tuple[int, str]], section: str) -> None: + rows: list[tuple[int, list[str]]] = [] + for number, line in table: + cells = split_cells(line) + if cells is None: + parsed.errors.append(at(number, "table line does not end with an unescaped pipe")) + if not rows: + return + continue + rows.append((number, cells)) + header_line, header = rows[0] + kind = tuple(header) + if kind not in (FILE_HEADER, CASE_HEADER, TRIAGE_HEADER, SUMMARY_HEADER): + parsed.errors.append(at(header_line, f"unrecognised table header {header!r}; this check does not know how to read it")) + return + if len(rows) < 2 or not all(SEPARATOR_CELL.match(cell) for cell in rows[1][1]) or len(rows[1][1]) != len(header): + parsed.errors.append(at(header_line, "table header is not followed by a separator row of the same width")) + return + for number, cells in rows[2:]: + if kind == SUMMARY_HEADER: + parsed.summaries.append((number, cells)) + elif kind == TRIAGE_HEADER: + name = NAME_CELL.match(cells[0]) if len(cells) == 3 else None + if name is None: + parsed.errors.append(at(number, "needs triage index row must be `file` | area | reason")) + continue + parsed.triage_rows.append(TriageRow(number, name.group("name"), cells[1], cells[2])) + else: + parse_row(parsed, number, cells, section, case=kind == CASE_HEADER) + + +def parse_map(text: str) -> ParsedMap: + parsed = ParsedMap() + section = "" + table: list[tuple[int, str]] = [] + in_section_list = False + in_fence = False + lines = text.splitlines() + for number, line in enumerate([*lines, ""], start=1): + if line.lstrip().startswith("```"): + in_fence = not in_fence + if in_fence: + continue + if line.startswith("|"): + table.append((number, line)) + continue + if table: + parse_table(parsed, table, section) + table = [] + if line.startswith("## "): + section = line[3:].strip() + if line.strip() == SECTION_LIST_LEAD: + in_section_list = True + elif in_section_list and line.startswith("- "): + item = SECTION_LIST_ITEM.match(line.rstrip()) + if item is None: + parsed.errors.append(at(number, "section list item must read `- <section heading> (<row count>)`")) + else: + parsed.section_list.append((number, item.group("name"), int(item.group("count")))) + elif in_section_list and (line.strip() or parsed.section_list): + # A blank line may separate the lead from its items; anything after them ends the list. + in_section_list = False + for number, line in enumerate(lines, start=1): + linked_arrows: set[int] = set() + for link in ISSUE_LINK.finditer(line): + parsed.links.append( + IssueLink(number, int(link.group("label")), link.group("url"), link.group("arrow") is not None) + ) + if link.group("arrow") is not None: + linked_arrows.add(link.start()) + if any(arrow.start() not in linked_arrows for arrow in OWNER_ARROW.finditer(line)): + parsed.errors.append( + at(number, "an owner arrow must be followed by an issue link with a complete URL, so its state can be checked") + ) + return parsed + + +def rust_code(text: str) -> str: + """Mask comments and literal tokens, preserving code and line boundaries.""" + parts: list[str] = [] + cursor = 0 + while token := RUST_NON_CODE.search(text, cursor): + start, end = token.span() + spelling = token.group() + if spelling == "/*": + depth = 1 + while depth: + delimiter = RUST_BLOCK_DELIMITER.search(text, end) + if delimiter is None: + end = len(text) + break + depth += 1 if delimiter.group() == "/*" else -1 + end = delimiter.end() + elif token.group("hashes") is not None: + closing = '"' + token.group("hashes") + close_at = text.find(closing, end) + end = len(text) if close_at < 0 else close_at + len(closing) + elif spelling == '"': + while end < len(text): + character = text[end] + end += 1 + if character == "\\": + end = min(end + 1, len(text)) + elif character == '"': + break + parts.append(text[cursor:start]) + # Comments separate tokens like whitespace. Literals remain opaque + # tokens so removing them cannot join code on either side into a fn. + replacement = " " if spelling.startswith("/") else "~" + parts.append(re.sub(r"[^\r\n]", replacement, text[start:end])) + cursor = end + parts.append(text[cursor:]) + return "".join(parts) + + +def ignored_rust_test_selected(occurrences: set[tuple[str, str, str, str]]) -> bool: + """Match CI against the proven target identity and qualified function name.""" + for package, kind, target, name in occurrences: + arguments = IGNORED.get((package, kind, target)) + if arguments is None: + continue + filters: list[str] = [] + for argument in arguments: + if argument == "--exact" or re.fullmatch(r"--test-threads=[1-9][0-9]*", argument): + continue + if argument.startswith("-"): + # A new selection/skip option needs explicit handling before + # the checker may claim that CI executes the cited test. + return False + filters.append(argument) + if not filters: + return True + if any(name == value if "--exact" in arguments else value in name for value in filters): + return True + return False + + +def rust_functions(text: str, reachable: dict[int, set[tuple[str, str, str, str]]]) -> tuple[set[str], set[str]]: + """Every source `fn` name, and the subset eligible for CI test citations.""" + lines = rust_code(text).splitlines() + functions: set[str] = set() + tests: set[str] = set() + for index, line in enumerate(lines): + found = RUST_FN.match(line) + if found is None: + continue + functions.add(found.group("name")) + attributes: list[str] = [] + cursor = index - 1 + while cursor >= 0: + above = lines[cursor].strip() + if above.endswith(("{", "}", ";")): + break + attributes.append(above) + cursor -= 1 + attribute_block = " ".join(reversed(attributes)) + occurrences = {entry for entry in reachable.get(index + 1, set()) if entry[3].rsplit("::", 1)[-1] == found.group("name")} + if occurrences and RUST_TEST_ATTRIBUTE.search(attribute_block) and ( + not RUST_IGNORE_ATTRIBUTE.search(attribute_block) + or ignored_rust_test_selected(occurrences) + ): + tests.add(found.group("name")) + return functions, tests + + +def discovered_python_tests(root: Path) -> dict[Path, set[str]] | str: + """Index the source of methods in CI's suite without executing the suite.""" + previous_cwd = Path.cwd() + previous_path = sys.path[:] + loader = unittest.TestLoader() + tests: dict[Path, set[str]] = {} + try: + # Match scripts/run_python_test_shard.py, including imports relative to root. + os.chdir(root) + sys.path.insert(0, str(root)) + pending = [loader.discover("tests", pattern="test_*.py")] + if loader.errors: + return "Python unittest discovery failed:\n" + "\n".join(loader.errors) + while pending: + test = pending.pop() + if isinstance(test, unittest.TestSuite): + pending.extend(test) + continue + method_name = test._testMethodName + method = getattr(test, method_name) + # Match TestCase.run before unwrapping: skip decorators attach the + # flag to the wrapper, and class-level flags may be inherited. + if getattr(type(test), "__unittest_skip__", False) or getattr(method, "__unittest_skip__", False): + continue + method = inspect.unwrap(method) + if not method_name.startswith(loader.testMethodPrefix) or not inspect.isroutine(method): + continue + source = inspect.getsourcefile(method) + if source is not None: + tests.setdefault(Path(source).resolve(), set()).add(method_name) + except Exception as error: + return f"Python unittest discovery failed: {error}" + finally: + os.chdir(previous_cwd) + sys.path[:] = previous_path + return tests + + +class References: + def __init__(self, root: Path) -> None: + self.root = root.resolve() + self.cache: dict[str, tuple[set[str], set[str]] | str] = {} + self.python_tests: dict[Path, set[str]] | str | None = None + self.rust_reachability: RustReachability | str | None = None + + def load(self, path: str, extension: str) -> tuple[set[str], set[str]] | str: + if path not in self.cache: + if ".." in PurePosixPath(path).parts: + self.cache[path] = "path leaves the repository" + else: + target = self.root / path + try: + text = target.read_text(encoding="utf-8") + except FileNotFoundError: + self.cache[path] = "no such file" + except (OSError, UnicodeDecodeError) as error: + self.cache[path] = f"unreadable: {error}" + else: + if extension == "rs": + if self.rust_reachability is None: + try: + self.rust_reachability = RustReachability(self.root, rust_code) + except (OSError, UnicodeDecodeError, ValueError, KeyError, TypeError, AttributeError) as error: + self.rust_reachability = f"cannot establish Cargo target reachability: {error}" + if isinstance(self.rust_reachability, str): + self.cache[path] = self.rust_reachability + else: + self.cache[path] = rust_functions(text, self.rust_reachability.occurrences(target)) + else: + if self.python_tests is None: + self.python_tests = discovered_python_tests(self.root) + if isinstance(self.python_tests, str): + self.cache[path] = self.python_tests + else: + functions = {found.group("name") for line in text.splitlines() if (found := PYTHON_DEF.match(line))} + self.cache[path] = functions, self.python_tests.get(target.resolve(), set()) + return self.cache[path] + + def problem(self, path: str, extension: str, name: str) -> str | None: + loaded = self.load(path, extension) + if isinstance(loaded, str): + return f"`{path}::{name}`: {loaded}" + functions, tests = loaded + keyword = "fn" if extension == "rs" else "def" + if name not in functions: + return f"`{path}::{name}`: no `{keyword} {name}` in that file" + if name not in tests: + wanted = ( + "an enabled function in a reachable Cargo target, a #[test]-style attribute and, " + "for #[ignore], selection by scripts/run_rust_test_shard.py; unsupported cfg/macro inclusion cannot prove reachability" + if extension == "rs" else "non-skipped membership in the discovered unittest suite" + ) + return f"`{path}::{name}`: `{keyword} {name}` exists but is not a test function ({wanted} is required)" + return None + + +def references_in(text: str) -> tuple[list[tuple[str, str, str]], list[str]]: + """Well-formed `path::name` references, and spans that look like one and are not.""" + good: list[tuple[str, str, str]] = [] + malformed: list[str] = [] + for span in CODE_SPAN.findall(text): + if "::" not in span: + continue + found = REFERENCE.match(span) + if found is not None: + good.append((found.group("path"), found.group("ext"), found.group("name"))) + continue + left = span.split("::", 1)[0] + if "/" in left or left.endswith((".rs", ".py")): + malformed.append(span) + return good, malformed + + +def check_references(text: str, root: Path) -> list[str]: + errors: list[str] = [] + references = References(root) + for number, line in enumerate(text.splitlines(), start=1): + good, malformed = references_in(line) + for span in malformed: + errors.append(at(number, f"`{span}` looks like a test reference but is not `<path>.rs|.py::<function>`")) + for path, extension, name in dict.fromkeys(good): + problem = references.problem(path, extension, name) + if problem is not None: + errors.append(at(number, problem)) + return errors + + +def check_rows(parsed: ParsedMap) -> list[str]: + errors: list[str] = [] + linked_lines = {link.line for link in parsed.links} + for label, rows in (("file", parsed.file_rows), ("case", parsed.case_rows)): + seen: dict[str, int] = {} + for row in rows: + if row.name in seen: + errors.append(at(row.line, f"{label} `{row.name}` already has a row at line {seen[row.name]}")) + seen.setdefault(row.name, row.line) + for row in [*parsed.file_rows, *parsed.case_rows]: + lead = next((status for status in STATUSES if row.text.startswith(status)), None) + if row.status == FULL: + if lead not in (None, FULL): + errors.append(at(row.line, f"status is full but the text leads with {lead!r}")) + elif lead != row.status: + errors.append(at(row.line, f"status is {row.status!r} but the text does not lead with it")) + if row.status in (FULL, PARTIAL) and not references_in(row.text)[0]: + errors.append(at(row.line, f"a {row.status} row must cite at least one `path::name` test")) + if row.status == OPEN_GAP and row.line not in linked_lines: + errors.append(at(row.line, "an open gap row must link the issue that closes the gap")) + return errors + + +def check_inventory(rows: list[Row], root: Path, manifest: Path, label: str) -> list[str]: + try: + text = (root / manifest).read_text(encoding="utf-8") + except (OSError, UnicodeDecodeError) as error: + return [f"{manifest.as_posix()}: cannot read deleted-{label} manifest: {error}"] + + errors: list[str] = [] + expected: set[str] = set() + for number, line in enumerate(text.splitlines(), start=1): + name = line.strip() + if not name or name.startswith("#"): + continue + if name in expected: + errors.append(f"{manifest.as_posix()}:{number}: duplicate deleted {label} `{name}`") + expected.add(name) + if not expected: + return [f"{manifest.as_posix()}: manifest has no deleted {label}s"] + + actual = {row.name for row in rows} + for name in sorted(expected - actual): + errors.append(at(1, f"deleted {label} `{name}` is missing from the {label} tables (listed in {manifest.as_posix()})")) + for row in rows: + if row.name not in expected: + errors.append(at(row.line, f"{label} `{row.name}` is not in the deleted-{label} manifest {manifest.as_posix()}")) + return errors + + +def check_triage_index(parsed: ParsedMap) -> list[str]: + errors: list[str] = [] + rows = {row.name: row for row in [*parsed.file_rows, *parsed.case_rows] if row.status == NEEDS_TRIAGE} + index: dict[str, TriageRow] = {} + for entry in parsed.triage_rows: + if entry.name in index: + errors.append(at(entry.line, f"`{entry.name}` is listed twice in the needs triage index")) + index.setdefault(entry.name, entry) + for name, row in rows.items(): + if name not in index: + errors.append(at(row.line, f"`{name}` is needs triage but missing from the needs triage index")) + for name, entry in index.items(): + row = rows.get(name) + if row is None: + errors.append(at(entry.line, f"`{name}` is in the needs triage index but has no needs triage row")) + elif entry.area != row.section: + errors.append(at(entry.line, f"area {entry.area!r} is not the section the row sits in, {row.section!r}")) + return errors + + +def check_counts(parsed: ParsedMap) -> list[str]: + errors: list[str] = [] + if len(parsed.summaries) != 1: + return [at(1, f"expected exactly one summary row under {' | '.join(SUMMARY_HEADER)}, found {len(parsed.summaries)}")] + line, cells = parsed.summaries[0] + if len(cells) != len(SUMMARY_HEADER) or not all(cell.isascii() and cell.isdigit() for cell in cells): + return [at(line, f"summary row must be {len(SUMMARY_HEADER)} whole numbers, found {cells!r}")] + declared = [int(cell) for cell in cells] + actual = Counter(row.status for row in parsed.file_rows) + if declared[0] != len(parsed.file_rows): + errors.append(at(line, f"summary says {declared[0]} rows, the file tables hold {len(parsed.file_rows)}")) + for status, count in zip(SUMMARY_STATUSES, declared[1:]): + if count != actual[status]: + errors.append(at(line, f"summary says {count} {status}, the rows say {actual[status]}")) + + per_section = Counter(row.section for row in parsed.file_rows) + if not parsed.section_list: + errors.append(at(1, f"no `{SECTION_LIST_LEAD}` list found")) + listed: set[str] = set() + for item_line, name, count in parsed.section_list: + if name in listed: + errors.append(at(item_line, f"section {name!r} is listed twice")) + listed.add(name) + if name not in per_section: + errors.append(at(item_line, f"section list names {name!r}, which has no file rows under a heading of that name")) + elif count != per_section[name]: + errors.append(at(item_line, f"section list says {count} rows in {name!r}, the section holds {per_section[name]}")) + for name in per_section: + if name not in listed and parsed.section_list: + errors.append(at(1, f"section {name!r} holds {per_section[name]} rows and is missing from the section list")) + return errors + + +def check_links(parsed: ParsedMap, repository: str) -> list[str]: + errors: list[str] = [] + for link in parsed.links: + expected = f"https://github.com/{repository}/issues/{link.number}" + if link.url != expected: + errors.append(at(link.line, f"[#{link.number}] links {link.url!r}, expected {expected}")) + return errors + + +def check_offline(text: str, root: Path, repository: str) -> tuple[ParsedMap, list[str]]: + parsed = parse_map(text) + errors = list(parsed.errors) + errors += check_rows(parsed) + errors += check_inventory(parsed.file_rows, root, FILE_MANIFEST, "file") + errors += check_inventory(parsed.case_rows, root, CASE_MANIFEST, "case") + errors += check_triage_index(parsed) + errors += check_counts(parsed) + errors += check_links(parsed, repository) + errors += check_references(text, root) + return parsed, errors + + +def owner_links(parsed: ParsedMap) -> dict[int, list[int]]: + """Issue number to the lines that name it as the owner of uncovered behaviour.""" + row_lines = {row.line for row in [*parsed.file_rows, *parsed.case_rows, *parsed.triage_rows]} + owners: dict[int, list[int]] = {} + for link in parsed.links: + if link.arrow or link.line in row_lines: + lines = owners.setdefault(link.number, []) + if link.line not in lines: + lines.append(link.line) + return owners + + +class StateUnknown(Exception): + """GitHub did not say whether the issue is open; carries the reason.""" + + +def fetch_issue_state(api_url: str, repository: str, number: int, token: str | None, deadline: float) -> str: + request = urllib.request.Request( + f"{api_url}/repos/{repository}/issues/{number}", + headers={ + "Accept": "application/vnd.github+json", + "X-GitHub-Api-Version": "2022-11-28", + "User-Agent": PREFIX, + }, + ) + if token: + request.add_header("Authorization", f"Bearer {token}") + reason = "no attempt was made" + for attempt in range(ATTEMPTS): + remaining = deadline - time.monotonic() + if remaining <= 0: + raise StateUnknown(f"ran out of time ({reason})") + try: + with urllib.request.urlopen(request, timeout=min(REQUEST_TIMEOUT_SECONDS, remaining)) as response: + payload = json.loads(response.read().decode("utf-8")) + except urllib.error.HTTPError as error: + reason = f"HTTP {error.code}" + if error.code == 404: + raise StateUnknown("HTTP 404: no such issue, or not visible to this token") from None + if error.code < 500: + raise StateUnknown(f"{reason} (rate limit or permissions; set GITHUB_TOKEN)") from None + except (OSError, http.client.HTTPException) as error: + # URLError and TimeoutError are OSErrors; a torn reply is an HTTPException. + if isinstance(getattr(error, "reason", None), ssl.SSLCertVerificationError): + raise StateUnknown( + f"TLS certificate verification failed ({error.reason.verify_message}); " + "this Python has no usable CA bundle, set SSL_CERT_FILE" + ) from None + reason = f"request failed: {error!r}" + except (UnicodeDecodeError, json.JSONDecodeError): + raise StateUnknown("reply was not JSON") from None + else: + state = payload.get("state") if isinstance(payload, dict) else None + if state not in ("open", "closed"): + raise StateUnknown(f"reply carried state {state!r}, expected 'open' or 'closed'") + return state + if attempt + 1 < ATTEMPTS: + time.sleep(max(0.0, min(0.5 * (attempt + 1), deadline - time.monotonic()))) + raise StateUnknown(reason) + + +def check_issues( + parsed: ParsedMap, api_url: str, repository: str, token: str | None, timeout_seconds: float +) -> tuple[list[str], list[str]]: + closed: list[str] = [] + unknown: list[str] = [] + deadline = time.monotonic() + timeout_seconds + for number, lines in sorted(owner_links(parsed).items()): + where = ", ".join(str(line) for line in lines) + try: + state = fetch_issue_state(api_url, repository, number, token, deadline) + except StateUnknown as error: + unknown.append(f"#{number} (map lines {where}): state unknown: {error}") + continue + if state == "closed": + closed.append( + f"#{number} is closed but the map names it as the owner of uncovered behaviour on " + f"{MAP.as_posix()} lines {where}; name the tests that landed, retire the remainder with a " + "reason, or point it at an open issue" + ) + return closed, unknown + + +def has_uncovered_remainder(row: Row) -> bool: + """Find coverage markers regardless of the punctuation preceding them.""" + for marker in re.finditer(r"\bnot covered(?:, no open issue)?:", row.text, re.IGNORECASE): + remainder = row.text[marker.end():].split(";", 1)[0].strip() + if remainder != "none with a native subject": + return True + return False + + +def summary(parsed: ParsedMap) -> str: + counts = Counter(row.status for row in parsed.file_rows) + owners_by_line: dict[int, list[int]] = {} + for number, owner_lines in sorted(owner_links(parsed).items()): + for line in owner_lines: + owners_by_line.setdefault(line, []).append(number) + lines = [ + f"{len(parsed.file_rows)} rows: {counts[FULL]} full, {counts[PARTIAL]} partial, {counts[RETIRED]} retired, " + f"{counts[OPEN_GAP]} open gap, {counts[NEEDS_TRIAGE]} needs triage.", + ] + for status in (OPEN_GAP, NEEDS_TRIAGE): + rows = [row for row in [*parsed.file_rows, *parsed.case_rows] if row.status == status] + if rows: + lines += ["", f"{status.capitalize()} ({len(rows)}):"] + for row in rows: + linked = ", ".join(f"#{number}" for number in owners_by_line.get(row.line, [])) + lines.append(f"- `{row.name}`" + (f" (links {linked})" if linked else "")) + owned = [row for row in [*parsed.file_rows, *parsed.case_rows] if row.status not in (OPEN_GAP, NEEDS_TRIAGE) and row.line in owners_by_line] + if owned: + lines += ["", f"Rows with a remainder owned by an open issue ({len(owned)}):"] + for row in owned: + owners = ", ".join(f"#{number}" for number in owners_by_line[row.line]) + lines.append(f"- `{row.name}` ({row.status}, owner {owners})") + unowned = [ + row for row in [*parsed.file_rows, *parsed.case_rows] + if row.status == PARTIAL and has_uncovered_remainder(row) and row.line not in owners_by_line + ] + if unowned: + lines += ["", f"Partial rows with an uncovered remainder and no owner issue ({len(unowned)}):"] + for row in unowned: + lines.append(f"- `{row.name}` (partial, no owner issue)") + return "\n".join(lines) + + +def positive_finite(value: str) -> float: + try: + number = float(value) + except ValueError: + raise argparse.ArgumentTypeError(f"{value!r} is not a number") from None + if not math.isfinite(number) or number <= 0: + raise argparse.ArgumentTypeError(f"{value!r} must be a finite number of seconds above zero") + return number + + +def http_url(value: str) -> str: + parts = urllib.parse.urlsplit(value) + if parts.scheme not in ("http", "https") or not parts.netloc: + raise argparse.ArgumentTypeError(f"{value!r} must be an http(s) URL") + return value.rstrip("/") + + +def repository_name(value: str) -> str: + if not re.fullmatch(r"[A-Za-z0-9_.-]+/[A-Za-z0-9_.-]+", value): + raise argparse.ArgumentTypeError(f"{value!r} must be <owner>/<name>") + return value + + +def main(argv: list[str] | None = None) -> int: + parser = argparse.ArgumentParser(description=__doc__.splitlines()[0]) + parser.add_argument("--root", type=Path, default=ROOT, help="repository root (default: this checkout)") + parser.add_argument("--check-issues", action="store_true", help="ask GitHub whether each owner issue is still open") + parser.add_argument("--summary", action="store_true", help="print the row counts and the rows that are still open") + parser.add_argument("--repository", type=repository_name, default=REPOSITORY, help=f"default: {REPOSITORY}") + parser.add_argument( + "--api-url", + type=http_url, + default=http_url(os.environ.get("GITHUB_API_URL") or DEFAULT_API_URL), + help=f"GitHub API base (default: $GITHUB_API_URL or {DEFAULT_API_URL})", + ) + parser.add_argument( + "--timeout-seconds", type=positive_finite, default=60.0, help="total time allowed for --check-issues (default: 60)" + ) + args = parser.parse_args(argv) + + path = args.root / MAP + try: + text = path.read_text(encoding="utf-8") + except (OSError, UnicodeDecodeError) as error: + print(f"{PREFIX}: cannot read {path}: {error}", file=sys.stderr) + return 1 + + parsed, errors = check_offline(text, args.root, args.repository) + unknown: list[str] = [] + if args.check_issues: + token = os.environ.get("GITHUB_TOKEN") or os.environ.get("GH_TOKEN") or None + closed, unknown = check_issues(parsed, args.api_url, args.repository, token, args.timeout_seconds) + errors += closed + + for error in errors: + print(f"{PREFIX}: {error}", file=sys.stderr) + for entry in unknown: + print(f"{PREFIX}: {entry}", file=sys.stderr) + if errors: + print(f"{PREFIX}: {len(errors)} problem(s) in {MAP.as_posix()}", file=sys.stderr) + return 1 + if unknown: + print(f"{PREFIX}: could not determine the state of {len(unknown)} issue(s); nothing is assumed open", file=sys.stderr) + return 2 + + if args.summary: + print(summary(parsed)) + else: + checked = f", {len(owner_links(parsed))} owner issue(s) open" if args.check_issues else "" + print(f"{PREFIX}: ok ({len(parsed.file_rows)} file rows, {len(parsed.case_rows)} case rows{checked})") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/scripts/check_e2e_scenarios.py b/scripts/check_e2e_scenarios.py new file mode 100644 index 000000000..5b474fbb9 --- /dev/null +++ b/scripts/check_e2e_scenarios.py @@ -0,0 +1,459 @@ +#!/usr/bin/env python3 +"""Hold the end-to-end and load scenario manifest to what the lanes run (#543). + +`test/e2e-scenarios.toml` lists every end-to-end and load scenario (#487's +catalogue, #521's scenarios, #474's), each with its lanes, its owner issue, +its pass criteria and whether it runs today. A scenario that runs cites its +evidence, and the evidence decides its lanes: + +- `tests`: `<package>::<binary>::<test path>` ids of gated tests. An id in + `test/prism-gated-tests.txt` runs in `pr` (the required + `prism-native-postgres` shards prove it executed); one in + `test/prism-nightly-gated-tests.txt` runs in `nightly`, and one in + `test/prism-weekly-gated-tests.txt` in `weekly` (the live-weekly job, #487 + L4). +- `presets`: load-harness presets in `crates/qbit-prism-load/presets`. A + `nightly` preset runs in `nightly`, a `weekly` one (the long soak, #575) + in `weekly`, a `manual` one in `dispatch`, and a `smoke` preset in `pr` + through the gated `load_smoke` or `faults` test (`SMOKE_BINARIES`). +- `unit_tests`: `<file>::<fn>` for an ungated `#[test]` or `#[tokio::test]` + that is not `#[ignore]`d; `cargo test --workspace` runs it in `pr`. +- `lane_checks`: names from `CHECKS` in `scripts/prism_shipped_image_lane.py`, + the checks lane L6 (#544) holds the shipped images to; they run in `L6`. +- `suites`: preset suites in `crates/qbit-prism-load/presets/suites.toml` + (#550); a suite runs in the lane it names (`L3` for both today). A + scenario that cites suites and presets cites only presets those suites + run, so a preset's own schedule and its suite membership are both claimed. + +This check fails when: + +- the manifest is malformed: an unknown key, a missing or empty field, a + duplicate id, an owner that is not `#<issue>`, or an undefined lane; +- a running scenario cites evidence no lane runs (an id in no gated list, an + unknown preset, a missing or ignored test, an L6 check the lane does not + define), or declares lanes other than + the ones its evidence runs in; +- an unexercised scenario cites evidence, or lacks a reason (#487's + definition of done: nothing unexercised without a linked issue and a + reason; the owner is the linked issue); +- a lane runs a scenario the manifest does not name: a gated test in one of + the end-to-end binaries (`SCENARIO_BINARIES`), an opt-in nightly or weekly + test, a preset, a suite, or an L6 check that no running scenario cites; +- a lane the manifest marks as running no longer has the workflow wiring + that runs it. + +CI runs it through tests/test_check_e2e_scenarios.py in the required +python-tests shards. + +Usage: python3 scripts/check_e2e_scenarios.py [--manifest test/e2e-scenarios.toml] +""" + +from __future__ import annotations + +import argparse +import ast +from collections import Counter +from pathlib import Path +import re +import sys +import tomllib + +sys.path.insert(0, str(Path(__file__).resolve().parent)) + +from check_gate_manifest import ManifestError, read_expected # noqa: E402 +from prism_load_matrix import SelectionError, load as load_presets, load_suites # noqa: E402 + + +ROOT = Path(__file__).resolve().parents[1] +PREFIX = "check-e2e-scenarios" +DEFAULT_MANIFEST = ROOT / "test" / "e2e-scenarios.toml" +SCHEMA = "qbit.e2e-scenarios.v1" +PR_LIST = Path("test/prism-gated-tests.txt") +NIGHTLY_LIST = Path("test/prism-nightly-gated-tests.txt") +WEEKLY_LIST = Path("test/prism-weekly-gated-tests.txt") +PRESETS = Path("crates/qbit-prism-load/presets") +L6_DRIVER = Path("scripts/prism_shipped_image_lane.py") +# Gated test binaries whose every test is an end-to-end scenario and so must +# be named in the manifest. The rest of the gated list is component tests. +SCENARIO_BINARIES = ( + "qbit-prism-load::faults::", + "qbit-prism-server::live_regtest::", + "qbit-prism-load::load_smoke::", +) +PRESET_LANES = {"nightly": "nightly", "weekly": "weekly", "manual": "dispatch", "smoke": "pr"} +# The gated test binaries that run the `smoke` presets in the PR shards: +# `load_smoke` runs `pr-smoke`, and `faults` runs `faults-pr-smoke` (#554). +SMOKE_BINARIES = ("qbit-prism-load::load_smoke::", "qbit-prism-load::faults::") +# The lanes this check can enumerate, and the workflow text that has to be +# present for each to run at all. A lane marked running that is not here needs +# this check taught how to read it. +LANE_WIRING = { + "pr": ( + ".github/workflows/ci.yml", + ("--expected test/prism-gated-tests.txt", "scripts/run_rust_test_shard.py"), + ), + "nightly": ( + ".github/workflows/prism-load-nightly.yml", + ( + "cron:", + "--expected test/prism-nightly-gated-tests.txt", + "scripts/prism_load_matrix.py", + ), + ), + "dispatch": ( + ".github/workflows/prism-load-nightly.yml", + ("workflow_dispatch:", "scripts/prism_load_matrix.py"), + ), + # The long soak's schedule (#575), whose plan selects the `weekly` presets, + # and the live-weekly job's, which runs the weekly gated list (#487 L4). + "weekly": ( + ".github/workflows/prism-load-nightly.yml", + ( + 'cron: "41 5 * * 6"', + "github.event.schedule == '41 5 * * 6' && 'weekly'", + 'cron: "43 3 * * 0"', + "--expected test/prism-weekly-gated-tests.txt", + ), + ), + "L6": ( + ".github/workflows/prism-load-nightly.yml", + ('cron: "43 3 * * 0"', "pull_request:", f"{L6_DRIVER} run"), + ), + # #550: the suites, planned by name, on the version bump, the + # release-candidate label, a v* tag, dispatch and the weekly schedule. + "L3": ( + ".github/workflows/prism-load-l3.yml", + ( + 'scripts/prism_load_matrix.py "suite:${SUITE}"', + "grep -qx VERSION", + "release-candidate", + 'tags: ["v*"]', + "cron:", + "suite=l3-reduced", + "scripts/prism_l3_promote.py", + ), + ), +} +LANE_KEYS = {"title", "runs", "owner", "workflow"} +SCENARIO_KEYS = { + "id", "title", "owner", "lanes", "criteria", "runs", "reason", + "tests", "presets", "unit_tests", "lane_checks", "suites", "notes", +} +EVIDENCE_KEYS = ("tests", "presets", "unit_tests", "lane_checks", "suites") +ID = re.compile(r"^[a-z0-9][a-z0-9-]*$") +ISSUE = re.compile(r"^#[1-9][0-9]*$") +UNIT_TEST = re.compile(r"^(crates/[A-Za-z0-9_./-]+\.rs)::([A-Za-z_][A-Za-z0-9_]*)$") +TEST_ATTRIBUTE = re.compile(r"^#\[(tokio::)?test\b") + + +class Lanes: + """What each running lane executes, read from the files that drive it.""" + + def __init__(self, root: Path) -> None: + self.root = root + self.pr = read_expected(root / PR_LIST) + self.nightly = read_expected(root / NIGHTLY_LIST) + self.weekly = read_expected(root / WEEKLY_LIST) + self.presets = load_presets(root / PRESETS) + self.l6_checks = read_lane_checks(root / L6_DRIVER) + self.suites = load_suites(root / PRESETS, self.presets) + + def test_lanes(self, test_id: str) -> set[str]: + lanes = set() + if test_id in self.pr: + lanes.add("pr") + if test_id in self.nightly: + lanes.add("nightly") + if test_id in self.weekly: + lanes.add("weekly") + return lanes + + def preset_lanes(self, name: str) -> set[str]: + preset = self.presets.get(name) + if preset is None: + return set() + lane = PRESET_LANES.get(preset.get("schedule")) + if lane == "pr" and not any(t.startswith(SMOKE_BINARIES) for t in self.pr): + return set() + return {lane} if lane else set() + + def suite_lanes(self, name: str) -> set[str]: + suite = self.suites.get(name) + return {suite["lane"]} if suite else set() + + def unit_test_problem(self, reference: str) -> str | None: + """Why `reference` is not a test `cargo test` runs, or None.""" + match = UNIT_TEST.match(reference) + if not match: + return f"{reference!r} is not <crates/...rs>::<fn>" + path = self.root / match.group(1) + if not path.is_file(): + return f"{match.group(1)} does not exist" + lines = path.read_text(encoding="utf-8").splitlines() + definition = re.compile( + rf"^\s*(pub(\([a-z]+\))?\s+)?(async\s+)?fn\s+{re.escape(match.group(2))}\s*[(<]" + ) + found = [number for number, line in enumerate(lines) if definition.match(line)] + if not found: + return f"{match.group(1)} defines no fn {match.group(2)}" + for number in found: + attributes = [] + above = number - 1 + while above >= 0 and lines[above].strip().startswith(("#[", "//")): + attributes.append(lines[above].strip()) + above -= 1 + if any(a.startswith("#[ignore") for a in attributes): + return f"{reference} is #[ignore]d, so no lane runs it" + if any(TEST_ATTRIBUTE.match(a) for a in attributes): + return None + return f"{reference} is not a #[test] or #[tokio::test]" + + +def read_lane_checks(path: Path) -> tuple[str, ...]: + """The driver's `CHECKS` tuple, read without importing the driver.""" + if not path.is_file(): + return () + for node in ast.parse(path.read_text(encoding="utf-8")).body: + if isinstance(node, ast.Assign) and any( + isinstance(target, ast.Name) and target.id == "CHECKS" for target in node.targets + ): + value = ast.literal_eval(node.value) + if not (isinstance(value, tuple) and all(text(item) for item in value)): + raise ValueError(f"{path}: CHECKS must be a tuple of names") + return value + raise ValueError(f"{path} defines no CHECKS") + + +def text(value: object) -> bool: + return isinstance(value, str) and bool(value.strip()) + + +def string_list(value: object) -> bool: + return isinstance(value, list) and all(text(item) for item in value) + + +def check_lanes(manifest: dict, root: Path) -> tuple[dict, list[str]]: + problems = [] + lanes = manifest.get("lanes") + if not isinstance(lanes, dict) or not lanes: + return {}, ["[lanes] is missing or empty"] + for name, lane in lanes.items(): + where = f"lane {name}" + if not isinstance(lane, dict): + problems.append(f"{where} is not a table") + continue + for key in sorted(set(lane) - LANE_KEYS): + problems.append(f"{where}: unknown key {key!r}") + if not text(lane.get("title")): + problems.append(f"{where}: title is missing or empty") + if not isinstance(lane.get("runs"), bool): + problems.append(f"{where}: runs must be true or false") + continue + if "owner" in lane and not (text(lane["owner"]) and ISSUE.match(lane["owner"])): + problems.append(f"{where}: owner {lane['owner']!r} is not #<issue>") + if lane["runs"]: + wiring = LANE_WIRING.get(name) + if wiring is None: + problems.append( + f"{where} is marked running, but this check cannot enumerate it; " + "teach scripts/check_e2e_scenarios.py what it runs" + ) + continue + workflow, needles = wiring + if lane.get("workflow") != workflow: + problems.append(f"{where}: workflow must be {workflow!r}") + path = root / workflow + body = path.read_text(encoding="utf-8") if path.is_file() else "" + for needle in needles: + if needle not in body: + problems.append(f"{where}: {workflow} no longer contains {needle!r}") + elif "owner" not in lane: + problems.append(f"{where} does not run yet and names no owner issue") + return lanes, problems + + +def check(manifest: dict, lanes_run: Lanes, root: Path) -> list[str]: + """Every reason the manifest and the lanes disagree; empty when they agree.""" + problems = [] + if manifest.get("schema") != SCHEMA: + problems.append(f"schema is {manifest.get('schema')!r}, not {SCHEMA!r}") + for key in sorted(set(manifest) - {"schema", "lanes", "scenario"}): + problems.append(f"unknown top-level key {key!r}") + lanes, lane_problems = check_lanes(manifest, root) + problems += lane_problems + scenarios = manifest.get("scenario") + if not isinstance(scenarios, list) or not scenarios: + return problems + ["no [[scenario]] entries"] + + ids = Counter(s.get("id") for s in scenarios if isinstance(s, dict)) + for name, count in sorted(ids.items(), key=lambda item: str(item[0])): + if count > 1: + problems.append(f"scenario id {name!r} appears {count} times") + cited: dict[str, set[str]] = {key: set() for key in EVIDENCE_KEYS} + + for index, scenario in enumerate(scenarios, start=1): + if not isinstance(scenario, dict): + problems.append(f"scenario {index} is not a table") + continue + name = scenario.get("id") + where = f"scenario {name}" if text(name) else f"scenario {index}" + if not (text(name) and ID.match(name)): + problems.append(f"{where}: id {name!r} is not lower-case kebab") + for key in sorted(set(scenario) - SCENARIO_KEYS): + problems.append(f"{where}: unknown key {key!r}") + for key in ("title", "criteria"): + if not text(scenario.get(key)): + problems.append(f"{where}: {key} is missing or empty") + owner = scenario.get("owner") + if not (text(owner) and ISSUE.match(owner)): + problems.append(f"{where}: owner {owner!r} is not #<issue>") + declared = scenario.get("lanes") + if not (string_list(declared) and declared): + problems.append(f"{where}: lanes must be a non-empty list of lane names") + declared = [] + for lane in declared: + if lane not in lanes: + problems.append(f"{where}: lane {lane!r} is not defined under [lanes]") + if len(set(declared)) != len(declared): + problems.append(f"{where}: lanes repeat") + evidence = {} + for key in EVIDENCE_KEYS: + value = scenario.get(key, []) + if not string_list(value): + problems.append(f"{where}: {key} must be a list of non-empty strings") + value = [] + if len(set(value)) != len(value): + problems.append(f"{where}: {key} repeats an entry") + evidence[key] = value + runs = scenario.get("runs") + if not isinstance(runs, bool): + problems.append(f"{where}: runs must be true or false") + continue + + if not runs: + if not text(scenario.get("reason")): + problems.append(f"{where} is unexercised and gives no reason") + if any(evidence.values()): + problems.append( + f"{where} is unexercised but cites evidence; set runs = true or drop it" + ) + continue + + if "reason" in scenario: + problems.append(f"{where} runs; reason is only for an unexercised scenario") + if not any(evidence.values()): + problems.append(f"{where} runs but cites no tests, presets, unit_tests, lane_checks or suites") + continue + derived: set[str] = set() + for test_id in evidence["tests"]: + found = lanes_run.test_lanes(test_id) + if not found: + problems.append( + f"{where}: {test_id} is in none of {PR_LIST}, {NIGHTLY_LIST} " + f"or {WEEKLY_LIST}, so no lane runs it" + ) + derived |= found + for preset in evidence["presets"]: + found = lanes_run.preset_lanes(preset) + if not found: + problems.append(f"{where}: preset {preset!r} is unknown or runs in no lane") + derived |= found + for reference in evidence["unit_tests"]: + problem = lanes_run.unit_test_problem(reference) + if problem: + problems.append(f"{where}: {problem}") + else: + derived.add("pr") + for name in evidence["lane_checks"]: + if name in lanes_run.l6_checks: + derived.add("L6") + else: + problems.append(f"{where}: {L6_DRIVER} defines no check {name!r}, so L6 does not run it") + suite_members: set[str] = set() + for suite in evidence["suites"]: + found = lanes_run.suite_lanes(suite) + if not found: + problems.append(f"{where}: suite {suite!r} is not in {PRESETS}/suites.toml") + else: + suite_members |= set(lanes_run.suites[suite]["presets"]) + derived |= found + if evidence["suites"]: + for preset in evidence["presets"]: + if preset not in suite_members: + problems.append( + f"{where}: preset {preset!r} is in none of its suites " + f"({', '.join(evidence['suites'])})" + ) + for key in EVIDENCE_KEYS: + cited[key] |= set(evidence[key]) + for lane in sorted(set(declared) - derived): + problems.append(f"{where} claims lane {lane!r}, but none of its evidence runs there") + for lane in sorted(derived - set(declared)): + problems.append(f"{where}: its evidence runs in {lane!r}; add it to lanes") + for lane in declared: + if lane in lanes and isinstance(lanes[lane], dict) and lanes[lane].get("runs") is False: + problems.append(f"{where} runs, but lane {lane!r} is marked as not running") + + for test_id in lanes_run.pr: + if test_id.startswith(SCENARIO_BINARIES) and test_id not in cited["tests"]: + problems.append(f"pr runs {test_id}, which no running scenario names") + for test_id in lanes_run.nightly: + if test_id not in cited["tests"]: + problems.append(f"nightly runs {test_id}, which no running scenario names") + for test_id in lanes_run.weekly: + if test_id not in cited["tests"]: + problems.append(f"weekly runs {test_id}, which no running scenario names") + for name in lanes_run.l6_checks: + if name not in cited["lane_checks"]: + problems.append(f"L6 runs check {name}, which no running scenario names") + for name, preset in sorted(lanes_run.presets.items()): + lane = PRESET_LANES.get(preset.get("schedule")) + if lane and name not in cited["presets"]: + problems.append(f"{lane} runs preset {name}, which no running scenario names") + for name, suite in sorted(lanes_run.suites.items()): + if name not in cited["suites"]: + problems.append(f"{suite['lane']} runs suite {name}, which no running scenario names") + return problems + + +def summary(manifest: dict) -> str: + scenarios = [s for s in manifest.get("scenario", []) if isinstance(s, dict)] + running = [s for s in scenarios if s.get("runs") is True] + lines = [f"{len(scenarios)} scenarios: {len(running)} run, {len(scenarios) - len(running)} unexercised"] + for lane in manifest.get("lanes", {}): + here = [s for s in scenarios if lane in s.get("lanes", [])] + runs = sum(1 for s in here if s.get("runs") is True) + lines.append(f" {lane}: {runs} run, {len(here) - runs} planned") + return "\n".join(lines) + + +def main(argv: list[str] | None = None) -> int: + parser = argparse.ArgumentParser( + description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter + ) + parser.add_argument("--manifest", type=Path, default=DEFAULT_MANIFEST) + parser.add_argument("--root", type=Path, default=ROOT, help=argparse.SUPPRESS) + args = parser.parse_args(argv) + try: + with args.manifest.open("rb") as handle: + manifest = tomllib.load(handle) + lanes_run = Lanes(args.root) + except FileNotFoundError as error: + print(f"{PREFIX}: {error.filename} is missing", file=sys.stderr) + return 1 + except (tomllib.TOMLDecodeError, ManifestError, SelectionError, OSError, ValueError) as error: + print(f"{PREFIX}: {error}", file=sys.stderr) + return 1 + problems = check(manifest, lanes_run, args.root) + for problem in problems: + print(f"{PREFIX}: {problem}", file=sys.stderr) + if problems: + print(f"{PREFIX}: {len(problems)} problem(s) in {args.manifest}", file=sys.stderr) + return 1 + print(summary(manifest)) + print(f"{PREFIX}: the manifest names every scenario the lanes run, and each claim holds") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/check_gate_env_reads.py b/scripts/check_gate_env_reads.py new file mode 100644 index 000000000..dee121338 --- /dev/null +++ b/scripts/check_gate_env_reads.py @@ -0,0 +1,108 @@ +#!/usr/bin/env python3 +"""Fail when a Rust file reads an integration-gate variable directly. + +Every test whose execution depends on `PRISM_TEST_DATABASE_URL`, +`PRISM_TEST_PG_BIN_DIR` or `QBITD_BIN` must take the value through the shared +gate crate (`crates/qbit-prism-test-gate`), which applies one decision table, +records the execution manifest and prints the skip line. The same goes for the +gate's own control variables, `PRISM_TEST_REQUIRE_INTEGRATION`, +`PRISM_TEST_GATE_MANIFEST` and `GITHUB_JOB`. A direct `std::env::var(...)` of +any of them elsewhere would let a new test pass vacuously again, unrecorded. + +The rule is syntactic and deliberately broad: outside the gate crate, no Rust +source line may contain a string literal that is exactly one of those names, +whether it feeds `env::var`, `env::var_os`, `option_env!`, a `const`, or +anything else. Doc comments and longer literals such as +`"set PRISM_TEST_DATABASE_URL for ..."` are unaffected. + +Usage: python3 scripts/check_gate_env_reads.py [--root DIR] + +Exits 0 when no such line exists and 1 otherwise, naming every offending +file and line. +""" + +from __future__ import annotations + +import argparse +from pathlib import Path +import re +import sys + + +ROOT = Path(__file__).resolve().parents[1] +PREFIX = "check-gate-env-reads" +GATE_CRATE = Path("crates") / "qbit-prism-test-gate" +GATE_VARIABLES = ( + "PRISM_TEST_DATABASE_URL", + "PRISM_TEST_PG_BIN_DIR", + "QBITD_BIN", + "PRISM_TEST_REQUIRE_INTEGRATION", + "PRISM_TEST_GATE_MANIFEST", + "GITHUB_JOB", +) +LITERAL = re.compile(r'"(' + "|".join(map(re.escape, GATE_VARIABLES)) + r')"') + + +def rust_sources(root: Path) -> list[Path]: + crates = root / "crates" + return sorted( + path + for path in crates.rglob("*.rs") + if "target" not in path.relative_to(root).parts + ) + + +def direct_reads(root: Path) -> list[tuple[Path, int, str]]: + """Every (file, line number, variable) outside the gate crate.""" + found = [] + for path in rust_sources(root): + relative = path.relative_to(root) + if relative.parts[: len(GATE_CRATE.parts)] == GATE_CRATE.parts: + continue + for number, line in enumerate( + path.read_text(encoding="utf-8").splitlines(), start=1 + ): + for match in LITERAL.finditer(line): + found.append((relative, number, match.group(1))) + return found + + +def main(argv: list[str] | None = None) -> int: + parser = argparse.ArgumentParser( + description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter + ) + parser.add_argument( + "--root", + type=Path, + default=ROOT, + help="checkout to scan (default: this checkout)", + ) + args = parser.parse_args(argv) + root = args.root.resolve() + if not (root / "crates").is_dir(): + print(f"{PREFIX}: {root / 'crates'} is not a directory", file=sys.stderr) + return 1 + found = direct_reads(root) + for relative, number, variable in found: + print( + f"{PREFIX}: {relative.as_posix()}:{number}: {variable} is read directly; " + f"take it through qbit_prism_test_gate instead", + file=sys.stderr, + ) + if found: + print( + f"{PREFIX}: {len(found)} direct read(s) of gate variables outside " + f"{GATE_CRATE.as_posix()}; see docs/prism-integration-test-gate.md", + file=sys.stderr, + ) + return 1 + scanned = len(rust_sources(root)) + print( + f"{PREFIX}: no direct reads of {', '.join(GATE_VARIABLES)} outside " + f"{GATE_CRATE.as_posix()} in {scanned} Rust files" + ) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/check_gate_manifest.py b/scripts/check_gate_manifest.py new file mode 100644 index 000000000..c321c0bcc --- /dev/null +++ b/scripts/check_gate_manifest.py @@ -0,0 +1,223 @@ +#!/usr/bin/env python3 +"""Prove that every gated test executed, from the gate's execution manifest. + +When `PRISM_TEST_GATE_MANIFEST` names a file, the shared integration gate +(`crates/qbit-prism-test-gate`) appends one line per decision: + + executed <package>::<binary>::<test path> + skipped <package>::<binary>::<test path> + failed <package>::<binary>::<test path> + +`test/prism-gated-tests.txt` is the checked-in list of every gated test the +`prism-native-postgres` job must execute; its length is the minimum count. +This check fails when: + +- any expected test has no `executed` line, +- any `skipped` or `failed` line appears, +- fewer distinct tests executed than the list holds, +- a test executed that is not in the list (adding a gated test means adding + it to the list, so the list stays the source of truth), +- the manifest holds a line the gate would not write, +- or, with `--log`, the test log contains the gate's skip prefix or the + wording of the skip messages the gate replaced. + +A test that executes more than once (a fixture opened in a loop, or one +binary run twice) counts once. The manifest is printed to standard output +and, when `--summary` names a file or `GITHUB_STEP_SUMMARY` is set, appended +to it as Markdown. + +Usage: python3 scripts/check_gate_manifest.py --manifest FILE + [--expected test/prism-gated-tests.txt] [--log FILE] [--summary FILE] +""" + +from __future__ import annotations + +import argparse +from collections import Counter +import os +from pathlib import Path +import re +import sys + + +ROOT = Path(__file__).resolve().parents[1] +PREFIX = "check-gate-manifest" +DEFAULT_EXPECTED = ROOT / "test" / "prism-gated-tests.txt" +KINDS = ("executed", "skipped", "failed") +# Keep in step with SKIP_PREFIX in crates/qbit-prism-test-gate/src/lib.rs. +SKIP_PREFIX = "[prism-test-gate] skipped" +# Wording of the per-file skip messages the shared gate replaced. A test that +# grows one of these back has bypassed the gate. +LEGACY_SKIP_PATTERNS = ( + re.compile(r"skipping [^\n]*(PRISM_TEST_DATABASE_URL|PRISM_TEST_PG_BIN_DIR|QBITD_BIN)"), + re.compile(r"PRISM_TEST_DATABASE_URL not set"), + re.compile(r"set PRISM_TEST_DATABASE_URL for "), + re.compile(r"SKIPPED: jsonb_ceiling_gate"), +) +TEST_ID = re.compile(r"^[A-Za-z0-9_-]+(::[A-Za-z0-9_]+)+$") + + +class ManifestError(Exception): + """A reason the proof fails, worded for whoever reads the CI log.""" + + +def read_expected(path: Path) -> list[str]: + try: + text = path.read_text(encoding="utf-8") + except FileNotFoundError: + raise ManifestError(f"expected list {path} is missing") from None + ids = [] + for number, raw in enumerate(text.splitlines(), start=1): + line = raw.split("#", 1)[0].strip() + if not line: + continue + if not TEST_ID.match(line): + raise ManifestError( + f"{path}:{number}: {line!r} is not a <package>::<binary>::<test> id" + ) + ids.append(line) + if not ids: + raise ManifestError(f"expected list {path} names no tests") + duplicates = sorted(name for name, count in Counter(ids).items() if count > 1) + if duplicates: + raise ManifestError( + f"expected list {path} repeats {', '.join(duplicates)}" + ) + if ids != sorted(ids): + raise ManifestError(f"expected list {path} must be sorted") + return ids + + +def read_manifest(path: Path) -> list[tuple[str, str]]: + try: + text = path.read_text(encoding="utf-8") + except FileNotFoundError: + raise ManifestError( + f"manifest {path} is missing: no gated test ran, or " + "PRISM_TEST_GATE_MANIFEST did not name it" + ) from None + lines = [] + for number, raw in enumerate(text.splitlines(), start=1): + parts = raw.split(" ") + if len(parts) != 2 or parts[0] not in KINDS or not TEST_ID.match(parts[1]): + raise ManifestError( + f"{path}:{number}: {raw!r} is not a line the gate writes " + f"(expected '<{'|'.join(KINDS)}> <package>::<binary>::<test>')" + ) + lines.append((parts[0], parts[1])) + if not lines: + raise ManifestError(f"manifest {path} is empty: no gated test ran") + return lines + + +def log_problems(text: str) -> list[str]: + """Every log line that shows a gated test skipping.""" + problems = [] + for line in text.splitlines(): + if SKIP_PREFIX in line or any(p.search(line) for p in LEGACY_SKIP_PATTERNS): + problems.append(line.rstrip()) + return problems + + +def check( + manifest: list[tuple[str, str]], expected: list[str] +) -> tuple[list[str], list[str]]: + """Returns (problems, notes). Empty problems means the proof holds.""" + problems = [] + notes = [] + executed = Counter(test for kind, test in manifest if kind == "executed") + for kind in ("skipped", "failed"): + for test in sorted({t for k, t in manifest if k == kind}): + problems.append(f"{test} was {kind}") + for test in expected: + if test not in executed: + problems.append(f"{test} did not execute") + unexpected = sorted(set(executed) - set(expected)) + for test in unexpected: + problems.append( + f"{test} executed but is not in the expected list; add it to " + f"{DEFAULT_EXPECTED.relative_to(ROOT).as_posix()}" + ) + if len(executed) < len(expected): + problems.append( + f"{len(executed)} distinct gated tests executed, below the minimum " + f"of {len(expected)}" + ) + repeated = sorted(test for test, count in executed.items() if count > 1) + for test in repeated: + notes.append(f"{test} executed {executed[test]} times; counted once") + return problems, notes + + +def render(manifest: list[tuple[str, str]], expected: list[str], problems: list[str], notes: list[str]) -> str: + executed = sorted({t for k, t in manifest if k == "executed"}) + lines = [ + "## Gated test execution manifest", + "", + f"{len(executed)} distinct gated tests executed; the expected list holds " + f"{len(expected)}; {len(manifest)} manifest lines.", + "", + ] + if problems: + lines += ["**FAILED**", ""] + [f"- {p}" for p in problems] + [""] + else: + lines += ["Every expected gated test executed.", ""] + if notes: + lines += [f"- {n}" for n in notes] + [""] + lines += ["```"] + [f"{kind} {test}" for kind, test in sorted(manifest)] + ["```", ""] + return "\n".join(lines) + + +def main(argv: list[str] | None = None) -> int: + parser = argparse.ArgumentParser( + description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter + ) + parser.add_argument("--manifest", type=Path, required=True, help="the gate's manifest") + parser.add_argument( + "--expected", + type=Path, + default=DEFAULT_EXPECTED, + help="checked-in list of gated tests (default: test/prism-gated-tests.txt)", + ) + parser.add_argument("--log", type=Path, help="test log to scan for skip lines") + parser.add_argument( + "--summary", + type=Path, + help="Markdown file to append the manifest to (default: $GITHUB_STEP_SUMMARY)", + ) + args = parser.parse_args(argv) + + try: + expected = read_expected(args.expected) + manifest = read_manifest(args.manifest) + log_text = args.log.read_text(encoding="utf-8", errors="replace") if args.log else "" + except (ManifestError, OSError) as error: + print(f"{PREFIX}: {error}", file=sys.stderr) + return 1 + + problems, notes = check(manifest, expected) + for line in log_problems(log_text): + problems.append(f"log shows a skipped gated test: {line}") + + report = render(manifest, expected, problems, notes) + print(report) + summary = args.summary or ( + Path(os.environ["GITHUB_STEP_SUMMARY"]) if os.environ.get("GITHUB_STEP_SUMMARY") else None + ) + if summary is not None: + with summary.open("a", encoding="utf-8") as handle: + handle.write(report) + for problem in problems: + print(f"{PREFIX}: {problem}", file=sys.stderr) + if problems: + print(f"{PREFIX}: {len(problems)} problem(s); the gated suite did not run in full", file=sys.stderr) + return 1 + print( + f"{PREFIX}: all {len(expected)} expected gated tests executed " + f"({len(manifest)} manifest lines, {len(notes)} repeated)" + ) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/check_prism_pool_acquires.py b/scripts/check_prism_pool_acquires.py new file mode 100644 index 000000000..e27030e21 --- /dev/null +++ b/scripts/check_prism_pool_acquires.py @@ -0,0 +1,329 @@ +#!/usr/bin/env python3 +"""Review gate for PRISM checkout sites, not a Rust type checker. + +Walk the production module graph, tokenize Rust (comments and SQL strings are +opaque), and inventory SQLx-shaped calls regardless of receiver names. Mutable +executor borrows are SQL on an existing connection; their nested acquire calls +are still scanned. Everything else needs an exact, counted classification. +Pool/Executor/Acquire/AuditReader helper signatures also enroll their callers, +so passing a pool to an existing generic helper cannot bypass the gate. +""" + +from __future__ import annotations + +import argparse +from collections import Counter +from dataclasses import dataclass +import json +from pathlib import Path +import re + +ROOT = Path(__file__).resolve().parents[1] +SOURCE = Path("crates/qbit-prism-server/src") +INVENTORY = Path("scripts/prism_pool_acquires.json") +OPERATIONS = { + "acquire", "try_acquire", "begin", "try_begin", "begin_with", "try_begin_with", "execute", "execute_many", + "fetch", "fetch_all", "fetch_one", "fetch_optional", "fetch_many", + "prepare", "prepare_with", "describe", +} +EXECUTORS = OPERATIONS - {"acquire", "try_acquire", "begin", "try_begin", "begin_with", "try_begin_with"} +CLASSES = {"timed", "delegated", "borrowed", "startup", "operator", "public", "compatibility", "non-database"} +TOKEN = re.compile( + r'\s+|//[^\n]*|/\*|(?:br|r)(?P<hash>\#*)"|(?:b|c)?"(?:\\.|[^"\\])*"' + r"|b?'(?:\\.|[^'\\\n])'|[A-Za-z_][A-Za-z_0-9]*|::|->|=>|[^\s]", + re.DOTALL, +) + + +@dataclass(frozen=True) +class Token: + value: str + line: int + + +def tokenize(source: str) -> list[Token]: + result = [] + pos = 0 + line = 1 + while pos < len(source): + match = TOKEN.match(source, pos) + if match is None: + raise ValueError(f"cannot tokenize line {line}") + value = match.group() + end = match.end() + if value == "/*": + depth = 1 + while depth: + comment = re.search(r"/\*|\*/", source[end:]) + if comment is None: + raise ValueError("unterminated block comment") + depth += 1 if comment.group() == "/*" else -1 + end += comment.end() + elif match.group("hash") is not None: + marker = '"' + match.group("hash") + close = source.find(marker, end) + if close < 0: + raise ValueError("unterminated raw string") + end = close + len(marker) + result.append(Token(source[pos:end], line)) + elif not value.isspace() and not value.startswith("//"): + result.append(Token(value, line)) + line += source[pos:end].count("\n") + pos = end + return result + + +def pairs(tokens: list[Token]) -> dict[int, int]: + stack = [] + result = {} + for i, token in enumerate(tokens): + if token.value in {"(", "[", "{"}: + stack.append(i) + elif token.value in {")", "]", "}"}: + if not stack: + raise ValueError(f"unmatched delimiter at line {token.line}") + start = stack.pop() + if tokens[start].value != {")": "(", "]": "[", "}": "{"}[token.value]: + raise ValueError(f"mismatched delimiter at line {token.line}") + result[start] = i + result[i] = start + if stack: + raise ValueError("unclosed delimiter") + return result + + +def production_tokens(source: str) -> list[Token]: + tokens = tokenize(source) + matching = pairs(tokens) + removed = set() + for i in range(len(tokens) - 2): + if tokens[i].value != "#" or tokens[i + 1].value != "[": + continue + end = matching[i + 1] + cfg = [t.value for t in tokens[i + 2:end]] + # Only unambiguously test-only items are removed. Other cfg branches + # remain in the census, including both platform implementations. + required_test = False + if cfg[:4] == ["cfg", "(", "all", "("]: + depth = 0 + for value in cfg[4:-2]: + if value == "(": + depth += 1 + elif value == ")": + depth -= 1 + elif value == "test" and depth == 0: + required_test = True + test_only = cfg == ["cfg", "(", "test", ")"] or required_test + if not test_only: + continue + j = end + 1 + while j < len(tokens): + value = tokens[j].value + if value in {"(", "["}: + j = matching[j] + 1 + continue + if value == "{": + j = matching[j] + 1 + break + if value in {";", ","}: + j += 1 + break + j += 1 + removed.update(range(i, j)) + return [t for i, t in enumerate(tokens) if i not in removed] + + +def modules(root: Path) -> dict[str, list[Token]]: + directory = root / SOURCE + pending = [directory / "lib.rs", directory / "main.rs", *sorted((directory / "bin").glob("*.rs"))] + found = {} + while pending: + path = pending.pop() + relative = path.relative_to(root).as_posix() + if relative in found or not path.exists(): + continue + tokens = production_tokens(path.read_text()) + if any(tokens[i].value == "include" and tokens[i + 1].value == "!" for i in range(len(tokens) - 1)): + raise ValueError(f"{relative}: include! needs explicit source-census support") + found[relative] = tokens + matching = pairs(tokens) + base = path.parent if path.name in {"lib.rs", "main.rs", "mod.rs"} else path.with_suffix("") + for i, token in enumerate(tokens[:-2]): + if token.value != "mod" or tokens[i + 2].value != ";": + continue + name = tokens[i + 1].value + # Inline modules change the directory of nested external modules. + parents = [tokens[j + 1].value for j in range(i - 2) + if tokens[j].value == "mod" and tokens[j + 2].value == "{" + and matching[j + 2] > i] + module_base = base.joinpath(*parents) + candidates = [module_base / f"{name}.rs", module_base / name / "mod.rs"] + # An immediately preceding #[path] may precede visibility tokens. + for j in range(max(0, i - 12), i): + if [t.value for t in tokens[j:j + 2]] == ["path", "="]: + candidates = [path.parent / json.loads(tokens[j + 2].value)] + existing = [p for p in candidates if p.is_file()] + if len(existing) != 1: + raise ValueError(f"cannot resolve {relative} mod {name}: {candidates}") + pending.append(existing[0]) + return found + + +def normalized(tokens: list[Token]) -> str: + return " ".join("STRING" if '"' in t.value else t.value for t in tokens) + + +def functions(tokens: list[Token]): + matching = pairs(tokens) + occurrences = Counter() + result = [] + for i, token in enumerate(tokens[:-1]): + if token.value != "fn" or tokens[i + 1].value == "(": + continue + name = tokens[i + 1].value + j = i + 2 + while j < len(tokens) and tokens[j].value not in {"{", ";"}: + j = matching[j] + 1 if tokens[j].value in {"(", "["} else j + 1 + if j == len(tokens) or tokens[j].value == ";": + continue + occurrences[name] += 1 + result.append((i, j, matching[j], f"{name}#{occurrences[name]}", name)) + return result + + +def census(root: Path) -> tuple[Counter, list[dict], int]: + sources = modules(root) + pool_types = {"PgPool", "Pool", "Executor", "Acquire", "AuditReader", "AuditConnection", "PoolConnection"} + changed = True + while changed: + before = len(pool_types) + for tokens in sources.values(): + for i in range(1, len(tokens) - 1): + if tokens[i].value == "as" and tokens[i - 1].value in pool_types: + pool_types.add(tokens[i + 1].value) + if tokens[i - 1].value == "type": + end = i + 1 + while end < len(tokens) and tokens[end].value != ";": + end += 1 + if any(t.value in pool_types for t in tokens[i + 1:end]): + pool_types.add(tokens[i].value) + changed = len(pool_types) != before + helpers = set() + for tokens in sources.values(): + for start, body, _, _, name in functions(tokens): + signature = {t.value for t in tokens[start:body]} + if start > 0 and tokens[start - 1].value == "async" and signature & pool_types: + helpers.add(name) + # Follow imported function aliases as well as direct/qualified helper calls. + changed = True + while changed: + before = len(helpers) + for tokens in sources.values(): + for i in range(1, len(tokens) - 1): + if tokens[i].value == "as" and tokens[i - 1].value in helpers: + helpers.add(tokens[i + 1].value) + changed = len(helpers) != before + sites = Counter() + details = [] + borrowed = 0 + for path, tokens in sorted(sources.items()): + matching = pairs(tokens) + definitions = functions(tokens) + declarations = {start + 1 for start, *_ in definitions} + for i in range(1, len(tokens) - 1): + name = tokens[i].value + if i in declarations or name not in OPERATIONS | helpers: + continue + opening = i + 1 + # Generic method calls (fetch_one::<...>) are uncommon, but must + # not disappear from the census if introduced. + if tokens[opening].value == "::" and tokens[opening + 1].value == "<": + opening += 2 + depth = 1 + while depth and opening + 1 < len(tokens): + opening += 1 + depth += (tokens[opening].value == "<") - (tokens[opening].value == ">") + opening += 1 + if opening >= len(tokens) or tokens[opening].value != "(": + continue + enclosing = [(body, label) for _, body, end, label, _ in definitions if body < i < end] + label = max(enclosing)[1] if enclosing else "<module>" + args = tokens[opening + 1:matching[opening]] + previous = tokens[i - 1].value + if name in EXECUTORS and previous == "." and [t.value for t in args[:2]] == ["&", "mut"]: + borrowed += 1 + continue + # A receiver for acquire/begin, or full arguments for executor and + # helper calls. Do not depend on a variable being named 'pool'. + receiver = [] + if previous == "." and name not in EXECUTORS: + j = i - 2 + while j >= 0: + value = tokens[j].value + if value in {")", "]"}: + j = matching[j] - 1 + continue + if value in {";", "=", "{", "}", ",", "(", "=>", "&", "return", "?", ":", "*", "mut"}: + break + j -= 1 + receiver = tokens[j + 1:i] + key = f"{path.removeprefix(str(SOURCE) + '/')}::{label}::{normalized(receiver)}{name}({normalized(args)})" + # Removing or disabling a timer changes the classified site too. + for j in range(i): + if tokens[j].value == "time_pool_acquire" and tokens[j + 1].value == "(" and matching[j + 1] > i: + first = j + 2 + end = first + while tokens[end].value != ",": + end = matching[end] + 1 if tokens[end].value in {"(", "[", "{"} else end + 1 + key += f" [timer: {normalized(tokens[first:end])}]" + break + sites[key] += 1 + details.append({"site": key, "line": tokens[i].line}) + return sites, details, borrowed + + +def check(sites: Counter, inventory: dict) -> list[str]: + errors = [] + entries = inventory["sites"] + policies = inventory["policies"] + for site, count in sites.items(): + entry = entries.get(site) + if entry is None: + errors.append(f"unclassified ({count}): {site}") + continue + expected, policy = entry + if expected != count: + errors.append(f"changed count {expected} -> {count}: {site}") + rule = policies.get(policy, {}) + if rule.get("class") not in CLASSES or not rule.get("owner") or not rule.get("reason"): + errors.append(f"classification requires class, owner and reason: {site}") + errors.extend(f"stale classification: {site}" for site in entries.keys() - sites.keys()) + used = {entry[1] for entry in entries.values()} + errors.extend(f"stale policy: {policy}" for policy in policies.keys() - used) + return errors + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--root", type=Path, default=ROOT) + parser.add_argument("--census", action="store_true", help="print source sites without changing classifications") + args = parser.parse_args() + sites, details, borrowed = census(args.root) + if args.census: + print(json.dumps({"sites": sites, "locations": details, "borrowed_statements": borrowed}, indent=2)) + return 0 + inventory = json.loads((args.root / INVENTORY).read_text()) + errors = check(sites, inventory) + if errors: + print("\n".join(errors)) + return 1 + counts = Counter() + for site, count in sites.items(): + policy = inventory["sites"][site][1] + counts[inventory["policies"][policy]["class"]] += count + print(f"PRISM checkout source census: {dict(sorted(counts.items()))}; {borrowed} mutable executor borrows (not fresh checkouts)") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/check_prism_settings.py b/scripts/check_prism_settings.py new file mode 100644 index 000000000..c78ba1ee9 --- /dev/null +++ b/scripts/check_prism_settings.py @@ -0,0 +1,93 @@ +#!/usr/bin/env python3 +"""Keep the native environment inventory and retired-setting guidance honest.""" +from pathlib import Path +import re +import subprocess +import sys + +ROOT = Path(__file__).resolve().parents[1] +NAME = re.compile(r"\bPRISM_[A-Z0-9_]+\b") +HISTORICAL = re.compile(r"<!-- retired-setting: (PRISM_[A-Z0-9_]+) -->") +OPERATOR_FILES = ("scripts/check-env.sh", ".env.example", "PRISM.md", "README.md") +GUIDANCE_TREES = ("doc", "docs") + + +def inventory(path): + return {line for line in path.read_text().splitlines() if line.startswith("PRISM_")} + + +def guidance_paths(root): + """Operator-facing files, excluding anything git ignores as generated output.""" + paths = [root / name for name in OPERATOR_FILES] + paths.extend(sorted((root / "crates").glob("*/README.md"))) + for tree in GUIDANCE_TREES: + paths.extend(sorted((root / tree).rglob("*"))) + paths = [path for path in paths if path.is_file()] + try: + # Outside a git checkout this exits non-zero with no output, so nothing is skipped. + result = subprocess.run( + ["git", "-C", str(root), "check-ignore", "-z", "--stdin"], + input="\0".join(str(path.relative_to(root)) for path in paths), + capture_output=True, text=True, check=False) + ignored = {root / name for name in result.stdout.split("\0") if name} + except FileNotFoundError: + ignored = set() + return [path for path in paths if path not in ignored] + + +def check(root): + source = root / "crates/qbit-prism-server/src" + known = inventory(source / "config/native-settings.txt") + retired = inventory(source / "config/retired-settings.txt") + errors = [] + overlap = known & retired + if overlap: + errors.append(f"native and retired inventories overlap: {', '.join(sorted(overlap))}") + referenced = set() + for path in source.rglob("*.rs"): + if ("tests" in path.parts or "bin" in path.parts or path.stem.endswith("tests") + or path.name in {"capacity.rs", "environment.rs"}): + continue + # Runtime modules put inline unit-test modules after their implementation; + # a cfg(test) declaration of an external module does not end runtime code. + runtime = re.split(r"#\[cfg\(test\)\]\s*mod\s+\w+\s*\{", path.read_text())[0] + referenced.update(re.findall(r'"(PRISM_[A-Z0-9_]+)"', runtime)) + generated = { + f"PRISM_PUBLIC_{kind}CACHE_{suffix}_SECONDS" + for kind in ("", "CONFIG_", "AGGREGATE_", "ARTIFACT_") + for suffix in ("TTL", "STALE_WHILE_REVALIDATE") + } + generated.update(f"{name}_FILE" for name in ( + "PRISM_MANIFEST_SIGNING_SEED_HEX", + "PRISM_LEDGER_ATTESTATION_SIGNING_SEED_HEX", + "PRISM_OPERATOR_BEARER_TOKEN", + )) + missing = referenced - known + if missing: + errors.append(f"native settings missing from inventory: {', '.join(sorted(missing))}") + unused = known - referenced - generated + if unused: + errors.append(f"inventory settings without a native reader: {', '.join(sorted(unused))}") + for path in guidance_paths(root): + for number, line in enumerate(path.read_text().splitlines(), 1): + # A patch's removed lines document removal, not live recommendations. + if path.suffix == ".patch" and line.startswith("-"): + continue + historical = set(HISTORICAL.findall(line)) if path.suffix == ".md" else set() + body = HISTORICAL.sub("", line) + for name in sorted((set(NAME.findall(body)) & retired) - historical): + errors.append(f"{path.relative_to(root)}:{number}: retired setting presented as live: {name}") + return errors + + +def main(): + errors = check(ROOT) + if errors: + print("\n".join(errors), file=sys.stderr) + return 1 + print("PRISM setting inventories and operator guidance agree") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/scripts/check_toolchain_pin.py b/scripts/check_toolchain_pin.py new file mode 100644 index 000000000..20d33c5e8 --- /dev/null +++ b/scripts/check_toolchain_pin.py @@ -0,0 +1,207 @@ +#!/usr/bin/env python3 +"""Fail when the Rust toolchain pin is held in more than one version. + +The pin lives in `rust-toolchain.toml` (`[toolchain].channel`, an exact stable +version). Two other places must agree with it: `[workspace.package].rust-version` +in the root `Cargo.toml`, which every workspace crate must inherit with +`rust-version.workspace = true`, and the builder stage of +`lab/prism/Dockerfile`, whose base image is `rust:<pin>-bookworm@sha256:<digest>`. + +Usage: python3 scripts/check_toolchain_pin.py [--root DIR] [--installed] [--print-pin] + +`--installed` additionally runs `rustc --version` and `cargo --version` and +fails unless both report the pinned version, so a CI job proves it is on the +pin rather than on whatever `stable` resolves to that day. `--print-pin` +prints the pinned version and nothing else, for shell steps that compare it +against something. Exits 0 when everything agrees and 1 otherwise, naming +each disagreement. +""" + +from __future__ import annotations + +import argparse +import os +from pathlib import Path +import re +import shutil +import subprocess +import sys +import tomllib + + +ROOT = Path(__file__).resolve().parents[1] +PREFIX = "check-toolchain-pin" +TOOLCHAIN_FILE = "rust-toolchain.toml" +DOCKERFILE = Path("lab") / "prism" / "Dockerfile" +REQUIRED_COMPONENTS = ("rustfmt", "clippy") +EXACT_VERSION = re.compile(r"^[0-9]+\.[0-9]+\.[0-9]+$") +BUILDER_FROM = re.compile( + r"^FROM\s+rust:(?P<version>[^\s@-]+)-bookworm@sha256:(?P<digest>[0-9a-f]+)\s+AS\s+build\s*$" +) + + +class PinError(Exception): + """One disagreement, worded for whoever reads the CI log.""" + + +def load_toml(path: Path) -> dict: + try: + with path.open("rb") as handle: + return tomllib.load(handle) + except FileNotFoundError: + raise PinError(f"{path} is missing") from None + except tomllib.TOMLDecodeError as error: + raise PinError(f"{path} is not valid TOML: {error}") from None + + +def pinned_version(root: Path) -> str: + data = load_toml(root / TOOLCHAIN_FILE) + toolchain = data.get("toolchain") + if not isinstance(toolchain, dict): + raise PinError(f"{TOOLCHAIN_FILE} has no [toolchain] table") + channel = toolchain.get("channel") + if not isinstance(channel, str) or not EXACT_VERSION.match(channel): + raise PinError( + f"{TOOLCHAIN_FILE}: channel must be an exact stable version such as " + f"1.98.1, got {channel!r}" + ) + if toolchain.get("profile") != "minimal": + raise PinError(f"{TOOLCHAIN_FILE}: profile must be \"minimal\", got {toolchain.get('profile')!r}") + components = toolchain.get("components") + if not isinstance(components, list) or any( + name not in components for name in REQUIRED_COMPONENTS + ): + raise PinError( + f"{TOOLCHAIN_FILE}: components must include {' and '.join(REQUIRED_COMPONENTS)}, " + f"got {components!r}" + ) + return channel + + +def disagreements(root: Path, pin: str) -> list[str]: + problems = [] + workspace = load_toml(root / "Cargo.toml") + package = workspace.get("workspace", {}).get("package", {}) + rust_version = package.get("rust-version") + if rust_version != pin: + problems.append( + f"Cargo.toml: [workspace.package].rust-version is {rust_version!r} but " + f"{TOOLCHAIN_FILE} pins {pin}" + ) + members = workspace.get("workspace", {}).get("members", []) + if not members: + problems.append("Cargo.toml: [workspace].members is empty") + for member in members: + manifest = root / member / "Cargo.toml" + try: + crate = load_toml(manifest) + except PinError as error: + problems.append(str(error)) + continue + setting = crate.get("package", {}).get("rust-version") + if setting != {"workspace": True}: + problems.append( + f"{member}/Cargo.toml: rust-version must be inherited with " + f"`rust-version.workspace = true`, got {setting!r}" + ) + dockerfile = root / DOCKERFILE + try: + lines = dockerfile.read_text(encoding="utf-8").splitlines() + except FileNotFoundError: + problems.append(f"{DOCKERFILE.as_posix()} is missing") + return problems + builders = [BUILDER_FROM.match(line) for line in lines if line.startswith("FROM ")] + builders = [match for match in builders if match] + if len(builders) != 1: + problems.append( + f"{DOCKERFILE.as_posix()}: expected exactly one " + "`FROM rust:<version>-bookworm@sha256:<digest> AS build` line, " + f"found {len(builders)}" + ) + else: + version = builders[0].group("version") + digest = builders[0].group("digest") + if version != pin: + problems.append( + f"{DOCKERFILE.as_posix()}: builder image is rust:{version}-bookworm but " + f"{TOOLCHAIN_FILE} pins {pin}" + ) + if len(digest) != 64: + problems.append( + f"{DOCKERFILE.as_posix()}: builder image digest must be 64 hex characters, " + f"got {len(digest)}" + ) + return problems + + +def installed_versions(pin: str) -> list[str]: + problems = [] + for tool in ("rustc", "cargo"): + binary = os.environ.get(tool.upper(), tool) + if shutil.which(binary) is None: + problems.append(f"{binary} is not on PATH") + continue + completed = subprocess.run( + [binary, "--version"], text=True, capture_output=True, check=False + ) + if completed.returncode != 0: + problems.append( + f"{binary} --version exited with status {completed.returncode}: " + f"{completed.stderr.strip()}" + ) + continue + words = completed.stdout.split() + reported = words[1] if len(words) > 1 else "" + if reported != pin: + problems.append( + f"{tool} reports {completed.stdout.strip()!r} but {TOOLCHAIN_FILE} pins {pin}" + ) + return problems + + +def main(argv: list[str] | None = None) -> int: + parser = argparse.ArgumentParser( + description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter + ) + parser.add_argument("--root", type=Path, default=ROOT, help="checkout to check (default: this checkout)") + parser.add_argument( + "--installed", + action="store_true", + help="also require rustc and cargo on PATH to report the pinned version", + ) + parser.add_argument( + "--print-pin", action="store_true", help="print the pinned version and exit" + ) + args = parser.parse_args(argv) + root = args.root.resolve() + + try: + pin = pinned_version(root) + except PinError as error: + print(f"{PREFIX}: {error}", file=sys.stderr) + return 1 + if args.print_pin: + print(pin) + return 0 + + problems = disagreements(root, pin) + if args.installed: + problems += installed_versions(pin) + for problem in problems: + print(f"{PREFIX}: {problem}", file=sys.stderr) + if problems: + print( + f"{PREFIX}: {len(problems)} disagreement(s) with the {pin} pin in " + f"{TOOLCHAIN_FILE}", + file=sys.stderr, + ) + return 1 + where = f"{TOOLCHAIN_FILE}, Cargo.toml rust-version, every crate, {DOCKERFILE.as_posix()}" + if args.installed: + where += ", installed rustc and cargo" + print(f"{PREFIX}: Rust {pin} is pinned consistently across {where}") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/check_version_skew.py b/scripts/check_version_skew.py new file mode 100644 index 000000000..fbc4d8a62 --- /dev/null +++ b/scripts/check_version_skew.py @@ -0,0 +1,134 @@ +#!/usr/bin/env python3 +"""Fail when VERSION and any workspace crate version diverge. + +The release identity is one number held in two places that must agree: the +`VERSION` file the bootstrap ships, and `[workspace.package].version` in the +root Cargo.toml, which every crate inherits with `version.workspace = true`. +Crate versions are resolved through `cargo metadata` rather than read from the +manifests, so a crate that opts back out of the workspace version with its own +literal is reported too. + +Usage: python3 scripts/check_version_skew.py [--root DIR] + +Exits 0 when every workspace crate matches VERSION. Exits 1 when any crate +differs, when VERSION is missing, empty, or not exactly one version line, or +when cargo cannot describe the workspace. Every mismatching crate is named +with both versions. Set CARGO to run a cargo other than the one on PATH. +""" + +from __future__ import annotations + +import argparse +import json +import os +from pathlib import Path +import re +import subprocess +import sys + + +ROOT = Path(__file__).resolve().parents[1] +PREFIX = "check-version-skew" +# Cargo's SemVer: MAJOR.MINOR.PATCH with optional -prerelease and +build parts. +VERSION_PATTERN = re.compile( + r"^[0-9]+\.[0-9]+\.[0-9]+(-[0-9A-Za-z.-]+)?(\+[0-9A-Za-z.-]+)?$" +) + + +class SkewError(Exception): + """A reason the check cannot pass, worded for whoever reads the CI log.""" + + +def read_version_file(path: Path) -> str: + try: + text = path.read_text(encoding="utf-8") + except FileNotFoundError: + raise SkewError(f"{path} is missing") from None + lines = text.splitlines() + if len(lines) != 1 or not VERSION_PATTERN.match(lines[0]): + raise SkewError( + f"{path} must hold exactly one SemVer line such as 3.0.0, got {text!r}" + ) + return lines[0] + + +def workspace_crate_versions(root: Path) -> dict[str, str]: + cargo = os.environ.get("CARGO", "cargo") + command = [ + cargo, + "metadata", + "--no-deps", + "--format-version", + "1", + "--manifest-path", + str(root / "Cargo.toml"), + ] + try: + completed = subprocess.run( + command, cwd=root, text=True, capture_output=True, check=False + ) + except OSError as error: + raise SkewError(f"cannot run {cargo}: {error}") from None + if completed.returncode != 0: + raise SkewError( + f"{' '.join(command)} exited with status {completed.returncode}:\n" + f"{completed.stderr.rstrip()}" + ) + try: + metadata = json.loads(completed.stdout) + members = set(metadata["workspace_members"]) + versions = { + package["name"]: package["version"] + for package in metadata["packages"] + if package["id"] in members + } + except (ValueError, KeyError, TypeError) as error: + raise SkewError(f"cannot read cargo metadata output: {error}") from None + if not versions: + raise SkewError(f"{root / 'Cargo.toml'} declares no workspace members") + return versions + + +def main(argv: list[str] | None = None) -> int: + parser = argparse.ArgumentParser( + description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter + ) + parser.add_argument( + "--root", + type=Path, + default=ROOT, + help="directory holding VERSION and the workspace Cargo.toml " + "(default: this checkout)", + ) + args = parser.parse_args(argv) + root = args.root.resolve() + + try: + expected = read_version_file(root / "VERSION") + crates = workspace_crate_versions(root) + except SkewError as error: + print(f"{PREFIX}: {error}", file=sys.stderr) + return 1 + + skewed = { + name: actual for name, actual in sorted(crates.items()) if actual != expected + } + for name, actual in skewed.items(): + print(f"{PREFIX}: {name} is {actual} but VERSION is {expected}", file=sys.stderr) + if skewed: + print( + f"{PREFIX}: {len(skewed)} of {len(crates)} workspace crates differ from " + f"VERSION {expected}; bump [workspace.package].version in Cargo.toml " + "and VERSION together", + file=sys.stderr, + ) + return 1 + print( + f"{PREFIX}: VERSION {expected} matches every workspace crate: " + f"{', '.join(sorted(crates))}" + ) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/generate_prism_alerts.py b/scripts/generate_prism_alerts.py new file mode 100644 index 000000000..8712c2320 --- /dev/null +++ b/scripts/generate_prism_alerts.py @@ -0,0 +1,226 @@ +#!/usr/bin/env python3 +"""Render migration tables and a review-only qbit-tools patch from the frozen snapshot.""" + +import argparse +import difflib +import hashlib +import json +from pathlib import Path +import re + +ROOT = Path(__file__).resolve().parents[1] +START = "<!-- generated-migration:start -->" +END = "<!-- generated-migration:end -->" + + +def load(path): + return json.loads((ROOT / path).read_text()) + + +def specification_digest(): + # Include the source inputs and the generator/fixture contract itself. A + # stale generator or deployment manifest must invalidate the checked-in + # patch just as surely as a changed rule specification. + return hashlib.sha256(b"".join((ROOT / path).read_bytes() for path in [ + "docs/prism-native-alert-rules.json", "docs/prism-postgres-alert-rules.json", + "docs/prism-postgres-exporter-queries.yaml", "docs/prism-alert-migration.json", + "tests/fixtures/prism-deployed-alerts.json", "scripts/generate_prism_alerts.py", + ])).hexdigest() + + +def postgres_rules(): + """D3: compare durable WAL prefixes, not the last reported replay latency.""" + target = 'job="qbit-postgres-primary",network="__NETWORK__"' + standby = target + ',application_name="prism_standby_1"' + identity = "job, instance, network, application_name" + + def metric(name): + return "pg_stat_replication_" + name + "{" + standby + "}" + + def unknown(sample, age=0): + # An absent/negative sample or missing recent history remains unknown. + # With 1s scrapes allow at most one missed interval (2s sample age). + elapsed = f"(time() - timestamp({sample}))" + return f"(max(({sample} < bool 0) + ({elapsed} < bool {age}) + ({elapsed} > bool {age + 2})) or vector(1))" + + common = [f"(max({name}{{{target}}} != bool {healthy}) or vector(1))" + for name, healthy in [("up", 1), ("pg_up", 1), ("pg_exporter_last_scrape_error", 0)]] + common += [f"((count(up{{{target}}}) != bool 1) or vector(1))", unknown(metric("count"))] + primary_hi, primary_lo = [metric("primary_flush_lsn_" + half) for half in ["hi", "lo"]] + replay_hi, replay_lo = [metric("replay_lsn_" + half) for half in ["hi", "lo"]] + old_hi, old_lo = primary_hi + " offset 5s", primary_lo + " offset 5s" + # Lexicographic comparison retains all 64 LSN bits, including low-word wrap. + behind = (f"({replay_hi} < bool on({identity}) {old_hi}) + " + f"(({replay_hi} == bool on({identity}) {old_hi}) * on({identity}) " + f"({replay_lo} < bool on({identity}) {old_lo}))") + lag_terms = common + [unknown(value) for value in [primary_hi, primary_lo, replay_hi, replay_lo]] + lag_terms += [unknown(value, 5) for value in [old_hi, old_lo]] + lag_terms += [unknown(metric("async")), f"(max({metric('async')} != bool 1) or vector(1))"] + lag_terms += [f"(max({behind}) or vector(1))"] + disconnected_terms = common + [f"(max({metric('count')} != bool 1) or vector(1))"] + original = load("docs/prism-postgres-alert-rules.json")["rules"] + for rule, terms in zip(original, [lag_terms, disconnected_terms], strict=True): + rule["expr"] = "clamp_max(" + " + ".join(terms) + ", 1)" + original[0]["description"] = ( + "D3: standby replay has not reached the primary durable WAL prefix observed five seconds earlier, " + "or position/history/exporter observations are unknown, stale or failed, or the required async topology is invalid. " + "Deployment-provided: requires primary SQL query metrics and one-second scrapes; verify in #281/#291. " + "Idle caught-up positions are healthy even when PostgreSQL reports NULL replay_lag. " + "The five-second comparison is sampled, with up to one scrape interval of timing uncertainty.") + original[1]["description"] = ( + "D3: the primary reports zero or ambiguous rows for prism_standby_1, or exporter/query observations " + "are missing, stale or failed. Deployment-provided: requires the primary exporter and one-second " + "scrapes; verify in #281/#291. This is the dedicated asynchronous HA standby, not the public read replica.") + return {"rules": original} + + +def migration_tables(): + migration = load("docs/prism-alert-migration.json") + lines = [START, "<!-- Run: python3 scripts/generate_prism_alerts.py -->", "", + "## Every deployed alert", "", + "78 definitions: **27 migrated, 34 unchanged external, 17 with no replacement**.", + "This includes definitions behind deployment Jinja flags; the snapshot does not record their effective values.", "", + "| Deployed UID | Deployed alert | Native rule / disposition | Reason |", + "| --- | --- | --- | --- |"] + for alert in migration["deployed_alerts"]: + replacement = ", ".join(f"`{name}`" for name in alert["replacement"]) or "no replacement" + lines.append(f"| `{alert['uid']}` | {alert['title']} | {replacement} ({alert['disposition']}) | {alert['reason']} |") + lines += ["", "## Every historical name", "", + "All **46** distinct tokens in the historical appendix are mapped below, including histogram suffixes and names retained by native producers.", "", + "| Historical 2.x.x name | Native replacement / disposition | Reason |", + "| --- | --- | --- |"] + for metric in migration["retired_names"]: + replacement = ", ".join(f"`{name}`" for name in metric["replacement"]) or "no replacement" + lines.append(f"| `{metric['name']}` | {replacement} | {metric['reason']} |") + lines += ["", "## Native rule evidence", "", + "The executable PromQL and Grafana conditions are in [prism-native-alert-rules.json](prism-native-alert-rules.json).", + "These are the only rewritten native rules; the historical appendix is not an active rule specification.", "", + "| Rule | Threshold evidence | Producer contract |", "| --- | --- | --- |"] + for rule in load("docs/prism-native-alert-rules.json")["rules"]: + refs = ", ".join(f"[{ref.split('/')[-1]}](../{ref})" for ref in rule["producer_refs"]) + lines.append(f"| {rule['title']} | {rule['basis']} | {refs} |") + return "\n".join(lines) + "\n\n" + END + + +def jinja_rules(rules): + # Documentation-only fields never enter the Grafana provisioning contract. + rows = [{key: value for key, value in rule.items() + if key not in {"role", "basis", "producer_refs"}} for rule in rules] + text = json.dumps(rows, indent=2).replace("__NETWORK__", '" ~ qbit_monitoring_stack_network ~ "') + overrides = { + "qbit-prism-connected-clients": ("qbit_monitoring_stack_prism_alert_connected_clients_threshold", "qbit_monitoring_stack_prism_alert_connected_clients_for", "10"), + "qbit-prism-block-candidate-oldest": ("qbit_monitoring_stack_prism_alert_block_candidate_age_warning_seconds", "qbit_monitoring_stack_prism_alert_block_candidate_age_for", "15"), + "qbit-prism-candidate-oldest-critical": ("qbit_monitoring_stack_prism_alert_block_candidate_age_critical_seconds", "qbit_monitoring_stack_prism_alert_block_candidate_age_for", "60"), + "qbit-prism-semantic-work-coverage": ("qbit_monitoring_stack_prism_alert_semantic_coverage_warning_ratio", "qbit_monitoring_stack_prism_alert_semantic_coverage_warning_for", "0.95"), + "qbit-prism-semantic-coverage-critical": ("qbit_monitoring_stack_prism_alert_semantic_coverage_critical_ratio", "qbit_monitoring_stack_prism_alert_semantic_coverage_critical_for", "0.5"), + } + canonical = {r["uid"] for r in rules} + missing = set(overrides) - canonical + if missing and (canonical & set(overrides)): + raise AssertionError(f"override target UID missing from rendered rules: {sorted(missing)}") + for uid, (threshold, dwell, literal) in overrides.items(): + marker = '"uid": "' + uid + '"' + start = text.find(marker) + if start < 0: continue + end = text.find('"uid": "', start + len(marker)) + block_end = end if end >= 0 else len(text) + block = text[start:block_end] + expr = re.search(r'("expr": ")(.*?)(",\n)', block) + if expr and literal in expr.group(2): + before, after = expr.group(2).split(literal, 1) + rebuilt = expr.group(1) + before + '" ~ (' + threshold + ' | default(' + literal + ')) ~ "' + after + expr.group(3) + block = block[:expr.start()] + rebuilt + block[expr.end():] + block = block.replace('"for": "3m"', '"for": (' + dwell + ' | default(\'3m\'))', 1) + text = text[:start] + block + text[block_end:] + return text + + +def deployment_template(snapshot): + manifest = load("tests/fixtures/prism-deployed-alerts.json") + assert hashlib.sha256(snapshot.encode()).hexdigest() == manifest["sha256"], "snapshot differs from the reviewed qbit-tools commit" + rules = load("docs/prism-native-alert-rules.json")["rules"] + ha = load("docs/prism-postgres-alert-rules.json")["rules"] + external_uids = {row["uid"] for row in manifest["alerts"] if row["external"]} + kept_uids = external_uids | {rule["uid"] for rule in rules} + deleted = [row["uid"] for row in manifest["alerts"] if row["uid"] not in kept_uids] + # Preserve deployment-owned sidecar, backup and Docker lifecycle definitions. + blocks = re.findall(r"\n \{\n.*?\n \}", snapshot, re.S) + sidecar = [block for block in blocks if '"prism_stratum_rule"' in block] + variables = [line for line in snapshot.splitlines() + if line.startswith(("{% set prism_stratum_", "{% set prism_backup_"))] + tail = snapshot[snapshot.index("{% if qbit_monitoring_stack_prism_backup_alerts_effective"):] + tail = tail[:tail.index('{% set qbit_monitoring_stack_alert_rule_group_name =')] + output = ["{# Native 3.x.x cutover draft, bootstrap #279. Apply only with the native image.", + " Specification SHA-256: " + specification_digest(), + " Generated from the reviewed a007a142 snapshot; backup, sidecar and lifecycle rules retain their producers.", + " Native numeric bounds marked provisional must be measured in #291 before production cutover.", + " The D3 group requires PRIMARY postgres_exporter query metrics; see bootstrap docs/prism-alert-migration.md.", + " Retired UIDs are explicitly deleted: omitting a provisioned rule would leave it active in Grafana.", + "#}"] + variables + output += ["{# D3 deployment-provided queries.yaml stanza; install on the PRIMARY exporter:", + (ROOT / "docs/prism-postgres-exporter-queries.yaml").read_text(), "#}", + "{% set prism_native_rules = " + jinja_rules([r for r in rules if r["role"] == "run"]) + " %}", + "{% set prism_public_read_alert_rules = " + jinja_rules([r for r in rules if r["role"] == "public-api"]) + " %}", + "{% set prism_database_alert_rules = " + jinja_rules(ha) + " %}", + "{% set prism_external_stratum_rules = [" + ",".join(sidecar) + "\n] %}", + "{% set alert_rules = prism_native_rules if (qbit_monitoring_stack_prism_alerts_enabled | bool) else [] %}", + "{% if (qbit_monitoring_stack_prism_alerts_enabled | bool) and (qbit_monitoring_stack_prism_stratum_alerts_enabled | bool) %}", + "{% set alert_rules = alert_rules + prism_external_stratum_rules %}", "{% endif %}", + "{% if qbit_monitoring_stack_prism_public_read_alerts_enabled | bool %}", + "{% set alert_rules = alert_rules + prism_public_read_alert_rules %}", "{% endif %}", + "{% if not (qbit_monitoring_stack_prism_alert_connected_clients_enabled | bool) %}", + "{% set alert_rules = alert_rules | rejectattr('connected_clients_rule', 'defined') | list %}", "{% endif %}", + tail, + '{% set qbit_monitoring_stack_alert_rule_group_name = "qbit-prism-alerts" %}', + '{% from "grafana-alert-rule-group.yml.j2" import render_alert_rule_group with context %}', + "{% set native_partitions = namespace(standard=[], paging=[]) %}", + "{% for rule in alert_rules %}", + "{% if rule.severity == 'critical' and (rule.labels | default({})).get('page') == 'true' %}", + "{% set native_partitions.paging = native_partitions.paging + [rule] %}", + "{% else %}", "{% set native_partitions.standard = native_partitions.standard + [rule] %}", + "{% endif %}", "{% endfor %}", "apiVersion: 1", "deleteRules:"] + for uid in deleted: + output += [" - orgId: 1", " uid: " + uid] + output += ["{# Disabled native rules must be deleted too, including reused legacy UIDs. #}", + "{% set emitted_uids = alert_rules | map(attribute='uid') | list %}", + "{% for rule in prism_native_rules + prism_public_read_alert_rules %}", + "{% if rule.uid not in emitted_uids %}", + " - orgId: 1", " uid: {{ rule.uid }}", "{% endif %}", "{% endfor %}", + "groups:", "{% if native_partitions.standard %}", + "{{ render_alert_rule_group('qbit-prism-alerts', qbit_monitoring_stack_grafana_alert_standard_evaluation_interval, native_partitions.standard) }}", "{% endif %}", + "{% if native_partitions.paging %}", + "{{ render_alert_rule_group('qbit-prism-alerts-paging', qbit_monitoring_stack_grafana_alert_paging_evaluation_interval, native_partitions.paging) }}", "{% endif %}", + "{{ render_alert_rule_group('qbit-prism-postgres-ha-alerts', qbit_monitoring_stack_grafana_alert_standard_evaluation_interval, prism_database_alert_rules) }}"] + return "\n".join(output) + "\n" + + +def update(path, value, check): + if check: + if path.read_text() != value: + raise SystemExit(f"stale generated artifact: {path.relative_to(ROOT)}") + else: + path.write_text(value) + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--snapshot", type=Path, help="read-only snapshot directory, required to regenerate the deployment patch") + parser.add_argument("--check", action="store_true") + args = parser.parse_args() + update(ROOT / "docs/prism-postgres-alert-rules.json", json.dumps(postgres_rules(), indent=2) + "\n", args.check) + doc = ROOT / "docs/prism-alert-migration.md" + before, rest = doc.read_text().split(START) + _, after = rest.split(END) + update(doc, before + migration_tables() + after, args.check) + if args.check: + assert specification_digest() in (ROOT / "docs/prism-alert-rules-qbit-tools.patch").read_text(), "deployment patch must be regenerated from the current specifications" + if args.snapshot: + snapshot = (args.snapshot / "grafana-alert-rules-prism.yml.j2").read_text() + target = "ansible/roles/qbit_monitoring_stack/templates/grafana-alert-rules-prism.yml.j2" + patch = "".join(difflib.unified_diff(snapshot.splitlines(True), deployment_template(snapshot).splitlines(True), + fromfile="a/" + target, tofile="b/" + target)) + update(ROOT / "docs/prism-alert-rules-qbit-tools.patch", patch, args.check) + + +if __name__ == "__main__": + main() diff --git a/scripts/generate_prism_metrics.py b/scripts/generate_prism_metrics.py new file mode 100644 index 000000000..5e65744f7 --- /dev/null +++ b/scripts/generate_prism_metrics.py @@ -0,0 +1,76 @@ +#!/usr/bin/env python3 +"""Generate the sole PRISM inventory from registry descriptors and compatibility annotations.""" + +import argparse +import json +from pathlib import Path +import re + +ROOT = Path(__file__).resolve().parents[1] +START = "<!-- generated-inventory:start -->" +END = "<!-- generated-inventory:end -->" + + +def generate(): + source = (ROOT / "crates/qbit-prism-server/src/metrics/registry.rs").read_text() + labels_source = (ROOT / "crates/qbit-prism-server/src/metrics/labels.rs").read_text() + metadata = json.loads((ROOT / "docs/prism-metric-metadata.json").read_text()) + enums = { + name: re.findall(r'=> "([^"]+)"', body) + for name, body in re.findall(r"labels!\((\w+)\s*\{(.*?)\}\);", labels_source, re.S) + } + registry = re.findall( + r'\w+: (Counter|Gauge|Histogram), "([^"]+)", "([^"]+)";', source + ) + suffixes = {suffix for _, suffix, _ in registry} + native_metadata = metadata["native"] + unknown_native = sorted(set(native_metadata) - suffixes) + if unknown_native: + raise SystemExit("metadata names absent from registry: " + ", ".join(unknown_native)) + rows = [] + for kind, suffix, meaning in registry: + annotation = metadata["native"].get(suffix, {}) + label_parts = [] + for key, enum in annotation.get("labels", {}).items(): + if enum not in enums: + raise SystemExit(f"metadata label enum {enum!r} for {suffix!r} is absent from labels.rs") + label_parts.append(f"`{key}=" + ",".join(enums[enum]) + "`") + labels = "; ".join(label_parts) or "none" + if annotation.get("status"): + meaning += " " + annotation["status"] + legacy = ", ".join(f"`{name}`" for name in annotation.get("replaces", [])) or "none" + rows.append(("qbit_prism_" + suffix, kind.lower(), labels, "run", meaning, legacy)) + for name, annotation in metadata["public"].items(): + if not name.startswith("qbit_prism_"): + raise SystemExit(f"public metadata family is not a PRISM family: {name}") + if not annotation.get("role") or not annotation.get("type") or not annotation.get("meaning"): + raise SystemExit(f"public metadata for {name!r} is missing type, role or meaning") + rows.append((name, annotation["type"], annotation["labels"], + annotation["role"], annotation["meaning"], f"`{name}`")) + assert len(rows) == len({row[0] for row in rows}), "duplicate family" + lines = [START, "<!-- Run: python3 scripts/generate_prism_metrics.py -->", "", + "| Family | Type | Labels | Role | Meaning / status | 2.x.x name replaced |", + "| --- | --- | --- | --- | --- | --- |"] + for name, *columns in sorted(rows): + lines.append("| `" + name + "` | " + " | ".join(columns) + " |") + return "\n".join(lines) + "\n\n" + END + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--check", action="store_true") + args = parser.parse_args() + path = ROOT / "docs/prism-native-metrics.md" + old = path.read_text() + before, rest = old.split(START) + _, after = rest.split(END) + new = before + generate() + after + if args.check: + if new != old: + raise SystemExit("PRISM inventory is stale: run python3 scripts/generate_prism_metrics.py") + else: + path.write_text(new) + + +if __name__ == "__main__": + main() diff --git a/scripts/measure_postgres_wal.py b/scripts/measure_postgres_wal.py new file mode 100755 index 000000000..a4b316503 --- /dev/null +++ b/scripts/measure_postgres_wal.py @@ -0,0 +1,55 @@ +#!/usr/bin/env python3 +"""Measure server-wide WAL for one caller-controlled operation. + +Each psql invocation has its own finite timeout; the measurement as a whole +may contain four bounded calls. WAL is server-wide for the bracket interval, +and operation_seconds ends immediately after the operation call. A timeout or +connection loss after an attempted operation leaves side effects uncertain; +this helper never retries or claims rollback. +""" +from __future__ import annotations +import argparse, json, os, subprocess, time +from urllib.parse import urlparse + +def psql(dsn: str, sql: str, timeout: float) -> str: + try: + env = {"PATH": os.environ.get("PATH", "")} + return subprocess.run(["psql", dsn, "-X", "-qAt", "-v", "ON_ERROR_STOP=1", "-c", sql], check=True, text=True, capture_output=True, env=env, timeout=timeout).stdout.strip() + except FileNotFoundError as exc: + raise RuntimeError("psql executable is unavailable") from exc + except subprocess.CalledProcessError as exc: + raise RuntimeError("PostgreSQL command failed") from exc + except subprocess.TimeoutExpired as exc: + raise RuntimeError("PostgreSQL command timed out") from exc + +def measure(dsn: str, sql: str, source: str, config: str, timeout_seconds: float = 10.0) -> dict: + if not (0 < timeout_seconds <= 300) or not float(timeout_seconds) == timeout_seconds: + raise ValueError("timeout_seconds must be finite and between 0 and 300") + parsed = urlparse(dsn) + if not sql.strip(): + raise ValueError("SQL operation must not be empty") + if parsed.query or parsed.password or parsed.hostname not in (None, "localhost", "127.0.0.1", "::1") or not dsn.startswith("postgresql://"): + raise ValueError("refusing non-local or ambiguous DSN; use explicit loopback target") + setup = time.monotonic() + try: start = psql(dsn, "select pg_current_wal_lsn()", timeout_seconds) + except RuntimeError as exc: raise RuntimeError(f"initial connection failed: {exc}") from exc + setup_seconds = time.monotonic()-setup + began = time.monotonic() + try: + psql(dsn, sql, timeout_seconds) + except RuntimeError as exc: + return {"outcome":"operation_failed", "execution_stage":"operation", "side_effect_status":"unknown", "measurement_scope":"server-wide", "source":source, "config":config, "operation_seconds":time.monotonic()-began, "setup_seconds":setup_seconds, "error":str(exc)} + operation_seconds = time.monotonic()-began + try: end = psql(dsn, "select pg_current_wal_lsn()", timeout_seconds) + except RuntimeError as exc: return {"outcome":"measurement_failed", "operation_status":"succeeded", "measurement_status":"failed", "execution_stage":"post-operation", "measurement_scope":"server-wide", "source":source, "config":config, "operation_seconds":operation_seconds, "setup_seconds":setup_seconds, "error":str(exc)} + try: delta = float(psql(dsn, f"select pg_wal_lsn_diff('{end}','{start}')", timeout_seconds)) + except RuntimeError as exc: return {"outcome":"measurement_failed", "operation_status":"succeeded", "measurement_status":"failed", "execution_stage":"post-operation", "measurement_scope":"server-wide", "source":source, "config":config, "operation_seconds":operation_seconds, "setup_seconds":setup_seconds, "error":str(exc)} + return {"outcome":"ok", "operation_status":"succeeded", "measurement_status":"succeeded", "measurement_scope":"server-wide", "source":source, "config":config, "start_lsn":start, "end_lsn":end, "wal_bytes":delta, "operation_seconds":operation_seconds, "setup_seconds":setup_seconds} + +def main() -> int: + ap=argparse.ArgumentParser(); ap.add_argument("--dsn", required=True); ap.add_argument("--sql", required=True); ap.add_argument("--source", default="caller-operation"); ap.add_argument("--config", required=True); ap.add_argument("--timeout-seconds", type=float, default=10.0) + try: result=measure(**vars(ap.parse_args())) + except RuntimeError as exc: print(json.dumps({"outcome":"connection_failed", "error":str(exc)})); return 1 + except ValueError as exc: print(json.dumps({"outcome":"invalid_input", "error":str(exc)})); return 1 + print(json.dumps(result, sort_keys=True)); return 0 if result["outcome"] == "ok" else 1 +if __name__ == "__main__": raise SystemExit(main()) diff --git a/scripts/prism-recovery-evidence-parallel.py b/scripts/prism-recovery-evidence-parallel.py new file mode 100644 index 000000000..1d658d744 --- /dev/null +++ b/scripts/prism-recovery-evidence-parallel.py @@ -0,0 +1,374 @@ +#!/usr/bin/env python3 +"""Export prism-recovery-evidence.sql's rows from parallel sessions (#712). + +Usage: python3 scripts/prism-recovery-evidence-parallel.py [--jobs N] > source.rows.jsonl + +Prints exactly the bytes `psql -XqAt -v ON_ERROR_STOP=1 -f +scripts/prism-recovery-evidence.sql` prints, so scripts/prism-recovery-evidence.py +summarizes them unchanged. Every session connects as that psql would: from +libpq's environment (PGSERVICE and the rest), or --dbname. None may prompt for +a password, so keep it in a password file (or PGPASSWORD). + +- A coordinator session runs the script's preamble, refusals included, in + the export transaction, and exports that transaction's snapshot. +- --jobs sessions import the snapshot, run the preamble again and export one + part each: a share_seq range of the shares, a header-hash range of the + share hashes (bytewise, as the script orders them), or the kinds between + or after those two, as the script exports them. +- Range boundaries are sampled from the tables, and only balance the work: + the first and last ranges are open, so every row is in exactly one range, + whatever the boundaries. Each share-hash range reads the whole table, so + there are --jobs of them, and four times as many share ranges. While + migration 2's share-hash backfill is pending, the script exports no share + hashes, so they are one part. +- The range parts sort under SET LOCAL work_mem (--work-mem). +- Parts wait under --work-dir and are copied to stdout in the script's order. + The last, which ends with the completion marker the summarizer requires, + is written only once every session has succeeded; a failure leaves no + marker, as a failed psql does. A killed export (SIGKILL) leaves its + .prism-recovery-evidence-* directory there; remove it. +Writers must be stopped, as for the serial export. +""" + +from __future__ import annotations + +import argparse +import concurrent.futures +import os +from pathlib import Path +import re +import shutil +import signal +import subprocess +import sys +import tempfile +import threading + +SCRIPT = Path(__file__).resolve().with_name("prism-recovery-evidence.sql") +# Where the script is cut. Each must appear exactly once, so an edit that +# rewrites one fails here rather than exporting something else. +BEGIN = "BEGIN ISOLATION LEVEL REPEATABLE READ READ ONLY;\n" +SHARES = ("SELECT jsonb_build_object('kind', 'shares', 'row', to_jsonb(s))\n" + "FROM qbit_share_ledger s ORDER BY share_seq;\n") +SHARES_ORDER = " ORDER BY share_seq;" +HASHES = "\\if :has_native_share_hashes\n" +HASHES_ORDER = ' ORDER BY header_hash COLLATE "C";' +# The 2.x mapping's header and the pattern its shares must match. +LEGACY_HEADER = "lower(right(share_id,64))" +LEGACY_MATCH = "share_id ~ '[0-9a-fA-F]{64}$'" +ENDIF = "\\endif\n" +COMMIT = "COMMIT;\n" +# Range boundaries come from a sample of about this many pages per table. +SAMPLE_PAGES = 1000 +SNAPSHOT = re.compile(r"[0-9A-F]+-[0-9A-F]+-[0-9]+") +WORK_MEM = re.compile(r"[1-9][0-9]*(kB|MB|GB)") +SHARE_BOUND = re.compile(r"-?[0-9]{1,19}") +HASH_BOUND = re.compile(r"[0-9a-f]{1,64}") + + +class ExportError(Exception): + """Why the export failed. What was written has no completion marker.""" + + +def _changed(what): + return ExportError(f"{SCRIPT.name}'s {what} changed; update {Path(__file__).name} with it") + + +class Script: + """prism-recovery-evidence.sql, cut where the parallel parts begin.""" + + def __init__(self, text): + if (text.count(SHARES) != 1 or text.count(HASHES) != 1 + or text.index(HASHES) < text.index(SHARES)): + raise _changed("shares or share_hashes export") + self.preamble, rest = text.split(SHARES) + self.middle, rest = rest.split(HASHES) + hashes, endif, self.tail = rest.partition(ENDIF) + self.hashes = HASHES + hashes + endif + if (self.preamble.count(BEGIN) != 1 or not endif or "\\if" in hashes + or hashes.count(HASHES_ORDER) != 2 or hashes.count(LEGACY_MATCH) != 1 + or hashes.count(f"{LEGACY_HEADER} AS header_hash") != 1 + or not self.tail.endswith(COMMIT)): + raise _changed("transaction or share_hashes export") + + def session(self, snapshot): + """The preamble, in a transaction that first imports `snapshot`.""" + return self.preamble.replace(BEGIN, f"{BEGIN}SET TRANSACTION SNAPSHOT '{snapshot}';\n") + + def parts(self, snapshot, share_bounds, hash_bounds, work_mem): + """(name, psql input) of every part, in the serial export's order.""" + session = self.session(snapshot) + sort = f"SET LOCAL work_mem = '{work_mem}';\n" + parts = [] + share_ranges = ranges([str(bound) for bound in share_bounds]) + for index, (lower, upper) in enumerate(share_ranges, 1): + where = predicate("share_seq", lower, upper) + parts.append((f"shares part {index} of {len(share_ranges)} ({where})", + session + sort + SHARES.replace(SHARES_ORDER, f" WHERE {where}{SHARES_ORDER}") + + COMMIT)) + parts.append(("the kinds between shares and share_hashes", session + self.middle + COMMIT)) + hash_ranges = ranges([f"'{bound}'" for bound in hash_bounds]) + for index, (lower, upper) in enumerate(hash_ranges, 1): + where = predicate('header_hash COLLATE "C"', lower, upper) + # Every 2.x range would match the pattern against the whole + # ledger, which costs more than the rest of its scan: test the + # range first. The header is that expression, so the guard + # admits exactly the shares whose header is in the range. + guard = predicate(f'{LEGACY_HEADER} COLLATE "C"', lower, upper) + hashes = self.hashes.replace(HASHES_ORDER, f" WHERE {where}{HASHES_ORDER}").replace( + LEGACY_MATCH, f"CASE WHEN {guard} THEN {LEGACY_MATCH} END") + parts.append((f"share_hashes part {index} of {len(hash_ranges)} ({where})", + session + sort + hashes + COMMIT)) + parts.append(("the kinds after share_hashes", session + self.tail)) + return parts + + +def ranges(bounds): + """Consecutive ranges between ascending bounds; the outer two are open.""" + edges = [None, *bounds, None] + return list(zip(edges, edges[1:])) + + +def predicate(key, lower, upper): + """SQL for lower <= key < upper. The last range also holds NULL, which + ORDER BY puts last, so the ranges cover every row in order.""" + if lower is None: + return "true" if upper is None else f"{key} < {upper}" + if upper is None: + return f"({key} >= {lower} OR {key} IS NULL)" + return f"{key} >= {lower} AND {key} < {upper}" + + +def sample(relation, key, parts, where=""): + """SQL for the boundaries splitting a sample of `relation` into `parts`.""" + if parts < 2: + return "SELECT NULL;\n" + fractions = ",".join(repr(index / parts) for index in range(1, parts)) + # A partitioned table's pages are its partitions'. + percent = (f"(SELECT least(100, 100.0 * {SAMPLE_PAGES} * current_setting('block_size')::int" + f" / greatest(1, sum(pg_relation_size(oid)))) FROM pg_class WHERE oid = '{relation}'::regclass" + f" OR oid IN (SELECT inhrelid FROM pg_inherits WHERE inhparent = '{relation}'::regclass))") + return (f"SELECT percentile_disc(ARRAY[{fractions}]::float8[]) WITHIN GROUP (ORDER BY {key})" + f" FROM {relation} TABLESAMPLE SYSTEM ({percent}) REPEATABLE (712){where};\n") + + +def bounds(line, pattern): + """The sampled boundaries in the array psql printed.""" + line = line.strip() + if not line: + return [] + if not (line.startswith("{") and line.endswith("}")): + raise ExportError(f"unexpected boundary sample {line!r}") + # Anything else (a NULL, a quoted value) is just not a boundary. + return [value for value in line[1:-1].split(",") if pattern.fullmatch(value)] + + +class Sessions: + """The psql processes running, so that a failure stops every one.""" + + def __init__(self, psql): + self.psql = psql + self.lock = threading.Lock() + self.running = set() + self.stopped = False + + def start(self, *args, stdin): + """A psql session whose rows the driver reads from its stdout.""" + with self.lock: + if self.stopped: + raise ExportError("stopped") + try: + process = subprocess.Popen([*self.psql, *args], stdin=stdin, stdout=subprocess.PIPE) + except OSError as error: + raise ExportError(f"cannot run {self.psql[0]}: {error}") from error + self.running.add(process) + return process + + def finish(self, process): + with self.lock: + self.running.discard(process) + + def stop(self): + with self.lock: + self.stopped = True + processes = list(self.running) + for process in processes: + if process.poll() is None: + process.terminate() + for process in processes: + process.wait() + + +class Coordinator: + """The session whose export transaction the parts share.""" + + def __init__(self, sessions, script): + self.sessions = sessions + self.process = sessions.start(stdin=subprocess.PIPE) + # Idle while the parts run: a server's idle-in-transaction limit must + # not end the snapshot before every part has imported it. + self.pending = script.preamble + "SET LOCAL idle_in_transaction_session_timeout = 0;\n" + + def ask(self, sql): + """The one row `sql` prints, run after anything pending.""" + try: + self.process.stdin.write((self.pending + sql).encode()) + self.process.stdin.flush() + line = self.process.stdout.readline() + except BrokenPipeError: + line = b"" + self.pending = "" + if not line: + raise ExportError(f"the export transaction failed (psql exited {self.process.wait()})") + return line.decode().rstrip("\n") + + def commit(self): + try: + self.process.stdin.write(COMMIT.encode()) + self.process.stdin.close() + except BrokenPipeError: + pass + status = self.process.wait() + self.sessions.finish(self.process) + if status != 0: + raise ExportError(f"the export transaction failed (psql exited {status})") + + +def run_part(sessions, name, sql, path): + script = path.with_suffix(".sql") + script.write_text(sql, encoding="utf-8") + # psql stops fetching, and still exits 0, when it cannot write a row, so + # it writes to a pipe instead, and a full disk fails the copy here. + process = sessions.start("--file", str(script), stdin=subprocess.DEVNULL) + try: + with path.open("wb") as rows: + shutil.copyfileobj(process.stdout, rows, 1 << 20) + except BaseException: + process.terminate() + raise + finally: + process.stdout.close() + status = process.wait() + sessions.finish(process) + if status != 0: + raise ExportError(f"{name} failed (psql exited {status})") + + +def export(args, out): + script = Script(SCRIPT.read_text(encoding="utf-8")) + # -w: concurrent sessions cannot share a password prompt; the password + # must come from a password file or PGPASSWORD. + psql = [args.psql, "-XqAtw", "-v", "ON_ERROR_STOP=1"] + if args.fetch_count is not None: + psql += ["-v", f"FETCH_COUNT={args.fetch_count}"] + if args.dbname is not None: + psql += ["--dbname", args.dbname] + sessions = Sessions(psql) + work = Path(tempfile.mkdtemp(prefix=".prism-recovery-evidence-", dir=args.work_dir)) + try: + coordinator = Coordinator(sessions, script) + exported = coordinator.ask( + "SELECT pg_export_snapshot(), :'has_native_share_hashes'::boolean, :'share_hashes_deferred'::boolean;\n") + snapshot, native, deferred = (exported.split("|") + ["", ""])[:3] + if (not SNAPSHOT.fullmatch(snapshot) or native not in ("t", "f") or deferred not in ("t", "f") + or exported.count("|") != 2): + raise ExportError(f"unexpected snapshot export {exported!r}") + share_bounds = sorted({int(bound) for bound in bounds(coordinator.ask( + sample("qbit_share_ledger", "share_seq", 4 * args.jobs)), SHARE_BOUND)}) + if deferred == "t": + # Pending at fence 2: the script exports none of the partial + # mapping, so splitting it would only run empty parts. + hash_sample = sample("qbit_prism_share_hashes", 'header_hash COLLATE "C"', 1) + elif native == "t": + hash_sample = sample("qbit_prism_share_hashes", 'header_hash COLLATE "C"', args.jobs) + else: + # The mapping the script derives from 2.x, which migration 002 backfills. + hash_sample = sample("qbit_share_ledger", f'{LEGACY_HEADER} COLLATE "C"', args.jobs, + f" WHERE accepted AND {LEGACY_MATCH}") + hash_bounds = sorted(set(bounds(coordinator.ask(hash_sample), HASH_BOUND))) + print(f"note: exporting snapshot {snapshot} in {args.jobs} sessions: shares in " + f"{len(share_bounds) + 1} ranges, share_hashes in {len(hash_bounds) + 1}" + f"{' (deferred: the share-hash backfill is pending)' if deferred == 't' else ''}", + file=sys.stderr, flush=True) + parts = script.parts(snapshot, share_bounds, hash_bounds, args.work_mem) + paths = [work / f"{index:05}.jsonl" for index in range(len(parts))] + # The closing kinds first, as they hold the audit bodies and the + # integrity report. Then the shares in order, so they are printed + # while the share hashes are still being exported. + last = len(parts) - 1 + middle = len(share_bounds) + 1 + order = [last, middle, *range(middle), *range(middle + 1, last)] + done = set() + written = 0 + with concurrent.futures.ThreadPoolExecutor(max_workers=args.jobs) as pool: + futures = {pool.submit(run_part, sessions, *parts[index], paths[index]): index + for index in order} + try: + for future in concurrent.futures.as_completed(futures): + future.result() + done.add(futures[future]) + # Write what is ready, in order, short of the marker's part. + while written < last and written in done: + with paths[written].open("rb") as rows: + shutil.copyfileobj(rows, out, 1 << 20) + paths[written].unlink() + written += 1 + except BaseException: + for future in futures: + future.cancel() + sessions.stop() + raise + coordinator.commit() + with paths[last].open("rb") as rows: + shutil.copyfileobj(rows, out, 1 << 20) + out.flush() + finally: + sessions.stop() + shutil.rmtree(work, ignore_errors=True) + + +def positive(value): + number = int(value) + if number < 1: + raise argparse.ArgumentTypeError(f"{value} is not a positive integer") + return number + + +def work_mem(value): + if not WORK_MEM.fullmatch(value): + raise argparse.ArgumentTypeError(f"{value!r} is not a work_mem such as 256MB") + return value + + +def main(argv=None): + parser = argparse.ArgumentParser(description=__doc__, + formatter_class=argparse.RawDescriptionHelpFormatter) + parser.add_argument("--jobs", type=positive, default=4, + help="sessions exporting parts at once, besides the coordinator (default 4)") + parser.add_argument("--work-mem", type=work_mem, default="256MB", + help="work_mem of each range part's sort (default 256MB)") + parser.add_argument("--work-dir", type=Path, default=Path("."), + help="where parts wait to be written in order; budget the whole export " + "there, as for its output (default: the current directory)") + parser.add_argument("--fetch-count", type=positive, + help="psql FETCH_COUNT of every session (default: the script's)") + parser.add_argument("--dbname", help="psql --dbname of every session (default: libpq's environment)") + parser.add_argument("--psql", default="psql", help="psql executable (default: psql)") + args = parser.parse_args(argv) + # Stop every session and remove the parts on these as on ^C. + for stop in (signal.SIGTERM, signal.SIGHUP): + signal.signal(stop, lambda signum, frame: sys.exit(128 + signum)) + try: + export(args, sys.stdout.buffer) + except (ExportError, OSError) as error: + if isinstance(error, OSError): + # Stdout may be what failed: discard what it still buffers rather + # than fail again flushing it at exit. + os.dup2(os.open(os.devnull, os.O_WRONLY), sys.stdout.fileno()) + parser.exit(1, f"recovery evidence export failed: {error}\n") + except KeyboardInterrupt: + parser.exit(130, "recovery evidence export interrupted\n") + + +if __name__ == "__main__": + main() diff --git a/scripts/prism-recovery-evidence.py b/scripts/prism-recovery-evidence.py new file mode 100644 index 000000000..164e13f82 --- /dev/null +++ b/scripts/prism-recovery-evidence.py @@ -0,0 +1,624 @@ +#!/usr/bin/env python3 +"""Summarize prism-recovery-evidence.sql JSONL without loading share history. + +Usage: python3 scripts/prism-recovery-evidence.py [--jobs N] source.rows.jsonl +No database access or mutation. The carry head is byte-compatible with 2.x's +_carry_forward_audit_head_locked (v2.0.2, 504846cc), including ASCII escaping. + +Carry integrity doesn't depend on which validator the database ran (#708): +- Rows landed before migration 011, on unmarked blocks, are re-checked here as + one running chain per payout program, as 2.x keeps their balances. +- A finding the database reports on such a row must be one its per-label rule + (2.x's, and 011's) also finds there; the per-program chain clears it. +- Findings on marked (as-issued) blocks come from the database's manifest + rule, which this export can't repeat, and fail as before. +- The summary keeps the database's report, less the findings cleared, so a 2.x + source and its migrated copy summarize alike. + +While migration 2's share-hash backfill is pending at fence 2 (`migrate +--defer-share-hashes`), the export holds no share_hashes rows and one +share_hashes_deferred record, the backfill's cursor, read last, after the +integrity report. The summary marks that kind deferred with it, in +records.share_hashes.deferred, and a note says so: compare share_hashes only +once 2 is recorded. + +Parsing is parallel by default (--jobs, the CPUs this process may use, at +most 8): worker processes parse and canonicalize newline-aligned ranges of +the file, and this process applies every check to their results in file +order, so the summary, its notes and every refusal are byte for byte the +serial summarizer's (--jobs 1). From a range a worker cannot read as a +text-mode read does (invalid UTF-8, a read error) on, the file is read +serially from its start, lines already checked included, so decoding fails +exactly where it would. +""" + +import argparse +import codecs +import collections +import hashlib +import io +import itertools +import json +import multiprocessing +import os +import pickle +import re +import sys +from concurrent.futures import ProcessPoolExecutor +from pathlib import Path + +INTEGER = re.compile(r"-?[0-9]+") +BALANCE_FIELDS = ("prior_balance", "candidate_balance", "carry_forward_balance") +KINDS = ( + "shares", "share_sequence", "sequences", "share_hashes", "blocks", "audits", "audit_bodies", "audit_snapshots", + "carry", "payouts", "candidates", "candidate_balances", + "ctv_sets", "ctv_artifacts", "ctv_checkpoints", "ctv_retry_progress", + "ctv_broadcast_attempts", + "cpfp_packages", "cpfp_retired_funding", "deferred_shares", + "fatal_state", "fatal_state_events", "policy_transitions", "chain_checkpoint", "cluster_config", + "payout_revision", "ledger_clock", "active_carry", +) +# The kinds the summary reads beyond hashing and counting. Their rows reach +# the checks whole; every other kind reaches them as canonical bytes only. +ROW_KINDS = ("blocks", "active_carry", "candidates") +# A dict, as the summary's own lookup is: an unhashable kind must raise the +# same TypeError, whose wording differs between dicts and sets. +_KIND_LOOKUP = dict.fromkeys(KINDS) +DEFAULT_CHUNK_BYTES = 32 << 20 +MAX_DEFAULT_JOBS = 8 + + +def _amount(row, field): + value = row[field] + if isinstance(value, bool) or not isinstance(value, (int, str)) or ( + isinstance(value, str) and not INTEGER.fullmatch(value)): + raise ValueError(f"active carry {field} is not an integer amount") + return int(value) + + +def _off_chain(row, prior, gross, onchain): + """Whether a row's stored balances differ from a chain at `prior`.""" + expected = (prior, prior + gross, prior + gross - onchain) + return any(_amount(row, f"{field}_sats") != value + for field, value in zip(BALANCE_FIELDS, expected)) + + +def _canonical(row): + return json.dumps(row, sort_keys=True, separators=(",", ":")).encode("utf-8") + + +class _Summary: + """Every check the summary makes, applied to records in file order. + + `line` takes one line as a text-mode read yields it. The parallel path + feeds the same state from parsed records, and from runs of rows that + need nothing but hashing and counting, which a run can do at once: no + state those checks read changes inside a run of hashed rows. + """ + + def __init__(self): + self.hashes = {kind: hashlib.sha256() for kind in KINDS} + self.counts = dict.fromkeys(KINDS, 0) + self.head = bytes(32) + self.last_share_seq = 0 + self.accepted = 0 + self.pending = 0 + self.unfinished = 0 + self.integrity = None + self.complete = False + self.marked_blocks = set() + self.carry_started = False + self.carry_order = None + # Running (gross - onchain) over earlier active rows, marked ones + # included, as the SQL rules sum it: per program (025) and per label. + self.by_program = {} + self.by_label = {} + self.legacy_breaks = 0 + self.label_breaks = set() + # The pending share-hash backfill's cursor, when the export defers + # the share_hashes kind. + self.deferred = None + + def line(self, line): + self.start() + record = json.loads(line) + kind, row = record["kind"], record["row"] + self.record(kind, row) + + def start(self): + """The check every line meets before it is parsed.""" + if self.complete: + raise ValueError("records follow completion marker") + + def record(self, kind, row, canonical=None): + """One parsed record; `canonical` is the row's canonical bytes, if known.""" + if kind == "complete": + if row is not True or self.integrity is None: + raise ValueError("invalid completion marker or missing integrity report") + self.complete = True + elif kind == "integrity": + if self.integrity is not None: + raise ValueError("duplicate integrity report") + self.integrity = row + elif kind in self.hashes: + if self.integrity is not None: + raise ValueError("accounting records follow integrity report") + if canonical is None: + canonical = _canonical(row) + self.hashes[kind].update(canonical + b"\n") + self.counts[kind] += 1 + if kind == "blocks": + if self.carry_started: + raise ValueError("block records follow active carry records") + if row.get("as_issued_audit_sha256") is not None: + self.marked_blocks.add(row["block_hash"]) + elif kind == "active_carry": + self.carry_started = True + self.head = hashlib.sha256(self.head + canonical).digest() + order = (row["block_height"], row["carry_forward_seq"]) + if self.carry_order is not None and order <= self.carry_order: + raise ValueError("active carry records are not in chain order") + self.carry_order = order + program = row["p2mr_program_hex"] + label = (row["recipient_id"], row["order_key"], program) + gross = _amount(row, "gross_amount_sats") + onchain = _amount(row, "onchain_amount_sats") + program_prior = self.by_program.get(program, 0) + label_prior = self.by_label.get(label, 0) + if row["block_hash"] not in self.marked_blocks: + if _off_chain(row, program_prior, gross, onchain): + self.legacy_breaks += 1 + if _off_chain(row, label_prior, gross, onchain): + self.label_breaks.add(row["carry_forward_seq"]) + self.by_program[program] = program_prior + gross - onchain + self.by_label[label] = label_prior + gross - onchain + elif kind == "shares": + if row["share_seq"] <= self.last_share_seq: + raise ValueError("share sequence is not strictly increasing") + self.last_share_seq = row["share_seq"] + self.accepted += int(row["accepted"]) + elif kind == "candidates": + self.pending += int(row["state"] == "pending") + # Unknown states must not make a drained-work check pass. + # `orphaned` (migration 015) is terminal and keeps its evidence. + self.unfinished += int(row["state"] not in ("submitted", "abandoned", "orphaned")) + elif kind == "share_hashes_deferred": + # Migration 2's share-hash backfill is pending: the export holds + # none of its partial mapping, and this is its cursor. Not an + # accounting record: the export reads it last, after the + # integrity report, to hold the cursor's lock as briefly as it + # can. + if self.deferred is not None: + raise ValueError("duplicate share_hashes deferral") + if not isinstance(row, dict): + raise ValueError("share_hashes deferral is not an object") + self.deferred = row + else: + raise ValueError(f"unknown evidence kind: {kind}") + + def rows(self, kind, count, blob): + """`count` rows of a kind that is only hashed and counted, whose + canonical bytes, each with its newline, are `blob`.""" + self.start() + if self.integrity is not None: + raise ValueError("accounting records follow integrity report") + self.hashes[kind].update(blob) + self.counts[kind] += count + + def shares(self, count, blob, first, last, accepted): + """A run of share rows whose integer sequences strictly increase + within it, from `first` to `last`, with `accepted` integer flags.""" + self.rows("shares", count, blob) + if first <= self.last_share_seq: + raise ValueError("share sequence is not strictly increasing") + self.last_share_seq = last + self.accepted += accepted + + def report(self, notes=None): + if not self.complete: + raise ValueError("incomplete export; psql must finish successfully") + integrity = self.integrity + if not isinstance(integrity, dict): + raise ValueError("carry-forward integrity report is not an object") + for field in ("mismatch_count", "current_drift_count", "checked_active_rows"): + value = integrity.get(field) + if isinstance(value, bool) or not isinstance(value, int): + raise ValueError(f"carry-forward integrity report lacks {field}") + if integrity["current_drift_count"] != 0: + raise ValueError("carry-forward integrity failure: current_drift_count") + if integrity["checked_active_rows"] != self.counts["active_carry"]: + raise ValueError("active carry count differs from integrity report") + mismatches = integrity.get("mismatches", []) + if not isinstance(mismatches, list) or len(mismatches) != integrity["mismatch_count"]: + raise ValueError("carry-forward integrity report does not list every mismatch") + if not all(isinstance(finding, dict) for finding in mismatches): + raise ValueError("carry-forward integrity report lists a mismatch that is not an object") + # Block, manifest and payout findings (no carry row) are as-issued only. + as_issued = [finding for finding in mismatches + if finding.get("carry_forward_seq") is None + or finding.get("block_hash") in self.marked_blocks] + if as_issued: + raise ValueError(f"carry-forward integrity failure: {len(as_issued)} as-issued finding(s)") + if self.legacy_breaks: + raise ValueError( + f"carry-forward integrity failure: {self.legacy_breaks} legacy row(s) break their payout program's chain") + # What is left are legacy findings. Each must be the per-label rule's + # (2.x's, and 011's), on a row whose program chain holds; any other is + # a rule this summary does not know, and fails. + unexplained = [finding for finding in mismatches + if finding["carry_forward_seq"] not in self.label_breaks] + if unexplained: + raise ValueError( + f"carry-forward integrity failure: {len(unexplained)} legacy finding(s) no chain rule explains") + if notes is not None and mismatches: + notes.append( + f"the database's per-label legacy carry rule reported {len(mismatches)} finding(s) that each " + "payout program's chain clears: a program paid under more than one label (#708)") + records = {kind: {"count": self.counts[kind], "sha256": self.hashes[kind].hexdigest()} + for kind in KINDS} + if self.deferred is not None: + if self.counts["share_hashes"]: + raise ValueError("share_hashes rows were exported beside their deferral") + records["share_hashes"]["deferred"] = self.deferred + if notes is not None: + notes.append( + "share_hashes deferred: migration 2's share-hash backfill is pending, so the export holds " + f"none of its mapping; the legacy shares from share_seq {self.deferred.get('next_seq')} up to " + f"{self.deferred.get('end_seq')} are not all mapped. Compare share_hashes once 2 is recorded") + return { + "schema": "qbit.prism.recovery-evidence.v1", + "records": records, + "accepted_shares": self.accepted, + "last_share_seq": self.last_share_seq, + "pending_candidates": self.pending, + "unfinished_candidates": self.unfinished, + "audit_chain_version": "qbit.prism.carry-forward-active-delta-chain.v1", + "audit_head_sha256": self.head.hex(), + # The database's report, less the findings cleared above: a clean + # report is unchanged. + "carry_forward_integrity": dict(integrity, mismatch_count=0, mismatches=[]), + } + + +def summarize(lines, notes=None, summary=None): + """Summarize `lines`, continuing `summary` if it already checked earlier ones.""" + if summary is None: + summary = _Summary() + for line in lines: + summary.line(line) + return summary.report(notes) + + +def _text_lines(text): + """`text`'s lines as a text-mode read yields them, newline included.""" + start = 0 + while True: + end = text.find("\n", start) + if end < 0: + if start < len(text): + yield text[start:] + return + yield text[start:end + 1] + start = end + 1 + + +def _share_fields(row): + """What the share checks read of a row: its own fields, or the row itself + when it is not an object, so they fail exactly as on the whole row.""" + if not isinstance(row, dict): + return row + return {field: row[field] for field in ("share_seq", "accepted") if field in row} + + +class _Items: + """A worker's results for one range, in file order: runs of rows that are + only hashed and counted, as their canonical bytes, and every other row + whole. A share run also carries its first and last sequence and its + accepted sum; its sequences strictly increase.""" + + def __init__(self): + self.items = [] + self.kind = None + self.blob = None + self.count = 0 + self.first = self.last = None + self.accepted = 0 + + def flush(self): + if self.kind == "shares": + self.items.append(("shares", self.count, self.blob, self.first, self.last, self.accepted)) + elif self.kind is not None: + self.items.append(("rows", self.kind, self.count, self.blob)) + self.kind = None + + def _append(self, kind, canonical): + if self.kind != kind: + self.flush() + self.kind, self.blob, self.count = kind, bytearray(), 0 + self.blob += canonical + self.blob += b"\n" + self.count += 1 + + def hashed(self, kind, canonical): + self._append(kind, canonical) + + def share(self, row, canonical): + sequence = row["share_seq"] + if self.kind == "shares" and sequence <= self.last: + # Out of order: a new run, whose first sequence the summary refuses. + self.flush() + if self.kind != "shares": + self.first, self.accepted = sequence, 0 + self._append("shares", canonical) + self.last = sequence + self.accepted += int(row["accepted"]) + + def record(self, kind, row, canonical): + self.flush() + self.items.append(("record", kind, row, canonical)) + + +def _fast_share(row): + """A share row the run checks can take at once: an object with an + integer sequence and a boolean or integer accepted flag.""" + return (type(row) is dict + and type(row.get("share_seq")) is int + and type(row.get("accepted")) in (bool, int)) + + +SERIAL = [("serial",)] + + +def _parse_range(path, start, end): + """Parse and canonicalize one newline-aligned byte range of the export, + from `start` to `end`, or to the end of the file when `end` is None. + + Returns where the range ended, how many lines it holds, and the items + `_replay` feeds to the summary in order. A range a worker cannot read + exactly as a text-mode read would (invalid UTF-8, a read error), or + whose rows raise something other than the summarizer's own refusals, is + `SERIAL`: the summary reads it, and everything after it, serially. + """ + try: + with open(path, "rb") as source: + source.seek(start) + data = source.read() if end is None else source.read(end - start) + text = data.decode("utf-8") + except (OSError, UnicodeDecodeError): + return None, 0, SERIAL + end = start + len(data) + del data + if "\r" in text: + # Universal newlines, as text mode reads them. + text = text.replace("\r\n", "\n").replace("\r", "\n") + results = _Items() + lines = 0 + for line in _text_lines(text): + lines += 1 + try: + record = json.loads(line) + kind, row = record["kind"], record["row"] + hashed = kind in _KIND_LOOKUP + except (ValueError, KeyError, TypeError, AttributeError) as error: + # Raised by the summary once every earlier row has been checked. + try: + pickle.dumps(error) + except Exception: + return end, lines, SERIAL + results.flush() + results.items.append(("error", error)) + return end, lines, results.items + except Exception: + # RecursionError, MemoryError: the serial reader raises it. + return end, lines, SERIAL + if not hashed: + results.record(kind, row, None) + continue + try: + canonical = _canonical(row) + except Exception: + # Never raised for json.loads output in practice; the serial + # summarizer raises it at this row, so let it. + return end, lines, SERIAL + if kind in ROW_KINDS: + results.record(kind, row, canonical) + elif kind == "shares": + if _fast_share(row): + results.share(row, canonical) + else: + results.record(kind, _share_fields(row), canonical) + else: + results.hashed(kind, canonical) + results.flush() + return end, lines, results.items + + +def _ranges(source, chunk_bytes): + """Byte ranges of `source` that each end just after a newline, but the + last, which is open: its worker reads to the end of the file, so a line + still being written is never split between two ranges.""" + start = 0 + while True: + source.seek(start + chunk_bytes) + if not source.readline().endswith(b"\n"): + yield start, None + return + end = source.tell() + yield start, end + start = end + + +class _Serial(Exception): + """A worker could not take its range: read it, and the rest, serially.""" + + +class _FullySerial(Exception): + """A refusal the serial reader might not reach: summarize from scratch.""" + + +def _replay(summary, items): + for item in items: + tag = item[0] + if tag == "rows": + summary.rows(item[1], item[2], item[3]) + elif tag == "shares": + summary.shares(*item[1:]) + elif tag == "record": + summary.start() + summary.record(item[1], item[2], item[3]) + elif tag == "error": + summary.start() + raise item[1] + else: + raise _Serial() + + +def _decode_lookahead(): + """How far past a line's end a text-mode read may have decoded before it + yields the line: one read chunk (TextIOWrapper's _CHUNK_SIZE, 8 KiB in + CPython) and the bytes of a character split across it. The window is at + least 1 MiB whatever the interpreter: a wider one only sends a refusal + near an invalid byte through the serial reader.""" + chunk = getattr(io.TextIOWrapper(io.BytesIO(), encoding="utf-8"), "_CHUNK_SIZE", 0) + return max(1 << 20, 2 * chunk) + 4 + + +def _decodes(path, start): + """Whether every byte a serial read may decode past `start`, before it + yields the line ending there, is valid UTF-8.""" + with open(path, "rb") as source: + source.seek(start) + data = source.read(_decode_lookahead()) + final = not source.read(1) + try: + codecs.getincrementaldecoder("utf-8")().decode(data, final) + except UnicodeDecodeError: + return False + return True + + +def default_jobs(): + """The CPUs this process may run on (its affinity), at most 8.""" + if hasattr(os, "process_cpu_count"): + cpus = os.process_cpu_count() + elif hasattr(os, "sched_getaffinity"): + cpus = len(os.sched_getaffinity(0)) + else: + cpus = os.cpu_count() + return max(1, min(cpus or 1, MAX_DEFAULT_JOBS)) + + +def _workers_available(): + """Whether worker processes can run `_parse_range`: forked, and able to + find it by name, as the pool sends it (a module loaded without a + sys.modules entry cannot be found, and the pool would wait forever).""" + module = sys.modules.get(_parse_range.__module__) + return (getattr(module, "_parse_range", None) is _parse_range + and "fork" in multiprocessing.get_all_start_methods()) + + +def summarize_file(path, notes=None, jobs=1, chunk_bytes=DEFAULT_CHUNK_BYTES): + """Summarize the export at `path`, parsing in `jobs` worker processes. + + The result, its notes and any refusal are the serial summarizer's. One + job, a file that is not a regular file, or a host that cannot start the + workers is the serial summarizer. + """ + summary, checked = None, 0 + if jobs > 1 and os.path.isfile(path) and _workers_available(): + summary = _Summary() + try: + with open(path, "rb") as source: + checked = _summarize_ranges(summary, path, source, jobs, chunk_bytes) + except _FullySerial: + summary, checked = None, 0 + if checked is None: + return summary.report(notes) + # The serial summarizer; or, from a range the workers could not take on, + # the rest of the file. Either way the file is read as the serial + # summarizer reads it, from the start and through the same call, so its + # decoding, any refusal and even an uncaught error are the serial ones. + # Lines the workers' ranges already checked are read but not checked + # again. + with open(path, encoding="utf-8") as source: + lines = iter(source) + collections.deque(itertools.islice(lines, checked), maxlen=0) + return summarize(lines, notes, summary) + + +def _summarize_ranges(summary, path, source, jobs, chunk_bytes): + """Feed `summary` from parallel workers in file order. Returns None once + the file is summarized, or the number of lines checked before a range + the workers could not take.""" + try: + pool = ProcessPoolExecutor(max_workers=jobs, mp_context=multiprocessing.get_context("fork")) + except Exception: + # No process pool on this host (no POSIX semaphores, say). + return 0 + try: + pending = collections.deque() + ranges = _ranges(source, chunk_bytes) + checked = 0 + while True: + while len(pending) < jobs + 2: + bounds = next(ranges, None) + if bounds is None: + break + try: + pending.append(pool.submit(_parse_range, path, *bounds)) + except Exception: + # A worker could not be started (the host is out of + # processes or memory): read the rest serially. + return checked + if not pending: + return None + try: + end, lines, items = pending.popleft().result() + except Exception: + # A worker that failed outright (killed, out of memory): + # read this range, and the rest, serially. + return checked + try: + _replay(summary, items) + except _Serial: + return checked + except Exception: + # Every byte up to `end` decoded, so a serial read meets this + # refusal too, unless it decodes an invalid byte just past it + # first; then only a serial pass from scratch is exact. + if not _decodes(path, end): + raise _FullySerial() from None + raise + checked += lines + finally: + pool.shutdown(wait=True, cancel_futures=True) + + +def _positive(value): + number = int(value) + if number < 1: + raise argparse.ArgumentTypeError(f"{value} is not a positive integer") + return number + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("evidence", type=Path) + parser.add_argument( + "--jobs", type=_positive, default=default_jobs(), + help=f"worker processes that parse the export (default: the CPU count, at most {MAX_DEFAULT_JOBS}); " + "1 summarizes serially. The summary is the same either way.") + parser.add_argument("--chunk-bytes", type=_positive, default=DEFAULT_CHUNK_BYTES, help=argparse.SUPPRESS) + args = parser.parse_args() + notes = [] + try: + report = summarize_file(args.evidence, notes, args.jobs, args.chunk_bytes) + except (OSError, ValueError, KeyError, TypeError, AttributeError) as error: + parser.exit(1, f"recovery evidence failed: {error}\n") + for note in notes: + print(f"note: {note}", file=sys.stderr) + print(json.dumps(report, sort_keys=True, indent=2)) + + +if __name__ == "__main__": + main() diff --git a/scripts/prism-recovery-evidence.sql b/scripts/prism-recovery-evidence.sql new file mode 100644 index 000000000..1fe3d8366 --- /dev/null +++ b/scripts/prism-recovery-evidence.sql @@ -0,0 +1,698 @@ +-- Read-only reconciliation evidence shared by both migration runbooks. +-- Run with psql -XqAt -v ON_ERROR_STOP=1 [-v FETCH_COUNT=<rows>]; feed stdout to +-- scripts/prism-recovery-evidence.py. Works on frozen 2.x and native schemas. +-- Writers must be stopped: the transaction gives this export one snapshot, +-- but cannot make separately taken backups or exports contemporaneous. +-- scripts/prism-recovery-evidence-parallel.py prints the same bytes from +-- sessions sharing this transaction's snapshot (#712). Each runs everything +-- before the shares export, then one part of the rest. It splits the shares +-- and share_hashes exports into ranges, so keep each a statement ending in +-- its ORDER BY. +-- Print each result in batches of FETCH_COUNT rows. Unset (0), psql buffers a +-- whole result before printing it, and the share history alone took 63 GB at +-- 65.9M shares (#705). A caller's -v FETCH_COUNT=<rows> is kept. psql 16 and +-- older batch only statements that start with SELECT or VALUES, so keep every +-- large export a plain SELECT, not WITH ... or TABLE. +SELECT :'FETCH_COUNT'::int = 0 AS prism_fetch_unset \gset +\if :prism_fetch_unset +\set FETCH_COUNT 10000 +\endif +BEGIN ISOLATION LEVEL REPEATABLE READ READ ONLY; +SET LOCAL TIME ZONE 'UTC'; +SET LOCAL bytea_output = 'hex'; +-- psql 16 and older batch through a cursor: plan for the whole result, as for +-- an ordinary query, not for its first rows. +SET LOCAL cursor_tuple_fraction = 1.0; + + +-- Match startup's source-resolution guard before any evidence query. A later +-- ledger (or temporary table) must not replace a missing local relation. +DO $source_schema$ +DECLARE + source_schema text := current_schema(); + foreign_objects text[]; +BEGIN + IF source_schema IS NULL THEN + RAISE EXCEPTION 'recovery search_path has no current schema'; + END IF; + SELECT array_agg(format('%I.%I', n.nspname, c.relname) ORDER BY n.nspname, c.relname) + INTO foreign_objects + FROM pg_catalog.pg_class c JOIN pg_catalog.pg_namespace n ON n.oid = c.relnamespace + WHERE n.nspname <> source_schema AND left(c.relname, 5) = 'qbit_' + AND pg_catalog.pg_table_is_visible(c.oid); + IF foreign_objects IS NOT NULL THEN + RAISE EXCEPTION 'recovery search_path resolves relations % outside the current schema %', foreign_objects, source_schema + USING HINT = 'Set search_path to the intended ledger schema and restore its missing tables and sequences before exporting again.'; + END IF; +END +$source_schema$; + +-- Startup requires every declared migration and refuses a database whose +-- capability declaration or source record is missing or unreadable +-- (require_schema_version, require_known_capabilities, require_migration_source). Refuse it here +-- too, so a restore that cannot start yields no evidence. Only the current +-- schema's own tables count: search_path must not resolve a lost table to +-- another ledger's. Frozen 2.x, whose #258 declaration of 2 remains valid, +-- has no native history and is not checked. Metadata is validated, +-- never exported: routine migration provenance must not change evidence. +DO $metadata$ +DECLARE + history regclass := to_regclass('qbit_prism_schema_migrations'); + hint constant text := 'Startup refuses this database. Restore the full backup, including the metadata tables of the current schema, then export again.'; + required_versions constant integer[] := ARRAY[2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26]; + applied integer[]; + missing integer[]; + metadata text; + relation record; + capability_table text; + source_table text; + capability record; + declared boolean := false; + offer_declared boolean := false; + startup_declared boolean := false; + orphan_declared boolean := false; + epoch_declared boolean := false; + lease_declared boolean := false; + fanout_lease_declared boolean := false; + hold_declared boolean := false; + -- Each capability startup understands (ledger::NATIVE_CAPABILITIES), to + -- the highest value it accepts: a migration unit test compares the two. + understood constant jsonb := '{"candidate_storage_version": 1, "candidate_offer_lifecycle": 1, "instance_offer_startup": 1, "candidate_orphan_disposition": 1, "chain_observation_epoch": 1, "candidate_claim_observed_lease": 1, "fanout_claim_observed_lease": 1, "block_submission_hold": 1, "share_hash_backfill_pending": 2}'; + backfill_fence integer; + backfill_cursor text; + epoch_state record; + source record; + source_rows bigint; +BEGIN + IF history IS NULL THEN + IF EXISTS (SELECT 1 FROM pg_catalog.pg_class + WHERE relnamespace = current_schema()::regnamespace + AND relname IN ('qbit_prism_cluster', 'qbit_prism_migration_source', + 'qbit_prism_share_hashes')) THEN + RAISE EXCEPTION 'native recovery is missing qbit_prism_schema_migrations' USING HINT = hint; + END IF; + RETURN; + END IF; + -- Mirrors require_migration_history: startup refuses any other storage, + -- columns, defaults, keys, indexes or behavior on the history table. + IF NOT EXISTS ( + SELECT 1 FROM pg_class c JOIN pg_am am ON am.oid=c.relam + WHERE c.oid=history + AND c.relnamespace=current_schema()::regnamespace + AND c.relkind='r' AND c.relpersistence='p' AND am.amname='heap' + AND NOT c.relrowsecurity AND NOT c.relforcerowsecurity + AND NOT EXISTS (SELECT 1 FROM pg_inherits WHERE inhrelid=c.oid OR inhparent=c.oid) + AND NOT EXISTS (SELECT 1 FROM pg_trigger WHERE tgrelid=c.oid) + AND NOT EXISTS (SELECT 1 FROM pg_rewrite WHERE ev_class=c.oid) + AND NOT EXISTS (SELECT 1 FROM pg_policy WHERE polrelid=c.oid) + AND (SELECT count(*) FROM pg_attribute WHERE attrelid=c.oid AND attnum>0 AND NOT attisdropped)=2 + AND EXISTS ( + SELECT 1 FROM pg_attribute a + WHERE a.attrelid=c.oid AND a.attname='version' AND NOT a.attisdropped + AND a.atttypid='pg_catalog.int4'::regtype AND a.atttypmod=-1 + AND a.attnotnull AND a.attidentity='' AND a.attgenerated='' + AND NOT EXISTS (SELECT 1 FROM pg_attrdef WHERE adrelid=c.oid AND adnum=a.attnum) + ) + AND EXISTS ( + SELECT 1 FROM pg_attribute a JOIN pg_attrdef d ON d.adrelid=a.attrelid AND d.adnum=a.attnum + WHERE a.attrelid=c.oid AND a.attname='applied_at' AND NOT a.attisdropped + AND a.atttypid='pg_catalog.timestamptz'::regtype AND a.atttypmod=-1 + AND a.attnotnull AND a.attidentity='' AND a.attgenerated='' + AND pg_get_expr(d.adbin,d.adrelid) IN ('clock_timestamp()', 'pg_catalog.clock_timestamp()') + AND NOT EXISTS ( + SELECT 1 FROM pg_depend dep WHERE dep.classid='pg_attrdef'::regclass + AND dep.objid=d.oid AND dep.refclassid='pg_proc'::regclass + AND dep.refobjid<>'pg_catalog.clock_timestamp()'::regprocedure + ) + ) + AND (SELECT count(*) FROM pg_constraint WHERE conrelid=c.oid OR confrelid=c.oid)=1 + AND (SELECT count(*) FROM pg_index WHERE indrelid=c.oid)=1 + AND EXISTS ( + SELECT 1 FROM pg_constraint k JOIN pg_index i ON i.indexrelid=k.conindid + JOIN pg_attribute a ON a.attrelid=c.oid AND a.attname='version' AND NOT a.attisdropped + JOIN pg_opclass op ON op.oid=i.indclass[0] + JOIN pg_am iam ON iam.oid=op.opcmethod + WHERE k.conrelid=c.oid AND k.contype='p' AND k.convalidated + AND NOT k.condeferrable AND NOT k.condeferred AND k.conkey=ARRAY[a.attnum] + AND i.indrelid=c.oid AND i.indisprimary AND i.indisunique + AND i.indisvalid AND i.indisready AND i.indislive + AND i.indnatts=1 AND i.indnkeyatts=1 AND i.indkey[0]=a.attnum + AND i.indexprs IS NULL AND i.indpred IS NULL + AND op.opcnamespace='pg_catalog'::regnamespace + AND op.opcname='int4_ops' AND iam.amname='btree' + ) + ) THEN + RAISE EXCEPTION 'native migration history must have the native logged-table, version primary-key and applied_at definitions, without extra columns, constraints, indexes, triggers, rules, inheritance or row security' USING HINT = hint; + END IF; + -- Keep this set aligned with ledger::REQUIRED_SCHEMA_VERSIONS: a migration + -- unit test compares the two, and the recovery regression removes each + -- version declared by the server in turn. + EXECUTE format('SELECT array_agg(version ORDER BY version) FROM %s', history) INTO applied; + -- Migration 2's share-hash backfill (#582) has not finished while its + -- cursor table exists, even with 2 recorded by hand. Its fence is read + -- with the other capabilities below, and decides: startup serves the + -- database with 2 unrecorded only at share_hash_backfill_pending = 2. + SELECT c.relkind::text INTO backfill_cursor FROM pg_catalog.pg_class c + WHERE c.relnamespace = current_schema()::regnamespace + AND c.relname = 'qbit_prism_share_hash_backfill'; + SELECT array_agg(version ORDER BY version) INTO missing + FROM unnest(required_versions) AS required(version) + WHERE NOT version = ANY(COALESCE(applied, ARRAY[]::integer[])) + AND NOT (version = 2 AND backfill_cursor IS NOT NULL); + IF missing IS NOT NULL THEN + RAISE EXCEPTION 'missing required native migrations % (found %)', missing, applied USING HINT = hint; + END IF; + FOREACH metadata IN ARRAY ARRAY['qbit_prism_schema_capabilities', 'qbit_prism_migration_source'] LOOP + SELECT n.nspname IS NOT DISTINCT FROM current_schema() AS in_current_schema, + format('%I.%I', n.nspname, c.relname) AS qualified, c.relkind::text AS kind, + c.relrowsecurity OR c.relforcerowsecurity AS row_security + INTO relation + FROM pg_catalog.pg_class c JOIN pg_catalog.pg_namespace n ON n.oid = c.relnamespace + WHERE c.oid = to_regclass(metadata); + IF NOT FOUND THEN + RAISE EXCEPTION 'database is at schema migration 6 but has no % in the current schema %', metadata, current_schema() USING HINT = hint; + ELSIF NOT relation.in_current_schema THEN + RAISE EXCEPTION '% resolves to %, outside the current schema %; refusing another schema''s metadata', metadata, relation.qualified, current_schema() USING HINT = hint; + ELSIF relation.kind <> 'r' THEN + RAISE EXCEPTION '% must be an ordinary table (found relation kind %)', metadata, relation.kind USING HINT = hint; + ELSIF metadata = 'qbit_prism_schema_capabilities' AND relation.row_security THEN + RAISE EXCEPTION 'qbit_prism_schema_capabilities has row-level security enabled or forced; refusing possibly hidden capability rows' USING HINT = hint; + END IF; + IF metadata = 'qbit_prism_schema_capabilities' THEN + capability_table := relation.qualified; + ELSE + source_table := relation.qualified; + END IF; + END LOOP; + -- Startup decodes capability as text and capability_value as int4, and + -- understands storage version 1, offer lifecycle 1, startup fence 1 and + -- orphan disposition 1 (015), chain observation epoch 1 (018), + -- candidate claim observed lease 1 (021), fanout claim observed lease 1 + -- (022), block submission hold 1 (023), and the share-hash backfill + -- fence 1 or 2 (#669), which a database declares only while 2's + -- backfill is pending: `understood` above. + -- Every format declaration is required on this native schema; a missing row is never repaired. + FOR capability IN EXECUTE format('SELECT capability, capability_value, pg_typeof(capability)::text AS name_type, pg_typeof(capability_value)::text AS value_type FROM %s ORDER BY capability DESC', capability_table) LOOP + IF capability.name_type <> 'text' OR capability.value_type <> 'integer' + OR capability.capability IS NULL OR capability.capability_value IS NULL THEN + RAISE EXCEPTION 'qbit_prism_schema_capabilities has an unreadable row: capability % (%), capability_value % (%)', capability.capability, capability.name_type, capability.capability_value, capability.value_type USING HINT = hint; + ELSIF NOT understood ? capability.capability THEN + RAISE EXCEPTION 'database declares capability % = %, which this server does not understand', capability.capability, capability.capability_value USING HINT = hint; + ELSIF capability.capability_value NOT BETWEEN 1 AND (understood ->> capability.capability)::integer THEN + RAISE EXCEPTION 'database declares % = %, but this server understands % 1 to % only', capability.capability, capability.capability_value, capability.capability, understood ->> capability.capability USING HINT = hint; + END IF; + declared := declared OR capability.capability = 'candidate_storage_version'; + offer_declared := offer_declared OR capability.capability = 'candidate_offer_lifecycle'; + startup_declared := startup_declared OR capability.capability = 'instance_offer_startup'; + orphan_declared := orphan_declared OR capability.capability = 'candidate_orphan_disposition'; + epoch_declared := epoch_declared OR capability.capability = 'chain_observation_epoch'; + lease_declared := lease_declared OR capability.capability = 'candidate_claim_observed_lease'; + fanout_lease_declared := fanout_lease_declared OR capability.capability = 'fanout_claim_observed_lease'; + hold_declared := hold_declared OR capability.capability = 'block_submission_hold'; + IF capability.capability = 'share_hash_backfill_pending' THEN + backfill_fence := capability.capability_value; + END IF; + END LOOP; + -- A pending share-hash backfill declares share_hash_backfill_pending + -- beside its cursor, and the transaction that records 2 removes both + -- (#669). At 1 startup refuses the cursor, and so does this export: + -- qbit_prism_share_hashes would be exported partial. At 2 the recent + -- range is mapped and startup serves the database while the rest is + -- mapped (`migrate --defer-share-hashes`), so the export reads it too: it + -- exports the cursor, and no share_hashes row, at its end. Only the + -- catalog is read here, never the cursor table: every session of the + -- parallel export runs this block, and a lock on the cursor held by one + -- that waits for the others would stall the transaction that records 2. + -- A declaration without its cursor has lost it, which startup refuses + -- too. + IF backfill_cursor IS NOT NULL THEN + IF backfill_cursor <> 'r' THEN + RAISE EXCEPTION 'a relation of kind % named qbit_prism_share_hash_backfill holds the name of migration 2''s share-hash backfill progress table in the current schema %', backfill_cursor, current_schema() + USING HINT = 'Startup refuses this database. Check what it holds, then rename or move it aside and export again.'; + ELSIF backfill_fence IS NULL THEN + RAISE EXCEPTION 'migration 2''s share-hash backfill has not finished: qbit_prism_share_hash_backfill exists in the current schema %, and a build before #669 started it', current_schema() + USING HINT = 'Startup refuses this database. Run plain qbit-prism-server migrate, which finishes the backfill before anything serves, then export again.'; + ELSIF backfill_fence <> 2 THEN + RAISE EXCEPTION 'migration 2''s share-hash backfill has not finished: qbit_prism_share_hash_backfill exists in the current schema %', current_schema() + USING HINT = 'Startup refuses this database. Run qbit-prism-server migrate --defer-share-hashes to permit serving with it pending, or plain qbit-prism-server migrate to finish it, then export again.'; + END IF; + ELSIF backfill_fence IS NOT NULL THEN + RAISE EXCEPTION 'database declares share_hash_backfill_pending = %, but migration 2''s share-hash backfill cursor qbit_prism_share_hash_backfill is gone, so legacy shares may be unmapped', backfill_fence USING HINT = hint; + END IF; + IF NOT declared THEN + RAISE EXCEPTION 'database is at schema migration 6 but qbit_prism_schema_capabilities has no candidate_storage_version row' USING HINT = hint; + END IF; + IF NOT offer_declared THEN + RAISE EXCEPTION 'database is at schema migration 11 but qbit_prism_schema_capabilities has no candidate_offer_lifecycle row' USING HINT = hint; + END IF; + IF NOT startup_declared THEN + RAISE EXCEPTION 'database is at schema migration 12 but has no instance_offer_startup declaration' USING HINT = hint; + END IF; + IF NOT orphan_declared THEN + RAISE EXCEPTION 'database is at schema migration 15 but does not declare candidate_orphan_disposition = 1' USING HINT = hint; + END IF; + IF NOT epoch_declared THEN + RAISE EXCEPTION 'database is at schema migration 18 but does not declare chain_observation_epoch = 1' USING HINT = hint; + END IF; + IF NOT lease_declared THEN + RAISE EXCEPTION 'database is at schema migration 21 but does not declare candidate_claim_observed_lease = 1' USING HINT = hint; + END IF; + IF NOT fanout_lease_declared THEN + RAISE EXCEPTION 'database is at schema migration 22 but does not declare fanout_claim_observed_lease = 1' USING HINT = hint; + END IF; + IF NOT hold_declared THEN + RAISE EXCEPTION 'database is at schema migration 23 but does not declare block_submission_hold = 1' USING HINT = hint; + END IF; + -- The singleton row startup decodes into MigrationSource. + EXECUTE format('SELECT concat_ws('','', pg_typeof(source_state), pg_typeof(source_release), pg_typeof(source_commit), pg_typeof(candidate_storage_version), pg_typeof(prior_schema_version), pg_typeof(migrated_by), pg_typeof(migrated_at)) AS types, source_state IS NULL OR prior_schema_version IS NULL OR migrated_by IS NULL OR migrated_at IS NULL AS incomplete FROM %s WHERE singleton LIMIT 1', source_table) INTO source; + GET DIAGNOSTICS source_rows = ROW_COUNT; + IF source_rows = 0 THEN + RAISE EXCEPTION 'database is at schema migration 6 but qbit_prism_migration_source has no singleton row' USING HINT = hint; + ELSIF source.types <> 'text,text,text,integer,integer,text,timestamp with time zone' OR source.incomplete THEN + RAISE EXCEPTION 'qbit_prism_migration_source has an unreadable singleton row (column types %, required value missing: %)', source.types, source.incomplete USING HINT = hint; + END IF; + SELECT count(*) INTO source_rows FROM qbit_prism_cluster WHERE singleton; + IF source_rows <> 1 THEN + RAISE EXCEPTION 'qbit_prism_cluster must contain exactly one singleton row' USING HINT = hint; + END IF; + EXECUTE 'SELECT chain_epoch,pg_typeof(chain_epoch)::text AS type FROM qbit_prism_cluster WHERE singleton' INTO epoch_state; + IF epoch_state.chain_epoch IS NULL OR epoch_state.chain_epoch < 0 OR epoch_state.type <> 'bigint' THEN + RAISE EXCEPTION 'migration 018 chain_epoch metadata is missing or unreadable; restore the full backup, never reset the epoch' USING HINT = hint; + END IF; +END +$metadata$; + +-- Frozen 2.x lacks these native tables. Skip absent tables before parsing +-- their queries only without native migration history. A native marker +-- requires the complete native evidence tables; loss must fail the export. +-- Empty native tables still hash identically to frozen 2.x. Set before the +-- first export: the parallel export runs everything above it in every +-- session, and its share-hash and closing streams branch on these flags. +SELECT (to_regclass('qbit_prism_cpfp_packages') IS NOT NULL + OR to_regclass('qbit_prism_schema_migrations') IS NOT NULL) AS has_cpfp_packages, + (to_regclass('qbit_prism_cpfp_retired_funding') IS NOT NULL + OR to_regclass('qbit_prism_schema_migrations') IS NOT NULL) AS has_cpfp_retired_funding, + (to_regclass('qbit_prism_deferred_shares') IS NOT NULL + OR to_regclass('qbit_prism_schema_migrations') IS NOT NULL) AS has_deferred_shares, + (to_regclass('qbit_prism_audit_snapshots') IS NOT NULL + OR to_regclass('qbit_prism_schema_migrations') IS NOT NULL) AS has_audit_snapshots, + (to_regclass('qbit_prism_share_hashes') IS NOT NULL + OR to_regclass('qbit_prism_schema_migrations') IS NOT NULL) AS has_native_share_hashes, + -- Checked above: a native cursor is a backfill at fence 2. + (to_regclass('qbit_prism_schema_migrations') IS NOT NULL + AND EXISTS (SELECT 1 FROM pg_catalog.pg_class c + WHERE c.relnamespace = current_schema()::regnamespace + AND c.relname = 'qbit_prism_share_hash_backfill')) AS share_hashes_deferred, + (to_regclass('qbit_prism_cluster') IS NOT NULL + OR to_regclass('qbit_prism_schema_migrations') IS NOT NULL) AS has_cluster, + (to_regclass('qbit_prism_fatal_state_events') IS NOT NULL + OR to_regclass('qbit_prism_schema_migrations') IS NOT NULL) AS has_fatal_state_events, + (to_regclass('qbit_prism_policy_transitions') IS NOT NULL + OR to_regclass('qbit_prism_schema_migrations') IS NOT NULL) AS has_policy_transitions +\gset + +SELECT jsonb_build_object('kind', 'shares', 'row', to_jsonb(s)) +FROM qbit_share_ledger s ORDER BY share_seq; +-- Rows alone do not preserve the next allocation, including gaps left by +-- rolled-back writes. Read sequence state without consuming a value. +SELECT jsonb_build_object('kind', 'share_sequence', 'row', jsonb_build_object( + 'last_value', last_value, 'is_called', is_called)) +FROM qbit_share_ledger_share_seq_seq; +-- Other durable allocators whose values appear in exported rows. Name them +-- explicitly so a missing sequence fails the export. Session and candidate +-- dispatch sequences only schedule work and are deliberately omitted. +SELECT jsonb_build_object('kind', 'sequences', 'row', jsonb_build_object( + 'sequence', name, 'last_value', last_value, 'is_called', is_called)) +FROM ( + SELECT 'qbit_payout_carry_forward_carry_forward_seq_seq' AS name, last_value, is_called + FROM qbit_payout_carry_forward_carry_forward_seq_seq + UNION ALL SELECT 'qbit_pool_payout_entries_payout_entry_seq_seq', last_value, is_called + FROM qbit_pool_payout_entries_payout_entry_seq_seq + UNION ALL SELECT 'qbit_ctv_fanout_broadcast_attempts_attempt_seq_seq', last_value, is_called + FROM qbit_ctv_fanout_broadcast_attempts_attempt_seq_seq + UNION ALL SELECT 'qbit_audit_publication_sequence_seq', last_value, is_called + FROM qbit_audit_publication_sequence_seq +) s ORDER BY name COLLATE "C"; + +SELECT jsonb_build_object('kind', 'blocks', 'row', jsonb_build_object( + 'block_hash', block_hash, 'block_height', block_height, + 'parent_hash', parent_hash, 'coinbase_txid', coinbase_txid, + 'payout_manifest_sha256', payout_manifest_sha256, + 'as_issued_audit_sha256', to_jsonb(b)->'as_issued_audit_sha256', + 'audit_publication_sequence', audit_publication_sequence, + 'chain_state', chain_state, 'maturity_state', maturity_state, + 'found_at', to_jsonb(found_at AT TIME ZONE 'UTC'), + 'matured_at', to_jsonb(matured_at AT TIME ZONE 'UTC'), + 'disconnected_at', to_jsonb(disconnected_at AT TIME ZONE 'UTC'), + 'inactive_since', to_jsonb((to_jsonb(b)->>'inactive_since')::timestamptz AT TIME ZONE 'UTC'))) +FROM qbit_pool_blocks b ORDER BY block_hash COLLATE "C"; + +-- Import authenticates added bytes against the declared digest. Normalize +-- absent bytes to that digest so frozen, migrated and imported rows agree. +-- Hash bytea directly rather than copying potentially large bodies into JSON. +-- Import derives reader metadata from those bytes. Once they authenticate, +-- export the stored metadata only if it disagrees, so valid imports and +-- pre-import rows agree. Parse text json: bundles can exceed jsonb limits. +-- No bundle field determines bits, so every schema exports them as stored. +-- The by-commitment lookup and the latest-evidence read break a block-height +-- tie on created_at, so a restore that rewinds or swaps creation order among +-- same-height audits serves a different public artifact. The column is set +-- once at landing and rewritten by no import, backfill or migration; render +-- it in UTC so the digest ignores the session TimeZone. +-- Native reconstruction inputs are fingerprinted separately below. +SELECT EXISTS ( + SELECT 1 FROM pg_catalog.pg_attribute + WHERE attrelid = to_regclass('qbit_pool_audit_bundles') + AND attname = 'canonical_audit_bytes' AND NOT attisdropped +) OR to_regclass('qbit_prism_schema_migrations') IS NOT NULL AS has_canonical_audit_bytes +\gset +\if :has_canonical_audit_bytes +SELECT jsonb_build_object('kind', 'audits', 'row', jsonb_build_object( + 'block_hash', a.block_hash, 'audit_bundle_sha256', a.audit_bundle_sha256, + 'coinbase_tx_hex', a.coinbase_tx_hex, 'found_block_bits', a.found_block_bits, + 'created_at', to_jsonb(a.created_at AT TIME ZONE 'UTC'), + 'canonical_audit_bytes_sha256', COALESCE(a.canonical_sha256, a.audit_bundle_sha256)) + || CASE WHEN a.canonical_sha256 IS DISTINCT FROM a.audit_bundle_sha256 + OR (a.schema_version, a.found_block_network_difficulty, + a.found_block_coinbase_value_sats::numeric, + a.audit_commitment_leaves_hex, a.witness_merkle_leaves_hex) + IS NOT DISTINCT FROM (c.schema, (c.found_block->>'network_difficulty')::numeric, + (c.found_block->>'coinbase_value_sats')::numeric, + -- Canonical JSON omits empty leaf arrays; import stores []. + COALESCE(c.audit_commitment_leaves_hex, '[]'), + COALESCE(c.witness_merkle_leaves_hex, '[]')) + THEN '{}'::jsonb ELSE jsonb_build_object('mismatched_metadata', jsonb_build_object( + 'schema_version', a.schema_version, + 'found_block_network_difficulty', a.found_block_network_difficulty, + 'found_block_coinbase_value_sats', a.found_block_coinbase_value_sats, + 'audit_commitment_leaves_hex', a.audit_commitment_leaves_hex, + 'witness_merkle_leaves_hex', a.witness_merkle_leaves_hex)) END) +FROM ( + SELECT block_hash, audit_bundle_sha256, coinbase_tx_hex, found_block_bits, created_at, + schema_version, found_block_network_difficulty, found_block_coinbase_value_sats, + audit_commitment_leaves_hex, witness_merkle_leaves_hex, canonical_audit_bytes, + encode(pg_catalog.sha256(canonical_audit_bytes), 'hex') AS canonical_sha256 + FROM qbit_pool_audit_bundles OFFSET 0 +) a +-- Decode only authenticated bytes; corrupt bytes can be invalid UTF-8. +LEFT JOIN LATERAL json_to_record(CASE WHEN a.canonical_sha256 = a.audit_bundle_sha256 + THEN convert_from(a.canonical_audit_bytes, 'UTF8')::json END) + AS c(schema text, found_block json, audit_commitment_leaves_hex jsonb, + witness_merkle_leaves_hex jsonb) ON true +ORDER BY a.block_hash COLLATE "C"; +\else +SELECT jsonb_build_object('kind', 'audits', 'row', jsonb_build_object( + 'block_hash', block_hash, 'audit_bundle_sha256', audit_bundle_sha256, + 'coinbase_tx_hex', coinbase_tx_hex, 'found_block_bits', found_block_bits, + 'created_at', to_jsonb(created_at AT TIME ZONE 'UTC'), + 'canonical_audit_bytes_sha256', audit_bundle_sha256)) +FROM qbit_pool_audit_bundles ORDER BY block_hash COLLATE "C"; +\endif + +SELECT jsonb_build_object('kind', 'carry', 'row', to_jsonb(c)) +FROM qbit_payout_carry_forward c ORDER BY carry_forward_seq; +SELECT jsonb_build_object('kind', 'payouts', 'row', to_jsonb(p)) +FROM qbit_pool_payout_entries p ORDER BY payout_entry_seq; +-- The claim lane serves attempt_count = 0 first and backs a retry off by the +-- count, so a rewound count reorders and hastens pending work; the count and +-- last_error are legacy columns carried verbatim. So is created_at: the fresh +-- lane serves the newest pending candidate first and the retry lane breaks +-- ties on it, so a restore that rewinds or swaps creation order submits a +-- different competing block first. It is set once at enqueue and never +-- rewritten; render it in UTC so the digest ignores the session TimeZone. +-- Claim ownership is not evidence, and neither is the clock-derived schedule: +-- migration 002 backfills next_attempt_at from clock_timestamp(), so only the +-- parked sentinel (infinity) is durable and clock-independent. +-- An offer reservation is durable authority, unlike a renewable claim. Keep +-- its owner/time and the observed proof/call/outcome, including unknowns, so +-- restoring a row cannot silently rewind or alter its one-offer evidence. +SELECT jsonb_build_object('kind', 'candidates', 'row', jsonb_build_object( + 'block_hash', block_hash, 'share_id', share_id, + 'candidate_sha256', candidate_sha256, 'state', state, + 'candidate', candidate, + 'proof_observed_at_ms', to_jsonb(o)->'proof_observed_at_ms', + 'offer_reserved_at', to_jsonb((to_jsonb(o)->>'offer_reserved_at')::timestamptz AT TIME ZONE 'UTC'), + 'offer_reserved_by', to_jsonb(o)->'offer_reserved_by', + 'offered_at_ms', to_jsonb(o)->'offered_at_ms', + 'offer_outcome', to_jsonb(o)->'offer_outcome', + 'offer_reply', to_jsonb(o)->'offer_reply', + 'block_bytes', to_jsonb(o)->'block_bytes', + 'window_anchor_ms', to_jsonb(o)->'window_anchor_ms', + 'window_prior_balances_sha256', to_jsonb(o)->'window_prior_balances_sha256', + 'window_first_share_seq', to_jsonb(o)->'window_first_share_seq', + 'window_last_share_seq', to_jsonb(o)->'window_last_share_seq', + 'window_share_count', to_jsonb(o)->'window_share_count', + 'window_snapshot_sha256', to_jsonb(o)->'window_snapshot_sha256', + 'storage_version', COALESCE(to_jsonb(o)->'storage_version', '1'::jsonb), + 'attempt_count', attempt_count, 'last_error', last_error, + 'created_at', to_jsonb(created_at AT TIME ZONE 'UTC'), + 'parked', CASE WHEN to_jsonb(o)->'next_attempt_at' = '"infinity"'::jsonb THEN true END)) +FROM qbit_block_candidate_outbox o ORDER BY block_hash COLLATE "C"; +-- Every unfinished candidate retains the as-issued balances needed to land +-- or reconcile its audit, including a row that must never be offered again. +-- Terminal rows, including `orphaned` (migration 015), release their window +-- references; their offer metadata remains in the candidate record above. +-- Ignore unreferenced snapshots, which ordinary garbage collection removes. +SELECT (to_regclass('qbit_prism_balance_snapshots') IS NOT NULL + OR to_regclass('qbit_prism_schema_migrations') IS NOT NULL) AS has_candidate_balances +\gset +\if :has_candidate_balances +SELECT jsonb_build_object('kind', 'candidate_balances', 'row', jsonb_build_object( + 'prior_balances_digest', b.prior_balances_digest, + 'balances_sha256', encode(pg_catalog.sha256(b.balances), 'hex'))) +FROM qbit_prism_balance_snapshots b +WHERE EXISTS (SELECT 1 FROM qbit_block_candidate_outbox c + WHERE c.state IN ('pending','offer_reserved','offered','reconciliation') + AND c.window_prior_balances_sha256 = b.prior_balances_digest) +ORDER BY b.prior_balances_digest COLLATE "C"; +\endif +SELECT jsonb_build_object('kind', 'ctv_sets', 'row', jsonb_build_object( + 'block_hash', block_hash, 'manifest_set_sha256', manifest_set_sha256, + 'manifest_set_json', manifest_set_json, 'manifest_set', manifest_set, + 'settlement_mode', settlement_mode, + 'parent_coinbase_txid', parent_coinbase_txid, 'parent_coinbase_tx_hex', parent_coinbase_tx_hex, + 'fanout_count', fanout_count, 'fanout_output_sum_sats', fanout_output_sum_sats, + 'covenant_output_value_sats', covenant_output_value_sats)) +FROM qbit_ctv_fanout_sets ORDER BY block_hash COLLATE "C"; +-- Keep the immutable payout payload separate from native progress below. +-- Ephemeral claim ownership is not part of recovery evidence. +SELECT jsonb_build_object('kind', 'ctv_artifacts', 'row', jsonb_build_object( + 'fanout_txid', fanout_txid, 'block_hash', block_hash, + 'manifest_set_sha256', manifest_set_sha256, + 'manifest_json', manifest_json, 'manifest', manifest, + 'manifest_sha256', manifest_sha256, 'precommitment_sha256', precommitment_sha256, + 'ctv_hash', ctv_hash, 'commitment_witness_leaf_hex', commitment_witness_leaf_hex, + 'chunk_index', chunk_index, 'chunk_count', chunk_count, + 'parent_coinbase_txid', parent_coinbase_txid, 'parent_coinbase_vout', parent_coinbase_vout, + 'fanout_tx_template_hex', fanout_tx_template_hex, 'fanout_tx_hex', fanout_tx_hex, + 'anchor_vout', anchor_vout, 'covenant_output_value_sats', covenant_output_value_sats, + 'fanout_output_sum_sats', fanout_output_sum_sats, + 'settlement_status', settlement_status)) +FROM qbit_ctv_fanout_artifacts ORDER BY fanout_txid COLLATE "C"; + +-- Missing legacy columns and freshly migrated NULL/zero checkpoints are +-- equivalent. Any native confirmation or spend-scan progress must survive +-- recovery, even when settlement_status does not change. +SELECT jsonb_build_object('kind', 'ctv_checkpoints', 'row', + checkpoint || jsonb_build_object('fanout_txid', fanout_txid)) +FROM ( + SELECT fanout_txid, jsonb_build_object( + 'confirmed_block_hash', to_jsonb(a)->'confirmed_block_hash', + 'confirmed_block_height', to_jsonb(a)->'confirmed_block_height', + 'confirmed_depth', COALESCE(to_jsonb(a)->'confirmed_depth', '0'::jsonb), + 'spend_scan_next_height', to_jsonb(a)->'spend_scan_next_height', + 'spend_scan_anchor_height', to_jsonb(a)->'spend_scan_anchor_height', + 'spend_scan_anchor_hash', to_jsonb(a)->'spend_scan_anchor_hash') AS checkpoint + FROM qbit_ctv_fanout_artifacts a +) progress +WHERE checkpoint <> '{"confirmed_block_hash":null,"confirmed_block_height":null,"confirmed_depth":0,"spend_scan_next_height":null,"spend_scan_anchor_height":null,"spend_scan_anchor_hash":null}'::jsonb +ORDER BY fanout_txid COLLATE "C"; + +-- Durable retry progress survives journal pruning (latest 32 attempts), so it +-- is evidence in its own right. Claim ownership and updated_at are not. +-- Missing legacy columns and freshly migrated defaults are equivalent; +-- timestamps are normalized to UTC so the session time zone cannot matter. +SELECT jsonb_build_object('kind', 'ctv_retry_progress', 'row', + progress || jsonb_build_object('fanout_txid', fanout_txid)) +FROM ( + SELECT fanout_txid, jsonb_build_object( + 'broadcast_attempt_count', COALESCE(r->'broadcast_attempt_count', '0'::jsonb), + 'broadcast_attempt_detail_count', COALESCE(r->'broadcast_attempt_detail_count', '0'::jsonb), + 'first_broadcast_attempt_at', to_jsonb((r->>'first_broadcast_attempt_at')::timestamptz AT TIME ZONE 'UTC'), + 'last_broadcast_attempt_at', to_jsonb((r->>'last_broadcast_attempt_at')::timestamptz AT TIME ZONE 'UTC'), + 'last_broadcast_attempt_status', r->'last_broadcast_attempt_status', + 'last_broadcast_package_tx_hexes', COALESCE(r->'last_broadcast_package_tx_hexes', '[]'::jsonb), + 'last_broadcast_package_txids', COALESCE(r->'last_broadcast_package_txids', '[]'::jsonb), + 'last_broadcast_submit_result', r->'last_broadcast_submit_result', + 'last_broadcast_error', r->'last_broadcast_error', + 'broadcast_attempt_status_counts', COALESCE(r->'broadcast_attempt_status_counts', '{}'::jsonb), + 'next_broadcast_attempt_at', to_jsonb((r->>'next_broadcast_attempt_at')::timestamptz AT TIME ZONE 'UTC'), + 'broadcast_retry_backoff_seconds', COALESCE(r->'broadcast_retry_backoff_seconds', '0'::jsonb)) AS progress + FROM (SELECT fanout_txid, to_jsonb(a) AS r FROM qbit_ctv_fanout_artifacts a) rows +) retry +WHERE progress <> '{"broadcast_attempt_count":0,"broadcast_attempt_detail_count":0,"first_broadcast_attempt_at":null,"last_broadcast_attempt_at":null,"last_broadcast_attempt_status":null,"last_broadcast_package_tx_hexes":[],"last_broadcast_package_txids":[],"last_broadcast_submit_result":null,"last_broadcast_error":null,"broadcast_attempt_status_counts":{},"next_broadcast_attempt_at":null,"broadcast_retry_backoff_seconds":0}'::jsonb +ORDER BY fanout_txid COLLATE "C"; + +SELECT jsonb_build_object('kind', 'ctv_broadcast_attempts', 'row', to_jsonb(a)) +FROM qbit_ctv_fanout_broadcast_attempts a ORDER BY attempt_seq; + +-- Native replay protection must survive recovery. Frozen 2.x exports the +-- exact mapping migration 002 will backfill, including its duplicate rule. +-- Native migration history prevents a missing table from being synthesized. +-- While 2's backfill is pending, at fence 2, the mapping holds only what is +-- mapped so far, so none of it is exported: share_hashes_deferred at the +-- end exports the cursor instead. +\if :has_native_share_hashes +SELECT jsonb_build_object('kind', 'share_hashes', 'row', jsonb_build_object( + 'header_hash', header_hash, 'share_id', share_id)) +FROM (SELECT header_hash, share_id FROM qbit_prism_share_hashes + WHERE NOT :'share_hashes_deferred'::boolean +) mapping ORDER BY header_hash COLLATE "C"; +\else +SELECT jsonb_build_object('kind', 'share_hashes', 'row', jsonb_build_object( + 'header_hash', header_hash, 'share_id', share_id)) +FROM ( + SELECT DISTINCT ON (lower(right(share_id,64))) + lower(right(share_id,64)) AS header_hash, share_id + FROM qbit_share_ledger + WHERE accepted AND share_id ~ '[0-9a-fA-F]{64}$' + ORDER BY lower(right(share_id,64)), share_seq +) legacy_hashes ORDER BY header_hash COLLATE "C"; +\endif +\if :has_audit_snapshots +-- Imported legacy rows gain canonical bytes and normalized metadata, but no +-- snapshot reference. Fingerprint native reconstruction inputs separately. +-- Audit headers and inline share windows can be megabytes a row, so fetch +-- them a few rows at a time. +\set prism_fetch_rows :FETCH_COUNT +\set FETCH_COUNT 16 +SELECT jsonb_build_object('kind', 'audit_bodies', 'row', jsonb_build_object( + 'block_hash', block_hash, 'audit_bundle', audit_bundle, + 'share_snapshot_sha256', share_snapshot_sha256)) +FROM qbit_pool_audit_bundles WHERE share_snapshot_sha256 IS NOT NULL +ORDER BY block_hash COLLATE "C"; + +SELECT jsonb_build_object('kind', 'audit_snapshots', 'row', jsonb_build_object( + 'snapshot_sha256', snapshot_sha256, + 'first_share_seq', first_share_seq, 'last_share_seq', last_share_seq, + 'anchor_ms', anchor_ms, 'share_count', share_count, 'inline_shares', inline_shares)) +FROM qbit_prism_audit_snapshots ORDER BY snapshot_sha256 COLLATE "C"; +\set FETCH_COUNT :prism_fetch_rows +\endif +\if :has_cpfp_packages +SELECT jsonb_build_object('kind', 'cpfp_packages', 'row', to_jsonb(p)) +FROM qbit_prism_cpfp_packages p ORDER BY fanout_txid COLLATE "C"; +\endif +\if :has_cpfp_retired_funding +SELECT jsonb_build_object('kind', 'cpfp_retired_funding', 'row', to_jsonb(r)) +FROM qbit_prism_cpfp_retired_funding r +ORDER BY funding_txid COLLATE "C", funding_vout; +\endif +\if :has_deferred_shares +SELECT jsonb_build_object('kind', 'deferred_shares', 'row', to_jsonb(d)) +FROM qbit_prism_deferred_shares d ORDER BY block_hash COLLATE "C"; +\endif +\if :has_cluster +-- A fresh migration has no halt. Exclude routine cluster metadata so that +-- migrating an unchanged legacy backup still produces identical evidence. +-- Historical halts can have an unknown set time; retain that NULL value. +SELECT jsonb_build_object('kind', 'fatal_state', 'row', jsonb_build_object( + 'fatal_error', fatal_error, 'fatal_error_set_at', to_jsonb(c)->'fatal_error_set_at')) +FROM qbit_prism_cluster c WHERE fatal_error IS NOT NULL ORDER BY singleton; +-- observe_chain_view rejects tips below this checkpoint. Migration defaults +-- (zero work, no tip) are omitted so fresh and migrated evidence still match. +SELECT jsonb_build_object('kind', 'chain_checkpoint', 'row', jsonb_build_object( + 'best_chainwork', best_chainwork::text, 'best_tip_hash', best_tip_hash, + 'best_tip_height', best_tip_height) || + CASE WHEN chain_epoch = 0 THEN '{}'::jsonb + ELSE jsonb_build_object('chain_epoch', chain_epoch) END) +FROM qbit_prism_cluster c +WHERE best_chainwork <> 0 OR best_tip_hash IS NOT NULL OR best_tip_height IS NOT NULL OR chain_epoch <> 0 +ORDER BY singleton; +-- Ledger::configure pins the frontend's consensus, payout and signing +-- configuration onto a NULL fingerprint and rejects a mismatch otherwise, so +-- a restore that loses the pin must be visible. NULL is the migration default. +SELECT jsonb_build_object('kind', 'cluster_config', 'row', jsonb_build_object( + 'config_fingerprint', config_fingerprint)) +FROM qbit_prism_cluster c WHERE config_fingerprint IS NOT NULL ORDER BY singleton; +-- Candidate admission, landing and reconciliation require exact equality with +-- this fence, so a rewound revision can revive fenced-out payout work. The +-- migration default of zero is omitted; every later revision is evidence. +SELECT jsonb_build_object('kind', 'payout_revision', 'row', jsonb_build_object( + 'payout_revision', payout_revision)) +FROM qbit_prism_cluster c WHERE payout_revision <> 0 ORDER BY singleton; +-- append and snapshot advance this monotonic barrier so share timestamps and +-- window anchors never depend on host clock sync; a rewound clock can stamp +-- new shares below existing anchors. Only the migration default of zero is omitted. +SELECT jsonb_build_object('kind', 'ledger_clock', 'row', jsonb_build_object( + 'ledger_clock_ms', ledger_clock_ms)) +FROM qbit_prism_cluster c WHERE ledger_clock_ms <> 0 ORDER BY singleton; +\endif +\if :has_fatal_state_events +SELECT jsonb_build_object('kind', 'fatal_state_events', 'row', to_jsonb(e)) +FROM qbit_prism_fatal_state_events e ORDER BY event_id; +-- Frozen 2.x has no such identity; omit its untouched native default so +-- frozen, migrated and imported exports agree. Absent still fails. +SELECT jsonb_build_object('kind', 'sequences', 'row', jsonb_build_object( + 'sequence', 'qbit_prism_fatal_state_events_event_id_seq', + 'last_value', last_value, 'is_called', is_called)) +FROM qbit_prism_fatal_state_events_event_id_seq WHERE is_called OR last_value <> 1; +\endif +\if :has_policy_transitions +SELECT jsonb_build_object('kind', 'policy_transitions', 'row', to_jsonb(t)) +FROM qbit_prism_policy_transitions t ORDER BY transition_id; +-- Preserve journal allocation, including gaps from rolled-back transitions. +-- Omit only the untouched native default, as for fatal-state history above. +SELECT jsonb_build_object('kind', 'sequences', 'row', jsonb_build_object( + 'sequence', 'qbit_prism_policy_transitions_transition_id_seq', + 'last_value', last_value, 'is_called', is_called)) +FROM qbit_prism_policy_transitions_transition_id_seq WHERE is_called OR last_value <> 1; +\endif + +-- Exact row shape/order used by 2.x _carry_forward_audit_head_locked. +-- Stream rows rather than constructing one unbounded json_agg value. +SELECT jsonb_build_object('kind', 'active_carry', 'row', jsonb_build_object( + 'carry_forward_seq', ledger.carry_forward_seq, + 'block_hash', ledger.block_hash, 'block_height', ledger.block_height, + 'recipient_id', ledger.miner_id, 'order_key', ledger.payout_order_key, + 'p2mr_program_hex', encode(ledger.p2mr_program, 'hex'), + 'gross_amount_sats', ledger.gross_amount_sats, + 'prior_balance_sats', ledger.prior_balance_sats::text, + 'candidate_balance_sats', ledger.candidate_balance_sats::text, + 'onchain_amount_sats', ledger.onchain_amount_sats, + 'settlement_fee_sats', ledger.settlement_fee_sats, + 'carry_forward_balance_sats', ledger.carry_forward_balance_sats::text, + 'action', ledger.action, 'maturity_state', ledger.maturity_state)) +FROM qbit_payout_carry_forward ledger +JOIN qbit_pool_blocks block ON block.block_hash = ledger.block_hash +WHERE ledger.maturity_state <> 'reversed' AND block.chain_state = 'confirmed' + AND block.maturity_state <> 'reversed' +ORDER BY ledger.block_height, ledger.carry_forward_seq; + +SELECT jsonb_build_object('kind', 'integrity', 'row', qbit_carry_forward_integrity_report()); +-- The pending backfill's cursor: the legacy shares from next_seq up to +-- end_seq are not all mapped yet. scripts/prism-recovery-evidence.py marks +-- the share_hashes kind deferred with it; compare that kind once 2 is +-- recorded. Read last, in the export's snapshot, so that the cursor table's +-- lock is held only until the commit right after it, and the transaction +-- that records 2 never waits long for an export. One that recorded 2 since +-- the snapshot has dropped the table, which fails the export: export again. +-- The recent range's columns are read through the row. +\if :share_hashes_deferred +DO $deferred$ +BEGIN + IF (SELECT count(*) FROM qbit_prism_share_hash_backfill WHERE singleton) <> 1 THEN + RAISE EXCEPTION 'qbit_prism_share_hash_backfill has no row: migration 2 created it with its cursor and nothing deletes the row, so it was edited or restored selectively' + USING HINT = 'Startup refuses this database. Restore the full backup, including the metadata tables of the current schema, then export again.'; + END IF; +END +$deferred$; +SELECT jsonb_build_object('kind', 'share_hashes_deferred', 'row', jsonb_build_object( + 'start_seq', start_seq, 'next_seq', next_seq, 'end_seq', end_seq, + 'recent_min_height', to_jsonb(b)->'recent_min_height', + 'recent_start_seq', to_jsonb(b)->'recent_start_seq')) +FROM qbit_prism_share_hash_backfill b WHERE singleton; +\endif +-- A final marker makes an interrupted/failed psql export fail closed. +SELECT jsonb_build_object('kind', 'complete', 'row', true); +COMMIT; diff --git a/scripts/prism-self-check.py b/scripts/prism-self-check.py deleted file mode 100755 index 9fae29259..000000000 --- a/scripts/prism-self-check.py +++ /dev/null @@ -1,1145 +0,0 @@ -#!/usr/bin/env python3 -"""Operator readiness checks for the PRISM pool profile.""" - -from __future__ import annotations - -import argparse -import base64 -import json -import math -import os -import shutil -import socket -import subprocess -import sys -import time -import urllib.error -import urllib.request -from dataclasses import dataclass -from decimal import Decimal, InvalidOperation, ROUND_CEILING -from pathlib import Path - - -ROOT_DIR = Path(__file__).resolve().parents[1] - - -PUBLIC_QBIT_CHAINS = {"mainnet", "testnet", "testnet3", "testnet4", "signet"} -QBIT_CHAIN_FLAGS = { - "mainnet": "-chain=main", - "regtest": "-regtest", - "testnet": "-testnet", - "testnet3": "-testnet3", - "testnet4": "-testnet4", - "signet": "-signet", -} -QBIT_RPC_CHAIN_ALIASES = {"main": "mainnet"} -BOOL_TRUE = {"1", "true", "yes", "on"} -BOOL_FALSE = {"0", "false", "no", "off"} -LAUNCH_READINESS_FLAG = "QBIT_MAINNET_LAUNCH_READINESS_CHECKS_ENABLED" -PRELAUNCH_TIP_AGE_NAME = "QBIT_MAINNET_PRELAUNCH_MAX_TIP_AGE_SECONDS" -MAX_SIGNED_INT64 = 9223372036854775807 -BITCOIN_CHAIN_FLAGS = { - "mainnet": "-chain=main", - "regtest": "-regtest", - "testnet": "-testnet", - "testnet3": "-testnet", - "testnet4": "-testnet4", - "signet": "-signet", -} - - -class MissingHighdiffNotificationError(RuntimeError): - """Raised when Stratum never advertises an initial difficulty.""" - - -@dataclass -class CheckRow: - status: str - name: str - detail: str - hint: str | None = None - - -class Reporter: - def __init__(self) -> None: - self.rows: list[CheckRow] = [] - - def pass_(self, name: str, detail: str) -> None: - self.rows.append(CheckRow("PASS", name, detail)) - - def warn(self, name: str, detail: str, *, hint: str | None = None) -> None: - self.rows.append(CheckRow("WARN", name, detail, hint)) - - def fail(self, name: str, detail: str, *, hint: str | None = None) -> None: - self.rows.append(CheckRow("FAIL", name, detail, hint)) - - @property - def failed(self) -> bool: - return any(row.status == "FAIL" for row in self.rows) - - def emit(self) -> None: - width = max((len(row.name) for row in self.rows), default=0) - for row in self.rows: - print(f"{row.status:<4} {row.name:<{width}} {row.detail}") - if row.hint: - print(f" {'':<{width}} hint: {row.hint}") - - -def env_file_args() -> list[str]: - upstream = ROOT_DIR / "config" / "upstream.env" - if not upstream.exists(): - upstream = ROOT_DIR / "config" / "upstream.env.example" - args = ["--env-file", str(upstream)] - deploy_env = os.environ.get("DEPLOY_ENV_FILE", "").strip() - if deploy_env: - deploy_path = Path(deploy_env) - if not deploy_path.is_absolute(): - deploy_path = ROOT_DIR / deploy_path - args.extend(["--env-file", str(deploy_path)]) - else: - local_env = ROOT_DIR / ".env" - if local_env.exists(): - args.extend(["--env-file", str(local_env)]) - return args - - -def compose_base_command() -> list[str]: - project_name = os.environ.get("COMPOSE_PROJECT_NAME", "qbit-mining-bootstrap") - return [ - "docker", - "compose", - *env_file_args(), - "-f", - str(ROOT_DIR / "compose.yaml"), - "--project-name", - project_name, - "--profile", - "prism", - ] - - -def run_command(command: list[str], *, timeout: int = 10) -> subprocess.CompletedProcess[str]: - return subprocess.run( - command, - cwd=ROOT_DIR, - text=True, - stdout=subprocess.PIPE, - stderr=subprocess.PIPE, - timeout=timeout, - check=False, - ) - - -def compose_environment(reporter: Reporter) -> dict[str, str]: - if shutil.which("docker") is None: - reporter.fail("docker.cli", "docker is not installed", hint="Install Docker with the Compose plugin.") - return {} - command = [*compose_base_command(), "config", "--environment"] - try: - completed = run_command(command, timeout=30) - except subprocess.TimeoutExpired: - reporter.fail("compose.config", "docker compose config timed out") - return {} - if completed.returncode != 0: - detail = (completed.stderr or completed.stdout).strip().splitlines() - reporter.fail( - "compose.config", - "docker compose config failed", - hint=detail[-1] if detail else "Run docker compose config for the full error.", - ) - return {} - reporter.pass_("compose.config", "PRISM compose environment resolved") - resolved: dict[str, str] = {} - for line in completed.stdout.splitlines(): - if "=" not in line: - continue - key, value = line.split("=", 1) - resolved[key] = value - return resolved - - -def env_value(env: dict[str, str], name: str, default: str = "") -> str: - return env.get(name, default) - - -def is_true(value: str) -> bool: - return value.strip().lower() in BOOL_TRUE - - -def optional_bool_env(env: dict[str, str], name: str) -> bool | None: - value = env.get(name, "") - if value == "": - return None - normalized = value.strip().lower() - if normalized in BOOL_TRUE: - return True - if normalized in BOOL_FALSE: - return False - raise ValueError(f"{name} must be a true/false style value, got {value!r}") - - -def launch_readiness_checks_enabled(env: dict[str, str]) -> bool: - enabled = optional_bool_env(env, LAUNCH_READINESS_FLAG) - if enabled is None: - return True - chain = env_value(env, "QBIT_CHAIN", "regtest").strip().lower() or "regtest" - if chain != "mainnet": - raise ValueError(f"{LAUNCH_READINESS_FLAG} is valid only for QBIT_CHAIN=mainnet") - return enabled - - -def authorized_mainnet_prelaunch(env: dict[str, str]) -> bool: - try: - return ( - env_value(env, "QBIT_CHAIN", "regtest").strip().lower() == "mainnet" - and optional_bool_env(env, "QBIT_PRODUCTION") is True - and optional_bool_env(env, "QBIT_TOOLS_PRODUCTION") is True - and optional_bool_env(env, "CKPOOL_NON_TEST_READINESS_GATE") is False - and launch_readiness_checks_enabled(env) is False - ) - except ValueError: - return False - - -def launch_readiness_checks_disabled(env: dict[str, str]) -> bool: - # static_checks reports malformed or incomplete authorization. Live checks - # must remain strict instead of treating it as prelaunch authorization. - return authorized_mainnet_prelaunch(env) - - -def report_launch_dependent_failure( - env: dict[str, str], - reporter: Reporter, - name: str, - detail: str, - *, - hint: str | None = None, -) -> None: - if launch_readiness_checks_disabled(env): - reporter.warn( - name, - f"{detail}; tolerated only because {LAUNCH_READINESS_FLAG}=0 (prelaunch)", - hint=f"Set {LAUNCH_READINESS_FLAG}=1 at launch; this condition will then fail the self-check.", - ) - else: - reporter.fail(name, detail, hint=hint) - - -def normalize_qbit_chain_name(value: object) -> str: - normalized = str(value).strip().lower() - return QBIT_RPC_CHAIN_ALIASES.get(normalized, normalized) - - -def production_mode(env: dict[str, str]) -> bool: - return is_true(env_value(env, "QBIT_PRODUCTION", "0")) or is_true(env_value(env, "QBIT_TOOLS_PRODUCTION", "0")) - - -def prelaunch_tip_age_seconds(env: dict[str, str]) -> int | None: - if PRELAUNCH_TIP_AGE_NAME not in env: - return None - - value = env[PRELAUNCH_TIP_AGE_NAME] - if value == "": - raise ValueError(f"{PRELAUNCH_TIP_AGE_NAME} must not be empty") - if not value.isascii() or not value.isdigit(): - raise ValueError(f"{PRELAUNCH_TIP_AGE_NAME} must be a positive integer") - normalized = value.lstrip("0") or "0" - if normalized == "0": - raise ValueError(f"{PRELAUNCH_TIP_AGE_NAME} must be greater than zero") - maximum = str(MAX_SIGNED_INT64) - if len(normalized) > len(maximum) or ( - len(normalized) == len(maximum) and normalized > maximum - ): - raise ValueError( - f"{PRELAUNCH_TIP_AGE_NAME} exceeds qbitd's signed 64-bit integer range" - ) - seconds = int(normalized) - production_enabled = optional_bool_env(env, "QBIT_PRODUCTION") - if production_enabled is not True: - raise ValueError(f"{PRELAUNCH_TIP_AGE_NAME} is valid only with QBIT_PRODUCTION=1") - if env_value(env, "QBIT_CHAIN", "regtest") != "mainnet": - raise ValueError(f"{PRELAUNCH_TIP_AGE_NAME} is valid only with QBIT_CHAIN=mainnet") - return seconds - - -def parse_decimal(value: str) -> Decimal: - try: - parsed = Decimal(value) - except InvalidOperation as exc: - raise ValueError(f"{value!r} is not a decimal number") from exc - if not parsed.is_finite(): - raise ValueError(f"{value!r} is not a finite decimal number") - return parsed - - -def parse_host_port(value: str, *, default_host: str, default_port: int) -> tuple[str, int]: - if not value: - return default_host, default_port - if value.isdigit(): - return default_host, int(value) - host, separator, port = value.rpartition(":") - if not separator or not port.isdigit(): - raise ValueError(f"expected HOST:PORT or PORT, got {value!r}") - return host or default_host, int(port) - - -def static_checks(env: dict[str, str], reporter: Reporter) -> None: - qbit_chain = env_value(env, "QBIT_CHAIN", "regtest") - prod = qbit_chain == "mainnet" or production_mode(env) - qbit_chain_flag = env_value(env, "QBIT_CHAIN_FLAG", "-regtest") - try: - launch_checks_enabled = launch_readiness_checks_enabled(env) - except ValueError as exc: - reporter.fail(f"env.{LAUNCH_READINESS_FLAG}", str(exc)) - else: - configured_launch_flag = env_value(env, LAUNCH_READINESS_FLAG) - if not configured_launch_flag: - reporter.pass_( - "launch.readiness", - f"{LAUNCH_READINESS_FLAG} is unset; launch-dependent checks remain strict", - ) - elif launch_checks_enabled: - reporter.pass_("launch.readiness", f"{LAUNCH_READINESS_FLAG}=1; launch checks are strict") - elif not authorized_mainnet_prelaunch(env): - reporter.fail( - "launch.readiness", - f"{LAUNCH_READINESS_FLAG}=0 requires QBIT_CHAIN=mainnet, " - "QBIT_PRODUCTION=1, QBIT_TOOLS_PRODUCTION=1, and " - "CKPOOL_NON_TEST_READINESS_GATE=0", - ) - else: - reporter.warn( - "launch.readiness", - f"{LAUNCH_READINESS_FLAG}=0; only explicit launch-dependent conditions are relaxed", - hint=f"Set {LAUNCH_READINESS_FLAG}=1 at launch.", - ) - if prod and qbit_chain == "regtest": - reporter.fail("qbit.chain", "production mode cannot use regtest", hint="Set QBIT_CHAIN and QBIT_CHAIN_FLAG for the live network.") - else: - reporter.pass_("qbit.chain", f"QBIT_CHAIN={qbit_chain} QBIT_CHAIN_FLAG={qbit_chain_flag}") - expected_chain_flag = QBIT_CHAIN_FLAGS.get(qbit_chain) - if expected_chain_flag is not None and qbit_chain_flag != expected_chain_flag: - reporter.fail("qbit.chain_flag", f"QBIT_CHAIN={qbit_chain} requires QBIT_CHAIN_FLAG={expected_chain_flag}") - elif qbit_chain not in QBIT_CHAIN_FLAGS: - reporter.fail("qbit.chain_flag", f"unknown QBIT_CHAIN={qbit_chain}") - else: - reporter.pass_("qbit.chain_flag", "chain flag matches the configured qbit chain") - - try: - tip_age_seconds = prelaunch_tip_age_seconds(env) - except ValueError as exc: - reporter.fail(f"env.{PRELAUNCH_TIP_AGE_NAME}", str(exc)) - else: - if tip_age_seconds is None: - reporter.pass_( - f"env.{PRELAUNCH_TIP_AGE_NAME}", - "not configured; qbitd uses its normal tip-age policy", - ) - else: - reporter.pass_( - f"env.{PRELAUNCH_TIP_AGE_NAME}", - f"validated reviewed duration of {tip_age_seconds} seconds", - ) - - bitcoin_chain = env_value(env, "BITCOIN_CHAIN", "regtest") - bitcoin_chain_flag = env_value(env, "BITCOIN_CHAIN_FLAG", "-regtest") - expected_bitcoin_chain_flag = BITCOIN_CHAIN_FLAGS.get(bitcoin_chain) - if expected_bitcoin_chain_flag is None: - reporter.fail("bitcoin.chain_flag", f"unknown BITCOIN_CHAIN={bitcoin_chain}") - elif bitcoin_chain_flag != expected_bitcoin_chain_flag: - reporter.fail( - "bitcoin.chain_flag", - f"BITCOIN_CHAIN={bitcoin_chain} requires BITCOIN_CHAIN_FLAG={expected_bitcoin_chain_flag}", - ) - else: - reporter.pass_("bitcoin.chain_flag", "chain flag matches the configured Bitcoin chain") - - expected_genesis_hash = env_value(env, "QBIT_EXPECTED_GENESIS_HASH").strip() - if qbit_chain == "mainnet" and not expected_genesis_hash: - reporter.fail( - "qbit.genesis_config", - "QBIT_CHAIN=mainnet requires QBIT_EXPECTED_GENESIS_HASH", - hint="Pin the final release genesis hash before starting the pool.", - ) - elif expected_genesis_hash and ( - len(expected_genesis_hash) != 64 - or any(character not in "0123456789abcdefABCDEF" for character in expected_genesis_hash) - ): - reporter.fail("qbit.genesis_config", "QBIT_EXPECTED_GENESIS_HASH must be 64 hex characters") - elif expected_genesis_hash: - reporter.pass_("qbit.genesis_config", expected_genesis_hash.lower()) - else: - reporter.pass_("qbit.genesis_config", "not pinned for this test chain") - - if prod: - qbit_git_commit = env_value(env, "QBIT_GIT_COMMIT").strip() - if len(qbit_git_commit) != 40 or any( - character not in "0123456789abcdefABCDEF" for character in qbit_git_commit - ): - reporter.fail( - "qbit.source_pin", - "production requires QBIT_GIT_COMMIT as exactly 40 hex characters", - ) - else: - reporter.pass_("qbit.source_pin", qbit_git_commit.lower()) - - if prod: - for name in ("CKPOOL_GIT_REF", "CPUMINER_GIT_REF"): - source_ref = env_value(env, name).strip() - if len(source_ref) != 40 or any( - character not in "0123456789abcdefABCDEF" for character in source_ref - ): - reporter.fail( - f"source.{name}", - f"production requires {name} as exactly 40 hex characters", - ) - else: - reporter.pass_(f"source.{name}", source_ref.lower()) - - required_keys = ( - "PRISM_MANIFEST_SIGNING_SEED_HEX", - "PRISM_LEDGER_ATTESTATION_SIGNING_SEED_HEX", - "PRISM_LEDGER_WRITER_PUBLIC_KEY_HEX", - ) - for name in required_keys: - if env_value(env, name): - reporter.pass_(f"env.{name}", "configured") - else: - reporter.fail( - f"env.{name}", - "missing", - hint="Set real PRISM signing material before accepting miners.", - ) - - writer_defaults = { - "PRISM_LEDGER_WRITER_ID": "prism-coordinator", - "PRISM_LEDGER_WRITER_EPOCH": "1", - } - for name, default in writer_defaults.items(): - value = env_value(env, name, default) - if value: - reporter.pass_(f"env.{name}", value) - else: - reporter.fail(f"env.{name}", "missing") - - forbidden_flags = ( - "PRISM_ALLOW_MEMORY_LEDGER", - "PRISM_ALLOW_TEST_SIGNING_SEEDS", - "PRISM_ALLOW_BUNDLE_EMBEDDED_LEDGER_KEY", - "PRISM_ALLOW_FIXED_LEDGER_SESSION_TOKEN", - ) - for name in forbidden_flags: - value = env_value(env, name, "0") - if prod and is_true(value): - reporter.fail(f"env.{name}", "forbidden in production", hint=f"Set {name}=0.") - elif is_true(value): - reporter.warn(f"env.{name}", "test-only bypass enabled", hint="Keep this disabled for deploys.") - else: - reporter.pass_(f"env.{name}", "disabled") - - default_database_url = ( - "postgresql://" - f"{env_value(env, 'PRISM_POSTGRES_USER', 'qbit')}:" - f"{env_value(env, 'PRISM_POSTGRES_PASSWORD', 'change-this')}" - f"@prism-postgres:5432/{env_value(env, 'PRISM_POSTGRES_DB', 'qbit')}" - ) - database_url = env_value(env, "PRISM_DATABASE_URL", default_database_url) - psql_command = env_value(env, "PRISM_POSTGRES_PSQL_COMMAND") - if database_url or psql_command: - reporter.pass_("ledger.database", "Postgres ledger configuration is present") - elif prod: - reporter.fail("ledger.database", "production requires PRISM_DATABASE_URL or PRISM_POSTGRES_PSQL_COMMAND") - else: - reporter.warn("ledger.database", "no Postgres ledger configuration found") - postgres_password = env_value(env, "PRISM_POSTGRES_PASSWORD", "change-this") - if prod and postgres_password == "change-this": - reporter.fail("ledger.database_auth", "production cannot use the default PRISM_POSTGRES_PASSWORD") - elif postgres_password == "change-this": - reporter.warn("ledger.database_auth", "using default PRISM_POSTGRES_PASSWORD", hint="Set a real password before deploy.") - else: - reporter.pass_("ledger.database_auth", "Postgres password is non-default") - - audit_dir = env_value(env, "PRISM_AUDIT_DIR", "/var/lib/qbit-prism/audit") - evidence_path = env_value(env, "PRISM_EVIDENCE_PATH", "/var/lib/qbit-prism/audit/prism-live-evidence.json") - if audit_dir and evidence_path: - reporter.pass_("audit.paths", f"dir={audit_dir} evidence={evidence_path}") - else: - reporter.fail("audit.paths", "PRISM_AUDIT_DIR and PRISM_EVIDENCE_PATH are required") - - qbit_user = env_value(env, "QBIT_RPC_USER", "qbitrpc") - qbit_password = env_value(env, "QBIT_RPC_PASSWORD", "change-this") - if not qbit_user or not qbit_password: - reporter.fail("qbit.rpc_auth", "QBIT_RPC_USER and QBIT_RPC_PASSWORD are required") - elif prod and qbit_password == "change-this": - reporter.fail("qbit.rpc_auth", "production cannot use the default qbit RPC password") - elif qbit_password == "change-this": - reporter.warn("qbit.rpc_auth", "using default qbit RPC password", hint="Set a real password before deploy.") - else: - reporter.pass_("qbit.rpc_auth", "RPC credentials configured") - - try: - min_ready = int(env_value(env, "PRISM_MIN_READY_MINERS", "3")) - if min_ready <= 0: - raise ValueError - reporter.pass_("mining.min_ready", f"{min_ready} miners") - except ValueError: - reporter.fail("mining.min_ready", "PRISM_MIN_READY_MINERS must be a positive integer") - - try: - share_diff = parse_decimal(env_value(env, "PRISM_STRATUM_SHARE_DIFF", "0.000000001")) - if share_diff <= 0: - raise ValueError("share difficulty must be positive") - reporter.pass_("mining.share_diff", f"{share_diff}") - except ValueError as exc: - reporter.fail("mining.share_diff", str(exc)) - - # Match the coordinator's env_nonnegative_float(): the runtime's float - # syntax and range, finite and non-negative (fractional seconds included). - stale_grace = env_value(env, "PRISM_STRATUM_STALE_GRACE_SECONDS", "3").strip() - try: - stale_grace_value = float(stale_grace) - if not math.isfinite(stale_grace_value) or stale_grace_value < 0: - raise ValueError("stale grace must be a finite non-negative number") - reporter.pass_("mining.stale_grace", f"{stale_grace_value:g} seconds") - except ValueError: - reporter.fail("mining.stale_grace", "PRISM_STRATUM_STALE_GRACE_SECONDS must be a finite non-negative number") - - if prod and bitcoin_chain != "regtest": - qbit_miner_address = env_value(env, "QBIT_MINER_ADDRESS", "auto").strip() - bitcoin_miner_address = env_value(env, "BITCOIN_MINER_ADDRESS", "auto").strip() - if not qbit_miner_address or qbit_miner_address == "auto": - reporter.fail("auxpow.qbit_payout", "production AuxPoW requires an explicit QBIT_MINER_ADDRESS") - else: - reporter.pass_("auxpow.qbit_payout", "explicit payout configured") - if not bitcoin_miner_address or bitcoin_miner_address == "auto": - reporter.fail( - "auxpow.bitcoin_payout", - "production AuxPoW requires an explicit BITCOIN_MINER_ADDRESS", - ) - else: - reporter.pass_("auxpow.bitcoin_payout", "explicit payout configured") - - if is_true(env_value(env, "PRISM_STRATUM_VARDIFF", "1")): - try: - min_diff = parse_decimal(env_value(env, "PRISM_STRATUM_VARDIFF_MIN_DIFF", "0.000000001")) - max_diff = parse_decimal(env_value(env, "PRISM_STRATUM_VARDIFF_MAX_DIFF", "1024")) - start_diff = parse_decimal(env_value(env, "PRISM_STRATUM_VARDIFF_START_DIFF", str(min_diff))) - if min_diff <= 0 or max_diff <= 0 or start_diff <= 0: - raise ValueError("vardiff difficulty values must be positive") - if min_diff > max_diff: - raise ValueError("PRISM_STRATUM_VARDIFF_MIN_DIFF exceeds PRISM_STRATUM_VARDIFF_MAX_DIFF") - if start_diff > max_diff: - raise ValueError("PRISM_STRATUM_VARDIFF_START_DIFF exceeds PRISM_STRATUM_VARDIFF_MAX_DIFF") - reporter.pass_("mining.vardiff", f"enabled start={start_diff} range={min_diff}..{max_diff}") - except ValueError as exc: - reporter.fail("mining.vardiff", str(exc)) - else: - reporter.warn("mining.vardiff", "vardiff is disabled") - - if prod: - difficulty_names = ( - "PRISM_STRATUM_SHARE_DIFF", - "PRISM_STRATUM_VARDIFF_MIN_DIFF", - "PRISM_STRATUM_VARDIFF_START_DIFF", - "PRISM_STRATUM_VARDIFF_MAX_DIFF", - ) - try: - production_difficulties: dict[str, Decimal] = {} - for name in difficulty_names: - raw_value = env_value(env, name).strip() - if not raw_value: - raise ValueError(f"production requires an explicit {name}") - value = parse_decimal(raw_value) - if value <= 0: - raise ValueError(f"{name} must be positive") - if value == Decimal("0.000000001"): - raise ValueError(f"{name} cannot use the lab-only 1e-9 difficulty") - production_difficulties[name] = value - if ( - production_difficulties["PRISM_STRATUM_VARDIFF_MIN_DIFF"] - > production_difficulties["PRISM_STRATUM_VARDIFF_START_DIFF"] - ): - raise ValueError("production vardiff minimum exceeds its start difficulty") - if ( - production_difficulties["PRISM_STRATUM_VARDIFF_START_DIFF"] - > production_difficulties["PRISM_STRATUM_VARDIFF_MAX_DIFF"] - ): - raise ValueError("production vardiff start exceeds its maximum difficulty") - except ValueError as exc: - reporter.fail("mining.production_difficulty", str(exc)) - else: - reporter.pass_( - "mining.production_difficulty", - "reviewed share and bounded vardiff values configured", - ) - - if env_value(env, "PRISM_STRATUM_HIGHDIFF_PORT"): - try: - highdiff_port = int(env_value(env, "PRISM_STRATUM_HIGHDIFF_PORT")) - if not 0 < highdiff_port < 65536: - raise ValueError("PRISM_STRATUM_HIGHDIFF_PORT must be a valid TCP port") - highdiff_min = parse_decimal(env_value(env, "PRISM_STRATUM_HIGHDIFF_MIN_DIFF", "500000")) - highdiff_start = parse_decimal(env_value(env, "PRISM_STRATUM_HIGHDIFF_START_DIFF", "500000")) - highdiff_max = parse_decimal(env_value(env, "PRISM_STRATUM_HIGHDIFF_MAX_DIFF", "4294967296")) - if highdiff_min <= 0 or highdiff_start <= 0 or highdiff_max <= 0: - raise ValueError("high-diff difficulty values must be positive") - if highdiff_min > highdiff_start: - raise ValueError("PRISM_STRATUM_HIGHDIFF_MIN_DIFF exceeds PRISM_STRATUM_HIGHDIFF_START_DIFF") - if highdiff_start > highdiff_max: - raise ValueError("PRISM_STRATUM_HIGHDIFF_START_DIFF exceeds PRISM_STRATUM_HIGHDIFF_MAX_DIFF") - # Match the coordinator: an unset OR empty fixed difficulty tracks - # the start difficulty (compose resolves the default to ""). - highdiff_share_raw = env_value(env, "PRISM_STRATUM_HIGHDIFF_SHARE_DIFF", "").strip() - highdiff_share = parse_decimal(highdiff_share_raw) if highdiff_share_raw else highdiff_start - if highdiff_share < highdiff_min or highdiff_share > highdiff_max: - raise ValueError("PRISM_STRATUM_HIGHDIFF_SHARE_DIFF is outside the min/max bounds") - reporter.pass_( - "mining.highdiff", - f"enabled port={highdiff_port} start={highdiff_start} range={highdiff_min}..{highdiff_max}", - ) - except ValueError as exc: - reporter.fail("mining.highdiff", str(exc)) - else: - reporter.pass_("mining.highdiff", "disabled (single stratum listener)") - - pool_fee_enabled = is_true(env_value(env, "PRISM_POOL_FEE_ENABLED", "0")) - if pool_fee_enabled: - fee_bps = env_value(env, "PRISM_POOL_FEE_BPS") - fee_address = env_value(env, "PRISM_POOL_FEE_ADDRESS") - fee_program = env_value(env, "PRISM_POOL_FEE_P2MR_PROGRAM_HEX") - if not fee_bps: - reporter.fail("pool.fee", "PRISM_POOL_FEE_ENABLED=1 requires PRISM_POOL_FEE_BPS") - elif not fee_address and not fee_program: - reporter.fail("pool.fee", "pool fee requires PRISM_POOL_FEE_ADDRESS or PRISM_POOL_FEE_P2MR_PROGRAM_HEX") - else: - reporter.pass_("pool.fee", f"enabled bps={fee_bps}") - else: - reporter.pass_("pool.fee", "disabled") - - # No whitespace normalization: mirror the coordinator, which rejects any - # value outside the exact allowed strings. - output_policy = env_value(env, "PRISM_COINBASE_OUTPUT_POLICY", "canonical") or "canonical" - if output_policy not in {"canonical", "pool-fee-first"}: - reporter.fail( - "coinbase.output_policy", - "PRISM_COINBASE_OUTPUT_POLICY must be one of: canonical, pool-fee-first", - ) - elif output_policy == "pool-fee-first" and not pool_fee_enabled: - reporter.fail( - "coinbase.output_policy", - "PRISM_COINBASE_OUTPUT_POLICY=pool-fee-first requires PRISM_POOL_FEE_ENABLED=1", - ) - else: - reporter.pass_("coinbase.output_policy", output_policy) - - if is_true(env_value(env, "PRISM_CTV_SETTLEMENT_ENABLED", "0")): - market_rate = env_value( - env, - "PRISM_CTV_FANOUT_FEE_MARKET_RATE_BITS_PER_1000_WEIGHT", - ).strip() - premium = env_value(env, "PRISM_CTV_FANOUT_FEE_PREMIUM_BPS", "12000").strip() - try: - premium_value = int(premium) - if premium_value <= 0: - raise ValueError - except ValueError: - reporter.fail("ctv.fee_premium", "PRISM_CTV_FANOUT_FEE_PREMIUM_BPS must be a positive integer") - else: - reporter.pass_("ctv.fee_premium", f"{premium_value} bps") - - if market_rate: - try: - market_rate_value = int(market_rate) - if market_rate_value <= 0: - raise ValueError - except ValueError: - reporter.fail( - "ctv.fee_source", - "PRISM_CTV_FANOUT_FEE_MARKET_RATE_BITS_PER_1000_WEIGHT must be a positive integer", - ) - else: - reporter.pass_("ctv.fee_source", f"explicit rate={market_rate_value} bits/1000 weight") - else: - if qbit_chain == "mainnet": - reporter.fail( - "ctv.fee_source", - "mainnet requires PRISM_CTV_FANOUT_FEE_MARKET_RATE_BITS_PER_1000_WEIGHT", - ) - else: - try: - estimate_target = int(env_value(env, "PRISM_CTV_FANOUT_FEE_ESTIMATE_TARGET_BLOCKS", "2")) - if estimate_target <= 0: - raise ValueError - except ValueError: - reporter.fail( - "ctv.fee_source", - "PRISM_CTV_FANOUT_FEE_ESTIMATE_TARGET_BLOCKS must be a positive integer", - ) - else: - reporter.warn( - "ctv.fee_source", - f"estimatesmartfee target={estimate_target}; live preflight required", - hint=( - "Fresh chains and chains with empty blocks have no empirical fee history. " - "Configure a positive explicit fanout fee rate until estimatesmartfee succeeds." - ), - ) - else: - reporter.pass_("ctv.fee_source", "CTV settlement disabled") - - -def highdiff_probe_target(env: dict[str, str]) -> tuple[str, int] | None: - """Host/port for probing the published high-diff listener. - - Only the host publish mapping counts: probing the container listen port - would pass even when miners cannot reach the published port. Returns None - when nothing usable is published (unset, empty, or the ephemeral - loopback-port-0 default used while the listener is disabled).""" - value = env_value(env, "PRISM_STRATUM_HIGHDIFF_PORT_HOST", "").strip() - if not value: - return None - host, port = parse_host_port(value, default_host="127.0.0.1", default_port=4334) - if port <= 0: - return None - return host, port - - -def tcp_connect_check(name: str, host: str, port: int, reporter: Reporter) -> None: - try: - with socket.create_connection((host, port), timeout=3): - pass - except OSError as exc: - reporter.fail(name, f"cannot connect to {host}:{port}: {exc}", hint="Start the PRISM profile or check host port mapping.") - else: - reporter.pass_(name, f"reachable at {host}:{port}") - - -def stratum_first_advertised_difficulty( - host: str, - port: int, - *, - username: str, - password: str = "x", - timeout: float = 10.0, -) -> Decimal: - """Subscribe and authorize like a rig, returning the first - mining.set_difficulty the pool advertises on this connection. - - This mirrors the marketplace verification handshake: NiceHash-style - probes judge the first advertised difficulty, so TCP reachability alone - is not evidence that a listener honors its floor.""" - deadline = time.monotonic() + timeout - with socket.create_connection((host, port), timeout=timeout) as sock: - sock.settimeout(timeout) - for request in ( - {"id": 1, "method": "mining.subscribe", "params": ["prism-self-check/1"]}, - {"id": 2, "method": "mining.authorize", "params": [username, password]}, - ): - sock.sendall((json.dumps(request) + "\n").encode()) - buffer = b"" - while time.monotonic() < deadline: - try: - chunk = sock.recv(4096) - except socket.timeout as exc: - raise MissingHighdiffNotificationError( - "timed out waiting for mining.set_difficulty" - ) from exc - if not chunk: - raise MissingHighdiffNotificationError( - "connection closed before mining.set_difficulty" - ) - buffer += chunk - while b"\n" in buffer: - line, _, buffer = buffer.partition(b"\n") - if not line.strip(): - continue - try: - message = json.loads(line) - except json.JSONDecodeError: - continue - if not isinstance(message, dict): - continue - if message.get("id") == 2 and (message.get("error") or message.get("result") is False): - raise RuntimeError(f"mining.authorize rejected: {message.get('error')}") - if message.get("method") == "mining.set_difficulty": - params = message.get("params") - if not isinstance(params, list) or not params: - raise RuntimeError("mining.set_difficulty carried no difficulty") - return parse_decimal(str(params[0])) - raise MissingHighdiffNotificationError("timed out waiting for mining.set_difficulty") - - -def check_highdiff_advertised_floor(env: dict[str, str], host: str, port: int, reporter: Reporter) -> None: - floor_raw = env_value(env, "PRISM_STRATUM_HIGHDIFF_MIN_DIFF", "").strip() or "500000" - try: - floor = parse_decimal(floor_raw) - except ValueError as exc: - reporter.fail("stratum.highdiff_floor", f"PRISM_STRATUM_HIGHDIFF_MIN_DIFF: {exc}") - return - username = env_value(env, "PRISM_USERNAME_FALLBACK_ADDRESS", "").strip() or "prism-self-check" - try: - advertised = stratum_first_advertised_difficulty(host, port, username=username) - except MissingHighdiffNotificationError as exc: - report_launch_dependent_failure( - env, - reporter, - "stratum.highdiff_floor", - f"stratum handshake with {host}:{port} did not advertise a difficulty: {exc}", - hint="Wait for a usable block template/job and check coordinator logs.", - ) - return - except (OSError, RuntimeError, ValueError) as exc: - reporter.fail( - "stratum.highdiff_floor", - f"stratum handshake with {host}:{port} failed: {exc}", - hint="Marketplace verification needs subscribe/authorize to answer with mining.set_difficulty; check coordinator logs and PRISM_USERNAME_FALLBACK_ADDRESS.", - ) - return - if advertised < floor: - reporter.fail( - "stratum.highdiff_floor", - f"first mining.set_difficulty advertises {advertised}, below the {floor} floor", - hint="Rental marketplaces judge the first advertised difficulty; the high-diff listener must clamp stamped jobs to its floor.", - ) - else: - reporter.pass_( - "stratum.highdiff_floor", - f"first mining.set_difficulty advertises {advertised} (floor {floor})", - ) - - -def qbit_rpc_call(env: dict[str, str], method: str, params: list[object] | None = None) -> object: - host, port = parse_host_port(env_value(env, "QBIT_RPC_PORT_HOST", "127.0.0.1:18452"), default_host="127.0.0.1", default_port=18452) - user = env_value(env, "QBIT_RPC_USER", "qbitrpc") - password = env_value(env, "QBIT_RPC_PASSWORD", "change-this") - auth = base64.b64encode(f"{user}:{password}".encode()).decode() - request = urllib.request.Request( - f"http://{host}:{port}/", - data=json.dumps( - { - "jsonrpc": "1.0", - "id": "prism-self-check", - "method": method, - "params": params or [], - } - ).encode(), - headers={"Authorization": f"Basic {auth}", "Content-Type": "application/json"}, - ) - with urllib.request.urlopen(request, timeout=5) as response: - payload = json.loads(response.read()) - if payload.get("error"): - raise RuntimeError(payload["error"]) - return payload.get("result") - - -def qbit_live_checks(env: dict[str, str], reporter: Reporter) -> None: - try: - blockchain_info = qbit_rpc_call(env, "getblockchaininfo") - except (OSError, urllib.error.URLError, RuntimeError, ValueError) as exc: - reporter.fail("qbit.rpc", f"RPC probe failed: {exc}", hint="Check qbitd, QBIT_RPC_PORT_HOST, and RPC credentials.") - return - if not isinstance(blockchain_info, dict): - reporter.fail("qbit.rpc", "getblockchaininfo returned a non-object response") - return - expected_chain = env_value(env, "QBIT_CHAIN", "regtest") - actual_chain = str(blockchain_info.get("chain", "")) - if normalize_qbit_chain_name(actual_chain) != normalize_qbit_chain_name(expected_chain): - reporter.fail("qbit.rpc_chain", f"qbitd reports chain={actual_chain}, expected {expected_chain}") - else: - reporter.pass_("qbit.rpc_chain", f"qbitd reports {actual_chain}") - public_chain = expected_chain in PUBLIC_QBIT_CHAINS - if ( - blockchain_info.get("initialblockdownload") is not False - if public_chain - else bool(blockchain_info.get("initialblockdownload")) - ): - report_launch_dependent_failure( - env, - reporter, - "qbit.ibd", - "qbitd is still in initial block download or did not report readiness", - ) - else: - reporter.pass_("qbit.ibd", "qbitd is not in initial block download") - if public_chain: - try: - blocks = int(blockchain_info["blocks"]) - headers = int(blockchain_info["headers"]) - if blocks < 0 or headers < 0: - raise ValueError("negative height") - except (KeyError, TypeError, ValueError) as exc: - reporter.fail("qbit.headers", f"qbitd did not report valid blocks and headers: {exc}") - else: - if blocks != headers: - reporter.fail("qbit.headers", f"qbitd is not caught up: blocks={blocks} headers={headers}") - else: - reporter.pass_("qbit.headers", f"blocks={blocks} headers={headers}") - - try: - template_rules = ["segwit"] - if expected_chain == "signet": - template_rules.append("signet") - template = qbit_rpc_call(env, "getblocktemplate", [{"rules": template_rules}]) - if not isinstance(template, dict) or not template.get("previousblockhash"): - raise ValueError("missing previousblockhash") - template_time = int(template["curtime"]) - max_age = int(env_value(env, "PRISM_TEMPLATE_MAX_AGE_SECONDS", "120")) - if max_age < 0: - raise ValueError("PRISM_TEMPLATE_MAX_AGE_SECONDS must be non-negative") - template_age = int(time.time()) - template_time - if template_age > max_age: - raise ValueError(f"template age {template_age}s exceeds {max_age}s") - except (OSError, urllib.error.URLError, RuntimeError, ValueError, KeyError) as exc: - reporter.fail("qbit.template", f"public-chain template preflight failed: {exc}") - else: - reporter.pass_("qbit.template", f"fresh template age={template_age}s") - - expected_genesis_hash = env_value(env, "QBIT_EXPECTED_GENESIS_HASH").strip().lower() - if expected_genesis_hash: - try: - actual_genesis_hash = str(qbit_rpc_call(env, "getblockhash", [0])).lower() - except (OSError, urllib.error.URLError, RuntimeError, ValueError) as exc: - reporter.fail("qbit.genesis", f"could not read genesis hash: {exc}") - else: - if actual_genesis_hash != expected_genesis_hash: - reporter.fail( - "qbit.genesis", - f"qbitd reports genesis={actual_genesis_hash}, expected {expected_genesis_hash}", - ) - else: - reporter.pass_("qbit.genesis", actual_genesis_hash) - - if ( - is_true(env_value(env, "PRISM_CTV_SETTLEMENT_ENABLED", "0")) - and not env_value( - env, - "PRISM_CTV_FANOUT_FEE_MARKET_RATE_BITS_PER_1000_WEIGHT", - ).strip() - ): - try: - estimate_target = int(env_value(env, "PRISM_CTV_FANOUT_FEE_ESTIMATE_TARGET_BLOCKS", "2")) - estimate = qbit_rpc_call(env, "estimatesmartfee", [estimate_target]) - if not isinstance(estimate, dict): - raise ValueError("returned a non-object response") - if estimate.get("errors"): - raise ValueError(f"returned errors: {estimate['errors']}") - fee_rate = parse_decimal(str(estimate.get("feerate", ""))) - if not fee_rate.is_finite() or fee_rate <= 0: - raise ValueError(f"returned invalid feerate: {estimate.get('feerate')!r}") - except (OSError, urllib.error.URLError, RuntimeError, ValueError) as exc: - reporter.fail( - "ctv.fee_estimator", - f"estimatesmartfee preflight failed: {exc}", - hint=( - "Fresh chains and empty blocks do not build fee-estimation history. Set " - "PRISM_CTV_FANOUT_FEE_MARKET_RATE_BITS_PER_1000_WEIGHT to a positive " - "operator-reviewed rate." - ), - ) - else: - reporter.pass_("ctv.fee_estimator", f"feerate={fee_rate} target={estimate_target}") - - configured_ctv_rate = env_value( - env, - "PRISM_CTV_FANOUT_FEE_MARKET_RATE_BITS_PER_1000_WEIGHT", - ).strip() - if is_true(env_value(env, "PRISM_CTV_SETTLEMENT_ENABLED", "0")) and configured_ctv_rate: - try: - mempool_info = qbit_rpc_call(env, "getmempoolinfo") - if not isinstance(mempool_info, dict): - raise ValueError("returned a non-object response") - relay_rates = [] - for name in ("minrelaytxfee", "mempoolminfee"): - if mempool_info.get(name) is None: - continue - rate = parse_decimal(str(mempool_info[name])) - if not rate.is_finite() or rate <= 0: - raise ValueError(f"{name} is not positive") - relay_rates.append( - int((rate * Decimal(100_000_000)).to_integral_value(rounding=ROUND_CEILING)) - ) - if not relay_rates: - raise ValueError("relay fee floor was not reported") - configured = int(configured_ctv_rate) - required = max(relay_rates) - if configured < required: - raise ValueError( - f"configured={configured} required={required} bits/1000 weight" - ) - except (OSError, urllib.error.URLError, RuntimeError, ValueError) as exc: - reporter.fail("ctv.relay_floor", f"CTV fee rate is below or cannot verify relay floor: {exc}") - else: - reporter.pass_("ctv.relay_floor", f"configured={configured} required={required}") - - try: - network_info = qbit_rpc_call(env, "getnetworkinfo") - peers = int(network_info.get("connections", 0)) if isinstance(network_info, dict) else 0 - except (OSError, urllib.error.URLError, RuntimeError, ValueError) as exc: - if expected_chain in PUBLIC_QBIT_CHAINS: - reporter.fail("qbit.peers", f"could not verify public-chain peer count: {exc}") - else: - reporter.warn("qbit.peers", f"could not read peer count: {exc}") - return - if expected_chain in PUBLIC_QBIT_CHAINS and peers <= 0: - reporter.fail("qbit.peers", "public-chain qbitd has no peers") - else: - reporter.pass_("qbit.peers", f"{peers} peers") - - -def compose_exec(service: str, code: str, *, timeout: int = 10) -> subprocess.CompletedProcess[str]: - return run_command([*compose_base_command(), "exec", "-T", service, "python3", "-c", code], timeout=timeout) - - -def check_ready_miner_threshold(payload: dict[str, object], env: dict[str, str], reporter: Reporter) -> None: - try: - ready_miner_count = int(payload.get("ready_miner_count", -1)) - except (TypeError, ValueError): - reporter.fail("coordinator.ready_miners", f"invalid ready_miner_count in /healthz payload: {payload.get('ready_miner_count')!r}") - return - try: - min_ready_miners = int(env_value(env, "PRISM_MIN_READY_MINERS", "3")) - except ValueError: - reporter.fail("coordinator.ready_miners", "PRISM_MIN_READY_MINERS must be an integer") - return - if ready_miner_count < min_ready_miners: - report_launch_dependent_failure( - env, - reporter, - "coordinator.ready_miners", - f"{ready_miner_count}/{min_ready_miners} miners ready", - hint="Connect miners and wait for accepted PRISM shares before treating the pool as block-ready.", - ) - else: - reporter.pass_("coordinator.ready_miners", f"{ready_miner_count}/{min_ready_miners} miners ready") - - -def coordinator_live_checks(env: dict[str, str], reporter: Reporter) -> None: - try: - stratum_host, stratum_port = parse_host_port( - env_value(env, "PRISM_STRATUM_PORT_HOST", "3340"), - default_host="127.0.0.1", - default_port=3340, - ) - except ValueError as exc: - reporter.fail("stratum.port", str(exc)) - else: - tcp_connect_check("stratum.tcp", stratum_host, stratum_port, reporter) - - if env_value(env, "PRISM_STRATUM_HIGHDIFF_PORT"): - try: - highdiff_target = highdiff_probe_target(env) - except ValueError as exc: - reporter.fail("stratum.highdiff_port", str(exc)) - else: - if highdiff_target is None: - reporter.fail( - "stratum.highdiff_port", - "PRISM_STRATUM_HIGHDIFF_PORT is set but PRISM_STRATUM_HIGHDIFF_PORT_HOST does not publish a host port", - hint="Set PRISM_STRATUM_HIGHDIFF_PORT_HOST (e.g. 4334) so miners can reach the high-diff listener.", - ) - else: - tcp_connect_check("stratum.highdiff_tcp", highdiff_target[0], highdiff_target[1], reporter) - check_highdiff_advertised_floor(env, highdiff_target[0], highdiff_target[1], reporter) - - health_code = r""" -import json -import os -import urllib.request -host = os.environ.get("PRISM_AUDIT_BIND", "127.0.0.1") or "127.0.0.1" -if host in ("0.0.0.0", "::"): - host = "127.0.0.1" -port = int(os.environ.get("PRISM_AUDIT_PORT", "0") or "0") -if port <= 0: - raise SystemExit("PRISM_AUDIT_PORT is disabled") -with urllib.request.urlopen(f"http://{host}:{port}/healthz", timeout=3) as response: - print(response.read().decode()) -""" - completed = compose_exec("prism-coordinator", health_code, timeout=10) - if completed.returncode != 0: - detail = (completed.stderr or completed.stdout).strip().splitlines() - reporter.fail( - "coordinator.healthz", - "could not read /healthz inside prism-coordinator", - hint=detail[-1] if detail else "Start prism-coordinator.", - ) - else: - try: - payload = json.loads(completed.stdout) - except json.JSONDecodeError as exc: - reporter.fail("coordinator.healthz", f"/healthz returned invalid JSON: {exc}") - else: - ready = payload.get("ok") is True - ready_miner_count = payload.get("ready_miner_count", "?") - accepted_share_count = payload.get("accepted_share_count", "?") - if ready: - reporter.pass_( - "coordinator.healthz", - f"ok, ready_miner_count={ready_miner_count}, accepted_share_count={accepted_share_count}", - ) - check_ready_miner_threshold(payload, env, reporter) - else: - reporter.fail("coordinator.healthz", f"unhealthy payload: {payload}") - - audit_code = r""" -import os -import pathlib -import tempfile -audit_dir = pathlib.Path(os.environ.get("PRISM_AUDIT_DIR", "/var/lib/qbit-prism/audit")) -audit_dir.mkdir(parents=True, exist_ok=True) -with tempfile.NamedTemporaryFile(prefix=".self-check-", dir=audit_dir, delete=True) as handle: - handle.write(b"ok") -""" - completed = compose_exec("prism-coordinator", audit_code, timeout=10) - if completed.returncode == 0: - reporter.pass_("audit.writable", "PRISM_AUDIT_DIR is writable inside prism-coordinator") - else: - detail = (completed.stderr or completed.stdout).strip().splitlines() - reporter.fail("audit.writable", "PRISM_AUDIT_DIR is not writable", hint=detail[-1] if detail else None) - - -def postgres_live_check(env: dict[str, str], reporter: Reporter) -> None: - user = env_value(env, "PRISM_POSTGRES_USER", "qbit") - database = env_value(env, "PRISM_POSTGRES_DB", "qbit") - completed = run_command( - [*compose_base_command(), "exec", "-T", "prism-postgres", "pg_isready", "-U", user, "-d", database], - timeout=10, - ) - if completed.returncode == 0: - reporter.pass_("postgres.ready", completed.stdout.strip() or "pg_isready passed") - else: - detail = (completed.stderr or completed.stdout).strip().splitlines() - reporter.fail("postgres.ready", "Postgres is not ready", hint=detail[-1] if detail else "Start prism-postgres.") - - -def live_checks(env: dict[str, str], reporter: Reporter) -> None: - qbit_live_checks(env, reporter) - postgres_live_check(env, reporter) - coordinator_live_checks(env, reporter) - - -def parse_args(argv: list[str]) -> argparse.Namespace: - parser = argparse.ArgumentParser(description=__doc__) - parser.add_argument( - "--skip-live", - action="store_true", - help="only check static PRISM configuration; skip qbitd, Postgres, Stratum, and coordinator probes", - ) - return parser.parse_args(argv) - - -def main(argv: list[str] | None = None) -> int: - args = parse_args(argv or sys.argv[1:]) - reporter = Reporter() - env = compose_environment(reporter) - if env: - static_checks(env, reporter) - if args.skip_live: - reporter.warn("live.probes", "skipped by --skip-live") - else: - live_checks(env, reporter) - reporter.emit() - return 1 if reporter.failed else 0 - - -if __name__ == "__main__": - raise SystemExit(main()) diff --git a/scripts/prism_capacity_evidence.py b/scripts/prism_capacity_evidence.py deleted file mode 100644 index d8c2150c1..000000000 --- a/scripts/prism_capacity_evidence.py +++ /dev/null @@ -1,108 +0,0 @@ -#!/usr/bin/env python3 -"""Validate a PRISM Stratum-to-Postgres capacity qualification artifact.""" - -from __future__ import annotations - -import argparse -import sys -from pathlib import Path - - -ROOT_DIR = Path(__file__).resolve().parents[1] -if str(ROOT_DIR) not in sys.path: - sys.path.insert(0, str(ROOT_DIR)) - -from lab.prism.capacity_evidence import ( # noqa: E402 - CONFIGURATION_KEYS, - DEFAULT_MAX_AGE_SECONDS, - CapacityEvidenceSummary, - EvidenceError, - load_capacity_evidence, - validate_capacity_evidence, -) - - -def _name_values(values: list[str], option: str) -> dict[str, str]: - expected: dict[str, str] = {} - for value in values: - key, separator, configured_value = value.partition("=") - if not separator or not key or not configured_value: - raise EvidenceError(f"{option} must use NAME=VALUE, got {value!r}") - expected[key] = configured_value - return expected - - -def _expected_configuration(values: list[str]) -> dict[str, str]: - expected = _name_values(values, "--expect") - unknown = sorted(set(expected) - set(CONFIGURATION_KEYS)) - if unknown: - raise EvidenceError(f"--expect contains unknown configuration keys: {', '.join(unknown)}") - return expected - - -def main(argv: list[str] | None = None) -> int: - parser = argparse.ArgumentParser(description=__doc__) - parser.add_argument("evidence_file", type=Path) - parser.add_argument( - "--expect", - action="append", - default=[], - metavar="NAME=VALUE", - help="bind evidence to one deployment configuration value", - ) - parser.add_argument("--expect-coordinator-revision") - parser.add_argument("--expect-coordinator-image-digest") - parser.add_argument("--expect-postgres-server-version") - parser.add_argument("--expect-database-profile-sha256") - parser.add_argument("--forecast-peak-shares-per-second") - parser.add_argument("--ack-p99-limit-milliseconds") - parser.add_argument( - "--max-age-seconds", - type=int, - default=DEFAULT_MAX_AGE_SECONDS, - help="maximum qualification-artifact age (default: 86400)", - ) - parser.add_argument( - "--allow-example-evidence-for-tests", - action="store_true", - help="accept committed example artifacts; never use for a deployment gate", - ) - args = parser.parse_args(argv) - - expected_subject_values = { - "coordinator_revision": args.expect_coordinator_revision, - "coordinator_image_digest": args.expect_coordinator_image_digest, - "postgres_server_version": args.expect_postgres_server_version, - "database_profile_sha256": args.expect_database_profile_sha256, - } - expected_subject = ( - {key: value for key, value in expected_subject_values.items() if value is not None} - or None - ) - try: - expected_configuration = _expected_configuration(args.expect) if args.expect else None - summary = load_capacity_evidence( - args.evidence_file, - expected_configuration=expected_configuration, - expected_subject=expected_subject, - expected_forecast_peak_shares_per_second=args.forecast_peak_shares_per_second, - expected_ack_p99_limit_milliseconds=args.ack_p99_limit_milliseconds, - max_age_seconds=args.max_age_seconds, - allow_example=args.allow_example_evidence_for_tests, - ) - except EvidenceError as exc: - print(f"capacity evidence invalid: {exc}", file=sys.stderr) - return 1 - print( - "capacity evidence valid: " - f"rate={summary.measured_shares_per_second} shares/s " - f"capacity={summary.capacity_multiple}x " - f"ACK p50={summary.ack_p50_milliseconds}ms " - f"p99={summary.ack_p99_milliseconds}ms " - f"committed={summary.acknowledged_shares}" - ) - return 0 - - -if __name__ == "__main__": - raise SystemExit(main()) diff --git a/scripts/prism_ha_qualification.py b/scripts/prism_ha_qualification.py new file mode 100644 index 000000000..978d48944 --- /dev/null +++ b/scripts/prism_ha_qualification.py @@ -0,0 +1,233 @@ +#!/usr/bin/env python3 +"""Bounded #281 preparation and disposable functional exercise (never qualification).""" + +import argparse +import hashlib +import json +import os +from pathlib import Path +import shutil +import signal +import subprocess +import tempfile + +ROOT = Path(__file__).resolve().parents[1] +FRONTENDS = ("prism-coordinator", "prism-coordinator-2") + + +def fixture_environment(): + # Compose must not read a real .env or inherit credential-bearing overrides. + return { + "PATH": os.environ.get("PATH", "/usr/bin:/bin"), + "HOME": os.environ.get("HOME", "/tmp"), + "PRISM_HA_INSTANCE_ID_1": "qual-east-281", + "PRISM_HA_INSTANCE_ID_2": "qual-west-281", + "PRISM_HA_RPC_HOST_1": "node-east.invalid", + "PRISM_HA_RPC_HOST_2": "node-west.invalid", + "PRISM_HA_RPC_URL_1": "http://node-east.invalid:19452/", + "PRISM_HA_RPC_URL_2": "http://node-west.invalid:19452/", + "PRISM_HA_AUDIT_BIND": "0.0.0.0", + "PRISM_HA_AUDIT_PORT": "18441", + "PRISM_HA_STRATUM_PORT_HOST_1": "127.0.0.1:18440", + "PRISM_HA_STRATUM_PORT_HOST_2": "127.0.0.1:18443", + "PRISM_HA_HEALTH_PORT_HOST_1": "127.0.0.1:18441", + "PRISM_HA_HEALTH_PORT_HOST_2": "127.0.0.1:18444", + "PRISM_HA_HIGHDIFF_PORT_HOST_1": "127.0.0.1:18442", + "PRISM_HA_HIGHDIFF_PORT_HOST_2": "127.0.0.1:18445", + "PRISM_STRATUM_HIGHDIFF_PORT": "18446", + "PRISM_DATABASE_URL": "postgresql://fixture:fixture@writer.invalid:5432/fixture", + "PRISM_PUBLIC_DATABASE_URL": "postgresql://fixture:fixture@public-reader.invalid:5432/fixture", + "PRISM_PUBLIC_REPLICA_MODE": "require", + } + + +def check_render(document, expected): + """Return only allowlisted nonsecret observations; never echo resolved config.""" + services = document["services"] + result = {} + for index, name in enumerate(FRONTENDS, 1): + service = services[name] + env = service["environment"] + for consumer, input_name in ( + ("PRISM_INSTANCE_ID", f"PRISM_HA_INSTANCE_ID_{index}"), + ("QBIT_RPC_URL", f"PRISM_HA_RPC_URL_{index}"), + ("PRISM_AUDIT_PORT", "PRISM_HA_AUDIT_PORT"), + ("PRISM_AUDIT_BIND", "PRISM_HA_AUDIT_BIND"), + ("PRISM_DATABASE_URL", "PRISM_DATABASE_URL"), + ("PRISM_STRATUM_HIGHDIFF_PORT", "PRISM_STRATUM_HIGHDIFF_PORT"), + ): + if env[consumer] != expected[input_name]: + raise ValueError(f"{name}: effective {consumer} differs from fixture") + ports = service["ports"] + expected_ports = { + ("127.0.0.1", int(env["PRISM_STRATUM_PORT"]), str(18440 if index == 1 else 18443)), + ("127.0.0.1", 18441, str(18441 if index == 1 else 18444)), + ("127.0.0.1", 18446, str(18442 if index == 1 else 18445)), + } + observed_ports = {(p.get("host_ip"), p["target"], str(p["published"])) for p in ports} + if expected_ports != observed_ports: + raise ValueError(f"{name}: expected loopback ports did not reach render") + result[name] = {"instance_id": env["PRISM_INSTANCE_ID"], "fixture_rpc_matches": True, + "fixture_writer_matches": True, "loopback_ports_match": True} + return result + + +def render(): + env = fixture_environment() + process = subprocess.run([ + "docker", "compose", "--env-file", os.devnull, "-f", "compose.yaml", + "-f", "compose.prism-ha.yaml", "--profile", "prism", "config", "--format", "json", + ], cwd=ROOT, env=env, capture_output=True, text=True, timeout=30) + if process.returncode: + raise RuntimeError("Compose render failed; resolved configuration is deliberately withheld") + return {"render": check_render(json.loads(process.stdout), env), + "runtime_startup": "not executed", "live_acceptance": "not established"} + + +def cleanup_owned(root, pg_bin): + """The wrapper's newly allocated, private parent is the ownership boundary.""" + entries = [] + for cluster_root in sorted(root.iterdir()): + if cluster_root.is_symlink() or not cluster_root.is_dir() or not cluster_root.name.startswith("prism-load-"): + raise RuntimeError("unexpected entry in owned resource directory; retained for inspection") + for name in ("standby", "primary"): + data = cluster_root / name + if data.is_symlink(): + raise RuntimeError("owned data path became a symlink; refusing cleanup") + if not data.exists(): + continue + pg = [str(pg_bin / "pg_ctl"), "-D", str(data)] + # initdb may fail before PG_VERSION; no postmaster could have started. + if not (data / "PG_VERSION").exists(): + if (data / "postmaster.pid").exists(): + raise RuntimeError("partial cluster has an unexplained postmaster; retained") + entries.append({"data": str(data), "state": "initialization incomplete, no postmaster"}) + continue + status = subprocess.run(pg + ["status"], capture_output=True, timeout=20).returncode + if status == 0: + subprocess.run(pg + ["-m", "immediate", "-w", "-t", "15", "stop"], + capture_output=True, timeout=20, check=True) + status = subprocess.run(pg + ["status"], capture_output=True, timeout=20).returncode + if status != 3 or (data / "postmaster.pid").exists(): + raise RuntimeError("owned PostgreSQL exit is unverified; retained data directory") + entries.append({"data": str(data), "state": "stopped"}) + shutil.rmtree(root) + return {"remaining_clusters": entries, "owned_parent_removed": not root.exists()} + + +def run(args): + for binary in (args.example_bin, args.runtime_tests, args.ack_tests, args.pg_bin_dir / "pg_ctl"): + if not binary.is_file() or not os.access(binary, os.X_OK): + raise ValueError("an explicitly supplied executable is unavailable") + if args.out.exists(): + raise ValueError("output directory must be new") + executable_hashes = {} + for name, binary in (("example", args.example_bin), ("runtime_tests", args.runtime_tests), ("ack_tests", args.ack_tests)): + digest = hashlib.sha256() + with binary.open("rb") as handle: + for chunk in iter(lambda: handle.read(1024 * 1024), b""): + digest.update(chunk) + executable_hashes[name] = digest.hexdigest() + head = subprocess.run(["git", "rev-parse", "HEAD"], cwd=ROOT, + capture_output=True, text=True, check=True).stdout.strip() + dirty = bool(subprocess.run(["git", "status", "--porcelain"], cwd=ROOT, + capture_output=True, text=True, check=True).stdout) + args.out.mkdir() + # Short paths keep the managed driver's Unix socket below PG's path limit. + root = Path(tempfile.mkdtemp(prefix="b281-", dir="/tmp")) + command = [str(args.example_bin), "--pg-bin-dir", str(args.pg_bin_dir), + "--runtime-tests", str(args.runtime_tests), "--ack-tests", str(args.ack_tests), + "--out", str(args.out / "functional")] + env = {"PATH": os.environ.get("PATH", "/usr/bin:/bin"), "TMPDIR": str(root), "LC_ALL": "C"} + process = None + status = None + failure = None + cleanup = None + census = { + "owned_parent": str(root), "process_pid": None, "process_exit": None, + "containers_started": [], "external_databases_used": [], + "checkout_at_invocation": {"head": head, "dirty": dirty}, + "executable_sha256": executable_hashes, + "provenance_note": "hashes identify executed binaries; checkout SHA alone does not certify build provenance", + "cleanup": {"owned_parent_removed": False, "state": "not yet verified"}, "failure": None, + } + try: + (args.out / "resource-census.json").write_text(json.dumps(census, indent=2) + "\n") + process = subprocess.Popen(command, env=env, stdout=subprocess.PIPE, stderr=subprocess.PIPE, + text=True, start_new_session=True) + census["process_pid"] = process.pid + (args.out / "resource-census.json").write_text(json.dumps(census, indent=2) + "\n") + try: + stdout, stderr = process.communicate(timeout=480) + except (subprocess.TimeoutExpired, KeyboardInterrupt): + # This process group was created by this invocation; never select by name. + os.killpg(process.pid, signal.SIGTERM) + try: + stdout, stderr = process.communicate(timeout=30) + except subprocess.TimeoutExpired: + os.killpg(process.pid, signal.SIGKILL) + stdout, stderr = process.communicate(timeout=10) + failure = "functional process exceeded deadline or was interrupted" + status = process.returncode + # Only fixture processes run with this cleared environment; no real DSNs. + (args.out / "functional-process.log").write_text(stdout + stderr) + finally: + if process is not None and process.poll() is None: + os.killpg(process.pid, signal.SIGTERM) + try: + process.wait(timeout=30) + except subprocess.TimeoutExpired: + os.killpg(process.pid, signal.SIGKILL) + process.wait(timeout=10) + status = process.returncode + # Keep startup diagnostics before removing verified-stopped fixture data. + # Every source is below this run's private parent, never a user database. + try: + for log in root.glob("prism-load-*/*.log"): + if log.is_file() and not log.is_symlink() and not log.parent.is_symlink(): + saved = args.out / "postgres-logs" / log.parent.name / log.name + saved.parent.mkdir(parents=True, exist_ok=True) + shutil.copyfile(log, saved) + except Exception: + diagnostic_failure = "could not retain owned PostgreSQL diagnostics" + failure = f"{failure}; {diagnostic_failure}" if failure else diagnostic_failure + try: + cleanup = cleanup_owned(root, args.pg_bin_dir) + except Exception as error: + cleanup = {"owned_parent_removed": False, "error": str(error), "retained_parent": str(root)} + census.update(process_exit=status, cleanup=cleanup, failure=failure) + (args.out / "resource-census.json").write_text(json.dumps(census, indent=2) + "\n") + if status != 0 or failure or not cleanup["owned_parent_removed"]: + raise RuntimeError("functional run failed or cleanup is unverified; inspect local evidence") + evidence = json.loads((args.out / "functional" / "ha-functional.json").read_text()) + if evidence["result"] != "passed" or not evidence["cleanup"]["complete"]: + raise RuntimeError("functional evidence is incomplete") + return {"functional": "passed", "report": str(args.out / "functional" / "ha-functional.json"), + "resource_census": str(args.out / "resource-census.json"), + "live_acceptance": "not established"} + + +def terminate_as_interrupt(_signum, _frame): + # Let run's existing interruption path stop only its own process group and + # verify its exact PG directories. A repeated TERM must not interrupt cleanup. + signal.signal(signal.SIGTERM, signal.SIG_IGN) + raise KeyboardInterrupt + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + subcommands = parser.add_subparsers(dest="command", required=True) + subcommands.add_parser("render", help="Check nondefault HA settings without starting services") + functional = subcommands.add_parser("run", help="Only after coordinating a resource slot") + for option in ("example-bin", "runtime-tests", "ack-tests", "pg-bin-dir", "out"): + functional.add_argument(f"--{option}", type=lambda p: Path(p).resolve(), required=True) + args = parser.parse_args() + previous = signal.signal(signal.SIGTERM, terminate_as_interrupt) + try: + print(json.dumps(render() if args.command == "render" else run(args), indent=2)) + finally: + signal.signal(signal.SIGTERM, previous) + + +if __name__ == "__main__": + main() diff --git a/scripts/prism_ha_readiness_probe.py b/scripts/prism_ha_readiness_probe.py new file mode 100644 index 000000000..53dc49fa3 --- /dev/null +++ b/scripts/prism_ha_readiness_probe.py @@ -0,0 +1,113 @@ +#!/usr/bin/env python3 +"""Deterministic simulator for the documented external HA readiness probe. + +This is qualification evidence only; it performs no network I/O or routing. +""" +from dataclasses import dataclass +import argparse +import json +import math +import numbers +from typing import Any + + +@dataclass(frozen=True) +class ProbeConfig: + interval_s: float = 2.0 + timeout_s: float = 1.0 + fall: int = 6 + rise: int = 2 + + def __post_init__(self) -> None: + if not math.isfinite(self.interval_s) or not math.isfinite(self.timeout_s): + raise ValueError("probe interval and timeout must be finite") + if self.interval_s <= 0 or self.timeout_s <= 0: + raise ValueError("probe interval and timeout must be positive") + if self.fall < 1 or self.rise < 1: + raise ValueError("fall and rise thresholds must be positive") + + +class ReadinessProbe: + """Monotonic hysteresis state machine: unknown, up, or down.""" + + def __init__(self, config: ProbeConfig = ProbeConfig()) -> None: + self.config = config + self.state = "unknown" + self.failures = 0 + self.successes = 0 + + @staticmethod + def classify(response: Any, elapsed_s: float, timeout_s: float) -> bool: + if ( + isinstance(elapsed_s, bool) + or not isinstance(elapsed_s, numbers.Real) + or not math.isfinite(elapsed_s) + or elapsed_s < 0 + or elapsed_s > timeout_s + or not isinstance(response, dict) + ): + return False + return response.get("status") == 200 and response.get("ok") is True + + def observe(self, response: Any, elapsed_s: float = 0.0) -> str: + good = self.classify(response, elapsed_s, self.config.timeout_s) + if good: + self.failures = 0 + self.successes += 1 + if self.state in ("unknown", "down") and self.successes >= self.config.rise: + self.state = "up" + else: + self.successes = 0 + self.failures += 1 + if self.failures >= self.config.fall: + self.state = "down" + return self.state + + +def run(sequence: list[tuple[Any, float]], config: ProbeConfig = ProbeConfig()) -> list[str]: + probe = ReadinessProbe(config) + return [probe.observe(response, elapsed) for response, elapsed in sequence] + + +def run_timeline( + sequence: list[tuple[float, Any, float]], config: ProbeConfig = ProbeConfig() +) -> list[str]: + """Run starts on a monotonic schedule; rejects overlapping/early probes.""" + probe = ReadinessProbe(config) + previous = None + previous_elapsed = 0.0 + states = [] + for started_at, response, elapsed in sequence: + if not math.isfinite(started_at) or started_at < 0: + raise ValueError("probe starts must be finite and non-negative") + if not math.isfinite(elapsed) or elapsed < 0: + raise ValueError("probe durations must be finite and non-negative") + if previous is not None and ( + started_at - previous < config.interval_s + or started_at < previous + previous_elapsed + ): + raise ValueError("probe starts must honor the configured monotonic interval") + previous = started_at + previous_elapsed = elapsed + states.append(probe.observe(response, elapsed)) + return states + + +def main() -> None: + # JSON-lines adapter for reproducible qualification, intentionally no I/O + # beyond stdin/stdout and no claim of real load-balancer behavior. + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--interval-s", type=float, default=2.0) + parser.add_argument("--timeout-s", type=float, default=1.0) + parser.add_argument("--fall", type=int, default=6) + parser.add_argument("--rise", type=int, default=2) + args = parser.parse_args() + config = ProbeConfig(args.interval_s, args.timeout_s, args.fall, args.rise) + probe = ReadinessProbe(config) + for line in __import__("sys").stdin: + item = json.loads(line) + print(json.dumps({"state": probe.observe(item["response"], item.get("elapsed_s", 0.0))})) + + +if __name__ == "__main__": + main() diff --git a/scripts/prism_l3_promote.py b/scripts/prism_l3_promote.py new file mode 100644 index 000000000..1645ccfdd --- /dev/null +++ b/scripts/prism_l3_promote.py @@ -0,0 +1,178 @@ +#!/usr/bin/env python3 +"""Decide whether a `v*` tag promotes a pull request's L3 run (#550). + +The full L3 suite (`suite:l3-full`, `.github/workflows/prism-load-l3.yml`) +runs on the version-bump pull request. When the release is tagged, the +tagged commit's tree is usually the tree that pull request's merge result +ran, so running the suite again measures nothing new (#487 cost lever P1). +This script looks for that run and prints the decision as JSON: + +- **promote** the newest run of the L3 workflow that + - ran on `pull_request` from this repository (not a fork), + - finished (conclusion `success` or `failure`, never cancelled, skipped or + timed out), + - uploaded `prism-l3-verdict-<tree>` for exactly this tree, whose + `verdict.json` (`qbit.prism.l3-verdict.v1`) is for suite `l3-full` and + `complete`: every planned run reached its gate. A complete run that + failed a gate is promoted too, and the tag run then fails with it: the + same tree would measure the same, and the verdict is carried, not + dropped; +- otherwise **rerun** the full suite. A lookup that fails for any reason is + a rerun, never a promotion: promoting needs positive evidence. + +The artifact's name carries the tree, so the lookup lists only candidates +for this tree and downloads the newest first. + +Usage: + python3 scripts/prism_l3_promote.py --repo OWNER/NAME --tree TREE \\ + --run-id ID [--out decision.json] + +Needs `gh` authenticated with `actions: read` (GH_TOKEN in the workflow). +Exit 0 with a decision either way; 2 on bad arguments. +""" + +from __future__ import annotations + +import argparse +import json +from pathlib import Path +import re +import subprocess +import sys +import tempfile +from typing import Callable + +WORKFLOW_PATH = ".github/workflows/prism-load-l3.yml" +VERDICT_ARTIFACT = "prism-l3-verdict-{tree}" +VERDICT_SCHEMA = "qbit.prism.l3-verdict.v1" +FULL_SUITE = "l3-full" +FINISHED = ("success", "failure") +TREE = re.compile(r"^[0-9a-f]{40}$") + +# gh(args) -> stdout; raises subprocess.CalledProcessError or OSError. +Gh = Callable[[list[str]], str] + + +def run_gh(args: list[str]) -> str: + return subprocess.run( + ["gh", *args], check=True, capture_output=True, text=True, timeout=120 + ).stdout + + +def download_verdict(gh: Gh, repo: str, run_id: int, name: str) -> dict: + with tempfile.TemporaryDirectory() as tmp: + gh(["run", "download", str(run_id), "--repo", repo, "-n", name, "-D", tmp]) + return json.loads((Path(tmp) / "verdict.json").read_text(encoding="utf-8")) + + +def run_rejection(run: dict, repo: str) -> str | None: + """Why `run` cannot be the source of a promotion, or None.""" + if run.get("event") != "pull_request": + return f"ran on {run.get('event')!r}, not pull_request" + if not str(run.get("path", "")).startswith(WORKFLOW_PATH): + return f"is a run of {run.get('path')!r}, not {WORKFLOW_PATH}" + head_repo = (run.get("head_repository") or {}).get("full_name") + if head_repo != repo: + return f"ran from {head_repo!r}, not this repository" + if run.get("status") != "completed" or run.get("conclusion") not in FINISHED: + return f"did not finish (status {run.get('status')!r}, conclusion {run.get('conclusion')!r})" + return None + + +def verdict_rejection(verdict: object, tree: str) -> str | None: + """Why the run's `verdict.json` does not show the full suite on `tree`.""" + if not isinstance(verdict, dict): + return "its verdict.json is not an object" + if verdict.get("schema") != VERDICT_SCHEMA: + return f"its verdict has schema {verdict.get('schema')!r}, not {VERDICT_SCHEMA}" + if verdict.get("tree") != tree: + return f"its verdict is for tree {verdict.get('tree')!r}" + if verdict.get("suite") != FULL_SUITE: + return f"ran suite {verdict.get('suite')!r}, not {FULL_SUITE}" + if verdict.get("complete") is not True: + return "did not bring every planned run to its gate" + if not isinstance(verdict.get("passed"), bool): + return "its verdict does not say whether the suite passed" + return None + + +def decide(gh: Gh, repo: str, tree: str, run_id: int) -> dict: + """The decision for `tree`: promote a pull request's run, or rerun.""" + name = VERDICT_ARTIFACT.format(tree=tree) + decision: dict = {"tree": tree, "artifact": name, "promote": False, "considered": []} + try: + listing = json.loads( + gh(["api", f"repos/{repo}/actions/artifacts?name={name}&per_page=100"]) + ) + except (subprocess.CalledProcessError, OSError, subprocess.TimeoutExpired, ValueError) as error: + decision["reason"] = f"rerun: the artifact lookup failed ({error})" + return decision + # An artifact that names no run cannot be traced to one, so it is not a + # candidate (and never crashes the lookup into a failed tag run). + artifacts = [ + a for a in listing.get("artifacts", []) + if isinstance(a, dict) + and not a.get("expired") + and isinstance((a.get("workflow_run") or {}).get("id"), int) + and a["workflow_run"]["id"] != run_id + ] + # Newest first; the API already lists them so, but a stale run must + # never shadow a newer one whatever order it returns. + artifacts.sort(key=lambda a: a.get("created_at", ""), reverse=True) + for artifact in artifacts: + source = artifact["workflow_run"]["id"] + try: + run = json.loads(gh(["api", f"repos/{repo}/actions/runs/{source}"])) + except (subprocess.CalledProcessError, OSError, subprocess.TimeoutExpired, ValueError) as error: + decision["considered"].append({"run": source, "rejected": f"lookup failed ({error})"}) + continue + why = run_rejection(run, repo) + verdict = None + if why is None: + try: + verdict = download_verdict(gh, repo, source, name) + except (subprocess.CalledProcessError, OSError, subprocess.TimeoutExpired, ValueError) as error: + why = f"its verdict could not be read ({error})" + else: + why = verdict_rejection(verdict, tree) + if why is not None: + decision["considered"].append({"run": source, "rejected": why}) + continue + decision.update( + promote=True, + source_run=source, + source_url=run.get("html_url"), + source_pull_requests=[pr.get("number") for pr in run.get("pull_requests") or []], + verdict=verdict, + reason=f"promote: run {source} ran suite {FULL_SUITE} on this tree and every planned " + f"run reached its gate ({'passed' if verdict['passed'] else 'not every run passed'})", + ) + return decision + decision["reason"] = ( + "rerun: no finished same-repository pull_request run of the full suite on this tree" + ) + return decision + + +def main(argv: list[str] | None = None) -> int: + parser = argparse.ArgumentParser( + description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter + ) + parser.add_argument("--repo", required=True) + parser.add_argument("--tree", required=True) + parser.add_argument("--run-id", type=int, required=True) + parser.add_argument("--out", type=Path) + args = parser.parse_args(argv) + if not TREE.match(args.tree): + print(f"prism-l3-promote: --tree {args.tree!r} is not a full tree id", file=sys.stderr) + return 2 + decision = decide(run_gh, args.repo, args.tree, args.run_id) + text = json.dumps(decision, indent=2) + "\n" + if args.out: + args.out.write_text(text, encoding="utf-8") + print(text, end="") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/prism_legacy_import_admission_gate.sh b/scripts/prism_legacy_import_admission_gate.sh new file mode 100755 index 000000000..438179037 --- /dev/null +++ b/scripts/prism_legacy_import_admission_gate.sh @@ -0,0 +1,417 @@ +#!/bin/bash +# The go-live gate for a frontend that admits traffic while `qbit-prism-server import-audits` is +# still storing legacy audit bodies. +# +# In production, `self-check` stops at legacy audit completeness and skips its local checks until +# the import has finished (#734). This script checks every fact `self-check` checks +# (`self_check()` and `self_check_local()` in crates/qbit-prism-server/src/tools.rs), with one +# exception: legacy completeness, which it reports as pending, together with whether an +# import-audits session can be seen. It still fails on any native audit row that self-check would +# count as incomplete. +# +# Run it on each frontend host, as the frontend's service user, with that frontend's operator +# environment exported (the environment `self-check` would read): +# set -a; . <the frontend's environment file>; set +a +# bash prism_legacy_import_admission_gate.sh +# It needs `qbit-prism-server`, `psql` and `python3` on PATH. Every query runs in a read-only +# transaction. psql is given the database URL without its password, which it reads from +# PGPASSWORD, so no credential appears in a command line or in the output. Exit 0 only when +# every check passes. +# +# Every query runs under statement_timeout 15 s and lock_timeout 5 s, except check 5's +# carry-forward integrity report, which takes about a minute at production size: 120 s on the +# writer, 600 s on a standby. The report holds a snapshot for that long, and on a primary with +# live miners every share updates the cluster row, so a long snapshot there slows the share +# append's lock (#738). Above about 200 shares/s, run the report elsewhere: +# PRISM_INTEGRITY_REPORT_DATABASE_URL run check 5 on this hot standby instead, which must +# have hot_standby_feedback off (or the report holds back the writer anyway). Pause its +# replay first (SELECT pg_wal_replay_pause(); until pg_get_wal_replay_pause_state() is +# 'paused'), or set max_standby_streaming_delay and max_standby_archive_delay to -1 +# there, or a recovery conflict can cancel the report. Never use a standby the writer +# waits on for apply (synchronous_commit = remote_apply): pausing it stalls every commit. +# The found-block offer standby (#529) waits for flush, which a pause doesn't stop. The +# gate refuses a URL that is not in recovery, is not a replica of this writer's database +# (system identifier and database name) on the writer's timeline, has hot_standby_feedback +# on, serves a writer that waits for apply, or is stale: its last replayed transaction is +# older than PRISM_INTEGRITY_REPORT_MAX_AGE_SECONDS (default 300). The live frontends' +# heartbeats keep the writer committing, so a following standby stays well inside that. +# Right after a failover a following standby can show its previous timeline until its +# next restartpoint; rerun then. The gate records both WAL positions. +# PRISM_GATE_INTEGRITY=skip skip check 5 and say so (WARN). Record the report from wherever +# it ran instead. +# These limits are the gate's own. PRISM_DATABASE_STATEMENT_TIMEOUT_MS sets the frontends' +# 15 s, which is too short for the report at production size (#737). +set -uo pipefail +: "${PRISM_DATABASE_URL:?export the frontend environment first}" +fail=0 +skipped="" +gate_tmp=$(mktemp -d) || exit 2 +trap 'rm -rf "$gate_tmp"' EXIT +caller_pgpassword_set=${PGPASSWORD+x} +caller_pgpassword=${PGPASSWORD-} +pass() { printf 'PASS %s\n' "$1"; } +bad() { printf 'FAIL %s\n' "$1"; fail=1; } +info() { printf 'INFO %s\n' "$1"; } +warn() { printf 'WARN %s\n' "$1"; } +trim() { + local value=$1 + value=${value#"${value%%[![:space:]]*}"} + printf '%s' "${value%"${value##*[![:space:]]}"}" +} + +# The URL without its password, and the password: from the user info or a `password` query +# parameter, percent-decoded as libpq decodes them. split_url NAME reads the variable NAME. +split_url() { + GATE_URL=${!1} python3 - <<'PY' +import os, sys, urllib.parse +parts = urllib.parse.urlsplit(os.environ["GATE_URL"].strip()) +if parts.scheme not in ("postgres", "postgresql"): + sys.exit(1) +userinfo, at, hosts = parts.netloc.rpartition("@") +user, _, password = userinfo.partition(":") +password = urllib.parse.unquote(password) +kept = [] +for item in parts.query.split("&") if parts.query else []: + key, _, value = item.partition("=") + if urllib.parse.unquote(key) == "password": + password = urllib.parse.unquote(value) + else: + kept.append(item) +if "\0" in password: + sys.exit(1) +netloc = f"{user}@{hosts}" if at else hosts +url = urllib.parse.urlunsplit(parts._replace(netloc=netloc, query="&".join(kept))) +sys.stdout.write(url + "\0" + password + "\0") +PY +} +if ! { IFS= read -r -d '' database && IFS= read -r -d '' password; } < <(split_url PRISM_DATABASE_URL); then + echo "PRISM_DATABASE_URL must be a postgres:// or postgresql:// URL" >&2 + exit 2 +fi +integrity_database="" +integrity_password="" +integrity_url=$(trim "${PRISM_INTEGRITY_REPORT_DATABASE_URL:-}") +if [ -n "$integrity_url" ] && ! { + IFS= read -r -d '' integrity_database && IFS= read -r -d '' integrity_password +} < <(split_url integrity_url); then + echo "PRISM_INTEGRITY_REPORT_DATABASE_URL must be a postgres:// or postgresql:// URL" >&2 + exit 2 +fi +max_age=$(trim "${PRISM_INTEGRITY_REPORT_MAX_AGE_SECONDS:-}") +max_age=${max_age:-300} +if ! [[ $max_age =~ ^[0-9]{1,9}$ ]] || (( 10#$max_age < 1 )); then + echo "PRISM_INTEGRITY_REPORT_MAX_AGE_SECONDS must be a whole number of seconds from 1 to 999999999" >&2 + exit 2 +fi +max_age=$(( 10#$max_age )) +if [ -n "$password" ]; then export PGPASSWORD=$password; fi +unset password +# psqlq SQL [psql -v name=value ...]: one read-only transaction against the frontend's +# database, or, with q_target=integrity, against PRISM_INTEGRITY_REPORT_DATABASE_URL. +# q_timeout overrides the 15 s statement timeout. psql's stderr is kept apart and only ever +# classified by last_error: it can name the host and user. +psqlq() { + local sql=$1 + shift + ( # A subshell, so the standby's credential stays in this process's environment, never argv. + target=$database + if [ "${q_target:-}" = integrity ]; then + target=$integrity_database + if [ -n "$integrity_password" ]; then + export PGPASSWORD=$integrity_password + elif [ -n "$caller_pgpassword_set" ]; then + export PGPASSWORD=$caller_pgpassword + else + unset PGPASSWORD + fi + fi + printf "SET default_transaction_read_only = on;\nSET statement_timeout = '%s';\nSET lock_timeout = '5s';\n%s;\n" \ + "${q_timeout:-15s}" "$sql" | + psql "$target" -XAtq -v ON_ERROR_STOP=1 "$@" 2>"$gate_tmp/psql.err" + ) +} +# Why the last psqlq failed, when psql said something recognizable, without echoing it. +last_error() { + local error + error=$(cat "$gate_tmp/psql.err" 2>/dev/null) + case $error in + *"canceling statement due to statement timeout"*) printf ' (statement timeout)' ;; + *"conflict with recovery"*) + printf ' (cancelled by a recovery conflict: pause the standby'"'"'s replay or set max_standby_streaming_delay = -1 there)' ;; + *"lock timeout"*) printf ' (lock timeout)' ;; + esac +} + +# 1. Configuration under production rules, the pool fee settling dust (#525) and the block +# submission state (#291). `check-config` runs all three, plus every other settings loader. +if out=$(qbit-prism-server check-config 2>&1); then + pass "check-config" +else + bad "check-config: $(tail -1 <<<"$out")" +fi +grep -E '^WARNING' <<<"$out" | sed 's/^/INFO check-config /' + +# 2. The live-instance census, classified as `live_instances()` classifies it. self-check fails +# only when the heartbeat read fails, and warns when HA is unknown or fewer than two +# frontends are live. Freshness is 3 health refreshes, at least 15 s. +refresh=$(trim "${PRISM_HEALTH_REFRESH_SECONDS:-}") +if [[ ${refresh:-2} =~ ^[0-9]+$ ]] && (( 10#${refresh:-2} >= 1 )); then + fresh=$(( 10#${refresh:-2} * 3 > 15 ? 10#${refresh:-2} * 3 : 15 )) +else + fresh=15 +fi +if census=$(psqlq "WITH sample AS (SELECT clock_timestamp() AS observed_at), +heartbeats AS ( + SELECT instance_id, extract(epoch FROM (observed_at - heartbeat_at)) AS age, status, + coalesce(jsonb_typeof(status) = 'object' + AND status->>'schema' = 'qbit.prism.audit-health.v1' + AND jsonb_typeof(status->'ready') = 'boolean', false) AS health + FROM qbit_prism_instances, sample) +SELECT count(*) FILTER (WHERE age BETWEEN 0 AND :fresh AND health) || '|' + || count(*) FILTER (WHERE age > :fresh) || '|' + || count(*) FILTER (WHERE age BETWEEN 0 AND :fresh AND NOT health + AND coalesce(status->>'state', '') IN ('starting', 'stopped')) || '|' + || count(*) FILTER (WHERE age < 0 OR (age <= :fresh AND NOT health + AND coalesce(status->>'state', '') NOT IN ('starting', 'stopped'))) || '|' + || coalesce(jsonb_agg(jsonb_build_object('id', instance_id, 'age_s', round(age::numeric, 1), + 'status', left(status::text, 80)) ORDER BY instance_id)::text, '[]') +FROM heartbeats" -v fresh="$fresh"); then + IFS='|' read -r live stale inactive unknown rows <<<"$census" + pass "heartbeat census read: $rows" + info "HA: $live live, $stale stale, $inactive inactive, $unknown unknown (freshness ${fresh}s)" + if [ "$unknown" != 0 ]; then + warn "Unrecognized or future-dated heartbeats; HA is unknown" + elif [ "$live" -lt 2 ]; then + warn "Fewer than two live frontends observed; do not present this deployment as HA" + fi +else + bad "heartbeat census read failed$(last_error)" +fi + +# 3. The cluster's block submission hold (#664): reported, never a failure. +info "submission hold: $(qbit-prism-server submission-hold show 2>&1 | tr '\n' ' ' | cut -c1-300)" + +# 4. Audit completeness, as `audit_completeness()` counts it, split by kind. Legacy rows (no +# snapshot) are what import-audits is storing. A native row (with a snapshot) that has no +# bytes and either no object body or no snapshot row is incomplete, and fails here as it +# would fail self-check. +if c=$(psqlq "SELECT count(*) FILTER (WHERE canonical_audit_bytes IS NULL AND share_snapshot_sha256 IS NULL) || '|' || count(*) FILTER (WHERE canonical_audit_bytes IS NULL AND share_snapshot_sha256 IS NOT NULL AND (jsonb_typeof(audit_bundle) IS DISTINCT FROM 'object' OR NOT EXISTS (SELECT 1 FROM qbit_prism_audit_snapshots s WHERE s.snapshot_sha256 = a.share_snapshot_sha256))) FROM qbit_pool_audit_bundles a"); then + IFS='|' read -r legacy native <<<"$c" + info "audit completeness: ${legacy} legacy rows pending" + if [ "$native" = 0 ]; then + pass "native audit rows complete" + else + bad "${native} native audit rows are incomplete" + fi + # Whether import-audits is running: a session whose last statement is the import's row read + # or write. Other sessions' statements need pg_read_all_stats to be seen. + if [ "${legacy:-0}" != 0 ]; then + if s=$(psqlq "SELECT count(*) FILTER (WHERE importing) || '|' || coalesce(round(extract(epoch FROM clock_timestamp() - max(query_start) FILTER (WHERE importing)))::text, '-') || '|' || count(*) FILTER (WHERE query = '<insufficient privilege>') FROM (SELECT query, query_start, (query LIKE 'UPDATE qbit_pool_audit_bundles SET schema_version=%' OR query LIKE 'SELECT block_hash,body_uri,audit_bundle,audit_bundle_sha256,coinbase_tx_hex FROM qbit_pool_audit_bundles%') AS importing FROM pg_stat_activity WHERE pid <> pg_backend_pid()) t"); then + IFS='|' read -r sessions age hidden <<<"$s" + if [ "$sessions" != 0 ]; then + info "import-audits: $sessions session(s), last statement ${age}s ago" + elif [ "$hidden" != 0 ]; then + info "import-audits progress not visible to this role (pg_read_all_stats shows it)" + else + warn "no import-audits session found: legacy completeness will not progress until it runs" + fi + else + warn "could not read pg_stat_activity for import-audits progress" + fi + fi +else + bad "audit completeness read failed$(last_error)" +fi + +# 5. Carry-forward integrity: mismatch_count and current_drift_count are 0. On the writer by +# default; on a hot standby with PRISM_INTEGRITY_REPORT_DATABASE_URL; skipped, with a WARN, +# with PRISM_GATE_INTEGRITY=skip. See the header for the standby's requirements. +integrity() { # integrity [q_target] + local r mismatch drift where="" limit=120s + if [ -n "${1:-}" ]; then where=" on the standby"; limit=600s; fi + if r=$(q_target=${1:-} q_timeout=$limit psqlq "SELECT (x->>'mismatch_count') || '|' || (x->>'current_drift_count') FROM (SELECT qbit_carry_forward_integrity_report() AS x) t"); then + IFS='|' read -r mismatch drift <<<"$r" + if [ "$mismatch" = 0 ] && [ "$drift" = 0 ]; then + pass "carry-forward integrity$where (mismatch 0, drift 0)" + else + bad "carry-forward integrity$where: mismatch_count=$mismatch current_drift_count=$drift" + fi + else + bad "carry-forward integrity report failed$where$(last_error)" + fi +} +integrity_mode=$(trim "${PRISM_GATE_INTEGRITY:-}") +identity_sql="(SELECT system_identifier FROM pg_control_system())::text || '|' || current_database()" +if [ -n "$integrity_mode" ] && [ "$integrity_mode" != skip ]; then + bad "PRISM_GATE_INTEGRITY must be empty or skip" +elif [ "$integrity_mode" = skip ]; then + warn "carry-forward integrity skipped (PRISM_GATE_INTEGRITY=skip): record the report from where it ran" + if [ -n "$integrity_database" ]; then + warn "PRISM_INTEGRITY_REPORT_DATABASE_URL is set but not used: PRISM_GATE_INTEGRITY=skip" + fi + skipped=" (carry-forward integrity skipped)" +elif [ -n "$integrity_database" ]; then + if ! w=$(psqlq "SELECT current_setting('synchronous_commit') || '|' || current_setting('synchronous_standby_names') || '|' || pg_current_wal_lsn()::text || '|' || $identity_sql || '|' || substring(pg_walfile_name(pg_current_wal_lsn()), 1, 8)"); then + bad "writer position read failed$(last_error)" + else + IFS='|' read -r commit_mode sync_names writer_lsn writer_system writer_db writer_tli <<<"$w" + # The standby's timeline: the WAL receiver's, when this role may read it, else its last + # restartpoint's (which can trail a timeline switch until the next restartpoint). WAL + # positions on different timelines aren't comparable, so a mismatch refuses. The age is + # clamped at 0 against clock skew between the hosts. + if ! r=$(q_target=integrity psqlq "SELECT pg_is_in_recovery()::text || '|' || coalesce(pg_last_wal_replay_lsn()::text, '-') || '|' || CASE WHEN pg_is_in_recovery() THEN pg_get_wal_replay_pause_state() ELSE '-' END || '|' || coalesce(greatest(0, round(extract(epoch FROM clock_timestamp() - pg_last_xact_replay_timestamp())))::text, '-') || '|' || current_setting('hot_standby_feedback') || '|' || current_setting('max_standby_streaming_delay') || '|' || current_setting('max_standby_archive_delay') || '|' || $identity_sql || '|' || lpad(upper(to_hex(coalesce((SELECT received_tli FROM pg_stat_wal_receiver), (SELECT timeline_id FROM pg_control_checkpoint())))), 8, '0')"); then + bad "integrity standby read failed$(last_error)" + else + IFS='|' read -r recovery lsn pause age feedback stream_delay archive_delay system db tli <<<"$r" + if [ "$recovery" != true ]; then + bad "PRISM_INTEGRITY_REPORT_DATABASE_URL is not a hot standby (in recovery: $recovery); unset it to run the report on the writer" + elif [ "$system" != "$writer_system" ] || [ "$db" != "$writer_db" ]; then + bad "the integrity standby is not a replica of this writer's database (its system identifier or database name differs)" + elif [ "$tli" != "$writer_tli" ]; then + bad "the integrity standby is on timeline $tli but the writer is on $writer_tli: after a failover, let it follow the writer's timeline (a follower shows it from its next restartpoint), or unset the URL" + elif [ "$commit_mode" = remote_apply ] && [ -n "$sync_names" ]; then + bad "the writer waits for standby apply (synchronous_commit=remote_apply): pausing one of its synchronous standbys stalls every commit, so resume this standby's replay now (SELECT pg_wal_replay_resume();) and use an asynchronous standby, or skip" + elif [ "$feedback" = on ]; then + bad "the integrity standby has hot_standby_feedback on: its report would hold back the writer's horizon all the same (#738); turn it off there, or unset the URL" + elif ! [[ $age =~ ^[0-9]+$ ]]; then + bad "the integrity standby has replayed no transaction yet; let it catch up, then pause it and rerun" + elif (( age > max_age )); then + bad "the integrity standby's last replayed transaction is ${age}s old (limit ${max_age}s): resume its replay until it catches up, then pause it and rerun" + else + info "integrity report on a standby: replayed to $lsn on timeline $tli (writer at $writer_lsn), replay state '$pause', last replayed transaction ${age}s ago" + if [ "$pause" != paused ] && ! { [ "$stream_delay" = -1 ] && [ "$archive_delay" = -1 ]; }; then + warn "the integrity standby's replay state is '$pause', not 'paused': a recovery conflict can cancel the report; pause it (SELECT pg_wal_replay_pause(); until pg_get_wal_replay_pause_state() is 'paused') or set max_standby_streaming_delay and max_standby_archive_delay to -1 there" + fi + integrity integrity + fi + fi + fi +else + integrity +fi + +# 6. Durability: fsync, full_page_writes and synchronous_commit are not off. +if d=$(psqlq "SELECT string_agg(name || '=' || setting, ' ' ORDER BY name) FROM pg_settings WHERE name IN ('fsync', 'full_page_writes', 'synchronous_commit')"); then + if grep -q '=off' <<<"$d"; then bad "durability: $d"; else pass "durability: $d"; fi +else + bad "durability read failed$(last_error)" +fi + +# 7. The found-block offer's standby (#529), on when PRISM_OFFER_STANDBY_APPLICATION_NAME is set +# and PRISM_OFFER_STANDBY_FLUSH_WAIT_MS is not 0 (default 250): the role can read replication +# positions, and exactly one streaming or catching-up standby by that name reports a flush +# position. +standby=$(trim "${PRISM_OFFER_STANDBY_APPLICATION_NAME:-}") +wait_ms=$(trim "${PRISM_OFFER_STANDBY_FLUSH_WAIT_MS:-}") +if [ -n "$standby" ] && ! [[ $wait_ms =~ ^\+?0+$ ]]; then + s=$(psqlq "SELECT pg_has_role(current_user, 'pg_read_all_stats', 'USAGE')::text || '|' || count(*) || '|' || count(flush_lsn) FROM pg_catalog.pg_stat_replication WHERE application_name = :'app' AND state IN ('streaming', 'catchup')" -v app="$standby") + if [ "$s" = "true|1|1" ]; then + pass "offer standby usable" + else + bad "offer standby not usable (readable|standbys|flushing = ${s:-error})" + fi +else + info "offer standby not configured" +fi + +# 8. The frontend's own readiness, through the same `healthcheck` self-check calls: HTTP 2xx and +# ok=true. It also stands for self-check's tool coordinator start and `refresh_once()`: this +# frontend passed the same startup gates and has published work on the observed tip. +if out=$(qbit-prism-server healthcheck 2>&1); then + pass "healthcheck (frontend ready on the observed tip)" +else + bad "healthcheck: $(tail -1 <<<"$out")" +fi + +# 9. The highdiff listener's floor, when PRISM_STRATUM_HIGHDIFF_PORT is set: within 15 s, the +# first difficulty it advertises is at least PRISM_STRATUM_HIGHDIFF_MIN_DIFF (default 500000). +# The probe authorizes as self-check does: PRISM_SELF_CHECK_ADDRESS, the username fallback +# (PRISM_USERNAME_FALLBACK_ADDRESS, or the built-in test-network address), the pool fee +# address, then the most recent miner. Like config::optional, a blank variable counts as +# unset and a set one is used verbatim. +highdiff_port=$(trim "${PRISM_STRATUM_HIGHDIFF_PORT:-}") +if [ -n "$highdiff_port" ]; then + user="" + for name in PRISM_SELF_CHECK_ADDRESS PRISM_USERNAME_FALLBACK_ADDRESS; do + if [ -z "$user" ] && [ -n "$(trim "${!name:-}")" ]; then user=${!name}; fi + done + if [ -z "$user" ]; then + case "$(trim "${QBIT_CHAIN:-regtest}" | tr '[:upper:]' '[:lower:]')" in + test | testnet | testnet3 | testnet4 | signet) + user=tq1zlsq9dpxz8mennhdpr9nf9s0f2tjtq6gxs9m84k6xglhkfp92q2zszzu4m3 ;; + esac + fi + if [ -z "$user" ] && [ -n "$(trim "${PRISM_POOL_FEE_ADDRESS:-}")" ]; then + user=$PRISM_POOL_FEE_ADDRESS + fi + [ -n "$user" ] || user=$(psqlq "SELECT miner_id FROM qbit_share_ledger ORDER BY share_seq DESC LIMIT 1") + bind=${PRISM_STRATUM_HIGHDIFF_BIND:-} + [ -n "$(trim "$bind")" ] || bind=${PRISM_STRATUM_BIND-127.0.0.1} + floor=$(trim "${PRISM_STRATUM_HIGHDIFF_MIN_DIFF:-}") + if [ -z "$user" ]; then + bad "highdiff probe: set PRISM_SELF_CHECK_ADDRESS to a valid P2MR address to probe highdiff on an empty pool" + elif res=$(python3 - "$bind" "$highdiff_port" "$user" "${floor:-500000}" <<'PY' 2>&1 +import json, math, socket, sys, time +host, port, user, floor = sys.argv[1], int(sys.argv[2]), sys.argv[3], float(sys.argv[4]) +host = {"0.0.0.0": "127.0.0.1", "::": "::1"}.get(host, host).strip("[]") +deadline = time.monotonic() + 15 + +def remaining(): + left = deadline - time.monotonic() + if left <= 0: + raise socket.timeout() + return left + +try: + conn = socket.create_connection((host, port), timeout=remaining()) + for message in ({"id": 1, "method": "mining.subscribe", "params": ["prism-admission-gate"]}, + {"id": 2, "method": "mining.authorize", "params": [user, "x"]}): + conn.settimeout(remaining()) + conn.sendall((json.dumps(message) + "\n").encode()) + pending = b"" + while True: + conn.settimeout(remaining()) + chunk = conn.recv(65536) + if not chunk: + sys.exit("Stratum probe disconnected before difficulty") + pending += chunk + while b"\n" in pending: + line, pending = pending.split(b"\n", 1) + if len(line) > 1024 * 1024: + sys.exit("oversized Stratum probe response") + value = json.loads(line) + if not isinstance(value, dict): + continue + if value.get("error") is not None: + sys.exit(f"Stratum probe rejected: {value['error']}") + if value.get("method") == "mining.set_difficulty": + params = value.get("params") + difficulty = params[0] if isinstance(params, list) and params else None + if (isinstance(difficulty, bool) or not isinstance(difficulty, (int, float)) + or not math.isfinite(difficulty) or difficulty <= 0): + sys.exit("invalid advertised difficulty") + difficulty = float(difficulty) + print(f"{difficulty} {'>=' if difficulty >= floor else '<'} {floor}") + sys.exit(0 if difficulty >= floor else 1) + if len(pending) > 1024 * 1024: + sys.exit("oversized Stratum probe response") +except socket.timeout: + sys.exit("Stratum difficulty probe timed out") +except OSError as error: + sys.exit(f"Stratum probe failed: {error}") +PY + ); then + pass "highdiff first difficulty $res" + else + bad "highdiff probe: $res" + fi +else + info "highdiff not configured on this host" +fi + +if [ "$fail" = 0 ]; then + echo "ADMISSION GATE: PASS (legacy audit import pending, reported above)$skipped" +else + echo "ADMISSION GATE: FAIL" +fi +exit "$fail" diff --git a/scripts/prism_legacy_parts_digest_check.py b/scripts/prism_legacy_parts_digest_check.py new file mode 100644 index 000000000..0e30778e4 --- /dev/null +++ b/scripts/prism_legacy_parts_digest_check.py @@ -0,0 +1,238 @@ +#!/usr/bin/env python3 +"""List the legacy audit rows `import-audits` refuses with "window proof +share_parts_digest_hex mismatch", without reading any share slot. + +2.x writes an audit-bundle.v2 body's window-proof `share_parts_digest_hex` as +sha256 of `json.dumps({"share_parts": parts}, separators=(",", ":"))`: each +part's keys in insertion order (`kind` first), non-ASCII escaped. 3.x +recomputes it from the parsed parts through `serde_json::json!`, and +serde_json without `preserve_order` sorts every object's keys, so 3.x refuses +every 2.x v2 body (crates/qbit-prism/src/audit_body_ref.rs). `import-audits` +reads a row's canonical sidecar before its body, so only a v2 body without one +in the import root reaches that check; prism_legacy_range_sidecars.py writes +the missing sidecars. + +For each row (`block_hash,body_uri,audit_bundle_sha256`) the status is: + has-sidecar the import root holds the row's canonical sidecar, which + import reads instead of the body (its bytes are verified by + prism_legacy_range_sidecars.py and by import, not here) + 2x-order-only the v2 digest is 2.x's: import refuses the body + matches-neither the v2 digest is neither form: import refuses the body + ok-in-3x the v2 digest is the form 3.x recomputes + no-parts-digest a v2 body without the digest, which 3.x does not check + no-window-proof not a v2 body (a body-ref or a plain bundle): not checked + body-missing, body-unreadable, body-digest-differs: the body itself +The model of 3.x keeps an inline part's shares as stored; `has_inline` flags +those bodies. + +`--affected-csv` writes the refused rows (2x-order-only, matches-neither) in +the input's format, for prism_legacy_range_sidecars.py --rows. The exit status +is 0 only when every row is has-sidecar, ok-in-3x, no-parts-digest or +no-window-proof; the summary on stderr counts each status. + +Usage: + prism_legacy_parts_digest_check.py --rows rows.csv \\ + --audit-root /var/lib/qbit-mining-pool/prism/audit \\ + --sidecar-dir <the directory passed to import-audits --root> \\ + [--affected-csv affected.csv] [--jobs 8] > report.jsonl + prism_legacy_parts_digest_check.py BODY.json [BODY.json ...] +""" + +from __future__ import annotations + +import argparse +import csv +import hashlib +import json +import multiprocessing +import os +import sys +from pathlib import Path + +V2_SCHEMA = "qbit.prism.audit-bundle.v2" +# 3.x's SharePart fields per kind (crates/qbit-prism/src/audit_body_ref.rs); +# serde skips a segment range's segment bounds when they are absent. +FIELDS = { + "segment": ("first_share_seq", "last_share_seq", "share_count", "sha256", "body_uri"), + "segment_range": ("segment_first_share_seq", "segment_last_share_seq", "first_share_seq", + "last_share_seq", "share_count", "range_sha256", "body_uri"), + "segment_prefix": ("first_share_seq", "last_share_seq", "share_count", "prefix_sha256", + "body_uri"), + "inline": ("first_share_seq", "last_share_seq", "share_count", "shares"), +} +OPTIONAL = ("segment_first_share_seq", "segment_last_share_seq") +REFUSED = ("2x-order-only", "matches-neither") +OK_STATUSES = {"has-sidecar", "ok-in-3x", "no-parts-digest", "no-window-proof"} +CSV_COLUMNS = ("block_hash", "body_uri", "audit_bundle_sha256") + + +def digest_2x(parts: list) -> str: + """2.x's share_parts_digest_hex (lab/prism/audit_artifacts.py on 2.x.x).""" + return hashlib.sha256( + json.dumps({"share_parts": parts}, separators=(",", ":")).encode("utf-8") + ).hexdigest() + + +def part_as_3x(part: dict) -> dict: + kind = part.get("kind") + out = {"kind": kind} + for field in FIELDS.get(kind, ()): + if field in OPTIONAL and part.get(field) is None: + continue + if field in part: + out[field] = part[field] + return out + + +def digest_3x(parts: list) -> str: + """What 3.x recomputes: the parsed parts' known fields, keys sorted, raw UTF-8.""" + view = {"share_parts": [part_as_3x(part) for part in parts]} + return hashlib.sha256( + json.dumps(view, separators=(",", ":"), sort_keys=True, ensure_ascii=False).encode("utf-8") + ).hexdigest() + + +def map_uri(uri: str, prefix: str, root: Path | None, base: Path | None) -> Path: + """prism_legacy_range_sidecars.py's mapping: the URI prefix becomes the + audit root, and a relative URI resolves against import's --root.""" + path = uri[len("file://"):] if uri.startswith("file://") else uri + if root is not None and prefix and ( + path == prefix or path.startswith(prefix.rstrip("/") + "/") + ): + return root / path[len(prefix):].lstrip("/") + candidate = Path(path) + return candidate if candidate.is_absolute() or base is None else base / candidate + + +def classify_body(path: Path, digest: str | None) -> dict: + result: dict = {"body": str(path)} + try: + body = json.loads(path.read_bytes()) + except FileNotFoundError: + result["status"] = "body-missing" + return result + except (OSError, ValueError) as exc: + result["status"] = "body-unreadable" + result["error"] = f"{type(exc).__name__}: {exc}"[:200] + return result + if (digest is not None and isinstance(body, dict) and "audit_bundle_sha256" in body + and str(body["audit_bundle_sha256"]).lower() != digest.lower()): + result["status"] = "body-digest-differs" + return result + if not isinstance(body, dict) or body.get("schema") != V2_SCHEMA: + result["status"] = "no-window-proof" + result["schema"] = body.get("schema") if isinstance(body, dict) else None + return result + if digest is not None and str(body.get("audit_bundle_sha256", "")).lower() != digest.lower(): + result["status"] = "body-digest-differs" + return result + proof = body.get("share_window_proof") + parts = proof.get("share_parts") if isinstance(proof, dict) else None + if not isinstance(parts, list) or not all(isinstance(part, dict) for part in parts): + result["status"] = "body-unreadable" + result["error"] = "share_window_proof.share_parts is not a list of objects" + return result + if proof.get("share_parts_digest_hex") is None: + result["status"] = "no-parts-digest" + return result + expected = str(proof["share_parts_digest_hex"]).lower() + result["parts"] = len(parts) + result["has_inline"] = any(part.get("kind") == "inline" for part in parts) + if digest_3x(parts) == expected: + result["status"] = "ok-in-3x" + elif digest_2x(parts) == expected: + result["status"] = "2x-order-only" + else: + result["status"] = "matches-neither" + return result + + +def classify_row(job: tuple[dict, dict]) -> dict: + row, options = job + block = (row.get("block_hash") or "").strip() + digest = (row.get("audit_bundle_sha256") or "").strip() + body_uri = (row.get("body_uri") or "").strip() + sidecar_dir = options["sidecar_dir"] + if sidecar_dir is not None and block and digest: + sidecar = sidecar_dir / f"prism-audit-bundle-canonical-{block}-{digest}.json.gz" + if os.path.lexists(sidecar): + result = {"status": "has-sidecar", "sidecar": str(sidecar)} + return {"block_hash": block, "audit_bundle_sha256": digest, **result} + if not body_uri: + result = {"status": "body-missing"} + else: + path = map_uri(body_uri, options["uri_prefix"], options["audit_root"], + sidecar_dir or options["audit_root"]) + try: + result = classify_body(path, digest or None) + except Exception as exc: # one malformed body must not end the run + result = {"body": str(path), "status": "body-unreadable", + "error": f"{type(exc).__name__}: {exc}"[:200]} + if block or digest: + result = {"block_hash": block, "audit_bundle_sha256": digest, **result} + return result + + +def main(argv: list[str] | None = None) -> int: + parser = argparse.ArgumentParser( + description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter + ) + parser.add_argument("bodies", nargs="*", type=Path, help="body files to check directly") + parser.add_argument("--rows", "--csv", type=Path, + help="CSV with header block_hash,body_uri,audit_bundle_sha256") + parser.add_argument("--audit-root", type=Path, help="where the audit tree is mounted") + parser.add_argument("--uri-prefix", default="/var/lib/qbit-mining-pool/prism/audit", + help="body URI prefix that --audit-root replaces") + parser.add_argument("--sidecar-dir", type=Path, + help="the import-audits --root directory: a row with a sidecar " + "there is not checked further") + parser.add_argument("--affected-csv", type=Path, + help="write the rows import refuses, in the --rows format") + parser.add_argument("--jobs", type=int, default=1, help="parallel processes (default 1)") + args = parser.parse_args(argv) + if args.rows is None and not args.bodies: + parser.error("give --rows or body files") + if args.jobs < 1: + parser.error("--jobs must be at least 1") + options = { + "audit_root": args.audit_root.absolute() if args.audit_root else None, + "uri_prefix": args.uri_prefix, + "sidecar_dir": args.sidecar_dir.absolute() if args.sidecar_dir else None, + } + rows: list[dict] = [{"body_uri": str(body.absolute())} for body in args.bodies] + if args.rows is not None: + with open(args.rows, newline="", encoding="utf-8") as handle: + reader = csv.DictReader(handle) + missing = set(CSV_COLUMNS) - set(reader.fieldnames or ()) + if missing: + parser.error(f"--rows lacks the column(s) {', '.join(sorted(missing))}") + rows.extend(reader) + jobs = [(row, options) for row in rows] + totals: dict[str, int] = {} + refused: list[dict] = [] + pool = multiprocessing.Pool(args.jobs) if args.jobs > 1 and len(jobs) > 1 else None + try: + results = pool.imap(classify_row, jobs) if pool else map(classify_row, jobs) + for row, result in zip(rows, results): + status = result["status"] + totals[status] = totals.get(status, 0) + 1 + if status in REFUSED and row.get("block_hash"): + refused.append({column: row.get(column, "") for column in CSV_COLUMNS}) + print(json.dumps(result, sort_keys=True, ensure_ascii=False)) + finally: + if pool: + pool.close() + pool.join() + if args.affected_csv is not None: + with open(args.affected_csv, "w", newline="", encoding="utf-8") as handle: + writer = csv.DictWriter(handle, fieldnames=CSV_COLUMNS) + writer.writeheader() + writer.writerows(refused) + summary = {"import_refuses": sum(totals.get(status, 0) for status in REFUSED), + "totals": dict(sorted(totals.items()))} + print(json.dumps(summary, sort_keys=True), file=sys.stderr) + return 0 if set(totals) <= OK_STATUSES else 3 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/scripts/prism_legacy_range_sidecars.py b/scripts/prism_legacy_range_sidecars.py new file mode 100644 index 000000000..4ae4d37d8 --- /dev/null +++ b/scripts/prism_legacy_range_sidecars.py @@ -0,0 +1,397 @@ +#!/usr/bin/env python3 +"""Prove legacy audit bodies whose share-segment range digests use Python's +escaped JSON, and write the canonical sidecars `import-audits` reads first. + +Background (#709): 1.x and 2.x hashed each `segment_range` and +`segment_prefix` part as sha256 of +`json.dumps({schema, first_share_seq, last_share_seq, share_count, shares}, +separators=(",", ":"))`, with the default ensure_ascii=True, so non-ASCII +text is written as \\uXXXX escapes. 3.x re-serializes the same shares with +serde_json, which writes raw UTF-8. A range whose shares carry any non-ASCII +string (a worker name, say) therefore fails 3.x's range check, and +`import-audits` refuses the body. The content itself is intact. + +For each row (`block_hash,body_uri,audit_bundle_sha256`) the tool: +1. checks an existing sidecar, if `--sidecar-dir` holds one: it must be one + gzip member whose bytes hash to the digest, because import reads it before + the body and stops on a bad one; +2. runs the 3.x canonicalizer on the body as it is: a body that already + verifies needs nothing; +3. otherwise rewrites the failing range parts of a body-ref or v2 body as + inline parts holding the slot's shares, and runs the canonicalizer on the + rewritten body-ref. The canonicalizer itself requires + sha256(canonical bundle) == audit_bundle_sha256, so a zero exit proves the + content is exactly what was committed; +4. with `--write`, gzips the proven canonical bytes (level 9, mtime 0) and + hard-links the complete file to + `<sidecar-dir>/prism-audit-bundle-canonical-<block>-<digest>.json.gz`, so + an existing file is never replaced and a partial one never appears. + `import-audits` re-verifies the sidecar against the digest, the coinbase + and the ledger key. + +Without `--write` it only classifies. The exit status is 0 only when every row +is `ok-in-3x`, `sidecar-ok`, `proven` or, with `--write`, `sidecar-written`; +the per-row report says what stopped the others. + +Usage: + prism_legacy_range_sidecars.py --rows rows.csv \\ + --audit-root /var/lib/qbit-mining-pool/prism/audit \\ + --canonicalizer target/release/qbit-prism-audit-canonicalize \\ + --sidecar-dir <the directory passed to import-audits --root> [--write] +""" + +from __future__ import annotations + +import argparse +import csv +import gzip +import hashlib +import io +import json +import os +import re +import shutil +import subprocess +import sys +import zlib +from pathlib import Path + +SEGMENT_SCHEMA = "qbit.prism.audit-share-segment.v1" +BODY_REF_SCHEMA = "qbit.prism.audit-body-ref.v1" +V2_SCHEMA = "qbit.prism.audit-bundle.v2" +# 3.x AcceptedShare field order (crates/qbit-prism/src/lib.rs); credit_policy +# is serialized only when set. +SHARE_FIELDS = ( + "share_seq", "share_id", "miner_id", "order_key", "p2mr_program_hex", + "share_difficulty", "network_difficulty", "template_height", "job_id", + "job_issued_at_ms", "accepted_at_ms", "ntime", +) +DIGEST_KEY = {"segment_range": "range_sha256", "segment_prefix": "prefix_sha256"} +# Range parts whose digest 3.x does not reproduce; the rewrite inlines them. +INLINED = ("python-escaped", "unexplained") +CSV_COLUMNS = {"block_hash", "body_uri", "audit_bundle_sha256"} +HEX64 = re.compile(r"[0-9a-f]{64}") +OK_STATUSES = {"ok-in-3x", "sidecar-ok", "proven", "sidecar-written"} + + +def sha256_hex(payload: bytes) -> str: + return hashlib.sha256(payload).hexdigest() + + +def compact(value: object, *, ascii_only: bool) -> bytes: + return json.dumps(value, separators=(",", ":"), ensure_ascii=ascii_only).encode("utf-8") + + +def accepted_share(share: dict) -> dict: + out = {key: share[key] for key in SHARE_FIELDS if key in share} + if share.get("credit_policy") is not None: + out["credit_policy"] = share["credit_policy"] + return out + + +def segment_payload(first: int, last: int, shares: list) -> dict: + return { + "schema": SEGMENT_SCHEMA, + "first_share_seq": first, + "last_share_seq": last, + "share_count": len(shares), + "shares": shares, + } + + +def map_uri(uri: str, prefix: str, root: Path, base: Path) -> Path: + path = uri[len("file://"):] if uri.startswith("file://") else uri + if prefix and (path == prefix or path.startswith(prefix.rstrip("/") + "/")): + return root / path[len(prefix):].lstrip("/") + candidate = Path(path) + return candidate if candidate.is_absolute() else base / candidate + + +def sidecar_path(sidecar_dir: Path, block: str, digest: str) -> Path: + return sidecar_dir / f"prism-audit-bundle-canonical-{block}-{digest}.json.gz" + + +def canonicalize(canonicalizer: str, body: Path) -> tuple[bytes | None, str]: + run = subprocess.run([canonicalizer, "--input", str(body)], capture_output=True) + if run.returncode != 0: + return None, run.stderr.decode("utf-8", "replace").strip()[-400:] + return run.stdout, "" + + +def slot_shares(slot: Path, slots: dict) -> list: + """A slot's shares, parsed once while consecutive parts read the same slot. + Only the latest slot is kept, which bounds memory to one slot.""" + if slot not in slots: + slots.clear() + slots[slot] = json.loads(slot.read_bytes())["shares"] + return slots[slot] + + +def probe_part(part: dict, prefix: str, root: Path, base: Path, slots: dict) -> dict: + """Classify one share part by which serialization reproduces its digest.""" + kind = part.get("kind") + probe = {"kind": kind, "first": part.get("first_share_seq"), "last": part.get("last_share_seq")} + if kind not in DIGEST_KEY: + probe["status"] = "not-a-range-part" + return probe + slot = map_uri(part.get("body_uri") or "", prefix, root, base) + probe["slot"] = str(slot) + if not slot.is_file(): + probe["status"] = "slot-missing" + return probe + first, last = int(part["first_share_seq"]), int(part["last_share_seq"]) + try: + selected = sorted( + (share for share in slot_shares(slot, slots) + if first <= int(share["share_seq"]) <= last), + key=lambda share: int(share["share_seq"]), + ) + except (OSError, KeyError, TypeError, ValueError) as exc: + probe["status"] = "slot-unreadable" + probe["error"] = f"{type(exc).__name__}: {exc}"[:200] + return probe + if len(selected) != int(part["share_count"]): + probe["status"] = f"slot-has-{len(selected)}-of-{part['share_count']}-shares" + return probe + shares = [accepted_share(share) for share in selected] + expected = str(part.get(DIGEST_KEY[kind]) or "").lower() + payload = segment_payload(first, last, shares) + if sha256_hex(compact(payload, ascii_only=False)) == expected: + probe["status"] = "ok-in-3x" + elif sha256_hex(compact(payload, ascii_only=True)) == expected: + probe["status"] = "python-escaped" + else: + probe["status"] = "unexplained" + probe["non_ascii"] = [ + {"share_seq": share.get("share_seq"), "field": key, "text": value} + for share in shares for key, value in share.items() + if isinstance(value, str) and not value.isascii() + ][:20] + if probe["status"] in INLINED: + probe["shares"] = shares + return probe + + +def rewrite(body: dict, digest: str, parts: list, probes: list, prefix: str, root: Path, + base: Path) -> dict: + """A body-ref copy with every failing range part inlined from its slot. + + A v2 body's own proof checks (its parts digest and reward manifest) are not + rerun on the copy; the bundle digest commits to the whole bundle, the + reward manifest included, so matching it proves the content as well.""" + new_parts = [] + for part, probe in zip(parts, probes): + if probe["status"] in INLINED: + new_parts.append({ + "kind": "inline", + "first_share_seq": part["first_share_seq"], + "last_share_seq": part["last_share_seq"], + "share_count": part["share_count"], + "shares": probe["shares"], + }) + else: + kept = dict(part) + if "body_uri" in kept: + kept["body_uri"] = str(map_uri(kept["body_uri"], prefix, root, base)) + new_parts.append(kept) + rewritten = { + "schema": BODY_REF_SCHEMA, + "audit_bundle_sha256": digest, + "share_count": body["share_count"], + "bundle_without_shares": body["bundle_without_shares"], + "share_parts": new_parts, + } + if "shares_key_index" in body: + rewritten["shares_key_index"] = body["shares_key_index"] + return rewritten + + +def write_sidecar(target: Path, canonical: bytes) -> str: + """Publish the gzip sidecar atomically, never replacing an existing file. + + The complete file is synced under a temporary name and hard-linked to the + target, which refuses an existing one. Import stops on a partial sidecar, + so there is no fallback for a filesystem without hard links: its OSError + reaches the caller, and the temporary file is removed either way.""" + sink = io.BytesIO() + with gzip.GzipFile(fileobj=sink, mode="wb", compresslevel=9, mtime=0) as handle: + handle.write(canonical) + temp = target.with_name(target.name + f".tmp-{os.getpid()}") + try: + with open(temp, "wb") as handle: + handle.write(sink.getvalue()) + handle.flush() + os.fsync(handle.fileno()) + os.link(temp, target) + except FileExistsError: + return "sidecar-conflict" + finally: + temp.unlink(missing_ok=True) + return "sidecar-written" + + +def existing_sidecar_status(target: Path, digest: str) -> str: + """`sidecar-ok` only for one complete gzip member, the form this tool + writes, whose bytes hash to the digest. Import reads the first member and + requires that hash, from a file that resolves inside its --root.""" + try: + # Python before 3.13 raises RuntimeError on a symlink loop. + if not target.resolve().is_relative_to(target.parent.resolve()): + return "sidecar-bad" + inflater = zlib.decompressobj(wbits=31) + payload = inflater.decompress(target.read_bytes()) + except (OSError, RuntimeError, zlib.error): + return "sidecar-bad" + if not inflater.eof or inflater.unused_data: + return "sidecar-bad" + return "sidecar-ok" if sha256_hex(payload) == digest else "sidecar-bad" + + +def share_parts(body: dict) -> list | None: + if body.get("schema") == BODY_REF_SCHEMA: + return body["share_parts"] + if body.get("schema") == V2_SCHEMA: + return body["share_window_proof"]["share_parts"] + return None + + +def process(row: dict, args: argparse.Namespace) -> dict: + # The sidecar name keeps the block hash as stored: import looks it up + # verbatim. Its digest check compares lowercase hex. + block = (row.get("block_hash") or "").strip() + digest = (row.get("audit_bundle_sha256") or "").strip() + result = {"block_hash": block, "audit_bundle_sha256": digest} + if not (HEX64.fullmatch(block.lower()) and HEX64.fullmatch(digest)): + result["status"] = "bad-row" + return result + target = sidecar_path(args.sidecar_dir, block, digest) if args.sidecar_dir else None + if target is not None and os.path.lexists(target): + result["sidecar"] = str(target) + result["status"] = existing_sidecar_status(target, digest) + return result + # A relative body URI resolves against import's --root, the sidecar + # directory. Import loads the body from its canonical path, so relative + # slot URIs resolve against the real file's directory. + body_uri = (row.get("body_uri") or "").strip() + body_base = args.sidecar_dir or args.audit_root + try: + # Python before 3.13 raises RuntimeError on a symlink loop. + body_path = map_uri(body_uri, args.uri_prefix, args.audit_root, body_base).resolve() + except (OSError, RuntimeError) as exc: + result["status"] = "body-unreadable" + result["error"] = f"{type(exc).__name__}: {exc}"[:200] + return result + result["body"] = str(body_path) + if not body_uri or not body_path.is_file(): + result["status"] = "body-missing" + return result + canonical, error = canonicalize(args.canonicalizer, body_path) + if canonical is not None and sha256_hex(canonical) == digest: + result["status"] = "ok-in-3x" + return result + result["original_error"] = error or "canonical bytes differ from the digest" + base = body_path.parent + try: + body = json.loads(body_path.read_bytes()) + parts = share_parts(body) + if parts is None: + result["status"] = "not-a-parted-body" + result["schema"] = body.get("schema") + return result + if str(body.get("audit_bundle_sha256", "")).lower() != digest: + result["status"] = "body-digest-differs" + return result + slots: dict = {} + probes = [probe_part(part, args.uri_prefix, args.audit_root, base, slots) + for part in parts] + slots.clear() + result["parts"] = [{k: v for k, v in probe.items() if k != "shares"} for probe in probes] + if any(probe["status"].startswith("slot-") for probe in probes): + result["status"] = "slot-incomplete" + return result + # Failing range parts are inlined; the others keep their slots, at the + # mapped paths. The canonicalizer's digest check is the proof either way. + rewritten = rewrite(body, digest, parts, probes, args.uri_prefix, args.audit_root, base) + except (OSError, AttributeError, KeyError, TypeError, ValueError) as exc: + result["status"] = "body-unreadable" + result["error"] = f"{type(exc).__name__}: {exc}"[:200] + return result + rewritten_path = args.work_dir / f"rewritten-{block}.json" + try: + args.work_dir.mkdir(parents=True, exist_ok=True) + rewritten_path.write_bytes(compact(rewritten, ascii_only=False)) + except OSError as exc: + result["status"] = "work-dir-unwritable" + result["error"] = f"{type(exc).__name__}: {exc}"[:200] + return result + canonical, error = canonicalize(args.canonicalizer, rewritten_path) + if canonical is None: + result["status"] = "canonicalizer-refused" + result["error"] = error + return result + if sha256_hex(canonical) != digest: + result["status"] = "digest-mismatch" + return result + if not args.write: + result["status"] = "proven" + return result + result["sidecar"] = str(target) + try: + result["status"] = write_sidecar(target, canonical) + except OSError as exc: + result["status"] = "sidecar-write-failed" + result["error"] = f"{type(exc).__name__}: {exc}"[:200] + return result + + +def main(argv: list[str] | None = None) -> int: + parser = argparse.ArgumentParser( + description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter + ) + parser.add_argument("--rows", required=True, type=Path, + help="CSV with header block_hash,body_uri,audit_bundle_sha256") + parser.add_argument("--audit-root", required=True, type=Path, + help="where the audit tree is mounted") + parser.add_argument("--uri-prefix", default="/var/lib/qbit-mining-pool/prism/audit", + help="body and slot URI prefix that --audit-root replaces") + parser.add_argument("--canonicalizer", required=True, + help="qbit-prism-audit-canonicalize built at the release pin") + parser.add_argument("--sidecar-dir", type=Path, + help="the import-audits --root directory, against which relative " + "body URIs resolve (else --audit-root); required with --write") + parser.add_argument("--work-dir", type=Path, default=Path("legacy-range-sidecars-work"), + help="where rewritten body-refs are kept for review") + parser.add_argument("--report", type=Path, default=Path("legacy-range-sidecars.jsonl")) + parser.add_argument("--write", action="store_true", help="write proven sidecars") + args = parser.parse_args(argv) + if args.write and args.sidecar_dir is None: + parser.error("--write needs --sidecar-dir") + if args.sidecar_dir is not None and not args.sidecar_dir.is_dir(): + parser.error(f"--sidecar-dir {args.sidecar_dir} is not a directory") + if shutil.which(args.canonicalizer) is None: + parser.error(f"--canonicalizer {args.canonicalizer} is not an executable") + # The canonicalizer resolves a rewritten body-ref's relative slot paths + # against the work directory, so every kept path is made absolute. + args.audit_root = args.audit_root.absolute() + args.work_dir = args.work_dir.absolute() + if args.sidecar_dir is not None: + args.sidecar_dir = args.sidecar_dir.absolute() + totals: dict[str, int] = {} + with open(args.rows, newline="", encoding="utf-8") as rows: + reader = csv.DictReader(rows) + missing = CSV_COLUMNS - set(reader.fieldnames or ()) + if missing: + parser.error(f"--rows lacks the column(s) {', '.join(sorted(missing))}") + with open(args.report, "w", encoding="utf-8") as report: + for row in reader: + result = process(row, args) + totals[result["status"]] = totals.get(result["status"], 0) + 1 + report.write(json.dumps(result, ensure_ascii=False) + "\n") + if result["status"] in ("sidecar-conflict", "sidecar-bad", "sidecar-write-failed"): + print(f"{result['status']}: {result['sidecar']}", file=sys.stderr) + print(json.dumps({"totals": totals, "report": str(args.report)}, sort_keys=True)) + return 0 if set(totals) <= OK_STATUSES else 3 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/scripts/prism_legacy_range_sidecars_sharded.sh b/scripts/prism_legacy_range_sidecars_sharded.sh new file mode 100755 index 000000000..a87d6fe33 --- /dev/null +++ b/scripts/prism_legacy_range_sidecars_sharded.sh @@ -0,0 +1,54 @@ +#!/bin/bash +# Run prism_legacy_range_sidecars.py --write over a rows CSV in N parallel, disjoint shards, +# publishing proven canonical sidecars into the import-audits --root directory. +# +# usage: prism_legacy_range_sidecars_sharded.sh ROWS.csv AUDIT_TREE IMPORT_ROOT CANONICALIZER TOOL OUT_DIR [N] +# ROWS.csv block_hash,body_uri,audit_bundle_sha256 (prism_legacy_parts_digest_check.py --affected-csv) +# AUDIT_TREE where the audit tree is mounted (the tool's --audit-root) +# IMPORT_ROOT the directory import-audits --root reads (the tool's --sidecar-dir) +# CANONICALIZER qbit-prism-audit-canonicalize built at the release pin; TOOL the sidecar tool +# OUT_DIR a new or empty directory outside IMPORT_ROOT for shard CSVs, work dirs and reports +# N shards, default 8 +# +# Shards stay apart: each has its own CSV (a round-robin split that keeps the header), its own +# --work-dir and its own --report; the tool's report otherwise defaults to one file in the +# current directory. In IMPORT_ROOT the tool writes <sidecar>.tmp-<pid> and publishes it with a +# hard link that refuses an existing file, so no sidecar is overwritten or seen half-written, +# and a rerun verifies the sidecars already there (sidecar-ok). Exit 0 only when every shard +# exits 0 and no temporary file is left in IMPORT_ROOT. +set -euo pipefail +[ $# -ge 6 ] || { sed -n '5,11p' "$0" >&2; exit 2; } +rows=$(realpath "$1"); tree=$(realpath "$2"); root=$(realpath "$3") +canon=$(realpath "$4"); tool=$(realpath "$5"); out=$(realpath -m "$6"); n=${7:-8} +[[ $n =~ ^[1-9][0-9]?$ ]] || { echo "N must be 1 to 99" >&2; exit 2; } +case "$out/" in "$root"/*) echo "OUT_DIR must not be inside IMPORT_ROOT" >&2; exit 2;; esac +mkdir -p "$out" +[ -z "$(ls -A "$out")" ] || { echo "OUT_DIR $out is not empty" >&2; exit 2; } +cd "$out" +total=$(( $(wc -l < "$rows") - 1 )) +[ "$total" -gt 0 ] || { echo "no rows in $rows: nothing to do"; exit 0; } +tail -n +2 "$rows" | split -e -n r/"$n" -d -a 2 - shard- +for f in shard-??; do { head -1 "$rows"; cat "$f"; } > "$f.csv"; rm "$f"; done +shards=(shard-*.csv) +sharded=$(( $(cat "${shards[@]}" | wc -l) - ${#shards[@]} )) +[ "$sharded" -eq "$total" ] || { echo "shard rows $sharded != input rows $total" >&2; exit 3; } +echo "$(date -u +%H:%M:%SZ) $total rows in ${#shards[@]} shards; writing into $root" +for f in "${shards[@]}"; do + i=${f#shard-}; i=${i%.csv} + ( set +e + python3 "$tool" --rows "$f" --audit-root "$tree" --canonicalizer "$canon" \ + --sidecar-dir "$root" --work-dir "work-$i" --report "report-$i.jsonl" --write \ + > "totals-$i.json" 2> "stderr-$i.log" + echo $? > "exit-$i" ) & +done +wait +echo "$(date -u +%H:%M:%SZ) done; shard exit codes: $(cat exit-* | tr '\n' ' ')" +echo "statuses over all shards:" +cat report-*.jsonl | python3 -c ' +import collections, json, sys +counts = collections.Counter(json.loads(line)["status"] for line in sys.stdin) +print(json.dumps(dict(sorted(counts.items())), indent=1))' +left=$(find "$root" -maxdepth 1 -name 'prism-audit-bundle-canonical-*.tmp-*' | wc -l) +echo "temporary files left in IMPORT_ROOT: $left" +exits=(exit-*) +[ "${#exits[@]}" -eq "${#shards[@]}" ] && ! grep -qv '^0$' "${exits[@]}" && [ "$left" -eq 0 ] diff --git a/scripts/prism_load_ab.py b/scripts/prism_load_ab.py new file mode 100644 index 000000000..ce8e201b5 --- /dev/null +++ b/scripts/prism_load_ab.py @@ -0,0 +1,1114 @@ +#!/usr/bin/env python3 +"""Run one load-harness preset against two builds, interleaved, on one host (#511). + +The release benchmark: `--base` and `--candidate` are git refs. Each is built +in release mode in its own clean detached worktree, and each run uses that +build's own harness and server (a harness never drives another build's +server). The series: + +- takes the host's benchmark lock for its whole length, so no other series + shares the host; +- records `pg_test_fsync` on the filesystem the harness builds its clusters + on, before and after the series; +- runs `--repeats` pairs, alternating which build goes first (base then + candidate, then candidate then base, ...), so a drift in the host over the + series does not land on one build; +- before each run waits `--cooldown-seconds`, then until the 1-minute load + average is under `--max-load` (giving up after `--load-wait-seconds`); +- samples the load average and MemAvailable every `--sample-seconds` during + each run and snapshots the process table at each end; +- kills a run that outlives `--run-ceiling-seconds`. + +Every flag the result depends on comes from the preset, checked in beside +this script and read from this checkout, never from a build's defaults. A +build whose harness predates a flag the preset pins runs without it only +when `crates/qbit-prism-load/legacy-flags.json` says its value is what that +build ran anyway; otherwise the pair is refused before anything runs, as is +a build whose harness has a result flag the preset does not pin. + +The series is written to `--out`: `manifest.json` (rewritten after every +run, so an interrupted series resumes with `--resume`), `builds/<label>/` +(the worktrees), `runs/<run id>/` (each harness's `--out`) with the run's +log, host samples and process snapshots beside it, `pg_test_fsync-*.txt`, +and `comparison.md`, `qbit-prism-load-compare`'s table. The exit status is +the comparison's: 0 the candidate meets #473's D1 rule in every gated phase, +1 it does not, 2 bad inputs or a failed build, 3 the series could not finish +(the host never went quiet). + +Usage: + python3 scripts/prism_load_ab.py --base 5d0042f6 --candidate origin/3.x.x \\ + --preset throughput-20k-window-1fe --out ab-throughput-20k-window-1fe +""" + +from __future__ import annotations + +import argparse +import contextlib +import datetime as dt +import fcntl +import hashlib +import json +import math +import os +from pathlib import Path +import platform +import re +import shutil +import signal +import socket +import subprocess +import sys +import threading +import time +import traceback +import urllib.parse +from typing import Any, Callable, Iterator + +from prism_load_matrix import load_aliases + +ROOT = Path(__file__).resolve().parents[1] +CRATE = ROOT / "crates" / "qbit-prism-load" +PRESETS = CRATE / "presets" +LEGACY_FLAGS = CRATE / "legacy-flags.json" +PRESET_SOURCE = CRATE / "src" / "preset.rs" +PRESET_SCHEMA = "qbit.prism.load-preset.v1" +LEGACY_SCHEMA = "qbit.prism.load-legacy-flags.v1" +MANIFEST_SCHEMA = "qbit.prism.load-ab.v1" +LABELS = ("base", "candidate") +FORMULAS = ("sessions-per-frontend-plus-16-min-128",) + +EXIT_PASS, EXIT_FAIL, EXIT_INPUT, EXIT_INCOMPLETE = 0, 1, 2, 3 + + +class DriverError(Exception): + """A refusal worded for the operator; exits 2.""" + + +class IncompleteSeries(Exception): + """The series could not finish on this host; exits 3.""" + + +# --- presets and flags ------------------------------------------------------ + + +def required_pg_binaries(source: Path = CRATE / "src" / "cluster.rs") -> tuple[str, ...]: + """The server binaries the harness refuses to start without, read from + `cluster.rs` so the list has one source.""" + match = re.search(r"pub const REQUIRED_BINARIES: \[&str; \d+\] = \[(.*?)\];", source.read_text(encoding="utf-8")) + if not match: + raise DriverError(f"{source}: REQUIRED_BINARIES not found") + return tuple(re.findall(r'"([a-z_]+)"', match.group(1))) + + +def operational_flags(source: Path = PRESET_SOURCE) -> frozenset[str]: + """The harness's operational flags, read from `preset.rs` so the list + has one source.""" + text = source.read_text(encoding="utf-8") + match = re.search(r"pub const OPERATIONAL_FLAGS: &\[&str\] = &\[(.*?)\];", text, re.S) + if not match: + raise DriverError(f"{source}: OPERATIONAL_FLAGS not found") + flags = frozenset(re.findall(r'"(--[a-z0-9-]+)"', match.group(1))) + if "--out" not in flags: + raise DriverError(f"{source}: OPERATIONAL_FLAGS does not name --out") + return flags + + +def load_preset(path: Path) -> dict[str, Any]: + try: + raw = path.read_bytes() + preset = json.loads(raw) + except (OSError, ValueError) as error: + raise DriverError(f"preset {path}: {error}") from error + if preset.get("schema") != PRESET_SCHEMA: + raise DriverError(f"preset {path}: schema is {preset.get('schema')!r}, not {PRESET_SCHEMA}") + if preset.get("name") != path.stem or not isinstance(preset.get("args"), dict): + raise DriverError(f"preset {path}: needs a name equal to its stem and an args object") + preset["_sha256"] = hashlib.sha256(raw).hexdigest() + preset["_path"] = str(path) + return preset + + +def load_legacy(path: Path = LEGACY_FLAGS) -> dict[str, dict[str, Any]]: + table = json.loads(path.read_text(encoding="utf-8")) + if table.get("schema") != LEGACY_SCHEMA: + raise DriverError(f"{path}: schema is {table.get('schema')!r}, not {LEGACY_SCHEMA}") + flags = table["flags"] + for flag, rule in flags.items(): + kinds = [k for k in ("equals", "equals_flag", "inert_when", "formula") if k in rule] + if len(kinds) != 1: + raise DriverError(f"{path}: {flag} needs exactly one rule, has {kinds}") + if "formula" in rule and rule["formula"] not in FORMULAS: + raise DriverError(f"{path}: {flag}: unknown formula {rule['formula']!r}") + if not re.fullmatch(r"[0-9a-f]{40}", str(rule.get("introduced", ""))): + raise DriverError(f"{path}: {flag} needs the full commit that introduced it, as introduced") + return flags + + +def load_report_fields(path: Path = LEGACY_FLAGS) -> dict[str, dict[str, Any]]: + """The side-report fields added after the oldest comparable build, each + with the commit that introduced it.""" + fields = json.loads(path.read_text(encoding="utf-8")).get("report_fields", {}) + for field, rule in fields.items(): + if not re.fullmatch(r"[0-9a-f]{40}", str(rule.get("introduced", ""))) or not rule.get("why"): + raise DriverError(f"{path}: report field {field} needs introduced (a full commit) and why") + return fields + + +def descends_from(commit: str, ancestor: str) -> bool: + """Whether `commit` has `ancestor` in its history; a history that cannot + be read (a shallow clone, an unknown commit) is refused, not guessed.""" + result = subprocess.run(["git", "merge-base", "--is-ancestor", ancestor, commit], + cwd=ROOT, capture_output=True, text=True, check=False) + if result.returncode not in (0, 1): + raise DriverError(f"cannot tell whether {commit[:8]} descends from {ancestor[:8]}: {result.stderr.strip()}") + return result.returncode == 0 + + +def same(a: Any, b: Any) -> bool: + """JSON equality with numbers compared as numbers (0 == 0.0), and a + boolean never equal to a number.""" + numeric = lambda v: isinstance(v, (int, float)) and not isinstance(v, bool) # noqa: E731 + if numeric(a) and numeric(b): + return float(a) == float(b) + return type(a) is type(b) and a == b + + +def legacy_holds(flag: str, rule: dict[str, Any], args: dict[str, Any]) -> tuple[bool, str]: + """Whether a build without `flag` runs what the preset's value asks.""" + value = args.get(flag) + if "equals" in rule: + return same(value, rule["equals"]), f"needs {json.dumps(rule['equals'])}" + if "equals_flag" in rule: + other = rule["equals_flag"] + return ( + other in args and same(value, args[other]), + f"needs the value of {other} ({json.dumps(args.get(other))})", + ) + if "inert_when" in rule: + wanted = rule["inert_when"] + holds = all(k in args and same(args[k], v) for k, v in wanted.items()) + return holds, "is read only when " + ", ".join( + f"{k} is not {json.dumps(v)}" for k, v in wanted.items() + ) + # sessions-per-frontend-plus-16-min-128 + sessions, frontends = args.get("--sessions"), args.get("--frontends") + if not all(isinstance(n, int) and not isinstance(n, bool) and n > 0 for n in (sessions, frontends)): + return False, "needs --sessions and --frontends to derive the old admission" + implied = max(math.ceil(sessions / frontends) + 16, 128) + return same(value, implied), f"needs {implied} (ceil(sessions / frontends) + 16, at least 128)" + + +def argv_words(flag: str, value: Any) -> list[str]: + """As `Preset::argv`: `true` bare, `false` and `null` omitted.""" + if value is True: + return [flag] + if value is False or value is None: + return [] + if isinstance(value, str): + return [flag, value] + if isinstance(value, (int, float)): + return [flag, json.dumps(value)] + raise DriverError(f"{flag} is {value!r}, not a string, number, boolean or null") + + +def resolve_argv( + args: dict[str, Any], + build_flags: frozenset[str], + legacy: dict[str, dict[str, Any]], + operational: frozenset[str], + label: str, + contains: Callable[[str], bool], +) -> tuple[list[str], list[str]]: + """The preset's command line for one build, and the flags it left off. + `contains` says whether the build's history has a commit: a build + lacking a flag it descends from the introduction of lost the flag, and + a legacy rule speaks only for builds that never had it.""" + words: list[str] = [] + dropped: list[str] = [] + problems: list[str] = [] + for flag in sorted(args): + value = args[flag] + if flag in build_flags: + words += argv_words(flag, value) + continue + rule = legacy.get(flag) + if rule is None: + problems.append( + f"its harness has no {flag} and legacy-flags.json has no rule for what it ran instead" + ) + continue + if contains(rule["introduced"]): + problems.append( + f"its harness has no {flag}, though its history has {rule['introduced'][:8]}, which " + "introduced it: the flag was removed, and a legacy rule speaks only for older builds" + ) + continue + holds, needs = legacy_holds(flag, rule, args) + if holds: + dropped.append(flag) + else: + problems.append( + f"its harness has no {flag}; the preset's {json.dumps(value)} is a workload it " + f"cannot run (leaving the flag off {needs})" + ) + unpinned = sorted(build_flags - set(args) - operational - {"--help", "--version"}) + for flag in unpinned: + problems.append(f"its harness has result flag {flag}, which the preset does not pin") + if problems: + raise DriverError(f"the {label} build cannot run this preset:\n - " + "\n - ".join(problems)) + return words, dropped + + +def help_flags(harness: Path) -> frozenset[str]: + """Every long flag a harness binary accepts, from its `--help`.""" + result = subprocess.run([str(harness), "--help"], capture_output=True, text=True, check=False) + if result.returncode != 0: + raise DriverError(f"{harness} --help exited {result.returncode}: {result.stderr.strip()}") + return parse_help_flags(result.stdout) + + +def parse_help_flags(text: str) -> frozenset[str]: + """The long flags clap's `--help` defines: the first on an indented + option line, never one a description mentions (`pg_config --bindir`).""" + return frozenset(re.findall(r"^\s+(?:-[A-Za-z0-9], )?(--[a-z0-9][a-z0-9-]*)", text, re.M)) + + +# --- host ------------------------------------------------------------------- + + +def load1() -> float: + return float(Path("/proc/loadavg").read_text().split()[0]) + + +def mem_available_mib() -> int | None: + with contextlib.suppress(OSError): + for line in Path("/proc/meminfo").read_text().splitlines(): + if line.startswith("MemAvailable:"): + return int(line.split()[1]) // 1024 + return None + + +def host_facts() -> dict[str, Any]: + mem_total = None + with contextlib.suppress(OSError): + for line in Path("/proc/meminfo").read_text().splitlines(): + if line.startswith("MemTotal:"): + mem_total = int(line.split()[1]) // 1024 + return { + "hostname": socket.gethostname(), + "nproc": os.cpu_count(), + "mem_total_mib": mem_total, + "kernel": platform.release(), + } + + +FSYNC_LINE = re.compile(r"^\s*fdatasync\s+([\d.]+) ops/sec\s+(\d+) usecs/op", re.M) + + +def parse_pg_test_fsync(text: str) -> dict[str, float] | None: + """The fdatasync line of the one-8 kB-write section, or `None`.""" + start = text.find("using one 8kB write") + if start < 0: + return None + section = text[start:] + end = section.find("\nCompare", 1) + match = FSYNC_LINE.search(section if end < 0 else section[:end]) + if not match: + return None + return {"fdatasync_ops_per_second": float(match.group(1)), "fdatasync_usecs_per_op": float(match.group(2))} + + +def record_fsync(pg_bin: Path, tmpdir: Path, out: Path, seconds: int) -> dict[str, Any]: + probe = tmpdir / "pg_test_fsync.out" + try: + result = subprocess.run( + [str(pg_bin / "pg_test_fsync"), "-s", str(seconds), "-f", str(probe)], + capture_output=True, + text=True, + check=False, + ) + except OSError as error: + raise DriverError(f"running {pg_bin / 'pg_test_fsync'}: {error}") from error + with contextlib.suppress(OSError): + probe.unlink() + out.write_text(f"# pg_test_fsync on {tmpdir}\n{result.stdout}{result.stderr}", encoding="utf-8") + parsed = parse_pg_test_fsync(result.stdout) if result.returncode == 0 else None + # A probe that failed, or printed no one-write fdatasync figure, leaves + # the series unplaced against the reference host: the host is broken, + # not the build, so the series stops as bad input with the probe's own + # output kept beside it. + if parsed is None: + raise DriverError( + f"pg_test_fsync on {tmpdir} exited {result.returncode} without a one-8 kB-write fdatasync " + f"figure; see {out}" + ) + return {"file": out.name, "exit_code": result.returncode, **parsed} + + +@contextlib.contextmanager +def series_lock(path: Path) -> Iterator[None]: + """The series directory's own lock; a second invocation on the same + `--out` is refused rather than queued.""" + with open(path, "a+") as handle: + try: + fcntl.flock(handle, fcntl.LOCK_EX | fcntl.LOCK_NB) + except BlockingIOError: + raise DriverError(f"another invocation is using {path.parent}") from None + try: + yield + finally: + fcntl.flock(handle, fcntl.LOCK_UN) + + +@contextlib.contextmanager +def benchmark_lock(path: Path) -> Iterator[None]: + with open(path, "a+") as handle: + try: + fcntl.flock(handle, fcntl.LOCK_EX | fcntl.LOCK_NB) + except BlockingIOError: + print(f"waiting for the benchmark lock {path}", file=sys.stderr, flush=True) + fcntl.flock(handle, fcntl.LOCK_EX) + try: + yield + finally: + fcntl.flock(handle, fcntl.LOCK_UN) + + +def wait_for_quiet( + max_load: float, + wait_seconds: float, + poll_seconds: float = 10.0, + read_load: Callable[[], float] = load1, + sleep: Callable[[float], None] = time.sleep, + now: Callable[[], float] = time.monotonic, +) -> float: + """Block until the 1-minute load is under `max_load`; the load seen.""" + deadline = now() + wait_seconds + while True: + load = read_load() + if load < max_load: + return load + if now() >= deadline: + raise IncompleteSeries( + f"the 1-minute load stayed at or above {max_load} for {wait_seconds:.0f} s " + f"(last {load:.2f}); the series stops rather than run on a loaded host" + ) + sleep(poll_seconds) + + +def interleaved(repeats: int) -> list[tuple[int, str]]: + """(repeat, label) in run order: base first on odd repeats, the + candidate first on even ones.""" + order = [] + for repeat in range(1, repeats + 1): + pair = LABELS if repeat % 2 else tuple(reversed(LABELS)) + order += [(repeat, label) for label in pair] + return order + + +# --- builds ----------------------------------------------------------------- + + +def git(*args: str, cwd: Path = ROOT) -> str: + result = subprocess.run(["git", *args], cwd=cwd, capture_output=True, text=True, check=False) + if result.returncode != 0: + raise DriverError(f"git {' '.join(args)}: {result.stderr.strip()}") + return result.stdout.strip() + + +# The binaries a build provides, and the record of a build this driver made: +# the commit it built and each binary's digest. `--skip-build` reuses a +# worktree's binaries only under that record. The harness takes its revision +# from the checkout it runs in and proves the server binary's provenance from +# Cargo's dep-info, but not its own, so a harness binary left from another +# commit, or copied in, would have its measurements attributed to this one. +BINARIES = ("qbit-prism-load", "qbit-prism-server") +BUILD_RECORD = ".prism-ab-build.json" + + +def build_record(commit: str, release: Path) -> dict[str, Any]: + digests = {} + for binary in BINARIES: + digest = hashlib.sha256() + with (release / binary).open("rb") as handle: + for chunk in iter(lambda: handle.read(1 << 20), b""): + digest.update(chunk) + digests[binary] = digest.hexdigest() + return {"commit": commit, "sha256": digests} + + +REDACTED = "<redacted>" + + +def redact_url_secrets(value: str) -> str: + """As the harness's `frontend::redact_url_secrets`: a URL keeps its user, + host and path but loses its password and the value of any query + parameter whose percent-decoded name is `password`; anything else is + returned as it is. The manifest and the log carry resolved command + lines, and a preset may pin an external `--database-url` with + credentials.""" + scheme, separator, rest = value.partition("://") + if not separator: + return value + ends = [index for index in (rest.find(mark) for mark in "/?#") if index >= 0] + authority, tail = rest[:min(ends, default=len(rest))], rest[min(ends, default=len(rest)):] + userinfo, at, host = authority.rpartition("@") + if at: + user, colon, _ = userinfo.partition(":") + authority = f"{user}:{REDACTED}@{host}" if colon else f"{userinfo}@{host}" + path, question, query = tail.partition("?") + if question: + query, hash_mark, fragment = query.partition("#") + pairs = [] + for pair in query.split("&"): + name, equals, _ = pair.partition("=") + # Decoded as SQLx reads a key (`+` a space, a malformed escape + # kept), compared ASCII case-insensitively. + decoded = urllib.parse.unquote_plus(name) + secret = equals and decoded.isascii() and decoded.lower() == "password" + pairs.append(f"{name}={REDACTED}" if secret else pair) + tail = f"{path}?{'&'.join(pairs)}" + (f"#{fragment}" if hash_mark else "") + return f"{scheme}://{authority}{tail}" + + +def prepare_build(label: str, ref: str, out: Path, skip_build: bool) -> dict[str, Any]: + commit = git("rev-parse", "--verify", f"{ref}^{{commit}}") + tree = out / "builds" / label + if tree.exists(): + head = git("rev-parse", "HEAD", cwd=tree) + if head != commit: + raise DriverError(f"{tree} is at {head}, not {ref} ({commit}); remove it or choose another --out") + else: + tree.parent.mkdir(parents=True, exist_ok=True) + git("worktree", "add", "--detach", str(tree), commit) + if git("status", "--porcelain", cwd=tree): + raise DriverError(f"{tree} has local changes; the harness would refuse its dirty tree") + release = tree / "target" / "release" + record = release / BUILD_RECORD + if not skip_build: + # A build that fails or is interrupted leaves no record behind it. + record.unlink(missing_ok=True) + env = {k: v for k, v in os.environ.items() if k != "CARGO_TARGET_DIR"} + print(f"building {label} ({ref}, {commit[:8]}) in {tree}", file=sys.stderr, flush=True) + result = subprocess.run( + ["cargo", "build", "--locked", "--release", "-p", "qbit-prism-server", "-p", "qbit-prism-load"], + cwd=tree, + env=env, + check=False, + ) + if result.returncode != 0: + raise DriverError(f"building {label} ({ref}) failed with exit {result.returncode}") + for binary in BINARIES: + if not (release / binary).is_file(): + raise DriverError(f"{release / binary} is missing") + built = build_record(commit, release) + if skip_build: + try: + recorded = json.loads(record.read_text(encoding="utf-8")) + except (OSError, ValueError): + recorded = None + if recorded != built: + raise DriverError( + f"--skip-build: {release} holds no build this driver made of {ref} ({commit[:8]}), " + "so its binaries cannot be attributed to that commit; run without --skip-build" + ) + else: + write_json(record, built) + return {"label": label, "ref": ref, "commit": commit, "worktree": str(tree.relative_to(out)), + "sha256": built["sha256"]} + + +# --- one run ---------------------------------------------------------------- + + +def write_json(path: Path, value: Any) -> None: + temporary = path.with_suffix(path.suffix + ".tmp") + temporary.write_text(json.dumps(value, indent=2) + "\n", encoding="utf-8") + temporary.replace(path) + + +def process_snapshot(path: Path) -> None: + # The command name, never the command line: on a shared host another + # process's arguments may carry a token or a password, and this file is + # attached with the series. + result = subprocess.run( + ["ps", "-eo", "pid,ppid,pcpu,rss,etime,comm", "--sort=-pcpu"], + capture_output=True, + text=True, + check=False, + ) + path.write_text("\n".join(result.stdout.splitlines()[:60]) + "\n", encoding="utf-8") + + +def group_alive(pgid: int) -> bool: + try: + os.killpg(pgid, 0) + except ProcessLookupError: + return False + except PermissionError: + return True + return True + + +def stop_group(child: subprocess.Popen, grace_seconds: float = 60.0, poll_seconds: float = 0.2) -> None: + """SIGTERM the child's process group, then SIGKILL whatever of it is + left after `grace_seconds`. The group, not the leader, is waited for: a + server or PostgreSQL the harness started can outlive the harness, and + the benchmark lock must not be released while one does.""" + pgid = child.pid + with contextlib.suppress(ProcessLookupError): + os.killpg(pgid, signal.SIGTERM) + deadline = time.monotonic() + grace_seconds + while True: + # Reap the leader, and any member orphaned to the driver as the + # subreaper, so a zombie does not keep the group alive. + child.poll() + reap_orphans(keep=child.pid) + if not group_alive(pgid): + return + if time.monotonic() >= deadline: + break + time.sleep(poll_seconds) + with contextlib.suppress(ProcessLookupError): + os.killpg(pgid, signal.SIGKILL) + child.wait() + # SIGKILL cannot be refused; what can linger is a zombie no init reaps, + # which holds no resources, so the wait for its reaping is bounded. + reaped_by = time.monotonic() + 10.0 + while group_alive(pgid) and time.monotonic() < reaped_by: + reap_orphans(keep=child.pid) + time.sleep(poll_seconds) + + +PR_SET_CHILD_SUBREAPER = 36 + + +def become_subreaper() -> bool: + """Make the driver the reaper of every orphan below it (Linux). The + harness's servers and PostgreSQL each call setsid(), so they leave the + harness's process group; as a subreaper the driver still finds them, as + its descendants, when the harness dies before stopping them.""" + try: + import ctypes + + libc = ctypes.CDLL(None, use_errno=True) + return libc.prctl(PR_SET_CHILD_SUBREAPER, 1, 0, 0, 0) == 0 + except (OSError, AttributeError): + return False + + +def descendants(root: int) -> list[int]: + """Every live or unreaped process below `root`, from /proc.""" + children: dict[int, list[int]] = {} + for entry in Path("/proc").iterdir(): + if not entry.name.isdigit(): + continue + try: + fields = (entry / "stat").read_text().rsplit(")", 1)[1].split() + except OSError: + continue + children.setdefault(int(fields[1]), []).append(int(entry.name)) + found, queue = [], list(children.get(root, [])) + while queue: + pid = queue.pop() + found.append(pid) + queue.extend(children.get(pid, [])) + return found + + +def _alive(pid: int) -> bool: + """Running, as opposed to gone or a zombie; reaps it if it is ours.""" + with contextlib.suppress(ChildProcessError, OSError): + os.waitpid(pid, os.WNOHANG) + try: + state = Path(f"/proc/{pid}/stat").read_text().rsplit(")", 1)[1].split()[0] + except OSError: + return False + return state != "Z" + + +def reap_orphans(keep: int | None = None) -> None: + """Collect the exit status of every finished process below the driver + that has become its child, except `keep` (the harness, whose status its + Popen collects).""" + for pid in descendants(os.getpid()): + if pid != keep: + _alive(pid) + + +def sweep_descendants(grace_seconds: float = 60.0, poll_seconds: float = 0.2) -> None: + """Stop every process still below the driver, whatever its session: + SIGTERM, then SIGKILL after `grace_seconds`, reaping each, and return + only once none is left. Runs only once a run's harness is gone, when + nothing below the driver belongs to anything but that run.""" + me = os.getpid() + live = lambda: [pid for pid in descendants(me) if _alive(pid)] # noqa: E731 + for pid in live(): + with contextlib.suppress(ProcessLookupError): + os.kill(pid, signal.SIGTERM) + deadline = time.monotonic() + grace_seconds + while live() and time.monotonic() < deadline: + time.sleep(poll_seconds) + # A process in uninterruptible I/O outlives SIGKILL until the I/O + # returns. The driver waits for it however long that takes, still + # holding the benchmark lock, and says so, rather than start the next + # run or free the host while it is alive. + warned = time.monotonic() + while remaining := live(): + for pid in remaining: + with contextlib.suppress(ProcessLookupError): + os.kill(pid, signal.SIGKILL) + if time.monotonic() - warned >= 10.0: + print(f"waiting for {', '.join(map(str, remaining))} to exit after SIGKILL; the benchmark " + "lock stays held until it does", file=sys.stderr, flush=True) + warned = time.monotonic() + time.sleep(poll_seconds) + + +def execute_run( + run_id: str, + command: list[str], + cwd: Path, + env: dict[str, str], + runs_dir: Path, + ceiling_seconds: float, + sample_seconds: float, +) -> dict[str, Any]: + """Run one harness with host sampling and a ceiling.""" + stem = runs_dir / run_id + samples: list[dict[str, Any]] = [] + stop = threading.Event() + + def sample() -> None: + with open(f"{stem}.host.jsonl", "w", encoding="utf-8") as handle: + while not stop.is_set(): + row = {"t": time.time(), "load1": load1(), "mem_available_mib": mem_available_mib()} + samples.append(row) + handle.write(json.dumps(row) + "\n") + handle.flush() + stop.wait(sample_seconds) + + process_snapshot(Path(f"{stem}.ps-before.txt")) + started = dt.datetime.now(dt.timezone.utc) + sampler = threading.Thread(target=sample, daemon=True) + sampler.start() + ceiling_hit = False + exit_code: int | None = None + with open(f"{stem}.log", "w", encoding="utf-8") as log: + child: subprocess.Popen | None = None + try: + # A termination signal that lands while Popen is forking would + # raise before `child` exists; it is held until the spawn + # returns and raised after it, where the `finally` sees the child. + with deferred_termination() as pending: + child = subprocess.Popen(command, cwd=cwd, env=env, stdout=log, stderr=subprocess.STDOUT, + start_new_session=True) + if pending: + raise Terminated(pending[0]) + try: + exit_code = child.wait(timeout=ceiling_seconds) + except subprocess.TimeoutExpired: + ceiling_hit = True + finally: + # However the wait ended (exit, ceiling, Ctrl-C, a signal), + # nothing of the run outlives it: a server or PostgreSQL the + # harness started can outlive the harness, and the benchmark lock + # must not be released while one does. Termination is held while + # the group is stopped and raised once it is. + with deferred_termination() as held: + if child is not None and (child.poll() is None or group_alive(child.pid)): + stop_group(child) + # The servers and PostgreSQL left the harness's group with + # setsid(); as the subreaper the driver finds them below it. + sweep_descendants() + stop.set() + if held: + raise Terminated(held[0]) + stop.set() + sampler.join() + process_snapshot(Path(f"{stem}.ps-after.txt")) + loads = [row["load1"] for row in samples] + mems = [row["mem_available_mib"] for row in samples if row["mem_available_mib"] is not None] + return { + "started_at": started.isoformat(), + "finished_at": dt.datetime.now(dt.timezone.utc).isoformat(), + "exit_code": exit_code, + "ceiling_hit": ceiling_hit, + "load_max": max(loads) if loads else None, + "mem_available_min_mib": min(mems) if mems else None, + } + + +# --- the series ------------------------------------------------------------- + + +def positive(kind: type) -> Callable[[str], Any]: + def parse(text: str) -> Any: + value = kind(text) + if not (value > 0 and math.isfinite(value)): + raise argparse.ArgumentTypeError(f"{text!r} is not a positive finite number") + return value + + return parse + + +def non_negative(text: str) -> float: + value = float(text) + if not (value >= 0 and math.isfinite(value)): + raise argparse.ArgumentTypeError(f"{text!r} is not a non-negative finite number") + return value + + +def parse_args(argv: list[str]) -> argparse.Namespace: + parser = argparse.ArgumentParser(description=__doc__.split("\n\n")[0]) + parser.add_argument("--base", required=True, help="git ref of the build compared against") + parser.add_argument("--candidate", required=True, help="git ref of the build under test") + parser.add_argument("--preset", required=True, help="a preset name under crates/qbit-prism-load/presets, or a path") + parser.add_argument("--out", required=True, type=Path, help="the series directory") + parser.add_argument("--repeats", type=positive(int), default=3) + parser.add_argument("--pg-bin-dir", type=Path, default=Path(os.environ.get("PG_BIN_DIR", "/usr/lib/postgresql/16/bin"))) + parser.add_argument("--cooldown-seconds", type=non_negative, default=60.0) + parser.add_argument("--max-load", type=positive(float), default=3.0, help="start a run only under this 1-minute load") + parser.add_argument("--load-wait-seconds", type=non_negative, default=3600.0) + parser.add_argument("--sample-seconds", type=positive(float), default=10.0) + parser.add_argument("--run-ceiling-seconds", type=positive(float), default=2400.0) + parser.add_argument("--fsync-seconds", type=positive(int), default=5, help="pg_test_fsync -s") + parser.add_argument("--lock-file", type=Path, default=Path("/tmp/qbit-prism-bench.lock")) + parser.add_argument("--tmpdir", type=Path, default=Path(os.environ.get("PRISM_LOAD_TMPDIR", "/tmp/pload-ab")), + help="the harness's TMPDIR, where it builds its clusters; kept short for the socket path") + parser.add_argument("--compare-bin", type=Path, help="a built qbit-prism-load-compare (default: build this checkout's)") + parser.add_argument("--resume", action="store_true", help="keep the runs an interrupted series already recorded") + parser.add_argument("--skip-build", action="store_true", + help="reuse the release builds this driver made in the worktrees, as it recorded them") + parser.add_argument("--dry-run", action="store_true", help="build and resolve both command lines, run nothing") + return parser.parse_args(argv) + + +# Settings a resumed series may change: none of them touches what is +# measured or how the host is isolated. Every other setting (the load gate, +# cooldown, sampling, ceiling, lock, the clusters' filesystem, the repeat +# count) must stay as the series began. +RESUME_FREE = ("resume", "skip_build", "dry_run", "compare_bin") + + +def settings_of(options: argparse.Namespace) -> dict[str, Any]: + """The settings the manifest records, as JSON values.""" + return { + key: str(value) if isinstance(value, Path) else value + for key, value in vars(options).items() + if key not in ("base", "candidate", "preset", "out") + } + + +def check_resumable(previous: dict[str, Any], options: argparse.Namespace) -> None: + problems = [] + host = host_facts() + for key, was in previous["host"].items(): + if host.get(key) != was: + problems.append(f"the host's {key} was {was}, not {host.get(key)}") + for key, now in settings_of(options).items(): + was = previous["settings"].get(key) + if key not in RESUME_FREE and was != now: + problems.append(f"--{key.replace('_', '-')} was {was}, not {now}") + if problems: + raise DriverError("cannot resume: " + "; ".join(problems) + "; start a new --out instead") + + +def preset_path(name: str) -> Path: + """A path as given; a name from the presets directory, a deprecated one + through `aliases.txt` with a warning, as the nightly matrix resolves it.""" + path = Path(name) + if path.suffix == ".json" or os.sep in name: + return path.resolve() + if not (PRESETS / f"{name}.json").exists(): + renamed = load_aliases(PRESETS).get(name) + if renamed: + print(f"warning: preset {name} is deprecated; it is now {renamed}", file=sys.stderr) + name = renamed + return PRESETS / f"{name}.json" + + +_built_comparator: Path | None = None + + +def compare_bin(explicit: Path | None) -> Path: + global _built_comparator + if explicit: + return explicit + if _built_comparator is not None: + return _built_comparator + built = subprocess.run( + ["cargo", "build", "--locked", "--release", "-p", "qbit-prism-load", "--bin", "qbit-prism-load-compare"], + cwd=ROOT, + check=False, + ) + if built.returncode != 0: + raise DriverError(f"building qbit-prism-load-compare failed with exit {built.returncode}") + _built_comparator = comparator_path(os.environ.get("CARGO_TARGET_DIR")) + return _built_comparator + + +def comparator_path(target_dir: str | None) -> Path: + """Where cargo, run from ROOT, puts the comparator: a relative + CARGO_TARGET_DIR is relative to that working directory, not ours.""" + target = ROOT / (target_dir or "target") + return target / "release" / "qbit-prism-load-compare" + + +def validate_preset(comparator: Path, preset: str) -> None: + try: + result = subprocess.run([str(comparator), "--validate-preset", "--preset", preset], + capture_output=True, text=True, check=False) + except OSError as error: + raise DriverError(f"running {comparator}: {error}") from error + if result.returncode != 0: + raise DriverError(f"preset {preset} is not valid: {result.stderr.strip() or result.stdout.strip()}") + + +def main(argv: list[str]) -> int: + options = parse_args(argv) + # Every path absolute: the harness runs with its worktree as its working + # directory, so a relative TMPDIR would name a different directory there + # than the one pg_test_fsync probed. + for key in ("tmpdir", "pg_bin_dir", "lock_file", "compare_bin"): + if getattr(options, key) is not None: + setattr(options, key, getattr(options, key).resolve()) + out: Path = options.out.resolve() + preset = load_preset(preset_path(options.preset)) + legacy = load_legacy() + operational = operational_flags() + manifest_path = out / "manifest.json" + if not options.dry_run: + missing = [ + str(options.pg_bin_dir / name) + for name in ("pg_test_fsync", *required_pg_binaries()) + if not os.access(options.pg_bin_dir / name, os.X_OK) + ] + if missing: + raise DriverError( + f"{', '.join(missing)}: not executable; pass --pg-bin-dir (PostgreSQL 16's bin directory)" + ) + + # The lock covers the builds and the comparator's build as well as the + # runs: a compile on the host distorts another series' measurement as + # much as a run does. The manifest is read under it too, so a second + # invocation that waited sees the series the first one wrote. + # One invocation per series, whatever --lock-file each names: the lock + # lives in the series directory itself. + out.mkdir(parents=True, exist_ok=True) + # Every invocation starts processes the cleanup must be able to find: + # the comparator and the builds compile, runs start servers. + if not become_subreaper(): + raise DriverError( + "could not become a child subreaper (prctl PR_SET_CHILD_SUBREAPER): without it a server or " + "PostgreSQL left by a killed harness would escape the cleanup and outlive its run" + ) + with series_lock(out / ".series.lock"), benchmark_lock(options.lock_file): + if manifest_path.exists() and not options.resume: + raise DriverError(f"{manifest_path} exists; pass --resume to continue that series or choose another --out") + out.mkdir(parents=True, exist_ok=True) + (out / "runs").mkdir(exist_ok=True) + previous = json.loads(manifest_path.read_text()) if manifest_path.exists() else None + if previous and previous["preset"]["sha256"] != preset["_sha256"]: + raise DriverError(f"{manifest_path} is a series of another preset ({previous['preset']['name']})") + if previous: + check_resumable(previous, options) + try: + # The full preset check (gates, budgets, timeout, every flag this + # harness would derive a setting or phase from), before any + # build, and inside the cleanup: it may compile the comparator. + validate_preset(compare_bin(options.compare_bin), preset["_path"]) + return run_series(options, out, preset, legacy, operational, manifest_path, previous) + finally: + # Whatever ended the series (a signal in a cargo build, a failed + # probe), nothing it started outlives the locks: rustc, a linker + # or a build script orphaned by an interrupted cargo is below the + # driver, as its subreaper, and is stopped here. + with deferred_termination() as held: + sweep_descendants() + if held: + raise Terminated(held[0]) + + +def run_series( + options: argparse.Namespace, + out: Path, + preset: dict[str, Any], + legacy: dict[str, dict[str, Any]], + operational: frozenset[str], + manifest_path: Path, + previous: dict[str, Any] | None, +) -> int: + builds = [] + report_fields = load_report_fields() + # The command lines as they run; only redacted copies are printed or + # written to the manifest. + commands: dict[str, list[str]] = {} + for label, ref in zip(LABELS, (options.base, options.candidate)): + build = prepare_build(label, ref, out, options.skip_build) + harness = out / build["worktree"] / "target" / "release" / "qbit-prism-load" + commands[label], build["dropped_legacy_flags"] = resolve_argv( + preset["args"], help_flags(harness), legacy, operational, label, + lambda introduced, commit=build["commit"]: descends_from(commit, introduced), + ) + build["argv"] = [redact_url_secrets(word) for word in commands[label]] + # The report fields this build's harness is too old to write; the + # comparator requires every other one. + build["predates_report_fields"] = sorted( + field for field, rule in report_fields.items() + if not descends_from(build["commit"], rule["introduced"]) + ) + builds.append(build) + if previous: + for old, new in zip(previous["builds"], builds): + if old["commit"] != new["commit"]: + raise DriverError(f"{manifest_path}: the {old['label']} build was {old['commit']}, not {new['commit']}") + # The same commit can build other binaries (a toolchain, RUSTFLAGS + # or another build input changed), and runs of both would be + # pooled under one label. + if old.get("sha256") != new.get("sha256"): + raise DriverError( + f"{manifest_path}: the {old['label']} build's binaries are not the ones its recorded " + "runs used; start a new series" + ) + + manifest = { + "schema": MANIFEST_SCHEMA, + "preset": {"name": preset["name"], "path": preset["_path"], "sha256": preset["_sha256"]}, + "host": host_facts(), + "settings": settings_of(options), + "pg_test_fsync": previous["pg_test_fsync"] if previous else {}, + "builds": builds, + "runs": previous["runs"] if previous else [], + } + for build in builds: + print(f"{build['label']} ({build['commit'][:8]}): {' '.join(build['argv'])}", file=sys.stderr) + if build["dropped_legacy_flags"]: + print(f" left off as legacy: {', '.join(build['dropped_legacy_flags'])}", file=sys.stderr) + if options.dry_run: + return EXIT_PASS + + options.tmpdir.mkdir(parents=True, exist_ok=True) + env = {**os.environ, "TMPDIR": str(options.tmpdir)} + done = {run["id"] for run in manifest["runs"]} + if "before" not in manifest["pg_test_fsync"]: + manifest["pg_test_fsync"]["before"] = record_fsync( + options.pg_bin_dir, options.tmpdir, out / "pg_test_fsync-before.txt", options.fsync_seconds) + write_json(manifest_path, manifest) + for repeat, label in interleaved(options.repeats): + run_id = f"{preset['name']}-{label}-r{repeat}" + if run_id in done: + continue + build = next(b for b in builds if b["label"] == label) + run_dir = out / "runs" / run_id + if run_dir.exists(): + shutil.rmtree(run_dir) + time.sleep(options.cooldown_seconds) + load_before = wait_for_quiet(options.max_load, options.load_wait_seconds) + mem_before = mem_available_mib() + release = out / build["worktree"] / "target" / "release" + command = [ + str(release / "qbit-prism-load"), + *commands[label], + "--server-bin", str(release / "qbit-prism-server"), + "--pg-bin-dir", str(options.pg_bin_dir), + "--out", str(run_dir), + ] + print(f"{run_id}: load {load_before:.2f}, running", file=sys.stderr, flush=True) + record = execute_run(run_id, command, out / build["worktree"], env, out / "runs", + options.run_ceiling_seconds, options.sample_seconds) + manifest["runs"].append({ + "id": run_id, + "build": label, + "repeat": repeat, + "dir": f"runs/{run_id}", + "load_before": load_before, + "mem_available_before_mib": mem_before, + **record, + }) + write_json(manifest_path, manifest) + print(f"{run_id}: exit {record['exit_code']}{' (ceiling)' if record['ceiling_hit'] else ''}", + file=sys.stderr, flush=True) + if "after" not in manifest["pg_test_fsync"]: + manifest["pg_test_fsync"]["after"] = record_fsync( + options.pg_bin_dir, options.tmpdir, out / "pg_test_fsync-after.txt", options.fsync_seconds) + write_json(manifest_path, manifest) + + comparator = compare_bin(options.compare_bin) + try: + summary = subprocess.run( + [str(comparator), "--manifest", str(manifest_path), "--preset", preset["_path"]], + capture_output=True, + text=True, + check=False, + ) + except OSError as error: + raise DriverError(f"running {comparator}: {error}") from error + (out / "comparison.md").write_text(summary.stdout, encoding="utf-8") + sys.stdout.write(summary.stdout) + sys.stderr.write(summary.stderr) + return {0: EXIT_PASS, 1: EXIT_FAIL}.get(summary.returncode, EXIT_INPUT) + + +class Terminated(BaseException): + """SIGTERM or SIGHUP reached the driver. Raised in the main thread, it + takes the same path as Ctrl-C: `execute_run` stops the harness's process + group (which `start_new_session` keeps out of reach of any signal sent + to ours) before the benchmark lock is released.""" + + def __init__(self, signum: int) -> None: + super().__init__(signal.Signals(signum).name) + self.signum = signum + + +# Set while a signal must not interrupt: the handler records it here +# instead of raising. +_deferred: list[int] | None = None + + +def on_termination(signum: int, frame: Any) -> None: + if _deferred is not None: + _deferred.append(signum) + return + raise Terminated(signum) + + +@contextlib.contextmanager +def deferred_termination() -> Iterator[list[int]]: + """Hold SIGTERM and SIGHUP for the block; the list yielded holds any + that arrived, for the caller to raise once it can clean up.""" + global _deferred + held: list[int] = [] + _deferred = held + try: + yield held + finally: + _deferred = None + + +def run_cli(argv: list[str]) -> int: + """`main` with every failure mapped to the documented exit status.""" + previous = {signum: signal.signal(signum, on_termination) for signum in (signal.SIGTERM, signal.SIGHUP)} + try: + return main(argv) + except DriverError as error: + print(f"prism_load_ab: {error}", file=sys.stderr) + return EXIT_INPUT + except IncompleteSeries as error: + print(f"prism_load_ab: {error}", file=sys.stderr) + return EXIT_INCOMPLETE + except Terminated as error: + print(f"prism_load_ab: stopped by {error}", file=sys.stderr) + return 128 + error.signum + except Exception: # noqa: BLE001 + # Python's own status for an uncaught exception is 1, the status of a + # candidate FAIL; an unexpected failure is an infrastructure problem. + traceback.print_exc() + return EXIT_INPUT + finally: + for signum, handler in previous.items(): + signal.signal(signum, handler) + + +if __name__ == "__main__": + sys.exit(run_cli(sys.argv[1:])) diff --git a/scripts/prism_load_bridge.py b/scripts/prism_load_bridge.py new file mode 100644 index 000000000..d71e0f328 --- /dev/null +++ b/scripts/prism_load_bridge.py @@ -0,0 +1,403 @@ +#!/usr/bin/env python3 +"""The bridging lane's comparison row: what the real node costs (#552, #487 S12). + +The nightly workflow's `bridging` job runs `--plan short` twice on one +runner, once on the fake node (`short-plan-fake-node`) and once on a real +regtest qbitd (`short-plan-real-node`). The two presets differ only in +`--node`. This script reads the two run directories that +`.github/scripts/prism-load-run.sh` wrote and reports, for each metric, +real minus fake: + +- `client_ack_latency` p50 and p99 in each phase both runs drove; +- the time to usable work on the external tips: the slowest session's time + (`time_to_usable_work.tips[].all_sessions_milliseconds`), p50 and max over + the tips; +- the pool node's `submitblock` and `getblocktemplate` latency through the + harness's relay. The fake node has no relay and records no latency, so its + side and the difference are unknown, never 0; +- peak RSS (`VmHWM`) of each frontend and of the frontends together, and of + the pool qbitd. The fake node runs inside the harness process, which is + not sampled, so its node RSS is unknown. + +It is a cost statement on this runner and this commit, not a D1 verdict +(#487 decisions 1 and 5): real-node rates are never D1 evidence, and the +short plan runs at 50 shares/s. + +A figure a side could not measure stays null with the reason. A side whose +harness or gate did not exit 0 still gets its numbers, but the row's +`trusted` is false and names why. This script only reports; the workflow +fails the job on the sides' gate exit codes. + +Writes the row as JSON (`--out`) and the Markdown table (`--summary`, which +is appended to, as GITHUB_STEP_SUMMARY is). Exits 0 once both are written, +2 on bad arguments or an unwritable output. + +Usage: + python3 scripts/prism_load_bridge.py --fake DIR --real DIR --order fake,real \ + --out bridge-row.json [--summary FILE] [--commit SHA] [--run-url URL] +""" + +from __future__ import annotations + +import argparse +import json +import math +from pathlib import Path +import sys + + +SCHEMA = "qbit.prism.load-bridge.v1" +REPORT_SCHEMA = "qbit.prism.load-harness.v1" +PRESETS = {"fake": "short-plan-fake-node", "real": "short-plan-real-node"} +STATEMENT = ( + "What the real node adds to the short plan on this runner and commit: real minus fake. " + "Not a D1 verdict (#487 decisions 1 and 5)." +) +MS = "milliseconds" +KIB = "KiB" +CLIENT_CLOCK = "client monotonic" +FAKE_HAS_NO_RELAY = "the fake node has no RPC relay and records no RPC latency" +FAKE_NODE_IN_HARNESS = "the fake node runs inside the harness process, whose RSS is not sampled" + + +def read_side(directory: Path, side: str) -> dict: + """One side's exit codes, report and host, or the reason each is missing.""" + result: dict = { + "preset": PRESETS[side], + "directory": str(directory), + "harness_exit_code": read_code(directory / "harness-exit-code"), + "gate_exit_code": read_code(directory / "gate-exit-code"), + "runner": None, + "report": None, + "report_error": None, + } + try: + host = json.loads((directory / "host.json").read_text(encoding="utf-8")) + result["runner"] = host.get("runner_label") if isinstance(host, dict) else None + except (OSError, ValueError): + pass + path = directory / "load-harness-report.json" + try: + report = json.loads(path.read_text(encoding="utf-8")) + except FileNotFoundError: + result["report_error"] = f"{path.name} is missing" + return result + except (OSError, ValueError) as error: + result["report_error"] = f"{path.name} is unreadable: {error}" + return result + if not isinstance(report, dict) or report.get("schema") != REPORT_SCHEMA: + schema = report.get("schema") if isinstance(report, dict) else None + result["report_error"] = f"{path.name} has schema {schema!r}, not {REPORT_SCHEMA}" + return result + name = (report.get("preset") or {}).get("name") + if name != PRESETS[side]: + result["report_error"] = f"{path.name} is from preset {name!r}, not {PRESETS[side]}" + return result + result["report"] = report + return result + + +def read_code(path: Path) -> int | None: + try: + return int(path.read_text(encoding="utf-8").strip()) + except (OSError, ValueError): + return None + + +def number(value: object) -> float | None: + if isinstance(value, bool) or not isinstance(value, (int, float)): + return None + return float(value) if math.isfinite(value) else None + + +class Figure: + """A measured value, or None with the reason it is missing.""" + + def __init__(self, value: float | None, reason: str | None = None) -> None: + self.value = value + self.reason = None if value is not None else (reason or "not reported") + + +def missing_report(side: dict) -> Figure | None: + if side["report"] is None: + return Figure(None, side["report_error"]) + return None + + +def phase_names(side: dict) -> list[str]: + report = side["report"] or {} + return [ + phase["name"] + for phase in report.get("phases") or [] + if isinstance(phase, dict) and isinstance(phase.get("name"), str) + ] + + +def ack_latency(side: dict, phase_name: str, statistic: str) -> Figure: + missing = missing_report(side) + if missing: + return missing + phase = next( + (p for p in side["report"].get("phases") or [] if isinstance(p, dict) and p.get("name") == phase_name), + None, + ) + if phase is None: + return Figure(None, f"the run has no {phase_name} phase") + latency = phase.get("client_ack_latency") or {} + if latency.get("unit") not in (None, MS): + return Figure(None, f"client_ack_latency is in {latency.get('unit')!r}, not {MS}") + value = number(latency.get(statistic)) + return Figure(value, latency.get("unavailable_reason") or f"client_ack_latency.{statistic} is null") + + +def tip_last_notify(side: dict, statistic: str) -> Figure: + """p50 or max over the external tips of the slowest session's time. + + A tip some session never got work on has no figure, so neither does the + statistic: the gate reads that as a failure, and a partial p50 would + report the run faster than it was. + """ + missing = missing_report(side) + if missing: + return missing + tips = (side["report"].get("time_to_usable_work") or {}).get("tips") + if not isinstance(tips, list) or not tips: + return Figure(None, "the run reported no external tip") + values = [] + for index, tip in enumerate(tips): + value = number(tip.get("all_sessions_milliseconds")) if isinstance(tip, dict) else None + if value is None: + reason = tip.get("all_sessions_unavailable_reason") if isinstance(tip, dict) else None + return Figure(None, f"tip {index}: {reason or 'no figure'}") + values.append(value) + values.sort() + if statistic == "max": + return Figure(values[-1]) + rank = max(1, math.ceil(0.5 * len(values))) + return Figure(values[rank - 1]) + + +def relay_latency(side: dict, method: str, statistic: str) -> Figure: + missing = missing_report(side) + if missing: + return missing + node = side["report"].get("node") or {} + if node.get("mode") != "qbitd": + return Figure(None, FAKE_HAS_NO_RELAY) + summary = ((node.get("relay") or {}).get("rpc_latency_milliseconds") or {}).get(method) + if not isinstance(summary, dict) or not summary.get("count"): + return Figure(None, f"no {method} went through the relay") + return Figure(number(summary.get(statistic)), f"relay {method} {statistic} is null") + + +def frontend_peaks(side: dict) -> tuple[list[Figure], str | None]: + """Each frontend's peak RSS over the run, in frontend order. + + `VmHWM` is a kernel peak per process, so the run's peak is the largest + any phase saw; a frontend restarted in `reconnect` is a new process, and + the maximum covers both lifetimes. + """ + if side["report"] is None: + return [], side["report_error"] + count = (side["report"].get("topology") or {}).get("frontends") + if not isinstance(count, int) or count < 1: + return [], "the report does not say how many frontends ran" + peaks: list[list[float]] = [[] for _ in range(count)] + for phase in side["report"].get("phases") or []: + for index, process in enumerate((phase or {}).get("processes") or []): + value = number((process or {}).get("peak_rss_kib")) + if index < count and value is not None: + peaks[index].append(value) + return [ + Figure(max(values)) if values else Figure(None, f"frontend {index}: no peak RSS was sampled") + for index, values in enumerate(peaks) + ], None + + +def node_peak(side: dict) -> Figure: + missing = missing_report(side) + if missing: + return missing + node = side["report"].get("node") or {} + if node.get("mode") != "qbitd": + return Figure(None, FAKE_NODE_IN_HARNESS) + value = number(((node.get("qbitd") or {}).get("peak_rss_kib") or {}).get("pool")) + return Figure(value, "the pool qbitd's VmHWM could not be read") + + +def metric(name: str, unit: str, clock: str | None, fake: Figure, real: Figure) -> dict: + difference = None + reason = None + if fake.value is not None and real.value is not None: + difference = real.value - fake.value + else: + reasons = [f"{side}: {figure.reason}" for side, figure in (("fake", fake), ("real", real)) if figure.value is None] + reason = "; ".join(reasons) + return { + "metric": name, + "unit": unit, + "clock": clock, + "fake": fake.value, + "real": real.value, + "real_minus_fake": difference, + "unavailable_reason": reason, + } + + +def metrics(fake: dict, real: dict) -> list[dict]: + rows = [] + phases = phase_names(fake) + [name for name in phase_names(real) if name not in phase_names(fake)] + for phase in phases: + for statistic in ("p50", "p99"): + rows.append(metric( + f"client_ack_latency {statistic} ({phase})", MS, + "client monotonic, submit written to acknowledgement read", + ack_latency(fake, phase, statistic), ack_latency(real, phase, statistic), + )) + for statistic in ("p50", "max"): + rows.append(metric( + f"time to usable work, slowest session, {statistic} over external tips", MS, + "client monotonic against the node's tip stamp", + tip_last_notify(fake, statistic), tip_last_notify(real, statistic), + )) + for method in ("submitblock", "getblocktemplate"): + for statistic in ("p50", "p99"): + rows.append(metric( + f"node {method} latency {statistic}", MS, "relay monotonic, request forwarded to reply", + relay_latency(fake, method, statistic), relay_latency(real, method, statistic), + )) + fake_peaks, fake_error = frontend_peaks(fake) + real_peaks, real_error = frontend_peaks(real) + for index in range(max(len(fake_peaks), len(real_peaks), 1)): + rows.append(metric( + f"frontend {index} peak RSS", KIB, None, + fake_peaks[index] if index < len(fake_peaks) else Figure(None, fake_error or f"no frontend {index}"), + real_peaks[index] if index < len(real_peaks) else Figure(None, real_error or f"no frontend {index}"), + )) + rows.append(metric( + "frontends peak RSS, summed", KIB, None, summed(fake_peaks, fake_error), summed(real_peaks, real_error), + )) + rows.append(metric("pool node peak RSS", KIB, None, node_peak(fake), node_peak(real))) + return rows + + +def summed(peaks: list[Figure], error: str | None) -> Figure: + if error or not peaks: + return Figure(None, error or "no frontend peak RSS") + unknown = [figure.reason for figure in peaks if figure.value is None] + if unknown: + return Figure(None, "; ".join(unknown)) + return Figure(sum(figure.value for figure in peaks)) + + +def trust(sides: dict[str, dict]) -> list[str]: + """Why the row cannot be taken at face value; empty when it can.""" + reasons = [] + for name, side in sides.items(): + if side["report"] is None: + reasons.append(f"{name}: {side['report_error']}") + for key in ("harness_exit_code", "gate_exit_code"): + code = side[key] + if code != 0: + reasons.append(f"{name}: {key} is {'missing' if code is None else code}") + runners = {side["runner"] for side in sides.values()} + if len(runners) != 1 or None in runners: + reasons.append(f"the runner labels differ or are missing: {sorted(map(str, runners))}") + return reasons + + +def build_row(fake_dir: Path, real_dir: Path, order: list[str], commit: str | None, run_url: str | None) -> dict: + sides = {"fake": read_side(fake_dir, "fake"), "real": read_side(real_dir, "real")} + untrusted = trust(sides) + return { + "schema": SCHEMA, + "statement": STATEMENT, + "commit": commit, + "run_url": run_url, + "runner": sides["fake"]["runner"] if sides["fake"]["runner"] == sides["real"]["runner"] else None, + "order": order, + "trusted": not untrusted, + "untrusted_reasons": untrusted, + "sides": { + name: {key: value for key, value in side.items() if key != "report"} + for name, side in sides.items() + }, + "metrics": metrics(sides["fake"], sides["real"]), + } + + +def show(value: float | None, unit: str) -> str: + if value is None: + return "unknown" + if unit == KIB: + return f"{value / 1024:,.1f} MiB" + return f"{value:,.1f} ms" + + +def show_difference(value: float | None, unit: str) -> str: + if value is None: + return "unknown" + text = show(abs(value), unit) + return f"+{text}" if value > 0 else (f"-{text}" if value < 0 else text) + + +def markdown(row: dict) -> str: + lines = [ + "### Bridging: what the real node costs (real minus fake)", + "", + f"{row['statement']} Commit `{row['commit'] or 'unknown'}` on `{row['runner'] or 'unknown runner'}`; " + f"order {' then '.join(row['order'])}.", + "", + ] + if row["trusted"]: + lines.append("Both sides completed, reconciled and passed their gates.") + else: + lines.append("**Not trusted:** " + "; ".join(row["untrusted_reasons"]) + ".") + lines += ["", "| Metric | Fake | Real | Real − fake |", "|---|---|---|---|"] + notes = [] + for entry in row["metrics"]: + difference = show_difference(entry["real_minus_fake"], entry["unit"]) + if entry["unavailable_reason"]: + notes.append(entry["unavailable_reason"]) + difference += f" [{len(notes)}]" + lines.append( + f"| {entry['metric']} | {show(entry['fake'], entry['unit'])} | " + f"{show(entry['real'], entry['unit'])} | {difference} |" + ) + if notes: + lines.append("") + lines += [f"{index}. {note}" for index, note in enumerate(notes, start=1)] + return "\n".join(lines) + "\n\n" + + +def main(argv: list[str] | None = None) -> int: + parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) + parser.add_argument("--fake", type=Path, required=True, help="the short-plan-fake-node run directory") + parser.add_argument("--real", type=Path, required=True, help="the short-plan-real-node run directory") + parser.add_argument("--order", required=True, help="the order the sides ran in: fake,real or real,fake") + parser.add_argument("--out", type=Path, required=True, help="where to write the JSON row") + parser.add_argument("--summary", type=Path, help="a Markdown file to append the table to") + parser.add_argument("--commit") + parser.add_argument("--run-url") + args = parser.parse_args(argv) + order = [side.strip() for side in args.order.split(",")] + if sorted(order) != ["fake", "real"]: + print(f"prism-load-bridge: --order {args.order!r} is not fake,real or real,fake", file=sys.stderr) + return 2 + row = build_row(args.fake, args.real, order, args.commit, args.run_url) + try: + args.out.parent.mkdir(parents=True, exist_ok=True) + args.out.write_text(json.dumps(row, indent=2, sort_keys=True) + "\n", encoding="utf-8") + table = markdown(row) + if args.summary: + with args.summary.open("a", encoding="utf-8") as handle: + handle.write(table) + except OSError as error: + print(f"prism-load-bridge: {error}", file=sys.stderr) + return 2 + print(table, end="") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/prism_load_matrix.py b/scripts/prism_load_matrix.py new file mode 100644 index 000000000..da22f89db --- /dev/null +++ b/scripts/prism_load_matrix.py @@ -0,0 +1,230 @@ +#!/usr/bin/env python3 +"""Plan the nightly load-harness matrix from the checked-in presets (#521). + +Reads `crates/qbit-prism-load/presets/*.json` and prints the GitHub Actions +matrix for a selection: + +- `nightly` (the nightly schedule's selection): every preset whose schedule + is `nightly`; +- `weekly` (the weekly schedule's selection): every preset whose schedule is + `weekly`, the long soak (#575) and the long fault set (#554); +- `all`: every `nightly` and `manual` preset (not the weekly soak, which + takes most of a runner's six hours); +- `suite:<name>`: a suite from `presets/suites.toml` (#550), which names its + presets whatever their schedules, how many times each runs and, optionally, + one runner for all of them; +- otherwise a comma-separated list of preset names, any schedule. A name + listed in `presets/aliases.txt` (a preset's deprecated name) selects the + preset it was renamed to, with a warning. + +Each entry carries the preset's name, runner and timeout, so which machine a +preset runs on and for how long is checked in beside the flags it pins. A +suite's entries also carry `repeat` (1-based) and `id`, `<preset>-r<repeat>`, +one job per repeat, and the suite's runner where it sets one. + +Usage: python3 scripts/prism_load_matrix.py <selection> [--presets DIR] +""" + +from __future__ import annotations + +import argparse +import json +from pathlib import Path +import re +import sys +import tomllib + + +ROOT = Path(__file__).resolve().parents[1] +PRESETS = ROOT / "crates" / "qbit-prism-load" / "presets" +SCHEMA = "qbit.prism.load-preset.v1" +SUITES_SCHEMA = "qbit.prism.load-suites.v1" +SUITE_KEYS = {"description", "lane", "issues", "repeats", "runner", "presets"} +SUITE_REQUIRED = SUITE_KEYS - {"runner"} +MAX_REPEATS = 5 +ISSUE = re.compile(r"^#[1-9][0-9]*$") + + +class SelectionError(Exception): + """A selection the presets cannot satisfy, worded for the workflow log.""" + + +def load(directory: Path) -> dict[str, dict]: + presets = {} + for path in sorted(directory.glob("*.json")): + preset = json.loads(path.read_text(encoding="utf-8")) + if preset.get("schema") != SCHEMA: + raise SelectionError(f"{path}: schema is {preset.get('schema')!r}, not {SCHEMA}") + if preset.get("name") != path.stem: + raise SelectionError(f"{path}: name {preset.get('name')!r} is not the file's stem") + presets[path.stem] = preset + if not presets: + raise SelectionError(f"no presets in {directory}") + return presets + + +def load_aliases(directory: Path) -> dict[str, str]: + """Deprecated preset names, `old new` per line, from `aliases.txt`.""" + path = directory / "aliases.txt" + aliases = {} + if not path.exists(): + return aliases + for number, raw in enumerate(path.read_text(encoding="utf-8").splitlines(), start=1): + line = raw.split("#", 1)[0].strip() + if not line: + continue + fields = line.split() + if len(fields) != 2: + raise SelectionError(f"{path}:{number}: {raw!r} is not `old new`") + aliases[fields[0]] = fields[1] + return aliases + + +def load_suites(directory: Path, presets: dict[str, dict]) -> dict[str, dict]: + """The suites in `suites.toml`, each checked against the presets (#550). + + A suite names presets by their current names only, each once: a + deprecated name would stop resolving when its alias is dropped, and a + repeat belongs in `repeats`. An unknown key, a missing one, a schema other + than this script's or a `repeats` outside 1..5 is refused, so a suite + never runs other than as written. + """ + path = directory / "suites.toml" + if not path.exists(): + return {} + try: + with path.open("rb") as handle: + data = tomllib.load(handle) + except tomllib.TOMLDecodeError as error: + raise SelectionError(f"{path}: {error}") from error + if data.get("schema") != SUITES_SCHEMA: + raise SelectionError(f"{path}: schema is {data.get('schema')!r}, not {SUITES_SCHEMA}") + extra = sorted(set(data) - {"schema", "suites"}) + if extra: + raise SelectionError(f"{path}: unknown top-level key(s) {', '.join(extra)}") + suites = data.get("suites") + if not isinstance(suites, dict) or not suites: + raise SelectionError(f"{path}: no [suites.<name>] tables") + for name, suite in suites.items(): + where = f"{path}: suite {name}" + if not isinstance(suite, dict): + raise SelectionError(f"{where} is not a table") + unknown = sorted(set(suite) - SUITE_KEYS) + if unknown: + raise SelectionError(f"{where}: unknown key(s) {', '.join(unknown)}") + missing = sorted(SUITE_REQUIRED - set(suite)) + if missing: + raise SelectionError(f"{where}: missing {', '.join(missing)}") + for key in ("description", "lane"): + if not (isinstance(suite[key], str) and suite[key].strip()): + raise SelectionError(f"{where}: {key} must be a non-empty string") + if "runner" in suite and not ( + isinstance(suite["runner"], str) and suite["runner"].startswith("blacksmith-") + ): + raise SelectionError(f"{where}: runner must be a blacksmith- runner label") + issues = suite["issues"] + if not (isinstance(issues, list) and issues and all( + isinstance(i, str) and ISSUE.match(i) for i in issues + )): + raise SelectionError(f"{where}: issues must be a non-empty list of #<issue>") + repeats = suite["repeats"] + if not (type(repeats) is int and 1 <= repeats <= MAX_REPEATS): + raise SelectionError(f"{where}: repeats must be an integer 1..{MAX_REPEATS}") + names = suite["presets"] + if not (isinstance(names, list) and names and all(isinstance(n, str) for n in names)): + raise SelectionError(f"{where}: presets must be a non-empty list of names") + if len(set(names)) != len(names): + raise SelectionError(f"{where}: presets repeat a name; use repeats") + unknown = [n for n in names if n not in presets] + if unknown: + raise SelectionError(f"{where}: unknown preset(s) {', '.join(unknown)}") + return suites + + +def select_suite(presets: dict[str, dict], suites: dict[str, dict], name: str) -> list[dict]: + suite = suites.get(name) + if suite is None: + raise SelectionError( + f"unknown suite {name!r}; choose from {', '.join(suites) or 'none (no suites.toml)'}" + ) + return [ + { + "preset": preset, + "runner": suite.get("runner", presets[preset]["runner"]), + "timeout_minutes": presets[preset]["timeout_minutes"], + "repeat": repeat, + "id": f"{preset}-r{repeat}", + } + for preset in suite["presets"] + for repeat in range(1, suite["repeats"] + 1) + ] + + +def select( + presets: dict[str, dict], + selection: str, + aliases: dict[str, str] | None = None, + suites: dict[str, dict] | None = None, +) -> list[dict]: + selection = selection.strip() + if selection.startswith("suite:"): + return select_suite(presets, suites or {}, selection.removeprefix("suite:").strip()) + if selection in ("nightly", "weekly"): + names = [name for name, p in presets.items() if p["schedule"] == selection] + elif selection == "all": + names = [name for name, p in presets.items() if p["schedule"] in ("nightly", "manual")] + else: + names = [] + for name in (name.strip() for name in selection.split(",")): + if not name: + continue + if name not in presets and name in (aliases or {}): + print( + f"prism-load-matrix: {name} is deprecated; running {aliases[name]}", + file=sys.stderr, + ) + name = aliases[name] + names.append(name) + unknown = [name for name in names if name not in presets] + if unknown: + raise SelectionError( + f"unknown preset(s) {', '.join(unknown)}; choose from {', '.join(presets)}" + ) + if not names: + raise SelectionError(f"selection {selection!r} names no preset") + return [ + { + "preset": name, + "runner": presets[name]["runner"], + "timeout_minutes": presets[name]["timeout_minutes"], + } + for name in sorted(set(names)) + ] + + +def main(argv: list[str] | None = None) -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("selection") + parser.add_argument("--presets", type=Path, default=PRESETS) + args = parser.parse_args(argv) + try: + presets = load(args.presets) + # Only a suite selection reads suites.toml, so the nightly's plan + # never depends on the L3 suites. + suites = ( + load_suites(args.presets, presets) + if args.selection.strip().startswith("suite:") + else {} + ) + matrix = { + "include": select(presets, args.selection, load_aliases(args.presets), suites) + } + except SelectionError as error: + print(f"prism-load-matrix: {error}", file=sys.stderr) + return 2 + print(json.dumps(matrix, separators=(",", ":"))) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/prism_load_probe.py b/scripts/prism_load_probe.py new file mode 100644 index 000000000..b585308c6 --- /dev/null +++ b/scripts/prism_load_probe.py @@ -0,0 +1,1440 @@ +#!/usr/bin/env python3 +"""The Blacksmith runner probe's helpers (#541 and #542, S1 and S2 of #487). + +`.github/workflows/prism-load-runner-probe.yml` measures what each runner +class is before any lane depends on it, and how noisy the harness is on it. +This script is every part of that workflow that is not a shell one-liner, so +it can be tested: + +- `matrix`: the probe and fsync-spread job matrices from the dispatch inputs, + refusing a class, a cache backend, a preset or a repeat count the workflow + cannot run, and a plan whose jobs would outlast the per-job ceiling. It + also prints the plan's spend ceiling; +- `measure`: run a command, sample the host's memory once a second while it + runs, and write its wall time, exit code and peak memory as JSON. With + `--timeout-seconds` it ends the command's whole process group at the + deadline and records `timed_out`. It exits with the command's code (124 on + a timeout); +- `row`: one probe job's result from the measurements, the harness's side + reports, the gate's exit codes and `pg_test_fsync`'s output, for each + preset and each in-job repeat; +- `fsync-row`: one fsync-spread VM's `pg_test_fsync` result; +- `host`: the host fingerprint, with what the kernel says about the block + device under a directory (its write cache and FUA support), which is the + evidence for whether `fsync` reaches stable storage; +- `table`: the per-class results table (Markdown) from every row; +- `variance`: per class, preset and fsync-cost band, the median, MAD and CV + of each headline metric over every run, over the per-VM medians (VM-to-VM) + and within each VM (run-to-run), as the JSON document `VARIANCE_SCHEMA` + names (#542; the S6 regression rule in #551 reads it), with the runner-size + evidence per class. A run that failed or never reported is unknown there, + never a zero; +- `verdict`: fail when the build or any planned run did not exit 0. + +Peak memory is the host's (MemTotal - MemAvailable), not one process's: the +harness, its frontends and both PostgreSQL clusters share the VM, and what +sizes a runner is whether all of them fit (#487 lesson 3). + +Usage: + prism_load_probe.py matrix --classes 8,16,32 --target-cache both --fsync-vms 3 \ + [--presets NAME,...] [--repeats N] [--in-job-repeats N] + prism_load_probe.py measure --out build.json [--timeout-seconds S] -- cargo build ... + prism_load_probe.py row --out row.json --class 8 ... --run PRESET=DIR [--run PRESET=DIR2 ...] + prism_load_probe.py fsync-row --out row.json --class 8 --vm 1 --fsync FILE + prism_load_probe.py host --out host.json --dir "$RUNNER_TEMP/pload" [--runner LABEL] + prism_load_probe.py table DIR [--expected FILE] + prism_load_probe.py variance DIR [--expected FILE] [--out FILE] [--markdown] + prism_load_probe.py verdict --probe DIR --presets NAME,... --in-job-repeats N +""" + +from __future__ import annotations + +import argparse +import json +import math +import os +from pathlib import Path +import re +import signal +import statistics +import subprocess +import sys +import time + +import prism_load_matrix + +# v2 (#542) adds the job's repeat index and commit and makes each preset's +# run a list, one entry per in-job repeat. v1 rows (#541) are still read: +# `load_rows` upgrades them to one repeat and one run per preset. +ROW_SCHEMA = "qbit.prism.runner-probe-row.v2" +ROW_SCHEMA_V1 = "qbit.prism.runner-probe-row.v1" +ROW_SCHEMA_PREFIX = "qbit.prism.runner-probe-row." +VARIANCE_SCHEMA = "qbit.prism.runner-probe-variance.v1" +# Blacksmith's x64 Ubuntu 24.04 classes; the probe is for 8, 16 and 32. +CLASSES = (2, 4, 8, 16, 32) +TARGET_CACHES = ("actions-cache", "sticky-disk") +MAX_FSYNC_VMS = 10 +# #542 asks for at least 10 runs of one commit per class; 20 separate jobs +# is the most one dispatch may ask for per class and cache backend. +MAX_REPEATS = 20 +MAX_IN_JOB_REPEATS = 5 +# A dispatch that would start more probe jobs than this is refused before +# any runner is spent: it is a typo more often than a plan. +MAX_PROBE_JOBS = 64 +# The default presets (#541: one short and one D1 at 20k). +PRESETS = ("short-plan-20k-window-1fe", "throughput-20k-window-1fe") +# A probe job's budget in minutes: setup (checkout, toolchain, PostgreSQL, +# the target restore), the release build, and the rows and uploads after +# the runs, around each run's own preset timeout. The job's timeout is their +# sum, so a run that hangs ends at its preset's timeout and the job still +# writes and uploads its row. +SETUP_MINUTES = 15 +BUILD_MINUTES = 60 +REPORT_MINUTES = 15 +# Slack between the runs' summed timeouts and the run step's own timeout: a +# fixed allowance, plus one minute per run for the cleanup after a run that +# timed out (end_group's 30 s SIGTERM grace and 5 s SIGKILL wait, the +# postmaster kill and the cluster removal), so every run can time out and +# the last one still writes its measurement. +RUN_STEP_SLACK_MINUTES = 5 +CLEANUP_MINUTES_PER_RUN = 1 +# The longest probe job the plan accepts. GitHub's hosted-runner limit is +# 360 minutes; Blacksmith's is what #541's duration job measures, so the +# probe stays inside the lower one until that is known. +MAX_JOB_MINUTES = 360 +FSYNC_JOB_MINUTES = 20 +# The plan and collate jobs, on 2 vCPU, for the spend ceiling. +OVERHEAD_JOB_MINUTES = 10 + 15 +# Blacksmith's list price per 2 vCPU x64 minute, larger classes billed in +# proportion (#487's cost section; S1 confirms the multiplier). +PRICE_PER_2VCPU_MINUTE_USD = 0.004 +# PostgreSQL's default wal_sync_method on Linux, so the line whose cost a +# commit pays. +WAL_SYNC_METHOD = "fdatasync" + + +class ProbeError(Exception): + """An input the probe cannot use, worded for the workflow log.""" + + +def runner_label(size: int) -> str: + return f"blacksmith-{size}vcpu-ubuntu-2404" + + +# --- matrix ------------------------------------------------------------- + + +def parse_classes(text: str) -> list[int]: + sizes = [] + for item in text.split(","): + item = item.strip() + if not item: + continue + if not item.isdigit() or int(item) not in CLASSES: + raise ProbeError( + f"runner class {item!r} is not one of {', '.join(map(str, CLASSES))}" + ) + sizes.append(int(item)) + if not sizes: + raise ProbeError(f"classes {text!r} names no runner class") + return sorted(set(sizes)) + + +def parse_target_caches(text: str) -> list[str]: + text = text.strip() + if text == "both": + return list(TARGET_CACHES) + if text not in TARGET_CACHES: + raise ProbeError( + f"target cache {text!r} is not one of both, {', '.join(TARGET_CACHES)}" + ) + return [text] + + +def parse_count(text: str, name: str, low: int, high: int) -> int: + """A whole number in low..high, read as decimal (a leading zero is not + octal here, and a sign, a fraction or an exponent is refused).""" + text = text.strip() + if not re.fullmatch(r"[0-9]{1,4}", text) or not low <= int(text) <= high: + raise ProbeError(f"{name} {text!r} is not a whole number {low}..{high}") + return int(text) + + +def parse_fsync_vms(text: str) -> int: + return parse_count(text, "fsync VMs", 0, MAX_FSYNC_VMS) + + +def load_presets(directory: Path | None = None) -> tuple[dict[str, dict], dict[str, str]]: + directory = directory or prism_load_matrix.PRESETS + try: + return (prism_load_matrix.load(directory), prism_load_matrix.load_aliases(directory)) + except prism_load_matrix.SelectionError as error: + raise ProbeError(str(error)) from error + + +def parse_presets(text: str, presets: dict[str, dict], aliases: dict[str, str]) -> list[str]: + """Checked-in preset names, in the order given, a deprecated name + resolved to the preset it was renamed to. The probe builds no qbitd, so a + real-node preset is refused here rather than failing on the runner.""" + names: list[str] = [] + unknown = [] + for item in text.split(","): + name = item.strip() + if not name: + continue + if name not in presets and name in aliases: + print(f"prism_load_probe: {name} is deprecated; running {aliases[name]}", + file=sys.stderr) + name = aliases[name] + if name not in presets: + unknown.append(name) + elif name not in names: + names.append(name) + if unknown: + raise ProbeError( + f"unknown preset(s) {', '.join(unknown)}; choose from {', '.join(presets)}" + ) + if not names: + raise ProbeError(f"presets {text!r} names no preset") + real_node = [n for n in names if (presets[n].get("args") or {}).get("--node", "fake") != "fake"] + if real_node: + raise ProbeError( + f"preset(s) {', '.join(real_node)} need a real node, which the probe does not build" + ) + return names + + +def preset_minutes(preset: dict) -> int: + minutes = preset.get("timeout_minutes") + if not isinstance(minutes, int) or isinstance(minutes, bool) or minutes <= 0: + raise ProbeError(f"preset {preset.get('name')!r} has no positive timeout_minutes") + return minutes + + +def matrices( + classes: str, + target_cache: str, + fsync_vms: str, + presets: str = ",".join(PRESETS), + repeats: str = "1", + in_job_repeats: str = "1", + presets_dir: Path | None = None, +) -> dict: + sizes = parse_classes(classes) + caches = parse_target_caches(target_cache) + vms = parse_fsync_vms(fsync_vms) + jobs_per_cell = parse_count(repeats, "repeats", 1, MAX_REPEATS) + iterations = parse_count(in_job_repeats, "in-job repeats", 1, MAX_IN_JOB_REPEATS) + known, aliases = load_presets(presets_dir) + names = parse_presets(presets, known, aliases) + run_minutes = {name: preset_minutes(known[name]) for name in names} + runs = len(names) * iterations + runs_minutes = (sum(run_minutes.values()) * iterations + RUN_STEP_SLACK_MINUTES + + CLEANUP_MINUTES_PER_RUN * runs) + job_minutes = SETUP_MINUTES + BUILD_MINUTES + runs_minutes + REPORT_MINUTES + if job_minutes > MAX_JOB_MINUTES: + overhead = job_minutes - sum(run_minutes.values()) * iterations + raise ProbeError( + f"a probe job would need {job_minutes} minutes ({len(names)} preset(s) x " + f"{iterations} in-job repeat(s) at their timeouts, plus {overhead} " + f"for setup, build and reports), over the {MAX_JOB_MINUTES}-minute ceiling; " + "choose fewer presets or in-job repeats and dispatch the rest separately" + ) + probe = [ + {"class": size, "runner": runner_label(size), "target_cache": cache, "repeat": repeat} + for size in sizes + for cache in caches + for repeat in range(1, jobs_per_cell + 1) + ] + if len(probe) > MAX_PROBE_JOBS: + raise ProbeError( + f"the plan asks for {len(probe)} probe jobs ({len(sizes)} class(es) x " + f"{len(caches)} cache backend(s) x {jobs_per_cell} repeat(s)), over {MAX_PROBE_JOBS}" + ) + fsync = [ + {"class": size, "runner": runner_label(size), "vm": vm} + for size in sizes + for vm in range(1, vms + 1) + ] + two_vcpu_minutes = ( + sum(job_minutes * e["class"] // 2 for e in probe) + + sum(FSYNC_JOB_MINUTES * e["class"] // 2 for e in fsync) + + OVERHEAD_JOB_MINUTES + ) + # An empty include list is not a valid matrix, so an fsync spread of 0 + # VMs is carried as a flag the workflow's `if:` reads instead. + return { + "probe": {"include": probe}, + "fsync": {"include": fsync or [{"class": 0, "runner": "", "vm": 0}]}, + "fsync_enabled": bool(fsync), + "presets": ",".join(names), + "in_job_repeats": iterations, + "runs_minutes": runs_minutes, + "job_minutes": job_minutes, + # What the jobs' timeouts cap, not what the runs are expected to + # cost; the duration hold is not included. + "ceiling": { + "probe_jobs": len(probe), + "fsync_jobs": len(fsync), + "two_vcpu_minutes": two_vcpu_minutes, + "usd": round(two_vcpu_minutes * PRICE_PER_2VCPU_MINUTE_USD, 2), + "price_per_two_vcpu_minute_usd": PRICE_PER_2VCPU_MINUTE_USD, + }, + } + + +def plan_summary(plan: dict) -> str: + ceiling = plan["ceiling"] + return "\n".join([ + "### Probe plan", + "", + f"- {ceiling['probe_jobs']} probe job(s): presets {plan['presets']}, " + f"{plan['in_job_repeats']} run(s) of each per job; each job times out at " + f"{plan['job_minutes']} min (runs {plan['runs_minutes']} min).", + f"- {ceiling['fsync_jobs']} fsync-spread job(s).", + f"- Spend ceiling: {ceiling['two_vcpu_minutes']:,} 2 vCPU minutes, about " + f"${ceiling['usd']:,.2f} at ${ceiling['price_per_two_vcpu_minute_usd']} per 2 vCPU " + "minute (list price, before free minutes; every job at its timeout, the " + "duration hold not included). Actual spend is the jobs' billed minutes.", + "", + ]) + + +# --- measure ------------------------------------------------------------ + + +def meminfo_kib(text: str) -> dict[str, int]: + values = {} + for line in text.splitlines(): + key, _, rest = line.partition(":") + fields = rest.split() + if fields and fields[0].isdigit(): + values[key.strip()] = int(fields[0]) + return values + + +def read_meminfo() -> dict[str, int]: + try: + return meminfo_kib(Path("/proc/meminfo").read_text(encoding="ascii")) + except OSError: + return {} + + +def used_kib(info: dict[str, int]) -> int | None: + if "MemTotal" in info and "MemAvailable" in info: + return info["MemTotal"] - info["MemAvailable"] + return None + + +def mib(kib: int | None) -> float | None: + return None if kib is None else round(kib / 1024, 1) + + +def process_state(pid: int | str) -> tuple[str, int] | None: + """(state, process group) from /proc/<pid>/stat, or None when it is gone.""" + try: + text = Path(f"/proc/{pid}/stat").read_text(encoding="ascii", errors="replace") + except OSError: + return None + fields = text[text.rindex(")") + 2:].split() + return fields[0], int(fields[2]) + + +def group_alive(pgid: int) -> bool: + """Whether any process of the group is still running. A zombie is not: + an orphan whose PID 1 does not reap it promptly stays in the group + as a zombie, and a kill(0) probe would wait on it for the whole grace.""" + proc = Path("/proc") + if proc.is_dir(): + for entry in proc.iterdir(): + if entry.name.isdigit(): + state = process_state(entry.name) + if state is not None and state[1] == pgid and state[0] not in ("Z", "X"): + return True + return False + try: + os.killpg(pgid, 0) + except ProcessLookupError: + return False + except PermissionError: + return True + return True + + +def end_group(child: subprocess.Popen, grace: float = 30.0) -> None: + """SIGTERM the child's process group, then SIGKILL what is left after + `grace` seconds: what a run that timed out started must not outlive it + and skew the next run's memory and CPU. (A PostgreSQL postmaster that + `pg_ctl` put in a session of its own is outside the group; the workflow + ends those by their data directory.)""" + for sig, wait in ((signal.SIGTERM, grace), (signal.SIGKILL, 5.0)): + try: + os.killpg(child.pid, sig) + except ProcessLookupError: + break + deadline = time.monotonic() + wait + while time.monotonic() < deadline: + child.poll() # reap the leader, so a zombie is not counted as alive + if not group_alive(child.pid): + return + time.sleep(0.2) + child.poll() + + +def measure(command: list[str], interval: float = 1.0, + timeout_seconds: float | None = None) -> dict: + baseline = read_meminfo() + samples = [used_kib(baseline)] + started = time.monotonic() + # Its own process group, so a timeout, or the step being cancelled, ends + # everything the command started and not just its first process. + child = subprocess.Popen(command, start_new_session=True) + + def forward(signum, _frame): + end_group(child, grace=10.0) + raise SystemExit(128 + signum) + + previous = {sig: signal.signal(sig, forward) for sig in (signal.SIGTERM, signal.SIGINT)} + timed_out = False + try: + while True: + try: + code = child.wait(timeout=interval) + break + except subprocess.TimeoutExpired: + samples.append(used_kib(read_meminfo())) + if timeout_seconds is not None and time.monotonic() - started >= timeout_seconds: + timed_out = True + end_group(child) + code = child.wait() + break + finally: + for sig, handler in previous.items(): + signal.signal(sig, handler) + wall = time.monotonic() - started + samples.append(used_kib(read_meminfo())) + known = [s for s in samples if s is not None] + return { + "command": command, + "exit_code": code, + "timed_out": timed_out, + "timeout_seconds": timeout_seconds, + "wall_seconds": round(wall, 1), + "samples": len(known), + "mem_total_mib": mib(baseline.get("MemTotal")), + "baseline_used_mib": mib(samples[0]), + # None, not 0, when /proc/meminfo could not be read (EP-OBSERVABILITY). + "peak_used_mib": mib(max(known)) if known else None, + } + + +# --- host --------------------------------------------------------------- + + +def read_text(path: Path) -> str | None: + try: + return path.read_text(encoding="utf-8", errors="replace").strip() + except OSError: + return None + + +def command_output(command: list[str]) -> str | None: + try: + done = subprocess.run(command, capture_output=True, text=True, timeout=30, check=False) + except (OSError, subprocess.TimeoutExpired): + return None + return done.stdout.strip() if done.returncode == 0 else None + + +def block_device(source: str | None) -> dict: + """The kernel's view of the device a filesystem sits on: `write back` in + write_cache with fua 0 means a flush has to be sent to reach media, and + whether the hypervisor honours it is what the probe cannot see from here.""" + if not source or not source.startswith("/dev/"): + return {"source": source} + name = Path(source).resolve().name + sys_block = Path("/sys/class/block") / name + queue = sys_block / "queue" + if not queue.exists(): + # A partition's queue is its parent disk's. + queue = sys_block.resolve().parent / "queue" + return { + "source": source, + "device": name, + "write_cache": read_text(queue / "write_cache"), + "fua": read_text(queue / "fua"), + "rotational": read_text(queue / "rotational"), + "model": read_text(sys_block.resolve().parent / "device" / "model") + or read_text(sys_block / "device" / "model"), + } + + +def host_facts(directory: Path) -> dict: + cpuinfo = read_text(Path("/proc/cpuinfo")) or "" + model = next((line.split(":", 1)[1].strip() for line in cpuinfo.splitlines() + if line.startswith("model name")), None) + info = read_meminfo() + mount = command_output(["findmnt", "-no", "SOURCE,FSTYPE,OPTIONS", "-T", str(directory)]) + source, fstype, options = (mount.split(None, 2) + [None, None, None])[:3] if mount else (None, None, None) + return { + "nproc": len([line for line in cpuinfo.splitlines() if line.startswith("processor")]) or None, + "cpu_model": model, + "mem_total_mib": mib(info.get("MemTotal")), + "kernel": read_text(Path("/proc/sys/kernel/osrelease")), + "directory": str(directory), + "filesystem": {"type": fstype, "options": options}, + "block_device": block_device(source), + } + + +# --- pg_test_fsync ------------------------------------------------------ + + +def parse_pg_test_fsync(text: str) -> dict: + """The one-8kB-write section's fdatasync line: the WAL flush a commit pays. + + Returns ops_per_second and usecs_per_op, both None when the output has no + such line (the method is unsupported) or is marked failed: the probe and + prism-load-run.sh append a line starting `pg_test_fsync failed` when it + exits nonzero, and a VM whose run failed after printing the one-write + section is unknown, not a sample.""" + if re.search(r"^pg_test_fsync failed", text, re.M): + return {"method": WAL_SYNC_METHOD, "ops_per_second": None, "usecs_per_op": None} + section = re.search( + r"Compare file sync methods using one 8kB write:(.*?)(?:\n\s*\n\S|\Z)", text, re.S + ) + body = section.group(1) if section else "" + match = re.search( + rf"^\s*{WAL_SYNC_METHOD}\s+([\d.]+)\s+ops/sec\s+([\d.]+)\s+usecs/op", body, re.M + ) + if not match: + return {"method": WAL_SYNC_METHOD, "ops_per_second": None, "usecs_per_op": None} + return { + "method": WAL_SYNC_METHOD, + "ops_per_second": float(match.group(1)), + "usecs_per_op": float(match.group(2)), + } + + +def read_fsync(path: Path) -> dict: + try: + return parse_pg_test_fsync(path.read_text(encoding="utf-8", errors="replace")) + except OSError: + return parse_pg_test_fsync("") + + +# --- the harness's outputs ---------------------------------------------- + + +def nearest_rank(values: list[float], quantile: float) -> float | None: + """The gate's nearest-rank percentile (qbit-prism-load's gate.rs).""" + if not values: + return None + ordered = sorted(values) + rank = min(max(math.ceil(quantile * len(ordered)), 1), len(ordered)) + return ordered[rank - 1] + + +def read_json(path: Path) -> dict | None: + try: + return json.loads(path.read_text(encoding="utf-8")) + except (OSError, ValueError): + return None + + +def read_int(path: Path) -> int | None: + try: + return int(path.read_text(encoding="ascii").strip()) + except (OSError, ValueError): + return None + + +def provenance(report: dict | None) -> str: + """`pass` when the run earned a qualification artifact with no override. + + The probe passes no --allow-* flag, so a refusal ends the run before it + writes a report; a report whose artifact_kind is `example` means one of + the checks was bypassed some other way.""" + if report is None: + return "no report (refused or failed before measuring)" + evidence = (report.get("versions") or {}).get("server_revision_evidence") or {} + problems = [] + if report.get("dirty"): + problems.append("dirty tree") + if evidence.get("status") != "established": + problems.append(f"revision {evidence.get('status', 'unknown')}") + if (report.get("versions") or {}).get("server_build_profile") != "release": + problems.append("server not release") + if report.get("artifact_kind") != "qualification": + problems.append(f"artifact_kind {report.get('artifact_kind')}") + return "pass" if not problems else "fail: " + ", ".join(problems) + + +def finite(value) -> float | int | None: + """A reported number, or None: JSON may carry NaN or Infinity, and a + boolean is not a measurement.""" + if isinstance(value, bool) or not isinstance(value, (int, float)): + return None + return value if math.isfinite(value) else None + + +def latency_ms(summary, key: str) -> float | None: + """A client latency percentile, only when the report says it is in + milliseconds: a value in another unit is unknown here, not rescaled.""" + if not isinstance(summary, dict) or summary.get("unit") != "milliseconds": + return None + return finite(summary.get(key)) + + +def unmeasured_reason(result: dict) -> str | None: + """Why a run's numbers are not a sample, or None when they are. Only a + harness exit 0 with a report, every provenance check passed and a gate + verdict is a measurement; a gate that failed on it (exit 1) still measured it, and + its shortfall is exactly what the runner-size pick reads.""" + if result.get("timed_out"): + return "timed out" + code = result.get("harness_exit_code") + if code is None: + return "no harness exit code" + if code != 0: + return f"harness exit {code}" + if result.get("provenance") != "pass": + return f"provenance {result.get('provenance')}" + # prism-load-run.sh's status is the gate's: 0 or 1 is a verdict, anything + # else (2, a panic, a gate that would not start) is not. + gate = result.get("gate_exit_code") + if gate not in (0, 1) or isinstance(gate, bool): + return f"gate exit {gate}" + return None + + +def run_result(directory: Path) -> dict: + report = read_json(directory / "load-harness-report.json") + measured = read_json(directory / "probe-measure.json") or {} + peak = finite(measured.get("peak_used_mib")) + total = finite(measured.get("mem_total_mib")) + result = { + "harness_exit_code": read_int(directory / "harness-exit-code"), + "gate_exit_code": measured.get("exit_code"), + "timed_out": measured.get("timed_out"), + "wall_seconds": measured.get("wall_seconds"), + "peak_used_mib": peak, + "baseline_used_mib": measured.get("baseline_used_mib"), + "mem_total_mib": total, + "headroom_mib": None if peak is None or total is None else round(total - peak, 1), + "fsync": read_fsync(directory / "pg_test_fsync.txt"), + "provenance": provenance(report), + "seed_seconds": None, + "seed_rows": None, + "phases": {}, + "tip_last_notify_p99_ms": None, + "tips_missing_a_session": None, + } + if report is not None: + seed = ((report.get("window") or {}).get("seed")) or {} + result["seed_seconds"] = seed.get("seconds") + result["seed_rows"] = seed.get("rows") + for phase in report.get("phases") or []: + ack = phase.get("client_ack_latency") + result["phases"][phase.get("name")] = { + "seconds": phase.get("duration_seconds"), + "shortfall": phase.get("shortfall"), + "completed": phase.get("completed"), + "achieved_rate_shares_per_second": finite( + phase.get("achieved_rate_shares_per_second")), + "ack_p50_ms": latency_ms(ack, "p50"), + "ack_p99_ms": latency_ms(ack, "p99"), + } + tips = ((report.get("time_to_usable_work") or {}).get("tips")) or [] + slowest = [t["all_sessions_milliseconds"] for t in tips + if isinstance(t.get("all_sessions_milliseconds"), (int, float))] + # As the gate does, a tip some session never got work on leaves the p99 + # unmeasured: a percentile over the fully served tips alone would + # understate delivery. + result["tips_missing_a_session"] = len(tips) - len(slowest) + if not result["tips_missing_a_session"]: + result["tip_last_notify_p99_ms"] = nearest_rank(slowest, 0.99) + result["unmeasured_reason"] = unmeasured_reason(result) + return result + + +def parse_run(text: str, known: set[str] | None = None) -> tuple[str, Path]: + known = set(PRESETS) | (known or set()) + name, sep, directory = text.partition("=") + if not sep or not directory or name not in known: + raise ProbeError(f"--run {text!r} is not <preset>=<directory> for a checked-in preset") + return name, Path(directory) + + +def build_row(args: argparse.Namespace, known: set[str] | None = None) -> dict: + # A preset given more than once is one run per in-job repeat, in order. + runs: dict[str, list[dict]] = {} + for text in args.run: + name, directory = parse_run(text, known) + runs.setdefault(name, []).append(run_result(directory)) + return { + "schema": ROW_SCHEMA, + "kind": "probe", + "class": args.size, + "runner": args.runner, + "target_cache": args.target_cache, + "repeat": args.repeat, + "commit": args.commit or None, + "target": { + "restored": args.target_restored == "true", + "bytes_before_build": args.target_bytes_before, + "bytes_after_build": args.target_bytes_after, + "restore_seconds": args.restore_seconds, + "save_seconds": args.save_seconds, + }, + "build": read_json(args.build) or {}, + "host": read_json(args.host) or {}, + "runs": runs, + } + + +def build_fsync_row(args: argparse.Namespace) -> dict: + return { + "schema": ROW_SCHEMA, + "kind": "fsync", + "class": args.size, + "runner": args.runner, + "vm": args.vm, + "host": read_json(args.host) or {}, + "fsync": read_fsync(args.fsync), + } + + +def upgrade_row(row: dict, path: Path) -> dict: + """A row in the current shape. A v1 row (#541) is one repeat whose + presets each ran once, with no commit recorded; any other runner-probe + schema is refused rather than misread.""" + schema = row.get("schema") + if schema == ROW_SCHEMA: + return row + if schema == ROW_SCHEMA_V1: + row = dict(row, schema=ROW_SCHEMA) + if row.get("kind") == "probe": + row["repeat"] = 1 + row["commit"] = None + # v1 runs carry no memory total, so their headroom stays unknown, + # and whether each is a sample is decided here as for v2 runs. + row["runs"] = {name: [dict(run, unmeasured_reason=unmeasured_reason(run))] + for name, run in (row.get("runs") or {}).items()} + return row + raise ProbeError(f"{path}: row schema {schema!r} is not {ROW_SCHEMA} or {ROW_SCHEMA_V1}") + + +# --- table -------------------------------------------------------------- + + +def cell(value, suffix: str = "") -> str: + if value is None: + return "unknown" + if isinstance(value, float): + value = f"{value:,.1f}" if value < 1000 else f"{value:,.0f}" + return f"{value}{suffix}" + + +def disk_line(host: dict) -> str: + fs = host.get("filesystem") or {} + dev = host.get("block_device") or {} + return (f"{cell(fs.get('type'))} on {cell(dev.get('source'))} " + f"(write_cache {cell(dev.get('write_cache'))}, fua {cell(dev.get('fua'))}, " + f"rotational {cell(dev.get('rotational'))}, model {cell(dev.get('model'))}; " + f"options {cell(fs.get('options'))})") + + +def load_rows(directory: Path) -> list[dict]: + rows = [] + for path in sorted(directory.rglob("*.json")): + row = read_json(path) + if isinstance(row, dict) and str(row.get("schema", "")).startswith(ROW_SCHEMA_PREFIX): + rows.append(upgrade_row(row, path)) + return rows + + +def expected_presets(expected: dict | None) -> list[str]: + presets = (expected or {}).get("presets") + if isinstance(presets, str) and presets: + return presets.split(",") + return list(PRESETS) + + +def expected_iterations(expected: dict | None) -> int: + value = (expected or {}).get("in_job_repeats") + return value if isinstance(value, int) and not isinstance(value, bool) and value > 0 else 1 + + +def probe_key(row: dict) -> tuple[int, str, int]: + return (row["class"], row["target_cache"], row.get("repeat", 1)) + + +def spread(samples: list[float | None]) -> str: + """The range over the VMs that measured, naming any that did not: a + failed pg_test_fsync is a missing sample, not a narrower range.""" + values = [v for v in samples if v is not None] + unknown = len(samples) - len(values) + missing = (f"; {unknown} of {len(samples)} VMs unknown (pg_test_fsync failed or " + f"the job wrote no row)") if unknown else "" + if not values: + return f"unknown{missing}" + low, high = min(values), max(values) + ratio = f", max/min {high / low:.2f}" if low > 0 else "" + return f"{cell(low)}–{cell(high)} over {len(values)} VMs{ratio}{missing}" + + +def table(rows: list[dict], expected: dict | None = None) -> str: + """The results table. `expected` is the plan job's `matrices()` output: + a job it lists that wrote no row (it failed before its row step, or its + upload did) is named as missing, never left out of the count.""" + probes = sorted((r for r in rows if r.get("kind") == "probe"), key=probe_key) + lines = ["## Runner probe (#541, #542)", ""] + if expected: + present = {probe_key(r) for r in probes} + absent = [e for e in expected["probe"]["include"] + if (e["class"], e["target_cache"], e.get("repeat", 1)) not in present] + for entry in absent: + lines += [f"### {entry['class']} vCPU · {entry['target_cache']} · repeat " + f"{entry.get('repeat', 1)}: no row " + "(the job failed before writing it; see its log)", ""] + if not probes: + lines += ["No probe row was produced.", ""] + # One column per preset and in-job repeat: the plan's, then any a row + # carries that the plan did not name. + iterations = expected_iterations(expected) if expected else 1 + for row in probes: + for runs in (row.get("runs") or {}).values(): + iterations = max(iterations, len(runs)) + # Without the plan, the presets the rows ran (a job's own summary has + # only its row), not the defaults, which a dispatch may not have asked for. + names = expected_presets(expected) if expected else [] + for row in probes: + names += [n for n in (row.get("runs") or {}) if n not in names] + names = names or list(PRESETS) + columns = [(name, i) for name in names for i in range(iterations)] + + def heading(name: str, i: int) -> str: + return name if iterations == 1 else f"{name} #{i + 1}" + + for row in probes: + build = row.get("build") or {} + target = row.get("target") or {} + host = row.get("host") or {} + lines += [ + f"### {row['class']} vCPU · {row['target_cache']} · repeat {row.get('repeat', 1)} " + f"({'warm' if target.get('restored') else 'cold'} target)", + "", + f"`{row['runner']}` · {host.get('cpu_model', 'unknown CPU')} · " + f"{cell(host.get('nproc'))} CPUs · {cell(host.get('mem_total_mib'), ' MiB')}", + "", + f"Cluster filesystem: {disk_line(host)}", + "", + "| Measure | " + " | ".join(heading(n, i) for n, i in columns) + " |", + "|---|" + "---|" * len(columns), + ] + runs = row.get("runs") or {} + + def per_run(label: str, pick) -> str: + return f"| {label} | " + " | ".join( + pick(runs[name][i]) if i < len(runs.get(name, [])) else "not run" + for name, i in columns + ) + " |" + + lines += [ + per_run("pg_test_fsync fdatasync, one 8 kB write", + lambda r: f"{cell(r['fsync']['ops_per_second'], ' ops/s')} " + f"({cell(r['fsync']['usecs_per_op'], ' µs/op')})"), + per_run("seeding", lambda r: f"{cell(r['seed_seconds'], ' s')} " + f"({cell(r['seed_rows'])} rows)"), + per_run("run wall time (pg_test_fsync, harness, gate)", + lambda r: cell(r["wall_seconds"], " s") + + (" (timed out)" if r.get("timed_out") else "")), + per_run("peak host memory used", + lambda r: f"{cell(r['peak_used_mib'], ' MiB')} " + f"(from {cell(r['baseline_used_mib'], ' MiB')})"), + per_run("shortfall per phase", lambda r: ", ".join( + f"{name} {cell(p.get('shortfall'))}" for name, p in r["phases"].items() + ) or "unknown"), + per_run("tip-to-last-notify p99", + lambda r: cell(r["tip_last_notify_p99_ms"], " ms") + + (f" ({r['tips_missing_a_session']} tips missing a session)" + if r.get("tips_missing_a_session") else "")), + per_run("provenance without an override", lambda r: r["provenance"]), + per_run("harness / gate exit", + lambda r: f"{cell(r['harness_exit_code'])} / {cell(r['gate_exit_code'])}"), + "", + f"Build: {cell(build.get('wall_seconds'), ' s')} (exit " + f"{cell(build.get('exit_code'))}, peak host memory " + f"{cell(build.get('peak_used_mib'), ' MiB')}). Target directory: " + f"{cell(target.get('bytes_before_build'), ' B')} before, " + f"{cell(target.get('bytes_after_build'), ' B')} after; restore " + f"{cell(target.get('restore_seconds'), ' s')}, save " + f"{cell(target.get('save_seconds'), ' s')}.", + "", + ] + by_vm: dict[tuple[int, int], float | None] = {} + for row in rows: + if row.get("kind") == "fsync": + by_vm[(row["class"], row["vm"])] = row["fsync"].get("ops_per_second") + if expected and expected.get("fsync_enabled"): + for entry in expected["fsync"]["include"]: + by_vm.setdefault((entry["class"], entry["vm"]), None) + by_class: dict[int, list[float | None]] = {} + for (size, _vm), ops in sorted(by_vm.items()): + by_class.setdefault(size, []).append(ops) + if by_class: + lines += [ + "### fdatasync across VMs of one class", + "", + "| Class | fdatasync ops/s, one 8 kB write |", + "|---|---|", + ] + for size in sorted(by_class): + lines.append(f"| {size} vCPU | {spread(by_class[size])} |") + lines.append("") + return "\n".join(lines) + + +# --- variance ----------------------------------------------------------- + +# fdatasync cost bands in µs per one-8kB-write op (pg_test_fsync), doubling +# from 125 µs: a VM's band is the median over its runs, so the regression +# rule compares a run only with runs whose commits cost about the same. +FSYNC_BAND_EDGES_USECS = (125, 250, 500, 1000, 2000, 4000) +UNKNOWN_BAND = "unknown" +ALL_BANDS = "all" +DEFAULT_HEADROOM_FRACTION = 0.25 +# Each metric's unit and clock, by the name's last part (a phase metric is +# `<phase>.<metric>`). +METRIC_UNITS = { + "wall_seconds": "seconds, the runner's monotonic clock, around pg_test_fsync, the harness and the gate", + "seed_seconds": "seconds, as the harness reports the window seed", + "peak_used_mib": "MiB, the host's MemTotal - MemAvailable sampled every second", + "headroom_mib": "MiB, MemTotal minus the peak used", + "tip_last_notify_p99_ms": "milliseconds, nearest-rank p99 of each tip's slowest session", + "shortfall": "offers no session could take", + "achieved_rate_shares_per_second": "shares per second, reconciled acknowledged count over the phase", + "ack_p50_ms": "milliseconds, the client's monotonic clock, accepted shares", + "ack_p99_ms": "milliseconds, the client's monotonic clock, accepted shares", +} +RUN_METRICS = ("wall_seconds", "seed_seconds", "peak_used_mib", "headroom_mib", + "tip_last_notify_p99_ms") +PHASE_METRICS = ("shortfall", "achieved_rate_shares_per_second", "ack_p50_ms", "ack_p99_ms") + + +def fsync_band(usecs: float | None) -> str: + if usecs is None: + return UNKNOWN_BAND + lower = 0 + for edge in FSYNC_BAND_EDGES_USECS: + if usecs < edge: + return f"{lower}-{edge}us" + lower = edge + return f"ge{lower}us" + + +def rounded(value: float | None) -> float | None: + return None if value is None else round(value, 4) + + +def stats(values: list[float], planned: int | None = None) -> dict: + """n, unknown, median, MAD (median absolute deviation from the median, + unscaled, in the metric's unit), CV (sample standard deviation over the + mean: None under two samples or at a zero mean), min and max.""" + n = len(values) + out = {"n": n, "unknown": None if planned is None else max(planned - n, 0), + "median": None, "mad": None, "cv": None, "min": None, "max": None} + if not n: + return out + median = statistics.median(values) + mean = statistics.fmean(values) + out.update( + median=rounded(median), + mad=rounded(statistics.median(abs(v - median) for v in values)), + cv=rounded(statistics.stdev(values) / abs(mean)) if n >= 2 and mean else None, + min=rounded(min(values)), + max=rounded(max(values)), + ) + return out + + +def run_metrics(run: dict) -> dict[str, float | None]: + metrics = {name: finite(run.get(name)) for name in RUN_METRICS} + for phase, values in (run.get("phases") or {}).items(): + for name in PHASE_METRICS: + metrics[f"{phase}.{name}"] = finite((values or {}).get(name)) + return metrics + + +def metric_unit(name: str) -> str: + return METRIC_UNITS.get(name.rsplit(".", 1)[-1], "unknown") + + +def job_band(row: dict | None) -> tuple[str, float | None]: + """The VM's fsync-cost band: the median fdatasync µs/op over every run + it made, whatever the preset. A VM with none measured is `unknown`.""" + if row is None: + return UNKNOWN_BAND, None + costs = [finite(run["fsync"].get("usecs_per_op")) + for runs in (row.get("runs") or {}).values() for run in runs] + costs = [c for c in costs if c is not None] + usecs = statistics.median(costs) if costs else None + return fsync_band(usecs), usecs + + +def gated_phases(preset: dict | None) -> list[str] | None: + phases = ((preset or {}).get("gates") or {}).get("phases") + return list(phases) if isinstance(phases, list) and phases else None + + +def size_evidence(runs: list[dict], planned: int, preset: dict | None, + headroom_fraction: float) -> dict: + """Whether the class holds the preset (#487 P5): every planned run + measured, zero shortfall in each gated phase of every run, and the + host's headroom at peak at least `headroom_fraction` of MemTotal and at + least the preset's own memory floor. False on any measured violation, + None while a run is unknown and nothing violated, True otherwise.""" + measured = [r for r in runs if r.get("unmeasured_reason") is None] + phases = gated_phases(preset) + floor = finite(((preset or {}).get("args") or {}).get("--min-mem-available-mib")) + totals = [t for t in (finite(r.get("mem_total_mib")) for r in measured) if t is not None] + peaks = [p for p in (finite(r.get("peak_used_mib")) for r in measured) if p is not None] + headrooms = [h for h in (finite(r.get("headroom_mib")) for r in measured) if h is not None] + # Each run against its own VM's MemTotal: one class's VMs need not all + # report the same total, and the smallest must not set a lenient bar + # for the others. + required_by_run = [ + (h, round(max(headroom_fraction * t, floor or 0), 1)) + for h, t in ((finite(r.get("headroom_mib")), finite(r.get("mem_total_mib"))) + for r in measured) + if h is not None and t is not None + ] + required = max((need for _h, need in required_by_run), default=None) + shortfalls: list[float | None] = [] + for run in measured: + names = phases or list((run.get("phases") or {})) + values = [finite(((run.get("phases") or {}).get(n) or {}).get("shortfall")) for n in names] + shortfalls.append(None if not values or None in values else max(values)) + known_shortfalls = [v for v in shortfalls if v is not None] + problems = [] + if any(v > 0 for v in known_shortfalls): + problems.append(f"{sum(v > 0 for v in known_shortfalls)} run(s) with gated shortfall") + short = sum(h < need for h, need in required_by_run) + if short: + problems.append(f"{short} run(s) under the headroom") + unknowns = [] + if len(measured) < planned: + unknowns.append(f"{planned - len(measured)} of {planned} run(s) unknown") + if len(known_shortfalls) < len(measured): + unknowns.append(f"{len(measured) - len(known_shortfalls)} run(s) with no gated shortfall") + if len(required_by_run) < len(measured) or required is None: + unknowns.append("memory headroom not measured on every run") + if problems: + fits, why = False, "; ".join(problems) + elif unknowns or not measured: + fits, why = None, "; ".join(unknowns) or "no run measured" + else: + fits, why = True, f"all {planned} run(s) measured, no gated shortfall, headroom held" + return { + "runs_planned": planned, + "runs_measured": len(measured), + "gated_phases": phases, + "gated_shortfall_max": max(known_shortfalls) if known_shortfalls else None, + "runs_with_gated_shortfall": sum(v > 0 for v in known_shortfalls), + "peak_used_mib_max": max(peaks) if peaks else None, + "mem_total_mib_min": min(totals) if totals else None, + "headroom_mib_min": min(headrooms) if headrooms else None, + # The most any run needed: each run's is max(fraction x its own + # MemTotal, the preset's floor). + "headroom_mib_required": required, + "headroom_fraction": headroom_fraction, + "preset_min_mem_available_mib": floor, + "fits": fits, + "why": why, + } + + +def group_document(size: int, preset_name: str, band: str, jobs: list[dict], + iterations: int, preset: dict | None, headroom_fraction: float) -> dict: + """One class, preset and band. Each job is {"row": row or None, "key": ...}; + a job with no row, or a planned run it did not record or did not + measure, is counted as unknown in every metric.""" + reasons: dict[str, int] = {} + runs_all: list[dict] = [] + per_job: list[list[dict]] = [] + planned = 0 + for job in jobs: + row = job["row"] + runs = list(((row or {}).get("runs") or {}).get(preset_name, [])) + job_planned = max(iterations, len(runs)) + planned += job_planned + if row is None: + reasons["job wrote no row"] = reasons.get("job wrote no row", 0) + job_planned + elif len(runs) < job_planned: + reasons["run not recorded"] = reasons.get("run not recorded", 0) + job_planned - len(runs) + for run in runs: + reason = run.get("unmeasured_reason") + if reason is not None: + reasons[reason] = reasons.get(reason, 0) + 1 + runs_all += runs + per_job.append([r for r in runs if r.get("unmeasured_reason") is None]) + measured = [r for jr in per_job for r in jr] + # The headline metrics are always present, so a group whose every run is + # unknown still says n 0 and how many are unknown for each. A preset + # whose gates name no phases gates every phase it drives: those are the + # phases any of the group's runs reported, measured or not (none, when + # no run got as far as a report). + phases = gated_phases(preset) or sorted( + {name for run in runs_all for name in (run.get("phases") or {})}) + seeded = set(RUN_METRICS) | {f"{phase}.{name}" for phase in phases + for name in PHASE_METRICS} + names = sorted(seeded | {n for r in measured for n in run_metrics(r)}) + metrics = {} + for name in names: + values = [v for v in (run_metrics(r).get(name) for r in measured) if v is not None] + job_values = [[v for v in (run_metrics(r).get(name) for r in jr) if v is not None] + for jr in per_job] + medians = [statistics.median(vs) for vs in job_values if vs] + within = [stats(vs) for vs in job_values if len(vs) >= 2] + cvs = [w["cv"] for w in within if w["cv"] is not None] + metrics[name] = { + "unit": metric_unit(name), + "all": stats(values, planned), + "vm_to_vm": stats(medians, len(jobs)), + "run_to_run": { + "jobs": len(within), + "median_mad": rounded(statistics.median(w["mad"] for w in within)), + "median_cv": rounded(statistics.median(cvs)) if cvs else None, + } if within else None, + } + fsync_costs = [c for c in (finite(r["fsync"].get("usecs_per_op")) for r in runs_all) + if c is not None] + disks: dict[tuple, int] = {} + for job in jobs: + dev = (((job["row"] or {}).get("host") or {}).get("block_device")) or {} + key = (dev.get("write_cache"), dev.get("fua"), dev.get("model")) + if job["row"] is not None: + disks[key] = disks.get(key, 0) + 1 + return { + "class": size, + "runner": runner_label(size), + "preset": preset_name, + "fsync_band": band, + "jobs": {"planned": len(jobs), "with_row": sum(j["row"] is not None for j in jobs)}, + "runs": {"planned": planned, "measured": len(measured), + "unknown": planned - len(measured), "unknown_reasons": reasons}, + "fsync_usecs_per_op": stats(fsync_costs, planned), + "disks": [{"write_cache": k[0], "fua": k[1], "model": k[2], "vms": n} + for k, n in sorted(disks.items(), key=lambda kv: str(kv[0]))], + "metrics": metrics, + "size_evidence": size_evidence(runs_all, planned, preset, headroom_fraction) + if band == ALL_BANDS else None, + } + + +def variance(rows: list[dict], expected: dict | None = None, + presets: dict[str, dict] | None = None, + headroom_fraction: float = DEFAULT_HEADROOM_FRACTION) -> dict: + """The variance document (`VARIANCE_SCHEMA`). Groups are per class, + preset and fsync band, and per class and preset over every band + (`fsync_band` "all", which also carries the runner-size evidence). + Without `expected` a job that wrote no row cannot be counted, and the + document says so (`jobs.planned` is null).""" + if not math.isfinite(headroom_fraction) or not 0 <= headroom_fraction < 1: + raise ProbeError(f"headroom fraction {headroom_fraction!r} is not in [0, 1)") + presets = presets or {} + probes = [r for r in rows if r.get("kind") == "probe"] + commits = sorted({r["commit"] for r in probes if r.get("commit")}) + if len(commits) > 1: + raise ProbeError(f"rows from more than one commit ({', '.join(commits)}): " + "variance is over repeats of one commit") + by_key = {} + for row in probes: + key = probe_key(row) + if key in by_key: + raise ProbeError(f"two rows for {key[0]} vCPU, {key[1]}, repeat {key[2]}") + by_key[key] = row + planned_keys = ([(e["class"], e["target_cache"], e.get("repeat", 1)) + for e in expected["probe"]["include"]] if expected else []) + keys = planned_keys + sorted(k for k in by_key if k not in planned_keys) + names = expected_presets(expected) if expected else [] + for row in probes: + names += [n for n in (row.get("runs") or {}) if n not in names] + iterations = expected_iterations(expected) + jobs = [] + for key in keys: + row = by_key.get(key) + band, usecs = job_band(row) + jobs.append({"key": key, "row": row, "band": band, "fsync_usecs_per_op": usecs}) + groups = [] + for size in sorted({k[0] for k in keys}): + for name in names: + class_jobs = [j for j in jobs if j["key"][0] == size] + groups.append(group_document(size, name, ALL_BANDS, class_jobs, iterations, + presets.get(name), headroom_fraction)) + for band in sorted({j["band"] for j in class_jobs}): + groups.append(group_document( + size, name, band, [j for j in class_jobs if j["band"] == band], + iterations, presets.get(name), headroom_fraction)) + return { + "schema": VARIANCE_SCHEMA, + "commit": commits[0] if commits else None, + "rows_without_commit": sum(1 for r in probes if not r.get("commit")), + "in_job_repeats": iterations, + "fsync_band_edges_usecs_per_op": list(FSYNC_BAND_EDGES_USECS), + "jobs": { + "planned": len(planned_keys) if expected else None, + "with_row": len(by_key), + "missing": [{"class": k[0], "target_cache": k[1], "repeat": k[2]} + for k in planned_keys if k not in by_key], + "by_vm": [{"class": j["key"][0], "target_cache": j["key"][1], + "repeat": j["key"][2], "fsync_band": j["band"], + "fsync_usecs_per_op": rounded(j["fsync_usecs_per_op"])} for j in jobs], + }, + "groups": groups, + } + + +def variance_markdown(document: dict) -> str: + """A one-screen view of the document's per-class groups.""" + def num(value): + if value is None: + return "unknown" + return f"{value:,.0f}" if abs(value) >= 1000 else f"{value:.4g}" + + def spread(metric: dict | None) -> str: + if not metric: + return "unknown" + rtr = (metric.get("run_to_run") or {}).get("median_cv") + return (f"{num(metric['all']['median'])} (n {metric['all']['n']}, CV " + f"{num(metric['all']['cv'])}; VM CV {num(metric['vm_to_vm']['cv'])}; " + f"run CV {num(rtr)})") + + planned = document["jobs"]["planned"] + lines = ["## Runner probe variance (#542)", "", + f"Commit {document['commit'] or 'unknown'}; {document['jobs']['with_row']} job(s) " + f"with a row of {'an unknown number' if planned is None else planned} planned. " + "CV is the sample standard deviation over the mean; VM CV is over each VM's " + "median, run CV the median within one VM. `unknown` is a run that failed, timed " + "out or never reported, never a zero.", "", + "| Class | Preset | Runs measured | fsync bands | steady_state rate, shares/s | " + "steady_state ACK p99, ms | peak used, MiB (MemTotal) | gated shortfall max | fits |", + "|---|---|---|---|---|---|---|---|---|"] + for group in document["groups"]: + if group["fsync_band"] != ALL_BANDS: + continue + bands = sorted({vm["fsync_band"] for vm in document["jobs"]["by_vm"] + if vm["class"] == group["class"]}) + m = group["metrics"] + e = group["size_evidence"] or {} + fits = {True: "yes", False: "no", None: "unknown"}[e.get("fits")] + lines.append( + f"| {group['class']} vCPU | {group['preset']} | {group['runs']['measured']} of " + f"{group['runs']['planned']} | {', '.join(bands)} | " + f"{spread(m.get('steady_state.achieved_rate_shares_per_second'))} | " + f"{spread(m.get('steady_state.ack_p99_ms'))} | " + f"{num(e.get('peak_used_mib_max'))} ({num(e.get('mem_total_mib_min'))}) | " + f"{num(e.get('gated_shortfall_max'))} | {fits}: {e.get('why', '')} |") + lines.append("") + return "\n".join(lines) + + +# --- verdict ------------------------------------------------------------ + + +def verdict(probe_dir: Path, presets: list[str], iterations: int) -> list[str]: + """What failed: the build, or a planned run whose gate did not exit 0 + (#487 lesson 5: the exit code is the verdict). A run with no + measurement file is a failure, not a pass.""" + failed = [] + names = ["build"] + [f"{name}/{i}/probe-measure" for name in presets + for i in range(1, iterations + 1)] + for name in names: + document = read_json(probe_dir / f"{name}.json") + code = document.get("exit_code") if isinstance(document, dict) else None + if code != 0 or (document or {}).get("timed_out"): + failed.append(f"{name}: exit {code}" + + (" (timed out)" if (document or {}).get("timed_out") else "")) + return failed + + +# --- entry point -------------------------------------------------------- + + +def optional_number(text: str) -> float | None: + return None if text.strip() == "" else float(text) + + +def optional_int(text: str) -> int | None: + return None if text.strip() == "" else int(text) + + +def fraction(text: str) -> float: + value = float(text) + if not math.isfinite(value) or not 0 <= value < 1: + raise argparse.ArgumentTypeError(f"{text!r} is not a fraction in [0, 1)") + return value + + +def positive_seconds(text: str) -> float: + value = float(text) + if not math.isfinite(value) or value <= 0: + raise argparse.ArgumentTypeError(f"{text!r} is not a positive number of seconds") + return value + + +def write_json(path: Path, document: dict) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(json.dumps(document, indent=2) + "\n", encoding="utf-8") + + +def read_expected(path: Path | None) -> dict | None: + if path is None: + return None + expected = read_json(path) + if expected is None: + raise ProbeError(f"--expected {path} is not readable JSON") + return expected + + +def main(argv: list[str] | None = None) -> int: + parser = argparse.ArgumentParser(description=__doc__, + formatter_class=argparse.RawDescriptionHelpFormatter) + sub = parser.add_subparsers(dest="action", required=True) + + m = sub.add_parser("matrix") + m.add_argument("--classes", required=True) + m.add_argument("--target-cache", required=True) + m.add_argument("--fsync-vms", required=True) + m.add_argument("--presets", default=",".join(PRESETS), + help="checked-in fake-node preset names, comma separated") + m.add_argument("--repeats", default="1", + help=f"probe jobs per class and cache backend, 1..{MAX_REPEATS}") + m.add_argument("--in-job-repeats", default="1", + help=f"runs of each preset inside one job, 1..{MAX_IN_JOB_REPEATS}") + m.add_argument("--summary", type=Path, default=None, + help="append the plan and its spend ceiling (Markdown) to this file") + + me = sub.add_parser("measure") + me.add_argument("--out", type=Path, required=True) + me.add_argument("--timeout-seconds", type=positive_seconds, default=None) + me.add_argument("command", nargs=argparse.REMAINDER) + + r = sub.add_parser("row") + r.add_argument("--out", type=Path, required=True) + r.add_argument("--class", dest="size", type=int, required=True) + r.add_argument("--runner", required=True) + r.add_argument("--target-cache", choices=TARGET_CACHES, required=True) + r.add_argument("--repeat", type=int, default=1) + r.add_argument("--commit", default=None) + r.add_argument("--target-restored", choices=("true", "false"), required=True) + r.add_argument("--target-bytes-before", type=optional_int, default=None) + r.add_argument("--target-bytes-after", type=optional_int, default=None) + r.add_argument("--restore-seconds", type=optional_number, default=None) + r.add_argument("--save-seconds", type=optional_number, default=None) + r.add_argument("--build", type=Path, required=True) + r.add_argument("--host", type=Path, required=True) + r.add_argument("--run", action="append", default=[], + help="PRESET=DIR, once per run; a preset given again is its next in-job repeat") + + f = sub.add_parser("fsync-row") + f.add_argument("--out", type=Path, required=True) + f.add_argument("--class", dest="size", type=int, required=True) + f.add_argument("--runner", required=True) + f.add_argument("--vm", type=int, required=True) + f.add_argument("--host", type=Path, required=True) + f.add_argument("--fsync", type=Path, required=True) + + h = sub.add_parser("host") + h.add_argument("--out", type=Path, required=True) + h.add_argument("--dir", type=Path, required=True) + h.add_argument("--runner", default=None, + help="the runner label the job asked for, recorded as runner_label") + + t = sub.add_parser("table") + t.add_argument("directory", type=Path) + t.add_argument("--expected", type=Path, default=None, + help="the plan job's matrices as JSON, so a job that wrote no row is counted") + + v = sub.add_parser("variance") + v.add_argument("directory", type=Path) + v.add_argument("--expected", type=Path, default=None, + help="the plan job's matrices as JSON, so a job that wrote no row is counted") + v.add_argument("--out", type=Path, default=None, + help="write the JSON document here (default: standard output)") + v.add_argument("--markdown", action="store_true", + help="print a Markdown summary on standard output (needs --out)") + v.add_argument("--headroom-fraction", type=fraction, default=DEFAULT_HEADROOM_FRACTION, + help="share of MemTotal that must stay available at peak for a class to fit") + + ve = sub.add_parser("verdict") + ve.add_argument("--probe", type=Path, required=True) + ve.add_argument("--presets", required=True) + ve.add_argument("--in-job-repeats", type=int, required=True) + + args = parser.parse_args(argv) + try: + if args.action == "matrix": + result = matrices(args.classes, args.target_cache, args.fsync_vms, + args.presets, args.repeats, args.in_job_repeats) + for key, value in result.items(): + # A string is an output the workflow reads as text (the + # preset list), not JSON: no quotes around it. + text = value if isinstance(value, str) else json.dumps(value, separators=(",", ":")) + print(f"{key}={text}") + if args.summary is not None: + with args.summary.open("a", encoding="utf-8") as summary: + summary.write(plan_summary(result) + "\n") + return 0 + if args.action == "measure": + command = args.command[1:] if args.command[:1] == ["--"] else args.command + if not command: + raise ProbeError("measure needs a command after --") + result = measure(command, timeout_seconds=args.timeout_seconds) + write_json(args.out, result) + if result["timed_out"]: + return 124 + return result["exit_code"] if result["exit_code"] >= 0 else 128 - result["exit_code"] + if args.action == "host": + facts = host_facts(args.dir) + if args.runner: + facts = {"runner_label": args.runner, **facts} + write_json(args.out, facts) + return 0 + if args.action == "row": + known, _aliases = load_presets() + write_json(args.out, build_row(args, set(known))) + return 0 + if args.action == "fsync-row": + write_json(args.out, build_fsync_row(args)) + return 0 + if args.action == "variance": + if args.markdown and args.out is None: + raise ProbeError("variance --markdown needs --out for the JSON document") + known, _aliases = load_presets() + document = variance(load_rows(args.directory), read_expected(args.expected), + known, args.headroom_fraction) + if args.out is None: + print(json.dumps(document, indent=2)) + else: + write_json(args.out, document) + if args.markdown: + print(variance_markdown(document)) + return 0 + if args.action == "verdict": + failed = verdict(args.probe, [n for n in args.presets.split(",") if n], + args.in_job_repeats) + if failed: + print("; ".join(failed), file=sys.stderr) + return 1 + return 0 + print(table(load_rows(args.directory), read_expected(args.expected))) + return 0 + except ProbeError as error: + print(f"prism_load_probe: {error}", file=sys.stderr) + return 2 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/scripts/prism_load_regress.py b/scripts/prism_load_regress.py new file mode 100644 index 000000000..0f4de1c07 --- /dev/null +++ b/scripts/prism_load_regress.py @@ -0,0 +1,780 @@ +#!/usr/bin/env python3 +"""The load lanes' regression rule, report-only (#551, S6 of #487). + +Compares a run's trend rows (scripts/prism_load_trend.py) with the trailing +rows of the same series on the `ci-evidence` branch, under the thresholds in +`test/prism-load-regression.toml` (its header states the rule). A series is +one lane, preset, preset sha256, runner class and fsync band; only runs whose +harness exited 0 count toward a baseline, and each run counts once, at its +latest attempt. + +For a regressed number the verdict names the commit range it appeared in: +from the newest earlier run of the series that was within the threshold (the +last good commit) to the oldest run of the unbroken regressed streak that +ends at this run (the first bad commit). On the first night that is the last +trended commit and this one; on later nights of the same regression the range +stays where it began. Flagged runs are left out of later baselines, so a +regression that persists does not become its own baseline, until it has +lasted `window` runs: the new level is then taken as accepted. + +Nothing here passes by default (EP-OBSERVABILITY): a number the run did not +measure, a run with no fsync cost (so no band), a series with fewer than +`min_baseline` usable runs and a run whose harness did not exit 0 each get an +explicit `unknown` or `not evaluated` with the reason. + +#542's variance document (`scripts/prism_load_probe.py variance`, schema +VARIANCE_SCHEMA), when given with --variance, supplies the spread. Its groups +are per runner class, preset and fsync band (#542's own doubling bands, from +prism_load_probe.fsync_band), and each metric carries `all` (every run, so +across VMs), `vm_to_vm` (the per-VM medians) and `run_to_run` (within each +VM) statistics. A nightly runs once on whichever VM it gets, so the spread a +night is compared with is `all.cv`, the coefficient of variation over every +run, as a fraction of the median. It is read from the group of the run's own +#542 band, or else the class's all-band group (the summary says which); a +number #542 does not measure (VARIANCE_METRICS lists the ones it does), or a +group with fewer than two runs of it, uses the provisional spread, and the +verdict stays provisional and names it. When `vm_to_vm.cv` is over `ab_ratio` +times `run_to_run.median_cv` the summary points at #511's same-VM A/B. + +A document of another schema, or another version of this one, is refused by +name in the verdict (EP-COMPAT) and the rule falls back to the provisional +spread. + +Usage: + prism_load_regress.py --history DIR --rows DIR [--config FILE] + [--variance FILE] [--json OUT] [--markdown OUT] [--github-output FILE] + +Exits 0 whether or not anything regressed (the rule is report-only), and 2 +when an input cannot be read. +""" + +from __future__ import annotations + +import argparse +from dataclasses import dataclass +import json +import math +from pathlib import Path +import statistics +from typing import Callable +import sys +import tomllib + +sys.path.insert(0, str(Path(__file__).resolve().parent)) + +import prism_load_probe as probe # noqa: E402 + +ROOT = Path(__file__).resolve().parents[1] +CONFIG = ROOT / "test" / "prism-load-regression.toml" +CONFIG_SCHEMA = "qbit.prism.load-regression-config.v1" +# The trend row this rule reads, written by scripts/prism_load_trend.py. +ROW_SCHEMA = "qbit.prism.load-trend-row.v1" +VERDICT_SCHEMA = "qbit.prism.load-regression-verdict.v1" +# #542's document, written by scripts/prism_load_probe.py `variance`. +VARIANCE_SCHEMA = probe.VARIANCE_SCHEMA +VARIANCE_SCHEMA_PREFIX = "qbit.prism.runner-probe-variance." +# This rule's metric keys and #542's names for them (a phase metric is +# `<phase>.<name>` there). The server-side ACK, lock waiters and frontend RSS +# are not in #542's runs, so they keep the provisional spread. +VARIANCE_METRICS = { + "achieved_rate_shares_per_second": "achieved_rate_shares_per_second", + "client_ack_p50_ms": "ack_p50_ms", + "client_ack_p99_ms": "ack_p99_ms", + "tip_last_notify_p99_ms": "tip_last_notify_p99_ms", +} +# 1.4826 * MAD estimates a normal sample's standard deviation. +MAD_SCALE = 1.4826 +WORSE = ("higher", "lower") +SCOPES = ("phase", "run") + + +class RegressError(Exception): + """An input the rule cannot use, worded for the workflow log.""" + + +@dataclass(frozen=True) +class Metric: + key: str + scope: str + worse: str + min_relative_change: float + min_absolute_change: float + + +@dataclass(frozen=True) +class Config: + provisional: bool + lanes: tuple[str, ...] + window: int + min_baseline: int + k: float + ab_ratio: float + band_edges: tuple[float, ...] + band_names: tuple[str, ...] + metrics: tuple[Metric, ...] + + +# --- config ------------------------------------------------------------- + + +def finite(value, where: str, minimum: float = 0.0, strict: bool = False) -> float: + if isinstance(value, bool) or not isinstance(value, (int, float)) or not math.isfinite(value): + raise RegressError(f"{where} is {value!r}, not a finite number") + if value < minimum or (strict and value == minimum): + raise RegressError(f"{where} is {value}, not {'above' if strict else 'at least'} {minimum}") + return float(value) + + +def whole(value, where: str, minimum: int) -> int: + if isinstance(value, bool) or not isinstance(value, int) or value < minimum: + raise RegressError(f"{where} is {value!r}, not a whole number of at least {minimum}") + return value + + +def known_keys(table: dict, allowed: set[str], where: str) -> None: + unknown = sorted(set(table) - allowed) + if unknown: + raise RegressError(f"{where}: unknown key(s) {', '.join(unknown)}") + + +def load_config(path: Path) -> Config: + try: + data = tomllib.loads(path.read_text(encoding="utf-8")) + except (OSError, tomllib.TOMLDecodeError) as error: + raise RegressError(f"{path}: {error}") from error + if data.get("schema") != CONFIG_SCHEMA: + raise RegressError(f"{path}: schema is {data.get('schema')!r}, not {CONFIG_SCHEMA}") + known_keys(data, {"schema", "provisional", "rule", "fsync_bands", "metric"}, str(path)) + if not isinstance(data.get("provisional"), bool): + raise RegressError(f"{path}: provisional must be true or false") + rule = data.get("rule") or {} + known_keys(rule, {"lanes", "window", "min_baseline", "k", "ab_ratio"}, f"{path} [rule]") + lanes = rule.get("lanes") + if not isinstance(lanes, list) or not lanes or not all(isinstance(x, str) and x for x in lanes): + raise RegressError(f"{path}: rule.lanes must be a non-empty list of lane names") + window = whole(rule.get("window"), f"{path}: rule.window", 1) + min_baseline = whole(rule.get("min_baseline"), f"{path}: rule.min_baseline", 1) + if min_baseline > window: + raise RegressError(f"{path}: rule.min_baseline {min_baseline} exceeds rule.window {window}") + k = finite(rule.get("k"), f"{path}: rule.k", strict=True) + ab_ratio = finite(rule.get("ab_ratio"), f"{path}: rule.ab_ratio", minimum=1.0) + bands = data.get("fsync_bands") or {} + known_keys(bands, {"edges_usecs", "names"}, f"{path} [fsync_bands]") + edges = bands.get("edges_usecs") + names = bands.get("names") + if not isinstance(edges, list) or not isinstance(names, list): + raise RegressError(f"{path}: fsync_bands needs edges_usecs and names lists") + edges = [finite(e, f"{path}: fsync_bands.edges_usecs", strict=True) for e in edges] + if any(b <= a for a, b in zip(edges, edges[1:])): + raise RegressError(f"{path}: fsync_bands.edges_usecs must increase strictly") + if len(names) != len(edges) + 1 or len(set(names)) != len(names) \ + or not all(isinstance(n, str) and n and n != "unknown" for n in names): + raise RegressError( + f"{path}: fsync_bands.names must be {len(edges) + 1} distinct names other than 'unknown'" + ) + metrics = [] + for index, raw in enumerate(data.get("metric") or []): + where = f"{path}: metric[{index}]" + known_keys(raw, {"key", "scope", "worse", "min_relative_change", "min_absolute_change"}, where) + if not isinstance(raw.get("key"), str) or not raw["key"]: + raise RegressError(f"{where}: key is missing") + if raw.get("scope") not in SCOPES: + raise RegressError(f"{where}: scope is {raw.get('scope')!r}, not one of {SCOPES}") + if raw.get("worse") not in WORSE: + raise RegressError(f"{where}: worse is {raw.get('worse')!r}, not one of {WORSE}") + metrics.append(Metric( + key=raw["key"], + scope=raw["scope"], + worse=raw["worse"], + min_relative_change=finite(raw.get("min_relative_change"), f"{where}.min_relative_change"), + min_absolute_change=finite(raw.get("min_absolute_change"), f"{where}.min_absolute_change"), + )) + if not metrics: + raise RegressError(f"{path}: no [[metric]] to compare") + if len({(m.scope, m.key) for m in metrics}) != len(metrics): + raise RegressError(f"{path}: a metric is listed twice") + return Config( + provisional=data["provisional"], + lanes=tuple(lanes), + window=window, + min_baseline=min_baseline, + k=k, + ab_ratio=ab_ratio, + band_edges=tuple(edges), + band_names=tuple(names), + metrics=tuple(metrics), + ) + + +def fsync_band(usecs_per_op, config: Config) -> str: + """The run's fsync band, or `unknown` when its cost was not measured.""" + if isinstance(usecs_per_op, bool) or not isinstance(usecs_per_op, (int, float)) \ + or not math.isfinite(usecs_per_op) or usecs_per_op <= 0: + return "unknown" + for edge, name in zip(config.band_edges, config.band_names): + if usecs_per_op < edge: + return name + return config.band_names[-1] + + +# --- variance (#542) ---------------------------------------------------- + + +def load_variance(path: Path | None) -> tuple[dict | None, str | None]: + """The variance document, or None and why it is not used.""" + if path is None: + return None, "no variance file given (#542's document is not checked in); the spread is provisional" + try: + data = json.loads(path.read_text(encoding="utf-8")) + except (OSError, ValueError) as error: + return None, f"variance file {path} could not be read ({error}); the spread is provisional" + schema = data.get("schema") if isinstance(data, dict) else None + if schema != VARIANCE_SCHEMA: + kind = ("another version of #542's document" if isinstance(schema, str) + and schema.startswith(VARIANCE_SCHEMA_PREFIX) else "not #542's document") + return None, (f"variance file {path} has schema {schema!r}, {kind}; this rule reads " + f"{VARIANCE_SCHEMA} only, so the spread is provisional") + if not isinstance(data.get("groups"), list): + return None, f"variance file {path} has no groups list; the spread is provisional" + return data, None + + +@dataclass(frozen=True) +class Spread: + """#542's spreads for one number, as fractions of its median.""" + all_cv: float + vm_to_vm_cv: float | None + run_to_run_cv: float | None + source: str + + +def variance_spread(variance: dict | None, runner: str | None, preset: str | None, + metric: Metric, phase: str | None, usecs_per_op) -> Spread | None: + """The run's spread from #542's document, or None when it has none.""" + name = VARIANCE_METRICS.get(metric.key) + if variance is None or name is None: + return None + full = name if metric.scope == "run" else f"{phase}.{name}" + band = probe.fsync_band(number(usecs_per_op)) + for wanted in (band, probe.ALL_BANDS): + if wanted == probe.UNKNOWN_BAND: + continue + for group in variance["groups"]: + if not isinstance(group, dict) or (group.get("runner"), group.get("preset"), + group.get("fsync_band")) != (runner, preset, wanted): + continue + stats = (group.get("metrics") or {}).get(full) or {} + every = stats.get("all") or {} + cv = positive(every.get("cv")) + if cv is None or not isinstance(every.get("n"), int) or every["n"] < 2: + continue + label = f"{wanted} band" if wanted != probe.ALL_BANDS else "all bands" + return Spread( + all_cv=cv, + vm_to_vm_cv=positive((stats.get("vm_to_vm") or {}).get("cv")), + run_to_run_cv=positive((stats.get("run_to_run") or {}).get("median_cv")), + source=f"variance (#542, {label}, n {every['n']})", + ) + return None + + +def positive(value) -> float | None: + if isinstance(value, bool) or not isinstance(value, (int, float)) or not math.isfinite(value): + return None + return float(value) if value >= 0 else None + + +# --- rows --------------------------------------------------------------- + + +def number(value) -> float | None: + if isinstance(value, bool) or not isinstance(value, (int, float)) or not math.isfinite(value): + return None + return float(value) + + +def read_row_files(directory: Path) -> list[dict]: + """The run's own rows: one JSON object per `*.json` file.""" + rows = [] + for path in sorted(directory.glob("*.json")): + try: + row = json.loads(path.read_text(encoding="utf-8")) + except (OSError, ValueError) as error: + raise RegressError(f"{path}: {error}") from error + if not isinstance(row, dict) or row.get("schema") != ROW_SCHEMA: + raise RegressError(f"{path}: not a {ROW_SCHEMA} row") + rows.append(row) + if not rows: + # No rows is no comparison, not "nothing regressed" (EP-OBSERVABILITY). + raise RegressError(f"{directory}: no trend rows to compare (the run left none)") + return rows + + +def read_history(directory: Path) -> tuple[list[dict], list[str]]: + """Every row under trend/*/*.jsonl, and notes on lines that were skipped. + + Lines of another schema, or that do not parse, are counted and left out: + they are kept on the branch (it is append-only) but never read as data of + this schema (EP-COMPAT).""" + rows, notes = [], [] + unparsed = foreign = 0 + for path in sorted((directory / "trend").glob("*/*.jsonl")): + for line in path.read_text(encoding="utf-8").splitlines(): + if not line.strip(): + continue + try: + row = json.loads(line) + except ValueError: + unparsed += 1 + continue + if not isinstance(row, dict) or row.get("schema") != ROW_SCHEMA: + foreign += 1 + continue + rows.append(row) + if unparsed: + notes.append(f"{unparsed} history line(s) did not parse and were skipped") + if foreign: + notes.append(f"{foreign} history row(s) of a schema other than {ROW_SCHEMA} were skipped") + return rows, notes + + +def latest_attempts(rows: list[dict]) -> list[dict]: + """One row per run and preset: its latest attempt, a measurement or a + `missing` placeholder (EP-STATE: an earlier attempt never stands in for + a later one, so a re-run that left no row hides the earlier measurement). + Only `preset` rows are returned; a run whose latest attempt is missing + has none.""" + return [r for r in latest_outcomes(rows) if r.get("kind") == "preset"] + + +def latest_outcomes(rows: list[dict]) -> list[dict]: + def rank(row: dict) -> tuple: + # At one attempt a measurement outranks a placeholder. + return (row.get("run_attempt") or 0, row.get("kind") == "preset") + + best: dict[tuple, dict] = {} + for row in rows: + if row.get("kind") not in ("preset", "missing"): + continue + # Not by lane: a re-run whose preset moved lanes still supersedes. + key = (row.get("repository"), row.get("run_id"), row.get("preset")) + if key not in best or rank(row) > rank(best[key]): + best[key] = row + return list(best.values()) + + +def order_key(row: dict) -> tuple: + return (row.get("recorded_at") or "", row.get("run_id") or 0, row.get("run_attempt") or 0) + + +def series_key(row: dict, config: Config) -> tuple: + fsync = row.get("fsync") or {} + return ( + row.get("lane"), + row.get("preset"), + row.get("preset_sha256"), + row.get("runner_class"), + fsync_band(fsync.get("usecs_per_op"), config), + ) + + +def completed(row: dict) -> bool: + return (row.get("outcome") or {}).get("harness_exit_code") == 0 + + +def value_of(row: dict, metric: Metric, phase: str | None) -> float | None: + headline = row.get("headline") or {} + if metric.scope == "run": + return number((headline.get("run") or {}).get(metric.key)) + return number(((headline.get("phases") or {}).get(phase) or {}).get(metric.key)) + + +def headline_of(row: dict, phase: str | None) -> dict: + headline = row.get("headline") or {} + if phase is None: + return dict(headline.get("run") or {}) + return dict((headline.get("phases") or {}).get(phase) or {}) + + +def artifact_phases(row: dict) -> list[str]: + phases = (row.get("headline") or {}).get("phases") or {} + return [name for name, phase in phases.items() if (phase or {}).get("in_artifact") is not False] + + +# --- the rule ----------------------------------------------------------- + + +@dataclass(frozen=True) +class Threshold: + median: float + spread: float + allowed: float + n: int + source: str + + def worse(self, value: float, worse: str) -> bool: + if worse == "higher": + return value > self.median + self.allowed + return value < self.median - self.allowed + + def limit(self, worse: str) -> float: + return self.median + self.allowed if worse == "higher" else self.median - self.allowed + + +def threshold(values: list[float], metric: Metric, config: Config, + measured: Spread | None) -> Threshold: + median = statistics.median(values) + if measured is not None: + spread, source = measured.all_cv * abs(median), measured.source + else: + mad = statistics.median(abs(v - median) for v in values) + spread, source = MAD_SCALE * mad, "provisional: the baseline's scaled MAD" + allowed = max(config.k * spread, metric.min_relative_change * abs(median), + metric.min_absolute_change) + return Threshold(median, spread, allowed, len(values), source) + + +def point(row: dict, value: float | None) -> dict: + return { + "run_id": row.get("run_id"), + "run_attempt": row.get("run_attempt"), + "run_url": row.get("run_url"), + "commit": row.get("commit"), + "recorded_at": row.get("recorded_at"), + "value": value, + } + + +def evaluate_number(current: dict, value: float, history: list[tuple[dict, float]], + metric: Metric, config: Config, + spread_for: Callable[[dict], Spread | None]) -> dict: + """One number of one run against its series. `history` is the series' + usable earlier runs with this number measured, oldest first. + `spread_for(row)` is #542's spread for a run (its own fsync band), or + None; each replayed run is judged with its own. + + The series is replayed oldest first so that every run's verdict is the + one its own trailing baseline gave it, and a flagged run is left out of + later baselines: a lasting regression stays flagged, with its range + where it began, instead of drifting into the median that judges it. + Once a streak of flagged runs is `window` long the new level is taken + as accepted and the streak becomes the baseline.""" + series = history + [(current, value)] + flagged = [False] * len(series) + # Each run's own verdict: `within`, `flagged`, `accepted` (a flagged run + # whose streak lasted a window) or `unjudged` (too few runs before it). + verdicts = ["unjudged"] * len(series) + # Whether any judgement, a replayed run's or this one's, fell back to the + # provisional spread: that keeps the verdict provisional. + provisional_spread = False + limit = None + for index, (_, observed) in enumerate(series): + streak = 0 + while streak < index and flagged[index - 1 - streak]: + streak += 1 + if streak >= config.window: + for accepted in range(index - streak, index): + flagged[accepted] = False + verdicts[accepted] = "accepted" + base = [v for j, (_, v) in enumerate(series[:index]) if not flagged[j]][-config.window:] + if len(base) < config.min_baseline: + limit = None + continue + spread = spread_for(series[index][0]) + provisional_spread = provisional_spread or spread is None + limit = threshold(base, metric, config, spread) + flagged[index] = limit.worse(observed, metric.worse) + verdicts[index] = "flagged" if flagged[index] else "within" + if limit is None: + usable = len([f for f in flagged[:-1] if not f]) + return {"status": "unknown", + "reason": f"the series has {usable} usable earlier run(s) with this number; " + f"the rule needs {config.min_baseline}"} + if not flagged[-1]: + return {"status": "within", "threshold": limit, "provisional_spread": provisional_spread} + streak = 1 + while streak < len(series) and flagged[-1 - streak]: + streak += 1 + # The last good run is the one before the streak, if it was judged within + # its own baseline, or else (never judged, or an accepted level) if it is + # within the limit that flagged this run; the verdict says which. + good, basis = None, None + if streak < len(series): + candidate = series[-1 - streak] + if verdicts[-1 - streak] == "within": + good, basis = candidate, "within its own baseline" + elif not limit.worse(candidate[1], metric.worse): + good, basis = candidate, ("within this run's limit (not itself judged: too few runs " + "before it)" if verdicts[-1 - streak] == "unjudged" + else "within this run's limit (an accepted level)") + return {"status": "regressed", "threshold": limit, "good": good, "good_basis": basis, + "first_bad": series[-streak], "streak": streak, "provisional_spread": provisional_spread} + + +def evaluate(current_rows: list[dict], history_rows: list[dict], config: Config, + variance: dict | None, variance_note: str | None) -> dict: + current = latest_attempts(current_rows) + current_ids = {(r.get("repository"), r.get("run_id")) for r in current} + history = [r for r in latest_attempts(history_rows) + if (r.get("repository"), r.get("run_id")) not in current_ids] + history.sort(key=order_key) + regressions, entries, notes = [], [], [] + # Numbers compared on the provisional spread: any keeps the verdict + # provisional, however the config is marked. + fallbacks: set[str] = set() + # A planned preset whose job left no row is shown, never silently absent. + for row in latest_outcomes(current_rows): + if row.get("kind") == "missing": + entries.append({"lane": row.get("lane"), "preset": row.get("preset"), "runner_class": None, + "fsync_band": "unknown", "run_id": row.get("run_id"), + "run_attempt": row.get("run_attempt"), "commit": row.get("commit"), + "status": "not evaluated", "missing": True, + "reason": "the preset's job left no row, so nothing was measured"}) + if variance_note: + notes.append(variance_note) + for row in sorted(current, key=lambda r: (r.get("lane") or "", r.get("preset") or "")): + lane, preset = row.get("lane"), row.get("preset") + runner = row.get("runner_class") + band = series_key(row, config)[4] + head = {"lane": lane, "preset": preset, "runner_class": runner, "fsync_band": band, + "run_id": row.get("run_id"), "run_attempt": row.get("run_attempt"), + "commit": row.get("commit")} + if lane not in config.lanes: + entries.append({**head, "status": "not evaluated", + "reason": f"the rule reads lanes {', '.join(config.lanes)}"}) + continue + if not completed(row): + code = (row.get("outcome") or {}).get("harness_exit_code") + entries.append({**head, "status": "not evaluated", + "reason": f"the harness exited {code if code is not None else 'unknown'}, " + "so its numbers are not a measurement of the commit"}) + continue + if band == "unknown": + entries.append({**head, "status": "unknown", + "reason": "pg_test_fsync's cost was not measured, so the run has no " + "fsync band to compare within"}) + continue + key = series_key(row, config) + series = [r for r in history if completed(r) and series_key(r, config) == key + and order_key(r) < order_key(row)] + counts = {"within": 0, "regressed": 0, "unknown": 0} + reasons: list[str] = [] + for metric in config.metrics: + phases = [None] if metric.scope == "run" else artifact_phases(row) + for phase in phases: + value = value_of(row, metric, phase) + label = metric.key if phase is None else f"{phase}.{metric.key}" + if value is None: + counts["unknown"] += 1 + reasons.append(f"{label}: not measured in this run") + continue + measured = variance_spread(variance, runner, preset, metric, phase, + (row.get("fsync") or {}).get("usecs_per_op")) + if measured and measured.run_to_run_cv is not None and measured.vm_to_vm_cv is not None \ + and measured.vm_to_vm_cv > config.ab_ratio * measured.run_to_run_cv: + notes.append( + f"{runner} {preset} {label}: VM-to-VM spread {measured.vm_to_vm_cv:.3g} is over " + f"{config.ab_ratio:g}x the run-to-run spread {measured.run_to_run_cv:.3g}; " + "#511's interleaved A/B on one VM is the comparison to trust for it") + earlier = [(r, v) for r in series if (v := value_of(r, metric, phase)) is not None] + def spread_for(judged: dict, metric=metric, phase=phase) -> Spread | None: + return variance_spread(variance, judged.get("runner_class"), judged.get("preset"), + metric, phase, (judged.get("fsync") or {}).get("usecs_per_op")) + + result = evaluate_number(row, value, earlier, metric, config, spread_for) + if result["status"] == "unknown": + counts["unknown"] += 1 + reasons.append(f"{label}: {result['reason']}") + continue + counts[result["status"]] += 1 + if result["provisional_spread"]: + fallbacks.add(f"{runner} {preset} {label}") + if result["status"] != "regressed": + continue + limit: Threshold = result["threshold"] + good, first_bad = result["good"], result["first_bad"] + regressions.append({ + **head, + "phase": phase, + "metric": metric.key, + "worse": metric.worse, + "current": point(row, value), + "baseline": {"median": limit.median, "spread": limit.spread, + "allowed_change": limit.allowed, "limit": limit.limit(metric.worse), + "runs": limit.n, "spread_source": limit.source}, + "last_good": {**point(*good), "basis": result["good_basis"]} if good else None, + "first_bad": point(*first_bad), + "runs_in_streak": result["streak"], + # Both runs' headline numbers for the phase (or the run), + # for the tracking issue's side-by-side summary. + "summaries": {"current": headline_of(row, phase), + "last_good": headline_of(good[0], phase) if good else None}, + "commit_range": { + "from": good[0].get("commit") if good else None, + "to": first_bad[0].get("commit"), + }, + }) + entries.append({**head, "status": "evaluated", "numbers": counts, "unknown": reasons}) + if variance is not None and fallbacks: + notes.append("the variance file has no spread (two or more runs) for " + + "; ".join(sorted(fallbacks)) + "; those use the provisional spread") + return { + "schema": VERDICT_SCHEMA, + "report_only": True, + "provisional": config.provisional or variance is None or bool(fallbacks), + "rule": {"window": config.window, "min_baseline": config.min_baseline, "k": config.k, + "lanes": list(config.lanes), "fsync_band_edges_usecs": list(config.band_edges), + "fsync_band_names": list(config.band_names)}, + "history_rows": len(history), + "regressions": regressions, + "runs": entries, + "notes": sorted(set(notes)), + } + + +# --- output ------------------------------------------------------------- + + +def fmt(value) -> str: + if value is None: + return "unknown" + if isinstance(value, float): + return f"{value:,.3f}".rstrip("0").rstrip(".") + return str(value) + + +def short(commit: str | None) -> str: + return commit[:12] if commit else "unknown" + + +def compare_link(repository: str | None, server: str, commit_range: dict) -> str: + start, end = commit_range.get("from"), commit_range.get("to") + if not start or not end: + return f"`{short(start)}..{short(end)}` (no earlier run within the threshold)" + text = f"`{short(start)}..{short(end)}`" + if repository: + return f"[{text}]({server}/{repository}/compare/{start}...{end})" + return text + + +def run_link(p: dict | None) -> str: + if not p: + return "none" + label = f"run {p.get('run_id')}" + (f" (attempt {p['run_attempt']})" if (p.get("run_attempt") or 1) > 1 else "") + return f"[{label}]({p['run_url']})" if p.get("run_url") else label + + +def markdown(verdict: dict, repository: str | None, server: str) -> str: + lines = ["### Load regression rule (#551): report-only", ""] + if verdict["provisional"]: + lines += ["Thresholds are **provisional** (the config is not yet calibrated against " + "#542's measured variance, or a number used the provisional spread; see the " + "notes); nothing here fails a job or blocks a merge.", ""] + rule = verdict["rule"] + lines += [f"Trailing median of up to {rule['window']} runs of the same lane, preset, runner " + f"class and fsync band (at least {rule['min_baseline']}), k = {rule['k']:g}; " + f"{verdict['history_rows']} earlier runs read from `ci-evidence`.", ""] + regressions = verdict["regressions"] + if regressions: + lines += [f"#### {len(regressions)} number(s) regressed", "", + "| lane | preset | runner | fsync band | phase | number | this run | last good run | " + "baseline median | limit | commit range |", + "|---|---|---|---|---|---|---|---|---|---|---|"] + for r in regressions: + good = r["last_good"] + lines.append( + f"| {r['lane']} | {r['preset']} | {r['runner_class']} | {r['fsync_band']} | " + f"{r['phase'] or 'run'} | {r['metric']} ({r['worse']} is worse) | " + f"{fmt(r['current']['value'])} ({run_link(r['current'])}) | " + f"{fmt(good['value']) if good else 'none'} ({run_link(good)}" + f"{'; ' + good['basis'] if good and good.get('basis') != 'within its own baseline' else ''}) | " + f"{fmt(r['baseline']['median'])} over {r['baseline']['runs']} runs | " + f"{fmt(r['baseline']['limit'])} ({r['baseline']['spread_source']}) | " + f"{compare_link(repository, server, r['commit_range'])} |") + lines.append("") + lines += ["The range runs from the newest earlier run within the threshold to the first " + "run of the unbroken regressed streak ending at this run.", ""] + shown = set() + for r in regressions: + key = (r["lane"], r["preset"], r["phase"], r["current"]["run_id"]) + if key in shown: + continue + shown.add(key) + current, good = r["summaries"]["current"], r["summaries"]["last_good"] or {} + lines += [f"<details><summary>{r['preset']} {r['phase'] or 'run'}: this run and the last " + "good run</summary>", "", + f"| number | last good ({run_link(r['last_good'])}) | this run ({run_link(r['current'])}) |", + "|---|---|---|"] + for name in sorted(set(current) | set(good)): + lines.append(f"| {name} | {fmt(good.get(name))} | {fmt(current.get(name))} |") + lines += ["", "</details>", ""] + else: + counted = [e["numbers"] for e in verdict["runs"] if e["status"] == "evaluated"] + within = sum(n["within"] for n in counted) + unknown = sum(n["unknown"] for n in counted) + if not within: + # Every number unknown is no verdict, not a pass (EP-OBSERVABILITY). + lines += ["**No verdict:** no number could be compared " + f"({unknown} without a verdict; see the reasons below).", ""] + else: + lines += [f"No number regressed: {within} within the threshold, {unknown} without a verdict.", ""] + missing = [e for e in verdict["runs"] if e.get("missing")] + if missing: + lines += [f"**{len(missing)} planned preset(s) left no measurement:** " + + ", ".join(e["preset"] or "unknown" for e in missing) + ".", ""] + lines += ["#### Runs", "", "| lane | preset | runner | fsync band | status |", "|---|---|---|---|---|"] + for entry in verdict["runs"]: + if entry["status"] == "evaluated": + n = entry["numbers"] + status = f"{n['within']} within, {n['regressed']} regressed, {n['unknown']} unknown" + else: + status = f"{entry['status']}: {entry['reason']}" + lines.append(f"| {entry['lane']} | {entry['preset']} | {entry['runner_class']} | " + f"{entry['fsync_band']} | {status} |") + unknown = [(e, u) for e in verdict["runs"] for u in e.get("unknown") or []] + if unknown: + lines += ["", "<details><summary>Numbers with no verdict</summary>", ""] + lines += [f"- {e['preset']}: {u}" for e, u in unknown] + lines += ["", "</details>"] + if verdict["notes"]: + lines += ["", "#### Notes", ""] + [f"- {note}" for note in verdict["notes"]] + return "\n".join(lines) + "\n" + + +def main(argv: list[str] | None = None) -> int: + parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) + parser.add_argument("--history", type=Path, required=True, + help="a checkout of ci-evidence (trend/<lane>/*.jsonl); may be empty") + parser.add_argument("--rows", type=Path, required=True, help="this run's row files (*.json)") + parser.add_argument("--config", type=Path, default=CONFIG) + parser.add_argument("--variance", type=Path, help="#542's variance file") + parser.add_argument("--json", type=Path, help="write the verdict here") + parser.add_argument("--markdown", type=Path, help="write the summary here") + parser.add_argument("--github-output", type=Path, help="append regressions=<n> here") + parser.add_argument("--repository", help="owner/name, for compare links") + parser.add_argument("--server-url", default="https://github.com") + args = parser.parse_args(argv) + try: + config = load_config(args.config) + history, history_notes = read_history(args.history) + rows = read_row_files(args.rows) + except (RegressError, OSError) as error: + print(f"prism-load-regress: {error}", file=sys.stderr) + return 2 + variance, variance_note = load_variance(args.variance) + verdict = evaluate(rows, history, config, variance, variance_note) + verdict["notes"] = sorted(set(verdict["notes"]) | set(history_notes)) + text = markdown(verdict, args.repository, args.server_url) + if args.json: + args.json.write_text(json.dumps(verdict, indent=2) + "\n", encoding="utf-8") + if args.markdown: + args.markdown.write_text(text, encoding="utf-8") + if args.github_output: + with args.github_output.open("a", encoding="utf-8") as out: + out.write(f"regressions={len(verdict['regressions'])}\n") + print(text, end="") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/prism_load_trend.py b/scripts/prism_load_trend.py new file mode 100644 index 000000000..3dc2e288f --- /dev/null +++ b/scripts/prism_load_trend.py @@ -0,0 +1,982 @@ +#!/usr/bin/env python3 +"""The load lanes' trend rows and evidence promotion (#551, S6 of #487). + +#487 decision 3 keeps the lanes' evidence in this repository: Actions +artifacts (90 days) are the working set, one JSON line per run on the orphan +branch `ci-evidence` is the trend, and raw bundles that must outlive the +artifacts are promoted to release assets. + +Layout of `ci-evidence`: `trend/<lane>/YYYY-MM.jsonl`, append-only, one row +per preset run (`kind: preset`), per planned preset whose job left no row +(`kind: missing`, its own kind so it can never stand in for, or block, the +preset's real row), per non-preset lane job of a scheduled run +(`kind: job`: the live regtest scenarios, the fuzzing, the shipped images) or +per later promotion (`kind: promotion`: a cited run, or a writer job's re-run +that promoted a bundle the recorded row lists as failed). The month is the +row's `recorded_at` month, in UTC. Rows follow ROW_SCHEMA; a reader skips +rows of another schema instead of guessing at them. + +Only the scheduled writer job and a `v*` tag job write (`contents: write` on +those jobs alone); `append` refuses a row from any other event, so a pull +request or a dispatch can build rows as artifacts but never record them. + +Subcommands: + +- `row`: one preset run's row, from the directory prism-load-run.sh wrote + (load-harness-report.json, host.json, pg_test_fsync.txt, the exit codes). + A number the run did not measure is null, never 0. +- `job-rows`: rows for a scheduled run's non-preset jobs, from the + workflow's `toJSON(needs)`, and a `kind: missing` row for each planned + preset whose job left no row. +- `fetch`: export `ci-evidence`'s trend files into a directory (empty when + the branch does not exist yet), for the regression rule to read. +- `promote`: upload the selected rows' bundles (their Actions artifacts, + tarred) to a release and record the asset URL in the row. +- `append`: add rows to `ci-evidence`, creating the orphan branch on the + first write. Each attempt fetches the tip, skips rows already there (by + identity: kind, run id, run attempt and preset or job), commits on top and + pushes without force; a rejected or unconfirmed push is retried from a + fresh fetch, so concurrent writers never lose a row and a retry never + duplicates one (EP-STATE). Attempts are bounded and share one deadline + (EP-ERRORS). +- `cite`: a maintainer's tool: promote a finished run's artifact to the + monthly pre-release and print (or, with --append, record) its + `kind: promotion` row. + +Promotion policy (`promote --select policy`): L3, L5 and `weekly` (the +Saturday soak, #575) rows always; an L2 +row when it is flagged, that is, when the regression rule found a regressed +number in it or its gate failed; no other lane. Monthly bundles go to the +`ci-evidence/YYYY-MM` pre-release, a tag's full-suite bundles to the tag's +release (`--release <tag>`). Asset names carry lane, preset, run id and +attempt, so `gh release upload --clobber` only ever replaces the same bundle +on a retry. + +Usage: + prism_load_trend.py row --dir OUT --preset NAME --lane auto --runner-class LABEL \\ + --commit SHA --run-id N --run-attempt N --event EVENT --repository OWNER/NAME \\ + [--artifact-name NAME --artifact-url URL] --out ROW.json + prism_load_trend.py job-rows --needs JSON --matrix JSON --rows DIR --run-id N ... + prism_load_trend.py fetch --out DIR + prism_load_trend.py promote --rows DIR --select policy --verdict VERDICT.json --repository R + prism_load_trend.py append --rows DIR + prism_load_trend.py cite --artifact-id N --lane L2 --preset NAME --repository R +""" + +from __future__ import annotations + +import argparse +from datetime import datetime, timezone +import hashlib +import json +import math +import os +from pathlib import Path +import re +import subprocess +import sys +import tarfile +import tempfile +import time +import zipfile + +sys.path.insert(0, str(Path(__file__).resolve().parent)) + +import prism_load_probe as probe # noqa: E402 +import prism_load_regress as regress # noqa: E402 + +ROOT = Path(__file__).resolve().parents[1] +PRESETS = ROOT / "crates" / "qbit-prism-load" / "presets" +ROW_SCHEMA = regress.ROW_SCHEMA +BRANCH = "ci-evidence" +# #487's lanes, plus the manifest's `weekly` (the Saturday soak, #575, which +# is not L5) and the nightly fuzzing. +LANES = ("L2", "L3", "L4", "L5", "L6", "weekly", "fuzz") +KINDS = ("preset", "missing", "job", "promotion") +# The events whose rows may be recorded: the schedules, and a `v*` tag push +# (a branch push never runs a lane that writes). `cite` is a maintainer's +# promotion of an earlier run, recorded only as a `promotion` row. +WRITE_EVENTS = ("schedule", "push") +# The non-preset jobs of prism-load-nightly.yml whose outcome is a trend row. +JOB_LANES = { + # #552's bridging pair: its outcome; its comparison row + # (scripts/prism_load_bridge.py) stays in its artifact for now. + "bridging": "L2", + "live-nightly": "L4", + "live-weekly": "L4", + "stratum-fuzz": "fuzz", + "shipped-images": "L6", +} +PROMOTE_ALWAYS = ("L3", "L5", "weekly") +BOT_NAME = "github-actions[bot]" +BOT_EMAIL = "41898282+github-actions[bot]@users.noreply.github.com" +README = """\ +# ci-evidence + +The PRISM load lanes' trend rows (#551, S6 of #487; decision 3). This orphan +branch is written only by the scheduled and `v*` tag jobs of the lane +workflows, through `scripts/prism_load_trend.py append` on `3.x.x`. It is +append-only: rows are added, never edited or removed, and a push is never +forced. + +- `trend/<lane>/YYYY-MM.jsonl`: one JSON row per line, schema + `qbit.prism.load-trend-row.v1`, month of the row's `recorded_at` (UTC). +- Raw bundles that outlive the 90-day Actions artifacts are release assets: + the rolling `ci-evidence/YYYY-MM` pre-release, or a tag's release for the + full suite. A row's `asset_url` points at its bundle once promoted. + +`scripts/prism_load_regress.py` reads these rows for the report-only +regression rule. +""" + + +class TrendError(Exception): + """An input or an operation the trend tooling cannot complete.""" + + +def utc_now() -> str: + return datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ") + + +def month_of(row: dict) -> str: + recorded = row.get("recorded_at") + if not isinstance(recorded, str) or not re.fullmatch(r"\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}Z", recorded): + raise TrendError(f"row {identity(row)} has recorded_at {recorded!r}, not UTC YYYY-MM-DDTHH:MM:SSZ") + return recorded[:7] + + +def identity(row: dict) -> tuple: + """What makes two rows the same record: a retried write of one of them is + skipped. A preset row is one run attempt of one preset. A job row is one + attempt of a job, with its outcome and commit: each job exports the + attempt it ran in, so a later attempt that re-ran the job has a new row, + and one that did not re-run it repeats the same row and is skipped.""" + kind = row.get("kind") + if kind in ("preset", "missing"): + return (kind, row.get("repository"), row.get("run_id"), row.get("run_attempt"), row.get("preset")) + if kind == "job": + return (kind, row.get("repository"), row.get("run_id"), row.get("run_attempt"), row.get("job"), + (row.get("outcome") or {}).get("result"), row.get("commit")) + return (kind, row.get("repository"), row.get("run_id"), row.get("run_attempt"), + row.get("preset"), row.get("asset_url")) + + +def validate(row: dict, events: tuple[str, ...] = WRITE_EVENTS) -> None: + if not isinstance(row, dict) or row.get("schema") != ROW_SCHEMA: + raise TrendError(f"not a {ROW_SCHEMA} row: {str(row)[:120]}") + if row.get("kind") not in KINDS: + raise TrendError(f"row {identity(row)}: kind {row.get('kind')!r} is not one of {KINDS}") + if row.get("lane") not in LANES: + raise TrendError(f"row {identity(row)}: lane {row.get('lane')!r} is not one of {LANES}") + for field in ("run_id", "run_attempt"): + if isinstance(row.get(field), bool) or not isinstance(row.get(field), int) or row[field] < 1: + raise TrendError(f"row {identity(row)}: {field} is {row.get(field)!r}") + if not isinstance(row.get("repository"), str) or "/" not in row["repository"]: + raise TrendError(f"row {identity(row)}: repository is {row.get('repository')!r}") + if row["kind"] == "job": + if JOB_LANES.get(row.get("job")) != row["lane"]: + raise TrendError(f"row {identity(row)}: job {row.get('job')!r} is not one of {sorted(JOB_LANES)} " + "in its lane") + elif not isinstance(row.get("preset"), str) or not row["preset"]: + raise TrendError(f"row {identity(row)}: preset is {row.get('preset')!r}") + # A promotion row is a maintainer's `cite`, or a writer job's re-run + # recovering a promotion its first attempt recorded as failed. + allowed = ("cite", *events) if row["kind"] == "promotion" else events + if row.get("event") not in allowed: + raise TrendError( + f"row {identity(row)}: event {row.get('event')!r} may not write to {BRANCH} " + f"(only {', '.join(allowed)}); pull request and dispatch runs produce artifacts only" + ) + # A `v*` tag push is the only push that writes; the ref is on the row. + if row["event"] == "push" and not str(row.get("ref") or "").startswith("refs/tags/v"): + raise TrendError(f"row {identity(row)}: a push of {row.get('ref')!r} may not write to {BRANCH} " + "(only a refs/tags/v* tag push)") + month_of(row) + + +# --- row ---------------------------------------------------------------- + + +def number(value) -> float | None: + return regress.number(value) + + +def integer(value) -> int | None: + if isinstance(value, bool) or not isinstance(value, int): + return None + return value + + +def server_ack(deltas) -> tuple[float | None, float | None]: + """Mean and bucketed p99 upper bound, in ms, of the frontends' accepted + share acknowledgements (qbit_prism_share_ack_seconds, receipt to response + write, which includes the share append). Unknown if any frontend's delta + is: a mean over the rest would describe a different set of shares.""" + if not isinstance(deltas, list) or not deltas: + return None, None + count = total = 0.0 + buckets: dict[float, float] = {} + for delta in deltas: + if not isinstance(delta, dict) or delta.get("unavailable_reason"): + return None, None + c = number((delta.get("counts") or {}).get("accepted")) + s = number((delta.get("sums") or {}).get("accepted")) + if c is None or s is None: + return None, None + count += c + total += s + for le, value in ((delta.get("bucket_deltas") or {}).get("accepted") or {}).items(): + try: + bound = float(le) + except ValueError: + continue + if number(value) is not None: + buckets[bound] = buckets.get(bound, 0.0) + value + if count <= 0: + return None, None + p99 = None + for bound in sorted(buckets): + if buckets[bound] >= 0.99 * count: + p99 = bound * 1000 if math.isfinite(bound) else None + break + return total / count * 1000, p99 + + +def peak_rss_mib(processes) -> float | None: + """The largest frontend peak RSS; unknown when any frontend's is.""" + if not isinstance(processes, list) or not processes: + return None + peaks = [integer((p or {}).get("peak_rss_kib")) for p in processes] + if any(p is None for p in peaks): + return None + return round(max(peaks) / 1024, 1) + + +def latency_ms(summary, field: str) -> float | None: + if not isinstance(summary, dict) or summary.get("unit") != "milliseconds": + return None + return number(summary.get(field)) + + +def phase_headline(phase: dict) -> dict: + order = phase.get("order_lock") or {} + ack = phase.get("client_ack_latency") + mean, p99 = server_ack(phase.get("server_share_ack_seconds")) + reconciliation = phase.get("reconciliation") or {} + return { + "in_artifact": phase.get("in_artifact") if isinstance(phase.get("in_artifact"), bool) else None, + "completed": phase.get("completed") if isinstance(phase.get("completed"), bool) else None, + "duration_seconds": number(phase.get("duration_seconds")), + "target_rate_shares_per_second": number(phase.get("target_rate_shares_per_second")), + "achieved_rate_shares_per_second": number(phase.get("achieved_rate_shares_per_second")), + "shortfall": integer(phase.get("shortfall")), + "client_ack_p50_ms": latency_ms(ack, "p50"), + "client_ack_p99_ms": latency_ms(ack, "p99"), + "server_ack_mean_ms": mean, + "server_ack_p99_upper_ms": p99, + "order_lock_max_waiters": integer(order.get("max_waiters")), + "order_lock_mean_waiters": number(order.get("mean_waiters")), + "peak_rss_mib": peak_rss_mib(phase.get("processes")), + "rejected_valid_shares": integer(phase.get("rejected_valid_shares")), + "missing_shares": integer(reconciliation.get("missing")), + } + + +def preset_facts(name: str, directory: Path) -> tuple[str | None, str | None]: + """The preset's schedule and file sha256 from the checkout.""" + path = directory / f"{name}.json" + try: + data = json.loads(path.read_text(encoding="utf-8")) + except (OSError, ValueError): + return None, None + return data.get("schedule"), hashlib.sha256(path.read_bytes()).hexdigest() + + +def lane_for(lane: str, schedule: str | None) -> str: + if lane != "auto": + return lane + # The weekly soak (#575) is the manifest's `weekly` lane (not L5, which + # is #556's soak and chaos); every other preset of the nightly + # workflow is L2's. + return "weekly" if schedule == "weekly" else "L2" + + +def host_summary(host: dict | None) -> dict | None: + if not isinstance(host, dict) or "error" in host: + return None + filesystem = host.get("filesystem") or {} + device = host.get("block_device") or {} + return { + "runner_label": host.get("runner_label"), + "nproc": host.get("nproc"), + "cpu_model": host.get("cpu_model"), + "mem_total_mib": host.get("mem_total_mib"), + "kernel": host.get("kernel"), + "filesystem_type": filesystem.get("type"), + "mount_options": filesystem.get("options"), + "write_cache": device.get("write_cache"), + "fua": device.get("fua"), + } + + +def base_row(args: argparse.Namespace, kind: str, lane: str) -> dict: + return { + "schema": ROW_SCHEMA, + "kind": kind, + "recorded_at": args.recorded_at or utc_now(), + "repository": args.repository, + "run_id": args.run_id, + "run_attempt": args.run_attempt, + "run_url": f"{args.server_url}/{args.repository}/actions/runs/{args.run_id}" + + (f"/attempts/{args.run_attempt}" if args.run_attempt > 1 else ""), + "event": args.event, + "ref": args.ref or None, + "commit": args.commit or None, + "lane": lane, + } + + +def build_row(args: argparse.Namespace) -> dict: + directory: Path = args.dir + report = probe.read_json(directory / "load-harness-report.json") + result = probe.run_result(directory) + gate = probe.read_int(directory / "gate-exit-code") + harness = result["harness_exit_code"] + schedule, sha256 = preset_facts(args.preset, args.presets) + config = regress.load_config(args.config) + fsync = result["fsync"] + phases = {} + if report is not None: + for phase in report.get("phases") or []: + if isinstance(phase, dict) and isinstance(phase.get("name"), str): + phases[phase["name"]] = phase_headline(phase) + if gate in (0, 1): + verdict = "pass" if gate == 0 else "fail" + else: + verdict = "no verdict" + durability = report.get("durability_findings") if report else None + row = base_row(args, "preset", lane_for(args.lane, schedule)) + row.update({ + "preset": args.preset, + "preset_sha256": sha256, + "runner_class": args.runner_class, + "fsync": { + "method": fsync.get("method"), + "ops_per_second": fsync.get("ops_per_second"), + "usecs_per_op": fsync.get("usecs_per_op"), + "band": regress.fsync_band(fsync.get("usecs_per_op"), config), + "band_edges_usecs": list(config.band_edges), + }, + "host": host_summary(probe.read_json(directory / "host.json")), + "outcome": { + "harness_exit_code": harness, + "gate_exit_code": gate, + "result": verdict, + "provenance": result["provenance"], + }, + "report": None if report is None else { + "schema": report.get("schema"), + "run_id": report.get("run_id"), + "started_at": report.get("started_at"), + "finished_at": report.get("finished_at"), + }, + "headline": { + "run": { + "tip_last_notify_p99_ms": result["tip_last_notify_p99_ms"], + "tips_missing_a_session": result["tips_missing_a_session"] if report else None, + "durability_findings": len(durability) if isinstance(durability, list) else None, + }, + "phases": phases, + }, + "artifact": {"name": args.artifact_name or None, "id": artifact_id_of(args.artifact_id, args.artifact_url), + "url": args.artifact_url or None}, + "promotion": None, + "asset_url": None, + }) + return row + + +def artifact_id_of(given: str | None, url: str | None) -> int | None: + """The artifact's ID: upload-artifact's `artifact-id`, else the tail of + its `artifact-url` (.../actions/runs/<run>/artifacts/<id>).""" + if given and str(given).isdigit(): + return int(given) + match = re.search(r"/artifacts/(\d+)$", url or "") + return int(match.group(1)) if match else None + + +def job_rows(args: argparse.Namespace) -> list[dict]: + """A row per non-preset job that ran, and a `missing` row per planned + preset that left none, so every scheduled run has rows.""" + try: + needs = json.loads(args.needs) + except ValueError as error: + raise TrendError(f"--needs is not JSON: {error}") from error + rows = [] + for job, lane in JOB_LANES.items(): + entry = needs.get(job) if isinstance(needs, dict) else None + if not isinstance(entry, dict) or entry.get("result") in (None, "skipped"): + continue + outputs = entry.get("outputs") or {} + # The attempt the job last ran in: a re-run of the evidence job alone + # leaves the other jobs' outputs, and so their attempts, as they were. + attempt = str(outputs.get("attempt") or "") + row = base_row(argparse.Namespace(**{**vars(args), "run_attempt": int(attempt) if attempt.isdigit() + and int(attempt) > 0 else args.run_attempt}), "job", lane) + row["commit"] = outputs.get("commit") or None + row.update({"job": job, "outcome": {"result": entry["result"]}, + "promotion": None, "asset_url": None}) + rows.append(row) + planned: list[str] = [] + if args.matrix: + try: + planned = [e["preset"] for e in json.loads(args.matrix)["include"]] + except (ValueError, KeyError, TypeError) as error: + raise TrendError(f"--matrix is not the plan's matrix: {error}") from error + # A row for a preset this attempt's plan does not name is an earlier + # attempt's (a re-run after the plan changed): it is not this run's + # measurement, so it is dropped before anything reads the directory. + for path in sorted(args.rows.glob("*.json")): + try: + row = json.loads(path.read_text(encoding="utf-8")) + except (OSError, ValueError): + continue + if row.get("kind") == "preset" and row.get("preset") not in planned: + print(f"prism-load-trend: dropping {path.name}: preset {row.get('preset')!r} is not in " + "this run's plan", file=sys.stderr) + path.unlink() + if args.matrix: + # The attempt each preset job last ran in (the jobs API's + # filter=latest, one {"name", "run_attempt"} per line): a row from an + # earlier attempt of a job that was re-run is stale, and the re-run + # that left no row is recorded as missing at its own attempt. + latest: dict[str, int] = {} + if args.jobs: + for line in args.jobs.read_text(encoding="utf-8").splitlines(): + try: + job = json.loads(line) + except ValueError as error: + raise TrendError(f"--jobs line {line[:80]!r} is not JSON") from error + if isinstance(job.get("run_attempt"), int): + latest[job.get("name")] = max(job["run_attempt"], latest.get(job.get("name"), 0)) + present: dict[str, int] = {} + for path in args.rows.glob("*.json"): + try: + row = json.loads(path.read_text(encoding="utf-8")) + except (OSError, ValueError): + continue + if row.get("kind") == "preset" and isinstance(row.get("run_attempt"), int): + present[row["preset"]] = max(row["run_attempt"], present.get(row["preset"], 0)) + for preset in planned: + if preset in present and present[preset] >= latest.get(preset, 0): + continue + schedule, sha256 = preset_facts(preset, args.presets) + # The row (its attempt and its run URL) is the preset job's latest + # attempt, which need not be this job's. + attempt = latest.get(preset) or args.run_attempt + row = base_row(argparse.Namespace(**{**vars(args), "run_attempt": attempt}), + "missing", lane_for("auto", schedule)) + row.update({ + "preset": preset, "preset_sha256": sha256, + "outcome": {"result": "missing: the preset's job left no row"}, + "promotion": None, "asset_url": None, + }) + rows.append(row) + return rows + + +# --- git ---------------------------------------------------------------- + + +class Deadline: + def __init__(self, seconds: float) -> None: + self.end = time.monotonic() + seconds + + def remaining(self) -> float: + return self.end - time.monotonic() + + def check(self, what: str) -> float: + left = self.remaining() + if left <= 0: + raise TrendError(f"the deadline passed before {what}") + return left + + +def run(command: list[str], deadline: Deadline, *, cwd: Path | None = None, check: bool = True, + input: bytes | None = None, env: dict | None = None, + stdout_path: Path | None = None) -> subprocess.CompletedProcess: + timeout = deadline.check(" ".join(command[:3])) + try: + if stdout_path is None: + done = subprocess.run(command, cwd=cwd, input=input, capture_output=True, timeout=timeout, + env=env, check=False) + else: + with stdout_path.open("wb") as out: + done = subprocess.run(command, cwd=cwd, input=input, stdout=out, stderr=subprocess.PIPE, + timeout=timeout, env=env, check=False) + except subprocess.TimeoutExpired as error: + raise TrendError(f"{' '.join(command[:4])} did not finish before the deadline") from error + if check and done.returncode != 0: + raise TrendError(f"{' '.join(command[:4])} exited {done.returncode}: " + f"{done.stderr.decode(errors='replace').strip()[-500:]}") + return done + + +class Branch: + """`ci-evidence` on a remote, read and written with plumbing only: no + checkout, so a shallow clone of any branch can write it.""" + + def __init__(self, repo: Path, remote: str, deadline: Deadline) -> None: + self.repo, self.remote, self.deadline = repo, remote, deadline + self.ref = f"refs/remotes/{remote}/{BRANCH}" + + def git(self, *args: str, check: bool = True, input: bytes | None = None, + env: dict | None = None) -> subprocess.CompletedProcess: + return run(["git", *args], self.deadline, cwd=self.repo, check=check, input=input, env=env) + + def fetch(self) -> str | None: + """The branch's tip, or None when it does not exist. A failure to ask + is an error, never read as a missing branch.""" + listed = self.git("ls-remote", "--heads", self.remote, f"refs/heads/{BRANCH}") + if not listed.stdout.strip(): + return None + self.git("fetch", "--no-tags", "--depth=1", self.remote, f"+refs/heads/{BRANCH}:{self.ref}") + return self.git("rev-parse", self.ref).stdout.decode().strip() + + def read(self, tip: str | None, path: str) -> str: + if tip is None: + return "" + done = self.git("cat-file", "-e", f"{tip}:{path}", check=False) + if done.returncode != 0: + return "" + return self.git("cat-file", "blob", f"{tip}:{path}").stdout.decode("utf-8") + + def files(self, tip: str | None, prefix: str) -> list[str]: + if tip is None: + return [] + out = self.git("ls-tree", "-r", "--name-only", tip, "--", prefix).stdout.decode() + return [line for line in out.splitlines() if line] + + def commit(self, tip: str | None, contents: dict[str, str], message: str) -> str: + with tempfile.TemporaryDirectory() as scratch: + env = {**os.environ, "GIT_INDEX_FILE": str(Path(scratch) / "index")} + env.setdefault("GIT_AUTHOR_NAME", BOT_NAME) + env.setdefault("GIT_AUTHOR_EMAIL", BOT_EMAIL) + env.setdefault("GIT_COMMITTER_NAME", env["GIT_AUTHOR_NAME"]) + env.setdefault("GIT_COMMITTER_EMAIL", env["GIT_AUTHOR_EMAIL"]) + if tip is not None: + self.git("read-tree", tip, env=env) + else: + self.git("read-tree", "--empty", env=env) + for path, text in contents.items(): + blob = self.git("hash-object", "-w", "--stdin", input=text.encode("utf-8")).stdout.decode().strip() + self.git("update-index", "--add", "--cacheinfo", f"100644,{blob},{path}", env=env) + tree = self.git("write-tree", env=env).stdout.decode().strip() + parents = ["-p", tip] if tip else [] + return self.git("commit-tree", tree, *parents, "-m", message, env=env).stdout.decode().strip() + + def push(self, commit: str) -> tuple[bool, str]: + done = self.git("push", "--no-verify", self.remote, f"{commit}:refs/heads/{BRANCH}", check=False) + return done.returncode == 0, done.stderr.decode(errors="replace").strip() + + +def lines_of(text: str) -> list[dict]: + rows = [] + for line in text.splitlines(): + try: + row = json.loads(line) + except ValueError: + continue + if isinstance(row, dict): + rows.append(row) + return rows + + +def recovered_promotion(row: dict, recorded: dict) -> dict | None: + """A `kind: promotion` row for a preset row whose recorded copy lacks + the asset this copy now has, or None.""" + if row.get("kind") != "preset" or (row.get("promotion") or {}).get("status") != "promoted" \ + or not row.get("asset_url") or recorded.get("asset_url") == row["asset_url"]: + return None + return {key: row.get(key) for key in ( + "schema", "recorded_at", "repository", "run_id", "run_attempt", "run_url", "event", + "ref", "commit", "lane", "preset", "artifact", "promotion", "asset_url")} | {"kind": "promotion"} + + +def append_rows(branch: Branch, rows: list[dict], attempts: int, backoff: float, + log=print) -> dict: + """Record `rows`, returning how many were added and skipped. Raises when + the rows could not be confirmed on the branch within the attempts.""" + for row in rows: + validate(row) + # A row listed twice in one batch is one record. + unique: dict[tuple, dict] = {} + for row in rows: + unique.setdefault(identity(row), row) + last_error = "no attempt made" + for attempt in range(1, attempts + 1): + tip = branch.fetch() + additions: dict[str, list[dict]] = {} + skipped = 0 + # Every month of each lane is read: a row rebuilt by a re-run months + # later has a later `recorded_at` but the same identity. + lanes: dict[str, dict[tuple, dict]] = {} + for key, row in unique.items(): + path = f"trend/{row['lane']}/{month_of(row)}.jsonl" + if row["lane"] not in lanes: + lanes[row["lane"]] = {identity(r): r for p in branch.files(tip, f"trend/{row['lane']}") + for r in lines_of(branch.read(tip, p))} + seen = lanes[row["lane"]] + if key in seen: + skipped += 1 + # A re-run that promoted a bundle the recorded row could not: + # the row stays as written, and the promotion is its own row. + recovered = recovered_promotion(row, seen[key]) + if recovered is not None and identity(recovered) not in seen: + additions.setdefault(path, []).append(recovered) + continue + additions.setdefault(path, []).append(row) + if not additions: + log(f"{BRANCH}: every row is already recorded ({skipped} skipped)") + return {"added": 0, "skipped": skipped, "attempts": attempt} + contents = {} + for path, new in additions.items(): + existing = branch.read(tip, path) + if existing and not existing.endswith("\n"): + existing += "\n" + contents[path] = existing + "".join(json.dumps(r, sort_keys=True) + "\n" for r in new) + if tip is None: + contents["README.md"] = README + added = sum(len(v) for v in additions.values()) + runs = sorted({str(r["run_id"]) for v in additions.values() for r in v}) + commit = branch.commit(tip, contents, f"ci-evidence: {added} row(s) from run {', '.join(runs)}") + pushed, error = branch.push(commit) + if pushed: + log(f"{BRANCH}: added {added} row(s), {skipped} already recorded" + + (" (created the branch)" if tip is None else "") + f", attempt {attempt}") + return {"added": added, "skipped": skipped, "attempts": attempt} + # Rejected (another writer moved the tip, or created the branch) or + # unconfirmed: the next attempt starts from a fresh fetch and skips + # whatever did land. + last_error = error + log(f"{BRANCH}: push attempt {attempt} of {attempts} was not accepted: {error[-300:]}") + if attempt < attempts: + time.sleep(min(backoff * attempt, max(branch.deadline.remaining() - 1, 0))) + raise TrendError(f"the rows were not recorded after {attempts} attempts: {last_error[-300:]}") + + +def export(branch: Branch, out: Path) -> str | None: + """Write the branch's trend files under `out`; returns the tip.""" + tip = branch.fetch() + out.mkdir(parents=True, exist_ok=True) + for path in branch.files(tip, "trend"): + target = out / path + target.parent.mkdir(parents=True, exist_ok=True) + target.write_text(branch.read(tip, path), encoding="utf-8") + return tip + + +# --- promotion ---------------------------------------------------------- + + +def asset_name(row: dict) -> str: + name = f"{row['lane']}-{row.get('preset') or row.get('job')}-run{row['run_id']}-attempt{row['run_attempt']}.tar.gz" + return re.sub(r"[^A-Za-z0-9._-]", "_", name) + + +def flagged(row: dict, verdict: dict | None) -> bool: + if (row.get("outcome") or {}).get("result") == "fail": + return True + for entry in (verdict or {}).get("regressions") or []: + if (entry.get("run_id"), entry.get("run_attempt"), entry.get("preset")) == \ + (row.get("run_id"), row.get("run_attempt"), row.get("preset")): + return True + return False + + +def selected(row: dict, select: str, verdict: dict | None) -> bool: + if row.get("kind") != "preset" or not (row.get("artifact") or {}).get("name"): + return False + if select == "none": + return False + if select == "all": + return True + if select == "flagged": + return flagged(row, verdict) + return row.get("lane") in PROMOTE_ALWAYS or (row.get("lane") == "L2" and flagged(row, verdict)) + + +def gh(args: list[str], deadline: Deadline, check: bool = True, + stdout_path: Path | None = None) -> subprocess.CompletedProcess: + return run(["gh", *args], deadline, check=check, stdout_path=stdout_path) + + +def gh_json(path: str, deadline: Deadline) -> dict: + done = gh(["api", path], deadline) + try: + return json.loads(done.stdout.decode()) + except ValueError as error: + raise TrendError(f"gh api {path} did not return JSON") from error + + +def download_artifact(row: dict, repository: str, dest: Path, deadline: Deadline) -> None: + """Fetch exactly the artifact the row names, by ID. `gh run download` + selects by run and name only, and a re-run's upload of the same name + replaces it, so a name could package another attempt's bundle under + this row's attempt.""" + artifact = row.get("artifact") or {} + ident = artifact_id_of(artifact.get("id"), artifact.get("url")) + if ident is None: + raise TrendError(f"row {identity(row)} has no artifact ID, so its bundle cannot be selected exactly") + meta = gh_json(f"repos/{repository}/actions/artifacts/{ident}", deadline) + if (meta.get("workflow_run") or {}).get("id") != row["run_id"] or meta.get("name") != artifact.get("name"): + raise TrendError(f"artifact {ident} is {meta.get('name')!r} of run " + f"{(meta.get('workflow_run') or {}).get('id')}, not the row's") + if meta.get("expired"): + raise TrendError(f"artifact {ident} has expired") + archive = dest.parent / f"{dest.name}.zip" + gh(["api", f"repos/{repository}/actions/artifacts/{ident}/zip"], deadline, stdout_path=archive) + with zipfile.ZipFile(archive) as bundle: + bundle.extractall(dest) + archive.unlink() + + +def attempt_of_artifact(meta: dict, repository: str, deadline: Deadline) -> int: + """The run attempt that uploaded an artifact: the latest attempt that had + started by the time the artifact was created.""" + run_id = (meta.get("workflow_run") or {}).get("id") + latest = gh_json(f"repos/{repository}/actions/runs/{run_id}", deadline).get("run_attempt") or 1 + created = meta.get("created_at") or "" + for attempt in range(latest, 0, -1): + started = gh_json(f"repos/{repository}/actions/runs/{run_id}/attempts/{attempt}", + deadline).get("run_started_at") or "" + if started and started <= created: + return attempt + raise TrendError(f"no attempt of run {run_id} started before artifact {meta.get('id')} was created") + + +def ensure_release(tag: str, repository: str, create: bool, deadline: Deadline) -> None: + if gh(["release", "view", tag, "--repo", repository, "--json", "tagName"], deadline, check=False).returncode == 0: + return + if not create: + raise TrendError(f"release {tag} does not exist, and only a ci-evidence/YYYY-MM pre-release is created here") + made = gh(["release", "create", tag, "--repo", repository, "--prerelease", "--latest=false", + "--title", f"CI evidence {tag.split('/', 1)[1]}", + "--notes", "Raw bundles of the PRISM load lanes' reduced-L3 jobs, flagged L2 runs, " + "soaks and cited runs (#551, #487 decision 3). Rows: the ci-evidence " + "branch, trend/<lane>/YYYY-MM.jsonl."], + deadline, check=False) + # Another writer may have created it between the view and the create: + # reconcile before calling it a failure. + if made.returncode != 0 and gh(["release", "view", tag, "--repo", repository, "--json", "tagName"], + deadline, check=False).returncode != 0: + raise TrendError(f"could not create release {tag}: {made.stderr.decode(errors='replace').strip()}") + + +def upload_bundle(row: dict, tag: str, repository: str, create: bool, deadline: Deadline, + attempts: int = 3) -> str: + """Upload the row's artifact as a tarball and return the asset's URL.""" + name = asset_name(row) + ensure_release(tag, repository, create, deadline) + with tempfile.TemporaryDirectory() as scratch: + bundle = Path(scratch) / "bundle" + download_artifact(row, repository, bundle, deadline) + tarball = Path(scratch) / name + with tarfile.open(tarball, "w:gz") as tar: + tar.add(bundle, arcname=name.removesuffix(".tar.gz")) + error = "" + for attempt in range(1, attempts + 1): + done = gh(["release", "upload", tag, str(tarball), "--clobber", "--repo", repository], + deadline, check=False) + error = done.stderr.decode(errors="replace").strip() + # An upload whose response was lost may still have landed: the + # release's asset list is the answer either way. + listed = gh(["release", "view", tag, "--repo", repository, "--json", "assets", + "--jq", f'.assets[] | select(.name == "{name}") | .url'], deadline, check=False) + url = listed.stdout.decode().strip() + if url: + return url + if attempt < attempts: + time.sleep(min(5 * attempt, max(deadline.remaining() - 1, 0))) + raise TrendError(f"asset {name} is not on release {tag} after {attempts} uploads: {error[-300:]}") + + +def promote(rows_dir: Path, select: str, verdict: dict | None, release: str, repository: str, + deadline: Deadline, uploader=upload_bundle, log=print) -> int: + """Promote the selected rows' bundles and write the result into each row + file. A failed promotion is recorded as failed, not left looking like a + row nobody selected; returns the number that failed.""" + failures = 0 + for path in sorted(rows_dir.glob("*.json")): + row = json.loads(path.read_text(encoding="utf-8")) + if not selected(row, select, verdict): + continue + monthly = release == "monthly" + tag = release + try: + tag = f"{BRANCH}/{month_of(row)}" if monthly else release + url = uploader(row, tag, repository, monthly, deadline) + row["promotion"] = {"status": "promoted", "release": tag, "asset": asset_name(row)} + row["asset_url"] = url + log(f"promoted {row.get('preset')} to {tag}: {url}") + # A full disk, a bad zip or a tar error while bundling is a failed + # promotion like any other, recorded on this row; the rest still run. + except (TrendError, OSError, zipfile.BadZipFile, tarfile.TarError) as error: + failures += 1 + row["promotion"] = {"status": "failed", "release": tag, "error": str(error)[-500:]} + row["asset_url"] = None + log(f"could not promote {row.get('preset')} to {tag}: {error}") + path.write_text(json.dumps(row, indent=2) + "\n", encoding="utf-8") + return failures + + +def read_rows(directory: Path) -> list[dict]: + rows = [] + for path in sorted(directory.glob("*.json")): + try: + rows.append(json.loads(path.read_text(encoding="utf-8"))) + except (OSError, ValueError) as error: + raise TrendError(f"{path}: {error}") from error + return rows + + +# --- main --------------------------------------------------------------- + + +def add_run_args(parser: argparse.ArgumentParser) -> None: + parser.add_argument("--run-id", type=int, required=True) + parser.add_argument("--run-attempt", type=int, default=1) + parser.add_argument("--event", required=True) + parser.add_argument("--repository", required=True) + parser.add_argument("--server-url", default="https://github.com") + parser.add_argument("--ref", default="") + parser.add_argument("--commit", default="") + parser.add_argument("--recorded-at", default=None, help="UTC; defaults to now") + parser.add_argument("--presets", type=Path, default=PRESETS) + + +def add_git_args(parser: argparse.ArgumentParser) -> None: + parser.add_argument("--repo", type=Path, default=Path.cwd(), help="a clone with push access") + parser.add_argument("--remote", default="origin") + parser.add_argument("--deadline-seconds", type=float, default=300.0) + + +def main(argv: list[str] | None = None) -> int: + parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) + sub = parser.add_subparsers(dest="action", required=True) + + r = sub.add_parser("row") + add_run_args(r) + r.add_argument("--dir", type=Path, required=True) + r.add_argument("--preset", required=True) + r.add_argument("--lane", default="auto", choices=("auto", *LANES)) + r.add_argument("--runner-class", required=True) + r.add_argument("--artifact-name", default="") + r.add_argument("--artifact-url", default="") + r.add_argument("--artifact-id", default="") + r.add_argument("--config", type=Path, default=regress.CONFIG) + r.add_argument("--out", type=Path, required=True) + + j = sub.add_parser("job-rows") + add_run_args(j) + j.add_argument("--needs", required=True, help="toJSON(needs)") + j.add_argument("--matrix", default="", help="the plan's matrix, to add missing preset rows") + j.add_argument("--rows", type=Path, required=True, help="the row directory to add to") + j.add_argument("--jobs", type=Path, default=None, + help="the run's jobs as JSON lines of {name, run_attempt} (jobs API, filter=latest)") + + f = sub.add_parser("fetch") + add_git_args(f) + f.add_argument("--out", type=Path, required=True) + + p = sub.add_parser("promote") + p.add_argument("--rows", type=Path, required=True) + p.add_argument("--select", choices=("policy", "all", "flagged", "none"), default="policy") + p.add_argument("--verdict", type=Path, help="the regression rule's verdict JSON") + p.add_argument("--release", default="monthly", help="`monthly` or an existing release's tag") + p.add_argument("--repository", required=True) + p.add_argument("--deadline-seconds", type=float, default=1500.0) + + a = sub.add_parser("append") + add_git_args(a) + a.add_argument("--rows", type=Path, required=True) + a.add_argument("--attempts", type=int, default=5) + a.add_argument("--backoff-seconds", type=float, default=5.0) + + c = sub.add_parser("cite") + add_git_args(c) + # The download, the tarball and up to three uploads, as `promote` allows. + c.set_defaults(deadline_seconds=1500.0) + c.add_argument("--artifact-id", type=int, required=True, + help="the artifact's ID (its URL's last part); the run and attempt are read from it") + c.add_argument("--lane", required=True, choices=LANES) + c.add_argument("--preset", required=True) + c.add_argument("--repository", required=True) + c.add_argument("--server-url", default="https://github.com") + c.add_argument("--append", action="store_true", help="also record the promotion row") + + args = parser.parse_args(argv) + try: + if args.action == "row": + row = build_row(args) + args.out.parent.mkdir(parents=True, exist_ok=True) + args.out.write_text(json.dumps(row, indent=2) + "\n", encoding="utf-8") + return 0 + if args.action == "job-rows": + args.rows.mkdir(parents=True, exist_ok=True) + for row in job_rows(args): + name = row.get("job") or f"missing-{row['preset']}-attempt{row['run_attempt']}" + (args.rows / f"{name}.json").write_text(json.dumps(row, indent=2) + "\n", encoding="utf-8") + return 0 + if args.action == "fetch": + tip = export(Branch(args.repo, args.remote, Deadline(args.deadline_seconds)), args.out) + print(f"{BRANCH}: {'at ' + tip if tip else 'does not exist yet; no history'}") + return 0 + if args.action == "promote": + verdict = json.loads(args.verdict.read_text(encoding="utf-8")) if args.verdict else None + failed = promote(args.rows, args.select, verdict, args.release, args.repository, + Deadline(args.deadline_seconds)) + return 1 if failed else 0 + if args.action == "append": + if args.attempts < 1: + raise TrendError("--attempts must be at least 1") + rows = read_rows(args.rows) + if not rows: + print(f"{BRANCH}: no rows to record") + return 0 + append_rows(Branch(args.repo, args.remote, Deadline(args.deadline_seconds)), rows, + args.attempts, args.backoff_seconds) + return 0 + if args.action == "cite": + deadline = Deadline(args.deadline_seconds) + meta = gh_json(f"repos/{args.repository}/actions/artifacts/{args.artifact_id}", deadline) + run_id = (meta.get("workflow_run") or {}).get("id") + if not isinstance(run_id, int): + raise TrendError(f"artifact {args.artifact_id} names no workflow run") + attempt = attempt_of_artifact(meta, args.repository, deadline) + row = { + "schema": ROW_SCHEMA, "kind": "promotion", "recorded_at": utc_now(), + "repository": args.repository, "run_id": run_id, "run_attempt": attempt, + "run_url": f"{args.server_url}/{args.repository}/actions/runs/{run_id}" + + (f"/attempts/{attempt}" if attempt > 1 else ""), + "event": "cite", "lane": args.lane, "preset": args.preset, + "artifact": {"name": meta.get("name"), "id": args.artifact_id, "url": None}, + } + tag = f"{BRANCH}/{month_of(row)}" + row["asset_url"] = upload_bundle(row, tag, args.repository, True, deadline) + row["promotion"] = {"status": "promoted", "release": tag, "asset": asset_name(row)} + print(json.dumps(row, sort_keys=True)) + if args.append: + append_rows(Branch(args.repo, args.remote, deadline), [row], 5, 5.0) + return 0 + except TrendError as error: + print(f"prism-load-trend: {error}", file=sys.stderr) + return 2 + except regress.RegressError as error: + print(f"prism-load-trend: {error}", file=sys.stderr) + return 2 + return 2 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/prism_pool_acquires.json b/scripts/prism_pool_acquires.json new file mode 100644 index 000000000..ce610a734 --- /dev/null +++ b/scripts/prism_pool_acquires.json @@ -0,0 +1,345 @@ +{ + "policies": { + "ledger": { + "class": "delegated", + "owner": "ledger", + "reason": "Ledger::acquire/::begin records once when metrics are attached; SQL/BEGIN uses that checkout. Operator Ledger instances may have no metrics." + }, + "timer": { + "class": "timed", + "owner": "metrics", + "reason": "Fresh pool checkout enclosed by time_pool_acquire; success ends before SQL/BEGIN, failure includes pending cancellation." + }, + "transaction": { + "class": "borrowed", + "owner": "ledger", + "reason": "BEGIN/rollback on an already acquired or detached connection; no additional pool checkout." + }, + "startup": { + "class": "startup", + "owner": "ledger startup/migrations", + "reason": "Schema, capability, provenance or session-setting gate before serving; intentionally excluded unless already using the timed transaction helper." + }, + "startup-generic": { + "class": "startup", + "owner": "ledger migrations", + "reason": "Generic Acquire helper: startup passes a pool, migration passes a borrowed connection. Exact callers are also guarded; no run-role work-loop caller." + }, + "public": { + "class": "public", + "owner": "public API", + "reason": "HTTP reads: public dispatch substitutes its isolated read pool, while private audit endpoints can retain the run pool. These operator/public API requests are explicitly excluded; public-role export policy remains separate." + }, + "public-hook": { + "class": "borrowed", + "owner": "public API", + "reason": "after_connect/before_acquire hook receives an existing PgConnection; SQL here is not another checkout." + }, + "operator": { + "class": "operator", + "owner": "operator recovery/tools", + "reason": "Read-only candidate inventory/recovery, fatal-state inspection, or self-check command; no run frontend call." + }, + "compatibility": { + "class": "compatibility", + "owner": "partition/rollup public API", + "reason": "Pool-only public wrapper has no metrics owner; retained for compatibility. The run frontend calls the metrics-aware entrypoint." + }, + "partition-lead": { + "class": "compatibility", + "owner": "share partitions", + "reason": "Pool-only public lead_rows utility; no production frontend caller. Metrics collector reads headroom on its existing timed transaction." + }, + "audit-public": { + "class": "compatibility", + "owner": "audit API", + "reason": "Public pool-only audit wrapper forwards None; production Ledger audit_bundle uses the metrics-aware reader. Connection-backed operator reconstruction borrows." + }, + "audit-forward": { + "class": "delegated", + "owner": "audit ledger", + "reason": "Forwards the optional metrics owner to AuditReader/verify_durable_range; each fresh checkout is timed at the leaf, borrowed connections add no sample." + }, + "maintenance-forward": { + "class": "delegated", + "owner": "server maintenance", + "reason": "Server/maintenance loop forwards its metrics owner to the timed checkout leaf; forwarding is not an additional observation." + }, + "audit-completeness": { + "class": "operator", + "owner": "operator audit tools", + "reason": "Generic Executor audit completeness query: operator import check uses a pool, self-check sample uses a borrowed connection; exact callers are guarded." + }, + "divergence-line": { + "class": "public", + "owner": "integrity report", + "reason": "Pool helper for the payout_divergence line (#478, #506), one plain read: the private audit endpoint (public) and self-check (operator) pass their pools; each caller is classified at its own site." + }, + "non-database": { + "class": "non-database", + "owner": "runtime", + "reason": "Lexical name collision with checkout/helper methods: CPU preparation, admission, runtime or command dispatch, rather than a fresh SQLx checkout at this call site." + }, + "claim-revocation": { + "class": "operator", + "owner": "candidate and fanout claim tests and operators", + "reason": "Generic Executor helpers that revoke candidate claims (#581) and CTV fanout claims (#654): the lease tests and an operator pass a pool or their own transaction; no run frontend call." + }, + "shielded-begin": { + "class": "borrowed", + "owner": "ledger", + "reason": "shielded_begin runs BEGIN on a connection the caller already checked out (and timed) in a task of its own (#482); no additional pool checkout." + } + }, + "sites": { + "ledger/archive.rs::seal#1::maintenance_connection(ledger)": [1,"ledger"], + "ledger/archive.rs::detach#1::maintenance_connection(ledger)": [1,"ledger"], + "ledger/archive.rs::drop_partition#1::maintenance_connection(ledger)": [1,"ledger"], + "ledger/append_admission.rs::begin#1::begin(& mut * * connection , Some ( statement . into ( ) ))": [1,"transaction"], + "ledger/audit.rs::audit_canonical_bytes_admitted#1::reader .connection(None)": [1,"audit-public"], + "ledger/audit.rs::audit_canonical_bytes_admitted#1::materialize_audit_row_with_metrics(& mut reader , & mut logical , permit , None , true)": [1,"audit-public"], + "ledger/audit.rs::materialize_audit_row_with_metrics#1::reader .connection(metrics)": [1,"audit-forward"], + "ledger/audit.rs::read_range#1::reader .connection(metrics)": [1,"audit-forward"], + "ledger/divergence.rs::reserve_offer_within#1::self .begin()": [2,"ledger"], + "ledger/divergence.rs::record_capture_disabled#1::self .begin()": [1,"ledger"], + "ledger/divergence.rs::as_issued_snapshot#1::self .acquire()": [1,"ledger"], + "ledger/divergence.rs::payout_divergence_line#1::fetch_one(pool)": [1,"divergence-line"], + "ledger/fatal_state.rs::inspect_fatal_state#1::read_fatal_state(pool . acquire ( ))": [1,"operator"], + "ledger/fatal_state.rs::fatal_state#1::read_fatal_state(self . acquire ( ))": [1,"ledger"], + "ledger/standby_durability.rs::await_standby_flush#1::self .acquire()": [2,"ledger"], + "ledger/standby_durability.rs::offer_standby_report#1::self .acquire()": [1,"ledger"], + "api.rs::audit#1::fetch_one(& state . pool)": [3,"public"], + "api.rs::audit#1::payout_divergence_line(& state . pool)": [1,"public"], + "api/charts.rs::hashrate_series#1::fetch_one(& state . pool)": [2,"public"], + "api/charts.rs::block_markers#1::fetch_one(& state . pool)": [1,"public"], + "api/public.rs::pool_summary#1::fetch_one(& state . pool)": [1,"public"], + "api/public.rs::miner#1::fetch_one(& state . pool)": [2,"public"], + "api/public.rs::miner#1::fetch_optional(& state . pool)": [1,"public"], + "api/public.rs::settlement_artifacts#1::fetch_one(& state . pool)": [1,"public"], + "api/public.rs::settlement_artifacts#1::fetch_optional(& state . pool)": [1,"public"], + "api/public_service.rs::read_pool#1::execute(connection)": [2,"public-hook"], + "api/public_service.rs::probe_once#1::fetch_one(& self . pool)": [2,"public"], + "api/read_models.rs::owed#1::fetch_one(& state . pool)": [1,"public"], + "api/read_models.rs::audit_payouts#1::fetch_one(& state . pool)": [1,"public"], + "api/read_models.rs::bundle#1::fetch_one(& state . pool)": [1,"public"], + "api/read_models.rs::bundle#1::fetch_optional(& state . pool)": [1,"public"], + "api/read_models.rs::bundle#1::materialize_audit_row(& state . pool , & mut value , Some ( permit ))": [1,"public"], + "api/read_models.rs::manifest_set#1::fetch_one(& state . pool)": [1,"public"], + "api/read_models.rs::fanout#1::fetch_one(& state . pool)": [1,"public"], + "api/read_models.rs::pending_fanouts#1::fetch_one(& state . pool)": [1,"public"], + "api/read_models.rs::artifact_document#1::fetch_optional(& state . pool)": [2,"public"], + "api/read_models.rs::artifact_document#1::audit_canonical_bytes_admitted(& state . pool , & block_hash , Some ( permit ))": [1,"public"], + "api/read_models.rs::artifact#1::fetch_optional(& state . pool)": [2,"public"], + "api/read_models.rs::blocks#1::fetch_one(& state . pool)": [1,"public"], + "api/read_models.rs::workers#1::fetch_one(& state . pool)": [1,"public"], + "api/read_models.rs::payouts#1::fetch_one(& state . pool)": [1,"public"], + "api/read_models.rs::reward_leaderboard#1::fetch_one(& state . pool)": [1,"public"], + "api/read_models.rs::leaderboard#1::fetch_one(& state . pool)": [1,"public"], + "api/read_models.rs::latest_evidence#1::fetch_optional(& state . pool)": [1,"public"], + "api/read_models.rs::latest_evidence#1::fetch_one(& state . pool)": [3,"public"], + "coordinator.rs::connect#1::fetch_one(& ledger . pool)": [2,"startup"], + "coordinator.rs::rebuild_claim_parts#1::match self . ledger .acquire()": [1,"ledger"], + "coordinator.rs::submit_loop_with#1::attempt .run(& self , lease)": [1,"non-database"], + "coordinator.rs::with_tracked_heartbeat#1::self . ledger .acquire()": [2,"ledger"], + "coordinator/issued_batcher.rs::with_dwell#1::run(ledger , receiver , shutdown . clone ( ) , dwell)": [1,"non-database"], + "coordinator/miner_submit.rs::deferred_share_disposition#1::self . ledger .acquire()": [1,"ledger"], + "coordinator/miner_submit.rs::persist_block_only#1::ledger .acquire()": [1,"ledger"], + "coordinator/refresh_window.rs::capture_refresh_window#1::prepare(std :: mem :: take ( & mut snapshot . snapshot . prior_balances ) , & admission ,)": [1,"non-database"], + "coordinator/work_ledger.rs::clocked_payout_revision#1::self .acquire()": [1,"ledger"], + "coordinator/work_ledger.rs::now_ms#1::self .acquire()": [1,"ledger"], + "coordinator/work_ledger.rs::window_held#1::self .acquire()": [1,"ledger"], + "ledger/archive.rs::lifecycle_guard#1::ledger .acquire()": [1,"ledger"], + "ledger/archive.rs::plan#1::ledger .begin()": [1,"ledger"], + "ledger/archive.rs::seal#1::ledger .acquire()": [2,"ledger"], + "ledger/archive.rs::seal#1::audit_canonical_bytes(& mut * connection , & block_hash)": [1,"transaction"], + "ledger/archive.rs::seal#1::ledger .begin()": [1,"ledger"], + "ledger/archive.rs::archive#1::ledger .acquire()": [1,"ledger"], + "ledger/archive.rs::archive#1::ledger .begin()": [1,"ledger"], + "ledger/archive.rs::verify#1::ledger .acquire()": [1,"ledger"], + "ledger/archive.rs::verify#1::ledger .begin()": [1,"ledger"], + "ledger/archive.rs::maintenance_connection#1::ledger .acquire()": [1,"ledger"], + "ledger/archive.rs::detach#1::ledger .acquire()": [2,"ledger"], + "ledger/archive.rs::attachment2#1::ledger .acquire()": [1,"ledger"], + "ledger/archive.rs::record_detached#1::ledger .begin()": [1,"ledger"], + "ledger/archive.rs::drop_partition#1::ledger .acquire()": [2,"ledger"], + "ledger/archive.rs::drop_partition#1::ledger .begin()": [1,"ledger"], + "ledger/archive.rs::restore#1::ledger .acquire()": [1,"ledger"], + "ledger/archive.rs::restore#1::ledger .begin()": [1,"ledger"], + "ledger/audit.rs::audit_completeness#1::fetch_one(executor)": [1,"audit-completeness"], + "ledger/audit.rs::connection#1::pool .acquire() [timer: metrics]": [1,"timer"], + "ledger/audit.rs::audit_canonical_bytes#1::audit_canonical_bytes_admitted(reader , block_hash , None)": [1,"audit-public"], + "ledger/audit.rs::materialize_audit_row#1::materialize_audit_row_with_metrics(reader , row , permit , None , false)": [1,"audit-public"], + "ledger/audit.rs::materialize_audit_row_with_metrics#1::read_range(& mut reader , snapshot . try_get ( STRING ) ? , snapshot . try_get ( STRING ) ? , snapshot . try_get ( STRING ) ? , metrics ,)": [1,"audit-forward"], + "ledger/audit.rs::audit_bundle#1::self .acquire()": [1,"ledger"], + "ledger/audit.rs::audit_bundle#1::materialize_audit_row_with_metrics(& self . pool , & mut logical , None , self . metrics . as_deref ( ) , false ,)": [1,"audit-forward"], + "ledger/audit.rs::verify_durable_range#1::pool .acquire() [timer: metrics]": [4,"timer"], + "ledger/blocks.rs::land_candidate_checked#1::self .acquire()": [1,"ledger"], + "ledger/blocks.rs::land_candidate_checked#1::verify_durable_range(& self . pool , & landing . snapshot , self . metrics . as_deref ( ))": [1,"audit-forward"], + "ledger/blocks.rs::land_candidate_checked#1::self .begin()": [1,"ledger"], + "ledger/blocks.rs::finish_candidate_counted_at_revision#1::self .begin()": [1,"ledger"], + "ledger/blocks.rs::orphan_candidate_at_revision#1::self .begin()": [1,"ledger"], + "ledger/blocks.rs::pool_blocks_for_reconcile#1::self .acquire()": [1,"ledger"], + "ledger/blocks.rs::reconcile_blocks_at_revision#1::self .begin()": [1,"ledger"], + "ledger/blocks.rs::claim_fanout#1::self .begin()": [1,"ledger"], + "ledger/blocks.rs::finish_fanout#1::self .begin()": [1,"ledger"], + "ledger/candidates.rs::enqueue_candidate_observed#1::self .begin()": [1,"ledger"], + "ledger/candidates.rs::claim_candidate#1::self .begin()": [1,"ledger"], + "ledger/candidates.rs::park_invalid_candidate#1::self .begin()": [1,"ledger"], + "ledger/candidates.rs::landed_audit#1::self .acquire()": [1,"ledger"], + "ledger/candidates.rs::record_landed_audit_bits#1::self .acquire()": [1,"ledger"], + "ledger/candidates.rs::renew_candidate_claim#1::self .begin()": [1,"ledger"], + "ledger/candidates.rs::release_candidate_claim#1::self .begin()": [1,"ledger"], + "ledger/candidates.rs::record_offer#1::self .begin()": [1,"ledger"], + "ledger/candidates.rs::release_unsent_offer#1::self .begin()": [1,"ledger"], + "ledger/candidates.rs::adopt_active_candidate#1::self .begin()": [1,"ledger"], + "ledger/candidates.rs::reconcile_candidate#1::self .begin()": [1,"ledger"], + "ledger/candidates.rs::list_candidates#1::read_only_pool(url)": [1,"operator"], + "ledger/candidates.rs::list_candidates#1::fetch_all(& pool)": [1,"operator"], + "ledger/candidates.rs::abandon_candidate#1::self .begin()": [1,"ledger"], + "ledger/candidates.rs::open#1::read_only_pool(url)": [1,"operator"], + "ledger/candidates.rs::rows#1::fetch_all(& self . pool)": [1,"operator"], + "ledger/candidates.rs::unfinished_states#1::fetch_all(& self . pool)": [1,"operator"], + "ledger/candidates.rs::claim_candidate_for_recovery#1::self .begin()": [1,"ledger"], + "ledger/candidates.rs::release_recovery_token#1::self .begin()": [1,"ledger"], + "ledger/candidates.rs::revoke_candidate_claims#1::execute(executor)": [1,"claim-revocation"], + "ledger/fanout.rs::revoke_fanout_claims#1::execute(executor)": [1,"claim-revocation"], + "ledger/connect.rs::release#1::pool .acquire() [timer: self . metrics . as_deref ( )]": [1,"timer"], + "ledger/connect.rs::drop#2::pool .acquire() [timer: metrics . as_deref ( )]": [1,"timer"], + "ledger/connect.rs::begin#1::shielded_begin(self . acquire ( ) . await ?)": [1,"shielded-begin"], + "ledger/connect.rs::begin#1::self .acquire()": [1,"ledger"], + "ledger/connect.rs::connect_inner#1::begin(& pool , metrics . as_deref ( ))": [1,"maintenance-forward"], + "ledger/connect.rs::connect_inner#1::apply_online_migration(& pool , pending , options . index_build , metrics . as_deref ( ) ,)": [1,"maintenance-forward"], + "ledger/connect.rs::connect_inner#1::require_schema_version(& pool)": [1,"startup"], + "ledger/connect.rs::connect_inner#1::require_known_capabilities(& pool)": [1,"startup"], + "ledger/connect.rs::connect_inner#1::require_migration_source(& ledger . pool)": [1,"startup"], + "ledger/connect.rs::connect_inner#1::ledger .begin()": [1,"ledger"], + "ledger/connect.rs::configure#1::self .begin()": [1,"ledger"], + "ledger/connect.rs::release_session_owner_reservations#1::self .acquire()": [1,"ledger"], + "ledger/connect.rs::new_session_id#1::self .begin()": [1,"ledger"], + "ledger/connect.rs::payout_revision#1::self .acquire()": [1,"ledger"], + "ledger/connect.rs::begin#2::shielded_begin(time_pool_acquire ( metrics , pool . acquire ( ) ) . await ?)": [1,"shielded-begin"], + "ledger/connect.rs::begin#2::pool .acquire() [timer: metrics]": [1,"timer"], + "ledger/connect.rs::shielded_begin#1::begin(connection , None)": [1,"transaction"], + "ledger/connect/acquire.rs::acquire#1::self . pool .acquire() [timer: self . metrics . as_deref ( )]": [1,"timer"], + "ledger/difficulty.rs::worker_difficulty#1::self .acquire()": [1,"ledger"], + "ledger/difficulty.rs::record_worker_difficulty#1::self .begin()": [1,"ledger"], + "ledger/difficulty.rs::lower_worker_difficulty#1::self .begin()": [1,"ledger"], + "ledger/difficulty.rs::prune_worker_difficulties#1::self .begin()": [1,"ledger"], + "ledger/difficulty.rs::share_accepted_at_ms#1::self .acquire()": [1,"ledger"], + "ledger/fanout.rs::renew_fanout_claim#1::self .begin()": [1,"ledger"], + "ledger/fanout.rs::release_fanout_claim#1::self .begin()": [1,"ledger"], + "ledger/fanout.rs::hand_back_fanout#1::self .begin()": [1,"ledger"], + "ledger/fanout.rs::record_fanout_scan#1::self .begin()": [1,"ledger"], + "ledger/fanout.rs::observe_fanout#1::self .begin()": [1,"ledger"], + "ledger/fanout.rs::halt_fanout_reorg#1::self .begin()": [1,"ledger"], + "ledger/fanout.rs::cpfp_package#1::self .acquire()": [1,"ledger"], + "ledger/fanout.rs::reserve_cpfp_funding#1::self .begin()": [1,"ledger"], + "ledger/fanout.rs::retire_unsigned_cpfp_funding#1::self .begin()": [1,"ledger"], + "ledger/fanout.rs::retired_cpfp_funding#1::self .acquire()": [1,"ledger"], + "ledger/fanout.rs::record_retired_cpfp_wallet_cleanup#1::self .begin()": [1,"ledger"], + "ledger/fanout.rs::save_cpfp_package#1::self .begin()": [1,"ledger"], + "ledger/fanout.rs::mark_cpfp_wallet_unlocked#1::self .begin()": [1,"ledger"], + "ledger/fanout.rs::mark_cpfp_wallet_lock_pending#1::self .begin()": [1,"ledger"], + "ledger/fatal_state.rs::inspect_fatal_state#1::pool .acquire()": [1,"operator"], + "ledger/fatal_state.rs::fatal_state#1::self .acquire()": [1,"ledger"], + "ledger/fatal_state.rs::clear_fatal_state_in#1::self .begin()": [1,"ledger"], + "ledger/submission_hold.rs::submission_hold#1::self .acquire()": [1,"ledger"], + "ledger/submission_hold.rs::health_reads#1::self .acquire()": [1,"ledger"], + "ledger/submission_hold.rs::set_submission_hold#1::self .begin()": [1,"ledger"], + "ledger/submission_hold.rs::clear_submission_hold#1::self .begin()": [1,"ledger"], + "ledger/instances.rs::heartbeat#1::self .acquire()": [1,"ledger"], + "ledger/jobs.rs::save_job#1::self .begin()": [1,"ledger"], + "ledger/jobs.rs::save_issued_job#1::self .begin()": [1,"ledger"], + "ledger/jobs.rs::job#1::self .acquire()": [1,"ledger"], + "ledger/jobs.rs::prune_expired_jobs#1::self .acquire()": [1,"ledger"], + "ledger/jobs/cleanup.rs::sweep_blobs#1::self .begin()": [1,"ledger"], + "ledger/jobs/compact_issued.rs::save_issued_job_compact#1::self .begin()": [1,"ledger"], + "ledger/jobs/compact_issued_batch.rs::drop#1::execute(& mut * connection , STRING)": [1,"transaction"], + "ledger/jobs/compact_issued_batch.rs::save_compact_batch#1::self .acquire()": [1,"ledger"], + "ledger/jobs/compact_issued_batch.rs::save_compact_batch#1::connection . connection . as_mut ( ) . unwrap ( ) .begin()": [1,"transaction"], + "ledger/jobs/prepared.rs::save_compact_prepared#1::self .begin()": [1,"ledger"], + "ledger/jobs/prepared.rs::compact_prepared_with_admission#1::self .acquire()": [1,"ledger"], + "ledger/migration.rs::inspect_source_schema#1::read_capabilities(& mut * * tx)": [1,"transaction"], + "ledger/migration.rs::read_capabilities#1::executor .acquire()": [1,"startup-generic"], + "ledger/migration.rs::require_migration_history#1::fetch_one(connection)": [1,"transaction"], + "ledger/migration.rs::migrate_schema#1::require_migration_source(& mut * * tx)": [1,"transaction"], + "ledger/migration.rs::migrate_schema#1::require_known_capabilities(& mut * * tx)": [1,"transaction"], + "ledger/migration.rs::require_schema_version#1::fetch_one(pool)": [1,"startup"], + "ledger/migration.rs::require_schema_version#1::pool .acquire()": [1,"startup"], + "ledger/migration.rs::require_schema_version#1::read_capabilities(& mut * connection)": [1,"startup-generic"], + "ledger/migration.rs::require_known_capabilities#1::executor .acquire()": [1,"startup-generic"], + "ledger/migration.rs::require_known_capabilities#1::read_capabilities(& mut * connection)": [2,"startup-generic"], + "ledger/migration.rs::read_migration_source#1::executor .acquire()": [1,"startup-generic"], + "ledger/migration.rs::require_migration_source#1::read_migration_source(executor)": [1,"startup-generic"], + "ledger/migration.rs::migration_source#1::self .acquire()": [1,"ledger"], + "ledger/migration.rs::pending_share_hash_backfill#1::self .acquire()": [1,"ledger"], + "ledger/migration.rs::backfill_share_hashes#1::self .acquire()": [1,"ledger"], + "ledger/migration.rs::import_legacy_audits_with#1::self .acquire()": [2,"ledger"], + "ledger/migration.rs::write_legacy_audit#1::self .acquire()": [1,"ledger"], + "ledger/migration.rs::write_legacy_audit#1::connection .begin()": [1,"transaction"], + "ledger/migration.rs::backfill_ctv#1::self .acquire()": [1,"ledger"], + "ledger/migration.rs::backfill_ctv#1::self .begin()": [1,"ledger"], + "ledger/migration/online.rs::apply_online_migration#1::pool .acquire() [timer: metrics]": [1,"timer"], + "ledger/migration/online.rs::apply_concurrently#1::connection .begin()": [1,"transaction"], + "ledger/migration/online.rs::apply_offline#1::connection .begin()": [1,"transaction"], + "ledger/migration/online.rs::drop_planned#1::describe(name)": [1,"non-database"], + "ledger/migration/online.rs::drop_planned#1::describe()": [1,"non-database"], + "ledger/migration/online.rs::verify_declared#1::describe()": [1,"non-database"], + "ledger/migration/online.rs::verify_declared#1::describe(name)": [2,"non-database"], + "ledger/migration/partition.rs::apply#1::connection .begin()": [1,"transaction"], + "ledger/migration/share_hashes.rs::map_batches#1::connection .begin()": [1,"transaction"], + "ledger/migration/share_hashes.rs::throttled#1::connection .begin()": [1,"transaction"], + "ledger/migration/share_hashes.rs::run_to_end#1::connection .begin()": [1,"transaction"], + "ledger/migration/share_hashes.rs::record_transaction#1::connection .begin()": [1,"transaction"], + "ledger/migration/share_hashes.rs::map_recent#1::connection .begin()": [1,"transaction"], + "ledger/policy_transition.rs::transition_policy_in#1::self .begin()": [1,"ledger"], + "ledger/signing_transition.rs::transition_signing_in#1::self .begin()": [1,"ledger"], + "ledger/window.rs::read_window_owned#1::self .begin()": [1,"ledger"], + "ledger/window.rs::chain_observation_state#1::self .acquire()": [1,"ledger"], + "ledger/window.rs::observe_chain_view_checked#1::self .begin()": [1,"ledger"], + "ledger/window.rs::append_checked#1::ensure_with_metrics(& self . pool , self . metrics . as_deref ( ))": [1,"maintenance-forward"], + "ledger/window.rs::append_prepared#1::acquire(& self . pool)": [1,"non-database"], + "ledger/window.rs::append_prepared#1::self .acquire()": [1,"ledger"], + "ledger/window.rs::append_prepared#1::connection .begin(Self :: APPEND_TRANSACTION_BEGIN)": [1,"transaction"], + "ledger/window.rs::snapshot_with_admission#1::self .begin()": [2,"ledger"], + "ledger/window.rs::snapshot_with_admission#1::advance(& mut tx , prior , weight , anchor_ms , cutoff , & completion)": [1,"transaction"], + "ledger/window/payout_state.rs::payout_state#1::self .begin()": [1,"ledger"], + "ledger/window/payout_state.rs::refresh_probe#1::self .begin()": [1,"ledger"], + "main.rs::main#1::prepare()": [1,"non-database"], + "metrics/collectors.rs::database#1::pool .acquire() [timer: Some ( metrics )]": [1,"timer"], + "metrics/collectors.rs::database#1::shielded_begin(connection)": [1,"shielded-begin"], + "metrics/collectors.rs::run#1::database(& pool , & metrics)": [1,"maintenance-forward"], + "partitions.rs::ensure#1::ensure_with_metrics(pool , None)": [1,"compatibility"], + "partitions.rs::ensure_with_metrics#1::begin(time_pool_acquire ( metrics , pool . acquire ( ) ) . await ? , None)": [1,"transaction"], + "partitions.rs::ensure_with_metrics#1::pool .acquire() [timer: metrics]": [1,"timer"], + "partitions.rs::lead_rows#1::fetch_one(pool)": [1,"partition-lead"], + "partitions.rs::run#1::run_with_metrics(pool , settings , shutdown , None)": [1,"compatibility"], + "partitions.rs::run_with_metrics#1::ensure_with_metrics(& pool , metrics . as_deref ( ))": [1,"maintenance-forward"], + "rollups.rs::advance#1::advance_with_metrics(pool , batch , None)": [1,"compatibility"], + "rollups.rs::advance_with_metrics#1::begin(time_pool_acquire ( metrics , pool . acquire ( ) ) . await ? , None)": [1,"transaction"], + "rollups.rs::advance_with_metrics#1::pool .acquire() [timer: metrics]": [1,"timer"], + "rollups.rs::run#1::run_with_metrics(pool , settings , shutdown , None)": [1,"compatibility"], + "rollups.rs::run_with_metrics#1::advance_with_metrics(& pool , settings . batch , metrics . as_deref ( ))": [1,"maintenance-forward"], + "server.rs::run#1::ensure_with_metrics(& coordinator . ledger . pool , Some ( & registry ))": [1,"maintenance-forward"], + "server.rs::run#1::run(coordinator . clone ( ) , shutdown_rx . clone ( ))": [1,"non-database"], + "server.rs::run#1::run_with_metrics(coordinator . ledger . pool . clone ( ) , settings , shutdown_rx . clone ( ) , Some ( registry . clone ( ) ) ,)": [1,"maintenance-forward"], + "server.rs::run#1::run_with_metrics(coordinator . ledger . pool . clone ( ) , partition_settings , shutdown_rx . clone ( ) , Some ( registry . clone ( ) ) ,)": [1,"maintenance-forward"], + "server.rs::run#1::runtime . clone ( ) .run(shutdown_rx . clone ( ))": [1,"non-database"], + "server.rs::run#1::run(metrics , coordinator . ledger . pool . clone ( ) , shutdown_rx . clone ( ) ,)": [1,"maintenance-forward"], + "stratum.rs::prepare_job#1::config . initial_job_limit .acquire()": [1,"non-database"], + "stratum.rs::prepare_job#1::config . rebuild_job_limit .acquire()": [1,"non-database"], + "stratum.rs::prepare_job#1::match config . rebuild_job_limit .try_acquire()": [1,"non-database"], + "stratum.rs::request#1::begin(metrics , is_submit , received_at)": [1,"non-database"], + "stratum.rs::run_listener#1::config . connection_limit .try_acquire()": [1,"non-database"], + "stratum.rs::timed_out#1::database(message)": [1,"non-database"], + "tools.rs::prepare#1::run(command , transition)": [1,"non-database"], + "tools.rs::run#1::run(Config :: from_env ( ) ?)": [1,"non-database"], + "tools.rs::run#1::audit_completeness(& ledger . pool)": [1,"operator"], + "tools.rs::run#1::run(args)": [1,"non-database"], + "tools.rs::sample_audit_completeness#1::audit_completeness(& mut connection)": [1,"operator"], + "tools.rs::self_check_local#1::coordinator . ledger . pool .begin()": [1,"operator"], + "tools.rs::self_check_local#1::payout_divergence_line(& coordinator . ledger . pool)": [1,"operator"], + "tools.rs::self_check_local#1::fetch_all(& coordinator . ledger . pool)": [1,"operator"], + "tools.rs::self_check_local#1::fetch_optional(& coordinator . ledger . pool)": [1,"operator"] + } +} diff --git a/scripts/prism_pool_wait_scenarios.py b/scripts/prism_pool_wait_scenarios.py new file mode 100644 index 000000000..f89af7675 --- /dev/null +++ b/scripts/prism_pool_wait_scenarios.py @@ -0,0 +1,169 @@ +"""Disposable PromQL regressions for #336; no database or deployment access.""" + +TITLE = "PrismDatabasePoolWaitHigh" +HISTOGRAM = "qbit_prism_database_pool_acquire_seconds" +BUCKETS = ("0.01", "0.025", "0.05", "0.1", "0.25", "0.5", "1", "2.5", "5", "10", "30", "+Inf") +IDENTITY = {"job": "qbit-prism", "instance": "one", "network": "mainnet"} +SCRAPE_GATE = (' and on(job, instance, network) ' + '(up{job="qbit-prism",network="mainnet"} == 1)') +SNAPSHOT_GATE = (' and on(job, instance, network) ' + '((qbit_prism_metrics_snapshot_available{job="qbit-prism",network="mainnet"} == 1) ' + 'and on(job, instance, network) ' + '(qbit_prism_metrics_snapshot_stale{job="qbit-prism",network="mainnet"} == 0))') +OLD_GATE = (' and on(job, instance, network) ' + '(qbit_prism_collector_available{job="qbit-prism",network="mainnet",' + 'collector="database"} == 1)') +NEGATIVE_CONTROL_SCENARIOS = ( + "pool wait: slow waits survive unavailable collector", + "pool wait: publisher freshness cannot reset live pool dwell", +) + + +def old_pool_wait_expression(expression): + """Restore both original guards, rather than adding a collector gate to up.""" + assert expression.endswith(SCRAPE_GATE), "pool wait rule must use current scrape availability" + return expression[:-len(SCRAPE_GATE)] + SNAPSHOT_GATE + OLD_GATE + + +def series_name(metric, **labels): + return metric + "{" + ",".join(f'{key}="{value}"' for key, value in sorted(labels.items())) + "}" + + +def target(instance="one", network="mainnet", job="qbit-prism", *, + slow=True, count="0+1x900", collector=0, available=1, stale=0, up=1, + outcomes=("failure",)): + """One-second scrapes; bucket/count pairs describe real cumulative waits.""" + labels = dict(job=job, instance=instance, network=network) + rows = {} + for name, value in [("collector_available", collector), + ("metrics_snapshot_available", available), + ("metrics_snapshot_stale", stale), ("up", up)]: + if value is not None: + extra = {"collector": "database"} if name == "collector_available" else {} + metric = name if name == "up" else "qbit_prism_" + name + rows[series_name(metric, **labels, **extra)] = ( + value if isinstance(value, str) else f"{value}+0x900") + if count is not None: + for outcome in outcomes: + result = dict(labels, result=outcome) + # Match the producer buckets: slow waits are in (1, 2.5], + # healthy waits in (0.05, 0.1]. + for le in BUCKETS: + rows[series_name(HISTOGRAM + "_bucket", **result, le=le)] = ( + "0+0x900" if float(le) < (2.5 if slow else 0.1) else count) + rows[series_name(HISTOGRAM + "_count", **result)] = count + return rows + + +def pool_wait_tests(rule, expression): + labels = {"severity": rule["severity"], "service": rule["service"], **rule["labels"]} + assert rule["evaluator"] == "gt" and rule["threshold"] == 0 + alert = {"alert": TITLE, "expr": f"({expression}) > 0", "for": rule["for"], "labels": labels} + old_expression = old_pool_wait_expression(expression) + tests = [] + + def samples(values): + return [{"labels": series_name("", **identity), "value": value} for identity, value in values] + + def expr_check(expr, values, at="6m"): + return {"expr": expr, "eval_time": at, "exp_samples": samples(values)} + + def alert_check(identities, at="6m"): + return {"alertname": TITLE, "eval_time": at, + "exp_alerts": [{"exp_labels": {**identity, **labels}, "exp_annotations": {}} + for identity in identities]} + + def scenario(name, rows, values, firing=(), *, old_values=None): + test = {"name": "pool wait: " + name, "interval": "1s", + "input_series": [{"series": key, "values": value} for key, value in rows.items()], + "promql_expr_test": [expr_check(expression, values)], + "alert_rule_test": [alert_check([], "2m"), alert_check(firing)]} + if old_values is not None: + test["promql_expr_test"].append(expr_check(old_expression, old_values)) + tests.append(test) + return test + + high, low = [(IDENTITY, 1)], [(IDENTITY, 0)] + scenario("slow waits survive unavailable collector", target(), high, [IDENTITY], old_values=[]) + scenario("successful slow waits also survive collector failure", + target(outcomes=("success",)), high, [IDENTITY], old_values=[]) + scenario("healthy low waits with unavailable collector", target(slow=False), low) + scenario("healthy low waits with available collector", target(slow=False, collector=1), low) + scenario("slow waits with available collector", target(collector=1), high, [IDENTITY], old_values=high) + scenario("collector series absent", target(collector=None), high, [IDENTITY], old_values=[]) + for name, kwargs in [("stale snapshot does not hide live waits", {"stale": 1}), + ("unpublished snapshot does not hide live waits", {"available": 0, "stale": 1}), + ("missing body availability does not hide live waits", {"available": None}), + ("missing body freshness does not hide live waits", {"stale": None}), + ("no snapshot series does not hide live waits", {"available": None, "stale": None})]: + scenario(name, target(collector=1, **kwargs), high, [IDENTITY], old_values=[]) + scenario("failed scrape suppresses pool waits", target(up=0), []) + scenario("missing scrape availability suppresses pool waits", target(up=None), []) + scenario("missing target", {}, []) + scenario("no histogram series", target(count=None), []) + scenario("zero observations cannot fabricate a percentile", target(count="0+0x900"), []) + scenario("old observations outside window", target(count="100+0x900"), []) + scenario("one isolated scrape cannot establish a rate", + target(count=" ".join(["_"] * 360 + ["1", "stale"])), []) + # Ten is a per-instance minimum; the two outcome series are combined. + scenario("nine observations are insufficient", target(count="0+0x239 9+0x660"), []) + scenario("ten observations are sufficient", target(count="0+0x239 10+0x660"), high) + scenario("counts combine outcomes", target(count="0+0x239 5+0x660", + outcomes=("success", "failure")), high) + scenario("counts never combine instances", { + **target("one", count="0+0x239 6+0x660"), + **target("two", count="0+0x239 6+0x660"), + }, []) + two = dict(IDENTITY, instance="two") + scenario("independent instances keep their labels", { + **target("one"), **target("two", slow=False, collector=1), + }, high + [(two, 0)], [IDENTITY], old_values=[(two, 0)]) + scenario("successful peer scrape cannot authorize failed instance", { + **target("one", up=0), **target("two", slow=False), + }, [(two, 0)]) + scenario("successful peer scrape cannot authorize missing instance", { + **target("one", up=None), **target("two", slow=False), + }, [(two, 0)]) + scenario("other job and network cannot supply sample count", { + **target(count="0+0x239 9+0x660"), + **target(network="signet"), **target(job="other-prism"), + }, []) + scenario("other job and network cannot supply scrape availability", { + **target(up=None), + **target(network="signet"), **target(job="other-prism"), + }, []) + # A blocked publisher can repeatedly age the cached body while successful + # scrapes continue to include new live acquisition observations. Even with + # a healthy collector, thirty-second body-freshness interruptions prevent + # the old rule from ever earning its three-minute dwell. + intermittent_stale = " ".join(f"{index % 2}+0x29" for index in range(31)) + for name, collector in [("sustained exhaustion survives periodic stale body", 0), + ("publisher freshness cannot reset live pool dwell", 1)]: + stalled = scenario(name, target(stale=intermittent_stale, collector=collector), + high, [IDENTITY], old_values=high if collector else []) + stalled["promql_expr_test"] += [expr_check(expression, high, "6m30s"), + expr_check(old_expression, [], "6m30s")] + stalled["alert_rule_test"] += [alert_check([IDENTITY], "6m30s"), + alert_check([IDENTITY], "7m")] + # The collector recovers at 6m01s, then only low waits are observed. + # Existing slow observations keep the alert active until the window drains. + rows = target(collector="0+0x360 1+0x539") + for le in ("0.1", "0.25", "0.5", "1"): + rows[series_name(HISTOGRAM + "_bucket", **IDENTITY, result="failure", le=le)] = ( + "0+0x360 1+1x539") + recovery = scenario("recovery drains the five minute window", rows, high, [IDENTITY], old_values=[]) + recovery["promql_expr_test"] += [expr_check(expression, high, "6m1s"), + expr_check(expression, low, "11m")] + recovery["alert_rule_test"] += [alert_check([IDENTITY], "6m1s"), alert_check([], "11m")] + # Repeated scrapes of frozen counters must resolve through the sample guard. + frozen = scenario("frozen histogram ages out", target(count="0+1x360 360+0x539"), high, [IDENTITY]) + frozen["promql_expr_test"].append(expr_check(expression, [], "11m")) + frozen["alert_rule_test"].append(alert_check([], "11m")) + # A failed scrape clears firing state even while the preceding observations + # remain inside the rate window. A successful scrape earns a new dwell. + restored = scenario("scrape recovery restarts dwell", target(up="1+0x360 0+0x118 1+0x420", + count="0+1x360 360+0x118 361+1x420"), high, [IDENTITY]) + restored["promql_expr_test"] += [expr_check(expression, [], "7m"), expr_check(expression, high, "8m")] + restored["alert_rule_test"] += [alert_check([], "7m"), alert_check([], "10m59s"), + alert_check([IDENTITY], "11m")] + return alert, tests diff --git a/scripts/prism_shipped_image_lane.py b/scripts/prism_shipped_image_lane.py new file mode 100644 index 000000000..7a20b9dfd --- /dev/null +++ b/scripts/prism_shipped_image_lane.py @@ -0,0 +1,1099 @@ +#!/usr/bin/env python3 +"""#487 lane L6 (#544): the shipped images under the Compose `prism` profile +with the HA overlay, against a real regtest node. + +The lane builds the images the way an operator does (`docker compose build` +from compose.yaml), brings up `compose.yaml` + `compose.prism-ha.yaml` with +the `prism` profile (qbitd, the PostgreSQL primary and its replica, two +mining frontends and the public API), and holds it to these checks, which +test/e2e-scenarios.toml cites as the lane's evidence: + +- `ha-frontends-ready`: both frontends answer `/healthz` with `ok: true` and + distinct instance IDs, and each one's `self-check` observes both as live. +- `cross-frontend-resume`: #281 criterion 1 on the images. Work issued by + frontend 1 is accepted once on frontend 2 over a real socket; another + worker cannot submit it, and a replay on either frontend is rejected. +- `cpuminer-found-block`: cpuminer-opt (`docker/real-miner`), a Stratum client + we did not write, has shares accepted on frontend 1 and solves a block the + node keeps on its active chain. +- `prism-miner-found-block`: the same for `qbit-prism-miner` on frontend 2. +- `found-block-audit-verifies`: each client's latest found block's audit + bundle, served identically by both frontends, verifies with + `qbit-prism-audit-verify` against the node's coinbase and the lane's + ledger writer key. +- `ledger-reconciles`: each client's ledger rows equal the shares it saw + acknowledged, with none of its submissions left unanswered (see + `reconcile_client`), each credited header is credited once, both frontends + wrote shares, and both frontends' `/audit/latest` report the ledger's + accepted-share count. +- `public-api-lists-found-blocks`: the public API, reading the replica, + lists every verified block. + +Usage (the workflow runs the three in order; `run` tears the stack down): + + python3 scripts/prism_shipped_image_lane.py prepare --work DIR + python3 scripts/prism_shipped_image_lane.py build --work DIR + python3 scripts/prism_shipped_image_lane.py run --work DIR --out OUT + +`changed` reads changed file names on stdin and prints `true` when one is a +path the lane watches (WATCHED_PATHS), `false` otherwise; the workflow's +`changes` job uses it to decide whether a PR or push runs the lane. + +`prepare` stages the pinned qbit source (scripts/prepare-qbit-source.sh) and +writes DIR/lane.env: fresh signing seeds for this run, the ledger writer key +derived from them, the public Stratum URL the public API requires, and the +frontends' instance IDs. Nothing else overrides the checked-in lab defaults +in the upstream pins (config/upstream.env, as the Makefile and +prepare-qbit-source.sh read it) and .env.example. Requires Docker Compose +2.24.4 or newer, openssl and a free host port set 3340-3344 and 18452. +""" + +from __future__ import annotations + +import argparse +import base64 +from dataclasses import dataclass, field +import fnmatch +from fractions import Fraction +import hashlib +import json +import os +from pathlib import Path +import re +import secrets +import socket +import subprocess +import sys +import threading +import time +import urllib.error +import urllib.request + +ROOT = Path(__file__).resolve().parents[1] +PREFIX = "prism-shipped-images" + +# The lane's named checks. test/e2e-scenarios.toml cites them as the L6 +# evidence and scripts/check_e2e_scenarios.py holds the two to each other. +CHECKS = ( + "ha-frontends-ready", + "cross-frontend-resume", + "cpuminer-found-block", + "prism-miner-found-block", + "found-block-audit-verifies", + "ledger-reconciles", + "public-api-lists-found-blocks", +) + +# What a PR or push must touch to run the lane: every Dockerfile, the Compose +# files, and what the images the lane builds are built or configured from. +# fnmatch patterns, whose `*` also matches `/`. +WATCHED_PATHS = ( + "Dockerfile", + "*/Dockerfile", + "compose*.yaml", + ".env.example", + "config/upstream.env", + "config/upstream.env.example", + "config/qbit/*", + "config/prism-postgres/*", + "docker/qbit/*", + "docker/real-miner/*", + "lab/prism/*", + "lab/real-miner/*", + "scripts/prepare-qbit-source.sh", + "scripts/prism_shipped_image_lane.py", + ".github/workflows/prism-load-nightly.yml", +) + +COMPOSE_FILES = ("compose.yaml", "compose.prism-ha.yaml") +# The upstream pins as the Makefile and scripts/prepare-qbit-source.sh pick +# them: config/upstream.env, or its example when a checkout has none. The +# images must build from the same pins as the qbit source they stage. +UPSTREAM_ENV_FILE = ( + "config/upstream.env" if (ROOT / "config/upstream.env").is_file() else "config/upstream.env.example" +) +ENV_FILES = (UPSTREAM_ENV_FILE, ".env.example") +INSTANCE_IDS = ("l6-frontend-1", "l6-frontend-2") +# compose.prism-ha.yaml's default host ports. +STRATUM_PORTS = (3340, 3343) +HEALTH_PORTS = (3341, 3344) +PUBLIC_API_PORT = 3342 +QBIT_RPC_PORT = 18452 +# The lab defaults in .env.example; the lane does not override them. +QBIT_RPC_USER = "qbitrpc" +QBIT_RPC_PASSWORD = "change-this" +POSTGRES_USER = "qbit" +POSTGRES_DB = "qbit" +COORDINATOR_IMAGE = "qbit-lab-prism-coordinator:local" +REAL_MINER_IMAGE = "qbit-lab-real-miner:local" +WALLET = "l6" +# The client timeout on each qbitd RPC. +RPC_TIMEOUT = 30.0 +# The bound on creating the lane's wallet, client timeout and wait included +# (#637). qbitd takes 24.5-30 s on the lane's runner, nearly all of it +# deriving the create-time P2MR keys, so a 30 s RPC races it; the lane waits +# for the wallet instead. Ten times the usual cost still leaves the 45-minute +# job most of what its cold build and the ten-minute scenario need. +WALLET_SECONDS = 300.0 +DIFF1_TARGET = 0xFFFF << 208 +# An Ed25519 private key in PKCS#8 DER is this prefix and the 32-byte seed. +ED25519_PKCS8_PREFIX = bytes.fromhex("302e020100300506032b657004220420") + +# cpuminer-opt prints its running totals after every share result, e.g. +# "12 Accepted 12 S0 R0 B3, 1.234 sec (5ms)" or "13 A13 S0 R0 BLOCK SOLVED 4, ...". +CPUMINER_RESULT = re.compile( + r"\b(?:A|Accepted )(\d+) (?:S|Stale )(\d+) (?:R|Rejected )(\d+) (?:B|BLOCK SOLVED )(\d+)," +) +CPUMINER_SUBMIT = re.compile(r"\b\d+ Submitted Diff ") +# Client logs kept in the artifact, and lines echoed to the job log, per client. +CLIENT_LOG_BYTES = 32 * 1024 * 1024 +CLIENT_ECHO_LINES = 400 + +# Addresses go into SQL literals, so hold them to the bech32 alphabet first. +ADDRESS = re.compile(r"^[a-z0-9]{8,120}$") + + +class LaneFailure(RuntimeError): + pass + + +class RpcTimeout(LaneFailure): + """A qbitd RPC that did not answer within its client timeout. + + Its outcome is unknown: the node carries on with the call after the + client gives up. + """ + + +def log(message: str) -> None: + print(f"{PREFIX}: {message}", flush=True) + + +# --------------------------------------------------------------------------- +# Pure helpers (unit-tested in tests/test_prism_shipped_image_lane.py). +# --------------------------------------------------------------------------- + + +def watched(paths) -> bool: + """Whether any changed path is one the lane watches.""" + return any( + fnmatch.fnmatchcase(path, pattern) + for path in (line.strip() for line in paths) if path + for pattern in WATCHED_PATHS + ) + + +def ed25519_public_key_hex(seed_hex: str) -> str: + """The Ed25519 public key for a 32-byte seed, as qbit-pool-builder prints it.""" + seed = bytes.fromhex(seed_hex) + if len(seed) != 32: + raise ValueError("an Ed25519 seed is 32 bytes") + der = subprocess.run( + ["openssl", "pkey", "-inform", "DER", "-pubout", "-outform", "DER"], + input=ED25519_PKCS8_PREFIX + seed, + capture_output=True, + check=True, + ).stdout + return der[-32:].hex() + + +def lane_env(qbit_src: Path, manifest_seed: str, attestation_seed: str, writer_key: str) -> str: + lines = [ + "# Written by scripts/prism_shipped_image_lane.py prepare; fresh seeds per run.", + f"QBIT_SRC_DIR={qbit_src}", + f"PRISM_MANIFEST_SIGNING_SEED_HEX={manifest_seed}", + f"PRISM_LEDGER_ATTESTATION_SIGNING_SEED_HEX={attestation_seed}", + f"PRISM_LEDGER_WRITER_PUBLIC_KEY_HEX={writer_key}", + f"PRISM_PUBLIC_STRATUM_URL=stratum+tcp://127.0.0.1:{STRATUM_PORTS[0]}", + f"PRISM_HA_INSTANCE_ID_1={INSTANCE_IDS[0]}", + f"PRISM_HA_INSTANCE_ID_2={INSTANCE_IDS[1]}", + ] + return "\n".join(lines) + "\n" + + +def read_env_file(path: Path) -> dict[str, str]: + values = {} + for line in path.read_text(encoding="utf-8").splitlines(): + line = line.strip() + if line and not line.startswith("#") and "=" in line: + key, value = line.split("=", 1) + values[key] = value + return values + + +def compose_command(project: str, env_file: Path, *args: str, profiles: tuple[str, ...] = ("prism",)) -> list[str]: + command = ["docker", "compose", "--project-name", project] + for name in (*ENV_FILES, str(env_file)): + command += ["--env-file", name] + for name in COMPOSE_FILES: + command += ["-f", name] + for profile in profiles: + command += ["--profile", profile] + return command + list(args) + + +@dataclass +class ClientTally: + """What one client saw: its submissions and the answers it got.""" + + submitted: int = 0 + accepted: int = 0 + rejected: int = 0 + stale: int = 0 + blocks: int = 0 + + @property + def unanswered(self) -> int: + return max(0, self.submitted - self.accepted - self.rejected - self.stale) + + +class CpuminerOutput: + """cpuminer-opt's totals, read a line at a time (its output can be large).""" + + def __init__(self) -> None: + self.tally = ClientTally() + + def feed(self, line: str) -> None: + if CPUMINER_SUBMIT.search(line): + self.tally.submitted += 1 + return + result = CPUMINER_RESULT.search(line) + if result: + # Each result line carries the running totals. + (self.tally.accepted, self.tally.stale, self.tally.rejected, + self.tally.blocks) = (int(value) for value in result.groups()) + + def result(self) -> ClientTally: + return self.tally + + +class PrismMinerOutput: + """qbit-prism-miner's JSON events: its summary has the totals.""" + + def __init__(self) -> None: + self.summary: dict | None = None + self.blocks = 0 + + def feed(self, line: str) -> None: + line = line.strip() + if not line.startswith("{"): + return + try: + event = json.loads(line) + except json.JSONDecodeError: + return + if event.get("event") == "summary": + self.summary = event + elif event.get("event") == "submit" and event.get("block_target_met") is True: + self.blocks += 1 + + def result(self) -> ClientTally: + if self.summary is None: + raise LaneFailure("qbit-prism-miner printed no summary") + return ClientTally( + submitted=int(self.summary["submitted"]), + accepted=int(self.summary["accepted"]), + rejected=int(self.summary["rejected"]), + blocks=self.blocks, + ) + + +def parse_cpuminer(output: str) -> ClientTally: + parser = CpuminerOutput() + for line in output.splitlines(): + parser.feed(line) + return parser.result() + + +def parse_prism_miner(output: str) -> ClientTally: + parser = PrismMinerOutput() + for line in output.splitlines(): + parser.feed(line) + return parser.result() + + +def reconcile_client(name: str, tally: ClientTally, ledger_rows: int) -> list[str]: + """Why a client's acknowledged shares and the ledger disagree; empty when they agree. + + Neither shipped client logs a share's identity, so this compares counts, + which is exact only when every submission was answered: each ledger row + is a submission, an answered one is acknowledged or refused, and a + refused share has no accepted row, so rows == acknowledged means every + acknowledged share is there. With a submission left unanswered, a lost + acknowledged share and a committed unanswered one would cancel out, so + that is a failure too, not a pass on bounds. (The load harness, L2, + reconciles share identities.) + """ + problems = [] + if tally.accepted == 0: + problems.append(f"{name}: no share was accepted") + if tally.unanswered: + problems.append( + f"{name}: {tally.unanswered} submissions were unanswered when it stopped, so counts cannot " + "prove every acknowledged share is in the ledger" + ) + if ledger_rows < tally.accepted: + problems.append( + f"{name}: {tally.accepted} shares acknowledged but only {ledger_rows} in the ledger" + ) + if ledger_rows > tally.accepted: + problems.append( + f"{name}: {ledger_rows} ledger rows but only {tally.accepted} shares acknowledged" + ) + return problems + + +def double_sha256(data: bytes) -> bytes: + return hashlib.sha256(hashlib.sha256(data).digest()).digest() + + +def difficulty_target(difficulty: float) -> int: + target = Fraction(DIFF1_TARGET) / Fraction(difficulty) + return min(int(target), (1 << 256) - 1) + + +def block_header(job: dict, extranonce1: bytes, extranonce2: bytes, nonce: int) -> bytes: + """The 80-byte header a notify describes, built as qbit-prism-miner builds it.""" + coinbase = bytes.fromhex(job["coinb1"]) + extranonce1 + extranonce2 + bytes.fromhex(job["coinb2"]) + merkle = double_sha256(coinbase) + for sibling in job["branch"]: + merkle = double_sha256(merkle + bytes.fromhex(sibling)) + previous = bytes.fromhex(job["prevhash"]) + previous = b"".join(previous[i : i + 4][::-1] for i in range(0, 32, 4)) + return ( + int(job["version"], 16).to_bytes(4, "little") + + previous + + merkle + + int(job["ntime"], 16).to_bytes(4, "little") + + int(job["nbits"], 16).to_bytes(4, "little") + + nonce.to_bytes(4, "little") + ) + + +def solve_share(job: dict, extranonce1: bytes, extranonce2_size: int, difficulty: float) -> tuple[str, str]: + """An (extranonce2, nonce) pair whose header meets half the share target. + + Half keeps the proof clear of any rounding at the boundary; on the lab's + share difficulty that takes a handful of hashes. + """ + target = difficulty_target(difficulty) // 2 + extranonce2 = (1).to_bytes(extranonce2_size, "little") + for nonce in range(1 << 32): + header = block_header(job, extranonce1, extranonce2, nonce) + if int.from_bytes(double_sha256(header), "little") <= target: + return extranonce2.hex(), f"{nonce:08x}" + raise LaneFailure("no nonce meets the share target") + + +def refusal_reason(answer: dict) -> str | None: + """The server's reason for refusing a submit, or None if it did not refuse.""" + error = answer.get("error") + if answer.get("result") is True or not error: + return None + if isinstance(error, list) and len(error) > 2 and isinstance(error[2], dict): + return error[2].get("reason_id") or str(error[1]) + if isinstance(error, list) and len(error) > 1: + return str(error[1]) + return str(error) + + +def notify_job(message: dict) -> dict: + params = message["params"] + keys = ("job_id", "prevhash", "coinb1", "coinb2", "branch", "version", "nbits", "ntime", "clean") + return dict(zip(keys, params)) + + +# --------------------------------------------------------------------------- +# Talking to the stack. +# --------------------------------------------------------------------------- + + +def run(command: list[str], *, check: bool = True, capture: bool = True, timeout: float | None = None, + input_text: str | None = None) -> subprocess.CompletedProcess: + process = subprocess.run( + command, cwd=ROOT, text=True, capture_output=capture, timeout=timeout, input=input_text + ) + if check and process.returncode != 0: + detail = (process.stderr or process.stdout or "").strip()[-4000:] if capture else "" + raise LaneFailure(f"{' '.join(command[:6])} ... exited {process.returncode}: {detail}") + return process + + +def http_json(url: str, *, timeout: float = 5.0) -> dict: + with urllib.request.urlopen(url, timeout=timeout) as response: + return json.loads(response.read()) + + +def until(what: str, seconds: float, probe, interval: float = 1.0): + deadline = time.monotonic() + seconds + last_error = None + while time.monotonic() < deadline: + try: + value = probe() + if value: + return value + except Exception as error: # noqa: BLE001 - reported when the wait expires + last_error = error + time.sleep(interval) + raise LaneFailure(f"timed out after {seconds:.0f}s waiting for {what}; last error: {last_error}") + + +class Rpc: + def __init__(self) -> None: + credentials = base64.b64encode(f"{QBIT_RPC_USER}:{QBIT_RPC_PASSWORD}".encode()).decode() + self.headers = {"Authorization": f"Basic {credentials}", "Content-Type": "application/json"} + + def call(self, method: str, params: list | None = None, wallet: str | None = None, + timeout: float | None = None): + timeout = RPC_TIMEOUT if timeout is None else timeout + url = f"http://127.0.0.1:{QBIT_RPC_PORT}/" + (f"wallet/{wallet}" if wallet else "") + body = json.dumps({"jsonrpc": "1.0", "id": method, "method": method, "params": params or []}) + request = urllib.request.Request(url, data=body.encode(), headers=self.headers) + try: + with urllib.request.urlopen(request, timeout=timeout) as response: + payload = json.loads(response.read()) + except urllib.error.HTTPError as error: + payload = json.loads(error.read() or b"{}") + except urllib.error.URLError as error: + # A connect timeout arrives wrapped; a read timeout (below) bare. + if isinstance(error.reason, TimeoutError): + raise RpcTimeout(f"qbit RPC {method} timed out after {timeout:g}s") from error + raise + except TimeoutError as error: + raise RpcTimeout(f"qbit RPC {method} timed out after {timeout:g}s") from error + if payload.get("error"): + raise LaneFailure(f"qbit RPC {method} failed: {payload['error']}") + return payload["result"] + + +def create_wallet(rpc: Rpc, name: str, seconds: float = WALLET_SECONDS) -> dict: + """Create wallet `name` on the node and return once the node has it loaded (#637). + + createwallet can outlast its client timeout, and the node finishes the + wallet after the client gives up, so a timeout is an unknown outcome: + the lane then waits for `listwallets` to show the wallet, which qbitd + does only once the wallet is created and loaded. It never issues + createwallet again, as the first one may already have made the wallet. + One deadline covers the call and the wait. Any other failure of the call + fails the lane. Returns how the wallet arrived, for the report. + """ + started = time.monotonic() + deadline = started + seconds + try: + rpc.call("createwallet", [name], timeout=min(RPC_TIMEOUT, seconds)) + return {"outcome": "created", "seconds": round(time.monotonic() - started, 1)} + except RpcTimeout as error: + log(f"{error}; the node finishes the wallet regardless, so waiting for listwallets to show {name!r}") + + def loaded(): + remaining = deadline - time.monotonic() + wallets = rpc.call("listwallets", timeout=max(0.1, min(RPC_TIMEOUT, remaining))) + if name not in wallets: + raise LaneFailure(f"listwallets shows {wallets}") + return True + + until(f"the node to load wallet {name!r} after createwallet timed out", deadline - time.monotonic(), loaded) + return {"outcome": "loaded after createwallet timed out", "seconds": round(time.monotonic() - started, 1)} + + +class Stratum: + """A line-delimited Stratum V1 client, just enough for the resume check.""" + + def __init__(self, port: int, timeout: float = 20.0) -> None: + self.sock = socket.create_connection(("127.0.0.1", port), timeout=timeout) + self.timeout = timeout + self.buffer = b"" + self.next_id = 1 + self.job: dict | None = None + self.difficulty: float | None = None + self.extranonce1 = b"" + self.extranonce2_size = 0 + + def close(self) -> None: + self.sock.close() + + def _message(self, deadline: float) -> dict: + while b"\n" not in self.buffer: + remaining = deadline - time.monotonic() + if remaining <= 0: + raise LaneFailure("Stratum read timed out") + self.sock.settimeout(remaining) + chunk = self.sock.recv(65536) + if not chunk: + raise LaneFailure("Stratum server closed the connection") + self.buffer += chunk + line, self.buffer = self.buffer.split(b"\n", 1) + message = json.loads(line) + if message.get("method") == "mining.notify": + self.job = notify_job(message) + elif message.get("method") == "mining.set_difficulty": + self.difficulty = float(message["params"][0]) + return message + + def request(self, method: str, params: list) -> dict: + request_id = self.next_id + self.next_id += 1 + self.sock.sendall((json.dumps({"id": request_id, "method": method, "params": params}) + "\n").encode()) + deadline = time.monotonic() + self.timeout + while True: + message = self._message(deadline) + if message.get("id") == request_id and "method" not in message: + return message + + def login(self, username: str) -> None: + subscribed = self.request("mining.subscribe", ["prism-shipped-images/1"]) + if subscribed.get("error"): + raise LaneFailure(f"subscribe rejected: {subscribed['error']}") + self.extranonce1 = bytes.fromhex(subscribed["result"][1]) + self.extranonce2_size = int(subscribed["result"][2]) + authorized = self.request("mining.authorize", [username, "x"]) + if authorized.get("result") is not True: + raise LaneFailure(f"authorize {username} rejected: {authorized.get('error')}") + deadline = time.monotonic() + self.timeout + while self.job is None or self.difficulty is None: + self._message(deadline) + + +# --------------------------------------------------------------------------- +# The lane. +# --------------------------------------------------------------------------- + + +@dataclass +class Lane: + work: Path + out: Path + project: str + load_seconds: int + cpuminer_threads: int + cpuminer_diff_multiplier: float + prism_miner_hashes_per_second: int + env_file: Path = field(init=False) + env: dict[str, str] = field(init=False) + rpc: Rpc = field(default_factory=Rpc) + report: dict = field(default_factory=dict) + checks: dict[str, dict] = field(default_factory=dict) + # The phase under way, and when it began, so a failure can name it. + current_phase: str | None = field(default=None, init=False) + phase_started: float = field(default=0.0, init=False) + + def __post_init__(self) -> None: + self.env_file = self.work / "lane.env" + self.env = read_env_file(self.env_file) + self.report = {"schema": "qbit.prism.shipped-image-lane.v1", "project": self.project, + "load_seconds": self.load_seconds, "phases": {}, "checks": self.checks} + + # -- helpers -------------------------------------------------------- + + def compose(self, *args: str, check: bool = True, timeout: float | None = 600, + profiles: tuple[str, ...] = ("prism",)) -> subprocess.CompletedProcess: + return run(compose_command(self.project, self.env_file, *args, profiles=profiles), + check=check, timeout=timeout) + + def psql(self, sql: str) -> str: + return self.compose("exec", "-T", "prism-postgres", "psql", "-XqAt", "-v", "ON_ERROR_STOP=1", + "-U", POSTGRES_USER, "-d", POSTGRES_DB, "-c", sql, timeout=60).stdout.strip() + + def scalar(self, sql: str) -> int: + return int(self.psql(sql) or 0) + + def passed(self, name: str, **evidence) -> None: + self.checks[name] = {"passed": True, **evidence} + log(f"PASS {name}: {json.dumps(evidence, sort_keys=True)}") + + def failed(self, name: str, problems: list[str], **evidence) -> None: + self.checks[name] = {"passed": False, "problems": problems, **evidence} + for problem in problems: + log(f"FAIL {name}: {problem}") + + def begin(self, name: str) -> float: + self.current_phase, self.phase_started = name, time.monotonic() + return self.phase_started + + def phase(self, name: str, started: float) -> None: + self.report["phases"][name] = round(time.monotonic() - started, 1) + + # -- phases --------------------------------------------------------- + + def start(self) -> dict[str, str]: + started = self.begin("node") + self.compose("up", "-d", "--no-build", "--wait", "--wait-timeout", "180", "qbitd") + until("the qbitd RPC", 60, lambda: self.rpc.call("getblockchaininfo")) + self.report["wallet"] = create_wallet(self.rpc, WALLET) + addresses = { + role: self.rpc.call("getnewaddress", ["", "p2mr"], wallet=WALLET) + for role in ("bootstrap", "cpuminer", "prism-miner", "resume", "thief") + } + # A fresh regtest chain is in initial block download until a block + # with a current timestamp arrives; the frontends refuse to serve + # work before that. + for role, address in addresses.items(): + if not ADDRESS.match(address): + raise LaneFailure(f"the node's {role} address {address!r} is not a bech32 string") + self.rpc.call("generatetoaddress", [1, addresses["bootstrap"]]) + self.phase("node", started) + started = self.begin("stack") + self.compose("up", "-d", "--no-build", "--wait", "--wait-timeout", "420") + self.phase("stack", started) + self.report["addresses"] = addresses + return addresses + + def self_check(self, service: str) -> dict: + process = self.compose("exec", "-T", service, "qbit-prism-server", "self-check", + check=False, timeout=120) + try: + report = json.loads(process.stdout) + except json.JSONDecodeError as error: + raise LaneFailure(f"{service} self-check printed no JSON report (exit {process.returncode}): " + f"{process.stderr.strip()[-2000:]}") from error + report["_exit"] = process.returncode + report["_stderr_tail"] = process.stderr.strip()[-2000:] + return report + + def check_frontends(self, self_checks: dict[str, dict]) -> None: + problems = [] + health = {} + for port in HEALTH_PORTS: + body = until(f"frontend /healthz on {port}", 180, + lambda port=port: (lambda b: b if b.get("ok") is True else None)( + http_json(f"http://127.0.0.1:{port}/healthz"))) + health[port] = body.get("instance_id") + if sorted(health.values()) != sorted(INSTANCE_IDS): + problems.append(f"healthz instance ids {health} are not {INSTANCE_IDS}") + observed = {} + for service, report in self_checks.items(): + live = report.get("live_instances") or {} + observed[service] = {"status": live.get("status"), "count": live.get("count"), + "instance_ids": live.get("instance_ids"), "ok": report.get("ok"), + "exit": report.get("_exit"), "error": report.get("_error"), + "stderr_tail": report.get("_stderr_tail")} + if live.get("status") != "observed" or live.get("count") != 2 or \ + sorted(live.get("instance_ids") or []) != sorted(INSTANCE_IDS): + problems.append(f"{service} self-check observes {observed[service]}, not both frontends") + if problems: + self.failed("ha-frontends-ready", problems, healthz=health, self_check=observed) + else: + self.passed("ha-frontends-ready", healthz=health, self_check=observed) + + def check_resume(self, addresses: dict[str, str]) -> None: + """#281 criterion 1: work issued on frontend 1, resumed on frontend 2.""" + self.begin("resume") + try: + self._check_resume(addresses) + except (LaneFailure, OSError, KeyError, ValueError) as error: + self.failed("cross-frontend-resume", [f"{type(error).__name__}: {error}"]) + + def _check_resume(self, addresses: dict[str, str]) -> None: + username = f"{addresses['resume']}.resume" + thief = f"{addresses['thief']}.thief" + answers = {} + issuer = Stratum(STRATUM_PORTS[0]) + try: + issuer.login(username) + job = issuer.job + extranonce2, nonce = solve_share(job, issuer.extranonce1, issuer.extranonce2_size, + issuer.difficulty) + issued_extranonce1 = issuer.extranonce1 + finally: + issuer.close() + submit = [username, job["job_id"], extranonce2, job["ntime"], nonce] + clients = [] + try: + # The other worker goes first, while the proof is still unspent: + # after the legitimate submit a refusal could be a mere duplicate + # and would not show that another worker cannot resume the job. + stranger = Stratum(STRATUM_PORTS[1]) + clients.append(stranger) + stranger.login(thief) + answers["other_worker_on_frontend_2"] = stranger.request("mining.submit", [thief, *submit[1:]]) + resumed = Stratum(STRATUM_PORTS[1]) + clients.append(resumed) + resumed.login(username) + answers["resumed_on_frontend_2"] = resumed.request("mining.submit", submit) + answers["replay_on_frontend_2"] = resumed.request("mining.submit", submit) + back = Stratum(STRATUM_PORTS[0]) + clients.append(back) + back.login(username) + answers["replay_on_frontend_1"] = back.request("mining.submit", submit) + finally: + for client in clients: + client.close() + problems = [] + if resumed.extranonce1 == issued_extranonce1: + problems.append("frontend 2 reused frontend 1's session extranonce") + if answers["resumed_on_frontend_2"].get("result") is not True: + problems.append(f"frontend 2 refused frontend 1's work: {answers['resumed_on_frontend_2']}") + for name in ("other_worker_on_frontend_2", "replay_on_frontend_2", "replay_on_frontend_1"): + if answers[name].get("result") is True: + problems.append(f"{name} was accepted: {answers[name]}") + if refusal_reason(answers["other_worker_on_frontend_2"]) in (None, "duplicate-share"): + problems.append("the other worker's submission was not refused as someone else's work: " + f"{answers['other_worker_on_frontend_2']}") + self.report["resume_answers"] = answers + self.report["resume_job"] = job["job_id"] + # The ledger half (one credited row, none for the other worker) is + # checked with the rest of the ledger after the load phase. + self.checks["cross-frontend-resume"] = {"passed": not problems, "problems": problems, + "answers": {k: v.get("result") for k, v in answers.items()}} + for problem in problems: + log(f"FAIL cross-frontend-resume: {problem}") + + def load(self, addresses: dict[str, str]) -> dict[str, ClientTally]: + started = self.begin("load") + network = f"{self.project}_default" + seconds = str(self.load_seconds) + clients = { + "cpuminer": [ + "docker", "run", "--rm", "--name", f"{self.project}-cpuminer", "--network", network, + REAL_MINER_IMAGE, "timeout", "-s", "INT", seconds, "cpuminer", "-a", "sha256d", + "-o", "stratum+tcp://prism-coordinator:3340", "-u", f"{addresses['cpuminer']}.cpuminer", + "-p", "x", "-t", str(self.cpuminer_threads), "--no-color", + "--diff-multiplier", f"{self.cpuminer_diff_multiplier:g}", + ], + "prism-miner": [ + "docker", "run", "--rm", "--name", f"{self.project}-prism-miner", "--network", network, + "--no-healthcheck", COORDINATOR_IMAGE, "qbit-prism-miner", + "--address", "prism-coordinator-2:3340", "--username", f"{addresses['prism-miner']}.prism-miner", + "--threads", "1", "--hashes-per-second", str(self.prism_miner_hashes_per_second), + "--duration-seconds", seconds, + ], + } + processes = {} + parsers = {"cpuminer": CpuminerOutput(), "prism-miner": PrismMinerOutput()} + readers = [] + for name, command in clients.items(): + log(f"starting {name} for {seconds}s") + process = subprocess.Popen(command, cwd=ROOT, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, + text=True, bufsize=1) + processes[name] = process + + def pump(process=process, parser=parsers[name], name=name): + # Parse every line, but keep the job log and the artifact + # bounded: a client that floods its pool must not fill either. + logged = echoed = 0 + with (self.out / f"{name}.log").open("w", encoding="utf-8") as sink: + for line in process.stdout: + parser.feed(line) + if logged < CLIENT_LOG_BYTES: + sink.write(line) + logged += len(line) + if logged >= CLIENT_LOG_BYTES: + sink.write(f"[{PREFIX}: log truncated at {CLIENT_LOG_BYTES} bytes]\n") + if echoed < CLIENT_ECHO_LINES: + print(f"[{name}] {line.rstrip()}", flush=True) + echoed += 1 + + reader = threading.Thread(target=pump, daemon=True) + reader.start() + readers.append(reader) + # Both frontends are serving a miner now; ask each what it sees. + time.sleep(min(60, self.load_seconds / 2)) + self_checks = {} + for service in ("prism-coordinator", "prism-coordinator-2"): + try: + self_checks[service] = self.self_check(service) + except (LaneFailure, subprocess.TimeoutExpired) as error: + self_checks[service] = {"_error": str(error)} + self.report["self_check"] = self_checks + codes = {} + for name, process in processes.items(): + try: + codes[name] = process.wait(timeout=self.load_seconds + 120) + except subprocess.TimeoutExpired: + process.kill() + codes[name] = process.wait() + for reader in readers: + reader.join(timeout=30) + self.report["client_exit"] = codes + self.phase("load", started) + # timeout(1) exits 124 when it stopped cpuminer at the deadline; any + # other status means cpuminer stopped on its own before then. + if codes["cpuminer"] != 124: + raise LaneFailure(f"cpuminer exited {codes['cpuminer']} before its {seconds}s run ended") + if codes["prism-miner"] != 0: + raise LaneFailure(f"qbit-prism-miner exited {codes['prism-miner']}") + self.check_frontends(self_checks) + return {name: parser.result() for name, parser in parsers.items()} + + def quiesce(self) -> int: + """The ledger's accepted count once it holds still and both frontends report it.""" + started = self.begin("quiesce") + + def settled(): + before = self.scalar("SELECT count(*) FROM qbit_share_ledger WHERE accepted") + latest = [http_json(f"http://127.0.0.1:{port}/audit/latest") for port in HEALTH_PORTS] + after = self.scalar("SELECT count(*) FROM qbit_share_ledger WHERE accepted") + counts = [body.get("accepted_share_count") for body in latest] + self.report["audit_latest_counts"] = counts + return after if before == after and all(count == after for count in counts) else None + + count = until("both frontends to report the ledger's accepted-share count", 90, settled, interval=2) + self.phase("quiesce", started) + return count + + def check_ledger(self, addresses: dict[str, str], tallies: dict[str, ClientTally], accepted: int) -> None: + self.begin("ledger") + rows = {role: self.scalar(f"SELECT count(*) FROM qbit_share_ledger WHERE accepted AND miner_id='{address}'") + for role, address in addresses.items()} + credited = self.scalar("SELECT count(*) FROM qbit_prism_share_hashes") + writers = { + writer: int(count) for writer, count in (line.split("|", 1) for line in self.psql( + "SELECT writer_id, count(*) FROM qbit_share_ledger WHERE accepted GROUP BY writer_id ORDER BY 1" + ).splitlines() if line) + } + problems = [] + for role, tally in tallies.items(): + problems += reconcile_client(role, tally, rows[role]) + if accepted != credited: + problems.append(f"{accepted} accepted shares but {credited} credited headers") + if not set(INSTANCE_IDS) <= set(writers): + problems.append(f"shares were written by {sorted(writers)}, not both frontends") + if rows["thief"] != 0: + problems.append(f"the other worker has {rows['thief']} credited shares") + if rows["bootstrap"] != 0: + problems.append(f"the bootstrap address has {rows['bootstrap']} credited shares") + resume = self.checks.get("cross-frontend-resume") + if resume is not None: + resume_problems = [] + if rows["resume"] != 1: + resume_problems.append(f"the resumed work is credited {rows['resume']} times, not once") + resume["ledger_rows"] = rows["resume"] + resume["problems"] += resume_problems + resume["passed"] = not resume["problems"] + for problem in resume_problems: + log(f"FAIL cross-frontend-resume: {problem}") + if resume["passed"]: + log(f"PASS cross-frontend-resume: {json.dumps(resume, sort_keys=True)}") + evidence = {"accepted": accepted, "credited_headers": credited, + "rows": rows, "writers": writers, + "clients": {role: tally.__dict__ | {"unanswered": tally.unanswered} + for role, tally in tallies.items()}} + if problems: + self.failed("ledger-reconciles", problems, **evidence) + else: + self.passed("ledger-reconciles", **evidence) + self.report["tallies"] = evidence["clients"] + + def check_blocks(self, addresses: dict[str, str]) -> None: + self.begin("blocks") + rows = [line.split("|") for line in self.psql( + "SELECT block_hash, block_height, solver_miner_id FROM qbit_pool_blocks " + "WHERE chain_state='confirmed' ORDER BY block_height" + ).splitlines() if line] + self.report["confirmed_blocks"] = len(rows) + latest = {} + for block_hash, height, solver in rows: + for role in ("cpuminer", "prism-miner"): + if solver == addresses[role]: + latest[role] = (block_hash, int(height)) + verified = {} + audit_problems = [] + for role, check in (("cpuminer", "cpuminer-found-block"), ("prism-miner", "prism-miner-found-block")): + count = sum(1 for _, _, solver in rows if solver == addresses[role]) + if role not in latest: + self.failed(check, [f"no confirmed block solved by {role}"], blocks=count) + continue + block_hash, height = latest[role] + active = self.rpc.call("getblockhash", [height]) + if active != block_hash: + self.failed(check, [f"{block_hash} at {height} is not on the node's active chain ({active})"], + blocks=count) + continue + self.passed(check, blocks=count, latest=block_hash, height=height) + try: + self.verify_audit(block_hash, height) + verified[role] = block_hash + except LaneFailure as error: + audit_problems.append(f"{role} block {block_hash}: {error}") + if audit_problems or len(verified) < 2: + if len(verified) < 2 and not audit_problems: + audit_problems.append("not every client has a found block to verify") + self.failed("found-block-audit-verifies", audit_problems, verified=verified) + else: + self.passed("found-block-audit-verifies", verified=verified) + self.check_public_blocks(list(verified.values())) + + def verify_audit(self, block_hash: str, height: int) -> None: + bundles = [] + for port in HEALTH_PORTS: + body = http_json(f"http://127.0.0.1:{port}/audit/blocks/{block_hash}/bundle", timeout=30) + bundles.append(body.get("audit_bundle")) + if bundles[0] is None or bundles[0] != bundles[1]: + raise LaneFailure("the frontends do not serve one identical audit bundle") + block = self.rpc.call("getblock", [block_hash, 2]) + coinbase = block["tx"][0].get("hex") or self.rpc.call( + "getrawtransaction", [block["tx"][0]["txid"], False, block_hash]) + path = self.out / f"audit-bundle-{height}-{block_hash[:16]}.json" + path.write_text(json.dumps(bundles[0]), encoding="utf-8") + path.chmod(0o644) + process = run(["docker", "run", "--rm", "--network", "none", "--no-healthcheck", + "-v", f"{path}:/work/bundle.json:ro", COORDINATOR_IMAGE, + "qbit-prism-audit-verify", "/work/bundle.json", "--coinbase-tx-hex", coinbase, + "--ledger-writer-public-key-hex", self.env["PRISM_LEDGER_WRITER_PUBLIC_KEY_HEX"]], + check=False, timeout=120) + (self.out / f"audit-verify-{height}.txt").write_text(process.stdout + process.stderr, encoding="utf-8") + if process.returncode != 0: + raise LaneFailure(f"qbit-prism-audit-verify exited {process.returncode}: {process.stderr.strip()}") + + def check_public_blocks(self, hashes: list[str]) -> None: + if not hashes: + self.failed("public-api-lists-found-blocks", ["no verified block to look for"]) + return + + def listed(): + seen, page = set(), 1 + while True: + body = http_json(f"http://127.0.0.1:{PUBLIC_API_PORT}/public/v1/blocks?limit=100&page={page}") + text = json.dumps(body) + seen |= {h for h in hashes if h in text} + pages = (body.get("pagination") or {}).get("total_pages") or 0 + if seen == set(hashes) or page >= pages: + return seen == set(hashes) + page += 1 + + try: + until("the public API to list the verified blocks", 120, listed, interval=3) + except LaneFailure as error: + self.failed("public-api-lists-found-blocks", [str(error)], blocks=hashes) + return + self.passed("public-api-lists-found-blocks", blocks=hashes) + + def collect(self) -> None: + for args, name in ((("ps", "--all"), "compose-ps.txt"), + (("logs", "--no-color", "--timestamps"), "compose-logs.txt")): + process = self.compose(*args, check=False, timeout=180) + (self.out / name).write_text(process.stdout + process.stderr, encoding="utf-8") + + def down(self) -> None: + for name in ("cpuminer", "prism-miner"): + run(["docker", "rm", "-f", f"{self.project}-{name}"], check=False) + self.compose("down", "-v", "--remove-orphans", check=False, timeout=300, + profiles=("prism", "real-miner-smoke")) + + def execute(self) -> bool: + started = time.monotonic() + error = failed_phase = None + try: + addresses = self.start() + self.check_resume(addresses) + tallies = self.load(addresses) + accepted = self.quiesce() + self.check_ledger(addresses, tallies, accepted) + self.check_blocks(addresses) + except (LaneFailure, subprocess.TimeoutExpired, OSError, KeyError, ValueError) as caught: + error = f"{type(caught).__name__}: {caught}" + failed_phase = self.current_phase + if failed_phase is not None: + # How long the failed phase ran, kept apart from the timings + # of phases that finished. + self.report["failed_phase_seconds"] = round(time.monotonic() - self.phase_started, 1) + log(f"ERROR in the {failed_phase} phase: {error}" if failed_phase else f"ERROR {error}") + finally: + try: + self.collect() + finally: + self.down() + self.phase("total", started) + missing = [name for name in CHECKS if name not in self.checks] + passed = error is None and not missing and all(c["passed"] for c in self.checks.values()) + self.report.update(error=error, failed_phase=failed_phase, not_reached=missing, passed=passed) + (self.out / "l6-report.json").write_text(json.dumps(self.report, indent=2, sort_keys=True) + "\n", + encoding="utf-8") + write_summary(self.report) + return passed + + +def write_summary(report: dict) -> None: + lines = ["### L6 shipped images: Compose `prism` profile with the HA overlay", "", + "| Check | Result |", "| --- | --- |"] + for name in CHECKS: + check = report["checks"].get(name) + if check is None: + result = "not reached" + elif check["passed"]: + result = "pass" + else: + result = "**fail**: " + "; ".join(check.get("problems", [])) + lines.append(f"| `{name}` | {result} |") + lines += ["", f"Phases (s): `{json.dumps(report['phases'])}`"] + if report.get("tallies"): + lines.append(f"Clients: `{json.dumps(report['tallies'], sort_keys=True)}`") + if report.get("error"): + # Name the phase: the checks it never reached say nothing about why. + where = f" in the `{report['failed_phase']}` phase" if report.get("failed_phase") else "" + after = f" after {report['failed_phase_seconds']}s" if "failed_phase_seconds" in report else "" + lines.append(f"Error{where}{after}: `{report['error']}`") + text = "\n".join(lines) + "\n" + print(text) + summary = os.environ.get("GITHUB_STEP_SUMMARY") + if summary: + with open(summary, "a", encoding="utf-8") as handle: + handle.write(text) + + +def prepare(work: Path) -> None: + work.mkdir(parents=True, exist_ok=True) + qbit_src = run(["bash", "scripts/prepare-qbit-source.sh"], timeout=900).stdout.strip().splitlines()[-1] + manifest_seed, attestation_seed = secrets.token_hex(32), secrets.token_hex(32) + text = lane_env(Path(qbit_src), manifest_seed, attestation_seed, ed25519_public_key_hex(attestation_seed)) + path = work / "lane.env" + path.write_text(text, encoding="utf-8") + path.chmod(0o600) + log(f"staged qbit source at {qbit_src}; wrote {path}") + + +def build(work: Path, project: str) -> None: + env_file = work / "lane.env" + started = time.monotonic() + # The frontends and the public API share one image; build it once. + for services, profiles in ((("qbitd", "prism-coordinator"), ("prism",)), + (("real-miner",), ("prism", "real-miner-smoke"))): + run(compose_command(project, env_file, "build", *services, profiles=profiles), + capture=False, timeout=2400) + log(f"built the images in {time.monotonic() - started:.0f}s") + + +def main(argv: list[str] | None = None) -> int: + parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) + commands = parser.add_subparsers(dest="command", required=True) + commands.add_parser("changed", help="print whether stdin's changed paths run the lane") + for name in ("prepare", "build", "run"): + command = commands.add_parser(name) + command.add_argument("--work", type=Path, required=True) + command.add_argument("--project", default="qbit-l6") + if name == "run": + command.add_argument("--out", type=Path, required=True) + command.add_argument("--load-seconds", type=int, default=240) + command.add_argument("--cpuminer-threads", type=int, default=2) + # cpuminer submits every hash that meets the pool's difficulty + # without waiting for answers. At the lab's 1e-9 floor that is + # most hashes: its connection floods, it never reads the next + # job and it mines a stale one for the rest of the run. Hashing + # locally to a harder target keeps it to a few shares a second + # until vardiff takes over; every share still meets regtest's + # network target, so each accepted one is a block. + command.add_argument("--cpuminer-diff-multiplier", type=float, default=1e7) + command.add_argument("--prism-miner-hashes-per-second", type=int, default=20000) + args = parser.parse_args(argv) + if args.command == "changed": + print("true" if watched(sys.stdin) else "false") + return 0 + work = args.work.resolve() + if args.command == "prepare": + prepare(work) + return 0 + if args.command == "build": + build(work, args.project) + return 0 + out = args.out.resolve() + out.mkdir(parents=True, exist_ok=True) + out.chmod(0o755) + lane = Lane(work=work, out=out, project=args.project, load_seconds=args.load_seconds, + cpuminer_threads=args.cpuminer_threads, + cpuminer_diff_multiplier=args.cpuminer_diff_multiplier, + prism_miner_hashes_per_second=args.prism_miner_hashes_per_second) + return 0 if lane.execute() else 1 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/scripts/run_python_test_shard.py b/scripts/run_python_test_shard.py index 08b64246f..f23df959d 100644 --- a/scripts/run_python_test_shard.py +++ b/scripts/run_python_test_shard.py @@ -3,16 +3,25 @@ Discovery sorts module paths; distribute its top-level suites round-robin so module/class fixtures stay together and new test modules join CI automatically. +Python citations in the deleted-test map must have a successful execution in +their assigned shard; unittest's successful exit for skipped fixtures is not +parity evidence. """ from __future__ import annotations import argparse +import inspect import os from pathlib import Path import sys import unittest +if __package__: + from .check_deleted_test_map import MAP, references_in +else: + from check_deleted_test_map import MAP, references_in + ROOT = Path(__file__).resolve().parents[1] @@ -29,6 +38,41 @@ def select_shard( return selected +def cited_test_ids(suite: unittest.TestSuite, root: Path) -> dict[str, set[str]]: + """Capture cited cases before unittest runs fixtures and consumes the suite.""" + citations = {} + for line in (root / MAP).read_text(encoding="utf-8").splitlines(): + for path, extension, name in references_in(line)[0]: + if extension == "py": + citations[(root.joinpath(path).resolve(), name)] = f"{path}::{name}" + expected: dict[str, set[str]] = {} + pending = [suite] + while pending: + test = pending.pop() + if isinstance(test, unittest.TestSuite): + pending.extend(test) + continue + name = test._testMethodName + method = inspect.unwrap(getattr(test, name)) + if not inspect.isroutine(method): + continue + source = inspect.getsourcefile(method) + citation = citations.get((Path(source).resolve(), name)) if source else None + if citation is not None: + expected.setdefault(citation, set()).add(test.id()) + return expected + + +class ParityTestResult(unittest.TextTestResult): + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + self.successful_ids: set[str] = set() + + def addSuccess(self, test): + super().addSuccess(test) + self.successful_ids.add(test.id()) + + def main(argv: list[str] | None = None) -> int: parser = argparse.ArgumentParser(description=__doc__) parser.add_argument("--shard-index", type=int, required=True) @@ -42,15 +86,21 @@ def main(argv: list[str] | None = None) -> int: suite = unittest.TestLoader().discover("tests", pattern="test_*.py") try: selected = select_shard(suite, args.shard_index, args.shard_count) + expected = cited_test_ids(selected, ROOT) except ValueError as error: parser.error(str(error)) + except (OSError, UnicodeDecodeError) as error: + parser.error(f"cannot read {MAP}: {error}") print( f"Python test shard {args.shard_index}/{args.shard_count}: " f"{selected.countTestCases()} of {suite.countTestCases()} tests", flush=True, ) - result = unittest.TextTestRunner(verbosity=2).run(selected) - return 0 if result.wasSuccessful() else 1 + result = unittest.TextTestRunner(verbosity=2, resultclass=ParityTestResult).run(selected) + missing = sorted(citation for citation, ids in expected.items() if not ids & result.successful_ids) + for citation in missing: + print(f"Python parity citation `{citation}`: no successful execution in this shard", file=sys.stderr) + return 0 if result.wasSuccessful() and not missing else 1 if __name__ == "__main__": diff --git a/scripts/run_rust_test_shard.py b/scripts/run_rust_test_shard.py new file mode 100644 index 000000000..09ce78e8b --- /dev/null +++ b/scripts/run_rust_test_shard.py @@ -0,0 +1,128 @@ +#!/usr/bin/env python3 +"""Run a deterministic shard of workspace Rust targets in required integration CI. + +Keep each test binary intact so its fixtures and process-wide locks retain +Cargo's normal semantics. Discover targets from Cargo so new suites join CI. +Balance shards by each target's typical duration, longest first. +""" + +from __future__ import annotations + +import argparse +import json +import os +from pathlib import Path +import shlex +import subprocess +import time + + +ROOT = Path(__file__).resolve().parents[1] +COMPACT_SCALE = ("qbit-prism-server", "test", "compact_runtime_scale") +# These ignored database contracts must be selected explicitly. Run each only +# on the shard that owns its ordinary test target; test/prism-native-tests.sh +# runs the same selections locally. +IGNORED = { + ("qbit-prism-server", "test", "stratum_admission_postgres"): [ + "--exact", + "ten_thousand_unsubscribed_connections_do_not_advance_postgres_sequence", + ], + ("qbit-prism-server", "test", "observability_database"): [], + # Run the real scale cases serially under optimized runtime code; each + # owns a private durable primary for its insert-LSN measurements. + COMPACT_SCALE: ["--test-threads=1"], + ("qbit-prism-server", "test", "issued_job_dependency"): ["--test-threads=2"], +} +# Typical seconds per target, build included, from the PostgreSQL shard logs +# of 2026-10-01 (#647). Round-robin put the first three on one shard, which +# then ran into the job's cap. Every other target averages about 20 s. +# Re-measure from the job summary's per-command table when shards drift. +DURATION_SECONDS = { + ("qbit-prism-server", "test", "live_regtest"): 700, + ("qbit-prism-server", "lib", "qbit_prism_server"): 330, + ("qbit-prism-server", "test", "candidate_storm_restart"): 270, + COMPACT_SCALE: 240, + ("qbit-prism-server", "test", "ledger_postgres"): 210, +} +DEFAULT_DURATION_SECONDS = 20 + + +def typical_seconds(target: tuple[str, str, str]) -> int: + return DURATION_SECONDS.get(target, DEFAULT_DURATION_SECONDS) + + +def workspace_targets(metadata: dict) -> list[tuple[str, str, str]]: + targets = [] + for package in metadata["packages"]: + if package["id"] not in metadata["workspace_members"]: + continue + for target in package["targets"]: + kinds = target["kind"] + if kinds == ["custom-build"]: + continue + if len(kinds) != 1 or kinds[0] not in ("lib", "bin", "test", "bench", "example"): + raise ValueError(f"unsupported target kind: {kinds}") + targets.append((package["name"], kinds[0], target["name"])) + return sorted(targets) + + +def shard_commands(metadata: dict, index: int, count: int) -> list[list[str]]: + if count < 1 or not 0 <= index < count: + raise ValueError("require shard-count > 0 and 0 <= shard-index < shard-count") + loads = [0] * count + shards: list[list[tuple[str, str, str]]] = [[] for _ in range(count)] + for target in sorted(workspace_targets(metadata), key=lambda target: (-typical_seconds(target), target)): + lightest = min(range(count), key=lambda shard: (loads[shard], shard)) + loads[lightest] += typical_seconds(target) + shards[lightest].append(target) + targets = sorted(shards[index]) + if not targets: + raise ValueError("shard contains no targets; reduce shard-count") + commands = [] + for package, kind, name in targets: + command = ["cargo", "test", "--locked", "-p", package, f"--{kind}"] + if kind != "lib": + command.append(name) + commands.append(command + ["--", "--nocapture"]) + if (package, kind, name) in IGNORED: + profile = ["--release"] if (package, kind, name) == COMPACT_SCALE else [] + commands.append(command + profile + ["--", "--ignored", "--nocapture"] + IGNORED[package, kind, name]) + return commands + + +def main(argv: list[str] | None = None) -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--shard-index", type=int, required=True) + parser.add_argument("--shard-count", type=int, required=True) + parser.add_argument("--dry-run", action="store_true", help="print commands without running tests") + args = parser.parse_args(argv) + try: + metadata = json.loads(subprocess.check_output( + ["cargo", "metadata", "--locked", "--no-deps", "--format-version", "1"], + cwd=ROOT, text=True, + )) + commands = shard_commands(metadata, args.shard_index, args.shard_count) + summary = None if args.dry_run else os.environ.get("GITHUB_STEP_SUMMARY") + if summary: + with open(summary, "a", encoding="utf-8") as handle: + handle.write("| Command | Seconds |\n| --- | ---: |\n") + for command in commands: + print(shlex.join(command), flush=True) + if not args.dry_run: + started = time.monotonic() + try: + subprocess.run(command, cwd=ROOT, check=True) + finally: + if summary: + # Per-command durations are the data for the next re-balance. + with open(summary, "a", encoding="utf-8") as handle: + handle.write(f"| `{shlex.join(command)}` | {time.monotonic() - started:.0f} |\n") + except ValueError as error: + parser.error(str(error)) + except subprocess.CalledProcessError as error: + return error.returncode if error.returncode > 0 else 1 + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/rust_test_reachability.py b/scripts/rust_test_reachability.py new file mode 100644 index 000000000..40517781d --- /dev/null +++ b/scripts/rust_test_reachability.py @@ -0,0 +1,259 @@ +"""Conservative, offline reachability proof for deleted-test Rust citations. + +Read Cargo manifests and follow module declarations, without building or running +repository code. This is deliberately not a Rust compiler: macro inclusion, +cfg_attr, feature/platform/custom cfg predicates and custom harnesses cannot +prove a citation. Only unconditional code and predicates composed of test, +all(), any() and not() are accepted. CI invokes each target explicitly, so +Cargo's default target-selection flags (test/bench) do not exclude a target. +""" + +from __future__ import annotations + +from collections.abc import Callable, Iterator +from pathlib import Path +import re +import tomllib + + +VISIBILITY = r"(?:pub(?:\([^)]*\))?\s+)?" +MODULE = re.compile(VISIBILITY + r"mod\s+([A-Za-z_][A-Za-z0-9_]*)\s*$") +FUNCTION = re.compile(VISIBILITY + r"(?:async\s+)?fn\s+([A-Za-z_][A-Za-z0-9_]*)\s*[(<]") +ATTRIBUTE = re.compile(r"#\s*(!?)\s*\[") + + +def manifest(path: Path) -> dict: + with path.open("rb") as source: + return tomllib.load(source) + + +def cargo_test_targets(root: Path) -> Iterator[tuple[tuple[str, str, str], Path]]: + """Roots selected by the shard runner, limited to provable workspace members.""" + root = root.resolve() + workspace = manifest(root / "Cargo.toml") + settings = workspace.get("workspace", {}) + members = {root} if "package" in workspace else set() + for pattern in settings.get("members", []): + members.update(path.resolve() for path in root.glob(pattern) if path.is_dir()) + for pattern in settings.get("exclude", []): + members.difference_update(path.resolve() for path in root.glob(pattern)) + for directory in sorted(members): + if not directory.is_relative_to(root): + raise ValueError("Cargo workspace member leaves the repository") + data = manifest(directory / "Cargo.toml") + package = data["package"] + edition = package.get("edition", "2015") + if isinstance(edition, dict): + edition = settings.get("package", {}).get("edition", "2015") + auto = edition != "2015" or not any(kind in data for kind in ("lib", "bin", "test", "bench", "example")) + features = {"default"} if "default" in data.get("features", {}) else set() + pending = list(data.get("features", {}).get("default", [])) + while pending: + feature = pending.pop() + if feature not in features: + features.add(feature) + pending.extend(data.get("features", {}).get(feature, [])) + for kind, folder, flag in ( + ("lib", "src", "autolib"), ("bin", "src/bin", "autobins"), + ("test", "tests", "autotests"), ("bench", "benches", "autobenches"), + ("example", "examples", "autoexamples"), + ): + inferred: dict[str, Path] = {} + if kind == "lib": + inferred[package["name"].replace("-", "_")] = directory / "src/lib.rs" + else: + for path in sorted((directory / folder).glob("*.rs")): + inferred[path.stem] = path + for path in sorted((directory / folder).glob("*/main.rs")): + inferred[path.parent.name] = path + if kind == "bin" and (directory / "src/main.rs").is_file(): + inferred[package["name"]] = directory / "src/main.rs" + targets = {name: {"path": str(path)} for name, path in inferred.items()} if package.get(flag, auto) else {} + explicit = [data[kind]] if kind == "lib" and kind in data else data.get(kind, []) + if kind == "lib" and explicit: + targets = {} + for target in explicit: + name = target.get("name", package["name"].replace("-", "_")) + path = directory / target["path"] if "path" in target else inferred.get(name) + if kind == "lib" and path is None: + path = directory / "src/lib.rs" + # An explicit target can rename an inferred source file as + # well as override the settings of an inferred target name. + if path is not None: + targets = {key: value for key, value in targets.items() if Path(value["path"]).resolve() != path.resolve()} + targets[name] = {**target, "path": str(path) if path is not None else ""} + for name, target in targets.items(): + if target.get("harness", True) is not True or not set(target.get("required-features", [])).issubset(features): + continue + path = Path(target["path"]).resolve() + if path.is_relative_to(root) and path.is_file(): + yield (package["name"], kind, name), path + + +def cfg_enabled(expression: str) -> bool | None: + """Prove target-independent test predicates; unknown is never treated as false.""" + tokens = re.findall(r"[A-Za-z_][A-Za-z0-9_]*|[^\s]", expression) + position = 0 + + def parse() -> bool: + nonlocal position + if position >= len(tokens): + raise ValueError("missing cfg predicate") + name = tokens[position] + position += 1 + if name == "test": + return True + if name not in ("all", "any", "not") or position >= len(tokens) or tokens[position] != "(": + raise ValueError("unsupported cfg predicate") + position += 1 + values = [] + while position < len(tokens) and tokens[position] != ")": + values.append(parse()) + if position < len(tokens) and tokens[position] == ",": + position += 1 + elif position >= len(tokens) or tokens[position] != ")": + raise ValueError("malformed cfg predicate") + if position >= len(tokens): + raise ValueError("unclosed cfg predicate") + position += 1 + if name == "not": + if len(values) != 1: + raise ValueError("not requires one predicate") + return not values[0] + return all(values) if name == "all" else any(values) + + try: + value = parse() + return value if position == len(tokens) else None + except ValueError: + return None + + +def attributes_enabled(attributes: list[str]) -> bool: + for attribute in attributes: + name = re.match(r"\s*([A-Za-z_][A-Za-z0-9_:]*)", attribute) + if name is None: + return False + if name[1] == "cfg": + predicate = re.fullmatch(r"\s*cfg\s*\((.*)\)\s*", attribute, re.DOTALL) + if predicate is None or cfg_enabled(predicate[1]) is not True: + return False + elif name[1] not in ( + "path", "test", "tokio::test", "ignore", "should_panic", "allow", "warn", "deny", "forbid", + "expect", "doc", "deprecated", "rustfmt::skip", "no_implicit_prelude", + ): + # In particular, cfg_attr or an unknown procedural attribute can + # remove/rewrite the module or function, or change its source path. + return False + return True + + +class RustReachability: + def __init__(self, root: Path, mask_code: Callable[[str], str]) -> None: + self.root = root.resolve() + self.mask_code = mask_code + self.functions: dict[Path, dict[int, set[tuple[str, str, str, str]]]] = {} + self.visited: set[tuple[tuple[str, str, str], Path, Path, str]] = set() + for self.target, path in cargo_test_targets(self.root): + self.file(path, path.parent) + + def file(self, path: Path, module_dir: Path, prefix: str = "", ancestors: frozenset[Path] = frozenset()) -> None: + path = path.resolve() + key = (self.target, path, module_dir.resolve(), prefix) + if not path.is_relative_to(self.root) or key in self.visited or path in ancestors: + return + self.visited.add(key) + text = path.read_text(encoding="utf-8") + code = self.mask_code(text) + # Delimiter pairs let the walk skip function bodies, macro token trees, + # impls and other non-module scopes rather than trusting indentation. + pairs: dict[int, int] = {} + stack: list[int] = [] + for index, character in enumerate(code): + if character in "([{": + stack.append(index) + elif character in ")]}": + if not stack or {"(": ")", "[": "]", "{": "}"}[code[stack[-1]]] != character: + return + pairs[stack.pop()] = index + if stack: + return + self.scope(path, text, code, pairs, 0, len(code), module_dir, path.parent, prefix, ancestors | {path}) + + def scope( + self, path: Path, text: str, code: str, pairs: dict[int, int], start: int, end: int, + module_dir: Path, path_dir: Path, prefix: str, ancestors: frozenset[Path], + ) -> None: + cursor = start + attributes: list[str] = [] + while cursor < end: + if code[cursor].isspace() or code[cursor] == ";": + cursor += 1 + continue + attribute = ATTRIBUTE.match(code, cursor) + if attribute: + close = pairs[attribute.end() - 1] + value = code[attribute.end():close] + if attribute[1]: + if not attributes_enabled([value]): + return + else: + attributes.append(value) + cursor = close + 1 + continue + item_start = cursor + while cursor < end and code[cursor] not in "{;": + cursor = pairs[cursor] + 1 if code[cursor] in "([" else cursor + 1 + if cursor == end: + return + header = code[item_start:cursor].strip() + module = MODULE.fullmatch(header) + enabled = attributes_enabled(attributes) + if enabled and module: + name = module[1] + custom_path = None + for value in attributes: + matched = re.fullmatch(r"\s*path\s*=\s*(~+)\s*", value) + if matched: + # Recover just this literal from the original source. + begin = code.rfind(value, start, item_start) + matched.start(1) + try: + custom_path = tomllib.loads("path = " + text[begin:begin + len(matched[1])])["path"] + except tomllib.TOMLDecodeError: + enabled = False + elif re.match(r"\s*path\b", value): + enabled = False + if enabled and code[cursor] == "{": + child_dir = path_dir / custom_path if custom_path is not None else module_dir / name + self.scope(path, text, code, pairs, cursor + 1, pairs[cursor], child_dir, child_dir, prefix + name + "::", ancestors) + elif enabled: + candidates = ( + [path_dir / custom_path] if custom_path is not None + else [module_dir / f"{name}.rs", module_dir / name / "mod.rs"] + ) + candidates = [candidate for candidate in candidates if candidate.is_file()] + if len(candidates) == 1: + child = candidates[0] + child_dir = child.parent if custom_path is not None or child.name == "mod.rs" else child.with_suffix("") + self.file(child, child_dir, prefix + name + "::", ancestors) + elif enabled and FUNCTION.match(header): + body = cursor + 1 + while body < end: + if code[body].isspace(): + body += 1 + continue + inner = ATTRIBUTE.match(code, body) + if inner is None or not inner[1]: + break + close = pairs[inner.end() - 1] + enabled = enabled and attributes_enabled([code[inner.end():close]]) + body = close + 1 + if enabled: + line = code.count("\n", 0, item_start) + 1 + name = FUNCTION.match(header)[1] + self.functions.setdefault(path, {}).setdefault(line, set()).add((*self.target, prefix + name)) + attributes = [] + cursor = pairs[cursor] + 1 if code[cursor] == "{" else cursor + 1 + + def occurrences(self, path: Path) -> dict[int, set[tuple[str, str, str, str]]]: + return self.functions.get(path.resolve(), {}) diff --git a/scripts/test_prism_alert_deployment.py b/scripts/test_prism_alert_deployment.py new file mode 100644 index 000000000..4882ebe4b --- /dev/null +++ b/scripts/test_prism_alert_deployment.py @@ -0,0 +1,195 @@ +#!/usr/bin/env python3 +"""Render the review-only patch and check provisioning against its frozen source. + +Requires Jinja2 and PyYAML. Uses synthetic deployment variables and the unchanged +shared Grafana macro from --snapshot; does not contact Grafana or qbit-tools. +""" + +import argparse +import hashlib +import json +from pathlib import Path +import subprocess +import tempfile +import re + +import jinja2 +from jinja2.meta import find_undeclared_variables +import yaml + +from generate_prism_alerts import ROOT, deployment_template, load + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--snapshot", type=Path, required=True) + args = parser.parse_args() + manifest = load("tests/fixtures/prism-deployed-alerts.json") + relative = Path("ansible/roles/qbit_monitoring_stack/templates/grafana-alert-rules-prism.yml.j2") + original = (args.snapshot / relative.name).read_bytes() + assert hashlib.sha256(original).hexdigest() == manifest["sha256"] + generated = deployment_template(original.decode()) + with tempfile.TemporaryDirectory(prefix="prism-provisioning-") as directory: + target = Path(directory) / relative + target.parent.mkdir(parents=True) + target.write_bytes(original) + patch = str(ROOT / "docs/prism-alert-rules-qbit-tools.patch") + subprocess.run(["git", "apply", "--check", patch], cwd=directory, check=True) + subprocess.run(["git", "apply", patch], cwd=directory, check=True) + assert target.read_text() == generated, "patch differs from the specification" + + env = jinja2.Environment(loader=jinja2.FileSystemLoader(args.snapshot), + undefined=jinja2.StrictUndefined) + env.filters.update(bool=bool, quote=json.dumps, to_json=json.dumps) + needed = set() + for template in [original.decode(), generated, + (args.snapshot / "grafana-alert-rule-group.yml.j2").read_text()]: + needed |= find_undeclared_variables(env.parse(template)) + variables = {key: 1 for key in needed if key != "undef"} + gates = [key for key in variables if key.endswith(("_enabled", "_effective"))] + for key in variables: + if key in gates: + variables[key] = True + elif key.endswith("_window"): + variables[key] = "5m" + elif key.endswith("_for"): + variables[key] = "3m" + elif key.endswith("_ratio"): + variables[key] = 0.5 + elif "evaluation_interval" in key: + variables[key] = "1m" + variables.update(qbit_monitoring_stack_network="mainnet", + qbit_monitoring_stack_grafana_dashboard_folder="Qbit", + qbit_monitoring_stack_prism_alert_preview_publication_warning_seconds="1", + qbit_monitoring_stack_prism_alert_preview_publication_critical_seconds="5", + qbit_monitoring_stack_prism_alert_lease_monitor_wake_recovery_threshold=0.5, + qbit_monitoring_stack_alert_rule_group_name="qbit-prism-alerts") + external = {row["uid"] for row in manifest["alerts"] if row["external"]} + all_old = {row["uid"] for row in manifest["alerts"]} + all_new = external | {row["uid"] for path in ["docs/prism-native-alert-rules.json", + "docs/prism-postgres-alert-rules.json"] + for row in load(path)["rules"]} + native_uids = {row["uid"] for row in load("docs/prism-native-alert-rules.json")["rules"]} + + def rules(document): + rows = [rule for group in document.get("groups", []) or [] for rule in group["rules"]] + result = {rule["uid"]: rule for rule in rows} + assert len(rows) == len(result), "duplicate UIDs" + return result + + combinations = [{}, {key: False for key in gates}] + [{key: False} for key in gates] + duration = re.compile(r"^[0-9]+(?:ms|s|m|h|d|w|y)$") + for index, overrides in enumerate(combinations): + values = {**variables, **overrides} + old = yaml.safe_load(env.from_string(original.decode()).render(**values)) + new = yaml.safe_load(env.from_string(generated).render(**values)) + before, after = rules(old), rules(new) + for rule in after.values(): + assert duration.fullmatch(str(rule["for"])), (rule["uid"], rule["for"]) + assert "{{" not in str(rule["for"]) and '"' not in str(rule["for"]) + assert new["apiVersion"] == 1 + assert before.keys() & external == after.keys() & external, overrides + for uid in before.keys() & external: + assert before[uid] == after[uid], (uid, overrides) + deletions = [row["uid"] for row in new["deleteRules"]] + assert len(deletions) == len(set(deletions)) + assert all(row["orgId"] == 1 for row in new["deleteRules"]) + assert set(deletions) == (all_old - all_new) | (native_uids - after.keys()), overrides + assert not after.keys() & set(deletions) + if index == 0: + assert len(before) == 78 and len(after) == 79 + assert len(deletions) == 26 + assert after["qbit-prism-candidate-oldest-critical"]["labels"]["severity"] == "critical" + assert after["qbit-prism-candidate-oldest-critical"]["labels"].get("page") == "true" + assert after["qbit-prism-semantic-coverage-critical"]["labels"]["severity"] == "critical" + assert after["qbit-prism-semantic-coverage-critical"]["labels"].get("page") == "true" + # #493: every native paging rule dwells, stays quiet on no data or + # evaluation errors, and conjoins an availability gate on its own + # target. External (sidecar) paging rules keep their own producers. + paging = {uid: rule for uid, rule in after.items() + if uid in native_uids and rule["labels"].get("page") == "true"} + assert {"qbit-prism-candidate-oldest-critical", "qbit-prism-semantic-coverage-critical", + "qbit-prism-revision-work-pending-critical", + "qbit-prism-candidate-landing-failed-critical", + "qbit-prism-work-refresh-stalled-critical", + "qbit-prism-database-unavailable", + "qbit-prism-block-submission-held"} <= set(paging), sorted(paging) + for uid, rule in paging.items(): + assert rule["for"] != "0s", uid + assert rule["noDataState"] == "OK", uid + assert rule["execErrState"] == "OK", uid + expr = json.dumps(rule) + assert " and on(job, instance, network) " in expr, uid + fresh = "qbit_prism_metrics_snapshot_available{" in expr and "qbit_prism_metrics_snapshot_stale{" in expr + scraped = "up{" in expr and "== 1" in expr + assert fresh or scraped, uid + assert paging["qbit-prism-revision-work-pending-critical"]["for"] == "1m" + # #525: a refresh that keeps failing pages on this frontend's own gauge. + assert paging["qbit-prism-work-refresh-stalled-critical"]["for"] == "1m" + # #581: a database outage pages on its own rule, on the live + # collector gauge and a successful scrape, never on the snapshot. + database = json.dumps(paging["qbit-prism-database-unavailable"]) + assert paging["qbit-prism-database-unavailable"]["for"] == "2m" + assert "collector=\\\"database\\\"" in database + assert "qbit_prism_metrics_snapshot" not in database + # #666: a frontend held by PRISM_BLOCK_SUBMIT_ENABLED pages a minute + # after it starts, on a successful scrape within two minutes: the + # gauge is fixed at start, so neither a stale snapshot nor one + # failed scrape may hide it or restart its dwell. + held = json.dumps(paging["qbit-prism-block-submission-held"]) + assert paging["qbit-prism-block-submission-held"]["for"] == "1m" + assert "last_over_time(qbit_prism_block_submission_enabled{" in held and "!= bool 1" in held + assert "max_over_time(up{" in held + assert "qbit_prism_metrics_snapshot" not in held + # #493: the tracking-unknown and unlanded warnings dwell and never page. + for uid, dwell in [("qbit-prism-revision-work-unknown", "2m"), + ("qbit-prism-accepted-block-unlanded", "3m"), + ("qbit-prism-block-candidate-stuck", "3m"), + ("qbit-prism-candidate-landing-failed", "3m")]: + assert uid not in paging and after[uid]["labels"].get("page") is None, uid + assert after[uid]["labels"]["severity"] == "warning" and after[uid]["for"] == dwell, uid + assert after["qbit-prism-revision-work-unknown"]["noDataState"] == "Alerting" + assert after["qbit-prism-accepted-block-unlanded"]["noDataState"] == "OK" + assert after["qbit-prism-revision-work-pending"]["for"] == "0s" + # #529: a found block offered without a confirmed standby copy + # warns at once, never pages, and is quiet on no data. + standby = after["qbit-prism-block-offer-standby-unconfirmed"] + assert "qbit-prism-block-offer-standby-unconfirmed" not in paging + assert standby["labels"]["severity"] == "warning" and standby["labels"].get("page") is None + assert standby["for"] == "0s" and standby["noDataState"] == "OK" + assert 'outcome=~\\"absent|lagging|failed\\"' in json.dumps(standby), json.dumps(standby) + # #493: the candidate paging rule reads the unacknowledged age, not the all-unfinished age. + critical = json.dumps(after["qbit-prism-candidate-oldest-critical"]) + assert "qbit_prism_block_candidate_oldest_unacknowledged_seconds{" in critical + assert " or on(job, instance, network) qbit_prism_block_candidate_oldest_pending_seconds{" in critical, "no mixed-version fallback" + assert paging["qbit-prism-candidate-landing-failed-critical"]["for"] == "1m" + # #602: slow landing windows ticket and warn, and never page; the + # pool-wide resubmission reject ratio warns. + for uid, severity, dwell in [("qbit-prism-landing-ack-p99-slow", "ticket", "1m"), + ("qbit-prism-landing-ack-p99-high", "warning", "1m"), + ("qbit-prism-pool-resubmission-reject-ratio", "warning", "5m")]: + assert uid not in paging and after[uid]["labels"].get("page") is None, uid + assert after[uid]["labels"]["severity"] == severity and after[uid]["for"] == dwell, uid + assert after[uid]["noDataState"] == after[uid]["execErrState"] == "OK", uid + if index == 1: + assert len(new["groups"]) == 1 and len(after) == 2 + for rule in load("docs/prism-postgres-alert-rules.json")["rules"]: + rendered = after[rule["uid"]] + assert rendered["noDataState"] == rendered["execErrState"] == "Alerting" + assert rendered["for"] == "1m" + tuned = dict(variables, qbit_monitoring_stack_prism_alert_connected_clients_for="9m", + qbit_monitoring_stack_prism_alert_block_candidate_age_for="7m", + qbit_monitoring_stack_prism_alert_semantic_coverage_warning_for="11m") + rendered = yaml.safe_load(env.from_string(generated).render(**tuned)) + tuned_rules = rules(rendered) + assert tuned_rules["qbit-prism-connected-clients"]["for"] == "9m" + assert tuned_rules["qbit-prism-block-candidate-oldest"]["for"] == "7m" + assert tuned_rules["qbit-prism-semantic-work-coverage"]["for"] == "11m" + assert (args.snapshot / relative.name).read_bytes() == original + print(f"Patch applies cleanly; {len(combinations)} Jinja gate combinations passed; " + "78 original / 79 proposed rules; 34 external definitions preserved; " + "26 baseline deletions plus every native UID disabled by its gate") + + +if __name__ == "__main__": + main() diff --git a/scripts/test_prism_alert_promql.py b/scripts/test_prism_alert_promql.py new file mode 100644 index 000000000..3055e968d --- /dev/null +++ b/scripts/test_prism_alert_promql.py @@ -0,0 +1,108 @@ +#!/usr/bin/env python3 +"""Check the authored PromQL and unknown/stale/availability contracts with promtool.""" + +import argparse +import json +from pathlib import Path +import re +import shutil +import subprocess +import tempfile + +from prism_pool_wait_scenarios import ( + NEGATIVE_CONTROL_SCENARIOS, TITLE as POOL_WAIT_TITLE, + old_pool_wait_expression, pool_wait_tests, +) + +ROOT = Path(__file__).resolve().parents[1] + + +def load(path): + return json.loads((ROOT / path).read_text()) + + +def check_pool_wait_negative_control(promtool, path, alert, expression, tests): + """The original guards must fail these otherwise passing positive cases.""" + old_alert = dict(alert, expr=f"({old_pool_wait_expression(expression)}) > 0") + negative_tests = [{key: value for key, value in test.items() if key != "promql_expr_test"} + for test in tests if test["name"] in NEGATIVE_CONTROL_SCENARIOS] + assert len(negative_tests) == len(NEGATIVE_CONTROL_SCENARIOS) + rules_path, tests_path = path / "old-pool-rule.yml", path / "old-pool-tests.yml" + rules_path.write_text(json.dumps({"groups": [{"name": "old-pool-rule", "rules": [old_alert]}]})) + tests_path.write_text(json.dumps({"rule_files": [str(rules_path)], + "evaluation_interval": "1s", "tests": negative_tests})) + subprocess.run([promtool, "check", "rules", str(rules_path)], check=True) + result = subprocess.run([promtool, "test", "rules", str(tests_path)], + capture_output=True, text=True) + output = result.stdout + result.stderr + # The same inputs and expectations have already passed with the new rule; + # only its expression changes. Do not accept a parser or process failure. + assert result.returncode == 1 and re.search(r"got:\s*\[\]", output) and all( + name in output for name in NEGATIVE_CONTROL_SCENARIOS + ), "old pool rule did not produce the expected false negatives:\n" + output + print(f"Original pool rule fails {len(negative_tests)} required positive scenarios, as expected") + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--promtool", default=shutil.which("promtool")) + parser.add_argument("--scenarios", type=Path, + default=ROOT / "tests/fixtures/prism-alert-scenarios.json") + args = parser.parse_args() + if not args.promtool: + parser.error("install promtool or pass --promtool /path/to/promtool") + rules = [rule for path in ["docs/prism-native-alert-rules.json", + "docs/prism-postgres-alert-rules.json"] + for rule in load(path)["rules"]] + expressions = {rule["title"]: rule["expr"].replace("__NETWORK__", "mainnet") for rule in rules} + pool_rule = next(rule for rule in rules if rule["title"] == POOL_WAIT_TITLE) + pool_alert, pool_tests = pool_wait_tests(pool_rule, expressions[POOL_WAIT_TITLE]) + tests = [] + for scenario in json.loads(args.scenarios.read_text()): + tests.append({ + "name": scenario["name"], "interval": "1s", + "input_series": [{"series": metric, "values": value if isinstance(value, str) else f"{value}+0x360"} + for metric, value in scenario["series"].items()], + "promql_expr_test": [{ + "expr": "sum(" + expressions[name] + ")", "eval_time": scenario.get("eval_time", "6m"), + "exp_samples": [] if value is None else [{"labels": "{}", "value": value}], + } for name, value in scenario["expect"].items()], + }) + if "alerts" in scenario: + tests[-1]["alert_rule_test"] = [{ + "eval_time": scenario.get("eval_time", "6m"), "alertname": name, + "exp_alerts": [{"exp_labels": labels, "exp_annotations": {}} + for labels in (active if isinstance(active, list) else ([{}] if active else []))], + } for name, active in scenario["alerts"].items()] + with tempfile.TemporaryDirectory(prefix="prism-alert-promql-") as directory: + path = Path(directory) + # JSON is valid YAML; no Python YAML dependency is needed by this check. + checked_rules = [{"record": "prism_contract_" + name.replace(" ", "_"), "expr": expr} + for name, expr in expressions.items()] + checked_rules.append(pool_alert) + tests.extend(pool_tests) + # Every rule is also an alert rule with its dwell, so a scenario may + # assert pending versus firing for any of them; the pool-wait rule is + # registered by pool_wait_tests above with its labels. + for rule in rules: + if rule["title"] == POOL_WAIT_TITLE: + continue + assert rule["evaluator"] == "gt" and rule["threshold"] == 0 + checked_rules.append({"alert": rule["title"], "expr": f"({expressions[rule['title']]}) > 0", + "for": rule["for"]}) + (path / "rules.yml").write_text(json.dumps({"groups": [{ + "name": "prism-contract", "rules": checked_rules, + }]})) + (path / "tests.yml").write_text(json.dumps({"rule_files": [str(path / "rules.yml")], + "evaluation_interval": "1s", "tests": tests})) + subprocess.run([args.promtool, "check", "rules", str(path / "rules.yml")], check=True) + subprocess.run([args.promtool, "test", "rules", str(path / "tests.yml")], check=True) + check_pool_wait_negative_control(args.promtool, path, pool_alert, + expressions[POOL_WAIT_TITLE], pool_tests) + print(f"{len(rules)} expressions; {len(tests)} scenarios; " + f"{sum(len(test['promql_expr_test']) for test in tests)} expression assertions; " + f"{sum(len(test.get('alert_rule_test', [])) for test in tests)} alert assertions passed") + + +if __name__ == "__main__": + main() diff --git a/scripts/test_prism_postgres_alerts.py b/scripts/test_prism_postgres_alerts.py new file mode 100644 index 000000000..f7521ecad --- /dev/null +++ b/scripts/test_prism_postgres_alerts.py @@ -0,0 +1,203 @@ +#!/usr/bin/env python3 +"""Exercise the exact D3 SQL on disposable asynchronous PostgreSQL 16 clusters. + +Requires PyYAML, PostgreSQL server tools, and promtool. The recorded SQL samples +feed PromQL at the required one-second cadence; exporter status is controlled by +the harness, not measured from a running postgres_exporter. Never connects to a +deployed database. Optional --report writes the observations and assertion count. +""" + +import argparse +import getpass +import json +import os +from pathlib import Path +import shutil +import socket +import subprocess +import sys +import tempfile +import time + +import yaml + +ROOT = Path(__file__).resolve().parents[1] + + +def free_port(): + with socket.socket() as listener: + listener.bind(("127.0.0.1", 0)) + return listener.getsockname()[1] + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--pg-bin-dir", type=Path, default=os.environ.get("PRISM_TEST_PG_BIN_DIR")) + parser.add_argument("--promtool", default=shutil.which("promtool")) + parser.add_argument("--report", type=Path) + args = parser.parse_args() + if not args.pg_bin_dir or not args.promtool: + parser.error("provide --pg-bin-dir (or PRISM_TEST_PG_BIN_DIR) and --promtool") + query = yaml.safe_load((ROOT / "docs/prism-postgres-exporter-queries.yaml").read_text())["pg_stat_replication"]["query"].strip().rstrip(";") + + def run(binary, *arguments): + result = subprocess.run([str(args.pg_bin_dir / binary), *map(str, arguments)], + env={**os.environ, "LC_ALL": "C"}, capture_output=True, + text=True, timeout=45) + if result.returncode: + raise RuntimeError(f"{binary}: {result.stdout}\n{result.stderr}") + return result.stdout.strip() + + def sql(port, statement): + return run("psql", "-X", "-h", "127.0.0.1", "-p", port, "-d", "postgres", + "-v", "ON_ERROR_STOP=1", "-At", "-c", statement) + + with tempfile.TemporaryDirectory(prefix="prism-d3-sql-") as directory: + root = Path(directory) + primary, standby = root / "primary", root / "standby" + primary_port, standby_port = free_port(), free_port() + while standby_port == primary_port: + standby_port = free_port() + active = [] + samples = [] + cases = [] + started = time.monotonic() + + def observe(): + row = json.loads(sql(primary_port, f"SELECT row_to_json(r) FROM ({query}) r")) + raw = sql(primary_port, "SELECT EXTRACT(EPOCH FROM replay_lag) FROM pg_stat_replication WHERE application_name='prism_standby_1'") + samples.append({"elapsed_seconds": round(time.monotonic() - started, 3), + "metrics": row, "raw_replay_lag": float(raw) if raw else None}) + return row + + def collect(seconds): + for _ in range(seconds): + tick = time.monotonic() + observe() + time.sleep(max(0, 1 - (time.monotonic() - tick))) + + def checkpoint(name, expected, alerts=None): + cases.append({"name": name, "sample": len(samples) - 1, + "expect": expected, "alerts": alerts}) + print(f"{name}: {samples[-1]}", flush=True) + + def start(data, port): + options = (f"-h 127.0.0.1 -p {port} -k {root} -c wal_level=replica " + "-c max_wal_senders=4 " + "-c wal_receiver_status_interval=1s -c checkpoint_timeout=1h -c autovacuum=off") + run("pg_ctl", "-D", data, "-l", root / (data.name + ".log"), "-o", options, "-w", "start") + active.append(data) + + try: + run("initdb", "-D", primary, "-A", "trust", "--no-locale", "-E", "UTF8") + start(primary, primary_port) + run("pg_basebackup", "-D", standby, "-d", + f"host=127.0.0.1 port={primary_port} user={getpass.getuser()} application_name=prism_standby_1", + "-R", "-X", "stream", "-c", "fast") + start(standby, standby_port) + sql(primary_port, "CREATE TABLE d3_probe(id integer); INSERT INTO d3_probe VALUES (1)") + for _ in range(30): + row = observe() + if row["count"] == 1 and row["replay_lag"] == 0 and samples[-1]["raw_replay_lag"] is None: + break + time.sleep(1) + else: + raise AssertionError("standby never became caught-up/idle with NULL raw replay_lag") + assert sql(primary_port, "SELECT sync_state FROM pg_stat_replication WHERE application_name='prism_standby_1'") == "async" + samples.clear() + collect(7) + checkpoint("SQL healthy idle NULL lag has caught-up WAL proof", (0, 0), (False, False)) + sql(standby_port, "SELECT pg_wal_replay_pause()") + sql(primary_port, "INSERT INTO d3_probe VALUES (2)") + collect(4) + checkpoint("SQL recent write remains inside five-second allowance", (0, 0), (False, False)) + collect(3) + checkpoint("SQL paused replay exceeds allowance but not dwell", (1, 0), (False, False)) + collect(62) + checkpoint("SQL paused replay passes one-minute dwell", (1, 0), (True, False)) + sql(standby_port, "SELECT pg_wal_replay_resume()") + for _ in range(15): + collect(1) + if samples[-1]["metrics"]["replay_lag"] == 0: + break + assert samples[-1]["metrics"]["replay_lag"] == 0 + checkpoint("SQL resumed replay is caught up despite old reported latency", (0, 0), (False, False)) + # A synchronous configuration violates the async ACK contract even + # while replay is caught up; preserve actual positions and expose + # the invalid topology separately instead of fabricating unknown WAL. + sql(primary_port, "ALTER SYSTEM SET synchronous_standby_names = 'FIRST 1 (prism_standby_1)'") + sql(primary_port, "SELECT pg_reload_conf()") + for _ in range(10): + collect(1) + if sql(primary_port, "SELECT sync_state FROM pg_stat_replication WHERE application_name='prism_standby_1'") == "sync": + break + else: + raise AssertionError("standby did not become synchronous") + collect(1) + assert samples[-1]["metrics"]["async"] == 0 + checkpoint("SQL synchronous standby violates required asynchronous topology", (1, 0)) + sql(primary_port, "ALTER SYSTEM SET synchronous_standby_names = 'FIRST 1 (another_standby)'") + sql(primary_port, "SELECT pg_reload_conf()") + for _ in range(10): + collect(1) + if sql(primary_port, "SELECT sync_state FROM pg_stat_replication WHERE application_name='prism_standby_1'") == "async": + break + else: + raise AssertionError("dedicated standby did not return to async state") + collect(1) + assert samples[-1]["metrics"]["async"] == 0 + checkpoint("SQL primary waiting for another standby also violates topology", (1, 0)) + sql(primary_port, "ALTER SYSTEM RESET synchronous_standby_names") + sql(primary_port, "SELECT pg_reload_conf()") + for _ in range(10): + collect(1) + if samples[-1]["metrics"]["async"] == 1: + break + assert samples[-1]["metrics"]["async"] == 1 + checkpoint("SQL asynchronous topology restored", (0, 0), (False, False)) + # The same primary-only query must fail on a recovery server, rather + # than silently produce a healthy zero when deployed to the wrong role. + try: + sql(standby_port, query) + except RuntimeError as error: + assert "recovery" in str(error), error + else: + raise AssertionError("primary query unexpectedly succeeded during recovery") + run("pg_ctl", "-D", standby, "-m", "immediate", "-w", "stop") + active.remove(standby) + collect(2) + row = samples[-1]["metrics"] + assert row["count"] == 0 and row["replay_lsn_hi"] == -1 and row["replay_lag"] == -1 + checkpoint("SQL disconnected standby is unknown and unavailable", (1, 1)) + + target = 'job="qbit-postgres-primary",instance="primary",network="mainnet"' + labels = target + ',application_name="prism_standby_1"' + series = {name + "{" + target + "}": value for name, value in + [("up", 1), ("pg_up", 1), ("pg_exporter_last_scrape_error", 0)]} + for name in samples[0]["metrics"]: + if name != "application_name": + series["pg_stat_replication_" + name + "{" + labels + "}"] = " ".join( + str(sample["metrics"][name]) for sample in samples) + titles = ["PrismHAStandbyReplayLagHigh", "PrismHAStandbyDisconnected"] + scenarios = [{"name": case["name"], "series": series, "eval_time": f"{case['sample']}s", + "expect": dict(zip(titles, case["expect"]))} for case in cases] + for scenario, case in zip(scenarios, cases): + if case["alerts"] is not None: + scenario["alerts"] = dict(zip(titles, case["alerts"])) + fixture = root / "scenarios.json" + fixture.write_text(json.dumps(scenarios)) + subprocess.run([sys.executable, str(ROOT / "scripts/test_prism_alert_promql.py"), + "--promtool", args.promtool, "--scenarios", str(fixture)], check=True) + if args.report: + args.report.write_text(json.dumps({"postgres": run("postgres", "--version"), + "samples": samples, "cases": cases, + "exporter": "not started; status inputs controlled by harness", + "wrong_role_query": "failed as expected"}, indent=2) + "\n") + print("D3 SQL and derived PromQL passed; disposable clusters cleaned on exit", flush=True) + finally: + for data in reversed(active): + run("pg_ctl", "-D", data, "-m", "immediate", "-w", "stop") + + +if __name__ == "__main__": + main() diff --git a/test/e2e-scenarios.toml b/test/e2e-scenarios.toml new file mode 100644 index 000000000..1094d4cea --- /dev/null +++ b/test/e2e-scenarios.toml @@ -0,0 +1,924 @@ +# End-to-end and load scenario manifest (#543, #487 S14). +# +# Every end-to-end and load scenario we care about before production, with the +# lane that runs it, the issue that owns it, what it must show to pass, and +# whether it runs today. scripts/check_e2e_scenarios.py holds this file to +# what the lanes actually run; its docstring has the rules. In short: +# +# - A scenario with `runs = true` cites its evidence, and the evidence decides +# its lanes: `tests` are gated test ids (test/prism-gated-tests.txt runs in +# `pr`, test/prism-nightly-gated-tests.txt in `nightly`, +# test/prism-weekly-gated-tests.txt in `weekly`), `presets` are +# crates/qbit-prism-load/presets (a nightly preset runs in `nightly`, a +# weekly one in `weekly`, a manual one in `dispatch`, `smoke` in `pr`), +# `unit_tests` are +# `<file>::<fn>` ungated tests that `cargo test --workspace` runs in `pr`, +# and `lane_checks` are the named checks of scripts/prism_shipped_image_lane.py, +# which run in `L6`. +# - A scenario with `runs = false` cites no evidence and gives a `reason`; its +# `owner` is the linked issue, and its `lanes` are the lanes it is planned +# for (#487's definition of done: nothing unexercised without both). +# - Every live_regtest and load_smoke test in the PR gated list, every opt-in +# nightly and weekly test, every preset and every L6 check must be cited by +# a running scenario. +# +# Scenarios whose fixtures belong to #521 or #474 are listed under those +# issues; #487's lanes run them under load without redefining what they +# assert (#487's division of work). + +schema = "qbit.e2e-scenarios.v1" + +# --------------------------------------------------------------------------- +# Lanes. `pr`, `nightly`, `dispatch`, `weekly` and L6 run today; L2 to L5 are #487's +# lanes that do not exist yet, each owned by the sub-issue that builds it. +# --------------------------------------------------------------------------- + +[lanes.pr] +title = "Every PR and push: the required ci.yml shards (#487 L0 and L1)" +runs = true +workflow = ".github/workflows/ci.yml" + +[lanes.nightly] +title = "Nightly on 3.x.x: nightly presets and the opt-in live variants (#521 item 6)" +runs = true +workflow = ".github/workflows/prism-load-nightly.yml" + +[lanes.dispatch] +title = "On demand only: the manual presets through workflow_dispatch" +runs = true +workflow = ".github/workflows/prism-load-nightly.yml" + +[lanes.weekly] +title = "Weekly on 3.x.x: the long soak's weekly presets (#575 item 2) and the live-weekly job's opt-in live_regtest tests (#487 L4)" +runs = true +workflow = ".github/workflows/prism-load-nightly.yml" + +[lanes.L2] +title = "Nightly load regression with repeats, a trend and a regression rule" +runs = false +owner = "#549" + +[lanes.L3] +title = "Production-window matrix: full on the version bump, reduced weekly (#550)" +runs = true +owner = "#550" +workflow = ".github/workflows/prism-load-l3.yml" + +[lanes.L4] +title = "Real-node scenarios under session load" +runs = false +owner = "#553" + +[lanes.L5] +title = "Two-hour soak and chaos" +runs = false +owner = "#556" + +[lanes.L6] +title = "Shipped images: the Compose prism profile with the HA overlay, weekly and on Dockerfile or Compose changes (#544)" +runs = true +workflow = ".github/workflows/prism-load-nightly.yml" + +# --------------------------------------------------------------------------- +# Real-node baseline in the PR shards (#487 "Where we are": live_regtest). +# --------------------------------------------------------------------------- + +[[scenario]] +id = "live-two-frontend-mining-failover-audit-reorg" +title = "Two server processes on a real qbitd: mining, frontend failover, audit and a reorg" +owner = "#487" +lanes = ["pr"] +criteria = "Both frontends mine real blocks; a miner fails over between them; the audit verifies against the real coinbase; a reorg is followed without duplicate credit." +runs = true +tests = ["qbit-prism-server::live_regtest::real_two_server_mining_failover_audit_and_reorg"] +notes = "Built with the native server (#244); #487 lists it as the L0 baseline." + +[[scenario]] +id = "live-ctv-maturity-and-confirmation" +title = "CTV payouts reach maturity, broadcast and confirm on a real qbitd" +owner = "#487" +lanes = ["pr"] +criteria = "A won block's CTV fanout matures, is broadcast and confirms on the real chain." +runs = true +tests = ["qbit-prism-server::live_regtest::real_ctv_maturity_broadcast_and_confirmation"] + +[[scenario]] +id = "live-ctv-cpfp-recovery" +title = "CTV CPFP funding recovers from an owner crash, an abandoned child and a spent reservation" +owner = "#487" +lanes = ["pr"] +criteria = "Each CPFP fault is repaired against the real wallet without losing or double-spending the reserved funding." +runs = true +tests = [ + "qbit-prism-server::live_regtest::cpfp_tests::real_cpfp_abandoned_child_repair_retains_unremovable_spent_lock", + "qbit-prism-server::live_regtest::cpfp_tests::real_cpfp_recovers_reserved_funding_after_owner_crash", + "qbit-prism-server::live_regtest::cpfp_tests::real_cpfp_replaces_unsigned_funding_spent_after_reservation_before_wallet_lock", +] + +[[scenario]] +id = "live-large-template-compact-runtime" +title = "A large real template: first insertion, resume and mining on the compact runtime" +owner = "#397" +lanes = ["pr"] +criteria = "The first large template is stored, resumed and mined with bounded JSONB and WAL." +runs = true +tests = ["qbit-prism-server::live_regtest::compact_runtime_tests::real_large_template_first_insertion_resumes_and_mines_with_bounded_jsonb_and_wal"] + +# --------------------------------------------------------------------------- +# Functional entries from #487's catalogue, mapped to the tests that cover +# them (#543). Only vardiff convergence had no test; #558 added it. +# --------------------------------------------------------------------------- + +[[scenario]] +id = "highdiff-listener" +title = "The high-difficulty listener: binding, floors, block-only proofs and shared admission" +owner = "#487" +lanes = ["pr"] +criteria = "The listener binds only when configured and fails startup on a held port; a valid block below the share target is still forwarded; a block-only proof waits for active-chain credit on a real node; the per-source cap is one budget across both listeners." +runs = true +tests = [ + "qbit-prism-server::live_regtest::highdiff_tests::observability_grace_counter_moves_only_after_real_durable_credit", + "qbit-prism-server::live_regtest::highdiff_tests::observed_tip_advance_fences_old_prepared_jobs_and_health", + "qbit-prism-server::live_regtest::highdiff_tests::real_highdiff_block_only_proof_waits_for_active_chain_credit", + "qbit-prism-server::live_regtest::highdiff_tests::real_share_height_queries_use_template_parent_height", + "qbit-prism-server::stratum_restart_bind::a_configured_highdiff_port_is_bound_and_accepts", + "qbit-prism-server::stratum_restart_bind::held_highdiff_port_fails_startup_before_registration", + "qbit-prism-server::stratum_restart_bind::without_a_highdiff_port_only_the_primary_listener_is_bound", +] +unit_tests = [ + "crates/qbit-prism-server/tests/stratum_protocol.rs::highdiff_floor_still_forwards_valid_block_below_share_target", + "crates/qbit-prism-server/tests/stratum_protocol.rs::per_source_cap_is_one_budget_across_the_default_and_highdiff_listeners", +] + +[[scenario]] +id = "version-rolling" +title = "Version rolling: mask negotiation, rolled submits and the mask kept across frontends" +owner = "#487" +lanes = ["pr"] +criteria = "mining.configure negotiates the template's mask; a share rolled under that mask is accepted, including after reconnecting to another listener that negotiated none; issued work keeps its original mask." +runs = true +tests = ["qbit-prism-server::compact_runtime_e2e::real_socket_reconnect_resumes_original_entropy_mask_and_submits_once"] +unit_tests = [ + "crates/qbit-prism-server/tests/stratum_codec.rs::version_mask_advertisement_is_authoritative", + "crates/qbit-prism-server/tests/stratum_protocol.rs::negotiation_reauthorization_retains_original_job_worker_and_grace_expires", + "crates/qbit-prism-server/tests/stratum_protocol.rs::reconnect_on_another_listener_preserves_original_entropy_mask_and_expiry", +] + +[[scenario]] +id = "malformed-frames-and-admission-limits" +title = "Malformed and oversized frames, and the admission limits" +owner = "#468" +lanes = ["pr"] +criteria = "Oversized frames close the connection; malformed frames, authorize cycling and unknown-job misses are charged to per-session budgets that disconnect; the per-source cap refuses at accept without backend work; a disconnect releases every permit; recommended budgets admit honest traffic; invalid settings are refused by name." +runs = true +tests = [ + "qbit-prism-server::stratum_admission_postgres::ten_thousand_unsubscribed_connections_do_not_advance_postgres_sequence", + "qbit-prism-server::stratum_admission_postgres::unknown_job_budget_bounds_ledger_queries_and_a_capped_source_runs_none", + "qbit-prism-server::stratum_admission_postgres::wrap_exhaustion_leaves_subscription_retryable_and_disconnect_releases_guard", +] +unit_tests = [ + "crates/qbit-prism-server/tests/stratum_admission.rs::malformed_subscribe_does_not_allocate_then_pipelined_subscribes_reuse_identity", + "crates/qbit-prism-server/tests/stratum_protocol.rs::oversized_frames_close_connection_and_fragmented_json_survives_timer_ticks", + "crates/qbit-prism-server/tests/stratum_protocol.rs::malformed_frame_budget_counts_frames_not_bytes_and_then_disconnects", + "crates/qbit-prism-server/tests/stratum_protocol.rs::every_unknown_job_miss_is_requeried_and_charged_until_the_budget_disconnects", + "crates/qbit-prism-server/tests/stratum_protocol.rs::authorize_budget_bounds_username_cycling_and_keeps_reauthorization_working", + "crates/qbit-prism-server/tests/stratum_protocol.rs::per_source_cap_refuses_at_accept_without_backend_work_and_frees_the_slot_on_close", + "crates/qbit-prism-server/tests/stratum_protocol.rs::per_source_cap_of_zero_admits_every_connection_from_one_address", + "crates/qbit-prism-server/tests/stratum_protocol.rs::a_budget_disconnect_releases_the_global_per_source_and_per_username_permits", + "crates/qbit-prism-server/tests/stratum_protocol.rs::recommended_budgets_admit_a_thousand_valid_submits_and_an_honest_stale_burst", + "crates/qbit-prism-server/tests/stratum_protocol.rs::admission_settings_reject_malformed_and_out_of_range_values_by_name", +] + +[[scenario]] +id = "cross-frontend-authorisation" +title = "Cross-frontend authorisation: a job issued by A submitted to B, a wrong worker, a duplicate on both (#281 criterion 1)" +owner = "#281" +lanes = ["pr", "L6"] +criteria = "Work issued by frontend A is accepted once on B over a real socket; another worker cannot resume or submit it; the same share replayed on B, or appended through a second instance's ledger, is rejected as a duplicate and credited once." +runs = true +tests = [ + "qbit-prism-server::compact_runtime_e2e::real_socket_reconnect_resumes_original_entropy_mask_and_submits_once", + "qbit-prism-server::compact_runtime_e2e::refresh_issue_resume_preserves_original_work_and_compact_storage", + "qbit-prism-server::ledger_postgres::global_duplicates_idempotence_and_config_fencing", +] +unit_tests = ["crates/qbit-prism-server/tests/stratum_protocol.rs::reconnect_on_another_listener_preserves_original_entropy_mask_and_expiry"] +lane_checks = ["cross-frontend-resume"] +notes = "L6 runs #281's Compose exercise of this criterion on the shipped images and the HA overlay: work issued by frontend 1 is accepted once on frontend 2, another worker's submission and a replay on either frontend are refused, and the ledger credits it once." + +[[scenario]] +id = "vardiff-retarget-steps" +title = "Vardiff retarget steps and their delivery" +owner = "#487" +lanes = ["pr"] +criteria = "One retarget uses accepted work, bounds its early and idle steps, is delivered or restored with its paired job, and a lowered idle difficulty is kept across reconnect." +runs = true +unit_tests = [ + "crates/qbit-prism-server/src/vardiff.rs::fast_arrival_uses_one_bounded_evidence_gate_and_normal_steps_after_delivery", + "crates/qbit-prism-server/src/vardiff.rs::normal_arrivals_do_not_repeatedly_test_the_initial_noise_gate", + "crates/qbit-prism-server/tests/stratum_codec.rs::vardiff_uses_actual_accepted_work_and_limits_idle_step", + "crates/qbit-prism-server/tests/stratum_protocol.rs::fast_arrival_retries_failed_paired_delivery_and_only_proven_difficulty_resumes", + "crates/qbit-prism-server/tests/stratum_protocol.rs::idle_retarget_lowers_reconnect_difficulty_without_renewing_evidence", +] + +[[scenario]] +id = "vardiff-convergence" +title = "Vardiff convergence per session over a real Stratum socket" +owner = "#558" +lanes = ["pr"] +criteria = "Sessions starting well below and well above their target settle within a stated number of retargets into the target share-interval band and do not oscillate." +runs = true +unit_tests = ["crates/qbit-prism-server/tests/vardiff_convergence.rs::sessions_below_and_above_their_target_settle_into_the_share_interval_band"] +notes = "Two miners 16x below and 16x above their target over a real socket to the native listener, each settling within 6 retarget periods and 4 adjustments (2 expected) into 1.5x of the target share interval, every adjustment closer to the target and none reversing direction, and changing at most once more over the next 8 retarget periods. It takes 8 s of wall time and little CPU, so it runs per PR." + +# --------------------------------------------------------------------------- +# #521's scenarios: #521 owns the fixtures and assertions; #487's L4 and L5 +# run them under load (S15, S16, S18). +# --------------------------------------------------------------------------- + +[[scenario]] +id = "r521-1-lost-same-height-race" +title = "#521 scenario 1: an own block loses a same-height race and is orphaned" +owner = "#521" +lanes = ["pr"] +criteria = "On a second qbitd, the own block is orphaned, credited once on the reorg back, with no duplicate header." +runs = true +tests = ["qbit-prism-server::live_regtest::two_node_tests::real_own_block_loses_same_height_race_is_orphaned_and_credited_once_on_reorg_back"] + +[[scenario]] +id = "r521-2-deep-reorg" +title = "#521 scenario 2: a reorg deeper than the orphan-confirmation depth, own blocks on both branches" +owner = "#521" +lanes = ["pr"] +criteria = "Credit, balances and fanouts are exact after the reorg." +runs = true +tests = ["qbit-prism-server::live_regtest::two_node_tests::real_deep_reorg_with_own_blocks_on_both_branches_keeps_credit_balances_and_fanouts_exact"] + +[[scenario]] +id = "r521-3-postgres-failover-mid-landing" +title = "#521 scenario 3: PostgreSQL failover with a found block mid-landing" +owner = "#521" +lanes = ["pr"] +criteria = "Fenced failover keeps every acknowledged share; async failover loses only the replication gap, and the block lands; with the standby flush wait (#529), a block offered at a replication cut survives the promotion and lands once." +runs = true +tests = [ + "qbit-prism-server::live_regtest::pg_failover_tests::offer_standby::a_found_block_offered_after_the_standby_flush_survives_a_replication_cut_at_submitblock", + "qbit-prism-server::live_regtest::pg_failover_tests::real_async_failover_mid_landing_loses_only_the_replication_gap", + "qbit-prism-server::live_regtest::pg_failover_tests::real_fenced_failover_mid_landing_keeps_every_acknowledged_share", +] + +[[scenario]] +id = "r521-4-qbitd-outage-around-submitblock" +title = "#521 scenario 4: a qbitd stop or crash around submitblock" +owner = "#521" +lanes = ["pr"] +criteria = "Each block is offered once and lands, or is reconciled, after the node returns." +runs = true +tests = [ + "qbit-prism-server::live_regtest::node_outage_tests::node_killed_after_accepting_reconciles_the_lost_block_once", + "qbit-prism-server::live_regtest::node_outage_tests::node_killed_before_submitblock_lands_the_block_once_after_restart", + "qbit-prism-server::live_regtest::node_outage_tests::node_restarted_into_warmup_before_submitblock_lands_the_block_once", + "qbit-prism-server::live_regtest::node_outage_tests::node_stopped_after_accepting_lands_the_block_once_after_resume", + "qbit-prism-server::live_regtest::node_outage_tests::node_stopped_before_submitblock_lands_the_block_once_after_resume", +] + +[[scenario]] +id = "r521-4-qbitd-reindex-nightly" +title = "#521 scenario 4, opt-in: the node is killed after accepting and restarted with -reindex" +owner = "#521" +lanes = ["nightly"] +criteria = "The accepted block is recovered from the block files and lands once." +runs = true +tests = ["qbit-prism-server::live_regtest::node_outage_tests::node_killed_after_accepting_and_reindexed_lands_the_block_once"] + +[[scenario]] +id = "r521-5-dense-cadence-soak" +title = "#521 scenario 5: dense-cadence landings and a 10-minute nightly soak on a real node" +owner = "#521" +lanes = ["nightly"] +criteria = "Every block found on live work is a candidate that settles (submitted, orphaned, or abandoned only for a superseded parent); each server's accepted-block-to-revision-work p99 stays within 5 s with no degraded landing; RSS stays flat." +runs = true +tests = ["qbit-prism-server::live_regtest::dense_soak_tests::dense_cadence_soak_lands_every_block_rebuilds_in_budget_and_holds_rss_flat"] + +[[scenario]] +id = "r521-6-pr-smoke" +title = "#521 item 6: the per-PR load-harness smoke (#487 L1's harness half)" +owner = "#521" +lanes = ["pr"] +criteria = "A debug frontend serves every session and every tip and the run reconciles exactly under the smoke preset's gates." +runs = true +tests = ["qbit-prism-load::load_smoke::the_smoke_preset_serves_every_session_every_tip_and_reconciles"] +presets = ["pr-smoke"] + +[[scenario]] +id = "r548-ctv-fanout-settlement-under-load" +title = "#548: CTV fanout settlement in the load harness (#487 S9)" +owner = "#548" +lanes = ["pr"] +criteria = "With --ctv-settlement, a fake-node run whose 400 payable recipients overflow the direct-output cap lands its block with 12 direct recipients and the rest in one fanout chunk, reconciles exactly, and its side report's settlement block counts the direct, fanout and carried recipients from the server's landing rows." +runs = true +tests = ["qbit-prism-load::ctv_settlement::a_ctv_run_pays_the_direct_overflow_through_fanout_and_counts_the_split"] +unit_tests = [ + "crates/qbit-prism-load/tests/harness.rs::ctv_settlement_sets_mainnets_ctv_keys_and_leaves_the_broadcaster_off", + "crates/qbit-prism-load/tests/harness.rs::the_settlement_block_keeps_unmeasured_apart_from_zero", +] +notes = "The economic assertions on spent fanout outputs stay with r521-7 on the real node; this adds the settlement path under load only." + +[[scenario]] +id = "r521-7-weighted-recipients-share-to-spend" +title = "#521 scenario 7: weighted multi-address shares to spent outputs (#487 L1's wallet half)" +owner = "#521" +lanes = ["pr"] +criteria = "40 weighted payout addresses receive exact PPLNS outputs through fanout, and the owning wallets spend them." +runs = true +tests = ["qbit-prism-server::live_regtest::weighted_recipients_tests::real_weighted_recipients_pay_exact_pplns_outputs_through_fanout_spend"] + +[[scenario]] +id = "r521-7-mainnet-floor-recipients-nightly" +title = "#521 scenario 7, opt-in: 130 mainnet-floor recipients through six blocks" +owner = "#521" +lanes = ["nightly"] +criteria = "As scenario 7, at the mainnet-floor recipient shape." +runs = true +tests = ["qbit-prism-server::live_regtest::weighted_recipients_tests::nightly_mainnet_floor_weighted_recipients_share_to_spend"] + +[[scenario]] +id = "real-node-harness-smoke" +title = "#547: the per-PR real-node harness smoke on a managed regtest qbitd pair" +owner = "#547" +lanes = ["pr"] +criteria = "Exit 0: every share reconciled against PostgreSQL, every block above the ramp exactly one peer mint or one accepted relay submission at the ramped bits, and the cadence band stated; fake-node mode byte-identical to before real-node mode existed." +runs = true +tests = ["qbit-prism-load::real_node::real_node_smoke_reconciles_against_postgres_and_the_chain"] +presets = ["real-node-smoke"] +unit_tests = ["crates/qbit-prism-load/tests/real_node.rs::fake_node_mode_is_unchanged"] + +[[scenario]] +id = "real-node-short-plan-pair" +title = "#547: the short plan on the real and the fake node, differing only in --node" +owner = "#547" +lanes = ["nightly"] +criteria = "Both exit 0 under the presets' gates; the real-node run's chain reconciles and its report states the cadence band." +runs = true +presets = ["short-plan-real-node", "short-plan-fake-node"] + +# #575's pre-release gaps on a real node: resource exhaustion (item 5) and +# the replay of recorded Stratum transcripts (item 3's automatable half). + +[[scenario]] +id = "b575-5-postgres-disk-full" +title = "#575 item 5: PostgreSQL's data volume fills under mining and is freed" +owner = "#575" +lanes = ["nightly"] +criteria = "Every acknowledged share is durable; the full state is refused explicitly, and PrismDatabaseUnavailable and at least one more rule hold for their `for` on every server (#581); once space is freed the same server processes accept shares and land a block with clean carry integrity." +runs = true +tests = ["qbit-prism-server::live_regtest::disk_exhaustion_tests::postgres_volume_full_refuses_shares_cleanly_and_recovers_once_space_is_freed"] + +[[scenario]] +id = "b575-5-wall-clock-jumps" +title = "#575 item 5: server and database wall clocks jump minutes and hours, including mid-landing" +owner = "#575" +lanes = ["nightly"] +criteria = "Every acknowledged share is credited once with a monotonic ledger clock; every block is offered exactly once, lands once and matches its audit bundle; a server far ahead of the node serves no work and PrismWorkRefreshStalledCritical fires; every outbox row finishes, and neither block held across a database clock step is taken over (#581)." +runs = true +tests = ["qbit-prism-server::live_regtest::clock_jump_tests::wall_clock_jumps_keep_windows_claims_and_payouts_and_land_every_block_once"] + +[[scenario]] +id = "b575-3-stratum-transcript-replay" +title = "#575 item 3: recorded Stratum client transcripts replayed against a live server" +owner = "#575" +lanes = ["pr"] +criteria = "Every replayed request of every corpus transcript gets an answer the recorded client accepts, every notification has the shape miners parse, and every accepted replayed share is in the ledger." +runs = true +tests = ["qbit-prism-server::live_regtest::transcript_replay_tests::recorded_stratum_transcripts_replay_compatibly_against_a_live_server"] +unit_tests = ["crates/qbit-prism-server/tests/support/live_transcript_replay.rs::stratum_transcript_corpus_is_well_formed_and_labelled"] + +# The load-harness presets #521 item 6 runs (fake node). Each preset states +# its own gates; #549 turns the nightly set into L2 with repeats and a trend. + +[[scenario]] +id = "load-d1-20k" +title = "D1 plan over the 20k window, uniform and Zipf(1.1) recipients" +owner = "#521" +lanes = ["nightly"] +criteria = "Exit 0 with exact reconciliation, no durability finding, shortfall 0 and #473's D1 verdict in steady state." +runs = true +presets = ["throughput-20k-window-1fe", "throughput-20k-window-1fe-500-addresses"] + +[[scenario]] +id = "load-d1-400k-fe1-async" +title = "#473's production-window baseline cell: 400k, one frontend, async standby" +owner = "#521" +lanes = ["nightly"] +criteria = "The preset's gates, with #473's D1 verdict." +runs = true +presets = ["throughput-400k-window-1fe-async"] + +[[scenario]] +id = "load-tip-delivery-275" +title = "#275's tip-delivery benchmark: 2,000 miners, 400k window, two frontends, retargeting tips" +owner = "#521" +lanes = ["nightly"] +criteria = "The preset's gates, with the tip-to-last-notify p99 within 2,500 ms." +runs = true +presets = ["tip-delivery-2000-miners-400k-2fe-retarget"] + +[[scenario]] +id = "load-tip-delivery-275-500-addresses" +title = "#275's tip-delivery benchmark with 500 Zipf(1.1) payout addresses" +owner = "#521" +lanes = ["dispatch"] +criteria = "The preset's gates, with the tip-to-last-notify p99 within 2,500 ms." +runs = true +presets = ["tip-delivery-2000-miners-400k-2fe-retarget-500-addresses"] + +[[scenario]] +id = "load-runner-probe-short-plan" +title = "#541's runner probe: the short plan over the 20k-window fixture on one frontend" +owner = "#541" +lanes = ["dispatch"] +criteria = "The preset's gates on the runner under probe: exact reconciliation, no acknowledged share missing, zero shortfall in each gated phase." +runs = true +presets = ["short-plan-20k-window-1fe"] + +[[scenario]] +id = "load-mainnet-floor" +title = "Mainnet-shaped floor: 130 recipients, one whale, bursty arrivals" +owner = "#521" +lanes = ["nightly"] +criteria = "The preset's gates, with the tip-to-last-notify p99 within 3,000 ms." +runs = true +presets = ["mainnet-shape-130-addresses"] + +[[scenario]] +id = "load-growth-5x" +title = "Mainnet floor scaled five times: 650 recipients, 2,000 sessions" +owner = "#521" +lanes = ["nightly"] +criteria = "The preset's gates, with the tip-to-last-notify p99 within 3,000 ms." +runs = true +presets = ["mainnet-shape-650-addresses"] + +[[scenario]] +id = "load-rental-churn" +title = "Rental churn over the mainnet-floor population: bursts, abrupt departures, reconnect storms" +owner = "#521" +lanes = ["nightly"] +criteria = "The preset's gates, including the churn tip-delivery and new-session first-job budgets." +runs = true +presets = ["rental-churn-bursts-and-storms"] + +[[scenario]] +id = "load-growth-20x" +title = "Mainnet floor scaled twenty times: 2,600 recipients, 8,000 sessions on four frontends" +owner = "#521" +lanes = ["dispatch"] +criteria = "The preset's gates, with the tip-to-last-notify p99 within 5,000 ms." +runs = true +presets = ["mainnet-shape-2600-addresses"] + +[[scenario]] +id = "load-d1-473-cells" +title = "#473's other D1 cells: 200k, 400k and 500k at 1, 2 and 4 frontends, async and sync" +owner = "#521" +lanes = ["dispatch"] +criteria = "Each preset's gates, with #473's D1 verdict." +runs = true +presets = [ + "throughput-200k-window-1fe-async", + "throughput-400k-window-2fe-async", + "throughput-400k-window-2fe-sync", + "throughput-400k-window-4fe-async", + "throughput-500k-window-1fe-async", + "throughput-500k-window-2fe-async", + "throughput-500k-window-4fe-async", +] +notes = "Also in L3 (#550): l3-full runs every one of them, l3-reduced the 400k async ones (l3-production-window-matrix)." + +# --------------------------------------------------------------------------- +# #474's scenarios. Its A item's CTV case is #521 scenario 7 (#524), which +# #474 A extends with the signed audit bundle and the owning wallets; A adds +# direct settlement and the 1,001-recipient case. B and C have not landed. +# --------------------------------------------------------------------------- + +[[scenario]] +id = "r474-a-direct-recipients-share-to-spend" +title = "#474 A: weighted recipients through direct coinbase settlement (CTV off) to spent outputs" +owner = "#474" +lanes = ["pr"] +criteria = "Six addresses from six wallets with unequal weights, carry below the floor and the pool fee's dust sweep: exact direct outputs, ledger accounts and signed audit bundle against an independent model; the owning wallets spend a payee output of every block." +runs = true +tests = ["qbit-prism-server::live_regtest::weighted_recipients_tests::real_weighted_recipients_pay_exact_direct_outputs_without_ctv"] + +[[scenario]] +id = "r474-a-1001-recipients" +title = "#474 A: 1,001 payable recipients across the 1,000-recipient chunk boundary" +owner = "#474" +lanes = ["nightly"] +criteria = "Seeded (Ledger::append) and Stratum-submitted recipient counts reported separately; all 1,001 recipients paid exactly across two fanout chunks per the model and the signed audit bundle; the owning wallets spend an output of each chunk." +runs = true +tests = ["qbit-prism-server::live_regtest::weighted_recipients_tests::recipient_boundary::nightly_1001_recipients_cross_the_fanout_chunk_boundary_to_spent_outputs"] + +[[scenario]] +id = "r474-b-async-promotion-under-live-miners" +title = "#474 B: async promotion while miners and frontend caches survive" +owner = "#474" +lanes = ["pr"] +criteria = "Set by #474: exact offered, ACKed, rejected and unknown share identities reconciled against the promoted ledger under D3." +runs = true +tests = ["qbit-prism-server::live_regtest::pg_failover_tests::async_live_miners::async_promotion_with_live_miners_accounts_for_every_share_and_lands_a_block"] +notes = "Four scripted miners on two frontends through a replication cut and a fence with a COMMIT held in flight; every ACKed-but-missing share named, sessions and kept work survive the promotion, and a post-promotion block pays a full read of the promoted window. The block solved on a job issued inside the gap must be refused as stale or pay only promoted rows (#619; refused before its offer since #645)." + +[[scenario]] +id = "r474-c-lost-node-replies" +title = "#474 C: the node accepted the operation but its reply was lost" +owner = "#474" +lanes = ["pr"] +criteria = "Set by #474: no fabricated rejection, duplicate credit or second payout after the owning frontend is killed." +runs = true +tests = [ + "qbit-prism-server::live_regtest::lost_reply_tests::lost_sendrawtransaction_reply_owner_killed_survivor_confirms_the_fanout_once", + "qbit-prism-server::live_regtest::lost_reply_tests::lost_submitblock_reply_owner_killed_survivor_lands_the_block_once", + "qbit-prism-server::live_regtest::lost_reply_tests::lost_submitpackage_reply_owner_killed_survivor_confirms_the_cpfp_package_once", + "qbit-prism-server::live_regtest::lost_reply_tests::share_credited_but_ack_lost_retry_is_deduplicated_exactly", +] +notes = "A proxy withholds qbitd's reply to submitblock, sendrawtransaction or submitpackage after the node accepted it; the owning frontend is killed before it journals the outcome and the survivor reconciles. The miner case withholds a durably credited share's acknowledgement and closes the connection; the retry is a duplicate on both frontends." + +[[scenario]] +id = "r619-gap-work-after-async-promotion" +title = "#619: work prepared in the replication gap is not payout authority after an async promotion" +owner = "#619" +lanes = ["pr"] +criteria = "After a D3 promotion that lost the gap and reissued its share_seq values, a block on a held gap job is refused stale-job before its enqueue (window_not_held, share not credited), a new session is not issued the gap work and the refresh rebuilds under writer_timeline; a fenced switchover with full replay and a job persisted before the cut keep their authority and land." +runs = true +tests = [ + "qbit-prism-server::b619_gap_work_after_promotion::a_block_on_a_job_issued_in_the_gap_is_refused_stale_job_before_its_offer", + "qbit-prism-server::b619_gap_work_after_promotion::a_candidate_accepted_before_the_promotion_cannot_be_published_after_it", + "qbit-prism-server::b619_gap_work_after_promotion::a_fenced_switchover_with_full_replay_keeps_pre_switch_work_and_lands_its_block", + "qbit-prism-server::b619_gap_work_after_promotion::a_job_persisted_before_the_cut_is_resumed_after_an_async_promotion_and_lands", + "qbit-prism-server::b619_gap_work_after_promotion::a_refusal_that_finishes_after_the_acknowledgement_deadline_is_still_counted_once", + "qbit-prism-server::b619_gap_work_after_promotion::work_prepared_in_the_gap_is_not_issued_to_a_new_session_after_an_async_promotion", +] +notes = "Disposable PostgreSQL 16 primary and async standby, one native frontend and the fake node, no qbitd. See docs/prism-async-promotion-gap-blocks.md." + +# --------------------------------------------------------------------------- +# #487's catalogue entries that no lane runs yet, under the sub-issue that +# builds each. +# --------------------------------------------------------------------------- + +[[scenario]] +id = "wallets-2000-cap" +title = "2,000 wallets at the production window on #521 scenario 7's fixture" +owner = "#553" +lanes = ["L4"] +criteria = "#521 scenario 7's assertions unchanged at 2,000 payees (two fanout chunks per block): attribution, coinbase, CTV outputs and carry-forward balances match an independent PPLNS recomputation, and the fanouts reach the wallets and are spent (decision 7)." +runs = false +reason = "Blocked by #621 and #622. In debug, since #604, the fixture opens all of its about 1,500 worker sessions per frontend, but at its 1 s reanchor every session always has a job rebuild queued, and a session answers a submit only after that rebuild completes, so some submit gets no reply within the fixture's 30 s (#621). In release every session opens, and a later round's new tip reaches some worker late while the frontends defer job preparation on stale tip polling (#622). The live_regtest case wallets_2000_share_to_spend_waits_on_submit_acks_until_621_is_fixed reproduces #621 by hand." + +[[scenario]] +id = "chain-events-under-load" +title = "#521 scenarios 1, 2, 4 and 5 at L4's session counts" +owner = "#553" +lanes = ["nightly", "weekly"] +criteria = "#521's assertions for each scenario, unchanged, under 2,000 share-only sessions at 100 shares/s; and the load's own: every acknowledged share durable and no refused share credited (ledger-outcome-unknown counts as unanswered), every refusal one a correct miner can earn (an allow-list), and every session on a ready frontend given work on every tip that lasts 15 s within 15 s." +runs = true +tests = [ + "qbit-prism-server::live_regtest::chain_events_load_tests::nightly_deep_reorg_under_2000_sessions_keeps_balances_exact_and_serves_every_session", + "qbit-prism-server::live_regtest::chain_events_load_tests::nightly_lost_race_under_2000_sessions_orphans_once_and_serves_every_session", + "qbit-prism-server::live_regtest::chain_events_load_tests::nightly_node_killed_after_accepting_under_2000_sessions_reconciles_the_lost_block_once", + "qbit-prism-server::live_regtest::chain_events_load_tests::weekly_dense_soak_under_2000_sessions_lands_every_block_and_holds_rss_flat", + "qbit-prism-server::live_regtest::chain_events_load_tests::weekly_node_killed_before_submitblock_under_2000_sessions_lands_the_block_once", + "qbit-prism-server::live_regtest::chain_events_load_tests::weekly_node_restarted_into_warmup_under_2000_sessions_lands_the_block_once", + "qbit-prism-server::live_regtest::chain_events_load_tests::weekly_node_stopped_after_accepting_under_2000_sessions_lands_the_block_once", + "qbit-prism-server::live_regtest::chain_events_load_tests::weekly_node_stopped_before_submitblock_under_2000_sessions_lands_the_block_once", +] +notes = "The frontends run #575's share-only settings and the production 60 s payout reanchor (scenario one-frontend-fast-reanchor runs the fixture's 1 s as #598's guard). Scenarios 1 and 2 and scenario 4's crash case run nightly; scenario 4's other four cases and scenario 5's 20-minute soak at 400 shares/s run weekly." + +[[scenario]] +id = "one-frontend-fast-reanchor" +title = "#598's guard: 2,000 sessions on one frontend at a 1 s reanchor" +owner = "#598" +lanes = ["nightly"] +criteria = "With at least 99% of 2,000 sessions on the one frontend holding work, each of three external tips 20 s apart reaches every session holding work within 15 s at the live fixture's 1 s reanchor and 5 shares/s, with the load's own checks. Before #598's fix almost no session got work on the 2nd and 3rd tips. Debug only: in release the fan-out outruns the reanchor even without the fix." +runs = true +tests = ["qbit-prism-server::live_regtest::chain_events_load_tests::nightly_2000_sessions_on_one_frontend_at_a_1s_reanchor_get_work_on_every_tip"] + +[[scenario]] +id = "external-tips-time-to-usable-work" +title = "External tips at a realistic cadence; time to usable work per frontend (#481)" +owner = "#553" +lanes = ["nightly", "weekly"] +criteria = "External tips from a second node at qbit's 60 s target spacing (seeded exponential gaps) to 2,000 sessions on two frontends: every tip that lasts 15 s reaches every session on both frontends within 15 s, and #481's time to usable work (node stamp to first notify on that parent) is reported per frontend and per tip, replaced tips counted apart." +runs = true +tests = [ + "qbit-prism-server::live_regtest::chain_events_load_tests::nightly_external_tips_at_target_spacing_reach_every_session_on_both_frontends", + "qbit-prism-server::live_regtest::chain_events_load_tests::weekly_external_tips_at_target_spacing_reach_every_session_on_both_frontends", +] +notes = "5 minutes at 100 shares/s nightly; 45 minutes at 400 shares/s weekly." + +[[scenario]] +id = "node-loses-peer" +title = "The node loses its peer while miners are connected" +owner = "#553" +lanes = ["nightly"] +criteria = "Regtest exempts the peer floor (readiness_rpc.rs pins both sides), so: with its only peer partitioned away, the pool mines on an isolated node, its own block there is deactivated when the node reorganizes onto the peer's longer chain and credited exactly once, with no payout divergence or duplicate header, and every one of 2,000 sessions gets work on the isolated block and on the healed chain within 15 s." +runs = true +tests = ["qbit-prism-server::live_regtest::chain_events_load_tests::nightly_node_loses_its_peer_under_2000_sessions_orphans_the_isolated_block_and_recovers"] + +[[scenario]] +id = "two-frontends-disagree-on-tip" +title = "Two frontends on two nodes that briefly disagree about the tip" +owner = "#553" +lanes = ["nightly"] +criteria = "Propagation lag and a partitioned split at equal work: the ledger takes one side's tip, the frontend on the other side reports itself unready and gives no work on its node's tip, no payout divergence or duplicate header follows, and after convergence every session on both frontends has work on the converged tip within 15 s." +runs = true +tests = ["qbit-prism-server::live_regtest::chain_events_load_tests::nightly_two_frontends_on_two_nodes_disagree_briefly_and_converge_under_2000_sessions"] + +[[scenario]] +id = "l2-load-regression" +title = "D1 at 20k and 100k with repeats against the trailing same-class baseline" +owner = "#549" +lanes = ["L2"] +criteria = "Median of three repeats within the noise-floor rule of the trailing baseline, per phase (steady_state, burst, reconnect, slow_database)." +runs = false +reason = "The nightly runs throughput-20k-window-1fe once and gates it alone; repeats, the trend and the regression rule are #549 and #551." + +[[scenario]] +id = "bridging-fake-and-real-node" +title = "The same short plan against the fake and the real node on one runner" +owner = "#552" +lanes = ["nightly"] +criteria = "Each run writes a comparison row stating the real node's cost as real minus fake (ACK latency, time to usable work, peak RSS; submitblock latency, which only the real node has, stays unknown on the fake side). It is not a D1 verdict." +runs = true +presets = ["short-plan-fake-node", "short-plan-real-node"] +notes = "prism-load-nightly.yml's bridging job runs both presets on one runner, alternating the order, and scripts/prism_load_bridge.py writes the row; tests/test_prism_load_bridge.py holds that job's wiring. The matrix still runs each half on its own runner (real-node-short-plan-pair)." + +[[scenario]] +id = "l3-production-window-matrix" +title = "#447's full production-window matrix on the version-bump PR and the tag, and its reduced weekly set" +owner = "#550" +lanes = ["dispatch", "L3"] +criteria = "Every cell's gates; the correctness and regression rows count toward #291, the rates do not (decision 5)." +runs = true +suites = ["l3-full", "l3-reduced"] +presets = [ + "throughput-400k-window-2fe-async-3-blocks", + "dense-cadence-400k-window-1fe-async", + "dense-cadence-400k-window-2fe-async", +] +notes = "l3-full runs every #473 cell three times: the eight rate cells (load-d1-473-cells and the nightly 400k baseline), the found-block run and dense cadence at 1 and 2 frontends, listed here. The 20k dense attempt (exit 6) and #473's flush and build controls are not presets. The reduced weekly suite (l3-reduced) is 400k at 1, 2 and 4 frontends, async." + +[[scenario]] +id = "session-points" +title = "2,000, 5,000 and 10,000 sessions at 500 shares/s, plus the 10,000-session burst" +owner = "#555" +lanes = ["dispatch", "L3"] +criteria = "Per-session cost and ACK path under connection count, weekly (decision 7)." +runs = true +suites = ["l3-reduced"] +presets = [ + "throughput-400k-window-2fe-async", + "session-point-5000-sessions-400k-2fe-async", + "session-point-10000-sessions-400k-2fe-async", +] +notes = "Capacity-envelope points, not D1 verdicts (decision 5): the 5,000 and 10,000 points gate reconciliation and a steady_state that places every offer, and report refused and unanswered shares. All three are the 400k window on 2 async frontends in the D1 plan; the 10,000 point's burst phase (2,000 shares/s for 60 s) is the burst variant. Admission is sessions per frontend + 16, as the 2,000-session cell ran it." + +[[scenario]] +id = "recipient-point-10000" +title = "10,000 distinct payout addresses in the window, ten fanout chunks per block" +owner = "#555" +lanes = ["L3"] +criteria = "Window-rebuild and payout-revision cost stays linear; seeded and submitted counts reported separately." +runs = false +reason = "No preset yet (#555): ten fanout chunks per block need CTV fanout settlement, which qbit-prism-load has had since #548 (--ctv-settlement, #644), so the preset can now be written with it on." +notes = "The 50,000-recipient run's trigger (#555): this point costs more than 25% more per recipient than the same preset at 1,000 recipients (one fanout chunk), run on the same runner class the same week, in window rebuild (the tip-to-last-notify p99 over its tips) or in payout revision (from a landing to every session holding the new revision's work, the median over landings). A fixed cost only lowers the per-recipient figure at the larger size, so a linear cost never fires it. Evaluated once this point's preset exists and both presets run weekly (crates/qbit-prism-load/README.md, L3)." + +[[scenario]] +id = "mainnet-shapes-reduced-l3" +title = "The mainnet-shape presets (130, 650 and 2,600 addresses) and rental churn in the reduced weekly L3" +owner = "#555" +lanes = ["nightly", "dispatch", "L3"] +criteria = "Each preset's gates on the L3 runner class (decision 11)." +runs = true +suites = ["l3-reduced"] +presets = [ + "mainnet-shape-130-addresses", + "mainnet-shape-650-addresses", + "mainnet-shape-2600-addresses", + "rental-churn-bursts-and-storms", +] +notes = "l3-reduced runs each once a week on the 32 vCPU class, beside their nightly (8 vCPU) or dispatch (16 vCPU) runs; rates compare only within one class. Not D1 verdicts." + +# --------------------------------------------------------------------------- +# Faults under load (#554, #487 S16). The load harness's `faults` phase +# (crates/qbit-prism-load/src/fault/) injects each fault while the sessions +# keep mining, then holds it to the criteria below on top of the run's own +# contract: every acknowledged share in PostgreSQL, no committed share left +# unanswered mid-run, one submitblock per block hash. The criteria follow +# D3's async loss policy and #585's shutdown and lease behaviour. +# --------------------------------------------------------------------------- + +[[scenario]] +id = "fault-sigterm-drain-offer-in-flight" +title = "SIGTERM of a frontend while a found block's submitblock is in flight" +owner = "#554" +lanes = ["nightly", "pr", "weekly"] +criteria = "The shutdown logs that it waits for the offer and no budget ALERT, the frontend exits 0 within 35 s, the block reaches the node exactly once, the node's accepted is recorded on the candidate before the process exits (#578, #585), and the frontend serves again after its relaunch; within 60 s of that the relaunch lands the block and every frontend serves work at the payout revision the landing committed, before the next fault starts (#686)." +runs = true +tests = ["qbit-prism-load::faults::faults_under_load_drain_an_offer_in_flight_and_outlast_a_settlement_lock_holder"] +presets = ["faults-long-real-node", "faults-pr-smoke", "faults-short-real-node"] +notes = "Per PR on the fake node with one frontend; nightly on the real node with two frontends and 500 sessions." + +[[scenario]] +id = "fault-settlement-lock-holder" +title = "A long SETTLEMENT_LOCK holder while the sessions keep mining, with a tip minted during the hold" +owner = "#554" +lanes = ["nightly", "pr", "weekly"] +criteria = "The lock is taken once every frontend serves current work with no settlement under way, or 60 s from the injection have passed, so a block an earlier fault landed is settled and republished first, and a reading under the lock finds every frontend serving work at the revision it keeps (#692); no share waits on the holder (answer p99 over the whole hold within --ack-p99-limit-ms); shares on retained work are accepted (at least 90%) until the refresh has been stalled for PRISM_SUBMIT_TIP_MAX_AGE_SECONDS, after which the server's refusals are counted; no session gets work on the new tip before the release, and every session has it within 10 s of the release. The share-ACK p50, p99 and max over the hold are recorded." +runs = true +tests = ["qbit-prism-load::faults::faults_under_load_drain_an_offer_in_flight_and_outlast_a_settlement_lock_holder"] +presets = ["faults-long-real-node", "faults-pr-smoke", "faults-short-real-node"] + +[[scenario]] +id = "fault-slow-database" +title = "The frontends' database path delayed by the proxy under load" +owner = "#554" +lanes = ["nightly", "weekly"] +criteria = "The delay is shown on the frontends' URL (SELECT 1 at least twice the one-way delay); every answer stays truthful (no loss, no divergence); no frontend exits; the second half of the recovery window offers the full rate with ACK p99 within max(--ack-p99-limit-ms, twice the baseline's)." +runs = true +presets = ["faults-long-real-node", "faults-short-real-node"] + +[[scenario]] +id = "fault-connection-exhaustion" +title = "PostgreSQL at max_connections with the frontends' idle backends terminated" +owner = "#554" +lanes = ["nightly", "weekly"] +criteria = "PostgreSQL refuses a connection for want of a slot (53300) and at least one frontend backend is terminated into the full server; no loss and no divergence; no frontend exits; full rate and normal ACK p99 in the second half of the recovery window. The frontends' pool-acquire outcomes are recorded." +runs = true +presets = ["faults-long-real-node", "faults-short-real-node"] + +[[scenario]] +id = "fault-frontend-sigkill-holding-a-block" +title = "SIGKILL of the frontend holding a found block the node has accepted, before its answer is recorded" +owner = "#554" +lanes = ["nightly", "weekly"] +criteria = "The node accepted the block and the frontend is killed inside its 1 s submitblock deadline; its in-flight shares are censused and re-offered and credited once; the block reached the node exactly once and lands without a second offer. The time until it lands is reported, not gated: the dead holder's claim waits for the 120 s candidate lease, which is #529's option-A gap that #585 left open." +runs = true +presets = ["faults-long-real-node", "faults-short-real-node"] + +[[scenario]] +id = "fault-rolling-restart" +title = "SIGTERM and relaunch of each frontend in turn, its sessions moved to the one serving" +owner = "#554" +lanes = ["nightly", "weekly"] +criteria = "Each frontend exits 0 within 35 s, every moved session has work on the serving frontend within 30 s, no committed share goes unanswered, and the full rate returns after the rebalance." +runs = true +presets = ["faults-long-real-node", "faults-short-real-node"] + +[[scenario]] +id = "fault-reconnect-storm-under-load" +title = "Half the sessions drop abruptly and return within 5 s, on the real node" +owner = "#554" +lanes = ["nightly", "weekly"] +criteria = "Every stormed session has work again within 10 s of its return; the abandoned submits are the storm's, never a divergence; full rate afterwards." +runs = true +presets = ["faults-long-real-node", "faults-short-real-node"] +notes = "The fake-node rental-churn shapes (load-rental-churn) run nightly beside it." + +[[scenario]] +id = "read-tier-independence" +title = "The public API and every /metrics scraped at a realistic rate throughout the faults" +owner = "#554" +lanes = ["nightly", "weekly"] +criteria = "A separate public-api process on the standby, polled at 5 requests a second, answers at least 99% 2xx with p99 at most 1 s over each fault and the whole phase (excluding a PostgreSQL at max_connections only when the public API reads the primary, whose slots that takes); every live frontend's /metrics answers within 1 s (#189's class, #469)." +runs = true +presets = ["faults-long-real-node", "faults-short-real-node"] + +[[scenario]] +id = "fault-primary-kill-under-load" +title = "PostgreSQL primary lost under mining load with async promotion, and the fenced switch (#474 B and #521 scenario 3 at load)" +owner = "#554" +lanes = ["dispatch", "nightly", "weekly"] +criteria = "Under D3 (primary-kill): a barrier shows the standby flushed the primary's position, replication is cut and the primary keeps acknowledging, then it is stopped immediately and the standby promoted; every acknowledged share missing on the promoted primary is listed by id, none was acknowledged before the barrier, each is absent from what the standby had received (promotion kept all of that), and the cut left WAL the standby never received; every frontend accepts a share on the new primary within 30 s of the promotion without a restart; a new standby streams afterwards. The fenced switch (primary-switch: writers fenced, the standby replayed through the old primary's flush position) loses nothing." +runs = true +presets = ["faults-failover-fake-node", "faults-long-real-node", "faults-short-real-node"] +notes = "Every writer reaches the primary through a writer endpoint the fault moves after the promotion; the proven gap losses are excused from the run's durability finding, any other loss is not." + +[[scenario]] +id = "fault-block-mid-landing-failover-under-load" +title = "A found block mid-landing across an async failover, under load" +owner = "#554" +lanes = ["dispatch", "nightly", "weekly"] +criteria = "The block's submitblock is held at the relay while replication is cut and the primary stopped, then let through; with #529's standby wait on, its reservation is on the promoted primary; it reaches the node exactly once and lands within 30 s of the promotion with outcome unknown or accepted (#585); blocks forwarded around the failover whose rows the gap took are counted with the node's answer." +runs = true +presets = ["faults-failover-fake-node", "faults-long-real-node", "faults-short-real-node"] + +[[scenario]] +id = "fault-wal-disk-full-under-load" +title = "The WAL volume fills under load, then space is freed and PostgreSQL returns" +owner = "#554" +lanes = ["dispatch", "weekly"] +criteria = "PostgreSQL PANICs on the full WAL volume (#575's DiskFullInjector), which is filled again for the hold whenever its crash recovery finds room; no acknowledged share is lost and none is acknowledged inside the intervals PostgreSQL's own log shows it down; every frontend accepts shares again within 30 s without a restart; every frontend's /metrics shows #575's paging condition (qbit_prism_collector_available{collector=\"database\"} == 0) during the outage; a PostgreSQL that cannot be brought back ends the run with its log." +runs = true +presets = ["faults-failover-fake-node", "faults-long-real-node"] +notes = "Needs fuse2fs, fuse3 and /dev/fuse on the runner; the primary's pg_wal is on the volume for the whole run." + +[[scenario]] +id = "fault-restart-with-candidate-backlog" +title = "Both frontends restarted over a pending candidate backlog (#187's class)" +owner = "#554" +lanes = ["dispatch", "nightly", "weekly"] +criteria = "While the node answers submitblock with its warmup error (#526), found blocks build a backlog of unfinished candidates; every frontend exits 0 within 35 s and is relaunched; once the node heals, every backlog row reaches a terminal state (a lost race is orphaned once the fault mints the winner's confirmations), none reaches the node twice, one lands through a single offer, and every row counted before the restart is there after it." +runs = true +presets = ["faults-failover-fake-node", "faults-long-real-node", "faults-short-real-node"] + +[[scenario]] +id = "soak-weekly-one-lifetime" +title = "#575 item 2: a 5.5 h soak of one server lifetime looping the mainnet-shape, rental-churn and tip-delivery workloads" +owner = "#575" +lanes = ["weekly"] +criteria = "No frontend restart; RSS window-peak slope at most 16 MiB/h after an hour and within 2x the warm-up peak; descriptors at most 2/h; connections at most 24 per frontend with no drift; pg_wal at most 2 GiB; at least three partition rollovers and two partitions archived and dropped by share-archive under the load; zero shortfall, tip delivery within budget, and exact reconciliation including the restored archives." +runs = true +presets = ["soak-weekly"] + +[[scenario]] +id = "soak-short-leak-gate" +title = "#575 item 2: a 25-minute soak whose RSS gate a leaking server fails" +owner = "#575" +lanes = ["dispatch"] +criteria = "The preset's gates; a server built with the test-only soak-leak-mutant feature (16 KiB per accepted share) fails the RSS window-peak slope and the 2x warm-up-peak bound." +runs = true +presets = ["soak-short"] + +[[scenario]] +id = "soak-smoke-rollover-retention" +title = "#575 item 2: a four-minute soak smoke with forced rollovers and the full share-archive cycle under load" +owner = "#575" +lanes = ["nightly", "dispatch"] +criteria = "Two rollovers; two partitions sealed, archived, verified, detached and dropped under load and restored for an exact reconciliation; every sample readable; every tip delivered." +runs = true +tests = ["qbit-prism-load::soak_smoke::the_soak_smoke_preset_rolls_over_archives_and_holds_its_gates"] +presets = ["soak-smoke"] + +[[scenario]] +id = "soak-two-hours" +title = "Two hours at steady state with tips, own blocks and a fault every few minutes" +owner = "#556" +lanes = ["L5"] +criteria = "#291's criteria on #278's metric families: RSS bound, restart count, late wakes, lock-wait and ACK distributions, exact reconciliation." +runs = false +reason = "Needs S16's faults (#554); #521 scenario 5's 10-minute soak (r521-5-dense-cadence-soak) is its base." + +[[scenario]] +id = "migration-lifecycle" +title = "A seeded 2.x.x-schema database migrated, mined on, reconciled and rolled back" +owner = "#545" +lanes = ["weekly"] +criteria = "Set by #545: the drained source's evidence and backup, an isolated restore reconciling exactly before any native acknowledgement, migrate and import-audits rehearsed then run for the cutover, both native servers landing their own blocks on the real node with the history untouched, and the post-ACK boundary of the migration guide's rollback section. #291's rehearsal stays the authority on production-sized data." +runs = true +tests = ["qbit-prism-server::live_regtest::migration_lifecycle_tests::weekly_2x_ledger_migrates_mines_reconciles_and_restores_in_isolation"] + +[[scenario]] +id = "cutover-rehearsal" +title = "A mainnet-shaped 2.x.x ledger cut over as the migration guide runs it, measured" +owner = "#575" +lanes = ["pr", "weekly"] +criteria = "Recovery evidence, every 2.x.x table's counts and sums, every address's balances (SQL and the running API) and the payout window are unchanged; a frontend on the result is ready, issues a first job and passes self-check; each step's wall time and longest lock are reported." +runs = true +tests = [ + "qbit-prism-server::migration_rollback::cutover_rehearsal_tests::mainnet_shaped_2x_dump_rehearses_the_cutover_and_keeps_every_balance", + "qbit-prism-server::live_regtest::migration_lifecycle_tests::weekly_mainnet_shaped_2x_ledger_cuts_over_measured_mines_and_restores_in_isolation", +] +notes = "The PR case rehearses a pg_dump of a PR-sized ledger. The weekly live_regtest variant cuts over 1x at 1/16 of the rows (about 4.1M) on a real regtest chain, at the server's default statement timeout, and `make prism-cutover-rehearsal` runs the same checks on an operator's snapshot." + +[[scenario]] +id = "external-target-load-through-a-balancer" +title = "#291: the load harness's client driving frontends it did not launch, through a TCP balancer" +owner = "#291" +lanes = ["pr"] +criteria = "Two external-target client processes drive two in-repo frontends through a round-robin TCP balancer, one asking for its own share difficulty with d=, and each mines at the difficulty the frontend advertised. A frontend SIGKILLed and relaunched mid-run costs reconnects, not acknowledgements: every disconnected session reconnects, no share is refused as mined wrong, every acknowledged share id is committed, every committed id was acknowledged or went unanswered, and the two processes' stats merge into their sums." +runs = true +tests = ["qbit-prism-load::external_frontend::external_load_through_a_balancer_reaches_the_in_repo_frontends_and_reconciles"] +unit_tests = [ + "crates/qbit-prism-load/tests/external_target.rs::a_run_mines_the_advertised_difficulty_and_counts_what_the_target_did", + "crates/qbit-prism-load/tests/external_target.rs::documents_from_two_processes_merge_into_their_sum", + "crates/qbit-prism-load/tests/external_target.rs::without_the_guard_flag_nothing_is_sent", +] +notes = "The fixture for #291's failover drill, load-balancer exercise and soak, which run `qbit-prism-load external` against the deployed pair behind the operator's balancer; those runs, and their reconciliation against the promoted primary, stay #291's." + +[[scenario]] +id = "shipped-image-lane" +title = "The Compose prism profile with the HA overlay under modest real-miner load" +owner = "#544" +lanes = ["L6"] +criteria = "Built by `docker compose build`, the stack runs both overlay frontends ready with distinct IDs that each one's self-check observes; cpuminer-opt on one frontend and qbit-prism-miner on the other each solve a block the node keeps, whose audit bundle both frontends serve identically and which verifies against the node's coinbase and the ledger key; each client's ledger rows equal the shares it saw acknowledged, with every submission answered; the public API lists the verified blocks; and #281 criterion 1 holds on the images." +runs = true +lane_checks = [ + "ha-frontends-ready", + "cross-frontend-resume", + "cpuminer-found-block", + "prism-miner-found-block", + "found-block-audit-verifies", + "ledger-reconciles", + "public-api-lists-found-blocks", +] +notes = "scripts/prism_shipped_image_lane.py; the report, client logs and Compose logs are the `prism-shipped-images` artifact." diff --git a/test/prism-cutover-rehearsal.sh b/test/prism-cutover-rehearsal.sh new file mode 100755 index 000000000..aa3e5fc8b --- /dev/null +++ b/test/prism-cutover-rehearsal.sh @@ -0,0 +1,131 @@ +#!/usr/bin/env bash +# The PRISM 2.x.x -> 3.x.x cutover rehearsal (#575), for operators. +# +# test/prism-cutover-rehearsal.sh rehearse rehearse the cutover on a pg_dump +# test/prism-cutover-rehearsal.sh dump write a mainnet-shaped 2.x.x dump +# test/prism-cutover-rehearsal.sh generated both: write a mainnet-shaped dump +# under WORKDIR and rehearse it +# +# rehearse restores DUMP (custom, directory, tar or plain format; a plain dump +# must be taken with --no-owner --no-privileges) into a +# private PostgreSQL cluster it creates under WORKDIR and removes afterwards, +# runs check-config, migrate and import-audits exactly as +# docs/prism-rust-migration.md does, checks the recovery evidence, balances, +# payout window and table sums against the source, starts one frontend +# (self-check, API balances, first Stratum job) and prints a PASS/FAIL report. +# It connects to no database but the one it creates. +# +# DUMP=<file> required: the pg_dump to rehearse +# LEDGER_KEY=<hex> required: the trusted PRISM_LEDGER_WRITER_PUBLIC_KEY_HEX +# AUDIT_ROOT=<dir> external audit bodies and canonical sidecars, mounted +# at their recorded paths (default: none) +# SCHEMA=<name> the ledger schema (default: the one holding qbit_share_ledger) +# ENV_FILE=<file> the reviewed production environment the real cutover +# uses (NAME=value lines): its synced node, chain, +# genesis pin, signing keys and pool fee. Only the +# database URL, listen ports and instance ID are +# replaced. Default: a lab configuration with test +# keys and a node serving the dump's own chain +# STATEMENT_TIMEOUT_MS=<ms> the PRISM_DATABASE_STATEMENT_TIMEOUT_MS the real +# migrate will run with (default: the server's 15000) +# REPORT=<file> also write the report as JSON +# WORKDIR=<dir> where the private cluster lives (default: TMPDIR) +# PG_BIN_DIR=<dir> PostgreSQL 16 server binaries (default: pg_config --bindir) +# +# dump writes OUT (custom format) and OUT's audit bodies to OUT.audits: +# +# OUT=<file> required +# SCALE=<n> 1 (mainnet 2.x.x, Jul 15 - Sep 28 2026) or more +# DENSITY=<fraction> share rows kept, each carrying 1/DENSITY of the +# difficulty (default 0.0625) +# +# generated takes SCALE, DENSITY, STATEMENT_TIMEOUT_MS, REPORT and WORKDIR +# (default: a new directory under TMPDIR, removed afterwards unless KEEP=1), +# writes the dump there and rehearses it with the generator's own ledger key: +# one command to rerun the rehearsal on any branch. +set -euo pipefail + +PRISM_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +cd "${PRISM_ROOT}" +mode="${1:-}" + +fail() { + echo "prism-cutover-rehearsal: $*" >&2 + exit 2 +} + +pg_bin="${PG_BIN_DIR:-}" +if [[ -z "${pg_bin}" ]] && command -v pg_config >/dev/null; then + pg_bin="$(pg_config --bindir)" +fi +[[ -n "${pg_bin}" && -x "${pg_bin}/initdb" ]] || fail "set PG_BIN_DIR to the PostgreSQL 16 server binaries (initdb not found)" + +# Only this run's own variables reach the test: nothing inherited selects +# another database. +export PRISM_TEST_PG_BIN_DIR="${pg_bin}" +export PRISM_TEST_REQUIRE_INTEGRATION=1 +unset PRISM_TEST_DATABASE_URL + +export_optional() { + local name="$1" value="$2" + if [[ -n "${value}" ]]; then + export "${name}=${value}" + else + unset "${name}" + fi +} + +# The ledger key the generator signs its history with (seed 22...22). +GENERATED_LEDGER_KEY=a09aa5f47a6759802ff955f8dc2d2a14a5c99d23be97f864127ff9383455a4f0 + +if [[ "${mode}" == generated ]]; then + workdir="${WORKDIR:+$(realpath -m "${WORKDIR}")}" + if [[ -z "${workdir}" ]]; then + workdir="$(mktemp -d "${TMPDIR:-/tmp}/prism-cutover-rehearsal.XXXXXX")" + if [[ "${KEEP:-0}" != 1 ]]; then + trap 'rm -rf -- "${workdir:?}"' EXIT + fi + fi + mkdir -p "${workdir}" + OUT="${workdir}/generated.dump" bash "$0" dump + DUMP="${workdir}/generated.dump" AUDIT_ROOT="${workdir}/generated.audits" \ + LEDGER_KEY="${GENERATED_LEDGER_KEY}" WORKDIR="${workdir}" bash "$0" rehearse + exit +fi + +case "${mode}" in + rehearse) + [[ -n "${DUMP:-}" ]] || fail "set DUMP to the pg_dump to rehearse" + [[ -e "${DUMP}" ]] || fail "DUMP ${DUMP} does not exist" + [[ -n "${LEDGER_KEY:-}" ]] || fail "set LEDGER_KEY to the trusted ledger public key (64 hex digits)" + PRISM_REHEARSAL_DUMP="$(realpath "${DUMP}")" + export PRISM_REHEARSAL_DUMP + export PRISM_REHEARSAL_LEDGER_PUBLIC_KEY_HEX="${LEDGER_KEY}" + # cargo runs the test from the crate directory: every path is absolute. + export_optional PRISM_REHEARSAL_AUDIT_ROOT "${AUDIT_ROOT:+$(realpath -m "${AUDIT_ROOT}")}" + export_optional PRISM_REHEARSAL_SCHEMA "${SCHEMA:-}" + export_optional PRISM_REHEARSAL_ENV_FILE "${ENV_FILE:+$(realpath -m "${ENV_FILE}")}" + report_path="" + if [[ -n "${REPORT:-}" ]]; then + report_path="$(realpath -m "${REPORT}")" + fi + export_optional PRISM_REHEARSAL_REPORT "${report_path}" + export_optional PRISM_REHEARSAL_WORKDIR "${WORKDIR:+$(realpath -m "${WORKDIR}")}" + export_optional PRISM_REHEARSAL_STATEMENT_TIMEOUT_MS "${STATEMENT_TIMEOUT_MS:-}" + test_name=cutover_rehearsal_tests::operator_dump_rehearsal + ;; + dump) + [[ -n "${OUT:-}" ]] || fail "set OUT to the dump file to write" + PRISM_REHEARSAL_GENERATE_DUMP="$(realpath -m "${OUT}")" + export PRISM_REHEARSAL_GENERATE_DUMP + export_optional PRISM_REHEARSAL_SCALE "${SCALE:-}" + export_optional PRISM_REHEARSAL_DENSITY "${DENSITY:-}" + test_name=cutover_rehearsal_tests::generate_mainnet_shaped_dump + ;; + *) + fail "usage: test/prism-cutover-rehearsal.sh rehearse|dump|generated (see the header)" + ;; +esac + +cargo test --locked --release -p qbit-prism-server --test migration_rollback -- \ + --ignored --exact --nocapture --test-threads=1 "${test_name}" diff --git a/test/prism-gated-tests.txt b/test/prism-gated-tests.txt new file mode 100644 index 000000000..83342491e --- /dev/null +++ b/test/prism-gated-tests.txt @@ -0,0 +1,932 @@ +# Every gated test the prism-native-postgres CI job must execute, one +# `<package>::<binary>::<test path>` id per line, sorted. Its length is the +# execution manifest with this list; a gated test that is added, renamed or +# minimum executed count. scripts/check_gate_manifest.py compares the gate's +# removed must be changed here too. See docs/prism-integration-test-gate.md. +# A live_regtest or load_smoke test is an end-to-end scenario: name it in +# test/e2e-scenarios.toml too (scripts/check_e2e_scenarios.py, #543). +qbit-prism-load::ctv_settlement::a_ctv_run_pays_the_direct_overflow_through_fanout_and_counts_the_split +qbit-prism-load::external_frontend::external_load_through_a_balancer_reaches_the_in_repo_frontends_and_reconciles +qbit-prism-load::faults::faults_under_load_drain_an_offer_in_flight_and_outlast_a_settlement_lock_holder +qbit-prism-load::harness::a_managed_cluster_that_fails_to_start_reports_the_servers_own_reason +qbit-prism-load::load_smoke::the_smoke_preset_serves_every_session_every_tip_and_reconciles +qbit-prism-load::quorum_replication::an_any_1_quorum_standby_on_postgres_16_is_observed_as_sync_like_first_1 +qbit-prism-load::real_node::real_node_smoke_reconciles_against_postgres_and_the_chain +qbit-prism-load::soak_sampling::soak_samples_a_prism_database_and_a_process_rolls_over_and_gates +qbit-prism-server::api_2x::block_views_markers_and_network_estimate_keep_2x_contracts +qbit-prism-server::api_2x::chart_rollups_match_raw_boundaries_tail_and_missing_progress +qbit-prism-server::api_2x::chart_rollups_match_raw_for_every_range_bucket_and_watermark +qbit-prism-server::api_2x::chart_rollups_read_only_the_partial_buckets_under_any_plan +qbit-prism-server::api_2x::content_addressed_bytes_and_legacy_fallback_headers_are_exact +qbit-prism-server::api_2x::fixture_migrations_do_not_share_advisory_locks +qbit-prism-server::api_2x::public_service_is_read_only_and_bounds_http_database_work +qbit-prism-server::api_database::accepted_public_blocks_and_earnings_follow_confirmed_chain_state +qbit-prism-server::api_database::shared_database_serves_all_contracts_and_global_reward_ranks +qbit-prism-server::append_admission::a_fenced_append_rereads_its_probe_after_waiting_on_a_cluster_row_writer +qbit-prism-server::append_admission::a_fenced_append_waits_out_a_revision_writer_and_compares_what_it_commits +qbit-prism-server::append_admission::a_halted_cluster_or_a_live_legacy_writer_refuses_the_append_before_its_fence +qbit-prism-server::append_admission::a_share_append_holds_order_lock_for_three_statements_and_its_commit +qbit-prism-server::append_admission::a_share_from_a_future_job_is_refused_and_its_write_rolls_back +qbit-prism-server::append_admission::append_dropped_outside_runtime_context_keeps_admission_until_cleanup +qbit-prism-server::append_admission::cancelled_transaction_keeps_admission_until_connection_cleanup +qbit-prism-server::append_admission::frontend_pool_keeps_read_headroom +qbit-prism-server::append_admission::min_connections_pool_dropped_outside_runtime_panics_once_like_sqlx_and_still_cleans_up +qbit-prism-server::append_admission::minimum_pool_keeps_read_headroom +qbit-prism-server::append_admission::missing_partition_retry_reacquires_admission_and_preserves_gate_deadline +qbit-prism-server::append_admission::queued_appends_recheck_revision_and_commit_gate +qbit-prism-server::append_admission::separate_pools_have_separate_admission_and_closing_wakes_queued_appends +qbit-prism-server::append_admission::shared_pool_queue_uses_original_deadline_and_cancels_without_sql +qbit-prism-server::append_admission::uncertain_commit_holds_admission_and_durable_replay_is_not_recredited +qbit-prism-server::artifact_admission::a_refused_artifact_is_never_cached_nor_shared_as_a_success +qbit-prism-server::artifact_admission::an_over_cap_artifact_request_is_refused_before_any_audit_read +qbit-prism-server::artifact_admission::artifact_permits_outlive_cancellation_and_survive_failure +qbit-prism-server::artifact_admission::artifact_route_load_keeps_pool_summary_within_its_read_deadline +qbit-prism-server::artifact_admission::rebuilds_are_bounded_independently_of_the_read_concurrency +qbit-prism-server::artifact_admission::sealed_and_unsealed_artifacts_are_byte_identical_under_admission +qbit-prism-server::audit_body_normalization::acquire_metrics::direct_callers_count_checkout_outcomes_and_recover +qbit-prism-server::audit_body_normalization::acquire_metrics::landing_counts_each_page_and_releases_before_blocking_work +qbit-prism-server::audit_body_normalization::acquire_metrics::legacy_import_pages_release_before_writes_and_audit_decode +qbit-prism-server::audit_body_normalization::audit_durable_range::durable_range_proof_accepts_partial_history_and_ignores_ineligible_endpoints +qbit-prism-server::audit_body_normalization::audit_durable_range::durable_range_proof_checks_bootstrap_against_the_anchored_ledger +qbit-prism-server::audit_body_normalization::audit_durable_range::durable_range_proof_refuses_a_window_that_differs_from_ledger_history_on_any_page +qbit-prism-server::audit_body_normalization::audit_durable_range::durable_range_proof_refuses_newest_truncation_of_a_multi_page_window +qbit-prism-server::audit_body_normalization::audit_durable_range::durable_range_proof_refuses_oldest_truncation_of_a_multi_page_window +qbit-prism-server::audit_body_normalization::audit_durable_range::durable_range_proof_retains_the_crossing_share_and_refuses_an_extra_prefix +qbit-prism-server::audit_body_normalization::imported_rows_serve_canonical_bytes_and_never_enter_native_reconstruction +qbit-prism-server::audit_body_normalization::landed_native_body_stores_no_share_copy_and_serves_identical_bytes +qbit-prism-server::audit_body_normalization::legacy_native_rows_with_manifest_shares_still_serve_identical_bytes +qbit-prism-server::audit_body_normalization::reconstruction_metrics::cancellation_distinguishes_pending_checkout_from_each_running_query +qbit-prism-server::audit_body_normalization::reconstruction_metrics::missing_data_and_decode_or_sql_errors_keep_successful_checkouts +qbit-prism-server::audit_body_normalization::reconstruction_metrics::reconstruction_releases_sole_connection_before_blocking_and_without_metrics +qbit-prism-server::audit_body_normalization::reconstruction_metrics::representations_count_only_their_fresh_checkouts_and_preserve_bytes +qbit-prism-server::audit_completeness::counts_distinguish_legacy_imports_and_reconstructible_native_bodies +qbit-prism-server::audit_completeness::import_cli_reports_remaining_audits_on_success_and_failure +qbit-prism-server::audit_completeness::self_check_reports_missing_audits_before_rpc_and_enforces_production +qbit-prism-server::audit_reconstruction_permit::a_queued_reconstruction_keeps_its_permit_after_its_caller_is_dropped +qbit-prism-server::audit_reconstruction_permit::a_second_reconstruction_waits_for_the_permit_and_both_rows_hydrate +qbit-prism-server::audit_reconstruction_permit::failed_and_cancelled_reconstructions_release_the_permit +qbit-prism-server::authorization_rpc::authorization_fallback_requires_a_definitive_address_result +qbit-prism-server::b275_persistence_measure::a_held_ledger_clock_update_does_not_block_persistence +qbit-prism-server::b275_persistence_measure::original_revision_fence_survives_lock_wait +qbit-prism-server::b275_persistence_measure::persistence_and_exact_rows_precede_three_second_stub_release +qbit-prism-server::b275_persistence_measure::retried_delivery_is_not_reported_as_failure_free +qbit-prism-server::b275_persistence_measure::small_public_delivery_measurement +qbit-prism-server::b275_persistence_measure::three_second_authority_row_hold_still_blocks_persistence +qbit-prism-server::b478_capture_overpay::a_bump_concurrent_with_the_reservation_is_bounded_not_bypassed +qbit-prism-server::b478_capture_overpay::a_capture_after_the_settlement_bump_lands_within_its_recorded_bound +qbit-prism-server::b478_capture_overpay::a_capture_over_the_ceiling_is_abandoned_unoffered_and_counted +qbit-prism-server::b478_capture_overpay::a_captured_candidate_whose_parent_changes_is_abandoned_unoffered +qbit-prism-server::b478_capture_overpay::a_zero_ceiling_turns_capture_off_at_submit_and_at_offer +qbit-prism-server::b478_capture_overpay::an_offer_before_the_settlement_bump_lands_divergent_and_is_recorded +qbit-prism-server::b478_capture_overpay::an_unknown_bound_abandons_nothing +qbit-prism-server::b478_capture_overpay::consecutive_base_order_landings_compound_and_every_sat_is_recorded +qbit-prism-server::b478_capture_overpay::consecutive_divergent_landings_compound_and_every_sat_is_recorded +qbit-prism-server::b478_capture_overpay::the_divergence_line_shows_the_live_positive_float +qbit-prism-server::b478_capture_overpay::two_consecutive_captures_each_land_within_their_bound +qbit-prism-server::b478_payout_integrity::a_capture_off_abandonment_is_recorded_as_disabled +qbit-prism-server::b478_payout_integrity::debt_realized_at_confirmation_is_recorded_when_landings_interleave +qbit-prism-server::b478_stale_revision_block::a_block_on_a_same_parent_revision_bumped_job_still_reaches_the_node +qbit-prism-server::b478_stale_revision_block::block_only_work_lands_its_block_but_earns_no_share_credit +qbit-prism-server::b574_ack_cap::a_block_carrying_append_whose_commit_is_in_flight_gets_the_share_grace +qbit-prism-server::b574_ack_cap::a_block_carrying_share_append_is_answered_at_the_cap_and_credited_once +qbit-prism-server::b574_ack_cap::a_captured_proof_is_answered_at_the_cap_and_credited_after_its_landing +qbit-prism-server::b574_ack_cap::a_captured_proof_whose_probe_outlasts_the_cap_is_still_enqueued_and_credited +qbit-prism-server::b602_order_lock_hold::a_landing_records_each_order_lock_hold_under_its_holder +qbit-prism-server::b602_order_lock_hold::acknowledgements_after_an_acceptance_are_read_in_the_landing_window +qbit-prism-server::b602_order_lock_hold::appends_commit_within_budget_while_a_settlement_is_held +qbit-prism-server::b619_gap_work_after_promotion::a_block_on_a_job_issued_in_the_gap_is_refused_stale_job_before_its_offer +qbit-prism-server::b619_gap_work_after_promotion::a_candidate_accepted_before_the_promotion_cannot_be_published_after_it +qbit-prism-server::b619_gap_work_after_promotion::a_fenced_switchover_with_full_replay_keeps_pre_switch_work_and_lands_its_block +qbit-prism-server::b619_gap_work_after_promotion::a_job_persisted_before_the_cut_is_resumed_after_an_async_promotion_and_lands +qbit-prism-server::b619_gap_work_after_promotion::a_refusal_that_finishes_after_the_acknowledgement_deadline_is_still_counted_once +qbit-prism-server::b619_gap_work_after_promotion::work_prepared_in_the_gap_is_not_issued_to_a_new_session_after_an_async_promotion +qbit-prism-server::candidate_dispatch_probe::canonical_probe_errors_remain_failures_and_pool_recovers +qbit-prism-server::candidate_dispatch_probe::canonical_probe_handles_churn_dead_entries_and_stale_statistics +qbit-prism-server::candidate_dispatch_probe::canonical_probe_plans_cover_exact_unfinished_and_retained_populations +qbit-prism-server::candidate_dispatch_probe::canonical_probe_preserves_slots_clocks_and_candidate_ownership +qbit-prism-server::candidate_storm::admission_serves_new_work_first_while_the_due_backlog_is_still_owed +qbit-prism-server::candidate_storm::claim_lanes_and_the_dispatch_probe_keep_their_indexes_over_unfinished_due_rows +qbit-prism-server::candidate_storm::drained_rows_cost_the_same_statements_at_the_baseline_and_storm_cardinalities +qbit-prism-server::candidate_storm::siblings_behind_a_decided_height_drain_once_each_at_a_constant_node_cost +qbit-prism-server::candidate_storm_restart::a_frontend_with_block_submission_disabled_sends_its_node_no_block_until_it_is_enabled +qbit-prism-server::candidate_storm_restart::a_held_candidate_lease_costs_the_drain_an_offer_and_never_a_wakeup +qbit-prism-server::candidate_storm_restart::a_held_ledger_holds_a_frontend_started_without_the_switch_until_it_is_cleared +qbit-prism-server::candidate_storm_restart::a_kill_before_the_first_offer_recovers_as_one_known_offer +qbit-prism-server::candidate_storm_restart::a_relaunched_frontend_recovers_one_row_at_the_same_statement_cost_at_both_cardinalities +qbit-prism-server::candidate_storm_restart::restart_between_claim_and_terminalization_offers_every_block_exactly_once +qbit-prism-server::candidate_window_migration::balance_snapshot_writer_is_canonical_idempotent_and_conflict_checked +qbit-prism-server::candidate_window_migration::configure_retains_the_pinned_cluster_fingerprint +qbit-prism-server::candidate_window_migration::legacy_2x_schema_with_terminal_outbox_rows_gains_007_and_keeps_every_row +qbit-prism-server::candidate_window_migration::migration_007_alone_is_applied_on_a_database_at_2_3_4_5_6_8_9_10 +qbit-prism-server::candidate_window_migration::migration_007_refuses_every_pre007_pending_shape_and_applies_nothing +qbit-prism-server::candidate_window_migration::outbox_window_check_accepts_exactly_the_three_states_and_indexes_the_digest +qbit-prism-server::candidate_window_migration::paged_reader_reports_a_failed_blocking_handoff_as_task_failed +qbit-prism-server::candidate_window_migration::read_window_permit_outlives_a_cancelled_page_and_is_released_off_the_runtime +qbit-prism-server::candidate_window_migration::share_probe_and_paged_reader_serve_a_callers_transaction +qbit-prism-server::candidate_window_migration::window_reference_from_snapshot_matches_the_landed_audit_snapshot_digest +qbit-prism-server::candidate_window_qualification::a_block_found_on_a_delta_built_window_after_a_retarget_lands +qbit-prism-server::candidate_window_qualification::incident_2_ack_latency_stays_flat_while_a_frontend_solves_a_large_window +qbit-prism-server::candidate_window_qualification::order_lock_hold_on_a_solve_does_not_grow_with_the_window +qbit-prism-server::candidate_window_switch::a_reissued_last_row_stamped_before_the_anchor_by_a_lagging_clock_passes_the_time_test +qbit-prism-server::candidate_window_switch::a_stored_snapshot_that_is_not_the_canonical_encoding_refuses_both_write_paths_atomically +qbit-prism-server::candidate_window_switch::as_issued_snapshot_preparation_runs_off_the_runtime_on_both_write_paths +qbit-prism-server::candidate_window_switch::candidate_preparation_never_waits_for_the_order_lock +qbit-prism-server::candidate_window_switch::claim_refuses_every_row_that_disagrees_with_its_document +qbit-prism-server::candidate_window_switch::configure_refuses_a_rotation_over_pending_foreign_signed_candidates +qbit-prism-server::candidate_window_switch::enqueue_alerts_but_publishes_when_the_window_prefix_row_is_missing +qbit-prism-server::candidate_window_switch::enqueue_refuses_a_fingerprint_the_writer_did_not_pin +qbit-prism-server::candidate_window_switch::enqueue_refuses_a_window_whose_last_row_this_primary_does_not_hold +qbit-prism-server::candidate_window_switch::enqueue_stores_a_reference_row_beside_the_block_bytes +qbit-prism-server::candidate_window_switch::landed_audit_is_authenticated_against_the_block +qbit-prism-server::candidate_window_switch::landing_refuses_parts_that_are_not_the_referenced_window +qbit-prism-server::candidate_window_switch::leased_enqueue_reinserts_the_as_issued_balance_snapshot +qbit-prism-server::candidate_window_switch::terminal_update_nulls_the_window_columns_and_block_bytes +qbit-prism-server::candidate_window_switch::the_durable_range_proof_runs_before_the_settlement_lock +qbit-prism-server::chain_epoch_migration::cancelled_epoch_migrator_rolls_back_before_commit +qbit-prism-server::chain_epoch_migration::epoch_metadata_refuses_missing_declarations_and_undeclared_collisions +qbit-prism-server::chain_epoch_migration::epoch_updates_are_atomic_and_accounting_does_not_advance_them +qbit-prism-server::chain_epoch_migration::epoch_upgrade_collision_and_rollback_leave_no_partial_metadata +qbit-prism-server::chain_epoch_migration::epoch_upgrade_requires_shutdown_preserves_checkpoint_and_survives_concurrent_restart +qbit-prism-server::checkout_probe_delivery::a_build_on_a_connection_killed_inside_the_probe_window_fails_once_then_serves_the_same_tip +qbit-prism-server::cluster_row_fences::a_key_share_fence_lets_the_share_clock_through_and_still_blocks_authority_writers +qbit-prism-server::compact_issued_batch::a_held_cohort_fence_does_not_hold_up_a_share_append +qbit-prism-server::compact_issued_batch::atomic_64_cohort_is_durable_while_three_second_settlement_stub_remains_held +qbit-prism-server::compact_issued_batch::canceled_insert_releases_pool_and_lost_commit_ack_reconciles_exact_identity +qbit-prism-server::compact_issued_batch::chain_transition_waits_for_issuance_then_fences_stale_revision_and_aba_epoch +qbit-prism-server::compact_issued_batch::conflicting_ids_and_late_insert_failure_roll_back_whole_batch_and_renewal +qbit-prism-server::compact_issued_batch::earliest_child_expiry_survives_cluster_dependency_blob_and_insert_waits +qbit-prism-server::compact_issued_batch::missing_and_corrupt_dependencies_and_batch_boundaries_never_write_children +qbit-prism-server::compact_issued_batch::one_transaction_preserves_exact_children_max_retention_and_idempotence +qbit-prism-server::compact_issued_batch::ordinary_authority_updates_wait_for_atomic_64_child_commit +qbit-prism-server::compact_issued_batch::original_deadline_includes_pool_wait_and_preserves_stricter_statement_limits +qbit-prism-server::compact_issued_batch::revision_configuration_and_fatal_state_revalidate_after_cluster_wait +qbit-prism-server::compact_issued_batch::two_frontends_overlap_exact_children_without_renewal_loss_or_partial_conflicts +qbit-prism-server::compact_issued_repair::blob_cleanup::blob_failure_preserves_committed_expiry_and_rolls_back_blobs_without_cursor_progress +qbit-prism-server::compact_issued_repair::blob_cleanup::bounded_pages_pass_live_prefixes_and_wrap_with_zero_expired_jobs +qbit-prism-server::compact_issued_repair::blob_cleanup::candidate_claim_expiry_retains_balances_until_terminal_outcome +qbit-prism-server::compact_issued_repair::blob_cleanup::cluster_fence_wait_uses_remaining_gc_deadline_and_preserves_cursor +qbit-prism-server::compact_issued_repair::blob_cleanup::collector_waits_for_repair_and_candidate_writers_in_lock_order +qbit-prism-server::compact_issued_repair::blob_cleanup::expiry_selection_rechecks_renewal_after_its_row_lock_wait +qbit-prism-server::compact_issued_repair::blob_cleanup::gc_before_repair_restores_exact_original_blobs_and_identity +qbit-prism-server::compact_issued_repair::blob_cleanup::halted_cluster_stops_blob_cleanup_but_expiry_still_commits +qbit-prism-server::compact_issued_repair::blob_cleanup::one_deadline_covers_pool_and_both_advisory_waits +qbit-prism-server::compact_issued_repair::blob_cleanup::repair_before_gc_retains_blobs_through_the_child_deadline +qbit-prism-server::compact_issued_repair::blob_cleanup::slow_expiry_keeps_advisory_locks_free_for_share_ack_and_candidate_enqueue +qbit-prism-server::compact_issued_repair::blob_cleanup::stricter_database_timeout_survives_failure_and_success +qbit-prism-server::compact_issued_repair::blob_cleanup::surviving_orphan_repair_and_gc_fence_both_orders_before_reference_scan +qbit-prism-server::compact_issued_repair::cancelled_or_failed_inserts_roll_back_record_blobs_and_child +qbit-prism-server::compact_issued_repair::child_deadline_covers_cluster_dependency_and_post_insert_waits +qbit-prism-server::compact_issued_repair::concurrent_repairs_publish_one_dependency_identity_and_fixed_children +qbit-prism-server::compact_issued_repair::conflicting_child_survivor_rolls_back_complete_dependency_restoration +qbit-prism-server::compact_issued_repair::conflicting_survivors_leave_no_partial_repair +qbit-prism-server::compact_issued_repair::current_revision_configuration_and_writable_fences_recheck_after_row_waits +qbit-prism-server::compact_issued_repair::direct_prepared_and_single_persistence_finish_before_settlement_stub_release +qbit-prism-server::compact_issued_repair::elapsed_original_is_identity_only_while_the_child_deadline_stays_live +qbit-prism-server::compact_issued_repair::hot_metadata_rejects_payload_column_and_original_identity_mismatches +qbit-prism-server::compact_issued_repair::hot_save_and_exact_retry_preserve_originals_and_child_deadlines +qbit-prism-server::compact_issued_repair::invalid_repair_inputs_and_child_boundaries_write_nothing +qbit-prism-server::compact_issued_repair::large_template_and_canonical_balances_match_the_existing_encoding_exactly +qbit-prism-server::compact_issued_repair::legacy_inline_calls_and_cross_kind_errors_keep_their_existing_contract +qbit-prism-server::compact_issued_repair::missing_record_restores_exact_inputs_with_surviving_or_missing_blobs +qbit-prism-server::compact_issued_repair::ordinary_authority_updates_wait_for_cold_repair_commit +qbit-prism-server::compact_issued_repair::surviving_record_never_repairs_missing_blobs_and_rejects_known_corruption +qbit-prism-server::compact_prepared_storage::cancellation_after_blob_inserts_rolls_back_all_effects +qbit-prism-server::compact_prepared_storage::compact_reader_and_retry_reject_payload_column_disagreement +qbit-prism-server::compact_prepared_storage::concurrent_retries_create_one_immutable_dependency +qbit-prism-server::compact_prepared_storage::configured_writer_refuses_a_reset_or_rotated_cluster_pin +qbit-prism-server::compact_prepared_storage::empty_and_nonempty_prepared_records_round_trip_original_inputs_and_exact_blobs +qbit-prism-server::compact_prepared_storage::fixed_expiry_is_checked_after_lock_and_post_insert_waits +qbit-prism-server::compact_prepared_storage::immutable_blob_conflicts_and_wrong_balance_digest_roll_back_atomically +qbit-prism-server::compact_prepared_storage::invalid_references_and_oversize_metadata_publish_nothing +qbit-prism-server::compact_prepared_storage::large_template_stays_out_of_compact_payload +qbit-prism-server::compact_prepared_storage::missing_or_corrupt_blobs_are_errors_not_misses +qbit-prism-server::compact_prepared_storage::old_inline_callers_remain_unchanged_and_only_explicit_legacy_rows_miss +qbit-prism-server::compact_prepared_storage::prepared_authority_is_fenced_before_checks_in_both_update_orders +qbit-prism-server::compact_prepared_storage::retries_preserve_original_identity_expiry_and_transaction_fence +qbit-prism-server::compact_runtime_e2e::a_database_stall_refuses_a_resume_as_the_databases +qbit-prism-server::compact_runtime_e2e::bootstrap_resume_keeps_each_workers_original_payout +qbit-prism-server::compact_runtime_e2e::cancelled_issued_save_releases_sql_resources_without_publishing +qbit-prism-server::compact_runtime_e2e::compact_reference_corruption_is_an_error_and_blobs_survive_collection +qbit-prism-server::compact_runtime_e2e::delayed_old_refresh_cannot_replace_new_tip_publication_or_resume_retired_work +qbit-prism-server::compact_runtime_e2e::issued_expiry_during_sql_wait_does_not_publish_or_renew_original_identity +qbit-prism-server::compact_runtime_e2e::malformed_issued_inputs_are_errors_and_missing_work_is_a_miss +qbit-prism-server::compact_runtime_e2e::missing_prepared_dependency_repairs_original_record_without_renewing_identity +qbit-prism-server::compact_runtime_e2e::real_socket_reconnect_resumes_original_entropy_mask_and_submits_once +qbit-prism-server::compact_runtime_e2e::refresh_issue_resume_preserves_original_work_and_compact_storage +qbit-prism-server::compact_runtime_e2e::resume_expiry_does_not_slide_and_expired_work_is_a_miss +qbit-prism-server::compact_runtime_e2e::resume_expiry_includes_blocked_share_read_and_releases_resources +qbit-prism-server::compact_runtime_e2e::resume_requires_the_frontends_published_revision_before_reconstruction +qbit-prism-server::compact_runtime_e2e::resumed_compact_work_authenticates_retained_share_rows +qbit-prism-server::compact_runtime_e2e::superseding_publication_during_completed_commit_wait_refuses_old_delivery +qbit-prism-server::compact_runtime_e2e::unchanged_refresh_during_completed_commit_wait_preserves_original_authority +qbit-prism-server::compact_runtime_e2e::unknown_issued_commit_is_observed_and_reconciled_without_reissuing +qbit-prism-server::compact_runtime_observer::a_broken_write_observer_never_recovers_as_a_healthy_zero +qbit-prism-server::compact_runtime_observer::completed_commit_pause_releases_database_locks_before_reply_delivery +qbit-prism-server::compact_runtime_observer::coordinator_refresh_is_observed_with_insert_lsn_and_real_jsonb_writes +qbit-prism-server::compact_runtime_observer::jsonb_probe_counts_unchanged_and_intermediate_row_writes +qbit-prism-server::compact_runtime_observer::observer_keeps_failed_and_unavailable_operations_distinct_from_zero +qbit-prism-server::compact_runtime_observer::row_observer_distinguishes_real_window_pages_from_probes_and_metadata +qbit-prism-server::compact_runtime_scale::actual_400k_refresh_and_cross_frontend_resume +qbit-prism-server::compact_runtime_scale::actual_500k_refresh_and_cross_frontend_resume +qbit-prism-server::compact_runtime_scale::actual_refresh_and_shared_cross_frontend_resume +qbit-prism-server::ctv_fanout_bookkeeping::a_cluster_hold_claims_and_sends_no_fanout_until_it_is_cleared +qbit-prism-server::ctv_fanout_bookkeeping::a_failed_attempt_whose_failure_is_lost_keeps_its_backoff +qbit-prism-server::ctv_fanout_bookkeeping::a_forward_database_clock_step_mid_attempt_keeps_the_holders_claim_and_settlement +qbit-prism-server::ctv_fanout_bookkeeping::a_frontend_with_block_submission_disabled_claims_and_sends_no_fanout +qbit-prism-server::ctv_fanout_bookkeeping::a_hold_set_mid_attempt_hands_the_fanout_claim_back_unsent +qbit-prism-server::ctv_fanout_bookkeeping::a_refused_completion_requeues_its_fanout_behind_the_rows_already_due +qbit-prism-server::ctv_fanout_bookkeeping::a_send_whose_completion_is_refused_is_recorded_as_an_attempt +qbit-prism-server::ctv_fanout_bookkeeping::a_shutdown_mid_attempt_hands_the_fanout_claim_back +qbit-prism-server::equal_work_scheduler::a_fresh_observation_after_completed_peer_aba_cannot_repeat_unchanged +qbit-prism-server::equal_work_scheduler::immediate_retry_budget_resets_only_after_an_external_tick +qbit-prism-server::equal_work_scheduler::non_retry_epoch_refusal_keeps_the_normal_poll_cadence +qbit-prism-server::equal_work_scheduler::peer_aba_during_the_immediate_proof_consumes_the_original_witness +qbit-prism-server::equal_work_scheduler::scheduled_equal_work_retry_issues_within_the_original_bound +qbit-prism-server::equal_work_scheduler::shutdown_after_refusal_prevents_the_immediate_attempt +qbit-prism-server::equal_work_tip::accounting_retries_keep_original_epoch_across_multiple_refusals_and_peer_aba +qbit-prism-server::equal_work_tip::accounting_revision_refusal_retains_only_an_uncommitted_transition +qbit-prism-server::equal_work_tip::cancelled_publication_and_cold_start_cannot_recreate_a_transition +qbit-prism-server::equal_work_tip::delayed_coherent_poll_cannot_reverse_a_newer_equal_work_tip +qbit-prism-server::equal_work_tip::delayed_transition_cannot_retry_after_peer_aba +qbit-prism-server::equal_work_tip::equal_work_replacement_preserves_strict_and_revision_fences +qbit-prism-server::equal_work_tip::equal_work_same_height_replacement_issues_within_two_seconds +qbit-prism-server::equal_work_tip::lower_work_consumes_an_accounting_retry_even_after_peer_aba +qbit-prism-server::equal_work_tip::lower_work_reconsideration_preserves_the_equal_work_predecessor +qbit-prism-server::equal_work_tip::unchanged_opposing_nodes_cannot_repeat_equal_work_replacements +qbit-prism-server::fanout_revision_bump::a_revision_bump_between_build_and_persist_refuses_the_stale_job +qbit-prism-server::import_audits_settlement_lock::a_busy_settlement_lock_fails_the_row_within_the_lock_timeout +qbit-prism-server::import_audits_settlement_lock::a_client_that_stalls_holding_settlement_lock_loses_it_within_the_lock_timeout +qbit-prism-server::import_audits_settlement_lock::a_large_legacy_audit_is_sent_without_holding_settlement_lock +qbit-prism-server::import_audits_settlement_lock::a_settlement_lock_holder_queued_behind_another_waiter_still_goes_first +qbit-prism-server::import_audits_settlement_lock::a_settlement_lock_holder_waiting_for_the_row_goes_first +qbit-prism-server::import_audits_settlement_lock::a_stop_after_the_last_row_ends_as_a_finished_import +qbit-prism-server::import_audits_settlement_lock::a_stop_before_the_lock_is_granted_rolls_the_row_back +qbit-prism-server::import_audits_settlement_lock::a_stop_between_rows_ends_the_import_before_the_next_row +qbit-prism-server::import_audits_settlement_lock::a_stop_during_a_write_rolls_the_row_back_at_once +qbit-prism-server::import_audits_settlement_lock::a_stop_while_the_server_runs_the_write_rolls_it_back_within_a_second +qbit-prism-server::import_audits_settlement_lock::a_write_that_stalls_mid_send_is_rolled_back_within_the_write_timeout +qbit-prism-server::import_audits_settlement_lock::the_import_audits_command_names_the_row_when_settlement_lock_stays_busy +qbit-prism-server::import_audits_settlement_lock::the_import_audits_command_stops_on_sigterm +qbit-prism-server::issued_job_dependency::conflicting_repair_and_invalid_dependency_boundaries_leave_no_child +qbit-prism-server::issued_job_dependency::exact_retry_is_immutable_and_conflicting_child_rolls_back_parent_extension +qbit-prism-server::issued_job_dependency::gc_rechecks_expiry_after_waiting_for_atomic_dependency_renewal +qbit-prism-server::issued_job_dependency::issued_save_extends_only_dependency_lifetime_through_the_child_deadline +qbit-prism-server::issued_job_dependency::pruned_dependency_returns_missing_without_committing_a_dangling_child +qbit-prism-server::issued_job_dependency::repair_preserves_original_revision_and_payload_under_a_new_current_revision +qbit-prism-server::issued_job_dependency::wrong_current_revision_rejects_before_renewal_or_repair +qbit-prism-server::jsonb_ceiling_gate::jsonb_ceiling_ratchet_at_reduced_sizes +qbit-prism-server::landing_malloc_trim::each_landing_trim_returns_the_landing_frontend_to_its_resident_floor +qbit-prism-server::landing_metrics::accepted_offer_waiting_for_build_capacity_is_reported_from_the_offer_reservation +qbit-prism-server::landing_metrics::active_proof_confirmation_and_maturity_bind_the_same_committed_revision +qbit-prism-server::landing_metrics::actual_build_deadline_counts_once_and_later_notify_is_degraded +qbit-prism-server::landing_metrics::cancelled_and_failed_terminal_collection_preserves_wait_until_durable_read +qbit-prism-server::landing_metrics::cancelled_commit_reply_keeps_unknown_without_fabricated_histogram +qbit-prism-server::landing_metrics::cancelled_orphan_commit_reply_reconciles_without_reopening_tombstone +qbit-prism-server::landing_metrics::cancelled_refresh_keeps_acceptance_without_fabricating_timeout_or_delivery +qbit-prism-server::landing_metrics::candidate_collector_keeps_an_acknowledged_lost_race_out_of_the_paging_age +qbit-prism-server::landing_metrics::committed_fatal_reconcile_preserves_processed_peer_revision +qbit-prism-server::landing_metrics::delayed_settlement_reply_cannot_publish_an_older_acceptance_at_a_newer_revision +qbit-prism-server::landing_metrics::failed_audit_landing_after_the_offer_is_reported_until_a_retry_lands +qbit-prism-server::landing_metrics::failed_template_and_second_acceptance_keep_oldest_wait_until_delivery +qbit-prism-server::landing_metrics::lost_commit_reply_timeouts_are_not_revision_work_failures +qbit-prism-server::landing_metrics::lost_offer_reply_starts_at_active_proof_and_is_not_a_work_build_timeout +qbit-prism-server::landing_metrics::lost_orphan_commit_reply_is_unknown_until_terminal_evidence +qbit-prism-server::landing_metrics::lost_race_stays_unlanded_until_its_orphan_proof_and_an_active_block_still_pages +qbit-prism-server::landing_metrics::node_rejected_block_pages_as_unacknowledged_but_a_side_chain_reply_does_not +qbit-prism-server::landing_metrics::operator_recovery_binds_first_confirmation_until_real_delivery +qbit-prism-server::landing_metrics::peer_confirmation_during_audit_reply_wait_cannot_invent_the_original_target +qbit-prism-server::landing_metrics::peer_orphan_evidence_closes_the_accepting_frontend_without_delivery +qbit-prism-server::landing_metrics::persistently_failing_post_offer_observation_is_reported_from_the_offer_reservation +qbit-prism-server::landing_metrics::real_offer_and_peer_proof_finish_once_only_after_notify_delivery +qbit-prism-server::landing_metrics::stale_reconcile_before_commit_does_not_poison_peer_landing +qbit-prism-server::ledger_cancelled_begin::session_allocation_dropped_inside_begin_returns_the_connection_clean +qbit-prism-server::ledger_cancelled_begin::share_append_cancelled_before_its_begin_batch_ends_retires_the_connection +qbit-prism-server::ledger_cancelled_begin::share_append_cancelled_inside_begin_retires_the_connection +qbit-prism-server::ledger_deadline_atomicity::backfill_item_failure_is_atomic +qbit-prism-server::ledger_deadline_atomicity::conflicting_audit_preserves_original +qbit-prism-server::ledger_deadline_atomicity::import_item_failure_preserves_completed_items +qbit-prism-server::ledger_deadline_atomicity::incomplete_audit_snapshot_returns_no_body +qbit-prism-server::ledger_deadline_atomicity::incomplete_window_is_typed_error +qbit-prism-server::ledger_deadline_atomicity::issued_job_late_expiry_rolls_back +qbit-prism-server::ledger_deadline_atomicity::landing_rows_roll_back_on_late_failure +qbit-prism-server::ledger_postgres::active_candidate_can_land_at_proven_new_chain_revision +qbit-prism-server::ledger_postgres::authority_writers::a_fatal_state_clear_waits_for_a_job_cohort_fence +qbit-prism-server::ledger_postgres::authority_writers::a_policy_transition_waits_for_a_job_cohort_fence +qbit-prism-server::ledger_postgres::authority_writers::configure_and_tip_observation_wait_for_a_job_cohort_fence +qbit-prism-server::ledger_postgres::authority_writers::settlement_and_reconcile_bumps_and_the_mature_block_halt_wait_for_a_job_cohort_fence +qbit-prism-server::ledger_postgres::authority_writers::the_deep_fanout_halt_waits_for_a_job_cohort_fence +qbit-prism-server::ledger_postgres::block_solvers::migration_016_attributes_a_2x_sources_blocks_as_its_single_statement_did +qbit-prism-server::ledger_postgres::block_solvers::migration_016_attributes_a_native_ledgers_blocks_when_016_is_missing +qbit-prism-server::ledger_postgres::candidate_dispatch_prioritizes_fresh_work_and_services_oldest_due_fairly +qbit-prism-server::ledger_postgres::candidate_mutations_reject_a_token_taken_over_while_waiting_for_its_row +qbit-prism-server::ledger_postgres::candidate_outbox_is_atomic_and_claims_recover_after_owner_loss +qbit-prism-server::ledger_postgres::candidate_processing_lock_allows_renewal_and_blocks_expired_takeover +qbit-prism-server::ledger_postgres::candidate_renewal_requires_a_live_pending_token +qbit-prism-server::ledger_postgres::candidates_cli::abandon_refuses_a_live_foreign_claim_and_succeeds_once_it_expires +qbit-prism-server::ledger_postgres::candidates_cli::abandon_refuses_a_parked_legacy_document_and_leaves_its_evidence_whole +qbit-prism-server::ledger_postgres::candidates_cli::abandon_refuses_a_terminal_row_and_a_pending_row_whose_block_landed +qbit-prism-server::ledger_postgres::candidates_cli::abandon_refuses_every_offered_state_and_leaves_the_row_byte_identical +qbit-prism-server::ledger_postgres::candidates_cli::abandon_refuses_unsupported_storage_versions_without_changing_evidence +qbit-prism-server::ledger_postgres::candidates_cli::abandon_registers_no_heartbeat_and_keeps_the_halt_guard +qbit-prism-server::ledger_postgres::candidates_cli::abandon_reports_claimed_when_lease_expires_before_diagnosis +qbit-prism-server::ledger_postgres::candidates_cli::abandon_terminalizes_a_pending_row_exactly_as_supersession_does +qbit-prism-server::ledger_postgres::candidates_cli::abandon_waits_for_inflight_landing_after_claim_expiry +qbit-prism-server::ledger_postgres::candidates_cli::list_reports_truncation_without_hiding_it_in_either_output_mode +qbit-prism-server::ledger_postgres::candidates_cli::list_separates_parked_rows_from_retrying_rows_and_shows_unknown_storage_versions +qbit-prism-server::ledger_postgres::candidates_cli::list_shows_every_unfinished_state_without_claiming_or_loading_payloads +qbit-prism-server::ledger_postgres::candidates_recover::recover_apply_enforces_the_deadline_and_leaves_the_candidate_recoverable +qbit-prism-server::ledger_postgres::candidates_recover::recover_apply_lands_the_listed_blocks_and_verifies_completed_ones_idempotently +qbit-prism-server::ledger_postgres::candidates_recover::recover_apply_refuses_a_live_foreign_claim_and_leaves_the_row_untouched +qbit-prism-server::ledger_postgres::candidates_recover::recover_apply_releases_a_committed_claim_when_its_reply_times_out +qbit-prism-server::ledger_postgres::candidates_recover::recover_apply_reports_changed_claim_ownership_after_deadline +qbit-prism-server::ledger_postgres::candidates_recover::recover_apply_reports_failed_claim_cleanup +qbit-prism-server::ledger_postgres::candidates_recover::recover_apply_revalidates_completed_blocks_after_earlier_landings +qbit-prism-server::ledger_postgres::candidates_recover::recover_apply_revalidates_completed_blocks_after_planning +qbit-prism-server::ledger_postgres::candidates_recover::recover_plan_fails_closed_on_missing_terminal_inactive_legacy_and_unlisted_parents +qbit-prism-server::ledger_postgres::candidates_recover::recover_plan_orders_parent_before_child_without_claiming_or_loading_payloads +qbit-prism-server::ledger_postgres::clock_steps::a_backward_database_clock_step_never_stretches_a_dead_fanout_holders_lease +qbit-prism-server::ledger_postgres::clock_steps::a_backward_database_clock_step_never_stretches_a_dead_holders_lease +qbit-prism-server::ledger_postgres::clock_steps::a_backward_database_clock_step_never_stretches_a_fanout_retry +qbit-prism-server::ledger_postgres::clock_steps::a_backward_database_clock_step_never_stretches_a_retry +qbit-prism-server::ledger_postgres::clock_steps::a_backward_step_then_one_renewal_never_strands_a_dead_fanout_holders_row +qbit-prism-server::ledger_postgres::clock_steps::a_backward_step_then_one_renewal_never_strands_a_dead_holders_row +qbit-prism-server::ledger_postgres::clock_steps::a_fanout_release_after_a_backward_step_never_stretches_its_next_attempt +qbit-prism-server::ledger_postgres::clock_steps::a_forward_database_clock_step_does_not_hand_a_live_claim_to_recovery +qbit-prism-server::ledger_postgres::clock_steps::a_forward_database_clock_step_does_not_let_abandon_take_a_live_claim +qbit-prism-server::ledger_postgres::clock_steps::a_forward_database_clock_step_never_ends_a_live_claim_early +qbit-prism-server::ledger_postgres::clock_steps::a_forward_database_clock_step_never_ends_a_live_fanout_claim_early +qbit-prism-server::ledger_postgres::clock_steps::a_recovery_claim_and_release_after_a_backward_step_never_stretch_a_retry +qbit-prism-server::ledger_postgres::concurrent_cpfp_allocations_exclude_active_and_retired_outpoints +qbit-prism-server::ledger_postgres::concurrent_instances_have_one_commit_order_and_stable_snapshots +qbit-prism-server::ledger_postgres::cpfp_retirement_requires_current_claim_and_preserves_signed_packages +qbit-prism-server::ledger_postgres::ctv_artifacts_wait_for_maturity_and_claims_are_fenced +qbit-prism-server::ledger_postgres::deferred_share_survives_confirmation_crash_and_is_credited_once +qbit-prism-server::ledger_postgres::durable_sessions_upgrade_async_commit_and_keep_remote_apply +qbit-prism-server::ledger_postgres::fanout_lane_plan::a_fanout_claim_poll_never_reads_the_settled_history +qbit-prism-server::ledger_postgres::fanout_lane_plan::migration_024_builds_the_lane_index_without_blocking_fanout_writes +qbit-prism-server::ledger_postgres::fatal_state::chain_change_rpc_timeout_and_cancellation_keep_the_halt +qbit-prism-server::ledger_postgres::fatal_state::clear_accepts_a_tip_extended_during_the_run +qbit-prism-server::ledger_postgres::fatal_state::clear_and_self_check_run_the_report_under_their_own_timeout +qbit-prism-server::ledger_postgres::fatal_state::clear_preserves_halt_on_integrity_failure_and_audit_insert_failure +qbit-prism-server::ledger_postgres::fatal_state::clear_refuses_a_chain_that_flapped_to_a_fork_during_its_observations +qbit-prism-server::ledger_postgres::fatal_state::clear_refuses_a_closing_tip_walk_that_would_run_into_the_commit +qbit-prism-server::ledger_postgres::fatal_state::clear_refuses_a_reorg_during_the_run +qbit-prism-server::ledger_postgres::fatal_state::clear_refuses_an_extension_with_a_pool_block_above_the_captured_height +qbit-prism-server::ledger_postgres::fatal_state::clear_refuses_live_starting_stale_and_unknown_instances +qbit-prism-server::ledger_postgres::fatal_state::clear_report_statement_ends_within_the_bound +qbit-prism-server::ledger_postgres::fatal_state::clear_rolls_back_reconciliation_if_mature_block_remains_disconnected +qbit-prism-server::ledger_postgres::fatal_state::clear_waits_for_a_commit_that_outlasts_its_bound +qbit-prism-server::ledger_postgres::fatal_state::deep_fanout_halt_names_recovery_and_refuses_unresolved_confirmation +qbit-prism-server::ledger_postgres::fatal_state::migration_and_show_preserve_unknown_legacy_set_time_without_registration +qbit-prism-server::ledger_postgres::fatal_state::one_shot_tools_register_no_heartbeat_and_keep_the_halt_guard +qbit-prism-server::ledger_postgres::fatal_state::recovery_rejects_wrong_cluster_and_unsuitable_node +qbit-prism-server::ledger_postgres::fatal_state::recovery_resolves_fee_address_before_verifying_cluster_fingerprint +qbit-prism-server::ledger_postgres::fatal_state::recovery_serializes_new_heartbeats_and_concurrent_clear +qbit-prism-server::ledger_postgres::fatal_state::self_check_and_clear_accept_a_programs_legacy_chain_across_case_labels +qbit-prism-server::ledger_postgres::fatal_state::show_and_clear_cli_resume_appends_and_record_operator_decision +qbit-prism-server::ledger_postgres::fixture_databases_isolate_advisory_keys_while_one_fixture_still_contends +qbit-prism-server::ledger_postgres::global_duplicates_idempotence_and_config_fencing +qbit-prism-server::ledger_postgres::index_trim::migration_013_builds_its_indexes_without_blocking_appends +qbit-prism-server::ledger_postgres::index_trim::migration_013_offline_builds_plain_indexes_in_one_transaction_with_every_instance_stopped +qbit-prism-server::ledger_postgres::index_trim::migration_013_offline_fails_fast_behind_an_open_writer_and_changes_nothing +qbit-prism-server::ledger_postgres::index_trim::migration_013_offline_refuses_a_live_instance_or_legacy_lease_before_any_ddl +qbit-prism-server::ledger_postgres::index_trim::migration_013_offline_reports_a_lock_timeout_in_its_guards_as_itself +qbit-prism-server::ledger_postgres::index_trim::migration_013_offline_rolls_back_a_partial_run_whole +qbit-prism-server::ledger_postgres::index_trim::migration_013_offline_takes_the_operators_build_memory_and_otherwise_never_lowers_the_servers +qbit-prism-server::ledger_postgres::index_trim::migration_013_refuses_a_drop_target_swapped_while_it_built +qbit-prism-server::ledger_postgres::index_trim::migration_013_refuses_redefined_drop_targets_before_any_ddl +qbit-prism-server::ledger_postgres::index_trim::migration_013_refuses_to_record_when_a_kept_index_moved_while_it_built +qbit-prism-server::ledger_postgres::index_trim::migration_013_replaces_the_release_indexes_of_a_frozen_2x_source_and_keeps_its_rows +qbit-prism-server::ledger_postgres::index_trim::migration_013_resumes_an_interrupted_build_keeps_its_own_index_and_refuses_another +qbit-prism-server::ledger_postgres::index_trim::migration_013_trims_the_indexes_of_an_empty_ledger_in_the_transaction_and_a_restart_keeps_them +qbit-prism-server::ledger_postgres::late_acceptance_after_abandoned_claim_recovers_deferred_credit +qbit-prism-server::ledger_postgres::legacy_audit_import_validates_envelope_hash_and_pinned_key +qbit-prism-server::ledger_postgres::migration_refuses_an_active_legacy_writer +qbit-prism-server::ledger_postgres::migration_refuses_undrained_legacy_block_candidates +qbit-prism-server::ledger_postgres::policy_transition::cli_loads_env_overrides_and_resolves_fee_addresses_without_registering +qbit-prism-server::ledger_postgres::policy_transition::cli_rejects_invalid_target_without_exposing_env_source +qbit-prism-server::ledger_postgres::policy_transition::ctv_fee_transition_can_repair_a_rate_below_the_live_floor +qbit-prism-server::ledger_postgres::policy_transition::event_failure_rolls_back_fingerprint_revision_and_candidate_disposition +qbit-prism-server::ledger_postgres::policy_transition::historical_audits_and_ctv_artifacts_remain_byte_identical +qbit-prism-server::ledger_postgres::policy_transition::offered_candidates_keep_original_evidence_and_unoffered_are_abandoned +qbit-prism-server::ledger_postgres::policy_transition::only_fee_fields_can_change_and_retry_does_not_bump_again +qbit-prism-server::ledger_postgres::policy_transition::policy_transition_cannot_clear_a_fatal_state +qbit-prism-server::ledger_postgres::policy_transition::registration_racing_the_stopped_scan_is_not_missed +qbit-prism-server::ledger_postgres::policy_transition::rejected_same_id_startup_cannot_authorize_a_policy_transition +qbit-prism-server::ledger_postgres::policy_transition::transition_waits_for_both_accounting_locks +qbit-prism-server::ledger_postgres::policy_transition::two_frontends_must_stop_and_old_work_and_startup_are_rejected +qbit-prism-server::ledger_postgres::reconciliation_watches_only_recent_blocks_and_one_mature_checkpoint +qbit-prism-server::ledger_postgres::session_sequence::cancelling_a_session_releases_only_its_reservation +qbit-prism-server::ledger_postgres::session_sequence::migration_009_preserves_preexisting_jobs_and_runs_once_for_two_frontends +qbit-prism-server::ledger_postgres::session_sequence::only_stopped_owners_are_reclaimed_and_old_cleanup_cannot_release_a_replacement +qbit-prism-server::ledger_postgres::session_sequence::stopped_guard_blocks_cleanup_and_owner_filter_preserves_other_reservations +qbit-prism-server::ledger_postgres::session_sequence::stopped_requires_no_pending_or_live_sessions_and_cannot_reclaim_another_incarnation +qbit-prism-server::ledger_postgres::session_sequence::two_frontends_race_across_wrap_without_duplicate_allocations +qbit-prism-server::ledger_postgres::session_sequence::wrap_reuses_expired_jobs_and_skips_live_jobs_and_sessions +qbit-prism-server::ledger_postgres::share_hash_backfill::a_batch_past_the_throttles_statement_timeout_is_retried_at_half_its_size +qbit-prism-server::ledger_postgres::share_hash_backfill::a_connect_queued_behind_a_deferred_claim_maps_nothing +qbit-prism-server::ledger_postgres::share_hash_backfill::a_crashed_resume_leaves_no_fence_for_an_earlier_runner_to_strand +qbit-prism-server::ledger_postgres::share_hash_backfill::a_deferred_migrate_stopped_before_its_recent_range_is_left_to_the_operator +qbit-prism-server::ledger_postgres::share_hash_backfill::a_frontend_with_init_schema_leaves_a_deferred_backfill_to_migrate +qbit-prism-server::ledger_postgres::share_hash_backfill::a_resume_leaves_no_fence_when_an_earlier_runner_finishes_the_backfill +qbit-prism-server::ledger_postgres::share_hash_backfill::a_runner_leaves_a_newer_releases_fence_with_its_cursor_and_record +qbit-prism-server::ledger_postgres::share_hash_backfill::an_archive_ready_native_partition_stays_attached_until_2_is_recorded +qbit-prism-server::ledger_postgres::share_hash_backfill::backfill_share_hashes_maps_the_rest_while_frontends_append_and_records_2 +qbit-prism-server::ledger_postgres::share_hash_backfill::backfill_share_hashes_refuses_a_backfill_that_does_not_permit_serving +qbit-prism-server::ledger_postgres::share_hash_backfill::backfill_share_hashes_refuses_to_record_2_over_a_native_share_repeating_a_legacy_header +qbit-prism-server::ledger_postgres::share_hash_backfill::backfill_share_hashes_runs_each_batch_under_its_statement_timeout_and_rests_between_batches +qbit-prism-server::ledger_postgres::share_hash_backfill::every_start_refuses_a_backfill_fence_on_a_migrated_database +qbit-prism-server::ledger_postgres::share_hash_backfill::migrate_defer_share_hashes_leaves_a_share_seq_sequence_at_or_past_the_end_untouched +qbit-prism-server::ledger_postgres::share_hash_backfill::migrate_defer_share_hashes_maps_the_recent_range_and_permits_serving +qbit-prism-server::ledger_postgres::share_hash_backfill::migrate_defer_share_hashes_moves_a_lagging_share_seq_sequence_up_to_the_frozen_end +qbit-prism-server::ledger_postgres::share_hash_backfill::migrate_defer_share_hashes_refuses_a_backfill_an_earlier_build_started +qbit-prism-server::ledger_postgres::share_hash_backfill::migrate_defer_share_hashes_resumes_a_backfill_plain_migrate_left_part_way +qbit-prism-server::ledger_postgres::share_hash_backfill::migrate_defer_share_hashes_with_offline_indexes_permits_serving_in_one_run +qbit-prism-server::ledger_postgres::share_hash_backfill::migrate_finishes_a_deferred_backfill_up_to_its_planned_end_while_frontends_append +qbit-prism-server::ledger_postgres::share_hash_backfill::migrate_refuses_a_double_credit_in_the_last_chunk_of_its_check +qbit-prism-server::ledger_postgres::share_hash_backfill::migrate_refuses_to_record_2_over_a_native_share_repeating_a_legacy_header +qbit-prism-server::ledger_postgres::share_hash_backfill::migration_002_backfill_resumes_from_its_last_committed_batch_after_an_interrupt +qbit-prism-server::ledger_postgres::share_hash_backfill::migration_002_fences_earlier_builds_until_its_backfill_is_recorded +qbit-prism-server::ledger_postgres::share_hash_backfill::migration_002_maps_a_populated_2x_ledger_in_batches_as_its_single_statement_did +qbit-prism-server::ledger_postgres::share_hash_backfill::migration_002_maps_a_row_past_its_planned_end_before_recording_2 +qbit-prism-server::ledger_postgres::share_hash_backfill::migration_002_refuses_a_2x_source_holding_the_backfill_progress_name_before_any_ddl +qbit-prism-server::ledger_postgres::share_hash_backfill::migration_002_refuses_a_backfill_fence_left_without_its_cursor +qbit-prism-server::ledger_postgres::share_hash_backfill::migrations_13_17_and_24_run_with_a_deferred_backfill_pending +qbit-prism-server::ledger_postgres::share_hash_backfill::plain_migrate_maps_a_deferred_backfill_at_the_default_throttle +qbit-prism-server::ledger_postgres::share_hash_backfill::plain_migrate_refuses_before_mapping_when_the_fence_reached_2_while_it_waited +qbit-prism-server::ledger_postgres::share_hash_backfill::self_check_reports_a_deferred_backfill_without_failing_on_it +qbit-prism-server::ledger_postgres::share_hash_backfill::share_archive_restore_detach_and_drop_refuse_while_a_backfill_is_pending +qbit-prism-server::ledger_postgres::share_hash_backfill::the_double_credit_check_runs_under_the_throttles_statement_timeout +qbit-prism-server::ledger_postgres::share_hash_backfill::the_record_of_2_stops_cleanly_behind_a_held_migration_lock_and_records_once_released +qbit-prism-server::ledger_postgres::share_hash_backfill::the_record_of_2_waits_out_a_reader_of_the_cursor_in_short_attempts +qbit-prism-server::ledger_postgres::share_partitions::migration_016_captures_old_frontend_solvers_during_online_conversion +qbit-prism-server::ledger_postgres::share_partitions::migration_017_converts_a_populated_ledger_online_and_resumes_from_every_stage +qbit-prism-server::ledger_postgres::share_partitions::migration_017_converts_an_empty_ledger_in_the_transaction_and_appends_route_to_the_first_cell +qbit-prism-server::ledger_postgres::share_partitions::migration_017_converts_empty_2x_ledgers_at_sequence_boundaries +qbit-prism-server::ledger_postgres::share_partitions::migration_017_moves_a_bound_whose_headroom_ran_out +qbit-prism-server::ledger_postgres::share_partitions::migration_017_refuses_a_reserved_partition_name_before_any_ddl +qbit-prism-server::ledger_postgres::share_partitions::migration_017_waits_for_an_open_writer_without_blocking_appends +qbit-prism-server::ledger_postgres::share_partitions::retained_legacy_header_duplicates_remain_exactly_replayable +qbit-prism-server::ledger_postgres::share_partitions::share_id_uniqueness_is_per_leaf_and_the_append_path_keeps_it_global +qbit-prism-server::ledger_postgres::shared_chainwork_rejects_lagging_nodes_and_fences_acknowledgements +qbit-prism-server::ledger_postgres::simultaneous_block_only_enqueue_has_one_winner +qbit-prism-server::ledger_postgres::stale_candidate_active_proof_cannot_overwrite_a_newer_reorg +qbit-prism-server::ledger_postgres::two_x::audit_range_query_uses_deadline_remaining_after_delayed_snapshot +qbit-prism-server::ledger_postgres::two_x::canonical_2x_sidecar_import_preserves_exact_bytes_and_fails_closed +qbit-prism-server::ledger_postgres::two_x::compact_bits_metadata_comes_from_durable_header_and_recovers_without_audit_changes +qbit-prism-server::ledger_postgres::two_x::corrupt_imported_canonical_bytes_are_refused_not_served_from_the_legacy_file +qbit-prism-server::ledger_postgres::two_x::domain_extra_column_is_refused_naming_it_and_rolls_back +qbit-prism-server::ledger_postgres::two_x::drifted_002_object_is_refused_on_a_258_source +qbit-prism-server::ledger_postgres::two_x::drifted_pre_258_source_is_refused_naming_each_object_and_rolls_back +qbit-prism-server::ledger_postgres::two_x::durable_worker_hints_order_by_shared_evidence_and_keep_original_ttl +qbit-prism-server::ledger_postgres::two_x::empty_database_migrates_as_fresh_with_or_without_an_operator_table +qbit-prism-server::ledger_postgres::two_x::empty_database_with_a_stray_native_table_is_refused_naming_it_before_any_ddl +qbit-prism-server::ledger_postgres::two_x::executable_extra_column_is_refused_naming_it_and_rolls_back +qbit-prism-server::ledger_postgres::two_x::extra_constraint_on_a_release_table_is_refused_naming_it_and_rolls_back +qbit-prism-server::ledger_postgres::two_x::extra_plain_or_custom_operator_index_is_refused_and_rolls_back +qbit-prism-server::ledger_postgres::two_x::extra_rewrite_rule_is_refused_naming_it_and_rolls_back +qbit-prism-server::ledger_postgres::two_x::extra_trigger_on_a_release_table_is_refused_naming_it_and_rolls_back +qbit-prism-server::ledger_postgres::two_x::extra_write_affecting_index_is_refused_naming_it_and_rolls_back +qbit-prism-server::ledger_postgres::two_x::frozen_258_source_refuses_pending_v1_and_v2_rows_and_migrates_terminal_rows +qbit-prism-server::ledger_postgres::two_x::frozen_pre_258_source_refuses_pending_v1_cleanly_and_migrates_terminal_rows +qbit-prism-server::ledger_postgres::two_x::imported_audit_decode_limit_outlives_a_dropped_request +qbit-prism-server::ledger_postgres::two_x::imported_ctv_audit_backfills_and_links_without_the_legacy_file +qbit-prism-server::ledger_postgres::two_x::imported_external_audit_is_served_from_canonical_bytes_without_the_legacy_file +qbit-prism-server::ledger_postgres::two_x::incoming_foreign_key_is_refused_naming_it_and_rolls_back +qbit-prism-server::ledger_postgres::two_x::inheritance_involving_a_release_table_is_refused_naming_it_and_rolls_back +qbit-prism-server::ledger_postgres::two_x::inline_only_legacy_audit_import_keeps_its_body_and_serves_canonical_bytes +qbit-prism-server::ledger_postgres::two_x::legacy_2x_failed_native_migration_rolls_back_seed_repair_and_all_ddl +qbit-prism-server::ledger_postgres::two_x::legacy_2x_upgrade_repairs_partial_carry_seed_and_preserves_shared_state +qbit-prism-server::ledger_postgres::two_x::migrate_role_without_create_on_the_database_is_told_the_grant_it_needs +qbit-prism-server::ledger_postgres::two_x::migrated_database_without_its_capability_declaration_is_refused_at_connect_and_at_migrate +qbit-prism-server::ledger_postgres::two_x::migrated_database_without_readable_source_metadata_is_refused_before_later_ddl +qbit-prism-server::ledger_postgres::two_x::migration_refuses_source_objects_resolved_from_a_later_schema +qbit-prism-server::ledger_postgres::two_x::migration_source_resolved_from_a_later_schema_is_refused +qbit-prism-server::ledger_postgres::two_x::native_candidate_version_two_is_refused_without_claiming_or_migrating +qbit-prism-server::ledger_postgres::two_x::native_capabilities_resolved_from_a_later_schema_are_refused +qbit-prism-server::ledger_postgres::two_x::native_capabilities_with_row_level_security_are_refused_before_writes +qbit-prism-server::ledger_postgres::two_x::native_capability_relation_must_be_an_ordinary_table +qbit-prism-server::ledger_postgres::two_x::native_migration_gap_collisions_are_refused_before_any_ddl +qbit-prism-server::ledger_postgres::two_x::native_migration_gaps_refuse_existing_tables_columns_and_constraints +qbit-prism-server::ledger_postgres::two_x::native_migration_history_drift_is_refused_before_later_ddl +qbit-prism-server::ledger_postgres::two_x::native_publication_ordinal_survives_reactivation_and_excludes_rejected_candidates +qbit-prism-server::ledger_postgres::two_x::native_record_with_3_and_not_2_is_refused_before_any_ddl_and_not_repaired +qbit-prism-server::ledger_postgres::two_x::newer_storage_version_or_capability_is_refused_at_migrate_and_at_connect +qbit-prism-server::ledger_postgres::two_x::non_native_source_refuses_preexisting_migration_history +qbit-prism-server::ledger_postgres::two_x::partial_001_source_is_refused_before_any_ddl_naming_the_objects_present +qbit-prism-server::ledger_postgres::two_x::partial_002_source_is_refused_naming_the_missing_object +qbit-prism-server::ledger_postgres::two_x::pre_006_native_outbox_row_security_is_refused_before_the_drain_check +qbit-prism-server::ledger_postgres::two_x::pre_006_native_schema_declaring_a_newer_capability_is_refused_before_any_ddl +qbit-prism-server::ledger_postgres::two_x::pre_006_native_schema_on_a_258_source_refuses_a_pending_v2_row_before_any_ddl +qbit-prism-server::ledger_postgres::two_x::pre_006_native_schema_on_a_pre_258_source_refuses_an_undrained_v1_row_with_the_v1_only_predicate +qbit-prism-server::ledger_postgres::two_x::pre_006_native_schema_refuses_a_malformed_storage_version_column +qbit-prism-server::ledger_postgres::two_x::pre_006_native_schema_refuses_preexisting_source_metadata +qbit-prism-server::ledger_postgres::two_x::pre_006_native_schema_with_009_applies_006_on_the_next_migrate +qbit-prism-server::ledger_postgres::two_x::pre_006_native_schema_with_only_native_pending_candidates_migrates_and_keeps_them_claimable +qbit-prism-server::ledger_postgres::two_x::release_constraint_left_not_valid_is_refused_naming_it_and_rolls_back +qbit-prism-server::ledger_postgres::two_x::release_foreign_key_with_disabled_enforcement_triggers_is_refused_naming_it_and_rolls_back +qbit-prism-server::ledger_postgres::two_x::required_extra_column_is_refused_naming_it_and_rolls_back +qbit-prism-server::ledger_postgres::two_x::reserved_name_held_by_a_relation_of_another_kind_is_refused_before_any_ddl +qbit-prism-server::ledger_postgres::two_x::row_level_security_on_a_release_table_is_refused_naming_it_and_rolls_back +qbit-prism-server::ledger_postgres::two_x::startup_without_initialize_refuses_a_pre_006_native_schema_with_009 +qbit-prism-server::ledger_postgres::two_x::startup_without_initialize_requires_the_current_schema_version +qbit-prism-server::ledger_postgres::two_x::superseded_inline_audit_body_is_not_loaded_for_imported_rows +qbit-prism-server::ledger_postgres::two_x::tolerated_source_differences_still_migrate +qbit-prism-server::ledger_postgres::two_x::two_x_source_with_a_stray_native_column_is_refused_naming_it_before_any_ddl +qbit-prism-server::ledger_postgres::two_x::two_x_source_with_stray_native_objects_is_refused_naming_each_before_any_ddl +qbit-prism-server::ledger_postgres::two_x::unknown_storage_version_row_is_parked_and_not_reclaimed_at_lease_expiry +qbit-prism-server::ledger_postgres::two_x::unlogged_release_table_or_sequence_is_refused_naming_it_and_rolls_back +qbit-prism-server::ledger_postgres::verified_landing_reconstructs_audit_and_reorgs_are_revision_fenced +qbit-prism-server::ledger_single_execution::broadcast_retry_requires_new_claim +qbit-prism-server::ledger_single_execution::candidate_backoff_requires_new_claim +qbit-prism-server::ledger_single_execution::candidate_terminal_timeout_executes_once +qbit-prism-server::ledger_single_execution::distinct_hashes_hold_independent_claims +qbit-prism-server::ledger_single_execution::fanout_journal_timeout_executes_once +qbit-prism-server::ledger_single_execution::restored_authority_requires_fresh_claim +qbit-prism-server::ledger_single_execution::superseded_landing_reports_failure +qbit-prism-server::ledger_wait_metrics::advisory_lock_and_pool_waits_are_recorded_by_outcome +qbit-prism-server::ledger_wait_metrics::unattached_ledger_keeps_results_and_error_shape +qbit-prism-server::live_regtest::compact_runtime_tests::real_large_template_first_insertion_resumes_and_mines_with_bounded_jsonb_and_wal +qbit-prism-server::live_regtest::cpfp_tests::real_cpfp_abandoned_child_repair_retains_unremovable_spent_lock +qbit-prism-server::live_regtest::cpfp_tests::real_cpfp_recovers_reserved_funding_after_owner_crash +qbit-prism-server::live_regtest::cpfp_tests::real_cpfp_replaces_unsigned_funding_spent_after_reservation_before_wallet_lock +qbit-prism-server::live_regtest::highdiff_tests::observability_grace_counter_moves_only_after_real_durable_credit +qbit-prism-server::live_regtest::highdiff_tests::observed_tip_advance_fences_old_prepared_jobs_and_health +qbit-prism-server::live_regtest::highdiff_tests::real_highdiff_block_only_proof_waits_for_active_chain_credit +qbit-prism-server::live_regtest::highdiff_tests::real_share_height_queries_use_template_parent_height +qbit-prism-server::live_regtest::lost_reply_tests::lost_sendrawtransaction_reply_owner_killed_survivor_confirms_the_fanout_once +qbit-prism-server::live_regtest::lost_reply_tests::lost_submitblock_reply_owner_killed_survivor_lands_the_block_once +qbit-prism-server::live_regtest::lost_reply_tests::lost_submitpackage_reply_owner_killed_survivor_confirms_the_cpfp_package_once +qbit-prism-server::live_regtest::lost_reply_tests::share_credited_but_ack_lost_retry_is_deduplicated_exactly +qbit-prism-server::live_regtest::node_outage_tests::node_killed_after_accepting_reconciles_the_lost_block_once +qbit-prism-server::live_regtest::node_outage_tests::node_killed_before_submitblock_lands_the_block_once_after_restart +qbit-prism-server::live_regtest::node_outage_tests::node_restarted_into_warmup_before_submitblock_lands_the_block_once +qbit-prism-server::live_regtest::node_outage_tests::node_stopped_after_accepting_lands_the_block_once_after_resume +qbit-prism-server::live_regtest::node_outage_tests::node_stopped_before_submitblock_lands_the_block_once_after_resume +qbit-prism-server::live_regtest::pg_failover_tests::async_live_miners::async_promotion_with_live_miners_accounts_for_every_share_and_lands_a_block +qbit-prism-server::live_regtest::pg_failover_tests::offer_standby::a_found_block_offered_after_the_standby_flush_survives_a_replication_cut_at_submitblock +qbit-prism-server::live_regtest::pg_failover_tests::real_async_failover_mid_landing_loses_only_the_replication_gap +qbit-prism-server::live_regtest::pg_failover_tests::real_fenced_failover_mid_landing_keeps_every_acknowledged_share +qbit-prism-server::live_regtest::real_ctv_maturity_broadcast_and_confirmation +qbit-prism-server::live_regtest::real_two_server_mining_failover_audit_and_reorg +qbit-prism-server::live_regtest::transcript_replay_tests::recorded_stratum_transcripts_replay_compatibly_against_a_live_server +qbit-prism-server::live_regtest::two_node_tests::real_deep_reorg_with_own_blocks_on_both_branches_keeps_credit_balances_and_fanouts_exact +qbit-prism-server::live_regtest::two_node_tests::real_own_block_loses_same_height_race_is_orphaned_and_credited_once_on_reorg_back +qbit-prism-server::live_regtest::weighted_recipients_tests::real_weighted_recipients_pay_exact_direct_outputs_without_ctv +qbit-prism-server::live_regtest::weighted_recipients_tests::real_weighted_recipients_pay_exact_pplns_outputs_through_fanout_spend +qbit-prism-server::lost_race_pin::a_failed_observation_never_settles_a_row_as_orphaned +qbit-prism-server::lost_race_pin::a_proven_orphan_is_settled_terminal_and_a_reorg_back_still_credits_it +qbit-prism-server::lost_race_pin::a_reactivated_orphan_is_counted_and_credited_once_across_a_flap +qbit-prism-server::lost_race_pin::a_reservation_only_orphan_stays_unknown_and_releases_its_payload +qbit-prism-server::lost_race_pin::a_stale_orphan_verdict_cannot_overwrite_a_newer_active_proof +qbit-prism-server::lost_race_pin::an_ordinary_confirmation_is_counted_once_across_a_reorg_and_back +qbit-prism-server::lost_race_pin::first_confirmation_is_counted_before_outbox_terminalization +qbit-prism-server::lost_race_pin::lost_tip_race_reconciles_the_orphan_and_never_delays_the_next_tips_work +qbit-prism-server::migration_rollback::a_2x_program_paid_under_case_labels_migrates_clean_and_summarizes_as_its_source +qbit-prism-server::migration_rollback::cutover_rehearsal_tests::mainnet_shaped_2x_dump_rehearses_the_cutover_and_keeps_every_balance +qbit-prism-server::migration_rollback::frozen_2x_backup_restore_reconciles_before_ack_and_exposes_post_ack_loss +qbit-prism-server::migration_rollback::frozen_2x_backup_restore_session_settings_stay_off_the_fixture_pool +qbit-prism-server::migration_rollback::frozen_2x_cli_import_restores_all_body_layouts_and_serves_exact_artifacts +qbit-prism-server::migration_rollback::legacy_ordinal_revert_refuses_native_schema_without_removing_columns +qbit-prism-server::migration_rollback::the_evidence_export_defers_share_hashes_while_the_backfill_permits_serving +qbit-prism-server::migration_rollback::the_parallel_evidence_export_prints_the_serial_export_byte_for_byte +qbit-prism-server::observability_database::collector_uses_real_schema_pending_rows_and_failed_read_semantics +qbit-prism-server::observability_database::database_privacy::candidate_identifiers_and_heights_never_enter_collector_http_metrics +qbit-prism-server::observability_database::live_pool_waits_survive_blocked_publication_through_http +qbit-prism-server::offer_latency::offer_reaches_the_node_within_the_bound_while_builders_and_landing_are_held +qbit-prism-server::offer_lifecycle::a_divergent_offered_landing_warns_with_both_digests_and_keeps_its_as_issued_accounts +qbit-prism-server::offer_lifecycle::an_orphan_settlement_that_deactivates_its_block_waits_for_a_job_cohort_fence +qbit-prism-server::offer_lifecycle::an_undisturbed_offered_landing_is_marked_without_a_divergence_alert +qbit-prism-server::offer_lifecycle::every_unfinished_state_is_claimable_retained_and_reserved_only_once +qbit-prism-server::offer_lifecycle::integrity_report_026_equals_001_on_clean_mismatch_and_drift_fixtures +qbit-prism-server::offer_lifecycle::integrity_validator_accepts_marked_as_issued_rows_and_names_every_corruption +qbit-prism-server::offer_lifecycle::integrity_validator_checks_the_issued_arithmetic_of_every_manifest_account +qbit-prism-server::offer_lifecycle::integrity_validator_follows_a_legacy_chain_per_payout_program_across_case_labels +qbit-prism-server::offer_lifecycle::landing_reads_the_lifecycle_state_under_the_claim_lock_not_from_the_claim +qbit-prism-server::offer_lifecycle::migrated_database_without_its_offer_lifecycle_declaration_is_refused_at_connect_and_at_migrate +qbit-prism-server::offer_lifecycle::migration_011_quarantines_attempted_pending_rows_and_keeps_never_attempted_ones +qbit-prism-server::offer_lifecycle::migration_011_refuses_a_live_claim_and_an_attempted_row_without_evidence +qbit-prism-server::offer_lifecycle::migration_011_refuses_idle_instances_until_explicitly_stopped +qbit-prism-server::offer_lifecycle::migration_011_refuses_unknown_checks_the_lifecycle_cannot_satisfy_and_changes_nothing +qbit-prism-server::offer_lifecycle::migration_011_rejects_pre_011_registration_queued_behind_cutover +qbit-prism-server::offer_lifecycle::migration_011_replaces_only_the_known_lifecycle_rules_and_keeps_operator_checks +qbit-prism-server::offer_lifecycle::migration_011_sql_refuses_a_legacy_rule_under_its_own_name_before_matching_it +qbit-prism-server::offer_lifecycle::migration_011_waits_for_in_flight_instance_registration +qbit-prism-server::offer_lifecycle::migration_012_fences_existing_011_and_requires_its_declaration +qbit-prism-server::offer_lifecycle::migration_025_runs_again_after_011_runs +qbit-prism-server::offer_lifecycle::migration_026_is_recorded_once_and_survives_remigrate +qbit-prism-server::offer_lifecycle::oldest_due_lane_and_dispatch_probe_use_the_unfinished_index_over_retained_history +qbit-prism-server::offer_lifecycle::orphaned_rows_are_terminal_keep_their_evidence_and_leave_the_pending_gauge +qbit-prism-server::offer_lifecycle::outbox_lifecycle_checks_accept_exactly_the_lifecycle_shapes +qbit-prism-server::offer_lifecycle::pending_divergent_landing_refuses_while_the_post_offer_landing_succeeds +qbit-prism-server::offer_lifecycle::signer_rotation_is_refused_at_every_unfinished_state_alone_and_accepted_once_all_are_terminal +qbit-prism-server::orphan_disposition_migration::migration_015_bounds_its_wait_for_the_outbox_lock +qbit-prism-server::orphan_disposition_migration::migration_015_refuses_foreign_lifecycle_checks_and_tampered_or_missing_rules_changing_nothing +qbit-prism-server::orphan_disposition_migration::migration_015_refuses_running_earlier_instances_before_touching_the_outbox +qbit-prism-server::orphan_disposition_migration::migration_015_upgrades_a_stopped_deployment_on_plain_and_dual_outboxes +qbit-prism-server::postgres_failover::acknowledged_shares_survive_synchronous_primary_loss_and_pool_reconnect +qbit-prism-server::postgres_failover::offer_standby_flush::offer_standby_wait_confirms_the_flush_returns_at_the_bound_and_never_waits_for_a_stopped_standby +qbit-prism-server::postgres_failover::public_replica::read_only_public_service_tracks_real_standby_stream_and_refuses_promotion +qbit-prism-server::public_database_config::direct_public_binary_authenticates_only_with_effective_reader_credentials +qbit-prism-server::qbit_prism_server::api::public_service::tests::every_sql_statement_uses_the_remaining_request_budget +qbit-prism-server::qbit_prism_server::coordinator::candidate_lease_tests::a_forward_database_clock_step_keeps_the_holders_lease_through_its_landing +qbit-prism-server::qbit_prism_server::coordinator::candidate_lease_tests::accepted_landing_failure_reaches_reconciliation_with_a_reason_never_abandoned +qbit-prism-server::qbit_prism_server::coordinator::candidate_lease_tests::canceled_or_lost_renewal_cannot_resume_queued_work +qbit-prism-server::qbit_prism_server::coordinator::candidate_lease_tests::checkout_tests::live_token_checkout_cancel_and_closed_pool_keep_renewal_failure +qbit-prism-server::qbit_prism_server::coordinator::candidate_lease_tests::checkout_tests::terminal_checkout_cancel_releases_slot_without_resuming_work +qbit-prism-server::qbit_prism_server::coordinator::candidate_lease_tests::checkout_tests::terminal_sql_failure_preserves_successful_checkouts +qbit-prism-server::qbit_prism_server::coordinator::candidate_lease_tests::claim_release_tests::a_deferred_release_never_frees_a_row_another_frontend_took +qbit-prism-server::qbit_prism_server::coordinator::candidate_lease_tests::claim_release_tests::a_failed_release_is_retried_after_the_attempt_stops_and_frees_the_row_before_its_lease +qbit-prism-server::qbit_prism_server::coordinator::candidate_lease_tests::claim_release_tests::shutdown_attempts_each_deferred_release_once +qbit-prism-server::qbit_prism_server::coordinator::candidate_lease_tests::crash_after_offer_before_landing_recovers_on_another_frontend_without_a_second_offer +qbit-prism-server::qbit_prism_server::coordinator::candidate_lease_tests::early_chain_probe_skips_only_proven_stale_work_and_recovers_active_blocks +qbit-prism-server::qbit_prism_server::coordinator::candidate_lease_tests::first_offer_sample_is_skipped_for_an_unknown_or_skewed_proof_time +qbit-prism-server::qbit_prism_server::coordinator::candidate_lease_tests::landing_transaction_renews_across_expiries_and_terminal_contention_finishes_once +qbit-prism-server::qbit_prism_server::coordinator::candidate_lease_tests::offer_not_sent_tests::a_crash_between_the_refused_connect_and_the_row_update_recovers_as_delivery_unknown +qbit-prism-server::qbit_prism_server::coordinator::candidate_lease_tests::offer_not_sent_tests::a_refused_submitblock_returns_the_row_to_pending_and_lands_the_block_once_when_the_node_is_back +qbit-prism-server::qbit_prism_server::coordinator::candidate_lease_tests::offer_not_sent_tests::a_reset_after_the_submitblock_was_written_stays_unknown_and_is_never_offered_again +qbit-prism-server::qbit_prism_server::coordinator::candidate_lease_tests::offer_not_sent_tests::offer_warmup_tests::a_submitblock_answered_in_warmup_returns_the_row_to_pending_and_lands_the_block_once_when_the_node_is_warm +qbit-prism-server::qbit_prism_server::coordinator::candidate_lease_tests::offer_not_sent_tests::offer_warmup_tests::a_warmup_answer_after_the_submitblock_ran_keeps_the_offer_and_never_offers_again +qbit-prism-server::qbit_prism_server::coordinator::candidate_lease_tests::offer_shutdown_tests::shutdown_after_the_offer_reservation_sends_and_records_the_block_once +qbit-prism-server::qbit_prism_server::coordinator::candidate_lease_tests::offer_standby_tests::a_found_block_is_offered_at_once_when_the_configured_standby_is_not_streaming +qbit-prism-server::qbit_prism_server::coordinator::candidate_lease_tests::offer_standby_tests::with_the_standby_wait_off_a_found_block_is_offered_without_a_wait +qbit-prism-server::qbit_prism_server::coordinator::candidate_lease_tests::saturated_build_capacity_keeps_the_lease_until_one_candidate_confirms +qbit-prism-server::qbit_prism_server::coordinator::candidate_lease_tests::shutdown_tests::submit_loop_shutdown_hands_the_claim_back_for_an_immediate_takeover +qbit-prism-server::qbit_prism_server::coordinator::candidate_lease_tests::submit_disabled_tests::a_cluster_hold_stops_an_enabled_frontend_until_it_is_cleared +qbit-prism-server::qbit_prism_server::coordinator::candidate_lease_tests::submit_disabled_tests::a_submit_loop_with_block_submission_disabled_claims_nothing_until_shutdown +qbit-prism-server::qbit_prism_server::coordinator::candidate_lease_tests::submit_disabled_tests::an_offer_on_a_frontend_with_block_submission_disabled_is_refused_before_its_reservation +qbit-prism-server::qbit_prism_server::coordinator::candidate_lease_tests::submit_disabled_tests::the_hold_fence_waits_for_a_hold_being_set_and_never_for_the_cluster_row +qbit-prism-server::qbit_prism_server::coordinator::candidate_lease_tests::submit_loop_shutdown_drops_waiting_work_and_allows_recovery +qbit-prism-server::qbit_prism_server::coordinator::candidate_lease_tests::token_loss_between_heartbeats_is_fenced_immediately_before_submitblock +qbit-prism-server::qbit_prism_server::coordinator::clocked_revision_tests::clocked_read_matches_the_separate_clock_and_revision_reads +qbit-prism-server::qbit_prism_server::coordinator::commit_reconcile_tests::commit_reconcile_append_queued_on_the_order_lock_is_aborted_at_the_deadline +qbit-prism-server::qbit_prism_server::coordinator::commit_reconcile_tests::commit_reconcile_backend_terminated_during_commit_is_unknown +qbit-prism-server::qbit_prism_server::coordinator::commit_reconcile_tests::commit_reconcile_block_append_queued_past_its_deadline_still_captures_its_block +qbit-prism-server::qbit_prism_server::coordinator::commit_reconcile_tests::commit_reconcile_block_only_confirmed_before_its_bound_is_accepted +qbit-prism-server::qbit_prism_server::coordinator::commit_reconcile_tests::commit_reconcile_block_only_enqueue_behind_the_order_lock_is_not_cut_off +qbit-prism-server::qbit_prism_server::coordinator::commit_reconcile_tests::commit_reconcile_block_only_enqueue_past_the_bound_is_unknown +qbit-prism-server::qbit_prism_server::coordinator::commit_reconcile_tests::commit_reconcile_block_only_poll_errors_are_retried_then_unknown +qbit-prism-server::qbit_prism_server::coordinator::commit_reconcile_tests::commit_reconcile_block_only_still_pending_at_its_bound_is_unknown +qbit-prism-server::qbit_prism_server::coordinator::commit_reconcile_tests::commit_reconcile_block_whose_revision_moves_before_its_commit_is_captured_and_lands_once +qbit-prism-server::qbit_prism_server::coordinator::commit_reconcile_tests::commit_reconcile_block_whose_revision_moves_with_capture_off_is_refused_as_before +qbit-prism-server::qbit_prism_server::coordinator::commit_reconcile_tests::commit_reconcile_commit_held_past_the_deadline_is_accepted_late +qbit-prism-server::qbit_prism_server::coordinator::commit_reconcile_tests::commit_reconcile_commit_held_past_the_grace_is_unknown_and_lands_later +qbit-prism-server::qbit_prism_server::coordinator::commit_reconcile_tests::commit_reconcile_commit_refused_by_the_server_is_rejected_without_a_row +qbit-prism-server::qbit_prism_server::coordinator::commit_reconcile_tests::commit_reconcile_credited_block_resubmitted_as_its_revision_moves_is_a_duplicate +qbit-prism-server::qbit_prism_server::coordinator::commit_reconcile_tests::commit_reconcile_ledger_hook_refusal_sends_no_commit +qbit-prism-server::qbit_prism_server::coordinator::commit_reconcile_tests::commit_reconcile_plain_share_whose_revision_moves_before_its_commit_writes_nothing +qbit-prism-server::qbit_prism_server::coordinator::compact_decode_tests::cancelled_and_expired_runtime_metadata_decoders_retain_capacity +qbit-prism-server::qbit_prism_server::coordinator::compact_decode_tests::cancelled_runtime_snapshot_decoders_retain_build_capacity +qbit-prism-server::qbit_prism_server::coordinator::compact_decode_tests::cancelled_shared_window_owner_forces_full_snapshot_without_cloning +qbit-prism-server::qbit_prism_server::coordinator::d2_below_target_tests::block_only_credit_survives_a_reorg_that_disconnects_its_block +qbit-prism-server::qbit_prism_server::coordinator::d2_below_target_tests::block_only_proof_is_credited_network_work_only_once_the_block_is_active +qbit-prism-server::qbit_prism_server::coordinator::d2_below_target_tests::block_only_proof_the_node_rejects_fails_the_submission_without_credit +qbit-prism-server::qbit_prism_server::coordinator::d2_below_target_tests::block_only_proof_with_a_lost_submitblock_reply_is_credited_once_by_reconciliation +qbit-prism-server::qbit_prism_server::coordinator::d2_below_target_tests::share_passing_block_proof_is_credited_its_assigned_difficulty_at_once +qbit-prism-server::qbit_prism_server::coordinator::d2_bootstrap_tests::a_bootstrap_block_still_pays_carried_forward_balances +qbit-prism-server::qbit_prism_server::coordinator::d2_bootstrap_tests::a_single_share_from_another_miner_takes_the_whole_coinbase +qbit-prism-server::qbit_prism_server::coordinator::d2_bootstrap_tests::accepted_shares_below_the_2xx_readiness_gate_still_pay_the_window +qbit-prism-server::qbit_prism_server::coordinator::d2_bootstrap_tests::an_empty_ledger_pays_the_solver_through_a_synthetic_bootstrap_share +qbit-prism-server::qbit_prism_server::coordinator::first_job_lane_tests::a_first_job_is_not_queued_behind_a_rebuild_storm +qbit-prism-server::qbit_prism_server::coordinator::issued_batcher::tests::postgres::active_cancellation_keeps_live_peers_and_reconciles_all_gone +qbit-prism-server::qbit_prism_server::coordinator::miner_submit::acquire_tests::a_failed_window_probe_is_unknown_never_stale_and_enqueues_nothing +qbit-prism-server::qbit_prism_server::coordinator::miner_submit::acquire_tests::block_only_initial_probe_sql_error_cancel_and_original_deadline +qbit-prism-server::qbit_prism_server::coordinator::miner_submit::acquire_tests::block_only_poll_checkout_cancel_and_deadline_stay_unknown +qbit-prism-server::qbit_prism_server::coordinator::miner_submit::acquire_tests::block_only_poll_missing_and_sql_error_stay_unknown +qbit-prism-server::qbit_prism_server::coordinator::miner_submit::acquire_tests::block_only_poll_sql_cancel_preserves_checkout_success +qbit-prism-server::qbit_prism_server::coordinator::miner_submit::acquire_tests::block_only_poll_terminal_results_preserve_checkout_success +qbit-prism-server::qbit_prism_server::coordinator::miner_submit::acquire_tests::chain_state_checkout_preserves_snapshot_errors_and_cancellation +qbit-prism-server::qbit_prism_server::coordinator::miner_submit::acquire_tests::clock_and_block_only_duplicate_checkouts_are_lazy_and_cancel_once +qbit-prism-server::qbit_prism_server::coordinator::miner_submit::acquire_tests::clock_sql_wait_error_and_cancel_keep_the_successful_checkout +qbit-prism-server::qbit_prism_server::coordinator::miner_submit::acquire_tests::coordinator_reads_without_metrics_keep_results_and_emit_nothing +qbit-prism-server::qbit_prism_server::coordinator::same_tip_republish_tests::a_build_overtaken_by_a_revision_or_fee_change_is_discarded +qbit-prism-server::qbit_prism_server::coordinator::same_tip_republish_tests::a_build_overtaken_by_a_same_tip_republication_is_delivered_as_its_publication +qbit-prism-server::qbit_prism_server::coordinator::same_tip_republish_tests::a_build_overtaken_by_a_tip_change_is_discarded +qbit-prism-server::qbit_prism_server::coordinator::storm_evidence_tests::a_held_and_renewed_lease_before_any_call_is_not_offer_evidence +qbit-prism-server::qbit_prism_server::coordinator::storm_evidence_tests::a_known_call_keeps_its_call_evidence_through_orphan_terminalization +qbit-prism-server::qbit_prism_server::coordinator::storm_evidence_tests::a_pre_offer_retry_records_no_call_and_leaves_the_row_pending +qbit-prism-server::qbit_prism_server::coordinator::storm_evidence_tests::a_pre_offer_supersession_terminalizes_without_any_call +qbit-prism-server::qbit_prism_server::coordinator::storm_evidence_tests::a_replay_adoption_after_restart_records_an_unknown_offer_with_no_call +qbit-prism-server::qbit_prism_server::coordinator::storm_evidence_tests::an_interrupted_reservation_orphans_unknown_without_inventing_a_call +qbit-prism-server::qbit_prism_server::coordinator::storm_evidence_tests::offer_path_records_a_counted_call_with_a_durable_timestamp_and_reply +qbit-prism-server::qbit_prism_server::coordinator::window_incident_tests::claim_rebuild_has_the_canonical_bytes_refresh_built_for_the_job +qbit-prism-server::qbit_prism_server::coordinator::window_incident_tests::incident_1_lease_renewal_is_not_delayed_by_a_large_window_claim +qbit-prism-server::qbit_prism_server::coordinator::window_ref_tests::a_refresh_over_an_empty_ledger_publishes_an_empty_window_reference +qbit-prism-server::qbit_prism_server::coordinator::window_ref_tests::a_refresh_publishes_the_reference_for_the_window_it_captured +qbit-prism-server::qbit_prism_server::coordinator::window_ref_tests::the_entry_point_holds_a_window_read_permit_only_while_it_reads +qbit-prism-server::qbit_prism_server::coordinator::window_ref_tests::the_entry_point_reads_without_occupying_a_blocking_thread +qbit-prism-server::qbit_prism_server::coordinator::window_switch_tests::a_node_error_after_acceptance_settles_the_row_in_reconciliation_with_the_reason +qbit-prism-server::qbit_prism_server::coordinator::window_switch_tests::a_parent_landing_late_diverges_the_child_which_lands_as_issued_with_additive_balances +qbit-prism-server::qbit_prism_server::coordinator::window_switch_tests::a_refused_candidate_is_a_definite_rejection_not_an_unknown_outcome +qbit-prism-server::qbit_prism_server::coordinator::window_switch_tests::a_reorg_during_the_landing_is_seen_by_the_fresh_post_landing_observation +qbit-prism-server::qbit_prism_server::coordinator::window_switch_tests::builder_version_or_signer_mismatch_retries_without_a_rebuild +qbit-prism-server::qbit_prism_server::coordinator::window_switch_tests::ctv_candidate_rebuilds_from_its_stored_settlement_inputs_not_configuration +qbit-prism-server::qbit_prism_server::coordinator::window_switch_tests::landed_audit_is_authenticated_and_the_claim_continues_to_submitblock +qbit-prism-server::qbit_prism_server::coordinator::window_switch_tests::leased_candidate_submits_first_then_lands_as_issued_before_any_terminal_outcome +qbit-prism-server::qbit_prism_server::coordinator::window_switch_tests::the_rebuild_and_landing_never_wait_for_the_order_lock +qbit-prism-server::qbit_prism_server::coordinator::window_switch_tests::window_read_failures_retry_with_an_alert_and_never_abandon +qbit-prism-server::qbit_prism_server::ledger::candidates::park_tests::advanced_state_is_not_parked_by_a_stale_decode +qbit-prism-server::qbit_prism_server::ledger::candidates::park_tests::decode_join_failure_is_not_parked +qbit-prism-server::qbit_prism_server::ledger::candidates::park_tests::failing_parking_update_reports_no_parking +qbit-prism-server::qbit_prism_server::ledger::candidates::park_tests::failure_before_the_parking_commit_reports_no_parking +qbit-prism-server::qbit_prism_server::ledger::candidates::park_tests::invalid_row_is_parked_with_its_hash_kind_and_diagnosis +qbit-prism-server::qbit_prism_server::ledger::candidates::park_tests::lost_parking_commit_reply_is_reported_unknown +qbit-prism-server::qbit_prism_server::ledger::candidates::park_tests::malformed_rows_park_in_every_unfinished_state_with_their_offer_evidence +qbit-prism-server::qbit_prism_server::ledger::candidates::park_tests::refused_parking_transaction_reports_no_parking +qbit-prism-server::qbit_prism_server::ledger::candidates::park_tests::replacement_owner_is_not_parked_by_a_stale_decode +qbit-prism-server::qbit_prism_server::ledger::candidates::park_tests::transient_decode_database_error_is_not_parked +qbit-prism-server::qbit_prism_server::ledger::candidates::park_tests::unclassified_decode_error_is_not_parked +qbit-prism-server::qbit_prism_server::ledger::candidates::park_tests::unknown_persisted_offer_outcome_parks_as_lifecycle +qbit-prism-server::qbit_prism_server::ledger::candidates::storm_fault_tests::a_competitor_at_the_candidates_height_orphans_only_at_the_threshold +qbit-prism-server::qbit_prism_server::ledger::candidates::storm_fault_tests::a_failed_block_hash_read_never_abandons_a_block_the_node_already_holds +qbit-prism-server::qbit_prism_server::ledger::candidates::storm_fault_tests::a_failed_or_shifting_chain_observation_leaves_the_drain_recoverable +qbit-prism-server::qbit_prism_server::ledger::candidates::storm_fault_tests::a_lost_landing_acknowledgement_rolls_back_and_the_next_attempt_lands +qbit-prism-server::qbit_prism_server::ledger::candidates::storm_fault_tests::a_lost_renewal_acknowledgement_cancels_only_that_attempt +qbit-prism-server::qbit_prism_server::ledger::candidates::storm_fault_tests::a_lost_terminal_commit_acknowledgement_stays_terminal +qbit-prism-server::qbit_prism_server::ledger::candidates::storm_fault_tests::every_post_offer_chain_observation_fault_keeps_one_offer_and_reconciles +qbit-prism-server::qbit_prism_server::ledger::candidates::storm_fault_tests::every_pre_offer_chain_observation_fault_leaves_the_row_unoffered +qbit-prism-server::qbit_prism_server::ledger::connect::acquire::job_tests::postgres_compact_read_releases_checkout_before_blocking_decode +qbit-prism-server::qbit_prism_server::ledger::connect::acquire::job_tests::postgres_job_checkouts_start_on_first_poll_and_recover +qbit-prism-server::qbit_prism_server::ledger::connect::acquire::job_tests::postgres_job_sql_wait_errors_and_cancellation_do_not_change_checkout_samples +qbit-prism-server::qbit_prism_server::ledger::connect::acquire::probe_tests::a_dead_idle_connection_inside_the_probe_window_fails_one_statement_then_recovers +qbit-prism-server::qbit_prism_server::ledger::connect::acquire::probe_tests::a_dead_idle_connection_is_replaced_when_the_idle_gap_reaches_the_threshold +qbit-prism-server::qbit_prism_server::ledger::connect::acquire::test_support::postgres_fixture_cleans_up_after_setup_error_and_body_panic +qbit-prism-server::qbit_prism_server::ledger::connect::acquire::tests::postgres_checkout_boundaries_and_callers +qbit-prism-server::qbit_prism_server::ledger::instances::live_instance_tests::heartbeat_sql_observes_empty_stale_and_missing_table +qbit-prism-server::qbit_prism_server::ledger::jobs::compact_issued_batch::tests::canceled_begin_cleans_untracked_transaction_before_pool_reuse +qbit-prism-server::qbit_prism_server::ledger::migration::share_hashes::postgres_tests::a_connection_lost_while_checking_a_failed_attempt_keeps_both_errors +qbit-prism-server::qbit_prism_server::ledger::migration::share_hashes::postgres_tests::a_cursor_dropped_after_its_look_up_reads_as_nothing_pending +qbit-prism-server::qbit_prism_server::ledger::migration::share_hashes::postgres_tests::a_fresh_read_connects_again_while_the_server_turns_it_away +qbit-prism-server::qbit_prism_server::ledger::migration::share_hashes::postgres_tests::a_record_attempt_that_committed_before_it_failed_finishes_the_backfill +qbit-prism-server::qbit_prism_server::ledger::migration::share_hashes::postgres_tests::a_record_unanswered_before_its_commit_stops_at_once_and_never_closes_its_connection +qbit-prism-server::qbit_prism_server::ledger::migration::share_hashes::postgres_tests::a_record_whose_commit_reply_never_comes_is_read_afresh_at_its_deadline +qbit-prism-server::qbit_prism_server::ledger::migration::share_hashes::postgres_tests::a_record_whose_connection_drops_after_its_commit_is_read_on_a_fresh_connection +qbit-prism-server::qbit_prism_server::ledger::migration::share_hashes::postgres_tests::a_record_whose_connection_drops_before_its_commit_stops_for_a_rerun +qbit-prism-server::qbit_prism_server::ledger::window::payout_state::tests::a_snapshot_reads_its_writer_timeline_in_the_transaction_that_reads_its_rows +qbit-prism-server::qbit_prism_server::ledger::window::payout_state::tests::cancellation_and_sql_timeout_release_probe_connection +qbit-prism-server::qbit_prism_server::ledger::window::payout_state::tests::cancelled_hash_keeps_build_admission_until_blocking_cleanup_finishes +qbit-prism-server::qbit_prism_server::ledger::window::payout_state::tests::failed_guard_precedes_locked_share_history +qbit-prism-server::qbit_prism_server::ledger::window::payout_state::tests::lost_or_cancelled_commit_reply_never_returns_a_successful_probe +qbit-prism-server::qbit_prism_server::ledger::window::payout_state::tests::one_checkout_five_executions_and_one_hash_per_probe +qbit-prism-server::qbit_prism_server::ledger::window::payout_state::tests::revision_cutoff_and_digest_share_one_snapshot_with_weaker_isolation_control +qbit-prism-server::qbit_prism_server::ledger::window::payout_state::tests::same_revision_balance_identity_and_accepted_cutoff_are_fresh +qbit-prism-server::qbit_prism_server::ledger::window::payout_state::tests::unavailable_read_only_and_invalid_values_are_errors +qbit-prism-server::qbit_prism_server::ledger::window::snapshot_delta::tests::adversarial::corrupted_retained_inputs_are_refused_not_assembled +qbit-prism-server::qbit_prism_server::ledger::window::snapshot_delta::tests::adversarial::legacy_fixture_random_walk_matches_full_reader_or_refuses_for_a_named_reason +qbit-prism-server::qbit_prism_server::ledger::window::snapshot_delta::tests::bulk_fixture_retroactive_insert_and_future_eligibility_are_not_delta_safe +qbit-prism-server::qbit_prism_server::ledger::window::snapshot_delta::tests::concurrent_detach_pending_cannot_authorize_retained_rows +qbit-prism-server::qbit_prism_server::ledger::window::snapshot_delta::tests::deferred_confirmation_credit_is_an_idempotent_delta +qbit-prism-server::qbit_prism_server::ledger::window::snapshot_delta::tests::delta_crossing_revision_and_balance_match_full_at_same_anchor +qbit-prism-server::qbit_prism_server::ledger::window::snapshot_delta::tests::delta_windows_pass_landing_rederivation_and_neighbours_fail +qbit-prism-server::qbit_prism_server::ledger::window::snapshot_delta::tests::healthy_pool_rotation_and_reconnect_preserve_history_evidence +qbit-prism-server::qbit_prism_server::ledger::window::snapshot_delta::tests::multi_page_delta_and_margin_match_full_reader +qbit-prism-server::qbit_prism_server::ledger::window::snapshot_delta::tests::oversized_delta_and_margin_take_the_full_scan +qbit-prism-server::qbit_prism_server::ledger::window::snapshot_delta::tests::partition_crossing_interior_detach_and_restore_fall_back +qbit-prism-server::qbit_prism_server::ledger::window::snapshot_delta::tests::physical_failover::physical_async_failover_replaced_history_matches_full_read +qbit-prism-server::qbit_prism_server::ledger::window::snapshot_delta::tests::physical_failover::physical_async_failover_with_retroactive_fixture_matches_full_read +qbit-prism-server::qbit_prism_server::ledger::window::snapshot_delta::tests::random_retargets_and_inserts_match_full_reader +qbit-prism-server::qbit_prism_server::ledger::window::snapshot_delta::tests::retargets_up_and_down_advance_with_exact_windows +qbit-prism-server::qbit_prism_server::ledger::window::snapshot_delta::tests::same_leaf_detach_and_reattach_invalidate_acquisition_proof +qbit-prism-server::qbit_prism_server::ledger::window::snapshot_delta::tests::short_history_retarget_regressions_fall_back_but_rollback_gaps_advance +qbit-prism-server::qbit_prism_server::partitions::acquire_tests::postgres_checkout_wait_error_cancel_and_reuse +qbit-prism-server::qbit_prism_server::partitions::acquire_tests::postgres_real_23514_fallback_counts_three_checkouts_and_one_effect +qbit-prism-server::qbit_prism_server::partitions::acquire_tests::postgres_sql_wait_error_cancel_and_shutdown_keep_checkout_success +qbit-prism-server::qbit_prism_server::rollups::tests::postgres_batch_and_no_work_each_count_one_checkout +qbit-prism-server::qbit_prism_server::rollups::tests::postgres_begin_delay_error_and_cancel_keep_checkout_success +qbit-prism-server::qbit_prism_server::rollups::tests::postgres_exhausted_checkout_errors_cancel_and_reuse +qbit-prism-server::qbit_prism_server::rollups::tests::postgres_run_shutdown_cancels_checkout_once +qbit-prism-server::qbit_prism_server::rollups::tests::postgres_run_shutdown_during_sql_preserves_success_and_compatibility +qbit-prism-server::qbit_prism_server::rollups::tests::postgres_sql_cancellation_retains_success_and_rolls_back +qbit-prism-server::qbit_prism_server::rollups::tests::postgres_sql_wait_and_failure_keep_completed_checkout +qbit-prism-server::readiness_rpc::another_frontend_payout_revision_captures_the_same_parent_block_but_fences_its_share +qbit-prism-server::readiness_rpc::cached_ctv_work_revalidates_live_floors_and_fences_old_underfunded_jobs +qbit-prism-server::readiness_rpc::coordinator_reports_wrap_exhaustion_truthfully +qbit-prism-server::readiness_rpc::observed_readiness_failure_closes_cached_work_and_candidate_settlement +qbit-prism-server::refresh_liveness::bootstrap_disconnect_after_prepared_commit +qbit-prism-server::refresh_liveness::bootstrap_disconnect_after_publication +qbit-prism-server::refresh_liveness::bootstrap_disconnect_before_refresh +qbit-prism-server::refresh_liveness::bootstrap_disconnect_during_chain_observation +qbit-prism-server::refresh_liveness::bootstrap_disconnect_during_template_fetch +qbit-prism-server::refresh_liveness::bootstrap_disconnect_during_window_read +qbit-prism-server::refresh_liveness::ctv_chunk_metrics_count_an_attempt_whose_completion_failed_to_persist +qbit-prism-server::refresh_liveness::ctv_releases_the_claim_when_a_revision_bump_refuses_a_successful_attempt +qbit-prism-server::refresh_liveness::ctv_settles_after_the_replacement_build_budget_when_publication_keeps_failing +qbit-prism-server::refresh_liveness::ctv_settles_when_the_observed_tip_predates_the_pass +qbit-prism-server::refresh_liveness::ctv_yields_after_current_chunk_and_a_later_pass_finishes_remaining_rows +qbit-prism-server::refresh_liveness::every_session_disconnects_then_new_tip_work_serves_a_new_identity +qbit-prism-server::refresh_liveness::new_tip_serves_work_while_ctv_completion_is_held_then_ctv_yields +qbit-prism-server::refresh_liveness::startup_claim_leaves_socket_share_appends_live_and_then_makes_progress +qbit-prism-server::refresh_liveness::window_disconnect_after_prepared_commit +qbit-prism-server::refresh_liveness::window_disconnect_after_publication +qbit-prism-server::refresh_liveness::window_disconnect_before_refresh +qbit-prism-server::refresh_liveness::window_disconnect_during_chain_observation +qbit-prism-server::refresh_liveness::window_disconnect_during_template_fetch +qbit-prism-server::refresh_liveness::window_disconnect_during_window_read +qbit-prism-server::refresh_window_split::cancelled_reservation_retains_bounded_cache_without_holding_build_admission +qbit-prism-server::refresh_window_split::delayed_snapshot_read_does_not_restart_reanchor_age +qbit-prism-server::refresh_window_split::difficulty_change_invalidates_the_retained_window_weight +qbit-prism-server::refresh_window_split::new_share_and_payout_revision_each_invalidate_once +qbit-prism-server::refresh_window_split::older_refresh_cannot_publish_after_newer_tip_observation +qbit-prism-server::refresh_window_split::shares_after_capture_enter_the_next_window_without_rewriting_the_selected_one +qbit-prism-server::refresh_window_split::transaction_churn_does_not_extend_original_reanchor_interval +qbit-prism-server::refresh_window_split::transaction_churn_reuses_window_and_preserves_as_issued_work +qbit-prism-server::refresh_window_split::unchanged_empty_and_nonempty_refresh_probe_execution_counts +qbit-prism-server::rollups_postgres::competing_frontends_fold_each_share_once_including_late_clocked_work +qbit-prism-server::rollups_postgres::failed_bucket_write_rolls_back_the_watermark_and_every_other_grain +qbit-prism-server::self_check_cli::postgres_self_check_samples_before_local_startup_and_reports_missing_table +qbit-prism-server::session_cleanup_metrics::closed_pool_failures_retain_reservations_and_unattached_cleanup_recovers +qbit-prism-server::session_cleanup_metrics::release_and_drop_count_once_and_preserve_replacement_tokens +qbit-prism-server::session_cleanup_metrics::runtime_shutdown_counts_only_polled_background_cleanup +qbit-prism-server::session_cleanup_metrics::sql_wait_and_errors_remain_successful_checkouts +qbit-prism-server::session_cleanup_metrics::unpolled_and_cancelled_release_keep_the_drop_fallback_and_recover +qbit-prism-server::share_archive::an_archived_blocks_audit_is_served_from_its_sealed_bytes +qbit-prism-server::share_archive::an_interrupted_concurrent_detach_is_finalized +qbit-prism-server::share_archive::an_interrupted_detach_is_held_to_every_condition_before_finalize +qbit-prism-server::share_archive::archive_and_verify_round_trip_chain_and_tamper_detection +qbit-prism-server::share_archive::archive_requires_durable_parent_directories_before_recording +qbit-prism-server::share_archive::archive_rewrite_waits_for_detach_and_catalog_reconciliation +qbit-prism-server::share_archive::archive_waits_for_an_append_that_drew_its_share_seq_before_the_bound +qbit-prism-server::share_archive::chain_repairs_are_written_and_certified_in_order +qbit-prism-server::share_archive::detach_and_drop_refuse_a_partition_that_outgrew_its_verified_archive +qbit-prism-server::share_archive::drop_refuses_when_the_recorded_archive_no_longer_reads_back +qbit-prism-server::share_archive::failed_forced_archive_preserves_the_recorded_files +qbit-prism-server::share_archive::plan_allows_long_maintenance_reads_without_changing_pool_timeouts +qbit-prism-server::share_archive::plan_keeps_candidate_and_audit_blockers_in_one_snapshot +qbit-prism-server::share_archive::plan_names_each_blocker_and_separates_unknown_from_clear +qbit-prism-server::share_archive::reconcile_and_drop_hold_a_hand_detached_partition_to_the_seal +qbit-prism-server::share_archive::reconcile_holds_a_hand_detached_partition_to_every_condition +qbit-prism-server::share_archive::rejected_share_ids_survive_detach_drop_and_restore +qbit-prism-server::share_archive::restore_import_rebuilds_hashes_and_rejects_conflicting_credits +qbit-prism-server::share_archive::restore_import_refuses_a_gap_above_the_attached_range +qbit-prism-server::share_archive::restore_import_rejects_duplicate_share_ids +qbit-prism-server::share_archive::restore_import_rejects_ids_credited_in_departed_partitions +qbit-prism-server::share_archive::restore_import_rejects_overlapping_cataloged_bounds +qbit-prism-server::share_archive::restore_import_rejects_rows_at_or_below_the_rollup_watermark +qbit-prism-server::share_archive::restore_import_waits_for_an_inflight_rollup +qbit-prism-server::share_archive::restore_rebuilds_the_partition_and_attach_returns_it_to_the_parent +qbit-prism-server::share_archive::restore_refuses_reattachment_without_a_recorded_manifest +qbit-prism-server::share_archive::rollup_gate_measures_the_newest_committed_row_not_the_bound +qbit-prism-server::share_archive::seal_is_idempotent_and_safe_to_run_concurrently +qbit-prism-server::share_archive::seal_stores_a_slow_artifact_without_changing_pool_timeouts +qbit-prism-server::share_archive::unfinished_candidate_window_pins_an_earlier_partition +qbit-prism-server::share_archive::verification_drains_appends_after_retaining_rejected_ids +qbit-prism-server::share_archive::verification_refuses_a_concurrent_archive_rewrite +qbit-prism-server::share_partitions_postgres::a_duplicate_below_the_floor_is_still_detected_across_a_partition_boundary_and_after_a_detach +qbit-prism-server::share_partitions_postgres::an_append_past_the_last_bound_attaches_the_lead_and_lands_the_share +qbit-prism-server::share_partitions_postgres::append_preserves_old_rejected_share_ids +qbit-prism-server::share_partitions_postgres::changing_the_partition_width_keeps_names_unique_and_bounds_contiguous +qbit-prism-server::share_partitions_postgres::landing_attributes_the_solver_on_the_block_row_and_readers_survive_a_detach +qbit-prism-server::share_partitions_postgres::latest_evidence_counts_lifetime_shares_across_the_rollup_watermark +qbit-prism-server::share_partitions_postgres::partition_ensure_restores_the_lead_and_creates_nothing_when_it_is_intact +qbit-prism-server::share_partitions_postgres::share_id_probes_are_bounded_and_the_vardiff_lookup_is_still_exact +qbit-prism-server::share_partitions_postgres::the_append_probe_is_pruned_to_the_leaves_between_the_floor_and_the_sequence +qbit-prism-server::share_partitions_postgres::the_probe_floor_tracks_the_attached_bounds_after_the_width_grows +qbit-prism-server::signing_transition::a_failed_journal_write_rolls_the_reset_back +qbit-prism-server::signing_transition::a_fresh_heartbeat_refuses_the_reset +qbit-prism-server::signing_transition::a_halted_cluster_refuses_the_reset +qbit-prism-server::signing_transition::a_killed_frontend_refuses_until_its_heartbeat_is_stale +qbit-prism-server::signing_transition::a_lock_wait_ends_at_the_lock_timeout_and_changes_nothing +qbit-prism-server::signing_transition::a_pending_candidate_refuses_the_reset +qbit-prism-server::signing_transition::an_old_key_start_after_the_reset_repins_and_the_reset_is_repeatable +qbit-prism-server::signing_transition::nothing_lands_between_the_checks_and_the_reset +qbit-prism-server::signing_transition::offered_candidates_with_the_same_keys_refuse_the_reset +qbit-prism-server::signing_transition::stale_rows_of_every_other_state_do_not_refuse +qbit-prism-server::signing_transition::the_command_waits_for_the_cluster_row_and_decides_after_it +qbit-prism-server::signing_transition::the_reset_is_journaled_and_the_first_new_key_frontend_pins +qbit-prism-server::signing_transition::the_signing_reset_waits_for_a_job_cohort_fence +qbit-prism-server::signing_transition::without_confirm_nothing_is_checked_or_changed +qbit-prism-server::stratum_admission_postgres::ten_thousand_unsubscribed_connections_do_not_advance_postgres_sequence +qbit-prism-server::stratum_admission_postgres::unknown_job_budget_bounds_ledger_queries_and_a_capped_source_runs_none +qbit-prism-server::stratum_admission_postgres::wrap_exhaustion_leaves_subscription_retryable_and_disconnect_releases_guard +qbit-prism-server::stratum_restart_bind::a_configured_highdiff_port_is_bound_and_accepts +qbit-prism-server::stratum_restart_bind::held_highdiff_port_fails_startup_before_registration +qbit-prism-server::stratum_restart_bind::held_primary_port_fails_startup_before_registration +qbit-prism-server::stratum_restart_bind::listeners_are_bound_while_the_node_is_slow +qbit-prism-server::stratum_restart_bind::without_a_highdiff_port_only_the_primary_listener_is_bound +qbit-prism-server::throughput_floor::share_append_throughput_floor +qbit-prism-server::window_balance_order::current_balance_reads_keep_the_sql_order_under_a_non_bytewise_collation +qbit-prism-server::window_read_oracle::a_u128_maximum_difficulty_is_decoded_and_closes_the_window +qbit-prism-server::window_read_oracle::empty_windows_return_no_shares +qbit-prism-server::window_read_oracle::rejected_rows_are_excluded_and_consume_no_weight +qbit-prism-server::window_read_oracle::rows_on_the_anchor_are_credited_and_rows_after_it_are_not +qbit-prism-server::window_read_oracle::the_anchor_barrier_holds_at_large_millisecond_values +qbit-prism-server::window_read_oracle::the_row_that_crosses_the_weight_boundary_is_credited +qbit-prism-server::window_read_oracle::window_matches_the_oracle_across_page_boundaries +qbit-prism-server::window_read_oracle::writer_rows_follow_integer_milliseconds_past_2_43 +qbit-prism-server::window_reference::as_issued_balances_and_signed_outputs_survive_current_revision_change +qbit-prism-server::window_reference::as_issued_snapshot_preserves_persisted_balance_order +qbit-prism-server::window_reference::ascending_pages_match_native_rows_and_filter_gaps +qbit-prism-server::window_reference::database_timeout_and_cancellation_return_the_single_connection +qbit-prism-server::window_reference::empty_window_never_queries_share_history_and_errors_keep_their_types +qbit-prism-server::window_reference::external_blobs_are_immutable_and_remain_deletable_for_gc +qbit-prism-server::window_reference::fatal_cluster_refuses_window_revision_reads +qbit-prism-server::window_reference::landed_009_coexists_with_008_in_both_orders_under_runner +qbit-prism-server::window_reference::migration_preserves_legacy_payloads_and_applies_missing_008_below_009 +qbit-prism-server::window_reference::missing_range_count_and_digest_fail_without_partial_windows +qbit-prism-server::window_reference::payout_state::revision_and_balance_digest_cannot_mix_across_a_blocked_read +qbit-prism-server::window_reference::payout_state::revision_only_change_and_balance_change_are_distinct_without_reading_a_window +qbit-prism-server::window_reference::payout_state::timeout_and_cancelled_balance_read_release_the_connection +qbit-prism-server::window_reference::payout_state::unavailable_read_only_and_corrupt_balances_remain_errors +qbit-prism-server::window_reference::revision_balances_and_pages_use_one_repeatable_read_snapshot +qbit-prism-server::window_reference::window_sql_check_accepts_exactly_the_three_null_states +qbit-prism-server::work_refresh_metric::failing_refreshes_grow_the_refresh_stall_gauge_while_coverage_reads_one diff --git a/test/prism-load-regression.toml b/test/prism-load-regression.toml new file mode 100644 index 000000000..3821e3cc1 --- /dev/null +++ b/test/prism-load-regression.toml @@ -0,0 +1,113 @@ +# The load lanes' regression rule (#551, S6 of #487), read by +# scripts/prism_load_regress.py. +# +# A run's headline number regresses when it is worse than the trailing median +# of the same series by more than the allowed change: +# +# allowed = max(k * spread, min_relative_change * |median|, min_absolute_change) +# +# where a series is one lane, preset (and the preset file's sha256, so a +# changed preset starts a new baseline), runner class and fsync band, and only +# runs whose harness exited 0 count toward it. `spread` is #542's measured +# spread for the runner class, preset and number (the coefficient of variation +# over every run, `all.cv`, times the median) when #542's variance document is +# given (scripts/prism_load_probe.py `variance --out`, checked in as +# test/prism-load-variance.json, which the evidence action passes when it +# exists), and otherwise the baseline's own scaled median absolute deviation +# (1.4826 * MAD), which is provisional. #542 measures the rate, the client ACK +# p50/p99 and the tip-to-last-notify p99; the other numbers stay provisional. +# +# PROVISIONAL: no measured variance document is checked in yet, so `k`, +# `window`, `min_baseline`, the per-metric floors and the fsync band edges +# below are placeholders chosen to be quiet, not calibrated. Set +# `provisional = false` once they are; a verdict still says provisional while +# any number it compared used the provisional spread. The rule is +# report-only either way: it opens or updates one tracking issue and never +# fails a job or blocks a merge (#487's "Results, evidence and regression +# detection"). When #542 finds VM-to-VM spread much larger than run-to-run +# spread on one VM (`ab_ratio` below), the summary says so: the path is then +# #511's interleaved A/B driver on one VM, not a wider threshold. + +schema = "qbit.prism.load-regression-config.v1" +provisional = true + +[rule] +# Lanes the rule reads. The weekly soak (#575) has its own bounds, still being +# calibrated (#575), and L3's rows are compared by #557's bundle, not here. +lanes = ["L2"] +# Trailing runs of the same series the median is taken over (#487: "the +# trailing 14-run median"). Flagged runs are left out of it; a streak of +# this many flagged runs is taken as the accepted new level. +window = 14 +# Fewer runs than this in the series gives no verdict ("unknown"), not a pass. +min_baseline = 5 +# Multiples of the spread a run may be worse by. +k = 3.0 +# The ratio of VM-to-VM to run-to-run spread above which the summary points at +# #511's same-VM A/B instead. Read only from a variance file. +ab_ratio = 2.0 + +[fsync_bands] +# pg_test_fsync's fdatasync cost for one 8 kB write, in microseconds per +# operation, on the filesystem under the harness's cluster. A run is compared +# only with runs in its own band; one whose cost was not measured is in no +# band and gets no verdict. `edges_usecs` are the upper bounds of every band +# but the last. +edges_usecs = [1000.0, 4000.0] +names = ["fast", "medium", "slow"] + +# The headline numbers the rule compares. `scope = "phase"` compares each of +# the run's artifact phases on its own; `worse` says which direction is a +# regression. Units are in the key. + +[[metric]] +key = "achieved_rate_shares_per_second" +scope = "phase" +worse = "lower" +min_relative_change = 0.05 +min_absolute_change = 1.0 + +[[metric]] +key = "client_ack_p50_ms" +scope = "phase" +worse = "higher" +min_relative_change = 0.25 +min_absolute_change = 2.0 + +[[metric]] +key = "client_ack_p99_ms" +scope = "phase" +worse = "higher" +min_relative_change = 0.25 +min_absolute_change = 5.0 + +# Server-side, from qbit_prism_share_ack_seconds{result="accepted"}: complete +# submit frame receipt to completed response write, which includes the share +# append. +[[metric]] +key = "server_ack_mean_ms" +scope = "phase" +worse = "higher" +min_relative_change = 0.25 +min_absolute_change = 1.0 + +[[metric]] +key = "order_lock_mean_waiters" +scope = "phase" +worse = "higher" +min_relative_change = 0.5 +min_absolute_change = 0.5 + +[[metric]] +key = "peak_rss_mib" +scope = "phase" +worse = "higher" +min_relative_change = 0.15 +min_absolute_change = 32.0 + +[[metric]] +key = "tip_last_notify_p99_ms" +scope = "run" +worse = "higher" +min_relative_change = 0.25 +min_absolute_change = 50.0 diff --git a/test/prism-native-tests.sh b/test/prism-native-tests.sh new file mode 100755 index 000000000..faa98a02d --- /dev/null +++ b/test/prism-native-tests.sh @@ -0,0 +1,178 @@ +#!/usr/bin/env bash +# Run the native PRISM test suites against a disposable PostgreSQL. +# +# test/prism-native-tests.sh every workspace test, the three +# explicit #[ignore] database targets, +# and the gate manifest check: the +# prism-native-postgres CI job +# test/prism-native-tests.sh live the real qbitd regtest suite +# test/prism-native-tests.sh replica the physical replica and failover suite +# test/prism-native-tests.sh cargo-args <cargo test arguments...> +# +# Database mode runs storm scenarios at the observed 3,120 candidates. Set +# PRISM_TEST_STORM_CANDIDATES=100 for the reduced cardinality CI uses; +# cargo-args mode keeps the Rust default unless explicitly overridden. +# +# Inputs: PRISM_TEST_DATABASE_URL (otherwise a private cluster is started from +# PRISM_TEST_PG_BIN_DIR or `pg_config --bindir`), PRISM_TEST_PG_BIN_DIR, and +# QBITD_BIN (otherwise QBIT_BIN_DIR/qbitd, otherwise qbitd on PATH). Once every +# input a mode needs is present, PRISM_TEST_REQUIRE_INTEGRATION=1 is exported +# so a gated test that cannot run fails instead of skipping, and the gate's +# execution manifest is written to PRISM_TEST_GATE_MANIFEST (default: a file in +# this run's scratch directory). See docs/prism-integration-test-gate.md. +set -euo pipefail + +PRISM_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +cd "${PRISM_ROOT}" +mode="${1:-database}" +case "${mode}" in + database|live|replica|cargo-args) ;; + *) + echo "Usage: test/prism-native-tests.sh [database|live|replica|cargo-args <cargo test arguments...>]" >&2 + exit 1 + ;; +esac + +# CI invokes the shard runner directly and keeps the Rust reader's default +# of 100. This local harness exercises the incident cardinality unless the +# caller supplies a value; preserve even an empty value for Rust to reject. +if [[ "${mode}" == database ]]; then + export PRISM_TEST_STORM_CANDIDATES="${PRISM_TEST_STORM_CANDIDATES-3120}" +fi + +prism_test_tmp="$(mktemp -d -t prism-native-tests.XXXXXX)" +prism_pg_bin="" +prism_cluster_started=0 +cleanup() { + if [[ "${prism_cluster_started}" == 1 ]]; then + "${prism_pg_bin}/pg_ctl" -D "${prism_test_tmp}/data" -m immediate stop >/dev/null 2>&1 || true + fi + rm -rf "${prism_test_tmp}" +} +trap cleanup EXIT + +prism_pg_bin="${PRISM_TEST_PG_BIN_DIR:-}" +if [[ -z "${prism_pg_bin}" ]] && command -v pg_config >/dev/null 2>&1; then + prism_pg_bin="$(pg_config --bindir)" +fi +if [[ -x "${prism_pg_bin}/initdb" && -x "${prism_pg_bin}/pg_ctl" ]]; then + export PRISM_TEST_PG_BIN_DIR="${prism_pg_bin}" +else + unset PRISM_TEST_PG_BIN_DIR + if [[ "${mode}" == replica ]]; then + echo 'Install PostgreSQL server tools and set PRISM_TEST_PG_BIN_DIR for physical replica tests.' >&2 + exit 1 + fi +fi + +if [[ -z "${PRISM_TEST_DATABASE_URL:-}" ]]; then + [[ -n "${PRISM_TEST_PG_BIN_DIR:-}" ]] || { + echo 'Set PRISM_TEST_DATABASE_URL or install PostgreSQL server tools for isolated native tests.' >&2 + exit 1 + } + "${prism_pg_bin}/initdb" -D "${prism_test_tmp}/data" -A trust --no-locale -E UTF8 > "${prism_test_tmp}/initdb.log" + # `initdb -A trust` admits any reader credential over TCP, valid or not, so a + # test whose subject is authentication fails. Keep trust for the bootstrap + # role that creates the fixtures; require SCRAM of every other TCP role. + prism_test_user="$(id -un)" + cat > "${prism_test_tmp}/data/pg_hba.conf" <<EOF +local all all trust +host all "${prism_test_user}" 127.0.0.1/32 trust +host all "${prism_test_user}" ::1/128 trust +host all all 127.0.0.1/32 scram-sha-256 +host all all ::1/128 scram-sha-256 +EOF + # A private socket directory and random loopback port avoid production DBs. + started=0 + for _ in 1 2 3 4 5; do + prism_test_port=$((20000 + RANDOM % 20000)) + if "${prism_pg_bin}/pg_ctl" -D "${prism_test_tmp}/data" -l "${prism_test_tmp}/postgres.log" \ + -o "-h 127.0.0.1 -p ${prism_test_port} -k ${prism_test_tmp}" start >/dev/null 2>&1; then + started=1 + prism_cluster_started=1 + break + fi + done + [[ "${started}" == 1 ]] || { cat "${prism_test_tmp}/postgres.log" >&2; exit 1; } + export PRISM_TEST_DATABASE_URL="postgresql://${prism_test_user}@127.0.0.1:${prism_test_port}/postgres" +fi + +# qbitd: QBITD_BIN, then QBIT_BIN_DIR, then PATH. Only the live suite refuses +# to go on without it; the other modes note its absence below. +qbitd="${QBITD_BIN:-${QBIT_BIN_DIR:+${QBIT_BIN_DIR}/qbitd}}" +if [[ -z "${qbitd}" ]]; then qbitd="$(command -v qbitd || true)"; fi +if [[ -n "${qbitd}" && -x "${qbitd}" ]]; then + export QBITD_BIN="${qbitd}" +else + unset QBITD_BIN + if [[ "${mode}" == live ]]; then + echo 'Set QBITD_BIN to a qbitd executable for native regtest integration.' >&2 + exit 1 + fi +fi + +# Required mode: once every input the mode needs is guaranteed, a gated test +# that cannot run fails instead of skipping, exactly as in CI. The default mode +# runs the whole workspace, so it needs all three inputs; the explicit modes +# have already refused to start without theirs, and a cargo-args run names its +# own tests, so a missing input there should fail loudly too. +required=1 +if [[ "${mode}" == database ]] && [[ -z "${PRISM_TEST_PG_BIN_DIR:-}" || -z "${QBITD_BIN:-}" ]]; then + required=0 +fi +if [[ "${required}" == 1 ]]; then + export PRISM_TEST_REQUIRE_INTEGRATION=1 +else + unset PRISM_TEST_REQUIRE_INTEGRATION + echo 'prism-native-tests: QBITD_BIN or PostgreSQL server tools are missing, so the gated tests that need them will skip; the manifest below is printed but not checked. Provide both for a run identical to CI.' >&2 +fi +export PRISM_TEST_GATE_MANIFEST="${PRISM_TEST_GATE_MANIFEST:-${prism_test_tmp}/gate-manifest.txt}" +# The manifest describes this run only. +: > "${PRISM_TEST_GATE_MANIFEST}" +log="${prism_test_tmp}/tests.log" + +run_tests() { + "$@" 2>&1 | tee -a "${log}" +} + +case "${mode}" in + live) + run_tests cargo test --locked -p qbit-prism-server --test live_regtest -- --nocapture + ;; + replica) + run_tests cargo test --locked -p qbit-prism-server --test postgres_failover -- --nocapture + ;; + cargo-args) + # Run an arbitrary cargo test invocation against the cluster this script + # prepared, so long or #[ignore]d runs can use the disposable database too. + shift + [[ $# -gt 0 ]] || { + echo 'Usage: test/prism-native-tests.sh cargo-args <cargo test arguments...>' >&2 + exit 1 + } + run_tests cargo test "$@" + ;; + database) + run_tests cargo test --locked --workspace --all-targets -- --nocapture + run_tests cargo test --locked -p qbit-prism-server --test stratum_admission_postgres \ + -- --ignored --nocapture --exact ten_thousand_unsubscribed_connections_do_not_advance_postgres_sequence + run_tests cargo test --locked -p qbit-prism-server --test observability_database -- --ignored --nocapture + run_tests cargo test --locked -p qbit-prism-server --test compact_runtime_scale --release \ + -- --ignored --nocapture --test-threads=1 + run_tests cargo test --locked -p qbit-prism-server --test issued_job_dependency -- --ignored --nocapture --test-threads=2 + ;; +esac + +if [[ "${mode}" == database && "${required}" == 1 ]]; then + python3 scripts/check_gate_manifest.py \ + --manifest "${PRISM_TEST_GATE_MANIFEST}" \ + --expected test/prism-gated-tests.txt \ + --log "${log}" +else + echo "prism-native-tests: gate manifest for this run (${PRISM_TEST_GATE_MANIFEST}):" + if [[ -s "${PRISM_TEST_GATE_MANIFEST}" ]]; then + sort "${PRISM_TEST_GATE_MANIFEST}" + else + echo '(no gated test recorded a decision)' + fi +fi diff --git a/test/prism-nightly-gated-tests.txt b/test/prism-nightly-gated-tests.txt new file mode 100644 index 000000000..29b519021 --- /dev/null +++ b/test/prism-nightly-gated-tests.txt @@ -0,0 +1,22 @@ +# Every opt-in (#[ignore]) gated test the nightly live-nightly job runs +# explicitly with --ignored --exact, one `<package>::<binary>::<test path>` id +# per line, sorted. The job runs each listed binary once, with its ids: +# qbit-prism-server::live_regtest and qbit-prism-load::soak_smoke (#575) +# today. The prism-native-postgres PR job never runs these, so they +# are listed here and not in test/prism-gated-tests.txt; the job proves each +# executed with scripts/check_gate_manifest.py (#521). +# Each id is also named in test/e2e-scenarios.toml (#543). +qbit-prism-load::soak_smoke::the_soak_smoke_preset_rolls_over_archives_and_holds_its_gates +qbit-prism-server::live_regtest::chain_events_load_tests::nightly_2000_sessions_on_one_frontend_at_a_1s_reanchor_get_work_on_every_tip +qbit-prism-server::live_regtest::chain_events_load_tests::nightly_deep_reorg_under_2000_sessions_keeps_balances_exact_and_serves_every_session +qbit-prism-server::live_regtest::chain_events_load_tests::nightly_external_tips_at_target_spacing_reach_every_session_on_both_frontends +qbit-prism-server::live_regtest::chain_events_load_tests::nightly_lost_race_under_2000_sessions_orphans_once_and_serves_every_session +qbit-prism-server::live_regtest::chain_events_load_tests::nightly_node_killed_after_accepting_under_2000_sessions_reconciles_the_lost_block_once +qbit-prism-server::live_regtest::chain_events_load_tests::nightly_node_loses_its_peer_under_2000_sessions_orphans_the_isolated_block_and_recovers +qbit-prism-server::live_regtest::chain_events_load_tests::nightly_two_frontends_on_two_nodes_disagree_briefly_and_converge_under_2000_sessions +qbit-prism-server::live_regtest::clock_jump_tests::wall_clock_jumps_keep_windows_claims_and_payouts_and_land_every_block_once +qbit-prism-server::live_regtest::dense_soak_tests::dense_cadence_soak_lands_every_block_rebuilds_in_budget_and_holds_rss_flat +qbit-prism-server::live_regtest::disk_exhaustion_tests::postgres_volume_full_refuses_shares_cleanly_and_recovers_once_space_is_freed +qbit-prism-server::live_regtest::node_outage_tests::node_killed_after_accepting_and_reindexed_lands_the_block_once +qbit-prism-server::live_regtest::weighted_recipients_tests::nightly_mainnet_floor_weighted_recipients_share_to_spend +qbit-prism-server::live_regtest::weighted_recipients_tests::recipient_boundary::nightly_1001_recipients_cross_the_fanout_chunk_boundary_to_spent_outputs diff --git a/test/prism-postgres-lib.sh b/test/prism-postgres-lib.sh deleted file mode 100644 index 0493cc034..000000000 --- a/test/prism-postgres-lib.sh +++ /dev/null @@ -1,43 +0,0 @@ -#!/usr/bin/env bash - -set -euo pipefail - -# Readiness gate for the throwaway Postgres containers the PRISM tests spin up. -# -# The official Postgres image initialises a fresh data directory by running a -# temporary server that listens on the unix socket only (listen_addresses=''), -# then shuts it down and starts the real one. `pg_isready` over that socket -# therefore reports "accepting connections" during initialisation, and the very -# next client call can hit `FATAL: database "qbit" does not exist`, `FATAL: the -# database system is shutting down`, or -- once the socket file is gone -- -# `connection to server on socket "/var/run/postgresql/.s.PGSQL.5432" failed: -# No such file or directory`. The window is short, so the race only bites when -# the image is already cached and the first probe lands inside it. -# -# Gate on the TCP listener instead (absent until the real server is up) and -# confirm with a real query over the socket the tests actually use. -wait_for_prism_postgres_container() { - local container="$1" - local user="${2:-qbit}" - local database="${3:-qbit}" - local timeout_seconds="${4:-120}" - local deadline=$((SECONDS + timeout_seconds)) - - while :; do - if docker exec "${container}" \ - pg_isready -h 127.0.0.1 -U "${user}" -d "${database}" >/dev/null 2>&1 && - echo 'SELECT 1;' | docker exec -i "${container}" \ - psql --no-psqlrc --set ON_ERROR_STOP=1 -U "${user}" -d "${database}" \ - >/dev/null 2>&1; then - return 0 - fi - - if [[ "${SECONDS}" -ge "${deadline}" ]]; then - echo "timed out waiting for PRISM Postgres container: ${container}" >&2 - docker logs "${container}" >&2 || true - return 1 - fi - - sleep 1 - done -} diff --git a/test/prism-weekly-gated-tests.txt b/test/prism-weekly-gated-tests.txt new file mode 100644 index 000000000..db334b8af --- /dev/null +++ b/test/prism-weekly-gated-tests.txt @@ -0,0 +1,14 @@ +# Every opt-in (#[ignore]) gated test the weekly live-weekly job of +# .github/workflows/prism-load-nightly.yml runs explicitly with --ignored +# --exact, one `<package>::<binary>::<test path>` id per line, sorted. Only +# qbit-prism-server::live_regtest ids: the job runs that one binary. Neither +# the prism-native-postgres PR job nor live-nightly runs these; the job proves +# each executed with scripts/check_gate_manifest.py (#487 L4). +qbit-prism-server::live_regtest::chain_events_load_tests::weekly_dense_soak_under_2000_sessions_lands_every_block_and_holds_rss_flat +qbit-prism-server::live_regtest::chain_events_load_tests::weekly_external_tips_at_target_spacing_reach_every_session_on_both_frontends +qbit-prism-server::live_regtest::chain_events_load_tests::weekly_node_killed_before_submitblock_under_2000_sessions_lands_the_block_once +qbit-prism-server::live_regtest::chain_events_load_tests::weekly_node_restarted_into_warmup_under_2000_sessions_lands_the_block_once +qbit-prism-server::live_regtest::chain_events_load_tests::weekly_node_stopped_after_accepting_under_2000_sessions_lands_the_block_once +qbit-prism-server::live_regtest::chain_events_load_tests::weekly_node_stopped_before_submitblock_under_2000_sessions_lands_the_block_once +qbit-prism-server::live_regtest::migration_lifecycle_tests::weekly_2x_ledger_migrates_mines_reconciles_and_restores_in_isolation +qbit-prism-server::live_regtest::migration_lifecycle_tests::weekly_mainnet_shaped_2x_ledger_cuts_over_measured_mines_and_restores_in_isolation diff --git a/test/test-prism-combined-regtest.sh b/test/test-prism-combined-regtest.sh deleted file mode 100644 index c392ea4ad..000000000 --- a/test/test-prism-combined-regtest.sh +++ /dev/null @@ -1,577 +0,0 @@ -#!/usr/bin/env bash -set -euo pipefail - -ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" -source "${ROOT_DIR}/test/test-lib.sh" -# shellcheck source=test/prism-postgres-lib.sh -source "${ROOT_DIR}/test/prism-postgres-lib.sh" - -DATADIR="${DATADIR:-$(mktemp -d -t qbit-prism-combined.XXXXXX)}" -RPC_USER="${RPC_USER:-qbitrpc}" -RPC_PASSWORD="${RPC_PASSWORD:-change-this}" -RPC_PORT="${RPC_PORT:-18456}" -WALLET_NAME="${WALLET_NAME:-prism-combined}" -STRATUM_PORT="${PRISM_STRATUM_PORT:-3342}" -AUDIT_PORT="${PRISM_AUDIT_PORT:-3343}" -MINER_COUNT=6 -MINER_TIMEOUT_SECONDS="${QBIT_PRISM_LIVE_MINER_TIMEOUT_SECONDS:-120}" -EVIDENCE_PATH="${DATADIR}/prism-live-evidence.json" -COORDINATOR_LOG="${DATADIR}/prism-coordinator.log" -POSTGRES_IMAGE="${QBIT_PRISM_POSTGRES_IMAGE:-postgres:16-alpine}" -POSTGRES_CONTAINER="${QBIT_PRISM_POSTGRES_CONTAINER:-qbit-prism-combined-pg-$$}" -MANIFEST_SIGNING_SEED_HEX="${QBIT_PRISM_MANIFEST_SIGNING_SEED_HEX:-4242424242424242424242424242424242424242424242424242424242424242}" -LEDGER_ATTESTATION_SIGNING_SEED_HEX="${QBIT_PRISM_LEDGER_ATTESTATION_SIGNING_SEED_HEX:-4343434343434343434343434343434343434343434343434343434343434343}" -LEDGER_WRITER_SESSION_TOKEN="${QBIT_PRISM_LEDGER_WRITER_SESSION_TOKEN:-prism-combined-regtest-session}" -ASERT_MOCKTIME_BASE="${QBIT_PRISM_LIVE_MOCKTIME_BASE:-1738713603}" -ASERT_TIME_STEP="${QBIT_PRISM_LIVE_ASERT_TIME_STEP:-1}" - -resolve_qbit_binaries -read -r -a CARGO_CMD <<< "${CARGO:-cargo}" -require_executable "${CARGO_CMD[0]}" -# When QBIT_PRISM_EXTERNAL_PSQL_COMMAND is set, use an already-running Postgres -# (e.g. a local cluster) instead of provisioning a Docker container. -EXTERNAL_PSQL="${QBIT_PRISM_EXTERNAL_PSQL_COMMAND:-}" -[[ -n "${EXTERNAL_PSQL}" ]] || require_executable docker -require_executable python3 -require_qbit_src_helper "test/functional/test_framework/blocktools.py" "the combined PRISM regtest proof" - -coordinator_pid="" -postgres_started=0 -miner_pids=() - -cleanup() { - for pid in "${miner_pids[@]:-}"; do - kill "${pid}" >/dev/null 2>&1 || true - done - if [[ -n "${coordinator_pid}" ]]; then - kill "${coordinator_pid}" >/dev/null 2>&1 || true - fi - if [[ "${postgres_started}" == "1" ]]; then - docker rm -f "${POSTGRES_CONTAINER}" >/dev/null 2>&1 || true - fi - stop_qbitd - remove_datadir "${DATADIR}" -} -trap cleanup EXIT - -if [[ "$(ulimit -n)" == "unlimited" ]]; then - ulimit -n 1024 || true -fi - -if [[ -n "${EXTERNAL_PSQL}" ]]; then - PSQL_COMMAND="${EXTERNAL_PSQL}" - deadline=$((SECONDS + 60)) - until echo 'SELECT 1;' | ${PSQL_COMMAND} >/dev/null 2>&1; do - if [[ "${SECONDS}" -ge "${deadline}" ]]; then - echo "timed out waiting for external PRISM Postgres" >&2 - exit 1 - fi - sleep 1 - done -else - docker rm -f "${POSTGRES_CONTAINER}" >/dev/null 2>&1 || true - docker run \ - --rm \ - --detach \ - --name "${POSTGRES_CONTAINER}" \ - -e POSTGRES_USER=qbit \ - -e POSTGRES_PASSWORD=qbit \ - -e POSTGRES_DB=qbit \ - "${POSTGRES_IMAGE}" >/dev/null - postgres_started=1 - PSQL_COMMAND="docker exec -i ${POSTGRES_CONTAINER} psql -U qbit -d qbit" - - wait_for_prism_postgres_container "${POSTGRES_CONTAINER}" -fi - -"${QBITD_BIN}" \ - -regtest \ - -asert \ - -p2mronly=1 \ - -daemonwait \ - -server \ - -listen=0 \ - -maxconnections=0 \ - -dnsseed=0 \ - -fixedseeds=0 \ - -rpcbind=127.0.0.1 \ - -rpcallowip=127.0.0.1 \ - -rpcport="${RPC_PORT}" \ - -rpcuser="${RPC_USER}" \ - -rpcpassword="${RPC_PASSWORD}" \ - -fallbackfee=0.00001000 \ - -datadir="${DATADIR}" >/dev/null - -wait_for_qbit_rpc_state ready 60 -qbit_rpc createwallet "${WALLET_NAME}" >/dev/null -qbit_rpc setmocktime "${ASERT_MOCKTIME_BASE}" >/dev/null - -miner_usernames=() -for miner_index in $(seq 1 "${MINER_COUNT}"); do - miner_usernames+=("$(qbit_rpc -rpcwallet="${WALLET_NAME}" getnewaddress "" p2mr)") -done - -initial_template_json="$(qbit_rpc getblocktemplate '{"rules":["segwit"]}')" -share_weights_json="$( - TEMPLATE_JSON="${initial_template_json}" \ - MINERS_JSON="$(printf '%s\n' "${miner_usernames[@]}" | python3 -c 'import json,sys; print(json.dumps([line.strip() for line in sys.stdin if line.strip()]))')" \ - python3 <<'PY' -import json -import os - -template = json.loads(os.environ["TEMPLATE_JSON"]) -miners = json.loads(os.environ["MINERS_JSON"]) - -def target_from_compact(bits_hex: str) -> int: - compact = int(bits_hex, 16) - size = compact >> 24 - mantissa = compact & 0x007fffff - if size <= 3: - return mantissa >> (8 * (3 - size)) - return mantissa << (8 * (size - 3)) - -pow_limit_target = target_from_compact("207fffff") -template_target = target_from_compact(template["bits"]) -network_difficulty = max(1, (pow_limit_target * 1_000_000) // template_target) -weights = [ - network_difficulty * 5, - network_difficulty, - max(1, network_difficulty // 4), - 10, - 1, - 1, -] -print(json.dumps(dict(zip(miners, weights)), separators=(",", ":"))) -PY -)" - -( - cd "${ROOT_DIR}" - export QBIT_RPC_HOST=127.0.0.1 - export QBIT_RPC_PORT="${RPC_PORT}" - export QBIT_RPC_USER="${RPC_USER}" - export QBIT_RPC_PASSWORD="${RPC_PASSWORD}" - export PRISM_STRATUM_BIND=127.0.0.1 - export PRISM_STRATUM_PORT="${STRATUM_PORT}" - export PRISM_AUDIT_BIND=127.0.0.1 - export PRISM_AUDIT_PORT="${AUDIT_PORT}" - export PRISM_MIN_READY_MINERS="${MINER_COUNT}" - export PRISM_EVIDENCE_PATH="${EVIDENCE_PATH}" - export PRISM_AUDIT_DIR="${DATADIR}" - export PRISM_STRATUM_SHARE_DIFF=0.000000001 - export PRISM_STRATUM_SHARE_WEIGHTS_JSON="${share_weights_json}" - export PRISM_STOP_AFTER_BLOCK=0 - export PRISM_MAX_BLOCKS=99 - export PRISM_POSTGRES_PSQL_COMMAND="${PSQL_COMMAND}" - export PRISM_POSTGRES_INIT_SCHEMA=1 - export PRISM_LEDGER_WRITER_SESSION_TOKEN="${LEDGER_WRITER_SESSION_TOKEN}" - export PRISM_ALLOW_FIXED_LEDGER_SESSION_TOKEN=1 - export PRISM_MANIFEST_SIGNING_SEED_HEX="${MANIFEST_SIGNING_SEED_HEX}" - export PRISM_LEDGER_ATTESTATION_SIGNING_SEED_HEX="${LEDGER_ATTESTATION_SIGNING_SEED_HEX}" - export PRISM_ALLOW_BUNDLE_EMBEDDED_LEDGER_KEY=1 - python3 lab/prism/prism_coordinator.py -) >"${COORDINATOR_LOG}" 2>&1 & -coordinator_pid="$!" - -STRATUM_PORT="${STRATUM_PORT}" python3 <<'PY' -import os -import socket -import time - -deadline = time.time() + 30 -port = int(os.environ["STRATUM_PORT"]) -while time.time() < deadline: - try: - with socket.create_connection(("127.0.0.1", port), timeout=0.5): - raise SystemExit(0) - except OSError: - time.sleep(0.25) -raise SystemExit("timed out waiting for PRISM Stratum port") -PY - -AUDIT_PORT="${AUDIT_PORT}" python3 <<'PY' -import os -import socket -import time - -deadline = time.time() + 30 -port = int(os.environ["AUDIT_PORT"]) -while time.time() < deadline: - try: - with socket.create_connection(("127.0.0.1", port), timeout=0.5): - raise SystemExit(0) - except OSError: - time.sleep(0.25) -raise SystemExit("timed out waiting for PRISM audit API port") -PY - -before_height="$(qbit_rpc getblockcount)" -for miner_index in $(seq 1 "${MINER_COUNT}"); do - miner_username="${miner_usernames[$((miner_index - 1))]}" - ( - cd "${ROOT_DIR}" - STRATUM_HOST=127.0.0.1 \ - STRATUM_PORT="${STRATUM_PORT}" \ - MINER_USERNAME="${miner_username}" \ - MINER_WALLET_NAME="prism-combined-miner-${miner_index}" \ - MINER_PASSWORD=x \ - MINER_TIMEOUT_SECONDS="${MINER_TIMEOUT_SECONDS}" \ - MINER_USERNAME_TIMEOUT_SECONDS=30 \ - QBIT_RPC_HOST=127.0.0.1 \ - QBIT_RPC_PORT="${RPC_PORT}" \ - QBIT_RPC_USER="${RPC_USER}" \ - QBIT_RPC_PASSWORD="${RPC_PASSWORD}" \ - python3 lab/miner-sim/miner_sim.py - ) >"${DATADIR}/miner-${miner_index}.log" 2>&1 & - miner_pids+=("$!") -done - -deadline=$((SECONDS + MINER_TIMEOUT_SECONDS)) -while [[ ! -f "${EVIDENCE_PATH}" && "${SECONDS}" -lt "${deadline}" ]]; do - if ! kill -0 "${coordinator_pid}" >/dev/null 2>&1; then - echo "PRISM coordinator exited before writing evidence" >&2 - cat "${COORDINATOR_LOG}" >&2 || true - exit 1 - fi - sleep 1 -done -if [[ ! -f "${EVIDENCE_PATH}" ]]; then - echo "timed out waiting for live PRISM evidence" >&2 - cat "${COORDINATOR_LOG}" >&2 || true - exit 1 -fi - -for pid in "${miner_pids[@]}"; do - wait "${pid}" || { - echo "a live PRISM miner exited unsuccessfully" >&2 - cat "${COORDINATOR_LOG}" >&2 || true - for miner_index in $(seq 1 "${MINER_COUNT}"); do - echo "--- miner ${miner_index} ---" >&2 - cat "${DATADIR}/miner-${miner_index}.log" >&2 || true - done - exit 1 - } -done -miner_pids=() - -after_height="$(qbit_rpc getblockcount)" -if [[ "${after_height}" -ne $((before_height + 1)) ]]; then - echo "expected first live PRISM block at ${before_height}+1, got ${after_height}" >&2 - exit 1 -fi -live_hash="$(qbit_rpc getblockhash "${after_height}")" - -EVIDENCE_PATH="${EVIDENCE_PATH}" MINER_COUNT="${MINER_COUNT}" python3 <<'PY' -import json -import os -from pathlib import Path - -evidence = json.loads(Path(os.environ["EVIDENCE_PATH"]).read_text(encoding="utf-8")) - -def local_audit_payload(path): - payload = json.loads(Path(path).read_text(encoding="utf-8")) - if payload.get("schema") == "qbit.prism.live-audit-bundle-envelope.v1": - return local_audit_payload(payload["body_uri"]) - if payload.get("schema") == "qbit.prism.audit-body-ref.v1": - return payload["bundle_without_shares"] - if payload.get("schema") == "qbit.prism.audit-bundle.v2": - return payload["bundle_without_shares"] - return payload - -bundle = local_audit_payload(evidence["audit_bundle_path"]) -if evidence["ledger_backend"] != "postgres-psql": - raise SystemExit("combined proof did not use the Postgres ledger") -if evidence["distinct_miner_count"] < int(os.environ["MINER_COUNT"]): - raise SystemExit("combined proof did not include all skewed miners") -if evidence["job_share_count"] < int(os.environ["MINER_COUNT"]): - raise SystemExit("live block did not use the 6-miner PRISM snapshot") -if evidence["audit_report"]["accrued_account_count"] < 1: - raise SystemExit("live skewed payout did not create an accrued account") -if not any(account["action"] == "accrued" for account in bundle["payout_policy_manifest"]["accounts"]): - raise SystemExit("live payout policy manifest has no accrued account") -PY - -build_submit_persist_direct_block() { - local phase="$1" - local suffix_hex="$2" - local mocktime="$3" - qbit_rpc setmocktime "${mocktime}" >/dev/null - ( - cd "${ROOT_DIR}" - PHASE="${phase}" \ - SUFFIX_HEX="${suffix_hex}" \ - DATADIR="${DATADIR}" \ - QBIT_RPC_HOST=127.0.0.1 \ - QBIT_RPC_PORT="${RPC_PORT}" \ - QBIT_RPC_USER="${RPC_USER}" \ - QBIT_RPC_PASSWORD="${RPC_PASSWORD}" \ - QBIT_SRC_DIR="${QBIT_SRC_DIR}" \ - PRISM_PSQL_COMMAND="${PSQL_COMMAND}" \ - LEDGER_WRITER_SESSION_TOKEN="${LEDGER_WRITER_SESSION_TOKEN}" \ - MANIFEST_SIGNING_SEED_HEX="${MANIFEST_SIGNING_SEED_HEX}" \ - LEDGER_ATTESTATION_SIGNING_SEED_HEX="${LEDGER_ATTESTATION_SIGNING_SEED_HEX}" \ - python3 <<'PY' -from __future__ import annotations - -import base64 -import io -import json -import os -import shlex -import subprocess -import sys -import time -import urllib.request -from pathlib import Path - -sys.path.insert(0, str(Path(os.environ["QBIT_SRC_DIR"]) / "test/functional")) -from test_framework.blocktools import create_block -from test_framework.messages import CBlock, CTransaction - -from lab.prism.share_ledger import PsqlShareLedger - - -def rpc(method: str, params: list[object] | None = None) -> object: - body = json.dumps({"jsonrpc": "1.0", "id": method, "method": method, "params": params or []}).encode() - credentials = f"{os.environ['QBIT_RPC_USER']}:{os.environ['QBIT_RPC_PASSWORD']}".encode() - request = urllib.request.Request( - f"http://{os.environ['QBIT_RPC_HOST']}:{os.environ['QBIT_RPC_PORT']}", - data=body, - headers={ - "Authorization": "Basic " + base64.b64encode(credentials).decode(), - "Content-Type": "application/json", - }, - ) - with urllib.request.urlopen(request, timeout=10) as response: - payload = json.loads(response.read()) - if payload["error"] is not None: - raise RuntimeError(f"qbit RPC {method} failed: {payload['error']}") - return payload["result"] - - -def target_from_compact(bits_hex: str) -> int: - compact = int(bits_hex, 16) - size = compact >> 24 - mantissa = compact & 0x007fffff - if size <= 3: - return mantissa >> (8 * (3 - size)) - return mantissa << (8 * (size - 3)) - - -phase = os.environ["PHASE"] -suffix_hex = os.environ["SUFFIX_HEX"] -datadir = Path(os.environ["DATADIR"]) -ledger = PsqlShareLedger( - psql_command=os.environ["PRISM_PSQL_COMMAND"], - writer_id="prism-coordinator", - writer_epoch=1, - writer_session_token=os.environ["LEDGER_WRITER_SESSION_TOKEN"], -) -template = rpc("getblocktemplate", [{"rules": ["segwit"]}]) -if template.get("transactions"): - raise SystemExit("combined proof expects an empty regtest mempool") -anchor_ms = int(time.time() * 1000) -pow_limit_target = target_from_compact("207fffff") -template_target = target_from_compact(template["bits"]) -network_difficulty = max(1, (pow_limit_target * 1_000_000) // template_target) -shares = [record.to_prism_json() for record in ledger.snapshot_at_job_issue(anchor_ms)] -prior_balances = ledger.current_prior_balances() -payload = { - "shares": shares, - "found_block": { - "block_height": int(template["height"]), - "coinbase_value_sats": int(template["coinbasevalue"]), - "network_difficulty": network_difficulty, - "anchor_job_issued_at_ms": anchor_ms, - }, - "prior_balances": prior_balances, - "coinbase_script_sig_suffix_hex": suffix_hex, -} -cargo = shlex.split(os.environ.get("PRISM_CARGO", os.environ.get("CARGO", "cargo"))) -bundle_json = subprocess.check_output( - cargo - + [ - "run", - "--quiet", - "-p", - "qbit-prism", - "--bin", - "qbit-prism-build-audit-bundle", - "--", - "--input", - "-", - "--signing-key-seed-hex", - os.environ["MANIFEST_SIGNING_SEED_HEX"], - "--ledger-signing-key-seed-hex", - os.environ["LEDGER_ATTESTATION_SIGNING_SEED_HEX"], - ], - input=json.dumps(payload), - text=True, -) -bundle = json.loads(bundle_json) -bundle_path = datadir / f"prism-combined-{phase}-bundle.json" -bundle_path.write_text(json.dumps(bundle, indent=2), encoding="utf-8") - -manifest = bundle["signed_coinbase_manifest"]["manifest"] -coinbase = CTransaction() -coinbase.deserialize(io.BytesIO(bytes.fromhex(manifest["coinbase_tx_hex"]))) -block = create_block( - hashprev=int(template["previousblockhash"], 16), - coinbase=coinbase, - ntime=int(template["curtime"]), - version=int(template["version"]), - tmpl=template, - txlist=[tx["data"] for tx in template.get("transactions", [])], -) -block.solve() -before_height = int(rpc("getblockcount")) -result = rpc("submitblock", [block.serialize().hex()]) -after_height = int(rpc("getblockcount")) -if result not in (None, "duplicate"): - raise SystemExit(f"submitblock rejected {phase}: {result}") -if after_height != before_height + 1: - raise SystemExit(f"submitblock did not advance height for {phase}: {before_height}->{after_height}") -block_hash = str(rpc("getblockhash", [after_height])) -raw_block = str(rpc("getblock", [block_hash, 0])) -onchain = CBlock() -onchain.deserialize(io.BytesIO(bytes.fromhex(raw_block))) -coinbase_hex = onchain.vtx[0].serialize_with_witness().hex() -if coinbase_hex != manifest["coinbase_tx_hex"]: - raise SystemExit(f"on-chain coinbase mismatch for {phase}") -report_json = subprocess.check_output( - cargo - + [ - "run", - "--quiet", - "-p", - "qbit-prism", - "--bin", - "qbit-prism-audit-verify", - "--", - str(bundle_path), - "--coinbase-tx-hex", - coinbase_hex, - "--ledger-writer-public-key-hex", - bundle["ledger_window_attestation"]["signature"]["public_key_hex"], - "--expected-coinbase-value-sats", - str(bundle["found_block"]["coinbase_value_sats"]), - ], - text=True, -) -report = json.loads(report_json) -ledger.persist_accepted_block( - block_hash=block_hash, - block_height=after_height, - parent_hash=str(template["previousblockhash"]), - final_bundle=bundle, - audit_report=report, -) -print(json.dumps({ - "phase": phase, - "block_hash": block_hash, - "block_height": after_height, - "bits": template["bits"], - "bundle_path": str(bundle_path), - "audit_report": report, - "prior_balance_count": len(prior_balances), -}, separators=(",", ":"))) -PY - ) -} - -second_json="$(build_submit_persist_direct_block second 222222223333333333333333 "$((ASERT_MOCKTIME_BASE + ASERT_TIME_STEP))")" -second_hash="$(SECOND_JSON="${second_json}" python3 -c 'import json,os; print(json.loads(os.environ["SECOND_JSON"])["block_hash"])')" -second_bits="$(SECOND_JSON="${second_json}" python3 -c 'import json,os; print(json.loads(os.environ["SECOND_JSON"])["bits"])')" -first_bits="$(TEMPLATE_JSON="${initial_template_json}" python3 -c 'import json,os; print(json.loads(os.environ["TEMPLATE_JSON"])["bits"])')" -if [[ "${first_bits}" == "${second_bits}" ]]; then - echo "expected ASERT compact target to change; saw ${first_bits},${second_bits}" >&2 - exit 1 -fi - -AUDIT_PORT="${AUDIT_PORT}" SECOND_HASH="${second_hash}" python3 <<'PY' -import json -import os -import urllib.request - -base = f"http://127.0.0.1:{os.environ['AUDIT_PORT']}" -with urllib.request.urlopen(base + "/owed-balances", timeout=10) as response: - owed = json.loads(response.read()) -if not owed["balances"]: - raise SystemExit("expected positive owed balances after skewed persisted blocks") -with urllib.request.urlopen(base + f"/audit/blocks/{os.environ['SECOND_HASH']}/bundle", timeout=10) as response: - bundle = json.loads(response.read()) -if bundle["block_hash"] != os.environ["SECOND_HASH"]: - raise SystemExit("second persisted bundle API mismatch") -PY - -qbit_rpc invalidateblock "${second_hash}" >/dev/null -reorg_height="$(qbit_rpc getblockcount)" -if [[ "${reorg_height}" -ne 1 ]]; then - echo "expected height 1 after invalidating second block, got ${reorg_height}" >&2 - exit 1 -fi - -PRISM_PSQL_COMMAND="${PSQL_COMMAND}" LEDGER_WRITER_SESSION_TOKEN="${LEDGER_WRITER_SESSION_TOKEN}" SECOND_HASH="${second_hash}" REORG_HEIGHT="${reorg_height}" python3 <<'PY' -import os -from lab.prism.share_ledger import PsqlShareLedger - -ledger = PsqlShareLedger( - psql_command=os.environ["PRISM_PSQL_COMMAND"], - writer_id="prism-coordinator", - writer_epoch=1, - writer_session_token=os.environ["LEDGER_WRITER_SESSION_TOKEN"], -) -result = ledger.reverse_immature_block( - block_hash=os.environ["SECOND_HASH"], - active_tip_height=int(os.environ["REORG_HEIGHT"]), -) -if result["reversed_count"] <= 0: - raise SystemExit("reverse_immature_pool_block did not reverse persisted rows") -PY - -AUDIT_PORT="${AUDIT_PORT}" SECOND_HASH="${second_hash}" python3 <<'PY' -import json -import os -import urllib.request - -base = f"http://127.0.0.1:{os.environ['AUDIT_PORT']}" -with urllib.request.urlopen(base + f"/audit/blocks/{os.environ['SECOND_HASH']}/payouts", timeout=10) as response: - payouts = json.loads(response.read()) -if not payouts["rows"] or {row["maturity_state"] for row in payouts["rows"]} != {"reversed"}: - raise SystemExit("disconnected block payout rows were not all reversed") -with urllib.request.urlopen(base + "/owed-balances", timeout=10) as response: - owed = json.loads(response.read()) -if not owed["balances"]: - raise SystemExit("expected first-block owed balances to remain after second-block reversal") -PY - -replacement_json="$(build_submit_persist_direct_block replacement 222222224444444444444444 "$((ASERT_MOCKTIME_BASE + ASERT_TIME_STEP))")" -replacement_hash="$(REPLACEMENT_JSON="${replacement_json}" python3 -c 'import json,os; print(json.loads(os.environ["REPLACEMENT_JSON"])["block_hash"])')" -if [[ "${replacement_hash}" == "${second_hash}" ]]; then - echo "replacement block unexpectedly matched disconnected block ${second_hash}" >&2 - exit 1 -fi -active_tip="$(qbit_rpc getbestblockhash)" -if [[ "${active_tip}" != "${replacement_hash}" ]]; then - echo "expected replacement ${replacement_hash} to be active tip, got ${active_tip}" >&2 - exit 1 -fi - -AUDIT_PORT="${AUDIT_PORT}" REPLACEMENT_HASH="${replacement_hash}" python3 <<'PY' -import json -import os -import urllib.request - -base = f"http://127.0.0.1:{os.environ['AUDIT_PORT']}" -with urllib.request.urlopen(base + f"/audit/blocks/{os.environ['REPLACEMENT_HASH']}/bundle", timeout=10) as response: - bundle = json.loads(response.read()) -if bundle["block_hash"] != os.environ["REPLACEMENT_HASH"]: - raise SystemExit("replacement persisted bundle API mismatch") -PY - -kill "${coordinator_pid}" >/dev/null 2>&1 || true -wait "${coordinator_pid}" >/dev/null 2>&1 || true -coordinator_pid="" - -printf 'prism combined regtest PASS live=%s second=%s replacement=%s bits=%s,%s\n' \ - "${live_hash}" "${second_hash}" "${replacement_hash}" "${first_bits}" "${second_bits}" diff --git a/test/test-prism-postgres-candidate-storage.sh b/test/test-prism-postgres-candidate-storage.sh deleted file mode 100755 index 33d21f5d2..000000000 --- a/test/test-prism-postgres-candidate-storage.sh +++ /dev/null @@ -1,65 +0,0 @@ -#!/usr/bin/env bash -# Real-PostgreSQL gate for chunked candidate bodies (issue #255). Provisions a -# disposable postgres container and drives tests/prism_postgres_candidate_gate.py -# through the psql subprocess backend and, when the host python has psycopg, -# the native pooled client too. -set -euo pipefail - -ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" -POSTGRES_IMAGE="${QBIT_PRISM_POSTGRES_IMAGE:-postgres:16-alpine}" -POSTGRES_CONTAINER="${QBIT_PRISM_POSTGRES_CONTAINER:-qbit-prism-candidate-pg-$$}" - -require_executable() { - command -v "$1" >/dev/null 2>&1 || { - echo "missing required executable: $1" >&2 - exit 1 - } -} - -cleanup() { - docker rm -f "${POSTGRES_CONTAINER}" >/dev/null 2>&1 || true -} -trap cleanup EXIT - -require_executable docker -require_executable python3 - -docker rm -f "${POSTGRES_CONTAINER}" >/dev/null 2>&1 || true -docker run \ - --rm \ - --detach \ - --name "${POSTGRES_CONTAINER}" \ - -p 127.0.0.1:0:5432 \ - -e POSTGRES_USER=qbit \ - -e POSTGRES_PASSWORD=qbit \ - -e POSTGRES_DB=qbit \ - "${POSTGRES_IMAGE}" >/dev/null - -deadline=$((SECONDS + 60)) -until docker exec "${POSTGRES_CONTAINER}" pg_isready -h 127.0.0.1 -U qbit -d qbit >/dev/null 2>&1; do - if [[ "${SECONDS}" -ge "${deadline}" ]]; then - echo "timed out waiting for PRISM Postgres container" >&2 - docker logs "${POSTGRES_CONTAINER}" >&2 || true - exit 1 - fi - sleep 1 -done - -HOST_PORT="$(docker port "${POSTGRES_CONTAINER}" 5432/tcp | head -n 1 | awk -F: '{print $NF}')" -DATABASE_URL="" -if [[ -n "${HOST_PORT}" ]]; then - DATABASE_URL="postgresql://qbit:qbit@127.0.0.1:${HOST_PORT}/qbit" -fi - -( - cd "${ROOT_DIR}" - PRISM_TEST_PSQL_COMMAND="docker exec -i ${POSTGRES_CONTAINER} psql -U qbit -d qbit" \ - PRISM_TEST_DATABASE_URL="${DATABASE_URL}" \ - python3 -m tests.prism_postgres_candidate_gate - if [[ -n "${DATABASE_URL}" ]] && python3 -c 'import psycopg' >/dev/null 2>&1; then - PRISM_TEST_DATABASE_URL="${DATABASE_URL}" \ - python3 -m unittest -v tests.test_prism_candidate_storage_races - fi -) - -echo "test-prism-postgres-candidate-storage: PASS" diff --git a/test/test-prism-postgres-ledger.sh b/test/test-prism-postgres-ledger.sh deleted file mode 100644 index a38573a7e..000000000 --- a/test/test-prism-postgres-ledger.sh +++ /dev/null @@ -1,2342 +0,0 @@ -#!/usr/bin/env bash -set -euo pipefail - -ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" -# shellcheck source=test/prism-postgres-lib.sh -source "${ROOT_DIR}/test/prism-postgres-lib.sh" - -POSTGRES_IMAGE="${QBIT_PRISM_POSTGRES_IMAGE:-postgres:16-alpine}" -POSTGRES_CONTAINER="${QBIT_PRISM_POSTGRES_CONTAINER:-qbit-prism-ledger-pg-$$}" - -require_executable() { - command -v "$1" >/dev/null 2>&1 || { - echo "missing required executable: $1" >&2 - exit 1 - } -} - -# When QBIT_PRISM_EXTERNAL_PSQL_COMMAND is set, run against an already-running -# Postgres (e.g. a local cluster) instead of provisioning a Docker container. -EXTERNAL_PSQL="${QBIT_PRISM_EXTERNAL_PSQL_COMMAND:-}" - -cleanup() { - if [[ -z "${EXTERNAL_PSQL}" ]]; then - docker rm -f "${POSTGRES_CONTAINER}" >/dev/null 2>&1 || true - fi -} -trap cleanup EXIT - -require_executable python3 - -if [[ -n "${EXTERNAL_PSQL}" ]]; then - PSQL_COMMAND="${EXTERNAL_PSQL}" - deadline=$((SECONDS + 60)) - until echo 'SELECT 1;' | ${PSQL_COMMAND} >/dev/null 2>&1; do - if [[ "${SECONDS}" -ge "${deadline}" ]]; then - echo "timed out waiting for external PRISM Postgres" >&2 - exit 1 - fi - sleep 1 - done -else - require_executable docker - docker rm -f "${POSTGRES_CONTAINER}" >/dev/null 2>&1 || true - docker run \ - --rm \ - --detach \ - --name "${POSTGRES_CONTAINER}" \ - -e POSTGRES_USER=qbit \ - -e POSTGRES_PASSWORD=qbit \ - -e POSTGRES_DB=qbit \ - "${POSTGRES_IMAGE}" >/dev/null - - wait_for_prism_postgres_container "${POSTGRES_CONTAINER}" - PSQL_COMMAND="docker exec -i ${POSTGRES_CONTAINER} psql -U qbit -d qbit" -fi - -# Every PsqlShareLedger session GUC rides PGOPTIONS on the psql path, and -# "docker exec" does not forward the caller's environment, so the container -# form silently drops them. The read-only session assertion at the end needs -# PGOPTIONS to actually arrive, so it runs through a command that asks docker -# to pass the variable through. An external psql inherits the environment -# already and needs no help. -if [[ -n "${EXTERNAL_PSQL}" ]]; then - PSQL_COMMAND_WITH_ENV="${PSQL_COMMAND}" -else - PSQL_COMMAND_WITH_ENV="docker exec -i -e PGOPTIONS ${POSTGRES_CONTAINER} psql -U qbit -d qbit" -fi - -if [[ -n "${EXTERNAL_PSQL}" ]]; then - GATE_IMAGE="external-postgres" - GATE_IMAGE_DIGEST="not-applicable" -else - GATE_IMAGE="$(docker inspect --format '{{.Config.Image}}' "${POSTGRES_CONTAINER}")" - GATE_IMAGE_DIGEST="$(docker inspect --format '{{.Image}}' "${POSTGRES_CONTAINER}")" -fi - -( - cd "${ROOT_DIR}" - PRISM_PSQL_COMMAND="${PSQL_COMMAND}" \ - PRISM_PSQL_COMMAND_WITH_ENV="${PSQL_COMMAND_WITH_ENV}" \ - python3 <<'PY' -from __future__ import annotations - -import copy -import hashlib -import json -import os -import tempfile -from pathlib import Path - -from lab.prism.share_ledger import ( - PendingShare, - PsqlShareLedger, - ShareReplayConflict, - SingleWriterShareLedger, -) -from tests.prism_window_rows_reference import assert_rows_match_aggregate - - -def pending( - index: int, - *, - job_ms: int, - accepted_ms: int, - share_id: str | None = None, - share_difficulty: int | None = None, - network_difficulty: int = 1000, - template_height: int = 10, -) -> PendingShare: - return PendingShare( - share_id=share_id or f"share-{index}", - miner_id=f"miner-{index}", - order_key=f"{index:04d}", - p2mr_program_hex=f"{index:02x}" * 32, - share_difficulty=share_difficulty if share_difficulty is not None else 100 + index, - network_difficulty=network_difficulty, - template_height=template_height, - job_id=f"job-{index}", - job_issued_at_ms=job_ms, - accepted_at_ms=accepted_ms, - ntime=1_700_000_000 + index, - ) - - -def fake_audit_bundle_bytes(payload) -> bytes: - return json.dumps(payload, separators=(",", ":")).encode() - - -def fake_audit_bundle_sha256(payload) -> str: - return hashlib.sha256(fake_audit_bundle_bytes(payload)).hexdigest() - - -def assert_equal(actual: object, expected: object, message: str) -> None: - if actual != expected: - raise SystemExit(f"{message}: expected {expected!r}, got {actual!r}") - - -def assert_current_matches_recomputed(runner: PsqlShareLedger, message: str) -> None: - """The summary-backed current balances must equal the O(history) recompute.""" - comparison = runner._run_json( - """ -SELECT json_build_object( - 'drift_count', (SELECT count(*) FROM qbit_carry_forward_current_drift()), - 'current', ( - SELECT COALESCE(json_agg(json_build_object( - 'recipient_id', miner_id, - 'order_key', payout_order_key, - 'p2mr_program_hex', encode(p2mr_program, 'hex'), - 'balance_sats', balance_sats::text - ) ORDER BY payout_order_key, miner_id, encode(p2mr_program, 'hex')), '[]'::json) - FROM qbit_current_carry_forward_balances() - ), - 'recomputed', ( - SELECT COALESCE(json_agg(json_build_object( - 'recipient_id', miner_id, - 'order_key', payout_order_key, - 'p2mr_program_hex', encode(p2mr_program, 'hex'), - 'balance_sats', balance_sats::text - ) ORDER BY payout_order_key, miner_id, encode(p2mr_program, 'hex')), '[]'::json) - FROM qbit_recomputed_carry_forward_balances() - ) -); -""" - ) - assert_equal(comparison["current"], comparison["recomputed"], message) - assert_equal(int(comparison["drift_count"]), 0, message + " (drift count)") - - -def share_ledger_identity(runner: PsqlShareLedger) -> dict[str, object]: - """The exact identity of qbit_share_ledger, for the #224 invariant. - - Every column of every row in sequence order, digested together with - each tuple's ctid and xmin -- so an UPDATE that rewrote a row to the - same values still changes it -- plus the sequence allocator, so an - INSERT that rolled back still changes it. - """ - return runner._run_json( - """ -SELECT json_build_object( - 'row_count', (SELECT count(*) FROM qbit_share_ledger), - 'digest', ( - SELECT md5(COALESCE(string_agg( - ctid::text || '|' || xmin::text || '|' || ledger::text, - E'\\n' ORDER BY share_seq - ), '')) - FROM qbit_share_ledger AS ledger - ), - 'sequence', ( - SELECT json_build_object('last_value', last_value, 'is_called', is_called) - FROM qbit_share_ledger_share_seq_seq - ) -); -""" - ) - - -def force_expired_idle_lease(runner: PsqlShareLedger) -> None: - runner._run_sql( - """ -UPDATE qbit_ledger_writer_lease -SET updated_at = clock_timestamp() - interval '6 minutes', - lease_expires_at = clock_timestamp() - interval '1 minute'; -""" - ) - - -psql = os.environ["PRISM_PSQL_COMMAND"] -psql_with_env = os.environ["PRISM_PSQL_COMMAND_WITH_ENV"] -ledger = PsqlShareLedger( - # This gate mutates legacy JSONB fixtures; the storage gate covers v2. - candidate_storage_version=1, - psql_command=psql, - writer_id="writer-a", - writer_epoch=1, - initialize_schema=True, -) -schema_smoke = ledger._run_json( - """ -SELECT json_build_object( - 'carry_forward_balance_rows', (SELECT count(*) FROM qbit_current_carry_forward_balances()), - 'owed_balance_rows', (SELECT count(*) FROM qbit_current_owed_balances()) -); -""" -) -assert_equal( - schema_smoke, - {"carry_forward_balance_rows": 0, "owed_balance_rows": 0}, - "schema initialization exposes current balance functions", -) - -# The live ACK path uses append_batch, including ordinary shares whose -# candidate value is JSON null. Exercise the real Postgres statement so JSONB -# null handling, FIFO sequence assignment, exact replay, and terminal outbox -# compaction cannot regress behind SQL-string unit tests. -batch_a = pending(90, job_ms=900, accepted_ms=901, share_id="batch-a") -batch_b = pending(91, job_ms=902, accepted_ms=903, share_id="batch-b") -batch_rows = ledger.append_batch([(batch_a, None), (batch_b, None)]) -assert_equal([row.share_id for row in batch_rows], ["batch-a", "batch-b"], "batch FIFO ids") -assert_equal([row.share_seq for row in batch_rows], [1, 2], "batch FIFO sequences") -assert_equal(ledger.pending_block_candidates(), [], "ordinary shares create no outbox rows") - -batch_c = pending(92, job_ms=904, accepted_ms=905, share_id="batch-c") -candidate_intent = { - "schema": "qbit.prism.block-candidate-intent.v1", - "block_hash_hex": "ab" * 32, - "block_hex": "00", -} -candidate_row = ledger.append_batch([(batch_c, candidate_intent)])[0] -assert_equal(candidate_row.share_seq, 3, "candidate share sequence") -assert_equal(ledger.append_batch([(batch_c, candidate_intent)])[0].share_seq, 3, "exact replay") -recovery_exact = ledger.append_recovered_share(batch_c) -assert_equal(recovery_exact.disposition, "exact_existing", "typed recovery exact replay") -assert_equal(recovery_exact.record.share_seq, 3, "typed recovery exact replay keeps sequence") -try: - ledger.append_recovered_share( - PendingShare(**{**batch_c.__dict__, "ntime": batch_c.ntime + 1}) - ) -except ShareReplayConflict: - pass -else: - raise SystemExit("typed recovery payload conflict was not rejected") -assert_equal(ledger.pending_block_candidates(), [candidate_intent], "pending candidate replay") -pending_rows = ledger.pending_block_candidate_rows() -assert_equal( - [ - {key: value for key, value in row.items() if key != "cursor"} - for row in pending_rows - ], - [ - { - "block_hash": "ab" * 32, - "candidate": candidate_intent, - "pool_block_exists": False, - } - ], - "pending candidate replay retains authoritative outbox key", -) -# Startup enumeration pages this read with a keyset cursor, so the cursor -# has to round-trip through the server exactly: a truncated stamp would -# make a page re-emit or skip its equal-timestamp peers. -assert_equal( - ledger.pending_block_candidate_rows(after_cursor=pending_rows[0]["cursor"]), - [], - "pending candidate cursor resumes strictly after its own row", -) -cursor_stamp, cursor_hash = pending_rows[0]["cursor"] -assert_equal(cursor_hash, "ab" * 32, "pending candidate cursor carries its row key") -assert_equal( - ledger._run_json( - "SELECT json_build_object('matched', (" - "SELECT count(*) FROM qbit_block_candidate_outbox " - f"WHERE created_at = '{cursor_stamp}'::timestamptz " - f"AND block_hash = '{cursor_hash}'))" - )["matched"], - 1, - "pending candidate cursor stamp round-trips at full precision", -) -assert ledger.mark_block_candidate_submitted(block_hash="ab" * 32) -assert_equal(ledger.pending_block_candidates(), [], "submitted candidate leaves pending set") -assert_equal(ledger.append_batch([(batch_c, candidate_intent)])[0].share_seq, 3, "compact exact replay") -batch_d = pending(93, job_ms=906, accepted_ms=907, share_id="batch-d") -candidate_only = { - **candidate_intent, - "block_hash_hex": "cd" * 32, - "credit_share_on_accept": True, -} -assert ledger.persist_block_candidate_intent(candidate_only) -assert_equal(ledger.pending_block_candidates(), [candidate_only], "candidate-only intent") -assert_equal(ledger.append_batch([(batch_d, candidate_only)])[0].share_seq, 4, "linked solver credit") -assert ledger.mark_block_candidate_submitted(block_hash="cd" * 32) -assert_equal(ledger.pending_block_candidates(), [], "linked candidate completion") -batch_recovered = pending(94, job_ms=908, accepted_ms=909, share_id="batch-recovered") -recovery_inserted = ledger.append_recovered_share(batch_recovered) -assert_equal(recovery_inserted.disposition, "inserted", "typed recovery inserts a new row") -assert_equal( - recovery_inserted.record.share_id, - "batch-recovered", - "typed recovery returns the inserted record", -) -poison_hash = "ef" * 32 -poison_candidate = { - **candidate_intent, - "block_hash_hex": poison_hash, -} -assert ledger.persist_block_candidate_intent(poison_candidate) -ledger._run_sql( - "UPDATE qbit_block_candidate_outbox " - "SET candidate = candidate - 'block_hash_hex' " - f"WHERE block_hash = '{poison_hash}';" -) -poison_rows = ledger.pending_block_candidate_rows() -assert_equal(poison_rows[0]["block_hash"], poison_hash, "poison row keeps durable key") -assert_equal( - poison_rows[0]["candidate"].get("block_hash_hex"), - None, - "poison fixture removes payload block hash", -) -assert ledger.mark_block_candidate_abandoned( - block_hash=poison_rows[0]["block_hash"], - error="invalid durable candidate intent", -) -assert_equal(ledger.pending_block_candidates(), [], "poison row quarantined by durable key") - -# The bulk terminal update replaces N single-hash abandonments on the storm -# path, so its fence, its exactly-pending predicate, its pool-block veto and -# its RETURNING-fed report all have to hold against the real table -- where -# state, completed_at and candidate are checked together in SQL and a -# half-applied terminal row would be rejected outright. -bulk_pending_a = "1a" * 32 -bulk_pending_b = "1b" * 32 -bulk_submitted = "1c" * 32 -bulk_landed = "1d" * 32 -bulk_missing = "99" * 32 -for bulk_hash in (bulk_pending_a, bulk_pending_b, bulk_submitted, bulk_landed): - assert ledger.persist_block_candidate_intent( - {**candidate_intent, "block_hash_hex": bulk_hash} - ) -assert ledger.mark_block_candidate_submitted(block_hash=bulk_submitted) -ledger._run_sql( - "INSERT INTO qbit_pool_blocks " - "(block_hash, block_height, parent_hash, coinbase_txid, payout_manifest_sha256) " - f"VALUES ('{bulk_landed}', 11, '{'00' * 32}', '{'11' * 32}', '{'22' * 32}');" -) -# The page read answers the landed-block question for every row it returns, -# so the caller never pays one round trip per pending hash. -assert_equal( - { - row["block_hash"]: row["pool_block_exists"] - for row in ledger.pending_block_candidate_rows(limit=32) - }, - {bulk_pending_a: False, bulk_pending_b: False, bulk_landed: True}, - "pending page reports authoritative pool-block existence", -) -assert_equal( - ledger.mark_block_candidates_abandoned(block_hashes=[], error="unused"), - (), - "empty bulk abandon page is a no-op", -) -bulk_abandoned = ledger.mark_block_candidates_abandoned( - block_hashes=[ - bulk_pending_b, - bulk_pending_a.upper(), - bulk_pending_a, - bulk_submitted, - bulk_landed, - bulk_missing, - ], - error="superseded by decided height", -) -assert_equal( - list(bulk_abandoned), - [bulk_pending_a, bulk_pending_b], - "bulk abandon reports exactly the pending rows it won", -) -assert_equal( - ledger.pending_block_candidates(), - [{**candidate_intent, "block_hash_hex": bulk_landed}], - "bulk abandon leaves the landed candidate pending", -) -bulk_rows = ledger._run_json( - "SELECT json_build_object('rows', COALESCE(json_agg(json_build_object(" - "'block_hash', block_hash, 'state', state, 'last_error', last_error, " - "'candidate_cleared', candidate IS NULL, " - "'completed', completed_at IS NOT NULL) ORDER BY block_hash), '[]'::json)) " - "FROM qbit_block_candidate_outbox WHERE block_hash = ANY(ARRAY[" - f"'{bulk_pending_a}', '{bulk_pending_b}', '{bulk_submitted}', '{bulk_landed}'" - "]::text[]);" -)["rows"] -assert_equal( - bulk_rows, - [ - { - "block_hash": bulk_pending_a, - "state": "abandoned", - "last_error": "superseded by decided height", - "candidate_cleared": True, - "completed": True, - }, - { - "block_hash": bulk_pending_b, - "state": "abandoned", - "last_error": "superseded by decided height", - "candidate_cleared": True, - "completed": True, - }, - { - "block_hash": bulk_submitted, - "state": "submitted", - "last_error": None, - "candidate_cleared": True, - "completed": True, - }, - { - "block_hash": bulk_landed, - "state": "pending", - "last_error": None, - "candidate_cleared": False, - "completed": False, - }, - ], - "bulk abandon writes the terminal column set and spares every other row", -) -assert_equal( - ledger.mark_block_candidates_abandoned( - block_hashes=[bulk_pending_a, bulk_pending_b], - error="second attempt", - ), - (), - "already-terminal rows are not re-won", -) -ledger._run_sql(f"DELETE FROM qbit_pool_blocks WHERE block_hash = '{bulk_landed}';") -# The stale-page case below asserts an exact page and terminal set. Isolate it -# from the pending row intentionally retained by the bulk-abandon case above. -ledger._run_sql("DELETE FROM qbit_block_candidate_outbox;") - -# Issue #211 removed the writer gate from the page read, which makes the page -# advisory by construction: a candidate can land between the snapshot and -# whatever the caller does with it. The case above inserted the pool block -# first, so its page was never stale. This one reads the page *before* the -# landing, so the set handed to the terminal write genuinely disagrees with -# the durable state -- and the veto has to come from the fenced UPDATE's own -# re-check rather than from the fact the caller carried. -stale_pending = "2a" * 32 -stale_landed = "2b" * 32 -for stale_hash in (stale_pending, stale_landed): - assert ledger.persist_block_candidate_intent( - {**candidate_intent, "block_hash_hex": stale_hash} - ) -stale_page = ledger.pending_block_candidate_rows(limit=32) -assert_equal( - {row["block_hash"]: row["pool_block_exists"] for row in stale_page}, - {stale_pending: False, stale_landed: False}, - "a page read before the landing reports no pool block for either row", -) -ledger._run_sql( - "INSERT INTO qbit_pool_blocks " - "(block_hash, block_height, parent_hash, coinbase_txid, payout_manifest_sha256) " - f"VALUES ('{stale_landed}', 12, '{'00' * 32}', '{'33' * 32}', '{'44' * 32}');" -) -assert_equal( - list( - ledger.mark_block_candidates_abandoned( - block_hashes=[str(row["block_hash"]) for row in stale_page], - error="acted on a stale page", - ) - ), - [stale_pending], - "a candidate that landed after the page read is refused by the fence", -) -assert_equal( - [row["block_hash"] for row in ledger.pending_block_candidate_rows(limit=32)], - [stale_landed], - "the landed candidate survived a stale page that named it", -) -ledger._run_sql(f"DELETE FROM qbit_pool_blocks WHERE block_hash = '{stale_landed}';") - -ledger._run_sql( - """ -DELETE FROM qbit_block_candidate_outbox; -DELETE FROM qbit_share_ledger; -ALTER SEQUENCE qbit_share_ledger_share_seq_seq RESTART WITH 1; -""" -) - -ledger._run_sql( - """ -ALTER TABLE qbit_ctv_fanout_artifacts - ALTER COLUMN anchor_vout SET NOT NULL; -""" -) -legacy_anchor_nullable = ledger._run_json( - """ -SELECT to_json(is_nullable) -FROM information_schema.columns -WHERE table_name = 'qbit_ctv_fanout_artifacts' - AND column_name = 'anchor_vout'; -""" -) -assert_equal(legacy_anchor_nullable, "NO", "old schema simulation makes anchor_vout not nullable") -ledger.release_writer_lease() -ledger = PsqlShareLedger( - # This gate mutates legacy JSONB fixtures; the storage gate covers v2. - candidate_storage_version=1, - psql_command=psql, - writer_id="writer-a", - writer_epoch=1, - initialize_schema=True, -) -repaired_anchor_nullable = ledger._run_json( - """ -SELECT to_json(is_nullable) -FROM information_schema.columns -WHERE table_name = 'qbit_ctv_fanout_artifacts' - AND column_name = 'anchor_vout'; -""" -) -assert_equal(repaired_anchor_nullable, "YES", "schema init repairs nullable CTV anchor_vout") - -first = ledger.append(pending(1, job_ms=1_700_000_000_500, accepted_ms=1_700_000_001_000)) -second = ledger.append(pending(2, job_ms=1_700_000_000_000, accepted_ms=1_700_000_001_100)) -assert_equal([first.share_seq, second.share_seq], [1, 2], "initial share sequence") - -snapshot = ledger.snapshot_at_job_issue(1_700_000_001_100) -assert_equal([share.share_seq for share in snapshot], [1, 2], "snapshot preserves insert order") - -try: - ledger.append(pending(99, job_ms=1_700_000_000_000, accepted_ms=1_700_000_001_200, share_id="share-1")) -except RuntimeError as exc: - if "duplicate share_id" not in str(exc): - raise -else: - raise SystemExit("duplicate share replay unexpectedly appended") - -third = ledger.append(pending(3, job_ms=1_700_000_000_000, accepted_ms=1_700_000_001_300)) -assert_equal(third.share_seq, 3, "duplicate replay must not consume a sequence value") - -future = ledger.append(pending(4, job_ms=1_700_000_000_000, accepted_ms=1_700_000_005_000)) -assert_equal(future.share_seq, 4, "future accepted share sequence") - -window = ledger.audit_share_window(anchor_job_issued_at_ms=1_700_000_001_300, network_difficulty=50) -assert_equal( - [row["share_id"] for row in window], - ["share-3", "share-2", "share-1"], - "audit window excludes future-accepted old-job share", -) - -# Eligible shares at this anchor, newest first: share-3 (difficulty 103, -# cumulative 103), share-2 (102, cumulative 205), share-1 (101, cumulative -# 306). The bounded snapshot must include the share that crosses the window -# weight and nothing past it. -for weight, expected_ids in [ - (1, ["share-3"]), - (103, ["share-3"]), - (104, ["share-2", "share-3"]), - (205, ["share-2", "share-3"]), - (206, ["share-1", "share-2", "share-3"]), -]: - bounded = ledger.snapshot_at_job_issue(1_700_000_001_300, window_weight=weight) - assert_equal( - [share.share_id for share in bounded], - expected_ids, - f"bounded snapshot crossing behavior at window weight {weight}", - ) -assert_equal( - [ - share.share_id - for share in ledger.snapshot_at_job_issue(1_700_000_001_300, window_weight=10**9) - ], - [share.share_id for share in ledger.snapshot_at_job_issue(1_700_000_001_300)], - "bounded snapshot with beyond-history weight matches the unbounded snapshot", -) - -try: - PsqlShareLedger(psql_command=psql, writer_id="writer-a", writer_epoch=2) -except RuntimeError as exc: - if "writer-a" not in str(exc): - raise -else: - raise SystemExit("same writer id with a different live epoch stole the lease") - -try: - PsqlShareLedger(psql_command=psql, writer_id="writer-b", writer_epoch=1) -except RuntimeError as exc: - if "writer-a" not in str(exc): - raise -else: - raise SystemExit("different writer stole an unexpired lease") - -force_expired_idle_lease(ledger) -idle_share = ledger.append(pending(50, job_ms=1_700_000_005_500, accepted_ms=1_700_000_005_600)) -assert_equal(idle_share.share_seq, 5, "same writer refreshes expired lease before append") - -try: - PsqlShareLedger(psql_command=psql, writer_id="writer-b", writer_epoch=1) -except RuntimeError as exc: - if "writer-a" not in str(exc): - raise -else: - raise SystemExit("different writer stole an unexpired lease after idle refresh") - -force_expired_idle_lease(ledger) -replacement = PsqlShareLedger( - psql_command=psql, - writer_id="writer-b", - writer_epoch=1, - audit_bundle_canonicalizer=fake_audit_bundle_bytes, -) -try: - ledger.append(pending(5, job_ms=1_700_000_006_000, accepted_ms=1_700_000_006_100)) -except RuntimeError as exc: - if "writer lease is not active" not in str(exc): - raise -else: - raise SystemExit("stale writer appended after replacement acquired the lease") - -force_expired_idle_lease(replacement) -try: - ledger.append(pending(51, job_ms=1_700_000_006_200, accepted_ms=1_700_000_006_300)) -except RuntimeError as exc: - if "writer lease is not active" not in str(exc): - raise -else: - raise SystemExit("stale writer reacquired after replacement lease expired") - -fifth = replacement.append(pending(5, job_ms=1_700_000_006_000, accepted_ms=1_700_000_006_100)) -assert_equal(fifth.share_seq, 6, "replacement writer resumes next sequence") - -below_height = replacement.append( - pending(6, job_ms=1_700_000_007_000, accepted_ms=1_700_000_007_100, template_height=19) -) -height_hit = replacement.append( - pending(7, job_ms=1_700_000_007_000, accepted_ms=1_700_000_007_100, template_height=20) -) -height_high = replacement.append( - pending(8, job_ms=1_700_000_007_000, accepted_ms=1_700_000_007_100, template_height=21) -) -assert_equal( - [below_height.share_seq, height_hit.share_seq, height_high.share_seq], - [7, 8, 9], - "template-height fixture sequence", -) -replacement._run_sql( - """ -INSERT INTO qbit_share_ledger ( - share_id, - miner_id, - payout_order_key, - p2mr_program, - share_difficulty, - network_difficulty, - template_height, - job_id, - job_issued_at, - ntime, - accepted_at, - accepted, - reject_reason, - writer_id, - writer_epoch -) VALUES ( - 'share-rejected-height-22', - 'miner-rejected', - 'rejected', - decode('99' || repeat('00', 31), 'hex'), - 1, - 1000, - 22, - 'job-rejected', - to_timestamp(1700000007.000), - 1700000009, - to_timestamp(1700000007.100), - false, - 'low-difficulty', - 'writer-b', - 1 -); -""" -) -since_height = replacement._run_json( - """ -SELECT COALESCE(json_agg(json_build_object( - 'share_id', share_id, - 'share_seq', share_seq, - 'template_height', template_height, - 'accepted', accepted -) ORDER BY share_seq ASC), '[]'::json) -FROM qbit_shares_since_template_height(20); -""" -) -assert_equal( - [row["share_id"] for row in since_height], - ["share-7", "share-8"], - "template-height query returns accepted shares at or above threshold in sequence order", -) - -replacement.append( - pending(10, job_ms=1_700_000_008_000, accepted_ms=1_700_000_008_000, share_difficulty=40) -) -replacement.append( - pending(11, job_ms=1_700_000_008_000, accepted_ms=1_700_000_008_000, share_difficulty=60) -) -replacement.append( - pending(12, job_ms=1_700_000_008_000, accepted_ms=1_700_000_008_000, share_difficulty=100) -) -exact_window = replacement.audit_share_window( - anchor_job_issued_at_ms=1_700_000_008_000, - network_difficulty=25, -) -assert_equal( - [row["share_id"] for row in exact_window], - ["share-12", "share-11", "share-10"], - "exact 8x window share order", -) -assert_equal( - [row["counted_difficulty"] for row in exact_window], - [100, 60, 40], - "exact 8x window counted difficulties", -) -assert_equal(sum(row["counted_difficulty"] for row in exact_window), 200, "exact 8x window total weight") - -replacement.append( - pending(13, job_ms=1_700_000_009_000, accepted_ms=1_700_000_009_000, share_difficulty=90) -) -replacement.append( - pending(14, job_ms=1_700_000_009_000, accepted_ms=1_700_000_009_000, share_difficulty=150) -) -partial_window = replacement.audit_share_window( - anchor_job_issued_at_ms=1_700_000_009_000, - network_difficulty=25, -) -assert_equal( - [row["share_id"] for row in partial_window], - ["share-14", "share-13"], - "partial oldest-share window order", -) -assert_equal([row["share_difficulty"] for row in partial_window], [150, 90], "partial window original difficulties") -assert_equal([row["counted_difficulty"] for row in partial_window], [150, 50], "partial oldest-share counted difficulty") -assert_equal(sum(row["counted_difficulty"] for row in partial_window), 200, "partial window total weight") - -bundle = { - "signed_coinbase_manifest": {"manifest": {"payout_count": 2}}, - "payout_policy_manifest": { - "accounts": [ - { - "recipient_id": "miner-a", - "order_key": "a", - "p2mr_program_hex": "aa" * 32, - "gross_amount_sats": 1000, - "prior_balance_sats": 0, - "candidate_balance_sats": 1000, - "onchain_amount_sats": 0, - "carry_forward_balance_sats": 1000, - "action": "accrued", - }, - { - "recipient_id": "miner-b", - "order_key": "b", - "p2mr_program_hex": "bb" * 32, - "gross_amount_sats": 50000, - "prior_balance_sats": 0, - "candidate_balance_sats": 50000, - "onchain_amount_sats": 50000, - "settlement_fee_sats": 500, - "carry_forward_balance_sats": 0, - "action": "onchain", - }, - ] - }, -} -def audit_report_for( - final_bundle, - *, - coinbase_txid: str = "11" * 32, - coinbase_manifest_sha256_hex: str = "22" * 32, - coinbase_tx_hex: str = "00", -): - return { - "coinbase_txid": coinbase_txid, - "coinbase_manifest_sha256_hex": coinbase_manifest_sha256_hex, - "audit_bundle_sha256_hex": fake_audit_bundle_sha256(final_bundle), - "coinbase_tx_hex": coinbase_tx_hex, - } - - -report = audit_report_for(bundle) -force_expired_idle_lease(replacement) -first_persist = replacement.persist_accepted_block( - block_hash="44" * 32, - block_height=7, - parent_hash="55" * 32, - final_bundle=bundle, - audit_report=report, -) -assert_equal( - [ - first_persist["block_count"], - first_persist["bundle_count"], - first_persist["payout_entry_count"], - first_persist["carry_forward_count"], - ], - [1, 1, 2, 2], - "initial accepted block persistence counts", -) -fanout_manifest_set = { - "settlement_mode": "ctv_fanout", - "parent_coinbase_txid": "11" * 32, - "fanout_count": 1, - "fanout_output_sum_sats": 487, - "covenant_output_value_sats": 500, - "manifests": [ - { - "fanout_txid": "12" * 32, - "parent_coinbase_txid": "11" * 32, - "parent_coinbase_tx_hex": "00", - "parent_coinbase_vout": 0, - "fanout_tx_hex": "02", - "commitment_witness_leaf_hex": "03", - "precommitment_sha256_hex": "13" * 32, - "covenant_output_value_sats": 500, - "precommitment": { - "settlement_mode": "ctv_fanout", - "chunk_index": 0, - "chunk_count": 1, - "ctv_hash_hex": "14" * 32, - "fanout_tx_template_hex": "04", - "fanout_fee_sats": 13, - "fanout_output_sum_sats": 487, - }, - } - ], -} -force_expired_idle_lease(replacement) -fanout_persist = replacement.persist_ctv_fanout_manifest_set( - block_hash="44" * 32, - manifest_set=fanout_manifest_set, - manifest_set_sha256="15" * 32, -) -assert_equal( - [fanout_persist["fanout_set_count"], fanout_persist["fanout_artifact_count"]], - [1, 1], - "same writer refreshes expired lease before CTV fanout persistence", -) -fanout_anchor_row = replacement._run_json( - """ -SELECT json_build_object( - 'anchor_vout_is_null', anchor_vout IS NULL, - 'fanout_output_sum_sats', fanout_output_sum_sats -) -FROM qbit_ctv_fanout_artifacts -WHERE fanout_txid = '""" + "12" * 32 + """'; -""" -) -assert_equal( - fanout_anchor_row, - {"anchor_vout_is_null": True, "fanout_output_sum_sats": 487}, - "fee-bearing CTV fanout persists with NULL anchor_vout", -) -replacement._run_sql("DELETE FROM qbit_ctv_fanout_artifacts WHERE fanout_txid = '" + "12" * 32 + "';") -force_expired_idle_lease(replacement) -fanout_backfill_lease_stealer = PsqlShareLedger( - psql_command=psql, - writer_id="fanout-backfill-lease-stealer", - writer_epoch=1, -) -try: - replacement.persist_ctv_fanout_manifest_set( - block_hash="44" * 32, - manifest_set=fanout_manifest_set, - manifest_set_sha256="15" * 32, - ) -except RuntimeError as exc: - if "writer lease is not active" not in str(exc): - raise -else: - raise SystemExit("stale CTV fanout backfill unexpectedly succeeded") -assert_equal( - fanout_backfill_lease_stealer._run_json( - """ -SELECT json_build_object( - 'artifact_count', - (SELECT count(*) FROM qbit_ctv_fanout_artifacts WHERE fanout_txid = '""" + "12" * 32 + """') -); -""" - )["artifact_count"], - 0, - "stale CTV fanout backfill does not insert missing artifacts", -) -force_expired_idle_lease(fanout_backfill_lease_stealer) -replacement = PsqlShareLedger( - psql_command=psql, - writer_id="writer-b", - writer_epoch=1, - audit_bundle_canonicalizer=fake_audit_bundle_bytes, -) -fanout_backfill = replacement.persist_ctv_fanout_manifest_set( - block_hash="44" * 32, - manifest_set=fanout_manifest_set, - manifest_set_sha256="15" * 32, -) -assert_equal( - [fanout_backfill["fanout_set_count"], fanout_backfill["fanout_artifact_count"]], - [1, 1], - "matching CTV fanout set backfills a missing artifact row", -) -force_expired_idle_lease(replacement) -fanout_status_update = replacement.update_ctv_fanout_status( - fanout_txid="12" * 32, - settlement_status="broadcastable", -) -assert_equal(fanout_status_update["updated_count"], 1, "same writer refreshes expired lease before CTV status update") -force_expired_idle_lease(replacement) -fanout_attempt = replacement.record_ctv_fanout_broadcast_attempt( - fanout_txid="12" * 32, - attempt_status="submitted", - package_tx_hexes=["02"], - package_txids=["16" * 32], - submit_result={"accepted": True}, -) -assert_equal( - [fanout_attempt["attempt_count"], fanout_attempt["updated_count"]], - [1, 1], - "same writer refreshes expired lease before CTV broadcast attempt", -) -fanout_status = replacement.ctv_fanout_status(fanout_txid="12" * 32) -assert_equal(fanout_status["settlement_status"], "broadcast_submitted", "CTV broadcast attempt updates status") -assert_equal( - [attempt["attempt_status"] for attempt in fanout_status["broadcast_attempts"]], - ["submitted"], - "CTV broadcast attempt is journaled", -) -# A second pending fanout on the same block, with its own attempts, proves the -# public pending-fanout page scopes broadcast_attempts to each row's fanout. -# The persisted manifest set is immutable, so the sibling is a row copy; it is -# removed again so later block-level checks see the original single fanout. -replacement._run_sql( - """ -INSERT INTO qbit_ctv_fanout_artifacts ( - fanout_txid, - block_hash, - manifest_set_sha256, - manifest_json, - manifest, - manifest_sha256, - precommitment_sha256, - ctv_hash, - commitment_witness_leaf_hex, - chunk_index, - chunk_count, - parent_coinbase_txid, - parent_coinbase_vout, - fanout_tx_template_hex, - fanout_tx_hex, - anchor_vout, - covenant_output_value_sats, - fanout_output_sum_sats, - settlement_status -) -SELECT - '""" + "17" * 32 + """', - block_hash, - manifest_set_sha256, - manifest_json, - manifest, - manifest_sha256, - precommitment_sha256, - ctv_hash, - commitment_witness_leaf_hex, - 1, - 2, - parent_coinbase_txid, - parent_coinbase_vout, - fanout_tx_template_hex, - fanout_tx_hex, - anchor_vout, - covenant_output_value_sats, - fanout_output_sum_sats, - 'broadcastable' -FROM qbit_ctv_fanout_artifacts -WHERE fanout_txid = '""" + "12" * 32 + """'; -""" -) -for sibling_package_txid in ("18" * 32, "19" * 32): - replacement.record_ctv_fanout_broadcast_attempt( - fanout_txid="17" * 32, - attempt_status="submitted", - package_tx_hexes=["02"], - package_txids=[sibling_package_txid], - submit_result={"accepted": True}, - ) -pending_fanout_page = replacement.dashboard_pending_fanout_rows(page=1, limit=15) -assert_equal( - [ - [ - row["fanout_txid"], - [attempt["package_txids"] for attempt in row["broadcast_attempts"]], - ] - for row in pending_fanout_page["rows"] - ], - [ - ["12" * 32, [["16" * 32]]], - ["17" * 32, [["18" * 32], ["19" * 32]]], - ], - "pending fanout rows carry only their own broadcast attempts, in order", -) -sibling_attempt_seqs = [attempt["attempt_seq"] for attempt in pending_fanout_page["rows"][1]["broadcast_attempts"]] -assert_equal(sibling_attempt_seqs, sorted(sibling_attempt_seqs), "pending fanout broadcast attempts ascend by attempt_seq") -replacement._run_sql( - "DELETE FROM qbit_ctv_fanout_broadcast_attempts WHERE fanout_txid = '" + "17" * 32 + "';\n" - "DELETE FROM qbit_ctv_fanout_artifacts WHERE fanout_txid = '" + "17" * 32 + "';" -) -duplicate_persist = replacement.persist_accepted_block( - block_hash="44" * 32, - block_height=7, - parent_hash="55" * 32, - final_bundle=bundle, - audit_report=report, -) -assert_equal( - [ - duplicate_persist["block_count"], - duplicate_persist["bundle_count"], - duplicate_persist["payout_entry_count"], - duplicate_persist["carry_forward_count"], - ], - [1, 1, 2, 2], - "duplicate accepted block verifies existing rows instead of reporting zero inserts", -) -bad_bundle = copy.deepcopy(bundle) -bad_bundle["found_block"] = {"bits": "207fffff"} -bad_report = audit_report_for(bad_bundle) -try: - replacement.persist_accepted_block( - block_hash="44" * 32, - block_height=7, - parent_hash="55" * 32, - final_bundle=bad_bundle, - audit_report=bad_report, - ) -except RuntimeError as exc: - if "existing audit bundle does not match payload" not in str(exc): - raise -else: - raise SystemExit("duplicate accepted block with mismatched audit digest unexpectedly succeeded") -owed = replacement.current_owed_balances() -assert_equal(owed, [], "prepared block must not affect owed balances before chain confirmation") -replacement._run_sql( - """ -INSERT INTO qbit_payout_carry_forward ( - block_height, - block_hash, - miner_id, - payout_order_key, - p2mr_program, - gross_amount_sats, - prior_balance_sats, - candidate_balance_sats, - onchain_amount_sats, - carry_forward_balance_sats, - action, - maturity_state -) VALUES - ( - 70, - NULL, - 'miner-unanchored-null', - 'unanchored-null', - decode('dd' || repeat('00', 31), 'hex'), - 777, - 0, - 777, - 0, - 777, - 'accrued', - 'immature' - ), - ( - 71, - '""" + "99" * 32 + """', - 'miner-unanchored-missing', - 'unanchored-missing', - decode('ee' || repeat('00', 31), 'hex'), - 888, - 0, - 888, - 0, - 888, - 'accrued', - 'immature' - ); -""" -) -unanchored_balances = replacement._run_json( - """ -SELECT COALESCE(json_agg(miner_id ORDER BY miner_id), '[]'::json) -FROM qbit_current_carry_forward_balances() -WHERE miner_id LIKE 'miner-unanchored-%'; -""" -) -assert_equal(unanchored_balances, [], "unanchored carry rows require a confirmed pool block") -unknown_confirmation = replacement.confirm_accepted_block( - block_hash="ff" * 32, - active_tip_height=7, -) -assert_equal( - unknown_confirmation, - {"backend": "postgres-psql", "confirmed_count": 0}, - "unknown block confirmation exposes no publication ordinal", -) -wrong_height_confirmation = replacement.confirm_accepted_block( - block_hash="44" * 32, - active_tip_height=8, -) -assert_equal( - wrong_height_confirmation, - {"backend": "postgres-psql", "confirmed_count": 0}, - "wrong-height confirmation exposes no publication ordinal", -) -confirmed = replacement.confirm_accepted_block(block_hash="44" * 32, active_tip_height=7) -assert_equal(confirmed["confirmed_count"], 1, "fresh confirmation reports the flip disposition") -first_publication_sequence = int(confirmed["audit_publication_sequence"]) -if first_publication_sequence <= 0: - raise SystemExit("first confirmed block received a non-positive publication ordinal") -allocator_after_flip = replacement._run_json( - "SELECT json_build_object('last_value', last_value, 'is_called', is_called) " - "FROM qbit_audit_publication_sequence_seq;" -) -confirmed_replay = replacement.confirm_accepted_block( - block_hash="44" * 32, - active_tip_height=7, -) -# The idempotent arm is the durable discriminator this contract exists for: -# the UPDATE matched nothing, so the row keeps the ordinal its flip -# allocated and the caller can tell a replay from a genuinely new -# confirmation that has to publish payout state (issue #61). -assert_equal( - confirmed_replay["confirmed_count"], - 2, - "exact confirmation replay reports the idempotent disposition", -) -assert_equal( - int(confirmed_replay["audit_publication_sequence"]), - first_publication_sequence, - "exact confirmation replay preserves publication ordinal", -) -assert_equal( - replacement._run_json( - "SELECT json_build_object('last_value', last_value, 'is_called', is_called) " - "FROM qbit_audit_publication_sequence_seq;" - ), - allocator_after_flip, - "exact confirmation replay burns no publication ordinal", -) -assert_equal( - replacement.dashboard_miner_pending_maturity_bits(recipient_id="miner-b"), - 49500, - "pending maturity totals net immature onchain outputs", -) -assert_equal( - replacement.dashboard_miner_pending_maturity_bits(recipient_id="miner-a"), - 0, - "pending maturity excludes accrued balances", -) -owed = replacement.current_owed_balances() -assert_equal([(row["recipient_id"], row["balance_sats"]) for row in owed], [("miner-a", 1000)], "owed balance before reversal") - -alias_block_hash = "ef" * 32 -alias_program_hex = "fa" * 32 -replacement._run_sql( - """ -INSERT INTO qbit_pool_blocks ( - block_hash, - block_height, - parent_hash, - coinbase_txid, - payout_manifest_sha256, - chain_state, - audit_publication_sequence -) VALUES ( - '""" + alias_block_hash + """', - 72, - '""" + "44" * 32 + """', - '""" + "46" * 32 + """', - '""" + "47" * 32 + """', - 'confirmed', - nextval('qbit_audit_publication_sequence_seq') -); - -INSERT INTO qbit_payout_carry_forward ( - block_height, - block_hash, - miner_id, - payout_order_key, - p2mr_program, - gross_amount_sats, - prior_balance_sats, - candidate_balance_sats, - onchain_amount_sats, - carry_forward_balance_sats, - action, - maturity_state -) VALUES - ( - 72, - '""" + alias_block_hash + """', - 'alias-miner-b', - '02', - decode('""" + alias_program_hex + """', 'hex'), - 200, - 0, - 200, - 0, - 200, - 'accrued', - 'immature' - ), - ( - 72, - '""" + alias_block_hash + """', - 'alias-miner-a', - '00', - decode('""" + alias_program_hex + """', 'hex'), - 100, - 0, - 100, - 0, - 100, - 'accrued', - 'immature' - ); -""" -) -alias_balances = replacement._run_json( - """ -SELECT COALESCE(json_agg(json_build_object( - 'recipient_id', miner_id, - 'order_key', payout_order_key, - 'balance_sats', balance_sats::text -) ORDER BY payout_order_key, miner_id), '[]'::json) -FROM qbit_current_carry_forward_balances() -WHERE p2mr_program = decode('""" + alias_program_hex + """', 'hex'); -""" -) -assert_equal( - alias_balances, - [{"recipient_id": "alias-miner-a", "order_key": "00", "balance_sats": "300"}], - "current carry balances aggregate same-payout-program aliases", -) -assert_current_matches_recomputed( - replacement, - "summary balances match recompute with same-payout-program aliases", -) -replacement._run_sql( - """ -UPDATE qbit_payout_carry_forward -SET maturity_state = 'reversed' -WHERE block_hash = '""" + alias_block_hash + """'; - -UPDATE qbit_pool_blocks -SET chain_state = 'reversed', - maturity_state = 'reversed', - disconnected_at = clock_timestamp() -WHERE block_hash = '""" + alias_block_hash + """'; -""" -) - -# Reorg visibility on the public block read model, against a real reversed -# row: the default view keeps hiding it, chain_state=reversed surfaces it -# with its disconnect time, and every pagination total describes exactly its -# own filtered set. -default_blocks = replacement.dashboard_blocks(page=1, limit=50) -all_blocks = replacement.dashboard_blocks(page=1, limit=50, chain_state="all") -reversed_blocks = replacement.dashboard_blocks(page=1, limit=50, chain_state="reversed") -if any(row["hash"] == alias_block_hash for row in default_blocks["rows"]): - raise SystemExit("default dashboard blocks view leaked a reversed block") -if any("chain_state" in row or "disconnected_at" in row for row in default_blocks["rows"]): - raise SystemExit("default dashboard blocks view leaked chain-state fields") -assert_equal( - all_blocks["pagination"]["total_count"], - default_blocks["pagination"]["total_count"] + reversed_blocks["pagination"]["total_count"], - "chain_state filters partition the block history", -) -reversed_rows_by_hash = {row["hash"]: row for row in reversed_blocks["rows"]} -if alias_block_hash not in reversed_rows_by_hash: - raise SystemExit("chain_state=reversed did not surface the reversed block") -reversed_row = reversed_rows_by_hash[alias_block_hash] -assert_equal(reversed_row["chain_state"], "reversed", "reversed block reports its chain state") -if not str(reversed_row["disconnected_at"] or "").endswith("Z"): - raise SystemExit("reversed block disconnected_at is not a UTC timestamp") -for row in all_blocks["rows"]: - if "chain_state" not in row or "disconnected_at" not in row: - raise SystemExit("chain_state=all rows must carry the state fields") - if (row["chain_state"] == "reversed") != (row["disconnected_at"] is not None): - raise SystemExit("disconnected_at must be set exactly for reversed rows") -paged_reversed = replacement.dashboard_blocks(page=1, limit=1, chain_state="reversed") -assert_equal( - paged_reversed["pagination"]["total_count"], - reversed_blocks["pagination"]["total_count"], - "reversed pagination total is limit-independent", -) -assert_equal(len(paged_reversed["rows"]), 1, "reversed page honours its limit") -reorg_snapshot = replacement.dashboard_pool_snapshot( - current_network_difficulty="1000", - generated_at="2026-01-01T00:00:00Z", -) -assert_equal( - reorg_snapshot["blocks_reversed_total"], - reversed_blocks["pagination"]["total_count"], - "pool snapshot reversed counter matches the reversed block view", -) -assert_equal( - reorg_snapshot["blocks_found_total"], - default_blocks["pagination"]["total_count"], - "blocks_found_total keeps excluding reversed blocks", -) - -zero_net_bundle = copy.deepcopy(bundle) -zero_net_bundle["signed_coinbase_manifest"]["manifest"]["payout_count"] = 1 -zero_net_bundle["payout_policy_manifest"]["accounts"] = [ - { - "recipient_id": "miner-a", - "order_key": "a", - "p2mr_program_hex": "aa" * 32, - "gross_amount_sats": 0, - "prior_balance_sats": 1000, - "candidate_balance_sats": 1000, - "onchain_amount_sats": 1000, - "carry_forward_balance_sats": 0, - "action": "onchain", - } -] -zero_net_report = audit_report_for(zero_net_bundle) -replacement.persist_accepted_block( - block_hash="45" * 32, - block_height=8, - parent_hash="44" * 32, - final_bundle=zero_net_bundle, - audit_report=zero_net_report, -) -force_expired_idle_lease(replacement) -second_confirmation = replacement.confirm_accepted_block( - block_hash="45" * 32, - active_tip_height=8, -) -second_publication_sequence = int(second_confirmation["audit_publication_sequence"]) -if second_publication_sequence == first_publication_sequence: - raise SystemExit("distinct confirmed blocks shared a publication ordinal") -zero_net_balances = replacement._run_json( - """ -SELECT COALESCE(json_agg(json_build_object( - 'recipient_id', miner_id, - 'balance_sats', balance_sats::text -) ORDER BY miner_id), '[]'::json) -FROM qbit_current_carry_forward_balances() -WHERE miner_id = 'miner-a'; -""" -) -assert_equal(zero_net_balances, [], "exact zero net carry replay omits current balance row") -force_expired_idle_lease(replacement) -replacement.reverse_immature_block(block_hash="45" * 32, active_tip_height=8) - -replacement.persist_accepted_block( - block_hash="46" * 32, - block_height=9, - parent_hash="45" * 32, - final_bundle=zero_net_bundle, - audit_report=zero_net_report, -) -# The stranded-prepared sweep finds exactly the rows the reorg watch read -# structurally cannot: prepared and buried past the reject depth floor. -# Nothing else re-examines them once their outbox row is gone. -assert_equal( - replacement.stranded_prepared_blocks( - active_tip_height=9 + 100, - min_depth=100, - ), - [{"block_hash": "46" * 32, "block_height": 9, "parent_hash": "45" * 32}], - "stranded prepared sweep finds a buried prepared row", -) -assert_equal( - replacement.stranded_prepared_blocks( - active_tip_height=9 + 99, - min_depth=100, - ), - [], - "stranded prepared sweep leaves rows inside the depth floor alone", -) -assert_equal( - replacement.stranded_prepared_blocks( - active_tip_height=9 + 100, - min_depth=100, - limit=0, - ), - [], - "stranded prepared sweep honours its page bound", -) -rejected_count = replacement.reject_prepared_block(block_hash="46" * 32, active_tip_height=8)["rejected_count"] -assert_equal(rejected_count, 3, "reject prepared block/payout/carry row count") -# The fenced rejection is what retires the row from the sweep. -assert_equal( - replacement.stranded_prepared_blocks( - active_tip_height=9 + 100, - min_depth=100, - ), - [], - "rejected block leaves the stranded prepared sweep", -) -rejected_state = replacement._run_json( - """ -SELECT json_build_object( - 'chain_state', chain_state, - 'maturity_state', maturity_state -) -FROM qbit_pool_blocks -WHERE block_hash = '""" + "46" * 32 + """'; -""" -) -assert_equal( - rejected_state, - {"chain_state": "rejected", "maturity_state": "reversed"}, - "rejected prepared block state", -) -# The reject flip stamps disconnected_at (its maturity_state 'reversed' -# requires it), but the public contract defines disconnected_at as a reorg -# disconnect time that is null for every non-reversed row -- a rejected -# block must not read as a reorg casualty. -rejected_all_view = replacement.dashboard_blocks(page=1, limit=50, chain_state="all") -rejected_public_row = next( - (row for row in rejected_all_view["rows"] if row["hash"] == "46" * 32), - None, -) -if rejected_public_row is None: - raise SystemExit("chain_state=all did not surface the rejected block") -assert_equal(rejected_public_row["chain_state"], "rejected", "rejected block reports its chain state") -if rejected_public_row["disconnected_at"] is not None: - raise SystemExit("rejected block must not report a reorg disconnect time") -assert_equal( - replacement.confirm_accepted_block( - block_hash="46" * 32, - active_tip_height=9, - ), - {"backend": "postgres-psql", "confirmed_count": -1}, - "rejected block confirmation reports the superseded disposition", -) - -try: - replacement._run_json("SELECT json_build_object('count', qbit_reverse_immature_pool_block('" + "44" * 32 + "', 7));") -except RuntimeError as exc: - if "function qbit_reverse_immature_pool_block" not in str(exc): - raise -else: - raise SystemExit("unfenced raw reversal function unexpectedly exists") - -inactive_count = replacement.mark_pool_block_inactive(block_hash="44" * 32, active_tip_height=7)["inactive_count"] -assert_equal(inactive_count, 1, "inactive block quarantine count") -assert_equal( - replacement.mark_pool_block_inactive( - block_hash="44" * 32, - active_tip_height=7, - )["inactive_count"], - 0, - "repeated inactive transition is count-zero", -) -assert_equal( - replacement.confirm_accepted_block( - block_hash="44" * 32, - active_tip_height=7, - ), - {"backend": "postgres-psql", "confirmed_count": -1}, - "inactive block confirmation reports the superseded disposition", -) -assert_equal(replacement.current_owed_balances(), [], "inactive owed balances are excluded") -reactivated = replacement.reactivate_pool_block(block_hash="44" * 32, active_tip_height=7) -assert_equal(reactivated["reactivated_count"], 1, "inactive block reactivation count") -reactivated_publication_sequence = int(reactivated["audit_publication_sequence"]) -assert_equal( - reactivated_publication_sequence, - first_publication_sequence, - "reactivated block preserves its published ordinal", -) -reactivated_confirmation = replacement.confirm_accepted_block( - block_hash="44" * 32, - active_tip_height=7, -) -# Reactivation restores 'confirmed' without returning the row to 'prepared', -# so a confirmation arriving after it is an idempotent replay too -- and it -# still addresses the ordinal the original flip allocated. -assert_equal( - reactivated_confirmation["confirmed_count"], - 2, - "confirmation after reactivation reports the idempotent disposition", -) -assert_equal( - int(reactivated_confirmation["audit_publication_sequence"]), - first_publication_sequence, - "confirmation after reactivation preserves the published ordinal", -) -if not replacement.current_owed_balances(): - raise SystemExit("reactivated block did not restore owed balances") -inactive_count = replacement.mark_pool_block_inactive(block_hash="44" * 32, active_tip_height=7)["inactive_count"] -assert_equal(inactive_count, 1, "inactive block can be quarantined again before final reversal") -reversed_count = replacement.reverse_immature_block(block_hash="44" * 32, active_tip_height=7)["reversed_count"] -assert_equal(reversed_count, 6, "reverse immature block/payout/carry/fanout row count") -assert_equal( - replacement.confirm_accepted_block( - block_hash="44" * 32, - active_tip_height=7, - ), - {"backend": "postgres-psql", "confirmed_count": -1}, - "reversed block confirmation reports the superseded disposition", -) -assert_equal(replacement.current_owed_balances(), [], "reversed owed balances are excluded") - -replacement.persist_accepted_block( - block_hash="66" * 32, - block_height=8, - parent_hash="44" * 32, - final_bundle=bundle, - audit_report=report, -) -replacement.confirm_accepted_block(block_hash="66" * 32, active_tip_height=8) - -# Issue #224: the reconciliation-owned mutations below (inactive quarantine, -# the maturation sweep, reactivation) move pool-block, payout-entry and carry -# state, and the coordinator relies on their never writing qbit_share_ledger -# to answer a confirmed mutation with a prior-balances reread instead of a -# full window rescan. Capture the share ledger's exact identity here and -# require it back after every transition, while the owed balance proves each -# transition actually happened. -share_ledger_identity_before = share_ledger_identity(replacement) - - -def assert_share_ledger_untouched(transition: str) -> None: - assert_equal( - share_ledger_identity(replacement), - share_ledger_identity_before, - f"qbit_share_ledger identity after {transition} (#224)", - ) - - -def owed_balances() -> list[tuple[str, int]]: - return [(row["recipient_id"], row["balance_sats"]) for row in replacement.current_owed_balances()] - - -assert_equal(owed_balances(), [("miner-a", 1000)], "confirmed block 66 owes its accrued balance") -inactive_count = replacement.mark_pool_block_inactive(block_hash="66" * 32, active_tip_height=8)["inactive_count"] -assert_equal(inactive_count, 1, "inactive block 66 quarantine count") -assert_equal(owed_balances(), [], "inactive block 66 owes nothing") -assert_share_ledger_untouched("inactive quarantine") -matured_while_inactive = replacement._run_json("SELECT json_build_object('count', qbit_mark_mature_pool_payouts(1008));")["count"] -assert_equal(matured_while_inactive, 0, "mature payout sweep ignores inactive blocks") -assert_share_ledger_untouched("maturation sweep over an inactive block") -inactive_count = replacement.mark_pool_block_inactive(block_hash="66" * 32, active_tip_height=1008)["inactive_count"] -assert_equal(inactive_count, 0, "height-mature inactive block quarantine is idempotent") -reactivated_count = replacement.reactivate_pool_block(block_hash="66" * 32, active_tip_height=1008)["reactivated_count"] -assert_equal(reactivated_count, 1, "height-mature inactive block reactivation count") -assert_equal(owed_balances(), [("miner-a", 1000)], "reactivated block 66 owes its accrued balance again") -assert_share_ledger_untouched("reactivation") -inactive_count = replacement.mark_pool_block_inactive(block_hash="66" * 32, active_tip_height=1008)["inactive_count"] -assert_equal(inactive_count, 1, "height-mature immature confirmed block can be quarantined") -assert_share_ledger_untouched("second inactive quarantine") -reactivated_count = replacement.reactivate_pool_block(block_hash="66" * 32, active_tip_height=1008)["reactivated_count"] -assert_equal(reactivated_count, 1, "height-mature re-quarantined block reactivation count") -assert_share_ledger_untouched("second reactivation") -matured_count = replacement._run_json("SELECT json_build_object('count', qbit_mark_mature_pool_payouts(1008));")["count"] -assert_equal(matured_count, 2, "mature payout entry count") -assert_share_ledger_untouched("maturation") -carry_states = replacement._run_json( - """ -SELECT COALESCE(json_agg(DISTINCT maturity_state ORDER BY maturity_state), '[]'::json) -FROM qbit_payout_carry_forward -WHERE block_hash = '""" + "66" * 32 + """'; -""" -) -assert_equal(carry_states, ["mature"], "mature payout sweep includes carry-forward rows") -assert_equal( - replacement.dashboard_miner_pending_maturity_bits(recipient_id="miner-b"), - 0, - "pending maturity excludes mature outputs", -) -try: - replacement.mark_pool_block_inactive(block_hash="66" * 32, active_tip_height=1008) -except RuntimeError as exc: - if "refusing to mark mature pool block inactive" not in str(exc): - raise -else: - raise SystemExit("mature block quarantine unexpectedly succeeded") -try: - replacement.reverse_immature_block(block_hash="66" * 32, active_tip_height=1008) -except RuntimeError as exc: - if "refusing to reverse mature pool block" not in str(exc): - raise -else: - raise SystemExit("mature block reversal unexpectedly succeeded") - -reorg_carry_block_a = "aa" * 32 -reorg_carry_block_b = "ab" * 32 -reorg_carry_miner = "miner-reorg-carry" -reorg_carry_bundle_a = copy.deepcopy(bundle) -reorg_carry_bundle_a["signed_coinbase_manifest"]["manifest"]["payout_count"] = 1 -reorg_carry_bundle_a["payout_policy_manifest"]["accounts"] = [ - { - "recipient_id": reorg_carry_miner, - "order_key": "reorg-carry", - "p2mr_program_hex": "cc" * 32, - "gross_amount_sats": 1000, - "prior_balance_sats": 0, - "candidate_balance_sats": 1000, - "onchain_amount_sats": 0, - "carry_forward_balance_sats": 1000, - "action": "accrued", - } -] -reorg_carry_bundle_b = copy.deepcopy(reorg_carry_bundle_a) -reorg_carry_bundle_b["payout_policy_manifest"]["accounts"][0]["gross_amount_sats"] = 250 -reorg_carry_bundle_b["payout_policy_manifest"]["accounts"][0]["prior_balance_sats"] = 1000 -reorg_carry_bundle_b["payout_policy_manifest"]["accounts"][0]["candidate_balance_sats"] = 1250 -reorg_carry_bundle_b["payout_policy_manifest"]["accounts"][0]["carry_forward_balance_sats"] = 1250 -reorg_carry_report_a = audit_report_for(reorg_carry_bundle_a) -reorg_carry_report_b = audit_report_for(reorg_carry_bundle_b) -replacement.persist_accepted_block( - block_hash=reorg_carry_block_a, - block_height=90, - parent_hash="66" * 32, - final_bundle=reorg_carry_bundle_a, - audit_report=reorg_carry_report_a, -) -replacement.confirm_accepted_block(block_hash=reorg_carry_block_a, active_tip_height=90) -replacement.persist_accepted_block( - block_hash=reorg_carry_block_b, - block_height=91, - parent_hash=reorg_carry_block_a, - final_bundle=reorg_carry_bundle_b, - audit_report=reorg_carry_report_b, -) -replacement.confirm_accepted_block(block_hash=reorg_carry_block_b, active_tip_height=91) -assert_equal( - replacement.reverse_immature_block(block_hash=reorg_carry_block_a, active_tip_height=91)["reversed_count"], - 3, - "reorg carry block A reversal count", -) -reorg_carry_balances = replacement._run_json( - """ -SELECT COALESCE(json_agg(json_build_object( - 'recipient_id', miner_id, - 'balance_sats', balance_sats::text -) ORDER BY miner_id), '[]'::json) -FROM qbit_current_carry_forward_balances() -WHERE miner_id = '""" + reorg_carry_miner + """'; -""" -) -assert_equal( - [(row["recipient_id"], int(row["balance_sats"])) for row in reorg_carry_balances], - [(reorg_carry_miner, 250)], - "later carry row remains independently valid after earlier row reversal", -) -assert_current_matches_recomputed( - replacement, - "summary balances match recompute while recorded chains are stale after mid-history reversal", -) -integrity_report = replacement.carry_forward_integrity_report() -assert_equal(integrity_report["mismatch_count"], 1, "replayed prior mismatch count") -assert_equal( - integrity_report["current_drift_count"], - 0, - "recorded-chain mismatches do not imply summary drift", -) -assert_equal( - integrity_report["audit_chain_version"], - "qbit.prism.carry-forward-active-delta-chain.v1", - "carry-forward audit chain version", -) -if integrity_report["audit_row_count"] <= 0: - raise SystemExit("carry-forward audit chain row count unexpectedly empty") -if len(integrity_report["audit_head_sha256"]) != 64: - raise SystemExit("carry-forward audit head is not a sha256 hex digest") -assert_equal( - integrity_report["mismatches"][0]["recipient_id"], - reorg_carry_miner, - "replayed prior mismatch recipient", -) -assert_equal( - integrity_report["mismatches"][0]["mismatch_reason"], - "prior_balance,candidate_balance,carry_forward_balance", - "replayed prior mismatch reason", -) -assert_equal( - replacement.reverse_immature_block(block_hash=reorg_carry_block_b, active_tip_height=91)["reversed_count"], - 3, - "reorg carry block B cleanup reversal count", -) -assert_equal( - replacement._run_json( - """ -SELECT COALESCE(json_agg(miner_id ORDER BY miner_id), '[]'::json) -FROM qbit_current_carry_forward_balances() -WHERE miner_id = '""" + reorg_carry_miner + """'; -""" - ), - [], - "reorg carry cleanup removes isolated miner from current balances", -) -assert_equal( - replacement.carry_forward_integrity_report()["mismatch_count"], - 0, - "reorg carry cleanup clears integrity mismatches", -) -assert_current_matches_recomputed( - replacement, - "summary balances match recompute after reorg cleanup", -) - -# Migration seeding: simulate a database that predates the transactionally -# maintained summary by clearing it, then let the next schema apply (ledger -# init below) rebuild it from carry history. -replacement._run_sql("DELETE FROM qbit_payout_carry_forward_current;") - -force_expired_idle_lease(replacement) -final_writer = PsqlShareLedger( - psql_command=psql, - writer_id="writer-c", - writer_epoch=1, - initialize_schema=True, - audit_bundle_canonicalizer=fake_audit_bundle_bytes, -) -assert_current_matches_recomputed( - final_writer, - "schema apply reseeds an empty summary from existing carry history", -) -if not final_writer._run_json( - "SELECT json_build_object('seeded', EXISTS (SELECT 1 FROM qbit_payout_carry_forward_current));" -)["seeded"]: - raise SystemExit("summary seeding left qbit_payout_carry_forward_current empty") -try: - replacement.persist_accepted_block( - block_hash="88" * 32, - block_height=9, - parent_hash="66" * 32, - final_bundle=bundle, - audit_report=report, - ) -except RuntimeError as exc: - if "writer lease is not active" not in str(exc): - raise -else: - raise SystemExit("stale writer persisted an accepted block after replacement acquired the lease") - -stale_bundle = copy.deepcopy(bundle) -stale_bundle["payout_policy_manifest"]["accounts"][0]["gross_amount_sats"] = 250 -stale_bundle["payout_policy_manifest"]["accounts"][0]["candidate_balance_sats"] = 250 -stale_bundle["payout_policy_manifest"]["accounts"][0]["carry_forward_balance_sats"] = 250 -stale_report = audit_report_for(stale_bundle) -final_writer.persist_accepted_block( - block_hash="77" * 32, - block_height=6, - parent_hash="66" * 32, - final_bundle=stale_bundle, - audit_report=stale_report, -) -try: - replacement.reverse_immature_block(block_hash="77" * 32, active_tip_height=6) -except RuntimeError as exc: - if "writer lease is not active" not in str(exc): - raise -else: - raise SystemExit("stale writer reversed a block after replacement acquired the lease") -assert_equal( - [(row["recipient_id"], row["balance_sats"]) for row in final_writer.current_owed_balances()], - [("miner-a", 1000)], - "out-of-order lower-height carry-forward persistence must not rewind current balance", -) -assert_equal( - final_writer.reverse_immature_block(block_hash="77" * 32, active_tip_height=6)["reversed_count"], - 5, - "active replacement writer can reverse its prepared block", -) - -final_writer._run_sql( - """ -UPDATE qbit_ledger_writer_lease -SET writer_id = 'deploy-writer', - writer_epoch = 7, - writer_session_token = 'old-deploy-session', - lease_expires_at = clock_timestamp() + interval '0.2 seconds', - updated_at = clock_timestamp() -WHERE singleton; -""" -) -deploy_writer = PsqlShareLedger( - psql_command=psql, - writer_id="deploy-writer", - writer_epoch=7, - lease_retry_max_sleep_seconds=0.05, -) -deploy_lease = deploy_writer._run_json( - """ -SELECT json_build_object( - 'writer_id', writer_id, - 'writer_epoch', writer_epoch, - 'writer_session_token', writer_session_token -) -FROM qbit_ledger_writer_lease -WHERE singleton; -""" -) -assert_equal( - [ - deploy_lease["writer_id"], - deploy_lease["writer_epoch"], - deploy_lease["writer_session_token"], - ], - ["deploy-writer", 7, deploy_writer._writer_session_token], - "same writer predecessor startup waits for lease expiry and then acquires", -) - -with tempfile.TemporaryDirectory() as audit_body_dir: - external_writer = PsqlShareLedger( - psql_command=psql, - writer_id="deploy-writer", - writer_epoch=7, - writer_session_token=deploy_writer._writer_session_token, - audit_body_dir=audit_body_dir, - audit_bundle_canonicalizer=fake_audit_bundle_bytes, - ) - external_block_hash = "d9" * 32 - commitment_leaf = "ab" * 32 - witness_leaf = "cd" * 32 - external_bundle = copy.deepcopy(bundle) - external_bundle["schema"] = "qbit.prism.audit-bundle.v1" - external_bundle["found_block"] = { - "network_difficulty": 2500, - "bits": "207fffff", - "coinbase_value_sats": 123456, - } - external_bundle["audit_commitment_leaves_hex"] = [commitment_leaf] - external_bundle["witness_merkle_leaves_hex"] = [witness_leaf] - external_report = { - "coinbase_txid": "da" * 32, - "coinbase_manifest_sha256_hex": "98" * 32, - "audit_bundle_sha256_hex": fake_audit_bundle_sha256(external_bundle), - "coinbase_tx_hex": "01", - } - external_persist = external_writer.persist_accepted_block( - block_hash=external_block_hash, - block_height=12, - parent_hash="97" * 32, - final_bundle=external_bundle, - audit_report=external_report, - ) - assert_equal( - [ - external_persist["block_count"], - external_persist["bundle_count"], - external_persist["payout_entry_count"], - external_persist["carry_forward_count"], - ], - [1, 1, 2, 2], - "externalized accepted block persistence counts", - ) - body_files = sorted(Path(audit_body_dir).glob("prism-audit-bundle-body-*.json")) - assert_equal(len(body_files), 1, "externalized audit body writes exactly one file") - if external_report["audit_bundle_sha256_hex"] not in body_files[0].name: - raise SystemExit("externalized audit body filename does not include bundle sha256") - - external_row = external_writer._run_json( - """ -SELECT json_build_object( - 'inline_body', audit_bundle IS NOT NULL, - 'body_uri', body_uri, - 'network_difficulty', found_block_network_difficulty::text, - 'bits', found_block_bits, - 'coinbase_value_sats', found_block_coinbase_value_sats, - 'commitment_leaf_count', jsonb_array_length(audit_commitment_leaves_hex), - 'witness_leaf_count', jsonb_array_length(witness_merkle_leaves_hex) -) -FROM qbit_pool_audit_bundles -WHERE block_hash = '""" + external_block_hash + """'; -""" - ) - assert_equal(external_row["inline_body"], False, "externalized row does not keep inline audit_bundle") - assert_equal(external_row["body_uri"], str(body_files[0].resolve()), "externalized row points at body file") - assert_equal( - [ - external_row["network_difficulty"], - external_row["bits"], - external_row["coinbase_value_sats"], - external_row["commitment_leaf_count"], - external_row["witness_leaf_count"], - ], - ["2500", "207fffff", 123456, 1, 1], - "externalized row stores promoted metadata", - ) - assert_equal( - external_writer.audit_bundle(block_hash=external_block_hash)["audit_bundle"], - external_bundle, - "audit_bundle reader resolves external body", - ) - assert_equal( - external_writer.audit_bundle_by_commitment(commitment_leaf_hex=commitment_leaf)["audit_bundle"], - external_bundle, - "commitment reader resolves external body from promoted leaves", - ) - assert_equal( - external_writer.dashboard_public_artifact(sha256=external_report["audit_bundle_sha256_hex"]), - external_bundle, - "public artifact reader resolves external body", - ) - external_blocks = external_writer.dashboard_blocks(page=1, limit=1) - assert_equal( - [ - external_blocks["rows"][0]["hash"], - external_blocks["rows"][0]["network_difficulty"], - external_blocks["rows"][0]["bits"], - external_blocks["rows"][0]["coinbase_value_bits"], - ], - [external_block_hash, "2500", "207fffff", 123456], - "dashboard blocks reads promoted metadata without inline bundle", - ) - duplicate_external = external_writer.persist_accepted_block( - block_hash=external_block_hash, - block_height=12, - parent_hash="97" * 32, - final_bundle=external_bundle, - audit_report=external_report, - ) - assert_equal(duplicate_external["bundle_count"], 1, "externalized duplicate verifies existing bundle") - assert_equal( - len(list(Path(audit_body_dir).glob("prism-audit-bundle-body-*.json"))), - 1, - "externalized duplicate reuses body file", - ) - body_files[0].unlink() - force_expired_idle_lease(external_writer) - external_successor = PsqlShareLedger( - psql_command=psql, - writer_id="external-successor", - writer_epoch=1, - audit_body_dir=audit_body_dir, - audit_bundle_canonicalizer=fake_audit_bundle_bytes, - ) - try: - external_writer.persist_accepted_block( - block_hash=external_block_hash, - block_height=12, - parent_hash="97" * 32, - final_bundle=external_bundle, - audit_report=external_report, - ) - except RuntimeError as exc: - if "writer lease is not active" not in str(exc): - raise - else: - raise SystemExit("stale externalized duplicate restored a missing body after losing lease") - assert_equal( - len(list(Path(audit_body_dir).glob("prism-audit-bundle-body-*.json"))), - 0, - "stale externalized duplicate does not restore missing body before lease", - ) - restored_external = external_successor.persist_accepted_block( - block_hash=external_block_hash, - block_height=12, - parent_hash="97" * 32, - final_bundle=external_bundle, - audit_report=external_report, - ) - assert_equal( - restored_external["bundle_count"], - 1, - "active successor externalized duplicate restores missing body after lease", - ) - restored_body_files = sorted(Path(audit_body_dir).glob("prism-audit-bundle-body-*.json")) - assert_equal(len(restored_body_files), 1, "externalized duplicate restores missing body file") - assert_equal( - external_successor.audit_bundle(block_hash=external_block_hash)["audit_bundle"], - external_bundle, - "restored external body remains readable", - ) - conflicting_bundle = copy.deepcopy(external_bundle) - conflicting_bundle["found_block"]["coinbase_value_sats"] = 123457 - conflicting_report = { - **external_report, - "audit_bundle_sha256_hex": fake_audit_bundle_sha256(conflicting_bundle), - } - try: - external_successor.persist_accepted_block( - block_hash=external_block_hash, - block_height=12, - parent_hash="97" * 32, - final_bundle=conflicting_bundle, - audit_report=conflicting_report, - ) - except RuntimeError as exc: - if "existing audit bundle does not match payload" not in str(exc): - raise - else: - raise SystemExit("externalized conflicting duplicate unexpectedly succeeded") - assert_equal( - len(list(Path(audit_body_dir).glob("prism-audit-bundle-body-*.json"))), - 1, - "externalized conflicting duplicate does not write an orphan body file", - ) - -external_successor.release_writer_lease() - -# A pre-ordinal writer confirms with a plain chain_state UPDATE that never -# mentions the publication ordinal. The BEFORE INSERT OR UPDATE trigger must -# assign the next ordinal for that write so the validated CHECK constraint -# stays satisfiable under a code-only rollback, while an ordinal-aware -# confirmation that assigns nextval() explicitly stays untouched. -legacy_confirm_hash = "e9" * 32 -replacement._run_sql( - """ -INSERT INTO qbit_pool_blocks ( - block_hash, - block_height, - parent_hash, - coinbase_txid, - payout_manifest_sha256, - chain_state -) VALUES ( - '""" + legacy_confirm_hash + """', - 95, - '""" + "ab" * 32 + """', - '""" + "4a" * 32 + """', - '""" + "4b" * 32 + """', - 'prepared' -); -""" -) -legacy_before = replacement._run_json( - """ -SELECT json_build_object( - 'ordinal', ( - SELECT audit_publication_sequence - FROM qbit_pool_blocks - WHERE block_hash = '""" + legacy_confirm_hash + """' - ), - 'max_ordinal', ( - SELECT COALESCE(MAX(audit_publication_sequence), 0) - FROM qbit_pool_blocks - ) -); -""" -) -assert_equal(legacy_before["ordinal"], None, "prepared legacy block carries no ordinal") -replacement._run_sql( - """ -UPDATE qbit_pool_blocks -SET chain_state = 'confirmed' -WHERE block_hash = '""" + legacy_confirm_hash + """'; -""" -) -legacy_after = replacement._run_json( - """ -SELECT json_build_object( - 'ordinal', ( - SELECT audit_publication_sequence - FROM qbit_pool_blocks - WHERE block_hash = '""" + legacy_confirm_hash + """' - ), - 'duplicate_ordinals', ( - SELECT count(*) - FROM ( - SELECT audit_publication_sequence - FROM qbit_pool_blocks - WHERE audit_publication_sequence IS NOT NULL - GROUP BY audit_publication_sequence - HAVING count(*) > 1 - ) AS duplicated - ) -); -""" -) -if legacy_after["ordinal"] is None: - raise SystemExit("legacy ordinal-less confirmation was not assigned an ordinal") -legacy_ordinal = int(legacy_after["ordinal"]) -if legacy_ordinal <= int(legacy_before["max_ordinal"]): - raise SystemExit("legacy confirmation ordinal did not extend publication order") -assert_equal( - legacy_after["duplicate_ordinals"], - 0, - "publication ordinals stay unique after a legacy confirmation", -) -explicit_confirm_hash = "e8" * 32 -replacement._run_sql( - """ -INSERT INTO qbit_pool_blocks ( - block_hash, - block_height, - parent_hash, - coinbase_txid, - payout_manifest_sha256, - chain_state -) VALUES ( - '""" + explicit_confirm_hash + """', - 96, - '""" + legacy_confirm_hash + """', - '""" + "4c" * 32 + """', - '""" + "4d" * 32 + """', - 'prepared' -); - -UPDATE qbit_pool_blocks -SET chain_state = 'confirmed', - audit_publication_sequence = nextval('qbit_audit_publication_sequence_seq') -WHERE block_hash = '""" + explicit_confirm_hash + """'; -""" -) -explicit_after = replacement._run_json( - """ -SELECT json_build_object( - 'ordinal', ( - SELECT audit_publication_sequence - FROM qbit_pool_blocks - WHERE block_hash = '""" + explicit_confirm_hash + """' - ) -); -""" -) -assert_equal( - int(explicit_after["ordinal"]), - legacy_ordinal + 1, - "explicit ordinal-aware confirmation continues the same allocation order", -) - -# Everything above runs on fixtures far smaller than one 4096-row cutoff page, -# so the recursive page step of the bounded window readers never executes. -# Seed enough unit-difficulty shares that the paged cutoff walk must cross page -# boundaries, then pin exact crossing behavior for both readers. With every -# share_difficulty = 1 the descending cumulative weight at the i-th newest -# share is exactly i, so a window weight of W must return precisely the W -# newest shares (all 9000 when W exceeds the history). -replacement._run_sql( - """ -DELETE FROM qbit_block_candidate_outbox; -DELETE FROM qbit_share_ledger; -ALTER SEQUENCE qbit_share_ledger_share_seq_seq RESTART WITH 1; -INSERT INTO qbit_share_ledger ( - share_id, miner_id, payout_order_key, p2mr_program, - share_difficulty, network_difficulty, template_height, job_id, - job_issued_at, ntime, accepted_at, accepted, writer_id, writer_epoch -) -SELECT - 'bulk-' || g, - 'miner-' || (g % 5), - lpad((g % 5)::text, 4, '0'), - decode(md5(g::text) || md5((g + 7)::text), 'hex'), - 1, - 1000, - 10, - 'job-bulk', - to_timestamp(1700000000) + (g * interval '1 millisecond'), - 1700000000, - to_timestamp(1700000000) + (g * interval '1 millisecond'), - TRUE, - 'writer-bulk', - 1 -FROM generate_series(1, 9000) AS g; -ANALYZE qbit_share_ledger; -""" -) -bulk_anchor_ms = 1_700_000_010_000 -for weight, expected_len in [ - (4095, 4095), - (4096, 4096), - (4097, 4097), - (8192, 8192), - (8193, 8193), - (9000, 9000), - (12000, 9000), -]: - bounded = replacement.snapshot_at_job_issue(bulk_anchor_ms, window_weight=weight) - assert_equal( - len(bounded), - expected_len, - f"multi-page bounded snapshot row count at window weight {weight}", - ) - assert_equal( - bounded[0].share_id, - f"bulk-{9000 - expected_len + 1}", - f"multi-page bounded snapshot oldest row at window weight {weight}", - ) - assert_equal( - bounded[-1].share_id, - "bulk-9000", - f"multi-page bounded snapshot newest row at window weight {weight}", - ) -assert_equal( - len(replacement.snapshot_at_job_issue(bulk_anchor_ms)), - 9000, - "multi-page unbounded snapshot matches the beyond-history bounded snapshot", -) -for network_difficulty, expected_len in [(512, 4096), (1024, 8192), (1025, 8200)]: - bulk_window = replacement.audit_share_window( - anchor_job_issued_at_ms=bulk_anchor_ms, - network_difficulty=network_difficulty, - ) - assert_equal( - len(bulk_window), - expected_len, - f"multi-page audit window row count at network difficulty {network_difficulty}", - ) - assert_equal( - bulk_window[0]["share_id"], - "bulk-9000", - f"multi-page audit window newest row at network difficulty {network_difficulty}", - ) - assert_equal( - bulk_window[-1]["share_id"], - f"bulk-{9000 - expected_len + 1}", - f"multi-page audit window crossing row at network difficulty {network_difficulty}", - ) - -# Issue #236: both payout-window reads now return one JSON object per row, -# decoded by the psql backend line by line from a spooled result instead of -# one json_agg value. The selection did not change, so the pre-#236 aggregate -# statement is the oracle: same rows, same order, same field values, across -# the 9000-row fixture (17 batches of 512, a boundary inside the crossing -# page), every exact weighted cutoff above, the unbounded full history, an -# empty window, and the delta's disjoint eligibility branches. -assert_equal(replacement.execution_backend, "psql-subprocess", "oracle check runs on the psql backend") -for weight, expected_len in [(4095, 4095), (4096, 4096), (4097, 4097), (8192, 8192), (9000, 9000), (12000, 9000)]: - assert_rows_match_aggregate( - replacement, - lambda: replacement.snapshot_at_job_issue(bulk_anchor_ms, window_weight=weight), - label=f"per-row bounded snapshot at window weight {weight}", - expected_len=expected_len, - ) -assert_rows_match_aggregate( - replacement, - lambda: replacement.snapshot_at_job_issue(bulk_anchor_ms), - label="per-row unbounded snapshot", - expected_len=9000, -) -assert_rows_match_aggregate( - replacement, - lambda: replacement.snapshot_at_job_issue(0, window_weight=64), - label="per-row empty window", - expected_len=0, -) -assert_rows_match_aggregate( - replacement, - lambda: replacement.snapshot_at_job_issue(1_700_000_000_001, window_weight=1), - label="per-row single-row window", - expected_len=1, -) -assert_rows_match_aggregate( - replacement, - lambda: replacement.snapshot_between_job_issues(1_700_000_004_000, 1_700_000_006_000), - label="per-row delta inside the bulk fixture", - expected_len=2000, -) -assert_rows_match_aggregate( - replacement, - lambda: replacement.snapshot_between_job_issues(0, bulk_anchor_ms), - label="per-row delta over the whole history", -) - -# A read-only ledger against this writable primary. Everything above proved -# the database accepts writes, which is exactly what makes it the right target: -# a standby would refuse these writes whoever asked, so it can never show that -# read_only=True is what refused them. -read_only_ledger = PsqlShareLedger( - psql_command=psql_with_env, - writer_id="public-read", - writer_epoch=1, - read_only=True, -) - -# release_writer_lease_fresh_connection issues an UPDATE over a one-shot psql -# connection, taking neither the writer gate nor the pool. The in-process gate -# cannot refuse it, so the server must. -try: - read_only_ledger.release_writer_lease_fresh_connection() -except RuntimeError as exc: - if "read-only transaction" not in str(exc): - raise -else: - raise SystemExit("read-only ledger wrote to a writable primary") - -# The read tier still reads: the session is read-only, not unusable. This goes -# through a read slot, which is the gate every public route takes; all_shares() -# and the other O(n) reads take the writer gate and are refused in-process, -# which is the pre-existing half of the promise. -assert_equal( - read_only_ledger.accepted_share_stats()["accepted_share_count"] > 0, - True, - "read-only ledger still serves reads", -) -read_only_ledger.close() - -# The same statement, the same identity, the same database, differing only in -# read_only: it lands. So the refusal above is the read-only session and not -# the statement, the schema, or the state of the lease row. -control_writer = PsqlShareLedger( - psql_command=psql_with_env, - writer_id="public-read", - writer_epoch=1, -) -assert_equal( - control_writer.release_writer_lease_fresh_connection(), - True, - "writable ledger of the same identity still releases its own lease", -) -control_writer.close() - -print("prism postgres ledger PASS shares=14 lease=replay startup-retry persist-fence sql-window bulk-abandon=fenced maturity=reorg carry-replay integrity multipage-window=9000 window-rows=oracle read-only-session share-ledger-identity=inactive+reactivate+mature") -PY - - PRISM_PSQL_COMMAND="${PSQL_COMMAND}" \ - python3 -m tests.prism_postgres_a1_gate - - PRISM_PSQL_COMMAND="${PSQL_COMMAND}" \ - python3 -m tests.prism_postgres_a1_migration_gate - - PRISM_PSQL_COMMAND="${PSQL_COMMAND}" \ - python3 -m tests.prism_postgres_a1_revert_gate - - PRISM_PSQL_COMMAND="${PSQL_COMMAND}" \ - QBIT_PRISM_GATE_IMAGE="${GATE_IMAGE}" \ - QBIT_PRISM_GATE_IMAGE_DIGEST="${GATE_IMAGE_DIGEST}" \ - python3 -m tests.prism_postgres_a1_process_gate -) diff --git a/test/test-prism-postgres-native-ledger.sh b/test/test-prism-postgres-native-ledger.sh deleted file mode 100755 index 48edeb324..000000000 --- a/test/test-prism-postgres-native-ledger.sh +++ /dev/null @@ -1,605 +0,0 @@ -#!/usr/bin/env bash -# End-to-end validation of the persistent pooled psycopg share-ledger backend -# (PRISM_POSTGRES_NATIVE_CLIENT=1) against a real Postgres: schema init, lease -# acquisition, batched synchronous share appends, duplicate handling, cached -# accepted stats, and cross-backend read consistency with the psql fallback. -# -# Requires docker and a host python3 with psycopg installed -# (python3 -m pip install 'psycopg[binary]'). -set -euo pipefail - -ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" -# shellcheck source=test/prism-postgres-lib.sh -source "${ROOT_DIR}/test/prism-postgres-lib.sh" - -POSTGRES_IMAGE="${QBIT_PRISM_POSTGRES_IMAGE:-postgres:16-alpine}" -POSTGRES_CONTAINER="${QBIT_PRISM_POSTGRES_CONTAINER:-qbit-prism-native-ledger-pg-$$}" - -require_executable() { - command -v "$1" >/dev/null 2>&1 || { - echo "missing required executable: $1" >&2 - exit 1 - } -} - -cleanup() { - docker rm -f "${POSTGRES_CONTAINER}" >/dev/null 2>&1 || true -} -trap cleanup EXIT - -require_executable docker -require_executable python3 - -python3 -c 'import psycopg' 2>/dev/null || { - echo "host python3 lacks psycopg; install with: python3 -m pip install 'psycopg[binary]'" >&2 - exit 1 -} - -docker rm -f "${POSTGRES_CONTAINER}" >/dev/null 2>&1 || true -docker run \ - --rm \ - --detach \ - --name "${POSTGRES_CONTAINER}" \ - -p 127.0.0.1:0:5432 \ - -e POSTGRES_USER=qbit \ - -e POSTGRES_PASSWORD=qbit \ - -e POSTGRES_DB=qbit \ - "${POSTGRES_IMAGE}" >/dev/null - -wait_for_prism_postgres_container "${POSTGRES_CONTAINER}" - -HOST_PORT="$(docker port "${POSTGRES_CONTAINER}" 5432/tcp | head -n 1 | awk -F: '{print $NF}')" -if [[ -z "${HOST_PORT}" ]]; then - echo "unable to resolve published Postgres port" >&2 - exit 1 -fi -DATABASE_URL="postgresql://qbit:qbit@127.0.0.1:${HOST_PORT}/qbit" - -# Wait for the published listener too (pg_isready above ran inside the -# container over the unix socket). -deadline=$((SECONDS + 60)) -until python3 - "$DATABASE_URL" <<'PY' >/dev/null 2>&1 -import sys -import psycopg - -with psycopg.connect(sys.argv[1], connect_timeout=3) as conn: - conn.execute("SELECT 1") -PY -do - if [[ "${SECONDS}" -ge "${deadline}" ]]; then - echo "timed out waiting for published PRISM Postgres port" >&2 - exit 1 - fi - sleep 1 -done - -( - cd "${ROOT_DIR}" - PRISM_TEST_DATABASE_URL="${DATABASE_URL}" \ - PRISM_TEST_PSQL_COMMAND="docker exec -i ${POSTGRES_CONTAINER} psql -U qbit -d qbit" \ - PRISM_TEST_PSQL_COMMAND_WITH_ENV="docker exec -i -e PGOPTIONS ${POSTGRES_CONTAINER} psql -U qbit -d qbit" \ - python3 <<'PY' -from __future__ import annotations - -import os -import threading -import time - -from lab.prism.share_ledger import ( - LedgerOperationTimeout, - PendingShare, - PsqlShareLedger, -) -from tests.prism_window_rows_reference import assert_rows_match_aggregate - - -def pending( - index: int, - *, - share_id: str | None = None, - miner_id: str | None = None, -) -> PendingShare: - return PendingShare( - share_id=share_id or f"share-{index}", - miner_id=miner_id or f"miner-{index % 4}", - order_key=f"{index:04d}", - p2mr_program_hex=f"{index % 256:02x}" * 32, - share_difficulty=100 + index, - network_difficulty=1_000, - template_height=10, - job_id=f"job-{index}", - job_issued_at_ms=1_700_000_000_000 + index, - accepted_at_ms=1_700_000_001_000 + index, - ntime=1_700_000_000 + index, - ) - - -def assert_equal(actual: object, expected: object, message: str) -> None: - if actual != expected: - raise SystemExit(f"{message}: expected {expected!r}, got {actual!r}") - - -database_url = os.environ["PRISM_TEST_DATABASE_URL"] -psql_command = os.environ["PRISM_TEST_PSQL_COMMAND"] -# Session GUCs reach a psql subprocess through PGOPTIONS, and "docker exec" -# does not forward the caller's environment, so the read-only assertions below -# use a command that asks docker to pass the variable through. -psql_command_with_env = os.environ["PRISM_TEST_PSQL_COMMAND_WITH_ENV"] - -ledger = PsqlShareLedger( - psql_command=psql_command, - database_url=database_url, - native_client_mode="1", - writer_id="writer-native", - writer_epoch=1, - initialize_schema=True, - accepted_stats_cache_seconds=60.0, -) -assert_equal(ledger.execution_backend, "psycopg-pool", "native backend selected") - -# Concurrent per-share appends exercise the pooled client under the writer -# lock from many threads, each caller getting its own canonical record back. -single_count = 16 -records: dict[int, object] = {} -errors: list[BaseException] = [] -barrier = threading.Barrier(8) - - -def run(worker_index: int) -> None: - try: - barrier.wait() - for offset in range(single_count // 8): - index = worker_index * (single_count // 8) + offset - records[index] = ledger.append(pending(index)) - except BaseException as exc: # noqa: BLE001 - surfaced below - errors.append(exc) - - -threads = [threading.Thread(target=run, args=(index,)) for index in range(8)] -for thread in threads: - thread.start() -for thread in threads: - thread.join(timeout=60) - -assert_equal(errors, [], "concurrent appends succeed") -assert_equal(len(records), single_count, "all appends returned records") -assert_equal( - sorted(record.share_seq for record in records.values()), - list(range(1, single_count + 1)), - "canonical share sequence is contiguous", -) -for index, record in records.items(): - assert_equal(record.share_id, f"share-{index}", "each caller got its own share back") -assert_equal( - ledger.accepted_share_stats()["accepted_share_count"], - single_count, - "cached share count before batch", -) - -# The coordinator's share-writer group commit lands one atomic append_batch -# statement; on the pooled client that is a single synchronously committed -# round trip. -share_count = 32 -batch_entries = [(pending(index), None) for index in range(single_count, share_count)] -batch_entries[0] = (pending(single_count, miner_id="miner-new"), None) -batch_entries[1] = (pending(single_count + 1, miner_id="miner-new"), None) -batch_records = ledger.append_batch(batch_entries) -assert_equal( - [record.share_id for record in batch_records], - [f"share-{index}" for index in range(single_count, share_count)], - "batch preserves submission order", -) -assert_equal( - sorted(record.share_seq for record in batch_records), - list(range(single_count + 1, share_count + 1)), - "batch extends the canonical sequence", -) -stats_before_replay = ledger.accepted_share_stats() -assert_equal(stats_before_replay["accepted_share_count"], share_count, "cached share count before replay") -assert_equal( - stats_before_replay["distinct_miner_count"], - 5, - "batch increments a new miner exactly once", -) - -# Replaying the exact same batch is idempotent: the same records come back -# and neither the database nor the accepted-share cache double-counts it. -replayed_records = ledger.append_batch(batch_entries) -assert_equal( - [(record.share_seq, record.share_id) for record in replayed_records], - [(record.share_seq, record.share_id) for record in batch_records], - "batch replay returns the originally committed records", -) -assert_equal( - ledger.accepted_share_stats()["accepted_share_count"], - share_count, - "batch replay leaves cached share count unchanged", -) - -try: - ledger.append(pending(999, share_id="share-0")) -except RuntimeError as exc: - if "duplicate share_id" not in str(exc): - raise -else: - raise SystemExit("duplicate share replay unexpectedly appended") - -# Stats reconcile against the database (the cache was advanced by the replay -# too, so force a fresh aggregate to prove the committed state is exact). -ledger._accepted_stats_cache_seconds = 0.0 -stats = ledger.accepted_share_stats() -assert_equal(stats["accepted_share_count"], share_count, "accepted share count") -assert_equal(stats["distinct_miner_count"], 5, "distinct miner count") -ledger._accepted_stats_cache_seconds = 60.0 - -snapshot = ledger.snapshot_at_job_issue(1_700_000_002_000) -assert_equal(len(snapshot), share_count, "snapshot returns all committed shares") - - -metrics = ledger.metrics() -assert_equal(metrics["shares"], share_count, "metrics share count from cached stats") - -# Issue #211: the replay enumeration is a read and must not queue on the -# coordinator-local writer gate. This is the production call shape -- the real -# statement, over the real pooled psycopg client, against a real server -- run -# while an accounting-shaped writer holds that gate for longer than the whole -# fast-call budget. Before the fix the page spent that budget on admission and -# never reached PostgreSQL. -candidate_hash = "ab" * 32 -second_candidate_hash = "cd" * 32 -candidate_intent = { - "schema": "qbit.prism.block-candidate-intent.v1", - "block_hash_hex": candidate_hash, - "block_hex": "00", -} -assert ledger.persist_block_candidate_intent(candidate_intent) -assert ledger.persist_block_candidate_intent( - {**candidate_intent, "block_hash_hex": second_candidate_hash} -) - -FAST_CALL_BUDGET_SECONDS = 1.0 -gate_taken = threading.Event() -release_gate = threading.Event() -holder_error: list[BaseException] = [] - - -def hold_writer_gate() -> None: - try: - with ledger._operation_gate(ledger._lock, "writer lock"): - gate_taken.set() - release_gate.wait(timeout=60) - except BaseException as exc: # noqa: BLE001 - surfaced below - holder_error.append(exc) - gate_taken.set() - - -holder = threading.Thread(target=hold_writer_gate, daemon=True) -holder.start() -if not gate_taken.wait(timeout=30): - raise SystemExit("the writer gate holder never acquired the gate") -if holder_error: - raise holder_error[0] - -try: - started = time.monotonic() - with ledger.operation_timeout(FAST_CALL_BUDGET_SECONDS): - page = ledger.pending_block_candidate_rows(limit=1) - enumeration_seconds = time.monotonic() - started - - # It completed, inside its own budget, with the gate still held. - assert_equal( - [row["block_hash"] for row in page], - [candidate_hash], - "pending page enumerated while the writer gate was held", - ) - assert_equal( - [row["pool_block_exists"] for row in page], - [False], - "pending page carried the landed-block fact from the same snapshot", - ) - if enumeration_seconds >= FAST_CALL_BUDGET_SECONDS: - raise SystemExit( - "pending page took " - f"{enumeration_seconds:.3f}s of a {FAST_CALL_BUDGET_SECONDS:g}s budget" - ) - - # Pagination stays exact across the same held gate: the cursor resumes - # strictly after its own row and the short page proves the end. - with ledger.operation_timeout(FAST_CALL_BUDGET_SECONDS): - second_page = ledger.pending_block_candidate_rows( - limit=1, - after_cursor=page[0]["cursor"], - ) - assert_equal( - [row["block_hash"] for row in second_page], - [second_candidate_hash], - "pending page cursor resumed strictly after its own row", - ) - with ledger.operation_timeout(FAST_CALL_BUDGET_SECONDS): - assert_equal( - ledger.pending_block_candidate_rows( - limit=1, - after_cursor=second_page[0]["cursor"], - ), - [], - "a cursor past every pending row proves the walk complete", - ) - - # The control. The same gate, the same budget, asked for writer admission - # instead: it times out, which is what the enumeration used to do. - try: - with ledger.operation_timeout(FAST_CALL_BUDGET_SECONDS): - ledger._acquire_operation_gate(ledger._lock, "writer lock") - except LedgerOperationTimeout as exc: - if "writer lock" not in str(exc): - raise - else: - ledger._lock.release() - raise SystemExit("the writer gate was not actually held") -finally: - release_gate.set() - holder.join(timeout=30) -if holder.is_alive(): - raise SystemExit("the writer gate holder never released the gate") -if holder_error: - raise holder_error[0] - -# The attribution the next budget exhaustion will be read from: no time on -# local admission, real time in PostgreSQL, and neither timeout counter armed. -read_gate_stats = ledger.ledger_read_gate_stats()["pending_block_candidate_headers"] -assert_equal(int(read_gate_stats["calls_total"]), 3, "read-slot calls counted") -assert_equal(int(read_gate_stats["gate_timeouts_total"]), 0, "no admission expiry") -assert_equal(int(read_gate_stats["execute_timeouts_total"]), 0, "no statement expiry") -if float(read_gate_stats["execute_seconds_total"]) <= 0.0: - raise SystemExit("pending page recorded no PostgreSQL execution time") -if float(read_gate_stats["gate_wait_seconds_max"]) >= FAST_CALL_BUDGET_SECONDS: - raise SystemExit( - "pending page charged " - f"{read_gate_stats['gate_wait_seconds_max']}s to local admission" - ) - -ledger._run_sql("DELETE FROM qbit_block_candidate_outbox;") - -try: - PsqlShareLedger( - psql_command=psql_command, - database_url=database_url, - native_client_mode="1", - writer_id="writer-other", - writer_epoch=1, - ) -except RuntimeError as exc: - if "writer-native" not in str(exc): - raise -else: - raise SystemExit("second writer stole an unexpired lease over the native client") - -# Issue #236: the pooled client consumes the payout-window reads one JSON -# object per row through fetchmany, in bounded batches, from one buffered -# SELECT. A 3000-row fixture crosses several 512-row batch boundaries; the -# pre-#236 json_agg statement over the same CTE is the oracle for every -# variant, exactly as on the psql backend. -ledger._run_script( - """ -INSERT INTO qbit_share_ledger ( - share_id, miner_id, payout_order_key, p2mr_program, - share_difficulty, network_difficulty, template_height, job_id, - job_issued_at, ntime, accepted_at, accepted, writer_id, writer_epoch -) -SELECT - 'bulk-' || g, - 'miner-' || (g % 7), - lpad((g % 7)::text, 4, '0'), - decode(md5(g::text) || md5((g + 7)::text), 'hex'), - 1 + (g % 3), - 1000, - 10, - 'job-bulk', - to_timestamp(1700001000) + (g * interval '1 millisecond'), - 1700001000, - to_timestamp(1700001000) + (g * interval '1 millisecond'), - TRUE, - 'writer-native', - 1 -FROM generate_series(1, 3000) AS g; -ANALYZE qbit_share_ledger; -""" -) -bulk_anchor_ms = 1_700_001_010_000 -assert_equal(ledger.execution_backend, "psycopg-pool", "oracle check runs on the pooled client") -for weight in (1, 512, 1023, 1024, 1025, 3000, 6000, 10**9): - assert_rows_match_aggregate( - ledger, - lambda: ledger.snapshot_at_job_issue(bulk_anchor_ms, window_weight=weight), - label=f"native per-row bounded snapshot at window weight {weight}", - ) -full_history = assert_rows_match_aggregate( - ledger, - lambda: ledger.snapshot_at_job_issue(bulk_anchor_ms), - label="native per-row unbounded snapshot", - expected_len=share_count + 3000, -) -assert_rows_match_aggregate( - ledger, - lambda: ledger.snapshot_at_job_issue(0, window_weight=64), - label="native per-row empty window", - expected_len=0, -) -assert_rows_match_aggregate( - ledger, - lambda: ledger.snapshot_between_job_issues(1_700_001_001_000, 1_700_001_002_000), - label="native per-row delta inside the bulk fixture", - expected_len=1000, -) -with ledger.operation_timeout(30.0): - assert_rows_match_aggregate( - ledger, - lambda: ledger.snapshot_at_job_issue(bulk_anchor_ms, window_weight=2048), - label="native per-row bounded snapshot under an armed deadline", - ) - -# One SELECT, one MVCC snapshot: a share appended while the rows are still -# being converted (between the first and second 512-row batches) is not in -# any part of that snapshot, and is in the next one. The between-batch hook -# is the ledger's own; a second pooled connection lands the append. -late_share = pending(share_count + 5000, share_id="appended-mid-decode") -late_share = PendingShare( - **{ - **late_share.__dict__, - "job_issued_at_ms": 1_700_001_005_000, - "accepted_at_ms": 1_700_001_005_000, - } -) -appended_during: list[object] = [] -original_hook = ledger._note_json_row_batch - - -def append_on_first_batch() -> None: - if not appended_during: - appended_during.append(ledger.append(late_share)) - original_hook() - - -ledger._note_json_row_batch = append_on_first_batch # type: ignore[method-assign] -try: - during = ledger.snapshot_at_job_issue(bulk_anchor_ms) -finally: - del ledger._note_json_row_batch -assert_equal(len(appended_during), 1, "the append landed during row conversion") -assert_equal( - [record.share_id for record in during], - [record.share_id for record in full_history], - "a share appended mid-decode is absent from the whole in-flight snapshot", -) -after = ledger.snapshot_at_job_issue(bulk_anchor_ms) -assert_equal(len(after), len(full_history) + 1, "the next snapshot includes the appended share") -assert_equal(after[-1].share_id, "appended-mid-decode", "the appended share is the newest row") -assert_equal( - ledger.ledger_read_gate_stats()["payout_window_snapshot"]["execute_timeouts_total"], - 0, - "no window read timed out", -) -print("prism postgres native ledger: OK window-rows=oracle mvcc=single-select", flush=True) - -# Restore the 32-share state the fallback and read-only checks below assert -# on: the bulk fixture and the mid-decode append were this section's only -# writes, and their sequence numbers sit above the contiguous 1..32 range. -ledger._run_script( - """ -DELETE FROM qbit_share_ledger -WHERE share_id LIKE 'bulk-%' OR share_id = 'appended-mid-decode'; -ANALYZE qbit_share_ledger; -""" -) -assert_equal( - len(ledger.snapshot_at_job_issue(bulk_anchor_ms)), - share_count, - "window fixture rows removed again", -) - -released = ledger.release_writer_lease() -assert_equal(released, True, "writer lease released") -ledger.close() - -# The psql subprocess fallback must see exactly the rows the native client -# committed (same schema, same data, interchangeable backends). -fallback = PsqlShareLedger( - psql_command=psql_command, - native_client_mode="0", - writer_id="writer-native", - writer_epoch=1, -) -assert_equal(fallback.execution_backend, "psql-subprocess", "fallback backend selected") -fallback_shares = fallback.all_shares() -assert_equal(len(fallback_shares), share_count, "fallback sees committed shares") -assert_equal( - sorted(share.share_seq for share in fallback_shares), - list(range(1, share_count + 1)), - "fallback sees the same canonical sequence", -) -fallback.release_writer_lease() -fallback.close() - -# A read-only ledger against this writable primary. The database has accepted -# every write above, which is what makes it the right target: a standby would -# refuse these writes whoever asked, so it could never show that read_only=True -# is what refused them. -read_only_ledger = PsqlShareLedger( - psql_command=psql_command_with_env, - database_url=database_url, - native_client_mode="1", - writer_id="public-read", - writer_epoch=1, - read_only=True, -) -assert_equal( - read_only_ledger.execution_backend, - "psycopg-pool", - "read-only ledger uses the pooled native client", -) - -# One instance, two kinds of connection. The pooled psycopg session runs the -# writer-lease upsert -- production's own gate-free write, reached here -# directly because a read-only ledger deliberately never runs it at startup. -try: - read_only_ledger._try_acquire_writer_lease() -except Exception as exc: - if "read-only transaction" not in str(exc): - raise -else: - raise SystemExit("read-only pooled session wrote to a writable primary") - -# ...and the one-shot psql connection runs the lease release, which takes -# neither the writer gate nor the pool, so nothing in-process can refuse it. -try: - read_only_ledger.release_writer_lease_fresh_connection() -except RuntimeError as exc: - if "read-only transaction" not in str(exc): - raise -else: - raise SystemExit("read-only psql connection wrote to a writable primary") - -# Reads still work: the session is read-only, not unusable. A read slot is the -# gate every public route takes; the O(n) reads take the writer gate and are -# refused in-process, which is the pre-existing half of the promise. -assert_equal( - read_only_ledger.accepted_share_stats()["accepted_share_count"], - share_count, - "read-only ledger still serves reads over the pool", -) -read_only_ledger.close() - -# The same two statements, the same identity, differing only in read_only. -control_writer = PsqlShareLedger( - psql_command=psql_command_with_env, - database_url=database_url, - native_client_mode="1", - writer_id="public-read", - writer_epoch=1, -) -assert_equal( - control_writer._try_acquire_writer_lease()["acquired"], - True, - "writable pooled session still acquires the lease", -) -assert_equal( - control_writer.release_writer_lease_fresh_connection(), - True, - "writable psql connection still releases the lease", -) -control_writer.close() - -print("prism postgres native ledger: OK read-only-session") -PY -) - -( - cd "${ROOT_DIR}" - PRISM_RECOVERY_TEST_DATABASE_URL="${DATABASE_URL}" \ - python3 -m unittest tests.test_prism_pending_block_recovery.NativeRecoveryTests -v - PRISM_TEST_DATABASE_URL="${DATABASE_URL}" \ - PRISM_TEST_PSQL_COMMAND_WITH_ENV="docker exec -i -e PGOPTIONS ${POSTGRES_CONTAINER} psql -U qbit -d qbit" \ - python3 -m unittest tests.test_prism_candidate_window tests.test_prism_statement_spool -v -) - -echo "test-prism-postgres-native-ledger: PASS" diff --git a/test/test-prism-postgres-scale.sh b/test/test-prism-postgres-scale.sh deleted file mode 100755 index 8ac089af6..000000000 --- a/test/test-prism-postgres-scale.sh +++ /dev/null @@ -1,500 +0,0 @@ -#!/usr/bin/env bash -set -euo pipefail - -# Named, reusable production-scale PRISM ledger fixture (issue #188 fix 6). -# -# Every latent O(history) charge in the 2026-08-14 landing livelock was -# invisible on the empty databases the ordinary tests run against. This -# harness seeds a scaled qbit_payout_carry_forward / qbit_share_ledger -# fixture with realistic miner skew, then asserts plan shape and per-query -# budgets for the hot query classes, timeout cancellation cleanliness, and -# the durable outbox format contract. Budgets are enforced on server-side -# planning+execution time (EXPLAIN ANALYZE) so per-call psql / docker-exec -# transport overhead on shared CI runners cannot flake the gate; wall-clock -# figures are printed alongside for the record. -# -# Scale knobs (defaults keep CI fast; raise them for production-shaped runs): -# QBIT_PRISM_SCALE_CARRY_ROWS carry-forward rows (default 120000; prod ~5000000) -# QBIT_PRISM_SCALE_SHARE_ROWS accepted share rows (default 200000; prod ~50000000) -# QBIT_PRISM_SCALE_MINERS distinct miners (default 13) -# QBIT_PRISM_SCALE_BLOCKS confirmed pool blocks (default 40) -# QBIT_PRISM_SCALE_PRIOR_BALANCES_BUDGET_MS warm budget (default 50) -# QBIT_PRISM_SCALE_OUTBOX_POLL_BUDGET_MS warm budget (default 50) -# QBIT_PRISM_SCALE_WINDOW_READ_BUDGET_MS warm budget (default 2000) - -ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" -# shellcheck source=test/prism-postgres-lib.sh -source "${ROOT_DIR}/test/prism-postgres-lib.sh" - -POSTGRES_IMAGE="${QBIT_PRISM_POSTGRES_IMAGE:-postgres:16-alpine}" -POSTGRES_CONTAINER="${QBIT_PRISM_POSTGRES_CONTAINER:-qbit-prism-scale-pg-$$}" - -require_executable() { - command -v "$1" >/dev/null 2>&1 || { - echo "missing required executable: $1" >&2 - exit 1 - } -} - -EXTERNAL_PSQL="${QBIT_PRISM_EXTERNAL_PSQL_COMMAND:-}" - -cleanup() { - if [[ -z "${EXTERNAL_PSQL}" ]]; then - docker rm -f "${POSTGRES_CONTAINER}" >/dev/null 2>&1 || true - fi -} -trap cleanup EXIT - -require_executable python3 - -if [[ -n "${EXTERNAL_PSQL}" ]]; then - PSQL_COMMAND="${EXTERNAL_PSQL}" - deadline=$((SECONDS + 60)) - until echo 'SELECT 1;' | ${PSQL_COMMAND} >/dev/null 2>&1; do - if [[ "${SECONDS}" -ge "${deadline}" ]]; then - echo "timed out waiting for external PRISM Postgres" >&2 - exit 1 - fi - sleep 1 - done -else - require_executable docker - docker rm -f "${POSTGRES_CONTAINER}" >/dev/null 2>&1 || true - docker run \ - --rm \ - --detach \ - --name "${POSTGRES_CONTAINER}" \ - -e POSTGRES_USER=qbit \ - -e POSTGRES_PASSWORD=qbit \ - -e POSTGRES_DB=qbit \ - "${POSTGRES_IMAGE}" >/dev/null - - wait_for_prism_postgres_container "${POSTGRES_CONTAINER}" - PSQL_COMMAND="docker exec -i ${POSTGRES_CONTAINER} psql -U qbit -d qbit" -fi - -( - cd "${ROOT_DIR}" - PRISM_PSQL_COMMAND="${PSQL_COMMAND}" \ - python3 <<'PY' -from __future__ import annotations - -import json -import os -import statistics -import time - -from lab.prism.share_ledger import LedgerOperationTimeout, PsqlShareLedger - -CARRY_ROWS = int(os.environ.get("QBIT_PRISM_SCALE_CARRY_ROWS", "120000")) -SHARE_ROWS = int(os.environ.get("QBIT_PRISM_SCALE_SHARE_ROWS", "200000")) -MINERS = max(3, int(os.environ.get("QBIT_PRISM_SCALE_MINERS", "13"))) -BLOCKS = max(2, int(os.environ.get("QBIT_PRISM_SCALE_BLOCKS", "40"))) -PRIOR_BUDGET_MS = float( - os.environ.get("QBIT_PRISM_SCALE_PRIOR_BALANCES_BUDGET_MS", "50") -) -OUTBOX_BUDGET_MS = float( - os.environ.get("QBIT_PRISM_SCALE_OUTBOX_POLL_BUDGET_MS", "50") -) -WINDOW_BUDGET_MS = float( - os.environ.get("QBIT_PRISM_SCALE_WINDOW_READ_BUDGET_MS", "2000") -) - - -def assert_true(condition: object, message: str) -> None: - if not condition: - raise SystemExit(f"scale fixture assertion failed: {message}") - - -def server_execution_ms(sql: str) -> float: - """Planning + execution time measured inside PostgreSQL. - - Wall-clock timing through the harness includes per-call psql (and, in - CI, docker exec) transport overhead of tens of milliseconds, which - would dominate the budgets on shared runners. Budgets gate the query - cost itself, so they are enforced on the server-side measurement; - wall-clock values are still reported for the record. - """ - plan = json.loads(ledger._run_sql("EXPLAIN (ANALYZE, FORMAT JSON) " + sql))[0] - return float(plan.get("Planning Time", 0.0)) + float(plan["Execution Time"]) - - -PRIOR_BALANCES_SQL = ( - "SELECT COALESCE(json_agg(json_build_object(" - "'m', miner_id, 'b', balance_sats::text)), '[]'::json) " - "FROM qbit_current_carry_forward_balances();" -) - - -ledger = PsqlShareLedger( - psql_command=os.environ["PRISM_PSQL_COMMAND"], - writer_id="scale-writer", - writer_epoch=1, - initialize_schema=True, -) - -print( - f"scale fixture: seeding carry={CARRY_ROWS} shares={SHARE_ROWS} " - f"miners={MINERS} blocks={BLOCKS}", - flush=True, -) -seed_started = time.monotonic() - -# Bulk seed with the summary trigger disabled, then rebuild through the ops -# resync hook -- exactly the documented migration/repair path. -# -# Miner skew: the three heaviest miners take ~60% of rows; the remainder is -# spread across the tail, approximating the production distribution. -ledger._run_sql( - f""" -ALTER TABLE qbit_payout_carry_forward DISABLE TRIGGER qbit_payout_carry_forward_current_sync; - -INSERT INTO qbit_pool_blocks ( - block_hash, block_height, parent_hash, coinbase_txid, - payout_manifest_sha256, chain_state, audit_publication_sequence -) -SELECT - 'scale-block-' || lpad(gs::text, 8, '0'), - gs, - 'scale-block-' || lpad((gs - 1)::text, 8, '0'), - md5('coinbase' || gs::text), - md5('manifest' || gs::text), - 'confirmed', - pg_catalog.nextval('qbit_audit_publication_sequence_seq') -FROM generate_series(1, {BLOCKS}) AS gs -ON CONFLICT (block_hash) DO NOTHING; - -INSERT INTO qbit_payout_carry_forward ( - block_height, block_hash, miner_id, payout_order_key, p2mr_program, - gross_amount_sats, prior_balance_sats, candidate_balance_sats, - onchain_amount_sats, carry_forward_balance_sats, action, maturity_state -) -SELECT - 1 + mod(gs, {BLOCKS}), - 'scale-block-' || lpad((1 + mod(gs, {BLOCKS}))::text, 8, '0'), - 'scale-miner-' || lpad(miner_idx::text, 4, '0'), - lpad(miner_idx::text, 4, '0'), - decode(lpad(to_hex(miner_idx + 1), 64, '0'), 'hex'), - 1000 + mod(gs, 50), - 0, - 1000 + mod(gs, 50), - CASE WHEN mod(gs, 7) = 0 THEN 1000 + mod(gs, 50) ELSE 0 END, - CASE WHEN mod(gs, 7) = 0 THEN 0 ELSE 1000 + mod(gs, 50) END, - CASE WHEN mod(gs, 7) = 0 THEN 'onchain' ELSE 'accrued' END, - 'immature' -FROM ( - SELECT - gs, - CASE - WHEN mod(gs, 10) < 3 THEN 1 - WHEN mod(gs, 10) < 5 THEN 2 - WHEN mod(gs, 10) < 6 THEN 3 - ELSE 4 + mod(gs, {MINERS} - 3) - END AS miner_idx - FROM generate_series(1, {CARRY_ROWS}) AS gs -) AS seeded; - -ALTER TABLE qbit_payout_carry_forward ENABLE TRIGGER qbit_payout_carry_forward_current_sync; - -INSERT INTO qbit_share_ledger ( - share_id, miner_id, payout_order_key, p2mr_program, - share_difficulty, network_difficulty, template_height, job_id, - job_issued_at, ntime, accepted_at, accepted, writer_id, writer_epoch -) -SELECT - 'scale-share-' || lpad(gs::text, 10, '0'), - 'scale-miner-' || lpad(miner_idx::text, 4, '0'), - lpad(miner_idx::text, 4, '0'), - decode(lpad(to_hex(miner_idx + 1), 64, '0'), 'hex'), - 64 + mod(gs, 512), - 100000, - 1 + mod(gs, {BLOCKS}), - 'scale-job-' || mod(gs, 1024)::text, - now() - make_interval(secs => ({SHARE_ROWS} - gs) * 0.25), - 1700000000 + gs, - now() - make_interval(secs => ({SHARE_ROWS} - gs) * 0.25), - true, - 'scale-writer', - 1 -FROM ( - SELECT - gs, - CASE - WHEN mod(gs, 10) < 3 THEN 1 - WHEN mod(gs, 10) < 5 THEN 2 - WHEN mod(gs, 10) < 6 THEN 3 - ELSE 4 + mod(gs, {MINERS} - 3) - END AS miner_idx - FROM generate_series(1, {SHARE_ROWS}) AS gs -) AS seeded; - -SELECT qbit_rebuild_carry_forward_current_balances(); -ANALYZE qbit_payout_carry_forward; -ANALYZE qbit_payout_carry_forward_current; -ANALYZE qbit_share_ledger; -ANALYZE qbit_pool_blocks; -""" -) -print( - f"scale fixture: seeded in {time.monotonic() - seed_started:.1f}s", - flush=True, -) - - -def timed_json(sql: str) -> tuple[float, object]: - started = time.monotonic() - result = ledger._run_json(sql) - return (time.monotonic() - started) * 1000.0, result - - -# --- Prior balances: budget + equivalence ----------------------------------- - -cold_ms, cold_balances = timed_json(PRIOR_BALANCES_SQL) -cold_server_ms = server_execution_ms(PRIOR_BALANCES_SQL) -warm_wall_samples = [] -warm_server_samples = [] -for _ in range(5): - warm_ms, _ = timed_json(PRIOR_BALANCES_SQL) - warm_wall_samples.append(warm_ms) - warm_server_samples.append(server_execution_ms(PRIOR_BALANCES_SQL)) -warm_wall_median = statistics.median(warm_wall_samples) -warm_server_median = statistics.median(warm_server_samples) -print( - f"scale fixture: prior balances cold wall={cold_ms:.1f}ms " - f"server={cold_server_ms:.1f}ms; warm medians " - f"wall={warm_wall_median:.1f}ms server={warm_server_median:.1f}ms " - f"budget={PRIOR_BUDGET_MS:.0f}ms", - flush=True, -) -assert_true(len(cold_balances) >= 3, "prior balances returned miner rows") -assert_true( - warm_server_median <= PRIOR_BUDGET_MS, - f"warm prior-balances server time {warm_server_median:.1f}ms exceeds " - f"{PRIOR_BUDGET_MS:.0f}ms budget", -) - -recompute_started = time.monotonic() -drift = ledger._run_json( - "SELECT json_build_object(" - "'drift_count', (SELECT count(*) FROM qbit_carry_forward_current_drift()));" -) -recompute_ms = (time.monotonic() - recompute_started) * 1000.0 -assert_true( - int(drift["drift_count"]) == 0, - "summary balances drift from recompute at scale", -) -print( - f"scale fixture: recompute cross-check {recompute_ms:.1f}ms drift=0", - flush=True, -) - -# --- Prior balances: plan shape ---------------------------------------------- -# Inline copy of the qbit_current_carry_forward_balances() body; the plan -# must never touch the O(history) qbit_payout_carry_forward table. - -plan_text = ledger._run_sql( - """ -EXPLAIN (FORMAT TEXT) -WITH balances AS ( - SELECT - (array_agg(current_balance.miner_id ORDER BY current_balance.payout_order_key, current_balance.miner_id))[1] AS miner_id, - (array_agg(current_balance.payout_order_key ORDER BY current_balance.payout_order_key, current_balance.miner_id))[1] AS payout_order_key, - current_balance.p2mr_program, - SUM(current_balance.balance_sats) AS balance_sats - FROM qbit_payout_carry_forward_current current_balance - WHERE current_balance.active_row_count > 0 - GROUP BY current_balance.p2mr_program - HAVING SUM(current_balance.balance_sats) <> 0 -) -SELECT balances.miner_id, balances.payout_order_key, balances.p2mr_program, balances.balance_sats -FROM balances -ORDER BY balances.payout_order_key, balances.miner_id, balances.p2mr_program; -""" -) -assert_true( - "qbit_payout_carry_forward " not in plan_text - and "on qbit_payout_carry_forward\n" not in plan_text - and not plan_text.rstrip().endswith("on qbit_payout_carry_forward"), - f"prior-balances plan touches the O(history) table:\n{plan_text}", -) -assert_true( - "qbit_payout_carry_forward_current" in plan_text, - f"prior-balances plan does not read the summary table:\n{plan_text}", -) - -# --- Mid-history reversal equivalence under trigger maintenance -------------- -# Reverse one seeded block the same way qbit_reverse_immature_pool_block -# does (block first, then carry rows) and require the trigger-maintained -# summary to keep matching the recompute. - -ledger._run_sql( - """ -UPDATE qbit_pool_blocks -SET chain_state = 'reversed', maturity_state = 'reversed', - disconnected_at = clock_timestamp() -WHERE block_hash = 'scale-block-00000002'; - -UPDATE qbit_payout_carry_forward -SET maturity_state = 'reversed' -WHERE block_hash = 'scale-block-00000002' - AND maturity_state = 'immature'; -""" -) -post_reversal_server_ms = statistics.median( - server_execution_ms(PRIOR_BALANCES_SQL) for _ in range(3) -) -drift = ledger._run_json( - "SELECT json_build_object(" - "'drift_count', (SELECT count(*) FROM qbit_carry_forward_current_drift()));" -) -assert_true( - int(drift["drift_count"]) == 0, - "summary drifts from recompute after mid-history reversal at scale", -) -assert_true( - post_reversal_server_ms <= PRIOR_BUDGET_MS, - f"post-reversal prior-balances server time {post_reversal_server_ms:.1f}ms " - f"exceeds {PRIOR_BUDGET_MS:.0f}ms budget", -) -print( - f"scale fixture: mid-history reversal kept drift=0 " - f"server={post_reversal_server_ms:.1f}ms", - flush=True, -) - -# --- Outbox poll: plan shape + budget --------------------------------------- - -intent = { - "schema": "qbit.prism.block-candidate-intent.v1", - "block_hash_hex": "ab" * 32, - "block_hex": "00", -} -assert_true( - ledger.persist_block_candidate_intent(intent), - "outbox intent persist", -) -poll_started = time.monotonic() -pending_rows = ledger.pending_block_candidate_rows() -poll_wall_ms = (time.monotonic() - poll_started) * 1000.0 -poll_server_ms = server_execution_ms( - "SELECT block_hash, candidate FROM qbit_block_candidate_outbox " - "WHERE state = 'pending' ORDER BY created_at, block_hash LIMIT 32;" -) -print( - f"scale fixture: outbox poll wall={poll_wall_ms:.1f}ms " - f"server={poll_server_ms:.1f}ms budget={OUTBOX_BUDGET_MS:.0f}ms", - flush=True, -) -assert_true( - poll_server_ms <= OUTBOX_BUDGET_MS, - f"outbox poll server time {poll_server_ms:.1f}ms exceeds " - f"{OUTBOX_BUDGET_MS:.0f}ms budget", -) -outbox_plan_text = ledger._run_sql( - """ -EXPLAIN (FORMAT TEXT) -SELECT block_hash, candidate -FROM qbit_block_candidate_outbox -WHERE state = 'pending' -ORDER BY created_at, block_hash -LIMIT 32; -""" -) -assert_true( - "qbit_block_candidate_outbox_pending_idx" in outbox_plan_text, - f"outbox poll does not use the pending partial index:\n{outbox_plan_text}", -) - -# --- Durable outbox format contract (replay compatibility) ------------------- -# A rollback replays rows written by the newer build; the payload must keep -# its schema-versioned shape and authoritative key so the previous release -# can digest it. - -assert_true(len(pending_rows) == 1, "one pending outbox row") -durable_row = pending_rows[0] -assert_true( - durable_row["block_hash"] == "ab" * 32, - "outbox row keeps the authoritative block-hash key", -) -candidate_payload = durable_row["candidate"] -assert_true( - candidate_payload.get("schema") == "qbit.prism.block-candidate-intent.v1", - "outbox candidate payload keeps the v1 intent schema tag", -) -assert_true( - candidate_payload.get("block_hash_hex") == "ab" * 32 - and "block_hex" in candidate_payload, - "outbox candidate payload keeps replay-critical fields", -) -assert_true( - ledger.mark_block_candidate_submitted(block_hash="ab" * 32), - "outbox terminal update", -) - -# --- Window read budget ------------------------------------------------------ - -anchor = ledger._run_json( - "SELECT json_build_object('anchor', to_char(max(job_issued_at), " - "'YYYY-MM-DD\"T\"HH24:MI:SS.US\"Z\"')) FROM qbit_share_ledger;" -)["anchor"] -window_sql = ( - "SELECT json_build_object('rows', count(*)) FROM qbit_audit_share_window(" - f"'{anchor}'::timestamptz, 100000::numeric);" -) -window_started = time.monotonic() -window_rows = ledger._run_json(window_sql) -window_wall_ms = (time.monotonic() - window_started) * 1000.0 -window_server_ms = server_execution_ms(window_sql) -assert_true(int(window_rows["rows"]) > 0, "window read returned shares") -assert_true( - window_server_ms <= WINDOW_BUDGET_MS, - f"window read server time {window_server_ms:.1f}ms exceeds " - f"{WINDOW_BUDGET_MS:.0f}ms budget", -) -print( - f"scale fixture: window read wall={window_wall_ms:.1f}ms " - f"server={window_server_ms:.1f}ms rows={window_rows['rows']}", - flush=True, -) - -# --- Timeout cancellation leaves no invisible query -------------------------- - -marker = "scale-cancellation-marker" -try: - with ledger.operation_timeout(0.2): - ledger._run_json( - f"SELECT json_build_object('{marker}', pg_sleep(5));" - ) - raise SystemExit("padded statement unexpectedly completed under 0.2s budget") -except LedgerOperationTimeout: - pass -deadline = time.monotonic() + 5.0 -lingering = None -while time.monotonic() < deadline: - lingering = ledger._run_json( - "SELECT json_build_object('active', count(*)) FROM pg_stat_activity " - f"WHERE state = 'active' AND query LIKE '%{marker}%' " - "AND query NOT LIKE '%pg_stat_activity%';" - ) - if int(lingering["active"]) == 0: - break - time.sleep(0.1) -assert_true( - lingering is not None and int(lingering["active"]) == 0, - f"cancelled statement still active on the server: {lingering}", -) -follow_up = ledger._run_json("SELECT json_build_object('ok', 1);") -assert_true(int(follow_up["ok"]) == 1, "ledger unusable after cancellation") -print("scale fixture: timeout cancellation left no active query", flush=True) - -ledger.close() -print( - "prism postgres scale PASS " - f"carry={CARRY_ROWS} shares={SHARE_ROWS} miners={MINERS} " - f"prior_cold_server_ms={cold_server_ms:.1f} " - f"prior_warm_server_ms={warm_server_median:.1f} " - f"window_server_ms={window_server_ms:.1f}", - flush=True, -) -PY -) diff --git a/test/test-prism-postgres-seed-guard.sh b/test/test-prism-postgres-seed-guard.sh deleted file mode 100755 index 3911b812b..000000000 --- a/test/test-prism-postgres-seed-guard.sh +++ /dev/null @@ -1,606 +0,0 @@ -#!/usr/bin/env bash -# Regression test for the carry-forward summary seed guard (#124). -# -# Reproduces the poisoning mechanism against a real Postgres: a per-statement -# autocommit schema apply commits the carry-forward summary sync triggers -# before the seeding block at the end of the file runs, a still-running writer -# confirms a block in that gap, and the summary ends up holding only the -# post-trigger delta. The old emptiness-based guard then treats that partial -# summary as already seeded and locks the damage in permanently. The hardened -# guard compares the summary against the carry history it summarizes and -# repairs it on the next apply, on both the psql-subprocess and the native -# psycopg apply paths. A second scenario proves the other half of the fix: -# a failing schema apply commits nothing on either path (the script wraps -# itself in one BEGIN/COMMIT transaction, and the psql backend additionally -# passes --single-transaction). -# -# Requires docker and a host python3 with psycopg installed -# (python3 -m pip install 'psycopg[binary]'). -set -euo pipefail - -ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" -POSTGRES_IMAGE="${QBIT_PRISM_POSTGRES_IMAGE:-postgres:16-alpine}" -POSTGRES_CONTAINER="${QBIT_PRISM_POSTGRES_CONTAINER:-qbit-prism-seed-guard-pg-$$}" - -require_executable() { - command -v "$1" >/dev/null 2>&1 || { - echo "missing required executable: $1" >&2 - exit 1 - } -} - -cleanup() { - docker rm -f "${POSTGRES_CONTAINER}" >/dev/null 2>&1 || true -} -trap cleanup EXIT - -require_executable docker -require_executable python3 - -# Both apply paths are exercised, so the native psycopg client is mandatory -# here rather than opportunistic. -python3 -c 'import psycopg' 2>/dev/null || { - echo "host python3 lacks psycopg; install with: python3 -m pip install 'psycopg[binary]'" >&2 - exit 1 -} - -docker rm -f "${POSTGRES_CONTAINER}" >/dev/null 2>&1 || true -docker run \ - --rm \ - --detach \ - --name "${POSTGRES_CONTAINER}" \ - -p 127.0.0.1:0:5432 \ - -e POSTGRES_USER=qbit \ - -e POSTGRES_PASSWORD=qbit \ - -e POSTGRES_DB=qbit \ - "${POSTGRES_IMAGE}" >/dev/null - -# The official image runs initdb against a temporary server that binds no TCP -# address, so a socket-only `pg_isready -U qbit -d qbit` reports ready during -# bootstrap and races the restart into the durable server. Gate on the TCP -# listener instead, and confirm with a real query rather than trusting the -# liveness probe alone. -deadline=$((SECONDS + 60)) -until docker exec "${POSTGRES_CONTAINER}" \ - pg_isready -h 127.0.0.1 -U qbit -d qbit >/dev/null 2>&1 \ - && echo 'SELECT 1;' | docker exec -i "${POSTGRES_CONTAINER}" \ - psql --no-psqlrc --set ON_ERROR_STOP=1 -U qbit -d qbit >/dev/null 2>&1; do - if [[ "${SECONDS}" -ge "${deadline}" ]]; then - echo "timed out waiting for PRISM Postgres container" >&2 - docker logs "${POSTGRES_CONTAINER}" >&2 || true - exit 1 - fi - sleep 1 -done - -HOST_PORT="$(docker port "${POSTGRES_CONTAINER}" 5432/tcp | head -n 1 | awk -F: '{print $NF}')" -if [[ -z "${HOST_PORT}" ]]; then - echo "unable to resolve published Postgres port" >&2 - exit 1 -fi -DATABASE_URL="postgresql://qbit:qbit@127.0.0.1:${HOST_PORT}/qbit" - -# The gate above ran inside the container; the native apply path connects over -# the published port from the host, so prove that reaches the same server. -deadline=$((SECONDS + 60)) -until python3 - "$DATABASE_URL" <<'PY' >/dev/null 2>&1 -import sys -import psycopg - -with psycopg.connect(sys.argv[1], connect_timeout=3) as conn: - conn.execute("SELECT 1") -PY -do - if [[ "${SECONDS}" -ge "${deadline}" ]]; then - echo "timed out waiting for published PRISM Postgres port" >&2 - docker logs "${POSTGRES_CONTAINER}" >&2 || true - exit 1 - fi - sleep 1 -done - -( - cd "${ROOT_DIR}" - PRISM_TEST_ROOT_DIR="${ROOT_DIR}" \ - PRISM_TEST_POSTGRES_CONTAINER="${POSTGRES_CONTAINER}" \ - PRISM_TEST_POSTGRES_PORT="${HOST_PORT}" \ - python3 <<'PY' -from __future__ import annotations - -import json -import os -import subprocess -import sys -import tempfile -from pathlib import Path - -ROOT_DIR = os.environ["PRISM_TEST_ROOT_DIR"] -sys.path.insert(0, ROOT_DIR) - -from lab.prism.share_ledger import PsqlShareLedger - -CONTAINER = os.environ["PRISM_TEST_POSTGRES_CONTAINER"] -HOST_PORT = os.environ["PRISM_TEST_POSTGRES_PORT"] -SCHEMA_PATH = Path(ROOT_DIR) / "crates" / "qbit-prism" / "sql" / "001_share_ledger.sql" - -# The hardened guard's first comment line. Splitting the schema here yields a -# head that installs every table, function and carry-forward summary sync -# trigger but no seeding block at all -- exactly the state a non-atomic apply -# leaves behind when it is interrupted between the triggers and the seed. -SEED_GUARD_MARKER = ( - "-- Seed or repair the summary from carry history whenever the summary" -) - -# The pre-#124 guard, verbatim. It seeds only when the summary is completely -# empty, so a partially populated summary reads as "already seeded". -OLD_SEED_GUARD = """ -DO $$ -BEGIN - IF NOT EXISTS (SELECT 1 FROM qbit_payout_carry_forward_current) - AND EXISTS ( - SELECT 1 - FROM qbit_payout_carry_forward carry - JOIN qbit_pool_blocks block - ON block.block_hash = carry.block_hash - WHERE carry.maturity_state <> 'reversed' - AND block.chain_state = 'confirmed' - AND block.maturity_state <> 'reversed' - ) - THEN - PERFORM qbit_rebuild_carry_forward_current_balances(); - END IF; -END -$$; -""" - -CONFIRMED_BLOCK_HASH = "aa" * 32 -PREPARED_BLOCK_HASH = "ee" * 32 - - -def assert_equal(actual: object, expected: object, message: str) -> None: - if actual != expected: - raise SystemExit(f"{message}: expected {expected!r}, got {actual!r}") - - -def psql_command(database: str) -> str: - return f"docker exec -i {CONTAINER} psql -U qbit -d {database}" - - -def database_url(database: str) -> str: - return f"postgresql://qbit:qbit@127.0.0.1:{HOST_PORT}/{database}" - - -def _psql(sql: str, *, database: str, extra_args: list[str]) -> str: - completed = subprocess.run( - [ - "docker", - "exec", - "-i", - CONTAINER, - "psql", - "--no-psqlrc", - "--set", - "ON_ERROR_STOP=1", - *extra_args, - "-U", - "qbit", - "-d", - database, - ], - input=sql, - text=True, - stdout=subprocess.PIPE, - stderr=subprocess.PIPE, - check=False, - ) - if completed.returncode != 0: - raise SystemExit( - f"psql on {database} failed (exit {completed.returncode}): " - f"{completed.stderr.strip()}" - ) - return completed.stdout - - -def psql_autocommit_apply(sql: str, *, database: str = "qbit") -> None: - """Apply SQL with no --single-transaction: one commit per statement. - - This is the operator error the old ops doc allowed, and the shape that - poisons the summary seed. - """ - _psql(sql, database=database, extra_args=[]) - - -def psql_json(sql: str, *, database: str = "qbit") -> object: - output = _psql( - sql, - database=database, - extra_args=["--tuples-only", "--no-align", "--quiet"], - ).strip() - if not output: - raise SystemExit(f"psql query on {database} returned no JSON") - return json.loads(output.splitlines()[-1]) - - -def summary_rows(*, database: str = "qbit") -> object: - return psql_json( - """ -SELECT COALESCE(json_agg(json_build_object( - 'miner_id', miner_id, - 'balance_sats', balance_sats::text, - 'active_row_count', active_row_count -) ORDER BY miner_id), '[]'::json) -FROM qbit_payout_carry_forward_current; -""", - database=database, - ) - - -def reported_balances(*, database: str = "qbit") -> object: - return psql_json( - """ -SELECT COALESCE(json_agg(json_build_object( - 'miner_id', miner_id, - 'balance_sats', balance_sats::text -) ORDER BY miner_id), '[]'::json) -FROM qbit_current_carry_forward_balances(); -""", - database=database, - ) - - -POISONED_SUMMARY = [ - {"miner_id": "miner-b", "balance_sats": "500", "active_row_count": 1} -] -REPAIRED_SUMMARY = [ - {"miner_id": "miner-a", "balance_sats": "1000", "active_row_count": 1}, - {"miner_id": "miner-b", "balance_sats": "500", "active_row_count": 1}, -] -REPAIRED_BALANCES = [ - {"miner_id": "miner-a", "balance_sats": "1000"}, - {"miner_id": "miner-b", "balance_sats": "500"}, -] - - -def assert_repaired(ledger: PsqlShareLedger, label: str) -> None: - assert_equal(summary_rows(), REPAIRED_SUMMARY, f"{label}: summary rebuilt from history") - assert_equal( - reported_balances(), - REPAIRED_BALANCES, - f"{label}: reported balances no longer under-report the pre-upgrade carry", - ) - assert_equal( - ledger._run_json( - """ -SELECT json_build_object( - 'drift_rows', (SELECT count(*) FROM qbit_carry_forward_current_drift()), - 'balance_rows', (SELECT count(*) FROM qbit_current_carry_forward_balances()) -); -""" - ), - {"drift_rows": 0, "balance_rows": 2}, - f"{label}: summary agrees with the recomputation it summarizes", - ) - - -# -------------------------------------------------------------------------- -# Scenario A: poison via a non-atomic apply, old guard locks it in, a new -# apply repairs it on both execution backends. -# -------------------------------------------------------------------------- - -schema_text = SCHEMA_PATH.read_text(encoding="utf-8") -schema_lines = schema_text.splitlines(keepends=True) -marker_indexes = [ - index - for index, line in enumerate(schema_lines) - if line.rstrip("\n") == SEED_GUARD_MARKER -] -assert_equal(len(marker_indexes), 1, "seed guard marker comment appears exactly once") -marker_index = marker_indexes[0] -schema_head = "".join(schema_lines[:marker_index]) -schema_tail = "".join(schema_lines[marker_index:]) -if "CREATE TRIGGER qbit_pool_blocks_carry_forward_current_sync" not in schema_head: - raise SystemExit("schema head is missing the carry-forward summary sync triggers") -if "CREATE OR REPLACE FUNCTION qbit_carry_forward_current_drift" not in schema_head: - raise SystemExit("schema head is missing the drift function the guard depends on") -if "qbit_rebuild_carry_forward_current_balances" not in schema_tail: - raise SystemExit("schema tail is missing the seed/repair guard") - -# The script wraps itself in one BEGIN/COMMIT transaction (that wrapper is -# half of the #124 fix). The interrupted-apply reproduction must strip the -# leading BEGIN so the head really does autocommit statement by statement, -# exactly like the pre-fix schema file did under a plain `psql -f`. -head_lines = schema_head.splitlines(keepends=True) -begin_indexes = [ - index for index, line in enumerate(head_lines) if line.rstrip("\n") == "BEGIN;" -] -assert_equal(len(begin_indexes), 1, "schema head contains exactly one leading BEGIN") -unwrapped_head = "".join(head_lines[begin_indexes[0] + 1 :]) - -# 1. The interrupted operator apply: every statement commits on its own and -# the run stops before the seeding block at the end of the file. -psql_autocommit_apply(unwrapped_head) - -# 2. Pre-upgrade carry history that never flowed through the summary triggers. -# session_replication_role = replica suppresses them for this session the -# way history written before they existed never reached them. It also -# suppresses the BEFORE trigger that assigns the publication ordinal, so -# the confirmed row carries one explicitly to satisfy the CHECK constraint. -psql_autocommit_apply( - """ -SET session_replication_role = replica; - -INSERT INTO qbit_pool_blocks ( - block_hash, - block_height, - parent_hash, - coinbase_txid, - payout_manifest_sha256, - chain_state, - audit_publication_sequence -) VALUES ( - '""" - + CONFIRMED_BLOCK_HASH - + """', - 100, - '""" - + "00" * 32 - + """', - '""" - + "ab" * 32 - + """', - '""" - + "ac" * 32 - + """', - 'confirmed', - nextval('qbit_audit_publication_sequence_seq') -); - -INSERT INTO qbit_pool_blocks ( - block_hash, - block_height, - parent_hash, - coinbase_txid, - payout_manifest_sha256, - chain_state -) VALUES ( - '""" - + PREPARED_BLOCK_HASH - + """', - 101, - '""" - + CONFIRMED_BLOCK_HASH - + """', - '""" - + "ef" * 32 - + """', - '""" - + "e0" * 32 - + """', - 'prepared' -); - -INSERT INTO qbit_payout_carry_forward ( - block_height, - block_hash, - miner_id, - payout_order_key, - p2mr_program, - gross_amount_sats, - prior_balance_sats, - candidate_balance_sats, - onchain_amount_sats, - carry_forward_balance_sats, - action, - maturity_state -) VALUES - ( - 100, - '""" - + CONFIRMED_BLOCK_HASH - + """', - 'miner-a', - 'a', - decode('11' || repeat('00', 31), 'hex'), - 1000, - 0, - 1000, - 0, - 1000, - 'accrued', - 'immature' - ), - ( - 101, - '""" - + PREPARED_BLOCK_HASH - + """', - 'miner-b', - 'b', - decode('22' || repeat('00', 31), 'hex'), - 500, - 0, - 500, - 0, - 500, - 'accrued', - 'immature' - ); - -SET session_replication_role = origin; -""" -) -assert_equal( - summary_rows(), - [], - "pre-upgrade carry history left the summary empty", -) - -# 3. The gap mutation: the still-running writer confirms the prepared block -# while the apply sits between the committed triggers and the seed. The -# summary receives that delta and nothing else. -psql_autocommit_apply( - "UPDATE qbit_pool_blocks SET chain_state = 'confirmed' " - "WHERE block_hash = '" + PREPARED_BLOCK_HASH + "';" -) -assert_equal( - summary_rows(), - POISONED_SUMMARY, - "the gap mutation leaves a summary holding only the post-trigger delta", -) -assert_equal( - reported_balances(), - [{"miner_id": "miner-b", "balance_sats": "500"}], - "the poisoned summary silently omits every pre-upgrade carry balance", -) - -# 4. The pre-#124 guard sees a non-empty summary and calls the poison seeded. -psql_autocommit_apply(OLD_SEED_GUARD) -assert_equal( - summary_rows(), - POISONED_SUMMARY, - "the old emptiness guard locks the poisoned summary in", -) - -# 5. Repair through the psql-subprocess apply path. -psql_ledger = PsqlShareLedger( - psql_command=psql_command("qbit"), - native_client_mode="0", - writer_id="seed-guard-psql", - writer_epoch=1, - initialize_schema=True, -) -assert_equal( - psql_ledger.execution_backend, "psql-subprocess", "psql apply path selected" -) -assert_repaired(psql_ledger, "psql apply path") -psql_ledger.release_writer_lease() -psql_ledger.close() - -# 6. Re-poison the summary the same way and repair through the native apply -# path, which wraps the script in the simple-query-protocol transaction -# rather than passing --single-transaction. -psql_autocommit_apply( - "DELETE FROM qbit_payout_carry_forward_current WHERE miner_id = 'miner-a';" -) -assert_equal( - summary_rows(), - POISONED_SUMMARY, - "re-poisoned summary before the native apply", -) -native_ledger = PsqlShareLedger( - psql_command=psql_command("qbit"), - database_url=database_url("qbit"), - native_client_mode="1", - writer_id="seed-guard-native", - writer_epoch=1, - initialize_schema=True, -) -assert_equal( - native_ledger.execution_backend, "psycopg-pool", "native apply path selected" -) -assert_repaired(native_ledger, "native apply path") -native_ledger.release_writer_lease() -native_ledger.close() - -# 7. The guard's drift-only branch: a summary whose active row-count total -# still matches the carry history but whose balances were corrupted. -# Row counts alone cannot catch this; the drift comparison must. -psql_autocommit_apply( - "UPDATE qbit_payout_carry_forward_current " - "SET balance_sats = balance_sats + 1 WHERE miner_id = 'miner-a';" -) -assert_equal( - reported_balances(), - [ - {"miner_id": "miner-a", "balance_sats": "1001"}, - {"miner_id": "miner-b", "balance_sats": "500"}, - ], - "balance-only corruption is visible in the reported balances", -) -drift_ledger = PsqlShareLedger( - psql_command=psql_command("qbit"), - native_client_mode="0", - writer_id="seed-guard-drift", - writer_epoch=1, - initialize_schema=True, -) -assert_repaired(drift_ledger, "drift-only corruption repair") -drift_ledger.release_writer_lease() -drift_ledger.close() - -# -------------------------------------------------------------------------- -# Scenario B: a failing apply commits nothing on either path. The failure is -# injected just before the script's own final COMMIT so it lands inside the -# apply's transaction: without atomicity (a pre-fix autocommit apply, or a -# client that neither honors the wrapper nor passes --single-transaction) -# every earlier statement would already be durable when the failure lands, -# which is exactly what opens the poisoning gap. -# -------------------------------------------------------------------------- - -ATOMIC_DATABASES = ( - ("qbit_psql_atomic", "0", "seed-guard-atomic-psql", False), - ("qbit_native_atomic", "1", "seed-guard-atomic-native", True), -) - -for atomic_database, _mode, _writer_id, _needs_url in ATOMIC_DATABASES: - # CREATE DATABASE cannot run inside a transaction block, so it goes - # through the maintenance command rather than the ledger. - psql_autocommit_apply(f"CREATE DATABASE {atomic_database};") - -schema_body_lines = schema_text.splitlines(keepends=True) -commit_indexes = [ - index - for index, line in enumerate(schema_body_lines) - if line.rstrip("\n") == "COMMIT;" -] -assert_equal( - len(commit_indexes), - 1, - "schema script contains exactly one top-level COMMIT", -) -failing_schema_text = ( - "".join(schema_body_lines[: commit_indexes[0]]) - + "SELECT 1/0;\n" - + "".join(schema_body_lines[commit_indexes[0] :]) -) - -with tempfile.TemporaryDirectory() as temporary_directory: - failing_schema_path = Path(temporary_directory) / "001_share_ledger_failing.sql" - failing_schema_path.write_text(failing_schema_text, encoding="utf-8") - for atomic_database, mode, writer_id, needs_url in ATOMIC_DATABASES: - failure: BaseException | None = None - try: - PsqlShareLedger( - psql_command=psql_command(atomic_database), - database_url=database_url(atomic_database) if needs_url else None, - native_client_mode=mode, - writer_id=writer_id, - writer_epoch=1, - initialize_schema=True, - schema_path=failing_schema_path, - ) - except Exception as exc: # noqa: BLE001 - the failure mode is the assertion - failure = exc - if failure is None: - raise SystemExit( - f"{atomic_database}: a failing schema apply unexpectedly succeeded" - ) - assert_equal( - psql_json( - """ -SELECT json_build_object( - 'pool_blocks', to_regclass('qbit_pool_blocks')::text, - 'carry_forward_current', to_regclass('qbit_payout_carry_forward_current')::text -); -""", - database=atomic_database, - ), - {"pool_blocks": None, "carry_forward_current": None}, - f"{atomic_database}: the failed apply committed nothing", - ) - -print("prism postgres seed guard: OK") -PY -) - -echo "test-prism-postgres-seed-guard: PASS" diff --git a/test/test-prism-postgres-throughput.sh b/test/test-prism-postgres-throughput.sh index 3d06819f5..083b12572 100644 --- a/test/test-prism-postgres-throughput.sh +++ b/test/test-prism-postgres-throughput.sh @@ -1,187 +1,123 @@ #!/usr/bin/env bash +# Full-size share-append throughput floor for the native ledger (issue #271). +# +# This runs the real production share path: every share is one `Ledger::append` +# transaction that takes the cluster-wide ORDER_LOCK advisory lock, so the rate +# it reports is the rate at which that serialized transaction retires, measured +# at 1, 2 and 4 concurrent appenders. +# +# It replaces the CPU-only audit-builder benchmark this script used to run. That +# benchmark still exists and is still worth running; it just answers a different +# question (builder CPU cost, no database): +# +# cargo run --release -p qbit-prism-server -- benchmark \ +# --shares 100000 --miners 100 --iterations 10 +# +# The numbers here are also NOT comparable with the 2.x.x +# `qbit.prism.postgres-throughput.v1` report, which timed one bulk +# `INSERT ... SELECT ... FROM generate_series`: one statement, one commit, no +# advisory lock, no concurrency. The report this script writes carries the +# schema `qbit.prism.postgres-throughput.v2` for that reason. +# +# RUN THIS AGAINST A DATABASE NOTHING ELSE IS APPENDING TO. +# ORDER_LOCK is a PostgreSQL advisory lock, and an advisory lock is scoped to +# the database, not to the per-run schema the test creates. Any other backend +# appending shares into the same database serializes against this run and +# roughly halves the rate reported here. The test does not trust the operator +# to get that right: every connection it opens carries a per-run +# `application_name`, the ORDER_LOCK sampler separates this run's waiters from +# everyone else's, and a level during which a foreign backend held or waited +# on the lock is marked `contaminated: true` in the report and in the failure +# message. A contaminated level is still measured and still gated, but its +# number describes two workloads sharing one lock and must not be carried +# forward as a property of the append path. Leaving PRISM_TEST_DATABASE_URL +# unset gives you a disposable cluster of your own, which is the safe default. +# +# A CANCELLED RUN LEAVES ITS SCHEMA BEHIND. +# The test drops its `prism_throughput_<uuid>` schema on every exit path it +# controls, but Ctrl-C kills the process before the drop runs. Adding a signal +# handler would mean a new dependency the test is not allowed to take, so the +# leftovers are swept by hand. On a server you keep, after cancelling a run: +# +# SELECT nspname FROM pg_namespace WHERE nspname LIKE 'prism\_throughput\_%'; +# DROP SCHEMA prism_throughput_0ee059ccc0314cbb8a814c0319f846d5 CASCADE; +# +# Do that only when no run is in flight; a live run's schema matches the same +# pattern. A disposable cluster (PRISM_TEST_DATABASE_URL unset) needs none of +# this: it is thrown away whole. +# +# Environment: +# PRISM_TEST_DATABASE_URL an existing PostgreSQL 16; when unset, a +# disposable cluster is started from the local +# server binaries and torn down afterwards. +# QBIT_PRISM_MIN_SHARES_PER_SEC the floor; unset uses the test's compiled-in +# constant. +# QBIT_PRISM_THROUGHPUT_* window size, shares per level, appender +# counts, lock sampling interval (1..=1000 ms), +# report path. Every one of these is inherited +# by the test process unchanged; this script +# reads none of them except to work out where +# the report should have landed. set -euo pipefail - -ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" -# shellcheck source=test/prism-postgres-lib.sh -source "${ROOT_DIR}/test/prism-postgres-lib.sh" - -POSTGRES_IMAGE="${QBIT_PRISM_POSTGRES_IMAGE:-postgres:16-alpine}" -POSTGRES_CONTAINER="${QBIT_PRISM_POSTGRES_CONTAINER:-qbit-prism-throughput-pg-$$}" -SHARE_COUNT="${QBIT_PRISM_THROUGHPUT_SHARES:-20000}" -MIN_SHARES_PER_SECOND="${QBIT_PRISM_MIN_SHARES_PER_SEC:-0}" -REPORT_PATH="${QBIT_PRISM_THROUGHPUT_REPORT:-${ROOT_DIR}/build/prism-postgres-throughput.json}" - -require_executable() { - command -v "$1" >/dev/null 2>&1 || { - echo "missing required executable: $1" >&2 - exit 1 - } -} - -cleanup() { - docker rm -f "${POSTGRES_CONTAINER}" >/dev/null 2>&1 || true -} -trap cleanup EXIT - -require_executable docker -require_executable python3 - -docker rm -f "${POSTGRES_CONTAINER}" >/dev/null 2>&1 || true -docker run \ - --rm \ - --detach \ - --name "${POSTGRES_CONTAINER}" \ - -e POSTGRES_USER=qbit \ - -e POSTGRES_PASSWORD=qbit \ - -e POSTGRES_DB=qbit \ - "${POSTGRES_IMAGE}" >/dev/null - -wait_for_prism_postgres_container "${POSTGRES_CONTAINER}" - -mkdir -p "$(dirname "${REPORT_PATH}")" - -( - cd "${ROOT_DIR}" - POSTGRES_CONTAINER="${POSTGRES_CONTAINER}" \ - SHARE_COUNT="${SHARE_COUNT}" \ - MIN_SHARES_PER_SECOND="${MIN_SHARES_PER_SECOND}" \ - REPORT_PATH="${REPORT_PATH}" \ - python3 <<'PY' -from __future__ import annotations - -import json -import os -import platform -import subprocess -import time -from pathlib import Path - - -ROOT_DIR = Path.cwd() -SCHEMA_PATH = ROOT_DIR / "crates/qbit-prism/sql/001_share_ledger.sql" -container = os.environ["POSTGRES_CONTAINER"] -share_count = int(os.environ["SHARE_COUNT"]) -min_shares_per_second = float(os.environ["MIN_SHARES_PER_SECOND"]) -report_path = Path(os.environ["REPORT_PATH"]) - -if share_count <= 0: - raise SystemExit("QBIT_PRISM_THROUGHPUT_SHARES must be positive") -if min_shares_per_second < 0: - raise SystemExit("QBIT_PRISM_MIN_SHARES_PER_SEC must be non-negative") - - -def psql(sql: str) -> str: - completed = subprocess.run( - [ - "docker", - "exec", - "-i", - container, - "psql", - "--no-psqlrc", - "--set", - "ON_ERROR_STOP=1", - "--tuples-only", - "--no-align", - "--quiet", - "-U", - "qbit", - "-d", - "qbit", - ], - input=sql, - text=True, - stdout=subprocess.PIPE, - stderr=subprocess.PIPE, - check=False, - ) - if completed.returncode != 0: - raise RuntimeError( - "psql command failed " - f"(exit {completed.returncode}): {completed.stderr.strip()}" - ) - return completed.stdout - - -psql(SCHEMA_PATH.read_text(encoding="utf-8")) -postgres_version = psql("SELECT version();").strip() - -insert_sql = f""" -TRUNCATE qbit_share_ledger RESTART IDENTITY CASCADE; -INSERT INTO qbit_share_ledger ( - share_id, - miner_id, - payout_order_key, - p2mr_program, - share_difficulty, - network_difficulty, - template_height, - job_id, - job_issued_at, - ntime, - accepted_at, - accepted, - writer_id, - writer_epoch +cd "$(dirname "${BASH_SOURCE[0]}")/.." + +# Each test writes its own report, so the two floor tests cannot overwrite each +# other. This script runs the full-size one. +test_name=share_append_throughput_floor_full_size + +# Release, unlike the CI-sized run: this is the "how fast can this host go" +# measurement, and a debug build would report the compiler's overhead as the +# ledger's. The CI floor is calibrated separately, in debug. +# Without the leading `test`: the direct call below adds it, and +# `prism-native-tests.sh cargo-args` runs `cargo test "$@"` itself. Passing +# `test` here too would make the second one a test-name filter, and the run +# would select nothing and exit 0 having measured nothing. +cargo_args=( + --locked --release -p qbit-prism-server --test throughput_floor -- + --ignored --exact "${test_name}" --nocapture ) -SELECT - 'bench-' || gs::text, - 'miner-' || (gs % 1024)::text, - lpad((gs % 1024)::text, 8, '0'), - decode(lpad(to_hex(gs), 64, '0'), 'hex'), - 1 + (gs % 4096), - 1000000, - 1000 + (gs / 1000), - 'job-' || (gs / 1000)::text, - to_timestamp(1700000000) + make_interval(secs => gs::double precision / 1000.0), - 1700000000 + gs, - to_timestamp(1700000000) + make_interval(secs => gs::double precision / 1000.0), - true, - 'throughput', - 1 -FROM generate_series(1, {share_count}) AS gs; -""" -started = time.perf_counter() -psql(insert_sql) -insert_seconds = time.perf_counter() - started -observed_count = int(psql("SELECT count(*) FROM qbit_share_ledger WHERE accepted;").strip()) -shares_per_second = observed_count / max(insert_seconds, 0.001) - -anchor_seconds = 1700000000 + (share_count / 1000.0) -explain_sql = f""" -EXPLAIN (ANALYZE, BUFFERS, FORMAT JSON) -SELECT * -FROM qbit_audit_share_window(to_timestamp({anchor_seconds:.3f}), 100000::numeric); -""" -explain_text = psql(explain_sql).strip() - -report = { - "schema": "qbit.prism.postgres-throughput.v1", - "share_count": observed_count, - "insert_seconds": insert_seconds, - "shares_per_second": shares_per_second, - "min_shares_per_second": min_shares_per_second, - "passed_minimum": shares_per_second >= min_shares_per_second, - "postgres_version": postgres_version, - "machine": { - "platform": platform.platform(), - "python": platform.python_version(), - "cpu_count": os.cpu_count(), - }, - "audit_window_explain_analyze": explain_text, -} -report_path.write_text(json.dumps(report, indent=2, sort_keys=True) + "\n", encoding="utf-8") - -print( - "prism postgres throughput " - f"shares={observed_count} seconds={insert_seconds:.3f} " - f"shares_per_second={shares_per_second:.1f} report={report_path}" -) -if shares_per_second < min_shares_per_second: - raise SystemExit( - "Postgres throughput below QBIT_PRISM_MIN_SHARES_PER_SEC: " - f"{shares_per_second:.1f} < {min_shares_per_second:.1f}" - ) -PY -) +# The test writes its report before it asserts the floor, so a run that fails +# the floor still leaves the evidence behind. `set -e` is suspended around the +# test for exactly that reason: the path is worth printing on a red run, and the +# status is re-raised immediately afterwards. +# A report from an earlier run must not be announced as this run's evidence. +# Timestamps cannot tell the two apart reliably -- a warm build and a small +# window can finish inside one filesystem timestamp tick, and then a fresh +# report looks no newer than the moment the run started -- so the earlier file +# is moved aside instead, and the presence of the report afterwards is proof +# that this run wrote it. It is put back if this run writes nothing, so a run +# that dies during compilation or cluster setup does not destroy the evidence +# the previous one produced. +report="${QBIT_PRISM_THROUGHPUT_REPORT:-$(pwd)/target/prism-postgres-throughput-${test_name}.json}" +previous="${report}.previous" +rm -f "${previous}" +if [[ -f "${report}" ]]; then + mv "${report}" "${previous}" +fi + +status=0 +if [[ -n "${PRISM_TEST_DATABASE_URL:-}" ]]; then + cargo test "${cargo_args[@]}" || status=$? +else + # prism-native-tests.sh initdb's a throwaway cluster on a random loopback + # port, exports PRISM_TEST_DATABASE_URL for the command it runs, and stops + # and removes it on exit. + test/prism-native-tests.sh cargo-args "${cargo_args[@]}" || status=$? +fi + +# Announce the report only if this run wrote one. A run that died before the +# test started — no database, no server binaries, a rejected variable — writes +# nothing, and printing a path to a file that does not exist sends the reader +# looking for evidence that was never produced. A file left by an earlier run is +# worse than nothing: it is evidence, just not of this run. +if [[ -f "${report}" ]]; then + echo "PRISM share-append throughput report: ${report}" + rm -f "${previous}" +elif [[ -f "${previous}" ]]; then + mv "${previous}" "${report}" + echo "PRISM share-append throughput: this run wrote no report; the file at ${report} is from an earlier run" +else + echo "PRISM share-append throughput: no report was written (expected it at ${report})" +fi +exit "${status}" diff --git a/test/test-prism-public-read-replica.sh b/test/test-prism-public-read-replica.sh deleted file mode 100755 index b49364a33..000000000 --- a/test/test-prism-public-read-replica.sh +++ /dev/null @@ -1,89 +0,0 @@ -#!/usr/bin/env bash -set -euo pipefail - -ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" -POSTGRES_IMAGE="${QBIT_PRISM_POSTGRES_IMAGE:-postgres:16-alpine}" -NETWORK="qbit-prism-pubrep-net-$$" -PRIMARY="qbit-prism-pubrep-primary-$$" -REPLICA="qbit-prism-pubrep-replica-$$" - -require_executable() { - command -v "$1" >/dev/null 2>&1 || { - echo "missing required executable: $1" >&2 - exit 1 - } -} - -cleanup() { - docker rm -f "${PRIMARY}" "${REPLICA}" >/dev/null 2>&1 || true - docker network rm "${NETWORK}" >/dev/null 2>&1 || true -} -trap cleanup EXIT - -require_executable python3 -require_executable docker - -docker network create "${NETWORK}" >/dev/null - -# The primary runs the same mounted pg_hba.conf the compose service uses, so -# the replication rules (and the local trust the healthcheck depends on) are -# the production-shaped ones. -docker run \ - --rm \ - --detach \ - --name "${PRIMARY}" \ - --network "${NETWORK}" \ - -e POSTGRES_USER=qbit \ - -e POSTGRES_PASSWORD=qbit \ - -e POSTGRES_DB=qbit \ - -v "${ROOT_DIR}/config/prism-postgres/pg_hba.conf:/etc/postgresql/pg_hba.conf:ro" \ - "${POSTGRES_IMAGE}" \ - postgres -c hba_file=/etc/postgresql/pg_hba.conf >/dev/null - -deadline=$((SECONDS + 60)) -until docker exec "${PRIMARY}" pg_isready -h 127.0.0.1 -p 5432 -U qbit -d qbit >/dev/null 2>&1 \ - && sleep 1 \ - && docker exec "${PRIMARY}" pg_isready -h 127.0.0.1 -p 5432 -U qbit -d qbit >/dev/null 2>&1; do - if [ "${SECONDS}" -ge "${deadline}" ]; then - echo "timed out waiting for the primary Postgres container" >&2 - docker logs "${PRIMARY}" >&2 || true - exit 1 - fi - sleep 1 -done - -# The standby reuses the compose entrypoint: pg_basebackup with slot creation, -# then the stock image entrypoint. The schema lands later, through streaming. -docker run \ - --rm \ - --detach \ - --name "${REPLICA}" \ - --network "${NETWORK}" \ - -e POSTGRES_USER=qbit \ - -e POSTGRES_PASSWORD=qbit \ - -e POSTGRES_DB=qbit \ - -e PRISM_POSTGRES_PRIMARY_HOST="${PRIMARY}" \ - -e PRISM_POSTGRES_PRIMARY_PORT=5432 \ - -e PRISM_POSTGRES_REPLICATION_SLOT=prism_public_replica_gate \ - -v "${ROOT_DIR}/config/prism-postgres/replica-entrypoint.sh:/usr/local/bin/prism-replica-entrypoint.sh:ro" \ - --entrypoint /usr/local/bin/prism-replica-entrypoint.sh \ - "${POSTGRES_IMAGE}" >/dev/null - -deadline=$((SECONDS + 120)) -until docker exec "${REPLICA}" pg_isready -h 127.0.0.1 -p 5432 -U qbit -d qbit >/dev/null 2>&1 \ - && [ "$(docker exec "${REPLICA}" psql -U qbit -d qbit -tAc 'SELECT pg_is_in_recovery()' 2>/dev/null | tr -d '[:space:]')" = "t" ]; do - if [ "${SECONDS}" -ge "${deadline}" ]; then - echo "timed out waiting for the replica standby container" >&2 - docker logs "${REPLICA}" >&2 || true - exit 1 - fi - sleep 1 -done - -( - cd "${ROOT_DIR}" - GATE_PRIMARY_PSQL_COMMAND="docker exec -i ${PRIMARY} psql -U qbit -d qbit" \ - GATE_REPLICA_PSQL_COMMAND="docker exec -i ${REPLICA} psql -U qbit -d qbit" \ - PYTHONPATH="${ROOT_DIR}" \ - python3 tests/prism_public_read_replica_gate.py -) diff --git a/test/test-prism-regtest.sh b/test/test-prism-regtest.sh deleted file mode 100755 index 5ee113a62..000000000 --- a/test/test-prism-regtest.sh +++ /dev/null @@ -1,628 +0,0 @@ -#!/usr/bin/env bash -set -euo pipefail - -ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" -source "${ROOT_DIR}/test/test-lib.sh" -DATADIR="${DATADIR:-$(mktemp -d -t qbit-prism-regtest.XXXXXX)}" -RPC_USER="${RPC_USER:-qbitrpc}" -RPC_PASSWORD="${RPC_PASSWORD:-change-this}" -RPC_PORT="${RPC_PORT:-18454}" -WALLET_NAME="${WALLET_NAME:-prism-regtest}" -REGTEST_BLOCKS="${QBIT_PRISM_REGTEST_BLOCKS:-2}" -ASERT_MOCKTIME_BASE="${QBIT_PRISM_REGTEST_MOCKTIME_BASE:-1738713603}" -ASERT_TIME_STEP="${QBIT_PRISM_REGTEST_ASERT_TIME_STEP:-1}" -MANIFEST_SIGNING_SEED_HEX="${QBIT_PRISM_MANIFEST_SIGNING_SEED_HEX:-4242424242424242424242424242424242424242424242424242424242424242}" -LEDGER_ATTESTATION_SIGNING_SEED_HEX="${QBIT_PRISM_LEDGER_ATTESTATION_SIGNING_SEED_HEX:-4343434343434343434343434343434343434343434343434343434343434343}" -COINBASE_SCRIPT_SIG_SUFFIX_HEX="${QBIT_PRISM_COINBASE_SCRIPT_SIG_SUFFIX_HEX:-111111112222222222222222}" -REORG_TEST="${QBIT_PRISM_REGTEST_REORG:-1}" -REORG_COINBASE_SCRIPT_SIG_SUFFIX_HEX="${QBIT_PRISM_REORG_COINBASE_SCRIPT_SIG_SUFFIX_HEX:-111111113333333333333333}" - -resolve_qbit_binaries -read -r -a CARGO_CMD <<< "${CARGO:-cargo}" -require_executable "${CARGO_CMD[0]}" -require_executable python3 -require_qbit_src_helper "test/functional/test_framework/blocktools.py" "the PRISM regtest smoke" -run_cargo() { - "${CARGO_CMD[@]}" "$@" -} - -ledger_public_key_from_bundle_json() { - BUNDLE_JSON="$1" python3 <<'PY' -import json -import os - -bundle = json.loads(os.environ["BUNDLE_JSON"]) -print(bundle["ledger_window_attestation"]["signature"]["public_key_hex"]) -PY -} - -cleanup() { - stop_qbitd - remove_datadir "${DATADIR}" -} -trap cleanup EXIT - -if [[ "$(ulimit -n)" == "unlimited" ]]; then - ulimit -n 1024 || true -fi - -if [[ "${REGTEST_BLOCKS}" -lt 1 ]]; then - echo "QBIT_PRISM_REGTEST_BLOCKS must be >= 1" >&2 - exit 1 -fi - -"${QBITD_BIN}" \ - -regtest \ - -asert \ - -p2mronly=1 \ - -daemonwait \ - -server \ - -listen=0 \ - -maxconnections=0 \ - -dnsseed=0 \ - -fixedseeds=0 \ - -rpcbind=127.0.0.1 \ - -rpcallowip=127.0.0.1 \ - -rpcport="${RPC_PORT}" \ - -rpcuser="${RPC_USER}" \ - -rpcpassword="${RPC_PASSWORD}" \ - -fallbackfee=0.00001000 \ - -datadir="${DATADIR}" >/dev/null - -wait_for_qbit_rpc_state ready 60 -qbit_rpc createwallet "${WALLET_NAME}" >/dev/null - -programs_json="$( - WALLET_NAME="${WALLET_NAME}" \ - QBIT_CLI_BIN="${QBIT_CLI_BIN}" \ - DATADIR="${DATADIR}" \ - RPC_USER="${RPC_USER}" \ - RPC_PASSWORD="${RPC_PASSWORD}" \ - RPC_PORT="${RPC_PORT}" \ - FIXTURE_PATH="${ROOT_DIR}/crates/qbit-prism/fixtures/power-law-accrual.prism-fixture.json" \ - python3 <<'PY' -import json -import os -import subprocess - -fixture = json.load(open(os.environ["FIXTURE_PATH"], encoding="utf-8")) - -def qbit_rpc(*args: str) -> str: - command = [ - os.environ["QBIT_CLI_BIN"], - "-regtest", - f"-datadir={os.environ['DATADIR']}", - f"-rpcuser={os.environ['RPC_USER']}", - f"-rpcpassword={os.environ['RPC_PASSWORD']}", - f"-rpcport={os.environ['RPC_PORT']}", - *args, - ] - return subprocess.check_output(command, text=True).strip() - -programs = [] -seen = set() -for _ in fixture["shares"]: - address = qbit_rpc(f"-rpcwallet={os.environ['WALLET_NAME']}", "getnewaddress", "", "p2mr") - if address in seen: - raise SystemExit(f"duplicate generated payout address: {address}") - seen.add(address) - info = json.loads(qbit_rpc("getaddressinfo", address)) - script = info["scriptPubKey"] - if not script.startswith("5220") or len(script) != 68: - raise SystemExit(f"non-P2MR script for {address}: {script}") - programs.append(script[4:]) - -print(json.dumps(programs, separators=(",", ":"))) -PY -)" - -prior_balances_json="[]" -bits_seen=() -block_hashes=() -bundle_paths=() -prior_before_paths=() - -for block_index in $(seq 1 "${REGTEST_BLOCKS}"); do - mocktime=$((ASERT_MOCKTIME_BASE + (block_index - 1) * ASERT_TIME_STEP)) - qbit_rpc setmocktime "${mocktime}" >/dev/null - - before_height="$(qbit_rpc getblockcount)" - template_json="$(qbit_rpc getblocktemplate '{"rules":["segwit"]}')" - template_bits="$(TEMPLATE_JSON="${template_json}" python3 - <<'PY' -import json -import os -print(json.loads(os.environ["TEMPLATE_JSON"])["bits"]) -PY -)" - bits_seen+=("${template_bits}") - - bundle_input_json="$( - TEMPLATE_JSON="${template_json}" \ - PROGRAMS_JSON="${programs_json}" \ - PRIOR_BALANCES_JSON="${prior_balances_json}" \ - BLOCK_INDEX="${block_index}" \ - FIXTURE_PATH="${ROOT_DIR}/crates/qbit-prism/fixtures/power-law-accrual.prism-fixture.json" \ - COINBASE_SCRIPT_SIG_SUFFIX_HEX="${COINBASE_SCRIPT_SIG_SUFFIX_HEX}" \ - python3 <<'PY' -import copy -import json -import os - -template = json.loads(os.environ["TEMPLATE_JSON"]) -fixture = json.load(open(os.environ["FIXTURE_PATH"], encoding="utf-8")) -programs = json.loads(os.environ["PROGRAMS_JSON"]) -prior_balances = json.loads(os.environ["PRIOR_BALANCES_JSON"]) -block_index = int(os.environ["BLOCK_INDEX"]) -if template.get("transactions"): - raise SystemExit("prism regtest smoke expects an empty regtest mempool") - -def target_from_compact(bits_hex: str) -> int: - compact = int(bits_hex, 16) - size = compact >> 24 - mantissa = compact & 0x007fffff - if size <= 3: - return mantissa >> (8 * (3 - size)) - return mantissa << (8 * (size - 3)) - -pow_limit_target = target_from_compact("207fffff") -template_target = target_from_compact(template["bits"]) -network_difficulty = max(1, (pow_limit_target * 1_000_000) // template_target) -share_difficulties = [ - network_difficulty * 5, - network_difficulty, - max(1, network_difficulty // 4), - 10, - 1, - 1, -] -if len(share_difficulties) != len(fixture["shares"]) or len(programs) != len(fixture["shares"]): - raise SystemExit("prism fixture shape changed; update test/test-prism-regtest.sh") - -found_block = copy.deepcopy(fixture["found_block"]) -found_block["block_height"] = int(template["height"]) -found_block["coinbase_value_sats"] = int(template["coinbasevalue"]) -found_block["network_difficulty"] = network_difficulty -time_offset_ms = (block_index - 1) * 10_000 -found_block["anchor_job_issued_at_ms"] = int(found_block["anchor_job_issued_at_ms"]) + time_offset_ms - -shares = copy.deepcopy(fixture["shares"]) -base_seq = (block_index - 1) * len(shares) -for offset, (share, program, difficulty) in enumerate(zip(shares, programs, share_difficulties), start=1): - share["share_seq"] = base_seq + offset - share["share_id"] = f"{share['share_id']}-block-{block_index}" - share["job_id"] = f"{share['job_id']}-block-{block_index}" - share["p2mr_program_hex"] = program - share["share_difficulty"] = difficulty - share["network_difficulty"] = network_difficulty - share["template_height"] = int(template["height"]) - 1 - share["job_issued_at_ms"] = int(share["job_issued_at_ms"]) + time_offset_ms - share["accepted_at_ms"] = int(share["accepted_at_ms"]) + time_offset_ms - share["ntime"] = int(template["curtime"]) - -print(json.dumps({ - "found_block": found_block, - "shares": shares, - "prior_balances": prior_balances, - "coinbase_script_sig_suffix_hex": os.environ["COINBASE_SCRIPT_SIG_SUFFIX_HEX"], -}, separators=(",", ":"))) -PY - )" - prior_before_path="${DATADIR}/prior-balances-before-${block_index}.json" - printf '%s\n' "${prior_balances_json}" >"${prior_before_path}" - prior_before_paths+=("${prior_before_path}") - - bundle_json="$( - run_cargo run --quiet -p qbit-prism --bin qbit-prism-build-audit-bundle -- \ - --input - \ - --signing-key-seed-hex "${MANIFEST_SIGNING_SEED_HEX}" \ - --ledger-signing-key-seed-hex "${LEDGER_ATTESTATION_SIGNING_SEED_HEX}" <<<"${bundle_input_json}" - )" - ledger_public_key_hex="$(ledger_public_key_from_bundle_json "${bundle_json}")" - bundle_path="${DATADIR}/prism-audit-bundle-${block_index}.json" - printf '%s\n' "${bundle_json}" >"${bundle_path}" - bundle_paths+=("${bundle_path}") - - block_hex="$( - TEMPLATE_JSON="${template_json}" \ - BUNDLE_JSON="${bundle_json}" \ - QBIT_SRC_DIR="${QBIT_SRC_DIR}" \ - python3 <<'PY' -import io -import json -import os -import sys -from pathlib import Path - -qbit_src = Path(os.environ["QBIT_SRC_DIR"]) -sys.path.insert(0, str(qbit_src / "test/functional")) - -from test_framework.blocktools import create_block -from test_framework.messages import CTransaction - -template = json.loads(os.environ["TEMPLATE_JSON"]) -bundle = json.loads(os.environ["BUNDLE_JSON"]) -manifest = bundle["signed_coinbase_manifest"]["manifest"] -coinbase = CTransaction() -coinbase.deserialize(io.BytesIO(bytes.fromhex(manifest["coinbase_tx_hex"]))) -block = create_block( - hashprev=int(template["previousblockhash"], 16), - coinbase=coinbase, - ntime=int(template["curtime"]), - version=int(template["version"]), - tmpl=template, - txlist=[tx["data"] for tx in template.get("transactions", [])], -) -block.solve() -print(block.serialize().hex()) -PY - )" - - result="$(qbit_rpc submitblock "${block_hex}")" - after_height="$(qbit_rpc getblockcount)" - if [[ -n "${result}" && "${result}" != "null" ]]; then - echo "submitblock returned failure for PRISM regtest block ${block_index}: ${result}" >&2 - exit 1 - fi - if [[ "${after_height}" -ne $((before_height + 1)) ]]; then - echo "expected blockcount ${before_height}+1 after submitblock, got ${after_height}" >&2 - exit 1 - fi - - block_hash="$(qbit_rpc getblockhash "${after_height}")" - block_hashes+=("${block_hash}") - raw_block="$(qbit_rpc getblock "${block_hash}" 0)" - coinbase_tx_hex="$( - RAW_BLOCK="${raw_block}" \ - BUNDLE_JSON="${bundle_json}" \ - QBIT_SRC_DIR="${QBIT_SRC_DIR}" \ - python3 <<'PY' -import io -import json -import os -import sys -from pathlib import Path - -qbit_src = Path(os.environ["QBIT_SRC_DIR"]) -sys.path.insert(0, str(qbit_src / "test/functional")) - -from test_framework.messages import CBlock - -block = CBlock() -block.deserialize(io.BytesIO(bytes.fromhex(os.environ["RAW_BLOCK"]))) -bundle = json.loads(os.environ["BUNDLE_JSON"]) -manifest = bundle["signed_coinbase_manifest"]["manifest"] -policy_manifest = bundle["payout_policy_manifest"] -coinbase = block.vtx[0] -coinbase_tx_hex = coinbase.serialize_with_witness().hex() -if coinbase_tx_hex != manifest["coinbase_tx_hex"]: - raise SystemExit("on-chain coinbase bytes do not match signed PRISM manifest coinbase_tx_hex") -if manifest["coinbase_script_sig_suffix_hex"] != bundle["coinbase_script_sig_suffix_hex"]: - raise SystemExit("manifest scriptSig suffix does not match audit bundle suffix") -if len(manifest["outputs"]) < 3: - raise SystemExit(f"expected at least 3 on-chain PRISM payouts, got {len(manifest['outputs'])}") -accrued = [account for account in policy_manifest["accounts"] if account["action"] == "accrued"] -if len(accrued) < 1: - raise SystemExit("expected at least one accrued below-floor account") -actual = [ - (txout.nValue, bytes(txout.scriptPubKey).hex()) - for txout in coinbase.vout[:len(manifest["outputs"])] -] -expected = [ - (output["amount_sats"], output["script_pubkey_hex"]) - for output in manifest["outputs"] -] -if actual != expected: - raise SystemExit("on-chain PRISM payout outputs do not match signed manifest") -if sum(value for value, _script in actual) != manifest["coinbase_value_sats"]: - raise SystemExit("on-chain PRISM payout sum does not equal coinbase value") -commitment = coinbase.vout[-1] -if commitment.nValue != 0 or bytes(commitment.scriptPubKey).hex() != manifest["witness_commitment_script_hex"]: - raise SystemExit("on-chain witness commitment does not match PRISM manifest") -print(coinbase_tx_hex) -PY - )" - - verify_report="$( - run_cargo run --quiet -p qbit-prism --bin qbit-prism-audit-verify -- \ - "${bundle_path}" \ - --coinbase-tx-hex "${coinbase_tx_hex}" \ - --ledger-writer-public-key-hex "${ledger_public_key_hex}" \ - --expected-coinbase-value-sats "$(BUNDLE_PATH="${bundle_path}" python3 -c 'import json,os; print(json.load(open(os.environ["BUNDLE_PATH"], encoding="utf-8"))["found_block"]["coinbase_value_sats"])')" - )" - VERIFY_REPORT="${verify_report}" python3 <<'PY' -import json -import os - -report = json.loads(os.environ["VERIFY_REPORT"]) -if report["schema"] != "qbit.prism.audit-verification-report.v1": - raise SystemExit("unexpected audit verifier report schema") -if report["onchain_output_count"] < 3: - raise SystemExit(f"expected at least 3 verified on-chain outputs, got {report['onchain_output_count']}") -if report["accrued_account_count"] < 1: - raise SystemExit("expected verifier to report accrued below-floor accounts") -PY - - prior_balances_json="$( - BUNDLE_JSON="${bundle_json}" python3 <<'PY' -import json -import os - -bundle = json.loads(os.environ["BUNDLE_JSON"]) -balances = [] -for account in bundle["payout_policy_manifest"]["accounts"]: - carry = int(account["carry_forward_balance_sats"]) - if carry > 0: - balances.append({ - "recipient_id": account["recipient_id"], - "order_key": account["order_key"], - "p2mr_program_hex": account["p2mr_program_hex"], - "balance_sats": carry, - }) -print(json.dumps(balances, separators=(",", ":"))) -PY - )" - - network_difficulty="$( - BUNDLE_JSON="${bundle_json}" python3 <<'PY' -import json -import os -print(json.loads(os.environ["BUNDLE_JSON"])["found_block"]["network_difficulty"]) -PY - )" - printf 'prism regtest block PASS index=%s height=%s bits=%s scaled_network_difficulty=%s block=%s\n' \ - "${block_index}" "${after_height}" "${template_bits}" "${network_difficulty}" "${block_hash}" -done - -if [[ "${REGTEST_BLOCKS}" -ge 2 ]]; then - unique_bits="$(printf '%s\n' "${bits_seen[@]}" | sort -u | wc -l | tr -d ' ')" - if [[ "${unique_bits}" -lt 2 ]]; then - echo "expected ASERT compact target to change across ${REGTEST_BLOCKS} blocks; saw: ${bits_seen[*]}" >&2 - exit 1 - fi -fi - -if [[ "${REORG_TEST}" == "1" && "${REGTEST_BLOCKS}" -ge 2 ]]; then - reorg_index="${REGTEST_BLOCKS}" - disconnected_hash="${block_hashes[$((reorg_index - 1))]}" - disconnected_bundle_path="${bundle_paths[$((reorg_index - 1))]}" - replacement_prior_balances_json="$(cat "${prior_before_paths[$((reorg_index - 1))]}")" - - qbit_rpc invalidateblock "${disconnected_hash}" >/dev/null - reorg_height="$(qbit_rpc getblockcount)" - if [[ "${reorg_height}" -ne $((reorg_index - 1)) ]]; then - echo "expected height $((reorg_index - 1)) after invalidating ${disconnected_hash}, got ${reorg_height}" >&2 - exit 1 - fi - - replacement_mocktime=$((ASERT_MOCKTIME_BASE + (reorg_index - 1) * ASERT_TIME_STEP)) - qbit_rpc setmocktime "${replacement_mocktime}" >/dev/null - before_height="$(qbit_rpc getblockcount)" - template_json="$(qbit_rpc getblocktemplate '{"rules":["segwit"]}')" - replacement_bits="$(TEMPLATE_JSON="${template_json}" python3 - <<'PY' -import json -import os -print(json.loads(os.environ["TEMPLATE_JSON"])["bits"]) -PY -)" - bundle_input_json="$( - TEMPLATE_JSON="${template_json}" \ - PROGRAMS_JSON="${programs_json}" \ - PRIOR_BALANCES_JSON="${replacement_prior_balances_json}" \ - BLOCK_INDEX="${reorg_index}" \ - FIXTURE_PATH="${ROOT_DIR}/crates/qbit-prism/fixtures/power-law-accrual.prism-fixture.json" \ - COINBASE_SCRIPT_SIG_SUFFIX_HEX="${REORG_COINBASE_SCRIPT_SIG_SUFFIX_HEX}" \ - python3 <<'PY' -import copy -import json -import os - -template = json.loads(os.environ["TEMPLATE_JSON"]) -fixture = json.load(open(os.environ["FIXTURE_PATH"], encoding="utf-8")) -programs = json.loads(os.environ["PROGRAMS_JSON"]) -prior_balances = json.loads(os.environ["PRIOR_BALANCES_JSON"]) -block_index = int(os.environ["BLOCK_INDEX"]) -if template.get("transactions"): - raise SystemExit("prism regtest reorg replacement expects an empty regtest mempool") - -def target_from_compact(bits_hex: str) -> int: - compact = int(bits_hex, 16) - size = compact >> 24 - mantissa = compact & 0x007fffff - if size <= 3: - return mantissa >> (8 * (3 - size)) - return mantissa << (8 * (size - 3)) - -pow_limit_target = target_from_compact("207fffff") -template_target = target_from_compact(template["bits"]) -network_difficulty = max(1, (pow_limit_target * 1_000_000) // template_target) -share_difficulties = [ - network_difficulty * 5, - network_difficulty, - max(1, network_difficulty // 4), - 10, - 1, - 1, -] -if len(share_difficulties) != len(fixture["shares"]) or len(programs) != len(fixture["shares"]): - raise SystemExit("prism fixture shape changed; update test/test-prism-regtest.sh") - -found_block = copy.deepcopy(fixture["found_block"]) -found_block["block_height"] = int(template["height"]) -found_block["coinbase_value_sats"] = int(template["coinbasevalue"]) -found_block["network_difficulty"] = network_difficulty -time_offset_ms = (block_index - 1) * 10_000 -found_block["anchor_job_issued_at_ms"] = int(found_block["anchor_job_issued_at_ms"]) + time_offset_ms - -shares = copy.deepcopy(fixture["shares"]) -base_seq = (block_index - 1) * len(shares) -for offset, (share, program, difficulty) in enumerate(zip(shares, programs, share_difficulties), start=1): - share["share_seq"] = base_seq + offset - share["share_id"] = f"{share['share_id']}-replacement-{block_index}" - share["job_id"] = f"{share['job_id']}-replacement-{block_index}" - share["p2mr_program_hex"] = program - share["share_difficulty"] = difficulty - share["network_difficulty"] = network_difficulty - share["template_height"] = int(template["height"]) - 1 - share["job_issued_at_ms"] = int(share["job_issued_at_ms"]) + time_offset_ms - share["accepted_at_ms"] = int(share["accepted_at_ms"]) + time_offset_ms - share["ntime"] = int(template["curtime"]) - -print(json.dumps({ - "found_block": found_block, - "shares": shares, - "prior_balances": prior_balances, - "coinbase_script_sig_suffix_hex": os.environ["COINBASE_SCRIPT_SIG_SUFFIX_HEX"], -}, separators=(",", ":"))) -PY - )" - - replacement_bundle_json="$( - run_cargo run --quiet -p qbit-prism --bin qbit-prism-build-audit-bundle -- \ - --input - \ - --signing-key-seed-hex "${MANIFEST_SIGNING_SEED_HEX}" \ - --ledger-signing-key-seed-hex "${LEDGER_ATTESTATION_SIGNING_SEED_HEX}" <<<"${bundle_input_json}" - )" - replacement_ledger_public_key_hex="$(ledger_public_key_from_bundle_json "${replacement_bundle_json}")" - replacement_bundle_path="${DATADIR}/prism-audit-bundle-replacement-${reorg_index}.json" - printf '%s\n' "${replacement_bundle_json}" >"${replacement_bundle_path}" - - block_hex="$( - TEMPLATE_JSON="${template_json}" \ - BUNDLE_JSON="${replacement_bundle_json}" \ - QBIT_SRC_DIR="${QBIT_SRC_DIR}" \ - python3 <<'PY' -import io -import json -import os -import sys -from pathlib import Path - -qbit_src = Path(os.environ["QBIT_SRC_DIR"]) -sys.path.insert(0, str(qbit_src / "test/functional")) - -from test_framework.blocktools import create_block -from test_framework.messages import CTransaction - -template = json.loads(os.environ["TEMPLATE_JSON"]) -bundle = json.loads(os.environ["BUNDLE_JSON"]) -manifest = bundle["signed_coinbase_manifest"]["manifest"] -coinbase = CTransaction() -coinbase.deserialize(io.BytesIO(bytes.fromhex(manifest["coinbase_tx_hex"]))) -block = create_block( - hashprev=int(template["previousblockhash"], 16), - coinbase=coinbase, - ntime=int(template["curtime"]), - version=int(template["version"]), - tmpl=template, - txlist=[tx["data"] for tx in template.get("transactions", [])], -) -block.solve() -print(block.serialize().hex()) -PY - )" - - result="$(qbit_rpc submitblock "${block_hex}")" - after_height="$(qbit_rpc getblockcount)" - if [[ -n "${result}" && "${result}" != "null" ]]; then - echo "submitblock returned failure for PRISM replacement block: ${result}" >&2 - exit 1 - fi - if [[ "${after_height}" -ne $((before_height + 1)) ]]; then - echo "expected replacement blockcount ${before_height}+1 after submitblock, got ${after_height}" >&2 - exit 1 - fi - - replacement_hash="$(qbit_rpc getblockhash "${after_height}")" - if [[ "${replacement_hash}" == "${disconnected_hash}" ]]; then - echo "replacement block hash unexpectedly equals disconnected block ${disconnected_hash}" >&2 - exit 1 - fi - active_tip="$(qbit_rpc getbestblockhash)" - if [[ "${active_tip}" != "${replacement_hash}" ]]; then - echo "expected active tip ${replacement_hash}, got ${active_tip}" >&2 - exit 1 - fi - - raw_block="$(qbit_rpc getblock "${replacement_hash}" 0)" - replacement_coinbase_tx_hex="$( - RAW_BLOCK="${raw_block}" \ - BUNDLE_JSON="${replacement_bundle_json}" \ - QBIT_SRC_DIR="${QBIT_SRC_DIR}" \ - python3 <<'PY' -import io -import json -import os -import sys -from pathlib import Path - -qbit_src = Path(os.environ["QBIT_SRC_DIR"]) -sys.path.insert(0, str(qbit_src / "test/functional")) - -from test_framework.messages import CBlock - -block = CBlock() -block.deserialize(io.BytesIO(bytes.fromhex(os.environ["RAW_BLOCK"]))) -bundle = json.loads(os.environ["BUNDLE_JSON"]) -manifest = bundle["signed_coinbase_manifest"]["manifest"] -coinbase_tx_hex = block.vtx[0].serialize_with_witness().hex() -if coinbase_tx_hex != manifest["coinbase_tx_hex"]: - raise SystemExit("replacement coinbase bytes do not match signed PRISM manifest") -print(coinbase_tx_hex) -PY - )" - - run_cargo run --quiet -p qbit-prism --bin qbit-prism-audit-verify -- \ - "${replacement_bundle_path}" \ - --coinbase-tx-hex "${replacement_coinbase_tx_hex}" \ - --ledger-writer-public-key-hex "${replacement_ledger_public_key_hex}" \ - --expected-coinbase-value-sats "$(BUNDLE_PATH="${replacement_bundle_path}" python3 -c 'import json,os; print(json.load(open(os.environ["BUNDLE_PATH"], encoding="utf-8"))["found_block"]["coinbase_value_sats"])')" >/dev/null - - reorg_input_json="$( - DISCONNECTED_HASH="${disconnected_hash}" \ - DISCONNECTED_HEIGHT="${reorg_index}" \ - DISCONNECTED_BUNDLE_PATH="${disconnected_bundle_path}" \ - REPLACEMENT_HASH="${replacement_hash}" \ - REPLACEMENT_HEIGHT="${reorg_index}" \ - REPLACEMENT_BUNDLE_PATH="${replacement_bundle_path}" \ - python3 <<'PY' -import json -import os - -with open(os.environ["DISCONNECTED_BUNDLE_PATH"], encoding="utf-8") as handle: - disconnected = json.load(handle) -with open(os.environ["REPLACEMENT_BUNDLE_PATH"], encoding="utf-8") as handle: - replacement = json.load(handle) -print(json.dumps({ - "disconnected_block_hash": os.environ["DISCONNECTED_HASH"], - "disconnected_block_height": int(os.environ["DISCONNECTED_HEIGHT"]), - "disconnected_payout_policy_manifest": disconnected["payout_policy_manifest"], - "replacement_block_hash": os.environ["REPLACEMENT_HASH"], - "replacement_block_height": int(os.environ["REPLACEMENT_HEIGHT"]), - "replacement_payout_policy_manifest": replacement["payout_policy_manifest"], -}, separators=(",", ":"))) -PY - )" - reorg_report="$( - run_cargo run --quiet -p qbit-prism --bin qbit-prism-reorg-verify -- \ - --input - <<<"${reorg_input_json}" - )" - VERIFY_REPORT="${reorg_report}" python3 <<'PY' -import json -import os - -report = json.loads(os.environ["VERIFY_REPORT"]) -if report["schema"] != "qbit.prism.reorg-verification-report.v1": - raise SystemExit("unexpected reorg verifier report schema") -if report["reversed_entry_count"] != report["disconnected_entry_count"]: - raise SystemExit("not all disconnected entries were reversed") -if report["replacement_entry_count"] < 1: - raise SystemExit("replacement branch produced no maturity entries") -PY - - printf 'prism reorg PASS disconnected=%s replacement=%s replacement_bits=%s\n' \ - "${disconnected_hash}" "${replacement_hash}" "${replacement_bits}" -fi - -active_tip="$(qbit_rpc getbestblockhash)" -printf 'prism regtest PASS blocks=%s asert_bits=%s active_tip=%s\n' \ - "${REGTEST_BLOCKS}" "$(IFS=,; echo "${bits_seen[*]}")" "${active_tip}" diff --git a/test/test-prism-stratum-regtest-live.sh b/test/test-prism-stratum-regtest-live.sh deleted file mode 100755 index da78da09c..000000000 --- a/test/test-prism-stratum-regtest-live.sh +++ /dev/null @@ -1,811 +0,0 @@ -#!/usr/bin/env bash -set -euo pipefail - -ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" -source "${ROOT_DIR}/test/test-lib.sh" -# shellcheck source=test/prism-postgres-lib.sh -source "${ROOT_DIR}/test/prism-postgres-lib.sh" -DATADIR="${DATADIR:-$(mktemp -d -t qbit-prism-live.XXXXXX)}" -RPC_USER="${RPC_USER:-qbitrpc}" -RPC_PASSWORD="${RPC_PASSWORD:-change-this}" -RPC_PORT="${RPC_PORT:-18455}" -WALLET_NAME="${WALLET_NAME:-prism-live}" -STRATUM_PORT="${PRISM_STRATUM_PORT:-3340}" -AUDIT_PORT="${PRISM_AUDIT_PORT:-3341}" -HIGHDIFF_PORT="${PRISM_STRATUM_HIGHDIFF_PORT:-4334}" -HIGHDIFF_START_DIFF="${PRISM_STRATUM_HIGHDIFF_START_DIFF:-500000}" -POWER_LAW_ENABLED="${QBIT_PRISM_LIVE_POWER_LAW:-0}" -if [[ "${POWER_LAW_ENABLED}" == "1" ]]; then - MINER_COUNT="${QBIT_PRISM_LIVE_MINERS:-6}" -else - MINER_COUNT="${QBIT_PRISM_LIVE_MINERS:-3}" -fi -TARGET_BLOCKS="${QBIT_PRISM_LIVE_BLOCKS:-1}" -EVIDENCE_PATH="${DATADIR}/prism-live-evidence.json" -COORDINATOR_LOG="${DATADIR}/prism-coordinator.log" -MINER_TIMEOUT_SECONDS="${QBIT_PRISM_LIVE_MINER_TIMEOUT_SECONDS:-90}" -AUDIT_API_ENABLED="${QBIT_PRISM_LIVE_AUDIT_API:-0}" -POSTGRES_ENABLED="${QBIT_PRISM_LIVE_POSTGRES:-0}" -NATIVE_POSTGRES="${QBIT_PRISM_LIVE_NATIVE_POSTGRES:-0}" -PRIMED_WINDOW="${QBIT_PRISM_LIVE_PRIMED_WINDOW:-0}" -RESTART_REPLAY="${QBIT_PRISM_LIVE_RESTART_REPLAY:-0}" -CRASH_MARKER="${DATADIR}/candidate-crash-marker.json" -POSTGRES_IMAGE="${QBIT_PRISM_POSTGRES_IMAGE:-postgres:16-alpine}" -POSTGRES_CONTAINER="${QBIT_PRISM_POSTGRES_CONTAINER:-qbit-prism-pg-$$}" -MANIFEST_SIGNING_SEED_HEX="${QBIT_PRISM_MANIFEST_SIGNING_SEED_HEX:-4242424242424242424242424242424242424242424242424242424242424242}" -LEDGER_ATTESTATION_SIGNING_SEED_HEX="${QBIT_PRISM_LEDGER_ATTESTATION_SIGNING_SEED_HEX:-4343434343434343434343434343434343434343434343434343434343434343}" -PSQL_COMMAND="" - -resolve_qbit_binaries -read -r -a CARGO_CMD <<< "${CARGO:-cargo}" -require_executable "${CARGO_CMD[0]}" -require_executable python3 -# When QBIT_PRISM_EXTERNAL_PSQL_COMMAND is set, use an already-running Postgres -# (e.g. a local cluster) instead of provisioning a Docker container. -EXTERNAL_PSQL="${QBIT_PRISM_EXTERNAL_PSQL_COMMAND:-}" -if [[ "${POSTGRES_ENABLED}" == "1" && -z "${EXTERNAL_PSQL}" ]]; then - require_executable docker -fi -run_cargo() { - "${CARGO_CMD[@]}" "$@" -} -if [[ "${POSTGRES_ENABLED}" == "1" && "${AUDIT_API_ENABLED}" != "1" ]]; then - echo "QBIT_PRISM_LIVE_POSTGRES=1 requires QBIT_PRISM_LIVE_AUDIT_API=1 for DB/API proof checks" >&2 - exit 1 -fi -if [[ "${TARGET_BLOCKS}" -gt 1 && "${AUDIT_API_ENABLED}" != "1" ]]; then - echo "QBIT_PRISM_LIVE_BLOCKS>1 requires QBIT_PRISM_LIVE_AUDIT_API=1" >&2 - exit 1 -fi -if [[ "${RESTART_REPLAY}" == "1" && ( "${PRIMED_WINDOW}" != "1" || "${NATIVE_POSTGRES}" != "1" || "${POSTGRES_ENABLED}" != "1" ) ]]; then - echo "restart replay requires primed-window and native-Postgres fixture modes" >&2 - exit 1 -fi -if [[ "${RESTART_REPLAY}" == "1" && -n "${EXTERNAL_PSQL}" && -z "${PRISM_DATABASE_URL:-}" ]]; then - echo "restart replay with external Postgres requires PRISM_DATABASE_URL" >&2 - exit 1 -fi -if [[ "${POWER_LAW_ENABLED}" == "1" && "${MINER_COUNT}" -lt 6 ]]; then - echo "QBIT_PRISM_LIVE_POWER_LAW=1 requires at least 6 miners" >&2 - exit 1 -fi -require_qbit_src_helper "test/functional/test_framework/blocktools.py" "the live PRISM Stratum regtest smoke" - -coordinator_pid="" -miner_pids=() -postgres_started=0 - -cleanup() { - for pid in "${miner_pids[@]:-}"; do - kill "${pid}" >/dev/null 2>&1 || true - done - if [[ -n "${coordinator_pid}" ]]; then - kill "${coordinator_pid}" >/dev/null 2>&1 || true - fi - if [[ "${postgres_started}" == "1" ]]; then - docker rm -f "${POSTGRES_CONTAINER}" >/dev/null 2>&1 || true - fi - stop_qbitd - remove_datadir "${DATADIR}" -} -trap cleanup EXIT - -if [[ "$(ulimit -n)" == "unlimited" ]]; then - ulimit -n 1024 || true -fi - -if [[ "${POSTGRES_ENABLED}" == "1" ]]; then - if [[ -n "${EXTERNAL_PSQL}" ]]; then - PSQL_COMMAND="${EXTERNAL_PSQL}" - deadline=$((SECONDS + 60)) - until echo 'SELECT 1;' | ${PSQL_COMMAND} >/dev/null 2>&1; do - if [[ "${SECONDS}" -ge "${deadline}" ]]; then - echo "timed out waiting for external PRISM Postgres" >&2 - exit 1 - fi - sleep 1 - done - else - docker rm -f "${POSTGRES_CONTAINER}" >/dev/null 2>&1 || true - postgres_port_args=() - if [[ "${NATIVE_POSTGRES}" == "1" ]]; then - postgres_port_args=(-p 127.0.0.1::5432) - fi - docker run \ - "${postgres_port_args[@]}" \ - --rm \ - --detach \ - --name "${POSTGRES_CONTAINER}" \ - -e POSTGRES_USER=qbit \ - -e POSTGRES_PASSWORD=qbit \ - -e POSTGRES_DB=qbit \ - "${POSTGRES_IMAGE}" >/dev/null - postgres_started=1 - PSQL_COMMAND="docker exec -i ${POSTGRES_CONTAINER} psql -U qbit -d qbit" - wait_for_prism_postgres_container "${POSTGRES_CONTAINER}" - fi -fi - -node_fixture_args=() -if [[ "${PRIMED_WINDOW}" == "1" ]]; then - # Keep ASERT near its anchor while mining enough blocks to separate the - # network target from the low Stratum share target. This is regtest only. - node_fixture_args=(-mocktime=1738714602) -fi -"${QBITD_BIN}" \ - "${node_fixture_args[@]}" \ - -regtest \ - -asert \ - -p2mronly=1 \ - -daemonwait \ - -server \ - -listen=0 \ - -maxconnections=0 \ - -dnsseed=0 \ - -fixedseeds=0 \ - -rpcbind=127.0.0.1 \ - -rpcallowip=127.0.0.1 \ - -rpcport="${RPC_PORT}" \ - -rpcuser="${RPC_USER}" \ - -rpcpassword="${RPC_PASSWORD}" \ - -fallbackfee=0.00001000 \ - -datadir="${DATADIR}" >/dev/null - -wait_for_qbit_rpc_state ready 60 -miner_usernames=() -if [[ "${PRIMED_WINDOW}" == "1" ]]; then - # Wallet-free deterministic P2MR programs are enough to verify the mined - # coinbase. The extra identity keeps difficulty probes out of miner state. - for miner_index in $(seq 1 "$((MINER_COUNT + 1))"); do - miner_usernames+=("$(python3 - "${miner_index}" "${QBIT_SRC_DIR}" <<'PYADDR' -import sys -sys.path.insert(0, sys.argv[2] + "/test/functional") -from test_framework.address import program_to_witness -print(program_to_witness(2, int(sys.argv[1]).to_bytes(32, "big"), main=False)) -PYADDR -)") - done - qbit_rpc generatetoaddress 1000 "${miner_usernames[0]}" >/dev/null -else - qbit_rpc createwallet "${WALLET_NAME}" >/dev/null - for _miner_index in $(seq 1 "$((MINER_COUNT + 1))"); do - miner_usernames+=("$(qbit_rpc -rpcwallet="${WALLET_NAME}" getnewaddress "" p2mr)") - done - # Exit IBD without adding any PRISM ledger rows. - qbit_rpc generatetoaddress 1 "${miner_usernames[0]}" >/dev/null -fi -before_height="$(qbit_rpc getblockcount)" - -start_coordinator() { -( - cd "${ROOT_DIR}" - export QBIT_RPC_HOST=127.0.0.1 - export QBIT_RPC_PORT="${RPC_PORT}" - export QBIT_RPC_USER="${RPC_USER}" - export QBIT_RPC_PASSWORD="${RPC_PASSWORD}" - export PRISM_STRATUM_BIND=127.0.0.1 - export PRISM_STRATUM_PORT="${STRATUM_PORT}" - export PRISM_STRATUM_HIGHDIFF_PORT="${HIGHDIFF_PORT}" - export PRISM_STRATUM_HIGHDIFF_START_DIFF="${HIGHDIFF_START_DIFF}" - export PRISM_STRATUM_HIGHDIFF_MIN_DIFF="${HIGHDIFF_START_DIFF}" - export PRISM_HOT_PATH_LOG=1 - export PRISM_MIN_READY_MINERS="${MINER_COUNT}" - export PRISM_EVIDENCE_PATH="${EVIDENCE_PATH}" - export PRISM_AUDIT_DIR="${DATADIR}" - export PRISM_MANIFEST_SIGNING_SEED_HEX="${MANIFEST_SIGNING_SEED_HEX}" - export PRISM_LEDGER_ATTESTATION_SIGNING_SEED_HEX="${LEDGER_ATTESTATION_SIGNING_SEED_HEX}" - export PRISM_ALLOW_BUNDLE_EMBEDDED_LEDGER_KEY=1 - if [[ "${PRIMED_WINDOW}" == "1" ]]; then - # The template intentionally uses historical node mocktime. Ledger and - # coordinator clocks stay real, and all lease budgets stay unchanged. - PRISM_TEMPLATE_MAX_AGE_SECONDS="$(python3 -c 'import time; print(max(120, int(time.time()) - 1738714602 + 120))')" - export PRISM_TEMPLATE_MAX_AGE_SECONDS - fi - export PRISM_STRATUM_SHARE_DIFF=0.000000001 - export PRISM_STOP_AFTER_BLOCK=1 - if [[ "${AUDIT_API_ENABLED}" == "1" ]]; then - export PRISM_AUDIT_BIND=127.0.0.1 - export PRISM_AUDIT_PORT="${AUDIT_PORT}" - export PRISM_STOP_AFTER_BLOCK=0 - export PRISM_MAX_BLOCKS="${QBIT_PRISM_COORDINATOR_MAX_BLOCKS:-99}" - else - export PRISM_MAX_BLOCKS="${TARGET_BLOCKS}" - fi - if [[ "${POSTGRES_ENABLED}" == "1" ]]; then - export PRISM_POSTGRES_PSQL_COMMAND="${PSQL_COMMAND}" - export PRISM_POSTGRES_INIT_SCHEMA=1 - if [[ "${NATIVE_POSTGRES}" == "1" ]]; then - if [[ -z "${EXTERNAL_PSQL}" ]]; then - postgres_port="$(docker port "${POSTGRES_CONTAINER}" 5432/tcp | head -1 | sed 's/.*://')" - export PRISM_DATABASE_URL="postgresql://qbit:qbit@127.0.0.1:${postgres_port}/qbit" - fi - export PRISM_POSTGRES_NATIVE_CLIENT=1 - fi - else - export PRISM_ALLOW_MEMORY_LEDGER=1 - fi - if [[ "${RESTART_REPLAY}" == "1" && ! -f "${CRASH_MARKER}" ]]; then - exec python3 tests/prism_regtest_crash_coordinator.py "${CRASH_MARKER}" - fi - exec python3 lab/prism/prism_coordinator.py -) >>"${COORDINATOR_LOG}" 2>&1 & -coordinator_pid="$!" -} -start_coordinator - -STRATUM_PORT="${STRATUM_PORT}" python3 <<'PY' -import os -import socket -import time - -deadline = time.time() + 30 -port = int(os.environ["STRATUM_PORT"]) -while time.time() < deadline: - try: - with socket.create_connection(("127.0.0.1", port), timeout=0.5): - raise SystemExit(0) - except OSError: - time.sleep(0.25) -raise SystemExit("timed out waiting for PRISM Stratum port") -PY -if ! kill -0 "${coordinator_pid}" >/dev/null 2>&1; then - echo "PRISM coordinator exited before opening Stratum port" >&2 - cat "${COORDINATOR_LOG}" >&2 || true - exit 1 -fi - -STRATUM_PORT="${HIGHDIFF_PORT}" python3 <<'PY' -import os -import socket -import time - -deadline = time.time() + 30 -port = int(os.environ["STRATUM_PORT"]) -while time.time() < deadline: - try: - with socket.create_connection(("127.0.0.1", port), timeout=0.5): - raise SystemExit(0) - except OSError: - time.sleep(0.25) -raise SystemExit("timed out waiting for PRISM high-diff Stratum port") -PY -if ! kill -0 "${coordinator_pid}" >/dev/null 2>&1; then - echo "PRISM coordinator exited before opening high-diff Stratum port" >&2 - cat "${COORDINATOR_LOG}" >&2 || true - exit 1 -fi - -# Handshake-only difficulty probes: the first mining.set_difficulty must be the -# listener's start difficulty (the high-diff floor is what rental marketplaces -# verify), and a password d= request must override it within listener bounds. -probe_first_difficulty() { - PROBE_PORT="$1" PROBE_USERNAME="$2" PROBE_PASSWORD="$3" PROBE_EXPECTED_DIFF="$4" python3 <<'PY' -import json -import os -import socket - -port = int(os.environ["PROBE_PORT"]) -username = os.environ["PROBE_USERNAME"] -password = os.environ["PROBE_PASSWORD"] -expected = float(os.environ["PROBE_EXPECTED_DIFF"]) - -with socket.create_connection(("127.0.0.1", port), timeout=15) as sock: - stream = sock.makefile("rw", encoding="utf-8", newline="\n") - - def send(payload): - stream.write(json.dumps(payload) + "\n") - stream.flush() - - send({"id": 1, "method": "mining.subscribe", "params": ["difficulty-probe/1.0"]}) - send({"id": 2, "method": "mining.authorize", "params": [username, password]}) - difficulty = None - saw_notify_before_difficulty = False - for _ in range(50): - line = stream.readline() - if not line: - break - message = json.loads(line) - if message.get("method") == "mining.set_difficulty": - difficulty = float(message["params"][0]) - break - if message.get("method") == "mining.notify": - saw_notify_before_difficulty = True - -if saw_notify_before_difficulty: - raise SystemExit(f"port {port}: mining.notify arrived before any mining.set_difficulty") -if difficulty is None: - raise SystemExit(f"port {port}: no mining.set_difficulty received") -if abs(difficulty - expected) > expected * 1e-9: - raise SystemExit(f"port {port}: first difficulty {difficulty} != expected {expected}") -print(f"difficulty probe PASS port={port} password={password!r} difficulty={difficulty}") -PY -} - -# The stamped job floor clamps the advertised share difficulty to the network -# difficulty (a share is never required to be harder than a block), while an -# explicit md= minimum and the high-diff wire floor still hold above it. On a -# fresh regtest chain the network difficulty sits below every configured -# value, so the first probe expects the network clamp. -network_difficulty="$(qbit_rpc getblocktemplate '{"rules":["segwit"]}' | python3 -c ' -import json, sys -bits = int(json.load(sys.stdin)["bits"], 16) -size, mantissa = bits >> 24, bits & 0x7fffff -target = mantissa << (8 * (size - 3)) if size > 3 else mantissa >> (8 * (3 - size)) -print(float(0xffff * 2**208 / target)) -')" -start_difficulty="${network_difficulty}" -if [[ "${PRIMED_WINDOW}" == "1" ]]; then - start_difficulty=0.000000001 -fi -probe_first_difficulty "${STRATUM_PORT}" "${miner_usernames[0]}" "x" "${start_difficulty}" -probe_first_difficulty "${HIGHDIFF_PORT}" "${miner_usernames[0]}" "x" "${HIGHDIFF_START_DIFF}" -# Password d=/md= bound vardiff but do not lift the first advertised job -# above the network clamp; only the high-diff listener floor is a wire -# guarantee that holds above it. -probe_first_difficulty "${STRATUM_PORT}" "${miner_usernames[$MINER_COUNT]}" "d=0.5,md=0.25" "${network_difficulty}" - -if [[ "${AUDIT_API_ENABLED}" == "1" ]]; then - AUDIT_PORT="${AUDIT_PORT}" python3 <<'PY' -import os -import socket -import time - -deadline = time.time() + 30 -port = int(os.environ["AUDIT_PORT"]) -while time.time() < deadline: - try: - with socket.create_connection(("127.0.0.1", port), timeout=0.5): - raise SystemExit(0) - except OSError: - time.sleep(0.25) -raise SystemExit("timed out waiting for PRISM audit API port") -PY - if ! kill -0 "${coordinator_pid}" >/dev/null 2>&1; then - echo "PRISM coordinator exited before opening audit API port" >&2 - cat "${COORDINATOR_LOG}" >&2 || true - exit 1 - fi -fi - -seed_miner_script=lab/miner-sim/miner_sim.py -if [[ "${PRIMED_WINDOW}" == "1" ]]; then - seed_miner_script=tests/prism_regtest_share_miner.py -fi - -for miner_index in $(seq 1 "${MINER_COUNT}"); do - miner_username="${miner_usernames[$((miner_index - 1))]}" - ( - cd "${ROOT_DIR}" - STRATUM_HOST=127.0.0.1 \ - STRATUM_PORT="${STRATUM_PORT}" \ - MINER_USERNAME="${miner_username}" \ - MINER_WALLET_NAME="prism-live-miner-${miner_index}" \ - MINER_PASSWORD=x \ - MINER_TIMEOUT_SECONDS="${MINER_TIMEOUT_SECONDS}" \ - MINER_USERNAME_TIMEOUT_SECONDS=30 \ - QBIT_RPC_HOST=127.0.0.1 \ - QBIT_RPC_PORT="${RPC_PORT}" \ - QBIT_RPC_USER="${RPC_USER}" \ - QBIT_RPC_PASSWORD="${RPC_PASSWORD}" \ - python3 "${seed_miner_script}" - ) >"${DATADIR}/miner-${miner_index}.log" 2>&1 & - miner_pids+=("$!") -done - -deadline=$((SECONDS + 30)) -while [[ ! -f "${EVIDENCE_PATH}" && "${SECONDS}" -lt "${deadline}" ]]; do - accepted_count="$(grep -c 'prism coordinator: accepted share' "${COORDINATOR_LOG}" 2>/dev/null || true)" - if [[ "${accepted_count}" -ge "${MINER_COUNT}" ]]; then - miner_username="${miner_usernames[0]}" - ( - cd "${ROOT_DIR}" - STRATUM_HOST=127.0.0.1 \ - STRATUM_PORT="${STRATUM_PORT}" \ - MINER_USERNAME="${miner_username}" \ - MINER_WALLET_NAME="prism-live-miner-block" \ - MINER_PASSWORD=x \ - MINER_TIMEOUT_SECONDS="${MINER_TIMEOUT_SECONDS}" \ - MINER_USERNAME_TIMEOUT_SECONDS=30 \ - QBIT_RPC_HOST=127.0.0.1 \ - QBIT_RPC_PORT="${RPC_PORT}" \ - QBIT_RPC_USER="${RPC_USER}" \ - QBIT_RPC_PASSWORD="${RPC_PASSWORD}" \ - python3 lab/miner-sim/miner_sim.py - ) >"${DATADIR}/miner-block.log" 2>&1 & - miner_pids+=("$!") - break - fi - if ! kill -0 "${coordinator_pid}" >/dev/null 2>&1; then - echo "PRISM coordinator exited before accepting ready shares" >&2 - cat "${COORDINATOR_LOG}" >&2 || true - exit 1 - fi - sleep 1 -done - -if [[ "${RESTART_REPLAY}" == "1" ]]; then - crash_deadline=$((SECONDS + MINER_TIMEOUT_SECONDS)) - while kill -0 "${coordinator_pid}" >/dev/null 2>&1; do - if [[ "${SECONDS}" -ge "${crash_deadline}" ]]; then - echo "candidate crash fixture did not reach the durable publication boundary" >&2 - cat "${COORDINATOR_LOG}" >&2 - exit 1 - fi - sleep 0.1 - done - crash_status=0 - wait "${coordinator_pid}" || crash_status=$? - coordinator_pid="" - if [[ "${crash_status}" != "75" || ! -f "${CRASH_MARKER}" ]]; then - echo "unexpected coordinator exit in crash fixture: ${crash_status}" >&2 - cat "${COORDINATOR_LOG}" >&2 - exit 1 - fi - # No miners remain to resubmit or solve a replacement. The successor must - # discover and finish exactly the candidate committed by its predecessor. - for pid in "${miner_pids[@]}"; do - kill "${pid}" >/dev/null 2>&1 || true - wait "${pid}" >/dev/null 2>&1 || true - done - miner_pids=() - start_coordinator -fi - -deadline=$((SECONDS + MINER_TIMEOUT_SECONDS)) -while [[ ! -f "${EVIDENCE_PATH}" && "${SECONDS}" -lt "${deadline}" ]]; do - if ! kill -0 "${coordinator_pid}" >/dev/null 2>&1; then - echo "PRISM coordinator exited before writing evidence" >&2 - cat "${COORDINATOR_LOG}" >&2 || true - exit 1 - fi - sleep 1 -done - -if [[ ! -f "${EVIDENCE_PATH}" ]]; then - echo "timed out waiting for live PRISM evidence" >&2 - cat "${COORDINATOR_LOG}" >&2 || true - for miner_index in $(seq 1 "${MINER_COUNT}"); do - echo "--- miner ${miner_index} ---" >&2 - cat "${DATADIR}/miner-${miner_index}.log" >&2 || true - done - echo "--- miner block ---" >&2 - cat "${DATADIR}/miner-block.log" >&2 || true - exit 1 -fi - -for pid in "${miner_pids[@]}"; do - wait "${pid}" || { - echo "a live PRISM miner exited unsuccessfully" >&2 - cat "${COORDINATOR_LOG}" >&2 || true - for miner_index in $(seq 1 "${MINER_COUNT}"); do - echo "--- miner ${miner_index} ---" >&2 - cat "${DATADIR}/miner-${miner_index}.log" >&2 || true - done - echo "--- miner block ---" >&2 - cat "${DATADIR}/miner-block.log" >&2 || true - exit 1 - } -done - -if [[ "${AUDIT_API_ENABLED}" == "1" ]]; then - RESTART_REPLAY="${RESTART_REPLAY}" CRASH_MARKER="${CRASH_MARKER}" \ - EVIDENCE_PATH="${EVIDENCE_PATH}" \ - DATADIR="${DATADIR}" \ - AUDIT_PORT="${AUDIT_PORT}" \ - POSTGRES_ENABLED="${POSTGRES_ENABLED}" \ - POSTGRES_CONTAINER="${POSTGRES_CONTAINER}" \ - PRISM_PSQL_COMMAND="${PSQL_COMMAND}" \ - python3 <<'PY' -import json -import os -import subprocess -import urllib.request -from pathlib import Path - -base_url = f"http://127.0.0.1:{os.environ['AUDIT_PORT']}" -evidence_path = Path(os.environ["EVIDENCE_PATH"]) -datadir = Path(os.environ["DATADIR"]) -evidence = json.loads(evidence_path.read_text(encoding="utf-8")) - -def local_audit_payload(path): - payload = json.loads(Path(path).read_text(encoding="utf-8")) - if payload.get("schema") == "qbit.prism.live-audit-bundle-envelope.v1": - return local_audit_payload(payload["body_uri"]) - if payload.get("schema") == "qbit.prism.audit-body-ref.v1": - return payload["bundle_without_shares"] - if payload.get("schema") == "qbit.prism.audit-bundle.v2": - return payload["bundle_without_shares"] - return payload - -bundle = local_audit_payload(evidence["audit_bundle_path"]) - -def get_json(path): - with urllib.request.urlopen(base_url + path, timeout=10) as response: - return json.loads(response.read()) - -def get_text(path): - with urllib.request.urlopen(base_url + path, timeout=10) as response: - return response.read().decode() - -latest = get_json("/audit/latest") -if latest["block_hash"] != evidence["block_hash"]: - raise SystemExit("audit API latest evidence block hash mismatch") - -owed = get_json("/owed-balances") -if owed["schema"] != "qbit.prism.owed-balances.v1": - raise SystemExit("unexpected owed-balances schema") - -import time -expected_metrics = ( - f"qbit_prism_accepted_shares_total {evidence['accepted_share_count']}", - "qbit_prism_blocks_accepted_total 1", -) -metrics_deadline = time.monotonic() + 20 -while True: - metrics = get_text("/metrics") - if all(expected in metrics for expected in expected_metrics): - break - if time.monotonic() >= metrics_deadline: - raise SystemExit(f"metrics cache did not publish expected counters: {expected_metrics}") - time.sleep(0.25) - -if os.environ["POSTGRES_ENABLED"] == "1": - import shlex - - psql_base = shlex.split(os.environ["PRISM_PSQL_COMMAND"]) - - def psql_json(sql): - raw = subprocess.check_output( - psql_base - + [ - "--no-psqlrc", - "--set", - "ON_ERROR_STOP=1", - "--tuples-only", - "--no-align", - "--quiet", - "--command", - sql, - ], - text=True, - ).strip() - return json.loads(raw.splitlines()[-1]) - - if os.environ.get("RESTART_REPLAY") == "1": - with open(os.environ["CRASH_MARKER"], encoding="utf-8") as handle: - crashed = json.load(handle) - if evidence["block_hash"] != crashed["block_hash"]: - raise SystemExit("restart finalized a different candidate") - replayed = psql_json(""" - SELECT json_build_object( - 'state', o.state, 'digest', o.candidate_sha256, - 'share_id', o.share_id, 'share_count', - (SELECT count(*) FROM qbit_share_ledger s WHERE s.share_id = o.share_id), - 'storage_version', COALESCE((to_jsonb(o)->>'storage_version')::integer, 1), - 'body_id', to_jsonb(o)->>'body_id' - ) FROM qbit_block_candidate_outbox o - WHERE o.block_hash = '%s' - """ % crashed["block_hash"]) - if replayed != {"state": "submitted", "digest": crashed["candidate_sha256"], - "share_id": crashed["share_id"], "share_count": 1, - "storage_version": crashed["storage_version"], "body_id": None}: - raise SystemExit(f"restart candidate identity or atomicity mismatch: {replayed}") - print("native candidate crash/restart replay PASS", flush=True) - - counts = psql_json( - """ - SELECT json_build_object( - 'shares', (SELECT count(*) FROM qbit_share_ledger WHERE accepted), - 'contiguous', ( - SELECT COALESCE(bool_and(share_seq = expected_seq), true) - FROM ( - SELECT share_seq, row_number() OVER (ORDER BY share_seq) AS expected_seq - FROM qbit_share_ledger - WHERE accepted - ) ordered - ), - 'blocks', (SELECT count(*) FROM qbit_pool_blocks), - 'payout_entries', (SELECT count(*) FROM qbit_pool_payout_entries), - 'bundles', (SELECT count(*) FROM qbit_pool_audit_bundles), - 'writer_rows', ( - SELECT count(*) - FROM qbit_share_ledger - WHERE writer_id = 'prism-coordinator' AND writer_epoch = 1 - ) - ); - """ - ) - if counts["shares"] != evidence["accepted_share_count"]: - raise SystemExit(f"persisted share count mismatch: {counts['shares']} != {evidence['accepted_share_count']}") - if counts["contiguous"] is not True: - raise SystemExit("persisted share_seq values are not contiguous") - if counts["blocks"] != 1 or counts["bundles"] != 1: - raise SystemExit(f"expected one persisted block and bundle, got {counts}") - if counts["writer_rows"] != evidence["accepted_share_count"]: - raise SystemExit("persisted shares do not all carry the expected writer identity") - - expected_payout_entries = len(bundle["payout_policy_manifest"]["accounts"]) - if counts["payout_entries"] != expected_payout_entries: - raise SystemExit(f"payout row count mismatch: {counts['payout_entries']} != {expected_payout_entries}") - - found = bundle["found_block"] - sql_window = psql_json( - f""" - SELECT COALESCE(json_agg(json_build_object( - 'share_seq', share_seq, - 'share_id', share_id, - 'miner_id', miner_id, - 'order_key', payout_order_key, - 'p2mr_program_hex', encode(p2mr_program, 'hex'), - 'share_difficulty', share_difficulty::text, - 'counted_difficulty', counted_difficulty::text - ) ORDER BY share_seq DESC), '[]'::json) - FROM qbit_audit_share_window( - to_timestamp({int(found['anchor_job_issued_at_ms'])}::double precision / 1000.0), - {int(found['network_difficulty'])}::numeric - ); - """ - ) - expected_window = [ - { - "share_seq": share["share_seq"], - "share_id": share["share_id"], - "miner_id": share["miner_id"], - "order_key": share["order_key"], - "p2mr_program_hex": share["p2mr_program_hex"], - "share_difficulty": str(share["share_difficulty"]), - "counted_difficulty": str(share["counted_difficulty"]), - } - for share in bundle["reward_manifest"]["shares"] - ] - if sql_window != expected_window: - raise SystemExit("persisted audit share window does not match submitted bundle") - - api_window = get_json( - "/audit/share-window" - f"?anchor_job_issued_at_ms={int(found['anchor_job_issued_at_ms'])}" - f"&network_difficulty={int(found['network_difficulty'])}" - ) - if [row["share_id"] for row in api_window["rows"]] != [row["share_id"] for row in bundle["reward_manifest"]["shares"]]: - raise SystemExit("audit API share-window rows do not match submitted bundle") - expected_api_window = [ - { - "share_seq": share["share_seq"], - "share_id": share["share_id"], - "miner_id": share["miner_id"], - "order_key": share["order_key"], - "p2mr_program_hex": share["p2mr_program_hex"], - "share_difficulty": share["share_difficulty"], - "counted_difficulty": share["counted_difficulty"], - } - for share in bundle["reward_manifest"]["shares"] - ] - actual_api_window = [ - { - "share_seq": row["share_seq"], - "share_id": row["share_id"], - "miner_id": row["miner_id"], - "order_key": row["order_key"], - "p2mr_program_hex": row["p2mr_program_hex"], - "share_difficulty": row["share_difficulty"], - "counted_difficulty": row["counted_difficulty"], - } - for row in api_window["rows"] - ] - if actual_api_window != expected_api_window: - raise SystemExit("audit API share-window row contents do not match submitted bundle") - - payouts = get_json(f"/audit/blocks/{evidence['block_hash']}/payouts") - if len(payouts["rows"]) != expected_payout_entries: - raise SystemExit("audit API payout row count does not match submitted bundle") - expected_payouts = [ - { - "miner_id": account["recipient_id"], - "order_key": account["order_key"], - "p2mr_program_hex": account["p2mr_program_hex"], - "onchain_amount_sats": account["onchain_amount_sats"], - "carry_forward_balance_sats": account["carry_forward_balance_sats"], - "action": account["action"], - "maturity_state": "immature", - } - for account in bundle["payout_policy_manifest"]["accounts"] - ] - actual_payouts = [ - { - "miner_id": row["miner_id"], - "order_key": row["order_key"], - "p2mr_program_hex": row["p2mr_program_hex"], - "onchain_amount_sats": row["onchain_amount_sats"], - "carry_forward_balance_sats": row["carry_forward_balance_sats"], - "action": row["action"], - "maturity_state": row["maturity_state"], - } - for row in payouts["rows"] - ] - if actual_payouts != expected_payouts: - raise SystemExit("audit API payout rows do not match submitted payout manifest") - - api_bundle = get_json(f"/audit/blocks/{evidence['block_hash']}/bundle") - if api_bundle["audit_bundle_sha256"] != evidence["audit_report"]["audit_bundle_sha256_hex"]: - raise SystemExit("audit API bundle digest does not match verifier report") - api_bundle_path = datadir / "prism-api-audit-bundle.json" - api_bundle_path.write_text(json.dumps(api_bundle["audit_bundle"], indent=2), encoding="utf-8") -PY - - if [[ "${POSTGRES_ENABLED}" == "1" ]]; then - run_cargo run --quiet -p qbit-prism --bin qbit-prism-audit-verify -- \ - "${DATADIR}/prism-api-audit-bundle.json" \ - --coinbase-tx-hex "$(EVIDENCE_PATH="${EVIDENCE_PATH}" python3 -c 'import json,os; print(json.load(open(os.environ["EVIDENCE_PATH"], encoding="utf-8"))["coinbase_tx_hex"])')" \ - --ledger-writer-public-key-hex "$(DATADIR="${DATADIR}" python3 -c 'import json,os; print(json.load(open(os.path.join(os.environ["DATADIR"], "prism-api-audit-bundle.json"), encoding="utf-8"))["ledger_window_attestation"]["signature"]["public_key_hex"])')" \ - --expected-coinbase-value-sats "$(DATADIR="${DATADIR}" python3 -c 'import json,os; print(json.load(open(os.path.join(os.environ["DATADIR"], "prism-api-audit-bundle.json"), encoding="utf-8"))["found_block"]["coinbase_value_sats"])')" >/dev/null - fi - - kill "${coordinator_pid}" >/dev/null 2>&1 || true - wait "${coordinator_pid}" >/dev/null 2>&1 || true - coordinator_pid="" -else - wait "${coordinator_pid}" || { - echo "PRISM coordinator exited unsuccessfully" >&2 - cat "${COORDINATOR_LOG}" >&2 || true - exit 1 - } - coordinator_pid="" -fi - -after_height="$(qbit_rpc getblockcount)" -# With the advertised difficulty clamped to the network difficulty, every -# accepted share is also a block, so a share already in flight when the -# coordinator begins its stop-after-block shutdown can land one extra block. -# Require at least the target; never fewer. -if [[ "${after_height}" -lt $((before_height + 1)) ]]; then - echo "expected live PRISM blockcount >= ${before_height}+1, got ${after_height}" >&2 - exit 1 -fi - -EVIDENCE_PATH="${EVIDENCE_PATH}" MINER_COUNT="${MINER_COUNT}" python3 <<'PY' -import json -import os - -evidence = json.load(open(os.environ["EVIDENCE_PATH"], encoding="utf-8")) -miner_count = int(os.environ["MINER_COUNT"]) -if evidence["schema"] != "qbit.prism.live-stratum-evidence.v1": - raise SystemExit("unexpected live evidence schema") -if evidence["distinct_miner_count"] < miner_count: - raise SystemExit(f"expected >= {miner_count} distinct miners, got {evidence['distinct_miner_count']}") -if evidence["accepted_share_count"] < miner_count: - raise SystemExit("live ledger did not record enough accepted shares") -if evidence["job_share_count"] < miner_count: - raise SystemExit("submitted block did not use a 3+ miner PRISM share snapshot") -if evidence["audit_report"]["schema"] != "qbit.prism.audit-verification-report.v1": - raise SystemExit("unexpected audit report schema") -if evidence["audit_report"]["onchain_output_count"] < 1: - raise SystemExit("live audit report has no on-chain outputs") -PY - -block_hash="$(qbit_rpc getblockhash "${after_height}")" -raw_block="$(qbit_rpc getblock "${block_hash}" 0)" -RAW_BLOCK="${raw_block}" \ -EVIDENCE_PATH="${EVIDENCE_PATH}" \ -QBIT_SRC_DIR="${QBIT_SRC_DIR}" \ -python3 <<'PY' -import io -import json -import os -import sys -from pathlib import Path - -qbit_src = Path(os.environ["QBIT_SRC_DIR"]) -sys.path.insert(0, str(qbit_src / "test/functional")) - -from test_framework.messages import CBlock - -evidence = json.load(open(os.environ["EVIDENCE_PATH"], encoding="utf-8")) -block = CBlock() -block.deserialize(io.BytesIO(bytes.fromhex(os.environ["RAW_BLOCK"]))) -coinbase_hex = block.vtx[0].serialize_with_witness().hex() -if coinbase_hex != evidence["coinbase_tx_hex"]: - raise SystemExit("on-chain live coinbase bytes do not match coordinator evidence") -PY - -printf 'prism live stratum regtest PASS height=%s block=%s evidence=%s\n' \ - "${after_height}" "${block_hash}" "${EVIDENCE_PATH}" diff --git a/tests/fixtures/fake_serve_builder.py b/tests/fixtures/fake_serve_builder.py deleted file mode 100644 index f3163e65f..000000000 --- a/tests/fixtures/fake_serve_builder.py +++ /dev/null @@ -1,405 +0,0 @@ -#!/usr/bin/env python3 -"""Deterministic stand-in for qbit-prism-build-audit-bundle in daemon tests. - -Invoked through a patched prism_tool_command: with --serve it speaks the -JSONL daemon protocol (echoing enough request state for assertions), and -without it it echoes the one-shot stdin payload, so the same command list -serves both the daemon path and its transparent one-shot fallback. -FAKE_SERVE_BUILDER_MODE selects daemon misbehavior for anomaly tests. - -prepare_window requests are served by the real Python fold -(``IncrementalShareWindow``), so mirror digests verify exactly and the -window-pipeline tests exercise the coordinator against byte-faithful daemon -behavior without a Rust build; the Rust implementation itself is proven by -the parity oracle's ``rust-daemon`` adapter. Two more real-daemon contracts -are mirrored so the coordinator sees them without a Rust build: the -declared integer widths (a prepare request carrying a value outside -``RUST_INTEGER_DOMAIN`` is answered ``out_of_range``, never folded and -never treated as malformed), and the stable ``rejection`` category beside -every ``fold_invalid``/``fallback`` error. Additional modes fault-inject -the prepare protocol: ``prepare-forget-windows`` answers every advance with -``needs_full``, ``prepare-fallback`` answers every advance with ``fallback``, -``prepare-error`` answers prepare requests with a generic error, -``crash-during-prepare`` dies on the first prepare request, and -``prepare-slow`` sleeps FAKE_SERVE_BUILDER_PREPARE_DELAY_SECONDS before every -prepare response, so a test can make the daemon exchange itself outlast a -budget that was consumed waiting for the daemon's lock. ``prepare-miscount`` -answers with byte-exact windows whose declared record_count is one too many. -""" - -import json -import os -import sys -import time -from pathlib import Path - - -def one_shot() -> None: - payload = json.load(sys.stdin) - json.dump({"received": payload, "transport": "one-shot"}, sys.stdout) - sys.stdout.flush() - - -def _fold_windows(): - """Import the shipped fold lazily; the repo root is not on script paths.""" - repo_root = Path(__file__).resolve().parents[2] - if str(repo_root) not in sys.path: - sys.path.insert(0, str(repo_root)) - from lab.prism.share_ledger import ( # noqa: PLC0415 - script-local import - AcceptedShareRecord, - IncrementalShareWindow, - IncrementalWindowFallback, - ) - - return AcceptedShareRecord, IncrementalShareWindow, IncrementalWindowFallback - - -def _declared_domain(): - """The real daemon's declared widths and rejection vocabulary, lazily. - - Taken from the parity oracle so the fake's ``out_of_range`` and - ``rejection`` answers are classified by the same tables the harness - holds the Rust daemon to, rather than by a third copy. - """ - repo_root = Path(__file__).resolve().parents[2] - if str(repo_root) not in sys.path: - sys.path.insert(0, str(repo_root)) - from tests.window_pipeline_parity import ( # noqa: PLC0415 - script-local import - RUST_INTEGER_DOMAIN, - rejection_category_for_message, - ) - - return RUST_INTEGER_DOMAIN, rejection_category_for_message - - -def _out_of_range(request: dict, domain) -> tuple[str, str] | None: - """The first prepare_window integer outside the declared widths. - - Mirrors the real daemon's classifier: the top-level fields first, then - each record's integer fields in declaration order; returns the field - path and the width it missed. - """ - checks = [ - ("window_weight", request.get("window_weight"), domain.window_weight), - ("page_size", request.get("page_size"), domain.page_size), - ( - "anchor_job_issued_at_ms", - request.get("anchor_job_issued_at_ms"), - domain.timestamps_ms, - ), - ] - for index, record in enumerate(request.get("records", [])): - for name in ( - "share_seq", - "share_difficulty", - "network_difficulty", - "template_height", - "job_issued_at_ms", - "accepted_at_ms", - "ntime", - ): - checks.append( - ( - f"records[{index}].{name}", - record.get(name), - domain.record_field_width(name), - ) - ) - for field, value, width in checks: - if isinstance(value, int) and not width.admits(value): - return field, width.name - return None - - -def _write_out_of_range(field: str, width: str) -> None: - _write_response( - { - "ok": False, - "request": "prepare_window", - "out_of_range": True, - "field": field, - "width": width, - "error": f"prepare_window {field} is outside the declared {width}", - } - ) - - -def _write_response(payload: dict, raw_section: bytes | None = None) -> None: - sys.stdout.write(json.dumps(payload) + "\n") - sys.stdout.flush() - if raw_section is not None: - sys.stdout.buffer.write(raw_section + b"\n") - sys.stdout.buffer.flush() - - -def _canonical_items(window) -> bytes: - return b",".join( - page.canonical_json_items - for page in window.pages - if page.canonical_json_items - ) - - -def _handle_prepare_window(request: dict, prepared: dict, mode: str) -> None: - if mode == "crash-during-prepare": - sys.exit(1) - if mode == "prepare-slow": - time.sleep( - float(os.environ.get("FAKE_SERVE_BUILDER_PREPARE_DELAY_SECONDS", "0")) - ) - if mode == "prepare-error": - _write_response( - { - "ok": False, - "request": "prepare_window", - "error": "injected prepare failure", - } - ) - return - miscount = 1 if mode == "prepare-miscount" else 0 - domain, category_for_message = _declared_domain() - out_of_range = _out_of_range(request, domain) - if out_of_range is not None: - # Like the real daemon: declined before anything is folded, state - # untouched, and the coordinator folds this window in-process. - _write_out_of_range(*out_of_range) - return - record_type, window_type, fallback_type = _fold_windows() - records = [ - record_type( - **{ - key: value - for key, value in record_json.items() - if value is not None or key == "credit_policy" - } - ) - for record_json in request.get("records", []) - ] - epoch = int(request.get("append_invalidation_epoch", 0)) - anchor = int(request["anchor_job_issued_at_ms"]) - prepare_mode = request.get("mode") - if prepare_mode == "full": - try: - window = window_type.from_full_snapshot( - records, - anchor_job_issued_at_ms=anchor, - window_weight=int(request["window_weight"]), - page_size=int(request.get("page_size", 512)), - ) - except ValueError as error: - _write_response( - { - "ok": False, - "request": "prepare_window", - "fold_invalid": True, - "rejection": category_for_message(str(error)), - "error": str(error), - } - ) - return - digest = window.json_records().canonical_json_sha256() - items = _canonical_items(window) - prepared.clear() - prepared[digest] = (window, epoch) - _write_response( - { - "ok": True, - "request": "prepare_window", - "share_snapshot_sha256": digest, - "record_count": window.record_count + miscount, - "added_rows": 0, - "expired_rows": 0, - "touched_pages": 0, - "window_items_len": len(items), - }, - items, - ) - return - if prepare_mode != "advance": - _write_response( - { - "ok": False, - "request": "prepare_window", - "error": f"unsupported prepare_window mode: {prepare_mode}", - } - ) - return - base_digest = str(request.get("base_digest", "")) - held = prepared.get(base_digest) - # Addressed by digest alone, like the real daemon: the epoch recorded - # beside the window is diagnostic, never an eviction rule, so a base the - # coordinator retagged still advances in place. - if mode == "prepare-forget-windows" or held is None: - _write_response( - { - "ok": False, - "request": "prepare_window", - "needs_full": True, - "error": f"prepared window {base_digest} is not held", - } - ) - return - if mode == "prepare-fallback": - _write_response( - { - "ok": False, - "request": "prepare_window", - "fallback": True, - "rejection": "delta_not_append", - "error": "injected advance invariant failure", - } - ) - return - base_window = held[0] - old_items = _canonical_items(base_window) - try: - advanced, stats = base_window.advance( - records, - anchor_job_issued_at_ms=anchor, - ) - except fallback_type as error: - _write_response( - { - "ok": False, - "request": "prepare_window", - "fallback": True, - "rejection": category_for_message(str(error)), - "error": str(error), - } - ) - return - base_epoch = held[1] - digest = advanced.json_records().canonical_json_sha256() - new_items = _canonical_items(advanced) - # Byte surgery from the old stream to the new: expiry only drops a - # prefix and appends only extend the tail, so the retained old suffix is - # a prefix of the new stream. Scanning for the smallest such drop keeps - # the fake honest against the fold it just ran. - drop = 0 - while drop <= len(old_items): - retained = old_items[drop:] - if new_items[: len(retained)] == retained: - break - drop += 1 - appended = new_items[len(old_items) - drop :] - prepared.clear() - prepared[digest] = (advanced, epoch) - _write_response( - { - "ok": True, - "request": "prepare_window", - "share_snapshot_sha256": digest, - "record_count": advanced.record_count + miscount, - "added_rows": stats.added_rows, - "expired_rows": stats.expired_rows, - "touched_pages": stats.touched_pages, - "retained_drop_bytes": drop, - "appended_items_len": len(appended), - "base_append_invalidation_epoch": base_epoch, - "append_invalidation_epoch": epoch, - }, - appended, - ) - - -def serve() -> None: - mode = os.environ.get("FAKE_SERVE_BUILDER_MODE", "ok") - protocol = int(os.environ.get("FAKE_SERVE_BUILDER_PROTOCOL", "2")) - if mode == "protocol-mismatch": - protocol = 99 - sys.stdout.write( - json.dumps( - { - "event": "handshake", - "tool": "qbit-prism-build-audit-bundle", - "protocol": protocol, - } - ) - + "\n" - ) - sys.stdout.flush() - if mode == "crash-before-response": - sys.stdin.readline() - sys.exit(1) - if mode == "hang-after-request": - sys.stdin.readline() - # Block until the coordinator kills the daemon (supersession) or - # closes stdin; never respond. - sys.stdin.read() - sys.exit(0) - cache: dict[str, dict[str, object]] = {} - prepared: dict[str, tuple[object, int]] = {} - hits = 0 - misses = 0 - for line in sys.stdin: - line = line.strip() - if not line: - continue - request = json.loads(line) - if request.get("request") == "prepare_window": - _handle_prepare_window(request, prepared, mode) - continue - key = request["window_key"]["share_snapshot_sha256"] - had_window = bool(request.get("compact_shares")) - if had_window: - cache[key] = { - "compact_share_identities": request.get( - "compact_share_identities", [] - ), - "compact_shares": request["compact_shares"], - } - while len(cache) > 2: - cache.pop(next(iter(cache))) - misses += 1 - elif key in cache: - hits += 1 - # Promote like the real daemon: a hit entry moves to - # most-recent position before any later eviction. - cache[key] = cache.pop(key) - elif key in prepared: - # The unified cache: a prepared window serves builds directly. - hits += 1 - else: - sys.stdout.write( - json.dumps( - { - "ok": False, - "error": f"share window {key} is not cached", - "needs_window": True, - } - ) - + "\n" - ) - sys.stdout.flush() - continue - response = { - "ok": True, - "summary": { - "found_block": request["found_block"], - "signed_coinbase_manifest": {"manifest": {}}, - "payout_policy_manifest": {"accounts": []}, - "window": cache.get(key, {"prepared": True}), - "request_had_window": had_window, - "transport": "serve", - }, - "metrics": { - "input_deserialization_seconds": 0.001, - "phases_seconds": {"payout_state_derivation": 0.002}, - "output_serialization_seconds": 0.003, - }, - "window_cache": { - "hit": not had_window, - "hits": hits, - "misses": misses, - "entries": len(cache) + len(prepared), - }, - } - sys.stdout.write(json.dumps(response) + "\n") - sys.stdout.flush() - - -if __name__ == "__main__": - if "--serve" in sys.argv[1:]: - serve() - else: - one_shot() diff --git a/tests/fixtures/mainnet-compose.env b/tests/fixtures/mainnet-compose.env index c52d2d944..480b059a1 100644 --- a/tests/fixtures/mainnet-compose.env +++ b/tests/fixtures/mainnet-compose.env @@ -14,6 +14,7 @@ PRISM_POSTGRES_DATA_SOURCE=/srv/qbit-mining-bootstrap/mainnet/postgres/data PRISM_POSTGRES_WAL_SOURCE=/srv/qbit-mining-bootstrap/mainnet/postgres/wal PRISM_AUDIT_DATA_SOURCE=/srv/qbit-mining-bootstrap/mainnet/prism/audit PRISM_POSTGRES_REPLICA_DATA_SOURCE=/srv/qbit-mining-bootstrap/mainnet/postgres-replica/data +PRISM_SECRETS_SOURCE=/srv/qbit-mining-bootstrap/mainnet/prism/secrets QBIT_PROVIDER=source QBIT_PRODUCTION=1 @@ -28,6 +29,7 @@ QBIT_LISTEN=1 QBIT_RPC_USER=render-qbit-rpc QBIT_RPC_PASSWORD=synthetic-qbit-password-7fb3a0 QBIT_RPC_PORT=19552 +QBIT_RPC_URL=https://rpc.example.invalid:8443/qbit/mainnet/rpc QBIT_P2P_PORT=19555 QBIT_RPC_PORT_HOST=127.0.0.1:19552 QBIT_P2P_PORT_HOST=0.0.0.0:19555 @@ -81,16 +83,34 @@ PRISM_POSTGRES_USER=render-prism PRISM_POSTGRES_PASSWORD=synthetic-postgres-password-1ae478 PRISM_POSTGRES_DB=render_prism PRISM_DATABASE_URL=postgresql://render-prism:synthetic-postgres-password-1ae478@prism-postgres:5432/render_prism +PRISM_DATABASE_STATEMENT_TIMEOUT_MS=23000 +PRISM_DATABASE_LOCK_TIMEOUT_MS=7000 +PRISM_SHARE_COMMIT_TIMEOUT_SECONDS=19 +PRISM_RPC_TIMEOUT_SECONDS=21 +PRISM_BLOCK_SUBMIT_RPC_TIMEOUT_SECONDS=2 +PRISM_CANDIDATE_ORPHAN_CONFIRMATIONS=17 +PRISM_PUBLIC_READ_STATEMENT_TIMEOUT_SECONDS=9 +PRISM_POSTGRES_READ_CONCURRENCY=3 +PRISM_PUBLIC_AUDIT_REBUILD_CONCURRENCY=2 +PRISM_PUBLIC_AUDIT_ARTIFACT_MAX_IN_FLIGHT=11 +PRISM_PAYOUT_MIN_OUTPUT_BITS=24576 +PRISM_PAYOUT_MIN_OUTPUT_SATS=24576 PRISM_PUBLIC_DATABASE_URL=postgresql://render-prism:synthetic-postgres-password-1ae478@prism-postgres-replica:5432/render_prism PRISM_LEDGER_WRITER_ID=render-writer PRISM_LEDGER_WRITER_EPOCH=1 PRISM_LEDGER_WRITER_SESSION_TOKEN= PRISM_LEDGER_WRITER_PUBLIC_KEY_HEX=cccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccc +# Stale direct seeds prove the production override clears them. PRISM_MANIFEST_SIGNING_SEED_HEX=dddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddd PRISM_LEDGER_ATTESTATION_SIGNING_SEED_HEX=eeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeee PRISM_CTV_SETTLEMENT_ENABLED=1 PRISM_CTV_FANOUT_FEE_MARKET_RATE_BITS_PER_1000_WEIGHT=1000 PRISM_CTV_FANOUT_FEE_PREMIUM_BPS=12000 +# Every settlement mode refuses to start without a pool fee (#525, #535). +# Address only: as the deploy role renders it, with no program beside it. +PRISM_POOL_FEE_ENABLED=1 +PRISM_POOL_FEE_BPS=200 +PRISM_POOL_FEE_ADDRESS=qb1syntheticmainnetpoolfeeaddress PRISM_CTV_BROADCASTER_ENABLED=1 PRISM_CTV_BROADCASTER_WALLET= PRISM_CTV_BROADCASTER_FEE_BITS=0 diff --git a/tests/fixtures/prism-alert-scenarios.json b/tests/fixtures/prism-alert-scenarios.json new file mode 100644 index 000000000..212698f53 --- /dev/null +++ b/tests/fixtures/prism-alert-scenarios.json @@ -0,0 +1,3768 @@ +[ + { + "name": "successful real zero is healthy", + "series": { + "qbit_prism_metrics_snapshot_available{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\"}": 1.0, + "qbit_prism_metrics_snapshot_stale{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\"}": 0.0, + "qbit_prism_collector_available{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\",collector=\"database\"}": 1.0, + "qbit_prism_collector_available{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\",collector=\"process\"}": 1.0, + "qbit_prism_block_candidates_pending{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\"}": 0.0, + "qbit_prism_block_candidate_oldest_pending_seconds{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\"}": 0.0, + "qbit_prism_process_resident_memory_bytes{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\"}": 0.0, + "qbit_prism_stratum_semantic_current_work_ratio{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\"}": 1.0, + "qbit_prism_stratum_oldest_pending_initial_job_seconds{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\"}": 0.0, + "qbit_prism_authorized_clients{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\"}": 1.0, + "qbit_prism_runtime_lag_seconds{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\"}": 0.0, + "qbit_prism_runtime_poll_lag_seconds{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\",task=\"refresh\"}": 0.0, + "qbit_prism_runtime_task_stalled{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\",task=\"refresh\"}": 0.0, + "up{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\"}": 1.0, + "qbit_prism_health_state{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\"}": 1.0, + "qbit_prism_block_candidate_oldest_unacknowledged_seconds{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\"}": 0, + "qbit_prism_block_candidate_oldest_landing_failed_seconds{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\"}": 0 + }, + "expect": { + "PrismBlockCandidateBacklog": 0, + "PrismBlockCandidateMetricsUnavailable": 0, + "PrismResidentMemoryHigh": 0, + "PrismProcessMetricsUnavailable": 0, + "PrismSemanticWorkCoverageLoss": 0, + "PrismWorkMetricsUnavailable": 0, + "PrismRuntimeLagHigh": 0, + "PrismMetricsNoData": 0, + "PrismBlockCandidateStuck": 0, + "PrismBlockCandidateOldestPendingCritical": 0, + "PrismBlockCandidateLandingFailed": 0, + "PrismBlockCandidateLandingFailedCritical": 0 + } + }, + { + "name": "unknown is not healthy", + "series": { + "qbit_prism_metrics_snapshot_available{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\"}": 1.0, + "qbit_prism_metrics_snapshot_stale{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\"}": 0.0, + "qbit_prism_collector_available{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\",collector=\"database\"}": 0.0, + "qbit_prism_collector_available{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\",collector=\"process\"}": 0.0, + "qbit_prism_block_candidates_pending{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\"}": -1.0, + "qbit_prism_block_candidate_oldest_pending_seconds{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\"}": -1.0, + "qbit_prism_process_resident_memory_bytes{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\"}": -1.0, + "qbit_prism_stratum_semantic_current_work_ratio{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\"}": -1.0, + "qbit_prism_stratum_oldest_pending_initial_job_seconds{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\"}": -1.0, + "qbit_prism_authorized_clients{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\"}": 1.0, + "qbit_prism_runtime_lag_seconds{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\"}": 0.0, + "qbit_prism_runtime_poll_lag_seconds{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\",task=\"refresh\"}": 0.0, + "qbit_prism_runtime_task_stalled{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\",task=\"refresh\"}": 0.0, + "up{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\"}": 1.0, + "qbit_prism_health_state{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\"}": 1.0, + "qbit_prism_block_candidate_oldest_unacknowledged_seconds{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\"}": -1, + "qbit_prism_block_candidate_oldest_landing_failed_seconds{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\"}": -1 + }, + "expect": { + "PrismBlockCandidateBacklog": null, + "PrismBlockCandidateMetricsUnavailable": 1, + "PrismResidentMemoryHigh": null, + "PrismProcessMetricsUnavailable": 1, + "PrismSemanticWorkCoverageLoss": null, + "PrismWorkMetricsUnavailable": 2, + "PrismBlockCandidateStuck": null, + "PrismBlockCandidateLandingFailed": null, + "PrismBlockCandidateLandingFailedCritical": null + } + }, + { + "name": "stale body does not fire threshold or mask runtime", + "series": { + "qbit_prism_metrics_snapshot_available{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\"}": 1.0, + "qbit_prism_metrics_snapshot_stale{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\"}": 1.0, + "qbit_prism_collector_available{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\",collector=\"database\"}": 1.0, + "qbit_prism_collector_available{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\",collector=\"process\"}": 1.0, + "qbit_prism_block_candidates_pending{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\"}": 10.0, + "qbit_prism_block_candidate_oldest_pending_seconds{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\"}": 0.0, + "qbit_prism_process_resident_memory_bytes{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\"}": 9000000000.0, + "qbit_prism_stratum_semantic_current_work_ratio{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\"}": 1.0, + "qbit_prism_stratum_oldest_pending_initial_job_seconds{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\"}": 0.0, + "qbit_prism_authorized_clients{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\"}": 1.0, + "qbit_prism_runtime_lag_seconds{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\"}": 0.0, + "qbit_prism_runtime_poll_lag_seconds{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\",task=\"refresh\"}": 3.0, + "qbit_prism_runtime_task_stalled{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\",task=\"refresh\"}": 0.0, + "up{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\"}": 1.0, + "qbit_prism_health_state{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\"}": 1.0 + }, + "expect": { + "PrismBlockCandidateBacklog": null, + "PrismResidentMemoryHigh": null, + "PrismMetricsSnapshotStale": 1, + "PrismRuntimeLagHigh": 1 + } + }, + { + "name": "fresh native pressure", + "series": { + "qbit_prism_metrics_snapshot_available{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\"}": 1.0, + "qbit_prism_metrics_snapshot_stale{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\"}": 0.0, + "qbit_prism_collector_available{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\",collector=\"database\"}": 1.0, + "qbit_prism_collector_available{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\",collector=\"process\"}": 1.0, + "qbit_prism_block_candidates_pending{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\"}": 5.0, + "qbit_prism_block_candidate_oldest_pending_seconds{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\"}": 60.0, + "qbit_prism_process_resident_memory_bytes{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\"}": 9000000000.0, + "qbit_prism_stratum_semantic_current_work_ratio{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\"}": 0.4, + "qbit_prism_stratum_oldest_pending_initial_job_seconds{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\"}": 0.0, + "qbit_prism_authorized_clients{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\"}": 1.0, + "qbit_prism_runtime_lag_seconds{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\"}": 0.0, + "qbit_prism_runtime_poll_lag_seconds{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\",task=\"refresh\"}": 0.0, + "qbit_prism_runtime_task_stalled{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\",task=\"refresh\"}": 0.0, + "up{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\"}": 1.0, + "qbit_prism_health_state{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\"}": 1.0, + "qbit_prism_block_candidate_oldest_unacknowledged_seconds{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\"}": 60, + "qbit_prism_block_candidate_oldest_landing_failed_seconds{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\"}": 0 + }, + "expect": { + "PrismBlockCandidateBacklog": 1, + "PrismBlockCandidateBacklogCritical": 1, + "PrismBlockCandidateOldestPendingCritical": 1, + "PrismResidentMemoryHigh": 1, + "PrismSemanticWorkCoverageLoss": 1, + "PrismSemanticWorkCoverageLossCritical": 1, + "PrismBlockCandidateStuck": 0, + "PrismBlockCandidateLandingFailedCritical": 0 + } + }, + { + "name": "candidate lost race in reconciliation never pages", + "series": { + "qbit_prism_block_candidate_oldest_pending_seconds{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\"}": "0+1x899 0x300", + "qbit_prism_block_candidate_oldest_unacknowledged_seconds{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\"}": "0x1200", + "qbit_prism_block_candidates_pending{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\"}": "1x1200", + "qbit_prism_metrics_snapshot_available{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\"}": "1x1200", + "qbit_prism_metrics_snapshot_stale{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\"}": "0x1200", + "qbit_prism_collector_available{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\",collector=\"database\"}": "1x1200", + "up{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\"}": "1x1200", + "qbit_prism_block_candidate_oldest_landing_failed_seconds{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\"}": "0x1200" + }, + "eval_time": "14m59s", + "expect": { + "PrismBlockCandidateOldestPendingCritical": 0, + "PrismBlockCandidateStuck": 0, + "PrismBlockCandidateOldestPending": 1, + "PrismBlockCandidateLandingFailedCritical": 0 + }, + "alerts": { + "PrismBlockCandidateOldestPendingCritical": [], + "PrismBlockCandidateStuck": [], + "PrismBlockCandidateOldestPending": [ + { + "job": "qbit-prism", + "instance": "one", + "network": "mainnet" + } + ], + "PrismBlockCandidateLandingFailedCritical": [] + } + }, + { + "name": "candidate lost race orphan proof clears the backlog", + "series": { + "qbit_prism_block_candidate_oldest_pending_seconds{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\"}": "0+1x899 0x300", + "qbit_prism_block_candidate_oldest_unacknowledged_seconds{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\"}": "0x1200", + "qbit_prism_block_candidates_pending{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\"}": "1x1200", + "qbit_prism_metrics_snapshot_available{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\"}": "1x1200", + "qbit_prism_metrics_snapshot_stale{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\"}": "0x1200", + "qbit_prism_collector_available{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\",collector=\"database\"}": "1x1200", + "up{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\"}": "1x1200", + "qbit_prism_block_candidate_oldest_landing_failed_seconds{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\"}": "0x1200" + }, + "eval_time": "20m", + "expect": { + "PrismBlockCandidateOldestPendingCritical": 0, + "PrismBlockCandidateStuck": 0, + "PrismBlockCandidateOldestPending": 0, + "PrismBlockCandidateLandingFailedCritical": 0 + }, + "alerts": { + "PrismBlockCandidateOldestPendingCritical": [], + "PrismBlockCandidateStuck": [], + "PrismBlockCandidateOldestPending": [], + "PrismBlockCandidateLandingFailedCritical": [] + } + }, + { + "name": "candidate pre-offer or unknown-outcome row is pending under the paging dwell", + "series": { + "qbit_prism_block_candidate_oldest_pending_seconds{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\"}": "0+1x360", + "qbit_prism_block_candidate_oldest_unacknowledged_seconds{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\"}": "0+1x360", + "qbit_prism_block_candidates_pending{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\"}": 1, + "qbit_prism_metrics_snapshot_available{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\"}": 1, + "qbit_prism_metrics_snapshot_stale{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\"}": 0, + "qbit_prism_collector_available{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\",collector=\"database\"}": 1, + "up{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\"}": 1, + "qbit_prism_block_candidate_oldest_landing_failed_seconds{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\"}": 0 + }, + "eval_time": "3m59s", + "expect": { + "PrismBlockCandidateOldestPendingCritical": 1, + "PrismBlockCandidateOldestPending": 1 + }, + "alerts": { + "PrismBlockCandidateOldestPendingCritical": [], + "PrismBlockCandidateOldestPending": [ + { + "job": "qbit-prism", + "instance": "one", + "network": "mainnet" + } + ] + } + }, + { + "name": "candidate pre-offer or unknown-outcome row still pages after the dwell", + "series": { + "qbit_prism_block_candidate_oldest_pending_seconds{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\"}": "0+1x360", + "qbit_prism_block_candidate_oldest_unacknowledged_seconds{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\"}": "0+1x360", + "qbit_prism_block_candidates_pending{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\"}": 1, + "qbit_prism_metrics_snapshot_available{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\"}": 1, + "qbit_prism_metrics_snapshot_stale{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\"}": 0, + "qbit_prism_collector_available{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\",collector=\"database\"}": 1, + "up{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\"}": 1, + "qbit_prism_block_candidate_oldest_landing_failed_seconds{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\"}": 0 + }, + "eval_time": "4m", + "expect": { + "PrismBlockCandidateOldestPendingCritical": 1, + "PrismBlockCandidateStuck": 0 + }, + "alerts": { + "PrismBlockCandidateOldestPendingCritical": [ + { + "job": "qbit-prism", + "instance": "one", + "network": "mainnet" + } + ], + "PrismBlockCandidateStuck": [] + } + }, + { + "name": "candidate stuck reconciliation row warns without paging", + "series": { + "qbit_prism_block_candidate_oldest_pending_seconds{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\"}": "1790+1x360", + "qbit_prism_block_candidate_oldest_unacknowledged_seconds{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\"}": 0, + "qbit_prism_block_candidates_pending{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\"}": 1, + "qbit_prism_metrics_snapshot_available{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\"}": 1, + "qbit_prism_metrics_snapshot_stale{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\"}": 0, + "qbit_prism_collector_available{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\",collector=\"database\"}": 1, + "up{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\"}": 1, + "qbit_prism_block_candidate_oldest_landing_failed_seconds{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\"}": 0 + }, + "eval_time": "6m", + "expect": { + "PrismBlockCandidateOldestPendingCritical": 0, + "PrismBlockCandidateStuck": 1, + "PrismBlockCandidateOldestPending": 1 + }, + "alerts": { + "PrismBlockCandidateOldestPendingCritical": [], + "PrismBlockCandidateStuck": [ + { + "job": "qbit-prism", + "instance": "one", + "network": "mainnet" + } + ], + "PrismBlockCandidateOldestPending": [ + { + "job": "qbit-prism", + "instance": "one", + "network": "mainnet" + } + ] + } + }, + { + "name": "candidate stuck reconciliation row is pending under the dwell", + "series": { + "qbit_prism_block_candidate_oldest_pending_seconds{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\"}": "1790+1x360", + "qbit_prism_block_candidate_oldest_unacknowledged_seconds{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\"}": 0, + "qbit_prism_block_candidates_pending{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\"}": 1, + "qbit_prism_metrics_snapshot_available{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\"}": 1, + "qbit_prism_metrics_snapshot_stale{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\"}": 0, + "qbit_prism_collector_available{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\",collector=\"database\"}": 1, + "up{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\"}": 1, + "qbit_prism_block_candidate_oldest_landing_failed_seconds{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\"}": 0 + }, + "eval_time": "2m", + "expect": { + "PrismBlockCandidateStuck": 1 + }, + "alerts": { + "PrismBlockCandidateStuck": [] + } + }, + { + "name": "candidate collector failure invalidates the unacknowledged age with the others", + "series": { + "qbit_prism_block_candidate_oldest_pending_seconds{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\"}": -1, + "qbit_prism_block_candidate_oldest_unacknowledged_seconds{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\"}": -1, + "qbit_prism_block_candidates_pending{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\"}": -1, + "qbit_prism_metrics_snapshot_available{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\"}": 1, + "qbit_prism_metrics_snapshot_stale{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\"}": 0, + "qbit_prism_collector_available{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\",collector=\"database\"}": 0, + "up{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\"}": 1, + "qbit_prism_block_candidate_oldest_landing_failed_seconds{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\"}": -1 + }, + "eval_time": "6m", + "expect": { + "PrismBlockCandidateMetricsUnavailable": 1, + "PrismBlockCandidateOldestPendingCritical": null, + "PrismBlockCandidateStuck": null, + "PrismBlockCandidateLandingFailedCritical": null + }, + "alerts": { + "PrismBlockCandidateOldestPendingCritical": [], + "PrismBlockCandidateStuck": [] + } + }, + { + "name": "candidate rejected block keeps paging like a pre-offer row", + "series": { + "qbit_prism_block_candidate_oldest_pending_seconds{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\"}": "0+1x360", + "qbit_prism_block_candidate_oldest_unacknowledged_seconds{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\"}": "0+1x360", + "qbit_prism_block_candidate_oldest_landing_failed_seconds{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\"}": 0, + "qbit_prism_block_candidates_pending{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\"}": 1, + "qbit_prism_metrics_snapshot_available{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\"}": 1, + "qbit_prism_metrics_snapshot_stale{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\"}": 0, + "qbit_prism_collector_available{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\",collector=\"database\"}": 1, + "up{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\"}": 1 + }, + "eval_time": "4m", + "expect": { + "PrismBlockCandidateOldestPendingCritical": 1, + "PrismBlockCandidateLandingFailedCritical": 0, + "PrismBlockCandidateStuck": 0 + }, + "alerts": { + "PrismBlockCandidateOldestPendingCritical": [ + { + "job": "qbit-prism", + "instance": "one", + "network": "mainnet" + } + ], + "PrismBlockCandidateLandingFailedCritical": [], + "PrismBlockCandidateStuck": [] + } + }, + { + "name": "candidate failing landing warns at 60s after its dwell", + "series": { + "qbit_prism_block_candidate_oldest_pending_seconds{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\"}": "0+1x420", + "qbit_prism_block_candidate_oldest_unacknowledged_seconds{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\"}": 0, + "qbit_prism_block_candidate_oldest_landing_failed_seconds{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\"}": "0+1x420", + "qbit_prism_block_candidates_pending{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\"}": 1, + "qbit_prism_metrics_snapshot_available{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\"}": 1, + "qbit_prism_metrics_snapshot_stale{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\"}": 0, + "qbit_prism_collector_available{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\",collector=\"database\"}": 1, + "up{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\"}": 1 + }, + "eval_time": "4m", + "expect": { + "PrismBlockCandidateLandingFailed": 1, + "PrismBlockCandidateLandingFailedCritical": 0, + "PrismBlockCandidateOldestPendingCritical": 0 + }, + "alerts": { + "PrismBlockCandidateLandingFailed": [ + { + "job": "qbit-prism", + "instance": "one", + "network": "mainnet" + } + ], + "PrismBlockCandidateLandingFailedCritical": [], + "PrismBlockCandidateOldestPendingCritical": [] + } + }, + { + "name": "candidate failing landing is pending under the paging dwell", + "series": { + "qbit_prism_block_candidate_oldest_pending_seconds{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\"}": "0+1x420", + "qbit_prism_block_candidate_oldest_unacknowledged_seconds{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\"}": 0, + "qbit_prism_block_candidate_oldest_landing_failed_seconds{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\"}": "0+1x420", + "qbit_prism_block_candidates_pending{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\"}": 1, + "qbit_prism_metrics_snapshot_available{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\"}": 1, + "qbit_prism_metrics_snapshot_stale{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\"}": 0, + "qbit_prism_collector_available{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\",collector=\"database\"}": 1, + "up{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\"}": 1 + }, + "eval_time": "6m6s", + "expect": { + "PrismBlockCandidateLandingFailedCritical": 1, + "PrismBlockCandidateOldestPendingCritical": 0 + }, + "alerts": { + "PrismBlockCandidateLandingFailedCritical": [], + "PrismBlockCandidateOldestPendingCritical": [] + } + }, + { + "name": "candidate failing landing pages at 307s plus one minute", + "series": { + "qbit_prism_block_candidate_oldest_pending_seconds{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\"}": "0+1x420", + "qbit_prism_block_candidate_oldest_unacknowledged_seconds{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\"}": 0, + "qbit_prism_block_candidate_oldest_landing_failed_seconds{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\"}": "0+1x420", + "qbit_prism_block_candidates_pending{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\"}": 1, + "qbit_prism_metrics_snapshot_available{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\"}": 1, + "qbit_prism_metrics_snapshot_stale{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\"}": 0, + "qbit_prism_collector_available{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\",collector=\"database\"}": 1, + "up{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\"}": 1 + }, + "eval_time": "6m8s", + "expect": { + "PrismBlockCandidateLandingFailedCritical": 1, + "PrismBlockCandidateOldestPendingCritical": 0 + }, + "alerts": { + "PrismBlockCandidateLandingFailedCritical": [ + { + "job": "qbit-prism", + "instance": "one", + "network": "mainnet" + } + ], + "PrismBlockCandidateOldestPendingCritical": [] + } + }, + { + "name": "candidate accepted block hung in offered before its landing is pending under the paging dwell", + "series": { + "qbit_prism_block_candidate_oldest_pending_seconds{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\"}": "0+1x420", + "qbit_prism_block_candidate_oldest_unacknowledged_seconds{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\"}": 0, + "qbit_prism_block_candidate_oldest_landing_failed_seconds{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\"}": "0+1x420", + "qbit_prism_block_candidates_pending{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\"}": 1, + "qbit_prism_accepted_block_unlanded_seconds{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\"}": "0+1x420", + "qbit_prism_accepted_block_revision_work_pending_seconds{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\"}": 0, + "qbit_prism_accepted_block_revision_work_tracking_unknown{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\"}": 0, + "qbit_prism_authorized_clients{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\"}": 1, + "qbit_prism_metrics_snapshot_available{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\"}": 1, + "qbit_prism_metrics_snapshot_stale{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\"}": 0, + "qbit_prism_collector_available{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\",collector=\"database\"}": 1, + "up{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\"}": 1 + }, + "eval_time": "6m6s", + "expect": { + "PrismBlockCandidateLandingFailed": 1, + "PrismBlockCandidateLandingFailedCritical": 1, + "PrismBlockCandidateOldestPendingCritical": 0, + "PrismAcceptedRevisionWorkPendingCritical": 0, + "PrismAcceptedBlockUnlanded": 0 + }, + "alerts": { + "PrismBlockCandidateLandingFailed": [ + { + "job": "qbit-prism", + "instance": "one", + "network": "mainnet" + } + ], + "PrismBlockCandidateLandingFailedCritical": [], + "PrismBlockCandidateOldestPendingCritical": [], + "PrismAcceptedRevisionWorkPendingCritical": [], + "PrismAcceptedBlockUnlanded": [] + } + }, + { + "name": "candidate accepted block hung in offered before its landing pages at 307s plus one minute", + "series": { + "qbit_prism_block_candidate_oldest_pending_seconds{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\"}": "0+1x420", + "qbit_prism_block_candidate_oldest_unacknowledged_seconds{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\"}": 0, + "qbit_prism_block_candidate_oldest_landing_failed_seconds{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\"}": "0+1x420", + "qbit_prism_block_candidates_pending{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\"}": 1, + "qbit_prism_accepted_block_unlanded_seconds{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\"}": "0+1x420", + "qbit_prism_accepted_block_revision_work_pending_seconds{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\"}": 0, + "qbit_prism_accepted_block_revision_work_tracking_unknown{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\"}": 0, + "qbit_prism_authorized_clients{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\"}": 1, + "qbit_prism_metrics_snapshot_available{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\"}": 1, + "qbit_prism_metrics_snapshot_stale{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\"}": 0, + "qbit_prism_collector_available{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\",collector=\"database\"}": 1, + "up{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\"}": 1 + }, + "eval_time": "6m8s", + "expect": { + "PrismBlockCandidateLandingFailed": 1, + "PrismBlockCandidateLandingFailedCritical": 1, + "PrismBlockCandidateOldestPendingCritical": 0, + "PrismAcceptedRevisionWorkPendingCritical": 0, + "PrismAcceptedBlockUnlanded": 0 + }, + "alerts": { + "PrismBlockCandidateLandingFailed": [ + { + "job": "qbit-prism", + "instance": "one", + "network": "mainnet" + } + ], + "PrismBlockCandidateLandingFailedCritical": [ + { + "job": "qbit-prism", + "instance": "one", + "network": "mainnet" + } + ], + "PrismBlockCandidateOldestPendingCritical": [], + "PrismAcceptedRevisionWorkPendingCritical": [], + "PrismAcceptedBlockUnlanded": [] + } + }, + { + "name": "candidate landing refused once then landed never reaches the dwell", + "series": { + "qbit_prism_block_candidate_oldest_pending_seconds{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\"}": "0+1x360", + "qbit_prism_block_candidate_oldest_unacknowledged_seconds{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\"}": 0, + "qbit_prism_block_candidate_oldest_landing_failed_seconds{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\"}": "0+1x9 0x350", + "qbit_prism_block_candidates_pending{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\"}": 1, + "qbit_prism_metrics_snapshot_available{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\"}": 1, + "qbit_prism_metrics_snapshot_stale{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\"}": 0, + "qbit_prism_collector_available{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\",collector=\"database\"}": 1, + "up{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\"}": 1 + }, + "eval_time": "6m", + "expect": { + "PrismBlockCandidateLandingFailed": 0, + "PrismBlockCandidateLandingFailedCritical": 0, + "PrismBlockCandidateOldestPendingCritical": 0 + }, + "alerts": { + "PrismBlockCandidateLandingFailed": [], + "PrismBlockCandidateLandingFailedCritical": [], + "PrismBlockCandidateOldestPendingCritical": [] + } + }, + { + "name": "candidate mixed-version rollout falls back to the all-unfinished age", + "series": { + "qbit_prism_block_candidate_oldest_pending_seconds{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\"}": "0+1x360", + "qbit_prism_block_candidates_pending{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\"}": 1, + "qbit_prism_metrics_snapshot_available{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\"}": 1, + "qbit_prism_metrics_snapshot_stale{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\"}": 0, + "qbit_prism_collector_available{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\",collector=\"database\"}": 1, + "up{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\"}": 1 + }, + "eval_time": "4m", + "expect": { + "PrismBlockCandidateOldestPendingCritical": 1, + "PrismBlockCandidateOldestPending": 1, + "PrismBlockCandidateMetricsUnavailable": 0 + }, + "alerts": { + "PrismBlockCandidateOldestPendingCritical": [ + { + "job": "qbit-prism", + "instance": "one", + "network": "mainnet" + } + ], + "PrismBlockCandidateOldestPending": [ + { + "job": "qbit-prism", + "instance": "one", + "network": "mainnet" + } + ] + } + }, + { + "name": "candidate mixed-version rollout fallback is pending under the dwell", + "series": { + "qbit_prism_block_candidate_oldest_pending_seconds{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\"}": "0+1x360", + "qbit_prism_block_candidates_pending{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\"}": 1, + "qbit_prism_metrics_snapshot_available{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\"}": 1, + "qbit_prism_metrics_snapshot_stale{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\"}": 0, + "qbit_prism_collector_available{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\",collector=\"database\"}": 1, + "up{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\"}": 1 + }, + "eval_time": "3m59s", + "expect": { + "PrismBlockCandidateOldestPendingCritical": 1 + }, + "alerts": { + "PrismBlockCandidateOldestPendingCritical": [] + } + }, + { + "name": "candidate upgraded target ignores the fallback for an acknowledged row", + "series": { + "qbit_prism_block_candidate_oldest_pending_seconds{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\"}": "0+1x360", + "qbit_prism_block_candidate_oldest_unacknowledged_seconds{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\"}": "0x360", + "qbit_prism_block_candidate_oldest_landing_failed_seconds{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\"}": 0, + "qbit_prism_block_candidates_pending{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\"}": 1, + "qbit_prism_metrics_snapshot_available{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\"}": 1, + "qbit_prism_metrics_snapshot_stale{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\"}": 0, + "qbit_prism_collector_available{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\",collector=\"database\"}": 1, + "up{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\"}": 1 + }, + "eval_time": "6m", + "expect": { + "PrismBlockCandidateOldestPendingCritical": 0, + "PrismBlockCandidateOldestPending": 1 + }, + "alerts": { + "PrismBlockCandidateOldestPendingCritical": [], + "PrismBlockCandidateOldestPending": [ + { + "job": "qbit-prism", + "instance": "one", + "network": "mainnet" + } + ] + } + }, + { + "name": "missing health family on live scrape", + "series": { + "qbit_prism_metrics_snapshot_available{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\"}": 1.0, + "qbit_prism_metrics_snapshot_stale{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\"}": 0.0, + "qbit_prism_collector_available{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\",collector=\"database\"}": 1.0, + "qbit_prism_collector_available{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\",collector=\"process\"}": 1.0, + "qbit_prism_block_candidates_pending{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\"}": 0.0, + "qbit_prism_block_candidate_oldest_pending_seconds{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\"}": 0.0, + "qbit_prism_process_resident_memory_bytes{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\"}": 0.0, + "qbit_prism_stratum_semantic_current_work_ratio{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\"}": 1.0, + "qbit_prism_stratum_oldest_pending_initial_job_seconds{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\"}": 0.0, + "qbit_prism_authorized_clients{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\"}": 1.0, + "qbit_prism_runtime_lag_seconds{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\"}": 0.0, + "qbit_prism_runtime_poll_lag_seconds{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\",task=\"refresh\"}": 0.0, + "qbit_prism_runtime_task_stalled{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\",task=\"refresh\"}": 0.0, + "up{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\"}": 1.0 + }, + "expect": { + "PrismMetricsNoData": 1 + } + }, + { + "name": "runtime sampler unknown", + "series": { + "qbit_prism_metrics_snapshot_available{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\"}": 1.0, + "qbit_prism_metrics_snapshot_stale{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\"}": 0.0, + "qbit_prism_collector_available{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\",collector=\"database\"}": 1.0, + "qbit_prism_collector_available{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\",collector=\"process\"}": 1.0, + "qbit_prism_block_candidates_pending{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\"}": 0.0, + "qbit_prism_block_candidate_oldest_pending_seconds{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\"}": 0.0, + "qbit_prism_process_resident_memory_bytes{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\"}": 0.0, + "qbit_prism_stratum_semantic_current_work_ratio{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\"}": 1.0, + "qbit_prism_stratum_oldest_pending_initial_job_seconds{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\"}": 0.0, + "qbit_prism_authorized_clients{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\"}": 1.0, + "qbit_prism_runtime_lag_seconds{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\"}": -1, + "qbit_prism_runtime_poll_lag_seconds{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\",task=\"refresh\"}": 0.0, + "qbit_prism_runtime_task_stalled{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\",task=\"refresh\"}": 0.0, + "up{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\"}": 1.0, + "qbit_prism_health_state{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\"}": 1.0 + }, + "expect": { + "PrismRuntimeLagHigh": 1 + } + }, + { + "name": "public scrape missing while peer is healthy", + "series": { + "qbit_prism_public_ledger_ready{job=\"qbit-prism-public-read\",instance=\"healthy\",network=\"mainnet\"}": 1, + "up{job=\"qbit-prism-public-read\",instance=\"healthy\",network=\"mainnet\"}": 1, + "up{job=\"qbit-prism-public-read\",instance=\"failed\",network=\"mainnet\"}": 0 + }, + "expect": { + "PrismPublicLedgerUnready": 1 + } + }, + { + "name": "public scrape fails while stale ready sample remains", + "series": { + "qbit_prism_public_ledger_ready{job=\"qbit-prism-public-read\",instance=\"failed\",network=\"mainnet\"}": 1, + "up{job=\"qbit-prism-public-read\",instance=\"failed\",network=\"mainnet\"}": 0 + }, + "expect": { + "PrismPublicLedgerUnready": 1 + } + }, + { + "name": "low-volume total ledger write failures remain visible", + "series": { + "qbit_prism_rejections_total{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\",reason_id=\"ledger-confirmation-failed\"}": "0+0x300 1+1x59", + "qbit_prism_metrics_snapshot_available{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\"}": 1, + "qbit_prism_metrics_snapshot_stale{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\"}": 0, + "up{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\"}": 1 + }, + "expect": { + "PrismShareAppendFailures": 1, + "PrismRejectRatioByReasonHigh": null + } + }, + { + "name": "unknown ledger outcomes page like write failures", + "series": { + "qbit_prism_rejections_total{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\",reason_id=\"ledger-outcome-unknown\"}": "0+0x300 1+1x59", + "qbit_prism_metrics_snapshot_available{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\"}": 1, + "qbit_prism_metrics_snapshot_stale{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\"}": 0, + "up{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\"}": 1 + }, + "expect": { + "PrismShareAppendFailures": 1, + "PrismRejectRatioByReasonHigh": null + } + }, + { + "name": "network client floor: six plus six is healthy", + "series": { + "up{job=\"qbit-prism\",instance=\"coordinator-0\",network=\"mainnet\"}": 1, + "qbit_prism_connections{job=\"qbit-prism\",instance=\"coordinator-0\",network=\"mainnet\"}": 6, + "qbit_prism_metrics_snapshot_available{job=\"qbit-prism\",instance=\"coordinator-0\",network=\"mainnet\"}": 1, + "qbit_prism_metrics_snapshot_stale{job=\"qbit-prism\",instance=\"coordinator-0\",network=\"mainnet\"}": 0, + "up{job=\"qbit-prism\",instance=\"coordinator-1\",network=\"mainnet\"}": 1, + "qbit_prism_connections{job=\"qbit-prism\",instance=\"coordinator-1\",network=\"mainnet\"}": 6, + "qbit_prism_metrics_snapshot_available{job=\"qbit-prism\",instance=\"coordinator-1\",network=\"mainnet\"}": 1, + "qbit_prism_metrics_snapshot_stale{job=\"qbit-prism\",instance=\"coordinator-1\",network=\"mainnet\"}": 0 + }, + "expect": { + "PRISM Connected Clients": 0 + } + }, + { + "name": "network client floor: six plus three is low", + "series": { + "up{job=\"qbit-prism\",instance=\"coordinator-0\",network=\"mainnet\"}": 1, + "qbit_prism_connections{job=\"qbit-prism\",instance=\"coordinator-0\",network=\"mainnet\"}": 6, + "qbit_prism_metrics_snapshot_available{job=\"qbit-prism\",instance=\"coordinator-0\",network=\"mainnet\"}": 1, + "qbit_prism_metrics_snapshot_stale{job=\"qbit-prism\",instance=\"coordinator-0\",network=\"mainnet\"}": 0, + "up{job=\"qbit-prism\",instance=\"coordinator-1\",network=\"mainnet\"}": 1, + "qbit_prism_connections{job=\"qbit-prism\",instance=\"coordinator-1\",network=\"mainnet\"}": 3, + "qbit_prism_metrics_snapshot_available{job=\"qbit-prism\",instance=\"coordinator-1\",network=\"mainnet\"}": 1, + "qbit_prism_metrics_snapshot_stale{job=\"qbit-prism\",instance=\"coordinator-1\",network=\"mainnet\"}": 0 + }, + "expect": { + "PRISM Connected Clients": 1 + } + }, + { + "name": "network client floor: partial stale data is unknown", + "series": { + "up{job=\"qbit-prism\",instance=\"coordinator-0\",network=\"mainnet\"}": 1, + "qbit_prism_connections{job=\"qbit-prism\",instance=\"coordinator-0\",network=\"mainnet\"}": 6, + "qbit_prism_metrics_snapshot_available{job=\"qbit-prism\",instance=\"coordinator-0\",network=\"mainnet\"}": 1, + "qbit_prism_metrics_snapshot_stale{job=\"qbit-prism\",instance=\"coordinator-0\",network=\"mainnet\"}": 0, + "up{job=\"qbit-prism\",instance=\"coordinator-1\",network=\"mainnet\"}": 1, + "qbit_prism_connections{job=\"qbit-prism\",instance=\"coordinator-1\",network=\"mainnet\"}": 6, + "qbit_prism_metrics_snapshot_available{job=\"qbit-prism\",instance=\"coordinator-1\",network=\"mainnet\"}": 1, + "qbit_prism_metrics_snapshot_stale{job=\"qbit-prism\",instance=\"coordinator-1\",network=\"mainnet\"}": 1 + }, + "expect": { + "PRISM Connected Clients": null + } + }, + { + "name": "D3 healthy idle has equal exact WAL positions", + "series": { + "up{job=\"qbit-postgres-primary\",instance=\"primary\",network=\"mainnet\"}": 1, + "pg_up{job=\"qbit-postgres-primary\",instance=\"primary\",network=\"mainnet\"}": 1, + "pg_exporter_last_scrape_error{job=\"qbit-postgres-primary\",instance=\"primary\",network=\"mainnet\"}": 0, + "pg_stat_replication_count{job=\"qbit-postgres-primary\",instance=\"primary\",network=\"mainnet\",application_name=\"prism_standby_1\"}": 1, + "pg_stat_replication_replay_lag{job=\"qbit-postgres-primary\",instance=\"primary\",network=\"mainnet\",application_name=\"prism_standby_1\"}": 0, + "pg_stat_replication_primary_flush_lsn_hi{job=\"qbit-postgres-primary\",instance=\"primary\",network=\"mainnet\",application_name=\"prism_standby_1\"}": 2097153, + "pg_stat_replication_primary_flush_lsn_lo{job=\"qbit-postgres-primary\",instance=\"primary\",network=\"mainnet\",application_name=\"prism_standby_1\"}": 100, + "pg_stat_replication_replay_lsn_hi{job=\"qbit-postgres-primary\",instance=\"primary\",network=\"mainnet\",application_name=\"prism_standby_1\"}": 2097153, + "pg_stat_replication_replay_lsn_lo{job=\"qbit-postgres-primary\",instance=\"primary\",network=\"mainnet\",application_name=\"prism_standby_1\"}": 100, + "pg_stat_replication_async{job=\"qbit-postgres-primary\",instance=\"primary\",network=\"mainnet\",application_name=\"prism_standby_1\"}": 1 + }, + "eval_time": "6m", + "expect": { + "PrismHAStandbyReplayLagHigh": 0, + "PrismHAStandbyDisconnected": 0 + } + }, + { + "name": "D3 caught up remains healthy with diagnostic NULL or old latency", + "series": { + "up{job=\"qbit-postgres-primary\",instance=\"primary\",network=\"mainnet\"}": 1, + "pg_up{job=\"qbit-postgres-primary\",instance=\"primary\",network=\"mainnet\"}": 1, + "pg_exporter_last_scrape_error{job=\"qbit-postgres-primary\",instance=\"primary\",network=\"mainnet\"}": 0, + "pg_stat_replication_count{job=\"qbit-postgres-primary\",instance=\"primary\",network=\"mainnet\",application_name=\"prism_standby_1\"}": 1, + "pg_stat_replication_replay_lag{job=\"qbit-postgres-primary\",instance=\"primary\",network=\"mainnet\",application_name=\"prism_standby_1\"}": -1, + "pg_stat_replication_primary_flush_lsn_hi{job=\"qbit-postgres-primary\",instance=\"primary\",network=\"mainnet\",application_name=\"prism_standby_1\"}": 2097153, + "pg_stat_replication_primary_flush_lsn_lo{job=\"qbit-postgres-primary\",instance=\"primary\",network=\"mainnet\",application_name=\"prism_standby_1\"}": 100, + "pg_stat_replication_replay_lsn_hi{job=\"qbit-postgres-primary\",instance=\"primary\",network=\"mainnet\",application_name=\"prism_standby_1\"}": 2097153, + "pg_stat_replication_replay_lsn_lo{job=\"qbit-postgres-primary\",instance=\"primary\",network=\"mainnet\",application_name=\"prism_standby_1\"}": 100, + "pg_stat_replication_async{job=\"qbit-postgres-primary\",instance=\"primary\",network=\"mainnet\",application_name=\"prism_standby_1\"}": 1 + }, + "eval_time": "6m", + "expect": { + "PrismHAStandbyReplayLagHigh": 0, + "PrismHAStandbyDisconnected": 0 + } + }, + { + "name": "D3 caught up remains healthy after a slow recovery", + "series": { + "up{job=\"qbit-postgres-primary\",instance=\"primary\",network=\"mainnet\"}": 1, + "pg_up{job=\"qbit-postgres-primary\",instance=\"primary\",network=\"mainnet\"}": 1, + "pg_exporter_last_scrape_error{job=\"qbit-postgres-primary\",instance=\"primary\",network=\"mainnet\"}": 0, + "pg_stat_replication_count{job=\"qbit-postgres-primary\",instance=\"primary\",network=\"mainnet\",application_name=\"prism_standby_1\"}": 1, + "pg_stat_replication_replay_lag{job=\"qbit-postgres-primary\",instance=\"primary\",network=\"mainnet\",application_name=\"prism_standby_1\"}": 99, + "pg_stat_replication_primary_flush_lsn_hi{job=\"qbit-postgres-primary\",instance=\"primary\",network=\"mainnet\",application_name=\"prism_standby_1\"}": 2097153, + "pg_stat_replication_primary_flush_lsn_lo{job=\"qbit-postgres-primary\",instance=\"primary\",network=\"mainnet\",application_name=\"prism_standby_1\"}": 100, + "pg_stat_replication_replay_lsn_hi{job=\"qbit-postgres-primary\",instance=\"primary\",network=\"mainnet\",application_name=\"prism_standby_1\"}": 2097153, + "pg_stat_replication_replay_lsn_lo{job=\"qbit-postgres-primary\",instance=\"primary\",network=\"mainnet\",application_name=\"prism_standby_1\"}": 100, + "pg_stat_replication_async{job=\"qbit-postgres-primary\",instance=\"primary\",network=\"mainnet\",application_name=\"prism_standby_1\"}": 1 + }, + "eval_time": "6m", + "expect": { + "PrismHAStandbyReplayLagHigh": 0, + "PrismHAStandbyDisconnected": 0 + } + }, + { + "name": "D3 stalled replay alerts despite tiny last-reported latency", + "series": { + "up{job=\"qbit-postgres-primary\",instance=\"primary\",network=\"mainnet\"}": 1, + "pg_up{job=\"qbit-postgres-primary\",instance=\"primary\",network=\"mainnet\"}": 1, + "pg_exporter_last_scrape_error{job=\"qbit-postgres-primary\",instance=\"primary\",network=\"mainnet\"}": 0, + "pg_stat_replication_count{job=\"qbit-postgres-primary\",instance=\"primary\",network=\"mainnet\",application_name=\"prism_standby_1\"}": 1, + "pg_stat_replication_replay_lag{job=\"qbit-postgres-primary\",instance=\"primary\",network=\"mainnet\",application_name=\"prism_standby_1\"}": 0.001, + "pg_stat_replication_primary_flush_lsn_hi{job=\"qbit-postgres-primary\",instance=\"primary\",network=\"mainnet\",application_name=\"prism_standby_1\"}": 2097153, + "pg_stat_replication_primary_flush_lsn_lo{job=\"qbit-postgres-primary\",instance=\"primary\",network=\"mainnet\",application_name=\"prism_standby_1\"}": 100, + "pg_stat_replication_replay_lsn_hi{job=\"qbit-postgres-primary\",instance=\"primary\",network=\"mainnet\",application_name=\"prism_standby_1\"}": 2097153, + "pg_stat_replication_replay_lsn_lo{job=\"qbit-postgres-primary\",instance=\"primary\",network=\"mainnet\",application_name=\"prism_standby_1\"}": 99, + "pg_stat_replication_async{job=\"qbit-postgres-primary\",instance=\"primary\",network=\"mainnet\",application_name=\"prism_standby_1\"}": 1 + }, + "eval_time": "6m", + "expect": { + "PrismHAStandbyReplayLagHigh": 1, + "PrismHAStandbyDisconnected": 0 + }, + "alerts": { + "PrismHAStandbyReplayLagHigh": true, + "PrismHAStandbyDisconnected": false + } + }, + { + "name": "D3 low word wrap is lagged", + "series": { + "up{job=\"qbit-postgres-primary\",instance=\"primary\",network=\"mainnet\"}": 1, + "pg_up{job=\"qbit-postgres-primary\",instance=\"primary\",network=\"mainnet\"}": 1, + "pg_exporter_last_scrape_error{job=\"qbit-postgres-primary\",instance=\"primary\",network=\"mainnet\"}": 0, + "pg_stat_replication_count{job=\"qbit-postgres-primary\",instance=\"primary\",network=\"mainnet\",application_name=\"prism_standby_1\"}": 1, + "pg_stat_replication_replay_lag{job=\"qbit-postgres-primary\",instance=\"primary\",network=\"mainnet\",application_name=\"prism_standby_1\"}": 0, + "pg_stat_replication_primary_flush_lsn_hi{job=\"qbit-postgres-primary\",instance=\"primary\",network=\"mainnet\",application_name=\"prism_standby_1\"}": 2097153, + "pg_stat_replication_primary_flush_lsn_lo{job=\"qbit-postgres-primary\",instance=\"primary\",network=\"mainnet\",application_name=\"prism_standby_1\"}": 0, + "pg_stat_replication_replay_lsn_hi{job=\"qbit-postgres-primary\",instance=\"primary\",network=\"mainnet\",application_name=\"prism_standby_1\"}": 2097152, + "pg_stat_replication_replay_lsn_lo{job=\"qbit-postgres-primary\",instance=\"primary\",network=\"mainnet\",application_name=\"prism_standby_1\"}": 4294967295, + "pg_stat_replication_async{job=\"qbit-postgres-primary\",instance=\"primary\",network=\"mainnet\",application_name=\"prism_standby_1\"}": 1 + }, + "eval_time": "6m", + "expect": { + "PrismHAStandbyReplayLagHigh": 1, + "PrismHAStandbyDisconnected": 0 + } + }, + { + "name": "D3 low word wrap has caught up", + "series": { + "up{job=\"qbit-postgres-primary\",instance=\"primary\",network=\"mainnet\"}": 1, + "pg_up{job=\"qbit-postgres-primary\",instance=\"primary\",network=\"mainnet\"}": 1, + "pg_exporter_last_scrape_error{job=\"qbit-postgres-primary\",instance=\"primary\",network=\"mainnet\"}": 0, + "pg_stat_replication_count{job=\"qbit-postgres-primary\",instance=\"primary\",network=\"mainnet\",application_name=\"prism_standby_1\"}": 1, + "pg_stat_replication_replay_lag{job=\"qbit-postgres-primary\",instance=\"primary\",network=\"mainnet\",application_name=\"prism_standby_1\"}": 0, + "pg_stat_replication_primary_flush_lsn_hi{job=\"qbit-postgres-primary\",instance=\"primary\",network=\"mainnet\",application_name=\"prism_standby_1\"}": 2097152, + "pg_stat_replication_primary_flush_lsn_lo{job=\"qbit-postgres-primary\",instance=\"primary\",network=\"mainnet\",application_name=\"prism_standby_1\"}": 4294967295, + "pg_stat_replication_replay_lsn_hi{job=\"qbit-postgres-primary\",instance=\"primary\",network=\"mainnet\",application_name=\"prism_standby_1\"}": 2097153, + "pg_stat_replication_replay_lsn_lo{job=\"qbit-postgres-primary\",instance=\"primary\",network=\"mainnet\",application_name=\"prism_standby_1\"}": 0, + "pg_stat_replication_async{job=\"qbit-postgres-primary\",instance=\"primary\",network=\"mainnet\",application_name=\"prism_standby_1\"}": 1 + }, + "eval_time": "6m", + "expect": { + "PrismHAStandbyReplayLagHigh": 0, + "PrismHAStandbyDisconnected": 0 + } + }, + { + "name": "D3 exact five-second history boundary", + "series": { + "up{job=\"qbit-postgres-primary\",instance=\"primary\",network=\"mainnet\"}": 1, + "pg_up{job=\"qbit-postgres-primary\",instance=\"primary\",network=\"mainnet\"}": 1, + "pg_exporter_last_scrape_error{job=\"qbit-postgres-primary\",instance=\"primary\",network=\"mainnet\"}": 0, + "pg_stat_replication_count{job=\"qbit-postgres-primary\",instance=\"primary\",network=\"mainnet\",application_name=\"prism_standby_1\"}": 1, + "pg_stat_replication_replay_lag{job=\"qbit-postgres-primary\",instance=\"primary\",network=\"mainnet\",application_name=\"prism_standby_1\"}": 0, + "pg_stat_replication_primary_flush_lsn_hi{job=\"qbit-postgres-primary\",instance=\"primary\",network=\"mainnet\",application_name=\"prism_standby_1\"}": 2097153, + "pg_stat_replication_primary_flush_lsn_lo{job=\"qbit-postgres-primary\",instance=\"primary\",network=\"mainnet\",application_name=\"prism_standby_1\"}": "100+1x360", + "pg_stat_replication_replay_lsn_hi{job=\"qbit-postgres-primary\",instance=\"primary\",network=\"mainnet\",application_name=\"prism_standby_1\"}": 2097153, + "pg_stat_replication_replay_lsn_lo{job=\"qbit-postgres-primary\",instance=\"primary\",network=\"mainnet\",application_name=\"prism_standby_1\"}": "95+1x360", + "pg_stat_replication_async{job=\"qbit-postgres-primary\",instance=\"primary\",network=\"mainnet\",application_name=\"prism_standby_1\"}": 1 + }, + "eval_time": "6m", + "expect": { + "PrismHAStandbyReplayLagHigh": 0, + "PrismHAStandbyDisconnected": 0 + } + }, + { + "name": "D3 six-second replay delay", + "series": { + "up{job=\"qbit-postgres-primary\",instance=\"primary\",network=\"mainnet\"}": 1, + "pg_up{job=\"qbit-postgres-primary\",instance=\"primary\",network=\"mainnet\"}": 1, + "pg_exporter_last_scrape_error{job=\"qbit-postgres-primary\",instance=\"primary\",network=\"mainnet\"}": 0, + "pg_stat_replication_count{job=\"qbit-postgres-primary\",instance=\"primary\",network=\"mainnet\",application_name=\"prism_standby_1\"}": 1, + "pg_stat_replication_replay_lag{job=\"qbit-postgres-primary\",instance=\"primary\",network=\"mainnet\",application_name=\"prism_standby_1\"}": 0, + "pg_stat_replication_primary_flush_lsn_hi{job=\"qbit-postgres-primary\",instance=\"primary\",network=\"mainnet\",application_name=\"prism_standby_1\"}": 2097153, + "pg_stat_replication_primary_flush_lsn_lo{job=\"qbit-postgres-primary\",instance=\"primary\",network=\"mainnet\",application_name=\"prism_standby_1\"}": "100+1x360", + "pg_stat_replication_replay_lsn_hi{job=\"qbit-postgres-primary\",instance=\"primary\",network=\"mainnet\",application_name=\"prism_standby_1\"}": 2097153, + "pg_stat_replication_replay_lsn_lo{job=\"qbit-postgres-primary\",instance=\"primary\",network=\"mainnet\",application_name=\"prism_standby_1\"}": "94+1x360", + "pg_stat_replication_async{job=\"qbit-postgres-primary\",instance=\"primary\",network=\"mainnet\",application_name=\"prism_standby_1\"}": 1 + }, + "eval_time": "6m", + "expect": { + "PrismHAStandbyReplayLagHigh": 1, + "PrismHAStandbyDisconnected": 0 + } + }, + { + "name": "D3 disconnected standby", + "series": { + "up{job=\"qbit-postgres-primary\",instance=\"primary\",network=\"mainnet\"}": 1, + "pg_up{job=\"qbit-postgres-primary\",instance=\"primary\",network=\"mainnet\"}": 1, + "pg_exporter_last_scrape_error{job=\"qbit-postgres-primary\",instance=\"primary\",network=\"mainnet\"}": 0, + "pg_stat_replication_count{job=\"qbit-postgres-primary\",instance=\"primary\",network=\"mainnet\",application_name=\"prism_standby_1\"}": 0, + "pg_stat_replication_replay_lag{job=\"qbit-postgres-primary\",instance=\"primary\",network=\"mainnet\",application_name=\"prism_standby_1\"}": -1, + "pg_stat_replication_primary_flush_lsn_hi{job=\"qbit-postgres-primary\",instance=\"primary\",network=\"mainnet\",application_name=\"prism_standby_1\"}": 2097153, + "pg_stat_replication_primary_flush_lsn_lo{job=\"qbit-postgres-primary\",instance=\"primary\",network=\"mainnet\",application_name=\"prism_standby_1\"}": 100, + "pg_stat_replication_replay_lsn_hi{job=\"qbit-postgres-primary\",instance=\"primary\",network=\"mainnet\",application_name=\"prism_standby_1\"}": -1, + "pg_stat_replication_replay_lsn_lo{job=\"qbit-postgres-primary\",instance=\"primary\",network=\"mainnet\",application_name=\"prism_standby_1\"}": -1, + "pg_stat_replication_async{job=\"qbit-postgres-primary\",instance=\"primary\",network=\"mainnet\",application_name=\"prism_standby_1\"}": 1 + }, + "eval_time": "6m", + "expect": { + "PrismHAStandbyReplayLagHigh": 1, + "PrismHAStandbyDisconnected": 1 + }, + "alerts": { + "PrismHAStandbyReplayLagHigh": true, + "PrismHAStandbyDisconnected": true + } + }, + { + "name": "D3 ambiguous standby identity", + "series": { + "up{job=\"qbit-postgres-primary\",instance=\"primary\",network=\"mainnet\"}": 1, + "pg_up{job=\"qbit-postgres-primary\",instance=\"primary\",network=\"mainnet\"}": 1, + "pg_exporter_last_scrape_error{job=\"qbit-postgres-primary\",instance=\"primary\",network=\"mainnet\"}": 0, + "pg_stat_replication_count{job=\"qbit-postgres-primary\",instance=\"primary\",network=\"mainnet\",application_name=\"prism_standby_1\"}": 2, + "pg_stat_replication_replay_lag{job=\"qbit-postgres-primary\",instance=\"primary\",network=\"mainnet\",application_name=\"prism_standby_1\"}": 0, + "pg_stat_replication_primary_flush_lsn_hi{job=\"qbit-postgres-primary\",instance=\"primary\",network=\"mainnet\",application_name=\"prism_standby_1\"}": 2097153, + "pg_stat_replication_primary_flush_lsn_lo{job=\"qbit-postgres-primary\",instance=\"primary\",network=\"mainnet\",application_name=\"prism_standby_1\"}": 100, + "pg_stat_replication_replay_lsn_hi{job=\"qbit-postgres-primary\",instance=\"primary\",network=\"mainnet\",application_name=\"prism_standby_1\"}": -1, + "pg_stat_replication_replay_lsn_lo{job=\"qbit-postgres-primary\",instance=\"primary\",network=\"mainnet\",application_name=\"prism_standby_1\"}": -1, + "pg_stat_replication_async{job=\"qbit-postgres-primary\",instance=\"primary\",network=\"mainnet\",application_name=\"prism_standby_1\"}": 1 + }, + "eval_time": "6m", + "expect": { + "PrismHAStandbyReplayLagHigh": 1, + "PrismHAStandbyDisconnected": 1 + } + }, + { + "name": "D3 failed database", + "series": { + "up{job=\"qbit-postgres-primary\",instance=\"primary\",network=\"mainnet\"}": 1, + "pg_up{job=\"qbit-postgres-primary\",instance=\"primary\",network=\"mainnet\"}": 0, + "pg_exporter_last_scrape_error{job=\"qbit-postgres-primary\",instance=\"primary\",network=\"mainnet\"}": 0, + "pg_stat_replication_count{job=\"qbit-postgres-primary\",instance=\"primary\",network=\"mainnet\",application_name=\"prism_standby_1\"}": 1, + "pg_stat_replication_replay_lag{job=\"qbit-postgres-primary\",instance=\"primary\",network=\"mainnet\",application_name=\"prism_standby_1\"}": 0, + "pg_stat_replication_primary_flush_lsn_hi{job=\"qbit-postgres-primary\",instance=\"primary\",network=\"mainnet\",application_name=\"prism_standby_1\"}": 2097153, + "pg_stat_replication_primary_flush_lsn_lo{job=\"qbit-postgres-primary\",instance=\"primary\",network=\"mainnet\",application_name=\"prism_standby_1\"}": 100, + "pg_stat_replication_replay_lsn_hi{job=\"qbit-postgres-primary\",instance=\"primary\",network=\"mainnet\",application_name=\"prism_standby_1\"}": 2097153, + "pg_stat_replication_replay_lsn_lo{job=\"qbit-postgres-primary\",instance=\"primary\",network=\"mainnet\",application_name=\"prism_standby_1\"}": 100, + "pg_stat_replication_async{job=\"qbit-postgres-primary\",instance=\"primary\",network=\"mainnet\",application_name=\"prism_standby_1\"}": 1 + }, + "eval_time": "6m", + "expect": { + "PrismHAStandbyReplayLagHigh": 1, + "PrismHAStandbyDisconnected": 1 + } + }, + { + "name": "D3 failed query with previous healthy samples", + "series": { + "up{job=\"qbit-postgres-primary\",instance=\"primary\",network=\"mainnet\"}": 1, + "pg_up{job=\"qbit-postgres-primary\",instance=\"primary\",network=\"mainnet\"}": 1, + "pg_exporter_last_scrape_error{job=\"qbit-postgres-primary\",instance=\"primary\",network=\"mainnet\"}": 1, + "pg_stat_replication_count{job=\"qbit-postgres-primary\",instance=\"primary\",network=\"mainnet\",application_name=\"prism_standby_1\"}": 1, + "pg_stat_replication_replay_lag{job=\"qbit-postgres-primary\",instance=\"primary\",network=\"mainnet\",application_name=\"prism_standby_1\"}": 0, + "pg_stat_replication_primary_flush_lsn_hi{job=\"qbit-postgres-primary\",instance=\"primary\",network=\"mainnet\",application_name=\"prism_standby_1\"}": 2097153, + "pg_stat_replication_primary_flush_lsn_lo{job=\"qbit-postgres-primary\",instance=\"primary\",network=\"mainnet\",application_name=\"prism_standby_1\"}": 100, + "pg_stat_replication_replay_lsn_hi{job=\"qbit-postgres-primary\",instance=\"primary\",network=\"mainnet\",application_name=\"prism_standby_1\"}": 2097153, + "pg_stat_replication_replay_lsn_lo{job=\"qbit-postgres-primary\",instance=\"primary\",network=\"mainnet\",application_name=\"prism_standby_1\"}": 100, + "pg_stat_replication_async{job=\"qbit-postgres-primary\",instance=\"primary\",network=\"mainnet\",application_name=\"prism_standby_1\"}": 1 + }, + "eval_time": "6m", + "expect": { + "PrismHAStandbyReplayLagHigh": 1, + "PrismHAStandbyDisconnected": 1 + } + }, + { + "name": "D3 failed exporter with previous healthy samples", + "series": { + "up{job=\"qbit-postgres-primary\",instance=\"primary\",network=\"mainnet\"}": 0, + "pg_up{job=\"qbit-postgres-primary\",instance=\"primary\",network=\"mainnet\"}": 1, + "pg_exporter_last_scrape_error{job=\"qbit-postgres-primary\",instance=\"primary\",network=\"mainnet\"}": 0, + "pg_stat_replication_count{job=\"qbit-postgres-primary\",instance=\"primary\",network=\"mainnet\",application_name=\"prism_standby_1\"}": 1, + "pg_stat_replication_replay_lag{job=\"qbit-postgres-primary\",instance=\"primary\",network=\"mainnet\",application_name=\"prism_standby_1\"}": 0, + "pg_stat_replication_primary_flush_lsn_hi{job=\"qbit-postgres-primary\",instance=\"primary\",network=\"mainnet\",application_name=\"prism_standby_1\"}": 2097153, + "pg_stat_replication_primary_flush_lsn_lo{job=\"qbit-postgres-primary\",instance=\"primary\",network=\"mainnet\",application_name=\"prism_standby_1\"}": 100, + "pg_stat_replication_replay_lsn_hi{job=\"qbit-postgres-primary\",instance=\"primary\",network=\"mainnet\",application_name=\"prism_standby_1\"}": 2097153, + "pg_stat_replication_replay_lsn_lo{job=\"qbit-postgres-primary\",instance=\"primary\",network=\"mainnet\",application_name=\"prism_standby_1\"}": 100, + "pg_stat_replication_async{job=\"qbit-postgres-primary\",instance=\"primary\",network=\"mainnet\",application_name=\"prism_standby_1\"}": 1 + }, + "eval_time": "6m", + "expect": { + "PrismHAStandbyReplayLagHigh": 1, + "PrismHAStandbyDisconnected": 1 + } + }, + { + "name": "D3 absent exporter", + "series": {}, + "expect": { + "PrismHAStandbyReplayLagHigh": 1, + "PrismHAStandbyDisconnected": 1 + }, + "alerts": { + "PrismHAStandbyReplayLagHigh": true, + "PrismHAStandbyDisconnected": true + } + }, + { + "name": "D3 missing replay position", + "series": { + "up{job=\"qbit-postgres-primary\",instance=\"primary\",network=\"mainnet\"}": 1, + "pg_up{job=\"qbit-postgres-primary\",instance=\"primary\",network=\"mainnet\"}": 1, + "pg_exporter_last_scrape_error{job=\"qbit-postgres-primary\",instance=\"primary\",network=\"mainnet\"}": 0, + "pg_stat_replication_count{job=\"qbit-postgres-primary\",instance=\"primary\",network=\"mainnet\",application_name=\"prism_standby_1\"}": 1, + "pg_stat_replication_replay_lag{job=\"qbit-postgres-primary\",instance=\"primary\",network=\"mainnet\",application_name=\"prism_standby_1\"}": 0, + "pg_stat_replication_primary_flush_lsn_hi{job=\"qbit-postgres-primary\",instance=\"primary\",network=\"mainnet\",application_name=\"prism_standby_1\"}": 2097153, + "pg_stat_replication_primary_flush_lsn_lo{job=\"qbit-postgres-primary\",instance=\"primary\",network=\"mainnet\",application_name=\"prism_standby_1\"}": 100, + "pg_stat_replication_replay_lsn_hi{job=\"qbit-postgres-primary\",instance=\"primary\",network=\"mainnet\",application_name=\"prism_standby_1\"}": 2097153, + "pg_stat_replication_async{job=\"qbit-postgres-primary\",instance=\"primary\",network=\"mainnet\",application_name=\"prism_standby_1\"}": 1 + }, + "eval_time": "6m", + "expect": { + "PrismHAStandbyReplayLagHigh": 1, + "PrismHAStandbyDisconnected": 0 + } + }, + { + "name": "D3 unknown replay position", + "series": { + "up{job=\"qbit-postgres-primary\",instance=\"primary\",network=\"mainnet\"}": 1, + "pg_up{job=\"qbit-postgres-primary\",instance=\"primary\",network=\"mainnet\"}": 1, + "pg_exporter_last_scrape_error{job=\"qbit-postgres-primary\",instance=\"primary\",network=\"mainnet\"}": 0, + "pg_stat_replication_count{job=\"qbit-postgres-primary\",instance=\"primary\",network=\"mainnet\",application_name=\"prism_standby_1\"}": 1, + "pg_stat_replication_replay_lag{job=\"qbit-postgres-primary\",instance=\"primary\",network=\"mainnet\",application_name=\"prism_standby_1\"}": 0, + "pg_stat_replication_primary_flush_lsn_hi{job=\"qbit-postgres-primary\",instance=\"primary\",network=\"mainnet\",application_name=\"prism_standby_1\"}": 2097153, + "pg_stat_replication_primary_flush_lsn_lo{job=\"qbit-postgres-primary\",instance=\"primary\",network=\"mainnet\",application_name=\"prism_standby_1\"}": 100, + "pg_stat_replication_replay_lsn_hi{job=\"qbit-postgres-primary\",instance=\"primary\",network=\"mainnet\",application_name=\"prism_standby_1\"}": 2097153, + "pg_stat_replication_replay_lsn_lo{job=\"qbit-postgres-primary\",instance=\"primary\",network=\"mainnet\",application_name=\"prism_standby_1\"}": -1, + "pg_stat_replication_async{job=\"qbit-postgres-primary\",instance=\"primary\",network=\"mainnet\",application_name=\"prism_standby_1\"}": 1 + }, + "eval_time": "6m", + "expect": { + "PrismHAStandbyReplayLagHigh": 1, + "PrismHAStandbyDisconnected": 0 + } + }, + { + "name": "D3 missing count query", + "series": { + "up{job=\"qbit-postgres-primary\",instance=\"primary\",network=\"mainnet\"}": 1, + "pg_up{job=\"qbit-postgres-primary\",instance=\"primary\",network=\"mainnet\"}": 1, + "pg_exporter_last_scrape_error{job=\"qbit-postgres-primary\",instance=\"primary\",network=\"mainnet\"}": 0, + "pg_stat_replication_replay_lag{job=\"qbit-postgres-primary\",instance=\"primary\",network=\"mainnet\",application_name=\"prism_standby_1\"}": 0, + "pg_stat_replication_primary_flush_lsn_hi{job=\"qbit-postgres-primary\",instance=\"primary\",network=\"mainnet\",application_name=\"prism_standby_1\"}": 2097153, + "pg_stat_replication_primary_flush_lsn_lo{job=\"qbit-postgres-primary\",instance=\"primary\",network=\"mainnet\",application_name=\"prism_standby_1\"}": 100, + "pg_stat_replication_replay_lsn_hi{job=\"qbit-postgres-primary\",instance=\"primary\",network=\"mainnet\",application_name=\"prism_standby_1\"}": 2097153, + "pg_stat_replication_replay_lsn_lo{job=\"qbit-postgres-primary\",instance=\"primary\",network=\"mainnet\",application_name=\"prism_standby_1\"}": 100, + "pg_stat_replication_async{job=\"qbit-postgres-primary\",instance=\"primary\",network=\"mainnet\",application_name=\"prism_standby_1\"}": 1 + }, + "eval_time": "6m", + "expect": { + "PrismHAStandbyReplayLagHigh": 1, + "PrismHAStandbyDisconnected": 1 + } + }, + { + "name": "D3 negative count", + "series": { + "up{job=\"qbit-postgres-primary\",instance=\"primary\",network=\"mainnet\"}": 1, + "pg_up{job=\"qbit-postgres-primary\",instance=\"primary\",network=\"mainnet\"}": 1, + "pg_exporter_last_scrape_error{job=\"qbit-postgres-primary\",instance=\"primary\",network=\"mainnet\"}": 0, + "pg_stat_replication_count{job=\"qbit-postgres-primary\",instance=\"primary\",network=\"mainnet\",application_name=\"prism_standby_1\"}": -1, + "pg_stat_replication_replay_lag{job=\"qbit-postgres-primary\",instance=\"primary\",network=\"mainnet\",application_name=\"prism_standby_1\"}": 0, + "pg_stat_replication_primary_flush_lsn_hi{job=\"qbit-postgres-primary\",instance=\"primary\",network=\"mainnet\",application_name=\"prism_standby_1\"}": 2097153, + "pg_stat_replication_primary_flush_lsn_lo{job=\"qbit-postgres-primary\",instance=\"primary\",network=\"mainnet\",application_name=\"prism_standby_1\"}": 100, + "pg_stat_replication_replay_lsn_hi{job=\"qbit-postgres-primary\",instance=\"primary\",network=\"mainnet\",application_name=\"prism_standby_1\"}": 2097153, + "pg_stat_replication_replay_lsn_lo{job=\"qbit-postgres-primary\",instance=\"primary\",network=\"mainnet\",application_name=\"prism_standby_1\"}": 100, + "pg_stat_replication_async{job=\"qbit-postgres-primary\",instance=\"primary\",network=\"mainnet\",application_name=\"prism_standby_1\"}": 1 + }, + "eval_time": "6m", + "expect": { + "PrismHAStandbyReplayLagHigh": 1, + "PrismHAStandbyDisconnected": 1 + } + }, + { + "name": "D3 stale count despite successful exporter", + "series": { + "up{job=\"qbit-postgres-primary\",instance=\"primary\",network=\"mainnet\"}": 1, + "pg_up{job=\"qbit-postgres-primary\",instance=\"primary\",network=\"mainnet\"}": 1, + "pg_exporter_last_scrape_error{job=\"qbit-postgres-primary\",instance=\"primary\",network=\"mainnet\"}": 0, + "pg_stat_replication_count{job=\"qbit-postgres-primary\",instance=\"primary\",network=\"mainnet\",application_name=\"prism_standby_1\"}": "1+0x357 _ _ _", + "pg_stat_replication_replay_lag{job=\"qbit-postgres-primary\",instance=\"primary\",network=\"mainnet\",application_name=\"prism_standby_1\"}": 0, + "pg_stat_replication_primary_flush_lsn_hi{job=\"qbit-postgres-primary\",instance=\"primary\",network=\"mainnet\",application_name=\"prism_standby_1\"}": 2097153, + "pg_stat_replication_primary_flush_lsn_lo{job=\"qbit-postgres-primary\",instance=\"primary\",network=\"mainnet\",application_name=\"prism_standby_1\"}": 100, + "pg_stat_replication_replay_lsn_hi{job=\"qbit-postgres-primary\",instance=\"primary\",network=\"mainnet\",application_name=\"prism_standby_1\"}": 2097153, + "pg_stat_replication_replay_lsn_lo{job=\"qbit-postgres-primary\",instance=\"primary\",network=\"mainnet\",application_name=\"prism_standby_1\"}": 100, + "pg_stat_replication_async{job=\"qbit-postgres-primary\",instance=\"primary\",network=\"mainnet\",application_name=\"prism_standby_1\"}": 1 + }, + "eval_time": "6m", + "expect": { + "PrismHAStandbyReplayLagHigh": 1, + "PrismHAStandbyDisconnected": 1 + } + }, + { + "name": "D3 stale replay position", + "series": { + "up{job=\"qbit-postgres-primary\",instance=\"primary\",network=\"mainnet\"}": 1, + "pg_up{job=\"qbit-postgres-primary\",instance=\"primary\",network=\"mainnet\"}": 1, + "pg_exporter_last_scrape_error{job=\"qbit-postgres-primary\",instance=\"primary\",network=\"mainnet\"}": 0, + "pg_stat_replication_count{job=\"qbit-postgres-primary\",instance=\"primary\",network=\"mainnet\",application_name=\"prism_standby_1\"}": 1, + "pg_stat_replication_replay_lag{job=\"qbit-postgres-primary\",instance=\"primary\",network=\"mainnet\",application_name=\"prism_standby_1\"}": 0, + "pg_stat_replication_primary_flush_lsn_hi{job=\"qbit-postgres-primary\",instance=\"primary\",network=\"mainnet\",application_name=\"prism_standby_1\"}": 2097153, + "pg_stat_replication_primary_flush_lsn_lo{job=\"qbit-postgres-primary\",instance=\"primary\",network=\"mainnet\",application_name=\"prism_standby_1\"}": 100, + "pg_stat_replication_replay_lsn_hi{job=\"qbit-postgres-primary\",instance=\"primary\",network=\"mainnet\",application_name=\"prism_standby_1\"}": 2097153, + "pg_stat_replication_replay_lsn_lo{job=\"qbit-postgres-primary\",instance=\"primary\",network=\"mainnet\",application_name=\"prism_standby_1\"}": "100+0x357 _ _ _", + "pg_stat_replication_async{job=\"qbit-postgres-primary\",instance=\"primary\",network=\"mainnet\",application_name=\"prism_standby_1\"}": 1 + }, + "eval_time": "6m", + "expect": { + "PrismHAStandbyReplayLagHigh": 1, + "PrismHAStandbyDisconnected": 0 + } + }, + { + "name": "D3 missing recent history", + "series": { + "up{job=\"qbit-postgres-primary\",instance=\"primary\",network=\"mainnet\"}": 1, + "pg_up{job=\"qbit-postgres-primary\",instance=\"primary\",network=\"mainnet\"}": 1, + "pg_exporter_last_scrape_error{job=\"qbit-postgres-primary\",instance=\"primary\",network=\"mainnet\"}": 0, + "pg_stat_replication_count{job=\"qbit-postgres-primary\",instance=\"primary\",network=\"mainnet\",application_name=\"prism_standby_1\"}": 1, + "pg_stat_replication_replay_lag{job=\"qbit-postgres-primary\",instance=\"primary\",network=\"mainnet\",application_name=\"prism_standby_1\"}": 0, + "pg_stat_replication_primary_flush_lsn_hi{job=\"qbit-postgres-primary\",instance=\"primary\",network=\"mainnet\",application_name=\"prism_standby_1\"}": 2097153, + "pg_stat_replication_primary_flush_lsn_lo{job=\"qbit-postgres-primary\",instance=\"primary\",network=\"mainnet\",application_name=\"prism_standby_1\"}": "100+0x352 _ _ _ _ _ _ 100 100", + "pg_stat_replication_replay_lsn_hi{job=\"qbit-postgres-primary\",instance=\"primary\",network=\"mainnet\",application_name=\"prism_standby_1\"}": 2097153, + "pg_stat_replication_replay_lsn_lo{job=\"qbit-postgres-primary\",instance=\"primary\",network=\"mainnet\",application_name=\"prism_standby_1\"}": 100, + "pg_stat_replication_async{job=\"qbit-postgres-primary\",instance=\"primary\",network=\"mainnet\",application_name=\"prism_standby_1\"}": 1 + }, + "eval_time": "6m", + "expect": { + "PrismHAStandbyReplayLagHigh": 1, + "PrismHAStandbyDisconnected": 0 + } + }, + { + "name": "D3 missing history at startup is unknown", + "series": { + "up{job=\"qbit-postgres-primary\",instance=\"primary\",network=\"mainnet\"}": 1, + "pg_up{job=\"qbit-postgres-primary\",instance=\"primary\",network=\"mainnet\"}": 1, + "pg_exporter_last_scrape_error{job=\"qbit-postgres-primary\",instance=\"primary\",network=\"mainnet\"}": 0, + "pg_stat_replication_count{job=\"qbit-postgres-primary\",instance=\"primary\",network=\"mainnet\",application_name=\"prism_standby_1\"}": 1, + "pg_stat_replication_replay_lag{job=\"qbit-postgres-primary\",instance=\"primary\",network=\"mainnet\",application_name=\"prism_standby_1\"}": 0, + "pg_stat_replication_primary_flush_lsn_hi{job=\"qbit-postgres-primary\",instance=\"primary\",network=\"mainnet\",application_name=\"prism_standby_1\"}": 2097153, + "pg_stat_replication_primary_flush_lsn_lo{job=\"qbit-postgres-primary\",instance=\"primary\",network=\"mainnet\",application_name=\"prism_standby_1\"}": 100, + "pg_stat_replication_replay_lsn_hi{job=\"qbit-postgres-primary\",instance=\"primary\",network=\"mainnet\",application_name=\"prism_standby_1\"}": 2097153, + "pg_stat_replication_replay_lsn_lo{job=\"qbit-postgres-primary\",instance=\"primary\",network=\"mainnet\",application_name=\"prism_standby_1\"}": 100, + "pg_stat_replication_async{job=\"qbit-postgres-primary\",instance=\"primary\",network=\"mainnet\",application_name=\"prism_standby_1\"}": 1 + }, + "eval_time": "3s", + "expect": { + "PrismHAStandbyReplayLagHigh": 1, + "PrismHAStandbyDisconnected": 0 + }, + "alerts": { + "PrismHAStandbyReplayLagHigh": false, + "PrismHAStandbyDisconnected": false + } + }, + { + "name": "D3 one-minute dwell has not yet elapsed", + "series": { + "up{job=\"qbit-postgres-primary\",instance=\"primary\",network=\"mainnet\"}": 1, + "pg_up{job=\"qbit-postgres-primary\",instance=\"primary\",network=\"mainnet\"}": 1, + "pg_exporter_last_scrape_error{job=\"qbit-postgres-primary\",instance=\"primary\",network=\"mainnet\"}": 0, + "pg_stat_replication_count{job=\"qbit-postgres-primary\",instance=\"primary\",network=\"mainnet\",application_name=\"prism_standby_1\"}": 1, + "pg_stat_replication_replay_lag{job=\"qbit-postgres-primary\",instance=\"primary\",network=\"mainnet\",application_name=\"prism_standby_1\"}": 0, + "pg_stat_replication_primary_flush_lsn_hi{job=\"qbit-postgres-primary\",instance=\"primary\",network=\"mainnet\",application_name=\"prism_standby_1\"}": 2097153, + "pg_stat_replication_primary_flush_lsn_lo{job=\"qbit-postgres-primary\",instance=\"primary\",network=\"mainnet\",application_name=\"prism_standby_1\"}": 100, + "pg_stat_replication_replay_lsn_hi{job=\"qbit-postgres-primary\",instance=\"primary\",network=\"mainnet\",application_name=\"prism_standby_1\"}": 2097153, + "pg_stat_replication_replay_lsn_lo{job=\"qbit-postgres-primary\",instance=\"primary\",network=\"mainnet\",application_name=\"prism_standby_1\"}": 99, + "pg_stat_replication_async{job=\"qbit-postgres-primary\",instance=\"primary\",network=\"mainnet\",application_name=\"prism_standby_1\"}": 1 + }, + "eval_time": "59s", + "expect": { + "PrismHAStandbyReplayLagHigh": 1, + "PrismHAStandbyDisconnected": 0 + }, + "alerts": { + "PrismHAStandbyReplayLagHigh": false, + "PrismHAStandbyDisconnected": false + } + }, + { + "name": "D3 one-minute dwell elapsed", + "series": { + "up{job=\"qbit-postgres-primary\",instance=\"primary\",network=\"mainnet\"}": 1, + "pg_up{job=\"qbit-postgres-primary\",instance=\"primary\",network=\"mainnet\"}": 1, + "pg_exporter_last_scrape_error{job=\"qbit-postgres-primary\",instance=\"primary\",network=\"mainnet\"}": 0, + "pg_stat_replication_count{job=\"qbit-postgres-primary\",instance=\"primary\",network=\"mainnet\",application_name=\"prism_standby_1\"}": 1, + "pg_stat_replication_replay_lag{job=\"qbit-postgres-primary\",instance=\"primary\",network=\"mainnet\",application_name=\"prism_standby_1\"}": 0, + "pg_stat_replication_primary_flush_lsn_hi{job=\"qbit-postgres-primary\",instance=\"primary\",network=\"mainnet\",application_name=\"prism_standby_1\"}": 2097153, + "pg_stat_replication_primary_flush_lsn_lo{job=\"qbit-postgres-primary\",instance=\"primary\",network=\"mainnet\",application_name=\"prism_standby_1\"}": 100, + "pg_stat_replication_replay_lsn_hi{job=\"qbit-postgres-primary\",instance=\"primary\",network=\"mainnet\",application_name=\"prism_standby_1\"}": 2097153, + "pg_stat_replication_replay_lsn_lo{job=\"qbit-postgres-primary\",instance=\"primary\",network=\"mainnet\",application_name=\"prism_standby_1\"}": 99, + "pg_stat_replication_async{job=\"qbit-postgres-primary\",instance=\"primary\",network=\"mainnet\",application_name=\"prism_standby_1\"}": 1 + }, + "eval_time": "60s", + "expect": { + "PrismHAStandbyReplayLagHigh": 1, + "PrismHAStandbyDisconnected": 0 + }, + "alerts": { + "PrismHAStandbyReplayLagHigh": true, + "PrismHAStandbyDisconnected": false + } + }, + { + "name": "D3 synchronous standby violates topology", + "series": { + "up{job=\"qbit-postgres-primary\",instance=\"primary\",network=\"mainnet\"}": 1, + "pg_up{job=\"qbit-postgres-primary\",instance=\"primary\",network=\"mainnet\"}": 1, + "pg_exporter_last_scrape_error{job=\"qbit-postgres-primary\",instance=\"primary\",network=\"mainnet\"}": 0, + "pg_stat_replication_count{job=\"qbit-postgres-primary\",instance=\"primary\",network=\"mainnet\",application_name=\"prism_standby_1\"}": 1, + "pg_stat_replication_replay_lag{job=\"qbit-postgres-primary\",instance=\"primary\",network=\"mainnet\",application_name=\"prism_standby_1\"}": 0, + "pg_stat_replication_primary_flush_lsn_hi{job=\"qbit-postgres-primary\",instance=\"primary\",network=\"mainnet\",application_name=\"prism_standby_1\"}": 2097153, + "pg_stat_replication_primary_flush_lsn_lo{job=\"qbit-postgres-primary\",instance=\"primary\",network=\"mainnet\",application_name=\"prism_standby_1\"}": 100, + "pg_stat_replication_replay_lsn_hi{job=\"qbit-postgres-primary\",instance=\"primary\",network=\"mainnet\",application_name=\"prism_standby_1\"}": 2097153, + "pg_stat_replication_replay_lsn_lo{job=\"qbit-postgres-primary\",instance=\"primary\",network=\"mainnet\",application_name=\"prism_standby_1\"}": 100, + "pg_stat_replication_async{job=\"qbit-postgres-primary\",instance=\"primary\",network=\"mainnet\",application_name=\"prism_standby_1\"}": 0 + }, + "eval_time": "6m", + "expect": { + "PrismHAStandbyReplayLagHigh": 1, + "PrismHAStandbyDisconnected": 0 + } + }, + { + "name": "D3 unknown async status", + "series": { + "up{job=\"qbit-postgres-primary\",instance=\"primary\",network=\"mainnet\"}": 1, + "pg_up{job=\"qbit-postgres-primary\",instance=\"primary\",network=\"mainnet\"}": 1, + "pg_exporter_last_scrape_error{job=\"qbit-postgres-primary\",instance=\"primary\",network=\"mainnet\"}": 0, + "pg_stat_replication_count{job=\"qbit-postgres-primary\",instance=\"primary\",network=\"mainnet\",application_name=\"prism_standby_1\"}": 1, + "pg_stat_replication_replay_lag{job=\"qbit-postgres-primary\",instance=\"primary\",network=\"mainnet\",application_name=\"prism_standby_1\"}": 0, + "pg_stat_replication_primary_flush_lsn_hi{job=\"qbit-postgres-primary\",instance=\"primary\",network=\"mainnet\",application_name=\"prism_standby_1\"}": 2097153, + "pg_stat_replication_primary_flush_lsn_lo{job=\"qbit-postgres-primary\",instance=\"primary\",network=\"mainnet\",application_name=\"prism_standby_1\"}": 100, + "pg_stat_replication_replay_lsn_hi{job=\"qbit-postgres-primary\",instance=\"primary\",network=\"mainnet\",application_name=\"prism_standby_1\"}": 2097153, + "pg_stat_replication_replay_lsn_lo{job=\"qbit-postgres-primary\",instance=\"primary\",network=\"mainnet\",application_name=\"prism_standby_1\"}": 100, + "pg_stat_replication_async{job=\"qbit-postgres-primary\",instance=\"primary\",network=\"mainnet\",application_name=\"prism_standby_1\"}": -1 + }, + "eval_time": "6m", + "expect": { + "PrismHAStandbyReplayLagHigh": 1, + "PrismHAStandbyDisconnected": 0 + } + }, + { + "name": "D3 stale async status", + "series": { + "up{job=\"qbit-postgres-primary\",instance=\"primary\",network=\"mainnet\"}": 1, + "pg_up{job=\"qbit-postgres-primary\",instance=\"primary\",network=\"mainnet\"}": 1, + "pg_exporter_last_scrape_error{job=\"qbit-postgres-primary\",instance=\"primary\",network=\"mainnet\"}": 0, + "pg_stat_replication_count{job=\"qbit-postgres-primary\",instance=\"primary\",network=\"mainnet\",application_name=\"prism_standby_1\"}": 1, + "pg_stat_replication_replay_lag{job=\"qbit-postgres-primary\",instance=\"primary\",network=\"mainnet\",application_name=\"prism_standby_1\"}": 0, + "pg_stat_replication_primary_flush_lsn_hi{job=\"qbit-postgres-primary\",instance=\"primary\",network=\"mainnet\",application_name=\"prism_standby_1\"}": 2097153, + "pg_stat_replication_primary_flush_lsn_lo{job=\"qbit-postgres-primary\",instance=\"primary\",network=\"mainnet\",application_name=\"prism_standby_1\"}": 100, + "pg_stat_replication_replay_lsn_hi{job=\"qbit-postgres-primary\",instance=\"primary\",network=\"mainnet\",application_name=\"prism_standby_1\"}": 2097153, + "pg_stat_replication_replay_lsn_lo{job=\"qbit-postgres-primary\",instance=\"primary\",network=\"mainnet\",application_name=\"prism_standby_1\"}": 100, + "pg_stat_replication_async{job=\"qbit-postgres-primary\",instance=\"primary\",network=\"mainnet\",application_name=\"prism_standby_1\"}": "1+0x357 _ _ _" + }, + "eval_time": "6m", + "expect": { + "PrismHAStandbyReplayLagHigh": 1, + "PrismHAStandbyDisconnected": 0 + } + }, + { + "name": "D3 missing async status", + "series": { + "up{job=\"qbit-postgres-primary\",instance=\"primary\",network=\"mainnet\"}": 1, + "pg_up{job=\"qbit-postgres-primary\",instance=\"primary\",network=\"mainnet\"}": 1, + "pg_exporter_last_scrape_error{job=\"qbit-postgres-primary\",instance=\"primary\",network=\"mainnet\"}": 0, + "pg_stat_replication_count{job=\"qbit-postgres-primary\",instance=\"primary\",network=\"mainnet\",application_name=\"prism_standby_1\"}": 1, + "pg_stat_replication_replay_lag{job=\"qbit-postgres-primary\",instance=\"primary\",network=\"mainnet\",application_name=\"prism_standby_1\"}": 0, + "pg_stat_replication_primary_flush_lsn_hi{job=\"qbit-postgres-primary\",instance=\"primary\",network=\"mainnet\",application_name=\"prism_standby_1\"}": 2097153, + "pg_stat_replication_primary_flush_lsn_lo{job=\"qbit-postgres-primary\",instance=\"primary\",network=\"mainnet\",application_name=\"prism_standby_1\"}": 100, + "pg_stat_replication_replay_lsn_hi{job=\"qbit-postgres-primary\",instance=\"primary\",network=\"mainnet\",application_name=\"prism_standby_1\"}": 2097153, + "pg_stat_replication_replay_lsn_lo{job=\"qbit-postgres-primary\",instance=\"primary\",network=\"mainnet\",application_name=\"prism_standby_1\"}": 100 + }, + "eval_time": "6m", + "expect": { + "PrismHAStandbyReplayLagHigh": 1, + "PrismHAStandbyDisconnected": 0 + } + }, + { + "name": "landing no pending delivery", + "series": { + "qbit_prism_accepted_block_revision_work_pending_seconds{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\"}": 0, + "up{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\"}": 1, + "qbit_prism_accepted_block_revision_work_tracking_unknown{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\"}": 0, + "qbit_prism_authorized_clients{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\"}": 1, + "qbit_prism_accepted_block_unlanded_seconds{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\"}": 0 + }, + "expect": { + "PrismAcceptedRevisionWorkPending": 0, + "PrismAcceptedRevisionWorkPendingCritical": 0, + "PrismAcceptedRevisionWorkTrackingUnknown": 0, + "PrismAcceptedBlockUnlanded": 0 + }, + "alerts": { + "PrismAcceptedRevisionWorkPending": [], + "PrismAcceptedRevisionWorkPendingCritical": [], + "PrismAcceptedRevisionWorkTrackingUnknown": [], + "PrismAcceptedBlockUnlanded": [] + } + }, + { + "name": "landing one second boundary", + "series": { + "qbit_prism_accepted_block_revision_work_pending_seconds{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\"}": 1, + "up{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\"}": 1, + "qbit_prism_accepted_block_revision_work_tracking_unknown{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\"}": 0, + "qbit_prism_authorized_clients{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\"}": 1 + }, + "expect": { + "PrismAcceptedRevisionWorkPending": 0, + "PrismAcceptedRevisionWorkPendingCritical": 0 + }, + "alerts": { + "PrismAcceptedRevisionWorkPending": [], + "PrismAcceptedRevisionWorkPendingCritical": [] + } + }, + { + "name": "landing early pending warning", + "series": { + "qbit_prism_accepted_block_revision_work_pending_seconds{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\"}": 1.01, + "up{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\"}": 1, + "qbit_prism_accepted_block_revision_work_tracking_unknown{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\"}": 0, + "qbit_prism_authorized_clients{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\"}": 1 + }, + "expect": { + "PrismAcceptedRevisionWorkPending": 1, + "PrismAcceptedRevisionWorkPendingCritical": 0, + "PrismAcceptedRevisionWorkTrackingUnknown": 0 + }, + "alerts": { + "PrismAcceptedRevisionWorkPending": [ + { + "job": "qbit-prism", + "instance": "one", + "network": "mainnet" + } + ], + "PrismAcceptedRevisionWorkPendingCritical": [], + "PrismAcceptedRevisionWorkTrackingUnknown": [] + } + }, + { + "name": "landing before incident critical floor", + "series": { + "qbit_prism_accepted_block_revision_work_pending_seconds{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\"}": 306.99, + "up{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\"}": 1, + "qbit_prism_accepted_block_revision_work_tracking_unknown{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\"}": 0, + "qbit_prism_authorized_clients{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\"}": 1 + }, + "expect": { + "PrismAcceptedRevisionWorkPending": 1, + "PrismAcceptedRevisionWorkPendingCritical": 0 + }, + "alerts": { + "PrismAcceptedRevisionWorkPending": [ + { + "job": "qbit-prism", + "instance": "one", + "network": "mainnet" + } + ], + "PrismAcceptedRevisionWorkPendingCritical": [] + } + }, + { + "name": "landing incident critical floor", + "series": { + "qbit_prism_accepted_block_revision_work_pending_seconds{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\"}": 307, + "up{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\"}": 1, + "qbit_prism_accepted_block_revision_work_tracking_unknown{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\"}": 0, + "qbit_prism_authorized_clients{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\"}": 1 + }, + "expect": { + "PrismAcceptedRevisionWorkPending": 1, + "PrismAcceptedRevisionWorkPendingCritical": 1 + }, + "alerts": { + "PrismAcceptedRevisionWorkPending": [ + { + "job": "qbit-prism", + "instance": "one", + "network": "mainnet" + } + ], + "PrismAcceptedRevisionWorkPendingCritical": [ + { + "job": "qbit-prism", + "instance": "one", + "network": "mainnet" + } + ] + } + }, + { + "name": "landing tracking unknown", + "series": { + "qbit_prism_accepted_block_revision_work_pending_seconds{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\"}": -1, + "up{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\"}": 1, + "qbit_prism_accepted_block_revision_work_tracking_unknown{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\"}": 1, + "qbit_prism_authorized_clients{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\"}": 1 + }, + "expect": { + "PrismAcceptedRevisionWorkPending": 0, + "PrismAcceptedRevisionWorkPendingCritical": null, + "PrismAcceptedRevisionWorkTrackingUnknown": 1 + }, + "alerts": { + "PrismAcceptedRevisionWorkPending": [], + "PrismAcceptedRevisionWorkPendingCritical": [], + "PrismAcceptedRevisionWorkTrackingUnknown": [ + { + "job": "qbit-prism", + "instance": "one", + "network": "mainnet" + } + ] + } + }, + { + "name": "landing missing gauge", + "series": { + "up{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\"}": 1 + }, + "expect": { + "PrismAcceptedRevisionWorkPending": 1, + "PrismAcceptedRevisionWorkPendingCritical": null, + "PrismAcceptedRevisionWorkTrackingUnknown": null + }, + "alerts": { + "PrismAcceptedRevisionWorkPending": [ + { + "job": "qbit-prism", + "instance": "one", + "network": "mainnet" + } + ], + "PrismAcceptedRevisionWorkPendingCritical": [], + "PrismAcceptedRevisionWorkTrackingUnknown": [] + } + }, + { + "name": "landing scrape down", + "series": { + "qbit_prism_accepted_block_revision_work_pending_seconds{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\"}": 0, + "up{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\"}": 0, + "qbit_prism_accepted_block_revision_work_tracking_unknown{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\"}": 0 + }, + "expect": { + "PrismAcceptedRevisionWorkPending": 1, + "PrismAcceptedRevisionWorkPendingCritical": null, + "PrismAcceptedRevisionWorkTrackingUnknown": 0 + }, + "alerts": { + "PrismAcceptedRevisionWorkPending": [ + { + "job": "qbit-prism", + "instance": "one", + "network": "mainnet" + } + ], + "PrismAcceptedRevisionWorkPendingCritical": [], + "PrismAcceptedRevisionWorkTrackingUnknown": [] + } + }, + { + "name": "landing one peer missing", + "series": { + "qbit_prism_accepted_block_revision_work_pending_seconds{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\"}": 0, + "up{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\"}": 1, + "up{job=\"qbit-prism\",instance=\"two\",network=\"mainnet\"}": 1, + "qbit_prism_accepted_block_revision_work_tracking_unknown{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\"}": 0 + }, + "expect": { + "PrismAcceptedRevisionWorkPending": 1, + "PrismAcceptedRevisionWorkTrackingUnknown": 0 + } + }, + { + "name": "landing success has no timeout", + "series": { + "qbit_prism_revision_work_build_timeouts_total{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\"}": "0+0x360" + }, + "eval_time": "2m", + "expect": { + "PrismRevisionWorkBuildTimeouts": 0 + }, + "alerts": { + "PrismRevisionWorkBuildTimeouts": [] + } + }, + { + "name": "landing actual timeout", + "series": { + "qbit_prism_revision_work_build_timeouts_total{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\"}": "0+0x60 1+0x299" + }, + "eval_time": "2m", + "expect": { + "PrismRevisionWorkBuildTimeouts": 1 + }, + "alerts": { + "PrismRevisionWorkBuildTimeouts": [ + { + "job": "qbit-prism", + "instance": "one", + "network": "mainnet" + } + ] + } + }, + { + "name": "landing process reset is not a timeout", + "series": { + "qbit_prism_revision_work_build_timeouts_total{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\"}": "5+0x60 0+0x299" + }, + "eval_time": "2m", + "expect": { + "PrismRevisionWorkBuildTimeouts": 0 + }, + "alerts": { + "PrismRevisionWorkBuildTimeouts": [] + } + }, + { + "name": "landing lost commit reply stays unknown with no timeout", + "series": { + "qbit_prism_accepted_block_revision_work_pending_seconds{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\"}": -1, + "up{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\"}": 1, + "qbit_prism_accepted_block_revision_work_tracking_unknown{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\"}": 1, + "qbit_prism_authorized_clients{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\"}": 1, + "qbit_prism_revision_work_build_timeouts_total{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\"}": "0+0x360" + }, + "eval_time": "2m", + "expect": { + "PrismAcceptedRevisionWorkPending": 0, + "PrismAcceptedRevisionWorkPendingCritical": null, + "PrismRevisionWorkBuildTimeouts": 0, + "PrismAcceptedRevisionWorkTrackingUnknown": 1 + }, + "alerts": { + "PrismAcceptedRevisionWorkPending": [], + "PrismAcceptedRevisionWorkPendingCritical": [], + "PrismRevisionWorkBuildTimeouts": [], + "PrismAcceptedRevisionWorkTrackingUnknown": [ + { + "job": "qbit-prism", + "instance": "one", + "network": "mainnet" + } + ] + } + }, + { + "name": "landing lost commit reply beside a known stall and a real timeout", + "series": { + "qbit_prism_accepted_block_revision_work_pending_seconds{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\"}": 308, + "up{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\"}": 1, + "qbit_prism_accepted_block_revision_work_tracking_unknown{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\"}": 1, + "qbit_prism_authorized_clients{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\"}": 1, + "qbit_prism_revision_work_build_timeouts_total{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\"}": "0+0x60 1+0x299" + }, + "eval_time": "2m", + "expect": { + "PrismAcceptedRevisionWorkPending": 1, + "PrismAcceptedRevisionWorkPendingCritical": 1, + "PrismRevisionWorkBuildTimeouts": 1, + "PrismAcceptedRevisionWorkTrackingUnknown": 1 + }, + "alerts": { + "PrismAcceptedRevisionWorkPending": [ + { + "job": "qbit-prism", + "instance": "one", + "network": "mainnet" + } + ], + "PrismAcceptedRevisionWorkPendingCritical": [ + { + "job": "qbit-prism", + "instance": "one", + "network": "mainnet" + } + ], + "PrismRevisionWorkBuildTimeouts": [ + { + "job": "qbit-prism", + "instance": "one", + "network": "mainnet" + } + ], + "PrismAcceptedRevisionWorkTrackingUnknown": [ + { + "job": "qbit-prism", + "instance": "one", + "network": "mainnet" + } + ] + } + }, + { + "name": "landing increasing age at 1s", + "series": { + "qbit_prism_accepted_block_revision_work_pending_seconds{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\"}": "0+1x360", + "up{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\"}": 1, + "qbit_prism_accepted_block_revision_work_tracking_unknown{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\"}": 0, + "qbit_prism_authorized_clients{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\"}": 1 + }, + "eval_time": "1s", + "expect": { + "PrismAcceptedRevisionWorkPending": 0, + "PrismAcceptedRevisionWorkPendingCritical": 0 + }, + "alerts": { + "PrismAcceptedRevisionWorkPending": [], + "PrismAcceptedRevisionWorkPendingCritical": [] + } + }, + { + "name": "landing increasing age at 2s", + "series": { + "qbit_prism_accepted_block_revision_work_pending_seconds{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\"}": "0+1x360", + "up{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\"}": 1, + "qbit_prism_accepted_block_revision_work_tracking_unknown{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\"}": 0, + "qbit_prism_authorized_clients{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\"}": 1 + }, + "eval_time": "2s", + "expect": { + "PrismAcceptedRevisionWorkPending": 1, + "PrismAcceptedRevisionWorkPendingCritical": 0 + }, + "alerts": { + "PrismAcceptedRevisionWorkPending": [ + { + "job": "qbit-prism", + "instance": "one", + "network": "mainnet" + } + ], + "PrismAcceptedRevisionWorkPendingCritical": [] + } + }, + { + "name": "landing increasing age at 306s", + "series": { + "qbit_prism_accepted_block_revision_work_pending_seconds{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\"}": "0+1x360", + "up{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\"}": 1, + "qbit_prism_accepted_block_revision_work_tracking_unknown{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\"}": 0, + "qbit_prism_authorized_clients{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\"}": 1 + }, + "eval_time": "306s", + "expect": { + "PrismAcceptedRevisionWorkPending": 1, + "PrismAcceptedRevisionWorkPendingCritical": 0 + }, + "alerts": { + "PrismAcceptedRevisionWorkPending": [ + { + "job": "qbit-prism", + "instance": "one", + "network": "mainnet" + } + ], + "PrismAcceptedRevisionWorkPendingCritical": [] + } + }, + { + "name": "landing increasing age at 307s", + "series": { + "qbit_prism_accepted_block_revision_work_pending_seconds{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\"}": "0+1x360", + "up{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\"}": 1, + "qbit_prism_accepted_block_revision_work_tracking_unknown{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\"}": 0, + "qbit_prism_authorized_clients{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\"}": 1 + }, + "eval_time": "307s", + "expect": { + "PrismAcceptedRevisionWorkPending": 1, + "PrismAcceptedRevisionWorkPendingCritical": 1 + }, + "alerts": { + "PrismAcceptedRevisionWorkPending": [ + { + "job": "qbit-prism", + "instance": "one", + "network": "mainnet" + } + ], + "PrismAcceptedRevisionWorkPendingCritical": [] + } + }, + { + "name": "landing unknown A and known B critical age", + "series": { + "qbit_prism_accepted_block_revision_work_pending_seconds{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\"}": 308, + "up{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\"}": 1, + "qbit_prism_accepted_block_revision_work_tracking_unknown{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\"}": 1, + "qbit_prism_authorized_clients{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\"}": 1 + }, + "eval_time": "2m", + "expect": { + "PrismAcceptedRevisionWorkPending": 1, + "PrismAcceptedRevisionWorkPendingCritical": 1, + "PrismAcceptedRevisionWorkTrackingUnknown": 1 + }, + "alerts": { + "PrismAcceptedRevisionWorkPending": [ + { + "job": "qbit-prism", + "instance": "one", + "network": "mainnet" + } + ], + "PrismAcceptedRevisionWorkPendingCritical": [ + { + "job": "qbit-prism", + "instance": "one", + "network": "mainnet" + } + ], + "PrismAcceptedRevisionWorkTrackingUnknown": [ + { + "job": "qbit-prism", + "instance": "one", + "network": "mainnet" + } + ] + } + }, + { + "name": "landing unknown A and known B subwarning age", + "series": { + "qbit_prism_accepted_block_revision_work_pending_seconds{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\"}": 0.5, + "up{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\"}": 1, + "qbit_prism_accepted_block_revision_work_tracking_unknown{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\"}": 1, + "qbit_prism_authorized_clients{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\"}": 1 + }, + "eval_time": "2m", + "expect": { + "PrismAcceptedRevisionWorkPending": 0, + "PrismAcceptedRevisionWorkPendingCritical": 0, + "PrismAcceptedRevisionWorkTrackingUnknown": 1 + }, + "alerts": { + "PrismAcceptedRevisionWorkPending": [], + "PrismAcceptedRevisionWorkPendingCritical": [], + "PrismAcceptedRevisionWorkTrackingUnknown": [ + { + "job": "qbit-prism", + "instance": "one", + "network": "mainnet" + } + ] + } + }, + { + "name": "landing missing tracking-unknown gauge", + "series": { + "qbit_prism_accepted_block_revision_work_pending_seconds{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\"}": 0, + "up{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\"}": 1, + "qbit_prism_authorized_clients{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\"}": 1 + }, + "eval_time": "2m", + "expect": { + "PrismAcceptedRevisionWorkPending": 0, + "PrismAcceptedRevisionWorkPendingCritical": 0, + "PrismAcceptedRevisionWorkTrackingUnknown": 1 + }, + "alerts": { + "PrismAcceptedRevisionWorkPending": [], + "PrismAcceptedRevisionWorkPendingCritical": [], + "PrismAcceptedRevisionWorkTrackingUnknown": [ + { + "job": "qbit-prism", + "instance": "one", + "network": "mainnet" + } + ] + } + }, + { + "name": "landing increasing age fires after the dwell", + "series": { + "qbit_prism_accepted_block_revision_work_pending_seconds{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\"}": "0+1x420", + "qbit_prism_accepted_block_revision_work_tracking_unknown{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\"}": 0, + "up{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\"}": 1, + "qbit_prism_authorized_clients{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\"}": 3 + }, + "expect": { + "PrismAcceptedRevisionWorkPending": 1, + "PrismAcceptedRevisionWorkPendingCritical": 1 + }, + "alerts": { + "PrismAcceptedRevisionWorkPending": [ + { + "job": "qbit-prism", + "instance": "one", + "network": "mainnet" + } + ], + "PrismAcceptedRevisionWorkPendingCritical": [ + { + "job": "qbit-prism", + "instance": "one", + "network": "mainnet" + } + ] + }, + "eval_time": "6m7s" + }, + { + "name": "landing critical pages a prolonged known stall with miners connected", + "series": { + "qbit_prism_accepted_block_revision_work_pending_seconds{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\"}": 400, + "qbit_prism_accepted_block_revision_work_tracking_unknown{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\"}": 0, + "up{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\"}": 1, + "qbit_prism_authorized_clients{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\"}": 3 + }, + "expect": { + "PrismAcceptedRevisionWorkPending": 1, + "PrismAcceptedRevisionWorkPendingCritical": 1 + }, + "alerts": { + "PrismAcceptedRevisionWorkPending": [ + { + "job": "qbit-prism", + "instance": "one", + "network": "mainnet" + } + ], + "PrismAcceptedRevisionWorkPendingCritical": [ + { + "job": "qbit-prism", + "instance": "one", + "network": "mainnet" + } + ] + } + }, + { + "name": "landing critical is pending under the dwell", + "series": { + "qbit_prism_accepted_block_revision_work_pending_seconds{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\"}": 400, + "qbit_prism_accepted_block_revision_work_tracking_unknown{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\"}": 0, + "up{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\"}": 1, + "qbit_prism_authorized_clients{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\"}": 3 + }, + "expect": { + "PrismAcceptedRevisionWorkPending": 1, + "PrismAcceptedRevisionWorkPendingCritical": 1 + }, + "alerts": { + "PrismAcceptedRevisionWorkPending": [ + { + "job": "qbit-prism", + "instance": "one", + "network": "mainnet" + } + ], + "PrismAcceptedRevisionWorkPendingCritical": [] + }, + "eval_time": "30s" + }, + { + "name": "landing critical fires once the dwell elapses", + "series": { + "qbit_prism_accepted_block_revision_work_pending_seconds{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\"}": 400, + "qbit_prism_accepted_block_revision_work_tracking_unknown{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\"}": 0, + "up{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\"}": 1, + "qbit_prism_authorized_clients{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\"}": 3 + }, + "expect": { + "PrismAcceptedRevisionWorkPendingCritical": 1 + }, + "alerts": { + "PrismAcceptedRevisionWorkPendingCritical": [ + { + "job": "qbit-prism", + "instance": "one", + "network": "mainnet" + } + ] + }, + "eval_time": "1m" + }, + { + "name": "landing critical needs connected miners", + "series": { + "qbit_prism_accepted_block_revision_work_pending_seconds{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\"}": 400, + "qbit_prism_accepted_block_revision_work_tracking_unknown{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\"}": 0, + "up{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\"}": 1, + "qbit_prism_authorized_clients{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\"}": 0 + }, + "expect": { + "PrismAcceptedRevisionWorkPending": 1, + "PrismAcceptedRevisionWorkPendingCritical": null + }, + "alerts": { + "PrismAcceptedRevisionWorkPending": [ + { + "job": "qbit-prism", + "instance": "one", + "network": "mainnet" + } + ], + "PrismAcceptedRevisionWorkPendingCritical": [] + } + }, + { + "name": "landing critical pages through a stale snapshot", + "series": { + "qbit_prism_accepted_block_revision_work_pending_seconds{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\"}": 400, + "qbit_prism_accepted_block_revision_work_tracking_unknown{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\"}": 0, + "up{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\"}": 1, + "qbit_prism_authorized_clients{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\"}": 3, + "qbit_prism_metrics_snapshot_available{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\"}": 1, + "qbit_prism_metrics_snapshot_stale{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\"}": 1 + }, + "expect": { + "PrismAcceptedRevisionWorkPending": 1, + "PrismAcceptedRevisionWorkPendingCritical": 1 + }, + "alerts": { + "PrismAcceptedRevisionWorkPending": [ + { + "job": "qbit-prism", + "instance": "one", + "network": "mainnet" + } + ], + "PrismAcceptedRevisionWorkPendingCritical": [ + { + "job": "qbit-prism", + "instance": "one", + "network": "mainnet" + } + ] + } + }, + { + "name": "landing critical pages through an unavailable snapshot", + "series": { + "qbit_prism_accepted_block_revision_work_pending_seconds{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\"}": 400, + "qbit_prism_accepted_block_revision_work_tracking_unknown{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\"}": 0, + "up{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\"}": 1, + "qbit_prism_authorized_clients{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\"}": 3, + "qbit_prism_metrics_snapshot_available{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\"}": 0, + "qbit_prism_metrics_snapshot_stale{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\"}": 0 + }, + "expect": { + "PrismAcceptedRevisionWorkPending": 1, + "PrismAcceptedRevisionWorkPendingCritical": 1 + }, + "alerts": { + "PrismAcceptedRevisionWorkPending": [ + { + "job": "qbit-prism", + "instance": "one", + "network": "mainnet" + } + ], + "PrismAcceptedRevisionWorkPendingCritical": [ + { + "job": "qbit-prism", + "instance": "one", + "network": "mainnet" + } + ] + } + }, + { + "name": "landing critical needs a scraped target", + "series": { + "qbit_prism_accepted_block_revision_work_pending_seconds{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\"}": 400, + "qbit_prism_accepted_block_revision_work_tracking_unknown{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\"}": 0, + "up{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\"}": 0, + "qbit_prism_authorized_clients{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\"}": 3 + }, + "expect": { + "PrismAcceptedRevisionWorkPending": 1, + "PrismAcceptedRevisionWorkPendingCritical": null + }, + "alerts": { + "PrismAcceptedRevisionWorkPending": [ + { + "job": "qbit-prism", + "instance": "one", + "network": "mainnet" + } + ], + "PrismAcceptedRevisionWorkPendingCritical": [] + } + }, + { + "name": "landing critical needs a known age", + "series": { + "qbit_prism_accepted_block_revision_work_pending_seconds{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\"}": -1, + "qbit_prism_accepted_block_revision_work_tracking_unknown{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\"}": 1, + "up{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\"}": 1, + "qbit_prism_authorized_clients{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\"}": 3 + }, + "expect": { + "PrismAcceptedRevisionWorkPending": 0, + "PrismAcceptedRevisionWorkPendingCritical": null, + "PrismAcceptedRevisionWorkTrackingUnknown": 1 + }, + "alerts": { + "PrismAcceptedRevisionWorkPending": [], + "PrismAcceptedRevisionWorkPendingCritical": [], + "PrismAcceptedRevisionWorkTrackingUnknown": [ + { + "job": "qbit-prism", + "instance": "one", + "network": "mainnet" + } + ] + } + }, + { + "name": "landing single-scrape miner flap restarts the dwell", + "series": { + "qbit_prism_accepted_block_revision_work_pending_seconds{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\"}": 400, + "qbit_prism_accepted_block_revision_work_tracking_unknown{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\"}": 0, + "up{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\"}": 1, + "qbit_prism_authorized_clients{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\"}": "3+0x30 0 3+0x400" + }, + "expect": { + "PrismAcceptedRevisionWorkPendingCritical": 1 + }, + "alerts": { + "PrismAcceptedRevisionWorkPendingCritical": [] + }, + "eval_time": "1m15s" + }, + { + "name": "landing no miner flap fires at the same evaluation", + "series": { + "qbit_prism_accepted_block_revision_work_pending_seconds{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\"}": 400, + "qbit_prism_accepted_block_revision_work_tracking_unknown{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\"}": 0, + "up{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\"}": 1, + "qbit_prism_authorized_clients{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\"}": 3 + }, + "expect": { + "PrismAcceptedRevisionWorkPendingCritical": 1 + }, + "alerts": { + "PrismAcceptedRevisionWorkPendingCritical": [ + { + "job": "qbit-prism", + "instance": "one", + "network": "mainnet" + } + ] + }, + "eval_time": "1m15s" + }, + { + "name": "landing stall outlasting a miner flap still pages", + "series": { + "qbit_prism_accepted_block_revision_work_pending_seconds{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\"}": 400, + "qbit_prism_accepted_block_revision_work_tracking_unknown{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\"}": 0, + "up{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\"}": 1, + "qbit_prism_authorized_clients{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\"}": "3+0x30 0 3+0x400" + }, + "expect": { + "PrismAcceptedRevisionWorkPendingCritical": 1 + }, + "alerts": { + "PrismAcceptedRevisionWorkPendingCritical": [ + { + "job": "qbit-prism", + "instance": "one", + "network": "mainnet" + } + ] + }, + "eval_time": "1m40s" + }, + { + "name": "landing one failed refresh is pending under the unknown dwell", + "series": { + "qbit_prism_accepted_block_revision_work_pending_seconds{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\"}": "0x30 -1 -1 0x327", + "qbit_prism_accepted_block_revision_work_tracking_unknown{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\"}": "0x30 1 1 0x327", + "up{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\"}": 1, + "qbit_prism_authorized_clients{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\"}": 1 + }, + "eval_time": "32s", + "expect": { + "PrismAcceptedRevisionWorkTrackingUnknown": 1, + "PrismAcceptedRevisionWorkPending": 0 + }, + "alerts": { + "PrismAcceptedRevisionWorkTrackingUnknown": [], + "PrismAcceptedRevisionWorkPending": [] + } + }, + { + "name": "landing one failed refresh never warns", + "series": { + "qbit_prism_accepted_block_revision_work_pending_seconds{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\"}": "0x30 -1 -1 0x327", + "qbit_prism_accepted_block_revision_work_tracking_unknown{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\"}": "0x30 1 1 0x327", + "up{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\"}": 1, + "qbit_prism_authorized_clients{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\"}": 1 + }, + "eval_time": "6m", + "expect": { + "PrismAcceptedRevisionWorkTrackingUnknown": 0, + "PrismAcceptedRevisionWorkPending": 0, + "PrismAcceptedRevisionWorkPendingCritical": 0 + }, + "alerts": { + "PrismAcceptedRevisionWorkTrackingUnknown": [], + "PrismAcceptedRevisionWorkPending": [], + "PrismAcceptedRevisionWorkPendingCritical": [] + } + }, + { + "name": "landing persistent unknown is pending under the dwell", + "series": { + "qbit_prism_accepted_block_revision_work_pending_seconds{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\"}": -1, + "qbit_prism_accepted_block_revision_work_tracking_unknown{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\"}": 1, + "up{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\"}": 1, + "qbit_prism_authorized_clients{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\"}": 1 + }, + "eval_time": "1m59s", + "expect": { + "PrismAcceptedRevisionWorkTrackingUnknown": 1, + "PrismAcceptedRevisionWorkPending": 0, + "PrismAcceptedRevisionWorkPendingCritical": null + }, + "alerts": { + "PrismAcceptedRevisionWorkTrackingUnknown": [], + "PrismAcceptedRevisionWorkPending": [], + "PrismAcceptedRevisionWorkPendingCritical": [] + } + }, + { + "name": "landing persistent unknown warns once the dwell elapses", + "series": { + "qbit_prism_accepted_block_revision_work_pending_seconds{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\"}": -1, + "qbit_prism_accepted_block_revision_work_tracking_unknown{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\"}": 1, + "up{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\"}": 1, + "qbit_prism_authorized_clients{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\"}": 1 + }, + "eval_time": "2m", + "expect": { + "PrismAcceptedRevisionWorkTrackingUnknown": 1, + "PrismAcceptedRevisionWorkPending": 0, + "PrismAcceptedRevisionWorkPendingCritical": null + }, + "alerts": { + "PrismAcceptedRevisionWorkTrackingUnknown": [ + { + "job": "qbit-prism", + "instance": "one", + "network": "mainnet" + } + ], + "PrismAcceptedRevisionWorkPending": [], + "PrismAcceptedRevisionWorkPendingCritical": [] + } + }, + { + "name": "landing saturation beside a known age warns after the dwell", + "series": { + "qbit_prism_accepted_block_revision_work_pending_seconds{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\"}": 5, + "qbit_prism_accepted_block_revision_work_tracking_unknown{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\"}": 1, + "qbit_prism_accepted_block_unlanded_seconds{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\"}": -1, + "up{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\"}": 1, + "qbit_prism_authorized_clients{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\"}": 1 + }, + "eval_time": "6m", + "expect": { + "PrismAcceptedRevisionWorkTrackingUnknown": 1, + "PrismAcceptedRevisionWorkPending": 1, + "PrismAcceptedBlockUnlanded": 0, + "PrismAcceptedRevisionWorkPendingCritical": 0 + }, + "alerts": { + "PrismAcceptedRevisionWorkTrackingUnknown": [ + { + "job": "qbit-prism", + "instance": "one", + "network": "mainnet" + } + ], + "PrismAcceptedRevisionWorkPending": [ + { + "job": "qbit-prism", + "instance": "one", + "network": "mainnet" + } + ], + "PrismAcceptedBlockUnlanded": [], + "PrismAcceptedRevisionWorkPendingCritical": [] + } + }, + { + "name": "landing budget warning is unchanged by the split", + "series": { + "qbit_prism_accepted_block_revision_work_pending_seconds{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\"}": 1.01, + "qbit_prism_accepted_block_revision_work_tracking_unknown{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\"}": 0, + "up{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\"}": 1, + "qbit_prism_authorized_clients{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\"}": 1 + }, + "eval_time": "1s", + "expect": { + "PrismAcceptedRevisionWorkPending": 1, + "PrismAcceptedRevisionWorkTrackingUnknown": 0 + }, + "alerts": { + "PrismAcceptedRevisionWorkPending": [ + { + "job": "qbit-prism", + "instance": "one", + "network": "mainnet" + } + ], + "PrismAcceptedRevisionWorkTrackingUnknown": [] + } + }, + { + "name": "landing lost race stays inside the unlanded threshold", + "series": { + "qbit_prism_accepted_block_unlanded_seconds{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\"}": "0+1x899 0x300", + "qbit_prism_accepted_block_revision_work_pending_seconds{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\"}": "0x1200", + "qbit_prism_accepted_block_revision_work_tracking_unknown{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\"}": "0x1200", + "up{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\"}": "1x1200", + "qbit_prism_authorized_clients{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\"}": "3x1200" + }, + "eval_time": "14m59s", + "expect": { + "PrismAcceptedBlockUnlanded": 0, + "PrismAcceptedRevisionWorkPending": 0, + "PrismAcceptedRevisionWorkTrackingUnknown": 0, + "PrismAcceptedRevisionWorkPendingCritical": 0 + }, + "alerts": { + "PrismAcceptedBlockUnlanded": [], + "PrismAcceptedRevisionWorkPending": [], + "PrismAcceptedRevisionWorkTrackingUnknown": [], + "PrismAcceptedRevisionWorkPendingCritical": [] + } + }, + { + "name": "landing lost race orphan proof clears the unlanded gauge", + "series": { + "qbit_prism_accepted_block_unlanded_seconds{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\"}": "0+1x899 0x300", + "qbit_prism_accepted_block_revision_work_pending_seconds{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\"}": "0x1200", + "qbit_prism_accepted_block_revision_work_tracking_unknown{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\"}": "0x1200", + "up{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\"}": "1x1200", + "qbit_prism_authorized_clients{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\"}": "3x1200" + }, + "eval_time": "20m", + "expect": { + "PrismAcceptedBlockUnlanded": 0, + "PrismAcceptedRevisionWorkPending": 0, + "PrismAcceptedRevisionWorkPendingCritical": 0 + }, + "alerts": { + "PrismAcceptedBlockUnlanded": [], + "PrismAcceptedRevisionWorkPending": [], + "PrismAcceptedRevisionWorkPendingCritical": [] + } + }, + { + "name": "landing stuck unlanded row is pending under the dwell", + "series": { + "qbit_prism_accepted_block_unlanded_seconds{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\"}": "1700+1x400", + "qbit_prism_accepted_block_revision_work_pending_seconds{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\"}": 0, + "qbit_prism_accepted_block_revision_work_tracking_unknown{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\"}": 0, + "up{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\"}": 1, + "qbit_prism_authorized_clients{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\"}": 3 + }, + "eval_time": "2m", + "expect": { + "PrismAcceptedBlockUnlanded": 1, + "PrismAcceptedRevisionWorkPending": 0, + "PrismAcceptedRevisionWorkPendingCritical": 0 + }, + "alerts": { + "PrismAcceptedBlockUnlanded": [], + "PrismAcceptedRevisionWorkPending": [], + "PrismAcceptedRevisionWorkPendingCritical": [] + } + }, + { + "name": "landing stuck unlanded row warns after the dwell", + "series": { + "qbit_prism_accepted_block_unlanded_seconds{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\"}": "1700+1x400", + "qbit_prism_accepted_block_revision_work_pending_seconds{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\"}": 0, + "qbit_prism_accepted_block_revision_work_tracking_unknown{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\"}": 0, + "up{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\"}": 1, + "qbit_prism_authorized_clients{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\"}": 3 + }, + "eval_time": "6m", + "expect": { + "PrismAcceptedBlockUnlanded": 1, + "PrismAcceptedRevisionWorkPending": 0, + "PrismAcceptedRevisionWorkPendingCritical": 0 + }, + "alerts": { + "PrismAcceptedBlockUnlanded": [ + { + "job": "qbit-prism", + "instance": "one", + "network": "mainnet" + } + ], + "PrismAcceptedRevisionWorkPending": [], + "PrismAcceptedRevisionWorkPendingCritical": [] + } + }, + { + "name": "landing saturated unlanded tracking is unknown not stuck", + "series": { + "qbit_prism_accepted_block_unlanded_seconds{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\"}": -1, + "qbit_prism_accepted_block_revision_work_pending_seconds{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\"}": -1, + "qbit_prism_accepted_block_revision_work_tracking_unknown{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\"}": 1, + "up{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\"}": 1, + "qbit_prism_authorized_clients{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\"}": 3 + }, + "eval_time": "6m", + "expect": { + "PrismAcceptedBlockUnlanded": 0, + "PrismAcceptedRevisionWorkTrackingUnknown": 1, + "PrismAcceptedRevisionWorkPending": 0, + "PrismAcceptedRevisionWorkPendingCritical": null + }, + "alerts": { + "PrismAcceptedBlockUnlanded": [], + "PrismAcceptedRevisionWorkTrackingUnknown": [ + { + "job": "qbit-prism", + "instance": "one", + "network": "mainnet" + } + ], + "PrismAcceptedRevisionWorkPending": [], + "PrismAcceptedRevisionWorkPendingCritical": [] + } + }, + { + "name": "advisory lock: slow order waits fire while settlement stays fast", + "series": { + "up{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\"}": 1.0, + "qbit_prism_metrics_snapshot_available{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\"}": 1.0, + "qbit_prism_metrics_snapshot_stale{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\"}": 0.0, + "qbit_prism_database_advisory_lock_wait_seconds_bucket{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\",lock=\"order\",result=\"success\",le=\"0.01\"}": "0+0x360", + "qbit_prism_database_advisory_lock_wait_seconds_bucket{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\",lock=\"order\",result=\"success\",le=\"0.025\"}": "0+0x360", + "qbit_prism_database_advisory_lock_wait_seconds_bucket{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\",lock=\"order\",result=\"success\",le=\"0.05\"}": "0+0x360", + "qbit_prism_database_advisory_lock_wait_seconds_bucket{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\",lock=\"order\",result=\"success\",le=\"0.1\"}": "0+0x360", + "qbit_prism_database_advisory_lock_wait_seconds_bucket{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\",lock=\"order\",result=\"success\",le=\"0.25\"}": "0+0x360", + "qbit_prism_database_advisory_lock_wait_seconds_bucket{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\",lock=\"order\",result=\"success\",le=\"0.5\"}": "0+0x360", + "qbit_prism_database_advisory_lock_wait_seconds_bucket{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\",lock=\"order\",result=\"success\",le=\"1\"}": "0+0x360", + "qbit_prism_database_advisory_lock_wait_seconds_bucket{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\",lock=\"order\",result=\"success\",le=\"2.5\"}": "0+1x360", + "qbit_prism_database_advisory_lock_wait_seconds_bucket{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\",lock=\"order\",result=\"success\",le=\"5\"}": "0+1x360", + "qbit_prism_database_advisory_lock_wait_seconds_bucket{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\",lock=\"order\",result=\"success\",le=\"10\"}": "0+1x360", + "qbit_prism_database_advisory_lock_wait_seconds_bucket{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\",lock=\"order\",result=\"success\",le=\"30\"}": "0+1x360", + "qbit_prism_database_advisory_lock_wait_seconds_bucket{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\",lock=\"order\",result=\"success\",le=\"+Inf\"}": "0+1x360", + "qbit_prism_database_advisory_lock_wait_seconds_count{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\",lock=\"order\",result=\"success\"}": "0+1x360", + "qbit_prism_database_advisory_lock_wait_seconds_bucket{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\",lock=\"settlement\",result=\"success\",le=\"0.01\"}": "0+0x360", + "qbit_prism_database_advisory_lock_wait_seconds_bucket{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\",lock=\"settlement\",result=\"success\",le=\"0.025\"}": "0+0x360", + "qbit_prism_database_advisory_lock_wait_seconds_bucket{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\",lock=\"settlement\",result=\"success\",le=\"0.05\"}": "0+0x360", + "qbit_prism_database_advisory_lock_wait_seconds_bucket{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\",lock=\"settlement\",result=\"success\",le=\"0.1\"}": "0+1x360", + "qbit_prism_database_advisory_lock_wait_seconds_bucket{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\",lock=\"settlement\",result=\"success\",le=\"0.25\"}": "0+1x360", + "qbit_prism_database_advisory_lock_wait_seconds_bucket{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\",lock=\"settlement\",result=\"success\",le=\"0.5\"}": "0+1x360", + "qbit_prism_database_advisory_lock_wait_seconds_bucket{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\",lock=\"settlement\",result=\"success\",le=\"1\"}": "0+1x360", + "qbit_prism_database_advisory_lock_wait_seconds_bucket{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\",lock=\"settlement\",result=\"success\",le=\"2.5\"}": "0+1x360", + "qbit_prism_database_advisory_lock_wait_seconds_bucket{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\",lock=\"settlement\",result=\"success\",le=\"5\"}": "0+1x360", + "qbit_prism_database_advisory_lock_wait_seconds_bucket{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\",lock=\"settlement\",result=\"success\",le=\"10\"}": "0+1x360", + "qbit_prism_database_advisory_lock_wait_seconds_bucket{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\",lock=\"settlement\",result=\"success\",le=\"30\"}": "0+1x360", + "qbit_prism_database_advisory_lock_wait_seconds_bucket{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\",lock=\"settlement\",result=\"success\",le=\"+Inf\"}": "0+1x360", + "qbit_prism_database_advisory_lock_wait_seconds_count{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\",lock=\"settlement\",result=\"success\"}": "0+1x360" + }, + "expect": { + "PrismDatabaseAdvisoryLockWaitHigh": 1, + "PrismDatabaseAdvisoryLockFailuresCritical": null + } + }, + { + "name": "advisory lock: fast waits on both locks stay quiet", + "series": { + "up{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\"}": 1.0, + "qbit_prism_metrics_snapshot_available{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\"}": 1.0, + "qbit_prism_metrics_snapshot_stale{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\"}": 0.0, + "qbit_prism_database_advisory_lock_wait_seconds_bucket{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\",lock=\"order\",result=\"success\",le=\"0.01\"}": "0+0x360", + "qbit_prism_database_advisory_lock_wait_seconds_bucket{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\",lock=\"order\",result=\"success\",le=\"0.025\"}": "0+0x360", + "qbit_prism_database_advisory_lock_wait_seconds_bucket{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\",lock=\"order\",result=\"success\",le=\"0.05\"}": "0+0x360", + "qbit_prism_database_advisory_lock_wait_seconds_bucket{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\",lock=\"order\",result=\"success\",le=\"0.1\"}": "0+1x360", + "qbit_prism_database_advisory_lock_wait_seconds_bucket{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\",lock=\"order\",result=\"success\",le=\"0.25\"}": "0+1x360", + "qbit_prism_database_advisory_lock_wait_seconds_bucket{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\",lock=\"order\",result=\"success\",le=\"0.5\"}": "0+1x360", + "qbit_prism_database_advisory_lock_wait_seconds_bucket{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\",lock=\"order\",result=\"success\",le=\"1\"}": "0+1x360", + "qbit_prism_database_advisory_lock_wait_seconds_bucket{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\",lock=\"order\",result=\"success\",le=\"2.5\"}": "0+1x360", + "qbit_prism_database_advisory_lock_wait_seconds_bucket{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\",lock=\"order\",result=\"success\",le=\"5\"}": "0+1x360", + "qbit_prism_database_advisory_lock_wait_seconds_bucket{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\",lock=\"order\",result=\"success\",le=\"10\"}": "0+1x360", + "qbit_prism_database_advisory_lock_wait_seconds_bucket{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\",lock=\"order\",result=\"success\",le=\"30\"}": "0+1x360", + "qbit_prism_database_advisory_lock_wait_seconds_bucket{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\",lock=\"order\",result=\"success\",le=\"+Inf\"}": "0+1x360", + "qbit_prism_database_advisory_lock_wait_seconds_count{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\",lock=\"order\",result=\"success\"}": "0+1x360", + "qbit_prism_database_advisory_lock_wait_seconds_bucket{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\",lock=\"order\",result=\"failure\",le=\"0.01\"}": "0+0x360", + "qbit_prism_database_advisory_lock_wait_seconds_bucket{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\",lock=\"order\",result=\"failure\",le=\"0.025\"}": "0+0x360", + "qbit_prism_database_advisory_lock_wait_seconds_bucket{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\",lock=\"order\",result=\"failure\",le=\"0.05\"}": "0+0x360", + "qbit_prism_database_advisory_lock_wait_seconds_bucket{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\",lock=\"order\",result=\"failure\",le=\"0.1\"}": "0+0x360", + "qbit_prism_database_advisory_lock_wait_seconds_bucket{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\",lock=\"order\",result=\"failure\",le=\"0.25\"}": "0+0x360", + "qbit_prism_database_advisory_lock_wait_seconds_bucket{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\",lock=\"order\",result=\"failure\",le=\"0.5\"}": "0+0x360", + "qbit_prism_database_advisory_lock_wait_seconds_bucket{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\",lock=\"order\",result=\"failure\",le=\"1\"}": "0+0x360", + "qbit_prism_database_advisory_lock_wait_seconds_bucket{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\",lock=\"order\",result=\"failure\",le=\"2.5\"}": "0+0x360", + "qbit_prism_database_advisory_lock_wait_seconds_bucket{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\",lock=\"order\",result=\"failure\",le=\"5\"}": "0+0x360", + "qbit_prism_database_advisory_lock_wait_seconds_bucket{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\",lock=\"order\",result=\"failure\",le=\"10\"}": "0+0x360", + "qbit_prism_database_advisory_lock_wait_seconds_bucket{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\",lock=\"order\",result=\"failure\",le=\"30\"}": "0+0x360", + "qbit_prism_database_advisory_lock_wait_seconds_bucket{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\",lock=\"order\",result=\"failure\",le=\"+Inf\"}": "0+0x360", + "qbit_prism_database_advisory_lock_wait_seconds_count{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\",lock=\"order\",result=\"failure\"}": "0+0x360", + "qbit_prism_database_advisory_lock_wait_seconds_bucket{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\",lock=\"settlement\",result=\"success\",le=\"0.01\"}": "0+0x360", + "qbit_prism_database_advisory_lock_wait_seconds_bucket{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\",lock=\"settlement\",result=\"success\",le=\"0.025\"}": "0+0x360", + "qbit_prism_database_advisory_lock_wait_seconds_bucket{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\",lock=\"settlement\",result=\"success\",le=\"0.05\"}": "0+0x360", + "qbit_prism_database_advisory_lock_wait_seconds_bucket{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\",lock=\"settlement\",result=\"success\",le=\"0.1\"}": "0+1x360", + "qbit_prism_database_advisory_lock_wait_seconds_bucket{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\",lock=\"settlement\",result=\"success\",le=\"0.25\"}": "0+1x360", + "qbit_prism_database_advisory_lock_wait_seconds_bucket{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\",lock=\"settlement\",result=\"success\",le=\"0.5\"}": "0+1x360", + "qbit_prism_database_advisory_lock_wait_seconds_bucket{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\",lock=\"settlement\",result=\"success\",le=\"1\"}": "0+1x360", + "qbit_prism_database_advisory_lock_wait_seconds_bucket{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\",lock=\"settlement\",result=\"success\",le=\"2.5\"}": "0+1x360", + "qbit_prism_database_advisory_lock_wait_seconds_bucket{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\",lock=\"settlement\",result=\"success\",le=\"5\"}": "0+1x360", + "qbit_prism_database_advisory_lock_wait_seconds_bucket{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\",lock=\"settlement\",result=\"success\",le=\"10\"}": "0+1x360", + "qbit_prism_database_advisory_lock_wait_seconds_bucket{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\",lock=\"settlement\",result=\"success\",le=\"30\"}": "0+1x360", + "qbit_prism_database_advisory_lock_wait_seconds_bucket{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\",lock=\"settlement\",result=\"success\",le=\"+Inf\"}": "0+1x360", + "qbit_prism_database_advisory_lock_wait_seconds_count{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\",lock=\"settlement\",result=\"success\"}": "0+1x360", + "qbit_prism_database_advisory_lock_wait_seconds_bucket{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\",lock=\"settlement\",result=\"failure\",le=\"0.01\"}": "0+0x360", + "qbit_prism_database_advisory_lock_wait_seconds_bucket{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\",lock=\"settlement\",result=\"failure\",le=\"0.025\"}": "0+0x360", + "qbit_prism_database_advisory_lock_wait_seconds_bucket{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\",lock=\"settlement\",result=\"failure\",le=\"0.05\"}": "0+0x360", + "qbit_prism_database_advisory_lock_wait_seconds_bucket{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\",lock=\"settlement\",result=\"failure\",le=\"0.1\"}": "0+0x360", + "qbit_prism_database_advisory_lock_wait_seconds_bucket{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\",lock=\"settlement\",result=\"failure\",le=\"0.25\"}": "0+0x360", + "qbit_prism_database_advisory_lock_wait_seconds_bucket{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\",lock=\"settlement\",result=\"failure\",le=\"0.5\"}": "0+0x360", + "qbit_prism_database_advisory_lock_wait_seconds_bucket{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\",lock=\"settlement\",result=\"failure\",le=\"1\"}": "0+0x360", + "qbit_prism_database_advisory_lock_wait_seconds_bucket{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\",lock=\"settlement\",result=\"failure\",le=\"2.5\"}": "0+0x360", + "qbit_prism_database_advisory_lock_wait_seconds_bucket{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\",lock=\"settlement\",result=\"failure\",le=\"5\"}": "0+0x360", + "qbit_prism_database_advisory_lock_wait_seconds_bucket{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\",lock=\"settlement\",result=\"failure\",le=\"10\"}": "0+0x360", + "qbit_prism_database_advisory_lock_wait_seconds_bucket{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\",lock=\"settlement\",result=\"failure\",le=\"30\"}": "0+0x360", + "qbit_prism_database_advisory_lock_wait_seconds_bucket{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\",lock=\"settlement\",result=\"failure\",le=\"+Inf\"}": "0+0x360", + "qbit_prism_database_advisory_lock_wait_seconds_count{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\",lock=\"settlement\",result=\"failure\"}": "0+0x360" + }, + "expect": { + "PrismDatabaseAdvisoryLockWaitHigh": 0, + "PrismDatabaseAdvisoryLockFailuresCritical": 0 + } + }, + { + "name": "advisory lock: migration waits alone never fire", + "series": { + "up{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\"}": 1.0, + "qbit_prism_metrics_snapshot_available{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\"}": 1.0, + "qbit_prism_metrics_snapshot_stale{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\"}": 0.0, + "qbit_prism_database_advisory_lock_wait_seconds_bucket{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\",lock=\"migration\",result=\"success\",le=\"0.01\"}": "0+0x360", + "qbit_prism_database_advisory_lock_wait_seconds_bucket{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\",lock=\"migration\",result=\"success\",le=\"0.025\"}": "0+0x360", + "qbit_prism_database_advisory_lock_wait_seconds_bucket{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\",lock=\"migration\",result=\"success\",le=\"0.05\"}": "0+0x360", + "qbit_prism_database_advisory_lock_wait_seconds_bucket{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\",lock=\"migration\",result=\"success\",le=\"0.1\"}": "0+0x360", + "qbit_prism_database_advisory_lock_wait_seconds_bucket{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\",lock=\"migration\",result=\"success\",le=\"0.25\"}": "0+0x360", + "qbit_prism_database_advisory_lock_wait_seconds_bucket{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\",lock=\"migration\",result=\"success\",le=\"0.5\"}": "0+0x360", + "qbit_prism_database_advisory_lock_wait_seconds_bucket{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\",lock=\"migration\",result=\"success\",le=\"1\"}": "0+0x360", + "qbit_prism_database_advisory_lock_wait_seconds_bucket{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\",lock=\"migration\",result=\"success\",le=\"2.5\"}": "0+1x360", + "qbit_prism_database_advisory_lock_wait_seconds_bucket{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\",lock=\"migration\",result=\"success\",le=\"5\"}": "0+1x360", + "qbit_prism_database_advisory_lock_wait_seconds_bucket{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\",lock=\"migration\",result=\"success\",le=\"10\"}": "0+1x360", + "qbit_prism_database_advisory_lock_wait_seconds_bucket{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\",lock=\"migration\",result=\"success\",le=\"30\"}": "0+1x360", + "qbit_prism_database_advisory_lock_wait_seconds_bucket{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\",lock=\"migration\",result=\"success\",le=\"+Inf\"}": "0+1x360", + "qbit_prism_database_advisory_lock_wait_seconds_count{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\",lock=\"migration\",result=\"success\"}": "0+1x360", + "qbit_prism_database_advisory_lock_wait_seconds_bucket{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\",lock=\"migration\",result=\"failure\",le=\"0.01\"}": "0+0x360", + "qbit_prism_database_advisory_lock_wait_seconds_bucket{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\",lock=\"migration\",result=\"failure\",le=\"0.025\"}": "0+0x360", + "qbit_prism_database_advisory_lock_wait_seconds_bucket{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\",lock=\"migration\",result=\"failure\",le=\"0.05\"}": "0+0x360", + "qbit_prism_database_advisory_lock_wait_seconds_bucket{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\",lock=\"migration\",result=\"failure\",le=\"0.1\"}": "0+0x360", + "qbit_prism_database_advisory_lock_wait_seconds_bucket{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\",lock=\"migration\",result=\"failure\",le=\"0.25\"}": "0+0x360", + "qbit_prism_database_advisory_lock_wait_seconds_bucket{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\",lock=\"migration\",result=\"failure\",le=\"0.5\"}": "0+0x360", + "qbit_prism_database_advisory_lock_wait_seconds_bucket{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\",lock=\"migration\",result=\"failure\",le=\"1\"}": "0+0x360", + "qbit_prism_database_advisory_lock_wait_seconds_bucket{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\",lock=\"migration\",result=\"failure\",le=\"2.5\"}": "0+1x360", + "qbit_prism_database_advisory_lock_wait_seconds_bucket{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\",lock=\"migration\",result=\"failure\",le=\"5\"}": "0+1x360", + "qbit_prism_database_advisory_lock_wait_seconds_bucket{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\",lock=\"migration\",result=\"failure\",le=\"10\"}": "0+1x360", + "qbit_prism_database_advisory_lock_wait_seconds_bucket{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\",lock=\"migration\",result=\"failure\",le=\"30\"}": "0+1x360", + "qbit_prism_database_advisory_lock_wait_seconds_bucket{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\",lock=\"migration\",result=\"failure\",le=\"+Inf\"}": "0+1x360", + "qbit_prism_database_advisory_lock_wait_seconds_count{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\",lock=\"migration\",result=\"failure\"}": "0+1x360" + }, + "expect": { + "PrismDatabaseAdvisoryLockWaitHigh": null, + "PrismDatabaseAdvisoryLockFailuresCritical": null + } + }, + { + "name": "advisory lock: stale snapshot body suppresses slow waits", + "series": { + "up{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\"}": 1.0, + "qbit_prism_metrics_snapshot_available{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\"}": 1.0, + "qbit_prism_metrics_snapshot_stale{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\"}": 1.0, + "qbit_prism_database_advisory_lock_wait_seconds_bucket{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\",lock=\"order\",result=\"success\",le=\"0.01\"}": "0+0x900", + "qbit_prism_database_advisory_lock_wait_seconds_bucket{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\",lock=\"order\",result=\"success\",le=\"0.025\"}": "0+0x900", + "qbit_prism_database_advisory_lock_wait_seconds_bucket{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\",lock=\"order\",result=\"success\",le=\"0.05\"}": "0+0x900", + "qbit_prism_database_advisory_lock_wait_seconds_bucket{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\",lock=\"order\",result=\"success\",le=\"0.1\"}": "0+0x900", + "qbit_prism_database_advisory_lock_wait_seconds_bucket{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\",lock=\"order\",result=\"success\",le=\"0.25\"}": "0+0x900", + "qbit_prism_database_advisory_lock_wait_seconds_bucket{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\",lock=\"order\",result=\"success\",le=\"0.5\"}": "0+0x900", + "qbit_prism_database_advisory_lock_wait_seconds_bucket{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\",lock=\"order\",result=\"success\",le=\"1\"}": "0+0x900", + "qbit_prism_database_advisory_lock_wait_seconds_bucket{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\",lock=\"order\",result=\"success\",le=\"2.5\"}": "0+0x239 10+0x660", + "qbit_prism_database_advisory_lock_wait_seconds_bucket{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\",lock=\"order\",result=\"success\",le=\"5\"}": "0+0x239 10+0x660", + "qbit_prism_database_advisory_lock_wait_seconds_bucket{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\",lock=\"order\",result=\"success\",le=\"10\"}": "0+0x239 10+0x660", + "qbit_prism_database_advisory_lock_wait_seconds_bucket{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\",lock=\"order\",result=\"success\",le=\"30\"}": "0+0x239 10+0x660", + "qbit_prism_database_advisory_lock_wait_seconds_bucket{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\",lock=\"order\",result=\"success\",le=\"+Inf\"}": "0+0x239 10+0x660", + "qbit_prism_database_advisory_lock_wait_seconds_count{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\",lock=\"order\",result=\"success\"}": "0+0x239 10+0x660", + "qbit_prism_database_advisory_lock_wait_seconds_bucket{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\",lock=\"order\",result=\"failure\",le=\"0.01\"}": "0+0x900", + "qbit_prism_database_advisory_lock_wait_seconds_bucket{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\",lock=\"order\",result=\"failure\",le=\"0.025\"}": "0+0x900", + "qbit_prism_database_advisory_lock_wait_seconds_bucket{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\",lock=\"order\",result=\"failure\",le=\"0.05\"}": "0+0x900", + "qbit_prism_database_advisory_lock_wait_seconds_bucket{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\",lock=\"order\",result=\"failure\",le=\"0.1\"}": "0+0x900", + "qbit_prism_database_advisory_lock_wait_seconds_bucket{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\",lock=\"order\",result=\"failure\",le=\"0.25\"}": "0+0x900", + "qbit_prism_database_advisory_lock_wait_seconds_bucket{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\",lock=\"order\",result=\"failure\",le=\"0.5\"}": "0+0x900", + "qbit_prism_database_advisory_lock_wait_seconds_bucket{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\",lock=\"order\",result=\"failure\",le=\"1\"}": "0+0x900", + "qbit_prism_database_advisory_lock_wait_seconds_bucket{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\",lock=\"order\",result=\"failure\",le=\"2.5\"}": "0+0x239 10+0x660", + "qbit_prism_database_advisory_lock_wait_seconds_bucket{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\",lock=\"order\",result=\"failure\",le=\"5\"}": "0+0x239 10+0x660", + "qbit_prism_database_advisory_lock_wait_seconds_bucket{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\",lock=\"order\",result=\"failure\",le=\"10\"}": "0+0x239 10+0x660", + "qbit_prism_database_advisory_lock_wait_seconds_bucket{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\",lock=\"order\",result=\"failure\",le=\"30\"}": "0+0x239 10+0x660", + "qbit_prism_database_advisory_lock_wait_seconds_bucket{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\",lock=\"order\",result=\"failure\",le=\"+Inf\"}": "0+0x239 10+0x660", + "qbit_prism_database_advisory_lock_wait_seconds_count{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\",lock=\"order\",result=\"failure\"}": "0+0x239 10+0x660" + }, + "expect": { + "PrismDatabaseAdvisoryLockWaitHigh": null, + "PrismDatabaseAdvisoryLockFailuresCritical": null + } + }, + { + "name": "advisory lock: unpublished snapshot body suppresses slow waits", + "series": { + "up{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\"}": 1.0, + "qbit_prism_metrics_snapshot_available{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\"}": 0.0, + "qbit_prism_metrics_snapshot_stale{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\"}": 1.0, + "qbit_prism_database_advisory_lock_wait_seconds_bucket{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\",lock=\"order\",result=\"success\",le=\"0.01\"}": "0+0x900", + "qbit_prism_database_advisory_lock_wait_seconds_bucket{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\",lock=\"order\",result=\"success\",le=\"0.025\"}": "0+0x900", + "qbit_prism_database_advisory_lock_wait_seconds_bucket{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\",lock=\"order\",result=\"success\",le=\"0.05\"}": "0+0x900", + "qbit_prism_database_advisory_lock_wait_seconds_bucket{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\",lock=\"order\",result=\"success\",le=\"0.1\"}": "0+0x900", + "qbit_prism_database_advisory_lock_wait_seconds_bucket{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\",lock=\"order\",result=\"success\",le=\"0.25\"}": "0+0x900", + "qbit_prism_database_advisory_lock_wait_seconds_bucket{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\",lock=\"order\",result=\"success\",le=\"0.5\"}": "0+0x900", + "qbit_prism_database_advisory_lock_wait_seconds_bucket{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\",lock=\"order\",result=\"success\",le=\"1\"}": "0+0x900", + "qbit_prism_database_advisory_lock_wait_seconds_bucket{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\",lock=\"order\",result=\"success\",le=\"2.5\"}": "0+0x239 10+0x660", + "qbit_prism_database_advisory_lock_wait_seconds_bucket{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\",lock=\"order\",result=\"success\",le=\"5\"}": "0+0x239 10+0x660", + "qbit_prism_database_advisory_lock_wait_seconds_bucket{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\",lock=\"order\",result=\"success\",le=\"10\"}": "0+0x239 10+0x660", + "qbit_prism_database_advisory_lock_wait_seconds_bucket{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\",lock=\"order\",result=\"success\",le=\"30\"}": "0+0x239 10+0x660", + "qbit_prism_database_advisory_lock_wait_seconds_bucket{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\",lock=\"order\",result=\"success\",le=\"+Inf\"}": "0+0x239 10+0x660", + "qbit_prism_database_advisory_lock_wait_seconds_count{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\",lock=\"order\",result=\"success\"}": "0+0x239 10+0x660", + "qbit_prism_database_advisory_lock_wait_seconds_bucket{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\",lock=\"order\",result=\"failure\",le=\"0.01\"}": "0+0x900", + "qbit_prism_database_advisory_lock_wait_seconds_bucket{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\",lock=\"order\",result=\"failure\",le=\"0.025\"}": "0+0x900", + "qbit_prism_database_advisory_lock_wait_seconds_bucket{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\",lock=\"order\",result=\"failure\",le=\"0.05\"}": "0+0x900", + "qbit_prism_database_advisory_lock_wait_seconds_bucket{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\",lock=\"order\",result=\"failure\",le=\"0.1\"}": "0+0x900", + "qbit_prism_database_advisory_lock_wait_seconds_bucket{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\",lock=\"order\",result=\"failure\",le=\"0.25\"}": "0+0x900", + "qbit_prism_database_advisory_lock_wait_seconds_bucket{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\",lock=\"order\",result=\"failure\",le=\"0.5\"}": "0+0x900", + "qbit_prism_database_advisory_lock_wait_seconds_bucket{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\",lock=\"order\",result=\"failure\",le=\"1\"}": "0+0x900", + "qbit_prism_database_advisory_lock_wait_seconds_bucket{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\",lock=\"order\",result=\"failure\",le=\"2.5\"}": "0+0x239 10+0x660", + "qbit_prism_database_advisory_lock_wait_seconds_bucket{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\",lock=\"order\",result=\"failure\",le=\"5\"}": "0+0x239 10+0x660", + "qbit_prism_database_advisory_lock_wait_seconds_bucket{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\",lock=\"order\",result=\"failure\",le=\"10\"}": "0+0x239 10+0x660", + "qbit_prism_database_advisory_lock_wait_seconds_bucket{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\",lock=\"order\",result=\"failure\",le=\"30\"}": "0+0x239 10+0x660", + "qbit_prism_database_advisory_lock_wait_seconds_bucket{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\",lock=\"order\",result=\"failure\",le=\"+Inf\"}": "0+0x239 10+0x660", + "qbit_prism_database_advisory_lock_wait_seconds_count{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\",lock=\"order\",result=\"failure\"}": "0+0x239 10+0x660" + }, + "expect": { + "PrismDatabaseAdvisoryLockWaitHigh": null, + "PrismDatabaseAdvisoryLockFailuresCritical": null + } + }, + { + "name": "advisory lock: missing snapshot series suppress slow waits", + "series": { + "up{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\"}": 1.0, + "qbit_prism_database_advisory_lock_wait_seconds_bucket{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\",lock=\"order\",result=\"success\",le=\"0.01\"}": "0+0x900", + "qbit_prism_database_advisory_lock_wait_seconds_bucket{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\",lock=\"order\",result=\"success\",le=\"0.025\"}": "0+0x900", + "qbit_prism_database_advisory_lock_wait_seconds_bucket{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\",lock=\"order\",result=\"success\",le=\"0.05\"}": "0+0x900", + "qbit_prism_database_advisory_lock_wait_seconds_bucket{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\",lock=\"order\",result=\"success\",le=\"0.1\"}": "0+0x900", + "qbit_prism_database_advisory_lock_wait_seconds_bucket{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\",lock=\"order\",result=\"success\",le=\"0.25\"}": "0+0x900", + "qbit_prism_database_advisory_lock_wait_seconds_bucket{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\",lock=\"order\",result=\"success\",le=\"0.5\"}": "0+0x900", + "qbit_prism_database_advisory_lock_wait_seconds_bucket{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\",lock=\"order\",result=\"success\",le=\"1\"}": "0+0x900", + "qbit_prism_database_advisory_lock_wait_seconds_bucket{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\",lock=\"order\",result=\"success\",le=\"2.5\"}": "0+0x239 10+0x660", + "qbit_prism_database_advisory_lock_wait_seconds_bucket{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\",lock=\"order\",result=\"success\",le=\"5\"}": "0+0x239 10+0x660", + "qbit_prism_database_advisory_lock_wait_seconds_bucket{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\",lock=\"order\",result=\"success\",le=\"10\"}": "0+0x239 10+0x660", + "qbit_prism_database_advisory_lock_wait_seconds_bucket{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\",lock=\"order\",result=\"success\",le=\"30\"}": "0+0x239 10+0x660", + "qbit_prism_database_advisory_lock_wait_seconds_bucket{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\",lock=\"order\",result=\"success\",le=\"+Inf\"}": "0+0x239 10+0x660", + "qbit_prism_database_advisory_lock_wait_seconds_count{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\",lock=\"order\",result=\"success\"}": "0+0x239 10+0x660", + "qbit_prism_database_advisory_lock_wait_seconds_bucket{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\",lock=\"order\",result=\"failure\",le=\"0.01\"}": "0+0x900", + "qbit_prism_database_advisory_lock_wait_seconds_bucket{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\",lock=\"order\",result=\"failure\",le=\"0.025\"}": "0+0x900", + "qbit_prism_database_advisory_lock_wait_seconds_bucket{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\",lock=\"order\",result=\"failure\",le=\"0.05\"}": "0+0x900", + "qbit_prism_database_advisory_lock_wait_seconds_bucket{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\",lock=\"order\",result=\"failure\",le=\"0.1\"}": "0+0x900", + "qbit_prism_database_advisory_lock_wait_seconds_bucket{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\",lock=\"order\",result=\"failure\",le=\"0.25\"}": "0+0x900", + "qbit_prism_database_advisory_lock_wait_seconds_bucket{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\",lock=\"order\",result=\"failure\",le=\"0.5\"}": "0+0x900", + "qbit_prism_database_advisory_lock_wait_seconds_bucket{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\",lock=\"order\",result=\"failure\",le=\"1\"}": "0+0x900", + "qbit_prism_database_advisory_lock_wait_seconds_bucket{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\",lock=\"order\",result=\"failure\",le=\"2.5\"}": "0+0x239 10+0x660", + "qbit_prism_database_advisory_lock_wait_seconds_bucket{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\",lock=\"order\",result=\"failure\",le=\"5\"}": "0+0x239 10+0x660", + "qbit_prism_database_advisory_lock_wait_seconds_bucket{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\",lock=\"order\",result=\"failure\",le=\"10\"}": "0+0x239 10+0x660", + "qbit_prism_database_advisory_lock_wait_seconds_bucket{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\",lock=\"order\",result=\"failure\",le=\"30\"}": "0+0x239 10+0x660", + "qbit_prism_database_advisory_lock_wait_seconds_bucket{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\",lock=\"order\",result=\"failure\",le=\"+Inf\"}": "0+0x239 10+0x660", + "qbit_prism_database_advisory_lock_wait_seconds_count{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\",lock=\"order\",result=\"failure\"}": "0+0x239 10+0x660" + }, + "expect": { + "PrismDatabaseAdvisoryLockWaitHigh": null, + "PrismDatabaseAdvisoryLockFailuresCritical": null + } + }, + { + "name": "advisory lock: failure ratio pages while waits stay fast", + "series": { + "up{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\"}": 1.0, + "qbit_prism_metrics_snapshot_available{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\"}": 1.0, + "qbit_prism_metrics_snapshot_stale{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\"}": 0.0, + "qbit_prism_database_advisory_lock_wait_seconds_bucket{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\",lock=\"order\",result=\"success\",le=\"0.01\"}": "0+0x360", + "qbit_prism_database_advisory_lock_wait_seconds_bucket{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\",lock=\"order\",result=\"success\",le=\"0.025\"}": "0+0x360", + "qbit_prism_database_advisory_lock_wait_seconds_bucket{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\",lock=\"order\",result=\"success\",le=\"0.05\"}": "0+0x360", + "qbit_prism_database_advisory_lock_wait_seconds_bucket{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\",lock=\"order\",result=\"success\",le=\"0.1\"}": "0+9x360", + "qbit_prism_database_advisory_lock_wait_seconds_bucket{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\",lock=\"order\",result=\"success\",le=\"0.25\"}": "0+9x360", + "qbit_prism_database_advisory_lock_wait_seconds_bucket{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\",lock=\"order\",result=\"success\",le=\"0.5\"}": "0+9x360", + "qbit_prism_database_advisory_lock_wait_seconds_bucket{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\",lock=\"order\",result=\"success\",le=\"1\"}": "0+9x360", + "qbit_prism_database_advisory_lock_wait_seconds_bucket{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\",lock=\"order\",result=\"success\",le=\"2.5\"}": "0+9x360", + "qbit_prism_database_advisory_lock_wait_seconds_bucket{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\",lock=\"order\",result=\"success\",le=\"5\"}": "0+9x360", + "qbit_prism_database_advisory_lock_wait_seconds_bucket{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\",lock=\"order\",result=\"success\",le=\"10\"}": "0+9x360", + "qbit_prism_database_advisory_lock_wait_seconds_bucket{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\",lock=\"order\",result=\"success\",le=\"30\"}": "0+9x360", + "qbit_prism_database_advisory_lock_wait_seconds_bucket{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\",lock=\"order\",result=\"success\",le=\"+Inf\"}": "0+9x360", + "qbit_prism_database_advisory_lock_wait_seconds_count{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\",lock=\"order\",result=\"success\"}": "0+9x360", + "qbit_prism_database_advisory_lock_wait_seconds_bucket{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\",lock=\"order\",result=\"failure\",le=\"0.01\"}": "0+0x360", + "qbit_prism_database_advisory_lock_wait_seconds_bucket{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\",lock=\"order\",result=\"failure\",le=\"0.025\"}": "0+0x360", + "qbit_prism_database_advisory_lock_wait_seconds_bucket{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\",lock=\"order\",result=\"failure\",le=\"0.05\"}": "0+0x360", + "qbit_prism_database_advisory_lock_wait_seconds_bucket{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\",lock=\"order\",result=\"failure\",le=\"0.1\"}": "0+1x360", + "qbit_prism_database_advisory_lock_wait_seconds_bucket{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\",lock=\"order\",result=\"failure\",le=\"0.25\"}": "0+1x360", + "qbit_prism_database_advisory_lock_wait_seconds_bucket{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\",lock=\"order\",result=\"failure\",le=\"0.5\"}": "0+1x360", + "qbit_prism_database_advisory_lock_wait_seconds_bucket{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\",lock=\"order\",result=\"failure\",le=\"1\"}": "0+1x360", + "qbit_prism_database_advisory_lock_wait_seconds_bucket{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\",lock=\"order\",result=\"failure\",le=\"2.5\"}": "0+1x360", + "qbit_prism_database_advisory_lock_wait_seconds_bucket{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\",lock=\"order\",result=\"failure\",le=\"5\"}": "0+1x360", + "qbit_prism_database_advisory_lock_wait_seconds_bucket{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\",lock=\"order\",result=\"failure\",le=\"10\"}": "0+1x360", + "qbit_prism_database_advisory_lock_wait_seconds_bucket{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\",lock=\"order\",result=\"failure\",le=\"30\"}": "0+1x360", + "qbit_prism_database_advisory_lock_wait_seconds_bucket{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\",lock=\"order\",result=\"failure\",le=\"+Inf\"}": "0+1x360", + "qbit_prism_database_advisory_lock_wait_seconds_count{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\",lock=\"order\",result=\"failure\"}": "0+1x360" + }, + "expect": { + "PrismDatabaseAdvisoryLockWaitHigh": 0, + "PrismDatabaseAdvisoryLockFailuresCritical": 1 + } + }, + { + "name": "advisory lock: failure ratio below five percent does not page", + "series": { + "up{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\"}": 1.0, + "qbit_prism_metrics_snapshot_available{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\"}": 1.0, + "qbit_prism_metrics_snapshot_stale{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\"}": 0.0, + "qbit_prism_database_advisory_lock_wait_seconds_bucket{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\",lock=\"order\",result=\"success\",le=\"0.01\"}": "0+0x360", + "qbit_prism_database_advisory_lock_wait_seconds_bucket{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\",lock=\"order\",result=\"success\",le=\"0.025\"}": "0+0x360", + "qbit_prism_database_advisory_lock_wait_seconds_bucket{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\",lock=\"order\",result=\"success\",le=\"0.05\"}": "0+0x360", + "qbit_prism_database_advisory_lock_wait_seconds_bucket{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\",lock=\"order\",result=\"success\",le=\"0.1\"}": "0+49x360", + "qbit_prism_database_advisory_lock_wait_seconds_bucket{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\",lock=\"order\",result=\"success\",le=\"0.25\"}": "0+49x360", + "qbit_prism_database_advisory_lock_wait_seconds_bucket{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\",lock=\"order\",result=\"success\",le=\"0.5\"}": "0+49x360", + "qbit_prism_database_advisory_lock_wait_seconds_bucket{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\",lock=\"order\",result=\"success\",le=\"1\"}": "0+49x360", + "qbit_prism_database_advisory_lock_wait_seconds_bucket{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\",lock=\"order\",result=\"success\",le=\"2.5\"}": "0+49x360", + "qbit_prism_database_advisory_lock_wait_seconds_bucket{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\",lock=\"order\",result=\"success\",le=\"5\"}": "0+49x360", + "qbit_prism_database_advisory_lock_wait_seconds_bucket{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\",lock=\"order\",result=\"success\",le=\"10\"}": "0+49x360", + "qbit_prism_database_advisory_lock_wait_seconds_bucket{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\",lock=\"order\",result=\"success\",le=\"30\"}": "0+49x360", + "qbit_prism_database_advisory_lock_wait_seconds_bucket{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\",lock=\"order\",result=\"success\",le=\"+Inf\"}": "0+49x360", + "qbit_prism_database_advisory_lock_wait_seconds_count{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\",lock=\"order\",result=\"success\"}": "0+49x360", + "qbit_prism_database_advisory_lock_wait_seconds_bucket{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\",lock=\"order\",result=\"failure\",le=\"0.01\"}": "0+0x360", + "qbit_prism_database_advisory_lock_wait_seconds_bucket{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\",lock=\"order\",result=\"failure\",le=\"0.025\"}": "0+0x360", + "qbit_prism_database_advisory_lock_wait_seconds_bucket{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\",lock=\"order\",result=\"failure\",le=\"0.05\"}": "0+0x360", + "qbit_prism_database_advisory_lock_wait_seconds_bucket{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\",lock=\"order\",result=\"failure\",le=\"0.1\"}": "0+1x360", + "qbit_prism_database_advisory_lock_wait_seconds_bucket{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\",lock=\"order\",result=\"failure\",le=\"0.25\"}": "0+1x360", + "qbit_prism_database_advisory_lock_wait_seconds_bucket{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\",lock=\"order\",result=\"failure\",le=\"0.5\"}": "0+1x360", + "qbit_prism_database_advisory_lock_wait_seconds_bucket{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\",lock=\"order\",result=\"failure\",le=\"1\"}": "0+1x360", + "qbit_prism_database_advisory_lock_wait_seconds_bucket{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\",lock=\"order\",result=\"failure\",le=\"2.5\"}": "0+1x360", + "qbit_prism_database_advisory_lock_wait_seconds_bucket{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\",lock=\"order\",result=\"failure\",le=\"5\"}": "0+1x360", + "qbit_prism_database_advisory_lock_wait_seconds_bucket{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\",lock=\"order\",result=\"failure\",le=\"10\"}": "0+1x360", + "qbit_prism_database_advisory_lock_wait_seconds_bucket{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\",lock=\"order\",result=\"failure\",le=\"30\"}": "0+1x360", + "qbit_prism_database_advisory_lock_wait_seconds_bucket{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\",lock=\"order\",result=\"failure\",le=\"+Inf\"}": "0+1x360", + "qbit_prism_database_advisory_lock_wait_seconds_count{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\",lock=\"order\",result=\"failure\"}": "0+1x360" + }, + "expect": { + "PrismDatabaseAdvisoryLockWaitHigh": 0, + "PrismDatabaseAdvisoryLockFailuresCritical": 0 + } + }, + { + "name": "advisory lock: settlement failures page while order stays clean", + "series": { + "up{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\"}": 1.0, + "qbit_prism_metrics_snapshot_available{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\"}": 1.0, + "qbit_prism_metrics_snapshot_stale{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\"}": 0.0, + "qbit_prism_database_advisory_lock_wait_seconds_bucket{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\",lock=\"order\",result=\"success\",le=\"0.01\"}": "0+0x360", + "qbit_prism_database_advisory_lock_wait_seconds_bucket{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\",lock=\"order\",result=\"success\",le=\"0.025\"}": "0+0x360", + "qbit_prism_database_advisory_lock_wait_seconds_bucket{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\",lock=\"order\",result=\"success\",le=\"0.05\"}": "0+0x360", + "qbit_prism_database_advisory_lock_wait_seconds_bucket{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\",lock=\"order\",result=\"success\",le=\"0.1\"}": "0+1x360", + "qbit_prism_database_advisory_lock_wait_seconds_bucket{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\",lock=\"order\",result=\"success\",le=\"0.25\"}": "0+1x360", + "qbit_prism_database_advisory_lock_wait_seconds_bucket{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\",lock=\"order\",result=\"success\",le=\"0.5\"}": "0+1x360", + "qbit_prism_database_advisory_lock_wait_seconds_bucket{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\",lock=\"order\",result=\"success\",le=\"1\"}": "0+1x360", + "qbit_prism_database_advisory_lock_wait_seconds_bucket{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\",lock=\"order\",result=\"success\",le=\"2.5\"}": "0+1x360", + "qbit_prism_database_advisory_lock_wait_seconds_bucket{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\",lock=\"order\",result=\"success\",le=\"5\"}": "0+1x360", + "qbit_prism_database_advisory_lock_wait_seconds_bucket{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\",lock=\"order\",result=\"success\",le=\"10\"}": "0+1x360", + "qbit_prism_database_advisory_lock_wait_seconds_bucket{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\",lock=\"order\",result=\"success\",le=\"30\"}": "0+1x360", + "qbit_prism_database_advisory_lock_wait_seconds_bucket{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\",lock=\"order\",result=\"success\",le=\"+Inf\"}": "0+1x360", + "qbit_prism_database_advisory_lock_wait_seconds_count{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\",lock=\"order\",result=\"success\"}": "0+1x360", + "qbit_prism_database_advisory_lock_wait_seconds_bucket{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\",lock=\"order\",result=\"failure\",le=\"0.01\"}": "0+0x360", + "qbit_prism_database_advisory_lock_wait_seconds_bucket{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\",lock=\"order\",result=\"failure\",le=\"0.025\"}": "0+0x360", + "qbit_prism_database_advisory_lock_wait_seconds_bucket{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\",lock=\"order\",result=\"failure\",le=\"0.05\"}": "0+0x360", + "qbit_prism_database_advisory_lock_wait_seconds_bucket{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\",lock=\"order\",result=\"failure\",le=\"0.1\"}": "0+0x360", + "qbit_prism_database_advisory_lock_wait_seconds_bucket{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\",lock=\"order\",result=\"failure\",le=\"0.25\"}": "0+0x360", + "qbit_prism_database_advisory_lock_wait_seconds_bucket{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\",lock=\"order\",result=\"failure\",le=\"0.5\"}": "0+0x360", + "qbit_prism_database_advisory_lock_wait_seconds_bucket{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\",lock=\"order\",result=\"failure\",le=\"1\"}": "0+0x360", + "qbit_prism_database_advisory_lock_wait_seconds_bucket{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\",lock=\"order\",result=\"failure\",le=\"2.5\"}": "0+0x360", + "qbit_prism_database_advisory_lock_wait_seconds_bucket{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\",lock=\"order\",result=\"failure\",le=\"5\"}": "0+0x360", + "qbit_prism_database_advisory_lock_wait_seconds_bucket{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\",lock=\"order\",result=\"failure\",le=\"10\"}": "0+0x360", + "qbit_prism_database_advisory_lock_wait_seconds_bucket{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\",lock=\"order\",result=\"failure\",le=\"30\"}": "0+0x360", + "qbit_prism_database_advisory_lock_wait_seconds_bucket{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\",lock=\"order\",result=\"failure\",le=\"+Inf\"}": "0+0x360", + "qbit_prism_database_advisory_lock_wait_seconds_count{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\",lock=\"order\",result=\"failure\"}": "0+0x360", + "qbit_prism_database_advisory_lock_wait_seconds_bucket{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\",lock=\"settlement\",result=\"success\",le=\"0.01\"}": "0+0x360", + "qbit_prism_database_advisory_lock_wait_seconds_bucket{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\",lock=\"settlement\",result=\"success\",le=\"0.025\"}": "0+0x360", + "qbit_prism_database_advisory_lock_wait_seconds_bucket{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\",lock=\"settlement\",result=\"success\",le=\"0.05\"}": "0+0x360", + "qbit_prism_database_advisory_lock_wait_seconds_bucket{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\",lock=\"settlement\",result=\"success\",le=\"0.1\"}": "0+9x360", + "qbit_prism_database_advisory_lock_wait_seconds_bucket{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\",lock=\"settlement\",result=\"success\",le=\"0.25\"}": "0+9x360", + "qbit_prism_database_advisory_lock_wait_seconds_bucket{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\",lock=\"settlement\",result=\"success\",le=\"0.5\"}": "0+9x360", + "qbit_prism_database_advisory_lock_wait_seconds_bucket{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\",lock=\"settlement\",result=\"success\",le=\"1\"}": "0+9x360", + "qbit_prism_database_advisory_lock_wait_seconds_bucket{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\",lock=\"settlement\",result=\"success\",le=\"2.5\"}": "0+9x360", + "qbit_prism_database_advisory_lock_wait_seconds_bucket{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\",lock=\"settlement\",result=\"success\",le=\"5\"}": "0+9x360", + "qbit_prism_database_advisory_lock_wait_seconds_bucket{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\",lock=\"settlement\",result=\"success\",le=\"10\"}": "0+9x360", + "qbit_prism_database_advisory_lock_wait_seconds_bucket{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\",lock=\"settlement\",result=\"success\",le=\"30\"}": "0+9x360", + "qbit_prism_database_advisory_lock_wait_seconds_bucket{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\",lock=\"settlement\",result=\"success\",le=\"+Inf\"}": "0+9x360", + "qbit_prism_database_advisory_lock_wait_seconds_count{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\",lock=\"settlement\",result=\"success\"}": "0+9x360", + "qbit_prism_database_advisory_lock_wait_seconds_bucket{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\",lock=\"settlement\",result=\"failure\",le=\"0.01\"}": "0+0x360", + "qbit_prism_database_advisory_lock_wait_seconds_bucket{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\",lock=\"settlement\",result=\"failure\",le=\"0.025\"}": "0+0x360", + "qbit_prism_database_advisory_lock_wait_seconds_bucket{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\",lock=\"settlement\",result=\"failure\",le=\"0.05\"}": "0+0x360", + "qbit_prism_database_advisory_lock_wait_seconds_bucket{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\",lock=\"settlement\",result=\"failure\",le=\"0.1\"}": "0+1x360", + "qbit_prism_database_advisory_lock_wait_seconds_bucket{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\",lock=\"settlement\",result=\"failure\",le=\"0.25\"}": "0+1x360", + "qbit_prism_database_advisory_lock_wait_seconds_bucket{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\",lock=\"settlement\",result=\"failure\",le=\"0.5\"}": "0+1x360", + "qbit_prism_database_advisory_lock_wait_seconds_bucket{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\",lock=\"settlement\",result=\"failure\",le=\"1\"}": "0+1x360", + "qbit_prism_database_advisory_lock_wait_seconds_bucket{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\",lock=\"settlement\",result=\"failure\",le=\"2.5\"}": "0+1x360", + "qbit_prism_database_advisory_lock_wait_seconds_bucket{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\",lock=\"settlement\",result=\"failure\",le=\"5\"}": "0+1x360", + "qbit_prism_database_advisory_lock_wait_seconds_bucket{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\",lock=\"settlement\",result=\"failure\",le=\"10\"}": "0+1x360", + "qbit_prism_database_advisory_lock_wait_seconds_bucket{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\",lock=\"settlement\",result=\"failure\",le=\"30\"}": "0+1x360", + "qbit_prism_database_advisory_lock_wait_seconds_bucket{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\",lock=\"settlement\",result=\"failure\",le=\"+Inf\"}": "0+1x360", + "qbit_prism_database_advisory_lock_wait_seconds_count{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\",lock=\"settlement\",result=\"failure\"}": "0+1x360" + }, + "expect": { + "PrismDatabaseAdvisoryLockWaitHigh": 0, + "PrismDatabaseAdvisoryLockFailuresCritical": 1 + } + }, + { + "name": "advisory lock: nine waits on a lock are insufficient", + "series": { + "up{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\"}": 1.0, + "qbit_prism_metrics_snapshot_available{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\"}": 1.0, + "qbit_prism_metrics_snapshot_stale{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\"}": 0.0, + "qbit_prism_database_advisory_lock_wait_seconds_bucket{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\",lock=\"order\",result=\"success\",le=\"0.01\"}": "0+0x900", + "qbit_prism_database_advisory_lock_wait_seconds_bucket{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\",lock=\"order\",result=\"success\",le=\"0.025\"}": "0+0x900", + "qbit_prism_database_advisory_lock_wait_seconds_bucket{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\",lock=\"order\",result=\"success\",le=\"0.05\"}": "0+0x900", + "qbit_prism_database_advisory_lock_wait_seconds_bucket{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\",lock=\"order\",result=\"success\",le=\"0.1\"}": "0+0x900", + "qbit_prism_database_advisory_lock_wait_seconds_bucket{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\",lock=\"order\",result=\"success\",le=\"0.25\"}": "0+0x900", + "qbit_prism_database_advisory_lock_wait_seconds_bucket{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\",lock=\"order\",result=\"success\",le=\"0.5\"}": "0+0x900", + "qbit_prism_database_advisory_lock_wait_seconds_bucket{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\",lock=\"order\",result=\"success\",le=\"1\"}": "0+0x900", + "qbit_prism_database_advisory_lock_wait_seconds_bucket{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\",lock=\"order\",result=\"success\",le=\"2.5\"}": "0+0x239 9+0x660", + "qbit_prism_database_advisory_lock_wait_seconds_bucket{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\",lock=\"order\",result=\"success\",le=\"5\"}": "0+0x239 9+0x660", + "qbit_prism_database_advisory_lock_wait_seconds_bucket{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\",lock=\"order\",result=\"success\",le=\"10\"}": "0+0x239 9+0x660", + "qbit_prism_database_advisory_lock_wait_seconds_bucket{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\",lock=\"order\",result=\"success\",le=\"30\"}": "0+0x239 9+0x660", + "qbit_prism_database_advisory_lock_wait_seconds_bucket{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\",lock=\"order\",result=\"success\",le=\"+Inf\"}": "0+0x239 9+0x660", + "qbit_prism_database_advisory_lock_wait_seconds_count{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\",lock=\"order\",result=\"success\"}": "0+0x239 9+0x660" + }, + "expect": { + "PrismDatabaseAdvisoryLockWaitHigh": null, + "PrismDatabaseAdvisoryLockFailuresCritical": null + } + }, + { + "name": "advisory lock: ten waits on a lock are sufficient", + "series": { + "up{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\"}": 1.0, + "qbit_prism_metrics_snapshot_available{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\"}": 1.0, + "qbit_prism_metrics_snapshot_stale{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\"}": 0.0, + "qbit_prism_database_advisory_lock_wait_seconds_bucket{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\",lock=\"order\",result=\"success\",le=\"0.01\"}": "0+0x900", + "qbit_prism_database_advisory_lock_wait_seconds_bucket{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\",lock=\"order\",result=\"success\",le=\"0.025\"}": "0+0x900", + "qbit_prism_database_advisory_lock_wait_seconds_bucket{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\",lock=\"order\",result=\"success\",le=\"0.05\"}": "0+0x900", + "qbit_prism_database_advisory_lock_wait_seconds_bucket{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\",lock=\"order\",result=\"success\",le=\"0.1\"}": "0+0x900", + "qbit_prism_database_advisory_lock_wait_seconds_bucket{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\",lock=\"order\",result=\"success\",le=\"0.25\"}": "0+0x900", + "qbit_prism_database_advisory_lock_wait_seconds_bucket{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\",lock=\"order\",result=\"success\",le=\"0.5\"}": "0+0x900", + "qbit_prism_database_advisory_lock_wait_seconds_bucket{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\",lock=\"order\",result=\"success\",le=\"1\"}": "0+0x900", + "qbit_prism_database_advisory_lock_wait_seconds_bucket{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\",lock=\"order\",result=\"success\",le=\"2.5\"}": "0+0x239 10+0x660", + "qbit_prism_database_advisory_lock_wait_seconds_bucket{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\",lock=\"order\",result=\"success\",le=\"5\"}": "0+0x239 10+0x660", + "qbit_prism_database_advisory_lock_wait_seconds_bucket{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\",lock=\"order\",result=\"success\",le=\"10\"}": "0+0x239 10+0x660", + "qbit_prism_database_advisory_lock_wait_seconds_bucket{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\",lock=\"order\",result=\"success\",le=\"30\"}": "0+0x239 10+0x660", + "qbit_prism_database_advisory_lock_wait_seconds_bucket{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\",lock=\"order\",result=\"success\",le=\"+Inf\"}": "0+0x239 10+0x660", + "qbit_prism_database_advisory_lock_wait_seconds_count{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\",lock=\"order\",result=\"success\"}": "0+0x239 10+0x660" + }, + "expect": { + "PrismDatabaseAdvisoryLockWaitHigh": 1, + "PrismDatabaseAdvisoryLockFailuresCritical": null + } + }, + { + "name": "advisory lock: locks never combine their sample minimum", + "series": { + "up{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\"}": 1.0, + "qbit_prism_metrics_snapshot_available{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\"}": 1.0, + "qbit_prism_metrics_snapshot_stale{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\"}": 0.0, + "qbit_prism_database_advisory_lock_wait_seconds_bucket{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\",lock=\"order\",result=\"success\",le=\"0.01\"}": "0+0x900", + "qbit_prism_database_advisory_lock_wait_seconds_bucket{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\",lock=\"order\",result=\"success\",le=\"0.025\"}": "0+0x900", + "qbit_prism_database_advisory_lock_wait_seconds_bucket{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\",lock=\"order\",result=\"success\",le=\"0.05\"}": "0+0x900", + "qbit_prism_database_advisory_lock_wait_seconds_bucket{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\",lock=\"order\",result=\"success\",le=\"0.1\"}": "0+0x900", + "qbit_prism_database_advisory_lock_wait_seconds_bucket{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\",lock=\"order\",result=\"success\",le=\"0.25\"}": "0+0x900", + "qbit_prism_database_advisory_lock_wait_seconds_bucket{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\",lock=\"order\",result=\"success\",le=\"0.5\"}": "0+0x900", + "qbit_prism_database_advisory_lock_wait_seconds_bucket{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\",lock=\"order\",result=\"success\",le=\"1\"}": "0+0x900", + "qbit_prism_database_advisory_lock_wait_seconds_bucket{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\",lock=\"order\",result=\"success\",le=\"2.5\"}": "0+0x239 6+0x660", + "qbit_prism_database_advisory_lock_wait_seconds_bucket{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\",lock=\"order\",result=\"success\",le=\"5\"}": "0+0x239 6+0x660", + "qbit_prism_database_advisory_lock_wait_seconds_bucket{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\",lock=\"order\",result=\"success\",le=\"10\"}": "0+0x239 6+0x660", + "qbit_prism_database_advisory_lock_wait_seconds_bucket{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\",lock=\"order\",result=\"success\",le=\"30\"}": "0+0x239 6+0x660", + "qbit_prism_database_advisory_lock_wait_seconds_bucket{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\",lock=\"order\",result=\"success\",le=\"+Inf\"}": "0+0x239 6+0x660", + "qbit_prism_database_advisory_lock_wait_seconds_count{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\",lock=\"order\",result=\"success\"}": "0+0x239 6+0x660", + "qbit_prism_database_advisory_lock_wait_seconds_bucket{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\",lock=\"settlement\",result=\"success\",le=\"0.01\"}": "0+0x900", + "qbit_prism_database_advisory_lock_wait_seconds_bucket{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\",lock=\"settlement\",result=\"success\",le=\"0.025\"}": "0+0x900", + "qbit_prism_database_advisory_lock_wait_seconds_bucket{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\",lock=\"settlement\",result=\"success\",le=\"0.05\"}": "0+0x900", + "qbit_prism_database_advisory_lock_wait_seconds_bucket{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\",lock=\"settlement\",result=\"success\",le=\"0.1\"}": "0+0x900", + "qbit_prism_database_advisory_lock_wait_seconds_bucket{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\",lock=\"settlement\",result=\"success\",le=\"0.25\"}": "0+0x900", + "qbit_prism_database_advisory_lock_wait_seconds_bucket{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\",lock=\"settlement\",result=\"success\",le=\"0.5\"}": "0+0x900", + "qbit_prism_database_advisory_lock_wait_seconds_bucket{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\",lock=\"settlement\",result=\"success\",le=\"1\"}": "0+0x900", + "qbit_prism_database_advisory_lock_wait_seconds_bucket{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\",lock=\"settlement\",result=\"success\",le=\"2.5\"}": "0+0x239 6+0x660", + "qbit_prism_database_advisory_lock_wait_seconds_bucket{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\",lock=\"settlement\",result=\"success\",le=\"5\"}": "0+0x239 6+0x660", + "qbit_prism_database_advisory_lock_wait_seconds_bucket{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\",lock=\"settlement\",result=\"success\",le=\"10\"}": "0+0x239 6+0x660", + "qbit_prism_database_advisory_lock_wait_seconds_bucket{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\",lock=\"settlement\",result=\"success\",le=\"30\"}": "0+0x239 6+0x660", + "qbit_prism_database_advisory_lock_wait_seconds_bucket{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\",lock=\"settlement\",result=\"success\",le=\"+Inf\"}": "0+0x239 6+0x660", + "qbit_prism_database_advisory_lock_wait_seconds_count{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\",lock=\"settlement\",result=\"success\"}": "0+0x239 6+0x660" + }, + "expect": { + "PrismDatabaseAdvisoryLockWaitHigh": null, + "PrismDatabaseAdvisoryLockFailuresCritical": null + } + }, + { + "name": "advisory lock: a failing lock below its own minimum cannot borrow samples", + "series": { + "up{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\"}": 1.0, + "qbit_prism_metrics_snapshot_available{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\"}": 1.0, + "qbit_prism_metrics_snapshot_stale{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\"}": 0.0, + "qbit_prism_database_advisory_lock_wait_seconds_bucket{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\",lock=\"order\",result=\"success\",le=\"0.01\"}": "0+0x900", + "qbit_prism_database_advisory_lock_wait_seconds_bucket{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\",lock=\"order\",result=\"success\",le=\"0.025\"}": "0+0x900", + "qbit_prism_database_advisory_lock_wait_seconds_bucket{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\",lock=\"order\",result=\"success\",le=\"0.05\"}": "0+0x900", + "qbit_prism_database_advisory_lock_wait_seconds_bucket{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\",lock=\"order\",result=\"success\",le=\"0.1\"}": "0+0x239 100+0x660", + "qbit_prism_database_advisory_lock_wait_seconds_bucket{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\",lock=\"order\",result=\"success\",le=\"0.25\"}": "0+0x239 100+0x660", + "qbit_prism_database_advisory_lock_wait_seconds_bucket{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\",lock=\"order\",result=\"success\",le=\"0.5\"}": "0+0x239 100+0x660", + "qbit_prism_database_advisory_lock_wait_seconds_bucket{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\",lock=\"order\",result=\"success\",le=\"1\"}": "0+0x239 100+0x660", + "qbit_prism_database_advisory_lock_wait_seconds_bucket{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\",lock=\"order\",result=\"success\",le=\"2.5\"}": "0+0x239 100+0x660", + "qbit_prism_database_advisory_lock_wait_seconds_bucket{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\",lock=\"order\",result=\"success\",le=\"5\"}": "0+0x239 100+0x660", + "qbit_prism_database_advisory_lock_wait_seconds_bucket{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\",lock=\"order\",result=\"success\",le=\"10\"}": "0+0x239 100+0x660", + "qbit_prism_database_advisory_lock_wait_seconds_bucket{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\",lock=\"order\",result=\"success\",le=\"30\"}": "0+0x239 100+0x660", + "qbit_prism_database_advisory_lock_wait_seconds_bucket{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\",lock=\"order\",result=\"success\",le=\"+Inf\"}": "0+0x239 100+0x660", + "qbit_prism_database_advisory_lock_wait_seconds_count{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\",lock=\"order\",result=\"success\"}": "0+0x239 100+0x660", + "qbit_prism_database_advisory_lock_wait_seconds_bucket{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\",lock=\"order\",result=\"failure\",le=\"0.01\"}": "0+0x900", + "qbit_prism_database_advisory_lock_wait_seconds_bucket{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\",lock=\"order\",result=\"failure\",le=\"0.025\"}": "0+0x900", + "qbit_prism_database_advisory_lock_wait_seconds_bucket{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\",lock=\"order\",result=\"failure\",le=\"0.05\"}": "0+0x900", + "qbit_prism_database_advisory_lock_wait_seconds_bucket{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\",lock=\"order\",result=\"failure\",le=\"0.1\"}": "0+0x900", + "qbit_prism_database_advisory_lock_wait_seconds_bucket{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\",lock=\"order\",result=\"failure\",le=\"0.25\"}": "0+0x900", + "qbit_prism_database_advisory_lock_wait_seconds_bucket{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\",lock=\"order\",result=\"failure\",le=\"0.5\"}": "0+0x900", + "qbit_prism_database_advisory_lock_wait_seconds_bucket{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\",lock=\"order\",result=\"failure\",le=\"1\"}": "0+0x900", + "qbit_prism_database_advisory_lock_wait_seconds_bucket{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\",lock=\"order\",result=\"failure\",le=\"2.5\"}": "0+0x900", + "qbit_prism_database_advisory_lock_wait_seconds_bucket{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\",lock=\"order\",result=\"failure\",le=\"5\"}": "0+0x900", + "qbit_prism_database_advisory_lock_wait_seconds_bucket{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\",lock=\"order\",result=\"failure\",le=\"10\"}": "0+0x900", + "qbit_prism_database_advisory_lock_wait_seconds_bucket{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\",lock=\"order\",result=\"failure\",le=\"30\"}": "0+0x900", + "qbit_prism_database_advisory_lock_wait_seconds_bucket{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\",lock=\"order\",result=\"failure\",le=\"+Inf\"}": "0+0x900", + "qbit_prism_database_advisory_lock_wait_seconds_count{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\",lock=\"order\",result=\"failure\"}": "0+0x900", + "qbit_prism_database_advisory_lock_wait_seconds_bucket{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\",lock=\"settlement\",result=\"success\",le=\"0.01\"}": "0+0x900", + "qbit_prism_database_advisory_lock_wait_seconds_bucket{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\",lock=\"settlement\",result=\"success\",le=\"0.025\"}": "0+0x900", + "qbit_prism_database_advisory_lock_wait_seconds_bucket{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\",lock=\"settlement\",result=\"success\",le=\"0.05\"}": "0+0x900", + "qbit_prism_database_advisory_lock_wait_seconds_bucket{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\",lock=\"settlement\",result=\"success\",le=\"0.1\"}": "0+0x900", + "qbit_prism_database_advisory_lock_wait_seconds_bucket{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\",lock=\"settlement\",result=\"success\",le=\"0.25\"}": "0+0x900", + "qbit_prism_database_advisory_lock_wait_seconds_bucket{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\",lock=\"settlement\",result=\"success\",le=\"0.5\"}": "0+0x900", + "qbit_prism_database_advisory_lock_wait_seconds_bucket{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\",lock=\"settlement\",result=\"success\",le=\"1\"}": "0+0x900", + "qbit_prism_database_advisory_lock_wait_seconds_bucket{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\",lock=\"settlement\",result=\"success\",le=\"2.5\"}": "0+0x239 1+0x660", + "qbit_prism_database_advisory_lock_wait_seconds_bucket{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\",lock=\"settlement\",result=\"success\",le=\"5\"}": "0+0x239 1+0x660", + "qbit_prism_database_advisory_lock_wait_seconds_bucket{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\",lock=\"settlement\",result=\"success\",le=\"10\"}": "0+0x239 1+0x660", + "qbit_prism_database_advisory_lock_wait_seconds_bucket{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\",lock=\"settlement\",result=\"success\",le=\"30\"}": "0+0x239 1+0x660", + "qbit_prism_database_advisory_lock_wait_seconds_bucket{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\",lock=\"settlement\",result=\"success\",le=\"+Inf\"}": "0+0x239 1+0x660", + "qbit_prism_database_advisory_lock_wait_seconds_count{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\",lock=\"settlement\",result=\"success\"}": "0+0x239 1+0x660", + "qbit_prism_database_advisory_lock_wait_seconds_bucket{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\",lock=\"settlement\",result=\"failure\",le=\"0.01\"}": "0+0x900", + "qbit_prism_database_advisory_lock_wait_seconds_bucket{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\",lock=\"settlement\",result=\"failure\",le=\"0.025\"}": "0+0x900", + "qbit_prism_database_advisory_lock_wait_seconds_bucket{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\",lock=\"settlement\",result=\"failure\",le=\"0.05\"}": "0+0x900", + "qbit_prism_database_advisory_lock_wait_seconds_bucket{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\",lock=\"settlement\",result=\"failure\",le=\"0.1\"}": "0+0x900", + "qbit_prism_database_advisory_lock_wait_seconds_bucket{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\",lock=\"settlement\",result=\"failure\",le=\"0.25\"}": "0+0x900", + "qbit_prism_database_advisory_lock_wait_seconds_bucket{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\",lock=\"settlement\",result=\"failure\",le=\"0.5\"}": "0+0x900", + "qbit_prism_database_advisory_lock_wait_seconds_bucket{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\",lock=\"settlement\",result=\"failure\",le=\"1\"}": "0+0x900", + "qbit_prism_database_advisory_lock_wait_seconds_bucket{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\",lock=\"settlement\",result=\"failure\",le=\"2.5\"}": "0+0x239 1+0x660", + "qbit_prism_database_advisory_lock_wait_seconds_bucket{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\",lock=\"settlement\",result=\"failure\",le=\"5\"}": "0+0x239 1+0x660", + "qbit_prism_database_advisory_lock_wait_seconds_bucket{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\",lock=\"settlement\",result=\"failure\",le=\"10\"}": "0+0x239 1+0x660", + "qbit_prism_database_advisory_lock_wait_seconds_bucket{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\",lock=\"settlement\",result=\"failure\",le=\"30\"}": "0+0x239 1+0x660", + "qbit_prism_database_advisory_lock_wait_seconds_bucket{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\",lock=\"settlement\",result=\"failure\",le=\"+Inf\"}": "0+0x239 1+0x660", + "qbit_prism_database_advisory_lock_wait_seconds_count{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\",lock=\"settlement\",result=\"failure\"}": "0+0x239 1+0x660" + }, + "expect": { + "PrismDatabaseAdvisoryLockWaitHigh": 0, + "PrismDatabaseAdvisoryLockFailuresCritical": 0 + } + }, + { + "name": "advisory lock: both locks breach at once", + "series": { + "up{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\"}": 1.0, + "qbit_prism_metrics_snapshot_available{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\"}": 1.0, + "qbit_prism_metrics_snapshot_stale{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\"}": 0.0, + "qbit_prism_database_advisory_lock_wait_seconds_bucket{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\",lock=\"order\",result=\"success\",le=\"0.01\"}": "0+0x360", + "qbit_prism_database_advisory_lock_wait_seconds_bucket{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\",lock=\"order\",result=\"success\",le=\"0.025\"}": "0+0x360", + "qbit_prism_database_advisory_lock_wait_seconds_bucket{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\",lock=\"order\",result=\"success\",le=\"0.05\"}": "0+0x360", + "qbit_prism_database_advisory_lock_wait_seconds_bucket{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\",lock=\"order\",result=\"success\",le=\"0.1\"}": "0+0x360", + "qbit_prism_database_advisory_lock_wait_seconds_bucket{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\",lock=\"order\",result=\"success\",le=\"0.25\"}": "0+0x360", + "qbit_prism_database_advisory_lock_wait_seconds_bucket{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\",lock=\"order\",result=\"success\",le=\"0.5\"}": "0+0x360", + "qbit_prism_database_advisory_lock_wait_seconds_bucket{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\",lock=\"order\",result=\"success\",le=\"1\"}": "0+0x360", + "qbit_prism_database_advisory_lock_wait_seconds_bucket{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\",lock=\"order\",result=\"success\",le=\"2.5\"}": "0+9x360", + "qbit_prism_database_advisory_lock_wait_seconds_bucket{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\",lock=\"order\",result=\"success\",le=\"5\"}": "0+9x360", + "qbit_prism_database_advisory_lock_wait_seconds_bucket{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\",lock=\"order\",result=\"success\",le=\"10\"}": "0+9x360", + "qbit_prism_database_advisory_lock_wait_seconds_bucket{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\",lock=\"order\",result=\"success\",le=\"30\"}": "0+9x360", + "qbit_prism_database_advisory_lock_wait_seconds_bucket{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\",lock=\"order\",result=\"success\",le=\"+Inf\"}": "0+9x360", + "qbit_prism_database_advisory_lock_wait_seconds_count{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\",lock=\"order\",result=\"success\"}": "0+9x360", + "qbit_prism_database_advisory_lock_wait_seconds_bucket{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\",lock=\"order\",result=\"failure\",le=\"0.01\"}": "0+0x360", + "qbit_prism_database_advisory_lock_wait_seconds_bucket{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\",lock=\"order\",result=\"failure\",le=\"0.025\"}": "0+0x360", + "qbit_prism_database_advisory_lock_wait_seconds_bucket{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\",lock=\"order\",result=\"failure\",le=\"0.05\"}": "0+0x360", + "qbit_prism_database_advisory_lock_wait_seconds_bucket{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\",lock=\"order\",result=\"failure\",le=\"0.1\"}": "0+0x360", + "qbit_prism_database_advisory_lock_wait_seconds_bucket{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\",lock=\"order\",result=\"failure\",le=\"0.25\"}": "0+0x360", + "qbit_prism_database_advisory_lock_wait_seconds_bucket{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\",lock=\"order\",result=\"failure\",le=\"0.5\"}": "0+0x360", + "qbit_prism_database_advisory_lock_wait_seconds_bucket{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\",lock=\"order\",result=\"failure\",le=\"1\"}": "0+0x360", + "qbit_prism_database_advisory_lock_wait_seconds_bucket{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\",lock=\"order\",result=\"failure\",le=\"2.5\"}": "0+1x360", + "qbit_prism_database_advisory_lock_wait_seconds_bucket{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\",lock=\"order\",result=\"failure\",le=\"5\"}": "0+1x360", + "qbit_prism_database_advisory_lock_wait_seconds_bucket{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\",lock=\"order\",result=\"failure\",le=\"10\"}": "0+1x360", + "qbit_prism_database_advisory_lock_wait_seconds_bucket{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\",lock=\"order\",result=\"failure\",le=\"30\"}": "0+1x360", + "qbit_prism_database_advisory_lock_wait_seconds_bucket{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\",lock=\"order\",result=\"failure\",le=\"+Inf\"}": "0+1x360", + "qbit_prism_database_advisory_lock_wait_seconds_count{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\",lock=\"order\",result=\"failure\"}": "0+1x360", + "qbit_prism_database_advisory_lock_wait_seconds_bucket{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\",lock=\"settlement\",result=\"success\",le=\"0.01\"}": "0+0x360", + "qbit_prism_database_advisory_lock_wait_seconds_bucket{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\",lock=\"settlement\",result=\"success\",le=\"0.025\"}": "0+0x360", + "qbit_prism_database_advisory_lock_wait_seconds_bucket{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\",lock=\"settlement\",result=\"success\",le=\"0.05\"}": "0+0x360", + "qbit_prism_database_advisory_lock_wait_seconds_bucket{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\",lock=\"settlement\",result=\"success\",le=\"0.1\"}": "0+0x360", + "qbit_prism_database_advisory_lock_wait_seconds_bucket{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\",lock=\"settlement\",result=\"success\",le=\"0.25\"}": "0+0x360", + "qbit_prism_database_advisory_lock_wait_seconds_bucket{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\",lock=\"settlement\",result=\"success\",le=\"0.5\"}": "0+0x360", + "qbit_prism_database_advisory_lock_wait_seconds_bucket{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\",lock=\"settlement\",result=\"success\",le=\"1\"}": "0+0x360", + "qbit_prism_database_advisory_lock_wait_seconds_bucket{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\",lock=\"settlement\",result=\"success\",le=\"2.5\"}": "0+9x360", + "qbit_prism_database_advisory_lock_wait_seconds_bucket{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\",lock=\"settlement\",result=\"success\",le=\"5\"}": "0+9x360", + "qbit_prism_database_advisory_lock_wait_seconds_bucket{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\",lock=\"settlement\",result=\"success\",le=\"10\"}": "0+9x360", + "qbit_prism_database_advisory_lock_wait_seconds_bucket{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\",lock=\"settlement\",result=\"success\",le=\"30\"}": "0+9x360", + "qbit_prism_database_advisory_lock_wait_seconds_bucket{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\",lock=\"settlement\",result=\"success\",le=\"+Inf\"}": "0+9x360", + "qbit_prism_database_advisory_lock_wait_seconds_count{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\",lock=\"settlement\",result=\"success\"}": "0+9x360", + "qbit_prism_database_advisory_lock_wait_seconds_bucket{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\",lock=\"settlement\",result=\"failure\",le=\"0.01\"}": "0+0x360", + "qbit_prism_database_advisory_lock_wait_seconds_bucket{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\",lock=\"settlement\",result=\"failure\",le=\"0.025\"}": "0+0x360", + "qbit_prism_database_advisory_lock_wait_seconds_bucket{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\",lock=\"settlement\",result=\"failure\",le=\"0.05\"}": "0+0x360", + "qbit_prism_database_advisory_lock_wait_seconds_bucket{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\",lock=\"settlement\",result=\"failure\",le=\"0.1\"}": "0+0x360", + "qbit_prism_database_advisory_lock_wait_seconds_bucket{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\",lock=\"settlement\",result=\"failure\",le=\"0.25\"}": "0+0x360", + "qbit_prism_database_advisory_lock_wait_seconds_bucket{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\",lock=\"settlement\",result=\"failure\",le=\"0.5\"}": "0+0x360", + "qbit_prism_database_advisory_lock_wait_seconds_bucket{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\",lock=\"settlement\",result=\"failure\",le=\"1\"}": "0+0x360", + "qbit_prism_database_advisory_lock_wait_seconds_bucket{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\",lock=\"settlement\",result=\"failure\",le=\"2.5\"}": "0+1x360", + "qbit_prism_database_advisory_lock_wait_seconds_bucket{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\",lock=\"settlement\",result=\"failure\",le=\"5\"}": "0+1x360", + "qbit_prism_database_advisory_lock_wait_seconds_bucket{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\",lock=\"settlement\",result=\"failure\",le=\"10\"}": "0+1x360", + "qbit_prism_database_advisory_lock_wait_seconds_bucket{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\",lock=\"settlement\",result=\"failure\",le=\"30\"}": "0+1x360", + "qbit_prism_database_advisory_lock_wait_seconds_bucket{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\",lock=\"settlement\",result=\"failure\",le=\"+Inf\"}": "0+1x360", + "qbit_prism_database_advisory_lock_wait_seconds_count{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\",lock=\"settlement\",result=\"failure\"}": "0+1x360" + }, + "expect": { + "PrismDatabaseAdvisoryLockWaitHigh": 2, + "PrismDatabaseAdvisoryLockFailuresCritical": 2 + } + }, + { + "name": "#525 a refresh that keeps failing pages while semantic coverage reads one", + "series": { + "qbit_prism_work_refresh_stalled_seconds{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\"}": "0x59 0+1x600", + "qbit_prism_stratum_semantic_current_work_ratio{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\"}": "1x600", + "qbit_prism_authorized_clients{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\"}": "1x600", + "qbit_prism_metrics_snapshot_available{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\"}": "1x600", + "qbit_prism_metrics_snapshot_stale{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\"}": "0x600" + }, + "eval_time": "4m30s", + "expect": { + "PrismWorkRefreshStalledCritical": 1, + "PrismSemanticWorkCoverageLossCritical": 0 + }, + "alerts": { + "PrismWorkRefreshStalledCritical": [ + { + "job": "qbit-prism", + "instance": "one", + "network": "mainnet" + } + ], + "PrismSemanticWorkCoverageLossCritical": [] + } + }, + { + "name": "#525 a refresh that keeps failing is pending inside the paging dwell", + "series": { + "qbit_prism_work_refresh_stalled_seconds{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\"}": "0x59 0+1x600", + "qbit_prism_stratum_semantic_current_work_ratio{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\"}": "1x600", + "qbit_prism_authorized_clients{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\"}": "1x600", + "qbit_prism_metrics_snapshot_available{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\"}": "1x600", + "qbit_prism_metrics_snapshot_stale{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\"}": "0x600" + }, + "eval_time": "3m30s", + "expect": { + "PrismWorkRefreshStalledCritical": 1, + "PrismSemanticWorkCoverageLossCritical": 0 + }, + "alerts": { + "PrismWorkRefreshStalledCritical": [] + } + }, + { + "name": "#525 a successful refresh resolves the page", + "series": { + "qbit_prism_work_refresh_stalled_seconds{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\"}": "0x59 0+1x200 0x400", + "qbit_prism_stratum_semantic_current_work_ratio{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\"}": "1x600", + "qbit_prism_authorized_clients{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\"}": "1x600", + "qbit_prism_metrics_snapshot_available{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\"}": "1x600", + "qbit_prism_metrics_snapshot_stale{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\"}": "0x600" + }, + "eval_time": "8m", + "expect": { + "PrismWorkRefreshStalledCritical": 0 + }, + "alerts": { + "PrismWorkRefreshStalledCritical": [] + } + }, + { + "name": "#525 refreshes that succeed within 120 seconds never page", + "series": { + "qbit_prism_work_refresh_stalled_seconds{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\"}": "0+1x110 0+1x110 0+1x110 0+1x110 0x160", + "qbit_prism_stratum_semantic_current_work_ratio{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\"}": "1x600", + "qbit_prism_authorized_clients{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\"}": "1x600", + "qbit_prism_metrics_snapshot_available{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\"}": "1x600", + "qbit_prism_metrics_snapshot_stale{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\"}": "0x600" + }, + "eval_time": "6m", + "expect": { + "PrismWorkRefreshStalledCritical": 0 + }, + "alerts": { + "PrismWorkRefreshStalledCritical": [] + } + }, + { + "name": "#525 an unpublished refresh age never pages", + "series": { + "qbit_prism_work_refresh_stalled_seconds{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\"}": "-1x600", + "qbit_prism_stratum_semantic_current_work_ratio{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\"}": "1x600", + "qbit_prism_authorized_clients{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\"}": "1x600", + "qbit_prism_metrics_snapshot_available{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\"}": "1x600", + "qbit_prism_metrics_snapshot_stale{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\"}": "0x600" + }, + "eval_time": "6m", + "expect": { + "PrismWorkRefreshStalledCritical": 0 + }, + "alerts": { + "PrismWorkRefreshStalledCritical": [] + } + }, + { + "name": "#525 a stale snapshot leaves the refresh rule to the snapshot rules", + "series": { + "qbit_prism_work_refresh_stalled_seconds{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\"}": "0+1x600", + "qbit_prism_stratum_semantic_current_work_ratio{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\"}": "1x600", + "qbit_prism_authorized_clients{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\"}": "1x600", + "qbit_prism_metrics_snapshot_available{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\"}": "1x600", + "qbit_prism_metrics_snapshot_stale{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\"}": "1x600" + }, + "eval_time": "6m", + "expect": { + "PrismWorkRefreshStalledCritical": null + }, + "alerts": { + "PrismWorkRefreshStalledCritical": [] + } + }, + { + "name": "#529 a block offered while the standby was not streaming warns at once", + "series": { + "qbit_prism_block_offer_standby_wait_total{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\",outcome=\"confirmed\"}": "0+1x3300", + "qbit_prism_block_offer_standby_wait_total{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\",outcome=\"absent\"}": "0x300 1x3000", + "qbit_prism_block_offer_standby_wait_total{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\",outcome=\"lagging\"}": "0x3300", + "qbit_prism_block_offer_standby_wait_total{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\",outcome=\"failed\"}": "0x3300", + "qbit_prism_metrics_snapshot_available{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\"}": "1x3300", + "qbit_prism_metrics_snapshot_stale{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\"}": "0x3300" + }, + "eval_time": "6m", + "expect": { + "PrismBlockOfferStandbyUnconfirmed": 1 + }, + "alerts": { + "PrismBlockOfferStandbyUnconfirmed": [ + { + "job": "qbit-prism", + "instance": "one", + "network": "mainnet" + } + ] + } + }, + { + "name": "#529 one unconfirmed offer keeps warning for its 30-minute window", + "series": { + "qbit_prism_block_offer_standby_wait_total{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\",outcome=\"confirmed\"}": "0+1x3300", + "qbit_prism_block_offer_standby_wait_total{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\",outcome=\"absent\"}": "0x300 1x3000", + "qbit_prism_block_offer_standby_wait_total{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\",outcome=\"lagging\"}": "0x3300", + "qbit_prism_block_offer_standby_wait_total{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\",outcome=\"failed\"}": "0x3300", + "qbit_prism_metrics_snapshot_available{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\"}": "1x3300", + "qbit_prism_metrics_snapshot_stale{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\"}": "0x3300" + }, + "eval_time": "30m", + "expect": { + "PrismBlockOfferStandbyUnconfirmed": 1 + }, + "alerts": { + "PrismBlockOfferStandbyUnconfirmed": [ + { + "job": "qbit-prism", + "instance": "one", + "network": "mainnet" + } + ] + } + }, + { + "name": "#529 offers the standby confirmed never warn", + "series": { + "qbit_prism_block_offer_standby_wait_total{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\",outcome=\"confirmed\"}": "0+1x3300", + "qbit_prism_block_offer_standby_wait_total{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\",outcome=\"absent\"}": "0x3300", + "qbit_prism_block_offer_standby_wait_total{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\",outcome=\"lagging\"}": "0x3300", + "qbit_prism_block_offer_standby_wait_total{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\",outcome=\"failed\"}": "0x3300", + "qbit_prism_metrics_snapshot_available{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\"}": "1x3300", + "qbit_prism_metrics_snapshot_stale{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\"}": "0x3300" + }, + "eval_time": "20m", + "expect": { + "PrismBlockOfferStandbyUnconfirmed": 0 + }, + "alerts": { + "PrismBlockOfferStandbyUnconfirmed": [] + } + }, + { + "name": "#529 the warning clears once its 30-minute window passes the offer", + "series": { + "qbit_prism_block_offer_standby_wait_total{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\",outcome=\"confirmed\"}": "0x3300", + "qbit_prism_block_offer_standby_wait_total{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\",outcome=\"absent\"}": "0x300 1x3000", + "qbit_prism_block_offer_standby_wait_total{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\",outcome=\"lagging\"}": "0x3300", + "qbit_prism_block_offer_standby_wait_total{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\",outcome=\"failed\"}": "0x3300", + "qbit_prism_metrics_snapshot_available{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\"}": "1x3300", + "qbit_prism_metrics_snapshot_stale{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\"}": "0x3300" + }, + "eval_time": "40m", + "expect": { + "PrismBlockOfferStandbyUnconfirmed": 0 + }, + "alerts": { + "PrismBlockOfferStandbyUnconfirmed": [] + } + }, + { + "name": "#529 lagging and failed waits warn on the frontend that offered", + "series": { + "qbit_prism_block_offer_standby_wait_total{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\",outcome=\"confirmed\"}": "0x3300", + "qbit_prism_block_offer_standby_wait_total{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\",outcome=\"absent\"}": "0x3300", + "qbit_prism_block_offer_standby_wait_total{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\",outcome=\"lagging\"}": "0x300 1x3000", + "qbit_prism_block_offer_standby_wait_total{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\",outcome=\"failed\"}": "0x3300", + "qbit_prism_metrics_snapshot_available{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\"}": "1x3300", + "qbit_prism_metrics_snapshot_stale{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\"}": "0x3300", + "qbit_prism_block_offer_standby_wait_total{job=\"qbit-prism\",instance=\"two\",network=\"mainnet\",outcome=\"confirmed\"}": "0x3300", + "qbit_prism_block_offer_standby_wait_total{job=\"qbit-prism\",instance=\"two\",network=\"mainnet\",outcome=\"absent\"}": "0x3300", + "qbit_prism_block_offer_standby_wait_total{job=\"qbit-prism\",instance=\"two\",network=\"mainnet\",outcome=\"lagging\"}": "0x3300", + "qbit_prism_block_offer_standby_wait_total{job=\"qbit-prism\",instance=\"two\",network=\"mainnet\",outcome=\"failed\"}": "0x300 2x3000", + "qbit_prism_metrics_snapshot_available{job=\"qbit-prism\",instance=\"two\",network=\"mainnet\"}": "1x3300", + "qbit_prism_metrics_snapshot_stale{job=\"qbit-prism\",instance=\"two\",network=\"mainnet\"}": "0x3300" + }, + "eval_time": "6m", + "expect": { + "PrismBlockOfferStandbyUnconfirmed": 2 + }, + "alerts": { + "PrismBlockOfferStandbyUnconfirmed": [ + { + "job": "qbit-prism", + "instance": "one", + "network": "mainnet" + }, + { + "job": "qbit-prism", + "instance": "two", + "network": "mainnet" + } + ] + } + }, + { + "name": "#529 a restart that resets the counter does not warn by itself", + "series": { + "qbit_prism_block_offer_standby_wait_total{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\",outcome=\"confirmed\"}": "0x3300", + "qbit_prism_block_offer_standby_wait_total{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\",outcome=\"absent\"}": "1x300 0x3000", + "qbit_prism_block_offer_standby_wait_total{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\",outcome=\"lagging\"}": "0x3300", + "qbit_prism_block_offer_standby_wait_total{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\",outcome=\"failed\"}": "0x3300", + "qbit_prism_metrics_snapshot_available{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\"}": "1x3300", + "qbit_prism_metrics_snapshot_stale{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\"}": "0x3300" + }, + "eval_time": "6m", + "expect": { + "PrismBlockOfferStandbyUnconfirmed": 0 + }, + "alerts": { + "PrismBlockOfferStandbyUnconfirmed": [] + } + }, + { + "name": "#529 a stale snapshot leaves the standby warning to the snapshot rules", + "series": { + "qbit_prism_block_offer_standby_wait_total{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\",outcome=\"confirmed\"}": "0x3300", + "qbit_prism_block_offer_standby_wait_total{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\",outcome=\"absent\"}": "0x300 1x3000", + "qbit_prism_block_offer_standby_wait_total{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\",outcome=\"lagging\"}": "0x3300", + "qbit_prism_block_offer_standby_wait_total{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\",outcome=\"failed\"}": "0x3300", + "qbit_prism_metrics_snapshot_available{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\"}": "1x3300", + "qbit_prism_metrics_snapshot_stale{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\"}": "1x3300" + }, + "eval_time": "6m", + "expect": { + "PrismBlockOfferStandbyUnconfirmed": null + }, + "alerts": { + "PrismBlockOfferStandbyUnconfirmed": [] + } + }, + { + "name": "#602 three slow landing windows in a row ticket", + "series": { + "qbit_prism_share_ack_slow_landing_windows{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\",p99_above_seconds=\"2\"}": "0x59 1x120 2x120 3x300", + "qbit_prism_share_ack_slow_landing_windows{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\",p99_above_seconds=\"10\"}": "0x600", + "qbit_prism_metrics_snapshot_available{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\"}": "1x600", + "qbit_prism_metrics_snapshot_stale{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\"}": "0x600" + }, + "eval_time": "6m30s", + "expect": { + "PrismLandingAckP99Slow": 1, + "PrismLandingAckP99High": 0 + }, + "alerts": { + "PrismLandingAckP99Slow": [ + { + "job": "qbit-prism", + "instance": "one", + "network": "mainnet", + "p99_above_seconds": "2" + } + ], + "PrismLandingAckP99High": [] + } + }, + { + "name": "#602 the third slow landing window is pending inside the ticket dwell", + "series": { + "qbit_prism_share_ack_slow_landing_windows{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\",p99_above_seconds=\"2\"}": "0x59 1x120 2x120 3x300", + "qbit_prism_share_ack_slow_landing_windows{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\",p99_above_seconds=\"10\"}": "0x600", + "qbit_prism_metrics_snapshot_available{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\"}": "1x600", + "qbit_prism_metrics_snapshot_stale{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\"}": "0x600" + }, + "eval_time": "5m20s", + "expect": { + "PrismLandingAckP99Slow": 1 + }, + "alerts": { + "PrismLandingAckP99Slow": [] + } + }, + { + "name": "#602 two slow landing windows in a row never ticket", + "series": { + "qbit_prism_share_ack_slow_landing_windows{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\",p99_above_seconds=\"2\"}": "0x59 1x120 2x420", + "qbit_prism_share_ack_slow_landing_windows{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\",p99_above_seconds=\"10\"}": "0x600", + "qbit_prism_metrics_snapshot_available{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\"}": "1x600", + "qbit_prism_metrics_snapshot_stale{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\"}": "0x600" + }, + "eval_time": "9m", + "expect": { + "PrismLandingAckP99Slow": 0, + "PrismLandingAckP99High": 0 + }, + "alerts": { + "PrismLandingAckP99Slow": [] + } + }, + { + "name": "#602 a landing window at or under 2 seconds resets the streak and resolves the ticket", + "series": { + "qbit_prism_share_ack_slow_landing_windows{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\",p99_above_seconds=\"2\"}": "3x300 0x300", + "qbit_prism_share_ack_slow_landing_windows{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\",p99_above_seconds=\"10\"}": "0x600", + "qbit_prism_metrics_snapshot_available{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\"}": "1x600", + "qbit_prism_metrics_snapshot_stale{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\"}": "0x600" + }, + "eval_time": "8m", + "expect": { + "PrismLandingAckP99Slow": 0 + }, + "alerts": { + "PrismLandingAckP99Slow": [] + } + }, + { + "name": "#602 one landing window over 10 seconds warns", + "series": { + "qbit_prism_share_ack_slow_landing_windows{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\",p99_above_seconds=\"2\"}": "0x59 1x540", + "qbit_prism_share_ack_slow_landing_windows{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\",p99_above_seconds=\"10\"}": "0x59 1x540", + "qbit_prism_metrics_snapshot_available{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\"}": "1x600", + "qbit_prism_metrics_snapshot_stale{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\"}": "0x600" + }, + "eval_time": "4m", + "expect": { + "PrismLandingAckP99Slow": 0, + "PrismLandingAckP99High": 1 + }, + "alerts": { + "PrismLandingAckP99Slow": [], + "PrismLandingAckP99High": [ + { + "job": "qbit-prism", + "instance": "one", + "network": "mainnet", + "p99_above_seconds": "10" + } + ] + } + }, + { + "name": "#602 no landing window verdict yet never fires", + "series": { + "qbit_prism_share_ack_slow_landing_windows{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\",p99_above_seconds=\"2\"}": "-1x600", + "qbit_prism_share_ack_slow_landing_windows{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\",p99_above_seconds=\"10\"}": "-1x600", + "qbit_prism_metrics_snapshot_available{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\"}": "1x600", + "qbit_prism_metrics_snapshot_stale{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\"}": "0x600" + }, + "eval_time": "8m", + "expect": { + "PrismLandingAckP99Slow": 0, + "PrismLandingAckP99High": 0 + }, + "alerts": { + "PrismLandingAckP99Slow": [], + "PrismLandingAckP99High": [] + } + }, + { + "name": "#602 a stale snapshot leaves the landing rules to the snapshot rules", + "series": { + "qbit_prism_share_ack_slow_landing_windows{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\",p99_above_seconds=\"2\"}": "3x600", + "qbit_prism_share_ack_slow_landing_windows{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\",p99_above_seconds=\"10\"}": "1x600", + "qbit_prism_metrics_snapshot_available{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\"}": "1x600", + "qbit_prism_metrics_snapshot_stale{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\"}": "1x600" + }, + "eval_time": "6m", + "expect": { + "PrismLandingAckP99Slow": null, + "PrismLandingAckP99High": null + }, + "alerts": { + "PrismLandingAckP99Slow": [], + "PrismLandingAckP99High": [] + } + }, + { + "name": "#602 duplicate, unknown-job and stale rejections spread over frontends and reasons warn pool-wide", + "series": { + "qbit_prism_accepted_shares_total{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\"}": "0+10x600", + "qbit_prism_rejections_total{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\",reason_id=\"duplicate-share\"}": "0+0.2x600", + "qbit_prism_rejections_total{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\",reason_id=\"unknown-job\"}": "0+0.2x600", + "qbit_prism_rejections_total{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\",reason_id=\"stale-job\"}": "0+0.2x600", + "qbit_prism_metrics_snapshot_available{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\"}": "1x600", + "qbit_prism_metrics_snapshot_stale{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\"}": "0x600", + "qbit_prism_accepted_shares_total{job=\"qbit-prism\",instance=\"two\",network=\"mainnet\"}": "0+10x600", + "qbit_prism_rejections_total{job=\"qbit-prism\",instance=\"two\",network=\"mainnet\",reason_id=\"duplicate-share\"}": "0+0.2x600", + "qbit_prism_rejections_total{job=\"qbit-prism\",instance=\"two\",network=\"mainnet\",reason_id=\"unknown-job\"}": "0+0.2x600", + "qbit_prism_rejections_total{job=\"qbit-prism\",instance=\"two\",network=\"mainnet\",reason_id=\"stale-job\"}": "0+0.2x600", + "qbit_prism_metrics_snapshot_available{job=\"qbit-prism\",instance=\"two\",network=\"mainnet\"}": "1x600", + "qbit_prism_metrics_snapshot_stale{job=\"qbit-prism\",instance=\"two\",network=\"mainnet\"}": "0x600" + }, + "eval_time": "10m", + "expect": { + "PrismPoolResubmissionRejectRatioHigh": 1, + "PrismRejectRatioByReasonHigh": 0 + }, + "alerts": { + "PrismPoolResubmissionRejectRatioHigh": [ + { + "network": "mainnet" + } + ], + "PrismRejectRatioByReasonHigh": [] + } + }, + { + "name": "#602 the pool reject ratio is pending inside its dwell", + "series": { + "qbit_prism_accepted_shares_total{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\"}": "0+10x600", + "qbit_prism_rejections_total{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\",reason_id=\"duplicate-share\"}": "0+0.2x600", + "qbit_prism_rejections_total{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\",reason_id=\"unknown-job\"}": "0+0.2x600", + "qbit_prism_rejections_total{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\",reason_id=\"stale-job\"}": "0+0.2x600", + "qbit_prism_metrics_snapshot_available{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\"}": "1x600", + "qbit_prism_metrics_snapshot_stale{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\"}": "0x600", + "qbit_prism_accepted_shares_total{job=\"qbit-prism\",instance=\"two\",network=\"mainnet\"}": "0+10x600", + "qbit_prism_rejections_total{job=\"qbit-prism\",instance=\"two\",network=\"mainnet\",reason_id=\"duplicate-share\"}": "0+0.2x600", + "qbit_prism_rejections_total{job=\"qbit-prism\",instance=\"two\",network=\"mainnet\",reason_id=\"unknown-job\"}": "0+0.2x600", + "qbit_prism_rejections_total{job=\"qbit-prism\",instance=\"two\",network=\"mainnet\",reason_id=\"stale-job\"}": "0+0.2x600", + "qbit_prism_metrics_snapshot_available{job=\"qbit-prism\",instance=\"two\",network=\"mainnet\"}": "1x600", + "qbit_prism_metrics_snapshot_stale{job=\"qbit-prism\",instance=\"two\",network=\"mainnet\"}": "0x600" + }, + "eval_time": "4m", + "expect": { + "PrismPoolResubmissionRejectRatioHigh": 1 + }, + "alerts": { + "PrismPoolResubmissionRejectRatioHigh": [] + } + }, + { + "name": "#602 fewer than 500 pool decisions never warn", + "series": { + "qbit_prism_accepted_shares_total{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\"}": "0+1x600", + "qbit_prism_rejections_total{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\",reason_id=\"duplicate-share\"}": "0+0.2x600", + "qbit_prism_metrics_snapshot_available{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\"}": "1x600", + "qbit_prism_metrics_snapshot_stale{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\"}": "0x600" + }, + "eval_time": "10m", + "expect": { + "PrismPoolResubmissionRejectRatioHigh": null + }, + "alerts": { + "PrismPoolResubmissionRejectRatioHigh": [] + } + }, + { + "name": "#602 other rejection reasons are not resubmission rejections", + "series": { + "qbit_prism_accepted_shares_total{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\"}": "0+10x600", + "qbit_prism_rejections_total{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\",reason_id=\"low-difficulty\"}": "0+2x600", + "qbit_prism_rejections_total{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\",reason_id=\"ledger-outcome-unknown\"}": "0+2x600", + "qbit_prism_rejections_total{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\",reason_id=\"duplicate-share\"}": "0+0.1x600", + "qbit_prism_metrics_snapshot_available{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\"}": "1x600", + "qbit_prism_metrics_snapshot_stale{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\"}": "0x600" + }, + "eval_time": "10m", + "expect": { + "PrismPoolResubmissionRejectRatioHigh": 0 + }, + "alerts": { + "PrismPoolResubmissionRejectRatioHigh": [] + } + }, + { + "name": "#602 a frontend with a stale snapshot is left out of the pool reject ratio", + "series": { + "qbit_prism_accepted_shares_total{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\"}": "0+10x600", + "qbit_prism_rejections_total{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\",reason_id=\"duplicate-share\"}": "0+0.1x600", + "qbit_prism_metrics_snapshot_available{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\"}": "1x600", + "qbit_prism_metrics_snapshot_stale{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\"}": "0x600", + "qbit_prism_accepted_shares_total{job=\"qbit-prism\",instance=\"two\",network=\"mainnet\"}": "0+10x600", + "qbit_prism_rejections_total{job=\"qbit-prism\",instance=\"two\",network=\"mainnet\",reason_id=\"duplicate-share\"}": "0+5x600", + "qbit_prism_metrics_snapshot_available{job=\"qbit-prism\",instance=\"two\",network=\"mainnet\"}": "1x600", + "qbit_prism_metrics_snapshot_stale{job=\"qbit-prism\",instance=\"two\",network=\"mainnet\"}": "1x600" + }, + "eval_time": "10m", + "expect": { + "PrismPoolResubmissionRejectRatioHigh": 0 + }, + "alerts": { + "PrismPoolResubmissionRejectRatioHigh": [] + } + }, + { + "name": "#581 a PostgreSQL outage pages on the database rule through a stale snapshot", + "series": { + "qbit_prism_collector_available{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\",collector=\"database\"}": "1x60 0x540", + "qbit_prism_metrics_snapshot_available{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\"}": "1x600", + "qbit_prism_metrics_snapshot_stale{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\"}": "0x60 1x15 0x15 1x15 0x15 1x480", + "up{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\"}": "1x600" + }, + "eval_time": "6m", + "expect": { + "PrismDatabaseUnavailable": 1, + "PrismBlockCandidateMetricsUnavailable": 1 + }, + "alerts": { + "PrismDatabaseUnavailable": [ + { + "job": "qbit-prism", + "instance": "one", + "network": "mainnet", + "collector": "database" + } + ] + } + }, + { + "name": "#581 the database rule dwells two minutes before it pages", + "series": { + "qbit_prism_collector_available{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\",collector=\"database\"}": "1x270 0x330", + "up{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\"}": "1x600" + }, + "eval_time": "6m", + "expect": { + "PrismDatabaseUnavailable": 1 + }, + "alerts": { + "PrismDatabaseUnavailable": [] + } + }, + { + "name": "#581 an available database collector does not page", + "series": { + "qbit_prism_collector_available{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\",collector=\"database\"}": "1x600", + "up{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\"}": "1x600" + }, + "eval_time": "6m", + "expect": { + "PrismDatabaseUnavailable": 0 + }, + "alerts": { + "PrismDatabaseUnavailable": [] + } + }, + { + "name": "#581 a failed scrape leaves the database rule to the endpoint rules", + "series": { + "qbit_prism_collector_available{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\",collector=\"database\"}": "0x600", + "up{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\"}": "0x600" + }, + "eval_time": "6m", + "expect": { + "PrismDatabaseUnavailable": null + }, + "alerts": { + "PrismDatabaseUnavailable": [] + } + }, + { + "name": "#581 ledger write failures stay visible through a stale snapshot", + "series": { + "qbit_prism_rejections_total{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\",reason_id=\"ledger-confirmation-failed\"}": "0+0x300 1+1x59", + "qbit_prism_metrics_snapshot_available{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\"}": 1, + "qbit_prism_metrics_snapshot_stale{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\"}": 1, + "up{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\"}": 1 + }, + "expect": { + "PrismShareAppendFailures": 1 + } + }, + { + "name": "#581 database refusals carry their own reason through a stale snapshot", + "series": { + "qbit_prism_accepted_shares_total{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\"}": "0+10x600", + "qbit_prism_rejections_total{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\",reason_id=\"backend-database-unavailable\"}": "0+5x600", + "qbit_prism_rejections_total{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\",reason_id=\"backend-rpc-unavailable\"}": "0x600", + "qbit_prism_metrics_snapshot_available{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\"}": "1x600", + "qbit_prism_metrics_snapshot_stale{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\"}": "1x600", + "up{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\"}": "1x600" + }, + "eval_time": "10m", + "expect": { + "PrismRejectRatioByReasonHigh": 1 + }, + "alerts": { + "PrismRejectRatioByReasonHigh": [ + { + "job": "qbit-prism", + "instance": "one", + "network": "mainnet", + "reason_id": "backend-database-unavailable" + } + ] + } + }, + { + "name": "#666 a frontend held for 70 seconds has paged, before it finds a block", + "series": { + "qbit_prism_block_submission_enabled{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\"}": "1x290 0x310", + "up{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\"}": "1x600" + }, + "eval_time": "6m", + "expect": { + "PrismBlockSubmissionHeld": 1 + }, + "alerts": { + "PrismBlockSubmissionHeld": [ + { + "job": "qbit-prism", + "instance": "one", + "network": "mainnet" + } + ] + } + }, + { + "name": "#666 a frontend held for 50 seconds is still pending", + "series": { + "qbit_prism_block_submission_enabled{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\"}": "1x310 0x290", + "up{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\"}": "1x600" + }, + "eval_time": "6m", + "expect": { + "PrismBlockSubmissionHeld": 1 + }, + "alerts": { + "PrismBlockSubmissionHeld": [] + } + }, + { + "name": "#666 a failed scrape every 50 seconds does not restart the dwell", + "series": { + "qbit_prism_block_submission_enabled{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\"}": "0x48 stale 0x48 stale 0x48 stale 0x48 stale 0x48 stale 0x48 stale 0x48 stale 0x48 stale 0x48 stale 0x48 stale 0x48 stale 0x48 stale", + "up{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\"}": "1x48 0 1x48 0 1x48 0 1x48 0 1x48 0 1x48 0 1x48 0 1x48 0 1x48 0 1x48 0 1x48 0 1x48 0" + }, + "eval_time": "6m", + "expect": { + "PrismBlockSubmissionHeld": 1 + }, + "alerts": { + "PrismBlockSubmissionHeld": [ + { + "job": "qbit-prism", + "instance": "one", + "network": "mainnet" + } + ] + } + }, + { + "name": "#666 a frontend that submits blocks does not page", + "series": { + "qbit_prism_block_submission_enabled{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\"}": "1x600", + "up{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\"}": "1x600" + }, + "eval_time": "6m", + "expect": { + "PrismBlockSubmissionHeld": 0 + }, + "alerts": { + "PrismBlockSubmissionHeld": [] + } + }, + { + "name": "#666 an unpublished setting pages, since nothing shows the frontend submits", + "series": { + "qbit_prism_block_submission_enabled{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\"}": -1.0, + "up{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\"}": 1 + }, + "eval_time": "6m", + "expect": { + "PrismBlockSubmissionHeld": 1 + }, + "alerts": { + "PrismBlockSubmissionHeld": [ + { + "job": "qbit-prism", + "instance": "one", + "network": "mainnet" + } + ] + } + }, + { + "name": "#666 a failed scrape leaves a held frontend to the endpoint rules", + "series": { + "qbit_prism_block_submission_enabled{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\"}": "0x600", + "up{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\"}": "0x600" + }, + "eval_time": "6m", + "expect": { + "PrismBlockSubmissionHeld": null + }, + "alerts": { + "PrismBlockSubmissionHeld": [] + } + }, + { + "name": "#666 an image without the gauge stays quiet", + "series": { + "up{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\"}": "1x600" + }, + "eval_time": "6m", + "expect": { + "PrismBlockSubmissionHeld": null + }, + "alerts": { + "PrismBlockSubmissionHeld": [] + } + }, + { + "name": "#666 only the held frontend of a pair pages", + "series": { + "qbit_prism_block_submission_enabled{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\"}": "0x600", + "qbit_prism_block_submission_enabled{job=\"qbit-prism\",instance=\"two\",network=\"mainnet\"}": "1x600", + "up{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\"}": "1x600", + "up{job=\"qbit-prism\",instance=\"two\",network=\"mainnet\"}": "1x600" + }, + "eval_time": "6m", + "expect": { + "PrismBlockSubmissionHeld": 1 + }, + "alerts": { + "PrismBlockSubmissionHeld": [ + { + "job": "qbit-prism", + "instance": "one", + "network": "mainnet" + } + ] + } + }, + { + "name": "#666 a held frontend keeps paging through a stale metrics snapshot", + "series": { + "qbit_prism_block_submission_enabled{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\"}": "0x600", + "qbit_prism_metrics_snapshot_available{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\"}": "1x600", + "qbit_prism_metrics_snapshot_stale{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\"}": "1x600", + "up{job=\"qbit-prism\",instance=\"one\",network=\"mainnet\"}": "1x600" + }, + "eval_time": "6m", + "expect": { + "PrismBlockSubmissionHeld": 1 + }, + "alerts": { + "PrismBlockSubmissionHeld": [ + { + "job": "qbit-prism", + "instance": "one", + "network": "mainnet" + } + ] + } + } +] diff --git a/tests/fixtures/prism-capacity-evidence.json b/tests/fixtures/prism-capacity-evidence.json index df123f450..c1947a356 100644 --- a/tests/fixtures/prism-capacity-evidence.json +++ b/tests/fixtures/prism-capacity-evidence.json @@ -1,5 +1,5 @@ { - "schema": "qbit-prism-capacity-evidence/v2", + "schema": "qbit-prism-capacity-evidence/v3", "artifact_kind": "example", "test_path": "stratum-to-postgres", "generated_at": "2026-07-13T12:00:00Z", @@ -27,11 +27,12 @@ "PRISM_STRATUM_VARDIFF_MAX_STEP_DOWN": "4", "PRISM_STRATUM_VARDIFF_EWMA_ALPHA": "0.4", "PRISM_STRATUM_VARDIFF_RETARGET_TOLERANCE": "0.25", - "PRISM_STRATUM_VARDIFF_IDLE_SWEEP_SECONDS": "15", - "PRISM_SHARE_COMMIT_BATCH_SIZE": 64, - "PRISM_SHARE_COMMIT_LINGER_MILLISECONDS": 5, "PRISM_SHARE_COMMIT_TIMEOUT_SECONDS": "15", - "PRISM_STRATUM_SEND_TIMEOUT_SECONDS": "20" + "PRISM_STRATUM_SEND_TIMEOUT_SECONDS": "20", + "PRISM_RUNTIME_WORKERS": 4, + "PRISM_DATABASE_MAX_CONNECTIONS": 16, + "PRISM_STRATUM_MAX_CONNECTIONS": 384, + "PRISM_JOB_BUILD_EXECUTOR_WORKERS": 4 }, "forecast_peak_shares_per_second": 100, "test_duration_seconds": 600, diff --git a/tests/fixtures/prism-deployed-alerts.json b/tests/fixtures/prism-deployed-alerts.json new file mode 100644 index 000000000..b5c695d93 --- /dev/null +++ b/tests/fixtures/prism-deployed-alerts.json @@ -0,0 +1,453 @@ +{ + "repository": "SwapLabsInc/qbit-tools", + "commit": "a007a14260548cb93e277882672d788fe9a3f95e", + "path": "ansible/roles/qbit_monitoring_stack/templates/grafana-alert-rules-prism.yml.j2", + "sha256": "8446692bfb4bffa7ad1cf9b775249aba15e75a3410cc403dedd475ed7ac47f38", + "metric_names": [ + "qbit_prism_accepted_block_preview_publication_seconds_bucket", + "qbit_prism_accepted_block_preview_publication_seconds_count", + "qbit_prism_accepted_parent_preview_wait_timeouts_total", + "qbit_prism_accepted_parent_unresolved_depth_max", + "qbit_prism_accepted_parent_unresolved_oldest_seconds", + "qbit_prism_accepted_parent_unresolved_transitions", + "qbit_prism_accepted_shares_total", + "qbit_prism_accepted_stats_reconcile_age_seconds", + "qbit_prism_accepted_stats_reconcile_failures_total", + "qbit_prism_backup_audit_snapshot_exists", + "qbit_prism_backup_collector_success", + "qbit_prism_backup_collector_timestamp_seconds", + "qbit_prism_backup_job_result", + "qbit_prism_backup_last_duration_seconds", + "qbit_prism_backup_last_success_bytes", + "qbit_prism_backup_last_success_timestamp_seconds", + "qbit_prism_backup_wal_failed_total", + "qbit_prism_backup_wal_filesystem_free_bytes", + "qbit_prism_backup_wal_filesystem_size_bytes", + "qbit_prism_backup_wal_generated_bytes_total", + "qbit_prism_backup_wal_last_archived_timestamp_seconds", + "qbit_prism_backup_wal_unarchived_segments", + "qbit_prism_block_candidate_cleanup_backpressure_active", + "qbit_prism_block_candidate_cleanup_backpressure_total", + "qbit_prism_block_candidate_cleanup_retry_backlog", + "qbit_prism_block_candidate_cleanup_retry_backlog_max", + "qbit_prism_block_candidate_cleanup_retry_oldest_seconds", + "qbit_prism_block_candidate_oldest_pending_seconds", + "qbit_prism_block_candidates_pending", + "qbit_prism_connected_clients", + "qbit_prism_coordinator_collector_success", + "qbit_prism_coordinator_collector_timestamp_seconds", + "qbit_prism_coordinator_unexpected_restarts_total", + "qbit_prism_job_build_orphan_evicted_total", + "qbit_prism_lease_heartbeat_monitor_exit_guarantee_breaches_total", + "qbit_prism_lease_heartbeat_monitor_late_wakes_total", + "qbit_prism_lease_heartbeat_monitor_wake_delay_window_max_seconds", + "qbit_prism_lease_heartbeat_policy_seconds", + "qbit_prism_lease_heartbeat_server_proven_age_seconds", + "qbit_prism_metrics_snapshot_age_seconds", + "qbit_prism_metrics_snapshot_available", + "qbit_prism_metrics_snapshot_stale", + "qbit_prism_public_ledger_ready", + "qbit_prism_public_responses_total", + "qbit_prism_public_staleness_refusals_total", + "qbit_prism_refresh_pending", + "qbit_prism_refresh_pending_age_seconds", + "qbit_prism_share_append_failures_total", + "qbit_prism_share_append_queue_depth", + "qbit_prism_shares_recovered_to_disk_total", + "qbit_prism_shares_replayed_total", + "qbit_prism_stratum_authorized_connections", + "qbit_prism_stratum_semantic_current_work_ratio" + ], + "alerts": [ + { + "uid": "qbit-prism-metrics-nodata", + "title": "PrismMetricsNoData", + "external": false + }, + { + "uid": "qbit-prism-metrics-endpoint-down", + "title": "PrismMetricsEndpointDown", + "external": false + }, + { + "uid": "qbit-prism-connected-clients", + "title": "PRISM Connected Clients", + "external": false + }, + { + "uid": "qbit-prism-stratum-nodata", + "title": "PrismStratumHealthNoData", + "external": true + }, + { + "uid": "qbit-prism-stratum-down-warning", + "title": "PrismStratumUnavailableWarning", + "external": true + }, + { + "uid": "qbit-prism-stratum-down", + "title": "PrismStratumUnavailable", + "external": true + }, + { + "uid": "qbit-prism-stratum-failure-rate", + "title": "PrismStratumCheckFailureRateHigh", + "external": true + }, + { + "uid": "qbit-prism-share-writer-backlog", + "title": "PrismShareWriterBacklog", + "external": false + }, + { + "uid": "qbit-prism-share-writer-backlog-critical", + "title": "PrismShareWriterBacklogCritical", + "external": false + }, + { + "uid": "qbit-prism-block-candidate-backlog", + "title": "PrismBlockCandidateBacklog", + "external": false + }, + { + "uid": "qbit-prism-shares-recovered-to-disk", + "title": "PrismSharesRecoveredToDisk", + "external": false + }, + { + "uid": "qbit-prism-share-append-failures", + "title": "PrismShareAppendFailures", + "external": false + }, + { + "uid": "qbit-prism-shares-replayed", + "title": "PrismSharesReplayed", + "external": false + }, + { + "uid": "qbit-prism-job-build-orphan-evicted", + "title": "PrismJobBuildOrphanEvictions", + "external": false + }, + { + "uid": "qbit-prism-stats-reconcile-stale", + "title": "PrismAcceptedStatsReconcileStale", + "external": false + }, + { + "uid": "qbit-prism-lease-proven-age", + "title": "PrismWriterLeaseServerProvenAgeNearCap", + "external": false + }, + { + "uid": "qbit-prism-lease-monitor-wake", + "title": "PrismWriterLeaseMonitorWakeDelayHigh", + "external": false + }, + { + "uid": "qbit-prism-lease-monitor-guarantee", + "title": "PrismWriterLeaseMonitorExitGuaranteeBreached", + "external": false + }, + { + "uid": "qbit-prism-lease-surplus-invalid", + "title": "PrismWriterLeaseStabilitySurplusExhausted", + "external": false + }, + { + "uid": "qbit-prism-preview-publish-latency", + "title": "PrismAcceptedPreviewPublicationLatencyHigh", + "external": false + }, + { + "uid": "qbit-prism-preview-publish-latency-crit", + "title": "PrismAcceptedPreviewPublicationLatencyCritical", + "external": false + }, + { + "uid": "qbit-prism-preview-publish-degraded", + "title": "PrismAcceptedPreviewPublicationDegraded", + "external": false + }, + { + "uid": "qbit-prism-parent-preview-timeouts", + "title": "PrismAcceptedParentPreviewWaitTimeouts", + "external": false + }, + { + "uid": "qbit-prism-parent-preview-timeouts-crit", + "title": "PrismAcceptedParentPreviewWaitTimeoutsCritical", + "external": false + }, + { + "uid": "qbit-prism-unresolved-parent-depth", + "title": "PrismUnresolvedParentDepthNonzero", + "external": false + }, + { + "uid": "qbit-prism-unresolved-parent-near-cap", + "title": "PrismUnresolvedParentDepthNearCap", + "external": false + }, + { + "uid": "qbit-prism-unresolved-parent-at-cap", + "title": "PrismUnresolvedParentDepthAtCap", + "external": false + }, + { + "uid": "qbit-prism-unresolved-parent-age", + "title": "PrismUnresolvedParentAgeOverBudget", + "external": false + }, + { + "uid": "qbit-prism-unresolved-parent-age-crit", + "title": "PrismUnresolvedParentAgeCritical", + "external": false + }, + { + "uid": "qbit-prism-candidate-backlog-critical", + "title": "PrismBlockCandidateBacklogCritical", + "external": false + }, + { + "uid": "qbit-prism-block-candidate-oldest", + "title": "PrismBlockCandidateOldestPending", + "external": false + }, + { + "uid": "qbit-prism-candidate-oldest-critical", + "title": "PrismBlockCandidateOldestPendingCritical", + "external": false + }, + { + "uid": "qbit-prism-candidate-metrics-unavailable", + "title": "PrismBlockCandidateMetricsUnavailable", + "external": false + }, + { + "uid": "qbit-prism-cleanup-backlog-depth", + "title": "PrismCollapseCleanupBacklog", + "external": false + }, + { + "uid": "qbit-prism-cleanup-backlog-oldest", + "title": "PrismCollapseCleanupBacklogOldest", + "external": false + }, + { + "uid": "qbit-prism-cleanup-backlog-near-bound", + "title": "PrismCollapseCleanupBacklogNearBound", + "external": false + }, + { + "uid": "qbit-prism-cleanup-backpressure-active", + "title": "PrismCollapseCleanupBackpressure", + "external": false + }, + { + "uid": "qbit-prism-cleanup-backpressure-engaged", + "title": "PrismCollapseCleanupBackpressureEngaged", + "external": false + }, + { + "uid": "qbit-prism-semantic-work-coverage", + "title": "PrismSemanticWorkCoverageLoss", + "external": false + }, + { + "uid": "qbit-prism-semantic-coverage-critical", + "title": "PrismSemanticWorkCoverageLossCritical", + "external": false + }, + { + "uid": "qbit-prism-refresh-pending", + "title": "PrismRefreshPendingPastDeadline", + "external": false + }, + { + "uid": "qbit-prism-refresh-pending-critical", + "title": "PrismRefreshPendingPastDeadlineCritical", + "external": false + }, + { + "uid": "qbit-prism-metrics-snapshot-stale", + "title": "PrismMetricsSnapshotStale", + "external": false + }, + { + "uid": "qbit-prism-metrics-snapshot-unavailable", + "title": "PrismMetricsSnapshotUnavailable", + "external": false + }, + { + "uid": "qbit-prism-metrics-snapshot-age", + "title": "PrismMetricsSnapshotAgeHigh", + "external": false + }, + { + "uid": "qbit-prism-public-staleness-refusal", + "title": "PrismPublicStalenessRefusalsIncreasing", + "external": false + }, + { + "uid": "qbit-prism-public-ledger-unready", + "title": "PrismPublicLedgerUnready", + "external": false + }, + { + "uid": "qbit-prism-public-5xx-sustained", + "title": "PrismPublic5xxSustained", + "external": false + }, + { + "uid": "qbit-prism-backup-metrics-nodata", + "title": "PrismBackupMetricsNoData", + "external": true + }, + { + "uid": "qbit-prism-backup-collector-stale", + "title": "PrismBackupCollectorStale", + "external": true + }, + { + "uid": "qbit-prism-backup-full-stale-warning", + "title": "PrismBackupFullStaleWarning", + "external": true + }, + { + "uid": "qbit-prism-backup-full-stale-critical", + "title": "PrismBackupFullStaleCritical", + "external": true + }, + { + "uid": "qbit-prism-backup-diff-stale-warning", + "title": "PrismBackupDifferentialStaleWarning", + "external": true + }, + { + "uid": "qbit-prism-backup-diff-stale-critical", + "title": "PrismBackupDifferentialStaleCritical", + "external": true + }, + { + "uid": "qbit-prism-backup-check-warning", + "title": "PrismBackupCheckWarning", + "external": true + }, + { + "uid": "qbit-prism-backup-check-critical", + "title": "PrismBackupCheckCritical", + "external": true + }, + { + "uid": "qbit-prism-backup-wal-stale-warning", + "title": "PrismBackupWalArchiveStaleWarning", + "external": true + }, + { + "uid": "qbit-prism-backup-wal-stale-critical", + "title": "PrismBackupWalArchiveStaleCritical", + "external": true + }, + { + "uid": "qbit-prism-backup-wal-fail-warning", + "title": "PrismBackupWalArchiveFailureWarning", + "external": true + }, + { + "uid": "qbit-prism-backup-wal-fail-critical", + "title": "PrismBackupWalArchiveFailureCritical", + "external": true + }, + { + "uid": "qbit-prism-backup-wal-space-warning", + "title": "PrismBackupWalFilesystemSpaceWarning", + "external": true + }, + { + "uid": "qbit-prism-backup-wal-space-critical", + "title": "PrismBackupWalFilesystemSpaceCritical", + "external": true + }, + { + "uid": "qbit-prism-backup-audit-stale-warning", + "title": "PrismBackupAuditSnapshotStaleWarning", + "external": true + }, + { + "uid": "qbit-prism-backup-audit-stale-critical", + "title": "PrismBackupAuditSnapshotStaleCritical", + "external": true + }, + { + "uid": "qbit-prism-backup-audit-slow-warning", + "title": "PrismBackupAuditDurationWarning", + "external": true + }, + { + "uid": "qbit-prism-backup-audit-slow-critical", + "title": "PrismBackupAuditDurationCritical", + "external": true + }, + { + "uid": "qbit-prism-backup-job-fail-warning", + "title": "PrismBackupScheduledJobFailureWarning", + "external": true + }, + { + "uid": "qbit-prism-backup-job-fail-critical", + "title": "PrismBackupScheduledJobFailureCritical", + "external": true + }, + { + "uid": "qbit-prism-backup-coverage-critical", + "title": "PrismBackupCombinedCoverageCritical", + "external": true + }, + { + "uid": "qbit-prism-backup-db-growth-warning", + "title": "PrismBackupDatabaseGrowthWarning", + "external": true + }, + { + "uid": "qbit-prism-backup-db-growth-critical", + "title": "PrismBackupDatabaseGrowthCritical", + "external": true + }, + { + "uid": "qbit-prism-backup-audit-growth-warning", + "title": "PrismBackupAuditGrowthWarning", + "external": true + }, + { + "uid": "qbit-prism-backup-audit-growth-critical", + "title": "PrismBackupAuditGrowthCritical", + "external": true + }, + { + "uid": "qbit-prism-backup-wal-growth-warning", + "title": "PrismBackupWalGenerationGrowthWarning", + "external": true + }, + { + "uid": "qbit-prism-backup-wal-growth-critical", + "title": "PrismBackupWalGenerationGrowthCritical", + "external": true + }, + { + "uid": "qbit-prism-coordinator-collector-stale", + "title": "PrismCoordinatorCollectorStale", + "external": true + }, + { + "uid": "qbit-prism-coordinator-restart", + "title": "PrismCoordinatorUnexpectedRestart", + "external": true + }, + { + "uid": "qbit-prism-coordinator-restart-loop", + "title": "PrismCoordinatorRestartLoop", + "external": true + } + ] +} diff --git a/tests/fixtures/prism-legacy-v2-audit-body/README.md b/tests/fixtures/prism-legacy-v2-audit-body/README.md new file mode 100644 index 000000000..e3a070b53 --- /dev/null +++ b/tests/fixtures/prism-legacy-v2-audit-body/README.md @@ -0,0 +1,49 @@ +# A 2.x audit-bundle.v2 body (#731) + +These three files are a v2 audit body and its two share-segment slots, as 2.x.x +publishes them. Two `segment_range` parts cover shares 1–4 and 5–6. The +window proof's `share_parts_digest_hex` is 2.x's: sha256 over the parts in +insertion order. 3.x recomputes that digest with sorted keys and refuses the +body with "window proof share_parts_digest_hex mismatch" (#731), which is how +`import-audits` refuses such a body when its canonical sidecar is missing. + +`tests/test_prism_legacy_parts_digest_check.py` reads them: + +- `scripts/prism_legacy_parts_digest_check.py` must call the body + `2x-order-only`, and its `digest_2x` must reproduce the stored digest; +- the real `qbit-prism-audit-canonicalize` must refuse the body, and accept it + once the digest is recomputed by the checker's `digest_3x`, with canonical + bytes whose sha256 is the body's `audit_bundle_sha256`; +- `scripts/prism_legacy_range_sidecars_sharded.sh` must turn copies of it into + proven canonical sidecars. + +## How it was made + +The generator needs 2.x.x's `lab/` tree, so it isn't in this repository. + +1. Build the bundle with this repository's `qbit-prism-build-audit-bundle + --canonical-output`. + - Input: `crates/qbit-prism/fixtures/power-law-accrual.prism-fixture.json`'s + `found_block` and `shares`, with the prior balance `audit_cli.rs` passes + (`miner-whale`, order key `01`, program `11`×32, 4,800 sats). + - Signing seeds: `42`×32, and `43`×32 for the ledger key. + - The canonical bundle is 10,999 bytes, with sha256 + `65b11e1b7e2025472fad2e4cd6b555eaba5eab2a4903e17179ba792d58780a4b`. +2. Publish it for block `00`×26 + `c8`×6 with 2.x.x's + `AuditArtifactStore.prepare_external_audit_body`, at 2.x.x `50e208ca`, with + `share_segment_size=4`. + - 2.x wrote the body, both slots and its own canonical sidecar. The + sidecar's decompressed bytes are the canonical bundle. +3. Make the body relocatable. + - 2.x named each slot by its absolute path under the temporary store root. + Each part's `body_uri` was rewritten to the production prefix, + `/var/lib/qbit-mining-pool/prism/audit/<slot name>`. + - `share_parts_digest_hex` was recomputed with 2.x's own + `AuditArtifactStore.share_parts_digest_hex`. + - The body was re-encoded with 2.x's `AuditArtifactStore.storage_json_bytes`. + That's the encoder 2.x writes bodies with; re-encoding the unmodified body + reproduces 2.x's bytes exactly. + +Two runs produced identical files. 2.x's v2 writer only emits `segment_range` +parts. Inline parts occur only in `audit-body-ref.v1` bodies, which carry no +parts digest, so this fixture has none. diff --git a/tests/fixtures/prism-legacy-v2-audit-body/prism-audit-bundle-body-0000000000000000000000000000000000000000000000000000c8c8c8c8c8c8-65b11e1b7e2025472fad2e4cd6b555eaba5eab2a4903e17179ba792d58780a4b.json b/tests/fixtures/prism-legacy-v2-audit-body/prism-audit-bundle-body-0000000000000000000000000000000000000000000000000000c8c8c8c8c8c8-65b11e1b7e2025472fad2e4cd6b555eaba5eab2a4903e17179ba792d58780a4b.json new file mode 100644 index 000000000..684ea899a --- /dev/null +++ b/tests/fixtures/prism-legacy-v2-audit-body/prism-audit-bundle-body-0000000000000000000000000000000000000000000000000000c8c8c8c8c8c8-65b11e1b7e2025472fad2e4cd6b555eaba5eab2a4903e17179ba792d58780a4b.json @@ -0,0 +1 @@ +{"schema":"qbit.prism.audit-bundle.v2","block_hash":"0000000000000000000000000000000000000000000000000000c8c8c8c8c8c8","audit_bundle_sha256":"65b11e1b7e2025472fad2e4cd6b555eaba5eab2a4903e17179ba792d58780a4b","logical_audit_bundle_schema":"qbit.prism.audit-bundle.v1","share_count":6,"shares_key_index":1,"bundle_without_shares":{"schema":"qbit.prism.audit-bundle.v1","found_block":{"block_height":200,"coinbase_value_sats":500000000,"network_difficulty":200000,"anchor_job_issued_at_ms":2005},"prior_balances":[{"recipient_id":"miner-whale","order_key":"01","p2mr_program_hex":"1111111111111111111111111111111111111111111111111111111111111111","balance_sats":4800}],"payout_policy":{"p2mr_spend_input_bytes":3680,"target_feerate_sats_per_byte":1,"safety_multiplier":4},"audit_commitment_leaves_hex":["492c8e5f83049d3f6b04a175a531f130a8c52c2fa944b06741f442587f365d3a"],"audit_commitment_root_hex":"492c8e5f83049d3f6b04a175a531f130a8c52c2fa944b06741f442587f365d3a","ledger_window_attestation":{"schema":"qbit.prism.ledger-window-attestation.v1","block_height":200,"coinbase_value_sats":500000000,"anchor_job_issued_at_ms":2005,"network_difficulty":200000,"share_slice_digest_hex":"fc39e87eaedeb6cb6442afbdf060ddc81a93846acc3ffdb47cf202c408c6a9d3","prior_balances_digest_hex":"66ce2409d37d280df5f6aa0392e3669febe60285215426269c83da6c3d731188","signature":{"algorithm":"ed25519","public_key_hex":"22fc297792f0b6ffc0bfcfdb7edb0c0aa14e025a365ec0e342e86e3829cb74b6","signature_hex":"9294bcd022f440c8ff2057bcaac1d71b125ff97e9d0a638fbc9c7a1d6c7ca47a56906c15ca1098751a3a7d3f1db71562c69efbd6aec1785d39521107c4e9780b"}},"reward_manifest":{"schema":"qbit.prism.reward-manifest.v1","block_height":200,"coinbase_value_sats":500000000,"network_difficulty":200000,"window_multiplier":8,"requested_window_weight":1600000,"counted_window_weight":1250012,"anchor_job_issued_at_ms":2005,"anchor_share_seq":6,"newest_share_seq":6,"oldest_share_seq":1,"included_share_count":6,"share_slice_digest_hex":"fc39e87eaedeb6cb6442afbdf060ddc81a93846acc3ffdb47cf202c408c6a9d3","shares":[{"share_seq":6,"share_id":"share-small-c","miner_id":"miner-small-c","order_key":"06","p2mr_program_hex":"6666666666666666666666666666666666666666666666666666666666666666","share_difficulty":1,"counted_difficulty":1,"job_issued_at_ms":2005,"accepted_at_ms":2005},{"share_seq":5,"share_id":"share-small-b","miner_id":"miner-small-b","order_key":"05","p2mr_program_hex":"5555555555555555555555555555555555555555555555555555555555555555","share_difficulty":1,"counted_difficulty":1,"job_issued_at_ms":2004,"accepted_at_ms":2004},{"share_seq":4,"share_id":"share-small-a","miner_id":"miner-small-a","order_key":"04","p2mr_program_hex":"4444444444444444444444444444444444444444444444444444444444444444","share_difficulty":10,"counted_difficulty":10,"job_issued_at_ms":2003,"accepted_at_ms":2003},{"share_seq":3,"share_id":"share-mid-b","miner_id":"miner-mid-b","order_key":"03","p2mr_program_hex":"3333333333333333333333333333333333333333333333333333333333333333","share_difficulty":50000,"counted_difficulty":50000,"job_issued_at_ms":2002,"accepted_at_ms":2002},{"share_seq":2,"share_id":"share-mid-a","miner_id":"miner-mid-a","order_key":"02","p2mr_program_hex":"2222222222222222222222222222222222222222222222222222222222222222","share_difficulty":200000,"counted_difficulty":200000,"job_issued_at_ms":2001,"accepted_at_ms":2001},{"share_seq":1,"share_id":"share-whale","miner_id":"miner-whale","order_key":"01","p2mr_program_hex":"1111111111111111111111111111111111111111111111111111111111111111","share_difficulty":1000000,"counted_difficulty":1000000,"job_issued_at_ms":2000,"accepted_at_ms":2000}],"entitlements":[{"recipient_id":"miner-mid-a","order_key":"02","p2mr_program_hex":"2222222222222222222222222222222222222222222222222222222222222222","weight":200000},{"recipient_id":"miner-mid-b","order_key":"03","p2mr_program_hex":"3333333333333333333333333333333333333333333333333333333333333333","weight":50000},{"recipient_id":"miner-small-a","order_key":"04","p2mr_program_hex":"4444444444444444444444444444444444444444444444444444444444444444","weight":10},{"recipient_id":"miner-small-b","order_key":"05","p2mr_program_hex":"5555555555555555555555555555555555555555555555555555555555555555","weight":1},{"recipient_id":"miner-small-c","order_key":"06","p2mr_program_hex":"6666666666666666666666666666666666666666666666666666666666666666","weight":1},{"recipient_id":"miner-whale","order_key":"01","p2mr_program_hex":"1111111111111111111111111111111111111111111111111111111111111111","weight":1000000}]},"payout_policy_manifest":{"schema":"qbit.prism.payout-policy.v1","block_height":200,"coinbase_value_sats":500000000,"min_output_sats":14720,"floor_formula":"3680 bytes/input * 1 sat/byte * 4x safety","accounts":[{"recipient_id":"miner-whale","order_key":"01","p2mr_program_hex":"1111111111111111111111111111111111111111111111111111111111111111","gross_amount_sats":399996160,"prior_balance_sats":4800,"candidate_balance_sats":400000960,"onchain_amount_sats":400000960,"carry_forward_balance_sats":0,"action":"onchain"},{"recipient_id":"miner-mid-a","order_key":"02","p2mr_program_hex":"2222222222222222222222222222222222222222222222222222222222222222","gross_amount_sats":79999232,"prior_balance_sats":0,"candidate_balance_sats":79999232,"onchain_amount_sats":79999232,"carry_forward_balance_sats":0,"action":"onchain"},{"recipient_id":"miner-mid-b","order_key":"03","p2mr_program_hex":"3333333333333333333333333333333333333333333333333333333333333333","gross_amount_sats":19999808,"prior_balance_sats":0,"candidate_balance_sats":19999808,"onchain_amount_sats":19999808,"carry_forward_balance_sats":0,"action":"onchain"},{"recipient_id":"miner-small-a","order_key":"04","p2mr_program_hex":"4444444444444444444444444444444444444444444444444444444444444444","gross_amount_sats":4000,"prior_balance_sats":0,"candidate_balance_sats":4000,"onchain_amount_sats":0,"carry_forward_balance_sats":4000,"action":"accrued"},{"recipient_id":"miner-small-b","order_key":"05","p2mr_program_hex":"5555555555555555555555555555555555555555555555555555555555555555","gross_amount_sats":400,"prior_balance_sats":0,"candidate_balance_sats":400,"onchain_amount_sats":0,"carry_forward_balance_sats":400,"action":"accrued"},{"recipient_id":"miner-small-c","order_key":"06","p2mr_program_hex":"6666666666666666666666666666666666666666666666666666666666666666","gross_amount_sats":400,"prior_balance_sats":0,"candidate_balance_sats":400,"onchain_amount_sats":0,"carry_forward_balance_sats":400,"action":"accrued"}],"onchain_entitlements":[{"recipient_id":"miner-whale","order_key":"01","p2mr_program_hex":"1111111111111111111111111111111111111111111111111111111111111111","weight":400000960},{"recipient_id":"miner-mid-a","order_key":"02","p2mr_program_hex":"2222222222222222222222222222222222222222222222222222222222222222","weight":79999232},{"recipient_id":"miner-mid-b","order_key":"03","p2mr_program_hex":"3333333333333333333333333333333333333333333333333333333333333333","weight":19999808}]},"signed_coinbase_manifest":{"manifest":{"schema":"qbit.prism.coinbase-manifest.v1","block_height":200,"coinbase_value_sats":500000000,"total_weight":500000000,"coinbase_weight_bytes":266,"payout_count":3,"coinbase_tx_hex":"020000000001010000000000000000000000000000000000000000000000000000000000000000ffffffff0302c800feffffff04c087d71700000000225220111111111111111111111111111111111111111111111111111111111111111100b1c404000000002252202222222222222222222222222222222222222222222222222222222222222222402c31010000000022522033333333333333333333333333333333333333333333333333333333333333330000000000000000266a24aa21a9edc8b15db7f1eb305ddbd87a3b6c321ee775ed4edf20f4fc4ab61953534ae6d3af0120492c8e5f83049d3f6b04a175a531f130a8c52c2fa944b06741f442587f365d3ac7000000","coinbase_txid":"7185c81e86cdeb04a96a6d35219b401de965fbc3dc459f9884b9cb9ecb0014ca","coinbase_wtxid":"0ef362e7e37037498b2e0b5fa437bcfcea69557da966a462bbd7d9e802ec0147","coinbase_script_sig_hex":"02c800","coinbase_script_sig_suffix_hex":"","witness_nonce_hex":"492c8e5f83049d3f6b04a175a531f130a8c52c2fa944b06741f442587f365d3a","witness_commitment_script_hex":"6a24aa21a9edc8b15db7f1eb305ddbd87a3b6c321ee775ed4edf20f4fc4ab61953534ae6d3af","outputs":[{"vout":0,"recipient_id":"miner-whale","order_key":"01","p2mr_program_hex":"1111111111111111111111111111111111111111111111111111111111111111","script_pubkey_hex":"52201111111111111111111111111111111111111111111111111111111111111111","weight":400000960,"amount_sats":400000960},{"vout":1,"recipient_id":"miner-mid-a","order_key":"02","p2mr_program_hex":"2222222222222222222222222222222222222222222222222222222222222222","script_pubkey_hex":"52202222222222222222222222222222222222222222222222222222222222222222","weight":79999232,"amount_sats":79999232},{"vout":2,"recipient_id":"miner-mid-b","order_key":"03","p2mr_program_hex":"3333333333333333333333333333333333333333333333333333333333333333","script_pubkey_hex":"52203333333333333333333333333333333333333333333333333333333333333333","weight":19999808,"amount_sats":19999808}]},"signature":{"algorithm":"ed25519","public_key_hex":"2152f8d19b791d24453242e15f2eab6cb7cffa7b6a5ed30097960e069881db12","signature_hex":"2cc0bfa1088b9b75708e2eebfc176aaf9c8d54f091780a680eefffe49bd04471e11c2c078c55e2a73de6a77f855742892084ee382612c48771430d702517a80d"}}},"share_window_proof":{"schema":"qbit.prism.window-completeness-proof.v1","share_segment_size":4,"first_share_seq":1,"last_share_seq":6,"share_count":6,"share_parts_digest_hex":"a7889efaf7c6207bd9c415e2224c51b47574eeafefe6f0ac9bd1c249d125fc2c","share_parts":[{"kind":"segment_range","segment_first_share_seq":1,"segment_last_share_seq":4,"first_share_seq":1,"last_share_seq":4,"share_count":4,"range_sha256":"45aa5921a074dcad701b0a4396e33c019ad650d680ffa2de89783e8df8dd559a","body_uri":"/var/lib/qbit-mining-pool/prism/audit/prism-audit-share-segment-slot-1-4.json"},{"kind":"segment_range","segment_first_share_seq":5,"segment_last_share_seq":8,"first_share_seq":5,"last_share_seq":6,"share_count":2,"range_sha256":"feea6e1fa0f23243b0accf5c633dfbddf84a000870afb920670bb8f1e183da51","body_uri":"/var/lib/qbit-mining-pool/prism/audit/prism-audit-share-segment-slot-5-8.json"}],"anchor_job_issued_at_ms":2005,"anchor_share_seq":6,"newest_share_seq":6,"oldest_share_seq":1,"included_share_count":6,"requested_window_weight":1600000,"counted_window_weight":1250012,"share_slice_digest_hex":"fc39e87eaedeb6cb6442afbdf060ddc81a93846acc3ffdb47cf202c408c6a9d3"}} \ No newline at end of file diff --git a/tests/fixtures/prism-legacy-v2-audit-body/prism-audit-share-segment-slot-1-4.json b/tests/fixtures/prism-legacy-v2-audit-body/prism-audit-share-segment-slot-1-4.json new file mode 100644 index 000000000..bd34b4bef --- /dev/null +++ b/tests/fixtures/prism-legacy-v2-audit-body/prism-audit-share-segment-slot-1-4.json @@ -0,0 +1 @@ +{"schema":"qbit.prism.audit-share-segment.v1","first_share_seq":1,"last_share_seq":4,"share_count":4,"shares":[{"share_seq":1,"share_id":"share-whale","miner_id":"miner-whale","order_key":"01","p2mr_program_hex":"1111111111111111111111111111111111111111111111111111111111111111","share_difficulty":1000000,"network_difficulty":200000,"template_height":199,"job_id":"job-2000","job_issued_at_ms":2000,"accepted_at_ms":2000,"ntime":1800000000},{"share_seq":2,"share_id":"share-mid-a","miner_id":"miner-mid-a","order_key":"02","p2mr_program_hex":"2222222222222222222222222222222222222222222222222222222222222222","share_difficulty":200000,"network_difficulty":200000,"template_height":199,"job_id":"job-2001","job_issued_at_ms":2001,"accepted_at_ms":2001,"ntime":1800000000},{"share_seq":3,"share_id":"share-mid-b","miner_id":"miner-mid-b","order_key":"03","p2mr_program_hex":"3333333333333333333333333333333333333333333333333333333333333333","share_difficulty":50000,"network_difficulty":200000,"template_height":199,"job_id":"job-2002","job_issued_at_ms":2002,"accepted_at_ms":2002,"ntime":1800000000},{"share_seq":4,"share_id":"share-small-a","miner_id":"miner-small-a","order_key":"04","p2mr_program_hex":"4444444444444444444444444444444444444444444444444444444444444444","share_difficulty":10,"network_difficulty":200000,"template_height":199,"job_id":"job-2003","job_issued_at_ms":2003,"accepted_at_ms":2003,"ntime":1800000000}]} \ No newline at end of file diff --git a/tests/fixtures/prism-legacy-v2-audit-body/prism-audit-share-segment-slot-5-8.json b/tests/fixtures/prism-legacy-v2-audit-body/prism-audit-share-segment-slot-5-8.json new file mode 100644 index 000000000..0eaac77b7 --- /dev/null +++ b/tests/fixtures/prism-legacy-v2-audit-body/prism-audit-share-segment-slot-5-8.json @@ -0,0 +1 @@ +{"schema":"qbit.prism.audit-share-segment.v1","first_share_seq":5,"last_share_seq":6,"share_count":2,"shares":[{"share_seq":5,"share_id":"share-small-b","miner_id":"miner-small-b","order_key":"05","p2mr_program_hex":"5555555555555555555555555555555555555555555555555555555555555555","share_difficulty":1,"network_difficulty":200000,"template_height":199,"job_id":"job-2004","job_issued_at_ms":2004,"accepted_at_ms":2004,"ntime":1800000000},{"share_seq":6,"share_id":"share-small-c","miner_id":"miner-small-c","order_key":"06","p2mr_program_hex":"6666666666666666666666666666666666666666666666666666666666666666","share_difficulty":1,"network_difficulty":200000,"template_height":199,"job_id":"job-2005","job_issued_at_ms":2005,"accepted_at_ms":2005,"ntime":1800000000}]} \ No newline at end of file diff --git a/tests/fixtures/prism-load-trend/README.md b/tests/fixtures/prism-load-trend/README.md new file mode 100644 index 000000000..9815e6db6 --- /dev/null +++ b/tests/fixtures/prism-load-trend/README.md @@ -0,0 +1,14 @@ +# prism-load-trend fixtures (#551) + +`regression/` is a synthetic series for `scripts/prism_load_regress.py`: +twelve nightly L2 rows of `throughput-20k-window-1fe` on one runner class and +fsync band (`history/trend/L2/2026-09.jsonl`, runs 1001 to 1012, within a few +percent of each other), and run 1013 (`current/`) as it would look with a +10 ms sleep inside the share append: server-side ACK mean 2 ms to 12.1 ms, +client ACK p50 10 ms to 22.75 ms and p99 40 ms to 91 ms. The rule must flag +those three numbers, and only those, with the commit range from run 1012's +commit to run 1013's. + + python3 scripts/prism_load_regress.py \ + --history tests/fixtures/prism-load-trend/regression/history \ + --rows tests/fixtures/prism-load-trend/regression/current diff --git a/tests/fixtures/prism-load-trend/regression/current/throughput-20k-window-1fe.json b/tests/fixtures/prism-load-trend/regression/current/throughput-20k-window-1fe.json new file mode 100644 index 000000000..b49947c48 --- /dev/null +++ b/tests/fixtures/prism-load-trend/regression/current/throughput-20k-window-1fe.json @@ -0,0 +1,66 @@ +{ + "schema": "qbit.prism.load-trend-row.v1", + "kind": "preset", + "recorded_at": "2026-09-13T02:40:00Z", + "repository": "Qbit-Org/qbit-mining-bootstrap", + "run_id": 1013, + "run_attempt": 1, + "run_url": "https://github.com/Qbit-Org/qbit-mining-bootstrap/actions/runs/1013", + "event": "schedule", + "ref": "3.x.x", + "commit": "03336910cebe160ef006593b9d30a729b5e3ab8f", + "lane": "L2", + "preset": "throughput-20k-window-1fe", + "preset_sha256": "b0dc655875203ad73922c5aaf15bd5cc4b49cd5ae86d1eb5362497dc783e8a3a", + "runner_class": "blacksmith-8vcpu-ubuntu-2404", + "fsync": { + "method": "fdatasync", + "ops_per_second": 2000.0, + "usecs_per_op": 500.0, + "band": "fast", + "band_edges_usecs": [ + 1000.0, + 4000.0 + ] + }, + "host": null, + "outcome": { + "harness_exit_code": 0, + "gate_exit_code": 0, + "result": "pass", + "provenance": "pass" + }, + "report": null, + "headline": { + "run": { + "tip_last_notify_p99_ms": null, + "tips_missing_a_session": null, + "durability_findings": 0 + }, + "phases": { + "steady_state": { + "in_artifact": true, + "completed": true, + "duration_seconds": 120.0, + "target_rate_shares_per_second": 500.0, + "achieved_rate_shares_per_second": 500.0, + "shortfall": 0, + "client_ack_p50_ms": 22.75, + "client_ack_p99_ms": 91.0, + "server_ack_mean_ms": 12.1, + "server_ack_p99_upper_ms": null, + "order_lock_max_waiters": 1, + "order_lock_mean_waiters": 0.1, + "peak_rss_mib": 180.0, + "rejected_valid_shares": 0, + "missing_shares": 0 + } + } + }, + "artifact": { + "name": "prism-load-throughput-20k-window-1fe", + "url": null + }, + "promotion": null, + "asset_url": null +} \ No newline at end of file diff --git a/tests/fixtures/prism-load-trend/regression/history/trend/L2/2026-09.jsonl b/tests/fixtures/prism-load-trend/regression/history/trend/L2/2026-09.jsonl new file mode 100644 index 000000000..a4d55a37c --- /dev/null +++ b/tests/fixtures/prism-load-trend/regression/history/trend/L2/2026-09.jsonl @@ -0,0 +1,12 @@ +{"artifact": {"name": "prism-load-throughput-20k-window-1fe", "url": null}, "asset_url": null, "commit": "2c3df3e638562aba088a2fec0e8a024a25093597", "event": "schedule", "fsync": {"band": "fast", "band_edges_usecs": [1000.0, 4000.0], "method": "fdatasync", "ops_per_second": 2000.0, "usecs_per_op": 500.0}, "headline": {"phases": {"steady_state": {"achieved_rate_shares_per_second": 500.0, "client_ack_p50_ms": 10.0, "client_ack_p99_ms": 40.0, "completed": true, "duration_seconds": 120.0, "in_artifact": true, "missing_shares": 0, "order_lock_max_waiters": 1, "order_lock_mean_waiters": 0.1, "peak_rss_mib": 180.0, "rejected_valid_shares": 0, "server_ack_mean_ms": 2.0, "server_ack_p99_upper_ms": null, "shortfall": 0, "target_rate_shares_per_second": 500.0}}, "run": {"durability_findings": 0, "tip_last_notify_p99_ms": null, "tips_missing_a_session": null}}, "host": null, "kind": "preset", "lane": "L2", "outcome": {"gate_exit_code": 0, "harness_exit_code": 0, "provenance": "pass", "result": "pass"}, "preset": "throughput-20k-window-1fe", "preset_sha256": "b0dc655875203ad73922c5aaf15bd5cc4b49cd5ae86d1eb5362497dc783e8a3a", "promotion": null, "recorded_at": "2026-09-01T02:40:00Z", "ref": "3.x.x", "report": null, "repository": "Qbit-Org/qbit-mining-bootstrap", "run_attempt": 1, "run_id": 1001, "run_url": "https://github.com/Qbit-Org/qbit-mining-bootstrap/actions/runs/1001", "runner_class": "blacksmith-8vcpu-ubuntu-2404", "schema": "qbit.prism.load-trend-row.v1"} +{"artifact": {"name": "prism-load-throughput-20k-window-1fe", "url": null}, "asset_url": null, "commit": "52b0a4375db7d5d955bd6c3bcb57cae4bdd0ca02", "event": "schedule", "fsync": {"band": "fast", "band_edges_usecs": [1000.0, 4000.0], "method": "fdatasync", "ops_per_second": 2000.0, "usecs_per_op": 500.0}, "headline": {"phases": {"steady_state": {"achieved_rate_shares_per_second": 500.0, "client_ack_p50_ms": 10.3, "client_ack_p99_ms": 41.2, "completed": true, "duration_seconds": 120.0, "in_artifact": true, "missing_shares": 0, "order_lock_max_waiters": 1, "order_lock_mean_waiters": 0.1, "peak_rss_mib": 180.0, "rejected_valid_shares": 0, "server_ack_mean_ms": 2.06, "server_ack_p99_upper_ms": null, "shortfall": 0, "target_rate_shares_per_second": 500.0}}, "run": {"durability_findings": 0, "tip_last_notify_p99_ms": null, "tips_missing_a_session": null}}, "host": null, "kind": "preset", "lane": "L2", "outcome": {"gate_exit_code": 0, "harness_exit_code": 0, "provenance": "pass", "result": "pass"}, "preset": "throughput-20k-window-1fe", "preset_sha256": "b0dc655875203ad73922c5aaf15bd5cc4b49cd5ae86d1eb5362497dc783e8a3a", "promotion": null, "recorded_at": "2026-09-02T02:40:00Z", "ref": "3.x.x", "report": null, "repository": "Qbit-Org/qbit-mining-bootstrap", "run_attempt": 1, "run_id": 1002, "run_url": "https://github.com/Qbit-Org/qbit-mining-bootstrap/actions/runs/1002", "runner_class": "blacksmith-8vcpu-ubuntu-2404", "schema": "qbit.prism.load-trend-row.v1"} +{"artifact": {"name": "prism-load-throughput-20k-window-1fe", "url": null}, "asset_url": null, "commit": "f715a31fd71128cc7581cb32a244cb19a3e9f0fb", "event": "schedule", "fsync": {"band": "fast", "band_edges_usecs": [1000.0, 4000.0], "method": "fdatasync", "ops_per_second": 2000.0, "usecs_per_op": 500.0}, "headline": {"phases": {"steady_state": {"achieved_rate_shares_per_second": 500.0, "client_ack_p50_ms": 9.8, "client_ack_p99_ms": 39.2, "completed": true, "duration_seconds": 120.0, "in_artifact": true, "missing_shares": 0, "order_lock_max_waiters": 1, "order_lock_mean_waiters": 0.1, "peak_rss_mib": 180.0, "rejected_valid_shares": 0, "server_ack_mean_ms": 1.96, "server_ack_p99_upper_ms": null, "shortfall": 0, "target_rate_shares_per_second": 500.0}}, "run": {"durability_findings": 0, "tip_last_notify_p99_ms": null, "tips_missing_a_session": null}}, "host": null, "kind": "preset", "lane": "L2", "outcome": {"gate_exit_code": 0, "harness_exit_code": 0, "provenance": "pass", "result": "pass"}, "preset": "throughput-20k-window-1fe", "preset_sha256": "b0dc655875203ad73922c5aaf15bd5cc4b49cd5ae86d1eb5362497dc783e8a3a", "promotion": null, "recorded_at": "2026-09-03T02:40:00Z", "ref": "3.x.x", "report": null, "repository": "Qbit-Org/qbit-mining-bootstrap", "run_attempt": 1, "run_id": 1003, "run_url": "https://github.com/Qbit-Org/qbit-mining-bootstrap/actions/runs/1003", "runner_class": "blacksmith-8vcpu-ubuntu-2404", "schema": "qbit.prism.load-trend-row.v1"} +{"artifact": {"name": "prism-load-throughput-20k-window-1fe", "url": null}, "asset_url": null, "commit": "04b198d21eb462175badfe2982b0e93c4e30dafe", "event": "schedule", "fsync": {"band": "fast", "band_edges_usecs": [1000.0, 4000.0], "method": "fdatasync", "ops_per_second": 2000.0, "usecs_per_op": 500.0}, "headline": {"phases": {"steady_state": {"achieved_rate_shares_per_second": 500.0, "client_ack_p50_ms": 10.1, "client_ack_p99_ms": 40.4, "completed": true, "duration_seconds": 120.0, "in_artifact": true, "missing_shares": 0, "order_lock_max_waiters": 1, "order_lock_mean_waiters": 0.1, "peak_rss_mib": 180.0, "rejected_valid_shares": 0, "server_ack_mean_ms": 2.02, "server_ack_p99_upper_ms": null, "shortfall": 0, "target_rate_shares_per_second": 500.0}}, "run": {"durability_findings": 0, "tip_last_notify_p99_ms": null, "tips_missing_a_session": null}}, "host": null, "kind": "preset", "lane": "L2", "outcome": {"gate_exit_code": 0, "harness_exit_code": 0, "provenance": "pass", "result": "pass"}, "preset": "throughput-20k-window-1fe", "preset_sha256": "b0dc655875203ad73922c5aaf15bd5cc4b49cd5ae86d1eb5362497dc783e8a3a", "promotion": null, "recorded_at": "2026-09-04T02:40:00Z", "ref": "3.x.x", "report": null, "repository": "Qbit-Org/qbit-mining-bootstrap", "run_attempt": 1, "run_id": 1004, "run_url": "https://github.com/Qbit-Org/qbit-mining-bootstrap/actions/runs/1004", "runner_class": "blacksmith-8vcpu-ubuntu-2404", "schema": "qbit.prism.load-trend-row.v1"} +{"artifact": {"name": "prism-load-throughput-20k-window-1fe", "url": null}, "asset_url": null, "commit": "a59a3ed671de879922ad09a87e320f561081e35a", "event": "schedule", "fsync": {"band": "fast", "band_edges_usecs": [1000.0, 4000.0], "method": "fdatasync", "ops_per_second": 2000.0, "usecs_per_op": 500.0}, "headline": {"phases": {"steady_state": {"achieved_rate_shares_per_second": 500.0, "client_ack_p50_ms": 9.7, "client_ack_p99_ms": 38.8, "completed": true, "duration_seconds": 120.0, "in_artifact": true, "missing_shares": 0, "order_lock_max_waiters": 1, "order_lock_mean_waiters": 0.1, "peak_rss_mib": 180.0, "rejected_valid_shares": 0, "server_ack_mean_ms": 1.94, "server_ack_p99_upper_ms": null, "shortfall": 0, "target_rate_shares_per_second": 500.0}}, "run": {"durability_findings": 0, "tip_last_notify_p99_ms": null, "tips_missing_a_session": null}}, "host": null, "kind": "preset", "lane": "L2", "outcome": {"gate_exit_code": 0, "harness_exit_code": 0, "provenance": "pass", "result": "pass"}, "preset": "throughput-20k-window-1fe", "preset_sha256": "b0dc655875203ad73922c5aaf15bd5cc4b49cd5ae86d1eb5362497dc783e8a3a", "promotion": null, "recorded_at": "2026-09-05T02:40:00Z", "ref": "3.x.x", "report": null, "repository": "Qbit-Org/qbit-mining-bootstrap", "run_attempt": 1, "run_id": 1005, "run_url": "https://github.com/Qbit-Org/qbit-mining-bootstrap/actions/runs/1005", "runner_class": "blacksmith-8vcpu-ubuntu-2404", "schema": "qbit.prism.load-trend-row.v1"} +{"artifact": {"name": "prism-load-throughput-20k-window-1fe", "url": null}, "asset_url": null, "commit": "85c3e31833f7847f47d6a055a4dfdfa463f72c4b", "event": "schedule", "fsync": {"band": "fast", "band_edges_usecs": [1000.0, 4000.0], "method": "fdatasync", "ops_per_second": 2000.0, "usecs_per_op": 500.0}, "headline": {"phases": {"steady_state": {"achieved_rate_shares_per_second": 500.0, "client_ack_p50_ms": 10.2, "client_ack_p99_ms": 40.8, "completed": true, "duration_seconds": 120.0, "in_artifact": true, "missing_shares": 0, "order_lock_max_waiters": 1, "order_lock_mean_waiters": 0.1, "peak_rss_mib": 180.0, "rejected_valid_shares": 0, "server_ack_mean_ms": 2.04, "server_ack_p99_upper_ms": null, "shortfall": 0, "target_rate_shares_per_second": 500.0}}, "run": {"durability_findings": 0, "tip_last_notify_p99_ms": null, "tips_missing_a_session": null}}, "host": null, "kind": "preset", "lane": "L2", "outcome": {"gate_exit_code": 0, "harness_exit_code": 0, "provenance": "pass", "result": "pass"}, "preset": "throughput-20k-window-1fe", "preset_sha256": "b0dc655875203ad73922c5aaf15bd5cc4b49cd5ae86d1eb5362497dc783e8a3a", "promotion": null, "recorded_at": "2026-09-06T02:40:00Z", "ref": "3.x.x", "report": null, "repository": "Qbit-Org/qbit-mining-bootstrap", "run_attempt": 1, "run_id": 1006, "run_url": "https://github.com/Qbit-Org/qbit-mining-bootstrap/actions/runs/1006", "runner_class": "blacksmith-8vcpu-ubuntu-2404", "schema": "qbit.prism.load-trend-row.v1"} +{"artifact": {"name": "prism-load-throughput-20k-window-1fe", "url": null}, "asset_url": null, "commit": "62d7f0c0ee10e6b68064b3fe4b4ea8948060f2b8", "event": "schedule", "fsync": {"band": "fast", "band_edges_usecs": [1000.0, 4000.0], "method": "fdatasync", "ops_per_second": 2000.0, "usecs_per_op": 500.0}, "headline": {"phases": {"steady_state": {"achieved_rate_shares_per_second": 500.0, "client_ack_p50_ms": 9.9, "client_ack_p99_ms": 39.6, "completed": true, "duration_seconds": 120.0, "in_artifact": true, "missing_shares": 0, "order_lock_max_waiters": 1, "order_lock_mean_waiters": 0.1, "peak_rss_mib": 180.0, "rejected_valid_shares": 0, "server_ack_mean_ms": 1.98, "server_ack_p99_upper_ms": null, "shortfall": 0, "target_rate_shares_per_second": 500.0}}, "run": {"durability_findings": 0, "tip_last_notify_p99_ms": null, "tips_missing_a_session": null}}, "host": null, "kind": "preset", "lane": "L2", "outcome": {"gate_exit_code": 0, "harness_exit_code": 0, "provenance": "pass", "result": "pass"}, "preset": "throughput-20k-window-1fe", "preset_sha256": "b0dc655875203ad73922c5aaf15bd5cc4b49cd5ae86d1eb5362497dc783e8a3a", "promotion": null, "recorded_at": "2026-09-07T02:40:00Z", "ref": "3.x.x", "report": null, "repository": "Qbit-Org/qbit-mining-bootstrap", "run_attempt": 1, "run_id": 1007, "run_url": "https://github.com/Qbit-Org/qbit-mining-bootstrap/actions/runs/1007", "runner_class": "blacksmith-8vcpu-ubuntu-2404", "schema": "qbit.prism.load-trend-row.v1"} +{"artifact": {"name": "prism-load-throughput-20k-window-1fe", "url": null}, "asset_url": null, "commit": "d46441627dd89e1583d90defabc6da612044030c", "event": "schedule", "fsync": {"band": "fast", "band_edges_usecs": [1000.0, 4000.0], "method": "fdatasync", "ops_per_second": 2000.0, "usecs_per_op": 500.0}, "headline": {"phases": {"steady_state": {"achieved_rate_shares_per_second": 500.0, "client_ack_p50_ms": 10.3, "client_ack_p99_ms": 41.2, "completed": true, "duration_seconds": 120.0, "in_artifact": true, "missing_shares": 0, "order_lock_max_waiters": 1, "order_lock_mean_waiters": 0.1, "peak_rss_mib": 180.0, "rejected_valid_shares": 0, "server_ack_mean_ms": 2.06, "server_ack_p99_upper_ms": null, "shortfall": 0, "target_rate_shares_per_second": 500.0}}, "run": {"durability_findings": 0, "tip_last_notify_p99_ms": null, "tips_missing_a_session": null}}, "host": null, "kind": "preset", "lane": "L2", "outcome": {"gate_exit_code": 0, "harness_exit_code": 0, "provenance": "pass", "result": "pass"}, "preset": "throughput-20k-window-1fe", "preset_sha256": "b0dc655875203ad73922c5aaf15bd5cc4b49cd5ae86d1eb5362497dc783e8a3a", "promotion": null, "recorded_at": "2026-09-08T02:40:00Z", "ref": "3.x.x", "report": null, "repository": "Qbit-Org/qbit-mining-bootstrap", "run_attempt": 1, "run_id": 1008, "run_url": "https://github.com/Qbit-Org/qbit-mining-bootstrap/actions/runs/1008", "runner_class": "blacksmith-8vcpu-ubuntu-2404", "schema": "qbit.prism.load-trend-row.v1"} +{"artifact": {"name": "prism-load-throughput-20k-window-1fe", "url": null}, "asset_url": null, "commit": "a867b8a4c0c15aa78783d2dca269b0ae85312072", "event": "schedule", "fsync": {"band": "fast", "band_edges_usecs": [1000.0, 4000.0], "method": "fdatasync", "ops_per_second": 2000.0, "usecs_per_op": 500.0}, "headline": {"phases": {"steady_state": {"achieved_rate_shares_per_second": 500.0, "client_ack_p50_ms": 10.0, "client_ack_p99_ms": 40.0, "completed": true, "duration_seconds": 120.0, "in_artifact": true, "missing_shares": 0, "order_lock_max_waiters": 1, "order_lock_mean_waiters": 0.1, "peak_rss_mib": 180.0, "rejected_valid_shares": 0, "server_ack_mean_ms": 2.0, "server_ack_p99_upper_ms": null, "shortfall": 0, "target_rate_shares_per_second": 500.0}}, "run": {"durability_findings": 0, "tip_last_notify_p99_ms": null, "tips_missing_a_session": null}}, "host": null, "kind": "preset", "lane": "L2", "outcome": {"gate_exit_code": 0, "harness_exit_code": 0, "provenance": "pass", "result": "pass"}, "preset": "throughput-20k-window-1fe", "preset_sha256": "b0dc655875203ad73922c5aaf15bd5cc4b49cd5ae86d1eb5362497dc783e8a3a", "promotion": null, "recorded_at": "2026-09-09T02:40:00Z", "ref": "3.x.x", "report": null, "repository": "Qbit-Org/qbit-mining-bootstrap", "run_attempt": 1, "run_id": 1009, "run_url": "https://github.com/Qbit-Org/qbit-mining-bootstrap/actions/runs/1009", "runner_class": "blacksmith-8vcpu-ubuntu-2404", "schema": "qbit.prism.load-trend-row.v1"} +{"artifact": {"name": "prism-load-throughput-20k-window-1fe", "url": null}, "asset_url": null, "commit": "0ec80db42273652f94be428a823bb18f87d4365d", "event": "schedule", "fsync": {"band": "fast", "band_edges_usecs": [1000.0, 4000.0], "method": "fdatasync", "ops_per_second": 2000.0, "usecs_per_op": 500.0}, "headline": {"phases": {"steady_state": {"achieved_rate_shares_per_second": 500.0, "client_ack_p50_ms": 9.8, "client_ack_p99_ms": 39.2, "completed": true, "duration_seconds": 120.0, "in_artifact": true, "missing_shares": 0, "order_lock_max_waiters": 1, "order_lock_mean_waiters": 0.1, "peak_rss_mib": 180.0, "rejected_valid_shares": 0, "server_ack_mean_ms": 1.96, "server_ack_p99_upper_ms": null, "shortfall": 0, "target_rate_shares_per_second": 500.0}}, "run": {"durability_findings": 0, "tip_last_notify_p99_ms": null, "tips_missing_a_session": null}}, "host": null, "kind": "preset", "lane": "L2", "outcome": {"gate_exit_code": 0, "harness_exit_code": 0, "provenance": "pass", "result": "pass"}, "preset": "throughput-20k-window-1fe", "preset_sha256": "b0dc655875203ad73922c5aaf15bd5cc4b49cd5ae86d1eb5362497dc783e8a3a", "promotion": null, "recorded_at": "2026-09-10T02:40:00Z", "ref": "3.x.x", "report": null, "repository": "Qbit-Org/qbit-mining-bootstrap", "run_attempt": 1, "run_id": 1010, "run_url": "https://github.com/Qbit-Org/qbit-mining-bootstrap/actions/runs/1010", "runner_class": "blacksmith-8vcpu-ubuntu-2404", "schema": "qbit.prism.load-trend-row.v1"} +{"artifact": {"name": "prism-load-throughput-20k-window-1fe", "url": null}, "asset_url": null, "commit": "06290d7d723174013690371f71dc1f0b65d64e2c", "event": "schedule", "fsync": {"band": "fast", "band_edges_usecs": [1000.0, 4000.0], "method": "fdatasync", "ops_per_second": 2000.0, "usecs_per_op": 500.0}, "headline": {"phases": {"steady_state": {"achieved_rate_shares_per_second": 500.0, "client_ack_p50_ms": 10.1, "client_ack_p99_ms": 40.4, "completed": true, "duration_seconds": 120.0, "in_artifact": true, "missing_shares": 0, "order_lock_max_waiters": 1, "order_lock_mean_waiters": 0.1, "peak_rss_mib": 180.0, "rejected_valid_shares": 0, "server_ack_mean_ms": 2.02, "server_ack_p99_upper_ms": null, "shortfall": 0, "target_rate_shares_per_second": 500.0}}, "run": {"durability_findings": 0, "tip_last_notify_p99_ms": null, "tips_missing_a_session": null}}, "host": null, "kind": "preset", "lane": "L2", "outcome": {"gate_exit_code": 0, "harness_exit_code": 0, "provenance": "pass", "result": "pass"}, "preset": "throughput-20k-window-1fe", "preset_sha256": "b0dc655875203ad73922c5aaf15bd5cc4b49cd5ae86d1eb5362497dc783e8a3a", "promotion": null, "recorded_at": "2026-09-11T02:40:00Z", "ref": "3.x.x", "report": null, "repository": "Qbit-Org/qbit-mining-bootstrap", "run_attempt": 1, "run_id": 1011, "run_url": "https://github.com/Qbit-Org/qbit-mining-bootstrap/actions/runs/1011", "runner_class": "blacksmith-8vcpu-ubuntu-2404", "schema": "qbit.prism.load-trend-row.v1"} +{"artifact": {"name": "prism-load-throughput-20k-window-1fe", "url": null}, "asset_url": null, "commit": "61794906afeb4df2bb55cd33ee335885a1c4eb93", "event": "schedule", "fsync": {"band": "fast", "band_edges_usecs": [1000.0, 4000.0], "method": "fdatasync", "ops_per_second": 2000.0, "usecs_per_op": 500.0}, "headline": {"phases": {"steady_state": {"achieved_rate_shares_per_second": 500.0, "client_ack_p50_ms": 9.9, "client_ack_p99_ms": 39.6, "completed": true, "duration_seconds": 120.0, "in_artifact": true, "missing_shares": 0, "order_lock_max_waiters": 1, "order_lock_mean_waiters": 0.1, "peak_rss_mib": 180.0, "rejected_valid_shares": 0, "server_ack_mean_ms": 1.98, "server_ack_p99_upper_ms": null, "shortfall": 0, "target_rate_shares_per_second": 500.0}}, "run": {"durability_findings": 0, "tip_last_notify_p99_ms": null, "tips_missing_a_session": null}}, "host": null, "kind": "preset", "lane": "L2", "outcome": {"gate_exit_code": 0, "harness_exit_code": 0, "provenance": "pass", "result": "pass"}, "preset": "throughput-20k-window-1fe", "preset_sha256": "b0dc655875203ad73922c5aaf15bd5cc4b49cd5ae86d1eb5362497dc783e8a3a", "promotion": null, "recorded_at": "2026-09-12T02:40:00Z", "ref": "3.x.x", "report": null, "repository": "Qbit-Org/qbit-mining-bootstrap", "run_attempt": 1, "run_id": 1012, "run_url": "https://github.com/Qbit-Org/qbit-mining-bootstrap/actions/runs/1012", "runner_class": "blacksmith-8vcpu-ubuntu-2404", "schema": "qbit.prism.load-trend-row.v1"} diff --git a/tests/fixtures/prism-load-trend/variance/generate.py b/tests/fixtures/prism-load-trend/variance/generate.py new file mode 100644 index 000000000..e2bdcee7a --- /dev/null +++ b/tests/fixtures/prism-load-trend/variance/generate.py @@ -0,0 +1,68 @@ +#!/usr/bin/env python3 +"""Write runner-probe-variance.json with scripts/prism_load_probe.py's own +`variance` (#542), from four synthetic 8 vCPU VMs of two runs each at the +regression fixture's fsync cost. tests/test_prism_load_regress.py checks the +checked-in file still equals this output, so it keeps #542's real shape. + +Usage: python3 tests/fixtures/prism-load-trend/variance/generate.py [--check] +""" + +from __future__ import annotations + +import json +from pathlib import Path +import sys + +HERE = Path(__file__).resolve().parent +ROOT = HERE.parents[3] +sys.path.insert(0, str(ROOT / "scripts")) + +import prism_load_probe as probe # noqa: E402 + +PRESET = "throughput-20k-window-1fe" +OUT = HERE / "runner-probe-variance.json" +# Per VM: (rate, ack p50, ack p99) for its two runs. VM 4 is a slower VM, so +# VM-to-VM spread is visible beside the run-to-run spread. +VMS = [ + [(500.0, 9.8, 39.0), (500.0, 10.1, 40.5)], + [(499.5, 10.0, 40.0), (500.0, 9.9, 41.0)], + [(500.0, 10.2, 39.5), (499.8, 10.0, 40.2)], + [(498.0, 11.0, 44.0), (498.5, 11.2, 45.0)], +] + + +def run(rate: float, p50: float, p99: float) -> dict: + result = { + "harness_exit_code": 0, "gate_exit_code": 0, "timed_out": False, + "wall_seconds": 600.0, "peak_used_mib": 9000.0, "baseline_used_mib": 1000.0, + "mem_total_mib": 32000.0, "headroom_mib": 23000.0, + "fsync": {"method": "fdatasync", "ops_per_second": 2000.0, "usecs_per_op": 500.0}, + "provenance": "pass", "seed_seconds": 10.0, "seed_rows": 20000, + "phases": {"steady_state": {"shortfall": 0, "ack_p50_ms": p50, "ack_p99_ms": p99, + "achieved_rate_shares_per_second": rate}}, + "tip_last_notify_p99_ms": None, "tips_missing_a_session": None, + } + result["unmeasured_reason"] = probe.unmeasured_reason(result) + return result + + +def document() -> dict: + rows = [{"schema": probe.ROW_SCHEMA, "kind": "probe", "class": 8, "runner": probe.runner_label(8), + "target_cache": "sticky-disk", "repeat": repeat, "commit": "f1x7ure", + "target": {}, "build": {}, + "host": {"block_device": {"write_cache": "write back", "fua": "0", "model": "m"}}, + "runs": {PRESET: [run(*r) for r in runs]}} + for repeat, runs in enumerate(VMS, start=1)] + expected = probe.matrices("8", "sticky-disk", "0", PRESET, str(len(VMS)), "2") + presets, _ = probe.load_presets() + return probe.variance(rows, expected, presets) + + +def text() -> str: + return json.dumps(document(), indent=2, sort_keys=True) + "\n" + + +if __name__ == "__main__": + if sys.argv[1:] == ["--check"]: + raise SystemExit(0 if OUT.read_text(encoding="utf-8") == text() else 1) + OUT.write_text(text(), encoding="utf-8") diff --git a/tests/fixtures/prism-load-trend/variance/runner-probe-variance.json b/tests/fixtures/prism-load-trend/variance/runner-probe-variance.json new file mode 100644 index 000000000..3b0ac6fb1 --- /dev/null +++ b/tests/fixtures/prism-load-trend/variance/runner-probe-variance.json @@ -0,0 +1,599 @@ +{ + "commit": "f1x7ure", + "fsync_band_edges_usecs_per_op": [ + 125, + 250, + 500, + 1000, + 2000, + 4000 + ], + "groups": [ + { + "class": 8, + "disks": [ + { + "fua": "0", + "model": "m", + "vms": 4, + "write_cache": "write back" + } + ], + "fsync_band": "all", + "fsync_usecs_per_op": { + "cv": 0.0, + "mad": 0.0, + "max": 500.0, + "median": 500.0, + "min": 500.0, + "n": 8, + "unknown": 0 + }, + "jobs": { + "planned": 4, + "with_row": 4 + }, + "metrics": { + "headroom_mib": { + "all": { + "cv": 0.0, + "mad": 0.0, + "max": 23000.0, + "median": 23000.0, + "min": 23000.0, + "n": 8, + "unknown": 0 + }, + "run_to_run": { + "jobs": 4, + "median_cv": 0.0, + "median_mad": 0.0 + }, + "unit": "MiB, MemTotal minus the peak used", + "vm_to_vm": { + "cv": 0.0, + "mad": 0.0, + "max": 23000.0, + "median": 23000.0, + "min": 23000.0, + "n": 4, + "unknown": 0 + } + }, + "peak_used_mib": { + "all": { + "cv": 0.0, + "mad": 0.0, + "max": 9000.0, + "median": 9000.0, + "min": 9000.0, + "n": 8, + "unknown": 0 + }, + "run_to_run": { + "jobs": 4, + "median_cv": 0.0, + "median_mad": 0.0 + }, + "unit": "MiB, the host's MemTotal - MemAvailable sampled every second", + "vm_to_vm": { + "cv": 0.0, + "mad": 0.0, + "max": 9000.0, + "median": 9000.0, + "min": 9000.0, + "n": 4, + "unknown": 0 + } + }, + "seed_seconds": { + "all": { + "cv": 0.0, + "mad": 0.0, + "max": 10.0, + "median": 10.0, + "min": 10.0, + "n": 8, + "unknown": 0 + }, + "run_to_run": { + "jobs": 4, + "median_cv": 0.0, + "median_mad": 0.0 + }, + "unit": "seconds, as the harness reports the window seed", + "vm_to_vm": { + "cv": 0.0, + "mad": 0.0, + "max": 10.0, + "median": 10.0, + "min": 10.0, + "n": 4, + "unknown": 0 + } + }, + "steady_state.achieved_rate_shares_per_second": { + "all": { + "cv": 0.0016, + "mad": 0.1, + "max": 500.0, + "median": 499.9, + "min": 498.0, + "n": 8, + "unknown": 0 + }, + "run_to_run": { + "jobs": 4, + "median_cv": 0.0005, + "median_mad": 0.175 + }, + "unit": "shares per second, reconciled acknowledged count over the phase", + "vm_to_vm": { + "cv": 0.0016, + "mad": 0.125, + "max": 500.0, + "median": 499.825, + "min": 498.25, + "n": 4, + "unknown": 0 + } + }, + "steady_state.ack_p50_ms": { + "all": { + "cv": 0.0512, + "mad": 0.15, + "max": 11.2, + "median": 10.05, + "min": 9.8, + "n": 8, + "unknown": 0 + }, + "run_to_run": { + "jobs": 4, + "median_cv": 0.0134, + "median_mad": 0.1 + }, + "unit": "milliseconds, the client's monotonic clock, accepted shares", + "vm_to_vm": { + "cv": 0.054, + "mad": 0.075, + "max": 11.1, + "median": 10.025, + "min": 9.95, + "n": 4, + "unknown": 0 + } + }, + "steady_state.ack_p99_ms": { + "all": { + "cv": 0.0527, + "mad": 0.75, + "max": 45.0, + "median": 40.35, + "min": 39.0, + "n": 8, + "unknown": 0 + }, + "run_to_run": { + "jobs": 4, + "median_cv": 0.0167, + "median_mad": 0.5 + }, + "unit": "milliseconds, the client's monotonic clock, accepted shares", + "vm_to_vm": { + "cv": 0.0549, + "mad": 0.375, + "max": 44.5, + "median": 40.175, + "min": 39.75, + "n": 4, + "unknown": 0 + } + }, + "steady_state.shortfall": { + "all": { + "cv": null, + "mad": 0.0, + "max": 0, + "median": 0.0, + "min": 0, + "n": 8, + "unknown": 0 + }, + "run_to_run": { + "jobs": 4, + "median_cv": null, + "median_mad": 0.0 + }, + "unit": "offers no session could take", + "vm_to_vm": { + "cv": null, + "mad": 0.0, + "max": 0.0, + "median": 0.0, + "min": 0.0, + "n": 4, + "unknown": 0 + } + }, + "tip_last_notify_p99_ms": { + "all": { + "cv": null, + "mad": null, + "max": null, + "median": null, + "min": null, + "n": 0, + "unknown": 8 + }, + "run_to_run": null, + "unit": "milliseconds, nearest-rank p99 of each tip's slowest session", + "vm_to_vm": { + "cv": null, + "mad": null, + "max": null, + "median": null, + "min": null, + "n": 0, + "unknown": 4 + } + }, + "wall_seconds": { + "all": { + "cv": 0.0, + "mad": 0.0, + "max": 600.0, + "median": 600.0, + "min": 600.0, + "n": 8, + "unknown": 0 + }, + "run_to_run": { + "jobs": 4, + "median_cv": 0.0, + "median_mad": 0.0 + }, + "unit": "seconds, the runner's monotonic clock, around pg_test_fsync, the harness and the gate", + "vm_to_vm": { + "cv": 0.0, + "mad": 0.0, + "max": 600.0, + "median": 600.0, + "min": 600.0, + "n": 4, + "unknown": 0 + } + } + }, + "preset": "throughput-20k-window-1fe", + "runner": "blacksmith-8vcpu-ubuntu-2404", + "runs": { + "measured": 8, + "planned": 8, + "unknown": 0, + "unknown_reasons": {} + }, + "size_evidence": { + "fits": true, + "gated_phases": [ + "steady_state" + ], + "gated_shortfall_max": 0, + "headroom_fraction": 0.25, + "headroom_mib_min": 23000.0, + "headroom_mib_required": 8000.0, + "mem_total_mib_min": 32000.0, + "peak_used_mib_max": 9000.0, + "preset_min_mem_available_mib": 6144, + "runs_measured": 8, + "runs_planned": 8, + "runs_with_gated_shortfall": 0, + "why": "all 8 run(s) measured, no gated shortfall, headroom held" + } + }, + { + "class": 8, + "disks": [ + { + "fua": "0", + "model": "m", + "vms": 4, + "write_cache": "write back" + } + ], + "fsync_band": "500-1000us", + "fsync_usecs_per_op": { + "cv": 0.0, + "mad": 0.0, + "max": 500.0, + "median": 500.0, + "min": 500.0, + "n": 8, + "unknown": 0 + }, + "jobs": { + "planned": 4, + "with_row": 4 + }, + "metrics": { + "headroom_mib": { + "all": { + "cv": 0.0, + "mad": 0.0, + "max": 23000.0, + "median": 23000.0, + "min": 23000.0, + "n": 8, + "unknown": 0 + }, + "run_to_run": { + "jobs": 4, + "median_cv": 0.0, + "median_mad": 0.0 + }, + "unit": "MiB, MemTotal minus the peak used", + "vm_to_vm": { + "cv": 0.0, + "mad": 0.0, + "max": 23000.0, + "median": 23000.0, + "min": 23000.0, + "n": 4, + "unknown": 0 + } + }, + "peak_used_mib": { + "all": { + "cv": 0.0, + "mad": 0.0, + "max": 9000.0, + "median": 9000.0, + "min": 9000.0, + "n": 8, + "unknown": 0 + }, + "run_to_run": { + "jobs": 4, + "median_cv": 0.0, + "median_mad": 0.0 + }, + "unit": "MiB, the host's MemTotal - MemAvailable sampled every second", + "vm_to_vm": { + "cv": 0.0, + "mad": 0.0, + "max": 9000.0, + "median": 9000.0, + "min": 9000.0, + "n": 4, + "unknown": 0 + } + }, + "seed_seconds": { + "all": { + "cv": 0.0, + "mad": 0.0, + "max": 10.0, + "median": 10.0, + "min": 10.0, + "n": 8, + "unknown": 0 + }, + "run_to_run": { + "jobs": 4, + "median_cv": 0.0, + "median_mad": 0.0 + }, + "unit": "seconds, as the harness reports the window seed", + "vm_to_vm": { + "cv": 0.0, + "mad": 0.0, + "max": 10.0, + "median": 10.0, + "min": 10.0, + "n": 4, + "unknown": 0 + } + }, + "steady_state.achieved_rate_shares_per_second": { + "all": { + "cv": 0.0016, + "mad": 0.1, + "max": 500.0, + "median": 499.9, + "min": 498.0, + "n": 8, + "unknown": 0 + }, + "run_to_run": { + "jobs": 4, + "median_cv": 0.0005, + "median_mad": 0.175 + }, + "unit": "shares per second, reconciled acknowledged count over the phase", + "vm_to_vm": { + "cv": 0.0016, + "mad": 0.125, + "max": 500.0, + "median": 499.825, + "min": 498.25, + "n": 4, + "unknown": 0 + } + }, + "steady_state.ack_p50_ms": { + "all": { + "cv": 0.0512, + "mad": 0.15, + "max": 11.2, + "median": 10.05, + "min": 9.8, + "n": 8, + "unknown": 0 + }, + "run_to_run": { + "jobs": 4, + "median_cv": 0.0134, + "median_mad": 0.1 + }, + "unit": "milliseconds, the client's monotonic clock, accepted shares", + "vm_to_vm": { + "cv": 0.054, + "mad": 0.075, + "max": 11.1, + "median": 10.025, + "min": 9.95, + "n": 4, + "unknown": 0 + } + }, + "steady_state.ack_p99_ms": { + "all": { + "cv": 0.0527, + "mad": 0.75, + "max": 45.0, + "median": 40.35, + "min": 39.0, + "n": 8, + "unknown": 0 + }, + "run_to_run": { + "jobs": 4, + "median_cv": 0.0167, + "median_mad": 0.5 + }, + "unit": "milliseconds, the client's monotonic clock, accepted shares", + "vm_to_vm": { + "cv": 0.0549, + "mad": 0.375, + "max": 44.5, + "median": 40.175, + "min": 39.75, + "n": 4, + "unknown": 0 + } + }, + "steady_state.shortfall": { + "all": { + "cv": null, + "mad": 0.0, + "max": 0, + "median": 0.0, + "min": 0, + "n": 8, + "unknown": 0 + }, + "run_to_run": { + "jobs": 4, + "median_cv": null, + "median_mad": 0.0 + }, + "unit": "offers no session could take", + "vm_to_vm": { + "cv": null, + "mad": 0.0, + "max": 0.0, + "median": 0.0, + "min": 0.0, + "n": 4, + "unknown": 0 + } + }, + "tip_last_notify_p99_ms": { + "all": { + "cv": null, + "mad": null, + "max": null, + "median": null, + "min": null, + "n": 0, + "unknown": 8 + }, + "run_to_run": null, + "unit": "milliseconds, nearest-rank p99 of each tip's slowest session", + "vm_to_vm": { + "cv": null, + "mad": null, + "max": null, + "median": null, + "min": null, + "n": 0, + "unknown": 4 + } + }, + "wall_seconds": { + "all": { + "cv": 0.0, + "mad": 0.0, + "max": 600.0, + "median": 600.0, + "min": 600.0, + "n": 8, + "unknown": 0 + }, + "run_to_run": { + "jobs": 4, + "median_cv": 0.0, + "median_mad": 0.0 + }, + "unit": "seconds, the runner's monotonic clock, around pg_test_fsync, the harness and the gate", + "vm_to_vm": { + "cv": 0.0, + "mad": 0.0, + "max": 600.0, + "median": 600.0, + "min": 600.0, + "n": 4, + "unknown": 0 + } + } + }, + "preset": "throughput-20k-window-1fe", + "runner": "blacksmith-8vcpu-ubuntu-2404", + "runs": { + "measured": 8, + "planned": 8, + "unknown": 0, + "unknown_reasons": {} + }, + "size_evidence": null + } + ], + "in_job_repeats": 2, + "jobs": { + "by_vm": [ + { + "class": 8, + "fsync_band": "500-1000us", + "fsync_usecs_per_op": 500.0, + "repeat": 1, + "target_cache": "sticky-disk" + }, + { + "class": 8, + "fsync_band": "500-1000us", + "fsync_usecs_per_op": 500.0, + "repeat": 2, + "target_cache": "sticky-disk" + }, + { + "class": 8, + "fsync_band": "500-1000us", + "fsync_usecs_per_op": 500.0, + "repeat": 3, + "target_cache": "sticky-disk" + }, + { + "class": 8, + "fsync_band": "500-1000us", + "fsync_usecs_per_op": 500.0, + "repeat": 4, + "target_cache": "sticky-disk" + } + ], + "missing": [], + "planned": 4, + "with_row": 4 + }, + "rows_without_commit": 0, + "schema": "qbit.prism.runner-probe-variance.v1" +} diff --git a/tests/fixtures/window_pipeline_parity/README.md b/tests/fixtures/window_pipeline_parity/README.md new file mode 100644 index 000000000..bef5d453f --- /dev/null +++ b/tests/fixtures/window_pipeline_parity/README.md @@ -0,0 +1,130 @@ +# Frozen payout-window corpus + +`reference.json` is a frozen corpus of 35 payout-window cases: inputs, +rejection categories, and the exact output bytes (canonical window bytes and +digest, per-record stream, compact spool tail, advance stats). The Python +differential oracle generated it on 2.x.x. 3.x.x removed that oracle, and two +Rust tests read the corpus instead: + +- `crates/qbit-prism/tests/window_frozen_vectors.rs` replays every case + against `qbit_prism::window::PayoutWindow`; +- `crates/qbit-prism/tests/window_daemon_gate.rs` replays every case through + the real `qbit-prism-build-audit-bundle --serve` daemon. + +Both run under `cargo test` and need no Python or other inputs. Both also +replay the cases in `supplementary.json` (see below) under a separate tally. + +## Pins + +- `reference.json` is pinned by its sha256, + `017c787d3b894d92702d65774e47224ce5a09838bcd1118549f740a21b406142`, in + `crates/qbit-prism/tests/support/window_corpus.rs`. +- `supplementary.json` is pinned there too, by its sha256, + `4a89967643938cfbec933b7f5582118e583245986eb2adf9e533d017ea252fc6`. +- Every case's input document is pinned by its `input_sha256` in + `reference.json`. That is Python's + `json.dumps(doc, sort_keys=True, separators=(",", ":"))` with its default + `ensure_ascii`. Both tests recompute it for all 35 cases before comparing any + output. + +Changing either file needs a reviewed commit that updates these pins. Never +edit expected values to make a test pass: a disagreement may be a payout bug. + +## `inputs-unpinned.json` + +Thirty cases carry their input in `pinned_literals.input`. The other five +(`bulk-seeded`, `multi-page-interior-cutoff`, `page-boundary-511`, +`page-boundary-512`, `page-boundary-513`) don't. The 2.x.x oracle generated +them with Python's seeded `random.Random`, which isn't ported to Rust. + +The sidecar holds those five input documents, exported from 2.x.x at +`504846cc0b72e8f86ed17f896d4ccbbe196a31dc`. Its `command` field records the +exact export command. That command checks each document against the frozen +`input_sha256` before writing anything. + +## `supplementary.json` + +The corpus never expires a whole pre-existing page at exactly +`window_weight`, so a `>` in place of that `>=` passes every corpus case. +`supplementary.json` holds four cases that pin the boundary. They were +exported from the same 2.x.x oracle at +`504846cc0b72e8f86ed17f896d4ccbbe196a31dc`, and each entry has exactly the +`reference.json` v2 shape: `why`, `input_sha256`, `pinned_literals` (input, +record JSONs, canonical bytes and spool tail), and every output field. + +| Case | Window before the advance | What the advance pins | +| --- | --- | --- | +| `advance-page-expiry-at-weight` | weight 2, page_size 2, one page | the old page expires with exactly the weight left | +| `advance-page-expiry-at-weight-second-page` | weight 4, page_size 2, two pages | the first page expires above the weight, the second with exactly the weight left | +| `advance-page-expiry-one-below-weight` | weight 3, page_size 2, one page | expiring the page would leave one below the weight, so it stays and one row expires | +| `advance-page-expiry-one-above-weight` | weight 3, page_size 2, one page | the page expires with one above the weight left | + +Every expected value, `touched_pages` included, is the oracle's output; none +is computed by hand. Both tests replay these cases with the same comparisons +as the corpus, under their own tally +(`4 cases, 0 rejections, 4 byte-compared, 0 declined, 4 with advance stats`), +so the corpus counts stay as frozen. + +The file's `command` field is the exact export. Run it in the extract's root. +It writes `supplementary.json` there, and exits non-zero if the oracle +rejects any case. Two runs produce identical bytes. + +## Compact spool tail + +Each frozen `spool_tail` is the compact build-request suffix the 2.x.x +coordinator spooled: +`,"compact_share_identities":[...],"compact_shares":[...]}`. The native test +compares it with the test's own encoder. The daemon gate also makes the +daemon's real decoder consume it: + +- It prefixes the frozen bytes with a `found_block` and a `window_key` and + sends the result as a `--serve` build request. +- The build summary must equal the one the same daemon builds from the window + it prepared from the case's full records. +- The summary's reward manifest lists every counted share with the fields the + compact format carries: `share_seq`, `share_id`, `miner_id`, `order_key`, + `p2mr_program_hex`, `share_difficulty`, `job_issued_at_ms`, + `accepted_at_ms` and `credit_policy`. The fields it doesn't carry + (`network_difficulty`, `template_height`, `job_id`, `ntime`) never reach the + summary. + +The five sidecar cases pin no spool literal. For them the gate sends the test +encoder's bytes, and only once those hash to the frozen `spool_tail_sha256`. +The empty window has no rows to upload and isn't sent. + +## Regenerating + +In an extract of 2.x.x at the pinned commit +(`git archive 504846cc0b72e8f86ed17f896d4ccbbe196a31dc | tar -x -C <dir>`), +with Python 3.14 and no database: + +1. Run the no-op test. It must report 29 tests OK: + `PYTHONDONTWRITEBYTECODE=1 python3 -m unittest tests.test_window_pipeline_parity -v`. +2. Regenerate the corpus with the file's own `regenerate_with` command: + `python3 -m tests.window_pipeline_parity regenerate`. +3. Export the sidecar with the `command` recorded in `inputs-unpinned.json`. + It writes `inputs-unpinned.json` in the extract's root. +4. Export the supplementary cases with the `command` recorded in + `supplementary.json`. It writes `supplementary.json` in the extract's root. +5. Copy the three files here and update the pins above. + +## Integer domain + +3.x.x carries integers at fixed widths: `share_seq` and `template_height` are +u64, difficulties and `window_weight` are u128, `ntime` is u32, timestamps are +i64, `page_size` is u64, and the total difficulty is u128. 2.x.x's Python +oracle had unbounded integers. Two cases lie outside these widths: + +| Case | First out-of-domain value | Width | +| --- | --- | --- | +| `difficulty-beyond-u128` | `window_weight` = 2^128 + 2^64 | u128 | +| `ntime-beyond-u32` | `records[0].ntime` = 2^32 | u32 | + +On 3.x.x their inputs can't be represented. The typed parse fails, and the +daemon answers `out_of_range` with that field and width. The tests assert this +decline explicitly, but +the frozen output bytes for these two cases came from unbounded integers and +aren't compared. + +`wide-integers` sits at the edges of the widths (2^64-1, 2^128-1) and must +match byte for byte. diff --git a/tests/fixtures/window_pipeline_parity/inputs-unpinned.json b/tests/fixtures/window_pipeline_parity/inputs-unpinned.json new file mode 100644 index 000000000..787fef10d --- /dev/null +++ b/tests/fixtures/window_pipeline_parity/inputs-unpinned.json @@ -0,0 +1 @@ +{"cases":{"bulk-seeded":{"advances":[{"anchor_job_issued_at_ms":1000010,"records":[{"accepted_at_ms":1000005,"credit_policy":null,"job_id":"job-4001","job_issued_at_ms":999970,"miner_id":"miner-8","network_difficulty":1000006,"ntime":1700004001,"order_key":"08:miner-8","p2mr_program_hex":"a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1","share_difficulty":19,"share_id":"share-4001","share_seq":4001,"template_height":800125},{"accepted_at_ms":1000003,"credit_policy":null,"job_id":"job-4002","job_issued_at_ms":999982,"miner_id":"miner-9","network_difficulty":1000007,"ntime":1700004002,"order_key":"09:miner-9","p2mr_program_hex":"a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2","share_difficulty":19,"share_id":"share-4002","share_seq":4002,"template_height":800125},{"accepted_at_ms":999970,"credit_policy":"stale-grace","job_id":"job-4003","job_issued_at_ms":1000003,"miner_id":"miner-10","network_difficulty":1000008,"ntime":1700004003,"order_key":"10:miner-10","p2mr_program_hex":"a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3","share_difficulty":101,"share_id":"share-4003","share_seq":4003,"template_height":800125},{"accepted_at_ms":1000004,"credit_policy":"stale-grace","job_id":"job-4004","job_issued_at_ms":999988,"miner_id":"miner-0","network_difficulty":1000009,"ntime":1700004004,"order_key":"00:miner-0","p2mr_program_hex":"a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4","share_difficulty":19,"share_id":"share-4004","share_seq":4004,"template_height":800125},{"accepted_at_ms":999976,"credit_policy":"stale-grace","job_id":"job-4005","job_issued_at_ms":1000001,"miner_id":"miner-1","network_difficulty":1000010,"ntime":1700004005,"order_key":"01:miner-1","p2mr_program_hex":"a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5","share_difficulty":19,"share_id":"share-4005","share_seq":4005,"template_height":800125},{"accepted_at_ms":999990,"credit_policy":null,"job_id":"job-4006","job_issued_at_ms":1000003,"miner_id":"miner-2","network_difficulty":1000011,"ntime":1700004006,"order_key":"02:miner-2","p2mr_program_hex":"a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6","share_difficulty":101,"share_id":"share-4006","share_seq":4006,"template_height":800125},{"accepted_at_ms":1000008,"credit_policy":"stale-grace","job_id":"job-4007","job_issued_at_ms":999986,"miner_id":"miner-3","network_difficulty":1000012,"ntime":1700004007,"order_key":"03:miner-3","p2mr_program_hex":"a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7","share_difficulty":2,"share_id":"share-4007","share_seq":4007,"template_height":800125},{"accepted_at_ms":1000004,"credit_policy":"stale-grace","job_id":"job-4008","job_issued_at_ms":999984,"miner_id":"miner-4","network_difficulty":1000013,"ntime":1700004008,"order_key":"04:miner-4","p2mr_program_hex":"a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8","share_difficulty":1000,"share_id":"share-4008","share_seq":4008,"template_height":800125},{"accepted_at_ms":1000001,"credit_policy":null,"job_id":"job-4009","job_issued_at_ms":1000005,"miner_id":"miner-5","network_difficulty":1000014,"ntime":1700004009,"order_key":"05:miner-5","p2mr_program_hex":"a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9","share_difficulty":3,"share_id":"share-4009","share_seq":4009,"template_height":800125},{"accepted_at_ms":1000003,"credit_policy":"stale-grace","job_id":"job-4010","job_issued_at_ms":1000001,"miner_id":"miner-6","network_difficulty":1000015,"ntime":1700004010,"order_key":"06:miner-6","p2mr_program_hex":"aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa","share_difficulty":1000,"share_id":"share-4010","share_seq":4010,"template_height":800125},{"accepted_at_ms":1000007,"credit_policy":null,"job_id":"job-4011","job_issued_at_ms":999977,"miner_id":"miner-7","network_difficulty":1000016,"ntime":1700004011,"order_key":"07:miner-7","p2mr_program_hex":"abababababababababababababababababababababababababababababababab","share_difficulty":1,"share_id":"share-4011","share_seq":4011,"template_height":800125},{"accepted_at_ms":999971,"credit_policy":null,"job_id":"job-4012","job_issued_at_ms":1000005,"miner_id":"miner-8","network_difficulty":1000000,"ntime":1700004012,"order_key":"08:miner-8","p2mr_program_hex":"acacacacacacacacacacacacacacacacacacacacacacacacacacacacacacacac","share_difficulty":1000,"share_id":"share-4012","share_seq":4012,"template_height":800125},{"accepted_at_ms":1000010,"credit_policy":null,"job_id":"job-4013","job_issued_at_ms":999988,"miner_id":"miner-9","network_difficulty":1000001,"ntime":1700004013,"order_key":"09:miner-9","p2mr_program_hex":"adadadadadadadadadadadadadadadadadadadadadadadadadadadadadadadad","share_difficulty":2,"share_id":"share-4013","share_seq":4013,"template_height":800125},{"accepted_at_ms":1000003,"credit_policy":"stale-grace","job_id":"job-4014","job_issued_at_ms":999999,"miner_id":"miner-10","network_difficulty":1000002,"ntime":1700004014,"order_key":"10:miner-10","p2mr_program_hex":"aeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeae","share_difficulty":7,"share_id":"share-4014","share_seq":4014,"template_height":800125},{"accepted_at_ms":999994,"credit_policy":null,"job_id":"job-4015","job_issued_at_ms":1000002,"miner_id":"miner-0","network_difficulty":1000003,"ntime":1700004015,"order_key":"00:miner-0","p2mr_program_hex":"afafafafafafafafafafafafafafafafafafafafafafafafafafafafafafafaf","share_difficulty":2,"share_id":"share-4015","share_seq":4015,"template_height":800125},{"accepted_at_ms":1000005,"credit_policy":null,"job_id":"job-4016","job_issued_at_ms":999986,"miner_id":"miner-1","network_difficulty":1000004,"ntime":1700004016,"order_key":"01:miner-1","p2mr_program_hex":"b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0","share_difficulty":7,"share_id":"share-4016","share_seq":4016,"template_height":800125},{"accepted_at_ms":1000006,"credit_policy":null,"job_id":"job-4017","job_issued_at_ms":999972,"miner_id":"miner-2","network_difficulty":1000005,"ntime":1700004017,"order_key":"02:miner-2","p2mr_program_hex":"b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1","share_difficulty":3,"share_id":"share-4017","share_seq":4017,"template_height":800125},{"accepted_at_ms":999989,"credit_policy":null,"job_id":"job-4018","job_issued_at_ms":1000008,"miner_id":"miner-3","network_difficulty":1000006,"ntime":1700004018,"order_key":"03:miner-3","p2mr_program_hex":"b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2","share_difficulty":7,"share_id":"share-4018","share_seq":4018,"template_height":800125},{"accepted_at_ms":1000005,"credit_policy":null,"job_id":"job-4019","job_issued_at_ms":1000003,"miner_id":"miner-4","network_difficulty":1000007,"ntime":1700004019,"order_key":"04:miner-4","p2mr_program_hex":"b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3","share_difficulty":1000,"share_id":"share-4019","share_seq":4019,"template_height":800125},{"accepted_at_ms":999981,"credit_policy":"stale-grace","job_id":"job-4020","job_issued_at_ms":1000004,"miner_id":"miner-5","network_difficulty":1000008,"ntime":1700004020,"order_key":"05:miner-5","p2mr_program_hex":"b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4","share_difficulty":3,"share_id":"share-4020","share_seq":4020,"template_height":800125},{"accepted_at_ms":1000003,"credit_policy":null,"job_id":"job-4021","job_issued_at_ms":999987,"miner_id":"miner-6","network_difficulty":1000009,"ntime":1700004021,"order_key":"06:miner-6","p2mr_program_hex":"b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5","share_difficulty":3,"share_id":"share-4021","share_seq":4021,"template_height":800125},{"accepted_at_ms":1000007,"credit_policy":null,"job_id":"job-4022","job_issued_at_ms":999971,"miner_id":"miner-7","network_difficulty":1000010,"ntime":1700004022,"order_key":"07:miner-7","p2mr_program_hex":"b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6","share_difficulty":101,"share_id":"share-4022","share_seq":4022,"template_height":800125},{"accepted_at_ms":1000004,"credit_policy":null,"job_id":"job-4023","job_issued_at_ms":999980,"miner_id":"miner-8","network_difficulty":1000011,"ntime":1700004023,"order_key":"08:miner-8","p2mr_program_hex":"b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7","share_difficulty":3,"share_id":"share-4023","share_seq":4023,"template_height":800125},{"accepted_at_ms":999976,"credit_policy":null,"job_id":"job-4024","job_issued_at_ms":1000005,"miner_id":"miner-9","network_difficulty":1000012,"ntime":1700004024,"order_key":"09:miner-9","p2mr_program_hex":"b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8","share_difficulty":1,"share_id":"share-4024","share_seq":4024,"template_height":800125},{"accepted_at_ms":999998,"credit_policy":null,"job_id":"job-4025","job_issued_at_ms":1000001,"miner_id":"miner-10","network_difficulty":1000013,"ntime":1700004025,"order_key":"10:miner-10","p2mr_program_hex":"b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9","share_difficulty":2,"share_id":"share-4025","share_seq":4025,"template_height":800125},{"accepted_at_ms":999985,"credit_policy":"stale-grace","job_id":"job-4026","job_issued_at_ms":1000002,"miner_id":"miner-0","network_difficulty":1000014,"ntime":1700004026,"order_key":"00:miner-0","p2mr_program_hex":"babababababababababababababababababababababababababababababababa","share_difficulty":1,"share_id":"share-4026","share_seq":4026,"template_height":800125},{"accepted_at_ms":1000005,"credit_policy":"stale-grace","job_id":"job-4027","job_issued_at_ms":1000008,"miner_id":"miner-1","network_difficulty":1000015,"ntime":1700004027,"order_key":"01:miner-1","p2mr_program_hex":"bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb","share_difficulty":101,"share_id":"share-4027","share_seq":4027,"template_height":800125},{"accepted_at_ms":1000010,"credit_policy":null,"job_id":"job-4028","job_issued_at_ms":1000004,"miner_id":"miner-2","network_difficulty":1000016,"ntime":1700004028,"order_key":"02:miner-2","p2mr_program_hex":"bcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbc","share_difficulty":1,"share_id":"share-4028","share_seq":4028,"template_height":800125},{"accepted_at_ms":999996,"credit_policy":null,"job_id":"job-4029","job_issued_at_ms":1000001,"miner_id":"miner-3","network_difficulty":1000000,"ntime":1700004029,"order_key":"03:miner-3","p2mr_program_hex":"bdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbd","share_difficulty":2,"share_id":"share-4029","share_seq":4029,"template_height":800125},{"accepted_at_ms":999995,"credit_policy":"stale-grace","job_id":"job-4030","job_issued_at_ms":1000005,"miner_id":"miner-4","network_difficulty":1000001,"ntime":1700004030,"order_key":"04:miner-4","p2mr_program_hex":"bebebebebebebebebebebebebebebebebebebebebebebebebebebebebebebebe","share_difficulty":3,"share_id":"share-4030","share_seq":4030,"template_height":800125},{"accepted_at_ms":1000006,"credit_policy":null,"job_id":"job-4031","job_issued_at_ms":999992,"miner_id":"miner-5","network_difficulty":1000002,"ntime":1700004031,"order_key":"05:miner-5","p2mr_program_hex":"bfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbf","share_difficulty":1000,"share_id":"share-4031","share_seq":4031,"template_height":800125},{"accepted_at_ms":1000008,"credit_policy":"stale-grace","job_id":"job-4032","job_issued_at_ms":999990,"miner_id":"miner-6","network_difficulty":1000003,"ntime":1700004032,"order_key":"06:miner-6","p2mr_program_hex":"c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0","share_difficulty":1000,"share_id":"share-4032","share_seq":4032,"template_height":800126},{"accepted_at_ms":999978,"credit_policy":null,"job_id":"job-4033","job_issued_at_ms":1000009,"miner_id":"miner-7","network_difficulty":1000004,"ntime":1700004033,"order_key":"07:miner-7","p2mr_program_hex":"c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1","share_difficulty":7,"share_id":"share-4033","share_seq":4033,"template_height":800126},{"accepted_at_ms":999995,"credit_policy":null,"job_id":"job-4034","job_issued_at_ms":1000001,"miner_id":"miner-8","network_difficulty":1000005,"ntime":1700004034,"order_key":"08:miner-8","p2mr_program_hex":"c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2","share_difficulty":101,"share_id":"share-4034","share_seq":4034,"template_height":800126},{"accepted_at_ms":1000003,"credit_policy":null,"job_id":"job-4035","job_issued_at_ms":1000003,"miner_id":"miner-9","network_difficulty":1000006,"ntime":1700004035,"order_key":"09:miner-9","p2mr_program_hex":"c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3","share_difficulty":101,"share_id":"share-4035","share_seq":4035,"template_height":800126},{"accepted_at_ms":999992,"credit_policy":"stale-grace","job_id":"job-4036","job_issued_at_ms":1000003,"miner_id":"miner-10","network_difficulty":1000007,"ntime":1700004036,"order_key":"10:miner-10","p2mr_program_hex":"c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4","share_difficulty":1,"share_id":"share-4036","share_seq":4036,"template_height":800126},{"accepted_at_ms":1000007,"credit_policy":null,"job_id":"job-4037","job_issued_at_ms":999996,"miner_id":"miner-0","network_difficulty":1000008,"ntime":1700004037,"order_key":"00:miner-0","p2mr_program_hex":"c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5","share_difficulty":101,"share_id":"share-4037","share_seq":4037,"template_height":800126},{"accepted_at_ms":999979,"credit_policy":"stale-grace","job_id":"job-4038","job_issued_at_ms":1000002,"miner_id":"miner-1","network_difficulty":1000009,"ntime":1700004038,"order_key":"01:miner-1","p2mr_program_hex":"c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6","share_difficulty":1000,"share_id":"share-4038","share_seq":4038,"template_height":800126},{"accepted_at_ms":1000008,"credit_policy":null,"job_id":"job-4039","job_issued_at_ms":1000009,"miner_id":"miner-2","network_difficulty":1000010,"ntime":1700004039,"order_key":"02:miner-2","p2mr_program_hex":"c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7","share_difficulty":3,"share_id":"share-4039","share_seq":4039,"template_height":800126},{"accepted_at_ms":999991,"credit_policy":null,"job_id":"job-4040","job_issued_at_ms":1000005,"miner_id":"miner-3","network_difficulty":1000011,"ntime":1700004040,"order_key":"03:miner-3","p2mr_program_hex":"c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8","share_difficulty":19,"share_id":"share-4040","share_seq":4040,"template_height":800126},{"accepted_at_ms":1000005,"credit_policy":null,"job_id":"job-4041","job_issued_at_ms":1000002,"miner_id":"miner-4","network_difficulty":1000012,"ntime":1700004041,"order_key":"04:miner-4","p2mr_program_hex":"c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9","share_difficulty":101,"share_id":"share-4041","share_seq":4041,"template_height":800126},{"accepted_at_ms":999988,"credit_policy":null,"job_id":"job-4042","job_issued_at_ms":1000002,"miner_id":"miner-5","network_difficulty":1000013,"ntime":1700004042,"order_key":"05:miner-5","p2mr_program_hex":"cacacacacacacacacacacacacacacacacacacacacacacacacacacacacacacaca","share_difficulty":3,"share_id":"share-4042","share_seq":4042,"template_height":800126},{"accepted_at_ms":1000004,"credit_policy":null,"job_id":"job-4043","job_issued_at_ms":999974,"miner_id":"miner-6","network_difficulty":1000014,"ntime":1700004043,"order_key":"06:miner-6","p2mr_program_hex":"cbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcb","share_difficulty":101,"share_id":"share-4043","share_seq":4043,"template_height":800126},{"accepted_at_ms":1000003,"credit_policy":null,"job_id":"job-4044","job_issued_at_ms":1000009,"miner_id":"miner-7","network_difficulty":1000015,"ntime":1700004044,"order_key":"07:miner-7","p2mr_program_hex":"cccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccc","share_difficulty":7,"share_id":"share-4044","share_seq":4044,"template_height":800126},{"accepted_at_ms":1000002,"credit_policy":null,"job_id":"job-4045","job_issued_at_ms":999977,"miner_id":"miner-8","network_difficulty":1000016,"ntime":1700004045,"order_key":"08:miner-8","p2mr_program_hex":"cdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcd","share_difficulty":19,"share_id":"share-4045","share_seq":4045,"template_height":800126},{"accepted_at_ms":999981,"credit_policy":null,"job_id":"job-4046","job_issued_at_ms":1000008,"miner_id":"miner-9","network_difficulty":1000000,"ntime":1700004046,"order_key":"09:miner-9","p2mr_program_hex":"cececececececececececececececececececececececececececececececece","share_difficulty":3,"share_id":"share-4046","share_seq":4046,"template_height":800126},{"accepted_at_ms":1000010,"credit_policy":null,"job_id":"job-4047","job_issued_at_ms":999984,"miner_id":"miner-10","network_difficulty":1000001,"ntime":1700004047,"order_key":"10:miner-10","p2mr_program_hex":"cfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcf","share_difficulty":101,"share_id":"share-4047","share_seq":4047,"template_height":800126},{"accepted_at_ms":1000002,"credit_policy":null,"job_id":"job-4048","job_issued_at_ms":1000003,"miner_id":"miner-0","network_difficulty":1000002,"ntime":1700004048,"order_key":"00:miner-0","p2mr_program_hex":"d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0","share_difficulty":19,"share_id":"share-4048","share_seq":4048,"template_height":800126},{"accepted_at_ms":1000007,"credit_policy":"stale-grace","job_id":"job-4049","job_issued_at_ms":999996,"miner_id":"miner-1","network_difficulty":1000003,"ntime":1700004049,"order_key":"01:miner-1","p2mr_program_hex":"d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1","share_difficulty":101,"share_id":"share-4049","share_seq":4049,"template_height":800126},{"accepted_at_ms":999970,"credit_policy":"stale-grace","job_id":"job-4050","job_issued_at_ms":1000006,"miner_id":"miner-2","network_difficulty":1000004,"ntime":1700004050,"order_key":"02:miner-2","p2mr_program_hex":"d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2","share_difficulty":19,"share_id":"share-4050","share_seq":4050,"template_height":800126},{"accepted_at_ms":999976,"credit_policy":null,"job_id":"job-4051","job_issued_at_ms":1000003,"miner_id":"miner-3","network_difficulty":1000005,"ntime":1700004051,"order_key":"03:miner-3","p2mr_program_hex":"d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3","share_difficulty":7,"share_id":"share-4051","share_seq":4051,"template_height":800126},{"accepted_at_ms":1000006,"credit_policy":null,"job_id":"job-4052","job_issued_at_ms":1000004,"miner_id":"miner-4","network_difficulty":1000006,"ntime":1700004052,"order_key":"04:miner-4","p2mr_program_hex":"d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4","share_difficulty":1,"share_id":"share-4052","share_seq":4052,"template_height":800126},{"accepted_at_ms":1000009,"credit_policy":null,"job_id":"job-4053","job_issued_at_ms":1000005,"miner_id":"miner-5","network_difficulty":1000007,"ntime":1700004053,"order_key":"05:miner-5","p2mr_program_hex":"d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5","share_difficulty":7,"share_id":"share-4053","share_seq":4053,"template_height":800126},{"accepted_at_ms":1000009,"credit_policy":null,"job_id":"job-4054","job_issued_at_ms":1000007,"miner_id":"miner-6","network_difficulty":1000008,"ntime":1700004054,"order_key":"06:miner-6","p2mr_program_hex":"d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6","share_difficulty":3,"share_id":"share-4054","share_seq":4054,"template_height":800126},{"accepted_at_ms":1000006,"credit_policy":null,"job_id":"job-4055","job_issued_at_ms":999999,"miner_id":"miner-7","network_difficulty":1000009,"ntime":1700004055,"order_key":"07:miner-7","p2mr_program_hex":"d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7","share_difficulty":2,"share_id":"share-4055","share_seq":4055,"template_height":800126},{"accepted_at_ms":1000010,"credit_policy":null,"job_id":"job-4056","job_issued_at_ms":999982,"miner_id":"miner-8","network_difficulty":1000010,"ntime":1700004056,"order_key":"08:miner-8","p2mr_program_hex":"d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8","share_difficulty":1,"share_id":"share-4056","share_seq":4056,"template_height":800126},{"accepted_at_ms":1000001,"credit_policy":null,"job_id":"job-4057","job_issued_at_ms":999998,"miner_id":"miner-9","network_difficulty":1000011,"ntime":1700004057,"order_key":"09:miner-9","p2mr_program_hex":"d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9","share_difficulty":7,"share_id":"share-4057","share_seq":4057,"template_height":800126},{"accepted_at_ms":1000001,"credit_policy":"stale-grace","job_id":"job-4058","job_issued_at_ms":1000004,"miner_id":"miner-10","network_difficulty":1000012,"ntime":1700004058,"order_key":"10:miner-10","p2mr_program_hex":"dadadadadadadadadadadadadadadadadadadadadadadadadadadadadadadada","share_difficulty":7,"share_id":"share-4058","share_seq":4058,"template_height":800126},{"accepted_at_ms":999976,"credit_policy":null,"job_id":"job-4059","job_issued_at_ms":1000009,"miner_id":"miner-0","network_difficulty":1000013,"ntime":1700004059,"order_key":"00:miner-0","p2mr_program_hex":"dbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdb","share_difficulty":2,"share_id":"share-4059","share_seq":4059,"template_height":800126},{"accepted_at_ms":999978,"credit_policy":null,"job_id":"job-4060","job_issued_at_ms":1000004,"miner_id":"miner-1","network_difficulty":1000014,"ntime":1700004060,"order_key":"01:miner-1","p2mr_program_hex":"dcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdc","share_difficulty":3,"share_id":"share-4060","share_seq":4060,"template_height":800126},{"accepted_at_ms":1000003,"credit_policy":null,"job_id":"job-4061","job_issued_at_ms":1000001,"miner_id":"miner-2","network_difficulty":1000015,"ntime":1700004061,"order_key":"02:miner-2","p2mr_program_hex":"dddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddd","share_difficulty":7,"share_id":"share-4061","share_seq":4061,"template_height":800126},{"accepted_at_ms":999991,"credit_policy":"stale-grace","job_id":"job-4062","job_issued_at_ms":1000008,"miner_id":"miner-3","network_difficulty":1000016,"ntime":1700004062,"order_key":"03:miner-3","p2mr_program_hex":"dededededededededededededededededededededededededededededededede","share_difficulty":2,"share_id":"share-4062","share_seq":4062,"template_height":800126},{"accepted_at_ms":999992,"credit_policy":null,"job_id":"job-4063","job_issued_at_ms":1000008,"miner_id":"miner-4","network_difficulty":1000000,"ntime":1700004063,"order_key":"04:miner-4","p2mr_program_hex":"dfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdf","share_difficulty":1000,"share_id":"share-4063","share_seq":4063,"template_height":800126},{"accepted_at_ms":1000002,"credit_policy":null,"job_id":"job-4064","job_issued_at_ms":999997,"miner_id":"miner-5","network_difficulty":1000001,"ntime":1700004064,"order_key":"05:miner-5","p2mr_program_hex":"e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0","share_difficulty":1000,"share_id":"share-4064","share_seq":4064,"template_height":800127},{"accepted_at_ms":999991,"credit_policy":null,"job_id":"job-4065","job_issued_at_ms":1000001,"miner_id":"miner-6","network_difficulty":1000002,"ntime":1700004065,"order_key":"06:miner-6","p2mr_program_hex":"e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1","share_difficulty":2,"share_id":"share-4065","share_seq":4065,"template_height":800127},{"accepted_at_ms":999990,"credit_policy":null,"job_id":"job-4066","job_issued_at_ms":1000001,"miner_id":"miner-7","network_difficulty":1000003,"ntime":1700004066,"order_key":"07:miner-7","p2mr_program_hex":"e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2","share_difficulty":7,"share_id":"share-4066","share_seq":4066,"template_height":800127},{"accepted_at_ms":1000003,"credit_policy":null,"job_id":"job-4067","job_issued_at_ms":999975,"miner_id":"miner-8","network_difficulty":1000004,"ntime":1700004067,"order_key":"08:miner-8","p2mr_program_hex":"e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3","share_difficulty":101,"share_id":"share-4067","share_seq":4067,"template_height":800127},{"accepted_at_ms":1000005,"credit_policy":null,"job_id":"job-4068","job_issued_at_ms":999992,"miner_id":"miner-9","network_difficulty":1000005,"ntime":1700004068,"order_key":"09:miner-9","p2mr_program_hex":"e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4","share_difficulty":19,"share_id":"share-4068","share_seq":4068,"template_height":800127},{"accepted_at_ms":999980,"credit_policy":"stale-grace","job_id":"job-4069","job_issued_at_ms":1000005,"miner_id":"miner-10","network_difficulty":1000006,"ntime":1700004069,"order_key":"10:miner-10","p2mr_program_hex":"e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5","share_difficulty":101,"share_id":"share-4069","share_seq":4069,"template_height":800127},{"accepted_at_ms":1000007,"credit_policy":null,"job_id":"job-4070","job_issued_at_ms":999975,"miner_id":"miner-0","network_difficulty":1000007,"ntime":1700004070,"order_key":"00:miner-0","p2mr_program_hex":"e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6","share_difficulty":101,"share_id":"share-4070","share_seq":4070,"template_height":800127},{"accepted_at_ms":1000003,"credit_policy":null,"job_id":"job-4071","job_issued_at_ms":999988,"miner_id":"miner-1","network_difficulty":1000008,"ntime":1700004071,"order_key":"01:miner-1","p2mr_program_hex":"e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7","share_difficulty":2,"share_id":"share-4071","share_seq":4071,"template_height":800127},{"accepted_at_ms":1000001,"credit_policy":null,"job_id":"job-4072","job_issued_at_ms":1000010,"miner_id":"miner-2","network_difficulty":1000009,"ntime":1700004072,"order_key":"02:miner-2","p2mr_program_hex":"e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8","share_difficulty":19,"share_id":"share-4072","share_seq":4072,"template_height":800127},{"accepted_at_ms":1000003,"credit_policy":"stale-grace","job_id":"job-4073","job_issued_at_ms":999973,"miner_id":"miner-3","network_difficulty":1000010,"ntime":1700004073,"order_key":"03:miner-3","p2mr_program_hex":"e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9","share_difficulty":19,"share_id":"share-4073","share_seq":4073,"template_height":800127},{"accepted_at_ms":1000009,"credit_policy":null,"job_id":"job-4074","job_issued_at_ms":999986,"miner_id":"miner-4","network_difficulty":1000011,"ntime":1700004074,"order_key":"04:miner-4","p2mr_program_hex":"eaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaea","share_difficulty":19,"share_id":"share-4074","share_seq":4074,"template_height":800127},{"accepted_at_ms":1000002,"credit_policy":null,"job_id":"job-4075","job_issued_at_ms":999975,"miner_id":"miner-5","network_difficulty":1000012,"ntime":1700004075,"order_key":"05:miner-5","p2mr_program_hex":"ebebebebebebebebebebebebebebebebebebebebebebebebebebebebebebebeb","share_difficulty":101,"share_id":"share-4075","share_seq":4075,"template_height":800127},{"accepted_at_ms":1000001,"credit_policy":null,"job_id":"job-4076","job_issued_at_ms":999989,"miner_id":"miner-6","network_difficulty":1000013,"ntime":1700004076,"order_key":"06:miner-6","p2mr_program_hex":"ecececececececececececececececececececececececececececececececec","share_difficulty":101,"share_id":"share-4076","share_seq":4076,"template_height":800127},{"accepted_at_ms":1000006,"credit_policy":null,"job_id":"job-4077","job_issued_at_ms":999981,"miner_id":"miner-7","network_difficulty":1000014,"ntime":1700004077,"order_key":"07:miner-7","p2mr_program_hex":"edededededededededededededededededededededededededededededededed","share_difficulty":7,"share_id":"share-4077","share_seq":4077,"template_height":800127},{"accepted_at_ms":999973,"credit_policy":"stale-grace","job_id":"job-4078","job_issued_at_ms":1000001,"miner_id":"miner-8","network_difficulty":1000015,"ntime":1700004078,"order_key":"08:miner-8","p2mr_program_hex":"eeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeee","share_difficulty":19,"share_id":"share-4078","share_seq":4078,"template_height":800127},{"accepted_at_ms":1000002,"credit_policy":null,"job_id":"job-4079","job_issued_at_ms":999983,"miner_id":"miner-9","network_difficulty":1000016,"ntime":1700004079,"order_key":"09:miner-9","p2mr_program_hex":"efefefefefefefefefefefefefefefefefefefefefefefefefefefefefefefef","share_difficulty":3,"share_id":"share-4079","share_seq":4079,"template_height":800127},{"accepted_at_ms":999995,"credit_policy":null,"job_id":"job-4080","job_issued_at_ms":1000005,"miner_id":"miner-10","network_difficulty":1000000,"ntime":1700004080,"order_key":"10:miner-10","p2mr_program_hex":"f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0","share_difficulty":101,"share_id":"share-4080","share_seq":4080,"template_height":800127},{"accepted_at_ms":1000010,"credit_policy":null,"job_id":"job-4081","job_issued_at_ms":999989,"miner_id":"miner-0","network_difficulty":1000001,"ntime":1700004081,"order_key":"00:miner-0","p2mr_program_hex":"f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1","share_difficulty":7,"share_id":"share-4081","share_seq":4081,"template_height":800127},{"accepted_at_ms":999972,"credit_policy":null,"job_id":"job-4082","job_issued_at_ms":1000004,"miner_id":"miner-1","network_difficulty":1000002,"ntime":1700004082,"order_key":"01:miner-1","p2mr_program_hex":"f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2","share_difficulty":101,"share_id":"share-4082","share_seq":4082,"template_height":800127},{"accepted_at_ms":1000007,"credit_policy":null,"job_id":"job-4083","job_issued_at_ms":999981,"miner_id":"miner-2","network_difficulty":1000003,"ntime":1700004083,"order_key":"02:miner-2","p2mr_program_hex":"f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3","share_difficulty":1000,"share_id":"share-4083","share_seq":4083,"template_height":800127},{"accepted_at_ms":1000008,"credit_policy":"stale-grace","job_id":"job-4084","job_issued_at_ms":999982,"miner_id":"miner-3","network_difficulty":1000004,"ntime":1700004084,"order_key":"03:miner-3","p2mr_program_hex":"f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4","share_difficulty":7,"share_id":"share-4084","share_seq":4084,"template_height":800127},{"accepted_at_ms":1000001,"credit_policy":null,"job_id":"job-4085","job_issued_at_ms":1000010,"miner_id":"miner-4","network_difficulty":1000005,"ntime":1700004085,"order_key":"04:miner-4","p2mr_program_hex":"f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5","share_difficulty":19,"share_id":"share-4085","share_seq":4085,"template_height":800127},{"accepted_at_ms":1000005,"credit_policy":null,"job_id":"job-4086","job_issued_at_ms":1000008,"miner_id":"miner-5","network_difficulty":1000006,"ntime":1700004086,"order_key":"05:miner-5","p2mr_program_hex":"f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6","share_difficulty":7,"share_id":"share-4086","share_seq":4086,"template_height":800127},{"accepted_at_ms":1000002,"credit_policy":null,"job_id":"job-4087","job_issued_at_ms":999974,"miner_id":"miner-6","network_difficulty":1000007,"ntime":1700004087,"order_key":"06:miner-6","p2mr_program_hex":"f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7","share_difficulty":7,"share_id":"share-4087","share_seq":4087,"template_height":800127},{"accepted_at_ms":1000001,"credit_policy":null,"job_id":"job-4088","job_issued_at_ms":1000005,"miner_id":"miner-7","network_difficulty":1000008,"ntime":1700004088,"order_key":"07:miner-7","p2mr_program_hex":"f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8","share_difficulty":1,"share_id":"share-4088","share_seq":4088,"template_height":800127},{"accepted_at_ms":999980,"credit_policy":null,"job_id":"job-4089","job_issued_at_ms":1000008,"miner_id":"miner-8","network_difficulty":1000009,"ntime":1700004089,"order_key":"08:miner-8","p2mr_program_hex":"f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9","share_difficulty":7,"share_id":"share-4089","share_seq":4089,"template_height":800127},{"accepted_at_ms":1000008,"credit_policy":null,"job_id":"job-4090","job_issued_at_ms":999990,"miner_id":"miner-9","network_difficulty":1000010,"ntime":1700004090,"order_key":"09:miner-9","p2mr_program_hex":"fafafafafafafafafafafafafafafafafafafafafafafafafafafafafafafafa","share_difficulty":101,"share_id":"share-4090","share_seq":4090,"template_height":800127},{"accepted_at_ms":1000001,"credit_policy":null,"job_id":"job-4091","job_issued_at_ms":999991,"miner_id":"miner-10","network_difficulty":1000011,"ntime":1700004091,"order_key":"10:miner-10","p2mr_program_hex":"fbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfb","share_difficulty":3,"share_id":"share-4091","share_seq":4091,"template_height":800127},{"accepted_at_ms":1000004,"credit_policy":null,"job_id":"job-4092","job_issued_at_ms":1000003,"miner_id":"miner-0","network_difficulty":1000012,"ntime":1700004092,"order_key":"00:miner-0","p2mr_program_hex":"fcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfc","share_difficulty":7,"share_id":"share-4092","share_seq":4092,"template_height":800127},{"accepted_at_ms":1000002,"credit_policy":null,"job_id":"job-4093","job_issued_at_ms":999994,"miner_id":"miner-1","network_difficulty":1000013,"ntime":1700004093,"order_key":"01:miner-1","p2mr_program_hex":"fdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfd","share_difficulty":1000,"share_id":"share-4093","share_seq":4093,"template_height":800127},{"accepted_at_ms":1000006,"credit_policy":null,"job_id":"job-4094","job_issued_at_ms":1000003,"miner_id":"miner-2","network_difficulty":1000014,"ntime":1700004094,"order_key":"02:miner-2","p2mr_program_hex":"fefefefefefefefefefefefefefefefefefefefefefefefefefefefefefefefe","share_difficulty":7,"share_id":"share-4094","share_seq":4094,"template_height":800127},{"accepted_at_ms":999977,"credit_policy":null,"job_id":"job-4095","job_issued_at_ms":1000004,"miner_id":"miner-3","network_difficulty":1000015,"ntime":1700004095,"order_key":"03:miner-3","p2mr_program_hex":"ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff","share_difficulty":2,"share_id":"share-4095","share_seq":4095,"template_height":800127},{"accepted_at_ms":999995,"credit_policy":null,"job_id":"job-4096","job_issued_at_ms":1000005,"miner_id":"miner-4","network_difficulty":1000016,"ntime":1700004096,"order_key":"04:miner-4","p2mr_program_hex":"0000000000000000000000000000000000000000000000000000000000000000","share_difficulty":1,"share_id":"share-4096","share_seq":4096,"template_height":800128},{"accepted_at_ms":999997,"credit_policy":null,"job_id":"job-4097","job_issued_at_ms":1000003,"miner_id":"miner-5","network_difficulty":1000000,"ntime":1700004097,"order_key":"05:miner-5","p2mr_program_hex":"0101010101010101010101010101010101010101010101010101010101010101","share_difficulty":19,"share_id":"share-4097","share_seq":4097,"template_height":800128},{"accepted_at_ms":1000002,"credit_policy":null,"job_id":"job-4098","job_issued_at_ms":999985,"miner_id":"miner-6","network_difficulty":1000001,"ntime":1700004098,"order_key":"06:miner-6","p2mr_program_hex":"0202020202020202020202020202020202020202020202020202020202020202","share_difficulty":1000,"share_id":"share-4098","share_seq":4098,"template_height":800128},{"accepted_at_ms":999998,"credit_policy":"stale-grace","job_id":"job-4099","job_issued_at_ms":1000008,"miner_id":"miner-7","network_difficulty":1000002,"ntime":1700004099,"order_key":"07:miner-7","p2mr_program_hex":"0303030303030303030303030303030303030303030303030303030303030303","share_difficulty":1000,"share_id":"share-4099","share_seq":4099,"template_height":800128},{"accepted_at_ms":1000007,"credit_policy":null,"job_id":"job-4100","job_issued_at_ms":999997,"miner_id":"miner-8","network_difficulty":1000003,"ntime":1700004100,"order_key":"08:miner-8","p2mr_program_hex":"0404040404040404040404040404040404040404040404040404040404040404","share_difficulty":2,"share_id":"share-4100","share_seq":4100,"template_height":800128},{"accepted_at_ms":1000006,"credit_policy":null,"job_id":"job-4101","job_issued_at_ms":1000006,"miner_id":"miner-9","network_difficulty":1000004,"ntime":1700004101,"order_key":"09:miner-9","p2mr_program_hex":"0505050505050505050505050505050505050505050505050505050505050505","share_difficulty":19,"share_id":"share-4101","share_seq":4101,"template_height":800128},{"accepted_at_ms":1000004,"credit_policy":"stale-grace","job_id":"job-4102","job_issued_at_ms":999987,"miner_id":"miner-10","network_difficulty":1000005,"ntime":1700004102,"order_key":"10:miner-10","p2mr_program_hex":"0606060606060606060606060606060606060606060606060606060606060606","share_difficulty":2,"share_id":"share-4102","share_seq":4102,"template_height":800128},{"accepted_at_ms":999973,"credit_policy":"stale-grace","job_id":"job-4103","job_issued_at_ms":1000004,"miner_id":"miner-0","network_difficulty":1000006,"ntime":1700004103,"order_key":"00:miner-0","p2mr_program_hex":"0707070707070707070707070707070707070707070707070707070707070707","share_difficulty":1,"share_id":"share-4103","share_seq":4103,"template_height":800128},{"accepted_at_ms":1000002,"credit_policy":null,"job_id":"job-4104","job_issued_at_ms":999974,"miner_id":"miner-1","network_difficulty":1000007,"ntime":1700004104,"order_key":"01:miner-1","p2mr_program_hex":"0808080808080808080808080808080808080808080808080808080808080808","share_difficulty":19,"share_id":"share-4104","share_seq":4104,"template_height":800128},{"accepted_at_ms":1000005,"credit_policy":"stale-grace","job_id":"job-4105","job_issued_at_ms":1000007,"miner_id":"miner-2","network_difficulty":1000008,"ntime":1700004105,"order_key":"02:miner-2","p2mr_program_hex":"0909090909090909090909090909090909090909090909090909090909090909","share_difficulty":19,"share_id":"share-4105","share_seq":4105,"template_height":800128},{"accepted_at_ms":1000007,"credit_policy":null,"job_id":"job-4106","job_issued_at_ms":999987,"miner_id":"miner-3","network_difficulty":1000009,"ntime":1700004106,"order_key":"03:miner-3","p2mr_program_hex":"0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a","share_difficulty":7,"share_id":"share-4106","share_seq":4106,"template_height":800128},{"accepted_at_ms":1000010,"credit_policy":null,"job_id":"job-4107","job_issued_at_ms":999996,"miner_id":"miner-4","network_difficulty":1000010,"ntime":1700004107,"order_key":"04:miner-4","p2mr_program_hex":"0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b","share_difficulty":1,"share_id":"share-4107","share_seq":4107,"template_height":800128},{"accepted_at_ms":999970,"credit_policy":"stale-grace","job_id":"job-4108","job_issued_at_ms":1000007,"miner_id":"miner-5","network_difficulty":1000011,"ntime":1700004108,"order_key":"05:miner-5","p2mr_program_hex":"0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c","share_difficulty":1,"share_id":"share-4108","share_seq":4108,"template_height":800128},{"accepted_at_ms":1000009,"credit_policy":null,"job_id":"job-4109","job_issued_at_ms":999972,"miner_id":"miner-6","network_difficulty":1000012,"ntime":1700004109,"order_key":"06:miner-6","p2mr_program_hex":"0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d","share_difficulty":3,"share_id":"share-4109","share_seq":4109,"template_height":800128},{"accepted_at_ms":999997,"credit_policy":null,"job_id":"job-4110","job_issued_at_ms":1000007,"miner_id":"miner-7","network_difficulty":1000013,"ntime":1700004110,"order_key":"07:miner-7","p2mr_program_hex":"0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e","share_difficulty":19,"share_id":"share-4110","share_seq":4110,"template_height":800128},{"accepted_at_ms":999982,"credit_policy":null,"job_id":"job-4111","job_issued_at_ms":1000001,"miner_id":"miner-8","network_difficulty":1000014,"ntime":1700004111,"order_key":"08:miner-8","p2mr_program_hex":"0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f","share_difficulty":1000,"share_id":"share-4111","share_seq":4111,"template_height":800128},{"accepted_at_ms":1000005,"credit_policy":null,"job_id":"job-4112","job_issued_at_ms":999981,"miner_id":"miner-9","network_difficulty":1000015,"ntime":1700004112,"order_key":"09:miner-9","p2mr_program_hex":"1010101010101010101010101010101010101010101010101010101010101010","share_difficulty":2,"share_id":"share-4112","share_seq":4112,"template_height":800128},{"accepted_at_ms":999986,"credit_policy":null,"job_id":"job-4113","job_issued_at_ms":1000003,"miner_id":"miner-10","network_difficulty":1000016,"ntime":1700004113,"order_key":"10:miner-10","p2mr_program_hex":"1111111111111111111111111111111111111111111111111111111111111111","share_difficulty":19,"share_id":"share-4113","share_seq":4113,"template_height":800128},{"accepted_at_ms":999970,"credit_policy":null,"job_id":"job-4114","job_issued_at_ms":1000008,"miner_id":"miner-0","network_difficulty":1000000,"ntime":1700004114,"order_key":"00:miner-0","p2mr_program_hex":"1212121212121212121212121212121212121212121212121212121212121212","share_difficulty":2,"share_id":"share-4114","share_seq":4114,"template_height":800128},{"accepted_at_ms":1000005,"credit_policy":"stale-grace","job_id":"job-4115","job_issued_at_ms":1000004,"miner_id":"miner-1","network_difficulty":1000001,"ntime":1700004115,"order_key":"01:miner-1","p2mr_program_hex":"1313131313131313131313131313131313131313131313131313131313131313","share_difficulty":2,"share_id":"share-4115","share_seq":4115,"template_height":800128},{"accepted_at_ms":1000006,"credit_policy":"stale-grace","job_id":"job-4116","job_issued_at_ms":999997,"miner_id":"miner-2","network_difficulty":1000002,"ntime":1700004116,"order_key":"02:miner-2","p2mr_program_hex":"1414141414141414141414141414141414141414141414141414141414141414","share_difficulty":1000,"share_id":"share-4116","share_seq":4116,"template_height":800128},{"accepted_at_ms":1000001,"credit_policy":null,"job_id":"job-4117","job_issued_at_ms":1000006,"miner_id":"miner-3","network_difficulty":1000003,"ntime":1700004117,"order_key":"03:miner-3","p2mr_program_hex":"1515151515151515151515151515151515151515151515151515151515151515","share_difficulty":3,"share_id":"share-4117","share_seq":4117,"template_height":800128},{"accepted_at_ms":999984,"credit_policy":null,"job_id":"job-4118","job_issued_at_ms":1000004,"miner_id":"miner-4","network_difficulty":1000004,"ntime":1700004118,"order_key":"04:miner-4","p2mr_program_hex":"1616161616161616161616161616161616161616161616161616161616161616","share_difficulty":2,"share_id":"share-4118","share_seq":4118,"template_height":800128},{"accepted_at_ms":999994,"credit_policy":null,"job_id":"job-4119","job_issued_at_ms":1000003,"miner_id":"miner-5","network_difficulty":1000005,"ntime":1700004119,"order_key":"05:miner-5","p2mr_program_hex":"1717171717171717171717171717171717171717171717171717171717171717","share_difficulty":19,"share_id":"share-4119","share_seq":4119,"template_height":800128},{"accepted_at_ms":1000007,"credit_policy":null,"job_id":"job-4120","job_issued_at_ms":1000005,"miner_id":"miner-6","network_difficulty":1000006,"ntime":1700004120,"order_key":"06:miner-6","p2mr_program_hex":"1818181818181818181818181818181818181818181818181818181818181818","share_difficulty":7,"share_id":"share-4120","share_seq":4120,"template_height":800128},{"accepted_at_ms":1000002,"credit_policy":null,"job_id":"job-4121","job_issued_at_ms":999979,"miner_id":"miner-7","network_difficulty":1000007,"ntime":1700004121,"order_key":"07:miner-7","p2mr_program_hex":"1919191919191919191919191919191919191919191919191919191919191919","share_difficulty":1000,"share_id":"share-4121","share_seq":4121,"template_height":800128},{"accepted_at_ms":999982,"credit_policy":null,"job_id":"job-4122","job_issued_at_ms":1000006,"miner_id":"miner-8","network_difficulty":1000008,"ntime":1700004122,"order_key":"08:miner-8","p2mr_program_hex":"1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a","share_difficulty":2,"share_id":"share-4122","share_seq":4122,"template_height":800128},{"accepted_at_ms":1000002,"credit_policy":null,"job_id":"job-4123","job_issued_at_ms":999983,"miner_id":"miner-9","network_difficulty":1000009,"ntime":1700004123,"order_key":"09:miner-9","p2mr_program_hex":"1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b","share_difficulty":1,"share_id":"share-4123","share_seq":4123,"template_height":800128},{"accepted_at_ms":999986,"credit_policy":null,"job_id":"job-4124","job_issued_at_ms":1000010,"miner_id":"miner-10","network_difficulty":1000010,"ntime":1700004124,"order_key":"10:miner-10","p2mr_program_hex":"1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c","share_difficulty":1000,"share_id":"share-4124","share_seq":4124,"template_height":800128},{"accepted_at_ms":1000009,"credit_policy":null,"job_id":"job-4125","job_issued_at_ms":1000010,"miner_id":"miner-0","network_difficulty":1000011,"ntime":1700004125,"order_key":"00:miner-0","p2mr_program_hex":"1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d","share_difficulty":7,"share_id":"share-4125","share_seq":4125,"template_height":800128},{"accepted_at_ms":1000008,"credit_policy":null,"job_id":"job-4126","job_issued_at_ms":999985,"miner_id":"miner-1","network_difficulty":1000012,"ntime":1700004126,"order_key":"01:miner-1","p2mr_program_hex":"1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e","share_difficulty":7,"share_id":"share-4126","share_seq":4126,"template_height":800128},{"accepted_at_ms":999970,"credit_policy":null,"job_id":"job-4127","job_issued_at_ms":1000010,"miner_id":"miner-2","network_difficulty":1000013,"ntime":1700004127,"order_key":"02:miner-2","p2mr_program_hex":"1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f","share_difficulty":1,"share_id":"share-4127","share_seq":4127,"template_height":800128},{"accepted_at_ms":999999,"credit_policy":null,"job_id":"job-4128","job_issued_at_ms":1000009,"miner_id":"miner-3","network_difficulty":1000014,"ntime":1700004128,"order_key":"03:miner-3","p2mr_program_hex":"2020202020202020202020202020202020202020202020202020202020202020","share_difficulty":101,"share_id":"share-4128","share_seq":4128,"template_height":800129},{"accepted_at_ms":1000004,"credit_policy":null,"job_id":"job-4129","job_issued_at_ms":1000009,"miner_id":"miner-4","network_difficulty":1000015,"ntime":1700004129,"order_key":"04:miner-4","p2mr_program_hex":"2121212121212121212121212121212121212121212121212121212121212121","share_difficulty":101,"share_id":"share-4129","share_seq":4129,"template_height":800129},{"accepted_at_ms":999992,"credit_policy":null,"job_id":"job-4130","job_issued_at_ms":1000007,"miner_id":"miner-5","network_difficulty":1000016,"ntime":1700004130,"order_key":"05:miner-5","p2mr_program_hex":"2222222222222222222222222222222222222222222222222222222222222222","share_difficulty":7,"share_id":"share-4130","share_seq":4130,"template_height":800129},{"accepted_at_ms":1000006,"credit_policy":null,"job_id":"job-4131","job_issued_at_ms":999986,"miner_id":"miner-6","network_difficulty":1000000,"ntime":1700004131,"order_key":"06:miner-6","p2mr_program_hex":"2323232323232323232323232323232323232323232323232323232323232323","share_difficulty":2,"share_id":"share-4131","share_seq":4131,"template_height":800129},{"accepted_at_ms":1000001,"credit_policy":null,"job_id":"job-4132","job_issued_at_ms":999984,"miner_id":"miner-7","network_difficulty":1000001,"ntime":1700004132,"order_key":"07:miner-7","p2mr_program_hex":"2424242424242424242424242424242424242424242424242424242424242424","share_difficulty":2,"share_id":"share-4132","share_seq":4132,"template_height":800129},{"accepted_at_ms":1000001,"credit_policy":null,"job_id":"job-4133","job_issued_at_ms":999983,"miner_id":"miner-8","network_difficulty":1000002,"ntime":1700004133,"order_key":"08:miner-8","p2mr_program_hex":"2525252525252525252525252525252525252525252525252525252525252525","share_difficulty":3,"share_id":"share-4133","share_seq":4133,"template_height":800129},{"accepted_at_ms":1000002,"credit_policy":null,"job_id":"job-4134","job_issued_at_ms":1000008,"miner_id":"miner-9","network_difficulty":1000003,"ntime":1700004134,"order_key":"09:miner-9","p2mr_program_hex":"2626262626262626262626262626262626262626262626262626262626262626","share_difficulty":7,"share_id":"share-4134","share_seq":4134,"template_height":800129},{"accepted_at_ms":999999,"credit_policy":null,"job_id":"job-4135","job_issued_at_ms":1000010,"miner_id":"miner-10","network_difficulty":1000004,"ntime":1700004135,"order_key":"10:miner-10","p2mr_program_hex":"2727272727272727272727272727272727272727272727272727272727272727","share_difficulty":7,"share_id":"share-4135","share_seq":4135,"template_height":800129},{"accepted_at_ms":999995,"credit_policy":null,"job_id":"job-4136","job_issued_at_ms":1000003,"miner_id":"miner-0","network_difficulty":1000005,"ntime":1700004136,"order_key":"00:miner-0","p2mr_program_hex":"2828282828282828282828282828282828282828282828282828282828282828","share_difficulty":7,"share_id":"share-4136","share_seq":4136,"template_height":800129},{"accepted_at_ms":1000001,"credit_policy":null,"job_id":"job-4137","job_issued_at_ms":1000008,"miner_id":"miner-1","network_difficulty":1000006,"ntime":1700004137,"order_key":"01:miner-1","p2mr_program_hex":"2929292929292929292929292929292929292929292929292929292929292929","share_difficulty":7,"share_id":"share-4137","share_seq":4137,"template_height":800129},{"accepted_at_ms":1000005,"credit_policy":"stale-grace","job_id":"job-4138","job_issued_at_ms":1000008,"miner_id":"miner-2","network_difficulty":1000007,"ntime":1700004138,"order_key":"02:miner-2","p2mr_program_hex":"2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a","share_difficulty":3,"share_id":"share-4138","share_seq":4138,"template_height":800129},{"accepted_at_ms":1000007,"credit_policy":null,"job_id":"job-4139","job_issued_at_ms":999977,"miner_id":"miner-3","network_difficulty":1000008,"ntime":1700004139,"order_key":"03:miner-3","p2mr_program_hex":"2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b","share_difficulty":101,"share_id":"share-4139","share_seq":4139,"template_height":800129},{"accepted_at_ms":999991,"credit_policy":"stale-grace","job_id":"job-4140","job_issued_at_ms":1000010,"miner_id":"miner-4","network_difficulty":1000009,"ntime":1700004140,"order_key":"04:miner-4","p2mr_program_hex":"2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c","share_difficulty":3,"share_id":"share-4140","share_seq":4140,"template_height":800129},{"accepted_at_ms":999986,"credit_policy":null,"job_id":"job-4141","job_issued_at_ms":1000001,"miner_id":"miner-5","network_difficulty":1000010,"ntime":1700004141,"order_key":"05:miner-5","p2mr_program_hex":"2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d","share_difficulty":7,"share_id":"share-4141","share_seq":4141,"template_height":800129},{"accepted_at_ms":1000004,"credit_policy":null,"job_id":"job-4142","job_issued_at_ms":999974,"miner_id":"miner-6","network_difficulty":1000011,"ntime":1700004142,"order_key":"06:miner-6","p2mr_program_hex":"2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e","share_difficulty":1000,"share_id":"share-4142","share_seq":4142,"template_height":800129},{"accepted_at_ms":1000008,"credit_policy":null,"job_id":"job-4143","job_issued_at_ms":999995,"miner_id":"miner-7","network_difficulty":1000012,"ntime":1700004143,"order_key":"07:miner-7","p2mr_program_hex":"2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f","share_difficulty":7,"share_id":"share-4143","share_seq":4143,"template_height":800129},{"accepted_at_ms":1000009,"credit_policy":null,"job_id":"job-4144","job_issued_at_ms":1000002,"miner_id":"miner-8","network_difficulty":1000013,"ntime":1700004144,"order_key":"08:miner-8","p2mr_program_hex":"3030303030303030303030303030303030303030303030303030303030303030","share_difficulty":1000,"share_id":"share-4144","share_seq":4144,"template_height":800129},{"accepted_at_ms":1000008,"credit_policy":null,"job_id":"job-4145","job_issued_at_ms":1000003,"miner_id":"miner-9","network_difficulty":1000014,"ntime":1700004145,"order_key":"09:miner-9","p2mr_program_hex":"3131313131313131313131313131313131313131313131313131313131313131","share_difficulty":1000,"share_id":"share-4145","share_seq":4145,"template_height":800129},{"accepted_at_ms":1000007,"credit_policy":null,"job_id":"job-4146","job_issued_at_ms":999997,"miner_id":"miner-10","network_difficulty":1000015,"ntime":1700004146,"order_key":"10:miner-10","p2mr_program_hex":"3232323232323232323232323232323232323232323232323232323232323232","share_difficulty":1,"share_id":"share-4146","share_seq":4146,"template_height":800129},{"accepted_at_ms":999991,"credit_policy":null,"job_id":"job-4147","job_issued_at_ms":1000010,"miner_id":"miner-0","network_difficulty":1000016,"ntime":1700004147,"order_key":"00:miner-0","p2mr_program_hex":"3333333333333333333333333333333333333333333333333333333333333333","share_difficulty":101,"share_id":"share-4147","share_seq":4147,"template_height":800129},{"accepted_at_ms":1000004,"credit_policy":null,"job_id":"job-4148","job_issued_at_ms":1000008,"miner_id":"miner-1","network_difficulty":1000000,"ntime":1700004148,"order_key":"01:miner-1","p2mr_program_hex":"3434343434343434343434343434343434343434343434343434343434343434","share_difficulty":1000,"share_id":"share-4148","share_seq":4148,"template_height":800129},{"accepted_at_ms":1000001,"credit_policy":null,"job_id":"job-4149","job_issued_at_ms":999991,"miner_id":"miner-2","network_difficulty":1000001,"ntime":1700004149,"order_key":"02:miner-2","p2mr_program_hex":"3535353535353535353535353535353535353535353535353535353535353535","share_difficulty":101,"share_id":"share-4149","share_seq":4149,"template_height":800129},{"accepted_at_ms":999984,"credit_policy":null,"job_id":"job-4150","job_issued_at_ms":1000001,"miner_id":"miner-3","network_difficulty":1000002,"ntime":1700004150,"order_key":"03:miner-3","p2mr_program_hex":"3636363636363636363636363636363636363636363636363636363636363636","share_difficulty":19,"share_id":"share-4150","share_seq":4150,"template_height":800129},{"accepted_at_ms":999982,"credit_policy":"stale-grace","job_id":"job-4151","job_issued_at_ms":1000007,"miner_id":"miner-4","network_difficulty":1000003,"ntime":1700004151,"order_key":"04:miner-4","p2mr_program_hex":"3737373737373737373737373737373737373737373737373737373737373737","share_difficulty":1,"share_id":"share-4151","share_seq":4151,"template_height":800129},{"accepted_at_ms":999976,"credit_policy":null,"job_id":"job-4152","job_issued_at_ms":1000009,"miner_id":"miner-5","network_difficulty":1000004,"ntime":1700004152,"order_key":"05:miner-5","p2mr_program_hex":"3838383838383838383838383838383838383838383838383838383838383838","share_difficulty":3,"share_id":"share-4152","share_seq":4152,"template_height":800129},{"accepted_at_ms":1000008,"credit_policy":null,"job_id":"job-4153","job_issued_at_ms":1000009,"miner_id":"miner-6","network_difficulty":1000005,"ntime":1700004153,"order_key":"06:miner-6","p2mr_program_hex":"3939393939393939393939393939393939393939393939393939393939393939","share_difficulty":101,"share_id":"share-4153","share_seq":4153,"template_height":800129},{"accepted_at_ms":999991,"credit_policy":null,"job_id":"job-4154","job_issued_at_ms":1000009,"miner_id":"miner-7","network_difficulty":1000006,"ntime":1700004154,"order_key":"07:miner-7","p2mr_program_hex":"3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a","share_difficulty":101,"share_id":"share-4154","share_seq":4154,"template_height":800129},{"accepted_at_ms":1000002,"credit_policy":"stale-grace","job_id":"job-4155","job_issued_at_ms":999999,"miner_id":"miner-8","network_difficulty":1000007,"ntime":1700004155,"order_key":"08:miner-8","p2mr_program_hex":"3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b","share_difficulty":1,"share_id":"share-4155","share_seq":4155,"template_height":800129},{"accepted_at_ms":999970,"credit_policy":null,"job_id":"job-4156","job_issued_at_ms":1000002,"miner_id":"miner-9","network_difficulty":1000008,"ntime":1700004156,"order_key":"09:miner-9","p2mr_program_hex":"3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c","share_difficulty":101,"share_id":"share-4156","share_seq":4156,"template_height":800129},{"accepted_at_ms":999971,"credit_policy":null,"job_id":"job-4157","job_issued_at_ms":1000007,"miner_id":"miner-10","network_difficulty":1000009,"ntime":1700004157,"order_key":"10:miner-10","p2mr_program_hex":"3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d","share_difficulty":2,"share_id":"share-4157","share_seq":4157,"template_height":800129},{"accepted_at_ms":999994,"credit_policy":null,"job_id":"job-4158","job_issued_at_ms":1000007,"miner_id":"miner-0","network_difficulty":1000010,"ntime":1700004158,"order_key":"00:miner-0","p2mr_program_hex":"3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e","share_difficulty":7,"share_id":"share-4158","share_seq":4158,"template_height":800129},{"accepted_at_ms":1000007,"credit_policy":null,"job_id":"job-4159","job_issued_at_ms":999991,"miner_id":"miner-1","network_difficulty":1000011,"ntime":1700004159,"order_key":"01:miner-1","p2mr_program_hex":"3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f","share_difficulty":3,"share_id":"share-4159","share_seq":4159,"template_height":800129},{"accepted_at_ms":1000004,"credit_policy":null,"job_id":"job-4160","job_issued_at_ms":1000008,"miner_id":"miner-2","network_difficulty":1000012,"ntime":1700004160,"order_key":"02:miner-2","p2mr_program_hex":"4040404040404040404040404040404040404040404040404040404040404040","share_difficulty":1000,"share_id":"share-4160","share_seq":4160,"template_height":800130},{"accepted_at_ms":999994,"credit_policy":null,"job_id":"job-4161","job_issued_at_ms":1000002,"miner_id":"miner-3","network_difficulty":1000013,"ntime":1700004161,"order_key":"03:miner-3","p2mr_program_hex":"4141414141414141414141414141414141414141414141414141414141414141","share_difficulty":19,"share_id":"share-4161","share_seq":4161,"template_height":800130},{"accepted_at_ms":999989,"credit_policy":null,"job_id":"job-4162","job_issued_at_ms":1000004,"miner_id":"miner-4","network_difficulty":1000014,"ntime":1700004162,"order_key":"04:miner-4","p2mr_program_hex":"4242424242424242424242424242424242424242424242424242424242424242","share_difficulty":7,"share_id":"share-4162","share_seq":4162,"template_height":800130},{"accepted_at_ms":999972,"credit_policy":null,"job_id":"job-4163","job_issued_at_ms":1000001,"miner_id":"miner-5","network_difficulty":1000015,"ntime":1700004163,"order_key":"05:miner-5","p2mr_program_hex":"4343434343434343434343434343434343434343434343434343434343434343","share_difficulty":2,"share_id":"share-4163","share_seq":4163,"template_height":800130},{"accepted_at_ms":1000003,"credit_policy":null,"job_id":"job-4164","job_issued_at_ms":1000001,"miner_id":"miner-6","network_difficulty":1000016,"ntime":1700004164,"order_key":"06:miner-6","p2mr_program_hex":"4444444444444444444444444444444444444444444444444444444444444444","share_difficulty":19,"share_id":"share-4164","share_seq":4164,"template_height":800130},{"accepted_at_ms":999985,"credit_policy":null,"job_id":"job-4165","job_issued_at_ms":1000005,"miner_id":"miner-7","network_difficulty":1000000,"ntime":1700004165,"order_key":"07:miner-7","p2mr_program_hex":"4545454545454545454545454545454545454545454545454545454545454545","share_difficulty":1000,"share_id":"share-4165","share_seq":4165,"template_height":800130},{"accepted_at_ms":1000005,"credit_policy":null,"job_id":"job-4166","job_issued_at_ms":999971,"miner_id":"miner-8","network_difficulty":1000001,"ntime":1700004166,"order_key":"08:miner-8","p2mr_program_hex":"4646464646464646464646464646464646464646464646464646464646464646","share_difficulty":3,"share_id":"share-4166","share_seq":4166,"template_height":800130},{"accepted_at_ms":999974,"credit_policy":null,"job_id":"job-4167","job_issued_at_ms":1000008,"miner_id":"miner-9","network_difficulty":1000002,"ntime":1700004167,"order_key":"09:miner-9","p2mr_program_hex":"4747474747474747474747474747474747474747474747474747474747474747","share_difficulty":7,"share_id":"share-4167","share_seq":4167,"template_height":800130},{"accepted_at_ms":1000003,"credit_policy":"stale-grace","job_id":"job-4168","job_issued_at_ms":999997,"miner_id":"miner-10","network_difficulty":1000003,"ntime":1700004168,"order_key":"10:miner-10","p2mr_program_hex":"4848484848484848484848484848484848484848484848484848484848484848","share_difficulty":1000,"share_id":"share-4168","share_seq":4168,"template_height":800130},{"accepted_at_ms":1000008,"credit_policy":null,"job_id":"job-4169","job_issued_at_ms":1000001,"miner_id":"miner-0","network_difficulty":1000004,"ntime":1700004169,"order_key":"00:miner-0","p2mr_program_hex":"4949494949494949494949494949494949494949494949494949494949494949","share_difficulty":1,"share_id":"share-4169","share_seq":4169,"template_height":800130},{"accepted_at_ms":999984,"credit_policy":null,"job_id":"job-4170","job_issued_at_ms":1000003,"miner_id":"miner-1","network_difficulty":1000005,"ntime":1700004170,"order_key":"01:miner-1","p2mr_program_hex":"4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a","share_difficulty":101,"share_id":"share-4170","share_seq":4170,"template_height":800130},{"accepted_at_ms":1000008,"credit_policy":null,"job_id":"job-4171","job_issued_at_ms":1000005,"miner_id":"miner-2","network_difficulty":1000006,"ntime":1700004171,"order_key":"02:miner-2","p2mr_program_hex":"4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b","share_difficulty":2,"share_id":"share-4171","share_seq":4171,"template_height":800130},{"accepted_at_ms":999992,"credit_policy":null,"job_id":"job-4172","job_issued_at_ms":1000001,"miner_id":"miner-3","network_difficulty":1000007,"ntime":1700004172,"order_key":"03:miner-3","p2mr_program_hex":"4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c","share_difficulty":19,"share_id":"share-4172","share_seq":4172,"template_height":800130},{"accepted_at_ms":999993,"credit_policy":null,"job_id":"job-4173","job_issued_at_ms":1000007,"miner_id":"miner-4","network_difficulty":1000008,"ntime":1700004173,"order_key":"04:miner-4","p2mr_program_hex":"4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d","share_difficulty":2,"share_id":"share-4173","share_seq":4173,"template_height":800130},{"accepted_at_ms":999989,"credit_policy":null,"job_id":"job-4174","job_issued_at_ms":1000006,"miner_id":"miner-5","network_difficulty":1000009,"ntime":1700004174,"order_key":"05:miner-5","p2mr_program_hex":"4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e","share_difficulty":101,"share_id":"share-4174","share_seq":4174,"template_height":800130},{"accepted_at_ms":1000003,"credit_policy":null,"job_id":"job-4175","job_issued_at_ms":999981,"miner_id":"miner-6","network_difficulty":1000010,"ntime":1700004175,"order_key":"06:miner-6","p2mr_program_hex":"4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f","share_difficulty":3,"share_id":"share-4175","share_seq":4175,"template_height":800130},{"accepted_at_ms":999980,"credit_policy":"stale-grace","job_id":"job-4176","job_issued_at_ms":1000001,"miner_id":"miner-7","network_difficulty":1000011,"ntime":1700004176,"order_key":"07:miner-7","p2mr_program_hex":"5050505050505050505050505050505050505050505050505050505050505050","share_difficulty":1,"share_id":"share-4176","share_seq":4176,"template_height":800130},{"accepted_at_ms":999982,"credit_policy":null,"job_id":"job-4177","job_issued_at_ms":1000005,"miner_id":"miner-8","network_difficulty":1000012,"ntime":1700004177,"order_key":"08:miner-8","p2mr_program_hex":"5151515151515151515151515151515151515151515151515151515151515151","share_difficulty":101,"share_id":"share-4177","share_seq":4177,"template_height":800130},{"accepted_at_ms":999990,"credit_policy":"stale-grace","job_id":"job-4178","job_issued_at_ms":1000002,"miner_id":"miner-9","network_difficulty":1000013,"ntime":1700004178,"order_key":"09:miner-9","p2mr_program_hex":"5252525252525252525252525252525252525252525252525252525252525252","share_difficulty":1,"share_id":"share-4178","share_seq":4178,"template_height":800130},{"accepted_at_ms":1000006,"credit_policy":"stale-grace","job_id":"job-4179","job_issued_at_ms":999999,"miner_id":"miner-10","network_difficulty":1000014,"ntime":1700004179,"order_key":"10:miner-10","p2mr_program_hex":"5353535353535353535353535353535353535353535353535353535353535353","share_difficulty":19,"share_id":"share-4179","share_seq":4179,"template_height":800130},{"accepted_at_ms":1000002,"credit_policy":null,"job_id":"job-4180","job_issued_at_ms":1000006,"miner_id":"miner-0","network_difficulty":1000015,"ntime":1700004180,"order_key":"00:miner-0","p2mr_program_hex":"5454545454545454545454545454545454545454545454545454545454545454","share_difficulty":101,"share_id":"share-4180","share_seq":4180,"template_height":800130},{"accepted_at_ms":1000005,"credit_policy":"stale-grace","job_id":"job-4181","job_issued_at_ms":999998,"miner_id":"miner-1","network_difficulty":1000016,"ntime":1700004181,"order_key":"01:miner-1","p2mr_program_hex":"5555555555555555555555555555555555555555555555555555555555555555","share_difficulty":19,"share_id":"share-4181","share_seq":4181,"template_height":800130},{"accepted_at_ms":999971,"credit_policy":"stale-grace","job_id":"job-4182","job_issued_at_ms":1000003,"miner_id":"miner-2","network_difficulty":1000000,"ntime":1700004182,"order_key":"02:miner-2","p2mr_program_hex":"5656565656565656565656565656565656565656565656565656565656565656","share_difficulty":1000,"share_id":"share-4182","share_seq":4182,"template_height":800130},{"accepted_at_ms":999974,"credit_policy":null,"job_id":"job-4183","job_issued_at_ms":1000007,"miner_id":"miner-3","network_difficulty":1000001,"ntime":1700004183,"order_key":"03:miner-3","p2mr_program_hex":"5757575757575757575757575757575757575757575757575757575757575757","share_difficulty":3,"share_id":"share-4183","share_seq":4183,"template_height":800130},{"accepted_at_ms":999997,"credit_policy":"stale-grace","job_id":"job-4184","job_issued_at_ms":1000003,"miner_id":"miner-4","network_difficulty":1000002,"ntime":1700004184,"order_key":"04:miner-4","p2mr_program_hex":"5858585858585858585858585858585858585858585858585858585858585858","share_difficulty":7,"share_id":"share-4184","share_seq":4184,"template_height":800130},{"accepted_at_ms":1000004,"credit_policy":null,"job_id":"job-4185","job_issued_at_ms":999982,"miner_id":"miner-5","network_difficulty":1000003,"ntime":1700004185,"order_key":"05:miner-5","p2mr_program_hex":"5959595959595959595959595959595959595959595959595959595959595959","share_difficulty":2,"share_id":"share-4185","share_seq":4185,"template_height":800130},{"accepted_at_ms":1000009,"credit_policy":null,"job_id":"job-4186","job_issued_at_ms":999983,"miner_id":"miner-6","network_difficulty":1000004,"ntime":1700004186,"order_key":"06:miner-6","p2mr_program_hex":"5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a","share_difficulty":101,"share_id":"share-4186","share_seq":4186,"template_height":800130},{"accepted_at_ms":1000006,"credit_policy":null,"job_id":"job-4187","job_issued_at_ms":1000010,"miner_id":"miner-7","network_difficulty":1000005,"ntime":1700004187,"order_key":"07:miner-7","p2mr_program_hex":"5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b","share_difficulty":2,"share_id":"share-4187","share_seq":4187,"template_height":800130},{"accepted_at_ms":999988,"credit_policy":"stale-grace","job_id":"job-4188","job_issued_at_ms":1000006,"miner_id":"miner-8","network_difficulty":1000006,"ntime":1700004188,"order_key":"08:miner-8","p2mr_program_hex":"5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c","share_difficulty":1,"share_id":"share-4188","share_seq":4188,"template_height":800130},{"accepted_at_ms":999988,"credit_policy":null,"job_id":"job-4189","job_issued_at_ms":1000005,"miner_id":"miner-9","network_difficulty":1000007,"ntime":1700004189,"order_key":"09:miner-9","p2mr_program_hex":"5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d","share_difficulty":101,"share_id":"share-4189","share_seq":4189,"template_height":800130},{"accepted_at_ms":1000006,"credit_policy":null,"job_id":"job-4190","job_issued_at_ms":999997,"miner_id":"miner-10","network_difficulty":1000008,"ntime":1700004190,"order_key":"10:miner-10","p2mr_program_hex":"5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e","share_difficulty":1000,"share_id":"share-4190","share_seq":4190,"template_height":800130},{"accepted_at_ms":1000010,"credit_policy":null,"job_id":"job-4191","job_issued_at_ms":999975,"miner_id":"miner-0","network_difficulty":1000009,"ntime":1700004191,"order_key":"00:miner-0","p2mr_program_hex":"5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f","share_difficulty":3,"share_id":"share-4191","share_seq":4191,"template_height":800130},{"accepted_at_ms":1000009,"credit_policy":null,"job_id":"job-4192","job_issued_at_ms":1000010,"miner_id":"miner-1","network_difficulty":1000010,"ntime":1700004192,"order_key":"01:miner-1","p2mr_program_hex":"6060606060606060606060606060606060606060606060606060606060606060","share_difficulty":101,"share_id":"share-4192","share_seq":4192,"template_height":800131},{"accepted_at_ms":1000008,"credit_policy":null,"job_id":"job-4193","job_issued_at_ms":999989,"miner_id":"miner-2","network_difficulty":1000011,"ntime":1700004193,"order_key":"02:miner-2","p2mr_program_hex":"6161616161616161616161616161616161616161616161616161616161616161","share_difficulty":1,"share_id":"share-4193","share_seq":4193,"template_height":800131},{"accepted_at_ms":1000002,"credit_policy":null,"job_id":"job-4194","job_issued_at_ms":999986,"miner_id":"miner-3","network_difficulty":1000012,"ntime":1700004194,"order_key":"03:miner-3","p2mr_program_hex":"6262626262626262626262626262626262626262626262626262626262626262","share_difficulty":7,"share_id":"share-4194","share_seq":4194,"template_height":800131},{"accepted_at_ms":999971,"credit_policy":null,"job_id":"job-4195","job_issued_at_ms":1000002,"miner_id":"miner-4","network_difficulty":1000013,"ntime":1700004195,"order_key":"04:miner-4","p2mr_program_hex":"6363636363636363636363636363636363636363636363636363636363636363","share_difficulty":1,"share_id":"share-4195","share_seq":4195,"template_height":800131},{"accepted_at_ms":999982,"credit_policy":null,"job_id":"job-4196","job_issued_at_ms":1000001,"miner_id":"miner-5","network_difficulty":1000014,"ntime":1700004196,"order_key":"05:miner-5","p2mr_program_hex":"6464646464646464646464646464646464646464646464646464646464646464","share_difficulty":1,"share_id":"share-4196","share_seq":4196,"template_height":800131},{"accepted_at_ms":1000006,"credit_policy":"stale-grace","job_id":"job-4197","job_issued_at_ms":999983,"miner_id":"miner-6","network_difficulty":1000015,"ntime":1700004197,"order_key":"06:miner-6","p2mr_program_hex":"6565656565656565656565656565656565656565656565656565656565656565","share_difficulty":2,"share_id":"share-4197","share_seq":4197,"template_height":800131},{"accepted_at_ms":999972,"credit_policy":null,"job_id":"job-4198","job_issued_at_ms":1000005,"miner_id":"miner-7","network_difficulty":1000016,"ntime":1700004198,"order_key":"07:miner-7","p2mr_program_hex":"6666666666666666666666666666666666666666666666666666666666666666","share_difficulty":3,"share_id":"share-4198","share_seq":4198,"template_height":800131},{"accepted_at_ms":1000009,"credit_policy":null,"job_id":"job-4199","job_issued_at_ms":999980,"miner_id":"miner-8","network_difficulty":1000000,"ntime":1700004199,"order_key":"08:miner-8","p2mr_program_hex":"6767676767676767676767676767676767676767676767676767676767676767","share_difficulty":1000,"share_id":"share-4199","share_seq":4199,"template_height":800131},{"accepted_at_ms":999979,"credit_policy":"stale-grace","job_id":"job-4200","job_issued_at_ms":1000007,"miner_id":"miner-9","network_difficulty":1000001,"ntime":1700004200,"order_key":"09:miner-9","p2mr_program_hex":"6868686868686868686868686868686868686868686868686868686868686868","share_difficulty":2,"share_id":"share-4200","share_seq":4200,"template_height":800131},{"accepted_at_ms":999986,"credit_policy":null,"job_id":"job-4201","job_issued_at_ms":1000009,"miner_id":"miner-10","network_difficulty":1000002,"ntime":1700004201,"order_key":"10:miner-10","p2mr_program_hex":"6969696969696969696969696969696969696969696969696969696969696969","share_difficulty":2,"share_id":"share-4201","share_seq":4201,"template_height":800131},{"accepted_at_ms":999978,"credit_policy":null,"job_id":"job-4202","job_issued_at_ms":1000010,"miner_id":"miner-0","network_difficulty":1000003,"ntime":1700004202,"order_key":"00:miner-0","p2mr_program_hex":"6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a","share_difficulty":101,"share_id":"share-4202","share_seq":4202,"template_height":800131},{"accepted_at_ms":1000008,"credit_policy":null,"job_id":"job-4203","job_issued_at_ms":1000006,"miner_id":"miner-1","network_difficulty":1000004,"ntime":1700004203,"order_key":"01:miner-1","p2mr_program_hex":"6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b","share_difficulty":1,"share_id":"share-4203","share_seq":4203,"template_height":800131},{"accepted_at_ms":1000003,"credit_policy":"stale-grace","job_id":"job-4204","job_issued_at_ms":999985,"miner_id":"miner-2","network_difficulty":1000005,"ntime":1700004204,"order_key":"02:miner-2","p2mr_program_hex":"6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c","share_difficulty":2,"share_id":"share-4204","share_seq":4204,"template_height":800131},{"accepted_at_ms":1000010,"credit_policy":null,"job_id":"job-4205","job_issued_at_ms":1000001,"miner_id":"miner-3","network_difficulty":1000006,"ntime":1700004205,"order_key":"03:miner-3","p2mr_program_hex":"6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d","share_difficulty":19,"share_id":"share-4205","share_seq":4205,"template_height":800131},{"accepted_at_ms":999993,"credit_policy":"stale-grace","job_id":"job-4206","job_issued_at_ms":1000006,"miner_id":"miner-4","network_difficulty":1000007,"ntime":1700004206,"order_key":"04:miner-4","p2mr_program_hex":"6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e","share_difficulty":1000,"share_id":"share-4206","share_seq":4206,"template_height":800131},{"accepted_at_ms":999977,"credit_policy":null,"job_id":"job-4207","job_issued_at_ms":1000009,"miner_id":"miner-5","network_difficulty":1000008,"ntime":1700004207,"order_key":"05:miner-5","p2mr_program_hex":"6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f","share_difficulty":3,"share_id":"share-4207","share_seq":4207,"template_height":800131},{"accepted_at_ms":999993,"credit_policy":"stale-grace","job_id":"job-4208","job_issued_at_ms":1000005,"miner_id":"miner-6","network_difficulty":1000009,"ntime":1700004208,"order_key":"06:miner-6","p2mr_program_hex":"7070707070707070707070707070707070707070707070707070707070707070","share_difficulty":19,"share_id":"share-4208","share_seq":4208,"template_height":800131},{"accepted_at_ms":1000001,"credit_policy":null,"job_id":"job-4209","job_issued_at_ms":999986,"miner_id":"miner-7","network_difficulty":1000010,"ntime":1700004209,"order_key":"07:miner-7","p2mr_program_hex":"7171717171717171717171717171717171717171717171717171717171717171","share_difficulty":19,"share_id":"share-4209","share_seq":4209,"template_height":800131},{"accepted_at_ms":1000009,"credit_policy":null,"job_id":"job-4210","job_issued_at_ms":999992,"miner_id":"miner-8","network_difficulty":1000011,"ntime":1700004210,"order_key":"08:miner-8","p2mr_program_hex":"7272727272727272727272727272727272727272727272727272727272727272","share_difficulty":3,"share_id":"share-4210","share_seq":4210,"template_height":800131},{"accepted_at_ms":999985,"credit_policy":null,"job_id":"job-4211","job_issued_at_ms":1000001,"miner_id":"miner-9","network_difficulty":1000012,"ntime":1700004211,"order_key":"09:miner-9","p2mr_program_hex":"7373737373737373737373737373737373737373737373737373737373737373","share_difficulty":19,"share_id":"share-4211","share_seq":4211,"template_height":800131},{"accepted_at_ms":999973,"credit_policy":null,"job_id":"job-4212","job_issued_at_ms":1000001,"miner_id":"miner-10","network_difficulty":1000013,"ntime":1700004212,"order_key":"10:miner-10","p2mr_program_hex":"7474747474747474747474747474747474747474747474747474747474747474","share_difficulty":2,"share_id":"share-4212","share_seq":4212,"template_height":800131},{"accepted_at_ms":1000009,"credit_policy":null,"job_id":"job-4213","job_issued_at_ms":1000010,"miner_id":"miner-0","network_difficulty":1000014,"ntime":1700004213,"order_key":"00:miner-0","p2mr_program_hex":"7575757575757575757575757575757575757575757575757575757575757575","share_difficulty":19,"share_id":"share-4213","share_seq":4213,"template_height":800131},{"accepted_at_ms":1000010,"credit_policy":null,"job_id":"job-4214","job_issued_at_ms":1000002,"miner_id":"miner-1","network_difficulty":1000015,"ntime":1700004214,"order_key":"01:miner-1","p2mr_program_hex":"7676767676767676767676767676767676767676767676767676767676767676","share_difficulty":101,"share_id":"share-4214","share_seq":4214,"template_height":800131},{"accepted_at_ms":1000003,"credit_policy":null,"job_id":"job-4215","job_issued_at_ms":999991,"miner_id":"miner-2","network_difficulty":1000016,"ntime":1700004215,"order_key":"02:miner-2","p2mr_program_hex":"7777777777777777777777777777777777777777777777777777777777777777","share_difficulty":1000,"share_id":"share-4215","share_seq":4215,"template_height":800131},{"accepted_at_ms":999984,"credit_policy":null,"job_id":"job-4216","job_issued_at_ms":1000002,"miner_id":"miner-3","network_difficulty":1000000,"ntime":1700004216,"order_key":"03:miner-3","p2mr_program_hex":"7878787878787878787878787878787878787878787878787878787878787878","share_difficulty":7,"share_id":"share-4216","share_seq":4216,"template_height":800131},{"accepted_at_ms":1000010,"credit_policy":"stale-grace","job_id":"job-4217","job_issued_at_ms":999989,"miner_id":"miner-4","network_difficulty":1000001,"ntime":1700004217,"order_key":"04:miner-4","p2mr_program_hex":"7979797979797979797979797979797979797979797979797979797979797979","share_difficulty":3,"share_id":"share-4217","share_seq":4217,"template_height":800131},{"accepted_at_ms":1000003,"credit_policy":null,"job_id":"job-4218","job_issued_at_ms":1000006,"miner_id":"miner-5","network_difficulty":1000002,"ntime":1700004218,"order_key":"05:miner-5","p2mr_program_hex":"7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a","share_difficulty":101,"share_id":"share-4218","share_seq":4218,"template_height":800131},{"accepted_at_ms":999976,"credit_policy":null,"job_id":"job-4219","job_issued_at_ms":1000010,"miner_id":"miner-6","network_difficulty":1000003,"ntime":1700004219,"order_key":"06:miner-6","p2mr_program_hex":"7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b","share_difficulty":1,"share_id":"share-4219","share_seq":4219,"template_height":800131},{"accepted_at_ms":1000003,"credit_policy":"stale-grace","job_id":"job-4220","job_issued_at_ms":999972,"miner_id":"miner-7","network_difficulty":1000004,"ntime":1700004220,"order_key":"07:miner-7","p2mr_program_hex":"7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c","share_difficulty":7,"share_id":"share-4220","share_seq":4220,"template_height":800131},{"accepted_at_ms":1000003,"credit_policy":"stale-grace","job_id":"job-4221","job_issued_at_ms":999996,"miner_id":"miner-8","network_difficulty":1000005,"ntime":1700004221,"order_key":"08:miner-8","p2mr_program_hex":"7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d","share_difficulty":1,"share_id":"share-4221","share_seq":4221,"template_height":800131},{"accepted_at_ms":999972,"credit_policy":null,"job_id":"job-4222","job_issued_at_ms":1000006,"miner_id":"miner-9","network_difficulty":1000006,"ntime":1700004222,"order_key":"09:miner-9","p2mr_program_hex":"7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e","share_difficulty":1,"share_id":"share-4222","share_seq":4222,"template_height":800131},{"accepted_at_ms":999985,"credit_policy":null,"job_id":"job-4223","job_issued_at_ms":1000006,"miner_id":"miner-10","network_difficulty":1000007,"ntime":1700004223,"order_key":"10:miner-10","p2mr_program_hex":"7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f","share_difficulty":1,"share_id":"share-4223","share_seq":4223,"template_height":800131},{"accepted_at_ms":1000008,"credit_policy":null,"job_id":"job-4224","job_issued_at_ms":999975,"miner_id":"miner-0","network_difficulty":1000008,"ntime":1700004224,"order_key":"00:miner-0","p2mr_program_hex":"8080808080808080808080808080808080808080808080808080808080808080","share_difficulty":101,"share_id":"share-4224","share_seq":4224,"template_height":800132},{"accepted_at_ms":1000001,"credit_policy":"stale-grace","job_id":"job-4225","job_issued_at_ms":999970,"miner_id":"miner-1","network_difficulty":1000009,"ntime":1700004225,"order_key":"01:miner-1","p2mr_program_hex":"8181818181818181818181818181818181818181818181818181818181818181","share_difficulty":1000,"share_id":"share-4225","share_seq":4225,"template_height":800132},{"accepted_at_ms":1000010,"credit_policy":"stale-grace","job_id":"job-4226","job_issued_at_ms":999977,"miner_id":"miner-2","network_difficulty":1000010,"ntime":1700004226,"order_key":"02:miner-2","p2mr_program_hex":"8282828282828282828282828282828282828282828282828282828282828282","share_difficulty":2,"share_id":"share-4226","share_seq":4226,"template_height":800132},{"accepted_at_ms":999987,"credit_policy":null,"job_id":"job-4227","job_issued_at_ms":1000005,"miner_id":"miner-3","network_difficulty":1000011,"ntime":1700004227,"order_key":"03:miner-3","p2mr_program_hex":"8383838383838383838383838383838383838383838383838383838383838383","share_difficulty":101,"share_id":"share-4227","share_seq":4227,"template_height":800132},{"accepted_at_ms":1000001,"credit_policy":"stale-grace","job_id":"job-4228","job_issued_at_ms":1000001,"miner_id":"miner-4","network_difficulty":1000012,"ntime":1700004228,"order_key":"04:miner-4","p2mr_program_hex":"8484848484848484848484848484848484848484848484848484848484848484","share_difficulty":2,"share_id":"share-4228","share_seq":4228,"template_height":800132},{"accepted_at_ms":999999,"credit_policy":"stale-grace","job_id":"job-4229","job_issued_at_ms":1000003,"miner_id":"miner-5","network_difficulty":1000013,"ntime":1700004229,"order_key":"05:miner-5","p2mr_program_hex":"8585858585858585858585858585858585858585858585858585858585858585","share_difficulty":2,"share_id":"share-4229","share_seq":4229,"template_height":800132},{"accepted_at_ms":1000006,"credit_policy":null,"job_id":"job-4230","job_issued_at_ms":1000001,"miner_id":"miner-6","network_difficulty":1000014,"ntime":1700004230,"order_key":"06:miner-6","p2mr_program_hex":"8686868686868686868686868686868686868686868686868686868686868686","share_difficulty":1000,"share_id":"share-4230","share_seq":4230,"template_height":800132},{"accepted_at_ms":999974,"credit_policy":null,"job_id":"job-4231","job_issued_at_ms":1000002,"miner_id":"miner-7","network_difficulty":1000015,"ntime":1700004231,"order_key":"07:miner-7","p2mr_program_hex":"8787878787878787878787878787878787878787878787878787878787878787","share_difficulty":7,"share_id":"share-4231","share_seq":4231,"template_height":800132},{"accepted_at_ms":999981,"credit_policy":null,"job_id":"job-4232","job_issued_at_ms":1000004,"miner_id":"miner-8","network_difficulty":1000016,"ntime":1700004232,"order_key":"08:miner-8","p2mr_program_hex":"8888888888888888888888888888888888888888888888888888888888888888","share_difficulty":2,"share_id":"share-4232","share_seq":4232,"template_height":800132},{"accepted_at_ms":1000001,"credit_policy":"stale-grace","job_id":"job-4233","job_issued_at_ms":999990,"miner_id":"miner-9","network_difficulty":1000000,"ntime":1700004233,"order_key":"09:miner-9","p2mr_program_hex":"8989898989898989898989898989898989898989898989898989898989898989","share_difficulty":1,"share_id":"share-4233","share_seq":4233,"template_height":800132},{"accepted_at_ms":999979,"credit_policy":null,"job_id":"job-4234","job_issued_at_ms":1000002,"miner_id":"miner-10","network_difficulty":1000001,"ntime":1700004234,"order_key":"10:miner-10","p2mr_program_hex":"8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a","share_difficulty":2,"share_id":"share-4234","share_seq":4234,"template_height":800132},{"accepted_at_ms":1000005,"credit_policy":null,"job_id":"job-4235","job_issued_at_ms":999974,"miner_id":"miner-0","network_difficulty":1000002,"ntime":1700004235,"order_key":"00:miner-0","p2mr_program_hex":"8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b","share_difficulty":1,"share_id":"share-4235","share_seq":4235,"template_height":800132},{"accepted_at_ms":1000008,"credit_policy":null,"job_id":"job-4236","job_issued_at_ms":1000003,"miner_id":"miner-1","network_difficulty":1000003,"ntime":1700004236,"order_key":"01:miner-1","p2mr_program_hex":"8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c","share_difficulty":7,"share_id":"share-4236","share_seq":4236,"template_height":800132},{"accepted_at_ms":999979,"credit_policy":null,"job_id":"job-4237","job_issued_at_ms":1000004,"miner_id":"miner-2","network_difficulty":1000004,"ntime":1700004237,"order_key":"02:miner-2","p2mr_program_hex":"8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d","share_difficulty":3,"share_id":"share-4237","share_seq":4237,"template_height":800132},{"accepted_at_ms":999979,"credit_policy":null,"job_id":"job-4238","job_issued_at_ms":1000007,"miner_id":"miner-3","network_difficulty":1000005,"ntime":1700004238,"order_key":"03:miner-3","p2mr_program_hex":"8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e","share_difficulty":3,"share_id":"share-4238","share_seq":4238,"template_height":800132},{"accepted_at_ms":1000002,"credit_policy":"stale-grace","job_id":"job-4239","job_issued_at_ms":1000008,"miner_id":"miner-4","network_difficulty":1000006,"ntime":1700004239,"order_key":"04:miner-4","p2mr_program_hex":"8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f","share_difficulty":101,"share_id":"share-4239","share_seq":4239,"template_height":800132},{"accepted_at_ms":999983,"credit_policy":null,"job_id":"job-4240","job_issued_at_ms":1000008,"miner_id":"miner-5","network_difficulty":1000007,"ntime":1700004240,"order_key":"05:miner-5","p2mr_program_hex":"9090909090909090909090909090909090909090909090909090909090909090","share_difficulty":19,"share_id":"share-4240","share_seq":4240,"template_height":800132},{"accepted_at_ms":999979,"credit_policy":null,"job_id":"job-4241","job_issued_at_ms":1000010,"miner_id":"miner-6","network_difficulty":1000008,"ntime":1700004241,"order_key":"06:miner-6","p2mr_program_hex":"9191919191919191919191919191919191919191919191919191919191919191","share_difficulty":19,"share_id":"share-4241","share_seq":4241,"template_height":800132},{"accepted_at_ms":1000009,"credit_policy":null,"job_id":"job-4242","job_issued_at_ms":999990,"miner_id":"miner-7","network_difficulty":1000009,"ntime":1700004242,"order_key":"07:miner-7","p2mr_program_hex":"9292929292929292929292929292929292929292929292929292929292929292","share_difficulty":1000,"share_id":"share-4242","share_seq":4242,"template_height":800132},{"accepted_at_ms":999985,"credit_policy":null,"job_id":"job-4243","job_issued_at_ms":1000005,"miner_id":"miner-8","network_difficulty":1000010,"ntime":1700004243,"order_key":"08:miner-8","p2mr_program_hex":"9393939393939393939393939393939393939393939393939393939393939393","share_difficulty":1,"share_id":"share-4243","share_seq":4243,"template_height":800132},{"accepted_at_ms":999995,"credit_policy":null,"job_id":"job-4244","job_issued_at_ms":1000004,"miner_id":"miner-9","network_difficulty":1000011,"ntime":1700004244,"order_key":"09:miner-9","p2mr_program_hex":"9494949494949494949494949494949494949494949494949494949494949494","share_difficulty":19,"share_id":"share-4244","share_seq":4244,"template_height":800132},{"accepted_at_ms":1000010,"credit_policy":"stale-grace","job_id":"job-4245","job_issued_at_ms":1000004,"miner_id":"miner-10","network_difficulty":1000012,"ntime":1700004245,"order_key":"10:miner-10","p2mr_program_hex":"9595959595959595959595959595959595959595959595959595959595959595","share_difficulty":2,"share_id":"share-4245","share_seq":4245,"template_height":800132},{"accepted_at_ms":1000001,"credit_policy":null,"job_id":"job-4246","job_issued_at_ms":999970,"miner_id":"miner-0","network_difficulty":1000013,"ntime":1700004246,"order_key":"00:miner-0","p2mr_program_hex":"9696969696969696969696969696969696969696969696969696969696969696","share_difficulty":19,"share_id":"share-4246","share_seq":4246,"template_height":800132},{"accepted_at_ms":1000002,"credit_policy":"stale-grace","job_id":"job-4247","job_issued_at_ms":999993,"miner_id":"miner-1","network_difficulty":1000014,"ntime":1700004247,"order_key":"01:miner-1","p2mr_program_hex":"9797979797979797979797979797979797979797979797979797979797979797","share_difficulty":1,"share_id":"share-4247","share_seq":4247,"template_height":800132},{"accepted_at_ms":1000007,"credit_policy":null,"job_id":"job-4248","job_issued_at_ms":999972,"miner_id":"miner-2","network_difficulty":1000015,"ntime":1700004248,"order_key":"02:miner-2","p2mr_program_hex":"9898989898989898989898989898989898989898989898989898989898989898","share_difficulty":19,"share_id":"share-4248","share_seq":4248,"template_height":800132},{"accepted_at_ms":1000010,"credit_policy":"stale-grace","job_id":"job-4249","job_issued_at_ms":999977,"miner_id":"miner-3","network_difficulty":1000016,"ntime":1700004249,"order_key":"03:miner-3","p2mr_program_hex":"9999999999999999999999999999999999999999999999999999999999999999","share_difficulty":7,"share_id":"share-4249","share_seq":4249,"template_height":800132},{"accepted_at_ms":1000009,"credit_policy":null,"job_id":"job-4250","job_issued_at_ms":999998,"miner_id":"miner-4","network_difficulty":1000000,"ntime":1700004250,"order_key":"04:miner-4","p2mr_program_hex":"9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a","share_difficulty":3,"share_id":"share-4250","share_seq":4250,"template_height":800132},{"accepted_at_ms":999970,"credit_policy":null,"job_id":"job-4251","job_issued_at_ms":1000010,"miner_id":"miner-5","network_difficulty":1000001,"ntime":1700004251,"order_key":"05:miner-5","p2mr_program_hex":"9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b","share_difficulty":3,"share_id":"share-4251","share_seq":4251,"template_height":800132},{"accepted_at_ms":1000006,"credit_policy":"stale-grace","job_id":"job-4252","job_issued_at_ms":999977,"miner_id":"miner-6","network_difficulty":1000002,"ntime":1700004252,"order_key":"06:miner-6","p2mr_program_hex":"9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c","share_difficulty":7,"share_id":"share-4252","share_seq":4252,"template_height":800132},{"accepted_at_ms":1000009,"credit_policy":null,"job_id":"job-4253","job_issued_at_ms":1000009,"miner_id":"miner-7","network_difficulty":1000003,"ntime":1700004253,"order_key":"07:miner-7","p2mr_program_hex":"9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d","share_difficulty":101,"share_id":"share-4253","share_seq":4253,"template_height":800132},{"accepted_at_ms":1000003,"credit_policy":null,"job_id":"job-4254","job_issued_at_ms":1000010,"miner_id":"miner-8","network_difficulty":1000004,"ntime":1700004254,"order_key":"08:miner-8","p2mr_program_hex":"9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e","share_difficulty":101,"share_id":"share-4254","share_seq":4254,"template_height":800132},{"accepted_at_ms":1000005,"credit_policy":null,"job_id":"job-4255","job_issued_at_ms":999985,"miner_id":"miner-9","network_difficulty":1000005,"ntime":1700004255,"order_key":"09:miner-9","p2mr_program_hex":"9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f","share_difficulty":1000,"share_id":"share-4255","share_seq":4255,"template_height":800132},{"accepted_at_ms":1000002,"credit_policy":null,"job_id":"job-4256","job_issued_at_ms":1000009,"miner_id":"miner-10","network_difficulty":1000006,"ntime":1700004256,"order_key":"10:miner-10","p2mr_program_hex":"a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0","share_difficulty":2,"share_id":"share-4256","share_seq":4256,"template_height":800133},{"accepted_at_ms":999986,"credit_policy":"stale-grace","job_id":"job-4257","job_issued_at_ms":1000006,"miner_id":"miner-0","network_difficulty":1000007,"ntime":1700004257,"order_key":"00:miner-0","p2mr_program_hex":"a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1","share_difficulty":1000,"share_id":"share-4257","share_seq":4257,"template_height":800133},{"accepted_at_ms":1000006,"credit_policy":null,"job_id":"job-4258","job_issued_at_ms":999972,"miner_id":"miner-1","network_difficulty":1000008,"ntime":1700004258,"order_key":"01:miner-1","p2mr_program_hex":"a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2","share_difficulty":1,"share_id":"share-4258","share_seq":4258,"template_height":800133},{"accepted_at_ms":1000001,"credit_policy":null,"job_id":"job-4259","job_issued_at_ms":1000001,"miner_id":"miner-2","network_difficulty":1000009,"ntime":1700004259,"order_key":"02:miner-2","p2mr_program_hex":"a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3","share_difficulty":19,"share_id":"share-4259","share_seq":4259,"template_height":800133},{"accepted_at_ms":999972,"credit_policy":null,"job_id":"job-4260","job_issued_at_ms":1000001,"miner_id":"miner-3","network_difficulty":1000010,"ntime":1700004260,"order_key":"03:miner-3","p2mr_program_hex":"a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4","share_difficulty":7,"share_id":"share-4260","share_seq":4260,"template_height":800133},{"accepted_at_ms":1000003,"credit_policy":null,"job_id":"job-4261","job_issued_at_ms":999986,"miner_id":"miner-4","network_difficulty":1000011,"ntime":1700004261,"order_key":"04:miner-4","p2mr_program_hex":"a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5","share_difficulty":1,"share_id":"share-4261","share_seq":4261,"template_height":800133},{"accepted_at_ms":999970,"credit_policy":null,"job_id":"job-4262","job_issued_at_ms":1000001,"miner_id":"miner-5","network_difficulty":1000012,"ntime":1700004262,"order_key":"05:miner-5","p2mr_program_hex":"a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6","share_difficulty":1,"share_id":"share-4262","share_seq":4262,"template_height":800133},{"accepted_at_ms":1000004,"credit_policy":null,"job_id":"job-4263","job_issued_at_ms":1000006,"miner_id":"miner-6","network_difficulty":1000013,"ntime":1700004263,"order_key":"06:miner-6","p2mr_program_hex":"a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7","share_difficulty":19,"share_id":"share-4263","share_seq":4263,"template_height":800133},{"accepted_at_ms":1000004,"credit_policy":null,"job_id":"job-4264","job_issued_at_ms":999988,"miner_id":"miner-7","network_difficulty":1000014,"ntime":1700004264,"order_key":"07:miner-7","p2mr_program_hex":"a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8","share_difficulty":19,"share_id":"share-4264","share_seq":4264,"template_height":800133},{"accepted_at_ms":1000001,"credit_policy":"stale-grace","job_id":"job-4265","job_issued_at_ms":999977,"miner_id":"miner-8","network_difficulty":1000015,"ntime":1700004265,"order_key":"08:miner-8","p2mr_program_hex":"a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9","share_difficulty":1,"share_id":"share-4265","share_seq":4265,"template_height":800133},{"accepted_at_ms":999976,"credit_policy":null,"job_id":"job-4266","job_issued_at_ms":1000002,"miner_id":"miner-9","network_difficulty":1000016,"ntime":1700004266,"order_key":"09:miner-9","p2mr_program_hex":"aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa","share_difficulty":101,"share_id":"share-4266","share_seq":4266,"template_height":800133},{"accepted_at_ms":1000010,"credit_policy":null,"job_id":"job-4267","job_issued_at_ms":999976,"miner_id":"miner-10","network_difficulty":1000000,"ntime":1700004267,"order_key":"10:miner-10","p2mr_program_hex":"abababababababababababababababababababababababababababababababab","share_difficulty":1000,"share_id":"share-4267","share_seq":4267,"template_height":800133},{"accepted_at_ms":999980,"credit_policy":null,"job_id":"job-4268","job_issued_at_ms":1000005,"miner_id":"miner-0","network_difficulty":1000001,"ntime":1700004268,"order_key":"00:miner-0","p2mr_program_hex":"acacacacacacacacacacacacacacacacacacacacacacacacacacacacacacacac","share_difficulty":1,"share_id":"share-4268","share_seq":4268,"template_height":800133},{"accepted_at_ms":1000004,"credit_policy":null,"job_id":"job-4269","job_issued_at_ms":999975,"miner_id":"miner-1","network_difficulty":1000002,"ntime":1700004269,"order_key":"01:miner-1","p2mr_program_hex":"adadadadadadadadadadadadadadadadadadadadadadadadadadadadadadadad","share_difficulty":2,"share_id":"share-4269","share_seq":4269,"template_height":800133},{"accepted_at_ms":999997,"credit_policy":null,"job_id":"job-4270","job_issued_at_ms":1000009,"miner_id":"miner-2","network_difficulty":1000003,"ntime":1700004270,"order_key":"02:miner-2","p2mr_program_hex":"aeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeae","share_difficulty":1000,"share_id":"share-4270","share_seq":4270,"template_height":800133},{"accepted_at_ms":999986,"credit_policy":null,"job_id":"job-4271","job_issued_at_ms":1000001,"miner_id":"miner-3","network_difficulty":1000004,"ntime":1700004271,"order_key":"03:miner-3","p2mr_program_hex":"afafafafafafafafafafafafafafafafafafafafafafafafafafafafafafafaf","share_difficulty":1000,"share_id":"share-4271","share_seq":4271,"template_height":800133},{"accepted_at_ms":999989,"credit_policy":"stale-grace","job_id":"job-4272","job_issued_at_ms":1000007,"miner_id":"miner-4","network_difficulty":1000005,"ntime":1700004272,"order_key":"04:miner-4","p2mr_program_hex":"b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0","share_difficulty":3,"share_id":"share-4272","share_seq":4272,"template_height":800133},{"accepted_at_ms":1000002,"credit_policy":null,"job_id":"job-4273","job_issued_at_ms":999997,"miner_id":"miner-5","network_difficulty":1000006,"ntime":1700004273,"order_key":"05:miner-5","p2mr_program_hex":"b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1","share_difficulty":1000,"share_id":"share-4273","share_seq":4273,"template_height":800133},{"accepted_at_ms":999986,"credit_policy":null,"job_id":"job-4274","job_issued_at_ms":1000010,"miner_id":"miner-6","network_difficulty":1000007,"ntime":1700004274,"order_key":"06:miner-6","p2mr_program_hex":"b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2","share_difficulty":2,"share_id":"share-4274","share_seq":4274,"template_height":800133},{"accepted_at_ms":999976,"credit_policy":null,"job_id":"job-4275","job_issued_at_ms":1000005,"miner_id":"miner-7","network_difficulty":1000008,"ntime":1700004275,"order_key":"07:miner-7","p2mr_program_hex":"b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3","share_difficulty":101,"share_id":"share-4275","share_seq":4275,"template_height":800133},{"accepted_at_ms":1000003,"credit_policy":null,"job_id":"job-4276","job_issued_at_ms":1000007,"miner_id":"miner-8","network_difficulty":1000009,"ntime":1700004276,"order_key":"08:miner-8","p2mr_program_hex":"b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4","share_difficulty":3,"share_id":"share-4276","share_seq":4276,"template_height":800133},{"accepted_at_ms":999979,"credit_policy":null,"job_id":"job-4277","job_issued_at_ms":1000002,"miner_id":"miner-9","network_difficulty":1000010,"ntime":1700004277,"order_key":"09:miner-9","p2mr_program_hex":"b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5","share_difficulty":101,"share_id":"share-4277","share_seq":4277,"template_height":800133},{"accepted_at_ms":1000006,"credit_policy":null,"job_id":"job-4278","job_issued_at_ms":1000007,"miner_id":"miner-10","network_difficulty":1000011,"ntime":1700004278,"order_key":"10:miner-10","p2mr_program_hex":"b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6","share_difficulty":1,"share_id":"share-4278","share_seq":4278,"template_height":800133},{"accepted_at_ms":1000002,"credit_policy":null,"job_id":"job-4279","job_issued_at_ms":1000009,"miner_id":"miner-0","network_difficulty":1000012,"ntime":1700004279,"order_key":"00:miner-0","p2mr_program_hex":"b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7","share_difficulty":101,"share_id":"share-4279","share_seq":4279,"template_height":800133},{"accepted_at_ms":999989,"credit_policy":null,"job_id":"job-4280","job_issued_at_ms":1000003,"miner_id":"miner-1","network_difficulty":1000013,"ntime":1700004280,"order_key":"01:miner-1","p2mr_program_hex":"b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8","share_difficulty":1,"share_id":"share-4280","share_seq":4280,"template_height":800133},{"accepted_at_ms":1000005,"credit_policy":null,"job_id":"job-4281","job_issued_at_ms":1000001,"miner_id":"miner-2","network_difficulty":1000014,"ntime":1700004281,"order_key":"02:miner-2","p2mr_program_hex":"b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9","share_difficulty":3,"share_id":"share-4281","share_seq":4281,"template_height":800133},{"accepted_at_ms":1000005,"credit_policy":null,"job_id":"job-4282","job_issued_at_ms":999989,"miner_id":"miner-3","network_difficulty":1000015,"ntime":1700004282,"order_key":"03:miner-3","p2mr_program_hex":"babababababababababababababababababababababababababababababababa","share_difficulty":19,"share_id":"share-4282","share_seq":4282,"template_height":800133},{"accepted_at_ms":1000001,"credit_policy":null,"job_id":"job-4283","job_issued_at_ms":1000010,"miner_id":"miner-4","network_difficulty":1000016,"ntime":1700004283,"order_key":"04:miner-4","p2mr_program_hex":"bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb","share_difficulty":1,"share_id":"share-4283","share_seq":4283,"template_height":800133},{"accepted_at_ms":1000005,"credit_policy":null,"job_id":"job-4284","job_issued_at_ms":999980,"miner_id":"miner-5","network_difficulty":1000000,"ntime":1700004284,"order_key":"05:miner-5","p2mr_program_hex":"bcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbc","share_difficulty":7,"share_id":"share-4284","share_seq":4284,"template_height":800133},{"accepted_at_ms":999975,"credit_policy":null,"job_id":"job-4285","job_issued_at_ms":1000005,"miner_id":"miner-6","network_difficulty":1000001,"ntime":1700004285,"order_key":"06:miner-6","p2mr_program_hex":"bdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbd","share_difficulty":101,"share_id":"share-4285","share_seq":4285,"template_height":800133},{"accepted_at_ms":1000002,"credit_policy":null,"job_id":"job-4286","job_issued_at_ms":999983,"miner_id":"miner-7","network_difficulty":1000002,"ntime":1700004286,"order_key":"07:miner-7","p2mr_program_hex":"bebebebebebebebebebebebebebebebebebebebebebebebebebebebebebebebe","share_difficulty":1000,"share_id":"share-4286","share_seq":4286,"template_height":800133},{"accepted_at_ms":1000009,"credit_policy":null,"job_id":"job-4287","job_issued_at_ms":1000006,"miner_id":"miner-8","network_difficulty":1000003,"ntime":1700004287,"order_key":"08:miner-8","p2mr_program_hex":"bfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbf","share_difficulty":2,"share_id":"share-4287","share_seq":4287,"template_height":800133},{"accepted_at_ms":999995,"credit_policy":null,"job_id":"job-4288","job_issued_at_ms":1000003,"miner_id":"miner-9","network_difficulty":1000004,"ntime":1700004288,"order_key":"09:miner-9","p2mr_program_hex":"c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0","share_difficulty":101,"share_id":"share-4288","share_seq":4288,"template_height":800134},{"accepted_at_ms":1000001,"credit_policy":"stale-grace","job_id":"job-4289","job_issued_at_ms":999996,"miner_id":"miner-10","network_difficulty":1000005,"ntime":1700004289,"order_key":"10:miner-10","p2mr_program_hex":"c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1","share_difficulty":3,"share_id":"share-4289","share_seq":4289,"template_height":800134},{"accepted_at_ms":999997,"credit_policy":"stale-grace","job_id":"job-4290","job_issued_at_ms":1000003,"miner_id":"miner-0","network_difficulty":1000006,"ntime":1700004290,"order_key":"00:miner-0","p2mr_program_hex":"c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2","share_difficulty":7,"share_id":"share-4290","share_seq":4290,"template_height":800134},{"accepted_at_ms":999979,"credit_policy":"stale-grace","job_id":"job-4291","job_issued_at_ms":1000010,"miner_id":"miner-1","network_difficulty":1000007,"ntime":1700004291,"order_key":"01:miner-1","p2mr_program_hex":"c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3","share_difficulty":2,"share_id":"share-4291","share_seq":4291,"template_height":800134},{"accepted_at_ms":999981,"credit_policy":null,"job_id":"job-4292","job_issued_at_ms":1000002,"miner_id":"miner-2","network_difficulty":1000008,"ntime":1700004292,"order_key":"02:miner-2","p2mr_program_hex":"c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4","share_difficulty":2,"share_id":"share-4292","share_seq":4292,"template_height":800134},{"accepted_at_ms":1000002,"credit_policy":null,"job_id":"job-4293","job_issued_at_ms":999979,"miner_id":"miner-3","network_difficulty":1000009,"ntime":1700004293,"order_key":"03:miner-3","p2mr_program_hex":"c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5","share_difficulty":2,"share_id":"share-4293","share_seq":4293,"template_height":800134},{"accepted_at_ms":999972,"credit_policy":null,"job_id":"job-4294","job_issued_at_ms":1000004,"miner_id":"miner-4","network_difficulty":1000010,"ntime":1700004294,"order_key":"04:miner-4","p2mr_program_hex":"c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6","share_difficulty":1,"share_id":"share-4294","share_seq":4294,"template_height":800134},{"accepted_at_ms":1000001,"credit_policy":"stale-grace","job_id":"job-4295","job_issued_at_ms":1000005,"miner_id":"miner-5","network_difficulty":1000011,"ntime":1700004295,"order_key":"05:miner-5","p2mr_program_hex":"c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7","share_difficulty":7,"share_id":"share-4295","share_seq":4295,"template_height":800134},{"accepted_at_ms":999984,"credit_policy":null,"job_id":"job-4296","job_issued_at_ms":1000004,"miner_id":"miner-6","network_difficulty":1000012,"ntime":1700004296,"order_key":"06:miner-6","p2mr_program_hex":"c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8","share_difficulty":3,"share_id":"share-4296","share_seq":4296,"template_height":800134},{"accepted_at_ms":1000009,"credit_policy":"stale-grace","job_id":"job-4297","job_issued_at_ms":999982,"miner_id":"miner-7","network_difficulty":1000013,"ntime":1700004297,"order_key":"07:miner-7","p2mr_program_hex":"c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9","share_difficulty":7,"share_id":"share-4297","share_seq":4297,"template_height":800134},{"accepted_at_ms":1000002,"credit_policy":null,"job_id":"job-4298","job_issued_at_ms":999981,"miner_id":"miner-8","network_difficulty":1000014,"ntime":1700004298,"order_key":"08:miner-8","p2mr_program_hex":"cacacacacacacacacacacacacacacacacacacacacacacacacacacacacacacaca","share_difficulty":1000,"share_id":"share-4298","share_seq":4298,"template_height":800134},{"accepted_at_ms":999996,"credit_policy":null,"job_id":"job-4299","job_issued_at_ms":1000004,"miner_id":"miner-9","network_difficulty":1000015,"ntime":1700004299,"order_key":"09:miner-9","p2mr_program_hex":"cbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcb","share_difficulty":19,"share_id":"share-4299","share_seq":4299,"template_height":800134},{"accepted_at_ms":1000010,"credit_policy":null,"job_id":"job-4300","job_issued_at_ms":1000003,"miner_id":"miner-10","network_difficulty":1000016,"ntime":1700004300,"order_key":"10:miner-10","p2mr_program_hex":"cccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccc","share_difficulty":3,"share_id":"share-4300","share_seq":4300,"template_height":800134},{"accepted_at_ms":1000006,"credit_policy":null,"job_id":"job-4301","job_issued_at_ms":1000003,"miner_id":"miner-0","network_difficulty":1000000,"ntime":1700004301,"order_key":"00:miner-0","p2mr_program_hex":"cdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcd","share_difficulty":3,"share_id":"share-4301","share_seq":4301,"template_height":800134},{"accepted_at_ms":999974,"credit_policy":"stale-grace","job_id":"job-4302","job_issued_at_ms":1000003,"miner_id":"miner-1","network_difficulty":1000001,"ntime":1700004302,"order_key":"01:miner-1","p2mr_program_hex":"cececececececececececececececececececececececececececececececece","share_difficulty":19,"share_id":"share-4302","share_seq":4302,"template_height":800134},{"accepted_at_ms":1000006,"credit_policy":"stale-grace","job_id":"job-4303","job_issued_at_ms":1000004,"miner_id":"miner-2","network_difficulty":1000002,"ntime":1700004303,"order_key":"02:miner-2","p2mr_program_hex":"cfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcf","share_difficulty":1,"share_id":"share-4303","share_seq":4303,"template_height":800134},{"accepted_at_ms":1000002,"credit_policy":"stale-grace","job_id":"job-4304","job_issued_at_ms":1000009,"miner_id":"miner-3","network_difficulty":1000003,"ntime":1700004304,"order_key":"03:miner-3","p2mr_program_hex":"d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0","share_difficulty":3,"share_id":"share-4304","share_seq":4304,"template_height":800134},{"accepted_at_ms":999971,"credit_policy":null,"job_id":"job-4305","job_issued_at_ms":1000005,"miner_id":"miner-4","network_difficulty":1000004,"ntime":1700004305,"order_key":"04:miner-4","p2mr_program_hex":"d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1","share_difficulty":2,"share_id":"share-4305","share_seq":4305,"template_height":800134},{"accepted_at_ms":999989,"credit_policy":null,"job_id":"job-4306","job_issued_at_ms":1000010,"miner_id":"miner-5","network_difficulty":1000005,"ntime":1700004306,"order_key":"05:miner-5","p2mr_program_hex":"d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2","share_difficulty":1,"share_id":"share-4306","share_seq":4306,"template_height":800134},{"accepted_at_ms":999981,"credit_policy":null,"job_id":"job-4307","job_issued_at_ms":1000008,"miner_id":"miner-6","network_difficulty":1000006,"ntime":1700004307,"order_key":"06:miner-6","p2mr_program_hex":"d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3","share_difficulty":19,"share_id":"share-4307","share_seq":4307,"template_height":800134},{"accepted_at_ms":999997,"credit_policy":null,"job_id":"job-4308","job_issued_at_ms":1000007,"miner_id":"miner-7","network_difficulty":1000007,"ntime":1700004308,"order_key":"07:miner-7","p2mr_program_hex":"d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4","share_difficulty":3,"share_id":"share-4308","share_seq":4308,"template_height":800134},{"accepted_at_ms":999999,"credit_policy":null,"job_id":"job-4309","job_issued_at_ms":1000010,"miner_id":"miner-8","network_difficulty":1000008,"ntime":1700004309,"order_key":"08:miner-8","p2mr_program_hex":"d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5","share_difficulty":101,"share_id":"share-4309","share_seq":4309,"template_height":800134},{"accepted_at_ms":1000006,"credit_policy":null,"job_id":"job-4310","job_issued_at_ms":1000002,"miner_id":"miner-9","network_difficulty":1000009,"ntime":1700004310,"order_key":"09:miner-9","p2mr_program_hex":"d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6","share_difficulty":3,"share_id":"share-4310","share_seq":4310,"template_height":800134},{"accepted_at_ms":1000001,"credit_policy":"stale-grace","job_id":"job-4311","job_issued_at_ms":999975,"miner_id":"miner-10","network_difficulty":1000010,"ntime":1700004311,"order_key":"10:miner-10","p2mr_program_hex":"d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7","share_difficulty":7,"share_id":"share-4311","share_seq":4311,"template_height":800134},{"accepted_at_ms":1000004,"credit_policy":null,"job_id":"job-4312","job_issued_at_ms":999985,"miner_id":"miner-0","network_difficulty":1000011,"ntime":1700004312,"order_key":"00:miner-0","p2mr_program_hex":"d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8","share_difficulty":19,"share_id":"share-4312","share_seq":4312,"template_height":800134},{"accepted_at_ms":999991,"credit_policy":null,"job_id":"job-4313","job_issued_at_ms":1000008,"miner_id":"miner-1","network_difficulty":1000012,"ntime":1700004313,"order_key":"01:miner-1","p2mr_program_hex":"d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9","share_difficulty":1000,"share_id":"share-4313","share_seq":4313,"template_height":800134},{"accepted_at_ms":999970,"credit_policy":"stale-grace","job_id":"job-4314","job_issued_at_ms":1000001,"miner_id":"miner-2","network_difficulty":1000013,"ntime":1700004314,"order_key":"02:miner-2","p2mr_program_hex":"dadadadadadadadadadadadadadadadadadadadadadadadadadadadadadadada","share_difficulty":3,"share_id":"share-4314","share_seq":4314,"template_height":800134},{"accepted_at_ms":999973,"credit_policy":"stale-grace","job_id":"job-4315","job_issued_at_ms":1000009,"miner_id":"miner-3","network_difficulty":1000014,"ntime":1700004315,"order_key":"03:miner-3","p2mr_program_hex":"dbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdb","share_difficulty":101,"share_id":"share-4315","share_seq":4315,"template_height":800134},{"accepted_at_ms":999987,"credit_policy":null,"job_id":"job-4316","job_issued_at_ms":1000002,"miner_id":"miner-4","network_difficulty":1000015,"ntime":1700004316,"order_key":"04:miner-4","p2mr_program_hex":"dcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdc","share_difficulty":2,"share_id":"share-4316","share_seq":4316,"template_height":800134},{"accepted_at_ms":999976,"credit_policy":null,"job_id":"job-4317","job_issued_at_ms":1000005,"miner_id":"miner-5","network_difficulty":1000016,"ntime":1700004317,"order_key":"05:miner-5","p2mr_program_hex":"dddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddd","share_difficulty":3,"share_id":"share-4317","share_seq":4317,"template_height":800134},{"accepted_at_ms":1000006,"credit_policy":null,"job_id":"job-4318","job_issued_at_ms":1000000,"miner_id":"miner-6","network_difficulty":1000000,"ntime":1700004318,"order_key":"06:miner-6","p2mr_program_hex":"dededededededededededededededededededededededededededededededede","share_difficulty":1,"share_id":"share-4318","share_seq":4318,"template_height":800134},{"accepted_at_ms":999993,"credit_policy":null,"job_id":"job-4319","job_issued_at_ms":1000008,"miner_id":"miner-7","network_difficulty":1000001,"ntime":1700004319,"order_key":"07:miner-7","p2mr_program_hex":"dfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdf","share_difficulty":7,"share_id":"share-4319","share_seq":4319,"template_height":800134},{"accepted_at_ms":1000007,"credit_policy":null,"job_id":"job-4320","job_issued_at_ms":999993,"miner_id":"miner-8","network_difficulty":1000002,"ntime":1700004320,"order_key":"08:miner-8","p2mr_program_hex":"e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0","share_difficulty":7,"share_id":"share-4320","share_seq":4320,"template_height":800135},{"accepted_at_ms":999988,"credit_policy":null,"job_id":"job-4321","job_issued_at_ms":1000001,"miner_id":"miner-9","network_difficulty":1000003,"ntime":1700004321,"order_key":"09:miner-9","p2mr_program_hex":"e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1","share_difficulty":3,"share_id":"share-4321","share_seq":4321,"template_height":800135},{"accepted_at_ms":999984,"credit_policy":null,"job_id":"job-4322","job_issued_at_ms":1000002,"miner_id":"miner-10","network_difficulty":1000004,"ntime":1700004322,"order_key":"10:miner-10","p2mr_program_hex":"e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2","share_difficulty":101,"share_id":"share-4322","share_seq":4322,"template_height":800135},{"accepted_at_ms":999997,"credit_policy":null,"job_id":"job-4323","job_issued_at_ms":1000006,"miner_id":"miner-0","network_difficulty":1000005,"ntime":1700004323,"order_key":"00:miner-0","p2mr_program_hex":"e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3","share_difficulty":3,"share_id":"share-4323","share_seq":4323,"template_height":800135},{"accepted_at_ms":1000009,"credit_policy":"stale-grace","job_id":"job-4324","job_issued_at_ms":1000003,"miner_id":"miner-1","network_difficulty":1000006,"ntime":1700004324,"order_key":"01:miner-1","p2mr_program_hex":"e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4","share_difficulty":3,"share_id":"share-4324","share_seq":4324,"template_height":800135},{"accepted_at_ms":999982,"credit_policy":null,"job_id":"job-4325","job_issued_at_ms":1000005,"miner_id":"miner-2","network_difficulty":1000007,"ntime":1700004325,"order_key":"02:miner-2","p2mr_program_hex":"e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5","share_difficulty":3,"share_id":"share-4325","share_seq":4325,"template_height":800135},{"accepted_at_ms":1000009,"credit_policy":null,"job_id":"job-4326","job_issued_at_ms":999999,"miner_id":"miner-3","network_difficulty":1000008,"ntime":1700004326,"order_key":"03:miner-3","p2mr_program_hex":"e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6","share_difficulty":7,"share_id":"share-4326","share_seq":4326,"template_height":800135},{"accepted_at_ms":1000002,"credit_policy":"stale-grace","job_id":"job-4327","job_issued_at_ms":999977,"miner_id":"miner-4","network_difficulty":1000009,"ntime":1700004327,"order_key":"04:miner-4","p2mr_program_hex":"e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7","share_difficulty":101,"share_id":"share-4327","share_seq":4327,"template_height":800135},{"accepted_at_ms":1000002,"credit_policy":null,"job_id":"job-4328","job_issued_at_ms":999985,"miner_id":"miner-5","network_difficulty":1000010,"ntime":1700004328,"order_key":"05:miner-5","p2mr_program_hex":"e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8","share_difficulty":1000,"share_id":"share-4328","share_seq":4328,"template_height":800135},{"accepted_at_ms":999991,"credit_policy":null,"job_id":"job-4329","job_issued_at_ms":1000009,"miner_id":"miner-6","network_difficulty":1000011,"ntime":1700004329,"order_key":"06:miner-6","p2mr_program_hex":"e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9","share_difficulty":7,"share_id":"share-4329","share_seq":4329,"template_height":800135},{"accepted_at_ms":999983,"credit_policy":null,"job_id":"job-4330","job_issued_at_ms":1000009,"miner_id":"miner-7","network_difficulty":1000012,"ntime":1700004330,"order_key":"07:miner-7","p2mr_program_hex":"eaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaea","share_difficulty":3,"share_id":"share-4330","share_seq":4330,"template_height":800135},{"accepted_at_ms":999997,"credit_policy":null,"job_id":"job-4331","job_issued_at_ms":1000001,"miner_id":"miner-8","network_difficulty":1000013,"ntime":1700004331,"order_key":"08:miner-8","p2mr_program_hex":"ebebebebebebebebebebebebebebebebebebebebebebebebebebebebebebebeb","share_difficulty":7,"share_id":"share-4331","share_seq":4331,"template_height":800135},{"accepted_at_ms":1000001,"credit_policy":"stale-grace","job_id":"job-4332","job_issued_at_ms":999970,"miner_id":"miner-9","network_difficulty":1000014,"ntime":1700004332,"order_key":"09:miner-9","p2mr_program_hex":"ecececececececececececececececececececececececececececececececec","share_difficulty":19,"share_id":"share-4332","share_seq":4332,"template_height":800135},{"accepted_at_ms":1000010,"credit_policy":null,"job_id":"job-4333","job_issued_at_ms":999977,"miner_id":"miner-10","network_difficulty":1000015,"ntime":1700004333,"order_key":"10:miner-10","p2mr_program_hex":"edededededededededededededededededededededededededededededededed","share_difficulty":1000,"share_id":"share-4333","share_seq":4333,"template_height":800135},{"accepted_at_ms":999985,"credit_policy":null,"job_id":"job-4334","job_issued_at_ms":1000006,"miner_id":"miner-0","network_difficulty":1000016,"ntime":1700004334,"order_key":"00:miner-0","p2mr_program_hex":"eeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeee","share_difficulty":2,"share_id":"share-4334","share_seq":4334,"template_height":800135},{"accepted_at_ms":999986,"credit_policy":null,"job_id":"job-4335","job_issued_at_ms":1000005,"miner_id":"miner-1","network_difficulty":1000000,"ntime":1700004335,"order_key":"01:miner-1","p2mr_program_hex":"efefefefefefefefefefefefefefefefefefefefefefefefefefefefefefefef","share_difficulty":101,"share_id":"share-4335","share_seq":4335,"template_height":800135},{"accepted_at_ms":999999,"credit_policy":null,"job_id":"job-4336","job_issued_at_ms":1000002,"miner_id":"miner-2","network_difficulty":1000001,"ntime":1700004336,"order_key":"02:miner-2","p2mr_program_hex":"f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0","share_difficulty":19,"share_id":"share-4336","share_seq":4336,"template_height":800135},{"accepted_at_ms":999975,"credit_policy":null,"job_id":"job-4337","job_issued_at_ms":1000007,"miner_id":"miner-3","network_difficulty":1000002,"ntime":1700004337,"order_key":"03:miner-3","p2mr_program_hex":"f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1","share_difficulty":3,"share_id":"share-4337","share_seq":4337,"template_height":800135},{"accepted_at_ms":1000008,"credit_policy":"stale-grace","job_id":"job-4338","job_issued_at_ms":1000002,"miner_id":"miner-4","network_difficulty":1000003,"ntime":1700004338,"order_key":"04:miner-4","p2mr_program_hex":"f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2","share_difficulty":1000,"share_id":"share-4338","share_seq":4338,"template_height":800135},{"accepted_at_ms":1000001,"credit_policy":null,"job_id":"job-4339","job_issued_at_ms":1000003,"miner_id":"miner-5","network_difficulty":1000004,"ntime":1700004339,"order_key":"05:miner-5","p2mr_program_hex":"f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3","share_difficulty":3,"share_id":"share-4339","share_seq":4339,"template_height":800135},{"accepted_at_ms":1000002,"credit_policy":null,"job_id":"job-4340","job_issued_at_ms":999997,"miner_id":"miner-6","network_difficulty":1000005,"ntime":1700004340,"order_key":"06:miner-6","p2mr_program_hex":"f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4","share_difficulty":1,"share_id":"share-4340","share_seq":4340,"template_height":800135},{"accepted_at_ms":1000001,"credit_policy":null,"job_id":"job-4341","job_issued_at_ms":1000001,"miner_id":"miner-7","network_difficulty":1000006,"ntime":1700004341,"order_key":"07:miner-7","p2mr_program_hex":"f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5","share_difficulty":19,"share_id":"share-4341","share_seq":4341,"template_height":800135},{"accepted_at_ms":1000004,"credit_policy":null,"job_id":"job-4342","job_issued_at_ms":999993,"miner_id":"miner-8","network_difficulty":1000007,"ntime":1700004342,"order_key":"08:miner-8","p2mr_program_hex":"f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6","share_difficulty":1,"share_id":"share-4342","share_seq":4342,"template_height":800135},{"accepted_at_ms":1000005,"credit_policy":"stale-grace","job_id":"job-4343","job_issued_at_ms":1000004,"miner_id":"miner-9","network_difficulty":1000008,"ntime":1700004343,"order_key":"09:miner-9","p2mr_program_hex":"f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7","share_difficulty":1000,"share_id":"share-4343","share_seq":4343,"template_height":800135},{"accepted_at_ms":1000005,"credit_policy":null,"job_id":"job-4344","job_issued_at_ms":999979,"miner_id":"miner-10","network_difficulty":1000009,"ntime":1700004344,"order_key":"10:miner-10","p2mr_program_hex":"f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8","share_difficulty":2,"share_id":"share-4344","share_seq":4344,"template_height":800135},{"accepted_at_ms":1000002,"credit_policy":null,"job_id":"job-4345","job_issued_at_ms":1000000,"miner_id":"miner-0","network_difficulty":1000010,"ntime":1700004345,"order_key":"00:miner-0","p2mr_program_hex":"f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9","share_difficulty":7,"share_id":"share-4345","share_seq":4345,"template_height":800135},{"accepted_at_ms":999978,"credit_policy":null,"job_id":"job-4346","job_issued_at_ms":1000010,"miner_id":"miner-1","network_difficulty":1000011,"ntime":1700004346,"order_key":"01:miner-1","p2mr_program_hex":"fafafafafafafafafafafafafafafafafafafafafafafafafafafafafafafafa","share_difficulty":1000,"share_id":"share-4346","share_seq":4346,"template_height":800135},{"accepted_at_ms":999985,"credit_policy":null,"job_id":"job-4347","job_issued_at_ms":1000006,"miner_id":"miner-2","network_difficulty":1000012,"ntime":1700004347,"order_key":"02:miner-2","p2mr_program_hex":"fbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfb","share_difficulty":7,"share_id":"share-4347","share_seq":4347,"template_height":800135},{"accepted_at_ms":999988,"credit_policy":null,"job_id":"job-4348","job_issued_at_ms":1000003,"miner_id":"miner-3","network_difficulty":1000013,"ntime":1700004348,"order_key":"03:miner-3","p2mr_program_hex":"fcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfc","share_difficulty":7,"share_id":"share-4348","share_seq":4348,"template_height":800135},{"accepted_at_ms":1000006,"credit_policy":null,"job_id":"job-4349","job_issued_at_ms":1000004,"miner_id":"miner-4","network_difficulty":1000014,"ntime":1700004349,"order_key":"04:miner-4","p2mr_program_hex":"fdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfd","share_difficulty":101,"share_id":"share-4349","share_seq":4349,"template_height":800135},{"accepted_at_ms":1000008,"credit_policy":null,"job_id":"job-4350","job_issued_at_ms":999988,"miner_id":"miner-5","network_difficulty":1000015,"ntime":1700004350,"order_key":"05:miner-5","p2mr_program_hex":"fefefefefefefefefefefefefefefefefefefefefefefefefefefefefefefefe","share_difficulty":7,"share_id":"share-4350","share_seq":4350,"template_height":800135},{"accepted_at_ms":1000001,"credit_policy":null,"job_id":"job-4351","job_issued_at_ms":1000004,"miner_id":"miner-6","network_difficulty":1000016,"ntime":1700004351,"order_key":"06:miner-6","p2mr_program_hex":"ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff","share_difficulty":101,"share_id":"share-4351","share_seq":4351,"template_height":800135},{"accepted_at_ms":1000010,"credit_policy":null,"job_id":"job-4352","job_issued_at_ms":999970,"miner_id":"miner-7","network_difficulty":1000000,"ntime":1700004352,"order_key":"07:miner-7","p2mr_program_hex":"0000000000000000000000000000000000000000000000000000000000000000","share_difficulty":1,"share_id":"share-4352","share_seq":4352,"template_height":800136},{"accepted_at_ms":1000006,"credit_policy":null,"job_id":"job-4353","job_issued_at_ms":1000003,"miner_id":"miner-8","network_difficulty":1000001,"ntime":1700004353,"order_key":"08:miner-8","p2mr_program_hex":"0101010101010101010101010101010101010101010101010101010101010101","share_difficulty":2,"share_id":"share-4353","share_seq":4353,"template_height":800136},{"accepted_at_ms":999991,"credit_policy":null,"job_id":"job-4354","job_issued_at_ms":1000007,"miner_id":"miner-9","network_difficulty":1000002,"ntime":1700004354,"order_key":"09:miner-9","p2mr_program_hex":"0202020202020202020202020202020202020202020202020202020202020202","share_difficulty":101,"share_id":"share-4354","share_seq":4354,"template_height":800136},{"accepted_at_ms":1000007,"credit_policy":null,"job_id":"job-4355","job_issued_at_ms":1000008,"miner_id":"miner-10","network_difficulty":1000003,"ntime":1700004355,"order_key":"10:miner-10","p2mr_program_hex":"0303030303030303030303030303030303030303030303030303030303030303","share_difficulty":1,"share_id":"share-4355","share_seq":4355,"template_height":800136},{"accepted_at_ms":1000002,"credit_policy":null,"job_id":"job-4356","job_issued_at_ms":999996,"miner_id":"miner-0","network_difficulty":1000004,"ntime":1700004356,"order_key":"00:miner-0","p2mr_program_hex":"0404040404040404040404040404040404040404040404040404040404040404","share_difficulty":7,"share_id":"share-4356","share_seq":4356,"template_height":800136},{"accepted_at_ms":1000009,"credit_policy":null,"job_id":"job-4357","job_issued_at_ms":999991,"miner_id":"miner-1","network_difficulty":1000005,"ntime":1700004357,"order_key":"01:miner-1","p2mr_program_hex":"0505050505050505050505050505050505050505050505050505050505050505","share_difficulty":1000,"share_id":"share-4357","share_seq":4357,"template_height":800136},{"accepted_at_ms":1000007,"credit_policy":"stale-grace","job_id":"job-4358","job_issued_at_ms":999984,"miner_id":"miner-2","network_difficulty":1000006,"ntime":1700004358,"order_key":"02:miner-2","p2mr_program_hex":"0606060606060606060606060606060606060606060606060606060606060606","share_difficulty":1,"share_id":"share-4358","share_seq":4358,"template_height":800136},{"accepted_at_ms":999976,"credit_policy":"stale-grace","job_id":"job-4359","job_issued_at_ms":1000008,"miner_id":"miner-3","network_difficulty":1000007,"ntime":1700004359,"order_key":"03:miner-3","p2mr_program_hex":"0707070707070707070707070707070707070707070707070707070707070707","share_difficulty":101,"share_id":"share-4359","share_seq":4359,"template_height":800136},{"accepted_at_ms":1000009,"credit_policy":null,"job_id":"job-4360","job_issued_at_ms":999982,"miner_id":"miner-4","network_difficulty":1000008,"ntime":1700004360,"order_key":"04:miner-4","p2mr_program_hex":"0808080808080808080808080808080808080808080808080808080808080808","share_difficulty":2,"share_id":"share-4360","share_seq":4360,"template_height":800136},{"accepted_at_ms":1000005,"credit_policy":"stale-grace","job_id":"job-4361","job_issued_at_ms":1000009,"miner_id":"miner-5","network_difficulty":1000009,"ntime":1700004361,"order_key":"05:miner-5","p2mr_program_hex":"0909090909090909090909090909090909090909090909090909090909090909","share_difficulty":3,"share_id":"share-4361","share_seq":4361,"template_height":800136},{"accepted_at_ms":1000010,"credit_policy":null,"job_id":"job-4362","job_issued_at_ms":1000008,"miner_id":"miner-6","network_difficulty":1000010,"ntime":1700004362,"order_key":"06:miner-6","p2mr_program_hex":"0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a","share_difficulty":1000,"share_id":"share-4362","share_seq":4362,"template_height":800136},{"accepted_at_ms":1000010,"credit_policy":"stale-grace","job_id":"job-4363","job_issued_at_ms":1000008,"miner_id":"miner-7","network_difficulty":1000011,"ntime":1700004363,"order_key":"07:miner-7","p2mr_program_hex":"0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b","share_difficulty":7,"share_id":"share-4363","share_seq":4363,"template_height":800136},{"accepted_at_ms":999972,"credit_policy":null,"job_id":"job-4364","job_issued_at_ms":1000004,"miner_id":"miner-8","network_difficulty":1000012,"ntime":1700004364,"order_key":"08:miner-8","p2mr_program_hex":"0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c","share_difficulty":3,"share_id":"share-4364","share_seq":4364,"template_height":800136},{"accepted_at_ms":999975,"credit_policy":null,"job_id":"job-4365","job_issued_at_ms":1000003,"miner_id":"miner-9","network_difficulty":1000013,"ntime":1700004365,"order_key":"09:miner-9","p2mr_program_hex":"0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d","share_difficulty":101,"share_id":"share-4365","share_seq":4365,"template_height":800136},{"accepted_at_ms":999972,"credit_policy":null,"job_id":"job-4366","job_issued_at_ms":1000006,"miner_id":"miner-10","network_difficulty":1000014,"ntime":1700004366,"order_key":"10:miner-10","p2mr_program_hex":"0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e","share_difficulty":1000,"share_id":"share-4366","share_seq":4366,"template_height":800136},{"accepted_at_ms":1000006,"credit_policy":null,"job_id":"job-4367","job_issued_at_ms":1000002,"miner_id":"miner-0","network_difficulty":1000015,"ntime":1700004367,"order_key":"00:miner-0","p2mr_program_hex":"0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f","share_difficulty":3,"share_id":"share-4367","share_seq":4367,"template_height":800136},{"accepted_at_ms":999981,"credit_policy":"stale-grace","job_id":"job-4368","job_issued_at_ms":1000003,"miner_id":"miner-1","network_difficulty":1000016,"ntime":1700004368,"order_key":"01:miner-1","p2mr_program_hex":"1010101010101010101010101010101010101010101010101010101010101010","share_difficulty":7,"share_id":"share-4368","share_seq":4368,"template_height":800136},{"accepted_at_ms":1000001,"credit_policy":null,"job_id":"job-4369","job_issued_at_ms":999980,"miner_id":"miner-2","network_difficulty":1000000,"ntime":1700004369,"order_key":"02:miner-2","p2mr_program_hex":"1111111111111111111111111111111111111111111111111111111111111111","share_difficulty":2,"share_id":"share-4369","share_seq":4369,"template_height":800136},{"accepted_at_ms":999977,"credit_policy":null,"job_id":"job-4370","job_issued_at_ms":1000004,"miner_id":"miner-3","network_difficulty":1000001,"ntime":1700004370,"order_key":"03:miner-3","p2mr_program_hex":"1212121212121212121212121212121212121212121212121212121212121212","share_difficulty":101,"share_id":"share-4370","share_seq":4370,"template_height":800136},{"accepted_at_ms":1000001,"credit_policy":"stale-grace","job_id":"job-4371","job_issued_at_ms":999990,"miner_id":"miner-4","network_difficulty":1000002,"ntime":1700004371,"order_key":"04:miner-4","p2mr_program_hex":"1313131313131313131313131313131313131313131313131313131313131313","share_difficulty":101,"share_id":"share-4371","share_seq":4371,"template_height":800136},{"accepted_at_ms":999985,"credit_policy":"stale-grace","job_id":"job-4372","job_issued_at_ms":1000007,"miner_id":"miner-5","network_difficulty":1000003,"ntime":1700004372,"order_key":"05:miner-5","p2mr_program_hex":"1414141414141414141414141414141414141414141414141414141414141414","share_difficulty":7,"share_id":"share-4372","share_seq":4372,"template_height":800136},{"accepted_at_ms":1000007,"credit_policy":null,"job_id":"job-4373","job_issued_at_ms":1000000,"miner_id":"miner-6","network_difficulty":1000004,"ntime":1700004373,"order_key":"06:miner-6","p2mr_program_hex":"1515151515151515151515151515151515151515151515151515151515151515","share_difficulty":1000,"share_id":"share-4373","share_seq":4373,"template_height":800136},{"accepted_at_ms":1000006,"credit_policy":null,"job_id":"job-4374","job_issued_at_ms":999977,"miner_id":"miner-7","network_difficulty":1000005,"ntime":1700004374,"order_key":"07:miner-7","p2mr_program_hex":"1616161616161616161616161616161616161616161616161616161616161616","share_difficulty":2,"share_id":"share-4374","share_seq":4374,"template_height":800136},{"accepted_at_ms":1000001,"credit_policy":null,"job_id":"job-4375","job_issued_at_ms":999984,"miner_id":"miner-8","network_difficulty":1000006,"ntime":1700004375,"order_key":"08:miner-8","p2mr_program_hex":"1717171717171717171717171717171717171717171717171717171717171717","share_difficulty":7,"share_id":"share-4375","share_seq":4375,"template_height":800136},{"accepted_at_ms":1000006,"credit_policy":null,"job_id":"job-4376","job_issued_at_ms":999989,"miner_id":"miner-9","network_difficulty":1000007,"ntime":1700004376,"order_key":"09:miner-9","p2mr_program_hex":"1818181818181818181818181818181818181818181818181818181818181818","share_difficulty":2,"share_id":"share-4376","share_seq":4376,"template_height":800136},{"accepted_at_ms":1000008,"credit_policy":"stale-grace","job_id":"job-4377","job_issued_at_ms":999971,"miner_id":"miner-10","network_difficulty":1000008,"ntime":1700004377,"order_key":"10:miner-10","p2mr_program_hex":"1919191919191919191919191919191919191919191919191919191919191919","share_difficulty":7,"share_id":"share-4377","share_seq":4377,"template_height":800136},{"accepted_at_ms":999989,"credit_policy":null,"job_id":"job-4378","job_issued_at_ms":1000005,"miner_id":"miner-0","network_difficulty":1000009,"ntime":1700004378,"order_key":"00:miner-0","p2mr_program_hex":"1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a","share_difficulty":7,"share_id":"share-4378","share_seq":4378,"template_height":800136},{"accepted_at_ms":1000009,"credit_policy":null,"job_id":"job-4379","job_issued_at_ms":1000006,"miner_id":"miner-1","network_difficulty":1000010,"ntime":1700004379,"order_key":"01:miner-1","p2mr_program_hex":"1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b","share_difficulty":101,"share_id":"share-4379","share_seq":4379,"template_height":800136},{"accepted_at_ms":1000003,"credit_policy":null,"job_id":"job-4380","job_issued_at_ms":999986,"miner_id":"miner-2","network_difficulty":1000011,"ntime":1700004380,"order_key":"02:miner-2","p2mr_program_hex":"1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c","share_difficulty":3,"share_id":"share-4380","share_seq":4380,"template_height":800136},{"accepted_at_ms":999981,"credit_policy":"stale-grace","job_id":"job-4381","job_issued_at_ms":1000009,"miner_id":"miner-3","network_difficulty":1000012,"ntime":1700004381,"order_key":"03:miner-3","p2mr_program_hex":"1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d","share_difficulty":3,"share_id":"share-4381","share_seq":4381,"template_height":800136},{"accepted_at_ms":999996,"credit_policy":null,"job_id":"job-4382","job_issued_at_ms":1000009,"miner_id":"miner-4","network_difficulty":1000013,"ntime":1700004382,"order_key":"04:miner-4","p2mr_program_hex":"1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e","share_difficulty":3,"share_id":"share-4382","share_seq":4382,"template_height":800136},{"accepted_at_ms":1000002,"credit_policy":null,"job_id":"job-4383","job_issued_at_ms":999973,"miner_id":"miner-5","network_difficulty":1000014,"ntime":1700004383,"order_key":"05:miner-5","p2mr_program_hex":"1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f","share_difficulty":19,"share_id":"share-4383","share_seq":4383,"template_height":800136},{"accepted_at_ms":1000009,"credit_policy":null,"job_id":"job-4384","job_issued_at_ms":1000001,"miner_id":"miner-6","network_difficulty":1000015,"ntime":1700004384,"order_key":"06:miner-6","p2mr_program_hex":"2020202020202020202020202020202020202020202020202020202020202020","share_difficulty":101,"share_id":"share-4384","share_seq":4384,"template_height":800137},{"accepted_at_ms":1000009,"credit_policy":null,"job_id":"job-4385","job_issued_at_ms":999996,"miner_id":"miner-7","network_difficulty":1000016,"ntime":1700004385,"order_key":"07:miner-7","p2mr_program_hex":"2121212121212121212121212121212121212121212121212121212121212121","share_difficulty":101,"share_id":"share-4385","share_seq":4385,"template_height":800137},{"accepted_at_ms":999970,"credit_policy":null,"job_id":"job-4386","job_issued_at_ms":1000010,"miner_id":"miner-8","network_difficulty":1000000,"ntime":1700004386,"order_key":"08:miner-8","p2mr_program_hex":"2222222222222222222222222222222222222222222222222222222222222222","share_difficulty":19,"share_id":"share-4386","share_seq":4386,"template_height":800137},{"accepted_at_ms":1000005,"credit_policy":null,"job_id":"job-4387","job_issued_at_ms":1000004,"miner_id":"miner-9","network_difficulty":1000001,"ntime":1700004387,"order_key":"09:miner-9","p2mr_program_hex":"2323232323232323232323232323232323232323232323232323232323232323","share_difficulty":1000,"share_id":"share-4387","share_seq":4387,"template_height":800137},{"accepted_at_ms":999989,"credit_policy":null,"job_id":"job-4388","job_issued_at_ms":1000004,"miner_id":"miner-10","network_difficulty":1000002,"ntime":1700004388,"order_key":"10:miner-10","p2mr_program_hex":"2424242424242424242424242424242424242424242424242424242424242424","share_difficulty":1000,"share_id":"share-4388","share_seq":4388,"template_height":800137},{"accepted_at_ms":999983,"credit_policy":null,"job_id":"job-4389","job_issued_at_ms":1000009,"miner_id":"miner-0","network_difficulty":1000003,"ntime":1700004389,"order_key":"00:miner-0","p2mr_program_hex":"2525252525252525252525252525252525252525252525252525252525252525","share_difficulty":1,"share_id":"share-4389","share_seq":4389,"template_height":800137},{"accepted_at_ms":999971,"credit_policy":null,"job_id":"job-4390","job_issued_at_ms":1000008,"miner_id":"miner-1","network_difficulty":1000004,"ntime":1700004390,"order_key":"01:miner-1","p2mr_program_hex":"2626262626262626262626262626262626262626262626262626262626262626","share_difficulty":1,"share_id":"share-4390","share_seq":4390,"template_height":800137},{"accepted_at_ms":999989,"credit_policy":null,"job_id":"job-4391","job_issued_at_ms":1000010,"miner_id":"miner-2","network_difficulty":1000005,"ntime":1700004391,"order_key":"02:miner-2","p2mr_program_hex":"2727272727272727272727272727272727272727272727272727272727272727","share_difficulty":19,"share_id":"share-4391","share_seq":4391,"template_height":800137},{"accepted_at_ms":999981,"credit_policy":"stale-grace","job_id":"job-4392","job_issued_at_ms":1000003,"miner_id":"miner-3","network_difficulty":1000006,"ntime":1700004392,"order_key":"03:miner-3","p2mr_program_hex":"2828282828282828282828282828282828282828282828282828282828282828","share_difficulty":3,"share_id":"share-4392","share_seq":4392,"template_height":800137},{"accepted_at_ms":1000007,"credit_policy":null,"job_id":"job-4393","job_issued_at_ms":1000004,"miner_id":"miner-4","network_difficulty":1000007,"ntime":1700004393,"order_key":"04:miner-4","p2mr_program_hex":"2929292929292929292929292929292929292929292929292929292929292929","share_difficulty":2,"share_id":"share-4393","share_seq":4393,"template_height":800137},{"accepted_at_ms":1000007,"credit_policy":"stale-grace","job_id":"job-4394","job_issued_at_ms":1000002,"miner_id":"miner-5","network_difficulty":1000008,"ntime":1700004394,"order_key":"05:miner-5","p2mr_program_hex":"2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a","share_difficulty":19,"share_id":"share-4394","share_seq":4394,"template_height":800137},{"accepted_at_ms":999974,"credit_policy":null,"job_id":"job-4395","job_issued_at_ms":1000006,"miner_id":"miner-6","network_difficulty":1000009,"ntime":1700004395,"order_key":"06:miner-6","p2mr_program_hex":"2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b","share_difficulty":101,"share_id":"share-4395","share_seq":4395,"template_height":800137},{"accepted_at_ms":1000007,"credit_policy":null,"job_id":"job-4396","job_issued_at_ms":999973,"miner_id":"miner-7","network_difficulty":1000010,"ntime":1700004396,"order_key":"07:miner-7","p2mr_program_hex":"2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c","share_difficulty":3,"share_id":"share-4396","share_seq":4396,"template_height":800137},{"accepted_at_ms":999993,"credit_policy":null,"job_id":"job-4397","job_issued_at_ms":1000005,"miner_id":"miner-8","network_difficulty":1000011,"ntime":1700004397,"order_key":"08:miner-8","p2mr_program_hex":"2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d","share_difficulty":19,"share_id":"share-4397","share_seq":4397,"template_height":800137},{"accepted_at_ms":999984,"credit_policy":"stale-grace","job_id":"job-4398","job_issued_at_ms":1000005,"miner_id":"miner-9","network_difficulty":1000012,"ntime":1700004398,"order_key":"09:miner-9","p2mr_program_hex":"2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e","share_difficulty":101,"share_id":"share-4398","share_seq":4398,"template_height":800137},{"accepted_at_ms":1000008,"credit_policy":null,"job_id":"job-4399","job_issued_at_ms":1000006,"miner_id":"miner-10","network_difficulty":1000013,"ntime":1700004399,"order_key":"10:miner-10","p2mr_program_hex":"2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f","share_difficulty":7,"share_id":"share-4399","share_seq":4399,"template_height":800137},{"accepted_at_ms":1000002,"credit_policy":null,"job_id":"job-4400","job_issued_at_ms":999990,"miner_id":"miner-0","network_difficulty":1000014,"ntime":1700004400,"order_key":"00:miner-0","p2mr_program_hex":"3030303030303030303030303030303030303030303030303030303030303030","share_difficulty":19,"share_id":"share-4400","share_seq":4400,"template_height":800137},{"accepted_at_ms":1000004,"credit_policy":null,"job_id":"job-4401","job_issued_at_ms":999991,"miner_id":"miner-1","network_difficulty":1000015,"ntime":1700004401,"order_key":"01:miner-1","p2mr_program_hex":"3131313131313131313131313131313131313131313131313131313131313131","share_difficulty":101,"share_id":"share-4401","share_seq":4401,"template_height":800137},{"accepted_at_ms":1000005,"credit_policy":null,"job_id":"job-4402","job_issued_at_ms":999981,"miner_id":"miner-2","network_difficulty":1000016,"ntime":1700004402,"order_key":"02:miner-2","p2mr_program_hex":"3232323232323232323232323232323232323232323232323232323232323232","share_difficulty":101,"share_id":"share-4402","share_seq":4402,"template_height":800137},{"accepted_at_ms":1000005,"credit_policy":null,"job_id":"job-4403","job_issued_at_ms":1000009,"miner_id":"miner-3","network_difficulty":1000000,"ntime":1700004403,"order_key":"03:miner-3","p2mr_program_hex":"3333333333333333333333333333333333333333333333333333333333333333","share_difficulty":3,"share_id":"share-4403","share_seq":4403,"template_height":800137},{"accepted_at_ms":1000008,"credit_policy":null,"job_id":"job-4404","job_issued_at_ms":999980,"miner_id":"miner-4","network_difficulty":1000001,"ntime":1700004404,"order_key":"04:miner-4","p2mr_program_hex":"3434343434343434343434343434343434343434343434343434343434343434","share_difficulty":3,"share_id":"share-4404","share_seq":4404,"template_height":800137},{"accepted_at_ms":1000006,"credit_policy":null,"job_id":"job-4405","job_issued_at_ms":1000003,"miner_id":"miner-5","network_difficulty":1000002,"ntime":1700004405,"order_key":"05:miner-5","p2mr_program_hex":"3535353535353535353535353535353535353535353535353535353535353535","share_difficulty":2,"share_id":"share-4405","share_seq":4405,"template_height":800137},{"accepted_at_ms":999985,"credit_policy":null,"job_id":"job-4406","job_issued_at_ms":1000004,"miner_id":"miner-6","network_difficulty":1000003,"ntime":1700004406,"order_key":"06:miner-6","p2mr_program_hex":"3636363636363636363636363636363636363636363636363636363636363636","share_difficulty":1,"share_id":"share-4406","share_seq":4406,"template_height":800137},{"accepted_at_ms":1000004,"credit_policy":null,"job_id":"job-4407","job_issued_at_ms":1000002,"miner_id":"miner-7","network_difficulty":1000004,"ntime":1700004407,"order_key":"07:miner-7","p2mr_program_hex":"3737373737373737373737373737373737373737373737373737373737373737","share_difficulty":3,"share_id":"share-4407","share_seq":4407,"template_height":800137},{"accepted_at_ms":999971,"credit_policy":null,"job_id":"job-4408","job_issued_at_ms":1000007,"miner_id":"miner-8","network_difficulty":1000005,"ntime":1700004408,"order_key":"08:miner-8","p2mr_program_hex":"3838383838383838383838383838383838383838383838383838383838383838","share_difficulty":1,"share_id":"share-4408","share_seq":4408,"template_height":800137},{"accepted_at_ms":999970,"credit_policy":null,"job_id":"job-4409","job_issued_at_ms":1000007,"miner_id":"miner-9","network_difficulty":1000006,"ntime":1700004409,"order_key":"09:miner-9","p2mr_program_hex":"3939393939393939393939393939393939393939393939393939393939393939","share_difficulty":3,"share_id":"share-4409","share_seq":4409,"template_height":800137},{"accepted_at_ms":1000009,"credit_policy":null,"job_id":"job-4410","job_issued_at_ms":999983,"miner_id":"miner-10","network_difficulty":1000007,"ntime":1700004410,"order_key":"10:miner-10","p2mr_program_hex":"3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a","share_difficulty":3,"share_id":"share-4410","share_seq":4410,"template_height":800137},{"accepted_at_ms":999991,"credit_policy":null,"job_id":"job-4411","job_issued_at_ms":1000002,"miner_id":"miner-0","network_difficulty":1000008,"ntime":1700004411,"order_key":"00:miner-0","p2mr_program_hex":"3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b","share_difficulty":3,"share_id":"share-4411","share_seq":4411,"template_height":800137},{"accepted_at_ms":1000009,"credit_policy":null,"job_id":"job-4412","job_issued_at_ms":1000008,"miner_id":"miner-1","network_difficulty":1000009,"ntime":1700004412,"order_key":"01:miner-1","p2mr_program_hex":"3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c","share_difficulty":101,"share_id":"share-4412","share_seq":4412,"template_height":800137},{"accepted_at_ms":1000006,"credit_policy":null,"job_id":"job-4413","job_issued_at_ms":1000007,"miner_id":"miner-2","network_difficulty":1000010,"ntime":1700004413,"order_key":"02:miner-2","p2mr_program_hex":"3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d","share_difficulty":2,"share_id":"share-4413","share_seq":4413,"template_height":800137},{"accepted_at_ms":1000002,"credit_policy":null,"job_id":"job-4414","job_issued_at_ms":999994,"miner_id":"miner-3","network_difficulty":1000011,"ntime":1700004414,"order_key":"03:miner-3","p2mr_program_hex":"3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e","share_difficulty":19,"share_id":"share-4414","share_seq":4414,"template_height":800137},{"accepted_at_ms":1000002,"credit_policy":null,"job_id":"job-4415","job_issued_at_ms":999977,"miner_id":"miner-4","network_difficulty":1000012,"ntime":1700004415,"order_key":"04:miner-4","p2mr_program_hex":"3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f","share_difficulty":7,"share_id":"share-4415","share_seq":4415,"template_height":800137},{"accepted_at_ms":999991,"credit_policy":null,"job_id":"job-4416","job_issued_at_ms":1000003,"miner_id":"miner-5","network_difficulty":1000013,"ntime":1700004416,"order_key":"05:miner-5","p2mr_program_hex":"4040404040404040404040404040404040404040404040404040404040404040","share_difficulty":1,"share_id":"share-4416","share_seq":4416,"template_height":800138},{"accepted_at_ms":1000005,"credit_policy":null,"job_id":"job-4417","job_issued_at_ms":999989,"miner_id":"miner-6","network_difficulty":1000014,"ntime":1700004417,"order_key":"06:miner-6","p2mr_program_hex":"4141414141414141414141414141414141414141414141414141414141414141","share_difficulty":1,"share_id":"share-4417","share_seq":4417,"template_height":800138},{"accepted_at_ms":1000009,"credit_policy":null,"job_id":"job-4418","job_issued_at_ms":1000005,"miner_id":"miner-7","network_difficulty":1000015,"ntime":1700004418,"order_key":"07:miner-7","p2mr_program_hex":"4242424242424242424242424242424242424242424242424242424242424242","share_difficulty":101,"share_id":"share-4418","share_seq":4418,"template_height":800138},{"accepted_at_ms":1000009,"credit_policy":"stale-grace","job_id":"job-4419","job_issued_at_ms":999983,"miner_id":"miner-8","network_difficulty":1000016,"ntime":1700004419,"order_key":"08:miner-8","p2mr_program_hex":"4343434343434343434343434343434343434343434343434343434343434343","share_difficulty":1,"share_id":"share-4419","share_seq":4419,"template_height":800138},{"accepted_at_ms":1000003,"credit_policy":null,"job_id":"job-4420","job_issued_at_ms":1000006,"miner_id":"miner-9","network_difficulty":1000000,"ntime":1700004420,"order_key":"09:miner-9","p2mr_program_hex":"4444444444444444444444444444444444444444444444444444444444444444","share_difficulty":19,"share_id":"share-4420","share_seq":4420,"template_height":800138},{"accepted_at_ms":1000010,"credit_policy":null,"job_id":"job-4421","job_issued_at_ms":1000000,"miner_id":"miner-10","network_difficulty":1000001,"ntime":1700004421,"order_key":"10:miner-10","p2mr_program_hex":"4545454545454545454545454545454545454545454545454545454545454545","share_difficulty":1,"share_id":"share-4421","share_seq":4421,"template_height":800138},{"accepted_at_ms":999997,"credit_policy":null,"job_id":"job-4422","job_issued_at_ms":1000010,"miner_id":"miner-0","network_difficulty":1000002,"ntime":1700004422,"order_key":"00:miner-0","p2mr_program_hex":"4646464646464646464646464646464646464646464646464646464646464646","share_difficulty":2,"share_id":"share-4422","share_seq":4422,"template_height":800138},{"accepted_at_ms":1000006,"credit_policy":null,"job_id":"job-4423","job_issued_at_ms":999979,"miner_id":"miner-1","network_difficulty":1000003,"ntime":1700004423,"order_key":"01:miner-1","p2mr_program_hex":"4747474747474747474747474747474747474747474747474747474747474747","share_difficulty":1,"share_id":"share-4423","share_seq":4423,"template_height":800138},{"accepted_at_ms":1000005,"credit_policy":null,"job_id":"job-4424","job_issued_at_ms":1000004,"miner_id":"miner-2","network_difficulty":1000004,"ntime":1700004424,"order_key":"02:miner-2","p2mr_program_hex":"4848484848484848484848484848484848484848484848484848484848484848","share_difficulty":7,"share_id":"share-4424","share_seq":4424,"template_height":800138},{"accepted_at_ms":1000003,"credit_policy":null,"job_id":"job-4425","job_issued_at_ms":999996,"miner_id":"miner-3","network_difficulty":1000005,"ntime":1700004425,"order_key":"03:miner-3","p2mr_program_hex":"4949494949494949494949494949494949494949494949494949494949494949","share_difficulty":19,"share_id":"share-4425","share_seq":4425,"template_height":800138},{"accepted_at_ms":1000005,"credit_policy":null,"job_id":"job-4426","job_issued_at_ms":999985,"miner_id":"miner-4","network_difficulty":1000006,"ntime":1700004426,"order_key":"04:miner-4","p2mr_program_hex":"4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a","share_difficulty":7,"share_id":"share-4426","share_seq":4426,"template_height":800138},{"accepted_at_ms":1000002,"credit_policy":"stale-grace","job_id":"job-4427","job_issued_at_ms":1000008,"miner_id":"miner-5","network_difficulty":1000007,"ntime":1700004427,"order_key":"05:miner-5","p2mr_program_hex":"4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b","share_difficulty":3,"share_id":"share-4427","share_seq":4427,"template_height":800138},{"accepted_at_ms":999989,"credit_policy":null,"job_id":"job-4428","job_issued_at_ms":1000001,"miner_id":"miner-6","network_difficulty":1000008,"ntime":1700004428,"order_key":"06:miner-6","p2mr_program_hex":"4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c","share_difficulty":1,"share_id":"share-4428","share_seq":4428,"template_height":800138},{"accepted_at_ms":999970,"credit_policy":null,"job_id":"job-4429","job_issued_at_ms":1000009,"miner_id":"miner-7","network_difficulty":1000009,"ntime":1700004429,"order_key":"07:miner-7","p2mr_program_hex":"4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d","share_difficulty":1000,"share_id":"share-4429","share_seq":4429,"template_height":800138},{"accepted_at_ms":1000007,"credit_policy":null,"job_id":"job-4430","job_issued_at_ms":999980,"miner_id":"miner-8","network_difficulty":1000010,"ntime":1700004430,"order_key":"08:miner-8","p2mr_program_hex":"4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e","share_difficulty":101,"share_id":"share-4430","share_seq":4430,"template_height":800138},{"accepted_at_ms":1000001,"credit_policy":null,"job_id":"job-4431","job_issued_at_ms":999979,"miner_id":"miner-9","network_difficulty":1000011,"ntime":1700004431,"order_key":"09:miner-9","p2mr_program_hex":"4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f","share_difficulty":7,"share_id":"share-4431","share_seq":4431,"template_height":800138},{"accepted_at_ms":999971,"credit_policy":"stale-grace","job_id":"job-4432","job_issued_at_ms":1000007,"miner_id":"miner-10","network_difficulty":1000012,"ntime":1700004432,"order_key":"10:miner-10","p2mr_program_hex":"5050505050505050505050505050505050505050505050505050505050505050","share_difficulty":3,"share_id":"share-4432","share_seq":4432,"template_height":800138},{"accepted_at_ms":1000007,"credit_policy":null,"job_id":"job-4433","job_issued_at_ms":999972,"miner_id":"miner-0","network_difficulty":1000013,"ntime":1700004433,"order_key":"00:miner-0","p2mr_program_hex":"5151515151515151515151515151515151515151515151515151515151515151","share_difficulty":19,"share_id":"share-4433","share_seq":4433,"template_height":800138},{"accepted_at_ms":1000008,"credit_policy":"stale-grace","job_id":"job-4434","job_issued_at_ms":999995,"miner_id":"miner-1","network_difficulty":1000014,"ntime":1700004434,"order_key":"01:miner-1","p2mr_program_hex":"5252525252525252525252525252525252525252525252525252525252525252","share_difficulty":7,"share_id":"share-4434","share_seq":4434,"template_height":800138},{"accepted_at_ms":999973,"credit_policy":null,"job_id":"job-4435","job_issued_at_ms":1000010,"miner_id":"miner-2","network_difficulty":1000015,"ntime":1700004435,"order_key":"02:miner-2","p2mr_program_hex":"5353535353535353535353535353535353535353535353535353535353535353","share_difficulty":101,"share_id":"share-4435","share_seq":4435,"template_height":800138},{"accepted_at_ms":999996,"credit_policy":null,"job_id":"job-4436","job_issued_at_ms":1000008,"miner_id":"miner-3","network_difficulty":1000016,"ntime":1700004436,"order_key":"03:miner-3","p2mr_program_hex":"5454545454545454545454545454545454545454545454545454545454545454","share_difficulty":2,"share_id":"share-4436","share_seq":4436,"template_height":800138},{"accepted_at_ms":999978,"credit_policy":"stale-grace","job_id":"job-4437","job_issued_at_ms":1000009,"miner_id":"miner-4","network_difficulty":1000000,"ntime":1700004437,"order_key":"04:miner-4","p2mr_program_hex":"5555555555555555555555555555555555555555555555555555555555555555","share_difficulty":3,"share_id":"share-4437","share_seq":4437,"template_height":800138},{"accepted_at_ms":999975,"credit_policy":null,"job_id":"job-4438","job_issued_at_ms":1000005,"miner_id":"miner-5","network_difficulty":1000001,"ntime":1700004438,"order_key":"05:miner-5","p2mr_program_hex":"5656565656565656565656565656565656565656565656565656565656565656","share_difficulty":2,"share_id":"share-4438","share_seq":4438,"template_height":800138},{"accepted_at_ms":999985,"credit_policy":"stale-grace","job_id":"job-4439","job_issued_at_ms":1000006,"miner_id":"miner-6","network_difficulty":1000002,"ntime":1700004439,"order_key":"06:miner-6","p2mr_program_hex":"5757575757575757575757575757575757575757575757575757575757575757","share_difficulty":3,"share_id":"share-4439","share_seq":4439,"template_height":800138},{"accepted_at_ms":1000010,"credit_policy":null,"job_id":"job-4440","job_issued_at_ms":1000004,"miner_id":"miner-7","network_difficulty":1000003,"ntime":1700004440,"order_key":"07:miner-7","p2mr_program_hex":"5858585858585858585858585858585858585858585858585858585858585858","share_difficulty":3,"share_id":"share-4440","share_seq":4440,"template_height":800138},{"accepted_at_ms":1000000,"credit_policy":"stale-grace","job_id":"job-4441","job_issued_at_ms":1000010,"miner_id":"miner-8","network_difficulty":1000004,"ntime":1700004441,"order_key":"08:miner-8","p2mr_program_hex":"5959595959595959595959595959595959595959595959595959595959595959","share_difficulty":7,"share_id":"share-4441","share_seq":4441,"template_height":800138},{"accepted_at_ms":1000001,"credit_policy":null,"job_id":"job-4442","job_issued_at_ms":999992,"miner_id":"miner-9","network_difficulty":1000005,"ntime":1700004442,"order_key":"09:miner-9","p2mr_program_hex":"5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a","share_difficulty":19,"share_id":"share-4442","share_seq":4442,"template_height":800138},{"accepted_at_ms":1000004,"credit_policy":null,"job_id":"job-4443","job_issued_at_ms":999995,"miner_id":"miner-10","network_difficulty":1000006,"ntime":1700004443,"order_key":"10:miner-10","p2mr_program_hex":"5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b","share_difficulty":3,"share_id":"share-4443","share_seq":4443,"template_height":800138},{"accepted_at_ms":1000001,"credit_policy":null,"job_id":"job-4444","job_issued_at_ms":1000007,"miner_id":"miner-0","network_difficulty":1000007,"ntime":1700004444,"order_key":"00:miner-0","p2mr_program_hex":"5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c","share_difficulty":2,"share_id":"share-4444","share_seq":4444,"template_height":800138},{"accepted_at_ms":1000002,"credit_policy":null,"job_id":"job-4445","job_issued_at_ms":999989,"miner_id":"miner-1","network_difficulty":1000008,"ntime":1700004445,"order_key":"01:miner-1","p2mr_program_hex":"5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d","share_difficulty":2,"share_id":"share-4445","share_seq":4445,"template_height":800138},{"accepted_at_ms":1000010,"credit_policy":null,"job_id":"job-4446","job_issued_at_ms":999995,"miner_id":"miner-2","network_difficulty":1000009,"ntime":1700004446,"order_key":"02:miner-2","p2mr_program_hex":"5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e","share_difficulty":2,"share_id":"share-4446","share_seq":4446,"template_height":800138},{"accepted_at_ms":999973,"credit_policy":"stale-grace","job_id":"job-4447","job_issued_at_ms":1000007,"miner_id":"miner-3","network_difficulty":1000010,"ntime":1700004447,"order_key":"03:miner-3","p2mr_program_hex":"5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f","share_difficulty":3,"share_id":"share-4447","share_seq":4447,"template_height":800138},{"accepted_at_ms":999977,"credit_policy":null,"job_id":"job-4448","job_issued_at_ms":1000002,"miner_id":"miner-4","network_difficulty":1000011,"ntime":1700004448,"order_key":"04:miner-4","p2mr_program_hex":"6060606060606060606060606060606060606060606060606060606060606060","share_difficulty":1,"share_id":"share-4448","share_seq":4448,"template_height":800139},{"accepted_at_ms":999976,"credit_policy":null,"job_id":"job-4449","job_issued_at_ms":1000001,"miner_id":"miner-5","network_difficulty":1000012,"ntime":1700004449,"order_key":"05:miner-5","p2mr_program_hex":"6161616161616161616161616161616161616161616161616161616161616161","share_difficulty":7,"share_id":"share-4449","share_seq":4449,"template_height":800139},{"accepted_at_ms":1000002,"credit_policy":null,"job_id":"job-4450","job_issued_at_ms":1000001,"miner_id":"miner-6","network_difficulty":1000013,"ntime":1700004450,"order_key":"06:miner-6","p2mr_program_hex":"6262626262626262626262626262626262626262626262626262626262626262","share_difficulty":1000,"share_id":"share-4450","share_seq":4450,"template_height":800139},{"accepted_at_ms":999987,"credit_policy":null,"job_id":"job-4451","job_issued_at_ms":1000010,"miner_id":"miner-7","network_difficulty":1000014,"ntime":1700004451,"order_key":"07:miner-7","p2mr_program_hex":"6363636363636363636363636363636363636363636363636363636363636363","share_difficulty":19,"share_id":"share-4451","share_seq":4451,"template_height":800139},{"accepted_at_ms":1000003,"credit_policy":null,"job_id":"job-4452","job_issued_at_ms":999998,"miner_id":"miner-8","network_difficulty":1000015,"ntime":1700004452,"order_key":"08:miner-8","p2mr_program_hex":"6464646464646464646464646464646464646464646464646464646464646464","share_difficulty":7,"share_id":"share-4452","share_seq":4452,"template_height":800139},{"accepted_at_ms":999981,"credit_policy":null,"job_id":"job-4453","job_issued_at_ms":1000009,"miner_id":"miner-9","network_difficulty":1000016,"ntime":1700004453,"order_key":"09:miner-9","p2mr_program_hex":"6565656565656565656565656565656565656565656565656565656565656565","share_difficulty":1,"share_id":"share-4453","share_seq":4453,"template_height":800139},{"accepted_at_ms":1000006,"credit_policy":null,"job_id":"job-4454","job_issued_at_ms":1000001,"miner_id":"miner-10","network_difficulty":1000000,"ntime":1700004454,"order_key":"10:miner-10","p2mr_program_hex":"6666666666666666666666666666666666666666666666666666666666666666","share_difficulty":1,"share_id":"share-4454","share_seq":4454,"template_height":800139},{"accepted_at_ms":1000008,"credit_policy":null,"job_id":"job-4455","job_issued_at_ms":999977,"miner_id":"miner-0","network_difficulty":1000001,"ntime":1700004455,"order_key":"00:miner-0","p2mr_program_hex":"6767676767676767676767676767676767676767676767676767676767676767","share_difficulty":2,"share_id":"share-4455","share_seq":4455,"template_height":800139},{"accepted_at_ms":999970,"credit_policy":null,"job_id":"job-4456","job_issued_at_ms":1000009,"miner_id":"miner-1","network_difficulty":1000002,"ntime":1700004456,"order_key":"01:miner-1","p2mr_program_hex":"6868686868686868686868686868686868686868686868686868686868686868","share_difficulty":1,"share_id":"share-4456","share_seq":4456,"template_height":800139},{"accepted_at_ms":999980,"credit_policy":null,"job_id":"job-4457","job_issued_at_ms":1000004,"miner_id":"miner-2","network_difficulty":1000003,"ntime":1700004457,"order_key":"02:miner-2","p2mr_program_hex":"6969696969696969696969696969696969696969696969696969696969696969","share_difficulty":1,"share_id":"share-4457","share_seq":4457,"template_height":800139},{"accepted_at_ms":1000009,"credit_policy":null,"job_id":"job-4458","job_issued_at_ms":1000005,"miner_id":"miner-3","network_difficulty":1000004,"ntime":1700004458,"order_key":"03:miner-3","p2mr_program_hex":"6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a","share_difficulty":101,"share_id":"share-4458","share_seq":4458,"template_height":800139},{"accepted_at_ms":1000003,"credit_policy":null,"job_id":"job-4459","job_issued_at_ms":999996,"miner_id":"miner-4","network_difficulty":1000005,"ntime":1700004459,"order_key":"04:miner-4","p2mr_program_hex":"6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b","share_difficulty":19,"share_id":"share-4459","share_seq":4459,"template_height":800139},{"accepted_at_ms":1000008,"credit_policy":null,"job_id":"job-4460","job_issued_at_ms":999978,"miner_id":"miner-5","network_difficulty":1000006,"ntime":1700004460,"order_key":"05:miner-5","p2mr_program_hex":"6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c","share_difficulty":7,"share_id":"share-4460","share_seq":4460,"template_height":800139},{"accepted_at_ms":999983,"credit_policy":null,"job_id":"job-4461","job_issued_at_ms":1000002,"miner_id":"miner-6","network_difficulty":1000007,"ntime":1700004461,"order_key":"06:miner-6","p2mr_program_hex":"6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d","share_difficulty":1000,"share_id":"share-4461","share_seq":4461,"template_height":800139},{"accepted_at_ms":1000002,"credit_policy":"stale-grace","job_id":"job-4462","job_issued_at_ms":1000002,"miner_id":"miner-7","network_difficulty":1000008,"ntime":1700004462,"order_key":"07:miner-7","p2mr_program_hex":"6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e","share_difficulty":3,"share_id":"share-4462","share_seq":4462,"template_height":800139},{"accepted_at_ms":1000003,"credit_policy":null,"job_id":"job-4463","job_issued_at_ms":999996,"miner_id":"miner-8","network_difficulty":1000009,"ntime":1700004463,"order_key":"08:miner-8","p2mr_program_hex":"6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f","share_difficulty":7,"share_id":"share-4463","share_seq":4463,"template_height":800139},{"accepted_at_ms":1000005,"credit_policy":"stale-grace","job_id":"job-4464","job_issued_at_ms":1000007,"miner_id":"miner-9","network_difficulty":1000010,"ntime":1700004464,"order_key":"09:miner-9","p2mr_program_hex":"7070707070707070707070707070707070707070707070707070707070707070","share_difficulty":7,"share_id":"share-4464","share_seq":4464,"template_height":800139},{"accepted_at_ms":1000007,"credit_policy":null,"job_id":"job-4465","job_issued_at_ms":999995,"miner_id":"miner-10","network_difficulty":1000011,"ntime":1700004465,"order_key":"10:miner-10","p2mr_program_hex":"7171717171717171717171717171717171717171717171717171717171717171","share_difficulty":101,"share_id":"share-4465","share_seq":4465,"template_height":800139},{"accepted_at_ms":1000006,"credit_policy":null,"job_id":"job-4466","job_issued_at_ms":999974,"miner_id":"miner-0","network_difficulty":1000012,"ntime":1700004466,"order_key":"00:miner-0","p2mr_program_hex":"7272727272727272727272727272727272727272727272727272727272727272","share_difficulty":19,"share_id":"share-4466","share_seq":4466,"template_height":800139},{"accepted_at_ms":999975,"credit_policy":null,"job_id":"job-4467","job_issued_at_ms":1000005,"miner_id":"miner-1","network_difficulty":1000013,"ntime":1700004467,"order_key":"01:miner-1","p2mr_program_hex":"7373737373737373737373737373737373737373737373737373737373737373","share_difficulty":1000,"share_id":"share-4467","share_seq":4467,"template_height":800139},{"accepted_at_ms":999996,"credit_policy":"stale-grace","job_id":"job-4468","job_issued_at_ms":1000008,"miner_id":"miner-2","network_difficulty":1000014,"ntime":1700004468,"order_key":"02:miner-2","p2mr_program_hex":"7474747474747474747474747474747474747474747474747474747474747474","share_difficulty":101,"share_id":"share-4468","share_seq":4468,"template_height":800139},{"accepted_at_ms":999975,"credit_policy":null,"job_id":"job-4469","job_issued_at_ms":1000007,"miner_id":"miner-3","network_difficulty":1000015,"ntime":1700004469,"order_key":"03:miner-3","p2mr_program_hex":"7575757575757575757575757575757575757575757575757575757575757575","share_difficulty":7,"share_id":"share-4469","share_seq":4469,"template_height":800139},{"accepted_at_ms":999982,"credit_policy":"stale-grace","job_id":"job-4470","job_issued_at_ms":1000010,"miner_id":"miner-4","network_difficulty":1000016,"ntime":1700004470,"order_key":"04:miner-4","p2mr_program_hex":"7676767676767676767676767676767676767676767676767676767676767676","share_difficulty":3,"share_id":"share-4470","share_seq":4470,"template_height":800139},{"accepted_at_ms":999973,"credit_policy":null,"job_id":"job-4471","job_issued_at_ms":1000008,"miner_id":"miner-5","network_difficulty":1000000,"ntime":1700004471,"order_key":"05:miner-5","p2mr_program_hex":"7777777777777777777777777777777777777777777777777777777777777777","share_difficulty":101,"share_id":"share-4471","share_seq":4471,"template_height":800139},{"accepted_at_ms":999998,"credit_policy":null,"job_id":"job-4472","job_issued_at_ms":1000004,"miner_id":"miner-6","network_difficulty":1000001,"ntime":1700004472,"order_key":"06:miner-6","p2mr_program_hex":"7878787878787878787878787878787878787878787878787878787878787878","share_difficulty":101,"share_id":"share-4472","share_seq":4472,"template_height":800139},{"accepted_at_ms":999992,"credit_policy":"stale-grace","job_id":"job-4473","job_issued_at_ms":1000001,"miner_id":"miner-7","network_difficulty":1000002,"ntime":1700004473,"order_key":"07:miner-7","p2mr_program_hex":"7979797979797979797979797979797979797979797979797979797979797979","share_difficulty":2,"share_id":"share-4473","share_seq":4473,"template_height":800139},{"accepted_at_ms":1000010,"credit_policy":null,"job_id":"job-4474","job_issued_at_ms":1000006,"miner_id":"miner-8","network_difficulty":1000003,"ntime":1700004474,"order_key":"08:miner-8","p2mr_program_hex":"7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a","share_difficulty":7,"share_id":"share-4474","share_seq":4474,"template_height":800139},{"accepted_at_ms":999973,"credit_policy":null,"job_id":"job-4475","job_issued_at_ms":1000002,"miner_id":"miner-9","network_difficulty":1000004,"ntime":1700004475,"order_key":"09:miner-9","p2mr_program_hex":"7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b","share_difficulty":3,"share_id":"share-4475","share_seq":4475,"template_height":800139},{"accepted_at_ms":999993,"credit_policy":null,"job_id":"job-4476","job_issued_at_ms":1000002,"miner_id":"miner-10","network_difficulty":1000005,"ntime":1700004476,"order_key":"10:miner-10","p2mr_program_hex":"7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c","share_difficulty":19,"share_id":"share-4476","share_seq":4476,"template_height":800139},{"accepted_at_ms":1000008,"credit_policy":null,"job_id":"job-4477","job_issued_at_ms":999990,"miner_id":"miner-0","network_difficulty":1000006,"ntime":1700004477,"order_key":"00:miner-0","p2mr_program_hex":"7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d","share_difficulty":19,"share_id":"share-4477","share_seq":4477,"template_height":800139},{"accepted_at_ms":999973,"credit_policy":null,"job_id":"job-4478","job_issued_at_ms":1000010,"miner_id":"miner-1","network_difficulty":1000007,"ntime":1700004478,"order_key":"01:miner-1","p2mr_program_hex":"7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e","share_difficulty":101,"share_id":"share-4478","share_seq":4478,"template_height":800139},{"accepted_at_ms":1000007,"credit_policy":"stale-grace","job_id":"job-4479","job_issued_at_ms":1000001,"miner_id":"miner-2","network_difficulty":1000008,"ntime":1700004479,"order_key":"02:miner-2","p2mr_program_hex":"7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f","share_difficulty":1,"share_id":"share-4479","share_seq":4479,"template_height":800139},{"accepted_at_ms":1000007,"credit_policy":null,"job_id":"job-4480","job_issued_at_ms":1000008,"miner_id":"miner-3","network_difficulty":1000009,"ntime":1700004480,"order_key":"03:miner-3","p2mr_program_hex":"8080808080808080808080808080808080808080808080808080808080808080","share_difficulty":1000,"share_id":"share-4480","share_seq":4480,"template_height":800140},{"accepted_at_ms":1000007,"credit_policy":null,"job_id":"job-4481","job_issued_at_ms":999991,"miner_id":"miner-4","network_difficulty":1000010,"ntime":1700004481,"order_key":"04:miner-4","p2mr_program_hex":"8181818181818181818181818181818181818181818181818181818181818181","share_difficulty":19,"share_id":"share-4481","share_seq":4481,"template_height":800140},{"accepted_at_ms":1000008,"credit_policy":null,"job_id":"job-4482","job_issued_at_ms":1000005,"miner_id":"miner-5","network_difficulty":1000011,"ntime":1700004482,"order_key":"05:miner-5","p2mr_program_hex":"8282828282828282828282828282828282828282828282828282828282828282","share_difficulty":7,"share_id":"share-4482","share_seq":4482,"template_height":800140},{"accepted_at_ms":1000001,"credit_policy":null,"job_id":"job-4483","job_issued_at_ms":999987,"miner_id":"miner-6","network_difficulty":1000012,"ntime":1700004483,"order_key":"06:miner-6","p2mr_program_hex":"8383838383838383838383838383838383838383838383838383838383838383","share_difficulty":1000,"share_id":"share-4483","share_seq":4483,"template_height":800140},{"accepted_at_ms":1000008,"credit_policy":"stale-grace","job_id":"job-4484","job_issued_at_ms":1000006,"miner_id":"miner-7","network_difficulty":1000013,"ntime":1700004484,"order_key":"07:miner-7","p2mr_program_hex":"8484848484848484848484848484848484848484848484848484848484848484","share_difficulty":3,"share_id":"share-4484","share_seq":4484,"template_height":800140},{"accepted_at_ms":1000001,"credit_policy":null,"job_id":"job-4485","job_issued_at_ms":1000003,"miner_id":"miner-8","network_difficulty":1000014,"ntime":1700004485,"order_key":"08:miner-8","p2mr_program_hex":"8585858585858585858585858585858585858585858585858585858585858585","share_difficulty":7,"share_id":"share-4485","share_seq":4485,"template_height":800140},{"accepted_at_ms":999995,"credit_policy":null,"job_id":"job-4486","job_issued_at_ms":1000007,"miner_id":"miner-9","network_difficulty":1000015,"ntime":1700004486,"order_key":"09:miner-9","p2mr_program_hex":"8686868686868686868686868686868686868686868686868686868686868686","share_difficulty":101,"share_id":"share-4486","share_seq":4486,"template_height":800140},{"accepted_at_ms":1000008,"credit_policy":null,"job_id":"job-4487","job_issued_at_ms":999970,"miner_id":"miner-10","network_difficulty":1000016,"ntime":1700004487,"order_key":"10:miner-10","p2mr_program_hex":"8787878787878787878787878787878787878787878787878787878787878787","share_difficulty":2,"share_id":"share-4487","share_seq":4487,"template_height":800140},{"accepted_at_ms":999978,"credit_policy":null,"job_id":"job-4488","job_issued_at_ms":1000009,"miner_id":"miner-0","network_difficulty":1000000,"ntime":1700004488,"order_key":"00:miner-0","p2mr_program_hex":"8888888888888888888888888888888888888888888888888888888888888888","share_difficulty":19,"share_id":"share-4488","share_seq":4488,"template_height":800140},{"accepted_at_ms":1000004,"credit_policy":null,"job_id":"job-4489","job_issued_at_ms":1000006,"miner_id":"miner-1","network_difficulty":1000001,"ntime":1700004489,"order_key":"01:miner-1","p2mr_program_hex":"8989898989898989898989898989898989898989898989898989898989898989","share_difficulty":3,"share_id":"share-4489","share_seq":4489,"template_height":800140},{"accepted_at_ms":1000002,"credit_policy":null,"job_id":"job-4490","job_issued_at_ms":999973,"miner_id":"miner-2","network_difficulty":1000002,"ntime":1700004490,"order_key":"02:miner-2","p2mr_program_hex":"8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a","share_difficulty":19,"share_id":"share-4490","share_seq":4490,"template_height":800140},{"accepted_at_ms":1000005,"credit_policy":null,"job_id":"job-4491","job_issued_at_ms":999975,"miner_id":"miner-3","network_difficulty":1000003,"ntime":1700004491,"order_key":"03:miner-3","p2mr_program_hex":"8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b","share_difficulty":1000,"share_id":"share-4491","share_seq":4491,"template_height":800140},{"accepted_at_ms":1000003,"credit_policy":null,"job_id":"job-4492","job_issued_at_ms":999988,"miner_id":"miner-4","network_difficulty":1000004,"ntime":1700004492,"order_key":"04:miner-4","p2mr_program_hex":"8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c","share_difficulty":2,"share_id":"share-4492","share_seq":4492,"template_height":800140},{"accepted_at_ms":999999,"credit_policy":null,"job_id":"job-4493","job_issued_at_ms":1000001,"miner_id":"miner-5","network_difficulty":1000005,"ntime":1700004493,"order_key":"05:miner-5","p2mr_program_hex":"8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d","share_difficulty":1000,"share_id":"share-4493","share_seq":4493,"template_height":800140},{"accepted_at_ms":1000004,"credit_policy":null,"job_id":"job-4494","job_issued_at_ms":1000000,"miner_id":"miner-6","network_difficulty":1000006,"ntime":1700004494,"order_key":"06:miner-6","p2mr_program_hex":"8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e","share_difficulty":7,"share_id":"share-4494","share_seq":4494,"template_height":800140},{"accepted_at_ms":1000004,"credit_policy":null,"job_id":"job-4495","job_issued_at_ms":999990,"miner_id":"miner-7","network_difficulty":1000007,"ntime":1700004495,"order_key":"07:miner-7","p2mr_program_hex":"8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f","share_difficulty":1000,"share_id":"share-4495","share_seq":4495,"template_height":800140},{"accepted_at_ms":1000007,"credit_policy":null,"job_id":"job-4496","job_issued_at_ms":1000004,"miner_id":"miner-8","network_difficulty":1000008,"ntime":1700004496,"order_key":"08:miner-8","p2mr_program_hex":"9090909090909090909090909090909090909090909090909090909090909090","share_difficulty":1,"share_id":"share-4496","share_seq":4496,"template_height":800140},{"accepted_at_ms":1000008,"credit_policy":null,"job_id":"job-4497","job_issued_at_ms":1000001,"miner_id":"miner-9","network_difficulty":1000009,"ntime":1700004497,"order_key":"09:miner-9","p2mr_program_hex":"9191919191919191919191919191919191919191919191919191919191919191","share_difficulty":19,"share_id":"share-4497","share_seq":4497,"template_height":800140},{"accepted_at_ms":999996,"credit_policy":"stale-grace","job_id":"job-4498","job_issued_at_ms":1000001,"miner_id":"miner-10","network_difficulty":1000010,"ntime":1700004498,"order_key":"10:miner-10","p2mr_program_hex":"9292929292929292929292929292929292929292929292929292929292929292","share_difficulty":3,"share_id":"share-4498","share_seq":4498,"template_height":800140},{"accepted_at_ms":1000004,"credit_policy":"stale-grace","job_id":"job-4499","job_issued_at_ms":1000003,"miner_id":"miner-0","network_difficulty":1000011,"ntime":1700004499,"order_key":"00:miner-0","p2mr_program_hex":"9393939393939393939393939393939393939393939393939393939393939393","share_difficulty":2,"share_id":"share-4499","share_seq":4499,"template_height":800140},{"accepted_at_ms":999993,"credit_policy":null,"job_id":"job-4500","job_issued_at_ms":1000003,"miner_id":"miner-1","network_difficulty":1000012,"ntime":1700004500,"order_key":"01:miner-1","p2mr_program_hex":"9494949494949494949494949494949494949494949494949494949494949494","share_difficulty":3,"share_id":"share-4500","share_seq":4500,"template_height":800140},{"accepted_at_ms":1000008,"credit_policy":null,"job_id":"job-4501","job_issued_at_ms":1000009,"miner_id":"miner-2","network_difficulty":1000013,"ntime":1700004501,"order_key":"02:miner-2","p2mr_program_hex":"9595959595959595959595959595959595959595959595959595959595959595","share_difficulty":101,"share_id":"share-4501","share_seq":4501,"template_height":800140},{"accepted_at_ms":1000008,"credit_policy":null,"job_id":"job-4502","job_issued_at_ms":999977,"miner_id":"miner-3","network_difficulty":1000014,"ntime":1700004502,"order_key":"03:miner-3","p2mr_program_hex":"9696969696969696969696969696969696969696969696969696969696969696","share_difficulty":1000,"share_id":"share-4502","share_seq":4502,"template_height":800140},{"accepted_at_ms":999987,"credit_policy":null,"job_id":"job-4503","job_issued_at_ms":1000006,"miner_id":"miner-4","network_difficulty":1000015,"ntime":1700004503,"order_key":"04:miner-4","p2mr_program_hex":"9797979797979797979797979797979797979797979797979797979797979797","share_difficulty":7,"share_id":"share-4503","share_seq":4503,"template_height":800140},{"accepted_at_ms":1000005,"credit_policy":null,"job_id":"job-4504","job_issued_at_ms":999997,"miner_id":"miner-5","network_difficulty":1000016,"ntime":1700004504,"order_key":"05:miner-5","p2mr_program_hex":"9898989898989898989898989898989898989898989898989898989898989898","share_difficulty":19,"share_id":"share-4504","share_seq":4504,"template_height":800140},{"accepted_at_ms":999984,"credit_policy":"stale-grace","job_id":"job-4505","job_issued_at_ms":1000004,"miner_id":"miner-6","network_difficulty":1000000,"ntime":1700004505,"order_key":"06:miner-6","p2mr_program_hex":"9999999999999999999999999999999999999999999999999999999999999999","share_difficulty":3,"share_id":"share-4505","share_seq":4505,"template_height":800140},{"accepted_at_ms":999970,"credit_policy":"stale-grace","job_id":"job-4506","job_issued_at_ms":1000003,"miner_id":"miner-7","network_difficulty":1000001,"ntime":1700004506,"order_key":"07:miner-7","p2mr_program_hex":"9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a","share_difficulty":19,"share_id":"share-4506","share_seq":4506,"template_height":800140},{"accepted_at_ms":1000002,"credit_policy":null,"job_id":"job-4507","job_issued_at_ms":1000003,"miner_id":"miner-8","network_difficulty":1000002,"ntime":1700004507,"order_key":"08:miner-8","p2mr_program_hex":"9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b","share_difficulty":19,"share_id":"share-4507","share_seq":4507,"template_height":800140},{"accepted_at_ms":1000004,"credit_policy":"stale-grace","job_id":"job-4508","job_issued_at_ms":1000003,"miner_id":"miner-9","network_difficulty":1000003,"ntime":1700004508,"order_key":"09:miner-9","p2mr_program_hex":"9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c","share_difficulty":19,"share_id":"share-4508","share_seq":4508,"template_height":800140},{"accepted_at_ms":999984,"credit_policy":null,"job_id":"job-4509","job_issued_at_ms":1000010,"miner_id":"miner-10","network_difficulty":1000004,"ntime":1700004509,"order_key":"10:miner-10","p2mr_program_hex":"9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d","share_difficulty":2,"share_id":"share-4509","share_seq":4509,"template_height":800140},{"accepted_at_ms":1000002,"credit_policy":null,"job_id":"job-4510","job_issued_at_ms":1000009,"miner_id":"miner-0","network_difficulty":1000005,"ntime":1700004510,"order_key":"00:miner-0","p2mr_program_hex":"9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e","share_difficulty":1,"share_id":"share-4510","share_seq":4510,"template_height":800140},{"accepted_at_ms":1000007,"credit_policy":null,"job_id":"job-4511","job_issued_at_ms":999973,"miner_id":"miner-1","network_difficulty":1000006,"ntime":1700004511,"order_key":"01:miner-1","p2mr_program_hex":"9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f","share_difficulty":3,"share_id":"share-4511","share_seq":4511,"template_height":800140},{"accepted_at_ms":1000008,"credit_policy":null,"job_id":"job-4512","job_issued_at_ms":999978,"miner_id":"miner-2","network_difficulty":1000007,"ntime":1700004512,"order_key":"02:miner-2","p2mr_program_hex":"a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0","share_difficulty":19,"share_id":"share-4512","share_seq":4512,"template_height":800141},{"accepted_at_ms":1000005,"credit_policy":null,"job_id":"job-4513","job_issued_at_ms":999972,"miner_id":"miner-3","network_difficulty":1000008,"ntime":1700004513,"order_key":"03:miner-3","p2mr_program_hex":"a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1","share_difficulty":19,"share_id":"share-4513","share_seq":4513,"template_height":800141},{"accepted_at_ms":1000005,"credit_policy":"stale-grace","job_id":"job-4514","job_issued_at_ms":999980,"miner_id":"miner-4","network_difficulty":1000009,"ntime":1700004514,"order_key":"04:miner-4","p2mr_program_hex":"a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2","share_difficulty":1000,"share_id":"share-4514","share_seq":4514,"template_height":800141},{"accepted_at_ms":1000006,"credit_policy":null,"job_id":"job-4515","job_issued_at_ms":999972,"miner_id":"miner-5","network_difficulty":1000010,"ntime":1700004515,"order_key":"05:miner-5","p2mr_program_hex":"a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3","share_difficulty":7,"share_id":"share-4515","share_seq":4515,"template_height":800141},{"accepted_at_ms":1000001,"credit_policy":null,"job_id":"job-4516","job_issued_at_ms":1000010,"miner_id":"miner-6","network_difficulty":1000011,"ntime":1700004516,"order_key":"06:miner-6","p2mr_program_hex":"a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4","share_difficulty":3,"share_id":"share-4516","share_seq":4516,"template_height":800141},{"accepted_at_ms":1000007,"credit_policy":"stale-grace","job_id":"job-4517","job_issued_at_ms":999979,"miner_id":"miner-7","network_difficulty":1000012,"ntime":1700004517,"order_key":"07:miner-7","p2mr_program_hex":"a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5","share_difficulty":1,"share_id":"share-4517","share_seq":4517,"template_height":800141},{"accepted_at_ms":1000002,"credit_policy":"stale-grace","job_id":"job-4518","job_issued_at_ms":999992,"miner_id":"miner-8","network_difficulty":1000013,"ntime":1700004518,"order_key":"08:miner-8","p2mr_program_hex":"a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6","share_difficulty":7,"share_id":"share-4518","share_seq":4518,"template_height":800141},{"accepted_at_ms":1000002,"credit_policy":null,"job_id":"job-4519","job_issued_at_ms":999980,"miner_id":"miner-9","network_difficulty":1000014,"ntime":1700004519,"order_key":"09:miner-9","p2mr_program_hex":"a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7","share_difficulty":1000,"share_id":"share-4519","share_seq":4519,"template_height":800141},{"accepted_at_ms":999994,"credit_policy":"stale-grace","job_id":"job-4520","job_issued_at_ms":1000003,"miner_id":"miner-10","network_difficulty":1000015,"ntime":1700004520,"order_key":"10:miner-10","p2mr_program_hex":"a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8","share_difficulty":7,"share_id":"share-4520","share_seq":4520,"template_height":800141},{"accepted_at_ms":1000009,"credit_policy":null,"job_id":"job-4521","job_issued_at_ms":1000010,"miner_id":"miner-0","network_difficulty":1000016,"ntime":1700004521,"order_key":"00:miner-0","p2mr_program_hex":"a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9","share_difficulty":19,"share_id":"share-4521","share_seq":4521,"template_height":800141},{"accepted_at_ms":1000005,"credit_policy":null,"job_id":"job-4522","job_issued_at_ms":1000006,"miner_id":"miner-1","network_difficulty":1000000,"ntime":1700004522,"order_key":"01:miner-1","p2mr_program_hex":"aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa","share_difficulty":1000,"share_id":"share-4522","share_seq":4522,"template_height":800141},{"accepted_at_ms":999975,"credit_policy":null,"job_id":"job-4523","job_issued_at_ms":1000004,"miner_id":"miner-2","network_difficulty":1000001,"ntime":1700004523,"order_key":"02:miner-2","p2mr_program_hex":"abababababababababababababababababababababababababababababababab","share_difficulty":19,"share_id":"share-4523","share_seq":4523,"template_height":800141},{"accepted_at_ms":1000010,"credit_policy":null,"job_id":"job-4524","job_issued_at_ms":1000009,"miner_id":"miner-3","network_difficulty":1000002,"ntime":1700004524,"order_key":"03:miner-3","p2mr_program_hex":"acacacacacacacacacacacacacacacacacacacacacacacacacacacacacacacac","share_difficulty":19,"share_id":"share-4524","share_seq":4524,"template_height":800141},{"accepted_at_ms":999981,"credit_policy":null,"job_id":"job-4525","job_issued_at_ms":1000002,"miner_id":"miner-4","network_difficulty":1000003,"ntime":1700004525,"order_key":"04:miner-4","p2mr_program_hex":"adadadadadadadadadadadadadadadadadadadadadadadadadadadadadadadad","share_difficulty":2,"share_id":"share-4525","share_seq":4525,"template_height":800141},{"accepted_at_ms":999984,"credit_policy":null,"job_id":"job-4526","job_issued_at_ms":1000006,"miner_id":"miner-5","network_difficulty":1000004,"ntime":1700004526,"order_key":"05:miner-5","p2mr_program_hex":"aeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeae","share_difficulty":101,"share_id":"share-4526","share_seq":4526,"template_height":800141},{"accepted_at_ms":1000009,"credit_policy":null,"job_id":"job-4527","job_issued_at_ms":999990,"miner_id":"miner-6","network_difficulty":1000005,"ntime":1700004527,"order_key":"06:miner-6","p2mr_program_hex":"afafafafafafafafafafafafafafafafafafafafafafafafafafafafafafafaf","share_difficulty":1000,"share_id":"share-4527","share_seq":4527,"template_height":800141},{"accepted_at_ms":1000008,"credit_policy":null,"job_id":"job-4528","job_issued_at_ms":999977,"miner_id":"miner-7","network_difficulty":1000006,"ntime":1700004528,"order_key":"07:miner-7","p2mr_program_hex":"b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0","share_difficulty":101,"share_id":"share-4528","share_seq":4528,"template_height":800141},{"accepted_at_ms":1000006,"credit_policy":null,"job_id":"job-4529","job_issued_at_ms":999990,"miner_id":"miner-8","network_difficulty":1000007,"ntime":1700004529,"order_key":"08:miner-8","p2mr_program_hex":"b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1","share_difficulty":2,"share_id":"share-4529","share_seq":4529,"template_height":800141},{"accepted_at_ms":1000008,"credit_policy":"stale-grace","job_id":"job-4530","job_issued_at_ms":999993,"miner_id":"miner-9","network_difficulty":1000008,"ntime":1700004530,"order_key":"09:miner-9","p2mr_program_hex":"b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2","share_difficulty":7,"share_id":"share-4530","share_seq":4530,"template_height":800141},{"accepted_at_ms":1000006,"credit_policy":null,"job_id":"job-4531","job_issued_at_ms":999976,"miner_id":"miner-10","network_difficulty":1000009,"ntime":1700004531,"order_key":"10:miner-10","p2mr_program_hex":"b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3","share_difficulty":101,"share_id":"share-4531","share_seq":4531,"template_height":800141},{"accepted_at_ms":1000010,"credit_policy":null,"job_id":"job-4532","job_issued_at_ms":999985,"miner_id":"miner-0","network_difficulty":1000010,"ntime":1700004532,"order_key":"00:miner-0","p2mr_program_hex":"b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4","share_difficulty":2,"share_id":"share-4532","share_seq":4532,"template_height":800141},{"accepted_at_ms":1000008,"credit_policy":null,"job_id":"job-4533","job_issued_at_ms":1000008,"miner_id":"miner-1","network_difficulty":1000011,"ntime":1700004533,"order_key":"01:miner-1","p2mr_program_hex":"b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5","share_difficulty":7,"share_id":"share-4533","share_seq":4533,"template_height":800141},{"accepted_at_ms":1000010,"credit_policy":null,"job_id":"job-4534","job_issued_at_ms":1000003,"miner_id":"miner-2","network_difficulty":1000012,"ntime":1700004534,"order_key":"02:miner-2","p2mr_program_hex":"b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6","share_difficulty":101,"share_id":"share-4534","share_seq":4534,"template_height":800141},{"accepted_at_ms":1000005,"credit_policy":"stale-grace","job_id":"job-4535","job_issued_at_ms":999971,"miner_id":"miner-3","network_difficulty":1000013,"ntime":1700004535,"order_key":"03:miner-3","p2mr_program_hex":"b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7","share_difficulty":19,"share_id":"share-4535","share_seq":4535,"template_height":800141},{"accepted_at_ms":999979,"credit_policy":null,"job_id":"job-4536","job_issued_at_ms":1000010,"miner_id":"miner-4","network_difficulty":1000014,"ntime":1700004536,"order_key":"04:miner-4","p2mr_program_hex":"b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8","share_difficulty":2,"share_id":"share-4536","share_seq":4536,"template_height":800141},{"accepted_at_ms":1000004,"credit_policy":null,"job_id":"job-4537","job_issued_at_ms":999985,"miner_id":"miner-5","network_difficulty":1000015,"ntime":1700004537,"order_key":"05:miner-5","p2mr_program_hex":"b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9","share_difficulty":1000,"share_id":"share-4537","share_seq":4537,"template_height":800141},{"accepted_at_ms":1000003,"credit_policy":"stale-grace","job_id":"job-4538","job_issued_at_ms":1000009,"miner_id":"miner-6","network_difficulty":1000016,"ntime":1700004538,"order_key":"06:miner-6","p2mr_program_hex":"babababababababababababababababababababababababababababababababa","share_difficulty":2,"share_id":"share-4538","share_seq":4538,"template_height":800141},{"accepted_at_ms":999982,"credit_policy":null,"job_id":"job-4539","job_issued_at_ms":1000009,"miner_id":"miner-7","network_difficulty":1000000,"ntime":1700004539,"order_key":"07:miner-7","p2mr_program_hex":"bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb","share_difficulty":101,"share_id":"share-4539","share_seq":4539,"template_height":800141},{"accepted_at_ms":1000006,"credit_policy":null,"job_id":"job-4540","job_issued_at_ms":999986,"miner_id":"miner-8","network_difficulty":1000001,"ntime":1700004540,"order_key":"08:miner-8","p2mr_program_hex":"bcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbc","share_difficulty":19,"share_id":"share-4540","share_seq":4540,"template_height":800141},{"accepted_at_ms":999972,"credit_policy":null,"job_id":"job-4541","job_issued_at_ms":1000006,"miner_id":"miner-9","network_difficulty":1000002,"ntime":1700004541,"order_key":"09:miner-9","p2mr_program_hex":"bdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbd","share_difficulty":1,"share_id":"share-4541","share_seq":4541,"template_height":800141},{"accepted_at_ms":1000002,"credit_policy":null,"job_id":"job-4542","job_issued_at_ms":999984,"miner_id":"miner-10","network_difficulty":1000003,"ntime":1700004542,"order_key":"10:miner-10","p2mr_program_hex":"bebebebebebebebebebebebebebebebebebebebebebebebebebebebebebebebe","share_difficulty":7,"share_id":"share-4542","share_seq":4542,"template_height":800141},{"accepted_at_ms":999988,"credit_policy":null,"job_id":"job-4543","job_issued_at_ms":1000006,"miner_id":"miner-0","network_difficulty":1000004,"ntime":1700004543,"order_key":"00:miner-0","p2mr_program_hex":"bfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbf","share_difficulty":3,"share_id":"share-4543","share_seq":4543,"template_height":800141},{"accepted_at_ms":999992,"credit_policy":null,"job_id":"job-4544","job_issued_at_ms":1000002,"miner_id":"miner-1","network_difficulty":1000005,"ntime":1700004544,"order_key":"01:miner-1","p2mr_program_hex":"c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0","share_difficulty":2,"share_id":"share-4544","share_seq":4544,"template_height":800142},{"accepted_at_ms":999993,"credit_policy":null,"job_id":"job-4545","job_issued_at_ms":1000001,"miner_id":"miner-2","network_difficulty":1000006,"ntime":1700004545,"order_key":"02:miner-2","p2mr_program_hex":"c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1","share_difficulty":101,"share_id":"share-4545","share_seq":4545,"template_height":800142},{"accepted_at_ms":999997,"credit_policy":null,"job_id":"job-4546","job_issued_at_ms":1000007,"miner_id":"miner-3","network_difficulty":1000007,"ntime":1700004546,"order_key":"03:miner-3","p2mr_program_hex":"c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2","share_difficulty":1000,"share_id":"share-4546","share_seq":4546,"template_height":800142},{"accepted_at_ms":1000006,"credit_policy":null,"job_id":"job-4547","job_issued_at_ms":1000003,"miner_id":"miner-4","network_difficulty":1000008,"ntime":1700004547,"order_key":"04:miner-4","p2mr_program_hex":"c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3","share_difficulty":2,"share_id":"share-4547","share_seq":4547,"template_height":800142},{"accepted_at_ms":999992,"credit_policy":"stale-grace","job_id":"job-4548","job_issued_at_ms":1000004,"miner_id":"miner-5","network_difficulty":1000009,"ntime":1700004548,"order_key":"05:miner-5","p2mr_program_hex":"c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4","share_difficulty":1000,"share_id":"share-4548","share_seq":4548,"template_height":800142},{"accepted_at_ms":999978,"credit_policy":null,"job_id":"job-4549","job_issued_at_ms":1000005,"miner_id":"miner-6","network_difficulty":1000010,"ntime":1700004549,"order_key":"06:miner-6","p2mr_program_hex":"c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5","share_difficulty":2,"share_id":"share-4549","share_seq":4549,"template_height":800142},{"accepted_at_ms":999980,"credit_policy":null,"job_id":"job-4550","job_issued_at_ms":1000007,"miner_id":"miner-7","network_difficulty":1000011,"ntime":1700004550,"order_key":"07:miner-7","p2mr_program_hex":"c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6","share_difficulty":1,"share_id":"share-4550","share_seq":4550,"template_height":800142},{"accepted_at_ms":999996,"credit_policy":null,"job_id":"job-4551","job_issued_at_ms":1000005,"miner_id":"miner-8","network_difficulty":1000012,"ntime":1700004551,"order_key":"08:miner-8","p2mr_program_hex":"c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7","share_difficulty":7,"share_id":"share-4551","share_seq":4551,"template_height":800142},{"accepted_at_ms":1000002,"credit_policy":"stale-grace","job_id":"job-4552","job_issued_at_ms":1000002,"miner_id":"miner-9","network_difficulty":1000013,"ntime":1700004552,"order_key":"09:miner-9","p2mr_program_hex":"c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8","share_difficulty":19,"share_id":"share-4552","share_seq":4552,"template_height":800142},{"accepted_at_ms":1000008,"credit_policy":null,"job_id":"job-4553","job_issued_at_ms":999995,"miner_id":"miner-10","network_difficulty":1000014,"ntime":1700004553,"order_key":"10:miner-10","p2mr_program_hex":"c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9","share_difficulty":1,"share_id":"share-4553","share_seq":4553,"template_height":800142},{"accepted_at_ms":1000004,"credit_policy":"stale-grace","job_id":"job-4554","job_issued_at_ms":999979,"miner_id":"miner-0","network_difficulty":1000015,"ntime":1700004554,"order_key":"00:miner-0","p2mr_program_hex":"cacacacacacacacacacacacacacacacacacacacacacacacacacacacacacacaca","share_difficulty":7,"share_id":"share-4554","share_seq":4554,"template_height":800142},{"accepted_at_ms":1000000,"credit_policy":null,"job_id":"job-4555","job_issued_at_ms":1000010,"miner_id":"miner-1","network_difficulty":1000016,"ntime":1700004555,"order_key":"01:miner-1","p2mr_program_hex":"cbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcb","share_difficulty":2,"share_id":"share-4555","share_seq":4555,"template_height":800142},{"accepted_at_ms":999973,"credit_policy":null,"job_id":"job-4556","job_issued_at_ms":1000004,"miner_id":"miner-2","network_difficulty":1000000,"ntime":1700004556,"order_key":"02:miner-2","p2mr_program_hex":"cccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccc","share_difficulty":1000,"share_id":"share-4556","share_seq":4556,"template_height":800142},{"accepted_at_ms":999995,"credit_policy":null,"job_id":"job-4557","job_issued_at_ms":1000004,"miner_id":"miner-3","network_difficulty":1000001,"ntime":1700004557,"order_key":"03:miner-3","p2mr_program_hex":"cdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcd","share_difficulty":7,"share_id":"share-4557","share_seq":4557,"template_height":800142},{"accepted_at_ms":1000001,"credit_policy":null,"job_id":"job-4558","job_issued_at_ms":1000010,"miner_id":"miner-4","network_difficulty":1000002,"ntime":1700004558,"order_key":"04:miner-4","p2mr_program_hex":"cececececececececececececececececececececececececececececececece","share_difficulty":3,"share_id":"share-4558","share_seq":4558,"template_height":800142},{"accepted_at_ms":999978,"credit_policy":null,"job_id":"job-4559","job_issued_at_ms":1000001,"miner_id":"miner-5","network_difficulty":1000003,"ntime":1700004559,"order_key":"05:miner-5","p2mr_program_hex":"cfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcf","share_difficulty":19,"share_id":"share-4559","share_seq":4559,"template_height":800142},{"accepted_at_ms":1000007,"credit_policy":null,"job_id":"job-4560","job_issued_at_ms":1000001,"miner_id":"miner-6","network_difficulty":1000004,"ntime":1700004560,"order_key":"06:miner-6","p2mr_program_hex":"d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0","share_difficulty":101,"share_id":"share-4560","share_seq":4560,"template_height":800142},{"accepted_at_ms":1000001,"credit_policy":null,"job_id":"job-4561","job_issued_at_ms":1000004,"miner_id":"miner-7","network_difficulty":1000005,"ntime":1700004561,"order_key":"07:miner-7","p2mr_program_hex":"d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1","share_difficulty":1,"share_id":"share-4561","share_seq":4561,"template_height":800142},{"accepted_at_ms":999978,"credit_policy":null,"job_id":"job-4562","job_issued_at_ms":1000008,"miner_id":"miner-8","network_difficulty":1000006,"ntime":1700004562,"order_key":"08:miner-8","p2mr_program_hex":"d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2","share_difficulty":3,"share_id":"share-4562","share_seq":4562,"template_height":800142},{"accepted_at_ms":999976,"credit_policy":null,"job_id":"job-4563","job_issued_at_ms":1000008,"miner_id":"miner-9","network_difficulty":1000007,"ntime":1700004563,"order_key":"09:miner-9","p2mr_program_hex":"d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3","share_difficulty":1,"share_id":"share-4563","share_seq":4563,"template_height":800142},{"accepted_at_ms":1000006,"credit_policy":null,"job_id":"job-4564","job_issued_at_ms":1000006,"miner_id":"miner-10","network_difficulty":1000008,"ntime":1700004564,"order_key":"10:miner-10","p2mr_program_hex":"d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4","share_difficulty":1,"share_id":"share-4564","share_seq":4564,"template_height":800142},{"accepted_at_ms":1000010,"credit_policy":"stale-grace","job_id":"job-4565","job_issued_at_ms":999976,"miner_id":"miner-0","network_difficulty":1000009,"ntime":1700004565,"order_key":"00:miner-0","p2mr_program_hex":"d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5","share_difficulty":7,"share_id":"share-4565","share_seq":4565,"template_height":800142},{"accepted_at_ms":1000004,"credit_policy":null,"job_id":"job-4566","job_issued_at_ms":1000008,"miner_id":"miner-1","network_difficulty":1000010,"ntime":1700004566,"order_key":"01:miner-1","p2mr_program_hex":"d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6","share_difficulty":3,"share_id":"share-4566","share_seq":4566,"template_height":800142},{"accepted_at_ms":1000006,"credit_policy":null,"job_id":"job-4567","job_issued_at_ms":999984,"miner_id":"miner-2","network_difficulty":1000011,"ntime":1700004567,"order_key":"02:miner-2","p2mr_program_hex":"d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7","share_difficulty":7,"share_id":"share-4567","share_seq":4567,"template_height":800142},{"accepted_at_ms":999977,"credit_policy":null,"job_id":"job-4568","job_issued_at_ms":1000009,"miner_id":"miner-3","network_difficulty":1000012,"ntime":1700004568,"order_key":"03:miner-3","p2mr_program_hex":"d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8","share_difficulty":19,"share_id":"share-4568","share_seq":4568,"template_height":800142},{"accepted_at_ms":1000010,"credit_policy":null,"job_id":"job-4569","job_issued_at_ms":999977,"miner_id":"miner-4","network_difficulty":1000013,"ntime":1700004569,"order_key":"04:miner-4","p2mr_program_hex":"d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9","share_difficulty":101,"share_id":"share-4569","share_seq":4569,"template_height":800142},{"accepted_at_ms":1000001,"credit_policy":null,"job_id":"job-4570","job_issued_at_ms":999995,"miner_id":"miner-5","network_difficulty":1000014,"ntime":1700004570,"order_key":"05:miner-5","p2mr_program_hex":"dadadadadadadadadadadadadadadadadadadadadadadadadadadadadadadada","share_difficulty":3,"share_id":"share-4570","share_seq":4570,"template_height":800142},{"accepted_at_ms":999993,"credit_policy":null,"job_id":"job-4571","job_issued_at_ms":1000003,"miner_id":"miner-6","network_difficulty":1000015,"ntime":1700004571,"order_key":"06:miner-6","p2mr_program_hex":"dbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdb","share_difficulty":3,"share_id":"share-4571","share_seq":4571,"template_height":800142},{"accepted_at_ms":999982,"credit_policy":"stale-grace","job_id":"job-4572","job_issued_at_ms":1000008,"miner_id":"miner-7","network_difficulty":1000016,"ntime":1700004572,"order_key":"07:miner-7","p2mr_program_hex":"dcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdc","share_difficulty":7,"share_id":"share-4572","share_seq":4572,"template_height":800142},{"accepted_at_ms":999986,"credit_policy":null,"job_id":"job-4573","job_issued_at_ms":1000004,"miner_id":"miner-8","network_difficulty":1000000,"ntime":1700004573,"order_key":"08:miner-8","p2mr_program_hex":"dddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddd","share_difficulty":1000,"share_id":"share-4573","share_seq":4573,"template_height":800142},{"accepted_at_ms":1000004,"credit_policy":null,"job_id":"job-4574","job_issued_at_ms":999972,"miner_id":"miner-9","network_difficulty":1000001,"ntime":1700004574,"order_key":"09:miner-9","p2mr_program_hex":"dededededededededededededededededededededededededededededededede","share_difficulty":19,"share_id":"share-4574","share_seq":4574,"template_height":800142},{"accepted_at_ms":1000008,"credit_policy":null,"job_id":"job-4575","job_issued_at_ms":999975,"miner_id":"miner-10","network_difficulty":1000002,"ntime":1700004575,"order_key":"10:miner-10","p2mr_program_hex":"dfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdf","share_difficulty":1,"share_id":"share-4575","share_seq":4575,"template_height":800142},{"accepted_at_ms":1000004,"credit_policy":null,"job_id":"job-4576","job_issued_at_ms":999990,"miner_id":"miner-0","network_difficulty":1000003,"ntime":1700004576,"order_key":"00:miner-0","p2mr_program_hex":"e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0","share_difficulty":3,"share_id":"share-4576","share_seq":4576,"template_height":800143},{"accepted_at_ms":999996,"credit_policy":"stale-grace","job_id":"job-4577","job_issued_at_ms":1000007,"miner_id":"miner-1","network_difficulty":1000004,"ntime":1700004577,"order_key":"01:miner-1","p2mr_program_hex":"e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1","share_difficulty":3,"share_id":"share-4577","share_seq":4577,"template_height":800143},{"accepted_at_ms":1000002,"credit_policy":null,"job_id":"job-4578","job_issued_at_ms":999983,"miner_id":"miner-2","network_difficulty":1000005,"ntime":1700004578,"order_key":"02:miner-2","p2mr_program_hex":"e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2","share_difficulty":1000,"share_id":"share-4578","share_seq":4578,"template_height":800143},{"accepted_at_ms":1000004,"credit_policy":"stale-grace","job_id":"job-4579","job_issued_at_ms":999993,"miner_id":"miner-3","network_difficulty":1000006,"ntime":1700004579,"order_key":"03:miner-3","p2mr_program_hex":"e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3","share_difficulty":101,"share_id":"share-4579","share_seq":4579,"template_height":800143},{"accepted_at_ms":1000001,"credit_policy":null,"job_id":"job-4580","job_issued_at_ms":999996,"miner_id":"miner-4","network_difficulty":1000007,"ntime":1700004580,"order_key":"04:miner-4","p2mr_program_hex":"e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4","share_difficulty":101,"share_id":"share-4580","share_seq":4580,"template_height":800143},{"accepted_at_ms":999983,"credit_policy":null,"job_id":"job-4581","job_issued_at_ms":1000006,"miner_id":"miner-5","network_difficulty":1000008,"ntime":1700004581,"order_key":"05:miner-5","p2mr_program_hex":"e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5","share_difficulty":19,"share_id":"share-4581","share_seq":4581,"template_height":800143},{"accepted_at_ms":999991,"credit_policy":null,"job_id":"job-4582","job_issued_at_ms":1000006,"miner_id":"miner-6","network_difficulty":1000009,"ntime":1700004582,"order_key":"06:miner-6","p2mr_program_hex":"e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6","share_difficulty":101,"share_id":"share-4582","share_seq":4582,"template_height":800143},{"accepted_at_ms":999994,"credit_policy":null,"job_id":"job-4583","job_issued_at_ms":1000009,"miner_id":"miner-7","network_difficulty":1000010,"ntime":1700004583,"order_key":"07:miner-7","p2mr_program_hex":"e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7","share_difficulty":7,"share_id":"share-4583","share_seq":4583,"template_height":800143},{"accepted_at_ms":999990,"credit_policy":null,"job_id":"job-4584","job_issued_at_ms":1000005,"miner_id":"miner-8","network_difficulty":1000011,"ntime":1700004584,"order_key":"08:miner-8","p2mr_program_hex":"e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8","share_difficulty":3,"share_id":"share-4584","share_seq":4584,"template_height":800143},{"accepted_at_ms":999984,"credit_policy":"stale-grace","job_id":"job-4585","job_issued_at_ms":1000004,"miner_id":"miner-9","network_difficulty":1000012,"ntime":1700004585,"order_key":"09:miner-9","p2mr_program_hex":"e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9","share_difficulty":101,"share_id":"share-4585","share_seq":4585,"template_height":800143},{"accepted_at_ms":1000009,"credit_policy":null,"job_id":"job-4586","job_issued_at_ms":1000009,"miner_id":"miner-10","network_difficulty":1000013,"ntime":1700004586,"order_key":"10:miner-10","p2mr_program_hex":"eaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaea","share_difficulty":19,"share_id":"share-4586","share_seq":4586,"template_height":800143},{"accepted_at_ms":999976,"credit_policy":null,"job_id":"job-4587","job_issued_at_ms":1000003,"miner_id":"miner-0","network_difficulty":1000014,"ntime":1700004587,"order_key":"00:miner-0","p2mr_program_hex":"ebebebebebebebebebebebebebebebebebebebebebebebebebebebebebebebeb","share_difficulty":7,"share_id":"share-4587","share_seq":4587,"template_height":800143},{"accepted_at_ms":999973,"credit_policy":null,"job_id":"job-4588","job_issued_at_ms":1000004,"miner_id":"miner-1","network_difficulty":1000015,"ntime":1700004588,"order_key":"01:miner-1","p2mr_program_hex":"ecececececececececececececececececececececececececececececececec","share_difficulty":101,"share_id":"share-4588","share_seq":4588,"template_height":800143},{"accepted_at_ms":1000006,"credit_policy":null,"job_id":"job-4589","job_issued_at_ms":1000008,"miner_id":"miner-2","network_difficulty":1000016,"ntime":1700004589,"order_key":"02:miner-2","p2mr_program_hex":"edededededededededededededededededededededededededededededededed","share_difficulty":19,"share_id":"share-4589","share_seq":4589,"template_height":800143},{"accepted_at_ms":999988,"credit_policy":null,"job_id":"job-4590","job_issued_at_ms":1000006,"miner_id":"miner-3","network_difficulty":1000000,"ntime":1700004590,"order_key":"03:miner-3","p2mr_program_hex":"eeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeee","share_difficulty":7,"share_id":"share-4590","share_seq":4590,"template_height":800143},{"accepted_at_ms":1000001,"credit_policy":"stale-grace","job_id":"job-4591","job_issued_at_ms":999974,"miner_id":"miner-4","network_difficulty":1000001,"ntime":1700004591,"order_key":"04:miner-4","p2mr_program_hex":"efefefefefefefefefefefefefefefefefefefefefefefefefefefefefefefef","share_difficulty":7,"share_id":"share-4591","share_seq":4591,"template_height":800143},{"accepted_at_ms":999971,"credit_policy":"stale-grace","job_id":"job-4592","job_issued_at_ms":1000007,"miner_id":"miner-5","network_difficulty":1000002,"ntime":1700004592,"order_key":"05:miner-5","p2mr_program_hex":"f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0","share_difficulty":1,"share_id":"share-4592","share_seq":4592,"template_height":800143},{"accepted_at_ms":1000002,"credit_policy":null,"job_id":"job-4593","job_issued_at_ms":999994,"miner_id":"miner-6","network_difficulty":1000003,"ntime":1700004593,"order_key":"06:miner-6","p2mr_program_hex":"f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1","share_difficulty":19,"share_id":"share-4593","share_seq":4593,"template_height":800143},{"accepted_at_ms":999988,"credit_policy":null,"job_id":"job-4594","job_issued_at_ms":1000008,"miner_id":"miner-7","network_difficulty":1000004,"ntime":1700004594,"order_key":"07:miner-7","p2mr_program_hex":"f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2","share_difficulty":2,"share_id":"share-4594","share_seq":4594,"template_height":800143},{"accepted_at_ms":999981,"credit_policy":null,"job_id":"job-4595","job_issued_at_ms":1000004,"miner_id":"miner-8","network_difficulty":1000005,"ntime":1700004595,"order_key":"08:miner-8","p2mr_program_hex":"f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3","share_difficulty":1000,"share_id":"share-4595","share_seq":4595,"template_height":800143},{"accepted_at_ms":1000006,"credit_policy":null,"job_id":"job-4596","job_issued_at_ms":999973,"miner_id":"miner-9","network_difficulty":1000006,"ntime":1700004596,"order_key":"09:miner-9","p2mr_program_hex":"f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4","share_difficulty":19,"share_id":"share-4596","share_seq":4596,"template_height":800143},{"accepted_at_ms":999975,"credit_policy":null,"job_id":"job-4597","job_issued_at_ms":1000004,"miner_id":"miner-10","network_difficulty":1000007,"ntime":1700004597,"order_key":"10:miner-10","p2mr_program_hex":"f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5","share_difficulty":7,"share_id":"share-4597","share_seq":4597,"template_height":800143},{"accepted_at_ms":999976,"credit_policy":null,"job_id":"job-4598","job_issued_at_ms":1000010,"miner_id":"miner-0","network_difficulty":1000008,"ntime":1700004598,"order_key":"00:miner-0","p2mr_program_hex":"f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6","share_difficulty":1,"share_id":"share-4598","share_seq":4598,"template_height":800143},{"accepted_at_ms":1000009,"credit_policy":null,"job_id":"job-4599","job_issued_at_ms":999970,"miner_id":"miner-1","network_difficulty":1000009,"ntime":1700004599,"order_key":"01:miner-1","p2mr_program_hex":"f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7","share_difficulty":2,"share_id":"share-4599","share_seq":4599,"template_height":800143},{"accepted_at_ms":999981,"credit_policy":null,"job_id":"job-4600","job_issued_at_ms":1000008,"miner_id":"miner-2","network_difficulty":1000010,"ntime":1700004600,"order_key":"02:miner-2","p2mr_program_hex":"f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8","share_difficulty":1,"share_id":"share-4600","share_seq":4600,"template_height":800143},{"accepted_at_ms":999986,"credit_policy":null,"job_id":"job-4601","job_issued_at_ms":1000006,"miner_id":"miner-3","network_difficulty":1000011,"ntime":1700004601,"order_key":"03:miner-3","p2mr_program_hex":"f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9","share_difficulty":7,"share_id":"share-4601","share_seq":4601,"template_height":800143},{"accepted_at_ms":999978,"credit_policy":null,"job_id":"job-4602","job_issued_at_ms":1000010,"miner_id":"miner-4","network_difficulty":1000012,"ntime":1700004602,"order_key":"04:miner-4","p2mr_program_hex":"fafafafafafafafafafafafafafafafafafafafafafafafafafafafafafafafa","share_difficulty":7,"share_id":"share-4602","share_seq":4602,"template_height":800143},{"accepted_at_ms":1000008,"credit_policy":"stale-grace","job_id":"job-4603","job_issued_at_ms":999988,"miner_id":"miner-5","network_difficulty":1000013,"ntime":1700004603,"order_key":"05:miner-5","p2mr_program_hex":"fbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfb","share_difficulty":3,"share_id":"share-4603","share_seq":4603,"template_height":800143},{"accepted_at_ms":1000001,"credit_policy":null,"job_id":"job-4604","job_issued_at_ms":1000008,"miner_id":"miner-6","network_difficulty":1000014,"ntime":1700004604,"order_key":"06:miner-6","p2mr_program_hex":"fcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfc","share_difficulty":1,"share_id":"share-4604","share_seq":4604,"template_height":800143},{"accepted_at_ms":999986,"credit_policy":null,"job_id":"job-4605","job_issued_at_ms":1000006,"miner_id":"miner-7","network_difficulty":1000015,"ntime":1700004605,"order_key":"07:miner-7","p2mr_program_hex":"fdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfd","share_difficulty":19,"share_id":"share-4605","share_seq":4605,"template_height":800143},{"accepted_at_ms":1000010,"credit_policy":null,"job_id":"job-4606","job_issued_at_ms":999985,"miner_id":"miner-8","network_difficulty":1000016,"ntime":1700004606,"order_key":"08:miner-8","p2mr_program_hex":"fefefefefefefefefefefefefefefefefefefefefefefefefefefefefefefefe","share_difficulty":1,"share_id":"share-4606","share_seq":4606,"template_height":800143},{"accepted_at_ms":999974,"credit_policy":null,"job_id":"job-4607","job_issued_at_ms":1000006,"miner_id":"miner-9","network_difficulty":1000000,"ntime":1700004607,"order_key":"09:miner-9","p2mr_program_hex":"ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff","share_difficulty":1,"share_id":"share-4607","share_seq":4607,"template_height":800143},{"accepted_at_ms":1000004,"credit_policy":null,"job_id":"job-4608","job_issued_at_ms":1000008,"miner_id":"miner-10","network_difficulty":1000001,"ntime":1700004608,"order_key":"10:miner-10","p2mr_program_hex":"0000000000000000000000000000000000000000000000000000000000000000","share_difficulty":7,"share_id":"share-4608","share_seq":4608,"template_height":800144},{"accepted_at_ms":1000001,"credit_policy":"stale-grace","job_id":"job-4609","job_issued_at_ms":999997,"miner_id":"miner-0","network_difficulty":1000002,"ntime":1700004609,"order_key":"00:miner-0","p2mr_program_hex":"0101010101010101010101010101010101010101010101010101010101010101","share_difficulty":3,"share_id":"share-4609","share_seq":4609,"template_height":800144},{"accepted_at_ms":999989,"credit_policy":"stale-grace","job_id":"job-4610","job_issued_at_ms":1000008,"miner_id":"miner-1","network_difficulty":1000003,"ntime":1700004610,"order_key":"01:miner-1","p2mr_program_hex":"0202020202020202020202020202020202020202020202020202020202020202","share_difficulty":3,"share_id":"share-4610","share_seq":4610,"template_height":800144},{"accepted_at_ms":1000001,"credit_policy":null,"job_id":"job-4611","job_issued_at_ms":999982,"miner_id":"miner-2","network_difficulty":1000004,"ntime":1700004611,"order_key":"02:miner-2","p2mr_program_hex":"0303030303030303030303030303030303030303030303030303030303030303","share_difficulty":1,"share_id":"share-4611","share_seq":4611,"template_height":800144},{"accepted_at_ms":1000001,"credit_policy":null,"job_id":"job-4612","job_issued_at_ms":999990,"miner_id":"miner-3","network_difficulty":1000005,"ntime":1700004612,"order_key":"03:miner-3","p2mr_program_hex":"0404040404040404040404040404040404040404040404040404040404040404","share_difficulty":7,"share_id":"share-4612","share_seq":4612,"template_height":800144},{"accepted_at_ms":1000001,"credit_policy":null,"job_id":"job-4613","job_issued_at_ms":999985,"miner_id":"miner-4","network_difficulty":1000006,"ntime":1700004613,"order_key":"04:miner-4","p2mr_program_hex":"0505050505050505050505050505050505050505050505050505050505050505","share_difficulty":19,"share_id":"share-4613","share_seq":4613,"template_height":800144},{"accepted_at_ms":1000002,"credit_policy":"stale-grace","job_id":"job-4614","job_issued_at_ms":1000004,"miner_id":"miner-5","network_difficulty":1000007,"ntime":1700004614,"order_key":"05:miner-5","p2mr_program_hex":"0606060606060606060606060606060606060606060606060606060606060606","share_difficulty":2,"share_id":"share-4614","share_seq":4614,"template_height":800144},{"accepted_at_ms":1000005,"credit_policy":null,"job_id":"job-4615","job_issued_at_ms":999971,"miner_id":"miner-6","network_difficulty":1000008,"ntime":1700004615,"order_key":"06:miner-6","p2mr_program_hex":"0707070707070707070707070707070707070707070707070707070707070707","share_difficulty":19,"share_id":"share-4615","share_seq":4615,"template_height":800144},{"accepted_at_ms":1000005,"credit_policy":null,"job_id":"job-4616","job_issued_at_ms":999991,"miner_id":"miner-7","network_difficulty":1000009,"ntime":1700004616,"order_key":"07:miner-7","p2mr_program_hex":"0808080808080808080808080808080808080808080808080808080808080808","share_difficulty":19,"share_id":"share-4616","share_seq":4616,"template_height":800144},{"accepted_at_ms":1000007,"credit_policy":null,"job_id":"job-4617","job_issued_at_ms":1000003,"miner_id":"miner-8","network_difficulty":1000010,"ntime":1700004617,"order_key":"08:miner-8","p2mr_program_hex":"0909090909090909090909090909090909090909090909090909090909090909","share_difficulty":101,"share_id":"share-4617","share_seq":4617,"template_height":800144},{"accepted_at_ms":1000002,"credit_policy":null,"job_id":"job-4618","job_issued_at_ms":999980,"miner_id":"miner-9","network_difficulty":1000011,"ntime":1700004618,"order_key":"09:miner-9","p2mr_program_hex":"0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a","share_difficulty":7,"share_id":"share-4618","share_seq":4618,"template_height":800144},{"accepted_at_ms":1000002,"credit_policy":"stale-grace","job_id":"job-4619","job_issued_at_ms":999999,"miner_id":"miner-10","network_difficulty":1000012,"ntime":1700004619,"order_key":"10:miner-10","p2mr_program_hex":"0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b","share_difficulty":7,"share_id":"share-4619","share_seq":4619,"template_height":800144},{"accepted_at_ms":999973,"credit_policy":"stale-grace","job_id":"job-4620","job_issued_at_ms":1000010,"miner_id":"miner-0","network_difficulty":1000013,"ntime":1700004620,"order_key":"00:miner-0","p2mr_program_hex":"0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c","share_difficulty":1,"share_id":"share-4620","share_seq":4620,"template_height":800144},{"accepted_at_ms":1000002,"credit_policy":"stale-grace","job_id":"job-4621","job_issued_at_ms":999996,"miner_id":"miner-1","network_difficulty":1000014,"ntime":1700004621,"order_key":"01:miner-1","p2mr_program_hex":"0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d","share_difficulty":101,"share_id":"share-4621","share_seq":4621,"template_height":800144},{"accepted_at_ms":1000000,"credit_policy":null,"job_id":"job-4622","job_issued_at_ms":1000010,"miner_id":"miner-2","network_difficulty":1000015,"ntime":1700004622,"order_key":"02:miner-2","p2mr_program_hex":"0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e","share_difficulty":1,"share_id":"share-4622","share_seq":4622,"template_height":800144},{"accepted_at_ms":1000001,"credit_policy":null,"job_id":"job-4623","job_issued_at_ms":999975,"miner_id":"miner-3","network_difficulty":1000016,"ntime":1700004623,"order_key":"03:miner-3","p2mr_program_hex":"0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f","share_difficulty":1,"share_id":"share-4623","share_seq":4623,"template_height":800144},{"accepted_at_ms":1000001,"credit_policy":null,"job_id":"job-4624","job_issued_at_ms":1000000,"miner_id":"miner-4","network_difficulty":1000000,"ntime":1700004624,"order_key":"04:miner-4","p2mr_program_hex":"1010101010101010101010101010101010101010101010101010101010101010","share_difficulty":1,"share_id":"share-4624","share_seq":4624,"template_height":800144},{"accepted_at_ms":999982,"credit_policy":null,"job_id":"job-4625","job_issued_at_ms":1000004,"miner_id":"miner-5","network_difficulty":1000001,"ntime":1700004625,"order_key":"05:miner-5","p2mr_program_hex":"1111111111111111111111111111111111111111111111111111111111111111","share_difficulty":19,"share_id":"share-4625","share_seq":4625,"template_height":800144},{"accepted_at_ms":1000009,"credit_policy":null,"job_id":"job-4626","job_issued_at_ms":999971,"miner_id":"miner-6","network_difficulty":1000002,"ntime":1700004626,"order_key":"06:miner-6","p2mr_program_hex":"1212121212121212121212121212121212121212121212121212121212121212","share_difficulty":2,"share_id":"share-4626","share_seq":4626,"template_height":800144},{"accepted_at_ms":1000010,"credit_policy":null,"job_id":"job-4627","job_issued_at_ms":999988,"miner_id":"miner-7","network_difficulty":1000003,"ntime":1700004627,"order_key":"07:miner-7","p2mr_program_hex":"1313131313131313131313131313131313131313131313131313131313131313","share_difficulty":1000,"share_id":"share-4627","share_seq":4627,"template_height":800144},{"accepted_at_ms":999972,"credit_policy":null,"job_id":"job-4628","job_issued_at_ms":1000004,"miner_id":"miner-8","network_difficulty":1000004,"ntime":1700004628,"order_key":"08:miner-8","p2mr_program_hex":"1414141414141414141414141414141414141414141414141414141414141414","share_difficulty":1000,"share_id":"share-4628","share_seq":4628,"template_height":800144},{"accepted_at_ms":1000007,"credit_policy":"stale-grace","job_id":"job-4629","job_issued_at_ms":1000006,"miner_id":"miner-9","network_difficulty":1000005,"ntime":1700004629,"order_key":"09:miner-9","p2mr_program_hex":"1515151515151515151515151515151515151515151515151515151515151515","share_difficulty":1,"share_id":"share-4629","share_seq":4629,"template_height":800144},{"accepted_at_ms":999989,"credit_policy":null,"job_id":"job-4630","job_issued_at_ms":1000002,"miner_id":"miner-10","network_difficulty":1000006,"ntime":1700004630,"order_key":"10:miner-10","p2mr_program_hex":"1616161616161616161616161616161616161616161616161616161616161616","share_difficulty":101,"share_id":"share-4630","share_seq":4630,"template_height":800144},{"accepted_at_ms":999992,"credit_policy":null,"job_id":"job-4631","job_issued_at_ms":1000001,"miner_id":"miner-0","network_difficulty":1000007,"ntime":1700004631,"order_key":"00:miner-0","p2mr_program_hex":"1717171717171717171717171717171717171717171717171717171717171717","share_difficulty":3,"share_id":"share-4631","share_seq":4631,"template_height":800144},{"accepted_at_ms":999972,"credit_policy":null,"job_id":"job-4632","job_issued_at_ms":1000007,"miner_id":"miner-1","network_difficulty":1000008,"ntime":1700004632,"order_key":"01:miner-1","p2mr_program_hex":"1818181818181818181818181818181818181818181818181818181818181818","share_difficulty":7,"share_id":"share-4632","share_seq":4632,"template_height":800144},{"accepted_at_ms":1000006,"credit_policy":null,"job_id":"job-4633","job_issued_at_ms":999996,"miner_id":"miner-2","network_difficulty":1000009,"ntime":1700004633,"order_key":"02:miner-2","p2mr_program_hex":"1919191919191919191919191919191919191919191919191919191919191919","share_difficulty":101,"share_id":"share-4633","share_seq":4633,"template_height":800144},{"accepted_at_ms":1000008,"credit_policy":null,"job_id":"job-4634","job_issued_at_ms":1000005,"miner_id":"miner-3","network_difficulty":1000010,"ntime":1700004634,"order_key":"03:miner-3","p2mr_program_hex":"1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a","share_difficulty":1,"share_id":"share-4634","share_seq":4634,"template_height":800144},{"accepted_at_ms":1000009,"credit_policy":null,"job_id":"job-4635","job_issued_at_ms":1000008,"miner_id":"miner-4","network_difficulty":1000011,"ntime":1700004635,"order_key":"04:miner-4","p2mr_program_hex":"1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b","share_difficulty":1,"share_id":"share-4635","share_seq":4635,"template_height":800144},{"accepted_at_ms":1000002,"credit_policy":null,"job_id":"job-4636","job_issued_at_ms":999991,"miner_id":"miner-5","network_difficulty":1000012,"ntime":1700004636,"order_key":"05:miner-5","p2mr_program_hex":"1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c","share_difficulty":3,"share_id":"share-4636","share_seq":4636,"template_height":800144},{"accepted_at_ms":999986,"credit_policy":null,"job_id":"job-4637","job_issued_at_ms":1000008,"miner_id":"miner-6","network_difficulty":1000013,"ntime":1700004637,"order_key":"06:miner-6","p2mr_program_hex":"1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d","share_difficulty":19,"share_id":"share-4637","share_seq":4637,"template_height":800144},{"accepted_at_ms":999976,"credit_policy":null,"job_id":"job-4638","job_issued_at_ms":1000007,"miner_id":"miner-7","network_difficulty":1000014,"ntime":1700004638,"order_key":"07:miner-7","p2mr_program_hex":"1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e","share_difficulty":7,"share_id":"share-4638","share_seq":4638,"template_height":800144},{"accepted_at_ms":1000004,"credit_policy":null,"job_id":"job-4639","job_issued_at_ms":999981,"miner_id":"miner-8","network_difficulty":1000015,"ntime":1700004639,"order_key":"08:miner-8","p2mr_program_hex":"1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f","share_difficulty":1,"share_id":"share-4639","share_seq":4639,"template_height":800144},{"accepted_at_ms":1000006,"credit_policy":null,"job_id":"job-4640","job_issued_at_ms":1000006,"miner_id":"miner-9","network_difficulty":1000016,"ntime":1700004640,"order_key":"09:miner-9","p2mr_program_hex":"2020202020202020202020202020202020202020202020202020202020202020","share_difficulty":101,"share_id":"share-4640","share_seq":4640,"template_height":800145},{"accepted_at_ms":999980,"credit_policy":null,"job_id":"job-4641","job_issued_at_ms":1000005,"miner_id":"miner-10","network_difficulty":1000000,"ntime":1700004641,"order_key":"10:miner-10","p2mr_program_hex":"2121212121212121212121212121212121212121212121212121212121212121","share_difficulty":101,"share_id":"share-4641","share_seq":4641,"template_height":800145},{"accepted_at_ms":1000010,"credit_policy":null,"job_id":"job-4642","job_issued_at_ms":999989,"miner_id":"miner-0","network_difficulty":1000001,"ntime":1700004642,"order_key":"00:miner-0","p2mr_program_hex":"2222222222222222222222222222222222222222222222222222222222222222","share_difficulty":1,"share_id":"share-4642","share_seq":4642,"template_height":800145},{"accepted_at_ms":999983,"credit_policy":null,"job_id":"job-4643","job_issued_at_ms":1000010,"miner_id":"miner-1","network_difficulty":1000002,"ntime":1700004643,"order_key":"01:miner-1","p2mr_program_hex":"2323232323232323232323232323232323232323232323232323232323232323","share_difficulty":19,"share_id":"share-4643","share_seq":4643,"template_height":800145},{"accepted_at_ms":999996,"credit_policy":null,"job_id":"job-4644","job_issued_at_ms":1000003,"miner_id":"miner-2","network_difficulty":1000003,"ntime":1700004644,"order_key":"02:miner-2","p2mr_program_hex":"2424242424242424242424242424242424242424242424242424242424242424","share_difficulty":1000,"share_id":"share-4644","share_seq":4644,"template_height":800145},{"accepted_at_ms":999992,"credit_policy":"stale-grace","job_id":"job-4645","job_issued_at_ms":1000008,"miner_id":"miner-3","network_difficulty":1000004,"ntime":1700004645,"order_key":"03:miner-3","p2mr_program_hex":"2525252525252525252525252525252525252525252525252525252525252525","share_difficulty":7,"share_id":"share-4645","share_seq":4645,"template_height":800145},{"accepted_at_ms":1000000,"credit_policy":null,"job_id":"job-4646","job_issued_at_ms":1000008,"miner_id":"miner-4","network_difficulty":1000005,"ntime":1700004646,"order_key":"04:miner-4","p2mr_program_hex":"2626262626262626262626262626262626262626262626262626262626262626","share_difficulty":1000,"share_id":"share-4646","share_seq":4646,"template_height":800145},{"accepted_at_ms":999994,"credit_policy":null,"job_id":"job-4647","job_issued_at_ms":1000004,"miner_id":"miner-5","network_difficulty":1000006,"ntime":1700004647,"order_key":"05:miner-5","p2mr_program_hex":"2727272727272727272727272727272727272727272727272727272727272727","share_difficulty":1,"share_id":"share-4647","share_seq":4647,"template_height":800145},{"accepted_at_ms":999971,"credit_policy":null,"job_id":"job-4648","job_issued_at_ms":1000008,"miner_id":"miner-6","network_difficulty":1000007,"ntime":1700004648,"order_key":"06:miner-6","p2mr_program_hex":"2828282828282828282828282828282828282828282828282828282828282828","share_difficulty":1000,"share_id":"share-4648","share_seq":4648,"template_height":800145},{"accepted_at_ms":999971,"credit_policy":null,"job_id":"job-4649","job_issued_at_ms":1000003,"miner_id":"miner-7","network_difficulty":1000008,"ntime":1700004649,"order_key":"07:miner-7","p2mr_program_hex":"2929292929292929292929292929292929292929292929292929292929292929","share_difficulty":101,"share_id":"share-4649","share_seq":4649,"template_height":800145},{"accepted_at_ms":1000010,"credit_policy":null,"job_id":"job-4650","job_issued_at_ms":1000001,"miner_id":"miner-8","network_difficulty":1000009,"ntime":1700004650,"order_key":"08:miner-8","p2mr_program_hex":"2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a","share_difficulty":7,"share_id":"share-4650","share_seq":4650,"template_height":800145},{"accepted_at_ms":999999,"credit_policy":"stale-grace","job_id":"job-4651","job_issued_at_ms":1000008,"miner_id":"miner-9","network_difficulty":1000010,"ntime":1700004651,"order_key":"09:miner-9","p2mr_program_hex":"2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b","share_difficulty":19,"share_id":"share-4651","share_seq":4651,"template_height":800145},{"accepted_at_ms":1000010,"credit_policy":null,"job_id":"job-4652","job_issued_at_ms":999995,"miner_id":"miner-10","network_difficulty":1000011,"ntime":1700004652,"order_key":"10:miner-10","p2mr_program_hex":"2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c","share_difficulty":1,"share_id":"share-4652","share_seq":4652,"template_height":800145},{"accepted_at_ms":1000005,"credit_policy":null,"job_id":"job-4653","job_issued_at_ms":1000008,"miner_id":"miner-0","network_difficulty":1000012,"ntime":1700004653,"order_key":"00:miner-0","p2mr_program_hex":"2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d","share_difficulty":2,"share_id":"share-4653","share_seq":4653,"template_height":800145},{"accepted_at_ms":1000006,"credit_policy":null,"job_id":"job-4654","job_issued_at_ms":999999,"miner_id":"miner-1","network_difficulty":1000013,"ntime":1700004654,"order_key":"01:miner-1","p2mr_program_hex":"2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e","share_difficulty":101,"share_id":"share-4654","share_seq":4654,"template_height":800145},{"accepted_at_ms":1000010,"credit_policy":null,"job_id":"job-4655","job_issued_at_ms":1000005,"miner_id":"miner-2","network_difficulty":1000014,"ntime":1700004655,"order_key":"02:miner-2","p2mr_program_hex":"2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f","share_difficulty":19,"share_id":"share-4655","share_seq":4655,"template_height":800145},{"accepted_at_ms":1000009,"credit_policy":"stale-grace","job_id":"job-4656","job_issued_at_ms":1000005,"miner_id":"miner-3","network_difficulty":1000015,"ntime":1700004656,"order_key":"03:miner-3","p2mr_program_hex":"3030303030303030303030303030303030303030303030303030303030303030","share_difficulty":3,"share_id":"share-4656","share_seq":4656,"template_height":800145},{"accepted_at_ms":1000003,"credit_policy":null,"job_id":"job-4657","job_issued_at_ms":1000003,"miner_id":"miner-4","network_difficulty":1000016,"ntime":1700004657,"order_key":"04:miner-4","p2mr_program_hex":"3131313131313131313131313131313131313131313131313131313131313131","share_difficulty":19,"share_id":"share-4657","share_seq":4657,"template_height":800145},{"accepted_at_ms":1000005,"credit_policy":null,"job_id":"job-4658","job_issued_at_ms":999986,"miner_id":"miner-5","network_difficulty":1000000,"ntime":1700004658,"order_key":"05:miner-5","p2mr_program_hex":"3232323232323232323232323232323232323232323232323232323232323232","share_difficulty":101,"share_id":"share-4658","share_seq":4658,"template_height":800145},{"accepted_at_ms":1000008,"credit_policy":null,"job_id":"job-4659","job_issued_at_ms":999992,"miner_id":"miner-6","network_difficulty":1000001,"ntime":1700004659,"order_key":"06:miner-6","p2mr_program_hex":"3333333333333333333333333333333333333333333333333333333333333333","share_difficulty":19,"share_id":"share-4659","share_seq":4659,"template_height":800145},{"accepted_at_ms":1000007,"credit_policy":"stale-grace","job_id":"job-4660","job_issued_at_ms":999975,"miner_id":"miner-7","network_difficulty":1000002,"ntime":1700004660,"order_key":"07:miner-7","p2mr_program_hex":"3434343434343434343434343434343434343434343434343434343434343434","share_difficulty":3,"share_id":"share-4660","share_seq":4660,"template_height":800145},{"accepted_at_ms":999992,"credit_policy":null,"job_id":"job-4661","job_issued_at_ms":1000007,"miner_id":"miner-8","network_difficulty":1000003,"ntime":1700004661,"order_key":"08:miner-8","p2mr_program_hex":"3535353535353535353535353535353535353535353535353535353535353535","share_difficulty":1000,"share_id":"share-4661","share_seq":4661,"template_height":800145},{"accepted_at_ms":999985,"credit_policy":null,"job_id":"job-4662","job_issued_at_ms":1000010,"miner_id":"miner-9","network_difficulty":1000004,"ntime":1700004662,"order_key":"09:miner-9","p2mr_program_hex":"3636363636363636363636363636363636363636363636363636363636363636","share_difficulty":1000,"share_id":"share-4662","share_seq":4662,"template_height":800145},{"accepted_at_ms":1000003,"credit_policy":null,"job_id":"job-4663","job_issued_at_ms":999997,"miner_id":"miner-10","network_difficulty":1000005,"ntime":1700004663,"order_key":"10:miner-10","p2mr_program_hex":"3737373737373737373737373737373737373737373737373737373737373737","share_difficulty":101,"share_id":"share-4663","share_seq":4663,"template_height":800145},{"accepted_at_ms":1000004,"credit_policy":null,"job_id":"job-4664","job_issued_at_ms":1000009,"miner_id":"miner-0","network_difficulty":1000006,"ntime":1700004664,"order_key":"00:miner-0","p2mr_program_hex":"3838383838383838383838383838383838383838383838383838383838383838","share_difficulty":1000,"share_id":"share-4664","share_seq":4664,"template_height":800145},{"accepted_at_ms":1000000,"credit_policy":"stale-grace","job_id":"job-4665","job_issued_at_ms":1000007,"miner_id":"miner-1","network_difficulty":1000007,"ntime":1700004665,"order_key":"01:miner-1","p2mr_program_hex":"3939393939393939393939393939393939393939393939393939393939393939","share_difficulty":1,"share_id":"share-4665","share_seq":4665,"template_height":800145},{"accepted_at_ms":1000003,"credit_policy":null,"job_id":"job-4666","job_issued_at_ms":999981,"miner_id":"miner-2","network_difficulty":1000008,"ntime":1700004666,"order_key":"02:miner-2","p2mr_program_hex":"3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a","share_difficulty":3,"share_id":"share-4666","share_seq":4666,"template_height":800145},{"accepted_at_ms":1000005,"credit_policy":null,"job_id":"job-4667","job_issued_at_ms":999974,"miner_id":"miner-3","network_difficulty":1000009,"ntime":1700004667,"order_key":"03:miner-3","p2mr_program_hex":"3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b","share_difficulty":1000,"share_id":"share-4667","share_seq":4667,"template_height":800145},{"accepted_at_ms":999988,"credit_policy":null,"job_id":"job-4668","job_issued_at_ms":1000001,"miner_id":"miner-4","network_difficulty":1000010,"ntime":1700004668,"order_key":"04:miner-4","p2mr_program_hex":"3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c","share_difficulty":1000,"share_id":"share-4668","share_seq":4668,"template_height":800145},{"accepted_at_ms":999980,"credit_policy":null,"job_id":"job-4669","job_issued_at_ms":1000003,"miner_id":"miner-5","network_difficulty":1000011,"ntime":1700004669,"order_key":"05:miner-5","p2mr_program_hex":"3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d","share_difficulty":7,"share_id":"share-4669","share_seq":4669,"template_height":800145},{"accepted_at_ms":1000001,"credit_policy":null,"job_id":"job-4670","job_issued_at_ms":1000006,"miner_id":"miner-6","network_difficulty":1000012,"ntime":1700004670,"order_key":"06:miner-6","p2mr_program_hex":"3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e","share_difficulty":2,"share_id":"share-4670","share_seq":4670,"template_height":800145},{"accepted_at_ms":1000006,"credit_policy":null,"job_id":"job-4671","job_issued_at_ms":1000000,"miner_id":"miner-7","network_difficulty":1000013,"ntime":1700004671,"order_key":"07:miner-7","p2mr_program_hex":"3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f","share_difficulty":3,"share_id":"share-4671","share_seq":4671,"template_height":800145},{"accepted_at_ms":1000005,"credit_policy":"stale-grace","job_id":"job-4672","job_issued_at_ms":1000001,"miner_id":"miner-8","network_difficulty":1000014,"ntime":1700004672,"order_key":"08:miner-8","p2mr_program_hex":"4040404040404040404040404040404040404040404040404040404040404040","share_difficulty":1,"share_id":"share-4672","share_seq":4672,"template_height":800146},{"accepted_at_ms":1000003,"credit_policy":null,"job_id":"job-4673","job_issued_at_ms":999977,"miner_id":"miner-9","network_difficulty":1000015,"ntime":1700004673,"order_key":"09:miner-9","p2mr_program_hex":"4141414141414141414141414141414141414141414141414141414141414141","share_difficulty":19,"share_id":"share-4673","share_seq":4673,"template_height":800146},{"accepted_at_ms":999972,"credit_policy":null,"job_id":"job-4674","job_issued_at_ms":1000009,"miner_id":"miner-10","network_difficulty":1000016,"ntime":1700004674,"order_key":"10:miner-10","p2mr_program_hex":"4242424242424242424242424242424242424242424242424242424242424242","share_difficulty":19,"share_id":"share-4674","share_seq":4674,"template_height":800146},{"accepted_at_ms":1000006,"credit_policy":null,"job_id":"job-4675","job_issued_at_ms":999994,"miner_id":"miner-0","network_difficulty":1000000,"ntime":1700004675,"order_key":"00:miner-0","p2mr_program_hex":"4343434343434343434343434343434343434343434343434343434343434343","share_difficulty":2,"share_id":"share-4675","share_seq":4675,"template_height":800146},{"accepted_at_ms":1000008,"credit_policy":null,"job_id":"job-4676","job_issued_at_ms":999988,"miner_id":"miner-1","network_difficulty":1000001,"ntime":1700004676,"order_key":"01:miner-1","p2mr_program_hex":"4444444444444444444444444444444444444444444444444444444444444444","share_difficulty":3,"share_id":"share-4676","share_seq":4676,"template_height":800146},{"accepted_at_ms":999978,"credit_policy":"stale-grace","job_id":"job-4677","job_issued_at_ms":1000009,"miner_id":"miner-2","network_difficulty":1000002,"ntime":1700004677,"order_key":"02:miner-2","p2mr_program_hex":"4545454545454545454545454545454545454545454545454545454545454545","share_difficulty":1000,"share_id":"share-4677","share_seq":4677,"template_height":800146},{"accepted_at_ms":999981,"credit_policy":null,"job_id":"job-4678","job_issued_at_ms":1000009,"miner_id":"miner-3","network_difficulty":1000003,"ntime":1700004678,"order_key":"03:miner-3","p2mr_program_hex":"4646464646464646464646464646464646464646464646464646464646464646","share_difficulty":2,"share_id":"share-4678","share_seq":4678,"template_height":800146},{"accepted_at_ms":1000008,"credit_policy":null,"job_id":"job-4679","job_issued_at_ms":999970,"miner_id":"miner-4","network_difficulty":1000004,"ntime":1700004679,"order_key":"04:miner-4","p2mr_program_hex":"4747474747474747474747474747474747474747474747474747474747474747","share_difficulty":3,"share_id":"share-4679","share_seq":4679,"template_height":800146},{"accepted_at_ms":1000010,"credit_policy":null,"job_id":"job-4680","job_issued_at_ms":1000005,"miner_id":"miner-5","network_difficulty":1000005,"ntime":1700004680,"order_key":"05:miner-5","p2mr_program_hex":"4848484848484848484848484848484848484848484848484848484848484848","share_difficulty":3,"share_id":"share-4680","share_seq":4680,"template_height":800146},{"accepted_at_ms":1000001,"credit_policy":null,"job_id":"job-4681","job_issued_at_ms":999995,"miner_id":"miner-6","network_difficulty":1000006,"ntime":1700004681,"order_key":"06:miner-6","p2mr_program_hex":"4949494949494949494949494949494949494949494949494949494949494949","share_difficulty":1,"share_id":"share-4681","share_seq":4681,"template_height":800146},{"accepted_at_ms":1000010,"credit_policy":null,"job_id":"job-4682","job_issued_at_ms":999986,"miner_id":"miner-7","network_difficulty":1000007,"ntime":1700004682,"order_key":"07:miner-7","p2mr_program_hex":"4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a","share_difficulty":7,"share_id":"share-4682","share_seq":4682,"template_height":800146},{"accepted_at_ms":1000007,"credit_policy":null,"job_id":"job-4683","job_issued_at_ms":999992,"miner_id":"miner-8","network_difficulty":1000008,"ntime":1700004683,"order_key":"08:miner-8","p2mr_program_hex":"4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b","share_difficulty":1000,"share_id":"share-4683","share_seq":4683,"template_height":800146},{"accepted_at_ms":1000003,"credit_policy":null,"job_id":"job-4684","job_issued_at_ms":1000008,"miner_id":"miner-9","network_difficulty":1000009,"ntime":1700004684,"order_key":"09:miner-9","p2mr_program_hex":"4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c","share_difficulty":1,"share_id":"share-4684","share_seq":4684,"template_height":800146},{"accepted_at_ms":999995,"credit_policy":null,"job_id":"job-4685","job_issued_at_ms":1000001,"miner_id":"miner-10","network_difficulty":1000010,"ntime":1700004685,"order_key":"10:miner-10","p2mr_program_hex":"4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d","share_difficulty":2,"share_id":"share-4685","share_seq":4685,"template_height":800146},{"accepted_at_ms":999997,"credit_policy":null,"job_id":"job-4686","job_issued_at_ms":1000005,"miner_id":"miner-0","network_difficulty":1000011,"ntime":1700004686,"order_key":"00:miner-0","p2mr_program_hex":"4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e","share_difficulty":1000,"share_id":"share-4686","share_seq":4686,"template_height":800146},{"accepted_at_ms":999999,"credit_policy":null,"job_id":"job-4687","job_issued_at_ms":1000010,"miner_id":"miner-1","network_difficulty":1000012,"ntime":1700004687,"order_key":"01:miner-1","p2mr_program_hex":"4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f","share_difficulty":2,"share_id":"share-4687","share_seq":4687,"template_height":800146},{"accepted_at_ms":999998,"credit_policy":null,"job_id":"job-4688","job_issued_at_ms":1000002,"miner_id":"miner-2","network_difficulty":1000013,"ntime":1700004688,"order_key":"02:miner-2","p2mr_program_hex":"5050505050505050505050505050505050505050505050505050505050505050","share_difficulty":1,"share_id":"share-4688","share_seq":4688,"template_height":800146},{"accepted_at_ms":999993,"credit_policy":null,"job_id":"job-4689","job_issued_at_ms":1000010,"miner_id":"miner-3","network_difficulty":1000014,"ntime":1700004689,"order_key":"03:miner-3","p2mr_program_hex":"5151515151515151515151515151515151515151515151515151515151515151","share_difficulty":101,"share_id":"share-4689","share_seq":4689,"template_height":800146},{"accepted_at_ms":1000006,"credit_policy":"stale-grace","job_id":"job-4690","job_issued_at_ms":999982,"miner_id":"miner-4","network_difficulty":1000015,"ntime":1700004690,"order_key":"04:miner-4","p2mr_program_hex":"5252525252525252525252525252525252525252525252525252525252525252","share_difficulty":7,"share_id":"share-4690","share_seq":4690,"template_height":800146},{"accepted_at_ms":999977,"credit_policy":null,"job_id":"job-4691","job_issued_at_ms":1000005,"miner_id":"miner-5","network_difficulty":1000016,"ntime":1700004691,"order_key":"05:miner-5","p2mr_program_hex":"5353535353535353535353535353535353535353535353535353535353535353","share_difficulty":3,"share_id":"share-4691","share_seq":4691,"template_height":800146},{"accepted_at_ms":999973,"credit_policy":"stale-grace","job_id":"job-4692","job_issued_at_ms":1000002,"miner_id":"miner-6","network_difficulty":1000000,"ntime":1700004692,"order_key":"06:miner-6","p2mr_program_hex":"5454545454545454545454545454545454545454545454545454545454545454","share_difficulty":19,"share_id":"share-4692","share_seq":4692,"template_height":800146},{"accepted_at_ms":999987,"credit_policy":null,"job_id":"job-4693","job_issued_at_ms":1000007,"miner_id":"miner-7","network_difficulty":1000001,"ntime":1700004693,"order_key":"07:miner-7","p2mr_program_hex":"5555555555555555555555555555555555555555555555555555555555555555","share_difficulty":19,"share_id":"share-4693","share_seq":4693,"template_height":800146},{"accepted_at_ms":999981,"credit_policy":null,"job_id":"job-4694","job_issued_at_ms":1000008,"miner_id":"miner-8","network_difficulty":1000002,"ntime":1700004694,"order_key":"08:miner-8","p2mr_program_hex":"5656565656565656565656565656565656565656565656565656565656565656","share_difficulty":3,"share_id":"share-4694","share_seq":4694,"template_height":800146},{"accepted_at_ms":1000009,"credit_policy":"stale-grace","job_id":"job-4695","job_issued_at_ms":1000008,"miner_id":"miner-9","network_difficulty":1000003,"ntime":1700004695,"order_key":"09:miner-9","p2mr_program_hex":"5757575757575757575757575757575757575757575757575757575757575757","share_difficulty":7,"share_id":"share-4695","share_seq":4695,"template_height":800146},{"accepted_at_ms":1000008,"credit_policy":null,"job_id":"job-4696","job_issued_at_ms":1000003,"miner_id":"miner-10","network_difficulty":1000004,"ntime":1700004696,"order_key":"10:miner-10","p2mr_program_hex":"5858585858585858585858585858585858585858585858585858585858585858","share_difficulty":19,"share_id":"share-4696","share_seq":4696,"template_height":800146},{"accepted_at_ms":999978,"credit_policy":null,"job_id":"job-4697","job_issued_at_ms":1000010,"miner_id":"miner-0","network_difficulty":1000005,"ntime":1700004697,"order_key":"00:miner-0","p2mr_program_hex":"5959595959595959595959595959595959595959595959595959595959595959","share_difficulty":1000,"share_id":"share-4697","share_seq":4697,"template_height":800146},{"accepted_at_ms":1000010,"credit_policy":null,"job_id":"job-4698","job_issued_at_ms":999976,"miner_id":"miner-1","network_difficulty":1000006,"ntime":1700004698,"order_key":"01:miner-1","p2mr_program_hex":"5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a","share_difficulty":1000,"share_id":"share-4698","share_seq":4698,"template_height":800146},{"accepted_at_ms":1000004,"credit_policy":null,"job_id":"job-4699","job_issued_at_ms":1000007,"miner_id":"miner-2","network_difficulty":1000007,"ntime":1700004699,"order_key":"02:miner-2","p2mr_program_hex":"5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b","share_difficulty":1,"share_id":"share-4699","share_seq":4699,"template_height":800146},{"accepted_at_ms":1000001,"credit_policy":null,"job_id":"job-4700","job_issued_at_ms":999971,"miner_id":"miner-3","network_difficulty":1000008,"ntime":1700004700,"order_key":"03:miner-3","p2mr_program_hex":"5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c","share_difficulty":1,"share_id":"share-4700","share_seq":4700,"template_height":800146}]}],"name":"bulk-seeded","page_size":512,"schema":"qbit-prism-window-pipeline-parity-input/v1","snapshot":{"anchor_job_issued_at_ms":1000000,"records":[{"accepted_at_ms":999986,"credit_policy":"stale-grace","job_id":"job-1897","job_issued_at_ms":999971,"miner_id":"miner-5","network_difficulty":1000010,"ntime":1700001897,"order_key":"05:miner-5","p2mr_program_hex":"6969696969696969696969696969696969696969696969696969696969696969","share_difficulty":7,"share_id":"share-1897","share_seq":1897,"template_height":800059},{"accepted_at_ms":999986,"credit_policy":null,"job_id":"job-3288","job_issued_at_ms":999999,"miner_id":"miner-10","network_difficulty":1000007,"ntime":1700003288,"order_key":"10:miner-10","p2mr_program_hex":"d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8","share_difficulty":19,"share_id":"share-3288","share_seq":3288,"template_height":800102},{"accepted_at_ms":999979,"credit_policy":null,"job_id":"job-2386","job_issued_at_ms":999974,"miner_id":"miner-10","network_difficulty":1000006,"ntime":1700002386,"order_key":"10:miner-10","p2mr_program_hex":"5252525252525252525252525252525252525252525252525252525252525252","share_difficulty":1,"share_id":"share-2386","share_seq":2386,"template_height":800074},{"accepted_at_ms":999999,"credit_policy":null,"job_id":"job-1510","job_issued_at_ms":999999,"miner_id":"miner-3","network_difficulty":1000014,"ntime":1700001510,"order_key":"03:miner-3","p2mr_program_hex":"e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6","share_difficulty":2,"share_id":"share-1510","share_seq":1510,"template_height":800047},{"accepted_at_ms":999989,"credit_policy":null,"job_id":"job-287","job_issued_at_ms":999984,"miner_id":"miner-1","network_difficulty":1000015,"ntime":1700000287,"order_key":"01:miner-1","p2mr_program_hex":"1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f","share_difficulty":2,"share_id":"share-287","share_seq":287,"template_height":800008},{"accepted_at_ms":999970,"credit_policy":null,"job_id":"job-2626","job_issued_at_ms":999992,"miner_id":"miner-8","network_difficulty":1000008,"ntime":1700002626,"order_key":"08:miner-8","p2mr_program_hex":"4242424242424242424242424242424242424242424242424242424242424242","share_difficulty":3,"share_id":"share-2626","share_seq":2626,"template_height":800082},{"accepted_at_ms":999987,"credit_policy":null,"job_id":"job-2221","job_issued_at_ms":999975,"miner_id":"miner-10","network_difficulty":1000011,"ntime":1700002221,"order_key":"10:miner-10","p2mr_program_hex":"adadadadadadadadadadadadadadadadadadadadadadadadadadadadadadadad","share_difficulty":2,"share_id":"share-2221","share_seq":2221,"template_height":800069},{"accepted_at_ms":999996,"credit_policy":null,"job_id":"job-2421","job_issued_at_ms":999978,"miner_id":"miner-1","network_difficulty":1000007,"ntime":1700002421,"order_key":"01:miner-1","p2mr_program_hex":"7575757575757575757575757575757575757575757575757575757575757575","share_difficulty":101,"share_id":"share-2421","share_seq":2421,"template_height":800075},{"accepted_at_ms":999993,"credit_policy":null,"job_id":"job-2050","job_issued_at_ms":999973,"miner_id":"miner-4","network_difficulty":1000010,"ntime":1700002050,"order_key":"04:miner-4","p2mr_program_hex":"0202020202020202020202020202020202020202020202020202020202020202","share_difficulty":101,"share_id":"share-2050","share_seq":2050,"template_height":800064},{"accepted_at_ms":1000000,"credit_policy":"stale-grace","job_id":"job-2914","job_issued_at_ms":999976,"miner_id":"miner-10","network_difficulty":1000007,"ntime":1700002914,"order_key":"10:miner-10","p2mr_program_hex":"6262626262626262626262626262626262626262626262626262626262626262","share_difficulty":2,"share_id":"share-2914","share_seq":2914,"template_height":800091},{"accepted_at_ms":999999,"credit_policy":null,"job_id":"job-3484","job_issued_at_ms":999982,"miner_id":"miner-8","network_difficulty":1000016,"ntime":1700003484,"order_key":"08:miner-8","p2mr_program_hex":"9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c","share_difficulty":1,"share_id":"share-3484","share_seq":3484,"template_height":800108},{"accepted_at_ms":1000135,"credit_policy":null,"job_id":"job-1059","job_issued_at_ms":1000229,"miner_id":"miner-3","network_difficulty":1000005,"ntime":1700001059,"order_key":"03:miner-3","p2mr_program_hex":"2323232323232323232323232323232323232323232323232323232323232323","share_difficulty":1,"share_id":"share-1059","share_seq":1059,"template_height":800033},{"accepted_at_ms":999986,"credit_policy":null,"job_id":"job-2130","job_issued_at_ms":999976,"miner_id":"miner-7","network_difficulty":1000005,"ntime":1700002130,"order_key":"07:miner-7","p2mr_program_hex":"5252525252525252525252525252525252525252525252525252525252525252","share_difficulty":1,"share_id":"share-2130","share_seq":2130,"template_height":800066},{"accepted_at_ms":999970,"credit_policy":null,"job_id":"job-1868","job_issued_at_ms":999992,"miner_id":"miner-9","network_difficulty":1000015,"ntime":1700001868,"order_key":"09:miner-9","p2mr_program_hex":"4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c","share_difficulty":1,"share_id":"share-1868","share_seq":1868,"template_height":800058},{"accepted_at_ms":999971,"credit_policy":null,"job_id":"job-1637","job_issued_at_ms":999971,"miner_id":"miner-9","network_difficulty":1000005,"ntime":1700001637,"order_key":"09:miner-9","p2mr_program_hex":"6565656565656565656565656565656565656565656565656565656565656565","share_difficulty":7,"share_id":"share-1637","share_seq":1637,"template_height":800051},{"accepted_at_ms":999985,"credit_policy":"stale-grace","job_id":"job-3931","job_issued_at_ms":999993,"miner_id":"miner-4","network_difficulty":1000004,"ntime":1700003931,"order_key":"04:miner-4","p2mr_program_hex":"5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b","share_difficulty":19,"share_id":"share-3931","share_seq":3931,"template_height":800122},{"accepted_at_ms":999987,"credit_policy":null,"job_id":"job-3210","job_issued_at_ms":999988,"miner_id":"miner-9","network_difficulty":1000014,"ntime":1700003210,"order_key":"09:miner-9","p2mr_program_hex":"8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a","share_difficulty":2,"share_id":"share-3210","share_seq":3210,"template_height":800100},{"accepted_at_ms":999974,"credit_policy":null,"job_id":"job-1777","job_issued_at_ms":999977,"miner_id":"miner-6","network_difficulty":1000009,"ntime":1700001777,"order_key":"06:miner-6","p2mr_program_hex":"f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1","share_difficulty":3,"share_id":"share-1777","share_seq":1777,"template_height":800055},{"accepted_at_ms":999987,"credit_policy":null,"job_id":"job-3474","job_issued_at_ms":999973,"miner_id":"miner-9","network_difficulty":1000006,"ntime":1700003474,"order_key":"09:miner-9","p2mr_program_hex":"9292929292929292929292929292929292929292929292929292929292929292","share_difficulty":3,"share_id":"share-3474","share_seq":3474,"template_height":800108},{"accepted_at_ms":999982,"credit_policy":null,"job_id":"job-2833","job_issued_at_ms":999997,"miner_id":"miner-6","network_difficulty":1000011,"ntime":1700002833,"order_key":"06:miner-6","p2mr_program_hex":"1111111111111111111111111111111111111111111111111111111111111111","share_difficulty":1000,"share_id":"share-2833","share_seq":2833,"template_height":800088},{"accepted_at_ms":999994,"credit_policy":"stale-grace","job_id":"job-2019","job_issued_at_ms":999988,"miner_id":"miner-6","network_difficulty":1000013,"ntime":1700002019,"order_key":"06:miner-6","p2mr_program_hex":"e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3","share_difficulty":1000,"share_id":"share-2019","share_seq":2019,"template_height":800063},{"accepted_at_ms":999992,"credit_policy":"stale-grace","job_id":"job-870","job_issued_at_ms":999989,"miner_id":"miner-1","network_difficulty":1000003,"ntime":1700000870,"order_key":"01:miner-1","p2mr_program_hex":"6666666666666666666666666666666666666666666666666666666666666666","share_difficulty":19,"share_id":"share-870","share_seq":870,"template_height":800027},{"accepted_at_ms":1000257,"credit_policy":"stale-grace","job_id":"job-23","job_issued_at_ms":999999,"miner_id":"miner-1","network_difficulty":1000006,"ntime":1700000023,"order_key":"01:miner-1","p2mr_program_hex":"1717171717171717171717171717171717171717171717171717171717171717","share_difficulty":19,"share_id":"share-23","share_seq":23,"template_height":800000},{"accepted_at_ms":999981,"credit_policy":null,"job_id":"job-3905","job_issued_at_ms":999974,"miner_id":"miner-0","network_difficulty":1000012,"ntime":1700003905,"order_key":"00:miner-0","p2mr_program_hex":"4141414141414141414141414141414141414141414141414141414141414141","share_difficulty":1000,"share_id":"share-3905","share_seq":3905,"template_height":800122},{"accepted_at_ms":999989,"credit_policy":null,"job_id":"job-2517","job_issued_at_ms":999980,"miner_id":"miner-9","network_difficulty":1000001,"ntime":1700002517,"order_key":"09:miner-9","p2mr_program_hex":"d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5","share_difficulty":101,"share_id":"share-2517","share_seq":2517,"template_height":800078},{"accepted_at_ms":999973,"credit_policy":null,"job_id":"job-2990","job_issued_at_ms":1000283,"miner_id":"miner-9","network_difficulty":1000015,"ntime":1700002990,"order_key":"09:miner-9","p2mr_program_hex":"aeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeae","share_difficulty":19,"share_id":"share-2990","share_seq":2990,"template_height":800093},{"accepted_at_ms":999998,"credit_policy":"stale-grace","job_id":"job-900","job_issued_at_ms":999996,"miner_id":"miner-9","network_difficulty":1000016,"ntime":1700000900,"order_key":"09:miner-9","p2mr_program_hex":"8484848484848484848484848484848484848484848484848484848484848484","share_difficulty":7,"share_id":"share-900","share_seq":900,"template_height":800028},{"accepted_at_ms":999983,"credit_policy":null,"job_id":"job-1063","job_issued_at_ms":999991,"miner_id":"miner-7","network_difficulty":1000009,"ntime":1700001063,"order_key":"07:miner-7","p2mr_program_hex":"2727272727272727272727272727272727272727272727272727272727272727","share_difficulty":3,"share_id":"share-1063","share_seq":1063,"template_height":800033},{"accepted_at_ms":999994,"credit_policy":"stale-grace","job_id":"job-1202","job_issued_at_ms":999993,"miner_id":"miner-3","network_difficulty":1000012,"ntime":1700001202,"order_key":"03:miner-3","p2mr_program_hex":"b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2","share_difficulty":101,"share_id":"share-1202","share_seq":1202,"template_height":800037},{"accepted_at_ms":999985,"credit_policy":null,"job_id":"job-2921","job_issued_at_ms":999974,"miner_id":"miner-6","network_difficulty":1000014,"ntime":1700002921,"order_key":"06:miner-6","p2mr_program_hex":"6969696969696969696969696969696969696969696969696969696969696969","share_difficulty":19,"share_id":"share-2921","share_seq":2921,"template_height":800091},{"accepted_at_ms":999993,"credit_policy":null,"job_id":"job-3285","job_issued_at_ms":999980,"miner_id":"miner-7","network_difficulty":1000004,"ntime":1700003285,"order_key":"07:miner-7","p2mr_program_hex":"d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5","share_difficulty":1,"share_id":"share-3285","share_seq":3285,"template_height":800102},{"accepted_at_ms":999979,"credit_policy":null,"job_id":"job-3762","job_issued_at_ms":999995,"miner_id":"miner-0","network_difficulty":1000005,"ntime":1700003762,"order_key":"00:miner-0","p2mr_program_hex":"b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2","share_difficulty":19,"share_id":"share-3762","share_seq":3762,"template_height":800117},{"accepted_at_ms":999980,"credit_policy":"stale-grace","job_id":"job-137","job_issued_at_ms":999976,"miner_id":"miner-5","network_difficulty":1000001,"ntime":1700000137,"order_key":"05:miner-5","p2mr_program_hex":"8989898989898989898989898989898989898989898989898989898989898989","share_difficulty":19,"share_id":"share-137","share_seq":137,"template_height":800004},{"accepted_at_ms":1000190,"credit_policy":"stale-grace","job_id":"job-388","job_issued_at_ms":1000409,"miner_id":"miner-3","network_difficulty":1000014,"ntime":1700000388,"order_key":"03:miner-3","p2mr_program_hex":"8484848484848484848484848484848484848484848484848484848484848484","share_difficulty":2,"share_id":"share-388","share_seq":388,"template_height":800012},{"accepted_at_ms":999987,"credit_policy":null,"job_id":"job-501","job_issued_at_ms":999980,"miner_id":"miner-6","network_difficulty":1000008,"ntime":1700000501,"order_key":"06:miner-6","p2mr_program_hex":"f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5","share_difficulty":7,"share_id":"share-501","share_seq":501,"template_height":800015},{"accepted_at_ms":999996,"credit_policy":null,"job_id":"job-840","job_issued_at_ms":999988,"miner_id":"miner-4","network_difficulty":1000007,"ntime":1700000840,"order_key":"04:miner-4","p2mr_program_hex":"4848484848484848484848484848484848484848484848484848484848484848","share_difficulty":101,"share_id":"share-840","share_seq":840,"template_height":800026},{"accepted_at_ms":999978,"credit_policy":null,"job_id":"job-3078","job_issued_at_ms":999993,"miner_id":"miner-9","network_difficulty":1000001,"ntime":1700003078,"order_key":"09:miner-9","p2mr_program_hex":"0606060606060606060606060606060606060606060606060606060606060606","share_difficulty":1000,"share_id":"share-3078","share_seq":3078,"template_height":800096},{"accepted_at_ms":999997,"credit_policy":null,"job_id":"job-1237","job_issued_at_ms":999980,"miner_id":"miner-5","network_difficulty":1000013,"ntime":1700001237,"order_key":"05:miner-5","p2mr_program_hex":"d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5","share_difficulty":7,"share_id":"share-1237","share_seq":1237,"template_height":800038},{"accepted_at_ms":999980,"credit_policy":null,"job_id":"job-1668","job_issued_at_ms":999975,"miner_id":"miner-7","network_difficulty":1000002,"ntime":1700001668,"order_key":"07:miner-7","p2mr_program_hex":"8484848484848484848484848484848484848484848484848484848484848484","share_difficulty":2,"share_id":"share-1668","share_seq":1668,"template_height":800052},{"accepted_at_ms":999972,"credit_policy":null,"job_id":"job-3786","job_issued_at_ms":999985,"miner_id":"miner-2","network_difficulty":1000012,"ntime":1700003786,"order_key":"02:miner-2","p2mr_program_hex":"cacacacacacacacacacacacacacacacacacacacacacacacacacacacacacacaca","share_difficulty":7,"share_id":"share-3786","share_seq":3786,"template_height":800118},{"accepted_at_ms":999981,"credit_policy":null,"job_id":"job-454","job_issued_at_ms":999980,"miner_id":"miner-3","network_difficulty":1000012,"ntime":1700000454,"order_key":"03:miner-3","p2mr_program_hex":"c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6","share_difficulty":101,"share_id":"share-454","share_seq":454,"template_height":800014},{"accepted_at_ms":999974,"credit_policy":null,"job_id":"job-779","job_issued_at_ms":999987,"miner_id":"miner-9","network_difficulty":1000014,"ntime":1700000779,"order_key":"09:miner-9","p2mr_program_hex":"0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b","share_difficulty":2,"share_id":"share-779","share_seq":779,"template_height":800024},{"accepted_at_ms":999998,"credit_policy":"stale-grace","job_id":"job-3022","job_issued_at_ms":999982,"miner_id":"miner-8","network_difficulty":1000013,"ntime":1700003022,"order_key":"08:miner-8","p2mr_program_hex":"cececececececececececececececececececececececececececececececece","share_difficulty":1000,"share_id":"share-3022","share_seq":3022,"template_height":800094},{"accepted_at_ms":999983,"credit_policy":null,"job_id":"job-3106","job_issued_at_ms":999972,"miner_id":"miner-4","network_difficulty":1000012,"ntime":1700003106,"order_key":"04:miner-4","p2mr_program_hex":"2222222222222222222222222222222222222222222222222222222222222222","share_difficulty":101,"share_id":"share-3106","share_seq":3106,"template_height":800097},{"accepted_at_ms":999985,"credit_policy":null,"job_id":"job-921","job_issued_at_ms":999982,"miner_id":"miner-8","network_difficulty":1000003,"ntime":1700000921,"order_key":"08:miner-8","p2mr_program_hex":"9999999999999999999999999999999999999999999999999999999999999999","share_difficulty":19,"share_id":"share-921","share_seq":921,"template_height":800028},{"accepted_at_ms":999997,"credit_policy":null,"job_id":"job-3190","job_issued_at_ms":999993,"miner_id":"miner-0","network_difficulty":1000011,"ntime":1700003190,"order_key":"00:miner-0","p2mr_program_hex":"7676767676767676767676767676767676767676767676767676767676767676","share_difficulty":7,"share_id":"share-3190","share_seq":3190,"template_height":800099},{"accepted_at_ms":999975,"credit_policy":null,"job_id":"job-3038","job_issued_at_ms":999974,"miner_id":"miner-2","network_difficulty":1000012,"ntime":1700003038,"order_key":"02:miner-2","p2mr_program_hex":"dededededededededededededededededededededededededededededededede","share_difficulty":2,"share_id":"share-3038","share_seq":3038,"template_height":800094},{"accepted_at_ms":999981,"credit_policy":null,"job_id":"job-636","job_issued_at_ms":999986,"miner_id":"miner-9","network_difficulty":1000007,"ntime":1700000636,"order_key":"09:miner-9","p2mr_program_hex":"7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c","share_difficulty":3,"share_id":"share-636","share_seq":636,"template_height":800019},{"accepted_at_ms":999971,"credit_policy":null,"job_id":"job-1128","job_issued_at_ms":999996,"miner_id":"miner-6","network_difficulty":1000006,"ntime":1700001128,"order_key":"06:miner-6","p2mr_program_hex":"6868686868686868686868686868686868686868686868686868686868686868","share_difficulty":101,"share_id":"share-1128","share_seq":1128,"template_height":800035},{"accepted_at_ms":999971,"credit_policy":null,"job_id":"job-803","job_issued_at_ms":999984,"miner_id":"miner-0","network_difficulty":1000004,"ntime":1700000803,"order_key":"00:miner-0","p2mr_program_hex":"2323232323232323232323232323232323232323232323232323232323232323","share_difficulty":7,"share_id":"share-803","share_seq":803,"template_height":800025},{"accepted_at_ms":999971,"credit_policy":null,"job_id":"job-3591","job_issued_at_ms":999998,"miner_id":"miner-5","network_difficulty":1000004,"ntime":1700003591,"order_key":"05:miner-5","p2mr_program_hex":"0707070707070707070707070707070707070707070707070707070707070707","share_difficulty":3,"share_id":"share-3591","share_seq":3591,"template_height":800112},{"accepted_at_ms":1000374,"credit_policy":null,"job_id":"job-553","job_issued_at_ms":999972,"miner_id":"miner-3","network_difficulty":1000009,"ntime":1700000553,"order_key":"03:miner-3","p2mr_program_hex":"2929292929292929292929292929292929292929292929292929292929292929","share_difficulty":1000,"share_id":"share-553","share_seq":553,"template_height":800017},{"accepted_at_ms":999977,"credit_policy":null,"job_id":"job-1767","job_issued_at_ms":999984,"miner_id":"miner-7","network_difficulty":1000016,"ntime":1700001767,"order_key":"07:miner-7","p2mr_program_hex":"e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7","share_difficulty":101,"share_id":"share-1767","share_seq":1767,"template_height":800055},{"accepted_at_ms":999970,"credit_policy":null,"job_id":"job-1221","job_issued_at_ms":999978,"miner_id":"miner-0","network_difficulty":1000014,"ntime":1700001221,"order_key":"00:miner-0","p2mr_program_hex":"c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5","share_difficulty":3,"share_id":"share-1221","share_seq":1221,"template_height":800038},{"accepted_at_ms":999981,"credit_policy":null,"job_id":"job-2205","job_issued_at_ms":999999,"miner_id":"miner-5","network_difficulty":1000012,"ntime":1700002205,"order_key":"05:miner-5","p2mr_program_hex":"9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d","share_difficulty":101,"share_id":"share-2205","share_seq":2205,"template_height":800068},{"accepted_at_ms":999979,"credit_policy":null,"job_id":"job-1989","job_issued_at_ms":999993,"miner_id":"miner-9","network_difficulty":1000000,"ntime":1700001989,"order_key":"09:miner-9","p2mr_program_hex":"c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5","share_difficulty":1,"share_id":"share-1989","share_seq":1989,"template_height":800062},{"accepted_at_ms":999983,"credit_policy":"stale-grace","job_id":"job-2458","job_issued_at_ms":999991,"miner_id":"miner-5","network_difficulty":1000010,"ntime":1700002458,"order_key":"05:miner-5","p2mr_program_hex":"9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a","share_difficulty":19,"share_id":"share-2458","share_seq":2458,"template_height":800076},{"accepted_at_ms":999973,"credit_policy":null,"job_id":"job-789","job_issued_at_ms":999975,"miner_id":"miner-8","network_difficulty":1000007,"ntime":1700000789,"order_key":"08:miner-8","p2mr_program_hex":"1515151515151515151515151515151515151515151515151515151515151515","share_difficulty":7,"share_id":"share-789","share_seq":789,"template_height":800024},{"accepted_at_ms":999981,"credit_policy":"stale-grace","job_id":"job-3577","job_issued_at_ms":999986,"miner_id":"miner-2","network_difficulty":1000007,"ntime":1700003577,"order_key":"02:miner-2","p2mr_program_hex":"f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9","share_difficulty":2,"share_id":"share-3577","share_seq":3577,"template_height":800111},{"accepted_at_ms":999989,"credit_policy":"stale-grace","job_id":"job-906","job_issued_at_ms":999971,"miner_id":"miner-4","network_difficulty":1000005,"ntime":1700000906,"order_key":"04:miner-4","p2mr_program_hex":"8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a","share_difficulty":101,"share_id":"share-906","share_seq":906,"template_height":800028},{"accepted_at_ms":999989,"credit_policy":null,"job_id":"job-2270","job_issued_at_ms":999999,"miner_id":"miner-4","network_difficulty":1000009,"ntime":1700002270,"order_key":"04:miner-4","p2mr_program_hex":"dededededededededededededededededededededededededededededededede","share_difficulty":101,"share_id":"share-2270","share_seq":2270,"template_height":800070},{"accepted_at_ms":999998,"credit_policy":null,"job_id":"job-124","job_issued_at_ms":999992,"miner_id":"miner-3","network_difficulty":1000005,"ntime":1700000124,"order_key":"03:miner-3","p2mr_program_hex":"7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c","share_difficulty":1000,"share_id":"share-124","share_seq":124,"template_height":800003},{"accepted_at_ms":999999,"credit_policy":"stale-grace","job_id":"job-2331","job_issued_at_ms":999970,"miner_id":"miner-10","network_difficulty":1000002,"ntime":1700002331,"order_key":"10:miner-10","p2mr_program_hex":"1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b","share_difficulty":3,"share_id":"share-2331","share_seq":2331,"template_height":800072},{"accepted_at_ms":999975,"credit_policy":null,"job_id":"job-818","job_issued_at_ms":999984,"miner_id":"miner-4","network_difficulty":1000002,"ntime":1700000818,"order_key":"04:miner-4","p2mr_program_hex":"3232323232323232323232323232323232323232323232323232323232323232","share_difficulty":3,"share_id":"share-818","share_seq":818,"template_height":800025},{"accepted_at_ms":999989,"credit_policy":null,"job_id":"job-2202","job_issued_at_ms":999991,"miner_id":"miner-2","network_difficulty":1000009,"ntime":1700002202,"order_key":"02:miner-2","p2mr_program_hex":"9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a","share_difficulty":1,"share_id":"share-2202","share_seq":2202,"template_height":800068},{"accepted_at_ms":1000043,"credit_policy":null,"job_id":"job-3788","job_issued_at_ms":1000487,"miner_id":"miner-4","network_difficulty":1000014,"ntime":1700003788,"order_key":"04:miner-4","p2mr_program_hex":"cccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccc","share_difficulty":1,"share_id":"share-3788","share_seq":3788,"template_height":800118},{"accepted_at_ms":999994,"credit_policy":null,"job_id":"job-914","job_issued_at_ms":999985,"miner_id":"miner-1","network_difficulty":1000013,"ntime":1700000914,"order_key":"01:miner-1","p2mr_program_hex":"9292929292929292929292929292929292929292929292929292929292929292","share_difficulty":7,"share_id":"share-914","share_seq":914,"template_height":800028},{"accepted_at_ms":999991,"credit_policy":null,"job_id":"job-2674","job_issued_at_ms":999989,"miner_id":"miner-1","network_difficulty":1000005,"ntime":1700002674,"order_key":"01:miner-1","p2mr_program_hex":"7272727272727272727272727272727272727272727272727272727272727272","share_difficulty":7,"share_id":"share-2674","share_seq":2674,"template_height":800083},{"accepted_at_ms":999999,"credit_policy":"stale-grace","job_id":"job-1275","job_issued_at_ms":999998,"miner_id":"miner-10","network_difficulty":1000000,"ntime":1700001275,"order_key":"10:miner-10","p2mr_program_hex":"fbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfb","share_difficulty":2,"share_id":"share-1275","share_seq":1275,"template_height":800039},{"accepted_at_ms":999974,"credit_policy":null,"job_id":"job-3552","job_issued_at_ms":999996,"miner_id":"miner-10","network_difficulty":1000016,"ntime":1700003552,"order_key":"10:miner-10","p2mr_program_hex":"e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0","share_difficulty":1000,"share_id":"share-3552","share_seq":3552,"template_height":800111},{"accepted_at_ms":999980,"credit_policy":"stale-grace","job_id":"job-2752","job_issued_at_ms":1000000,"miner_id":"miner-2","network_difficulty":1000015,"ntime":1700002752,"order_key":"02:miner-2","p2mr_program_hex":"c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0","share_difficulty":1000,"share_id":"share-2752","share_seq":2752,"template_height":800086},{"accepted_at_ms":1000266,"credit_policy":"stale-grace","job_id":"job-628","job_issued_at_ms":999970,"miner_id":"miner-1","network_difficulty":1000016,"ntime":1700000628,"order_key":"01:miner-1","p2mr_program_hex":"7474747474747474747474747474747474747474747474747474747474747474","share_difficulty":101,"share_id":"share-628","share_seq":628,"template_height":800019},{"accepted_at_ms":999995,"credit_policy":null,"job_id":"job-45","job_issued_at_ms":999990,"miner_id":"miner-1","network_difficulty":1000011,"ntime":1700000045,"order_key":"01:miner-1","p2mr_program_hex":"2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d","share_difficulty":101,"share_id":"share-45","share_seq":45,"template_height":800001},{"accepted_at_ms":999996,"credit_policy":"stale-grace","job_id":"job-173","job_issued_at_ms":999998,"miner_id":"miner-8","network_difficulty":1000003,"ntime":1700000173,"order_key":"08:miner-8","p2mr_program_hex":"adadadadadadadadadadadadadadadadadadadadadadadadadadadadadadadad","share_difficulty":19,"share_id":"share-173","share_seq":173,"template_height":800005},{"accepted_at_ms":999986,"credit_policy":"stale-grace","job_id":"job-1326","job_issued_at_ms":1000135,"miner_id":"miner-6","network_difficulty":1000000,"ntime":1700001326,"order_key":"06:miner-6","p2mr_program_hex":"2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e","share_difficulty":1,"share_id":"share-1326","share_seq":1326,"template_height":800041},{"accepted_at_ms":999971,"credit_policy":null,"job_id":"job-634","job_issued_at_ms":999971,"miner_id":"miner-7","network_difficulty":1000005,"ntime":1700000634,"order_key":"07:miner-7","p2mr_program_hex":"7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a","share_difficulty":1,"share_id":"share-634","share_seq":634,"template_height":800019},{"accepted_at_ms":999991,"credit_policy":null,"job_id":"job-3707","job_issued_at_ms":999974,"miner_id":"miner-0","network_difficulty":1000001,"ntime":1700003707,"order_key":"00:miner-0","p2mr_program_hex":"7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b","share_difficulty":1000,"share_id":"share-3707","share_seq":3707,"template_height":800115},{"accepted_at_ms":999979,"credit_policy":null,"job_id":"job-833","job_issued_at_ms":999997,"miner_id":"miner-8","network_difficulty":1000000,"ntime":1700000833,"order_key":"08:miner-8","p2mr_program_hex":"4141414141414141414141414141414141414141414141414141414141414141","share_difficulty":3,"share_id":"share-833","share_seq":833,"template_height":800026},{"accepted_at_ms":999986,"credit_policy":null,"job_id":"job-886","job_issued_at_ms":999983,"miner_id":"miner-6","network_difficulty":1000002,"ntime":1700000886,"order_key":"06:miner-6","p2mr_program_hex":"7676767676767676767676767676767676767676767676767676767676767676","share_difficulty":2,"share_id":"share-886","share_seq":886,"template_height":800027},{"accepted_at_ms":999982,"credit_policy":null,"job_id":"job-75","job_issued_at_ms":999979,"miner_id":"miner-9","network_difficulty":1000007,"ntime":1700000075,"order_key":"09:miner-9","p2mr_program_hex":"4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b","share_difficulty":1000,"share_id":"share-75","share_seq":75,"template_height":800002},{"accepted_at_ms":999996,"credit_policy":null,"job_id":"job-1325","job_issued_at_ms":999986,"miner_id":"miner-5","network_difficulty":1000016,"ntime":1700001325,"order_key":"05:miner-5","p2mr_program_hex":"2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d","share_difficulty":1,"share_id":"share-1325","share_seq":1325,"template_height":800041},{"accepted_at_ms":999998,"credit_policy":null,"job_id":"job-3325","job_issued_at_ms":999994,"miner_id":"miner-3","network_difficulty":1000010,"ntime":1700003325,"order_key":"03:miner-3","p2mr_program_hex":"fdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfd","share_difficulty":2,"share_id":"share-3325","share_seq":3325,"template_height":800103},{"accepted_at_ms":999974,"credit_policy":null,"job_id":"job-3654","job_issued_at_ms":999999,"miner_id":"miner-2","network_difficulty":1000016,"ntime":1700003654,"order_key":"02:miner-2","p2mr_program_hex":"4646464646464646464646464646464646464646464646464646464646464646","share_difficulty":1,"share_id":"share-3654","share_seq":3654,"template_height":800114},{"accepted_at_ms":999994,"credit_policy":null,"job_id":"job-285","job_issued_at_ms":999999,"miner_id":"miner-10","network_difficulty":1000013,"ntime":1700000285,"order_key":"10:miner-10","p2mr_program_hex":"1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d","share_difficulty":1,"share_id":"share-285","share_seq":285,"template_height":800008},{"accepted_at_ms":999975,"credit_policy":null,"job_id":"job-3534","job_issued_at_ms":999977,"miner_id":"miner-3","network_difficulty":1000015,"ntime":1700003534,"order_key":"03:miner-3","p2mr_program_hex":"cececececececececececececececececececececececececececececececece","share_difficulty":19,"share_id":"share-3534","share_seq":3534,"template_height":800110},{"accepted_at_ms":999995,"credit_policy":null,"job_id":"job-2262","job_issued_at_ms":999983,"miner_id":"miner-7","network_difficulty":1000001,"ntime":1700002262,"order_key":"07:miner-7","p2mr_program_hex":"d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6","share_difficulty":7,"share_id":"share-2262","share_seq":2262,"template_height":800070},{"accepted_at_ms":999982,"credit_policy":null,"job_id":"job-3551","job_issued_at_ms":999987,"miner_id":"miner-9","network_difficulty":1000015,"ntime":1700003551,"order_key":"09:miner-9","p2mr_program_hex":"dfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdf","share_difficulty":1000,"share_id":"share-3551","share_seq":3551,"template_height":800110},{"accepted_at_ms":999992,"credit_policy":null,"job_id":"job-1477","job_issued_at_ms":999983,"miner_id":"miner-3","network_difficulty":1000015,"ntime":1700001477,"order_key":"03:miner-3","p2mr_program_hex":"c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5","share_difficulty":19,"share_id":"share-1477","share_seq":1477,"template_height":800046},{"accepted_at_ms":1000000,"credit_policy":null,"job_id":"job-3460","job_issued_at_ms":999987,"miner_id":"miner-6","network_difficulty":1000009,"ntime":1700003460,"order_key":"06:miner-6","p2mr_program_hex":"8484848484848484848484848484848484848484848484848484848484848484","share_difficulty":1,"share_id":"share-3460","share_seq":3460,"template_height":800108},{"accepted_at_ms":1000000,"credit_policy":"stale-grace","job_id":"job-3258","job_issued_at_ms":999981,"miner_id":"miner-2","network_difficulty":1000011,"ntime":1700003258,"order_key":"02:miner-2","p2mr_program_hex":"babababababababababababababababababababababababababababababababa","share_difficulty":19,"share_id":"share-3258","share_seq":3258,"template_height":800101},{"accepted_at_ms":999971,"credit_policy":"stale-grace","job_id":"job-3153","job_issued_at_ms":1000450,"miner_id":"miner-7","network_difficulty":1000008,"ntime":1700003153,"order_key":"07:miner-7","p2mr_program_hex":"5151515151515151515151515151515151515151515151515151515151515151","share_difficulty":2,"share_id":"share-3153","share_seq":3153,"template_height":800098},{"accepted_at_ms":999999,"credit_policy":null,"job_id":"job-2871","job_issued_at_ms":999971,"miner_id":"miner-0","network_difficulty":1000015,"ntime":1700002871,"order_key":"00:miner-0","p2mr_program_hex":"3737373737373737373737373737373737373737373737373737373737373737","share_difficulty":3,"share_id":"share-2871","share_seq":2871,"template_height":800089},{"accepted_at_ms":999979,"credit_policy":null,"job_id":"job-680","job_issued_at_ms":999973,"miner_id":"miner-9","network_difficulty":1000000,"ntime":1700000680,"order_key":"09:miner-9","p2mr_program_hex":"a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8","share_difficulty":2,"share_id":"share-680","share_seq":680,"template_height":800021},{"accepted_at_ms":999993,"credit_policy":null,"job_id":"job-1820","job_issued_at_ms":999996,"miner_id":"miner-5","network_difficulty":1000001,"ntime":1700001820,"order_key":"05:miner-5","p2mr_program_hex":"1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c","share_difficulty":3,"share_id":"share-1820","share_seq":1820,"template_height":800056},{"accepted_at_ms":999975,"credit_policy":null,"job_id":"job-229","job_issued_at_ms":1000191,"miner_id":"miner-9","network_difficulty":1000008,"ntime":1700000229,"order_key":"09:miner-9","p2mr_program_hex":"e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5","share_difficulty":2,"share_id":"share-229","share_seq":229,"template_height":800007},{"accepted_at_ms":999979,"credit_policy":null,"job_id":"job-3003","job_issued_at_ms":999984,"miner_id":"miner-0","network_difficulty":1000011,"ntime":1700003003,"order_key":"00:miner-0","p2mr_program_hex":"bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb","share_difficulty":101,"share_id":"share-3003","share_seq":3003,"template_height":800093},{"accepted_at_ms":1000377,"credit_policy":"stale-grace","job_id":"job-3132","job_issued_at_ms":999996,"miner_id":"miner-8","network_difficulty":1000004,"ntime":1700003132,"order_key":"08:miner-8","p2mr_program_hex":"3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c","share_difficulty":7,"share_id":"share-3132","share_seq":3132,"template_height":800097},{"accepted_at_ms":1000237,"credit_policy":null,"job_id":"job-2624","job_issued_at_ms":999999,"miner_id":"miner-6","network_difficulty":1000006,"ntime":1700002624,"order_key":"06:miner-6","p2mr_program_hex":"4040404040404040404040404040404040404040404040404040404040404040","share_difficulty":2,"share_id":"share-2624","share_seq":2624,"template_height":800082},{"accepted_at_ms":1000072,"credit_policy":null,"job_id":"job-1720","job_issued_at_ms":999981,"miner_id":"miner-4","network_difficulty":1000003,"ntime":1700001720,"order_key":"04:miner-4","p2mr_program_hex":"b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8","share_difficulty":3,"share_id":"share-1720","share_seq":1720,"template_height":800053},{"accepted_at_ms":999974,"credit_policy":"stale-grace","job_id":"job-3071","job_issued_at_ms":999998,"miner_id":"miner-2","network_difficulty":1000011,"ntime":1700003071,"order_key":"02:miner-2","p2mr_program_hex":"ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff","share_difficulty":1000,"share_id":"share-3071","share_seq":3071,"template_height":800095},{"accepted_at_ms":999994,"credit_policy":null,"job_id":"job-1534","job_issued_at_ms":999973,"miner_id":"miner-5","network_difficulty":1000004,"ntime":1700001534,"order_key":"05:miner-5","p2mr_program_hex":"fefefefefefefefefefefefefefefefefefefefefefefefefefefefefefefefe","share_difficulty":19,"share_id":"share-1534","share_seq":1534,"template_height":800047},{"accepted_at_ms":999985,"credit_policy":null,"job_id":"job-1647","job_issued_at_ms":999985,"miner_id":"miner-8","network_difficulty":1000015,"ntime":1700001647,"order_key":"08:miner-8","p2mr_program_hex":"6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f","share_difficulty":2,"share_id":"share-1647","share_seq":1647,"template_height":800051},{"accepted_at_ms":999992,"credit_policy":null,"job_id":"job-1794","job_issued_at_ms":999981,"miner_id":"miner-1","network_difficulty":1000009,"ntime":1700001794,"order_key":"01:miner-1","p2mr_program_hex":"0202020202020202020202020202020202020202020202020202020202020202","share_difficulty":1000,"share_id":"share-1794","share_seq":1794,"template_height":800056},{"accepted_at_ms":999982,"credit_policy":null,"job_id":"job-783","job_issued_at_ms":999974,"miner_id":"miner-2","network_difficulty":1000001,"ntime":1700000783,"order_key":"02:miner-2","p2mr_program_hex":"0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f","share_difficulty":3,"share_id":"share-783","share_seq":783,"template_height":800024},{"accepted_at_ms":999980,"credit_policy":null,"job_id":"job-3378","job_issued_at_ms":999970,"miner_id":"miner-1","network_difficulty":1000012,"ntime":1700003378,"order_key":"01:miner-1","p2mr_program_hex":"3232323232323232323232323232323232323232323232323232323232323232","share_difficulty":7,"share_id":"share-3378","share_seq":3378,"template_height":800105},{"accepted_at_ms":999994,"credit_policy":null,"job_id":"job-2359","job_issued_at_ms":999977,"miner_id":"miner-5","network_difficulty":1000013,"ntime":1700002359,"order_key":"05:miner-5","p2mr_program_hex":"3737373737373737373737373737373737373737373737373737373737373737","share_difficulty":3,"share_id":"share-2359","share_seq":2359,"template_height":800073},{"accepted_at_ms":999989,"credit_policy":null,"job_id":"job-340","job_issued_at_ms":999984,"miner_id":"miner-10","network_difficulty":1000000,"ntime":1700000340,"order_key":"10:miner-10","p2mr_program_hex":"5454545454545454545454545454545454545454545454545454545454545454","share_difficulty":7,"share_id":"share-340","share_seq":340,"template_height":800010},{"accepted_at_ms":999970,"credit_policy":null,"job_id":"job-1939","job_issued_at_ms":999972,"miner_id":"miner-3","network_difficulty":1000001,"ntime":1700001939,"order_key":"03:miner-3","p2mr_program_hex":"9393939393939393939393939393939393939393939393939393939393939393","share_difficulty":2,"share_id":"share-1939","share_seq":1939,"template_height":800060},{"accepted_at_ms":999988,"credit_policy":null,"job_id":"job-3246","job_issued_at_ms":999973,"miner_id":"miner-1","network_difficulty":1000016,"ntime":1700003246,"order_key":"01:miner-1","p2mr_program_hex":"aeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeae","share_difficulty":2,"share_id":"share-3246","share_seq":3246,"template_height":800101},{"accepted_at_ms":999985,"credit_policy":null,"job_id":"job-2127","job_issued_at_ms":1000000,"miner_id":"miner-4","network_difficulty":1000002,"ntime":1700002127,"order_key":"04:miner-4","p2mr_program_hex":"4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f","share_difficulty":1000,"share_id":"share-2127","share_seq":2127,"template_height":800066},{"accepted_at_ms":999991,"credit_policy":null,"job_id":"job-1949","job_issued_at_ms":999970,"miner_id":"miner-2","network_difficulty":1000011,"ntime":1700001949,"order_key":"02:miner-2","p2mr_program_hex":"9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d","share_difficulty":19,"share_id":"share-1949","share_seq":1949,"template_height":800060},{"accepted_at_ms":999998,"credit_policy":null,"job_id":"job-734","job_issued_at_ms":999978,"miner_id":"miner-8","network_difficulty":1000003,"ntime":1700000734,"order_key":"08:miner-8","p2mr_program_hex":"dededededededededededededededededededededededededededededededede","share_difficulty":19,"share_id":"share-734","share_seq":734,"template_height":800022},{"accepted_at_ms":999979,"credit_policy":"stale-grace","job_id":"job-3896","job_issued_at_ms":999991,"miner_id":"miner-2","network_difficulty":1000003,"ntime":1700003896,"order_key":"02:miner-2","p2mr_program_hex":"3838383838383838383838383838383838383838383838383838383838383838","share_difficulty":19,"share_id":"share-3896","share_seq":3896,"template_height":800121},{"accepted_at_ms":999985,"credit_policy":null,"job_id":"job-1236","job_issued_at_ms":999978,"miner_id":"miner-4","network_difficulty":1000012,"ntime":1700001236,"order_key":"04:miner-4","p2mr_program_hex":"d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4","share_difficulty":2,"share_id":"share-1236","share_seq":1236,"template_height":800038},{"accepted_at_ms":999976,"credit_policy":"stale-grace","job_id":"job-1904","job_issued_at_ms":999991,"miner_id":"miner-1","network_difficulty":1000000,"ntime":1700001904,"order_key":"01:miner-1","p2mr_program_hex":"7070707070707070707070707070707070707070707070707070707070707070","share_difficulty":3,"share_id":"share-1904","share_seq":1904,"template_height":800059},{"accepted_at_ms":999981,"credit_policy":null,"job_id":"job-1576","job_issued_at_ms":999973,"miner_id":"miner-3","network_difficulty":1000012,"ntime":1700001576,"order_key":"03:miner-3","p2mr_program_hex":"2828282828282828282828282828282828282828282828282828282828282828","share_difficulty":19,"share_id":"share-1576","share_seq":1576,"template_height":800049},{"accepted_at_ms":999977,"credit_policy":null,"job_id":"job-1611","job_issued_at_ms":999989,"miner_id":"miner-5","network_difficulty":1000013,"ntime":1700001611,"order_key":"05:miner-5","p2mr_program_hex":"4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b","share_difficulty":1000,"share_id":"share-1611","share_seq":1611,"template_height":800050},{"accepted_at_ms":999983,"credit_policy":null,"job_id":"job-609","job_issued_at_ms":999973,"miner_id":"miner-4","network_difficulty":1000014,"ntime":1700000609,"order_key":"04:miner-4","p2mr_program_hex":"6161616161616161616161616161616161616161616161616161616161616161","share_difficulty":1000,"share_id":"share-609","share_seq":609,"template_height":800019},{"accepted_at_ms":999992,"credit_policy":null,"job_id":"job-3018","job_issued_at_ms":999996,"miner_id":"miner-4","network_difficulty":1000009,"ntime":1700003018,"order_key":"04:miner-4","p2mr_program_hex":"cacacacacacacacacacacacacacacacacacacacacacacacacacacacacacacaca","share_difficulty":19,"share_id":"share-3018","share_seq":3018,"template_height":800094},{"accepted_at_ms":999984,"credit_policy":null,"job_id":"job-1032","job_issued_at_ms":999988,"miner_id":"miner-9","network_difficulty":1000012,"ntime":1700001032,"order_key":"09:miner-9","p2mr_program_hex":"0808080808080808080808080808080808080808080808080808080808080808","share_difficulty":1,"share_id":"share-1032","share_seq":1032,"template_height":800032},{"accepted_at_ms":999999,"credit_policy":null,"job_id":"job-1254","job_issued_at_ms":999989,"miner_id":"miner-0","network_difficulty":1000013,"ntime":1700001254,"order_key":"00:miner-0","p2mr_program_hex":"e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6","share_difficulty":1000,"share_id":"share-1254","share_seq":1254,"template_height":800039},{"accepted_at_ms":999992,"credit_policy":"stale-grace","job_id":"job-2219","job_issued_at_ms":999987,"miner_id":"miner-8","network_difficulty":1000009,"ntime":1700002219,"order_key":"08:miner-8","p2mr_program_hex":"abababababababababababababababababababababababababababababababab","share_difficulty":19,"share_id":"share-2219","share_seq":2219,"template_height":800069},{"accepted_at_ms":999973,"credit_policy":null,"job_id":"job-3898","job_issued_at_ms":999989,"miner_id":"miner-4","network_difficulty":1000005,"ntime":1700003898,"order_key":"04:miner-4","p2mr_program_hex":"3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a","share_difficulty":7,"share_id":"share-3898","share_seq":3898,"template_height":800121},{"accepted_at_ms":999995,"credit_policy":null,"job_id":"job-1538","job_issued_at_ms":999970,"miner_id":"miner-9","network_difficulty":1000008,"ntime":1700001538,"order_key":"09:miner-9","p2mr_program_hex":"0202020202020202020202020202020202020202020202020202020202020202","share_difficulty":101,"share_id":"share-1538","share_seq":1538,"template_height":800048},{"accepted_at_ms":999990,"credit_policy":null,"job_id":"job-857","job_issued_at_ms":1000158,"miner_id":"miner-10","network_difficulty":1000007,"ntime":1700000857,"order_key":"10:miner-10","p2mr_program_hex":"5959595959595959595959595959595959595959595959595959595959595959","share_difficulty":101,"share_id":"share-857","share_seq":857,"template_height":800026},{"accepted_at_ms":999997,"credit_policy":null,"job_id":"job-1133","job_issued_at_ms":999982,"miner_id":"miner-0","network_difficulty":1000011,"ntime":1700001133,"order_key":"00:miner-0","p2mr_program_hex":"6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d","share_difficulty":101,"share_id":"share-1133","share_seq":1133,"template_height":800035},{"accepted_at_ms":999989,"credit_policy":null,"job_id":"job-2279","job_issued_at_ms":999973,"miner_id":"miner-2","network_difficulty":1000001,"ntime":1700002279,"order_key":"02:miner-2","p2mr_program_hex":"e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7","share_difficulty":101,"share_id":"share-2279","share_seq":2279,"template_height":800071},{"accepted_at_ms":999976,"credit_policy":null,"job_id":"job-1963","job_issued_at_ms":999984,"miner_id":"miner-5","network_difficulty":1000008,"ntime":1700001963,"order_key":"05:miner-5","p2mr_program_hex":"abababababababababababababababababababababababababababababababab","share_difficulty":19,"share_id":"share-1963","share_seq":1963,"template_height":800061},{"accepted_at_ms":1000318,"credit_policy":null,"job_id":"job-982","job_issued_at_ms":1000314,"miner_id":"miner-3","network_difficulty":1000013,"ntime":1700000982,"order_key":"03:miner-3","p2mr_program_hex":"d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6","share_difficulty":101,"share_id":"share-982","share_seq":982,"template_height":800030},{"accepted_at_ms":999974,"credit_policy":null,"job_id":"job-905","job_issued_at_ms":999998,"miner_id":"miner-3","network_difficulty":1000004,"ntime":1700000905,"order_key":"03:miner-3","p2mr_program_hex":"8989898989898989898989898989898989898989898989898989898989898989","share_difficulty":19,"share_id":"share-905","share_seq":905,"template_height":800028},{"accepted_at_ms":999985,"credit_policy":"stale-grace","job_id":"job-2896","job_issued_at_ms":999994,"miner_id":"miner-3","network_difficulty":1000006,"ntime":1700002896,"order_key":"03:miner-3","p2mr_program_hex":"5050505050505050505050505050505050505050505050505050505050505050","share_difficulty":1000,"share_id":"share-2896","share_seq":2896,"template_height":800090},{"accepted_at_ms":1000097,"credit_policy":null,"job_id":"job-108","job_issued_at_ms":1000351,"miner_id":"miner-9","network_difficulty":1000006,"ntime":1700000108,"order_key":"09:miner-9","p2mr_program_hex":"6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c","share_difficulty":1000,"share_id":"share-108","share_seq":108,"template_height":800003},{"accepted_at_ms":999984,"credit_policy":null,"job_id":"job-1748","job_issued_at_ms":999991,"miner_id":"miner-10","network_difficulty":1000014,"ntime":1700001748,"order_key":"10:miner-10","p2mr_program_hex":"d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4","share_difficulty":19,"share_id":"share-1748","share_seq":1748,"template_height":800054},{"accepted_at_ms":1000000,"credit_policy":"stale-grace","job_id":"job-2879","job_issued_at_ms":999993,"miner_id":"miner-8","network_difficulty":1000006,"ntime":1700002879,"order_key":"08:miner-8","p2mr_program_hex":"3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f","share_difficulty":1000,"share_id":"share-2879","share_seq":2879,"template_height":800089},{"accepted_at_ms":999999,"credit_policy":null,"job_id":"job-2723","job_issued_at_ms":999978,"miner_id":"miner-6","network_difficulty":1000003,"ntime":1700002723,"order_key":"06:miner-6","p2mr_program_hex":"a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3","share_difficulty":101,"share_id":"share-2723","share_seq":2723,"template_height":800085},{"accepted_at_ms":999977,"credit_policy":null,"job_id":"job-2493","job_issued_at_ms":999975,"miner_id":"miner-7","network_difficulty":1000011,"ntime":1700002493,"order_key":"07:miner-7","p2mr_program_hex":"bdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbd","share_difficulty":3,"share_id":"share-2493","share_seq":2493,"template_height":800077},{"accepted_at_ms":999995,"credit_policy":null,"job_id":"job-410","job_issued_at_ms":999997,"miner_id":"miner-3","network_difficulty":1000002,"ntime":1700000410,"order_key":"03:miner-3","p2mr_program_hex":"9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a","share_difficulty":2,"share_id":"share-410","share_seq":410,"template_height":800012},{"accepted_at_ms":999985,"credit_policy":null,"job_id":"job-2209","job_issued_at_ms":999979,"miner_id":"miner-9","network_difficulty":1000016,"ntime":1700002209,"order_key":"09:miner-9","p2mr_program_hex":"a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1","share_difficulty":1000,"share_id":"share-2209","share_seq":2209,"template_height":800069},{"accepted_at_ms":1000000,"credit_policy":null,"job_id":"job-3133","job_issued_at_ms":999987,"miner_id":"miner-9","network_difficulty":1000005,"ntime":1700003133,"order_key":"09:miner-9","p2mr_program_hex":"3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d","share_difficulty":1000,"share_id":"share-3133","share_seq":3133,"template_height":800097},{"accepted_at_ms":999975,"credit_policy":null,"job_id":"job-2687","job_issued_at_ms":999999,"miner_id":"miner-3","network_difficulty":1000001,"ntime":1700002687,"order_key":"03:miner-3","p2mr_program_hex":"7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f","share_difficulty":101,"share_id":"share-2687","share_seq":2687,"template_height":800083},{"accepted_at_ms":1000365,"credit_policy":null,"job_id":"job-2676","job_issued_at_ms":1000339,"miner_id":"miner-3","network_difficulty":1000007,"ntime":1700002676,"order_key":"03:miner-3","p2mr_program_hex":"7474747474747474747474747474747474747474747474747474747474747474","share_difficulty":1,"share_id":"share-2676","share_seq":2676,"template_height":800083},{"accepted_at_ms":999978,"credit_policy":null,"job_id":"job-3432","job_issued_at_ms":999972,"miner_id":"miner-0","network_difficulty":1000015,"ntime":1700003432,"order_key":"00:miner-0","p2mr_program_hex":"6868686868686868686868686868686868686868686868686868686868686868","share_difficulty":1,"share_id":"share-3432","share_seq":3432,"template_height":800107},{"accepted_at_ms":999995,"credit_policy":"stale-grace","job_id":"job-2697","job_issued_at_ms":999990,"miner_id":"miner-2","network_difficulty":1000011,"ntime":1700002697,"order_key":"02:miner-2","p2mr_program_hex":"8989898989898989898989898989898989898989898989898989898989898989","share_difficulty":1,"share_id":"share-2697","share_seq":2697,"template_height":800084},{"accepted_at_ms":999978,"credit_policy":null,"job_id":"job-130","job_issued_at_ms":999977,"miner_id":"miner-9","network_difficulty":1000011,"ntime":1700000130,"order_key":"09:miner-9","p2mr_program_hex":"8282828282828282828282828282828282828282828282828282828282828282","share_difficulty":1000,"share_id":"share-130","share_seq":130,"template_height":800004},{"accepted_at_ms":999994,"credit_policy":"stale-grace","job_id":"job-1803","job_issued_at_ms":999986,"miner_id":"miner-10","network_difficulty":1000001,"ntime":1700001803,"order_key":"10:miner-10","p2mr_program_hex":"0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b","share_difficulty":2,"share_id":"share-1803","share_seq":1803,"template_height":800056},{"accepted_at_ms":999971,"credit_policy":null,"job_id":"job-2302","job_issued_at_ms":999988,"miner_id":"miner-3","network_difficulty":1000007,"ntime":1700002302,"order_key":"03:miner-3","p2mr_program_hex":"fefefefefefefefefefefefefefefefefefefefefefefefefefefefefefefefe","share_difficulty":2,"share_id":"share-2302","share_seq":2302,"template_height":800071},{"accepted_at_ms":999990,"credit_policy":null,"job_id":"job-3067","job_issued_at_ms":999995,"miner_id":"miner-9","network_difficulty":1000007,"ntime":1700003067,"order_key":"09:miner-9","p2mr_program_hex":"fbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfb","share_difficulty":7,"share_id":"share-3067","share_seq":3067,"template_height":800095},{"accepted_at_ms":999985,"credit_policy":null,"job_id":"job-2672","job_issued_at_ms":999998,"miner_id":"miner-10","network_difficulty":1000003,"ntime":1700002672,"order_key":"10:miner-10","p2mr_program_hex":"7070707070707070707070707070707070707070707070707070707070707070","share_difficulty":3,"share_id":"share-2672","share_seq":2672,"template_height":800083},{"accepted_at_ms":1000343,"credit_policy":null,"job_id":"job-2640","job_issued_at_ms":999997,"miner_id":"miner-0","network_difficulty":1000005,"ntime":1700002640,"order_key":"00:miner-0","p2mr_program_hex":"5050505050505050505050505050505050505050505050505050505050505050","share_difficulty":19,"share_id":"share-2640","share_seq":2640,"template_height":800082},{"accepted_at_ms":999978,"credit_policy":null,"job_id":"job-1533","job_issued_at_ms":999983,"miner_id":"miner-4","network_difficulty":1000003,"ntime":1700001533,"order_key":"04:miner-4","p2mr_program_hex":"fdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfd","share_difficulty":3,"share_id":"share-1533","share_seq":1533,"template_height":800047},{"accepted_at_ms":999992,"credit_policy":null,"job_id":"job-846","job_issued_at_ms":999971,"miner_id":"miner-10","network_difficulty":1000013,"ntime":1700000846,"order_key":"10:miner-10","p2mr_program_hex":"4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e","share_difficulty":1,"share_id":"share-846","share_seq":846,"template_height":800026},{"accepted_at_ms":999995,"credit_policy":null,"job_id":"job-1758","job_issued_at_ms":999997,"miner_id":"miner-9","network_difficulty":1000007,"ntime":1700001758,"order_key":"09:miner-9","p2mr_program_hex":"dededededededededededededededededededededededededededededededede","share_difficulty":19,"share_id":"share-1758","share_seq":1758,"template_height":800054},{"accepted_at_ms":999971,"credit_policy":null,"job_id":"job-1656","job_issued_at_ms":999996,"miner_id":"miner-6","network_difficulty":1000007,"ntime":1700001656,"order_key":"06:miner-6","p2mr_program_hex":"7878787878787878787878787878787878787878787878787878787878787878","share_difficulty":1,"share_id":"share-1656","share_seq":1656,"template_height":800051},{"accepted_at_ms":999991,"credit_policy":null,"job_id":"job-2551","job_issued_at_ms":999970,"miner_id":"miner-10","network_difficulty":1000001,"ntime":1700002551,"order_key":"10:miner-10","p2mr_program_hex":"f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7","share_difficulty":3,"share_id":"share-2551","share_seq":2551,"template_height":800079},{"accepted_at_ms":999983,"credit_policy":"stale-grace","job_id":"job-1075","job_issued_at_ms":999999,"miner_id":"miner-8","network_difficulty":1000004,"ntime":1700001075,"order_key":"08:miner-8","p2mr_program_hex":"3333333333333333333333333333333333333333333333333333333333333333","share_difficulty":1000,"share_id":"share-1075","share_seq":1075,"template_height":800033},{"accepted_at_ms":999975,"credit_policy":null,"job_id":"job-1155","job_issued_at_ms":999976,"miner_id":"miner-0","network_difficulty":1000016,"ntime":1700001155,"order_key":"00:miner-0","p2mr_program_hex":"8383838383838383838383838383838383838383838383838383838383838383","share_difficulty":3,"share_id":"share-1155","share_seq":1155,"template_height":800036},{"accepted_at_ms":999993,"credit_policy":"stale-grace","job_id":"job-443","job_issued_at_ms":999990,"miner_id":"miner-3","network_difficulty":1000001,"ntime":1700000443,"order_key":"03:miner-3","p2mr_program_hex":"bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb","share_difficulty":7,"share_id":"share-443","share_seq":443,"template_height":800013},{"accepted_at_ms":999998,"credit_policy":null,"job_id":"job-2667","job_issued_at_ms":999974,"miner_id":"miner-5","network_difficulty":1000015,"ntime":1700002667,"order_key":"05:miner-5","p2mr_program_hex":"6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b","share_difficulty":1,"share_id":"share-2667","share_seq":2667,"template_height":800083},{"accepted_at_ms":999979,"credit_policy":null,"job_id":"job-3759","job_issued_at_ms":999983,"miner_id":"miner-8","network_difficulty":1000002,"ntime":1700003759,"order_key":"08:miner-8","p2mr_program_hex":"afafafafafafafafafafafafafafafafafafafafafafafafafafafafafafafaf","share_difficulty":3,"share_id":"share-3759","share_seq":3759,"template_height":800117},{"accepted_at_ms":999983,"credit_policy":null,"job_id":"job-3048","job_issued_at_ms":999992,"miner_id":"miner-1","network_difficulty":1000005,"ntime":1700003048,"order_key":"01:miner-1","p2mr_program_hex":"e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8","share_difficulty":101,"share_id":"share-3048","share_seq":3048,"template_height":800095},{"accepted_at_ms":999997,"credit_policy":null,"job_id":"job-2570","job_issued_at_ms":999981,"miner_id":"miner-7","network_difficulty":1000003,"ntime":1700002570,"order_key":"07:miner-7","p2mr_program_hex":"0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a","share_difficulty":1,"share_id":"share-2570","share_seq":2570,"template_height":800080},{"accepted_at_ms":999990,"credit_policy":null,"job_id":"job-2939","job_issued_at_ms":999984,"miner_id":"miner-2","network_difficulty":1000015,"ntime":1700002939,"order_key":"02:miner-2","p2mr_program_hex":"7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b","share_difficulty":7,"share_id":"share-2939","share_seq":2939,"template_height":800091},{"accepted_at_ms":999980,"credit_policy":null,"job_id":"job-850","job_issued_at_ms":999998,"miner_id":"miner-3","network_difficulty":1000000,"ntime":1700000850,"order_key":"03:miner-3","p2mr_program_hex":"5252525252525252525252525252525252525252525252525252525252525252","share_difficulty":19,"share_id":"share-850","share_seq":850,"template_height":800026},{"accepted_at_ms":999981,"credit_policy":"stale-grace","job_id":"job-2757","job_issued_at_ms":999998,"miner_id":"miner-7","network_difficulty":1000003,"ntime":1700002757,"order_key":"07:miner-7","p2mr_program_hex":"c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5","share_difficulty":1,"share_id":"share-2757","share_seq":2757,"template_height":800086},{"accepted_at_ms":999977,"credit_policy":null,"job_id":"job-1552","job_issued_at_ms":999987,"miner_id":"miner-1","network_difficulty":1000005,"ntime":1700001552,"order_key":"01:miner-1","p2mr_program_hex":"1010101010101010101010101010101010101010101010101010101010101010","share_difficulty":7,"share_id":"share-1552","share_seq":1552,"template_height":800048},{"accepted_at_ms":1000000,"credit_policy":null,"job_id":"job-3008","job_issued_at_ms":999984,"miner_id":"miner-5","network_difficulty":1000016,"ntime":1700003008,"order_key":"05:miner-5","p2mr_program_hex":"c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0","share_difficulty":19,"share_id":"share-3008","share_seq":3008,"template_height":800094},{"accepted_at_ms":999985,"credit_policy":null,"job_id":"job-1092","job_issued_at_ms":999989,"miner_id":"miner-3","network_difficulty":1000004,"ntime":1700001092,"order_key":"03:miner-3","p2mr_program_hex":"4444444444444444444444444444444444444444444444444444444444444444","share_difficulty":7,"share_id":"share-1092","share_seq":1092,"template_height":800034},{"accepted_at_ms":999975,"credit_policy":"stale-grace","job_id":"job-309","job_issued_at_ms":999986,"miner_id":"miner-1","network_difficulty":1000003,"ntime":1700000309,"order_key":"01:miner-1","p2mr_program_hex":"3535353535353535353535353535353535353535353535353535353535353535","share_difficulty":3,"share_id":"share-309","share_seq":309,"template_height":800009},{"accepted_at_ms":999986,"credit_policy":null,"job_id":"job-2480","job_issued_at_ms":999998,"miner_id":"miner-5","network_difficulty":1000015,"ntime":1700002480,"order_key":"05:miner-5","p2mr_program_hex":"b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0","share_difficulty":3,"share_id":"share-2480","share_seq":2480,"template_height":800077},{"accepted_at_ms":999995,"credit_policy":null,"job_id":"job-3200","job_issued_at_ms":999989,"miner_id":"miner-10","network_difficulty":1000004,"ntime":1700003200,"order_key":"10:miner-10","p2mr_program_hex":"8080808080808080808080808080808080808080808080808080808080808080","share_difficulty":2,"share_id":"share-3200","share_seq":3200,"template_height":800100},{"accepted_at_ms":999991,"credit_policy":"stale-grace","job_id":"job-2455","job_issued_at_ms":999989,"miner_id":"miner-2","network_difficulty":1000007,"ntime":1700002455,"order_key":"02:miner-2","p2mr_program_hex":"9797979797979797979797979797979797979797979797979797979797979797","share_difficulty":1000,"share_id":"share-2455","share_seq":2455,"template_height":800076},{"accepted_at_ms":999992,"credit_policy":null,"job_id":"job-1199","job_issued_at_ms":999993,"miner_id":"miner-0","network_difficulty":1000009,"ntime":1700001199,"order_key":"00:miner-0","p2mr_program_hex":"afafafafafafafafafafafafafafafafafafafafafafafafafafafafafafafaf","share_difficulty":2,"share_id":"share-1199","share_seq":1199,"template_height":800037},{"accepted_at_ms":999987,"credit_policy":null,"job_id":"job-2783","job_issued_at_ms":999975,"miner_id":"miner-0","network_difficulty":1000012,"ntime":1700002783,"order_key":"00:miner-0","p2mr_program_hex":"dfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdf","share_difficulty":2,"share_id":"share-2783","share_seq":2783,"template_height":800086},{"accepted_at_ms":1000162,"credit_policy":null,"job_id":"job-2598","job_issued_at_ms":1000358,"miner_id":"miner-2","network_difficulty":1000014,"ntime":1700002598,"order_key":"02:miner-2","p2mr_program_hex":"2626262626262626262626262626262626262626262626262626262626262626","share_difficulty":2,"share_id":"share-2598","share_seq":2598,"template_height":800081},{"accepted_at_ms":999981,"credit_policy":null,"job_id":"job-541","job_issued_at_ms":999981,"miner_id":"miner-2","network_difficulty":1000014,"ntime":1700000541,"order_key":"02:miner-2","p2mr_program_hex":"1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d","share_difficulty":19,"share_id":"share-541","share_seq":541,"template_height":800016},{"accepted_at_ms":999989,"credit_policy":null,"job_id":"job-674","job_issued_at_ms":999989,"miner_id":"miner-3","network_difficulty":1000011,"ntime":1700000674,"order_key":"03:miner-3","p2mr_program_hex":"a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2","share_difficulty":2,"share_id":"share-674","share_seq":674,"template_height":800021},{"accepted_at_ms":999987,"credit_policy":"stale-grace","job_id":"job-1374","job_issued_at_ms":999973,"miner_id":"miner-10","network_difficulty":1000014,"ntime":1700001374,"order_key":"10:miner-10","p2mr_program_hex":"5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e","share_difficulty":2,"share_id":"share-1374","share_seq":1374,"template_height":800042},{"accepted_at_ms":999990,"credit_policy":"stale-grace","job_id":"job-2854","job_issued_at_ms":999979,"miner_id":"miner-5","network_difficulty":1000015,"ntime":1700002854,"order_key":"05:miner-5","p2mr_program_hex":"2626262626262626262626262626262626262626262626262626262626262626","share_difficulty":101,"share_id":"share-2854","share_seq":2854,"template_height":800089},{"accepted_at_ms":999977,"credit_policy":"stale-grace","job_id":"job-2592","job_issued_at_ms":999986,"miner_id":"miner-7","network_difficulty":1000008,"ntime":1700002592,"order_key":"07:miner-7","p2mr_program_hex":"2020202020202020202020202020202020202020202020202020202020202020","share_difficulty":101,"share_id":"share-2592","share_seq":2592,"template_height":800081},{"accepted_at_ms":999999,"credit_policy":null,"job_id":"job-257","job_issued_at_ms":1000038,"miner_id":"miner-4","network_difficulty":1000002,"ntime":1700000257,"order_key":"04:miner-4","p2mr_program_hex":"0101010101010101010101010101010101010101010101010101010101010101","share_difficulty":7,"share_id":"share-257","share_seq":257,"template_height":800008},{"accepted_at_ms":999973,"credit_policy":null,"job_id":"job-1219","job_issued_at_ms":999976,"miner_id":"miner-9","network_difficulty":1000012,"ntime":1700001219,"order_key":"09:miner-9","p2mr_program_hex":"c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3","share_difficulty":101,"share_id":"share-1219","share_seq":1219,"template_height":800038},{"accepted_at_ms":999978,"credit_policy":null,"job_id":"job-3983","job_issued_at_ms":999970,"miner_id":"miner-1","network_difficulty":1000005,"ntime":1700003983,"order_key":"01:miner-1","p2mr_program_hex":"8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f","share_difficulty":3,"share_id":"share-3983","share_seq":3983,"template_height":800124},{"accepted_at_ms":999995,"credit_policy":null,"job_id":"job-3284","job_issued_at_ms":999998,"miner_id":"miner-6","network_difficulty":1000003,"ntime":1700003284,"order_key":"06:miner-6","p2mr_program_hex":"d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4","share_difficulty":1000,"share_id":"share-3284","share_seq":3284,"template_height":800102},{"accepted_at_ms":1000154,"credit_policy":null,"job_id":"job-1435","job_issued_at_ms":999996,"miner_id":"miner-5","network_difficulty":1000007,"ntime":1700001435,"order_key":"05:miner-5","p2mr_program_hex":"9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b","share_difficulty":7,"share_id":"share-1435","share_seq":1435,"template_height":800044},{"accepted_at_ms":999979,"credit_policy":"stale-grace","job_id":"job-1378","job_issued_at_ms":999989,"miner_id":"miner-3","network_difficulty":1000001,"ntime":1700001378,"order_key":"03:miner-3","p2mr_program_hex":"6262626262626262626262626262626262626262626262626262626262626262","share_difficulty":1000,"share_id":"share-1378","share_seq":1378,"template_height":800043},{"accepted_at_ms":999986,"credit_policy":null,"job_id":"job-1290","job_issued_at_ms":999985,"miner_id":"miner-3","network_difficulty":1000015,"ntime":1700001290,"order_key":"03:miner-3","p2mr_program_hex":"0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a","share_difficulty":1000,"share_id":"share-1290","share_seq":1290,"template_height":800040},{"accepted_at_ms":999976,"credit_policy":null,"job_id":"job-1094","job_issued_at_ms":999981,"miner_id":"miner-5","network_difficulty":1000006,"ntime":1700001094,"order_key":"05:miner-5","p2mr_program_hex":"4646464646464646464646464646464646464646464646464646464646464646","share_difficulty":19,"share_id":"share-1094","share_seq":1094,"template_height":800034},{"accepted_at_ms":999975,"credit_policy":"stale-grace","job_id":"job-2496","job_issued_at_ms":999996,"miner_id":"miner-10","network_difficulty":1000014,"ntime":1700002496,"order_key":"10:miner-10","p2mr_program_hex":"c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0","share_difficulty":19,"share_id":"share-2496","share_seq":2496,"template_height":800078},{"accepted_at_ms":999981,"credit_policy":"stale-grace","job_id":"job-3440","job_issued_at_ms":999981,"miner_id":"miner-8","network_difficulty":1000006,"ntime":1700003440,"order_key":"08:miner-8","p2mr_program_hex":"7070707070707070707070707070707070707070707070707070707070707070","share_difficulty":2,"share_id":"share-3440","share_seq":3440,"template_height":800107},{"accepted_at_ms":999973,"credit_policy":null,"job_id":"job-3998","job_issued_at_ms":999981,"miner_id":"miner-5","network_difficulty":1000003,"ntime":1700003998,"order_key":"05:miner-5","p2mr_program_hex":"9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e","share_difficulty":101,"share_id":"share-3998","share_seq":3998,"template_height":800124},{"accepted_at_ms":1000442,"credit_policy":null,"job_id":"job-3580","job_issued_at_ms":1000347,"miner_id":"miner-5","network_difficulty":1000010,"ntime":1700003580,"order_key":"05:miner-5","p2mr_program_hex":"fcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfc","share_difficulty":3,"share_id":"share-3580","share_seq":3580,"template_height":800111},{"accepted_at_ms":999977,"credit_policy":"stale-grace","job_id":"job-681","job_issued_at_ms":999998,"miner_id":"miner-10","network_difficulty":1000001,"ntime":1700000681,"order_key":"10:miner-10","p2mr_program_hex":"a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9","share_difficulty":3,"share_id":"share-681","share_seq":681,"template_height":800021},{"accepted_at_ms":999972,"credit_policy":null,"job_id":"job-3758","job_issued_at_ms":999988,"miner_id":"miner-7","network_difficulty":1000001,"ntime":1700003758,"order_key":"07:miner-7","p2mr_program_hex":"aeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeae","share_difficulty":19,"share_id":"share-3758","share_seq":3758,"template_height":800117},{"accepted_at_ms":999989,"credit_policy":null,"job_id":"job-76","job_issued_at_ms":999975,"miner_id":"miner-10","network_difficulty":1000008,"ntime":1700000076,"order_key":"10:miner-10","p2mr_program_hex":"4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c","share_difficulty":1,"share_id":"share-76","share_seq":76,"template_height":800002},{"accepted_at_ms":999978,"credit_policy":null,"job_id":"job-3890","job_issued_at_ms":999982,"miner_id":"miner-7","network_difficulty":1000014,"ntime":1700003890,"order_key":"07:miner-7","p2mr_program_hex":"3232323232323232323232323232323232323232323232323232323232323232","share_difficulty":19,"share_id":"share-3890","share_seq":3890,"template_height":800121},{"accepted_at_ms":999992,"credit_policy":"stale-grace","job_id":"job-56","job_issued_at_ms":999992,"miner_id":"miner-1","network_difficulty":1000005,"ntime":1700000056,"order_key":"01:miner-1","p2mr_program_hex":"3838383838383838383838383838383838383838383838383838383838383838","share_difficulty":19,"share_id":"share-56","share_seq":56,"template_height":800001},{"accepted_at_ms":999977,"credit_policy":"stale-grace","job_id":"job-2173","job_issued_at_ms":999988,"miner_id":"miner-6","network_difficulty":1000014,"ntime":1700002173,"order_key":"06:miner-6","p2mr_program_hex":"7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d","share_difficulty":3,"share_id":"share-2173","share_seq":2173,"template_height":800067},{"accepted_at_ms":999992,"credit_policy":null,"job_id":"job-2429","job_issued_at_ms":999976,"miner_id":"miner-9","network_difficulty":1000015,"ntime":1700002429,"order_key":"09:miner-9","p2mr_program_hex":"7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d","share_difficulty":3,"share_id":"share-2429","share_seq":2429,"template_height":800075},{"accepted_at_ms":999992,"credit_policy":null,"job_id":"job-2131","job_issued_at_ms":1000000,"miner_id":"miner-8","network_difficulty":1000006,"ntime":1700002131,"order_key":"08:miner-8","p2mr_program_hex":"5353535353535353535353535353535353535353535353535353535353535353","share_difficulty":1,"share_id":"share-2131","share_seq":2131,"template_height":800066},{"accepted_at_ms":999971,"credit_policy":"stale-grace","job_id":"job-2442","job_issued_at_ms":999987,"miner_id":"miner-0","network_difficulty":1000011,"ntime":1700002442,"order_key":"00:miner-0","p2mr_program_hex":"8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a","share_difficulty":3,"share_id":"share-2442","share_seq":2442,"template_height":800076},{"accepted_at_ms":999980,"credit_policy":"stale-grace","job_id":"job-3865","job_issued_at_ms":999983,"miner_id":"miner-4","network_difficulty":1000006,"ntime":1700003865,"order_key":"04:miner-4","p2mr_program_hex":"1919191919191919191919191919191919191919191919191919191919191919","share_difficulty":7,"share_id":"share-3865","share_seq":3865,"template_height":800120},{"accepted_at_ms":999988,"credit_policy":null,"job_id":"job-3247","job_issued_at_ms":999971,"miner_id":"miner-2","network_difficulty":1000000,"ntime":1700003247,"order_key":"02:miner-2","p2mr_program_hex":"afafafafafafafafafafafafafafafafafafafafafafafafafafafafafafafaf","share_difficulty":101,"share_id":"share-3247","share_seq":3247,"template_height":800101},{"accepted_at_ms":999981,"credit_policy":null,"job_id":"job-498","job_issued_at_ms":999972,"miner_id":"miner-3","network_difficulty":1000005,"ntime":1700000498,"order_key":"03:miner-3","p2mr_program_hex":"f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2","share_difficulty":1,"share_id":"share-498","share_seq":498,"template_height":800015},{"accepted_at_ms":999998,"credit_policy":null,"job_id":"job-1682","job_issued_at_ms":999982,"miner_id":"miner-10","network_difficulty":1000016,"ntime":1700001682,"order_key":"10:miner-10","p2mr_program_hex":"9292929292929292929292929292929292929292929292929292929292929292","share_difficulty":19,"share_id":"share-1682","share_seq":1682,"template_height":800052},{"accepted_at_ms":999971,"credit_policy":"stale-grace","job_id":"job-1124","job_issued_at_ms":999996,"miner_id":"miner-2","network_difficulty":1000002,"ntime":1700001124,"order_key":"02:miner-2","p2mr_program_hex":"6464646464646464646464646464646464646464646464646464646464646464","share_difficulty":101,"share_id":"share-1124","share_seq":1124,"template_height":800035},{"accepted_at_ms":999978,"credit_policy":null,"job_id":"job-1443","job_issued_at_ms":999993,"miner_id":"miner-2","network_difficulty":1000015,"ntime":1700001443,"order_key":"02:miner-2","p2mr_program_hex":"a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3","share_difficulty":19,"share_id":"share-1443","share_seq":1443,"template_height":800045},{"accepted_at_ms":999976,"credit_policy":"stale-grace","job_id":"job-669","job_issued_at_ms":999978,"miner_id":"miner-9","network_difficulty":1000006,"ntime":1700000669,"order_key":"09:miner-9","p2mr_program_hex":"9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d","share_difficulty":1000,"share_id":"share-669","share_seq":669,"template_height":800020},{"accepted_at_ms":999987,"credit_policy":null,"job_id":"job-1021","job_issued_at_ms":999977,"miner_id":"miner-9","network_difficulty":1000001,"ntime":1700001021,"order_key":"09:miner-9","p2mr_program_hex":"fdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfd","share_difficulty":1,"share_id":"share-1021","share_seq":1021,"template_height":800031},{"accepted_at_ms":1000000,"credit_policy":null,"job_id":"job-1291","job_issued_at_ms":1000000,"miner_id":"miner-4","network_difficulty":1000016,"ntime":1700001291,"order_key":"04:miner-4","p2mr_program_hex":"0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b","share_difficulty":3,"share_id":"share-1291","share_seq":1291,"template_height":800040},{"accepted_at_ms":999977,"credit_policy":null,"job_id":"job-618","job_issued_at_ms":999983,"miner_id":"miner-2","network_difficulty":1000006,"ntime":1700000618,"order_key":"02:miner-2","p2mr_program_hex":"6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a","share_difficulty":3,"share_id":"share-618","share_seq":618,"template_height":800019},{"accepted_at_ms":1000250,"credit_policy":"stale-grace","job_id":"job-3678","job_issued_at_ms":1000205,"miner_id":"miner-4","network_difficulty":1000006,"ntime":1700003678,"order_key":"04:miner-4","p2mr_program_hex":"5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e","share_difficulty":1000,"share_id":"share-3678","share_seq":3678,"template_height":800114},{"accepted_at_ms":999987,"credit_policy":null,"job_id":"job-402","job_issued_at_ms":999975,"miner_id":"miner-6","network_difficulty":1000011,"ntime":1700000402,"order_key":"06:miner-6","p2mr_program_hex":"9292929292929292929292929292929292929292929292929292929292929292","share_difficulty":7,"share_id":"share-402","share_seq":402,"template_height":800012},{"accepted_at_ms":999994,"credit_policy":null,"job_id":"job-2932","job_issued_at_ms":999983,"miner_id":"miner-6","network_difficulty":1000008,"ntime":1700002932,"order_key":"06:miner-6","p2mr_program_hex":"7474747474747474747474747474747474747474747474747474747474747474","share_difficulty":1,"share_id":"share-2932","share_seq":2932,"template_height":800091},{"accepted_at_ms":999985,"credit_policy":null,"job_id":"job-2435","job_issued_at_ms":999993,"miner_id":"miner-4","network_difficulty":1000004,"ntime":1700002435,"order_key":"04:miner-4","p2mr_program_hex":"8383838383838383838383838383838383838383838383838383838383838383","share_difficulty":3,"share_id":"share-2435","share_seq":2435,"template_height":800076},{"accepted_at_ms":999984,"credit_policy":null,"job_id":"job-2293","job_issued_at_ms":999983,"miner_id":"miner-5","network_difficulty":1000015,"ntime":1700002293,"order_key":"05:miner-5","p2mr_program_hex":"f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5","share_difficulty":1,"share_id":"share-2293","share_seq":2293,"template_height":800071},{"accepted_at_ms":1000000,"credit_policy":null,"job_id":"job-82","job_issued_at_ms":999988,"miner_id":"miner-5","network_difficulty":1000014,"ntime":1700000082,"order_key":"05:miner-5","p2mr_program_hex":"5252525252525252525252525252525252525252525252525252525252525252","share_difficulty":2,"share_id":"share-82","share_seq":82,"template_height":800002},{"accepted_at_ms":999981,"credit_policy":null,"job_id":"job-325","job_issued_at_ms":999986,"miner_id":"miner-6","network_difficulty":1000002,"ntime":1700000325,"order_key":"06:miner-6","p2mr_program_hex":"4545454545454545454545454545454545454545454545454545454545454545","share_difficulty":1000,"share_id":"share-325","share_seq":325,"template_height":800010},{"accepted_at_ms":999995,"credit_policy":null,"job_id":"job-2660","job_issued_at_ms":999980,"miner_id":"miner-9","network_difficulty":1000008,"ntime":1700002660,"order_key":"09:miner-9","p2mr_program_hex":"6464646464646464646464646464646464646464646464646464646464646464","share_difficulty":2,"share_id":"share-2660","share_seq":2660,"template_height":800083},{"accepted_at_ms":999974,"credit_policy":"stale-grace","job_id":"job-1380","job_issued_at_ms":999988,"miner_id":"miner-5","network_difficulty":1000003,"ntime":1700001380,"order_key":"05:miner-5","p2mr_program_hex":"6464646464646464646464646464646464646464646464646464646464646464","share_difficulty":2,"share_id":"share-1380","share_seq":1380,"template_height":800043},{"accepted_at_ms":999971,"credit_policy":null,"job_id":"job-557","job_issued_at_ms":999983,"miner_id":"miner-7","network_difficulty":1000013,"ntime":1700000557,"order_key":"07:miner-7","p2mr_program_hex":"2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d","share_difficulty":101,"share_id":"share-557","share_seq":557,"template_height":800017},{"accepted_at_ms":999983,"credit_policy":null,"job_id":"job-2582","job_issued_at_ms":999991,"miner_id":"miner-8","network_difficulty":1000015,"ntime":1700002582,"order_key":"08:miner-8","p2mr_program_hex":"1616161616161616161616161616161616161616161616161616161616161616","share_difficulty":7,"share_id":"share-2582","share_seq":2582,"template_height":800080},{"accepted_at_ms":999982,"credit_policy":"stale-grace","job_id":"job-3069","job_issued_at_ms":999992,"miner_id":"miner-0","network_difficulty":1000009,"ntime":1700003069,"order_key":"00:miner-0","p2mr_program_hex":"fdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfd","share_difficulty":7,"share_id":"share-3069","share_seq":3069,"template_height":800095},{"accepted_at_ms":999992,"credit_policy":null,"job_id":"job-2710","job_issued_at_ms":999985,"miner_id":"miner-4","network_difficulty":1000007,"ntime":1700002710,"order_key":"04:miner-4","p2mr_program_hex":"9696969696969696969696969696969696969696969696969696969696969696","share_difficulty":1000,"share_id":"share-2710","share_seq":2710,"template_height":800084},{"accepted_at_ms":999993,"credit_policy":null,"job_id":"job-2011","job_issued_at_ms":999982,"miner_id":"miner-9","network_difficulty":1000005,"ntime":1700002011,"order_key":"09:miner-9","p2mr_program_hex":"dbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdb","share_difficulty":19,"share_id":"share-2011","share_seq":2011,"template_height":800062},{"accepted_at_ms":999978,"credit_policy":null,"job_id":"job-1810","job_issued_at_ms":999985,"miner_id":"miner-6","network_difficulty":1000008,"ntime":1700001810,"order_key":"06:miner-6","p2mr_program_hex":"1212121212121212121212121212121212121212121212121212121212121212","share_difficulty":3,"share_id":"share-1810","share_seq":1810,"template_height":800056},{"accepted_at_ms":1000460,"credit_policy":null,"job_id":"job-86","job_issued_at_ms":1000042,"miner_id":"miner-9","network_difficulty":1000001,"ntime":1700000086,"order_key":"09:miner-9","p2mr_program_hex":"5656565656565656565656565656565656565656565656565656565656565656","share_difficulty":1000,"share_id":"share-86","share_seq":86,"template_height":800002},{"accepted_at_ms":999982,"credit_policy":null,"job_id":"job-107","job_issued_at_ms":999988,"miner_id":"miner-8","network_difficulty":1000005,"ntime":1700000107,"order_key":"08:miner-8","p2mr_program_hex":"6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b","share_difficulty":2,"share_id":"share-107","share_seq":107,"template_height":800003},{"accepted_at_ms":999992,"credit_policy":null,"job_id":"job-1746","job_issued_at_ms":999983,"miner_id":"miner-8","network_difficulty":1000012,"ntime":1700001746,"order_key":"08:miner-8","p2mr_program_hex":"d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2","share_difficulty":2,"share_id":"share-1746","share_seq":1746,"template_height":800054},{"accepted_at_ms":999994,"credit_policy":null,"job_id":"job-3578","job_issued_at_ms":999990,"miner_id":"miner-3","network_difficulty":1000008,"ntime":1700003578,"order_key":"03:miner-3","p2mr_program_hex":"fafafafafafafafafafafafafafafafafafafafafafafafafafafafafafafafa","share_difficulty":101,"share_id":"share-3578","share_seq":3578,"template_height":800111},{"accepted_at_ms":999993,"credit_policy":null,"job_id":"job-800","job_issued_at_ms":999979,"miner_id":"miner-8","network_difficulty":1000001,"ntime":1700000800,"order_key":"08:miner-8","p2mr_program_hex":"2020202020202020202020202020202020202020202020202020202020202020","share_difficulty":101,"share_id":"share-800","share_seq":800,"template_height":800025},{"accepted_at_ms":999992,"credit_policy":null,"job_id":"job-916","job_issued_at_ms":999976,"miner_id":"miner-3","network_difficulty":1000015,"ntime":1700000916,"order_key":"03:miner-3","p2mr_program_hex":"9494949494949494949494949494949494949494949494949494949494949494","share_difficulty":19,"share_id":"share-916","share_seq":916,"template_height":800028},{"accepted_at_ms":999980,"credit_policy":null,"job_id":"job-3873","job_issued_at_ms":999991,"miner_id":"miner-1","network_difficulty":1000014,"ntime":1700003873,"order_key":"01:miner-1","p2mr_program_hex":"2121212121212121212121212121212121212121212121212121212121212121","share_difficulty":3,"share_id":"share-3873","share_seq":3873,"template_height":800121},{"accepted_at_ms":999985,"credit_policy":null,"job_id":"job-2201","job_issued_at_ms":999983,"miner_id":"miner-1","network_difficulty":1000008,"ntime":1700002201,"order_key":"01:miner-1","p2mr_program_hex":"9999999999999999999999999999999999999999999999999999999999999999","share_difficulty":19,"share_id":"share-2201","share_seq":2201,"template_height":800068},{"accepted_at_ms":999998,"credit_policy":null,"job_id":"job-386","job_issued_at_ms":1000000,"miner_id":"miner-1","network_difficulty":1000012,"ntime":1700000386,"order_key":"01:miner-1","p2mr_program_hex":"8282828282828282828282828282828282828282828282828282828282828282","share_difficulty":3,"share_id":"share-386","share_seq":386,"template_height":800012},{"accepted_at_ms":999995,"credit_policy":null,"job_id":"job-3843","job_issued_at_ms":999981,"miner_id":"miner-4","network_difficulty":1000001,"ntime":1700003843,"order_key":"04:miner-4","p2mr_program_hex":"0303030303030303030303030303030303030303030303030303030303030303","share_difficulty":1,"share_id":"share-3843","share_seq":3843,"template_height":800120},{"accepted_at_ms":999980,"credit_policy":null,"job_id":"job-3308","job_issued_at_ms":999994,"miner_id":"miner-8","network_difficulty":1000010,"ntime":1700003308,"order_key":"08:miner-8","p2mr_program_hex":"ecececececececececececececececececececececececececececececececec","share_difficulty":1,"share_id":"share-3308","share_seq":3308,"template_height":800103},{"accepted_at_ms":999988,"credit_policy":null,"job_id":"job-1078","job_issued_at_ms":999992,"miner_id":"miner-0","network_difficulty":1000007,"ntime":1700001078,"order_key":"00:miner-0","p2mr_program_hex":"3636363636363636363636363636363636363636363636363636363636363636","share_difficulty":3,"share_id":"share-1078","share_seq":1078,"template_height":800033},{"accepted_at_ms":999999,"credit_policy":null,"job_id":"job-473","job_issued_at_ms":999991,"miner_id":"miner-0","network_difficulty":1000014,"ntime":1700000473,"order_key":"00:miner-0","p2mr_program_hex":"d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9","share_difficulty":1000,"share_id":"share-473","share_seq":473,"template_height":800014},{"accepted_at_ms":1000000,"credit_policy":null,"job_id":"job-342","job_issued_at_ms":999970,"miner_id":"miner-1","network_difficulty":1000002,"ntime":1700000342,"order_key":"01:miner-1","p2mr_program_hex":"5656565656565656565656565656565656565656565656565656565656565656","share_difficulty":3,"share_id":"share-342","share_seq":342,"template_height":800010},{"accepted_at_ms":999994,"credit_policy":null,"job_id":"job-3974","job_issued_at_ms":999985,"miner_id":"miner-3","network_difficulty":1000013,"ntime":1700003974,"order_key":"03:miner-3","p2mr_program_hex":"8686868686868686868686868686868686868686868686868686868686868686","share_difficulty":2,"share_id":"share-3974","share_seq":3974,"template_height":800124},{"accepted_at_ms":999995,"credit_policy":null,"job_id":"job-1052","job_issued_at_ms":1000168,"miner_id":"miner-7","network_difficulty":1000015,"ntime":1700001052,"order_key":"07:miner-7","p2mr_program_hex":"1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c","share_difficulty":101,"share_id":"share-1052","share_seq":1052,"template_height":800032},{"accepted_at_ms":999991,"credit_policy":"stale-grace","job_id":"job-1159","job_issued_at_ms":999994,"miner_id":"miner-4","network_difficulty":1000003,"ntime":1700001159,"order_key":"04:miner-4","p2mr_program_hex":"8787878787878787878787878787878787878787878787878787878787878787","share_difficulty":1000,"share_id":"share-1159","share_seq":1159,"template_height":800036},{"accepted_at_ms":999999,"credit_policy":"stale-grace","job_id":"job-3779","job_issued_at_ms":999975,"miner_id":"miner-6","network_difficulty":1000005,"ntime":1700003779,"order_key":"06:miner-6","p2mr_program_hex":"c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3","share_difficulty":7,"share_id":"share-3779","share_seq":3779,"template_height":800118},{"accepted_at_ms":999990,"credit_policy":null,"job_id":"job-3887","job_issued_at_ms":999987,"miner_id":"miner-4","network_difficulty":1000011,"ntime":1700003887,"order_key":"04:miner-4","p2mr_program_hex":"2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f","share_difficulty":101,"share_id":"share-3887","share_seq":3887,"template_height":800121},{"accepted_at_ms":999979,"credit_policy":"stale-grace","job_id":"job-2688","job_issued_at_ms":999987,"miner_id":"miner-4","network_difficulty":1000002,"ntime":1700002688,"order_key":"04:miner-4","p2mr_program_hex":"8080808080808080808080808080808080808080808080808080808080808080","share_difficulty":19,"share_id":"share-2688","share_seq":2688,"template_height":800084},{"accepted_at_ms":999993,"credit_policy":"stale-grace","job_id":"job-1263","job_issued_at_ms":999983,"miner_id":"miner-9","network_difficulty":1000005,"ntime":1700001263,"order_key":"09:miner-9","p2mr_program_hex":"efefefefefefefefefefefefefefefefefefefefefefefefefefefefefefefef","share_difficulty":1,"share_id":"share-1263","share_seq":1263,"template_height":800039},{"accepted_at_ms":999971,"credit_policy":"stale-grace","job_id":"job-3172","job_issued_at_ms":999995,"miner_id":"miner-4","network_difficulty":1000010,"ntime":1700003172,"order_key":"04:miner-4","p2mr_program_hex":"6464646464646464646464646464646464646464646464646464646464646464","share_difficulty":3,"share_id":"share-3172","share_seq":3172,"template_height":800099},{"accepted_at_ms":999979,"credit_policy":null,"job_id":"job-421","job_issued_at_ms":1000000,"miner_id":"miner-3","network_difficulty":1000013,"ntime":1700000421,"order_key":"03:miner-3","p2mr_program_hex":"a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5","share_difficulty":7,"share_id":"share-421","share_seq":421,"template_height":800013},{"accepted_at_ms":999982,"credit_policy":null,"job_id":"job-1920","job_issued_at_ms":999978,"miner_id":"miner-6","network_difficulty":1000016,"ntime":1700001920,"order_key":"06:miner-6","p2mr_program_hex":"8080808080808080808080808080808080808080808080808080808080808080","share_difficulty":7,"share_id":"share-1920","share_seq":1920,"template_height":800060},{"accepted_at_ms":999975,"credit_policy":null,"job_id":"job-3178","job_issued_at_ms":999973,"miner_id":"miner-10","network_difficulty":1000016,"ntime":1700003178,"order_key":"10:miner-10","p2mr_program_hex":"6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a","share_difficulty":101,"share_id":"share-3178","share_seq":3178,"template_height":800099},{"accepted_at_ms":999978,"credit_policy":"stale-grace","job_id":"job-411","job_issued_at_ms":999993,"miner_id":"miner-4","network_difficulty":1000003,"ntime":1700000411,"order_key":"04:miner-4","p2mr_program_hex":"9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b","share_difficulty":101,"share_id":"share-411","share_seq":411,"template_height":800012},{"accepted_at_ms":999982,"credit_policy":"stale-grace","job_id":"job-3811","job_issued_at_ms":999980,"miner_id":"miner-5","network_difficulty":1000003,"ntime":1700003811,"order_key":"05:miner-5","p2mr_program_hex":"e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3","share_difficulty":101,"share_id":"share-3811","share_seq":3811,"template_height":800119},{"accepted_at_ms":999999,"credit_policy":null,"job_id":"job-953","job_issued_at_ms":999975,"miner_id":"miner-7","network_difficulty":1000001,"ntime":1700000953,"order_key":"07:miner-7","p2mr_program_hex":"b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9","share_difficulty":1,"share_id":"share-953","share_seq":953,"template_height":800029},{"accepted_at_ms":999997,"credit_policy":null,"job_id":"job-1068","job_issued_at_ms":999982,"miner_id":"miner-1","network_difficulty":1000014,"ntime":1700001068,"order_key":"01:miner-1","p2mr_program_hex":"2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c","share_difficulty":3,"share_id":"share-1068","share_seq":1068,"template_height":800033},{"accepted_at_ms":999990,"credit_policy":null,"job_id":"job-298","job_issued_at_ms":999971,"miner_id":"miner-1","network_difficulty":1000009,"ntime":1700000298,"order_key":"01:miner-1","p2mr_program_hex":"2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a","share_difficulty":19,"share_id":"share-298","share_seq":298,"template_height":800009},{"accepted_at_ms":999976,"credit_policy":null,"job_id":"job-3477","job_issued_at_ms":999973,"miner_id":"miner-1","network_difficulty":1000009,"ntime":1700003477,"order_key":"01:miner-1","p2mr_program_hex":"9595959595959595959595959595959595959595959595959595959595959595","share_difficulty":1,"share_id":"share-3477","share_seq":3477,"template_height":800108},{"accepted_at_ms":999992,"credit_policy":null,"job_id":"job-3401","job_issued_at_ms":999997,"miner_id":"miner-2","network_difficulty":1000001,"ntime":1700003401,"order_key":"02:miner-2","p2mr_program_hex":"4949494949494949494949494949494949494949494949494949494949494949","share_difficulty":101,"share_id":"share-3401","share_seq":3401,"template_height":800106},{"accepted_at_ms":999975,"credit_policy":null,"job_id":"job-1889","job_issued_at_ms":999988,"miner_id":"miner-8","network_difficulty":1000002,"ntime":1700001889,"order_key":"08:miner-8","p2mr_program_hex":"6161616161616161616161616161616161616161616161616161616161616161","share_difficulty":2,"share_id":"share-1889","share_seq":1889,"template_height":800059},{"accepted_at_ms":1000000,"credit_policy":null,"job_id":"job-260","job_issued_at_ms":999981,"miner_id":"miner-7","network_difficulty":1000005,"ntime":1700000260,"order_key":"07:miner-7","p2mr_program_hex":"0404040404040404040404040404040404040404040404040404040404040404","share_difficulty":1000,"share_id":"share-260","share_seq":260,"template_height":800008},{"accepted_at_ms":999985,"credit_policy":null,"job_id":"job-3441","job_issued_at_ms":999990,"miner_id":"miner-9","network_difficulty":1000007,"ntime":1700003441,"order_key":"09:miner-9","p2mr_program_hex":"7171717171717171717171717171717171717171717171717171717171717171","share_difficulty":1000,"share_id":"share-3441","share_seq":3441,"template_height":800107},{"accepted_at_ms":999975,"credit_policy":null,"job_id":"job-2005","job_issued_at_ms":999989,"miner_id":"miner-3","network_difficulty":1000016,"ntime":1700002005,"order_key":"03:miner-3","p2mr_program_hex":"d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5","share_difficulty":101,"share_id":"share-2005","share_seq":2005,"template_height":800062},{"accepted_at_ms":999987,"credit_policy":"stale-grace","job_id":"job-2546","job_issued_at_ms":999977,"miner_id":"miner-5","network_difficulty":1000013,"ntime":1700002546,"order_key":"05:miner-5","p2mr_program_hex":"f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2","share_difficulty":2,"share_id":"share-2546","share_seq":2546,"template_height":800079},{"accepted_at_ms":999978,"credit_policy":"stale-grace","job_id":"job-1976","job_issued_at_ms":999971,"miner_id":"miner-7","network_difficulty":1000004,"ntime":1700001976,"order_key":"07:miner-7","p2mr_program_hex":"b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8","share_difficulty":101,"share_id":"share-1976","share_seq":1976,"template_height":800061},{"accepted_at_ms":999972,"credit_policy":null,"job_id":"job-585","job_issued_at_ms":999973,"miner_id":"miner-2","network_difficulty":1000007,"ntime":1700000585,"order_key":"02:miner-2","p2mr_program_hex":"4949494949494949494949494949494949494949494949494949494949494949","share_difficulty":1000,"share_id":"share-585","share_seq":585,"template_height":800018},{"accepted_at_ms":999977,"credit_policy":null,"job_id":"job-427","job_issued_at_ms":999991,"miner_id":"miner-9","network_difficulty":1000002,"ntime":1700000427,"order_key":"09:miner-9","p2mr_program_hex":"abababababababababababababababababababababababababababababababab","share_difficulty":3,"share_id":"share-427","share_seq":427,"template_height":800013},{"accepted_at_ms":1000393,"credit_policy":null,"job_id":"job-2078","job_issued_at_ms":999982,"miner_id":"miner-10","network_difficulty":1000004,"ntime":1700002078,"order_key":"10:miner-10","p2mr_program_hex":"1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e","share_difficulty":1000,"share_id":"share-2078","share_seq":2078,"template_height":800064},{"accepted_at_ms":999991,"credit_policy":null,"job_id":"job-375","job_issued_at_ms":999988,"miner_id":"miner-1","network_difficulty":1000001,"ntime":1700000375,"order_key":"01:miner-1","p2mr_program_hex":"7777777777777777777777777777777777777777777777777777777777777777","share_difficulty":1000,"share_id":"share-375","share_seq":375,"template_height":800011},{"accepted_at_ms":999988,"credit_policy":null,"job_id":"job-1905","job_issued_at_ms":999983,"miner_id":"miner-2","network_difficulty":1000001,"ntime":1700001905,"order_key":"02:miner-2","p2mr_program_hex":"7171717171717171717171717171717171717171717171717171717171717171","share_difficulty":3,"share_id":"share-1905","share_seq":1905,"template_height":800059},{"accepted_at_ms":999992,"credit_policy":null,"job_id":"job-270","job_issued_at_ms":999982,"miner_id":"miner-6","network_difficulty":1000015,"ntime":1700000270,"order_key":"06:miner-6","p2mr_program_hex":"0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e","share_difficulty":101,"share_id":"share-270","share_seq":270,"template_height":800008},{"accepted_at_ms":1000431,"credit_policy":"stale-grace","job_id":"job-2367","job_issued_at_ms":999994,"miner_id":"miner-2","network_difficulty":1000004,"ntime":1700002367,"order_key":"02:miner-2","p2mr_program_hex":"3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f","share_difficulty":7,"share_id":"share-2367","share_seq":2367,"template_height":800073},{"accepted_at_ms":999974,"credit_policy":null,"job_id":"job-915","job_issued_at_ms":999996,"miner_id":"miner-2","network_difficulty":1000014,"ntime":1700000915,"order_key":"02:miner-2","p2mr_program_hex":"9393939393939393939393939393939393939393939393939393939393939393","share_difficulty":3,"share_id":"share-915","share_seq":915,"template_height":800028},{"accepted_at_ms":999972,"credit_policy":null,"job_id":"job-1811","job_issued_at_ms":999990,"miner_id":"miner-7","network_difficulty":1000009,"ntime":1700001811,"order_key":"07:miner-7","p2mr_program_hex":"1313131313131313131313131313131313131313131313131313131313131313","share_difficulty":19,"share_id":"share-1811","share_seq":1811,"template_height":800056},{"accepted_at_ms":999987,"credit_policy":null,"job_id":"job-2095","job_issued_at_ms":999976,"miner_id":"miner-5","network_difficulty":1000004,"ntime":1700002095,"order_key":"05:miner-5","p2mr_program_hex":"2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f","share_difficulty":2,"share_id":"share-2095","share_seq":2095,"template_height":800065},{"accepted_at_ms":999988,"credit_policy":null,"job_id":"job-3397","job_issued_at_ms":999980,"miner_id":"miner-9","network_difficulty":1000014,"ntime":1700003397,"order_key":"09:miner-9","p2mr_program_hex":"4545454545454545454545454545454545454545454545454545454545454545","share_difficulty":7,"share_id":"share-3397","share_seq":3397,"template_height":800106},{"accepted_at_ms":999981,"credit_policy":"stale-grace","job_id":"job-839","job_issued_at_ms":999997,"miner_id":"miner-3","network_difficulty":1000006,"ntime":1700000839,"order_key":"03:miner-3","p2mr_program_hex":"4747474747474747474747474747474747474747474747474747474747474747","share_difficulty":101,"share_id":"share-839","share_seq":839,"template_height":800026},{"accepted_at_ms":999974,"credit_policy":"stale-grace","job_id":"job-2461","job_issued_at_ms":999998,"miner_id":"miner-8","network_difficulty":1000013,"ntime":1700002461,"order_key":"08:miner-8","p2mr_program_hex":"9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d","share_difficulty":1000,"share_id":"share-2461","share_seq":2461,"template_height":800076},{"accepted_at_ms":999980,"credit_policy":"stale-grace","job_id":"job-3256","job_issued_at_ms":999986,"miner_id":"miner-0","network_difficulty":1000009,"ntime":1700003256,"order_key":"00:miner-0","p2mr_program_hex":"b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8","share_difficulty":2,"share_id":"share-3256","share_seq":3256,"template_height":800101},{"accepted_at_ms":999976,"credit_policy":null,"job_id":"job-430","job_issued_at_ms":999995,"miner_id":"miner-1","network_difficulty":1000005,"ntime":1700000430,"order_key":"01:miner-1","p2mr_program_hex":"aeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeae","share_difficulty":2,"share_id":"share-430","share_seq":430,"template_height":800013},{"accepted_at_ms":999983,"credit_policy":null,"job_id":"job-1969","job_issued_at_ms":1000000,"miner_id":"miner-0","network_difficulty":1000014,"ntime":1700001969,"order_key":"00:miner-0","p2mr_program_hex":"b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1","share_difficulty":19,"share_id":"share-1969","share_seq":1969,"template_height":800061},{"accepted_at_ms":999980,"credit_policy":null,"job_id":"job-1044","job_issued_at_ms":999997,"miner_id":"miner-10","network_difficulty":1000007,"ntime":1700001044,"order_key":"10:miner-10","p2mr_program_hex":"1414141414141414141414141414141414141414141414141414141414141414","share_difficulty":19,"share_id":"share-1044","share_seq":1044,"template_height":800032},{"accepted_at_ms":999975,"credit_policy":null,"job_id":"job-70","job_issued_at_ms":999973,"miner_id":"miner-4","network_difficulty":1000002,"ntime":1700000070,"order_key":"04:miner-4","p2mr_program_hex":"4646464646464646464646464646464646464646464646464646464646464646","share_difficulty":1000,"share_id":"share-70","share_seq":70,"template_height":800002},{"accepted_at_ms":999993,"credit_policy":null,"job_id":"job-1455","job_issued_at_ms":999997,"miner_id":"miner-3","network_difficulty":1000010,"ntime":1700001455,"order_key":"03:miner-3","p2mr_program_hex":"afafafafafafafafafafafafafafafafafafafafafafafafafafafafafafafaf","share_difficulty":7,"share_id":"share-1455","share_seq":1455,"template_height":800045},{"accepted_at_ms":999998,"credit_policy":null,"job_id":"job-2917","job_issued_at_ms":999995,"miner_id":"miner-2","network_difficulty":1000010,"ntime":1700002917,"order_key":"02:miner-2","p2mr_program_hex":"6565656565656565656565656565656565656565656565656565656565656565","share_difficulty":7,"share_id":"share-2917","share_seq":2917,"template_height":800091},{"accepted_at_ms":999973,"credit_policy":null,"job_id":"job-2305","job_issued_at_ms":999996,"miner_id":"miner-6","network_difficulty":1000010,"ntime":1700002305,"order_key":"06:miner-6","p2mr_program_hex":"0101010101010101010101010101010101010101010101010101010101010101","share_difficulty":1000,"share_id":"share-2305","share_seq":2305,"template_height":800072},{"accepted_at_ms":999984,"credit_policy":null,"job_id":"job-531","job_issued_at_ms":999997,"miner_id":"miner-3","network_difficulty":1000004,"ntime":1700000531,"order_key":"03:miner-3","p2mr_program_hex":"1313131313131313131313131313131313131313131313131313131313131313","share_difficulty":2,"share_id":"share-531","share_seq":531,"template_height":800016},{"accepted_at_ms":999995,"credit_policy":null,"job_id":"job-1506","job_issued_at_ms":999978,"miner_id":"miner-10","network_difficulty":1000010,"ntime":1700001506,"order_key":"10:miner-10","p2mr_program_hex":"e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2","share_difficulty":7,"share_id":"share-1506","share_seq":1506,"template_height":800047},{"accepted_at_ms":999979,"credit_policy":null,"job_id":"job-1530","job_issued_at_ms":999999,"miner_id":"miner-1","network_difficulty":1000000,"ntime":1700001530,"order_key":"01:miner-1","p2mr_program_hex":"fafafafafafafafafafafafafafafafafafafafafafafafafafafafafafafafa","share_difficulty":1,"share_id":"share-1530","share_seq":1530,"template_height":800047},{"accepted_at_ms":999992,"credit_policy":"stale-grace","job_id":"job-1841","job_issued_at_ms":999979,"miner_id":"miner-4","network_difficulty":1000005,"ntime":1700001841,"order_key":"04:miner-4","p2mr_program_hex":"3131313131313131313131313131313131313131313131313131313131313131","share_difficulty":2,"share_id":"share-1841","share_seq":1841,"template_height":800057},{"accepted_at_ms":999972,"credit_policy":null,"job_id":"job-3359","job_issued_at_ms":999975,"miner_id":"miner-4","network_difficulty":1000010,"ntime":1700003359,"order_key":"04:miner-4","p2mr_program_hex":"1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f","share_difficulty":7,"share_id":"share-3359","share_seq":3359,"template_height":800104},{"accepted_at_ms":999992,"credit_policy":"stale-grace","job_id":"job-1490","job_issued_at_ms":999990,"miner_id":"miner-5","network_difficulty":1000011,"ntime":1700001490,"order_key":"05:miner-5","p2mr_program_hex":"d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2","share_difficulty":101,"share_id":"share-1490","share_seq":1490,"template_height":800046},{"accepted_at_ms":999990,"credit_policy":null,"job_id":"job-3422","job_issued_at_ms":999976,"miner_id":"miner-1","network_difficulty":1000005,"ntime":1700003422,"order_key":"01:miner-1","p2mr_program_hex":"5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e","share_difficulty":3,"share_id":"share-3422","share_seq":3422,"template_height":800106},{"accepted_at_ms":999981,"credit_policy":null,"job_id":"job-1877","job_issued_at_ms":999997,"miner_id":"miner-7","network_difficulty":1000007,"ntime":1700001877,"order_key":"07:miner-7","p2mr_program_hex":"5555555555555555555555555555555555555555555555555555555555555555","share_difficulty":7,"share_id":"share-1877","share_seq":1877,"template_height":800058},{"accepted_at_ms":999987,"credit_policy":null,"job_id":"job-564","job_issued_at_ms":999990,"miner_id":"miner-3","network_difficulty":1000003,"ntime":1700000564,"order_key":"03:miner-3","p2mr_program_hex":"3434343434343434343434343434343434343434343434343434343434343434","share_difficulty":2,"share_id":"share-564","share_seq":564,"template_height":800017},{"accepted_at_ms":999973,"credit_policy":null,"job_id":"job-3952","job_issued_at_ms":999982,"miner_id":"miner-3","network_difficulty":1000008,"ntime":1700003952,"order_key":"03:miner-3","p2mr_program_hex":"7070707070707070707070707070707070707070707070707070707070707070","share_difficulty":7,"share_id":"share-3952","share_seq":3952,"template_height":800123},{"accepted_at_ms":999987,"credit_policy":null,"job_id":"job-1359","job_issued_at_ms":999974,"miner_id":"miner-6","network_difficulty":1000016,"ntime":1700001359,"order_key":"06:miner-6","p2mr_program_hex":"4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f","share_difficulty":1,"share_id":"share-1359","share_seq":1359,"template_height":800042},{"accepted_at_ms":999985,"credit_policy":null,"job_id":"job-3276","job_issued_at_ms":1000000,"miner_id":"miner-9","network_difficulty":1000012,"ntime":1700003276,"order_key":"09:miner-9","p2mr_program_hex":"cccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccc","share_difficulty":19,"share_id":"share-3276","share_seq":3276,"template_height":800102},{"accepted_at_ms":999976,"credit_policy":null,"job_id":"job-1004","job_issued_at_ms":999989,"miner_id":"miner-3","network_difficulty":1000001,"ntime":1700001004,"order_key":"03:miner-3","p2mr_program_hex":"ecececececececececececececececececececececececececececececececec","share_difficulty":7,"share_id":"share-1004","share_seq":1004,"template_height":800031},{"accepted_at_ms":999995,"credit_policy":null,"job_id":"job-2716","job_issued_at_ms":1000161,"miner_id":"miner-10","network_difficulty":1000013,"ntime":1700002716,"order_key":"10:miner-10","p2mr_program_hex":"9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c","share_difficulty":2,"share_id":"share-2716","share_seq":2716,"template_height":800084},{"accepted_at_ms":999981,"credit_policy":null,"job_id":"job-1750","job_issued_at_ms":1000000,"miner_id":"miner-1","network_difficulty":1000016,"ntime":1700001750,"order_key":"01:miner-1","p2mr_program_hex":"d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6","share_difficulty":1000,"share_id":"share-1750","share_seq":1750,"template_height":800054},{"accepted_at_ms":999986,"credit_policy":null,"job_id":"job-2981","job_issued_at_ms":999988,"miner_id":"miner-0","network_difficulty":1000006,"ntime":1700002981,"order_key":"00:miner-0","p2mr_program_hex":"a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5","share_difficulty":1,"share_id":"share-2981","share_seq":2981,"template_height":800093},{"accepted_at_ms":999987,"credit_policy":null,"job_id":"job-1498","job_issued_at_ms":999996,"miner_id":"miner-2","network_difficulty":1000002,"ntime":1700001498,"order_key":"02:miner-2","p2mr_program_hex":"dadadadadadadadadadadadadadadadadadadadadadadadadadadadadadadada","share_difficulty":1,"share_id":"share-1498","share_seq":1498,"template_height":800046},{"accepted_at_ms":999986,"credit_policy":null,"job_id":"job-1521","job_issued_at_ms":999995,"miner_id":"miner-3","network_difficulty":1000008,"ntime":1700001521,"order_key":"03:miner-3","p2mr_program_hex":"f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1","share_difficulty":19,"share_id":"share-1521","share_seq":1521,"template_height":800047},{"accepted_at_ms":999987,"credit_policy":null,"job_id":"job-3614","job_issued_at_ms":999999,"miner_id":"miner-6","network_difficulty":1000010,"ntime":1700003614,"order_key":"06:miner-6","p2mr_program_hex":"1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e","share_difficulty":19,"share_id":"share-3614","share_seq":3614,"template_height":800112},{"accepted_at_ms":999987,"credit_policy":null,"job_id":"job-1972","job_issued_at_ms":999999,"miner_id":"miner-3","network_difficulty":1000000,"ntime":1700001972,"order_key":"03:miner-3","p2mr_program_hex":"b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4","share_difficulty":19,"share_id":"share-1972","share_seq":1972,"template_height":800061},{"accepted_at_ms":999998,"credit_policy":null,"job_id":"job-2342","job_issued_at_ms":999990,"miner_id":"miner-10","network_difficulty":1000013,"ntime":1700002342,"order_key":"10:miner-10","p2mr_program_hex":"2626262626262626262626262626262626262626262626262626262626262626","share_difficulty":19,"share_id":"share-2342","share_seq":2342,"template_height":800073},{"accepted_at_ms":999985,"credit_policy":null,"job_id":"job-2836","job_issued_at_ms":999990,"miner_id":"miner-9","network_difficulty":1000014,"ntime":1700002836,"order_key":"09:miner-9","p2mr_program_hex":"1414141414141414141414141414141414141414141414141414141414141414","share_difficulty":2,"share_id":"share-2836","share_seq":2836,"template_height":800088},{"accepted_at_ms":999982,"credit_policy":null,"job_id":"job-657","job_issued_at_ms":999971,"miner_id":"miner-8","network_difficulty":1000011,"ntime":1700000657,"order_key":"08:miner-8","p2mr_program_hex":"9191919191919191919191919191919191919191919191919191919191919191","share_difficulty":1,"share_id":"share-657","share_seq":657,"template_height":800020},{"accepted_at_ms":999994,"credit_policy":"stale-grace","job_id":"job-239","job_issued_at_ms":999991,"miner_id":"miner-8","network_difficulty":1000001,"ntime":1700000239,"order_key":"08:miner-8","p2mr_program_hex":"efefefefefefefefefefefefefefefefefefefefefefefefefefefefefefefef","share_difficulty":1,"share_id":"share-239","share_seq":239,"template_height":800007},{"accepted_at_ms":999987,"credit_policy":null,"job_id":"job-3687","job_issued_at_ms":999972,"miner_id":"miner-2","network_difficulty":1000015,"ntime":1700003687,"order_key":"02:miner-2","p2mr_program_hex":"6767676767676767676767676767676767676767676767676767676767676767","share_difficulty":1,"share_id":"share-3687","share_seq":3687,"template_height":800115},{"accepted_at_ms":999985,"credit_policy":"stale-grace","job_id":"job-2882","job_issued_at_ms":999977,"miner_id":"miner-0","network_difficulty":1000009,"ntime":1700002882,"order_key":"00:miner-0","p2mr_program_hex":"4242424242424242424242424242424242424242424242424242424242424242","share_difficulty":1000,"share_id":"share-2882","share_seq":2882,"template_height":800090},{"accepted_at_ms":999985,"credit_policy":null,"job_id":"job-761","job_issued_at_ms":999985,"miner_id":"miner-2","network_difficulty":1000013,"ntime":1700000761,"order_key":"02:miner-2","p2mr_program_hex":"f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9","share_difficulty":3,"share_id":"share-761","share_seq":761,"template_height":800023},{"accepted_at_ms":999999,"credit_policy":null,"job_id":"job-389","job_issued_at_ms":999992,"miner_id":"miner-4","network_difficulty":1000015,"ntime":1700000389,"order_key":"04:miner-4","p2mr_program_hex":"8585858585858585858585858585858585858585858585858585858585858585","share_difficulty":1,"share_id":"share-389","share_seq":389,"template_height":800012},{"accepted_at_ms":999996,"credit_policy":null,"job_id":"job-1091","job_issued_at_ms":999992,"miner_id":"miner-2","network_difficulty":1000003,"ntime":1700001091,"order_key":"02:miner-2","p2mr_program_hex":"4343434343434343434343434343434343434343434343434343434343434343","share_difficulty":3,"share_id":"share-1091","share_seq":1091,"template_height":800034},{"accepted_at_ms":999987,"credit_policy":null,"job_id":"job-1906","job_issued_at_ms":999990,"miner_id":"miner-3","network_difficulty":1000002,"ntime":1700001906,"order_key":"03:miner-3","p2mr_program_hex":"7272727272727272727272727272727272727272727272727272727272727272","share_difficulty":19,"share_id":"share-1906","share_seq":1906,"template_height":800059},{"accepted_at_ms":1000000,"credit_policy":null,"job_id":"job-852","job_issued_at_ms":1000113,"miner_id":"miner-5","network_difficulty":1000002,"ntime":1700000852,"order_key":"05:miner-5","p2mr_program_hex":"5454545454545454545454545454545454545454545454545454545454545454","share_difficulty":19,"share_id":"share-852","share_seq":852,"template_height":800026},{"accepted_at_ms":1000000,"credit_policy":null,"job_id":"job-2729","job_issued_at_ms":999991,"miner_id":"miner-1","network_difficulty":1000009,"ntime":1700002729,"order_key":"01:miner-1","p2mr_program_hex":"a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9","share_difficulty":1000,"share_id":"share-2729","share_seq":2729,"template_height":800085},{"accepted_at_ms":999976,"credit_policy":null,"job_id":"job-2608","job_issued_at_ms":999985,"miner_id":"miner-1","network_difficulty":1000007,"ntime":1700002608,"order_key":"01:miner-1","p2mr_program_hex":"3030303030303030303030303030303030303030303030303030303030303030","share_difficulty":19,"share_id":"share-2608","share_seq":2608,"template_height":800081},{"accepted_at_ms":999976,"credit_policy":null,"job_id":"job-2656","job_issued_at_ms":999988,"miner_id":"miner-5","network_difficulty":1000004,"ntime":1700002656,"order_key":"05:miner-5","p2mr_program_hex":"6060606060606060606060606060606060606060606060606060606060606060","share_difficulty":3,"share_id":"share-2656","share_seq":2656,"template_height":800083},{"accepted_at_ms":999985,"credit_policy":null,"job_id":"job-391","job_issued_at_ms":999972,"miner_id":"miner-6","network_difficulty":1000000,"ntime":1700000391,"order_key":"06:miner-6","p2mr_program_hex":"8787878787878787878787878787878787878787878787878787878787878787","share_difficulty":7,"share_id":"share-391","share_seq":391,"template_height":800012},{"accepted_at_ms":999999,"credit_policy":"stale-grace","job_id":"job-1274","job_issued_at_ms":1000000,"miner_id":"miner-9","network_difficulty":1000016,"ntime":1700001274,"order_key":"09:miner-9","p2mr_program_hex":"fafafafafafafafafafafafafafafafafafafafafafafafafafafafafafafafa","share_difficulty":2,"share_id":"share-1274","share_seq":1274,"template_height":800039},{"accepted_at_ms":999994,"credit_policy":null,"job_id":"job-3967","job_issued_at_ms":999977,"miner_id":"miner-7","network_difficulty":1000006,"ntime":1700003967,"order_key":"07:miner-7","p2mr_program_hex":"7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f","share_difficulty":19,"share_id":"share-3967","share_seq":3967,"template_height":800123},{"accepted_at_ms":999981,"credit_policy":null,"job_id":"job-1108","job_issued_at_ms":999982,"miner_id":"miner-8","network_difficulty":1000003,"ntime":1700001108,"order_key":"08:miner-8","p2mr_program_hex":"5454545454545454545454545454545454545454545454545454545454545454","share_difficulty":7,"share_id":"share-1108","share_seq":1108,"template_height":800034},{"accepted_at_ms":999990,"credit_policy":null,"job_id":"job-1429","job_issued_at_ms":999998,"miner_id":"miner-10","network_difficulty":1000001,"ntime":1700001429,"order_key":"10:miner-10","p2mr_program_hex":"9595959595959595959595959595959595959595959595959595959595959595","share_difficulty":1000,"share_id":"share-1429","share_seq":1429,"template_height":800044},{"accepted_at_ms":999997,"credit_policy":null,"job_id":"job-996","job_issued_at_ms":999985,"miner_id":"miner-6","network_difficulty":1000010,"ntime":1700000996,"order_key":"06:miner-6","p2mr_program_hex":"e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4","share_difficulty":19,"share_id":"share-996","share_seq":996,"template_height":800031},{"accepted_at_ms":1000150,"credit_policy":"stale-grace","job_id":"job-3060","job_issued_at_ms":999976,"miner_id":"miner-2","network_difficulty":1000000,"ntime":1700003060,"order_key":"02:miner-2","p2mr_program_hex":"f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4","share_difficulty":7,"share_id":"share-3060","share_seq":3060,"template_height":800095},{"accepted_at_ms":999998,"credit_policy":null,"job_id":"job-2139","job_issued_at_ms":999983,"miner_id":"miner-5","network_difficulty":1000014,"ntime":1700002139,"order_key":"05:miner-5","p2mr_program_hex":"5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b","share_difficulty":19,"share_id":"share-2139","share_seq":2139,"template_height":800066},{"accepted_at_ms":999996,"credit_policy":"stale-grace","job_id":"job-3725","job_issued_at_ms":999994,"miner_id":"miner-7","network_difficulty":1000002,"ntime":1700003725,"order_key":"07:miner-7","p2mr_program_hex":"8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d","share_difficulty":1000,"share_id":"share-3725","share_seq":3725,"template_height":800116},{"accepted_at_ms":999990,"credit_policy":"stale-grace","job_id":"job-1669","job_issued_at_ms":999989,"miner_id":"miner-8","network_difficulty":1000003,"ntime":1700001669,"order_key":"08:miner-8","p2mr_program_hex":"8585858585858585858585858585858585858585858585858585858585858585","share_difficulty":101,"share_id":"share-1669","share_seq":1669,"template_height":800052},{"accepted_at_ms":999997,"credit_policy":"stale-grace","job_id":"job-1320","job_issued_at_ms":999985,"miner_id":"miner-0","network_difficulty":1000011,"ntime":1700001320,"order_key":"00:miner-0","p2mr_program_hex":"2828282828282828282828282828282828282828282828282828282828282828","share_difficulty":101,"share_id":"share-1320","share_seq":1320,"template_height":800041},{"accepted_at_ms":1000000,"credit_policy":"stale-grace","job_id":"job-3371","job_issued_at_ms":999971,"miner_id":"miner-5","network_difficulty":1000005,"ntime":1700003371,"order_key":"05:miner-5","p2mr_program_hex":"2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b","share_difficulty":2,"share_id":"share-3371","share_seq":3371,"template_height":800105},{"accepted_at_ms":999977,"credit_policy":null,"job_id":"job-306","job_issued_at_ms":999993,"miner_id":"miner-9","network_difficulty":1000000,"ntime":1700000306,"order_key":"09:miner-9","p2mr_program_hex":"3232323232323232323232323232323232323232323232323232323232323232","share_difficulty":19,"share_id":"share-306","share_seq":306,"template_height":800009},{"accepted_at_ms":999985,"credit_policy":null,"job_id":"job-2726","job_issued_at_ms":999987,"miner_id":"miner-9","network_difficulty":1000006,"ntime":1700002726,"order_key":"09:miner-9","p2mr_program_hex":"a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6","share_difficulty":1000,"share_id":"share-2726","share_seq":2726,"template_height":800085},{"accepted_at_ms":999983,"credit_policy":null,"job_id":"job-3058","job_issued_at_ms":999989,"miner_id":"miner-0","network_difficulty":1000015,"ntime":1700003058,"order_key":"00:miner-0","p2mr_program_hex":"f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2","share_difficulty":7,"share_id":"share-3058","share_seq":3058,"template_height":800095},{"accepted_at_ms":999981,"credit_policy":null,"job_id":"job-1646","job_issued_at_ms":1000294,"miner_id":"miner-7","network_difficulty":1000014,"ntime":1700001646,"order_key":"07:miner-7","p2mr_program_hex":"6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e","share_difficulty":2,"share_id":"share-1646","share_seq":1646,"template_height":800051},{"accepted_at_ms":999993,"credit_policy":null,"job_id":"job-1780","job_issued_at_ms":1000000,"miner_id":"miner-9","network_difficulty":1000012,"ntime":1700001780,"order_key":"09:miner-9","p2mr_program_hex":"f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4","share_difficulty":1000,"share_id":"share-1780","share_seq":1780,"template_height":800055},{"accepted_at_ms":999981,"credit_policy":null,"job_id":"job-2047","job_issued_at_ms":999999,"miner_id":"miner-1","network_difficulty":1000007,"ntime":1700002047,"order_key":"01:miner-1","p2mr_program_hex":"ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff","share_difficulty":3,"share_id":"share-2047","share_seq":2047,"template_height":800063},{"accepted_at_ms":999986,"credit_policy":null,"job_id":"job-292","job_issued_at_ms":999970,"miner_id":"miner-6","network_difficulty":1000003,"ntime":1700000292,"order_key":"06:miner-6","p2mr_program_hex":"2424242424242424242424242424242424242424242424242424242424242424","share_difficulty":2,"share_id":"share-292","share_seq":292,"template_height":800009},{"accepted_at_ms":999980,"credit_policy":"stale-grace","job_id":"job-1319","job_issued_at_ms":999988,"miner_id":"miner-10","network_difficulty":1000010,"ntime":1700001319,"order_key":"10:miner-10","p2mr_program_hex":"2727272727272727272727272727272727272727272727272727272727272727","share_difficulty":1000,"share_id":"share-1319","share_seq":1319,"template_height":800041},{"accepted_at_ms":999999,"credit_policy":null,"job_id":"job-766","job_issued_at_ms":999975,"miner_id":"miner-7","network_difficulty":1000001,"ntime":1700000766,"order_key":"07:miner-7","p2mr_program_hex":"fefefefefefefefefefefefefefefefefefefefefefefefefefefefefefefefe","share_difficulty":7,"share_id":"share-766","share_seq":766,"template_height":800023},{"accepted_at_ms":999971,"credit_policy":null,"job_id":"job-1440","job_issued_at_ms":999989,"miner_id":"miner-10","network_difficulty":1000012,"ntime":1700001440,"order_key":"10:miner-10","p2mr_program_hex":"a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0","share_difficulty":2,"share_id":"share-1440","share_seq":1440,"template_height":800045},{"accepted_at_ms":999981,"credit_policy":null,"job_id":"job-3196","job_issued_at_ms":999990,"miner_id":"miner-6","network_difficulty":1000000,"ntime":1700003196,"order_key":"06:miner-6","p2mr_program_hex":"7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c","share_difficulty":101,"share_id":"share-3196","share_seq":3196,"template_height":800099},{"accepted_at_ms":999998,"credit_policy":"stale-grace","job_id":"job-1798","job_issued_at_ms":999992,"miner_id":"miner-5","network_difficulty":1000013,"ntime":1700001798,"order_key":"05:miner-5","p2mr_program_hex":"0606060606060606060606060606060606060606060606060606060606060606","share_difficulty":101,"share_id":"share-1798","share_seq":1798,"template_height":800056},{"accepted_at_ms":999972,"credit_policy":"stale-grace","job_id":"job-811","job_issued_at_ms":999976,"miner_id":"miner-8","network_difficulty":1000012,"ntime":1700000811,"order_key":"08:miner-8","p2mr_program_hex":"2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b","share_difficulty":2,"share_id":"share-811","share_seq":811,"template_height":800025},{"accepted_at_ms":999983,"credit_policy":"stale-grace","job_id":"job-672","job_issued_at_ms":999984,"miner_id":"miner-1","network_difficulty":1000009,"ntime":1700000672,"order_key":"01:miner-1","p2mr_program_hex":"a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0","share_difficulty":2,"share_id":"share-672","share_seq":672,"template_height":800021},{"accepted_at_ms":999977,"credit_policy":null,"job_id":"job-3261","job_issued_at_ms":999999,"miner_id":"miner-5","network_difficulty":1000014,"ntime":1700003261,"order_key":"05:miner-5","p2mr_program_hex":"bdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbd","share_difficulty":19,"share_id":"share-3261","share_seq":3261,"template_height":800101},{"accepted_at_ms":999974,"credit_policy":null,"job_id":"job-1632","job_issued_at_ms":999998,"miner_id":"miner-4","network_difficulty":1000000,"ntime":1700001632,"order_key":"04:miner-4","p2mr_program_hex":"6060606060606060606060606060606060606060606060606060606060606060","share_difficulty":3,"share_id":"share-1632","share_seq":1632,"template_height":800051},{"accepted_at_ms":1000481,"credit_policy":null,"job_id":"job-2001","job_issued_at_ms":1000321,"miner_id":"miner-10","network_difficulty":1000012,"ntime":1700002001,"order_key":"10:miner-10","p2mr_program_hex":"d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1","share_difficulty":1,"share_id":"share-2001","share_seq":2001,"template_height":800062},{"accepted_at_ms":999996,"credit_policy":null,"job_id":"job-1986","job_issued_at_ms":999973,"miner_id":"miner-6","network_difficulty":1000014,"ntime":1700001986,"order_key":"06:miner-6","p2mr_program_hex":"c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2","share_difficulty":19,"share_id":"share-1986","share_seq":1986,"template_height":800062},{"accepted_at_ms":999986,"credit_policy":null,"job_id":"job-3126","job_issued_at_ms":1000372,"miner_id":"miner-2","network_difficulty":1000015,"ntime":1700003126,"order_key":"02:miner-2","p2mr_program_hex":"3636363636363636363636363636363636363636363636363636363636363636","share_difficulty":7,"share_id":"share-3126","share_seq":3126,"template_height":800097},{"accepted_at_ms":999991,"credit_policy":null,"job_id":"job-1577","job_issued_at_ms":999998,"miner_id":"miner-4","network_difficulty":1000013,"ntime":1700001577,"order_key":"04:miner-4","p2mr_program_hex":"2929292929292929292929292929292929292929292929292929292929292929","share_difficulty":19,"share_id":"share-1577","share_seq":1577,"template_height":800049},{"accepted_at_ms":999978,"credit_policy":null,"job_id":"job-3949","job_issued_at_ms":999991,"miner_id":"miner-0","network_difficulty":1000005,"ntime":1700003949,"order_key":"00:miner-0","p2mr_program_hex":"6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d","share_difficulty":19,"share_id":"share-3949","share_seq":3949,"template_height":800123},{"accepted_at_ms":999989,"credit_policy":null,"job_id":"job-1966","job_issued_at_ms":999977,"miner_id":"miner-8","network_difficulty":1000011,"ntime":1700001966,"order_key":"08:miner-8","p2mr_program_hex":"aeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeae","share_difficulty":19,"share_id":"share-1966","share_seq":1966,"template_height":800061},{"accepted_at_ms":999975,"credit_policy":"stale-grace","job_id":"job-3499","job_issued_at_ms":999978,"miner_id":"miner-1","network_difficulty":1000014,"ntime":1700003499,"order_key":"01:miner-1","p2mr_program_hex":"abababababababababababababababababababababababababababababababab","share_difficulty":7,"share_id":"share-3499","share_seq":3499,"template_height":800109},{"accepted_at_ms":999979,"credit_policy":null,"job_id":"job-2304","job_issued_at_ms":999979,"miner_id":"miner-5","network_difficulty":1000009,"ntime":1700002304,"order_key":"05:miner-5","p2mr_program_hex":"0000000000000000000000000000000000000000000000000000000000000000","share_difficulty":3,"share_id":"share-2304","share_seq":2304,"template_height":800072},{"accepted_at_ms":999996,"credit_policy":null,"job_id":"job-1968","job_issued_at_ms":999995,"miner_id":"miner-10","network_difficulty":1000013,"ntime":1700001968,"order_key":"10:miner-10","p2mr_program_hex":"b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0","share_difficulty":19,"share_id":"share-1968","share_seq":1968,"template_height":800061},{"accepted_at_ms":999973,"credit_policy":null,"job_id":"job-955","job_issued_at_ms":999989,"miner_id":"miner-9","network_difficulty":1000003,"ntime":1700000955,"order_key":"09:miner-9","p2mr_program_hex":"bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb","share_difficulty":7,"share_id":"share-955","share_seq":955,"template_height":800029},{"accepted_at_ms":1000000,"credit_policy":null,"job_id":"job-2488","job_issued_at_ms":999985,"miner_id":"miner-2","network_difficulty":1000006,"ntime":1700002488,"order_key":"02:miner-2","p2mr_program_hex":"b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8","share_difficulty":19,"share_id":"share-2488","share_seq":2488,"template_height":800077},{"accepted_at_ms":999999,"credit_policy":null,"job_id":"job-1513","job_issued_at_ms":999997,"miner_id":"miner-6","network_difficulty":1000000,"ntime":1700001513,"order_key":"06:miner-6","p2mr_program_hex":"e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9","share_difficulty":101,"share_id":"share-1513","share_seq":1513,"template_height":800047},{"accepted_at_ms":999991,"credit_policy":null,"job_id":"job-3688","job_issued_at_ms":999999,"miner_id":"miner-3","network_difficulty":1000016,"ntime":1700003688,"order_key":"03:miner-3","p2mr_program_hex":"6868686868686868686868686868686868686868686868686868686868686868","share_difficulty":1000,"share_id":"share-3688","share_seq":3688,"template_height":800115},{"accepted_at_ms":999986,"credit_policy":null,"job_id":"job-2260","job_issued_at_ms":999989,"miner_id":"miner-5","network_difficulty":1000016,"ntime":1700002260,"order_key":"05:miner-5","p2mr_program_hex":"d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4","share_difficulty":19,"share_id":"share-2260","share_seq":2260,"template_height":800070},{"accepted_at_ms":999973,"credit_policy":"stale-grace","job_id":"job-212","job_issued_at_ms":999980,"miner_id":"miner-3","network_difficulty":1000008,"ntime":1700000212,"order_key":"03:miner-3","p2mr_program_hex":"d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4","share_difficulty":19,"share_id":"share-212","share_seq":212,"template_height":800006},{"accepted_at_ms":999971,"credit_policy":null,"job_id":"job-379","job_issued_at_ms":999993,"miner_id":"miner-5","network_difficulty":1000005,"ntime":1700000379,"order_key":"05:miner-5","p2mr_program_hex":"7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b","share_difficulty":2,"share_id":"share-379","share_seq":379,"template_height":800011},{"accepted_at_ms":999996,"credit_policy":null,"job_id":"job-3588","job_issued_at_ms":999983,"miner_id":"miner-2","network_difficulty":1000001,"ntime":1700003588,"order_key":"02:miner-2","p2mr_program_hex":"0404040404040404040404040404040404040404040404040404040404040404","share_difficulty":2,"share_id":"share-3588","share_seq":3588,"template_height":800112},{"accepted_at_ms":999994,"credit_policy":null,"job_id":"job-2009","job_issued_at_ms":999984,"miner_id":"miner-7","network_difficulty":1000003,"ntime":1700002009,"order_key":"07:miner-7","p2mr_program_hex":"d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9","share_difficulty":2,"share_id":"share-2009","share_seq":2009,"template_height":800062},{"accepted_at_ms":999977,"credit_policy":null,"job_id":"job-1144","job_issued_at_ms":999979,"miner_id":"miner-0","network_difficulty":1000005,"ntime":1700001144,"order_key":"00:miner-0","p2mr_program_hex":"7878787878787878787878787878787878787878787878787878787878787878","share_difficulty":1000,"share_id":"share-1144","share_seq":1144,"template_height":800035},{"accepted_at_ms":999977,"credit_policy":null,"job_id":"job-1826","job_issued_at_ms":999977,"miner_id":"miner-0","network_difficulty":1000007,"ntime":1700001826,"order_key":"00:miner-0","p2mr_program_hex":"2222222222222222222222222222222222222222222222222222222222222222","share_difficulty":101,"share_id":"share-1826","share_seq":1826,"template_height":800057},{"accepted_at_ms":999980,"credit_policy":null,"job_id":"job-343","job_issued_at_ms":999993,"miner_id":"miner-2","network_difficulty":1000003,"ntime":1700000343,"order_key":"02:miner-2","p2mr_program_hex":"5757575757575757575757575757575757575757575757575757575757575757","share_difficulty":3,"share_id":"share-343","share_seq":343,"template_height":800010},{"accepted_at_ms":1000394,"credit_policy":null,"job_id":"job-3871","job_issued_at_ms":999998,"miner_id":"miner-10","network_difficulty":1000012,"ntime":1700003871,"order_key":"10:miner-10","p2mr_program_hex":"1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f","share_difficulty":3,"share_id":"share-3871","share_seq":3871,"template_height":800120},{"accepted_at_ms":999982,"credit_policy":"stale-grace","job_id":"job-1205","job_issued_at_ms":999992,"miner_id":"miner-6","network_difficulty":1000015,"ntime":1700001205,"order_key":"06:miner-6","p2mr_program_hex":"b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5","share_difficulty":3,"share_id":"share-1205","share_seq":1205,"template_height":800037},{"accepted_at_ms":999977,"credit_policy":null,"job_id":"job-828","job_issued_at_ms":999978,"miner_id":"miner-3","network_difficulty":1000012,"ntime":1700000828,"order_key":"03:miner-3","p2mr_program_hex":"3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c","share_difficulty":1,"share_id":"share-828","share_seq":828,"template_height":800025},{"accepted_at_ms":1000000,"credit_policy":null,"job_id":"job-1491","job_issued_at_ms":999972,"miner_id":"miner-6","network_difficulty":1000012,"ntime":1700001491,"order_key":"06:miner-6","p2mr_program_hex":"d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3","share_difficulty":2,"share_id":"share-1491","share_seq":1491,"template_height":800046},{"accepted_at_ms":999990,"credit_policy":null,"job_id":"job-2183","job_issued_at_ms":999979,"miner_id":"miner-5","network_difficulty":1000007,"ntime":1700002183,"order_key":"05:miner-5","p2mr_program_hex":"8787878787878787878787878787878787878787878787878787878787878787","share_difficulty":2,"share_id":"share-2183","share_seq":2183,"template_height":800068},{"accepted_at_ms":999992,"credit_policy":"stale-grace","job_id":"job-240","job_issued_at_ms":999995,"miner_id":"miner-9","network_difficulty":1000002,"ntime":1700000240,"order_key":"09:miner-9","p2mr_program_hex":"f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0","share_difficulty":2,"share_id":"share-240","share_seq":240,"template_height":800007},{"accepted_at_ms":999981,"credit_policy":null,"job_id":"job-2749","job_issued_at_ms":999999,"miner_id":"miner-10","network_difficulty":1000012,"ntime":1700002749,"order_key":"10:miner-10","p2mr_program_hex":"bdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbd","share_difficulty":2,"share_id":"share-2749","share_seq":2749,"template_height":800085},{"accepted_at_ms":999985,"credit_policy":null,"job_id":"job-1984","job_issued_at_ms":999974,"miner_id":"miner-4","network_difficulty":1000012,"ntime":1700001984,"order_key":"04:miner-4","p2mr_program_hex":"c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0","share_difficulty":101,"share_id":"share-1984","share_seq":1984,"template_height":800062},{"accepted_at_ms":999992,"credit_policy":null,"job_id":"job-723","job_issued_at_ms":999972,"miner_id":"miner-8","network_difficulty":1000009,"ntime":1700000723,"order_key":"08:miner-8","p2mr_program_hex":"d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3","share_difficulty":1000,"share_id":"share-723","share_seq":723,"template_height":800022},{"accepted_at_ms":999997,"credit_policy":null,"job_id":"job-3525","job_issued_at_ms":999983,"miner_id":"miner-5","network_difficulty":1000006,"ntime":1700003525,"order_key":"05:miner-5","p2mr_program_hex":"c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5","share_difficulty":2,"share_id":"share-3525","share_seq":3525,"template_height":800110},{"accepted_at_ms":999997,"credit_policy":null,"job_id":"job-2390","job_issued_at_ms":999981,"miner_id":"miner-3","network_difficulty":1000010,"ntime":1700002390,"order_key":"03:miner-3","p2mr_program_hex":"5656565656565656565656565656565656565656565656565656565656565656","share_difficulty":1,"share_id":"share-2390","share_seq":2390,"template_height":800074},{"accepted_at_ms":999988,"credit_policy":null,"job_id":"job-1944","job_issued_at_ms":999995,"miner_id":"miner-8","network_difficulty":1000006,"ntime":1700001944,"order_key":"08:miner-8","p2mr_program_hex":"9898989898989898989898989898989898989898989898989898989898989898","share_difficulty":2,"share_id":"share-1944","share_seq":1944,"template_height":800060},{"accepted_at_ms":999986,"credit_policy":null,"job_id":"job-2841","job_issued_at_ms":1000428,"miner_id":"miner-3","network_difficulty":1000002,"ntime":1700002841,"order_key":"03:miner-3","p2mr_program_hex":"1919191919191919191919191919191919191919191919191919191919191919","share_difficulty":7,"share_id":"share-2841","share_seq":2841,"template_height":800088},{"accepted_at_ms":999974,"credit_policy":null,"job_id":"job-3598","job_issued_at_ms":999975,"miner_id":"miner-1","network_difficulty":1000011,"ntime":1700003598,"order_key":"01:miner-1","p2mr_program_hex":"0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e","share_difficulty":3,"share_id":"share-3598","share_seq":3598,"template_height":800112},{"accepted_at_ms":999991,"credit_policy":null,"job_id":"job-2861","job_issued_at_ms":999989,"miner_id":"miner-1","network_difficulty":1000005,"ntime":1700002861,"order_key":"01:miner-1","p2mr_program_hex":"2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d","share_difficulty":101,"share_id":"share-2861","share_seq":2861,"template_height":800089},{"accepted_at_ms":999996,"credit_policy":null,"job_id":"job-1014","job_issued_at_ms":999991,"miner_id":"miner-2","network_difficulty":1000011,"ntime":1700001014,"order_key":"02:miner-2","p2mr_program_hex":"f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6","share_difficulty":3,"share_id":"share-1014","share_seq":1014,"template_height":800031},{"accepted_at_ms":999988,"credit_policy":"stale-grace","job_id":"job-1310","job_issued_at_ms":999981,"miner_id":"miner-1","network_difficulty":1000001,"ntime":1700001310,"order_key":"01:miner-1","p2mr_program_hex":"1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e","share_difficulty":2,"share_id":"share-1310","share_seq":1310,"template_height":800040},{"accepted_at_ms":999972,"credit_policy":null,"job_id":"job-2711","job_issued_at_ms":999971,"miner_id":"miner-5","network_difficulty":1000008,"ntime":1700002711,"order_key":"05:miner-5","p2mr_program_hex":"9797979797979797979797979797979797979797979797979797979797979797","share_difficulty":1,"share_id":"share-2711","share_seq":2711,"template_height":800084},{"accepted_at_ms":999999,"credit_policy":null,"job_id":"job-2002","job_issued_at_ms":999978,"miner_id":"miner-0","network_difficulty":1000013,"ntime":1700002002,"order_key":"00:miner-0","p2mr_program_hex":"d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2","share_difficulty":101,"share_id":"share-2002","share_seq":2002,"template_height":800062},{"accepted_at_ms":999974,"credit_policy":null,"job_id":"job-2195","job_issued_at_ms":999999,"miner_id":"miner-6","network_difficulty":1000002,"ntime":1700002195,"order_key":"06:miner-6","p2mr_program_hex":"9393939393939393939393939393939393939393939393939393939393939393","share_difficulty":3,"share_id":"share-2195","share_seq":2195,"template_height":800068},{"accepted_at_ms":999983,"credit_policy":null,"job_id":"job-1227","job_issued_at_ms":999970,"miner_id":"miner-6","network_difficulty":1000003,"ntime":1700001227,"order_key":"06:miner-6","p2mr_program_hex":"cbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcb","share_difficulty":19,"share_id":"share-1227","share_seq":1227,"template_height":800038},{"accepted_at_ms":999974,"credit_policy":null,"job_id":"job-1822","job_issued_at_ms":999987,"miner_id":"miner-7","network_difficulty":1000003,"ntime":1700001822,"order_key":"07:miner-7","p2mr_program_hex":"1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e","share_difficulty":7,"share_id":"share-1822","share_seq":1822,"template_height":800056},{"accepted_at_ms":999996,"credit_policy":null,"job_id":"job-3806","job_issued_at_ms":999982,"miner_id":"miner-0","network_difficulty":1000015,"ntime":1700003806,"order_key":"00:miner-0","p2mr_program_hex":"dededededededededededededededededededededededededededededededede","share_difficulty":3,"share_id":"share-3806","share_seq":3806,"template_height":800118},{"accepted_at_ms":999991,"credit_policy":"stale-grace","job_id":"job-2171","job_issued_at_ms":999970,"miner_id":"miner-4","network_difficulty":1000012,"ntime":1700002171,"order_key":"04:miner-4","p2mr_program_hex":"7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b","share_difficulty":2,"share_id":"share-2171","share_seq":2171,"template_height":800067},{"accepted_at_ms":999992,"credit_policy":null,"job_id":"job-1849","job_issued_at_ms":999996,"miner_id":"miner-1","network_difficulty":1000013,"ntime":1700001849,"order_key":"01:miner-1","p2mr_program_hex":"3939393939393939393939393939393939393939393939393939393939393939","share_difficulty":2,"share_id":"share-1849","share_seq":1849,"template_height":800057},{"accepted_at_ms":999979,"credit_policy":null,"job_id":"job-484","job_issued_at_ms":999985,"miner_id":"miner-0","network_difficulty":1000008,"ntime":1700000484,"order_key":"00:miner-0","p2mr_program_hex":"e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4","share_difficulty":7,"share_id":"share-484","share_seq":484,"template_height":800015},{"accepted_at_ms":999993,"credit_policy":"stale-grace","job_id":"job-1177","job_issued_at_ms":999971,"miner_id":"miner-0","network_difficulty":1000004,"ntime":1700001177,"order_key":"00:miner-0","p2mr_program_hex":"9999999999999999999999999999999999999999999999999999999999999999","share_difficulty":2,"share_id":"share-1177","share_seq":1177,"template_height":800036},{"accepted_at_ms":999988,"credit_policy":null,"job_id":"job-3044","job_issued_at_ms":999991,"miner_id":"miner-8","network_difficulty":1000001,"ntime":1700003044,"order_key":"08:miner-8","p2mr_program_hex":"e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4","share_difficulty":1,"share_id":"share-3044","share_seq":3044,"template_height":800095},{"accepted_at_ms":999997,"credit_policy":null,"job_id":"job-302","job_issued_at_ms":999991,"miner_id":"miner-5","network_difficulty":1000013,"ntime":1700000302,"order_key":"05:miner-5","p2mr_program_hex":"2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e","share_difficulty":3,"share_id":"share-302","share_seq":302,"template_height":800009},{"accepted_at_ms":999983,"credit_policy":null,"job_id":"job-932","job_issued_at_ms":999970,"miner_id":"miner-8","network_difficulty":1000014,"ntime":1700000932,"order_key":"08:miner-8","p2mr_program_hex":"a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4","share_difficulty":2,"share_id":"share-932","share_seq":932,"template_height":800029},{"accepted_at_ms":999974,"credit_policy":null,"job_id":"job-3199","job_issued_at_ms":999970,"miner_id":"miner-9","network_difficulty":1000003,"ntime":1700003199,"order_key":"09:miner-9","p2mr_program_hex":"7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f","share_difficulty":1000,"share_id":"share-3199","share_seq":3199,"template_height":800099},{"accepted_at_ms":999970,"credit_policy":null,"job_id":"job-679","job_issued_at_ms":999983,"miner_id":"miner-8","network_difficulty":1000016,"ntime":1700000679,"order_key":"08:miner-8","p2mr_program_hex":"a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7","share_difficulty":1000,"share_id":"share-679","share_seq":679,"template_height":800021},{"accepted_at_ms":999995,"credit_policy":null,"job_id":"job-3279","job_issued_at_ms":999990,"miner_id":"miner-1","network_difficulty":1000015,"ntime":1700003279,"order_key":"01:miner-1","p2mr_program_hex":"cfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcf","share_difficulty":1000,"share_id":"share-3279","share_seq":3279,"template_height":800102},{"accepted_at_ms":1000412,"credit_policy":null,"job_id":"job-1446","job_issued_at_ms":999983,"miner_id":"miner-5","network_difficulty":1000001,"ntime":1700001446,"order_key":"05:miner-5","p2mr_program_hex":"a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6","share_difficulty":3,"share_id":"share-1446","share_seq":1446,"template_height":800045},{"accepted_at_ms":999972,"credit_policy":"stale-grace","job_id":"job-3002","job_issued_at_ms":999995,"miner_id":"miner-10","network_difficulty":1000010,"ntime":1700003002,"order_key":"10:miner-10","p2mr_program_hex":"babababababababababababababababababababababababababababababababa","share_difficulty":1,"share_id":"share-3002","share_seq":3002,"template_height":800093},{"accepted_at_ms":999995,"credit_policy":null,"job_id":"job-928","job_issued_at_ms":999973,"miner_id":"miner-4","network_difficulty":1000010,"ntime":1700000928,"order_key":"04:miner-4","p2mr_program_hex":"a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0","share_difficulty":1000,"share_id":"share-928","share_seq":928,"template_height":800029},{"accepted_at_ms":999996,"credit_policy":"stale-grace","job_id":"job-3437","job_issued_at_ms":999987,"miner_id":"miner-5","network_difficulty":1000003,"ntime":1700003437,"order_key":"05:miner-5","p2mr_program_hex":"6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d","share_difficulty":1000,"share_id":"share-3437","share_seq":3437,"template_height":800107},{"accepted_at_ms":999986,"credit_policy":null,"job_id":"job-2601","job_issued_at_ms":999985,"miner_id":"miner-5","network_difficulty":1000000,"ntime":1700002601,"order_key":"05:miner-5","p2mr_program_hex":"2929292929292929292929292929292929292929292929292929292929292929","share_difficulty":1,"share_id":"share-2601","share_seq":2601,"template_height":800081},{"accepted_at_ms":999977,"credit_policy":null,"job_id":"job-397","job_issued_at_ms":999975,"miner_id":"miner-1","network_difficulty":1000006,"ntime":1700000397,"order_key":"01:miner-1","p2mr_program_hex":"8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d","share_difficulty":3,"share_id":"share-397","share_seq":397,"template_height":800012},{"accepted_at_ms":999993,"credit_policy":"stale-grace","job_id":"job-629","job_issued_at_ms":999974,"miner_id":"miner-2","network_difficulty":1000000,"ntime":1700000629,"order_key":"02:miner-2","p2mr_program_hex":"7575757575757575757575757575757575757575757575757575757575757575","share_difficulty":2,"share_id":"share-629","share_seq":629,"template_height":800019},{"accepted_at_ms":999977,"credit_policy":null,"job_id":"job-1028","job_issued_at_ms":999993,"miner_id":"miner-5","network_difficulty":1000008,"ntime":1700001028,"order_key":"05:miner-5","p2mr_program_hex":"0404040404040404040404040404040404040404040404040404040404040404","share_difficulty":1000,"share_id":"share-1028","share_seq":1028,"template_height":800032},{"accepted_at_ms":999979,"credit_policy":null,"job_id":"job-1938","job_issued_at_ms":999992,"miner_id":"miner-2","network_difficulty":1000000,"ntime":1700001938,"order_key":"02:miner-2","p2mr_program_hex":"9292929292929292929292929292929292929292929292929292929292929292","share_difficulty":3,"share_id":"share-1938","share_seq":1938,"template_height":800060},{"accepted_at_ms":1000000,"credit_policy":"stale-grace","job_id":"job-3061","job_issued_at_ms":999970,"miner_id":"miner-3","network_difficulty":1000001,"ntime":1700003061,"order_key":"03:miner-3","p2mr_program_hex":"f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5","share_difficulty":1000,"share_id":"share-3061","share_seq":3061,"template_height":800095},{"accepted_at_ms":1000470,"credit_policy":null,"job_id":"job-1416","job_issued_at_ms":999988,"miner_id":"miner-8","network_difficulty":1000005,"ntime":1700001416,"order_key":"08:miner-8","p2mr_program_hex":"8888888888888888888888888888888888888888888888888888888888888888","share_difficulty":1,"share_id":"share-1416","share_seq":1416,"template_height":800044},{"accepted_at_ms":999996,"credit_policy":null,"job_id":"job-1599","job_issued_at_ms":999993,"miner_id":"miner-4","network_difficulty":1000001,"ntime":1700001599,"order_key":"04:miner-4","p2mr_program_hex":"3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f","share_difficulty":19,"share_id":"share-1599","share_seq":1599,"template_height":800049},{"accepted_at_ms":999998,"credit_policy":null,"job_id":"job-2860","job_issued_at_ms":999989,"miner_id":"miner-0","network_difficulty":1000004,"ntime":1700002860,"order_key":"00:miner-0","p2mr_program_hex":"2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c","share_difficulty":7,"share_id":"share-2860","share_seq":2860,"template_height":800089},{"accepted_at_ms":999993,"credit_policy":null,"job_id":"job-3429","job_issued_at_ms":999986,"miner_id":"miner-8","network_difficulty":1000012,"ntime":1700003429,"order_key":"08:miner-8","p2mr_program_hex":"6565656565656565656565656565656565656565656565656565656565656565","share_difficulty":2,"share_id":"share-3429","share_seq":3429,"template_height":800107},{"accepted_at_ms":1000000,"credit_policy":null,"job_id":"job-3290","job_issued_at_ms":999980,"miner_id":"miner-1","network_difficulty":1000009,"ntime":1700003290,"order_key":"01:miner-1","p2mr_program_hex":"dadadadadadadadadadadadadadadadadadadadadadadadadadadadadadadada","share_difficulty":2,"share_id":"share-3290","share_seq":3290,"template_height":800102},{"accepted_at_ms":999987,"credit_policy":null,"job_id":"job-705","job_issued_at_ms":999988,"miner_id":"miner-1","network_difficulty":1000008,"ntime":1700000705,"order_key":"01:miner-1","p2mr_program_hex":"c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1","share_difficulty":1000,"share_id":"share-705","share_seq":705,"template_height":800022},{"accepted_at_ms":999994,"credit_policy":null,"job_id":"job-424","job_issued_at_ms":999974,"miner_id":"miner-6","network_difficulty":1000016,"ntime":1700000424,"order_key":"06:miner-6","p2mr_program_hex":"a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8","share_difficulty":19,"share_id":"share-424","share_seq":424,"template_height":800013},{"accepted_at_ms":1000257,"credit_policy":"stale-grace","job_id":"job-3520","job_issued_at_ms":1000454,"miner_id":"miner-0","network_difficulty":1000001,"ntime":1700003520,"order_key":"00:miner-0","p2mr_program_hex":"c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0","share_difficulty":7,"share_id":"share-3520","share_seq":3520,"template_height":800110},{"accepted_at_ms":999982,"credit_policy":null,"job_id":"job-3571","job_issued_at_ms":999976,"miner_id":"miner-7","network_difficulty":1000001,"ntime":1700003571,"order_key":"07:miner-7","p2mr_program_hex":"f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3","share_difficulty":1000,"share_id":"share-3571","share_seq":3571,"template_height":800111},{"accepted_at_ms":999996,"credit_policy":"stale-grace","job_id":"job-3917","job_issued_at_ms":999972,"miner_id":"miner-1","network_difficulty":1000007,"ntime":1700003917,"order_key":"01:miner-1","p2mr_program_hex":"4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d","share_difficulty":7,"share_id":"share-3917","share_seq":3917,"template_height":800122},{"accepted_at_ms":999992,"credit_policy":null,"job_id":"job-2052","job_issued_at_ms":1000000,"miner_id":"miner-6","network_difficulty":1000012,"ntime":1700002052,"order_key":"06:miner-6","p2mr_program_hex":"0404040404040404040404040404040404040404040404040404040404040404","share_difficulty":101,"share_id":"share-2052","share_seq":2052,"template_height":800064},{"accepted_at_ms":999998,"credit_policy":null,"job_id":"job-1771","job_issued_at_ms":999976,"miner_id":"miner-0","network_difficulty":1000003,"ntime":1700001771,"order_key":"00:miner-0","p2mr_program_hex":"ebebebebebebebebebebebebebebebebebebebebebebebebebebebebebebebeb","share_difficulty":7,"share_id":"share-1771","share_seq":1771,"template_height":800055},{"accepted_at_ms":999971,"credit_policy":null,"job_id":"job-1945","job_issued_at_ms":999988,"miner_id":"miner-9","network_difficulty":1000007,"ntime":1700001945,"order_key":"09:miner-9","p2mr_program_hex":"9999999999999999999999999999999999999999999999999999999999999999","share_difficulty":3,"share_id":"share-1945","share_seq":1945,"template_height":800060},{"accepted_at_ms":999979,"credit_policy":null,"job_id":"job-883","job_issued_at_ms":999995,"miner_id":"miner-3","network_difficulty":1000016,"ntime":1700000883,"order_key":"03:miner-3","p2mr_program_hex":"7373737373737373737373737373737373737373737373737373737373737373","share_difficulty":2,"share_id":"share-883","share_seq":883,"template_height":800027},{"accepted_at_ms":999992,"credit_policy":null,"job_id":"job-1736","job_issued_at_ms":1000207,"miner_id":"miner-9","network_difficulty":1000002,"ntime":1700001736,"order_key":"09:miner-9","p2mr_program_hex":"c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8","share_difficulty":2,"share_id":"share-1736","share_seq":1736,"template_height":800054},{"accepted_at_ms":999971,"credit_policy":null,"job_id":"job-2828","job_issued_at_ms":999978,"miner_id":"miner-1","network_difficulty":1000006,"ntime":1700002828,"order_key":"01:miner-1","p2mr_program_hex":"0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c","share_difficulty":101,"share_id":"share-2828","share_seq":2828,"template_height":800088},{"accepted_at_ms":1000000,"credit_policy":"stale-grace","job_id":"job-1233","job_issued_at_ms":999986,"miner_id":"miner-1","network_difficulty":1000009,"ntime":1700001233,"order_key":"01:miner-1","p2mr_program_hex":"d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1","share_difficulty":1,"share_id":"share-1233","share_seq":1233,"template_height":800038},{"accepted_at_ms":999995,"credit_policy":null,"job_id":"job-3740","job_issued_at_ms":999972,"miner_id":"miner-0","network_difficulty":1000000,"ntime":1700003740,"order_key":"00:miner-0","p2mr_program_hex":"9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c","share_difficulty":19,"share_id":"share-3740","share_seq":3740,"template_height":800116},{"accepted_at_ms":999977,"credit_policy":"stale-grace","job_id":"job-1714","job_issued_at_ms":999987,"miner_id":"miner-9","network_difficulty":1000014,"ntime":1700001714,"order_key":"09:miner-9","p2mr_program_hex":"b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2","share_difficulty":3,"share_id":"share-1714","share_seq":1714,"template_height":800053},{"accepted_at_ms":999999,"credit_policy":null,"job_id":"job-2485","job_issued_at_ms":999992,"miner_id":"miner-10","network_difficulty":1000003,"ntime":1700002485,"order_key":"10:miner-10","p2mr_program_hex":"b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5","share_difficulty":3,"share_id":"share-2485","share_seq":2485,"template_height":800077},{"accepted_at_ms":999997,"credit_policy":"stale-grace","job_id":"job-999","job_issued_at_ms":1000272,"miner_id":"miner-9","network_difficulty":1000013,"ntime":1700000999,"order_key":"09:miner-9","p2mr_program_hex":"e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7","share_difficulty":3,"share_id":"share-999","share_seq":999,"template_height":800031},{"accepted_at_ms":999972,"credit_policy":"stale-grace","job_id":"job-1786","job_issued_at_ms":999984,"miner_id":"miner-4","network_difficulty":1000001,"ntime":1700001786,"order_key":"04:miner-4","p2mr_program_hex":"fafafafafafafafafafafafafafafafafafafafafafafafafafafafafafafafa","share_difficulty":19,"share_id":"share-1786","share_seq":1786,"template_height":800055},{"accepted_at_ms":999974,"credit_policy":null,"job_id":"job-7","job_issued_at_ms":999973,"miner_id":"miner-7","network_difficulty":1000007,"ntime":1700000007,"order_key":"07:miner-7","p2mr_program_hex":"0707070707070707070707070707070707070707070707070707070707070707","share_difficulty":2,"share_id":"share-7","share_seq":7,"template_height":800000},{"accepted_at_ms":1000365,"credit_policy":"stale-grace","job_id":"job-171","job_issued_at_ms":999981,"miner_id":"miner-6","network_difficulty":1000001,"ntime":1700000171,"order_key":"06:miner-6","p2mr_program_hex":"abababababababababababababababababababababababababababababababab","share_difficulty":7,"share_id":"share-171","share_seq":171,"template_height":800005},{"accepted_at_ms":999993,"credit_policy":null,"job_id":"job-1450","job_issued_at_ms":999984,"miner_id":"miner-9","network_difficulty":1000005,"ntime":1700001450,"order_key":"09:miner-9","p2mr_program_hex":"aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa","share_difficulty":1000,"share_id":"share-1450","share_seq":1450,"template_height":800045},{"accepted_at_ms":999993,"credit_policy":null,"job_id":"job-12","job_issued_at_ms":999990,"miner_id":"miner-1","network_difficulty":1000012,"ntime":1700000012,"order_key":"01:miner-1","p2mr_program_hex":"0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c","share_difficulty":1000,"share_id":"share-12","share_seq":12,"template_height":800000},{"accepted_at_ms":999986,"credit_policy":"stale-grace","job_id":"job-1563","job_issued_at_ms":1000267,"miner_id":"miner-1","network_difficulty":1000016,"ntime":1700001563,"order_key":"01:miner-1","p2mr_program_hex":"1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b","share_difficulty":3,"share_id":"share-1563","share_seq":1563,"template_height":800048},{"accepted_at_ms":999978,"credit_policy":null,"job_id":"job-254","job_issued_at_ms":999980,"miner_id":"miner-1","network_difficulty":1000016,"ntime":1700000254,"order_key":"01:miner-1","p2mr_program_hex":"fefefefefefefefefefefefefefefefefefefefefefefefefefefefefefefefe","share_difficulty":19,"share_id":"share-254","share_seq":254,"template_height":800007},{"accepted_at_ms":999992,"credit_policy":null,"job_id":"job-1605","job_issued_at_ms":999993,"miner_id":"miner-10","network_difficulty":1000007,"ntime":1700001605,"order_key":"10:miner-10","p2mr_program_hex":"4545454545454545454545454545454545454545454545454545454545454545","share_difficulty":19,"share_id":"share-1605","share_seq":1605,"template_height":800050},{"accepted_at_ms":999975,"credit_policy":null,"job_id":"job-2150","job_issued_at_ms":999977,"miner_id":"miner-5","network_difficulty":1000008,"ntime":1700002150,"order_key":"05:miner-5","p2mr_program_hex":"6666666666666666666666666666666666666666666666666666666666666666","share_difficulty":2,"share_id":"share-2150","share_seq":2150,"template_height":800067},{"accepted_at_ms":999983,"credit_policy":null,"job_id":"job-2322","job_issued_at_ms":999974,"miner_id":"miner-1","network_difficulty":1000010,"ntime":1700002322,"order_key":"01:miner-1","p2mr_program_hex":"1212121212121212121212121212121212121212121212121212121212121212","share_difficulty":3,"share_id":"share-2322","share_seq":2322,"template_height":800072},{"accepted_at_ms":999978,"credit_policy":null,"job_id":"job-3501","job_issued_at_ms":999984,"miner_id":"miner-3","network_difficulty":1000016,"ntime":1700003501,"order_key":"03:miner-3","p2mr_program_hex":"adadadadadadadadadadadadadadadadadadadadadadadadadadadadadadadad","share_difficulty":3,"share_id":"share-3501","share_seq":3501,"template_height":800109},{"accepted_at_ms":999989,"credit_policy":null,"job_id":"job-2911","job_issued_at_ms":999990,"miner_id":"miner-7","network_difficulty":1000004,"ntime":1700002911,"order_key":"07:miner-7","p2mr_program_hex":"5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f","share_difficulty":101,"share_id":"share-2911","share_seq":2911,"template_height":800090},{"accepted_at_ms":999981,"credit_policy":null,"job_id":"job-851","job_issued_at_ms":999987,"miner_id":"miner-4","network_difficulty":1000001,"ntime":1700000851,"order_key":"04:miner-4","p2mr_program_hex":"5353535353535353535353535353535353535353535353535353535353535353","share_difficulty":1000,"share_id":"share-851","share_seq":851,"template_height":800026},{"accepted_at_ms":999971,"credit_policy":null,"job_id":"job-3399","job_issued_at_ms":999981,"miner_id":"miner-0","network_difficulty":1000016,"ntime":1700003399,"order_key":"00:miner-0","p2mr_program_hex":"4747474747474747474747474747474747474747474747474747474747474747","share_difficulty":1000,"share_id":"share-3399","share_seq":3399,"template_height":800106},{"accepted_at_ms":999983,"credit_policy":null,"job_id":"job-1292","job_issued_at_ms":999984,"miner_id":"miner-5","network_difficulty":1000000,"ntime":1700001292,"order_key":"05:miner-5","p2mr_program_hex":"0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c","share_difficulty":3,"share_id":"share-1292","share_seq":1292,"template_height":800040},{"accepted_at_ms":999971,"credit_policy":"stale-grace","job_id":"job-801","job_issued_at_ms":999991,"miner_id":"miner-9","network_difficulty":1000002,"ntime":1700000801,"order_key":"09:miner-9","p2mr_program_hex":"2121212121212121212121212121212121212121212121212121212121212121","share_difficulty":3,"share_id":"share-801","share_seq":801,"template_height":800025},{"accepted_at_ms":999996,"credit_policy":null,"job_id":"job-2046","job_issued_at_ms":999976,"miner_id":"miner-0","network_difficulty":1000006,"ntime":1700002046,"order_key":"00:miner-0","p2mr_program_hex":"fefefefefefefefefefefefefefefefefefefefefefefefefefefefefefefefe","share_difficulty":101,"share_id":"share-2046","share_seq":2046,"template_height":800063},{"accepted_at_ms":999975,"credit_policy":"stale-grace","job_id":"job-1772","job_issued_at_ms":999986,"miner_id":"miner-1","network_difficulty":1000004,"ntime":1700001772,"order_key":"01:miner-1","p2mr_program_hex":"ecececececececececececececececececececececececececececececececec","share_difficulty":19,"share_id":"share-1772","share_seq":1772,"template_height":800055},{"accepted_at_ms":999991,"credit_policy":null,"job_id":"job-3626","job_issued_at_ms":999982,"miner_id":"miner-7","network_difficulty":1000005,"ntime":1700003626,"order_key":"07:miner-7","p2mr_program_hex":"2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a","share_difficulty":1000,"share_id":"share-3626","share_seq":3626,"template_height":800113},{"accepted_at_ms":999987,"credit_policy":null,"job_id":"job-452","job_issued_at_ms":999995,"miner_id":"miner-1","network_difficulty":1000010,"ntime":1700000452,"order_key":"01:miner-1","p2mr_program_hex":"c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4","share_difficulty":3,"share_id":"share-452","share_seq":452,"template_height":800014},{"accepted_at_ms":1000415,"credit_policy":"stale-grace","job_id":"job-1106","job_issued_at_ms":999988,"miner_id":"miner-6","network_difficulty":1000001,"ntime":1700001106,"order_key":"06:miner-6","p2mr_program_hex":"5252525252525252525252525252525252525252525252525252525252525252","share_difficulty":19,"share_id":"share-1106","share_seq":1106,"template_height":800034},{"accepted_at_ms":999970,"credit_policy":null,"job_id":"job-2056","job_issued_at_ms":999980,"miner_id":"miner-10","network_difficulty":1000016,"ntime":1700002056,"order_key":"10:miner-10","p2mr_program_hex":"0808080808080808080808080808080808080808080808080808080808080808","share_difficulty":7,"share_id":"share-2056","share_seq":2056,"template_height":800064},{"accepted_at_ms":999995,"credit_policy":null,"job_id":"job-230","job_issued_at_ms":999984,"miner_id":"miner-10","network_difficulty":1000009,"ntime":1700000230,"order_key":"10:miner-10","p2mr_program_hex":"e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6","share_difficulty":19,"share_id":"share-230","share_seq":230,"template_height":800007},{"accepted_at_ms":999982,"credit_policy":null,"job_id":"job-2426","job_issued_at_ms":999990,"miner_id":"miner-6","network_difficulty":1000012,"ntime":1700002426,"order_key":"06:miner-6","p2mr_program_hex":"7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a","share_difficulty":101,"share_id":"share-2426","share_seq":2426,"template_height":800075},{"accepted_at_ms":999990,"credit_policy":"stale-grace","job_id":"job-589","job_issued_at_ms":999989,"miner_id":"miner-6","network_difficulty":1000011,"ntime":1700000589,"order_key":"06:miner-6","p2mr_program_hex":"4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d","share_difficulty":101,"share_id":"share-589","share_seq":589,"template_height":800018},{"accepted_at_ms":999984,"credit_policy":null,"job_id":"job-2673","job_issued_at_ms":999985,"miner_id":"miner-0","network_difficulty":1000004,"ntime":1700002673,"order_key":"00:miner-0","p2mr_program_hex":"7171717171717171717171717171717171717171717171717171717171717171","share_difficulty":101,"share_id":"share-2673","share_seq":2673,"template_height":800083},{"accepted_at_ms":999970,"credit_policy":"stale-grace","job_id":"job-155","job_issued_at_ms":999995,"miner_id":"miner-1","network_difficulty":1000002,"ntime":1700000155,"order_key":"01:miner-1","p2mr_program_hex":"9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b","share_difficulty":1,"share_id":"share-155","share_seq":155,"template_height":800004},{"accepted_at_ms":999981,"credit_policy":"stale-grace","job_id":"job-3946","job_issued_at_ms":999987,"miner_id":"miner-8","network_difficulty":1000002,"ntime":1700003946,"order_key":"08:miner-8","p2mr_program_hex":"6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a","share_difficulty":7,"share_id":"share-3946","share_seq":3946,"template_height":800123},{"accepted_at_ms":999989,"credit_policy":"stale-grace","job_id":"job-3604","job_issued_at_ms":999977,"miner_id":"miner-7","network_difficulty":1000000,"ntime":1700003604,"order_key":"07:miner-7","p2mr_program_hex":"1414141414141414141414141414141414141414141414141414141414141414","share_difficulty":101,"share_id":"share-3604","share_seq":3604,"template_height":800112},{"accepted_at_ms":999977,"credit_policy":null,"job_id":"job-2286","job_issued_at_ms":999999,"miner_id":"miner-9","network_difficulty":1000008,"ntime":1700002286,"order_key":"09:miner-9","p2mr_program_hex":"eeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeee","share_difficulty":7,"share_id":"share-2286","share_seq":2286,"template_height":800071},{"accepted_at_ms":999988,"credit_policy":"stale-grace","job_id":"job-561","job_issued_at_ms":999991,"miner_id":"miner-0","network_difficulty":1000000,"ntime":1700000561,"order_key":"00:miner-0","p2mr_program_hex":"3131313131313131313131313131313131313131313131313131313131313131","share_difficulty":7,"share_id":"share-561","share_seq":561,"template_height":800017},{"accepted_at_ms":999975,"credit_policy":null,"job_id":"job-2410","job_issued_at_ms":999984,"miner_id":"miner-1","network_difficulty":1000013,"ntime":1700002410,"order_key":"01:miner-1","p2mr_program_hex":"6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a","share_difficulty":1,"share_id":"share-2410","share_seq":2410,"template_height":800075},{"accepted_at_ms":999983,"credit_policy":null,"job_id":"job-3033","job_issued_at_ms":999991,"miner_id":"miner-8","network_difficulty":1000007,"ntime":1700003033,"order_key":"08:miner-8","p2mr_program_hex":"d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9","share_difficulty":3,"share_id":"share-3033","share_seq":3033,"template_height":800094},{"accepted_at_ms":999996,"credit_policy":null,"job_id":"job-3326","job_issued_at_ms":999980,"miner_id":"miner-4","network_difficulty":1000011,"ntime":1700003326,"order_key":"04:miner-4","p2mr_program_hex":"fefefefefefefefefefefefefefefefefefefefefefefefefefefefefefefefe","share_difficulty":2,"share_id":"share-3326","share_seq":3326,"template_height":800103},{"accepted_at_ms":999987,"credit_policy":null,"job_id":"job-1026","job_issued_at_ms":999984,"miner_id":"miner-3","network_difficulty":1000006,"ntime":1700001026,"order_key":"03:miner-3","p2mr_program_hex":"0202020202020202020202020202020202020202020202020202020202020202","share_difficulty":1,"share_id":"share-1026","share_seq":1026,"template_height":800032},{"accepted_at_ms":999997,"credit_policy":null,"job_id":"job-588","job_issued_at_ms":999977,"miner_id":"miner-5","network_difficulty":1000010,"ntime":1700000588,"order_key":"05:miner-5","p2mr_program_hex":"4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c","share_difficulty":1,"share_id":"share-588","share_seq":588,"template_height":800018},{"accepted_at_ms":999983,"credit_policy":null,"job_id":"job-1311","job_issued_at_ms":999996,"miner_id":"miner-2","network_difficulty":1000002,"ntime":1700001311,"order_key":"02:miner-2","p2mr_program_hex":"1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f","share_difficulty":1,"share_id":"share-1311","share_seq":1311,"template_height":800040},{"accepted_at_ms":999985,"credit_policy":null,"job_id":"job-2955","job_issued_at_ms":999972,"miner_id":"miner-7","network_difficulty":1000014,"ntime":1700002955,"order_key":"07:miner-7","p2mr_program_hex":"8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b","share_difficulty":7,"share_id":"share-2955","share_seq":2955,"template_height":800092},{"accepted_at_ms":1000104,"credit_policy":null,"job_id":"job-36","job_issued_at_ms":999970,"miner_id":"miner-3","network_difficulty":1000002,"ntime":1700000036,"order_key":"03:miner-3","p2mr_program_hex":"2424242424242424242424242424242424242424242424242424242424242424","share_difficulty":101,"share_id":"share-36","share_seq":36,"template_height":800001},{"accepted_at_ms":999986,"credit_policy":null,"job_id":"job-2380","job_issued_at_ms":999998,"miner_id":"miner-4","network_difficulty":1000000,"ntime":1700002380,"order_key":"04:miner-4","p2mr_program_hex":"4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c","share_difficulty":3,"share_id":"share-2380","share_seq":2380,"template_height":800074},{"accepted_at_ms":999971,"credit_policy":null,"job_id":"job-2272","job_issued_at_ms":999971,"miner_id":"miner-6","network_difficulty":1000011,"ntime":1700002272,"order_key":"06:miner-6","p2mr_program_hex":"e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0","share_difficulty":101,"share_id":"share-2272","share_seq":2272,"template_height":800071},{"accepted_at_ms":999983,"credit_policy":"stale-grace","job_id":"job-2158","job_issued_at_ms":999976,"miner_id":"miner-2","network_difficulty":1000016,"ntime":1700002158,"order_key":"02:miner-2","p2mr_program_hex":"6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e","share_difficulty":1000,"share_id":"share-2158","share_seq":2158,"template_height":800067},{"accepted_at_ms":999975,"credit_policy":null,"job_id":"job-384","job_issued_at_ms":999986,"miner_id":"miner-10","network_difficulty":1000010,"ntime":1700000384,"order_key":"10:miner-10","p2mr_program_hex":"8080808080808080808080808080808080808080808080808080808080808080","share_difficulty":19,"share_id":"share-384","share_seq":384,"template_height":800012},{"accepted_at_ms":999975,"credit_policy":"stale-grace","job_id":"job-1808","job_issued_at_ms":999988,"miner_id":"miner-4","network_difficulty":1000006,"ntime":1700001808,"order_key":"04:miner-4","p2mr_program_hex":"1010101010101010101010101010101010101010101010101010101010101010","share_difficulty":19,"share_id":"share-1808","share_seq":1808,"template_height":800056},{"accepted_at_ms":999971,"credit_policy":null,"job_id":"job-179","job_issued_at_ms":1000000,"miner_id":"miner-3","network_difficulty":1000009,"ntime":1700000179,"order_key":"03:miner-3","p2mr_program_hex":"b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3","share_difficulty":7,"share_id":"share-179","share_seq":179,"template_height":800005},{"accepted_at_ms":999992,"credit_policy":null,"job_id":"job-385","job_issued_at_ms":999977,"miner_id":"miner-0","network_difficulty":1000011,"ntime":1700000385,"order_key":"00:miner-0","p2mr_program_hex":"8181818181818181818181818181818181818181818181818181818181818181","share_difficulty":1000,"share_id":"share-385","share_seq":385,"template_height":800012},{"accepted_at_ms":1000229,"credit_policy":null,"job_id":"job-838","job_issued_at_ms":1000095,"miner_id":"miner-2","network_difficulty":1000005,"ntime":1700000838,"order_key":"02:miner-2","p2mr_program_hex":"4646464646464646464646464646464646464646464646464646464646464646","share_difficulty":101,"share_id":"share-838","share_seq":838,"template_height":800026},{"accepted_at_ms":999978,"credit_policy":null,"job_id":"job-1386","job_issued_at_ms":999980,"miner_id":"miner-0","network_difficulty":1000009,"ntime":1700001386,"order_key":"00:miner-0","p2mr_program_hex":"6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a","share_difficulty":1000,"share_id":"share-1386","share_seq":1386,"template_height":800043},{"accepted_at_ms":1000000,"credit_policy":null,"job_id":"job-3034","job_issued_at_ms":999979,"miner_id":"miner-9","network_difficulty":1000008,"ntime":1700003034,"order_key":"09:miner-9","p2mr_program_hex":"dadadadadadadadadadadadadadadadadadadadadadadadadadadadadadadada","share_difficulty":1000,"share_id":"share-3034","share_seq":3034,"template_height":800094},{"accepted_at_ms":999988,"credit_policy":null,"job_id":"job-625","job_issued_at_ms":1000451,"miner_id":"miner-9","network_difficulty":1000013,"ntime":1700000625,"order_key":"09:miner-9","p2mr_program_hex":"7171717171717171717171717171717171717171717171717171717171717171","share_difficulty":19,"share_id":"share-625","share_seq":625,"template_height":800019},{"accepted_at_ms":999999,"credit_policy":null,"job_id":"job-3301","job_issued_at_ms":999995,"miner_id":"miner-1","network_difficulty":1000003,"ntime":1700003301,"order_key":"01:miner-1","p2mr_program_hex":"e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5","share_difficulty":1000,"share_id":"share-3301","share_seq":3301,"template_height":800103},{"accepted_at_ms":999985,"credit_policy":"stale-grace","job_id":"job-1136","job_issued_at_ms":999995,"miner_id":"miner-3","network_difficulty":1000014,"ntime":1700001136,"order_key":"03:miner-3","p2mr_program_hex":"7070707070707070707070707070707070707070707070707070707070707070","share_difficulty":7,"share_id":"share-1136","share_seq":1136,"template_height":800035},{"accepted_at_ms":999980,"credit_policy":null,"job_id":"job-3693","job_issued_at_ms":999987,"miner_id":"miner-8","network_difficulty":1000004,"ntime":1700003693,"order_key":"08:miner-8","p2mr_program_hex":"6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d","share_difficulty":3,"share_id":"share-3693","share_seq":3693,"template_height":800115},{"accepted_at_ms":999974,"credit_policy":null,"job_id":"job-1965","job_issued_at_ms":999983,"miner_id":"miner-7","network_difficulty":1000010,"ntime":1700001965,"order_key":"07:miner-7","p2mr_program_hex":"adadadadadadadadadadadadadadadadadadadadadadadadadadadadadadadad","share_difficulty":19,"share_id":"share-1965","share_seq":1965,"template_height":800061},{"accepted_at_ms":999973,"credit_policy":"stale-grace","job_id":"job-1515","job_issued_at_ms":999983,"miner_id":"miner-8","network_difficulty":1000002,"ntime":1700001515,"order_key":"08:miner-8","p2mr_program_hex":"ebebebebebebebebebebebebebebebebebebebebebebebebebebebebebebebeb","share_difficulty":7,"share_id":"share-1515","share_seq":1515,"template_height":800047},{"accepted_at_ms":999980,"credit_policy":null,"job_id":"job-3505","job_issued_at_ms":999983,"miner_id":"miner-7","network_difficulty":1000003,"ntime":1700003505,"order_key":"07:miner-7","p2mr_program_hex":"b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1","share_difficulty":19,"share_id":"share-3505","share_seq":3505,"template_height":800109},{"accepted_at_ms":999977,"credit_policy":"stale-grace","job_id":"job-1636","job_issued_at_ms":999990,"miner_id":"miner-8","network_difficulty":1000004,"ntime":1700001636,"order_key":"08:miner-8","p2mr_program_hex":"6464646464646464646464646464646464646464646464646464646464646464","share_difficulty":101,"share_id":"share-1636","share_seq":1636,"template_height":800051},{"accepted_at_ms":999999,"credit_policy":null,"job_id":"job-606","job_issued_at_ms":999998,"miner_id":"miner-1","network_difficulty":1000011,"ntime":1700000606,"order_key":"01:miner-1","p2mr_program_hex":"5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e","share_difficulty":1000,"share_id":"share-606","share_seq":606,"template_height":800018},{"accepted_at_ms":999977,"credit_policy":null,"job_id":"job-3650","job_issued_at_ms":999997,"miner_id":"miner-9","network_difficulty":1000012,"ntime":1700003650,"order_key":"09:miner-9","p2mr_program_hex":"4242424242424242424242424242424242424242424242424242424242424242","share_difficulty":3,"share_id":"share-3650","share_seq":3650,"template_height":800114},{"accepted_at_ms":999994,"credit_policy":"stale-grace","job_id":"job-755","job_issued_at_ms":999993,"miner_id":"miner-7","network_difficulty":1000007,"ntime":1700000755,"order_key":"07:miner-7","p2mr_program_hex":"f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3","share_difficulty":19,"share_id":"share-755","share_seq":755,"template_height":800023},{"accepted_at_ms":999999,"credit_policy":null,"job_id":"job-3434","job_issued_at_ms":999971,"miner_id":"miner-2","network_difficulty":1000000,"ntime":1700003434,"order_key":"02:miner-2","p2mr_program_hex":"6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a","share_difficulty":3,"share_id":"share-3434","share_seq":3434,"template_height":800107},{"accepted_at_ms":999998,"credit_policy":"stale-grace","job_id":"job-521","job_issued_at_ms":999989,"miner_id":"miner-4","network_difficulty":1000011,"ntime":1700000521,"order_key":"04:miner-4","p2mr_program_hex":"0909090909090909090909090909090909090909090909090909090909090909","share_difficulty":3,"share_id":"share-521","share_seq":521,"template_height":800016},{"accepted_at_ms":999999,"credit_policy":null,"job_id":"job-2189","job_issued_at_ms":999987,"miner_id":"miner-0","network_difficulty":1000013,"ntime":1700002189,"order_key":"00:miner-0","p2mr_program_hex":"8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d","share_difficulty":2,"share_id":"share-2189","share_seq":2189,"template_height":800068},{"accepted_at_ms":999975,"credit_policy":"stale-grace","job_id":"job-2244","job_issued_at_ms":999988,"miner_id":"miner-0","network_difficulty":1000000,"ntime":1700002244,"order_key":"00:miner-0","p2mr_program_hex":"c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4","share_difficulty":2,"share_id":"share-2244","share_seq":2244,"template_height":800070},{"accepted_at_ms":999978,"credit_policy":"stale-grace","job_id":"job-2180","job_issued_at_ms":999995,"miner_id":"miner-2","network_difficulty":1000004,"ntime":1700002180,"order_key":"02:miner-2","p2mr_program_hex":"8484848484848484848484848484848484848484848484848484848484848484","share_difficulty":7,"share_id":"share-2180","share_seq":2180,"template_height":800068},{"accepted_at_ms":999998,"credit_policy":null,"job_id":"job-442","job_issued_at_ms":999983,"miner_id":"miner-2","network_difficulty":1000000,"ntime":1700000442,"order_key":"02:miner-2","p2mr_program_hex":"babababababababababababababababababababababababababababababababa","share_difficulty":1000,"share_id":"share-442","share_seq":442,"template_height":800013},{"accepted_at_ms":999996,"credit_policy":"stale-grace","job_id":"job-3680","job_issued_at_ms":999999,"miner_id":"miner-6","network_difficulty":1000008,"ntime":1700003680,"order_key":"06:miner-6","p2mr_program_hex":"6060606060606060606060606060606060606060606060606060606060606060","share_difficulty":1,"share_id":"share-3680","share_seq":3680,"template_height":800115},{"accepted_at_ms":999970,"credit_policy":null,"job_id":"job-730","job_issued_at_ms":999982,"miner_id":"miner-4","network_difficulty":1000016,"ntime":1700000730,"order_key":"04:miner-4","p2mr_program_hex":"dadadadadadadadadadadadadadadadadadadadadadadadadadadadadadadada","share_difficulty":7,"share_id":"share-730","share_seq":730,"template_height":800022},{"accepted_at_ms":999975,"credit_policy":null,"job_id":"job-704","job_issued_at_ms":999992,"miner_id":"miner-0","network_difficulty":1000007,"ntime":1700000704,"order_key":"00:miner-0","p2mr_program_hex":"c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0","share_difficulty":7,"share_id":"share-704","share_seq":704,"template_height":800022},{"accepted_at_ms":999990,"credit_policy":null,"job_id":"job-3497","job_issued_at_ms":999971,"miner_id":"miner-10","network_difficulty":1000012,"ntime":1700003497,"order_key":"10:miner-10","p2mr_program_hex":"a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9","share_difficulty":1000,"share_id":"share-3497","share_seq":3497,"template_height":800109},{"accepted_at_ms":999982,"credit_policy":null,"job_id":"job-545","job_issued_at_ms":999981,"miner_id":"miner-6","network_difficulty":1000001,"ntime":1700000545,"order_key":"06:miner-6","p2mr_program_hex":"2121212121212121212121212121212121212121212121212121212121212121","share_difficulty":3,"share_id":"share-545","share_seq":545,"template_height":800017},{"accepted_at_ms":999997,"credit_policy":null,"job_id":"job-2815","job_issued_at_ms":999973,"miner_id":"miner-10","network_difficulty":1000010,"ntime":1700002815,"order_key":"10:miner-10","p2mr_program_hex":"ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff","share_difficulty":3,"share_id":"share-2815","share_seq":2815,"template_height":800087},{"accepted_at_ms":999997,"credit_policy":null,"job_id":"job-1423","job_issued_at_ms":999976,"miner_id":"miner-4","network_difficulty":1000012,"ntime":1700001423,"order_key":"04:miner-4","p2mr_program_hex":"8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f","share_difficulty":19,"share_id":"share-1423","share_seq":1423,"template_height":800044},{"accepted_at_ms":999978,"credit_policy":"stale-grace","job_id":"job-1235","job_issued_at_ms":999973,"miner_id":"miner-3","network_difficulty":1000011,"ntime":1700001235,"order_key":"03:miner-3","p2mr_program_hex":"d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3","share_difficulty":19,"share_id":"share-1235","share_seq":1235,"template_height":800038},{"accepted_at_ms":999982,"credit_policy":"stale-grace","job_id":"job-2973","job_issued_at_ms":999991,"miner_id":"miner-3","network_difficulty":1000015,"ntime":1700002973,"order_key":"03:miner-3","p2mr_program_hex":"9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d","share_difficulty":7,"share_id":"share-2973","share_seq":2973,"template_height":800092},{"accepted_at_ms":999987,"credit_policy":null,"job_id":"job-3741","job_issued_at_ms":999979,"miner_id":"miner-1","network_difficulty":1000001,"ntime":1700003741,"order_key":"01:miner-1","p2mr_program_hex":"9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d","share_difficulty":1000,"share_id":"share-3741","share_seq":3741,"template_height":800116},{"accepted_at_ms":999992,"credit_policy":null,"job_id":"job-782","job_issued_at_ms":999971,"miner_id":"miner-1","network_difficulty":1000000,"ntime":1700000782,"order_key":"01:miner-1","p2mr_program_hex":"0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e","share_difficulty":7,"share_id":"share-782","share_seq":782,"template_height":800024},{"accepted_at_ms":999993,"credit_policy":null,"job_id":"job-2193","job_issued_at_ms":999993,"miner_id":"miner-4","network_difficulty":1000000,"ntime":1700002193,"order_key":"04:miner-4","p2mr_program_hex":"9191919191919191919191919191919191919191919191919191919191919191","share_difficulty":3,"share_id":"share-2193","share_seq":2193,"template_height":800068},{"accepted_at_ms":999984,"credit_policy":null,"job_id":"job-2622","job_issued_at_ms":999984,"miner_id":"miner-4","network_difficulty":1000004,"ntime":1700002622,"order_key":"04:miner-4","p2mr_program_hex":"3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e","share_difficulty":3,"share_id":"share-2622","share_seq":2622,"template_height":800081},{"accepted_at_ms":999991,"credit_policy":null,"job_id":"job-3964","job_issued_at_ms":999987,"miner_id":"miner-4","network_difficulty":1000003,"ntime":1700003964,"order_key":"04:miner-4","p2mr_program_hex":"7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c","share_difficulty":19,"share_id":"share-3964","share_seq":3964,"template_height":800123},{"accepted_at_ms":999988,"credit_policy":"stale-grace","job_id":"job-547","job_issued_at_ms":999977,"miner_id":"miner-8","network_difficulty":1000003,"ntime":1700000547,"order_key":"08:miner-8","p2mr_program_hex":"2323232323232323232323232323232323232323232323232323232323232323","share_difficulty":101,"share_id":"share-547","share_seq":547,"template_height":800017},{"accepted_at_ms":999991,"credit_policy":null,"job_id":"job-3766","job_issued_at_ms":999972,"miner_id":"miner-4","network_difficulty":1000009,"ntime":1700003766,"order_key":"04:miner-4","p2mr_program_hex":"b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6","share_difficulty":101,"share_id":"share-3766","share_seq":3766,"template_height":800117},{"accepted_at_ms":999995,"credit_policy":null,"job_id":"job-344","job_issued_at_ms":1000497,"miner_id":"miner-3","network_difficulty":1000004,"ntime":1700000344,"order_key":"03:miner-3","p2mr_program_hex":"5858585858585858585858585858585858585858585858585858585858585858","share_difficulty":2,"share_id":"share-344","share_seq":344,"template_height":800010},{"accepted_at_ms":999996,"credit_policy":null,"job_id":"job-3163","job_issued_at_ms":999993,"miner_id":"miner-6","network_difficulty":1000001,"ntime":1700003163,"order_key":"06:miner-6","p2mr_program_hex":"5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b","share_difficulty":101,"share_id":"share-3163","share_seq":3163,"template_height":800098},{"accepted_at_ms":1000000,"credit_policy":null,"job_id":"job-875","job_issued_at_ms":999972,"miner_id":"miner-6","network_difficulty":1000008,"ntime":1700000875,"order_key":"06:miner-6","p2mr_program_hex":"6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b","share_difficulty":3,"share_id":"share-875","share_seq":875,"template_height":800027},{"accepted_at_ms":999986,"credit_policy":null,"job_id":"job-471","job_issued_at_ms":999975,"miner_id":"miner-9","network_difficulty":1000012,"ntime":1700000471,"order_key":"09:miner-9","p2mr_program_hex":"d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7","share_difficulty":101,"share_id":"share-471","share_seq":471,"template_height":800014},{"accepted_at_ms":999993,"credit_policy":null,"job_id":"job-366","job_issued_at_ms":999972,"miner_id":"miner-3","network_difficulty":1000009,"ntime":1700000366,"order_key":"03:miner-3","p2mr_program_hex":"6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e","share_difficulty":19,"share_id":"share-366","share_seq":366,"template_height":800011},{"accepted_at_ms":999995,"credit_policy":null,"job_id":"job-3026","job_issued_at_ms":999982,"miner_id":"miner-1","network_difficulty":1000000,"ntime":1700003026,"order_key":"01:miner-1","p2mr_program_hex":"d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2","share_difficulty":19,"share_id":"share-3026","share_seq":3026,"template_height":800094},{"accepted_at_ms":999983,"credit_policy":null,"job_id":"job-3140","job_issued_at_ms":999999,"miner_id":"miner-5","network_difficulty":1000012,"ntime":1700003140,"order_key":"05:miner-5","p2mr_program_hex":"4444444444444444444444444444444444444444444444444444444444444444","share_difficulty":1,"share_id":"share-3140","share_seq":3140,"template_height":800098},{"accepted_at_ms":999990,"credit_policy":null,"job_id":"job-952","job_issued_at_ms":999990,"miner_id":"miner-6","network_difficulty":1000000,"ntime":1700000952,"order_key":"06:miner-6","p2mr_program_hex":"b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8","share_difficulty":2,"share_id":"share-952","share_seq":952,"template_height":800029},{"accepted_at_ms":999984,"credit_policy":"stale-grace","job_id":"job-3530","job_issued_at_ms":999984,"miner_id":"miner-10","network_difficulty":1000011,"ntime":1700003530,"order_key":"10:miner-10","p2mr_program_hex":"cacacacacacacacacacacacacacacacacacacacacacacacacacacacacacacaca","share_difficulty":1000,"share_id":"share-3530","share_seq":3530,"template_height":800110},{"accepted_at_ms":999988,"credit_policy":null,"job_id":"job-2773","job_issued_at_ms":999985,"miner_id":"miner-1","network_difficulty":1000002,"ntime":1700002773,"order_key":"01:miner-1","p2mr_program_hex":"d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5","share_difficulty":101,"share_id":"share-2773","share_seq":2773,"template_height":800086},{"accepted_at_ms":1000367,"credit_policy":null,"job_id":"job-1339","job_issued_at_ms":1000148,"miner_id":"miner-8","network_difficulty":1000013,"ntime":1700001339,"order_key":"08:miner-8","p2mr_program_hex":"3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b","share_difficulty":1,"share_id":"share-1339","share_seq":1339,"template_height":800041},{"accepted_at_ms":999998,"credit_policy":null,"job_id":"job-2233","job_issued_at_ms":999973,"miner_id":"miner-0","network_difficulty":1000006,"ntime":1700002233,"order_key":"00:miner-0","p2mr_program_hex":"b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9","share_difficulty":1,"share_id":"share-2233","share_seq":2233,"template_height":800069},{"accepted_at_ms":1000232,"credit_policy":"stale-grace","job_id":"job-2952","job_issued_at_ms":1000054,"miner_id":"miner-4","network_difficulty":1000011,"ntime":1700002952,"order_key":"04:miner-4","p2mr_program_hex":"8888888888888888888888888888888888888888888888888888888888888888","share_difficulty":19,"share_id":"share-2952","share_seq":2952,"template_height":800092},{"accepted_at_ms":999997,"credit_policy":"stale-grace","job_id":"job-1286","job_issued_at_ms":999975,"miner_id":"miner-10","network_difficulty":1000011,"ntime":1700001286,"order_key":"10:miner-10","p2mr_program_hex":"0606060606060606060606060606060606060606060606060606060606060606","share_difficulty":1,"share_id":"share-1286","share_seq":1286,"template_height":800040},{"accepted_at_ms":1000112,"credit_policy":null,"job_id":"job-3455","job_issued_at_ms":999999,"miner_id":"miner-1","network_difficulty":1000004,"ntime":1700003455,"order_key":"01:miner-1","p2mr_program_hex":"7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f","share_difficulty":3,"share_id":"share-3455","share_seq":3455,"template_height":800107},{"accepted_at_ms":999994,"credit_policy":null,"job_id":"job-1117","job_issued_at_ms":999983,"miner_id":"miner-6","network_difficulty":1000012,"ntime":1700001117,"order_key":"06:miner-6","p2mr_program_hex":"5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d","share_difficulty":2,"share_id":"share-1117","share_seq":1117,"template_height":800034},{"accepted_at_ms":999987,"credit_policy":"stale-grace","job_id":"job-3134","job_issued_at_ms":999994,"miner_id":"miner-10","network_difficulty":1000006,"ntime":1700003134,"order_key":"10:miner-10","p2mr_program_hex":"3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e","share_difficulty":2,"share_id":"share-3134","share_seq":3134,"template_height":800097},{"accepted_at_ms":999996,"credit_policy":"stale-grace","job_id":"job-2107","job_issued_at_ms":999981,"miner_id":"miner-6","network_difficulty":1000016,"ntime":1700002107,"order_key":"06:miner-6","p2mr_program_hex":"3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b","share_difficulty":3,"share_id":"share-2107","share_seq":2107,"template_height":800065},{"accepted_at_ms":999992,"credit_policy":null,"job_id":"job-3452","job_issued_at_ms":999989,"miner_id":"miner-9","network_difficulty":1000001,"ntime":1700003452,"order_key":"09:miner-9","p2mr_program_hex":"7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c","share_difficulty":2,"share_id":"share-3452","share_seq":3452,"template_height":800107},{"accepted_at_ms":999984,"credit_policy":null,"job_id":"job-3733","job_issued_at_ms":999998,"miner_id":"miner-4","network_difficulty":1000010,"ntime":1700003733,"order_key":"04:miner-4","p2mr_program_hex":"9595959595959595959595959595959595959595959595959595959595959595","share_difficulty":19,"share_id":"share-3733","share_seq":3733,"template_height":800116},{"accepted_at_ms":999971,"credit_policy":"stale-grace","job_id":"job-3358","job_issued_at_ms":999970,"miner_id":"miner-3","network_difficulty":1000009,"ntime":1700003358,"order_key":"03:miner-3","p2mr_program_hex":"1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e","share_difficulty":2,"share_id":"share-3358","share_seq":3358,"template_height":800104},{"accepted_at_ms":999993,"credit_policy":null,"job_id":"job-1486","job_issued_at_ms":999981,"miner_id":"miner-1","network_difficulty":1000007,"ntime":1700001486,"order_key":"01:miner-1","p2mr_program_hex":"cececececececececececececececececececececececececececececececece","share_difficulty":19,"share_id":"share-1486","share_seq":1486,"template_height":800046},{"accepted_at_ms":999997,"credit_policy":null,"job_id":"job-2229","job_issued_at_ms":999972,"miner_id":"miner-7","network_difficulty":1000002,"ntime":1700002229,"order_key":"07:miner-7","p2mr_program_hex":"b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5","share_difficulty":7,"share_id":"share-2229","share_seq":2229,"template_height":800069},{"accepted_at_ms":999971,"credit_policy":null,"job_id":"job-978","job_issued_at_ms":999997,"miner_id":"miner-10","network_difficulty":1000009,"ntime":1700000978,"order_key":"10:miner-10","p2mr_program_hex":"d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2","share_difficulty":7,"share_id":"share-978","share_seq":978,"template_height":800030},{"accepted_at_ms":999985,"credit_policy":null,"job_id":"job-3570","job_issued_at_ms":999999,"miner_id":"miner-6","network_difficulty":1000000,"ntime":1700003570,"order_key":"06:miner-6","p2mr_program_hex":"f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2","share_difficulty":19,"share_id":"share-3570","share_seq":3570,"template_height":800111},{"accepted_at_ms":999971,"credit_policy":null,"job_id":"job-3822","job_issued_at_ms":999984,"miner_id":"miner-5","network_difficulty":1000014,"ntime":1700003822,"order_key":"05:miner-5","p2mr_program_hex":"eeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeee","share_difficulty":2,"share_id":"share-3822","share_seq":3822,"template_height":800119},{"accepted_at_ms":999987,"credit_policy":null,"job_id":"job-612","job_issued_at_ms":999974,"miner_id":"miner-7","network_difficulty":1000000,"ntime":1700000612,"order_key":"07:miner-7","p2mr_program_hex":"6464646464646464646464646464646464646464646464646464646464646464","share_difficulty":2,"share_id":"share-612","share_seq":612,"template_height":800019},{"accepted_at_ms":999979,"credit_policy":"stale-grace","job_id":"job-202","job_issued_at_ms":999997,"miner_id":"miner-4","network_difficulty":1000015,"ntime":1700000202,"order_key":"04:miner-4","p2mr_program_hex":"cacacacacacacacacacacacacacacacacacacacacacacacacacacacacacacaca","share_difficulty":101,"share_id":"share-202","share_seq":202,"template_height":800006},{"accepted_at_ms":999999,"credit_policy":"stale-grace","job_id":"job-456","job_issued_at_ms":999985,"miner_id":"miner-5","network_difficulty":1000014,"ntime":1700000456,"order_key":"05:miner-5","p2mr_program_hex":"c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8","share_difficulty":7,"share_id":"share-456","share_seq":456,"template_height":800014},{"accepted_at_ms":999977,"credit_policy":null,"job_id":"job-3286","job_issued_at_ms":999979,"miner_id":"miner-8","network_difficulty":1000005,"ntime":1700003286,"order_key":"08:miner-8","p2mr_program_hex":"d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6","share_difficulty":19,"share_id":"share-3286","share_seq":3286,"template_height":800102},{"accepted_at_ms":999979,"credit_policy":null,"job_id":"job-3834","job_issued_at_ms":999977,"miner_id":"miner-6","network_difficulty":1000009,"ntime":1700003834,"order_key":"06:miner-6","p2mr_program_hex":"fafafafafafafafafafafafafafafafafafafafafafafafafafafafafafafafa","share_difficulty":2,"share_id":"share-3834","share_seq":3834,"template_height":800119},{"accepted_at_ms":999970,"credit_policy":null,"job_id":"job-3364","job_issued_at_ms":999982,"miner_id":"miner-9","network_difficulty":1000015,"ntime":1700003364,"order_key":"09:miner-9","p2mr_program_hex":"2424242424242424242424242424242424242424242424242424242424242424","share_difficulty":101,"share_id":"share-3364","share_seq":3364,"template_height":800105},{"accepted_at_ms":1000256,"credit_policy":null,"job_id":"job-2547","job_issued_at_ms":999983,"miner_id":"miner-6","network_difficulty":1000014,"ntime":1700002547,"order_key":"06:miner-6","p2mr_program_hex":"f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3","share_difficulty":2,"share_id":"share-2547","share_seq":2547,"template_height":800079},{"accepted_at_ms":999997,"credit_policy":null,"job_id":"job-2599","job_issued_at_ms":999981,"miner_id":"miner-3","network_difficulty":1000015,"ntime":1700002599,"order_key":"03:miner-3","p2mr_program_hex":"2727272727272727272727272727272727272727272727272727272727272727","share_difficulty":101,"share_id":"share-2599","share_seq":2599,"template_height":800081},{"accepted_at_ms":999989,"credit_policy":"stale-grace","job_id":"job-802","job_issued_at_ms":999987,"miner_id":"miner-10","network_difficulty":1000003,"ntime":1700000802,"order_key":"10:miner-10","p2mr_program_hex":"2222222222222222222222222222222222222222222222222222222222222222","share_difficulty":101,"share_id":"share-802","share_seq":802,"template_height":800025},{"accepted_at_ms":999979,"credit_policy":null,"job_id":"job-447","job_issued_at_ms":999979,"miner_id":"miner-7","network_difficulty":1000005,"ntime":1700000447,"order_key":"07:miner-7","p2mr_program_hex":"bfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbf","share_difficulty":1000,"share_id":"share-447","share_seq":447,"template_height":800013},{"accepted_at_ms":999974,"credit_policy":null,"job_id":"job-416","job_issued_at_ms":1000000,"miner_id":"miner-9","network_difficulty":1000008,"ntime":1700000416,"order_key":"09:miner-9","p2mr_program_hex":"a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0","share_difficulty":101,"share_id":"share-416","share_seq":416,"template_height":800013},{"accepted_at_ms":1000146,"credit_policy":null,"job_id":"job-2394","job_issued_at_ms":999981,"miner_id":"miner-7","network_difficulty":1000014,"ntime":1700002394,"order_key":"07:miner-7","p2mr_program_hex":"5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a","share_difficulty":3,"share_id":"share-2394","share_seq":2394,"template_height":800074},{"accepted_at_ms":999998,"credit_policy":null,"job_id":"job-1360","job_issued_at_ms":999981,"miner_id":"miner-7","network_difficulty":1000000,"ntime":1700001360,"order_key":"07:miner-7","p2mr_program_hex":"5050505050505050505050505050505050505050505050505050505050505050","share_difficulty":19,"share_id":"share-1360","share_seq":1360,"template_height":800042},{"accepted_at_ms":999999,"credit_policy":null,"job_id":"job-3968","job_issued_at_ms":1000000,"miner_id":"miner-8","network_difficulty":1000007,"ntime":1700003968,"order_key":"08:miner-8","p2mr_program_hex":"8080808080808080808080808080808080808080808080808080808080808080","share_difficulty":19,"share_id":"share-3968","share_seq":3968,"template_height":800124},{"accepted_at_ms":999999,"credit_policy":null,"job_id":"job-1642","job_issued_at_ms":999998,"miner_id":"miner-3","network_difficulty":1000010,"ntime":1700001642,"order_key":"03:miner-3","p2mr_program_hex":"6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a","share_difficulty":19,"share_id":"share-1642","share_seq":1642,"template_height":800051},{"accepted_at_ms":999976,"credit_policy":null,"job_id":"job-825","job_issued_at_ms":999992,"miner_id":"miner-0","network_difficulty":1000009,"ntime":1700000825,"order_key":"00:miner-0","p2mr_program_hex":"3939393939393939393939393939393939393939393939393939393939393939","share_difficulty":3,"share_id":"share-825","share_seq":825,"template_height":800025},{"accepted_at_ms":1000000,"credit_policy":null,"job_id":"job-2695","job_issued_at_ms":999980,"miner_id":"miner-0","network_difficulty":1000009,"ntime":1700002695,"order_key":"00:miner-0","p2mr_program_hex":"8787878787878787878787878787878787878787878787878787878787878787","share_difficulty":1000,"share_id":"share-2695","share_seq":2695,"template_height":800084},{"accepted_at_ms":999990,"credit_policy":null,"job_id":"job-3102","job_issued_at_ms":999983,"miner_id":"miner-0","network_difficulty":1000008,"ntime":1700003102,"order_key":"00:miner-0","p2mr_program_hex":"1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e","share_difficulty":2,"share_id":"share-3102","share_seq":3102,"template_height":800096},{"accepted_at_ms":999988,"credit_policy":"stale-grace","job_id":"job-865","job_issued_at_ms":999999,"miner_id":"miner-7","network_difficulty":1000015,"ntime":1700000865,"order_key":"07:miner-7","p2mr_program_hex":"6161616161616161616161616161616161616161616161616161616161616161","share_difficulty":1,"share_id":"share-865","share_seq":865,"template_height":800027},{"accepted_at_ms":999996,"credit_policy":null,"job_id":"job-2069","job_issued_at_ms":999995,"miner_id":"miner-1","network_difficulty":1000012,"ntime":1700002069,"order_key":"01:miner-1","p2mr_program_hex":"1515151515151515151515151515151515151515151515151515151515151515","share_difficulty":2,"share_id":"share-2069","share_seq":2069,"template_height":800064},{"accepted_at_ms":999980,"credit_policy":null,"job_id":"job-350","job_issued_at_ms":1000000,"miner_id":"miner-9","network_difficulty":1000010,"ntime":1700000350,"order_key":"09:miner-9","p2mr_program_hex":"5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e","share_difficulty":1,"share_id":"share-350","share_seq":350,"template_height":800010},{"accepted_at_ms":999983,"credit_policy":null,"job_id":"job-3794","job_issued_at_ms":999999,"miner_id":"miner-10","network_difficulty":1000003,"ntime":1700003794,"order_key":"10:miner-10","p2mr_program_hex":"d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2","share_difficulty":19,"share_id":"share-3794","share_seq":3794,"template_height":800118},{"accepted_at_ms":999986,"credit_policy":null,"job_id":"job-3302","job_issued_at_ms":999970,"miner_id":"miner-2","network_difficulty":1000004,"ntime":1700003302,"order_key":"02:miner-2","p2mr_program_hex":"e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6","share_difficulty":3,"share_id":"share-3302","share_seq":3302,"template_height":800103},{"accepted_at_ms":1000423,"credit_policy":null,"job_id":"job-2537","job_issued_at_ms":999998,"miner_id":"miner-7","network_difficulty":1000004,"ntime":1700002537,"order_key":"07:miner-7","p2mr_program_hex":"e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9","share_difficulty":1,"share_id":"share-2537","share_seq":2537,"template_height":800079},{"accepted_at_ms":999986,"credit_policy":null,"job_id":"job-2988","job_issued_at_ms":1000147,"miner_id":"miner-7","network_difficulty":1000013,"ntime":1700002988,"order_key":"07:miner-7","p2mr_program_hex":"acacacacacacacacacacacacacacacacacacacacacacacacacacacacacacacac","share_difficulty":19,"share_id":"share-2988","share_seq":2988,"template_height":800093},{"accepted_at_ms":999979,"credit_policy":"stale-grace","job_id":"job-2834","job_issued_at_ms":999998,"miner_id":"miner-7","network_difficulty":1000012,"ntime":1700002834,"order_key":"07:miner-7","p2mr_program_hex":"1212121212121212121212121212121212121212121212121212121212121212","share_difficulty":2,"share_id":"share-2834","share_seq":2834,"template_height":800088},{"accepted_at_ms":999995,"credit_policy":null,"job_id":"job-2284","job_issued_at_ms":999984,"miner_id":"miner-7","network_difficulty":1000006,"ntime":1700002284,"order_key":"07:miner-7","p2mr_program_hex":"ecececececececececececececececececececececececececececececececec","share_difficulty":19,"share_id":"share-2284","share_seq":2284,"template_height":800071},{"accepted_at_ms":999983,"credit_policy":null,"job_id":"job-2149","job_issued_at_ms":999972,"miner_id":"miner-4","network_difficulty":1000007,"ntime":1700002149,"order_key":"04:miner-4","p2mr_program_hex":"6565656565656565656565656565656565656565656565656565656565656565","share_difficulty":1,"share_id":"share-2149","share_seq":2149,"template_height":800067},{"accepted_at_ms":999978,"credit_policy":"stale-grace","job_id":"job-2552","job_issued_at_ms":999970,"miner_id":"miner-0","network_difficulty":1000002,"ntime":1700002552,"order_key":"00:miner-0","p2mr_program_hex":"f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8","share_difficulty":1000,"share_id":"share-2552","share_seq":2552,"template_height":800079},{"accepted_at_ms":1000000,"credit_policy":null,"job_id":"job-700","job_issued_at_ms":999976,"miner_id":"miner-7","network_difficulty":1000003,"ntime":1700000700,"order_key":"07:miner-7","p2mr_program_hex":"bcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbc","share_difficulty":2,"share_id":"share-700","share_seq":700,"template_height":800021},{"accepted_at_ms":999997,"credit_policy":null,"job_id":"job-3040","job_issued_at_ms":999988,"miner_id":"miner-4","network_difficulty":1000014,"ntime":1700003040,"order_key":"04:miner-4","p2mr_program_hex":"e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0","share_difficulty":2,"share_id":"share-3040","share_seq":3040,"template_height":800095},{"accepted_at_ms":1000000,"credit_policy":null,"job_id":"job-3081","job_issued_at_ms":999989,"miner_id":"miner-1","network_difficulty":1000004,"ntime":1700003081,"order_key":"01:miner-1","p2mr_program_hex":"0909090909090909090909090909090909090909090909090909090909090909","share_difficulty":19,"share_id":"share-3081","share_seq":3081,"template_height":800096},{"accepted_at_ms":999988,"credit_policy":null,"job_id":"job-3115","job_issued_at_ms":999976,"miner_id":"miner-2","network_difficulty":1000004,"ntime":1700003115,"order_key":"02:miner-2","p2mr_program_hex":"2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b","share_difficulty":2,"share_id":"share-3115","share_seq":3115,"template_height":800097},{"accepted_at_ms":999994,"credit_policy":"stale-grace","job_id":"job-2281","job_issued_at_ms":1000000,"miner_id":"miner-4","network_difficulty":1000003,"ntime":1700002281,"order_key":"04:miner-4","p2mr_program_hex":"e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9","share_difficulty":1000,"share_id":"share-2281","share_seq":2281,"template_height":800071},{"accepted_at_ms":999991,"credit_policy":null,"job_id":"job-1551","job_issued_at_ms":999993,"miner_id":"miner-0","network_difficulty":1000004,"ntime":1700001551,"order_key":"00:miner-0","p2mr_program_hex":"0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f","share_difficulty":3,"share_id":"share-1551","share_seq":1551,"template_height":800048},{"accepted_at_ms":999974,"credit_policy":null,"job_id":"job-2032","job_issued_at_ms":999995,"miner_id":"miner-8","network_difficulty":1000009,"ntime":1700002032,"order_key":"08:miner-8","p2mr_program_hex":"f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0","share_difficulty":7,"share_id":"share-2032","share_seq":2032,"template_height":800063},{"accepted_at_ms":999970,"credit_policy":null,"job_id":"job-2261","job_issued_at_ms":999974,"miner_id":"miner-6","network_difficulty":1000000,"ntime":1700002261,"order_key":"06:miner-6","p2mr_program_hex":"d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5","share_difficulty":3,"share_id":"share-2261","share_seq":2261,"template_height":800070},{"accepted_at_ms":999979,"credit_policy":null,"job_id":"job-3136","job_issued_at_ms":999992,"miner_id":"miner-1","network_difficulty":1000008,"ntime":1700003136,"order_key":"01:miner-1","p2mr_program_hex":"4040404040404040404040404040404040404040404040404040404040404040","share_difficulty":1,"share_id":"share-3136","share_seq":3136,"template_height":800098},{"accepted_at_ms":999987,"credit_policy":null,"job_id":"job-1717","job_issued_at_ms":999992,"miner_id":"miner-1","network_difficulty":1000000,"ntime":1700001717,"order_key":"01:miner-1","p2mr_program_hex":"b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5","share_difficulty":1,"share_id":"share-1717","share_seq":1717,"template_height":800053},{"accepted_at_ms":1000000,"credit_policy":null,"job_id":"job-57","job_issued_at_ms":999989,"miner_id":"miner-2","network_difficulty":1000006,"ntime":1700000057,"order_key":"02:miner-2","p2mr_program_hex":"3939393939393939393939393939393939393939393939393939393939393939","share_difficulty":101,"share_id":"share-57","share_seq":57,"template_height":800001},{"accepted_at_ms":999982,"credit_policy":null,"job_id":"job-1190","job_issued_at_ms":999986,"miner_id":"miner-2","network_difficulty":1000000,"ntime":1700001190,"order_key":"02:miner-2","p2mr_program_hex":"a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6","share_difficulty":7,"share_id":"share-1190","share_seq":1190,"template_height":800037},{"accepted_at_ms":999995,"credit_policy":null,"job_id":"job-3877","job_issued_at_ms":999995,"miner_id":"miner-5","network_difficulty":1000001,"ntime":1700003877,"order_key":"05:miner-5","p2mr_program_hex":"2525252525252525252525252525252525252525252525252525252525252525","share_difficulty":3,"share_id":"share-3877","share_seq":3877,"template_height":800121},{"accepted_at_ms":999970,"credit_policy":null,"job_id":"job-3160","job_issued_at_ms":999998,"miner_id":"miner-3","network_difficulty":1000015,"ntime":1700003160,"order_key":"03:miner-3","p2mr_program_hex":"5858585858585858585858585858585858585858585858585858585858585858","share_difficulty":19,"share_id":"share-3160","share_seq":3160,"template_height":800098},{"accepted_at_ms":999972,"credit_policy":null,"job_id":"job-2934","job_issued_at_ms":999993,"miner_id":"miner-8","network_difficulty":1000010,"ntime":1700002934,"order_key":"08:miner-8","p2mr_program_hex":"7676767676767676767676767676767676767676767676767676767676767676","share_difficulty":19,"share_id":"share-2934","share_seq":2934,"template_height":800091},{"accepted_at_ms":999974,"credit_policy":"stale-grace","job_id":"job-3987","job_issued_at_ms":999988,"miner_id":"miner-5","network_difficulty":1000009,"ntime":1700003987,"order_key":"05:miner-5","p2mr_program_hex":"9393939393939393939393939393939393939393939393939393939393939393","share_difficulty":3,"share_id":"share-3987","share_seq":3987,"template_height":800124},{"accepted_at_ms":999994,"credit_policy":"stale-grace","job_id":"job-2678","job_issued_at_ms":999982,"miner_id":"miner-5","network_difficulty":1000009,"ntime":1700002678,"order_key":"05:miner-5","p2mr_program_hex":"7676767676767676767676767676767676767676767676767676767676767676","share_difficulty":7,"share_id":"share-2678","share_seq":2678,"template_height":800083},{"accepted_at_ms":999997,"credit_policy":null,"job_id":"job-3808","job_issued_at_ms":999995,"miner_id":"miner-2","network_difficulty":1000000,"ntime":1700003808,"order_key":"02:miner-2","p2mr_program_hex":"e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0","share_difficulty":101,"share_id":"share-3808","share_seq":3808,"template_height":800119},{"accepted_at_ms":999992,"credit_policy":null,"job_id":"job-2721","job_issued_at_ms":999996,"miner_id":"miner-4","network_difficulty":1000001,"ntime":1700002721,"order_key":"04:miner-4","p2mr_program_hex":"a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1","share_difficulty":101,"share_id":"share-2721","share_seq":2721,"template_height":800085},{"accepted_at_ms":999986,"credit_policy":null,"job_id":"job-3916","job_issued_at_ms":999989,"miner_id":"miner-0","network_difficulty":1000006,"ntime":1700003916,"order_key":"00:miner-0","p2mr_program_hex":"4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c","share_difficulty":101,"share_id":"share-3916","share_seq":3916,"template_height":800122},{"accepted_at_ms":999979,"credit_policy":null,"job_id":"job-4","job_issued_at_ms":999978,"miner_id":"miner-4","network_difficulty":1000004,"ntime":1700000004,"order_key":"04:miner-4","p2mr_program_hex":"0404040404040404040404040404040404040404040404040404040404040404","share_difficulty":7,"share_id":"share-4","share_seq":4,"template_height":800000},{"accepted_at_ms":999971,"credit_policy":null,"job_id":"job-1630","job_issued_at_ms":999976,"miner_id":"miner-2","network_difficulty":1000015,"ntime":1700001630,"order_key":"02:miner-2","p2mr_program_hex":"5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e","share_difficulty":19,"share_id":"share-1630","share_seq":1630,"template_height":800050},{"accepted_at_ms":999995,"credit_policy":null,"job_id":"job-1883","job_issued_at_ms":999977,"miner_id":"miner-2","network_difficulty":1000013,"ntime":1700001883,"order_key":"02:miner-2","p2mr_program_hex":"5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b","share_difficulty":7,"share_id":"share-1883","share_seq":1883,"template_height":800058},{"accepted_at_ms":999971,"credit_policy":null,"job_id":"job-1614","job_issued_at_ms":999994,"miner_id":"miner-8","network_difficulty":1000016,"ntime":1700001614,"order_key":"08:miner-8","p2mr_program_hex":"4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e","share_difficulty":19,"share_id":"share-1614","share_seq":1614,"template_height":800050},{"accepted_at_ms":999970,"credit_policy":null,"job_id":"job-1216","job_issued_at_ms":999999,"miner_id":"miner-6","network_difficulty":1000009,"ntime":1700001216,"order_key":"06:miner-6","p2mr_program_hex":"c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0","share_difficulty":1,"share_id":"share-1216","share_seq":1216,"template_height":800038},{"accepted_at_ms":999993,"credit_policy":"stale-grace","job_id":"job-1641","job_issued_at_ms":999979,"miner_id":"miner-2","network_difficulty":1000009,"ntime":1700001641,"order_key":"02:miner-2","p2mr_program_hex":"6969696969696969696969696969696969696969696969696969696969696969","share_difficulty":19,"share_id":"share-1641","share_seq":1641,"template_height":800051},{"accepted_at_ms":999975,"credit_policy":null,"job_id":"job-827","job_issued_at_ms":999997,"miner_id":"miner-2","network_difficulty":1000011,"ntime":1700000827,"order_key":"02:miner-2","p2mr_program_hex":"3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b","share_difficulty":19,"share_id":"share-827","share_seq":827,"template_height":800025},{"accepted_at_ms":999991,"credit_policy":null,"job_id":"job-3535","job_issued_at_ms":999997,"miner_id":"miner-4","network_difficulty":1000016,"ntime":1700003535,"order_key":"04:miner-4","p2mr_program_hex":"cfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcf","share_difficulty":19,"share_id":"share-3535","share_seq":3535,"template_height":800110},{"accepted_at_ms":999976,"credit_policy":null,"job_id":"job-2385","job_issued_at_ms":999971,"miner_id":"miner-9","network_difficulty":1000005,"ntime":1700002385,"order_key":"09:miner-9","p2mr_program_hex":"5151515151515151515151515151515151515151515151515151515151515151","share_difficulty":1000,"share_id":"share-2385","share_seq":2385,"template_height":800074},{"accepted_at_ms":999983,"credit_policy":null,"job_id":"job-1030","job_issued_at_ms":999983,"miner_id":"miner-7","network_difficulty":1000010,"ntime":1700001030,"order_key":"07:miner-7","p2mr_program_hex":"0606060606060606060606060606060606060606060606060606060606060606","share_difficulty":1,"share_id":"share-1030","share_seq":1030,"template_height":800032},{"accepted_at_ms":999985,"credit_policy":null,"job_id":"job-1659","job_issued_at_ms":999983,"miner_id":"miner-9","network_difficulty":1000010,"ntime":1700001659,"order_key":"09:miner-9","p2mr_program_hex":"7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b","share_difficulty":1000,"share_id":"share-1659","share_seq":1659,"template_height":800051},{"accepted_at_ms":999988,"credit_policy":"stale-grace","job_id":"job-1054","job_issued_at_ms":999971,"miner_id":"miner-9","network_difficulty":1000000,"ntime":1700001054,"order_key":"09:miner-9","p2mr_program_hex":"1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e","share_difficulty":1,"share_id":"share-1054","share_seq":1054,"template_height":800032},{"accepted_at_ms":999971,"credit_policy":null,"job_id":"job-550","job_issued_at_ms":999973,"miner_id":"miner-0","network_difficulty":1000006,"ntime":1700000550,"order_key":"00:miner-0","p2mr_program_hex":"2626262626262626262626262626262626262626262626262626262626262626","share_difficulty":3,"share_id":"share-550","share_seq":550,"template_height":800017},{"accepted_at_ms":999981,"credit_policy":null,"job_id":"job-1353","job_issued_at_ms":999988,"miner_id":"miner-0","network_difficulty":1000010,"ntime":1700001353,"order_key":"00:miner-0","p2mr_program_hex":"4949494949494949494949494949494949494949494949494949494949494949","share_difficulty":1000,"share_id":"share-1353","share_seq":1353,"template_height":800042},{"accepted_at_ms":999970,"credit_policy":null,"job_id":"job-2038","job_issued_at_ms":999988,"miner_id":"miner-3","network_difficulty":1000015,"ntime":1700002038,"order_key":"03:miner-3","p2mr_program_hex":"f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6","share_difficulty":1,"share_id":"share-2038","share_seq":2038,"template_height":800063},{"accepted_at_ms":999994,"credit_policy":null,"job_id":"job-777","job_issued_at_ms":999971,"miner_id":"miner-7","network_difficulty":1000012,"ntime":1700000777,"order_key":"07:miner-7","p2mr_program_hex":"0909090909090909090909090909090909090909090909090909090909090909","share_difficulty":3,"share_id":"share-777","share_seq":777,"template_height":800024},{"accepted_at_ms":1000180,"credit_policy":"stale-grace","job_id":"job-1212","job_issued_at_ms":999979,"miner_id":"miner-2","network_difficulty":1000005,"ntime":1700001212,"order_key":"02:miner-2","p2mr_program_hex":"bcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbc","share_difficulty":1,"share_id":"share-1212","share_seq":1212,"template_height":800037},{"accepted_at_ms":999986,"credit_policy":null,"job_id":"job-16","job_issued_at_ms":999984,"miner_id":"miner-5","network_difficulty":1000016,"ntime":1700000016,"order_key":"05:miner-5","p2mr_program_hex":"1010101010101010101010101010101010101010101010101010101010101010","share_difficulty":1000,"share_id":"share-16","share_seq":16,"template_height":800000},{"accepted_at_ms":1000391,"credit_policy":null,"job_id":"job-3340","job_issued_at_ms":999976,"miner_id":"miner-7","network_difficulty":1000008,"ntime":1700003340,"order_key":"07:miner-7","p2mr_program_hex":"0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c","share_difficulty":1,"share_id":"share-3340","share_seq":3340,"template_height":800104},{"accepted_at_ms":999985,"credit_policy":null,"job_id":"job-3780","job_issued_at_ms":999991,"miner_id":"miner-7","network_difficulty":1000006,"ntime":1700003780,"order_key":"07:miner-7","p2mr_program_hex":"c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4","share_difficulty":1000,"share_id":"share-3780","share_seq":3780,"template_height":800118},{"accepted_at_ms":999991,"credit_policy":null,"job_id":"job-918","job_issued_at_ms":999995,"miner_id":"miner-5","network_difficulty":1000000,"ntime":1700000918,"order_key":"05:miner-5","p2mr_program_hex":"9696969696969696969696969696969696969696969696969696969696969696","share_difficulty":2,"share_id":"share-918","share_seq":918,"template_height":800028},{"accepted_at_ms":999971,"credit_policy":"stale-grace","job_id":"job-2311","job_issued_at_ms":1000076,"miner_id":"miner-1","network_difficulty":1000016,"ntime":1700002311,"order_key":"01:miner-1","p2mr_program_hex":"0707070707070707070707070707070707070707070707070707070707070707","share_difficulty":1000,"share_id":"share-2311","share_seq":2311,"template_height":800072},{"accepted_at_ms":999985,"credit_policy":null,"job_id":"job-3355","job_issued_at_ms":999979,"miner_id":"miner-0","network_difficulty":1000006,"ntime":1700003355,"order_key":"00:miner-0","p2mr_program_hex":"1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b","share_difficulty":19,"share_id":"share-3355","share_seq":3355,"template_height":800104},{"accepted_at_ms":1000000,"credit_policy":"stale-grace","job_id":"job-1335","job_issued_at_ms":999987,"miner_id":"miner-4","network_difficulty":1000009,"ntime":1700001335,"order_key":"04:miner-4","p2mr_program_hex":"3737373737373737373737373737373737373737373737373737373737373737","share_difficulty":1000,"share_id":"share-1335","share_seq":1335,"template_height":800041},{"accepted_at_ms":999971,"credit_policy":null,"job_id":"job-3609","job_issued_at_ms":999981,"miner_id":"miner-1","network_difficulty":1000005,"ntime":1700003609,"order_key":"01:miner-1","p2mr_program_hex":"1919191919191919191919191919191919191919191919191919191919191919","share_difficulty":1,"share_id":"share-3609","share_seq":3609,"template_height":800112},{"accepted_at_ms":999985,"credit_policy":"stale-grace","job_id":"job-1324","job_issued_at_ms":999994,"miner_id":"miner-4","network_difficulty":1000015,"ntime":1700001324,"order_key":"04:miner-4","p2mr_program_hex":"2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c","share_difficulty":1000,"share_id":"share-1324","share_seq":1324,"template_height":800041},{"accepted_at_ms":1000124,"credit_policy":null,"job_id":"job-1294","job_issued_at_ms":1000286,"miner_id":"miner-7","network_difficulty":1000002,"ntime":1700001294,"order_key":"07:miner-7","p2mr_program_hex":"0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e","share_difficulty":19,"share_id":"share-1294","share_seq":1294,"template_height":800040},{"accepted_at_ms":999983,"credit_policy":null,"job_id":"job-3729","job_issued_at_ms":999983,"miner_id":"miner-0","network_difficulty":1000006,"ntime":1700003729,"order_key":"00:miner-0","p2mr_program_hex":"9191919191919191919191919191919191919191919191919191919191919191","share_difficulty":19,"share_id":"share-3729","share_seq":3729,"template_height":800116},{"accepted_at_ms":999985,"credit_policy":null,"job_id":"job-1970","job_issued_at_ms":999973,"miner_id":"miner-1","network_difficulty":1000015,"ntime":1700001970,"order_key":"01:miner-1","p2mr_program_hex":"b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2","share_difficulty":101,"share_id":"share-1970","share_seq":1970,"template_height":800061},{"accepted_at_ms":1000357,"credit_policy":null,"job_id":"job-791","job_issued_at_ms":1000447,"miner_id":"miner-10","network_difficulty":1000009,"ntime":1700000791,"order_key":"10:miner-10","p2mr_program_hex":"1717171717171717171717171717171717171717171717171717171717171717","share_difficulty":1000,"share_id":"share-791","share_seq":791,"template_height":800024},{"accepted_at_ms":999980,"credit_policy":"stale-grace","job_id":"job-2020","job_issued_at_ms":999996,"miner_id":"miner-7","network_difficulty":1000014,"ntime":1700002020,"order_key":"07:miner-7","p2mr_program_hex":"e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4","share_difficulty":3,"share_id":"share-2020","share_seq":2020,"template_height":800063},{"accepted_at_ms":1000394,"credit_policy":null,"job_id":"job-2376","job_issued_at_ms":1000039,"miner_id":"miner-0","network_difficulty":1000013,"ntime":1700002376,"order_key":"00:miner-0","p2mr_program_hex":"4848484848484848484848484848484848484848484848484848484848484848","share_difficulty":3,"share_id":"share-2376","share_seq":2376,"template_height":800074},{"accepted_at_ms":999989,"credit_policy":null,"job_id":"job-307","job_issued_at_ms":999983,"miner_id":"miner-10","network_difficulty":1000001,"ntime":1700000307,"order_key":"10:miner-10","p2mr_program_hex":"3333333333333333333333333333333333333333333333333333333333333333","share_difficulty":1,"share_id":"share-307","share_seq":307,"template_height":800009},{"accepted_at_ms":999983,"credit_policy":null,"job_id":"job-2444","job_issued_at_ms":999999,"miner_id":"miner-2","network_difficulty":1000013,"ntime":1700002444,"order_key":"02:miner-2","p2mr_program_hex":"8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c","share_difficulty":7,"share_id":"share-2444","share_seq":2444,"template_height":800076},{"accepted_at_ms":999980,"credit_policy":null,"job_id":"job-2796","job_issued_at_ms":999987,"miner_id":"miner-2","network_difficulty":1000008,"ntime":1700002796,"order_key":"02:miner-2","p2mr_program_hex":"ecececececececececececececececececececececececececececececececec","share_difficulty":3,"share_id":"share-2796","share_seq":2796,"template_height":800087},{"accepted_at_ms":999975,"credit_policy":"stale-grace","job_id":"job-2677","job_issued_at_ms":999981,"miner_id":"miner-4","network_difficulty":1000008,"ntime":1700002677,"order_key":"04:miner-4","p2mr_program_hex":"7575757575757575757575757575757575757575757575757575757575757575","share_difficulty":101,"share_id":"share-2677","share_seq":2677,"template_height":800083},{"accepted_at_ms":999980,"credit_policy":null,"job_id":"job-1385","job_issued_at_ms":999997,"miner_id":"miner-10","network_difficulty":1000008,"ntime":1700001385,"order_key":"10:miner-10","p2mr_program_hex":"6969696969696969696969696969696969696969696969696969696969696969","share_difficulty":1,"share_id":"share-1385","share_seq":1385,"template_height":800043},{"accepted_at_ms":999982,"credit_policy":null,"job_id":"job-3184","job_issued_at_ms":999986,"miner_id":"miner-5","network_difficulty":1000005,"ntime":1700003184,"order_key":"05:miner-5","p2mr_program_hex":"7070707070707070707070707070707070707070707070707070707070707070","share_difficulty":19,"share_id":"share-3184","share_seq":3184,"template_height":800099},{"accepted_at_ms":999978,"credit_policy":"stale-grace","job_id":"job-225","job_issued_at_ms":999974,"miner_id":"miner-5","network_difficulty":1000004,"ntime":1700000225,"order_key":"05:miner-5","p2mr_program_hex":"e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1","share_difficulty":3,"share_id":"share-225","share_seq":225,"template_height":800007},{"accepted_at_ms":999982,"credit_policy":null,"job_id":"job-2292","job_issued_at_ms":999989,"miner_id":"miner-4","network_difficulty":1000014,"ntime":1700002292,"order_key":"04:miner-4","p2mr_program_hex":"f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4","share_difficulty":19,"share_id":"share-2292","share_seq":2292,"template_height":800071},{"accepted_at_ms":999994,"credit_policy":null,"job_id":"job-3992","job_issued_at_ms":999982,"miner_id":"miner-10","network_difficulty":1000014,"ntime":1700003992,"order_key":"10:miner-10","p2mr_program_hex":"9898989898989898989898989898989898989898989898989898989898989898","share_difficulty":2,"share_id":"share-3992","share_seq":3992,"template_height":800124},{"accepted_at_ms":999983,"credit_policy":null,"job_id":"job-1398","job_issued_at_ms":999981,"miner_id":"miner-1","network_difficulty":1000004,"ntime":1700001398,"order_key":"01:miner-1","p2mr_program_hex":"7676767676767676767676767676767676767676767676767676767676767676","share_difficulty":7,"share_id":"share-1398","share_seq":1398,"template_height":800043},{"accepted_at_ms":1000000,"credit_policy":null,"job_id":"job-788","job_issued_at_ms":999986,"miner_id":"miner-7","network_difficulty":1000006,"ntime":1700000788,"order_key":"07:miner-7","p2mr_program_hex":"1414141414141414141414141414141414141414141414141414141414141414","share_difficulty":7,"share_id":"share-788","share_seq":788,"template_height":800024},{"accepted_at_ms":999970,"credit_policy":null,"job_id":"job-1527","job_issued_at_ms":999994,"miner_id":"miner-9","network_difficulty":1000014,"ntime":1700001527,"order_key":"09:miner-9","p2mr_program_hex":"f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7","share_difficulty":1,"share_id":"share-1527","share_seq":1527,"template_height":800047},{"accepted_at_ms":999994,"credit_policy":null,"job_id":"job-3664","job_issued_at_ms":999989,"miner_id":"miner-1","network_difficulty":1000009,"ntime":1700003664,"order_key":"01:miner-1","p2mr_program_hex":"5050505050505050505050505050505050505050505050505050505050505050","share_difficulty":7,"share_id":"share-3664","share_seq":3664,"template_height":800114},{"accepted_at_ms":999973,"credit_policy":null,"job_id":"job-2486","job_issued_at_ms":999971,"miner_id":"miner-0","network_difficulty":1000004,"ntime":1700002486,"order_key":"00:miner-0","p2mr_program_hex":"b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6","share_difficulty":19,"share_id":"share-2486","share_seq":2486,"template_height":800077},{"accepted_at_ms":999998,"credit_policy":"stale-grace","job_id":"job-390","job_issued_at_ms":999996,"miner_id":"miner-5","network_difficulty":1000016,"ntime":1700000390,"order_key":"05:miner-5","p2mr_program_hex":"8686868686868686868686868686868686868686868686868686868686868686","share_difficulty":7,"share_id":"share-390","share_seq":390,"template_height":800012},{"accepted_at_ms":999992,"credit_policy":null,"job_id":"job-2024","job_issued_at_ms":999993,"miner_id":"miner-0","network_difficulty":1000001,"ntime":1700002024,"order_key":"00:miner-0","p2mr_program_hex":"e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8","share_difficulty":2,"share_id":"share-2024","share_seq":2024,"template_height":800063},{"accepted_at_ms":999977,"credit_policy":null,"job_id":"job-1048","job_issued_at_ms":999993,"miner_id":"miner-3","network_difficulty":1000011,"ntime":1700001048,"order_key":"03:miner-3","p2mr_program_hex":"1818181818181818181818181818181818181818181818181818181818181818","share_difficulty":19,"share_id":"share-1048","share_seq":1048,"template_height":800032},{"accepted_at_ms":999999,"credit_policy":null,"job_id":"job-3222","job_issued_at_ms":999988,"miner_id":"miner-10","network_difficulty":1000009,"ntime":1700003222,"order_key":"10:miner-10","p2mr_program_hex":"9696969696969696969696969696969696969696969696969696969696969696","share_difficulty":1000,"share_id":"share-3222","share_seq":3222,"template_height":800100},{"accepted_at_ms":999993,"credit_policy":"stale-grace","job_id":"job-3320","job_issued_at_ms":999979,"miner_id":"miner-9","network_difficulty":1000005,"ntime":1700003320,"order_key":"09:miner-9","p2mr_program_hex":"f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8","share_difficulty":1,"share_id":"share-3320","share_seq":3320,"template_height":800103},{"accepted_at_ms":999992,"credit_policy":null,"job_id":"job-2538","job_issued_at_ms":999989,"miner_id":"miner-8","network_difficulty":1000005,"ntime":1700002538,"order_key":"08:miner-8","p2mr_program_hex":"eaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaea","share_difficulty":1000,"share_id":"share-2538","share_seq":2538,"template_height":800079},{"accepted_at_ms":999973,"credit_policy":null,"job_id":"job-1352","job_issued_at_ms":999986,"miner_id":"miner-10","network_difficulty":1000009,"ntime":1700001352,"order_key":"10:miner-10","p2mr_program_hex":"4848484848484848484848484848484848484848484848484848484848484848","share_difficulty":101,"share_id":"share-1352","share_seq":1352,"template_height":800042},{"accepted_at_ms":999975,"credit_policy":null,"job_id":"job-51","job_issued_at_ms":999982,"miner_id":"miner-7","network_difficulty":1000000,"ntime":1700000051,"order_key":"07:miner-7","p2mr_program_hex":"3333333333333333333333333333333333333333333333333333333333333333","share_difficulty":7,"share_id":"share-51","share_seq":51,"template_height":800001},{"accepted_at_ms":999991,"credit_policy":null,"job_id":"job-722","job_issued_at_ms":999972,"miner_id":"miner-7","network_difficulty":1000008,"ntime":1700000722,"order_key":"07:miner-7","p2mr_program_hex":"d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2","share_difficulty":19,"share_id":"share-722","share_seq":722,"template_height":800022},{"accepted_at_ms":999977,"credit_policy":"stale-grace","job_id":"job-794","job_issued_at_ms":999979,"miner_id":"miner-2","network_difficulty":1000012,"ntime":1700000794,"order_key":"02:miner-2","p2mr_program_hex":"1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a","share_difficulty":3,"share_id":"share-794","share_seq":794,"template_height":800024},{"accepted_at_ms":999993,"credit_policy":null,"job_id":"job-1535","job_issued_at_ms":1000000,"miner_id":"miner-6","network_difficulty":1000005,"ntime":1700001535,"order_key":"06:miner-6","p2mr_program_hex":"ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff","share_difficulty":3,"share_id":"share-1535","share_seq":1535,"template_height":800047},{"accepted_at_ms":999983,"credit_policy":"stale-grace","job_id":"job-676","job_issued_at_ms":999980,"miner_id":"miner-5","network_difficulty":1000013,"ntime":1700000676,"order_key":"05:miner-5","p2mr_program_hex":"a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4","share_difficulty":19,"share_id":"share-676","share_seq":676,"template_height":800021},{"accepted_at_ms":1000000,"credit_policy":null,"job_id":"job-1727","job_issued_at_ms":999981,"miner_id":"miner-0","network_difficulty":1000010,"ntime":1700001727,"order_key":"00:miner-0","p2mr_program_hex":"bfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbf","share_difficulty":1,"share_id":"share-1727","share_seq":1727,"template_height":800053},{"accepted_at_ms":999984,"credit_policy":"stale-grace","job_id":"job-3293","job_issued_at_ms":999988,"miner_id":"miner-4","network_difficulty":1000012,"ntime":1700003293,"order_key":"04:miner-4","p2mr_program_hex":"dddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddd","share_difficulty":1000,"share_id":"share-3293","share_seq":3293,"template_height":800102},{"accepted_at_ms":999997,"credit_policy":null,"job_id":"job-2885","job_issued_at_ms":999979,"miner_id":"miner-3","network_difficulty":1000012,"ntime":1700002885,"order_key":"03:miner-3","p2mr_program_hex":"4545454545454545454545454545454545454545454545454545454545454545","share_difficulty":1,"share_id":"share-2885","share_seq":2885,"template_height":800090},{"accepted_at_ms":999971,"credit_policy":null,"job_id":"job-2275","job_issued_at_ms":999976,"miner_id":"miner-9","network_difficulty":1000014,"ntime":1700002275,"order_key":"09:miner-9","p2mr_program_hex":"e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3","share_difficulty":101,"share_id":"share-2275","share_seq":2275,"template_height":800071},{"accepted_at_ms":999978,"credit_policy":null,"job_id":"job-3615","job_issued_at_ms":999996,"miner_id":"miner-7","network_difficulty":1000011,"ntime":1700003615,"order_key":"07:miner-7","p2mr_program_hex":"1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f","share_difficulty":7,"share_id":"share-3615","share_seq":3615,"template_height":800112},{"accepted_at_ms":999987,"credit_policy":null,"job_id":"job-2662","job_issued_at_ms":999994,"miner_id":"miner-0","network_difficulty":1000010,"ntime":1700002662,"order_key":"00:miner-0","p2mr_program_hex":"6666666666666666666666666666666666666666666666666666666666666666","share_difficulty":3,"share_id":"share-2662","share_seq":2662,"template_height":800083},{"accepted_at_ms":999972,"credit_policy":"stale-grace","job_id":"job-1016","job_issued_at_ms":999978,"miner_id":"miner-4","network_difficulty":1000013,"ntime":1700001016,"order_key":"04:miner-4","p2mr_program_hex":"f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8","share_difficulty":19,"share_id":"share-1016","share_seq":1016,"template_height":800031},{"accepted_at_ms":999988,"credit_policy":"stale-grace","job_id":"job-1567","job_issued_at_ms":999988,"miner_id":"miner-5","network_difficulty":1000003,"ntime":1700001567,"order_key":"05:miner-5","p2mr_program_hex":"1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f","share_difficulty":1000,"share_id":"share-1567","share_seq":1567,"template_height":800048},{"accepted_at_ms":999980,"credit_policy":null,"job_id":"job-2744","job_issued_at_ms":999981,"miner_id":"miner-5","network_difficulty":1000007,"ntime":1700002744,"order_key":"05:miner-5","p2mr_program_hex":"b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8","share_difficulty":101,"share_id":"share-2744","share_seq":2744,"template_height":800085},{"accepted_at_ms":999979,"credit_policy":null,"job_id":"job-144","job_issued_at_ms":999979,"miner_id":"miner-1","network_difficulty":1000008,"ntime":1700000144,"order_key":"01:miner-1","p2mr_program_hex":"9090909090909090909090909090909090909090909090909090909090909090","share_difficulty":7,"share_id":"share-144","share_seq":144,"template_height":800004},{"accepted_at_ms":1000000,"credit_policy":null,"job_id":"job-1193","job_issued_at_ms":999994,"miner_id":"miner-5","network_difficulty":1000003,"ntime":1700001193,"order_key":"05:miner-5","p2mr_program_hex":"a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9","share_difficulty":2,"share_id":"share-1193","share_seq":1193,"template_height":800037},{"accepted_at_ms":999970,"credit_policy":null,"job_id":"job-2145","job_issued_at_ms":999991,"miner_id":"miner-0","network_difficulty":1000003,"ntime":1700002145,"order_key":"00:miner-0","p2mr_program_hex":"6161616161616161616161616161616161616161616161616161616161616161","share_difficulty":7,"share_id":"share-2145","share_seq":2145,"template_height":800067},{"accepted_at_ms":999999,"credit_policy":null,"job_id":"job-3764","job_issued_at_ms":999985,"miner_id":"miner-2","network_difficulty":1000007,"ntime":1700003764,"order_key":"02:miner-2","p2mr_program_hex":"b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4","share_difficulty":3,"share_id":"share-3764","share_seq":3764,"template_height":800117},{"accepted_at_ms":999981,"credit_policy":null,"job_id":"job-3734","job_issued_at_ms":999981,"miner_id":"miner-5","network_difficulty":1000011,"ntime":1700003734,"order_key":"05:miner-5","p2mr_program_hex":"9696969696969696969696969696969696969696969696969696969696969696","share_difficulty":101,"share_id":"share-3734","share_seq":3734,"template_height":800116},{"accepted_at_ms":999989,"credit_policy":null,"job_id":"job-1713","job_issued_at_ms":999977,"miner_id":"miner-8","network_difficulty":1000013,"ntime":1700001713,"order_key":"08:miner-8","p2mr_program_hex":"b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1","share_difficulty":2,"share_id":"share-1713","share_seq":1713,"template_height":800053},{"accepted_at_ms":999976,"credit_policy":null,"job_id":"job-3176","job_issued_at_ms":999995,"miner_id":"miner-8","network_difficulty":1000014,"ntime":1700003176,"order_key":"08:miner-8","p2mr_program_hex":"6868686868686868686868686868686868686868686868686868686868686868","share_difficulty":19,"share_id":"share-3176","share_seq":3176,"template_height":800099},{"accepted_at_ms":999997,"credit_policy":null,"job_id":"job-2541","job_issued_at_ms":999997,"miner_id":"miner-0","network_difficulty":1000008,"ntime":1700002541,"order_key":"00:miner-0","p2mr_program_hex":"edededededededededededededededededededededededededededededededed","share_difficulty":2,"share_id":"share-2541","share_seq":2541,"template_height":800079},{"accepted_at_ms":999983,"credit_policy":null,"job_id":"job-2392","job_issued_at_ms":999981,"miner_id":"miner-5","network_difficulty":1000012,"ntime":1700002392,"order_key":"05:miner-5","p2mr_program_hex":"5858585858585858585858585858585858585858585858585858585858585858","share_difficulty":2,"share_id":"share-2392","share_seq":2392,"template_height":800074},{"accepted_at_ms":999974,"credit_policy":null,"job_id":"job-3448","job_issued_at_ms":999973,"miner_id":"miner-5","network_difficulty":1000014,"ntime":1700003448,"order_key":"05:miner-5","p2mr_program_hex":"7878787878787878787878787878787878787878787878787878787878787878","share_difficulty":3,"share_id":"share-3448","share_seq":3448,"template_height":800107},{"accepted_at_ms":999995,"credit_policy":null,"job_id":"job-3876","job_issued_at_ms":999990,"miner_id":"miner-4","network_difficulty":1000000,"ntime":1700003876,"order_key":"04:miner-4","p2mr_program_hex":"2424242424242424242424242424242424242424242424242424242424242424","share_difficulty":3,"share_id":"share-3876","share_seq":3876,"template_height":800121},{"accepted_at_ms":999997,"credit_policy":null,"job_id":"job-1672","job_issued_at_ms":999973,"miner_id":"miner-0","network_difficulty":1000006,"ntime":1700001672,"order_key":"00:miner-0","p2mr_program_hex":"8888888888888888888888888888888888888888888888888888888888888888","share_difficulty":7,"share_id":"share-1672","share_seq":1672,"template_height":800052},{"accepted_at_ms":999984,"credit_policy":null,"job_id":"job-1725","job_issued_at_ms":999982,"miner_id":"miner-9","network_difficulty":1000008,"ntime":1700001725,"order_key":"09:miner-9","p2mr_program_hex":"bdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbd","share_difficulty":2,"share_id":"share-1725","share_seq":1725,"template_height":800053},{"accepted_at_ms":999990,"credit_policy":null,"job_id":"job-2457","job_issued_at_ms":999973,"miner_id":"miner-4","network_difficulty":1000009,"ntime":1700002457,"order_key":"04:miner-4","p2mr_program_hex":"9999999999999999999999999999999999999999999999999999999999999999","share_difficulty":7,"share_id":"share-2457","share_seq":2457,"template_height":800076},{"accepted_at_ms":999988,"credit_policy":null,"job_id":"job-542","job_issued_at_ms":999989,"miner_id":"miner-3","network_difficulty":1000015,"ntime":1700000542,"order_key":"03:miner-3","p2mr_program_hex":"1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e","share_difficulty":7,"share_id":"share-542","share_seq":542,"template_height":800016},{"accepted_at_ms":999997,"credit_policy":null,"job_id":"job-1382","job_issued_at_ms":999978,"miner_id":"miner-7","network_difficulty":1000005,"ntime":1700001382,"order_key":"07:miner-7","p2mr_program_hex":"6666666666666666666666666666666666666666666666666666666666666666","share_difficulty":2,"share_id":"share-1382","share_seq":1382,"template_height":800043},{"accepted_at_ms":999993,"credit_policy":"stale-grace","job_id":"job-1043","job_issued_at_ms":999971,"miner_id":"miner-9","network_difficulty":1000006,"ntime":1700001043,"order_key":"09:miner-9","p2mr_program_hex":"1313131313131313131313131313131313131313131313131313131313131313","share_difficulty":19,"share_id":"share-1043","share_seq":1043,"template_height":800032},{"accepted_at_ms":999970,"credit_policy":null,"job_id":"job-1186","job_issued_at_ms":999982,"miner_id":"miner-9","network_difficulty":1000013,"ntime":1700001186,"order_key":"09:miner-9","p2mr_program_hex":"a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2","share_difficulty":1,"share_id":"share-1186","share_seq":1186,"template_height":800037},{"accepted_at_ms":1000000,"credit_policy":null,"job_id":"job-3595","job_issued_at_ms":999998,"miner_id":"miner-9","network_difficulty":1000008,"ntime":1700003595,"order_key":"09:miner-9","p2mr_program_hex":"0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b","share_difficulty":1000,"share_id":"share-3595","share_seq":3595,"template_height":800112},{"accepted_at_ms":1000000,"credit_policy":null,"job_id":"job-2062","job_issued_at_ms":999994,"miner_id":"miner-5","network_difficulty":1000005,"ntime":1700002062,"order_key":"05:miner-5","p2mr_program_hex":"0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e","share_difficulty":1,"share_id":"share-2062","share_seq":2062,"template_height":800064},{"accepted_at_ms":999976,"credit_policy":"stale-grace","job_id":"job-3721","job_issued_at_ms":999986,"miner_id":"miner-3","network_difficulty":1000015,"ntime":1700003721,"order_key":"03:miner-3","p2mr_program_hex":"8989898989898989898989898989898989898989898989898989898989898989","share_difficulty":2,"share_id":"share-3721","share_seq":3721,"template_height":800116},{"accepted_at_ms":999973,"credit_policy":null,"job_id":"job-1484","job_issued_at_ms":999979,"miner_id":"miner-10","network_difficulty":1000005,"ntime":1700001484,"order_key":"10:miner-10","p2mr_program_hex":"cccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccc","share_difficulty":101,"share_id":"share-1484","share_seq":1484,"template_height":800046},{"accepted_at_ms":999988,"credit_policy":null,"job_id":"job-2530","job_issued_at_ms":999971,"miner_id":"miner-0","network_difficulty":1000014,"ntime":1700002530,"order_key":"00:miner-0","p2mr_program_hex":"e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2","share_difficulty":1000,"share_id":"share-2530","share_seq":2530,"template_height":800079},{"accepted_at_ms":999978,"credit_policy":null,"job_id":"job-1546","job_issued_at_ms":999976,"miner_id":"miner-6","network_difficulty":1000016,"ntime":1700001546,"order_key":"06:miner-6","p2mr_program_hex":"0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a","share_difficulty":7,"share_id":"share-1546","share_seq":1546,"template_height":800048},{"accepted_at_ms":999979,"credit_policy":null,"job_id":"job-1893","job_issued_at_ms":999996,"miner_id":"miner-1","network_difficulty":1000006,"ntime":1700001893,"order_key":"01:miner-1","p2mr_program_hex":"6565656565656565656565656565656565656565656565656565656565656565","share_difficulty":1,"share_id":"share-1893","share_seq":1893,"template_height":800059},{"accepted_at_ms":999997,"credit_policy":"stale-grace","job_id":"job-3988","job_issued_at_ms":999985,"miner_id":"miner-6","network_difficulty":1000010,"ntime":1700003988,"order_key":"06:miner-6","p2mr_program_hex":"9494949494949494949494949494949494949494949494949494949494949494","share_difficulty":3,"share_id":"share-3988","share_seq":3988,"template_height":800124},{"accepted_at_ms":999997,"credit_policy":null,"job_id":"job-3637","job_issued_at_ms":999978,"miner_id":"miner-7","network_difficulty":1000016,"ntime":1700003637,"order_key":"07:miner-7","p2mr_program_hex":"3535353535353535353535353535353535353535353535353535353535353535","share_difficulty":1000,"share_id":"share-3637","share_seq":3637,"template_height":800113},{"accepted_at_ms":999974,"credit_policy":null,"job_id":"job-3915","job_issued_at_ms":999998,"miner_id":"miner-10","network_difficulty":1000005,"ntime":1700003915,"order_key":"10:miner-10","p2mr_program_hex":"4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b","share_difficulty":7,"share_id":"share-3915","share_seq":3915,"template_height":800122},{"accepted_at_ms":999979,"credit_policy":null,"job_id":"job-320","job_issued_at_ms":999993,"miner_id":"miner-1","network_difficulty":1000014,"ntime":1700000320,"order_key":"01:miner-1","p2mr_program_hex":"4040404040404040404040404040404040404040404040404040404040404040","share_difficulty":101,"share_id":"share-320","share_seq":320,"template_height":800010},{"accepted_at_ms":999976,"credit_policy":null,"job_id":"job-847","job_issued_at_ms":1000000,"miner_id":"miner-0","network_difficulty":1000014,"ntime":1700000847,"order_key":"00:miner-0","p2mr_program_hex":"4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f","share_difficulty":19,"share_id":"share-847","share_seq":847,"template_height":800026},{"accepted_at_ms":999974,"credit_policy":null,"job_id":"job-2994","job_issued_at_ms":999993,"miner_id":"miner-2","network_difficulty":1000002,"ntime":1700002994,"order_key":"02:miner-2","p2mr_program_hex":"b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2","share_difficulty":101,"share_id":"share-2994","share_seq":2994,"template_height":800093},{"accepted_at_ms":999996,"credit_policy":null,"job_id":"job-1321","job_issued_at_ms":1000202,"miner_id":"miner-1","network_difficulty":1000012,"ntime":1700001321,"order_key":"01:miner-1","p2mr_program_hex":"2929292929292929292929292929292929292929292929292929292929292929","share_difficulty":1,"share_id":"share-1321","share_seq":1321,"template_height":800041},{"accepted_at_ms":999981,"credit_policy":null,"job_id":"job-1848","job_issued_at_ms":999982,"miner_id":"miner-0","network_difficulty":1000012,"ntime":1700001848,"order_key":"00:miner-0","p2mr_program_hex":"3838383838383838383838383838383838383838383838383838383838383838","share_difficulty":7,"share_id":"share-1848","share_seq":1848,"template_height":800057},{"accepted_at_ms":999976,"credit_policy":null,"job_id":"job-341","job_issued_at_ms":999996,"miner_id":"miner-0","network_difficulty":1000001,"ntime":1700000341,"order_key":"00:miner-0","p2mr_program_hex":"5555555555555555555555555555555555555555555555555555555555555555","share_difficulty":101,"share_id":"share-341","share_seq":341,"template_height":800010},{"accepted_at_ms":999975,"credit_policy":"stale-grace","job_id":"job-3933","job_issued_at_ms":999976,"miner_id":"miner-6","network_difficulty":1000006,"ntime":1700003933,"order_key":"06:miner-6","p2mr_program_hex":"5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d","share_difficulty":2,"share_id":"share-3933","share_seq":3933,"template_height":800122},{"accepted_at_ms":999977,"credit_policy":null,"job_id":"job-2949","job_issued_at_ms":999972,"miner_id":"miner-1","network_difficulty":1000008,"ntime":1700002949,"order_key":"01:miner-1","p2mr_program_hex":"8585858585858585858585858585858585858585858585858585858585858585","share_difficulty":3,"share_id":"share-2949","share_seq":2949,"template_height":800092},{"accepted_at_ms":999996,"credit_policy":null,"job_id":"job-2647","job_issued_at_ms":999975,"miner_id":"miner-7","network_difficulty":1000012,"ntime":1700002647,"order_key":"07:miner-7","p2mr_program_hex":"5757575757575757575757575757575757575757575757575757575757575757","share_difficulty":1000,"share_id":"share-2647","share_seq":2647,"template_height":800082},{"accepted_at_ms":999991,"credit_policy":null,"job_id":"job-3368","job_issued_at_ms":999982,"miner_id":"miner-2","network_difficulty":1000002,"ntime":1700003368,"order_key":"02:miner-2","p2mr_program_hex":"2828282828282828282828282828282828282828282828282828282828282828","share_difficulty":2,"share_id":"share-3368","share_seq":3368,"template_height":800105},{"accepted_at_ms":999997,"credit_policy":null,"job_id":"job-3354","job_issued_at_ms":999989,"miner_id":"miner-10","network_difficulty":1000005,"ntime":1700003354,"order_key":"10:miner-10","p2mr_program_hex":"1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a","share_difficulty":101,"share_id":"share-3354","share_seq":3354,"template_height":800104},{"accepted_at_ms":999997,"credit_policy":null,"job_id":"job-2558","job_issued_at_ms":999989,"miner_id":"miner-6","network_difficulty":1000008,"ntime":1700002558,"order_key":"06:miner-6","p2mr_program_hex":"fefefefefefefefefefefefefefefefefefefefefefefefefefefefefefefefe","share_difficulty":101,"share_id":"share-2558","share_seq":2558,"template_height":800079},{"accepted_at_ms":999987,"credit_policy":"stale-grace","job_id":"job-169","job_issued_at_ms":999996,"miner_id":"miner-4","network_difficulty":1000016,"ntime":1700000169,"order_key":"04:miner-4","p2mr_program_hex":"a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9","share_difficulty":101,"share_id":"share-169","share_seq":169,"template_height":800005},{"accepted_at_ms":999988,"credit_policy":null,"job_id":"job-1997","job_issued_at_ms":999989,"miner_id":"miner-6","network_difficulty":1000008,"ntime":1700001997,"order_key":"06:miner-6","p2mr_program_hex":"cdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcd","share_difficulty":2,"share_id":"share-1997","share_seq":1997,"template_height":800062},{"accepted_at_ms":999990,"credit_policy":null,"job_id":"job-1289","job_issued_at_ms":999983,"miner_id":"miner-2","network_difficulty":1000014,"ntime":1700001289,"order_key":"02:miner-2","p2mr_program_hex":"0909090909090909090909090909090909090909090909090909090909090909","share_difficulty":19,"share_id":"share-1289","share_seq":1289,"template_height":800040},{"accepted_at_ms":999987,"credit_policy":"stale-grace","job_id":"job-2787","job_issued_at_ms":999995,"miner_id":"miner-4","network_difficulty":1000016,"ntime":1700002787,"order_key":"04:miner-4","p2mr_program_hex":"e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3","share_difficulty":19,"share_id":"share-2787","share_seq":2787,"template_height":800087},{"accepted_at_ms":999999,"credit_policy":null,"job_id":"job-738","job_issued_at_ms":999994,"miner_id":"miner-1","network_difficulty":1000007,"ntime":1700000738,"order_key":"01:miner-1","p2mr_program_hex":"e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2","share_difficulty":2,"share_id":"share-738","share_seq":738,"template_height":800023},{"accepted_at_ms":999987,"credit_policy":null,"job_id":"job-1975","job_issued_at_ms":999982,"miner_id":"miner-6","network_difficulty":1000003,"ntime":1700001975,"order_key":"06:miner-6","p2mr_program_hex":"b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7","share_difficulty":3,"share_id":"share-1975","share_seq":1975,"template_height":800061},{"accepted_at_ms":999990,"credit_policy":null,"job_id":"job-2682","job_issued_at_ms":999990,"miner_id":"miner-9","network_difficulty":1000013,"ntime":1700002682,"order_key":"09:miner-9","p2mr_program_hex":"7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a","share_difficulty":19,"share_id":"share-2682","share_seq":2682,"template_height":800083},{"accepted_at_ms":999971,"credit_policy":null,"job_id":"job-1249","job_issued_at_ms":999983,"miner_id":"miner-6","network_difficulty":1000008,"ntime":1700001249,"order_key":"06:miner-6","p2mr_program_hex":"e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1","share_difficulty":7,"share_id":"share-1249","share_seq":1249,"template_height":800039},{"accepted_at_ms":999984,"credit_policy":null,"job_id":"job-706","job_issued_at_ms":999970,"miner_id":"miner-2","network_difficulty":1000009,"ntime":1700000706,"order_key":"02:miner-2","p2mr_program_hex":"c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2","share_difficulty":101,"share_id":"share-706","share_seq":706,"template_height":800022},{"accepted_at_ms":999983,"credit_policy":null,"job_id":"job-2894","job_issued_at_ms":999993,"miner_id":"miner-1","network_difficulty":1000004,"ntime":1700002894,"order_key":"01:miner-1","p2mr_program_hex":"4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e","share_difficulty":1000,"share_id":"share-2894","share_seq":2894,"template_height":800090},{"accepted_at_ms":999986,"credit_policy":null,"job_id":"job-2618","job_issued_at_ms":999977,"miner_id":"miner-0","network_difficulty":1000000,"ntime":1700002618,"order_key":"00:miner-0","p2mr_program_hex":"3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a","share_difficulty":2,"share_id":"share-2618","share_seq":2618,"template_height":800081},{"accepted_at_ms":999988,"credit_policy":"stale-grace","job_id":"job-2847","job_issued_at_ms":999977,"miner_id":"miner-9","network_difficulty":1000008,"ntime":1700002847,"order_key":"09:miner-9","p2mr_program_hex":"1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f","share_difficulty":7,"share_id":"share-2847","share_seq":2847,"template_height":800088},{"accepted_at_ms":1000164,"credit_policy":null,"job_id":"job-2737","job_issued_at_ms":999974,"miner_id":"miner-9","network_difficulty":1000000,"ntime":1700002737,"order_key":"09:miner-9","p2mr_program_hex":"b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1","share_difficulty":1000,"share_id":"share-2737","share_seq":2737,"template_height":800085},{"accepted_at_ms":999987,"credit_policy":null,"job_id":"job-1845","job_issued_at_ms":999980,"miner_id":"miner-8","network_difficulty":1000009,"ntime":1700001845,"order_key":"08:miner-8","p2mr_program_hex":"3535353535353535353535353535353535353535353535353535353535353535","share_difficulty":2,"share_id":"share-1845","share_seq":1845,"template_height":800057},{"accepted_at_ms":999972,"credit_policy":null,"job_id":"job-2521","job_issued_at_ms":999984,"miner_id":"miner-2","network_difficulty":1000005,"ntime":1700002521,"order_key":"02:miner-2","p2mr_program_hex":"d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9","share_difficulty":19,"share_id":"share-2521","share_seq":2521,"template_height":800078},{"accepted_at_ms":999978,"credit_policy":null,"job_id":"job-1422","job_issued_at_ms":999988,"miner_id":"miner-3","network_difficulty":1000011,"ntime":1700001422,"order_key":"03:miner-3","p2mr_program_hex":"8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e","share_difficulty":101,"share_id":"share-1422","share_seq":1422,"template_height":800044},{"accepted_at_ms":999995,"credit_policy":null,"job_id":"job-345","job_issued_at_ms":999986,"miner_id":"miner-4","network_difficulty":1000005,"ntime":1700000345,"order_key":"04:miner-4","p2mr_program_hex":"5959595959595959595959595959595959595959595959595959595959595959","share_difficulty":19,"share_id":"share-345","share_seq":345,"template_height":800010},{"accepted_at_ms":999984,"credit_policy":null,"job_id":"job-1562","job_issued_at_ms":999972,"miner_id":"miner-0","network_difficulty":1000015,"ntime":1700001562,"order_key":"00:miner-0","p2mr_program_hex":"1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a","share_difficulty":19,"share_id":"share-1562","share_seq":1562,"template_height":800048},{"accepted_at_ms":999978,"credit_policy":null,"job_id":"job-1071","job_issued_at_ms":999988,"miner_id":"miner-4","network_difficulty":1000000,"ntime":1700001071,"order_key":"04:miner-4","p2mr_program_hex":"2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f","share_difficulty":2,"share_id":"share-1071","share_seq":1071,"template_height":800033},{"accepted_at_ms":999984,"credit_policy":null,"job_id":"job-3392","job_issued_at_ms":999984,"miner_id":"miner-4","network_difficulty":1000009,"ntime":1700003392,"order_key":"04:miner-4","p2mr_program_hex":"4040404040404040404040404040404040404040404040404040404040404040","share_difficulty":1,"share_id":"share-3392","share_seq":3392,"template_height":800106},{"accepted_at_ms":999974,"credit_policy":null,"job_id":"job-1594","job_issued_at_ms":999978,"miner_id":"miner-10","network_difficulty":1000013,"ntime":1700001594,"order_key":"10:miner-10","p2mr_program_hex":"3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a","share_difficulty":19,"share_id":"share-1594","share_seq":1594,"template_height":800049},{"accepted_at_ms":1000267,"credit_policy":null,"job_id":"job-2356","job_issued_at_ms":999999,"miner_id":"miner-2","network_difficulty":1000010,"ntime":1700002356,"order_key":"02:miner-2","p2mr_program_hex":"3434343434343434343434343434343434343434343434343434343434343434","share_difficulty":3,"share_id":"share-2356","share_seq":2356,"template_height":800073},{"accepted_at_ms":999989,"credit_policy":null,"job_id":"job-106","job_issued_at_ms":999986,"miner_id":"miner-7","network_difficulty":1000004,"ntime":1700000106,"order_key":"07:miner-7","p2mr_program_hex":"6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a","share_difficulty":1000,"share_id":"share-106","share_seq":106,"template_height":800003},{"accepted_at_ms":999990,"credit_policy":null,"job_id":"job-3996","job_issued_at_ms":999998,"miner_id":"miner-3","network_difficulty":1000001,"ntime":1700003996,"order_key":"03:miner-3","p2mr_program_hex":"9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c","share_difficulty":101,"share_id":"share-3996","share_seq":3996,"template_height":800124},{"accepted_at_ms":999987,"credit_policy":null,"job_id":"job-1246","job_issued_at_ms":999980,"miner_id":"miner-3","network_difficulty":1000005,"ntime":1700001246,"order_key":"03:miner-3","p2mr_program_hex":"dededededededededededededededededededededededededededededededede","share_difficulty":2,"share_id":"share-1246","share_seq":1246,"template_height":800038},{"accepted_at_ms":999977,"credit_policy":null,"job_id":"job-2620","job_issued_at_ms":999986,"miner_id":"miner-2","network_difficulty":1000002,"ntime":1700002620,"order_key":"02:miner-2","p2mr_program_hex":"3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c","share_difficulty":101,"share_id":"share-2620","share_seq":2620,"template_height":800081},{"accepted_at_ms":999980,"credit_policy":null,"job_id":"job-520","job_issued_at_ms":999973,"miner_id":"miner-3","network_difficulty":1000010,"ntime":1700000520,"order_key":"03:miner-3","p2mr_program_hex":"0808080808080808080808080808080808080808080808080808080808080808","share_difficulty":7,"share_id":"share-520","share_seq":520,"template_height":800016},{"accepted_at_ms":999989,"credit_policy":null,"job_id":"job-1135","job_issued_at_ms":999986,"miner_id":"miner-2","network_difficulty":1000013,"ntime":1700001135,"order_key":"02:miner-2","p2mr_program_hex":"6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f","share_difficulty":3,"share_id":"share-1135","share_seq":1135,"template_height":800035},{"accepted_at_ms":999977,"credit_policy":null,"job_id":"job-2208","job_issued_at_ms":1000400,"miner_id":"miner-8","network_difficulty":1000015,"ntime":1700002208,"order_key":"08:miner-8","p2mr_program_hex":"a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0","share_difficulty":7,"share_id":"share-2208","share_seq":2208,"template_height":800069},{"accepted_at_ms":999989,"credit_policy":"stale-grace","job_id":"job-3130","job_issued_at_ms":999985,"miner_id":"miner-6","network_difficulty":1000002,"ntime":1700003130,"order_key":"06:miner-6","p2mr_program_hex":"3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a","share_difficulty":1,"share_id":"share-3130","share_seq":3130,"template_height":800097},{"accepted_at_ms":999979,"credit_policy":null,"job_id":"job-3086","job_issued_at_ms":999972,"miner_id":"miner-6","network_difficulty":1000009,"ntime":1700003086,"order_key":"06:miner-6","p2mr_program_hex":"0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e","share_difficulty":1,"share_id":"share-3086","share_seq":3086,"template_height":800096},{"accepted_at_ms":999994,"credit_policy":null,"job_id":"job-253","job_issued_at_ms":999981,"miner_id":"miner-0","network_difficulty":1000015,"ntime":1700000253,"order_key":"00:miner-0","p2mr_program_hex":"fdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfd","share_difficulty":3,"share_id":"share-253","share_seq":253,"template_height":800007},{"accepted_at_ms":999986,"credit_policy":null,"job_id":"job-1512","job_issued_at_ms":999977,"miner_id":"miner-5","network_difficulty":1000016,"ntime":1700001512,"order_key":"05:miner-5","p2mr_program_hex":"e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8","share_difficulty":3,"share_id":"share-1512","share_seq":1512,"template_height":800047},{"accepted_at_ms":999994,"credit_policy":null,"job_id":"job-571","job_issued_at_ms":999977,"miner_id":"miner-10","network_difficulty":1000010,"ntime":1700000571,"order_key":"10:miner-10","p2mr_program_hex":"3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b","share_difficulty":101,"share_id":"share-571","share_seq":571,"template_height":800017},{"accepted_at_ms":999978,"credit_policy":null,"job_id":"job-512","job_issued_at_ms":999990,"miner_id":"miner-6","network_difficulty":1000002,"ntime":1700000512,"order_key":"06:miner-6","p2mr_program_hex":"0000000000000000000000000000000000000000000000000000000000000000","share_difficulty":1,"share_id":"share-512","share_seq":512,"template_height":800016},{"accepted_at_ms":999992,"credit_policy":null,"job_id":"job-3343","job_issued_at_ms":999972,"miner_id":"miner-10","network_difficulty":1000011,"ntime":1700003343,"order_key":"10:miner-10","p2mr_program_hex":"0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f","share_difficulty":19,"share_id":"share-3343","share_seq":3343,"template_height":800104},{"accepted_at_ms":999972,"credit_policy":null,"job_id":"job-578","job_issued_at_ms":999996,"miner_id":"miner-6","network_difficulty":1000000,"ntime":1700000578,"order_key":"06:miner-6","p2mr_program_hex":"4242424242424242424242424242424242424242424242424242424242424242","share_difficulty":7,"share_id":"share-578","share_seq":578,"template_height":800018},{"accepted_at_ms":1000224,"credit_policy":null,"job_id":"job-1705","job_issued_at_ms":999989,"miner_id":"miner-0","network_difficulty":1000005,"ntime":1700001705,"order_key":"00:miner-0","p2mr_program_hex":"a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9","share_difficulty":1,"share_id":"share-1705","share_seq":1705,"template_height":800053},{"accepted_at_ms":999987,"credit_policy":null,"job_id":"job-2368","job_issued_at_ms":999984,"miner_id":"miner-3","network_difficulty":1000005,"ntime":1700002368,"order_key":"03:miner-3","p2mr_program_hex":"4040404040404040404040404040404040404040404040404040404040404040","share_difficulty":7,"share_id":"share-2368","share_seq":2368,"template_height":800074},{"accepted_at_ms":999975,"credit_policy":null,"job_id":"job-2504","job_issued_at_ms":999995,"miner_id":"miner-7","network_difficulty":1000005,"ntime":1700002504,"order_key":"07:miner-7","p2mr_program_hex":"c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8","share_difficulty":101,"share_id":"share-2504","share_seq":2504,"template_height":800078},{"accepted_at_ms":999998,"credit_policy":null,"job_id":"job-3986","job_issued_at_ms":999982,"miner_id":"miner-4","network_difficulty":1000008,"ntime":1700003986,"order_key":"04:miner-4","p2mr_program_hex":"9292929292929292929292929292929292929292929292929292929292929292","share_difficulty":19,"share_id":"share-3986","share_seq":3986,"template_height":800124},{"accepted_at_ms":999997,"credit_policy":null,"job_id":"job-358","job_issued_at_ms":999982,"miner_id":"miner-6","network_difficulty":1000001,"ntime":1700000358,"order_key":"06:miner-6","p2mr_program_hex":"6666666666666666666666666666666666666666666666666666666666666666","share_difficulty":3,"share_id":"share-358","share_seq":358,"template_height":800011},{"accepted_at_ms":999977,"credit_policy":null,"job_id":"job-2175","job_issued_at_ms":999993,"miner_id":"miner-8","network_difficulty":1000016,"ntime":1700002175,"order_key":"08:miner-8","p2mr_program_hex":"7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f","share_difficulty":3,"share_id":"share-2175","share_seq":2175,"template_height":800067},{"accepted_at_ms":999983,"credit_policy":null,"job_id":"job-1255","job_issued_at_ms":999971,"miner_id":"miner-1","network_difficulty":1000014,"ntime":1700001255,"order_key":"01:miner-1","p2mr_program_hex":"e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7","share_difficulty":19,"share_id":"share-1255","share_seq":1255,"template_height":800039},{"accepted_at_ms":999970,"credit_policy":null,"job_id":"job-3035","job_issued_at_ms":999998,"miner_id":"miner-10","network_difficulty":1000009,"ntime":1700003035,"order_key":"10:miner-10","p2mr_program_hex":"dbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdb","share_difficulty":2,"share_id":"share-3035","share_seq":3035,"template_height":800094},{"accepted_at_ms":1000000,"credit_policy":"stale-grace","job_id":"job-1730","job_issued_at_ms":1000000,"miner_id":"miner-3","network_difficulty":1000013,"ntime":1700001730,"order_key":"03:miner-3","p2mr_program_hex":"c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2","share_difficulty":1,"share_id":"share-1730","share_seq":1730,"template_height":800054},{"accepted_at_ms":999976,"credit_policy":"stale-grace","job_id":"job-1163","job_issued_at_ms":999979,"miner_id":"miner-8","network_difficulty":1000007,"ntime":1700001163,"order_key":"08:miner-8","p2mr_program_hex":"8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b","share_difficulty":3,"share_id":"share-1163","share_seq":1163,"template_height":800036},{"accepted_at_ms":999984,"credit_policy":"stale-grace","job_id":"job-1172","job_issued_at_ms":999996,"miner_id":"miner-6","network_difficulty":1000016,"ntime":1700001172,"order_key":"06:miner-6","p2mr_program_hex":"9494949494949494949494949494949494949494949494949494949494949494","share_difficulty":101,"share_id":"share-1172","share_seq":1172,"template_height":800036},{"accepted_at_ms":999973,"credit_policy":null,"job_id":"job-673","job_issued_at_ms":999985,"miner_id":"miner-2","network_difficulty":1000010,"ntime":1700000673,"order_key":"02:miner-2","p2mr_program_hex":"a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1","share_difficulty":1,"share_id":"share-673","share_seq":673,"template_height":800021},{"accepted_at_ms":999987,"credit_policy":"stale-grace","job_id":"job-2748","job_issued_at_ms":999998,"miner_id":"miner-9","network_difficulty":1000011,"ntime":1700002748,"order_key":"09:miner-9","p2mr_program_hex":"bcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbc","share_difficulty":2,"share_id":"share-2748","share_seq":2748,"template_height":800085},{"accepted_at_ms":999988,"credit_policy":null,"job_id":"job-754","job_issued_at_ms":999981,"miner_id":"miner-6","network_difficulty":1000006,"ntime":1700000754,"order_key":"06:miner-6","p2mr_program_hex":"f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2","share_difficulty":3,"share_id":"share-754","share_seq":754,"template_height":800023},{"accepted_at_ms":999994,"credit_policy":null,"job_id":"job-3203","job_issued_at_ms":1000397,"miner_id":"miner-2","network_difficulty":1000007,"ntime":1700003203,"order_key":"02:miner-2","p2mr_program_hex":"8383838383838383838383838383838383838383838383838383838383838383","share_difficulty":7,"share_id":"share-3203","share_seq":3203,"template_height":800100},{"accepted_at_ms":999980,"credit_policy":null,"job_id":"job-502","job_issued_at_ms":999991,"miner_id":"miner-7","network_difficulty":1000009,"ntime":1700000502,"order_key":"07:miner-7","p2mr_program_hex":"f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6","share_difficulty":101,"share_id":"share-502","share_seq":502,"template_height":800015},{"accepted_at_ms":999995,"credit_policy":"stale-grace","job_id":"job-1122","job_issued_at_ms":999991,"miner_id":"miner-0","network_difficulty":1000000,"ntime":1700001122,"order_key":"00:miner-0","p2mr_program_hex":"6262626262626262626262626262626262626262626262626262626262626262","share_difficulty":2,"share_id":"share-1122","share_seq":1122,"template_height":800035},{"accepted_at_ms":999971,"credit_policy":null,"job_id":"job-3391","job_issued_at_ms":999977,"miner_id":"miner-3","network_difficulty":1000008,"ntime":1700003391,"order_key":"03:miner-3","p2mr_program_hex":"3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f","share_difficulty":3,"share_id":"share-3391","share_seq":3391,"template_height":800105},{"accepted_at_ms":999989,"credit_policy":null,"job_id":"job-3237","job_issued_at_ms":999973,"miner_id":"miner-3","network_difficulty":1000007,"ntime":1700003237,"order_key":"03:miner-3","p2mr_program_hex":"a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5","share_difficulty":3,"share_id":"share-3237","share_seq":3237,"template_height":800101},{"accepted_at_ms":999979,"credit_policy":null,"job_id":"job-579","job_issued_at_ms":999978,"miner_id":"miner-7","network_difficulty":1000001,"ntime":1700000579,"order_key":"07:miner-7","p2mr_program_hex":"4343434343434343434343434343434343434343434343434343434343434343","share_difficulty":2,"share_id":"share-579","share_seq":579,"template_height":800018},{"accepted_at_ms":999991,"credit_policy":null,"job_id":"job-400","job_issued_at_ms":999989,"miner_id":"miner-4","network_difficulty":1000009,"ntime":1700000400,"order_key":"04:miner-4","p2mr_program_hex":"9090909090909090909090909090909090909090909090909090909090909090","share_difficulty":19,"share_id":"share-400","share_seq":400,"template_height":800012},{"accepted_at_ms":999990,"credit_policy":"stale-grace","job_id":"job-2858","job_issued_at_ms":999998,"miner_id":"miner-9","network_difficulty":1000002,"ntime":1700002858,"order_key":"09:miner-9","p2mr_program_hex":"2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a","share_difficulty":2,"share_id":"share-2858","share_seq":2858,"template_height":800089},{"accepted_at_ms":1000201,"credit_policy":null,"job_id":"job-3137","job_issued_at_ms":1000231,"miner_id":"miner-2","network_difficulty":1000009,"ntime":1700003137,"order_key":"02:miner-2","p2mr_program_hex":"4141414141414141414141414141414141414141414141414141414141414141","share_difficulty":2,"share_id":"share-3137","share_seq":3137,"template_height":800098},{"accepted_at_ms":999981,"credit_policy":"stale-grace","job_id":"job-1875","job_issued_at_ms":1000051,"miner_id":"miner-5","network_difficulty":1000005,"ntime":1700001875,"order_key":"05:miner-5","p2mr_program_hex":"5353535353535353535353535353535353535353535353535353535353535353","share_difficulty":7,"share_id":"share-1875","share_seq":1875,"template_height":800058},{"accepted_at_ms":999986,"credit_policy":null,"job_id":"job-667","job_issued_at_ms":999999,"miner_id":"miner-7","network_difficulty":1000004,"ntime":1700000667,"order_key":"07:miner-7","p2mr_program_hex":"9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b","share_difficulty":19,"share_id":"share-667","share_seq":667,"template_height":800020},{"accepted_at_ms":999990,"credit_policy":null,"job_id":"job-3524","job_issued_at_ms":999990,"miner_id":"miner-4","network_difficulty":1000005,"ntime":1700003524,"order_key":"04:miner-4","p2mr_program_hex":"c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4","share_difficulty":1,"share_id":"share-3524","share_seq":3524,"template_height":800110},{"accepted_at_ms":999991,"credit_policy":null,"job_id":"job-951","job_issued_at_ms":1000000,"miner_id":"miner-5","network_difficulty":1000016,"ntime":1700000951,"order_key":"05:miner-5","p2mr_program_hex":"b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7","share_difficulty":2,"share_id":"share-951","share_seq":951,"template_height":800029},{"accepted_at_ms":1000157,"credit_policy":"stale-grace","job_id":"job-2416","job_issued_at_ms":1000350,"miner_id":"miner-7","network_difficulty":1000002,"ntime":1700002416,"order_key":"07:miner-7","p2mr_program_hex":"7070707070707070707070707070707070707070707070707070707070707070","share_difficulty":1,"share_id":"share-2416","share_seq":2416,"template_height":800075},{"accepted_at_ms":999982,"credit_policy":null,"job_id":"job-3404","job_issued_at_ms":999978,"miner_id":"miner-5","network_difficulty":1000004,"ntime":1700003404,"order_key":"05:miner-5","p2mr_program_hex":"4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c","share_difficulty":2,"share_id":"share-3404","share_seq":3404,"template_height":800106},{"accepted_at_ms":999973,"credit_policy":null,"job_id":"job-1518","job_issued_at_ms":999978,"miner_id":"miner-0","network_difficulty":1000005,"ntime":1700001518,"order_key":"00:miner-0","p2mr_program_hex":"eeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeee","share_difficulty":101,"share_id":"share-1518","share_seq":1518,"template_height":800047},{"accepted_at_ms":999980,"credit_policy":null,"job_id":"job-3697","job_issued_at_ms":999971,"miner_id":"miner-1","network_difficulty":1000008,"ntime":1700003697,"order_key":"01:miner-1","p2mr_program_hex":"7171717171717171717171717171717171717171717171717171717171717171","share_difficulty":7,"share_id":"share-3697","share_seq":3697,"template_height":800115},{"accepted_at_ms":999971,"credit_policy":null,"job_id":"job-1392","job_issued_at_ms":999976,"miner_id":"miner-6","network_difficulty":1000015,"ntime":1700001392,"order_key":"06:miner-6","p2mr_program_hex":"7070707070707070707070707070707070707070707070707070707070707070","share_difficulty":1000,"share_id":"share-1392","share_seq":1392,"template_height":800043},{"accepted_at_ms":999990,"credit_policy":null,"job_id":"job-2346","job_issued_at_ms":999981,"miner_id":"miner-3","network_difficulty":1000000,"ntime":1700002346,"order_key":"03:miner-3","p2mr_program_hex":"2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a","share_difficulty":7,"share_id":"share-2346","share_seq":2346,"template_height":800073},{"accepted_at_ms":999977,"credit_policy":null,"job_id":"job-2574","job_issued_at_ms":999974,"miner_id":"miner-0","network_difficulty":1000007,"ntime":1700002574,"order_key":"00:miner-0","p2mr_program_hex":"0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e","share_difficulty":3,"share_id":"share-2574","share_seq":2574,"template_height":800080},{"accepted_at_ms":999998,"credit_policy":null,"job_id":"job-3907","job_issued_at_ms":999990,"miner_id":"miner-2","network_difficulty":1000014,"ntime":1700003907,"order_key":"02:miner-2","p2mr_program_hex":"4343434343434343434343434343434343434343434343434343434343434343","share_difficulty":1000,"share_id":"share-3907","share_seq":3907,"template_height":800122},{"accepted_at_ms":999987,"credit_policy":null,"job_id":"job-2477","job_issued_at_ms":1000000,"miner_id":"miner-2","network_difficulty":1000012,"ntime":1700002477,"order_key":"02:miner-2","p2mr_program_hex":"adadadadadadadadadadadadadadadadadadadadadadadadadadadadadadadad","share_difficulty":19,"share_id":"share-2477","share_seq":2477,"template_height":800077},{"accepted_at_ms":999973,"credit_policy":null,"job_id":"job-1930","job_issued_at_ms":999992,"miner_id":"miner-5","network_difficulty":1000009,"ntime":1700001930,"order_key":"05:miner-5","p2mr_program_hex":"8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a","share_difficulty":1,"share_id":"share-1930","share_seq":1930,"template_height":800060},{"accepted_at_ms":999984,"credit_policy":"stale-grace","job_id":"job-2560","job_issued_at_ms":999992,"miner_id":"miner-8","network_difficulty":1000010,"ntime":1700002560,"order_key":"08:miner-8","p2mr_program_hex":"0000000000000000000000000000000000000000000000000000000000000000","share_difficulty":7,"share_id":"share-2560","share_seq":2560,"template_height":800080},{"accepted_at_ms":1000000,"credit_policy":null,"job_id":"job-991","job_issued_at_ms":999997,"miner_id":"miner-1","network_difficulty":1000005,"ntime":1700000991,"order_key":"01:miner-1","p2mr_program_hex":"dfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdf","share_difficulty":7,"share_id":"share-991","share_seq":991,"template_height":800030},{"accepted_at_ms":999974,"credit_policy":"stale-grace","job_id":"job-3389","job_issued_at_ms":999970,"miner_id":"miner-1","network_difficulty":1000006,"ntime":1700003389,"order_key":"01:miner-1","p2mr_program_hex":"3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d","share_difficulty":19,"share_id":"share-3389","share_seq":3389,"template_height":800105},{"accepted_at_ms":999981,"credit_policy":"stale-grace","job_id":"job-2536","job_issued_at_ms":999999,"miner_id":"miner-6","network_difficulty":1000003,"ntime":1700002536,"order_key":"06:miner-6","p2mr_program_hex":"e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8","share_difficulty":7,"share_id":"share-2536","share_seq":2536,"template_height":800079},{"accepted_at_ms":999992,"credit_policy":"stale-grace","job_id":"job-422","job_issued_at_ms":999996,"miner_id":"miner-4","network_difficulty":1000014,"ntime":1700000422,"order_key":"04:miner-4","p2mr_program_hex":"a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6","share_difficulty":101,"share_id":"share-422","share_seq":422,"template_height":800013},{"accepted_at_ms":999983,"credit_policy":null,"job_id":"job-3925","job_issued_at_ms":999998,"miner_id":"miner-9","network_difficulty":1000015,"ntime":1700003925,"order_key":"09:miner-9","p2mr_program_hex":"5555555555555555555555555555555555555555555555555555555555555555","share_difficulty":101,"share_id":"share-3925","share_seq":3925,"template_height":800122},{"accepted_at_ms":999970,"credit_policy":null,"job_id":"job-228","job_issued_at_ms":999987,"miner_id":"miner-8","network_difficulty":1000007,"ntime":1700000228,"order_key":"08:miner-8","p2mr_program_hex":"e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4","share_difficulty":1000,"share_id":"share-228","share_seq":228,"template_height":800007},{"accepted_at_ms":999982,"credit_policy":null,"job_id":"job-2012","job_issued_at_ms":999978,"miner_id":"miner-10","network_difficulty":1000006,"ntime":1700002012,"order_key":"10:miner-10","p2mr_program_hex":"dcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdc","share_difficulty":101,"share_id":"share-2012","share_seq":2012,"template_height":800062},{"accepted_at_ms":1000247,"credit_policy":null,"job_id":"job-3682","job_issued_at_ms":1000302,"miner_id":"miner-8","network_difficulty":1000010,"ntime":1700003682,"order_key":"08:miner-8","p2mr_program_hex":"6262626262626262626262626262626262626262626262626262626262626262","share_difficulty":2,"share_id":"share-3682","share_seq":3682,"template_height":800115},{"accepted_at_ms":1000063,"credit_policy":null,"job_id":"job-1854","job_issued_at_ms":999996,"miner_id":"miner-6","network_difficulty":1000001,"ntime":1700001854,"order_key":"06:miner-6","p2mr_program_hex":"3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e","share_difficulty":2,"share_id":"share-1854","share_seq":1854,"template_height":800057},{"accepted_at_ms":999983,"credit_policy":null,"job_id":"job-1601","job_issued_at_ms":999988,"miner_id":"miner-6","network_difficulty":1000003,"ntime":1700001601,"order_key":"06:miner-6","p2mr_program_hex":"4141414141414141414141414141414141414141414141414141414141414141","share_difficulty":3,"share_id":"share-1601","share_seq":1601,"template_height":800050},{"accepted_at_ms":999977,"credit_policy":null,"job_id":"job-1182","job_issued_at_ms":999979,"miner_id":"miner-5","network_difficulty":1000009,"ntime":1700001182,"order_key":"05:miner-5","p2mr_program_hex":"9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e","share_difficulty":3,"share_id":"share-1182","share_seq":1182,"template_height":800036},{"accepted_at_ms":999996,"credit_policy":null,"job_id":"job-2125","job_issued_at_ms":999997,"miner_id":"miner-2","network_difficulty":1000000,"ntime":1700002125,"order_key":"02:miner-2","p2mr_program_hex":"4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d","share_difficulty":1,"share_id":"share-2125","share_seq":2125,"template_height":800066},{"accepted_at_ms":999991,"credit_policy":null,"job_id":"job-495","job_issued_at_ms":999978,"miner_id":"miner-0","network_difficulty":1000002,"ntime":1700000495,"order_key":"00:miner-0","p2mr_program_hex":"efefefefefefefefefefefefefefefefefefefefefefefefefefefefefefefef","share_difficulty":1,"share_id":"share-495","share_seq":495,"template_height":800015},{"accepted_at_ms":999971,"credit_policy":null,"job_id":"job-3644","job_issued_at_ms":1000000,"miner_id":"miner-3","network_difficulty":1000006,"ntime":1700003644,"order_key":"03:miner-3","p2mr_program_hex":"3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c","share_difficulty":3,"share_id":"share-3644","share_seq":3644,"template_height":800113},{"accepted_at_ms":999995,"credit_policy":null,"job_id":"job-877","job_issued_at_ms":999975,"miner_id":"miner-8","network_difficulty":1000010,"ntime":1700000877,"order_key":"08:miner-8","p2mr_program_hex":"6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d","share_difficulty":1000,"share_id":"share-877","share_seq":877,"template_height":800027},{"accepted_at_ms":999996,"credit_policy":null,"job_id":"job-2999","job_issued_at_ms":1000198,"miner_id":"miner-7","network_difficulty":1000007,"ntime":1700002999,"order_key":"07:miner-7","p2mr_program_hex":"b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7","share_difficulty":1000,"share_id":"share-2999","share_seq":2999,"template_height":800093},{"accepted_at_ms":999975,"credit_policy":null,"job_id":"job-3092","job_issued_at_ms":999971,"miner_id":"miner-1","network_difficulty":1000015,"ntime":1700003092,"order_key":"01:miner-1","p2mr_program_hex":"1414141414141414141414141414141414141414141414141414141414141414","share_difficulty":2,"share_id":"share-3092","share_seq":3092,"template_height":800096},{"accepted_at_ms":999980,"credit_policy":null,"job_id":"job-3307","job_issued_at_ms":999970,"miner_id":"miner-7","network_difficulty":1000009,"ntime":1700003307,"order_key":"07:miner-7","p2mr_program_hex":"ebebebebebebebebebebebebebebebebebebebebebebebebebebebebebebebeb","share_difficulty":1000,"share_id":"share-3307","share_seq":3307,"template_height":800103},{"accepted_at_ms":999978,"credit_policy":null,"job_id":"job-1857","job_issued_at_ms":999998,"miner_id":"miner-9","network_difficulty":1000004,"ntime":1700001857,"order_key":"09:miner-9","p2mr_program_hex":"4141414141414141414141414141414141414141414141414141414141414141","share_difficulty":1,"share_id":"share-1857","share_seq":1857,"template_height":800058},{"accepted_at_ms":999994,"credit_policy":null,"job_id":"job-3656","job_issued_at_ms":999983,"miner_id":"miner-4","network_difficulty":1000001,"ntime":1700003656,"order_key":"04:miner-4","p2mr_program_hex":"4848484848484848484848484848484848484848484848484848484848484848","share_difficulty":19,"share_id":"share-3656","share_seq":3656,"template_height":800114},{"accepted_at_ms":999979,"credit_policy":null,"job_id":"job-2123","job_issued_at_ms":999985,"miner_id":"miner-0","network_difficulty":1000015,"ntime":1700002123,"order_key":"00:miner-0","p2mr_program_hex":"4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b","share_difficulty":7,"share_id":"share-2123","share_seq":2123,"template_height":800066},{"accepted_at_ms":999996,"credit_policy":null,"job_id":"job-2897","job_issued_at_ms":999972,"miner_id":"miner-4","network_difficulty":1000007,"ntime":1700002897,"order_key":"04:miner-4","p2mr_program_hex":"5151515151515151515151515151515151515151515151515151515151515151","share_difficulty":1000,"share_id":"share-2897","share_seq":2897,"template_height":800090},{"accepted_at_ms":999973,"credit_policy":null,"job_id":"job-3080","job_issued_at_ms":999983,"miner_id":"miner-0","network_difficulty":1000003,"ntime":1700003080,"order_key":"00:miner-0","p2mr_program_hex":"0808080808080808080808080808080808080808080808080808080808080808","share_difficulty":2,"share_id":"share-3080","share_seq":3080,"template_height":800096},{"accepted_at_ms":999982,"credit_policy":null,"job_id":"job-3676","job_issued_at_ms":999984,"miner_id":"miner-2","network_difficulty":1000004,"ntime":1700003676,"order_key":"02:miner-2","p2mr_program_hex":"5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c","share_difficulty":2,"share_id":"share-3676","share_seq":3676,"template_height":800114},{"accepted_at_ms":999981,"credit_policy":null,"job_id":"job-3897","job_issued_at_ms":999976,"miner_id":"miner-3","network_difficulty":1000004,"ntime":1700003897,"order_key":"03:miner-3","p2mr_program_hex":"3939393939393939393939393939393939393939393939393939393939393939","share_difficulty":101,"share_id":"share-3897","share_seq":3897,"template_height":800121},{"accepted_at_ms":999997,"credit_policy":null,"job_id":"job-983","job_issued_at_ms":999977,"miner_id":"miner-4","network_difficulty":1000014,"ntime":1700000983,"order_key":"04:miner-4","p2mr_program_hex":"d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7","share_difficulty":101,"share_id":"share-983","share_seq":983,"template_height":800030},{"accepted_at_ms":999987,"credit_policy":"stale-grace","job_id":"job-1066","job_issued_at_ms":999986,"miner_id":"miner-10","network_difficulty":1000012,"ntime":1700001066,"order_key":"10:miner-10","p2mr_program_hex":"2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a","share_difficulty":1000,"share_id":"share-1066","share_seq":1066,"template_height":800033},{"accepted_at_ms":999975,"credit_policy":null,"job_id":"job-2741","job_issued_at_ms":999982,"miner_id":"miner-2","network_difficulty":1000004,"ntime":1700002741,"order_key":"02:miner-2","p2mr_program_hex":"b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5","share_difficulty":1,"share_id":"share-2741","share_seq":2741,"template_height":800085},{"accepted_at_ms":999976,"credit_policy":null,"job_id":"job-1035","job_issued_at_ms":999982,"miner_id":"miner-1","network_difficulty":1000015,"ntime":1700001035,"order_key":"01:miner-1","p2mr_program_hex":"0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b","share_difficulty":19,"share_id":"share-1035","share_seq":1035,"template_height":800032},{"accepted_at_ms":999986,"credit_policy":null,"job_id":"job-1985","job_issued_at_ms":999993,"miner_id":"miner-5","network_difficulty":1000013,"ntime":1700001985,"order_key":"05:miner-5","p2mr_program_hex":"c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1","share_difficulty":1000,"share_id":"share-1985","share_seq":1985,"template_height":800062},{"accepted_at_ms":999975,"credit_policy":null,"job_id":"job-702","job_issued_at_ms":999975,"miner_id":"miner-9","network_difficulty":1000005,"ntime":1700000702,"order_key":"09:miner-9","p2mr_program_hex":"bebebebebebebebebebebebebebebebebebebebebebebebebebebebebebebebe","share_difficulty":7,"share_id":"share-702","share_seq":702,"template_height":800021},{"accepted_at_ms":999979,"credit_policy":"stale-grace","job_id":"job-1648","job_issued_at_ms":999986,"miner_id":"miner-9","network_difficulty":1000016,"ntime":1700001648,"order_key":"09:miner-9","p2mr_program_hex":"7070707070707070707070707070707070707070707070707070707070707070","share_difficulty":19,"share_id":"share-1648","share_seq":1648,"template_height":800051},{"accepted_at_ms":999976,"credit_policy":null,"job_id":"job-2908","job_issued_at_ms":999995,"miner_id":"miner-4","network_difficulty":1000001,"ntime":1700002908,"order_key":"04:miner-4","p2mr_program_hex":"5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c","share_difficulty":19,"share_id":"share-2908","share_seq":2908,"template_height":800090},{"accepted_at_ms":999974,"credit_policy":null,"job_id":"job-351","job_issued_at_ms":999975,"miner_id":"miner-10","network_difficulty":1000011,"ntime":1700000351,"order_key":"10:miner-10","p2mr_program_hex":"5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f","share_difficulty":2,"share_id":"share-351","share_seq":351,"template_height":800010},{"accepted_at_ms":999980,"credit_policy":"stale-grace","job_id":"job-3523","job_issued_at_ms":999979,"miner_id":"miner-3","network_difficulty":1000004,"ntime":1700003523,"order_key":"03:miner-3","p2mr_program_hex":"c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3","share_difficulty":1000,"share_id":"share-3523","share_seq":3523,"template_height":800110},{"accepted_at_ms":999985,"credit_policy":null,"job_id":"job-3493","job_issued_at_ms":999983,"miner_id":"miner-6","network_difficulty":1000008,"ntime":1700003493,"order_key":"06:miner-6","p2mr_program_hex":"a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5","share_difficulty":2,"share_id":"share-3493","share_seq":3493,"template_height":800109},{"accepted_at_ms":999998,"credit_policy":null,"job_id":"job-981","job_issued_at_ms":999987,"miner_id":"miner-2","network_difficulty":1000012,"ntime":1700000981,"order_key":"02:miner-2","p2mr_program_hex":"d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5","share_difficulty":1,"share_id":"share-981","share_seq":981,"template_height":800030},{"accepted_at_ms":999992,"credit_policy":null,"job_id":"job-174","job_issued_at_ms":999984,"miner_id":"miner-9","network_difficulty":1000004,"ntime":1700000174,"order_key":"09:miner-9","p2mr_program_hex":"aeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeae","share_difficulty":101,"share_id":"share-174","share_seq":174,"template_height":800005},{"accepted_at_ms":999988,"credit_policy":null,"job_id":"job-3932","job_issued_at_ms":1000000,"miner_id":"miner-5","network_difficulty":1000005,"ntime":1700003932,"order_key":"05:miner-5","p2mr_program_hex":"5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c","share_difficulty":7,"share_id":"share-3932","share_seq":3932,"template_height":800122},{"accepted_at_ms":999997,"credit_policy":null,"job_id":"job-1141","job_issued_at_ms":999972,"miner_id":"miner-8","network_difficulty":1000002,"ntime":1700001141,"order_key":"08:miner-8","p2mr_program_hex":"7575757575757575757575757575757575757575757575757575757575757575","share_difficulty":3,"share_id":"share-1141","share_seq":1141,"template_height":800035},{"accepted_at_ms":999979,"credit_policy":null,"job_id":"job-316","job_issued_at_ms":999975,"miner_id":"miner-8","network_difficulty":1000010,"ntime":1700000316,"order_key":"08:miner-8","p2mr_program_hex":"3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c","share_difficulty":1000,"share_id":"share-316","share_seq":316,"template_height":800009},{"accepted_at_ms":999980,"credit_policy":null,"job_id":"job-2132","job_issued_at_ms":999995,"miner_id":"miner-9","network_difficulty":1000007,"ntime":1700002132,"order_key":"09:miner-9","p2mr_program_hex":"5454545454545454545454545454545454545454545454545454545454545454","share_difficulty":101,"share_id":"share-2132","share_seq":2132,"template_height":800066},{"accepted_at_ms":1000489,"credit_policy":"stale-grace","job_id":"job-3994","job_issued_at_ms":1000384,"miner_id":"miner-1","network_difficulty":1000016,"ntime":1700003994,"order_key":"01:miner-1","p2mr_program_hex":"9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a","share_difficulty":19,"share_id":"share-3994","share_seq":3994,"template_height":800124},{"accepted_at_ms":1000238,"credit_policy":null,"job_id":"job-3344","job_issued_at_ms":999989,"miner_id":"miner-0","network_difficulty":1000012,"ntime":1700003344,"order_key":"00:miner-0","p2mr_program_hex":"1010101010101010101010101010101010101010101010101010101010101010","share_difficulty":1000,"share_id":"share-3344","share_seq":3344,"template_height":800104},{"accepted_at_ms":999998,"credit_policy":null,"job_id":"job-2328","job_issued_at_ms":999983,"miner_id":"miner-7","network_difficulty":1000016,"ntime":1700002328,"order_key":"07:miner-7","p2mr_program_hex":"1818181818181818181818181818181818181818181818181818181818181818","share_difficulty":1000,"share_id":"share-2328","share_seq":2328,"template_height":800072},{"accepted_at_ms":999982,"credit_policy":null,"job_id":"job-1505","job_issued_at_ms":999999,"miner_id":"miner-9","network_difficulty":1000009,"ntime":1700001505,"order_key":"09:miner-9","p2mr_program_hex":"e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1","share_difficulty":19,"share_id":"share-1505","share_seq":1505,"template_height":800047},{"accepted_at_ms":999985,"credit_policy":null,"job_id":"job-1595","job_issued_at_ms":999998,"miner_id":"miner-0","network_difficulty":1000014,"ntime":1700001595,"order_key":"00:miner-0","p2mr_program_hex":"3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b","share_difficulty":3,"share_id":"share-1595","share_seq":1595,"template_height":800049},{"accepted_at_ms":999985,"credit_policy":null,"job_id":"job-1060","job_issued_at_ms":999999,"miner_id":"miner-4","network_difficulty":1000006,"ntime":1700001060,"order_key":"04:miner-4","p2mr_program_hex":"2424242424242424242424242424242424242424242424242424242424242424","share_difficulty":3,"share_id":"share-1060","share_seq":1060,"template_height":800033},{"accepted_at_ms":999985,"credit_policy":null,"job_id":"job-221","job_issued_at_ms":999989,"miner_id":"miner-1","network_difficulty":1000000,"ntime":1700000221,"order_key":"01:miner-1","p2mr_program_hex":"dddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddd","share_difficulty":1000,"share_id":"share-221","share_seq":221,"template_height":800006},{"accepted_at_ms":999973,"credit_policy":"stale-grace","job_id":"job-2413","job_issued_at_ms":999976,"miner_id":"miner-4","network_difficulty":1000016,"ntime":1700002413,"order_key":"04:miner-4","p2mr_program_hex":"6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d","share_difficulty":1000,"share_id":"share-2413","share_seq":2413,"template_height":800075},{"accepted_at_ms":999990,"credit_policy":null,"job_id":"job-2831","job_issued_at_ms":999993,"miner_id":"miner-4","network_difficulty":1000009,"ntime":1700002831,"order_key":"04:miner-4","p2mr_program_hex":"0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f","share_difficulty":101,"share_id":"share-2831","share_seq":2831,"template_height":800088},{"accepted_at_ms":999990,"credit_policy":null,"job_id":"job-3634","job_issued_at_ms":1000438,"miner_id":"miner-4","network_difficulty":1000013,"ntime":1700003634,"order_key":"04:miner-4","p2mr_program_hex":"3232323232323232323232323232323232323232323232323232323232323232","share_difficulty":7,"share_id":"share-3634","share_seq":3634,"template_height":800113},{"accepted_at_ms":1000132,"credit_policy":"stale-grace","job_id":"job-2140","job_issued_at_ms":1000304,"miner_id":"miner-6","network_difficulty":1000015,"ntime":1700002140,"order_key":"06:miner-6","p2mr_program_hex":"5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c","share_difficulty":2,"share_id":"share-2140","share_seq":2140,"template_height":800066},{"accepted_at_ms":999978,"credit_policy":null,"job_id":"job-1369","job_issued_at_ms":999992,"miner_id":"miner-5","network_difficulty":1000009,"ntime":1700001369,"order_key":"05:miner-5","p2mr_program_hex":"5959595959595959595959595959595959595959595959595959595959595959","share_difficulty":3,"share_id":"share-1369","share_seq":1369,"template_height":800042},{"accepted_at_ms":999977,"credit_policy":null,"job_id":"job-2571","job_issued_at_ms":999975,"miner_id":"miner-8","network_difficulty":1000004,"ntime":1700002571,"order_key":"08:miner-8","p2mr_program_hex":"0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b","share_difficulty":2,"share_id":"share-2571","share_seq":2571,"template_height":800080},{"accepted_at_ms":999983,"credit_policy":null,"job_id":"job-1305","job_issued_at_ms":999978,"miner_id":"miner-7","network_difficulty":1000013,"ntime":1700001305,"order_key":"07:miner-7","p2mr_program_hex":"1919191919191919191919191919191919191919191919191919191919191919","share_difficulty":101,"share_id":"share-1305","share_seq":1305,"template_height":800040},{"accepted_at_ms":999984,"credit_policy":null,"job_id":"job-1379","job_issued_at_ms":999986,"miner_id":"miner-4","network_difficulty":1000002,"ntime":1700001379,"order_key":"04:miner-4","p2mr_program_hex":"6363636363636363636363636363636363636363636363636363636363636363","share_difficulty":101,"share_id":"share-1379","share_seq":1379,"template_height":800043},{"accepted_at_ms":999985,"credit_policy":null,"job_id":"job-3736","job_issued_at_ms":999990,"miner_id":"miner-7","network_difficulty":1000013,"ntime":1700003736,"order_key":"07:miner-7","p2mr_program_hex":"9898989898989898989898989898989898989898989898989898989898989898","share_difficulty":101,"share_id":"share-3736","share_seq":3736,"template_height":800116},{"accepted_at_ms":999973,"credit_policy":null,"job_id":"job-3241","job_issued_at_ms":999996,"miner_id":"miner-7","network_difficulty":1000011,"ntime":1700003241,"order_key":"07:miner-7","p2mr_program_hex":"a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9","share_difficulty":1000,"share_id":"share-3241","share_seq":3241,"template_height":800101},{"accepted_at_ms":999971,"credit_policy":null,"job_id":"job-1847","job_issued_at_ms":999993,"miner_id":"miner-10","network_difficulty":1000011,"ntime":1700001847,"order_key":"10:miner-10","p2mr_program_hex":"3737373737373737373737373737373737373737373737373737373737373737","share_difficulty":7,"share_id":"share-1847","share_seq":1847,"template_height":800057},{"accepted_at_ms":999992,"credit_policy":null,"job_id":"job-246","job_issued_at_ms":999985,"miner_id":"miner-4","network_difficulty":1000008,"ntime":1700000246,"order_key":"04:miner-4","p2mr_program_hex":"f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6","share_difficulty":2,"share_id":"share-246","share_seq":246,"template_height":800007},{"accepted_at_ms":999972,"credit_policy":"stale-grace","job_id":"job-1532","job_issued_at_ms":999995,"miner_id":"miner-3","network_difficulty":1000002,"ntime":1700001532,"order_key":"03:miner-3","p2mr_program_hex":"fcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfc","share_difficulty":3,"share_id":"share-1532","share_seq":1532,"template_height":800047},{"accepted_at_ms":999981,"credit_policy":"stale-grace","job_id":"job-2657","job_issued_at_ms":999987,"miner_id":"miner-6","network_difficulty":1000005,"ntime":1700002657,"order_key":"06:miner-6","p2mr_program_hex":"6161616161616161616161616161616161616161616161616161616161616161","share_difficulty":3,"share_id":"share-2657","share_seq":2657,"template_height":800083},{"accepted_at_ms":999970,"credit_policy":null,"job_id":"job-3471","job_issued_at_ms":999994,"miner_id":"miner-6","network_difficulty":1000003,"ntime":1700003471,"order_key":"06:miner-6","p2mr_program_hex":"8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f","share_difficulty":3,"share_id":"share-3471","share_seq":3471,"template_height":800108},{"accepted_at_ms":999977,"credit_policy":null,"job_id":"job-232","job_issued_at_ms":999977,"miner_id":"miner-1","network_difficulty":1000011,"ntime":1700000232,"order_key":"01:miner-1","p2mr_program_hex":"e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8","share_difficulty":2,"share_id":"share-232","share_seq":232,"template_height":800007},{"accepted_at_ms":999978,"credit_policy":null,"job_id":"job-1301","job_issued_at_ms":999981,"miner_id":"miner-3","network_difficulty":1000009,"ntime":1700001301,"order_key":"03:miner-3","p2mr_program_hex":"1515151515151515151515151515151515151515151515151515151515151515","share_difficulty":1000,"share_id":"share-1301","share_seq":1301,"template_height":800040},{"accepted_at_ms":999994,"credit_policy":null,"job_id":"job-2102","job_issued_at_ms":999996,"miner_id":"miner-1","network_difficulty":1000011,"ntime":1700002102,"order_key":"01:miner-1","p2mr_program_hex":"3636363636363636363636363636363636363636363636363636363636363636","share_difficulty":1000,"share_id":"share-2102","share_seq":2102,"template_height":800065},{"accepted_at_ms":999989,"credit_policy":"stale-grace","job_id":"job-3567","job_issued_at_ms":999986,"miner_id":"miner-3","network_difficulty":1000014,"ntime":1700003567,"order_key":"03:miner-3","p2mr_program_hex":"efefefefefefefefefefefefefefefefefefefefefefefefefefefefefefefef","share_difficulty":7,"share_id":"share-3567","share_seq":3567,"template_height":800111},{"accepted_at_ms":999974,"credit_policy":null,"job_id":"job-3108","job_issued_at_ms":999986,"miner_id":"miner-6","network_difficulty":1000014,"ntime":1700003108,"order_key":"06:miner-6","p2mr_program_hex":"2424242424242424242424242424242424242424242424242424242424242424","share_difficulty":101,"share_id":"share-3108","share_seq":3108,"template_height":800097},{"accepted_at_ms":999986,"credit_policy":null,"job_id":"job-2314","job_issued_at_ms":999994,"miner_id":"miner-4","network_difficulty":1000002,"ntime":1700002314,"order_key":"04:miner-4","p2mr_program_hex":"0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a","share_difficulty":3,"share_id":"share-2314","share_seq":2314,"template_height":800072},{"accepted_at_ms":999996,"credit_policy":null,"job_id":"job-1685","job_issued_at_ms":999986,"miner_id":"miner-2","network_difficulty":1000002,"ntime":1700001685,"order_key":"02:miner-2","p2mr_program_hex":"9595959595959595959595959595959595959595959595959595959595959595","share_difficulty":2,"share_id":"share-1685","share_seq":1685,"template_height":800052},{"accepted_at_ms":999981,"credit_policy":null,"job_id":"job-2578","job_issued_at_ms":999971,"miner_id":"miner-4","network_difficulty":1000011,"ntime":1700002578,"order_key":"04:miner-4","p2mr_program_hex":"1212121212121212121212121212121212121212121212121212121212121212","share_difficulty":3,"share_id":"share-2578","share_seq":2578,"template_height":800080},{"accepted_at_ms":999977,"credit_policy":null,"job_id":"job-1261","job_issued_at_ms":999994,"miner_id":"miner-7","network_difficulty":1000003,"ntime":1700001261,"order_key":"07:miner-7","p2mr_program_hex":"edededededededededededededededededededededededededededededededed","share_difficulty":19,"share_id":"share-1261","share_seq":1261,"template_height":800039},{"accepted_at_ms":999984,"credit_policy":"stale-grace","job_id":"job-3225","job_issued_at_ms":999990,"miner_id":"miner-2","network_difficulty":1000012,"ntime":1700003225,"order_key":"02:miner-2","p2mr_program_hex":"9999999999999999999999999999999999999999999999999999999999999999","share_difficulty":19,"share_id":"share-3225","share_seq":3225,"template_height":800100},{"accepted_at_ms":999989,"credit_policy":"stale-grace","job_id":"job-3052","job_issued_at_ms":999992,"miner_id":"miner-5","network_difficulty":1000009,"ntime":1700003052,"order_key":"05:miner-5","p2mr_program_hex":"ecececececececececececececececececececececececececececececececec","share_difficulty":7,"share_id":"share-3052","share_seq":3052,"template_height":800095},{"accepted_at_ms":999982,"credit_policy":null,"job_id":"job-3913","job_issued_at_ms":999981,"miner_id":"miner-8","network_difficulty":1000003,"ntime":1700003913,"order_key":"08:miner-8","p2mr_program_hex":"4949494949494949494949494949494949494949494949494949494949494949","share_difficulty":3,"share_id":"share-3913","share_seq":3913,"template_height":800122},{"accepted_at_ms":999977,"credit_policy":"stale-grace","job_id":"job-1304","job_issued_at_ms":999985,"miner_id":"miner-6","network_difficulty":1000012,"ntime":1700001304,"order_key":"06:miner-6","p2mr_program_hex":"1818181818181818181818181818181818181818181818181818181818181818","share_difficulty":3,"share_id":"share-1304","share_seq":1304,"template_height":800040},{"accepted_at_ms":999975,"credit_policy":null,"job_id":"job-1844","job_issued_at_ms":999970,"miner_id":"miner-7","network_difficulty":1000008,"ntime":1700001844,"order_key":"07:miner-7","p2mr_program_hex":"3434343434343434343434343434343434343434343434343434343434343434","share_difficulty":3,"share_id":"share-1844","share_seq":1844,"template_height":800057},{"accepted_at_ms":999998,"credit_policy":null,"job_id":"job-660","job_issued_at_ms":999988,"miner_id":"miner-0","network_difficulty":1000014,"ntime":1700000660,"order_key":"00:miner-0","p2mr_program_hex":"9494949494949494949494949494949494949494949494949494949494949494","share_difficulty":19,"share_id":"share-660","share_seq":660,"template_height":800020},{"accepted_at_ms":1000453,"credit_policy":"stale-grace","job_id":"job-3796","job_issued_at_ms":1000487,"miner_id":"miner-1","network_difficulty":1000005,"ntime":1700003796,"order_key":"01:miner-1","p2mr_program_hex":"d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4","share_difficulty":19,"share_id":"share-3796","share_seq":3796,"template_height":800118},{"accepted_at_ms":999996,"credit_policy":null,"job_id":"job-3912","job_issued_at_ms":999988,"miner_id":"miner-7","network_difficulty":1000002,"ntime":1700003912,"order_key":"07:miner-7","p2mr_program_hex":"4848484848484848484848484848484848484848484848484848484848484848","share_difficulty":19,"share_id":"share-3912","share_seq":3912,"template_height":800122},{"accepted_at_ms":999975,"credit_policy":null,"job_id":"job-623","job_issued_at_ms":999996,"miner_id":"miner-7","network_difficulty":1000011,"ntime":1700000623,"order_key":"07:miner-7","p2mr_program_hex":"6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f","share_difficulty":19,"share_id":"share-623","share_seq":623,"template_height":800019},{"accepted_at_ms":999989,"credit_policy":null,"job_id":"job-2612","job_issued_at_ms":999994,"miner_id":"miner-5","network_difficulty":1000011,"ntime":1700002612,"order_key":"05:miner-5","p2mr_program_hex":"3434343434343434343434343434343434343434343434343434343434343434","share_difficulty":7,"share_id":"share-2612","share_seq":2612,"template_height":800081},{"accepted_at_ms":999990,"credit_policy":null,"job_id":"job-1007","job_issued_at_ms":999981,"miner_id":"miner-6","network_difficulty":1000004,"ntime":1700001007,"order_key":"06:miner-6","p2mr_program_hex":"efefefefefefefefefefefefefefefefefefefefefefefefefefefefefefefef","share_difficulty":19,"share_id":"share-1007","share_seq":1007,"template_height":800031},{"accepted_at_ms":1000266,"credit_policy":"stale-grace","job_id":"job-3809","job_issued_at_ms":999986,"miner_id":"miner-3","network_difficulty":1000001,"ntime":1700003809,"order_key":"03:miner-3","p2mr_program_hex":"e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1","share_difficulty":2,"share_id":"share-3809","share_seq":3809,"template_height":800119},{"accepted_at_ms":999987,"credit_policy":null,"job_id":"job-3855","job_issued_at_ms":999999,"miner_id":"miner-5","network_difficulty":1000013,"ntime":1700003855,"order_key":"05:miner-5","p2mr_program_hex":"0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f","share_difficulty":7,"share_id":"share-3855","share_seq":3855,"template_height":800120},{"accepted_at_ms":999991,"credit_policy":null,"job_id":"job-1628","job_issued_at_ms":999991,"miner_id":"miner-0","network_difficulty":1000013,"ntime":1700001628,"order_key":"00:miner-0","p2mr_program_hex":"5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c","share_difficulty":101,"share_id":"share-1628","share_seq":1628,"template_height":800050},{"accepted_at_ms":999971,"credit_policy":null,"job_id":"job-2549","job_issued_at_ms":1000231,"miner_id":"miner-8","network_difficulty":1000016,"ntime":1700002549,"order_key":"08:miner-8","p2mr_program_hex":"f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5","share_difficulty":1000,"share_id":"share-2549","share_seq":2549,"template_height":800079},{"accepted_at_ms":999978,"credit_policy":"stale-grace","job_id":"job-3417","job_issued_at_ms":999990,"miner_id":"miner-7","network_difficulty":1000000,"ntime":1700003417,"order_key":"07:miner-7","p2mr_program_hex":"5959595959595959595959595959595959595959595959595959595959595959","share_difficulty":1000,"share_id":"share-3417","share_seq":3417,"template_height":800106},{"accepted_at_ms":999986,"credit_policy":null,"job_id":"job-3384","job_issued_at_ms":1000155,"miner_id":"miner-7","network_difficulty":1000001,"ntime":1700003384,"order_key":"07:miner-7","p2mr_program_hex":"3838383838383838383838383838383838383838383838383838383838383838","share_difficulty":101,"share_id":"share-3384","share_seq":3384,"template_height":800105},{"accepted_at_ms":999981,"credit_policy":null,"job_id":"job-264","job_issued_at_ms":999989,"miner_id":"miner-0","network_difficulty":1000009,"ntime":1700000264,"order_key":"00:miner-0","p2mr_program_hex":"0808080808080808080808080808080808080808080808080808080808080808","share_difficulty":2,"share_id":"share-264","share_seq":264,"template_height":800008},{"accepted_at_ms":999990,"credit_policy":"stale-grace","job_id":"job-3144","job_issued_at_ms":999979,"miner_id":"miner-9","network_difficulty":1000016,"ntime":1700003144,"order_key":"09:miner-9","p2mr_program_hex":"4848484848484848484848484848484848484848484848484848484848484848","share_difficulty":3,"share_id":"share-3144","share_seq":3144,"template_height":800098},{"accepted_at_ms":999974,"credit_policy":null,"job_id":"job-2681","job_issued_at_ms":999987,"miner_id":"miner-8","network_difficulty":1000012,"ntime":1700002681,"order_key":"08:miner-8","p2mr_program_hex":"7979797979797979797979797979797979797979797979797979797979797979","share_difficulty":3,"share_id":"share-2681","share_seq":2681,"template_height":800083},{"accepted_at_ms":999993,"credit_policy":null,"job_id":"job-3566","job_issued_at_ms":999996,"miner_id":"miner-2","network_difficulty":1000013,"ntime":1700003566,"order_key":"02:miner-2","p2mr_program_hex":"eeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeee","share_difficulty":1000,"share_id":"share-3566","share_seq":3566,"template_height":800111},{"accepted_at_ms":1000315,"credit_policy":null,"job_id":"job-762","job_issued_at_ms":999986,"miner_id":"miner-3","network_difficulty":1000014,"ntime":1700000762,"order_key":"03:miner-3","p2mr_program_hex":"fafafafafafafafafafafafafafafafafafafafafafafafafafafafafafafafa","share_difficulty":7,"share_id":"share-762","share_seq":762,"template_height":800023},{"accepted_at_ms":999982,"credit_policy":"stale-grace","job_id":"job-1571","job_issued_at_ms":999983,"miner_id":"miner-9","network_difficulty":1000007,"ntime":1700001571,"order_key":"09:miner-9","p2mr_program_hex":"2323232323232323232323232323232323232323232323232323232323232323","share_difficulty":7,"share_id":"share-1571","share_seq":1571,"template_height":800049},{"accepted_at_ms":999988,"credit_policy":null,"job_id":"job-3390","job_issued_at_ms":999996,"miner_id":"miner-2","network_difficulty":1000007,"ntime":1700003390,"order_key":"02:miner-2","p2mr_program_hex":"3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e","share_difficulty":2,"share_id":"share-3390","share_seq":3390,"template_height":800105},{"accepted_at_ms":999997,"credit_policy":null,"job_id":"job-3638","job_issued_at_ms":999977,"miner_id":"miner-8","network_difficulty":1000000,"ntime":1700003638,"order_key":"08:miner-8","p2mr_program_hex":"3636363636363636363636363636363636363636363636363636363636363636","share_difficulty":1000,"share_id":"share-3638","share_seq":3638,"template_height":800113},{"accepted_at_ms":999997,"credit_policy":null,"job_id":"job-3088","job_issued_at_ms":999986,"miner_id":"miner-8","network_difficulty":1000011,"ntime":1700003088,"order_key":"08:miner-8","p2mr_program_hex":"1010101010101010101010101010101010101010101010101010101010101010","share_difficulty":3,"share_id":"share-3088","share_seq":3088,"template_height":800096},{"accepted_at_ms":999985,"credit_policy":null,"job_id":"job-2186","job_issued_at_ms":999995,"miner_id":"miner-8","network_difficulty":1000010,"ntime":1700002186,"order_key":"08:miner-8","p2mr_program_hex":"8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a","share_difficulty":3,"share_id":"share-2186","share_seq":2186,"template_height":800068},{"accepted_at_ms":999975,"credit_policy":null,"job_id":"job-974","job_issued_at_ms":999998,"miner_id":"miner-6","network_difficulty":1000005,"ntime":1700000974,"order_key":"06:miner-6","p2mr_program_hex":"cececececececececececececececececececececececececececececececece","share_difficulty":2,"share_id":"share-974","share_seq":974,"template_height":800030},{"accepted_at_ms":999972,"credit_policy":null,"job_id":"job-1157","job_issued_at_ms":999980,"miner_id":"miner-2","network_difficulty":1000001,"ntime":1700001157,"order_key":"02:miner-2","p2mr_program_hex":"8585858585858585858585858585858585858585858585858585858585858585","share_difficulty":7,"share_id":"share-1157","share_seq":1157,"template_height":800036},{"accepted_at_ms":999978,"credit_policy":null,"job_id":"job-80","job_issued_at_ms":999993,"miner_id":"miner-3","network_difficulty":1000012,"ntime":1700000080,"order_key":"03:miner-3","p2mr_program_hex":"5050505050505050505050505050505050505050505050505050505050505050","share_difficulty":3,"share_id":"share-80","share_seq":80,"template_height":800002},{"accepted_at_ms":999985,"credit_policy":null,"job_id":"job-3738","job_issued_at_ms":999978,"miner_id":"miner-9","network_difficulty":1000015,"ntime":1700003738,"order_key":"09:miner-9","p2mr_program_hex":"9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a","share_difficulty":2,"share_id":"share-3738","share_seq":3738,"template_height":800116},{"accepted_at_ms":1000455,"credit_policy":null,"job_id":"job-3456","job_issued_at_ms":1000047,"miner_id":"miner-2","network_difficulty":1000005,"ntime":1700003456,"order_key":"02:miner-2","p2mr_program_hex":"8080808080808080808080808080808080808080808080808080808080808080","share_difficulty":19,"share_id":"share-3456","share_seq":3456,"template_height":800108},{"accepted_at_ms":999990,"credit_policy":null,"job_id":"job-3342","job_issued_at_ms":999993,"miner_id":"miner-9","network_difficulty":1000010,"ntime":1700003342,"order_key":"09:miner-9","p2mr_program_hex":"0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e","share_difficulty":19,"share_id":"share-3342","share_seq":3342,"template_height":800104},{"accepted_at_ms":999999,"credit_policy":null,"job_id":"job-3875","job_issued_at_ms":999983,"miner_id":"miner-3","network_difficulty":1000016,"ntime":1700003875,"order_key":"03:miner-3","p2mr_program_hex":"2323232323232323232323232323232323232323232323232323232323232323","share_difficulty":19,"share_id":"share-3875","share_seq":3875,"template_height":800121},{"accepted_at_ms":999988,"credit_policy":null,"job_id":"job-741","job_issued_at_ms":999973,"miner_id":"miner-4","network_difficulty":1000010,"ntime":1700000741,"order_key":"04:miner-4","p2mr_program_hex":"e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5","share_difficulty":101,"share_id":"share-741","share_seq":741,"template_height":800023},{"accepted_at_ms":999985,"credit_policy":null,"job_id":"job-2714","job_issued_at_ms":999983,"miner_id":"miner-8","network_difficulty":1000011,"ntime":1700002714,"order_key":"08:miner-8","p2mr_program_hex":"9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a","share_difficulty":2,"share_id":"share-2714","share_seq":2714,"template_height":800084},{"accepted_at_ms":999972,"credit_policy":null,"job_id":"job-3797","job_issued_at_ms":999999,"miner_id":"miner-2","network_difficulty":1000006,"ntime":1700003797,"order_key":"02:miner-2","p2mr_program_hex":"d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5","share_difficulty":1,"share_id":"share-3797","share_seq":3797,"template_height":800118},{"accepted_at_ms":999990,"credit_policy":null,"job_id":"job-937","job_issued_at_ms":999983,"miner_id":"miner-2","network_difficulty":1000002,"ntime":1700000937,"order_key":"02:miner-2","p2mr_program_hex":"a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9","share_difficulty":1,"share_id":"share-937","share_seq":937,"template_height":800029},{"accepted_at_ms":999992,"credit_policy":null,"job_id":"job-1323","job_issued_at_ms":999986,"miner_id":"miner-3","network_difficulty":1000014,"ntime":1700001323,"order_key":"03:miner-3","p2mr_program_hex":"2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b","share_difficulty":1,"share_id":"share-1323","share_seq":1323,"template_height":800041},{"accepted_at_ms":999982,"credit_policy":"stale-grace","job_id":"job-2280","job_issued_at_ms":999999,"miner_id":"miner-3","network_difficulty":1000002,"ntime":1700002280,"order_key":"03:miner-3","p2mr_program_hex":"e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8","share_difficulty":1000,"share_id":"share-2280","share_seq":2280,"template_height":800071},{"accepted_at_ms":999982,"credit_policy":null,"job_id":"job-819","job_issued_at_ms":999993,"miner_id":"miner-5","network_difficulty":1000003,"ntime":1700000819,"order_key":"05:miner-5","p2mr_program_hex":"3333333333333333333333333333333333333333333333333333333333333333","share_difficulty":19,"share_id":"share-819","share_seq":819,"template_height":800025},{"accepted_at_ms":999994,"credit_policy":null,"job_id":"job-3532","job_issued_at_ms":999985,"miner_id":"miner-1","network_difficulty":1000013,"ntime":1700003532,"order_key":"01:miner-1","p2mr_program_hex":"cccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccc","share_difficulty":1,"share_id":"share-3532","share_seq":3532,"template_height":800110},{"accepted_at_ms":999973,"credit_policy":null,"job_id":"job-2361","job_issued_at_ms":999993,"miner_id":"miner-7","network_difficulty":1000015,"ntime":1700002361,"order_key":"07:miner-7","p2mr_program_hex":"3939393939393939393939393939393939393939393939393939393939393939","share_difficulty":101,"share_id":"share-2361","share_seq":2361,"template_height":800073},{"accepted_at_ms":999987,"credit_policy":null,"job_id":"job-2848","job_issued_at_ms":999999,"miner_id":"miner-10","network_difficulty":1000009,"ntime":1700002848,"order_key":"10:miner-10","p2mr_program_hex":"2020202020202020202020202020202020202020202020202020202020202020","share_difficulty":3,"share_id":"share-2848","share_seq":2848,"template_height":800089},{"accepted_at_ms":999994,"credit_policy":null,"job_id":"job-1111","job_issued_at_ms":999988,"miner_id":"miner-0","network_difficulty":1000006,"ntime":1700001111,"order_key":"00:miner-0","p2mr_program_hex":"5757575757575757575757575757575757575757575757575757575757575757","share_difficulty":1000,"share_id":"share-1111","share_seq":1111,"template_height":800034},{"accepted_at_ms":999985,"credit_policy":null,"job_id":"job-363","job_issued_at_ms":1000000,"miner_id":"miner-0","network_difficulty":1000006,"ntime":1700000363,"order_key":"00:miner-0","p2mr_program_hex":"6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b","share_difficulty":1,"share_id":"share-363","share_seq":363,"template_height":800011},{"accepted_at_ms":999980,"credit_policy":null,"job_id":"job-3751","job_issued_at_ms":1000255,"miner_id":"miner-0","network_difficulty":1000011,"ntime":1700003751,"order_key":"00:miner-0","p2mr_program_hex":"a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7","share_difficulty":3,"share_id":"share-3751","share_seq":3751,"template_height":800117},{"accepted_at_ms":999982,"credit_policy":null,"job_id":"job-3549","job_issued_at_ms":999994,"miner_id":"miner-7","network_difficulty":1000013,"ntime":1700003549,"order_key":"07:miner-7","p2mr_program_hex":"dddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddd","share_difficulty":1000,"share_id":"share-3549","share_seq":3549,"template_height":800110},{"accepted_at_ms":999970,"credit_policy":null,"job_id":"job-3881","job_issued_at_ms":999980,"miner_id":"miner-9","network_difficulty":1000005,"ntime":1700003881,"order_key":"09:miner-9","p2mr_program_hex":"2929292929292929292929292929292929292929292929292929292929292929","share_difficulty":1000,"share_id":"share-3881","share_seq":3881,"template_height":800121},{"accepted_at_ms":999981,"credit_policy":null,"job_id":"job-122","job_issued_at_ms":999998,"miner_id":"miner-1","network_difficulty":1000003,"ntime":1700000122,"order_key":"01:miner-1","p2mr_program_hex":"7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a","share_difficulty":101,"share_id":"share-122","share_seq":122,"template_height":800003},{"accepted_at_ms":999994,"credit_policy":"stale-grace","job_id":"job-2091","job_issued_at_ms":999979,"miner_id":"miner-1","network_difficulty":1000000,"ntime":1700002091,"order_key":"01:miner-1","p2mr_program_hex":"2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b","share_difficulty":7,"share_id":"share-2091","share_seq":2091,"template_height":800065},{"accepted_at_ms":999996,"credit_policy":null,"job_id":"job-3443","job_issued_at_ms":999987,"miner_id":"miner-0","network_difficulty":1000009,"ntime":1700003443,"order_key":"00:miner-0","p2mr_program_hex":"7373737373737373737373737373737373737373737373737373737373737373","share_difficulty":1,"share_id":"share-3443","share_seq":3443,"template_height":800107},{"accepted_at_ms":999988,"credit_policy":null,"job_id":"job-3146","job_issued_at_ms":999994,"miner_id":"miner-0","network_difficulty":1000001,"ntime":1700003146,"order_key":"00:miner-0","p2mr_program_hex":"4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a","share_difficulty":101,"share_id":"share-3146","share_seq":3146,"template_height":800098},{"accepted_at_ms":999989,"credit_policy":null,"job_id":"job-1580","job_issued_at_ms":999971,"miner_id":"miner-7","network_difficulty":1000016,"ntime":1700001580,"order_key":"07:miner-7","p2mr_program_hex":"2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c","share_difficulty":7,"share_id":"share-1580","share_seq":1580,"template_height":800049},{"accepted_at_ms":999983,"credit_policy":null,"job_id":"job-3675","job_issued_at_ms":999977,"miner_id":"miner-1","network_difficulty":1000003,"ntime":1700003675,"order_key":"01:miner-1","p2mr_program_hex":"5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b","share_difficulty":1000,"share_id":"share-3675","share_seq":3675,"template_height":800114},{"accepted_at_ms":999982,"credit_policy":null,"job_id":"job-2713","job_issued_at_ms":999996,"miner_id":"miner-7","network_difficulty":1000010,"ntime":1700002713,"order_key":"07:miner-7","p2mr_program_hex":"9999999999999999999999999999999999999999999999999999999999999999","share_difficulty":7,"share_id":"share-2713","share_seq":2713,"template_height":800084},{"accepted_at_ms":999970,"credit_policy":null,"job_id":"job-1494","job_issued_at_ms":999974,"miner_id":"miner-9","network_difficulty":1000015,"ntime":1700001494,"order_key":"09:miner-9","p2mr_program_hex":"d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6","share_difficulty":19,"share_id":"share-1494","share_seq":1494,"template_height":800046},{"accepted_at_ms":999979,"credit_policy":null,"job_id":"job-3454","job_issued_at_ms":999977,"miner_id":"miner-0","network_difficulty":1000003,"ntime":1700003454,"order_key":"00:miner-0","p2mr_program_hex":"7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e","share_difficulty":101,"share_id":"share-3454","share_seq":3454,"template_height":800107},{"accepted_at_ms":999978,"credit_policy":null,"job_id":"job-1624","job_issued_at_ms":999988,"miner_id":"miner-7","network_difficulty":1000009,"ntime":1700001624,"order_key":"07:miner-7","p2mr_program_hex":"5858585858585858585858585858585858585858585858585858585858585858","share_difficulty":1,"share_id":"share-1624","share_seq":1624,"template_height":800050},{"accepted_at_ms":1000110,"credit_policy":null,"job_id":"job-1828","job_issued_at_ms":999970,"miner_id":"miner-2","network_difficulty":1000009,"ntime":1700001828,"order_key":"02:miner-2","p2mr_program_hex":"2424242424242424242424242424242424242424242424242424242424242424","share_difficulty":3,"share_id":"share-1828","share_seq":1828,"template_height":800057},{"accepted_at_ms":999970,"credit_policy":null,"job_id":"job-289","job_issued_at_ms":999983,"miner_id":"miner-3","network_difficulty":1000000,"ntime":1700000289,"order_key":"03:miner-3","p2mr_program_hex":"2121212121212121212121212121212121212121212121212121212121212121","share_difficulty":3,"share_id":"share-289","share_seq":289,"template_height":800009},{"accepted_at_ms":999998,"credit_policy":"stale-grace","job_id":"job-1188","job_issued_at_ms":999999,"miner_id":"miner-0","network_difficulty":1000015,"ntime":1700001188,"order_key":"00:miner-0","p2mr_program_hex":"a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4","share_difficulty":1000,"share_id":"share-1188","share_seq":1188,"template_height":800037},{"accepted_at_ms":999975,"credit_policy":"stale-grace","job_id":"job-901","job_issued_at_ms":999989,"miner_id":"miner-10","network_difficulty":1000000,"ntime":1700000901,"order_key":"10:miner-10","p2mr_program_hex":"8585858585858585858585858585858585858585858585858585858585858585","share_difficulty":101,"share_id":"share-901","share_seq":901,"template_height":800028},{"accepted_at_ms":999992,"credit_policy":"stale-grace","job_id":"job-2225","job_issued_at_ms":999986,"miner_id":"miner-3","network_difficulty":1000015,"ntime":1700002225,"order_key":"03:miner-3","p2mr_program_hex":"b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1","share_difficulty":1,"share_id":"share-2225","share_seq":2225,"template_height":800069},{"accepted_at_ms":999986,"credit_policy":null,"job_id":"job-1278","job_issued_at_ms":999978,"miner_id":"miner-2","network_difficulty":1000003,"ntime":1700001278,"order_key":"02:miner-2","p2mr_program_hex":"fefefefefefefefefefefefefefefefefefefefefefefefefefefefefefefefe","share_difficulty":3,"share_id":"share-1278","share_seq":1278,"template_height":800039},{"accepted_at_ms":999993,"credit_policy":null,"job_id":"job-795","job_issued_at_ms":999994,"miner_id":"miner-3","network_difficulty":1000013,"ntime":1700000795,"order_key":"03:miner-3","p2mr_program_hex":"1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b","share_difficulty":2,"share_id":"share-795","share_seq":795,"template_height":800024},{"accepted_at_ms":999987,"credit_policy":"stale-grace","job_id":"job-842","job_issued_at_ms":999993,"miner_id":"miner-6","network_difficulty":1000009,"ntime":1700000842,"order_key":"06:miner-6","p2mr_program_hex":"4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a","share_difficulty":1000,"share_id":"share-842","share_seq":842,"template_height":800026},{"accepted_at_ms":999976,"credit_policy":"stale-grace","job_id":"job-3747","job_issued_at_ms":999993,"miner_id":"miner-7","network_difficulty":1000007,"ntime":1700003747,"order_key":"07:miner-7","p2mr_program_hex":"a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3","share_difficulty":7,"share_id":"share-3747","share_seq":3747,"template_height":800117},{"accepted_at_ms":999970,"credit_policy":null,"job_id":"job-692","job_issued_at_ms":999979,"miner_id":"miner-10","network_difficulty":1000012,"ntime":1700000692,"order_key":"10:miner-10","p2mr_program_hex":"b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4","share_difficulty":2,"share_id":"share-692","share_seq":692,"template_height":800021},{"accepted_at_ms":999991,"credit_policy":null,"job_id":"job-1482","job_issued_at_ms":999974,"miner_id":"miner-8","network_difficulty":1000003,"ntime":1700001482,"order_key":"08:miner-8","p2mr_program_hex":"cacacacacacacacacacacacacacacacacacacacacacacacacacacacacacacaca","share_difficulty":19,"share_id":"share-1482","share_seq":1482,"template_height":800046},{"accepted_at_ms":999994,"credit_policy":null,"job_id":"job-3300","job_issued_at_ms":999978,"miner_id":"miner-0","network_difficulty":1000002,"ntime":1700003300,"order_key":"00:miner-0","p2mr_program_hex":"e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4","share_difficulty":19,"share_id":"share-3300","share_seq":3300,"template_height":800103},{"accepted_at_ms":1000108,"credit_policy":null,"job_id":"job-863","job_issued_at_ms":999999,"miner_id":"miner-5","network_difficulty":1000013,"ntime":1700000863,"order_key":"05:miner-5","p2mr_program_hex":"5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f","share_difficulty":1000,"share_id":"share-863","share_seq":863,"template_height":800026},{"accepted_at_ms":999995,"credit_policy":null,"job_id":"job-509","job_issued_at_ms":999982,"miner_id":"miner-3","network_difficulty":1000016,"ntime":1700000509,"order_key":"03:miner-3","p2mr_program_hex":"fdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfd","share_difficulty":1,"share_id":"share-509","share_seq":509,"template_height":800015},{"accepted_at_ms":999990,"credit_policy":null,"job_id":"job-2823","job_issued_at_ms":999972,"miner_id":"miner-7","network_difficulty":1000001,"ntime":1700002823,"order_key":"07:miner-7","p2mr_program_hex":"0707070707070707070707070707070707070707070707070707070707070707","share_difficulty":19,"share_id":"share-2823","share_seq":2823,"template_height":800088},{"accepted_at_ms":999982,"credit_policy":"stale-grace","job_id":"job-569","job_issued_at_ms":999989,"miner_id":"miner-8","network_difficulty":1000008,"ntime":1700000569,"order_key":"08:miner-8","p2mr_program_hex":"3939393939393939393939393939393939393939393939393939393939393939","share_difficulty":2,"share_id":"share-569","share_seq":569,"template_height":800017},{"accepted_at_ms":999988,"credit_policy":null,"job_id":"job-1622","job_issued_at_ms":1000494,"miner_id":"miner-5","network_difficulty":1000007,"ntime":1700001622,"order_key":"05:miner-5","p2mr_program_hex":"5656565656565656565656565656565656565656565656565656565656565656","share_difficulty":2,"share_id":"share-1622","share_seq":1622,"template_height":800050},{"accepted_at_ms":999976,"credit_policy":null,"job_id":"job-2474","job_issued_at_ms":999986,"miner_id":"miner-10","network_difficulty":1000009,"ntime":1700002474,"order_key":"10:miner-10","p2mr_program_hex":"aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa","share_difficulty":2,"share_id":"share-2474","share_seq":2474,"template_height":800077},{"accepted_at_ms":999980,"credit_policy":null,"job_id":"job-2609","job_issued_at_ms":999999,"miner_id":"miner-2","network_difficulty":1000008,"ntime":1700002609,"order_key":"02:miner-2","p2mr_program_hex":"3131313131313131313131313131313131313131313131313131313131313131","share_difficulty":1,"share_id":"share-2609","share_seq":2609,"template_height":800081},{"accepted_at_ms":1000000,"credit_policy":null,"job_id":"job-2312","job_issued_at_ms":999970,"miner_id":"miner-2","network_difficulty":1000000,"ntime":1700002312,"order_key":"02:miner-2","p2mr_program_hex":"0808080808080808080808080808080808080808080808080808080808080808","share_difficulty":19,"share_id":"share-2312","share_seq":2312,"template_height":800072},{"accepted_at_ms":999983,"credit_policy":"stale-grace","job_id":"job-2114","job_issued_at_ms":999991,"miner_id":"miner-2","network_difficulty":1000006,"ntime":1700002114,"order_key":"02:miner-2","p2mr_program_hex":"4242424242424242424242424242424242424242424242424242424242424242","share_difficulty":1000,"share_id":"share-2114","share_seq":2114,"template_height":800066},{"accepted_at_ms":999982,"credit_policy":null,"job_id":"job-1910","job_issued_at_ms":999978,"miner_id":"miner-7","network_difficulty":1000006,"ntime":1700001910,"order_key":"07:miner-7","p2mr_program_hex":"7676767676767676767676767676767676767676767676767676767676767676","share_difficulty":7,"share_id":"share-1910","share_seq":1910,"template_height":800059},{"accepted_at_ms":999976,"credit_policy":null,"job_id":"job-3187","job_issued_at_ms":999997,"miner_id":"miner-8","network_difficulty":1000008,"ntime":1700003187,"order_key":"08:miner-8","p2mr_program_hex":"7373737373737373737373737373737373737373737373737373737373737373","share_difficulty":7,"share_id":"share-3187","share_seq":3187,"template_height":800099},{"accepted_at_ms":999970,"credit_policy":null,"job_id":"job-219","job_issued_at_ms":999971,"miner_id":"miner-10","network_difficulty":1000015,"ntime":1700000219,"order_key":"10:miner-10","p2mr_program_hex":"dbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdb","share_difficulty":1000,"share_id":"share-219","share_seq":219,"template_height":800006},{"accepted_at_ms":999992,"credit_policy":null,"job_id":"job-523","job_issued_at_ms":999994,"miner_id":"miner-6","network_difficulty":1000013,"ntime":1700000523,"order_key":"06:miner-6","p2mr_program_hex":"0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b","share_difficulty":101,"share_id":"share-523","share_seq":523,"template_height":800016},{"accepted_at_ms":1000396,"credit_policy":null,"job_id":"job-835","job_issued_at_ms":999970,"miner_id":"miner-10","network_difficulty":1000002,"ntime":1700000835,"order_key":"10:miner-10","p2mr_program_hex":"4343434343434343434343434343434343434343434343434343434343434343","share_difficulty":1,"share_id":"share-835","share_seq":835,"template_height":800026},{"accepted_at_ms":999988,"credit_policy":"stale-grace","job_id":"job-1670","job_issued_at_ms":999978,"miner_id":"miner-9","network_difficulty":1000004,"ntime":1700001670,"order_key":"09:miner-9","p2mr_program_hex":"8686868686868686868686868686868686868686868686868686868686868686","share_difficulty":101,"share_id":"share-1670","share_seq":1670,"template_height":800052},{"accepted_at_ms":999997,"credit_policy":null,"job_id":"job-1080","job_issued_at_ms":999972,"miner_id":"miner-2","network_difficulty":1000009,"ntime":1700001080,"order_key":"02:miner-2","p2mr_program_hex":"3838383838383838383838383838383838383838383838383838383838383838","share_difficulty":7,"share_id":"share-1080","share_seq":1080,"template_height":800033},{"accepted_at_ms":999984,"credit_policy":null,"job_id":"job-858","job_issued_at_ms":999979,"miner_id":"miner-0","network_difficulty":1000008,"ntime":1700000858,"order_key":"00:miner-0","p2mr_program_hex":"5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a","share_difficulty":3,"share_id":"share-858","share_seq":858,"template_height":800026},{"accepted_at_ms":999970,"credit_policy":"stale-grace","job_id":"job-1700","job_issued_at_ms":999996,"miner_id":"miner-6","network_difficulty":1000000,"ntime":1700001700,"order_key":"06:miner-6","p2mr_program_hex":"a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4","share_difficulty":19,"share_id":"share-1700","share_seq":1700,"template_height":800053},{"accepted_at_ms":999986,"credit_policy":null,"job_id":"job-2579","job_issued_at_ms":999979,"miner_id":"miner-5","network_difficulty":1000012,"ntime":1700002579,"order_key":"05:miner-5","p2mr_program_hex":"1313131313131313131313131313131313131313131313131313131313131313","share_difficulty":2,"share_id":"share-2579","share_seq":2579,"template_height":800080},{"accepted_at_ms":999984,"credit_policy":null,"job_id":"job-648","job_issued_at_ms":999997,"miner_id":"miner-10","network_difficulty":1000002,"ntime":1700000648,"order_key":"10:miner-10","p2mr_program_hex":"8888888888888888888888888888888888888888888888888888888888888888","share_difficulty":101,"share_id":"share-648","share_seq":648,"template_height":800020},{"accepted_at_ms":999979,"credit_policy":null,"job_id":"job-2358","job_issued_at_ms":999988,"miner_id":"miner-4","network_difficulty":1000012,"ntime":1700002358,"order_key":"04:miner-4","p2mr_program_hex":"3636363636363636363636363636363636363636363636363636363636363636","share_difficulty":7,"share_id":"share-2358","share_seq":2358,"template_height":800073},{"accepted_at_ms":999997,"credit_policy":null,"job_id":"job-2906","job_issued_at_ms":999976,"miner_id":"miner-2","network_difficulty":1000016,"ntime":1700002906,"order_key":"02:miner-2","p2mr_program_hex":"5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a","share_difficulty":7,"share_id":"share-2906","share_seq":2906,"template_height":800090},{"accepted_at_ms":999986,"credit_policy":null,"job_id":"job-1610","job_issued_at_ms":999984,"miner_id":"miner-4","network_difficulty":1000012,"ntime":1700001610,"order_key":"04:miner-4","p2mr_program_hex":"4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a","share_difficulty":1000,"share_id":"share-1610","share_seq":1610,"template_height":800050},{"accepted_at_ms":1000121,"credit_policy":null,"job_id":"job-2807","job_issued_at_ms":999997,"miner_id":"miner-2","network_difficulty":1000002,"ntime":1700002807,"order_key":"02:miner-2","p2mr_program_hex":"f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7","share_difficulty":101,"share_id":"share-2807","share_seq":2807,"template_height":800087},{"accepted_at_ms":1000082,"credit_policy":null,"job_id":"job-1718","job_issued_at_ms":999993,"miner_id":"miner-2","network_difficulty":1000001,"ntime":1700001718,"order_key":"02:miner-2","p2mr_program_hex":"b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6","share_difficulty":101,"share_id":"share-1718","share_seq":1718,"template_height":800053},{"accepted_at_ms":999996,"credit_policy":null,"job_id":"job-2294","job_issued_at_ms":999987,"miner_id":"miner-6","network_difficulty":1000016,"ntime":1700002294,"order_key":"06:miner-6","p2mr_program_hex":"f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6","share_difficulty":7,"share_id":"share-2294","share_seq":2294,"template_height":800071},{"accepted_at_ms":999979,"credit_policy":null,"job_id":"job-1579","job_issued_at_ms":999991,"miner_id":"miner-6","network_difficulty":1000015,"ntime":1700001579,"order_key":"06:miner-6","p2mr_program_hex":"2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b","share_difficulty":2,"share_id":"share-1579","share_seq":1579,"template_height":800049},{"accepted_at_ms":999988,"credit_policy":null,"job_id":"job-3709","job_issued_at_ms":999978,"miner_id":"miner-2","network_difficulty":1000003,"ntime":1700003709,"order_key":"02:miner-2","p2mr_program_hex":"7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d","share_difficulty":1000,"share_id":"share-3709","share_seq":3709,"template_height":800115},{"accepted_at_ms":999980,"credit_policy":null,"job_id":"job-3492","job_issued_at_ms":999971,"miner_id":"miner-5","network_difficulty":1000007,"ntime":1700003492,"order_key":"05:miner-5","p2mr_program_hex":"a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4","share_difficulty":101,"share_id":"share-3492","share_seq":3492,"template_height":800109},{"accepted_at_ms":999975,"credit_policy":null,"job_id":"job-3708","job_issued_at_ms":999986,"miner_id":"miner-1","network_difficulty":1000002,"ntime":1700003708,"order_key":"01:miner-1","p2mr_program_hex":"7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c","share_difficulty":2,"share_id":"share-3708","share_seq":3708,"template_height":800115},{"accepted_at_ms":999997,"credit_policy":null,"job_id":"job-562","job_issued_at_ms":999979,"miner_id":"miner-1","network_difficulty":1000001,"ntime":1700000562,"order_key":"01:miner-1","p2mr_program_hex":"3232323232323232323232323232323232323232323232323232323232323232","share_difficulty":19,"share_id":"share-562","share_seq":562,"template_height":800017},{"accepted_at_ms":999985,"credit_policy":null,"job_id":"job-2178","job_issued_at_ms":999978,"miner_id":"miner-0","network_difficulty":1000002,"ntime":1700002178,"order_key":"00:miner-0","p2mr_program_hex":"8282828282828282828282828282828282828282828282828282828282828282","share_difficulty":2,"share_id":"share-2178","share_seq":2178,"template_height":800068},{"accepted_at_ms":999995,"credit_policy":null,"job_id":"job-3240","job_issued_at_ms":1000000,"miner_id":"miner-6","network_difficulty":1000010,"ntime":1700003240,"order_key":"06:miner-6","p2mr_program_hex":"a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8","share_difficulty":1,"share_id":"share-3240","share_seq":3240,"template_height":800101},{"accepted_at_ms":999981,"credit_policy":null,"job_id":"job-2812","job_issued_at_ms":999984,"miner_id":"miner-7","network_difficulty":1000007,"ntime":1700002812,"order_key":"07:miner-7","p2mr_program_hex":"fcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfc","share_difficulty":1000,"share_id":"share-2812","share_seq":2812,"template_height":800087},{"accepted_at_ms":999991,"credit_policy":"stale-grace","job_id":"job-1958","job_issued_at_ms":999979,"miner_id":"miner-0","network_difficulty":1000003,"ntime":1700001958,"order_key":"00:miner-0","p2mr_program_hex":"a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6","share_difficulty":3,"share_id":"share-1958","share_seq":1958,"template_height":800061},{"accepted_at_ms":999986,"credit_policy":null,"job_id":"job-2135","job_issued_at_ms":999996,"miner_id":"miner-1","network_difficulty":1000010,"ntime":1700002135,"order_key":"01:miner-1","p2mr_program_hex":"5757575757575757575757575757575757575757575757575757575757575757","share_difficulty":101,"share_id":"share-2135","share_seq":2135,"template_height":800066},{"accepted_at_ms":999970,"credit_policy":"stale-grace","job_id":"job-1256","job_issued_at_ms":999995,"miner_id":"miner-2","network_difficulty":1000015,"ntime":1700001256,"order_key":"02:miner-2","p2mr_program_hex":"e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8","share_difficulty":19,"share_id":"share-1256","share_seq":1256,"template_height":800039},{"accepted_at_ms":999997,"credit_policy":null,"job_id":"job-2926","job_issued_at_ms":999974,"miner_id":"miner-0","network_difficulty":1000002,"ntime":1700002926,"order_key":"00:miner-0","p2mr_program_hex":"6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e","share_difficulty":2,"share_id":"share-2926","share_seq":2926,"template_height":800091},{"accepted_at_ms":999999,"credit_policy":null,"job_id":"job-1756","job_issued_at_ms":999970,"miner_id":"miner-7","network_difficulty":1000005,"ntime":1700001756,"order_key":"07:miner-7","p2mr_program_hex":"dcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdc","share_difficulty":3,"share_id":"share-1756","share_seq":1756,"template_height":800054},{"accepted_at_ms":999973,"credit_policy":null,"job_id":"job-2596","job_issued_at_ms":999999,"miner_id":"miner-0","network_difficulty":1000012,"ntime":1700002596,"order_key":"00:miner-0","p2mr_program_hex":"2424242424242424242424242424242424242424242424242424242424242424","share_difficulty":2,"share_id":"share-2596","share_seq":2596,"template_height":800081},{"accepted_at_ms":999984,"credit_policy":null,"job_id":"job-3070","job_issued_at_ms":999980,"miner_id":"miner-1","network_difficulty":1000010,"ntime":1700003070,"order_key":"01:miner-1","p2mr_program_hex":"fefefefefefefefefefefefefefefefefefefefefefefefefefefefefefefefe","share_difficulty":1,"share_id":"share-3070","share_seq":3070,"template_height":800095},{"accepted_at_ms":999987,"credit_policy":null,"job_id":"job-980","job_issued_at_ms":999989,"miner_id":"miner-1","network_difficulty":1000011,"ntime":1700000980,"order_key":"01:miner-1","p2mr_program_hex":"d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4","share_difficulty":1000,"share_id":"share-980","share_seq":980,"template_height":800030},{"accepted_at_ms":999984,"credit_policy":null,"job_id":"job-2179","job_issued_at_ms":999993,"miner_id":"miner-1","network_difficulty":1000003,"ntime":1700002179,"order_key":"01:miner-1","p2mr_program_hex":"8383838383838383838383838383838383838383838383838383838383838383","share_difficulty":2,"share_id":"share-2179","share_seq":2179,"template_height":800068},{"accepted_at_ms":999986,"credit_policy":null,"job_id":"job-486","job_issued_at_ms":999975,"miner_id":"miner-2","network_difficulty":1000010,"ntime":1700000486,"order_key":"02:miner-2","p2mr_program_hex":"e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6","share_difficulty":3,"share_id":"share-486","share_seq":486,"template_height":800015},{"accepted_at_ms":999975,"credit_policy":"stale-grace","job_id":"job-2203","job_issued_at_ms":999978,"miner_id":"miner-3","network_difficulty":1000010,"ntime":1700002203,"order_key":"03:miner-3","p2mr_program_hex":"9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b","share_difficulty":3,"share_id":"share-2203","share_seq":2203,"template_height":800068},{"accepted_at_ms":1000151,"credit_policy":null,"job_id":"job-1980","job_issued_at_ms":1000460,"miner_id":"miner-0","network_difficulty":1000008,"ntime":1700001980,"order_key":"00:miner-0","p2mr_program_hex":"bcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbc","share_difficulty":19,"share_id":"share-1980","share_seq":1980,"template_height":800061},{"accepted_at_ms":1000453,"credit_policy":null,"job_id":"job-3268","job_issued_at_ms":1000249,"miner_id":"miner-1","network_difficulty":1000004,"ntime":1700003268,"order_key":"01:miner-1","p2mr_program_hex":"c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4","share_difficulty":2,"share_id":"share-3268","share_seq":3268,"template_height":800102},{"accepted_at_ms":999983,"credit_policy":"stale-grace","job_id":"job-933","job_issued_at_ms":1000259,"miner_id":"miner-9","network_difficulty":1000015,"ntime":1700000933,"order_key":"09:miner-9","p2mr_program_hex":"a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5","share_difficulty":1,"share_id":"share-933","share_seq":933,"template_height":800029},{"accepted_at_ms":999975,"credit_policy":null,"job_id":"job-2876","job_issued_at_ms":999992,"miner_id":"miner-5","network_difficulty":1000003,"ntime":1700002876,"order_key":"05:miner-5","p2mr_program_hex":"3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c","share_difficulty":2,"share_id":"share-2876","share_seq":2876,"template_height":800089},{"accepted_at_ms":999993,"credit_policy":null,"job_id":"job-453","job_issued_at_ms":999988,"miner_id":"miner-2","network_difficulty":1000011,"ntime":1700000453,"order_key":"02:miner-2","p2mr_program_hex":"c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5","share_difficulty":101,"share_id":"share-453","share_seq":453,"template_height":800014},{"accepted_at_ms":999985,"credit_policy":"stale-grace","job_id":"job-114","job_issued_at_ms":999982,"miner_id":"miner-4","network_difficulty":1000012,"ntime":1700000114,"order_key":"04:miner-4","p2mr_program_hex":"7272727272727272727272727272727272727272727272727272727272727272","share_difficulty":3,"share_id":"share-114","share_seq":114,"template_height":800003},{"accepted_at_ms":999985,"credit_policy":null,"job_id":"job-2572","job_issued_at_ms":999982,"miner_id":"miner-9","network_difficulty":1000005,"ntime":1700002572,"order_key":"09:miner-9","p2mr_program_hex":"0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c","share_difficulty":101,"share_id":"share-2572","share_seq":2572,"template_height":800080},{"accepted_at_ms":999984,"credit_policy":null,"job_id":"job-3372","job_issued_at_ms":999995,"miner_id":"miner-6","network_difficulty":1000006,"ntime":1700003372,"order_key":"06:miner-6","p2mr_program_hex":"2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c","share_difficulty":3,"share_id":"share-3372","share_seq":3372,"template_height":800105},{"accepted_at_ms":999970,"credit_policy":null,"job_id":"job-3473","job_issued_at_ms":999975,"miner_id":"miner-8","network_difficulty":1000005,"ntime":1700003473,"order_key":"08:miner-8","p2mr_program_hex":"9191919191919191919191919191919191919191919191919191919191919191","share_difficulty":3,"share_id":"share-3473","share_seq":3473,"template_height":800108},{"accepted_at_ms":999988,"credit_policy":null,"job_id":"job-2191","job_issued_at_ms":999976,"miner_id":"miner-2","network_difficulty":1000015,"ntime":1700002191,"order_key":"02:miner-2","p2mr_program_hex":"8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f","share_difficulty":7,"share_id":"share-2191","share_seq":2191,"template_height":800068},{"accepted_at_ms":999979,"credit_policy":null,"job_id":"job-2963","job_issued_at_ms":999991,"miner_id":"miner-4","network_difficulty":1000005,"ntime":1700002963,"order_key":"04:miner-4","p2mr_program_hex":"9393939393939393939393939393939393939393939393939393939393939393","share_difficulty":7,"share_id":"share-2963","share_seq":2963,"template_height":800092},{"accepted_at_ms":999970,"credit_policy":null,"job_id":"job-1166","job_issued_at_ms":999973,"miner_id":"miner-0","network_difficulty":1000010,"ntime":1700001166,"order_key":"00:miner-0","p2mr_program_hex":"8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e","share_difficulty":101,"share_id":"share-1166","share_seq":1166,"template_height":800036},{"accepted_at_ms":999997,"credit_policy":null,"job_id":"job-359","job_issued_at_ms":999992,"miner_id":"miner-7","network_difficulty":1000002,"ntime":1700000359,"order_key":"07:miner-7","p2mr_program_hex":"6767676767676767676767676767676767676767676767676767676767676767","share_difficulty":1000,"share_id":"share-359","share_seq":359,"template_height":800011},{"accepted_at_ms":999989,"credit_policy":null,"job_id":"job-3782","job_issued_at_ms":999973,"miner_id":"miner-9","network_difficulty":1000008,"ntime":1700003782,"order_key":"09:miner-9","p2mr_program_hex":"c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6","share_difficulty":101,"share_id":"share-3782","share_seq":3782,"template_height":800118},{"accepted_at_ms":999984,"credit_policy":null,"job_id":"job-2234","job_issued_at_ms":1000193,"miner_id":"miner-1","network_difficulty":1000007,"ntime":1700002234,"order_key":"01:miner-1","p2mr_program_hex":"babababababababababababababababababababababababababababababababa","share_difficulty":101,"share_id":"share-2234","share_seq":2234,"template_height":800069},{"accepted_at_ms":999988,"credit_policy":"stale-grace","job_id":"job-853","job_issued_at_ms":999983,"miner_id":"miner-6","network_difficulty":1000003,"ntime":1700000853,"order_key":"06:miner-6","p2mr_program_hex":"5555555555555555555555555555555555555555555555555555555555555555","share_difficulty":3,"share_id":"share-853","share_seq":853,"template_height":800026},{"accepted_at_ms":999986,"credit_policy":"stale-grace","job_id":"job-2003","job_issued_at_ms":999988,"miner_id":"miner-1","network_difficulty":1000014,"ntime":1700002003,"order_key":"01:miner-1","p2mr_program_hex":"d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3","share_difficulty":2,"share_id":"share-2003","share_seq":2003,"template_height":800062},{"accepted_at_ms":999990,"credit_policy":"stale-grace","job_id":"job-1695","job_issued_at_ms":999985,"miner_id":"miner-1","network_difficulty":1000012,"ntime":1700001695,"order_key":"01:miner-1","p2mr_program_hex":"9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f","share_difficulty":1000,"share_id":"share-1695","share_seq":1695,"template_height":800052},{"accepted_at_ms":999994,"credit_policy":null,"job_id":"job-1167","job_issued_at_ms":999981,"miner_id":"miner-1","network_difficulty":1000011,"ntime":1700001167,"order_key":"01:miner-1","p2mr_program_hex":"8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f","share_difficulty":7,"share_id":"share-1167","share_seq":1167,"template_height":800036},{"accepted_at_ms":999993,"credit_policy":null,"job_id":"job-796","job_issued_at_ms":1000000,"miner_id":"miner-4","network_difficulty":1000014,"ntime":1700000796,"order_key":"04:miner-4","p2mr_program_hex":"1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c","share_difficulty":7,"share_id":"share-796","share_seq":796,"template_height":800024},{"accepted_at_ms":999997,"credit_policy":null,"job_id":"job-1840","job_issued_at_ms":999995,"miner_id":"miner-3","network_difficulty":1000004,"ntime":1700001840,"order_key":"03:miner-3","p2mr_program_hex":"3030303030303030303030303030303030303030303030303030303030303030","share_difficulty":2,"share_id":"share-1840","share_seq":1840,"template_height":800057},{"accepted_at_ms":1000000,"credit_policy":null,"job_id":"job-2146","job_issued_at_ms":999972,"miner_id":"miner-1","network_difficulty":1000004,"ntime":1700002146,"order_key":"01:miner-1","p2mr_program_hex":"6262626262626262626262626262626262626262626262626262626262626262","share_difficulty":101,"share_id":"share-2146","share_seq":2146,"template_height":800067},{"accepted_at_ms":999994,"credit_policy":null,"job_id":"job-1239","job_issued_at_ms":999993,"miner_id":"miner-7","network_difficulty":1000015,"ntime":1700001239,"order_key":"07:miner-7","p2mr_program_hex":"d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7","share_difficulty":2,"share_id":"share-1239","share_seq":1239,"template_height":800038},{"accepted_at_ms":999989,"credit_policy":"stale-grace","job_id":"job-927","job_issued_at_ms":999978,"miner_id":"miner-3","network_difficulty":1000009,"ntime":1700000927,"order_key":"03:miner-3","p2mr_program_hex":"9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f","share_difficulty":101,"share_id":"share-927","share_seq":927,"template_height":800028},{"accepted_at_ms":999970,"credit_policy":null,"job_id":"job-1200","job_issued_at_ms":999989,"miner_id":"miner-1","network_difficulty":1000010,"ntime":1700001200,"order_key":"01:miner-1","p2mr_program_hex":"b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0","share_difficulty":19,"share_id":"share-1200","share_seq":1200,"template_height":800037},{"accepted_at_ms":1000319,"credit_policy":null,"job_id":"job-1293","job_issued_at_ms":1000069,"miner_id":"miner-6","network_difficulty":1000001,"ntime":1700001293,"order_key":"06:miner-6","p2mr_program_hex":"0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d","share_difficulty":3,"share_id":"share-1293","share_seq":1293,"template_height":800040},{"accepted_at_ms":999973,"credit_policy":null,"job_id":"job-2316","job_issued_at_ms":999973,"miner_id":"miner-6","network_difficulty":1000004,"ntime":1700002316,"order_key":"06:miner-6","p2mr_program_hex":"0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c","share_difficulty":7,"share_id":"share-2316","share_seq":2316,"template_height":800072},{"accepted_at_ms":999978,"credit_policy":"stale-grace","job_id":"job-3427","job_issued_at_ms":1000339,"miner_id":"miner-6","network_difficulty":1000010,"ntime":1700003427,"order_key":"06:miner-6","p2mr_program_hex":"6363636363636363636363636363636363636363636363636363636363636363","share_difficulty":1,"share_id":"share-3427","share_seq":3427,"template_height":800107},{"accepted_at_ms":999983,"credit_policy":"stale-grace","job_id":"job-3770","job_issued_at_ms":999979,"miner_id":"miner-8","network_difficulty":1000013,"ntime":1700003770,"order_key":"08:miner-8","p2mr_program_hex":"babababababababababababababababababababababababababababababababa","share_difficulty":19,"share_id":"share-3770","share_seq":3770,"template_height":800117},{"accepted_at_ms":999978,"credit_policy":"stale-grace","job_id":"job-1586","job_issued_at_ms":999984,"miner_id":"miner-2","network_difficulty":1000005,"ntime":1700001586,"order_key":"02:miner-2","p2mr_program_hex":"3232323232323232323232323232323232323232323232323232323232323232","share_difficulty":2,"share_id":"share-1586","share_seq":1586,"template_height":800049},{"accepted_at_ms":1000000,"credit_policy":null,"job_id":"job-2778","job_issued_at_ms":999973,"miner_id":"miner-6","network_difficulty":1000007,"ntime":1700002778,"order_key":"06:miner-6","p2mr_program_hex":"dadadadadadadadadadadadadadadadadadadadadadadadadadadadadadadada","share_difficulty":101,"share_id":"share-2778","share_seq":2778,"template_height":800086},{"accepted_at_ms":999996,"credit_policy":"stale-grace","job_id":"job-2922","job_issued_at_ms":999997,"miner_id":"miner-7","network_difficulty":1000015,"ntime":1700002922,"order_key":"07:miner-7","p2mr_program_hex":"6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a","share_difficulty":101,"share_id":"share-2922","share_seq":2922,"template_height":800091},{"accepted_at_ms":999977,"credit_policy":null,"job_id":"job-3206","job_issued_at_ms":999975,"miner_id":"miner-5","network_difficulty":1000010,"ntime":1700003206,"order_key":"05:miner-5","p2mr_program_hex":"8686868686868686868686868686868686868686868686868686868686868686","share_difficulty":19,"share_id":"share-3206","share_seq":3206,"template_height":800100},{"accepted_at_ms":999970,"credit_policy":"stale-grace","job_id":"job-3576","job_issued_at_ms":999993,"miner_id":"miner-1","network_difficulty":1000006,"ntime":1700003576,"order_key":"01:miner-1","p2mr_program_hex":"f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8","share_difficulty":3,"share_id":"share-3576","share_seq":3576,"template_height":800111},{"accepted_at_ms":999994,"credit_policy":null,"job_id":"job-3267","job_issued_at_ms":999974,"miner_id":"miner-0","network_difficulty":1000003,"ntime":1700003267,"order_key":"00:miner-0","p2mr_program_hex":"c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3","share_difficulty":19,"share_id":"share-3267","share_seq":3267,"template_height":800102},{"accepted_at_ms":999974,"credit_policy":null,"job_id":"job-3652","job_issued_at_ms":999984,"miner_id":"miner-0","network_difficulty":1000014,"ntime":1700003652,"order_key":"00:miner-0","p2mr_program_hex":"4444444444444444444444444444444444444444444444444444444444444444","share_difficulty":1,"share_id":"share-3652","share_seq":3652,"template_height":800114},{"accepted_at_ms":999995,"credit_policy":null,"job_id":"job-716","job_issued_at_ms":999998,"miner_id":"miner-1","network_difficulty":1000002,"ntime":1700000716,"order_key":"01:miner-1","p2mr_program_hex":"cccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccc","share_difficulty":7,"share_id":"share-716","share_seq":716,"template_height":800022},{"accepted_at_ms":999987,"credit_policy":null,"job_id":"job-2369","job_issued_at_ms":999981,"miner_id":"miner-4","network_difficulty":1000006,"ntime":1700002369,"order_key":"04:miner-4","p2mr_program_hex":"4141414141414141414141414141414141414141414141414141414141414141","share_difficulty":1000,"share_id":"share-2369","share_seq":2369,"template_height":800074},{"accepted_at_ms":999992,"credit_policy":null,"job_id":"job-2299","job_issued_at_ms":999986,"miner_id":"miner-0","network_difficulty":1000004,"ntime":1700002299,"order_key":"00:miner-0","p2mr_program_hex":"fbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfb","share_difficulty":1000,"share_id":"share-2299","share_seq":2299,"template_height":800071},{"accepted_at_ms":999991,"credit_policy":null,"job_id":"job-206","job_issued_at_ms":999977,"miner_id":"miner-8","network_difficulty":1000002,"ntime":1700000206,"order_key":"08:miner-8","p2mr_program_hex":"cececececececececececececececececececececececececececececececece","share_difficulty":2,"share_id":"share-206","share_seq":206,"template_height":800006},{"accepted_at_ms":999977,"credit_policy":null,"job_id":"job-3660","job_issued_at_ms":999984,"miner_id":"miner-8","network_difficulty":1000005,"ntime":1700003660,"order_key":"08:miner-8","p2mr_program_hex":"4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c","share_difficulty":101,"share_id":"share-3660","share_seq":3660,"template_height":800114},{"accepted_at_ms":999990,"credit_policy":null,"job_id":"job-187","job_issued_at_ms":999990,"miner_id":"miner-0","network_difficulty":1000000,"ntime":1700000187,"order_key":"00:miner-0","p2mr_program_hex":"bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb","share_difficulty":1,"share_id":"share-187","share_seq":187,"template_height":800005},{"accepted_at_ms":999998,"credit_policy":"stale-grace","job_id":"job-2888","job_issued_at_ms":999997,"miner_id":"miner-6","network_difficulty":1000015,"ntime":1700002888,"order_key":"06:miner-6","p2mr_program_hex":"4848484848484848484848484848484848484848484848484848484848484848","share_difficulty":2,"share_id":"share-2888","share_seq":2888,"template_height":800090},{"accepted_at_ms":999982,"credit_policy":null,"job_id":"job-3755","job_issued_at_ms":999988,"miner_id":"miner-4","network_difficulty":1000015,"ntime":1700003755,"order_key":"04:miner-4","p2mr_program_hex":"abababababababababababababababababababababababababababababababab","share_difficulty":19,"share_id":"share-3755","share_seq":3755,"template_height":800117},{"accepted_at_ms":999981,"credit_policy":null,"job_id":"job-2539","job_issued_at_ms":1000390,"miner_id":"miner-9","network_difficulty":1000006,"ntime":1700002539,"order_key":"09:miner-9","p2mr_program_hex":"ebebebebebebebebebebebebebebebebebebebebebebebebebebebebebebebeb","share_difficulty":2,"share_id":"share-2539","share_seq":2539,"template_height":800079},{"accepted_at_ms":999976,"credit_policy":null,"job_id":"job-2984","job_issued_at_ms":999992,"miner_id":"miner-3","network_difficulty":1000009,"ntime":1700002984,"order_key":"03:miner-3","p2mr_program_hex":"a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8","share_difficulty":1000,"share_id":"share-2984","share_seq":2984,"template_height":800093},{"accepted_at_ms":999979,"credit_policy":null,"job_id":"job-3981","job_issued_at_ms":999979,"miner_id":"miner-10","network_difficulty":1000003,"ntime":1700003981,"order_key":"10:miner-10","p2mr_program_hex":"8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d","share_difficulty":2,"share_id":"share-3981","share_seq":3981,"template_height":800124},{"accepted_at_ms":999982,"credit_policy":"stale-grace","job_id":"job-1288","job_issued_at_ms":999977,"miner_id":"miner-1","network_difficulty":1000013,"ntime":1700001288,"order_key":"01:miner-1","p2mr_program_hex":"0808080808080808080808080808080808080808080808080808080808080808","share_difficulty":1,"share_id":"share-1288","share_seq":1288,"template_height":800040},{"accepted_at_ms":999988,"credit_policy":"stale-grace","job_id":"job-1404","job_issued_at_ms":999991,"miner_id":"miner-7","network_difficulty":1000010,"ntime":1700001404,"order_key":"07:miner-7","p2mr_program_hex":"7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c","share_difficulty":101,"share_id":"share-1404","share_seq":1404,"template_height":800043},{"accepted_at_ms":999994,"credit_policy":null,"job_id":"job-2826","job_issued_at_ms":999988,"miner_id":"miner-10","network_difficulty":1000004,"ntime":1700002826,"order_key":"10:miner-10","p2mr_program_hex":"0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a","share_difficulty":2,"share_id":"share-2826","share_seq":2826,"template_height":800088},{"accepted_at_ms":999988,"credit_policy":null,"job_id":"job-624","job_issued_at_ms":999970,"miner_id":"miner-8","network_difficulty":1000012,"ntime":1700000624,"order_key":"08:miner-8","p2mr_program_hex":"7070707070707070707070707070707070707070707070707070707070707070","share_difficulty":7,"share_id":"share-624","share_seq":624,"template_height":800019},{"accepted_at_ms":999975,"credit_policy":"stale-grace","job_id":"job-1168","job_issued_at_ms":999998,"miner_id":"miner-2","network_difficulty":1000012,"ntime":1700001168,"order_key":"02:miner-2","p2mr_program_hex":"9090909090909090909090909090909090909090909090909090909090909090","share_difficulty":101,"share_id":"share-1168","share_seq":1168,"template_height":800036},{"accepted_at_ms":999997,"credit_policy":null,"job_id":"job-2515","job_issued_at_ms":999985,"miner_id":"miner-7","network_difficulty":1000016,"ntime":1700002515,"order_key":"07:miner-7","p2mr_program_hex":"d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3","share_difficulty":2,"share_id":"share-2515","share_seq":2515,"template_height":800078},{"accepted_at_ms":999972,"credit_policy":null,"job_id":"job-2066","job_issued_at_ms":999998,"miner_id":"miner-9","network_difficulty":1000009,"ntime":1700002066,"order_key":"09:miner-9","p2mr_program_hex":"1212121212121212121212121212121212121212121212121212121212121212","share_difficulty":1000,"share_id":"share-2066","share_seq":2066,"template_height":800064},{"accepted_at_ms":999990,"credit_policy":null,"job_id":"job-3513","job_issued_at_ms":999993,"miner_id":"miner-4","network_difficulty":1000011,"ntime":1700003513,"order_key":"04:miner-4","p2mr_program_hex":"b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9","share_difficulty":7,"share_id":"share-3513","share_seq":3513,"template_height":800109},{"accepted_at_ms":999981,"credit_policy":null,"job_id":"job-2325","job_issued_at_ms":999977,"miner_id":"miner-4","network_difficulty":1000013,"ntime":1700002325,"order_key":"04:miner-4","p2mr_program_hex":"1515151515151515151515151515151515151515151515151515151515151515","share_difficulty":101,"share_id":"share-2325","share_seq":2325,"template_height":800072},{"accepted_at_ms":999979,"credit_policy":null,"job_id":"job-1884","job_issued_at_ms":999988,"miner_id":"miner-3","network_difficulty":1000014,"ntime":1700001884,"order_key":"03:miner-3","p2mr_program_hex":"5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c","share_difficulty":3,"share_id":"share-1884","share_seq":1884,"template_height":800058},{"accepted_at_ms":999976,"credit_policy":"stale-grace","job_id":"job-1297","job_issued_at_ms":999990,"miner_id":"miner-10","network_difficulty":1000005,"ntime":1700001297,"order_key":"10:miner-10","p2mr_program_hex":"1111111111111111111111111111111111111111111111111111111111111111","share_difficulty":101,"share_id":"share-1297","share_seq":1297,"template_height":800040},{"accepted_at_ms":999983,"credit_policy":null,"job_id":"job-474","job_issued_at_ms":999997,"miner_id":"miner-1","network_difficulty":1000015,"ntime":1700000474,"order_key":"01:miner-1","p2mr_program_hex":"dadadadadadadadadadadadadadadadadadadadadadadadadadadadadadadada","share_difficulty":19,"share_id":"share-474","share_seq":474,"template_height":800014},{"accepted_at_ms":1000323,"credit_policy":"stale-grace","job_id":"job-3150","job_issued_at_ms":1000216,"miner_id":"miner-4","network_difficulty":1000005,"ntime":1700003150,"order_key":"04:miner-4","p2mr_program_hex":"4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e","share_difficulty":1,"share_id":"share-3150","share_seq":3150,"template_height":800098},{"accepted_at_ms":999989,"credit_policy":"stale-grace","job_id":"job-61","job_issued_at_ms":999972,"miner_id":"miner-6","network_difficulty":1000010,"ntime":1700000061,"order_key":"06:miner-6","p2mr_program_hex":"3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d","share_difficulty":2,"share_id":"share-61","share_seq":61,"template_height":800001},{"accepted_at_ms":999997,"credit_policy":null,"job_id":"job-2448","job_issued_at_ms":999978,"miner_id":"miner-6","network_difficulty":1000000,"ntime":1700002448,"order_key":"06:miner-6","p2mr_program_hex":"9090909090909090909090909090909090909090909090909090909090909090","share_difficulty":2,"share_id":"share-2448","share_seq":2448,"template_height":800076},{"accepted_at_ms":1000228,"credit_policy":"stale-grace","job_id":"job-3179","job_issued_at_ms":1000078,"miner_id":"miner-0","network_difficulty":1000000,"ntime":1700003179,"order_key":"00:miner-0","p2mr_program_hex":"6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b","share_difficulty":2,"share_id":"share-3179","share_seq":3179,"template_height":800099},{"accepted_at_ms":999974,"credit_policy":"stale-grace","job_id":"job-95","job_issued_at_ms":999985,"miner_id":"miner-7","network_difficulty":1000010,"ntime":1700000095,"order_key":"07:miner-7","p2mr_program_hex":"5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f","share_difficulty":19,"share_id":"share-95","share_seq":95,"template_height":800002},{"accepted_at_ms":999981,"credit_policy":null,"job_id":"job-451","job_issued_at_ms":999982,"miner_id":"miner-0","network_difficulty":1000009,"ntime":1700000451,"order_key":"00:miner-0","p2mr_program_hex":"c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3","share_difficulty":1,"share_id":"share-451","share_seq":451,"template_height":800014},{"accepted_at_ms":999981,"credit_policy":null,"job_id":"job-595","job_issued_at_ms":999992,"miner_id":"miner-1","network_difficulty":1000000,"ntime":1700000595,"order_key":"01:miner-1","p2mr_program_hex":"5353535353535353535353535353535353535353535353535353535353535353","share_difficulty":1,"share_id":"share-595","share_seq":595,"template_height":800018},{"accepted_at_ms":999998,"credit_policy":null,"job_id":"job-1156","job_issued_at_ms":999994,"miner_id":"miner-1","network_difficulty":1000000,"ntime":1700001156,"order_key":"01:miner-1","p2mr_program_hex":"8484848484848484848484848484848484848484848484848484848484848484","share_difficulty":1000,"share_id":"share-1156","share_seq":1156,"template_height":800036},{"accepted_at_ms":999994,"credit_policy":null,"job_id":"job-2253","job_issued_at_ms":999977,"miner_id":"miner-9","network_difficulty":1000009,"ntime":1700002253,"order_key":"09:miner-9","p2mr_program_hex":"cdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcd","share_difficulty":2,"share_id":"share-2253","share_seq":2253,"template_height":800070},{"accepted_at_ms":999992,"credit_policy":null,"job_id":"job-3292","job_issued_at_ms":999972,"miner_id":"miner-3","network_difficulty":1000011,"ntime":1700003292,"order_key":"03:miner-3","p2mr_program_hex":"dcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdc","share_difficulty":2,"share_id":"share-3292","share_seq":3292,"template_height":800102},{"accepted_at_ms":1000096,"credit_policy":null,"job_id":"job-258","job_issued_at_ms":999997,"miner_id":"miner-5","network_difficulty":1000003,"ntime":1700000258,"order_key":"05:miner-5","p2mr_program_hex":"0202020202020202020202020202020202020202020202020202020202020202","share_difficulty":2,"share_id":"share-258","share_seq":258,"template_height":800008},{"accepted_at_ms":999973,"credit_policy":null,"job_id":"job-3536","job_issued_at_ms":999976,"miner_id":"miner-5","network_difficulty":1000000,"ntime":1700003536,"order_key":"05:miner-5","p2mr_program_hex":"d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0","share_difficulty":19,"share_id":"share-3536","share_seq":3536,"template_height":800110},{"accepted_at_ms":999971,"credit_policy":null,"job_id":"job-3979","job_issued_at_ms":999975,"miner_id":"miner-8","network_difficulty":1000001,"ntime":1700003979,"order_key":"08:miner-8","p2mr_program_hex":"8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b","share_difficulty":19,"share_id":"share-3979","share_seq":3979,"template_height":800124},{"accepted_at_ms":999987,"credit_policy":null,"job_id":"job-2669","job_issued_at_ms":999990,"miner_id":"miner-7","network_difficulty":1000000,"ntime":1700002669,"order_key":"07:miner-7","p2mr_program_hex":"6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d","share_difficulty":1000,"share_id":"share-2669","share_seq":2669,"template_height":800083},{"accepted_at_ms":999996,"credit_policy":null,"job_id":"job-2370","job_issued_at_ms":999973,"miner_id":"miner-5","network_difficulty":1000007,"ntime":1700002370,"order_key":"05:miner-5","p2mr_program_hex":"4242424242424242424242424242424242424242424242424242424242424242","share_difficulty":1,"share_id":"share-2370","share_seq":2370,"template_height":800074},{"accepted_at_ms":999979,"credit_policy":null,"job_id":"job-2768","job_issued_at_ms":999970,"miner_id":"miner-7","network_difficulty":1000014,"ntime":1700002768,"order_key":"07:miner-7","p2mr_program_hex":"d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0","share_difficulty":19,"share_id":"share-2768","share_seq":2768,"template_height":800086},{"accepted_at_ms":999973,"credit_policy":"stale-grace","job_id":"job-419","job_issued_at_ms":999975,"miner_id":"miner-1","network_difficulty":1000011,"ntime":1700000419,"order_key":"01:miner-1","p2mr_program_hex":"a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3","share_difficulty":3,"share_id":"share-419","share_seq":419,"template_height":800013},{"accepted_at_ms":999973,"credit_policy":null,"job_id":"job-1724","job_issued_at_ms":999978,"miner_id":"miner-8","network_difficulty":1000007,"ntime":1700001724,"order_key":"08:miner-8","p2mr_program_hex":"bcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbc","share_difficulty":19,"share_id":"share-1724","share_seq":1724,"template_height":800053},{"accepted_at_ms":1000310,"credit_policy":null,"job_id":"job-1448","job_issued_at_ms":1000359,"miner_id":"miner-7","network_difficulty":1000003,"ntime":1700001448,"order_key":"07:miner-7","p2mr_program_hex":"a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8","share_difficulty":1000,"share_id":"share-1448","share_seq":1448,"template_height":800045},{"accepted_at_ms":1000336,"credit_policy":null,"job_id":"job-1340","job_issued_at_ms":999976,"miner_id":"miner-9","network_difficulty":1000014,"ntime":1700001340,"order_key":"09:miner-9","p2mr_program_hex":"3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c","share_difficulty":2,"share_id":"share-1340","share_seq":1340,"template_height":800041},{"accepted_at_ms":1000337,"credit_policy":null,"job_id":"job-3864","job_issued_at_ms":999992,"miner_id":"miner-3","network_difficulty":1000005,"ntime":1700003864,"order_key":"03:miner-3","p2mr_program_hex":"1818181818181818181818181818181818181818181818181818181818181818","share_difficulty":3,"share_id":"share-3864","share_seq":3864,"template_height":800120},{"accepted_at_ms":999988,"credit_policy":null,"job_id":"job-293","job_issued_at_ms":999978,"miner_id":"miner-7","network_difficulty":1000004,"ntime":1700000293,"order_key":"07:miner-7","p2mr_program_hex":"2525252525252525252525252525252525252525252525252525252525252525","share_difficulty":3,"share_id":"share-293","share_seq":293,"template_height":800009},{"accepted_at_ms":999986,"credit_policy":null,"job_id":"job-2372","job_issued_at_ms":999996,"miner_id":"miner-7","network_difficulty":1000009,"ntime":1700002372,"order_key":"07:miner-7","p2mr_program_hex":"4444444444444444444444444444444444444444444444444444444444444444","share_difficulty":2,"share_id":"share-2372","share_seq":2372,"template_height":800074},{"accepted_at_ms":999987,"credit_policy":null,"job_id":"job-1125","job_issued_at_ms":999974,"miner_id":"miner-3","network_difficulty":1000003,"ntime":1700001125,"order_key":"03:miner-3","p2mr_program_hex":"6565656565656565656565656565656565656565656565656565656565656565","share_difficulty":101,"share_id":"share-1125","share_seq":1125,"template_height":800035},{"accepted_at_ms":999972,"credit_policy":null,"job_id":"job-2212","job_issued_at_ms":999998,"miner_id":"miner-1","network_difficulty":1000002,"ntime":1700002212,"order_key":"01:miner-1","p2mr_program_hex":"a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4","share_difficulty":101,"share_id":"share-2212","share_seq":2212,"template_height":800069},{"accepted_at_ms":1000154,"credit_policy":"stale-grace","job_id":"job-2391","job_issued_at_ms":999970,"miner_id":"miner-4","network_difficulty":1000011,"ntime":1700002391,"order_key":"04:miner-4","p2mr_program_hex":"5757575757575757575757575757575757575757575757575757575757575757","share_difficulty":7,"share_id":"share-2391","share_seq":2391,"template_height":800074},{"accepted_at_ms":999996,"credit_policy":null,"job_id":"job-3173","job_issued_at_ms":999975,"miner_id":"miner-5","network_difficulty":1000011,"ntime":1700003173,"order_key":"05:miner-5","p2mr_program_hex":"6565656565656565656565656565656565656565656565656565656565656565","share_difficulty":101,"share_id":"share-3173","share_seq":3173,"template_height":800099},{"accepted_at_ms":999993,"credit_policy":null,"job_id":"job-2867","job_issued_at_ms":999995,"miner_id":"miner-7","network_difficulty":1000011,"ntime":1700002867,"order_key":"07:miner-7","p2mr_program_hex":"3333333333333333333333333333333333333333333333333333333333333333","share_difficulty":7,"share_id":"share-2867","share_seq":2867,"template_height":800089},{"accepted_at_ms":999975,"credit_policy":null,"job_id":"job-1620","job_issued_at_ms":999986,"miner_id":"miner-3","network_difficulty":1000005,"ntime":1700001620,"order_key":"03:miner-3","p2mr_program_hex":"5454545454545454545454545454545454545454545454545454545454545454","share_difficulty":19,"share_id":"share-1620","share_seq":1620,"template_height":800050},{"accepted_at_ms":999996,"credit_policy":null,"job_id":"job-1426","job_issued_at_ms":999977,"miner_id":"miner-7","network_difficulty":1000015,"ntime":1700001426,"order_key":"07:miner-7","p2mr_program_hex":"9292929292929292929292929292929292929292929292929292929292929292","share_difficulty":3,"share_id":"share-1426","share_seq":1426,"template_height":800044},{"accepted_at_ms":1000000,"credit_policy":null,"job_id":"job-3642","job_issued_at_ms":999977,"miner_id":"miner-1","network_difficulty":1000004,"ntime":1700003642,"order_key":"01:miner-1","p2mr_program_hex":"3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a","share_difficulty":3,"share_id":"share-3642","share_seq":3642,"template_height":800113},{"accepted_at_ms":999988,"credit_policy":null,"job_id":"job-3461","job_issued_at_ms":999982,"miner_id":"miner-7","network_difficulty":1000010,"ntime":1700003461,"order_key":"07:miner-7","p2mr_program_hex":"8585858585858585858585858585858585858585858585858585858585858585","share_difficulty":19,"share_id":"share-3461","share_seq":3461,"template_height":800108},{"accepted_at_ms":999976,"credit_policy":null,"job_id":"job-15","job_issued_at_ms":1000183,"miner_id":"miner-4","network_difficulty":1000015,"ntime":1700000015,"order_key":"04:miner-4","p2mr_program_hex":"0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f","share_difficulty":3,"share_id":"share-15","share_seq":15,"template_height":800000},{"accepted_at_ms":999984,"credit_policy":null,"job_id":"job-1307","job_issued_at_ms":999999,"miner_id":"miner-9","network_difficulty":1000015,"ntime":1700001307,"order_key":"09:miner-9","p2mr_program_hex":"1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b","share_difficulty":19,"share_id":"share-1307","share_seq":1307,"template_height":800040},{"accepted_at_ms":999990,"credit_policy":null,"job_id":"job-2875","job_issued_at_ms":999973,"miner_id":"miner-4","network_difficulty":1000002,"ntime":1700002875,"order_key":"04:miner-4","p2mr_program_hex":"3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b","share_difficulty":19,"share_id":"share-2875","share_seq":2875,"template_height":800089},{"accepted_at_ms":999990,"credit_policy":"stale-grace","job_id":"job-3953","job_issued_at_ms":999970,"miner_id":"miner-4","network_difficulty":1000009,"ntime":1700003953,"order_key":"04:miner-4","p2mr_program_hex":"7171717171717171717171717171717171717171717171717171717171717171","share_difficulty":2,"share_id":"share-3953","share_seq":3953,"template_height":800123},{"accepted_at_ms":999972,"credit_policy":null,"job_id":"job-2276","job_issued_at_ms":999971,"miner_id":"miner-10","network_difficulty":1000015,"ntime":1700002276,"order_key":"10:miner-10","p2mr_program_hex":"e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4","share_difficulty":3,"share_id":"share-2276","share_seq":2276,"template_height":800071},{"accepted_at_ms":999978,"credit_policy":null,"job_id":"job-24","job_issued_at_ms":999996,"miner_id":"miner-2","network_difficulty":1000007,"ntime":1700000024,"order_key":"02:miner-2","p2mr_program_hex":"1818181818181818181818181818181818181818181818181818181818181818","share_difficulty":1,"share_id":"share-24","share_seq":24,"template_height":800000},{"accepted_at_ms":999990,"credit_policy":null,"job_id":"job-775","job_issued_at_ms":999986,"miner_id":"miner-5","network_difficulty":1000010,"ntime":1700000775,"order_key":"05:miner-5","p2mr_program_hex":"0707070707070707070707070707070707070707070707070707070707070707","share_difficulty":1,"share_id":"share-775","share_seq":775,"template_height":800024},{"accepted_at_ms":999993,"credit_policy":null,"job_id":"job-768","job_issued_at_ms":999987,"miner_id":"miner-9","network_difficulty":1000003,"ntime":1700000768,"order_key":"09:miner-9","p2mr_program_hex":"0000000000000000000000000000000000000000000000000000000000000000","share_difficulty":2,"share_id":"share-768","share_seq":768,"template_height":800024},{"accepted_at_ms":999976,"credit_policy":null,"job_id":"job-3942","job_issued_at_ms":999990,"miner_id":"miner-4","network_difficulty":1000015,"ntime":1700003942,"order_key":"04:miner-4","p2mr_program_hex":"6666666666666666666666666666666666666666666666666666666666666666","share_difficulty":101,"share_id":"share-3942","share_seq":3942,"template_height":800123},{"accepted_at_ms":1000070,"credit_policy":null,"job_id":"job-301","job_issued_at_ms":999996,"miner_id":"miner-4","network_difficulty":1000012,"ntime":1700000301,"order_key":"04:miner-4","p2mr_program_hex":"2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d","share_difficulty":2,"share_id":"share-301","share_seq":301,"template_height":800009},{"accepted_at_ms":999987,"credit_policy":null,"job_id":"job-1919","job_issued_at_ms":999976,"miner_id":"miner-5","network_difficulty":1000015,"ntime":1700001919,"order_key":"05:miner-5","p2mr_program_hex":"7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f","share_difficulty":19,"share_id":"share-1919","share_seq":1919,"template_height":800059},{"accepted_at_ms":999981,"credit_policy":"stale-grace","job_id":"job-2408","job_issued_at_ms":999987,"miner_id":"miner-10","network_difficulty":1000011,"ntime":1700002408,"order_key":"10:miner-10","p2mr_program_hex":"6868686868686868686868686868686868686868686868686868686868686868","share_difficulty":1,"share_id":"share-2408","share_seq":2408,"template_height":800075},{"accepted_at_ms":999998,"credit_policy":null,"job_id":"job-3827","job_issued_at_ms":999972,"miner_id":"miner-10","network_difficulty":1000002,"ntime":1700003827,"order_key":"10:miner-10","p2mr_program_hex":"f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3","share_difficulty":2,"share_id":"share-3827","share_seq":3827,"template_height":800119},{"accepted_at_ms":999980,"credit_policy":null,"job_id":"job-322","job_issued_at_ms":999971,"miner_id":"miner-3","network_difficulty":1000016,"ntime":1700000322,"order_key":"03:miner-3","p2mr_program_hex":"4242424242424242424242424242424242424242424242424242424242424242","share_difficulty":2,"share_id":"share-322","share_seq":322,"template_height":800010},{"accepted_at_ms":999975,"credit_policy":null,"job_id":"job-2433","job_issued_at_ms":999976,"miner_id":"miner-2","network_difficulty":1000002,"ntime":1700002433,"order_key":"02:miner-2","p2mr_program_hex":"8181818181818181818181818181818181818181818181818181818181818181","share_difficulty":7,"share_id":"share-2433","share_seq":2433,"template_height":800076},{"accepted_at_ms":1000000,"credit_policy":null,"job_id":"job-1666","job_issued_at_ms":999983,"miner_id":"miner-5","network_difficulty":1000000,"ntime":1700001666,"order_key":"05:miner-5","p2mr_program_hex":"8282828282828282828282828282828282828282828282828282828282828282","share_difficulty":7,"share_id":"share-1666","share_seq":1666,"template_height":800052},{"accepted_at_ms":999999,"credit_policy":null,"job_id":"job-1084","job_issued_at_ms":999984,"miner_id":"miner-6","network_difficulty":1000013,"ntime":1700001084,"order_key":"06:miner-6","p2mr_program_hex":"3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c","share_difficulty":19,"share_id":"share-1084","share_seq":1084,"template_height":800033},{"accepted_at_ms":999999,"credit_policy":null,"job_id":"job-2915","job_issued_at_ms":999993,"miner_id":"miner-0","network_difficulty":1000008,"ntime":1700002915,"order_key":"00:miner-0","p2mr_program_hex":"6363636363636363636363636363636363636363636363636363636363636363","share_difficulty":19,"share_id":"share-2915","share_seq":2915,"template_height":800091},{"accepted_at_ms":999988,"credit_policy":null,"job_id":"job-3104","job_issued_at_ms":999981,"miner_id":"miner-2","network_difficulty":1000010,"ntime":1700003104,"order_key":"02:miner-2","p2mr_program_hex":"2020202020202020202020202020202020202020202020202020202020202020","share_difficulty":2,"share_id":"share-3104","share_seq":3104,"template_height":800097},{"accepted_at_ms":999993,"credit_policy":null,"job_id":"job-2632","job_issued_at_ms":999979,"miner_id":"miner-3","network_difficulty":1000014,"ntime":1700002632,"order_key":"03:miner-3","p2mr_program_hex":"4848484848484848484848484848484848484848484848484848484848484848","share_difficulty":2,"share_id":"share-2632","share_seq":2632,"template_height":800082},{"accepted_at_ms":999971,"credit_policy":null,"job_id":"job-3094","job_issued_at_ms":999990,"miner_id":"miner-3","network_difficulty":1000000,"ntime":1700003094,"order_key":"03:miner-3","p2mr_program_hex":"1616161616161616161616161616161616161616161616161616161616161616","share_difficulty":1,"share_id":"share-3094","share_seq":3094,"template_height":800096},{"accepted_at_ms":999983,"credit_policy":"stale-grace","job_id":"job-1536","job_issued_at_ms":999989,"miner_id":"miner-7","network_difficulty":1000006,"ntime":1700001536,"order_key":"07:miner-7","p2mr_program_hex":"0000000000000000000000000000000000000000000000000000000000000000","share_difficulty":19,"share_id":"share-1536","share_seq":1536,"template_height":800048},{"accepted_at_ms":999991,"credit_policy":null,"job_id":"job-994","job_issued_at_ms":999973,"miner_id":"miner-4","network_difficulty":1000008,"ntime":1700000994,"order_key":"04:miner-4","p2mr_program_hex":"e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2","share_difficulty":3,"share_id":"share-994","share_seq":994,"template_height":800031},{"accepted_at_ms":999985,"credit_policy":null,"job_id":"job-2040","job_issued_at_ms":999994,"miner_id":"miner-5","network_difficulty":1000000,"ntime":1700002040,"order_key":"05:miner-5","p2mr_program_hex":"f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8","share_difficulty":19,"share_id":"share-2040","share_seq":2040,"template_height":800063},{"accepted_at_ms":999974,"credit_policy":null,"job_id":"job-620","job_issued_at_ms":999973,"miner_id":"miner-4","network_difficulty":1000008,"ntime":1700000620,"order_key":"04:miner-4","p2mr_program_hex":"6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c","share_difficulty":19,"share_id":"share-620","share_seq":620,"template_height":800019},{"accepted_at_ms":999988,"credit_policy":null,"job_id":"job-2817","job_issued_at_ms":999973,"miner_id":"miner-1","network_difficulty":1000012,"ntime":1700002817,"order_key":"01:miner-1","p2mr_program_hex":"0101010101010101010101010101010101010101010101010101010101010101","share_difficulty":101,"share_id":"share-2817","share_seq":2817,"template_height":800088},{"accepted_at_ms":999972,"credit_policy":null,"job_id":"job-1519","job_issued_at_ms":999972,"miner_id":"miner-1","network_difficulty":1000006,"ntime":1700001519,"order_key":"01:miner-1","p2mr_program_hex":"efefefefefefefefefefefefefefefefefefefefefefefefefefefefefefefef","share_difficulty":101,"share_id":"share-1519","share_seq":1519,"template_height":800047},{"accepted_at_ms":999983,"credit_policy":null,"job_id":"job-3145","job_issued_at_ms":999978,"miner_id":"miner-10","network_difficulty":1000000,"ntime":1700003145,"order_key":"10:miner-10","p2mr_program_hex":"4949494949494949494949494949494949494949494949494949494949494949","share_difficulty":1000,"share_id":"share-3145","share_seq":3145,"template_height":800098},{"accepted_at_ms":999978,"credit_policy":"stale-grace","job_id":"job-132","job_issued_at_ms":999995,"miner_id":"miner-0","network_difficulty":1000013,"ntime":1700000132,"order_key":"00:miner-0","p2mr_program_hex":"8484848484848484848484848484848484848484848484848484848484848484","share_difficulty":101,"share_id":"share-132","share_seq":132,"template_height":800004},{"accepted_at_ms":999999,"credit_policy":null,"job_id":"job-74","job_issued_at_ms":999989,"miner_id":"miner-8","network_difficulty":1000006,"ntime":1700000074,"order_key":"08:miner-8","p2mr_program_hex":"4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a","share_difficulty":19,"share_id":"share-74","share_seq":74,"template_height":800002},{"accepted_at_ms":999991,"credit_policy":"stale-grace","job_id":"job-2192","job_issued_at_ms":999998,"miner_id":"miner-3","network_difficulty":1000016,"ntime":1700002192,"order_key":"03:miner-3","p2mr_program_hex":"9090909090909090909090909090909090909090909090909090909090909090","share_difficulty":2,"share_id":"share-2192","share_seq":2192,"template_height":800068},{"accepted_at_ms":999992,"credit_policy":null,"job_id":"job-3526","job_issued_at_ms":999980,"miner_id":"miner-6","network_difficulty":1000007,"ntime":1700003526,"order_key":"06:miner-6","p2mr_program_hex":"c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6","share_difficulty":1000,"share_id":"share-3526","share_seq":3526,"template_height":800110},{"accepted_at_ms":999972,"credit_policy":null,"job_id":"job-3768","job_issued_at_ms":999976,"miner_id":"miner-6","network_difficulty":1000011,"ntime":1700003768,"order_key":"06:miner-6","p2mr_program_hex":"b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8","share_difficulty":101,"share_id":"share-3768","share_seq":3768,"template_height":800117},{"accepted_at_ms":1000052,"credit_policy":null,"job_id":"job-1471","job_issued_at_ms":999983,"miner_id":"miner-8","network_difficulty":1000009,"ntime":1700001471,"order_key":"08:miner-8","p2mr_program_hex":"bfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbf","share_difficulty":1000,"share_id":"share-1471","share_seq":1471,"template_height":800045},{"accepted_at_ms":999974,"credit_policy":null,"job_id":"job-1252","job_issued_at_ms":999970,"miner_id":"miner-9","network_difficulty":1000011,"ntime":1700001252,"order_key":"09:miner-9","p2mr_program_hex":"e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4","share_difficulty":19,"share_id":"share-1252","share_seq":1252,"template_height":800039},{"accepted_at_ms":999983,"credit_policy":null,"job_id":"job-2174","job_issued_at_ms":999970,"miner_id":"miner-7","network_difficulty":1000015,"ntime":1700002174,"order_key":"07:miner-7","p2mr_program_hex":"7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e","share_difficulty":2,"share_id":"share-2174","share_seq":2174,"template_height":800067},{"accepted_at_ms":999992,"credit_policy":null,"job_id":"job-934","job_issued_at_ms":999997,"miner_id":"miner-10","network_difficulty":1000016,"ntime":1700000934,"order_key":"10:miner-10","p2mr_program_hex":"a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6","share_difficulty":101,"share_id":"share-934","share_seq":934,"template_height":800029},{"accepted_at_ms":999984,"credit_policy":"stale-grace","job_id":"job-750","job_issued_at_ms":999985,"miner_id":"miner-2","network_difficulty":1000002,"ntime":1700000750,"order_key":"02:miner-2","p2mr_program_hex":"eeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeee","share_difficulty":1,"share_id":"share-750","share_seq":750,"template_height":800023},{"accepted_at_ms":999998,"credit_policy":null,"job_id":"job-1088","job_issued_at_ms":999987,"miner_id":"miner-10","network_difficulty":1000000,"ntime":1700001088,"order_key":"10:miner-10","p2mr_program_hex":"4040404040404040404040404040404040404040404040404040404040404040","share_difficulty":1,"share_id":"share-1088","share_seq":1088,"template_height":800034},{"accepted_at_ms":999976,"credit_policy":null,"job_id":"job-1428","job_issued_at_ms":999979,"miner_id":"miner-9","network_difficulty":1000000,"ntime":1700001428,"order_key":"09:miner-9","p2mr_program_hex":"9494949494949494949494949494949494949494949494949494949494949494","share_difficulty":101,"share_id":"share-1428","share_seq":1428,"template_height":800044},{"accepted_at_ms":999972,"credit_policy":null,"job_id":"job-1419","job_issued_at_ms":999997,"miner_id":"miner-0","network_difficulty":1000008,"ntime":1700001419,"order_key":"00:miner-0","p2mr_program_hex":"8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b","share_difficulty":1000,"share_id":"share-1419","share_seq":1419,"template_height":800044},{"accepted_at_ms":999993,"credit_policy":null,"job_id":"job-11","job_issued_at_ms":999993,"miner_id":"miner-0","network_difficulty":1000011,"ntime":1700000011,"order_key":"00:miner-0","p2mr_program_hex":"0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b","share_difficulty":101,"share_id":"share-11","share_seq":11,"template_height":800000},{"accepted_at_ms":999997,"credit_policy":null,"job_id":"job-3646","job_issued_at_ms":999982,"miner_id":"miner-5","network_difficulty":1000008,"ntime":1700003646,"order_key":"05:miner-5","p2mr_program_hex":"3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e","share_difficulty":2,"share_id":"share-3646","share_seq":3646,"template_height":800113},{"accepted_at_ms":999991,"credit_policy":null,"job_id":"job-465","job_issued_at_ms":1000444,"miner_id":"miner-3","network_difficulty":1000006,"ntime":1700000465,"order_key":"03:miner-3","p2mr_program_hex":"d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1","share_difficulty":1,"share_id":"share-465","share_seq":465,"template_height":800014},{"accepted_at_ms":999995,"credit_policy":"stale-grace","job_id":"job-3103","job_issued_at_ms":999997,"miner_id":"miner-1","network_difficulty":1000009,"ntime":1700003103,"order_key":"01:miner-1","p2mr_program_hex":"1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f","share_difficulty":3,"share_id":"share-3103","share_seq":3103,"template_height":800096},{"accepted_at_ms":999992,"credit_policy":null,"job_id":"job-1119","job_issued_at_ms":999978,"miner_id":"miner-8","network_difficulty":1000014,"ntime":1700001119,"order_key":"08:miner-8","p2mr_program_hex":"5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f","share_difficulty":101,"share_id":"share-1119","share_seq":1119,"template_height":800034},{"accepted_at_ms":999972,"credit_policy":null,"job_id":"job-2919","job_issued_at_ms":999999,"miner_id":"miner-4","network_difficulty":1000012,"ntime":1700002919,"order_key":"04:miner-4","p2mr_program_hex":"6767676767676767676767676767676767676767676767676767676767676767","share_difficulty":19,"share_id":"share-2919","share_seq":2919,"template_height":800091},{"accepted_at_ms":1000085,"credit_policy":null,"job_id":"job-904","job_issued_at_ms":1000251,"miner_id":"miner-2","network_difficulty":1000003,"ntime":1700000904,"order_key":"02:miner-2","p2mr_program_hex":"8888888888888888888888888888888888888888888888888888888888888888","share_difficulty":19,"share_id":"share-904","share_seq":904,"template_height":800028},{"accepted_at_ms":999989,"credit_policy":null,"job_id":"job-2082","job_issued_at_ms":999979,"miner_id":"miner-3","network_difficulty":1000008,"ntime":1700002082,"order_key":"03:miner-3","p2mr_program_hex":"2222222222222222222222222222222222222222222222222222222222222222","share_difficulty":1000,"share_id":"share-2082","share_seq":2082,"template_height":800065},{"accepted_at_ms":999991,"credit_policy":"stale-grace","job_id":"job-3719","job_issued_at_ms":999981,"miner_id":"miner-1","network_difficulty":1000013,"ntime":1700003719,"order_key":"01:miner-1","p2mr_program_hex":"8787878787878787878787878787878787878787878787878787878787878787","share_difficulty":3,"share_id":"share-3719","share_seq":3719,"template_height":800116},{"accepted_at_ms":999989,"credit_policy":null,"job_id":"job-319","job_issued_at_ms":999976,"miner_id":"miner-0","network_difficulty":1000013,"ntime":1700000319,"order_key":"00:miner-0","p2mr_program_hex":"3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f","share_difficulty":101,"share_id":"share-319","share_seq":319,"template_height":800009},{"accepted_at_ms":1000284,"credit_policy":null,"job_id":"job-1395","job_issued_at_ms":999988,"miner_id":"miner-9","network_difficulty":1000001,"ntime":1700001395,"order_key":"09:miner-9","p2mr_program_hex":"7373737373737373737373737373737373737373737373737373737373737373","share_difficulty":3,"share_id":"share-1395","share_seq":1395,"template_height":800043},{"accepted_at_ms":999994,"credit_policy":"stale-grace","job_id":"job-1879","job_issued_at_ms":999972,"miner_id":"miner-9","network_difficulty":1000009,"ntime":1700001879,"order_key":"09:miner-9","p2mr_program_hex":"5757575757575757575757575757575757575757575757575757575757575757","share_difficulty":19,"share_id":"share-1879","share_seq":1879,"template_height":800058},{"accepted_at_ms":1000062,"credit_policy":null,"job_id":"job-3746","job_issued_at_ms":999991,"miner_id":"miner-6","network_difficulty":1000006,"ntime":1700003746,"order_key":"06:miner-6","p2mr_program_hex":"a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2","share_difficulty":3,"share_id":"share-3746","share_seq":3746,"template_height":800117},{"accepted_at_ms":999977,"credit_policy":null,"job_id":"job-438","job_issued_at_ms":999981,"miner_id":"miner-9","network_difficulty":1000013,"ntime":1700000438,"order_key":"09:miner-9","p2mr_program_hex":"b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6","share_difficulty":3,"share_id":"share-438","share_seq":438,"template_height":800013},{"accepted_at_ms":999990,"credit_policy":null,"job_id":"job-3833","job_issued_at_ms":999988,"miner_id":"miner-5","network_difficulty":1000008,"ntime":1700003833,"order_key":"05:miner-5","p2mr_program_hex":"f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9","share_difficulty":1,"share_id":"share-3833","share_seq":3833,"template_height":800119},{"accepted_at_ms":999988,"credit_policy":null,"job_id":"job-2087","job_issued_at_ms":999999,"miner_id":"miner-8","network_difficulty":1000013,"ntime":1700002087,"order_key":"08:miner-8","p2mr_program_hex":"2727272727272727272727272727272727272727272727272727272727272727","share_difficulty":7,"share_id":"share-2087","share_seq":2087,"template_height":800065},{"accepted_at_ms":999972,"credit_policy":null,"job_id":"job-3257","job_issued_at_ms":999988,"miner_id":"miner-1","network_difficulty":1000010,"ntime":1700003257,"order_key":"01:miner-1","p2mr_program_hex":"b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9","share_difficulty":19,"share_id":"share-3257","share_seq":3257,"template_height":800101},{"accepted_at_ms":999995,"credit_policy":null,"job_id":"job-2763","job_issued_at_ms":999983,"miner_id":"miner-2","network_difficulty":1000009,"ntime":1700002763,"order_key":"02:miner-2","p2mr_program_hex":"cbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcb","share_difficulty":7,"share_id":"share-2763","share_seq":2763,"template_height":800086},{"accepted_at_ms":999981,"credit_policy":null,"job_id":"job-119","job_issued_at_ms":999975,"miner_id":"miner-9","network_difficulty":1000000,"ntime":1700000119,"order_key":"09:miner-9","p2mr_program_hex":"7777777777777777777777777777777777777777777777777777777777777777","share_difficulty":3,"share_id":"share-119","share_seq":119,"template_height":800003},{"accepted_at_ms":999998,"credit_policy":"stale-grace","job_id":"job-1855","job_issued_at_ms":999998,"miner_id":"miner-7","network_difficulty":1000002,"ntime":1700001855,"order_key":"07:miner-7","p2mr_program_hex":"3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f","share_difficulty":2,"share_id":"share-1855","share_seq":1855,"template_height":800057},{"accepted_at_ms":999970,"credit_policy":"stale-grace","job_id":"job-1964","job_issued_at_ms":999988,"miner_id":"miner-6","network_difficulty":1000009,"ntime":1700001964,"order_key":"06:miner-6","p2mr_program_hex":"acacacacacacacacacacacacacacacacacacacacacacacacacacacacacacacac","share_difficulty":1000,"share_id":"share-1964","share_seq":1964,"template_height":800061},{"accepted_at_ms":999976,"credit_policy":null,"job_id":"job-1977","job_issued_at_ms":999998,"miner_id":"miner-8","network_difficulty":1000005,"ntime":1700001977,"order_key":"08:miner-8","p2mr_program_hex":"b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9","share_difficulty":7,"share_id":"share-1977","share_seq":1977,"template_height":800061},{"accepted_at_ms":999999,"credit_policy":null,"job_id":"job-1721","job_issued_at_ms":999986,"miner_id":"miner-5","network_difficulty":1000004,"ntime":1700001721,"order_key":"05:miner-5","p2mr_program_hex":"b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9","share_difficulty":7,"share_id":"share-1721","share_seq":1721,"template_height":800053},{"accepted_at_ms":999988,"credit_policy":null,"job_id":"job-3085","job_issued_at_ms":999989,"miner_id":"miner-5","network_difficulty":1000008,"ntime":1700003085,"order_key":"05:miner-5","p2mr_program_hex":"0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d","share_difficulty":19,"share_id":"share-3085","share_seq":3085,"template_height":800096},{"accepted_at_ms":999983,"credit_policy":null,"job_id":"job-154","job_issued_at_ms":999975,"miner_id":"miner-0","network_difficulty":1000001,"ntime":1700000154,"order_key":"00:miner-0","p2mr_program_hex":"9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a","share_difficulty":101,"share_id":"share-154","share_seq":154,"template_height":800004},{"accepted_at_ms":999989,"credit_policy":"stale-grace","job_id":"job-3013","job_issued_at_ms":999997,"miner_id":"miner-10","network_difficulty":1000004,"ntime":1700003013,"order_key":"10:miner-10","p2mr_program_hex":"c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5","share_difficulty":7,"share_id":"share-3013","share_seq":3013,"template_height":800094},{"accepted_at_ms":999976,"credit_policy":null,"job_id":"job-2507","job_issued_at_ms":999989,"miner_id":"miner-10","network_difficulty":1000008,"ntime":1700002507,"order_key":"10:miner-10","p2mr_program_hex":"cbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcb","share_difficulty":19,"share_id":"share-2507","share_seq":2507,"template_height":800078},{"accepted_at_ms":999992,"credit_policy":null,"job_id":"job-3639","job_issued_at_ms":999974,"miner_id":"miner-9","network_difficulty":1000001,"ntime":1700003639,"order_key":"09:miner-9","p2mr_program_hex":"3737373737373737373737373737373737373737373737373737373737373737","share_difficulty":1,"share_id":"share-3639","share_seq":3639,"template_height":800113},{"accepted_at_ms":1000195,"credit_policy":null,"job_id":"job-1819","job_issued_at_ms":999984,"miner_id":"miner-4","network_difficulty":1000000,"ntime":1700001819,"order_key":"04:miner-4","p2mr_program_hex":"1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b","share_difficulty":3,"share_id":"share-1819","share_seq":1819,"template_height":800056},{"accepted_at_ms":999992,"credit_policy":null,"job_id":"job-3402","job_issued_at_ms":999992,"miner_id":"miner-3","network_difficulty":1000002,"ntime":1700003402,"order_key":"03:miner-3","p2mr_program_hex":"4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a","share_difficulty":7,"share_id":"share-3402","share_seq":3402,"template_height":800106},{"accepted_at_ms":999981,"credit_policy":null,"job_id":"job-727","job_issued_at_ms":1000000,"miner_id":"miner-1","network_difficulty":1000013,"ntime":1700000727,"order_key":"01:miner-1","p2mr_program_hex":"d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7","share_difficulty":3,"share_id":"share-727","share_seq":727,"template_height":800022},{"accepted_at_ms":999994,"credit_policy":null,"job_id":"job-3767","job_issued_at_ms":999971,"miner_id":"miner-5","network_difficulty":1000010,"ntime":1700003767,"order_key":"05:miner-5","p2mr_program_hex":"b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7","share_difficulty":101,"share_id":"share-3767","share_seq":3767,"template_height":800117},{"accepted_at_ms":999977,"credit_policy":null,"job_id":"job-1961","job_issued_at_ms":999981,"miner_id":"miner-3","network_difficulty":1000006,"ntime":1700001961,"order_key":"03:miner-3","p2mr_program_hex":"a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9","share_difficulty":3,"share_id":"share-1961","share_seq":1961,"template_height":800061},{"accepted_at_ms":999990,"credit_policy":null,"job_id":"job-1483","job_issued_at_ms":999982,"miner_id":"miner-9","network_difficulty":1000004,"ntime":1700001483,"order_key":"09:miner-9","p2mr_program_hex":"cbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcb","share_difficulty":101,"share_id":"share-1483","share_seq":1483,"template_height":800046},{"accepted_at_ms":999984,"credit_policy":null,"job_id":"job-2214","job_issued_at_ms":999985,"miner_id":"miner-3","network_difficulty":1000004,"ntime":1700002214,"order_key":"03:miner-3","p2mr_program_hex":"a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6","share_difficulty":1,"share_id":"share-2214","share_seq":2214,"template_height":800069},{"accepted_at_ms":999993,"credit_policy":"stale-grace","job_id":"job-752","job_issued_at_ms":999990,"miner_id":"miner-4","network_difficulty":1000004,"ntime":1700000752,"order_key":"04:miner-4","p2mr_program_hex":"f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0","share_difficulty":2,"share_id":"share-752","share_seq":752,"template_height":800023},{"accepted_at_ms":999998,"credit_policy":null,"job_id":"job-3765","job_issued_at_ms":1000000,"miner_id":"miner-3","network_difficulty":1000008,"ntime":1700003765,"order_key":"03:miner-3","p2mr_program_hex":"b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5","share_difficulty":101,"share_id":"share-3765","share_seq":3765,"template_height":800117},{"accepted_at_ms":999978,"credit_policy":"stale-grace","job_id":"job-2614","job_issued_at_ms":999990,"miner_id":"miner-7","network_difficulty":1000013,"ntime":1700002614,"order_key":"07:miner-7","p2mr_program_hex":"3636363636363636363636363636363636363636363636363636363636363636","share_difficulty":19,"share_id":"share-2614","share_seq":2614,"template_height":800081},{"accepted_at_ms":999975,"credit_policy":null,"job_id":"job-2159","job_issued_at_ms":999992,"miner_id":"miner-3","network_difficulty":1000000,"ntime":1700002159,"order_key":"03:miner-3","p2mr_program_hex":"6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f","share_difficulty":7,"share_id":"share-2159","share_seq":2159,"template_height":800067},{"accepted_at_ms":999987,"credit_policy":null,"job_id":"job-3338","job_issued_at_ms":999985,"miner_id":"miner-5","network_difficulty":1000006,"ntime":1700003338,"order_key":"05:miner-5","p2mr_program_hex":"0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a","share_difficulty":7,"share_id":"share-3338","share_seq":3338,"template_height":800104},{"accepted_at_ms":999987,"credit_policy":null,"job_id":"job-2415","job_issued_at_ms":999970,"miner_id":"miner-6","network_difficulty":1000001,"ntime":1700002415,"order_key":"06:miner-6","p2mr_program_hex":"6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f","share_difficulty":101,"share_id":"share-2415","share_seq":2415,"template_height":800075},{"accepted_at_ms":999972,"credit_policy":"stale-grace","job_id":"job-3139","job_issued_at_ms":999984,"miner_id":"miner-4","network_difficulty":1000011,"ntime":1700003139,"order_key":"04:miner-4","p2mr_program_hex":"4343434343434343434343434343434343434343434343434343434343434343","share_difficulty":1,"share_id":"share-3139","share_seq":3139,"template_height":800098},{"accepted_at_ms":999982,"credit_policy":null,"job_id":"job-458","job_issued_at_ms":1000000,"miner_id":"miner-7","network_difficulty":1000016,"ntime":1700000458,"order_key":"07:miner-7","p2mr_program_hex":"cacacacacacacacacacacacacacacacacacacacacacacacacacacacacacacaca","share_difficulty":1000,"share_id":"share-458","share_seq":458,"template_height":800014},{"accepted_at_ms":999984,"credit_policy":null,"job_id":"job-1768","job_issued_at_ms":999973,"miner_id":"miner-8","network_difficulty":1000000,"ntime":1700001768,"order_key":"08:miner-8","p2mr_program_hex":"e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8","share_difficulty":1,"share_id":"share-1768","share_seq":1768,"template_height":800055},{"accepted_at_ms":999999,"credit_policy":null,"job_id":"job-2708","job_issued_at_ms":999983,"miner_id":"miner-2","network_difficulty":1000005,"ntime":1700002708,"order_key":"02:miner-2","p2mr_program_hex":"9494949494949494949494949494949494949494949494949494949494949494","share_difficulty":1,"share_id":"share-2708","share_seq":2708,"template_height":800084},{"accepted_at_ms":999996,"credit_policy":null,"job_id":"job-3641","job_issued_at_ms":999998,"miner_id":"miner-0","network_difficulty":1000003,"ntime":1700003641,"order_key":"00:miner-0","p2mr_program_hex":"3939393939393939393939393939393939393939393939393939393939393939","share_difficulty":1000,"share_id":"share-3641","share_seq":3641,"template_height":800113},{"accepted_at_ms":999984,"credit_policy":null,"job_id":"job-2167","job_issued_at_ms":999998,"miner_id":"miner-0","network_difficulty":1000008,"ntime":1700002167,"order_key":"00:miner-0","p2mr_program_hex":"7777777777777777777777777777777777777777777777777777777777777777","share_difficulty":1,"share_id":"share-2167","share_seq":2167,"template_height":800067},{"accepted_at_ms":999993,"credit_policy":null,"job_id":"job-3914","job_issued_at_ms":999981,"miner_id":"miner-9","network_difficulty":1000004,"ntime":1700003914,"order_key":"09:miner-9","p2mr_program_hex":"4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a","share_difficulty":7,"share_id":"share-3914","share_seq":3914,"template_height":800122},{"accepted_at_ms":1000477,"credit_policy":null,"job_id":"job-2133","job_issued_at_ms":999993,"miner_id":"miner-10","network_difficulty":1000008,"ntime":1700002133,"order_key":"10:miner-10","p2mr_program_hex":"5555555555555555555555555555555555555555555555555555555555555555","share_difficulty":3,"share_id":"share-2133","share_seq":2133,"template_height":800066},{"accepted_at_ms":999993,"credit_policy":"stale-grace","job_id":"job-887","job_issued_at_ms":999971,"miner_id":"miner-7","network_difficulty":1000003,"ntime":1700000887,"order_key":"07:miner-7","p2mr_program_hex":"7777777777777777777777777777777777777777777777777777777777777777","share_difficulty":101,"share_id":"share-887","share_seq":887,"template_height":800027},{"accepted_at_ms":999972,"credit_policy":null,"job_id":"job-401","job_issued_at_ms":999978,"miner_id":"miner-5","network_difficulty":1000010,"ntime":1700000401,"order_key":"05:miner-5","p2mr_program_hex":"9191919191919191919191919191919191919191919191919191919191919191","share_difficulty":1,"share_id":"share-401","share_seq":401,"template_height":800012},{"accepted_at_ms":999996,"credit_policy":"stale-grace","job_id":"job-39","job_issued_at_ms":999990,"miner_id":"miner-6","network_difficulty":1000005,"ntime":1700000039,"order_key":"06:miner-6","p2mr_program_hex":"2727272727272727272727272727272727272727272727272727272727272727","share_difficulty":3,"share_id":"share-39","share_seq":39,"template_height":800001},{"accepted_at_ms":999989,"credit_policy":null,"job_id":"job-1644","job_issued_at_ms":999978,"miner_id":"miner-5","network_difficulty":1000012,"ntime":1700001644,"order_key":"05:miner-5","p2mr_program_hex":"6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c","share_difficulty":101,"share_id":"share-1644","share_seq":1644,"template_height":800051},{"accepted_at_ms":999989,"credit_policy":null,"job_id":"job-1053","job_issued_at_ms":999994,"miner_id":"miner-8","network_difficulty":1000016,"ntime":1700001053,"order_key":"08:miner-8","p2mr_program_hex":"1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d","share_difficulty":101,"share_id":"share-1053","share_seq":1053,"template_height":800032},{"accepted_at_ms":999983,"credit_policy":"stale-grace","job_id":"job-1127","job_issued_at_ms":999995,"miner_id":"miner-5","network_difficulty":1000005,"ntime":1700001127,"order_key":"05:miner-5","p2mr_program_hex":"6767676767676767676767676767676767676767676767676767676767676767","share_difficulty":2,"share_id":"share-1127","share_seq":1127,"template_height":800035},{"accepted_at_ms":999992,"credit_policy":null,"job_id":"job-3266","job_issued_at_ms":999975,"miner_id":"miner-10","network_difficulty":1000002,"ntime":1700003266,"order_key":"10:miner-10","p2mr_program_hex":"c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2","share_difficulty":1000,"share_id":"share-3266","share_seq":3266,"template_height":800102},{"accepted_at_ms":999979,"credit_policy":null,"job_id":"job-3000","job_issued_at_ms":999976,"miner_id":"miner-8","network_difficulty":1000008,"ntime":1700003000,"order_key":"08:miner-8","p2mr_program_hex":"b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8","share_difficulty":1,"share_id":"share-3000","share_seq":3000,"template_height":800093},{"accepted_at_ms":999984,"credit_policy":null,"job_id":"job-3157","job_issued_at_ms":999998,"miner_id":"miner-0","network_difficulty":1000012,"ntime":1700003157,"order_key":"00:miner-0","p2mr_program_hex":"5555555555555555555555555555555555555555555555555555555555555555","share_difficulty":1000,"share_id":"share-3157","share_seq":3157,"template_height":800098},{"accepted_at_ms":999970,"credit_policy":null,"job_id":"job-3420","job_issued_at_ms":1000099,"miner_id":"miner-10","network_difficulty":1000003,"ntime":1700003420,"order_key":"10:miner-10","p2mr_program_hex":"5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c","share_difficulty":7,"share_id":"share-3420","share_seq":3420,"template_height":800106},{"accepted_at_ms":999983,"credit_policy":"stale-grace","job_id":"job-3891","job_issued_at_ms":999986,"miner_id":"miner-8","network_difficulty":1000015,"ntime":1700003891,"order_key":"08:miner-8","p2mr_program_hex":"3333333333333333333333333333333333333333333333333333333333333333","share_difficulty":19,"share_id":"share-3891","share_seq":3891,"template_height":800121},{"accepted_at_ms":999987,"credit_policy":null,"job_id":"job-854","job_issued_at_ms":999988,"miner_id":"miner-7","network_difficulty":1000004,"ntime":1700000854,"order_key":"07:miner-7","p2mr_program_hex":"5656565656565656565656565656565656565656565656565656565656565656","share_difficulty":1,"share_id":"share-854","share_seq":854,"template_height":800026},{"accepted_at_ms":999971,"credit_policy":null,"job_id":"job-3858","job_issued_at_ms":1000286,"miner_id":"miner-8","network_difficulty":1000016,"ntime":1700003858,"order_key":"08:miner-8","p2mr_program_hex":"1212121212121212121212121212121212121212121212121212121212121212","share_difficulty":1,"share_id":"share-3858","share_seq":3858,"template_height":800120},{"accepted_at_ms":999983,"credit_policy":null,"job_id":"job-2544","job_issued_at_ms":999976,"miner_id":"miner-3","network_difficulty":1000011,"ntime":1700002544,"order_key":"03:miner-3","p2mr_program_hex":"f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0","share_difficulty":3,"share_id":"share-2544","share_seq":2544,"template_height":800079},{"accepted_at_ms":999998,"credit_policy":"stale-grace","job_id":"job-1873","job_issued_at_ms":999999,"miner_id":"miner-3","network_difficulty":1000003,"ntime":1700001873,"order_key":"03:miner-3","p2mr_program_hex":"5151515151515151515151515151515151515151515151515151515151515151","share_difficulty":7,"share_id":"share-1873","share_seq":1873,"template_height":800058},{"accepted_at_ms":999970,"credit_policy":null,"job_id":"job-2785","job_issued_at_ms":999997,"miner_id":"miner-2","network_difficulty":1000014,"ntime":1700002785,"order_key":"02:miner-2","p2mr_program_hex":"e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1","share_difficulty":1000,"share_id":"share-2785","share_seq":2785,"template_height":800087},{"accepted_at_ms":999980,"credit_policy":"stale-grace","job_id":"job-1077","job_issued_at_ms":999991,"miner_id":"miner-10","network_difficulty":1000006,"ntime":1700001077,"order_key":"10:miner-10","p2mr_program_hex":"3535353535353535353535353535353535353535353535353535353535353535","share_difficulty":2,"share_id":"share-1077","share_seq":1077,"template_height":800033},{"accepted_at_ms":999974,"credit_policy":null,"job_id":"job-2964","job_issued_at_ms":999996,"miner_id":"miner-5","network_difficulty":1000006,"ntime":1700002964,"order_key":"05:miner-5","p2mr_program_hex":"9494949494949494949494949494949494949494949494949494949494949494","share_difficulty":101,"share_id":"share-2964","share_seq":2964,"template_height":800092},{"accepted_at_ms":999983,"credit_policy":null,"job_id":"job-2717","job_issued_at_ms":999978,"miner_id":"miner-0","network_difficulty":1000014,"ntime":1700002717,"order_key":"00:miner-0","p2mr_program_hex":"9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d","share_difficulty":101,"share_id":"share-2717","share_seq":2717,"template_height":800084},{"accepted_at_ms":999984,"credit_policy":null,"job_id":"job-1741","job_issued_at_ms":999974,"miner_id":"miner-3","network_difficulty":1000007,"ntime":1700001741,"order_key":"03:miner-3","p2mr_program_hex":"cdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcd","share_difficulty":1,"share_id":"share-1741","share_seq":1741,"template_height":800054},{"accepted_at_ms":1000135,"credit_policy":null,"job_id":"job-2761","job_issued_at_ms":1000409,"miner_id":"miner-0","network_difficulty":1000007,"ntime":1700002761,"order_key":"00:miner-0","p2mr_program_hex":"c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9","share_difficulty":2,"share_id":"share-2761","share_seq":2761,"template_height":800086},{"accepted_at_ms":999977,"credit_policy":null,"job_id":"job-492","job_issued_at_ms":999980,"miner_id":"miner-8","network_difficulty":1000016,"ntime":1700000492,"order_key":"08:miner-8","p2mr_program_hex":"ecececececececececececececececececececececececececececececececec","share_difficulty":3,"share_id":"share-492","share_seq":492,"template_height":800015},{"accepted_at_ms":999988,"credit_policy":"stale-grace","job_id":"job-2350","job_issued_at_ms":1000000,"miner_id":"miner-7","network_difficulty":1000004,"ntime":1700002350,"order_key":"07:miner-7","p2mr_program_hex":"2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e","share_difficulty":19,"share_id":"share-2350","share_seq":2350,"template_height":800073},{"accepted_at_ms":999999,"credit_policy":"stale-grace","job_id":"job-2946","job_issued_at_ms":999995,"miner_id":"miner-9","network_difficulty":1000005,"ntime":1700002946,"order_key":"09:miner-9","p2mr_program_hex":"8282828282828282828282828282828282828282828282828282828282828282","share_difficulty":2,"share_id":"share-2946","share_seq":2946,"template_height":800092},{"accepted_at_ms":999972,"credit_policy":null,"job_id":"job-946","job_issued_at_ms":999994,"miner_id":"miner-0","network_difficulty":1000011,"ntime":1700000946,"order_key":"00:miner-0","p2mr_program_hex":"b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2","share_difficulty":7,"share_id":"share-946","share_seq":946,"template_height":800029},{"accepted_at_ms":999995,"credit_policy":null,"job_id":"job-2685","job_issued_at_ms":999979,"miner_id":"miner-1","network_difficulty":1000016,"ntime":1700002685,"order_key":"01:miner-1","p2mr_program_hex":"7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d","share_difficulty":7,"share_id":"share-2685","share_seq":2685,"template_height":800083},{"accepted_at_ms":999971,"credit_policy":null,"job_id":"job-1903","job_issued_at_ms":999988,"miner_id":"miner-0","network_difficulty":1000016,"ntime":1700001903,"order_key":"00:miner-0","p2mr_program_hex":"6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f","share_difficulty":1,"share_id":"share-1903","share_seq":1903,"template_height":800059},{"accepted_at_ms":999987,"credit_policy":null,"job_id":"job-746","job_issued_at_ms":999993,"miner_id":"miner-9","network_difficulty":1000015,"ntime":1700000746,"order_key":"09:miner-9","p2mr_program_hex":"eaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaea","share_difficulty":2,"share_id":"share-746","share_seq":746,"template_height":800023},{"accepted_at_ms":999990,"credit_policy":null,"job_id":"job-1033","job_issued_at_ms":999970,"miner_id":"miner-10","network_difficulty":1000013,"ntime":1700001033,"order_key":"10:miner-10","p2mr_program_hex":"0909090909090909090909090909090909090909090909090909090909090909","share_difficulty":3,"share_id":"share-1033","share_seq":1033,"template_height":800032},{"accepted_at_ms":999980,"credit_policy":null,"job_id":"job-1759","job_issued_at_ms":999982,"miner_id":"miner-10","network_difficulty":1000008,"ntime":1700001759,"order_key":"10:miner-10","p2mr_program_hex":"dfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdf","share_difficulty":101,"share_id":"share-1759","share_seq":1759,"template_height":800054},{"accepted_at_ms":999983,"credit_policy":null,"job_id":"job-3538","job_issued_at_ms":999971,"miner_id":"miner-7","network_difficulty":1000002,"ntime":1700003538,"order_key":"07:miner-7","p2mr_program_hex":"d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2","share_difficulty":19,"share_id":"share-3538","share_seq":3538,"template_height":800110},{"accepted_at_ms":999994,"credit_policy":"stale-grace","job_id":"job-973","job_issued_at_ms":999999,"miner_id":"miner-5","network_difficulty":1000004,"ntime":1700000973,"order_key":"05:miner-5","p2mr_program_hex":"cdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcd","share_difficulty":1,"share_id":"share-973","share_seq":973,"template_height":800030},{"accepted_at_ms":999985,"credit_policy":null,"job_id":"job-2028","job_issued_at_ms":999993,"miner_id":"miner-4","network_difficulty":1000005,"ntime":1700002028,"order_key":"04:miner-4","p2mr_program_hex":"ecececececececececececececececececececececececececececececececec","share_difficulty":101,"share_id":"share-2028","share_seq":2028,"template_height":800063},{"accepted_at_ms":999970,"credit_policy":null,"job_id":"job-3007","job_issued_at_ms":999995,"miner_id":"miner-4","network_difficulty":1000015,"ntime":1700003007,"order_key":"04:miner-4","p2mr_program_hex":"bfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbf","share_difficulty":3,"share_id":"share-3007","share_seq":3007,"template_height":800093},{"accepted_at_ms":999998,"credit_policy":null,"job_id":"job-1936","job_issued_at_ms":999996,"miner_id":"miner-0","network_difficulty":1000015,"ntime":1700001936,"order_key":"00:miner-0","p2mr_program_hex":"9090909090909090909090909090909090909090909090909090909090909090","share_difficulty":3,"share_id":"share-1936","share_seq":1936,"template_height":800060},{"accepted_at_ms":999973,"credit_policy":"stale-grace","job_id":"job-69","job_issued_at_ms":999992,"miner_id":"miner-3","network_difficulty":1000001,"ntime":1700000069,"order_key":"03:miner-3","p2mr_program_hex":"4545454545454545454545454545454545454545454545454545454545454545","share_difficulty":19,"share_id":"share-69","share_seq":69,"template_height":800002},{"accepted_at_ms":999989,"credit_policy":null,"job_id":"job-2446","job_issued_at_ms":999980,"miner_id":"miner-4","network_difficulty":1000015,"ntime":1700002446,"order_key":"04:miner-4","p2mr_program_hex":"8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e","share_difficulty":19,"share_id":"share-2446","share_seq":2446,"template_height":800076},{"accepted_at_ms":999999,"credit_policy":null,"job_id":"job-2893","job_issued_at_ms":999983,"miner_id":"miner-0","network_difficulty":1000003,"ntime":1700002893,"order_key":"00:miner-0","p2mr_program_hex":"4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d","share_difficulty":2,"share_id":"share-2893","share_seq":2893,"template_height":800090},{"accepted_at_ms":999974,"credit_policy":null,"job_id":"job-2884","job_issued_at_ms":999983,"miner_id":"miner-2","network_difficulty":1000011,"ntime":1700002884,"order_key":"02:miner-2","p2mr_program_hex":"4444444444444444444444444444444444444444444444444444444444444444","share_difficulty":101,"share_id":"share-2884","share_seq":2884,"template_height":800090},{"accepted_at_ms":999983,"credit_policy":null,"job_id":"job-2993","job_issued_at_ms":999986,"miner_id":"miner-1","network_difficulty":1000001,"ntime":1700002993,"order_key":"01:miner-1","p2mr_program_hex":"b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1","share_difficulty":1,"share_id":"share-2993","share_seq":2993,"template_height":800093},{"accepted_at_ms":999999,"credit_policy":null,"job_id":"job-2494","job_issued_at_ms":999974,"miner_id":"miner-8","network_difficulty":1000012,"ntime":1700002494,"order_key":"08:miner-8","p2mr_program_hex":"bebebebebebebebebebebebebebebebebebebebebebebebebebebebebebebebe","share_difficulty":1000,"share_id":"share-2494","share_seq":2494,"template_height":800077},{"accepted_at_ms":1000000,"credit_policy":null,"job_id":"job-3506","job_issued_at_ms":999988,"miner_id":"miner-8","network_difficulty":1000004,"ntime":1700003506,"order_key":"08:miner-8","p2mr_program_hex":"b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2","share_difficulty":1000,"share_id":"share-3506","share_seq":3506,"template_height":800109},{"accepted_at_ms":999989,"credit_policy":null,"job_id":"job-2661","job_issued_at_ms":999991,"miner_id":"miner-10","network_difficulty":1000009,"ntime":1700002661,"order_key":"10:miner-10","p2mr_program_hex":"6565656565656565656565656565656565656565656565656565656565656565","share_difficulty":19,"share_id":"share-2661","share_seq":2661,"template_height":800083},{"accepted_at_ms":999995,"credit_policy":null,"job_id":"job-731","job_issued_at_ms":999986,"miner_id":"miner-5","network_difficulty":1000000,"ntime":1700000731,"order_key":"05:miner-5","p2mr_program_hex":"dbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdb","share_difficulty":1000,"share_id":"share-731","share_seq":731,"template_height":800022},{"accepted_at_ms":999970,"credit_policy":null,"job_id":"job-3382","job_issued_at_ms":1000000,"miner_id":"miner-5","network_difficulty":1000016,"ntime":1700003382,"order_key":"05:miner-5","p2mr_program_hex":"3636363636363636363636363636363636363636363636363636363636363636","share_difficulty":2,"share_id":"share-3382","share_seq":3382,"template_height":800105},{"accepted_at_ms":999999,"credit_policy":null,"job_id":"job-767","job_issued_at_ms":999994,"miner_id":"miner-8","network_difficulty":1000002,"ntime":1700000767,"order_key":"08:miner-8","p2mr_program_hex":"ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff","share_difficulty":3,"share_id":"share-767","share_seq":767,"template_height":800023},{"accepted_at_ms":999992,"credit_policy":null,"job_id":"job-1598","job_issued_at_ms":1000000,"miner_id":"miner-3","network_difficulty":1000000,"ntime":1700001598,"order_key":"03:miner-3","p2mr_program_hex":"3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e","share_difficulty":7,"share_id":"share-1598","share_seq":1598,"template_height":800049},{"accepted_at_ms":999978,"credit_policy":null,"job_id":"job-817","job_issued_at_ms":999983,"miner_id":"miner-3","network_difficulty":1000001,"ntime":1700000817,"order_key":"03:miner-3","p2mr_program_hex":"3131313131313131313131313131313131313131313131313131313131313131","share_difficulty":2,"share_id":"share-817","share_seq":817,"template_height":800025},{"accepted_at_ms":999981,"credit_policy":null,"job_id":"job-3977","job_issued_at_ms":1000338,"miner_id":"miner-6","network_difficulty":1000016,"ntime":1700003977,"order_key":"06:miner-6","p2mr_program_hex":"8989898989898989898989898989898989898989898989898989898989898989","share_difficulty":3,"share_id":"share-3977","share_seq":3977,"template_height":800124},{"accepted_at_ms":999998,"credit_policy":null,"job_id":"job-1181","job_issued_at_ms":999977,"miner_id":"miner-4","network_difficulty":1000008,"ntime":1700001181,"order_key":"04:miner-4","p2mr_program_hex":"9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d","share_difficulty":7,"share_id":"share-1181","share_seq":1181,"template_height":800036},{"accepted_at_ms":999998,"credit_policy":null,"job_id":"job-1929","job_issued_at_ms":999976,"miner_id":"miner-4","network_difficulty":1000008,"ntime":1700001929,"order_key":"04:miner-4","p2mr_program_hex":"8989898989898989898989898989898989898989898989898989898989898989","share_difficulty":1000,"share_id":"share-1929","share_seq":1929,"template_height":800060},{"accepted_at_ms":999995,"credit_policy":null,"job_id":"job-3135","job_issued_at_ms":999993,"miner_id":"miner-0","network_difficulty":1000007,"ntime":1700003135,"order_key":"00:miner-0","p2mr_program_hex":"3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f","share_difficulty":1000,"share_id":"share-3135","share_seq":3135,"template_height":800097},{"accepted_at_ms":999976,"credit_policy":null,"job_id":"job-3217","job_issued_at_ms":999983,"miner_id":"miner-5","network_difficulty":1000004,"ntime":1700003217,"order_key":"05:miner-5","p2mr_program_hex":"9191919191919191919191919191919191919191919191919191919191919191","share_difficulty":3,"share_id":"share-3217","share_seq":3217,"template_height":800100},{"accepted_at_ms":999986,"credit_policy":null,"job_id":"job-3934","job_issued_at_ms":999974,"miner_id":"miner-7","network_difficulty":1000007,"ntime":1700003934,"order_key":"07:miner-7","p2mr_program_hex":"5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e","share_difficulty":101,"share_id":"share-3934","share_seq":3934,"template_height":800122},{"accepted_at_ms":999986,"credit_policy":null,"job_id":"job-1039","job_issued_at_ms":999987,"miner_id":"miner-5","network_difficulty":1000002,"ntime":1700001039,"order_key":"05:miner-5","p2mr_program_hex":"0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f","share_difficulty":3,"share_id":"share-1039","share_seq":1039,"template_height":800032},{"accepted_at_ms":999993,"credit_policy":null,"job_id":"job-2645","job_issued_at_ms":999990,"miner_id":"miner-5","network_difficulty":1000010,"ntime":1700002645,"order_key":"05:miner-5","p2mr_program_hex":"5555555555555555555555555555555555555555555555555555555555555555","share_difficulty":1000,"share_id":"share-2645","share_seq":2645,"template_height":800082},{"accepted_at_ms":999975,"credit_policy":"stale-grace","job_id":"job-5","job_issued_at_ms":999998,"miner_id":"miner-5","network_difficulty":1000005,"ntime":1700000005,"order_key":"05:miner-5","p2mr_program_hex":"0505050505050505050505050505050505050505050505050505050505050505","share_difficulty":7,"share_id":"share-5","share_seq":5,"template_height":800000},{"accepted_at_ms":999987,"credit_policy":null,"job_id":"job-1782","job_issued_at_ms":999998,"miner_id":"miner-0","network_difficulty":1000014,"ntime":1700001782,"order_key":"00:miner-0","p2mr_program_hex":"f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6","share_difficulty":2,"share_id":"share-1782","share_seq":1782,"template_height":800055},{"accepted_at_ms":999986,"credit_policy":null,"job_id":"job-3388","job_issued_at_ms":999988,"miner_id":"miner-0","network_difficulty":1000005,"ntime":1700003388,"order_key":"00:miner-0","p2mr_program_hex":"3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c","share_difficulty":19,"share_id":"share-3388","share_seq":3388,"template_height":800105},{"accepted_at_ms":999977,"credit_policy":null,"job_id":"job-1213","job_issued_at_ms":999992,"miner_id":"miner-3","network_difficulty":1000006,"ntime":1700001213,"order_key":"03:miner-3","p2mr_program_hex":"bdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbd","share_difficulty":7,"share_id":"share-1213","share_seq":1213,"template_height":800037},{"accepted_at_ms":999991,"credit_policy":null,"job_id":"job-1991","job_issued_at_ms":999991,"miner_id":"miner-0","network_difficulty":1000002,"ntime":1700001991,"order_key":"00:miner-0","p2mr_program_hex":"c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7","share_difficulty":2,"share_id":"share-1991","share_seq":1991,"template_height":800062},{"accepted_at_ms":999991,"credit_policy":null,"job_id":"job-158","job_issued_at_ms":999971,"miner_id":"miner-4","network_difficulty":1000005,"ntime":1700000158,"order_key":"04:miner-4","p2mr_program_hex":"9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e","share_difficulty":7,"share_id":"share-158","share_seq":158,"template_height":800004},{"accepted_at_ms":1000073,"credit_policy":null,"job_id":"job-3098","job_issued_at_ms":1000382,"miner_id":"miner-7","network_difficulty":1000004,"ntime":1700003098,"order_key":"07:miner-7","p2mr_program_hex":"1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a","share_difficulty":7,"share_id":"share-3098","share_seq":3098,"template_height":800096},{"accepted_at_ms":999985,"credit_policy":null,"job_id":"job-1673","job_issued_at_ms":999994,"miner_id":"miner-1","network_difficulty":1000007,"ntime":1700001673,"order_key":"01:miner-1","p2mr_program_hex":"8989898989898989898989898989898989898989898989898989898989898989","share_difficulty":2,"share_id":"share-1673","share_seq":1673,"template_height":800052},{"accepted_at_ms":999997,"credit_policy":null,"job_id":"job-736","job_issued_at_ms":1000000,"miner_id":"miner-10","network_difficulty":1000005,"ntime":1700000736,"order_key":"10:miner-10","p2mr_program_hex":"e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0","share_difficulty":7,"share_id":"share-736","share_seq":736,"template_height":800023},{"accepted_at_ms":999984,"credit_policy":null,"job_id":"job-383","job_issued_at_ms":999972,"miner_id":"miner-9","network_difficulty":1000009,"ntime":1700000383,"order_key":"09:miner-9","p2mr_program_hex":"7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f","share_difficulty":2,"share_id":"share-383","share_seq":383,"template_height":800011},{"accepted_at_ms":999980,"credit_policy":null,"job_id":"job-3113","job_issued_at_ms":999995,"miner_id":"miner-0","network_difficulty":1000002,"ntime":1700003113,"order_key":"00:miner-0","p2mr_program_hex":"2929292929292929292929292929292929292929292929292929292929292929","share_difficulty":19,"share_id":"share-3113","share_seq":3113,"template_height":800097},{"accepted_at_ms":999970,"credit_policy":null,"job_id":"job-510","job_issued_at_ms":999990,"miner_id":"miner-4","network_difficulty":1000000,"ntime":1700000510,"order_key":"04:miner-4","p2mr_program_hex":"fefefefefefefefefefefefefefefefefefefefefefefefefefefefefefefefe","share_difficulty":1,"share_id":"share-510","share_seq":510,"template_height":800015},{"accepted_at_ms":999983,"credit_policy":null,"job_id":"job-2850","job_issued_at_ms":999985,"miner_id":"miner-1","network_difficulty":1000011,"ntime":1700002850,"order_key":"01:miner-1","p2mr_program_hex":"2222222222222222222222222222222222222222222222222222222222222222","share_difficulty":2,"share_id":"share-2850","share_seq":2850,"template_height":800089},{"accepted_at_ms":999988,"credit_policy":null,"job_id":"job-2096","job_issued_at_ms":999988,"miner_id":"miner-6","network_difficulty":1000005,"ntime":1700002096,"order_key":"06:miner-6","p2mr_program_hex":"3030303030303030303030303030303030303030303030303030303030303030","share_difficulty":2,"share_id":"share-2096","share_seq":2096,"template_height":800065},{"accepted_at_ms":999976,"credit_policy":null,"job_id":"job-1540","job_issued_at_ms":999974,"miner_id":"miner-0","network_difficulty":1000010,"ntime":1700001540,"order_key":"00:miner-0","p2mr_program_hex":"0404040404040404040404040404040404040404040404040404040404040404","share_difficulty":1,"share_id":"share-1540","share_seq":1540,"template_height":800048},{"accepted_at_ms":999985,"credit_policy":null,"job_id":"job-1962","job_issued_at_ms":999981,"miner_id":"miner-4","network_difficulty":1000007,"ntime":1700001962,"order_key":"04:miner-4","p2mr_program_hex":"aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa","share_difficulty":7,"share_id":"share-1962","share_seq":1962,"template_height":800061},{"accepted_at_ms":999987,"credit_policy":null,"job_id":"job-3001","job_issued_at_ms":999979,"miner_id":"miner-9","network_difficulty":1000009,"ntime":1700003001,"order_key":"09:miner-9","p2mr_program_hex":"b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9","share_difficulty":3,"share_id":"share-3001","share_seq":3001,"template_height":800093},{"accepted_at_ms":999980,"credit_policy":null,"job_id":"job-2652","job_issued_at_ms":999976,"miner_id":"miner-1","network_difficulty":1000000,"ntime":1700002652,"order_key":"01:miner-1","p2mr_program_hex":"5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c","share_difficulty":1,"share_id":"share-2652","share_seq":2652,"template_height":800082},{"accepted_at_ms":999988,"credit_policy":null,"job_id":"job-1327","job_issued_at_ms":1000466,"miner_id":"miner-7","network_difficulty":1000001,"ntime":1700001327,"order_key":"07:miner-7","p2mr_program_hex":"2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f","share_difficulty":19,"share_id":"share-1327","share_seq":1327,"template_height":800041},{"accepted_at_ms":1000355,"credit_policy":null,"job_id":"job-466","job_issued_at_ms":999999,"miner_id":"miner-4","network_difficulty":1000007,"ntime":1700000466,"order_key":"04:miner-4","p2mr_program_hex":"d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2","share_difficulty":2,"share_id":"share-466","share_seq":466,"template_height":800014},{"accepted_at_ms":1000279,"credit_policy":null,"job_id":"job-2268","job_issued_at_ms":999988,"miner_id":"miner-2","network_difficulty":1000007,"ntime":1700002268,"order_key":"02:miner-2","p2mr_program_hex":"dcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdc","share_difficulty":101,"share_id":"share-2268","share_seq":2268,"template_height":800070},{"accepted_at_ms":999982,"credit_policy":null,"job_id":"job-1504","job_issued_at_ms":999981,"miner_id":"miner-8","network_difficulty":1000008,"ntime":1700001504,"order_key":"08:miner-8","p2mr_program_hex":"e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0","share_difficulty":3,"share_id":"share-1504","share_seq":1504,"template_height":800047},{"accepted_at_ms":999990,"credit_policy":null,"job_id":"job-235","job_issued_at_ms":999999,"miner_id":"miner-4","network_difficulty":1000014,"ntime":1700000235,"order_key":"04:miner-4","p2mr_program_hex":"ebebebebebebebebebebebebebebebebebebebebebebebebebebebebebebebeb","share_difficulty":19,"share_id":"share-235","share_seq":235,"template_height":800007},{"accepted_at_ms":999986,"credit_policy":"stale-grace","job_id":"job-874","job_issued_at_ms":999985,"miner_id":"miner-5","network_difficulty":1000007,"ntime":1700000874,"order_key":"05:miner-5","p2mr_program_hex":"6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a","share_difficulty":1,"share_id":"share-874","share_seq":874,"template_height":800027},{"accepted_at_ms":999999,"credit_policy":null,"job_id":"job-2163","job_issued_at_ms":999986,"miner_id":"miner-7","network_difficulty":1000004,"ntime":1700002163,"order_key":"07:miner-7","p2mr_program_hex":"7373737373737373737373737373737373737373737373737373737373737373","share_difficulty":1,"share_id":"share-2163","share_seq":2163,"template_height":800067},{"accepted_at_ms":999988,"credit_policy":"stale-grace","job_id":"job-3306","job_issued_at_ms":1000196,"miner_id":"miner-6","network_difficulty":1000008,"ntime":1700003306,"order_key":"06:miner-6","p2mr_program_hex":"eaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaea","share_difficulty":101,"share_id":"share-3306","share_seq":3306,"template_height":800103},{"accepted_at_ms":999979,"credit_policy":null,"job_id":"job-517","job_issued_at_ms":999999,"miner_id":"miner-0","network_difficulty":1000007,"ntime":1700000517,"order_key":"00:miner-0","p2mr_program_hex":"0505050505050505050505050505050505050505050505050505050505050505","share_difficulty":3,"share_id":"share-517","share_seq":517,"template_height":800016},{"accepted_at_ms":999973,"credit_policy":null,"job_id":"job-2968","job_issued_at_ms":999976,"miner_id":"miner-9","network_difficulty":1000010,"ntime":1700002968,"order_key":"09:miner-9","p2mr_program_hex":"9898989898989898989898989898989898989898989898989898989898989898","share_difficulty":7,"share_id":"share-2968","share_seq":2968,"template_height":800092},{"accepted_at_ms":1000000,"credit_policy":"stale-grace","job_id":"job-2089","job_issued_at_ms":999980,"miner_id":"miner-10","network_difficulty":1000015,"ntime":1700002089,"order_key":"10:miner-10","p2mr_program_hex":"2929292929292929292929292929292929292929292929292929292929292929","share_difficulty":19,"share_id":"share-2089","share_seq":2089,"template_height":800065},{"accepted_at_ms":999997,"credit_policy":null,"job_id":"job-3127","job_issued_at_ms":999973,"miner_id":"miner-3","network_difficulty":1000016,"ntime":1700003127,"order_key":"03:miner-3","p2mr_program_hex":"3737373737373737373737373737373737373737373737373737373737373737","share_difficulty":3,"share_id":"share-3127","share_seq":3127,"template_height":800097},{"accepted_at_ms":999997,"credit_policy":null,"job_id":"job-2236","job_issued_at_ms":999970,"miner_id":"miner-3","network_difficulty":1000009,"ntime":1700002236,"order_key":"03:miner-3","p2mr_program_hex":"bcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbc","share_difficulty":101,"share_id":"share-2236","share_seq":2236,"template_height":800069},{"accepted_at_ms":999983,"credit_policy":null,"job_id":"job-2033","job_issued_at_ms":999975,"miner_id":"miner-9","network_difficulty":1000010,"ntime":1700002033,"order_key":"09:miner-9","p2mr_program_hex":"f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1","share_difficulty":7,"share_id":"share-2033","share_seq":2033,"template_height":800063},{"accepted_at_ms":999990,"credit_policy":"stale-grace","job_id":"job-3810","job_issued_at_ms":999975,"miner_id":"miner-4","network_difficulty":1000002,"ntime":1700003810,"order_key":"04:miner-4","p2mr_program_hex":"e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2","share_difficulty":19,"share_id":"share-3810","share_seq":3810,"template_height":800119},{"accepted_at_ms":999988,"credit_policy":"stale-grace","job_id":"job-2767","job_issued_at_ms":999975,"miner_id":"miner-6","network_difficulty":1000013,"ntime":1700002767,"order_key":"06:miner-6","p2mr_program_hex":"cfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcf","share_difficulty":3,"share_id":"share-2767","share_seq":2767,"template_height":800086},{"accepted_at_ms":999974,"credit_policy":"stale-grace","job_id":"job-3653","job_issued_at_ms":1000000,"miner_id":"miner-1","network_difficulty":1000015,"ntime":1700003653,"order_key":"01:miner-1","p2mr_program_hex":"4545454545454545454545454545454545454545454545454545454545454545","share_difficulty":2,"share_id":"share-3653","share_seq":3653,"template_height":800114},{"accepted_at_ms":999973,"credit_policy":null,"job_id":"job-308","job_issued_at_ms":999992,"miner_id":"miner-0","network_difficulty":1000002,"ntime":1700000308,"order_key":"00:miner-0","p2mr_program_hex":"3434343434343434343434343434343434343434343434343434343434343434","share_difficulty":2,"share_id":"share-308","share_seq":308,"template_height":800009},{"accepted_at_ms":999973,"credit_policy":"stale-grace","job_id":"job-866","job_issued_at_ms":999977,"miner_id":"miner-8","network_difficulty":1000016,"ntime":1700000866,"order_key":"08:miner-8","p2mr_program_hex":"6262626262626262626262626262626262626262626262626262626262626262","share_difficulty":7,"share_id":"share-866","share_seq":866,"template_height":800027},{"accepted_at_ms":999995,"credit_policy":null,"job_id":"job-3667","job_issued_at_ms":999974,"miner_id":"miner-4","network_difficulty":1000012,"ntime":1700003667,"order_key":"04:miner-4","p2mr_program_hex":"5353535353535353535353535353535353535353535353535353535353535353","share_difficulty":101,"share_id":"share-3667","share_seq":3667,"template_height":800114},{"accepted_at_ms":999982,"credit_policy":null,"job_id":"job-529","job_issued_at_ms":999976,"miner_id":"miner-1","network_difficulty":1000002,"ntime":1700000529,"order_key":"01:miner-1","p2mr_program_hex":"1111111111111111111111111111111111111111111111111111111111111111","share_difficulty":1000,"share_id":"share-529","share_seq":529,"template_height":800016},{"accepted_at_ms":1000166,"credit_policy":null,"job_id":"job-63","job_issued_at_ms":1000442,"miner_id":"miner-8","network_difficulty":1000012,"ntime":1700000063,"order_key":"08:miner-8","p2mr_program_hex":"3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f","share_difficulty":3,"share_id":"share-63","share_seq":63,"template_height":800001},{"accepted_at_ms":1000251,"credit_policy":"stale-grace","job_id":"job-986","job_issued_at_ms":1000023,"miner_id":"miner-7","network_difficulty":1000000,"ntime":1700000986,"order_key":"07:miner-7","p2mr_program_hex":"dadadadadadadadadadadadadadadadadadadadadadadadadadadadadadadada","share_difficulty":1,"share_id":"share-986","share_seq":986,"template_height":800030},{"accepted_at_ms":1000294,"credit_policy":null,"job_id":"job-3980","job_issued_at_ms":1000154,"miner_id":"miner-9","network_difficulty":1000002,"ntime":1700003980,"order_key":"09:miner-9","p2mr_program_hex":"8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c","share_difficulty":1,"share_id":"share-3980","share_seq":3980,"template_height":800124},{"accepted_at_ms":999988,"credit_policy":null,"job_id":"job-3248","job_issued_at_ms":999987,"miner_id":"miner-3","network_difficulty":1000001,"ntime":1700003248,"order_key":"03:miner-3","p2mr_program_hex":"b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0","share_difficulty":3,"share_id":"share-3248","share_seq":3248,"template_height":800101},{"accepted_at_ms":999997,"credit_policy":null,"job_id":"job-464","job_issued_at_ms":999988,"miner_id":"miner-2","network_difficulty":1000005,"ntime":1700000464,"order_key":"02:miner-2","p2mr_program_hex":"d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0","share_difficulty":3,"share_id":"share-464","share_seq":464,"template_height":800014},{"accepted_at_ms":999985,"credit_policy":null,"job_id":"job-1336","job_issued_at_ms":999976,"miner_id":"miner-5","network_difficulty":1000010,"ntime":1700001336,"order_key":"05:miner-5","p2mr_program_hex":"3838383838383838383838383838383838383838383838383838383838383838","share_difficulty":2,"share_id":"share-1336","share_seq":1336,"template_height":800041},{"accepted_at_ms":999994,"credit_policy":null,"job_id":"job-1407","job_issued_at_ms":999983,"miner_id":"miner-10","network_difficulty":1000013,"ntime":1700001407,"order_key":"10:miner-10","p2mr_program_hex":"7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f","share_difficulty":101,"share_id":"share-1407","share_seq":1407,"template_height":800043},{"accepted_at_ms":999984,"credit_policy":null,"job_id":"job-266","job_issued_at_ms":999974,"miner_id":"miner-2","network_difficulty":1000011,"ntime":1700000266,"order_key":"02:miner-2","p2mr_program_hex":"0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a","share_difficulty":7,"share_id":"share-266","share_seq":266,"template_height":800008},{"accepted_at_ms":999990,"credit_policy":null,"job_id":"job-1300","job_issued_at_ms":999972,"miner_id":"miner-2","network_difficulty":1000008,"ntime":1700001300,"order_key":"02:miner-2","p2mr_program_hex":"1414141414141414141414141414141414141414141414141414141414141414","share_difficulty":101,"share_id":"share-1300","share_seq":1300,"template_height":800040},{"accepted_at_ms":999973,"credit_policy":null,"job_id":"job-3194","job_issued_at_ms":1000000,"miner_id":"miner-4","network_difficulty":1000015,"ntime":1700003194,"order_key":"04:miner-4","p2mr_program_hex":"7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a","share_difficulty":1,"share_id":"share-3194","share_seq":3194,"template_height":800099},{"accepted_at_ms":999994,"credit_policy":null,"job_id":"job-601","job_issued_at_ms":1000000,"miner_id":"miner-7","network_difficulty":1000006,"ntime":1700000601,"order_key":"07:miner-7","p2mr_program_hex":"5959595959595959595959595959595959595959595959595959595959595959","share_difficulty":1000,"share_id":"share-601","share_seq":601,"template_height":800018},{"accepted_at_ms":999970,"credit_policy":null,"job_id":"job-810","job_issued_at_ms":999987,"miner_id":"miner-7","network_difficulty":1000011,"ntime":1700000810,"order_key":"07:miner-7","p2mr_program_hex":"2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a","share_difficulty":3,"share_id":"share-810","share_seq":810,"template_height":800025},{"accepted_at_ms":999998,"credit_policy":null,"job_id":"job-2022","job_issued_at_ms":999986,"miner_id":"miner-9","network_difficulty":1000016,"ntime":1700002022,"order_key":"09:miner-9","p2mr_program_hex":"e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6","share_difficulty":3,"share_id":"share-2022","share_seq":2022,"template_height":800063},{"accepted_at_ms":999976,"credit_policy":null,"job_id":"job-3152","job_issued_at_ms":999978,"miner_id":"miner-6","network_difficulty":1000007,"ntime":1700003152,"order_key":"06:miner-6","p2mr_program_hex":"5050505050505050505050505050505050505050505050505050505050505050","share_difficulty":3,"share_id":"share-3152","share_seq":3152,"template_height":800098},{"accepted_at_ms":999993,"credit_policy":"stale-grace","job_id":"job-1497","job_issued_at_ms":999990,"miner_id":"miner-1","network_difficulty":1000001,"ntime":1700001497,"order_key":"01:miner-1","p2mr_program_hex":"d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9","share_difficulty":2,"share_id":"share-1497","share_seq":1497,"template_height":800046},{"accepted_at_ms":999971,"credit_policy":null,"job_id":"job-1481","job_issued_at_ms":999997,"miner_id":"miner-7","network_difficulty":1000002,"ntime":1700001481,"order_key":"07:miner-7","p2mr_program_hex":"c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9","share_difficulty":101,"share_id":"share-1481","share_seq":1481,"template_height":800046},{"accepted_at_ms":1000000,"credit_policy":null,"job_id":"job-3950","job_issued_at_ms":999986,"miner_id":"miner-1","network_difficulty":1000006,"ntime":1700003950,"order_key":"01:miner-1","p2mr_program_hex":"6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e","share_difficulty":7,"share_id":"share-3950","share_seq":3950,"template_height":800123},{"accepted_at_ms":999979,"credit_policy":null,"job_id":"job-25","job_issued_at_ms":999971,"miner_id":"miner-3","network_difficulty":1000008,"ntime":1700000025,"order_key":"03:miner-3","p2mr_program_hex":"1919191919191919191919191919191919191919191919191919191919191919","share_difficulty":7,"share_id":"share-25","share_seq":25,"template_height":800000},{"accepted_at_ms":999972,"credit_policy":null,"job_id":"job-3182","job_issued_at_ms":999979,"miner_id":"miner-3","network_difficulty":1000003,"ntime":1700003182,"order_key":"03:miner-3","p2mr_program_hex":"6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e","share_difficulty":19,"share_id":"share-3182","share_seq":3182,"template_height":800099},{"accepted_at_ms":999978,"credit_policy":null,"job_id":"job-1467","job_issued_at_ms":999993,"miner_id":"miner-4","network_difficulty":1000005,"ntime":1700001467,"order_key":"04:miner-4","p2mr_program_hex":"bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb","share_difficulty":1,"share_id":"share-1467","share_seq":1467,"template_height":800045},{"accepted_at_ms":1000147,"credit_policy":null,"job_id":"job-3330","job_issued_at_ms":999992,"miner_id":"miner-8","network_difficulty":1000015,"ntime":1700003330,"order_key":"08:miner-8","p2mr_program_hex":"0202020202020202020202020202020202020202020202020202020202020202","share_difficulty":101,"share_id":"share-3330","share_seq":3330,"template_height":800104},{"accepted_at_ms":1000039,"credit_policy":null,"job_id":"job-3411","job_issued_at_ms":1000275,"miner_id":"miner-1","network_difficulty":1000011,"ntime":1700003411,"order_key":"01:miner-1","p2mr_program_hex":"5353535353535353535353535353535353535353535353535353535353535353","share_difficulty":7,"share_id":"share-3411","share_seq":3411,"template_height":800106},{"accepted_at_ms":999980,"credit_policy":null,"job_id":"job-519","job_issued_at_ms":999975,"miner_id":"miner-2","network_difficulty":1000009,"ntime":1700000519,"order_key":"02:miner-2","p2mr_program_hex":"0707070707070707070707070707070707070707070707070707070707070707","share_difficulty":1,"share_id":"share-519","share_seq":519,"template_height":800016},{"accepted_at_ms":999995,"credit_policy":"stale-grace","job_id":"job-2694","job_issued_at_ms":999973,"miner_id":"miner-10","network_difficulty":1000008,"ntime":1700002694,"order_key":"10:miner-10","p2mr_program_hex":"8686868686868686868686868686868686868686868686868686868686868686","share_difficulty":7,"share_id":"share-2694","share_seq":2694,"template_height":800084},{"accepted_at_ms":999972,"credit_policy":null,"job_id":"job-241","job_issued_at_ms":999986,"miner_id":"miner-10","network_difficulty":1000003,"ntime":1700000241,"order_key":"10:miner-10","p2mr_program_hex":"f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1","share_difficulty":101,"share_id":"share-241","share_seq":241,"template_height":800007},{"accepted_at_ms":999984,"credit_policy":null,"job_id":"job-1160","job_issued_at_ms":999981,"miner_id":"miner-5","network_difficulty":1000004,"ntime":1700001160,"order_key":"05:miner-5","p2mr_program_hex":"8888888888888888888888888888888888888888888888888888888888888888","share_difficulty":1000,"share_id":"share-1160","share_seq":1160,"template_height":800036},{"accepted_at_ms":1000181,"credit_policy":null,"job_id":"job-894","job_issued_at_ms":1000000,"miner_id":"miner-3","network_difficulty":1000010,"ntime":1700000894,"order_key":"03:miner-3","p2mr_program_hex":"7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e","share_difficulty":1000,"share_id":"share-894","share_seq":894,"template_height":800027},{"accepted_at_ms":999990,"credit_policy":null,"job_id":"job-67","job_issued_at_ms":1000383,"miner_id":"miner-1","network_difficulty":1000016,"ntime":1700000067,"order_key":"01:miner-1","p2mr_program_hex":"4343434343434343434343434343434343434343434343434343434343434343","share_difficulty":3,"share_id":"share-67","share_seq":67,"template_height":800002},{"accepted_at_ms":999982,"credit_policy":null,"job_id":"job-2206","job_issued_at_ms":999994,"miner_id":"miner-6","network_difficulty":1000013,"ntime":1700002206,"order_key":"06:miner-6","p2mr_program_hex":"9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e","share_difficulty":2,"share_id":"share-2206","share_seq":2206,"template_height":800068},{"accepted_at_ms":999984,"credit_policy":"stale-grace","job_id":"job-2801","job_issued_at_ms":999994,"miner_id":"miner-7","network_difficulty":1000013,"ntime":1700002801,"order_key":"07:miner-7","p2mr_program_hex":"f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1","share_difficulty":19,"share_id":"share-2801","share_seq":2801,"template_height":800087},{"accepted_at_ms":999989,"credit_policy":null,"job_id":"job-3511","job_issued_at_ms":999971,"miner_id":"miner-2","network_difficulty":1000009,"ntime":1700003511,"order_key":"02:miner-2","p2mr_program_hex":"b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7","share_difficulty":101,"share_id":"share-3511","share_seq":3511,"template_height":800109},{"accepted_at_ms":999996,"credit_policy":null,"job_id":"job-1683","job_issued_at_ms":999978,"miner_id":"miner-0","network_difficulty":1000000,"ntime":1700001683,"order_key":"00:miner-0","p2mr_program_hex":"9393939393939393939393939393939393939393939393939393939393939393","share_difficulty":2,"share_id":"share-1683","share_seq":1683,"template_height":800052},{"accepted_at_ms":999980,"credit_policy":null,"job_id":"job-2336","job_issued_at_ms":999980,"miner_id":"miner-4","network_difficulty":1000007,"ntime":1700002336,"order_key":"04:miner-4","p2mr_program_hex":"2020202020202020202020202020202020202020202020202020202020202020","share_difficulty":101,"share_id":"share-2336","share_seq":2336,"template_height":800073},{"accepted_at_ms":999996,"credit_policy":null,"job_id":"job-371","job_issued_at_ms":999983,"miner_id":"miner-8","network_difficulty":1000014,"ntime":1700000371,"order_key":"08:miner-8","p2mr_program_hex":"7373737373737373737373737373737373737373737373737373737373737373","share_difficulty":1000,"share_id":"share-371","share_seq":371,"template_height":800011},{"accepted_at_ms":999974,"credit_policy":null,"job_id":"job-1606","job_issued_at_ms":999973,"miner_id":"miner-0","network_difficulty":1000008,"ntime":1700001606,"order_key":"00:miner-0","p2mr_program_hex":"4646464646464646464646464646464646464646464646464646464646464646","share_difficulty":19,"share_id":"share-1606","share_seq":1606,"template_height":800050},{"accepted_at_ms":999975,"credit_policy":"stale-grace","job_id":"job-1762","job_issued_at_ms":999978,"miner_id":"miner-2","network_difficulty":1000011,"ntime":1700001762,"order_key":"02:miner-2","p2mr_program_hex":"e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2","share_difficulty":1,"share_id":"share-1762","share_seq":1762,"template_height":800055},{"accepted_at_ms":999985,"credit_policy":null,"job_id":"job-3481","job_issued_at_ms":999994,"miner_id":"miner-5","network_difficulty":1000013,"ntime":1700003481,"order_key":"05:miner-5","p2mr_program_hex":"9999999999999999999999999999999999999999999999999999999999999999","share_difficulty":3,"share_id":"share-3481","share_seq":3481,"template_height":800108},{"accepted_at_ms":999982,"credit_policy":null,"job_id":"job-2023","job_issued_at_ms":999990,"miner_id":"miner-10","network_difficulty":1000000,"ntime":1700002023,"order_key":"10:miner-10","p2mr_program_hex":"e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7","share_difficulty":3,"share_id":"share-2023","share_seq":2023,"template_height":800063},{"accepted_at_ms":999970,"credit_policy":null,"job_id":"job-2556","job_issued_at_ms":999976,"miner_id":"miner-4","network_difficulty":1000006,"ntime":1700002556,"order_key":"04:miner-4","p2mr_program_hex":"fcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfc","share_difficulty":1,"share_id":"share-2556","share_seq":2556,"template_height":800079},{"accepted_at_ms":999970,"credit_policy":null,"job_id":"job-3730","job_issued_at_ms":999982,"miner_id":"miner-1","network_difficulty":1000007,"ntime":1700003730,"order_key":"01:miner-1","p2mr_program_hex":"9292929292929292929292929292929292929292929292929292929292929292","share_difficulty":101,"share_id":"share-3730","share_seq":3730,"template_height":800116},{"accepted_at_ms":999990,"credit_policy":"stale-grace","job_id":"job-3303","job_issued_at_ms":999989,"miner_id":"miner-3","network_difficulty":1000005,"ntime":1700003303,"order_key":"03:miner-3","p2mr_program_hex":"e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7","share_difficulty":3,"share_id":"share-3303","share_seq":3303,"template_height":800103},{"accepted_at_ms":999973,"credit_policy":null,"job_id":"job-3218","job_issued_at_ms":999984,"miner_id":"miner-6","network_difficulty":1000005,"ntime":1700003218,"order_key":"06:miner-6","p2mr_program_hex":"9292929292929292929292929292929292929292929292929292929292929292","share_difficulty":1,"share_id":"share-3218","share_seq":3218,"template_height":800100},{"accepted_at_ms":999989,"credit_policy":null,"job_id":"job-2490","job_issued_at_ms":999979,"miner_id":"miner-4","network_difficulty":1000008,"ntime":1700002490,"order_key":"04:miner-4","p2mr_program_hex":"babababababababababababababababababababababababababababababababa","share_difficulty":1,"share_id":"share-2490","share_seq":2490,"template_height":800077},{"accepted_at_ms":1000043,"credit_policy":"stale-grace","job_id":"job-1697","job_issued_at_ms":999971,"miner_id":"miner-3","network_difficulty":1000014,"ntime":1700001697,"order_key":"03:miner-3","p2mr_program_hex":"a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1","share_difficulty":2,"share_id":"share-1697","share_seq":1697,"template_height":800053},{"accepted_at_ms":999993,"credit_policy":null,"job_id":"job-2319","job_issued_at_ms":999974,"miner_id":"miner-9","network_difficulty":1000007,"ntime":1700002319,"order_key":"09:miner-9","p2mr_program_hex":"0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f","share_difficulty":101,"share_id":"share-2319","share_seq":2319,"template_height":800072},{"accepted_at_ms":999978,"credit_policy":null,"job_id":"job-3213","job_issued_at_ms":999986,"miner_id":"miner-1","network_difficulty":1000000,"ntime":1700003213,"order_key":"01:miner-1","p2mr_program_hex":"8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d","share_difficulty":7,"share_id":"share-3213","share_seq":3213,"template_height":800100},{"accepted_at_ms":999992,"credit_policy":"stale-grace","job_id":"job-2242","job_issued_at_ms":999972,"miner_id":"miner-9","network_difficulty":1000015,"ntime":1700002242,"order_key":"09:miner-9","p2mr_program_hex":"c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2","share_difficulty":101,"share_id":"share-2242","share_seq":2242,"template_height":800070},{"accepted_at_ms":999981,"credit_policy":null,"job_id":"job-2611","job_issued_at_ms":999980,"miner_id":"miner-4","network_difficulty":1000010,"ntime":1700002611,"order_key":"04:miner-4","p2mr_program_hex":"3333333333333333333333333333333333333333333333333333333333333333","share_difficulty":1000,"share_id":"share-2611","share_seq":2611,"template_height":800081},{"accepted_at_ms":999997,"credit_policy":"stale-grace","job_id":"job-3020","job_issued_at_ms":999987,"miner_id":"miner-6","network_difficulty":1000011,"ntime":1700003020,"order_key":"06:miner-6","p2mr_program_hex":"cccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccc","share_difficulty":101,"share_id":"share-3020","share_seq":3020,"template_height":800094},{"accepted_at_ms":999975,"credit_policy":null,"job_id":"job-3043","job_issued_at_ms":999987,"miner_id":"miner-7","network_difficulty":1000000,"ntime":1700003043,"order_key":"07:miner-7","p2mr_program_hex":"e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3","share_difficulty":1,"share_id":"share-3043","share_seq":3043,"template_height":800095},{"accepted_at_ms":999987,"credit_policy":null,"job_id":"job-2692","job_issued_at_ms":999996,"miner_id":"miner-8","network_difficulty":1000006,"ntime":1700002692,"order_key":"08:miner-8","p2mr_program_hex":"8484848484848484848484848484848484848484848484848484848484848484","share_difficulty":7,"share_id":"share-2692","share_seq":2692,"template_height":800084},{"accepted_at_ms":999999,"credit_policy":null,"job_id":"job-489","job_issued_at_ms":999977,"miner_id":"miner-5","network_difficulty":1000013,"ntime":1700000489,"order_key":"05:miner-5","p2mr_program_hex":"e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9","share_difficulty":1000,"share_id":"share-489","share_seq":489,"template_height":800015},{"accepted_at_ms":999975,"credit_policy":null,"job_id":"job-1318","job_issued_at_ms":999971,"miner_id":"miner-9","network_difficulty":1000009,"ntime":1700001318,"order_key":"09:miner-9","p2mr_program_hex":"2626262626262626262626262626262626262626262626262626262626262626","share_difficulty":7,"share_id":"share-1318","share_seq":1318,"template_height":800041},{"accepted_at_ms":999979,"credit_policy":null,"job_id":"job-683","job_issued_at_ms":999987,"miner_id":"miner-1","network_difficulty":1000003,"ntime":1700000683,"order_key":"01:miner-1","p2mr_program_hex":"abababababababababababababababababababababababababababababababab","share_difficulty":1000,"share_id":"share-683","share_seq":683,"template_height":800021},{"accepted_at_ms":999996,"credit_policy":null,"job_id":"job-2565","job_issued_at_ms":999994,"miner_id":"miner-2","network_difficulty":1000015,"ntime":1700002565,"order_key":"02:miner-2","p2mr_program_hex":"0505050505050505050505050505050505050505050505050505050505050505","share_difficulty":19,"share_id":"share-2565","share_seq":2565,"template_height":800080},{"accepted_at_ms":999970,"credit_policy":null,"job_id":"job-170","job_issued_at_ms":999995,"miner_id":"miner-5","network_difficulty":1000000,"ntime":1700000170,"order_key":"05:miner-5","p2mr_program_hex":"aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa","share_difficulty":3,"share_id":"share-170","share_seq":170,"template_height":800005},{"accepted_at_ms":1000459,"credit_policy":null,"job_id":"job-3416","job_issued_at_ms":999987,"miner_id":"miner-6","network_difficulty":1000016,"ntime":1700003416,"order_key":"06:miner-6","p2mr_program_hex":"5858585858585858585858585858585858585858585858585858585858585858","share_difficulty":1,"share_id":"share-3416","share_seq":3416,"template_height":800106},{"accepted_at_ms":999979,"credit_policy":null,"job_id":"job-1650","job_issued_at_ms":999993,"miner_id":"miner-0","network_difficulty":1000001,"ntime":1700001650,"order_key":"00:miner-0","p2mr_program_hex":"7272727272727272727272727272727272727272727272727272727272727272","share_difficulty":2,"share_id":"share-1650","share_seq":1650,"template_height":800051},{"accepted_at_ms":999975,"credit_policy":null,"job_id":"job-576","job_issued_at_ms":999979,"miner_id":"miner-4","network_difficulty":1000015,"ntime":1700000576,"order_key":"04:miner-4","p2mr_program_hex":"4040404040404040404040404040404040404040404040404040404040404040","share_difficulty":1,"share_id":"share-576","share_seq":576,"template_height":800018},{"accepted_at_ms":999980,"credit_policy":"stale-grace","job_id":"job-3005","job_issued_at_ms":999977,"miner_id":"miner-2","network_difficulty":1000013,"ntime":1700003005,"order_key":"02:miner-2","p2mr_program_hex":"bdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbd","share_difficulty":101,"share_id":"share-3005","share_seq":3005,"template_height":800093},{"accepted_at_ms":999974,"credit_policy":null,"job_id":"job-2568","job_issued_at_ms":999973,"miner_id":"miner-5","network_difficulty":1000001,"ntime":1700002568,"order_key":"05:miner-5","p2mr_program_hex":"0808080808080808080808080808080808080808080808080808080808080808","share_difficulty":2,"share_id":"share-2568","share_seq":2568,"template_height":800080},{"accepted_at_ms":999984,"credit_policy":null,"job_id":"job-2564","job_issued_at_ms":999979,"miner_id":"miner-1","network_difficulty":1000014,"ntime":1700002564,"order_key":"01:miner-1","p2mr_program_hex":"0404040404040404040404040404040404040404040404040404040404040404","share_difficulty":101,"share_id":"share-2564","share_seq":2564,"template_height":800080},{"accepted_at_ms":999999,"credit_policy":null,"job_id":"job-3075","job_issued_at_ms":999978,"miner_id":"miner-6","network_difficulty":1000015,"ntime":1700003075,"order_key":"06:miner-6","p2mr_program_hex":"0303030303030303030303030303030303030303030303030303030303030303","share_difficulty":7,"share_id":"share-3075","share_seq":3075,"template_height":800096},{"accepted_at_ms":999971,"credit_policy":null,"job_id":"job-21","job_issued_at_ms":999971,"miner_id":"miner-10","network_difficulty":1000004,"ntime":1700000021,"order_key":"10:miner-10","p2mr_program_hex":"1515151515151515151515151515151515151515151515151515151515151515","share_difficulty":1,"share_id":"share-21","share_seq":21,"template_height":800000},{"accepted_at_ms":999978,"credit_policy":null,"job_id":"job-2653","job_issued_at_ms":1000000,"miner_id":"miner-2","network_difficulty":1000001,"ntime":1700002653,"order_key":"02:miner-2","p2mr_program_hex":"5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d","share_difficulty":3,"share_id":"share-2653","share_seq":2653,"template_height":800082},{"accepted_at_ms":999976,"credit_policy":null,"job_id":"job-3629","job_issued_at_ms":999985,"miner_id":"miner-10","network_difficulty":1000008,"ntime":1700003629,"order_key":"10:miner-10","p2mr_program_hex":"2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d","share_difficulty":101,"share_id":"share-3629","share_seq":3629,"template_height":800113},{"accepted_at_ms":999989,"credit_policy":null,"job_id":"job-3605","job_issued_at_ms":999975,"miner_id":"miner-8","network_difficulty":1000001,"ntime":1700003605,"order_key":"08:miner-8","p2mr_program_hex":"1515151515151515151515151515151515151515151515151515151515151515","share_difficulty":19,"share_id":"share-3605","share_seq":3605,"template_height":800112},{"accepted_at_ms":999973,"credit_policy":null,"job_id":"job-3966","job_issued_at_ms":999972,"miner_id":"miner-6","network_difficulty":1000005,"ntime":1700003966,"order_key":"06:miner-6","p2mr_program_hex":"7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e","share_difficulty":101,"share_id":"share-3966","share_seq":3966,"template_height":800123},{"accepted_at_ms":999990,"credit_policy":null,"job_id":"job-143","job_issued_at_ms":999970,"miner_id":"miner-0","network_difficulty":1000007,"ntime":1700000143,"order_key":"00:miner-0","p2mr_program_hex":"8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f","share_difficulty":2,"share_id":"share-143","share_seq":143,"template_height":800004},{"accepted_at_ms":999986,"credit_policy":null,"job_id":"job-1788","job_issued_at_ms":999989,"miner_id":"miner-6","network_difficulty":1000003,"ntime":1700001788,"order_key":"06:miner-6","p2mr_program_hex":"fcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfc","share_difficulty":2,"share_id":"share-1788","share_seq":1788,"template_height":800055},{"accepted_at_ms":999997,"credit_policy":"stale-grace","job_id":"job-2469","job_issued_at_ms":999996,"miner_id":"miner-5","network_difficulty":1000004,"ntime":1700002469,"order_key":"05:miner-5","p2mr_program_hex":"a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5","share_difficulty":101,"share_id":"share-2469","share_seq":2469,"template_height":800077},{"accepted_at_ms":999995,"credit_policy":null,"job_id":"job-1462","job_issued_at_ms":999985,"miner_id":"miner-10","network_difficulty":1000000,"ntime":1700001462,"order_key":"10:miner-10","p2mr_program_hex":"b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6","share_difficulty":19,"share_id":"share-1462","share_seq":1462,"template_height":800045},{"accepted_at_ms":1000000,"credit_policy":null,"job_id":"job-2185","job_issued_at_ms":999976,"miner_id":"miner-7","network_difficulty":1000009,"ntime":1700002185,"order_key":"07:miner-7","p2mr_program_hex":"8989898989898989898989898989898989898989898989898989898989898989","share_difficulty":1000,"share_id":"share-2185","share_seq":2185,"template_height":800068},{"accepted_at_ms":999971,"credit_policy":null,"job_id":"job-22","job_issued_at_ms":999987,"miner_id":"miner-0","network_difficulty":1000005,"ntime":1700000022,"order_key":"00:miner-0","p2mr_program_hex":"1616161616161616161616161616161616161616161616161616161616161616","share_difficulty":7,"share_id":"share-22","share_seq":22,"template_height":800000},{"accepted_at_ms":999975,"credit_policy":null,"job_id":"job-872","job_issued_at_ms":999993,"miner_id":"miner-3","network_difficulty":1000005,"ntime":1700000872,"order_key":"03:miner-3","p2mr_program_hex":"6868686868686868686868686868686868686868686868686868686868686868","share_difficulty":1000,"share_id":"share-872","share_seq":872,"template_height":800027},{"accepted_at_ms":999991,"credit_policy":"stale-grace","job_id":"job-444","job_issued_at_ms":999981,"miner_id":"miner-4","network_difficulty":1000002,"ntime":1700000444,"order_key":"04:miner-4","p2mr_program_hex":"bcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbc","share_difficulty":3,"share_id":"share-444","share_seq":444,"template_height":800013},{"accepted_at_ms":999996,"credit_policy":null,"job_id":"job-2654","job_issued_at_ms":999989,"miner_id":"miner-3","network_difficulty":1000002,"ntime":1700002654,"order_key":"03:miner-3","p2mr_program_hex":"5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e","share_difficulty":19,"share_id":"share-2654","share_seq":2654,"template_height":800082},{"accepted_at_ms":999971,"credit_policy":null,"job_id":"job-535","job_issued_at_ms":999985,"miner_id":"miner-7","network_difficulty":1000008,"ntime":1700000535,"order_key":"07:miner-7","p2mr_program_hex":"1717171717171717171717171717171717171717171717171717171717171717","share_difficulty":19,"share_id":"share-535","share_seq":535,"template_height":800016},{"accepted_at_ms":999982,"credit_policy":null,"job_id":"job-3406","job_issued_at_ms":999971,"miner_id":"miner-7","network_difficulty":1000006,"ntime":1700003406,"order_key":"07:miner-7","p2mr_program_hex":"4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e","share_difficulty":101,"share_id":"share-3406","share_seq":3406,"template_height":800106},{"accepted_at_ms":999984,"credit_policy":"stale-grace","job_id":"job-3233","job_issued_at_ms":999972,"miner_id":"miner-10","network_difficulty":1000003,"ntime":1700003233,"order_key":"10:miner-10","p2mr_program_hex":"a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1","share_difficulty":3,"share_id":"share-3233","share_seq":3233,"template_height":800101},{"accepted_at_ms":999998,"credit_policy":"stale-grace","job_id":"job-2567","job_issued_at_ms":999998,"miner_id":"miner-4","network_difficulty":1000000,"ntime":1700002567,"order_key":"04:miner-4","p2mr_program_hex":"0707070707070707070707070707070707070707070707070707070707070707","share_difficulty":101,"share_id":"share-2567","share_seq":2567,"template_height":800080},{"accepted_at_ms":999990,"credit_policy":"stale-grace","job_id":"job-3446","job_issued_at_ms":999971,"miner_id":"miner-3","network_difficulty":1000012,"ntime":1700003446,"order_key":"03:miner-3","p2mr_program_hex":"7676767676767676767676767676767676767676767676767676767676767676","share_difficulty":1000,"share_id":"share-3446","share_seq":3446,"template_height":800107},{"accepted_at_ms":999979,"credit_policy":null,"job_id":"job-617","job_issued_at_ms":999997,"miner_id":"miner-1","network_difficulty":1000005,"ntime":1700000617,"order_key":"01:miner-1","p2mr_program_hex":"6969696969696969696969696969696969696969696969696969696969696969","share_difficulty":19,"share_id":"share-617","share_seq":617,"template_height":800019},{"accepted_at_ms":999995,"credit_policy":"stale-grace","job_id":"job-159","job_issued_at_ms":999994,"miner_id":"miner-5","network_difficulty":1000006,"ntime":1700000159,"order_key":"05:miner-5","p2mr_program_hex":"9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f","share_difficulty":2,"share_id":"share-159","share_seq":159,"template_height":800004},{"accepted_at_ms":999988,"credit_policy":null,"job_id":"job-3249","job_issued_at_ms":999991,"miner_id":"miner-4","network_difficulty":1000002,"ntime":1700003249,"order_key":"04:miner-4","p2mr_program_hex":"b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1","share_difficulty":2,"share_id":"share-3249","share_seq":3249,"template_height":800101},{"accepted_at_ms":999979,"credit_policy":null,"job_id":"job-3056","job_issued_at_ms":1000041,"miner_id":"miner-9","network_difficulty":1000013,"ntime":1700003056,"order_key":"09:miner-9","p2mr_program_hex":"f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0","share_difficulty":19,"share_id":"share-3056","share_seq":3056,"template_height":800095},{"accepted_at_ms":999978,"credit_policy":null,"job_id":"job-3082","job_issued_at_ms":999996,"miner_id":"miner-2","network_difficulty":1000005,"ntime":1700003082,"order_key":"02:miner-2","p2mr_program_hex":"0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a","share_difficulty":1000,"share_id":"share-3082","share_seq":3082,"template_height":800096},{"accepted_at_ms":999974,"credit_policy":null,"job_id":"job-3879","job_issued_at_ms":999987,"miner_id":"miner-7","network_difficulty":1000003,"ntime":1700003879,"order_key":"07:miner-7","p2mr_program_hex":"2727272727272727272727272727272727272727272727272727272727272727","share_difficulty":7,"share_id":"share-3879","share_seq":3879,"template_height":800121},{"accepted_at_ms":999999,"credit_policy":null,"job_id":"job-3710","job_issued_at_ms":999991,"miner_id":"miner-3","network_difficulty":1000004,"ntime":1700003710,"order_key":"03:miner-3","p2mr_program_hex":"7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e","share_difficulty":19,"share_id":"share-3710","share_seq":3710,"template_height":800115},{"accepted_at_ms":999997,"credit_policy":null,"job_id":"job-2418","job_issued_at_ms":999989,"miner_id":"miner-9","network_difficulty":1000004,"ntime":1700002418,"order_key":"09:miner-9","p2mr_program_hex":"7272727272727272727272727272727272727272727272727272727272727272","share_difficulty":7,"share_id":"share-2418","share_seq":2418,"template_height":800075},{"accepted_at_ms":999998,"credit_policy":"stale-grace","job_id":"job-1370","job_issued_at_ms":999975,"miner_id":"miner-6","network_difficulty":1000010,"ntime":1700001370,"order_key":"06:miner-6","p2mr_program_hex":"5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a","share_difficulty":101,"share_id":"share-1370","share_seq":1370,"template_height":800042},{"accepted_at_ms":999999,"credit_policy":null,"job_id":"job-2453","job_issued_at_ms":999974,"miner_id":"miner-0","network_difficulty":1000005,"ntime":1700002453,"order_key":"00:miner-0","p2mr_program_hex":"9595959595959595959595959595959595959595959595959595959595959595","share_difficulty":2,"share_id":"share-2453","share_seq":2453,"template_height":800076},{"accepted_at_ms":999986,"credit_policy":null,"job_id":"job-2743","job_issued_at_ms":999994,"miner_id":"miner-4","network_difficulty":1000006,"ntime":1700002743,"order_key":"04:miner-4","p2mr_program_hex":"b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7","share_difficulty":1000,"share_id":"share-2743","share_seq":2743,"template_height":800085},{"accepted_at_ms":999986,"credit_policy":null,"job_id":"job-3065","job_issued_at_ms":999997,"miner_id":"miner-7","network_difficulty":1000005,"ntime":1700003065,"order_key":"07:miner-7","p2mr_program_hex":"f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9","share_difficulty":1,"share_id":"share-3065","share_seq":3065,"template_height":800095},{"accepted_at_ms":999995,"credit_policy":null,"job_id":"job-2770","job_issued_at_ms":999980,"miner_id":"miner-9","network_difficulty":1000016,"ntime":1700002770,"order_key":"09:miner-9","p2mr_program_hex":"d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2","share_difficulty":7,"share_id":"share-2770","share_seq":2770,"template_height":800086},{"accepted_at_ms":999999,"credit_policy":null,"job_id":"job-3183","job_issued_at_ms":999989,"miner_id":"miner-4","network_difficulty":1000004,"ntime":1700003183,"order_key":"04:miner-4","p2mr_program_hex":"6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f","share_difficulty":7,"share_id":"share-3183","share_seq":3183,"template_height":800099},{"accepted_at_ms":1000482,"credit_policy":"stale-grace","job_id":"job-975","job_issued_at_ms":999971,"miner_id":"miner-7","network_difficulty":1000006,"ntime":1700000975,"order_key":"07:miner-7","p2mr_program_hex":"cfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcf","share_difficulty":7,"share_id":"share-975","share_seq":975,"template_height":800030},{"accepted_at_ms":999975,"credit_policy":null,"job_id":"job-1116","job_issued_at_ms":999989,"miner_id":"miner-5","network_difficulty":1000011,"ntime":1700001116,"order_key":"05:miner-5","p2mr_program_hex":"5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c","share_difficulty":101,"share_id":"share-1116","share_seq":1116,"template_height":800034},{"accepted_at_ms":999970,"credit_policy":null,"job_id":"job-1710","job_issued_at_ms":999979,"miner_id":"miner-5","network_difficulty":1000010,"ntime":1700001710,"order_key":"05:miner-5","p2mr_program_hex":"aeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeae","share_difficulty":3,"share_id":"share-1710","share_seq":1710,"template_height":800053},{"accepted_at_ms":999990,"credit_policy":null,"job_id":"job-2220","job_issued_at_ms":999977,"miner_id":"miner-9","network_difficulty":1000010,"ntime":1700002220,"order_key":"09:miner-9","p2mr_program_hex":"acacacacacacacacacacacacacacacacacacacacacacacacacacacacacacacac","share_difficulty":3,"share_id":"share-2220","share_seq":2220,"template_height":800069},{"accepted_at_ms":999984,"credit_policy":"stale-grace","job_id":"job-2548","job_issued_at_ms":999982,"miner_id":"miner-7","network_difficulty":1000015,"ntime":1700002548,"order_key":"07:miner-7","p2mr_program_hex":"f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4","share_difficulty":101,"share_id":"share-2548","share_seq":2548,"template_height":800079},{"accepted_at_ms":999971,"credit_policy":null,"job_id":"job-1590","job_issued_at_ms":999976,"miner_id":"miner-6","network_difficulty":1000009,"ntime":1700001590,"order_key":"06:miner-6","p2mr_program_hex":"3636363636363636363636363636363636363636363636363636363636363636","share_difficulty":1,"share_id":"share-1590","share_seq":1590,"template_height":800049},{"accepted_at_ms":999972,"credit_policy":"stale-grace","job_id":"job-1389","job_issued_at_ms":999999,"miner_id":"miner-3","network_difficulty":1000012,"ntime":1700001389,"order_key":"03:miner-3","p2mr_program_hex":"6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d","share_difficulty":1,"share_id":"share-1389","share_seq":1389,"template_height":800043},{"accepted_at_ms":999990,"credit_policy":"stale-grace","job_id":"job-3278","job_issued_at_ms":999998,"miner_id":"miner-0","network_difficulty":1000014,"ntime":1700003278,"order_key":"00:miner-0","p2mr_program_hex":"cececececececececececececececececececececececececececececececece","share_difficulty":1,"share_id":"share-3278","share_seq":3278,"template_height":800102},{"accepted_at_ms":999980,"credit_policy":null,"job_id":"job-3322","job_issued_at_ms":999975,"miner_id":"miner-0","network_difficulty":1000007,"ntime":1700003322,"order_key":"00:miner-0","p2mr_program_hex":"fafafafafafafafafafafafafafafafafafafafafafafafafafafafafafafafa","share_difficulty":3,"share_id":"share-3322","share_seq":3322,"template_height":800103},{"accepted_at_ms":999997,"credit_policy":"stale-grace","job_id":"job-2476","job_issued_at_ms":999994,"miner_id":"miner-1","network_difficulty":1000011,"ntime":1700002476,"order_key":"01:miner-1","p2mr_program_hex":"acacacacacacacacacacacacacacacacacacacacacacacacacacacacacacacac","share_difficulty":2,"share_id":"share-2476","share_seq":2476,"template_height":800077},{"accepted_at_ms":1000000,"credit_policy":null,"job_id":"job-3818","job_issued_at_ms":999990,"miner_id":"miner-1","network_difficulty":1000010,"ntime":1700003818,"order_key":"01:miner-1","p2mr_program_hex":"eaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaea","share_difficulty":1000,"share_id":"share-3818","share_seq":3818,"template_height":800119},{"accepted_at_ms":999979,"credit_policy":"stale-grace","job_id":"job-3735","job_issued_at_ms":999973,"miner_id":"miner-6","network_difficulty":1000012,"ntime":1700003735,"order_key":"06:miner-6","p2mr_program_hex":"9797979797979797979797979797979797979797979797979797979797979797","share_difficulty":2,"share_id":"share-3735","share_seq":3735,"template_height":800116},{"accepted_at_ms":999974,"credit_policy":null,"job_id":"job-527","job_issued_at_ms":999988,"miner_id":"miner-10","network_difficulty":1000000,"ntime":1700000527,"order_key":"10:miner-10","p2mr_program_hex":"0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f","share_difficulty":19,"share_id":"share-527","share_seq":527,"template_height":800016},{"accepted_at_ms":999996,"credit_policy":null,"job_id":"job-191","job_issued_at_ms":999987,"miner_id":"miner-4","network_difficulty":1000004,"ntime":1700000191,"order_key":"04:miner-4","p2mr_program_hex":"bfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbf","share_difficulty":101,"share_id":"share-191","share_seq":191,"template_height":800005},{"accepted_at_ms":999984,"credit_policy":null,"job_id":"job-1878","job_issued_at_ms":999995,"miner_id":"miner-8","network_difficulty":1000008,"ntime":1700001878,"order_key":"08:miner-8","p2mr_program_hex":"5656565656565656565656565656565656565656565656565656565656565656","share_difficulty":1000,"share_id":"share-1878","share_seq":1878,"template_height":800058},{"accepted_at_ms":999988,"credit_policy":"stale-grace","job_id":"job-429","job_issued_at_ms":999981,"miner_id":"miner-0","network_difficulty":1000004,"ntime":1700000429,"order_key":"00:miner-0","p2mr_program_hex":"adadadadadadadadadadadadadadadadadadadadadadadadadadadadadadadad","share_difficulty":3,"share_id":"share-429","share_seq":429,"template_height":800013},{"accepted_at_ms":999984,"credit_policy":null,"job_id":"job-2665","job_issued_at_ms":999980,"miner_id":"miner-3","network_difficulty":1000013,"ntime":1700002665,"order_key":"03:miner-3","p2mr_program_hex":"6969696969696969696969696969696969696969696969696969696969696969","share_difficulty":101,"share_id":"share-2665","share_seq":2665,"template_height":800083},{"accepted_at_ms":999993,"credit_policy":null,"job_id":"job-1095","job_issued_at_ms":999975,"miner_id":"miner-6","network_difficulty":1000007,"ntime":1700001095,"order_key":"06:miner-6","p2mr_program_hex":"4747474747474747474747474747474747474747474747474747474747474747","share_difficulty":3,"share_id":"share-1095","share_seq":1095,"template_height":800034},{"accepted_at_ms":999986,"credit_policy":null,"job_id":"job-1816","job_issued_at_ms":999989,"miner_id":"miner-1","network_difficulty":1000014,"ntime":1700001816,"order_key":"01:miner-1","p2mr_program_hex":"1818181818181818181818181818181818181818181818181818181818181818","share_difficulty":19,"share_id":"share-1816","share_seq":1816,"template_height":800056},{"accepted_at_ms":999986,"credit_policy":null,"job_id":"job-420","job_issued_at_ms":999992,"miner_id":"miner-2","network_difficulty":1000012,"ntime":1700000420,"order_key":"02:miner-2","p2mr_program_hex":"a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4","share_difficulty":1000,"share_id":"share-420","share_seq":420,"template_height":800013},{"accepted_at_ms":999995,"credit_policy":"stale-grace","job_id":"job-1626","job_issued_at_ms":999999,"miner_id":"miner-9","network_difficulty":1000011,"ntime":1700001626,"order_key":"09:miner-9","p2mr_program_hex":"5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a","share_difficulty":7,"share_id":"share-1626","share_seq":1626,"template_height":800050},{"accepted_at_ms":999994,"credit_policy":null,"job_id":"job-972","job_issued_at_ms":999988,"miner_id":"miner-4","network_difficulty":1000003,"ntime":1700000972,"order_key":"04:miner-4","p2mr_program_hex":"cccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccc","share_difficulty":1,"share_id":"share-972","share_seq":972,"template_height":800030},{"accepted_at_ms":999989,"credit_policy":null,"job_id":"job-909","job_issued_at_ms":999994,"miner_id":"miner-7","network_difficulty":1000008,"ntime":1700000909,"order_key":"07:miner-7","p2mr_program_hex":"8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d","share_difficulty":101,"share_id":"share-909","share_seq":909,"template_height":800028},{"accepted_at_ms":999998,"credit_policy":null,"job_id":"job-3883","job_issued_at_ms":999983,"miner_id":"miner-0","network_difficulty":1000007,"ntime":1700003883,"order_key":"00:miner-0","p2mr_program_hex":"2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b","share_difficulty":2,"share_id":"share-3883","share_seq":3883,"template_height":800121},{"accepted_at_ms":999972,"credit_policy":null,"job_id":"job-396","job_issued_at_ms":999988,"miner_id":"miner-0","network_difficulty":1000005,"ntime":1700000396,"order_key":"00:miner-0","p2mr_program_hex":"8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c","share_difficulty":1,"share_id":"share-396","share_seq":396,"template_height":800012},{"accepted_at_ms":999997,"credit_policy":null,"job_id":"job-1823","job_issued_at_ms":999990,"miner_id":"miner-8","network_difficulty":1000004,"ntime":1700001823,"order_key":"08:miner-8","p2mr_program_hex":"1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f","share_difficulty":2,"share_id":"share-1823","share_seq":1823,"template_height":800056},{"accepted_at_ms":999999,"credit_policy":null,"job_id":"job-2184","job_issued_at_ms":999988,"miner_id":"miner-6","network_difficulty":1000008,"ntime":1700002184,"order_key":"06:miner-6","p2mr_program_hex":"8888888888888888888888888888888888888888888888888888888888888888","share_difficulty":19,"share_id":"share-2184","share_seq":2184,"template_height":800068},{"accepted_at_ms":999992,"credit_policy":null,"job_id":"job-461","job_issued_at_ms":999970,"miner_id":"miner-10","network_difficulty":1000002,"ntime":1700000461,"order_key":"10:miner-10","p2mr_program_hex":"cdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcd","share_difficulty":7,"share_id":"share-461","share_seq":461,"template_height":800014},{"accepted_at_ms":999998,"credit_policy":null,"job_id":"job-3227","job_issued_at_ms":999995,"miner_id":"miner-4","network_difficulty":1000014,"ntime":1700003227,"order_key":"04:miner-4","p2mr_program_hex":"9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b","share_difficulty":1,"share_id":"share-3227","share_seq":3227,"template_height":800100},{"accepted_at_ms":999971,"credit_policy":null,"job_id":"job-3349","job_issued_at_ms":999981,"miner_id":"miner-5","network_difficulty":1000000,"ntime":1700003349,"order_key":"05:miner-5","p2mr_program_hex":"1515151515151515151515151515151515151515151515151515151515151515","share_difficulty":1,"share_id":"share-3349","share_seq":3349,"template_height":800104},{"accepted_at_ms":999977,"credit_policy":null,"job_id":"job-3965","job_issued_at_ms":1000000,"miner_id":"miner-5","network_difficulty":1000004,"ntime":1700003965,"order_key":"05:miner-5","p2mr_program_hex":"7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d","share_difficulty":19,"share_id":"share-3965","share_seq":3965,"template_height":800123},{"accepted_at_ms":999971,"credit_policy":null,"job_id":"job-1315","job_issued_at_ms":999986,"miner_id":"miner-6","network_difficulty":1000006,"ntime":1700001315,"order_key":"06:miner-6","p2mr_program_hex":"2323232323232323232323232323232323232323232323232323232323232323","share_difficulty":2,"share_id":"share-1315","share_seq":1315,"template_height":800041},{"accepted_at_ms":999989,"credit_policy":null,"job_id":"job-2257","job_issued_at_ms":999972,"miner_id":"miner-2","network_difficulty":1000013,"ntime":1700002257,"order_key":"02:miner-2","p2mr_program_hex":"d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1","share_difficulty":19,"share_id":"share-2257","share_seq":2257,"template_height":800070},{"accepted_at_ms":999996,"credit_policy":null,"job_id":"job-1769","job_issued_at_ms":999990,"miner_id":"miner-9","network_difficulty":1000001,"ntime":1700001769,"order_key":"09:miner-9","p2mr_program_hex":"e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9","share_difficulty":1,"share_id":"share-1769","share_seq":1769,"template_height":800055},{"accepted_at_ms":999972,"credit_policy":null,"job_id":"job-3236","job_issued_at_ms":999989,"miner_id":"miner-2","network_difficulty":1000006,"ntime":1700003236,"order_key":"02:miner-2","p2mr_program_hex":"a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4","share_difficulty":1000,"share_id":"share-3236","share_seq":3236,"template_height":800101},{"accepted_at_ms":999995,"credit_policy":null,"job_id":"job-3820","job_issued_at_ms":999984,"miner_id":"miner-3","network_difficulty":1000012,"ntime":1700003820,"order_key":"03:miner-3","p2mr_program_hex":"ecececececececececececececececececececececececececececececececec","share_difficulty":2,"share_id":"share-3820","share_seq":3820,"template_height":800119},{"accepted_at_ms":999990,"credit_policy":"stale-grace","job_id":"job-3107","job_issued_at_ms":999997,"miner_id":"miner-5","network_difficulty":1000013,"ntime":1700003107,"order_key":"05:miner-5","p2mr_program_hex":"2323232323232323232323232323232323232323232323232323232323232323","share_difficulty":1,"share_id":"share-3107","share_seq":3107,"template_height":800097},{"accepted_at_ms":999984,"credit_policy":null,"job_id":"job-1029","job_issued_at_ms":999974,"miner_id":"miner-6","network_difficulty":1000009,"ntime":1700001029,"order_key":"06:miner-6","p2mr_program_hex":"0505050505050505050505050505050505050505050505050505050505050505","share_difficulty":2,"share_id":"share-1029","share_seq":1029,"template_height":800032},{"accepted_at_ms":1000479,"credit_policy":null,"job_id":"job-577","job_issued_at_ms":999999,"miner_id":"miner-5","network_difficulty":1000016,"ntime":1700000577,"order_key":"05:miner-5","p2mr_program_hex":"4141414141414141414141414141414141414141414141414141414141414141","share_difficulty":3,"share_id":"share-577","share_seq":577,"template_height":800018},{"accepted_at_ms":999976,"credit_policy":null,"job_id":"job-3074","job_issued_at_ms":999994,"miner_id":"miner-5","network_difficulty":1000014,"ntime":1700003074,"order_key":"05:miner-5","p2mr_program_hex":"0202020202020202020202020202020202020202020202020202020202020202","share_difficulty":3,"share_id":"share-3074","share_seq":3074,"template_height":800096},{"accepted_at_ms":999973,"credit_policy":null,"job_id":"job-261","job_issued_at_ms":999998,"miner_id":"miner-8","network_difficulty":1000006,"ntime":1700000261,"order_key":"08:miner-8","p2mr_program_hex":"0505050505050505050505050505050505050505050505050505050505050505","share_difficulty":3,"share_id":"share-261","share_seq":261,"template_height":800008},{"accepted_at_ms":999982,"credit_policy":null,"job_id":"job-2283","job_issued_at_ms":999980,"miner_id":"miner-6","network_difficulty":1000005,"ntime":1700002283,"order_key":"06:miner-6","p2mr_program_hex":"ebebebebebebebebebebebebebebebebebebebebebebebebebebebebebebebeb","share_difficulty":1000,"share_id":"share-2283","share_seq":2283,"template_height":800071},{"accepted_at_ms":999983,"credit_policy":null,"job_id":"job-3051","job_issued_at_ms":999994,"miner_id":"miner-4","network_difficulty":1000008,"ntime":1700003051,"order_key":"04:miner-4","p2mr_program_hex":"ebebebebebebebebebebebebebebebebebebebebebebebebebebebebebebebeb","share_difficulty":3,"share_id":"share-3051","share_seq":3051,"template_height":800095},{"accepted_at_ms":999999,"credit_policy":null,"job_id":"job-273","job_issued_at_ms":999974,"miner_id":"miner-9","network_difficulty":1000001,"ntime":1700000273,"order_key":"09:miner-9","p2mr_program_hex":"1111111111111111111111111111111111111111111111111111111111111111","share_difficulty":101,"share_id":"share-273","share_seq":273,"template_height":800008},{"accepted_at_ms":999990,"credit_policy":"stale-grace","job_id":"job-2105","job_issued_at_ms":999993,"miner_id":"miner-4","network_difficulty":1000014,"ntime":1700002105,"order_key":"04:miner-4","p2mr_program_hex":"3939393939393939393939393939393939393939393939393939393939393939","share_difficulty":7,"share_id":"share-2105","share_seq":2105,"template_height":800065},{"accepted_at_ms":999972,"credit_policy":null,"job_id":"job-2746","job_issued_at_ms":999999,"miner_id":"miner-7","network_difficulty":1000009,"ntime":1700002746,"order_key":"07:miner-7","p2mr_program_hex":"babababababababababababababababababababababababababababababababa","share_difficulty":7,"share_id":"share-2746","share_seq":2746,"template_height":800085},{"accepted_at_ms":999995,"credit_policy":null,"job_id":"job-1639","job_issued_at_ms":999995,"miner_id":"miner-0","network_difficulty":1000007,"ntime":1700001639,"order_key":"00:miner-0","p2mr_program_hex":"6767676767676767676767676767676767676767676767676767676767676767","share_difficulty":1,"share_id":"share-1639","share_seq":1639,"template_height":800051},{"accepted_at_ms":999983,"credit_policy":"stale-grace","job_id":"job-3049","job_issued_at_ms":999986,"miner_id":"miner-2","network_difficulty":1000006,"ntime":1700003049,"order_key":"02:miner-2","p2mr_program_hex":"e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9","share_difficulty":19,"share_id":"share-3049","share_seq":3049,"template_height":800095},{"accepted_at_ms":999972,"credit_policy":null,"job_id":"job-43","job_issued_at_ms":999973,"miner_id":"miner-10","network_difficulty":1000009,"ntime":1700000043,"order_key":"10:miner-10","p2mr_program_hex":"2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b","share_difficulty":2,"share_id":"share-43","share_seq":43,"template_height":800001},{"accepted_at_ms":999970,"credit_policy":null,"job_id":"job-3174","job_issued_at_ms":999975,"miner_id":"miner-6","network_difficulty":1000012,"ntime":1700003174,"order_key":"06:miner-6","p2mr_program_hex":"6666666666666666666666666666666666666666666666666666666666666666","share_difficulty":1000,"share_id":"share-3174","share_seq":3174,"template_height":800099},{"accepted_at_ms":999999,"credit_policy":null,"job_id":"job-3976","job_issued_at_ms":999995,"miner_id":"miner-5","network_difficulty":1000015,"ntime":1700003976,"order_key":"05:miner-5","p2mr_program_hex":"8888888888888888888888888888888888888888888888888888888888888888","share_difficulty":3,"share_id":"share-3976","share_seq":3976,"template_height":800124},{"accepted_at_ms":999993,"credit_policy":null,"job_id":"job-1890","job_issued_at_ms":1000234,"miner_id":"miner-9","network_difficulty":1000003,"ntime":1700001890,"order_key":"09:miner-9","p2mr_program_hex":"6262626262626262626262626262626262626262626262626262626262626262","share_difficulty":19,"share_id":"share-1890","share_seq":1890,"template_height":800059},{"accepted_at_ms":999995,"credit_policy":null,"job_id":"job-1296","job_issued_at_ms":999979,"miner_id":"miner-9","network_difficulty":1000004,"ntime":1700001296,"order_key":"09:miner-9","p2mr_program_hex":"1010101010101010101010101010101010101010101010101010101010101010","share_difficulty":19,"share_id":"share-1296","share_seq":1296,"template_height":800040},{"accepted_at_ms":999988,"credit_policy":null,"job_id":"job-3123","job_issued_at_ms":999985,"miner_id":"miner-10","network_difficulty":1000012,"ntime":1700003123,"order_key":"10:miner-10","p2mr_program_hex":"3333333333333333333333333333333333333333333333333333333333333333","share_difficulty":101,"share_id":"share-3123","share_seq":3123,"template_height":800097},{"accepted_at_ms":999973,"credit_policy":"stale-grace","job_id":"job-3838","job_issued_at_ms":999978,"miner_id":"miner-10","network_difficulty":1000013,"ntime":1700003838,"order_key":"10:miner-10","p2mr_program_hex":"fefefefefefefefefefefefefefefefefefefefefefefefefefefefefefefefe","share_difficulty":3,"share_id":"share-3838","share_seq":3838,"template_height":800119},{"accepted_at_ms":999999,"credit_policy":null,"job_id":"job-2274","job_issued_at_ms":999993,"miner_id":"miner-8","network_difficulty":1000013,"ntime":1700002274,"order_key":"08:miner-8","p2mr_program_hex":"e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2","share_difficulty":19,"share_id":"share-2274","share_seq":2274,"template_height":800071},{"accepted_at_ms":999985,"credit_policy":null,"job_id":"job-641","job_issued_at_ms":999999,"miner_id":"miner-3","network_difficulty":1000012,"ntime":1700000641,"order_key":"03:miner-3","p2mr_program_hex":"8181818181818181818181818181818181818181818181818181818181818181","share_difficulty":3,"share_id":"share-641","share_seq":641,"template_height":800020},{"accepted_at_ms":999971,"credit_policy":null,"job_id":"job-2190","job_issued_at_ms":999999,"miner_id":"miner-1","network_difficulty":1000014,"ntime":1700002190,"order_key":"01:miner-1","p2mr_program_hex":"8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e","share_difficulty":19,"share_id":"share-2190","share_seq":2190,"template_height":800068},{"accepted_at_ms":999981,"credit_policy":"stale-grace","job_id":"job-3962","job_issued_at_ms":999992,"miner_id":"miner-2","network_difficulty":1000001,"ntime":1700003962,"order_key":"02:miner-2","p2mr_program_hex":"7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a","share_difficulty":1,"share_id":"share-3962","share_seq":3962,"template_height":800123},{"accepted_at_ms":999996,"credit_policy":null,"job_id":"job-3617","job_issued_at_ms":999975,"miner_id":"miner-9","network_difficulty":1000013,"ntime":1700003617,"order_key":"09:miner-9","p2mr_program_hex":"2121212121212121212121212121212121212121212121212121212121212121","share_difficulty":1000,"share_id":"share-3617","share_seq":3617,"template_height":800113},{"accepted_at_ms":999972,"credit_policy":null,"job_id":"job-689","job_issued_at_ms":999974,"miner_id":"miner-7","network_difficulty":1000009,"ntime":1700000689,"order_key":"07:miner-7","p2mr_program_hex":"b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1","share_difficulty":7,"share_id":"share-689","share_seq":689,"template_height":800021},{"accepted_at_ms":999998,"credit_policy":null,"job_id":"job-2954","job_issued_at_ms":999988,"miner_id":"miner-6","network_difficulty":1000013,"ntime":1700002954,"order_key":"06:miner-6","p2mr_program_hex":"8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a","share_difficulty":7,"share_id":"share-2954","share_seq":2954,"template_height":800092},{"accepted_at_ms":999972,"credit_policy":"stale-grace","job_id":"job-2907","job_issued_at_ms":999983,"miner_id":"miner-3","network_difficulty":1000000,"ntime":1700002907,"order_key":"03:miner-3","p2mr_program_hex":"5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b","share_difficulty":3,"share_id":"share-2907","share_seq":2907,"template_height":800090},{"accepted_at_ms":999978,"credit_policy":null,"job_id":"job-2277","job_issued_at_ms":999999,"miner_id":"miner-0","network_difficulty":1000016,"ntime":1700002277,"order_key":"00:miner-0","p2mr_program_hex":"e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5","share_difficulty":19,"share_id":"share-2277","share_seq":2277,"template_height":800071},{"accepted_at_ms":999996,"credit_policy":null,"job_id":"job-627","job_issued_at_ms":999989,"miner_id":"miner-0","network_difficulty":1000015,"ntime":1700000627,"order_key":"00:miner-0","p2mr_program_hex":"7373737373737373737373737373737373737373737373737373737373737373","share_difficulty":3,"share_id":"share-627","share_seq":627,"template_height":800019},{"accepted_at_ms":999993,"credit_policy":null,"job_id":"job-1649","job_issued_at_ms":999982,"miner_id":"miner-10","network_difficulty":1000000,"ntime":1700001649,"order_key":"10:miner-10","p2mr_program_hex":"7171717171717171717171717171717171717171717171717171717171717171","share_difficulty":3,"share_id":"share-1649","share_seq":1649,"template_height":800051},{"accepted_at_ms":999994,"credit_policy":null,"job_id":"job-2877","job_issued_at_ms":999971,"miner_id":"miner-6","network_difficulty":1000004,"ntime":1700002877,"order_key":"06:miner-6","p2mr_program_hex":"3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d","share_difficulty":3,"share_id":"share-2877","share_seq":2877,"template_height":800089},{"accepted_at_ms":999981,"credit_policy":null,"job_id":"job-2363","job_issued_at_ms":1000000,"miner_id":"miner-9","network_difficulty":1000000,"ntime":1700002363,"order_key":"09:miner-9","p2mr_program_hex":"3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b","share_difficulty":19,"share_id":"share-2363","share_seq":2363,"template_height":800073},{"accepted_at_ms":1000114,"credit_policy":null,"job_id":"job-3799","job_issued_at_ms":1000174,"miner_id":"miner-4","network_difficulty":1000008,"ntime":1700003799,"order_key":"04:miner-4","p2mr_program_hex":"d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7","share_difficulty":7,"share_id":"share-3799","share_seq":3799,"template_height":800118},{"accepted_at_ms":999972,"credit_policy":"stale-grace","job_id":"job-3169","job_issued_at_ms":999972,"miner_id":"miner-1","network_difficulty":1000007,"ntime":1700003169,"order_key":"01:miner-1","p2mr_program_hex":"6161616161616161616161616161616161616161616161616161616161616161","share_difficulty":19,"share_id":"share-3169","share_seq":3169,"template_height":800099},{"accepted_at_ms":999991,"credit_policy":null,"job_id":"job-2498","job_issued_at_ms":1000244,"miner_id":"miner-1","network_difficulty":1000016,"ntime":1700002498,"order_key":"01:miner-1","p2mr_program_hex":"c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2","share_difficulty":3,"share_id":"share-2498","share_seq":2498,"template_height":800078},{"accepted_at_ms":999988,"credit_policy":null,"job_id":"job-3829","job_issued_at_ms":999979,"miner_id":"miner-1","network_difficulty":1000004,"ntime":1700003829,"order_key":"01:miner-1","p2mr_program_hex":"f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5","share_difficulty":19,"share_id":"share-3829","share_seq":3829,"template_height":800119},{"accepted_at_ms":999998,"credit_policy":null,"job_id":"job-2075","job_issued_at_ms":999990,"miner_id":"miner-7","network_difficulty":1000001,"ntime":1700002075,"order_key":"07:miner-7","p2mr_program_hex":"1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b","share_difficulty":1000,"share_id":"share-2075","share_seq":2075,"template_height":800064},{"accepted_at_ms":999993,"credit_policy":null,"job_id":"job-3253","job_issued_at_ms":999979,"miner_id":"miner-8","network_difficulty":1000006,"ntime":1700003253,"order_key":"08:miner-8","p2mr_program_hex":"b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5","share_difficulty":101,"share_id":"share-3253","share_seq":3253,"template_height":800101},{"accepted_at_ms":999997,"credit_policy":null,"job_id":"job-18","job_issued_at_ms":999994,"miner_id":"miner-7","network_difficulty":1000001,"ntime":1700000018,"order_key":"07:miner-7","p2mr_program_hex":"1212121212121212121212121212121212121212121212121212121212121212","share_difficulty":101,"share_id":"share-18","share_seq":18,"template_height":800000},{"accepted_at_ms":999984,"credit_policy":null,"job_id":"job-3550","job_issued_at_ms":999996,"miner_id":"miner-8","network_difficulty":1000014,"ntime":1700003550,"order_key":"08:miner-8","p2mr_program_hex":"dededededededededededededededededededededededededededededededede","share_difficulty":1000,"share_id":"share-3550","share_seq":3550,"template_height":800110},{"accepted_at_ms":999974,"credit_policy":null,"job_id":"job-2484","job_issued_at_ms":999974,"miner_id":"miner-9","network_difficulty":1000002,"ntime":1700002484,"order_key":"09:miner-9","p2mr_program_hex":"b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4","share_difficulty":19,"share_id":"share-2484","share_seq":2484,"template_height":800077},{"accepted_at_ms":1000245,"credit_policy":null,"job_id":"job-3901","job_issued_at_ms":1000062,"miner_id":"miner-7","network_difficulty":1000008,"ntime":1700003901,"order_key":"07:miner-7","p2mr_program_hex":"3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d","share_difficulty":7,"share_id":"share-3901","share_seq":3901,"template_height":800121},{"accepted_at_ms":999996,"credit_policy":null,"job_id":"job-790","job_issued_at_ms":999976,"miner_id":"miner-9","network_difficulty":1000008,"ntime":1700000790,"order_key":"09:miner-9","p2mr_program_hex":"1616161616161616161616161616161616161616161616161616161616161616","share_difficulty":101,"share_id":"share-790","share_seq":790,"template_height":800024},{"accepted_at_ms":999974,"credit_policy":null,"job_id":"job-2701","job_issued_at_ms":999984,"miner_id":"miner-6","network_difficulty":1000015,"ntime":1700002701,"order_key":"06:miner-6","p2mr_program_hex":"8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d","share_difficulty":1000,"share_id":"share-2701","share_seq":2701,"template_height":800084},{"accepted_at_ms":999994,"credit_policy":null,"job_id":"job-957","job_issued_at_ms":999996,"miner_id":"miner-0","network_difficulty":1000005,"ntime":1700000957,"order_key":"00:miner-0","p2mr_program_hex":"bdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbd","share_difficulty":3,"share_id":"share-957","share_seq":957,"template_height":800029},{"accepted_at_ms":999981,"credit_policy":null,"job_id":"job-2278","job_issued_at_ms":999994,"miner_id":"miner-1","network_difficulty":1000000,"ntime":1700002278,"order_key":"01:miner-1","p2mr_program_hex":"e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6","share_difficulty":101,"share_id":"share-2278","share_seq":2278,"template_height":800071},{"accepted_at_ms":999994,"credit_policy":"stale-grace","job_id":"job-3817","job_issued_at_ms":999978,"miner_id":"miner-0","network_difficulty":1000009,"ntime":1700003817,"order_key":"00:miner-0","p2mr_program_hex":"e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9","share_difficulty":7,"share_id":"share-3817","share_seq":3817,"template_height":800119},{"accepted_at_ms":999993,"credit_policy":null,"job_id":"job-2045","job_issued_at_ms":999982,"miner_id":"miner-10","network_difficulty":1000005,"ntime":1700002045,"order_key":"10:miner-10","p2mr_program_hex":"fdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfd","share_difficulty":3,"share_id":"share-2045","share_seq":2045,"template_height":800063},{"accepted_at_ms":999979,"credit_policy":null,"job_id":"job-1619","job_issued_at_ms":999983,"miner_id":"miner-2","network_difficulty":1000004,"ntime":1700001619,"order_key":"02:miner-2","p2mr_program_hex":"5353535353535353535353535353535353535353535353535353535353535353","share_difficulty":1000,"share_id":"share-1619","share_seq":1619,"template_height":800050},{"accepted_at_ms":999993,"credit_policy":null,"job_id":"job-180","job_issued_at_ms":999999,"miner_id":"miner-4","network_difficulty":1000010,"ntime":1700000180,"order_key":"04:miner-4","p2mr_program_hex":"b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4","share_difficulty":7,"share_id":"share-180","share_seq":180,"template_height":800005},{"accepted_at_ms":1000000,"credit_policy":null,"job_id":"job-2745","job_issued_at_ms":999986,"miner_id":"miner-6","network_difficulty":1000008,"ntime":1700002745,"order_key":"06:miner-6","p2mr_program_hex":"b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9","share_difficulty":1,"share_id":"share-2745","share_seq":2745,"template_height":800085},{"accepted_at_ms":999975,"credit_policy":null,"job_id":"job-1037","job_issued_at_ms":999984,"miner_id":"miner-3","network_difficulty":1000000,"ntime":1700001037,"order_key":"03:miner-3","p2mr_program_hex":"0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d","share_difficulty":19,"share_id":"share-1037","share_seq":1037,"template_height":800032},{"accepted_at_ms":999980,"credit_policy":"stale-grace","job_id":"job-3597","job_issued_at_ms":999989,"miner_id":"miner-0","network_difficulty":1000010,"ntime":1700003597,"order_key":"00:miner-0","p2mr_program_hex":"0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d","share_difficulty":1000,"share_id":"share-3597","share_seq":3597,"template_height":800112},{"accepted_at_ms":999994,"credit_policy":"stale-grace","job_id":"job-504","job_issued_at_ms":999974,"miner_id":"miner-9","network_difficulty":1000011,"ntime":1700000504,"order_key":"09:miner-9","p2mr_program_hex":"f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8","share_difficulty":3,"share_id":"share-504","share_seq":504,"template_height":800015},{"accepted_at_ms":999983,"credit_policy":null,"job_id":"job-1629","job_issued_at_ms":999980,"miner_id":"miner-1","network_difficulty":1000014,"ntime":1700001629,"order_key":"01:miner-1","p2mr_program_hex":"5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d","share_difficulty":1000,"share_id":"share-1629","share_seq":1629,"template_height":800050},{"accepted_at_ms":999991,"credit_policy":"stale-grace","job_id":"job-2471","job_issued_at_ms":999998,"miner_id":"miner-7","network_difficulty":1000006,"ntime":1700002471,"order_key":"07:miner-7","p2mr_program_hex":"a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7","share_difficulty":19,"share_id":"share-2471","share_seq":2471,"template_height":800077},{"accepted_at_ms":999999,"credit_policy":null,"job_id":"job-2920","job_issued_at_ms":999970,"miner_id":"miner-5","network_difficulty":1000013,"ntime":1700002920,"order_key":"05:miner-5","p2mr_program_hex":"6868686868686868686868686868686868686868686868686868686868686868","share_difficulty":1000,"share_id":"share-2920","share_seq":2920,"template_height":800091},{"accepted_at_ms":999998,"credit_policy":null,"job_id":"job-312","job_issued_at_ms":999999,"miner_id":"miner-4","network_difficulty":1000006,"ntime":1700000312,"order_key":"04:miner-4","p2mr_program_hex":"3838383838383838383838383838383838383838383838383838383838383838","share_difficulty":1000,"share_id":"share-312","share_seq":312,"template_height":800009},{"accepted_at_ms":999982,"credit_policy":null,"job_id":"job-3503","job_issued_at_ms":999973,"miner_id":"miner-5","network_difficulty":1000001,"ntime":1700003503,"order_key":"05:miner-5","p2mr_program_hex":"afafafafafafafafafafafafafafafafafafafafafafafafafafafafafafafaf","share_difficulty":3,"share_id":"share-3503","share_seq":3503,"template_height":800109},{"accepted_at_ms":999974,"credit_policy":"stale-grace","job_id":"job-1943","job_issued_at_ms":999984,"miner_id":"miner-7","network_difficulty":1000005,"ntime":1700001943,"order_key":"07:miner-7","p2mr_program_hex":"9797979797979797979797979797979797979797979797979797979797979797","share_difficulty":3,"share_id":"share-1943","share_seq":1943,"template_height":800060},{"accepted_at_ms":999991,"credit_policy":null,"job_id":"job-1709","job_issued_at_ms":999998,"miner_id":"miner-4","network_difficulty":1000009,"ntime":1700001709,"order_key":"04:miner-4","p2mr_program_hex":"adadadadadadadadadadadadadadadadadadadadadadadadadadadadadadadad","share_difficulty":7,"share_id":"share-1709","share_seq":1709,"template_height":800053},{"accepted_at_ms":999981,"credit_policy":null,"job_id":"job-417","job_issued_at_ms":999998,"miner_id":"miner-10","network_difficulty":1000009,"ntime":1700000417,"order_key":"10:miner-10","p2mr_program_hex":"a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1","share_difficulty":2,"share_id":"share-417","share_seq":417,"template_height":800013},{"accepted_at_ms":999974,"credit_policy":null,"job_id":"job-1589","job_issued_at_ms":999996,"miner_id":"miner-5","network_difficulty":1000008,"ntime":1700001589,"order_key":"05:miner-5","p2mr_program_hex":"3535353535353535353535353535353535353535353535353535353535353535","share_difficulty":1000,"share_id":"share-1589","share_seq":1589,"template_height":800049},{"accepted_at_ms":999992,"credit_policy":"stale-grace","job_id":"job-2844","job_issued_at_ms":999995,"miner_id":"miner-6","network_difficulty":1000005,"ntime":1700002844,"order_key":"06:miner-6","p2mr_program_hex":"1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c","share_difficulty":1,"share_id":"share-2844","share_seq":2844,"template_height":800088},{"accepted_at_ms":999988,"credit_policy":"stale-grace","job_id":"job-2735","job_issued_at_ms":999990,"miner_id":"miner-7","network_difficulty":1000015,"ntime":1700002735,"order_key":"07:miner-7","p2mr_program_hex":"afafafafafafafafafafafafafafafafafafafafafafafafafafafafafafafaf","share_difficulty":19,"share_id":"share-2735","share_seq":2735,"template_height":800085},{"accepted_at_ms":999982,"credit_policy":"stale-grace","job_id":"job-1732","job_issued_at_ms":999974,"miner_id":"miner-5","network_difficulty":1000015,"ntime":1700001732,"order_key":"05:miner-5","p2mr_program_hex":"c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4","share_difficulty":1,"share_id":"share-1732","share_seq":1732,"template_height":800054},{"accepted_at_ms":999990,"credit_policy":null,"job_id":"job-3658","job_issued_at_ms":999971,"miner_id":"miner-6","network_difficulty":1000003,"ntime":1700003658,"order_key":"06:miner-6","p2mr_program_hex":"4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a","share_difficulty":1,"share_id":"share-3658","share_seq":3658,"template_height":800114},{"accepted_at_ms":999995,"credit_policy":null,"job_id":"job-1631","job_issued_at_ms":999980,"miner_id":"miner-3","network_difficulty":1000016,"ntime":1700001631,"order_key":"03:miner-3","p2mr_program_hex":"5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f","share_difficulty":19,"share_id":"share-1631","share_seq":1631,"template_height":800050},{"accepted_at_ms":999978,"credit_policy":null,"job_id":"job-1406","job_issued_at_ms":999992,"miner_id":"miner-9","network_difficulty":1000012,"ntime":1700001406,"order_key":"09:miner-9","p2mr_program_hex":"7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e","share_difficulty":101,"share_id":"share-1406","share_seq":1406,"template_height":800043},{"accepted_at_ms":999996,"credit_policy":null,"job_id":"job-2941","job_issued_at_ms":999983,"miner_id":"miner-4","network_difficulty":1000000,"ntime":1700002941,"order_key":"04:miner-4","p2mr_program_hex":"7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d","share_difficulty":3,"share_id":"share-2941","share_seq":2941,"template_height":800091},{"accepted_at_ms":999991,"credit_policy":"stale-grace","job_id":"job-2576","job_issued_at_ms":999994,"miner_id":"miner-2","network_difficulty":1000009,"ntime":1700002576,"order_key":"02:miner-2","p2mr_program_hex":"1010101010101010101010101010101010101010101010101010101010101010","share_difficulty":2,"share_id":"share-2576","share_seq":2576,"template_height":800080},{"accepted_at_ms":999970,"credit_policy":"stale-grace","job_id":"job-3091","job_issued_at_ms":1000000,"miner_id":"miner-0","network_difficulty":1000014,"ntime":1700003091,"order_key":"00:miner-0","p2mr_program_hex":"1313131313131313131313131313131313131313131313131313131313131313","share_difficulty":19,"share_id":"share-3091","share_seq":3091,"template_height":800096},{"accepted_at_ms":999992,"credit_policy":null,"job_id":"job-3963","job_issued_at_ms":999991,"miner_id":"miner-3","network_difficulty":1000002,"ntime":1700003963,"order_key":"03:miner-3","p2mr_program_hex":"7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b","share_difficulty":7,"share_id":"share-3963","share_seq":3963,"template_height":800123},{"accepted_at_ms":999975,"credit_policy":null,"job_id":"job-2198","job_issued_at_ms":999978,"miner_id":"miner-9","network_difficulty":1000005,"ntime":1700002198,"order_key":"09:miner-9","p2mr_program_hex":"9696969696969696969696969696969696969696969696969696969696969696","share_difficulty":19,"share_id":"share-2198","share_seq":2198,"template_height":800068},{"accepted_at_ms":999988,"credit_policy":null,"job_id":"job-326","job_issued_at_ms":999985,"miner_id":"miner-7","network_difficulty":1000003,"ntime":1700000326,"order_key":"07:miner-7","p2mr_program_hex":"4646464646464646464646464646464646464646464646464646464646464646","share_difficulty":3,"share_id":"share-326","share_seq":326,"template_height":800010},{"accepted_at_ms":999974,"credit_policy":null,"job_id":"job-2873","job_issued_at_ms":999984,"miner_id":"miner-2","network_difficulty":1000000,"ntime":1700002873,"order_key":"02:miner-2","p2mr_program_hex":"3939393939393939393939393939393939393939393939393939393939393939","share_difficulty":101,"share_id":"share-2873","share_seq":2873,"template_height":800089},{"accepted_at_ms":999977,"credit_policy":null,"job_id":"job-3418","job_issued_at_ms":999993,"miner_id":"miner-8","network_difficulty":1000001,"ntime":1700003418,"order_key":"08:miner-8","p2mr_program_hex":"5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a","share_difficulty":2,"share_id":"share-3418","share_seq":3418,"template_height":800106},{"accepted_at_ms":999988,"credit_policy":"stale-grace","job_id":"job-1818","job_issued_at_ms":999991,"miner_id":"miner-3","network_difficulty":1000016,"ntime":1700001818,"order_key":"03:miner-3","p2mr_program_hex":"1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a","share_difficulty":1,"share_id":"share-1818","share_seq":1818,"template_height":800056},{"accepted_at_ms":999998,"credit_policy":null,"job_id":"job-2979","job_issued_at_ms":999970,"miner_id":"miner-9","network_difficulty":1000004,"ntime":1700002979,"order_key":"09:miner-9","p2mr_program_hex":"a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3","share_difficulty":1000,"share_id":"share-2979","share_seq":2979,"template_height":800093},{"accepted_at_ms":1000038,"credit_policy":null,"job_id":"job-968","job_issued_at_ms":1000057,"miner_id":"miner-0","network_difficulty":1000016,"ntime":1700000968,"order_key":"00:miner-0","p2mr_program_hex":"c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8","share_difficulty":1000,"share_id":"share-968","share_seq":968,"template_height":800030},{"accepted_at_ms":999986,"credit_policy":null,"job_id":"job-3403","job_issued_at_ms":999987,"miner_id":"miner-4","network_difficulty":1000003,"ntime":1700003403,"order_key":"04:miner-4","p2mr_program_hex":"4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b","share_difficulty":19,"share_id":"share-3403","share_seq":3403,"template_height":800106},{"accepted_at_ms":1000031,"credit_policy":null,"job_id":"job-2509","job_issued_at_ms":1000024,"miner_id":"miner-1","network_difficulty":1000010,"ntime":1700002509,"order_key":"01:miner-1","p2mr_program_hex":"cdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcd","share_difficulty":101,"share_id":"share-2509","share_seq":2509,"template_height":800078},{"accepted_at_ms":999999,"credit_policy":null,"job_id":"job-203","job_issued_at_ms":999998,"miner_id":"miner-5","network_difficulty":1000016,"ntime":1700000203,"order_key":"05:miner-5","p2mr_program_hex":"cbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcb","share_difficulty":101,"share_id":"share-203","share_seq":203,"template_height":800006},{"accepted_at_ms":999996,"credit_policy":null,"job_id":"job-3908","job_issued_at_ms":999991,"miner_id":"miner-3","network_difficulty":1000015,"ntime":1700003908,"order_key":"03:miner-3","p2mr_program_hex":"4444444444444444444444444444444444444444444444444444444444444444","share_difficulty":2,"share_id":"share-3908","share_seq":3908,"template_height":800122},{"accepted_at_ms":999990,"credit_policy":null,"job_id":"job-445","job_issued_at_ms":999987,"miner_id":"miner-5","network_difficulty":1000003,"ntime":1700000445,"order_key":"05:miner-5","p2mr_program_hex":"bdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbd","share_difficulty":19,"share_id":"share-445","share_seq":445,"template_height":800013},{"accepted_at_ms":999972,"credit_policy":null,"job_id":"job-1901","job_issued_at_ms":999973,"miner_id":"miner-9","network_difficulty":1000014,"ntime":1700001901,"order_key":"09:miner-9","p2mr_program_hex":"6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d","share_difficulty":101,"share_id":"share-1901","share_seq":1901,"template_height":800059},{"accepted_at_ms":999971,"credit_policy":null,"job_id":"job-149","job_issued_at_ms":999982,"miner_id":"miner-6","network_difficulty":1000013,"ntime":1700000149,"order_key":"06:miner-6","p2mr_program_hex":"9595959595959595959595959595959595959595959595959595959595959595","share_difficulty":2,"share_id":"share-149","share_seq":149,"template_height":800004},{"accepted_at_ms":999997,"credit_policy":"stale-grace","job_id":"job-2799","job_issued_at_ms":999987,"miner_id":"miner-5","network_difficulty":1000011,"ntime":1700002799,"order_key":"05:miner-5","p2mr_program_hex":"efefefefefefefefefefefefefefefefefefefefefefefefefefefefefefefef","share_difficulty":1000,"share_id":"share-2799","share_seq":2799,"template_height":800087},{"accepted_at_ms":999996,"credit_policy":"stale-grace","job_id":"job-2287","job_issued_at_ms":1000012,"miner_id":"miner-10","network_difficulty":1000009,"ntime":1700002287,"order_key":"10:miner-10","p2mr_program_hex":"efefefefefefefefefefefefefefefefefefefefefefefefefefefefefefefef","share_difficulty":2,"share_id":"share-2287","share_seq":2287,"template_height":800071},{"accepted_at_ms":999975,"credit_policy":null,"job_id":"job-1006","job_issued_at_ms":999974,"miner_id":"miner-5","network_difficulty":1000003,"ntime":1700001006,"order_key":"05:miner-5","p2mr_program_hex":"eeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeee","share_difficulty":7,"share_id":"share-1006","share_seq":1006,"template_height":800031},{"accepted_at_ms":999975,"credit_policy":null,"job_id":"job-286","job_issued_at_ms":999992,"miner_id":"miner-0","network_difficulty":1000014,"ntime":1700000286,"order_key":"00:miner-0","p2mr_program_hex":"1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e","share_difficulty":19,"share_id":"share-286","share_seq":286,"template_height":800008},{"accepted_at_ms":999984,"credit_policy":null,"job_id":"job-335","job_issued_at_ms":999998,"miner_id":"miner-5","network_difficulty":1000012,"ntime":1700000335,"order_key":"05:miner-5","p2mr_program_hex":"4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f","share_difficulty":19,"share_id":"share-335","share_seq":335,"template_height":800010},{"accepted_at_ms":999993,"credit_policy":null,"job_id":"job-3010","job_issued_at_ms":999982,"miner_id":"miner-7","network_difficulty":1000001,"ntime":1700003010,"order_key":"07:miner-7","p2mr_program_hex":"c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2","share_difficulty":101,"share_id":"share-3010","share_seq":3010,"template_height":800094},{"accepted_at_ms":999998,"credit_policy":"stale-grace","job_id":"job-1381","job_issued_at_ms":999981,"miner_id":"miner-6","network_difficulty":1000004,"ntime":1700001381,"order_key":"06:miner-6","p2mr_program_hex":"6565656565656565656565656565656565656565656565656565656565656565","share_difficulty":101,"share_id":"share-1381","share_seq":1381,"template_height":800043},{"accepted_at_ms":999997,"credit_policy":null,"job_id":"job-3790","job_issued_at_ms":999987,"miner_id":"miner-6","network_difficulty":1000016,"ntime":1700003790,"order_key":"06:miner-6","p2mr_program_hex":"cececececececececececececececececececececececececececececececece","share_difficulty":101,"share_id":"share-3790","share_seq":3790,"template_height":800118},{"accepted_at_ms":999979,"credit_policy":"stale-grace","job_id":"job-3141","job_issued_at_ms":999988,"miner_id":"miner-6","network_difficulty":1000013,"ntime":1700003141,"order_key":"06:miner-6","p2mr_program_hex":"4545454545454545454545454545454545454545454545454545454545454545","share_difficulty":2,"share_id":"share-3141","share_seq":3141,"template_height":800098},{"accepted_at_ms":999986,"credit_policy":null,"job_id":"job-992","job_issued_at_ms":999989,"miner_id":"miner-2","network_difficulty":1000006,"ntime":1700000992,"order_key":"02:miner-2","p2mr_program_hex":"e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0","share_difficulty":7,"share_id":"share-992","share_seq":992,"template_height":800031},{"accepted_at_ms":999985,"credit_policy":null,"job_id":"job-339","job_issued_at_ms":999984,"miner_id":"miner-9","network_difficulty":1000016,"ntime":1700000339,"order_key":"09:miner-9","p2mr_program_hex":"5353535353535353535353535353535353535353535353535353535353535353","share_difficulty":1,"share_id":"share-339","share_seq":339,"template_height":800010},{"accepted_at_ms":999976,"credit_policy":"stale-grace","job_id":"job-3628","job_issued_at_ms":999988,"miner_id":"miner-9","network_difficulty":1000007,"ntime":1700003628,"order_key":"09:miner-9","p2mr_program_hex":"2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c","share_difficulty":3,"share_id":"share-3628","share_seq":3628,"template_height":800113},{"accepted_at_ms":999982,"credit_policy":null,"job_id":"job-3436","job_issued_at_ms":999997,"miner_id":"miner-4","network_difficulty":1000002,"ntime":1700003436,"order_key":"04:miner-4","p2mr_program_hex":"6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c","share_difficulty":2,"share_id":"share-3436","share_seq":3436,"template_height":800107},{"accepted_at_ms":1000000,"credit_policy":null,"job_id":"job-1377","job_issued_at_ms":999983,"miner_id":"miner-2","network_difficulty":1000000,"ntime":1700001377,"order_key":"02:miner-2","p2mr_program_hex":"6161616161616161616161616161616161616161616161616161616161616161","share_difficulty":1000,"share_id":"share-1377","share_seq":1377,"template_height":800043},{"accepted_at_ms":999991,"credit_policy":null,"job_id":"job-2617","job_issued_at_ms":999995,"miner_id":"miner-10","network_difficulty":1000016,"ntime":1700002617,"order_key":"10:miner-10","p2mr_program_hex":"3939393939393939393939393939393939393939393939393939393939393939","share_difficulty":3,"share_id":"share-2617","share_seq":2617,"template_height":800081},{"accepted_at_ms":999988,"credit_policy":null,"job_id":"job-1675","job_issued_at_ms":999988,"miner_id":"miner-3","network_difficulty":1000009,"ntime":1700001675,"order_key":"03:miner-3","p2mr_program_hex":"8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b","share_difficulty":2,"share_id":"share-1675","share_seq":1675,"template_height":800052},{"accepted_at_ms":999989,"credit_policy":null,"job_id":"job-1981","job_issued_at_ms":999971,"miner_id":"miner-1","network_difficulty":1000009,"ntime":1700001981,"order_key":"01:miner-1","p2mr_program_hex":"bdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbd","share_difficulty":101,"share_id":"share-1981","share_seq":1981,"template_height":800061},{"accepted_at_ms":999976,"credit_policy":null,"job_id":"job-3159","job_issued_at_ms":999974,"miner_id":"miner-2","network_difficulty":1000014,"ntime":1700003159,"order_key":"02:miner-2","p2mr_program_hex":"5757575757575757575757575757575757575757575757575757575757575757","share_difficulty":3,"share_id":"share-3159","share_seq":3159,"template_height":800098},{"accepted_at_ms":999991,"credit_policy":"stale-grace","job_id":"job-1501","job_issued_at_ms":999970,"miner_id":"miner-5","network_difficulty":1000005,"ntime":1700001501,"order_key":"05:miner-5","p2mr_program_hex":"dddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddd","share_difficulty":1000,"share_id":"share-1501","share_seq":1501,"template_height":800046},{"accepted_at_ms":999987,"credit_policy":null,"job_id":"job-619","job_issued_at_ms":999972,"miner_id":"miner-3","network_difficulty":1000007,"ntime":1700000619,"order_key":"03:miner-3","p2mr_program_hex":"6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b","share_difficulty":3,"share_id":"share-619","share_seq":619,"template_height":800019},{"accepted_at_ms":999992,"credit_policy":"stale-grace","job_id":"job-708","job_issued_at_ms":999976,"miner_id":"miner-4","network_difficulty":1000011,"ntime":1700000708,"order_key":"04:miner-4","p2mr_program_hex":"c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4","share_difficulty":19,"share_id":"share-708","share_seq":708,"template_height":800022},{"accepted_at_ms":999973,"credit_policy":null,"job_id":"job-220","job_issued_at_ms":999979,"miner_id":"miner-0","network_difficulty":1000016,"ntime":1700000220,"order_key":"00:miner-0","p2mr_program_hex":"dcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdc","share_difficulty":19,"share_id":"share-220","share_seq":220,"template_height":800006},{"accepted_at_ms":999986,"credit_policy":null,"job_id":"job-3100","job_issued_at_ms":999995,"miner_id":"miner-9","network_difficulty":1000006,"ntime":1700003100,"order_key":"09:miner-9","p2mr_program_hex":"1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c","share_difficulty":19,"share_id":"share-3100","share_seq":3100,"template_height":800096},{"accepted_at_ms":999977,"credit_policy":null,"job_id":"job-2177","job_issued_at_ms":999978,"miner_id":"miner-10","network_difficulty":1000001,"ntime":1700002177,"order_key":"10:miner-10","p2mr_program_hex":"8181818181818181818181818181818181818181818181818181818181818181","share_difficulty":101,"share_id":"share-2177","share_seq":2177,"template_height":800068},{"accepted_at_ms":999981,"credit_policy":null,"job_id":"job-131","job_issued_at_ms":999976,"miner_id":"miner-10","network_difficulty":1000012,"ntime":1700000131,"order_key":"10:miner-10","p2mr_program_hex":"8383838383838383838383838383838383838383838383838383838383838383","share_difficulty":7,"share_id":"share-131","share_seq":131,"template_height":800004},{"accepted_at_ms":999984,"credit_policy":null,"job_id":"job-807","job_issued_at_ms":999985,"miner_id":"miner-4","network_difficulty":1000008,"ntime":1700000807,"order_key":"04:miner-4","p2mr_program_hex":"2727272727272727272727272727272727272727272727272727272727272727","share_difficulty":7,"share_id":"share-807","share_seq":807,"template_height":800025},{"accepted_at_ms":999992,"credit_policy":"stale-grace","job_id":"job-869","job_issued_at_ms":999996,"miner_id":"miner-0","network_difficulty":1000002,"ntime":1700000869,"order_key":"00:miner-0","p2mr_program_hex":"6565656565656565656565656565656565656565656565656565656565656565","share_difficulty":3,"share_id":"share-869","share_seq":869,"template_height":800027},{"accepted_at_ms":999972,"credit_policy":null,"job_id":"job-1850","job_issued_at_ms":999996,"miner_id":"miner-2","network_difficulty":1000014,"ntime":1700001850,"order_key":"02:miner-2","p2mr_program_hex":"3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a","share_difficulty":1,"share_id":"share-1850","share_seq":1850,"template_height":800057},{"accepted_at_ms":999982,"credit_policy":"stale-grace","job_id":"job-1452","job_issued_at_ms":999992,"miner_id":"miner-0","network_difficulty":1000007,"ntime":1700001452,"order_key":"00:miner-0","p2mr_program_hex":"acacacacacacacacacacacacacacacacacacacacacacacacacacacacacacacac","share_difficulty":2,"share_id":"share-1452","share_seq":1452,"template_height":800045},{"accepted_at_ms":1000000,"credit_policy":null,"job_id":"job-3553","job_issued_at_ms":1000000,"miner_id":"miner-0","network_difficulty":1000000,"ntime":1700003553,"order_key":"00:miner-0","p2mr_program_hex":"e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1","share_difficulty":1,"share_id":"share-3553","share_seq":3553,"template_height":800111},{"accepted_at_ms":999996,"credit_policy":"stale-grace","job_id":"job-1171","job_issued_at_ms":999997,"miner_id":"miner-5","network_difficulty":1000015,"ntime":1700001171,"order_key":"05:miner-5","p2mr_program_hex":"9393939393939393939393939393939393939393939393939393939393939393","share_difficulty":2,"share_id":"share-1171","share_seq":1171,"template_height":800036},{"accepted_at_ms":999988,"credit_policy":"stale-grace","job_id":"job-1609","job_issued_at_ms":999993,"miner_id":"miner-3","network_difficulty":1000011,"ntime":1700001609,"order_key":"03:miner-3","p2mr_program_hex":"4949494949494949494949494949494949494949494949494949494949494949","share_difficulty":1,"share_id":"share-1609","share_seq":1609,"template_height":800050},{"accepted_at_ms":999988,"credit_policy":null,"job_id":"job-2542","job_issued_at_ms":999979,"miner_id":"miner-1","network_difficulty":1000009,"ntime":1700002542,"order_key":"01:miner-1","p2mr_program_hex":"eeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeee","share_difficulty":3,"share_id":"share-2542","share_seq":2542,"template_height":800079},{"accepted_at_ms":999972,"credit_policy":null,"job_id":"job-3421","job_issued_at_ms":999983,"miner_id":"miner-0","network_difficulty":1000004,"ntime":1700003421,"order_key":"00:miner-0","p2mr_program_hex":"5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d","share_difficulty":7,"share_id":"share-3421","share_seq":3421,"template_height":800106},{"accepted_at_ms":999993,"credit_policy":"stale-grace","job_id":"job-1409","job_issued_at_ms":999973,"miner_id":"miner-1","network_difficulty":1000015,"ntime":1700001409,"order_key":"01:miner-1","p2mr_program_hex":"8181818181818181818181818181818181818181818181818181818181818181","share_difficulty":101,"share_id":"share-1409","share_seq":1409,"template_height":800044},{"accepted_at_ms":999981,"credit_policy":null,"job_id":"job-2315","job_issued_at_ms":999972,"miner_id":"miner-5","network_difficulty":1000003,"ntime":1700002315,"order_key":"05:miner-5","p2mr_program_hex":"0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b","share_difficulty":3,"share_id":"share-2315","share_seq":2315,"template_height":800072},{"accepted_at_ms":999991,"credit_policy":null,"job_id":"job-533","job_issued_at_ms":999975,"miner_id":"miner-5","network_difficulty":1000006,"ntime":1700000533,"order_key":"05:miner-5","p2mr_program_hex":"1515151515151515151515151515151515151515151515151515151515151515","share_difficulty":3,"share_id":"share-533","share_seq":533,"template_height":800016},{"accepted_at_ms":999974,"credit_policy":null,"job_id":"job-2603","job_issued_at_ms":999971,"miner_id":"miner-7","network_difficulty":1000002,"ntime":1700002603,"order_key":"07:miner-7","p2mr_program_hex":"2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b","share_difficulty":1000,"share_id":"share-2603","share_seq":2603,"template_height":800081},{"accepted_at_ms":1000229,"credit_policy":null,"job_id":"job-608","job_issued_at_ms":999993,"miner_id":"miner-3","network_difficulty":1000013,"ntime":1700000608,"order_key":"03:miner-3","p2mr_program_hex":"6060606060606060606060606060606060606060606060606060606060606060","share_difficulty":1,"share_id":"share-608","share_seq":608,"template_height":800019},{"accepted_at_ms":999987,"credit_policy":null,"job_id":"job-28","job_issued_at_ms":999979,"miner_id":"miner-6","network_difficulty":1000011,"ntime":1700000028,"order_key":"06:miner-6","p2mr_program_hex":"1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c","share_difficulty":1,"share_id":"share-28","share_seq":28,"template_height":800000},{"accepted_at_ms":999987,"credit_policy":null,"job_id":"job-1100","job_issued_at_ms":999977,"miner_id":"miner-0","network_difficulty":1000012,"ntime":1700001100,"order_key":"00:miner-0","p2mr_program_hex":"4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c","share_difficulty":1000,"share_id":"share-1100","share_seq":1100,"template_height":800034},{"accepted_at_ms":999984,"credit_policy":null,"job_id":"job-2747","job_issued_at_ms":999973,"miner_id":"miner-8","network_difficulty":1000010,"ntime":1700002747,"order_key":"08:miner-8","p2mr_program_hex":"bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb","share_difficulty":7,"share_id":"share-2747","share_seq":2747,"template_height":800085},{"accepted_at_ms":999973,"credit_policy":null,"job_id":"job-1872","job_issued_at_ms":999977,"miner_id":"miner-2","network_difficulty":1000002,"ntime":1700001872,"order_key":"02:miner-2","p2mr_program_hex":"5050505050505050505050505050505050505050505050505050505050505050","share_difficulty":7,"share_id":"share-1872","share_seq":1872,"template_height":800058},{"accepted_at_ms":999973,"credit_policy":"stale-grace","job_id":"job-1517","job_issued_at_ms":999989,"miner_id":"miner-10","network_difficulty":1000004,"ntime":1700001517,"order_key":"10:miner-10","p2mr_program_hex":"edededededededededededededededededededededededededededededededed","share_difficulty":2,"share_id":"share-1517","share_seq":1517,"template_height":800047},{"accepted_at_ms":999974,"credit_policy":null,"job_id":"job-3558","job_issued_at_ms":999984,"miner_id":"miner-5","network_difficulty":1000005,"ntime":1700003558,"order_key":"05:miner-5","p2mr_program_hex":"e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6","share_difficulty":1000,"share_id":"share-3558","share_seq":3558,"template_height":800111},{"accepted_at_ms":999999,"credit_policy":null,"job_id":"job-3015","job_issued_at_ms":999990,"miner_id":"miner-1","network_difficulty":1000006,"ntime":1700003015,"order_key":"01:miner-1","p2mr_program_hex":"c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7","share_difficulty":101,"share_id":"share-3015","share_seq":3015,"template_height":800094},{"accepted_at_ms":999990,"credit_policy":null,"job_id":"job-1214","job_issued_at_ms":999988,"miner_id":"miner-4","network_difficulty":1000007,"ntime":1700001214,"order_key":"04:miner-4","p2mr_program_hex":"bebebebebebebebebebebebebebebebebebebebebebebebebebebebebebebebe","share_difficulty":2,"share_id":"share-1214","share_seq":1214,"template_height":800037},{"accepted_at_ms":999973,"credit_policy":null,"job_id":"job-2259","job_issued_at_ms":999987,"miner_id":"miner-4","network_difficulty":1000015,"ntime":1700002259,"order_key":"04:miner-4","p2mr_program_hex":"d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3","share_difficulty":1000,"share_id":"share-2259","share_seq":2259,"template_height":800070},{"accepted_at_ms":1000147,"credit_policy":null,"job_id":"job-3522","job_issued_at_ms":999983,"miner_id":"miner-2","network_difficulty":1000003,"ntime":1700003522,"order_key":"02:miner-2","p2mr_program_hex":"c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2","share_difficulty":1000,"share_id":"share-3522","share_seq":3522,"template_height":800110},{"accepted_at_ms":999996,"credit_policy":null,"job_id":"job-3557","job_issued_at_ms":999992,"miner_id":"miner-4","network_difficulty":1000004,"ntime":1700003557,"order_key":"04:miner-4","p2mr_program_hex":"e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5","share_difficulty":1,"share_id":"share-3557","share_seq":3557,"template_height":800111},{"accepted_at_ms":999991,"credit_policy":null,"job_id":"job-1662","job_issued_at_ms":1000321,"miner_id":"miner-1","network_difficulty":1000013,"ntime":1700001662,"order_key":"01:miner-1","p2mr_program_hex":"7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e","share_difficulty":3,"share_id":"share-1662","share_seq":1662,"template_height":800051},{"accepted_at_ms":999997,"credit_policy":null,"job_id":"job-2750","job_issued_at_ms":999981,"miner_id":"miner-0","network_difficulty":1000013,"ntime":1700002750,"order_key":"00:miner-0","p2mr_program_hex":"bebebebebebebebebebebebebebebebebebebebebebebebebebebebebebebebe","share_difficulty":19,"share_id":"share-2750","share_seq":2750,"template_height":800085},{"accepted_at_ms":1000414,"credit_policy":null,"job_id":"job-3380","job_issued_at_ms":999978,"miner_id":"miner-3","network_difficulty":1000014,"ntime":1700003380,"order_key":"03:miner-3","p2mr_program_hex":"3434343434343434343434343434343434343434343434343434343434343434","share_difficulty":1000,"share_id":"share-3380","share_seq":3380,"template_height":800105},{"accepted_at_ms":999970,"credit_policy":null,"job_id":"job-481","job_issued_at_ms":999983,"miner_id":"miner-8","network_difficulty":1000005,"ntime":1700000481,"order_key":"08:miner-8","p2mr_program_hex":"e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1","share_difficulty":7,"share_id":"share-481","share_seq":481,"template_height":800015},{"accepted_at_ms":999978,"credit_policy":null,"job_id":"job-639","job_issued_at_ms":999994,"miner_id":"miner-1","network_difficulty":1000010,"ntime":1700000639,"order_key":"01:miner-1","p2mr_program_hex":"7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f","share_difficulty":3,"share_id":"share-639","share_seq":639,"template_height":800019},{"accepted_at_ms":999993,"credit_policy":null,"job_id":"job-222","job_issued_at_ms":999993,"miner_id":"miner-2","network_difficulty":1000001,"ntime":1700000222,"order_key":"02:miner-2","p2mr_program_hex":"dededededededededededededededededededededededededededededededede","share_difficulty":101,"share_id":"share-222","share_seq":222,"template_height":800006},{"accepted_at_ms":999971,"credit_policy":null,"job_id":"job-177","job_issued_at_ms":999971,"miner_id":"miner-1","network_difficulty":1000007,"ntime":1700000177,"order_key":"01:miner-1","p2mr_program_hex":"b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1","share_difficulty":7,"share_id":"share-177","share_seq":177,"template_height":800005},{"accepted_at_ms":999996,"credit_policy":null,"job_id":"job-944","job_issued_at_ms":999974,"miner_id":"miner-9","network_difficulty":1000009,"ntime":1700000944,"order_key":"09:miner-9","p2mr_program_hex":"b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0","share_difficulty":1000,"share_id":"share-944","share_seq":944,"template_height":800029},{"accepted_at_ms":999989,"credit_policy":null,"job_id":"job-2093","job_issued_at_ms":999985,"miner_id":"miner-3","network_difficulty":1000002,"ntime":1700002093,"order_key":"03:miner-3","p2mr_program_hex":"2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d","share_difficulty":1,"share_id":"share-2093","share_seq":2093,"template_height":800065},{"accepted_at_ms":999974,"credit_policy":null,"job_id":"job-1342","job_issued_at_ms":999988,"miner_id":"miner-0","network_difficulty":1000016,"ntime":1700001342,"order_key":"00:miner-0","p2mr_program_hex":"3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e","share_difficulty":2,"share_id":"share-1342","share_seq":1342,"template_height":800041},{"accepted_at_ms":999991,"credit_policy":"stale-grace","job_id":"job-111","job_issued_at_ms":999986,"miner_id":"miner-1","network_difficulty":1000009,"ntime":1700000111,"order_key":"01:miner-1","p2mr_program_hex":"6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f","share_difficulty":3,"share_id":"share-111","share_seq":111,"template_height":800003},{"accepted_at_ms":999989,"credit_policy":null,"job_id":"job-3114","job_issued_at_ms":999972,"miner_id":"miner-1","network_difficulty":1000003,"ntime":1700003114,"order_key":"01:miner-1","p2mr_program_hex":"2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a","share_difficulty":2,"share_id":"share-3114","share_seq":3114,"template_height":800097},{"accepted_at_ms":999971,"credit_policy":null,"job_id":"job-3105","job_issued_at_ms":999978,"miner_id":"miner-3","network_difficulty":1000011,"ntime":1700003105,"order_key":"03:miner-3","p2mr_program_hex":"2121212121212121212121212121212121212121212121212121212121212121","share_difficulty":2,"share_id":"share-3105","share_seq":3105,"template_height":800097},{"accepted_at_ms":999975,"credit_policy":null,"job_id":"job-3521","job_issued_at_ms":1000000,"miner_id":"miner-1","network_difficulty":1000002,"ntime":1700003521,"order_key":"01:miner-1","p2mr_program_hex":"c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1","share_difficulty":7,"share_id":"share-3521","share_seq":3521,"template_height":800110},{"accepted_at_ms":999981,"credit_policy":"stale-grace","job_id":"job-3177","job_issued_at_ms":999975,"miner_id":"miner-9","network_difficulty":1000015,"ntime":1700003177,"order_key":"09:miner-9","p2mr_program_hex":"6969696969696969696969696969696969696969696969696969696969696969","share_difficulty":3,"share_id":"share-3177","share_seq":3177,"template_height":800099},{"accepted_at_ms":999999,"credit_policy":null,"job_id":"job-3945","job_issued_at_ms":999974,"miner_id":"miner-7","network_difficulty":1000001,"ntime":1700003945,"order_key":"07:miner-7","p2mr_program_hex":"6969696969696969696969696969696969696969696969696969696969696969","share_difficulty":7,"share_id":"share-3945","share_seq":3945,"template_height":800123},{"accepted_at_ms":999979,"credit_policy":"stale-grace","job_id":"job-2109","job_issued_at_ms":999982,"miner_id":"miner-8","network_difficulty":1000001,"ntime":1700002109,"order_key":"08:miner-8","p2mr_program_hex":"3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d","share_difficulty":101,"share_id":"share-2109","share_seq":2109,"template_height":800065},{"accepted_at_ms":999996,"credit_policy":null,"job_id":"job-1299","job_issued_at_ms":999975,"miner_id":"miner-1","network_difficulty":1000007,"ntime":1700001299,"order_key":"01:miner-1","p2mr_program_hex":"1313131313131313131313131313131313131313131313131313131313131313","share_difficulty":7,"share_id":"share-1299","share_seq":1299,"template_height":800040},{"accepted_at_ms":999984,"credit_policy":null,"job_id":"job-3120","job_issued_at_ms":999977,"miner_id":"miner-7","network_difficulty":1000009,"ntime":1700003120,"order_key":"07:miner-7","p2mr_program_hex":"3030303030303030303030303030303030303030303030303030303030303030","share_difficulty":7,"share_id":"share-3120","share_seq":3120,"template_height":800097},{"accepted_at_ms":999986,"credit_policy":null,"job_id":"job-3727","job_issued_at_ms":999974,"miner_id":"miner-9","network_difficulty":1000004,"ntime":1700003727,"order_key":"09:miner-9","p2mr_program_hex":"8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f","share_difficulty":1,"share_id":"share-3727","share_seq":3727,"template_height":800116},{"accepted_at_ms":999995,"credit_policy":null,"job_id":"job-1123","job_issued_at_ms":999984,"miner_id":"miner-1","network_difficulty":1000001,"ntime":1700001123,"order_key":"01:miner-1","p2mr_program_hex":"6363636363636363636363636363636363636363636363636363636363636363","share_difficulty":19,"share_id":"share-1123","share_seq":1123,"template_height":800035},{"accepted_at_ms":999990,"credit_policy":null,"job_id":"job-3900","job_issued_at_ms":999996,"miner_id":"miner-6","network_difficulty":1000007,"ntime":1700003900,"order_key":"06:miner-6","p2mr_program_hex":"3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c","share_difficulty":3,"share_id":"share-3900","share_seq":3900,"template_height":800121},{"accepted_at_ms":999977,"credit_policy":null,"job_id":"job-3960","job_issued_at_ms":999991,"miner_id":"miner-0","network_difficulty":1000016,"ntime":1700003960,"order_key":"00:miner-0","p2mr_program_hex":"7878787878787878787878787878787878787878787878787878787878787878","share_difficulty":1000,"share_id":"share-3960","share_seq":3960,"template_height":800123},{"accepted_at_ms":999980,"credit_policy":null,"job_id":"job-3669","job_issued_at_ms":999997,"miner_id":"miner-6","network_difficulty":1000014,"ntime":1700003669,"order_key":"06:miner-6","p2mr_program_hex":"5555555555555555555555555555555555555555555555555555555555555555","share_difficulty":3,"share_id":"share-3669","share_seq":3669,"template_height":800114},{"accepted_at_ms":999972,"credit_policy":null,"job_id":"job-1456","job_issued_at_ms":999982,"miner_id":"miner-4","network_difficulty":1000011,"ntime":1700001456,"order_key":"04:miner-4","p2mr_program_hex":"b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0","share_difficulty":2,"share_id":"share-1456","share_seq":1456,"template_height":800045},{"accepted_at_ms":999982,"credit_policy":null,"job_id":"job-3423","job_issued_at_ms":999983,"miner_id":"miner-2","network_difficulty":1000006,"ntime":1700003423,"order_key":"02:miner-2","p2mr_program_hex":"5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f","share_difficulty":3,"share_id":"share-3423","share_seq":3423,"template_height":800106},{"accepted_at_ms":999986,"credit_policy":null,"job_id":"job-2224","job_issued_at_ms":999984,"miner_id":"miner-2","network_difficulty":1000014,"ntime":1700002224,"order_key":"02:miner-2","p2mr_program_hex":"b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0","share_difficulty":7,"share_id":"share-2224","share_seq":2224,"template_height":800069},{"accepted_at_ms":999994,"credit_policy":null,"job_id":"job-399","job_issued_at_ms":999985,"miner_id":"miner-3","network_difficulty":1000008,"ntime":1700000399,"order_key":"03:miner-3","p2mr_program_hex":"8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f","share_difficulty":101,"share_id":"share-399","share_seq":399,"template_height":800012},{"accepted_at_ms":999983,"credit_policy":null,"job_id":"job-3294","job_issued_at_ms":999996,"miner_id":"miner-5","network_difficulty":1000013,"ntime":1700003294,"order_key":"05:miner-5","p2mr_program_hex":"dededededededededededededededededededededededededededededededede","share_difficulty":101,"share_id":"share-3294","share_seq":3294,"template_height":800102},{"accepted_at_ms":999974,"credit_policy":null,"job_id":"job-3951","job_issued_at_ms":999975,"miner_id":"miner-2","network_difficulty":1000007,"ntime":1700003951,"order_key":"02:miner-2","p2mr_program_hex":"6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f","share_difficulty":101,"share_id":"share-3951","share_seq":3951,"template_height":800123},{"accepted_at_ms":1000000,"credit_policy":"stale-grace","job_id":"job-2051","job_issued_at_ms":999976,"miner_id":"miner-5","network_difficulty":1000011,"ntime":1700002051,"order_key":"05:miner-5","p2mr_program_hex":"0303030303030303030303030303030303030303030303030303030303030303","share_difficulty":3,"share_id":"share-2051","share_seq":2051,"template_height":800064},{"accepted_at_ms":999996,"credit_policy":"stale-grace","job_id":"job-1036","job_issued_at_ms":999980,"miner_id":"miner-2","network_difficulty":1000016,"ntime":1700001036,"order_key":"02:miner-2","p2mr_program_hex":"0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c","share_difficulty":1000,"share_id":"share-1036","share_seq":1036,"template_height":800032},{"accepted_at_ms":999981,"credit_policy":null,"job_id":"job-3970","job_issued_at_ms":999996,"miner_id":"miner-10","network_difficulty":1000009,"ntime":1700003970,"order_key":"10:miner-10","p2mr_program_hex":"8282828282828282828282828282828282828282828282828282828282828282","share_difficulty":19,"share_id":"share-3970","share_seq":3970,"template_height":800124},{"accepted_at_ms":999999,"credit_policy":null,"job_id":"job-3324","job_issued_at_ms":999978,"miner_id":"miner-2","network_difficulty":1000009,"ntime":1700003324,"order_key":"02:miner-2","p2mr_program_hex":"fcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfc","share_difficulty":101,"share_id":"share-3324","share_seq":3324,"template_height":800103},{"accepted_at_ms":999986,"credit_policy":null,"job_id":"job-1574","job_issued_at_ms":999971,"miner_id":"miner-1","network_difficulty":1000010,"ntime":1700001574,"order_key":"01:miner-1","p2mr_program_hex":"2626262626262626262626262626262626262626262626262626262626262626","share_difficulty":1000,"share_id":"share-1574","share_seq":1574,"template_height":800049},{"accepted_at_ms":999973,"credit_policy":null,"job_id":"job-1792","job_issued_at_ms":1000000,"miner_id":"miner-10","network_difficulty":1000007,"ntime":1700001792,"order_key":"10:miner-10","p2mr_program_hex":"0000000000000000000000000000000000000000000000000000000000000000","share_difficulty":101,"share_id":"share-1792","share_seq":1792,"template_height":800056},{"accepted_at_ms":999988,"credit_policy":null,"job_id":"job-920","job_issued_at_ms":999974,"miner_id":"miner-7","network_difficulty":1000002,"ntime":1700000920,"order_key":"07:miner-7","p2mr_program_hex":"9898989898989898989898989898989898989898989898989898989898989898","share_difficulty":19,"share_id":"share-920","share_seq":920,"template_height":800028},{"accepted_at_ms":999972,"credit_policy":null,"job_id":"job-392","job_issued_at_ms":999979,"miner_id":"miner-7","network_difficulty":1000001,"ntime":1700000392,"order_key":"07:miner-7","p2mr_program_hex":"8888888888888888888888888888888888888888888888888888888888888888","share_difficulty":101,"share_id":"share-392","share_seq":392,"template_height":800012},{"accepted_at_ms":999996,"credit_policy":null,"job_id":"job-1334","job_issued_at_ms":1000414,"miner_id":"miner-3","network_difficulty":1000008,"ntime":1700001334,"order_key":"03:miner-3","p2mr_program_hex":"3636363636363636363636363636363636363636363636363636363636363636","share_difficulty":1000,"share_id":"share-1334","share_seq":1334,"template_height":800041},{"accepted_at_ms":999984,"credit_policy":null,"job_id":"job-3985","job_issued_at_ms":999996,"miner_id":"miner-3","network_difficulty":1000007,"ntime":1700003985,"order_key":"03:miner-3","p2mr_program_hex":"9191919191919191919191919191919191919191919191919191919191919191","share_difficulty":101,"share_id":"share-3985","share_seq":3985,"template_height":800124},{"accepted_at_ms":999988,"credit_policy":null,"job_id":"job-1173","job_issued_at_ms":999979,"miner_id":"miner-7","network_difficulty":1000000,"ntime":1700001173,"order_key":"07:miner-7","p2mr_program_hex":"9595959595959595959595959595959595959595959595959595959595959595","share_difficulty":3,"share_id":"share-1173","share_seq":1173,"template_height":800036},{"accepted_at_ms":1000000,"credit_policy":null,"job_id":"job-2013","job_issued_at_ms":999978,"miner_id":"miner-0","network_difficulty":1000007,"ntime":1700002013,"order_key":"00:miner-0","p2mr_program_hex":"dddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddd","share_difficulty":101,"share_id":"share-2013","share_seq":2013,"template_height":800062},{"accepted_at_ms":999991,"credit_policy":null,"job_id":"job-2970","job_issued_at_ms":999973,"miner_id":"miner-0","network_difficulty":1000012,"ntime":1700002970,"order_key":"00:miner-0","p2mr_program_hex":"9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a","share_difficulty":2,"share_id":"share-2970","share_seq":2970,"template_height":800092},{"accepted_at_ms":999970,"credit_policy":null,"job_id":"job-2619","job_issued_at_ms":999972,"miner_id":"miner-1","network_difficulty":1000001,"ntime":1700002619,"order_key":"01:miner-1","p2mr_program_hex":"3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b","share_difficulty":3,"share_id":"share-2619","share_seq":2619,"template_height":800081},{"accepted_at_ms":999977,"credit_policy":"stale-grace","job_id":"job-2054","job_issued_at_ms":999999,"miner_id":"miner-8","network_difficulty":1000014,"ntime":1700002054,"order_key":"08:miner-8","p2mr_program_hex":"0606060606060606060606060606060606060606060606060606060606060606","share_difficulty":7,"share_id":"share-2054","share_seq":2054,"template_height":800064},{"accepted_at_ms":999976,"credit_policy":null,"job_id":"job-2929","job_issued_at_ms":999970,"miner_id":"miner-3","network_difficulty":1000005,"ntime":1700002929,"order_key":"03:miner-3","p2mr_program_hex":"7171717171717171717171717171717171717171717171717171717171717171","share_difficulty":19,"share_id":"share-2929","share_seq":2929,"template_height":800091},{"accepted_at_ms":999985,"credit_policy":null,"job_id":"job-2068","job_issued_at_ms":999976,"miner_id":"miner-0","network_difficulty":1000011,"ntime":1700002068,"order_key":"00:miner-0","p2mr_program_hex":"1414141414141414141414141414141414141414141414141414141414141414","share_difficulty":19,"share_id":"share-2068","share_seq":2068,"template_height":800064},{"accepted_at_ms":999978,"credit_policy":null,"job_id":"job-1231","job_issued_at_ms":1000244,"miner_id":"miner-10","network_difficulty":1000007,"ntime":1700001231,"order_key":"10:miner-10","p2mr_program_hex":"cfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcf","share_difficulty":3,"share_id":"share-1231","share_seq":1231,"template_height":800038},{"accepted_at_ms":999976,"credit_policy":null,"job_id":"job-1242","job_issued_at_ms":999993,"miner_id":"miner-10","network_difficulty":1000001,"ntime":1700001242,"order_key":"10:miner-10","p2mr_program_hex":"dadadadadadadadadadadadadadadadadadadadadadadadadadadadadadadada","share_difficulty":1000,"share_id":"share-1242","share_seq":1242,"template_height":800038},{"accepted_at_ms":999980,"credit_policy":"stale-grace","job_id":"job-1153","job_issued_at_ms":999983,"miner_id":"miner-9","network_difficulty":1000014,"ntime":1700001153,"order_key":"09:miner-9","p2mr_program_hex":"8181818181818181818181818181818181818181818181818181818181818181","share_difficulty":1,"share_id":"share-1153","share_seq":1153,"template_height":800036},{"accepted_at_ms":999994,"credit_policy":null,"job_id":"job-1273","job_issued_at_ms":999993,"miner_id":"miner-8","network_difficulty":1000015,"ntime":1700001273,"order_key":"08:miner-8","p2mr_program_hex":"f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9","share_difficulty":7,"share_id":"share-1273","share_seq":1273,"template_height":800039},{"accepted_at_ms":999979,"credit_policy":"stale-grace","job_id":"job-1813","job_issued_at_ms":999998,"miner_id":"miner-9","network_difficulty":1000011,"ntime":1700001813,"order_key":"09:miner-9","p2mr_program_hex":"1515151515151515151515151515151515151515151515151515151515151515","share_difficulty":2,"share_id":"share-1813","share_seq":1813,"template_height":800056},{"accepted_at_ms":999994,"credit_policy":null,"job_id":"job-1707","job_issued_at_ms":999975,"miner_id":"miner-2","network_difficulty":1000007,"ntime":1700001707,"order_key":"02:miner-2","p2mr_program_hex":"abababababababababababababababababababababababababababababababab","share_difficulty":1000,"share_id":"share-1707","share_seq":1707,"template_height":800053},{"accepted_at_ms":999982,"credit_policy":null,"job_id":"job-2966","job_issued_at_ms":999985,"miner_id":"miner-7","network_difficulty":1000008,"ntime":1700002966,"order_key":"07:miner-7","p2mr_program_hex":"9696969696969696969696969696969696969696969696969696969696969696","share_difficulty":19,"share_id":"share-2966","share_seq":2966,"template_height":800092},{"accepted_at_ms":999977,"credit_policy":null,"job_id":"job-275","job_issued_at_ms":999990,"miner_id":"miner-0","network_difficulty":1000003,"ntime":1700000275,"order_key":"00:miner-0","p2mr_program_hex":"1313131313131313131313131313131313131313131313131313131313131313","share_difficulty":19,"share_id":"share-275","share_seq":275,"template_height":800008},{"accepted_at_ms":999988,"credit_policy":null,"job_id":"job-2067","job_issued_at_ms":999974,"miner_id":"miner-10","network_difficulty":1000010,"ntime":1700002067,"order_key":"10:miner-10","p2mr_program_hex":"1313131313131313131313131313131313131313131313131313131313131313","share_difficulty":19,"share_id":"share-2067","share_seq":2067,"template_height":800064},{"accepted_at_ms":999974,"credit_policy":null,"job_id":"job-2531","job_issued_at_ms":1000271,"miner_id":"miner-1","network_difficulty":1000015,"ntime":1700002531,"order_key":"01:miner-1","p2mr_program_hex":"e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3","share_difficulty":2,"share_id":"share-2531","share_seq":2531,"template_height":800079},{"accepted_at_ms":999980,"credit_policy":null,"job_id":"job-1749","job_issued_at_ms":999973,"miner_id":"miner-0","network_difficulty":1000015,"ntime":1700001749,"order_key":"00:miner-0","p2mr_program_hex":"d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5","share_difficulty":3,"share_id":"share-1749","share_seq":1749,"template_height":800054},{"accepted_at_ms":999984,"credit_policy":"stale-grace","job_id":"job-2514","job_issued_at_ms":999973,"miner_id":"miner-6","network_difficulty":1000015,"ntime":1700002514,"order_key":"06:miner-6","p2mr_program_hex":"d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2","share_difficulty":101,"share_id":"share-2514","share_seq":2514,"template_height":800078},{"accepted_at_ms":999976,"credit_policy":null,"job_id":"job-252","job_issued_at_ms":999989,"miner_id":"miner-10","network_difficulty":1000014,"ntime":1700000252,"order_key":"10:miner-10","p2mr_program_hex":"fcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfc","share_difficulty":19,"share_id":"share-252","share_seq":252,"template_height":800007},{"accepted_at_ms":999980,"credit_policy":null,"job_id":"job-1815","job_issued_at_ms":999976,"miner_id":"miner-0","network_difficulty":1000013,"ntime":1700001815,"order_key":"00:miner-0","p2mr_program_hex":"1717171717171717171717171717171717171717171717171717171717171717","share_difficulty":19,"share_id":"share-1815","share_seq":1815,"template_height":800056},{"accepted_at_ms":999972,"credit_policy":null,"job_id":"job-2250","job_issued_at_ms":999975,"miner_id":"miner-6","network_difficulty":1000006,"ntime":1700002250,"order_key":"06:miner-6","p2mr_program_hex":"cacacacacacacacacacacacacacacacacacacacacacacacacacacacacacacaca","share_difficulty":101,"share_id":"share-2250","share_seq":2250,"template_height":800070},{"accepted_at_ms":999970,"credit_policy":null,"job_id":"job-892","job_issued_at_ms":999978,"miner_id":"miner-1","network_difficulty":1000008,"ntime":1700000892,"order_key":"01:miner-1","p2mr_program_hex":"7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c","share_difficulty":1,"share_id":"share-892","share_seq":892,"template_height":800027},{"accepted_at_ms":999998,"credit_policy":"stale-grace","job_id":"job-684","job_issued_at_ms":999986,"miner_id":"miner-2","network_difficulty":1000004,"ntime":1700000684,"order_key":"02:miner-2","p2mr_program_hex":"acacacacacacacacacacacacacacacacacacacacacacacacacacacacacacacac","share_difficulty":1000,"share_id":"share-684","share_seq":684,"template_height":800021},{"accepted_at_ms":999970,"credit_policy":null,"job_id":"job-496","job_issued_at_ms":999999,"miner_id":"miner-1","network_difficulty":1000003,"ntime":1700000496,"order_key":"01:miner-1","p2mr_program_hex":"f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0","share_difficulty":3,"share_id":"share-496","share_seq":496,"template_height":800015},{"accepted_at_ms":999994,"credit_policy":null,"job_id":"job-2980","job_issued_at_ms":999970,"miner_id":"miner-10","network_difficulty":1000005,"ntime":1700002980,"order_key":"10:miner-10","p2mr_program_hex":"a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4","share_difficulty":3,"share_id":"share-2980","share_seq":2980,"template_height":800093},{"accepted_at_ms":999975,"credit_policy":null,"job_id":"job-3899","job_issued_at_ms":999979,"miner_id":"miner-5","network_difficulty":1000006,"ntime":1700003899,"order_key":"05:miner-5","p2mr_program_hex":"3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b","share_difficulty":19,"share_id":"share-3899","share_seq":3899,"template_height":800121},{"accepted_at_ms":999973,"credit_policy":null,"job_id":"job-1143","job_issued_at_ms":999997,"miner_id":"miner-10","network_difficulty":1000004,"ntime":1700001143,"order_key":"10:miner-10","p2mr_program_hex":"7777777777777777777777777777777777777777777777777777777777777777","share_difficulty":101,"share_id":"share-1143","share_seq":1143,"template_height":800035},{"accepted_at_ms":999995,"credit_policy":null,"job_id":"job-1003","job_issued_at_ms":999993,"miner_id":"miner-2","network_difficulty":1000000,"ntime":1700001003,"order_key":"02:miner-2","p2mr_program_hex":"ebebebebebebebebebebebebebebebebebebebebebebebebebebebebebebebeb","share_difficulty":3,"share_id":"share-1003","share_seq":1003,"template_height":800031},{"accepted_at_ms":1000470,"credit_policy":null,"job_id":"job-3681","job_issued_at_ms":1000320,"miner_id":"miner-7","network_difficulty":1000009,"ntime":1700003681,"order_key":"07:miner-7","p2mr_program_hex":"6161616161616161616161616161616161616161616161616161616161616161","share_difficulty":1000,"share_id":"share-3681","share_seq":3681,"template_height":800115},{"accepted_at_ms":999974,"credit_policy":"stale-grace","job_id":"job-881","job_issued_at_ms":999970,"miner_id":"miner-1","network_difficulty":1000014,"ntime":1700000881,"order_key":"01:miner-1","p2mr_program_hex":"7171717171717171717171717171717171717171717171717171717171717171","share_difficulty":1000,"share_id":"share-881","share_seq":881,"template_height":800027},{"accepted_at_ms":999976,"credit_policy":null,"job_id":"job-3840","job_issued_at_ms":999989,"miner_id":"miner-1","network_difficulty":1000015,"ntime":1700003840,"order_key":"01:miner-1","p2mr_program_hex":"0000000000000000000000000000000000000000000000000000000000000000","share_difficulty":2,"share_id":"share-3840","share_seq":3840,"template_height":800120},{"accepted_at_ms":999984,"credit_policy":null,"job_id":"job-2557","job_issued_at_ms":999972,"miner_id":"miner-5","network_difficulty":1000007,"ntime":1700002557,"order_key":"05:miner-5","p2mr_program_hex":"fdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfd","share_difficulty":1,"share_id":"share-2557","share_seq":2557,"template_height":800079},{"accepted_at_ms":999994,"credit_policy":"stale-grace","job_id":"job-1466","job_issued_at_ms":1000136,"miner_id":"miner-3","network_difficulty":1000004,"ntime":1700001466,"order_key":"03:miner-3","p2mr_program_hex":"babababababababababababababababababababababababababababababababa","share_difficulty":2,"share_id":"share-1466","share_seq":1466,"template_height":800045},{"accepted_at_ms":999989,"credit_policy":null,"job_id":"job-1982","job_issued_at_ms":999985,"miner_id":"miner-2","network_difficulty":1000010,"ntime":1700001982,"order_key":"02:miner-2","p2mr_program_hex":"bebebebebebebebebebebebebebebebebebebebebebebebebebebebebebebebe","share_difficulty":19,"share_id":"share-1982","share_seq":1982,"template_height":800061},{"accepted_at_ms":999976,"credit_policy":null,"job_id":"job-3812","job_issued_at_ms":999994,"miner_id":"miner-6","network_difficulty":1000004,"ntime":1700003812,"order_key":"06:miner-6","p2mr_program_hex":"e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4","share_difficulty":1,"share_id":"share-3812","share_seq":3812,"template_height":800119},{"accepted_at_ms":999982,"credit_policy":null,"job_id":"job-1330","job_issued_at_ms":999976,"miner_id":"miner-10","network_difficulty":1000004,"ntime":1700001330,"order_key":"10:miner-10","p2mr_program_hex":"3232323232323232323232323232323232323232323232323232323232323232","share_difficulty":19,"share_id":"share-1330","share_seq":1330,"template_height":800041},{"accepted_at_ms":999994,"credit_policy":null,"job_id":"job-3151","job_issued_at_ms":999973,"miner_id":"miner-5","network_difficulty":1000006,"ntime":1700003151,"order_key":"05:miner-5","p2mr_program_hex":"4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f","share_difficulty":1000,"share_id":"share-3151","share_seq":3151,"template_height":800098},{"accepted_at_ms":999982,"credit_policy":null,"job_id":"job-189","job_issued_at_ms":999984,"miner_id":"miner-2","network_difficulty":1000002,"ntime":1700000189,"order_key":"02:miner-2","p2mr_program_hex":"bdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbd","share_difficulty":7,"share_id":"share-189","share_seq":189,"template_height":800005},{"accepted_at_ms":999983,"credit_policy":null,"job_id":"job-721","job_issued_at_ms":999986,"miner_id":"miner-6","network_difficulty":1000007,"ntime":1700000721,"order_key":"06:miner-6","p2mr_program_hex":"d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1","share_difficulty":7,"share_id":"share-721","share_seq":721,"template_height":800022},{"accepted_at_ms":999994,"credit_policy":null,"job_id":"job-3291","job_issued_at_ms":999999,"miner_id":"miner-2","network_difficulty":1000010,"ntime":1700003291,"order_key":"02:miner-2","p2mr_program_hex":"dbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdb","share_difficulty":2,"share_id":"share-3291","share_seq":3291,"template_height":800102},{"accepted_at_ms":999986,"credit_policy":"stale-grace","job_id":"job-2793","job_issued_at_ms":1000000,"miner_id":"miner-10","network_difficulty":1000005,"ntime":1700002793,"order_key":"10:miner-10","p2mr_program_hex":"e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9","share_difficulty":1,"share_id":"share-2793","share_seq":2793,"template_height":800087},{"accepted_at_ms":999990,"credit_policy":null,"job_id":"job-272","job_issued_at_ms":999988,"miner_id":"miner-8","network_difficulty":1000000,"ntime":1700000272,"order_key":"08:miner-8","p2mr_program_hex":"1010101010101010101010101010101010101010101010101010101010101010","share_difficulty":2,"share_id":"share-272","share_seq":272,"template_height":800008},{"accepted_at_ms":1000000,"credit_policy":null,"job_id":"job-414","job_issued_at_ms":999979,"miner_id":"miner-7","network_difficulty":1000006,"ntime":1700000414,"order_key":"07:miner-7","p2mr_program_hex":"9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e","share_difficulty":1000,"share_id":"share-414","share_seq":414,"template_height":800012},{"accepted_at_ms":999995,"credit_policy":null,"job_id":"job-3084","job_issued_at_ms":999977,"miner_id":"miner-4","network_difficulty":1000007,"ntime":1700003084,"order_key":"04:miner-4","p2mr_program_hex":"0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c","share_difficulty":19,"share_id":"share-3084","share_seq":3084,"template_height":800096},{"accepted_at_ms":999978,"credit_policy":null,"job_id":"job-3486","job_issued_at_ms":999974,"miner_id":"miner-10","network_difficulty":1000001,"ntime":1700003486,"order_key":"10:miner-10","p2mr_program_hex":"9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e","share_difficulty":7,"share_id":"share-3486","share_seq":3486,"template_height":800108},{"accepted_at_ms":999976,"credit_policy":null,"job_id":"job-213","job_issued_at_ms":1000480,"miner_id":"miner-4","network_difficulty":1000009,"ntime":1700000213,"order_key":"04:miner-4","p2mr_program_hex":"d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5","share_difficulty":2,"share_id":"share-213","share_seq":213,"template_height":800006},{"accepted_at_ms":1000000,"credit_policy":null,"job_id":"job-3636","job_issued_at_ms":999973,"miner_id":"miner-6","network_difficulty":1000015,"ntime":1700003636,"order_key":"06:miner-6","p2mr_program_hex":"3434343434343434343434343434343434343434343434343434343434343434","share_difficulty":7,"share_id":"share-3636","share_seq":3636,"template_height":800113},{"accepted_at_ms":999974,"credit_policy":null,"job_id":"job-742","job_issued_at_ms":999991,"miner_id":"miner-5","network_difficulty":1000011,"ntime":1700000742,"order_key":"05:miner-5","p2mr_program_hex":"e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6","share_difficulty":7,"share_id":"share-742","share_seq":742,"template_height":800023},{"accepted_at_ms":999974,"credit_policy":"stale-grace","job_id":"job-3275","job_issued_at_ms":999972,"miner_id":"miner-8","network_difficulty":1000011,"ntime":1700003275,"order_key":"08:miner-8","p2mr_program_hex":"cbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcb","share_difficulty":3,"share_id":"share-3275","share_seq":3275,"template_height":800102},{"accepted_at_ms":999989,"credit_policy":null,"job_id":"job-1874","job_issued_at_ms":999977,"miner_id":"miner-4","network_difficulty":1000004,"ntime":1700001874,"order_key":"04:miner-4","p2mr_program_hex":"5252525252525252525252525252525252525252525252525252525252525252","share_difficulty":3,"share_id":"share-1874","share_seq":1874,"template_height":800058},{"accepted_at_ms":999989,"credit_policy":null,"job_id":"job-876","job_issued_at_ms":999987,"miner_id":"miner-7","network_difficulty":1000009,"ntime":1700000876,"order_key":"07:miner-7","p2mr_program_hex":"6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c","share_difficulty":7,"share_id":"share-876","share_seq":876,"template_height":800027},{"accepted_at_ms":999990,"credit_policy":null,"job_id":"job-690","job_issued_at_ms":999984,"miner_id":"miner-8","network_difficulty":1000010,"ntime":1700000690,"order_key":"08:miner-8","p2mr_program_hex":"b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2","share_difficulty":101,"share_id":"share-690","share_seq":690,"template_height":800021},{"accepted_at_ms":999985,"credit_policy":null,"job_id":"job-3662","job_issued_at_ms":999989,"miner_id":"miner-10","network_difficulty":1000007,"ntime":1700003662,"order_key":"10:miner-10","p2mr_program_hex":"4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e","share_difficulty":1000,"share_id":"share-3662","share_seq":3662,"template_height":800114},{"accepted_at_ms":999995,"credit_policy":"stale-grace","job_id":"job-556","job_issued_at_ms":999997,"miner_id":"miner-6","network_difficulty":1000012,"ntime":1700000556,"order_key":"06:miner-6","p2mr_program_hex":"2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c","share_difficulty":3,"share_id":"share-556","share_seq":556,"template_height":800017},{"accepted_at_ms":999994,"credit_policy":null,"job_id":"job-3180","job_issued_at_ms":1000433,"miner_id":"miner-1","network_difficulty":1000001,"ntime":1700003180,"order_key":"01:miner-1","p2mr_program_hex":"6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c","share_difficulty":3,"share_id":"share-3180","share_seq":3180,"template_height":800099},{"accepted_at_ms":999984,"credit_policy":null,"job_id":"job-2373","job_issued_at_ms":999981,"miner_id":"miner-8","network_difficulty":1000010,"ntime":1700002373,"order_key":"08:miner-8","p2mr_program_hex":"4545454545454545454545454545454545454545454545454545454545454545","share_difficulty":19,"share_id":"share-2373","share_seq":2373,"template_height":800074},{"accepted_at_ms":999993,"credit_policy":null,"job_id":"job-330","job_issued_at_ms":999982,"miner_id":"miner-0","network_difficulty":1000007,"ntime":1700000330,"order_key":"00:miner-0","p2mr_program_hex":"4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a","share_difficulty":1000,"share_id":"share-330","share_seq":330,"template_height":800010},{"accepted_at_ms":999974,"credit_policy":null,"job_id":"job-2918","job_issued_at_ms":999996,"miner_id":"miner-3","network_difficulty":1000011,"ntime":1700002918,"order_key":"03:miner-3","p2mr_program_hex":"6666666666666666666666666666666666666666666666666666666666666666","share_difficulty":1,"share_id":"share-2918","share_seq":2918,"template_height":800091},{"accepted_at_ms":999977,"credit_policy":null,"job_id":"job-2562","job_issued_at_ms":999989,"miner_id":"miner-10","network_difficulty":1000012,"ntime":1700002562,"order_key":"10:miner-10","p2mr_program_hex":"0202020202020202020202020202020202020202020202020202020202020202","share_difficulty":3,"share_id":"share-2562","share_seq":2562,"template_height":800080},{"accepted_at_ms":999977,"credit_policy":"stale-grace","job_id":"job-1285","job_issued_at_ms":999986,"miner_id":"miner-9","network_difficulty":1000010,"ntime":1700001285,"order_key":"09:miner-9","p2mr_program_hex":"0505050505050505050505050505050505050505050505050505050505050505","share_difficulty":7,"share_id":"share-1285","share_seq":1285,"template_height":800040},{"accepted_at_ms":999978,"credit_policy":null,"job_id":"job-166","job_issued_at_ms":999988,"miner_id":"miner-1","network_difficulty":1000013,"ntime":1700000166,"order_key":"01:miner-1","p2mr_program_hex":"a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6","share_difficulty":3,"share_id":"share-166","share_seq":166,"template_height":800005},{"accepted_at_ms":999976,"credit_policy":null,"job_id":"job-2090","job_issued_at_ms":1000000,"miner_id":"miner-0","network_difficulty":1000016,"ntime":1700002090,"order_key":"00:miner-0","p2mr_program_hex":"2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a","share_difficulty":1,"share_id":"share-2090","share_seq":2090,"template_height":800065},{"accepted_at_ms":999983,"credit_policy":"stale-grace","job_id":"job-3546","job_issued_at_ms":1000142,"miner_id":"miner-4","network_difficulty":1000010,"ntime":1700003546,"order_key":"04:miner-4","p2mr_program_hex":"dadadadadadadadadadadadadadadadadadadadadadadadadadadadadadadada","share_difficulty":19,"share_id":"share-3546","share_seq":3546,"template_height":800110},{"accepted_at_ms":999981,"credit_policy":"stale-grace","job_id":"job-331","job_issued_at_ms":999988,"miner_id":"miner-1","network_difficulty":1000008,"ntime":1700000331,"order_key":"01:miner-1","p2mr_program_hex":"4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b","share_difficulty":3,"share_id":"share-331","share_seq":331,"template_height":800010},{"accepted_at_ms":999970,"credit_policy":null,"job_id":"job-3125","job_issued_at_ms":999972,"miner_id":"miner-1","network_difficulty":1000014,"ntime":1700003125,"order_key":"01:miner-1","p2mr_program_hex":"3535353535353535353535353535353535353535353535353535353535353535","share_difficulty":7,"share_id":"share-3125","share_seq":3125,"template_height":800097},{"accepted_at_ms":1000295,"credit_policy":"stale-grace","job_id":"job-740","job_issued_at_ms":999984,"miner_id":"miner-3","network_difficulty":1000009,"ntime":1700000740,"order_key":"03:miner-3","p2mr_program_hex":"e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4","share_difficulty":7,"share_id":"share-740","share_seq":740,"template_height":800023},{"accepted_at_ms":999970,"credit_policy":null,"job_id":"job-3739","job_issued_at_ms":999999,"miner_id":"miner-10","network_difficulty":1000016,"ntime":1700003739,"order_key":"10:miner-10","p2mr_program_hex":"9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b","share_difficulty":2,"share_id":"share-3739","share_seq":3739,"template_height":800116},{"accepted_at_ms":999980,"credit_policy":null,"job_id":"job-2468","job_issued_at_ms":999979,"miner_id":"miner-4","network_difficulty":1000003,"ntime":1700002468,"order_key":"04:miner-4","p2mr_program_hex":"a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4","share_difficulty":1000,"share_id":"share-2468","share_seq":2468,"template_height":800077},{"accepted_at_ms":999972,"credit_policy":null,"job_id":"job-3643","job_issued_at_ms":999997,"miner_id":"miner-2","network_difficulty":1000005,"ntime":1700003643,"order_key":"02:miner-2","p2mr_program_hex":"3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b","share_difficulty":1000,"share_id":"share-3643","share_seq":3643,"template_height":800113},{"accepted_at_ms":999991,"credit_policy":null,"job_id":"job-2347","job_issued_at_ms":999978,"miner_id":"miner-4","network_difficulty":1000001,"ntime":1700002347,"order_key":"04:miner-4","p2mr_program_hex":"2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b","share_difficulty":101,"share_id":"share-2347","share_seq":2347,"template_height":800073},{"accepted_at_ms":999990,"credit_policy":null,"job_id":"job-1203","job_issued_at_ms":999982,"miner_id":"miner-4","network_difficulty":1000013,"ntime":1700001203,"order_key":"04:miner-4","p2mr_program_hex":"b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3","share_difficulty":1000,"share_id":"share-1203","share_seq":1203,"template_height":800037},{"accepted_at_ms":999975,"credit_policy":null,"job_id":"job-3077","job_issued_at_ms":1000000,"miner_id":"miner-8","network_difficulty":1000000,"ntime":1700003077,"order_key":"08:miner-8","p2mr_program_hex":"0505050505050505050505050505050505050505050505050505050505050505","share_difficulty":1,"share_id":"share-3077","share_seq":3077,"template_height":800096},{"accepted_at_ms":999985,"credit_policy":"stale-grace","job_id":"job-2335","job_issued_at_ms":999974,"miner_id":"miner-3","network_difficulty":1000006,"ntime":1700002335,"order_key":"03:miner-3","p2mr_program_hex":"1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f","share_difficulty":1000,"share_id":"share-2335","share_seq":2335,"template_height":800072},{"accepted_at_ms":1000000,"credit_policy":null,"job_id":"job-2584","job_issued_at_ms":999995,"miner_id":"miner-10","network_difficulty":1000000,"ntime":1700002584,"order_key":"10:miner-10","p2mr_program_hex":"1818181818181818181818181818181818181818181818181818181818181818","share_difficulty":1,"share_id":"share-2584","share_seq":2584,"template_height":800080},{"accepted_at_ms":999972,"credit_policy":null,"job_id":"job-1487","job_issued_at_ms":1000000,"miner_id":"miner-2","network_difficulty":1000008,"ntime":1700001487,"order_key":"02:miner-2","p2mr_program_hex":"cfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcf","share_difficulty":7,"share_id":"share-1487","share_seq":1487,"template_height":800046},{"accepted_at_ms":999993,"credit_policy":null,"job_id":"job-3167","job_issued_at_ms":999992,"miner_id":"miner-10","network_difficulty":1000005,"ntime":1700003167,"order_key":"10:miner-10","p2mr_program_hex":"5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f","share_difficulty":101,"share_id":"share-3167","share_seq":3167,"template_height":800098},{"accepted_at_ms":999972,"credit_policy":"stale-grace","job_id":"job-1591","job_issued_at_ms":999986,"miner_id":"miner-7","network_difficulty":1000010,"ntime":1700001591,"order_key":"07:miner-7","p2mr_program_hex":"3737373737373737373737373737373737373737373737373737373737373737","share_difficulty":7,"share_id":"share-1591","share_seq":1591,"template_height":800049},{"accepted_at_ms":999989,"credit_policy":null,"job_id":"job-1348","job_issued_at_ms":999992,"miner_id":"miner-6","network_difficulty":1000005,"ntime":1700001348,"order_key":"06:miner-6","p2mr_program_hex":"4444444444444444444444444444444444444444444444444444444444444444","share_difficulty":3,"share_id":"share-1348","share_seq":1348,"template_height":800042},{"accepted_at_ms":999984,"credit_policy":"stale-grace","job_id":"job-3066","job_issued_at_ms":999994,"miner_id":"miner-8","network_difficulty":1000006,"ntime":1700003066,"order_key":"08:miner-8","p2mr_program_hex":"fafafafafafafafafafafafafafafafafafafafafafafafafafafafafafafafa","share_difficulty":101,"share_id":"share-3066","share_seq":3066,"template_height":800095},{"accepted_at_ms":999982,"credit_policy":null,"job_id":"job-3011","job_issued_at_ms":999980,"miner_id":"miner-8","network_difficulty":1000002,"ntime":1700003011,"order_key":"08:miner-8","p2mr_program_hex":"c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3","share_difficulty":101,"share_id":"share-3011","share_seq":3011,"template_height":800094},{"accepted_at_ms":999971,"credit_policy":null,"job_id":"job-2094","job_issued_at_ms":999983,"miner_id":"miner-4","network_difficulty":1000003,"ntime":1700002094,"order_key":"04:miner-4","p2mr_program_hex":"2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e","share_difficulty":1,"share_id":"share-2094","share_seq":2094,"template_height":800065},{"accepted_at_ms":999972,"credit_policy":null,"job_id":"job-3129","job_issued_at_ms":999971,"miner_id":"miner-5","network_difficulty":1000001,"ntime":1700003129,"order_key":"05:miner-5","p2mr_program_hex":"3939393939393939393939393939393939393939393939393939393939393939","share_difficulty":101,"share_id":"share-3129","share_seq":3129,"template_height":800097},{"accepted_at_ms":999981,"credit_policy":"stale-grace","job_id":"job-3439","job_issued_at_ms":999972,"miner_id":"miner-7","network_difficulty":1000005,"ntime":1700003439,"order_key":"07:miner-7","p2mr_program_hex":"6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f","share_difficulty":1000,"share_id":"share-3439","share_seq":3439,"template_height":800107},{"accepted_at_ms":999981,"credit_policy":"stale-grace","job_id":"job-3659","job_issued_at_ms":999971,"miner_id":"miner-7","network_difficulty":1000004,"ntime":1700003659,"order_key":"07:miner-7","p2mr_program_hex":"4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b","share_difficulty":3,"share_id":"share-3659","share_seq":3659,"template_height":800114},{"accepted_at_ms":999978,"credit_policy":null,"job_id":"job-2482","job_issued_at_ms":999974,"miner_id":"miner-7","network_difficulty":1000000,"ntime":1700002482,"order_key":"07:miner-7","p2mr_program_hex":"b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2","share_difficulty":101,"share_id":"share-2482","share_seq":2482,"template_height":800077},{"accepted_at_ms":999998,"credit_policy":null,"job_id":"job-3801","job_issued_at_ms":999994,"miner_id":"miner-6","network_difficulty":1000010,"ntime":1700003801,"order_key":"06:miner-6","p2mr_program_hex":"d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9","share_difficulty":19,"share_id":"share-3801","share_seq":3801,"template_height":800118},{"accepted_at_ms":999984,"credit_policy":null,"job_id":"job-753","job_issued_at_ms":999971,"miner_id":"miner-5","network_difficulty":1000005,"ntime":1700000753,"order_key":"05:miner-5","p2mr_program_hex":"f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1","share_difficulty":101,"share_id":"share-753","share_seq":753,"template_height":800023},{"accepted_at_ms":999992,"credit_policy":null,"job_id":"job-404","job_issued_at_ms":999988,"miner_id":"miner-8","network_difficulty":1000013,"ntime":1700000404,"order_key":"08:miner-8","p2mr_program_hex":"9494949494949494949494949494949494949494949494949494949494949494","share_difficulty":19,"share_id":"share-404","share_seq":404,"template_height":800012},{"accepted_at_ms":999983,"credit_policy":null,"job_id":"job-1015","job_issued_at_ms":999981,"miner_id":"miner-3","network_difficulty":1000012,"ntime":1700001015,"order_key":"03:miner-3","p2mr_program_hex":"f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7","share_difficulty":101,"share_id":"share-1015","share_seq":1015,"template_height":800031},{"accepted_at_ms":999989,"credit_policy":"stale-grace","job_id":"job-3239","job_issued_at_ms":999996,"miner_id":"miner-5","network_difficulty":1000009,"ntime":1700003239,"order_key":"05:miner-5","p2mr_program_hex":"a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7","share_difficulty":2,"share_id":"share-3239","share_seq":3239,"template_height":800101},{"accepted_at_ms":999978,"credit_policy":null,"job_id":"job-661","job_issued_at_ms":1000000,"miner_id":"miner-1","network_difficulty":1000015,"ntime":1700000661,"order_key":"01:miner-1","p2mr_program_hex":"9595959595959595959595959595959595959595959595959595959595959595","share_difficulty":101,"share_id":"share-661","share_seq":661,"template_height":800020},{"accepted_at_ms":999975,"credit_policy":null,"job_id":"job-2707","job_issued_at_ms":999996,"miner_id":"miner-1","network_difficulty":1000004,"ntime":1700002707,"order_key":"01:miner-1","p2mr_program_hex":"9393939393939393939393939393939393939393939393939393939393939393","share_difficulty":7,"share_id":"share-2707","share_seq":2707,"template_height":800084},{"accepted_at_ms":999977,"credit_policy":null,"job_id":"job-3716","job_issued_at_ms":999977,"miner_id":"miner-9","network_difficulty":1000010,"ntime":1700003716,"order_key":"09:miner-9","p2mr_program_hex":"8484848484848484848484848484848484848484848484848484848484848484","share_difficulty":7,"share_id":"share-3716","share_seq":3716,"template_height":800116},{"accepted_at_ms":999993,"credit_policy":null,"job_id":"job-2621","job_issued_at_ms":999993,"miner_id":"miner-3","network_difficulty":1000003,"ntime":1700002621,"order_key":"03:miner-3","p2mr_program_hex":"3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d","share_difficulty":1000,"share_id":"share-2621","share_seq":2621,"template_height":800081},{"accepted_at_ms":999973,"credit_policy":"stale-grace","job_id":"job-2300","job_issued_at_ms":999993,"miner_id":"miner-1","network_difficulty":1000005,"ntime":1700002300,"order_key":"01:miner-1","p2mr_program_hex":"fcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfc","share_difficulty":1000,"share_id":"share-2300","share_seq":2300,"template_height":800071},{"accepted_at_ms":999994,"credit_policy":null,"job_id":"job-1862","job_issued_at_ms":999995,"miner_id":"miner-3","network_difficulty":1000009,"ntime":1700001862,"order_key":"03:miner-3","p2mr_program_hex":"4646464646464646464646464646464646464646464646464646464646464646","share_difficulty":1000,"share_id":"share-1862","share_seq":1862,"template_height":800058},{"accepted_at_ms":999995,"credit_policy":null,"job_id":"job-3498","job_issued_at_ms":999989,"miner_id":"miner-0","network_difficulty":1000013,"ntime":1700003498,"order_key":"00:miner-0","p2mr_program_hex":"aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa","share_difficulty":3,"share_id":"share-3498","share_seq":3498,"template_height":800109},{"accepted_at_ms":999997,"credit_policy":"stale-grace","job_id":"job-3717","job_issued_at_ms":999996,"miner_id":"miner-10","network_difficulty":1000011,"ntime":1700003717,"order_key":"10:miner-10","p2mr_program_hex":"8585858585858585858585858585858585858585858585858585858585858585","share_difficulty":19,"share_id":"share-3717","share_seq":3717,"template_height":800116},{"accepted_at_ms":999992,"credit_policy":null,"job_id":"job-711","job_issued_at_ms":999983,"miner_id":"miner-7","network_difficulty":1000014,"ntime":1700000711,"order_key":"07:miner-7","p2mr_program_hex":"c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7","share_difficulty":1,"share_id":"share-711","share_seq":711,"template_height":800022},{"accepted_at_ms":999975,"credit_policy":null,"job_id":"job-2522","job_issued_at_ms":999990,"miner_id":"miner-3","network_difficulty":1000006,"ntime":1700002522,"order_key":"03:miner-3","p2mr_program_hex":"dadadadadadadadadadadadadadadadadadadadadadadadadadadadadadadada","share_difficulty":1,"share_id":"share-2522","share_seq":2522,"template_height":800078},{"accepted_at_ms":1000198,"credit_policy":null,"job_id":"job-3444","job_issued_at_ms":1000461,"miner_id":"miner-1","network_difficulty":1000010,"ntime":1700003444,"order_key":"01:miner-1","p2mr_program_hex":"7474747474747474747474747474747474747474747474747474747474747474","share_difficulty":2,"share_id":"share-3444","share_seq":3444,"template_height":800107},{"accepted_at_ms":999979,"credit_policy":null,"job_id":"job-1999","job_issued_at_ms":999970,"miner_id":"miner-8","network_difficulty":1000010,"ntime":1700001999,"order_key":"08:miner-8","p2mr_program_hex":"cfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcf","share_difficulty":1,"share_id":"share-1999","share_seq":1999,"template_height":800062},{"accepted_at_ms":999989,"credit_policy":null,"job_id":"job-735","job_issued_at_ms":999999,"miner_id":"miner-9","network_difficulty":1000004,"ntime":1700000735,"order_key":"09:miner-9","p2mr_program_hex":"dfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdf","share_difficulty":101,"share_id":"share-735","share_seq":735,"template_height":800022},{"accepted_at_ms":999975,"credit_policy":null,"job_id":"job-145","job_issued_at_ms":999973,"miner_id":"miner-2","network_difficulty":1000009,"ntime":1700000145,"order_key":"02:miner-2","p2mr_program_hex":"9191919191919191919191919191919191919191919191919191919191919191","share_difficulty":101,"share_id":"share-145","share_seq":145,"template_height":800004},{"accepted_at_ms":999991,"credit_policy":null,"job_id":"job-3383","job_issued_at_ms":999987,"miner_id":"miner-6","network_difficulty":1000000,"ntime":1700003383,"order_key":"06:miner-6","p2mr_program_hex":"3737373737373737373737373737373737373737373737373737373737373737","share_difficulty":1000,"share_id":"share-3383","share_seq":3383,"template_height":800105},{"accepted_at_ms":999981,"credit_policy":null,"job_id":"job-2073","job_issued_at_ms":999983,"miner_id":"miner-5","network_difficulty":1000016,"ntime":1700002073,"order_key":"05:miner-5","p2mr_program_hex":"1919191919191919191919191919191919191919191919191919191919191919","share_difficulty":2,"share_id":"share-2073","share_seq":2073,"template_height":800064},{"accepted_at_ms":999996,"credit_policy":"stale-grace","job_id":"job-73","job_issued_at_ms":999978,"miner_id":"miner-7","network_difficulty":1000005,"ntime":1700000073,"order_key":"07:miner-7","p2mr_program_hex":"4949494949494949494949494949494949494949494949494949494949494949","share_difficulty":7,"share_id":"share-73","share_seq":73,"template_height":800002},{"accepted_at_ms":1000000,"credit_policy":null,"job_id":"job-2354","job_issued_at_ms":999984,"miner_id":"miner-0","network_difficulty":1000008,"ntime":1700002354,"order_key":"00:miner-0","p2mr_program_hex":"3232323232323232323232323232323232323232323232323232323232323232","share_difficulty":2,"share_id":"share-2354","share_seq":2354,"template_height":800073},{"accepted_at_ms":999984,"credit_policy":null,"job_id":"job-2814","job_issued_at_ms":999992,"miner_id":"miner-9","network_difficulty":1000009,"ntime":1700002814,"order_key":"09:miner-9","p2mr_program_hex":"fefefefefefefefefefefefefefefefefefefefefefefefefefefefefefefefe","share_difficulty":3,"share_id":"share-2814","share_seq":2814,"template_height":800087},{"accepted_at_ms":999978,"credit_policy":null,"job_id":"job-3776","job_issued_at_ms":999994,"miner_id":"miner-3","network_difficulty":1000002,"ntime":1700003776,"order_key":"03:miner-3","p2mr_program_hex":"c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0","share_difficulty":19,"share_id":"share-3776","share_seq":3776,"template_height":800118},{"accepted_at_ms":1000468,"credit_policy":null,"job_id":"job-3488","job_issued_at_ms":999976,"miner_id":"miner-1","network_difficulty":1000003,"ntime":1700003488,"order_key":"01:miner-1","p2mr_program_hex":"a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0","share_difficulty":3,"share_id":"share-3488","share_seq":3488,"template_height":800109},{"accepted_at_ms":999999,"credit_policy":null,"job_id":"job-2953","job_issued_at_ms":999974,"miner_id":"miner-5","network_difficulty":1000012,"ntime":1700002953,"order_key":"05:miner-5","p2mr_program_hex":"8989898989898989898989898989898989898989898989898989898989898989","share_difficulty":19,"share_id":"share-2953","share_seq":2953,"template_height":800092},{"accepted_at_ms":999983,"credit_policy":null,"job_id":"job-2119","job_issued_at_ms":999976,"miner_id":"miner-7","network_difficulty":1000011,"ntime":1700002119,"order_key":"07:miner-7","p2mr_program_hex":"4747474747474747474747474747474747474747474747474747474747474747","share_difficulty":1,"share_id":"share-2119","share_seq":2119,"template_height":800066},{"accepted_at_ms":1000140,"credit_policy":null,"job_id":"job-2151","job_issued_at_ms":1000283,"miner_id":"miner-6","network_difficulty":1000009,"ntime":1700002151,"order_key":"06:miner-6","p2mr_program_hex":"6767676767676767676767676767676767676767676767676767676767676767","share_difficulty":1,"share_id":"share-2151","share_seq":2151,"template_height":800067},{"accepted_at_ms":1000000,"credit_policy":"stale-grace","job_id":"job-3023","job_issued_at_ms":999973,"miner_id":"miner-9","network_difficulty":1000014,"ntime":1700003023,"order_key":"09:miner-9","p2mr_program_hex":"cfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcf","share_difficulty":101,"share_id":"share-3023","share_seq":3023,"template_height":800094},{"accepted_at_ms":999980,"credit_policy":null,"job_id":"job-2217","job_issued_at_ms":999971,"miner_id":"miner-6","network_difficulty":1000007,"ntime":1700002217,"order_key":"06:miner-6","p2mr_program_hex":"a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9","share_difficulty":7,"share_id":"share-2217","share_seq":2217,"template_height":800069},{"accepted_at_ms":999987,"credit_policy":null,"job_id":"job-2604","job_issued_at_ms":999979,"miner_id":"miner-8","network_difficulty":1000003,"ntime":1700002604,"order_key":"08:miner-8","p2mr_program_hex":"2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c","share_difficulty":2,"share_id":"share-2604","share_seq":2604,"template_height":800081},{"accepted_at_ms":999973,"credit_policy":null,"job_id":"job-387","job_issued_at_ms":1000000,"miner_id":"miner-2","network_difficulty":1000013,"ntime":1700000387,"order_key":"02:miner-2","p2mr_program_hex":"8383838383838383838383838383838383838383838383838383838383838383","share_difficulty":101,"share_id":"share-387","share_seq":387,"template_height":800012},{"accepted_at_ms":999992,"credit_policy":"stale-grace","job_id":"job-136","job_issued_at_ms":999997,"miner_id":"miner-4","network_difficulty":1000000,"ntime":1700000136,"order_key":"04:miner-4","p2mr_program_hex":"8888888888888888888888888888888888888888888888888888888888888888","share_difficulty":101,"share_id":"share-136","share_seq":136,"template_height":800004},{"accepted_at_ms":999994,"credit_policy":null,"job_id":"job-3219","job_issued_at_ms":999979,"miner_id":"miner-7","network_difficulty":1000006,"ntime":1700003219,"order_key":"07:miner-7","p2mr_program_hex":"9393939393939393939393939393939393939393939393939393939393939393","share_difficulty":19,"share_id":"share-3219","share_seq":3219,"template_height":800100},{"accepted_at_ms":999984,"credit_policy":null,"job_id":"job-2594","job_issued_at_ms":999970,"miner_id":"miner-9","network_difficulty":1000010,"ntime":1700002594,"order_key":"09:miner-9","p2mr_program_hex":"2222222222222222222222222222222222222222222222222222222222222222","share_difficulty":1,"share_id":"share-2594","share_seq":2594,"template_height":800081},{"accepted_at_ms":1000277,"credit_policy":null,"job_id":"job-2810","job_issued_at_ms":1000480,"miner_id":"miner-5","network_difficulty":1000005,"ntime":1700002810,"order_key":"05:miner-5","p2mr_program_hex":"fafafafafafafafafafafafafafafafafafafafafafafafafafafafafafafafa","share_difficulty":101,"share_id":"share-2810","share_seq":2810,"template_height":800087},{"accepted_at_ms":999978,"credit_policy":null,"job_id":"job-805","job_issued_at_ms":999974,"miner_id":"miner-2","network_difficulty":1000006,"ntime":1700000805,"order_key":"02:miner-2","p2mr_program_hex":"2525252525252525252525252525252525252525252525252525252525252525","share_difficulty":3,"share_id":"share-805","share_seq":805,"template_height":800025},{"accepted_at_ms":999993,"credit_policy":null,"job_id":"job-1655","job_issued_at_ms":999984,"miner_id":"miner-5","network_difficulty":1000006,"ntime":1700001655,"order_key":"05:miner-5","p2mr_program_hex":"7777777777777777777777777777777777777777777777777777777777777777","share_difficulty":7,"share_id":"share-1655","share_seq":1655,"template_height":800051},{"accepted_at_ms":999998,"credit_policy":null,"job_id":"job-3449","job_issued_at_ms":999986,"miner_id":"miner-6","network_difficulty":1000015,"ntime":1700003449,"order_key":"06:miner-6","p2mr_program_hex":"7979797979797979797979797979797979797979797979797979797979797979","share_difficulty":2,"share_id":"share-3449","share_seq":3449,"template_height":800107},{"accepted_at_ms":999983,"credit_policy":null,"job_id":"job-1121","job_issued_at_ms":999996,"miner_id":"miner-10","network_difficulty":1000016,"ntime":1700001121,"order_key":"10:miner-10","p2mr_program_hex":"6161616161616161616161616161616161616161616161616161616161616161","share_difficulty":101,"share_id":"share-1121","share_seq":1121,"template_height":800035},{"accepted_at_ms":999984,"credit_policy":null,"job_id":"job-1234","job_issued_at_ms":999991,"miner_id":"miner-2","network_difficulty":1000010,"ntime":1700001234,"order_key":"02:miner-2","p2mr_program_hex":"d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2","share_difficulty":2,"share_id":"share-1234","share_seq":1234,"template_height":800038},{"accepted_at_ms":999986,"credit_policy":null,"job_id":"job-1284","job_issued_at_ms":999979,"miner_id":"miner-8","network_difficulty":1000009,"ntime":1700001284,"order_key":"08:miner-8","p2mr_program_hex":"0404040404040404040404040404040404040404040404040404040404040404","share_difficulty":101,"share_id":"share-1284","share_seq":1284,"template_height":800040},{"accepted_at_ms":999972,"credit_policy":null,"job_id":"job-2786","job_issued_at_ms":1000000,"miner_id":"miner-3","network_difficulty":1000015,"ntime":1700002786,"order_key":"03:miner-3","p2mr_program_hex":"e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2","share_difficulty":19,"share_id":"share-2786","share_seq":2786,"template_height":800087},{"accepted_at_ms":999974,"credit_policy":null,"job_id":"job-1864","job_issued_at_ms":999979,"miner_id":"miner-5","network_difficulty":1000011,"ntime":1700001864,"order_key":"05:miner-5","p2mr_program_hex":"4848484848484848484848484848484848484848484848484848484848484848","share_difficulty":3,"share_id":"share-1864","share_seq":1864,"template_height":800058},{"accepted_at_ms":1000119,"credit_policy":null,"job_id":"job-3250","job_issued_at_ms":1000056,"miner_id":"miner-5","network_difficulty":1000003,"ntime":1700003250,"order_key":"05:miner-5","p2mr_program_hex":"b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2","share_difficulty":7,"share_id":"share-3250","share_seq":3250,"template_height":800101},{"accepted_at_ms":999975,"credit_policy":null,"job_id":"job-2256","job_issued_at_ms":999999,"miner_id":"miner-1","network_difficulty":1000012,"ntime":1700002256,"order_key":"01:miner-1","p2mr_program_hex":"d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0","share_difficulty":1000,"share_id":"share-2256","share_seq":2256,"template_height":800070},{"accepted_at_ms":999979,"credit_policy":null,"job_id":"job-3209","job_issued_at_ms":1000461,"miner_id":"miner-8","network_difficulty":1000013,"ntime":1700003209,"order_key":"08:miner-8","p2mr_program_hex":"8989898989898989898989898989898989898989898989898989898989898989","share_difficulty":7,"share_id":"share-3209","share_seq":3209,"template_height":800100},{"accepted_at_ms":1000011,"credit_policy":null,"job_id":"job-3956","job_issued_at_ms":1000107,"miner_id":"miner-7","network_difficulty":1000012,"ntime":1700003956,"order_key":"07:miner-7","p2mr_program_hex":"7474747474747474747474747474747474747474747474747474747474747474","share_difficulty":101,"share_id":"share-3956","share_seq":3956,"template_height":800123},{"accepted_at_ms":999995,"credit_policy":null,"job_id":"job-2216","job_issued_at_ms":999978,"miner_id":"miner-5","network_difficulty":1000006,"ntime":1700002216,"order_key":"05:miner-5","p2mr_program_hex":"a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8","share_difficulty":2,"share_id":"share-2216","share_seq":2216,"template_height":800069},{"accepted_at_ms":999987,"credit_policy":null,"job_id":"job-1244","job_issued_at_ms":999985,"miner_id":"miner-1","network_difficulty":1000003,"ntime":1700001244,"order_key":"01:miner-1","p2mr_program_hex":"dcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdc","share_difficulty":1000,"share_id":"share-1244","share_seq":1244,"template_height":800038},{"accepted_at_ms":999971,"credit_policy":null,"job_id":"job-2535","job_issued_at_ms":999995,"miner_id":"miner-5","network_difficulty":1000002,"ntime":1700002535,"order_key":"05:miner-5","p2mr_program_hex":"e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7","share_difficulty":1,"share_id":"share-2535","share_seq":2535,"template_height":800079},{"accepted_at_ms":999972,"credit_policy":null,"job_id":"job-323","job_issued_at_ms":999976,"miner_id":"miner-4","network_difficulty":1000000,"ntime":1700000323,"order_key":"04:miner-4","p2mr_program_hex":"4343434343434343434343434343434343434343434343434343434343434343","share_difficulty":1000,"share_id":"share-323","share_seq":323,"template_height":800010},{"accepted_at_ms":999978,"credit_policy":null,"job_id":"job-3698","job_issued_at_ms":999975,"miner_id":"miner-2","network_difficulty":1000009,"ntime":1700003698,"order_key":"02:miner-2","p2mr_program_hex":"7272727272727272727272727272727272727272727272727272727272727272","share_difficulty":1000,"share_id":"share-3698","share_seq":3698,"template_height":800115},{"accepted_at_ms":999992,"credit_policy":null,"job_id":"job-1941","job_issued_at_ms":999990,"miner_id":"miner-5","network_difficulty":1000003,"ntime":1700001941,"order_key":"05:miner-5","p2mr_program_hex":"9595959595959595959595959595959595959595959595959595959595959595","share_difficulty":7,"share_id":"share-1941","share_seq":1941,"template_height":800060},{"accepted_at_ms":999974,"credit_policy":null,"job_id":"job-1549","job_issued_at_ms":999974,"miner_id":"miner-9","network_difficulty":1000002,"ntime":1700001549,"order_key":"09:miner-9","p2mr_program_hex":"0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d","share_difficulty":7,"share_id":"share-1549","share_seq":1549,"template_height":800048},{"accepted_at_ms":999992,"credit_policy":null,"job_id":"job-1110","job_issued_at_ms":999998,"miner_id":"miner-10","network_difficulty":1000005,"ntime":1700001110,"order_key":"10:miner-10","p2mr_program_hex":"5656565656565656565656565656565656565656565656565656565656565656","share_difficulty":3,"share_id":"share-1110","share_seq":1110,"template_height":800034},{"accepted_at_ms":999971,"credit_policy":"stale-grace","job_id":"job-1926","job_issued_at_ms":999974,"miner_id":"miner-1","network_difficulty":1000005,"ntime":1700001926,"order_key":"01:miner-1","p2mr_program_hex":"8686868686868686868686868686868686868686868686868686868686868686","share_difficulty":7,"share_id":"share-1926","share_seq":1926,"template_height":800060},{"accepted_at_ms":1000087,"credit_policy":"stale-grace","job_id":"job-3353","job_issued_at_ms":1000297,"miner_id":"miner-9","network_difficulty":1000004,"ntime":1700003353,"order_key":"09:miner-9","p2mr_program_hex":"1919191919191919191919191919191919191919191919191919191919191919","share_difficulty":3,"share_id":"share-3353","share_seq":3353,"template_height":800104},{"accepted_at_ms":999990,"credit_policy":null,"job_id":"job-776","job_issued_at_ms":999984,"miner_id":"miner-6","network_difficulty":1000011,"ntime":1700000776,"order_key":"06:miner-6","p2mr_program_hex":"0808080808080808080808080808080808080808080808080808080808080808","share_difficulty":7,"share_id":"share-776","share_seq":776,"template_height":800024},{"accepted_at_ms":999972,"credit_policy":"stale-grace","job_id":"job-3143","job_issued_at_ms":1000277,"miner_id":"miner-8","network_difficulty":1000015,"ntime":1700003143,"order_key":"08:miner-8","p2mr_program_hex":"4747474747474747474747474747474747474747474747474747474747474747","share_difficulty":1000,"share_id":"share-3143","share_seq":3143,"template_height":800098},{"accepted_at_ms":999980,"credit_policy":null,"job_id":"job-3848","job_issued_at_ms":999983,"miner_id":"miner-9","network_difficulty":1000006,"ntime":1700003848,"order_key":"09:miner-9","p2mr_program_hex":"0808080808080808080808080808080808080808080808080808080808080808","share_difficulty":101,"share_id":"share-3848","share_seq":3848,"template_height":800120},{"accepted_at_ms":999980,"credit_policy":null,"job_id":"job-3854","job_issued_at_ms":999982,"miner_id":"miner-4","network_difficulty":1000012,"ntime":1700003854,"order_key":"04:miner-4","p2mr_program_hex":"0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e","share_difficulty":19,"share_id":"share-3854","share_seq":3854,"template_height":800120},{"accepted_at_ms":1000169,"credit_policy":"stale-grace","job_id":"job-1312","job_issued_at_ms":999986,"miner_id":"miner-3","network_difficulty":1000003,"ntime":1700001312,"order_key":"03:miner-3","p2mr_program_hex":"2020202020202020202020202020202020202020202020202020202020202020","share_difficulty":101,"share_id":"share-1312","share_seq":1312,"template_height":800041},{"accepted_at_ms":999983,"credit_policy":null,"job_id":"job-370","job_issued_at_ms":999985,"miner_id":"miner-7","network_difficulty":1000013,"ntime":1700000370,"order_key":"07:miner-7","p2mr_program_hex":"7272727272727272727272727272727272727272727272727272727272727272","share_difficulty":7,"share_id":"share-370","share_seq":370,"template_height":800011},{"accepted_at_ms":1000227,"credit_policy":"stale-grace","job_id":"job-1262","job_issued_at_ms":1000462,"miner_id":"miner-8","network_difficulty":1000004,"ntime":1700001262,"order_key":"08:miner-8","p2mr_program_hex":"eeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeee","share_difficulty":19,"share_id":"share-1262","share_seq":1262,"template_height":800039},{"accepted_at_ms":999985,"credit_policy":"stale-grace","job_id":"job-525","job_issued_at_ms":999985,"miner_id":"miner-8","network_difficulty":1000015,"ntime":1700000525,"order_key":"08:miner-8","p2mr_program_hex":"0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d","share_difficulty":3,"share_id":"share-525","share_seq":525,"template_height":800016},{"accepted_at_ms":999993,"credit_policy":null,"job_id":"job-1083","job_issued_at_ms":1000119,"miner_id":"miner-5","network_difficulty":1000012,"ntime":1700001083,"order_key":"05:miner-5","p2mr_program_hex":"3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b","share_difficulty":19,"share_id":"share-1083","share_seq":1083,"template_height":800033},{"accepted_at_ms":999995,"credit_policy":null,"job_id":"job-829","job_issued_at_ms":999979,"miner_id":"miner-4","network_difficulty":1000013,"ntime":1700000829,"order_key":"04:miner-4","p2mr_program_hex":"3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d","share_difficulty":1,"share_id":"share-829","share_seq":829,"template_height":800025},{"accepted_at_ms":999972,"credit_policy":null,"job_id":"job-1645","job_issued_at_ms":999996,"miner_id":"miner-6","network_difficulty":1000013,"ntime":1700001645,"order_key":"06:miner-6","p2mr_program_hex":"6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d","share_difficulty":2,"share_id":"share-1645","share_seq":1645,"template_height":800051},{"accepted_at_ms":999989,"credit_policy":null,"job_id":"job-1503","job_issued_at_ms":999977,"miner_id":"miner-7","network_difficulty":1000007,"ntime":1700001503,"order_key":"07:miner-7","p2mr_program_hex":"dfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdf","share_difficulty":19,"share_id":"share-1503","share_seq":1503,"template_height":800046},{"accepted_at_ms":999992,"credit_policy":null,"job_id":"job-3802","job_issued_at_ms":999985,"miner_id":"miner-7","network_difficulty":1000011,"ntime":1700003802,"order_key":"07:miner-7","p2mr_program_hex":"dadadadadadadadadadadadadadadadadadadadadadadadadadadadadadadada","share_difficulty":3,"share_id":"share-3802","share_seq":3802,"template_height":800118},{"accepted_at_ms":999979,"credit_policy":"stale-grace","job_id":"job-91","job_issued_at_ms":999973,"miner_id":"miner-3","network_difficulty":1000006,"ntime":1700000091,"order_key":"03:miner-3","p2mr_program_hex":"5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b","share_difficulty":1000,"share_id":"share-91","share_seq":91,"template_height":800002},{"accepted_at_ms":999989,"credit_policy":null,"job_id":"job-348","job_issued_at_ms":999995,"miner_id":"miner-7","network_difficulty":1000008,"ntime":1700000348,"order_key":"07:miner-7","p2mr_program_hex":"5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c","share_difficulty":2,"share_id":"share-348","share_seq":348,"template_height":800010},{"accepted_at_ms":999974,"credit_policy":"stale-grace","job_id":"job-1364","job_issued_at_ms":999974,"miner_id":"miner-0","network_difficulty":1000004,"ntime":1700001364,"order_key":"00:miner-0","p2mr_program_hex":"5454545454545454545454545454545454545454545454545454545454545454","share_difficulty":7,"share_id":"share-1364","share_seq":1364,"template_height":800042},{"accepted_at_ms":999996,"credit_policy":null,"job_id":"job-2738","job_issued_at_ms":999990,"miner_id":"miner-10","network_difficulty":1000001,"ntime":1700002738,"order_key":"10:miner-10","p2mr_program_hex":"b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2","share_difficulty":2,"share_id":"share-2738","share_seq":2738,"template_height":800085},{"accepted_at_ms":1000036,"credit_policy":null,"job_id":"job-2693","job_issued_at_ms":999982,"miner_id":"miner-9","network_difficulty":1000007,"ntime":1700002693,"order_key":"09:miner-9","p2mr_program_hex":"8585858585858585858585858585858585858585858585858585858585858585","share_difficulty":101,"share_id":"share-2693","share_seq":2693,"template_height":800084},{"accepted_at_ms":999992,"credit_policy":null,"job_id":"job-630","job_issued_at_ms":999984,"miner_id":"miner-3","network_difficulty":1000001,"ntime":1700000630,"order_key":"03:miner-3","p2mr_program_hex":"7676767676767676767676767676767676767676767676767676767676767676","share_difficulty":101,"share_id":"share-630","share_seq":630,"template_height":800019},{"accepted_at_ms":999996,"credit_policy":null,"job_id":"job-2593","job_issued_at_ms":999980,"miner_id":"miner-8","network_difficulty":1000009,"ntime":1700002593,"order_key":"08:miner-8","p2mr_program_hex":"2121212121212121212121212121212121212121212121212121212121212121","share_difficulty":3,"share_id":"share-2593","share_seq":2593,"template_height":800081},{"accepted_at_ms":999971,"credit_policy":null,"job_id":"job-2718","job_issued_at_ms":999999,"miner_id":"miner-1","network_difficulty":1000015,"ntime":1700002718,"order_key":"01:miner-1","p2mr_program_hex":"9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e","share_difficulty":3,"share_id":"share-2718","share_seq":2718,"template_height":800084},{"accepted_at_ms":999996,"credit_policy":null,"job_id":"job-2569","job_issued_at_ms":999971,"miner_id":"miner-6","network_difficulty":1000002,"ntime":1700002569,"order_key":"06:miner-6","p2mr_program_hex":"0909090909090909090909090909090909090909090909090909090909090909","share_difficulty":7,"share_id":"share-2569","share_seq":2569,"template_height":800080},{"accepted_at_ms":999993,"credit_policy":null,"job_id":"job-1583","job_issued_at_ms":999987,"miner_id":"miner-10","network_difficulty":1000002,"ntime":1700001583,"order_key":"10:miner-10","p2mr_program_hex":"2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f","share_difficulty":3,"share_id":"share-1583","share_seq":1583,"template_height":800049},{"accepted_at_ms":999999,"credit_policy":null,"job_id":"job-940","job_issued_at_ms":999974,"miner_id":"miner-5","network_difficulty":1000005,"ntime":1700000940,"order_key":"05:miner-5","p2mr_program_hex":"acacacacacacacacacacacacacacacacacacacacacacacacacacacacacacacac","share_difficulty":1,"share_id":"share-940","share_seq":940,"template_height":800029},{"accepted_at_ms":999977,"credit_policy":null,"job_id":"job-3781","job_issued_at_ms":999976,"miner_id":"miner-8","network_difficulty":1000007,"ntime":1700003781,"order_key":"08:miner-8","p2mr_program_hex":"c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5","share_difficulty":101,"share_id":"share-3781","share_seq":3781,"template_height":800118},{"accepted_at_ms":1000141,"credit_policy":null,"job_id":"job-747","job_issued_at_ms":1000255,"miner_id":"miner-10","network_difficulty":1000016,"ntime":1700000747,"order_key":"10:miner-10","p2mr_program_hex":"ebebebebebebebebebebebebebebebebebebebebebebebebebebebebebebebeb","share_difficulty":7,"share_id":"share-747","share_seq":747,"template_height":800023},{"accepted_at_ms":999996,"credit_policy":"stale-grace","job_id":"job-2420","job_issued_at_ms":999992,"miner_id":"miner-0","network_difficulty":1000006,"ntime":1700002420,"order_key":"00:miner-0","p2mr_program_hex":"7474747474747474747474747474747474747474747474747474747474747474","share_difficulty":2,"share_id":"share-2420","share_seq":2420,"template_height":800075},{"accepted_at_ms":999979,"credit_policy":null,"job_id":"job-2465","job_issued_at_ms":999991,"miner_id":"miner-1","network_difficulty":1000000,"ntime":1700002465,"order_key":"01:miner-1","p2mr_program_hex":"a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1","share_difficulty":7,"share_id":"share-2465","share_seq":2465,"template_height":800077},{"accepted_at_ms":999988,"credit_policy":null,"job_id":"job-1281","job_issued_at_ms":999979,"miner_id":"miner-5","network_difficulty":1000006,"ntime":1700001281,"order_key":"05:miner-5","p2mr_program_hex":"0101010101010101010101010101010101010101010101010101010101010101","share_difficulty":1000,"share_id":"share-1281","share_seq":1281,"template_height":800040},{"accepted_at_ms":999981,"credit_policy":null,"job_id":"job-3110","job_issued_at_ms":999984,"miner_id":"miner-8","network_difficulty":1000016,"ntime":1700003110,"order_key":"08:miner-8","p2mr_program_hex":"2626262626262626262626262626262626262626262626262626262626262626","share_difficulty":1000,"share_id":"share-3110","share_seq":3110,"template_height":800097},{"accepted_at_ms":999982,"credit_policy":null,"job_id":"job-435","job_issued_at_ms":999997,"miner_id":"miner-6","network_difficulty":1000010,"ntime":1700000435,"order_key":"06:miner-6","p2mr_program_hex":"b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3","share_difficulty":2,"share_id":"share-435","share_seq":435,"template_height":800013},{"accepted_at_ms":1000306,"credit_policy":null,"job_id":"job-2951","job_issued_at_ms":1000311,"miner_id":"miner-3","network_difficulty":1000010,"ntime":1700002951,"order_key":"03:miner-3","p2mr_program_hex":"8787878787878787878787878787878787878787878787878787878787878787","share_difficulty":2,"share_id":"share-2951","share_seq":2951,"template_height":800092},{"accepted_at_ms":999997,"credit_policy":null,"job_id":"job-2644","job_issued_at_ms":999990,"miner_id":"miner-4","network_difficulty":1000009,"ntime":1700002644,"order_key":"04:miner-4","p2mr_program_hex":"5454545454545454545454545454545454545454545454545454545454545454","share_difficulty":2,"share_id":"share-2644","share_seq":2644,"template_height":800082},{"accepted_at_ms":999998,"credit_policy":"stale-grace","job_id":"job-1134","job_issued_at_ms":999996,"miner_id":"miner-1","network_difficulty":1000012,"ntime":1700001134,"order_key":"01:miner-1","p2mr_program_hex":"6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e","share_difficulty":3,"share_id":"share-1134","share_seq":1134,"template_height":800035},{"accepted_at_ms":999991,"credit_policy":null,"job_id":"job-1742","job_issued_at_ms":999974,"miner_id":"miner-4","network_difficulty":1000008,"ntime":1700001742,"order_key":"04:miner-4","p2mr_program_hex":"cececececececececececececececececececececececececececececececece","share_difficulty":7,"share_id":"share-1742","share_seq":1742,"template_height":800054},{"accepted_at_ms":999983,"credit_policy":"stale-grace","job_id":"job-3959","job_issued_at_ms":999974,"miner_id":"miner-10","network_difficulty":1000015,"ntime":1700003959,"order_key":"10:miner-10","p2mr_program_hex":"7777777777777777777777777777777777777777777777777777777777777777","share_difficulty":1,"share_id":"share-3959","share_seq":3959,"template_height":800123},{"accepted_at_ms":999974,"credit_policy":null,"job_id":"job-2266","job_issued_at_ms":999983,"miner_id":"miner-0","network_difficulty":1000005,"ntime":1700002266,"order_key":"00:miner-0","p2mr_program_hex":"dadadadadadadadadadadadadadadadadadadadadadadadadadadadadadadada","share_difficulty":3,"share_id":"share-2266","share_seq":2266,"template_height":800070},{"accepted_at_ms":999975,"credit_policy":null,"job_id":"job-3297","job_issued_at_ms":999974,"miner_id":"miner-8","network_difficulty":1000016,"ntime":1700003297,"order_key":"08:miner-8","p2mr_program_hex":"e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1","share_difficulty":1000,"share_id":"share-3297","share_seq":3297,"template_height":800103},{"accepted_at_ms":999993,"credit_policy":null,"job_id":"job-3813","job_issued_at_ms":999996,"miner_id":"miner-7","network_difficulty":1000005,"ntime":1700003813,"order_key":"07:miner-7","p2mr_program_hex":"e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5","share_difficulty":101,"share_id":"share-3813","share_seq":3813,"template_height":800119},{"accepted_at_ms":999983,"credit_policy":null,"job_id":"job-2387","job_issued_at_ms":999977,"miner_id":"miner-0","network_difficulty":1000007,"ntime":1700002387,"order_key":"00:miner-0","p2mr_program_hex":"5353535353535353535353535353535353535353535353535353535353535353","share_difficulty":1,"share_id":"share-2387","share_seq":2387,"template_height":800074},{"accepted_at_ms":999995,"credit_policy":null,"job_id":"job-772","job_issued_at_ms":999990,"miner_id":"miner-2","network_difficulty":1000007,"ntime":1700000772,"order_key":"02:miner-2","p2mr_program_hex":"0404040404040404040404040404040404040404040404040404040404040404","share_difficulty":19,"share_id":"share-772","share_seq":772,"template_height":800024},{"accepted_at_ms":1000231,"credit_policy":null,"job_id":"job-1557","job_issued_at_ms":1000188,"miner_id":"miner-6","network_difficulty":1000010,"ntime":1700001557,"order_key":"06:miner-6","p2mr_program_hex":"1515151515151515151515151515151515151515151515151515151515151515","share_difficulty":1,"share_id":"share-1557","share_seq":1557,"template_height":800048},{"accepted_at_ms":999998,"credit_policy":null,"job_id":"job-84","job_issued_at_ms":999984,"miner_id":"miner-7","network_difficulty":1000016,"ntime":1700000084,"order_key":"07:miner-7","p2mr_program_hex":"5454545454545454545454545454545454545454545454545454545454545454","share_difficulty":101,"share_id":"share-84","share_seq":84,"template_height":800002},{"accepted_at_ms":999976,"credit_policy":null,"job_id":"job-2927","job_issued_at_ms":999992,"miner_id":"miner-1","network_difficulty":1000003,"ntime":1700002927,"order_key":"01:miner-1","p2mr_program_hex":"6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f","share_difficulty":7,"share_id":"share-2927","share_seq":2927,"template_height":800091},{"accepted_at_ms":999990,"credit_policy":null,"job_id":"job-3272","job_issued_at_ms":999970,"miner_id":"miner-5","network_difficulty":1000008,"ntime":1700003272,"order_key":"05:miner-5","p2mr_program_hex":"c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8","share_difficulty":3,"share_id":"share-3272","share_seq":3272,"template_height":800102},{"accepted_at_ms":999983,"credit_policy":null,"job_id":"job-3947","job_issued_at_ms":1000000,"miner_id":"miner-9","network_difficulty":1000003,"ntime":1700003947,"order_key":"09:miner-9","p2mr_program_hex":"6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b","share_difficulty":1000,"share_id":"share-3947","share_seq":3947,"template_height":800123},{"accepted_at_ms":999996,"credit_policy":null,"job_id":"job-2983","job_issued_at_ms":999973,"miner_id":"miner-2","network_difficulty":1000008,"ntime":1700002983,"order_key":"02:miner-2","p2mr_program_hex":"a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7","share_difficulty":7,"share_id":"share-2983","share_seq":2983,"template_height":800093},{"accepted_at_ms":999987,"credit_policy":null,"job_id":"job-1388","job_issued_at_ms":999971,"miner_id":"miner-2","network_difficulty":1000011,"ntime":1700001388,"order_key":"02:miner-2","p2mr_program_hex":"6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c","share_difficulty":1000,"share_id":"share-1388","share_seq":1388,"template_height":800043},{"accepted_at_ms":999994,"credit_policy":null,"job_id":"job-1744","job_issued_at_ms":999998,"miner_id":"miner-6","network_difficulty":1000010,"ntime":1700001744,"order_key":"06:miner-6","p2mr_program_hex":"d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0","share_difficulty":19,"share_id":"share-1744","share_seq":1744,"template_height":800054},{"accepted_at_ms":999983,"credit_policy":null,"job_id":"job-2006","job_issued_at_ms":999998,"miner_id":"miner-4","network_difficulty":1000000,"ntime":1700002006,"order_key":"04:miner-4","p2mr_program_hex":"d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6","share_difficulty":7,"share_id":"share-2006","share_seq":2006,"template_height":800062},{"accepted_at_ms":999970,"credit_policy":"stale-grace","job_id":"job-3599","job_issued_at_ms":999998,"miner_id":"miner-2","network_difficulty":1000012,"ntime":1700003599,"order_key":"02:miner-2","p2mr_program_hex":"0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f","share_difficulty":1000,"share_id":"share-3599","share_seq":3599,"template_height":800112},{"accepted_at_ms":999981,"credit_policy":"stale-grace","job_id":"job-2675","job_issued_at_ms":999992,"miner_id":"miner-2","network_difficulty":1000006,"ntime":1700002675,"order_key":"02:miner-2","p2mr_program_hex":"7373737373737373737373737373737373737373737373737373737373737373","share_difficulty":2,"share_id":"share-2675","share_seq":2675,"template_height":800083},{"accepted_at_ms":999994,"credit_policy":null,"job_id":"job-682","job_issued_at_ms":999977,"miner_id":"miner-0","network_difficulty":1000002,"ntime":1700000682,"order_key":"00:miner-0","p2mr_program_hex":"aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa","share_difficulty":3,"share_id":"share-682","share_seq":682,"template_height":800021},{"accepted_at_ms":999972,"credit_policy":null,"job_id":"job-1027","job_issued_at_ms":999978,"miner_id":"miner-4","network_difficulty":1000007,"ntime":1700001027,"order_key":"04:miner-4","p2mr_program_hex":"0303030303030303030303030303030303030303030303030303030303030303","share_difficulty":3,"share_id":"share-1027","share_seq":1027,"template_height":800032},{"accepted_at_ms":999975,"credit_policy":null,"job_id":"job-1298","job_issued_at_ms":1000363,"miner_id":"miner-0","network_difficulty":1000006,"ntime":1700001298,"order_key":"00:miner-0","p2mr_program_hex":"1212121212121212121212121212121212121212121212121212121212121212","share_difficulty":101,"share_id":"share-1298","share_seq":1298,"template_height":800040},{"accepted_at_ms":999992,"credit_policy":null,"job_id":"job-1747","job_issued_at_ms":1000000,"miner_id":"miner-9","network_difficulty":1000013,"ntime":1700001747,"order_key":"09:miner-9","p2mr_program_hex":"d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3","share_difficulty":19,"share_id":"share-1747","share_seq":1747,"template_height":800054},{"accepted_at_ms":999994,"credit_policy":null,"job_id":"job-440","job_issued_at_ms":999998,"miner_id":"miner-0","network_difficulty":1000015,"ntime":1700000440,"order_key":"00:miner-0","p2mr_program_hex":"b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8","share_difficulty":19,"share_id":"share-440","share_seq":440,"template_height":800013},{"accepted_at_ms":999990,"credit_policy":null,"job_id":"job-1801","job_issued_at_ms":999974,"miner_id":"miner-8","network_difficulty":1000016,"ntime":1700001801,"order_key":"08:miner-8","p2mr_program_hex":"0909090909090909090909090909090909090909090909090909090909090909","share_difficulty":101,"share_id":"share-1801","share_seq":1801,"template_height":800056},{"accepted_at_ms":999978,"credit_policy":null,"job_id":"job-2306","job_issued_at_ms":999974,"miner_id":"miner-7","network_difficulty":1000011,"ntime":1700002306,"order_key":"07:miner-7","p2mr_program_hex":"0202020202020202020202020202020202020202020202020202020202020202","share_difficulty":101,"share_id":"share-2306","share_seq":2306,"template_height":800072},{"accepted_at_ms":999974,"credit_policy":"stale-grace","job_id":"job-1865","job_issued_at_ms":999985,"miner_id":"miner-6","network_difficulty":1000012,"ntime":1700001865,"order_key":"06:miner-6","p2mr_program_hex":"4949494949494949494949494949494949494949494949494949494949494949","share_difficulty":1,"share_id":"share-1865","share_seq":1865,"template_height":800058},{"accepted_at_ms":1000499,"credit_policy":null,"job_id":"job-310","job_issued_at_ms":999974,"miner_id":"miner-2","network_difficulty":1000004,"ntime":1700000310,"order_key":"02:miner-2","p2mr_program_hex":"3636363636363636363636363636363636363636363636363636363636363636","share_difficulty":2,"share_id":"share-310","share_seq":310,"template_height":800009},{"accepted_at_ms":999994,"credit_policy":null,"job_id":"job-403","job_issued_at_ms":999989,"miner_id":"miner-7","network_difficulty":1000012,"ntime":1700000403,"order_key":"07:miner-7","p2mr_program_hex":"9393939393939393939393939393939393939393939393939393939393939393","share_difficulty":1000,"share_id":"share-403","share_seq":403,"template_height":800012},{"accepted_at_ms":999998,"credit_policy":null,"job_id":"job-2782","job_issued_at_ms":999994,"miner_id":"miner-10","network_difficulty":1000011,"ntime":1700002782,"order_key":"10:miner-10","p2mr_program_hex":"dededededededededededededededededededededededededededededededede","share_difficulty":101,"share_id":"share-2782","share_seq":2782,"template_height":800086},{"accepted_at_ms":999987,"credit_policy":null,"job_id":"job-1554","job_issued_at_ms":999974,"miner_id":"miner-3","network_difficulty":1000007,"ntime":1700001554,"order_key":"03:miner-3","p2mr_program_hex":"1212121212121212121212121212121212121212121212121212121212121212","share_difficulty":1000,"share_id":"share-1554","share_seq":1554,"template_height":800048},{"accepted_at_ms":999991,"credit_policy":null,"job_id":"job-1096","job_issued_at_ms":999979,"miner_id":"miner-7","network_difficulty":1000008,"ntime":1700001096,"order_key":"07:miner-7","p2mr_program_hex":"4848484848484848484848484848484848484848484848484848484848484848","share_difficulty":19,"share_id":"share-1096","share_seq":1096,"template_height":800034},{"accepted_at_ms":999972,"credit_policy":null,"job_id":"job-2142","job_issued_at_ms":999970,"miner_id":"miner-8","network_difficulty":1000000,"ntime":1700002142,"order_key":"08:miner-8","p2mr_program_hex":"5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e","share_difficulty":1,"share_id":"share-2142","share_seq":2142,"template_height":800066},{"accepted_at_ms":999980,"credit_policy":null,"job_id":"job-2590","job_issued_at_ms":999998,"miner_id":"miner-5","network_difficulty":1000006,"ntime":1700002590,"order_key":"05:miner-5","p2mr_program_hex":"1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e","share_difficulty":1000,"share_id":"share-2590","share_seq":2590,"template_height":800080},{"accepted_at_ms":999991,"credit_policy":"stale-grace","job_id":"job-822","job_issued_at_ms":999995,"miner_id":"miner-8","network_difficulty":1000006,"ntime":1700000822,"order_key":"08:miner-8","p2mr_program_hex":"3636363636363636363636363636363636363636363636363636363636363636","share_difficulty":19,"share_id":"share-822","share_seq":822,"template_height":800025},{"accepted_at_ms":999982,"credit_policy":null,"job_id":"job-1473","job_issued_at_ms":999998,"miner_id":"miner-10","network_difficulty":1000011,"ntime":1700001473,"order_key":"10:miner-10","p2mr_program_hex":"c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1","share_difficulty":19,"share_id":"share-1473","share_seq":1473,"template_height":800046},{"accepted_at_ms":999972,"credit_policy":null,"job_id":"job-1800","job_issued_at_ms":999995,"miner_id":"miner-7","network_difficulty":1000015,"ntime":1700001800,"order_key":"07:miner-7","p2mr_program_hex":"0808080808080808080808080808080808080808080808080808080808080808","share_difficulty":1000,"share_id":"share-1800","share_seq":1800,"template_height":800056},{"accepted_at_ms":999979,"credit_policy":"stale-grace","job_id":"job-372","job_issued_at_ms":999976,"miner_id":"miner-9","network_difficulty":1000015,"ntime":1700000372,"order_key":"09:miner-9","p2mr_program_hex":"7474747474747474747474747474747474747474747474747474747474747474","share_difficulty":3,"share_id":"share-372","share_seq":372,"template_height":800011},{"accepted_at_ms":999988,"credit_policy":null,"job_id":"job-1911","job_issued_at_ms":999984,"miner_id":"miner-8","network_difficulty":1000007,"ntime":1700001911,"order_key":"08:miner-8","p2mr_program_hex":"7777777777777777777777777777777777777777777777777777777777777777","share_difficulty":1,"share_id":"share-1911","share_seq":1911,"template_height":800059},{"accepted_at_ms":999978,"credit_policy":"stale-grace","job_id":"job-729","job_issued_at_ms":999999,"miner_id":"miner-3","network_difficulty":1000015,"ntime":1700000729,"order_key":"03:miner-3","p2mr_program_hex":"d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9","share_difficulty":2,"share_id":"share-729","share_seq":729,"template_height":800022},{"accepted_at_ms":999970,"credit_policy":null,"job_id":"job-587","job_issued_at_ms":999973,"miner_id":"miner-4","network_difficulty":1000009,"ntime":1700000587,"order_key":"04:miner-4","p2mr_program_hex":"4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b","share_difficulty":3,"share_id":"share-587","share_seq":587,"template_height":800018},{"accepted_at_ms":999979,"credit_policy":null,"job_id":"job-110","job_issued_at_ms":999978,"miner_id":"miner-0","network_difficulty":1000008,"ntime":1700000110,"order_key":"00:miner-0","p2mr_program_hex":"6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e","share_difficulty":2,"share_id":"share-110","share_seq":110,"template_height":800003},{"accepted_at_ms":999994,"credit_policy":null,"job_id":"job-1371","job_issued_at_ms":999976,"miner_id":"miner-7","network_difficulty":1000011,"ntime":1700001371,"order_key":"07:miner-7","p2mr_program_hex":"5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b","share_difficulty":19,"share_id":"share-1371","share_seq":1371,"template_height":800042},{"accepted_at_ms":999990,"credit_policy":null,"job_id":"job-764","job_issued_at_ms":999978,"miner_id":"miner-5","network_difficulty":1000016,"ntime":1700000764,"order_key":"05:miner-5","p2mr_program_hex":"fcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfc","share_difficulty":101,"share_id":"share-764","share_seq":764,"template_height":800023},{"accepted_at_ms":999981,"credit_policy":null,"job_id":"job-1162","job_issued_at_ms":999999,"miner_id":"miner-7","network_difficulty":1000006,"ntime":1700001162,"order_key":"07:miner-7","p2mr_program_hex":"8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a","share_difficulty":2,"share_id":"share-1162","share_seq":1162,"template_height":800036},{"accepted_at_ms":999997,"credit_policy":"stale-grace","job_id":"job-956","job_issued_at_ms":999974,"miner_id":"miner-10","network_difficulty":1000004,"ntime":1700000956,"order_key":"10:miner-10","p2mr_program_hex":"bcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbc","share_difficulty":19,"share_id":"share-956","share_seq":956,"template_height":800029},{"accepted_at_ms":999997,"credit_policy":null,"job_id":"job-1260","job_issued_at_ms":999995,"miner_id":"miner-6","network_difficulty":1000002,"ntime":1700001260,"order_key":"06:miner-6","p2mr_program_hex":"ecececececececececececececececececececececececececececececececec","share_difficulty":101,"share_id":"share-1260","share_seq":1260,"template_height":800039},{"accepted_at_ms":999987,"credit_policy":null,"job_id":"job-178","job_issued_at_ms":999979,"miner_id":"miner-2","network_difficulty":1000008,"ntime":1700000178,"order_key":"02:miner-2","p2mr_program_hex":"b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2","share_difficulty":2,"share_id":"share-178","share_seq":178,"template_height":800005},{"accepted_at_ms":999988,"credit_policy":null,"job_id":"job-233","job_issued_at_ms":999996,"miner_id":"miner-2","network_difficulty":1000012,"ntime":1700000233,"order_key":"02:miner-2","p2mr_program_hex":"e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9","share_difficulty":2,"share_id":"share-233","share_seq":233,"template_height":800007},{"accepted_at_ms":1000163,"credit_policy":null,"job_id":"job-3815","job_issued_at_ms":1000067,"miner_id":"miner-9","network_difficulty":1000007,"ntime":1700003815,"order_key":"09:miner-9","p2mr_program_hex":"e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7","share_difficulty":1000,"share_id":"share-3815","share_seq":3815,"template_height":800119},{"accepted_at_ms":1000326,"credit_policy":"stale-grace","job_id":"job-3451","job_issued_at_ms":1000065,"miner_id":"miner-8","network_difficulty":1000000,"ntime":1700003451,"order_key":"08:miner-8","p2mr_program_hex":"7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b","share_difficulty":101,"share_id":"share-3451","share_seq":3451,"template_height":800107},{"accepted_at_ms":999988,"credit_policy":null,"job_id":"job-393","job_issued_at_ms":999993,"miner_id":"miner-8","network_difficulty":1000002,"ntime":1700000393,"order_key":"08:miner-8","p2mr_program_hex":"8989898989898989898989898989898989898989898989898989898989898989","share_difficulty":1000,"share_id":"share-393","share_seq":393,"template_height":800012},{"accepted_at_ms":999974,"credit_policy":null,"job_id":"job-763","job_issued_at_ms":999986,"miner_id":"miner-4","network_difficulty":1000015,"ntime":1700000763,"order_key":"04:miner-4","p2mr_program_hex":"fbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfb","share_difficulty":7,"share_id":"share-763","share_seq":763,"template_height":800023},{"accepted_at_ms":999993,"credit_policy":null,"job_id":"job-1346","job_issued_at_ms":999998,"miner_id":"miner-4","network_difficulty":1000003,"ntime":1700001346,"order_key":"04:miner-4","p2mr_program_hex":"4242424242424242424242424242424242424242424242424242424242424242","share_difficulty":1,"share_id":"share-1346","share_seq":1346,"template_height":800042},{"accepted_at_ms":999987,"credit_policy":null,"job_id":"job-1993","job_issued_at_ms":999979,"miner_id":"miner-2","network_difficulty":1000004,"ntime":1700001993,"order_key":"02:miner-2","p2mr_program_hex":"c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9","share_difficulty":19,"share_id":"share-1993","share_seq":1993,"template_height":800062},{"accepted_at_ms":999996,"credit_policy":"stale-grace","job_id":"job-941","job_issued_at_ms":999972,"miner_id":"miner-6","network_difficulty":1000006,"ntime":1700000941,"order_key":"06:miner-6","p2mr_program_hex":"adadadadadadadadadadadadadadadadadadadadadadadadadadadadadadadad","share_difficulty":1,"share_id":"share-941","share_seq":941,"template_height":800029},{"accepted_at_ms":999986,"credit_policy":null,"job_id":"job-3142","job_issued_at_ms":999990,"miner_id":"miner-7","network_difficulty":1000014,"ntime":1700003142,"order_key":"07:miner-7","p2mr_program_hex":"4646464646464646464646464646464646464646464646464646464646464646","share_difficulty":2,"share_id":"share-3142","share_seq":3142,"template_height":800098},{"accepted_at_ms":1000092,"credit_policy":null,"job_id":"job-1784","job_issued_at_ms":1000144,"miner_id":"miner-2","network_difficulty":1000016,"ntime":1700001784,"order_key":"02:miner-2","p2mr_program_hex":"f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8","share_difficulty":19,"share_id":"share-1784","share_seq":1784,"template_height":800055},{"accepted_at_ms":999977,"credit_policy":null,"job_id":"job-3350","job_issued_at_ms":1000000,"miner_id":"miner-6","network_difficulty":1000001,"ntime":1700003350,"order_key":"06:miner-6","p2mr_program_hex":"1616161616161616161616161616161616161616161616161616161616161616","share_difficulty":2,"share_id":"share-3350","share_seq":3350,"template_height":800104},{"accepted_at_ms":999993,"credit_policy":"stale-grace","job_id":"job-1469","job_issued_at_ms":999988,"miner_id":"miner-6","network_difficulty":1000007,"ntime":1700001469,"order_key":"06:miner-6","p2mr_program_hex":"bdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbd","share_difficulty":1,"share_id":"share-1469","share_seq":1469,"template_height":800045},{"accepted_at_ms":1000000,"credit_policy":"stale-grace","job_id":"job-3923","job_issued_at_ms":999992,"miner_id":"miner-7","network_difficulty":1000013,"ntime":1700003923,"order_key":"07:miner-7","p2mr_program_hex":"5353535353535353535353535353535353535353535353535353535353535353","share_difficulty":2,"share_id":"share-3923","share_seq":3923,"template_height":800122},{"accepted_at_ms":999988,"credit_policy":null,"job_id":"job-313","job_issued_at_ms":999982,"miner_id":"miner-5","network_difficulty":1000007,"ntime":1700000313,"order_key":"05:miner-5","p2mr_program_hex":"3939393939393939393939393939393939393939393939393939393939393939","share_difficulty":3,"share_id":"share-313","share_seq":313,"template_height":800009},{"accepted_at_ms":999975,"credit_policy":null,"job_id":"job-2332","job_issued_at_ms":999978,"miner_id":"miner-0","network_difficulty":1000003,"ntime":1700002332,"order_key":"00:miner-0","p2mr_program_hex":"1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c","share_difficulty":2,"share_id":"share-2332","share_seq":2332,"template_height":800072},{"accepted_at_ms":999988,"credit_policy":"stale-grace","job_id":"job-3868","job_issued_at_ms":999979,"miner_id":"miner-7","network_difficulty":1000009,"ntime":1700003868,"order_key":"07:miner-7","p2mr_program_hex":"1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c","share_difficulty":2,"share_id":"share-3868","share_seq":3868,"template_height":800120},{"accepted_at_ms":999996,"credit_policy":null,"job_id":"job-193","job_issued_at_ms":999995,"miner_id":"miner-6","network_difficulty":1000006,"ntime":1700000193,"order_key":"06:miner-6","p2mr_program_hex":"c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1","share_difficulty":3,"share_id":"share-193","share_seq":193,"template_height":800006},{"accepted_at_ms":999971,"credit_policy":"stale-grace","job_id":"job-2863","job_issued_at_ms":999979,"miner_id":"miner-3","network_difficulty":1000007,"ntime":1700002863,"order_key":"03:miner-3","p2mr_program_hex":"2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f","share_difficulty":1000,"share_id":"share-2863","share_seq":2863,"template_height":800089},{"accepted_at_ms":1000000,"credit_policy":"stale-grace","job_id":"job-2450","job_issued_at_ms":999984,"miner_id":"miner-8","network_difficulty":1000002,"ntime":1700002450,"order_key":"08:miner-8","p2mr_program_hex":"9292929292929292929292929292929292929292929292929292929292929292","share_difficulty":1,"share_id":"share-2450","share_seq":2450,"template_height":800076},{"accepted_at_ms":999977,"credit_policy":null,"job_id":"job-959","job_issued_at_ms":999977,"miner_id":"miner-2","network_difficulty":1000007,"ntime":1700000959,"order_key":"02:miner-2","p2mr_program_hex":"bfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbf","share_difficulty":19,"share_id":"share-959","share_seq":959,"template_height":800029},{"accepted_at_ms":999983,"credit_policy":null,"job_id":"job-126","job_issued_at_ms":999970,"miner_id":"miner-5","network_difficulty":1000007,"ntime":1700000126,"order_key":"05:miner-5","p2mr_program_hex":"7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e","share_difficulty":7,"share_id":"share-126","share_seq":126,"template_height":800003},{"accepted_at_ms":999979,"credit_policy":null,"job_id":"job-1329","job_issued_at_ms":1000000,"miner_id":"miner-9","network_difficulty":1000003,"ntime":1700001329,"order_key":"09:miner-9","p2mr_program_hex":"3131313131313131313131313131313131313131313131313131313131313131","share_difficulty":1000,"share_id":"share-1329","share_seq":1329,"template_height":800041},{"accepted_at_ms":999993,"credit_policy":null,"job_id":"job-3053","job_issued_at_ms":999970,"miner_id":"miner-6","network_difficulty":1000010,"ntime":1700003053,"order_key":"06:miner-6","p2mr_program_hex":"edededededededededededededededededededededededededededededededed","share_difficulty":1000,"share_id":"share-3053","share_seq":3053,"template_height":800095},{"accepted_at_ms":999972,"credit_policy":null,"job_id":"job-2819","job_issued_at_ms":999989,"miner_id":"miner-3","network_difficulty":1000014,"ntime":1700002819,"order_key":"03:miner-3","p2mr_program_hex":"0303030303030303030303030303030303030303030303030303030303030303","share_difficulty":3,"share_id":"share-2819","share_seq":2819,"template_height":800088},{"accepted_at_ms":999990,"credit_policy":null,"job_id":"job-291","job_issued_at_ms":999991,"miner_id":"miner-5","network_difficulty":1000002,"ntime":1700000291,"order_key":"05:miner-5","p2mr_program_hex":"2323232323232323232323232323232323232323232323232323232323232323","share_difficulty":101,"share_id":"share-291","share_seq":291,"template_height":800009},{"accepted_at_ms":999986,"credit_policy":"stale-grace","job_id":"job-2412","job_issued_at_ms":1000052,"miner_id":"miner-3","network_difficulty":1000015,"ntime":1700002412,"order_key":"03:miner-3","p2mr_program_hex":"6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c","share_difficulty":3,"share_id":"share-2412","share_seq":2412,"template_height":800075},{"accepted_at_ms":999991,"credit_policy":null,"job_id":"job-2650","job_issued_at_ms":999973,"miner_id":"miner-10","network_difficulty":1000015,"ntime":1700002650,"order_key":"10:miner-10","p2mr_program_hex":"5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a","share_difficulty":1000,"share_id":"share-2650","share_seq":2650,"template_height":800082},{"accepted_at_ms":999993,"credit_policy":null,"job_id":"job-1295","job_issued_at_ms":999970,"miner_id":"miner-8","network_difficulty":1000003,"ntime":1700001295,"order_key":"08:miner-8","p2mr_program_hex":"0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f","share_difficulty":1,"share_id":"share-1295","share_seq":1295,"template_height":800040},{"accepted_at_ms":999977,"credit_policy":null,"job_id":"job-1079","job_issued_at_ms":999979,"miner_id":"miner-1","network_difficulty":1000008,"ntime":1700001079,"order_key":"01:miner-1","p2mr_program_hex":"3737373737373737373737373737373737373737373737373737373737373737","share_difficulty":1000,"share_id":"share-1079","share_seq":1079,"template_height":800033},{"accepted_at_ms":999985,"credit_policy":"stale-grace","job_id":"job-2991","job_issued_at_ms":999997,"miner_id":"miner-10","network_difficulty":1000016,"ntime":1700002991,"order_key":"10:miner-10","p2mr_program_hex":"afafafafafafafafafafafafafafafafafafafafafafafafafafafafafafafaf","share_difficulty":7,"share_id":"share-2991","share_seq":2991,"template_height":800093},{"accepted_at_ms":999988,"credit_policy":null,"job_id":"job-2492","job_issued_at_ms":999975,"miner_id":"miner-6","network_difficulty":1000010,"ntime":1700002492,"order_key":"06:miner-6","p2mr_program_hex":"bcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbc","share_difficulty":19,"share_id":"share-2492","share_seq":2492,"template_height":800077},{"accepted_at_ms":999994,"credit_policy":null,"job_id":"job-46","job_issued_at_ms":999988,"miner_id":"miner-2","network_difficulty":1000012,"ntime":1700000046,"order_key":"02:miner-2","p2mr_program_hex":"2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e","share_difficulty":19,"share_id":"share-46","share_seq":46,"template_height":800001},{"accepted_at_ms":999983,"credit_policy":null,"job_id":"job-476","job_issued_at_ms":999984,"miner_id":"miner-3","network_difficulty":1000000,"ntime":1700000476,"order_key":"03:miner-3","p2mr_program_hex":"dcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdc","share_difficulty":7,"share_id":"share-476","share_seq":476,"template_height":800014},{"accepted_at_ms":999987,"credit_policy":"stale-grace","job_id":"job-216","job_issued_at_ms":1000475,"miner_id":"miner-7","network_difficulty":1000012,"ntime":1700000216,"order_key":"07:miner-7","p2mr_program_hex":"d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8","share_difficulty":2,"share_id":"share-216","share_seq":216,"template_height":800006},{"accepted_at_ms":999977,"credit_policy":null,"job_id":"job-3663","job_issued_at_ms":999990,"miner_id":"miner-0","network_difficulty":1000008,"ntime":1700003663,"order_key":"00:miner-0","p2mr_program_hex":"4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f","share_difficulty":3,"share_id":"share-3663","share_seq":3663,"template_height":800114},{"accepted_at_ms":999997,"credit_policy":null,"job_id":"job-35","job_issued_at_ms":999999,"miner_id":"miner-2","network_difficulty":1000001,"ntime":1700000035,"order_key":"02:miner-2","p2mr_program_hex":"2323232323232323232323232323232323232323232323232323232323232323","share_difficulty":19,"share_id":"share-35","share_seq":35,"template_height":800001},{"accepted_at_ms":999992,"credit_policy":null,"job_id":"job-1607","job_issued_at_ms":999985,"miner_id":"miner-1","network_difficulty":1000009,"ntime":1700001607,"order_key":"01:miner-1","p2mr_program_hex":"4747474747474747474747474747474747474747474747474747474747474747","share_difficulty":3,"share_id":"share-1607","share_seq":1607,"template_height":800050},{"accepted_at_ms":999997,"credit_policy":null,"job_id":"job-1065","job_issued_at_ms":999971,"miner_id":"miner-9","network_difficulty":1000011,"ntime":1700001065,"order_key":"09:miner-9","p2mr_program_hex":"2929292929292929292929292929292929292929292929292929292929292929","share_difficulty":1,"share_id":"share-1065","share_seq":1065,"template_height":800033},{"accepted_at_ms":999997,"credit_policy":null,"job_id":"job-3544","job_issued_at_ms":999997,"miner_id":"miner-2","network_difficulty":1000008,"ntime":1700003544,"order_key":"02:miner-2","p2mr_program_hex":"d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8","share_difficulty":3,"share_id":"share-3544","share_seq":3544,"template_height":800110},{"accepted_at_ms":999992,"credit_policy":null,"job_id":"job-2483","job_issued_at_ms":999981,"miner_id":"miner-8","network_difficulty":1000001,"ntime":1700002483,"order_key":"08:miner-8","p2mr_program_hex":"b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3","share_difficulty":3,"share_id":"share-2483","share_seq":2483,"template_height":800077},{"accepted_at_ms":999970,"credit_policy":null,"job_id":"job-2384","job_issued_at_ms":999971,"miner_id":"miner-8","network_difficulty":1000004,"ntime":1700002384,"order_key":"08:miner-8","p2mr_program_hex":"5050505050505050505050505050505050505050505050505050505050505050","share_difficulty":2,"share_id":"share-2384","share_seq":2384,"template_height":800074},{"accepted_at_ms":1000150,"credit_policy":null,"job_id":"job-2573","job_issued_at_ms":1000173,"miner_id":"miner-10","network_difficulty":1000006,"ntime":1700002573,"order_key":"10:miner-10","p2mr_program_hex":"0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d","share_difficulty":1000,"share_id":"share-2573","share_seq":2573,"template_height":800080},{"accepted_at_ms":999978,"credit_policy":null,"job_id":"job-2414","job_issued_at_ms":999988,"miner_id":"miner-5","network_difficulty":1000000,"ntime":1700002414,"order_key":"05:miner-5","p2mr_program_hex":"6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e","share_difficulty":7,"share_id":"share-2414","share_seq":2414,"template_height":800075},{"accepted_at_ms":999980,"credit_policy":null,"job_id":"job-439","job_issued_at_ms":999990,"miner_id":"miner-10","network_difficulty":1000014,"ntime":1700000439,"order_key":"10:miner-10","p2mr_program_hex":"b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7","share_difficulty":1000,"share_id":"share-439","share_seq":439,"template_height":800013},{"accepted_at_ms":999981,"credit_policy":null,"job_id":"job-2597","job_issued_at_ms":999986,"miner_id":"miner-1","network_difficulty":1000013,"ntime":1700002597,"order_key":"01:miner-1","p2mr_program_hex":"2525252525252525252525252525252525252525252525252525252525252525","share_difficulty":19,"share_id":"share-2597","share_seq":2597,"template_height":800081},{"accepted_at_ms":999986,"credit_policy":null,"job_id":"job-943","job_issued_at_ms":999977,"miner_id":"miner-8","network_difficulty":1000008,"ntime":1700000943,"order_key":"08:miner-8","p2mr_program_hex":"afafafafafafafafafafafafafafafafafafafafafafafafafafafafafafafaf","share_difficulty":3,"share_id":"share-943","share_seq":943,"template_height":800029},{"accepted_at_ms":999991,"credit_policy":null,"job_id":"job-1421","job_issued_at_ms":999992,"miner_id":"miner-2","network_difficulty":1000010,"ntime":1700001421,"order_key":"02:miner-2","p2mr_program_hex":"8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d","share_difficulty":101,"share_id":"share-1421","share_seq":1421,"template_height":800044},{"accepted_at_ms":1000449,"credit_policy":null,"job_id":"job-1145","job_issued_at_ms":999988,"miner_id":"miner-1","network_difficulty":1000006,"ntime":1700001145,"order_key":"01:miner-1","p2mr_program_hex":"7979797979797979797979797979797979797979797979797979797979797979","share_difficulty":101,"share_id":"share-1145","share_seq":1145,"template_height":800035},{"accepted_at_ms":999995,"credit_policy":null,"job_id":"job-1918","job_issued_at_ms":999970,"miner_id":"miner-4","network_difficulty":1000014,"ntime":1700001918,"order_key":"04:miner-4","p2mr_program_hex":"7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e","share_difficulty":1000,"share_id":"share-1918","share_seq":1918,"template_height":800059},{"accepted_at_ms":999999,"credit_policy":null,"job_id":"job-201","job_issued_at_ms":999984,"miner_id":"miner-3","network_difficulty":1000014,"ntime":1700000201,"order_key":"03:miner-3","p2mr_program_hex":"c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9","share_difficulty":2,"share_id":"share-201","share_seq":201,"template_height":800006},{"accepted_at_ms":1000165,"credit_policy":null,"job_id":"job-528","job_issued_at_ms":999992,"miner_id":"miner-0","network_difficulty":1000001,"ntime":1700000528,"order_key":"00:miner-0","p2mr_program_hex":"1010101010101010101010101010101010101010101010101010101010101010","share_difficulty":1000,"share_id":"share-528","share_seq":528,"template_height":800016},{"accepted_at_ms":999984,"credit_policy":null,"job_id":"job-1902","job_issued_at_ms":999975,"miner_id":"miner-10","network_difficulty":1000015,"ntime":1700001902,"order_key":"10:miner-10","p2mr_program_hex":"6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e","share_difficulty":1000,"share_id":"share-1902","share_seq":1902,"template_height":800059},{"accepted_at_ms":999994,"credit_policy":"stale-grace","job_id":"job-3453","job_issued_at_ms":999994,"miner_id":"miner-10","network_difficulty":1000002,"ntime":1700003453,"order_key":"10:miner-10","p2mr_program_hex":"7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d","share_difficulty":2,"share_id":"share-3453","share_seq":3453,"template_height":800107},{"accepted_at_ms":999983,"credit_policy":null,"job_id":"job-3109","job_issued_at_ms":999980,"miner_id":"miner-7","network_difficulty":1000015,"ntime":1700003109,"order_key":"07:miner-7","p2mr_program_hex":"2525252525252525252525252525252525252525252525252525252525252525","share_difficulty":3,"share_id":"share-3109","share_seq":3109,"template_height":800097},{"accepted_at_ms":999976,"credit_policy":null,"job_id":"job-1653","job_issued_at_ms":999979,"miner_id":"miner-3","network_difficulty":1000004,"ntime":1700001653,"order_key":"03:miner-3","p2mr_program_hex":"7575757575757575757575757575757575757575757575757575757575757575","share_difficulty":7,"share_id":"share-1653","share_seq":1653,"template_height":800051},{"accepted_at_ms":999973,"credit_policy":null,"job_id":"job-3304","job_issued_at_ms":999982,"miner_id":"miner-4","network_difficulty":1000006,"ntime":1700003304,"order_key":"04:miner-4","p2mr_program_hex":"e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8","share_difficulty":19,"share_id":"share-3304","share_seq":3304,"template_height":800103},{"accepted_at_ms":999985,"credit_policy":null,"job_id":"job-1020","job_issued_at_ms":999995,"miner_id":"miner-8","network_difficulty":1000000,"ntime":1700001020,"order_key":"08:miner-8","p2mr_program_hex":"fcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfc","share_difficulty":101,"share_id":"share-1020","share_seq":1020,"template_height":800031},{"accepted_at_ms":999973,"credit_policy":null,"job_id":"job-2804","job_issued_at_ms":999976,"miner_id":"miner-10","network_difficulty":1000016,"ntime":1700002804,"order_key":"10:miner-10","p2mr_program_hex":"f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4","share_difficulty":1000,"share_id":"share-2804","share_seq":2804,"template_height":800087},{"accepted_at_ms":999972,"credit_policy":null,"job_id":"job-831","job_issued_at_ms":999975,"miner_id":"miner-6","network_difficulty":1000015,"ntime":1700000831,"order_key":"06:miner-6","p2mr_program_hex":"3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f","share_difficulty":1,"share_id":"share-831","share_seq":831,"template_height":800025},{"accepted_at_ms":999991,"credit_policy":null,"job_id":"job-1322","job_issued_at_ms":999981,"miner_id":"miner-2","network_difficulty":1000013,"ntime":1700001322,"order_key":"02:miner-2","p2mr_program_hex":"2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a","share_difficulty":19,"share_id":"share-1322","share_seq":1322,"template_height":800041},{"accepted_at_ms":999970,"credit_policy":"stale-grace","job_id":"job-2524","job_issued_at_ms":999985,"miner_id":"miner-5","network_difficulty":1000008,"ntime":1700002524,"order_key":"05:miner-5","p2mr_program_hex":"dcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdc","share_difficulty":1,"share_id":"share-2524","share_seq":2524,"template_height":800078},{"accepted_at_ms":999986,"credit_policy":null,"job_id":"job-1894","job_issued_at_ms":999987,"miner_id":"miner-2","network_difficulty":1000007,"ntime":1700001894,"order_key":"02:miner-2","p2mr_program_hex":"6666666666666666666666666666666666666666666666666666666666666666","share_difficulty":3,"share_id":"share-1894","share_seq":1894,"template_height":800059},{"accepted_at_ms":999995,"credit_policy":null,"job_id":"job-295","job_issued_at_ms":999989,"miner_id":"miner-9","network_difficulty":1000006,"ntime":1700000295,"order_key":"09:miner-9","p2mr_program_hex":"2727272727272727272727272727272727272727272727272727272727272727","share_difficulty":7,"share_id":"share-295","share_seq":295,"template_height":800009},{"accepted_at_ms":999995,"credit_policy":null,"job_id":"job-1276","job_issued_at_ms":999991,"miner_id":"miner-0","network_difficulty":1000001,"ntime":1700001276,"order_key":"00:miner-0","p2mr_program_hex":"fcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfc","share_difficulty":19,"share_id":"share-1276","share_seq":1276,"template_height":800039},{"accepted_at_ms":999998,"credit_policy":"stale-grace","job_id":"job-2241","job_issued_at_ms":999999,"miner_id":"miner-8","network_difficulty":1000014,"ntime":1700002241,"order_key":"08:miner-8","p2mr_program_hex":"c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1","share_difficulty":3,"share_id":"share-2241","share_seq":2241,"template_height":800070},{"accepted_at_ms":999996,"credit_policy":null,"job_id":"job-804","job_issued_at_ms":999986,"miner_id":"miner-1","network_difficulty":1000005,"ntime":1700000804,"order_key":"01:miner-1","p2mr_program_hex":"2424242424242424242424242424242424242424242424242424242424242424","share_difficulty":1000,"share_id":"share-804","share_seq":804,"template_height":800025},{"accepted_at_ms":999996,"credit_policy":null,"job_id":"job-1694","job_issued_at_ms":999985,"miner_id":"miner-0","network_difficulty":1000011,"ntime":1700001694,"order_key":"00:miner-0","p2mr_program_hex":"9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e","share_difficulty":1000,"share_id":"share-1694","share_seq":1694,"template_height":800052},{"accepted_at_ms":999986,"credit_policy":"stale-grace","job_id":"job-2290","job_issued_at_ms":999973,"miner_id":"miner-2","network_difficulty":1000012,"ntime":1700002290,"order_key":"02:miner-2","p2mr_program_hex":"f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2","share_difficulty":1,"share_id":"share-2290","share_seq":2290,"template_height":800071},{"accepted_at_ms":999997,"credit_policy":null,"job_id":"job-1545","job_issued_at_ms":999978,"miner_id":"miner-5","network_difficulty":1000015,"ntime":1700001545,"order_key":"05:miner-5","p2mr_program_hex":"0909090909090909090909090909090909090909090909090909090909090909","share_difficulty":3,"share_id":"share-1545","share_seq":1545,"template_height":800048},{"accepted_at_ms":999976,"credit_policy":null,"job_id":"job-94","job_issued_at_ms":999979,"miner_id":"miner-6","network_difficulty":1000009,"ntime":1700000094,"order_key":"06:miner-6","p2mr_program_hex":"5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e","share_difficulty":19,"share_id":"share-94","share_seq":94,"template_height":800002},{"accepted_at_ms":999970,"credit_policy":null,"job_id":"job-719","job_issued_at_ms":999996,"miner_id":"miner-4","network_difficulty":1000005,"ntime":1700000719,"order_key":"04:miner-4","p2mr_program_hex":"cfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcf","share_difficulty":7,"share_id":"share-719","share_seq":719,"template_height":800022},{"accepted_at_ms":999987,"credit_policy":null,"job_id":"job-276","job_issued_at_ms":999973,"miner_id":"miner-1","network_difficulty":1000004,"ntime":1700000276,"order_key":"01:miner-1","p2mr_program_hex":"1414141414141414141414141414141414141414141414141414141414141414","share_difficulty":19,"share_id":"share-276","share_seq":276,"template_height":800008},{"accepted_at_ms":1000397,"credit_policy":null,"job_id":"job-1461","job_issued_at_ms":1000055,"miner_id":"miner-9","network_difficulty":1000016,"ntime":1700001461,"order_key":"09:miner-9","p2mr_program_hex":"b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5","share_difficulty":101,"share_id":"share-1461","share_seq":1461,"template_height":800045},{"accepted_at_ms":999971,"credit_policy":null,"job_id":"job-2245","job_issued_at_ms":1000000,"miner_id":"miner-1","network_difficulty":1000001,"ntime":1700002245,"order_key":"01:miner-1","p2mr_program_hex":"c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5","share_difficulty":7,"share_id":"share-2245","share_seq":2245,"template_height":800070},{"accepted_at_ms":999992,"credit_policy":null,"job_id":"job-3317","job_issued_at_ms":999997,"miner_id":"miner-6","network_difficulty":1000002,"ntime":1700003317,"order_key":"06:miner-6","p2mr_program_hex":"f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5","share_difficulty":7,"share_id":"share-3317","share_seq":3317,"template_height":800103},{"accepted_at_ms":999981,"credit_policy":null,"job_id":"job-3761","job_issued_at_ms":999972,"miner_id":"miner-10","network_difficulty":1000004,"ntime":1700003761,"order_key":"10:miner-10","p2mr_program_hex":"b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1","share_difficulty":1,"share_id":"share-3761","share_seq":3761,"template_height":800117},{"accepted_at_ms":999979,"credit_policy":null,"job_id":"job-1218","job_issued_at_ms":999979,"miner_id":"miner-8","network_difficulty":1000011,"ntime":1700001218,"order_key":"08:miner-8","p2mr_program_hex":"c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2","share_difficulty":19,"share_id":"share-1218","share_seq":1218,"template_height":800038},{"accepted_at_ms":999970,"credit_policy":null,"job_id":"job-1537","job_issued_at_ms":999976,"miner_id":"miner-8","network_difficulty":1000007,"ntime":1700001537,"order_key":"08:miner-8","p2mr_program_hex":"0101010101010101010101010101010101010101010101010101010101010101","share_difficulty":19,"share_id":"share-1537","share_seq":1537,"template_height":800048},{"accepted_at_ms":999988,"credit_policy":null,"job_id":"job-347","job_issued_at_ms":999981,"miner_id":"miner-6","network_difficulty":1000007,"ntime":1700000347,"order_key":"06:miner-6","p2mr_program_hex":"5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b","share_difficulty":3,"share_id":"share-347","share_seq":347,"template_height":800010},{"accepted_at_ms":999979,"credit_policy":"stale-grace","job_id":"job-896","job_issued_at_ms":999979,"miner_id":"miner-5","network_difficulty":1000012,"ntime":1700000896,"order_key":"05:miner-5","p2mr_program_hex":"8080808080808080808080808080808080808080808080808080808080808080","share_difficulty":1,"share_id":"share-896","share_seq":896,"template_height":800028},{"accepted_at_ms":999996,"credit_policy":null,"job_id":"job-3593","job_issued_at_ms":999989,"miner_id":"miner-7","network_difficulty":1000006,"ntime":1700003593,"order_key":"07:miner-7","p2mr_program_hex":"0909090909090909090909090909090909090909090909090909090909090909","share_difficulty":101,"share_id":"share-3593","share_seq":3593,"template_height":800112},{"accepted_at_ms":999993,"credit_policy":null,"job_id":"job-163","job_issued_at_ms":999998,"miner_id":"miner-9","network_difficulty":1000010,"ntime":1700000163,"order_key":"09:miner-9","p2mr_program_hex":"a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3","share_difficulty":2,"share_id":"share-163","share_seq":163,"template_height":800005},{"accepted_at_ms":999985,"credit_policy":null,"job_id":"job-55","job_issued_at_ms":999990,"miner_id":"miner-0","network_difficulty":1000004,"ntime":1700000055,"order_key":"00:miner-0","p2mr_program_hex":"3737373737373737373737373737373737373737373737373737373737373737","share_difficulty":19,"share_id":"share-55","share_seq":55,"template_height":800001},{"accepted_at_ms":999986,"credit_policy":null,"job_id":"job-1937","job_issued_at_ms":999976,"miner_id":"miner-1","network_difficulty":1000016,"ntime":1700001937,"order_key":"01:miner-1","p2mr_program_hex":"9191919191919191919191919191919191919191919191919191919191919191","share_difficulty":2,"share_id":"share-1937","share_seq":1937,"template_height":800060},{"accepted_at_ms":999980,"credit_policy":null,"job_id":"job-3037","job_issued_at_ms":999984,"miner_id":"miner-1","network_difficulty":1000011,"ntime":1700003037,"order_key":"01:miner-1","p2mr_program_hex":"dddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddd","share_difficulty":101,"share_id":"share-3037","share_seq":3037,"template_height":800094},{"accepted_at_ms":999978,"credit_policy":null,"job_id":"job-3519","job_issued_at_ms":999972,"miner_id":"miner-10","network_difficulty":1000000,"ntime":1700003519,"order_key":"10:miner-10","p2mr_program_hex":"bfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbf","share_difficulty":2,"share_id":"share-3519","share_seq":3519,"template_height":800109},{"accepted_at_ms":999974,"credit_policy":null,"job_id":"job-1149","job_issued_at_ms":999994,"miner_id":"miner-5","network_difficulty":1000010,"ntime":1700001149,"order_key":"05:miner-5","p2mr_program_hex":"7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d","share_difficulty":1,"share_id":"share-1149","share_seq":1149,"template_height":800035},{"accepted_at_ms":999989,"credit_policy":"stale-grace","job_id":"job-1735","job_issued_at_ms":999996,"miner_id":"miner-8","network_difficulty":1000001,"ntime":1700001735,"order_key":"08:miner-8","p2mr_program_hex":"c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7","share_difficulty":1000,"share_id":"share-1735","share_seq":1735,"template_height":800054},{"accepted_at_ms":999989,"credit_policy":null,"job_id":"job-935","job_issued_at_ms":999989,"miner_id":"miner-0","network_difficulty":1000000,"ntime":1700000935,"order_key":"00:miner-0","p2mr_program_hex":"a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7","share_difficulty":3,"share_id":"share-935","share_seq":935,"template_height":800029},{"accepted_at_ms":999996,"credit_policy":"stale-grace","job_id":"job-192","job_issued_at_ms":999990,"miner_id":"miner-5","network_difficulty":1000005,"ntime":1700000192,"order_key":"05:miner-5","p2mr_program_hex":"c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0","share_difficulty":7,"share_id":"share-192","share_seq":192,"template_height":800006},{"accepted_at_ms":1000153,"credit_policy":"stale-grace","job_id":"job-190","job_issued_at_ms":999984,"miner_id":"miner-3","network_difficulty":1000003,"ntime":1700000190,"order_key":"03:miner-3","p2mr_program_hex":"bebebebebebebebebebebebebebebebebebebebebebebebebebebebebebebebe","share_difficulty":7,"share_id":"share-190","share_seq":190,"template_height":800005},{"accepted_at_ms":999987,"credit_policy":null,"job_id":"job-2943","job_issued_at_ms":999987,"miner_id":"miner-6","network_difficulty":1000002,"ntime":1700002943,"order_key":"06:miner-6","p2mr_program_hex":"7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f","share_difficulty":101,"share_id":"share-2943","share_seq":2943,"template_height":800091},{"accepted_at_ms":999983,"credit_policy":"stale-grace","job_id":"job-2064","job_issued_at_ms":999999,"miner_id":"miner-7","network_difficulty":1000007,"ntime":1700002064,"order_key":"07:miner-7","p2mr_program_hex":"1010101010101010101010101010101010101010101010101010101010101010","share_difficulty":7,"share_id":"share-2064","share_seq":2064,"template_height":800064},{"accepted_at_ms":999996,"credit_policy":null,"job_id":"job-2588","job_issued_at_ms":999998,"miner_id":"miner-3","network_difficulty":1000004,"ntime":1700002588,"order_key":"03:miner-3","p2mr_program_hex":"1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c","share_difficulty":19,"share_id":"share-2588","share_seq":2588,"template_height":800080},{"accepted_at_ms":999997,"credit_policy":"stale-grace","job_id":"job-2887","job_issued_at_ms":999974,"miner_id":"miner-5","network_difficulty":1000014,"ntime":1700002887,"order_key":"05:miner-5","p2mr_program_hex":"4747474747474747474747474747474747474747474747474747474747474747","share_difficulty":101,"share_id":"share-2887","share_seq":2887,"template_height":800090},{"accepted_at_ms":999999,"credit_policy":null,"job_id":"job-3508","job_issued_at_ms":999994,"miner_id":"miner-10","network_difficulty":1000006,"ntime":1700003508,"order_key":"10:miner-10","p2mr_program_hex":"b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4","share_difficulty":1000,"share_id":"share-3508","share_seq":3508,"template_height":800109},{"accepted_at_ms":1000000,"credit_policy":null,"job_id":"job-3622","job_issued_at_ms":999978,"miner_id":"miner-3","network_difficulty":1000001,"ntime":1700003622,"order_key":"03:miner-3","p2mr_program_hex":"2626262626262626262626262626262626262626262626262626262626262626","share_difficulty":1,"share_id":"share-3622","share_seq":3622,"template_height":800113},{"accepted_at_ms":1000000,"credit_policy":null,"job_id":"job-1723","job_issued_at_ms":999988,"miner_id":"miner-7","network_difficulty":1000006,"ntime":1700001723,"order_key":"07:miner-7","p2mr_program_hex":"bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb","share_difficulty":7,"share_id":"share-1723","share_seq":1723,"template_height":800053},{"accepted_at_ms":999974,"credit_policy":null,"job_id":"job-3198","job_issued_at_ms":999994,"miner_id":"miner-8","network_difficulty":1000002,"ntime":1700003198,"order_key":"08:miner-8","p2mr_program_hex":"7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e","share_difficulty":1000,"share_id":"share-3198","share_seq":3198,"template_height":800099},{"accepted_at_ms":999994,"credit_policy":null,"job_id":"job-2838","job_issued_at_ms":999990,"miner_id":"miner-0","network_difficulty":1000016,"ntime":1700002838,"order_key":"00:miner-0","p2mr_program_hex":"1616161616161616161616161616161616161616161616161616161616161616","share_difficulty":1,"share_id":"share-2838","share_seq":2838,"template_height":800088},{"accepted_at_ms":999984,"credit_policy":null,"job_id":"job-117","job_issued_at_ms":999998,"miner_id":"miner-7","network_difficulty":1000015,"ntime":1700000117,"order_key":"07:miner-7","p2mr_program_hex":"7575757575757575757575757575757575757575757575757575757575757575","share_difficulty":101,"share_id":"share-117","share_seq":117,"template_height":800003},{"accepted_at_ms":999998,"credit_policy":null,"job_id":"job-1658","job_issued_at_ms":999991,"miner_id":"miner-8","network_difficulty":1000009,"ntime":1700001658,"order_key":"08:miner-8","p2mr_program_hex":"7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a","share_difficulty":1000,"share_id":"share-1658","share_seq":1658,"template_height":800051},{"accepted_at_ms":999980,"credit_policy":null,"job_id":"job-3684","job_issued_at_ms":999996,"miner_id":"miner-10","network_difficulty":1000012,"ntime":1700003684,"order_key":"10:miner-10","p2mr_program_hex":"6464646464646464646464646464646464646464646464646464646464646464","share_difficulty":3,"share_id":"share-3684","share_seq":3684,"template_height":800115},{"accepted_at_ms":999986,"credit_policy":null,"job_id":"job-2950","job_issued_at_ms":999988,"miner_id":"miner-2","network_difficulty":1000009,"ntime":1700002950,"order_key":"02:miner-2","p2mr_program_hex":"8686868686868686868686868686868686868686868686868686868686868686","share_difficulty":19,"share_id":"share-2950","share_seq":2950,"template_height":800092},{"accepted_at_ms":999989,"credit_policy":"stale-grace","job_id":"job-2273","job_issued_at_ms":1000173,"miner_id":"miner-7","network_difficulty":1000012,"ntime":1700002273,"order_key":"07:miner-7","p2mr_program_hex":"e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1","share_difficulty":1,"share_id":"share-2273","share_seq":2273,"template_height":800071},{"accepted_at_ms":999987,"credit_policy":null,"job_id":"job-1414","job_issued_at_ms":999993,"miner_id":"miner-6","network_difficulty":1000003,"ntime":1700001414,"order_key":"06:miner-6","p2mr_program_hex":"8686868686868686868686868686868686868686868686868686868686868686","share_difficulty":3,"share_id":"share-1414","share_seq":1414,"template_height":800044},{"accepted_at_ms":999991,"credit_policy":null,"job_id":"job-2798","job_issued_at_ms":999977,"miner_id":"miner-4","network_difficulty":1000010,"ntime":1700002798,"order_key":"04:miner-4","p2mr_program_hex":"eeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeee","share_difficulty":2,"share_id":"share-2798","share_seq":2798,"template_height":800087},{"accepted_at_ms":999971,"credit_policy":null,"job_id":"job-2945","job_issued_at_ms":999976,"miner_id":"miner-8","network_difficulty":1000004,"ntime":1700002945,"order_key":"08:miner-8","p2mr_program_hex":"8181818181818181818181818181818181818181818181818181818181818181","share_difficulty":1000,"share_id":"share-2945","share_seq":2945,"template_height":800092},{"accepted_at_ms":999984,"credit_policy":null,"job_id":"job-152","job_issued_at_ms":1000000,"miner_id":"miner-9","network_difficulty":1000016,"ntime":1700000152,"order_key":"09:miner-9","p2mr_program_hex":"9898989898989898989898989898989898989898989898989898989898989898","share_difficulty":2,"share_id":"share-152","share_seq":152,"template_height":800004},{"accepted_at_ms":999998,"credit_policy":null,"job_id":"job-770","job_issued_at_ms":1000000,"miner_id":"miner-0","network_difficulty":1000005,"ntime":1700000770,"order_key":"00:miner-0","p2mr_program_hex":"0202020202020202020202020202020202020202020202020202020202020202","share_difficulty":101,"share_id":"share-770","share_seq":770,"template_height":800024},{"accepted_at_ms":999998,"credit_policy":null,"job_id":"job-2809","job_issued_at_ms":999995,"miner_id":"miner-4","network_difficulty":1000004,"ntime":1700002809,"order_key":"04:miner-4","p2mr_program_hex":"f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9","share_difficulty":19,"share_id":"share-2809","share_seq":2809,"template_height":800087},{"accepted_at_ms":999979,"credit_policy":"stale-grace","job_id":"job-948","job_issued_at_ms":999985,"miner_id":"miner-2","network_difficulty":1000013,"ntime":1700000948,"order_key":"02:miner-2","p2mr_program_hex":"b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4","share_difficulty":3,"share_id":"share-948","share_seq":948,"template_height":800029},{"accepted_at_ms":999990,"credit_policy":null,"job_id":"job-140","job_issued_at_ms":999981,"miner_id":"miner-8","network_difficulty":1000004,"ntime":1700000140,"order_key":"08:miner-8","p2mr_program_hex":"8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c","share_difficulty":1,"share_id":"share-140","share_seq":140,"template_height":800004},{"accepted_at_ms":999981,"credit_policy":null,"job_id":"job-632","job_issued_at_ms":999976,"miner_id":"miner-5","network_difficulty":1000003,"ntime":1700000632,"order_key":"05:miner-5","p2mr_program_hex":"7878787878787878787878787878787878787878787878787878787878787878","share_difficulty":1,"share_id":"share-632","share_seq":632,"template_height":800019},{"accepted_at_ms":999995,"credit_policy":null,"job_id":"job-2779","job_issued_at_ms":999998,"miner_id":"miner-7","network_difficulty":1000008,"ntime":1700002779,"order_key":"07:miner-7","p2mr_program_hex":"dbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdb","share_difficulty":101,"share_id":"share-2779","share_seq":2779,"template_height":800086},{"accepted_at_ms":999992,"credit_policy":"stale-grace","job_id":"job-3841","job_issued_at_ms":999973,"miner_id":"miner-2","network_difficulty":1000016,"ntime":1700003841,"order_key":"02:miner-2","p2mr_program_hex":"0101010101010101010101010101010101010101010101010101010101010101","share_difficulty":1,"share_id":"share-3841","share_seq":3841,"template_height":800120},{"accepted_at_ms":999977,"credit_policy":null,"job_id":"job-3686","job_issued_at_ms":999974,"miner_id":"miner-1","network_difficulty":1000014,"ntime":1700003686,"order_key":"01:miner-1","p2mr_program_hex":"6666666666666666666666666666666666666666666666666666666666666666","share_difficulty":7,"share_id":"share-3686","share_seq":3686,"template_height":800115},{"accepted_at_ms":999979,"credit_policy":null,"job_id":"job-2223","job_issued_at_ms":999988,"miner_id":"miner-1","network_difficulty":1000013,"ntime":1700002223,"order_key":"01:miner-1","p2mr_program_hex":"afafafafafafafafafafafafafafafafafafafafafafafafafafafafafafafaf","share_difficulty":19,"share_id":"share-2223","share_seq":2223,"template_height":800069},{"accepted_at_ms":999986,"credit_policy":null,"job_id":"job-1056","job_issued_at_ms":1000315,"miner_id":"miner-0","network_difficulty":1000002,"ntime":1700001056,"order_key":"00:miner-0","p2mr_program_hex":"2020202020202020202020202020202020202020202020202020202020202020","share_difficulty":19,"share_id":"share-1056","share_seq":1056,"template_height":800033},{"accepted_at_ms":999984,"credit_policy":null,"job_id":"job-592","job_issued_at_ms":999988,"miner_id":"miner-9","network_difficulty":1000014,"ntime":1700000592,"order_key":"09:miner-9","p2mr_program_hex":"5050505050505050505050505050505050505050505050505050505050505050","share_difficulty":101,"share_id":"share-592","share_seq":592,"template_height":800018},{"accepted_at_ms":999987,"credit_policy":null,"job_id":"job-1838","job_issued_at_ms":999977,"miner_id":"miner-1","network_difficulty":1000002,"ntime":1700001838,"order_key":"01:miner-1","p2mr_program_hex":"2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e","share_difficulty":1000,"share_id":"share-1838","share_seq":1838,"template_height":800057},{"accepted_at_ms":999989,"credit_policy":"stale-grace","job_id":"job-1663","job_issued_at_ms":999989,"miner_id":"miner-2","network_difficulty":1000014,"ntime":1700001663,"order_key":"02:miner-2","p2mr_program_hex":"7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f","share_difficulty":3,"share_id":"share-1663","share_seq":1663,"template_height":800051},{"accepted_at_ms":999976,"credit_policy":null,"job_id":"job-3750","job_issued_at_ms":999975,"miner_id":"miner-10","network_difficulty":1000010,"ntime":1700003750,"order_key":"10:miner-10","p2mr_program_hex":"a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6","share_difficulty":1,"share_id":"share-3750","share_seq":3750,"template_height":800117},{"accepted_at_ms":999981,"credit_policy":null,"job_id":"job-2540","job_issued_at_ms":999990,"miner_id":"miner-10","network_difficulty":1000007,"ntime":1700002540,"order_key":"10:miner-10","p2mr_program_hex":"ecececececececececececececececececececececececececececececececec","share_difficulty":3,"share_id":"share-2540","share_seq":2540,"template_height":800079},{"accepted_at_ms":999979,"credit_policy":"stale-grace","job_id":"job-3121","job_issued_at_ms":1000373,"miner_id":"miner-8","network_difficulty":1000010,"ntime":1700003121,"order_key":"08:miner-8","p2mr_program_hex":"3131313131313131313131313131313131313131313131313131313131313131","share_difficulty":1,"share_id":"share-3121","share_seq":3121,"template_height":800097},{"accepted_at_ms":999998,"credit_policy":"stale-grace","job_id":"job-1541","job_issued_at_ms":999973,"miner_id":"miner-1","network_difficulty":1000011,"ntime":1700001541,"order_key":"01:miner-1","p2mr_program_hex":"0505050505050505050505050505050505050505050505050505050505050505","share_difficulty":7,"share_id":"share-1541","share_seq":1541,"template_height":800048},{"accepted_at_ms":999970,"credit_policy":"stale-grace","job_id":"job-105","job_issued_at_ms":999976,"miner_id":"miner-6","network_difficulty":1000003,"ntime":1700000105,"order_key":"06:miner-6","p2mr_program_hex":"6969696969696969696969696969696969696969696969696969696969696969","share_difficulty":3,"share_id":"share-105","share_seq":105,"template_height":800003},{"accepted_at_ms":999973,"credit_policy":"stale-grace","job_id":"job-2891","job_issued_at_ms":999975,"miner_id":"miner-9","network_difficulty":1000001,"ntime":1700002891,"order_key":"09:miner-9","p2mr_program_hex":"4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b","share_difficulty":1,"share_id":"share-2891","share_seq":2891,"template_height":800090},{"accepted_at_ms":1000227,"credit_policy":null,"job_id":"job-3691","job_issued_at_ms":1000373,"miner_id":"miner-6","network_difficulty":1000002,"ntime":1700003691,"order_key":"06:miner-6","p2mr_program_hex":"6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b","share_difficulty":101,"share_id":"share-3691","share_seq":3691,"template_height":800115},{"accepted_at_ms":999985,"credit_policy":null,"job_id":"job-2784","job_issued_at_ms":999981,"miner_id":"miner-1","network_difficulty":1000013,"ntime":1700002784,"order_key":"01:miner-1","p2mr_program_hex":"e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0","share_difficulty":7,"share_id":"share-2784","share_seq":2784,"template_height":800087},{"accepted_at_ms":999976,"credit_policy":null,"job_id":"job-2018","job_issued_at_ms":999990,"miner_id":"miner-5","network_difficulty":1000012,"ntime":1700002018,"order_key":"05:miner-5","p2mr_program_hex":"e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2","share_difficulty":19,"share_id":"share-2018","share_seq":2018,"template_height":800063},{"accepted_at_ms":999998,"credit_policy":null,"job_id":"job-687","job_issued_at_ms":999991,"miner_id":"miner-5","network_difficulty":1000007,"ntime":1700000687,"order_key":"05:miner-5","p2mr_program_hex":"afafafafafafafafafafafafafafafafafafafafafafafafafafafafafafafaf","share_difficulty":1,"share_id":"share-687","share_seq":687,"template_height":800021},{"accepted_at_ms":999983,"credit_policy":null,"job_id":"job-1728","job_issued_at_ms":999978,"miner_id":"miner-1","network_difficulty":1000011,"ntime":1700001728,"order_key":"01:miner-1","p2mr_program_hex":"c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0","share_difficulty":7,"share_id":"share-1728","share_seq":1728,"template_height":800054},{"accepted_at_ms":999988,"credit_policy":null,"job_id":"job-2295","job_issued_at_ms":999984,"miner_id":"miner-7","network_difficulty":1000000,"ntime":1700002295,"order_key":"07:miner-7","p2mr_program_hex":"f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7","share_difficulty":101,"share_id":"share-2295","share_seq":2295,"template_height":800071},{"accepted_at_ms":999989,"credit_policy":null,"job_id":"job-1474","job_issued_at_ms":999974,"miner_id":"miner-0","network_difficulty":1000012,"ntime":1700001474,"order_key":"00:miner-0","p2mr_program_hex":"c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2","share_difficulty":1,"share_id":"share-1474","share_seq":1474,"template_height":800046},{"accepted_at_ms":999992,"credit_policy":null,"job_id":"job-3073","job_issued_at_ms":999994,"miner_id":"miner-4","network_difficulty":1000013,"ntime":1700003073,"order_key":"04:miner-4","p2mr_program_hex":"0101010101010101010101010101010101010101010101010101010101010101","share_difficulty":7,"share_id":"share-3073","share_seq":3073,"template_height":800096},{"accepted_at_ms":999976,"credit_policy":null,"job_id":"job-315","job_issued_at_ms":999998,"miner_id":"miner-7","network_difficulty":1000009,"ntime":1700000315,"order_key":"07:miner-7","p2mr_program_hex":"3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b","share_difficulty":1000,"share_id":"share-315","share_seq":315,"template_height":800009},{"accepted_at_ms":999972,"credit_policy":null,"job_id":"job-1495","job_issued_at_ms":999973,"miner_id":"miner-10","network_difficulty":1000016,"ntime":1700001495,"order_key":"10:miner-10","p2mr_program_hex":"d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7","share_difficulty":1000,"share_id":"share-1495","share_seq":1495,"template_height":800046},{"accepted_at_ms":999988,"credit_policy":null,"job_id":"job-1210","job_issued_at_ms":999977,"miner_id":"miner-0","network_difficulty":1000003,"ntime":1700001210,"order_key":"00:miner-0","p2mr_program_hex":"babababababababababababababababababababababababababababababababa","share_difficulty":19,"share_id":"share-1210","share_seq":1210,"template_height":800037},{"accepted_at_ms":999996,"credit_policy":null,"job_id":"job-3223","job_issued_at_ms":999997,"miner_id":"miner-0","network_difficulty":1000010,"ntime":1700003223,"order_key":"00:miner-0","p2mr_program_hex":"9797979797979797979797979797979797979797979797979797979797979797","share_difficulty":2,"share_id":"share-3223","share_seq":3223,"template_height":800100},{"accepted_at_ms":999985,"credit_policy":"stale-grace","job_id":"job-3679","job_issued_at_ms":999997,"miner_id":"miner-5","network_difficulty":1000007,"ntime":1700003679,"order_key":"05:miner-5","p2mr_program_hex":"5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f","share_difficulty":2,"share_id":"share-3679","share_seq":3679,"template_height":800114},{"accepted_at_ms":999977,"credit_policy":"stale-grace","job_id":"job-494","job_issued_at_ms":999994,"miner_id":"miner-10","network_difficulty":1000001,"ntime":1700000494,"order_key":"10:miner-10","p2mr_program_hex":"eeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeee","share_difficulty":3,"share_id":"share-494","share_seq":494,"template_height":800015},{"accepted_at_ms":999999,"credit_policy":null,"job_id":"job-3651","job_issued_at_ms":999995,"miner_id":"miner-10","network_difficulty":1000013,"ntime":1700003651,"order_key":"10:miner-10","p2mr_program_hex":"4343434343434343434343434343434343434343434343434343434343434343","share_difficulty":1000,"share_id":"share-3651","share_seq":3651,"template_height":800114},{"accepted_at_ms":999991,"credit_policy":null,"job_id":"job-1375","job_issued_at_ms":999992,"miner_id":"miner-0","network_difficulty":1000015,"ntime":1700001375,"order_key":"00:miner-0","p2mr_program_hex":"5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f","share_difficulty":3,"share_id":"share-1375","share_seq":1375,"template_height":800042},{"accepted_at_ms":999976,"credit_policy":null,"job_id":"job-1679","job_issued_at_ms":999985,"miner_id":"miner-7","network_difficulty":1000013,"ntime":1700001679,"order_key":"07:miner-7","p2mr_program_hex":"8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f","share_difficulty":101,"share_id":"share-1679","share_seq":1679,"template_height":800052},{"accepted_at_ms":999986,"credit_policy":null,"job_id":"job-2532","job_issued_at_ms":999995,"miner_id":"miner-2","network_difficulty":1000016,"ntime":1700002532,"order_key":"02:miner-2","p2mr_program_hex":"e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4","share_difficulty":101,"share_id":"share-2532","share_seq":2532,"template_height":800079},{"accepted_at_ms":999971,"credit_policy":null,"job_id":"job-594","job_issued_at_ms":999990,"miner_id":"miner-0","network_difficulty":1000016,"ntime":1700000594,"order_key":"00:miner-0","p2mr_program_hex":"5252525252525252525252525252525252525252525252525252525252525252","share_difficulty":3,"share_id":"share-594","share_seq":594,"template_height":800018},{"accepted_at_ms":999984,"credit_policy":null,"job_id":"job-3975","job_issued_at_ms":999987,"miner_id":"miner-4","network_difficulty":1000014,"ntime":1700003975,"order_key":"04:miner-4","p2mr_program_hex":"8787878787878787878787878787878787878787878787878787878787878787","share_difficulty":3,"share_id":"share-3975","share_seq":3975,"template_height":800124},{"accepted_at_ms":1000431,"credit_policy":null,"job_id":"job-2138","job_issued_at_ms":1000144,"miner_id":"miner-4","network_difficulty":1000013,"ntime":1700002138,"order_key":"04:miner-4","p2mr_program_hex":"5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a","share_difficulty":1000,"share_id":"share-2138","share_seq":2138,"template_height":800066},{"accepted_at_ms":999994,"credit_policy":"stale-grace","job_id":"job-3260","job_issued_at_ms":1000385,"miner_id":"miner-4","network_difficulty":1000013,"ntime":1700003260,"order_key":"04:miner-4","p2mr_program_hex":"bcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbc","share_difficulty":7,"share_id":"share-3260","share_seq":3260,"template_height":800101},{"accepted_at_ms":999983,"credit_policy":null,"job_id":"job-3606","job_issued_at_ms":999988,"miner_id":"miner-9","network_difficulty":1000002,"ntime":1700003606,"order_key":"09:miner-9","p2mr_program_hex":"1616161616161616161616161616161616161616161616161616161616161616","share_difficulty":2,"share_id":"share-3606","share_seq":3606,"template_height":800112},{"accepted_at_ms":1000236,"credit_policy":null,"job_id":"job-2732","job_issued_at_ms":1000316,"miner_id":"miner-4","network_difficulty":1000012,"ntime":1700002732,"order_key":"04:miner-4","p2mr_program_hex":"acacacacacacacacacacacacacacacacacacacacacacacacacacacacacacacac","share_difficulty":101,"share_id":"share-2732","share_seq":2732,"template_height":800085},{"accepted_at_ms":999995,"credit_policy":"stale-grace","job_id":"job-1148","job_issued_at_ms":999987,"miner_id":"miner-4","network_difficulty":1000009,"ntime":1700001148,"order_key":"04:miner-4","p2mr_program_hex":"7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c","share_difficulty":19,"share_id":"share-1148","share_seq":1148,"template_height":800035},{"accepted_at_ms":999996,"credit_policy":"stale-grace","job_id":"job-3408","job_issued_at_ms":999970,"miner_id":"miner-9","network_difficulty":1000008,"ntime":1700003408,"order_key":"09:miner-9","p2mr_program_hex":"5050505050505050505050505050505050505050505050505050505050505050","share_difficulty":19,"share_id":"share-3408","share_seq":3408,"template_height":800106},{"accepted_at_ms":999981,"credit_policy":null,"job_id":"job-1253","job_issued_at_ms":999971,"miner_id":"miner-10","network_difficulty":1000012,"ntime":1700001253,"order_key":"10:miner-10","p2mr_program_hex":"e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5","share_difficulty":3,"share_id":"share-1253","share_seq":1253,"template_height":800039},{"accepted_at_ms":999978,"credit_policy":"stale-grace","job_id":"job-1924","job_issued_at_ms":1000000,"miner_id":"miner-10","network_difficulty":1000003,"ntime":1700001924,"order_key":"10:miner-10","p2mr_program_hex":"8484848484848484848484848484848484848484848484848484848484848484","share_difficulty":1,"share_id":"share-1924","share_seq":1924,"template_height":800060},{"accepted_at_ms":999973,"credit_policy":null,"job_id":"job-1009","job_issued_at_ms":999994,"miner_id":"miner-8","network_difficulty":1000006,"ntime":1700001009,"order_key":"08:miner-8","p2mr_program_hex":"f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1","share_difficulty":3,"share_id":"share-1009","share_seq":1009,"template_height":800031},{"accepted_at_ms":999970,"credit_policy":null,"job_id":"job-748","job_issued_at_ms":999987,"miner_id":"miner-0","network_difficulty":1000000,"ntime":1700000748,"order_key":"00:miner-0","p2mr_program_hex":"ecececececececececececececececececececececececececececececececec","share_difficulty":1,"share_id":"share-748","share_seq":748,"template_height":800023},{"accepted_at_ms":999994,"credit_policy":null,"job_id":"job-3327","job_issued_at_ms":1000413,"miner_id":"miner-5","network_difficulty":1000012,"ntime":1700003327,"order_key":"05:miner-5","p2mr_program_hex":"ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff","share_difficulty":3,"share_id":"share-3327","share_seq":3327,"template_height":800103},{"accepted_at_ms":1000000,"credit_policy":null,"job_id":"job-183","job_issued_at_ms":999989,"miner_id":"miner-7","network_difficulty":1000013,"ntime":1700000183,"order_key":"07:miner-7","p2mr_program_hex":"b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7","share_difficulty":2,"share_id":"share-183","share_seq":183,"template_height":800005},{"accepted_at_ms":999971,"credit_policy":null,"job_id":"job-2898","job_issued_at_ms":999998,"miner_id":"miner-5","network_difficulty":1000008,"ntime":1700002898,"order_key":"05:miner-5","p2mr_program_hex":"5252525252525252525252525252525252525252525252525252525252525252","share_difficulty":1000,"share_id":"share-2898","share_seq":2898,"template_height":800090},{"accepted_at_ms":999982,"credit_policy":null,"job_id":"job-3271","job_issued_at_ms":999979,"miner_id":"miner-4","network_difficulty":1000007,"ntime":1700003271,"order_key":"04:miner-4","p2mr_program_hex":"c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7","share_difficulty":7,"share_id":"share-3271","share_seq":3271,"template_height":800102},{"accepted_at_ms":999979,"credit_policy":null,"job_id":"job-168","job_issued_at_ms":999970,"miner_id":"miner-3","network_difficulty":1000015,"ntime":1700000168,"order_key":"03:miner-3","p2mr_program_hex":"a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8","share_difficulty":2,"share_id":"share-168","share_seq":168,"template_height":800005},{"accepted_at_ms":999993,"credit_policy":null,"job_id":"job-1634","job_issued_at_ms":999990,"miner_id":"miner-6","network_difficulty":1000002,"ntime":1700001634,"order_key":"06:miner-6","p2mr_program_hex":"6262626262626262626262626262626262626262626262626262626262626262","share_difficulty":2,"share_id":"share-1634","share_seq":1634,"template_height":800051},{"accepted_at_ms":999985,"credit_policy":null,"job_id":"job-367","job_issued_at_ms":999970,"miner_id":"miner-4","network_difficulty":1000010,"ntime":1700000367,"order_key":"04:miner-4","p2mr_program_hex":"6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f","share_difficulty":3,"share_id":"share-367","share_seq":367,"template_height":800011},{"accepted_at_ms":999987,"credit_policy":null,"job_id":"job-1425","job_issued_at_ms":999983,"miner_id":"miner-6","network_difficulty":1000014,"ntime":1700001425,"order_key":"06:miner-6","p2mr_program_hex":"9191919191919191919191919191919191919191919191919191919191919191","share_difficulty":2,"share_id":"share-1425","share_seq":1425,"template_height":800044},{"accepted_at_ms":1000000,"credit_policy":"stale-grace","job_id":"job-2035","job_issued_at_ms":999978,"miner_id":"miner-0","network_difficulty":1000012,"ntime":1700002035,"order_key":"00:miner-0","p2mr_program_hex":"f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3","share_difficulty":1,"share_id":"share-2035","share_seq":2035,"template_height":800063},{"accepted_at_ms":999987,"credit_policy":null,"job_id":"job-2759","job_issued_at_ms":999997,"miner_id":"miner-9","network_difficulty":1000005,"ntime":1700002759,"order_key":"09:miner-9","p2mr_program_hex":"c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7","share_difficulty":3,"share_id":"share-2759","share_seq":2759,"template_height":800086},{"accepted_at_ms":999999,"credit_policy":null,"job_id":"job-2606","job_issued_at_ms":999972,"miner_id":"miner-10","network_difficulty":1000005,"ntime":1700002606,"order_key":"10:miner-10","p2mr_program_hex":"2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e","share_difficulty":101,"share_id":"share-2606","share_seq":2606,"template_height":800081},{"accepted_at_ms":999986,"credit_policy":null,"job_id":"job-102","job_issued_at_ms":999988,"miner_id":"miner-3","network_difficulty":1000000,"ntime":1700000102,"order_key":"03:miner-3","p2mr_program_hex":"6666666666666666666666666666666666666666666666666666666666666666","share_difficulty":3,"share_id":"share-102","share_seq":102,"template_height":800003},{"accepted_at_ms":999982,"credit_policy":null,"job_id":"job-3228","job_issued_at_ms":999981,"miner_id":"miner-5","network_difficulty":1000015,"ntime":1700003228,"order_key":"05:miner-5","p2mr_program_hex":"9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c","share_difficulty":1000,"share_id":"share-3228","share_seq":3228,"template_height":800100},{"accepted_at_ms":1000000,"credit_policy":null,"job_id":"job-2825","job_issued_at_ms":999992,"miner_id":"miner-9","network_difficulty":1000003,"ntime":1700002825,"order_key":"09:miner-9","p2mr_program_hex":"0909090909090909090909090909090909090909090909090909090909090909","share_difficulty":1000,"share_id":"share-2825","share_seq":2825,"template_height":800088},{"accepted_at_ms":999978,"credit_policy":null,"job_id":"job-2116","job_issued_at_ms":999993,"miner_id":"miner-4","network_difficulty":1000008,"ntime":1700002116,"order_key":"04:miner-4","p2mr_program_hex":"4444444444444444444444444444444444444444444444444444444444444444","share_difficulty":1,"share_id":"share-2116","share_seq":2116,"template_height":800066},{"accepted_at_ms":999989,"credit_policy":null,"job_id":"job-2065","job_issued_at_ms":999970,"miner_id":"miner-8","network_difficulty":1000008,"ntime":1700002065,"order_key":"08:miner-8","p2mr_program_hex":"1111111111111111111111111111111111111111111111111111111111111111","share_difficulty":1,"share_id":"share-2065","share_seq":2065,"template_height":800064},{"accepted_at_ms":999990,"credit_policy":"stale-grace","job_id":"job-431","job_issued_at_ms":999988,"miner_id":"miner-2","network_difficulty":1000006,"ntime":1700000431,"order_key":"02:miner-2","p2mr_program_hex":"afafafafafafafafafafafafafafafafafafafafafafafafafafafafafafafaf","share_difficulty":1,"share_id":"share-431","share_seq":431,"template_height":800013},{"accepted_at_ms":999972,"credit_policy":null,"job_id":"job-1179","job_issued_at_ms":999971,"miner_id":"miner-2","network_difficulty":1000006,"ntime":1700001179,"order_key":"02:miner-2","p2mr_program_hex":"9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b","share_difficulty":19,"share_id":"share-1179","share_seq":1179,"template_height":800036},{"accepted_at_ms":999989,"credit_policy":null,"job_id":"job-2513","job_issued_at_ms":999975,"miner_id":"miner-5","network_difficulty":1000014,"ntime":1700002513,"order_key":"05:miner-5","p2mr_program_hex":"d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1","share_difficulty":7,"share_id":"share-2513","share_seq":2513,"template_height":800078},{"accepted_at_ms":999995,"credit_policy":null,"job_id":"job-665","job_issued_at_ms":999993,"miner_id":"miner-5","network_difficulty":1000002,"ntime":1700000665,"order_key":"05:miner-5","p2mr_program_hex":"9999999999999999999999999999999999999999999999999999999999999999","share_difficulty":2,"share_id":"share-665","share_seq":665,"template_height":800020},{"accepted_at_ms":999986,"credit_policy":null,"job_id":"job-567","job_issued_at_ms":999971,"miner_id":"miner-6","network_difficulty":1000006,"ntime":1700000567,"order_key":"06:miner-6","p2mr_program_hex":"3737373737373737373737373737373737373737373737373737373737373737","share_difficulty":19,"share_id":"share-567","share_seq":567,"template_height":800017},{"accepted_at_ms":999977,"credit_policy":null,"job_id":"job-77","job_issued_at_ms":999979,"miner_id":"miner-0","network_difficulty":1000009,"ntime":1700000077,"order_key":"00:miner-0","p2mr_program_hex":"4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d","share_difficulty":1,"share_id":"share-77","share_seq":77,"template_height":800002},{"accepted_at_ms":999990,"credit_policy":"stale-grace","job_id":"job-282","job_issued_at_ms":999991,"miner_id":"miner-7","network_difficulty":1000010,"ntime":1700000282,"order_key":"07:miner-7","p2mr_program_hex":"1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a","share_difficulty":3,"share_id":"share-282","share_seq":282,"template_height":800008},{"accepted_at_ms":999980,"credit_policy":null,"job_id":"job-294","job_issued_at_ms":999980,"miner_id":"miner-8","network_difficulty":1000005,"ntime":1700000294,"order_key":"08:miner-8","p2mr_program_hex":"2626262626262626262626262626262626262626262626262626262626262626","share_difficulty":19,"share_id":"share-294","share_seq":294,"template_height":800009},{"accepted_at_ms":1000458,"credit_policy":null,"job_id":"job-745","job_issued_at_ms":1000487,"miner_id":"miner-8","network_difficulty":1000014,"ntime":1700000745,"order_key":"08:miner-8","p2mr_program_hex":"e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9","share_difficulty":3,"share_id":"share-745","share_seq":745,"template_height":800023},{"accepted_at_ms":999997,"credit_policy":null,"job_id":"job-836","job_issued_at_ms":999985,"miner_id":"miner-0","network_difficulty":1000003,"ntime":1700000836,"order_key":"00:miner-0","p2mr_program_hex":"4444444444444444444444444444444444444444444444444444444444444444","share_difficulty":19,"share_id":"share-836","share_seq":836,"template_height":800026},{"accepted_at_ms":999986,"credit_policy":"stale-grace","job_id":"job-2753","job_issued_at_ms":999971,"miner_id":"miner-3","network_difficulty":1000016,"ntime":1700002753,"order_key":"03:miner-3","p2mr_program_hex":"c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1","share_difficulty":1,"share_id":"share-2753","share_seq":2753,"template_height":800086},{"accepted_at_ms":999975,"credit_policy":null,"job_id":"job-2816","job_issued_at_ms":999975,"miner_id":"miner-0","network_difficulty":1000011,"ntime":1700002816,"order_key":"00:miner-0","p2mr_program_hex":"0000000000000000000000000000000000000000000000000000000000000000","share_difficulty":7,"share_id":"share-2816","share_seq":2816,"template_height":800088},{"accepted_at_ms":999987,"credit_policy":"stale-grace","job_id":"job-1640","job_issued_at_ms":999990,"miner_id":"miner-1","network_difficulty":1000008,"ntime":1700001640,"order_key":"01:miner-1","p2mr_program_hex":"6868686868686868686868686868686868686868686868686868686868686868","share_difficulty":3,"share_id":"share-1640","share_seq":1640,"template_height":800051},{"accepted_at_ms":999972,"credit_policy":"stale-grace","job_id":"job-563","job_issued_at_ms":999974,"miner_id":"miner-2","network_difficulty":1000002,"ntime":1700000563,"order_key":"02:miner-2","p2mr_program_hex":"3333333333333333333333333333333333333333333333333333333333333333","share_difficulty":1,"share_id":"share-563","share_seq":563,"template_height":800017},{"accepted_at_ms":999983,"credit_policy":null,"job_id":"job-2845","job_issued_at_ms":1000194,"miner_id":"miner-7","network_difficulty":1000006,"ntime":1700002845,"order_key":"07:miner-7","p2mr_program_hex":"1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d","share_difficulty":7,"share_id":"share-2845","share_seq":2845,"template_height":800088},{"accepted_at_ms":999981,"credit_policy":null,"job_id":"job-2924","job_issued_at_ms":999972,"miner_id":"miner-9","network_difficulty":1000000,"ntime":1700002924,"order_key":"09:miner-9","p2mr_program_hex":"6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c","share_difficulty":3,"share_id":"share-2924","share_seq":2924,"template_height":800091},{"accepted_at_ms":999987,"credit_policy":null,"job_id":"job-1196","job_issued_at_ms":999972,"miner_id":"miner-8","network_difficulty":1000006,"ntime":1700001196,"order_key":"08:miner-8","p2mr_program_hex":"acacacacacacacacacacacacacacacacacacacacacacacacacacacacacacacac","share_difficulty":3,"share_id":"share-1196","share_seq":1196,"template_height":800037},{"accepted_at_ms":999984,"credit_policy":null,"job_id":"job-3021","job_issued_at_ms":999985,"miner_id":"miner-7","network_difficulty":1000012,"ntime":1700003021,"order_key":"07:miner-7","p2mr_program_hex":"cdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcd","share_difficulty":19,"share_id":"share-3021","share_seq":3021,"template_height":800094},{"accepted_at_ms":999992,"credit_policy":null,"job_id":"job-1729","job_issued_at_ms":999974,"miner_id":"miner-2","network_difficulty":1000012,"ntime":1700001729,"order_key":"02:miner-2","p2mr_program_hex":"c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1","share_difficulty":7,"share_id":"share-1729","share_seq":1729,"template_height":800054},{"accepted_at_ms":999988,"credit_policy":null,"job_id":"job-1593","job_issued_at_ms":999980,"miner_id":"miner-9","network_difficulty":1000012,"ntime":1700001593,"order_key":"09:miner-9","p2mr_program_hex":"3939393939393939393939393939393939393939393939393939393939393939","share_difficulty":3,"share_id":"share-1593","share_seq":1593,"template_height":800049},{"accepted_at_ms":999997,"credit_policy":null,"job_id":"job-1349","job_issued_at_ms":999986,"miner_id":"miner-7","network_difficulty":1000006,"ntime":1700001349,"order_key":"07:miner-7","p2mr_program_hex":"4545454545454545454545454545454545454545454545454545454545454545","share_difficulty":7,"share_id":"share-1349","share_seq":1349,"template_height":800042},{"accepted_at_ms":999970,"credit_policy":null,"job_id":"job-1871","job_issued_at_ms":999971,"miner_id":"miner-1","network_difficulty":1000001,"ntime":1700001871,"order_key":"01:miner-1","p2mr_program_hex":"4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f","share_difficulty":101,"share_id":"share-1871","share_seq":1871,"template_height":800058},{"accepted_at_ms":999983,"credit_policy":null,"job_id":"job-3079","job_issued_at_ms":999996,"miner_id":"miner-10","network_difficulty":1000002,"ntime":1700003079,"order_key":"10:miner-10","p2mr_program_hex":"0707070707070707070707070707070707070707070707070707070707070707","share_difficulty":19,"share_id":"share-3079","share_seq":3079,"template_height":800096},{"accepted_at_ms":999972,"credit_policy":null,"job_id":"job-1891","job_issued_at_ms":999988,"miner_id":"miner-10","network_difficulty":1000004,"ntime":1700001891,"order_key":"10:miner-10","p2mr_program_hex":"6363636363636363636363636363636363636363636363636363636363636363","share_difficulty":1000,"share_id":"share-1891","share_seq":1891,"template_height":800059},{"accepted_at_ms":999991,"credit_policy":null,"job_id":"job-930","job_issued_at_ms":999972,"miner_id":"miner-6","network_difficulty":1000012,"ntime":1700000930,"order_key":"06:miner-6","p2mr_program_hex":"a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2","share_difficulty":1000,"share_id":"share-930","share_seq":930,"template_height":800029},{"accepted_at_ms":999979,"credit_policy":null,"job_id":"job-3785","job_issued_at_ms":999973,"miner_id":"miner-1","network_difficulty":1000011,"ntime":1700003785,"order_key":"01:miner-1","p2mr_program_hex":"c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9","share_difficulty":101,"share_id":"share-3785","share_seq":3785,"template_height":800118},{"accepted_at_ms":999992,"credit_policy":null,"job_id":"job-19","job_issued_at_ms":999992,"miner_id":"miner-8","network_difficulty":1000002,"ntime":1700000019,"order_key":"08:miner-8","p2mr_program_hex":"1313131313131313131313131313131313131313131313131313131313131313","share_difficulty":1000,"share_id":"share-19","share_seq":19,"template_height":800000},{"accepted_at_ms":999989,"credit_policy":null,"job_id":"job-2842","job_issued_at_ms":999972,"miner_id":"miner-4","network_difficulty":1000003,"ntime":1700002842,"order_key":"04:miner-4","p2mr_program_hex":"1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a","share_difficulty":101,"share_id":"share-2842","share_seq":2842,"template_height":800088},{"accepted_at_ms":999986,"credit_policy":"stale-grace","job_id":"job-2794","job_issued_at_ms":999989,"miner_id":"miner-0","network_difficulty":1000006,"ntime":1700002794,"order_key":"00:miner-0","p2mr_program_hex":"eaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaea","share_difficulty":1,"share_id":"share-2794","share_seq":2794,"template_height":800087},{"accepted_at_ms":999974,"credit_policy":"stale-grace","job_id":"job-1344","job_issued_at_ms":999992,"miner_id":"miner-2","network_difficulty":1000001,"ntime":1700001344,"order_key":"02:miner-2","p2mr_program_hex":"4040404040404040404040404040404040404040404040404040404040404040","share_difficulty":2,"share_id":"share-1344","share_seq":1344,"template_height":800042},{"accepted_at_ms":999983,"credit_policy":null,"job_id":"job-3166","job_issued_at_ms":999983,"miner_id":"miner-9","network_difficulty":1000004,"ntime":1700003166,"order_key":"09:miner-9","p2mr_program_hex":"5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e","share_difficulty":3,"share_id":"share-3166","share_seq":3166,"template_height":800098},{"accepted_at_ms":999985,"credit_policy":null,"job_id":"job-626","job_issued_at_ms":999974,"miner_id":"miner-10","network_difficulty":1000014,"ntime":1700000626,"order_key":"10:miner-10","p2mr_program_hex":"7272727272727272727272727272727272727272727272727272727272727272","share_difficulty":101,"share_id":"share-626","share_seq":626,"template_height":800019},{"accepted_at_ms":999983,"credit_policy":null,"job_id":"job-1881","job_issued_at_ms":999975,"miner_id":"miner-0","network_difficulty":1000011,"ntime":1700001881,"order_key":"00:miner-0","p2mr_program_hex":"5959595959595959595959595959595959595959595959595959595959595959","share_difficulty":1000,"share_id":"share-1881","share_seq":1881,"template_height":800058},{"accepted_at_ms":999996,"credit_policy":"stale-grace","job_id":"job-3029","job_issued_at_ms":999981,"miner_id":"miner-4","network_difficulty":1000003,"ntime":1700003029,"order_key":"04:miner-4","p2mr_program_hex":"d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5","share_difficulty":2,"share_id":"share-3029","share_seq":3029,"template_height":800094},{"accepted_at_ms":999979,"credit_policy":null,"job_id":"job-2852","job_issued_at_ms":999982,"miner_id":"miner-3","network_difficulty":1000013,"ntime":1700002852,"order_key":"03:miner-3","p2mr_program_hex":"2424242424242424242424242424242424242424242424242424242424242424","share_difficulty":1,"share_id":"share-2852","share_seq":2852,"template_height":800089},{"accepted_at_ms":999982,"credit_policy":null,"job_id":"job-2343","job_issued_at_ms":999999,"miner_id":"miner-0","network_difficulty":1000014,"ntime":1700002343,"order_key":"00:miner-0","p2mr_program_hex":"2727272727272727272727272727272727272727272727272727272727272727","share_difficulty":19,"share_id":"share-2343","share_seq":2343,"template_height":800073},{"accepted_at_ms":999975,"credit_policy":null,"job_id":"job-714","job_issued_at_ms":999986,"miner_id":"miner-10","network_difficulty":1000000,"ntime":1700000714,"order_key":"10:miner-10","p2mr_program_hex":"cacacacacacacacacacacacacacacacacacacacacacacacacacacacacacacaca","share_difficulty":101,"share_id":"share-714","share_seq":714,"template_height":800022},{"accepted_at_ms":999974,"credit_policy":null,"job_id":"job-744","job_issued_at_ms":999995,"miner_id":"miner-7","network_difficulty":1000013,"ntime":1700000744,"order_key":"07:miner-7","p2mr_program_hex":"e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8","share_difficulty":2,"share_id":"share-744","share_seq":744,"template_height":800023},{"accepted_at_ms":999975,"credit_policy":"stale-grace","job_id":"job-1604","job_issued_at_ms":1000000,"miner_id":"miner-9","network_difficulty":1000006,"ntime":1700001604,"order_key":"09:miner-9","p2mr_program_hex":"4444444444444444444444444444444444444444444444444444444444444444","share_difficulty":1,"share_id":"share-1604","share_seq":1604,"template_height":800050},{"accepted_at_ms":999978,"credit_policy":null,"job_id":"job-1793","job_issued_at_ms":999992,"miner_id":"miner-0","network_difficulty":1000008,"ntime":1700001793,"order_key":"00:miner-0","p2mr_program_hex":"0101010101010101010101010101010101010101010101010101010101010101","share_difficulty":2,"share_id":"share-1793","share_seq":1793,"template_height":800056},{"accepted_at_ms":1000485,"credit_policy":null,"job_id":"job-1150","job_issued_at_ms":999984,"miner_id":"miner-6","network_difficulty":1000011,"ntime":1700001150,"order_key":"06:miner-6","p2mr_program_hex":"7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e","share_difficulty":1000,"share_id":"share-1150","share_seq":1150,"template_height":800035},{"accepted_at_ms":1000315,"credit_policy":null,"job_id":"job-635","job_issued_at_ms":1000104,"miner_id":"miner-8","network_difficulty":1000006,"ntime":1700000635,"order_key":"08:miner-8","p2mr_program_hex":"7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b","share_difficulty":1,"share_id":"share-635","share_seq":635,"template_height":800019},{"accepted_at_ms":999988,"credit_policy":null,"job_id":"job-1105","job_issued_at_ms":999975,"miner_id":"miner-5","network_difficulty":1000000,"ntime":1700001105,"order_key":"05:miner-5","p2mr_program_hex":"5151515151515151515151515151515151515151515151515151515151515151","share_difficulty":19,"share_id":"share-1105","share_seq":1105,"template_height":800034},{"accepted_at_ms":999971,"credit_policy":null,"job_id":"job-3689","job_issued_at_ms":999991,"miner_id":"miner-4","network_difficulty":1000000,"ntime":1700003689,"order_key":"04:miner-4","p2mr_program_hex":"6969696969696969696969696969696969696969696969696969696969696969","share_difficulty":2,"share_id":"share-3689","share_seq":3689,"template_height":800115},{"accepted_at_ms":999998,"credit_policy":null,"job_id":"job-3118","job_issued_at_ms":999986,"miner_id":"miner-5","network_difficulty":1000007,"ntime":1700003118,"order_key":"05:miner-5","p2mr_program_hex":"2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e","share_difficulty":7,"share_id":"share-3118","share_seq":3118,"template_height":800097},{"accepted_at_ms":1000000,"credit_policy":"stale-grace","job_id":"job-586","job_issued_at_ms":999974,"miner_id":"miner-3","network_difficulty":1000008,"ntime":1700000586,"order_key":"03:miner-3","p2mr_program_hex":"4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a","share_difficulty":1000,"share_id":"share-586","share_seq":586,"template_height":800018},{"accepted_at_ms":999989,"credit_policy":null,"job_id":"job-2298","job_issued_at_ms":999999,"miner_id":"miner-10","network_difficulty":1000003,"ntime":1700002298,"order_key":"10:miner-10","p2mr_program_hex":"fafafafafafafafafafafafafafafafafafafafafafafafafafafafafafafafa","share_difficulty":2,"share_id":"share-2298","share_seq":2298,"template_height":800071},{"accepted_at_ms":999987,"credit_policy":null,"job_id":"job-3348","job_issued_at_ms":999975,"miner_id":"miner-4","network_difficulty":1000016,"ntime":1700003348,"order_key":"04:miner-4","p2mr_program_hex":"1414141414141414141414141414141414141414141414141414141414141414","share_difficulty":7,"share_id":"share-3348","share_seq":3348,"template_height":800104},{"accepted_at_ms":999984,"credit_policy":null,"job_id":"job-2472","job_issued_at_ms":999974,"miner_id":"miner-8","network_difficulty":1000007,"ntime":1700002472,"order_key":"08:miner-8","p2mr_program_hex":"a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8","share_difficulty":7,"share_id":"share-2472","share_seq":2472,"template_height":800077},{"accepted_at_ms":999992,"credit_policy":null,"job_id":"job-832","job_issued_at_ms":999993,"miner_id":"miner-7","network_difficulty":1000016,"ntime":1700000832,"order_key":"07:miner-7","p2mr_program_hex":"4040404040404040404040404040404040404040404040404040404040404040","share_difficulty":1000,"share_id":"share-832","share_seq":832,"template_height":800026},{"accepted_at_ms":999974,"credit_policy":null,"job_id":"job-139","job_issued_at_ms":999982,"miner_id":"miner-7","network_difficulty":1000003,"ntime":1700000139,"order_key":"07:miner-7","p2mr_program_hex":"8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b","share_difficulty":7,"share_id":"share-139","share_seq":139,"template_height":800004},{"accepted_at_ms":999978,"credit_policy":null,"job_id":"job-3047","job_issued_at_ms":999972,"miner_id":"miner-0","network_difficulty":1000004,"ntime":1700003047,"order_key":"00:miner-0","p2mr_program_hex":"e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7","share_difficulty":7,"share_id":"share-3047","share_seq":3047,"template_height":800095},{"accepted_at_ms":999985,"credit_policy":null,"job_id":"job-2365","job_issued_at_ms":999995,"miner_id":"miner-0","network_difficulty":1000002,"ntime":1700002365,"order_key":"00:miner-0","p2mr_program_hex":"3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d","share_difficulty":3,"share_id":"share-2365","share_seq":2365,"template_height":800073},{"accepted_at_ms":999995,"credit_policy":null,"job_id":"job-1453","job_issued_at_ms":999995,"miner_id":"miner-1","network_difficulty":1000008,"ntime":1700001453,"order_key":"01:miner-1","p2mr_program_hex":"adadadadadadadadadadadadadadadadadadadadadadadadadadadadadadadad","share_difficulty":19,"share_id":"share-1453","share_seq":1453,"template_height":800045},{"accepted_at_ms":999987,"credit_policy":null,"job_id":"job-3800","job_issued_at_ms":999997,"miner_id":"miner-5","network_difficulty":1000009,"ntime":1700003800,"order_key":"05:miner-5","p2mr_program_hex":"d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8","share_difficulty":1000,"share_id":"share-3800","share_seq":3800,"template_height":800118},{"accepted_at_ms":999978,"credit_policy":null,"job_id":"job-1754","job_issued_at_ms":999982,"miner_id":"miner-5","network_difficulty":1000003,"ntime":1700001754,"order_key":"05:miner-5","p2mr_program_hex":"dadadadadadadadadadadadadadadadadadadadadadadadadadadadadadadada","share_difficulty":7,"share_id":"share-1754","share_seq":1754,"template_height":800054},{"accepted_at_ms":999990,"credit_policy":null,"job_id":"job-602","job_issued_at_ms":999979,"miner_id":"miner-8","network_difficulty":1000007,"ntime":1700000602,"order_key":"08:miner-8","p2mr_program_hex":"5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a","share_difficulty":101,"share_id":"share-602","share_seq":602,"template_height":800018},{"accepted_at_ms":999999,"credit_policy":null,"job_id":"job-2309","job_issued_at_ms":999994,"miner_id":"miner-10","network_difficulty":1000014,"ntime":1700002309,"order_key":"10:miner-10","p2mr_program_hex":"0505050505050505050505050505050505050505050505050505050505050505","share_difficulty":101,"share_id":"share-2309","share_seq":2309,"template_height":800072},{"accepted_at_ms":999977,"credit_policy":null,"job_id":"job-1485","job_issued_at_ms":999996,"miner_id":"miner-0","network_difficulty":1000006,"ntime":1700001485,"order_key":"00:miner-0","p2mr_program_hex":"cdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcd","share_difficulty":1000,"share_id":"share-1485","share_seq":1485,"template_height":800046},{"accepted_at_ms":1000404,"credit_policy":null,"job_id":"job-3885","job_issued_at_ms":1000133,"miner_id":"miner-2","network_difficulty":1000009,"ntime":1700003885,"order_key":"02:miner-2","p2mr_program_hex":"2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d","share_difficulty":1000,"share_id":"share-3885","share_seq":3885,"template_height":800121},{"accepted_at_ms":999973,"credit_policy":null,"job_id":"job-1851","job_issued_at_ms":999978,"miner_id":"miner-3","network_difficulty":1000015,"ntime":1700001851,"order_key":"03:miner-3","p2mr_program_hex":"3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b","share_difficulty":101,"share_id":"share-1851","share_seq":1851,"template_height":800057},{"accepted_at_ms":999983,"credit_policy":null,"job_id":"job-14","job_issued_at_ms":999988,"miner_id":"miner-3","network_difficulty":1000014,"ntime":1700000014,"order_key":"03:miner-3","p2mr_program_hex":"0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e","share_difficulty":19,"share_id":"share-14","share_seq":14,"template_height":800000},{"accepted_at_ms":999975,"credit_policy":null,"job_id":"job-3400","job_issued_at_ms":999982,"miner_id":"miner-1","network_difficulty":1000000,"ntime":1700003400,"order_key":"01:miner-1","p2mr_program_hex":"4848484848484848484848484848484848484848484848484848484848484848","share_difficulty":2,"share_id":"share-3400","share_seq":3400,"template_height":800106},{"accepted_at_ms":999975,"credit_policy":"stale-grace","job_id":"job-2874","job_issued_at_ms":999995,"miner_id":"miner-3","network_difficulty":1000001,"ntime":1700002874,"order_key":"03:miner-3","p2mr_program_hex":"3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a","share_difficulty":1,"share_id":"share-2874","share_seq":2874,"template_height":800089},{"accepted_at_ms":1000342,"credit_policy":null,"job_id":"job-65","job_issued_at_ms":999982,"miner_id":"miner-10","network_difficulty":1000014,"ntime":1700000065,"order_key":"10:miner-10","p2mr_program_hex":"4141414141414141414141414141414141414141414141414141414141414141","share_difficulty":1,"share_id":"share-65","share_seq":65,"template_height":800002},{"accepted_at_ms":1000000,"credit_policy":null,"job_id":"job-1979","job_issued_at_ms":999992,"miner_id":"miner-10","network_difficulty":1000007,"ntime":1700001979,"order_key":"10:miner-10","p2mr_program_hex":"bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb","share_difficulty":2,"share_id":"share-1979","share_seq":1979,"template_height":800061},{"accepted_at_ms":1000228,"credit_policy":null,"job_id":"job-917","job_issued_at_ms":999984,"miner_id":"miner-4","network_difficulty":1000016,"ntime":1700000917,"order_key":"04:miner-4","p2mr_program_hex":"9595959595959595959595959595959595959595959595959595959595959595","share_difficulty":101,"share_id":"share-917","share_seq":917,"template_height":800028},{"accepted_at_ms":999999,"credit_policy":null,"job_id":"job-813","job_issued_at_ms":999973,"miner_id":"miner-10","network_difficulty":1000014,"ntime":1700000813,"order_key":"10:miner-10","p2mr_program_hex":"2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d","share_difficulty":1000,"share_id":"share-813","share_seq":813,"template_height":800025},{"accepted_at_ms":999977,"credit_policy":null,"job_id":"job-2405","job_issued_at_ms":999980,"miner_id":"miner-7","network_difficulty":1000008,"ntime":1700002405,"order_key":"07:miner-7","p2mr_program_hex":"6565656565656565656565656565656565656565656565656565656565656565","share_difficulty":3,"share_id":"share-2405","share_seq":2405,"template_height":800075},{"accepted_at_ms":1000000,"credit_policy":null,"job_id":"job-175","job_issued_at_ms":999986,"miner_id":"miner-10","network_difficulty":1000005,"ntime":1700000175,"order_key":"10:miner-10","p2mr_program_hex":"afafafafafafafafafafafafafafafafafafafafafafafafafafafafafafafaf","share_difficulty":101,"share_id":"share-175","share_seq":175,"template_height":800005},{"accepted_at_ms":999981,"credit_policy":null,"job_id":"job-1001","job_issued_at_ms":999998,"miner_id":"miner-0","network_difficulty":1000015,"ntime":1700001001,"order_key":"00:miner-0","p2mr_program_hex":"e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9","share_difficulty":101,"share_id":"share-1001","share_seq":1001,"template_height":800031},{"accepted_at_ms":1000000,"credit_policy":null,"job_id":"job-2164","job_issued_at_ms":999986,"miner_id":"miner-8","network_difficulty":1000005,"ntime":1700002164,"order_key":"08:miner-8","p2mr_program_hex":"7474747474747474747474747474747474747474747474747474747474747474","share_difficulty":3,"share_id":"share-2164","share_seq":2164,"template_height":800067},{"accepted_at_ms":999975,"credit_policy":null,"job_id":"job-1561","job_issued_at_ms":999987,"miner_id":"miner-10","network_difficulty":1000014,"ntime":1700001561,"order_key":"10:miner-10","p2mr_program_hex":"1919191919191919191919191919191919191919191919191919191919191919","share_difficulty":19,"share_id":"share-1561","share_seq":1561,"template_height":800048},{"accepted_at_ms":999993,"credit_policy":"stale-grace","job_id":"job-2977","job_issued_at_ms":999996,"miner_id":"miner-7","network_difficulty":1000002,"ntime":1700002977,"order_key":"07:miner-7","p2mr_program_hex":"a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1","share_difficulty":7,"share_id":"share-2977","share_seq":2977,"template_height":800093},{"accepted_at_ms":999985,"credit_policy":null,"job_id":"job-2630","job_issued_at_ms":999992,"miner_id":"miner-1","network_difficulty":1000012,"ntime":1700002630,"order_key":"01:miner-1","p2mr_program_hex":"4646464646464646464646464646464646464646464646464646464646464646","share_difficulty":7,"share_id":"share-2630","share_seq":2630,"template_height":800082},{"accepted_at_ms":999970,"credit_policy":null,"job_id":"job-693","job_issued_at_ms":999991,"miner_id":"miner-0","network_difficulty":1000013,"ntime":1700000693,"order_key":"00:miner-0","p2mr_program_hex":"b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5","share_difficulty":101,"share_id":"share-693","share_seq":693,"template_height":800021},{"accepted_at_ms":999981,"credit_policy":"stale-grace","job_id":"job-3666","job_issued_at_ms":999989,"miner_id":"miner-3","network_difficulty":1000011,"ntime":1700003666,"order_key":"03:miner-3","p2mr_program_hex":"5252525252525252525252525252525252525252525252525252525252525252","share_difficulty":2,"share_id":"share-3666","share_seq":3666,"template_height":800114},{"accepted_at_ms":999985,"credit_policy":null,"job_id":"job-3630","job_issued_at_ms":999983,"miner_id":"miner-0","network_difficulty":1000009,"ntime":1700003630,"order_key":"00:miner-0","p2mr_program_hex":"2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e","share_difficulty":1,"share_id":"share-3630","share_seq":3630,"template_height":800113},{"accepted_at_ms":999971,"credit_policy":null,"job_id":"job-382","job_issued_at_ms":999990,"miner_id":"miner-8","network_difficulty":1000008,"ntime":1700000382,"order_key":"08:miner-8","p2mr_program_hex":"7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e","share_difficulty":2,"share_id":"share-382","share_seq":382,"template_height":800011},{"accepted_at_ms":999979,"credit_policy":null,"job_id":"job-2030","job_issued_at_ms":999988,"miner_id":"miner-6","network_difficulty":1000007,"ntime":1700002030,"order_key":"06:miner-6","p2mr_program_hex":"eeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeee","share_difficulty":3,"share_id":"share-2030","share_seq":2030,"template_height":800063},{"accepted_at_ms":999982,"credit_policy":null,"job_id":"job-209","job_issued_at_ms":999981,"miner_id":"miner-0","network_difficulty":1000005,"ntime":1700000209,"order_key":"00:miner-0","p2mr_program_hex":"d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1","share_difficulty":1000,"share_id":"share-209","share_seq":209,"template_height":800006},{"accepted_at_ms":999981,"credit_policy":"stale-grace","job_id":"job-3563","job_issued_at_ms":999989,"miner_id":"miner-10","network_difficulty":1000010,"ntime":1700003563,"order_key":"10:miner-10","p2mr_program_hex":"ebebebebebebebebebebebebebebebebebebebebebebebebebebebebebebebeb","share_difficulty":1,"share_id":"share-3563","share_seq":3563,"template_height":800111},{"accepted_at_ms":999983,"credit_policy":null,"job_id":"job-987","job_issued_at_ms":999983,"miner_id":"miner-8","network_difficulty":1000001,"ntime":1700000987,"order_key":"08:miner-8","p2mr_program_hex":"dbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdb","share_difficulty":101,"share_id":"share-987","share_seq":987,"template_height":800030},{"accepted_at_ms":999999,"credit_policy":null,"job_id":"job-655","job_issued_at_ms":999971,"miner_id":"miner-6","network_difficulty":1000009,"ntime":1700000655,"order_key":"06:miner-6","p2mr_program_hex":"8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f","share_difficulty":1000,"share_id":"share-655","share_seq":655,"template_height":800020},{"accepted_at_ms":999992,"credit_policy":null,"job_id":"job-1104","job_issued_at_ms":999977,"miner_id":"miner-4","network_difficulty":1000016,"ntime":1700001104,"order_key":"04:miner-4","p2mr_program_hex":"5050505050505050505050505050505050505050505050505050505050505050","share_difficulty":7,"share_id":"share-1104","share_seq":1104,"template_height":800034},{"accepted_at_ms":999981,"credit_policy":null,"job_id":"job-3478","job_issued_at_ms":999996,"miner_id":"miner-2","network_difficulty":1000010,"ntime":1700003478,"order_key":"02:miner-2","p2mr_program_hex":"9696969696969696969696969696969696969696969696969696969696969696","share_difficulty":1,"share_id":"share-3478","share_seq":3478,"template_height":800108},{"accepted_at_ms":999980,"credit_policy":"stale-grace","job_id":"job-1243","job_issued_at_ms":999985,"miner_id":"miner-0","network_difficulty":1000002,"ntime":1700001243,"order_key":"00:miner-0","p2mr_program_hex":"dbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdb","share_difficulty":3,"share_id":"share-1243","share_seq":1243,"template_height":800038},{"accepted_at_ms":999981,"credit_policy":null,"job_id":"job-505","job_issued_at_ms":999970,"miner_id":"miner-10","network_difficulty":1000012,"ntime":1700000505,"order_key":"10:miner-10","p2mr_program_hex":"f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9","share_difficulty":101,"share_id":"share-505","share_seq":505,"template_height":800015},{"accepted_at_ms":999975,"credit_policy":null,"job_id":"job-3377","job_issued_at_ms":999994,"miner_id":"miner-0","network_difficulty":1000011,"ntime":1700003377,"order_key":"00:miner-0","p2mr_program_hex":"3131313131313131313131313131313131313131313131313131313131313131","share_difficulty":1000,"share_id":"share-3377","share_seq":3377,"template_height":800105},{"accepted_at_ms":999990,"credit_policy":"stale-grace","job_id":"job-3575","job_issued_at_ms":999989,"miner_id":"miner-0","network_difficulty":1000005,"ntime":1700003575,"order_key":"00:miner-0","p2mr_program_hex":"f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7","share_difficulty":3,"share_id":"share-3575","share_seq":3575,"template_height":800111},{"accepted_at_ms":1000236,"credit_policy":null,"job_id":"job-3585","job_issued_at_ms":1000310,"miner_id":"miner-10","network_difficulty":1000015,"ntime":1700003585,"order_key":"10:miner-10","p2mr_program_hex":"0101010101010101010101010101010101010101010101010101010101010101","share_difficulty":19,"share_id":"share-3585","share_seq":3585,"template_height":800112},{"accepted_at_ms":999976,"credit_policy":null,"job_id":"job-1791","job_issued_at_ms":999985,"miner_id":"miner-9","network_difficulty":1000006,"ntime":1700001791,"order_key":"09:miner-9","p2mr_program_hex":"ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff","share_difficulty":1,"share_id":"share-1791","share_seq":1791,"template_height":800055},{"accepted_at_ms":999979,"credit_policy":"stale-grace","job_id":"job-1176","job_issued_at_ms":999990,"miner_id":"miner-10","network_difficulty":1000003,"ntime":1700001176,"order_key":"10:miner-10","p2mr_program_hex":"9898989898989898989898989898989898989898989898989898989898989898","share_difficulty":3,"share_id":"share-1176","share_seq":1176,"template_height":800036},{"accepted_at_ms":999984,"credit_policy":null,"job_id":"job-26","job_issued_at_ms":999974,"miner_id":"miner-4","network_difficulty":1000009,"ntime":1700000026,"order_key":"04:miner-4","p2mr_program_hex":"1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a","share_difficulty":19,"share_id":"share-26","share_seq":26,"template_height":800000},{"accepted_at_ms":999970,"credit_policy":"stale-grace","job_id":"job-2837","job_issued_at_ms":999978,"miner_id":"miner-10","network_difficulty":1000015,"ntime":1700002837,"order_key":"10:miner-10","p2mr_program_hex":"1515151515151515151515151515151515151515151515151515151515151515","share_difficulty":1000,"share_id":"share-2837","share_seq":2837,"template_height":800088},{"accepted_at_ms":999984,"credit_policy":null,"job_id":"job-929","job_issued_at_ms":999982,"miner_id":"miner-5","network_difficulty":1000011,"ntime":1700000929,"order_key":"05:miner-5","p2mr_program_hex":"a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1","share_difficulty":101,"share_id":"share-929","share_seq":929,"template_height":800029},{"accepted_at_ms":999979,"credit_policy":"stale-grace","job_id":"job-2478","job_issued_at_ms":999992,"miner_id":"miner-3","network_difficulty":1000013,"ntime":1700002478,"order_key":"03:miner-3","p2mr_program_hex":"aeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeae","share_difficulty":3,"share_id":"share-2478","share_seq":2478,"template_height":800077},{"accepted_at_ms":999980,"credit_policy":null,"job_id":"job-3321","job_issued_at_ms":999995,"miner_id":"miner-10","network_difficulty":1000006,"ntime":1700003321,"order_key":"10:miner-10","p2mr_program_hex":"f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9","share_difficulty":7,"share_id":"share-3321","share_seq":3321,"template_height":800103},{"accepted_at_ms":999987,"credit_policy":"stale-grace","job_id":"job-3204","job_issued_at_ms":999990,"miner_id":"miner-3","network_difficulty":1000008,"ntime":1700003204,"order_key":"03:miner-3","p2mr_program_hex":"8484848484848484848484848484848484848484848484848484848484848484","share_difficulty":19,"share_id":"share-3204","share_seq":3204,"template_height":800100},{"accepted_at_ms":999986,"credit_policy":null,"job_id":"job-3958","job_issued_at_ms":999980,"miner_id":"miner-9","network_difficulty":1000014,"ntime":1700003958,"order_key":"09:miner-9","p2mr_program_hex":"7676767676767676767676767676767676767676767676767676767676767676","share_difficulty":3,"share_id":"share-3958","share_seq":3958,"template_height":800123},{"accepted_at_ms":999983,"credit_policy":null,"job_id":"job-3205","job_issued_at_ms":999976,"miner_id":"miner-4","network_difficulty":1000009,"ntime":1700003205,"order_key":"04:miner-4","p2mr_program_hex":"8585858585858585858585858585858585858585858585858585858585858585","share_difficulty":101,"share_id":"share-3205","share_seq":3205,"template_height":800100},{"accepted_at_ms":999978,"credit_policy":null,"job_id":"job-3771","job_issued_at_ms":999973,"miner_id":"miner-9","network_difficulty":1000014,"ntime":1700003771,"order_key":"09:miner-9","p2mr_program_hex":"bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb","share_difficulty":101,"share_id":"share-3771","share_seq":3771,"template_height":800117},{"accepted_at_ms":999990,"credit_policy":null,"job_id":"job-506","job_issued_at_ms":999975,"miner_id":"miner-0","network_difficulty":1000013,"ntime":1700000506,"order_key":"00:miner-0","p2mr_program_hex":"fafafafafafafafafafafafafafafafafafafafafafafafafafafafafafafafa","share_difficulty":1,"share_id":"share-506","share_seq":506,"template_height":800015},{"accepted_at_ms":999997,"credit_policy":null,"job_id":"job-2271","job_issued_at_ms":999986,"miner_id":"miner-5","network_difficulty":1000010,"ntime":1700002271,"order_key":"05:miner-5","p2mr_program_hex":"dfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdf","share_difficulty":7,"share_id":"share-2271","share_seq":2271,"template_height":800070},{"accepted_at_ms":999974,"credit_policy":"stale-grace","job_id":"job-3175","job_issued_at_ms":999995,"miner_id":"miner-7","network_difficulty":1000013,"ntime":1700003175,"order_key":"07:miner-7","p2mr_program_hex":"6767676767676767676767676767676767676767676767676767676767676767","share_difficulty":2,"share_id":"share-3175","share_seq":3175,"template_height":800099},{"accepted_at_ms":999971,"credit_policy":null,"job_id":"job-1","job_issued_at_ms":999996,"miner_id":"miner-1","network_difficulty":1000001,"ntime":1700000001,"order_key":"01:miner-1","p2mr_program_hex":"0101010101010101010101010101010101010101010101010101010101010101","share_difficulty":2,"share_id":"share-1","share_seq":1,"template_height":800000},{"accepted_at_ms":999994,"credit_policy":"stale-grace","job_id":"job-3313","job_issued_at_ms":999974,"miner_id":"miner-2","network_difficulty":1000015,"ntime":1700003313,"order_key":"02:miner-2","p2mr_program_hex":"f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1","share_difficulty":3,"share_id":"share-3313","share_seq":3313,"template_height":800103},{"accepted_at_ms":999986,"credit_policy":null,"job_id":"job-314","job_issued_at_ms":999973,"miner_id":"miner-6","network_difficulty":1000008,"ntime":1700000314,"order_key":"06:miner-6","p2mr_program_hex":"3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a","share_difficulty":19,"share_id":"share-314","share_seq":314,"template_height":800009},{"accepted_at_ms":999975,"credit_policy":null,"job_id":"job-2853","job_issued_at_ms":1000265,"miner_id":"miner-4","network_difficulty":1000014,"ntime":1700002853,"order_key":"04:miner-4","p2mr_program_hex":"2525252525252525252525252525252525252525252525252525252525252525","share_difficulty":7,"share_id":"share-2853","share_seq":2853,"template_height":800089},{"accepted_at_ms":999978,"credit_policy":null,"job_id":"job-2143","job_issued_at_ms":999975,"miner_id":"miner-9","network_difficulty":1000001,"ntime":1700002143,"order_key":"09:miner-9","p2mr_program_hex":"5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f","share_difficulty":19,"share_id":"share-2143","share_seq":2143,"template_height":800066},{"accepted_at_ms":999974,"credit_policy":null,"job_id":"job-2199","job_issued_at_ms":999994,"miner_id":"miner-10","network_difficulty":1000006,"ntime":1700002199,"order_key":"10:miner-10","p2mr_program_hex":"9797979797979797979797979797979797979797979797979797979797979797","share_difficulty":1000,"share_id":"share-2199","share_seq":2199,"template_height":800068},{"accepted_at_ms":999995,"credit_policy":null,"job_id":"job-1804","job_issued_at_ms":999976,"miner_id":"miner-0","network_difficulty":1000002,"ntime":1700001804,"order_key":"00:miner-0","p2mr_program_hex":"0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c","share_difficulty":1,"share_id":"share-1804","share_seq":1804,"template_height":800056},{"accepted_at_ms":999987,"credit_policy":null,"job_id":"job-2452","job_issued_at_ms":999990,"miner_id":"miner-10","network_difficulty":1000004,"ntime":1700002452,"order_key":"10:miner-10","p2mr_program_hex":"9494949494949494949494949494949494949494949494949494949494949494","share_difficulty":7,"share_id":"share-2452","share_seq":2452,"template_height":800076},{"accepted_at_ms":999986,"credit_policy":null,"job_id":"job-3221","job_issued_at_ms":999975,"miner_id":"miner-9","network_difficulty":1000008,"ntime":1700003221,"order_key":"09:miner-9","p2mr_program_hex":"9595959595959595959595959595959595959595959595959595959595959595","share_difficulty":2,"share_id":"share-3221","share_seq":3221,"template_height":800100},{"accepted_at_ms":999992,"credit_policy":null,"job_id":"job-3068","job_issued_at_ms":999998,"miner_id":"miner-10","network_difficulty":1000008,"ntime":1700003068,"order_key":"10:miner-10","p2mr_program_hex":"fcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfc","share_difficulty":2,"share_id":"share-3068","share_seq":3068,"template_height":800095},{"accepted_at_ms":999979,"credit_policy":null,"job_id":"job-823","job_issued_at_ms":999971,"miner_id":"miner-9","network_difficulty":1000007,"ntime":1700000823,"order_key":"09:miner-9","p2mr_program_hex":"3737373737373737373737373737373737373737373737373737373737373737","share_difficulty":7,"share_id":"share-823","share_seq":823,"template_height":800025},{"accepted_at_ms":999998,"credit_policy":null,"job_id":"job-2318","job_issued_at_ms":999999,"miner_id":"miner-8","network_difficulty":1000006,"ntime":1700002318,"order_key":"08:miner-8","p2mr_program_hex":"0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e","share_difficulty":19,"share_id":"share-2318","share_seq":2318,"template_height":800072},{"accepted_at_ms":999983,"credit_policy":"stale-grace","job_id":"job-1691","job_issued_at_ms":999978,"miner_id":"miner-8","network_difficulty":1000008,"ntime":1700001691,"order_key":"08:miner-8","p2mr_program_hex":"9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b","share_difficulty":2,"share_id":"share-1691","share_seq":1691,"template_height":800052},{"accepted_at_ms":999971,"credit_policy":null,"job_id":"job-1667","job_issued_at_ms":999980,"miner_id":"miner-6","network_difficulty":1000001,"ntime":1700001667,"order_key":"06:miner-6","p2mr_program_hex":"8383838383838383838383838383838383838383838383838383838383838383","share_difficulty":2,"share_id":"share-1667","share_seq":1667,"template_height":800052},{"accepted_at_ms":999990,"credit_policy":"stale-grace","job_id":"job-584","job_issued_at_ms":1000000,"miner_id":"miner-1","network_difficulty":1000006,"ntime":1700000584,"order_key":"01:miner-1","p2mr_program_hex":"4848484848484848484848484848484848484848484848484848484848484848","share_difficulty":3,"share_id":"share-584","share_seq":584,"template_height":800018},{"accepted_at_ms":999992,"credit_policy":null,"job_id":"job-2857","job_issued_at_ms":1000047,"miner_id":"miner-8","network_difficulty":1000001,"ntime":1700002857,"order_key":"08:miner-8","p2mr_program_hex":"2929292929292929292929292929292929292929292929292929292929292929","share_difficulty":1,"share_id":"share-2857","share_seq":2857,"template_height":800089},{"accepted_at_ms":999978,"credit_policy":null,"job_id":"job-2987","job_issued_at_ms":999997,"miner_id":"miner-6","network_difficulty":1000012,"ntime":1700002987,"order_key":"06:miner-6","p2mr_program_hex":"abababababababababababababababababababababababababababababababab","share_difficulty":3,"share_id":"share-2987","share_seq":2987,"template_height":800093},{"accepted_at_ms":999980,"credit_policy":null,"job_id":"job-566","job_issued_at_ms":999986,"miner_id":"miner-5","network_difficulty":1000005,"ntime":1700000566,"order_key":"05:miner-5","p2mr_program_hex":"3636363636363636363636363636363636363636363636363636363636363636","share_difficulty":1,"share_id":"share-566","share_seq":566,"template_height":800017},{"accepted_at_ms":999989,"credit_policy":null,"job_id":"job-432","job_issued_at_ms":999998,"miner_id":"miner-3","network_difficulty":1000007,"ntime":1700000432,"order_key":"03:miner-3","p2mr_program_hex":"b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0","share_difficulty":2,"share_id":"share-432","share_seq":432,"template_height":800013},{"accepted_at_ms":999971,"credit_policy":null,"job_id":"job-1740","job_issued_at_ms":999996,"miner_id":"miner-2","network_difficulty":1000006,"ntime":1700001740,"order_key":"02:miner-2","p2mr_program_hex":"cccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccc","share_difficulty":1,"share_id":"share-1740","share_seq":1740,"template_height":800054},{"accepted_at_ms":1000122,"credit_policy":null,"job_id":"job-1922","job_issued_at_ms":1000385,"miner_id":"miner-8","network_difficulty":1000001,"ntime":1700001922,"order_key":"08:miner-8","p2mr_program_hex":"8282828282828282828282828282828282828282828282828282828282828282","share_difficulty":7,"share_id":"share-1922","share_seq":1922,"template_height":800060},{"accepted_at_ms":999986,"credit_policy":null,"job_id":"job-2","job_issued_at_ms":999972,"miner_id":"miner-2","network_difficulty":1000002,"ntime":1700000002,"order_key":"02:miner-2","p2mr_program_hex":"0202020202020202020202020202020202020202020202020202020202020202","share_difficulty":101,"share_id":"share-2","share_seq":2,"template_height":800000},{"accepted_at_ms":999993,"credit_policy":"stale-grace","job_id":"job-3502","job_issued_at_ms":999992,"miner_id":"miner-4","network_difficulty":1000000,"ntime":1700003502,"order_key":"04:miner-4","p2mr_program_hex":"aeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeae","share_difficulty":19,"share_id":"share-3502","share_seq":3502,"template_height":800109},{"accepted_at_ms":999977,"credit_policy":null,"job_id":"job-1041","job_issued_at_ms":999986,"miner_id":"miner-7","network_difficulty":1000004,"ntime":1700001041,"order_key":"07:miner-7","p2mr_program_hex":"1111111111111111111111111111111111111111111111111111111111111111","share_difficulty":7,"share_id":"share-1041","share_seq":1041,"template_height":800032},{"accepted_at_ms":999988,"credit_policy":null,"job_id":"job-798","job_issued_at_ms":999983,"miner_id":"miner-6","network_difficulty":1000016,"ntime":1700000798,"order_key":"06:miner-6","p2mr_program_hex":"1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e","share_difficulty":1000,"share_id":"share-798","share_seq":798,"template_height":800024},{"accepted_at_ms":999983,"credit_policy":null,"job_id":"job-34","job_issued_at_ms":999984,"miner_id":"miner-1","network_difficulty":1000000,"ntime":1700000034,"order_key":"01:miner-1","p2mr_program_hex":"2222222222222222222222222222222222222222222222222222222222222222","share_difficulty":7,"share_id":"share-34","share_seq":34,"template_height":800001},{"accepted_at_ms":999993,"credit_policy":null,"job_id":"job-2944","job_issued_at_ms":999975,"miner_id":"miner-7","network_difficulty":1000003,"ntime":1700002944,"order_key":"07:miner-7","p2mr_program_hex":"8080808080808080808080808080808080808080808080808080808080808080","share_difficulty":1000,"share_id":"share-2944","share_seq":2944,"template_height":800092},{"accepted_at_ms":999988,"credit_policy":null,"job_id":"job-913","job_issued_at_ms":999987,"miner_id":"miner-0","network_difficulty":1000012,"ntime":1700000913,"order_key":"00:miner-0","p2mr_program_hex":"9191919191919191919191919191919191919191919191919191919191919191","share_difficulty":3,"share_id":"share-913","share_seq":913,"template_height":800028},{"accepted_at_ms":999986,"credit_policy":null,"job_id":"job-1308","job_issued_at_ms":999974,"miner_id":"miner-10","network_difficulty":1000016,"ntime":1700001308,"order_key":"10:miner-10","p2mr_program_hex":"1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c","share_difficulty":7,"share_id":"share-1308","share_seq":1308,"template_height":800040},{"accepted_at_ms":999985,"credit_policy":null,"job_id":"job-1399","job_issued_at_ms":999970,"miner_id":"miner-2","network_difficulty":1000005,"ntime":1700001399,"order_key":"02:miner-2","p2mr_program_hex":"7777777777777777777777777777777777777777777777777777777777777777","share_difficulty":101,"share_id":"share-1399","share_seq":1399,"template_height":800043},{"accepted_at_ms":999987,"credit_policy":"stale-grace","job_id":"job-205","job_issued_at_ms":999998,"miner_id":"miner-7","network_difficulty":1000001,"ntime":1700000205,"order_key":"07:miner-7","p2mr_program_hex":"cdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcd","share_difficulty":101,"share_id":"share-205","share_seq":205,"template_height":800006},{"accepted_at_ms":1000291,"credit_policy":null,"job_id":"job-2447","job_issued_at_ms":999985,"miner_id":"miner-5","network_difficulty":1000016,"ntime":1700002447,"order_key":"05:miner-5","p2mr_program_hex":"8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f","share_difficulty":1000,"share_id":"share-2447","share_seq":2447,"template_height":800076},{"accepted_at_ms":999982,"credit_policy":null,"job_id":"job-360","job_issued_at_ms":999987,"miner_id":"miner-8","network_difficulty":1000003,"ntime":1700000360,"order_key":"08:miner-8","p2mr_program_hex":"6868686868686868686868686868686868686868686868686868686868686868","share_difficulty":19,"share_id":"share-360","share_seq":360,"template_height":800011},{"accepted_at_ms":1000379,"credit_policy":"stale-grace","job_id":"job-2176","job_issued_at_ms":1000333,"miner_id":"miner-9","network_difficulty":1000000,"ntime":1700002176,"order_key":"09:miner-9","p2mr_program_hex":"8080808080808080808080808080808080808080808080808080808080808080","share_difficulty":7,"share_id":"share-2176","share_seq":2176,"template_height":800068},{"accepted_at_ms":999992,"credit_policy":null,"job_id":"job-1058","job_issued_at_ms":1000151,"miner_id":"miner-2","network_difficulty":1000004,"ntime":1700001058,"order_key":"02:miner-2","p2mr_program_hex":"2222222222222222222222222222222222222222222222222222222222222222","share_difficulty":7,"share_id":"share-1058","share_seq":1058,"template_height":800033},{"accepted_at_ms":999997,"credit_policy":null,"job_id":"job-1306","job_issued_at_ms":999996,"miner_id":"miner-8","network_difficulty":1000014,"ntime":1700001306,"order_key":"08:miner-8","p2mr_program_hex":"1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a","share_difficulty":2,"share_id":"share-1306","share_seq":1306,"template_height":800040},{"accepted_at_ms":999986,"credit_policy":null,"job_id":"job-3702","job_issued_at_ms":999987,"miner_id":"miner-6","network_difficulty":1000013,"ntime":1700003702,"order_key":"06:miner-6","p2mr_program_hex":"7676767676767676767676767676767676767676767676767676767676767676","share_difficulty":1000,"share_id":"share-3702","share_seq":3702,"template_height":800115},{"accepted_at_ms":999973,"credit_policy":"stale-grace","job_id":"job-262","job_issued_at_ms":999974,"miner_id":"miner-9","network_difficulty":1000007,"ntime":1700000262,"order_key":"09:miner-9","p2mr_program_hex":"0606060606060606060606060606060606060606060606060606060606060606","share_difficulty":19,"share_id":"share-262","share_seq":262,"template_height":800008},{"accepted_at_ms":999974,"credit_policy":null,"job_id":"job-3050","job_issued_at_ms":1000189,"miner_id":"miner-3","network_difficulty":1000007,"ntime":1700003050,"order_key":"03:miner-3","p2mr_program_hex":"eaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaea","share_difficulty":2,"share_id":"share-3050","share_seq":3050,"template_height":800095},{"accepted_at_ms":999991,"credit_policy":null,"job_id":"job-1161","job_issued_at_ms":999970,"miner_id":"miner-6","network_difficulty":1000005,"ntime":1700001161,"order_key":"06:miner-6","p2mr_program_hex":"8989898989898989898989898989898989898989898989898989898989898989","share_difficulty":19,"share_id":"share-1161","share_seq":1161,"template_height":800036},{"accepted_at_ms":999970,"credit_policy":null,"job_id":"job-455","job_issued_at_ms":999982,"miner_id":"miner-4","network_difficulty":1000013,"ntime":1700000455,"order_key":"04:miner-4","p2mr_program_hex":"c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7","share_difficulty":1,"share_id":"share-455","share_seq":455,"template_height":800014},{"accepted_at_ms":1000000,"credit_policy":"stale-grace","job_id":"job-844","job_issued_at_ms":999983,"miner_id":"miner-8","network_difficulty":1000011,"ntime":1700000844,"order_key":"08:miner-8","p2mr_program_hex":"4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c","share_difficulty":1000,"share_id":"share-844","share_seq":844,"template_height":800026},{"accepted_at_ms":999988,"credit_policy":null,"job_id":"job-115","job_issued_at_ms":999985,"miner_id":"miner-5","network_difficulty":1000013,"ntime":1700000115,"order_key":"05:miner-5","p2mr_program_hex":"7373737373737373737373737373737373737373737373737373737373737373","share_difficulty":19,"share_id":"share-115","share_seq":115,"template_height":800003},{"accepted_at_ms":999992,"credit_policy":null,"job_id":"job-1302","job_issued_at_ms":999996,"miner_id":"miner-4","network_difficulty":1000010,"ntime":1700001302,"order_key":"04:miner-4","p2mr_program_hex":"1616161616161616161616161616161616161616161616161616161616161616","share_difficulty":1000,"share_id":"share-1302","share_seq":1302,"template_height":800040},{"accepted_at_ms":1000450,"credit_policy":"stale-grace","job_id":"job-2344","job_issued_at_ms":1000408,"miner_id":"miner-1","network_difficulty":1000015,"ntime":1700002344,"order_key":"01:miner-1","p2mr_program_hex":"2828282828282828282828282828282828282828282828282828282828282828","share_difficulty":19,"share_id":"share-2344","share_seq":2344,"template_height":800073},{"accepted_at_ms":999980,"credit_policy":null,"job_id":"job-2742","job_issued_at_ms":999986,"miner_id":"miner-3","network_difficulty":1000005,"ntime":1700002742,"order_key":"03:miner-3","p2mr_program_hex":"b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6","share_difficulty":19,"share_id":"share-2742","share_seq":2742,"template_height":800085},{"accepted_at_ms":999992,"credit_policy":null,"job_id":"job-369","job_issued_at_ms":999994,"miner_id":"miner-6","network_difficulty":1000012,"ntime":1700000369,"order_key":"06:miner-6","p2mr_program_hex":"7171717171717171717171717171717171717171717171717171717171717171","share_difficulty":101,"share_id":"share-369","share_seq":369,"template_height":800011},{"accepted_at_ms":999982,"credit_policy":null,"job_id":"job-480","job_issued_at_ms":999993,"miner_id":"miner-7","network_difficulty":1000004,"ntime":1700000480,"order_key":"07:miner-7","p2mr_program_hex":"e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0","share_difficulty":101,"share_id":"share-480","share_seq":480,"template_height":800015},{"accepted_at_ms":999983,"credit_policy":"stale-grace","job_id":"job-2491","job_issued_at_ms":999988,"miner_id":"miner-5","network_difficulty":1000009,"ntime":1700002491,"order_key":"05:miner-5","p2mr_program_hex":"bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb","share_difficulty":19,"share_id":"share-2491","share_seq":2491,"template_height":800077},{"accepted_at_ms":999990,"credit_policy":null,"job_id":"job-100","job_issued_at_ms":999995,"miner_id":"miner-1","network_difficulty":1000015,"ntime":1700000100,"order_key":"01:miner-1","p2mr_program_hex":"6464646464646464646464646464646464646464646464646464646464646464","share_difficulty":3,"share_id":"share-100","share_seq":100,"template_height":800003},{"accepted_at_ms":999978,"credit_policy":"stale-grace","job_id":"job-3435","job_issued_at_ms":999971,"miner_id":"miner-3","network_difficulty":1000001,"ntime":1700003435,"order_key":"03:miner-3","p2mr_program_hex":"6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b","share_difficulty":1,"share_id":"share-3435","share_seq":3435,"template_height":800107},{"accepted_at_ms":999996,"credit_policy":null,"job_id":"job-1074","job_issued_at_ms":999982,"miner_id":"miner-7","network_difficulty":1000003,"ntime":1700001074,"order_key":"07:miner-7","p2mr_program_hex":"3232323232323232323232323232323232323232323232323232323232323232","share_difficulty":19,"share_id":"share-1074","share_seq":1074,"template_height":800033},{"accepted_at_ms":999986,"credit_policy":null,"job_id":"job-3265","job_issued_at_ms":999988,"miner_id":"miner-9","network_difficulty":1000001,"ntime":1700003265,"order_key":"09:miner-9","p2mr_program_hex":"c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1","share_difficulty":1,"share_id":"share-3265","share_seq":3265,"template_height":800102},{"accepted_at_ms":999976,"credit_policy":"stale-grace","job_id":"job-3743","job_issued_at_ms":999996,"miner_id":"miner-3","network_difficulty":1000003,"ntime":1700003743,"order_key":"03:miner-3","p2mr_program_hex":"9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f","share_difficulty":7,"share_id":"share-3743","share_seq":3743,"template_height":800116},{"accepted_at_ms":999992,"credit_policy":null,"job_id":"job-1799","job_issued_at_ms":999991,"miner_id":"miner-6","network_difficulty":1000014,"ntime":1700001799,"order_key":"06:miner-6","p2mr_program_hex":"0707070707070707070707070707070707070707070707070707070707070707","share_difficulty":1000,"share_id":"share-1799","share_seq":1799,"template_height":800056},{"accepted_at_ms":999980,"credit_policy":"stale-grace","job_id":"job-600","job_issued_at_ms":999976,"miner_id":"miner-6","network_difficulty":1000005,"ntime":1700000600,"order_key":"06:miner-6","p2mr_program_hex":"5858585858585858585858585858585858585858585858585858585858585858","share_difficulty":1,"share_id":"share-600","share_seq":600,"template_height":800018},{"accepted_at_ms":999975,"credit_policy":"stale-grace","job_id":"job-1773","job_issued_at_ms":999988,"miner_id":"miner-2","network_difficulty":1000005,"ntime":1700001773,"order_key":"02:miner-2","p2mr_program_hex":"edededededededededededededededededededededededededededededededed","share_difficulty":19,"share_id":"share-1773","share_seq":1773,"template_height":800055},{"accepted_at_ms":999996,"credit_policy":null,"job_id":"job-2690","job_issued_at_ms":999981,"miner_id":"miner-6","network_difficulty":1000004,"ntime":1700002690,"order_key":"06:miner-6","p2mr_program_hex":"8282828282828282828282828282828282828282828282828282828282828282","share_difficulty":2,"share_id":"share-2690","share_seq":2690,"template_height":800084},{"accepted_at_ms":999994,"credit_policy":null,"job_id":"job-3607","job_issued_at_ms":999971,"miner_id":"miner-10","network_difficulty":1000003,"ntime":1700003607,"order_key":"10:miner-10","p2mr_program_hex":"1717171717171717171717171717171717171717171717171717171717171717","share_difficulty":19,"share_id":"share-3607","share_seq":3607,"template_height":800112},{"accepted_at_ms":999980,"credit_policy":null,"job_id":"job-3825","job_issued_at_ms":999992,"miner_id":"miner-8","network_difficulty":1000000,"ntime":1700003825,"order_key":"08:miner-8","p2mr_program_hex":"f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1","share_difficulty":101,"share_id":"share-3825","share_seq":3825,"template_height":800119},{"accepted_at_ms":1000109,"credit_policy":"stale-grace","job_id":"job-1458","job_issued_at_ms":999983,"miner_id":"miner-6","network_difficulty":1000013,"ntime":1700001458,"order_key":"06:miner-6","p2mr_program_hex":"b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2","share_difficulty":2,"share_id":"share-1458","share_seq":1458,"template_height":800045},{"accepted_at_ms":999994,"credit_policy":"stale-grace","job_id":"job-1548","job_issued_at_ms":999998,"miner_id":"miner-8","network_difficulty":1000001,"ntime":1700001548,"order_key":"08:miner-8","p2mr_program_hex":"0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c","share_difficulty":3,"share_id":"share-1548","share_seq":1548,"template_height":800048},{"accepted_at_ms":999988,"credit_policy":null,"job_id":"job-3874","job_issued_at_ms":999975,"miner_id":"miner-2","network_difficulty":1000015,"ntime":1700003874,"order_key":"02:miner-2","p2mr_program_hex":"2222222222222222222222222222222222222222222222222222222222222222","share_difficulty":1000,"share_id":"share-3874","share_seq":3874,"template_height":800121},{"accepted_at_ms":999992,"credit_policy":null,"job_id":"job-1258","job_issued_at_ms":999981,"miner_id":"miner-4","network_difficulty":1000000,"ntime":1700001258,"order_key":"04:miner-4","p2mr_program_hex":"eaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaea","share_difficulty":19,"share_id":"share-1258","share_seq":1258,"template_height":800039},{"accepted_at_ms":999978,"credit_policy":null,"job_id":"job-2813","job_issued_at_ms":999999,"miner_id":"miner-8","network_difficulty":1000008,"ntime":1700002813,"order_key":"08:miner-8","p2mr_program_hex":"fdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfd","share_difficulty":101,"share_id":"share-2813","share_seq":2813,"template_height":800087},{"accepted_at_ms":1000000,"credit_policy":null,"job_id":"job-1802","job_issued_at_ms":999988,"miner_id":"miner-9","network_difficulty":1000000,"ntime":1700001802,"order_key":"09:miner-9","p2mr_program_hex":"0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a","share_difficulty":7,"share_id":"share-1802","share_seq":1802,"template_height":800056},{"accepted_at_ms":999994,"credit_policy":null,"job_id":"job-546","job_issued_at_ms":999986,"miner_id":"miner-7","network_difficulty":1000002,"ntime":1700000546,"order_key":"07:miner-7","p2mr_program_hex":"2222222222222222222222222222222222222222222222222222222222222222","share_difficulty":19,"share_id":"share-546","share_seq":546,"template_height":800017},{"accepted_at_ms":999970,"credit_policy":"stale-grace","job_id":"job-3458","job_issued_at_ms":999993,"miner_id":"miner-4","network_difficulty":1000007,"ntime":1700003458,"order_key":"04:miner-4","p2mr_program_hex":"8282828282828282828282828282828282828282828282828282828282828282","share_difficulty":2,"share_id":"share-3458","share_seq":3458,"template_height":800108},{"accepted_at_ms":999987,"credit_policy":null,"job_id":"job-2912","job_issued_at_ms":999984,"miner_id":"miner-8","network_difficulty":1000005,"ntime":1700002912,"order_key":"08:miner-8","p2mr_program_hex":"6060606060606060606060606060606060606060606060606060606060606060","share_difficulty":3,"share_id":"share-2912","share_seq":2912,"template_height":800091},{"accepted_at_ms":999989,"credit_policy":null,"job_id":"job-491","job_issued_at_ms":999975,"miner_id":"miner-7","network_difficulty":1000015,"ntime":1700000491,"order_key":"07:miner-7","p2mr_program_hex":"ebebebebebebebebebebebebebebebebebebebebebebebebebebebebebebebeb","share_difficulty":19,"share_id":"share-491","share_seq":491,"template_height":800015},{"accepted_at_ms":999989,"credit_policy":null,"job_id":"job-249","job_issued_at_ms":999979,"miner_id":"miner-7","network_difficulty":1000011,"ntime":1700000249,"order_key":"07:miner-7","p2mr_program_hex":"f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9","share_difficulty":1000,"share_id":"share-249","share_seq":249,"template_height":800007},{"accepted_at_ms":999985,"credit_policy":"stale-grace","job_id":"job-1921","job_issued_at_ms":999970,"miner_id":"miner-7","network_difficulty":1000000,"ntime":1700001921,"order_key":"07:miner-7","p2mr_program_hex":"8181818181818181818181818181818181818181818181818181818181818181","share_difficulty":2,"share_id":"share-1921","share_seq":1921,"template_height":800060},{"accepted_at_ms":999978,"credit_policy":null,"job_id":"job-2923","job_issued_at_ms":1000000,"miner_id":"miner-8","network_difficulty":1000016,"ntime":1700002923,"order_key":"08:miner-8","p2mr_program_hex":"6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b","share_difficulty":101,"share_id":"share-2923","share_seq":2923,"template_height":800091},{"accepted_at_ms":999977,"credit_policy":null,"job_id":"job-1781","job_issued_at_ms":999996,"miner_id":"miner-10","network_difficulty":1000013,"ntime":1700001781,"order_key":"10:miner-10","p2mr_program_hex":"f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5","share_difficulty":101,"share_id":"share-1781","share_seq":1781,"template_height":800055},{"accepted_at_ms":999976,"credit_policy":null,"job_id":"job-2166","job_issued_at_ms":999989,"miner_id":"miner-10","network_difficulty":1000007,"ntime":1700002166,"order_key":"10:miner-10","p2mr_program_hex":"7676767676767676767676767676767676767676767676767676767676767676","share_difficulty":1000,"share_id":"share-2166","share_seq":2166,"template_height":800067},{"accepted_at_ms":999994,"credit_policy":null,"job_id":"job-324","job_issued_at_ms":999998,"miner_id":"miner-5","network_difficulty":1000001,"ntime":1700000324,"order_key":"05:miner-5","p2mr_program_hex":"4444444444444444444444444444444444444444444444444444444444444444","share_difficulty":2,"share_id":"share-324","share_seq":324,"template_height":800010},{"accepted_at_ms":1000000,"credit_policy":null,"job_id":"job-1470","job_issued_at_ms":999988,"miner_id":"miner-7","network_difficulty":1000008,"ntime":1700001470,"order_key":"07:miner-7","p2mr_program_hex":"bebebebebebebebebebebebebebebebebebebebebebebebebebebebebebebebe","share_difficulty":2,"share_id":"share-1470","share_seq":1470,"template_height":800045},{"accepted_at_ms":999973,"credit_policy":null,"job_id":"job-3170","job_issued_at_ms":999999,"miner_id":"miner-2","network_difficulty":1000008,"ntime":1700003170,"order_key":"02:miner-2","p2mr_program_hex":"6262626262626262626262626262626262626262626262626262626262626262","share_difficulty":7,"share_id":"share-3170","share_seq":3170,"template_height":800099},{"accepted_at_ms":999971,"credit_policy":"stale-grace","job_id":"job-2859","job_issued_at_ms":999978,"miner_id":"miner-10","network_difficulty":1000003,"ntime":1700002859,"order_key":"10:miner-10","p2mr_program_hex":"2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b","share_difficulty":3,"share_id":"share-2859","share_seq":2859,"template_height":800089},{"accepted_at_ms":999971,"credit_policy":null,"job_id":"job-2438","job_issued_at_ms":999971,"miner_id":"miner-7","network_difficulty":1000007,"ntime":1700002438,"order_key":"07:miner-7","p2mr_program_hex":"8686868686868686868686868686868686868686868686868686868686868686","share_difficulty":19,"share_id":"share-2438","share_seq":2438,"template_height":800076},{"accepted_at_ms":999997,"credit_policy":null,"job_id":"job-2642","job_issued_at_ms":999986,"miner_id":"miner-2","network_difficulty":1000007,"ntime":1700002642,"order_key":"02:miner-2","p2mr_program_hex":"5252525252525252525252525252525252525252525252525252525252525252","share_difficulty":101,"share_id":"share-2642","share_seq":2642,"template_height":800082},{"accepted_at_ms":999974,"credit_policy":null,"job_id":"job-3789","job_issued_at_ms":999974,"miner_id":"miner-5","network_difficulty":1000015,"ntime":1700003789,"order_key":"05:miner-5","p2mr_program_hex":"cdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcd","share_difficulty":19,"share_id":"share-3789","share_seq":3789,"template_height":800118},{"accepted_at_ms":999995,"credit_policy":"stale-grace","job_id":"job-1031","job_issued_at_ms":999973,"miner_id":"miner-8","network_difficulty":1000011,"ntime":1700001031,"order_key":"08:miner-8","p2mr_program_hex":"0707070707070707070707070707070707070707070707070707070707070707","share_difficulty":101,"share_id":"share-1031","share_seq":1031,"template_height":800032},{"accepted_at_ms":999972,"credit_policy":null,"job_id":"job-1185","job_issued_at_ms":999975,"miner_id":"miner-8","network_difficulty":1000012,"ntime":1700001185,"order_key":"08:miner-8","p2mr_program_hex":"a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1","share_difficulty":1000,"share_id":"share-1185","share_seq":1185,"template_height":800037},{"accepted_at_ms":999975,"credit_policy":null,"job_id":"job-3831","job_issued_at_ms":999974,"miner_id":"miner-3","network_difficulty":1000006,"ntime":1700003831,"order_key":"03:miner-3","p2mr_program_hex":"f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7","share_difficulty":101,"share_id":"share-3831","share_seq":3831,"template_height":800119},{"accepted_at_ms":999992,"credit_policy":null,"job_id":"job-1356","job_issued_at_ms":999973,"miner_id":"miner-3","network_difficulty":1000013,"ntime":1700001356,"order_key":"03:miner-3","p2mr_program_hex":"4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c","share_difficulty":7,"share_id":"share-1356","share_seq":1356,"template_height":800042},{"accepted_at_ms":999973,"credit_policy":"stale-grace","job_id":"job-1387","job_issued_at_ms":999981,"miner_id":"miner-1","network_difficulty":1000010,"ntime":1700001387,"order_key":"01:miner-1","p2mr_program_hex":"6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b","share_difficulty":3,"share_id":"share-1387","share_seq":1387,"template_height":800043},{"accepted_at_ms":1000196,"credit_policy":"stale-grace","job_id":"job-3212","job_issued_at_ms":999987,"miner_id":"miner-0","network_difficulty":1000016,"ntime":1700003212,"order_key":"00:miner-0","p2mr_program_hex":"8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c","share_difficulty":1,"share_id":"share-3212","share_seq":3212,"template_height":800100},{"accepted_at_ms":999990,"credit_policy":null,"job_id":"job-2965","job_issued_at_ms":999986,"miner_id":"miner-6","network_difficulty":1000007,"ntime":1700002965,"order_key":"06:miner-6","p2mr_program_hex":"9595959595959595959595959595959595959595959595959595959595959595","share_difficulty":101,"share_id":"share-2965","share_seq":2965,"template_height":800092},{"accepted_at_ms":999985,"credit_policy":null,"job_id":"job-3712","job_issued_at_ms":999978,"miner_id":"miner-5","network_difficulty":1000006,"ntime":1700003712,"order_key":"05:miner-5","p2mr_program_hex":"8080808080808080808080808080808080808080808080808080808080808080","share_difficulty":1,"share_id":"share-3712","share_seq":3712,"template_height":800116},{"accepted_at_ms":999974,"credit_policy":null,"job_id":"job-573","job_issued_at_ms":999978,"miner_id":"miner-1","network_difficulty":1000012,"ntime":1700000573,"order_key":"01:miner-1","p2mr_program_hex":"3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d","share_difficulty":19,"share_id":"share-573","share_seq":573,"template_height":800017},{"accepted_at_ms":999973,"credit_policy":null,"job_id":"job-1112","job_issued_at_ms":999974,"miner_id":"miner-1","network_difficulty":1000007,"ntime":1700001112,"order_key":"01:miner-1","p2mr_program_hex":"5858585858585858585858585858585858585858585858585858585858585858","share_difficulty":19,"share_id":"share-1112","share_seq":1112,"template_height":800034},{"accepted_at_ms":999993,"credit_policy":null,"job_id":"job-1526","job_issued_at_ms":999995,"miner_id":"miner-8","network_difficulty":1000013,"ntime":1700001526,"order_key":"08:miner-8","p2mr_program_hex":"f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6","share_difficulty":3,"share_id":"share-1526","share_seq":1526,"template_height":800047},{"accepted_at_ms":1000000,"credit_policy":"stale-grace","job_id":"job-151","job_issued_at_ms":999993,"miner_id":"miner-8","network_difficulty":1000015,"ntime":1700000151,"order_key":"08:miner-8","p2mr_program_hex":"9797979797979797979797979797979797979797979797979797979797979797","share_difficulty":2,"share_id":"share-151","share_seq":151,"template_height":800004},{"accepted_at_ms":999980,"credit_policy":null,"job_id":"job-1465","job_issued_at_ms":999988,"miner_id":"miner-2","network_difficulty":1000003,"ntime":1700001465,"order_key":"02:miner-2","p2mr_program_hex":"b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9","share_difficulty":3,"share_id":"share-1465","share_seq":1465,"template_height":800045},{"accepted_at_ms":999982,"credit_policy":null,"job_id":"job-181","job_issued_at_ms":1000431,"miner_id":"miner-5","network_difficulty":1000011,"ntime":1700000181,"order_key":"05:miner-5","p2mr_program_hex":"b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5","share_difficulty":3,"share_id":"share-181","share_seq":181,"template_height":800005},{"accepted_at_ms":999977,"credit_policy":null,"job_id":"job-3909","job_issued_at_ms":999985,"miner_id":"miner-4","network_difficulty":1000016,"ntime":1700003909,"order_key":"04:miner-4","p2mr_program_hex":"4545454545454545454545454545454545454545454545454545454545454545","share_difficulty":2,"share_id":"share-3909","share_seq":3909,"template_height":800122},{"accepted_at_ms":999971,"credit_policy":null,"job_id":"job-1018","job_issued_at_ms":999982,"miner_id":"miner-6","network_difficulty":1000015,"ntime":1700001018,"order_key":"06:miner-6","p2mr_program_hex":"fafafafafafafafafafafafafafafafafafafafafafafafafafafafafafafafa","share_difficulty":3,"share_id":"share-1018","share_seq":1018,"template_height":800031},{"accepted_at_ms":999989,"credit_policy":null,"job_id":"job-1973","job_issued_at_ms":999999,"miner_id":"miner-4","network_difficulty":1000001,"ntime":1700001973,"order_key":"04:miner-4","p2mr_program_hex":"b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5","share_difficulty":2,"share_id":"share-1973","share_seq":1973,"template_height":800061},{"accepted_at_ms":999988,"credit_policy":null,"job_id":"job-1806","job_issued_at_ms":999987,"miner_id":"miner-2","network_difficulty":1000004,"ntime":1700001806,"order_key":"02:miner-2","p2mr_program_hex":"0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e","share_difficulty":1,"share_id":"share-1806","share_seq":1806,"template_height":800056},{"accepted_at_ms":999998,"credit_policy":"stale-grace","job_id":"job-147","job_issued_at_ms":999995,"miner_id":"miner-4","network_difficulty":1000011,"ntime":1700000147,"order_key":"04:miner-4","p2mr_program_hex":"9393939393939393939393939393939393939393939393939393939393939393","share_difficulty":7,"share_id":"share-147","share_seq":147,"template_height":800004},{"accepted_at_ms":999997,"credit_policy":null,"job_id":"job-3531","job_issued_at_ms":999980,"miner_id":"miner-0","network_difficulty":1000012,"ntime":1700003531,"order_key":"00:miner-0","p2mr_program_hex":"cbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcb","share_difficulty":2,"share_id":"share-3531","share_seq":3531,"template_height":800110},{"accepted_at_ms":999988,"credit_policy":null,"job_id":"job-642","job_issued_at_ms":999987,"miner_id":"miner-4","network_difficulty":1000013,"ntime":1700000642,"order_key":"04:miner-4","p2mr_program_hex":"8282828282828282828282828282828282828282828282828282828282828282","share_difficulty":101,"share_id":"share-642","share_seq":642,"template_height":800020},{"accepted_at_ms":999987,"credit_policy":"stale-grace","job_id":"job-1582","job_issued_at_ms":999990,"miner_id":"miner-9","network_difficulty":1000001,"ntime":1700001582,"order_key":"09:miner-9","p2mr_program_hex":"2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e","share_difficulty":7,"share_id":"share-1582","share_seq":1582,"template_height":800049},{"accepted_at_ms":999997,"credit_policy":null,"job_id":"job-2731","job_issued_at_ms":1000000,"miner_id":"miner-3","network_difficulty":1000011,"ntime":1700002731,"order_key":"03:miner-3","p2mr_program_hex":"abababababababababababababababababababababababababababababababab","share_difficulty":1000,"share_id":"share-2731","share_seq":2731,"template_height":800085},{"accepted_at_ms":999973,"credit_policy":null,"job_id":"job-1113","job_issued_at_ms":999976,"miner_id":"miner-2","network_difficulty":1000008,"ntime":1700001113,"order_key":"02:miner-2","p2mr_program_hex":"5959595959595959595959595959595959595959595959595959595959595959","share_difficulty":2,"share_id":"share-1113","share_seq":1113,"template_height":800034},{"accepted_at_ms":999981,"credit_policy":null,"job_id":"job-2505","job_issued_at_ms":999987,"miner_id":"miner-8","network_difficulty":1000006,"ntime":1700002505,"order_key":"08:miner-8","p2mr_program_hex":"c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9","share_difficulty":3,"share_id":"share-2505","share_seq":2505,"template_height":800078},{"accepted_at_ms":999995,"credit_policy":null,"job_id":"job-2648","job_issued_at_ms":999992,"miner_id":"miner-8","network_difficulty":1000013,"ntime":1700002648,"order_key":"08:miner-8","p2mr_program_hex":"5858585858585858585858585858585858585858585858585858585858585858","share_difficulty":1000,"share_id":"share-2648","share_seq":2648,"template_height":800082},{"accepted_at_ms":1000264,"credit_policy":null,"job_id":"job-2910","job_issued_at_ms":999981,"miner_id":"miner-6","network_difficulty":1000003,"ntime":1700002910,"order_key":"06:miner-6","p2mr_program_hex":"5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e","share_difficulty":2,"share_id":"share-2910","share_seq":2910,"template_height":800090},{"accepted_at_ms":999999,"credit_policy":null,"job_id":"job-135","job_issued_at_ms":999989,"miner_id":"miner-3","network_difficulty":1000016,"ntime":1700000135,"order_key":"03:miner-3","p2mr_program_hex":"8787878787878787878787878787878787878787878787878787878787878787","share_difficulty":101,"share_id":"share-135","share_seq":135,"template_height":800004},{"accepted_at_ms":999988,"credit_policy":"stale-grace","job_id":"job-485","job_issued_at_ms":999982,"miner_id":"miner-1","network_difficulty":1000009,"ntime":1700000485,"order_key":"01:miner-1","p2mr_program_hex":"e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5","share_difficulty":19,"share_id":"share-485","share_seq":485,"template_height":800015},{"accepted_at_ms":999977,"credit_policy":null,"job_id":"job-1553","job_issued_at_ms":999984,"miner_id":"miner-2","network_difficulty":1000006,"ntime":1700001553,"order_key":"02:miner-2","p2mr_program_hex":"1111111111111111111111111111111111111111111111111111111111111111","share_difficulty":1,"share_id":"share-1553","share_seq":1553,"template_height":800048},{"accepted_at_ms":999978,"credit_policy":null,"job_id":"job-1427","job_issued_at_ms":999974,"miner_id":"miner-8","network_difficulty":1000016,"ntime":1700001427,"order_key":"08:miner-8","p2mr_program_hex":"9393939393939393939393939393939393939393939393939393939393939393","share_difficulty":3,"share_id":"share-1427","share_seq":1427,"template_height":800044},{"accepted_at_ms":999988,"credit_policy":null,"job_id":"job-1935","job_issued_at_ms":999979,"miner_id":"miner-10","network_difficulty":1000014,"ntime":1700001935,"order_key":"10:miner-10","p2mr_program_hex":"8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f","share_difficulty":2,"share_id":"share-1935","share_seq":1935,"template_height":800060},{"accepted_at_ms":999984,"credit_policy":null,"job_id":"job-2255","job_issued_at_ms":999978,"miner_id":"miner-0","network_difficulty":1000011,"ntime":1700002255,"order_key":"00:miner-0","p2mr_program_hex":"cfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcf","share_difficulty":1,"share_id":"share-2255","share_seq":2255,"template_height":800070},{"accepted_at_ms":999981,"credit_policy":null,"job_id":"job-508","job_issued_at_ms":999970,"miner_id":"miner-2","network_difficulty":1000015,"ntime":1700000508,"order_key":"02:miner-2","p2mr_program_hex":"fcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfc","share_difficulty":1,"share_id":"share-508","share_seq":508,"template_height":800015},{"accepted_at_ms":999987,"credit_policy":null,"job_id":"job-3496","job_issued_at_ms":999987,"miner_id":"miner-9","network_difficulty":1000011,"ntime":1700003496,"order_key":"09:miner-9","p2mr_program_hex":"a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8","share_difficulty":1,"share_id":"share-3496","share_seq":3496,"template_height":800109},{"accepted_at_ms":999996,"credit_policy":null,"job_id":"job-3311","job_issued_at_ms":999983,"miner_id":"miner-0","network_difficulty":1000013,"ntime":1700003311,"order_key":"00:miner-0","p2mr_program_hex":"efefefefefefefefefefefefefefefefefefefefefefefefefefefefefefefef","share_difficulty":1,"share_id":"share-3311","share_seq":3311,"template_height":800103},{"accepted_at_ms":999999,"credit_policy":null,"job_id":"job-3955","job_issued_at_ms":999990,"miner_id":"miner-6","network_difficulty":1000011,"ntime":1700003955,"order_key":"06:miner-6","p2mr_program_hex":"7373737373737373737373737373737373737373737373737373737373737373","share_difficulty":1000,"share_id":"share-3955","share_seq":3955,"template_height":800123},{"accepted_at_ms":999988,"credit_policy":null,"job_id":"job-3419","job_issued_at_ms":999987,"miner_id":"miner-9","network_difficulty":1000002,"ntime":1700003419,"order_key":"09:miner-9","p2mr_program_hex":"5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b","share_difficulty":2,"share_id":"share-3419","share_seq":3419,"template_height":800106},{"accepted_at_ms":999978,"credit_policy":null,"job_id":"job-2222","job_issued_at_ms":999981,"miner_id":"miner-0","network_difficulty":1000012,"ntime":1700002222,"order_key":"00:miner-0","p2mr_program_hex":"aeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeae","share_difficulty":3,"share_id":"share-2222","share_seq":2222,"template_height":800069},{"accepted_at_ms":999979,"credit_policy":null,"job_id":"job-49","job_issued_at_ms":999998,"miner_id":"miner-5","network_difficulty":1000015,"ntime":1700000049,"order_key":"05:miner-5","p2mr_program_hex":"3131313131313131313131313131313131313131313131313131313131313131","share_difficulty":7,"share_id":"share-49","share_seq":49,"template_height":800001},{"accepted_at_ms":999979,"credit_policy":"stale-grace","job_id":"job-3045","job_issued_at_ms":1000000,"miner_id":"miner-9","network_difficulty":1000002,"ntime":1700003045,"order_key":"09:miner-9","p2mr_program_hex":"e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5","share_difficulty":19,"share_id":"share-3045","share_seq":3045,"template_height":800095},{"accepted_at_ms":999993,"credit_policy":null,"job_id":"job-277","job_issued_at_ms":999987,"miner_id":"miner-2","network_difficulty":1000005,"ntime":1700000277,"order_key":"02:miner-2","p2mr_program_hex":"1515151515151515151515151515151515151515151515151515151515151515","share_difficulty":101,"share_id":"share-277","share_seq":277,"template_height":800008},{"accepted_at_ms":999987,"credit_policy":null,"job_id":"job-3861","job_issued_at_ms":999978,"miner_id":"miner-0","network_difficulty":1000002,"ntime":1700003861,"order_key":"00:miner-0","p2mr_program_hex":"1515151515151515151515151515151515151515151515151515151515151515","share_difficulty":1000,"share_id":"share-3861","share_seq":3861,"template_height":800120},{"accepted_at_ms":999979,"credit_policy":null,"job_id":"job-332","job_issued_at_ms":999991,"miner_id":"miner-2","network_difficulty":1000009,"ntime":1700000332,"order_key":"02:miner-2","p2mr_program_hex":"4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c","share_difficulty":101,"share_id":"share-332","share_seq":332,"template_height":800010},{"accepted_at_ms":999996,"credit_policy":"stale-grace","job_id":"job-2985","job_issued_at_ms":999991,"miner_id":"miner-4","network_difficulty":1000010,"ntime":1700002985,"order_key":"04:miner-4","p2mr_program_hex":"a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9","share_difficulty":19,"share_id":"share-2985","share_seq":2985,"template_height":800093},{"accepted_at_ms":999972,"credit_policy":null,"job_id":"job-338","job_issued_at_ms":999995,"miner_id":"miner-8","network_difficulty":1000015,"ntime":1700000338,"order_key":"08:miner-8","p2mr_program_hex":"5252525252525252525252525252525252525252525252525252525252525252","share_difficulty":1000,"share_id":"share-338","share_seq":338,"template_height":800010},{"accepted_at_ms":999980,"credit_policy":null,"job_id":"job-2940","job_issued_at_ms":999998,"miner_id":"miner-3","network_difficulty":1000016,"ntime":1700002940,"order_key":"03:miner-3","p2mr_program_hex":"7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c","share_difficulty":3,"share_id":"share-2940","share_seq":2940,"template_height":800091},{"accepted_at_ms":999987,"credit_policy":"stale-grace","job_id":"job-530","job_issued_at_ms":999982,"miner_id":"miner-2","network_difficulty":1000003,"ntime":1700000530,"order_key":"02:miner-2","p2mr_program_hex":"1212121212121212121212121212121212121212121212121212121212121212","share_difficulty":1,"share_id":"share-530","share_seq":530,"template_height":800016},{"accepted_at_ms":999973,"credit_policy":null,"job_id":"job-2868","job_issued_at_ms":999986,"miner_id":"miner-8","network_difficulty":1000012,"ntime":1700002868,"order_key":"08:miner-8","p2mr_program_hex":"3434343434343434343434343434343434343434343434343434343434343434","share_difficulty":101,"share_id":"share-2868","share_seq":2868,"template_height":800089},{"accepted_at_ms":999988,"credit_policy":null,"job_id":"job-30","job_issued_at_ms":999979,"miner_id":"miner-8","network_difficulty":1000013,"ntime":1700000030,"order_key":"08:miner-8","p2mr_program_hex":"1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e","share_difficulty":7,"share_id":"share-30","share_seq":30,"template_height":800000},{"accepted_at_ms":999995,"credit_policy":null,"job_id":"job-532","job_issued_at_ms":999974,"miner_id":"miner-4","network_difficulty":1000005,"ntime":1700000532,"order_key":"04:miner-4","p2mr_program_hex":"1414141414141414141414141414141414141414141414141414141414141414","share_difficulty":19,"share_id":"share-532","share_seq":532,"template_height":800016},{"accepted_at_ms":999985,"credit_policy":null,"job_id":"job-615","job_issued_at_ms":999974,"miner_id":"miner-10","network_difficulty":1000003,"ntime":1700000615,"order_key":"10:miner-10","p2mr_program_hex":"6767676767676767676767676767676767676767676767676767676767676767","share_difficulty":2,"share_id":"share-615","share_seq":615,"template_height":800019},{"accepted_at_ms":999983,"credit_policy":null,"job_id":"job-1397","job_issued_at_ms":999993,"miner_id":"miner-0","network_difficulty":1000003,"ntime":1700001397,"order_key":"00:miner-0","p2mr_program_hex":"7575757575757575757575757575757575757575757575757575757575757575","share_difficulty":3,"share_id":"share-1397","share_seq":1397,"template_height":800043},{"accepted_at_ms":999983,"credit_policy":null,"job_id":"job-820","job_issued_at_ms":999987,"miner_id":"miner-6","network_difficulty":1000004,"ntime":1700000820,"order_key":"06:miner-6","p2mr_program_hex":"3434343434343434343434343434343434343434343434343434343434343434","share_difficulty":1000,"share_id":"share-820","share_seq":820,"template_height":800025},{"accepted_at_ms":999975,"credit_policy":null,"job_id":"job-2975","job_issued_at_ms":999994,"miner_id":"miner-5","network_difficulty":1000000,"ntime":1700002975,"order_key":"05:miner-5","p2mr_program_hex":"9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f","share_difficulty":19,"share_id":"share-2975","share_seq":2975,"template_height":800092},{"accepted_at_ms":999986,"credit_policy":null,"job_id":"job-1789","job_issued_at_ms":999986,"miner_id":"miner-7","network_difficulty":1000004,"ntime":1700001789,"order_key":"07:miner-7","p2mr_program_hex":"fdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfd","share_difficulty":7,"share_id":"share-1789","share_seq":1789,"template_height":800055},{"accepted_at_ms":999979,"credit_policy":"stale-grace","job_id":"job-1010","job_issued_at_ms":999997,"miner_id":"miner-9","network_difficulty":1000007,"ntime":1700001010,"order_key":"09:miner-9","p2mr_program_hex":"f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2","share_difficulty":3,"share_id":"share-1010","share_seq":1010,"template_height":800031},{"accepted_at_ms":999999,"credit_policy":null,"job_id":"job-2048","job_issued_at_ms":999975,"miner_id":"miner-2","network_difficulty":1000008,"ntime":1700002048,"order_key":"02:miner-2","p2mr_program_hex":"0000000000000000000000000000000000000000000000000000000000000000","share_difficulty":2,"share_id":"share-2048","share_seq":2048,"template_height":800064},{"accepted_at_ms":999971,"credit_policy":"stale-grace","job_id":"job-2230","job_issued_at_ms":999970,"miner_id":"miner-8","network_difficulty":1000003,"ntime":1700002230,"order_key":"08:miner-8","p2mr_program_hex":"b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6","share_difficulty":3,"share_id":"share-2230","share_seq":2230,"template_height":800069},{"accepted_at_ms":999985,"credit_policy":null,"job_id":"job-1178","job_issued_at_ms":999994,"miner_id":"miner-1","network_difficulty":1000005,"ntime":1700001178,"order_key":"01:miner-1","p2mr_program_hex":"9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a","share_difficulty":3,"share_id":"share-1178","share_seq":1178,"template_height":800036},{"accepted_at_ms":999981,"credit_policy":null,"job_id":"job-3518","job_issued_at_ms":999970,"miner_id":"miner-9","network_difficulty":1000016,"ntime":1700003518,"order_key":"09:miner-9","p2mr_program_hex":"bebebebebebebebebebebebebebebebebebebebebebebebebebebebebebebebe","share_difficulty":7,"share_id":"share-3518","share_seq":3518,"template_height":800109},{"accepted_at_ms":999988,"credit_policy":"stale-grace","job_id":"job-1931","job_issued_at_ms":999991,"miner_id":"miner-6","network_difficulty":1000010,"ntime":1700001931,"order_key":"06:miner-6","p2mr_program_hex":"8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b","share_difficulty":3,"share_id":"share-1931","share_seq":1931,"template_height":800060},{"accepted_at_ms":999994,"credit_policy":null,"job_id":"job-3618","job_issued_at_ms":999992,"miner_id":"miner-10","network_difficulty":1000014,"ntime":1700003618,"order_key":"10:miner-10","p2mr_program_hex":"2222222222222222222222222222222222222222222222222222222222222222","share_difficulty":1000,"share_id":"share-3618","share_seq":3618,"template_height":800113},{"accepted_at_ms":999989,"credit_policy":null,"job_id":"job-2740","job_issued_at_ms":999998,"miner_id":"miner-1","network_difficulty":1000003,"ntime":1700002740,"order_key":"01:miner-1","p2mr_program_hex":"b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4","share_difficulty":3,"share_id":"share-2740","share_seq":2740,"template_height":800085},{"accepted_at_ms":999985,"credit_policy":null,"job_id":"job-2345","job_issued_at_ms":999997,"miner_id":"miner-2","network_difficulty":1000016,"ntime":1700002345,"order_key":"02:miner-2","p2mr_program_hex":"2929292929292929292929292929292929292929292929292929292929292929","share_difficulty":2,"share_id":"share-2345","share_seq":2345,"template_height":800073},{"accepted_at_ms":1000387,"credit_policy":null,"job_id":"job-2475","job_issued_at_ms":999996,"miner_id":"miner-0","network_difficulty":1000010,"ntime":1700002475,"order_key":"00:miner-0","p2mr_program_hex":"abababababababababababababababababababababababababababababababab","share_difficulty":7,"share_id":"share-2475","share_seq":2475,"template_height":800077},{"accepted_at_ms":999986,"credit_policy":null,"job_id":"job-1990","job_issued_at_ms":999992,"miner_id":"miner-10","network_difficulty":1000001,"ntime":1700001990,"order_key":"10:miner-10","p2mr_program_hex":"c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6","share_difficulty":7,"share_id":"share-1990","share_seq":1990,"template_height":800062},{"accepted_at_ms":999987,"credit_policy":null,"job_id":"job-1859","job_issued_at_ms":999984,"miner_id":"miner-0","network_difficulty":1000006,"ntime":1700001859,"order_key":"00:miner-0","p2mr_program_hex":"4343434343434343434343434343434343434343434343434343434343434343","share_difficulty":101,"share_id":"share-1859","share_seq":1859,"template_height":800058},{"accepted_at_ms":999990,"credit_policy":null,"job_id":"job-3097","job_issued_at_ms":999996,"miner_id":"miner-6","network_difficulty":1000003,"ntime":1700003097,"order_key":"06:miner-6","p2mr_program_hex":"1919191919191919191919191919191919191919191919191919191919191919","share_difficulty":2,"share_id":"share-3097","share_seq":3097,"template_height":800096},{"accepted_at_ms":999990,"credit_policy":null,"job_id":"job-3369","job_issued_at_ms":999981,"miner_id":"miner-3","network_difficulty":1000003,"ntime":1700003369,"order_key":"03:miner-3","p2mr_program_hex":"2929292929292929292929292929292929292929292929292929292929292929","share_difficulty":1,"share_id":"share-3369","share_seq":3369,"template_height":800105},{"accepted_at_ms":999990,"credit_policy":null,"job_id":"job-2154","job_issued_at_ms":999972,"miner_id":"miner-9","network_difficulty":1000012,"ntime":1700002154,"order_key":"09:miner-9","p2mr_program_hex":"6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a","share_difficulty":19,"share_id":"share-2154","share_seq":2154,"template_height":800067},{"accepted_at_ms":999979,"credit_policy":null,"job_id":"job-1664","job_issued_at_ms":999981,"miner_id":"miner-3","network_difficulty":1000015,"ntime":1700001664,"order_key":"03:miner-3","p2mr_program_hex":"8080808080808080808080808080808080808080808080808080808080808080","share_difficulty":3,"share_id":"share-1664","share_seq":1664,"template_height":800052},{"accepted_at_ms":999982,"credit_policy":null,"job_id":"job-2071","job_issued_at_ms":999972,"miner_id":"miner-3","network_difficulty":1000014,"ntime":1700002071,"order_key":"03:miner-3","p2mr_program_hex":"1717171717171717171717171717171717171717171717171717171717171717","share_difficulty":101,"share_id":"share-2071","share_seq":2071,"template_height":800064},{"accepted_at_ms":999992,"credit_policy":null,"job_id":"job-3845","job_issued_at_ms":999981,"miner_id":"miner-6","network_difficulty":1000003,"ntime":1700003845,"order_key":"06:miner-6","p2mr_program_hex":"0505050505050505050505050505050505050505050505050505050505050505","share_difficulty":1000,"share_id":"share-3845","share_seq":3845,"template_height":800120},{"accepted_at_ms":999971,"credit_policy":null,"job_id":"job-2074","job_issued_at_ms":999980,"miner_id":"miner-6","network_difficulty":1000000,"ntime":1700002074,"order_key":"06:miner-6","p2mr_program_hex":"1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a","share_difficulty":1000,"share_id":"share-2074","share_seq":2074,"template_height":800064},{"accepted_at_ms":999999,"credit_policy":null,"job_id":"job-296","job_issued_at_ms":999995,"miner_id":"miner-10","network_difficulty":1000007,"ntime":1700000296,"order_key":"10:miner-10","p2mr_program_hex":"2828282828282828282828282828282828282828282828282828282828282828","share_difficulty":2,"share_id":"share-296","share_seq":296,"template_height":800009},{"accepted_at_ms":999975,"credit_policy":null,"job_id":"job-1017","job_issued_at_ms":999990,"miner_id":"miner-5","network_difficulty":1000014,"ntime":1700001017,"order_key":"05:miner-5","p2mr_program_hex":"f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9","share_difficulty":101,"share_id":"share-1017","share_seq":1017,"template_height":800031},{"accepted_at_ms":999978,"credit_policy":null,"job_id":"job-2194","job_issued_at_ms":999987,"miner_id":"miner-5","network_difficulty":1000001,"ntime":1700002194,"order_key":"05:miner-5","p2mr_program_hex":"9292929292929292929292929292929292929292929292929292929292929292","share_difficulty":7,"share_id":"share-2194","share_seq":2194,"template_height":800068},{"accepted_at_ms":999986,"credit_policy":null,"job_id":"job-3467","job_issued_at_ms":1000000,"miner_id":"miner-2","network_difficulty":1000016,"ntime":1700003467,"order_key":"02:miner-2","p2mr_program_hex":"8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b","share_difficulty":7,"share_id":"share-3467","share_seq":3467,"template_height":800108},{"accepted_at_ms":999993,"credit_policy":null,"job_id":"job-1086","job_issued_at_ms":999976,"miner_id":"miner-8","network_difficulty":1000015,"ntime":1700001086,"order_key":"08:miner-8","p2mr_program_hex":"3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e","share_difficulty":1000,"share_id":"share-1086","share_seq":1086,"template_height":800033},{"accepted_at_ms":999993,"credit_policy":"stale-grace","job_id":"job-2600","job_issued_at_ms":999987,"miner_id":"miner-4","network_difficulty":1000016,"ntime":1700002600,"order_key":"04:miner-4","p2mr_program_hex":"2828282828282828282828282828282828282828282828282828282828282828","share_difficulty":101,"share_id":"share-2600","share_seq":2600,"template_height":800081},{"accepted_at_ms":999989,"credit_policy":null,"job_id":"job-2200","job_issued_at_ms":999994,"miner_id":"miner-0","network_difficulty":1000007,"ntime":1700002200,"order_key":"00:miner-0","p2mr_program_hex":"9898989898989898989898989898989898989898989898989898989898989898","share_difficulty":101,"share_id":"share-2200","share_seq":2200,"template_height":800068},{"accepted_at_ms":1000129,"credit_policy":null,"job_id":"job-861","job_issued_at_ms":999971,"miner_id":"miner-3","network_difficulty":1000011,"ntime":1700000861,"order_key":"03:miner-3","p2mr_program_hex":"5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d","share_difficulty":7,"share_id":"share-861","share_seq":861,"template_height":800026},{"accepted_at_ms":999987,"credit_policy":null,"job_id":"job-757","job_issued_at_ms":999987,"miner_id":"miner-9","network_difficulty":1000009,"ntime":1700000757,"order_key":"09:miner-9","p2mr_program_hex":"f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5","share_difficulty":3,"share_id":"share-757","share_seq":757,"template_height":800023},{"accepted_at_ms":999970,"credit_policy":null,"job_id":"job-497","job_issued_at_ms":999983,"miner_id":"miner-2","network_difficulty":1000004,"ntime":1700000497,"order_key":"02:miner-2","p2mr_program_hex":"f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1","share_difficulty":7,"share_id":"share-497","share_seq":497,"template_height":800015},{"accepted_at_ms":1000346,"credit_policy":"stale-grace","job_id":"job-1207","job_issued_at_ms":1000246,"miner_id":"miner-8","network_difficulty":1000000,"ntime":1700001207,"order_key":"08:miner-8","p2mr_program_hex":"b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7","share_difficulty":7,"share_id":"share-1207","share_seq":1207,"template_height":800037},{"accepted_at_ms":999972,"credit_policy":null,"job_id":"job-2528","job_issued_at_ms":999989,"miner_id":"miner-9","network_difficulty":1000012,"ntime":1700002528,"order_key":"09:miner-9","p2mr_program_hex":"e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0","share_difficulty":3,"share_id":"share-2528","share_seq":2528,"template_height":800079},{"accepted_at_ms":999981,"credit_policy":null,"job_id":"job-1796","job_issued_at_ms":1000000,"miner_id":"miner-3","network_difficulty":1000011,"ntime":1700001796,"order_key":"03:miner-3","p2mr_program_hex":"0404040404040404040404040404040404040404040404040404040404040404","share_difficulty":19,"share_id":"share-1796","share_seq":1796,"template_height":800056},{"accepted_at_ms":999995,"credit_policy":"stale-grace","job_id":"job-3798","job_issued_at_ms":999989,"miner_id":"miner-3","network_difficulty":1000007,"ntime":1700003798,"order_key":"03:miner-3","p2mr_program_hex":"d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6","share_difficulty":1000,"share_id":"share-3798","share_seq":3798,"template_height":800118},{"accepted_at_ms":999974,"credit_policy":null,"job_id":"job-1025","job_issued_at_ms":999974,"miner_id":"miner-2","network_difficulty":1000005,"ntime":1700001025,"order_key":"02:miner-2","p2mr_program_hex":"0101010101010101010101010101010101010101010101010101010101010101","share_difficulty":1,"share_id":"share-1025","share_seq":1025,"template_height":800032},{"accepted_at_ms":999979,"credit_policy":null,"job_id":"job-1165","job_issued_at_ms":999985,"miner_id":"miner-10","network_difficulty":1000009,"ntime":1700001165,"order_key":"10:miner-10","p2mr_program_hex":"8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d","share_difficulty":101,"share_id":"share-1165","share_seq":1165,"template_height":800036},{"accepted_at_ms":999972,"credit_policy":null,"job_id":"job-3584","job_issued_at_ms":999989,"miner_id":"miner-9","network_difficulty":1000014,"ntime":1700003584,"order_key":"09:miner-9","p2mr_program_hex":"0000000000000000000000000000000000000000000000000000000000000000","share_difficulty":1000,"share_id":"share-3584","share_seq":3584,"template_height":800112},{"accepted_at_ms":999993,"credit_policy":"stale-grace","job_id":"job-2508","job_issued_at_ms":999987,"miner_id":"miner-0","network_difficulty":1000009,"ntime":1700002508,"order_key":"00:miner-0","p2mr_program_hex":"cccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccc","share_difficulty":7,"share_id":"share-2508","share_seq":2508,"template_height":800078},{"accepted_at_ms":1000236,"credit_policy":null,"job_id":"job-3244","job_issued_at_ms":999978,"miner_id":"miner-10","network_difficulty":1000014,"ntime":1700003244,"order_key":"10:miner-10","p2mr_program_hex":"acacacacacacacacacacacacacacacacacacacacacacacacacacacacacacacac","share_difficulty":19,"share_id":"share-3244","share_seq":3244,"template_height":800101},{"accepted_at_ms":999999,"credit_policy":null,"job_id":"job-1738","job_issued_at_ms":999978,"miner_id":"miner-0","network_difficulty":1000004,"ntime":1700001738,"order_key":"00:miner-0","p2mr_program_hex":"cacacacacacacacacacacacacacacacacacacacacacacacacacacacacacacaca","share_difficulty":3,"share_id":"share-1738","share_seq":1738,"template_height":800054},{"accepted_at_ms":999982,"credit_policy":null,"job_id":"job-2839","job_issued_at_ms":999985,"miner_id":"miner-1","network_difficulty":1000000,"ntime":1700002839,"order_key":"01:miner-1","p2mr_program_hex":"1717171717171717171717171717171717171717171717171717171717171717","share_difficulty":1000,"share_id":"share-2839","share_seq":2839,"template_height":800088},{"accepted_at_ms":999996,"credit_policy":null,"job_id":"job-2658","job_issued_at_ms":999972,"miner_id":"miner-7","network_difficulty":1000006,"ntime":1700002658,"order_key":"07:miner-7","p2mr_program_hex":"6262626262626262626262626262626262626262626262626262626262626262","share_difficulty":101,"share_id":"share-2658","share_seq":2658,"template_height":800083},{"accepted_at_ms":999980,"credit_policy":null,"job_id":"job-3927","job_issued_at_ms":999997,"miner_id":"miner-0","network_difficulty":1000000,"ntime":1700003927,"order_key":"00:miner-0","p2mr_program_hex":"5757575757575757575757575757575757575757575757575757575757575757","share_difficulty":1000,"share_id":"share-3927","share_seq":3927,"template_height":800122},{"accepted_at_ms":1000102,"credit_policy":null,"job_id":"job-582","job_issued_at_ms":1000424,"miner_id":"miner-10","network_difficulty":1000004,"ntime":1700000582,"order_key":"10:miner-10","p2mr_program_hex":"4646464646464646464646464646464646464646464646464646464646464646","share_difficulty":101,"share_id":"share-582","share_seq":582,"template_height":800018},{"accepted_at_ms":999989,"credit_policy":null,"job_id":"job-3919","job_issued_at_ms":999997,"miner_id":"miner-3","network_difficulty":1000009,"ntime":1700003919,"order_key":"03:miner-3","p2mr_program_hex":"4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f","share_difficulty":1,"share_id":"share-3919","share_seq":3919,"template_height":800122},{"accepted_at_ms":999976,"credit_policy":null,"job_id":"job-2034","job_issued_at_ms":999987,"miner_id":"miner-10","network_difficulty":1000011,"ntime":1700002034,"order_key":"10:miner-10","p2mr_program_hex":"f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2","share_difficulty":7,"share_id":"share-2034","share_seq":2034,"template_height":800063},{"accepted_at_ms":999988,"credit_policy":null,"job_id":"job-1770","job_issued_at_ms":999978,"miner_id":"miner-10","network_difficulty":1000002,"ntime":1700001770,"order_key":"10:miner-10","p2mr_program_hex":"eaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaea","share_difficulty":3,"share_id":"share-1770","share_seq":1770,"template_height":800055},{"accepted_at_ms":999993,"credit_policy":null,"job_id":"job-2366","job_issued_at_ms":999978,"miner_id":"miner-1","network_difficulty":1000003,"ntime":1700002366,"order_key":"01:miner-1","p2mr_program_hex":"3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e","share_difficulty":7,"share_id":"share-2366","share_seq":2366,"template_height":800073},{"accepted_at_ms":999998,"credit_policy":"stale-grace","job_id":"job-300","job_issued_at_ms":999999,"miner_id":"miner-3","network_difficulty":1000011,"ntime":1700000300,"order_key":"03:miner-3","p2mr_program_hex":"2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c","share_difficulty":19,"share_id":"share-300","share_seq":300,"template_height":800009},{"accepted_at_ms":1000410,"credit_policy":null,"job_id":"job-652","job_issued_at_ms":1000020,"miner_id":"miner-3","network_difficulty":1000006,"ntime":1700000652,"order_key":"03:miner-3","p2mr_program_hex":"8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c","share_difficulty":1000,"share_id":"share-652","share_seq":652,"template_height":800020},{"accepted_at_ms":999996,"credit_policy":null,"job_id":"job-554","job_issued_at_ms":999976,"miner_id":"miner-4","network_difficulty":1000010,"ntime":1700000554,"order_key":"04:miner-4","p2mr_program_hex":"2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a","share_difficulty":19,"share_id":"share-554","share_seq":554,"template_height":800017},{"accepted_at_ms":999987,"credit_policy":null,"job_id":"job-3926","job_issued_at_ms":999971,"miner_id":"miner-10","network_difficulty":1000016,"ntime":1700003926,"order_key":"10:miner-10","p2mr_program_hex":"5656565656565656565656565656565656565656565656565656565656565656","share_difficulty":1,"share_id":"share-3926","share_seq":3926,"template_height":800122},{"accepted_at_ms":999997,"credit_policy":null,"job_id":"job-1130","job_issued_at_ms":999997,"miner_id":"miner-8","network_difficulty":1000008,"ntime":1700001130,"order_key":"08:miner-8","p2mr_program_hex":"6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a","share_difficulty":7,"share_id":"share-1130","share_seq":1130,"template_height":800035},{"accepted_at_ms":999983,"credit_policy":null,"job_id":"job-3207","job_issued_at_ms":999988,"miner_id":"miner-6","network_difficulty":1000011,"ntime":1700003207,"order_key":"06:miner-6","p2mr_program_hex":"8787878787878787878787878787878787878787878787878787878787878787","share_difficulty":1,"share_id":"share-3207","share_seq":3207,"template_height":800100},{"accepted_at_ms":999988,"credit_policy":null,"job_id":"job-2440","job_issued_at_ms":999992,"miner_id":"miner-9","network_difficulty":1000009,"ntime":1700002440,"order_key":"09:miner-9","p2mr_program_hex":"8888888888888888888888888888888888888888888888888888888888888888","share_difficulty":7,"share_id":"share-2440","share_seq":2440,"template_height":800076},{"accepted_at_ms":999980,"credit_policy":"stale-grace","job_id":"job-2680","job_issued_at_ms":999974,"miner_id":"miner-7","network_difficulty":1000011,"ntime":1700002680,"order_key":"07:miner-7","p2mr_program_hex":"7878787878787878787878787878787878787878787878787878787878787878","share_difficulty":3,"share_id":"share-2680","share_seq":2680,"template_height":800083},{"accepted_at_ms":999981,"credit_policy":null,"job_id":"job-3076","job_issued_at_ms":999980,"miner_id":"miner-7","network_difficulty":1000016,"ntime":1700003076,"order_key":"07:miner-7","p2mr_program_hex":"0404040404040404040404040404040404040404040404040404040404040404","share_difficulty":3,"share_id":"share-3076","share_seq":3076,"template_height":800096},{"accepted_at_ms":999996,"credit_policy":null,"job_id":"job-1137","job_issued_at_ms":999972,"miner_id":"miner-4","network_difficulty":1000015,"ntime":1700001137,"order_key":"04:miner-4","p2mr_program_hex":"7171717171717171717171717171717171717171717171717171717171717171","share_difficulty":101,"share_id":"share-1137","share_seq":1137,"template_height":800035},{"accepted_at_ms":999999,"credit_policy":null,"job_id":"job-1367","job_issued_at_ms":1000000,"miner_id":"miner-3","network_difficulty":1000007,"ntime":1700001367,"order_key":"03:miner-3","p2mr_program_hex":"5757575757575757575757575757575757575757575757575757575757575757","share_difficulty":101,"share_id":"share-1367","share_seq":1367,"template_height":800042},{"accepted_at_ms":999996,"credit_policy":"stale-grace","job_id":"job-2510","job_issued_at_ms":999979,"miner_id":"miner-2","network_difficulty":1000011,"ntime":1700002510,"order_key":"02:miner-2","p2mr_program_hex":"cececececececececececececececececececececececececececececececece","share_difficulty":1,"share_id":"share-2510","share_seq":2510,"template_height":800078},{"accepted_at_ms":999978,"credit_policy":null,"job_id":"job-516","job_issued_at_ms":999993,"miner_id":"miner-10","network_difficulty":1000006,"ntime":1700000516,"order_key":"10:miner-10","p2mr_program_hex":"0404040404040404040404040404040404040404040404040404040404040404","share_difficulty":7,"share_id":"share-516","share_seq":516,"template_height":800016},{"accepted_at_ms":999988,"credit_policy":null,"job_id":"job-3856","job_issued_at_ms":999979,"miner_id":"miner-6","network_difficulty":1000014,"ntime":1700003856,"order_key":"06:miner-6","p2mr_program_hex":"1010101010101010101010101010101010101010101010101010101010101010","share_difficulty":7,"share_id":"share-3856","share_seq":3856,"template_height":800120},{"accepted_at_ms":999989,"credit_policy":null,"job_id":"job-1303","job_issued_at_ms":999997,"miner_id":"miner-5","network_difficulty":1000011,"ntime":1700001303,"order_key":"05:miner-5","p2mr_program_hex":"1717171717171717171717171717171717171717171717171717171717171717","share_difficulty":1000,"share_id":"share-1303","share_seq":1303,"template_height":800040},{"accepted_at_ms":999970,"credit_policy":null,"job_id":"job-985","job_issued_at_ms":999989,"miner_id":"miner-6","network_difficulty":1000016,"ntime":1700000985,"order_key":"06:miner-6","p2mr_program_hex":"d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9","share_difficulty":2,"share_id":"share-985","share_seq":985,"template_height":800030},{"accepted_at_ms":999981,"credit_policy":null,"job_id":"job-2196","job_issued_at_ms":999984,"miner_id":"miner-7","network_difficulty":1000003,"ntime":1700002196,"order_key":"07:miner-7","p2mr_program_hex":"9494949494949494949494949494949494949494949494949494949494949494","share_difficulty":1000,"share_id":"share-2196","share_seq":2196,"template_height":800068},{"accepted_at_ms":999993,"credit_policy":null,"job_id":"job-265","job_issued_at_ms":999973,"miner_id":"miner-1","network_difficulty":1000010,"ntime":1700000265,"order_key":"01:miner-1","p2mr_program_hex":"0909090909090909090909090909090909090909090909090909090909090909","share_difficulty":2,"share_id":"share-265","share_seq":265,"template_height":800008},{"accepted_at_ms":999997,"credit_policy":null,"job_id":"job-37","job_issued_at_ms":999998,"miner_id":"miner-4","network_difficulty":1000003,"ntime":1700000037,"order_key":"04:miner-4","p2mr_program_hex":"2525252525252525252525252525252525252525252525252525252525252525","share_difficulty":1,"share_id":"share-37","share_seq":37,"template_height":800001},{"accepted_at_ms":999984,"credit_policy":"stale-grace","job_id":"job-186","job_issued_at_ms":999984,"miner_id":"miner-10","network_difficulty":1000016,"ntime":1700000186,"order_key":"10:miner-10","p2mr_program_hex":"babababababababababababababababababababababababababababababababa","share_difficulty":19,"share_id":"share-186","share_seq":186,"template_height":800005},{"accepted_at_ms":999974,"credit_policy":null,"job_id":"job-2291","job_issued_at_ms":999993,"miner_id":"miner-3","network_difficulty":1000013,"ntime":1700002291,"order_key":"03:miner-3","p2mr_program_hex":"f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3","share_difficulty":1000,"share_id":"share-2291","share_seq":2291,"template_height":800071},{"accepted_at_ms":1000358,"credit_policy":null,"job_id":"job-2249","job_issued_at_ms":1000432,"miner_id":"miner-5","network_difficulty":1000005,"ntime":1700002249,"order_key":"05:miner-5","p2mr_program_hex":"c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9","share_difficulty":3,"share_id":"share-2249","share_seq":2249,"template_height":800070},{"accepted_at_ms":999979,"credit_policy":null,"job_id":"job-724","job_issued_at_ms":999985,"miner_id":"miner-9","network_difficulty":1000010,"ntime":1700000724,"order_key":"09:miner-9","p2mr_program_hex":"d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4","share_difficulty":101,"share_id":"share-724","share_seq":724,"template_height":800022},{"accepted_at_ms":999988,"credit_policy":null,"job_id":"job-1222","job_issued_at_ms":999976,"miner_id":"miner-1","network_difficulty":1000015,"ntime":1700001222,"order_key":"01:miner-1","p2mr_program_hex":"c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6","share_difficulty":1,"share_id":"share-1222","share_seq":1222,"template_height":800038},{"accepted_at_ms":999988,"credit_policy":"stale-grace","job_id":"job-1863","job_issued_at_ms":999988,"miner_id":"miner-4","network_difficulty":1000010,"ntime":1700001863,"order_key":"04:miner-4","p2mr_program_hex":"4747474747474747474747474747474747474747474747474747474747474747","share_difficulty":7,"share_id":"share-1863","share_seq":1863,"template_height":800058},{"accepted_at_ms":1000000,"credit_policy":null,"job_id":"job-128","job_issued_at_ms":999981,"miner_id":"miner-7","network_difficulty":1000009,"ntime":1700000128,"order_key":"07:miner-7","p2mr_program_hex":"8080808080808080808080808080808080808080808080808080808080808080","share_difficulty":1000,"share_id":"share-128","share_seq":128,"template_height":800004},{"accepted_at_ms":999977,"credit_policy":"stale-grace","job_id":"job-2886","job_issued_at_ms":999978,"miner_id":"miner-4","network_difficulty":1000013,"ntime":1700002886,"order_key":"04:miner-4","p2mr_program_hex":"4646464646464646464646464646464646464646464646464646464646464646","share_difficulty":1,"share_id":"share-2886","share_seq":2886,"template_height":800090},{"accepted_at_ms":999973,"credit_policy":null,"job_id":"job-1361","job_issued_at_ms":999987,"miner_id":"miner-8","network_difficulty":1000001,"ntime":1700001361,"order_key":"08:miner-8","p2mr_program_hex":"5151515151515151515151515151515151515151515151515151515151515151","share_difficulty":1,"share_id":"share-1361","share_seq":1361,"template_height":800042},{"accepted_at_ms":999975,"credit_policy":"stale-grace","job_id":"job-1279","job_issued_at_ms":999991,"miner_id":"miner-3","network_difficulty":1000004,"ntime":1700001279,"order_key":"03:miner-3","p2mr_program_hex":"ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff","share_difficulty":7,"share_id":"share-1279","share_seq":1279,"template_height":800039},{"accepted_at_ms":999995,"credit_policy":null,"job_id":"job-2456","job_issued_at_ms":1000000,"miner_id":"miner-3","network_difficulty":1000008,"ntime":1700002456,"order_key":"03:miner-3","p2mr_program_hex":"9898989898989898989898989898989898989898989898989898989898989898","share_difficulty":1000,"share_id":"share-2456","share_seq":2456,"template_height":800076},{"accepted_at_ms":999999,"credit_policy":null,"job_id":"job-1542","job_issued_at_ms":999981,"miner_id":"miner-2","network_difficulty":1000012,"ntime":1700001542,"order_key":"02:miner-2","p2mr_program_hex":"0606060606060606060606060606060606060606060606060606060606060606","share_difficulty":7,"share_id":"share-1542","share_seq":1542,"template_height":800048},{"accepted_at_ms":999996,"credit_policy":null,"job_id":"job-1418","job_issued_at_ms":999976,"miner_id":"miner-10","network_difficulty":1000007,"ntime":1700001418,"order_key":"10:miner-10","p2mr_program_hex":"8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a","share_difficulty":19,"share_id":"share-1418","share_seq":1418,"template_height":800044},{"accepted_at_ms":999987,"credit_policy":null,"job_id":"job-1699","job_issued_at_ms":999983,"miner_id":"miner-5","network_difficulty":1000016,"ntime":1700001699,"order_key":"05:miner-5","p2mr_program_hex":"a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3","share_difficulty":3,"share_id":"share-1699","share_seq":1699,"template_height":800053},{"accepted_at_ms":999990,"credit_policy":null,"job_id":"job-1657","job_issued_at_ms":999993,"miner_id":"miner-7","network_difficulty":1000008,"ntime":1700001657,"order_key":"07:miner-7","p2mr_program_hex":"7979797979797979797979797979797979797979797979797979797979797979","share_difficulty":2,"share_id":"share-1657","share_seq":1657,"template_height":800051},{"accepted_at_ms":999986,"credit_policy":null,"job_id":"job-441","job_issued_at_ms":999983,"miner_id":"miner-1","network_difficulty":1000016,"ntime":1700000441,"order_key":"01:miner-1","p2mr_program_hex":"b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9","share_difficulty":7,"share_id":"share-441","share_seq":441,"template_height":800013},{"accepted_at_ms":999979,"credit_policy":null,"job_id":"job-1046","job_issued_at_ms":999988,"miner_id":"miner-1","network_difficulty":1000009,"ntime":1700001046,"order_key":"01:miner-1","p2mr_program_hex":"1616161616161616161616161616161616161616161616161616161616161616","share_difficulty":1,"share_id":"share-1046","share_seq":1046,"template_height":800032},{"accepted_at_ms":1000000,"credit_policy":null,"job_id":"job-751","job_issued_at_ms":999992,"miner_id":"miner-3","network_difficulty":1000003,"ntime":1700000751,"order_key":"03:miner-3","p2mr_program_hex":"efefefefefefefefefefefefefefefefefefefefefefefefefefefefefefefef","share_difficulty":101,"share_id":"share-751","share_seq":751,"template_height":800023},{"accepted_at_ms":999987,"credit_policy":"stale-grace","job_id":"job-658","job_issued_at_ms":999983,"miner_id":"miner-9","network_difficulty":1000012,"ntime":1700000658,"order_key":"09:miner-9","p2mr_program_hex":"9292929292929292929292929292929292929292929292929292929292929292","share_difficulty":19,"share_id":"share-658","share_seq":658,"template_height":800020},{"accepted_at_ms":999970,"credit_policy":"stale-grace","job_id":"job-3450","job_issued_at_ms":1000148,"miner_id":"miner-7","network_difficulty":1000016,"ntime":1700003450,"order_key":"07:miner-7","p2mr_program_hex":"7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a","share_difficulty":1,"share_id":"share-3450","share_seq":3450,"template_height":800107},{"accepted_at_ms":999993,"credit_policy":"stale-grace","job_id":"job-1846","job_issued_at_ms":999984,"miner_id":"miner-9","network_difficulty":1000010,"ntime":1700001846,"order_key":"09:miner-9","p2mr_program_hex":"3636363636363636363636363636363636363636363636363636363636363636","share_difficulty":3,"share_id":"share-1846","share_seq":1846,"template_height":800057},{"accepted_at_ms":999992,"credit_policy":null,"job_id":"job-880","job_issued_at_ms":999988,"miner_id":"miner-0","network_difficulty":1000013,"ntime":1700000880,"order_key":"00:miner-0","p2mr_program_hex":"7070707070707070707070707070707070707070707070707070707070707070","share_difficulty":101,"share_id":"share-880","share_seq":880,"template_height":800027},{"accepted_at_ms":999994,"credit_policy":null,"job_id":"job-1831","job_issued_at_ms":999987,"miner_id":"miner-5","network_difficulty":1000012,"ntime":1700001831,"order_key":"05:miner-5","p2mr_program_hex":"2727272727272727272727272727272727272727272727272727272727272727","share_difficulty":1,"share_id":"share-1831","share_seq":1831,"template_height":800057},{"accepted_at_ms":999983,"credit_policy":null,"job_id":"job-1908","job_issued_at_ms":999976,"miner_id":"miner-5","network_difficulty":1000004,"ntime":1700001908,"order_key":"05:miner-5","p2mr_program_hex":"7474747474747474747474747474747474747474747474747474747474747474","share_difficulty":1000,"share_id":"share-1908","share_seq":1908,"template_height":800059},{"accepted_at_ms":999979,"credit_policy":null,"job_id":"job-1946","job_issued_at_ms":999970,"miner_id":"miner-10","network_difficulty":1000008,"ntime":1700001946,"order_key":"10:miner-10","p2mr_program_hex":"9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a","share_difficulty":101,"share_id":"share-1946","share_seq":1946,"template_height":800060},{"accepted_at_ms":999979,"credit_policy":null,"job_id":"job-278","job_issued_at_ms":999989,"miner_id":"miner-3","network_difficulty":1000006,"ntime":1700000278,"order_key":"03:miner-3","p2mr_program_hex":"1616161616161616161616161616161616161616161616161616161616161616","share_difficulty":7,"share_id":"share-278","share_seq":278,"template_height":800008},{"accepted_at_ms":999986,"credit_policy":null,"job_id":"job-990","job_issued_at_ms":999971,"miner_id":"miner-0","network_difficulty":1000004,"ntime":1700000990,"order_key":"00:miner-0","p2mr_program_hex":"dededededededededededededededededededededededededededededededede","share_difficulty":7,"share_id":"share-990","share_seq":990,"template_height":800030},{"accepted_at_ms":999999,"credit_policy":"stale-grace","job_id":"job-426","job_issued_at_ms":999974,"miner_id":"miner-8","network_difficulty":1000001,"ntime":1700000426,"order_key":"08:miner-8","p2mr_program_hex":"aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa","share_difficulty":19,"share_id":"share-426","share_seq":426,"template_height":800013},{"accepted_at_ms":999984,"credit_policy":null,"job_id":"job-1251","job_issued_at_ms":999977,"miner_id":"miner-8","network_difficulty":1000010,"ntime":1700001251,"order_key":"08:miner-8","p2mr_program_hex":"e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3","share_difficulty":3,"share_id":"share-1251","share_seq":1251,"template_height":800039},{"accepted_at_ms":999990,"credit_policy":null,"job_id":"job-3352","job_issued_at_ms":999984,"miner_id":"miner-8","network_difficulty":1000003,"ntime":1700003352,"order_key":"08:miner-8","p2mr_program_hex":"1818181818181818181818181818181818181818181818181818181818181818","share_difficulty":101,"share_id":"share-3352","share_seq":3352,"template_height":800104},{"accepted_at_ms":999993,"credit_policy":null,"job_id":"job-2393","job_issued_at_ms":999999,"miner_id":"miner-6","network_difficulty":1000013,"ntime":1700002393,"order_key":"06:miner-6","p2mr_program_hex":"5959595959595959595959595959595959595959595959595959595959595959","share_difficulty":1,"share_id":"share-2393","share_seq":2393,"template_height":800074},{"accepted_at_ms":999972,"credit_policy":null,"job_id":"job-3888","job_issued_at_ms":999970,"miner_id":"miner-5","network_difficulty":1000012,"ntime":1700003888,"order_key":"05:miner-5","p2mr_program_hex":"3030303030303030303030303030303030303030303030303030303030303030","share_difficulty":101,"share_id":"share-3888","share_seq":3888,"template_height":800121},{"accepted_at_ms":999975,"credit_policy":null,"job_id":"job-3704","job_issued_at_ms":1000000,"miner_id":"miner-8","network_difficulty":1000015,"ntime":1700003704,"order_key":"08:miner-8","p2mr_program_hex":"7878787878787878787878787878787878787878787878787878787878787878","share_difficulty":7,"share_id":"share-3704","share_seq":3704,"template_height":800115},{"accepted_at_ms":999977,"credit_policy":null,"job_id":"job-156","job_issued_at_ms":999977,"miner_id":"miner-2","network_difficulty":1000003,"ntime":1700000156,"order_key":"02:miner-2","p2mr_program_hex":"9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c","share_difficulty":1000,"share_id":"share-156","share_seq":156,"template_height":800004},{"accepted_at_ms":999984,"credit_policy":null,"job_id":"job-199","job_issued_at_ms":1000000,"miner_id":"miner-1","network_difficulty":1000012,"ntime":1700000199,"order_key":"01:miner-1","p2mr_program_hex":"c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7","share_difficulty":101,"share_id":"share-199","share_seq":199,"template_height":800006},{"accepted_at_ms":999987,"credit_policy":"stale-grace","job_id":"job-760","job_issued_at_ms":999986,"miner_id":"miner-1","network_difficulty":1000012,"ntime":1700000760,"order_key":"01:miner-1","p2mr_program_hex":"f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8","share_difficulty":3,"share_id":"share-760","share_seq":760,"template_height":800023},{"accepted_at_ms":999988,"credit_policy":null,"job_id":"job-311","job_issued_at_ms":999979,"miner_id":"miner-3","network_difficulty":1000005,"ntime":1700000311,"order_key":"03:miner-3","p2mr_program_hex":"3737373737373737373737373737373737373737373737373737373737373737","share_difficulty":2,"share_id":"share-311","share_seq":311,"template_height":800009},{"accepted_at_ms":999978,"credit_policy":null,"job_id":"job-3672","job_issued_at_ms":999989,"miner_id":"miner-9","network_difficulty":1000000,"ntime":1700003672,"order_key":"09:miner-9","p2mr_program_hex":"5858585858585858585858585858585858585858585858585858585858585858","share_difficulty":3,"share_id":"share-3672","share_seq":3672,"template_height":800114},{"accepted_at_ms":999979,"credit_policy":"stale-grace","job_id":"job-3027","job_issued_at_ms":999996,"miner_id":"miner-2","network_difficulty":1000001,"ntime":1700003027,"order_key":"02:miner-2","p2mr_program_hex":"d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3","share_difficulty":7,"share_id":"share-3027","share_seq":3027,"template_height":800094},{"accepted_at_ms":1000259,"credit_policy":null,"job_id":"job-893","job_issued_at_ms":999997,"miner_id":"miner-2","network_difficulty":1000009,"ntime":1700000893,"order_key":"02:miner-2","p2mr_program_hex":"7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d","share_difficulty":2,"share_id":"share-893","share_seq":893,"template_height":800027},{"accepted_at_ms":999993,"credit_policy":"stale-grace","job_id":"job-771","job_issued_at_ms":999982,"miner_id":"miner-1","network_difficulty":1000006,"ntime":1700000771,"order_key":"01:miner-1","p2mr_program_hex":"0303030303030303030303030303030303030303030303030303030303030303","share_difficulty":101,"share_id":"share-771","share_seq":771,"template_height":800024},{"accepted_at_ms":999983,"credit_policy":null,"job_id":"job-2307","job_issued_at_ms":999998,"miner_id":"miner-8","network_difficulty":1000012,"ntime":1700002307,"order_key":"08:miner-8","p2mr_program_hex":"0303030303030303030303030303030303030303030303030303030303030303","share_difficulty":2,"share_id":"share-2307","share_seq":2307,"template_height":800072},{"accepted_at_ms":999984,"credit_policy":"stale-grace","job_id":"job-3494","job_issued_at_ms":999997,"miner_id":"miner-7","network_difficulty":1000009,"ntime":1700003494,"order_key":"07:miner-7","p2mr_program_hex":"a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6","share_difficulty":3,"share_id":"share-3494","share_seq":3494,"template_height":800109},{"accepted_at_ms":999987,"credit_policy":"stale-grace","job_id":"job-1952","job_issued_at_ms":999979,"miner_id":"miner-5","network_difficulty":1000014,"ntime":1700001952,"order_key":"05:miner-5","p2mr_program_hex":"a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0","share_difficulty":2,"share_id":"share-1952","share_seq":1952,"template_height":800061},{"accepted_at_ms":999972,"credit_policy":null,"job_id":"job-1722","job_issued_at_ms":999990,"miner_id":"miner-6","network_difficulty":1000005,"ntime":1700001722,"order_key":"06:miner-6","p2mr_program_hex":"babababababababababababababababababababababababababababababababa","share_difficulty":2,"share_id":"share-1722","share_seq":1722,"template_height":800053},{"accepted_at_ms":999980,"credit_policy":null,"job_id":"job-3319","job_issued_at_ms":999989,"miner_id":"miner-8","network_difficulty":1000004,"ntime":1700003319,"order_key":"08:miner-8","p2mr_program_hex":"f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7","share_difficulty":19,"share_id":"share-3319","share_seq":3319,"template_height":800103},{"accepted_at_ms":999975,"credit_policy":null,"job_id":"job-710","job_issued_at_ms":999971,"miner_id":"miner-6","network_difficulty":1000013,"ntime":1700000710,"order_key":"06:miner-6","p2mr_program_hex":"c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6","share_difficulty":3,"share_id":"share-710","share_seq":710,"template_height":800022},{"accepted_at_ms":999981,"credit_policy":null,"job_id":"job-3715","job_issued_at_ms":999971,"miner_id":"miner-8","network_difficulty":1000009,"ntime":1700003715,"order_key":"08:miner-8","p2mr_program_hex":"8383838383838383838383838383838383838383838383838383838383838383","share_difficulty":7,"share_id":"share-3715","share_seq":3715,"template_height":800116},{"accepted_at_ms":999993,"credit_policy":null,"job_id":"job-3784","job_issued_at_ms":999972,"miner_id":"miner-0","network_difficulty":1000010,"ntime":1700003784,"order_key":"00:miner-0","p2mr_program_hex":"c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8","share_difficulty":101,"share_id":"share-3784","share_seq":3784,"template_height":800118},{"accepted_at_ms":999991,"credit_policy":null,"job_id":"job-354","job_issued_at_ms":999992,"miner_id":"miner-2","network_difficulty":1000014,"ntime":1700000354,"order_key":"02:miner-2","p2mr_program_hex":"6262626262626262626262626262626262626262626262626262626262626262","share_difficulty":1000,"share_id":"share-354","share_seq":354,"template_height":800011},{"accepted_at_ms":999995,"credit_policy":null,"job_id":"job-2489","job_issued_at_ms":999985,"miner_id":"miner-3","network_difficulty":1000007,"ntime":1700002489,"order_key":"03:miner-3","p2mr_program_hex":"b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9","share_difficulty":101,"share_id":"share-2489","share_seq":2489,"template_height":800077},{"accepted_at_ms":1000317,"credit_policy":null,"job_id":"job-2704","job_issued_at_ms":999974,"miner_id":"miner-9","network_difficulty":1000001,"ntime":1700002704,"order_key":"09:miner-9","p2mr_program_hex":"9090909090909090909090909090909090909090909090909090909090909090","share_difficulty":101,"share_id":"share-2704","share_seq":2704,"template_height":800084},{"accepted_at_ms":999971,"credit_policy":null,"job_id":"job-1309","job_issued_at_ms":999975,"miner_id":"miner-0","network_difficulty":1000000,"ntime":1700001309,"order_key":"00:miner-0","p2mr_program_hex":"1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d","share_difficulty":3,"share_id":"share-1309","share_seq":1309,"template_height":800040},{"accepted_at_ms":999976,"credit_policy":null,"job_id":"job-1587","job_issued_at_ms":999978,"miner_id":"miner-3","network_difficulty":1000006,"ntime":1700001587,"order_key":"03:miner-3","p2mr_program_hex":"3333333333333333333333333333333333333333333333333333333333333333","share_difficulty":101,"share_id":"share-1587","share_seq":1587,"template_height":800049},{"accepted_at_ms":999992,"credit_policy":null,"job_id":"job-1776","job_issued_at_ms":999972,"miner_id":"miner-5","network_difficulty":1000008,"ntime":1700001776,"order_key":"05:miner-5","p2mr_program_hex":"f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0","share_difficulty":2,"share_id":"share-1776","share_seq":1776,"template_height":800055},{"accepted_at_ms":999990,"credit_policy":null,"job_id":"job-150","job_issued_at_ms":1000000,"miner_id":"miner-7","network_difficulty":1000014,"ntime":1700000150,"order_key":"07:miner-7","p2mr_program_hex":"9696969696969696969696969696969696969696969696969696969696969696","share_difficulty":101,"share_id":"share-150","share_seq":150,"template_height":800004},{"accepted_at_ms":999990,"credit_policy":"stale-grace","job_id":"job-2827","job_issued_at_ms":999970,"miner_id":"miner-0","network_difficulty":1000005,"ntime":1700002827,"order_key":"00:miner-0","p2mr_program_hex":"0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b","share_difficulty":3,"share_id":"share-2827","share_seq":2827,"template_height":800088},{"accepted_at_ms":999988,"credit_policy":null,"job_id":"job-3475","job_issued_at_ms":999979,"miner_id":"miner-10","network_difficulty":1000007,"ntime":1700003475,"order_key":"10:miner-10","p2mr_program_hex":"9393939393939393939393939393939393939393939393939393939393939393","share_difficulty":3,"share_id":"share-3475","share_seq":3475,"template_height":800108},{"accepted_at_ms":999982,"credit_policy":null,"job_id":"job-172","job_issued_at_ms":999998,"miner_id":"miner-7","network_difficulty":1000002,"ntime":1700000172,"order_key":"07:miner-7","p2mr_program_hex":"acacacacacacacacacacacacacacacacacacacacacacacacacacacacacacacac","share_difficulty":2,"share_id":"share-172","share_seq":172,"template_height":800005},{"accepted_at_ms":999986,"credit_policy":null,"job_id":"job-2736","job_issued_at_ms":999978,"miner_id":"miner-8","network_difficulty":1000016,"ntime":1700002736,"order_key":"08:miner-8","p2mr_program_hex":"b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0","share_difficulty":19,"share_id":"share-2736","share_seq":2736,"template_height":800085},{"accepted_at_ms":999986,"credit_policy":null,"job_id":"job-47","job_issued_at_ms":999975,"miner_id":"miner-3","network_difficulty":1000013,"ntime":1700000047,"order_key":"03:miner-3","p2mr_program_hex":"2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f","share_difficulty":3,"share_id":"share-47","share_seq":47,"template_height":800001},{"accepted_at_ms":999989,"credit_policy":null,"job_id":"job-925","job_issued_at_ms":999977,"miner_id":"miner-1","network_difficulty":1000007,"ntime":1700000925,"order_key":"01:miner-1","p2mr_program_hex":"9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d","share_difficulty":1,"share_id":"share-925","share_seq":925,"template_height":800028},{"accepted_at_ms":999988,"credit_policy":null,"job_id":"job-544","job_issued_at_ms":999984,"miner_id":"miner-5","network_difficulty":1000000,"ntime":1700000544,"order_key":"05:miner-5","p2mr_program_hex":"2020202020202020202020202020202020202020202020202020202020202020","share_difficulty":1000,"share_id":"share-544","share_seq":544,"template_height":800017},{"accepted_at_ms":999975,"credit_policy":null,"job_id":"job-1971","job_issued_at_ms":999972,"miner_id":"miner-2","network_difficulty":1000016,"ntime":1700001971,"order_key":"02:miner-2","p2mr_program_hex":"b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3","share_difficulty":3,"share_id":"share-1971","share_seq":1971,"template_height":800061},{"accepted_at_ms":999975,"credit_policy":null,"job_id":"job-2097","job_issued_at_ms":999999,"miner_id":"miner-7","network_difficulty":1000006,"ntime":1700002097,"order_key":"07:miner-7","p2mr_program_hex":"3131313131313131313131313131313131313131313131313131313131313131","share_difficulty":101,"share_id":"share-2097","share_seq":2097,"template_height":800065},{"accepted_at_ms":999993,"credit_policy":null,"job_id":"job-780","job_issued_at_ms":999996,"miner_id":"miner-10","network_difficulty":1000015,"ntime":1700000780,"order_key":"10:miner-10","p2mr_program_hex":"0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c","share_difficulty":101,"share_id":"share-780","share_seq":780,"template_height":800024},{"accepted_at_ms":999970,"credit_policy":null,"job_id":"job-3632","job_issued_at_ms":999972,"miner_id":"miner-2","network_difficulty":1000011,"ntime":1700003632,"order_key":"02:miner-2","p2mr_program_hex":"3030303030303030303030303030303030303030303030303030303030303030","share_difficulty":1,"share_id":"share-3632","share_seq":3632,"template_height":800113},{"accepted_at_ms":999976,"credit_policy":null,"job_id":"job-3465","job_issued_at_ms":999989,"miner_id":"miner-0","network_difficulty":1000014,"ntime":1700003465,"order_key":"00:miner-0","p2mr_program_hex":"8989898989898989898989898989898989898989898989898989898989898989","share_difficulty":101,"share_id":"share-3465","share_seq":3465,"template_height":800108},{"accepted_at_ms":999994,"credit_policy":"stale-grace","job_id":"job-428","job_issued_at_ms":999998,"miner_id":"miner-10","network_difficulty":1000003,"ntime":1700000428,"order_key":"10:miner-10","p2mr_program_hex":"acacacacacacacacacacacacacacacacacacacacacacacacacacacacacacacac","share_difficulty":1,"share_id":"share-428","share_seq":428,"template_height":800013},{"accepted_at_ms":999976,"credit_policy":null,"job_id":"job-2663","job_issued_at_ms":999976,"miner_id":"miner-1","network_difficulty":1000011,"ntime":1700002663,"order_key":"01:miner-1","p2mr_program_hex":"6767676767676767676767676767676767676767676767676767676767676767","share_difficulty":1,"share_id":"share-2663","share_seq":2663,"template_height":800083},{"accepted_at_ms":999972,"credit_policy":null,"job_id":"job-1907","job_issued_at_ms":999977,"miner_id":"miner-4","network_difficulty":1000003,"ntime":1700001907,"order_key":"04:miner-4","p2mr_program_hex":"7373737373737373737373737373737373737373737373737373737373737373","share_difficulty":3,"share_id":"share-1907","share_seq":1907,"template_height":800059},{"accepted_at_ms":999970,"credit_policy":null,"job_id":"job-3447","job_issued_at_ms":999997,"miner_id":"miner-4","network_difficulty":1000013,"ntime":1700003447,"order_key":"04:miner-4","p2mr_program_hex":"7777777777777777777777777777777777777777777777777777777777777777","share_difficulty":7,"share_id":"share-3447","share_seq":3447,"template_height":800107},{"accepted_at_ms":999991,"credit_policy":null,"job_id":"job-1215","job_issued_at_ms":999985,"miner_id":"miner-5","network_difficulty":1000008,"ntime":1700001215,"order_key":"05:miner-5","p2mr_program_hex":"bfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbf","share_difficulty":1,"share_id":"share-1215","share_seq":1215,"template_height":800037},{"accepted_at_ms":999997,"credit_policy":null,"job_id":"job-3415","job_issued_at_ms":999977,"miner_id":"miner-5","network_difficulty":1000015,"ntime":1700003415,"order_key":"05:miner-5","p2mr_program_hex":"5757575757575757575757575757575757575757575757575757575757575757","share_difficulty":1,"share_id":"share-3415","share_seq":3415,"template_height":800106},{"accepted_at_ms":1000230,"credit_policy":"stale-grace","job_id":"job-3032","job_issued_at_ms":999997,"miner_id":"miner-7","network_difficulty":1000006,"ntime":1700003032,"order_key":"07:miner-7","p2mr_program_hex":"d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8","share_difficulty":7,"share_id":"share-3032","share_seq":3032,"template_height":800094},{"accepted_at_ms":999977,"credit_policy":null,"job_id":"job-1089","job_issued_at_ms":999979,"miner_id":"miner-0","network_difficulty":1000001,"ntime":1700001089,"order_key":"00:miner-0","p2mr_program_hex":"4141414141414141414141414141414141414141414141414141414141414141","share_difficulty":1,"share_id":"share-1089","share_seq":1089,"template_height":800034},{"accepted_at_ms":999985,"credit_policy":null,"job_id":"job-1588","job_issued_at_ms":999995,"miner_id":"miner-4","network_difficulty":1000007,"ntime":1700001588,"order_key":"04:miner-4","p2mr_program_hex":"3434343434343434343434343434343434343434343434343434343434343434","share_difficulty":2,"share_id":"share-1588","share_seq":1588,"template_height":800049},{"accepted_at_ms":999997,"credit_policy":null,"job_id":"job-3565","job_issued_at_ms":999985,"miner_id":"miner-1","network_difficulty":1000012,"ntime":1700003565,"order_key":"01:miner-1","p2mr_program_hex":"edededededededededededededededededededededededededededededededed","share_difficulty":101,"share_id":"share-3565","share_seq":3565,"template_height":800111},{"accepted_at_ms":999996,"credit_policy":null,"job_id":"job-1829","job_issued_at_ms":1000000,"miner_id":"miner-3","network_difficulty":1000010,"ntime":1700001829,"order_key":"03:miner-3","p2mr_program_hex":"2525252525252525252525252525252525252525252525252525252525252525","share_difficulty":101,"share_id":"share-1829","share_seq":1829,"template_height":800057},{"accepted_at_ms":999984,"credit_policy":null,"job_id":"job-3472","job_issued_at_ms":999989,"miner_id":"miner-7","network_difficulty":1000004,"ntime":1700003472,"order_key":"07:miner-7","p2mr_program_hex":"9090909090909090909090909090909090909090909090909090909090909090","share_difficulty":7,"share_id":"share-3472","share_seq":3472,"template_height":800108},{"accepted_at_ms":999988,"credit_policy":null,"job_id":"job-1158","job_issued_at_ms":999978,"miner_id":"miner-3","network_difficulty":1000002,"ntime":1700001158,"order_key":"03:miner-3","p2mr_program_hex":"8686868686868686868686868686868686868686868686868686868686868686","share_difficulty":3,"share_id":"share-1158","share_seq":1158,"template_height":800036},{"accepted_at_ms":999994,"credit_policy":null,"job_id":"job-2913","job_issued_at_ms":1000312,"miner_id":"miner-9","network_difficulty":1000006,"ntime":1700002913,"order_key":"09:miner-9","p2mr_program_hex":"6161616161616161616161616161616161616161616161616161616161616161","share_difficulty":3,"share_id":"share-2913","share_seq":2913,"template_height":800091},{"accepted_at_ms":999970,"credit_policy":null,"job_id":"job-1055","job_issued_at_ms":999976,"miner_id":"miner-10","network_difficulty":1000001,"ntime":1700001055,"order_key":"10:miner-10","p2mr_program_hex":"1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f","share_difficulty":1000,"share_id":"share-1055","share_seq":1055,"template_height":800032},{"accepted_at_ms":999980,"credit_policy":null,"job_id":"job-101","job_issued_at_ms":999988,"miner_id":"miner-2","network_difficulty":1000016,"ntime":1700000101,"order_key":"02:miner-2","p2mr_program_hex":"6565656565656565656565656565656565656565656565656565656565656565","share_difficulty":1000,"share_id":"share-101","share_seq":101,"template_height":800003},{"accepted_at_ms":999987,"credit_policy":null,"job_id":"job-1317","job_issued_at_ms":999987,"miner_id":"miner-8","network_difficulty":1000008,"ntime":1700001317,"order_key":"08:miner-8","p2mr_program_hex":"2525252525252525252525252525252525252525252525252525252525252525","share_difficulty":101,"share_id":"share-1317","share_seq":1317,"template_height":800041},{"accepted_at_ms":1000000,"credit_policy":null,"job_id":"job-3424","job_issued_at_ms":999982,"miner_id":"miner-3","network_difficulty":1000007,"ntime":1700003424,"order_key":"03:miner-3","p2mr_program_hex":"6060606060606060606060606060606060606060606060606060606060606060","share_difficulty":7,"share_id":"share-3424","share_seq":3424,"template_height":800107},{"accepted_at_ms":999978,"credit_policy":null,"job_id":"job-3318","job_issued_at_ms":999989,"miner_id":"miner-7","network_difficulty":1000003,"ntime":1700003318,"order_key":"07:miner-7","p2mr_program_hex":"f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6","share_difficulty":3,"share_id":"share-3318","share_seq":3318,"template_height":800103},{"accepted_at_ms":999978,"credit_policy":null,"job_id":"job-1856","job_issued_at_ms":999978,"miner_id":"miner-8","network_difficulty":1000003,"ntime":1700001856,"order_key":"08:miner-8","p2mr_program_hex":"4040404040404040404040404040404040404040404040404040404040404040","share_difficulty":1,"share_id":"share-1856","share_seq":1856,"template_height":800058},{"accepted_at_ms":999970,"credit_policy":null,"job_id":"job-305","job_issued_at_ms":999991,"miner_id":"miner-8","network_difficulty":1000016,"ntime":1700000305,"order_key":"08:miner-8","p2mr_program_hex":"3131313131313131313131313131313131313131313131313131313131313131","share_difficulty":2,"share_id":"share-305","share_seq":305,"template_height":800009},{"accepted_at_ms":999982,"credit_policy":null,"job_id":"job-2724","job_issued_at_ms":999999,"miner_id":"miner-7","network_difficulty":1000004,"ntime":1700002724,"order_key":"07:miner-7","p2mr_program_hex":"a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4","share_difficulty":101,"share_id":"share-2724","share_seq":2724,"template_height":800085},{"accepted_at_ms":999988,"credit_policy":null,"job_id":"job-1403","job_issued_at_ms":999975,"miner_id":"miner-6","network_difficulty":1000009,"ntime":1700001403,"order_key":"06:miner-6","p2mr_program_hex":"7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b","share_difficulty":3,"share_id":"share-1403","share_seq":1403,"template_height":800043},{"accepted_at_ms":999990,"credit_policy":null,"job_id":"job-3997","job_issued_at_ms":1000170,"miner_id":"miner-4","network_difficulty":1000002,"ntime":1700003997,"order_key":"04:miner-4","p2mr_program_hex":"9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d","share_difficulty":19,"share_id":"share-3997","share_seq":3997,"template_height":800124},{"accepted_at_ms":1000065,"credit_policy":null,"job_id":"job-1839","job_issued_at_ms":1000087,"miner_id":"miner-2","network_difficulty":1000003,"ntime":1700001839,"order_key":"02:miner-2","p2mr_program_hex":"2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f","share_difficulty":1,"share_id":"share-1839","share_seq":1839,"template_height":800057},{"accepted_at_ms":999976,"credit_policy":null,"job_id":"job-2780","job_issued_at_ms":999998,"miner_id":"miner-8","network_difficulty":1000009,"ntime":1700002780,"order_key":"08:miner-8","p2mr_program_hex":"dcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdc","share_difficulty":7,"share_id":"share-2780","share_seq":2780,"template_height":800086},{"accepted_at_ms":999984,"credit_policy":"stale-grace","job_id":"job-758","job_issued_at_ms":999974,"miner_id":"miner-10","network_difficulty":1000010,"ntime":1700000758,"order_key":"10:miner-10","p2mr_program_hex":"f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6","share_difficulty":1000,"share_id":"share-758","share_seq":758,"template_height":800023},{"accepted_at_ms":999983,"credit_policy":null,"job_id":"job-2000","job_issued_at_ms":999985,"miner_id":"miner-9","network_difficulty":1000011,"ntime":1700002000,"order_key":"09:miner-9","p2mr_program_hex":"d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0","share_difficulty":19,"share_id":"share-2000","share_seq":2000,"template_height":800062},{"accepted_at_ms":999983,"credit_policy":null,"job_id":"job-2423","job_issued_at_ms":999982,"miner_id":"miner-3","network_difficulty":1000009,"ntime":1700002423,"order_key":"03:miner-3","p2mr_program_hex":"7777777777777777777777777777777777777777777777777777777777777777","share_difficulty":3,"share_id":"share-2423","share_seq":2423,"template_height":800075},{"accepted_at_ms":1000259,"credit_policy":null,"job_id":"job-2025","job_issued_at_ms":1000183,"miner_id":"miner-1","network_difficulty":1000002,"ntime":1700002025,"order_key":"01:miner-1","p2mr_program_hex":"e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9","share_difficulty":101,"share_id":"share-2025","share_seq":2025,"template_height":800063},{"accepted_at_ms":999993,"credit_policy":"stale-grace","job_id":"job-1572","job_issued_at_ms":999999,"miner_id":"miner-10","network_difficulty":1000008,"ntime":1700001572,"order_key":"10:miner-10","p2mr_program_hex":"2424242424242424242424242424242424242424242424242424242424242424","share_difficulty":3,"share_id":"share-1572","share_seq":1572,"template_height":800049},{"accepted_at_ms":999983,"credit_policy":null,"job_id":"job-699","job_issued_at_ms":999993,"miner_id":"miner-6","network_difficulty":1000002,"ntime":1700000699,"order_key":"06:miner-6","p2mr_program_hex":"bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb","share_difficulty":7,"share_id":"share-699","share_seq":699,"template_height":800021},{"accepted_at_ms":999989,"credit_policy":"stale-grace","job_id":"job-462","job_issued_at_ms":999971,"miner_id":"miner-0","network_difficulty":1000003,"ntime":1700000462,"order_key":"00:miner-0","p2mr_program_hex":"cececececececececececececececececececececececececececececececece","share_difficulty":7,"share_id":"share-462","share_seq":462,"template_height":800014},{"accepted_at_ms":999987,"credit_policy":null,"job_id":"job-2400","job_issued_at_ms":999982,"miner_id":"miner-2","network_difficulty":1000003,"ntime":1700002400,"order_key":"02:miner-2","p2mr_program_hex":"6060606060606060606060606060606060606060606060606060606060606060","share_difficulty":101,"share_id":"share-2400","share_seq":2400,"template_height":800075},{"accepted_at_ms":999974,"credit_policy":null,"job_id":"job-540","job_issued_at_ms":999990,"miner_id":"miner-1","network_difficulty":1000013,"ntime":1700000540,"order_key":"01:miner-1","p2mr_program_hex":"1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c","share_difficulty":3,"share_id":"share-540","share_seq":540,"template_height":800016},{"accepted_at_ms":999988,"credit_policy":null,"job_id":"job-218","job_issued_at_ms":999991,"miner_id":"miner-9","network_difficulty":1000014,"ntime":1700000218,"order_key":"09:miner-9","p2mr_program_hex":"dadadadadadadadadadadadadadadadadadadadadadadadadadadadadadadada","share_difficulty":19,"share_id":"share-218","share_seq":218,"template_height":800006},{"accepted_at_ms":999971,"credit_policy":"stale-grace","job_id":"job-1057","job_issued_at_ms":999987,"miner_id":"miner-1","network_difficulty":1000003,"ntime":1700001057,"order_key":"01:miner-1","p2mr_program_hex":"2121212121212121212121212121212121212121212121212121212121212121","share_difficulty":2,"share_id":"share-1057","share_seq":1057,"template_height":800033},{"accepted_at_ms":999992,"credit_policy":null,"job_id":"job-3469","job_issued_at_ms":999987,"miner_id":"miner-4","network_difficulty":1000001,"ntime":1700003469,"order_key":"04:miner-4","p2mr_program_hex":"8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d","share_difficulty":19,"share_id":"share-3469","share_seq":3469,"template_height":800108},{"accepted_at_ms":999996,"credit_policy":"stale-grace","job_id":"job-2808","job_issued_at_ms":999987,"miner_id":"miner-3","network_difficulty":1000003,"ntime":1700002808,"order_key":"03:miner-3","p2mr_program_hex":"f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8","share_difficulty":19,"share_id":"share-2808","share_seq":2808,"template_height":800087},{"accepted_at_ms":999985,"credit_policy":null,"job_id":"job-1259","job_issued_at_ms":999986,"miner_id":"miner-5","network_difficulty":1000001,"ntime":1700001259,"order_key":"05:miner-5","p2mr_program_hex":"ebebebebebebebebebebebebebebebebebebebebebebebebebebebebebebebeb","share_difficulty":3,"share_id":"share-1259","share_seq":1259,"template_height":800039},{"accepted_at_ms":999988,"credit_policy":null,"job_id":"job-2800","job_issued_at_ms":999990,"miner_id":"miner-6","network_difficulty":1000012,"ntime":1700002800,"order_key":"06:miner-6","p2mr_program_hex":"f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0","share_difficulty":1,"share_id":"share-2800","share_seq":2800,"template_height":800087},{"accepted_at_ms":999989,"credit_policy":null,"job_id":"job-2017","job_issued_at_ms":999992,"miner_id":"miner-4","network_difficulty":1000011,"ntime":1700002017,"order_key":"04:miner-4","p2mr_program_hex":"e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1","share_difficulty":101,"share_id":"share-2017","share_seq":2017,"template_height":800063},{"accepted_at_ms":999976,"credit_policy":null,"job_id":"job-54","job_issued_at_ms":999989,"miner_id":"miner-10","network_difficulty":1000003,"ntime":1700000054,"order_key":"10:miner-10","p2mr_program_hex":"3636363636363636363636363636363636363636363636363636363636363636","share_difficulty":19,"share_id":"share-54","share_seq":54,"template_height":800001},{"accepted_at_ms":999971,"credit_policy":null,"job_id":"job-68","job_issued_at_ms":999973,"miner_id":"miner-2","network_difficulty":1000000,"ntime":1700000068,"order_key":"02:miner-2","p2mr_program_hex":"4444444444444444444444444444444444444444444444444444444444444444","share_difficulty":2,"share_id":"share-68","share_seq":68,"template_height":800002},{"accepted_at_ms":1000284,"credit_policy":"stale-grace","job_id":"job-3633","job_issued_at_ms":1000258,"miner_id":"miner-3","network_difficulty":1000012,"ntime":1700003633,"order_key":"03:miner-3","p2mr_program_hex":"3131313131313131313131313131313131313131313131313131313131313131","share_difficulty":3,"share_id":"share-3633","share_seq":3633,"template_height":800113},{"accepted_at_ms":999977,"credit_policy":"stale-grace","job_id":"job-112","job_issued_at_ms":999980,"miner_id":"miner-2","network_difficulty":1000010,"ntime":1700000112,"order_key":"02:miner-2","p2mr_program_hex":"7070707070707070707070707070707070707070707070707070707070707070","share_difficulty":7,"share_id":"share-112","share_seq":112,"template_height":800003},{"accepted_at_ms":999973,"credit_policy":null,"job_id":"job-1914","job_issued_at_ms":999996,"miner_id":"miner-0","network_difficulty":1000010,"ntime":1700001914,"order_key":"00:miner-0","p2mr_program_hex":"7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a","share_difficulty":19,"share_id":"share-1914","share_seq":1914,"template_height":800059},{"accepted_at_ms":999976,"credit_policy":null,"job_id":"job-2147","job_issued_at_ms":999999,"miner_id":"miner-2","network_difficulty":1000005,"ntime":1700002147,"order_key":"02:miner-2","p2mr_program_hex":"6363636363636363636363636363636363636363636363636363636363636363","share_difficulty":2,"share_id":"share-2147","share_seq":2147,"template_height":800067},{"accepted_at_ms":1000274,"credit_policy":null,"job_id":"job-599","job_issued_at_ms":999999,"miner_id":"miner-5","network_difficulty":1000004,"ntime":1700000599,"order_key":"05:miner-5","p2mr_program_hex":"5757575757575757575757575757575757575757575757575757575757575757","share_difficulty":3,"share_id":"share-599","share_seq":599,"template_height":800018},{"accepted_at_ms":999980,"credit_policy":null,"job_id":"job-1195","job_issued_at_ms":999976,"miner_id":"miner-7","network_difficulty":1000005,"ntime":1700001195,"order_key":"07:miner-7","p2mr_program_hex":"abababababababababababababababababababababababababababababababab","share_difficulty":7,"share_id":"share-1195","share_seq":1195,"template_height":800037},{"accepted_at_ms":999976,"credit_policy":null,"job_id":"job-1925","job_issued_at_ms":999973,"miner_id":"miner-0","network_difficulty":1000004,"ntime":1700001925,"order_key":"00:miner-0","p2mr_program_hex":"8585858585858585858585858585858585858585858585858585858585858585","share_difficulty":101,"share_id":"share-1925","share_seq":1925,"template_height":800060},{"accepted_at_ms":999980,"credit_policy":null,"job_id":"job-2371","job_issued_at_ms":999978,"miner_id":"miner-6","network_difficulty":1000008,"ntime":1700002371,"order_key":"06:miner-6","p2mr_program_hex":"4343434343434343434343434343434343434343434343434343434343434343","share_difficulty":1,"share_id":"share-2371","share_seq":2371,"template_height":800074},{"accepted_at_ms":999996,"credit_policy":null,"job_id":"job-899","job_issued_at_ms":999994,"miner_id":"miner-8","network_difficulty":1000015,"ntime":1700000899,"order_key":"08:miner-8","p2mr_program_hex":"8383838383838383838383838383838383838383838383838383838383838383","share_difficulty":3,"share_id":"share-899","share_seq":899,"template_height":800028},{"accepted_at_ms":999994,"credit_policy":"stale-grace","job_id":"job-2872","job_issued_at_ms":999974,"miner_id":"miner-1","network_difficulty":1000016,"ntime":1700002872,"order_key":"01:miner-1","p2mr_program_hex":"3838383838383838383838383838383838383838383838383838383838383838","share_difficulty":1,"share_id":"share-2872","share_seq":2872,"template_height":800089},{"accepted_at_ms":999983,"credit_policy":null,"job_id":"job-3263","job_issued_at_ms":999971,"miner_id":"miner-7","network_difficulty":1000016,"ntime":1700003263,"order_key":"07:miner-7","p2mr_program_hex":"bfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbf","share_difficulty":1000,"share_id":"share-3263","share_seq":3263,"template_height":800101},{"accepted_at_ms":999985,"credit_policy":null,"job_id":"job-2545","job_issued_at_ms":999980,"miner_id":"miner-4","network_difficulty":1000012,"ntime":1700002545,"order_key":"04:miner-4","p2mr_program_hex":"f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1","share_difficulty":1000,"share_id":"share-2545","share_seq":2545,"template_height":800079},{"accepted_at_ms":999971,"credit_policy":null,"job_id":"job-949","job_issued_at_ms":999978,"miner_id":"miner-3","network_difficulty":1000014,"ntime":1700000949,"order_key":"03:miner-3","p2mr_program_hex":"b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5","share_difficulty":101,"share_id":"share-949","share_seq":949,"template_height":800029},{"accepted_at_ms":999999,"credit_policy":null,"job_id":"job-1269","job_issued_at_ms":999976,"miner_id":"miner-4","network_difficulty":1000011,"ntime":1700001269,"order_key":"04:miner-4","p2mr_program_hex":"f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5","share_difficulty":1000,"share_id":"share-1269","share_seq":1269,"template_height":800039},{"accepted_at_ms":999984,"credit_policy":null,"job_id":"job-2771","job_issued_at_ms":999971,"miner_id":"miner-10","network_difficulty":1000000,"ntime":1700002771,"order_key":"10:miner-10","p2mr_program_hex":"d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3","share_difficulty":3,"share_id":"share-2771","share_seq":2771,"template_height":800086},{"accepted_at_ms":1000000,"credit_policy":null,"job_id":"job-1715","job_issued_at_ms":999986,"miner_id":"miner-10","network_difficulty":1000015,"ntime":1700001715,"order_key":"10:miner-10","p2mr_program_hex":"b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3","share_difficulty":19,"share_id":"share-1715","share_seq":1715,"template_height":800053},{"accepted_at_ms":999980,"credit_policy":null,"job_id":"job-1511","job_issued_at_ms":999982,"miner_id":"miner-4","network_difficulty":1000015,"ntime":1700001511,"order_key":"04:miner-4","p2mr_program_hex":"e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7","share_difficulty":19,"share_id":"share-1511","share_seq":1511,"template_height":800047},{"accepted_at_ms":999984,"credit_policy":null,"job_id":"job-406","job_issued_at_ms":999996,"miner_id":"miner-10","network_difficulty":1000015,"ntime":1700000406,"order_key":"10:miner-10","p2mr_program_hex":"9696969696969696969696969696969696969696969696969696969696969696","share_difficulty":1,"share_id":"share-406","share_seq":406,"template_height":800012},{"accepted_at_ms":999986,"credit_policy":null,"job_id":"job-2181","job_issued_at_ms":999988,"miner_id":"miner-3","network_difficulty":1000005,"ntime":1700002181,"order_key":"03:miner-3","p2mr_program_hex":"8585858585858585858585858585858585858585858585858585858585858585","share_difficulty":7,"share_id":"share-2181","share_seq":2181,"template_height":800068},{"accepted_at_ms":999998,"credit_policy":null,"job_id":"job-691","job_issued_at_ms":999971,"miner_id":"miner-9","network_difficulty":1000011,"ntime":1700000691,"order_key":"09:miner-9","p2mr_program_hex":"b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3","share_difficulty":19,"share_id":"share-691","share_seq":691,"template_height":800021},{"accepted_at_ms":999992,"credit_policy":null,"job_id":"job-3872","job_issued_at_ms":999985,"miner_id":"miner-0","network_difficulty":1000013,"ntime":1700003872,"order_key":"00:miner-0","p2mr_program_hex":"2020202020202020202020202020202020202020202020202020202020202020","share_difficulty":101,"share_id":"share-3872","share_seq":3872,"template_height":800121},{"accepted_at_ms":999988,"credit_policy":null,"job_id":"job-1885","job_issued_at_ms":999974,"miner_id":"miner-4","network_difficulty":1000015,"ntime":1700001885,"order_key":"04:miner-4","p2mr_program_hex":"5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d","share_difficulty":1000,"share_id":"share-1885","share_seq":1885,"template_height":800058},{"accepted_at_ms":999986,"credit_policy":null,"job_id":"job-3428","job_issued_at_ms":999972,"miner_id":"miner-7","network_difficulty":1000011,"ntime":1700003428,"order_key":"07:miner-7","p2mr_program_hex":"6464646464646464646464646464646464646464646464646464646464646464","share_difficulty":1000,"share_id":"share-3428","share_seq":3428,"template_height":800107},{"accepted_at_ms":999973,"credit_policy":null,"job_id":"job-2120","job_issued_at_ms":999976,"miner_id":"miner-8","network_difficulty":1000012,"ntime":1700002120,"order_key":"08:miner-8","p2mr_program_hex":"4848484848484848484848484848484848484848484848484848484848484848","share_difficulty":1000,"share_id":"share-2120","share_seq":2120,"template_height":800066},{"accepted_at_ms":999972,"credit_policy":"stale-grace","job_id":"job-2111","job_issued_at_ms":999980,"miner_id":"miner-10","network_difficulty":1000003,"ntime":1700002111,"order_key":"10:miner-10","p2mr_program_hex":"3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f","share_difficulty":2,"share_id":"share-2111","share_seq":2111,"template_height":800065},{"accepted_at_ms":999978,"credit_policy":null,"job_id":"job-2649","job_issued_at_ms":999976,"miner_id":"miner-9","network_difficulty":1000014,"ntime":1700002649,"order_key":"09:miner-9","p2mr_program_hex":"5959595959595959595959595959595959595959595959595959595959595959","share_difficulty":19,"share_id":"share-2649","share_seq":2649,"template_height":800082},{"accepted_at_ms":1000463,"credit_policy":null,"job_id":"job-1674","job_issued_at_ms":1000420,"miner_id":"miner-2","network_difficulty":1000008,"ntime":1700001674,"order_key":"02:miner-2","p2mr_program_hex":"8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a","share_difficulty":19,"share_id":"share-1674","share_seq":1674,"template_height":800052},{"accepted_at_ms":1000000,"credit_policy":"stale-grace","job_id":"job-2725","job_issued_at_ms":999980,"miner_id":"miner-8","network_difficulty":1000005,"ntime":1700002725,"order_key":"08:miner-8","p2mr_program_hex":"a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5","share_difficulty":2,"share_id":"share-2725","share_seq":2725,"template_height":800085},{"accepted_at_ms":999988,"credit_policy":"stale-grace","job_id":"job-809","job_issued_at_ms":1000000,"miner_id":"miner-6","network_difficulty":1000010,"ntime":1700000809,"order_key":"06:miner-6","p2mr_program_hex":"2929292929292929292929292929292929292929292929292929292929292929","share_difficulty":1000,"share_id":"share-809","share_seq":809,"template_height":800025},{"accepted_at_ms":1000238,"credit_policy":null,"job_id":"job-2684","job_issued_at_ms":1000473,"miner_id":"miner-0","network_difficulty":1000015,"ntime":1700002684,"order_key":"00:miner-0","p2mr_program_hex":"7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c","share_difficulty":1000,"share_id":"share-2684","share_seq":2684,"template_height":800083},{"accepted_at_ms":999998,"credit_policy":null,"job_id":"job-808","job_issued_at_ms":999994,"miner_id":"miner-5","network_difficulty":1000009,"ntime":1700000808,"order_key":"05:miner-5","p2mr_program_hex":"2828282828282828282828282828282828282828282828282828282828282828","share_difficulty":101,"share_id":"share-808","share_seq":808,"template_height":800025},{"accepted_at_ms":999977,"credit_policy":null,"job_id":"job-1140","job_issued_at_ms":999990,"miner_id":"miner-7","network_difficulty":1000001,"ntime":1700001140,"order_key":"07:miner-7","p2mr_program_hex":"7474747474747474747474747474747474747474747474747474747474747474","share_difficulty":1000,"share_id":"share-1140","share_seq":1140,"template_height":800035},{"accepted_at_ms":999971,"credit_policy":null,"job_id":"job-3921","job_issued_at_ms":999970,"miner_id":"miner-5","network_difficulty":1000011,"ntime":1700003921,"order_key":"05:miner-5","p2mr_program_hex":"5151515151515151515151515151515151515151515151515151515151515151","share_difficulty":2,"share_id":"share-3921","share_seq":3921,"template_height":800122},{"accepted_at_ms":999975,"credit_policy":null,"job_id":"job-328","job_issued_at_ms":999983,"miner_id":"miner-9","network_difficulty":1000005,"ntime":1700000328,"order_key":"09:miner-9","p2mr_program_hex":"4848484848484848484848484848484848484848484848484848484848484848","share_difficulty":3,"share_id":"share-328","share_seq":328,"template_height":800010},{"accepted_at_ms":999987,"credit_policy":null,"job_id":"job-2317","job_issued_at_ms":999977,"miner_id":"miner-7","network_difficulty":1000005,"ntime":1700002317,"order_key":"07:miner-7","p2mr_program_hex":"0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d","share_difficulty":101,"share_id":"share-2317","share_seq":2317,"template_height":800072},{"accepted_at_ms":999984,"credit_policy":"stale-grace","job_id":"job-2007","job_issued_at_ms":999970,"miner_id":"miner-5","network_difficulty":1000001,"ntime":1700002007,"order_key":"05:miner-5","p2mr_program_hex":"d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7","share_difficulty":1,"share_id":"share-2007","share_seq":2007,"template_height":800062},{"accepted_at_ms":999970,"credit_policy":null,"job_id":"job-2197","job_issued_at_ms":999986,"miner_id":"miner-8","network_difficulty":1000004,"ntime":1700002197,"order_key":"08:miner-8","p2mr_program_hex":"9595959595959595959595959595959595959595959595959595959595959595","share_difficulty":1,"share_id":"share-2197","share_seq":2197,"template_height":800068},{"accepted_at_ms":999976,"credit_policy":"stale-grace","job_id":"job-81","job_issued_at_ms":999996,"miner_id":"miner-4","network_difficulty":1000013,"ntime":1700000081,"order_key":"04:miner-4","p2mr_program_hex":"5151515151515151515151515151515151515151515151515151515151515151","share_difficulty":1000,"share_id":"share-81","share_seq":81,"template_height":800002},{"accepted_at_ms":999999,"credit_policy":"stale-grace","job_id":"job-483","job_issued_at_ms":999974,"miner_id":"miner-10","network_difficulty":1000007,"ntime":1700000483,"order_key":"10:miner-10","p2mr_program_hex":"e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3","share_difficulty":101,"share_id":"share-483","share_seq":483,"template_height":800015},{"accepted_at_ms":999995,"credit_policy":null,"job_id":"job-1433","job_issued_at_ms":999975,"miner_id":"miner-3","network_difficulty":1000005,"ntime":1700001433,"order_key":"03:miner-3","p2mr_program_hex":"9999999999999999999999999999999999999999999999999999999999999999","share_difficulty":1,"share_id":"share-1433","share_seq":1433,"template_height":800044},{"accepted_at_ms":999987,"credit_policy":"stale-grace","job_id":"job-3904","job_issued_at_ms":999996,"miner_id":"miner-10","network_difficulty":1000011,"ntime":1700003904,"order_key":"10:miner-10","p2mr_program_hex":"4040404040404040404040404040404040404040404040404040404040404040","share_difficulty":2,"share_id":"share-3904","share_seq":3904,"template_height":800122},{"accepted_at_ms":999996,"credit_policy":null,"job_id":"job-2464","job_issued_at_ms":999977,"miner_id":"miner-0","network_difficulty":1000016,"ntime":1700002464,"order_key":"00:miner-0","p2mr_program_hex":"a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0","share_difficulty":1,"share_id":"share-2464","share_seq":2464,"template_height":800077},{"accepted_at_ms":999996,"credit_policy":"stale-grace","job_id":"job-1569","job_issued_at_ms":999973,"miner_id":"miner-7","network_difficulty":1000005,"ntime":1700001569,"order_key":"07:miner-7","p2mr_program_hex":"2121212121212121212121212121212121212121212121212121212121212121","share_difficulty":101,"share_id":"share-1569","share_seq":1569,"template_height":800049},{"accepted_at_ms":999998,"credit_policy":null,"job_id":"job-1843","job_issued_at_ms":999972,"miner_id":"miner-6","network_difficulty":1000007,"ntime":1700001843,"order_key":"06:miner-6","p2mr_program_hex":"3333333333333333333333333333333333333333333333333333333333333333","share_difficulty":101,"share_id":"share-1843","share_seq":1843,"template_height":800057},{"accepted_at_ms":999987,"credit_policy":null,"job_id":"job-3431","job_issued_at_ms":999990,"miner_id":"miner-10","network_difficulty":1000014,"ntime":1700003431,"order_key":"10:miner-10","p2mr_program_hex":"6767676767676767676767676767676767676767676767676767676767676767","share_difficulty":19,"share_id":"share-3431","share_seq":3431,"template_height":800107},{"accepted_at_ms":999994,"credit_policy":null,"job_id":"job-694","job_issued_at_ms":999970,"miner_id":"miner-1","network_difficulty":1000014,"ntime":1700000694,"order_key":"01:miner-1","p2mr_program_hex":"b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6","share_difficulty":1000,"share_id":"share-694","share_seq":694,"template_height":800021},{"accepted_at_ms":999973,"credit_policy":null,"job_id":"job-3425","job_issued_at_ms":1000000,"miner_id":"miner-4","network_difficulty":1000008,"ntime":1700003425,"order_key":"04:miner-4","p2mr_program_hex":"6161616161616161616161616161616161616161616161616161616161616161","share_difficulty":1000,"share_id":"share-3425","share_seq":3425,"template_height":800107},{"accepted_at_ms":999989,"credit_policy":null,"job_id":"job-3695","job_issued_at_ms":999981,"miner_id":"miner-10","network_difficulty":1000006,"ntime":1700003695,"order_key":"10:miner-10","p2mr_program_hex":"6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f","share_difficulty":1,"share_id":"share-3695","share_seq":3695,"template_height":800115},{"accepted_at_ms":999991,"credit_policy":null,"job_id":"job-769","job_issued_at_ms":999973,"miner_id":"miner-10","network_difficulty":1000004,"ntime":1700000769,"order_key":"10:miner-10","p2mr_program_hex":"0101010101010101010101010101010101010101010101010101010101010101","share_difficulty":7,"share_id":"share-769","share_seq":769,"template_height":800024},{"accepted_at_ms":999982,"credit_policy":null,"job_id":"job-1383","job_issued_at_ms":999986,"miner_id":"miner-8","network_difficulty":1000006,"ntime":1700001383,"order_key":"08:miner-8","p2mr_program_hex":"6767676767676767676767676767676767676767676767676767676767676767","share_difficulty":19,"share_id":"share-1383","share_seq":1383,"template_height":800043},{"accepted_at_ms":999984,"credit_policy":null,"job_id":"job-165","job_issued_at_ms":999973,"miner_id":"miner-0","network_difficulty":1000012,"ntime":1700000165,"order_key":"00:miner-0","p2mr_program_hex":"a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5","share_difficulty":1000,"share_id":"share-165","share_seq":165,"template_height":800005},{"accepted_at_ms":999983,"credit_policy":null,"job_id":"job-1558","job_issued_at_ms":999985,"miner_id":"miner-7","network_difficulty":1000011,"ntime":1700001558,"order_key":"07:miner-7","p2mr_program_hex":"1616161616161616161616161616161616161616161616161616161616161616","share_difficulty":1000,"share_id":"share-1558","share_seq":1558,"template_height":800048},{"accepted_at_ms":999989,"credit_policy":"stale-grace","job_id":"job-3773","job_issued_at_ms":999972,"miner_id":"miner-0","network_difficulty":1000016,"ntime":1700003773,"order_key":"00:miner-0","p2mr_program_hex":"bdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbd","share_difficulty":1,"share_id":"share-3773","share_seq":3773,"template_height":800117},{"accepted_at_ms":999992,"credit_policy":null,"job_id":"job-1824","job_issued_at_ms":999989,"miner_id":"miner-9","network_difficulty":1000005,"ntime":1700001824,"order_key":"09:miner-9","p2mr_program_hex":"2020202020202020202020202020202020202020202020202020202020202020","share_difficulty":19,"share_id":"share-1824","share_seq":1824,"template_height":800057},{"accepted_at_ms":999975,"credit_policy":null,"job_id":"job-2671","job_issued_at_ms":999973,"miner_id":"miner-9","network_difficulty":1000002,"ntime":1700002671,"order_key":"09:miner-9","p2mr_program_hex":"6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f","share_difficulty":101,"share_id":"share-2671","share_seq":2671,"template_height":800083},{"accepted_at_ms":999989,"credit_policy":null,"job_id":"job-1444","job_issued_at_ms":999976,"miner_id":"miner-3","network_difficulty":1000016,"ntime":1700001444,"order_key":"03:miner-3","p2mr_program_hex":"a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4","share_difficulty":1000,"share_id":"share-1444","share_seq":1444,"template_height":800045},{"accepted_at_ms":999986,"credit_policy":"stale-grace","job_id":"job-2595","job_issued_at_ms":999997,"miner_id":"miner-10","network_difficulty":1000011,"ntime":1700002595,"order_key":"10:miner-10","p2mr_program_hex":"2323232323232323232323232323232323232323232323232323232323232323","share_difficulty":7,"share_id":"share-2595","share_seq":2595,"template_height":800081},{"accepted_at_ms":999998,"credit_policy":null,"job_id":"job-355","job_issued_at_ms":999991,"miner_id":"miner-3","network_difficulty":1000015,"ntime":1700000355,"order_key":"03:miner-3","p2mr_program_hex":"6363636363636363636363636363636363636363636363636363636363636363","share_difficulty":101,"share_id":"share-355","share_seq":355,"template_height":800011},{"accepted_at_ms":999973,"credit_policy":"stale-grace","job_id":"job-2308","job_issued_at_ms":999993,"miner_id":"miner-9","network_difficulty":1000013,"ntime":1700002308,"order_key":"09:miner-9","p2mr_program_hex":"0404040404040404040404040404040404040404040404040404040404040404","share_difficulty":1000,"share_id":"share-2308","share_seq":2308,"template_height":800072},{"accepted_at_ms":999998,"credit_policy":null,"job_id":"job-888","job_issued_at_ms":999984,"miner_id":"miner-8","network_difficulty":1000004,"ntime":1700000888,"order_key":"08:miner-8","p2mr_program_hex":"7878787878787878787878787878787878787878787878787878787878787878","share_difficulty":101,"share_id":"share-888","share_seq":888,"template_height":800027},{"accepted_at_ms":999970,"credit_policy":"stale-grace","job_id":"job-613","job_issued_at_ms":999998,"miner_id":"miner-8","network_difficulty":1000001,"ntime":1700000613,"order_key":"08:miner-8","p2mr_program_hex":"6565656565656565656565656565656565656565656565656565656565656565","share_difficulty":1000,"share_id":"share-613","share_seq":613,"template_height":800019},{"accepted_at_ms":999971,"credit_policy":null,"job_id":"job-3601","job_issued_at_ms":1000317,"miner_id":"miner-4","network_difficulty":1000014,"ntime":1700003601,"order_key":"04:miner-4","p2mr_program_hex":"1111111111111111111111111111111111111111111111111111111111111111","share_difficulty":19,"share_id":"share-3601","share_seq":3601,"template_height":800112},{"accepted_at_ms":999987,"credit_policy":null,"job_id":"job-104","job_issued_at_ms":999977,"miner_id":"miner-5","network_difficulty":1000002,"ntime":1700000104,"order_key":"05:miner-5","p2mr_program_hex":"6868686868686868686868686868686868686868686868686868686868686868","share_difficulty":2,"share_id":"share-104","share_seq":104,"template_height":800003},{"accepted_at_ms":999980,"credit_policy":null,"job_id":"job-2085","job_issued_at_ms":999980,"miner_id":"miner-6","network_difficulty":1000011,"ntime":1700002085,"order_key":"06:miner-6","p2mr_program_hex":"2525252525252525252525252525252525252525252525252525252525252525","share_difficulty":101,"share_id":"share-2085","share_seq":2085,"template_height":800065},{"accepted_at_ms":999980,"credit_policy":"stale-grace","job_id":"job-3600","job_issued_at_ms":999999,"miner_id":"miner-3","network_difficulty":1000013,"ntime":1700003600,"order_key":"03:miner-3","p2mr_program_hex":"1010101010101010101010101010101010101010101010101010101010101010","share_difficulty":3,"share_id":"share-3600","share_seq":3600,"template_height":800112},{"accepted_at_ms":999995,"credit_policy":null,"job_id":"job-3635","job_issued_at_ms":999979,"miner_id":"miner-5","network_difficulty":1000014,"ntime":1700003635,"order_key":"05:miner-5","p2mr_program_hex":"3333333333333333333333333333333333333333333333333333333333333333","share_difficulty":19,"share_id":"share-3635","share_seq":3635,"template_height":800113},{"accepted_at_ms":999974,"credit_policy":null,"job_id":"job-912","job_issued_at_ms":999991,"miner_id":"miner-10","network_difficulty":1000011,"ntime":1700000912,"order_key":"10:miner-10","p2mr_program_hex":"9090909090909090909090909090909090909090909090909090909090909090","share_difficulty":19,"share_id":"share-912","share_seq":912,"template_height":800028},{"accepted_at_ms":999983,"credit_policy":null,"job_id":"job-3928","job_issued_at_ms":999992,"miner_id":"miner-1","network_difficulty":1000001,"ntime":1700003928,"order_key":"01:miner-1","p2mr_program_hex":"5858585858585858585858585858585858585858585858585858585858585858","share_difficulty":1000,"share_id":"share-3928","share_seq":3928,"template_height":800122},{"accepted_at_ms":999977,"credit_policy":null,"job_id":"job-3393","job_issued_at_ms":1000011,"miner_id":"miner-5","network_difficulty":1000010,"ntime":1700003393,"order_key":"05:miner-5","p2mr_program_hex":"4141414141414141414141414141414141414141414141414141414141414141","share_difficulty":7,"share_id":"share-3393","share_seq":3393,"template_height":800106},{"accepted_at_ms":999995,"credit_policy":null,"job_id":"job-653","job_issued_at_ms":999970,"miner_id":"miner-4","network_difficulty":1000007,"ntime":1700000653,"order_key":"04:miner-4","p2mr_program_hex":"8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d","share_difficulty":1000,"share_id":"share-653","share_seq":653,"template_height":800020},{"accepted_at_ms":999991,"credit_policy":null,"job_id":"job-2527","job_issued_at_ms":999991,"miner_id":"miner-8","network_difficulty":1000011,"ntime":1700002527,"order_key":"08:miner-8","p2mr_program_hex":"dfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdf","share_difficulty":7,"share_id":"share-2527","share_seq":2527,"template_height":800078},{"accepted_at_ms":999975,"credit_policy":null,"job_id":"job-3886","job_issued_at_ms":999988,"miner_id":"miner-3","network_difficulty":1000010,"ntime":1700003886,"order_key":"03:miner-3","p2mr_program_hex":"2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e","share_difficulty":2,"share_id":"share-3886","share_seq":3886,"template_height":800121},{"accepted_at_ms":1000000,"credit_policy":null,"job_id":"job-3574","job_issued_at_ms":1000268,"miner_id":"miner-10","network_difficulty":1000004,"ntime":1700003574,"order_key":"10:miner-10","p2mr_program_hex":"f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6","share_difficulty":1,"share_id":"share-3574","share_seq":3574,"template_height":800111},{"accepted_at_ms":999975,"credit_policy":null,"job_id":"job-1525","job_issued_at_ms":1000300,"miner_id":"miner-7","network_difficulty":1000012,"ntime":1700001525,"order_key":"07:miner-7","p2mr_program_hex":"f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5","share_difficulty":7,"share_id":"share-1525","share_seq":1525,"template_height":800047},{"accepted_at_ms":999998,"credit_policy":null,"job_id":"job-336","job_issued_at_ms":999984,"miner_id":"miner-6","network_difficulty":1000013,"ntime":1700000336,"order_key":"06:miner-6","p2mr_program_hex":"5050505050505050505050505050505050505050505050505050505050505050","share_difficulty":7,"share_id":"share-336","share_seq":336,"template_height":800010},{"accepted_at_ms":1000024,"credit_policy":"stale-grace","job_id":"job-1184","job_issued_at_ms":999981,"miner_id":"miner-7","network_difficulty":1000011,"ntime":1700001184,"order_key":"07:miner-7","p2mr_program_hex":"a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0","share_difficulty":7,"share_id":"share-1184","share_seq":1184,"template_height":800037},{"accepted_at_ms":999997,"credit_policy":null,"job_id":"job-1316","job_issued_at_ms":999970,"miner_id":"miner-7","network_difficulty":1000007,"ntime":1700001316,"order_key":"07:miner-7","p2mr_program_hex":"2424242424242424242424242424242424242424242424242424242424242424","share_difficulty":1000,"share_id":"share-1316","share_seq":1316,"template_height":800041},{"accepted_at_ms":999972,"credit_policy":null,"job_id":"job-864","job_issued_at_ms":999987,"miner_id":"miner-6","network_difficulty":1000014,"ntime":1700000864,"order_key":"06:miner-6","p2mr_program_hex":"6060606060606060606060606060606060606060606060606060606060606060","share_difficulty":1000,"share_id":"share-864","share_seq":864,"template_height":800027},{"accepted_at_ms":999984,"credit_policy":null,"job_id":"job-2881","job_issued_at_ms":999997,"miner_id":"miner-10","network_difficulty":1000008,"ntime":1700002881,"order_key":"10:miner-10","p2mr_program_hex":"4141414141414141414141414141414141414141414141414141414141414141","share_difficulty":101,"share_id":"share-2881","share_seq":2881,"template_height":800090},{"accepted_at_ms":999997,"credit_policy":null,"job_id":"job-715","job_issued_at_ms":999982,"miner_id":"miner-0","network_difficulty":1000001,"ntime":1700000715,"order_key":"00:miner-0","p2mr_program_hex":"cbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcb","share_difficulty":1000,"share_id":"share-715","share_seq":715,"template_height":800022},{"accepted_at_ms":999995,"credit_policy":"stale-grace","job_id":"job-3587","job_issued_at_ms":999983,"miner_id":"miner-1","network_difficulty":1000000,"ntime":1700003587,"order_key":"01:miner-1","p2mr_program_hex":"0303030303030303030303030303030303030303030303030303030303030303","share_difficulty":19,"share_id":"share-3587","share_seq":3587,"template_height":800112},{"accepted_at_ms":999990,"credit_policy":null,"job_id":"job-3329","job_issued_at_ms":999975,"miner_id":"miner-7","network_difficulty":1000014,"ntime":1700003329,"order_key":"07:miner-7","p2mr_program_hex":"0101010101010101010101010101010101010101010101010101010101010101","share_difficulty":19,"share_id":"share-3329","share_seq":3329,"template_height":800104},{"accepted_at_ms":999976,"credit_policy":null,"job_id":"job-1180","job_issued_at_ms":999972,"miner_id":"miner-3","network_difficulty":1000007,"ntime":1700001180,"order_key":"03:miner-3","p2mr_program_hex":"9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c","share_difficulty":7,"share_id":"share-1180","share_seq":1180,"template_height":800036},{"accepted_at_ms":999974,"credit_policy":null,"job_id":"job-1442","job_issued_at_ms":999990,"miner_id":"miner-1","network_difficulty":1000014,"ntime":1700001442,"order_key":"01:miner-1","p2mr_program_hex":"a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2","share_difficulty":1000,"share_id":"share-1442","share_seq":1442,"template_height":800045},{"accepted_at_ms":999971,"credit_policy":null,"job_id":"job-78","job_issued_at_ms":999989,"miner_id":"miner-1","network_difficulty":1000010,"ntime":1700000078,"order_key":"01:miner-1","p2mr_program_hex":"4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e","share_difficulty":1,"share_id":"share-78","share_seq":78,"template_height":800002},{"accepted_at_ms":999976,"credit_policy":null,"job_id":"job-2204","job_issued_at_ms":999996,"miner_id":"miner-4","network_difficulty":1000011,"ntime":1700002204,"order_key":"04:miner-4","p2mr_program_hex":"9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c","share_difficulty":2,"share_id":"share-2204","share_seq":2204,"template_height":800068},{"accepted_at_ms":999986,"credit_policy":null,"job_id":"job-3778","job_issued_at_ms":999995,"miner_id":"miner-5","network_difficulty":1000004,"ntime":1700003778,"order_key":"05:miner-5","p2mr_program_hex":"c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2","share_difficulty":19,"share_id":"share-3778","share_seq":3778,"template_height":800118},{"accepted_at_ms":999977,"credit_policy":null,"job_id":"job-408","job_issued_at_ms":999980,"miner_id":"miner-1","network_difficulty":1000000,"ntime":1700000408,"order_key":"01:miner-1","p2mr_program_hex":"9898989898989898989898989898989898989898989898989898989898989898","share_difficulty":2,"share_id":"share-408","share_seq":408,"template_height":800012},{"accepted_at_ms":1000109,"credit_policy":null,"job_id":"job-129","job_issued_at_ms":999998,"miner_id":"miner-8","network_difficulty":1000010,"ntime":1700000129,"order_key":"08:miner-8","p2mr_program_hex":"8181818181818181818181818181818181818181818181818181818181818181","share_difficulty":101,"share_id":"share-129","share_seq":129,"template_height":800004},{"accepted_at_ms":999980,"credit_policy":null,"job_id":"job-3122","job_issued_at_ms":999999,"miner_id":"miner-9","network_difficulty":1000011,"ntime":1700003122,"order_key":"09:miner-9","p2mr_program_hex":"3232323232323232323232323232323232323232323232323232323232323232","share_difficulty":7,"share_id":"share-3122","share_seq":3122,"template_height":800097},{"accepted_at_ms":999998,"credit_policy":null,"job_id":"job-259","job_issued_at_ms":999973,"miner_id":"miner-6","network_difficulty":1000004,"ntime":1700000259,"order_key":"06:miner-6","p2mr_program_hex":"0303030303030303030303030303030303030303030303030303030303030303","share_difficulty":1000,"share_id":"share-259","share_seq":259,"template_height":800008},{"accepted_at_ms":999998,"credit_policy":"stale-grace","job_id":"job-1147","job_issued_at_ms":999992,"miner_id":"miner-3","network_difficulty":1000008,"ntime":1700001147,"order_key":"03:miner-3","p2mr_program_hex":"7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b","share_difficulty":3,"share_id":"share-1147","share_seq":1147,"template_height":800035},{"accepted_at_ms":999986,"credit_policy":null,"job_id":"job-1858","job_issued_at_ms":999994,"miner_id":"miner-10","network_difficulty":1000005,"ntime":1700001858,"order_key":"10:miner-10","p2mr_program_hex":"4242424242424242424242424242424242424242424242424242424242424242","share_difficulty":1000,"share_id":"share-1858","share_seq":1858,"template_height":800058},{"accepted_at_ms":999982,"credit_policy":null,"job_id":"job-3569","job_issued_at_ms":1000172,"miner_id":"miner-5","network_difficulty":1000016,"ntime":1700003569,"order_key":"05:miner-5","p2mr_program_hex":"f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1","share_difficulty":3,"share_id":"share-3569","share_seq":3569,"template_height":800111},{"accepted_at_ms":999979,"credit_policy":null,"job_id":"job-3769","job_issued_at_ms":999970,"miner_id":"miner-7","network_difficulty":1000012,"ntime":1700003769,"order_key":"07:miner-7","p2mr_program_hex":"b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9","share_difficulty":7,"share_id":"share-3769","share_seq":3769,"template_height":800117},{"accepted_at_ms":999990,"credit_policy":null,"job_id":"job-2878","job_issued_at_ms":999989,"miner_id":"miner-7","network_difficulty":1000005,"ntime":1700002878,"order_key":"07:miner-7","p2mr_program_hex":"3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e","share_difficulty":1000,"share_id":"share-2878","share_seq":2878,"template_height":800089},{"accepted_at_ms":999998,"credit_policy":null,"job_id":"job-1436","job_issued_at_ms":999977,"miner_id":"miner-6","network_difficulty":1000008,"ntime":1700001436,"order_key":"06:miner-6","p2mr_program_hex":"9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c","share_difficulty":2,"share_id":"share-1436","share_seq":1436,"template_height":800044},{"accepted_at_ms":999987,"credit_policy":null,"job_id":"job-2246","job_issued_at_ms":999987,"miner_id":"miner-2","network_difficulty":1000002,"ntime":1700002246,"order_key":"02:miner-2","p2mr_program_hex":"c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6","share_difficulty":2,"share_id":"share-2246","share_seq":2246,"template_height":800070},{"accepted_at_ms":999987,"credit_policy":null,"job_id":"job-1555","job_issued_at_ms":999974,"miner_id":"miner-4","network_difficulty":1000008,"ntime":1700001555,"order_key":"04:miner-4","p2mr_program_hex":"1313131313131313131313131313131313131313131313131313131313131313","share_difficulty":3,"share_id":"share-1555","share_seq":1555,"template_height":800048},{"accepted_at_ms":999974,"credit_policy":null,"job_id":"job-1338","job_issued_at_ms":1000152,"miner_id":"miner-7","network_difficulty":1000012,"ntime":1700001338,"order_key":"07:miner-7","p2mr_program_hex":"3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a","share_difficulty":7,"share_id":"share-1338","share_seq":1338,"template_height":800041},{"accepted_at_ms":999983,"credit_policy":null,"job_id":"job-1408","job_issued_at_ms":999981,"miner_id":"miner-0","network_difficulty":1000014,"ntime":1700001408,"order_key":"00:miner-0","p2mr_program_hex":"8080808080808080808080808080808080808080808080808080808080808080","share_difficulty":2,"share_id":"share-1408","share_seq":1408,"template_height":800044},{"accepted_at_ms":999980,"credit_policy":null,"job_id":"job-2049","job_issued_at_ms":999975,"miner_id":"miner-3","network_difficulty":1000009,"ntime":1700002049,"order_key":"03:miner-3","p2mr_program_hex":"0101010101010101010101010101010101010101010101010101010101010101","share_difficulty":1,"share_id":"share-2049","share_seq":2049,"template_height":800064},{"accepted_at_ms":999993,"credit_policy":null,"job_id":"job-1107","job_issued_at_ms":999971,"miner_id":"miner-7","network_difficulty":1000002,"ntime":1700001107,"order_key":"07:miner-7","p2mr_program_hex":"5353535353535353535353535353535353535353535353535353535353535353","share_difficulty":19,"share_id":"share-1107","share_seq":1107,"template_height":800034},{"accepted_at_ms":999992,"credit_policy":null,"job_id":"job-2651","job_issued_at_ms":999997,"miner_id":"miner-0","network_difficulty":1000016,"ntime":1700002651,"order_key":"00:miner-0","p2mr_program_hex":"5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b","share_difficulty":2,"share_id":"share-2651","share_seq":2651,"template_height":800082},{"accepted_at_ms":999977,"credit_policy":null,"job_id":"job-53","job_issued_at_ms":999981,"miner_id":"miner-9","network_difficulty":1000002,"ntime":1700000053,"order_key":"09:miner-9","p2mr_program_hex":"3535353535353535353535353535353535353535353535353535353535353535","share_difficulty":7,"share_id":"share-53","share_seq":53,"template_height":800001},{"accepted_at_ms":999981,"credit_policy":null,"job_id":"job-398","job_issued_at_ms":999978,"miner_id":"miner-2","network_difficulty":1000007,"ntime":1700000398,"order_key":"02:miner-2","p2mr_program_hex":"8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e","share_difficulty":101,"share_id":"share-398","share_seq":398,"template_height":800012},{"accepted_at_ms":999988,"credit_policy":null,"job_id":"job-2533","job_issued_at_ms":1000149,"miner_id":"miner-3","network_difficulty":1000000,"ntime":1700002533,"order_key":"03:miner-3","p2mr_program_hex":"e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5","share_difficulty":2,"share_id":"share-2533","share_seq":2533,"template_height":800079},{"accepted_at_ms":999982,"credit_policy":null,"job_id":"job-3119","job_issued_at_ms":999984,"miner_id":"miner-6","network_difficulty":1000008,"ntime":1700003119,"order_key":"06:miner-6","p2mr_program_hex":"2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f","share_difficulty":19,"share_id":"share-3119","share_seq":3119,"template_height":800097},{"accepted_at_ms":999983,"credit_policy":"stale-grace","job_id":"job-3491","job_issued_at_ms":999987,"miner_id":"miner-4","network_difficulty":1000006,"ntime":1700003491,"order_key":"04:miner-4","p2mr_program_hex":"a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3","share_difficulty":19,"share_id":"share-3491","share_seq":3491,"template_height":800109},{"accepted_at_ms":999982,"credit_policy":"stale-grace","job_id":"job-1861","job_issued_at_ms":999994,"miner_id":"miner-2","network_difficulty":1000008,"ntime":1700001861,"order_key":"02:miner-2","p2mr_program_hex":"4545454545454545454545454545454545454545454545454545454545454545","share_difficulty":101,"share_id":"share-1861","share_seq":1861,"template_height":800058},{"accepted_at_ms":999991,"credit_policy":null,"job_id":"job-543","job_issued_at_ms":999971,"miner_id":"miner-4","network_difficulty":1000016,"ntime":1700000543,"order_key":"04:miner-4","p2mr_program_hex":"1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f","share_difficulty":101,"share_id":"share-543","share_seq":543,"template_height":800016},{"accepted_at_ms":999979,"credit_policy":null,"job_id":"job-697","job_issued_at_ms":999991,"miner_id":"miner-4","network_difficulty":1000000,"ntime":1700000697,"order_key":"04:miner-4","p2mr_program_hex":"b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9","share_difficulty":1000,"share_id":"share-697","share_seq":697,"template_height":800021},{"accepted_at_ms":999972,"credit_policy":null,"job_id":"job-695","job_issued_at_ms":999970,"miner_id":"miner-2","network_difficulty":1000015,"ntime":1700000695,"order_key":"02:miner-2","p2mr_program_hex":"b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7","share_difficulty":101,"share_id":"share-695","share_seq":695,"template_height":800021},{"accepted_at_ms":999991,"credit_policy":null,"job_id":"job-3072","job_issued_at_ms":999974,"miner_id":"miner-3","network_difficulty":1000012,"ntime":1700003072,"order_key":"03:miner-3","p2mr_program_hex":"0000000000000000000000000000000000000000000000000000000000000000","share_difficulty":1,"share_id":"share-3072","share_seq":3072,"template_height":800096},{"accepted_at_ms":999991,"credit_policy":null,"job_id":"job-2555","job_issued_at_ms":999980,"miner_id":"miner-3","network_difficulty":1000005,"ntime":1700002555,"order_key":"03:miner-3","p2mr_program_hex":"fbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfb","share_difficulty":3,"share_id":"share-2555","share_seq":2555,"template_height":800079},{"accepted_at_ms":999983,"credit_policy":null,"job_id":"job-2128","job_issued_at_ms":999991,"miner_id":"miner-5","network_difficulty":1000003,"ntime":1700002128,"order_key":"05:miner-5","p2mr_program_hex":"5050505050505050505050505050505050505050505050505050505050505050","share_difficulty":3,"share_id":"share-2128","share_seq":2128,"template_height":800066},{"accepted_at_ms":999984,"credit_policy":null,"job_id":"job-1923","job_issued_at_ms":999991,"miner_id":"miner-9","network_difficulty":1000002,"ntime":1700001923,"order_key":"09:miner-9","p2mr_program_hex":"8383838383838383838383838383838383838383838383838383838383838383","share_difficulty":1000,"share_id":"share-1923","share_seq":1923,"template_height":800060},{"accepted_at_ms":999973,"credit_policy":"stale-grace","job_id":"job-1680","job_issued_at_ms":999985,"miner_id":"miner-8","network_difficulty":1000014,"ntime":1700001680,"order_key":"08:miner-8","p2mr_program_hex":"9090909090909090909090909090909090909090909090909090909090909090","share_difficulty":1000,"share_id":"share-1680","share_seq":1680,"template_height":800052},{"accepted_at_ms":999995,"credit_policy":"stale-grace","job_id":"job-409","job_issued_at_ms":999987,"miner_id":"miner-2","network_difficulty":1000001,"ntime":1700000409,"order_key":"02:miner-2","p2mr_program_hex":"9999999999999999999999999999999999999999999999999999999999999999","share_difficulty":1,"share_id":"share-409","share_seq":409,"template_height":800012},{"accepted_at_ms":999992,"credit_policy":null,"job_id":"job-707","job_issued_at_ms":999986,"miner_id":"miner-3","network_difficulty":1000010,"ntime":1700000707,"order_key":"03:miner-3","p2mr_program_hex":"c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3","share_difficulty":1,"share_id":"share-707","share_seq":707,"template_height":800022},{"accepted_at_ms":999982,"credit_policy":null,"job_id":"job-1384","job_issued_at_ms":999993,"miner_id":"miner-9","network_difficulty":1000007,"ntime":1700001384,"order_key":"09:miner-9","p2mr_program_hex":"6868686868686868686868686868686868686868686868686868686868686868","share_difficulty":3,"share_id":"share-1384","share_seq":1384,"template_height":800043},{"accepted_at_ms":999999,"credit_policy":null,"job_id":"job-3699","job_issued_at_ms":999990,"miner_id":"miner-3","network_difficulty":1000010,"ntime":1700003699,"order_key":"03:miner-3","p2mr_program_hex":"7373737373737373737373737373737373737373737373737373737373737373","share_difficulty":3,"share_id":"share-3699","share_seq":3699,"template_height":800115},{"accepted_at_ms":999979,"credit_policy":null,"job_id":"job-871","job_issued_at_ms":999999,"miner_id":"miner-2","network_difficulty":1000004,"ntime":1700000871,"order_key":"02:miner-2","p2mr_program_hex":"6767676767676767676767676767676767676767676767676767676767676767","share_difficulty":19,"share_id":"share-871","share_seq":871,"template_height":800027},{"accepted_at_ms":999973,"credit_policy":null,"job_id":"job-3063","job_issued_at_ms":999972,"miner_id":"miner-5","network_difficulty":1000003,"ntime":1700003063,"order_key":"05:miner-5","p2mr_program_hex":"f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7","share_difficulty":1,"share_id":"share-3063","share_seq":3063,"template_height":800095},{"accepted_at_ms":1000226,"credit_policy":"stale-grace","job_id":"job-3623","job_issued_at_ms":1000098,"miner_id":"miner-4","network_difficulty":1000002,"ntime":1700003623,"order_key":"04:miner-4","p2mr_program_hex":"2727272727272727272727272727272727272727272727272727272727272727","share_difficulty":1,"share_id":"share-3623","share_seq":3623,"template_height":800113},{"accepted_at_ms":999978,"credit_policy":null,"job_id":"job-2686","job_issued_at_ms":999975,"miner_id":"miner-2","network_difficulty":1000000,"ntime":1700002686,"order_key":"02:miner-2","p2mr_program_hex":"7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e","share_difficulty":19,"share_id":"share-2686","share_seq":2686,"template_height":800083},{"accepted_at_ms":999999,"credit_policy":null,"job_id":"job-1019","job_issued_at_ms":999996,"miner_id":"miner-7","network_difficulty":1000016,"ntime":1700001019,"order_key":"07:miner-7","p2mr_program_hex":"fbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfb","share_difficulty":2,"share_id":"share-1019","share_seq":1019,"template_height":800031},{"accepted_at_ms":999986,"credit_policy":"stale-grace","job_id":"job-3312","job_issued_at_ms":999996,"miner_id":"miner-1","network_difficulty":1000014,"ntime":1700003312,"order_key":"01:miner-1","p2mr_program_hex":"f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0","share_difficulty":19,"share_id":"share-3312","share_seq":3312,"template_height":800103},{"accepted_at_ms":999990,"credit_policy":"stale-grace","job_id":"job-3720","job_issued_at_ms":999996,"miner_id":"miner-2","network_difficulty":1000014,"ntime":1700003720,"order_key":"02:miner-2","p2mr_program_hex":"8888888888888888888888888888888888888888888888888888888888888888","share_difficulty":1000,"share_id":"share-3720","share_seq":3720,"template_height":800116},{"accepted_at_ms":999982,"credit_policy":null,"job_id":"job-1995","job_issued_at_ms":999977,"miner_id":"miner-4","network_difficulty":1000006,"ntime":1700001995,"order_key":"04:miner-4","p2mr_program_hex":"cbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcb","share_difficulty":3,"share_id":"share-1995","share_seq":1995,"template_height":800062},{"accepted_at_ms":1000194,"credit_policy":null,"job_id":"job-3457","job_issued_at_ms":999977,"miner_id":"miner-3","network_difficulty":1000006,"ntime":1700003457,"order_key":"03:miner-3","p2mr_program_hex":"8181818181818181818181818181818181818181818181818181818181818181","share_difficulty":1,"share_id":"share-3457","share_seq":3457,"template_height":800108},{"accepted_at_ms":999999,"credit_policy":null,"job_id":"job-2379","job_issued_at_ms":999970,"miner_id":"miner-3","network_difficulty":1000016,"ntime":1700002379,"order_key":"03:miner-3","p2mr_program_hex":"4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b","share_difficulty":2,"share_id":"share-2379","share_seq":2379,"template_height":800074},{"accepted_at_ms":999971,"credit_policy":null,"job_id":"job-1366","job_issued_at_ms":999984,"miner_id":"miner-2","network_difficulty":1000006,"ntime":1700001366,"order_key":"02:miner-2","p2mr_program_hex":"5656565656565656565656565656565656565656565656565656565656565656","share_difficulty":1,"share_id":"share-1366","share_seq":1366,"template_height":800042},{"accepted_at_ms":1000193,"credit_policy":null,"job_id":"job-2407","job_issued_at_ms":1000268,"miner_id":"miner-9","network_difficulty":1000010,"ntime":1700002407,"order_key":"09:miner-9","p2mr_program_hex":"6767676767676767676767676767676767676767676767676767676767676767","share_difficulty":2,"share_id":"share-2407","share_seq":2407,"template_height":800075},{"accepted_at_ms":999971,"credit_policy":null,"job_id":"job-572","job_issued_at_ms":999994,"miner_id":"miner-0","network_difficulty":1000011,"ntime":1700000572,"order_key":"00:miner-0","p2mr_program_hex":"3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c","share_difficulty":7,"share_id":"share-572","share_seq":572,"template_height":800017},{"accepted_at_ms":999992,"credit_policy":null,"job_id":"job-274","job_issued_at_ms":999978,"miner_id":"miner-10","network_difficulty":1000002,"ntime":1700000274,"order_key":"10:miner-10","p2mr_program_hex":"1212121212121212121212121212121212121212121212121212121212121212","share_difficulty":101,"share_id":"share-274","share_seq":274,"template_height":800008},{"accepted_at_ms":999971,"credit_policy":null,"job_id":"job-1547","job_issued_at_ms":999975,"miner_id":"miner-7","network_difficulty":1000000,"ntime":1700001547,"order_key":"07:miner-7","p2mr_program_hex":"0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b","share_difficulty":101,"share_id":"share-1547","share_seq":1547,"template_height":800048},{"accepted_at_ms":999971,"credit_policy":null,"job_id":"job-1337","job_issued_at_ms":999979,"miner_id":"miner-6","network_difficulty":1000011,"ntime":1700001337,"order_key":"06:miner-6","p2mr_program_hex":"3939393939393939393939393939393939393939393939393939393939393939","share_difficulty":1,"share_id":"share-1337","share_seq":1337,"template_height":800041},{"accepted_at_ms":999998,"credit_policy":null,"job_id":"job-3624","job_issued_at_ms":1000030,"miner_id":"miner-5","network_difficulty":1000003,"ntime":1700003624,"order_key":"05:miner-5","p2mr_program_hex":"2828282828282828282828282828282828282828282828282828282828282828","share_difficulty":7,"share_id":"share-3624","share_seq":3624,"template_height":800113},{"accepted_at_ms":999975,"credit_policy":null,"job_id":"job-2529","job_issued_at_ms":999991,"miner_id":"miner-10","network_difficulty":1000013,"ntime":1700002529,"order_key":"10:miner-10","p2mr_program_hex":"e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1","share_difficulty":101,"share_id":"share-2529","share_seq":2529,"template_height":800079},{"accepted_at_ms":999998,"credit_policy":null,"job_id":"job-2585","job_issued_at_ms":999985,"miner_id":"miner-0","network_difficulty":1000001,"ntime":1700002585,"order_key":"00:miner-0","p2mr_program_hex":"1919191919191919191919191919191919191919191919191919191919191919","share_difficulty":7,"share_id":"share-2585","share_seq":2585,"template_height":800080},{"accepted_at_ms":999971,"credit_policy":null,"job_id":"job-373","job_issued_at_ms":999981,"miner_id":"miner-10","network_difficulty":1000016,"ntime":1700000373,"order_key":"10:miner-10","p2mr_program_hex":"7575757575757575757575757575757575757575757575757575757575757575","share_difficulty":101,"share_id":"share-373","share_seq":373,"template_height":800011},{"accepted_at_ms":999995,"credit_policy":null,"job_id":"job-1090","job_issued_at_ms":999991,"miner_id":"miner-1","network_difficulty":1000002,"ntime":1700001090,"order_key":"01:miner-1","p2mr_program_hex":"4242424242424242424242424242424242424242424242424242424242424242","share_difficulty":3,"share_id":"share-1090","share_seq":1090,"template_height":800034},{"accepted_at_ms":1000487,"credit_policy":null,"job_id":"job-2454","job_issued_at_ms":1000054,"miner_id":"miner-1","network_difficulty":1000006,"ntime":1700002454,"order_key":"01:miner-1","p2mr_program_hex":"9696969696969696969696969696969696969696969696969696969696969696","share_difficulty":19,"share_id":"share-2454","share_seq":2454,"template_height":800076},{"accepted_at_ms":999997,"credit_policy":null,"job_id":"job-3701","job_issued_at_ms":999995,"miner_id":"miner-5","network_difficulty":1000012,"ntime":1700003701,"order_key":"05:miner-5","p2mr_program_hex":"7575757575757575757575757575757575757575757575757575757575757575","share_difficulty":3,"share_id":"share-3701","share_seq":3701,"template_height":800115},{"accepted_at_ms":999978,"credit_policy":null,"job_id":"job-2227","job_issued_at_ms":999988,"miner_id":"miner-5","network_difficulty":1000000,"ntime":1700002227,"order_key":"05:miner-5","p2mr_program_hex":"b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3","share_difficulty":1,"share_id":"share-2227","share_seq":2227,"template_height":800069},{"accepted_at_ms":999994,"credit_policy":"stale-grace","job_id":"job-1401","job_issued_at_ms":999975,"miner_id":"miner-4","network_difficulty":1000007,"ntime":1700001401,"order_key":"04:miner-4","p2mr_program_hex":"7979797979797979797979797979797979797979797979797979797979797979","share_difficulty":2,"share_id":"share-1401","share_seq":1401,"template_height":800043},{"accepted_at_ms":1000117,"credit_policy":null,"job_id":"job-3438","job_issued_at_ms":999989,"miner_id":"miner-6","network_difficulty":1000004,"ntime":1700003438,"order_key":"06:miner-6","p2mr_program_hex":"6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e","share_difficulty":101,"share_id":"share-3438","share_seq":3438,"template_height":800107},{"accepted_at_ms":999995,"credit_policy":null,"job_id":"job-1954","job_issued_at_ms":999971,"miner_id":"miner-7","network_difficulty":1000016,"ntime":1700001954,"order_key":"07:miner-7","p2mr_program_hex":"a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2","share_difficulty":7,"share_id":"share-1954","share_seq":1954,"template_height":800061},{"accepted_at_ms":999986,"credit_policy":null,"job_id":"job-654","job_issued_at_ms":999989,"miner_id":"miner-5","network_difficulty":1000008,"ntime":1700000654,"order_key":"05:miner-5","p2mr_program_hex":"8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e","share_difficulty":1,"share_id":"share-654","share_seq":654,"template_height":800020},{"accepted_at_ms":999971,"credit_policy":null,"job_id":"job-2519","job_issued_at_ms":999999,"miner_id":"miner-0","network_difficulty":1000003,"ntime":1700002519,"order_key":"00:miner-0","p2mr_program_hex":"d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7","share_difficulty":3,"share_id":"share-2519","share_seq":2519,"template_height":800078},{"accepted_at_ms":999980,"credit_policy":"stale-grace","job_id":"job-1686","job_issued_at_ms":999983,"miner_id":"miner-3","network_difficulty":1000003,"ntime":1700001686,"order_key":"03:miner-3","p2mr_program_hex":"9696969696969696969696969696969696969696969696969696969696969696","share_difficulty":7,"share_id":"share-1686","share_seq":1686,"template_height":800052},{"accepted_at_ms":999974,"credit_policy":null,"job_id":"job-826","job_issued_at_ms":999983,"miner_id":"miner-1","network_difficulty":1000010,"ntime":1700000826,"order_key":"01:miner-1","p2mr_program_hex":"3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a","share_difficulty":7,"share_id":"share-826","share_seq":826,"template_height":800025},{"accepted_at_ms":999979,"credit_policy":null,"job_id":"job-3772","job_issued_at_ms":999972,"miner_id":"miner-10","network_difficulty":1000015,"ntime":1700003772,"order_key":"10:miner-10","p2mr_program_hex":"bcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbc","share_difficulty":2,"share_id":"share-3772","share_seq":3772,"template_height":800117},{"accepted_at_ms":999972,"credit_policy":null,"job_id":"job-2930","job_issued_at_ms":999999,"miner_id":"miner-4","network_difficulty":1000006,"ntime":1700002930,"order_key":"04:miner-4","p2mr_program_hex":"7272727272727272727272727272727272727272727272727272727272727272","share_difficulty":1,"share_id":"share-2930","share_seq":2930,"template_height":800091},{"accepted_at_ms":999983,"credit_policy":null,"job_id":"job-1152","job_issued_at_ms":1000273,"miner_id":"miner-8","network_difficulty":1000013,"ntime":1700001152,"order_key":"08:miner-8","p2mr_program_hex":"8080808080808080808080808080808080808080808080808080808080808080","share_difficulty":1000,"share_id":"share-1152","share_seq":1152,"template_height":800036},{"accepted_at_ms":999972,"credit_policy":null,"job_id":"job-2057","job_issued_at_ms":1000389,"miner_id":"miner-0","network_difficulty":1000000,"ntime":1700002057,"order_key":"00:miner-0","p2mr_program_hex":"0909090909090909090909090909090909090909090909090909090909090909","share_difficulty":7,"share_id":"share-2057","share_seq":2057,"template_height":800064},{"accepted_at_ms":1000238,"credit_policy":null,"job_id":"job-3850","job_issued_at_ms":1000453,"miner_id":"miner-0","network_difficulty":1000008,"ntime":1700003850,"order_key":"00:miner-0","p2mr_program_hex":"0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a","share_difficulty":7,"share_id":"share-3850","share_seq":3850,"template_height":800120},{"accepted_at_ms":999985,"credit_policy":null,"job_id":"job-2766","job_issued_at_ms":999999,"miner_id":"miner-5","network_difficulty":1000012,"ntime":1700002766,"order_key":"05:miner-5","p2mr_program_hex":"cececececececececececececececececececececececececececececececece","share_difficulty":1,"share_id":"share-2766","share_seq":2766,"template_height":800086},{"accepted_at_ms":999971,"credit_policy":null,"job_id":"job-2942","job_issued_at_ms":999978,"miner_id":"miner-5","network_difficulty":1000001,"ntime":1700002942,"order_key":"05:miner-5","p2mr_program_hex":"7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e","share_difficulty":3,"share_id":"share-2942","share_seq":2942,"template_height":800091},{"accepted_at_ms":999993,"credit_policy":null,"job_id":"job-837","job_issued_at_ms":999996,"miner_id":"miner-1","network_difficulty":1000004,"ntime":1700000837,"order_key":"01:miner-1","p2mr_program_hex":"4545454545454545454545454545454545454545454545454545454545454545","share_difficulty":1,"share_id":"share-837","share_seq":837,"template_height":800026},{"accepted_at_ms":999998,"credit_policy":null,"job_id":"job-2646","job_issued_at_ms":999988,"miner_id":"miner-6","network_difficulty":1000011,"ntime":1700002646,"order_key":"06:miner-6","p2mr_program_hex":"5656565656565656565656565656565656565656565656565656565656565656","share_difficulty":7,"share_id":"share-2646","share_seq":2646,"template_height":800082},{"accepted_at_ms":999971,"credit_policy":null,"job_id":"job-2481","job_issued_at_ms":999982,"miner_id":"miner-6","network_difficulty":1000016,"ntime":1700002481,"order_key":"06:miner-6","p2mr_program_hex":"b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1","share_difficulty":1,"share_id":"share-2481","share_seq":2481,"template_height":800077},{"accepted_at_ms":999996,"credit_policy":null,"job_id":"job-1837","job_issued_at_ms":999983,"miner_id":"miner-0","network_difficulty":1000001,"ntime":1700001837,"order_key":"00:miner-0","p2mr_program_hex":"2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d","share_difficulty":19,"share_id":"share-1837","share_seq":1837,"template_height":800057},{"accepted_at_ms":1000285,"credit_policy":null,"job_id":"job-1778","job_issued_at_ms":1000026,"miner_id":"miner-7","network_difficulty":1000010,"ntime":1700001778,"order_key":"07:miner-7","p2mr_program_hex":"f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2","share_difficulty":1000,"share_id":"share-1778","share_seq":1778,"template_height":800055},{"accepted_at_ms":999992,"credit_policy":null,"job_id":"job-3944","job_issued_at_ms":999982,"miner_id":"miner-6","network_difficulty":1000000,"ntime":1700003944,"order_key":"06:miner-6","p2mr_program_hex":"6868686868686868686868686868686868686868686868686868686868686868","share_difficulty":19,"share_id":"share-3944","share_seq":3944,"template_height":800123},{"accepted_at_ms":999995,"credit_policy":null,"job_id":"job-3603","job_issued_at_ms":999980,"miner_id":"miner-6","network_difficulty":1000016,"ntime":1700003603,"order_key":"06:miner-6","p2mr_program_hex":"1313131313131313131313131313131313131313131313131313131313131313","share_difficulty":19,"share_id":"share-3603","share_seq":3603,"template_height":800112},{"accepted_at_ms":999973,"credit_policy":null,"job_id":"job-3480","job_issued_at_ms":999975,"miner_id":"miner-4","network_difficulty":1000012,"ntime":1700003480,"order_key":"04:miner-4","p2mr_program_hex":"9898989898989898989898989898989898989898989898989898989898989898","share_difficulty":101,"share_id":"share-3480","share_seq":3480,"template_height":800108},{"accepted_at_ms":999979,"credit_policy":"stale-grace","job_id":"job-3957","job_issued_at_ms":999976,"miner_id":"miner-8","network_difficulty":1000013,"ntime":1700003957,"order_key":"08:miner-8","p2mr_program_hex":"7575757575757575757575757575757575757575757575757575757575757575","share_difficulty":101,"share_id":"share-3957","share_seq":3957,"template_height":800123},{"accepted_at_ms":1000000,"credit_policy":null,"job_id":"job-773","job_issued_at_ms":999978,"miner_id":"miner-3","network_difficulty":1000008,"ntime":1700000773,"order_key":"03:miner-3","p2mr_program_hex":"0505050505050505050505050505050505050505050505050505050505050505","share_difficulty":1,"share_id":"share-773","share_seq":773,"template_height":800024},{"accepted_at_ms":999971,"credit_policy":"stale-grace","job_id":"job-197","job_issued_at_ms":999997,"miner_id":"miner-10","network_difficulty":1000010,"ntime":1700000197,"order_key":"10:miner-10","p2mr_program_hex":"c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5","share_difficulty":7,"share_id":"share-197","share_seq":197,"template_height":800006},{"accepted_at_ms":999976,"credit_policy":null,"job_id":"job-3625","job_issued_at_ms":999972,"miner_id":"miner-6","network_difficulty":1000004,"ntime":1700003625,"order_key":"06:miner-6","p2mr_program_hex":"2929292929292929292929292929292929292929292929292929292929292929","share_difficulty":1,"share_id":"share-3625","share_seq":3625,"template_height":800113},{"accepted_at_ms":999976,"credit_policy":null,"job_id":"job-1698","job_issued_at_ms":999974,"miner_id":"miner-4","network_difficulty":1000015,"ntime":1700001698,"order_key":"04:miner-4","p2mr_program_hex":"a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2","share_difficulty":7,"share_id":"share-1698","share_seq":1698,"template_height":800053},{"accepted_at_ms":999988,"credit_policy":null,"job_id":"job-2501","job_issued_at_ms":1000000,"miner_id":"miner-4","network_difficulty":1000002,"ntime":1700002501,"order_key":"04:miner-4","p2mr_program_hex":"c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5","share_difficulty":2,"share_id":"share-2501","share_seq":2501,"template_height":800078},{"accepted_at_ms":999981,"credit_policy":null,"job_id":"job-797","job_issued_at_ms":999985,"miner_id":"miner-5","network_difficulty":1000015,"ntime":1700000797,"order_key":"05:miner-5","p2mr_program_hex":"1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d","share_difficulty":101,"share_id":"share-797","share_seq":797,"template_height":800024},{"accepted_at_ms":999980,"credit_policy":null,"job_id":"job-3413","job_issued_at_ms":999979,"miner_id":"miner-3","network_difficulty":1000013,"ntime":1700003413,"order_key":"03:miner-3","p2mr_program_hex":"5555555555555555555555555555555555555555555555555555555555555555","share_difficulty":1000,"share_id":"share-3413","share_seq":3413,"template_height":800106},{"accepted_at_ms":999979,"credit_policy":null,"job_id":"job-2511","job_issued_at_ms":999994,"miner_id":"miner-3","network_difficulty":1000012,"ntime":1700002511,"order_key":"03:miner-3","p2mr_program_hex":"cfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcf","share_difficulty":1000,"share_id":"share-2511","share_seq":2511,"template_height":800078},{"accepted_at_ms":999974,"credit_policy":null,"job_id":"job-1126","job_issued_at_ms":999985,"miner_id":"miner-4","network_difficulty":1000004,"ntime":1700001126,"order_key":"04:miner-4","p2mr_program_hex":"6666666666666666666666666666666666666666666666666666666666666666","share_difficulty":3,"share_id":"share-1126","share_seq":1126,"template_height":800035},{"accepted_at_ms":999998,"credit_policy":"stale-grace","job_id":"job-2506","job_issued_at_ms":999994,"miner_id":"miner-9","network_difficulty":1000007,"ntime":1700002506,"order_key":"09:miner-9","p2mr_program_hex":"cacacacacacacacacacacacacacacacacacacacacacacacacacacacacacacaca","share_difficulty":2,"share_id":"share-2506","share_seq":2506,"template_height":800078},{"accepted_at_ms":999989,"credit_policy":null,"job_id":"job-942","job_issued_at_ms":999974,"miner_id":"miner-7","network_difficulty":1000007,"ntime":1700000942,"order_key":"07:miner-7","p2mr_program_hex":"aeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeae","share_difficulty":7,"share_id":"share-942","share_seq":942,"template_height":800029},{"accepted_at_ms":999991,"credit_policy":null,"job_id":"job-2439","job_issued_at_ms":999990,"miner_id":"miner-8","network_difficulty":1000008,"ntime":1700002439,"order_key":"08:miner-8","p2mr_program_hex":"8787878787878787878787878787878787878787878787878787878787878787","share_difficulty":1000,"share_id":"share-2439","share_seq":2439,"template_height":800076},{"accepted_at_ms":999981,"credit_policy":null,"job_id":"job-3245","job_issued_at_ms":999998,"miner_id":"miner-0","network_difficulty":1000015,"ntime":1700003245,"order_key":"00:miner-0","p2mr_program_hex":"adadadadadadadadadadadadadadadadadadadadadadadadadadadadadadadad","share_difficulty":1000,"share_id":"share-3245","share_seq":3245,"template_height":800101},{"accepted_at_ms":999998,"credit_policy":null,"job_id":"job-806","job_issued_at_ms":999980,"miner_id":"miner-3","network_difficulty":1000007,"ntime":1700000806,"order_key":"03:miner-3","p2mr_program_hex":"2626262626262626262626262626262626262626262626262626262626262626","share_difficulty":3,"share_id":"share-806","share_seq":806,"template_height":800025},{"accepted_at_ms":999997,"credit_policy":null,"job_id":"job-3316","job_issued_at_ms":999975,"miner_id":"miner-5","network_difficulty":1000001,"ntime":1700003316,"order_key":"05:miner-5","p2mr_program_hex":"f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4","share_difficulty":1,"share_id":"share-3316","share_seq":3316,"template_height":800103},{"accepted_at_ms":999997,"credit_policy":null,"job_id":"job-3362","job_issued_at_ms":999974,"miner_id":"miner-7","network_difficulty":1000013,"ntime":1700003362,"order_key":"07:miner-7","p2mr_program_hex":"2222222222222222222222222222222222222222222222222222222222222222","share_difficulty":7,"share_id":"share-3362","share_seq":3362,"template_height":800105},{"accepted_at_ms":999991,"credit_policy":"stale-grace","job_id":"job-902","job_issued_at_ms":1000178,"miner_id":"miner-0","network_difficulty":1000001,"ntime":1700000902,"order_key":"00:miner-0","p2mr_program_hex":"8686868686868686868686868686868686868686868686868686868686868686","share_difficulty":1000,"share_id":"share-902","share_seq":902,"template_height":800028},{"accepted_at_ms":999972,"credit_policy":null,"job_id":"job-1508","job_issued_at_ms":999991,"miner_id":"miner-1","network_difficulty":1000012,"ntime":1700001508,"order_key":"01:miner-1","p2mr_program_hex":"e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4","share_difficulty":7,"share_id":"share-1508","share_seq":1508,"template_height":800047},{"accepted_at_ms":999993,"credit_policy":null,"job_id":"job-1368","job_issued_at_ms":999974,"miner_id":"miner-4","network_difficulty":1000008,"ntime":1700001368,"order_key":"04:miner-4","p2mr_program_hex":"5858585858585858585858585858585858585858585858585858585858585858","share_difficulty":2,"share_id":"share-1368","share_seq":1368,"template_height":800042},{"accepted_at_ms":999972,"credit_policy":"stale-grace","job_id":"job-2938","job_issued_at_ms":999994,"miner_id":"miner-1","network_difficulty":1000014,"ntime":1700002938,"order_key":"01:miner-1","p2mr_program_hex":"7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a","share_difficulty":1,"share_id":"share-2938","share_seq":2938,"template_height":800091},{"accepted_at_ms":999996,"credit_policy":null,"job_id":"job-575","job_issued_at_ms":999996,"miner_id":"miner-3","network_difficulty":1000014,"ntime":1700000575,"order_key":"03:miner-3","p2mr_program_hex":"3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f","share_difficulty":101,"share_id":"share-575","share_seq":575,"template_height":800017},{"accepted_at_ms":999986,"credit_policy":null,"job_id":"job-2470","job_issued_at_ms":999987,"miner_id":"miner-6","network_difficulty":1000005,"ntime":1700002470,"order_key":"06:miner-6","p2mr_program_hex":"a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6","share_difficulty":1000,"share_id":"share-2470","share_seq":2470,"template_height":800077},{"accepted_at_ms":999991,"credit_policy":null,"job_id":"job-2523","job_issued_at_ms":999982,"miner_id":"miner-4","network_difficulty":1000007,"ntime":1700002523,"order_key":"04:miner-4","p2mr_program_hex":"dbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdb","share_difficulty":19,"share_id":"share-2523","share_seq":2523,"template_height":800078},{"accepted_at_ms":999996,"credit_policy":null,"job_id":"job-48","job_issued_at_ms":999974,"miner_id":"miner-4","network_difficulty":1000014,"ntime":1700000048,"order_key":"04:miner-4","p2mr_program_hex":"3030303030303030303030303030303030303030303030303030303030303030","share_difficulty":101,"share_id":"share-48","share_seq":48,"template_height":800001},{"accepted_at_ms":999975,"credit_policy":"stale-grace","job_id":"job-2228","job_issued_at_ms":999970,"miner_id":"miner-6","network_difficulty":1000001,"ntime":1700002228,"order_key":"06:miner-6","p2mr_program_hex":"b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4","share_difficulty":3,"share_id":"share-2228","share_seq":2228,"template_height":800069},{"accepted_at_ms":999974,"credit_policy":null,"job_id":"job-2265","job_issued_at_ms":999978,"miner_id":"miner-10","network_difficulty":1000004,"ntime":1700002265,"order_key":"10:miner-10","p2mr_program_hex":"d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9","share_difficulty":7,"share_id":"share-2265","share_seq":2265,"template_height":800070},{"accepted_at_ms":999979,"credit_policy":null,"job_id":"job-3414","job_issued_at_ms":999993,"miner_id":"miner-4","network_difficulty":1000014,"ntime":1700003414,"order_key":"04:miner-4","p2mr_program_hex":"5656565656565656565656565656565656565656565656565656565656565656","share_difficulty":1,"share_id":"share-3414","share_seq":3414,"template_height":800106},{"accepted_at_ms":999990,"credit_policy":"stale-grace","job_id":"job-2903","job_issued_at_ms":999995,"miner_id":"miner-10","network_difficulty":1000013,"ntime":1700002903,"order_key":"10:miner-10","p2mr_program_hex":"5757575757575757575757575757575757575757575757575757575757575757","share_difficulty":19,"share_id":"share-2903","share_seq":2903,"template_height":800090},{"accepted_at_ms":999998,"credit_policy":null,"job_id":"job-1757","job_issued_at_ms":999976,"miner_id":"miner-8","network_difficulty":1000006,"ntime":1700001757,"order_key":"08:miner-8","p2mr_program_hex":"dddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddd","share_difficulty":1,"share_id":"share-1757","share_seq":1757,"template_height":800054},{"accepted_at_ms":999979,"credit_policy":null,"job_id":"job-2170","job_issued_at_ms":999985,"miner_id":"miner-3","network_difficulty":1000011,"ntime":1700002170,"order_key":"03:miner-3","p2mr_program_hex":"7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a","share_difficulty":2,"share_id":"share-2170","share_seq":2170,"template_height":800067},{"accepted_at_ms":999982,"credit_policy":null,"job_id":"job-3961","job_issued_at_ms":1000000,"miner_id":"miner-1","network_difficulty":1000000,"ntime":1700003961,"order_key":"01:miner-1","p2mr_program_hex":"7979797979797979797979797979797979797979797979797979797979797979","share_difficulty":7,"share_id":"share-3961","share_seq":3961,"template_height":800123},{"accepted_at_ms":999976,"credit_policy":null,"job_id":"job-176","job_issued_at_ms":999975,"miner_id":"miner-0","network_difficulty":1000006,"ntime":1700000176,"order_key":"00:miner-0","p2mr_program_hex":"b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0","share_difficulty":1,"share_id":"share-176","share_seq":176,"template_height":800005},{"accepted_at_ms":999997,"credit_policy":null,"job_id":"job-3581","job_issued_at_ms":999971,"miner_id":"miner-6","network_difficulty":1000011,"ntime":1700003581,"order_key":"06:miner-6","p2mr_program_hex":"fdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfd","share_difficulty":1000,"share_id":"share-3581","share_seq":3581,"template_height":800111},{"accepted_at_ms":999977,"credit_policy":"stale-grace","job_id":"job-1493","job_issued_at_ms":999990,"miner_id":"miner-8","network_difficulty":1000014,"ntime":1700001493,"order_key":"08:miner-8","p2mr_program_hex":"d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5","share_difficulty":2,"share_id":"share-1493","share_seq":1493,"template_height":800046},{"accepted_at_ms":999994,"credit_policy":null,"job_id":"job-3365","job_issued_at_ms":999971,"miner_id":"miner-10","network_difficulty":1000016,"ntime":1700003365,"order_key":"10:miner-10","p2mr_program_hex":"2525252525252525252525252525252525252525252525252525252525252525","share_difficulty":3,"share_id":"share-3365","share_seq":3365,"template_height":800105},{"accepted_at_ms":999981,"credit_policy":"stale-grace","job_id":"job-3674","job_issued_at_ms":999976,"miner_id":"miner-0","network_difficulty":1000002,"ntime":1700003674,"order_key":"00:miner-0","p2mr_program_hex":"5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a","share_difficulty":1000,"share_id":"share-3674","share_seq":3674,"template_height":800114},{"accepted_at_ms":999973,"credit_policy":null,"job_id":"job-1365","job_issued_at_ms":999976,"miner_id":"miner-1","network_difficulty":1000005,"ntime":1700001365,"order_key":"01:miner-1","p2mr_program_hex":"5555555555555555555555555555555555555555555555555555555555555555","share_difficulty":2,"share_id":"share-1365","share_seq":1365,"template_height":800042},{"accepted_at_ms":999996,"credit_policy":null,"job_id":"job-1711","job_issued_at_ms":999997,"miner_id":"miner-6","network_difficulty":1000011,"ntime":1700001711,"order_key":"06:miner-6","p2mr_program_hex":"afafafafafafafafafafafafafafafafafafafafafafafafafafafafafafafaf","share_difficulty":101,"share_id":"share-1711","share_seq":1711,"template_height":800053},{"accepted_at_ms":999974,"credit_policy":null,"job_id":"job-1217","job_issued_at_ms":999980,"miner_id":"miner-7","network_difficulty":1000010,"ntime":1700001217,"order_key":"07:miner-7","p2mr_program_hex":"c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1","share_difficulty":101,"share_id":"share-1217","share_seq":1217,"template_height":800038},{"accepted_at_ms":999970,"credit_policy":"stale-grace","job_id":"job-1103","job_issued_at_ms":999984,"miner_id":"miner-3","network_difficulty":1000015,"ntime":1700001103,"order_key":"03:miner-3","p2mr_program_hex":"4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f","share_difficulty":7,"share_id":"share-1103","share_seq":1103,"template_height":800034},{"accepted_at_ms":999981,"credit_policy":null,"job_id":"job-1895","job_issued_at_ms":999976,"miner_id":"miner-3","network_difficulty":1000008,"ntime":1700001895,"order_key":"03:miner-3","p2mr_program_hex":"6767676767676767676767676767676767676767676767676767676767676767","share_difficulty":1,"share_id":"share-1895","share_seq":1895,"template_height":800059},{"accepted_at_ms":999992,"credit_policy":null,"job_id":"job-1120","job_issued_at_ms":999985,"miner_id":"miner-9","network_difficulty":1000015,"ntime":1700001120,"order_key":"09:miner-9","p2mr_program_hex":"6060606060606060606060606060606060606060606060606060606060606060","share_difficulty":19,"share_id":"share-1120","share_seq":1120,"template_height":800035},{"accepted_at_ms":1000416,"credit_policy":null,"job_id":"job-2141","job_issued_at_ms":1000246,"miner_id":"miner-7","network_difficulty":1000016,"ntime":1700002141,"order_key":"07:miner-7","p2mr_program_hex":"5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d","share_difficulty":101,"share_id":"share-2141","share_seq":2141,"template_height":800066},{"accepted_at_ms":1000000,"credit_policy":null,"job_id":"job-2254","job_issued_at_ms":999973,"miner_id":"miner-10","network_difficulty":1000010,"ntime":1700002254,"order_key":"10:miner-10","p2mr_program_hex":"cececececececececececececececececececececececececececececececece","share_difficulty":1000,"share_id":"share-2254","share_seq":2254,"template_height":800070},{"accepted_at_ms":999987,"credit_policy":null,"job_id":"job-816","job_issued_at_ms":999977,"miner_id":"miner-2","network_difficulty":1000000,"ntime":1700000816,"order_key":"02:miner-2","p2mr_program_hex":"3030303030303030303030303030303030303030303030303030303030303030","share_difficulty":1,"share_id":"share-816","share_seq":816,"template_height":800025},{"accepted_at_ms":999996,"credit_policy":null,"job_id":"job-3155","job_issued_at_ms":999997,"miner_id":"miner-9","network_difficulty":1000010,"ntime":1700003155,"order_key":"09:miner-9","p2mr_program_hex":"5353535353535353535353535353535353535353535353535353535353535353","share_difficulty":1,"share_id":"share-3155","share_seq":3155,"template_height":800098},{"accepted_at_ms":999983,"credit_policy":null,"job_id":"job-50","job_issued_at_ms":999980,"miner_id":"miner-6","network_difficulty":1000016,"ntime":1700000050,"order_key":"06:miner-6","p2mr_program_hex":"3232323232323232323232323232323232323232323232323232323232323232","share_difficulty":101,"share_id":"share-50","share_seq":50,"template_height":800001},{"accepted_at_ms":999975,"credit_policy":null,"job_id":"job-3791","job_issued_at_ms":999985,"miner_id":"miner-7","network_difficulty":1000000,"ntime":1700003791,"order_key":"07:miner-7","p2mr_program_hex":"cfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcf","share_difficulty":101,"share_id":"share-3791","share_seq":3791,"template_height":800118},{"accepted_at_ms":999998,"credit_policy":null,"job_id":"job-3235","job_issued_at_ms":999988,"miner_id":"miner-1","network_difficulty":1000005,"ntime":1700003235,"order_key":"01:miner-1","p2mr_program_hex":"a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3","share_difficulty":1000,"share_id":"share-3235","share_seq":3235,"template_height":800101},{"accepted_at_ms":999986,"credit_policy":null,"job_id":"job-2402","job_issued_at_ms":999979,"miner_id":"miner-4","network_difficulty":1000005,"ntime":1700002402,"order_key":"04:miner-4","p2mr_program_hex":"6262626262626262626262626262626262626262626262626262626262626262","share_difficulty":7,"share_id":"share-2402","share_seq":2402,"template_height":800075},{"accepted_at_ms":999981,"credit_policy":null,"job_id":"job-3361","job_issued_at_ms":999979,"miner_id":"miner-6","network_difficulty":1000012,"ntime":1700003361,"order_key":"06:miner-6","p2mr_program_hex":"2121212121212121212121212121212121212121212121212121212121212121","share_difficulty":19,"share_id":"share-3361","share_seq":3361,"template_height":800105},{"accepted_at_ms":999983,"credit_policy":"stale-grace","job_id":"job-631","job_issued_at_ms":999974,"miner_id":"miner-4","network_difficulty":1000002,"ntime":1700000631,"order_key":"04:miner-4","p2mr_program_hex":"7777777777777777777777777777777777777777777777777777777777777777","share_difficulty":3,"share_id":"share-631","share_seq":631,"template_height":800019},{"accepted_at_ms":1000407,"credit_policy":"stale-grace","job_id":"job-297","job_issued_at_ms":1000389,"miner_id":"miner-0","network_difficulty":1000008,"ntime":1700000297,"order_key":"00:miner-0","p2mr_program_hex":"2929292929292929292929292929292929292929292929292929292929292929","share_difficulty":1000,"share_id":"share-297","share_seq":297,"template_height":800009},{"accepted_at_ms":1000172,"credit_policy":"stale-grace","job_id":"job-377","job_issued_at_ms":1000324,"miner_id":"miner-3","network_difficulty":1000003,"ntime":1700000377,"order_key":"03:miner-3","p2mr_program_hex":"7979797979797979797979797979797979797979797979797979797979797979","share_difficulty":1,"share_id":"share-377","share_seq":377,"template_height":800011},{"accepted_at_ms":999977,"credit_policy":null,"job_id":"job-1860","job_issued_at_ms":999975,"miner_id":"miner-1","network_difficulty":1000007,"ntime":1700001860,"order_key":"01:miner-1","p2mr_program_hex":"4444444444444444444444444444444444444444444444444444444444444444","share_difficulty":2,"share_id":"share-1860","share_seq":1860,"template_height":800058},{"accepted_at_ms":999984,"credit_policy":null,"job_id":"job-2077","job_issued_at_ms":999978,"miner_id":"miner-9","network_difficulty":1000003,"ntime":1700002077,"order_key":"09:miner-9","p2mr_program_hex":"1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d","share_difficulty":3,"share_id":"share-2077","share_seq":2077,"template_height":800064},{"accepted_at_ms":999996,"credit_policy":null,"job_id":"job-3328","job_issued_at_ms":999993,"miner_id":"miner-6","network_difficulty":1000013,"ntime":1700003328,"order_key":"06:miner-6","p2mr_program_hex":"0000000000000000000000000000000000000000000000000000000000000000","share_difficulty":1,"share_id":"share-3328","share_seq":3328,"template_height":800104},{"accepted_at_ms":999974,"credit_policy":null,"job_id":"job-1363","job_issued_at_ms":1000405,"miner_id":"miner-10","network_difficulty":1000003,"ntime":1700001363,"order_key":"10:miner-10","p2mr_program_hex":"5353535353535353535353535353535353535353535353535353535353535353","share_difficulty":2,"share_id":"share-1363","share_seq":1363,"template_height":800042},{"accepted_at_ms":999996,"credit_policy":null,"job_id":"job-2655","job_issued_at_ms":999973,"miner_id":"miner-4","network_difficulty":1000003,"ntime":1700002655,"order_key":"04:miner-4","p2mr_program_hex":"5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f","share_difficulty":7,"share_id":"share-2655","share_seq":2655,"template_height":800082},{"accepted_at_ms":999996,"credit_policy":null,"job_id":"job-1940","job_issued_at_ms":1000000,"miner_id":"miner-4","network_difficulty":1000002,"ntime":1700001940,"order_key":"04:miner-4","p2mr_program_hex":"9494949494949494949494949494949494949494949494949494949494949494","share_difficulty":1000,"share_id":"share-1940","share_seq":1940,"template_height":800060},{"accepted_at_ms":999990,"credit_policy":null,"job_id":"job-3310","job_issued_at_ms":999970,"miner_id":"miner-10","network_difficulty":1000012,"ntime":1700003310,"order_key":"10:miner-10","p2mr_program_hex":"eeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeee","share_difficulty":7,"share_id":"share-3310","share_seq":3310,"template_height":800103},{"accepted_at_ms":999982,"credit_policy":null,"job_id":"job-3554","job_issued_at_ms":999970,"miner_id":"miner-1","network_difficulty":1000001,"ntime":1700003554,"order_key":"01:miner-1","p2mr_program_hex":"e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2","share_difficulty":101,"share_id":"share-3554","share_seq":3554,"template_height":800111},{"accepted_at_ms":999989,"credit_policy":null,"job_id":"job-1665","job_issued_at_ms":999979,"miner_id":"miner-4","network_difficulty":1000016,"ntime":1700001665,"order_key":"04:miner-4","p2mr_program_hex":"8181818181818181818181818181818181818181818181818181818181818181","share_difficulty":1,"share_id":"share-1665","share_seq":1665,"template_height":800052},{"accepted_at_ms":999999,"credit_policy":"stale-grace","job_id":"job-3869","job_issued_at_ms":999972,"miner_id":"miner-8","network_difficulty":1000010,"ntime":1700003869,"order_key":"08:miner-8","p2mr_program_hex":"1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d","share_difficulty":19,"share_id":"share-3869","share_seq":3869,"template_height":800120},{"accepted_at_ms":999978,"credit_policy":"stale-grace","job_id":"job-2664","job_issued_at_ms":999996,"miner_id":"miner-2","network_difficulty":1000012,"ntime":1700002664,"order_key":"02:miner-2","p2mr_program_hex":"6868686868686868686868686868686868686868686868686868686868686868","share_difficulty":7,"share_id":"share-2664","share_seq":2664,"template_height":800083},{"accepted_at_ms":999996,"credit_policy":null,"job_id":"job-4000","job_issued_at_ms":999972,"miner_id":"miner-7","network_difficulty":1000005,"ntime":1700004000,"order_key":"07:miner-7","p2mr_program_hex":"a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0","share_difficulty":101,"share_id":"share-4000","share_seq":4000,"template_height":800125},{"accepted_at_ms":999994,"credit_policy":"stale-grace","job_id":"job-670","job_issued_at_ms":999980,"miner_id":"miner-10","network_difficulty":1000007,"ntime":1700000670,"order_key":"10:miner-10","p2mr_program_hex":"9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e","share_difficulty":3,"share_id":"share-670","share_seq":670,"template_height":800020},{"accepted_at_ms":999981,"credit_policy":"stale-grace","job_id":"job-2424","job_issued_at_ms":999994,"miner_id":"miner-4","network_difficulty":1000010,"ntime":1700002424,"order_key":"04:miner-4","p2mr_program_hex":"7878787878787878787878787878787878787878787878787878787878787878","share_difficulty":7,"share_id":"share-2424","share_seq":2424,"template_height":800075},{"accepted_at_ms":999990,"credit_policy":null,"job_id":"job-1853","job_issued_at_ms":999989,"miner_id":"miner-5","network_difficulty":1000000,"ntime":1700001853,"order_key":"05:miner-5","p2mr_program_hex":"3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d","share_difficulty":7,"share_id":"share-1853","share_seq":1853,"template_height":800057},{"accepted_at_ms":999988,"credit_policy":null,"job_id":"job-1331","job_issued_at_ms":999974,"miner_id":"miner-0","network_difficulty":1000005,"ntime":1700001331,"order_key":"00:miner-0","p2mr_program_hex":"3333333333333333333333333333333333333333333333333333333333333333","share_difficulty":2,"share_id":"share-1331","share_seq":1331,"template_height":800041},{"accepted_at_ms":999972,"credit_policy":null,"job_id":"job-1671","job_issued_at_ms":999981,"miner_id":"miner-10","network_difficulty":1000005,"ntime":1700001671,"order_key":"10:miner-10","p2mr_program_hex":"8787878787878787878787878787878787878787878787878787878787878787","share_difficulty":1,"share_id":"share-1671","share_seq":1671,"template_height":800052},{"accepted_at_ms":999989,"credit_policy":null,"job_id":"job-604","job_issued_at_ms":999974,"miner_id":"miner-10","network_difficulty":1000009,"ntime":1700000604,"order_key":"10:miner-10","p2mr_program_hex":"5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c","share_difficulty":19,"share_id":"share-604","share_seq":604,"template_height":800018},{"accepted_at_ms":999995,"credit_policy":null,"job_id":"job-1415","job_issued_at_ms":999987,"miner_id":"miner-7","network_difficulty":1000004,"ntime":1700001415,"order_key":"07:miner-7","p2mr_program_hex":"8787878787878787878787878787878787878787878787878787878787878787","share_difficulty":1000,"share_id":"share-1415","share_seq":1415,"template_height":800044},{"accepted_at_ms":999978,"credit_policy":"stale-grace","job_id":"job-2769","job_issued_at_ms":999992,"miner_id":"miner-8","network_difficulty":1000015,"ntime":1700002769,"order_key":"08:miner-8","p2mr_program_hex":"d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1","share_difficulty":1,"share_id":"share-2769","share_seq":2769,"template_height":800086},{"accepted_at_ms":999997,"credit_policy":"stale-grace","job_id":"job-42","job_issued_at_ms":999980,"miner_id":"miner-9","network_difficulty":1000008,"ntime":1700000042,"order_key":"09:miner-9","p2mr_program_hex":"2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a","share_difficulty":1000,"share_id":"share-42","share_seq":42,"template_height":800001},{"accepted_at_ms":999998,"credit_policy":null,"job_id":"job-2936","job_issued_at_ms":999973,"miner_id":"miner-10","network_difficulty":1000012,"ntime":1700002936,"order_key":"10:miner-10","p2mr_program_hex":"7878787878787878787878787878787878787878787878787878787878787878","share_difficulty":7,"share_id":"share-2936","share_seq":2936,"template_height":800091},{"accepted_at_ms":999970,"credit_policy":null,"job_id":"job-2352","job_issued_at_ms":999978,"miner_id":"miner-9","network_difficulty":1000006,"ntime":1700002352,"order_key":"09:miner-9","p2mr_program_hex":"3030303030303030303030303030303030303030303030303030303030303030","share_difficulty":7,"share_id":"share-2352","share_seq":2352,"template_height":800073},{"accepted_at_ms":999995,"credit_policy":null,"job_id":"job-2375","job_issued_at_ms":999975,"miner_id":"miner-10","network_difficulty":1000012,"ntime":1700002375,"order_key":"10:miner-10","p2mr_program_hex":"4747474747474747474747474747474747474747474747474747474747474747","share_difficulty":7,"share_id":"share-2375","share_seq":2375,"template_height":800074},{"accepted_at_ms":999996,"credit_policy":null,"job_id":"job-1372","job_issued_at_ms":1000484,"miner_id":"miner-8","network_difficulty":1000012,"ntime":1700001372,"order_key":"08:miner-8","p2mr_program_hex":"5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c","share_difficulty":7,"share_id":"share-1372","share_seq":1372,"template_height":800042},{"accepted_at_ms":999982,"credit_policy":null,"job_id":"job-3211","job_issued_at_ms":999979,"miner_id":"miner-10","network_difficulty":1000015,"ntime":1700003211,"order_key":"10:miner-10","p2mr_program_hex":"8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b","share_difficulty":7,"share_id":"share-3211","share_seq":3211,"template_height":800100},{"accepted_at_ms":999994,"credit_policy":"stale-grace","job_id":"job-333","job_issued_at_ms":999982,"miner_id":"miner-3","network_difficulty":1000010,"ntime":1700000333,"order_key":"03:miner-3","p2mr_program_hex":"4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d","share_difficulty":2,"share_id":"share-333","share_seq":333,"template_height":800010},{"accepted_at_ms":999973,"credit_policy":null,"job_id":"job-449","job_issued_at_ms":999990,"miner_id":"miner-9","network_difficulty":1000007,"ntime":1700000449,"order_key":"09:miner-9","p2mr_program_hex":"c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1","share_difficulty":1,"share_id":"share-449","share_seq":449,"template_height":800014},{"accepted_at_ms":999980,"credit_policy":null,"job_id":"job-3880","job_issued_at_ms":999973,"miner_id":"miner-8","network_difficulty":1000004,"ntime":1700003880,"order_key":"08:miner-8","p2mr_program_hex":"2828282828282828282828282828282828282828282828282828282828282828","share_difficulty":1000,"share_id":"share-3880","share_seq":3880,"template_height":800121},{"accepted_at_ms":999990,"credit_policy":"stale-grace","job_id":"job-878","job_issued_at_ms":999981,"miner_id":"miner-9","network_difficulty":1000011,"ntime":1700000878,"order_key":"09:miner-9","p2mr_program_hex":"6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e","share_difficulty":1,"share_id":"share-878","share_seq":878,"template_height":800027},{"accepted_at_ms":999974,"credit_policy":"stale-grace","job_id":"job-2015","job_issued_at_ms":999991,"miner_id":"miner-2","network_difficulty":1000009,"ntime":1700002015,"order_key":"02:miner-2","p2mr_program_hex":"dfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdf","share_difficulty":3,"share_id":"share-2015","share_seq":2015,"template_height":800062},{"accepted_at_ms":999992,"credit_policy":null,"job_id":"job-1093","job_issued_at_ms":999977,"miner_id":"miner-4","network_difficulty":1000005,"ntime":1700001093,"order_key":"04:miner-4","p2mr_program_hex":"4545454545454545454545454545454545454545454545454545454545454545","share_difficulty":3,"share_id":"share-1093","share_seq":1093,"template_height":800034},{"accepted_at_ms":999974,"credit_policy":null,"job_id":"job-3528","job_issued_at_ms":999998,"miner_id":"miner-8","network_difficulty":1000009,"ntime":1700003528,"order_key":"08:miner-8","p2mr_program_hex":"c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8","share_difficulty":7,"share_id":"share-3528","share_seq":3528,"template_height":800110},{"accepted_at_ms":1000000,"credit_policy":null,"job_id":"job-967","job_issued_at_ms":999996,"miner_id":"miner-10","network_difficulty":1000015,"ntime":1700000967,"order_key":"10:miner-10","p2mr_program_hex":"c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7","share_difficulty":19,"share_id":"share-967","share_seq":967,"template_height":800030},{"accepted_at_ms":999980,"credit_policy":null,"job_id":"job-3462","job_issued_at_ms":1000000,"miner_id":"miner-8","network_difficulty":1000011,"ntime":1700003462,"order_key":"08:miner-8","p2mr_program_hex":"8686868686868686868686868686868686868686868686868686868686868686","share_difficulty":1,"share_id":"share-3462","share_seq":3462,"template_height":800108},{"accepted_at_ms":999990,"credit_policy":null,"job_id":"job-2136","job_issued_at_ms":999988,"miner_id":"miner-2","network_difficulty":1000011,"ntime":1700002136,"order_key":"02:miner-2","p2mr_program_hex":"5858585858585858585858585858585858585858585858585858585858585858","share_difficulty":3,"share_id":"share-2136","share_seq":2136,"template_height":800066},{"accepted_at_ms":1000000,"credit_policy":null,"job_id":"job-950","job_issued_at_ms":999975,"miner_id":"miner-4","network_difficulty":1000015,"ntime":1700000950,"order_key":"04:miner-4","p2mr_program_hex":"b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6","share_difficulty":2,"share_id":"share-950","share_seq":950,"template_height":800029},{"accepted_at_ms":999997,"credit_policy":null,"job_id":"job-799","job_issued_at_ms":999977,"miner_id":"miner-7","network_difficulty":1000000,"ntime":1700000799,"order_key":"07:miner-7","p2mr_program_hex":"1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f","share_difficulty":3,"share_id":"share-799","share_seq":799,"template_height":800024},{"accepted_at_ms":999979,"credit_policy":null,"job_id":"job-2683","job_issued_at_ms":999971,"miner_id":"miner-10","network_difficulty":1000014,"ntime":1700002683,"order_key":"10:miner-10","p2mr_program_hex":"7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b","share_difficulty":3,"share_id":"share-2683","share_seq":2683,"template_height":800083},{"accepted_at_ms":999982,"credit_policy":null,"job_id":"job-1690","job_issued_at_ms":999986,"miner_id":"miner-7","network_difficulty":1000007,"ntime":1700001690,"order_key":"07:miner-7","p2mr_program_hex":"9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a","share_difficulty":1000,"share_id":"share-1690","share_seq":1690,"template_height":800052},{"accepted_at_ms":999976,"credit_policy":null,"job_id":"job-2948","job_issued_at_ms":999986,"miner_id":"miner-0","network_difficulty":1000007,"ntime":1700002948,"order_key":"00:miner-0","p2mr_program_hex":"8484848484848484848484848484848484848484848484848484848484848484","share_difficulty":7,"share_id":"share-2948","share_seq":2948,"template_height":800092},{"accepted_at_ms":999983,"credit_policy":null,"job_id":"job-3464","job_issued_at_ms":999972,"miner_id":"miner-10","network_difficulty":1000013,"ntime":1700003464,"order_key":"10:miner-10","p2mr_program_hex":"8888888888888888888888888888888888888888888888888888888888888888","share_difficulty":19,"share_id":"share-3464","share_seq":3464,"template_height":800108},{"accepted_at_ms":999982,"credit_policy":null,"job_id":"job-2297","job_issued_at_ms":999993,"miner_id":"miner-9","network_difficulty":1000002,"ntime":1700002297,"order_key":"09:miner-9","p2mr_program_hex":"f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9","share_difficulty":3,"share_id":"share-2297","share_seq":2297,"template_height":800071},{"accepted_at_ms":999997,"credit_policy":null,"job_id":"job-784","job_issued_at_ms":999990,"miner_id":"miner-3","network_difficulty":1000002,"ntime":1700000784,"order_key":"03:miner-3","p2mr_program_hex":"1010101010101010101010101010101010101010101010101010101010101010","share_difficulty":1,"share_id":"share-784","share_seq":784,"template_height":800024},{"accepted_at_ms":999998,"credit_policy":null,"job_id":"job-979","job_issued_at_ms":999993,"miner_id":"miner-0","network_difficulty":1000010,"ntime":1700000979,"order_key":"00:miner-0","p2mr_program_hex":"d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3","share_difficulty":19,"share_id":"share-979","share_seq":979,"template_height":800030},{"accepted_at_ms":999970,"credit_policy":null,"job_id":"job-3857","job_issued_at_ms":999997,"miner_id":"miner-7","network_difficulty":1000015,"ntime":1700003857,"order_key":"07:miner-7","p2mr_program_hex":"1111111111111111111111111111111111111111111111111111111111111111","share_difficulty":7,"share_id":"share-3857","share_seq":3857,"template_height":800120},{"accepted_at_ms":999989,"credit_policy":null,"job_id":"job-255","job_issued_at_ms":999972,"miner_id":"miner-2","network_difficulty":1000000,"ntime":1700000255,"order_key":"02:miner-2","p2mr_program_hex":"ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff","share_difficulty":101,"share_id":"share-255","share_seq":255,"template_height":800007},{"accepted_at_ms":999999,"credit_policy":null,"job_id":"job-703","job_issued_at_ms":999986,"miner_id":"miner-10","network_difficulty":1000006,"ntime":1700000703,"order_key":"10:miner-10","p2mr_program_hex":"bfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbf","share_difficulty":3,"share_id":"share-703","share_seq":703,"template_height":800021},{"accepted_at_ms":999971,"credit_policy":null,"job_id":"job-1013","job_issued_at_ms":999987,"miner_id":"miner-1","network_difficulty":1000010,"ntime":1700001013,"order_key":"01:miner-1","p2mr_program_hex":"f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5","share_difficulty":101,"share_id":"share-1013","share_seq":1013,"template_height":800031},{"accepted_at_ms":999999,"credit_policy":null,"job_id":"job-675","job_issued_at_ms":999997,"miner_id":"miner-4","network_difficulty":1000012,"ntime":1700000675,"order_key":"04:miner-4","p2mr_program_hex":"a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3","share_difficulty":101,"share_id":"share-675","share_seq":675,"template_height":800021},{"accepted_at_ms":999971,"credit_policy":null,"job_id":"job-3006","job_issued_at_ms":999989,"miner_id":"miner-3","network_difficulty":1000014,"ntime":1700003006,"order_key":"03:miner-3","p2mr_program_hex":"bebebebebebebebebebebebebebebebebebebebebebebebebebebebebebebebe","share_difficulty":7,"share_id":"share-3006","share_seq":3006,"template_height":800093},{"accepted_at_ms":999977,"credit_policy":null,"job_id":"job-2382","job_issued_at_ms":999994,"miner_id":"miner-6","network_difficulty":1000002,"ntime":1700002382,"order_key":"06:miner-6","p2mr_program_hex":"4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e","share_difficulty":1000,"share_id":"share-2382","share_seq":2382,"template_height":800074},{"accepted_at_ms":999980,"credit_policy":null,"job_id":"job-3332","job_issued_at_ms":999973,"miner_id":"miner-10","network_difficulty":1000000,"ntime":1700003332,"order_key":"10:miner-10","p2mr_program_hex":"0404040404040404040404040404040404040404040404040404040404040404","share_difficulty":1000,"share_id":"share-3332","share_seq":3332,"template_height":800104},{"accepted_at_ms":999977,"credit_policy":"stale-grace","job_id":"job-931","job_issued_at_ms":999978,"miner_id":"miner-7","network_difficulty":1000013,"ntime":1700000931,"order_key":"07:miner-7","p2mr_program_hex":"a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3","share_difficulty":101,"share_id":"share-931","share_seq":931,"template_height":800029},{"accepted_at_ms":999978,"credit_policy":"stale-grace","job_id":"job-3954","job_issued_at_ms":999988,"miner_id":"miner-5","network_difficulty":1000010,"ntime":1700003954,"order_key":"05:miner-5","p2mr_program_hex":"7272727272727272727272727272727272727272727272727272727272727272","share_difficulty":1000,"share_id":"share-3954","share_seq":3954,"template_height":800123},{"accepted_at_ms":999973,"credit_policy":null,"job_id":"job-3832","job_issued_at_ms":999972,"miner_id":"miner-4","network_difficulty":1000007,"ntime":1700003832,"order_key":"04:miner-4","p2mr_program_hex":"f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8","share_difficulty":19,"share_id":"share-3832","share_seq":3832,"template_height":800119},{"accepted_at_ms":1000467,"credit_policy":null,"job_id":"job-664","job_issued_at_ms":999975,"miner_id":"miner-4","network_difficulty":1000001,"ntime":1700000664,"order_key":"04:miner-4","p2mr_program_hex":"9898989898989898989898989898989898989898989898989898989898989898","share_difficulty":7,"share_id":"share-664","share_seq":664,"template_height":800020},{"accepted_at_ms":999988,"credit_policy":null,"job_id":"job-3732","job_issued_at_ms":999973,"miner_id":"miner-3","network_difficulty":1000009,"ntime":1700003732,"order_key":"03:miner-3","p2mr_program_hex":"9494949494949494949494949494949494949494949494949494949494949494","share_difficulty":7,"share_id":"share-3732","share_seq":3732,"template_height":800116},{"accepted_at_ms":999981,"credit_policy":"stale-grace","job_id":"job-1608","job_issued_at_ms":999975,"miner_id":"miner-2","network_difficulty":1000010,"ntime":1700001608,"order_key":"02:miner-2","p2mr_program_hex":"4848484848484848484848484848484848484848484848484848484848484848","share_difficulty":2,"share_id":"share-1608","share_seq":1608,"template_height":800050},{"accepted_at_ms":999991,"credit_policy":null,"job_id":"job-3745","job_issued_at_ms":999981,"miner_id":"miner-5","network_difficulty":1000005,"ntime":1700003745,"order_key":"05:miner-5","p2mr_program_hex":"a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1","share_difficulty":2,"share_id":"share-3745","share_seq":3745,"template_height":800117},{"accepted_at_ms":999975,"credit_policy":null,"job_id":"job-713","job_issued_at_ms":1000000,"miner_id":"miner-9","network_difficulty":1000016,"ntime":1700000713,"order_key":"09:miner-9","p2mr_program_hex":"c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9","share_difficulty":2,"share_id":"share-713","share_seq":713,"template_height":800022},{"accepted_at_ms":999986,"credit_policy":null,"job_id":"job-2462","job_issued_at_ms":999999,"miner_id":"miner-9","network_difficulty":1000014,"ntime":1700002462,"order_key":"09:miner-9","p2mr_program_hex":"9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e","share_difficulty":2,"share_id":"share-2462","share_seq":2462,"template_height":800076},{"accepted_at_ms":999989,"credit_policy":"stale-grace","job_id":"job-395","job_issued_at_ms":999985,"miner_id":"miner-10","network_difficulty":1000004,"ntime":1700000395,"order_key":"10:miner-10","p2mr_program_hex":"8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b","share_difficulty":3,"share_id":"share-395","share_seq":395,"template_height":800012},{"accepted_at_ms":999999,"credit_policy":null,"job_id":"job-646","job_issued_at_ms":999978,"miner_id":"miner-8","network_difficulty":1000000,"ntime":1700000646,"order_key":"08:miner-8","p2mr_program_hex":"8686868686868686868686868686868686868686868686868686868686868686","share_difficulty":101,"share_id":"share-646","share_seq":646,"template_height":800020},{"accepted_at_ms":999980,"credit_policy":null,"job_id":"job-3346","job_issued_at_ms":999983,"miner_id":"miner-2","network_difficulty":1000014,"ntime":1700003346,"order_key":"02:miner-2","p2mr_program_hex":"1212121212121212121212121212121212121212121212121212121212121212","share_difficulty":7,"share_id":"share-3346","share_seq":3346,"template_height":800104},{"accepted_at_ms":999991,"credit_policy":null,"job_id":"job-2341","job_issued_at_ms":999982,"miner_id":"miner-9","network_difficulty":1000012,"ntime":1700002341,"order_key":"09:miner-9","p2mr_program_hex":"2525252525252525252525252525252525252525252525252525252525252525","share_difficulty":2,"share_id":"share-2341","share_seq":2341,"template_height":800073},{"accepted_at_ms":999997,"credit_policy":null,"job_id":"job-1688","job_issued_at_ms":999996,"miner_id":"miner-5","network_difficulty":1000005,"ntime":1700001688,"order_key":"05:miner-5","p2mr_program_hex":"9898989898989898989898989898989898989898989898989898989898989898","share_difficulty":1,"share_id":"share-1688","share_seq":1688,"template_height":800052},{"accepted_at_ms":999983,"credit_policy":null,"job_id":"job-552","job_issued_at_ms":999997,"miner_id":"miner-2","network_difficulty":1000008,"ntime":1700000552,"order_key":"02:miner-2","p2mr_program_hex":"2828282828282828282828282828282828282828282828282828282828282828","share_difficulty":2,"share_id":"share-552","share_seq":552,"template_height":800017},{"accepted_at_ms":999971,"credit_policy":null,"job_id":"job-3099","job_issued_at_ms":999999,"miner_id":"miner-8","network_difficulty":1000005,"ntime":1700003099,"order_key":"08:miner-8","p2mr_program_hex":"1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b","share_difficulty":3,"share_id":"share-3099","share_seq":3099,"template_height":800096},{"accepted_at_ms":999986,"credit_policy":null,"job_id":"job-2581","job_issued_at_ms":999988,"miner_id":"miner-7","network_difficulty":1000014,"ntime":1700002581,"order_key":"07:miner-7","p2mr_program_hex":"1515151515151515151515151515151515151515151515151515151515151515","share_difficulty":3,"share_id":"share-2581","share_seq":2581,"template_height":800080},{"accepted_at_ms":999994,"credit_policy":null,"job_id":"job-3870","job_issued_at_ms":999992,"miner_id":"miner-9","network_difficulty":1000011,"ntime":1700003870,"order_key":"09:miner-9","p2mr_program_hex":"1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e","share_difficulty":19,"share_id":"share-3870","share_seq":3870,"template_height":800120},{"accepted_at_ms":999972,"credit_policy":null,"job_id":"job-1008","job_issued_at_ms":999998,"miner_id":"miner-7","network_difficulty":1000005,"ntime":1700001008,"order_key":"07:miner-7","p2mr_program_hex":"f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0","share_difficulty":3,"share_id":"share-1008","share_seq":1008,"template_height":800031},{"accepted_at_ms":999999,"credit_policy":null,"job_id":"job-2916","job_issued_at_ms":999979,"miner_id":"miner-1","network_difficulty":1000009,"ntime":1700002916,"order_key":"01:miner-1","p2mr_program_hex":"6464646464646464646464646464646464646464646464646464646464646464","share_difficulty":101,"share_id":"share-2916","share_seq":2916,"template_height":800091},{"accepted_at_ms":999983,"credit_policy":null,"job_id":"job-911","job_issued_at_ms":999987,"miner_id":"miner-9","network_difficulty":1000010,"ntime":1700000911,"order_key":"09:miner-9","p2mr_program_hex":"8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f","share_difficulty":2,"share_id":"share-911","share_seq":911,"template_height":800028},{"accepted_at_ms":999983,"credit_policy":null,"job_id":"job-3345","job_issued_at_ms":999975,"miner_id":"miner-1","network_difficulty":1000013,"ntime":1700003345,"order_key":"01:miner-1","p2mr_program_hex":"1111111111111111111111111111111111111111111111111111111111111111","share_difficulty":19,"share_id":"share-3345","share_seq":3345,"template_height":800104},{"accepted_at_ms":1000495,"credit_policy":null,"job_id":"job-2210","job_issued_at_ms":999999,"miner_id":"miner-10","network_difficulty":1000000,"ntime":1700002210,"order_key":"10:miner-10","p2mr_program_hex":"a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2","share_difficulty":101,"share_id":"share-2210","share_seq":2210,"template_height":800069},{"accepted_at_ms":1000091,"credit_policy":null,"job_id":"job-1834","job_issued_at_ms":1000132,"miner_id":"miner-8","network_difficulty":1000015,"ntime":1700001834,"order_key":"08:miner-8","p2mr_program_hex":"2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a","share_difficulty":3,"share_id":"share-1834","share_seq":1834,"template_height":800057},{"accepted_at_ms":999988,"credit_policy":null,"job_id":"job-3787","job_issued_at_ms":999993,"miner_id":"miner-3","network_difficulty":1000013,"ntime":1700003787,"order_key":"03:miner-3","p2mr_program_hex":"cbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcb","share_difficulty":101,"share_id":"share-3787","share_seq":3787,"template_height":800118},{"accepted_at_ms":999976,"credit_policy":null,"job_id":"job-2282","job_issued_at_ms":999985,"miner_id":"miner-5","network_difficulty":1000004,"ntime":1700002282,"order_key":"05:miner-5","p2mr_program_hex":"eaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaea","share_difficulty":101,"share_id":"share-2282","share_seq":2282,"template_height":800071},{"accepted_at_ms":999977,"credit_policy":null,"job_id":"job-2303","job_issued_at_ms":999988,"miner_id":"miner-4","network_difficulty":1000008,"ntime":1700002303,"order_key":"04:miner-4","p2mr_program_hex":"ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff","share_difficulty":3,"share_id":"share-2303","share_seq":2303,"template_height":800071},{"accepted_at_ms":999971,"credit_policy":null,"job_id":"job-3586","job_issued_at_ms":999991,"miner_id":"miner-0","network_difficulty":1000016,"ntime":1700003586,"order_key":"00:miner-0","p2mr_program_hex":"0202020202020202020202020202020202020202020202020202020202020202","share_difficulty":101,"share_id":"share-3586","share_seq":3586,"template_height":800112},{"accepted_at_ms":999992,"credit_policy":null,"job_id":"job-3903","job_issued_at_ms":999990,"miner_id":"miner-9","network_difficulty":1000010,"ntime":1700003903,"order_key":"09:miner-9","p2mr_program_hex":"3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f","share_difficulty":2,"share_id":"share-3903","share_seq":3903,"template_height":800121},{"accepted_at_ms":999983,"credit_policy":null,"job_id":"job-2865","job_issued_at_ms":999973,"miner_id":"miner-5","network_difficulty":1000009,"ntime":1700002865,"order_key":"05:miner-5","p2mr_program_hex":"3131313131313131313131313131313131313131313131313131313131313131","share_difficulty":2,"share_id":"share-2865","share_seq":2865,"template_height":800089},{"accepted_at_ms":999989,"credit_policy":null,"job_id":"job-3792","job_issued_at_ms":999986,"miner_id":"miner-8","network_difficulty":1000001,"ntime":1700003792,"order_key":"08:miner-8","p2mr_program_hex":"d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0","share_difficulty":1000,"share_id":"share-3792","share_seq":3792,"template_height":800118},{"accepted_at_ms":999999,"credit_policy":"stale-grace","job_id":"job-781","job_issued_at_ms":999974,"miner_id":"miner-0","network_difficulty":1000016,"ntime":1700000781,"order_key":"00:miner-0","p2mr_program_hex":"0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d","share_difficulty":2,"share_id":"share-781","share_seq":781,"template_height":800024},{"accepted_at_ms":999992,"credit_policy":null,"job_id":"job-3795","job_issued_at_ms":999980,"miner_id":"miner-0","network_difficulty":1000004,"ntime":1700003795,"order_key":"00:miner-0","p2mr_program_hex":"d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3","share_difficulty":101,"share_id":"share-3795","share_seq":3795,"template_height":800118},{"accepted_at_ms":999986,"credit_policy":null,"job_id":"job-3269","job_issued_at_ms":999993,"miner_id":"miner-2","network_difficulty":1000005,"ntime":1700003269,"order_key":"02:miner-2","p2mr_program_hex":"c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5","share_difficulty":2,"share_id":"share-3269","share_seq":3269,"template_height":800102},{"accepted_at_ms":999977,"credit_policy":null,"job_id":"job-598","job_issued_at_ms":1000312,"miner_id":"miner-4","network_difficulty":1000003,"ntime":1700000598,"order_key":"04:miner-4","p2mr_program_hex":"5656565656565656565656565656565656565656565656565656565656565656","share_difficulty":19,"share_id":"share-598","share_seq":598,"template_height":800018},{"accepted_at_ms":999992,"credit_policy":null,"job_id":"job-1660","job_issued_at_ms":999987,"miner_id":"miner-10","network_difficulty":1000011,"ntime":1700001660,"order_key":"10:miner-10","p2mr_program_hex":"7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c","share_difficulty":19,"share_id":"share-1660","share_seq":1660,"template_height":800051},{"accepted_at_ms":999978,"credit_policy":"stale-grace","job_id":"job-2639","job_issued_at_ms":999986,"miner_id":"miner-10","network_difficulty":1000004,"ntime":1700002639,"order_key":"10:miner-10","p2mr_program_hex":"4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f","share_difficulty":101,"share_id":"share-2639","share_seq":2639,"template_height":800082},{"accepted_at_ms":999997,"credit_policy":null,"job_id":"job-2925","job_issued_at_ms":999981,"miner_id":"miner-10","network_difficulty":1000001,"ntime":1700002925,"order_key":"10:miner-10","p2mr_program_hex":"6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d","share_difficulty":101,"share_id":"share-2925","share_seq":2925,"template_height":800091},{"accepted_at_ms":999975,"credit_policy":null,"job_id":"job-72","job_issued_at_ms":999987,"miner_id":"miner-6","network_difficulty":1000004,"ntime":1700000072,"order_key":"06:miner-6","p2mr_program_hex":"4848484848484848484848484848484848484848484848484848484848484848","share_difficulty":2,"share_id":"share-72","share_seq":72,"template_height":800002},{"accepted_at_ms":999999,"credit_policy":null,"job_id":"job-3315","job_issued_at_ms":1000496,"miner_id":"miner-4","network_difficulty":1000000,"ntime":1700003315,"order_key":"04:miner-4","p2mr_program_hex":"f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3","share_difficulty":7,"share_id":"share-3315","share_seq":3315,"template_height":800103},{"accepted_at_ms":999977,"credit_policy":null,"job_id":"job-2797","job_issued_at_ms":999975,"miner_id":"miner-3","network_difficulty":1000009,"ntime":1700002797,"order_key":"03:miner-3","p2mr_program_hex":"edededededededededededededededededededededededededededededededed","share_difficulty":1000,"share_id":"share-2797","share_seq":2797,"template_height":800087},{"accepted_at_ms":999976,"credit_policy":null,"job_id":"job-1438","job_issued_at_ms":999970,"miner_id":"miner-8","network_difficulty":1000010,"ntime":1700001438,"order_key":"08:miner-8","p2mr_program_hex":"9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e","share_difficulty":2,"share_id":"share-1438","share_seq":1438,"template_height":800044},{"accepted_at_ms":999981,"credit_policy":null,"job_id":"job-3264","job_issued_at_ms":1000000,"miner_id":"miner-8","network_difficulty":1000000,"ntime":1700003264,"order_key":"08:miner-8","p2mr_program_hex":"c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0","share_difficulty":2,"share_id":"share-3264","share_seq":3264,"template_height":800102},{"accepted_at_ms":999991,"credit_policy":null,"job_id":"job-2437","job_issued_at_ms":999976,"miner_id":"miner-6","network_difficulty":1000006,"ntime":1700002437,"order_key":"06:miner-6","p2mr_program_hex":"8585858585858585858585858585858585858585858585858585858585858585","share_difficulty":1000,"share_id":"share-2437","share_seq":2437,"template_height":800076},{"accepted_at_ms":999998,"credit_policy":null,"job_id":"job-3116","job_issued_at_ms":999997,"miner_id":"miner-3","network_difficulty":1000005,"ntime":1700003116,"order_key":"03:miner-3","p2mr_program_hex":"2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c","share_difficulty":101,"share_id":"share-3116","share_seq":3116,"template_height":800097},{"accepted_at_ms":999986,"credit_policy":"stale-grace","job_id":"job-2822","job_issued_at_ms":999981,"miner_id":"miner-6","network_difficulty":1000000,"ntime":1700002822,"order_key":"06:miner-6","p2mr_program_hex":"0606060606060606060606060606060606060606060606060606060606060606","share_difficulty":1,"share_id":"share-2822","share_seq":2822,"template_height":800088},{"accepted_at_ms":999997,"credit_policy":"stale-grace","job_id":"job-685","job_issued_at_ms":1000354,"miner_id":"miner-3","network_difficulty":1000005,"ntime":1700000685,"order_key":"03:miner-3","p2mr_program_hex":"adadadadadadadadadadadadadadadadadadadadadadadadadadadadadadadad","share_difficulty":1,"share_id":"share-685","share_seq":685,"template_height":800021},{"accepted_at_ms":1000000,"credit_policy":null,"job_id":"job-3370","job_issued_at_ms":999984,"miner_id":"miner-4","network_difficulty":1000004,"ntime":1700003370,"order_key":"04:miner-4","p2mr_program_hex":"2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a","share_difficulty":1000,"share_id":"share-3370","share_seq":3370,"template_height":800105},{"accepted_at_ms":999995,"credit_policy":null,"job_id":"job-633","job_issued_at_ms":999997,"miner_id":"miner-6","network_difficulty":1000004,"ntime":1700000633,"order_key":"06:miner-6","p2mr_program_hex":"7979797979797979797979797979797979797979797979797979797979797979","share_difficulty":3,"share_id":"share-633","share_seq":633,"template_height":800019},{"accepted_at_ms":999993,"credit_policy":null,"job_id":"job-3517","job_issued_at_ms":999997,"miner_id":"miner-8","network_difficulty":1000015,"ntime":1700003517,"order_key":"08:miner-8","p2mr_program_hex":"bdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbd","share_difficulty":101,"share_id":"share-3517","share_seq":3517,"template_height":800109},{"accepted_at_ms":999992,"credit_policy":"stale-grace","job_id":"job-200","job_issued_at_ms":999996,"miner_id":"miner-2","network_difficulty":1000013,"ntime":1700000200,"order_key":"02:miner-2","p2mr_program_hex":"c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8","share_difficulty":1,"share_id":"share-200","share_seq":200,"template_height":800006},{"accepted_at_ms":999987,"credit_policy":"stale-grace","job_id":"job-1488","job_issued_at_ms":999974,"miner_id":"miner-3","network_difficulty":1000009,"ntime":1700001488,"order_key":"03:miner-3","p2mr_program_hex":"d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0","share_difficulty":1000,"share_id":"share-1488","share_seq":1488,"template_height":800046},{"accepted_at_ms":999986,"credit_policy":null,"job_id":"job-2148","job_issued_at_ms":999987,"miner_id":"miner-3","network_difficulty":1000006,"ntime":1700002148,"order_key":"03:miner-3","p2mr_program_hex":"6464646464646464646464646464646464646464646464646464646464646464","share_difficulty":2,"share_id":"share-2148","share_seq":2148,"template_height":800067},{"accepted_at_ms":999973,"credit_policy":null,"job_id":"job-2374","job_issued_at_ms":999988,"miner_id":"miner-9","network_difficulty":1000011,"ntime":1700002374,"order_key":"09:miner-9","p2mr_program_hex":"4646464646464646464646464646464646464646464646464646464646464646","share_difficulty":3,"share_id":"share-2374","share_seq":2374,"template_height":800074},{"accepted_at_ms":999977,"credit_policy":null,"job_id":"job-357","job_issued_at_ms":999972,"miner_id":"miner-5","network_difficulty":1000000,"ntime":1700000357,"order_key":"05:miner-5","p2mr_program_hex":"6565656565656565656565656565656565656565656565656565656565656565","share_difficulty":101,"share_id":"share-357","share_seq":357,"template_height":800011},{"accepted_at_ms":999975,"credit_policy":null,"job_id":"job-2869","job_issued_at_ms":999984,"miner_id":"miner-9","network_difficulty":1000013,"ntime":1700002869,"order_key":"09:miner-9","p2mr_program_hex":"3535353535353535353535353535353535353535353535353535353535353535","share_difficulty":2,"share_id":"share-2869","share_seq":2869,"template_height":800089},{"accepted_at_ms":999976,"credit_policy":null,"job_id":"job-459","job_issued_at_ms":999972,"miner_id":"miner-8","network_difficulty":1000000,"ntime":1700000459,"order_key":"08:miner-8","p2mr_program_hex":"cbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcb","share_difficulty":7,"share_id":"share-459","share_seq":459,"template_height":800014},{"accepted_at_ms":999977,"credit_policy":null,"job_id":"job-1189","job_issued_at_ms":999977,"miner_id":"miner-1","network_difficulty":1000016,"ntime":1700001189,"order_key":"01:miner-1","p2mr_program_hex":"a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5","share_difficulty":2,"share_id":"share-1189","share_seq":1189,"template_height":800037},{"accepted_at_ms":999975,"credit_policy":null,"job_id":"job-2060","job_issued_at_ms":999993,"miner_id":"miner-3","network_difficulty":1000003,"ntime":1700002060,"order_key":"03:miner-3","p2mr_program_hex":"0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c","share_difficulty":101,"share_id":"share-2060","share_seq":2060,"template_height":800064},{"accepted_at_ms":999973,"credit_policy":null,"job_id":"job-3647","job_issued_at_ms":999976,"miner_id":"miner-6","network_difficulty":1000009,"ntime":1700003647,"order_key":"06:miner-6","p2mr_program_hex":"3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f","share_difficulty":101,"share_id":"share-3647","share_seq":3647,"template_height":800113},{"accepted_at_ms":999987,"credit_policy":null,"job_id":"job-1393","job_issued_at_ms":999978,"miner_id":"miner-7","network_difficulty":1000016,"ntime":1700001393,"order_key":"07:miner-7","p2mr_program_hex":"7171717171717171717171717171717171717171717171717171717171717171","share_difficulty":3,"share_id":"share-1393","share_seq":1393,"template_height":800043},{"accepted_at_ms":999970,"credit_policy":null,"job_id":"job-378","job_issued_at_ms":999997,"miner_id":"miner-4","network_difficulty":1000004,"ntime":1700000378,"order_key":"04:miner-4","p2mr_program_hex":"7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a","share_difficulty":2,"share_id":"share-378","share_seq":378,"template_height":800011},{"accepted_at_ms":999989,"credit_policy":"stale-grace","job_id":"job-2330","job_issued_at_ms":999996,"miner_id":"miner-9","network_difficulty":1000001,"ntime":1700002330,"order_key":"09:miner-9","p2mr_program_hex":"1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a","share_difficulty":1,"share_id":"share-2330","share_seq":2330,"template_height":800072},{"accepted_at_ms":999985,"credit_policy":null,"job_id":"job-1413","job_issued_at_ms":999987,"miner_id":"miner-5","network_difficulty":1000002,"ntime":1700001413,"order_key":"05:miner-5","p2mr_program_hex":"8585858585858585858585858585858585858585858585858585858585858585","share_difficulty":101,"share_id":"share-1413","share_seq":1413,"template_height":800044},{"accepted_at_ms":999970,"credit_policy":null,"job_id":"job-1886","job_issued_at_ms":999975,"miner_id":"miner-5","network_difficulty":1000016,"ntime":1700001886,"order_key":"05:miner-5","p2mr_program_hex":"5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e","share_difficulty":1,"share_id":"share-1886","share_seq":1886,"template_height":800058},{"accepted_at_ms":999994,"credit_policy":null,"job_id":"job-3753","job_issued_at_ms":999991,"miner_id":"miner-2","network_difficulty":1000013,"ntime":1700003753,"order_key":"02:miner-2","p2mr_program_hex":"a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9","share_difficulty":1,"share_id":"share-3753","share_seq":3753,"template_height":800117},{"accepted_at_ms":999987,"credit_policy":null,"job_id":"job-2459","job_issued_at_ms":999993,"miner_id":"miner-6","network_difficulty":1000011,"ntime":1700002459,"order_key":"06:miner-6","p2mr_program_hex":"9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b","share_difficulty":1,"share_id":"share-2459","share_seq":2459,"template_height":800076},{"accepted_at_ms":999999,"credit_policy":null,"job_id":"job-3805","job_issued_at_ms":999989,"miner_id":"miner-10","network_difficulty":1000014,"ntime":1700003805,"order_key":"10:miner-10","p2mr_program_hex":"dddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddd","share_difficulty":3,"share_id":"share-3805","share_seq":3805,"template_height":800118},{"accepted_at_ms":999987,"credit_policy":"stale-grace","job_id":"job-3590","job_issued_at_ms":999971,"miner_id":"miner-4","network_difficulty":1000003,"ntime":1700003590,"order_key":"04:miner-4","p2mr_program_hex":"0606060606060606060606060606060606060606060606060606060606060606","share_difficulty":3,"share_id":"share-3590","share_seq":3590,"template_height":800112},{"accepted_at_ms":999986,"credit_policy":null,"job_id":"job-891","job_issued_at_ms":999999,"miner_id":"miner-0","network_difficulty":1000007,"ntime":1700000891,"order_key":"00:miner-0","p2mr_program_hex":"7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b","share_difficulty":1000,"share_id":"share-891","share_seq":891,"template_height":800027},{"accepted_at_ms":999999,"credit_policy":null,"job_id":"job-160","job_issued_at_ms":999996,"miner_id":"miner-6","network_difficulty":1000007,"ntime":1700000160,"order_key":"06:miner-6","p2mr_program_hex":"a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0","share_difficulty":3,"share_id":"share-160","share_seq":160,"template_height":800005},{"accepted_at_ms":999987,"credit_policy":null,"job_id":"job-88","job_issued_at_ms":999996,"miner_id":"miner-0","network_difficulty":1000003,"ntime":1700000088,"order_key":"00:miner-0","p2mr_program_hex":"5858585858585858585858585858585858585858585858585858585858585858","share_difficulty":1000,"share_id":"share-88","share_seq":88,"template_height":800002},{"accepted_at_ms":999985,"credit_policy":null,"job_id":"job-867","job_issued_at_ms":999970,"miner_id":"miner-9","network_difficulty":1000000,"ntime":1700000867,"order_key":"09:miner-9","p2mr_program_hex":"6363636363636363636363636363636363636363636363636363636363636363","share_difficulty":19,"share_id":"share-867","share_seq":867,"template_height":800027},{"accepted_at_ms":999978,"credit_policy":null,"job_id":"job-3090","job_issued_at_ms":999985,"miner_id":"miner-10","network_difficulty":1000013,"ntime":1700003090,"order_key":"10:miner-10","p2mr_program_hex":"1212121212121212121212121212121212121212121212121212121212121212","share_difficulty":1,"share_id":"share-3090","share_seq":3090,"template_height":800096},{"accepted_at_ms":999990,"credit_policy":null,"job_id":"job-2764","job_issued_at_ms":999974,"miner_id":"miner-3","network_difficulty":1000010,"ntime":1700002764,"order_key":"03:miner-3","p2mr_program_hex":"cccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccc","share_difficulty":3,"share_id":"share-2764","share_seq":2764,"template_height":800086},{"accepted_at_ms":999982,"credit_policy":"stale-grace","job_id":"job-1613","job_issued_at_ms":999997,"miner_id":"miner-7","network_difficulty":1000015,"ntime":1700001613,"order_key":"07:miner-7","p2mr_program_hex":"4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d","share_difficulty":1,"share_id":"share-1613","share_seq":1613,"template_height":800050},{"accepted_at_ms":999978,"credit_policy":null,"job_id":"job-668","job_issued_at_ms":999979,"miner_id":"miner-8","network_difficulty":1000005,"ntime":1700000668,"order_key":"08:miner-8","p2mr_program_hex":"9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c","share_difficulty":1000,"share_id":"share-668","share_seq":668,"template_height":800020},{"accepted_at_ms":999988,"credit_policy":"stale-grace","job_id":"job-3510","job_issued_at_ms":999979,"miner_id":"miner-1","network_difficulty":1000008,"ntime":1700003510,"order_key":"01:miner-1","p2mr_program_hex":"b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6","share_difficulty":2,"share_id":"share-3510","share_seq":3510,"template_height":800109},{"accepted_at_ms":999970,"credit_policy":null,"job_id":"job-1677","job_issued_at_ms":999977,"miner_id":"miner-5","network_difficulty":1000011,"ntime":1700001677,"order_key":"05:miner-5","p2mr_program_hex":"8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d","share_difficulty":2,"share_id":"share-1677","share_seq":1677,"template_height":800052},{"accepted_at_ms":999987,"credit_policy":null,"job_id":"job-830","job_issued_at_ms":999973,"miner_id":"miner-5","network_difficulty":1000014,"ntime":1700000830,"order_key":"05:miner-5","p2mr_program_hex":"3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e","share_difficulty":7,"share_id":"share-830","share_seq":830,"template_height":800025},{"accepted_at_ms":999994,"credit_policy":null,"job_id":"job-3347","job_issued_at_ms":999974,"miner_id":"miner-3","network_difficulty":1000015,"ntime":1700003347,"order_key":"03:miner-3","p2mr_program_hex":"1313131313131313131313131313131313131313131313131313131313131313","share_difficulty":19,"share_id":"share-3347","share_seq":3347,"template_height":800104},{"accepted_at_ms":999986,"credit_policy":"stale-grace","job_id":"job-2004","job_issued_at_ms":999987,"miner_id":"miner-2","network_difficulty":1000015,"ntime":1700002004,"order_key":"02:miner-2","p2mr_program_hex":"d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4","share_difficulty":1000,"share_id":"share-2004","share_seq":2004,"template_height":800062},{"accepted_at_ms":999972,"credit_policy":null,"job_id":"job-3396","job_issued_at_ms":999974,"miner_id":"miner-8","network_difficulty":1000013,"ntime":1700003396,"order_key":"08:miner-8","p2mr_program_hex":"4444444444444444444444444444444444444444444444444444444444444444","share_difficulty":1,"share_id":"share-3396","share_seq":3396,"template_height":800106},{"accepted_at_ms":999984,"credit_policy":"stale-grace","job_id":"job-1516","job_issued_at_ms":999989,"miner_id":"miner-9","network_difficulty":1000003,"ntime":1700001516,"order_key":"09:miner-9","p2mr_program_hex":"ecececececececececececececececececececececececececececececececec","share_difficulty":1000,"share_id":"share-1516","share_seq":1516,"template_height":800047},{"accepted_at_ms":999988,"credit_policy":null,"job_id":"job-3971","job_issued_at_ms":999974,"miner_id":"miner-0","network_difficulty":1000010,"ntime":1700003971,"order_key":"00:miner-0","p2mr_program_hex":"8383838383838383838383838383838383838383838383838383838383838383","share_difficulty":3,"share_id":"share-3971","share_seq":3971,"template_height":800124},{"accepted_at_ms":999986,"credit_policy":null,"job_id":"job-2830","job_issued_at_ms":999976,"miner_id":"miner-3","network_difficulty":1000008,"ntime":1700002830,"order_key":"03:miner-3","p2mr_program_hex":"0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e","share_difficulty":7,"share_id":"share-2830","share_seq":2830,"template_height":800088},{"accepted_at_ms":999974,"credit_policy":"stale-grace","job_id":"job-468","job_issued_at_ms":999994,"miner_id":"miner-6","network_difficulty":1000009,"ntime":1700000468,"order_key":"06:miner-6","p2mr_program_hex":"d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4","share_difficulty":101,"share_id":"share-468","share_seq":468,"template_height":800014},{"accepted_at_ms":999980,"credit_policy":null,"job_id":"job-3041","job_issued_at_ms":999982,"miner_id":"miner-5","network_difficulty":1000015,"ntime":1700003041,"order_key":"05:miner-5","p2mr_program_hex":"e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1","share_difficulty":19,"share_id":"share-3041","share_seq":3041,"template_height":800095},{"accepted_at_ms":999997,"credit_policy":"stale-grace","job_id":"job-1896","job_issued_at_ms":999983,"miner_id":"miner-4","network_difficulty":1000009,"ntime":1700001896,"order_key":"04:miner-4","p2mr_program_hex":"6868686868686868686868686868686868686868686868686868686868686868","share_difficulty":1,"share_id":"share-1896","share_seq":1896,"template_height":800059},{"accepted_at_ms":999980,"credit_policy":null,"job_id":"job-3274","job_issued_at_ms":1000201,"miner_id":"miner-7","network_difficulty":1000010,"ntime":1700003274,"order_key":"07:miner-7","p2mr_program_hex":"cacacacacacacacacacacacacacacacacacacacacacacacacacacacacacacaca","share_difficulty":2,"share_id":"share-3274","share_seq":3274,"template_height":800102},{"accepted_at_ms":1000000,"credit_policy":"stale-grace","job_id":"job-1539","job_issued_at_ms":999976,"miner_id":"miner-10","network_difficulty":1000009,"ntime":1700001539,"order_key":"10:miner-10","p2mr_program_hex":"0303030303030303030303030303030303030303030303030303030303030303","share_difficulty":2,"share_id":"share-1539","share_seq":1539,"template_height":800048},{"accepted_at_ms":999990,"credit_policy":null,"job_id":"job-3281","job_issued_at_ms":1000330,"miner_id":"miner-3","network_difficulty":1000000,"ntime":1700003281,"order_key":"03:miner-3","p2mr_program_hex":"d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1","share_difficulty":19,"share_id":"share-3281","share_seq":3281,"template_height":800102},{"accepted_at_ms":999993,"credit_policy":null,"job_id":"job-640","job_issued_at_ms":999991,"miner_id":"miner-2","network_difficulty":1000011,"ntime":1700000640,"order_key":"02:miner-2","p2mr_program_hex":"8080808080808080808080808080808080808080808080808080808080808080","share_difficulty":3,"share_id":"share-640","share_seq":640,"template_height":800020},{"accepted_at_ms":1000000,"credit_policy":null,"job_id":"job-245","job_issued_at_ms":999991,"miner_id":"miner-3","network_difficulty":1000007,"ntime":1700000245,"order_key":"03:miner-3","p2mr_program_hex":"f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5","share_difficulty":19,"share_id":"share-245","share_seq":245,"template_height":800007},{"accepted_at_ms":999985,"credit_policy":null,"job_id":"job-2098","job_issued_at_ms":999992,"miner_id":"miner-8","network_difficulty":1000007,"ntime":1700002098,"order_key":"08:miner-8","p2mr_program_hex":"3232323232323232323232323232323232323232323232323232323232323232","share_difficulty":1000,"share_id":"share-2098","share_seq":2098,"template_height":800065},{"accepted_at_ms":999998,"credit_policy":null,"job_id":"job-1867","job_issued_at_ms":999999,"miner_id":"miner-8","network_difficulty":1000014,"ntime":1700001867,"order_key":"08:miner-8","p2mr_program_hex":"4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b","share_difficulty":3,"share_id":"share-1867","share_seq":1867,"template_height":800058},{"accepted_at_ms":999979,"credit_policy":null,"job_id":"job-1351","job_issued_at_ms":999983,"miner_id":"miner-9","network_difficulty":1000008,"ntime":1700001351,"order_key":"09:miner-9","p2mr_program_hex":"4747474747474747474747474747474747474747474747474747474747474747","share_difficulty":7,"share_id":"share-1351","share_seq":1351,"template_height":800042},{"accepted_at_ms":999972,"credit_policy":"stale-grace","job_id":"job-1358","job_issued_at_ms":999981,"miner_id":"miner-5","network_difficulty":1000015,"ntime":1700001358,"order_key":"05:miner-5","p2mr_program_hex":"4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e","share_difficulty":101,"share_id":"share-1358","share_seq":1358,"template_height":800042},{"accepted_at_ms":999998,"credit_policy":null,"job_id":"job-2712","job_issued_at_ms":999978,"miner_id":"miner-6","network_difficulty":1000009,"ntime":1700002712,"order_key":"06:miner-6","p2mr_program_hex":"9898989898989898989898989898989898989898989898989898989898989898","share_difficulty":3,"share_id":"share-2712","share_seq":2712,"template_height":800084},{"accepted_at_ms":999988,"credit_policy":"stale-grace","job_id":"job-785","job_issued_at_ms":999995,"miner_id":"miner-4","network_difficulty":1000003,"ntime":1700000785,"order_key":"04:miner-4","p2mr_program_hex":"1111111111111111111111111111111111111111111111111111111111111111","share_difficulty":19,"share_id":"share-785","share_seq":785,"template_height":800024},{"accepted_at_ms":999996,"credit_policy":null,"job_id":"job-116","job_issued_at_ms":999979,"miner_id":"miner-6","network_difficulty":1000014,"ntime":1700000116,"order_key":"06:miner-6","p2mr_program_hex":"7474747474747474747474747474747474747474747474747474747474747474","share_difficulty":7,"share_id":"share-116","share_seq":116,"template_height":800003},{"accepted_at_ms":999995,"credit_policy":null,"job_id":"job-1805","job_issued_at_ms":999993,"miner_id":"miner-1","network_difficulty":1000003,"ntime":1700001805,"order_key":"01:miner-1","p2mr_program_hex":"0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d","share_difficulty":3,"share_id":"share-1805","share_seq":1805,"template_height":800056},{"accepted_at_ms":999978,"credit_policy":null,"job_id":"job-3589","job_issued_at_ms":999972,"miner_id":"miner-3","network_difficulty":1000002,"ntime":1700003589,"order_key":"03:miner-3","p2mr_program_hex":"0505050505050505050505050505050505050505050505050505050505050505","share_difficulty":1000,"share_id":"share-3589","share_seq":3589,"template_height":800112},{"accepted_at_ms":999991,"credit_policy":null,"job_id":"job-425","job_issued_at_ms":999994,"miner_id":"miner-7","network_difficulty":1000000,"ntime":1700000425,"order_key":"07:miner-7","p2mr_program_hex":"a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9","share_difficulty":19,"share_id":"share-425","share_seq":425,"template_height":800013},{"accepted_at_ms":999982,"credit_policy":null,"job_id":"job-418","job_issued_at_ms":999977,"miner_id":"miner-0","network_difficulty":1000010,"ntime":1700000418,"order_key":"00:miner-0","p2mr_program_hex":"a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2","share_difficulty":1,"share_id":"share-418","share_seq":418,"template_height":800013},{"accepted_at_ms":999979,"credit_policy":null,"job_id":"job-210","job_issued_at_ms":999985,"miner_id":"miner-1","network_difficulty":1000006,"ntime":1700000210,"order_key":"01:miner-1","p2mr_program_hex":"d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2","share_difficulty":7,"share_id":"share-210","share_seq":210,"template_height":800006},{"accepted_at_ms":999996,"credit_policy":null,"job_id":"job-3168","job_issued_at_ms":999990,"miner_id":"miner-0","network_difficulty":1000006,"ntime":1700003168,"order_key":"00:miner-0","p2mr_program_hex":"6060606060606060606060606060606060606060606060606060606060606060","share_difficulty":2,"share_id":"share-3168","share_seq":3168,"template_height":800099},{"accepted_at_ms":999980,"credit_policy":"stale-grace","job_id":"job-1138","job_issued_at_ms":999997,"miner_id":"miner-5","network_difficulty":1000016,"ntime":1700001138,"order_key":"05:miner-5","p2mr_program_hex":"7272727272727272727272727272727272727272727272727272727272727272","share_difficulty":7,"share_id":"share-1138","share_seq":1138,"template_height":800035},{"accepted_at_ms":999972,"credit_policy":null,"job_id":"job-3696","job_issued_at_ms":999985,"miner_id":"miner-0","network_difficulty":1000007,"ntime":1700003696,"order_key":"00:miner-0","p2mr_program_hex":"7070707070707070707070707070707070707070707070707070707070707070","share_difficulty":7,"share_id":"share-3696","share_seq":3696,"template_height":800115},{"accepted_at_ms":999991,"credit_policy":null,"job_id":"job-1169","job_issued_at_ms":999972,"miner_id":"miner-3","network_difficulty":1000013,"ntime":1700001169,"order_key":"03:miner-3","p2mr_program_hex":"9191919191919191919191919191919191919191919191919191919191919191","share_difficulty":7,"share_id":"share-1169","share_seq":1169,"template_height":800036},{"accepted_at_ms":999988,"credit_policy":null,"job_id":"job-2862","job_issued_at_ms":999998,"miner_id":"miner-2","network_difficulty":1000006,"ntime":1700002862,"order_key":"02:miner-2","p2mr_program_hex":"2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e","share_difficulty":3,"share_id":"share-2862","share_seq":2862,"template_height":800089},{"accepted_at_ms":999997,"credit_policy":"stale-grace","job_id":"job-29","job_issued_at_ms":999982,"miner_id":"miner-7","network_difficulty":1000012,"ntime":1700000029,"order_key":"07:miner-7","p2mr_program_hex":"1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d","share_difficulty":1000,"share_id":"share-29","share_seq":29,"template_height":800000},{"accepted_at_ms":999987,"credit_policy":"stale-grace","job_id":"job-1146","job_issued_at_ms":999988,"miner_id":"miner-2","network_difficulty":1000007,"ntime":1700001146,"order_key":"02:miner-2","p2mr_program_hex":"7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a","share_difficulty":1000,"share_id":"share-1146","share_seq":1146,"template_height":800035},{"accepted_at_ms":999985,"credit_policy":null,"job_id":"job-2079","job_issued_at_ms":999972,"miner_id":"miner-0","network_difficulty":1000005,"ntime":1700002079,"order_key":"00:miner-0","p2mr_program_hex":"1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f","share_difficulty":19,"share_id":"share-2079","share_seq":2079,"template_height":800064},{"accepted_at_ms":999971,"credit_policy":"stale-grace","job_id":"job-1983","job_issued_at_ms":999986,"miner_id":"miner-3","network_difficulty":1000011,"ntime":1700001983,"order_key":"03:miner-3","p2mr_program_hex":"bfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbf","share_difficulty":2,"share_id":"share-1983","share_seq":1983,"template_height":800061},{"accepted_at_ms":999973,"credit_policy":"stale-grace","job_id":"job-2399","job_issued_at_ms":999990,"miner_id":"miner-1","network_difficulty":1000002,"ntime":1700002399,"order_key":"01:miner-1","p2mr_program_hex":"5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f","share_difficulty":3,"share_id":"share-2399","share_seq":2399,"template_height":800074},{"accepted_at_ms":999982,"credit_policy":"stale-grace","job_id":"job-2696","job_issued_at_ms":999995,"miner_id":"miner-1","network_difficulty":1000010,"ntime":1700002696,"order_key":"01:miner-1","p2mr_program_hex":"8888888888888888888888888888888888888888888888888888888888888888","share_difficulty":7,"share_id":"share-2696","share_seq":2696,"template_height":800084},{"accepted_at_ms":999988,"credit_policy":"stale-grace","job_id":"job-1087","job_issued_at_ms":999995,"miner_id":"miner-9","network_difficulty":1000016,"ntime":1700001087,"order_key":"09:miner-9","p2mr_program_hex":"3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f","share_difficulty":7,"share_id":"share-1087","share_seq":1087,"template_height":800033},{"accepted_at_ms":999982,"credit_policy":null,"job_id":"job-141","job_issued_at_ms":999980,"miner_id":"miner-9","network_difficulty":1000005,"ntime":1700000141,"order_key":"09:miner-9","p2mr_program_hex":"8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d","share_difficulty":2,"share_id":"share-141","share_seq":141,"template_height":800004},{"accepted_at_ms":999998,"credit_policy":null,"job_id":"job-3936","job_issued_at_ms":999994,"miner_id":"miner-9","network_difficulty":1000009,"ntime":1700003936,"order_key":"09:miner-9","p2mr_program_hex":"6060606060606060606060606060606060606060606060606060606060606060","share_difficulty":1000,"share_id":"share-3936","share_seq":3936,"template_height":800123},{"accepted_at_ms":999970,"credit_policy":null,"job_id":"job-644","job_issued_at_ms":999972,"miner_id":"miner-6","network_difficulty":1000015,"ntime":1700000644,"order_key":"06:miner-6","p2mr_program_hex":"8484848484848484848484848484848484848484848484848484848484848484","share_difficulty":19,"share_id":"share-644","share_seq":644,"template_height":800020},{"accepted_at_ms":999984,"credit_policy":"stale-grace","job_id":"job-989","job_issued_at_ms":999984,"miner_id":"miner-10","network_difficulty":1000003,"ntime":1700000989,"order_key":"10:miner-10","p2mr_program_hex":"dddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddd","share_difficulty":1000,"share_id":"share-989","share_seq":989,"template_height":800030},{"accepted_at_ms":999997,"credit_policy":"stale-grace","job_id":"job-2900","job_issued_at_ms":999994,"miner_id":"miner-7","network_difficulty":1000010,"ntime":1700002900,"order_key":"07:miner-7","p2mr_program_hex":"5454545454545454545454545454545454545454545454545454545454545454","share_difficulty":1000,"share_id":"share-2900","share_seq":2900,"template_height":800090},{"accepted_at_ms":999971,"credit_policy":null,"job_id":"job-3282","job_issued_at_ms":999972,"miner_id":"miner-4","network_difficulty":1000001,"ntime":1700003282,"order_key":"04:miner-4","p2mr_program_hex":"d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2","share_difficulty":7,"share_id":"share-3282","share_seq":3282,"template_height":800102},{"accepted_at_ms":999999,"credit_policy":null,"job_id":"job-3700","job_issued_at_ms":999987,"miner_id":"miner-4","network_difficulty":1000011,"ntime":1700003700,"order_key":"04:miner-4","p2mr_program_hex":"7474747474747474747474747474747474747474747474747474747474747474","share_difficulty":19,"share_id":"share-3700","share_seq":3700,"template_height":800115},{"accepted_at_ms":999995,"credit_policy":null,"job_id":"job-2289","job_issued_at_ms":999986,"miner_id":"miner-1","network_difficulty":1000011,"ntime":1700002289,"order_key":"01:miner-1","p2mr_program_hex":"f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1","share_difficulty":2,"share_id":"share-2289","share_seq":2289,"template_height":800071},{"accepted_at_ms":999974,"credit_policy":null,"job_id":"job-3226","job_issued_at_ms":999978,"miner_id":"miner-3","network_difficulty":1000013,"ntime":1700003226,"order_key":"03:miner-3","p2mr_program_hex":"9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a","share_difficulty":101,"share_id":"share-3226","share_seq":3226,"template_height":800100},{"accepted_at_ms":999995,"credit_policy":null,"job_id":"job-2086","job_issued_at_ms":999986,"miner_id":"miner-7","network_difficulty":1000012,"ntime":1700002086,"order_key":"07:miner-7","p2mr_program_hex":"2626262626262626262626262626262626262626262626262626262626262626","share_difficulty":1,"share_id":"share-2086","share_seq":2086,"template_height":800065},{"accepted_at_ms":1000263,"credit_policy":null,"job_id":"job-3929","job_issued_at_ms":1000079,"miner_id":"miner-2","network_difficulty":1000002,"ntime":1700003929,"order_key":"02:miner-2","p2mr_program_hex":"5959595959595959595959595959595959595959595959595959595959595959","share_difficulty":3,"share_id":"share-3929","share_seq":3929,"template_height":800122},{"accepted_at_ms":999992,"credit_policy":null,"job_id":"job-3754","job_issued_at_ms":999990,"miner_id":"miner-3","network_difficulty":1000014,"ntime":1700003754,"order_key":"03:miner-3","p2mr_program_hex":"aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa","share_difficulty":3,"share_id":"share-3754","share_seq":3754,"template_height":800117},{"accepted_at_ms":999978,"credit_policy":"stale-grace","job_id":"job-3611","job_issued_at_ms":999970,"miner_id":"miner-3","network_difficulty":1000007,"ntime":1700003611,"order_key":"03:miner-3","p2mr_program_hex":"1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b","share_difficulty":1000,"share_id":"share-3611","share_seq":3611,"template_height":800112},{"accepted_at_ms":999982,"credit_policy":null,"job_id":"job-2467","job_issued_at_ms":999992,"miner_id":"miner-3","network_difficulty":1000002,"ntime":1700002467,"order_key":"03:miner-3","p2mr_program_hex":"a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3","share_difficulty":1,"share_id":"share-2467","share_seq":2467,"template_height":800077},{"accepted_at_ms":999985,"credit_policy":null,"job_id":"job-434","job_issued_at_ms":1000342,"miner_id":"miner-5","network_difficulty":1000009,"ntime":1700000434,"order_key":"05:miner-5","p2mr_program_hex":"b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2","share_difficulty":101,"share_id":"share-434","share_seq":434,"template_height":800013},{"accepted_at_ms":999973,"credit_policy":null,"job_id":"job-1651","job_issued_at_ms":999980,"miner_id":"miner-1","network_difficulty":1000002,"ntime":1700001651,"order_key":"01:miner-1","p2mr_program_hex":"7373737373737373737373737373737373737373737373737373737373737373","share_difficulty":1000,"share_id":"share-1651","share_seq":1651,"template_height":800051},{"accepted_at_ms":999980,"credit_policy":null,"job_id":"job-2362","job_issued_at_ms":999979,"miner_id":"miner-8","network_difficulty":1000016,"ntime":1700002362,"order_key":"08:miner-8","p2mr_program_hex":"3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a","share_difficulty":1000,"share_id":"share-2362","share_seq":2362,"template_height":800073},{"accepted_at_ms":999979,"credit_policy":null,"job_id":"job-3147","job_issued_at_ms":999990,"miner_id":"miner-1","network_difficulty":1000002,"ntime":1700003147,"order_key":"01:miner-1","p2mr_program_hex":"4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b","share_difficulty":3,"share_id":"share-3147","share_seq":3147,"template_height":800098},{"accepted_at_ms":999998,"credit_policy":"stale-grace","job_id":"job-1712","job_issued_at_ms":999987,"miner_id":"miner-7","network_difficulty":1000012,"ntime":1700001712,"order_key":"07:miner-7","p2mr_program_hex":"b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0","share_difficulty":19,"share_id":"share-1712","share_seq":1712,"template_height":800053},{"accepted_at_ms":999977,"credit_policy":"stale-grace","job_id":"job-2566","job_issued_at_ms":999981,"miner_id":"miner-3","network_difficulty":1000016,"ntime":1700002566,"order_key":"03:miner-3","p2mr_program_hex":"0606060606060606060606060606060606060606060606060606060606060606","share_difficulty":19,"share_id":"share-2566","share_seq":2566,"template_height":800080},{"accepted_at_ms":999973,"credit_policy":"stale-grace","job_id":"job-470","job_issued_at_ms":999991,"miner_id":"miner-8","network_difficulty":1000011,"ntime":1700000470,"order_key":"08:miner-8","p2mr_program_hex":"d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6","share_difficulty":1000,"share_id":"share-470","share_seq":470,"template_height":800014},{"accepted_at_ms":999980,"credit_policy":null,"job_id":"job-1751","job_issued_at_ms":999975,"miner_id":"miner-2","network_difficulty":1000000,"ntime":1700001751,"order_key":"02:miner-2","p2mr_program_hex":"d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7","share_difficulty":2,"share_id":"share-1751","share_seq":1751,"template_height":800054},{"accepted_at_ms":999970,"credit_policy":"stale-grace","job_id":"job-3516","job_issued_at_ms":999985,"miner_id":"miner-7","network_difficulty":1000014,"ntime":1700003516,"order_key":"07:miner-7","p2mr_program_hex":"bcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbc","share_difficulty":7,"share_id":"share-3516","share_seq":3516,"template_height":800109},{"accepted_at_ms":999994,"credit_policy":null,"job_id":"job-1616","job_issued_at_ms":999993,"miner_id":"miner-10","network_difficulty":1000001,"ntime":1700001616,"order_key":"10:miner-10","p2mr_program_hex":"5050505050505050505050505050505050505050505050505050505050505050","share_difficulty":7,"share_id":"share-1616","share_seq":1616,"template_height":800050},{"accepted_at_ms":999983,"credit_policy":null,"job_id":"job-696","job_issued_at_ms":999986,"miner_id":"miner-3","network_difficulty":1000016,"ntime":1700000696,"order_key":"03:miner-3","p2mr_program_hex":"b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8","share_difficulty":19,"share_id":"share-696","share_seq":696,"template_height":800021},{"accepted_at_ms":999989,"credit_policy":null,"job_id":"job-1681","job_issued_at_ms":999995,"miner_id":"miner-9","network_difficulty":1000015,"ntime":1700001681,"order_key":"09:miner-9","p2mr_program_hex":"9191919191919191919191919191919191919191919191919191919191919191","share_difficulty":1000,"share_id":"share-1681","share_seq":1681,"template_height":800052},{"accepted_at_ms":999981,"credit_policy":null,"job_id":"job-1684","job_issued_at_ms":999977,"miner_id":"miner-1","network_difficulty":1000001,"ntime":1700001684,"order_key":"01:miner-1","p2mr_program_hex":"9494949494949494949494949494949494949494949494949494949494949494","share_difficulty":3,"share_id":"share-1684","share_seq":1684,"template_height":800052},{"accepted_at_ms":999984,"credit_policy":"stale-grace","job_id":"job-2029","job_issued_at_ms":999997,"miner_id":"miner-5","network_difficulty":1000006,"ntime":1700002029,"order_key":"05:miner-5","p2mr_program_hex":"edededededededededededededededededededededededededededededededed","share_difficulty":1000,"share_id":"share-2029","share_seq":2029,"template_height":800063},{"accepted_at_ms":999980,"credit_policy":null,"job_id":"job-3280","job_issued_at_ms":1000493,"miner_id":"miner-2","network_difficulty":1000016,"ntime":1700003280,"order_key":"02:miner-2","p2mr_program_hex":"d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0","share_difficulty":7,"share_id":"share-3280","share_seq":3280,"template_height":800102},{"accepted_at_ms":999983,"credit_policy":null,"job_id":"job-1763","job_issued_at_ms":999995,"miner_id":"miner-3","network_difficulty":1000012,"ntime":1700001763,"order_key":"03:miner-3","p2mr_program_hex":"e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3","share_difficulty":3,"share_id":"share-1763","share_seq":1763,"template_height":800055},{"accepted_at_ms":999984,"credit_policy":null,"job_id":"job-1959","job_issued_at_ms":999972,"miner_id":"miner-1","network_difficulty":1000004,"ntime":1700001959,"order_key":"01:miner-1","p2mr_program_hex":"a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7","share_difficulty":1000,"share_id":"share-1959","share_seq":1959,"template_height":800061},{"accepted_at_ms":999990,"credit_policy":null,"job_id":"job-1247","job_issued_at_ms":999997,"miner_id":"miner-4","network_difficulty":1000006,"ntime":1700001247,"order_key":"04:miner-4","p2mr_program_hex":"dfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdf","share_difficulty":19,"share_id":"share-1247","share_seq":1247,"template_height":800038},{"accepted_at_ms":999993,"credit_policy":null,"job_id":"job-2637","job_issued_at_ms":1000491,"miner_id":"miner-8","network_difficulty":1000002,"ntime":1700002637,"order_key":"08:miner-8","p2mr_program_hex":"4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d","share_difficulty":1,"share_id":"share-2637","share_seq":2637,"template_height":800082},{"accepted_at_ms":1000018,"credit_policy":null,"job_id":"job-1022","job_issued_at_ms":1000199,"miner_id":"miner-10","network_difficulty":1000002,"ntime":1700001022,"order_key":"10:miner-10","p2mr_program_hex":"fefefefefefefefefefefefefefefefefefefefefefefefefefefefefefefefe","share_difficulty":1000,"share_id":"share-1022","share_seq":1022,"template_height":800031},{"accepted_at_ms":999999,"credit_policy":"stale-grace","job_id":"job-1955","job_issued_at_ms":999994,"miner_id":"miner-8","network_difficulty":1000000,"ntime":1700001955,"order_key":"08:miner-8","p2mr_program_hex":"a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3","share_difficulty":1,"share_id":"share-1955","share_seq":1955,"template_height":800061},{"accepted_at_ms":999970,"credit_policy":null,"job_id":"job-1343","job_issued_at_ms":999975,"miner_id":"miner-1","network_difficulty":1000000,"ntime":1700001343,"order_key":"01:miner-1","p2mr_program_hex":"3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f","share_difficulty":1000,"share_id":"share-1343","share_seq":1343,"template_height":800041},{"accepted_at_ms":999995,"credit_policy":null,"job_id":"job-848","job_issued_at_ms":999982,"miner_id":"miner-1","network_difficulty":1000015,"ntime":1700000848,"order_key":"01:miner-1","p2mr_program_hex":"5050505050505050505050505050505050505050505050505050505050505050","share_difficulty":19,"share_id":"share-848","share_seq":848,"template_height":800026},{"accepted_at_ms":999972,"credit_policy":null,"job_id":"job-493","job_issued_at_ms":999990,"miner_id":"miner-9","network_difficulty":1000000,"ntime":1700000493,"order_key":"09:miner-9","p2mr_program_hex":"edededededededededededededededededededededededededededededededed","share_difficulty":19,"share_id":"share-493","share_seq":493,"template_height":800015},{"accepted_at_ms":1000000,"credit_policy":null,"job_id":"job-787","job_issued_at_ms":999980,"miner_id":"miner-6","network_difficulty":1000005,"ntime":1700000787,"order_key":"06:miner-6","p2mr_program_hex":"1313131313131313131313131313131313131313131313131313131313131313","share_difficulty":101,"share_id":"share-787","share_seq":787,"template_height":800024},{"accepted_at_ms":999987,"credit_policy":null,"job_id":"job-1987","job_issued_at_ms":999993,"miner_id":"miner-7","network_difficulty":1000015,"ntime":1700001987,"order_key":"07:miner-7","p2mr_program_hex":"c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3","share_difficulty":7,"share_id":"share-1987","share_seq":1987,"template_height":800062},{"accepted_at_ms":999995,"credit_policy":null,"job_id":"job-885","job_issued_at_ms":999978,"miner_id":"miner-5","network_difficulty":1000001,"ntime":1700000885,"order_key":"05:miner-5","p2mr_program_hex":"7575757575757575757575757575757575757575757575757575757575757575","share_difficulty":2,"share_id":"share-885","share_seq":885,"template_height":800027},{"accepted_at_ms":999971,"credit_policy":null,"job_id":"job-231","job_issued_at_ms":999993,"miner_id":"miner-0","network_difficulty":1000010,"ntime":1700000231,"order_key":"00:miner-0","p2mr_program_hex":"e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7","share_difficulty":7,"share_id":"share-231","share_seq":231,"template_height":800007},{"accepted_at_ms":999982,"credit_policy":null,"job_id":"job-3367","job_issued_at_ms":999984,"miner_id":"miner-1","network_difficulty":1000001,"ntime":1700003367,"order_key":"01:miner-1","p2mr_program_hex":"2727272727272727272727272727272727272727272727272727272727272727","share_difficulty":2,"share_id":"share-3367","share_seq":3367,"template_height":800105},{"accepted_at_ms":999981,"credit_policy":null,"job_id":"job-467","job_issued_at_ms":999975,"miner_id":"miner-5","network_difficulty":1000008,"ntime":1700000467,"order_key":"05:miner-5","p2mr_program_hex":"d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3","share_difficulty":1,"share_id":"share-467","share_seq":467,"template_height":800014},{"accepted_at_ms":999991,"credit_policy":"stale-grace","job_id":"job-922","job_issued_at_ms":999993,"miner_id":"miner-9","network_difficulty":1000004,"ntime":1700000922,"order_key":"09:miner-9","p2mr_program_hex":"9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a","share_difficulty":2,"share_id":"share-922","share_seq":922,"template_height":800028},{"accepted_at_ms":999976,"credit_policy":null,"job_id":"job-2434","job_issued_at_ms":1000275,"miner_id":"miner-3","network_difficulty":1000003,"ntime":1700002434,"order_key":"03:miner-3","p2mr_program_hex":"8282828282828282828282828282828282828282828282828282828282828282","share_difficulty":2,"share_id":"share-2434","share_seq":2434,"template_height":800076},{"accepted_at_ms":999973,"credit_policy":null,"job_id":"job-3889","job_issued_at_ms":999978,"miner_id":"miner-6","network_difficulty":1000013,"ntime":1700003889,"order_key":"06:miner-6","p2mr_program_hex":"3131313131313131313131313131313131313131313131313131313131313131","share_difficulty":19,"share_id":"share-3889","share_seq":3889,"template_height":800121},{"accepted_at_ms":999970,"credit_policy":null,"job_id":"job-3445","job_issued_at_ms":999979,"miner_id":"miner-2","network_difficulty":1000011,"ntime":1700003445,"order_key":"02:miner-2","p2mr_program_hex":"7575757575757575757575757575757575757575757575757575757575757575","share_difficulty":1000,"share_id":"share-3445","share_seq":3445,"template_height":800107},{"accepted_at_ms":999998,"credit_policy":null,"job_id":"job-3046","job_issued_at_ms":999994,"miner_id":"miner-10","network_difficulty":1000003,"ntime":1700003046,"order_key":"10:miner-10","p2mr_program_hex":"e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6","share_difficulty":3,"share_id":"share-3046","share_seq":3046,"template_height":800095},{"accepted_at_ms":999978,"credit_policy":null,"job_id":"job-3039","job_issued_at_ms":999987,"miner_id":"miner-3","network_difficulty":1000013,"ntime":1700003039,"order_key":"03:miner-3","p2mr_program_hex":"dfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdf","share_difficulty":101,"share_id":"share-3039","share_seq":3039,"template_height":800094},{"accepted_at_ms":999984,"credit_policy":"stale-grace","job_id":"job-2525","job_issued_at_ms":999975,"miner_id":"miner-6","network_difficulty":1000009,"ntime":1700002525,"order_key":"06:miner-6","p2mr_program_hex":"dddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddd","share_difficulty":1,"share_id":"share-2525","share_seq":2525,"template_height":800078},{"accepted_at_ms":999990,"credit_policy":null,"job_id":"job-3559","job_issued_at_ms":999975,"miner_id":"miner-6","network_difficulty":1000006,"ntime":1700003559,"order_key":"06:miner-6","p2mr_program_hex":"e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7","share_difficulty":2,"share_id":"share-3559","share_seq":3559,"template_height":800111},{"accepted_at_ms":999970,"credit_policy":null,"job_id":"job-2703","job_issued_at_ms":999974,"miner_id":"miner-8","network_difficulty":1000000,"ntime":1700002703,"order_key":"08:miner-8","p2mr_program_hex":"8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f","share_difficulty":101,"share_id":"share-2703","share_seq":2703,"template_height":800084},{"accepted_at_ms":999999,"credit_policy":"stale-grace","job_id":"job-522","job_issued_at_ms":999980,"miner_id":"miner-5","network_difficulty":1000012,"ntime":1700000522,"order_key":"05:miner-5","p2mr_program_hex":"0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a","share_difficulty":1000,"share_id":"share-522","share_seq":522,"template_height":800016},{"accepted_at_ms":999980,"credit_policy":null,"job_id":"job-2840","job_issued_at_ms":999981,"miner_id":"miner-2","network_difficulty":1000001,"ntime":1700002840,"order_key":"02:miner-2","p2mr_program_hex":"1818181818181818181818181818181818181818181818181818181818181818","share_difficulty":7,"share_id":"share-2840","share_seq":2840,"template_height":800088},{"accepted_at_ms":999980,"credit_policy":null,"job_id":"job-1509","job_issued_at_ms":999985,"miner_id":"miner-2","network_difficulty":1000013,"ntime":1700001509,"order_key":"02:miner-2","p2mr_program_hex":"e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5","share_difficulty":2,"share_id":"share-1509","share_seq":1509,"template_height":800047},{"accepted_at_ms":999984,"credit_policy":null,"job_id":"job-1341","job_issued_at_ms":999983,"miner_id":"miner-10","network_difficulty":1000015,"ntime":1700001341,"order_key":"10:miner-10","p2mr_program_hex":"3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d","share_difficulty":7,"share_id":"share-1341","share_seq":1341,"template_height":800041},{"accepted_at_ms":1000115,"credit_policy":null,"job_id":"job-2998","job_issued_at_ms":999985,"miner_id":"miner-6","network_difficulty":1000006,"ntime":1700002998,"order_key":"06:miner-6","p2mr_program_hex":"b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6","share_difficulty":7,"share_id":"share-2998","share_seq":2998,"template_height":800093},{"accepted_at_ms":999973,"credit_policy":"stale-grace","job_id":"job-2587","job_issued_at_ms":999992,"miner_id":"miner-2","network_difficulty":1000003,"ntime":1700002587,"order_key":"02:miner-2","p2mr_program_hex":"1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b","share_difficulty":2,"share_id":"share-2587","share_seq":2587,"template_height":800080},{"accepted_at_ms":999998,"credit_policy":null,"job_id":"job-1719","job_issued_at_ms":1000378,"miner_id":"miner-3","network_difficulty":1000002,"ntime":1700001719,"order_key":"03:miner-3","p2mr_program_hex":"b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7","share_difficulty":1,"share_id":"share-1719","share_seq":1719,"template_height":800053},{"accepted_at_ms":999976,"credit_policy":null,"job_id":"job-2843","job_issued_at_ms":999996,"miner_id":"miner-5","network_difficulty":1000004,"ntime":1700002843,"order_key":"05:miner-5","p2mr_program_hex":"1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b","share_difficulty":19,"share_id":"share-2843","share_seq":2843,"template_height":800088},{"accepted_at_ms":999976,"credit_policy":null,"job_id":"job-2856","job_issued_at_ms":999989,"miner_id":"miner-7","network_difficulty":1000000,"ntime":1700002856,"order_key":"07:miner-7","p2mr_program_hex":"2828282828282828282828282828282828282828282828282828282828282828","share_difficulty":3,"share_id":"share-2856","share_seq":2856,"template_height":800089},{"accepted_at_ms":999981,"credit_policy":null,"job_id":"job-3859","job_issued_at_ms":999987,"miner_id":"miner-9","network_difficulty":1000000,"ntime":1700003859,"order_key":"09:miner-9","p2mr_program_hex":"1313131313131313131313131313131313131313131313131313131313131313","share_difficulty":7,"share_id":"share-3859","share_seq":3859,"template_height":800120},{"accepted_at_ms":999995,"credit_policy":null,"job_id":"job-1678","job_issued_at_ms":999999,"miner_id":"miner-6","network_difficulty":1000012,"ntime":1700001678,"order_key":"06:miner-6","p2mr_program_hex":"8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e","share_difficulty":19,"share_id":"share-1678","share_seq":1678,"template_height":800052},{"accepted_at_ms":999985,"credit_policy":"stale-grace","job_id":"job-2976","job_issued_at_ms":999983,"miner_id":"miner-6","network_difficulty":1000001,"ntime":1700002976,"order_key":"06:miner-6","p2mr_program_hex":"a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0","share_difficulty":2,"share_id":"share-2976","share_seq":2976,"template_height":800093},{"accepted_at_ms":999975,"credit_policy":null,"job_id":"job-3004","job_issued_at_ms":999983,"miner_id":"miner-1","network_difficulty":1000012,"ntime":1700003004,"order_key":"01:miner-1","p2mr_program_hex":"bcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbc","share_difficulty":1,"share_id":"share-3004","share_seq":3004,"template_height":800093},{"accepted_at_ms":999974,"credit_policy":null,"job_id":"job-1047","job_issued_at_ms":999998,"miner_id":"miner-2","network_difficulty":1000010,"ntime":1700001047,"order_key":"02:miner-2","p2mr_program_hex":"1717171717171717171717171717171717171717171717171717171717171717","share_difficulty":3,"share_id":"share-1047","share_seq":1047,"template_height":800032},{"accepted_at_ms":999990,"credit_policy":null,"job_id":"job-33","job_issued_at_ms":999995,"miner_id":"miner-0","network_difficulty":1000016,"ntime":1700000033,"order_key":"00:miner-0","p2mr_program_hex":"2121212121212121212121212121212121212121212121212121212121212121","share_difficulty":1,"share_id":"share-33","share_seq":33,"template_height":800001},{"accepted_at_ms":999994,"credit_policy":null,"job_id":"job-207","job_issued_at_ms":999999,"miner_id":"miner-9","network_difficulty":1000003,"ntime":1700000207,"order_key":"09:miner-9","p2mr_program_hex":"cfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcf","share_difficulty":1,"share_id":"share-207","share_seq":207,"template_height":800006},{"accepted_at_ms":999986,"credit_policy":null,"job_id":"job-103","job_issued_at_ms":999996,"miner_id":"miner-4","network_difficulty":1000001,"ntime":1700000103,"order_key":"04:miner-4","p2mr_program_hex":"6767676767676767676767676767676767676767676767676767676767676767","share_difficulty":1000,"share_id":"share-103","share_seq":103,"template_height":800003},{"accepted_at_ms":999998,"credit_policy":"stale-grace","job_id":"job-718","job_issued_at_ms":999992,"miner_id":"miner-3","network_difficulty":1000004,"ntime":1700000718,"order_key":"03:miner-3","p2mr_program_hex":"cececececececececececececececececececececececececececececececece","share_difficulty":101,"share_id":"share-718","share_seq":718,"template_height":800022},{"accepted_at_ms":999995,"credit_policy":null,"job_id":"job-3529","job_issued_at_ms":999979,"miner_id":"miner-9","network_difficulty":1000010,"ntime":1700003529,"order_key":"09:miner-9","p2mr_program_hex":"c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9","share_difficulty":1000,"share_id":"share-3529","share_seq":3529,"template_height":800110},{"accepted_at_ms":1000107,"credit_policy":null,"job_id":"job-2577","job_issued_at_ms":999978,"miner_id":"miner-3","network_difficulty":1000010,"ntime":1700002577,"order_key":"03:miner-3","p2mr_program_hex":"1111111111111111111111111111111111111111111111111111111111111111","share_difficulty":101,"share_id":"share-2577","share_seq":2577,"template_height":800080},{"accepted_at_ms":999988,"credit_policy":null,"job_id":"job-976","job_issued_at_ms":999992,"miner_id":"miner-8","network_difficulty":1000007,"ntime":1700000976,"order_key":"08:miner-8","p2mr_program_hex":"d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0","share_difficulty":19,"share_id":"share-976","share_seq":976,"template_height":800030},{"accepted_at_ms":999986,"credit_policy":null,"job_id":"job-85","job_issued_at_ms":999993,"miner_id":"miner-8","network_difficulty":1000000,"ntime":1700000085,"order_key":"08:miner-8","p2mr_program_hex":"5555555555555555555555555555555555555555555555555555555555555555","share_difficulty":7,"share_id":"share-85","share_seq":85,"template_height":800002},{"accepted_at_ms":1000000,"credit_policy":null,"job_id":"job-3512","job_issued_at_ms":999970,"miner_id":"miner-3","network_difficulty":1000010,"ntime":1700003512,"order_key":"03:miner-3","p2mr_program_hex":"b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8","share_difficulty":1000,"share_id":"share-3512","share_seq":3512,"template_height":800109},{"accepted_at_ms":999982,"credit_policy":null,"job_id":"job-3774","job_issued_at_ms":999981,"miner_id":"miner-1","network_difficulty":1000000,"ntime":1700003774,"order_key":"01:miner-1","p2mr_program_hex":"bebebebebebebebebebebebebebebebebebebebebebebebebebebebebebebebe","share_difficulty":1,"share_id":"share-3774","share_seq":3774,"template_height":800117},{"accepted_at_ms":999993,"credit_policy":null,"job_id":"job-142","job_issued_at_ms":999991,"miner_id":"miner-10","network_difficulty":1000006,"ntime":1700000142,"order_key":"10:miner-10","p2mr_program_hex":"8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e","share_difficulty":19,"share_id":"share-142","share_seq":142,"template_height":800004},{"accepted_at_ms":999972,"credit_policy":null,"job_id":"job-2364","job_issued_at_ms":999970,"miner_id":"miner-10","network_difficulty":1000001,"ntime":1700002364,"order_key":"10:miner-10","p2mr_program_hex":"3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c","share_difficulty":1000,"share_id":"share-2364","share_seq":2364,"template_height":800073},{"accepted_at_ms":999984,"credit_policy":"stale-grace","job_id":"job-3309","job_issued_at_ms":999979,"miner_id":"miner-9","network_difficulty":1000011,"ntime":1700003309,"order_key":"09:miner-9","p2mr_program_hex":"edededededededededededededededededededededededededededededededed","share_difficulty":101,"share_id":"share-3309","share_seq":3309,"template_height":800103},{"accepted_at_ms":999974,"credit_policy":null,"job_id":"job-2962","job_issued_at_ms":999976,"miner_id":"miner-3","network_difficulty":1000004,"ntime":1700002962,"order_key":"03:miner-3","p2mr_program_hex":"9292929292929292929292929292929292929292929292929292929292929292","share_difficulty":1000,"share_id":"share-2962","share_seq":2962,"template_height":800092},{"accepted_at_ms":999994,"credit_policy":"stale-grace","job_id":"job-3165","job_issued_at_ms":999977,"miner_id":"miner-8","network_difficulty":1000003,"ntime":1700003165,"order_key":"08:miner-8","p2mr_program_hex":"5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d","share_difficulty":3,"share_id":"share-3165","share_seq":3165,"template_height":800098},{"accepted_at_ms":999978,"credit_policy":null,"job_id":"job-2722","job_issued_at_ms":999971,"miner_id":"miner-5","network_difficulty":1000002,"ntime":1700002722,"order_key":"05:miner-5","p2mr_program_hex":"a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2","share_difficulty":3,"share_id":"share-2722","share_seq":2722,"template_height":800085},{"accepted_at_ms":999985,"credit_policy":"stale-grace","job_id":"job-526","job_issued_at_ms":999991,"miner_id":"miner-9","network_difficulty":1000016,"ntime":1700000526,"order_key":"09:miner-9","p2mr_program_hex":"0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e","share_difficulty":101,"share_id":"share-526","share_seq":526,"template_height":800016},{"accepted_at_ms":999981,"credit_policy":null,"job_id":"job-1900","job_issued_at_ms":999971,"miner_id":"miner-8","network_difficulty":1000013,"ntime":1700001900,"order_key":"08:miner-8","p2mr_program_hex":"6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c","share_difficulty":7,"share_id":"share-1900","share_seq":1900,"template_height":800059},{"accepted_at_ms":999989,"credit_policy":null,"job_id":"job-3583","job_issued_at_ms":999982,"miner_id":"miner-8","network_difficulty":1000013,"ntime":1700003583,"order_key":"08:miner-8","p2mr_program_hex":"ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff","share_difficulty":3,"share_id":"share-3583","share_seq":3583,"template_height":800111},{"accepted_at_ms":999983,"credit_policy":"stale-grace","job_id":"job-2583","job_issued_at_ms":999985,"miner_id":"miner-9","network_difficulty":1000016,"ntime":1700002583,"order_key":"09:miner-9","p2mr_program_hex":"1717171717171717171717171717171717171717171717171717171717171717","share_difficulty":19,"share_id":"share-2583","share_seq":2583,"template_height":800080},{"accepted_at_ms":999996,"credit_policy":"stale-grace","job_id":"job-1283","job_issued_at_ms":999985,"miner_id":"miner-7","network_difficulty":1000008,"ntime":1700001283,"order_key":"07:miner-7","p2mr_program_hex":"0303030303030303030303030303030303030303030303030303030303030303","share_difficulty":101,"share_id":"share-1283","share_seq":1283,"template_height":800040},{"accepted_at_ms":999973,"credit_policy":null,"job_id":"job-3910","job_issued_at_ms":999989,"miner_id":"miner-5","network_difficulty":1000000,"ntime":1700003910,"order_key":"05:miner-5","p2mr_program_hex":"4646464646464646464646464646464646464646464646464646464646464646","share_difficulty":101,"share_id":"share-3910","share_seq":3910,"template_height":800122},{"accepted_at_ms":999995,"credit_policy":null,"job_id":"job-663","job_issued_at_ms":999991,"miner_id":"miner-3","network_difficulty":1000000,"ntime":1700000663,"order_key":"03:miner-3","p2mr_program_hex":"9797979797979797979797979797979797979797979797979797979797979797","share_difficulty":7,"share_id":"share-663","share_seq":663,"template_height":800020},{"accepted_at_ms":999971,"credit_policy":"stale-grace","job_id":"job-3504","job_issued_at_ms":999995,"miner_id":"miner-6","network_difficulty":1000002,"ntime":1700003504,"order_key":"06:miner-6","p2mr_program_hex":"b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0","share_difficulty":1,"share_id":"share-3504","share_seq":3504,"template_height":800109},{"accepted_at_ms":999993,"credit_policy":"stale-grace","job_id":"job-223","job_issued_at_ms":999986,"miner_id":"miner-3","network_difficulty":1000002,"ntime":1700000223,"order_key":"03:miner-3","p2mr_program_hex":"dfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdf","share_difficulty":1,"share_id":"share-223","share_seq":223,"template_height":800006},{"accepted_at_ms":999984,"credit_policy":null,"job_id":"job-3019","job_issued_at_ms":999985,"miner_id":"miner-5","network_difficulty":1000010,"ntime":1700003019,"order_key":"05:miner-5","p2mr_program_hex":"cbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcb","share_difficulty":1,"share_id":"share-3019","share_seq":3019,"template_height":800094},{"accepted_at_ms":999982,"credit_policy":null,"job_id":"job-3748","job_issued_at_ms":999991,"miner_id":"miner-8","network_difficulty":1000008,"ntime":1700003748,"order_key":"08:miner-8","p2mr_program_hex":"a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4","share_difficulty":1,"share_id":"share-3748","share_seq":3748,"template_height":800117},{"accepted_at_ms":999997,"credit_policy":null,"job_id":"job-2806","job_issued_at_ms":999997,"miner_id":"miner-1","network_difficulty":1000001,"ntime":1700002806,"order_key":"01:miner-1","p2mr_program_hex":"f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6","share_difficulty":1000,"share_id":"share-2806","share_seq":2806,"template_height":800087},{"accepted_at_ms":1000219,"credit_policy":null,"job_id":"job-2636","job_issued_at_ms":1000034,"miner_id":"miner-7","network_difficulty":1000001,"ntime":1700002636,"order_key":"07:miner-7","p2mr_program_hex":"4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c","share_difficulty":7,"share_id":"share-2636","share_seq":2636,"template_height":800082},{"accepted_at_ms":999992,"credit_policy":null,"job_id":"job-3476","job_issued_at_ms":999989,"miner_id":"miner-0","network_difficulty":1000008,"ntime":1700003476,"order_key":"00:miner-0","p2mr_program_hex":"9494949494949494949494949494949494949494949494949494949494949494","share_difficulty":1,"share_id":"share-3476","share_seq":3476,"template_height":800108},{"accepted_at_ms":999978,"credit_policy":null,"job_id":"job-2631","job_issued_at_ms":999984,"miner_id":"miner-2","network_difficulty":1000013,"ntime":1700002631,"order_key":"02:miner-2","p2mr_program_hex":"4747474747474747474747474747474747474747474747474747474747474747","share_difficulty":1000,"share_id":"share-2631","share_seq":2631,"template_height":800082},{"accepted_at_ms":999978,"credit_policy":null,"job_id":"job-2931","job_issued_at_ms":999994,"miner_id":"miner-5","network_difficulty":1000007,"ntime":1700002931,"order_key":"05:miner-5","p2mr_program_hex":"7373737373737373737373737373737373737373737373737373737373737373","share_difficulty":1,"share_id":"share-2931","share_seq":2931,"template_height":800091},{"accepted_at_ms":999993,"credit_policy":null,"job_id":"job-2792","job_issued_at_ms":999979,"miner_id":"miner-9","network_difficulty":1000004,"ntime":1700002792,"order_key":"09:miner-9","p2mr_program_hex":"e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8","share_difficulty":7,"share_id":"share-2792","share_seq":2792,"template_height":800087},{"accepted_at_ms":999998,"credit_policy":null,"job_id":"job-849","job_issued_at_ms":999988,"miner_id":"miner-2","network_difficulty":1000016,"ntime":1700000849,"order_key":"02:miner-2","p2mr_program_hex":"5151515151515151515151515151515151515151515151515151515151515151","share_difficulty":2,"share_id":"share-849","share_seq":849,"template_height":800026},{"accepted_at_ms":1000000,"credit_policy":"stale-grace","job_id":"job-3395","job_issued_at_ms":999984,"miner_id":"miner-7","network_difficulty":1000012,"ntime":1700003395,"order_key":"07:miner-7","p2mr_program_hex":"4343434343434343434343434343434343434343434343434343434343434343","share_difficulty":19,"share_id":"share-3395","share_seq":3395,"template_height":800106},{"accepted_at_ms":999999,"credit_policy":"stale-grace","job_id":"job-3124","job_issued_at_ms":999980,"miner_id":"miner-0","network_difficulty":1000013,"ntime":1700003124,"order_key":"00:miner-0","p2mr_program_hex":"3434343434343434343434343434343434343434343434343434343434343434","share_difficulty":101,"share_id":"share-3124","share_seq":3124,"template_height":800097},{"accepted_at_ms":999990,"credit_policy":"stale-grace","job_id":"job-765","job_issued_at_ms":999982,"miner_id":"miner-6","network_difficulty":1000000,"ntime":1700000765,"order_key":"06:miner-6","p2mr_program_hex":"fdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfd","share_difficulty":1000,"share_id":"share-765","share_seq":765,"template_height":800023},{"accepted_at_ms":999982,"credit_policy":null,"job_id":"job-3208","job_issued_at_ms":999984,"miner_id":"miner-7","network_difficulty":1000012,"ntime":1700003208,"order_key":"07:miner-7","p2mr_program_hex":"8888888888888888888888888888888888888888888888888888888888888888","share_difficulty":101,"share_id":"share-3208","share_seq":3208,"template_height":800100},{"accepted_at_ms":1000294,"credit_policy":null,"job_id":"job-971","job_issued_at_ms":1000306,"miner_id":"miner-3","network_difficulty":1000002,"ntime":1700000971,"order_key":"03:miner-3","p2mr_program_hex":"cbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcb","share_difficulty":19,"share_id":"share-971","share_seq":971,"template_height":800030},{"accepted_at_ms":999974,"credit_policy":null,"job_id":"job-405","job_issued_at_ms":999972,"miner_id":"miner-9","network_difficulty":1000014,"ntime":1700000405,"order_key":"09:miner-9","p2mr_program_hex":"9595959595959595959595959595959595959595959595959595959595959595","share_difficulty":1000,"share_id":"share-405","share_seq":405,"template_height":800012},{"accepted_at_ms":999980,"credit_policy":null,"job_id":"job-3938","job_issued_at_ms":999982,"miner_id":"miner-0","network_difficulty":1000011,"ntime":1700003938,"order_key":"00:miner-0","p2mr_program_hex":"6262626262626262626262626262626262626262626262626262626262626262","share_difficulty":1,"share_id":"share-3938","share_seq":3938,"template_height":800123},{"accepted_at_ms":999985,"credit_policy":"stale-grace","job_id":"job-970","job_issued_at_ms":999988,"miner_id":"miner-2","network_difficulty":1000001,"ntime":1700000970,"order_key":"02:miner-2","p2mr_program_hex":"cacacacacacacacacacacacacacacacacacacacacacacacacacacacacacacaca","share_difficulty":7,"share_id":"share-970","share_seq":970,"template_height":800030},{"accepted_at_ms":999992,"credit_policy":null,"job_id":"job-353","job_issued_at_ms":999985,"miner_id":"miner-1","network_difficulty":1000013,"ntime":1700000353,"order_key":"01:miner-1","p2mr_program_hex":"6161616161616161616161616161616161616161616161616161616161616161","share_difficulty":1,"share_id":"share-353","share_seq":353,"template_height":800011},{"accepted_at_ms":999975,"credit_policy":null,"job_id":"job-3892","job_issued_at_ms":999970,"miner_id":"miner-9","network_difficulty":1000016,"ntime":1700003892,"order_key":"09:miner-9","p2mr_program_hex":"3434343434343434343434343434343434343434343434343434343434343434","share_difficulty":7,"share_id":"share-3892","share_seq":3892,"template_height":800121},{"accepted_at_ms":1000302,"credit_policy":null,"job_id":"job-2781","job_issued_at_ms":1000321,"miner_id":"miner-9","network_difficulty":1000010,"ntime":1700002781,"order_key":"09:miner-9","p2mr_program_hex":"dddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddd","share_difficulty":3,"share_id":"share-2781","share_seq":2781,"template_height":800086},{"accepted_at_ms":999974,"credit_policy":null,"job_id":"job-1082","job_issued_at_ms":999985,"miner_id":"miner-4","network_difficulty":1000011,"ntime":1700001082,"order_key":"04:miner-4","p2mr_program_hex":"3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a","share_difficulty":1,"share_id":"share-1082","share_seq":1082,"template_height":800033},{"accepted_at_ms":999982,"credit_policy":"stale-grace","job_id":"job-2411","job_issued_at_ms":999995,"miner_id":"miner-2","network_difficulty":1000014,"ntime":1700002411,"order_key":"02:miner-2","p2mr_program_hex":"6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b","share_difficulty":7,"share_id":"share-2411","share_seq":2411,"template_height":800075},{"accepted_at_ms":999997,"credit_policy":null,"job_id":"job-2691","job_issued_at_ms":999992,"miner_id":"miner-7","network_difficulty":1000005,"ntime":1700002691,"order_key":"07:miner-7","p2mr_program_hex":"8383838383838383838383838383838383838383838383838383838383838383","share_difficulty":1,"share_id":"share-2691","share_seq":2691,"template_height":800084},{"accepted_at_ms":999990,"credit_policy":null,"job_id":"job-2995","job_issued_at_ms":999998,"miner_id":"miner-3","network_difficulty":1000003,"ntime":1700002995,"order_key":"03:miner-3","p2mr_program_hex":"b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3","share_difficulty":2,"share_id":"share-2995","share_seq":2995,"template_height":800093},{"accepted_at_ms":999987,"credit_policy":null,"job_id":"job-3042","job_issued_at_ms":999986,"miner_id":"miner-6","network_difficulty":1000016,"ntime":1700003042,"order_key":"06:miner-6","p2mr_program_hex":"e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2","share_difficulty":1000,"share_id":"share-3042","share_seq":3042,"template_height":800095},{"accepted_at_ms":999978,"credit_policy":"stale-grace","job_id":"job-2441","job_issued_at_ms":999971,"miner_id":"miner-10","network_difficulty":1000010,"ntime":1700002441,"order_key":"10:miner-10","p2mr_program_hex":"8989898989898989898989898989898989898989898989898989898989898989","share_difficulty":1000,"share_id":"share-2441","share_seq":2441,"template_height":800076},{"accepted_at_ms":999970,"credit_policy":null,"job_id":"job-2526","job_issued_at_ms":999992,"miner_id":"miner-7","network_difficulty":1000010,"ntime":1700002526,"order_key":"07:miner-7","p2mr_program_hex":"dededededededededededededededededededededededededededededededede","share_difficulty":19,"share_id":"share-2526","share_seq":2526,"template_height":800078},{"accepted_at_ms":999994,"credit_policy":null,"job_id":"job-1559","job_issued_at_ms":999972,"miner_id":"miner-8","network_difficulty":1000012,"ntime":1700001559,"order_key":"08:miner-8","p2mr_program_hex":"1717171717171717171717171717171717171717171717171717171717171717","share_difficulty":101,"share_id":"share-1559","share_seq":1559,"template_height":800048},{"accepted_at_ms":999985,"credit_policy":null,"job_id":"job-2037","job_issued_at_ms":999982,"miner_id":"miner-2","network_difficulty":1000014,"ntime":1700002037,"order_key":"02:miner-2","p2mr_program_hex":"f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5","share_difficulty":7,"share_id":"share-2037","share_seq":2037,"template_height":800063},{"accepted_at_ms":999974,"credit_policy":null,"job_id":"job-1917","job_issued_at_ms":999978,"miner_id":"miner-3","network_difficulty":1000013,"ntime":1700001917,"order_key":"03:miner-3","p2mr_program_hex":"7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d","share_difficulty":7,"share_id":"share-1917","share_seq":1917,"template_height":800059},{"accepted_at_ms":999970,"credit_policy":null,"job_id":"job-2008","job_issued_at_ms":999987,"miner_id":"miner-6","network_difficulty":1000002,"ntime":1700002008,"order_key":"06:miner-6","p2mr_program_hex":"d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8","share_difficulty":1000,"share_id":"share-2008","share_seq":2008,"template_height":800062},{"accepted_at_ms":999976,"credit_policy":"stale-grace","job_id":"job-3940","job_issued_at_ms":999995,"miner_id":"miner-2","network_difficulty":1000013,"ntime":1700003940,"order_key":"02:miner-2","p2mr_program_hex":"6464646464646464646464646464646464646464646464646464646464646464","share_difficulty":2,"share_id":"share-3940","share_seq":3940,"template_height":800123},{"accepted_at_ms":999984,"credit_policy":"stale-grace","job_id":"job-1520","job_issued_at_ms":999974,"miner_id":"miner-2","network_difficulty":1000007,"ntime":1700001520,"order_key":"02:miner-2","p2mr_program_hex":"f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0","share_difficulty":7,"share_id":"share-1520","share_seq":1520,"template_height":800047},{"accepted_at_ms":999997,"credit_policy":null,"job_id":"job-79","job_issued_at_ms":999975,"miner_id":"miner-2","network_difficulty":1000011,"ntime":1700000079,"order_key":"02:miner-2","p2mr_program_hex":"4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f","share_difficulty":1000,"share_id":"share-79","share_seq":79,"template_height":800002},{"accepted_at_ms":999990,"credit_policy":null,"job_id":"job-1223","job_issued_at_ms":1000000,"miner_id":"miner-2","network_difficulty":1000016,"ntime":1700001223,"order_key":"02:miner-2","p2mr_program_hex":"c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7","share_difficulty":1000,"share_id":"share-1223","share_seq":1223,"template_height":800038},{"accepted_at_ms":999981,"credit_policy":null,"job_id":"job-815","job_issued_at_ms":999985,"miner_id":"miner-1","network_difficulty":1000016,"ntime":1700000815,"order_key":"01:miner-1","p2mr_program_hex":"2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f","share_difficulty":2,"share_id":"share-815","share_seq":815,"template_height":800025},{"accepted_at_ms":1000000,"credit_policy":null,"job_id":"job-269","job_issued_at_ms":999995,"miner_id":"miner-5","network_difficulty":1000014,"ntime":1700000269,"order_key":"05:miner-5","p2mr_program_hex":"0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d","share_difficulty":2,"share_id":"share-269","share_seq":269,"template_height":800008},{"accepted_at_ms":999977,"credit_policy":null,"job_id":"job-645","job_issued_at_ms":999977,"miner_id":"miner-7","network_difficulty":1000016,"ntime":1700000645,"order_key":"07:miner-7","p2mr_program_hex":"8585858585858585858585858585858585858585858585858585858585858585","share_difficulty":101,"share_id":"share-645","share_seq":645,"template_height":800020},{"accepted_at_ms":999986,"credit_policy":null,"job_id":"job-215","job_issued_at_ms":999994,"miner_id":"miner-6","network_difficulty":1000011,"ntime":1700000215,"order_key":"06:miner-6","p2mr_program_hex":"d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7","share_difficulty":2,"share_id":"share-215","share_seq":215,"template_height":800006},{"accepted_at_ms":1000060,"credit_policy":null,"job_id":"job-2427","job_issued_at_ms":999988,"miner_id":"miner-7","network_difficulty":1000013,"ntime":1700002427,"order_key":"07:miner-7","p2mr_program_hex":"7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b","share_difficulty":7,"share_id":"share-2427","share_seq":2427,"template_height":800075},{"accepted_at_ms":999988,"credit_policy":null,"job_id":"job-3826","job_issued_at_ms":999986,"miner_id":"miner-9","network_difficulty":1000001,"ntime":1700003826,"order_key":"09:miner-9","p2mr_program_hex":"f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2","share_difficulty":1000,"share_id":"share-3826","share_seq":3826,"template_height":800119},{"accepted_at_ms":999997,"credit_policy":"stale-grace","job_id":"job-3819","job_issued_at_ms":999986,"miner_id":"miner-2","network_difficulty":1000011,"ntime":1700003819,"order_key":"02:miner-2","p2mr_program_hex":"ebebebebebebebebebebebebebebebebebebebebebebebebebebebebebebebeb","share_difficulty":2,"share_id":"share-3819","share_seq":3819,"template_height":800119},{"accepted_at_ms":999972,"credit_policy":null,"job_id":"job-146","job_issued_at_ms":999975,"miner_id":"miner-3","network_difficulty":1000010,"ntime":1700000146,"order_key":"03:miner-3","p2mr_program_hex":"9292929292929292929292929292929292929292929292929292929292929292","share_difficulty":3,"share_id":"share-146","share_seq":146,"template_height":800004},{"accepted_at_ms":999991,"credit_policy":null,"job_id":"job-3379","job_issued_at_ms":999995,"miner_id":"miner-2","network_difficulty":1000013,"ntime":1700003379,"order_key":"02:miner-2","p2mr_program_hex":"3333333333333333333333333333333333333333333333333333333333333333","share_difficulty":1,"share_id":"share-3379","share_seq":3379,"template_height":800105},{"accepted_at_ms":999977,"credit_policy":null,"job_id":"job-3930","job_issued_at_ms":999977,"miner_id":"miner-3","network_difficulty":1000003,"ntime":1700003930,"order_key":"03:miner-3","p2mr_program_hex":"5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a","share_difficulty":1,"share_id":"share-3930","share_seq":3930,"template_height":800122},{"accepted_at_ms":1000230,"credit_policy":null,"job_id":"job-3731","job_issued_at_ms":1000428,"miner_id":"miner-2","network_difficulty":1000008,"ntime":1700003731,"order_key":"02:miner-2","p2mr_program_hex":"9393939393939393939393939393939393939393939393939393939393939393","share_difficulty":7,"share_id":"share-3731","share_seq":3731,"template_height":800116},{"accepted_at_ms":1000000,"credit_policy":"stale-grace","job_id":"job-1232","job_issued_at_ms":999973,"miner_id":"miner-0","network_difficulty":1000008,"ntime":1700001232,"order_key":"00:miner-0","p2mr_program_hex":"d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0","share_difficulty":1,"share_id":"share-1232","share_seq":1232,"template_height":800038},{"accepted_at_ms":999997,"credit_policy":null,"job_id":"job-2072","job_issued_at_ms":999982,"miner_id":"miner-4","network_difficulty":1000015,"ntime":1700002072,"order_key":"04:miner-4","p2mr_program_hex":"1818181818181818181818181818181818181818181818181818181818181818","share_difficulty":1000,"share_id":"share-2072","share_seq":2072,"template_height":800064},{"accepted_at_ms":999972,"credit_policy":null,"job_id":"job-1267","job_issued_at_ms":999993,"miner_id":"miner-2","network_difficulty":1000009,"ntime":1700001267,"order_key":"02:miner-2","p2mr_program_hex":"f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3","share_difficulty":7,"share_id":"share-1267","share_seq":1267,"template_height":800039},{"accepted_at_ms":1000258,"credit_policy":"stale-grace","job_id":"job-227","job_issued_at_ms":999973,"miner_id":"miner-7","network_difficulty":1000006,"ntime":1700000227,"order_key":"07:miner-7","p2mr_program_hex":"e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3","share_difficulty":19,"share_id":"share-227","share_seq":227,"template_height":800007},{"accepted_at_ms":1000057,"credit_policy":"stale-grace","job_id":"job-3426","job_issued_at_ms":1000223,"miner_id":"miner-5","network_difficulty":1000009,"ntime":1700003426,"order_key":"05:miner-5","p2mr_program_hex":"6262626262626262626262626262626262626262626262626262626262626262","share_difficulty":101,"share_id":"share-3426","share_seq":3426,"template_height":800107},{"accepted_at_ms":999980,"credit_policy":null,"job_id":"job-3807","job_issued_at_ms":999988,"miner_id":"miner-1","network_difficulty":1000016,"ntime":1700003807,"order_key":"01:miner-1","p2mr_program_hex":"dfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdf","share_difficulty":3,"share_id":"share-3807","share_seq":3807,"template_height":800118},{"accepted_at_ms":999986,"credit_policy":null,"job_id":"job-3842","job_issued_at_ms":999982,"miner_id":"miner-3","network_difficulty":1000000,"ntime":1700003842,"order_key":"03:miner-3","p2mr_program_hex":"0202020202020202020202020202020202020202020202020202020202020202","share_difficulty":101,"share_id":"share-3842","share_seq":3842,"template_height":800120},{"accepted_at_ms":999974,"credit_policy":null,"job_id":"job-251","job_issued_at_ms":999981,"miner_id":"miner-9","network_difficulty":1000013,"ntime":1700000251,"order_key":"09:miner-9","p2mr_program_hex":"fbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfb","share_difficulty":101,"share_id":"share-251","share_seq":251,"template_height":800007},{"accepted_at_ms":999973,"credit_policy":null,"job_id":"job-2080","job_issued_at_ms":1000051,"miner_id":"miner-1","network_difficulty":1000006,"ntime":1700002080,"order_key":"01:miner-1","p2mr_program_hex":"2020202020202020202020202020202020202020202020202020202020202020","share_difficulty":19,"share_id":"share-2080","share_seq":2080,"template_height":800065},{"accepted_at_ms":1000000,"credit_policy":"stale-grace","job_id":"job-3846","job_issued_at_ms":999996,"miner_id":"miner-7","network_difficulty":1000004,"ntime":1700003846,"order_key":"07:miner-7","p2mr_program_hex":"0606060606060606060606060606060606060606060606060606060606060606","share_difficulty":1000,"share_id":"share-3846","share_seq":3846,"template_height":800120},{"accepted_at_ms":999990,"credit_policy":null,"job_id":"job-2727","job_issued_at_ms":999970,"miner_id":"miner-10","network_difficulty":1000007,"ntime":1700002727,"order_key":"10:miner-10","p2mr_program_hex":"a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7","share_difficulty":3,"share_id":"share-2727","share_seq":2727,"template_height":800085},{"accepted_at_ms":999998,"credit_policy":null,"job_id":"job-1625","job_issued_at_ms":999985,"miner_id":"miner-8","network_difficulty":1000010,"ntime":1700001625,"order_key":"08:miner-8","p2mr_program_hex":"5959595959595959595959595959595959595959595959595959595959595959","share_difficulty":19,"share_id":"share-1625","share_seq":1625,"template_height":800050},{"accepted_at_ms":999993,"credit_policy":null,"job_id":"job-733","job_issued_at_ms":1000070,"miner_id":"miner-7","network_difficulty":1000002,"ntime":1700000733,"order_key":"07:miner-7","p2mr_program_hex":"dddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddd","share_difficulty":101,"share_id":"share-733","share_seq":733,"template_height":800022},{"accepted_at_ms":999994,"credit_policy":null,"job_id":"job-649","job_issued_at_ms":999979,"miner_id":"miner-0","network_difficulty":1000003,"ntime":1700000649,"order_key":"00:miner-0","p2mr_program_hex":"8989898989898989898989898989898989898989898989898989898989898989","share_difficulty":7,"share_id":"share-649","share_seq":649,"template_height":800020},{"accepted_at_ms":999973,"credit_policy":null,"job_id":"job-3579","job_issued_at_ms":999976,"miner_id":"miner-4","network_difficulty":1000009,"ntime":1700003579,"order_key":"04:miner-4","p2mr_program_hex":"fbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfb","share_difficulty":1000,"share_id":"share-3579","share_seq":3579,"template_height":800111},{"accepted_at_ms":999979,"credit_policy":null,"job_id":"job-3463","job_issued_at_ms":999970,"miner_id":"miner-9","network_difficulty":1000012,"ntime":1700003463,"order_key":"09:miner-9","p2mr_program_hex":"8787878787878787878787878787878787878787878787878787878787878787","share_difficulty":7,"share_id":"share-3463","share_seq":3463,"template_height":800108},{"accepted_at_ms":999974,"credit_policy":null,"job_id":"job-90","job_issued_at_ms":999990,"miner_id":"miner-2","network_difficulty":1000005,"ntime":1700000090,"order_key":"02:miner-2","p2mr_program_hex":"5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a","share_difficulty":1,"share_id":"share-90","share_seq":90,"template_height":800002},{"accepted_at_ms":999989,"credit_policy":null,"job_id":"job-437","job_issued_at_ms":999985,"miner_id":"miner-8","network_difficulty":1000012,"ntime":1700000437,"order_key":"08:miner-8","p2mr_program_hex":"b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5","share_difficulty":19,"share_id":"share-437","share_seq":437,"template_height":800013},{"accepted_at_ms":999980,"credit_policy":null,"job_id":"job-3882","job_issued_at_ms":999985,"miner_id":"miner-10","network_difficulty":1000006,"ntime":1700003882,"order_key":"10:miner-10","p2mr_program_hex":"2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a","share_difficulty":101,"share_id":"share-3882","share_seq":3882,"template_height":800121},{"accepted_at_ms":999991,"credit_policy":"stale-grace","job_id":"job-2010","job_issued_at_ms":999999,"miner_id":"miner-8","network_difficulty":1000004,"ntime":1700002010,"order_key":"08:miner-8","p2mr_program_hex":"dadadadadadadadadadadadadadadadadadadadadadadadadadadadadadadada","share_difficulty":7,"share_id":"share-2010","share_seq":2010,"template_height":800062},{"accepted_at_ms":999989,"credit_policy":null,"job_id":"job-720","job_issued_at_ms":999981,"miner_id":"miner-5","network_difficulty":1000006,"ntime":1700000720,"order_key":"05:miner-5","p2mr_program_hex":"d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0","share_difficulty":101,"share_id":"share-720","share_seq":720,"template_height":800022},{"accepted_at_ms":999978,"credit_policy":"stale-grace","job_id":"job-1635","job_issued_at_ms":999994,"miner_id":"miner-7","network_difficulty":1000003,"ntime":1700001635,"order_key":"07:miner-7","p2mr_program_hex":"6363636363636363636363636363636363636363636363636363636363636363","share_difficulty":101,"share_id":"share-1635","share_seq":1635,"template_height":800051},{"accepted_at_ms":999988,"credit_policy":null,"job_id":"job-3334","job_issued_at_ms":999978,"miner_id":"miner-1","network_difficulty":1000002,"ntime":1700003334,"order_key":"01:miner-1","p2mr_program_hex":"0606060606060606060606060606060606060606060606060606060606060606","share_difficulty":1000,"share_id":"share-3334","share_seq":3334,"template_height":800104},{"accepted_at_ms":999982,"credit_policy":null,"job_id":"job-701","job_issued_at_ms":999973,"miner_id":"miner-8","network_difficulty":1000004,"ntime":1700000701,"order_key":"08:miner-8","p2mr_program_hex":"bdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbd","share_difficulty":3,"share_id":"share-701","share_seq":701,"template_height":800021},{"accepted_at_ms":999972,"credit_policy":"stale-grace","job_id":"job-2156","job_issued_at_ms":999998,"miner_id":"miner-0","network_difficulty":1000014,"ntime":1700002156,"order_key":"00:miner-0","p2mr_program_hex":"6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c","share_difficulty":1,"share_id":"share-2156","share_seq":2156,"template_height":800067},{"accepted_at_ms":999982,"credit_policy":null,"job_id":"job-1948","job_issued_at_ms":999995,"miner_id":"miner-1","network_difficulty":1000010,"ntime":1700001948,"order_key":"01:miner-1","p2mr_program_hex":"9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c","share_difficulty":3,"share_id":"share-1948","share_seq":1948,"template_height":800060},{"accepted_at_ms":999973,"credit_policy":"stale-grace","job_id":"job-2329","job_issued_at_ms":999981,"miner_id":"miner-8","network_difficulty":1000000,"ntime":1700002329,"order_key":"08:miner-8","p2mr_program_hex":"1919191919191919191919191919191919191919191919191919191919191919","share_difficulty":1000,"share_id":"share-2329","share_seq":2329,"template_height":800072},{"accepted_at_ms":999991,"credit_policy":"stale-grace","job_id":"job-3298","job_issued_at_ms":1000000,"miner_id":"miner-9","network_difficulty":1000000,"ntime":1700003298,"order_key":"09:miner-9","p2mr_program_hex":"e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2","share_difficulty":3,"share_id":"share-3298","share_seq":3298,"template_height":800103},{"accepted_at_ms":1000323,"credit_policy":null,"job_id":"job-3064","job_issued_at_ms":999998,"miner_id":"miner-6","network_difficulty":1000004,"ntime":1700003064,"order_key":"06:miner-6","p2mr_program_hex":"f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8","share_difficulty":101,"share_id":"share-3064","share_seq":3064,"template_height":800095},{"accepted_at_ms":999974,"credit_policy":null,"job_id":"job-1951","job_issued_at_ms":999979,"miner_id":"miner-4","network_difficulty":1000013,"ntime":1700001951,"order_key":"04:miner-4","p2mr_program_hex":"9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f","share_difficulty":1,"share_id":"share-1951","share_seq":1951,"template_height":800060},{"accepted_at_ms":999976,"credit_policy":"stale-grace","job_id":"job-739","job_issued_at_ms":999978,"miner_id":"miner-2","network_difficulty":1000008,"ntime":1700000739,"order_key":"02:miner-2","p2mr_program_hex":"e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3","share_difficulty":19,"share_id":"share-739","share_seq":739,"template_height":800023},{"accepted_at_ms":999982,"credit_policy":null,"job_id":"job-2021","job_issued_at_ms":999977,"miner_id":"miner-8","network_difficulty":1000015,"ntime":1700002021,"order_key":"08:miner-8","p2mr_program_hex":"e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5","share_difficulty":3,"share_id":"share-2021","share_seq":2021,"template_height":800063},{"accepted_at_ms":999988,"credit_policy":null,"job_id":"job-2699","job_issued_at_ms":999983,"miner_id":"miner-4","network_difficulty":1000013,"ntime":1700002699,"order_key":"04:miner-4","p2mr_program_hex":"8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b","share_difficulty":101,"share_id":"share-2699","share_seq":2699,"template_height":800084},{"accepted_at_ms":999994,"credit_policy":null,"job_id":"job-2460","job_issued_at_ms":999973,"miner_id":"miner-7","network_difficulty":1000012,"ntime":1700002460,"order_key":"07:miner-7","p2mr_program_hex":"9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c","share_difficulty":1000,"share_id":"share-2460","share_seq":2460,"template_height":800076},{"accepted_at_ms":999993,"credit_policy":"stale-grace","job_id":"job-2775","job_issued_at_ms":999977,"miner_id":"miner-3","network_difficulty":1000004,"ntime":1700002775,"order_key":"03:miner-3","p2mr_program_hex":"d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7","share_difficulty":1000,"share_id":"share-2775","share_seq":2775,"template_height":800086},{"accepted_at_ms":999991,"credit_policy":"stale-grace","job_id":"job-2789","job_issued_at_ms":999990,"miner_id":"miner-6","network_difficulty":1000001,"ntime":1700002789,"order_key":"06:miner-6","p2mr_program_hex":"e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5","share_difficulty":7,"share_id":"share-2789","share_seq":2789,"template_height":800087},{"accepted_at_ms":999992,"credit_policy":null,"job_id":"job-1034","job_issued_at_ms":999986,"miner_id":"miner-0","network_difficulty":1000014,"ntime":1700001034,"order_key":"00:miner-0","p2mr_program_hex":"0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a","share_difficulty":2,"share_id":"share-1034","share_seq":1034,"template_height":800032},{"accepted_at_ms":999980,"credit_policy":null,"job_id":"job-364","job_issued_at_ms":999987,"miner_id":"miner-1","network_difficulty":1000007,"ntime":1700000364,"order_key":"01:miner-1","p2mr_program_hex":"6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c","share_difficulty":3,"share_id":"share-364","share_seq":364,"template_height":800011},{"accepted_at_ms":999993,"credit_policy":"stale-grace","job_id":"job-2360","job_issued_at_ms":999980,"miner_id":"miner-6","network_difficulty":1000014,"ntime":1700002360,"order_key":"06:miner-6","p2mr_program_hex":"3838383838383838383838383838383838383838383838383838383838383838","share_difficulty":1000,"share_id":"share-2360","share_seq":2360,"template_height":800073},{"accepted_at_ms":999973,"credit_policy":"stale-grace","job_id":"job-303","job_issued_at_ms":999972,"miner_id":"miner-6","network_difficulty":1000014,"ntime":1700000303,"order_key":"06:miner-6","p2mr_program_hex":"2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f","share_difficulty":7,"share_id":"share-303","share_seq":303,"template_height":800009},{"accepted_at_ms":999985,"credit_policy":"stale-grace","job_id":"job-3724","job_issued_at_ms":999987,"miner_id":"miner-6","network_difficulty":1000001,"ntime":1700003724,"order_key":"06:miner-6","p2mr_program_hex":"8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c","share_difficulty":2,"share_id":"share-3724","share_seq":3724,"template_height":800116},{"accepted_at_ms":999989,"credit_policy":"stale-grace","job_id":"job-1373","job_issued_at_ms":999996,"miner_id":"miner-9","network_difficulty":1000013,"ntime":1700001373,"order_key":"09:miner-9","p2mr_program_hex":"5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d","share_difficulty":19,"share_id":"share-1373","share_seq":1373,"template_height":800042},{"accepted_at_ms":999988,"credit_policy":null,"job_id":"job-3117","job_issued_at_ms":999976,"miner_id":"miner-4","network_difficulty":1000006,"ntime":1700003117,"order_key":"04:miner-4","p2mr_program_hex":"2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d","share_difficulty":19,"share_id":"share-3117","share_seq":3117,"template_height":800097},{"accepted_at_ms":999970,"credit_policy":null,"job_id":"job-503","job_issued_at_ms":999997,"miner_id":"miner-8","network_difficulty":1000010,"ntime":1700000503,"order_key":"08:miner-8","p2mr_program_hex":"f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7","share_difficulty":7,"share_id":"share-503","share_seq":503,"template_height":800015},{"accepted_at_ms":999977,"credit_policy":null,"job_id":"job-1328","job_issued_at_ms":999970,"miner_id":"miner-8","network_difficulty":1000002,"ntime":1700001328,"order_key":"08:miner-8","p2mr_program_hex":"3030303030303030303030303030303030303030303030303030303030303030","share_difficulty":3,"share_id":"share-1328","share_seq":1328,"template_height":800041},{"accepted_at_ms":999991,"credit_policy":null,"job_id":"job-1960","job_issued_at_ms":999992,"miner_id":"miner-2","network_difficulty":1000005,"ntime":1700001960,"order_key":"02:miner-2","p2mr_program_hex":"a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8","share_difficulty":2,"share_id":"share-1960","share_seq":1960,"template_height":800061},{"accepted_at_ms":999999,"credit_policy":null,"job_id":"job-1070","job_issued_at_ms":999989,"miner_id":"miner-3","network_difficulty":1000016,"ntime":1700001070,"order_key":"03:miner-3","p2mr_program_hex":"2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e","share_difficulty":19,"share_id":"share-1070","share_seq":1070,"template_height":800033},{"accepted_at_ms":1000242,"credit_policy":"stale-grace","job_id":"job-71","job_issued_at_ms":999989,"miner_id":"miner-5","network_difficulty":1000003,"ntime":1700000071,"order_key":"05:miner-5","p2mr_program_hex":"4747474747474747474747474747474747474747474747474747474747474747","share_difficulty":3,"share_id":"share-71","share_seq":71,"template_height":800002},{"accepted_at_ms":999987,"credit_policy":null,"job_id":"job-2502","job_issued_at_ms":999986,"miner_id":"miner-5","network_difficulty":1000003,"ntime":1700002502,"order_key":"05:miner-5","p2mr_program_hex":"c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6","share_difficulty":7,"share_id":"share-2502","share_seq":2502,"template_height":800078},{"accepted_at_ms":999986,"credit_policy":null,"job_id":"job-3277","job_issued_at_ms":999993,"miner_id":"miner-10","network_difficulty":1000013,"ntime":1700003277,"order_key":"10:miner-10","p2mr_program_hex":"cdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcd","share_difficulty":19,"share_id":"share-3277","share_seq":3277,"template_height":800102},{"accepted_at_ms":1000000,"credit_policy":null,"job_id":"job-2832","job_issued_at_ms":999973,"miner_id":"miner-5","network_difficulty":1000010,"ntime":1700002832,"order_key":"05:miner-5","p2mr_program_hex":"1010101010101010101010101010101010101010101010101010101010101010","share_difficulty":1,"share_id":"share-2832","share_seq":2832,"template_height":800088},{"accepted_at_ms":999986,"credit_policy":null,"job_id":"job-1209","job_issued_at_ms":999975,"miner_id":"miner-10","network_difficulty":1000002,"ntime":1700001209,"order_key":"10:miner-10","p2mr_program_hex":"b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9","share_difficulty":101,"share_id":"share-1209","share_seq":1209,"template_height":800037},{"accepted_at_ms":999999,"credit_policy":null,"job_id":"job-2061","job_issued_at_ms":999977,"miner_id":"miner-4","network_difficulty":1000004,"ntime":1700002061,"order_key":"04:miner-4","p2mr_program_hex":"0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d","share_difficulty":2,"share_id":"share-2061","share_seq":2061,"template_height":800064},{"accepted_at_ms":999995,"credit_policy":null,"job_id":"job-2607","job_issued_at_ms":999992,"miner_id":"miner-0","network_difficulty":1000006,"ntime":1700002607,"order_key":"00:miner-0","p2mr_program_hex":"2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f","share_difficulty":2,"share_id":"share-2607","share_seq":2607,"template_height":800081},{"accepted_at_ms":999976,"credit_policy":null,"job_id":"job-614","job_issued_at_ms":999998,"miner_id":"miner-9","network_difficulty":1000002,"ntime":1700000614,"order_key":"09:miner-9","p2mr_program_hex":"6666666666666666666666666666666666666666666666666666666666666666","share_difficulty":101,"share_id":"share-614","share_seq":614,"template_height":800019},{"accepted_at_ms":999994,"credit_policy":null,"job_id":"job-3752","job_issued_at_ms":999992,"miner_id":"miner-1","network_difficulty":1000012,"ntime":1700003752,"order_key":"01:miner-1","p2mr_program_hex":"a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8","share_difficulty":101,"share_id":"share-3752","share_seq":3752,"template_height":800117},{"accepted_at_ms":999995,"credit_policy":null,"job_id":"job-3373","job_issued_at_ms":999975,"miner_id":"miner-7","network_difficulty":1000007,"ntime":1700003373,"order_key":"07:miner-7","p2mr_program_hex":"2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d","share_difficulty":3,"share_id":"share-3373","share_seq":3373,"template_height":800105},{"accepted_at_ms":999971,"credit_policy":"stale-grace","job_id":"job-2989","job_issued_at_ms":999992,"miner_id":"miner-8","network_difficulty":1000014,"ntime":1700002989,"order_key":"08:miner-8","p2mr_program_hex":"adadadadadadadadadadadadadadadadadadadadadadadadadadadadadadadad","share_difficulty":101,"share_id":"share-2989","share_seq":2989,"template_height":800093},{"accepted_at_ms":999979,"credit_policy":null,"job_id":"job-3154","job_issued_at_ms":999978,"miner_id":"miner-8","network_difficulty":1000009,"ntime":1700003154,"order_key":"08:miner-8","p2mr_program_hex":"5252525252525252525252525252525252525252525252525252525252525252","share_difficulty":3,"share_id":"share-3154","share_seq":3154,"template_height":800098},{"accepted_at_ms":999989,"credit_policy":"stale-grace","job_id":"job-3335","job_issued_at_ms":999974,"miner_id":"miner-2","network_difficulty":1000003,"ntime":1700003335,"order_key":"02:miner-2","p2mr_program_hex":"0707070707070707070707070707070707070707070707070707070707070707","share_difficulty":2,"share_id":"share-3335","share_seq":3335,"template_height":800104},{"accepted_at_ms":999999,"credit_policy":null,"job_id":"job-2982","job_issued_at_ms":999985,"miner_id":"miner-1","network_difficulty":1000007,"ntime":1700002982,"order_key":"01:miner-1","p2mr_program_hex":"a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6","share_difficulty":1000,"share_id":"share-2982","share_seq":2982,"template_height":800093},{"accepted_at_ms":999985,"credit_policy":null,"job_id":"job-1737","job_issued_at_ms":999975,"miner_id":"miner-10","network_difficulty":1000003,"ntime":1700001737,"order_key":"10:miner-10","p2mr_program_hex":"c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9","share_difficulty":101,"share_id":"share-1737","share_seq":1737,"template_height":800054},{"accepted_at_ms":999984,"credit_policy":null,"job_id":"job-845","job_issued_at_ms":999993,"miner_id":"miner-9","network_difficulty":1000012,"ntime":1700000845,"order_key":"09:miner-9","p2mr_program_hex":"4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d","share_difficulty":101,"share_id":"share-845","share_seq":845,"template_height":800026},{"accepted_at_ms":999976,"credit_policy":null,"job_id":"job-2758","job_issued_at_ms":999988,"miner_id":"miner-8","network_difficulty":1000004,"ntime":1700002758,"order_key":"08:miner-8","p2mr_program_hex":"c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6","share_difficulty":1000,"share_id":"share-2758","share_seq":2758,"template_height":800086},{"accepted_at_ms":999997,"credit_policy":"stale-grace","job_id":"job-3376","job_issued_at_ms":999998,"miner_id":"miner-10","network_difficulty":1000010,"ntime":1700003376,"order_key":"10:miner-10","p2mr_program_hex":"3030303030303030303030303030303030303030303030303030303030303030","share_difficulty":7,"share_id":"share-3376","share_seq":3376,"template_height":800105},{"accepted_at_ms":999973,"credit_policy":null,"job_id":"job-2264","job_issued_at_ms":999976,"miner_id":"miner-9","network_difficulty":1000003,"ntime":1700002264,"order_key":"09:miner-9","p2mr_program_hex":"d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8","share_difficulty":101,"share_id":"share-2264","share_seq":2264,"template_height":800070},{"accepted_at_ms":999975,"credit_policy":null,"job_id":"job-3990","job_issued_at_ms":999993,"miner_id":"miner-8","network_difficulty":1000012,"ntime":1700003990,"order_key":"08:miner-8","p2mr_program_hex":"9696969696969696969696969696969696969696969696969696969696969696","share_difficulty":1000,"share_id":"share-3990","share_seq":3990,"template_height":800124},{"accepted_at_ms":999983,"credit_policy":null,"job_id":"job-475","job_issued_at_ms":999987,"miner_id":"miner-2","network_difficulty":1000016,"ntime":1700000475,"order_key":"02:miner-2","p2mr_program_hex":"dbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdb","share_difficulty":19,"share_id":"share-475","share_seq":475,"template_height":800014},{"accepted_at_ms":999970,"credit_policy":null,"job_id":"job-167","job_issued_at_ms":999992,"miner_id":"miner-2","network_difficulty":1000014,"ntime":1700000167,"order_key":"02:miner-2","p2mr_program_hex":"a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7","share_difficulty":7,"share_id":"share-167","share_seq":167,"template_height":800005},{"accepted_at_ms":999998,"credit_policy":"stale-grace","job_id":"job-568","job_issued_at_ms":999970,"miner_id":"miner-7","network_difficulty":1000007,"ntime":1700000568,"order_key":"07:miner-7","p2mr_program_hex":"3838383838383838383838383838383838383838383838383838383838383838","share_difficulty":7,"share_id":"share-568","share_seq":568,"template_height":800017},{"accepted_at_ms":999986,"credit_policy":null,"job_id":"job-2633","job_issued_at_ms":999984,"miner_id":"miner-4","network_difficulty":1000015,"ntime":1700002633,"order_key":"04:miner-4","p2mr_program_hex":"4949494949494949494949494949494949494949494949494949494949494949","share_difficulty":3,"share_id":"share-2633","share_seq":2633,"template_height":800082},{"accepted_at_ms":1000000,"credit_policy":null,"job_id":"job-3149","job_issued_at_ms":999979,"miner_id":"miner-3","network_difficulty":1000004,"ntime":1700003149,"order_key":"03:miner-3","p2mr_program_hex":"4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d","share_difficulty":101,"share_id":"share-3149","share_seq":3149,"template_height":800098},{"accepted_at_ms":999972,"credit_policy":null,"job_id":"job-334","job_issued_at_ms":999976,"miner_id":"miner-4","network_difficulty":1000011,"ntime":1700000334,"order_key":"04:miner-4","p2mr_program_hex":"4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e","share_difficulty":101,"share_id":"share-334","share_seq":334,"template_height":800010},{"accepted_at_ms":999970,"credit_policy":"stale-grace","job_id":"job-98","job_issued_at_ms":1000000,"miner_id":"miner-10","network_difficulty":1000013,"ntime":1700000098,"order_key":"10:miner-10","p2mr_program_hex":"6262626262626262626262626262626262626262626262626262626262626262","share_difficulty":2,"share_id":"share-98","share_seq":98,"template_height":800003},{"accepted_at_ms":1000340,"credit_policy":null,"job_id":"job-365","job_issued_at_ms":999988,"miner_id":"miner-2","network_difficulty":1000008,"ntime":1700000365,"order_key":"02:miner-2","p2mr_program_hex":"6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d","share_difficulty":3,"share_id":"share-365","share_seq":365,"template_height":800011},{"accepted_at_ms":999982,"credit_policy":null,"job_id":"job-3539","job_issued_at_ms":999996,"miner_id":"miner-8","network_difficulty":1000003,"ntime":1700003539,"order_key":"08:miner-8","p2mr_program_hex":"d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3","share_difficulty":1,"share_id":"share-3539","share_seq":3539,"template_height":800110},{"accepted_at_ms":999981,"credit_policy":null,"job_id":"job-962","job_issued_at_ms":999994,"miner_id":"miner-5","network_difficulty":1000010,"ntime":1700000962,"order_key":"05:miner-5","p2mr_program_hex":"c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2","share_difficulty":1000,"share_id":"share-962","share_seq":962,"template_height":800030},{"accepted_at_ms":999977,"credit_policy":null,"job_id":"job-3665","job_issued_at_ms":999973,"miner_id":"miner-2","network_difficulty":1000010,"ntime":1700003665,"order_key":"02:miner-2","p2mr_program_hex":"5151515151515151515151515151515151515151515151515151515151515151","share_difficulty":1000,"share_id":"share-3665","share_seq":3665,"template_height":800114},{"accepted_at_ms":999978,"credit_policy":null,"job_id":"job-329","job_issued_at_ms":999978,"miner_id":"miner-10","network_difficulty":1000006,"ntime":1700000329,"order_key":"10:miner-10","p2mr_program_hex":"4949494949494949494949494949494949494949494949494949494949494949","share_difficulty":2,"share_id":"share-329","share_seq":329,"template_height":800010},{"accepted_at_ms":999989,"credit_policy":null,"job_id":"job-2070","job_issued_at_ms":1000000,"miner_id":"miner-2","network_difficulty":1000013,"ntime":1700002070,"order_key":"02:miner-2","p2mr_program_hex":"1616161616161616161616161616161616161616161616161616161616161616","share_difficulty":3,"share_id":"share-2070","share_seq":2070,"template_height":800064},{"accepted_at_ms":999981,"credit_policy":"stale-grace","job_id":"job-2351","job_issued_at_ms":999974,"miner_id":"miner-8","network_difficulty":1000005,"ntime":1700002351,"order_key":"08:miner-8","p2mr_program_hex":"2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f","share_difficulty":1000,"share_id":"share-2351","share_seq":2351,"template_height":800073},{"accepted_at_ms":999972,"credit_policy":"stale-grace","job_id":"job-157","job_issued_at_ms":999982,"miner_id":"miner-3","network_difficulty":1000004,"ntime":1700000157,"order_key":"03:miner-3","p2mr_program_hex":"9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d","share_difficulty":7,"share_id":"share-157","share_seq":157,"template_height":800004},{"accepted_at_ms":999973,"credit_policy":"stale-grace","job_id":"job-873","job_issued_at_ms":999986,"miner_id":"miner-4","network_difficulty":1000006,"ntime":1700000873,"order_key":"04:miner-4","p2mr_program_hex":"6969696969696969696969696969696969696969696969696969696969696969","share_difficulty":101,"share_id":"share-873","share_seq":873,"template_height":800027},{"accepted_at_ms":999976,"credit_policy":null,"job_id":"job-17","job_issued_at_ms":999997,"miner_id":"miner-6","network_difficulty":1000000,"ntime":1700000017,"order_key":"06:miner-6","p2mr_program_hex":"1111111111111111111111111111111111111111111111111111111111111111","share_difficulty":1,"share_id":"share-17","share_seq":17,"template_height":800000},{"accepted_at_ms":999991,"credit_policy":null,"job_id":"job-3381","job_issued_at_ms":999987,"miner_id":"miner-4","network_difficulty":1000015,"ntime":1700003381,"order_key":"04:miner-4","p2mr_program_hex":"3535353535353535353535353535353535353535353535353535353535353535","share_difficulty":1,"share_id":"share-3381","share_seq":3381,"template_height":800105},{"accepted_at_ms":999970,"credit_policy":null,"job_id":"job-1347","job_issued_at_ms":999986,"miner_id":"miner-5","network_difficulty":1000004,"ntime":1700001347,"order_key":"05:miner-5","p2mr_program_hex":"4343434343434343434343434343434343434343434343434343434343434343","share_difficulty":1,"share_id":"share-1347","share_seq":1347,"template_height":800042},{"accepted_at_ms":999980,"credit_policy":null,"job_id":"job-3648","job_issued_at_ms":999999,"miner_id":"miner-7","network_difficulty":1000010,"ntime":1700003648,"order_key":"07:miner-7","p2mr_program_hex":"4040404040404040404040404040404040404040404040404040404040404040","share_difficulty":1,"share_id":"share-3648","share_seq":3648,"template_height":800114},{"accepted_at_ms":1000000,"credit_policy":"stale-grace","job_id":"job-2790","job_issued_at_ms":999975,"miner_id":"miner-7","network_difficulty":1000002,"ntime":1700002790,"order_key":"07:miner-7","p2mr_program_hex":"e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6","share_difficulty":101,"share_id":"share-2790","share_seq":2790,"template_height":800087},{"accepted_at_ms":999988,"credit_policy":"stale-grace","job_id":"job-2969","job_issued_at_ms":999980,"miner_id":"miner-10","network_difficulty":1000011,"ntime":1700002969,"order_key":"10:miner-10","p2mr_program_hex":"9999999999999999999999999999999999999999999999999999999999999999","share_difficulty":3,"share_id":"share-2969","share_seq":2969,"template_height":800092},{"accepted_at_ms":999996,"credit_policy":null,"job_id":"job-1410","job_issued_at_ms":999989,"miner_id":"miner-2","network_difficulty":1000016,"ntime":1700001410,"order_key":"02:miner-2","p2mr_program_hex":"8282828282828282828282828282828282828282828282828282828282828282","share_difficulty":3,"share_id":"share-1410","share_seq":1410,"template_height":800044},{"accepted_at_ms":999980,"credit_policy":null,"job_id":"job-1915","job_issued_at_ms":999989,"miner_id":"miner-1","network_difficulty":1000011,"ntime":1700001915,"order_key":"01:miner-1","p2mr_program_hex":"7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b","share_difficulty":1,"share_id":"share-1915","share_seq":1915,"template_height":800059},{"accepted_at_ms":999978,"credit_policy":null,"job_id":"job-3742","job_issued_at_ms":999992,"miner_id":"miner-2","network_difficulty":1000002,"ntime":1700003742,"order_key":"02:miner-2","p2mr_program_hex":"9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e","share_difficulty":7,"share_id":"share-3742","share_seq":3742,"template_height":800116},{"accepted_at_ms":999998,"credit_policy":null,"job_id":"job-2824","job_issued_at_ms":999996,"miner_id":"miner-8","network_difficulty":1000002,"ntime":1700002824,"order_key":"08:miner-8","p2mr_program_hex":"0808080808080808080808080808080808080808080808080808080808080808","share_difficulty":19,"share_id":"share-2824","share_seq":2824,"template_height":800088},{"accepted_at_ms":999995,"credit_policy":"stale-grace","job_id":"job-2161","job_issued_at_ms":999995,"miner_id":"miner-5","network_difficulty":1000002,"ntime":1700002161,"order_key":"05:miner-5","p2mr_program_hex":"7171717171717171717171717171717171717171717171717171717171717171","share_difficulty":3,"share_id":"share-2161","share_seq":2161,"template_height":800067},{"accepted_at_ms":999971,"credit_policy":"stale-grace","job_id":"job-3631","job_issued_at_ms":999990,"miner_id":"miner-1","network_difficulty":1000010,"ntime":1700003631,"order_key":"01:miner-1","p2mr_program_hex":"2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f","share_difficulty":1,"share_id":"share-3631","share_seq":3631,"template_height":800113},{"accepted_at_ms":999993,"credit_policy":null,"job_id":"job-217","job_issued_at_ms":999992,"miner_id":"miner-8","network_difficulty":1000013,"ntime":1700000217,"order_key":"08:miner-8","p2mr_program_hex":"d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9","share_difficulty":3,"share_id":"share-217","share_seq":217,"template_height":800006},{"accepted_at_ms":999995,"credit_policy":null,"job_id":"job-1785","job_issued_at_ms":999974,"miner_id":"miner-3","network_difficulty":1000000,"ntime":1700001785,"order_key":"03:miner-3","p2mr_program_hex":"f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9","share_difficulty":1000,"share_id":"share-1785","share_seq":1785,"template_height":800055},{"accepted_at_ms":999995,"credit_policy":null,"job_id":"job-120","job_issued_at_ms":999980,"miner_id":"miner-10","network_difficulty":1000001,"ntime":1700000120,"order_key":"10:miner-10","p2mr_program_hex":"7878787878787878787878787878787878787878787878787878787878787878","share_difficulty":2,"share_id":"share-120","share_seq":120,"template_height":800003},{"accepted_at_ms":999997,"credit_policy":null,"job_id":"job-1139","job_issued_at_ms":999982,"miner_id":"miner-6","network_difficulty":1000000,"ntime":1700001139,"order_key":"06:miner-6","p2mr_program_hex":"7373737373737373737373737373737373737373737373737373737373737373","share_difficulty":3,"share_id":"share-1139","share_seq":1139,"template_height":800035},{"accepted_at_ms":999981,"credit_policy":null,"job_id":"job-1597","job_issued_at_ms":999999,"miner_id":"miner-2","network_difficulty":1000016,"ntime":1700001597,"order_key":"02:miner-2","p2mr_program_hex":"3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d","share_difficulty":3,"share_id":"share-1597","share_seq":1597,"template_height":800049},{"accepted_at_ms":999991,"credit_policy":"stale-grace","job_id":"job-1142","job_issued_at_ms":999981,"miner_id":"miner-9","network_difficulty":1000003,"ntime":1700001142,"order_key":"09:miner-9","p2mr_program_hex":"7676767676767676767676767676767676767676767676767676767676767676","share_difficulty":1000,"share_id":"share-1142","share_seq":1142,"template_height":800035},{"accepted_at_ms":999983,"credit_policy":"stale-grace","job_id":"job-3556","job_issued_at_ms":999977,"miner_id":"miner-3","network_difficulty":1000003,"ntime":1700003556,"order_key":"03:miner-3","p2mr_program_hex":"e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4","share_difficulty":1000,"share_id":"share-3556","share_seq":3556,"template_height":800111},{"accepted_at_ms":999996,"credit_policy":"stale-grace","job_id":"job-2383","job_issued_at_ms":999974,"miner_id":"miner-7","network_difficulty":1000003,"ntime":1700002383,"order_key":"07:miner-7","p2mr_program_hex":"4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f","share_difficulty":2,"share_id":"share-2383","share_seq":2383,"template_height":800074},{"accepted_at_ms":999987,"credit_policy":"stale-grace","job_id":"job-1996","job_issued_at_ms":999971,"miner_id":"miner-5","network_difficulty":1000007,"ntime":1700001996,"order_key":"05:miner-5","p2mr_program_hex":"cccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccc","share_difficulty":1,"share_id":"share-1996","share_seq":1996,"template_height":800062},{"accepted_at_ms":999973,"credit_policy":"stale-grace","job_id":"job-268","job_issued_at_ms":999989,"miner_id":"miner-4","network_difficulty":1000013,"ntime":1700000268,"order_key":"04:miner-4","p2mr_program_hex":"0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c","share_difficulty":3,"share_id":"share-268","share_seq":268,"template_height":800008},{"accepted_at_ms":1000000,"credit_policy":null,"job_id":"job-2895","job_issued_at_ms":999987,"miner_id":"miner-2","network_difficulty":1000005,"ntime":1700002895,"order_key":"02:miner-2","p2mr_program_hex":"4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f","share_difficulty":1000,"share_id":"share-2895","share_seq":2895,"template_height":800090},{"accepted_at_ms":999977,"credit_policy":null,"job_id":"job-637","job_issued_at_ms":1000000,"miner_id":"miner-10","network_difficulty":1000008,"ntime":1700000637,"order_key":"10:miner-10","p2mr_program_hex":"7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d","share_difficulty":101,"share_id":"share-637","share_seq":637,"template_height":800019},{"accepted_at_ms":999988,"credit_policy":null,"job_id":"job-3602","job_issued_at_ms":999973,"miner_id":"miner-5","network_difficulty":1000015,"ntime":1700003602,"order_key":"05:miner-5","p2mr_program_hex":"1212121212121212121212121212121212121212121212121212121212121212","share_difficulty":19,"share_id":"share-3602","share_seq":3602,"template_height":800112},{"accepted_at_ms":999984,"credit_policy":null,"job_id":"job-2634","job_issued_at_ms":999978,"miner_id":"miner-5","network_difficulty":1000016,"ntime":1700002634,"order_key":"05:miner-5","p2mr_program_hex":"4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a","share_difficulty":3,"share_id":"share-2634","share_seq":2634,"template_height":800082},{"accepted_at_ms":1000258,"credit_policy":null,"job_id":"job-3572","job_issued_at_ms":999998,"miner_id":"miner-8","network_difficulty":1000002,"ntime":1700003572,"order_key":"08:miner-8","p2mr_program_hex":"f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4","share_difficulty":3,"share_id":"share-3572","share_seq":3572,"template_height":800111},{"accepted_at_ms":999984,"credit_policy":null,"job_id":"job-1061","job_issued_at_ms":1000000,"miner_id":"miner-5","network_difficulty":1000007,"ntime":1700001061,"order_key":"05:miner-5","p2mr_program_hex":"2525252525252525252525252525252525252525252525252525252525252525","share_difficulty":19,"share_id":"share-1061","share_seq":1061,"template_height":800033},{"accepted_at_ms":999981,"credit_policy":null,"job_id":"job-555","job_issued_at_ms":999988,"miner_id":"miner-5","network_difficulty":1000011,"ntime":1700000555,"order_key":"05:miner-5","p2mr_program_hex":"2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b","share_difficulty":1,"share_id":"share-555","share_seq":555,"template_height":800017},{"accepted_at_ms":999994,"credit_policy":"stale-grace","job_id":"job-346","job_issued_at_ms":999992,"miner_id":"miner-5","network_difficulty":1000006,"ntime":1700000346,"order_key":"05:miner-5","p2mr_program_hex":"5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a","share_difficulty":101,"share_id":"share-346","share_seq":346,"template_height":800010},{"accepted_at_ms":999990,"credit_policy":null,"job_id":"job-2168","job_issued_at_ms":999975,"miner_id":"miner-1","network_difficulty":1000009,"ntime":1700002168,"order_key":"01:miner-1","p2mr_program_hex":"7878787878787878787878787878787878787878787878787878787878787878","share_difficulty":3,"share_id":"share-2168","share_seq":2168,"template_height":800067},{"accepted_at_ms":999994,"credit_policy":"stale-grace","job_id":"job-3627","job_issued_at_ms":999996,"miner_id":"miner-8","network_difficulty":1000006,"ntime":1700003627,"order_key":"08:miner-8","p2mr_program_hex":"2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b","share_difficulty":2,"share_id":"share-3627","share_seq":3627,"template_height":800113},{"accepted_at_ms":999990,"credit_policy":null,"job_id":"job-2112","job_issued_at_ms":1000150,"miner_id":"miner-0","network_difficulty":1000004,"ntime":1700002112,"order_key":"00:miner-0","p2mr_program_hex":"4040404040404040404040404040404040404040404040404040404040404040","share_difficulty":1,"share_id":"share-2112","share_seq":2112,"template_height":800066},{"accepted_at_ms":999999,"credit_policy":null,"job_id":"job-1489","job_issued_at_ms":999990,"miner_id":"miner-4","network_difficulty":1000010,"ntime":1700001489,"order_key":"04:miner-4","p2mr_program_hex":"d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1","share_difficulty":7,"share_id":"share-1489","share_seq":1489,"template_height":800046},{"accepted_at_ms":999998,"credit_policy":null,"job_id":"job-3232","job_issued_at_ms":999982,"miner_id":"miner-9","network_difficulty":1000002,"ntime":1700003232,"order_key":"09:miner-9","p2mr_program_hex":"a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0","share_difficulty":101,"share_id":"share-3232","share_seq":3232,"template_height":800101},{"accepted_at_ms":999988,"credit_policy":null,"job_id":"job-87","job_issued_at_ms":999997,"miner_id":"miner-10","network_difficulty":1000002,"ntime":1700000087,"order_key":"10:miner-10","p2mr_program_hex":"5757575757575757575757575757575757575757575757575757575757575757","share_difficulty":1000,"share_id":"share-87","share_seq":87,"template_height":800002},{"accepted_at_ms":999985,"credit_policy":null,"job_id":"job-204","job_issued_at_ms":999986,"miner_id":"miner-6","network_difficulty":1000000,"ntime":1700000204,"order_key":"06:miner-6","p2mr_program_hex":"cccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccc","share_difficulty":2,"share_id":"share-204","share_seq":204,"template_height":800006},{"accepted_at_ms":999990,"credit_policy":null,"job_id":"job-732","job_issued_at_ms":999973,"miner_id":"miner-6","network_difficulty":1000001,"ntime":1700000732,"order_key":"06:miner-6","p2mr_program_hex":"dcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdc","share_difficulty":2,"share_id":"share-732","share_seq":732,"template_height":800022},{"accepted_at_ms":999983,"credit_policy":null,"job_id":"job-511","job_issued_at_ms":999989,"miner_id":"miner-5","network_difficulty":1000001,"ntime":1700000511,"order_key":"05:miner-5","p2mr_program_hex":"ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff","share_difficulty":19,"share_id":"share-511","share_seq":511,"template_height":800015},{"accepted_at_ms":999973,"credit_policy":null,"job_id":"job-1898","job_issued_at_ms":999992,"miner_id":"miner-6","network_difficulty":1000011,"ntime":1700001898,"order_key":"06:miner-6","p2mr_program_hex":"6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a","share_difficulty":1000,"share_id":"share-1898","share_seq":1898,"template_height":800059},{"accepted_at_ms":999997,"credit_policy":null,"job_id":"job-621","job_issued_at_ms":999989,"miner_id":"miner-5","network_difficulty":1000009,"ntime":1700000621,"order_key":"05:miner-5","p2mr_program_hex":"6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d","share_difficulty":101,"share_id":"share-621","share_seq":621,"template_height":800019},{"accepted_at_ms":999989,"credit_policy":null,"job_id":"job-3533","job_issued_at_ms":999986,"miner_id":"miner-2","network_difficulty":1000014,"ntime":1700003533,"order_key":"02:miner-2","p2mr_program_hex":"cdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcd","share_difficulty":101,"share_id":"share-3533","share_seq":3533,"template_height":800110},{"accepted_at_ms":999988,"credit_policy":null,"job_id":"job-743","job_issued_at_ms":999971,"miner_id":"miner-6","network_difficulty":1000012,"ntime":1700000743,"order_key":"06:miner-6","p2mr_program_hex":"e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7","share_difficulty":19,"share_id":"share-743","share_seq":743,"template_height":800023},{"accepted_at_ms":999999,"credit_policy":null,"job_id":"job-688","job_issued_at_ms":1000000,"miner_id":"miner-6","network_difficulty":1000008,"ntime":1700000688,"order_key":"06:miner-6","p2mr_program_hex":"b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0","share_difficulty":2,"share_id":"share-688","share_seq":688,"template_height":800021},{"accepted_at_ms":999987,"credit_policy":null,"job_id":"job-1411","job_issued_at_ms":999990,"miner_id":"miner-3","network_difficulty":1000000,"ntime":1700001411,"order_key":"03:miner-3","p2mr_program_hex":"8383838383838383838383838383838383838383838383838383838383838383","share_difficulty":2,"share_id":"share-1411","share_seq":1411,"template_height":800044},{"accepted_at_ms":1000000,"credit_policy":null,"job_id":"job-376","job_issued_at_ms":999973,"miner_id":"miner-2","network_difficulty":1000002,"ntime":1700000376,"order_key":"02:miner-2","p2mr_program_hex":"7878787878787878787878787878787878787878787878787878787878787878","share_difficulty":2,"share_id":"share-376","share_seq":376,"template_height":800011},{"accepted_at_ms":999988,"credit_policy":null,"job_id":"job-3895","job_issued_at_ms":999994,"miner_id":"miner-1","network_difficulty":1000002,"ntime":1700003895,"order_key":"01:miner-1","p2mr_program_hex":"3737373737373737373737373737373737373737373737373737373737373737","share_difficulty":7,"share_id":"share-3895","share_seq":3895,"template_height":800121},{"accepted_at_ms":999977,"credit_policy":null,"job_id":"job-3016","job_issued_at_ms":1000295,"miner_id":"miner-2","network_difficulty":1000007,"ntime":1700003016,"order_key":"02:miner-2","p2mr_program_hex":"c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8","share_difficulty":7,"share_id":"share-3016","share_seq":3016,"template_height":800094},{"accepted_at_ms":999996,"credit_policy":null,"job_id":"job-1779","job_issued_at_ms":999999,"miner_id":"miner-8","network_difficulty":1000011,"ntime":1700001779,"order_key":"08:miner-8","p2mr_program_hex":"f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3","share_difficulty":7,"share_id":"share-1779","share_seq":1779,"template_height":800055},{"accepted_at_ms":999995,"credit_policy":null,"job_id":"job-3852","job_issued_at_ms":999993,"miner_id":"miner-2","network_difficulty":1000010,"ntime":1700003852,"order_key":"02:miner-2","p2mr_program_hex":"0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c","share_difficulty":7,"share_id":"share-3852","share_seq":3852,"template_height":800120},{"accepted_at_ms":999990,"credit_policy":"stale-grace","job_id":"job-2355","job_issued_at_ms":999993,"miner_id":"miner-1","network_difficulty":1000009,"ntime":1700002355,"order_key":"01:miner-1","p2mr_program_hex":"3333333333333333333333333333333333333333333333333333333333333333","share_difficulty":7,"share_id":"share-2355","share_seq":2355,"template_height":800073},{"accepted_at_ms":999990,"credit_policy":"stale-grace","job_id":"job-3722","job_issued_at_ms":999974,"miner_id":"miner-4","network_difficulty":1000016,"ntime":1700003722,"order_key":"04:miner-4","p2mr_program_hex":"8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a","share_difficulty":101,"share_id":"share-3722","share_seq":3722,"template_height":800116},{"accepted_at_ms":999988,"credit_policy":null,"job_id":"job-3621","job_issued_at_ms":999972,"miner_id":"miner-2","network_difficulty":1000000,"ntime":1700003621,"order_key":"02:miner-2","p2mr_program_hex":"2525252525252525252525252525252525252525252525252525252525252525","share_difficulty":19,"share_id":"share-3621","share_seq":3621,"template_height":800113},{"accepted_at_ms":999990,"credit_policy":null,"job_id":"job-2401","job_issued_at_ms":999978,"miner_id":"miner-3","network_difficulty":1000004,"ntime":1700002401,"order_key":"03:miner-3","p2mr_program_hex":"6161616161616161616161616161616161616161616161616161616161616161","share_difficulty":1,"share_id":"share-2401","share_seq":2401,"template_height":800075},{"accepted_at_ms":1000000,"credit_policy":null,"job_id":"job-2182","job_issued_at_ms":999992,"miner_id":"miner-4","network_difficulty":1000006,"ntime":1700002182,"order_key":"04:miner-4","p2mr_program_hex":"8686868686868686868686868686868686868686868686868686868686868686","share_difficulty":3,"share_id":"share-2182","share_seq":2182,"template_height":800068},{"accepted_at_ms":999990,"credit_policy":"stale-grace","job_id":"job-2207","job_issued_at_ms":999994,"miner_id":"miner-7","network_difficulty":1000014,"ntime":1700002207,"order_key":"07:miner-7","p2mr_program_hex":"9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f","share_difficulty":19,"share_id":"share-2207","share_seq":2207,"template_height":800068},{"accepted_at_ms":999982,"credit_policy":null,"job_id":"job-3847","job_issued_at_ms":999998,"miner_id":"miner-8","network_difficulty":1000005,"ntime":1700003847,"order_key":"08:miner-8","p2mr_program_hex":"0707070707070707070707070707070707070707070707070707070707070707","share_difficulty":3,"share_id":"share-3847","share_seq":3847,"template_height":800120},{"accepted_at_ms":999999,"credit_policy":null,"job_id":"job-2821","job_issued_at_ms":999999,"miner_id":"miner-5","network_difficulty":1000016,"ntime":1700002821,"order_key":"05:miner-5","p2mr_program_hex":"0505050505050505050505050505050505050505050505050505050505050505","share_difficulty":1,"share_id":"share-2821","share_seq":2821,"template_height":800088},{"accepted_at_ms":999985,"credit_policy":null,"job_id":"job-998","job_issued_at_ms":999984,"miner_id":"miner-8","network_difficulty":1000012,"ntime":1700000998,"order_key":"08:miner-8","p2mr_program_hex":"e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6","share_difficulty":3,"share_id":"share-998","share_seq":998,"template_height":800031},{"accepted_at_ms":999985,"credit_policy":null,"job_id":"job-882","job_issued_at_ms":999992,"miner_id":"miner-2","network_difficulty":1000015,"ntime":1700000882,"order_key":"02:miner-2","p2mr_program_hex":"7272727272727272727272727272727272727272727272727272727272727272","share_difficulty":7,"share_id":"share-882","share_seq":882,"template_height":800027},{"accepted_at_ms":999990,"credit_policy":null,"job_id":"job-3259","job_issued_at_ms":999997,"miner_id":"miner-3","network_difficulty":1000012,"ntime":1700003259,"order_key":"03:miner-3","p2mr_program_hex":"bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb","share_difficulty":101,"share_id":"share-3259","share_seq":3259,"template_height":800101},{"accepted_at_ms":1000000,"credit_policy":null,"job_id":"job-184","job_issued_at_ms":999980,"miner_id":"miner-8","network_difficulty":1000014,"ntime":1700000184,"order_key":"08:miner-8","p2mr_program_hex":"b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8","share_difficulty":3,"share_id":"share-184","share_seq":184,"template_height":800005},{"accepted_at_ms":1000031,"credit_policy":null,"job_id":"job-1000","job_issued_at_ms":1000073,"miner_id":"miner-10","network_difficulty":1000014,"ntime":1700001000,"order_key":"10:miner-10","p2mr_program_hex":"e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8","share_difficulty":19,"share_id":"share-1000","share_seq":1000,"template_height":800031},{"accepted_at_ms":999986,"credit_policy":null,"job_id":"job-1417","job_issued_at_ms":999998,"miner_id":"miner-9","network_difficulty":1000006,"ntime":1700001417,"order_key":"09:miner-9","p2mr_program_hex":"8989898989898989898989898989898989898989898989898989898989898989","share_difficulty":2,"share_id":"share-1417","share_seq":1417,"template_height":800044},{"accepted_at_ms":999992,"credit_policy":null,"job_id":"job-2629","job_issued_at_ms":999977,"miner_id":"miner-0","network_difficulty":1000011,"ntime":1700002629,"order_key":"00:miner-0","p2mr_program_hex":"4545454545454545454545454545454545454545454545454545454545454545","share_difficulty":7,"share_id":"share-2629","share_seq":2629,"template_height":800082},{"accepted_at_ms":999991,"credit_policy":null,"job_id":"job-1528","job_issued_at_ms":999997,"miner_id":"miner-10","network_difficulty":1000015,"ntime":1700001528,"order_key":"10:miner-10","p2mr_program_hex":"f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8","share_difficulty":101,"share_id":"share-1528","share_seq":1528,"template_height":800047},{"accepted_at_ms":999987,"credit_policy":"stale-grace","job_id":"job-2115","job_issued_at_ms":999992,"miner_id":"miner-3","network_difficulty":1000007,"ntime":1700002115,"order_key":"03:miner-3","p2mr_program_hex":"4343434343434343434343434343434343434343434343434343434343434343","share_difficulty":1000,"share_id":"share-2115","share_seq":2115,"template_height":800066},{"accepted_at_ms":999972,"credit_policy":"stale-grace","job_id":"job-3922","job_issued_at_ms":999978,"miner_id":"miner-6","network_difficulty":1000012,"ntime":1700003922,"order_key":"06:miner-6","p2mr_program_hex":"5252525252525252525252525252525252525252525252525252525252525252","share_difficulty":2,"share_id":"share-3922","share_seq":3922,"template_height":800122},{"accepted_at_ms":999984,"credit_policy":"stale-grace","job_id":"job-3374","job_issued_at_ms":999976,"miner_id":"miner-8","network_difficulty":1000008,"ntime":1700003374,"order_key":"08:miner-8","p2mr_program_hex":"2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e","share_difficulty":1,"share_id":"share-3374","share_seq":3374,"template_height":800105},{"accepted_at_ms":999985,"credit_policy":null,"job_id":"job-603","job_issued_at_ms":999997,"miner_id":"miner-9","network_difficulty":1000008,"ntime":1700000603,"order_key":"09:miner-9","p2mr_program_hex":"5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b","share_difficulty":2,"share_id":"share-603","share_seq":603,"template_height":800018},{"accepted_at_ms":999979,"credit_policy":null,"job_id":"job-2389","job_issued_at_ms":999976,"miner_id":"miner-2","network_difficulty":1000009,"ntime":1700002389,"order_key":"02:miner-2","p2mr_program_hex":"5555555555555555555555555555555555555555555555555555555555555555","share_difficulty":3,"share_id":"share-2389","share_seq":2389,"template_height":800074},{"accepted_at_ms":999997,"credit_policy":"stale-grace","job_id":"job-1454","job_issued_at_ms":999980,"miner_id":"miner-2","network_difficulty":1000009,"ntime":1700001454,"order_key":"02:miner-2","p2mr_program_hex":"aeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeae","share_difficulty":101,"share_id":"share-1454","share_seq":1454,"template_height":800045},{"accepted_at_ms":999975,"credit_policy":null,"job_id":"job-3918","job_issued_at_ms":999988,"miner_id":"miner-2","network_difficulty":1000008,"ntime":1700003918,"order_key":"02:miner-2","p2mr_program_hex":"4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e","share_difficulty":1000,"share_id":"share-3918","share_seq":3918,"template_height":800122},{"accepted_at_ms":1000000,"credit_policy":null,"job_id":"job-2436","job_issued_at_ms":999977,"miner_id":"miner-5","network_difficulty":1000005,"ntime":1700002436,"order_key":"05:miner-5","p2mr_program_hex":"8484848484848484848484848484848484848484848484848484848484848484","share_difficulty":3,"share_id":"share-2436","share_seq":2436,"template_height":800076},{"accepted_at_ms":999997,"credit_policy":null,"job_id":"job-3723","job_issued_at_ms":999978,"miner_id":"miner-5","network_difficulty":1000000,"ntime":1700003723,"order_key":"05:miner-5","p2mr_program_hex":"8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b","share_difficulty":101,"share_id":"share-3723","share_seq":3723,"template_height":800116},{"accepted_at_ms":999989,"credit_policy":"stale-grace","job_id":"job-138","job_issued_at_ms":999991,"miner_id":"miner-6","network_difficulty":1000002,"ntime":1700000138,"order_key":"06:miner-6","p2mr_program_hex":"8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a","share_difficulty":1000,"share_id":"share-138","share_seq":138,"template_height":800004},{"accepted_at_ms":999981,"credit_policy":"stale-grace","job_id":"job-3017","job_issued_at_ms":999981,"miner_id":"miner-3","network_difficulty":1000008,"ntime":1700003017,"order_key":"03:miner-3","p2mr_program_hex":"c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9","share_difficulty":3,"share_id":"share-3017","share_seq":3017,"template_height":800094},{"accepted_at_ms":999975,"credit_policy":null,"job_id":"job-412","job_issued_at_ms":999990,"miner_id":"miner-5","network_difficulty":1000004,"ntime":1700000412,"order_key":"05:miner-5","p2mr_program_hex":"9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c","share_difficulty":3,"share_id":"share-412","share_seq":412,"template_height":800012},{"accepted_at_ms":999995,"credit_policy":null,"job_id":"job-2213","job_issued_at_ms":999998,"miner_id":"miner-2","network_difficulty":1000003,"ntime":1700002213,"order_key":"02:miner-2","p2mr_program_hex":"a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5","share_difficulty":1000,"share_id":"share-2213","share_seq":2213,"template_height":800069},{"accepted_at_ms":999980,"credit_policy":null,"job_id":"job-2043","job_issued_at_ms":999976,"miner_id":"miner-8","network_difficulty":1000003,"ntime":1700002043,"order_key":"08:miner-8","p2mr_program_hex":"fbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfb","share_difficulty":2,"share_id":"share-2043","share_seq":2043,"template_height":800063},{"accepted_at_ms":999980,"credit_policy":null,"job_id":"job-2795","job_issued_at_ms":999994,"miner_id":"miner-1","network_difficulty":1000007,"ntime":1700002795,"order_key":"01:miner-1","p2mr_program_hex":"ebebebebebebebebebebebebebebebebebebebebebebebebebebebebebebebeb","share_difficulty":3,"share_id":"share-2795","share_seq":2795,"template_height":800087},{"accepted_at_ms":999989,"credit_policy":null,"job_id":"job-1869","job_issued_at_ms":999976,"miner_id":"miner-10","network_difficulty":1000016,"ntime":1700001869,"order_key":"10:miner-10","p2mr_program_hex":"4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d","share_difficulty":101,"share_id":"share-1869","share_seq":1869,"template_height":800058},{"accepted_at_ms":999984,"credit_policy":"stale-grace","job_id":"job-659","job_issued_at_ms":999983,"miner_id":"miner-10","network_difficulty":1000013,"ntime":1700000659,"order_key":"10:miner-10","p2mr_program_hex":"9393939393939393939393939393939393939393939393939393939393939393","share_difficulty":19,"share_id":"share-659","share_seq":659,"template_height":800020},{"accepted_at_ms":999980,"credit_policy":null,"job_id":"job-2422","job_issued_at_ms":1000000,"miner_id":"miner-2","network_difficulty":1000008,"ntime":1700002422,"order_key":"02:miner-2","p2mr_program_hex":"7676767676767676767676767676767676767676767676767676767676767676","share_difficulty":7,"share_id":"share-2422","share_seq":2422,"template_height":800075},{"accepted_at_ms":999987,"credit_policy":null,"job_id":"job-1643","job_issued_at_ms":999982,"miner_id":"miner-4","network_difficulty":1000011,"ntime":1700001643,"order_key":"04:miner-4","p2mr_program_hex":"6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b","share_difficulty":7,"share_id":"share-1643","share_seq":1643,"template_height":800051},{"accepted_at_ms":999985,"credit_policy":null,"job_id":"job-93","job_issued_at_ms":999992,"miner_id":"miner-5","network_difficulty":1000008,"ntime":1700000093,"order_key":"05:miner-5","p2mr_program_hex":"5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d","share_difficulty":1000,"share_id":"share-93","share_seq":93,"template_height":800002},{"accepted_at_ms":999970,"credit_policy":null,"job_id":"job-31","job_issued_at_ms":999974,"miner_id":"miner-9","network_difficulty":1000014,"ntime":1700000031,"order_key":"09:miner-9","p2mr_program_hex":"1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f","share_difficulty":19,"share_id":"share-31","share_seq":31,"template_height":800000},{"accepted_at_ms":1000000,"credit_policy":null,"job_id":"job-2155","job_issued_at_ms":1000220,"miner_id":"miner-10","network_difficulty":1000013,"ntime":1700002155,"order_key":"10:miner-10","p2mr_program_hex":"6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b","share_difficulty":1000,"share_id":"share-2155","share_seq":2155,"template_height":800067},{"accepted_at_ms":999977,"credit_policy":"stale-grace","job_id":"job-824","job_issued_at_ms":999988,"miner_id":"miner-10","network_difficulty":1000008,"ntime":1700000824,"order_key":"10:miner-10","p2mr_program_hex":"3838383838383838383838383838383838383838383838383838383838383838","share_difficulty":7,"share_id":"share-824","share_seq":824,"template_height":800025},{"accepted_at_ms":999997,"credit_policy":null,"job_id":"job-1687","job_issued_at_ms":999992,"miner_id":"miner-4","network_difficulty":1000004,"ntime":1700001687,"order_key":"04:miner-4","p2mr_program_hex":"9797979797979797979797979797979797979797979797979797979797979797","share_difficulty":1000,"share_id":"share-1687","share_seq":1687,"template_height":800052},{"accepted_at_ms":999978,"credit_policy":null,"job_id":"job-1468","job_issued_at_ms":999981,"miner_id":"miner-5","network_difficulty":1000006,"ntime":1700001468,"order_key":"05:miner-5","p2mr_program_hex":"bcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbc","share_difficulty":2,"share_id":"share-1468","share_seq":1468,"template_height":800045},{"accepted_at_ms":999985,"credit_policy":"stale-grace","job_id":"job-3161","job_issued_at_ms":999978,"miner_id":"miner-4","network_difficulty":1000016,"ntime":1700003161,"order_key":"04:miner-4","p2mr_program_hex":"5959595959595959595959595959595959595959595959595959595959595959","share_difficulty":1,"share_id":"share-3161","share_seq":3161,"template_height":800098},{"accepted_at_ms":999988,"credit_policy":null,"job_id":"job-3028","job_issued_at_ms":999996,"miner_id":"miner-3","network_difficulty":1000002,"ntime":1700003028,"order_key":"03:miner-3","p2mr_program_hex":"d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4","share_difficulty":1000,"share_id":"share-3028","share_seq":3028,"template_height":800094},{"accepted_at_ms":999990,"credit_policy":null,"job_id":"job-1449","job_issued_at_ms":999991,"miner_id":"miner-8","network_difficulty":1000004,"ntime":1700001449,"order_key":"08:miner-8","p2mr_program_hex":"a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9","share_difficulty":1,"share_id":"share-1449","share_seq":1449,"template_height":800045},{"accepted_at_ms":999987,"credit_policy":null,"job_id":"job-963","job_issued_at_ms":1000246,"miner_id":"miner-6","network_difficulty":1000011,"ntime":1700000963,"order_key":"06:miner-6","p2mr_program_hex":"c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3","share_difficulty":3,"share_id":"share-963","share_seq":963,"template_height":800030},{"accepted_at_ms":999984,"credit_policy":null,"job_id":"job-2957","job_issued_at_ms":1000000,"miner_id":"miner-9","network_difficulty":1000016,"ntime":1700002957,"order_key":"09:miner-9","p2mr_program_hex":"8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d","share_difficulty":1000,"share_id":"share-2957","share_seq":2957,"template_height":800092},{"accepted_at_ms":999972,"credit_policy":null,"job_id":"job-1460","job_issued_at_ms":999996,"miner_id":"miner-8","network_difficulty":1000015,"ntime":1700001460,"order_key":"08:miner-8","p2mr_program_hex":"b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4","share_difficulty":7,"share_id":"share-1460","share_seq":1460,"template_height":800045},{"accepted_at_ms":999995,"credit_policy":null,"job_id":"job-1480","job_issued_at_ms":999995,"miner_id":"miner-6","network_difficulty":1000001,"ntime":1700001480,"order_key":"06:miner-6","p2mr_program_hex":"c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8","share_difficulty":2,"share_id":"share-1480","share_seq":1480,"template_height":800046},{"accepted_at_ms":999985,"credit_policy":"stale-grace","job_id":"job-1596","job_issued_at_ms":999973,"miner_id":"miner-1","network_difficulty":1000015,"ntime":1700001596,"order_key":"01:miner-1","p2mr_program_hex":"3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c","share_difficulty":2,"share_id":"share-1596","share_seq":1596,"template_height":800049},{"accepted_at_ms":999974,"credit_policy":null,"job_id":"job-3844","job_issued_at_ms":999993,"miner_id":"miner-5","network_difficulty":1000002,"ntime":1700003844,"order_key":"05:miner-5","p2mr_program_hex":"0404040404040404040404040404040404040404040404040404040404040404","share_difficulty":3,"share_id":"share-3844","share_seq":3844,"template_height":800120},{"accepted_at_ms":999980,"credit_policy":null,"job_id":"job-2730","job_issued_at_ms":999978,"miner_id":"miner-2","network_difficulty":1000010,"ntime":1700002730,"order_key":"02:miner-2","p2mr_program_hex":"aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa","share_difficulty":1,"share_id":"share-2730","share_seq":2730,"template_height":800085},{"accepted_at_ms":999983,"credit_policy":null,"job_id":"job-1050","job_issued_at_ms":999985,"miner_id":"miner-5","network_difficulty":1000013,"ntime":1700001050,"order_key":"05:miner-5","p2mr_program_hex":"1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a","share_difficulty":3,"share_id":"share-1050","share_seq":1050,"template_height":800032},{"accepted_at_ms":999988,"credit_policy":null,"job_id":"job-457","job_issued_at_ms":999975,"miner_id":"miner-6","network_difficulty":1000015,"ntime":1700000457,"order_key":"06:miner-6","p2mr_program_hex":"c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9","share_difficulty":101,"share_id":"share-457","share_seq":457,"template_height":800014},{"accepted_at_ms":999971,"credit_policy":"stale-grace","job_id":"job-2083","job_issued_at_ms":999972,"miner_id":"miner-4","network_difficulty":1000009,"ntime":1700002083,"order_key":"04:miner-4","p2mr_program_hex":"2323232323232323232323232323232323232323232323232323232323232323","share_difficulty":1000,"share_id":"share-2083","share_seq":2083,"template_height":800065},{"accepted_at_ms":999983,"credit_policy":null,"job_id":"job-1764","job_issued_at_ms":999993,"miner_id":"miner-4","network_difficulty":1000013,"ntime":1700001764,"order_key":"04:miner-4","p2mr_program_hex":"e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4","share_difficulty":101,"share_id":"share-1764","share_seq":1764,"template_height":800055},{"accepted_at_ms":999977,"credit_policy":null,"job_id":"job-3830","job_issued_at_ms":999980,"miner_id":"miner-2","network_difficulty":1000005,"ntime":1700003830,"order_key":"02:miner-2","p2mr_program_hex":"f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6","share_difficulty":19,"share_id":"share-3830","share_seq":3830,"template_height":800119},{"accepted_at_ms":999994,"credit_policy":null,"job_id":"job-1661","job_issued_at_ms":999981,"miner_id":"miner-0","network_difficulty":1000012,"ntime":1700001661,"order_key":"00:miner-0","p2mr_program_hex":"7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d","share_difficulty":101,"share_id":"share-1661","share_seq":1661,"template_height":800051},{"accepted_at_ms":999983,"credit_policy":null,"job_id":"job-862","job_issued_at_ms":999975,"miner_id":"miner-4","network_difficulty":1000012,"ntime":1700000862,"order_key":"04:miner-4","p2mr_program_hex":"5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e","share_difficulty":101,"share_id":"share-862","share_seq":862,"template_height":800026},{"accepted_at_ms":999982,"credit_policy":null,"job_id":"job-1543","job_issued_at_ms":999990,"miner_id":"miner-3","network_difficulty":1000013,"ntime":1700001543,"order_key":"03:miner-3","p2mr_program_hex":"0707070707070707070707070707070707070707070707070707070707070707","share_difficulty":2,"share_id":"share-1543","share_seq":1543,"template_height":800048},{"accepted_at_ms":999999,"credit_policy":null,"job_id":"job-407","job_issued_at_ms":999978,"miner_id":"miner-0","network_difficulty":1000016,"ntime":1700000407,"order_key":"00:miner-0","p2mr_program_hex":"9797979797979797979797979797979797979797979797979797979797979797","share_difficulty":1000,"share_id":"share-407","share_seq":407,"template_height":800012},{"accepted_at_ms":999998,"credit_policy":null,"job_id":"job-1523","job_issued_at_ms":999976,"miner_id":"miner-5","network_difficulty":1000010,"ntime":1700001523,"order_key":"05:miner-5","p2mr_program_hex":"f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3","share_difficulty":1000,"share_id":"share-1523","share_seq":1523,"template_height":800047},{"accepted_at_ms":999994,"credit_policy":null,"job_id":"job-515","job_issued_at_ms":999984,"miner_id":"miner-9","network_difficulty":1000005,"ntime":1700000515,"order_key":"09:miner-9","p2mr_program_hex":"0303030303030303030303030303030303030303030303030303030303030303","share_difficulty":101,"share_id":"share-515","share_seq":515,"template_height":800016},{"accepted_at_ms":999974,"credit_policy":"stale-grace","job_id":"job-164","job_issued_at_ms":999986,"miner_id":"miner-10","network_difficulty":1000011,"ntime":1700000164,"order_key":"10:miner-10","p2mr_program_hex":"a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4","share_difficulty":1000,"share_id":"share-164","share_seq":164,"template_height":800005},{"accepted_at_ms":999997,"credit_policy":null,"job_id":"job-3024","job_issued_at_ms":999991,"miner_id":"miner-10","network_difficulty":1000015,"ntime":1700003024,"order_key":"10:miner-10","p2mr_program_hex":"d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0","share_difficulty":101,"share_id":"share-3024","share_seq":3024,"template_height":800094},{"accepted_at_ms":999970,"credit_policy":null,"job_id":"job-3816","job_issued_at_ms":999990,"miner_id":"miner-10","network_difficulty":1000008,"ntime":1700003816,"order_key":"10:miner-10","p2mr_program_hex":"e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8","share_difficulty":1,"share_id":"share-3816","share_seq":3816,"template_height":800119},{"accepted_at_ms":999978,"credit_policy":null,"job_id":"job-1755","job_issued_at_ms":999971,"miner_id":"miner-6","network_difficulty":1000004,"ntime":1700001755,"order_key":"06:miner-6","p2mr_program_hex":"dbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdb","share_difficulty":1000,"share_id":"share-1755","share_seq":1755,"template_height":800054},{"accepted_at_ms":999998,"credit_policy":null,"job_id":"job-698","job_issued_at_ms":999976,"miner_id":"miner-5","network_difficulty":1000001,"ntime":1700000698,"order_key":"05:miner-5","p2mr_program_hex":"babababababababababababababababababababababababababababababababa","share_difficulty":1,"share_id":"share-698","share_seq":698,"template_height":800021},{"accepted_at_ms":999973,"credit_policy":null,"job_id":"job-2099","job_issued_at_ms":999970,"miner_id":"miner-9","network_difficulty":1000008,"ntime":1700002099,"order_key":"09:miner-9","p2mr_program_hex":"3333333333333333333333333333333333333333333333333333333333333333","share_difficulty":1000,"share_id":"share-2099","share_seq":2099,"template_height":800065},{"accepted_at_ms":999981,"credit_policy":null,"job_id":"job-868","job_issued_at_ms":999977,"miner_id":"miner-10","network_difficulty":1000001,"ntime":1700000868,"order_key":"10:miner-10","p2mr_program_hex":"6464646464646464646464646464646464646464646464646464646464646464","share_difficulty":2,"share_id":"share-868","share_seq":868,"template_height":800027},{"accepted_at_ms":999975,"credit_policy":null,"job_id":"job-1412","job_issued_at_ms":999992,"miner_id":"miner-4","network_difficulty":1000001,"ntime":1700001412,"order_key":"04:miner-4","p2mr_program_hex":"8484848484848484848484848484848484848484848484848484848484848484","share_difficulty":19,"share_id":"share-1412","share_seq":1412,"template_height":800044},{"accepted_at_ms":999982,"credit_policy":null,"job_id":"job-1424","job_issued_at_ms":999985,"miner_id":"miner-5","network_difficulty":1000013,"ntime":1700001424,"order_key":"05:miner-5","p2mr_program_hex":"9090909090909090909090909090909090909090909090909090909090909090","share_difficulty":19,"share_id":"share-1424","share_seq":1424,"template_height":800044},{"accepted_at_ms":999975,"credit_policy":null,"job_id":"job-2739","job_issued_at_ms":999976,"miner_id":"miner-0","network_difficulty":1000002,"ntime":1700002739,"order_key":"00:miner-0","p2mr_program_hex":"b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3","share_difficulty":1,"share_id":"share-2739","share_seq":2739,"template_height":800085},{"accepted_at_ms":999981,"credit_policy":null,"job_id":"job-3055","job_issued_at_ms":999988,"miner_id":"miner-8","network_difficulty":1000012,"ntime":1700003055,"order_key":"08:miner-8","p2mr_program_hex":"efefefefefefefefefefefefefefefefefefefefefefefefefefefefefefefef","share_difficulty":101,"share_id":"share-3055","share_seq":3055,"template_height":800095},{"accepted_at_ms":999994,"credit_policy":null,"job_id":"job-2231","job_issued_at_ms":999991,"miner_id":"miner-9","network_difficulty":1000004,"ntime":1700002231,"order_key":"09:miner-9","p2mr_program_hex":"b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7","share_difficulty":3,"share_id":"share-2231","share_seq":2231,"template_height":800069},{"accepted_at_ms":999976,"credit_policy":null,"job_id":"job-919","job_issued_at_ms":999984,"miner_id":"miner-6","network_difficulty":1000001,"ntime":1700000919,"order_key":"06:miner-6","p2mr_program_hex":"9797979797979797979797979797979797979797979797979797979797979797","share_difficulty":2,"share_id":"share-919","share_seq":919,"template_height":800028},{"accepted_at_ms":999973,"credit_policy":"stale-grace","job_id":"job-3866","job_issued_at_ms":999974,"miner_id":"miner-5","network_difficulty":1000007,"ntime":1700003866,"order_key":"05:miner-5","p2mr_program_hex":"1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a","share_difficulty":19,"share_id":"share-3866","share_seq":3866,"template_height":800120},{"accepted_at_ms":999974,"credit_policy":null,"job_id":"job-3893","job_issued_at_ms":999970,"miner_id":"miner-10","network_difficulty":1000000,"ntime":1700003893,"order_key":"10:miner-10","p2mr_program_hex":"3535353535353535353535353535353535353535353535353535353535353535","share_difficulty":3,"share_id":"share-3893","share_seq":3893,"template_height":800121},{"accepted_at_ms":1000211,"credit_policy":null,"job_id":"job-3385","job_issued_at_ms":1000223,"miner_id":"miner-8","network_difficulty":1000002,"ntime":1700003385,"order_key":"08:miner-8","p2mr_program_hex":"3939393939393939393939393939393939393939393939393939393939393939","share_difficulty":7,"share_id":"share-3385","share_seq":3385,"template_height":800105},{"accepted_at_ms":999988,"credit_policy":"stale-grace","job_id":"job-433","job_issued_at_ms":999981,"miner_id":"miner-4","network_difficulty":1000008,"ntime":1700000433,"order_key":"04:miner-4","p2mr_program_hex":"b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1","share_difficulty":2,"share_id":"share-433","share_seq":433,"template_height":800013},{"accepted_at_ms":999979,"credit_policy":null,"job_id":"job-3025","job_issued_at_ms":999990,"miner_id":"miner-0","network_difficulty":1000016,"ntime":1700003025,"order_key":"00:miner-0","p2mr_program_hex":"d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1","share_difficulty":19,"share_id":"share-3025","share_seq":3025,"template_height":800094},{"accepted_at_ms":999974,"credit_policy":null,"job_id":"job-1354","job_issued_at_ms":999983,"miner_id":"miner-1","network_difficulty":1000011,"ntime":1700001354,"order_key":"01:miner-1","p2mr_program_hex":"4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a","share_difficulty":7,"share_id":"share-1354","share_seq":1354,"template_height":800042},{"accepted_at_ms":999992,"credit_policy":"stale-grace","job_id":"job-2113","job_issued_at_ms":999987,"miner_id":"miner-1","network_difficulty":1000005,"ntime":1700002113,"order_key":"01:miner-1","p2mr_program_hex":"4141414141414141414141414141414141414141414141414141414141414141","share_difficulty":2,"share_id":"share-2113","share_seq":2113,"template_height":800066},{"accepted_at_ms":1000039,"credit_policy":"stale-grace","job_id":"job-2313","job_issued_at_ms":999981,"miner_id":"miner-3","network_difficulty":1000001,"ntime":1700002313,"order_key":"03:miner-3","p2mr_program_hex":"0909090909090909090909090909090909090909090909090909090909090909","share_difficulty":2,"share_id":"share-2313","share_seq":2313,"template_height":800072},{"accepted_at_ms":999980,"credit_policy":null,"job_id":"job-2121","job_issued_at_ms":999974,"miner_id":"miner-9","network_difficulty":1000013,"ntime":1700002121,"order_key":"09:miner-9","p2mr_program_hex":"4949494949494949494949494949494949494949494949494949494949494949","share_difficulty":19,"share_id":"share-2121","share_seq":2121,"template_height":800066},{"accepted_at_ms":999984,"credit_policy":null,"job_id":"job-2404","job_issued_at_ms":999983,"miner_id":"miner-6","network_difficulty":1000007,"ntime":1700002404,"order_key":"06:miner-6","p2mr_program_hex":"6464646464646464646464646464646464646464646464646464646464646464","share_difficulty":101,"share_id":"share-2404","share_seq":2404,"template_height":800075},{"accepted_at_ms":1000000,"credit_policy":null,"job_id":"job-3386","job_issued_at_ms":999985,"miner_id":"miner-9","network_difficulty":1000003,"ntime":1700003386,"order_key":"09:miner-9","p2mr_program_hex":"3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a","share_difficulty":19,"share_id":"share-3386","share_seq":3386,"template_height":800105},{"accepted_at_ms":999978,"credit_policy":null,"job_id":"job-237","job_issued_at_ms":999990,"miner_id":"miner-6","network_difficulty":1000016,"ntime":1700000237,"order_key":"06:miner-6","p2mr_program_hex":"edededededededededededededededededededededededededededededededed","share_difficulty":19,"share_id":"share-237","share_seq":237,"template_height":800007},{"accepted_at_ms":999973,"credit_policy":null,"job_id":"job-123","job_issued_at_ms":999990,"miner_id":"miner-2","network_difficulty":1000004,"ntime":1700000123,"order_key":"02:miner-2","p2mr_program_hex":"7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b","share_difficulty":1,"share_id":"share-123","share_seq":123,"template_height":800003},{"accepted_at_ms":999978,"credit_policy":null,"job_id":"job-2772","job_issued_at_ms":999989,"miner_id":"miner-0","network_difficulty":1000001,"ntime":1700002772,"order_key":"00:miner-0","p2mr_program_hex":"d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4","share_difficulty":1000,"share_id":"share-2772","share_seq":2772,"template_height":800086},{"accepted_at_ms":999974,"credit_policy":null,"job_id":"job-2589","job_issued_at_ms":999989,"miner_id":"miner-4","network_difficulty":1000005,"ntime":1700002589,"order_key":"04:miner-4","p2mr_program_hex":"1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d","share_difficulty":7,"share_id":"share-2589","share_seq":2589,"template_height":800080},{"accepted_at_ms":999998,"credit_policy":null,"job_id":"job-643","job_issued_at_ms":999988,"miner_id":"miner-5","network_difficulty":1000014,"ntime":1700000643,"order_key":"05:miner-5","p2mr_program_hex":"8383838383838383838383838383838383838383838383838383838383838383","share_difficulty":101,"share_id":"share-643","share_seq":643,"template_height":800020},{"accepted_at_ms":999993,"credit_policy":null,"job_id":"job-1617","job_issued_at_ms":999991,"miner_id":"miner-0","network_difficulty":1000002,"ntime":1700001617,"order_key":"00:miner-0","p2mr_program_hex":"5151515151515151515151515151515151515151515151515151515151515151","share_difficulty":1,"share_id":"share-1617","share_seq":1617,"template_height":800050},{"accepted_at_ms":999996,"credit_policy":"stale-grace","job_id":"job-3158","job_issued_at_ms":999975,"miner_id":"miner-1","network_difficulty":1000013,"ntime":1700003158,"order_key":"01:miner-1","p2mr_program_hex":"5656565656565656565656565656565656565656565656565656565656565656","share_difficulty":1000,"share_id":"share-3158","share_seq":3158,"template_height":800098},{"accepted_at_ms":999989,"credit_policy":null,"job_id":"job-1974","job_issued_at_ms":999974,"miner_id":"miner-5","network_difficulty":1000002,"ntime":1700001974,"order_key":"05:miner-5","p2mr_program_hex":"b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6","share_difficulty":2,"share_id":"share-1974","share_seq":1974,"template_height":800061},{"accepted_at_ms":999978,"credit_policy":null,"job_id":"job-299","job_issued_at_ms":1000099,"miner_id":"miner-2","network_difficulty":1000010,"ntime":1700000299,"order_key":"02:miner-2","p2mr_program_hex":"2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b","share_difficulty":1000,"share_id":"share-299","share_seq":299,"template_height":800009},{"accepted_at_ms":999988,"credit_policy":null,"job_id":"job-737","job_issued_at_ms":999996,"miner_id":"miner-0","network_difficulty":1000006,"ntime":1700000737,"order_key":"00:miner-0","p2mr_program_hex":"e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1","share_difficulty":7,"share_id":"share-737","share_seq":737,"template_height":800023},{"accepted_at_ms":999979,"credit_policy":null,"job_id":"job-558","job_issued_at_ms":999977,"miner_id":"miner-8","network_difficulty":1000014,"ntime":1700000558,"order_key":"08:miner-8","p2mr_program_hex":"2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e","share_difficulty":2,"share_id":"share-558","share_seq":558,"template_height":800017},{"accepted_at_ms":999970,"credit_policy":"stale-grace","job_id":"job-3655","job_issued_at_ms":999977,"miner_id":"miner-3","network_difficulty":1000000,"ntime":1700003655,"order_key":"03:miner-3","p2mr_program_hex":"4747474747474747474747474747474747474747474747474747474747474747","share_difficulty":19,"share_id":"share-3655","share_seq":3655,"template_height":800114},{"accepted_at_ms":999980,"credit_policy":null,"job_id":"job-2760","job_issued_at_ms":999982,"miner_id":"miner-10","network_difficulty":1000006,"ntime":1700002760,"order_key":"10:miner-10","p2mr_program_hex":"c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8","share_difficulty":7,"share_id":"share-2760","share_seq":2760,"template_height":800086},{"accepted_at_ms":999989,"credit_policy":null,"job_id":"job-3262","job_issued_at_ms":1000282,"miner_id":"miner-6","network_difficulty":1000015,"ntime":1700003262,"order_key":"06:miner-6","p2mr_program_hex":"bebebebebebebebebebebebebebebebebebebebebebebebebebebebebebebebe","share_difficulty":19,"share_id":"share-3262","share_seq":3262,"template_height":800101},{"accepted_at_ms":999973,"credit_policy":null,"job_id":"job-890","job_issued_at_ms":999989,"miner_id":"miner-10","network_difficulty":1000006,"ntime":1700000890,"order_key":"10:miner-10","p2mr_program_hex":"7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a","share_difficulty":101,"share_id":"share-890","share_seq":890,"template_height":800027},{"accepted_at_ms":999993,"credit_policy":null,"job_id":"job-1953","job_issued_at_ms":999989,"miner_id":"miner-6","network_difficulty":1000015,"ntime":1700001953,"order_key":"06:miner-6","p2mr_program_hex":"a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1","share_difficulty":101,"share_id":"share-1953","share_seq":1953,"template_height":800061},{"accepted_at_ms":1000268,"credit_policy":null,"job_id":"job-1115","job_issued_at_ms":999975,"miner_id":"miner-4","network_difficulty":1000010,"ntime":1700001115,"order_key":"04:miner-4","p2mr_program_hex":"5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b","share_difficulty":3,"share_id":"share-1115","share_seq":1115,"template_height":800034},{"accepted_at_ms":1000483,"credit_policy":null,"job_id":"job-2978","job_issued_at_ms":1000000,"miner_id":"miner-8","network_difficulty":1000003,"ntime":1700002978,"order_key":"08:miner-8","p2mr_program_hex":"a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2","share_difficulty":3,"share_id":"share-2978","share_seq":2978,"template_height":800093},{"accepted_at_ms":999992,"credit_policy":null,"job_id":"job-1708","job_issued_at_ms":999974,"miner_id":"miner-3","network_difficulty":1000008,"ntime":1700001708,"order_key":"03:miner-3","p2mr_program_hex":"acacacacacacacacacacacacacacacacacacacacacacacacacacacacacacacac","share_difficulty":2,"share_id":"share-1708","share_seq":1708,"template_height":800053},{"accepted_at_ms":999984,"credit_policy":null,"job_id":"job-436","job_issued_at_ms":999979,"miner_id":"miner-7","network_difficulty":1000011,"ntime":1700000436,"order_key":"07:miner-7","p2mr_program_hex":"b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4","share_difficulty":1,"share_id":"share-436","share_seq":436,"template_height":800013},{"accepted_at_ms":999981,"credit_policy":null,"job_id":"job-3836","job_issued_at_ms":999989,"miner_id":"miner-8","network_difficulty":1000011,"ntime":1700003836,"order_key":"08:miner-8","p2mr_program_hex":"fcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfc","share_difficulty":1000,"share_id":"share-3836","share_seq":3836,"template_height":800119},{"accepted_at_ms":1000389,"credit_policy":null,"job_id":"job-3083","job_issued_at_ms":1000183,"miner_id":"miner-3","network_difficulty":1000006,"ntime":1700003083,"order_key":"03:miner-3","p2mr_program_hex":"0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b","share_difficulty":2,"share_id":"share-3083","share_seq":3083,"template_height":800096},{"accepted_at_ms":999975,"credit_policy":null,"job_id":"job-671","job_issued_at_ms":999979,"miner_id":"miner-0","network_difficulty":1000008,"ntime":1700000671,"order_key":"00:miner-0","p2mr_program_hex":"9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f","share_difficulty":1,"share_id":"share-671","share_seq":671,"template_height":800020},{"accepted_at_ms":1000242,"credit_policy":"stale-grace","job_id":"job-2419","job_issued_at_ms":1000175,"miner_id":"miner-10","network_difficulty":1000005,"ntime":1700002419,"order_key":"10:miner-10","p2mr_program_hex":"7373737373737373737373737373737373737373737373737373737373737373","share_difficulty":3,"share_id":"share-2419","share_seq":2419,"template_height":800075},{"accepted_at_ms":999983,"credit_policy":null,"job_id":"job-2788","job_issued_at_ms":999989,"miner_id":"miner-5","network_difficulty":1000000,"ntime":1700002788,"order_key":"05:miner-5","p2mr_program_hex":"e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4","share_difficulty":1000,"share_id":"share-2788","share_seq":2788,"template_height":800087},{"accepted_at_ms":999995,"credit_policy":null,"job_id":"job-2846","job_issued_at_ms":999990,"miner_id":"miner-8","network_difficulty":1000007,"ntime":1700002846,"order_key":"08:miner-8","p2mr_program_hex":"1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e","share_difficulty":3,"share_id":"share-2846","share_seq":2846,"template_height":800088},{"accepted_at_ms":999983,"credit_policy":"stale-grace","job_id":"job-1947","job_issued_at_ms":999992,"miner_id":"miner-0","network_difficulty":1000009,"ntime":1700001947,"order_key":"00:miner-0","p2mr_program_hex":"9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b","share_difficulty":2,"share_id":"share-1947","share_seq":1947,"template_height":800060},{"accepted_at_ms":999979,"credit_policy":null,"job_id":"job-3112","job_issued_at_ms":999979,"miner_id":"miner-10","network_difficulty":1000001,"ntime":1700003112,"order_key":"10:miner-10","p2mr_program_hex":"2828282828282828282828282828282828282828282828282828282828282828","share_difficulty":7,"share_id":"share-3112","share_seq":3112,"template_height":800097},{"accepted_at_ms":999995,"credit_policy":null,"job_id":"job-248","job_issued_at_ms":999982,"miner_id":"miner-6","network_difficulty":1000010,"ntime":1700000248,"order_key":"06:miner-6","p2mr_program_hex":"f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8","share_difficulty":3,"share_id":"share-248","share_seq":248,"template_height":800007},{"accepted_at_ms":999990,"credit_policy":null,"job_id":"job-2103","job_issued_at_ms":999992,"miner_id":"miner-2","network_difficulty":1000012,"ntime":1700002103,"order_key":"02:miner-2","p2mr_program_hex":"3737373737373737373737373737373737373737373737373737373737373737","share_difficulty":1,"share_id":"share-2103","share_seq":2103,"template_height":800065},{"accepted_at_ms":999978,"credit_policy":null,"job_id":"job-3726","job_issued_at_ms":999990,"miner_id":"miner-8","network_difficulty":1000003,"ntime":1700003726,"order_key":"08:miner-8","p2mr_program_hex":"8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e","share_difficulty":7,"share_id":"share-3726","share_seq":3726,"template_height":800116},{"accepted_at_ms":999973,"credit_policy":"stale-grace","job_id":"job-3479","job_issued_at_ms":999983,"miner_id":"miner-3","network_difficulty":1000011,"ntime":1700003479,"order_key":"03:miner-3","p2mr_program_hex":"9797979797979797979797979797979797979797979797979797979797979797","share_difficulty":7,"share_id":"share-3479","share_seq":3479,"template_height":800108},{"accepted_at_ms":999972,"credit_policy":null,"job_id":"job-3507","job_issued_at_ms":999989,"miner_id":"miner-9","network_difficulty":1000005,"ntime":1700003507,"order_key":"09:miner-9","p2mr_program_hex":"b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3","share_difficulty":7,"share_id":"share-3507","share_seq":3507,"template_height":800109},{"accepted_at_ms":1000139,"credit_policy":null,"job_id":"job-3224","job_issued_at_ms":999998,"miner_id":"miner-1","network_difficulty":1000011,"ntime":1700003224,"order_key":"01:miner-1","p2mr_program_hex":"9898989898989898989898989898989898989898989898989898989898989898","share_difficulty":1000,"share_id":"share-3224","share_seq":3224,"template_height":800100},{"accepted_at_ms":999984,"credit_policy":null,"job_id":"job-1967","job_issued_at_ms":999973,"miner_id":"miner-9","network_difficulty":1000012,"ntime":1700001967,"order_key":"09:miner-9","p2mr_program_hex":"afafafafafafafafafafafafafafafafafafafafafafafafafafafafafafafaf","share_difficulty":1000,"share_id":"share-1967","share_seq":1967,"template_height":800061},{"accepted_at_ms":999977,"credit_policy":null,"job_id":"job-1220","job_issued_at_ms":999980,"miner_id":"miner-10","network_difficulty":1000013,"ntime":1700001220,"order_key":"10:miner-10","p2mr_program_hex":"c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4","share_difficulty":2,"share_id":"share-1220","share_seq":1220,"template_height":800038},{"accepted_at_ms":999996,"credit_policy":"stale-grace","job_id":"job-243","job_issued_at_ms":999996,"miner_id":"miner-1","network_difficulty":1000005,"ntime":1700000243,"order_key":"01:miner-1","p2mr_program_hex":"f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3","share_difficulty":3,"share_id":"share-243","share_seq":243,"template_height":800007},{"accepted_at_ms":999970,"credit_policy":null,"job_id":"job-1830","job_issued_at_ms":999978,"miner_id":"miner-4","network_difficulty":1000011,"ntime":1700001830,"order_key":"04:miner-4","p2mr_program_hex":"2626262626262626262626262626262626262626262626262626262626262626","share_difficulty":101,"share_id":"share-1830","share_seq":1830,"template_height":800057},{"accepted_at_ms":999986,"credit_policy":null,"job_id":"job-3884","job_issued_at_ms":999982,"miner_id":"miner-1","network_difficulty":1000008,"ntime":1700003884,"order_key":"01:miner-1","p2mr_program_hex":"2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c","share_difficulty":1,"share_id":"share-3884","share_seq":3884,"template_height":800121},{"accepted_at_ms":999997,"credit_policy":null,"job_id":"job-3728","job_issued_at_ms":999999,"miner_id":"miner-10","network_difficulty":1000005,"ntime":1700003728,"order_key":"10:miner-10","p2mr_program_hex":"9090909090909090909090909090909090909090909090909090909090909090","share_difficulty":1000,"share_id":"share-3728","share_seq":3728,"template_height":800116},{"accepted_at_ms":999980,"credit_policy":null,"job_id":"job-263","job_issued_at_ms":999983,"miner_id":"miner-10","network_difficulty":1000008,"ntime":1700000263,"order_key":"10:miner-10","p2mr_program_hex":"0707070707070707070707070707070707070707070707070707070707070707","share_difficulty":3,"share_id":"share-263","share_seq":263,"template_height":800008},{"accepted_at_ms":999994,"credit_policy":null,"job_id":"job-1191","job_issued_at_ms":999995,"miner_id":"miner-3","network_difficulty":1000001,"ntime":1700001191,"order_key":"03:miner-3","p2mr_program_hex":"a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7","share_difficulty":19,"share_id":"share-1191","share_seq":1191,"template_height":800037},{"accepted_at_ms":999984,"credit_policy":null,"job_id":"job-2333","job_issued_at_ms":999980,"miner_id":"miner-1","network_difficulty":1000004,"ntime":1700002333,"order_key":"01:miner-1","p2mr_program_hex":"1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d","share_difficulty":2,"share_id":"share-2333","share_seq":2333,"template_height":800072},{"accepted_at_ms":999973,"credit_policy":"stale-grace","job_id":"job-2904","job_issued_at_ms":999992,"miner_id":"miner-0","network_difficulty":1000014,"ntime":1700002904,"order_key":"00:miner-0","p2mr_program_hex":"5858585858585858585858585858585858585858585858585858585858585858","share_difficulty":1000,"share_id":"share-2904","share_seq":2904,"template_height":800090},{"accepted_at_ms":999977,"credit_policy":null,"job_id":"job-1023","job_issued_at_ms":999996,"miner_id":"miner-0","network_difficulty":1000003,"ntime":1700001023,"order_key":"00:miner-0","p2mr_program_hex":"ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff","share_difficulty":7,"share_id":"share-1023","share_seq":1023,"template_height":800031},{"accepted_at_ms":999981,"credit_policy":null,"job_id":"job-2334","job_issued_at_ms":999992,"miner_id":"miner-2","network_difficulty":1000005,"ntime":1700002334,"order_key":"02:miner-2","p2mr_program_hex":"1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e","share_difficulty":1000,"share_id":"share-2334","share_seq":2334,"template_height":800072},{"accepted_at_ms":999993,"credit_policy":null,"job_id":"job-2602","job_issued_at_ms":999973,"miner_id":"miner-6","network_difficulty":1000001,"ntime":1700002602,"order_key":"06:miner-6","p2mr_program_hex":"2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a","share_difficulty":1000,"share_id":"share-2602","share_seq":2602,"template_height":800081},{"accepted_at_ms":999989,"credit_policy":null,"job_id":"job-1238","job_issued_at_ms":999998,"miner_id":"miner-6","network_difficulty":1000014,"ntime":1700001238,"order_key":"06:miner-6","p2mr_program_hex":"d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6","share_difficulty":101,"share_id":"share-1238","share_seq":1238,"template_height":800038},{"accepted_at_ms":999994,"credit_policy":"stale-grace","job_id":"job-2550","job_issued_at_ms":999991,"miner_id":"miner-9","network_difficulty":1000000,"ntime":1700002550,"order_key":"09:miner-9","p2mr_program_hex":"f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6","share_difficulty":2,"share_id":"share-2550","share_seq":2550,"template_height":800079},{"accepted_at_ms":999989,"credit_policy":null,"job_id":"job-1350","job_issued_at_ms":999989,"miner_id":"miner-8","network_difficulty":1000007,"ntime":1700001350,"order_key":"08:miner-8","p2mr_program_hex":"4646464646464646464646464646464646464646464646464646464646464646","share_difficulty":1,"share_id":"share-1350","share_seq":1350,"template_height":800042},{"accepted_at_ms":999970,"credit_policy":null,"job_id":"job-3299","job_issued_at_ms":999977,"miner_id":"miner-10","network_difficulty":1000001,"ntime":1700003299,"order_key":"10:miner-10","p2mr_program_hex":"e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3","share_difficulty":19,"share_id":"share-3299","share_seq":3299,"template_height":800103},{"accepted_at_ms":999988,"credit_policy":null,"job_id":"job-1842","job_issued_at_ms":999986,"miner_id":"miner-5","network_difficulty":1000006,"ntime":1700001842,"order_key":"05:miner-5","p2mr_program_hex":"3232323232323232323232323232323232323232323232323232323232323232","share_difficulty":3,"share_id":"share-1842","share_seq":1842,"template_height":800057},{"accepted_at_ms":999995,"credit_policy":null,"job_id":"job-814","job_issued_at_ms":999982,"miner_id":"miner-0","network_difficulty":1000015,"ntime":1700000814,"order_key":"00:miner-0","p2mr_program_hex":"2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e","share_difficulty":7,"share_id":"share-814","share_seq":814,"template_height":800025},{"accepted_at_ms":999990,"credit_policy":"stale-grace","job_id":"job-678","job_issued_at_ms":999975,"miner_id":"miner-7","network_difficulty":1000015,"ntime":1700000678,"order_key":"07:miner-7","p2mr_program_hex":"a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6","share_difficulty":3,"share_id":"share-678","share_seq":678,"template_height":800021},{"accepted_at_ms":999983,"credit_policy":null,"job_id":"job-749","job_issued_at_ms":999995,"miner_id":"miner-1","network_difficulty":1000001,"ntime":1700000749,"order_key":"01:miner-1","p2mr_program_hex":"edededededededededededededededededededededededededededededededed","share_difficulty":3,"share_id":"share-749","share_seq":749,"template_height":800023},{"accepted_at_ms":999985,"credit_policy":null,"job_id":"job-565","job_issued_at_ms":999993,"miner_id":"miner-4","network_difficulty":1000004,"ntime":1700000565,"order_key":"04:miner-4","p2mr_program_hex":"3535353535353535353535353535353535353535353535353535353535353535","share_difficulty":101,"share_id":"share-565","share_seq":565,"template_height":800017},{"accepted_at_ms":999985,"credit_policy":"stale-grace","job_id":"job-3582","job_issued_at_ms":999975,"miner_id":"miner-7","network_difficulty":1000012,"ntime":1700003582,"order_key":"07:miner-7","p2mr_program_hex":"fefefefefefefefefefefefefefefefefefefefefefefefefefefefefefefefe","share_difficulty":19,"share_id":"share-3582","share_seq":3582,"template_height":800111},{"accepted_at_ms":999996,"credit_policy":null,"job_id":"job-3594","job_issued_at_ms":999986,"miner_id":"miner-8","network_difficulty":1000007,"ntime":1700003594,"order_key":"08:miner-8","p2mr_program_hex":"0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a","share_difficulty":1000,"share_id":"share-3594","share_seq":3594,"template_height":800112},{"accepted_at_ms":999997,"credit_policy":null,"job_id":"job-3823","job_issued_at_ms":999999,"miner_id":"miner-6","network_difficulty":1000015,"ntime":1700003823,"order_key":"06:miner-6","p2mr_program_hex":"efefefefefefefefefefefefefefefefefefefefefefefefefefefefefefefef","share_difficulty":7,"share_id":"share-3823","share_seq":3823,"template_height":800119},{"accepted_at_ms":999993,"credit_policy":null,"job_id":"job-2137","job_issued_at_ms":999992,"miner_id":"miner-3","network_difficulty":1000012,"ntime":1700002137,"order_key":"03:miner-3","p2mr_program_hex":"5959595959595959595959595959595959595959595959595959595959595959","share_difficulty":3,"share_id":"share-2137","share_seq":2137,"template_height":800066},{"accepted_at_ms":999977,"credit_policy":null,"job_id":"job-2110","job_issued_at_ms":999988,"miner_id":"miner-9","network_difficulty":1000002,"ntime":1700002110,"order_key":"09:miner-9","p2mr_program_hex":"3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e","share_difficulty":101,"share_id":"share-2110","share_seq":2110,"template_height":800065},{"accepted_at_ms":999988,"credit_policy":null,"job_id":"job-1753","job_issued_at_ms":999972,"miner_id":"miner-4","network_difficulty":1000002,"ntime":1700001753,"order_key":"04:miner-4","p2mr_program_hex":"d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9","share_difficulty":19,"share_id":"share-1753","share_seq":1753,"template_height":800054},{"accepted_at_ms":999997,"credit_policy":"stale-grace","job_id":"job-539","job_issued_at_ms":999989,"miner_id":"miner-0","network_difficulty":1000012,"ntime":1700000539,"order_key":"00:miner-0","p2mr_program_hex":"1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b","share_difficulty":19,"share_id":"share-539","share_seq":539,"template_height":800016},{"accepted_at_ms":999970,"credit_policy":null,"job_id":"job-89","job_issued_at_ms":999980,"miner_id":"miner-1","network_difficulty":1000004,"ntime":1700000089,"order_key":"01:miner-1","p2mr_program_hex":"5959595959595959595959595959595959595959595959595959595959595959","share_difficulty":3,"share_id":"share-89","share_seq":89,"template_height":800002},{"accepted_at_ms":1000000,"credit_policy":null,"job_id":"job-1257","job_issued_at_ms":999977,"miner_id":"miner-3","network_difficulty":1000016,"ntime":1700001257,"order_key":"03:miner-3","p2mr_program_hex":"e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9","share_difficulty":19,"share_id":"share-1257","share_seq":1257,"template_height":800039},{"accepted_at_ms":999998,"credit_policy":null,"job_id":"job-1600","job_issued_at_ms":999971,"miner_id":"miner-5","network_difficulty":1000002,"ntime":1700001600,"order_key":"05:miner-5","p2mr_program_hex":"4040404040404040404040404040404040404040404040404040404040404040","share_difficulty":7,"share_id":"share-1600","share_seq":1600,"template_height":800050},{"accepted_at_ms":999991,"credit_policy":"stale-grace","job_id":"job-3009","job_issued_at_ms":999973,"miner_id":"miner-6","network_difficulty":1000000,"ntime":1700003009,"order_key":"06:miner-6","p2mr_program_hex":"c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1","share_difficulty":101,"share_id":"share-3009","share_seq":3009,"template_height":800094},{"accepted_at_ms":999995,"credit_policy":null,"job_id":"job-304","job_issued_at_ms":1000222,"miner_id":"miner-7","network_difficulty":1000015,"ntime":1700000304,"order_key":"07:miner-7","p2mr_program_hex":"3030303030303030303030303030303030303030303030303030303030303030","share_difficulty":19,"share_id":"share-304","share_seq":304,"template_height":800009},{"accepted_at_ms":999988,"credit_policy":null,"job_id":"job-647","job_issued_at_ms":999993,"miner_id":"miner-9","network_difficulty":1000001,"ntime":1700000647,"order_key":"09:miner-9","p2mr_program_hex":"8787878787878787878787878787878787878787878787878787878787878787","share_difficulty":1000,"share_id":"share-647","share_seq":647,"template_height":800020},{"accepted_at_ms":999989,"credit_policy":null,"job_id":"job-1230","job_issued_at_ms":999977,"miner_id":"miner-9","network_difficulty":1000006,"ntime":1700001230,"order_key":"09:miner-9","p2mr_program_hex":"cececececececececececececececececececececececececececececececece","share_difficulty":101,"share_id":"share-1230","share_seq":1230,"template_height":800038},{"accepted_at_ms":999994,"credit_policy":"stale-grace","job_id":"job-2409","job_issued_at_ms":999983,"miner_id":"miner-0","network_difficulty":1000012,"ntime":1700002409,"order_key":"00:miner-0","p2mr_program_hex":"6969696969696969696969696969696969696969696969696969696969696969","share_difficulty":1,"share_id":"share-2409","share_seq":2409,"template_height":800075},{"accepted_at_ms":999978,"credit_policy":"stale-grace","job_id":"job-534","job_issued_at_ms":999996,"miner_id":"miner-6","network_difficulty":1000007,"ntime":1700000534,"order_key":"06:miner-6","p2mr_program_hex":"1616161616161616161616161616161616161616161616161616161616161616","share_difficulty":101,"share_id":"share-534","share_seq":534,"template_height":800016},{"accepted_at_ms":999981,"credit_policy":null,"job_id":"job-945","job_issued_at_ms":999977,"miner_id":"miner-10","network_difficulty":1000010,"ntime":1700000945,"order_key":"10:miner-10","p2mr_program_hex":"b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1","share_difficulty":3,"share_id":"share-945","share_seq":945,"template_height":800029},{"accepted_at_ms":999995,"credit_policy":null,"job_id":"job-3935","job_issued_at_ms":999978,"miner_id":"miner-8","network_difficulty":1000008,"ntime":1700003935,"order_key":"08:miner-8","p2mr_program_hex":"5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f","share_difficulty":2,"share_id":"share-3935","share_seq":3935,"template_height":800122},{"accepted_at_ms":999978,"credit_policy":null,"job_id":"job-1560","job_issued_at_ms":999995,"miner_id":"miner-9","network_difficulty":1000013,"ntime":1700001560,"order_key":"09:miner-9","p2mr_program_hex":"1818181818181818181818181818181818181818181818181818181818181818","share_difficulty":7,"share_id":"share-1560","share_seq":1560,"template_height":800048},{"accepted_at_ms":1000000,"credit_policy":null,"job_id":"job-3613","job_issued_at_ms":999995,"miner_id":"miner-5","network_difficulty":1000009,"ntime":1700003613,"order_key":"05:miner-5","p2mr_program_hex":"1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d","share_difficulty":101,"share_id":"share-3613","share_seq":3613,"template_height":800112},{"accepted_at_ms":999985,"credit_policy":null,"job_id":"job-2340","job_issued_at_ms":999997,"miner_id":"miner-8","network_difficulty":1000011,"ntime":1700002340,"order_key":"08:miner-8","p2mr_program_hex":"2424242424242424242424242424242424242424242424242424242424242424","share_difficulty":1000,"share_id":"share-2340","share_seq":2340,"template_height":800073},{"accepted_at_ms":999984,"credit_policy":null,"job_id":"job-2974","job_issued_at_ms":999971,"miner_id":"miner-4","network_difficulty":1000016,"ntime":1700002974,"order_key":"04:miner-4","p2mr_program_hex":"9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e","share_difficulty":1,"share_id":"share-2974","share_seq":2974,"template_height":800092},{"accepted_at_ms":1000243,"credit_policy":"stale-grace","job_id":"job-2417","job_issued_at_ms":1000453,"miner_id":"miner-8","network_difficulty":1000003,"ntime":1700002417,"order_key":"08:miner-8","p2mr_program_hex":"7171717171717171717171717171717171717171717171717171717171717171","share_difficulty":1000,"share_id":"share-2417","share_seq":2417,"template_height":800075},{"accepted_at_ms":999995,"credit_policy":null,"job_id":"job-1204","job_issued_at_ms":999989,"miner_id":"miner-5","network_difficulty":1000014,"ntime":1700001204,"order_key":"05:miner-5","p2mr_program_hex":"b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4","share_difficulty":1,"share_id":"share-1204","share_seq":1204,"template_height":800037},{"accepted_at_ms":999993,"credit_policy":null,"job_id":"job-712","job_issued_at_ms":999970,"miner_id":"miner-8","network_difficulty":1000015,"ntime":1700000712,"order_key":"08:miner-8","p2mr_program_hex":"c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8","share_difficulty":19,"share_id":"share-712","share_seq":712,"template_height":800022},{"accepted_at_ms":999980,"credit_policy":null,"job_id":"job-2126","job_issued_at_ms":999999,"miner_id":"miner-3","network_difficulty":1000001,"ntime":1700002126,"order_key":"03:miner-3","p2mr_program_hex":"4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e","share_difficulty":101,"share_id":"share-2126","share_seq":2126,"template_height":800066},{"accepted_at_ms":999990,"credit_policy":"stale-grace","job_id":"job-3156","job_issued_at_ms":999979,"miner_id":"miner-10","network_difficulty":1000011,"ntime":1700003156,"order_key":"10:miner-10","p2mr_program_hex":"5454545454545454545454545454545454545454545454545454545454545454","share_difficulty":3,"share_id":"share-3156","share_seq":3156,"template_height":800098},{"accepted_at_ms":999995,"credit_policy":null,"job_id":"job-2829","job_issued_at_ms":999981,"miner_id":"miner-2","network_difficulty":1000007,"ntime":1700002829,"order_key":"02:miner-2","p2mr_program_hex":"0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d","share_difficulty":3,"share_id":"share-2829","share_seq":2829,"template_height":800088},{"accepted_at_ms":999971,"credit_policy":null,"job_id":"job-1787","job_issued_at_ms":999970,"miner_id":"miner-5","network_difficulty":1000002,"ntime":1700001787,"order_key":"05:miner-5","p2mr_program_hex":"fbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfb","share_difficulty":1000,"share_id":"share-1787","share_seq":1787,"template_height":800055},{"accepted_at_ms":999994,"credit_policy":null,"job_id":"job-2188","job_issued_at_ms":999974,"miner_id":"miner-10","network_difficulty":1000012,"ntime":1700002188,"order_key":"10:miner-10","p2mr_program_hex":"8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c","share_difficulty":101,"share_id":"share-2188","share_seq":2188,"template_height":800068},{"accepted_at_ms":999998,"credit_policy":null,"job_id":"job-121","job_issued_at_ms":999981,"miner_id":"miner-0","network_difficulty":1000002,"ntime":1700000121,"order_key":"00:miner-0","p2mr_program_hex":"7979797979797979797979797979797979797979797979797979797979797979","share_difficulty":101,"share_id":"share-121","share_seq":121,"template_height":800003},{"accepted_at_ms":999977,"credit_policy":null,"job_id":"job-1627","job_issued_at_ms":999972,"miner_id":"miner-10","network_difficulty":1000012,"ntime":1700001627,"order_key":"10:miner-10","p2mr_program_hex":"5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b","share_difficulty":19,"share_id":"share-1627","share_seq":1627,"template_height":800050},{"accepted_at_ms":999980,"credit_policy":null,"job_id":"job-958","job_issued_at_ms":999973,"miner_id":"miner-1","network_difficulty":1000006,"ntime":1700000958,"order_key":"01:miner-1","p2mr_program_hex":"bebebebebebebebebebebebebebebebebebebebebebebebebebebebebebebebe","share_difficulty":1,"share_id":"share-958","share_seq":958,"template_height":800029},{"accepted_at_ms":999991,"credit_policy":null,"job_id":"job-2803","job_issued_at_ms":999981,"miner_id":"miner-9","network_difficulty":1000015,"ntime":1700002803,"order_key":"09:miner-9","p2mr_program_hex":"f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3","share_difficulty":2,"share_id":"share-2803","share_seq":2803,"template_height":800087},{"accepted_at_ms":999970,"credit_policy":null,"job_id":"job-3948","job_issued_at_ms":999986,"miner_id":"miner-10","network_difficulty":1000004,"ntime":1700003948,"order_key":"10:miner-10","p2mr_program_hex":"6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c","share_difficulty":101,"share_id":"share-3948","share_seq":3948,"template_height":800123},{"accepted_at_ms":999983,"credit_policy":null,"job_id":"job-59","job_issued_at_ms":999980,"miner_id":"miner-4","network_difficulty":1000008,"ntime":1700000059,"order_key":"04:miner-4","p2mr_program_hex":"3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b","share_difficulty":3,"share_id":"share-59","share_seq":59,"template_height":800001},{"accepted_at_ms":999975,"credit_policy":null,"job_id":"job-2252","job_issued_at_ms":1000000,"miner_id":"miner-8","network_difficulty":1000008,"ntime":1700002252,"order_key":"08:miner-8","p2mr_program_hex":"cccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccc","share_difficulty":19,"share_id":"share-2252","share_seq":2252,"template_height":800070},{"accepted_at_ms":999997,"credit_policy":"stale-grace","job_id":"job-650","job_issued_at_ms":999974,"miner_id":"miner-1","network_difficulty":1000004,"ntime":1700000650,"order_key":"01:miner-1","p2mr_program_hex":"8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a","share_difficulty":1000,"share_id":"share-650","share_seq":650,"template_height":800020},{"accepted_at_ms":999997,"credit_policy":null,"job_id":"job-2811","job_issued_at_ms":999996,"miner_id":"miner-6","network_difficulty":1000006,"ntime":1700002811,"order_key":"06:miner-6","p2mr_program_hex":"fbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfb","share_difficulty":1,"share_id":"share-2811","share_seq":2811,"template_height":800087},{"accepted_at_ms":999991,"credit_policy":"stale-grace","job_id":"job-3036","job_issued_at_ms":999976,"miner_id":"miner-0","network_difficulty":1000010,"ntime":1700003036,"order_key":"00:miner-0","p2mr_program_hex":"dcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdc","share_difficulty":2,"share_id":"share-3036","share_seq":3036,"template_height":800094},{"accepted_at_ms":999990,"credit_policy":"stale-grace","job_id":"job-279","job_issued_at_ms":999976,"miner_id":"miner-4","network_difficulty":1000007,"ntime":1700000279,"order_key":"04:miner-4","p2mr_program_hex":"1717171717171717171717171717171717171717171717171717171717171717","share_difficulty":101,"share_id":"share-279","share_seq":279,"template_height":800008},{"accepted_at_ms":999974,"credit_policy":null,"job_id":"job-1396","job_issued_at_ms":999997,"miner_id":"miner-10","network_difficulty":1000002,"ntime":1700001396,"order_key":"10:miner-10","p2mr_program_hex":"7474747474747474747474747474747474747474747474747474747474747474","share_difficulty":2,"share_id":"share-1396","share_seq":1396,"template_height":800043},{"accepted_at_ms":999972,"credit_policy":null,"job_id":"job-352","job_issued_at_ms":999983,"miner_id":"miner-0","network_difficulty":1000012,"ntime":1700000352,"order_key":"00:miner-0","p2mr_program_hex":"6060606060606060606060606060606060606060606060606060606060606060","share_difficulty":7,"share_id":"share-352","share_seq":352,"template_height":800011},{"accepted_at_ms":999985,"credit_policy":"stale-grace","job_id":"job-3982","job_issued_at_ms":999986,"miner_id":"miner-0","network_difficulty":1000004,"ntime":1700003982,"order_key":"00:miner-0","p2mr_program_hex":"8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e","share_difficulty":1,"share_id":"share-3982","share_seq":3982,"template_height":800124},{"accepted_at_ms":999984,"credit_policy":null,"job_id":"job-284","job_issued_at_ms":999985,"miner_id":"miner-9","network_difficulty":1000012,"ntime":1700000284,"order_key":"09:miner-9","p2mr_program_hex":"1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c","share_difficulty":19,"share_id":"share-284","share_seq":284,"template_height":800008},{"accepted_at_ms":1000000,"credit_policy":null,"job_id":"job-2986","job_issued_at_ms":999987,"miner_id":"miner-5","network_difficulty":1000011,"ntime":1700002986,"order_key":"05:miner-5","p2mr_program_hex":"aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa","share_difficulty":7,"share_id":"share-2986","share_seq":2986,"template_height":800093},{"accepted_at_ms":999974,"credit_policy":null,"job_id":"job-250","job_issued_at_ms":999981,"miner_id":"miner-8","network_difficulty":1000012,"ntime":1700000250,"order_key":"08:miner-8","p2mr_program_hex":"fafafafafafafafafafafafafafafafafafafafafafafafafafafafafafafafa","share_difficulty":2,"share_id":"share-250","share_seq":250,"template_height":800007},{"accepted_at_ms":1000078,"credit_policy":null,"job_id":"job-2765","job_issued_at_ms":999980,"miner_id":"miner-4","network_difficulty":1000011,"ntime":1700002765,"order_key":"04:miner-4","p2mr_program_hex":"cdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcd","share_difficulty":1,"share_id":"share-2765","share_seq":2765,"template_height":800086},{"accepted_at_ms":999998,"credit_policy":null,"job_id":"job-2909","job_issued_at_ms":999983,"miner_id":"miner-5","network_difficulty":1000002,"ntime":1700002909,"order_key":"05:miner-5","p2mr_program_hex":"5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d","share_difficulty":1,"share_id":"share-2909","share_seq":2909,"template_height":800090},{"accepted_at_ms":999970,"credit_policy":null,"job_id":"job-500","job_issued_at_ms":999989,"miner_id":"miner-5","network_difficulty":1000007,"ntime":1700000500,"order_key":"05:miner-5","p2mr_program_hex":"f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4","share_difficulty":101,"share_id":"share-500","share_seq":500,"template_height":800015},{"accepted_at_ms":999979,"credit_policy":null,"job_id":"job-524","job_issued_at_ms":999988,"miner_id":"miner-7","network_difficulty":1000014,"ntime":1700000524,"order_key":"07:miner-7","p2mr_program_hex":"0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c","share_difficulty":1,"share_id":"share-524","share_seq":524,"template_height":800016},{"accepted_at_ms":999998,"credit_policy":"stale-grace","job_id":"job-1766","job_issued_at_ms":999997,"miner_id":"miner-6","network_difficulty":1000015,"ntime":1700001766,"order_key":"06:miner-6","p2mr_program_hex":"e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6","share_difficulty":7,"share_id":"share-1766","share_seq":1766,"template_height":800055},{"accepted_at_ms":999986,"credit_policy":null,"job_id":"job-1618","job_issued_at_ms":999976,"miner_id":"miner-1","network_difficulty":1000003,"ntime":1700001618,"order_key":"01:miner-1","p2mr_program_hex":"5252525252525252525252525252525252525252525252525252525252525252","share_difficulty":101,"share_id":"share-1618","share_seq":1618,"template_height":800050},{"accepted_at_ms":999978,"credit_policy":null,"job_id":"job-1956","job_issued_at_ms":999974,"miner_id":"miner-9","network_difficulty":1000001,"ntime":1700001956,"order_key":"09:miner-9","p2mr_program_hex":"a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4","share_difficulty":1000,"share_id":"share-1956","share_seq":1956,"template_height":800061},{"accepted_at_ms":999986,"credit_policy":null,"job_id":"job-3059","job_issued_at_ms":999986,"miner_id":"miner-1","network_difficulty":1000016,"ntime":1700003059,"order_key":"01:miner-1","p2mr_program_hex":"f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3","share_difficulty":19,"share_id":"share-3059","share_seq":3059,"template_height":800095},{"accepted_at_ms":999979,"credit_policy":"stale-grace","job_id":"job-1045","job_issued_at_ms":999982,"miner_id":"miner-0","network_difficulty":1000008,"ntime":1700001045,"order_key":"00:miner-0","p2mr_program_hex":"1515151515151515151515151515151515151515151515151515151515151515","share_difficulty":1,"share_id":"share-1045","share_seq":1045,"template_height":800032},{"accepted_at_ms":999973,"credit_policy":null,"job_id":"job-2463","job_issued_at_ms":999998,"miner_id":"miner-10","network_difficulty":1000015,"ntime":1700002463,"order_key":"10:miner-10","p2mr_program_hex":"9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f","share_difficulty":1000,"share_id":"share-2463","share_seq":2463,"template_height":800076},{"accepted_at_ms":999997,"credit_policy":null,"job_id":"job-2534","job_issued_at_ms":999978,"miner_id":"miner-4","network_difficulty":1000001,"ntime":1700002534,"order_key":"04:miner-4","p2mr_program_hex":"e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6","share_difficulty":2,"share_id":"share-2534","share_seq":2534,"template_height":800079},{"accepted_at_ms":999974,"credit_policy":null,"job_id":"job-2251","job_issued_at_ms":999974,"miner_id":"miner-7","network_difficulty":1000007,"ntime":1700002251,"order_key":"07:miner-7","p2mr_program_hex":"cbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcb","share_difficulty":3,"share_id":"share-2251","share_seq":2251,"template_height":800070},{"accepted_at_ms":999978,"credit_policy":null,"job_id":"job-1988","job_issued_at_ms":999976,"miner_id":"miner-8","network_difficulty":1000016,"ntime":1700001988,"order_key":"08:miner-8","p2mr_program_hex":"c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4","share_difficulty":7,"share_id":"share-1988","share_seq":1988,"template_height":800062},{"accepted_at_ms":999980,"credit_policy":null,"job_id":"job-2044","job_issued_at_ms":999985,"miner_id":"miner-9","network_difficulty":1000004,"ntime":1700002044,"order_key":"09:miner-9","p2mr_program_hex":"fcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfc","share_difficulty":3,"share_id":"share-2044","share_seq":2044,"template_height":800063},{"accepted_at_ms":999989,"credit_policy":null,"job_id":"job-256","job_issued_at_ms":999986,"miner_id":"miner-3","network_difficulty":1000001,"ntime":1700000256,"order_key":"03:miner-3","p2mr_program_hex":"0000000000000000000000000000000000000000000000000000000000000000","share_difficulty":1,"share_id":"share-256","share_seq":256,"template_height":800008},{"accepted_at_ms":999995,"credit_policy":null,"job_id":"job-1612","job_issued_at_ms":999981,"miner_id":"miner-6","network_difficulty":1000014,"ntime":1700001612,"order_key":"06:miner-6","p2mr_program_hex":"4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c","share_difficulty":7,"share_id":"share-1612","share_seq":1612,"template_height":800050},{"accepted_at_ms":999976,"credit_policy":null,"job_id":"job-3645","job_issued_at_ms":999993,"miner_id":"miner-4","network_difficulty":1000007,"ntime":1700003645,"order_key":"04:miner-4","p2mr_program_hex":"3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d","share_difficulty":2,"share_id":"share-3645","share_seq":3645,"template_height":800113},{"accepted_at_ms":999992,"credit_policy":"stale-grace","job_id":"job-2689","job_issued_at_ms":999982,"miner_id":"miner-5","network_difficulty":1000003,"ntime":1700002689,"order_key":"05:miner-5","p2mr_program_hex":"8181818181818181818181818181818181818181818181818181818181818181","share_difficulty":101,"share_id":"share-2689","share_seq":2689,"template_height":800084},{"accepted_at_ms":999992,"credit_policy":null,"job_id":"job-3485","job_issued_at_ms":999983,"miner_id":"miner-9","network_difficulty":1000000,"ntime":1700003485,"order_key":"09:miner-9","p2mr_program_hex":"9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d","share_difficulty":101,"share_id":"share-3485","share_seq":3485,"template_height":800108},{"accepted_at_ms":999999,"credit_policy":null,"job_id":"job-677","job_issued_at_ms":999974,"miner_id":"miner-6","network_difficulty":1000014,"ntime":1700000677,"order_key":"06:miner-6","p2mr_program_hex":"a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5","share_difficulty":3,"share_id":"share-677","share_seq":677,"template_height":800021},{"accepted_at_ms":999991,"credit_policy":null,"job_id":"job-1927","job_issued_at_ms":999979,"miner_id":"miner-2","network_difficulty":1000006,"ntime":1700001927,"order_key":"02:miner-2","p2mr_program_hex":"8787878787878787878787878787878787878787878787878787878787878787","share_difficulty":19,"share_id":"share-1927","share_seq":1927,"template_height":800060},{"accepted_at_ms":999988,"credit_policy":null,"job_id":"job-3924","job_issued_at_ms":999970,"miner_id":"miner-8","network_difficulty":1000014,"ntime":1700003924,"order_key":"08:miner-8","p2mr_program_hex":"5454545454545454545454545454545454545454545454545454545454545454","share_difficulty":101,"share_id":"share-3924","share_seq":3924,"template_height":800122},{"accepted_at_ms":999996,"credit_policy":null,"job_id":"job-238","job_issued_at_ms":999973,"miner_id":"miner-7","network_difficulty":1000000,"ntime":1700000238,"order_key":"07:miner-7","p2mr_program_hex":"eeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeee","share_difficulty":3,"share_id":"share-238","share_seq":238,"template_height":800007},{"accepted_at_ms":999974,"credit_policy":null,"job_id":"job-1445","job_issued_at_ms":999999,"miner_id":"miner-4","network_difficulty":1000000,"ntime":1700001445,"order_key":"04:miner-4","p2mr_program_hex":"a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5","share_difficulty":19,"share_id":"share-1445","share_seq":1445,"template_height":800045},{"accepted_at_ms":999985,"credit_policy":null,"job_id":"job-3828","job_issued_at_ms":999991,"miner_id":"miner-0","network_difficulty":1000003,"ntime":1700003828,"order_key":"00:miner-0","p2mr_program_hex":"f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4","share_difficulty":101,"share_id":"share-3828","share_seq":3828,"template_height":800119},{"accepted_at_ms":999992,"credit_policy":null,"job_id":"job-1437","job_issued_at_ms":999980,"miner_id":"miner-7","network_difficulty":1000009,"ntime":1700001437,"order_key":"07:miner-7","p2mr_program_hex":"9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d","share_difficulty":101,"share_id":"share-1437","share_seq":1437,"template_height":800044},{"accepted_at_ms":999989,"credit_policy":null,"job_id":"job-64","job_issued_at_ms":999972,"miner_id":"miner-9","network_difficulty":1000013,"ntime":1700000064,"order_key":"09:miner-9","p2mr_program_hex":"4040404040404040404040404040404040404040404040404040404040404040","share_difficulty":3,"share_id":"share-64","share_seq":64,"template_height":800002},{"accepted_at_ms":1000155,"credit_policy":"stale-grace","job_id":"job-2628","job_issued_at_ms":1000416,"miner_id":"miner-10","network_difficulty":1000010,"ntime":1700002628,"order_key":"10:miner-10","p2mr_program_hex":"4444444444444444444444444444444444444444444444444444444444444444","share_difficulty":2,"share_id":"share-2628","share_seq":2628,"template_height":800082},{"accepted_at_ms":999971,"credit_policy":null,"job_id":"job-3668","job_issued_at_ms":999996,"miner_id":"miner-5","network_difficulty":1000013,"ntime":1700003668,"order_key":"05:miner-5","p2mr_program_hex":"5454545454545454545454545454545454545454545454545454545454545454","share_difficulty":7,"share_id":"share-3668","share_seq":3668,"template_height":800114},{"accepted_at_ms":999973,"credit_policy":null,"job_id":"job-1265","job_issued_at_ms":999973,"miner_id":"miner-0","network_difficulty":1000007,"ntime":1700001265,"order_key":"00:miner-0","p2mr_program_hex":"f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1","share_difficulty":3,"share_id":"share-1265","share_seq":1265,"template_height":800039},{"accepted_at_ms":999991,"credit_policy":null,"job_id":"job-27","job_issued_at_ms":999994,"miner_id":"miner-5","network_difficulty":1000010,"ntime":1700000027,"order_key":"05:miner-5","p2mr_program_hex":"1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b","share_difficulty":101,"share_id":"share-27","share_seq":27,"template_height":800000},{"accepted_at_ms":999986,"credit_policy":"stale-grace","job_id":"job-898","job_issued_at_ms":999989,"miner_id":"miner-7","network_difficulty":1000014,"ntime":1700000898,"order_key":"07:miner-7","p2mr_program_hex":"8282828282828282828282828282828282828282828282828282828282828282","share_difficulty":19,"share_id":"share-898","share_seq":898,"template_height":800028},{"accepted_at_ms":999999,"credit_policy":null,"job_id":"job-1073","job_issued_at_ms":999978,"miner_id":"miner-6","network_difficulty":1000002,"ntime":1700001073,"order_key":"06:miner-6","p2mr_program_hex":"3131313131313131313131313131313131313131313131313131313131313131","share_difficulty":2,"share_id":"share-1073","share_seq":1073,"template_height":800033},{"accepted_at_ms":999973,"credit_policy":null,"job_id":"job-40","job_issued_at_ms":999992,"miner_id":"miner-7","network_difficulty":1000006,"ntime":1700000040,"order_key":"07:miner-7","p2mr_program_hex":"2828282828282828282828282828282828282828282828282828282828282828","share_difficulty":7,"share_id":"share-40","share_seq":40,"template_height":800001},{"accepted_at_ms":999998,"credit_policy":"stale-grace","job_id":"job-2849","job_issued_at_ms":999993,"miner_id":"miner-0","network_difficulty":1000010,"ntime":1700002849,"order_key":"00:miner-0","p2mr_program_hex":"2121212121212121212121212121212121212121212121212121212121212121","share_difficulty":19,"share_id":"share-2849","share_seq":2849,"template_height":800089},{"accepted_at_ms":999978,"credit_policy":null,"job_id":"job-2905","job_issued_at_ms":999979,"miner_id":"miner-1","network_difficulty":1000015,"ntime":1700002905,"order_key":"01:miner-1","p2mr_program_hex":"5959595959595959595959595959595959595959595959595959595959595959","share_difficulty":7,"share_id":"share-2905","share_seq":2905,"template_height":800090},{"accepted_at_ms":999983,"credit_policy":null,"job_id":"job-2818","job_issued_at_ms":999977,"miner_id":"miner-2","network_difficulty":1000013,"ntime":1700002818,"order_key":"02:miner-2","p2mr_program_hex":"0202020202020202020202020202020202020202020202020202020202020202","share_difficulty":1000,"share_id":"share-2818","share_seq":2818,"template_height":800088},{"accepted_at_ms":999986,"credit_policy":"stale-grace","job_id":"job-2928","job_issued_at_ms":999992,"miner_id":"miner-2","network_difficulty":1000004,"ntime":1700002928,"order_key":"02:miner-2","p2mr_program_hex":"7070707070707070707070707070707070707070707070707070707070707070","share_difficulty":3,"share_id":"share-2928","share_seq":2928,"template_height":800091},{"accepted_at_ms":999991,"credit_policy":null,"job_id":"job-2751","job_issued_at_ms":999996,"miner_id":"miner-1","network_difficulty":1000014,"ntime":1700002751,"order_key":"01:miner-1","p2mr_program_hex":"bfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbf","share_difficulty":19,"share_id":"share-2751","share_seq":2751,"template_height":800085},{"accepted_at_ms":999999,"credit_policy":null,"job_id":"job-1472","job_issued_at_ms":999971,"miner_id":"miner-9","network_difficulty":1000010,"ntime":1700001472,"order_key":"09:miner-9","p2mr_program_hex":"c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0","share_difficulty":1000,"share_id":"share-1472","share_seq":1472,"template_height":800046},{"accepted_at_ms":999986,"credit_policy":null,"job_id":"job-879","job_issued_at_ms":999985,"miner_id":"miner-10","network_difficulty":1000012,"ntime":1700000879,"order_key":"10:miner-10","p2mr_program_hex":"6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f","share_difficulty":19,"share_id":"share-879","share_seq":879,"template_height":800027},{"accepted_at_ms":999971,"credit_policy":null,"job_id":"job-3839","job_issued_at_ms":999980,"miner_id":"miner-0","network_difficulty":1000014,"ntime":1700003839,"order_key":"00:miner-0","p2mr_program_hex":"ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff","share_difficulty":19,"share_id":"share-3839","share_seq":3839,"template_height":800119},{"accepted_at_ms":999993,"credit_policy":"stale-grace","job_id":"job-3616","job_issued_at_ms":999971,"miner_id":"miner-8","network_difficulty":1000012,"ntime":1700003616,"order_key":"08:miner-8","p2mr_program_hex":"2020202020202020202020202020202020202020202020202020202020202020","share_difficulty":101,"share_id":"share-3616","share_seq":3616,"template_height":800113},{"accepted_at_ms":999970,"credit_policy":null,"job_id":"job-2027","job_issued_at_ms":999998,"miner_id":"miner-3","network_difficulty":1000004,"ntime":1700002027,"order_key":"03:miner-3","p2mr_program_hex":"ebebebebebebebebebebebebebebebebebebebebebebebebebebebebebebebeb","share_difficulty":2,"share_id":"share-2027","share_seq":2027,"template_height":800063},{"accepted_at_ms":999998,"credit_policy":null,"job_id":"job-1430","job_issued_at_ms":999998,"miner_id":"miner-0","network_difficulty":1000002,"ntime":1700001430,"order_key":"00:miner-0","p2mr_program_hex":"9696969696969696969696969696969696969696969696969696969696969696","share_difficulty":101,"share_id":"share-1430","share_seq":1430,"template_height":800044},{"accepted_at_ms":999997,"credit_policy":null,"job_id":"job-3466","job_issued_at_ms":999984,"miner_id":"miner-1","network_difficulty":1000015,"ntime":1700003466,"order_key":"01:miner-1","p2mr_program_hex":"8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a","share_difficulty":1000,"share_id":"share-3466","share_seq":3466,"template_height":800108},{"accepted_at_ms":999990,"credit_policy":null,"job_id":"job-2776","job_issued_at_ms":999980,"miner_id":"miner-4","network_difficulty":1000005,"ntime":1700002776,"order_key":"04:miner-4","p2mr_program_hex":"d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8","share_difficulty":2,"share_id":"share-2776","share_seq":2776,"template_height":800086},{"accepted_at_ms":999993,"credit_policy":null,"job_id":"job-1909","job_issued_at_ms":999992,"miner_id":"miner-6","network_difficulty":1000005,"ntime":1700001909,"order_key":"06:miner-6","p2mr_program_hex":"7575757575757575757575757575757575757575757575757575757575757575","share_difficulty":1,"share_id":"share-1909","share_seq":1909,"template_height":800059},{"accepted_at_ms":999989,"credit_policy":null,"job_id":"job-977","job_issued_at_ms":999974,"miner_id":"miner-9","network_difficulty":1000008,"ntime":1700000977,"order_key":"09:miner-9","p2mr_program_hex":"d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1","share_difficulty":3,"share_id":"share-977","share_seq":977,"template_height":800030},{"accepted_at_ms":999997,"credit_policy":null,"job_id":"job-3351","job_issued_at_ms":999971,"miner_id":"miner-7","network_difficulty":1000002,"ntime":1700003351,"order_key":"07:miner-7","p2mr_program_hex":"1717171717171717171717171717171717171717171717171717171717171717","share_difficulty":1000,"share_id":"share-3351","share_seq":3351,"template_height":800104},{"accepted_at_ms":999983,"credit_policy":null,"job_id":"job-774","job_issued_at_ms":999992,"miner_id":"miner-4","network_difficulty":1000009,"ntime":1700000774,"order_key":"04:miner-4","p2mr_program_hex":"0606060606060606060606060606060606060606060606060606060606060606","share_difficulty":101,"share_id":"share-774","share_seq":774,"template_height":800024},{"accepted_at_ms":999997,"credit_policy":null,"job_id":"job-3640","job_issued_at_ms":999984,"miner_id":"miner-10","network_difficulty":1000002,"ntime":1700003640,"order_key":"10:miner-10","p2mr_program_hex":"3838383838383838383838383838383838383838383838383838383838383838","share_difficulty":19,"share_id":"share-3640","share_seq":3640,"template_height":800113},{"accepted_at_ms":999991,"credit_policy":null,"job_id":"job-1175","job_issued_at_ms":999978,"miner_id":"miner-9","network_difficulty":1000002,"ntime":1700001175,"order_key":"09:miner-9","p2mr_program_hex":"9797979797979797979797979797979797979797979797979797979797979797","share_difficulty":3,"share_id":"share-1175","share_seq":1175,"template_height":800036},{"accepted_at_ms":999976,"credit_policy":null,"job_id":"job-1362","job_issued_at_ms":999996,"miner_id":"miner-9","network_difficulty":1000002,"ntime":1700001362,"order_key":"09:miner-9","p2mr_program_hex":"5252525252525252525252525252525252525252525252525252525252525252","share_difficulty":7,"share_id":"share-1362","share_seq":1362,"template_height":800042},{"accepted_at_ms":999994,"credit_policy":null,"job_id":"job-1376","job_issued_at_ms":999975,"miner_id":"miner-1","network_difficulty":1000016,"ntime":1700001376,"order_key":"01:miner-1","p2mr_program_hex":"6060606060606060606060606060606060606060606060606060606060606060","share_difficulty":19,"share_id":"share-1376","share_seq":1376,"template_height":800043},{"accepted_at_ms":999996,"credit_policy":null,"job_id":"job-3863","job_issued_at_ms":999974,"miner_id":"miner-2","network_difficulty":1000004,"ntime":1700003863,"order_key":"02:miner-2","p2mr_program_hex":"1717171717171717171717171717171717171717171717171717171717171717","share_difficulty":3,"share_id":"share-3863","share_seq":3863,"template_height":800120},{"accepted_at_ms":999982,"credit_policy":null,"job_id":"job-3216","job_issued_at_ms":999994,"miner_id":"miner-4","network_difficulty":1000003,"ntime":1700003216,"order_key":"04:miner-4","p2mr_program_hex":"9090909090909090909090909090909090909090909090909090909090909090","share_difficulty":2,"share_id":"share-3216","share_seq":3216,"template_height":800100},{"accepted_at_ms":999998,"credit_policy":null,"job_id":"job-1703","job_issued_at_ms":999976,"miner_id":"miner-9","network_difficulty":1000003,"ntime":1700001703,"order_key":"09:miner-9","p2mr_program_hex":"a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7","share_difficulty":2,"share_id":"share-1703","share_seq":1703,"template_height":800053},{"accepted_at_ms":999970,"credit_policy":"stale-grace","job_id":"job-1550","job_issued_at_ms":999995,"miner_id":"miner-10","network_difficulty":1000003,"ntime":1700001550,"order_key":"10:miner-10","p2mr_program_hex":"0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e","share_difficulty":7,"share_id":"share-1550","share_seq":1550,"template_height":800048},{"accepted_at_ms":1000353,"credit_policy":null,"job_id":"job-1109","job_issued_at_ms":999986,"miner_id":"miner-9","network_difficulty":1000004,"ntime":1700001109,"order_key":"09:miner-9","p2mr_program_hex":"5555555555555555555555555555555555555555555555555555555555555555","share_difficulty":3,"share_id":"share-1109","share_seq":1109,"template_height":800034},{"accepted_at_ms":999973,"credit_policy":"stale-grace","job_id":"job-3162","job_issued_at_ms":999980,"miner_id":"miner-5","network_difficulty":1000000,"ntime":1700003162,"order_key":"05:miner-5","p2mr_program_hex":"5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a","share_difficulty":1000,"share_id":"share-3162","share_seq":3162,"template_height":800098},{"accepted_at_ms":999995,"credit_policy":null,"job_id":"job-1012","job_issued_at_ms":999991,"miner_id":"miner-0","network_difficulty":1000009,"ntime":1700001012,"order_key":"00:miner-0","p2mr_program_hex":"f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4","share_difficulty":101,"share_id":"share-1012","share_seq":1012,"template_height":800031},{"accepted_at_ms":999976,"credit_policy":"stale-grace","job_id":"job-1434","job_issued_at_ms":999992,"miner_id":"miner-4","network_difficulty":1000006,"ntime":1700001434,"order_key":"04:miner-4","p2mr_program_hex":"9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a","share_difficulty":19,"share_id":"share-1434","share_seq":1434,"template_height":800044},{"accepted_at_ms":999988,"credit_policy":"stale-grace","job_id":"job-1706","job_issued_at_ms":1000449,"miner_id":"miner-1","network_difficulty":1000006,"ntime":1700001706,"order_key":"01:miner-1","p2mr_program_hex":"aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa","share_difficulty":19,"share_id":"share-1706","share_seq":1706,"template_height":800053},{"accepted_at_ms":999984,"credit_policy":null,"job_id":"job-1151","job_issued_at_ms":999990,"miner_id":"miner-7","network_difficulty":1000012,"ntime":1700001151,"order_key":"07:miner-7","p2mr_program_hex":"7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f","share_difficulty":3,"share_id":"share-1151","share_seq":1151,"template_height":800035},{"accepted_at_ms":999998,"credit_policy":null,"job_id":"job-3692","job_issued_at_ms":999995,"miner_id":"miner-7","network_difficulty":1000003,"ntime":1700003692,"order_key":"07:miner-7","p2mr_program_hex":"6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c","share_difficulty":3,"share_id":"share-3692","share_seq":3692,"template_height":800115},{"accepted_at_ms":999971,"credit_policy":"stale-grace","job_id":"job-38","job_issued_at_ms":999997,"miner_id":"miner-5","network_difficulty":1000004,"ntime":1700000038,"order_key":"05:miner-5","p2mr_program_hex":"2626262626262626262626262626262626262626262626262626262626262626","share_difficulty":7,"share_id":"share-38","share_seq":38,"template_height":800001},{"accepted_at_ms":999989,"credit_policy":null,"job_id":"job-3057","job_issued_at_ms":999991,"miner_id":"miner-10","network_difficulty":1000014,"ntime":1700003057,"order_key":"10:miner-10","p2mr_program_hex":"f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1","share_difficulty":1000,"share_id":"share-3057","share_seq":3057,"template_height":800095},{"accepted_at_ms":999972,"credit_policy":null,"job_id":"job-2187","job_issued_at_ms":999970,"miner_id":"miner-9","network_difficulty":1000011,"ntime":1700002187,"order_key":"09:miner-9","p2mr_program_hex":"8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b","share_difficulty":2,"share_id":"share-2187","share_seq":2187,"template_height":800068},{"accepted_at_ms":999970,"credit_policy":null,"job_id":"job-2495","job_issued_at_ms":999993,"miner_id":"miner-9","network_difficulty":1000013,"ntime":1700002495,"order_key":"09:miner-9","p2mr_program_hex":"bfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbf","share_difficulty":19,"share_id":"share-2495","share_seq":2495,"template_height":800077},{"accepted_at_ms":999986,"credit_policy":null,"job_id":"job-2388","job_issued_at_ms":999988,"miner_id":"miner-1","network_difficulty":1000008,"ntime":1700002388,"order_key":"01:miner-1","p2mr_program_hex":"5454545454545454545454545454545454545454545454545454545454545454","share_difficulty":2,"share_id":"share-2388","share_seq":2388,"template_height":800074},{"accepted_at_ms":999988,"credit_policy":null,"job_id":"job-859","job_issued_at_ms":999973,"miner_id":"miner-1","network_difficulty":1000009,"ntime":1700000859,"order_key":"01:miner-1","p2mr_program_hex":"5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b","share_difficulty":19,"share_id":"share-859","share_seq":859,"template_height":800026},{"accepted_at_ms":999975,"credit_policy":null,"job_id":"job-2659","job_issued_at_ms":999991,"miner_id":"miner-8","network_difficulty":1000007,"ntime":1700002659,"order_key":"08:miner-8","p2mr_program_hex":"6363636363636363636363636363636363636363636363636363636363636363","share_difficulty":19,"share_id":"share-2659","share_seq":2659,"template_height":800083},{"accepted_at_ms":999979,"credit_policy":null,"job_id":"job-3540","job_issued_at_ms":999970,"miner_id":"miner-9","network_difficulty":1000004,"ntime":1700003540,"order_key":"09:miner-9","p2mr_program_hex":"d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4","share_difficulty":101,"share_id":"share-3540","share_seq":3540,"template_height":800110},{"accepted_at_ms":999982,"credit_policy":null,"job_id":"job-3661","job_issued_at_ms":999995,"miner_id":"miner-9","network_difficulty":1000006,"ntime":1700003661,"order_key":"09:miner-9","p2mr_program_hex":"4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d","share_difficulty":19,"share_id":"share-3661","share_seq":3661,"template_height":800114},{"accepted_at_ms":1000000,"credit_policy":"stale-grace","job_id":"job-1888","job_issued_at_ms":999975,"miner_id":"miner-7","network_difficulty":1000001,"ntime":1700001888,"order_key":"07:miner-7","p2mr_program_hex":"6060606060606060606060606060606060606060606060606060606060606060","share_difficulty":101,"share_id":"share-1888","share_seq":1888,"template_height":800059},{"accepted_at_ms":999988,"credit_policy":null,"job_id":"job-8","job_issued_at_ms":999978,"miner_id":"miner-8","network_difficulty":1000008,"ntime":1700000008,"order_key":"08:miner-8","p2mr_program_hex":"0808080808080808080808080808080808080808080808080808080808080808","share_difficulty":19,"share_id":"share-8","share_seq":8,"template_height":800000},{"accepted_at_ms":999991,"credit_policy":null,"job_id":"job-1280","job_issued_at_ms":999988,"miner_id":"miner-4","network_difficulty":1000005,"ntime":1700001280,"order_key":"04:miner-4","p2mr_program_hex":"0000000000000000000000000000000000000000000000000000000000000000","share_difficulty":2,"share_id":"share-1280","share_seq":1280,"template_height":800040},{"accepted_at_ms":999991,"credit_policy":null,"job_id":"job-3714","job_issued_at_ms":999976,"miner_id":"miner-7","network_difficulty":1000008,"ntime":1700003714,"order_key":"07:miner-7","p2mr_program_hex":"8282828282828282828282828282828282828282828282828282828282828282","share_difficulty":2,"share_id":"share-3714","share_seq":3714,"template_height":800116},{"accepted_at_ms":1000000,"credit_policy":"stale-grace","job_id":"job-477","job_issued_at_ms":999990,"miner_id":"miner-4","network_difficulty":1000001,"ntime":1700000477,"order_key":"04:miner-4","p2mr_program_hex":"dddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddd","share_difficulty":1000,"share_id":"share-477","share_seq":477,"template_height":800014},{"accepted_at_ms":999998,"credit_policy":null,"job_id":"job-2398","job_issued_at_ms":999999,"miner_id":"miner-0","network_difficulty":1000001,"ntime":1700002398,"order_key":"00:miner-0","p2mr_program_hex":"5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e","share_difficulty":3,"share_id":"share-2398","share_seq":2398,"template_height":800074},{"accepted_at_ms":999991,"credit_policy":null,"job_id":"job-96","job_issued_at_ms":999994,"miner_id":"miner-8","network_difficulty":1000011,"ntime":1700000096,"order_key":"08:miner-8","p2mr_program_hex":"6060606060606060606060606060606060606060606060606060606060606060","share_difficulty":7,"share_id":"share-96","share_seq":96,"template_height":800003},{"accepted_at_ms":999991,"credit_policy":null,"job_id":"job-134","job_issued_at_ms":999971,"miner_id":"miner-2","network_difficulty":1000015,"ntime":1700000134,"order_key":"02:miner-2","p2mr_program_hex":"8686868686868686868686868686868686868686868686868686868686868686","share_difficulty":101,"share_id":"share-134","share_seq":134,"template_height":800004},{"accepted_at_ms":999987,"credit_policy":null,"job_id":"job-2935","job_issued_at_ms":999973,"miner_id":"miner-9","network_difficulty":1000011,"ntime":1700002935,"order_key":"09:miner-9","p2mr_program_hex":"7777777777777777777777777777777777777777777777777777777777777777","share_difficulty":3,"share_id":"share-2935","share_seq":2935,"template_height":800091},{"accepted_at_ms":999985,"credit_policy":null,"job_id":"job-236","job_issued_at_ms":999988,"miner_id":"miner-5","network_difficulty":1000015,"ntime":1700000236,"order_key":"05:miner-5","p2mr_program_hex":"ecececececececececececececececececececececececececececececececec","share_difficulty":19,"share_id":"share-236","share_seq":236,"template_height":800007},{"accepted_at_ms":999984,"credit_policy":null,"job_id":"job-2165","job_issued_at_ms":999993,"miner_id":"miner-9","network_difficulty":1000006,"ntime":1700002165,"order_key":"09:miner-9","p2mr_program_hex":"7575757575757575757575757575757575757575757575757575757575757575","share_difficulty":3,"share_id":"share-2165","share_seq":2165,"template_height":800067},{"accepted_at_ms":999974,"credit_policy":"stale-grace","job_id":"job-3991","job_issued_at_ms":999978,"miner_id":"miner-9","network_difficulty":1000013,"ntime":1700003991,"order_key":"09:miner-9","p2mr_program_hex":"9797979797979797979797979797979797979797979797979797979797979797","share_difficulty":7,"share_id":"share-3991","share_seq":3991,"template_height":800124},{"accepted_at_ms":999989,"credit_policy":"stale-grace","job_id":"job-413","job_issued_at_ms":999990,"miner_id":"miner-6","network_difficulty":1000005,"ntime":1700000413,"order_key":"06:miner-6","p2mr_program_hex":"9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d","share_difficulty":19,"share_id":"share-413","share_seq":413,"template_height":800012},{"accepted_at_ms":999999,"credit_policy":null,"job_id":"job-2638","job_issued_at_ms":999984,"miner_id":"miner-9","network_difficulty":1000003,"ntime":1700002638,"order_key":"09:miner-9","p2mr_program_hex":"4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e","share_difficulty":19,"share_id":"share-2638","share_seq":2638,"template_height":800082},{"accepted_at_ms":999983,"credit_policy":"stale-grace","job_id":"job-2101","job_issued_at_ms":1000000,"miner_id":"miner-0","network_difficulty":1000010,"ntime":1700002101,"order_key":"00:miner-0","p2mr_program_hex":"3535353535353535353535353535353535353535353535353535353535353535","share_difficulty":1,"share_id":"share-2101","share_seq":2101,"template_height":800065},{"accepted_at_ms":999981,"credit_policy":"stale-grace","job_id":"job-2670","job_issued_at_ms":999982,"miner_id":"miner-8","network_difficulty":1000001,"ntime":1700002670,"order_key":"08:miner-8","p2mr_program_hex":"6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e","share_difficulty":2,"share_id":"share-2670","share_seq":2670,"template_height":800083},{"accepted_at_ms":999999,"credit_policy":null,"job_id":"job-2076","job_issued_at_ms":999999,"miner_id":"miner-8","network_difficulty":1000002,"ntime":1700002076,"order_key":"08:miner-8","p2mr_program_hex":"1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c","share_difficulty":101,"share_id":"share-2076","share_seq":2076,"template_height":800064},{"accepted_at_ms":999992,"credit_policy":"stale-grace","job_id":"job-2084","job_issued_at_ms":999983,"miner_id":"miner-5","network_difficulty":1000010,"ntime":1700002084,"order_key":"05:miner-5","p2mr_program_hex":"2424242424242424242424242424242424242424242424242424242424242424","share_difficulty":101,"share_id":"share-2084","share_seq":2084,"template_height":800065},{"accepted_at_ms":999999,"credit_policy":null,"job_id":"job-2451","job_issued_at_ms":999995,"miner_id":"miner-9","network_difficulty":1000003,"ntime":1700002451,"order_key":"09:miner-9","p2mr_program_hex":"9393939393939393939393939393939393939393939393939393939393939393","share_difficulty":1,"share_id":"share-2451","share_seq":2451,"template_height":800076},{"accepted_at_ms":999982,"credit_policy":null,"job_id":"job-2232","job_issued_at_ms":999976,"miner_id":"miner-10","network_difficulty":1000005,"ntime":1700002232,"order_key":"10:miner-10","p2mr_program_hex":"b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8","share_difficulty":3,"share_id":"share-2232","share_seq":2232,"template_height":800069},{"accepted_at_ms":999994,"credit_policy":null,"job_id":"job-961","job_issued_at_ms":999998,"miner_id":"miner-4","network_difficulty":1000009,"ntime":1700000961,"order_key":"04:miner-4","p2mr_program_hex":"c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1","share_difficulty":7,"share_id":"share-961","share_seq":961,"template_height":800030},{"accepted_at_ms":999981,"credit_policy":null,"job_id":"job-1390","job_issued_at_ms":999996,"miner_id":"miner-4","network_difficulty":1000013,"ntime":1700001390,"order_key":"04:miner-4","p2mr_program_hex":"6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e","share_difficulty":2,"share_id":"share-1390","share_seq":1390,"template_height":800043},{"accepted_at_ms":999983,"credit_policy":null,"job_id":"job-1584","job_issued_at_ms":999997,"miner_id":"miner-0","network_difficulty":1000003,"ntime":1700001584,"order_key":"00:miner-0","p2mr_program_hex":"3030303030303030303030303030303030303030303030303030303030303030","share_difficulty":2,"share_id":"share-1584","share_seq":1584,"template_height":800049},{"accepted_at_ms":999987,"credit_policy":null,"job_id":"job-2324","job_issued_at_ms":999988,"miner_id":"miner-3","network_difficulty":1000012,"ntime":1700002324,"order_key":"03:miner-3","p2mr_program_hex":"1414141414141414141414141414141414141414141414141414141414141414","share_difficulty":101,"share_id":"share-2324","share_seq":2324,"template_height":800072},{"accepted_at_ms":999999,"credit_policy":null,"job_id":"job-1192","job_issued_at_ms":999995,"miner_id":"miner-4","network_difficulty":1000002,"ntime":1700001192,"order_key":"04:miner-4","p2mr_program_hex":"a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8","share_difficulty":1,"share_id":"share-1192","share_seq":1192,"template_height":800037},{"accepted_at_ms":999984,"credit_policy":null,"job_id":"job-3995","job_issued_at_ms":999986,"miner_id":"miner-2","network_difficulty":1000000,"ntime":1700003995,"order_key":"02:miner-2","p2mr_program_hex":"9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b","share_difficulty":19,"share_id":"share-3995","share_seq":3995,"template_height":800124},{"accepted_at_ms":999982,"credit_policy":"stale-grace","job_id":"job-1573","job_issued_at_ms":999988,"miner_id":"miner-0","network_difficulty":1000009,"ntime":1700001573,"order_key":"00:miner-0","p2mr_program_hex":"2525252525252525252525252525252525252525252525252525252525252525","share_difficulty":7,"share_id":"share-1573","share_seq":1573,"template_height":800049},{"accepted_at_ms":999982,"credit_policy":"stale-grace","job_id":"job-1887","job_issued_at_ms":999975,"miner_id":"miner-6","network_difficulty":1000000,"ntime":1700001887,"order_key":"06:miner-6","p2mr_program_hex":"5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f","share_difficulty":7,"share_id":"share-1887","share_seq":1887,"template_height":800058},{"accepted_at_ms":999970,"credit_policy":null,"job_id":"job-2240","job_issued_at_ms":999976,"miner_id":"miner-7","network_difficulty":1000013,"ntime":1700002240,"order_key":"07:miner-7","p2mr_program_hex":"c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0","share_difficulty":3,"share_id":"share-2240","share_seq":2240,"template_height":800070},{"accepted_at_ms":999992,"credit_policy":null,"job_id":"job-1024","job_issued_at_ms":999980,"miner_id":"miner-1","network_difficulty":1000004,"ntime":1700001024,"order_key":"01:miner-1","p2mr_program_hex":"0000000000000000000000000000000000000000000000000000000000000000","share_difficulty":3,"share_id":"share-1024","share_seq":1024,"template_height":800032},{"accepted_at_ms":999972,"credit_policy":"stale-grace","job_id":"job-1696","job_issued_at_ms":999976,"miner_id":"miner-2","network_difficulty":1000013,"ntime":1700001696,"order_key":"02:miner-2","p2mr_program_hex":"a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0","share_difficulty":3,"share_id":"share-1696","share_seq":1696,"template_height":800053},{"accepted_at_ms":999979,"credit_policy":"stale-grace","job_id":"job-3171","job_issued_at_ms":999985,"miner_id":"miner-3","network_difficulty":1000009,"ntime":1700003171,"order_key":"03:miner-3","p2mr_program_hex":"6363636363636363636363636363636363636363636363636363636363636363","share_difficulty":1,"share_id":"share-3171","share_seq":3171,"template_height":800099},{"accepted_at_ms":999977,"credit_policy":null,"job_id":"job-415","job_issued_at_ms":999980,"miner_id":"miner-8","network_difficulty":1000007,"ntime":1700000415,"order_key":"08:miner-8","p2mr_program_hex":"9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f","share_difficulty":2,"share_id":"share-415","share_seq":415,"template_height":800012},{"accepted_at_ms":999993,"credit_policy":null,"job_id":"job-3242","job_issued_at_ms":999987,"miner_id":"miner-8","network_difficulty":1000012,"ntime":1700003242,"order_key":"08:miner-8","p2mr_program_hex":"aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa","share_difficulty":1,"share_id":"share-3242","share_seq":3242,"template_height":800101},{"accepted_at_ms":999998,"credit_policy":"stale-grace","job_id":"job-3030","job_issued_at_ms":1000440,"miner_id":"miner-5","network_difficulty":1000004,"ntime":1700003030,"order_key":"05:miner-5","p2mr_program_hex":"d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6","share_difficulty":19,"share_id":"share-3030","share_seq":3030,"template_height":800094},{"accepted_at_ms":999998,"credit_policy":null,"job_id":"job-60","job_issued_at_ms":999990,"miner_id":"miner-5","network_difficulty":1000009,"ntime":1700000060,"order_key":"05:miner-5","p2mr_program_hex":"3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c","share_difficulty":3,"share_id":"share-60","share_seq":60,"template_height":800001},{"accepted_at_ms":999975,"credit_policy":null,"job_id":"job-1268","job_issued_at_ms":999981,"miner_id":"miner-3","network_difficulty":1000010,"ntime":1700001268,"order_key":"03:miner-3","p2mr_program_hex":"f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4","share_difficulty":3,"share_id":"share-1268","share_seq":1268,"template_height":800039},{"accepted_at_ms":999995,"credit_policy":null,"job_id":"job-988","job_issued_at_ms":999971,"miner_id":"miner-9","network_difficulty":1000002,"ntime":1700000988,"order_key":"09:miner-9","p2mr_program_hex":"dcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdc","share_difficulty":3,"share_id":"share-988","share_seq":988,"template_height":800030},{"accepted_at_ms":999991,"credit_policy":null,"job_id":"job-3","job_issued_at_ms":999974,"miner_id":"miner-3","network_difficulty":1000003,"ntime":1700000003,"order_key":"03:miner-3","p2mr_program_hex":"0303030303030303030303030303030303030303030303030303030303030303","share_difficulty":1000,"share_id":"share-3","share_seq":3,"template_height":800000},{"accepted_at_ms":999994,"credit_policy":null,"job_id":"job-1743","job_issued_at_ms":999980,"miner_id":"miner-5","network_difficulty":1000009,"ntime":1700001743,"order_key":"05:miner-5","p2mr_program_hex":"cfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcf","share_difficulty":19,"share_id":"share-1743","share_seq":1743,"template_height":800054},{"accepted_at_ms":999998,"credit_policy":null,"job_id":"job-1556","job_issued_at_ms":999980,"miner_id":"miner-5","network_difficulty":1000009,"ntime":1700001556,"order_key":"05:miner-5","p2mr_program_hex":"1414141414141414141414141414141414141414141414141414141414141414","share_difficulty":7,"share_id":"share-1556","share_seq":1556,"template_height":800048},{"accepted_at_ms":999988,"credit_policy":null,"job_id":"job-2323","job_issued_at_ms":999971,"miner_id":"miner-2","network_difficulty":1000011,"ntime":1700002323,"order_key":"02:miner-2","p2mr_program_hex":"1313131313131313131313131313131313131313131313131313131313131313","share_difficulty":19,"share_id":"share-2323","share_seq":2323,"template_height":800072},{"accepted_at_ms":999971,"credit_policy":null,"job_id":"job-1603","job_issued_at_ms":999995,"miner_id":"miner-8","network_difficulty":1000005,"ntime":1700001603,"order_key":"08:miner-8","p2mr_program_hex":"4343434343434343434343434343434343434343434343434343434343434343","share_difficulty":7,"share_id":"share-1603","share_seq":1603,"template_height":800050},{"accepted_at_ms":999988,"credit_policy":null,"job_id":"job-1565","job_issued_at_ms":999970,"miner_id":"miner-3","network_difficulty":1000001,"ntime":1700001565,"order_key":"03:miner-3","p2mr_program_hex":"1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d","share_difficulty":101,"share_id":"share-1565","share_seq":1565,"template_height":800048},{"accepted_at_ms":999977,"credit_policy":null,"job_id":"job-2381","job_issued_at_ms":999985,"miner_id":"miner-5","network_difficulty":1000001,"ntime":1700002381,"order_key":"05:miner-5","p2mr_program_hex":"4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d","share_difficulty":3,"share_id":"share-2381","share_seq":2381,"template_height":800074},{"accepted_at_ms":999985,"credit_policy":null,"job_id":"job-1531","job_issued_at_ms":999983,"miner_id":"miner-2","network_difficulty":1000001,"ntime":1700001531,"order_key":"02:miner-2","p2mr_program_hex":"fbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfb","share_difficulty":2,"share_id":"share-1531","share_seq":1531,"template_height":800047},{"accepted_at_ms":999980,"credit_policy":"stale-grace","job_id":"job-3673","job_issued_at_ms":999970,"miner_id":"miner-10","network_difficulty":1000001,"ntime":1700003673,"order_key":"10:miner-10","p2mr_program_hex":"5959595959595959595959595959595959595959595959595959595959595959","share_difficulty":1000,"share_id":"share-3673","share_seq":3673,"template_height":800114},{"accepted_at_ms":999985,"credit_policy":null,"job_id":"job-2296","job_issued_at_ms":999974,"miner_id":"miner-8","network_difficulty":1000001,"ntime":1700002296,"order_key":"08:miner-8","p2mr_program_hex":"f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8","share_difficulty":1000,"share_id":"share-2296","share_seq":2296,"template_height":800071},{"accepted_at_ms":999991,"credit_policy":null,"job_id":"job-2706","job_issued_at_ms":999972,"miner_id":"miner-0","network_difficulty":1000003,"ntime":1700002706,"order_key":"00:miner-0","p2mr_program_hex":"9292929292929292929292929292929292929292929292929292929292929292","share_difficulty":3,"share_id":"share-2706","share_seq":2706,"template_height":800084},{"accepted_at_ms":1000147,"credit_policy":null,"job_id":"job-3860","job_issued_at_ms":999988,"miner_id":"miner-10","network_difficulty":1000001,"ntime":1700003860,"order_key":"10:miner-10","p2mr_program_hex":"1414141414141414141414141414141414141414141414141414141414141414","share_difficulty":101,"share_id":"share-3860","share_seq":3860,"template_height":800120},{"accepted_at_ms":999972,"credit_policy":null,"job_id":"job-2160","job_issued_at_ms":999970,"miner_id":"miner-4","network_difficulty":1000001,"ntime":1700002160,"order_key":"04:miner-4","p2mr_program_hex":"7070707070707070707070707070707070707070707070707070707070707070","share_difficulty":1000,"share_id":"share-2160","share_seq":2160,"template_height":800067},{"accepted_at_ms":999972,"credit_policy":"stale-grace","job_id":"job-3202","job_issued_at_ms":999990,"miner_id":"miner-1","network_difficulty":1000006,"ntime":1700003202,"order_key":"01:miner-1","p2mr_program_hex":"8282828282828282828282828282828282828282828282828282828282828282","share_difficulty":1,"share_id":"share-3202","share_seq":3202,"template_height":800100},{"accepted_at_ms":1000481,"credit_policy":"stale-grace","job_id":"job-1812","job_issued_at_ms":999976,"miner_id":"miner-8","network_difficulty":1000010,"ntime":1700001812,"order_key":"08:miner-8","p2mr_program_hex":"1414141414141414141414141414141414141414141414141414141414141414","share_difficulty":3,"share_id":"share-1812","share_seq":1812,"template_height":800056},{"accepted_at_ms":999995,"credit_policy":null,"job_id":"job-2039","job_issued_at_ms":999972,"miner_id":"miner-4","network_difficulty":1000016,"ntime":1700002039,"order_key":"04:miner-4","p2mr_program_hex":"f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7","share_difficulty":1000,"share_id":"share-2039","share_seq":2039,"template_height":800063},{"accepted_at_ms":999971,"credit_policy":"stale-grace","job_id":"job-597","job_issued_at_ms":999977,"miner_id":"miner-3","network_difficulty":1000002,"ntime":1700000597,"order_key":"03:miner-3","p2mr_program_hex":"5555555555555555555555555555555555555555555555555555555555555555","share_difficulty":1,"share_id":"share-597","share_seq":597,"template_height":800018},{"accepted_at_ms":999979,"credit_policy":null,"job_id":"job-947","job_issued_at_ms":999999,"miner_id":"miner-1","network_difficulty":1000012,"ntime":1700000947,"order_key":"01:miner-1","p2mr_program_hex":"b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3","share_difficulty":19,"share_id":"share-947","share_seq":947,"template_height":800029},{"accepted_at_ms":999979,"credit_policy":null,"job_id":"job-66","job_issued_at_ms":999984,"miner_id":"miner-0","network_difficulty":1000015,"ntime":1700000066,"order_key":"00:miner-0","p2mr_program_hex":"4242424242424242424242424242424242424242424242424242424242424242","share_difficulty":7,"share_id":"share-66","share_seq":66,"template_height":800002},{"accepted_at_ms":999974,"credit_policy":null,"job_id":"job-3336","job_issued_at_ms":999974,"miner_id":"miner-3","network_difficulty":1000004,"ntime":1700003336,"order_key":"03:miner-3","p2mr_program_hex":"0808080808080808080808080808080808080808080808080808080808080808","share_difficulty":101,"share_id":"share-3336","share_seq":3336,"template_height":800104},{"accepted_at_ms":999979,"credit_policy":null,"job_id":"job-3314","job_issued_at_ms":999995,"miner_id":"miner-3","network_difficulty":1000016,"ntime":1700003314,"order_key":"03:miner-3","p2mr_program_hex":"f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2","share_difficulty":7,"share_id":"share-3314","share_seq":3314,"template_height":800103},{"accepted_at_ms":999998,"credit_policy":null,"job_id":"job-2715","job_issued_at_ms":999990,"miner_id":"miner-9","network_difficulty":1000012,"ntime":1700002715,"order_key":"09:miner-9","p2mr_program_hex":"9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b","share_difficulty":1000,"share_id":"share-2715","share_seq":2715,"template_height":800084},{"accepted_at_ms":999996,"credit_policy":null,"job_id":"job-726","job_issued_at_ms":999971,"miner_id":"miner-0","network_difficulty":1000012,"ntime":1700000726,"order_key":"00:miner-0","p2mr_program_hex":"d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6","share_difficulty":101,"share_id":"share-726","share_seq":726,"template_height":800022},{"accepted_at_ms":999973,"credit_policy":null,"job_id":"job-856","job_issued_at_ms":999985,"miner_id":"miner-9","network_difficulty":1000006,"ntime":1700000856,"order_key":"09:miner-9","p2mr_program_hex":"5858585858585858585858585858585858585858585858585858585858585858","share_difficulty":7,"share_id":"share-856","share_seq":856,"template_height":800026},{"accepted_at_ms":999998,"credit_policy":"stale-grace","job_id":"job-2432","job_issued_at_ms":999975,"miner_id":"miner-1","network_difficulty":1000001,"ntime":1700002432,"order_key":"01:miner-1","p2mr_program_hex":"8080808080808080808080808080808080808080808080808080808080808080","share_difficulty":2,"share_id":"share-2432","share_seq":2432,"template_height":800076},{"accepted_at_ms":999987,"credit_policy":null,"job_id":"job-2554","job_issued_at_ms":999992,"miner_id":"miner-2","network_difficulty":1000004,"ntime":1700002554,"order_key":"02:miner-2","p2mr_program_hex":"fafafafafafafafafafafafafafafafafafafafafafafafafafafafafafafafa","share_difficulty":1,"share_id":"share-2554","share_seq":2554,"template_height":800079},{"accepted_at_ms":999972,"credit_policy":null,"job_id":"job-2327","job_issued_at_ms":999986,"miner_id":"miner-6","network_difficulty":1000015,"ntime":1700002327,"order_key":"06:miner-6","p2mr_program_hex":"1717171717171717171717171717171717171717171717171717171717171717","share_difficulty":2,"share_id":"share-2327","share_seq":2327,"template_height":800072},{"accepted_at_ms":999973,"credit_policy":null,"job_id":"job-1431","job_issued_at_ms":999978,"miner_id":"miner-1","network_difficulty":1000003,"ntime":1700001431,"order_key":"01:miner-1","p2mr_program_hex":"9797979797979797979797979797979797979797979797979797979797979797","share_difficulty":101,"share_id":"share-1431","share_seq":1431,"template_height":800044},{"accepted_at_ms":999992,"credit_policy":null,"job_id":"job-1882","job_issued_at_ms":999980,"miner_id":"miner-1","network_difficulty":1000012,"ntime":1700001882,"order_key":"01:miner-1","p2mr_program_hex":"5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a","share_difficulty":1,"share_id":"share-1882","share_seq":1882,"template_height":800058},{"accepted_at_ms":999982,"credit_policy":"stale-grace","job_id":"job-656","job_issued_at_ms":999975,"miner_id":"miner-7","network_difficulty":1000010,"ntime":1700000656,"order_key":"07:miner-7","p2mr_program_hex":"9090909090909090909090909090909090909090909090909090909090909090","share_difficulty":1000,"share_id":"share-656","share_seq":656,"template_height":800020},{"accepted_at_ms":999982,"credit_policy":null,"job_id":"job-965","job_issued_at_ms":999986,"miner_id":"miner-8","network_difficulty":1000013,"ntime":1700000965,"order_key":"08:miner-8","p2mr_program_hex":"c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5","share_difficulty":101,"share_id":"share-965","share_seq":965,"template_height":800030},{"accepted_at_ms":999972,"credit_policy":"stale-grace","job_id":"job-812","job_issued_at_ms":999998,"miner_id":"miner-9","network_difficulty":1000013,"ntime":1700000812,"order_key":"09:miner-9","p2mr_program_hex":"2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c","share_difficulty":2,"share_id":"share-812","share_seq":812,"template_height":800025},{"accepted_at_ms":999986,"credit_policy":null,"job_id":"job-3835","job_issued_at_ms":999981,"miner_id":"miner-7","network_difficulty":1000010,"ntime":1700003835,"order_key":"07:miner-7","p2mr_program_hex":"fbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfb","share_difficulty":101,"share_id":"share-3835","share_seq":3835,"template_height":800119},{"accepted_at_ms":999994,"credit_policy":null,"job_id":"job-2059","job_issued_at_ms":999982,"miner_id":"miner-2","network_difficulty":1000002,"ntime":1700002059,"order_key":"02:miner-2","p2mr_program_hex":"0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b","share_difficulty":101,"share_id":"share-2059","share_seq":2059,"template_height":800064},{"accepted_at_ms":999999,"credit_policy":null,"job_id":"job-1198","job_issued_at_ms":999990,"miner_id":"miner-10","network_difficulty":1000008,"ntime":1700001198,"order_key":"10:miner-10","p2mr_program_hex":"aeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeae","share_difficulty":2,"share_id":"share-1198","share_seq":1198,"template_height":800037},{"accepted_at_ms":999997,"credit_policy":null,"job_id":"job-1734","job_issued_at_ms":999981,"miner_id":"miner-7","network_difficulty":1000000,"ntime":1700001734,"order_key":"07:miner-7","p2mr_program_hex":"c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6","share_difficulty":2,"share_id":"share-1734","share_seq":1734,"template_height":800054},{"accepted_at_ms":999987,"credit_policy":null,"job_id":"job-3989","job_issued_at_ms":999998,"miner_id":"miner-7","network_difficulty":1000011,"ntime":1700003989,"order_key":"07:miner-7","p2mr_program_hex":"9595959595959595959595959595959595959595959595959595959595959595","share_difficulty":19,"share_id":"share-3989","share_seq":3989,"template_height":800124},{"accepted_at_ms":999970,"credit_policy":null,"job_id":"job-58","job_issued_at_ms":999971,"miner_id":"miner-3","network_difficulty":1000007,"ntime":1700000058,"order_key":"03:miner-3","p2mr_program_hex":"3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a","share_difficulty":7,"share_id":"share-58","share_seq":58,"template_height":800001},{"accepted_at_ms":999993,"credit_policy":null,"job_id":"job-224","job_issued_at_ms":999970,"miner_id":"miner-4","network_difficulty":1000003,"ntime":1700000224,"order_key":"04:miner-4","p2mr_program_hex":"e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0","share_difficulty":3,"share_id":"share-224","share_seq":224,"template_height":800007},{"accepted_at_ms":1000380,"credit_policy":null,"job_id":"job-616","job_issued_at_ms":1000398,"miner_id":"miner-0","network_difficulty":1000004,"ntime":1700000616,"order_key":"00:miner-0","p2mr_program_hex":"6868686868686868686868686868686868686868686868686868686868686868","share_difficulty":101,"share_id":"share-616","share_seq":616,"template_height":800019},{"accepted_at_ms":999997,"credit_policy":null,"job_id":"job-3657","job_issued_at_ms":999976,"miner_id":"miner-5","network_difficulty":1000002,"ntime":1700003657,"order_key":"05:miner-5","p2mr_program_hex":"4949494949494949494949494949494949494949494949494949494949494949","share_difficulty":1,"share_id":"share-3657","share_seq":3657,"template_height":800114},{"accepted_at_ms":999988,"credit_policy":null,"job_id":"job-463","job_issued_at_ms":999990,"miner_id":"miner-1","network_difficulty":1000004,"ntime":1700000463,"order_key":"01:miner-1","p2mr_program_hex":"cfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcf","share_difficulty":1,"share_id":"share-463","share_seq":463,"template_height":800014},{"accepted_at_ms":999974,"credit_policy":null,"job_id":"job-2016","job_issued_at_ms":999978,"miner_id":"miner-3","network_difficulty":1000010,"ntime":1700002016,"order_key":"03:miner-3","p2mr_program_hex":"e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0","share_difficulty":2,"share_id":"share-2016","share_seq":2016,"template_height":800063},{"accepted_at_ms":999976,"credit_policy":null,"job_id":"job-3542","job_issued_at_ms":999976,"miner_id":"miner-0","network_difficulty":1000006,"ntime":1700003542,"order_key":"00:miner-0","p2mr_program_hex":"d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6","share_difficulty":101,"share_id":"share-3542","share_seq":3542,"template_height":800110},{"accepted_at_ms":999995,"credit_policy":null,"job_id":"job-3412","job_issued_at_ms":999974,"miner_id":"miner-2","network_difficulty":1000012,"ntime":1700003412,"order_key":"02:miner-2","p2mr_program_hex":"5454545454545454545454545454545454545454545454545454545454545454","share_difficulty":19,"share_id":"share-3412","share_seq":3412,"template_height":800106},{"accepted_at_ms":999996,"credit_policy":null,"job_id":"job-1076","job_issued_at_ms":999994,"miner_id":"miner-9","network_difficulty":1000005,"ntime":1700001076,"order_key":"09:miner-9","p2mr_program_hex":"3434343434343434343434343434343434343434343434343434343434343434","share_difficulty":19,"share_id":"share-1076","share_seq":1076,"template_height":800033},{"accepted_at_ms":999998,"credit_policy":"stale-grace","job_id":"job-1164","job_issued_at_ms":999983,"miner_id":"miner-9","network_difficulty":1000008,"ntime":1700001164,"order_key":"09:miner-9","p2mr_program_hex":"8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c","share_difficulty":101,"share_id":"share-1164","share_seq":1164,"template_height":800036},{"accepted_at_ms":999986,"credit_policy":"stale-grace","job_id":"job-214","job_issued_at_ms":999991,"miner_id":"miner-5","network_difficulty":1000010,"ntime":1700000214,"order_key":"05:miner-5","p2mr_program_hex":"d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6","share_difficulty":1000,"share_id":"share-214","share_seq":214,"template_height":800006},{"accepted_at_ms":999995,"credit_policy":null,"job_id":"job-3405","job_issued_at_ms":1000150,"miner_id":"miner-6","network_difficulty":1000005,"ntime":1700003405,"order_key":"06:miner-6","p2mr_program_hex":"4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d","share_difficulty":2,"share_id":"share-3405","share_seq":3405,"template_height":800106},{"accepted_at_ms":999997,"credit_policy":"stale-grace","job_id":"job-3969","job_issued_at_ms":1000000,"miner_id":"miner-9","network_difficulty":1000008,"ntime":1700003969,"order_key":"09:miner-9","p2mr_program_hex":"8181818181818181818181818181818181818181818181818181818181818181","share_difficulty":7,"share_id":"share-3969","share_seq":3969,"template_height":800124},{"accepted_at_ms":999976,"credit_policy":"stale-grace","job_id":"job-3515","job_issued_at_ms":999987,"miner_id":"miner-6","network_difficulty":1000013,"ntime":1700003515,"order_key":"06:miner-6","p2mr_program_hex":"bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb","share_difficulty":101,"share_id":"share-3515","share_seq":3515,"template_height":800109},{"accepted_at_ms":999970,"credit_policy":null,"job_id":"job-1765","job_issued_at_ms":999977,"miner_id":"miner-5","network_difficulty":1000014,"ntime":1700001765,"order_key":"05:miner-5","p2mr_program_hex":"e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5","share_difficulty":1000,"share_id":"share-1765","share_seq":1765,"template_height":800055},{"accepted_at_ms":999975,"credit_policy":null,"job_id":"job-2153","job_issued_at_ms":999988,"miner_id":"miner-8","network_difficulty":1000011,"ntime":1700002153,"order_key":"08:miner-8","p2mr_program_hex":"6969696969696969696969696969696969696969696969696969696969696969","share_difficulty":3,"share_id":"share-2153","share_seq":2153,"template_height":800067},{"accepted_at_ms":999977,"credit_policy":null,"job_id":"job-2575","job_issued_at_ms":999977,"miner_id":"miner-1","network_difficulty":1000008,"ntime":1700002575,"order_key":"01:miner-1","p2mr_program_hex":"0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f","share_difficulty":2,"share_id":"share-2575","share_seq":2575,"template_height":800080},{"accepted_at_ms":999979,"credit_policy":null,"job_id":"job-226","job_issued_at_ms":999982,"miner_id":"miner-6","network_difficulty":1000005,"ntime":1700000226,"order_key":"06:miner-6","p2mr_program_hex":"e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2","share_difficulty":2,"share_id":"share-226","share_seq":226,"template_height":800007},{"accepted_at_ms":1000000,"credit_policy":null,"job_id":"job-2733","job_issued_at_ms":999973,"miner_id":"miner-5","network_difficulty":1000013,"ntime":1700002733,"order_key":"05:miner-5","p2mr_program_hex":"adadadadadadadadadadadadadadadadadadadadadadadadadadadadadadadad","share_difficulty":1,"share_id":"share-2733","share_seq":2733,"template_height":800085},{"accepted_at_ms":999972,"credit_policy":null,"job_id":"job-2124","job_issued_at_ms":999987,"miner_id":"miner-1","network_difficulty":1000016,"ntime":1700002124,"order_key":"01:miner-1","p2mr_program_hex":"4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c","share_difficulty":3,"share_id":"share-2124","share_seq":2124,"template_height":800066},{"accepted_at_ms":999985,"credit_policy":null,"job_id":"job-1570","job_issued_at_ms":999980,"miner_id":"miner-8","network_difficulty":1000006,"ntime":1700001570,"order_key":"08:miner-8","p2mr_program_hex":"2222222222222222222222222222222222222222222222222222222222222222","share_difficulty":3,"share_id":"share-1570","share_seq":1570,"template_height":800049},{"accepted_at_ms":999982,"credit_policy":null,"job_id":"job-2870","job_issued_at_ms":999973,"miner_id":"miner-10","network_difficulty":1000014,"ntime":1700002870,"order_key":"10:miner-10","p2mr_program_hex":"3636363636363636363636363636363636363636363636363636363636363636","share_difficulty":7,"share_id":"share-2870","share_seq":2870,"template_height":800089},{"accepted_at_ms":1000000,"credit_policy":"stale-grace","job_id":"job-113","job_issued_at_ms":999982,"miner_id":"miner-3","network_difficulty":1000011,"ntime":1700000113,"order_key":"03:miner-3","p2mr_program_hex":"7171717171717171717171717171717171717171717171717171717171717171","share_difficulty":101,"share_id":"share-113","share_seq":113,"template_height":800003},{"accepted_at_ms":999990,"credit_policy":null,"job_id":"job-1457","job_issued_at_ms":999991,"miner_id":"miner-5","network_difficulty":1000012,"ntime":1700001457,"order_key":"05:miner-5","p2mr_program_hex":"b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1","share_difficulty":1,"share_id":"share-1457","share_seq":1457,"template_height":800045},{"accepted_at_ms":999975,"credit_policy":null,"job_id":"job-3181","job_issued_at_ms":999998,"miner_id":"miner-2","network_difficulty":1000002,"ntime":1700003181,"order_key":"02:miner-2","p2mr_program_hex":"6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d","share_difficulty":7,"share_id":"share-3181","share_seq":3181,"template_height":800099},{"accepted_at_ms":999995,"credit_policy":null,"job_id":"job-125","job_issued_at_ms":999995,"miner_id":"miner-4","network_difficulty":1000006,"ntime":1700000125,"order_key":"04:miner-4","p2mr_program_hex":"7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d","share_difficulty":1000,"share_id":"share-125","share_seq":125,"template_height":800003},{"accepted_at_ms":999972,"credit_policy":null,"job_id":"job-97","job_issued_at_ms":1000000,"miner_id":"miner-9","network_difficulty":1000012,"ntime":1700000097,"order_key":"09:miner-9","p2mr_program_hex":"6161616161616161616161616161616161616161616161616161616161616161","share_difficulty":1000,"share_id":"share-97","share_seq":97,"template_height":800003},{"accepted_at_ms":999978,"credit_policy":null,"job_id":"job-3252","job_issued_at_ms":999986,"miner_id":"miner-7","network_difficulty":1000005,"ntime":1700003252,"order_key":"07:miner-7","p2mr_program_hex":"b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4","share_difficulty":101,"share_id":"share-3252","share_seq":3252,"template_height":800101},{"accepted_at_ms":999971,"credit_policy":null,"job_id":"job-423","job_issued_at_ms":999997,"miner_id":"miner-5","network_difficulty":1000015,"ntime":1700000423,"order_key":"05:miner-5","p2mr_program_hex":"a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7","share_difficulty":101,"share_id":"share-423","share_seq":423,"template_height":800013},{"accepted_at_ms":999998,"credit_policy":null,"job_id":"job-2288","job_issued_at_ms":999975,"miner_id":"miner-0","network_difficulty":1000010,"ntime":1700002288,"order_key":"00:miner-0","p2mr_program_hex":"f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0","share_difficulty":1,"share_id":"share-2288","share_seq":2288,"template_height":800071},{"accepted_at_ms":999979,"credit_policy":null,"job_id":"job-3339","job_issued_at_ms":999984,"miner_id":"miner-6","network_difficulty":1000007,"ntime":1700003339,"order_key":"06:miner-6","p2mr_program_hex":"0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b","share_difficulty":1,"share_id":"share-3339","share_seq":3339,"template_height":800104},{"accepted_at_ms":999977,"credit_policy":null,"job_id":"job-83","job_issued_at_ms":999981,"miner_id":"miner-6","network_difficulty":1000015,"ntime":1700000083,"order_key":"06:miner-6","p2mr_program_hex":"5353535353535353535353535353535353535353535353535353535353535353","share_difficulty":7,"share_id":"share-83","share_seq":83,"template_height":800002},{"accepted_at_ms":999987,"credit_policy":null,"job_id":"job-1992","job_issued_at_ms":999981,"miner_id":"miner-1","network_difficulty":1000003,"ntime":1700001992,"order_key":"01:miner-1","p2mr_program_hex":"c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8","share_difficulty":3,"share_id":"share-1992","share_seq":1992,"template_height":800062},{"accepted_at_ms":999978,"credit_policy":null,"job_id":"job-3468","job_issued_at_ms":999994,"miner_id":"miner-3","network_difficulty":1000000,"ntime":1700003468,"order_key":"03:miner-3","p2mr_program_hex":"8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c","share_difficulty":7,"share_id":"share-3468","share_seq":3468,"template_height":800108},{"accepted_at_ms":999992,"credit_policy":null,"job_id":"job-2263","job_issued_at_ms":999974,"miner_id":"miner-8","network_difficulty":1000002,"ntime":1700002263,"order_key":"08:miner-8","p2mr_program_hex":"d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7","share_difficulty":3,"share_id":"share-2263","share_seq":2263,"template_height":800070},{"accepted_at_ms":999978,"credit_policy":null,"job_id":"job-793","job_issued_at_ms":999995,"miner_id":"miner-1","network_difficulty":1000011,"ntime":1700000793,"order_key":"01:miner-1","p2mr_program_hex":"1919191919191919191919191919191919191919191919191919191919191919","share_difficulty":3,"share_id":"share-793","share_seq":793,"template_height":800024},{"accepted_at_ms":999984,"credit_policy":null,"job_id":"job-1248","job_issued_at_ms":999995,"miner_id":"miner-5","network_difficulty":1000007,"ntime":1700001248,"order_key":"05:miner-5","p2mr_program_hex":"e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0","share_difficulty":1000,"share_id":"share-1248","share_seq":1248,"template_height":800039},{"accepted_at_ms":999978,"credit_policy":null,"job_id":"job-1206","job_issued_at_ms":999994,"miner_id":"miner-7","network_difficulty":1000016,"ntime":1700001206,"order_key":"07:miner-7","p2mr_program_hex":"b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6","share_difficulty":3,"share_id":"share-1206","share_seq":1206,"template_height":800037},{"accepted_at_ms":999977,"credit_policy":null,"job_id":"job-2762","job_issued_at_ms":999977,"miner_id":"miner-1","network_difficulty":1000008,"ntime":1700002762,"order_key":"01:miner-1","p2mr_program_hex":"cacacacacacacacacacacacacacacacacacacacacacacacacacacacacacacaca","share_difficulty":3,"share_id":"share-2762","share_seq":2762,"template_height":800086},{"accepted_at_ms":999990,"credit_policy":null,"job_id":"job-1345","job_issued_at_ms":999996,"miner_id":"miner-3","network_difficulty":1000002,"ntime":1700001345,"order_key":"03:miner-3","p2mr_program_hex":"4141414141414141414141414141414141414141414141414141414141414141","share_difficulty":1000,"share_id":"share-1345","share_seq":1345,"template_height":800042},{"accepted_at_ms":1000426,"credit_policy":null,"job_id":"job-32","job_issued_at_ms":999997,"miner_id":"miner-10","network_difficulty":1000015,"ntime":1700000032,"order_key":"10:miner-10","p2mr_program_hex":"2020202020202020202020202020202020202020202020202020202020202020","share_difficulty":101,"share_id":"share-32","share_seq":32,"template_height":800001},{"accepted_at_ms":999984,"credit_policy":null,"job_id":"job-1692","job_issued_at_ms":999973,"miner_id":"miner-9","network_difficulty":1000009,"ntime":1700001692,"order_key":"09:miner-9","p2mr_program_hex":"9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c","share_difficulty":101,"share_id":"share-1692","share_seq":1692,"template_height":800052},{"accepted_at_ms":999975,"credit_policy":"stale-grace","job_id":"job-2031","job_issued_at_ms":999980,"miner_id":"miner-7","network_difficulty":1000008,"ntime":1700002031,"order_key":"07:miner-7","p2mr_program_hex":"efefefefefefefefefefefefefefefefefefefefefefefefefefefefefefefef","share_difficulty":1,"share_id":"share-2031","share_seq":2031,"template_height":800063},{"accepted_at_ms":999989,"credit_policy":"stale-grace","job_id":"job-327","job_issued_at_ms":999995,"miner_id":"miner-8","network_difficulty":1000004,"ntime":1700000327,"order_key":"08:miner-8","p2mr_program_hex":"4747474747474747474747474747474747474747474747474747474747474747","share_difficulty":7,"share_id":"share-327","share_seq":327,"template_height":800010},{"accepted_at_ms":999970,"credit_policy":null,"job_id":"job-1760","job_issued_at_ms":999996,"miner_id":"miner-0","network_difficulty":1000009,"ntime":1700001760,"order_key":"00:miner-0","p2mr_program_hex":"e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0","share_difficulty":101,"share_id":"share-1760","share_seq":1760,"template_height":800055},{"accepted_at_ms":999998,"credit_policy":null,"job_id":"job-3490","job_issued_at_ms":999996,"miner_id":"miner-3","network_difficulty":1000005,"ntime":1700003490,"order_key":"03:miner-3","p2mr_program_hex":"a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2","share_difficulty":3,"share_id":"share-3490","share_seq":3490,"template_height":800109},{"accepted_at_ms":1000270,"credit_policy":"stale-grace","job_id":"job-1835","job_issued_at_ms":999982,"miner_id":"miner-9","network_difficulty":1000016,"ntime":1700001835,"order_key":"09:miner-9","p2mr_program_hex":"2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b","share_difficulty":7,"share_id":"share-1835","share_seq":1835,"template_height":800057},{"accepted_at_ms":999972,"credit_policy":null,"job_id":"job-1998","job_issued_at_ms":999979,"miner_id":"miner-7","network_difficulty":1000009,"ntime":1700001998,"order_key":"07:miner-7","p2mr_program_hex":"cececececececececececececececececececececececececececececececece","share_difficulty":2,"share_id":"share-1998","share_seq":1998,"template_height":800062},{"accepted_at_ms":999970,"credit_policy":null,"job_id":"job-3387","job_issued_at_ms":999974,"miner_id":"miner-10","network_difficulty":1000004,"ntime":1700003387,"order_key":"10:miner-10","p2mr_program_hex":"3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b","share_difficulty":3,"share_id":"share-3387","share_seq":3387,"template_height":800105},{"accepted_at_ms":999990,"credit_policy":null,"job_id":"job-2728","job_issued_at_ms":999994,"miner_id":"miner-0","network_difficulty":1000008,"ntime":1700002728,"order_key":"00:miner-0","p2mr_program_hex":"a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8","share_difficulty":19,"share_id":"share-2728","share_seq":2728,"template_height":800085},{"accepted_at_ms":999982,"credit_policy":null,"job_id":"job-1827","job_issued_at_ms":999971,"miner_id":"miner-1","network_difficulty":1000008,"ntime":1700001827,"order_key":"01:miner-1","p2mr_program_hex":"2323232323232323232323232323232323232323232323232323232323232323","share_difficulty":2,"share_id":"share-1827","share_seq":1827,"template_height":800057},{"accepted_at_ms":999978,"credit_policy":null,"job_id":"job-479","job_issued_at_ms":999977,"miner_id":"miner-6","network_difficulty":1000003,"ntime":1700000479,"order_key":"06:miner-6","p2mr_program_hex":"dfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdf","share_difficulty":2,"share_id":"share-479","share_seq":479,"template_height":800014},{"accepted_at_ms":999998,"credit_policy":null,"job_id":"job-3705","job_issued_at_ms":999987,"miner_id":"miner-9","network_difficulty":1000016,"ntime":1700003705,"order_key":"09:miner-9","p2mr_program_hex":"7979797979797979797979797979797979797979797979797979797979797979","share_difficulty":1,"share_id":"share-3705","share_seq":3705,"template_height":800115},{"accepted_at_ms":999995,"credit_policy":null,"job_id":"job-3220","job_issued_at_ms":999970,"miner_id":"miner-8","network_difficulty":1000007,"ntime":1700003220,"order_key":"08:miner-8","p2mr_program_hex":"9494949494949494949494949494949494949494949494949494949494949494","share_difficulty":1000,"share_id":"share-3220","share_seq":3220,"template_height":800100},{"accepted_at_ms":999990,"credit_policy":"stale-grace","job_id":"job-374","job_issued_at_ms":999989,"miner_id":"miner-0","network_difficulty":1000000,"ntime":1700000374,"order_key":"00:miner-0","p2mr_program_hex":"7676767676767676767676767676767676767676767676767676767676767676","share_difficulty":2,"share_id":"share-374","share_seq":374,"template_height":800011},{"accepted_at_ms":999998,"credit_policy":"stale-grace","job_id":"job-1459","job_issued_at_ms":1000102,"miner_id":"miner-7","network_difficulty":1000014,"ntime":1700001459,"order_key":"07:miner-7","p2mr_program_hex":"b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3","share_difficulty":1000,"share_id":"share-1459","share_seq":1459,"template_height":800045},{"accepted_at_ms":999980,"credit_policy":null,"job_id":"job-1978","job_issued_at_ms":999972,"miner_id":"miner-9","network_difficulty":1000006,"ntime":1700001978,"order_key":"09:miner-9","p2mr_program_hex":"babababababababababababababababababababababababababababababababa","share_difficulty":1,"share_id":"share-1978","share_seq":1978,"template_height":800061},{"accepted_at_ms":999989,"credit_policy":null,"job_id":"job-2473","job_issued_at_ms":999989,"miner_id":"miner-9","network_difficulty":1000008,"ntime":1700002473,"order_key":"09:miner-9","p2mr_program_hex":"a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9","share_difficulty":19,"share_id":"share-2473","share_seq":2473,"template_height":800077},{"accepted_at_ms":999970,"credit_policy":"stale-grace","job_id":"job-756","job_issued_at_ms":999977,"miner_id":"miner-8","network_difficulty":1000008,"ntime":1700000756,"order_key":"08:miner-8","p2mr_program_hex":"f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4","share_difficulty":19,"share_id":"share-756","share_seq":756,"template_height":800023},{"accepted_at_ms":999992,"credit_policy":null,"job_id":"job-593","job_issued_at_ms":999999,"miner_id":"miner-10","network_difficulty":1000015,"ntime":1700000593,"order_key":"10:miner-10","p2mr_program_hex":"5151515151515151515151515151515151515151515151515151515151515151","share_difficulty":19,"share_id":"share-593","share_seq":593,"template_height":800018},{"accepted_at_ms":999981,"credit_policy":null,"job_id":"job-2956","job_issued_at_ms":999972,"miner_id":"miner-8","network_difficulty":1000015,"ntime":1700002956,"order_key":"08:miner-8","p2mr_program_hex":"8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c","share_difficulty":101,"share_id":"share-2956","share_seq":2956,"template_height":800092},{"accepted_at_ms":999980,"credit_policy":null,"job_id":"job-3489","job_issued_at_ms":999982,"miner_id":"miner-2","network_difficulty":1000004,"ntime":1700003489,"order_key":"02:miner-2","p2mr_program_hex":"a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1","share_difficulty":101,"share_id":"share-3489","share_seq":3489,"template_height":800109},{"accepted_at_ms":999987,"credit_policy":null,"job_id":"job-2520","job_issued_at_ms":999986,"miner_id":"miner-1","network_difficulty":1000004,"ntime":1700002520,"order_key":"01:miner-1","p2mr_program_hex":"d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8","share_difficulty":3,"share_id":"share-2520","share_seq":2520,"template_height":800078},{"accepted_at_ms":999985,"credit_policy":"stale-grace","job_id":"job-2889","job_issued_at_ms":999985,"miner_id":"miner-7","network_difficulty":1000016,"ntime":1700002889,"order_key":"07:miner-7","p2mr_program_hex":"4949494949494949494949494949494949494949494949494949494949494949","share_difficulty":19,"share_id":"share-2889","share_seq":2889,"template_height":800090},{"accepted_at_ms":999987,"credit_policy":"stale-grace","job_id":"job-2353","job_issued_at_ms":999996,"miner_id":"miner-10","network_difficulty":1000007,"ntime":1700002353,"order_key":"10:miner-10","p2mr_program_hex":"3131313131313131313131313131313131313131313131313131313131313131","share_difficulty":3,"share_id":"share-2353","share_seq":2353,"template_height":800073},{"accepted_at_ms":1000000,"credit_policy":null,"job_id":"job-10","job_issued_at_ms":999976,"miner_id":"miner-10","network_difficulty":1000010,"ntime":1700000010,"order_key":"10:miner-10","p2mr_program_hex":"0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a","share_difficulty":3,"share_id":"share-10","share_seq":10,"template_height":800000},{"accepted_at_ms":999971,"credit_policy":"stale-grace","job_id":"job-3564","job_issued_at_ms":999995,"miner_id":"miner-0","network_difficulty":1000011,"ntime":1700003564,"order_key":"00:miner-0","p2mr_program_hex":"ecececececececececececececececececececececececececececececececec","share_difficulty":3,"share_id":"share-3564","share_seq":3564,"template_height":800111},{"accepted_at_ms":999990,"credit_policy":"stale-grace","job_id":"job-717","job_issued_at_ms":999986,"miner_id":"miner-2","network_difficulty":1000003,"ntime":1700000717,"order_key":"02:miner-2","p2mr_program_hex":"cdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcd","share_difficulty":3,"share_id":"share-717","share_seq":717,"template_height":800022},{"accepted_at_ms":999977,"credit_policy":null,"job_id":"job-651","job_issued_at_ms":1000000,"miner_id":"miner-2","network_difficulty":1000005,"ntime":1700000651,"order_key":"02:miner-2","p2mr_program_hex":"8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b","share_difficulty":19,"share_id":"share-651","share_seq":651,"template_height":800020},{"accepted_at_ms":999996,"credit_policy":null,"job_id":"job-3201","job_issued_at_ms":999976,"miner_id":"miner-0","network_difficulty":1000005,"ntime":1700003201,"order_key":"00:miner-0","p2mr_program_hex":"8181818181818181818181818181818181818181818181818181818181818181","share_difficulty":3,"share_id":"share-3201","share_seq":3201,"template_height":800100},{"accepted_at_ms":999994,"credit_policy":null,"job_id":"job-939","job_issued_at_ms":999993,"miner_id":"miner-4","network_difficulty":1000004,"ntime":1700000939,"order_key":"04:miner-4","p2mr_program_hex":"abababababababababababababababababababababababababababababababab","share_difficulty":101,"share_id":"share-939","share_seq":939,"template_height":800029},{"accepted_at_ms":999981,"credit_policy":null,"job_id":"job-482","job_issued_at_ms":999970,"miner_id":"miner-9","network_difficulty":1000006,"ntime":1700000482,"order_key":"09:miner-9","p2mr_program_hex":"e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2","share_difficulty":19,"share_id":"share-482","share_seq":482,"template_height":800015},{"accepted_at_ms":999998,"credit_policy":null,"job_id":"job-1592","job_issued_at_ms":999971,"miner_id":"miner-8","network_difficulty":1000011,"ntime":1700001592,"order_key":"08:miner-8","p2mr_program_hex":"3838383838383838383838383838383838383838383838383838383838383838","share_difficulty":3,"share_id":"share-1592","share_seq":1592,"template_height":800049},{"accepted_at_ms":999977,"credit_policy":null,"job_id":"job-2215","job_issued_at_ms":999970,"miner_id":"miner-4","network_difficulty":1000005,"ntime":1700002215,"order_key":"04:miner-4","p2mr_program_hex":"a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7","share_difficulty":2,"share_id":"share-2215","share_seq":2215,"template_height":800069},{"accepted_at_ms":999989,"credit_policy":null,"job_id":"job-964","job_issued_at_ms":999986,"miner_id":"miner-7","network_difficulty":1000012,"ntime":1700000964,"order_key":"07:miner-7","p2mr_program_hex":"c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4","share_difficulty":1,"share_id":"share-964","share_seq":964,"template_height":800030},{"accepted_at_ms":999998,"credit_policy":"stale-grace","job_id":"job-2258","job_issued_at_ms":999980,"miner_id":"miner-3","network_difficulty":1000014,"ntime":1700002258,"order_key":"03:miner-3","p2mr_program_hex":"d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2","share_difficulty":2,"share_id":"share-2258","share_seq":2258,"template_height":800070},{"accepted_at_ms":1000000,"credit_policy":null,"job_id":"job-2855","job_issued_at_ms":999987,"miner_id":"miner-6","network_difficulty":1000016,"ntime":1700002855,"order_key":"06:miner-6","p2mr_program_hex":"2727272727272727272727272727272727272727272727272727272727272727","share_difficulty":19,"share_id":"share-2855","share_seq":2855,"template_height":800089},{"accepted_at_ms":999981,"credit_policy":"stale-grace","job_id":"job-1102","job_issued_at_ms":999981,"miner_id":"miner-2","network_difficulty":1000014,"ntime":1700001102,"order_key":"02:miner-2","p2mr_program_hex":"4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e","share_difficulty":3,"share_id":"share-1102","share_seq":1102,"template_height":800034},{"accepted_at_ms":999972,"credit_policy":null,"job_id":"job-2122","job_issued_at_ms":999994,"miner_id":"miner-10","network_difficulty":1000014,"ntime":1700002122,"order_key":"10:miner-10","p2mr_program_hex":"4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a","share_difficulty":1000,"share_id":"share-2122","share_seq":2122,"template_height":800066},{"accepted_at_ms":999973,"credit_policy":null,"job_id":"job-3185","job_issued_at_ms":999971,"miner_id":"miner-6","network_difficulty":1000006,"ntime":1700003185,"order_key":"06:miner-6","p2mr_program_hex":"7171717171717171717171717171717171717171717171717171717171717171","share_difficulty":7,"share_id":"share-3185","share_seq":3185,"template_height":800099},{"accepted_at_ms":999996,"credit_policy":"stale-grace","job_id":"job-3849","job_issued_at_ms":999978,"miner_id":"miner-10","network_difficulty":1000007,"ntime":1700003849,"order_key":"10:miner-10","p2mr_program_hex":"0909090909090909090909090909090909090909090909090909090909090909","share_difficulty":7,"share_id":"share-3849","share_seq":3849,"template_height":800120},{"accepted_at_ms":999982,"credit_policy":null,"job_id":"job-3973","job_issued_at_ms":999977,"miner_id":"miner-2","network_difficulty":1000012,"ntime":1700003973,"order_key":"02:miner-2","p2mr_program_hex":"8585858585858585858585858585858585858585858585858585858585858585","share_difficulty":3,"share_id":"share-3973","share_seq":3973,"template_height":800124},{"accepted_at_ms":999986,"credit_policy":"stale-grace","job_id":"job-247","job_issued_at_ms":999987,"miner_id":"miner-5","network_difficulty":1000009,"ntime":1700000247,"order_key":"05:miner-5","p2mr_program_hex":"f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7","share_difficulty":7,"share_id":"share-247","share_seq":247,"template_height":800007},{"accepted_at_ms":999989,"credit_policy":null,"job_id":"job-2641","job_issued_at_ms":1000000,"miner_id":"miner-1","network_difficulty":1000006,"ntime":1700002641,"order_key":"01:miner-1","p2mr_program_hex":"5151515151515151515151515151515151515151515151515151515151515151","share_difficulty":3,"share_id":"share-2641","share_seq":2641,"template_height":800082},{"accepted_at_ms":999974,"credit_policy":null,"job_id":"job-472","job_issued_at_ms":999988,"miner_id":"miner-10","network_difficulty":1000013,"ntime":1700000472,"order_key":"10:miner-10","p2mr_program_hex":"d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8","share_difficulty":101,"share_id":"share-472","share_seq":472,"template_height":800014},{"accepted_at_ms":999973,"credit_policy":null,"job_id":"job-3677","job_issued_at_ms":999971,"miner_id":"miner-3","network_difficulty":1000005,"ntime":1700003677,"order_key":"03:miner-3","p2mr_program_hex":"5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d","share_difficulty":3,"share_id":"share-3677","share_seq":3677,"template_height":800114},{"accepted_at_ms":1000307,"credit_policy":null,"job_id":"job-3189","job_issued_at_ms":1000302,"miner_id":"miner-10","network_difficulty":1000010,"ntime":1700003189,"order_key":"10:miner-10","p2mr_program_hex":"7575757575757575757575757575757575757575757575757575757575757575","share_difficulty":101,"share_id":"share-3189","share_seq":3189,"template_height":800099},{"accepted_at_ms":999972,"credit_policy":"stale-grace","job_id":"job-1912","job_issued_at_ms":999984,"miner_id":"miner-9","network_difficulty":1000008,"ntime":1700001912,"order_key":"09:miner-9","p2mr_program_hex":"7878787878787878787878787878787878787878787878787878787878787878","share_difficulty":1,"share_id":"share-1912","share_seq":1912,"template_height":800059},{"accepted_at_ms":999992,"credit_policy":null,"job_id":"job-1568","job_issued_at_ms":999995,"miner_id":"miner-6","network_difficulty":1000004,"ntime":1700001568,"order_key":"06:miner-6","p2mr_program_hex":"2020202020202020202020202020202020202020202020202020202020202020","share_difficulty":1,"share_id":"share-1568","share_seq":1568,"template_height":800049},{"accepted_at_ms":999981,"credit_policy":null,"job_id":"job-3610","job_issued_at_ms":999985,"miner_id":"miner-2","network_difficulty":1000006,"ntime":1700003610,"order_key":"02:miner-2","p2mr_program_hex":"1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a","share_difficulty":101,"share_id":"share-3610","share_seq":3610,"template_height":800112},{"accepted_at_ms":999982,"credit_policy":null,"job_id":"job-3164","job_issued_at_ms":999999,"miner_id":"miner-7","network_difficulty":1000002,"ntime":1700003164,"order_key":"07:miner-7","p2mr_program_hex":"5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c","share_difficulty":19,"share_id":"share-3164","share_seq":3164,"template_height":800098},{"accepted_at_ms":999975,"credit_policy":null,"job_id":"job-590","job_issued_at_ms":999993,"miner_id":"miner-7","network_difficulty":1000012,"ntime":1700000590,"order_key":"07:miner-7","p2mr_program_hex":"4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e","share_difficulty":19,"share_id":"share-590","share_seq":590,"template_height":800018},{"accepted_at_ms":999980,"credit_policy":null,"job_id":"job-2237","job_issued_at_ms":999996,"miner_id":"miner-4","network_difficulty":1000010,"ntime":1700002237,"order_key":"04:miner-4","p2mr_program_hex":"bdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbd","share_difficulty":19,"share_id":"share-2237","share_seq":2237,"template_height":800069},{"accepted_at_ms":999982,"credit_policy":null,"job_id":"job-3430","job_issued_at_ms":999983,"miner_id":"miner-9","network_difficulty":1000013,"ntime":1700003430,"order_key":"09:miner-9","p2mr_program_hex":"6666666666666666666666666666666666666666666666666666666666666666","share_difficulty":1,"share_id":"share-3430","share_seq":3430,"template_height":800107},{"accepted_at_ms":999984,"credit_policy":null,"job_id":"job-884","job_issued_at_ms":999989,"miner_id":"miner-4","network_difficulty":1000000,"ntime":1700000884,"order_key":"04:miner-4","p2mr_program_hex":"7474747474747474747474747474747474747474747474747474747474747474","share_difficulty":3,"share_id":"share-884","share_seq":884,"template_height":800027},{"accepted_at_ms":999988,"credit_policy":null,"job_id":"job-560","job_issued_at_ms":999991,"miner_id":"miner-10","network_difficulty":1000016,"ntime":1700000560,"order_key":"10:miner-10","p2mr_program_hex":"3030303030303030303030303030303030303030303030303030303030303030","share_difficulty":1,"share_id":"share-560","share_seq":560,"template_height":800017},{"accepted_at_ms":999971,"credit_policy":"stale-grace","job_id":"job-1731","job_issued_at_ms":999994,"miner_id":"miner-4","network_difficulty":1000014,"ntime":1700001731,"order_key":"04:miner-4","p2mr_program_hex":"c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3","share_difficulty":2,"share_id":"share-1731","share_seq":1731,"template_height":800054},{"accepted_at_ms":999997,"credit_policy":null,"job_id":"job-605","job_issued_at_ms":999984,"miner_id":"miner-0","network_difficulty":1000010,"ntime":1700000605,"order_key":"00:miner-0","p2mr_program_hex":"5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d","share_difficulty":1,"share_id":"share-605","share_seq":605,"template_height":800018},{"accepted_at_ms":999977,"credit_policy":null,"job_id":"job-1892","job_issued_at_ms":999970,"miner_id":"miner-0","network_difficulty":1000005,"ntime":1700001892,"order_key":"00:miner-0","p2mr_program_hex":"6464646464646464646464646464646464646464646464646464646464646464","share_difficulty":1000,"share_id":"share-1892","share_seq":1892,"template_height":800059},{"accepted_at_ms":999973,"credit_policy":null,"job_id":"job-1002","job_issued_at_ms":999977,"miner_id":"miner-1","network_difficulty":1000016,"ntime":1700001002,"order_key":"01:miner-1","p2mr_program_hex":"eaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaea","share_difficulty":3,"share_id":"share-1002","share_seq":1002,"template_height":800031},{"accepted_at_ms":999978,"credit_policy":null,"job_id":"job-194","job_issued_at_ms":999993,"miner_id":"miner-7","network_difficulty":1000007,"ntime":1700000194,"order_key":"07:miner-7","p2mr_program_hex":"c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2","share_difficulty":19,"share_id":"share-194","share_seq":194,"template_height":800006},{"accepted_at_ms":999990,"credit_policy":null,"job_id":"job-1775","job_issued_at_ms":999977,"miner_id":"miner-4","network_difficulty":1000007,"ntime":1700001775,"order_key":"04:miner-4","p2mr_program_hex":"efefefefefefefefefefefefefefefefefefefefefefefefefefefefefefefef","share_difficulty":1000,"share_id":"share-1775","share_seq":1775,"template_height":800055},{"accepted_at_ms":999979,"credit_policy":null,"job_id":"job-1852","job_issued_at_ms":999974,"miner_id":"miner-4","network_difficulty":1000016,"ntime":1700001852,"order_key":"04:miner-4","p2mr_program_hex":"3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c","share_difficulty":1,"share_id":"share-1852","share_seq":1852,"template_height":800057},{"accepted_at_ms":999999,"credit_policy":null,"job_id":"job-3937","job_issued_at_ms":999994,"miner_id":"miner-10","network_difficulty":1000010,"ntime":1700003937,"order_key":"10:miner-10","p2mr_program_hex":"6161616161616161616161616161616161616161616161616161616161616161","share_difficulty":2,"share_id":"share-3937","share_seq":3937,"template_height":800123},{"accepted_at_ms":999997,"credit_policy":null,"job_id":"job-1761","job_issued_at_ms":1000000,"miner_id":"miner-1","network_difficulty":1000010,"ntime":1700001761,"order_key":"01:miner-1","p2mr_program_hex":"e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1","share_difficulty":19,"share_id":"share-1761","share_seq":1761,"template_height":800055},{"accepted_at_ms":999991,"credit_policy":null,"job_id":"job-3487","job_issued_at_ms":999988,"miner_id":"miner-0","network_difficulty":1000002,"ntime":1700003487,"order_key":"00:miner-0","p2mr_program_hex":"9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f","share_difficulty":19,"share_id":"share-3487","share_seq":3487,"template_height":800108},{"accepted_at_ms":999996,"credit_policy":null,"job_id":"job-538","job_issued_at_ms":999996,"miner_id":"miner-10","network_difficulty":1000011,"ntime":1700000538,"order_key":"10:miner-10","p2mr_program_hex":"1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a","share_difficulty":19,"share_id":"share-538","share_seq":538,"template_height":800016},{"accepted_at_ms":999975,"credit_policy":null,"job_id":"job-1499","job_issued_at_ms":999977,"miner_id":"miner-3","network_difficulty":1000003,"ntime":1700001499,"order_key":"03:miner-3","p2mr_program_hex":"dbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdb","share_difficulty":19,"share_id":"share-1499","share_seq":1499,"template_height":800046},{"accepted_at_ms":999977,"credit_policy":null,"job_id":"job-638","job_issued_at_ms":999993,"miner_id":"miner-0","network_difficulty":1000009,"ntime":1700000638,"order_key":"00:miner-0","p2mr_program_hex":"7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e","share_difficulty":7,"share_id":"share-638","share_seq":638,"template_height":800019},{"accepted_at_ms":999971,"credit_policy":null,"job_id":"job-1870","job_issued_at_ms":999999,"miner_id":"miner-0","network_difficulty":1000000,"ntime":1700001870,"order_key":"00:miner-0","p2mr_program_hex":"4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e","share_difficulty":19,"share_id":"share-1870","share_seq":1870,"template_height":800058},{"accepted_at_ms":1000000,"credit_policy":"stale-grace","job_id":"job-349","job_issued_at_ms":999974,"miner_id":"miner-8","network_difficulty":1000009,"ntime":1700000349,"order_key":"08:miner-8","p2mr_program_hex":"5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d","share_difficulty":101,"share_id":"share-349","share_seq":349,"template_height":800010},{"accepted_at_ms":999990,"credit_policy":null,"job_id":"job-2235","job_issued_at_ms":999978,"miner_id":"miner-2","network_difficulty":1000008,"ntime":1700002235,"order_key":"02:miner-2","p2mr_program_hex":"bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb","share_difficulty":19,"share_id":"share-2235","share_seq":2235,"template_height":800069},{"accepted_at_ms":999996,"credit_policy":null,"job_id":"job-1821","job_issued_at_ms":999987,"miner_id":"miner-6","network_difficulty":1000002,"ntime":1700001821,"order_key":"06:miner-6","p2mr_program_hex":"1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d","share_difficulty":7,"share_id":"share-1821","share_seq":1821,"template_height":800056},{"accepted_at_ms":999977,"credit_policy":null,"job_id":"job-1693","job_issued_at_ms":999982,"miner_id":"miner-10","network_difficulty":1000010,"ntime":1700001693,"order_key":"10:miner-10","p2mr_program_hex":"9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d","share_difficulty":2,"share_id":"share-1693","share_seq":1693,"template_height":800052},{"accepted_at_ms":999989,"credit_policy":null,"job_id":"job-446","job_issued_at_ms":999990,"miner_id":"miner-6","network_difficulty":1000004,"ntime":1700000446,"order_key":"06:miner-6","p2mr_program_hex":"bebebebebebebebebebebebebebebebebebebebebebebebebebebebebebebebe","share_difficulty":7,"share_id":"share-446","share_seq":446,"template_height":800013},{"accepted_at_ms":999983,"credit_policy":null,"job_id":"job-2960","job_issued_at_ms":999983,"miner_id":"miner-1","network_difficulty":1000002,"ntime":1700002960,"order_key":"01:miner-1","p2mr_program_hex":"9090909090909090909090909090909090909090909090909090909090909090","share_difficulty":3,"share_id":"share-2960","share_seq":2960,"template_height":800092},{"accepted_at_ms":999996,"credit_policy":"stale-grace","job_id":"job-283","job_issued_at_ms":999985,"miner_id":"miner-8","network_difficulty":1000011,"ntime":1700000283,"order_key":"08:miner-8","p2mr_program_hex":"1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b","share_difficulty":1,"share_id":"share-283","share_seq":283,"template_height":800008},{"accepted_at_ms":999972,"credit_policy":null,"job_id":"job-2104","job_issued_at_ms":999983,"miner_id":"miner-3","network_difficulty":1000013,"ntime":1700002104,"order_key":"03:miner-3","p2mr_program_hex":"3838383838383838383838383838383838383838383838383838383838383838","share_difficulty":101,"share_id":"share-2104","share_seq":2104,"template_height":800065},{"accepted_at_ms":999970,"credit_policy":null,"job_id":"job-2425","job_issued_at_ms":999972,"miner_id":"miner-5","network_difficulty":1000011,"ntime":1700002425,"order_key":"05:miner-5","p2mr_program_hex":"7979797979797979797979797979797979797979797979797979797979797979","share_difficulty":1000,"share_id":"share-2425","share_seq":2425,"template_height":800075},{"accepted_at_ms":999990,"credit_policy":null,"job_id":"job-1880","job_issued_at_ms":999976,"miner_id":"miner-10","network_difficulty":1000010,"ntime":1700001880,"order_key":"10:miner-10","p2mr_program_hex":"5858585858585858585858585858585858585858585858585858585858585858","share_difficulty":3,"share_id":"share-1880","share_seq":1880,"template_height":800058},{"accepted_at_ms":999989,"credit_policy":null,"job_id":"job-3096","job_issued_at_ms":999989,"miner_id":"miner-5","network_difficulty":1000002,"ntime":1700003096,"order_key":"05:miner-5","p2mr_program_hex":"1818181818181818181818181818181818181818181818181818181818181818","share_difficulty":2,"share_id":"share-3096","share_seq":3096,"template_height":800096},{"accepted_at_ms":999982,"credit_policy":null,"job_id":"job-2864","job_issued_at_ms":999995,"miner_id":"miner-4","network_difficulty":1000008,"ntime":1700002864,"order_key":"04:miner-4","p2mr_program_hex":"3030303030303030303030303030303030303030303030303030303030303030","share_difficulty":19,"share_id":"share-2864","share_seq":2864,"template_height":800089},{"accepted_at_ms":999986,"credit_policy":null,"job_id":"job-1241","job_issued_at_ms":999983,"miner_id":"miner-9","network_difficulty":1000000,"ntime":1700001241,"order_key":"09:miner-9","p2mr_program_hex":"d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9","share_difficulty":1,"share_id":"share-1241","share_seq":1241,"template_height":800038},{"accepted_at_ms":1000000,"credit_policy":null,"job_id":"job-2967","job_issued_at_ms":1000000,"miner_id":"miner-8","network_difficulty":1000009,"ntime":1700002967,"order_key":"08:miner-8","p2mr_program_hex":"9797979797979797979797979797979797979797979797979797979797979797","share_difficulty":7,"share_id":"share-2967","share_seq":2967,"template_height":800092},{"accepted_at_ms":999973,"credit_policy":"stale-grace","job_id":"job-3188","job_issued_at_ms":999984,"miner_id":"miner-9","network_difficulty":1000009,"ntime":1700003188,"order_key":"09:miner-9","p2mr_program_hex":"7474747474747474747474747474747474747474747474747474747474747474","share_difficulty":101,"share_id":"share-3188","share_seq":3188,"template_height":800099},{"accepted_at_ms":999993,"credit_policy":null,"job_id":"job-3690","job_issued_at_ms":999983,"miner_id":"miner-5","network_difficulty":1000001,"ntime":1700003690,"order_key":"05:miner-5","p2mr_program_hex":"6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a","share_difficulty":3,"share_id":"share-3690","share_seq":3690,"template_height":800115},{"accepted_at_ms":999971,"credit_policy":null,"job_id":"job-1774","job_issued_at_ms":999993,"miner_id":"miner-3","network_difficulty":1000006,"ntime":1700001774,"order_key":"03:miner-3","p2mr_program_hex":"eeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeee","share_difficulty":1000,"share_id":"share-1774","share_seq":1774,"template_height":800055},{"accepted_at_ms":999979,"credit_policy":"stale-grace","job_id":"job-1575","job_issued_at_ms":1000310,"miner_id":"miner-2","network_difficulty":1000011,"ntime":1700001575,"order_key":"02:miner-2","p2mr_program_hex":"2727272727272727272727272727272727272727272727272727272727272727","share_difficulty":7,"share_id":"share-1575","share_seq":1575,"template_height":800049},{"accepted_at_ms":999971,"credit_policy":null,"job_id":"job-118","job_issued_at_ms":999971,"miner_id":"miner-8","network_difficulty":1000016,"ntime":1700000118,"order_key":"08:miner-8","p2mr_program_hex":"7676767676767676767676767676767676767676767676767676767676767676","share_difficulty":2,"share_id":"share-118","share_seq":118,"template_height":800003},{"accepted_at_ms":999972,"credit_policy":null,"job_id":"job-2719","job_issued_at_ms":999998,"miner_id":"miner-2","network_difficulty":1000016,"ntime":1700002719,"order_key":"02:miner-2","p2mr_program_hex":"9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f","share_difficulty":1000,"share_id":"share-2719","share_seq":2719,"template_height":800084},{"accepted_at_ms":999988,"credit_policy":null,"job_id":"job-52","job_issued_at_ms":999974,"miner_id":"miner-8","network_difficulty":1000001,"ntime":1700000052,"order_key":"08:miner-8","p2mr_program_hex":"3434343434343434343434343434343434343434343434343434343434343434","share_difficulty":19,"share_id":"share-52","share_seq":52,"template_height":800001},{"accepted_at_ms":999990,"credit_policy":"stale-grace","job_id":"job-1502","job_issued_at_ms":999974,"miner_id":"miner-6","network_difficulty":1000006,"ntime":1700001502,"order_key":"06:miner-6","p2mr_program_hex":"dededededededededededededededededededededededededededededededede","share_difficulty":7,"share_id":"share-1502","share_seq":1502,"template_height":800046},{"accepted_at_ms":999990,"credit_policy":null,"job_id":"job-1394","job_issued_at_ms":999975,"miner_id":"miner-8","network_difficulty":1000000,"ntime":1700001394,"order_key":"08:miner-8","p2mr_program_hex":"7272727272727272727272727272727272727272727272727272727272727272","share_difficulty":3,"share_id":"share-1394","share_seq":1394,"template_height":800043},{"accepted_at_ms":999996,"credit_policy":null,"job_id":"job-3911","job_issued_at_ms":999973,"miner_id":"miner-6","network_difficulty":1000001,"ntime":1700003911,"order_key":"06:miner-6","p2mr_program_hex":"4747474747474747474747474747474747474747474747474747474747474747","share_difficulty":101,"share_id":"share-3911","share_seq":3911,"template_height":800122},{"accepted_at_ms":999976,"credit_policy":null,"job_id":"job-3356","job_issued_at_ms":1000000,"miner_id":"miner-1","network_difficulty":1000007,"ntime":1700003356,"order_key":"01:miner-1","p2mr_program_hex":"1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c","share_difficulty":19,"share_id":"share-3356","share_seq":3356,"template_height":800104},{"accepted_at_ms":999981,"credit_policy":null,"job_id":"job-2117","job_issued_at_ms":999973,"miner_id":"miner-5","network_difficulty":1000009,"ntime":1700002117,"order_key":"05:miner-5","p2mr_program_hex":"4545454545454545454545454545454545454545454545454545454545454545","share_difficulty":1,"share_id":"share-2117","share_seq":2117,"template_height":800066},{"accepted_at_ms":999987,"credit_policy":null,"job_id":"job-1866","job_issued_at_ms":999993,"miner_id":"miner-7","network_difficulty":1000013,"ntime":1700001866,"order_key":"07:miner-7","p2mr_program_hex":"4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a","share_difficulty":1,"share_id":"share-1866","share_seq":1866,"template_height":800058},{"accepted_at_ms":999973,"credit_policy":null,"job_id":"job-709","job_issued_at_ms":999997,"miner_id":"miner-5","network_difficulty":1000012,"ntime":1700000709,"order_key":"05:miner-5","p2mr_program_hex":"c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5","share_difficulty":1,"share_id":"share-709","share_seq":709,"template_height":800022},{"accepted_at_ms":999997,"credit_policy":null,"job_id":"job-1733","job_issued_at_ms":999984,"miner_id":"miner-6","network_difficulty":1000016,"ntime":1700001733,"order_key":"06:miner-6","p2mr_program_hex":"c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5","share_difficulty":7,"share_id":"share-1733","share_seq":1733,"template_height":800054},{"accepted_at_ms":999982,"credit_policy":null,"job_id":"job-3459","job_issued_at_ms":999974,"miner_id":"miner-5","network_difficulty":1000008,"ntime":1700003459,"order_key":"05:miner-5","p2mr_program_hex":"8383838383838383838383838383838383838383838383838383838383838383","share_difficulty":1,"share_id":"share-3459","share_seq":3459,"template_height":800108},{"accepted_at_ms":999992,"credit_policy":null,"job_id":"job-3363","job_issued_at_ms":999970,"miner_id":"miner-8","network_difficulty":1000014,"ntime":1700003363,"order_key":"08:miner-8","p2mr_program_hex":"2323232323232323232323232323232323232323232323232323232323232323","share_difficulty":2,"share_id":"share-3363","share_seq":3363,"template_height":800105},{"accepted_at_ms":1000106,"credit_policy":"stale-grace","job_id":"job-92","job_issued_at_ms":999980,"miner_id":"miner-4","network_difficulty":1000007,"ntime":1700000092,"order_key":"04:miner-4","p2mr_program_hex":"5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c","share_difficulty":101,"share_id":"share-92","share_seq":92,"template_height":800002},{"accepted_at_ms":1000483,"credit_policy":null,"job_id":"job-1420","job_issued_at_ms":999983,"miner_id":"miner-1","network_difficulty":1000009,"ntime":1700001420,"order_key":"01:miner-1","p2mr_program_hex":"8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c","share_difficulty":3,"share_id":"share-1420","share_seq":1420,"template_height":800044},{"accepted_at_ms":999978,"credit_policy":null,"job_id":"job-2326","job_issued_at_ms":999970,"miner_id":"miner-5","network_difficulty":1000014,"ntime":1700002326,"order_key":"05:miner-5","p2mr_program_hex":"1616161616161616161616161616161616161616161616161616161616161616","share_difficulty":7,"share_id":"share-2326","share_seq":2326,"template_height":800072},{"accepted_at_ms":999978,"credit_policy":"stale-grace","job_id":"job-570","job_issued_at_ms":999982,"miner_id":"miner-9","network_difficulty":1000009,"ntime":1700000570,"order_key":"09:miner-9","p2mr_program_hex":"3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a","share_difficulty":101,"share_id":"share-570","share_seq":570,"template_height":800017},{"accepted_at_ms":999974,"credit_policy":null,"job_id":"job-1194","job_issued_at_ms":999984,"miner_id":"miner-6","network_difficulty":1000004,"ntime":1700001194,"order_key":"06:miner-6","p2mr_program_hex":"aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa","share_difficulty":1,"share_id":"share-1194","share_seq":1194,"template_height":800037},{"accepted_at_ms":999984,"credit_policy":null,"job_id":"job-860","job_issued_at_ms":1000089,"miner_id":"miner-2","network_difficulty":1000010,"ntime":1700000860,"order_key":"02:miner-2","p2mr_program_hex":"5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c","share_difficulty":1000,"share_id":"share-860","share_seq":860,"template_height":800026},{"accepted_at_ms":999972,"credit_policy":null,"job_id":"job-897","job_issued_at_ms":999985,"miner_id":"miner-6","network_difficulty":1000013,"ntime":1700000897,"order_key":"06:miner-6","p2mr_program_hex":"8181818181818181818181818181818181818181818181818181818181818181","share_difficulty":3,"share_id":"share-897","share_seq":897,"template_height":800028},{"accepted_at_ms":999975,"credit_policy":"stale-grace","job_id":"job-2108","job_issued_at_ms":999977,"miner_id":"miner-7","network_difficulty":1000000,"ntime":1700002108,"order_key":"07:miner-7","p2mr_program_hex":"3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c","share_difficulty":19,"share_id":"share-2108","share_seq":2108,"template_height":800065},{"accepted_at_ms":999999,"credit_policy":null,"job_id":"job-1790","job_issued_at_ms":999985,"miner_id":"miner-8","network_difficulty":1000005,"ntime":1700001790,"order_key":"08:miner-8","p2mr_program_hex":"fefefefefefefefefefefefefefefefefefefefefefefefefefefefefefefefe","share_difficulty":19,"share_id":"share-1790","share_seq":1790,"template_height":800055},{"accepted_at_ms":999997,"credit_policy":null,"job_id":"job-1313","job_issued_at_ms":999987,"miner_id":"miner-4","network_difficulty":1000004,"ntime":1700001313,"order_key":"04:miner-4","p2mr_program_hex":"2121212121212121212121212121212121212121212121212121212121212121","share_difficulty":101,"share_id":"share-1313","share_seq":1313,"template_height":800041},{"accepted_at_ms":999979,"credit_policy":null,"job_id":"job-2635","job_issued_at_ms":999987,"miner_id":"miner-6","network_difficulty":1000000,"ntime":1700002635,"order_key":"06:miner-6","p2mr_program_hex":"4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b","share_difficulty":19,"share_id":"share-2635","share_seq":2635,"template_height":800082},{"accepted_at_ms":999978,"credit_policy":null,"job_id":"job-1062","job_issued_at_ms":999975,"miner_id":"miner-6","network_difficulty":1000008,"ntime":1700001062,"order_key":"06:miner-6","p2mr_program_hex":"2626262626262626262626262626262626262626262626262626262626262626","share_difficulty":1,"share_id":"share-1062","share_seq":1062,"template_height":800033},{"accepted_at_ms":999990,"credit_policy":null,"job_id":"job-1496","job_issued_at_ms":999973,"miner_id":"miner-0","network_difficulty":1000000,"ntime":1700001496,"order_key":"00:miner-0","p2mr_program_hex":"d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8","share_difficulty":1,"share_id":"share-1496","share_seq":1496,"template_height":800046},{"accepted_at_ms":999973,"credit_policy":null,"job_id":"job-2497","job_issued_at_ms":999993,"miner_id":"miner-0","network_difficulty":1000015,"ntime":1700002497,"order_key":"00:miner-0","p2mr_program_hex":"c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1","share_difficulty":1,"share_id":"share-2497","share_seq":2497,"template_height":800078},{"accepted_at_ms":999971,"credit_policy":null,"job_id":"job-1064","job_issued_at_ms":999981,"miner_id":"miner-8","network_difficulty":1000010,"ntime":1700001064,"order_key":"08:miner-8","p2mr_program_hex":"2828282828282828282828282828282828282828282828282828282828282828","share_difficulty":7,"share_id":"share-1064","share_seq":1064,"template_height":800033},{"accepted_at_ms":999976,"credit_policy":null,"job_id":"job-3054","job_issued_at_ms":999980,"miner_id":"miner-7","network_difficulty":1000011,"ntime":1700003054,"order_key":"07:miner-7","p2mr_program_hex":"eeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeee","share_difficulty":2,"share_id":"share-3054","share_seq":3054,"template_height":800095},{"accepted_at_ms":999985,"credit_policy":null,"job_id":"job-2357","job_issued_at_ms":999978,"miner_id":"miner-3","network_difficulty":1000011,"ntime":1700002357,"order_key":"03:miner-3","p2mr_program_hex":"3535353535353535353535353535353535353535353535353535353535353535","share_difficulty":101,"share_id":"share-2357","share_seq":2357,"template_height":800073},{"accepted_at_ms":999979,"credit_policy":null,"job_id":"job-2397","job_issued_at_ms":999996,"miner_id":"miner-10","network_difficulty":1000000,"ntime":1700002397,"order_key":"10:miner-10","p2mr_program_hex":"5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d","share_difficulty":3,"share_id":"share-2397","share_seq":2397,"template_height":800074},{"accepted_at_ms":999996,"credit_policy":null,"job_id":"job-3867","job_issued_at_ms":999992,"miner_id":"miner-6","network_difficulty":1000008,"ntime":1700003867,"order_key":"06:miner-6","p2mr_program_hex":"1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b","share_difficulty":3,"share_id":"share-3867","share_seq":3867,"template_height":800120},{"accepted_at_ms":999971,"credit_policy":null,"job_id":"job-3978","job_issued_at_ms":999975,"miner_id":"miner-7","network_difficulty":1000000,"ntime":1700003978,"order_key":"07:miner-7","p2mr_program_hex":"8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a","share_difficulty":1,"share_id":"share-3978","share_seq":3978,"template_height":800124},{"accepted_at_ms":999990,"credit_policy":"stale-grace","job_id":"job-1098","job_issued_at_ms":999989,"miner_id":"miner-9","network_difficulty":1000010,"ntime":1700001098,"order_key":"09:miner-9","p2mr_program_hex":"4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a","share_difficulty":2,"share_id":"share-1098","share_seq":1098,"template_height":800034},{"accepted_at_ms":999991,"credit_policy":null,"job_id":"job-1833","job_issued_at_ms":999985,"miner_id":"miner-7","network_difficulty":1000014,"ntime":1700001833,"order_key":"07:miner-7","p2mr_program_hex":"2929292929292929292929292929292929292929292929292929292929292929","share_difficulty":1000,"share_id":"share-1833","share_seq":1833,"template_height":800057},{"accepted_at_ms":1000017,"credit_policy":"stale-grace","job_id":"job-1439","job_issued_at_ms":999983,"miner_id":"miner-9","network_difficulty":1000011,"ntime":1700001439,"order_key":"09:miner-9","p2mr_program_hex":"9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f","share_difficulty":1000,"share_id":"share-1439","share_seq":1439,"template_height":800044},{"accepted_at_ms":999977,"credit_policy":null,"job_id":"job-514","job_issued_at_ms":999990,"miner_id":"miner-8","network_difficulty":1000004,"ntime":1700000514,"order_key":"08:miner-8","p2mr_program_hex":"0202020202020202020202020202020202020202020202020202020202020202","share_difficulty":2,"share_id":"share-514","share_seq":514,"template_height":800016},{"accepted_at_ms":999981,"credit_policy":null,"job_id":"job-2563","job_issued_at_ms":999970,"miner_id":"miner-0","network_difficulty":1000013,"ntime":1700002563,"order_key":"00:miner-0","p2mr_program_hex":"0303030303030303030303030303030303030303030303030303030303030303","share_difficulty":1,"share_id":"share-2563","share_seq":2563,"template_height":800080},{"accepted_at_ms":999989,"credit_policy":null,"job_id":"job-2702","job_issued_at_ms":999984,"miner_id":"miner-7","network_difficulty":1000016,"ntime":1700002702,"order_key":"07:miner-7","p2mr_program_hex":"8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e","share_difficulty":2,"share_id":"share-2702","share_seq":2702,"template_height":800084},{"accepted_at_ms":999998,"credit_policy":null,"job_id":"job-3711","job_issued_at_ms":1000165,"miner_id":"miner-4","network_difficulty":1000005,"ntime":1700003711,"order_key":"04:miner-4","p2mr_program_hex":"7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f","share_difficulty":1,"share_id":"share-3711","share_seq":3711,"template_height":800115},{"accepted_at_ms":999987,"credit_policy":null,"job_id":"job-507","job_issued_at_ms":999991,"miner_id":"miner-1","network_difficulty":1000014,"ntime":1700000507,"order_key":"01:miner-1","p2mr_program_hex":"fbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfb","share_difficulty":7,"share_id":"share-507","share_seq":507,"template_height":800015},{"accepted_at_ms":1000454,"credit_policy":null,"job_id":"job-2169","job_issued_at_ms":1000411,"miner_id":"miner-2","network_difficulty":1000010,"ntime":1700002169,"order_key":"02:miner-2","p2mr_program_hex":"7979797979797979797979797979797979797979797979797979797979797979","share_difficulty":1000,"share_id":"share-2169","share_seq":2169,"template_height":800067},{"accepted_at_ms":999974,"credit_policy":null,"job_id":"job-1011","job_issued_at_ms":999979,"miner_id":"miner-10","network_difficulty":1000008,"ntime":1700001011,"order_key":"10:miner-10","p2mr_program_hex":"f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3","share_difficulty":1000,"share_id":"share-1011","share_seq":1011,"template_height":800031},{"accepted_at_ms":999982,"credit_policy":"stale-grace","job_id":"job-2615","job_issued_at_ms":999999,"miner_id":"miner-8","network_difficulty":1000014,"ntime":1700002615,"order_key":"08:miner-8","p2mr_program_hex":"3737373737373737373737373737373737373737373737373737373737373737","share_difficulty":3,"share_id":"share-2615","share_seq":2615,"template_height":800081},{"accepted_at_ms":999986,"credit_policy":null,"job_id":"job-666","job_issued_at_ms":999984,"miner_id":"miner-6","network_difficulty":1000003,"ntime":1700000666,"order_key":"06:miner-6","p2mr_program_hex":"9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a","share_difficulty":7,"share_id":"share-666","share_seq":666,"template_height":800020},{"accepted_at_ms":999974,"credit_policy":null,"job_id":"job-2543","job_issued_at_ms":999988,"miner_id":"miner-2","network_difficulty":1000010,"ntime":1700002543,"order_key":"02:miner-2","p2mr_program_hex":"efefefefefefefefefefefefefefefefefefefefefefefefefefefefefefefef","share_difficulty":3,"share_id":"share-2543","share_seq":2543,"template_height":800079},{"accepted_at_ms":1000038,"credit_policy":"stale-grace","job_id":"job-3713","job_issued_at_ms":999996,"miner_id":"miner-6","network_difficulty":1000007,"ntime":1700003713,"order_key":"06:miner-6","p2mr_program_hex":"8181818181818181818181818181818181818181818181818181818181818181","share_difficulty":1000,"share_id":"share-3713","share_seq":3713,"template_height":800116},{"accepted_at_ms":999996,"credit_policy":"stale-grace","job_id":"job-2791","job_issued_at_ms":999999,"miner_id":"miner-8","network_difficulty":1000003,"ntime":1700002791,"order_key":"08:miner-8","p2mr_program_hex":"e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7","share_difficulty":2,"share_id":"share-2791","share_seq":2791,"template_height":800087},{"accepted_at_ms":999971,"credit_policy":null,"job_id":"job-3229","job_issued_at_ms":999982,"miner_id":"miner-6","network_difficulty":1000016,"ntime":1700003229,"order_key":"06:miner-6","p2mr_program_hex":"9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d","share_difficulty":19,"share_id":"share-3229","share_seq":3229,"template_height":800100},{"accepted_at_ms":999998,"credit_policy":null,"job_id":"job-368","job_issued_at_ms":999973,"miner_id":"miner-5","network_difficulty":1000011,"ntime":1700000368,"order_key":"05:miner-5","p2mr_program_hex":"7070707070707070707070707070707070707070707070707070707070707070","share_difficulty":7,"share_id":"share-368","share_seq":368,"template_height":800011},{"accepted_at_ms":999971,"credit_policy":null,"job_id":"job-2679","job_issued_at_ms":999983,"miner_id":"miner-6","network_difficulty":1000010,"ntime":1700002679,"order_key":"06:miner-6","p2mr_program_hex":"7777777777777777777777777777777777777777777777777777777777777777","share_difficulty":3,"share_id":"share-2679","share_seq":2679,"template_height":800083},{"accepted_at_ms":999988,"credit_policy":null,"job_id":"job-2586","job_issued_at_ms":999973,"miner_id":"miner-1","network_difficulty":1000002,"ntime":1700002586,"order_key":"01:miner-1","p2mr_program_hex":"1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a","share_difficulty":1000,"share_id":"share-2586","share_seq":2586,"template_height":800080},{"accepted_at_ms":999988,"credit_policy":null,"job_id":"job-2959","job_issued_at_ms":999976,"miner_id":"miner-0","network_difficulty":1000001,"ntime":1700002959,"order_key":"00:miner-0","p2mr_program_hex":"8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f","share_difficulty":1,"share_id":"share-2959","share_seq":2959,"template_height":800092},{"accepted_at_ms":999984,"credit_policy":null,"job_id":"job-607","job_issued_at_ms":999985,"miner_id":"miner-2","network_difficulty":1000012,"ntime":1700000607,"order_key":"02:miner-2","p2mr_program_hex":"5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f","share_difficulty":101,"share_id":"share-607","share_seq":607,"template_height":800018},{"accepted_at_ms":999982,"credit_policy":null,"job_id":"job-1201","job_issued_at_ms":999991,"miner_id":"miner-2","network_difficulty":1000011,"ntime":1700001201,"order_key":"02:miner-2","p2mr_program_hex":"b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1","share_difficulty":101,"share_id":"share-1201","share_seq":1201,"template_height":800037},{"accepted_at_ms":999988,"credit_policy":"stale-grace","job_id":"job-3295","job_issued_at_ms":999992,"miner_id":"miner-6","network_difficulty":1000014,"ntime":1700003295,"order_key":"06:miner-6","p2mr_program_hex":"dfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdf","share_difficulty":7,"share_id":"share-3295","share_seq":3295,"template_height":800102},{"accepted_at_ms":999990,"credit_policy":null,"job_id":"job-3851","job_issued_at_ms":999996,"miner_id":"miner-1","network_difficulty":1000009,"ntime":1700003851,"order_key":"01:miner-1","p2mr_program_hex":"0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b","share_difficulty":101,"share_id":"share-3851","share_seq":3851,"template_height":800120},{"accepted_at_ms":999973,"credit_policy":null,"job_id":"job-3197","job_issued_at_ms":999981,"miner_id":"miner-7","network_difficulty":1000001,"ntime":1700003197,"order_key":"07:miner-7","p2mr_program_hex":"7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d","share_difficulty":19,"share_id":"share-3197","share_seq":3197,"template_height":800099},{"accepted_at_ms":999972,"credit_policy":"stale-grace","job_id":"job-778","job_issued_at_ms":999982,"miner_id":"miner-8","network_difficulty":1000013,"ntime":1700000778,"order_key":"08:miner-8","p2mr_program_hex":"0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a","share_difficulty":1,"share_id":"share-778","share_seq":778,"template_height":800024},{"accepted_at_ms":999979,"credit_policy":null,"job_id":"job-62","job_issued_at_ms":999981,"miner_id":"miner-7","network_difficulty":1000011,"ntime":1700000062,"order_key":"07:miner-7","p2mr_program_hex":"3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e","share_difficulty":2,"share_id":"share-62","share_seq":62,"template_height":800001},{"accepted_at_ms":999973,"credit_policy":"stale-grace","job_id":"job-3757","job_issued_at_ms":999980,"miner_id":"miner-6","network_difficulty":1000000,"ntime":1700003757,"order_key":"06:miner-6","p2mr_program_hex":"adadadadadadadadadadadadadadadadadadadadadadadadadadadadadadadad","share_difficulty":19,"share_id":"share-3757","share_seq":3757,"template_height":800117},{"accepted_at_ms":999992,"credit_policy":null,"job_id":"job-3803","job_issued_at_ms":999978,"miner_id":"miner-8","network_difficulty":1000012,"ntime":1700003803,"order_key":"08:miner-8","p2mr_program_hex":"dbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdb","share_difficulty":101,"share_id":"share-3803","share_seq":3803,"template_height":800118},{"accepted_at_ms":999980,"credit_policy":"stale-grace","job_id":"job-280","job_issued_at_ms":999982,"miner_id":"miner-5","network_difficulty":1000008,"ntime":1700000280,"order_key":"05:miner-5","p2mr_program_hex":"1818181818181818181818181818181818181818181818181818181818181818","share_difficulty":7,"share_id":"share-280","share_seq":280,"template_height":800008},{"accepted_at_ms":999992,"credit_policy":null,"job_id":"job-936","job_issued_at_ms":999994,"miner_id":"miner-1","network_difficulty":1000001,"ntime":1700000936,"order_key":"01:miner-1","p2mr_program_hex":"a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8","share_difficulty":1,"share_id":"share-936","share_seq":936,"template_height":800029},{"accepted_at_ms":999982,"credit_policy":null,"job_id":"job-318","job_issued_at_ms":999983,"miner_id":"miner-10","network_difficulty":1000012,"ntime":1700000318,"order_key":"10:miner-10","p2mr_program_hex":"3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e","share_difficulty":1000,"share_id":"share-318","share_seq":318,"template_height":800009},{"accepted_at_ms":999987,"credit_policy":null,"job_id":"job-1114","job_issued_at_ms":1000000,"miner_id":"miner-3","network_difficulty":1000009,"ntime":1700001114,"order_key":"03:miner-3","p2mr_program_hex":"5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a","share_difficulty":3,"share_id":"share-1114","share_seq":1114,"template_height":800034},{"accepted_at_ms":999978,"credit_policy":"stale-grace","job_id":"job-2162","job_issued_at_ms":999996,"miner_id":"miner-6","network_difficulty":1000003,"ntime":1700002162,"order_key":"06:miner-6","p2mr_program_hex":"7272727272727272727272727272727272727272727272727272727272727272","share_difficulty":101,"share_id":"share-2162","share_seq":2162,"template_height":800067},{"accepted_at_ms":999983,"credit_policy":null,"job_id":"job-3138","job_issued_at_ms":999971,"miner_id":"miner-3","network_difficulty":1000010,"ntime":1700003138,"order_key":"03:miner-3","p2mr_program_hex":"4242424242424242424242424242424242424242424242424242424242424242","share_difficulty":1000,"share_id":"share-3138","share_seq":3138,"template_height":800098},{"accepted_at_ms":999976,"credit_policy":null,"job_id":"job-2880","job_issued_at_ms":999997,"miner_id":"miner-9","network_difficulty":1000007,"ntime":1700002880,"order_key":"09:miner-9","p2mr_program_hex":"4040404040404040404040404040404040404040404040404040404040404040","share_difficulty":3,"share_id":"share-2880","share_seq":2880,"template_height":800090},{"accepted_at_ms":999979,"credit_policy":null,"job_id":"job-3943","job_issued_at_ms":1000166,"miner_id":"miner-5","network_difficulty":1000016,"ntime":1700003943,"order_key":"05:miner-5","p2mr_program_hex":"6767676767676767676767676767676767676767676767676767676767676767","share_difficulty":3,"share_id":"share-3943","share_seq":3943,"template_height":800123},{"accepted_at_ms":999999,"credit_policy":null,"job_id":"job-3541","job_issued_at_ms":999971,"miner_id":"miner-10","network_difficulty":1000005,"ntime":1700003541,"order_key":"10:miner-10","p2mr_program_hex":"d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5","share_difficulty":19,"share_id":"share-3541","share_seq":3541,"template_height":800110},{"accepted_at_ms":999998,"credit_policy":null,"job_id":"job-3878","job_issued_at_ms":999997,"miner_id":"miner-6","network_difficulty":1000002,"ntime":1700003878,"order_key":"06:miner-6","p2mr_program_hex":"2626262626262626262626262626262626262626262626262626262626262626","share_difficulty":101,"share_id":"share-3878","share_seq":3878,"template_height":800121},{"accepted_at_ms":999975,"credit_policy":null,"job_id":"job-2947","job_issued_at_ms":999981,"miner_id":"miner-10","network_difficulty":1000006,"ntime":1700002947,"order_key":"10:miner-10","p2mr_program_hex":"8383838383838383838383838383838383838383838383838383838383838383","share_difficulty":3,"share_id":"share-2947","share_seq":2947,"template_height":800092},{"accepted_at_ms":999980,"credit_policy":null,"job_id":"job-2627","job_issued_at_ms":999981,"miner_id":"miner-9","network_difficulty":1000009,"ntime":1700002627,"order_key":"09:miner-9","p2mr_program_hex":"4343434343434343434343434343434343434343434343434343434343434343","share_difficulty":101,"share_id":"share-2627","share_seq":2627,"template_height":800082},{"accepted_at_ms":999992,"credit_policy":null,"job_id":"job-1957","job_issued_at_ms":999992,"miner_id":"miner-10","network_difficulty":1000002,"ntime":1700001957,"order_key":"10:miner-10","p2mr_program_hex":"a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5","share_difficulty":1000,"share_id":"share-1957","share_seq":1957,"template_height":800061},{"accepted_at_ms":999979,"credit_policy":"stale-grace","job_id":"job-3984","job_issued_at_ms":999979,"miner_id":"miner-2","network_difficulty":1000006,"ntime":1700003984,"order_key":"02:miner-2","p2mr_program_hex":"9090909090909090909090909090909090909090909090909090909090909090","share_difficulty":19,"share_id":"share-3984","share_seq":3984,"template_height":800124},{"accepted_at_ms":1000378,"credit_policy":"stale-grace","job_id":"job-1876","job_issued_at_ms":999984,"miner_id":"miner-6","network_difficulty":1000006,"ntime":1700001876,"order_key":"06:miner-6","p2mr_program_hex":"5454545454545454545454545454545454545454545454545454545454545454","share_difficulty":1,"share_id":"share-1876","share_seq":1876,"template_height":800058},{"accepted_at_ms":999978,"credit_policy":null,"job_id":"job-1916","job_issued_at_ms":999993,"miner_id":"miner-2","network_difficulty":1000012,"ntime":1700001916,"order_key":"02:miner-2","p2mr_program_hex":"7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c","share_difficulty":19,"share_id":"share-1916","share_seq":1916,"template_height":800059},{"accepted_at_ms":999996,"credit_policy":null,"job_id":"job-1795","job_issued_at_ms":999976,"miner_id":"miner-2","network_difficulty":1000010,"ntime":1700001795,"order_key":"02:miner-2","p2mr_program_hex":"0303030303030303030303030303030303030303030303030303030303030303","share_difficulty":1000,"share_id":"share-1795","share_seq":1795,"template_height":800056},{"accepted_at_ms":999989,"credit_policy":null,"job_id":"job-1132","job_issued_at_ms":999998,"miner_id":"miner-10","network_difficulty":1000010,"ntime":1700001132,"order_key":"10:miner-10","p2mr_program_hex":"6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c","share_difficulty":3,"share_id":"share-1132","share_seq":1132,"template_height":800035},{"accepted_at_ms":999982,"credit_policy":"stale-grace","job_id":"job-3238","job_issued_at_ms":999998,"miner_id":"miner-4","network_difficulty":1000008,"ntime":1700003238,"order_key":"04:miner-4","p2mr_program_hex":"a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6","share_difficulty":3,"share_id":"share-3238","share_seq":3238,"template_height":800101},{"accepted_at_ms":999973,"credit_policy":null,"job_id":"job-2445","job_issued_at_ms":999976,"miner_id":"miner-3","network_difficulty":1000014,"ntime":1700002445,"order_key":"03:miner-3","p2mr_program_hex":"8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d","share_difficulty":19,"share_id":"share-2445","share_seq":2445,"template_height":800076},{"accepted_at_ms":999987,"credit_policy":null,"job_id":"job-1332","job_issued_at_ms":999998,"miner_id":"miner-1","network_difficulty":1000006,"ntime":1700001332,"order_key":"01:miner-1","p2mr_program_hex":"3434343434343434343434343434343434343434343434343434343434343434","share_difficulty":1,"share_id":"share-1332","share_seq":1332,"template_height":800041},{"accepted_at_ms":999992,"credit_policy":null,"job_id":"job-2709","job_issued_at_ms":999971,"miner_id":"miner-3","network_difficulty":1000006,"ntime":1700002709,"order_key":"03:miner-3","p2mr_program_hex":"9595959595959595959595959595959595959595959595959595959595959595","share_difficulty":2,"share_id":"share-2709","share_seq":2709,"template_height":800084},{"accepted_at_ms":999998,"credit_policy":null,"job_id":"job-1197","job_issued_at_ms":999988,"miner_id":"miner-9","network_difficulty":1000007,"ntime":1700001197,"order_key":"09:miner-9","p2mr_program_hex":"adadadadadadadadadadadadadadadadadadadadadadadadadadadadadadadad","share_difficulty":7,"share_id":"share-1197","share_seq":1197,"template_height":800037},{"accepted_at_ms":999972,"credit_policy":null,"job_id":"job-380","job_issued_at_ms":999989,"miner_id":"miner-6","network_difficulty":1000006,"ntime":1700000380,"order_key":"06:miner-6","p2mr_program_hex":"7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c","share_difficulty":7,"share_id":"share-380","share_seq":380,"template_height":800011},{"accepted_at_ms":1000363,"credit_policy":null,"job_id":"job-3920","job_issued_at_ms":1000205,"miner_id":"miner-4","network_difficulty":1000010,"ntime":1700003920,"order_key":"04:miner-4","p2mr_program_hex":"5050505050505050505050505050505050505050505050505050505050505050","share_difficulty":1000,"share_id":"share-3920","share_seq":3920,"template_height":800122},{"accepted_at_ms":1000155,"credit_policy":null,"job_id":"job-924","job_issued_at_ms":999983,"miner_id":"miner-0","network_difficulty":1000006,"ntime":1700000924,"order_key":"00:miner-0","p2mr_program_hex":"9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c","share_difficulty":7,"share_id":"share-924","share_seq":924,"template_height":800028},{"accepted_at_ms":999970,"credit_policy":null,"job_id":"job-1245","job_issued_at_ms":999975,"miner_id":"miner-2","network_difficulty":1000004,"ntime":1700001245,"order_key":"02:miner-2","p2mr_program_hex":"dddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddd","share_difficulty":19,"share_id":"share-1245","share_seq":1245,"template_height":800038},{"accepted_at_ms":999999,"credit_policy":null,"job_id":"job-2516","job_issued_at_ms":999974,"miner_id":"miner-8","network_difficulty":1000000,"ntime":1700002516,"order_key":"08:miner-8","p2mr_program_hex":"d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4","share_difficulty":101,"share_id":"share-2516","share_seq":2516,"template_height":800078},{"accepted_at_ms":999997,"credit_policy":null,"job_id":"job-195","job_issued_at_ms":999988,"miner_id":"miner-8","network_difficulty":1000008,"ntime":1700000195,"order_key":"08:miner-8","p2mr_program_hex":"c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3","share_difficulty":3,"share_id":"share-195","share_seq":195,"template_height":800006},{"accepted_at_ms":999978,"credit_policy":"stale-grace","job_id":"job-3760","job_issued_at_ms":999993,"miner_id":"miner-9","network_difficulty":1000003,"ntime":1700003760,"order_key":"09:miner-9","p2mr_program_hex":"b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0","share_difficulty":101,"share_id":"share-3760","share_seq":3760,"template_height":800117},{"accepted_at_ms":999999,"credit_policy":"stale-grace","job_id":"job-3703","job_issued_at_ms":1000471,"miner_id":"miner-7","network_difficulty":1000014,"ntime":1700003703,"order_key":"07:miner-7","p2mr_program_hex":"7777777777777777777777777777777777777777777777777777777777777777","share_difficulty":19,"share_id":"share-3703","share_seq":3703,"template_height":800115},{"accepted_at_ms":999987,"credit_policy":null,"job_id":"job-3547","job_issued_at_ms":999993,"miner_id":"miner-5","network_difficulty":1000011,"ntime":1700003547,"order_key":"05:miner-5","p2mr_program_hex":"dbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdb","share_difficulty":101,"share_id":"share-3547","share_seq":3547,"template_height":800110},{"accepted_at_ms":1000173,"credit_policy":"stale-grace","job_id":"job-1950","job_issued_at_ms":999975,"miner_id":"miner-3","network_difficulty":1000012,"ntime":1700001950,"order_key":"03:miner-3","p2mr_program_hex":"9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e","share_difficulty":7,"share_id":"share-1950","share_seq":1950,"template_height":800060},{"accepted_at_ms":999981,"credit_policy":null,"job_id":"job-2243","job_issued_at_ms":999991,"miner_id":"miner-10","network_difficulty":1000016,"ntime":1700002243,"order_key":"10:miner-10","p2mr_program_hex":"c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3","share_difficulty":2,"share_id":"share-2243","share_seq":2243,"template_height":800070},{"accepted_at_ms":999999,"credit_policy":null,"job_id":"job-954","job_issued_at_ms":999975,"miner_id":"miner-8","network_difficulty":1000002,"ntime":1700000954,"order_key":"08:miner-8","p2mr_program_hex":"babababababababababababababababababababababababababababababababa","share_difficulty":3,"share_id":"share-954","share_seq":954,"template_height":800029},{"accepted_at_ms":999993,"credit_policy":"stale-grace","job_id":"job-2961","job_issued_at_ms":999981,"miner_id":"miner-2","network_difficulty":1000003,"ntime":1700002961,"order_key":"02:miner-2","p2mr_program_hex":"9191919191919191919191919191919191919191919191919191919191919191","share_difficulty":101,"share_id":"share-2961","share_seq":2961,"template_height":800092},{"accepted_at_ms":999983,"credit_policy":null,"job_id":"job-211","job_issued_at_ms":999996,"miner_id":"miner-2","network_difficulty":1000007,"ntime":1700000211,"order_key":"02:miner-2","p2mr_program_hex":"d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3","share_difficulty":2,"share_id":"share-211","share_seq":211,"template_height":800006},{"accepted_at_ms":999971,"credit_policy":null,"job_id":"job-3694","job_issued_at_ms":999974,"miner_id":"miner-9","network_difficulty":1000005,"ntime":1700003694,"order_key":"09:miner-9","p2mr_program_hex":"6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e","share_difficulty":7,"share_id":"share-3694","share_seq":3694,"template_height":800115},{"accepted_at_ms":999975,"credit_policy":null,"job_id":"job-2835","job_issued_at_ms":999975,"miner_id":"miner-8","network_difficulty":1000013,"ntime":1700002835,"order_key":"08:miner-8","p2mr_program_hex":"1313131313131313131313131313131313131313131313131313131313131313","share_difficulty":101,"share_id":"share-2835","share_seq":2835,"template_height":800088},{"accepted_at_ms":999977,"credit_policy":null,"job_id":"job-610","job_issued_at_ms":999983,"miner_id":"miner-5","network_difficulty":1000015,"ntime":1700000610,"order_key":"05:miner-5","p2mr_program_hex":"6262626262626262626262626262626262626262626262626262626262626262","share_difficulty":2,"share_id":"share-610","share_seq":610,"template_height":800019},{"accepted_at_ms":999970,"credit_policy":null,"job_id":"job-2512","job_issued_at_ms":999988,"miner_id":"miner-4","network_difficulty":1000013,"ntime":1700002512,"order_key":"04:miner-4","p2mr_program_hex":"d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0","share_difficulty":2,"share_id":"share-2512","share_seq":2512,"template_height":800078},{"accepted_at_ms":999976,"credit_policy":null,"job_id":"job-3287","job_issued_at_ms":999979,"miner_id":"miner-9","network_difficulty":1000006,"ntime":1700003287,"order_key":"09:miner-9","p2mr_program_hex":"d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7","share_difficulty":19,"share_id":"share-3287","share_seq":3287,"template_height":800102},{"accepted_at_ms":999980,"credit_policy":null,"job_id":"job-1129","job_issued_at_ms":999993,"miner_id":"miner-7","network_difficulty":1000007,"ntime":1700001129,"order_key":"07:miner-7","p2mr_program_hex":"6969696969696969696969696969696969696969696969696969696969696969","share_difficulty":7,"share_id":"share-1129","share_seq":1129,"template_height":800035},{"accepted_at_ms":999992,"credit_policy":null,"job_id":"job-591","job_issued_at_ms":999994,"miner_id":"miner-8","network_difficulty":1000013,"ntime":1700000591,"order_key":"08:miner-8","p2mr_program_hex":"4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f","share_difficulty":19,"share_id":"share-591","share_seq":591,"template_height":800018},{"accepted_at_ms":999980,"credit_policy":null,"job_id":"job-3649","job_issued_at_ms":999977,"miner_id":"miner-8","network_difficulty":1000011,"ntime":1700003649,"order_key":"08:miner-8","p2mr_program_hex":"4141414141414141414141414141414141414141414141414141414141414141","share_difficulty":7,"share_id":"share-3649","share_seq":3649,"template_height":800114},{"accepted_at_ms":999982,"credit_policy":"stale-grace","job_id":"job-3737","job_issued_at_ms":999976,"miner_id":"miner-8","network_difficulty":1000014,"ntime":1700003737,"order_key":"08:miner-8","p2mr_program_hex":"9999999999999999999999999999999999999999999999999999999999999999","share_difficulty":2,"share_id":"share-3737","share_seq":3737,"template_height":800116},{"accepted_at_ms":999971,"credit_policy":null,"job_id":"job-843","job_issued_at_ms":999977,"miner_id":"miner-7","network_difficulty":1000010,"ntime":1700000843,"order_key":"07:miner-7","p2mr_program_hex":"4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b","share_difficulty":1,"share_id":"share-843","share_seq":843,"template_height":800026},{"accepted_at_ms":1000000,"credit_policy":null,"job_id":"job-3333","job_issued_at_ms":999972,"miner_id":"miner-0","network_difficulty":1000001,"ntime":1700003333,"order_key":"00:miner-0","p2mr_program_hex":"0505050505050505050505050505050505050505050505050505050505050505","share_difficulty":19,"share_id":"share-3333","share_seq":3333,"template_height":800104},{"accepted_at_ms":999979,"credit_policy":null,"job_id":"job-3410","job_issued_at_ms":999980,"miner_id":"miner-0","network_difficulty":1000010,"ntime":1700003410,"order_key":"00:miner-0","p2mr_program_hex":"5252525252525252525252525252525252525252525252525252525252525252","share_difficulty":1,"share_id":"share-3410","share_seq":3410,"template_height":800106},{"accepted_at_ms":1000349,"credit_policy":null,"job_id":"job-923","job_issued_at_ms":999992,"miner_id":"miner-10","network_difficulty":1000005,"ntime":1700000923,"order_key":"10:miner-10","p2mr_program_hex":"9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b","share_difficulty":1000,"share_id":"share-923","share_seq":923,"template_height":800028},{"accepted_at_ms":999974,"credit_policy":null,"job_id":"job-1716","job_issued_at_ms":999990,"miner_id":"miner-0","network_difficulty":1000016,"ntime":1700001716,"order_key":"00:miner-0","p2mr_program_hex":"b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4","share_difficulty":7,"share_id":"share-1716","share_seq":1716,"template_height":800053},{"accepted_at_ms":999995,"credit_policy":"stale-grace","job_id":"job-3331","job_issued_at_ms":999991,"miner_id":"miner-9","network_difficulty":1000016,"ntime":1700003331,"order_key":"09:miner-9","p2mr_program_hex":"0303030303030303030303030303030303030303030303030303030303030303","share_difficulty":1000,"share_id":"share-3331","share_seq":3331,"template_height":800104},{"accepted_at_ms":1000000,"credit_policy":null,"job_id":"job-1277","job_issued_at_ms":999987,"miner_id":"miner-1","network_difficulty":1000002,"ntime":1700001277,"order_key":"01:miner-1","p2mr_program_hex":"fdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfd","share_difficulty":101,"share_id":"share-1277","share_seq":1277,"template_height":800039},{"accepted_at_ms":999998,"credit_policy":null,"job_id":"job-2134","job_issued_at_ms":999999,"miner_id":"miner-0","network_difficulty":1000009,"ntime":1700002134,"order_key":"00:miner-0","p2mr_program_hex":"5656565656565656565656565656565656565656565656565656565656565656","share_difficulty":1,"share_id":"share-2134","share_seq":2134,"template_height":800066},{"accepted_at_ms":999974,"credit_policy":"stale-grace","job_id":"job-2698","job_issued_at_ms":999975,"miner_id":"miner-3","network_difficulty":1000012,"ntime":1700002698,"order_key":"03:miner-3","p2mr_program_hex":"8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a","share_difficulty":7,"share_id":"share-2698","share_seq":2698,"template_height":800084},{"accepted_at_ms":999995,"credit_policy":"stale-grace","job_id":"job-3862","job_issued_at_ms":999977,"miner_id":"miner-1","network_difficulty":1000003,"ntime":1700003862,"order_key":"01:miner-1","p2mr_program_hex":"1616161616161616161616161616161616161616161616161616161616161616","share_difficulty":1000,"share_id":"share-3862","share_seq":3862,"template_height":800120},{"accepted_at_ms":999994,"credit_policy":null,"job_id":"job-1783","job_issued_at_ms":999988,"miner_id":"miner-1","network_difficulty":1000015,"ntime":1700001783,"order_key":"01:miner-1","p2mr_program_hex":"f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7","share_difficulty":19,"share_id":"share-1783","share_seq":1783,"template_height":800055},{"accepted_at_ms":999991,"credit_policy":null,"job_id":"job-3296","job_issued_at_ms":999983,"miner_id":"miner-7","network_difficulty":1000015,"ntime":1700003296,"order_key":"07:miner-7","p2mr_program_hex":"e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0","share_difficulty":1000,"share_id":"share-3296","share_seq":3296,"template_height":800103},{"accepted_at_ms":999994,"credit_policy":"stale-grace","job_id":"job-1405","job_issued_at_ms":999970,"miner_id":"miner-8","network_difficulty":1000011,"ntime":1700001405,"order_key":"08:miner-8","p2mr_program_hex":"7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d","share_difficulty":1000,"share_id":"share-1405","share_seq":1405,"template_height":800043},{"accepted_at_ms":999997,"credit_policy":"stale-grace","job_id":"job-3972","job_issued_at_ms":999970,"miner_id":"miner-1","network_difficulty":1000011,"ntime":1700003972,"order_key":"01:miner-1","p2mr_program_hex":"8484848484848484848484848484848484848484848484848484848484848484","share_difficulty":1,"share_id":"share-3972","share_seq":3972,"template_height":800124},{"accepted_at_ms":999996,"credit_policy":null,"job_id":"job-1154","job_issued_at_ms":999984,"miner_id":"miner-10","network_difficulty":1000015,"ntime":1700001154,"order_key":"10:miner-10","p2mr_program_hex":"8282828282828282828282828282828282828282828282828282828282828282","share_difficulty":19,"share_id":"share-1154","share_seq":1154,"template_height":800036},{"accepted_at_ms":999986,"credit_policy":null,"job_id":"job-2055","job_issued_at_ms":999989,"miner_id":"miner-9","network_difficulty":1000015,"ntime":1700002055,"order_key":"09:miner-9","p2mr_program_hex":"0707070707070707070707070707070707070707070707070707070707070707","share_difficulty":2,"share_id":"share-2055","share_seq":2055,"template_height":800064},{"accepted_at_ms":999988,"credit_policy":null,"job_id":"job-2081","job_issued_at_ms":999981,"miner_id":"miner-2","network_difficulty":1000007,"ntime":1700002081,"order_key":"02:miner-2","p2mr_program_hex":"2121212121212121212121212121212121212121212121212121212121212121","share_difficulty":3,"share_id":"share-2081","share_seq":2081,"template_height":800065},{"accepted_at_ms":999980,"credit_policy":null,"job_id":"job-2377","job_issued_at_ms":999997,"miner_id":"miner-1","network_difficulty":1000014,"ntime":1700002377,"order_key":"01:miner-1","p2mr_program_hex":"4949494949494949494949494949494949494949494949494949494949494949","share_difficulty":1000,"share_id":"share-2377","share_seq":2377,"template_height":800074},{"accepted_at_ms":1000000,"credit_policy":"stale-grace","job_id":"job-3398","job_issued_at_ms":999994,"miner_id":"miner-10","network_difficulty":1000015,"ntime":1700003398,"order_key":"10:miner-10","p2mr_program_hex":"4646464646464646464646464646464646464646464646464646464646464646","share_difficulty":19,"share_id":"share-3398","share_seq":3398,"template_height":800106},{"accepted_at_ms":999993,"credit_policy":null,"job_id":"job-997","job_issued_at_ms":999997,"miner_id":"miner-7","network_difficulty":1000011,"ntime":1700000997,"order_key":"07:miner-7","p2mr_program_hex":"e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5","share_difficulty":3,"share_id":"share-997","share_seq":997,"template_height":800031},{"accepted_at_ms":999980,"credit_policy":null,"job_id":"job-1479","job_issued_at_ms":999974,"miner_id":"miner-5","network_difficulty":1000000,"ntime":1700001479,"order_key":"05:miner-5","p2mr_program_hex":"c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7","share_difficulty":2,"share_id":"share-1479","share_seq":1479,"template_height":800046},{"accepted_at_ms":999988,"credit_policy":null,"job_id":"job-1314","job_issued_at_ms":999978,"miner_id":"miner-5","network_difficulty":1000005,"ntime":1700001314,"order_key":"05:miner-5","p2mr_program_hex":"2222222222222222222222222222222222222222222222222222222222222222","share_difficulty":7,"share_id":"share-1314","share_seq":1314,"template_height":800041},{"accepted_at_ms":999990,"credit_policy":null,"job_id":"job-2338","job_issued_at_ms":999974,"miner_id":"miner-6","network_difficulty":1000009,"ntime":1700002338,"order_key":"06:miner-6","p2mr_program_hex":"2222222222222222222222222222222222222222222222222222222222222222","share_difficulty":19,"share_id":"share-2338","share_seq":2338,"template_height":800073},{"accepted_at_ms":999986,"credit_policy":null,"job_id":"job-267","job_issued_at_ms":999996,"miner_id":"miner-3","network_difficulty":1000012,"ntime":1700000267,"order_key":"03:miner-3","p2mr_program_hex":"0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b","share_difficulty":101,"share_id":"share-267","share_seq":267,"template_height":800008},{"accepted_at_ms":999972,"credit_policy":null,"job_id":"job-3783","job_issued_at_ms":999979,"miner_id":"miner-10","network_difficulty":1000009,"ntime":1700003783,"order_key":"10:miner-10","p2mr_program_hex":"c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7","share_difficulty":3,"share_id":"share-3783","share_seq":3783,"template_height":800118},{"accepted_at_ms":999997,"credit_policy":null,"job_id":"job-487","job_issued_at_ms":1000000,"miner_id":"miner-3","network_difficulty":1000011,"ntime":1700000487,"order_key":"03:miner-3","p2mr_program_hex":"e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7","share_difficulty":3,"share_id":"share-487","share_seq":487,"template_height":800015},{"accepted_at_ms":999986,"credit_policy":null,"job_id":"job-2042","job_issued_at_ms":999983,"miner_id":"miner-7","network_difficulty":1000002,"ntime":1700002042,"order_key":"07:miner-7","p2mr_program_hex":"fafafafafafafafafafafafafafafafafafafafafafafafafafafafafafafafa","share_difficulty":2,"share_id":"share-2042","share_seq":2042,"template_height":800063},{"accepted_at_ms":999971,"credit_policy":null,"job_id":"job-549","job_issued_at_ms":999981,"miner_id":"miner-10","network_difficulty":1000005,"ntime":1700000549,"order_key":"10:miner-10","p2mr_program_hex":"2525252525252525252525252525252525252525252525252525252525252525","share_difficulty":7,"share_id":"share-549","share_seq":549,"template_height":800017},{"accepted_at_ms":999982,"credit_policy":null,"job_id":"job-1287","job_issued_at_ms":999995,"miner_id":"miner-0","network_difficulty":1000012,"ntime":1700001287,"order_key":"00:miner-0","p2mr_program_hex":"0707070707070707070707070707070707070707070707070707070707070707","share_difficulty":19,"share_id":"share-1287","share_seq":1287,"template_height":800040},{"accepted_at_ms":999979,"credit_policy":null,"job_id":"job-2820","job_issued_at_ms":999975,"miner_id":"miner-4","network_difficulty":1000015,"ntime":1700002820,"order_key":"04:miner-4","p2mr_program_hex":"0404040404040404040404040404040404040404040404040404040404040404","share_difficulty":2,"share_id":"share-2820","share_seq":2820,"template_height":800088},{"accepted_at_ms":999991,"credit_policy":"stale-grace","job_id":"job-1226","job_issued_at_ms":999987,"miner_id":"miner-5","network_difficulty":1000002,"ntime":1700001226,"order_key":"05:miner-5","p2mr_program_hex":"cacacacacacacacacacacacacacacacacacacacacacacacacacacacacacacaca","share_difficulty":3,"share_id":"share-1226","share_seq":1226,"template_height":800038},{"accepted_at_ms":999989,"credit_policy":null,"job_id":"job-109","job_issued_at_ms":1000082,"miner_id":"miner-10","network_difficulty":1000007,"ntime":1700000109,"order_key":"10:miner-10","p2mr_program_hex":"6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d","share_difficulty":1000,"share_id":"share-109","share_seq":109,"template_height":800003},{"accepted_at_ms":999999,"credit_policy":null,"job_id":"job-786","job_issued_at_ms":999976,"miner_id":"miner-5","network_difficulty":1000004,"ntime":1700000786,"order_key":"05:miner-5","p2mr_program_hex":"1212121212121212121212121212121212121212121212121212121212121212","share_difficulty":101,"share_id":"share-786","share_seq":786,"template_height":800024},{"accepted_at_ms":999990,"credit_policy":null,"job_id":"job-3062","job_issued_at_ms":999975,"miner_id":"miner-4","network_difficulty":1000002,"ntime":1700003062,"order_key":"04:miner-4","p2mr_program_hex":"f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6","share_difficulty":19,"share_id":"share-3062","share_seq":3062,"template_height":800095},{"accepted_at_ms":999990,"credit_policy":null,"job_id":"job-1578","job_issued_at_ms":999987,"miner_id":"miner-5","network_difficulty":1000014,"ntime":1700001578,"order_key":"05:miner-5","p2mr_program_hex":"2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a","share_difficulty":2,"share_id":"share-1578","share_seq":1578,"template_height":800049},{"accepted_at_ms":999990,"credit_policy":"stale-grace","job_id":"job-2805","job_issued_at_ms":999995,"miner_id":"miner-0","network_difficulty":1000000,"ntime":1700002805,"order_key":"00:miner-0","p2mr_program_hex":"f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5","share_difficulty":3,"share_id":"share-2805","share_seq":2805,"template_height":800087},{"accepted_at_ms":999991,"credit_policy":null,"job_id":"job-2058","job_issued_at_ms":999976,"miner_id":"miner-1","network_difficulty":1000001,"ntime":1700002058,"order_key":"01:miner-1","p2mr_program_hex":"0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a","share_difficulty":7,"share_id":"share-2058","share_seq":2058,"template_height":800064},{"accepted_at_ms":999989,"credit_policy":null,"job_id":"job-2643","job_issued_at_ms":999981,"miner_id":"miner-3","network_difficulty":1000008,"ntime":1700002643,"order_key":"03:miner-3","p2mr_program_hex":"5353535353535353535353535353535353535353535353535353535353535353","share_difficulty":1,"share_id":"share-2643","share_seq":2643,"template_height":800082},{"accepted_at_ms":999974,"credit_policy":null,"job_id":"job-2851","job_issued_at_ms":999988,"miner_id":"miner-2","network_difficulty":1000012,"ntime":1700002851,"order_key":"02:miner-2","p2mr_program_hex":"2323232323232323232323232323232323232323232323232323232323232323","share_difficulty":7,"share_id":"share-2851","share_seq":2851,"template_height":800089},{"accepted_at_ms":999999,"credit_policy":"stale-grace","job_id":"job-662","job_issued_at_ms":999990,"miner_id":"miner-2","network_difficulty":1000016,"ntime":1700000662,"order_key":"02:miner-2","p2mr_program_hex":"9696969696969696969696969696969696969696969696969696969696969696","share_difficulty":2,"share_id":"share-662","share_seq":662,"template_height":800020},{"accepted_at_ms":999980,"credit_policy":null,"job_id":"job-1507","job_issued_at_ms":999975,"miner_id":"miner-0","network_difficulty":1000011,"ntime":1700001507,"order_key":"00:miner-0","p2mr_program_hex":"e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3","share_difficulty":19,"share_id":"share-1507","share_seq":1507,"template_height":800047},{"accepted_at_ms":999998,"credit_policy":"stale-grace","job_id":"job-2777","job_issued_at_ms":999993,"miner_id":"miner-5","network_difficulty":1000006,"ntime":1700002777,"order_key":"05:miner-5","p2mr_program_hex":"d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9","share_difficulty":3,"share_id":"share-2777","share_seq":2777,"template_height":800086},{"accepted_at_ms":999992,"credit_policy":null,"job_id":"job-969","job_issued_at_ms":999973,"miner_id":"miner-1","network_difficulty":1000000,"ntime":1700000969,"order_key":"01:miner-1","p2mr_program_hex":"c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9","share_difficulty":2,"share_id":"share-969","share_seq":969,"template_height":800030},{"accepted_at_ms":999993,"credit_policy":null,"job_id":"job-1097","job_issued_at_ms":999989,"miner_id":"miner-8","network_difficulty":1000009,"ntime":1700001097,"order_key":"08:miner-8","p2mr_program_hex":"4949494949494949494949494949494949494949494949494949494949494949","share_difficulty":19,"share_id":"share-1097","share_seq":1097,"template_height":800034},{"accepted_at_ms":999975,"credit_policy":null,"job_id":"job-2088","job_issued_at_ms":999973,"miner_id":"miner-9","network_difficulty":1000014,"ntime":1700002088,"order_key":"09:miner-9","p2mr_program_hex":"2828282828282828282828282828282828282828282828282828282828282828","share_difficulty":1000,"share_id":"share-2088","share_seq":2088,"template_height":800065},{"accepted_at_ms":999986,"credit_policy":"stale-grace","job_id":"job-3718","job_issued_at_ms":999984,"miner_id":"miner-0","network_difficulty":1000012,"ntime":1700003718,"order_key":"00:miner-0","p2mr_program_hex":"8686868686868686868686868686868686868686868686868686868686868686","share_difficulty":1,"share_id":"share-3718","share_seq":3718,"template_height":800116},{"accepted_at_ms":999998,"credit_policy":null,"job_id":"job-490","job_issued_at_ms":999990,"miner_id":"miner-6","network_difficulty":1000014,"ntime":1700000490,"order_key":"06:miner-6","p2mr_program_hex":"eaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaea","share_difficulty":2,"share_id":"share-490","share_seq":490,"template_height":800015},{"accepted_at_ms":999971,"credit_policy":"stale-grace","job_id":"job-1676","job_issued_at_ms":999971,"miner_id":"miner-4","network_difficulty":1000010,"ntime":1700001676,"order_key":"04:miner-4","p2mr_program_hex":"8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c","share_difficulty":2,"share_id":"share-1676","share_seq":1676,"template_height":800052},{"accepted_at_ms":999979,"credit_policy":"stale-grace","job_id":"job-3619","job_issued_at_ms":999974,"miner_id":"miner-0","network_difficulty":1000015,"ntime":1700003619,"order_key":"00:miner-0","p2mr_program_hex":"2323232323232323232323232323232323232323232323232323232323232323","share_difficulty":1000,"share_id":"share-3619","share_seq":3619,"template_height":800113},{"accepted_at_ms":999981,"credit_policy":null,"job_id":"job-499","job_issued_at_ms":999976,"miner_id":"miner-4","network_difficulty":1000006,"ntime":1700000499,"order_key":"04:miner-4","p2mr_program_hex":"f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3","share_difficulty":1000,"share_id":"share-499","share_seq":499,"template_height":800015},{"accepted_at_ms":999974,"credit_policy":null,"job_id":"job-2320","job_issued_at_ms":999985,"miner_id":"miner-10","network_difficulty":1000008,"ntime":1700002320,"order_key":"10:miner-10","p2mr_program_hex":"1010101010101010101010101010101010101010101010101010101010101010","share_difficulty":19,"share_id":"share-2320","share_seq":2320,"template_height":800072},{"accepted_at_ms":999998,"credit_policy":null,"job_id":"job-1042","job_issued_at_ms":999999,"miner_id":"miner-8","network_difficulty":1000005,"ntime":1700001042,"order_key":"08:miner-8","p2mr_program_hex":"1212121212121212121212121212121212121212121212121212121212121212","share_difficulty":101,"share_id":"share-1042","share_seq":1042,"template_height":800032},{"accepted_at_ms":999973,"credit_policy":null,"job_id":"job-133","job_issued_at_ms":999990,"miner_id":"miner-1","network_difficulty":1000014,"ntime":1700000133,"order_key":"01:miner-1","p2mr_program_hex":"8585858585858585858585858585858585858585858585858585858585858585","share_difficulty":1,"share_id":"share-133","share_seq":133,"template_height":800004},{"accepted_at_ms":999995,"credit_policy":null,"job_id":"job-2026","job_issued_at_ms":999986,"miner_id":"miner-2","network_difficulty":1000003,"ntime":1700002026,"order_key":"02:miner-2","p2mr_program_hex":"eaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaea","share_difficulty":7,"share_id":"share-2026","share_seq":2026,"template_height":800063},{"accepted_at_ms":999981,"credit_policy":"stale-grace","job_id":"job-3254","job_issued_at_ms":999994,"miner_id":"miner-9","network_difficulty":1000007,"ntime":1700003254,"order_key":"09:miner-9","p2mr_program_hex":"b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6","share_difficulty":2,"share_id":"share-3254","share_seq":3254,"template_height":800101},{"accepted_at_ms":999971,"credit_policy":"stale-grace","job_id":"job-1654","job_issued_at_ms":999996,"miner_id":"miner-4","network_difficulty":1000005,"ntime":1700001654,"order_key":"04:miner-4","p2mr_program_hex":"7676767676767676767676767676767676767676767676767676767676767676","share_difficulty":1000,"share_id":"share-1654","share_seq":1654,"template_height":800051},{"accepted_at_ms":999995,"credit_policy":"stale-grace","job_id":"job-3409","job_issued_at_ms":999986,"miner_id":"miner-10","network_difficulty":1000009,"ntime":1700003409,"order_key":"10:miner-10","p2mr_program_hex":"5151515151515151515151515151515151515151515151515151515151515151","share_difficulty":2,"share_id":"share-3409","share_seq":3409,"template_height":800106},{"accepted_at_ms":999970,"credit_policy":null,"job_id":"job-1085","job_issued_at_ms":999988,"miner_id":"miner-7","network_difficulty":1000014,"ntime":1700001085,"order_key":"07:miner-7","p2mr_program_hex":"3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d","share_difficulty":7,"share_id":"share-1085","share_seq":1085,"template_height":800033},{"accepted_at_ms":999981,"credit_policy":null,"job_id":"job-518","job_issued_at_ms":999989,"miner_id":"miner-1","network_difficulty":1000008,"ntime":1700000518,"order_key":"01:miner-1","p2mr_program_hex":"0606060606060606060606060606060606060606060606060606060606060606","share_difficulty":1000,"share_id":"share-518","share_seq":518,"template_height":800016},{"accepted_at_ms":999972,"credit_policy":null,"job_id":"job-1131","job_issued_at_ms":999972,"miner_id":"miner-9","network_difficulty":1000009,"ntime":1700001131,"order_key":"09:miner-9","p2mr_program_hex":"6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b","share_difficulty":3,"share_id":"share-1131","share_seq":1131,"template_height":800035},{"accepted_at_ms":1000000,"credit_policy":null,"job_id":"job-3993","job_issued_at_ms":999992,"miner_id":"miner-0","network_difficulty":1000015,"ntime":1700003993,"order_key":"00:miner-0","p2mr_program_hex":"9999999999999999999999999999999999999999999999999999999999999999","share_difficulty":1000,"share_id":"share-3993","share_seq":3993,"template_height":800124},{"accepted_at_ms":999972,"credit_policy":null,"job_id":"job-1282","job_issued_at_ms":999971,"miner_id":"miner-6","network_difficulty":1000007,"ntime":1700001282,"order_key":"06:miner-6","p2mr_program_hex":"0202020202020202020202020202020202020202020202020202020202020202","share_difficulty":1000,"share_id":"share-1282","share_seq":1282,"template_height":800040},{"accepted_at_ms":999991,"credit_policy":null,"job_id":"job-2802","job_issued_at_ms":999988,"miner_id":"miner-8","network_difficulty":1000014,"ntime":1700002802,"order_key":"08:miner-8","p2mr_program_hex":"f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2","share_difficulty":2,"share_id":"share-2802","share_seq":2802,"template_height":800087},{"accepted_at_ms":1000269,"credit_policy":null,"job_id":"job-2616","job_issued_at_ms":999979,"miner_id":"miner-9","network_difficulty":1000015,"ntime":1700002616,"order_key":"09:miner-9","p2mr_program_hex":"3838383838383838383838383838383838383838383838383838383838383838","share_difficulty":1000,"share_id":"share-2616","share_seq":2616,"template_height":800081},{"accepted_at_ms":999993,"credit_policy":null,"job_id":"job-3804","job_issued_at_ms":999983,"miner_id":"miner-9","network_difficulty":1000013,"ntime":1700003804,"order_key":"09:miner-9","p2mr_program_hex":"dcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdc","share_difficulty":2,"share_id":"share-3804","share_seq":3804,"template_height":800118},{"accepted_at_ms":999975,"credit_policy":null,"job_id":"job-1704","job_issued_at_ms":999972,"miner_id":"miner-10","network_difficulty":1000004,"ntime":1700001704,"order_key":"10:miner-10","p2mr_program_hex":"a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8","share_difficulty":19,"share_id":"share-1704","share_seq":1704,"template_height":800053},{"accepted_at_ms":999976,"credit_policy":null,"job_id":"job-3683","job_issued_at_ms":999999,"miner_id":"miner-9","network_difficulty":1000011,"ntime":1700003683,"order_key":"09:miner-9","p2mr_program_hex":"6363636363636363636363636363636363636363636363636363636363636363","share_difficulty":3,"share_id":"share-3683","share_seq":3683,"template_height":800115},{"accepted_at_ms":999972,"credit_policy":null,"job_id":"job-3744","job_issued_at_ms":999993,"miner_id":"miner-4","network_difficulty":1000004,"ntime":1700003744,"order_key":"04:miner-4","p2mr_program_hex":"a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0","share_difficulty":1000,"share_id":"share-3744","share_seq":3744,"template_height":800117},{"accepted_at_ms":999984,"credit_policy":null,"job_id":"job-3289","job_issued_at_ms":999985,"miner_id":"miner-0","network_difficulty":1000008,"ntime":1700003289,"order_key":"00:miner-0","p2mr_program_hex":"d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9","share_difficulty":7,"share_id":"share-3289","share_seq":3289,"template_height":800102},{"accepted_at_ms":999975,"credit_policy":null,"job_id":"job-3561","job_issued_at_ms":999972,"miner_id":"miner-8","network_difficulty":1000008,"ntime":1700003561,"order_key":"08:miner-8","p2mr_program_hex":"e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9","share_difficulty":1000,"share_id":"share-3561","share_seq":3561,"template_height":800111},{"accepted_at_ms":999984,"credit_policy":null,"job_id":"job-1451","job_issued_at_ms":999989,"miner_id":"miner-10","network_difficulty":1000006,"ntime":1700001451,"order_key":"10:miner-10","p2mr_program_hex":"abababababababababababababababababababababababababababababababab","share_difficulty":2,"share_id":"share-1451","share_seq":1451,"template_height":800045},{"accepted_at_ms":999987,"credit_policy":"stale-grace","job_id":"job-3483","job_issued_at_ms":999996,"miner_id":"miner-7","network_difficulty":1000015,"ntime":1700003483,"order_key":"07:miner-7","p2mr_program_hex":"9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b","share_difficulty":1,"share_id":"share-3483","share_seq":3483,"template_height":800108},{"accepted_at_ms":999974,"credit_policy":null,"job_id":"job-926","job_issued_at_ms":999981,"miner_id":"miner-2","network_difficulty":1000008,"ntime":1700000926,"order_key":"02:miner-2","p2mr_program_hex":"9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e","share_difficulty":1000,"share_id":"share-926","share_seq":926,"template_height":800028},{"accepted_at_ms":999982,"credit_policy":null,"job_id":"job-3193","job_issued_at_ms":999999,"miner_id":"miner-3","network_difficulty":1000014,"ntime":1700003193,"order_key":"03:miner-3","p2mr_program_hex":"7979797979797979797979797979797979797979797979797979797979797979","share_difficulty":1,"share_id":"share-3193","share_seq":3193,"template_height":800099},{"accepted_at_ms":999989,"credit_policy":"stale-grace","job_id":"job-3509","job_issued_at_ms":999972,"miner_id":"miner-0","network_difficulty":1000007,"ntime":1700003509,"order_key":"00:miner-0","p2mr_program_hex":"b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5","share_difficulty":1000,"share_id":"share-3509","share_seq":3509,"template_height":800109},{"accepted_at_ms":999990,"credit_policy":null,"job_id":"job-3824","job_issued_at_ms":999979,"miner_id":"miner-7","network_difficulty":1000016,"ntime":1700003824,"order_key":"07:miner-7","p2mr_program_hex":"f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0","share_difficulty":1000,"share_id":"share-3824","share_seq":3824,"template_height":800119},{"accepted_at_ms":999970,"credit_policy":null,"job_id":"job-2395","job_issued_at_ms":999976,"miner_id":"miner-8","network_difficulty":1000015,"ntime":1700002395,"order_key":"08:miner-8","p2mr_program_hex":"5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b","share_difficulty":101,"share_id":"share-2395","share_seq":2395,"template_height":800074},{"accepted_at_ms":999994,"credit_policy":null,"job_id":"job-2996","job_issued_at_ms":1000090,"miner_id":"miner-4","network_difficulty":1000004,"ntime":1700002996,"order_key":"04:miner-4","p2mr_program_hex":"b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4","share_difficulty":101,"share_id":"share-2996","share_seq":2996,"template_height":800093},{"accepted_at_ms":999973,"credit_policy":null,"job_id":"job-2349","job_issued_at_ms":999991,"miner_id":"miner-6","network_difficulty":1000003,"ntime":1700002349,"order_key":"06:miner-6","p2mr_program_hex":"2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d","share_difficulty":19,"share_id":"share-2349","share_seq":2349,"template_height":800073},{"accepted_at_ms":1000444,"credit_policy":null,"job_id":"job-2106","job_issued_at_ms":1000222,"miner_id":"miner-5","network_difficulty":1000015,"ntime":1700002106,"order_key":"05:miner-5","p2mr_program_hex":"3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a","share_difficulty":2,"share_id":"share-2106","share_seq":2106,"template_height":800065},{"accepted_at_ms":999998,"credit_policy":null,"job_id":"job-2041","job_issued_at_ms":999977,"miner_id":"miner-6","network_difficulty":1000001,"ntime":1700002041,"order_key":"06:miner-6","p2mr_program_hex":"f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9","share_difficulty":3,"share_id":"share-2041","share_seq":2041,"template_height":800063},{"accepted_at_ms":999973,"credit_policy":null,"job_id":"job-3545","job_issued_at_ms":999978,"miner_id":"miner-3","network_difficulty":1000009,"ntime":1700003545,"order_key":"03:miner-3","p2mr_program_hex":"d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9","share_difficulty":19,"share_id":"share-3545","share_seq":3545,"template_height":800110},{"accepted_at_ms":999976,"credit_policy":null,"job_id":"job-1899","job_issued_at_ms":999988,"miner_id":"miner-7","network_difficulty":1000012,"ntime":1700001899,"order_key":"07:miner-7","p2mr_program_hex":"6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b","share_difficulty":7,"share_id":"share-1899","share_seq":1899,"template_height":800059},{"accepted_at_ms":999973,"credit_policy":"stale-grace","job_id":"job-6","job_issued_at_ms":999989,"miner_id":"miner-6","network_difficulty":1000006,"ntime":1700000006,"order_key":"06:miner-6","p2mr_program_hex":"0606060606060606060606060606060606060606060606060606060606060606","share_difficulty":1000,"share_id":"share-6","share_seq":6,"template_height":800000},{"accepted_at_ms":999977,"credit_policy":null,"job_id":"job-725","job_issued_at_ms":1000153,"miner_id":"miner-10","network_difficulty":1000011,"ntime":1700000725,"order_key":"10:miner-10","p2mr_program_hex":"d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5","share_difficulty":1000,"share_id":"share-725","share_seq":725,"template_height":800022},{"accepted_at_ms":999996,"credit_policy":null,"job_id":"job-3706","job_issued_at_ms":999996,"miner_id":"miner-10","network_difficulty":1000000,"ntime":1700003706,"order_key":"10:miner-10","p2mr_program_hex":"7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a","share_difficulty":7,"share_id":"share-3706","share_seq":3706,"template_height":800115},{"accepted_at_ms":999995,"credit_policy":"stale-grace","job_id":"job-1817","job_issued_at_ms":999986,"miner_id":"miner-2","network_difficulty":1000015,"ntime":1700001817,"order_key":"02:miner-2","p2mr_program_hex":"1919191919191919191919191919191919191919191919191919191919191919","share_difficulty":7,"share_id":"share-1817","share_seq":1817,"template_height":800056},{"accepted_at_ms":999978,"credit_policy":null,"job_id":"job-1208","job_issued_at_ms":999970,"miner_id":"miner-9","network_difficulty":1000001,"ntime":1700001208,"order_key":"09:miner-9","p2mr_program_hex":"b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8","share_difficulty":7,"share_id":"share-1208","share_seq":1208,"template_height":800037},{"accepted_at_ms":999985,"credit_policy":null,"job_id":"job-3482","job_issued_at_ms":999987,"miner_id":"miner-6","network_difficulty":1000014,"ntime":1700003482,"order_key":"06:miner-6","p2mr_program_hex":"9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a","share_difficulty":7,"share_id":"share-3482","share_seq":3482,"template_height":800108},{"accepted_at_ms":999984,"credit_policy":null,"job_id":"job-1183","job_issued_at_ms":999996,"miner_id":"miner-6","network_difficulty":1000010,"ntime":1700001183,"order_key":"06:miner-6","p2mr_program_hex":"9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f","share_difficulty":7,"share_id":"share-1183","share_seq":1183,"template_height":800036},{"accepted_at_ms":1000000,"credit_policy":null,"job_id":"job-1224","job_issued_at_ms":999996,"miner_id":"miner-3","network_difficulty":1000000,"ntime":1700001224,"order_key":"03:miner-3","p2mr_program_hex":"c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8","share_difficulty":19,"share_id":"share-1224","share_seq":1224,"template_height":800038},{"accepted_at_ms":999979,"credit_policy":null,"job_id":"job-1478","job_issued_at_ms":999974,"miner_id":"miner-4","network_difficulty":1000016,"ntime":1700001478,"order_key":"04:miner-4","p2mr_program_hex":"c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6","share_difficulty":1,"share_id":"share-1478","share_seq":1478,"template_height":800046},{"accepted_at_ms":999977,"credit_policy":null,"job_id":"job-2518","job_issued_at_ms":999983,"miner_id":"miner-10","network_difficulty":1000002,"ntime":1700002518,"order_key":"10:miner-10","p2mr_program_hex":"d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6","share_difficulty":1000,"share_id":"share-2518","share_seq":2518,"template_height":800078},{"accepted_at_ms":999979,"credit_policy":null,"job_id":"job-1652","job_issued_at_ms":999975,"miner_id":"miner-2","network_difficulty":1000003,"ntime":1700001652,"order_key":"02:miner-2","p2mr_program_hex":"7474747474747474747474747474747474747474747474747474747474747474","share_difficulty":2,"share_id":"share-1652","share_seq":1652,"template_height":800051},{"accepted_at_ms":999970,"credit_policy":null,"job_id":"job-3337","job_issued_at_ms":999977,"miner_id":"miner-4","network_difficulty":1000005,"ntime":1700003337,"order_key":"04:miner-4","p2mr_program_hex":"0909090909090909090909090909090909090909090909090909090909090909","share_difficulty":2,"share_id":"share-3337","share_seq":3337,"template_height":800104},{"accepted_at_ms":999987,"credit_policy":null,"job_id":"job-581","job_issued_at_ms":999992,"miner_id":"miner-9","network_difficulty":1000003,"ntime":1700000581,"order_key":"09:miner-9","p2mr_program_hex":"4545454545454545454545454545454545454545454545454545454545454545","share_difficulty":1,"share_id":"share-581","share_seq":581,"template_height":800018},{"accepted_at_ms":999977,"credit_policy":null,"job_id":"job-2992","job_issued_at_ms":999994,"miner_id":"miner-0","network_difficulty":1000000,"ntime":1700002992,"order_key":"00:miner-0","p2mr_program_hex":"b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0","share_difficulty":101,"share_id":"share-2992","share_seq":2992,"template_height":800093},{"accepted_at_ms":999998,"credit_policy":null,"job_id":"job-3031","job_issued_at_ms":999991,"miner_id":"miner-6","network_difficulty":1000005,"ntime":1700003031,"order_key":"06:miner-6","p2mr_program_hex":"d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7","share_difficulty":101,"share_id":"share-3031","share_seq":3031,"template_height":800094},{"accepted_at_ms":999979,"credit_policy":null,"job_id":"job-2997","job_issued_at_ms":999998,"miner_id":"miner-5","network_difficulty":1000005,"ntime":1700002997,"order_key":"05:miner-5","p2mr_program_hex":"b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5","share_difficulty":7,"share_id":"share-2997","share_seq":2997,"template_height":800093},{"accepted_at_ms":999997,"credit_policy":null,"job_id":"job-208","job_issued_at_ms":999989,"miner_id":"miner-10","network_difficulty":1000004,"ntime":1700000208,"order_key":"10:miner-10","p2mr_program_hex":"d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0","share_difficulty":1,"share_id":"share-208","share_seq":208,"template_height":800006},{"accepted_at_ms":999996,"credit_policy":null,"job_id":"job-362","job_issued_at_ms":999989,"miner_id":"miner-10","network_difficulty":1000005,"ntime":1700000362,"order_key":"10:miner-10","p2mr_program_hex":"6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a","share_difficulty":2,"share_id":"share-362","share_seq":362,"template_height":800011},{"accepted_at_ms":999993,"credit_policy":"stale-grace","job_id":"job-2449","job_issued_at_ms":999992,"miner_id":"miner-7","network_difficulty":1000001,"ntime":1700002449,"order_key":"07:miner-7","p2mr_program_hex":"9191919191919191919191919191919191919191919191919191919191919191","share_difficulty":19,"share_id":"share-2449","share_seq":2449,"template_height":800076},{"accepted_at_ms":999973,"credit_policy":null,"job_id":"job-1400","job_issued_at_ms":999980,"miner_id":"miner-3","network_difficulty":1000006,"ntime":1700001400,"order_key":"03:miner-3","p2mr_program_hex":"7878787878787878787878787878787878787878787878787878787878787878","share_difficulty":1000,"share_id":"share-1400","share_seq":1400,"template_height":800043},{"accepted_at_ms":999982,"credit_policy":null,"job_id":"job-3305","job_issued_at_ms":999972,"miner_id":"miner-5","network_difficulty":1000007,"ntime":1700003305,"order_key":"05:miner-5","p2mr_program_hex":"e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9","share_difficulty":2,"share_id":"share-3305","share_seq":3305,"template_height":800103},{"accepted_at_ms":999974,"credit_policy":"stale-grace","job_id":"job-1934","job_issued_at_ms":999971,"miner_id":"miner-9","network_difficulty":1000013,"ntime":1700001934,"order_key":"09:miner-9","p2mr_program_hex":"8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e","share_difficulty":2,"share_id":"share-1934","share_seq":1934,"template_height":800060},{"accepted_at_ms":999998,"credit_policy":null,"job_id":"job-2666","job_issued_at_ms":999998,"miner_id":"miner-4","network_difficulty":1000014,"ntime":1700002666,"order_key":"04:miner-4","p2mr_program_hex":"6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a","share_difficulty":7,"share_id":"share-2666","share_seq":2666,"template_height":800083},{"accepted_at_ms":999997,"credit_policy":null,"job_id":"job-537","job_issued_at_ms":999976,"miner_id":"miner-9","network_difficulty":1000010,"ntime":1700000537,"order_key":"09:miner-9","p2mr_program_hex":"1919191919191919191919191919191919191919191919191919191919191919","share_difficulty":19,"share_id":"share-537","share_seq":537,"template_height":800016},{"accepted_at_ms":999990,"credit_policy":"stale-grace","job_id":"job-3255","job_issued_at_ms":999988,"miner_id":"miner-10","network_difficulty":1000008,"ntime":1700003255,"order_key":"10:miner-10","p2mr_program_hex":"b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7","share_difficulty":2,"share_id":"share-3255","share_seq":3255,"template_height":800101},{"accepted_at_ms":999984,"credit_policy":null,"job_id":"job-3273","job_issued_at_ms":999982,"miner_id":"miner-6","network_difficulty":1000009,"ntime":1700003273,"order_key":"06:miner-6","p2mr_program_hex":"c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9","share_difficulty":19,"share_id":"share-3273","share_seq":3273,"template_height":800102},{"accepted_at_ms":999982,"credit_policy":null,"job_id":"job-198","job_issued_at_ms":999978,"miner_id":"miner-0","network_difficulty":1000011,"ntime":1700000198,"order_key":"00:miner-0","p2mr_program_hex":"c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6","share_difficulty":1000,"share_id":"share-198","share_seq":198,"template_height":800006},{"accepted_at_ms":999980,"credit_policy":null,"job_id":"job-1250","job_issued_at_ms":999993,"miner_id":"miner-7","network_difficulty":1000009,"ntime":1700001250,"order_key":"07:miner-7","p2mr_program_hex":"e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2","share_difficulty":1,"share_id":"share-1250","share_seq":1250,"template_height":800039},{"accepted_at_ms":1000075,"credit_policy":"stale-grace","job_id":"job-1464","job_issued_at_ms":999981,"miner_id":"miner-1","network_difficulty":1000002,"ntime":1700001464,"order_key":"01:miner-1","p2mr_program_hex":"b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8","share_difficulty":101,"share_id":"share-1464","share_seq":1464,"template_height":800045},{"accepted_at_ms":999988,"credit_policy":null,"job_id":"job-2700","job_issued_at_ms":999990,"miner_id":"miner-5","network_difficulty":1000014,"ntime":1700002700,"order_key":"05:miner-5","p2mr_program_hex":"8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c","share_difficulty":1,"share_id":"share-2700","share_seq":2700,"template_height":800084},{"accepted_at_ms":999989,"credit_policy":null,"job_id":"job-1240","job_issued_at_ms":999979,"miner_id":"miner-8","network_difficulty":1000016,"ntime":1700001240,"order_key":"08:miner-8","p2mr_program_hex":"d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8","share_difficulty":3,"share_id":"share-1240","share_seq":1240,"template_height":800038},{"accepted_at_ms":999977,"credit_policy":null,"job_id":"job-3670","job_issued_at_ms":1000243,"miner_id":"miner-7","network_difficulty":1000015,"ntime":1700003670,"order_key":"07:miner-7","p2mr_program_hex":"5656565656565656565656565656565656565656565656565656565656565656","share_difficulty":19,"share_id":"share-3670","share_seq":3670,"template_height":800114},{"accepted_at_ms":999978,"credit_policy":null,"job_id":"job-966","job_issued_at_ms":999977,"miner_id":"miner-9","network_difficulty":1000014,"ntime":1700000966,"order_key":"09:miner-9","p2mr_program_hex":"c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6","share_difficulty":2,"share_id":"share-966","share_seq":966,"template_height":800030},{"accepted_at_ms":999983,"credit_policy":null,"job_id":"job-460","job_issued_at_ms":999986,"miner_id":"miner-9","network_difficulty":1000001,"ntime":1700000460,"order_key":"09:miner-9","p2mr_program_hex":"cccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccc","share_difficulty":2,"share_id":"share-460","share_seq":460,"template_height":800014},{"accepted_at_ms":999987,"credit_policy":null,"job_id":"job-290","job_issued_at_ms":999989,"miner_id":"miner-4","network_difficulty":1000001,"ntime":1700000290,"order_key":"04:miner-4","p2mr_program_hex":"2222222222222222222222222222222222222222222222222222222222222222","share_difficulty":101,"share_id":"share-290","share_seq":290,"template_height":800009},{"accepted_at_ms":999973,"credit_policy":null,"job_id":"job-3230","job_issued_at_ms":999977,"miner_id":"miner-7","network_difficulty":1000000,"ntime":1700003230,"order_key":"07:miner-7","p2mr_program_hex":"9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e","share_difficulty":2,"share_id":"share-3230","share_seq":3230,"template_height":800100},{"accepted_at_ms":999970,"credit_policy":null,"job_id":"job-3837","job_issued_at_ms":999996,"miner_id":"miner-9","network_difficulty":1000012,"ntime":1700003837,"order_key":"09:miner-9","p2mr_program_hex":"fdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfd","share_difficulty":7,"share_id":"share-3837","share_seq":3837,"template_height":800119},{"accepted_at_ms":999993,"credit_policy":null,"job_id":"job-3777","job_issued_at_ms":999990,"miner_id":"miner-4","network_difficulty":1000003,"ntime":1700003777,"order_key":"04:miner-4","p2mr_program_hex":"c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1","share_difficulty":3,"share_id":"share-3777","share_seq":3777,"template_height":800118},{"accepted_at_ms":999985,"credit_policy":null,"job_id":"job-1432","job_issued_at_ms":999974,"miner_id":"miner-2","network_difficulty":1000004,"ntime":1700001432,"order_key":"02:miner-2","p2mr_program_hex":"9898989898989898989898989898989898989898989898989898989898989898","share_difficulty":2,"share_id":"share-1432","share_seq":1432,"template_height":800044},{"accepted_at_ms":999983,"credit_policy":null,"job_id":"job-2144","job_issued_at_ms":999983,"miner_id":"miner-10","network_difficulty":1000002,"ntime":1700002144,"order_key":"10:miner-10","p2mr_program_hex":"6060606060606060606060606060606060606060606060606060606060606060","share_difficulty":3,"share_id":"share-2144","share_seq":2144,"template_height":800067},{"accepted_at_ms":999985,"credit_policy":"stale-grace","job_id":"job-2610","job_issued_at_ms":999976,"miner_id":"miner-3","network_difficulty":1000009,"ntime":1700002610,"order_key":"03:miner-3","p2mr_program_hex":"3232323232323232323232323232323232323232323232323232323232323232","share_difficulty":1000,"share_id":"share-2610","share_seq":2610,"template_height":800081},{"accepted_at_ms":999993,"credit_policy":null,"job_id":"job-1933","job_issued_at_ms":999998,"miner_id":"miner-8","network_difficulty":1000012,"ntime":1700001933,"order_key":"08:miner-8","p2mr_program_hex":"8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d","share_difficulty":3,"share_id":"share-1933","share_seq":1933,"template_height":800060},{"accepted_at_ms":999978,"credit_policy":null,"job_id":"job-792","job_issued_at_ms":1000000,"miner_id":"miner-0","network_difficulty":1000010,"ntime":1700000792,"order_key":"00:miner-0","p2mr_program_hex":"1818181818181818181818181818181818181818181818181818181818181818","share_difficulty":3,"share_id":"share-792","share_seq":792,"template_height":800024},{"accepted_at_ms":999989,"credit_policy":null,"job_id":"job-2755","job_issued_at_ms":999989,"miner_id":"miner-5","network_difficulty":1000001,"ntime":1700002755,"order_key":"05:miner-5","p2mr_program_hex":"c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3","share_difficulty":7,"share_id":"share-2755","share_seq":2755,"template_height":800086},{"accepted_at_ms":999985,"credit_policy":null,"job_id":"job-99","job_issued_at_ms":999985,"miner_id":"miner-0","network_difficulty":1000014,"ntime":1700000099,"order_key":"00:miner-0","p2mr_program_hex":"6363636363636363636363636363636363636363636363636363636363636363","share_difficulty":1000,"share_id":"share-99","share_seq":99,"template_height":800003},{"accepted_at_ms":999987,"credit_policy":"stale-grace","job_id":"job-2267","job_issued_at_ms":999997,"miner_id":"miner-1","network_difficulty":1000006,"ntime":1700002267,"order_key":"01:miner-1","p2mr_program_hex":"dbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdb","share_difficulty":19,"share_id":"share-2267","share_seq":2267,"template_height":800070},{"accepted_at_ms":999989,"credit_policy":null,"job_id":"job-1038","job_issued_at_ms":1000000,"miner_id":"miner-4","network_difficulty":1000001,"ntime":1700001038,"order_key":"04:miner-4","p2mr_program_hex":"0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e","share_difficulty":1,"share_id":"share-1038","share_seq":1038,"template_height":800032},{"accepted_at_ms":1000000,"credit_policy":null,"job_id":"job-1402","job_issued_at_ms":999977,"miner_id":"miner-5","network_difficulty":1000008,"ntime":1700001402,"order_key":"05:miner-5","p2mr_program_hex":"7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a","share_difficulty":1000,"share_id":"share-1402","share_seq":1402,"template_height":800043},{"accepted_at_ms":999998,"credit_policy":null,"job_id":"job-1081","job_issued_at_ms":999985,"miner_id":"miner-3","network_difficulty":1000010,"ntime":1700001081,"order_key":"03:miner-3","p2mr_program_hex":"3939393939393939393939393939393939393939393939393939393939393939","share_difficulty":101,"share_id":"share-1081","share_seq":1081,"template_height":800033},{"accepted_at_ms":999975,"credit_policy":null,"job_id":"job-821","job_issued_at_ms":999990,"miner_id":"miner-7","network_difficulty":1000005,"ntime":1700000821,"order_key":"07:miner-7","p2mr_program_hex":"3535353535353535353535353535353535353535353535353535353535353535","share_difficulty":1,"share_id":"share-821","share_seq":821,"template_height":800025},{"accepted_at_ms":999995,"credit_policy":null,"job_id":"job-1099","job_issued_at_ms":999980,"miner_id":"miner-10","network_difficulty":1000011,"ntime":1700001099,"order_key":"10:miner-10","p2mr_program_hex":"4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b","share_difficulty":19,"share_id":"share-1099","share_seq":1099,"template_height":800034},{"accepted_at_ms":999995,"credit_policy":null,"job_id":"job-1615","job_issued_at_ms":999974,"miner_id":"miner-9","network_difficulty":1000000,"ntime":1700001615,"order_key":"09:miner-9","p2mr_program_hex":"4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f","share_difficulty":19,"share_id":"share-1615","share_seq":1615,"template_height":800050},{"accepted_at_ms":999976,"credit_policy":null,"job_id":"job-448","job_issued_at_ms":999979,"miner_id":"miner-8","network_difficulty":1000006,"ntime":1700000448,"order_key":"08:miner-8","p2mr_program_hex":"c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0","share_difficulty":2,"share_id":"share-448","share_seq":448,"template_height":800014},{"accepted_at_ms":999978,"credit_policy":"stale-grace","job_id":"job-1187","job_issued_at_ms":999977,"miner_id":"miner-10","network_difficulty":1000014,"ntime":1700001187,"order_key":"10:miner-10","p2mr_program_hex":"a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3","share_difficulty":2,"share_id":"share-1187","share_seq":1187,"template_height":800037},{"accepted_at_ms":999976,"credit_policy":null,"job_id":"job-2625","job_issued_at_ms":1000198,"miner_id":"miner-7","network_difficulty":1000007,"ntime":1700002625,"order_key":"07:miner-7","p2mr_program_hex":"4141414141414141414141414141414141414141414141414141414141414141","share_difficulty":7,"share_id":"share-2625","share_seq":2625,"template_height":800082},{"accepted_at_ms":1000269,"credit_policy":null,"job_id":"job-1566","job_issued_at_ms":1000088,"miner_id":"miner-4","network_difficulty":1000002,"ntime":1700001566,"order_key":"04:miner-4","p2mr_program_hex":"1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e","share_difficulty":7,"share_id":"share-1566","share_seq":1566,"template_height":800048},{"accepted_at_ms":999972,"credit_policy":null,"job_id":"job-1623","job_issued_at_ms":999999,"miner_id":"miner-6","network_difficulty":1000008,"ntime":1700001623,"order_key":"06:miner-6","p2mr_program_hex":"5757575757575757575757575757575757575757575757575757575757575757","share_difficulty":1000,"share_id":"share-1623","share_seq":1623,"template_height":800050},{"accepted_at_ms":999999,"credit_policy":null,"job_id":"job-381","job_issued_at_ms":999975,"miner_id":"miner-7","network_difficulty":1000007,"ntime":1700000381,"order_key":"07:miner-7","p2mr_program_hex":"7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d","share_difficulty":1000,"share_id":"share-381","share_seq":381,"template_height":800011},{"accepted_at_ms":999975,"credit_policy":null,"job_id":"job-2157","job_issued_at_ms":999974,"miner_id":"miner-1","network_difficulty":1000015,"ntime":1700002157,"order_key":"01:miner-1","p2mr_program_hex":"6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d","share_difficulty":1000,"share_id":"share-2157","share_seq":2157,"template_height":800067},{"accepted_at_ms":999995,"credit_policy":null,"job_id":"job-13","job_issued_at_ms":999984,"miner_id":"miner-2","network_difficulty":1000013,"ntime":1700000013,"order_key":"02:miner-2","p2mr_program_hex":"0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d","share_difficulty":101,"share_id":"share-13","share_seq":13,"template_height":800000},{"accepted_at_ms":999993,"credit_policy":null,"job_id":"job-337","job_issued_at_ms":999982,"miner_id":"miner-7","network_difficulty":1000014,"ntime":1700000337,"order_key":"07:miner-7","p2mr_program_hex":"5151515151515151515151515151515151515151515151515151515151515151","share_difficulty":1,"share_id":"share-337","share_seq":337,"template_height":800010},{"accepted_at_ms":999995,"credit_policy":null,"job_id":"job-1564","job_issued_at_ms":999974,"miner_id":"miner-2","network_difficulty":1000000,"ntime":1700001564,"order_key":"02:miner-2","p2mr_program_hex":"1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c","share_difficulty":3,"share_id":"share-1564","share_seq":1564,"template_height":800048},{"accepted_at_ms":999984,"credit_policy":null,"job_id":"job-2238","job_issued_at_ms":999979,"miner_id":"miner-5","network_difficulty":1000011,"ntime":1700002238,"order_key":"05:miner-5","p2mr_program_hex":"bebebebebebebebebebebebebebebebebebebebebebebebebebebebebebebebe","share_difficulty":7,"share_id":"share-2238","share_seq":2238,"template_height":800069},{"accepted_at_ms":1000429,"credit_policy":null,"job_id":"job-2890","job_issued_at_ms":999971,"miner_id":"miner-8","network_difficulty":1000000,"ntime":1700002890,"order_key":"08:miner-8","p2mr_program_hex":"4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a","share_difficulty":1000,"share_id":"share-2890","share_seq":2890,"template_height":800090},{"accepted_at_ms":999983,"credit_policy":null,"job_id":"job-2937","job_issued_at_ms":1000000,"miner_id":"miner-0","network_difficulty":1000013,"ntime":1700002937,"order_key":"00:miner-0","p2mr_program_hex":"7979797979797979797979797979797979797979797979797979797979797979","share_difficulty":19,"share_id":"share-2937","share_seq":2937,"template_height":800091},{"accepted_at_ms":999990,"credit_policy":null,"job_id":"job-1752","job_issued_at_ms":999976,"miner_id":"miner-3","network_difficulty":1000001,"ntime":1700001752,"order_key":"03:miner-3","p2mr_program_hex":"d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8","share_difficulty":1,"share_id":"share-1752","share_seq":1752,"template_height":800054},{"accepted_at_ms":999999,"credit_policy":null,"job_id":"job-995","job_issued_at_ms":999996,"miner_id":"miner-5","network_difficulty":1000009,"ntime":1700000995,"order_key":"05:miner-5","p2mr_program_hex":"e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3","share_difficulty":19,"share_id":"share-995","share_seq":995,"template_height":800031},{"accepted_at_ms":999991,"credit_policy":"stale-grace","job_id":"job-2754","job_issued_at_ms":999985,"miner_id":"miner-4","network_difficulty":1000000,"ntime":1700002754,"order_key":"04:miner-4","p2mr_program_hex":"c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2","share_difficulty":7,"share_id":"share-2754","share_seq":2754,"template_height":800086},{"accepted_at_ms":999992,"credit_policy":null,"job_id":"job-2479","job_issued_at_ms":999972,"miner_id":"miner-4","network_difficulty":1000014,"ntime":1700002479,"order_key":"04:miner-4","p2mr_program_hex":"afafafafafafafafafafafafafafafafafafafafafafafafafafafafafafafaf","share_difficulty":7,"share_id":"share-2479","share_seq":2479,"template_height":800077},{"accepted_at_ms":999986,"credit_policy":null,"job_id":"job-2403","job_issued_at_ms":999987,"miner_id":"miner-5","network_difficulty":1000006,"ntime":1700002403,"order_key":"05:miner-5","p2mr_program_hex":"6363636363636363636363636363636363636363636363636363636363636363","share_difficulty":1000,"share_id":"share-2403","share_seq":2403,"template_height":800075},{"accepted_at_ms":999982,"credit_policy":null,"job_id":"job-244","job_issued_at_ms":999998,"miner_id":"miner-2","network_difficulty":1000006,"ntime":1700000244,"order_key":"02:miner-2","p2mr_program_hex":"f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4","share_difficulty":3,"share_id":"share-244","share_seq":244,"template_height":800007},{"accepted_at_ms":999985,"credit_policy":null,"job_id":"job-1266","job_issued_at_ms":999983,"miner_id":"miner-1","network_difficulty":1000008,"ntime":1700001266,"order_key":"01:miner-1","p2mr_program_hex":"f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2","share_difficulty":7,"share_id":"share-1266","share_seq":1266,"template_height":800039},{"accepted_at_ms":999998,"credit_policy":"stale-grace","job_id":"job-1544","job_issued_at_ms":999978,"miner_id":"miner-4","network_difficulty":1000014,"ntime":1700001544,"order_key":"04:miner-4","p2mr_program_hex":"0808080808080808080808080808080808080808080808080808080808080808","share_difficulty":3,"share_id":"share-1544","share_seq":1544,"template_height":800048},{"accepted_at_ms":999986,"credit_policy":null,"job_id":"job-3251","job_issued_at_ms":999988,"miner_id":"miner-6","network_difficulty":1000004,"ntime":1700003251,"order_key":"06:miner-6","p2mr_program_hex":"b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3","share_difficulty":19,"share_id":"share-3251","share_seq":3251,"template_height":800101},{"accepted_at_ms":999982,"credit_policy":null,"job_id":"job-889","job_issued_at_ms":999974,"miner_id":"miner-9","network_difficulty":1000005,"ntime":1700000889,"order_key":"09:miner-9","p2mr_program_hex":"7979797979797979797979797979797979797979797979797979797979797979","share_difficulty":101,"share_id":"share-889","share_seq":889,"template_height":800027},{"accepted_at_ms":999997,"credit_policy":null,"job_id":"job-3555","job_issued_at_ms":999992,"miner_id":"miner-2","network_difficulty":1000002,"ntime":1700003555,"order_key":"02:miner-2","p2mr_program_hex":"e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3","share_difficulty":7,"share_id":"share-3555","share_seq":3555,"template_height":800111},{"accepted_at_ms":999972,"credit_policy":null,"job_id":"job-2958","job_issued_at_ms":999972,"miner_id":"miner-10","network_difficulty":1000000,"ntime":1700002958,"order_key":"10:miner-10","p2mr_program_hex":"8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e","share_difficulty":7,"share_id":"share-2958","share_seq":2958,"template_height":800092},{"accepted_at_ms":999973,"credit_policy":"stale-grace","job_id":"job-3394","job_issued_at_ms":999986,"miner_id":"miner-6","network_difficulty":1000011,"ntime":1700003394,"order_key":"06:miner-6","p2mr_program_hex":"4242424242424242424242424242424242424242424242424242424242424242","share_difficulty":1,"share_id":"share-3394","share_seq":3394,"template_height":800106},{"accepted_at_ms":999982,"credit_policy":"stale-grace","job_id":"job-2443","job_issued_at_ms":999991,"miner_id":"miner-1","network_difficulty":1000012,"ntime":1700002443,"order_key":"01:miner-1","p2mr_program_hex":"8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b","share_difficulty":101,"share_id":"share-2443","share_seq":2443,"template_height":800076},{"accepted_at_ms":999991,"credit_policy":null,"job_id":"job-984","job_issued_at_ms":999997,"miner_id":"miner-5","network_difficulty":1000015,"ntime":1700000984,"order_key":"05:miner-5","p2mr_program_hex":"d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8","share_difficulty":1000,"share_id":"share-984","share_seq":984,"template_height":800030},{"accepted_at_ms":999970,"credit_policy":null,"job_id":"job-2063","job_issued_at_ms":999979,"miner_id":"miner-6","network_difficulty":1000006,"ntime":1700002063,"order_key":"06:miner-6","p2mr_program_hex":"0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f","share_difficulty":1000,"share_id":"share-2063","share_seq":2063,"template_height":800064},{"accepted_at_ms":999984,"credit_policy":null,"job_id":"job-1726","job_issued_at_ms":999974,"miner_id":"miner-10","network_difficulty":1000009,"ntime":1700001726,"order_key":"10:miner-10","p2mr_program_hex":"bebebebebebebebebebebebebebebebebebebebebebebebebebebebebebebebe","share_difficulty":19,"share_id":"share-1726","share_seq":1726,"template_height":800053},{"accepted_at_ms":999999,"credit_policy":"stale-grace","job_id":"job-993","job_issued_at_ms":999979,"miner_id":"miner-3","network_difficulty":1000007,"ntime":1700000993,"order_key":"03:miner-3","p2mr_program_hex":"e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1","share_difficulty":1,"share_id":"share-993","share_seq":993,"template_height":800031},{"accepted_at_ms":999990,"credit_policy":null,"job_id":"job-2613","job_issued_at_ms":999972,"miner_id":"miner-6","network_difficulty":1000012,"ntime":1700002613,"order_key":"06:miner-6","p2mr_program_hex":"3535353535353535353535353535353535353535353535353535353535353535","share_difficulty":19,"share_id":"share-2613","share_seq":2613,"template_height":800081},{"accepted_at_ms":999989,"credit_policy":null,"job_id":"job-841","job_issued_at_ms":999974,"miner_id":"miner-5","network_difficulty":1000008,"ntime":1700000841,"order_key":"05:miner-5","p2mr_program_hex":"4949494949494949494949494949494949494949494949494949494949494949","share_difficulty":1000,"share_id":"share-841","share_seq":841,"template_height":800026},{"accepted_at_ms":999984,"credit_policy":"stale-grace","job_id":"job-908","job_issued_at_ms":999996,"miner_id":"miner-6","network_difficulty":1000007,"ntime":1700000908,"order_key":"06:miner-6","p2mr_program_hex":"8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c","share_difficulty":1000,"share_id":"share-908","share_seq":908,"template_height":800028},{"accepted_at_ms":999970,"credit_policy":null,"job_id":"job-3234","job_issued_at_ms":999971,"miner_id":"miner-0","network_difficulty":1000004,"ntime":1700003234,"order_key":"00:miner-0","p2mr_program_hex":"a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2","share_difficulty":7,"share_id":"share-3234","share_seq":3234,"template_height":800101},{"accepted_at_ms":999971,"credit_policy":null,"job_id":"job-1101","job_issued_at_ms":999970,"miner_id":"miner-1","network_difficulty":1000013,"ntime":1700001101,"order_key":"01:miner-1","p2mr_program_hex":"4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d","share_difficulty":7,"share_id":"share-1101","share_seq":1101,"template_height":800034},{"accepted_at_ms":999971,"credit_policy":"stale-grace","job_id":"job-1745","job_issued_at_ms":999976,"miner_id":"miner-7","network_difficulty":1000011,"ntime":1700001745,"order_key":"07:miner-7","p2mr_program_hex":"d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1","share_difficulty":2,"share_id":"share-1745","share_seq":1745,"template_height":800054},{"accepted_at_ms":999995,"credit_policy":null,"job_id":"job-2866","job_issued_at_ms":999972,"miner_id":"miner-6","network_difficulty":1000010,"ntime":1700002866,"order_key":"06:miner-6","p2mr_program_hex":"3232323232323232323232323232323232323232323232323232323232323232","share_difficulty":1,"share_id":"share-2866","share_seq":2866,"template_height":800089},{"accepted_at_ms":999998,"credit_policy":null,"job_id":"job-2378","job_issued_at_ms":999974,"miner_id":"miner-2","network_difficulty":1000015,"ntime":1700002378,"order_key":"02:miner-2","p2mr_program_hex":"4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a","share_difficulty":101,"share_id":"share-2378","share_seq":2378,"template_height":800074},{"accepted_at_ms":999985,"credit_policy":"stale-grace","job_id":"job-1932","job_issued_at_ms":1000279,"miner_id":"miner-7","network_difficulty":1000011,"ntime":1700001932,"order_key":"07:miner-7","p2mr_program_hex":"8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c","share_difficulty":1000,"share_id":"share-1932","share_seq":1932,"template_height":800060},{"accepted_at_ms":999994,"credit_policy":null,"job_id":"job-3407","job_issued_at_ms":999977,"miner_id":"miner-8","network_difficulty":1000007,"ntime":1700003407,"order_key":"08:miner-8","p2mr_program_hex":"4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f","share_difficulty":3,"share_id":"share-3407","share_seq":3407,"template_height":800106},{"accepted_at_ms":999990,"credit_policy":"stale-grace","job_id":"job-3375","job_issued_at_ms":999975,"miner_id":"miner-9","network_difficulty":1000009,"ntime":1700003375,"order_key":"09:miner-9","p2mr_program_hex":"2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f","share_difficulty":19,"share_id":"share-3375","share_seq":3375,"template_height":800105},{"accepted_at_ms":999991,"credit_policy":"stale-grace","job_id":"job-596","job_issued_at_ms":999986,"miner_id":"miner-2","network_difficulty":1000001,"ntime":1700000596,"order_key":"02:miner-2","p2mr_program_hex":"5454545454545454545454545454545454545454545454545454545454545454","share_difficulty":19,"share_id":"share-596","share_seq":596,"template_height":800018},{"accepted_at_ms":999985,"credit_policy":null,"job_id":"job-2428","job_issued_at_ms":999978,"miner_id":"miner-8","network_difficulty":1000014,"ntime":1700002428,"order_key":"08:miner-8","p2mr_program_hex":"7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c","share_difficulty":19,"share_id":"share-2428","share_seq":2428,"template_height":800075},{"accepted_at_ms":999972,"credit_policy":"stale-grace","job_id":"job-1814","job_issued_at_ms":999995,"miner_id":"miner-10","network_difficulty":1000012,"ntime":1700001814,"order_key":"10:miner-10","p2mr_program_hex":"1616161616161616161616161616161616161616161616161616161616161616","share_difficulty":101,"share_id":"share-1814","share_seq":1814,"template_height":800056},{"accepted_at_ms":999983,"credit_policy":null,"job_id":"job-1621","job_issued_at_ms":999980,"miner_id":"miner-4","network_difficulty":1000006,"ntime":1700001621,"order_key":"04:miner-4","p2mr_program_hex":"5555555555555555555555555555555555555555555555555555555555555555","share_difficulty":2,"share_id":"share-1621","share_seq":1621,"template_height":800050},{"accepted_at_ms":1000000,"credit_policy":null,"job_id":"job-2487","job_issued_at_ms":999972,"miner_id":"miner-1","network_difficulty":1000005,"ntime":1700002487,"order_key":"01:miner-1","p2mr_program_hex":"b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7","share_difficulty":7,"share_id":"share-2487","share_seq":2487,"template_height":800077},{"accepted_at_ms":999996,"credit_policy":null,"job_id":"job-1463","job_issued_at_ms":999993,"miner_id":"miner-0","network_difficulty":1000001,"ntime":1700001463,"order_key":"00:miner-0","p2mr_program_hex":"b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7","share_difficulty":19,"share_id":"share-1463","share_seq":1463,"template_height":800045},{"accepted_at_ms":999985,"credit_policy":null,"job_id":"job-2211","job_issued_at_ms":999994,"miner_id":"miner-0","network_difficulty":1000001,"ntime":1700002211,"order_key":"00:miner-0","p2mr_program_hex":"a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3","share_difficulty":1000,"share_id":"share-2211","share_seq":2211,"template_height":800069},{"accepted_at_ms":999992,"credit_policy":"stale-grace","job_id":"job-2499","job_issued_at_ms":999973,"miner_id":"miner-2","network_difficulty":1000000,"ntime":1700002499,"order_key":"02:miner-2","p2mr_program_hex":"c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3","share_difficulty":7,"share_id":"share-2499","share_seq":2499,"template_height":800078},{"accepted_at_ms":999976,"credit_policy":null,"job_id":"job-288","job_issued_at_ms":999983,"miner_id":"miner-2","network_difficulty":1000016,"ntime":1700000288,"order_key":"02:miner-2","p2mr_program_hex":"2020202020202020202020202020202020202020202020202020202020202020","share_difficulty":1000,"share_id":"share-288","share_seq":288,"template_height":800009},{"accepted_at_ms":999990,"credit_policy":null,"job_id":"job-3939","job_issued_at_ms":999983,"miner_id":"miner-1","network_difficulty":1000012,"ntime":1700003939,"order_key":"01:miner-1","p2mr_program_hex":"6363636363636363636363636363636363636363636363636363636363636363","share_difficulty":3,"share_id":"share-3939","share_seq":3939,"template_height":800123},{"accepted_at_ms":999974,"credit_policy":null,"job_id":"job-1807","job_issued_at_ms":999991,"miner_id":"miner-3","network_difficulty":1000005,"ntime":1700001807,"order_key":"03:miner-3","p2mr_program_hex":"0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f","share_difficulty":1,"share_id":"share-1807","share_seq":1807,"template_height":800056},{"accepted_at_ms":999995,"credit_policy":null,"job_id":"job-3470","job_issued_at_ms":999997,"miner_id":"miner-5","network_difficulty":1000002,"ntime":1700003470,"order_key":"05:miner-5","p2mr_program_hex":"8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e","share_difficulty":2,"share_id":"share-3470","share_seq":3470,"template_height":800108},{"accepted_at_ms":999982,"credit_policy":null,"job_id":"job-3087","job_issued_at_ms":999985,"miner_id":"miner-7","network_difficulty":1000010,"ntime":1700003087,"order_key":"07:miner-7","p2mr_program_hex":"0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f","share_difficulty":2,"share_id":"share-3087","share_seq":3087,"template_height":800096},{"accepted_at_ms":999973,"credit_policy":null,"job_id":"job-3527","job_issued_at_ms":999981,"miner_id":"miner-7","network_difficulty":1000008,"ntime":1700003527,"order_key":"07:miner-7","p2mr_program_hex":"c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7","share_difficulty":101,"share_id":"share-3527","share_seq":3527,"template_height":800110},{"accepted_at_ms":999982,"credit_policy":null,"job_id":"job-1638","job_issued_at_ms":999993,"miner_id":"miner-10","network_difficulty":1000006,"ntime":1700001638,"order_key":"10:miner-10","p2mr_program_hex":"6666666666666666666666666666666666666666666666666666666666666666","share_difficulty":101,"share_id":"share-1638","share_seq":1638,"template_height":800051},{"accepted_at_ms":1000000,"credit_policy":"stale-grace","job_id":"job-1529","job_issued_at_ms":999975,"miner_id":"miner-0","network_difficulty":1000016,"ntime":1700001529,"order_key":"00:miner-0","p2mr_program_hex":"f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9","share_difficulty":1000,"share_id":"share-1529","share_seq":1529,"template_height":800047},{"accepted_at_ms":999996,"credit_policy":null,"job_id":"job-2430","job_issued_at_ms":999975,"miner_id":"miner-10","network_difficulty":1000016,"ntime":1700002430,"order_key":"10:miner-10","p2mr_program_hex":"7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e","share_difficulty":2,"share_id":"share-2430","share_seq":2430,"template_height":800075},{"accepted_at_ms":999973,"credit_policy":null,"job_id":"job-910","job_issued_at_ms":999991,"miner_id":"miner-8","network_difficulty":1000009,"ntime":1700000910,"order_key":"08:miner-8","p2mr_program_hex":"8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e","share_difficulty":2,"share_id":"share-910","share_seq":910,"template_height":800028},{"accepted_at_ms":999995,"credit_policy":null,"job_id":"job-1333","job_issued_at_ms":999977,"miner_id":"miner-2","network_difficulty":1000007,"ntime":1700001333,"order_key":"02:miner-2","p2mr_program_hex":"3535353535353535353535353535353535353535353535353535353535353535","share_difficulty":101,"share_id":"share-1333","share_seq":1333,"template_height":800041},{"accepted_at_ms":999996,"credit_policy":null,"job_id":"job-2239","job_issued_at_ms":999992,"miner_id":"miner-6","network_difficulty":1000012,"ntime":1700002239,"order_key":"06:miner-6","p2mr_program_hex":"bfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbf","share_difficulty":101,"share_id":"share-2239","share_seq":2239,"template_height":800069},{"accepted_at_ms":999974,"credit_policy":null,"job_id":"job-9","job_issued_at_ms":999984,"miner_id":"miner-9","network_difficulty":1000009,"ntime":1700000009,"order_key":"09:miner-9","p2mr_program_hex":"0909090909090909090909090909090909090909090909090909090909090909","share_difficulty":19,"share_id":"share-9","share_seq":9,"template_height":800000},{"accepted_at_ms":999994,"credit_policy":null,"job_id":"job-3573","job_issued_at_ms":999974,"miner_id":"miner-9","network_difficulty":1000003,"ntime":1700003573,"order_key":"09:miner-9","p2mr_program_hex":"f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5","share_difficulty":1,"share_id":"share-3573","share_seq":3573,"template_height":800111},{"accepted_at_ms":999992,"credit_policy":null,"job_id":"job-3500","job_issued_at_ms":999971,"miner_id":"miner-2","network_difficulty":1000015,"ntime":1700003500,"order_key":"02:miner-2","p2mr_program_hex":"acacacacacacacacacacacacacacacacacacacacacacacacacacacacacacacac","share_difficulty":1000,"share_id":"share-3500","share_seq":3500,"template_height":800109},{"accepted_at_ms":999973,"credit_policy":null,"job_id":"job-188","job_issued_at_ms":999986,"miner_id":"miner-1","network_difficulty":1000001,"ntime":1700000188,"order_key":"01:miner-1","p2mr_program_hex":"bcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbc","share_difficulty":19,"share_id":"share-188","share_seq":188,"template_height":800005},{"accepted_at_ms":999992,"credit_policy":null,"job_id":"job-2339","job_issued_at_ms":999976,"miner_id":"miner-7","network_difficulty":1000010,"ntime":1700002339,"order_key":"07:miner-7","p2mr_program_hex":"2323232323232323232323232323232323232323232323232323232323232323","share_difficulty":101,"share_id":"share-2339","share_seq":2339,"template_height":800073},{"accepted_at_ms":999990,"credit_policy":null,"job_id":"job-580","job_issued_at_ms":999975,"miner_id":"miner-8","network_difficulty":1000002,"ntime":1700000580,"order_key":"08:miner-8","p2mr_program_hex":"4444444444444444444444444444444444444444444444444444444444444444","share_difficulty":101,"share_id":"share-580","share_seq":580,"template_height":800018},{"accepted_at_ms":999988,"credit_policy":null,"job_id":"job-1689","job_issued_at_ms":999974,"miner_id":"miner-6","network_difficulty":1000006,"ntime":1700001689,"order_key":"06:miner-6","p2mr_program_hex":"9999999999999999999999999999999999999999999999999999999999999999","share_difficulty":101,"share_id":"share-1689","share_seq":1689,"template_height":800052},{"accepted_at_ms":999997,"credit_policy":null,"job_id":"job-1913","job_issued_at_ms":999984,"miner_id":"miner-10","network_difficulty":1000009,"ntime":1700001913,"order_key":"10:miner-10","p2mr_program_hex":"7979797979797979797979797979797979797979797979797979797979797979","share_difficulty":7,"share_id":"share-1913","share_seq":1913,"template_height":800059},{"accepted_at_ms":999984,"credit_policy":null,"job_id":"job-2226","job_issued_at_ms":999980,"miner_id":"miner-4","network_difficulty":1000016,"ntime":1700002226,"order_key":"04:miner-4","p2mr_program_hex":"b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2","share_difficulty":101,"share_id":"share-2226","share_seq":2226,"template_height":800069},{"accepted_at_ms":999980,"credit_policy":null,"job_id":"job-960","job_issued_at_ms":999988,"miner_id":"miner-3","network_difficulty":1000008,"ntime":1700000960,"order_key":"03:miner-3","p2mr_program_hex":"c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0","share_difficulty":19,"share_id":"share-960","share_seq":960,"template_height":800030},{"accepted_at_ms":999995,"credit_policy":null,"job_id":"job-3442","job_issued_at_ms":1000480,"miner_id":"miner-10","network_difficulty":1000008,"ntime":1700003442,"order_key":"10:miner-10","p2mr_program_hex":"7272727272727272727272727272727272727272727272727272727272727272","share_difficulty":19,"share_id":"share-3442","share_seq":3442,"template_height":800107},{"accepted_at_ms":999979,"credit_policy":null,"job_id":"job-1492","job_issued_at_ms":999999,"miner_id":"miner-7","network_difficulty":1000013,"ntime":1700001492,"order_key":"07:miner-7","p2mr_program_hex":"d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4","share_difficulty":3,"share_id":"share-1492","share_seq":1492,"template_height":800046},{"accepted_at_ms":1000000,"credit_policy":null,"job_id":"job-3089","job_issued_at_ms":999979,"miner_id":"miner-9","network_difficulty":1000012,"ntime":1700003089,"order_key":"09:miner-9","p2mr_program_hex":"1111111111111111111111111111111111111111111111111111111111111111","share_difficulty":3,"share_id":"share-3089","share_seq":3089,"template_height":800096},{"accepted_at_ms":999988,"credit_policy":null,"job_id":"job-2705","job_issued_at_ms":1000000,"miner_id":"miner-10","network_difficulty":1000002,"ntime":1700002705,"order_key":"10:miner-10","p2mr_program_hex":"9191919191919191919191919191919191919191919191919191919191919191","share_difficulty":1000,"share_id":"share-2705","share_seq":2705,"template_height":800084},{"accepted_at_ms":999990,"credit_policy":null,"job_id":"job-148","job_issued_at_ms":999970,"miner_id":"miner-5","network_difficulty":1000012,"ntime":1700000148,"order_key":"05:miner-5","p2mr_program_hex":"9494949494949494949494949494949494949494949494949494949494949494","share_difficulty":19,"share_id":"share-148","share_seq":148,"template_height":800004},{"accepted_at_ms":999995,"credit_policy":null,"job_id":"job-44","job_issued_at_ms":999970,"miner_id":"miner-0","network_difficulty":1000010,"ntime":1700000044,"order_key":"00:miner-0","p2mr_program_hex":"2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c","share_difficulty":1000,"share_id":"share-44","share_seq":44,"template_height":800001},{"accepted_at_ms":999992,"credit_policy":null,"job_id":"job-2014","job_issued_at_ms":999989,"miner_id":"miner-1","network_difficulty":1000008,"ntime":1700002014,"order_key":"01:miner-1","p2mr_program_hex":"dededededededededededededededededededededededededededededededede","share_difficulty":1000,"share_id":"share-2014","share_seq":2014,"template_height":800062},{"accepted_at_ms":999972,"credit_policy":null,"job_id":"job-855","job_issued_at_ms":999996,"miner_id":"miner-8","network_difficulty":1000005,"ntime":1700000855,"order_key":"08:miner-8","p2mr_program_hex":"5757575757575757575757575757575757575757575757575757575757575757","share_difficulty":2,"share_id":"share-855","share_seq":855,"template_height":800026},{"accepted_at_ms":999986,"credit_policy":null,"job_id":"job-1701","job_issued_at_ms":999986,"miner_id":"miner-7","network_difficulty":1000001,"ntime":1700001701,"order_key":"07:miner-7","p2mr_program_hex":"a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5","share_difficulty":3,"share_id":"share-1701","share_seq":1701,"template_height":800053},{"accepted_at_ms":999994,"credit_policy":"stale-grace","job_id":"job-2406","job_issued_at_ms":999993,"miner_id":"miner-8","network_difficulty":1000009,"ntime":1700002406,"order_key":"08:miner-8","p2mr_program_hex":"6666666666666666666666666666666666666666666666666666666666666666","share_difficulty":1,"share_id":"share-2406","share_seq":2406,"template_height":800075},{"accepted_at_ms":999991,"credit_policy":null,"job_id":"job-3592","job_issued_at_ms":999980,"miner_id":"miner-6","network_difficulty":1000005,"ntime":1700003592,"order_key":"06:miner-6","p2mr_program_hex":"0808080808080808080808080808080808080808080808080808080808080808","share_difficulty":3,"share_id":"share-3592","share_seq":3592,"template_height":800112},{"accepted_at_ms":1000000,"credit_policy":null,"job_id":"job-574","job_issued_at_ms":999979,"miner_id":"miner-2","network_difficulty":1000013,"ntime":1700000574,"order_key":"02:miner-2","p2mr_program_hex":"3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e","share_difficulty":19,"share_id":"share-574","share_seq":574,"template_height":800017},{"accepted_at_ms":999997,"credit_policy":"stale-grace","job_id":"job-3671","job_issued_at_ms":999984,"miner_id":"miner-8","network_difficulty":1000016,"ntime":1700003671,"order_key":"08:miner-8","p2mr_program_hex":"5757575757575757575757575757575757575757575757575757575757575757","share_difficulty":3,"share_id":"share-3671","share_seq":3671,"template_height":800114},{"accepted_at_ms":999984,"credit_policy":null,"job_id":"job-1475","job_issued_at_ms":999983,"miner_id":"miner-1","network_difficulty":1000013,"ntime":1700001475,"order_key":"01:miner-1","p2mr_program_hex":"c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3","share_difficulty":2,"share_id":"share-1475","share_seq":1475,"template_height":800046},{"accepted_at_ms":999993,"credit_policy":"stale-grace","job_id":"job-3560","job_issued_at_ms":999976,"miner_id":"miner-7","network_difficulty":1000007,"ntime":1700003560,"order_key":"07:miner-7","p2mr_program_hex":"e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8","share_difficulty":101,"share_id":"share-3560","share_seq":3560,"template_height":800111},{"accepted_at_ms":999979,"credit_policy":"stale-grace","job_id":"job-153","job_issued_at_ms":999983,"miner_id":"miner-10","network_difficulty":1000000,"ntime":1700000153,"order_key":"10:miner-10","p2mr_program_hex":"9999999999999999999999999999999999999999999999999999999999999999","share_difficulty":1000,"share_id":"share-153","share_seq":153,"template_height":800004},{"accepted_at_ms":999974,"credit_policy":null,"job_id":"job-3608","job_issued_at_ms":999984,"miner_id":"miner-0","network_difficulty":1000004,"ntime":1700003608,"order_key":"00:miner-0","p2mr_program_hex":"1818181818181818181818181818181818181818181818181818181818181818","share_difficulty":1,"share_id":"share-3608","share_seq":3608,"template_height":800112},{"accepted_at_ms":999999,"credit_policy":"stale-grace","job_id":"job-3562","job_issued_at_ms":999973,"miner_id":"miner-9","network_difficulty":1000009,"ntime":1700003562,"order_key":"09:miner-9","p2mr_program_hex":"eaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaea","share_difficulty":7,"share_id":"share-3562","share_seq":3562,"template_height":800111},{"accepted_at_ms":999978,"credit_policy":null,"job_id":"job-1500","job_issued_at_ms":999971,"miner_id":"miner-4","network_difficulty":1000004,"ntime":1700001500,"order_key":"04:miner-4","p2mr_program_hex":"dcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdc","share_difficulty":101,"share_id":"share-1500","share_seq":1500,"template_height":800046},{"accepted_at_ms":999993,"credit_policy":"stale-grace","job_id":"job-1225","job_issued_at_ms":999999,"miner_id":"miner-4","network_difficulty":1000001,"ntime":1700001225,"order_key":"04:miner-4","p2mr_program_hex":"c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9","share_difficulty":1,"share_id":"share-1225","share_seq":1225,"template_height":800038},{"accepted_at_ms":999979,"credit_policy":"stale-grace","job_id":"job-2152","job_issued_at_ms":999971,"miner_id":"miner-7","network_difficulty":1000010,"ntime":1700002152,"order_key":"07:miner-7","p2mr_program_hex":"6868686868686868686868686868686868686868686868686868686868686868","share_difficulty":2,"share_id":"share-2152","share_seq":2152,"template_height":800067},{"accepted_at_ms":999980,"credit_policy":null,"job_id":"job-185","job_issued_at_ms":999989,"miner_id":"miner-9","network_difficulty":1000015,"ntime":1700000185,"order_key":"09:miner-9","p2mr_program_hex":"b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9","share_difficulty":7,"share_id":"share-185","share_seq":185,"template_height":800005},{"accepted_at_ms":999989,"credit_policy":null,"job_id":"job-1522","job_issued_at_ms":999998,"miner_id":"miner-4","network_difficulty":1000009,"ntime":1700001522,"order_key":"04:miner-4","p2mr_program_hex":"f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2","share_difficulty":3,"share_id":"share-1522","share_seq":1522,"template_height":800047},{"accepted_at_ms":999988,"credit_policy":null,"job_id":"job-583","job_issued_at_ms":999974,"miner_id":"miner-0","network_difficulty":1000005,"ntime":1700000583,"order_key":"00:miner-0","p2mr_program_hex":"4747474747474747474747474747474747474747474747474747474747474747","share_difficulty":101,"share_id":"share-583","share_seq":583,"template_height":800018},{"accepted_at_ms":999982,"credit_policy":null,"job_id":"job-2734","job_issued_at_ms":999985,"miner_id":"miner-6","network_difficulty":1000014,"ntime":1700002734,"order_key":"06:miner-6","p2mr_program_hex":"aeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeae","share_difficulty":101,"share_id":"share-2734","share_seq":2734,"template_height":800085},{"accepted_at_ms":999973,"credit_policy":null,"job_id":"job-1118","job_issued_at_ms":999994,"miner_id":"miner-7","network_difficulty":1000013,"ntime":1700001118,"order_key":"07:miner-7","p2mr_program_hex":"5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e","share_difficulty":19,"share_id":"share-1118","share_seq":1118,"template_height":800034},{"accepted_at_ms":999982,"credit_policy":null,"job_id":"job-907","job_issued_at_ms":999974,"miner_id":"miner-5","network_difficulty":1000006,"ntime":1700000907,"order_key":"05:miner-5","p2mr_program_hex":"8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b","share_difficulty":3,"share_id":"share-907","share_seq":907,"template_height":800028},{"accepted_at_ms":999999,"credit_policy":null,"job_id":"job-2901","job_issued_at_ms":999983,"miner_id":"miner-8","network_difficulty":1000011,"ntime":1700002901,"order_key":"08:miner-8","p2mr_program_hex":"5555555555555555555555555555555555555555555555555555555555555555","share_difficulty":101,"share_id":"share-2901","share_seq":2901,"template_height":800090},{"accepted_at_ms":999978,"credit_policy":null,"job_id":"job-1228","job_issued_at_ms":999975,"miner_id":"miner-7","network_difficulty":1000004,"ntime":1700001228,"order_key":"07:miner-7","p2mr_program_hex":"cccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccc","share_difficulty":101,"share_id":"share-1228","share_seq":1228,"template_height":800038},{"accepted_at_ms":999980,"credit_policy":null,"job_id":"job-2172","job_issued_at_ms":999997,"miner_id":"miner-5","network_difficulty":1000013,"ntime":1700002172,"order_key":"05:miner-5","p2mr_program_hex":"7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c","share_difficulty":1000,"share_id":"share-2172","share_seq":2172,"template_height":800067},{"accepted_at_ms":999991,"credit_policy":null,"job_id":"job-1174","job_issued_at_ms":999996,"miner_id":"miner-8","network_difficulty":1000001,"ntime":1700001174,"order_key":"08:miner-8","p2mr_program_hex":"9696969696969696969696969696969696969696969696969696969696969696","share_difficulty":7,"share_id":"share-1174","share_seq":1174,"template_height":800036},{"accepted_at_ms":999983,"credit_policy":null,"job_id":"job-1447","job_issued_at_ms":999991,"miner_id":"miner-6","network_difficulty":1000002,"ntime":1700001447,"order_key":"06:miner-6","p2mr_program_hex":"a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7","share_difficulty":3,"share_id":"share-1447","share_seq":1447,"template_height":800045},{"accepted_at_ms":999985,"credit_policy":"stale-grace","job_id":"job-3360","job_issued_at_ms":999981,"miner_id":"miner-5","network_difficulty":1000011,"ntime":1700003360,"order_key":"05:miner-5","p2mr_program_hex":"2020202020202020202020202020202020202020202020202020202020202020","share_difficulty":3,"share_id":"share-3360","share_seq":3360,"template_height":800105},{"accepted_at_ms":999981,"credit_policy":null,"job_id":"job-2971","job_issued_at_ms":999988,"miner_id":"miner-1","network_difficulty":1000013,"ntime":1700002971,"order_key":"01:miner-1","p2mr_program_hex":"9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b","share_difficulty":1,"share_id":"share-2971","share_seq":2971,"template_height":800092},{"accepted_at_ms":999973,"credit_policy":null,"job_id":"job-3775","job_issued_at_ms":999990,"miner_id":"miner-2","network_difficulty":1000001,"ntime":1700003775,"order_key":"02:miner-2","p2mr_program_hex":"bfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbf","share_difficulty":101,"share_id":"share-3775","share_seq":3775,"template_height":800117},{"accepted_at_ms":1000261,"credit_policy":"stale-grace","job_id":"job-1602","job_issued_at_ms":999996,"miner_id":"miner-7","network_difficulty":1000004,"ntime":1700001602,"order_key":"07:miner-7","p2mr_program_hex":"4242424242424242424242424242424242424242424242424242424242424242","share_difficulty":2,"share_id":"share-1602","share_seq":1602,"template_height":800050},{"accepted_at_ms":999971,"credit_policy":null,"job_id":"job-548","job_issued_at_ms":999991,"miner_id":"miner-9","network_difficulty":1000004,"ntime":1700000548,"order_key":"09:miner-9","p2mr_program_hex":"2424242424242424242424242424242424242424242424242424242424242424","share_difficulty":7,"share_id":"share-548","share_seq":548,"template_height":800017},{"accepted_at_ms":999984,"credit_policy":null,"job_id":"job-41","job_issued_at_ms":999977,"miner_id":"miner-8","network_difficulty":1000007,"ntime":1700000041,"order_key":"08:miner-8","p2mr_program_hex":"2929292929292929292929292929292929292929292929292929292929292929","share_difficulty":3,"share_id":"share-41","share_seq":41,"template_height":800001},{"accepted_at_ms":999981,"credit_policy":null,"job_id":"job-2591","job_issued_at_ms":999993,"miner_id":"miner-6","network_difficulty":1000007,"ntime":1700002591,"order_key":"06:miner-6","p2mr_program_hex":"1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f","share_difficulty":1,"share_id":"share-2591","share_seq":2591,"template_height":800080},{"accepted_at_ms":999997,"credit_policy":"stale-grace","job_id":"job-2092","job_issued_at_ms":999971,"miner_id":"miner-2","network_difficulty":1000001,"ntime":1700002092,"order_key":"02:miner-2","p2mr_program_hex":"2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c","share_difficulty":19,"share_id":"share-2092","share_seq":2092,"template_height":800065},{"accepted_at_ms":999970,"credit_policy":null,"job_id":"job-2431","job_issued_at_ms":999972,"miner_id":"miner-0","network_difficulty":1000000,"ntime":1700002431,"order_key":"00:miner-0","p2mr_program_hex":"7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f","share_difficulty":1,"share_id":"share-2431","share_seq":2431,"template_height":800075},{"accepted_at_ms":999972,"credit_policy":"stale-grace","job_id":"job-1211","job_issued_at_ms":1000049,"miner_id":"miner-1","network_difficulty":1000004,"ntime":1700001211,"order_key":"01:miner-1","p2mr_program_hex":"bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb","share_difficulty":1,"share_id":"share-1211","share_seq":1211,"template_height":800037},{"accepted_at_ms":999996,"credit_policy":"stale-grace","job_id":"job-1271","job_issued_at_ms":999992,"miner_id":"miner-6","network_difficulty":1000013,"ntime":1700001271,"order_key":"06:miner-6","p2mr_program_hex":"f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7","share_difficulty":19,"share_id":"share-1271","share_seq":1271,"template_height":800039},{"accepted_at_ms":999984,"credit_policy":"stale-grace","job_id":"job-2036","job_issued_at_ms":999983,"miner_id":"miner-1","network_difficulty":1000013,"ntime":1700002036,"order_key":"01:miner-1","p2mr_program_hex":"f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4","share_difficulty":1000,"share_id":"share-2036","share_seq":2036,"template_height":800063},{"accepted_at_ms":999977,"credit_policy":null,"job_id":"job-3128","job_issued_at_ms":999994,"miner_id":"miner-4","network_difficulty":1000000,"ntime":1700003128,"order_key":"04:miner-4","p2mr_program_hex":"3838383838383838383838383838383838383838383838383838383838383838","share_difficulty":7,"share_id":"share-3128","share_seq":3128,"template_height":800097},{"accepted_at_ms":999988,"credit_policy":null,"job_id":"job-3999","job_issued_at_ms":999970,"miner_id":"miner-6","network_difficulty":1000004,"ntime":1700003999,"order_key":"06:miner-6","p2mr_program_hex":"9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f","share_difficulty":1,"share_id":"share-3999","share_seq":3999,"template_height":800124},{"accepted_at_ms":999982,"credit_policy":null,"job_id":"job-551","job_issued_at_ms":999988,"miner_id":"miner-1","network_difficulty":1000007,"ntime":1700000551,"order_key":"01:miner-1","p2mr_program_hex":"2727272727272727272727272727272727272727272727272727272727272727","share_difficulty":7,"share_id":"share-551","share_seq":551,"template_height":800017},{"accepted_at_ms":999999,"credit_policy":"stale-grace","job_id":"job-2466","job_issued_at_ms":999977,"miner_id":"miner-2","network_difficulty":1000001,"ntime":1700002466,"order_key":"02:miner-2","p2mr_program_hex":"a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2","share_difficulty":1000,"share_id":"share-2466","share_seq":2466,"template_height":800077},{"accepted_at_ms":999990,"credit_policy":null,"job_id":"job-3612","job_issued_at_ms":999994,"miner_id":"miner-4","network_difficulty":1000008,"ntime":1700003612,"order_key":"04:miner-4","p2mr_program_hex":"1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c","share_difficulty":1,"share_id":"share-3612","share_seq":3612,"template_height":800112},{"accepted_at_ms":999986,"credit_policy":null,"job_id":"job-161","job_issued_at_ms":999991,"miner_id":"miner-7","network_difficulty":1000008,"ntime":1700000161,"order_key":"07:miner-7","p2mr_program_hex":"a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1","share_difficulty":19,"share_id":"share-161","share_seq":161,"template_height":800005},{"accepted_at_ms":999992,"credit_policy":null,"job_id":"job-182","job_issued_at_ms":999974,"miner_id":"miner-6","network_difficulty":1000012,"ntime":1700000182,"order_key":"06:miner-6","p2mr_program_hex":"b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6","share_difficulty":19,"share_id":"share-182","share_seq":182,"template_height":800005},{"accepted_at_ms":999974,"credit_policy":null,"job_id":"job-3894","job_issued_at_ms":1000401,"miner_id":"miner-0","network_difficulty":1000001,"ntime":1700003894,"order_key":"00:miner-0","p2mr_program_hex":"3636363636363636363636363636363636363636363636363636363636363636","share_difficulty":2,"share_id":"share-3894","share_seq":3894,"template_height":800121},{"accepted_at_ms":1000500,"credit_policy":null,"job_id":"job-3014","job_issued_at_ms":1000239,"miner_id":"miner-0","network_difficulty":1000005,"ntime":1700003014,"order_key":"00:miner-0","p2mr_program_hex":"c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6","share_difficulty":19,"share_id":"share-3014","share_seq":3014,"template_height":800094},{"accepted_at_ms":999980,"credit_policy":null,"job_id":"job-3537","job_issued_at_ms":999985,"miner_id":"miner-6","network_difficulty":1000001,"ntime":1700003537,"order_key":"06:miner-6","p2mr_program_hex":"d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1","share_difficulty":7,"share_id":"share-3537","share_seq":3537,"template_height":800110},{"accepted_at_ms":999985,"credit_policy":null,"job_id":"job-1836","job_issued_at_ms":999999,"miner_id":"miner-10","network_difficulty":1000000,"ntime":1700001836,"order_key":"10:miner-10","p2mr_program_hex":"2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c","share_difficulty":101,"share_id":"share-1836","share_seq":1836,"template_height":800057},{"accepted_at_ms":999993,"credit_policy":null,"job_id":"job-242","job_issued_at_ms":999973,"miner_id":"miner-0","network_difficulty":1000004,"ntime":1700000242,"order_key":"00:miner-0","p2mr_program_hex":"f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2","share_difficulty":7,"share_id":"share-242","share_seq":242,"template_height":800007},{"accepted_at_ms":999990,"credit_policy":null,"job_id":"job-1476","job_issued_at_ms":999970,"miner_id":"miner-2","network_difficulty":1000014,"ntime":1700001476,"order_key":"02:miner-2","p2mr_program_hex":"c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4","share_difficulty":2,"share_id":"share-1476","share_seq":1476,"template_height":800046},{"accepted_at_ms":999986,"credit_policy":"stale-grace","job_id":"job-1585","job_issued_at_ms":999979,"miner_id":"miner-1","network_difficulty":1000004,"ntime":1700001585,"order_key":"01:miner-1","p2mr_program_hex":"3131313131313131313131313131313131313131313131313131313131313131","share_difficulty":1000,"share_id":"share-1585","share_seq":1585,"template_height":800049},{"accepted_at_ms":999995,"credit_policy":"stale-grace","job_id":"job-3192","job_issued_at_ms":999997,"miner_id":"miner-2","network_difficulty":1000013,"ntime":1700003192,"order_key":"02:miner-2","p2mr_program_hex":"7878787878787878787878787878787878787878787878787878787878787878","share_difficulty":19,"share_id":"share-3192","share_seq":3192,"template_height":800099},{"accepted_at_ms":999988,"credit_policy":null,"job_id":"job-2883","job_issued_at_ms":1000466,"miner_id":"miner-1","network_difficulty":1000010,"ntime":1700002883,"order_key":"01:miner-1","p2mr_program_hex":"4343434343434343434343434343434343434343434343434343434343434343","share_difficulty":1,"share_id":"share-2883","share_seq":2883,"template_height":800090},{"accepted_at_ms":999984,"credit_policy":"stale-grace","job_id":"job-1832","job_issued_at_ms":999970,"miner_id":"miner-6","network_difficulty":1000013,"ntime":1700001832,"order_key":"06:miner-6","p2mr_program_hex":"2828282828282828282828282828282828282828282828282828282828282828","share_difficulty":1000,"share_id":"share-1832","share_seq":1832,"template_height":800057},{"accepted_at_ms":999998,"credit_policy":null,"job_id":"job-488","job_issued_at_ms":999999,"miner_id":"miner-4","network_difficulty":1000012,"ntime":1700000488,"order_key":"04:miner-4","p2mr_program_hex":"e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8","share_difficulty":3,"share_id":"share-488","share_seq":488,"template_height":800015},{"accepted_at_ms":999995,"credit_policy":null,"job_id":"job-2668","job_issued_at_ms":1000000,"miner_id":"miner-6","network_difficulty":1000016,"ntime":1700002668,"order_key":"06:miner-6","p2mr_program_hex":"6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c","share_difficulty":101,"share_id":"share-2668","share_seq":2668,"template_height":800083},{"accepted_at_ms":999988,"credit_policy":null,"job_id":"job-3543","job_issued_at_ms":999970,"miner_id":"miner-1","network_difficulty":1000007,"ntime":1700003543,"order_key":"01:miner-1","p2mr_program_hex":"d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7","share_difficulty":2,"share_id":"share-3543","share_seq":3543,"template_height":800110},{"accepted_at_ms":999974,"credit_policy":null,"job_id":"job-2553","job_issued_at_ms":999971,"miner_id":"miner-1","network_difficulty":1000003,"ntime":1700002553,"order_key":"01:miner-1","p2mr_program_hex":"f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9","share_difficulty":3,"share_id":"share-2553","share_seq":2553,"template_height":800079},{"accepted_at_ms":999977,"credit_policy":null,"job_id":"job-3366","job_issued_at_ms":999997,"miner_id":"miner-0","network_difficulty":1000000,"ntime":1700003366,"order_key":"00:miner-0","p2mr_program_hex":"2626262626262626262626262626262626262626262626262626262626262626","share_difficulty":2,"share_id":"share-3366","share_seq":3366,"template_height":800105},{"accepted_at_ms":999987,"credit_policy":null,"job_id":"job-3231","job_issued_at_ms":999988,"miner_id":"miner-8","network_difficulty":1000001,"ntime":1700003231,"order_key":"08:miner-8","p2mr_program_hex":"9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f","share_difficulty":19,"share_id":"share-3231","share_seq":3231,"template_height":800100},{"accepted_at_ms":999998,"credit_policy":null,"job_id":"job-1391","job_issued_at_ms":1000280,"miner_id":"miner-5","network_difficulty":1000014,"ntime":1700001391,"order_key":"05:miner-5","p2mr_program_hex":"6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f","share_difficulty":19,"share_id":"share-1391","share_seq":1391,"template_height":800043},{"accepted_at_ms":999973,"credit_policy":"stale-grace","job_id":"job-469","job_issued_at_ms":999979,"miner_id":"miner-7","network_difficulty":1000010,"ntime":1700000469,"order_key":"07:miner-7","p2mr_program_hex":"d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5","share_difficulty":1,"share_id":"share-469","share_seq":469,"template_height":800014},{"accepted_at_ms":1000320,"credit_policy":null,"job_id":"job-3853","job_issued_at_ms":999997,"miner_id":"miner-3","network_difficulty":1000011,"ntime":1700003853,"order_key":"03:miner-3","p2mr_program_hex":"0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d","share_difficulty":2,"share_id":"share-3853","share_seq":3853,"template_height":800120},{"accepted_at_ms":999976,"credit_policy":null,"job_id":"job-281","job_issued_at_ms":1000142,"miner_id":"miner-6","network_difficulty":1000009,"ntime":1700000281,"order_key":"06:miner-6","p2mr_program_hex":"1919191919191919191919191919191919191919191919191919191919191919","share_difficulty":7,"share_id":"share-281","share_seq":281,"template_height":800008},{"accepted_at_ms":999972,"credit_policy":"stale-grace","job_id":"job-321","job_issued_at_ms":999972,"miner_id":"miner-2","network_difficulty":1000015,"ntime":1700000321,"order_key":"02:miner-2","p2mr_program_hex":"4141414141414141414141414141414141414141414141414141414141414141","share_difficulty":101,"share_id":"share-321","share_seq":321,"template_height":800010},{"accepted_at_ms":999999,"credit_policy":null,"job_id":"job-2559","job_issued_at_ms":1000000,"miner_id":"miner-7","network_difficulty":1000009,"ntime":1700002559,"order_key":"07:miner-7","p2mr_program_hex":"ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff","share_difficulty":3,"share_id":"share-2559","share_seq":2559,"template_height":800079},{"accepted_at_ms":999989,"credit_policy":null,"job_id":"job-20","job_issued_at_ms":999990,"miner_id":"miner-9","network_difficulty":1000003,"ntime":1700000020,"order_key":"09:miner-9","p2mr_program_hex":"1414141414141414141414141414141414141414141414141414141414141414","share_difficulty":1000,"share_id":"share-20","share_seq":20,"template_height":800000},{"accepted_at_ms":999976,"credit_policy":null,"job_id":"job-1994","job_issued_at_ms":999999,"miner_id":"miner-3","network_difficulty":1000005,"ntime":1700001994,"order_key":"03:miner-3","p2mr_program_hex":"cacacacacacacacacacacacacacacacacacacacacacacacacacacacacacacaca","share_difficulty":2,"share_id":"share-1994","share_seq":1994,"template_height":800062},{"accepted_at_ms":999988,"credit_policy":null,"job_id":"job-3433","job_issued_at_ms":999980,"miner_id":"miner-1","network_difficulty":1000016,"ntime":1700003433,"order_key":"01:miner-1","p2mr_program_hex":"6969696969696969696969696969696969696969696969696969696969696969","share_difficulty":2,"share_id":"share-3433","share_seq":3433,"template_height":800107},{"accepted_at_ms":999999,"credit_policy":null,"job_id":"job-2774","job_issued_at_ms":999998,"miner_id":"miner-2","network_difficulty":1000003,"ntime":1700002774,"order_key":"02:miner-2","p2mr_program_hex":"d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6","share_difficulty":3,"share_id":"share-2774","share_seq":2774,"template_height":800086},{"accepted_at_ms":999977,"credit_policy":"stale-grace","job_id":"job-1524","job_issued_at_ms":999976,"miner_id":"miner-6","network_difficulty":1000011,"ntime":1700001524,"order_key":"06:miner-6","p2mr_program_hex":"f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4","share_difficulty":1000,"share_id":"share-1524","share_seq":1524,"template_height":800047},{"accepted_at_ms":1000236,"credit_policy":null,"job_id":"job-2348","job_issued_at_ms":999990,"miner_id":"miner-5","network_difficulty":1000002,"ntime":1700002348,"order_key":"05:miner-5","p2mr_program_hex":"2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c","share_difficulty":3,"share_id":"share-2348","share_seq":2348,"template_height":800073},{"accepted_at_ms":999989,"credit_policy":null,"job_id":"job-3341","job_issued_at_ms":999973,"miner_id":"miner-8","network_difficulty":1000009,"ntime":1700003341,"order_key":"08:miner-8","p2mr_program_hex":"0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d","share_difficulty":2,"share_id":"share-3341","share_seq":3341,"template_height":800104},{"accepted_at_ms":999997,"credit_policy":null,"job_id":"job-938","job_issued_at_ms":999991,"miner_id":"miner-3","network_difficulty":1000003,"ntime":1700000938,"order_key":"03:miner-3","p2mr_program_hex":"aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa","share_difficulty":3,"share_id":"share-938","share_seq":938,"template_height":800029},{"accepted_at_ms":999984,"credit_policy":null,"job_id":"job-2396","job_issued_at_ms":999991,"miner_id":"miner-9","network_difficulty":1000016,"ntime":1700002396,"order_key":"09:miner-9","p2mr_program_hex":"5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c","share_difficulty":7,"share_id":"share-2396","share_seq":2396,"template_height":800074},{"accepted_at_ms":999973,"credit_policy":null,"job_id":"job-3548","job_issued_at_ms":999975,"miner_id":"miner-6","network_difficulty":1000012,"ntime":1700003548,"order_key":"06:miner-6","p2mr_program_hex":"dcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdc","share_difficulty":1000,"share_id":"share-3548","share_seq":3548,"template_height":800110},{"accepted_at_ms":999972,"credit_policy":null,"job_id":"job-2756","job_issued_at_ms":999990,"miner_id":"miner-6","network_difficulty":1000002,"ntime":1700002756,"order_key":"06:miner-6","p2mr_program_hex":"c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4","share_difficulty":19,"share_id":"share-2756","share_seq":2756,"template_height":800086},{"accepted_at_ms":999993,"credit_policy":"stale-grace","job_id":"job-450","job_issued_at_ms":1000313,"miner_id":"miner-10","network_difficulty":1000008,"ntime":1700000450,"order_key":"10:miner-10","p2mr_program_hex":"c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2","share_difficulty":2,"share_id":"share-450","share_seq":450,"template_height":800014},{"accepted_at_ms":999980,"credit_policy":null,"job_id":"job-611","job_issued_at_ms":999989,"miner_id":"miner-6","network_difficulty":1000016,"ntime":1700000611,"order_key":"06:miner-6","p2mr_program_hex":"6363636363636363636363636363636363636363636363636363636363636363","share_difficulty":1,"share_id":"share-611","share_seq":611,"template_height":800019},{"accepted_at_ms":999997,"credit_policy":null,"job_id":"job-2100","job_issued_at_ms":999980,"miner_id":"miner-10","network_difficulty":1000009,"ntime":1700002100,"order_key":"10:miner-10","p2mr_program_hex":"3434343434343434343434343434343434343434343434343434343434343434","share_difficulty":2,"share_id":"share-2100","share_seq":2100,"template_height":800065},{"accepted_at_ms":999975,"credit_policy":null,"job_id":"job-2218","job_issued_at_ms":999976,"miner_id":"miner-7","network_difficulty":1000008,"ntime":1700002218,"order_key":"07:miner-7","p2mr_program_hex":"aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa","share_difficulty":1,"share_id":"share-2218","share_seq":2218,"template_height":800069},{"accepted_at_ms":999991,"credit_policy":null,"job_id":"job-3568","job_issued_at_ms":999981,"miner_id":"miner-4","network_difficulty":1000015,"ntime":1700003568,"order_key":"04:miner-4","p2mr_program_hex":"f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0","share_difficulty":101,"share_id":"share-3568","share_seq":3568,"template_height":800111},{"accepted_at_ms":999991,"credit_policy":null,"job_id":"job-3357","job_issued_at_ms":999973,"miner_id":"miner-2","network_difficulty":1000008,"ntime":1700003357,"order_key":"02:miner-2","p2mr_program_hex":"1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d","share_difficulty":1000,"share_id":"share-3357","share_seq":3357,"template_height":800104},{"accepted_at_ms":999975,"credit_policy":null,"job_id":"job-1229","job_issued_at_ms":999979,"miner_id":"miner-8","network_difficulty":1000005,"ntime":1700001229,"order_key":"08:miner-8","p2mr_program_hex":"cdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcd","share_difficulty":7,"share_id":"share-1229","share_seq":1229,"template_height":800038},{"accepted_at_ms":999982,"credit_policy":null,"job_id":"job-1355","job_issued_at_ms":999988,"miner_id":"miner-2","network_difficulty":1000012,"ntime":1700001355,"order_key":"02:miner-2","p2mr_program_hex":"4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b","share_difficulty":101,"share_id":"share-1355","share_seq":1355,"template_height":800042},{"accepted_at_ms":999977,"credit_policy":"stale-grace","job_id":"job-1005","job_issued_at_ms":999976,"miner_id":"miner-4","network_difficulty":1000002,"ntime":1700001005,"order_key":"04:miner-4","p2mr_program_hex":"edededededededededededededededededededededededededededededededed","share_difficulty":19,"share_id":"share-1005","share_seq":1005,"template_height":800031},{"accepted_at_ms":1000216,"credit_policy":null,"job_id":"job-1270","job_issued_at_ms":999995,"miner_id":"miner-5","network_difficulty":1000012,"ntime":1700001270,"order_key":"05:miner-5","p2mr_program_hex":"f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6","share_difficulty":101,"share_id":"share-1270","share_seq":1270,"template_height":800039},{"accepted_at_ms":999979,"credit_policy":null,"job_id":"job-1739","job_issued_at_ms":999984,"miner_id":"miner-1","network_difficulty":1000005,"ntime":1700001739,"order_key":"01:miner-1","p2mr_program_hex":"cbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcb","share_difficulty":7,"share_id":"share-1739","share_seq":1739,"template_height":800054},{"accepted_at_ms":999990,"credit_policy":null,"job_id":"job-478","job_issued_at_ms":999989,"miner_id":"miner-5","network_difficulty":1000002,"ntime":1700000478,"order_key":"05:miner-5","p2mr_program_hex":"dededededededededededededededededededededededededededededededede","share_difficulty":2,"share_id":"share-478","share_seq":478,"template_height":800014},{"accepted_at_ms":999990,"credit_policy":"stale-grace","job_id":"job-3243","job_issued_at_ms":999975,"miner_id":"miner-9","network_difficulty":1000013,"ntime":1700003243,"order_key":"09:miner-9","p2mr_program_hex":"abababababababababababababababababababababababababababababababab","share_difficulty":2,"share_id":"share-3243","share_seq":3243,"template_height":800101},{"accepted_at_ms":999996,"credit_policy":null,"job_id":"job-2561","job_issued_at_ms":999999,"miner_id":"miner-9","network_difficulty":1000011,"ntime":1700002561,"order_key":"09:miner-9","p2mr_program_hex":"0101010101010101010101010101010101010101010101010101010101010101","share_difficulty":19,"share_id":"share-2561","share_seq":2561,"template_height":800080},{"accepted_at_ms":999975,"credit_policy":null,"job_id":"job-1264","job_issued_at_ms":999977,"miner_id":"miner-10","network_difficulty":1000006,"ntime":1700001264,"order_key":"10:miner-10","p2mr_program_hex":"f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0","share_difficulty":1000,"share_id":"share-1264","share_seq":1264,"template_height":800039},{"accepted_at_ms":999988,"credit_policy":null,"job_id":"job-3814","job_issued_at_ms":999995,"miner_id":"miner-8","network_difficulty":1000006,"ntime":1700003814,"order_key":"08:miner-8","p2mr_program_hex":"e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6","share_difficulty":7,"share_id":"share-3814","share_seq":3814,"template_height":800119},{"accepted_at_ms":1000000,"credit_policy":null,"job_id":"job-1040","job_issued_at_ms":999995,"miner_id":"miner-6","network_difficulty":1000003,"ntime":1700001040,"order_key":"06:miner-6","p2mr_program_hex":"1010101010101010101010101010101010101010101010101010101010101010","share_difficulty":7,"share_id":"share-1040","share_seq":1040,"template_height":800032},{"accepted_at_ms":999976,"credit_policy":null,"job_id":"job-513","job_issued_at_ms":999998,"miner_id":"miner-7","network_difficulty":1000003,"ntime":1700000513,"order_key":"07:miner-7","p2mr_program_hex":"0101010101010101010101010101010101010101010101010101010101010101","share_difficulty":3,"share_id":"share-513","share_seq":513,"template_height":800016},{"accepted_at_ms":999972,"credit_policy":null,"job_id":"job-3514","job_issued_at_ms":999992,"miner_id":"miner-5","network_difficulty":1000012,"ntime":1700003514,"order_key":"05:miner-5","p2mr_program_hex":"babababababababababababababababababababababababababababababababa","share_difficulty":7,"share_id":"share-3514","share_seq":3514,"template_height":800109},{"accepted_at_ms":999988,"credit_policy":null,"job_id":"job-2605","job_issued_at_ms":999999,"miner_id":"miner-9","network_difficulty":1000004,"ntime":1700002605,"order_key":"09:miner-9","p2mr_program_hex":"2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d","share_difficulty":101,"share_id":"share-2605","share_seq":2605,"template_height":800081},{"accepted_at_ms":999970,"credit_policy":null,"job_id":"job-196","job_issued_at_ms":999986,"miner_id":"miner-9","network_difficulty":1000009,"ntime":1700000196,"order_key":"09:miner-9","p2mr_program_hex":"c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4","share_difficulty":1000,"share_id":"share-196","share_seq":196,"template_height":800006},{"accepted_at_ms":999981,"credit_policy":null,"job_id":"job-1633","job_issued_at_ms":999995,"miner_id":"miner-5","network_difficulty":1000001,"ntime":1700001633,"order_key":"05:miner-5","p2mr_program_hex":"6161616161616161616161616161616161616161616161616161616161616161","share_difficulty":1,"share_id":"share-1633","share_seq":1633,"template_height":800051},{"accepted_at_ms":999998,"credit_policy":null,"job_id":"job-728","job_issued_at_ms":999994,"miner_id":"miner-2","network_difficulty":1000014,"ntime":1700000728,"order_key":"02:miner-2","p2mr_program_hex":"d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8","share_difficulty":19,"share_id":"share-728","share_seq":728,"template_height":800022},{"accepted_at_ms":999981,"credit_policy":"stale-grace","job_id":"job-3095","job_issued_at_ms":999989,"miner_id":"miner-4","network_difficulty":1000001,"ntime":1700003095,"order_key":"04:miner-4","p2mr_program_hex":"1717171717171717171717171717171717171717171717171717171717171717","share_difficulty":1,"share_id":"share-3095","share_seq":3095,"template_height":800096},{"accepted_at_ms":999973,"credit_policy":null,"job_id":"job-3685","job_issued_at_ms":999984,"miner_id":"miner-0","network_difficulty":1000013,"ntime":1700003685,"order_key":"00:miner-0","p2mr_program_hex":"6565656565656565656565656565656565656565656565656565656565656565","share_difficulty":101,"share_id":"share-3685","share_seq":3685,"template_height":800115},{"accepted_at_ms":999978,"credit_policy":null,"job_id":"job-1942","job_issued_at_ms":999999,"miner_id":"miner-6","network_difficulty":1000004,"ntime":1700001942,"order_key":"06:miner-6","p2mr_program_hex":"9696969696969696969696969696969696969696969696969696969696969696","share_difficulty":101,"share_id":"share-1942","share_seq":1942,"template_height":800060},{"accepted_at_ms":999984,"credit_policy":null,"job_id":"job-3191","job_issued_at_ms":999978,"miner_id":"miner-1","network_difficulty":1000012,"ntime":1700003191,"order_key":"01:miner-1","p2mr_program_hex":"7777777777777777777777777777777777777777777777777777777777777777","share_difficulty":19,"share_id":"share-3191","share_seq":3191,"template_height":800099},{"accepted_at_ms":999993,"credit_policy":null,"job_id":"job-2129","job_issued_at_ms":999984,"miner_id":"miner-6","network_difficulty":1000004,"ntime":1700002129,"order_key":"06:miner-6","p2mr_program_hex":"5151515151515151515151515151515151515151515151515151515151515151","share_difficulty":1000,"share_id":"share-2129","share_seq":2129,"template_height":800066},{"accepted_at_ms":999982,"credit_policy":null,"job_id":"job-361","job_issued_at_ms":999976,"miner_id":"miner-9","network_difficulty":1000004,"ntime":1700000361,"order_key":"09:miner-9","p2mr_program_hex":"6969696969696969696969696969696969696969696969696969696969696969","share_difficulty":101,"share_id":"share-361","share_seq":361,"template_height":800011},{"accepted_at_ms":999987,"credit_policy":"stale-grace","job_id":"job-3148","job_issued_at_ms":999993,"miner_id":"miner-2","network_difficulty":1000003,"ntime":1700003148,"order_key":"02:miner-2","p2mr_program_hex":"4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c","share_difficulty":19,"share_id":"share-3148","share_seq":3148,"template_height":800098},{"accepted_at_ms":999987,"credit_policy":null,"job_id":"job-271","job_issued_at_ms":999977,"miner_id":"miner-7","network_difficulty":1000016,"ntime":1700000271,"order_key":"07:miner-7","p2mr_program_hex":"0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f","share_difficulty":1000,"share_id":"share-271","share_seq":271,"template_height":800008},{"accepted_at_ms":999977,"credit_policy":null,"job_id":"job-1051","job_issued_at_ms":999994,"miner_id":"miner-6","network_difficulty":1000014,"ntime":1700001051,"order_key":"06:miner-6","p2mr_program_hex":"1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b","share_difficulty":101,"share_id":"share-1051","share_seq":1051,"template_height":800032},{"accepted_at_ms":999971,"credit_policy":null,"job_id":"job-2720","job_issued_at_ms":999971,"miner_id":"miner-3","network_difficulty":1000000,"ntime":1700002720,"order_key":"03:miner-3","p2mr_program_hex":"a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0","share_difficulty":3,"share_id":"share-2720","share_seq":2720,"template_height":800085},{"accepted_at_ms":999998,"credit_policy":null,"job_id":"job-2503","job_issued_at_ms":999975,"miner_id":"miner-6","network_difficulty":1000004,"ntime":1700002503,"order_key":"06:miner-6","p2mr_program_hex":"c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7","share_difficulty":19,"share_id":"share-2503","share_seq":2503,"template_height":800078},{"accepted_at_ms":999971,"credit_policy":null,"job_id":"job-3093","job_issued_at_ms":999999,"miner_id":"miner-2","network_difficulty":1000016,"ntime":1700003093,"order_key":"02:miner-2","p2mr_program_hex":"1515151515151515151515151515151515151515151515151515151515151515","share_difficulty":19,"share_id":"share-3093","share_seq":3093,"template_height":800096},{"accepted_at_ms":1000000,"credit_policy":null,"job_id":"job-559","job_issued_at_ms":999986,"miner_id":"miner-9","network_difficulty":1000015,"ntime":1700000559,"order_key":"09:miner-9","p2mr_program_hex":"2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f","share_difficulty":1,"share_id":"share-559","share_seq":559,"template_height":800017},{"accepted_at_ms":999983,"credit_policy":null,"job_id":"job-1441","job_issued_at_ms":999986,"miner_id":"miner-0","network_difficulty":1000013,"ntime":1700001441,"order_key":"00:miner-0","p2mr_program_hex":"a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1","share_difficulty":19,"share_id":"share-1441","share_seq":1441,"template_height":800045},{"accepted_at_ms":999989,"credit_policy":null,"job_id":"job-2933","job_issued_at_ms":999976,"miner_id":"miner-7","network_difficulty":1000009,"ntime":1700002933,"order_key":"07:miner-7","p2mr_program_hex":"7575757575757575757575757575757575757575757575757575757575757575","share_difficulty":1,"share_id":"share-2933","share_seq":2933,"template_height":800091},{"accepted_at_ms":999983,"credit_policy":null,"job_id":"job-3283","job_issued_at_ms":999975,"miner_id":"miner-5","network_difficulty":1000002,"ntime":1700003283,"order_key":"05:miner-5","p2mr_program_hex":"d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3","share_difficulty":2,"share_id":"share-3283","share_seq":3283,"template_height":800102},{"accepted_at_ms":999970,"credit_policy":null,"job_id":"job-1049","job_issued_at_ms":999970,"miner_id":"miner-4","network_difficulty":1000012,"ntime":1700001049,"order_key":"04:miner-4","p2mr_program_hex":"1919191919191919191919191919191919191919191919191919191919191919","share_difficulty":3,"share_id":"share-1049","share_seq":1049,"template_height":800032},{"accepted_at_ms":999995,"credit_policy":null,"job_id":"job-622","job_issued_at_ms":999976,"miner_id":"miner-6","network_difficulty":1000010,"ntime":1700000622,"order_key":"06:miner-6","p2mr_program_hex":"6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e","share_difficulty":19,"share_id":"share-622","share_seq":622,"template_height":800019},{"accepted_at_ms":999982,"credit_policy":null,"job_id":"job-3596","job_issued_at_ms":999999,"miner_id":"miner-10","network_difficulty":1000009,"ntime":1700003596,"order_key":"10:miner-10","p2mr_program_hex":"0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c","share_difficulty":1,"share_id":"share-3596","share_seq":3596,"template_height":800112},{"accepted_at_ms":999971,"credit_policy":"stale-grace","job_id":"job-3214","job_issued_at_ms":999993,"miner_id":"miner-2","network_difficulty":1000001,"ntime":1700003214,"order_key":"02:miner-2","p2mr_program_hex":"8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e","share_difficulty":2,"share_id":"share-3214","share_seq":3214,"template_height":800100},{"accepted_at_ms":999990,"credit_policy":null,"job_id":"job-1797","job_issued_at_ms":1000315,"miner_id":"miner-4","network_difficulty":1000012,"ntime":1700001797,"order_key":"04:miner-4","p2mr_program_hex":"0505050505050505050505050505050505050505050505050505050505050505","share_difficulty":19,"share_id":"share-1797","share_seq":1797,"template_height":800056},{"accepted_at_ms":999983,"credit_policy":"stale-grace","job_id":"job-3756","job_issued_at_ms":999983,"miner_id":"miner-5","network_difficulty":1000016,"ntime":1700003756,"order_key":"05:miner-5","p2mr_program_hex":"acacacacacacacacacacacacacacacacacacacacacacacacacacacacacacacac","share_difficulty":1,"share_id":"share-3756","share_seq":3756,"template_height":800117},{"accepted_at_ms":999971,"credit_policy":"stale-grace","job_id":"job-356","job_issued_at_ms":999978,"miner_id":"miner-4","network_difficulty":1000016,"ntime":1700000356,"order_key":"04:miner-4","p2mr_program_hex":"6464646464646464646464646464646464646464646464646464646464646464","share_difficulty":101,"share_id":"share-356","share_seq":356,"template_height":800011},{"accepted_at_ms":999994,"credit_policy":null,"job_id":"job-2580","job_issued_at_ms":999996,"miner_id":"miner-6","network_difficulty":1000013,"ntime":1700002580,"order_key":"06:miner-6","p2mr_program_hex":"1414141414141414141414141414141414141414141414141414141414141414","share_difficulty":101,"share_id":"share-2580","share_seq":2580,"template_height":800080},{"accepted_at_ms":999983,"credit_policy":null,"job_id":"job-3749","job_issued_at_ms":999992,"miner_id":"miner-9","network_difficulty":1000009,"ntime":1700003749,"order_key":"09:miner-9","p2mr_program_hex":"a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5","share_difficulty":2,"share_id":"share-3749","share_seq":3749,"template_height":800117},{"accepted_at_ms":999996,"credit_policy":"stale-grace","job_id":"job-2623","job_issued_at_ms":999973,"miner_id":"miner-5","network_difficulty":1000005,"ntime":1700002623,"order_key":"05:miner-5","p2mr_program_hex":"3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f","share_difficulty":1000,"share_id":"share-2623","share_seq":2623,"template_height":800081},{"accepted_at_ms":999988,"credit_policy":null,"job_id":"job-1809","job_issued_at_ms":999974,"miner_id":"miner-5","network_difficulty":1000007,"ntime":1700001809,"order_key":"05:miner-5","p2mr_program_hex":"1111111111111111111111111111111111111111111111111111111111111111","share_difficulty":2,"share_id":"share-1809","share_seq":1809,"template_height":800056},{"accepted_at_ms":999993,"credit_policy":null,"job_id":"job-127","job_issued_at_ms":999987,"miner_id":"miner-6","network_difficulty":1000008,"ntime":1700000127,"order_key":"06:miner-6","p2mr_program_hex":"7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f","share_difficulty":7,"share_id":"share-127","share_seq":127,"template_height":800003},{"accepted_at_ms":999971,"credit_policy":null,"job_id":"job-3902","job_issued_at_ms":1000000,"miner_id":"miner-8","network_difficulty":1000009,"ntime":1700003902,"order_key":"08:miner-8","p2mr_program_hex":"3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e","share_difficulty":7,"share_id":"share-3902","share_seq":3902,"template_height":800121},{"accepted_at_ms":999986,"credit_policy":null,"job_id":"job-3012","job_issued_at_ms":999977,"miner_id":"miner-9","network_difficulty":1000003,"ntime":1700003012,"order_key":"09:miner-9","p2mr_program_hex":"c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4","share_difficulty":19,"share_id":"share-3012","share_seq":3012,"template_height":800094},{"accepted_at_ms":999989,"credit_policy":null,"job_id":"job-2269","job_issued_at_ms":999983,"miner_id":"miner-3","network_difficulty":1000008,"ntime":1700002269,"order_key":"03:miner-3","p2mr_program_hex":"dddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddd","share_difficulty":2,"share_id":"share-2269","share_seq":2269,"template_height":800070},{"accepted_at_ms":999970,"credit_policy":null,"job_id":"job-1357","job_issued_at_ms":999978,"miner_id":"miner-4","network_difficulty":1000014,"ntime":1700001357,"order_key":"04:miner-4","p2mr_program_hex":"4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d","share_difficulty":19,"share_id":"share-1357","share_seq":1357,"template_height":800042},{"accepted_at_ms":999980,"credit_policy":"stale-grace","job_id":"job-2301","job_issued_at_ms":999986,"miner_id":"miner-2","network_difficulty":1000006,"ntime":1700002301,"order_key":"02:miner-2","p2mr_program_hex":"fdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfd","share_difficulty":7,"share_id":"share-2301","share_seq":2301,"template_height":800071},{"accepted_at_ms":999995,"credit_policy":null,"job_id":"job-2892","job_issued_at_ms":999983,"miner_id":"miner-10","network_difficulty":1000002,"ntime":1700002892,"order_key":"10:miner-10","p2mr_program_hex":"4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c","share_difficulty":2,"share_id":"share-2892","share_seq":2892,"template_height":800090},{"accepted_at_ms":999972,"credit_policy":null,"job_id":"job-2899","job_issued_at_ms":999991,"miner_id":"miner-6","network_difficulty":1000009,"ntime":1700002899,"order_key":"06:miner-6","p2mr_program_hex":"5353535353535353535353535353535353535353535353535353535353535353","share_difficulty":2,"share_id":"share-2899","share_seq":2899,"template_height":800090},{"accepted_at_ms":999994,"credit_policy":"stale-grace","job_id":"job-2248","job_issued_at_ms":999980,"miner_id":"miner-4","network_difficulty":1000004,"ntime":1700002248,"order_key":"04:miner-4","p2mr_program_hex":"c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8","share_difficulty":101,"share_id":"share-2248","share_seq":2248,"template_height":800070},{"accepted_at_ms":999997,"credit_policy":null,"job_id":"job-1581","job_issued_at_ms":999990,"miner_id":"miner-8","network_difficulty":1000000,"ntime":1700001581,"order_key":"08:miner-8","p2mr_program_hex":"2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d","share_difficulty":2,"share_id":"share-1581","share_seq":1581,"template_height":800049},{"accepted_at_ms":999995,"credit_policy":null,"job_id":"job-2321","job_issued_at_ms":999981,"miner_id":"miner-0","network_difficulty":1000009,"ntime":1700002321,"order_key":"00:miner-0","p2mr_program_hex":"1111111111111111111111111111111111111111111111111111111111111111","share_difficulty":1000,"share_id":"share-2321","share_seq":2321,"template_height":800072},{"accepted_at_ms":999973,"credit_policy":"stale-grace","job_id":"job-3793","job_issued_at_ms":999978,"miner_id":"miner-9","network_difficulty":1000002,"ntime":1700003793,"order_key":"09:miner-9","p2mr_program_hex":"d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1","share_difficulty":1000,"share_id":"share-3793","share_seq":3793,"template_height":800118},{"accepted_at_ms":999995,"credit_policy":null,"job_id":"job-536","job_issued_at_ms":999982,"miner_id":"miner-8","network_difficulty":1000009,"ntime":1700000536,"order_key":"08:miner-8","p2mr_program_hex":"1818181818181818181818181818181818181818181818181818181818181818","share_difficulty":7,"share_id":"share-536","share_seq":536,"template_height":800016},{"accepted_at_ms":999985,"credit_policy":null,"job_id":"job-834","job_issued_at_ms":999981,"miner_id":"miner-9","network_difficulty":1000001,"ntime":1700000834,"order_key":"09:miner-9","p2mr_program_hex":"4242424242424242424242424242424242424242424242424242424242424242","share_difficulty":1000,"share_id":"share-834","share_seq":834,"template_height":800026},{"accepted_at_ms":999996,"credit_policy":null,"job_id":"job-3821","job_issued_at_ms":999973,"miner_id":"miner-4","network_difficulty":1000013,"ntime":1700003821,"order_key":"04:miner-4","p2mr_program_hex":"edededededededededededededededededededededededededededededededed","share_difficulty":1,"share_id":"share-3821","share_seq":3821,"template_height":800119},{"accepted_at_ms":999989,"credit_policy":null,"job_id":"job-1067","job_issued_at_ms":999970,"miner_id":"miner-0","network_difficulty":1000013,"ntime":1700001067,"order_key":"00:miner-0","p2mr_program_hex":"2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b","share_difficulty":3,"share_id":"share-1067","share_seq":1067,"template_height":800033},{"accepted_at_ms":999995,"credit_policy":null,"job_id":"job-3111","job_issued_at_ms":999983,"miner_id":"miner-9","network_difficulty":1000000,"ntime":1700003111,"order_key":"09:miner-9","p2mr_program_hex":"2727272727272727272727272727272727272727272727272727272727272727","share_difficulty":3,"share_id":"share-3111","share_seq":3111,"template_height":800097},{"accepted_at_ms":999973,"credit_policy":"stale-grace","job_id":"job-759","job_issued_at_ms":999980,"miner_id":"miner-0","network_difficulty":1000011,"ntime":1700000759,"order_key":"00:miner-0","p2mr_program_hex":"f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7","share_difficulty":1000,"share_id":"share-759","share_seq":759,"template_height":800023},{"accepted_at_ms":999996,"credit_policy":null,"job_id":"job-394","job_issued_at_ms":999989,"miner_id":"miner-9","network_difficulty":1000003,"ntime":1700000394,"order_key":"09:miner-9","p2mr_program_hex":"8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a","share_difficulty":101,"share_id":"share-394","share_seq":394,"template_height":800012},{"accepted_at_ms":999978,"credit_policy":null,"job_id":"job-162","job_issued_at_ms":999977,"miner_id":"miner-8","network_difficulty":1000009,"ntime":1700000162,"order_key":"08:miner-8","p2mr_program_hex":"a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2","share_difficulty":1,"share_id":"share-162","share_seq":162,"template_height":800005},{"accepted_at_ms":999977,"credit_policy":"stale-grace","job_id":"job-317","job_issued_at_ms":999972,"miner_id":"miner-9","network_difficulty":1000011,"ntime":1700000317,"order_key":"09:miner-9","p2mr_program_hex":"3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d","share_difficulty":19,"share_id":"share-317","share_seq":317,"template_height":800009},{"accepted_at_ms":999991,"credit_policy":null,"job_id":"job-1170","job_issued_at_ms":999997,"miner_id":"miner-4","network_difficulty":1000014,"ntime":1700001170,"order_key":"04:miner-4","p2mr_program_hex":"9292929292929292929292929292929292929292929292929292929292929292","share_difficulty":2,"share_id":"share-1170","share_seq":1170,"template_height":800036},{"accepted_at_ms":999992,"credit_policy":"stale-grace","job_id":"job-2053","job_issued_at_ms":999997,"miner_id":"miner-7","network_difficulty":1000013,"ntime":1700002053,"order_key":"07:miner-7","p2mr_program_hex":"0505050505050505050505050505050505050505050505050505050505050505","share_difficulty":1,"share_id":"share-2053","share_seq":2053,"template_height":800064},{"accepted_at_ms":999996,"credit_policy":null,"job_id":"job-903","job_issued_at_ms":999976,"miner_id":"miner-1","network_difficulty":1000002,"ntime":1700000903,"order_key":"01:miner-1","p2mr_program_hex":"8787878787878787878787878787878787878787878787878787878787878787","share_difficulty":7,"share_id":"share-903","share_seq":903,"template_height":800028},{"accepted_at_ms":999998,"credit_policy":null,"job_id":"job-1702","job_issued_at_ms":999998,"miner_id":"miner-8","network_difficulty":1000002,"ntime":1700001702,"order_key":"08:miner-8","p2mr_program_hex":"a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6","share_difficulty":101,"share_id":"share-1702","share_seq":1702,"template_height":800053},{"accepted_at_ms":999983,"credit_policy":"stale-grace","job_id":"job-2500","job_issued_at_ms":999975,"miner_id":"miner-3","network_difficulty":1000001,"ntime":1700002500,"order_key":"03:miner-3","p2mr_program_hex":"c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4","share_difficulty":19,"share_id":"share-2500","share_seq":2500,"template_height":800078},{"accepted_at_ms":999986,"credit_policy":null,"job_id":"job-3186","job_issued_at_ms":999995,"miner_id":"miner-7","network_difficulty":1000007,"ntime":1700003186,"order_key":"07:miner-7","p2mr_program_hex":"7272727272727272727272727272727272727272727272727272727272727272","share_difficulty":3,"share_id":"share-3186","share_seq":3186,"template_height":800099},{"accepted_at_ms":999983,"credit_policy":null,"job_id":"job-1069","job_issued_at_ms":999993,"miner_id":"miner-2","network_difficulty":1000015,"ntime":1700001069,"order_key":"02:miner-2","p2mr_program_hex":"2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d","share_difficulty":3,"share_id":"share-1069","share_seq":1069,"template_height":800033},{"accepted_at_ms":999985,"credit_policy":null,"job_id":"job-3763","job_issued_at_ms":999989,"miner_id":"miner-1","network_difficulty":1000006,"ntime":1700003763,"order_key":"01:miner-1","p2mr_program_hex":"b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3","share_difficulty":1,"share_id":"share-3763","share_seq":3763,"template_height":800117},{"accepted_at_ms":999970,"credit_policy":null,"job_id":"job-1825","job_issued_at_ms":999970,"miner_id":"miner-10","network_difficulty":1000006,"ntime":1700001825,"order_key":"10:miner-10","p2mr_program_hex":"2121212121212121212121212121212121212121212121212121212121212121","share_difficulty":101,"share_id":"share-1825","share_seq":1825,"template_height":800057},{"accepted_at_ms":999996,"credit_policy":null,"job_id":"job-1072","job_issued_at_ms":999998,"miner_id":"miner-5","network_difficulty":1000001,"ntime":1700001072,"order_key":"05:miner-5","p2mr_program_hex":"3030303030303030303030303030303030303030303030303030303030303030","share_difficulty":19,"share_id":"share-1072","share_seq":1072,"template_height":800033},{"accepted_at_ms":1000487,"credit_policy":null,"job_id":"job-234","job_issued_at_ms":999986,"miner_id":"miner-3","network_difficulty":1000013,"ntime":1700000234,"order_key":"03:miner-3","p2mr_program_hex":"eaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaea","share_difficulty":2,"share_id":"share-234","share_seq":234,"template_height":800007},{"accepted_at_ms":999985,"credit_policy":null,"job_id":"job-686","job_issued_at_ms":1000225,"miner_id":"miner-4","network_difficulty":1000006,"ntime":1700000686,"order_key":"04:miner-4","p2mr_program_hex":"aeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeae","share_difficulty":3,"share_id":"share-686","share_seq":686,"template_height":800021},{"accepted_at_ms":999989,"credit_policy":"stale-grace","job_id":"job-2285","job_issued_at_ms":999985,"miner_id":"miner-8","network_difficulty":1000007,"ntime":1700002285,"order_key":"08:miner-8","p2mr_program_hex":"edededededededededededededededededededededededededededededededed","share_difficulty":101,"share_id":"share-2285","share_seq":2285,"template_height":800071},{"accepted_at_ms":999987,"credit_policy":null,"job_id":"job-895","job_issued_at_ms":999993,"miner_id":"miner-4","network_difficulty":1000011,"ntime":1700000895,"order_key":"04:miner-4","p2mr_program_hex":"7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f","share_difficulty":2,"share_id":"share-895","share_seq":895,"template_height":800027},{"accepted_at_ms":999973,"credit_policy":null,"job_id":"job-2247","job_issued_at_ms":999992,"miner_id":"miner-3","network_difficulty":1000003,"ntime":1700002247,"order_key":"03:miner-3","p2mr_program_hex":"c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7","share_difficulty":2,"share_id":"share-2247","share_seq":2247,"template_height":800070},{"accepted_at_ms":999978,"credit_policy":"stale-grace","job_id":"job-2310","job_issued_at_ms":999990,"miner_id":"miner-0","network_difficulty":1000015,"ntime":1700002310,"order_key":"00:miner-0","p2mr_program_hex":"0606060606060606060606060606060606060606060606060606060606060606","share_difficulty":3,"share_id":"share-2310","share_seq":2310,"template_height":800072},{"accepted_at_ms":999989,"credit_policy":"stale-grace","job_id":"job-3495","job_issued_at_ms":999977,"miner_id":"miner-8","network_difficulty":1000010,"ntime":1700003495,"order_key":"08:miner-8","p2mr_program_hex":"a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7","share_difficulty":1000,"share_id":"share-3495","share_seq":3495,"template_height":800109},{"accepted_at_ms":999984,"credit_policy":null,"job_id":"job-1928","job_issued_at_ms":999995,"miner_id":"miner-3","network_difficulty":1000007,"ntime":1700001928,"order_key":"03:miner-3","p2mr_program_hex":"8888888888888888888888888888888888888888888888888888888888888888","share_difficulty":1000,"share_id":"share-1928","share_seq":1928,"template_height":800060},{"accepted_at_ms":999999,"credit_policy":null,"job_id":"job-3941","job_issued_at_ms":999997,"miner_id":"miner-3","network_difficulty":1000014,"ntime":1700003941,"order_key":"03:miner-3","p2mr_program_hex":"6565656565656565656565656565656565656565656565656565656565656565","share_difficulty":2,"share_id":"share-3941","share_seq":3941,"template_height":800123},{"accepted_at_ms":999988,"credit_policy":null,"job_id":"job-3101","job_issued_at_ms":999976,"miner_id":"miner-10","network_difficulty":1000007,"ntime":1700003101,"order_key":"10:miner-10","p2mr_program_hex":"1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d","share_difficulty":2,"share_id":"share-3101","share_seq":3101,"template_height":800096},{"accepted_at_ms":999998,"credit_policy":null,"job_id":"job-3906","job_issued_at_ms":1000000,"miner_id":"miner-1","network_difficulty":1000013,"ntime":1700003906,"order_key":"01:miner-1","p2mr_program_hex":"4242424242424242424242424242424242424242424242424242424242424242","share_difficulty":7,"share_id":"share-3906","share_seq":3906,"template_height":800122},{"accepted_at_ms":1000213,"credit_policy":null,"job_id":"job-2337","job_issued_at_ms":999973,"miner_id":"miner-5","network_difficulty":1000008,"ntime":1700002337,"order_key":"05:miner-5","p2mr_program_hex":"2121212121212121212121212121212121212121212121212121212121212121","share_difficulty":2,"share_id":"share-2337","share_seq":2337,"template_height":800073},{"accepted_at_ms":999994,"credit_policy":"stale-grace","job_id":"job-3195","job_issued_at_ms":999980,"miner_id":"miner-5","network_difficulty":1000016,"ntime":1700003195,"order_key":"05:miner-5","p2mr_program_hex":"7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b","share_difficulty":7,"share_id":"share-3195","share_seq":3195,"template_height":800099},{"accepted_at_ms":999984,"credit_policy":null,"job_id":"job-3270","job_issued_at_ms":999984,"miner_id":"miner-3","network_difficulty":1000006,"ntime":1700003270,"order_key":"03:miner-3","p2mr_program_hex":"c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6","share_difficulty":3,"share_id":"share-3270","share_seq":3270,"template_height":800102},{"accepted_at_ms":999984,"credit_policy":null,"job_id":"job-2118","job_issued_at_ms":999999,"miner_id":"miner-6","network_difficulty":1000010,"ntime":1700002118,"order_key":"06:miner-6","p2mr_program_hex":"4646464646464646464646464646464646464646464646464646464646464646","share_difficulty":1,"share_id":"share-2118","share_seq":2118,"template_height":800066},{"accepted_at_ms":999997,"credit_policy":null,"job_id":"job-3215","job_issued_at_ms":999988,"miner_id":"miner-3","network_difficulty":1000002,"ntime":1700003215,"order_key":"03:miner-3","p2mr_program_hex":"8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f","share_difficulty":19,"share_id":"share-3215","share_seq":3215,"template_height":800100},{"accepted_at_ms":999997,"credit_policy":"stale-grace","job_id":"job-3131","job_issued_at_ms":999979,"miner_id":"miner-7","network_difficulty":1000003,"ntime":1700003131,"order_key":"07:miner-7","p2mr_program_hex":"3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b","share_difficulty":3,"share_id":"share-3131","share_seq":3131,"template_height":800097},{"accepted_at_ms":999989,"credit_policy":"stale-grace","job_id":"job-1272","job_issued_at_ms":999970,"miner_id":"miner-7","network_difficulty":1000014,"ntime":1700001272,"order_key":"07:miner-7","p2mr_program_hex":"f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8","share_difficulty":1000,"share_id":"share-1272","share_seq":1272,"template_height":800039},{"accepted_at_ms":999991,"credit_policy":"stale-grace","job_id":"job-3323","job_issued_at_ms":999994,"miner_id":"miner-1","network_difficulty":1000008,"ntime":1700003323,"order_key":"01:miner-1","p2mr_program_hex":"fbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfb","share_difficulty":19,"share_id":"share-3323","share_seq":3323,"template_height":800103},{"accepted_at_ms":999983,"credit_policy":"stale-grace","job_id":"job-2972","job_issued_at_ms":999977,"miner_id":"miner-2","network_difficulty":1000014,"ntime":1700002972,"order_key":"02:miner-2","p2mr_program_hex":"9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c","share_difficulty":7,"share_id":"share-2972","share_seq":2972,"template_height":800092},{"accepted_at_ms":999997,"credit_policy":"stale-grace","job_id":"job-3620","job_issued_at_ms":999979,"miner_id":"miner-1","network_difficulty":1000016,"ntime":1700003620,"order_key":"01:miner-1","p2mr_program_hex":"2424242424242424242424242424242424242424242424242424242424242424","share_difficulty":7,"share_id":"share-3620","share_seq":3620,"template_height":800113},{"accepted_at_ms":999996,"credit_policy":null,"job_id":"job-1514","job_issued_at_ms":999980,"miner_id":"miner-7","network_difficulty":1000001,"ntime":1700001514,"order_key":"07:miner-7","p2mr_program_hex":"eaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaea","share_difficulty":1000,"share_id":"share-1514","share_seq":1514,"template_height":800047},{"accepted_at_ms":999986,"credit_policy":"stale-grace","job_id":"job-2902","job_issued_at_ms":999997,"miner_id":"miner-9","network_difficulty":1000012,"ntime":1700002902,"order_key":"09:miner-9","p2mr_program_hex":"5656565656565656565656565656565656565656565656565656565656565656","share_difficulty":101,"share_id":"share-2902","share_seq":2902,"template_height":800090}]},"window_weight":450000},"multi-page-interior-cutoff":{"advances":[],"name":"multi-page-interior-cutoff","page_size":512,"schema":"qbit-prism-window-pipeline-parity-input/v1","snapshot":{"anchor_job_issued_at_ms":90000,"records":[{"accepted_at_ms":89974,"credit_policy":null,"job_id":"job-830","job_issued_at_ms":89977,"miner_id":"miner-5","network_difficulty":1000014,"ntime":1700000830,"order_key":"05:miner-5","p2mr_program_hex":"3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e","share_difficulty":1,"share_id":"share-830","share_seq":830,"template_height":800025},{"accepted_at_ms":89995,"credit_policy":null,"job_id":"job-1615","job_issued_at_ms":89970,"miner_id":"miner-9","network_difficulty":1000000,"ntime":1700001615,"order_key":"09:miner-9","p2mr_program_hex":"4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f","share_difficulty":3,"share_id":"share-1615","share_seq":1615,"template_height":800050},{"accepted_at_ms":89978,"credit_policy":"stale-grace","job_id":"job-76","job_issued_at_ms":89997,"miner_id":"miner-10","network_difficulty":1000008,"ntime":1700000076,"order_key":"10:miner-10","p2mr_program_hex":"4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c","share_difficulty":3,"share_id":"share-76","share_seq":76,"template_height":800002},{"accepted_at_ms":89987,"credit_policy":null,"job_id":"job-736","job_issued_at_ms":89988,"miner_id":"miner-10","network_difficulty":1000005,"ntime":1700000736,"order_key":"10:miner-10","p2mr_program_hex":"e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0","share_difficulty":3,"share_id":"share-736","share_seq":736,"template_height":800023},{"accepted_at_ms":89974,"credit_policy":"stale-grace","job_id":"job-582","job_issued_at_ms":89988,"miner_id":"miner-10","network_difficulty":1000004,"ntime":1700000582,"order_key":"10:miner-10","p2mr_program_hex":"4646464646464646464646464646464646464646464646464646464646464646","share_difficulty":3,"share_id":"share-582","share_seq":582,"template_height":800018},{"accepted_at_ms":89998,"credit_policy":null,"job_id":"job-131","job_issued_at_ms":89977,"miner_id":"miner-10","network_difficulty":1000012,"ntime":1700000131,"order_key":"10:miner-10","p2mr_program_hex":"8383838383838383838383838383838383838383838383838383838383838383","share_difficulty":1,"share_id":"share-131","share_seq":131,"template_height":800004},{"accepted_at_ms":89974,"credit_policy":null,"job_id":"job-1017","job_issued_at_ms":89979,"miner_id":"miner-5","network_difficulty":1000014,"ntime":1700001017,"order_key":"05:miner-5","p2mr_program_hex":"f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9","share_difficulty":3,"share_id":"share-1017","share_seq":1017,"template_height":800031},{"accepted_at_ms":89994,"credit_policy":null,"job_id":"job-296","job_issued_at_ms":89971,"miner_id":"miner-10","network_difficulty":1000007,"ntime":1700000296,"order_key":"10:miner-10","p2mr_program_hex":"2828282828282828282828282828282828282828282828282828282828282828","share_difficulty":19,"share_id":"share-296","share_seq":296,"template_height":800009},{"accepted_at_ms":89994,"credit_policy":null,"job_id":"job-1303","job_issued_at_ms":89996,"miner_id":"miner-5","network_difficulty":1000011,"ntime":1700001303,"order_key":"05:miner-5","p2mr_program_hex":"1717171717171717171717171717171717171717171717171717171717171717","share_difficulty":2,"share_id":"share-1303","share_seq":1303,"template_height":800040},{"accepted_at_ms":89975,"credit_policy":"stale-grace","job_id":"job-572","job_issued_at_ms":89980,"miner_id":"miner-0","network_difficulty":1000011,"ntime":1700000572,"order_key":"00:miner-0","p2mr_program_hex":"3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c","share_difficulty":2,"share_id":"share-572","share_seq":572,"template_height":800017},{"accepted_at_ms":89999,"credit_policy":null,"job_id":"job-1057","job_issued_at_ms":89995,"miner_id":"miner-1","network_difficulty":1000003,"ntime":1700001057,"order_key":"01:miner-1","p2mr_program_hex":"2121212121212121212121212121212121212121212121212121212121212121","share_difficulty":2,"share_id":"share-1057","share_seq":1057,"template_height":800033},{"accepted_at_ms":89993,"credit_policy":null,"job_id":"job-1046","job_issued_at_ms":89983,"miner_id":"miner-1","network_difficulty":1000009,"ntime":1700001046,"order_key":"01:miner-1","p2mr_program_hex":"1616161616161616161616161616161616161616161616161616161616161616","share_difficulty":2,"share_id":"share-1046","share_seq":1046,"template_height":800032},{"accepted_at_ms":89984,"credit_policy":null,"job_id":"job-617","job_issued_at_ms":89972,"miner_id":"miner-1","network_difficulty":1000005,"ntime":1700000617,"order_key":"01:miner-1","p2mr_program_hex":"6969696969696969696969696969696969696969696969696969696969696969","share_difficulty":2,"share_id":"share-617","share_seq":617,"template_height":800019},{"accepted_at_ms":89976,"credit_policy":"stale-grace","job_id":"job-29","job_issued_at_ms":90000,"miner_id":"miner-7","network_difficulty":1000012,"ntime":1700000029,"order_key":"07:miner-7","p2mr_program_hex":"1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d","share_difficulty":1,"share_id":"share-29","share_seq":29,"template_height":800000},{"accepted_at_ms":89990,"credit_policy":null,"job_id":"job-1740","job_issued_at_ms":89997,"miner_id":"miner-2","network_difficulty":1000006,"ntime":1700001740,"order_key":"02:miner-2","p2mr_program_hex":"cccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccc","share_difficulty":19,"share_id":"share-1740","share_seq":1740,"template_height":800054},{"accepted_at_ms":89973,"credit_policy":"stale-grace","job_id":"job-793","job_issued_at_ms":89996,"miner_id":"miner-1","network_difficulty":1000011,"ntime":1700000793,"order_key":"01:miner-1","p2mr_program_hex":"1919191919191919191919191919191919191919191919191919191919191919","share_difficulty":2,"share_id":"share-793","share_seq":793,"template_height":800024},{"accepted_at_ms":89998,"credit_policy":null,"job_id":"job-1257","job_issued_at_ms":89990,"miner_id":"miner-3","network_difficulty":1000016,"ntime":1700001257,"order_key":"03:miner-3","p2mr_program_hex":"e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9","share_difficulty":19,"share_id":"share-1257","share_seq":1257,"template_height":800039},{"accepted_at_ms":89979,"credit_policy":null,"job_id":"job-919","job_issued_at_ms":89987,"miner_id":"miner-6","network_difficulty":1000001,"ntime":1700000919,"order_key":"06:miner-6","p2mr_program_hex":"9797979797979797979797979797979797979797979797979797979797979797","share_difficulty":3,"share_id":"share-919","share_seq":919,"template_height":800028},{"accepted_at_ms":89999,"credit_policy":null,"job_id":"job-714","job_issued_at_ms":89984,"miner_id":"miner-10","network_difficulty":1000000,"ntime":1700000714,"order_key":"10:miner-10","p2mr_program_hex":"cacacacacacacacacacacacacacacacacacacacacacacacacacacacacacacaca","share_difficulty":2,"share_id":"share-714","share_seq":714,"template_height":800022},{"accepted_at_ms":89993,"credit_policy":"stale-grace","job_id":"job-1162","job_issued_at_ms":89997,"miner_id":"miner-7","network_difficulty":1000006,"ntime":1700001162,"order_key":"07:miner-7","p2mr_program_hex":"8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a","share_difficulty":19,"share_id":"share-1162","share_seq":1162,"template_height":800036},{"accepted_at_ms":89989,"credit_policy":"stale-grace","job_id":"job-599","job_issued_at_ms":89981,"miner_id":"miner-5","network_difficulty":1000004,"ntime":1700000599,"order_key":"05:miner-5","p2mr_program_hex":"5757575757575757575757575757575757575757575757575757575757575757","share_difficulty":1,"share_id":"share-599","share_seq":599,"template_height":800018},{"accepted_at_ms":89997,"credit_policy":null,"job_id":"job-1206","job_issued_at_ms":89971,"miner_id":"miner-7","network_difficulty":1000016,"ntime":1700001206,"order_key":"07:miner-7","p2mr_program_hex":"b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6","share_difficulty":2,"share_id":"share-1206","share_seq":1206,"template_height":800037},{"accepted_at_ms":89989,"credit_policy":null,"job_id":"job-1338","job_issued_at_ms":89992,"miner_id":"miner-7","network_difficulty":1000012,"ntime":1700001338,"order_key":"07:miner-7","p2mr_program_hex":"3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a","share_difficulty":1,"share_id":"share-1338","share_seq":1338,"template_height":800041},{"accepted_at_ms":90000,"credit_policy":null,"job_id":"job-620","job_issued_at_ms":89994,"miner_id":"miner-4","network_difficulty":1000008,"ntime":1700000620,"order_key":"04:miner-4","p2mr_program_hex":"6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c","share_difficulty":1,"share_id":"share-620","share_seq":620,"template_height":800019},{"accepted_at_ms":89990,"credit_policy":"stale-grace","job_id":"job-670","job_issued_at_ms":89984,"miner_id":"miner-10","network_difficulty":1000007,"ntime":1700000670,"order_key":"10:miner-10","p2mr_program_hex":"9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e","share_difficulty":3,"share_id":"share-670","share_seq":670,"template_height":800020},{"accepted_at_ms":89981,"credit_policy":"stale-grace","job_id":"job-1385","job_issued_at_ms":89982,"miner_id":"miner-10","network_difficulty":1000008,"ntime":1700001385,"order_key":"10:miner-10","p2mr_program_hex":"6969696969696969696969696969696969696969696969696969696969696969","share_difficulty":7,"share_id":"share-1385","share_seq":1385,"template_height":800043},{"accepted_at_ms":89991,"credit_policy":"stale-grace","job_id":"job-472","job_issued_at_ms":89994,"miner_id":"miner-10","network_difficulty":1000013,"ntime":1700000472,"order_key":"10:miner-10","p2mr_program_hex":"d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8","share_difficulty":7,"share_id":"share-472","share_seq":472,"template_height":800014},{"accepted_at_ms":89995,"credit_policy":null,"job_id":"job-799","job_issued_at_ms":89994,"miner_id":"miner-7","network_difficulty":1000000,"ntime":1700000799,"order_key":"07:miner-7","p2mr_program_hex":"1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f","share_difficulty":7,"share_id":"share-799","share_seq":799,"template_height":800024},{"accepted_at_ms":89975,"credit_policy":null,"job_id":"job-1614","job_issued_at_ms":89996,"miner_id":"miner-8","network_difficulty":1000016,"ntime":1700001614,"order_key":"08:miner-8","p2mr_program_hex":"4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e","share_difficulty":3,"share_id":"share-1614","share_seq":1614,"template_height":800050},{"accepted_at_ms":89978,"credit_policy":null,"job_id":"job-1025","job_issued_at_ms":89972,"miner_id":"miner-2","network_difficulty":1000005,"ntime":1700001025,"order_key":"02:miner-2","p2mr_program_hex":"0101010101010101010101010101010101010101010101010101010101010101","share_difficulty":2,"share_id":"share-1025","share_seq":1025,"template_height":800032},{"accepted_at_ms":89973,"credit_policy":null,"job_id":"job-175","job_issued_at_ms":90000,"miner_id":"miner-10","network_difficulty":1000005,"ntime":1700000175,"order_key":"10:miner-10","p2mr_program_hex":"afafafafafafafafafafafafafafafafafafafafafafafafafafafafafafafaf","share_difficulty":2,"share_id":"share-175","share_seq":175,"template_height":800005},{"accepted_at_ms":89984,"credit_policy":null,"job_id":"job-865","job_issued_at_ms":89994,"miner_id":"miner-7","network_difficulty":1000015,"ntime":1700000865,"order_key":"07:miner-7","p2mr_program_hex":"6161616161616161616161616161616161616161616161616161616161616161","share_difficulty":3,"share_id":"share-865","share_seq":865,"template_height":800027},{"accepted_at_ms":89978,"credit_policy":null,"job_id":"job-1640","job_issued_at_ms":89998,"miner_id":"miner-1","network_difficulty":1000008,"ntime":1700001640,"order_key":"01:miner-1","p2mr_program_hex":"6868686868686868686868686868686868686868686868686868686868686868","share_difficulty":2,"share_id":"share-1640","share_seq":1640,"template_height":800051},{"accepted_at_ms":89998,"credit_policy":null,"job_id":"job-245","job_issued_at_ms":89994,"miner_id":"miner-3","network_difficulty":1000007,"ntime":1700000245,"order_key":"03:miner-3","p2mr_program_hex":"f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5","share_difficulty":2,"share_id":"share-245","share_seq":245,"template_height":800007},{"accepted_at_ms":89988,"credit_policy":null,"job_id":"job-867","job_issued_at_ms":89988,"miner_id":"miner-9","network_difficulty":1000000,"ntime":1700000867,"order_key":"09:miner-9","p2mr_program_hex":"6363636363636363636363636363636363636363636363636363636363636363","share_difficulty":2,"share_id":"share-867","share_seq":867,"template_height":800027},{"accepted_at_ms":89970,"credit_policy":null,"job_id":"job-187","job_issued_at_ms":89992,"miner_id":"miner-0","network_difficulty":1000000,"ntime":1700000187,"order_key":"00:miner-0","p2mr_program_hex":"bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb","share_difficulty":2,"share_id":"share-187","share_seq":187,"template_height":800005},{"accepted_at_ms":89998,"credit_policy":"stale-grace","job_id":"job-1751","job_issued_at_ms":90000,"miner_id":"miner-2","network_difficulty":1000000,"ntime":1700001751,"order_key":"02:miner-2","p2mr_program_hex":"d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7","share_difficulty":2,"share_id":"share-1751","share_seq":1751,"template_height":800054},{"accepted_at_ms":89973,"credit_policy":null,"job_id":"job-798","job_issued_at_ms":89980,"miner_id":"miner-6","network_difficulty":1000016,"ntime":1700000798,"order_key":"06:miner-6","p2mr_program_hex":"1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e","share_difficulty":3,"share_id":"share-798","share_seq":798,"template_height":800024},{"accepted_at_ms":89978,"credit_policy":null,"job_id":"job-820","job_issued_at_ms":89981,"miner_id":"miner-6","network_difficulty":1000004,"ntime":1700000820,"order_key":"06:miner-6","p2mr_program_hex":"3434343434343434343434343434343434343434343434343434343434343434","share_difficulty":2,"share_id":"share-820","share_seq":820,"template_height":800025},{"accepted_at_ms":89974,"credit_policy":null,"job_id":"job-1007","job_issued_at_ms":89989,"miner_id":"miner-6","network_difficulty":1000004,"ntime":1700001007,"order_key":"06:miner-6","p2mr_program_hex":"efefefefefefefefefefefefefefefefefefefefefefefefefefefefefefefef","share_difficulty":2,"share_id":"share-1007","share_seq":1007,"template_height":800031},{"accepted_at_ms":89988,"credit_policy":"stale-grace","job_id":"job-1733","job_issued_at_ms":89992,"miner_id":"miner-6","network_difficulty":1000016,"ntime":1700001733,"order_key":"06:miner-6","p2mr_program_hex":"c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5","share_difficulty":2,"share_id":"share-1733","share_seq":1733,"template_height":800054},{"accepted_at_ms":89984,"credit_policy":"stale-grace","job_id":"job-312","job_issued_at_ms":89972,"miner_id":"miner-4","network_difficulty":1000006,"ntime":1700000312,"order_key":"04:miner-4","p2mr_program_hex":"3838383838383838383838383838383838383838383838383838383838383838","share_difficulty":19,"share_id":"share-312","share_seq":312,"template_height":800009},{"accepted_at_ms":89980,"credit_policy":"stale-grace","job_id":"job-1246","job_issued_at_ms":89998,"miner_id":"miner-3","network_difficulty":1000005,"ntime":1700001246,"order_key":"03:miner-3","p2mr_program_hex":"dededededededededededededededededededededededededededededededede","share_difficulty":1,"share_id":"share-1246","share_seq":1246,"template_height":800038},{"accepted_at_ms":89989,"credit_policy":null,"job_id":"job-755","job_issued_at_ms":90000,"miner_id":"miner-7","network_difficulty":1000007,"ntime":1700000755,"order_key":"07:miner-7","p2mr_program_hex":"f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3","share_difficulty":7,"share_id":"share-755","share_seq":755,"template_height":800023},{"accepted_at_ms":89987,"credit_policy":null,"job_id":"job-1161","job_issued_at_ms":89985,"miner_id":"miner-6","network_difficulty":1000005,"ntime":1700001161,"order_key":"06:miner-6","p2mr_program_hex":"8989898989898989898989898989898989898989898989898989898989898989","share_difficulty":1,"share_id":"share-1161","share_seq":1161,"template_height":800036},{"accepted_at_ms":89999,"credit_policy":null,"job_id":"job-1658","job_issued_at_ms":89984,"miner_id":"miner-8","network_difficulty":1000009,"ntime":1700001658,"order_key":"08:miner-8","p2mr_program_hex":"7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a","share_difficulty":3,"share_id":"share-1658","share_seq":1658,"template_height":800051},{"accepted_at_ms":89996,"credit_policy":"stale-grace","job_id":"job-1348","job_issued_at_ms":89989,"miner_id":"miner-6","network_difficulty":1000005,"ntime":1700001348,"order_key":"06:miner-6","p2mr_program_hex":"4444444444444444444444444444444444444444444444444444444444444444","share_difficulty":7,"share_id":"share-1348","share_seq":1348,"template_height":800042},{"accepted_at_ms":89998,"credit_policy":"stale-grace","job_id":"job-1048","job_issued_at_ms":89998,"miner_id":"miner-3","network_difficulty":1000011,"ntime":1700001048,"order_key":"03:miner-3","p2mr_program_hex":"1818181818181818181818181818181818181818181818181818181818181818","share_difficulty":19,"share_id":"share-1048","share_seq":1048,"template_height":800032},{"accepted_at_ms":89982,"credit_policy":null,"job_id":"job-1666","job_issued_at_ms":90000,"miner_id":"miner-5","network_difficulty":1000000,"ntime":1700001666,"order_key":"05:miner-5","p2mr_program_hex":"8282828282828282828282828282828282828282828282828282828282828282","share_difficulty":19,"share_id":"share-1666","share_seq":1666,"template_height":800052},{"accepted_at_ms":89981,"credit_policy":null,"job_id":"job-1047","job_issued_at_ms":89986,"miner_id":"miner-2","network_difficulty":1000010,"ntime":1700001047,"order_key":"02:miner-2","p2mr_program_hex":"1717171717171717171717171717171717171717171717171717171717171717","share_difficulty":3,"share_id":"share-1047","share_seq":1047,"template_height":800032},{"accepted_at_ms":89972,"credit_policy":null,"job_id":"job-935","job_issued_at_ms":89979,"miner_id":"miner-0","network_difficulty":1000000,"ntime":1700000935,"order_key":"00:miner-0","p2mr_program_hex":"a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7","share_difficulty":3,"share_id":"share-935","share_seq":935,"template_height":800029},{"accepted_at_ms":89975,"credit_policy":null,"job_id":"job-858","job_issued_at_ms":89981,"miner_id":"miner-0","network_difficulty":1000008,"ntime":1700000858,"order_key":"00:miner-0","p2mr_program_hex":"5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a","share_difficulty":7,"share_id":"share-858","share_seq":858,"template_height":800026},{"accepted_at_ms":89973,"credit_policy":"stale-grace","job_id":"job-1157","job_issued_at_ms":89979,"miner_id":"miner-2","network_difficulty":1000001,"ntime":1700001157,"order_key":"02:miner-2","p2mr_program_hex":"8585858585858585858585858585858585858585858585858585858585858585","share_difficulty":3,"share_id":"share-1157","share_seq":1157,"template_height":800036},{"accepted_at_ms":89982,"credit_policy":null,"job_id":"job-1487","job_issued_at_ms":89978,"miner_id":"miner-2","network_difficulty":1000008,"ntime":1700001487,"order_key":"02:miner-2","p2mr_program_hex":"cfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcf","share_difficulty":3,"share_id":"share-1487","share_seq":1487,"template_height":800046},{"accepted_at_ms":89992,"credit_policy":null,"job_id":"job-937","job_issued_at_ms":89981,"miner_id":"miner-2","network_difficulty":1000002,"ntime":1700000937,"order_key":"02:miner-2","p2mr_program_hex":"a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9","share_difficulty":7,"share_id":"share-937","share_seq":937,"template_height":800029},{"accepted_at_ms":89972,"credit_policy":null,"job_id":"job-657","job_issued_at_ms":89979,"miner_id":"miner-8","network_difficulty":1000011,"ntime":1700000657,"order_key":"08:miner-8","p2mr_program_hex":"9191919191919191919191919191919191919191919191919191919191919191","share_difficulty":7,"share_id":"share-657","share_seq":657,"template_height":800020},{"accepted_at_ms":89981,"credit_policy":null,"job_id":"job-246","job_issued_at_ms":89972,"miner_id":"miner-4","network_difficulty":1000008,"ntime":1700000246,"order_key":"04:miner-4","p2mr_program_hex":"f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6","share_difficulty":19,"share_id":"share-246","share_seq":246,"template_height":800007},{"accepted_at_ms":89979,"credit_policy":null,"job_id":"job-232","job_issued_at_ms":89998,"miner_id":"miner-1","network_difficulty":1000011,"ntime":1700000232,"order_key":"01:miner-1","p2mr_program_hex":"e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8","share_difficulty":1,"share_id":"share-232","share_seq":232,"template_height":800007},{"accepted_at_ms":89999,"credit_policy":"stale-grace","job_id":"job-1282","job_issued_at_ms":89976,"miner_id":"miner-6","network_difficulty":1000007,"ntime":1700001282,"order_key":"06:miner-6","p2mr_program_hex":"0202020202020202020202020202020202020202020202020202020202020202","share_difficulty":1,"share_id":"share-1282","share_seq":1282,"template_height":800040},{"accepted_at_ms":89972,"credit_policy":null,"job_id":"job-112","job_issued_at_ms":89980,"miner_id":"miner-2","network_difficulty":1000010,"ntime":1700000112,"order_key":"02:miner-2","p2mr_program_hex":"7070707070707070707070707070707070707070707070707070707070707070","share_difficulty":2,"share_id":"share-112","share_seq":112,"template_height":800003},{"accepted_at_ms":89984,"credit_policy":null,"job_id":"job-37","job_issued_at_ms":89998,"miner_id":"miner-4","network_difficulty":1000003,"ntime":1700000037,"order_key":"04:miner-4","p2mr_program_hex":"2525252525252525252525252525252525252525252525252525252525252525","share_difficulty":19,"share_id":"share-37","share_seq":37,"template_height":800001},{"accepted_at_ms":89971,"credit_policy":null,"job_id":"job-734","job_issued_at_ms":89995,"miner_id":"miner-8","network_difficulty":1000003,"ntime":1700000734,"order_key":"08:miner-8","p2mr_program_hex":"dededededededededededededededededededededededededededededededede","share_difficulty":2,"share_id":"share-734","share_seq":734,"template_height":800022},{"accepted_at_ms":89985,"credit_policy":"stale-grace","job_id":"job-127","job_issued_at_ms":89994,"miner_id":"miner-6","network_difficulty":1000008,"ntime":1700000127,"order_key":"06:miner-6","p2mr_program_hex":"7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f","share_difficulty":3,"share_id":"share-127","share_seq":127,"template_height":800003},{"accepted_at_ms":89972,"credit_policy":"stale-grace","job_id":"job-896","job_issued_at_ms":89987,"miner_id":"miner-5","network_difficulty":1000012,"ntime":1700000896,"order_key":"05:miner-5","p2mr_program_hex":"8080808080808080808080808080808080808080808080808080808080808080","share_difficulty":2,"share_id":"share-896","share_seq":896,"template_height":800028},{"accepted_at_ms":89980,"credit_policy":null,"job_id":"job-1386","job_issued_at_ms":90000,"miner_id":"miner-0","network_difficulty":1000009,"ntime":1700001386,"order_key":"00:miner-0","p2mr_program_hex":"6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a","share_difficulty":1,"share_id":"share-1386","share_seq":1386,"template_height":800043},{"accepted_at_ms":89990,"credit_policy":null,"job_id":"job-1732","job_issued_at_ms":89996,"miner_id":"miner-5","network_difficulty":1000015,"ntime":1700001732,"order_key":"05:miner-5","p2mr_program_hex":"c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4","share_difficulty":7,"share_id":"share-1732","share_seq":1732,"template_height":800054},{"accepted_at_ms":89992,"credit_policy":null,"job_id":"job-476","job_issued_at_ms":89981,"miner_id":"miner-3","network_difficulty":1000000,"ntime":1700000476,"order_key":"03:miner-3","p2mr_program_hex":"dcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdc","share_difficulty":3,"share_id":"share-476","share_seq":476,"template_height":800014},{"accepted_at_ms":89978,"credit_policy":null,"job_id":"job-613","job_issued_at_ms":89986,"miner_id":"miner-8","network_difficulty":1000001,"ntime":1700000613,"order_key":"08:miner-8","p2mr_program_hex":"6565656565656565656565656565656565656565656565656565656565656565","share_difficulty":3,"share_id":"share-613","share_seq":613,"template_height":800019},{"accepted_at_ms":89977,"credit_policy":null,"job_id":"job-1653","job_issued_at_ms":89981,"miner_id":"miner-3","network_difficulty":1000004,"ntime":1700001653,"order_key":"03:miner-3","p2mr_program_hex":"7575757575757575757575757575757575757575757575757575757575757575","share_difficulty":3,"share_id":"share-1653","share_seq":1653,"template_height":800051},{"accepted_at_ms":89997,"credit_policy":null,"job_id":"job-906","job_issued_at_ms":89975,"miner_id":"miner-4","network_difficulty":1000005,"ntime":1700000906,"order_key":"04:miner-4","p2mr_program_hex":"8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a","share_difficulty":3,"share_id":"share-906","share_seq":906,"template_height":800028},{"accepted_at_ms":89999,"credit_policy":null,"job_id":"job-377","job_issued_at_ms":89972,"miner_id":"miner-3","network_difficulty":1000003,"ntime":1700000377,"order_key":"03:miner-3","p2mr_program_hex":"7979797979797979797979797979797979797979797979797979797979797979","share_difficulty":19,"share_id":"share-377","share_seq":377,"template_height":800011},{"accepted_at_ms":89988,"credit_policy":null,"job_id":"job-1753","job_issued_at_ms":89992,"miner_id":"miner-4","network_difficulty":1000002,"ntime":1700001753,"order_key":"04:miner-4","p2mr_program_hex":"d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9","share_difficulty":19,"share_id":"share-1753","share_seq":1753,"template_height":800054},{"accepted_at_ms":89983,"credit_policy":null,"job_id":"job-1793","job_issued_at_ms":89986,"miner_id":"miner-0","network_difficulty":1000008,"ntime":1700001793,"order_key":"00:miner-0","p2mr_program_hex":"0101010101010101010101010101010101010101010101010101010101010101","share_difficulty":3,"share_id":"share-1793","share_seq":1793,"template_height":800056},{"accepted_at_ms":89996,"credit_policy":null,"job_id":"job-247","job_issued_at_ms":89984,"miner_id":"miner-5","network_difficulty":1000009,"ntime":1700000247,"order_key":"05:miner-5","p2mr_program_hex":"f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7","share_difficulty":19,"share_id":"share-247","share_seq":247,"template_height":800007},{"accepted_at_ms":89990,"credit_policy":null,"job_id":"job-1065","job_issued_at_ms":89971,"miner_id":"miner-9","network_difficulty":1000011,"ntime":1700001065,"order_key":"09:miner-9","p2mr_program_hex":"2929292929292929292929292929292929292929292929292929292929292929","share_difficulty":7,"share_id":"share-1065","share_seq":1065,"template_height":800033},{"accepted_at_ms":89978,"credit_policy":null,"job_id":"job-356","job_issued_at_ms":89983,"miner_id":"miner-4","network_difficulty":1000016,"ntime":1700000356,"order_key":"04:miner-4","p2mr_program_hex":"6464646464646464646464646464646464646464646464646464646464646464","share_difficulty":7,"share_id":"share-356","share_seq":356,"template_height":800011},{"accepted_at_ms":89974,"credit_policy":"stale-grace","job_id":"job-267","job_issued_at_ms":89995,"miner_id":"miner-3","network_difficulty":1000012,"ntime":1700000267,"order_key":"03:miner-3","p2mr_program_hex":"0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b","share_difficulty":2,"share_id":"share-267","share_seq":267,"template_height":800008},{"accepted_at_ms":89983,"credit_policy":null,"job_id":"job-759","job_issued_at_ms":89972,"miner_id":"miner-0","network_difficulty":1000011,"ntime":1700000759,"order_key":"00:miner-0","p2mr_program_hex":"f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7","share_difficulty":2,"share_id":"share-759","share_seq":759,"template_height":800023},{"accepted_at_ms":89970,"credit_policy":null,"job_id":"job-1639","job_issued_at_ms":89984,"miner_id":"miner-0","network_difficulty":1000007,"ntime":1700001639,"order_key":"00:miner-0","p2mr_program_hex":"6767676767676767676767676767676767676767676767676767676767676767","share_difficulty":1,"share_id":"share-1639","share_seq":1639,"template_height":800051},{"accepted_at_ms":89970,"credit_policy":null,"job_id":"job-696","job_issued_at_ms":89986,"miner_id":"miner-3","network_difficulty":1000016,"ntime":1700000696,"order_key":"03:miner-3","p2mr_program_hex":"b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8","share_difficulty":19,"share_id":"share-696","share_seq":696,"template_height":800021},{"accepted_at_ms":89970,"credit_policy":"stale-grace","job_id":"job-975","job_issued_at_ms":89994,"miner_id":"miner-7","network_difficulty":1000006,"ntime":1700000975,"order_key":"07:miner-7","p2mr_program_hex":"cfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcf","share_difficulty":2,"share_id":"share-975","share_seq":975,"template_height":800030},{"accepted_at_ms":89989,"credit_policy":null,"job_id":"job-761","job_issued_at_ms":89992,"miner_id":"miner-2","network_difficulty":1000013,"ntime":1700000761,"order_key":"02:miner-2","p2mr_program_hex":"f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9","share_difficulty":2,"share_id":"share-761","share_seq":761,"template_height":800023},{"accepted_at_ms":89972,"credit_policy":null,"job_id":"job-32","job_issued_at_ms":89990,"miner_id":"miner-10","network_difficulty":1000015,"ntime":1700000032,"order_key":"10:miner-10","p2mr_program_hex":"2020202020202020202020202020202020202020202020202020202020202020","share_difficulty":3,"share_id":"share-32","share_seq":32,"template_height":800001},{"accepted_at_ms":89979,"credit_policy":null,"job_id":"job-1599","job_issued_at_ms":89977,"miner_id":"miner-4","network_difficulty":1000001,"ntime":1700001599,"order_key":"04:miner-4","p2mr_program_hex":"3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f","share_difficulty":2,"share_id":"share-1599","share_seq":1599,"template_height":800049},{"accepted_at_ms":89978,"credit_policy":"stale-grace","job_id":"job-1294","job_issued_at_ms":89989,"miner_id":"miner-7","network_difficulty":1000002,"ntime":1700001294,"order_key":"07:miner-7","p2mr_program_hex":"0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e","share_difficulty":3,"share_id":"share-1294","share_seq":1294,"template_height":800040},{"accepted_at_ms":89983,"credit_policy":null,"job_id":"job-1570","job_issued_at_ms":89999,"miner_id":"miner-8","network_difficulty":1000006,"ntime":1700001570,"order_key":"08:miner-8","p2mr_program_hex":"2222222222222222222222222222222222222222222222222222222222222222","share_difficulty":19,"share_id":"share-1570","share_seq":1570,"template_height":800049},{"accepted_at_ms":90000,"credit_policy":"stale-grace","job_id":"job-829","job_issued_at_ms":89991,"miner_id":"miner-4","network_difficulty":1000013,"ntime":1700000829,"order_key":"04:miner-4","p2mr_program_hex":"3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d","share_difficulty":19,"share_id":"share-829","share_seq":829,"template_height":800025},{"accepted_at_ms":89980,"credit_policy":"stale-grace","job_id":"job-1674","job_issued_at_ms":89985,"miner_id":"miner-2","network_difficulty":1000008,"ntime":1700001674,"order_key":"02:miner-2","p2mr_program_hex":"8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a","share_difficulty":19,"share_id":"share-1674","share_seq":1674,"template_height":800052},{"accepted_at_ms":89973,"credit_policy":"stale-grace","job_id":"job-368","job_issued_at_ms":89979,"miner_id":"miner-5","network_difficulty":1000011,"ntime":1700000368,"order_key":"05:miner-5","p2mr_program_hex":"7070707070707070707070707070707070707070707070707070707070707070","share_difficulty":19,"share_id":"share-368","share_seq":368,"template_height":800011},{"accepted_at_ms":89986,"credit_policy":null,"job_id":"job-1253","job_issued_at_ms":89973,"miner_id":"miner-10","network_difficulty":1000012,"ntime":1700001253,"order_key":"10:miner-10","p2mr_program_hex":"e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5","share_difficulty":1,"share_id":"share-1253","share_seq":1253,"template_height":800039},{"accepted_at_ms":89992,"credit_policy":null,"job_id":"job-178","job_issued_at_ms":89984,"miner_id":"miner-2","network_difficulty":1000008,"ntime":1700000178,"order_key":"02:miner-2","p2mr_program_hex":"b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2","share_difficulty":7,"share_id":"share-178","share_seq":178,"template_height":800005},{"accepted_at_ms":89991,"credit_policy":null,"job_id":"job-899","job_issued_at_ms":89983,"miner_id":"miner-8","network_difficulty":1000015,"ntime":1700000899,"order_key":"08:miner-8","p2mr_program_hex":"8383838383838383838383838383838383838383838383838383838383838383","share_difficulty":2,"share_id":"share-899","share_seq":899,"template_height":800028},{"accepted_at_ms":89973,"credit_policy":null,"job_id":"job-1714","job_issued_at_ms":89973,"miner_id":"miner-9","network_difficulty":1000014,"ntime":1700001714,"order_key":"09:miner-9","p2mr_program_hex":"b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2","share_difficulty":19,"share_id":"share-1714","share_seq":1714,"template_height":800053},{"accepted_at_ms":89975,"credit_policy":null,"job_id":"job-1073","job_issued_at_ms":89982,"miner_id":"miner-6","network_difficulty":1000002,"ntime":1700001073,"order_key":"06:miner-6","p2mr_program_hex":"3131313131313131313131313131313131313131313131313131313131313131","share_difficulty":1,"share_id":"share-1073","share_seq":1073,"template_height":800033},{"accepted_at_ms":89998,"credit_policy":"stale-grace","job_id":"job-50","job_issued_at_ms":89993,"miner_id":"miner-6","network_difficulty":1000016,"ntime":1700000050,"order_key":"06:miner-6","p2mr_program_hex":"3232323232323232323232323232323232323232323232323232323232323232","share_difficulty":19,"share_id":"share-50","share_seq":50,"template_height":800001},{"accepted_at_ms":89998,"credit_policy":null,"job_id":"job-256","job_issued_at_ms":89983,"miner_id":"miner-3","network_difficulty":1000001,"ntime":1700000256,"order_key":"03:miner-3","p2mr_program_hex":"0000000000000000000000000000000000000000000000000000000000000000","share_difficulty":19,"share_id":"share-256","share_seq":256,"template_height":800008},{"accepted_at_ms":89995,"credit_policy":null,"job_id":"job-1148","job_issued_at_ms":90000,"miner_id":"miner-4","network_difficulty":1000009,"ntime":1700001148,"order_key":"04:miner-4","p2mr_program_hex":"7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c","share_difficulty":2,"share_id":"share-1148","share_seq":1148,"template_height":800035},{"accepted_at_ms":89996,"credit_policy":null,"job_id":"job-1247","job_issued_at_ms":89990,"miner_id":"miner-4","network_difficulty":1000006,"ntime":1700001247,"order_key":"04:miner-4","p2mr_program_hex":"dfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdf","share_difficulty":3,"share_id":"share-1247","share_seq":1247,"template_height":800038},{"accepted_at_ms":89971,"credit_policy":"stale-grace","job_id":"job-706","job_issued_at_ms":89985,"miner_id":"miner-2","network_difficulty":1000009,"ntime":1700000706,"order_key":"02:miner-2","p2mr_program_hex":"c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2","share_difficulty":3,"share_id":"share-706","share_seq":706,"template_height":800022},{"accepted_at_ms":89976,"credit_policy":null,"job_id":"job-1786","job_issued_at_ms":89986,"miner_id":"miner-4","network_difficulty":1000001,"ntime":1700001786,"order_key":"04:miner-4","p2mr_program_hex":"fafafafafafafafafafafafafafafafafafafafafafafafafafafafafafafafa","share_difficulty":2,"share_id":"share-1786","share_seq":1786,"template_height":800055},{"accepted_at_ms":89972,"credit_policy":null,"job_id":"job-427","job_issued_at_ms":89978,"miner_id":"miner-9","network_difficulty":1000002,"ntime":1700000427,"order_key":"09:miner-9","p2mr_program_hex":"abababababababababababababababababababababababababababababababab","share_difficulty":7,"share_id":"share-427","share_seq":427,"template_height":800013},{"accepted_at_ms":89976,"credit_policy":null,"job_id":"job-575","job_issued_at_ms":89993,"miner_id":"miner-3","network_difficulty":1000014,"ntime":1700000575,"order_key":"03:miner-3","p2mr_program_hex":"3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f","share_difficulty":2,"share_id":"share-575","share_seq":575,"template_height":800017},{"accepted_at_ms":89970,"credit_policy":null,"job_id":"job-31","job_issued_at_ms":89988,"miner_id":"miner-9","network_difficulty":1000014,"ntime":1700000031,"order_key":"09:miner-9","p2mr_program_hex":"1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f","share_difficulty":1,"share_id":"share-31","share_seq":31,"template_height":800000},{"accepted_at_ms":89990,"credit_policy":null,"job_id":"job-249","job_issued_at_ms":89971,"miner_id":"miner-7","network_difficulty":1000011,"ntime":1700000249,"order_key":"07:miner-7","p2mr_program_hex":"f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9","share_difficulty":2,"share_id":"share-249","share_seq":249,"template_height":800007},{"accepted_at_ms":89987,"credit_policy":null,"job_id":"job-466","job_issued_at_ms":89985,"miner_id":"miner-4","network_difficulty":1000007,"ntime":1700000466,"order_key":"04:miner-4","p2mr_program_hex":"d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2","share_difficulty":19,"share_id":"share-466","share_seq":466,"template_height":800014},{"accepted_at_ms":89975,"credit_policy":null,"job_id":"job-1072","job_issued_at_ms":89976,"miner_id":"miner-5","network_difficulty":1000001,"ntime":1700001072,"order_key":"05:miner-5","p2mr_program_hex":"3030303030303030303030303030303030303030303030303030303030303030","share_difficulty":7,"share_id":"share-1072","share_seq":1072,"template_height":800033},{"accepted_at_ms":89975,"credit_policy":null,"job_id":"job-550","job_issued_at_ms":89971,"miner_id":"miner-0","network_difficulty":1000006,"ntime":1700000550,"order_key":"00:miner-0","p2mr_program_hex":"2626262626262626262626262626262626262626262626262626262626262626","share_difficulty":3,"share_id":"share-550","share_seq":550,"template_height":800017},{"accepted_at_ms":89995,"credit_policy":"stale-grace","job_id":"job-470","job_issued_at_ms":89976,"miner_id":"miner-8","network_difficulty":1000011,"ntime":1700000470,"order_key":"08:miner-8","p2mr_program_hex":"d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6","share_difficulty":2,"share_id":"share-470","share_seq":470,"template_height":800014},{"accepted_at_ms":89979,"credit_policy":null,"job_id":"job-673","job_issued_at_ms":89989,"miner_id":"miner-2","network_difficulty":1000010,"ntime":1700000673,"order_key":"02:miner-2","p2mr_program_hex":"a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1","share_difficulty":2,"share_id":"share-673","share_seq":673,"template_height":800021},{"accepted_at_ms":89984,"credit_policy":null,"job_id":"job-200","job_issued_at_ms":89993,"miner_id":"miner-2","network_difficulty":1000013,"ntime":1700000200,"order_key":"02:miner-2","p2mr_program_hex":"c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8","share_difficulty":2,"share_id":"share-200","share_seq":200,"template_height":800006},{"accepted_at_ms":89978,"credit_policy":null,"job_id":"job-849","job_issued_at_ms":89998,"miner_id":"miner-2","network_difficulty":1000016,"ntime":1700000849,"order_key":"02:miner-2","p2mr_program_hex":"5151515151515151515151515151515151515151515151515151515151515151","share_difficulty":1,"share_id":"share-849","share_seq":849,"template_height":800026},{"accepted_at_ms":89984,"credit_policy":null,"job_id":"job-811","job_issued_at_ms":89970,"miner_id":"miner-8","network_difficulty":1000012,"ntime":1700000811,"order_key":"08:miner-8","p2mr_program_hex":"2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b","share_difficulty":3,"share_id":"share-811","share_seq":811,"template_height":800025},{"accepted_at_ms":89991,"credit_policy":null,"job_id":"job-1604","job_issued_at_ms":89991,"miner_id":"miner-9","network_difficulty":1000006,"ntime":1700001604,"order_key":"09:miner-9","p2mr_program_hex":"4444444444444444444444444444444444444444444444444444444444444444","share_difficulty":1,"share_id":"share-1604","share_seq":1604,"template_height":800050},{"accepted_at_ms":89970,"credit_policy":null,"job_id":"job-394","job_issued_at_ms":89970,"miner_id":"miner-9","network_difficulty":1000003,"ntime":1700000394,"order_key":"09:miner-9","p2mr_program_hex":"8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a","share_difficulty":2,"share_id":"share-394","share_seq":394,"template_height":800012},{"accepted_at_ms":89997,"credit_policy":null,"job_id":"job-115","job_issued_at_ms":89987,"miner_id":"miner-5","network_difficulty":1000013,"ntime":1700000115,"order_key":"05:miner-5","p2mr_program_hex":"7373737373737373737373737373737373737373737373737373737373737373","share_difficulty":7,"share_id":"share-115","share_seq":115,"template_height":800003},{"accepted_at_ms":89975,"credit_policy":null,"job_id":"job-339","job_issued_at_ms":89970,"miner_id":"miner-9","network_difficulty":1000016,"ntime":1700000339,"order_key":"09:miner-9","p2mr_program_hex":"5353535353535353535353535353535353535353535353535353535353535353","share_difficulty":2,"share_id":"share-339","share_seq":339,"template_height":800010},{"accepted_at_ms":89998,"credit_policy":null,"job_id":"job-568","job_issued_at_ms":89995,"miner_id":"miner-7","network_difficulty":1000007,"ntime":1700000568,"order_key":"07:miner-7","p2mr_program_hex":"3838383838383838383838383838383838383838383838383838383838383838","share_difficulty":3,"share_id":"share-568","share_seq":568,"template_height":800017},{"accepted_at_ms":89999,"credit_policy":"stale-grace","job_id":"job-1336","job_issued_at_ms":89984,"miner_id":"miner-5","network_difficulty":1000010,"ntime":1700001336,"order_key":"05:miner-5","p2mr_program_hex":"3838383838383838383838383838383838383838383838383838383838383838","share_difficulty":2,"share_id":"share-1336","share_seq":1336,"template_height":800041},{"accepted_at_ms":89998,"credit_policy":null,"job_id":"job-46","job_issued_at_ms":89973,"miner_id":"miner-2","network_difficulty":1000012,"ntime":1700000046,"order_key":"02:miner-2","p2mr_program_hex":"2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e","share_difficulty":19,"share_id":"share-46","share_seq":46,"template_height":800001},{"accepted_at_ms":89971,"credit_policy":null,"job_id":"job-675","job_issued_at_ms":89979,"miner_id":"miner-4","network_difficulty":1000012,"ntime":1700000675,"order_key":"04:miner-4","p2mr_program_hex":"a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3","share_difficulty":7,"share_id":"share-675","share_seq":675,"template_height":800021},{"accepted_at_ms":89997,"credit_policy":null,"job_id":"job-791","job_issued_at_ms":89980,"miner_id":"miner-10","network_difficulty":1000009,"ntime":1700000791,"order_key":"10:miner-10","p2mr_program_hex":"1717171717171717171717171717171717171717171717171717171717171717","share_difficulty":1,"share_id":"share-791","share_seq":791,"template_height":800024},{"accepted_at_ms":89997,"credit_policy":null,"job_id":"job-756","job_issued_at_ms":89984,"miner_id":"miner-8","network_difficulty":1000008,"ntime":1700000756,"order_key":"08:miner-8","p2mr_program_hex":"f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4","share_difficulty":1,"share_id":"share-756","share_seq":756,"template_height":800023},{"accepted_at_ms":89976,"credit_policy":"stale-grace","job_id":"job-1225","job_issued_at_ms":89973,"miner_id":"miner-4","network_difficulty":1000001,"ntime":1700001225,"order_key":"04:miner-4","p2mr_program_hex":"c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9","share_difficulty":1,"share_id":"share-1225","share_seq":1225,"template_height":800038},{"accepted_at_ms":89989,"credit_policy":null,"job_id":"job-1500","job_issued_at_ms":89979,"miner_id":"miner-4","network_difficulty":1000004,"ntime":1700001500,"order_key":"04:miner-4","p2mr_program_hex":"dcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdc","share_difficulty":19,"share_id":"share-1500","share_seq":1500,"template_height":800046},{"accepted_at_ms":89990,"credit_policy":null,"job_id":"job-776","job_issued_at_ms":89990,"miner_id":"miner-6","network_difficulty":1000011,"ntime":1700000776,"order_key":"06:miner-6","p2mr_program_hex":"0808080808080808080808080808080808080808080808080808080808080808","share_difficulty":19,"share_id":"share-776","share_seq":776,"template_height":800024},{"accepted_at_ms":89974,"credit_policy":null,"job_id":"job-387","job_issued_at_ms":89983,"miner_id":"miner-2","network_difficulty":1000013,"ntime":1700000387,"order_key":"02:miner-2","p2mr_program_hex":"8383838383838383838383838383838383838383838383838383838383838383","share_difficulty":3,"share_id":"share-387","share_seq":387,"template_height":800012},{"accepted_at_ms":89977,"credit_policy":"stale-grace","job_id":"job-625","job_issued_at_ms":89990,"miner_id":"miner-9","network_difficulty":1000013,"ntime":1700000625,"order_key":"09:miner-9","p2mr_program_hex":"7171717171717171717171717171717171717171717171717171717171717171","share_difficulty":2,"share_id":"share-625","share_seq":625,"template_height":800019},{"accepted_at_ms":89997,"credit_policy":"stale-grace","job_id":"job-1525","job_issued_at_ms":89987,"miner_id":"miner-7","network_difficulty":1000012,"ntime":1700001525,"order_key":"07:miner-7","p2mr_program_hex":"f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5","share_difficulty":1,"share_id":"share-1525","share_seq":1525,"template_height":800047},{"accepted_at_ms":89973,"credit_policy":null,"job_id":"job-1430","job_issued_at_ms":89975,"miner_id":"miner-0","network_difficulty":1000002,"ntime":1700001430,"order_key":"00:miner-0","p2mr_program_hex":"9696969696969696969696969696969696969696969696969696969696969696","share_difficulty":7,"share_id":"share-1430","share_seq":1430,"template_height":800044},{"accepted_at_ms":89994,"credit_policy":null,"job_id":"job-211","job_issued_at_ms":89991,"miner_id":"miner-2","network_difficulty":1000007,"ntime":1700000211,"order_key":"02:miner-2","p2mr_program_hex":"d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3","share_difficulty":19,"share_id":"share-211","share_seq":211,"template_height":800006},{"accepted_at_ms":89981,"credit_policy":null,"job_id":"job-251","job_issued_at_ms":89990,"miner_id":"miner-9","network_difficulty":1000013,"ntime":1700000251,"order_key":"09:miner-9","p2mr_program_hex":"fbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfb","share_difficulty":19,"share_id":"share-251","share_seq":251,"template_height":800007},{"accepted_at_ms":89981,"credit_policy":"stale-grace","job_id":"job-1112","job_issued_at_ms":89996,"miner_id":"miner-1","network_difficulty":1000007,"ntime":1700001112,"order_key":"01:miner-1","p2mr_program_hex":"5858585858585858585858585858585858585858585858585858585858585858","share_difficulty":1,"share_id":"share-1112","share_seq":1112,"template_height":800034},{"accepted_at_ms":89989,"credit_policy":null,"job_id":"job-254","job_issued_at_ms":89984,"miner_id":"miner-1","network_difficulty":1000016,"ntime":1700000254,"order_key":"01:miner-1","p2mr_program_hex":"fefefefefefefefefefefefefefefefefefefefefefefefefefefefefefefefe","share_difficulty":3,"share_id":"share-254","share_seq":254,"template_height":800007},{"accepted_at_ms":89980,"credit_policy":null,"job_id":"job-996","job_issued_at_ms":89988,"miner_id":"miner-6","network_difficulty":1000010,"ntime":1700000996,"order_key":"06:miner-6","p2mr_program_hex":"e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4","share_difficulty":1,"share_id":"share-996","share_seq":996,"template_height":800031},{"accepted_at_ms":89972,"credit_policy":null,"job_id":"job-1672","job_issued_at_ms":89975,"miner_id":"miner-0","network_difficulty":1000006,"ntime":1700001672,"order_key":"00:miner-0","p2mr_program_hex":"8888888888888888888888888888888888888888888888888888888888888888","share_difficulty":1,"share_id":"share-1672","share_seq":1672,"template_height":800052},{"accepted_at_ms":89992,"credit_policy":"stale-grace","job_id":"job-1478","job_issued_at_ms":89990,"miner_id":"miner-4","network_difficulty":1000016,"ntime":1700001478,"order_key":"04:miner-4","p2mr_program_hex":"c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6","share_difficulty":19,"share_id":"share-1478","share_seq":1478,"template_height":800046},{"accepted_at_ms":89986,"credit_policy":null,"job_id":"job-1638","job_issued_at_ms":89970,"miner_id":"miner-10","network_difficulty":1000006,"ntime":1700001638,"order_key":"10:miner-10","p2mr_program_hex":"6666666666666666666666666666666666666666666666666666666666666666","share_difficulty":7,"share_id":"share-1638","share_seq":1638,"template_height":800051},{"accepted_at_ms":89976,"credit_policy":"stale-grace","job_id":"job-1104","job_issued_at_ms":89976,"miner_id":"miner-4","network_difficulty":1000016,"ntime":1700001104,"order_key":"04:miner-4","p2mr_program_hex":"5050505050505050505050505050505050505050505050505050505050505050","share_difficulty":3,"share_id":"share-1104","share_seq":1104,"template_height":800034},{"accepted_at_ms":89984,"credit_policy":"stale-grace","job_id":"job-850","job_issued_at_ms":89976,"miner_id":"miner-3","network_difficulty":1000000,"ntime":1700000850,"order_key":"03:miner-3","p2mr_program_hex":"5252525252525252525252525252525252525252525252525252525252525252","share_difficulty":19,"share_id":"share-850","share_seq":850,"template_height":800026},{"accepted_at_ms":89973,"credit_policy":null,"job_id":"job-240","job_issued_at_ms":89983,"miner_id":"miner-9","network_difficulty":1000002,"ntime":1700000240,"order_key":"09:miner-9","p2mr_program_hex":"f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0","share_difficulty":19,"share_id":"share-240","share_seq":240,"template_height":800007},{"accepted_at_ms":89995,"credit_policy":"stale-grace","job_id":"job-953","job_issued_at_ms":89971,"miner_id":"miner-7","network_difficulty":1000001,"ntime":1700000953,"order_key":"07:miner-7","p2mr_program_hex":"b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9","share_difficulty":1,"share_id":"share-953","share_seq":953,"template_height":800029},{"accepted_at_ms":89992,"credit_policy":null,"job_id":"job-526","job_issued_at_ms":89987,"miner_id":"miner-9","network_difficulty":1000016,"ntime":1700000526,"order_key":"09:miner-9","p2mr_program_hex":"0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e","share_difficulty":19,"share_id":"share-526","share_seq":526,"template_height":800016},{"accepted_at_ms":89972,"credit_policy":null,"job_id":"job-1609","job_issued_at_ms":89987,"miner_id":"miner-3","network_difficulty":1000011,"ntime":1700001609,"order_key":"03:miner-3","p2mr_program_hex":"4949494949494949494949494949494949494949494949494949494949494949","share_difficulty":19,"share_id":"share-1609","share_seq":1609,"template_height":800050},{"accepted_at_ms":89999,"credit_policy":null,"job_id":"job-556","job_issued_at_ms":89997,"miner_id":"miner-6","network_difficulty":1000012,"ntime":1700000556,"order_key":"06:miner-6","p2mr_program_hex":"2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c","share_difficulty":7,"share_id":"share-556","share_seq":556,"template_height":800017},{"accepted_at_ms":89999,"credit_policy":"stale-grace","job_id":"job-1428","job_issued_at_ms":89986,"miner_id":"miner-9","network_difficulty":1000000,"ntime":1700001428,"order_key":"09:miner-9","p2mr_program_hex":"9494949494949494949494949494949494949494949494949494949494949494","share_difficulty":2,"share_id":"share-1428","share_seq":1428,"template_height":800044},{"accepted_at_ms":89991,"credit_policy":null,"job_id":"job-645","job_issued_at_ms":89975,"miner_id":"miner-7","network_difficulty":1000016,"ntime":1700000645,"order_key":"07:miner-7","p2mr_program_hex":"8585858585858585858585858585858585858585858585858585858585858585","share_difficulty":1,"share_id":"share-645","share_seq":645,"template_height":800020},{"accepted_at_ms":89997,"credit_policy":null,"job_id":"job-615","job_issued_at_ms":89993,"miner_id":"miner-10","network_difficulty":1000003,"ntime":1700000615,"order_key":"10:miner-10","p2mr_program_hex":"6767676767676767676767676767676767676767676767676767676767676767","share_difficulty":19,"share_id":"share-615","share_seq":615,"template_height":800019},{"accepted_at_ms":89971,"credit_policy":"stale-grace","job_id":"job-113","job_issued_at_ms":89971,"miner_id":"miner-3","network_difficulty":1000011,"ntime":1700000113,"order_key":"03:miner-3","p2mr_program_hex":"7171717171717171717171717171717171717171717171717171717171717171","share_difficulty":7,"share_id":"share-113","share_seq":113,"template_height":800003},{"accepted_at_ms":89984,"credit_policy":null,"job_id":"job-1198","job_issued_at_ms":89971,"miner_id":"miner-10","network_difficulty":1000008,"ntime":1700001198,"order_key":"10:miner-10","p2mr_program_hex":"aeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeae","share_difficulty":2,"share_id":"share-1198","share_seq":1198,"template_height":800037},{"accepted_at_ms":89974,"credit_policy":null,"job_id":"job-1651","job_issued_at_ms":89986,"miner_id":"miner-1","network_difficulty":1000002,"ntime":1700001651,"order_key":"01:miner-1","p2mr_program_hex":"7373737373737373737373737373737373737373737373737373737373737373","share_difficulty":19,"share_id":"share-1651","share_seq":1651,"template_height":800051},{"accepted_at_ms":89980,"credit_policy":null,"job_id":"job-1708","job_issued_at_ms":89980,"miner_id":"miner-3","network_difficulty":1000008,"ntime":1700001708,"order_key":"03:miner-3","p2mr_program_hex":"acacacacacacacacacacacacacacacacacacacacacacacacacacacacacacacac","share_difficulty":2,"share_id":"share-1708","share_seq":1708,"template_height":800053},{"accepted_at_ms":89990,"credit_policy":"stale-grace","job_id":"job-1712","job_issued_at_ms":89992,"miner_id":"miner-7","network_difficulty":1000012,"ntime":1700001712,"order_key":"07:miner-7","p2mr_program_hex":"b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0","share_difficulty":1,"share_id":"share-1712","share_seq":1712,"template_height":800053},{"accepted_at_ms":89996,"credit_policy":null,"job_id":"job-626","job_issued_at_ms":89997,"miner_id":"miner-10","network_difficulty":1000014,"ntime":1700000626,"order_key":"10:miner-10","p2mr_program_hex":"7272727272727272727272727272727272727272727272727272727272727272","share_difficulty":19,"share_id":"share-626","share_seq":626,"template_height":800019},{"accepted_at_ms":89984,"credit_policy":null,"job_id":"job-593","job_issued_at_ms":89973,"miner_id":"miner-10","network_difficulty":1000015,"ntime":1700000593,"order_key":"10:miner-10","p2mr_program_hex":"5151515151515151515151515151515151515151515151515151515151515151","share_difficulty":7,"share_id":"share-593","share_seq":593,"template_height":800018},{"accepted_at_ms":89988,"credit_policy":"stale-grace","job_id":"job-223","job_issued_at_ms":89994,"miner_id":"miner-3","network_difficulty":1000002,"ntime":1700000223,"order_key":"03:miner-3","p2mr_program_hex":"dfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdf","share_difficulty":19,"share_id":"share-223","share_seq":223,"template_height":800006},{"accepted_at_ms":89973,"credit_policy":null,"job_id":"job-1136","job_issued_at_ms":89979,"miner_id":"miner-3","network_difficulty":1000014,"ntime":1700001136,"order_key":"03:miner-3","p2mr_program_hex":"7070707070707070707070707070707070707070707070707070707070707070","share_difficulty":3,"share_id":"share-1136","share_seq":1136,"template_height":800035},{"accepted_at_ms":89993,"credit_policy":"stale-grace","job_id":"job-1242","job_issued_at_ms":89990,"miner_id":"miner-10","network_difficulty":1000001,"ntime":1700001242,"order_key":"10:miner-10","p2mr_program_hex":"dadadadadadadadadadadadadadadadadadadadadadadadadadadadadadadada","share_difficulty":2,"share_id":"share-1242","share_seq":1242,"template_height":800038},{"accepted_at_ms":89991,"credit_policy":null,"job_id":"job-982","job_issued_at_ms":89987,"miner_id":"miner-3","network_difficulty":1000013,"ntime":1700000982,"order_key":"03:miner-3","p2mr_program_hex":"d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6","share_difficulty":3,"share_id":"share-982","share_seq":982,"template_height":800030},{"accepted_at_ms":89976,"credit_policy":"stale-grace","job_id":"job-491","job_issued_at_ms":89976,"miner_id":"miner-7","network_difficulty":1000015,"ntime":1700000491,"order_key":"07:miner-7","p2mr_program_hex":"ebebebebebebebebebebebebebebebebebebebebebebebebebebebebebebebeb","share_difficulty":2,"share_id":"share-491","share_seq":491,"template_height":800015},{"accepted_at_ms":89999,"credit_policy":"stale-grace","job_id":"job-144","job_issued_at_ms":89992,"miner_id":"miner-1","network_difficulty":1000008,"ntime":1700000144,"order_key":"01:miner-1","p2mr_program_hex":"9090909090909090909090909090909090909090909090909090909090909090","share_difficulty":2,"share_id":"share-144","share_seq":144,"template_height":800004},{"accepted_at_ms":89989,"credit_policy":null,"job_id":"job-1132","job_issued_at_ms":89989,"miner_id":"miner-10","network_difficulty":1000010,"ntime":1700001132,"order_key":"10:miner-10","p2mr_program_hex":"6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c","share_difficulty":7,"share_id":"share-1132","share_seq":1132,"template_height":800035},{"accepted_at_ms":89996,"credit_policy":null,"job_id":"job-517","job_issued_at_ms":89994,"miner_id":"miner-0","network_difficulty":1000007,"ntime":1700000517,"order_key":"00:miner-0","p2mr_program_hex":"0505050505050505050505050505050505050505050505050505050505050505","share_difficulty":2,"share_id":"share-517","share_seq":517,"template_height":800016},{"accepted_at_ms":89974,"credit_policy":null,"job_id":"job-581","job_issued_at_ms":89972,"miner_id":"miner-9","network_difficulty":1000003,"ntime":1700000581,"order_key":"09:miner-9","p2mr_program_hex":"4545454545454545454545454545454545454545454545454545454545454545","share_difficulty":3,"share_id":"share-581","share_seq":581,"template_height":800018},{"accepted_at_ms":89986,"credit_policy":null,"job_id":"job-1492","job_issued_at_ms":89972,"miner_id":"miner-7","network_difficulty":1000013,"ntime":1700001492,"order_key":"07:miner-7","p2mr_program_hex":"d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4","share_difficulty":1,"share_id":"share-1492","share_seq":1492,"template_height":800046},{"accepted_at_ms":89996,"credit_policy":"stale-grace","job_id":"job-99","job_issued_at_ms":89980,"miner_id":"miner-0","network_difficulty":1000014,"ntime":1700000099,"order_key":"00:miner-0","p2mr_program_hex":"6363636363636363636363636363636363636363636363636363636363636363","share_difficulty":2,"share_id":"share-99","share_seq":99,"template_height":800003},{"accepted_at_ms":89995,"credit_policy":"stale-grace","job_id":"job-90","job_issued_at_ms":89988,"miner_id":"miner-2","network_difficulty":1000005,"ntime":1700000090,"order_key":"02:miner-2","p2mr_program_hex":"5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a","share_difficulty":1,"share_id":"share-90","share_seq":90,"template_height":800002},{"accepted_at_ms":89997,"credit_policy":null,"job_id":"job-325","job_issued_at_ms":89974,"miner_id":"miner-6","network_difficulty":1000002,"ntime":1700000325,"order_key":"06:miner-6","p2mr_program_hex":"4545454545454545454545454545454545454545454545454545454545454545","share_difficulty":7,"share_id":"share-325","share_seq":325,"template_height":800010},{"accepted_at_ms":89996,"credit_policy":"stale-grace","job_id":"job-1779","job_issued_at_ms":89978,"miner_id":"miner-8","network_difficulty":1000011,"ntime":1700001779,"order_key":"08:miner-8","p2mr_program_hex":"f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3","share_difficulty":1,"share_id":"share-1779","share_seq":1779,"template_height":800055},{"accepted_at_ms":89989,"credit_policy":"stale-grace","job_id":"job-519","job_issued_at_ms":89975,"miner_id":"miner-2","network_difficulty":1000009,"ntime":1700000519,"order_key":"02:miner-2","p2mr_program_hex":"0707070707070707070707070707070707070707070707070707070707070707","share_difficulty":19,"share_id":"share-519","share_seq":519,"template_height":800016},{"accepted_at_ms":89981,"credit_policy":"stale-grace","job_id":"job-57","job_issued_at_ms":89991,"miner_id":"miner-2","network_difficulty":1000006,"ntime":1700000057,"order_key":"02:miner-2","p2mr_program_hex":"3939393939393939393939393939393939393939393939393939393939393939","share_difficulty":1,"share_id":"share-57","share_seq":57,"template_height":800001},{"accepted_at_ms":89984,"credit_policy":null,"job_id":"job-1507","job_issued_at_ms":89976,"miner_id":"miner-0","network_difficulty":1000011,"ntime":1700001507,"order_key":"00:miner-0","p2mr_program_hex":"e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3","share_difficulty":19,"share_id":"share-1507","share_seq":1507,"template_height":800047},{"accepted_at_ms":89977,"credit_policy":null,"job_id":"job-680","job_issued_at_ms":89995,"miner_id":"miner-9","network_difficulty":1000000,"ntime":1700000680,"order_key":"09:miner-9","p2mr_program_hex":"a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8","share_difficulty":3,"share_id":"share-680","share_seq":680,"template_height":800021},{"accepted_at_ms":89999,"credit_policy":null,"job_id":"job-884","job_issued_at_ms":89971,"miner_id":"miner-4","network_difficulty":1000000,"ntime":1700000884,"order_key":"04:miner-4","p2mr_program_hex":"7474747474747474747474747474747474747474747474747474747474747474","share_difficulty":19,"share_id":"share-884","share_seq":884,"template_height":800027},{"accepted_at_ms":89994,"credit_policy":null,"job_id":"job-1532","job_issued_at_ms":89990,"miner_id":"miner-3","network_difficulty":1000002,"ntime":1700001532,"order_key":"03:miner-3","p2mr_program_hex":"fcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfc","share_difficulty":1,"share_id":"share-1532","share_seq":1532,"template_height":800047},{"accepted_at_ms":89983,"credit_policy":"stale-grace","job_id":"job-1305","job_issued_at_ms":89978,"miner_id":"miner-7","network_difficulty":1000013,"ntime":1700001305,"order_key":"07:miner-7","p2mr_program_hex":"1919191919191919191919191919191919191919191919191919191919191919","share_difficulty":2,"share_id":"share-1305","share_seq":1305,"template_height":800040},{"accepted_at_ms":89988,"credit_policy":null,"job_id":"job-741","job_issued_at_ms":89996,"miner_id":"miner-4","network_difficulty":1000010,"ntime":1700000741,"order_key":"04:miner-4","p2mr_program_hex":"e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5","share_difficulty":19,"share_id":"share-741","share_seq":741,"template_height":800023},{"accepted_at_ms":89981,"credit_policy":null,"job_id":"job-940","job_issued_at_ms":89978,"miner_id":"miner-5","network_difficulty":1000005,"ntime":1700000940,"order_key":"05:miner-5","p2mr_program_hex":"acacacacacacacacacacacacacacacacacacacacacacacacacacacacacacacac","share_difficulty":2,"share_id":"share-940","share_seq":940,"template_height":800029},{"accepted_at_ms":89981,"credit_policy":null,"job_id":"job-147","job_issued_at_ms":89970,"miner_id":"miner-4","network_difficulty":1000011,"ntime":1700000147,"order_key":"04:miner-4","p2mr_program_hex":"9393939393939393939393939393939393939393939393939393939393939393","share_difficulty":1,"share_id":"share-147","share_seq":147,"template_height":800004},{"accepted_at_ms":89970,"credit_policy":null,"job_id":"job-347","job_issued_at_ms":89998,"miner_id":"miner-6","network_difficulty":1000007,"ntime":1700000347,"order_key":"06:miner-6","p2mr_program_hex":"5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b","share_difficulty":1,"share_id":"share-347","share_seq":347,"template_height":800010},{"accepted_at_ms":89978,"credit_policy":"stale-grace","job_id":"job-1741","job_issued_at_ms":89997,"miner_id":"miner-3","network_difficulty":1000007,"ntime":1700001741,"order_key":"03:miner-3","p2mr_program_hex":"cdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcd","share_difficulty":3,"share_id":"share-1741","share_seq":1741,"template_height":800054},{"accepted_at_ms":89977,"credit_policy":null,"job_id":"job-745","job_issued_at_ms":89983,"miner_id":"miner-8","network_difficulty":1000014,"ntime":1700000745,"order_key":"08:miner-8","p2mr_program_hex":"e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9","share_difficulty":19,"share_id":"share-745","share_seq":745,"template_height":800023},{"accepted_at_ms":89986,"credit_policy":null,"job_id":"job-942","job_issued_at_ms":89973,"miner_id":"miner-7","network_difficulty":1000007,"ntime":1700000942,"order_key":"07:miner-7","p2mr_program_hex":"aeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeae","share_difficulty":3,"share_id":"share-942","share_seq":942,"template_height":800029},{"accepted_at_ms":89973,"credit_policy":null,"job_id":"job-623","job_issued_at_ms":89987,"miner_id":"miner-7","network_difficulty":1000011,"ntime":1700000623,"order_key":"07:miner-7","p2mr_program_hex":"6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f","share_difficulty":1,"share_id":"share-623","share_seq":623,"template_height":800019},{"accepted_at_ms":89985,"credit_policy":null,"job_id":"job-85","job_issued_at_ms":89994,"miner_id":"miner-8","network_difficulty":1000000,"ntime":1700000085,"order_key":"08:miner-8","p2mr_program_hex":"5555555555555555555555555555555555555555555555555555555555555555","share_difficulty":1,"share_id":"share-85","share_seq":85,"template_height":800002},{"accepted_at_ms":89976,"credit_policy":null,"job_id":"job-1790","job_issued_at_ms":90000,"miner_id":"miner-8","network_difficulty":1000005,"ntime":1700001790,"order_key":"08:miner-8","p2mr_program_hex":"fefefefefefefefefefefefefefefefefefefefefefefefefefefefefefefefe","share_difficulty":1,"share_id":"share-1790","share_seq":1790,"template_height":800055},{"accepted_at_ms":89987,"credit_policy":null,"job_id":"job-771","job_issued_at_ms":89990,"miner_id":"miner-1","network_difficulty":1000006,"ntime":1700000771,"order_key":"01:miner-1","p2mr_program_hex":"0303030303030303030303030303030303030303030303030303030303030303","share_difficulty":7,"share_id":"share-771","share_seq":771,"template_height":800024},{"accepted_at_ms":89993,"credit_policy":null,"job_id":"job-1576","job_issued_at_ms":89979,"miner_id":"miner-3","network_difficulty":1000012,"ntime":1700001576,"order_key":"03:miner-3","p2mr_program_hex":"2828282828282828282828282828282828282828282828282828282828282828","share_difficulty":2,"share_id":"share-1576","share_seq":1576,"template_height":800049},{"accepted_at_ms":89974,"credit_policy":null,"job_id":"job-350","job_issued_at_ms":89971,"miner_id":"miner-9","network_difficulty":1000010,"ntime":1700000350,"order_key":"09:miner-9","p2mr_program_hex":"5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e","share_difficulty":19,"share_id":"share-350","share_seq":350,"template_height":800010},{"accepted_at_ms":89980,"credit_policy":"stale-grace","job_id":"job-1762","job_issued_at_ms":89984,"miner_id":"miner-2","network_difficulty":1000011,"ntime":1700001762,"order_key":"02:miner-2","p2mr_program_hex":"e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2","share_difficulty":2,"share_id":"share-1762","share_seq":1762,"template_height":800055},{"accepted_at_ms":89981,"credit_policy":"stale-grace","job_id":"job-234","job_issued_at_ms":89981,"miner_id":"miner-3","network_difficulty":1000013,"ntime":1700000234,"order_key":"03:miner-3","p2mr_program_hex":"eaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaea","share_difficulty":7,"share_id":"share-234","share_seq":234,"template_height":800007},{"accepted_at_ms":89976,"credit_policy":null,"job_id":"job-1417","job_issued_at_ms":89980,"miner_id":"miner-9","network_difficulty":1000006,"ntime":1700001417,"order_key":"09:miner-9","p2mr_program_hex":"8989898989898989898989898989898989898989898989898989898989898989","share_difficulty":1,"share_id":"share-1417","share_seq":1417,"template_height":800044},{"accepted_at_ms":89984,"credit_policy":null,"job_id":"job-678","job_issued_at_ms":89988,"miner_id":"miner-7","network_difficulty":1000015,"ntime":1700000678,"order_key":"07:miner-7","p2mr_program_hex":"a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6","share_difficulty":7,"share_id":"share-678","share_seq":678,"template_height":800021},{"accepted_at_ms":89983,"credit_policy":null,"job_id":"job-737","job_issued_at_ms":89979,"miner_id":"miner-0","network_difficulty":1000006,"ntime":1700000737,"order_key":"00:miner-0","p2mr_program_hex":"e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1","share_difficulty":7,"share_id":"share-737","share_seq":737,"template_height":800023},{"accepted_at_ms":89974,"credit_policy":null,"job_id":"job-1204","job_issued_at_ms":89992,"miner_id":"miner-5","network_difficulty":1000014,"ntime":1700001204,"order_key":"05:miner-5","p2mr_program_hex":"b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4","share_difficulty":2,"share_id":"share-1204","share_seq":1204,"template_height":800037},{"accepted_at_ms":89986,"credit_policy":null,"job_id":"job-392","job_issued_at_ms":89995,"miner_id":"miner-7","network_difficulty":1000001,"ntime":1700000392,"order_key":"07:miner-7","p2mr_program_hex":"8888888888888888888888888888888888888888888888888888888888888888","share_difficulty":2,"share_id":"share-392","share_seq":392,"template_height":800012},{"accepted_at_ms":89971,"credit_policy":"stale-grace","job_id":"job-483","job_issued_at_ms":89995,"miner_id":"miner-10","network_difficulty":1000007,"ntime":1700000483,"order_key":"10:miner-10","p2mr_program_hex":"e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3","share_difficulty":3,"share_id":"share-483","share_seq":483,"template_height":800015},{"accepted_at_ms":89989,"credit_policy":null,"job_id":"job-317","job_issued_at_ms":89986,"miner_id":"miner-9","network_difficulty":1000011,"ntime":1700000317,"order_key":"09:miner-9","p2mr_program_hex":"3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d","share_difficulty":19,"share_id":"share-317","share_seq":317,"template_height":800009},{"accepted_at_ms":89975,"credit_policy":null,"job_id":"job-1555","job_issued_at_ms":89975,"miner_id":"miner-4","network_difficulty":1000008,"ntime":1700001555,"order_key":"04:miner-4","p2mr_program_hex":"1313131313131313131313131313131313131313131313131313131313131313","share_difficulty":19,"share_id":"share-1555","share_seq":1555,"template_height":800048},{"accepted_at_ms":89980,"credit_policy":null,"job_id":"job-1571","job_issued_at_ms":89990,"miner_id":"miner-9","network_difficulty":1000007,"ntime":1700001571,"order_key":"09:miner-9","p2mr_program_hex":"2323232323232323232323232323232323232323232323232323232323232323","share_difficulty":7,"share_id":"share-1571","share_seq":1571,"template_height":800049},{"accepted_at_ms":89973,"credit_policy":null,"job_id":"job-422","job_issued_at_ms":89983,"miner_id":"miner-4","network_difficulty":1000014,"ntime":1700000422,"order_key":"04:miner-4","p2mr_program_hex":"a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6","share_difficulty":2,"share_id":"share-422","share_seq":422,"template_height":800013},{"accepted_at_ms":89972,"credit_policy":"stale-grace","job_id":"job-697","job_issued_at_ms":89997,"miner_id":"miner-4","network_difficulty":1000000,"ntime":1700000697,"order_key":"04:miner-4","p2mr_program_hex":"b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9","share_difficulty":2,"share_id":"share-697","share_seq":697,"template_height":800021},{"accepted_at_ms":89976,"credit_policy":null,"job_id":"job-1580","job_issued_at_ms":89986,"miner_id":"miner-7","network_difficulty":1000016,"ntime":1700001580,"order_key":"07:miner-7","p2mr_program_hex":"2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c","share_difficulty":3,"share_id":"share-1580","share_seq":1580,"template_height":800049},{"accepted_at_ms":89974,"credit_policy":null,"job_id":"job-261","job_issued_at_ms":89983,"miner_id":"miner-8","network_difficulty":1000006,"ntime":1700000261,"order_key":"08:miner-8","p2mr_program_hex":"0505050505050505050505050505050505050505050505050505050505050505","share_difficulty":19,"share_id":"share-261","share_seq":261,"template_height":800008},{"accepted_at_ms":89976,"credit_policy":null,"job_id":"job-136","job_issued_at_ms":89977,"miner_id":"miner-4","network_difficulty":1000000,"ntime":1700000136,"order_key":"04:miner-4","p2mr_program_hex":"8888888888888888888888888888888888888888888888888888888888888888","share_difficulty":19,"share_id":"share-136","share_seq":136,"template_height":800004},{"accepted_at_ms":89993,"credit_policy":"stale-grace","job_id":"job-83","job_issued_at_ms":89977,"miner_id":"miner-6","network_difficulty":1000015,"ntime":1700000083,"order_key":"06:miner-6","p2mr_program_hex":"5353535353535353535353535353535353535353535353535353535353535353","share_difficulty":1,"share_id":"share-83","share_seq":83,"template_height":800002},{"accepted_at_ms":89994,"credit_policy":"stale-grace","job_id":"job-1616","job_issued_at_ms":89983,"miner_id":"miner-10","network_difficulty":1000001,"ntime":1700001616,"order_key":"10:miner-10","p2mr_program_hex":"5050505050505050505050505050505050505050505050505050505050505050","share_difficulty":19,"share_id":"share-1616","share_seq":1616,"template_height":800050},{"accepted_at_ms":89971,"credit_policy":null,"job_id":"job-1431","job_issued_at_ms":89970,"miner_id":"miner-1","network_difficulty":1000003,"ntime":1700001431,"order_key":"01:miner-1","p2mr_program_hex":"9797979797979797979797979797979797979797979797979797979797979797","share_difficulty":19,"share_id":"share-1431","share_seq":1431,"template_height":800044},{"accepted_at_ms":89970,"credit_policy":null,"job_id":"job-967","job_issued_at_ms":89975,"miner_id":"miner-10","network_difficulty":1000015,"ntime":1700000967,"order_key":"10:miner-10","p2mr_program_hex":"c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7","share_difficulty":7,"share_id":"share-967","share_seq":967,"template_height":800030},{"accepted_at_ms":89976,"credit_policy":null,"job_id":"job-10","job_issued_at_ms":89978,"miner_id":"miner-10","network_difficulty":1000010,"ntime":1700000010,"order_key":"10:miner-10","p2mr_program_hex":"0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a","share_difficulty":3,"share_id":"share-10","share_seq":10,"template_height":800000},{"accepted_at_ms":89998,"credit_policy":null,"job_id":"job-1504","job_issued_at_ms":89977,"miner_id":"miner-8","network_difficulty":1000008,"ntime":1700001504,"order_key":"08:miner-8","p2mr_program_hex":"e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0","share_difficulty":7,"share_id":"share-1504","share_seq":1504,"template_height":800047},{"accepted_at_ms":89982,"credit_policy":null,"job_id":"job-1514","job_issued_at_ms":89971,"miner_id":"miner-7","network_difficulty":1000001,"ntime":1700001514,"order_key":"07:miner-7","p2mr_program_hex":"eaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaea","share_difficulty":7,"share_id":"share-1514","share_seq":1514,"template_height":800047},{"accepted_at_ms":89981,"credit_policy":null,"job_id":"job-646","job_issued_at_ms":89992,"miner_id":"miner-8","network_difficulty":1000000,"ntime":1700000646,"order_key":"08:miner-8","p2mr_program_hex":"8686868686868686868686868686868686868686868686868686868686868686","share_difficulty":7,"share_id":"share-646","share_seq":646,"template_height":800020},{"accepted_at_ms":89995,"credit_policy":null,"job_id":"job-1365","job_issued_at_ms":89992,"miner_id":"miner-1","network_difficulty":1000005,"ntime":1700001365,"order_key":"01:miner-1","p2mr_program_hex":"5555555555555555555555555555555555555555555555555555555555555555","share_difficulty":3,"share_id":"share-1365","share_seq":1365,"template_height":800042},{"accepted_at_ms":89995,"credit_policy":null,"job_id":"job-123","job_issued_at_ms":89999,"miner_id":"miner-2","network_difficulty":1000004,"ntime":1700000123,"order_key":"02:miner-2","p2mr_program_hex":"7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b","share_difficulty":3,"share_id":"share-123","share_seq":123,"template_height":800003},{"accepted_at_ms":89980,"credit_policy":null,"job_id":"job-838","job_issued_at_ms":89975,"miner_id":"miner-2","network_difficulty":1000005,"ntime":1700000838,"order_key":"02:miner-2","p2mr_program_hex":"4646464646464646464646464646464646464646464646464646464646464646","share_difficulty":1,"share_id":"share-838","share_seq":838,"template_height":800026},{"accepted_at_ms":89981,"credit_policy":null,"job_id":"job-1380","job_issued_at_ms":89997,"miner_id":"miner-5","network_difficulty":1000003,"ntime":1700001380,"order_key":"05:miner-5","p2mr_program_hex":"6464646464646464646464646464646464646464646464646464646464646464","share_difficulty":3,"share_id":"share-1380","share_seq":1380,"template_height":800043},{"accepted_at_ms":89980,"credit_policy":null,"job_id":"job-1438","job_issued_at_ms":89974,"miner_id":"miner-8","network_difficulty":1000010,"ntime":1700001438,"order_key":"08:miner-8","p2mr_program_hex":"9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e","share_difficulty":19,"share_id":"share-1438","share_seq":1438,"template_height":800044},{"accepted_at_ms":89994,"credit_policy":"stale-grace","job_id":"job-886","job_issued_at_ms":89983,"miner_id":"miner-6","network_difficulty":1000002,"ntime":1700000886,"order_key":"06:miner-6","p2mr_program_hex":"7676767676767676767676767676767676767676767676767676767676767676","share_difficulty":3,"share_id":"share-886","share_seq":886,"template_height":800027},{"accepted_at_ms":89987,"credit_policy":null,"job_id":"job-1632","job_issued_at_ms":89991,"miner_id":"miner-4","network_difficulty":1000000,"ntime":1700001632,"order_key":"04:miner-4","p2mr_program_hex":"6060606060606060606060606060606060606060606060606060606060606060","share_difficulty":7,"share_id":"share-1632","share_seq":1632,"template_height":800051},{"accepted_at_ms":89970,"credit_policy":null,"job_id":"job-1220","job_issued_at_ms":89981,"miner_id":"miner-10","network_difficulty":1000013,"ntime":1700001220,"order_key":"10:miner-10","p2mr_program_hex":"c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4","share_difficulty":1,"share_id":"share-1220","share_seq":1220,"template_height":800038},{"accepted_at_ms":89988,"credit_policy":null,"job_id":"job-1384","job_issued_at_ms":89975,"miner_id":"miner-9","network_difficulty":1000007,"ntime":1700001384,"order_key":"09:miner-9","p2mr_program_hex":"6868686868686868686868686868686868686868686868686868686868686868","share_difficulty":7,"share_id":"share-1384","share_seq":1384,"template_height":800043},{"accepted_at_ms":89998,"credit_policy":"stale-grace","job_id":"job-290","job_issued_at_ms":89973,"miner_id":"miner-4","network_difficulty":1000001,"ntime":1700000290,"order_key":"04:miner-4","p2mr_program_hex":"2222222222222222222222222222222222222222222222222222222222222222","share_difficulty":19,"share_id":"share-290","share_seq":290,"template_height":800009},{"accepted_at_ms":89994,"credit_policy":null,"job_id":"job-959","job_issued_at_ms":89982,"miner_id":"miner-2","network_difficulty":1000007,"ntime":1700000959,"order_key":"02:miner-2","p2mr_program_hex":"bfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbf","share_difficulty":3,"share_id":"share-959","share_seq":959,"template_height":800029},{"accepted_at_ms":89979,"credit_policy":"stale-grace","job_id":"job-1269","job_issued_at_ms":89989,"miner_id":"miner-4","network_difficulty":1000011,"ntime":1700001269,"order_key":"04:miner-4","p2mr_program_hex":"f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5","share_difficulty":1,"share_id":"share-1269","share_seq":1269,"template_height":800039},{"accepted_at_ms":89972,"credit_policy":"stale-grace","job_id":"job-363","job_issued_at_ms":89986,"miner_id":"miner-0","network_difficulty":1000006,"ntime":1700000363,"order_key":"00:miner-0","p2mr_program_hex":"6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b","share_difficulty":1,"share_id":"share-363","share_seq":363,"template_height":800011},{"accepted_at_ms":89974,"credit_policy":null,"job_id":"job-418","job_issued_at_ms":89990,"miner_id":"miner-0","network_difficulty":1000010,"ntime":1700000418,"order_key":"00:miner-0","p2mr_program_hex":"a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2","share_difficulty":7,"share_id":"share-418","share_seq":418,"template_height":800013},{"accepted_at_ms":89974,"credit_policy":null,"job_id":"job-1160","job_issued_at_ms":89983,"miner_id":"miner-5","network_difficulty":1000004,"ntime":1700001160,"order_key":"05:miner-5","p2mr_program_hex":"8888888888888888888888888888888888888888888888888888888888888888","share_difficulty":1,"share_id":"share-1160","share_seq":1160,"template_height":800036},{"accepted_at_ms":89972,"credit_policy":null,"job_id":"job-1440","job_issued_at_ms":89973,"miner_id":"miner-10","network_difficulty":1000012,"ntime":1700001440,"order_key":"10:miner-10","p2mr_program_hex":"a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0","share_difficulty":7,"share_id":"share-1440","share_seq":1440,"template_height":800045},{"accepted_at_ms":89977,"credit_policy":"stale-grace","job_id":"job-437","job_issued_at_ms":89976,"miner_id":"miner-8","network_difficulty":1000012,"ntime":1700000437,"order_key":"08:miner-8","p2mr_program_hex":"b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5","share_difficulty":19,"share_id":"share-437","share_seq":437,"template_height":800013},{"accepted_at_ms":89971,"credit_policy":null,"job_id":"job-122","job_issued_at_ms":89984,"miner_id":"miner-1","network_difficulty":1000003,"ntime":1700000122,"order_key":"01:miner-1","p2mr_program_hex":"7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a","share_difficulty":2,"share_id":"share-122","share_seq":122,"template_height":800003},{"accepted_at_ms":89974,"credit_policy":null,"job_id":"job-672","job_issued_at_ms":89976,"miner_id":"miner-1","network_difficulty":1000009,"ntime":1700000672,"order_key":"01:miner-1","p2mr_program_hex":"a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0","share_difficulty":7,"share_id":"share-672","share_seq":672,"template_height":800021},{"accepted_at_ms":89997,"credit_policy":"stale-grace","job_id":"job-931","job_issued_at_ms":90000,"miner_id":"miner-7","network_difficulty":1000013,"ntime":1700000931,"order_key":"07:miner-7","p2mr_program_hex":"a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3","share_difficulty":3,"share_id":"share-931","share_seq":931,"template_height":800029},{"accepted_at_ms":89973,"credit_policy":null,"job_id":"job-1118","job_issued_at_ms":89997,"miner_id":"miner-7","network_difficulty":1000013,"ntime":1700001118,"order_key":"07:miner-7","p2mr_program_hex":"5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e","share_difficulty":3,"share_id":"share-1118","share_seq":1118,"template_height":800034},{"accepted_at_ms":89994,"credit_policy":"stale-grace","job_id":"job-1705","job_issued_at_ms":89976,"miner_id":"miner-0","network_difficulty":1000005,"ntime":1700001705,"order_key":"00:miner-0","p2mr_program_hex":"a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9","share_difficulty":7,"share_id":"share-1705","share_seq":1705,"template_height":800053},{"accepted_at_ms":89989,"credit_policy":null,"job_id":"job-1011","job_issued_at_ms":89978,"miner_id":"miner-10","network_difficulty":1000008,"ntime":1700001011,"order_key":"10:miner-10","p2mr_program_hex":"f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3","share_difficulty":3,"share_id":"share-1011","share_seq":1011,"template_height":800031},{"accepted_at_ms":89998,"credit_policy":null,"job_id":"job-637","job_issued_at_ms":89970,"miner_id":"miner-10","network_difficulty":1000008,"ntime":1700000637,"order_key":"10:miner-10","p2mr_program_hex":"7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d","share_difficulty":19,"share_id":"share-637","share_seq":637,"template_height":800019},{"accepted_at_ms":89993,"credit_policy":null,"job_id":"job-1477","job_issued_at_ms":89987,"miner_id":"miner-3","network_difficulty":1000015,"ntime":1700001477,"order_key":"03:miner-3","p2mr_program_hex":"c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5","share_difficulty":1,"share_id":"share-1477","share_seq":1477,"template_height":800046},{"accepted_at_ms":89979,"credit_policy":"stale-grace","job_id":"job-172","job_issued_at_ms":89996,"miner_id":"miner-7","network_difficulty":1000002,"ntime":1700000172,"order_key":"07:miner-7","p2mr_program_hex":"acacacacacacacacacacacacacacacacacacacacacacacacacacacacacacacac","share_difficulty":7,"share_id":"share-172","share_seq":172,"template_height":800005},{"accepted_at_ms":89991,"credit_policy":null,"job_id":"job-945","job_issued_at_ms":89970,"miner_id":"miner-10","network_difficulty":1000010,"ntime":1700000945,"order_key":"10:miner-10","p2mr_program_hex":"b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1","share_difficulty":2,"share_id":"share-945","share_seq":945,"template_height":800029},{"accepted_at_ms":89986,"credit_policy":null,"job_id":"job-423","job_issued_at_ms":89994,"miner_id":"miner-5","network_difficulty":1000015,"ntime":1700000423,"order_key":"05:miner-5","p2mr_program_hex":"a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7","share_difficulty":3,"share_id":"share-423","share_seq":423,"template_height":800013},{"accepted_at_ms":89988,"credit_policy":null,"job_id":"job-1462","job_issued_at_ms":89986,"miner_id":"miner-10","network_difficulty":1000000,"ntime":1700001462,"order_key":"10:miner-10","p2mr_program_hex":"b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6","share_difficulty":1,"share_id":"share-1462","share_seq":1462,"template_height":800045},{"accepted_at_ms":89999,"credit_policy":null,"job_id":"job-384","job_issued_at_ms":89982,"miner_id":"miner-10","network_difficulty":1000010,"ntime":1700000384,"order_key":"10:miner-10","p2mr_program_hex":"8080808080808080808080808080808080808080808080808080808080808080","share_difficulty":1,"share_id":"share-384","share_seq":384,"template_height":800012},{"accepted_at_ms":89996,"credit_policy":"stale-grace","job_id":"job-892","job_issued_at_ms":89989,"miner_id":"miner-1","network_difficulty":1000008,"ntime":1700000892,"order_key":"01:miner-1","p2mr_program_hex":"7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c","share_difficulty":7,"share_id":"share-892","share_seq":892,"template_height":800027},{"accepted_at_ms":89999,"credit_policy":null,"job_id":"job-1061","job_issued_at_ms":89988,"miner_id":"miner-5","network_difficulty":1000007,"ntime":1700001061,"order_key":"05:miner-5","p2mr_program_hex":"2525252525252525252525252525252525252525252525252525252525252525","share_difficulty":2,"share_id":"share-1061","share_seq":1061,"template_height":800033},{"accepted_at_ms":89970,"credit_policy":null,"job_id":"job-1451","job_issued_at_ms":89984,"miner_id":"miner-10","network_difficulty":1000006,"ntime":1700001451,"order_key":"10:miner-10","p2mr_program_hex":"abababababababababababababababababababababababababababababababab","share_difficulty":3,"share_id":"share-1451","share_seq":1451,"template_height":800045},{"accepted_at_ms":89984,"credit_policy":null,"job_id":"job-1620","job_issued_at_ms":89999,"miner_id":"miner-3","network_difficulty":1000005,"ntime":1700001620,"order_key":"03:miner-3","p2mr_program_hex":"5454545454545454545454545454545454545454545454545454545454545454","share_difficulty":2,"share_id":"share-1620","share_seq":1620,"template_height":800050},{"accepted_at_ms":89994,"credit_policy":null,"job_id":"job-1649","job_issued_at_ms":89983,"miner_id":"miner-10","network_difficulty":1000000,"ntime":1700001649,"order_key":"10:miner-10","p2mr_program_hex":"7171717171717171717171717171717171717171717171717171717171717171","share_difficulty":1,"share_id":"share-1649","share_seq":1649,"template_height":800051},{"accepted_at_ms":89984,"credit_policy":null,"job_id":"job-790","job_issued_at_ms":89983,"miner_id":"miner-9","network_difficulty":1000008,"ntime":1700000790,"order_key":"09:miner-9","p2mr_program_hex":"1616161616161616161616161616161616161616161616161616161616161616","share_difficulty":2,"share_id":"share-790","share_seq":790,"template_height":800024},{"accepted_at_ms":89994,"credit_policy":null,"job_id":"job-1497","job_issued_at_ms":89974,"miner_id":"miner-1","network_difficulty":1000001,"ntime":1700001497,"order_key":"01:miner-1","p2mr_program_hex":"d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9","share_difficulty":2,"share_id":"share-1497","share_seq":1497,"template_height":800046},{"accepted_at_ms":89978,"credit_policy":null,"job_id":"job-1313","job_issued_at_ms":89991,"miner_id":"miner-4","network_difficulty":1000004,"ntime":1700001313,"order_key":"04:miner-4","p2mr_program_hex":"2121212121212121212121212121212121212121212121212121212121212121","share_difficulty":19,"share_id":"share-1313","share_seq":1313,"template_height":800041},{"accepted_at_ms":89987,"credit_policy":"stale-grace","job_id":"job-1702","job_issued_at_ms":89993,"miner_id":"miner-8","network_difficulty":1000002,"ntime":1700001702,"order_key":"08:miner-8","p2mr_program_hex":"a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6","share_difficulty":7,"share_id":"share-1702","share_seq":1702,"template_height":800053},{"accepted_at_ms":89988,"credit_policy":"stale-grace","job_id":"job-45","job_issued_at_ms":89982,"miner_id":"miner-1","network_difficulty":1000011,"ntime":1700000045,"order_key":"01:miner-1","p2mr_program_hex":"2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d","share_difficulty":2,"share_id":"share-45","share_seq":45,"template_height":800001},{"accepted_at_ms":89992,"credit_policy":null,"job_id":"job-1189","job_issued_at_ms":89993,"miner_id":"miner-1","network_difficulty":1000016,"ntime":1700001189,"order_key":"01:miner-1","p2mr_program_hex":"a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5","share_difficulty":3,"share_id":"share-1189","share_seq":1189,"template_height":800037},{"accepted_at_ms":89992,"credit_policy":"stale-grace","job_id":"job-1498","job_issued_at_ms":89996,"miner_id":"miner-2","network_difficulty":1000002,"ntime":1700001498,"order_key":"02:miner-2","p2mr_program_hex":"dadadadadadadadadadadadadadadadadadadadadadadadadadadadadadadada","share_difficulty":19,"share_id":"share-1498","share_seq":1498,"template_height":800046},{"accepted_at_ms":89984,"credit_policy":"stale-grace","job_id":"job-1022","job_issued_at_ms":89999,"miner_id":"miner-10","network_difficulty":1000002,"ntime":1700001022,"order_key":"10:miner-10","p2mr_program_hex":"fefefefefefefefefefefefefefefefefefefefefefefefefefefefefefefefe","share_difficulty":1,"share_id":"share-1022","share_seq":1022,"template_height":800031},{"accepted_at_ms":89995,"credit_policy":"stale-grace","job_id":"job-168","job_issued_at_ms":89977,"miner_id":"miner-3","network_difficulty":1000015,"ntime":1700000168,"order_key":"03:miner-3","p2mr_program_hex":"a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8","share_difficulty":1,"share_id":"share-168","share_seq":168,"template_height":800005},{"accepted_at_ms":89994,"credit_policy":null,"job_id":"job-173","job_issued_at_ms":89976,"miner_id":"miner-8","network_difficulty":1000003,"ntime":1700000173,"order_key":"08:miner-8","p2mr_program_hex":"adadadadadadadadadadadadadadadadadadadadadadadadadadadadadadadad","share_difficulty":1,"share_id":"share-173","share_seq":173,"template_height":800005},{"accepted_at_ms":89989,"credit_policy":null,"job_id":"job-1610","job_issued_at_ms":89991,"miner_id":"miner-4","network_difficulty":1000012,"ntime":1700001610,"order_key":"04:miner-4","p2mr_program_hex":"4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a","share_difficulty":19,"share_id":"share-1610","share_seq":1610,"template_height":800050},{"accepted_at_ms":89985,"credit_policy":null,"job_id":"job-970","job_issued_at_ms":89987,"miner_id":"miner-2","network_difficulty":1000001,"ntime":1700000970,"order_key":"02:miner-2","p2mr_program_hex":"cacacacacacacacacacacacacacacacacacacacacacacacacacacacacacacaca","share_difficulty":19,"share_id":"share-970","share_seq":970,"template_height":800030},{"accepted_at_ms":89988,"credit_policy":"stale-grace","job_id":"job-1485","job_issued_at_ms":89995,"miner_id":"miner-0","network_difficulty":1000006,"ntime":1700001485,"order_key":"00:miner-0","p2mr_program_hex":"cdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcd","share_difficulty":2,"share_id":"share-1485","share_seq":1485,"template_height":800046},{"accepted_at_ms":89995,"credit_policy":null,"job_id":"job-1254","job_issued_at_ms":89995,"miner_id":"miner-0","network_difficulty":1000013,"ntime":1700001254,"order_key":"00:miner-0","p2mr_program_hex":"e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6","share_difficulty":7,"share_id":"share-1254","share_seq":1254,"template_height":800039},{"accepted_at_ms":89991,"credit_policy":null,"job_id":"job-1619","job_issued_at_ms":89993,"miner_id":"miner-2","network_difficulty":1000004,"ntime":1700001619,"order_key":"02:miner-2","p2mr_program_hex":"5353535353535353535353535353535353535353535353535353535353535353","share_difficulty":1,"share_id":"share-1619","share_seq":1619,"template_height":800050},{"accepted_at_ms":89998,"credit_policy":null,"job_id":"job-380","job_issued_at_ms":89982,"miner_id":"miner-6","network_difficulty":1000006,"ntime":1700000380,"order_key":"06:miner-6","p2mr_program_hex":"7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c","share_difficulty":7,"share_id":"share-380","share_seq":380,"template_height":800011},{"accepted_at_ms":89989,"credit_policy":null,"job_id":"job-352","job_issued_at_ms":89984,"miner_id":"miner-0","network_difficulty":1000012,"ntime":1700000352,"order_key":"00:miner-0","p2mr_program_hex":"6060606060606060606060606060606060606060606060606060606060606060","share_difficulty":2,"share_id":"share-352","share_seq":352,"template_height":800011},{"accepted_at_ms":89985,"credit_policy":null,"job_id":"job-1593","job_issued_at_ms":89986,"miner_id":"miner-9","network_difficulty":1000012,"ntime":1700001593,"order_key":"09:miner-9","p2mr_program_hex":"3939393939393939393939393939393939393939393939393939393939393939","share_difficulty":3,"share_id":"share-1593","share_seq":1593,"template_height":800049},{"accepted_at_ms":89981,"credit_policy":null,"job_id":"job-231","job_issued_at_ms":89973,"miner_id":"miner-0","network_difficulty":1000010,"ntime":1700000231,"order_key":"00:miner-0","p2mr_program_hex":"e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7","share_difficulty":7,"share_id":"share-231","share_seq":231,"template_height":800007},{"accepted_at_ms":89973,"credit_policy":null,"job_id":"job-1426","job_issued_at_ms":89983,"miner_id":"miner-7","network_difficulty":1000015,"ntime":1700001426,"order_key":"07:miner-7","p2mr_program_hex":"9292929292929292929292929292929292929292929292929292929292929292","share_difficulty":7,"share_id":"share-1426","share_seq":1426,"template_height":800044},{"accepted_at_ms":89975,"credit_policy":"stale-grace","job_id":"job-1540","job_issued_at_ms":89984,"miner_id":"miner-0","network_difficulty":1000010,"ntime":1700001540,"order_key":"00:miner-0","p2mr_program_hex":"0404040404040404040404040404040404040404040404040404040404040404","share_difficulty":7,"share_id":"share-1540","share_seq":1540,"template_height":800048},{"accepted_at_ms":89997,"credit_policy":null,"job_id":"job-908","job_issued_at_ms":89980,"miner_id":"miner-6","network_difficulty":1000007,"ntime":1700000908,"order_key":"06:miner-6","p2mr_program_hex":"8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c","share_difficulty":3,"share_id":"share-908","share_seq":908,"template_height":800028},{"accepted_at_ms":89998,"credit_policy":"stale-grace","job_id":"job-26","job_issued_at_ms":89979,"miner_id":"miner-4","network_difficulty":1000009,"ntime":1700000026,"order_key":"04:miner-4","p2mr_program_hex":"1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a","share_difficulty":1,"share_id":"share-26","share_seq":26,"template_height":800000},{"accepted_at_ms":89986,"credit_policy":null,"job_id":"job-1405","job_issued_at_ms":89983,"miner_id":"miner-8","network_difficulty":1000011,"ntime":1700001405,"order_key":"08:miner-8","p2mr_program_hex":"7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d","share_difficulty":1,"share_id":"share-1405","share_seq":1405,"template_height":800043},{"accepted_at_ms":89997,"credit_policy":null,"job_id":"job-1310","job_issued_at_ms":89970,"miner_id":"miner-1","network_difficulty":1000001,"ntime":1700001310,"order_key":"01:miner-1","p2mr_program_hex":"1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e","share_difficulty":19,"share_id":"share-1310","share_seq":1310,"template_height":800040},{"accepted_at_ms":89983,"credit_policy":null,"job_id":"job-199","job_issued_at_ms":89973,"miner_id":"miner-1","network_difficulty":1000012,"ntime":1700000199,"order_key":"01:miner-1","p2mr_program_hex":"c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7","share_difficulty":19,"share_id":"share-199","share_seq":199,"template_height":800006},{"accepted_at_ms":89990,"credit_policy":null,"job_id":"job-98","job_issued_at_ms":89970,"miner_id":"miner-10","network_difficulty":1000013,"ntime":1700000098,"order_key":"10:miner-10","p2mr_program_hex":"6262626262626262626262626262626262626262626262626262626262626262","share_difficulty":3,"share_id":"share-98","share_seq":98,"template_height":800003},{"accepted_at_ms":89974,"credit_policy":"stale-grace","job_id":"job-954","job_issued_at_ms":89988,"miner_id":"miner-8","network_difficulty":1000002,"ntime":1700000954,"order_key":"08:miner-8","p2mr_program_hex":"babababababababababababababababababababababababababababababababa","share_difficulty":2,"share_id":"share-954","share_seq":954,"template_height":800029},{"accepted_at_ms":90000,"credit_policy":null,"job_id":"job-1518","job_issued_at_ms":89975,"miner_id":"miner-0","network_difficulty":1000005,"ntime":1700001518,"order_key":"00:miner-0","p2mr_program_hex":"eeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeee","share_difficulty":1,"share_id":"share-1518","share_seq":1518,"template_height":800047},{"accepted_at_ms":89974,"credit_policy":null,"job_id":"job-415","job_issued_at_ms":89984,"miner_id":"miner-8","network_difficulty":1000007,"ntime":1700000415,"order_key":"08:miner-8","p2mr_program_hex":"9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f","share_difficulty":1,"share_id":"share-415","share_seq":415,"template_height":800012},{"accepted_at_ms":89970,"credit_policy":"stale-grace","job_id":"job-192","job_issued_at_ms":89980,"miner_id":"miner-5","network_difficulty":1000005,"ntime":1700000192,"order_key":"05:miner-5","p2mr_program_hex":"c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0","share_difficulty":2,"share_id":"share-192","share_seq":192,"template_height":800006},{"accepted_at_ms":89985,"credit_policy":null,"job_id":"job-1544","job_issued_at_ms":89984,"miner_id":"miner-4","network_difficulty":1000014,"ntime":1700001544,"order_key":"04:miner-4","p2mr_program_hex":"0808080808080808080808080808080808080808080808080808080808080808","share_difficulty":3,"share_id":"share-1544","share_seq":1544,"template_height":800048},{"accepted_at_ms":89995,"credit_policy":"stale-grace","job_id":"job-326","job_issued_at_ms":89983,"miner_id":"miner-7","network_difficulty":1000003,"ntime":1700000326,"order_key":"07:miner-7","p2mr_program_hex":"4646464646464646464646464646464646464646464646464646464646464646","share_difficulty":19,"share_id":"share-326","share_seq":326,"template_height":800010},{"accepted_at_ms":89995,"credit_policy":null,"job_id":"job-320","job_issued_at_ms":89978,"miner_id":"miner-1","network_difficulty":1000014,"ntime":1700000320,"order_key":"01:miner-1","p2mr_program_hex":"4040404040404040404040404040404040404040404040404040404040404040","share_difficulty":1,"share_id":"share-320","share_seq":320,"template_height":800010},{"accepted_at_ms":89988,"credit_policy":null,"job_id":"job-905","job_issued_at_ms":89974,"miner_id":"miner-3","network_difficulty":1000004,"ntime":1700000905,"order_key":"03:miner-3","p2mr_program_hex":"8989898989898989898989898989898989898989898989898989898989898989","share_difficulty":2,"share_id":"share-905","share_seq":905,"template_height":800028},{"accepted_at_ms":89980,"credit_policy":null,"job_id":"job-1680","job_issued_at_ms":89971,"miner_id":"miner-8","network_difficulty":1000014,"ntime":1700001680,"order_key":"08:miner-8","p2mr_program_hex":"9090909090909090909090909090909090909090909090909090909090909090","share_difficulty":2,"share_id":"share-1680","share_seq":1680,"template_height":800052},{"accepted_at_ms":89993,"credit_policy":null,"job_id":"job-980","job_issued_at_ms":89999,"miner_id":"miner-1","network_difficulty":1000011,"ntime":1700000980,"order_key":"01:miner-1","p2mr_program_hex":"d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4","share_difficulty":3,"share_id":"share-980","share_seq":980,"template_height":800030},{"accepted_at_ms":89987,"credit_policy":"stale-grace","job_id":"job-911","job_issued_at_ms":89990,"miner_id":"miner-9","network_difficulty":1000010,"ntime":1700000911,"order_key":"09:miner-9","p2mr_program_hex":"8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f","share_difficulty":2,"share_id":"share-911","share_seq":911,"template_height":800028},{"accepted_at_ms":89977,"credit_policy":"stale-grace","job_id":"job-578","job_issued_at_ms":89985,"miner_id":"miner-6","network_difficulty":1000000,"ntime":1700000578,"order_key":"06:miner-6","p2mr_program_hex":"4242424242424242424242424242424242424242424242424242424242424242","share_difficulty":7,"share_id":"share-578","share_seq":578,"template_height":800018},{"accepted_at_ms":89975,"credit_policy":null,"job_id":"job-786","job_issued_at_ms":89996,"miner_id":"miner-5","network_difficulty":1000004,"ntime":1700000786,"order_key":"05:miner-5","p2mr_program_hex":"1212121212121212121212121212121212121212121212121212121212121212","share_difficulty":19,"share_id":"share-786","share_seq":786,"template_height":800024},{"accepted_at_ms":89981,"credit_policy":"stale-grace","job_id":"job-1606","job_issued_at_ms":89990,"miner_id":"miner-0","network_difficulty":1000008,"ntime":1700001606,"order_key":"00:miner-0","p2mr_program_hex":"4646464646464646464646464646464646464646464646464646464646464646","share_difficulty":2,"share_id":"share-1606","share_seq":1606,"template_height":800050},{"accepted_at_ms":89982,"credit_policy":null,"job_id":"job-1038","job_issued_at_ms":89974,"miner_id":"miner-4","network_difficulty":1000001,"ntime":1700001038,"order_key":"04:miner-4","p2mr_program_hex":"0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e","share_difficulty":7,"share_id":"share-1038","share_seq":1038,"template_height":800032},{"accepted_at_ms":89971,"credit_policy":"stale-grace","job_id":"job-1178","job_issued_at_ms":89993,"miner_id":"miner-1","network_difficulty":1000005,"ntime":1700001178,"order_key":"01:miner-1","p2mr_program_hex":"9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a","share_difficulty":7,"share_id":"share-1178","share_seq":1178,"template_height":800036},{"accepted_at_ms":89995,"credit_policy":null,"job_id":"job-1479","job_issued_at_ms":89999,"miner_id":"miner-5","network_difficulty":1000000,"ntime":1700001479,"order_key":"05:miner-5","p2mr_program_hex":"c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7","share_difficulty":19,"share_id":"share-1479","share_seq":1479,"template_height":800046},{"accepted_at_ms":89995,"credit_policy":null,"job_id":"job-66","job_issued_at_ms":89994,"miner_id":"miner-0","network_difficulty":1000015,"ntime":1700000066,"order_key":"00:miner-0","p2mr_program_hex":"4242424242424242424242424242424242424242424242424242424242424242","share_difficulty":19,"share_id":"share-66","share_seq":66,"template_height":800002},{"accepted_at_ms":89975,"credit_policy":null,"job_id":"job-265","job_issued_at_ms":89983,"miner_id":"miner-1","network_difficulty":1000010,"ntime":1700000265,"order_key":"01:miner-1","p2mr_program_hex":"0909090909090909090909090909090909090909090909090909090909090909","share_difficulty":3,"share_id":"share-265","share_seq":265,"template_height":800008},{"accepted_at_ms":89970,"credit_policy":"stale-grace","job_id":"job-1503","job_issued_at_ms":89986,"miner_id":"miner-7","network_difficulty":1000007,"ntime":1700001503,"order_key":"07:miner-7","p2mr_program_hex":"dfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdf","share_difficulty":2,"share_id":"share-1503","share_seq":1503,"template_height":800046},{"accepted_at_ms":89985,"credit_policy":null,"job_id":"job-689","job_issued_at_ms":89970,"miner_id":"miner-7","network_difficulty":1000009,"ntime":1700000689,"order_key":"07:miner-7","p2mr_program_hex":"b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1","share_difficulty":19,"share_id":"share-689","share_seq":689,"template_height":800021},{"accepted_at_ms":89999,"credit_policy":null,"job_id":"job-1340","job_issued_at_ms":89992,"miner_id":"miner-9","network_difficulty":1000014,"ntime":1700001340,"order_key":"09:miner-9","p2mr_program_hex":"3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c","share_difficulty":7,"share_id":"share-1340","share_seq":1340,"template_height":800041},{"accepted_at_ms":89977,"credit_policy":null,"job_id":"job-1196","job_issued_at_ms":89998,"miner_id":"miner-8","network_difficulty":1000006,"ntime":1700001196,"order_key":"08:miner-8","p2mr_program_hex":"acacacacacacacacacacacacacacacacacacacacacacacacacacacacacacacac","share_difficulty":19,"share_id":"share-1196","share_seq":1196,"template_height":800037},{"accepted_at_ms":89998,"credit_policy":null,"job_id":"job-148","job_issued_at_ms":89998,"miner_id":"miner-5","network_difficulty":1000012,"ntime":1700000148,"order_key":"05:miner-5","p2mr_program_hex":"9494949494949494949494949494949494949494949494949494949494949494","share_difficulty":19,"share_id":"share-148","share_seq":148,"template_height":800004},{"accepted_at_ms":89996,"credit_policy":null,"job_id":"job-1521","job_issued_at_ms":89988,"miner_id":"miner-3","network_difficulty":1000008,"ntime":1700001521,"order_key":"03:miner-3","p2mr_program_hex":"f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1","share_difficulty":2,"share_id":"share-1521","share_seq":1521,"template_height":800047},{"accepted_at_ms":89981,"credit_policy":null,"job_id":"job-1284","job_issued_at_ms":89999,"miner_id":"miner-8","network_difficulty":1000009,"ntime":1700001284,"order_key":"08:miner-8","p2mr_program_hex":"0404040404040404040404040404040404040404040404040404040404040404","share_difficulty":3,"share_id":"share-1284","share_seq":1284,"template_height":800040},{"accepted_at_ms":89991,"credit_policy":null,"job_id":"job-346","job_issued_at_ms":89977,"miner_id":"miner-5","network_difficulty":1000006,"ntime":1700000346,"order_key":"05:miner-5","p2mr_program_hex":"5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a","share_difficulty":19,"share_id":"share-346","share_seq":346,"template_height":800010},{"accepted_at_ms":89991,"credit_policy":null,"job_id":"job-24","job_issued_at_ms":89992,"miner_id":"miner-2","network_difficulty":1000007,"ntime":1700000024,"order_key":"02:miner-2","p2mr_program_hex":"1818181818181818181818181818181818181818181818181818181818181818","share_difficulty":19,"share_id":"share-24","share_seq":24,"template_height":800000},{"accepted_at_ms":89973,"credit_policy":null,"job_id":"job-1404","job_issued_at_ms":89999,"miner_id":"miner-7","network_difficulty":1000010,"ntime":1700001404,"order_key":"07:miner-7","p2mr_program_hex":"7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c","share_difficulty":3,"share_id":"share-1404","share_seq":1404,"template_height":800043},{"accepted_at_ms":89997,"credit_policy":"stale-grace","job_id":"job-1260","job_issued_at_ms":89982,"miner_id":"miner-6","network_difficulty":1000002,"ntime":1700001260,"order_key":"06:miner-6","p2mr_program_hex":"ecececececececececececececececececececececececececececececececec","share_difficulty":3,"share_id":"share-1260","share_seq":1260,"template_height":800039},{"accepted_at_ms":89995,"credit_policy":null,"job_id":"job-1352","job_issued_at_ms":89984,"miner_id":"miner-10","network_difficulty":1000009,"ntime":1700001352,"order_key":"10:miner-10","p2mr_program_hex":"4848484848484848484848484848484848484848484848484848484848484848","share_difficulty":7,"share_id":"share-1352","share_seq":1352,"template_height":800042},{"accepted_at_ms":89992,"credit_policy":null,"job_id":"job-158","job_issued_at_ms":89991,"miner_id":"miner-4","network_difficulty":1000005,"ntime":1700000158,"order_key":"04:miner-4","p2mr_program_hex":"9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e","share_difficulty":19,"share_id":"share-158","share_seq":158,"template_height":800004},{"accepted_at_ms":89992,"credit_policy":"stale-grace","job_id":"job-1441","job_issued_at_ms":89988,"miner_id":"miner-0","network_difficulty":1000013,"ntime":1700001441,"order_key":"00:miner-0","p2mr_program_hex":"a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1","share_difficulty":3,"share_id":"share-1441","share_seq":1441,"template_height":800045},{"accepted_at_ms":89990,"credit_policy":"stale-grace","job_id":"job-533","job_issued_at_ms":89980,"miner_id":"miner-5","network_difficulty":1000006,"ntime":1700000533,"order_key":"05:miner-5","p2mr_program_hex":"1515151515151515151515151515151515151515151515151515151515151515","share_difficulty":1,"share_id":"share-533","share_seq":533,"template_height":800016},{"accepted_at_ms":89973,"credit_policy":null,"job_id":"job-1302","job_issued_at_ms":89995,"miner_id":"miner-4","network_difficulty":1000010,"ntime":1700001302,"order_key":"04:miner-4","p2mr_program_hex":"1616161616161616161616161616161616161616161616161616161616161616","share_difficulty":3,"share_id":"share-1302","share_seq":1302,"template_height":800040},{"accepted_at_ms":89995,"credit_policy":null,"job_id":"job-1000","job_issued_at_ms":89988,"miner_id":"miner-10","network_difficulty":1000014,"ntime":1700001000,"order_key":"10:miner-10","p2mr_program_hex":"e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8","share_difficulty":19,"share_id":"share-1000","share_seq":1000,"template_height":800031},{"accepted_at_ms":89979,"credit_policy":"stale-grace","job_id":"job-19","job_issued_at_ms":89987,"miner_id":"miner-8","network_difficulty":1000002,"ntime":1700000019,"order_key":"08:miner-8","p2mr_program_hex":"1313131313131313131313131313131313131313131313131313131313131313","share_difficulty":2,"share_id":"share-19","share_seq":19,"template_height":800000},{"accepted_at_ms":89985,"credit_policy":null,"job_id":"job-914","job_issued_at_ms":89975,"miner_id":"miner-1","network_difficulty":1000013,"ntime":1700000914,"order_key":"01:miner-1","p2mr_program_hex":"9292929292929292929292929292929292929292929292929292929292929292","share_difficulty":2,"share_id":"share-914","share_seq":914,"template_height":800028},{"accepted_at_ms":89972,"credit_policy":null,"job_id":"job-406","job_issued_at_ms":89991,"miner_id":"miner-10","network_difficulty":1000015,"ntime":1700000406,"order_key":"10:miner-10","p2mr_program_hex":"9696969696969696969696969696969696969696969696969696969696969696","share_difficulty":3,"share_id":"share-406","share_seq":406,"template_height":800012},{"accepted_at_ms":89988,"credit_policy":"stale-grace","job_id":"job-780","job_issued_at_ms":89996,"miner_id":"miner-10","network_difficulty":1000015,"ntime":1700000780,"order_key":"10:miner-10","p2mr_program_hex":"0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c","share_difficulty":19,"share_id":"share-780","share_seq":780,"template_height":800024},{"accepted_at_ms":89996,"credit_policy":null,"job_id":"job-12","job_issued_at_ms":89978,"miner_id":"miner-1","network_difficulty":1000012,"ntime":1700000012,"order_key":"01:miner-1","p2mr_program_hex":"0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c","share_difficulty":2,"share_id":"share-12","share_seq":12,"template_height":800000},{"accepted_at_ms":89970,"credit_policy":null,"job_id":"job-1495","job_issued_at_ms":89981,"miner_id":"miner-10","network_difficulty":1000016,"ntime":1700001495,"order_key":"10:miner-10","p2mr_program_hex":"d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7","share_difficulty":7,"share_id":"share-1495","share_seq":1495,"template_height":800046},{"accepted_at_ms":89984,"credit_policy":null,"job_id":"job-1411","job_issued_at_ms":89979,"miner_id":"miner-3","network_difficulty":1000000,"ntime":1700001411,"order_key":"03:miner-3","p2mr_program_hex":"8383838383838383838383838383838383838383838383838383838383838383","share_difficulty":1,"share_id":"share-1411","share_seq":1411,"template_height":800044},{"accepted_at_ms":89991,"credit_policy":null,"job_id":"job-924","job_issued_at_ms":89983,"miner_id":"miner-0","network_difficulty":1000006,"ntime":1700000924,"order_key":"00:miner-0","p2mr_program_hex":"9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c","share_difficulty":19,"share_id":"share-924","share_seq":924,"template_height":800028},{"accepted_at_ms":89994,"credit_policy":null,"job_id":"job-1590","job_issued_at_ms":89977,"miner_id":"miner-6","network_difficulty":1000009,"ntime":1700001590,"order_key":"06:miner-6","p2mr_program_hex":"3636363636363636363636363636363636363636363636363636363636363636","share_difficulty":3,"share_id":"share-1590","share_seq":1590,"template_height":800049},{"accepted_at_ms":89988,"credit_policy":"stale-grace","job_id":"job-1526","job_issued_at_ms":89981,"miner_id":"miner-8","network_difficulty":1000013,"ntime":1700001526,"order_key":"08:miner-8","p2mr_program_hex":"f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6","share_difficulty":7,"share_id":"share-1526","share_seq":1526,"template_height":800047},{"accepted_at_ms":89990,"credit_policy":"stale-grace","job_id":"job-1","job_issued_at_ms":89986,"miner_id":"miner-1","network_difficulty":1000001,"ntime":1700000001,"order_key":"01:miner-1","p2mr_program_hex":"0101010101010101010101010101010101010101010101010101010101010101","share_difficulty":3,"share_id":"share-1","share_seq":1,"template_height":800000},{"accepted_at_ms":89990,"credit_policy":"stale-grace","job_id":"job-1450","job_issued_at_ms":89993,"miner_id":"miner-9","network_difficulty":1000005,"ntime":1700001450,"order_key":"09:miner-9","p2mr_program_hex":"aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa","share_difficulty":1,"share_id":"share-1450","share_seq":1450,"template_height":800045},{"accepted_at_ms":89983,"credit_policy":null,"job_id":"job-1106","job_issued_at_ms":89976,"miner_id":"miner-6","network_difficulty":1000001,"ntime":1700001106,"order_key":"06:miner-6","p2mr_program_hex":"5252525252525252525252525252525252525252525252525252525252525252","share_difficulty":2,"share_id":"share-1106","share_seq":1106,"template_height":800034},{"accepted_at_ms":89997,"credit_policy":"stale-grace","job_id":"job-1691","job_issued_at_ms":89988,"miner_id":"miner-8","network_difficulty":1000008,"ntime":1700001691,"order_key":"08:miner-8","p2mr_program_hex":"9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b","share_difficulty":2,"share_id":"share-1691","share_seq":1691,"template_height":800052},{"accepted_at_ms":89981,"credit_policy":"stale-grace","job_id":"job-1655","job_issued_at_ms":89996,"miner_id":"miner-5","network_difficulty":1000006,"ntime":1700001655,"order_key":"05:miner-5","p2mr_program_hex":"7777777777777777777777777777777777777777777777777777777777777777","share_difficulty":1,"share_id":"share-1655","share_seq":1655,"template_height":800051},{"accepted_at_ms":89993,"credit_policy":null,"job_id":"job-631","job_issued_at_ms":89976,"miner_id":"miner-4","network_difficulty":1000002,"ntime":1700000631,"order_key":"04:miner-4","p2mr_program_hex":"7777777777777777777777777777777777777777777777777777777777777777","share_difficulty":19,"share_id":"share-631","share_seq":631,"template_height":800019},{"accepted_at_ms":89981,"credit_policy":null,"job_id":"job-659","job_issued_at_ms":89988,"miner_id":"miner-10","network_difficulty":1000013,"ntime":1700000659,"order_key":"10:miner-10","p2mr_program_hex":"9393939393939393939393939393939393939393939393939393939393939393","share_difficulty":1,"share_id":"share-659","share_seq":659,"template_height":800020},{"accepted_at_ms":89989,"credit_policy":null,"job_id":"job-513","job_issued_at_ms":89971,"miner_id":"miner-7","network_difficulty":1000003,"ntime":1700000513,"order_key":"07:miner-7","p2mr_program_hex":"0101010101010101010101010101010101010101010101010101010101010101","share_difficulty":7,"share_id":"share-513","share_seq":513,"template_height":800016},{"accepted_at_ms":89993,"credit_policy":"stale-grace","job_id":"job-1421","job_issued_at_ms":89979,"miner_id":"miner-2","network_difficulty":1000010,"ntime":1700001421,"order_key":"02:miner-2","p2mr_program_hex":"8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d","share_difficulty":3,"share_id":"share-1421","share_seq":1421,"template_height":800044},{"accepted_at_ms":89989,"credit_policy":null,"job_id":"job-1641","job_issued_at_ms":89993,"miner_id":"miner-2","network_difficulty":1000009,"ntime":1700001641,"order_key":"02:miner-2","p2mr_program_hex":"6969696969696969696969696969696969696969696969696969696969696969","share_difficulty":3,"share_id":"share-1641","share_seq":1641,"template_height":800051},{"accepted_at_ms":89999,"credit_policy":"stale-grace","job_id":"job-1013","job_issued_at_ms":89999,"miner_id":"miner-1","network_difficulty":1000010,"ntime":1700001013,"order_key":"01:miner-1","p2mr_program_hex":"f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5","share_difficulty":2,"share_id":"share-1013","share_seq":1013,"template_height":800031},{"accepted_at_ms":89985,"credit_policy":null,"job_id":"job-691","job_issued_at_ms":89973,"miner_id":"miner-9","network_difficulty":1000011,"ntime":1700000691,"order_key":"09:miner-9","p2mr_program_hex":"b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3","share_difficulty":7,"share_id":"share-691","share_seq":691,"template_height":800021},{"accepted_at_ms":89991,"credit_policy":null,"job_id":"job-179","job_issued_at_ms":89989,"miner_id":"miner-3","network_difficulty":1000009,"ntime":1700000179,"order_key":"03:miner-3","p2mr_program_hex":"b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3","share_difficulty":7,"share_id":"share-179","share_seq":179,"template_height":800005},{"accepted_at_ms":89984,"credit_policy":null,"job_id":"job-1654","job_issued_at_ms":89985,"miner_id":"miner-4","network_difficulty":1000005,"ntime":1700001654,"order_key":"04:miner-4","p2mr_program_hex":"7676767676767676767676767676767676767676767676767676767676767676","share_difficulty":1,"share_id":"share-1654","share_seq":1654,"template_height":800051},{"accepted_at_ms":89987,"credit_policy":"stale-grace","job_id":"job-441","job_issued_at_ms":89995,"miner_id":"miner-1","network_difficulty":1000016,"ntime":1700000441,"order_key":"01:miner-1","p2mr_program_hex":"b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9","share_difficulty":1,"share_id":"share-441","share_seq":441,"template_height":800013},{"accepted_at_ms":89973,"credit_policy":null,"job_id":"job-167","job_issued_at_ms":89976,"miner_id":"miner-2","network_difficulty":1000014,"ntime":1700000167,"order_key":"02:miner-2","p2mr_program_hex":"a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7","share_difficulty":3,"share_id":"share-167","share_seq":167,"template_height":800005},{"accepted_at_ms":89986,"credit_policy":null,"job_id":"job-362","job_issued_at_ms":89992,"miner_id":"miner-10","network_difficulty":1000005,"ntime":1700000362,"order_key":"10:miner-10","p2mr_program_hex":"6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a","share_difficulty":2,"share_id":"share-362","share_seq":362,"template_height":800011},{"accepted_at_ms":90000,"credit_policy":null,"job_id":"job-751","job_issued_at_ms":89976,"miner_id":"miner-3","network_difficulty":1000003,"ntime":1700000751,"order_key":"03:miner-3","p2mr_program_hex":"efefefefefefefefefefefefefefefefefefefefefefefefefefefefefefefef","share_difficulty":2,"share_id":"share-751","share_seq":751,"template_height":800023},{"accepted_at_ms":89990,"credit_policy":null,"job_id":"job-502","job_issued_at_ms":89995,"miner_id":"miner-7","network_difficulty":1000009,"ntime":1700000502,"order_key":"07:miner-7","p2mr_program_hex":"f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6","share_difficulty":2,"share_id":"share-502","share_seq":502,"template_height":800015},{"accepted_at_ms":89974,"credit_policy":null,"job_id":"job-1408","job_issued_at_ms":89991,"miner_id":"miner-0","network_difficulty":1000014,"ntime":1700001408,"order_key":"00:miner-0","p2mr_program_hex":"8080808080808080808080808080808080808080808080808080808080808080","share_difficulty":19,"share_id":"share-1408","share_seq":1408,"template_height":800044},{"accepted_at_ms":89974,"credit_policy":null,"job_id":"job-553","job_issued_at_ms":89992,"miner_id":"miner-3","network_difficulty":1000009,"ntime":1700000553,"order_key":"03:miner-3","p2mr_program_hex":"2929292929292929292929292929292929292929292929292929292929292929","share_difficulty":2,"share_id":"share-553","share_seq":553,"template_height":800017},{"accepted_at_ms":89999,"credit_policy":"stale-grace","job_id":"job-446","job_issued_at_ms":89987,"miner_id":"miner-6","network_difficulty":1000004,"ntime":1700000446,"order_key":"06:miner-6","p2mr_program_hex":"bebebebebebebebebebebebebebebebebebebebebebebebebebebebebebebebe","share_difficulty":1,"share_id":"share-446","share_seq":446,"template_height":800013},{"accepted_at_ms":89997,"credit_policy":null,"job_id":"job-885","job_issued_at_ms":89979,"miner_id":"miner-5","network_difficulty":1000001,"ntime":1700000885,"order_key":"05:miner-5","p2mr_program_hex":"7575757575757575757575757575757575757575757575757575757575757575","share_difficulty":7,"share_id":"share-885","share_seq":885,"template_height":800027},{"accepted_at_ms":89979,"credit_policy":"stale-grace","job_id":"job-1676","job_issued_at_ms":89972,"miner_id":"miner-4","network_difficulty":1000010,"ntime":1700001676,"order_key":"04:miner-4","p2mr_program_hex":"8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c","share_difficulty":1,"share_id":"share-1676","share_seq":1676,"template_height":800052},{"accepted_at_ms":89999,"credit_policy":null,"job_id":"job-1235","job_issued_at_ms":89973,"miner_id":"miner-3","network_difficulty":1000011,"ntime":1700001235,"order_key":"03:miner-3","p2mr_program_hex":"d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3","share_difficulty":2,"share_id":"share-1235","share_seq":1235,"template_height":800038},{"accepted_at_ms":89974,"credit_policy":null,"job_id":"job-719","job_issued_at_ms":89987,"miner_id":"miner-4","network_difficulty":1000005,"ntime":1700000719,"order_key":"04:miner-4","p2mr_program_hex":"cfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcf","share_difficulty":1,"share_id":"share-719","share_seq":719,"template_height":800022},{"accepted_at_ms":89973,"credit_policy":null,"job_id":"job-927","job_issued_at_ms":89976,"miner_id":"miner-3","network_difficulty":1000009,"ntime":1700000927,"order_key":"03:miner-3","p2mr_program_hex":"9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f","share_difficulty":7,"share_id":"share-927","share_seq":927,"template_height":800028},{"accepted_at_ms":89998,"credit_policy":"stale-grace","job_id":"job-1693","job_issued_at_ms":89989,"miner_id":"miner-10","network_difficulty":1000010,"ntime":1700001693,"order_key":"10:miner-10","p2mr_program_hex":"9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d","share_difficulty":3,"share_id":"share-1693","share_seq":1693,"template_height":800052},{"accepted_at_ms":89983,"credit_policy":null,"job_id":"job-35","job_issued_at_ms":89982,"miner_id":"miner-2","network_difficulty":1000001,"ntime":1700000035,"order_key":"02:miner-2","p2mr_program_hex":"2323232323232323232323232323232323232323232323232323232323232323","share_difficulty":3,"share_id":"share-35","share_seq":35,"template_height":800001},{"accepted_at_ms":89990,"credit_policy":null,"job_id":"job-1125","job_issued_at_ms":89981,"miner_id":"miner-3","network_difficulty":1000003,"ntime":1700001125,"order_key":"03:miner-3","p2mr_program_hex":"6565656565656565656565656565656565656565656565656565656565656565","share_difficulty":2,"share_id":"share-1125","share_seq":1125,"template_height":800035},{"accepted_at_ms":89970,"credit_policy":null,"job_id":"job-1764","job_issued_at_ms":89991,"miner_id":"miner-4","network_difficulty":1000013,"ntime":1700001764,"order_key":"04:miner-4","p2mr_program_hex":"e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4","share_difficulty":3,"share_id":"share-1764","share_seq":1764,"template_height":800055},{"accepted_at_ms":89972,"credit_policy":"stale-grace","job_id":"job-1394","job_issued_at_ms":89972,"miner_id":"miner-8","network_difficulty":1000000,"ntime":1700001394,"order_key":"08:miner-8","p2mr_program_hex":"7272727272727272727272727272727272727272727272727272727272727272","share_difficulty":2,"share_id":"share-1394","share_seq":1394,"template_height":800043},{"accepted_at_ms":89983,"credit_policy":null,"job_id":"job-1249","job_issued_at_ms":89981,"miner_id":"miner-6","network_difficulty":1000008,"ntime":1700001249,"order_key":"06:miner-6","p2mr_program_hex":"e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1","share_difficulty":2,"share_id":"share-1249","share_seq":1249,"template_height":800039},{"accepted_at_ms":89995,"credit_policy":null,"job_id":"job-1456","job_issued_at_ms":89975,"miner_id":"miner-4","network_difficulty":1000011,"ntime":1700001456,"order_key":"04:miner-4","p2mr_program_hex":"b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0","share_difficulty":3,"share_id":"share-1456","share_seq":1456,"template_height":800045},{"accepted_at_ms":89991,"credit_policy":null,"job_id":"job-1330","job_issued_at_ms":89977,"miner_id":"miner-10","network_difficulty":1000004,"ntime":1700001330,"order_key":"10:miner-10","p2mr_program_hex":"3232323232323232323232323232323232323232323232323232323232323232","share_difficulty":1,"share_id":"share-1330","share_seq":1330,"template_height":800041},{"accepted_at_ms":89984,"credit_policy":null,"job_id":"job-371","job_issued_at_ms":89997,"miner_id":"miner-8","network_difficulty":1000014,"ntime":1700000371,"order_key":"08:miner-8","p2mr_program_hex":"7373737373737373737373737373737373737373737373737373737373737373","share_difficulty":7,"share_id":"share-371","share_seq":371,"template_height":800011},{"accepted_at_ms":89985,"credit_policy":null,"job_id":"job-1222","job_issued_at_ms":89986,"miner_id":"miner-1","network_difficulty":1000015,"ntime":1700001222,"order_key":"01:miner-1","p2mr_program_hex":"c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6","share_difficulty":19,"share_id":"share-1222","share_seq":1222,"template_height":800038},{"accepted_at_ms":89975,"credit_policy":null,"job_id":"job-448","job_issued_at_ms":89993,"miner_id":"miner-8","network_difficulty":1000006,"ntime":1700000448,"order_key":"08:miner-8","p2mr_program_hex":"c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0","share_difficulty":19,"share_id":"share-448","share_seq":448,"template_height":800014},{"accepted_at_ms":89996,"credit_policy":null,"job_id":"job-1749","job_issued_at_ms":89994,"miner_id":"miner-0","network_difficulty":1000015,"ntime":1700001749,"order_key":"00:miner-0","p2mr_program_hex":"d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5","share_difficulty":19,"share_id":"share-1749","share_seq":1749,"template_height":800054},{"accepted_at_ms":89978,"credit_policy":null,"job_id":"job-600","job_issued_at_ms":89976,"miner_id":"miner-6","network_difficulty":1000005,"ntime":1700000600,"order_key":"06:miner-6","p2mr_program_hex":"5858585858585858585858585858585858585858585858585858585858585858","share_difficulty":19,"share_id":"share-600","share_seq":600,"template_height":800018},{"accepted_at_ms":89995,"credit_policy":null,"job_id":"job-1325","job_issued_at_ms":89978,"miner_id":"miner-5","network_difficulty":1000016,"ntime":1700001325,"order_key":"05:miner-5","p2mr_program_hex":"2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d","share_difficulty":19,"share_id":"share-1325","share_seq":1325,"template_height":800041},{"accepted_at_ms":89973,"credit_policy":null,"job_id":"job-1026","job_issued_at_ms":89992,"miner_id":"miner-3","network_difficulty":1000006,"ntime":1700001026,"order_key":"03:miner-3","p2mr_program_hex":"0202020202020202020202020202020202020202020202020202020202020202","share_difficulty":2,"share_id":"share-1026","share_seq":1026,"template_height":800032},{"accepted_at_ms":89971,"credit_policy":"stale-grace","job_id":"job-220","job_issued_at_ms":89993,"miner_id":"miner-0","network_difficulty":1000016,"ntime":1700000220,"order_key":"00:miner-0","p2mr_program_hex":"dcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdc","share_difficulty":19,"share_id":"share-220","share_seq":220,"template_height":800006},{"accepted_at_ms":89990,"credit_policy":null,"job_id":"job-1588","job_issued_at_ms":89990,"miner_id":"miner-4","network_difficulty":1000007,"ntime":1700001588,"order_key":"04:miner-4","p2mr_program_hex":"3434343434343434343434343434343434343434343434343434343434343434","share_difficulty":3,"share_id":"share-1588","share_seq":1588,"template_height":800049},{"accepted_at_ms":89984,"credit_policy":null,"job_id":"job-480","job_issued_at_ms":89974,"miner_id":"miner-7","network_difficulty":1000004,"ntime":1700000480,"order_key":"07:miner-7","p2mr_program_hex":"e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0","share_difficulty":1,"share_id":"share-480","share_seq":480,"template_height":800015},{"accepted_at_ms":89998,"credit_policy":null,"job_id":"job-335","job_issued_at_ms":89980,"miner_id":"miner-5","network_difficulty":1000012,"ntime":1700000335,"order_key":"05:miner-5","p2mr_program_hex":"4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f","share_difficulty":3,"share_id":"share-335","share_seq":335,"template_height":800010},{"accepted_at_ms":89988,"credit_policy":null,"job_id":"job-1058","job_issued_at_ms":89983,"miner_id":"miner-2","network_difficulty":1000004,"ntime":1700001058,"order_key":"02:miner-2","p2mr_program_hex":"2222222222222222222222222222222222222222222222222222222222222222","share_difficulty":1,"share_id":"share-1058","share_seq":1058,"template_height":800033},{"accepted_at_ms":89980,"credit_policy":"stale-grace","job_id":"job-1060","job_issued_at_ms":89976,"miner_id":"miner-4","network_difficulty":1000006,"ntime":1700001060,"order_key":"04:miner-4","p2mr_program_hex":"2424242424242424242424242424242424242424242424242424242424242424","share_difficulty":7,"share_id":"share-1060","share_seq":1060,"template_height":800033},{"accepted_at_ms":89981,"credit_policy":null,"job_id":"job-1210","job_issued_at_ms":89970,"miner_id":"miner-0","network_difficulty":1000003,"ntime":1700001210,"order_key":"00:miner-0","p2mr_program_hex":"babababababababababababababababababababababababababababababababa","share_difficulty":2,"share_id":"share-1210","share_seq":1210,"template_height":800037},{"accepted_at_ms":89996,"credit_policy":null,"job_id":"job-117","job_issued_at_ms":89996,"miner_id":"miner-7","network_difficulty":1000015,"ntime":1700000117,"order_key":"07:miner-7","p2mr_program_hex":"7575757575757575757575757575757575757575757575757575757575757575","share_difficulty":2,"share_id":"share-117","share_seq":117,"template_height":800003},{"accepted_at_ms":89996,"credit_policy":null,"job_id":"job-545","job_issued_at_ms":89987,"miner_id":"miner-6","network_difficulty":1000001,"ntime":1700000545,"order_key":"06:miner-6","p2mr_program_hex":"2121212121212121212121212121212121212121212121212121212121212121","share_difficulty":2,"share_id":"share-545","share_seq":545,"template_height":800017},{"accepted_at_ms":89976,"credit_policy":null,"job_id":"job-644","job_issued_at_ms":89978,"miner_id":"miner-6","network_difficulty":1000015,"ntime":1700000644,"order_key":"06:miner-6","p2mr_program_hex":"8484848484848484848484848484848484848484848484848484848484848484","share_difficulty":3,"share_id":"share-644","share_seq":644,"template_height":800020},{"accepted_at_ms":89996,"credit_policy":null,"job_id":"job-925","job_issued_at_ms":89981,"miner_id":"miner-1","network_difficulty":1000007,"ntime":1700000925,"order_key":"01:miner-1","p2mr_program_hex":"9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d","share_difficulty":19,"share_id":"share-925","share_seq":925,"template_height":800028},{"accepted_at_ms":89980,"credit_policy":null,"job_id":"job-282","job_issued_at_ms":89981,"miner_id":"miner-7","network_difficulty":1000010,"ntime":1700000282,"order_key":"07:miner-7","p2mr_program_hex":"1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a","share_difficulty":19,"share_id":"share-282","share_seq":282,"template_height":800008},{"accepted_at_ms":89994,"credit_policy":null,"job_id":"job-432","job_issued_at_ms":89980,"miner_id":"miner-3","network_difficulty":1000007,"ntime":1700000432,"order_key":"03:miner-3","p2mr_program_hex":"b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0","share_difficulty":19,"share_id":"share-432","share_seq":432,"template_height":800013},{"accepted_at_ms":89994,"credit_policy":null,"job_id":"job-1327","job_issued_at_ms":89990,"miner_id":"miner-7","network_difficulty":1000001,"ntime":1700001327,"order_key":"07:miner-7","p2mr_program_hex":"2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f","share_difficulty":3,"share_id":"share-1327","share_seq":1327,"template_height":800041},{"accepted_at_ms":89983,"credit_policy":null,"job_id":"job-870","job_issued_at_ms":89970,"miner_id":"miner-1","network_difficulty":1000003,"ntime":1700000870,"order_key":"01:miner-1","p2mr_program_hex":"6666666666666666666666666666666666666666666666666666666666666666","share_difficulty":19,"share_id":"share-870","share_seq":870,"template_height":800027},{"accepted_at_ms":89976,"credit_policy":null,"job_id":"job-1368","job_issued_at_ms":89985,"miner_id":"miner-4","network_difficulty":1000008,"ntime":1700001368,"order_key":"04:miner-4","p2mr_program_hex":"5858585858585858585858585858585858585858585858585858585858585858","share_difficulty":1,"share_id":"share-1368","share_seq":1368,"template_height":800042},{"accepted_at_ms":89970,"credit_policy":null,"job_id":"job-1108","job_issued_at_ms":89991,"miner_id":"miner-8","network_difficulty":1000003,"ntime":1700001108,"order_key":"08:miner-8","p2mr_program_hex":"5454545454545454545454545454545454545454545454545454545454545454","share_difficulty":3,"share_id":"share-1108","share_seq":1108,"template_height":800034},{"accepted_at_ms":89993,"credit_policy":"stale-grace","job_id":"job-760","job_issued_at_ms":89977,"miner_id":"miner-1","network_difficulty":1000012,"ntime":1700000760,"order_key":"01:miner-1","p2mr_program_hex":"f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8","share_difficulty":3,"share_id":"share-760","share_seq":760,"template_height":800023},{"accepted_at_ms":90000,"credit_policy":"stale-grace","job_id":"job-275","job_issued_at_ms":89990,"miner_id":"miner-0","network_difficulty":1000003,"ntime":1700000275,"order_key":"00:miner-0","p2mr_program_hex":"1313131313131313131313131313131313131313131313131313131313131313","share_difficulty":3,"share_id":"share-275","share_seq":275,"template_height":800008},{"accepted_at_ms":89971,"credit_policy":null,"job_id":"job-992","job_issued_at_ms":89993,"miner_id":"miner-2","network_difficulty":1000006,"ntime":1700000992,"order_key":"02:miner-2","p2mr_program_hex":"e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0","share_difficulty":1,"share_id":"share-992","share_seq":992,"template_height":800031},{"accepted_at_ms":89998,"credit_policy":null,"job_id":"job-485","job_issued_at_ms":89991,"miner_id":"miner-1","network_difficulty":1000009,"ntime":1700000485,"order_key":"01:miner-1","p2mr_program_hex":"e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5","share_difficulty":1,"share_id":"share-485","share_seq":485,"template_height":800015},{"accepted_at_ms":89999,"credit_policy":null,"job_id":"job-244","job_issued_at_ms":89979,"miner_id":"miner-2","network_difficulty":1000006,"ntime":1700000244,"order_key":"02:miner-2","p2mr_program_hex":"f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4","share_difficulty":2,"share_id":"share-244","share_seq":244,"template_height":800007},{"accepted_at_ms":89973,"credit_policy":null,"job_id":"job-917","job_issued_at_ms":89994,"miner_id":"miner-4","network_difficulty":1000016,"ntime":1700000917,"order_key":"04:miner-4","p2mr_program_hex":"9595959595959595959595959595959595959595959595959595959595959595","share_difficulty":19,"share_id":"share-917","share_seq":917,"template_height":800028},{"accepted_at_ms":89973,"credit_policy":null,"job_id":"job-1731","job_issued_at_ms":89978,"miner_id":"miner-4","network_difficulty":1000014,"ntime":1700001731,"order_key":"04:miner-4","p2mr_program_hex":"c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3","share_difficulty":7,"share_id":"share-1731","share_seq":1731,"template_height":800054},{"accepted_at_ms":89976,"credit_policy":null,"job_id":"job-735","job_issued_at_ms":89980,"miner_id":"miner-9","network_difficulty":1000004,"ntime":1700000735,"order_key":"09:miner-9","p2mr_program_hex":"dfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdf","share_difficulty":3,"share_id":"share-735","share_seq":735,"template_height":800022},{"accepted_at_ms":89976,"credit_policy":null,"job_id":"job-82","job_issued_at_ms":89980,"miner_id":"miner-5","network_difficulty":1000014,"ntime":1700000082,"order_key":"05:miner-5","p2mr_program_hex":"5252525252525252525252525252525252525252525252525252525252525252","share_difficulty":19,"share_id":"share-82","share_seq":82,"template_height":800002},{"accepted_at_ms":90000,"credit_policy":null,"job_id":"job-856","job_issued_at_ms":89986,"miner_id":"miner-9","network_difficulty":1000006,"ntime":1700000856,"order_key":"09:miner-9","p2mr_program_hex":"5858585858585858585858585858585858585858585858585858585858585858","share_difficulty":7,"share_id":"share-856","share_seq":856,"template_height":800026},{"accepted_at_ms":89978,"credit_policy":null,"job_id":"job-285","job_issued_at_ms":89973,"miner_id":"miner-10","network_difficulty":1000013,"ntime":1700000285,"order_key":"10:miner-10","p2mr_program_hex":"1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d","share_difficulty":7,"share_id":"share-285","share_seq":285,"template_height":800008},{"accepted_at_ms":89978,"credit_policy":null,"job_id":"job-1517","job_issued_at_ms":89997,"miner_id":"miner-10","network_difficulty":1000004,"ntime":1700001517,"order_key":"10:miner-10","p2mr_program_hex":"edededededededededededededededededededededededededededededededed","share_difficulty":19,"share_id":"share-1517","share_seq":1517,"template_height":800047},{"accepted_at_ms":89973,"credit_policy":null,"job_id":"job-778","job_issued_at_ms":89986,"miner_id":"miner-8","network_difficulty":1000013,"ntime":1700000778,"order_key":"08:miner-8","p2mr_program_hex":"0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a","share_difficulty":2,"share_id":"share-778","share_seq":778,"template_height":800024},{"accepted_at_ms":89998,"credit_policy":null,"job_id":"job-1466","job_issued_at_ms":89980,"miner_id":"miner-3","network_difficulty":1000004,"ntime":1700001466,"order_key":"03:miner-3","p2mr_program_hex":"babababababababababababababababababababababababababababababababa","share_difficulty":3,"share_id":"share-1466","share_seq":1466,"template_height":800045},{"accepted_at_ms":89971,"credit_policy":null,"job_id":"job-977","job_issued_at_ms":89998,"miner_id":"miner-9","network_difficulty":1000008,"ntime":1700000977,"order_key":"09:miner-9","p2mr_program_hex":"d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1","share_difficulty":1,"share_id":"share-977","share_seq":977,"template_height":800030},{"accepted_at_ms":90000,"credit_policy":null,"job_id":"job-1600","job_issued_at_ms":89974,"miner_id":"miner-5","network_difficulty":1000002,"ntime":1700001600,"order_key":"05:miner-5","p2mr_program_hex":"4040404040404040404040404040404040404040404040404040404040404040","share_difficulty":3,"share_id":"share-1600","share_seq":1600,"template_height":800050},{"accepted_at_ms":89982,"credit_policy":"stale-grace","job_id":"job-1259","job_issued_at_ms":89996,"miner_id":"miner-5","network_difficulty":1000001,"ntime":1700001259,"order_key":"05:miner-5","p2mr_program_hex":"ebebebebebebebebebebebebebebebebebebebebebebebebebebebebebebebeb","share_difficulty":3,"share_id":"share-1259","share_seq":1259,"template_height":800039},{"accepted_at_ms":89990,"credit_policy":null,"job_id":"job-1648","job_issued_at_ms":90000,"miner_id":"miner-9","network_difficulty":1000016,"ntime":1700001648,"order_key":"09:miner-9","p2mr_program_hex":"7070707070707070707070707070707070707070707070707070707070707070","share_difficulty":7,"share_id":"share-1648","share_seq":1648,"template_height":800051},{"accepted_at_ms":89983,"credit_policy":null,"job_id":"job-1032","job_issued_at_ms":89970,"miner_id":"miner-9","network_difficulty":1000012,"ntime":1700001032,"order_key":"09:miner-9","p2mr_program_hex":"0808080808080808080808080808080808080808080808080808080808080808","share_difficulty":1,"share_id":"share-1032","share_seq":1032,"template_height":800032},{"accepted_at_ms":89977,"credit_policy":null,"job_id":"job-445","job_issued_at_ms":89999,"miner_id":"miner-5","network_difficulty":1000003,"ntime":1700000445,"order_key":"05:miner-5","p2mr_program_hex":"bdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbd","share_difficulty":7,"share_id":"share-445","share_seq":445,"template_height":800013},{"accepted_at_ms":89996,"credit_policy":null,"job_id":"job-642","job_issued_at_ms":89986,"miner_id":"miner-4","network_difficulty":1000013,"ntime":1700000642,"order_key":"04:miner-4","p2mr_program_hex":"8282828282828282828282828282828282828282828282828282828282828282","share_difficulty":1,"share_id":"share-642","share_seq":642,"template_height":800020},{"accepted_at_ms":89984,"credit_policy":null,"job_id":"job-464","job_issued_at_ms":89988,"miner_id":"miner-2","network_difficulty":1000005,"ntime":1700000464,"order_key":"02:miner-2","p2mr_program_hex":"d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0","share_difficulty":7,"share_id":"share-464","share_seq":464,"template_height":800014},{"accepted_at_ms":89979,"credit_policy":null,"job_id":"job-1445","job_issued_at_ms":89988,"miner_id":"miner-4","network_difficulty":1000000,"ntime":1700001445,"order_key":"04:miner-4","p2mr_program_hex":"a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5","share_difficulty":1,"share_id":"share-1445","share_seq":1445,"template_height":800045},{"accepted_at_ms":89992,"credit_policy":null,"job_id":"job-716","job_issued_at_ms":89978,"miner_id":"miner-1","network_difficulty":1000002,"ntime":1700000716,"order_key":"01:miner-1","p2mr_program_hex":"cccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccc","share_difficulty":1,"share_id":"share-716","share_seq":716,"template_height":800022},{"accepted_at_ms":89975,"credit_policy":null,"job_id":"job-1769","job_issued_at_ms":89984,"miner_id":"miner-9","network_difficulty":1000001,"ntime":1700001769,"order_key":"09:miner-9","p2mr_program_hex":"e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9","share_difficulty":3,"share_id":"share-1769","share_seq":1769,"template_height":800055},{"accepted_at_ms":89978,"credit_policy":null,"job_id":"job-105","job_issued_at_ms":89990,"miner_id":"miner-6","network_difficulty":1000003,"ntime":1700000105,"order_key":"06:miner-6","p2mr_program_hex":"6969696969696969696969696969696969696969696969696969696969696969","share_difficulty":3,"share_id":"share-105","share_seq":105,"template_height":800003},{"accepted_at_ms":89973,"credit_policy":null,"job_id":"job-236","job_issued_at_ms":89982,"miner_id":"miner-5","network_difficulty":1000015,"ntime":1700000236,"order_key":"05:miner-5","p2mr_program_hex":"ecececececececececececececececececececececececececececececececec","share_difficulty":3,"share_id":"share-236","share_seq":236,"template_height":800007},{"accepted_at_ms":89971,"credit_policy":null,"job_id":"job-1586","job_issued_at_ms":89998,"miner_id":"miner-2","network_difficulty":1000005,"ntime":1700001586,"order_key":"02:miner-2","p2mr_program_hex":"3232323232323232323232323232323232323232323232323232323232323232","share_difficulty":1,"share_id":"share-1586","share_seq":1586,"template_height":800049},{"accepted_at_ms":89989,"credit_policy":null,"job_id":"job-238","job_issued_at_ms":89995,"miner_id":"miner-7","network_difficulty":1000000,"ntime":1700000238,"order_key":"07:miner-7","p2mr_program_hex":"eeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeee","share_difficulty":19,"share_id":"share-238","share_seq":238,"template_height":800007},{"accepted_at_ms":89973,"credit_policy":null,"job_id":"job-1326","job_issued_at_ms":89981,"miner_id":"miner-6","network_difficulty":1000000,"ntime":1700001326,"order_key":"06:miner-6","p2mr_program_hex":"2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e","share_difficulty":7,"share_id":"share-1326","share_seq":1326,"template_height":800041},{"accepted_at_ms":89981,"credit_policy":"stale-grace","job_id":"job-1375","job_issued_at_ms":89999,"miner_id":"miner-0","network_difficulty":1000015,"ntime":1700001375,"order_key":"00:miner-0","p2mr_program_hex":"5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f","share_difficulty":7,"share_id":"share-1375","share_seq":1375,"template_height":800042},{"accepted_at_ms":89993,"credit_policy":null,"job_id":"job-1766","job_issued_at_ms":89975,"miner_id":"miner-6","network_difficulty":1000015,"ntime":1700001766,"order_key":"06:miner-6","p2mr_program_hex":"e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6","share_difficulty":19,"share_id":"share-1766","share_seq":1766,"template_height":800055},{"accepted_at_ms":89986,"credit_policy":null,"job_id":"job-1716","job_issued_at_ms":89987,"miner_id":"miner-0","network_difficulty":1000016,"ntime":1700001716,"order_key":"00:miner-0","p2mr_program_hex":"b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4","share_difficulty":1,"share_id":"share-1716","share_seq":1716,"template_height":800053},{"accepted_at_ms":89985,"credit_policy":null,"job_id":"job-1627","job_issued_at_ms":89974,"miner_id":"miner-10","network_difficulty":1000012,"ntime":1700001627,"order_key":"10:miner-10","p2mr_program_hex":"5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b","share_difficulty":3,"share_id":"share-1627","share_seq":1627,"template_height":800050},{"accepted_at_ms":89973,"credit_policy":null,"job_id":"job-375","job_issued_at_ms":89997,"miner_id":"miner-1","network_difficulty":1000001,"ntime":1700000375,"order_key":"01:miner-1","p2mr_program_hex":"7777777777777777777777777777777777777777777777777777777777777777","share_difficulty":3,"share_id":"share-375","share_seq":375,"template_height":800011},{"accepted_at_ms":90000,"credit_policy":null,"job_id":"job-1563","job_issued_at_ms":89976,"miner_id":"miner-1","network_difficulty":1000016,"ntime":1700001563,"order_key":"01:miner-1","p2mr_program_hex":"1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b","share_difficulty":3,"share_id":"share-1563","share_seq":1563,"template_height":800048},{"accepted_at_ms":89979,"credit_policy":"stale-grace","job_id":"job-998","job_issued_at_ms":89982,"miner_id":"miner-8","network_difficulty":1000012,"ntime":1700000998,"order_key":"08:miner-8","p2mr_program_hex":"e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6","share_difficulty":3,"share_id":"share-998","share_seq":998,"template_height":800031},{"accepted_at_ms":89974,"credit_policy":null,"job_id":"job-1311","job_issued_at_ms":89988,"miner_id":"miner-2","network_difficulty":1000002,"ntime":1700001311,"order_key":"02:miner-2","p2mr_program_hex":"1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f","share_difficulty":3,"share_id":"share-1311","share_seq":1311,"template_height":800040},{"accepted_at_ms":90000,"credit_policy":"stale-grace","job_id":"job-479","job_issued_at_ms":89972,"miner_id":"miner-6","network_difficulty":1000003,"ntime":1700000479,"order_key":"06:miner-6","p2mr_program_hex":"dfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdf","share_difficulty":7,"share_id":"share-479","share_seq":479,"template_height":800014},{"accepted_at_ms":89972,"credit_policy":null,"job_id":"job-1543","job_issued_at_ms":89985,"miner_id":"miner-3","network_difficulty":1000013,"ntime":1700001543,"order_key":"03:miner-3","p2mr_program_hex":"0707070707070707070707070707070707070707070707070707070707070707","share_difficulty":2,"share_id":"share-1543","share_seq":1543,"template_height":800048},{"accepted_at_ms":89990,"credit_policy":null,"job_id":"job-1234","job_issued_at_ms":89997,"miner_id":"miner-2","network_difficulty":1000010,"ntime":1700001234,"order_key":"02:miner-2","p2mr_program_hex":"d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2","share_difficulty":2,"share_id":"share-1234","share_seq":1234,"template_height":800038},{"accepted_at_ms":89980,"credit_policy":null,"job_id":"job-1567","job_issued_at_ms":89983,"miner_id":"miner-5","network_difficulty":1000003,"ntime":1700001567,"order_key":"05:miner-5","p2mr_program_hex":"1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f","share_difficulty":2,"share_id":"share-1567","share_seq":1567,"template_height":800048},{"accepted_at_ms":89978,"credit_policy":null,"job_id":"job-151","job_issued_at_ms":89971,"miner_id":"miner-8","network_difficulty":1000015,"ntime":1700000151,"order_key":"08:miner-8","p2mr_program_hex":"9797979797979797979797979797979797979797979797979797979797979797","share_difficulty":2,"share_id":"share-151","share_seq":151,"template_height":800004},{"accepted_at_ms":89986,"credit_policy":null,"job_id":"job-1371","job_issued_at_ms":89990,"miner_id":"miner-7","network_difficulty":1000011,"ntime":1700001371,"order_key":"07:miner-7","p2mr_program_hex":"5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b","share_difficulty":19,"share_id":"share-1371","share_seq":1371,"template_height":800042},{"accepted_at_ms":89975,"credit_policy":"stale-grace","job_id":"job-163","job_issued_at_ms":89988,"miner_id":"miner-9","network_difficulty":1000010,"ntime":1700000163,"order_key":"09:miner-9","p2mr_program_hex":"a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3","share_difficulty":7,"share_id":"share-163","share_seq":163,"template_height":800005},{"accepted_at_ms":89972,"credit_policy":null,"job_id":"job-176","job_issued_at_ms":89976,"miner_id":"miner-0","network_difficulty":1000006,"ntime":1700000176,"order_key":"00:miner-0","p2mr_program_hex":"b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0","share_difficulty":1,"share_id":"share-176","share_seq":176,"template_height":800005},{"accepted_at_ms":89990,"credit_policy":null,"job_id":"job-1556","job_issued_at_ms":89997,"miner_id":"miner-5","network_difficulty":1000009,"ntime":1700001556,"order_key":"05:miner-5","p2mr_program_hex":"1414141414141414141414141414141414141414141414141414141414141414","share_difficulty":2,"share_id":"share-1556","share_seq":1556,"template_height":800048},{"accepted_at_ms":89990,"credit_policy":"stale-grace","job_id":"job-1278","job_issued_at_ms":89994,"miner_id":"miner-2","network_difficulty":1000003,"ntime":1700001278,"order_key":"02:miner-2","p2mr_program_hex":"fefefefefefefefefefefefefefefefefefefefefefefefefefefefefefefefe","share_difficulty":2,"share_id":"share-1278","share_seq":1278,"template_height":800039},{"accepted_at_ms":89988,"credit_policy":"stale-grace","job_id":"job-153","job_issued_at_ms":89996,"miner_id":"miner-10","network_difficulty":1000000,"ntime":1700000153,"order_key":"10:miner-10","p2mr_program_hex":"9999999999999999999999999999999999999999999999999999999999999999","share_difficulty":19,"share_id":"share-153","share_seq":153,"template_height":800004},{"accepted_at_ms":89974,"credit_policy":null,"job_id":"job-1547","job_issued_at_ms":89977,"miner_id":"miner-7","network_difficulty":1000000,"ntime":1700001547,"order_key":"07:miner-7","p2mr_program_hex":"0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b","share_difficulty":7,"share_id":"share-1547","share_seq":1547,"template_height":800048},{"accepted_at_ms":89987,"credit_policy":"stale-grace","job_id":"job-621","job_issued_at_ms":89976,"miner_id":"miner-5","network_difficulty":1000009,"ntime":1700000621,"order_key":"05:miner-5","p2mr_program_hex":"6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d","share_difficulty":2,"share_id":"share-621","share_seq":621,"template_height":800019},{"accepted_at_ms":89989,"credit_policy":null,"job_id":"job-5","job_issued_at_ms":89981,"miner_id":"miner-5","network_difficulty":1000005,"ntime":1700000005,"order_key":"05:miner-5","p2mr_program_hex":"0505050505050505050505050505050505050505050505050505050505050505","share_difficulty":1,"share_id":"share-5","share_seq":5,"template_height":800000},{"accepted_at_ms":89999,"credit_policy":null,"job_id":"job-248","job_issued_at_ms":89983,"miner_id":"miner-6","network_difficulty":1000010,"ntime":1700000248,"order_key":"06:miner-6","p2mr_program_hex":"f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8","share_difficulty":3,"share_id":"share-248","share_seq":248,"template_height":800007},{"accepted_at_ms":89979,"credit_policy":null,"job_id":"job-552","job_issued_at_ms":89984,"miner_id":"miner-2","network_difficulty":1000008,"ntime":1700000552,"order_key":"02:miner-2","p2mr_program_hex":"2828282828282828282828282828282828282828282828282828282828282828","share_difficulty":1,"share_id":"share-552","share_seq":552,"template_height":800017},{"accepted_at_ms":89985,"credit_policy":null,"job_id":"job-1476","job_issued_at_ms":89985,"miner_id":"miner-2","network_difficulty":1000014,"ntime":1700001476,"order_key":"02:miner-2","p2mr_program_hex":"c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4","share_difficulty":7,"share_id":"share-1476","share_seq":1476,"template_height":800046},{"accepted_at_ms":89986,"credit_policy":null,"job_id":"job-1155","job_issued_at_ms":89994,"miner_id":"miner-0","network_difficulty":1000016,"ntime":1700001155,"order_key":"00:miner-0","p2mr_program_hex":"8383838383838383838383838383838383838383838383838383838383838383","share_difficulty":19,"share_id":"share-1155","share_seq":1155,"template_height":800036},{"accepted_at_ms":89997,"credit_policy":"stale-grace","job_id":"job-1251","job_issued_at_ms":89997,"miner_id":"miner-8","network_difficulty":1000010,"ntime":1700001251,"order_key":"08:miner-8","p2mr_program_hex":"e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3","share_difficulty":3,"share_id":"share-1251","share_seq":1251,"template_height":800039},{"accepted_at_ms":89989,"credit_policy":null,"job_id":"job-52","job_issued_at_ms":89985,"miner_id":"miner-8","network_difficulty":1000001,"ntime":1700000052,"order_key":"08:miner-8","p2mr_program_hex":"3434343434343434343434343434343434343434343434343434343434343434","share_difficulty":2,"share_id":"share-52","share_seq":52,"template_height":800001},{"accepted_at_ms":89981,"credit_policy":"stale-grace","job_id":"job-1434","job_issued_at_ms":89998,"miner_id":"miner-4","network_difficulty":1000006,"ntime":1700001434,"order_key":"04:miner-4","p2mr_program_hex":"9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a","share_difficulty":19,"share_id":"share-1434","share_seq":1434,"template_height":800044},{"accepted_at_ms":89976,"credit_policy":"stale-grace","job_id":"job-1515","job_issued_at_ms":89993,"miner_id":"miner-8","network_difficulty":1000002,"ntime":1700001515,"order_key":"08:miner-8","p2mr_program_hex":"ebebebebebebebebebebebebebebebebebebebebebebebebebebebebebebebeb","share_difficulty":7,"share_id":"share-1515","share_seq":1515,"template_height":800047},{"accepted_at_ms":89982,"credit_policy":null,"job_id":"job-1626","job_issued_at_ms":89989,"miner_id":"miner-9","network_difficulty":1000011,"ntime":1700001626,"order_key":"09:miner-9","p2mr_program_hex":"5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a","share_difficulty":3,"share_id":"share-1626","share_seq":1626,"template_height":800050},{"accepted_at_ms":90000,"credit_policy":"stale-grace","job_id":"job-334","job_issued_at_ms":89996,"miner_id":"miner-4","network_difficulty":1000011,"ntime":1700000334,"order_key":"04:miner-4","p2mr_program_hex":"4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e","share_difficulty":1,"share_id":"share-334","share_seq":334,"template_height":800010},{"accepted_at_ms":89978,"credit_policy":null,"job_id":"job-1799","job_issued_at_ms":89985,"miner_id":"miner-6","network_difficulty":1000014,"ntime":1700001799,"order_key":"06:miner-6","p2mr_program_hex":"0707070707070707070707070707070707070707070707070707070707070707","share_difficulty":7,"share_id":"share-1799","share_seq":1799,"template_height":800056},{"accepted_at_ms":89977,"credit_policy":null,"job_id":"job-408","job_issued_at_ms":89990,"miner_id":"miner-1","network_difficulty":1000000,"ntime":1700000408,"order_key":"01:miner-1","p2mr_program_hex":"9898989898989898989898989898989898989898989898989898989898989898","share_difficulty":7,"share_id":"share-408","share_seq":408,"template_height":800012},{"accepted_at_ms":89999,"credit_policy":"stale-grace","job_id":"job-632","job_issued_at_ms":89984,"miner_id":"miner-5","network_difficulty":1000003,"ntime":1700000632,"order_key":"05:miner-5","p2mr_program_hex":"7878787878787878787878787878787878787878787878787878787878787878","share_difficulty":7,"share_id":"share-632","share_seq":632,"template_height":800019},{"accepted_at_ms":89984,"credit_policy":"stale-grace","job_id":"job-679","job_issued_at_ms":90000,"miner_id":"miner-8","network_difficulty":1000016,"ntime":1700000679,"order_key":"08:miner-8","p2mr_program_hex":"a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7","share_difficulty":3,"share_id":"share-679","share_seq":679,"template_height":800021},{"accepted_at_ms":89974,"credit_policy":null,"job_id":"job-555","job_issued_at_ms":89972,"miner_id":"miner-5","network_difficulty":1000011,"ntime":1700000555,"order_key":"05:miner-5","p2mr_program_hex":"2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b","share_difficulty":3,"share_id":"share-555","share_seq":555,"template_height":800017},{"accepted_at_ms":89980,"credit_policy":"stale-grace","job_id":"job-203","job_issued_at_ms":89994,"miner_id":"miner-5","network_difficulty":1000016,"ntime":1700000203,"order_key":"05:miner-5","p2mr_program_hex":"cbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcb","share_difficulty":19,"share_id":"share-203","share_seq":203,"template_height":800006},{"accepted_at_ms":89990,"credit_policy":null,"job_id":"job-74","job_issued_at_ms":89985,"miner_id":"miner-8","network_difficulty":1000006,"ntime":1700000074,"order_key":"08:miner-8","p2mr_program_hex":"4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a","share_difficulty":3,"share_id":"share-74","share_seq":74,"template_height":800002},{"accepted_at_ms":89985,"credit_policy":null,"job_id":"job-796","job_issued_at_ms":89993,"miner_id":"miner-4","network_difficulty":1000014,"ntime":1700000796,"order_key":"04:miner-4","p2mr_program_hex":"1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c","share_difficulty":7,"share_id":"share-796","share_seq":796,"template_height":800024},{"accepted_at_ms":89998,"credit_policy":null,"job_id":"job-1314","job_issued_at_ms":89986,"miner_id":"miner-5","network_difficulty":1000005,"ntime":1700001314,"order_key":"05:miner-5","p2mr_program_hex":"2222222222222222222222222222222222222222222222222222222222222222","share_difficulty":19,"share_id":"share-1314","share_seq":1314,"template_height":800041},{"accepted_at_ms":89989,"credit_policy":null,"job_id":"job-718","job_issued_at_ms":89998,"miner_id":"miner-3","network_difficulty":1000004,"ntime":1700000718,"order_key":"03:miner-3","p2mr_program_hex":"cececececececececececececececececececececececececececececececece","share_difficulty":2,"share_id":"share-718","share_seq":718,"template_height":800022},{"accepted_at_ms":89985,"credit_policy":null,"job_id":"job-1797","job_issued_at_ms":89982,"miner_id":"miner-4","network_difficulty":1000012,"ntime":1700001797,"order_key":"04:miner-4","p2mr_program_hex":"0505050505050505050505050505050505050505050505050505050505050505","share_difficulty":1,"share_id":"share-1797","share_seq":1797,"template_height":800056},{"accepted_at_ms":89975,"credit_policy":null,"job_id":"job-27","job_issued_at_ms":89979,"miner_id":"miner-5","network_difficulty":1000010,"ntime":1700000027,"order_key":"05:miner-5","p2mr_program_hex":"1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b","share_difficulty":7,"share_id":"share-27","share_seq":27,"template_height":800000},{"accepted_at_ms":89974,"credit_policy":null,"job_id":"job-792","job_issued_at_ms":89977,"miner_id":"miner-0","network_difficulty":1000010,"ntime":1700000792,"order_key":"00:miner-0","p2mr_program_hex":"1818181818181818181818181818181818181818181818181818181818181818","share_difficulty":2,"share_id":"share-792","share_seq":792,"template_height":800024},{"accepted_at_ms":89998,"credit_policy":null,"job_id":"job-1201","job_issued_at_ms":89993,"miner_id":"miner-2","network_difficulty":1000011,"ntime":1700001201,"order_key":"02:miner-2","p2mr_program_hex":"b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1","share_difficulty":7,"share_id":"share-1201","share_seq":1201,"template_height":800037},{"accepted_at_ms":89990,"credit_policy":null,"job_id":"job-781","job_issued_at_ms":89971,"miner_id":"miner-0","network_difficulty":1000016,"ntime":1700000781,"order_key":"00:miner-0","p2mr_program_hex":"0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d","share_difficulty":19,"share_id":"share-781","share_seq":781,"template_height":800024},{"accepted_at_ms":89997,"credit_policy":null,"job_id":"job-1542","job_issued_at_ms":90000,"miner_id":"miner-2","network_difficulty":1000012,"ntime":1700001542,"order_key":"02:miner-2","p2mr_program_hex":"0606060606060606060606060606060606060606060606060606060606060606","share_difficulty":1,"share_id":"share-1542","share_seq":1542,"template_height":800048},{"accepted_at_ms":89971,"credit_policy":null,"job_id":"job-1241","job_issued_at_ms":89981,"miner_id":"miner-9","network_difficulty":1000000,"ntime":1700001241,"order_key":"09:miner-9","p2mr_program_hex":"d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9","share_difficulty":7,"share_id":"share-1241","share_seq":1241,"template_height":800038},{"accepted_at_ms":89992,"credit_policy":"stale-grace","job_id":"job-538","job_issued_at_ms":89990,"miner_id":"miner-10","network_difficulty":1000011,"ntime":1700000538,"order_key":"10:miner-10","p2mr_program_hex":"1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a","share_difficulty":19,"share_id":"share-538","share_seq":538,"template_height":800016},{"accepted_at_ms":89990,"credit_policy":null,"job_id":"job-142","job_issued_at_ms":89991,"miner_id":"miner-10","network_difficulty":1000006,"ntime":1700000142,"order_key":"10:miner-10","p2mr_program_hex":"8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e","share_difficulty":2,"share_id":"share-142","share_seq":142,"template_height":800004},{"accepted_at_ms":89984,"credit_policy":null,"job_id":"job-315","job_issued_at_ms":89991,"miner_id":"miner-7","network_difficulty":1000009,"ntime":1700000315,"order_key":"07:miner-7","p2mr_program_hex":"3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b","share_difficulty":3,"share_id":"share-315","share_seq":315,"template_height":800009},{"accepted_at_ms":89981,"credit_policy":null,"job_id":"job-731","job_issued_at_ms":89987,"miner_id":"miner-5","network_difficulty":1000000,"ntime":1700000731,"order_key":"05:miner-5","p2mr_program_hex":"dbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdb","share_difficulty":7,"share_id":"share-731","share_seq":731,"template_height":800022},{"accepted_at_ms":89990,"credit_policy":null,"job_id":"job-488","job_issued_at_ms":89982,"miner_id":"miner-4","network_difficulty":1000012,"ntime":1700000488,"order_key":"04:miner-4","p2mr_program_hex":"e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8","share_difficulty":7,"share_id":"share-488","share_seq":488,"template_height":800015},{"accepted_at_ms":89985,"credit_policy":null,"job_id":"job-349","job_issued_at_ms":89986,"miner_id":"miner-8","network_difficulty":1000009,"ntime":1700000349,"order_key":"08:miner-8","p2mr_program_hex":"5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d","share_difficulty":2,"share_id":"share-349","share_seq":349,"template_height":800010},{"accepted_at_ms":89973,"credit_policy":null,"job_id":"job-71","job_issued_at_ms":89975,"miner_id":"miner-5","network_difficulty":1000003,"ntime":1700000071,"order_key":"05:miner-5","p2mr_program_hex":"4747474747474747474747474747474747474747474747474747474747474747","share_difficulty":7,"share_id":"share-71","share_seq":71,"template_height":800002},{"accepted_at_ms":89981,"credit_policy":"stale-grace","job_id":"job-1678","job_issued_at_ms":89984,"miner_id":"miner-6","network_difficulty":1000012,"ntime":1700001678,"order_key":"06:miner-6","p2mr_program_hex":"8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e","share_difficulty":2,"share_id":"share-1678","share_seq":1678,"template_height":800052},{"accepted_at_ms":89995,"credit_policy":null,"job_id":"job-1037","job_issued_at_ms":89972,"miner_id":"miner-3","network_difficulty":1000000,"ntime":1700001037,"order_key":"03:miner-3","p2mr_program_hex":"0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d","share_difficulty":1,"share_id":"share-1037","share_seq":1037,"template_height":800032},{"accepted_at_ms":89985,"credit_policy":null,"job_id":"job-1697","job_issued_at_ms":89979,"miner_id":"miner-3","network_difficulty":1000014,"ntime":1700001697,"order_key":"03:miner-3","p2mr_program_hex":"a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1","share_difficulty":2,"share_id":"share-1697","share_seq":1697,"template_height":800053},{"accepted_at_ms":89996,"credit_policy":null,"job_id":"job-1221","job_issued_at_ms":89973,"miner_id":"miner-0","network_difficulty":1000014,"ntime":1700001221,"order_key":"00:miner-0","p2mr_program_hex":"c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5","share_difficulty":1,"share_id":"share-1221","share_seq":1221,"template_height":800038},{"accepted_at_ms":89991,"credit_policy":"stale-grace","job_id":"job-482","job_issued_at_ms":89997,"miner_id":"miner-9","network_difficulty":1000006,"ntime":1700000482,"order_key":"09:miner-9","p2mr_program_hex":"e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2","share_difficulty":7,"share_id":"share-482","share_seq":482,"template_height":800015},{"accepted_at_ms":89980,"credit_policy":null,"job_id":"job-1743","job_issued_at_ms":89985,"miner_id":"miner-5","network_difficulty":1000009,"ntime":1700001743,"order_key":"05:miner-5","p2mr_program_hex":"cfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcf","share_difficulty":3,"share_id":"share-1743","share_seq":1743,"template_height":800054},{"accepted_at_ms":89974,"credit_policy":"stale-grace","job_id":"job-882","job_issued_at_ms":89989,"miner_id":"miner-2","network_difficulty":1000015,"ntime":1700000882,"order_key":"02:miner-2","p2mr_program_hex":"7272727272727272727272727272727272727272727272727272727272727272","share_difficulty":2,"share_id":"share-882","share_seq":882,"template_height":800027},{"accepted_at_ms":89997,"credit_policy":null,"job_id":"job-828","job_issued_at_ms":89985,"miner_id":"miner-3","network_difficulty":1000012,"ntime":1700000828,"order_key":"03:miner-3","p2mr_program_hex":"3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c","share_difficulty":7,"share_id":"share-828","share_seq":828,"template_height":800025},{"accepted_at_ms":89998,"credit_policy":null,"job_id":"job-227","job_issued_at_ms":89977,"miner_id":"miner-7","network_difficulty":1000006,"ntime":1700000227,"order_key":"07:miner-7","p2mr_program_hex":"e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3","share_difficulty":1,"share_id":"share-227","share_seq":227,"template_height":800007},{"accepted_at_ms":89980,"credit_policy":null,"job_id":"job-993","job_issued_at_ms":89982,"miner_id":"miner-3","network_difficulty":1000007,"ntime":1700000993,"order_key":"03:miner-3","p2mr_program_hex":"e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1","share_difficulty":1,"share_id":"share-993","share_seq":993,"template_height":800031},{"accepted_at_ms":89984,"credit_policy":null,"job_id":"job-725","job_issued_at_ms":89985,"miner_id":"miner-10","network_difficulty":1000011,"ntime":1700000725,"order_key":"10:miner-10","p2mr_program_hex":"d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5","share_difficulty":1,"share_id":"share-725","share_seq":725,"template_height":800022},{"accepted_at_ms":89984,"credit_policy":null,"job_id":"job-264","job_issued_at_ms":89983,"miner_id":"miner-0","network_difficulty":1000009,"ntime":1700000264,"order_key":"00:miner-0","p2mr_program_hex":"0808080808080808080808080808080808080808080808080808080808080808","share_difficulty":3,"share_id":"share-264","share_seq":264,"template_height":800008},{"accepted_at_ms":89993,"credit_policy":null,"job_id":"job-75","job_issued_at_ms":89987,"miner_id":"miner-9","network_difficulty":1000007,"ntime":1700000075,"order_key":"09:miner-9","p2mr_program_hex":"4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b","share_difficulty":1,"share_id":"share-75","share_seq":75,"template_height":800002},{"accepted_at_ms":89977,"credit_policy":null,"job_id":"job-1333","job_issued_at_ms":89986,"miner_id":"miner-2","network_difficulty":1000007,"ntime":1700001333,"order_key":"02:miner-2","p2mr_program_hex":"3535353535353535353535353535353535353535353535353535353535353535","share_difficulty":19,"share_id":"share-1333","share_seq":1333,"template_height":800041},{"accepted_at_ms":89989,"credit_policy":"stale-grace","job_id":"job-785","job_issued_at_ms":89982,"miner_id":"miner-4","network_difficulty":1000003,"ntime":1700000785,"order_key":"04:miner-4","p2mr_program_hex":"1111111111111111111111111111111111111111111111111111111111111111","share_difficulty":3,"share_id":"share-785","share_seq":785,"template_height":800024},{"accepted_at_ms":89979,"credit_policy":null,"job_id":"job-1258","job_issued_at_ms":89979,"miner_id":"miner-4","network_difficulty":1000000,"ntime":1700001258,"order_key":"04:miner-4","p2mr_program_hex":"eaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaea","share_difficulty":19,"share_id":"share-1258","share_seq":1258,"template_height":800039},{"accepted_at_ms":89985,"credit_policy":null,"job_id":"job-742","job_issued_at_ms":89976,"miner_id":"miner-5","network_difficulty":1000011,"ntime":1700000742,"order_key":"05:miner-5","p2mr_program_hex":"e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6","share_difficulty":1,"share_id":"share-742","share_seq":742,"template_height":800023},{"accepted_at_ms":89988,"credit_policy":null,"job_id":"job-1145","job_issued_at_ms":89980,"miner_id":"miner-1","network_difficulty":1000006,"ntime":1700001145,"order_key":"01:miner-1","p2mr_program_hex":"7979797979797979797979797979797979797979797979797979797979797979","share_difficulty":19,"share_id":"share-1145","share_seq":1145,"template_height":800035},{"accepted_at_ms":89980,"credit_policy":"stale-grace","job_id":"job-1200","job_issued_at_ms":89972,"miner_id":"miner-1","network_difficulty":1000010,"ntime":1700001200,"order_key":"01:miner-1","p2mr_program_hex":"b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0","share_difficulty":1,"share_id":"share-1200","share_seq":1200,"template_height":800037},{"accepted_at_ms":89979,"credit_policy":null,"job_id":"job-242","job_issued_at_ms":89985,"miner_id":"miner-0","network_difficulty":1000004,"ntime":1700000242,"order_key":"00:miner-0","p2mr_program_hex":"f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2","share_difficulty":3,"share_id":"share-242","share_seq":242,"template_height":800007},{"accepted_at_ms":89998,"credit_policy":null,"job_id":"job-974","job_issued_at_ms":89979,"miner_id":"miner-6","network_difficulty":1000005,"ntime":1700000974,"order_key":"06:miner-6","p2mr_program_hex":"cececececececececececececececececececececececececececececececece","share_difficulty":2,"share_id":"share-974","share_seq":974,"template_height":800030},{"accepted_at_ms":89997,"credit_policy":null,"job_id":"job-567","job_issued_at_ms":89990,"miner_id":"miner-6","network_difficulty":1000006,"ntime":1700000567,"order_key":"06:miner-6","p2mr_program_hex":"3737373737373737373737373737373737373737373737373737373737373737","share_difficulty":3,"share_id":"share-567","share_seq":567,"template_height":800017},{"accepted_at_ms":89980,"credit_policy":null,"job_id":"job-890","job_issued_at_ms":89999,"miner_id":"miner-10","network_difficulty":1000006,"ntime":1700000890,"order_key":"10:miner-10","p2mr_program_hex":"7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a","share_difficulty":2,"share_id":"share-890","share_seq":890,"template_height":800027},{"accepted_at_ms":89981,"credit_policy":null,"job_id":"job-419","job_issued_at_ms":89999,"miner_id":"miner-1","network_difficulty":1000011,"ntime":1700000419,"order_key":"01:miner-1","p2mr_program_hex":"a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3","share_difficulty":3,"share_id":"share-419","share_seq":419,"template_height":800013},{"accepted_at_ms":89971,"credit_policy":null,"job_id":"job-529","job_issued_at_ms":89992,"miner_id":"miner-1","network_difficulty":1000002,"ntime":1700000529,"order_key":"01:miner-1","p2mr_program_hex":"1111111111111111111111111111111111111111111111111111111111111111","share_difficulty":1,"share_id":"share-529","share_seq":529,"template_height":800016},{"accepted_at_ms":89980,"credit_policy":null,"job_id":"job-280","job_issued_at_ms":89993,"miner_id":"miner-5","network_difficulty":1000008,"ntime":1700000280,"order_key":"05:miner-5","p2mr_program_hex":"1818181818181818181818181818181818181818181818181818181818181818","share_difficulty":19,"share_id":"share-280","share_seq":280,"template_height":800008},{"accepted_at_ms":89977,"credit_policy":null,"job_id":"job-292","job_issued_at_ms":89975,"miner_id":"miner-6","network_difficulty":1000003,"ntime":1700000292,"order_key":"06:miner-6","p2mr_program_hex":"2424242424242424242424242424242424242424242424242424242424242424","share_difficulty":7,"share_id":"share-292","share_seq":292,"template_height":800009},{"accepted_at_ms":89994,"credit_policy":null,"job_id":"job-1539","job_issued_at_ms":89974,"miner_id":"miner-10","network_difficulty":1000009,"ntime":1700001539,"order_key":"10:miner-10","p2mr_program_hex":"0303030303030303030303030303030303030303030303030303030303030303","share_difficulty":3,"share_id":"share-1539","share_seq":1539,"template_height":800048},{"accepted_at_ms":89999,"credit_policy":null,"job_id":"job-554","job_issued_at_ms":89986,"miner_id":"miner-4","network_difficulty":1000010,"ntime":1700000554,"order_key":"04:miner-4","p2mr_program_hex":"2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a","share_difficulty":2,"share_id":"share-554","share_seq":554,"template_height":800017},{"accepted_at_ms":89990,"credit_policy":null,"job_id":"job-1505","job_issued_at_ms":89984,"miner_id":"miner-9","network_difficulty":1000009,"ntime":1700001505,"order_key":"09:miner-9","p2mr_program_hex":"e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1","share_difficulty":19,"share_id":"share-1505","share_seq":1505,"template_height":800047},{"accepted_at_ms":89970,"credit_policy":null,"job_id":"job-1696","job_issued_at_ms":89971,"miner_id":"miner-2","network_difficulty":1000013,"ntime":1700001696,"order_key":"02:miner-2","p2mr_program_hex":"a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0","share_difficulty":2,"share_id":"share-1696","share_seq":1696,"template_height":800053},{"accepted_at_ms":89983,"credit_policy":null,"job_id":"job-405","job_issued_at_ms":89981,"miner_id":"miner-9","network_difficulty":1000014,"ntime":1700000405,"order_key":"09:miner-9","p2mr_program_hex":"9595959595959595959595959595959595959595959595959595959595959595","share_difficulty":7,"share_id":"share-405","share_seq":405,"template_height":800012},{"accepted_at_ms":89979,"credit_policy":"stale-grace","job_id":"job-1044","job_issued_at_ms":89992,"miner_id":"miner-10","network_difficulty":1000007,"ntime":1700001044,"order_key":"10:miner-10","p2mr_program_hex":"1414141414141414141414141414141414141414141414141414141414141414","share_difficulty":7,"share_id":"share-1044","share_seq":1044,"template_height":800032},{"accepted_at_ms":89988,"credit_policy":"stale-grace","job_id":"job-1670","job_issued_at_ms":89983,"miner_id":"miner-9","network_difficulty":1000004,"ntime":1700001670,"order_key":"09:miner-9","p2mr_program_hex":"8686868686868686868686868686868686868686868686868686868686868686","share_difficulty":2,"share_id":"share-1670","share_seq":1670,"template_height":800052},{"accepted_at_ms":89970,"credit_policy":null,"job_id":"job-809","job_issued_at_ms":89999,"miner_id":"miner-6","network_difficulty":1000010,"ntime":1700000809,"order_key":"06:miner-6","p2mr_program_hex":"2929292929292929292929292929292929292929292929292929292929292929","share_difficulty":3,"share_id":"share-809","share_seq":809,"template_height":800025},{"accepted_at_ms":89999,"credit_policy":"stale-grace","job_id":"job-478","job_issued_at_ms":89981,"miner_id":"miner-5","network_difficulty":1000002,"ntime":1700000478,"order_key":"05:miner-5","p2mr_program_hex":"dededededededededededededededededededededededededededededededede","share_difficulty":19,"share_id":"share-478","share_seq":478,"template_height":800014},{"accepted_at_ms":89979,"credit_policy":null,"job_id":"job-399","job_issued_at_ms":89998,"miner_id":"miner-3","network_difficulty":1000008,"ntime":1700000399,"order_key":"03:miner-3","p2mr_program_hex":"8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f","share_difficulty":2,"share_id":"share-399","share_seq":399,"template_height":800012},{"accepted_at_ms":89993,"credit_policy":null,"job_id":"job-1142","job_issued_at_ms":89982,"miner_id":"miner-9","network_difficulty":1000003,"ntime":1700001142,"order_key":"09:miner-9","p2mr_program_hex":"7676767676767676767676767676767676767676767676767676767676767676","share_difficulty":1,"share_id":"share-1142","share_seq":1142,"template_height":800035},{"accepted_at_ms":89979,"credit_policy":null,"job_id":"job-1226","job_issued_at_ms":89991,"miner_id":"miner-5","network_difficulty":1000002,"ntime":1700001226,"order_key":"05:miner-5","p2mr_program_hex":"cacacacacacacacacacacacacacacacacacacacacacacacacacacacacacacaca","share_difficulty":3,"share_id":"share-1226","share_seq":1226,"template_height":800038},{"accepted_at_ms":89981,"credit_policy":null,"job_id":"job-1010","job_issued_at_ms":89989,"miner_id":"miner-9","network_difficulty":1000007,"ntime":1700001010,"order_key":"09:miner-9","p2mr_program_hex":"f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2","share_difficulty":2,"share_id":"share-1010","share_seq":1010,"template_height":800031},{"accepted_at_ms":90000,"credit_policy":"stale-grace","job_id":"job-1355","job_issued_at_ms":89975,"miner_id":"miner-2","network_difficulty":1000012,"ntime":1700001355,"order_key":"02:miner-2","p2mr_program_hex":"4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b","share_difficulty":2,"share_id":"share-1355","share_seq":1355,"template_height":800042},{"accepted_at_ms":89991,"credit_policy":null,"job_id":"job-1482","job_issued_at_ms":89997,"miner_id":"miner-8","network_difficulty":1000003,"ntime":1700001482,"order_key":"08:miner-8","p2mr_program_hex":"cacacacacacacacacacacacacacacacacacacacacacacacacacacacacacacaca","share_difficulty":2,"share_id":"share-1482","share_seq":1482,"template_height":800046},{"accepted_at_ms":89975,"credit_policy":null,"job_id":"job-851","job_issued_at_ms":89997,"miner_id":"miner-4","network_difficulty":1000001,"ntime":1700000851,"order_key":"04:miner-4","p2mr_program_hex":"5353535353535353535353535353535353535353535353535353535353535353","share_difficulty":1,"share_id":"share-851","share_seq":851,"template_height":800026},{"accepted_at_ms":89971,"credit_policy":null,"job_id":"job-367","job_issued_at_ms":89983,"miner_id":"miner-4","network_difficulty":1000010,"ntime":1700000367,"order_key":"04:miner-4","p2mr_program_hex":"6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f","share_difficulty":1,"share_id":"share-367","share_seq":367,"template_height":800011},{"accepted_at_ms":89985,"credit_policy":"stale-grace","job_id":"job-546","job_issued_at_ms":89981,"miner_id":"miner-7","network_difficulty":1000002,"ntime":1700000546,"order_key":"07:miner-7","p2mr_program_hex":"2222222222222222222222222222222222222222222222222222222222222222","share_difficulty":7,"share_id":"share-546","share_seq":546,"template_height":800017},{"accepted_at_ms":89993,"credit_policy":"stale-grace","job_id":"job-1018","job_issued_at_ms":89988,"miner_id":"miner-6","network_difficulty":1000015,"ntime":1700001018,"order_key":"06:miner-6","p2mr_program_hex":"fafafafafafafafafafafafafafafafafafafafafafafafafafafafafafafafa","share_difficulty":3,"share_id":"share-1018","share_seq":1018,"template_height":800031},{"accepted_at_ms":89995,"credit_policy":null,"job_id":"job-1677","job_issued_at_ms":89972,"miner_id":"miner-5","network_difficulty":1000011,"ntime":1700001677,"order_key":"05:miner-5","p2mr_program_hex":"8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d","share_difficulty":3,"share_id":"share-1677","share_seq":1677,"template_height":800052},{"accepted_at_ms":89982,"credit_policy":null,"job_id":"job-1644","job_issued_at_ms":89983,"miner_id":"miner-5","network_difficulty":1000012,"ntime":1700001644,"order_key":"05:miner-5","p2mr_program_hex":"6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c","share_difficulty":19,"share_id":"share-1644","share_seq":1644,"template_height":800051},{"accepted_at_ms":89986,"credit_policy":null,"job_id":"job-1722","job_issued_at_ms":89987,"miner_id":"miner-6","network_difficulty":1000005,"ntime":1700001722,"order_key":"06:miner-6","p2mr_program_hex":"babababababababababababababababababababababababababababababababa","share_difficulty":3,"share_id":"share-1722","share_seq":1722,"template_height":800053},{"accepted_at_ms":89995,"credit_policy":null,"job_id":"job-1499","job_issued_at_ms":89984,"miner_id":"miner-3","network_difficulty":1000003,"ntime":1700001499,"order_key":"03:miner-3","p2mr_program_hex":"dbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdb","share_difficulty":7,"share_id":"share-1499","share_seq":1499,"template_height":800046},{"accepted_at_ms":89979,"credit_policy":null,"job_id":"job-171","job_issued_at_ms":89988,"miner_id":"miner-6","network_difficulty":1000001,"ntime":1700000171,"order_key":"06:miner-6","p2mr_program_hex":"abababababababababababababababababababababababababababababababab","share_difficulty":3,"share_id":"share-171","share_seq":171,"template_height":800005},{"accepted_at_ms":89985,"credit_policy":null,"job_id":"job-143","job_issued_at_ms":89972,"miner_id":"miner-0","network_difficulty":1000007,"ntime":1700000143,"order_key":"00:miner-0","p2mr_program_hex":"8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f","share_difficulty":3,"share_id":"share-143","share_seq":143,"template_height":800004},{"accepted_at_ms":89978,"credit_policy":null,"job_id":"job-1471","job_issued_at_ms":89976,"miner_id":"miner-8","network_difficulty":1000009,"ntime":1700001471,"order_key":"08:miner-8","p2mr_program_hex":"bfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbf","share_difficulty":1,"share_id":"share-1471","share_seq":1471,"template_height":800045},{"accepted_at_ms":90000,"credit_policy":null,"job_id":"job-1794","job_issued_at_ms":89999,"miner_id":"miner-1","network_difficulty":1000009,"ntime":1700001794,"order_key":"01:miner-1","p2mr_program_hex":"0202020202020202020202020202020202020202020202020202020202020202","share_difficulty":7,"share_id":"share-1794","share_seq":1794,"template_height":800056},{"accepted_at_ms":89975,"credit_policy":"stale-grace","job_id":"job-1213","job_issued_at_ms":89984,"miner_id":"miner-3","network_difficulty":1000006,"ntime":1700001213,"order_key":"03:miner-3","p2mr_program_hex":"bdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbd","share_difficulty":2,"share_id":"share-1213","share_seq":1213,"template_height":800037},{"accepted_at_ms":89979,"credit_policy":null,"job_id":"job-664","job_issued_at_ms":89977,"miner_id":"miner-4","network_difficulty":1000001,"ntime":1700000664,"order_key":"04:miner-4","p2mr_program_hex":"9898989898989898989898989898989898989898989898989898989898989898","share_difficulty":3,"share_id":"share-664","share_seq":664,"template_height":800020},{"accepted_at_ms":89975,"credit_policy":"stale-grace","job_id":"job-531","job_issued_at_ms":89974,"miner_id":"miner-3","network_difficulty":1000004,"ntime":1700000531,"order_key":"03:miner-3","p2mr_program_hex":"1313131313131313131313131313131313131313131313131313131313131313","share_difficulty":19,"share_id":"share-531","share_seq":531,"template_height":800016},{"accepted_at_ms":89995,"credit_policy":null,"job_id":"job-1318","job_issued_at_ms":90000,"miner_id":"miner-9","network_difficulty":1000009,"ntime":1700001318,"order_key":"09:miner-9","p2mr_program_hex":"2626262626262626262626262626262626262626262626262626262626262626","share_difficulty":19,"share_id":"share-1318","share_seq":1318,"template_height":800041},{"accepted_at_ms":89983,"credit_policy":null,"job_id":"job-273","job_issued_at_ms":89971,"miner_id":"miner-9","network_difficulty":1000001,"ntime":1700000273,"order_key":"09:miner-9","p2mr_program_hex":"1111111111111111111111111111111111111111111111111111111111111111","share_difficulty":7,"share_id":"share-273","share_seq":273,"template_height":800008},{"accepted_at_ms":89998,"credit_policy":"stale-grace","job_id":"job-1359","job_issued_at_ms":89996,"miner_id":"miner-6","network_difficulty":1000016,"ntime":1700001359,"order_key":"06:miner-6","p2mr_program_hex":"4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f","share_difficulty":2,"share_id":"share-1359","share_seq":1359,"template_height":800042},{"accepted_at_ms":89978,"credit_policy":"stale-grace","job_id":"job-1656","job_issued_at_ms":89989,"miner_id":"miner-6","network_difficulty":1000007,"ntime":1700001656,"order_key":"06:miner-6","p2mr_program_hex":"7878787878787878787878787878787878787878787878787878787878787878","share_difficulty":3,"share_id":"share-1656","share_seq":1656,"template_height":800051},{"accepted_at_ms":89984,"credit_policy":null,"job_id":"job-9","job_issued_at_ms":89993,"miner_id":"miner-9","network_difficulty":1000009,"ntime":1700000009,"order_key":"09:miner-9","p2mr_program_hex":"0909090909090909090909090909090909090909090909090909090909090909","share_difficulty":7,"share_id":"share-9","share_seq":9,"template_height":800000},{"accepted_at_ms":89981,"credit_policy":null,"job_id":"job-881","job_issued_at_ms":89993,"miner_id":"miner-1","network_difficulty":1000014,"ntime":1700000881,"order_key":"01:miner-1","p2mr_program_hex":"7171717171717171717171717171717171717171717171717171717171717171","share_difficulty":1,"share_id":"share-881","share_seq":881,"template_height":800027},{"accepted_at_ms":89995,"credit_policy":null,"job_id":"job-1725","job_issued_at_ms":89993,"miner_id":"miner-9","network_difficulty":1000008,"ntime":1700001725,"order_key":"09:miner-9","p2mr_program_hex":"bdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbd","share_difficulty":19,"share_id":"share-1725","share_seq":1725,"template_height":800053},{"accepted_at_ms":89988,"credit_policy":null,"job_id":"job-1033","job_issued_at_ms":89988,"miner_id":"miner-10","network_difficulty":1000013,"ntime":1700001033,"order_key":"10:miner-10","p2mr_program_hex":"0909090909090909090909090909090909090909090909090909090909090909","share_difficulty":2,"share_id":"share-1033","share_seq":1033,"template_height":800032},{"accepted_at_ms":89991,"credit_policy":null,"job_id":"job-1715","job_issued_at_ms":89992,"miner_id":"miner-10","network_difficulty":1000015,"ntime":1700001715,"order_key":"10:miner-10","p2mr_program_hex":"b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3","share_difficulty":1,"share_id":"share-1715","share_seq":1715,"template_height":800053},{"accepted_at_ms":89985,"credit_policy":null,"job_id":"job-698","job_issued_at_ms":89980,"miner_id":"miner-5","network_difficulty":1000001,"ntime":1700000698,"order_key":"05:miner-5","p2mr_program_hex":"babababababababababababababababababababababababababababababababa","share_difficulty":1,"share_id":"share-698","share_seq":698,"template_height":800021},{"accepted_at_ms":89973,"credit_policy":"stale-grace","job_id":"job-340","job_issued_at_ms":89989,"miner_id":"miner-10","network_difficulty":1000000,"ntime":1700000340,"order_key":"10:miner-10","p2mr_program_hex":"5454545454545454545454545454545454545454545454545454545454545454","share_difficulty":7,"share_id":"share-340","share_seq":340,"template_height":800010},{"accepted_at_ms":89978,"credit_policy":"stale-grace","job_id":"job-864","job_issued_at_ms":89984,"miner_id":"miner-6","network_difficulty":1000014,"ntime":1700000864,"order_key":"06:miner-6","p2mr_program_hex":"6060606060606060606060606060606060606060606060606060606060606060","share_difficulty":3,"share_id":"share-864","share_seq":864,"template_height":800027},{"accepted_at_ms":89998,"credit_policy":null,"job_id":"job-739","job_issued_at_ms":89990,"miner_id":"miner-2","network_difficulty":1000008,"ntime":1700000739,"order_key":"02:miner-2","p2mr_program_hex":"e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3","share_difficulty":7,"share_id":"share-739","share_seq":739,"template_height":800023},{"accepted_at_ms":89997,"credit_policy":null,"job_id":"job-712","job_issued_at_ms":89984,"miner_id":"miner-8","network_difficulty":1000015,"ntime":1700000712,"order_key":"08:miner-8","p2mr_program_hex":"c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8","share_difficulty":2,"share_id":"share-712","share_seq":712,"template_height":800022},{"accepted_at_ms":89982,"credit_policy":"stale-grace","job_id":"job-42","job_issued_at_ms":89980,"miner_id":"miner-9","network_difficulty":1000008,"ntime":1700000042,"order_key":"09:miner-9","p2mr_program_hex":"2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a","share_difficulty":19,"share_id":"share-42","share_seq":42,"template_height":800001},{"accepted_at_ms":89978,"credit_policy":null,"job_id":"job-695","job_issued_at_ms":89982,"miner_id":"miner-2","network_difficulty":1000015,"ntime":1700000695,"order_key":"02:miner-2","p2mr_program_hex":"b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7","share_difficulty":3,"share_id":"share-695","share_seq":695,"template_height":800021},{"accepted_at_ms":89974,"credit_policy":null,"job_id":"job-1092","job_issued_at_ms":89990,"miner_id":"miner-3","network_difficulty":1000004,"ntime":1700001092,"order_key":"03:miner-3","p2mr_program_hex":"4444444444444444444444444444444444444444444444444444444444444444","share_difficulty":2,"share_id":"share-1092","share_seq":1092,"template_height":800034},{"accepted_at_ms":89989,"credit_policy":null,"job_id":"job-961","job_issued_at_ms":89979,"miner_id":"miner-4","network_difficulty":1000009,"ntime":1700000961,"order_key":"04:miner-4","p2mr_program_hex":"c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1","share_difficulty":3,"share_id":"share-961","share_seq":961,"template_height":800030},{"accepted_at_ms":89994,"credit_policy":null,"job_id":"job-196","job_issued_at_ms":89979,"miner_id":"miner-9","network_difficulty":1000009,"ntime":1700000196,"order_key":"09:miner-9","p2mr_program_hex":"c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4","share_difficulty":3,"share_id":"share-196","share_seq":196,"template_height":800006},{"accepted_at_ms":89972,"credit_policy":"stale-grace","job_id":"job-1597","job_issued_at_ms":89971,"miner_id":"miner-2","network_difficulty":1000016,"ntime":1700001597,"order_key":"02:miner-2","p2mr_program_hex":"3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d","share_difficulty":1,"share_id":"share-1597","share_seq":1597,"template_height":800049},{"accepted_at_ms":89989,"credit_policy":null,"job_id":"job-1141","job_issued_at_ms":89981,"miner_id":"miner-8","network_difficulty":1000002,"ntime":1700001141,"order_key":"08:miner-8","p2mr_program_hex":"7575757575757575757575757575757575757575757575757575757575757575","share_difficulty":3,"share_id":"share-1141","share_seq":1141,"template_height":800035},{"accepted_at_ms":89979,"credit_policy":null,"job_id":"job-189","job_issued_at_ms":90000,"miner_id":"miner-2","network_difficulty":1000002,"ntime":1700000189,"order_key":"02:miner-2","p2mr_program_hex":"bdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbd","share_difficulty":2,"share_id":"share-189","share_seq":189,"template_height":800005},{"accepted_at_ms":89992,"credit_policy":null,"job_id":"job-130","job_issued_at_ms":90000,"miner_id":"miner-9","network_difficulty":1000011,"ntime":1700000130,"order_key":"09:miner-9","p2mr_program_hex":"8282828282828282828282828282828282828282828282828282828282828282","share_difficulty":1,"share_id":"share-130","share_seq":130,"template_height":800004},{"accepted_at_ms":89970,"credit_policy":null,"job_id":"job-15","job_issued_at_ms":89998,"miner_id":"miner-4","network_difficulty":1000015,"ntime":1700000015,"order_key":"04:miner-4","p2mr_program_hex":"0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f","share_difficulty":1,"share_id":"share-15","share_seq":15,"template_height":800000},{"accepted_at_ms":89982,"credit_policy":null,"job_id":"job-324","job_issued_at_ms":89996,"miner_id":"miner-5","network_difficulty":1000001,"ntime":1700000324,"order_key":"05:miner-5","p2mr_program_hex":"4444444444444444444444444444444444444444444444444444444444444444","share_difficulty":2,"share_id":"share-324","share_seq":324,"template_height":800010},{"accepted_at_ms":89990,"credit_policy":null,"job_id":"job-1158","job_issued_at_ms":89996,"miner_id":"miner-3","network_difficulty":1000002,"ntime":1700001158,"order_key":"03:miner-3","p2mr_program_hex":"8686868686868686868686868686868686868686868686868686868686868686","share_difficulty":19,"share_id":"share-1158","share_seq":1158,"template_height":800036},{"accepted_at_ms":89990,"credit_policy":"stale-grace","job_id":"job-1742","job_issued_at_ms":89992,"miner_id":"miner-4","network_difficulty":1000008,"ntime":1700001742,"order_key":"04:miner-4","p2mr_program_hex":"cececececececececececececececececececececececececececececececece","share_difficulty":1,"share_id":"share-1742","share_seq":1742,"template_height":800054},{"accepted_at_ms":89984,"credit_policy":null,"job_id":"job-1519","job_issued_at_ms":89972,"miner_id":"miner-1","network_difficulty":1000006,"ntime":1700001519,"order_key":"01:miner-1","p2mr_program_hex":"efefefefefefefefefefefefefefefefefefefefefefefefefefefefefefefef","share_difficulty":7,"share_id":"share-1519","share_seq":1519,"template_height":800047},{"accepted_at_ms":89978,"credit_policy":"stale-grace","job_id":"job-812","job_issued_at_ms":89996,"miner_id":"miner-9","network_difficulty":1000013,"ntime":1700000812,"order_key":"09:miner-9","p2mr_program_hex":"2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c","share_difficulty":1,"share_id":"share-812","share_seq":812,"template_height":800025},{"accepted_at_ms":89999,"credit_policy":null,"job_id":"job-1424","job_issued_at_ms":89984,"miner_id":"miner-5","network_difficulty":1000013,"ntime":1700001424,"order_key":"05:miner-5","p2mr_program_hex":"9090909090909090909090909090909090909090909090909090909090909090","share_difficulty":19,"share_id":"share-1424","share_seq":1424,"template_height":800044},{"accepted_at_ms":89988,"credit_policy":null,"job_id":"job-536","job_issued_at_ms":89992,"miner_id":"miner-8","network_difficulty":1000009,"ntime":1700000536,"order_key":"08:miner-8","p2mr_program_hex":"1818181818181818181818181818181818181818181818181818181818181818","share_difficulty":7,"share_id":"share-536","share_seq":536,"template_height":800016},{"accepted_at_ms":89980,"credit_policy":null,"job_id":"job-1596","job_issued_at_ms":89980,"miner_id":"miner-1","network_difficulty":1000015,"ntime":1700001596,"order_key":"01:miner-1","p2mr_program_hex":"3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c","share_difficulty":1,"share_id":"share-1596","share_seq":1596,"template_height":800049},{"accepted_at_ms":89981,"credit_policy":"stale-grace","job_id":"job-1290","job_issued_at_ms":89978,"miner_id":"miner-3","network_difficulty":1000015,"ntime":1700001290,"order_key":"03:miner-3","p2mr_program_hex":"0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a","share_difficulty":1,"share_id":"share-1290","share_seq":1290,"template_height":800040},{"accepted_at_ms":89972,"credit_policy":null,"job_id":"job-65","job_issued_at_ms":89976,"miner_id":"miner-10","network_difficulty":1000014,"ntime":1700000065,"order_key":"10:miner-10","p2mr_program_hex":"4141414141414141414141414141414141414141414141414141414141414141","share_difficulty":7,"share_id":"share-65","share_seq":65,"template_height":800002},{"accepted_at_ms":89982,"credit_policy":null,"job_id":"job-1589","job_issued_at_ms":89972,"miner_id":"miner-5","network_difficulty":1000008,"ntime":1700001589,"order_key":"05:miner-5","p2mr_program_hex":"3535353535353535353535353535353535353535353535353535353535353535","share_difficulty":3,"share_id":"share-1589","share_seq":1589,"template_height":800049},{"accepted_at_ms":89999,"credit_policy":null,"job_id":"job-1568","job_issued_at_ms":89995,"miner_id":"miner-6","network_difficulty":1000004,"ntime":1700001568,"order_key":"06:miner-6","p2mr_program_hex":"2020202020202020202020202020202020202020202020202020202020202020","share_difficulty":19,"share_id":"share-1568","share_seq":1568,"template_height":800049},{"accepted_at_ms":89971,"credit_policy":null,"job_id":"job-330","job_issued_at_ms":89988,"miner_id":"miner-0","network_difficulty":1000007,"ntime":1700000330,"order_key":"00:miner-0","p2mr_program_hex":"4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a","share_difficulty":2,"share_id":"share-330","share_seq":330,"template_height":800010},{"accepted_at_ms":89982,"credit_policy":null,"job_id":"job-853","job_issued_at_ms":89993,"miner_id":"miner-6","network_difficulty":1000003,"ntime":1700000853,"order_key":"06:miner-6","p2mr_program_hex":"5555555555555555555555555555555555555555555555555555555555555555","share_difficulty":1,"share_id":"share-853","share_seq":853,"template_height":800026},{"accepted_at_ms":89985,"credit_policy":null,"job_id":"job-651","job_issued_at_ms":89992,"miner_id":"miner-2","network_difficulty":1000005,"ntime":1700000651,"order_key":"02:miner-2","p2mr_program_hex":"8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b","share_difficulty":19,"share_id":"share-651","share_seq":651,"template_height":800020},{"accepted_at_ms":89973,"credit_policy":null,"job_id":"job-1088","job_issued_at_ms":89993,"miner_id":"miner-10","network_difficulty":1000000,"ntime":1700001088,"order_key":"10:miner-10","p2mr_program_hex":"4040404040404040404040404040404040404040404040404040404040404040","share_difficulty":1,"share_id":"share-1088","share_seq":1088,"template_height":800034},{"accepted_at_ms":89986,"credit_policy":null,"job_id":"job-495","job_issued_at_ms":89993,"miner_id":"miner-0","network_difficulty":1000002,"ntime":1700000495,"order_key":"00:miner-0","p2mr_program_hex":"efefefefefefefefefefefefefefefefefefefefefefefefefefefefefefefef","share_difficulty":2,"share_id":"share-495","share_seq":495,"template_height":800015},{"accepted_at_ms":89974,"credit_policy":null,"job_id":"job-1461","job_issued_at_ms":89991,"miner_id":"miner-9","network_difficulty":1000016,"ntime":1700001461,"order_key":"09:miner-9","p2mr_program_hex":"b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5","share_difficulty":1,"share_id":"share-1461","share_seq":1461,"template_height":800045},{"accepted_at_ms":89976,"credit_policy":"stale-grace","job_id":"job-184","job_issued_at_ms":89984,"miner_id":"miner-8","network_difficulty":1000014,"ntime":1700000184,"order_key":"08:miner-8","p2mr_program_hex":"b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8","share_difficulty":7,"share_id":"share-184","share_seq":184,"template_height":800005},{"accepted_at_ms":89978,"credit_policy":null,"job_id":"job-1578","job_issued_at_ms":89974,"miner_id":"miner-5","network_difficulty":1000014,"ntime":1700001578,"order_key":"05:miner-5","p2mr_program_hex":"2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a","share_difficulty":1,"share_id":"share-1578","share_seq":1578,"template_height":800049},{"accepted_at_ms":89997,"credit_policy":null,"job_id":"job-1719","job_issued_at_ms":89980,"miner_id":"miner-3","network_difficulty":1000002,"ntime":1700001719,"order_key":"03:miner-3","p2mr_program_hex":"b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7","share_difficulty":19,"share_id":"share-1719","share_seq":1719,"template_height":800053},{"accepted_at_ms":89996,"credit_policy":null,"job_id":"job-366","job_issued_at_ms":89983,"miner_id":"miner-3","network_difficulty":1000009,"ntime":1700000366,"order_key":"03:miner-3","p2mr_program_hex":"6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e","share_difficulty":2,"share_id":"share-366","share_seq":366,"template_height":800011},{"accepted_at_ms":89972,"credit_policy":null,"job_id":"job-1613","job_issued_at_ms":89980,"miner_id":"miner-7","network_difficulty":1000015,"ntime":1700001613,"order_key":"07:miner-7","p2mr_program_hex":"4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d","share_difficulty":3,"share_id":"share-1613","share_seq":1613,"template_height":800050},{"accepted_at_ms":89987,"credit_policy":null,"job_id":"job-1420","job_issued_at_ms":89993,"miner_id":"miner-1","network_difficulty":1000009,"ntime":1700001420,"order_key":"01:miner-1","p2mr_program_hex":"8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c","share_difficulty":7,"share_id":"share-1420","share_seq":1420,"template_height":800044},{"accepted_at_ms":89983,"credit_policy":null,"job_id":"job-1650","job_issued_at_ms":89990,"miner_id":"miner-0","network_difficulty":1000001,"ntime":1700001650,"order_key":"00:miner-0","p2mr_program_hex":"7272727272727272727272727272727272727272727272727272727272727272","share_difficulty":2,"share_id":"share-1650","share_seq":1650,"template_height":800051},{"accepted_at_ms":89978,"credit_policy":"stale-grace","job_id":"job-895","job_issued_at_ms":89998,"miner_id":"miner-4","network_difficulty":1000011,"ntime":1700000895,"order_key":"04:miner-4","p2mr_program_hex":"7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f","share_difficulty":2,"share_id":"share-895","share_seq":895,"template_height":800027},{"accepted_at_ms":89981,"credit_policy":"stale-grace","job_id":"job-1005","job_issued_at_ms":89985,"miner_id":"miner-4","network_difficulty":1000002,"ntime":1700001005,"order_key":"04:miner-4","p2mr_program_hex":"edededededededededededededededededededededededededededededededed","share_difficulty":19,"share_id":"share-1005","share_seq":1005,"template_height":800031},{"accepted_at_ms":89979,"credit_policy":null,"job_id":"job-962","job_issued_at_ms":89984,"miner_id":"miner-5","network_difficulty":1000010,"ntime":1700000962,"order_key":"05:miner-5","p2mr_program_hex":"c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2","share_difficulty":1,"share_id":"share-962","share_seq":962,"template_height":800030},{"accepted_at_ms":89990,"credit_policy":null,"job_id":"job-1607","job_issued_at_ms":89974,"miner_id":"miner-1","network_difficulty":1000009,"ntime":1700001607,"order_key":"01:miner-1","p2mr_program_hex":"4747474747474747474747474747474747474747474747474747474747474747","share_difficulty":19,"share_id":"share-1607","share_seq":1607,"template_height":800050},{"accepted_at_ms":89995,"credit_policy":null,"job_id":"job-1190","job_issued_at_ms":89993,"miner_id":"miner-2","network_difficulty":1000000,"ntime":1700001190,"order_key":"02:miner-2","p2mr_program_hex":"a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6","share_difficulty":1,"share_id":"share-1190","share_seq":1190,"template_height":800037},{"accepted_at_ms":89987,"credit_policy":"stale-grace","job_id":"job-410","job_issued_at_ms":89985,"miner_id":"miner-3","network_difficulty":1000002,"ntime":1700000410,"order_key":"03:miner-3","p2mr_program_hex":"9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a","share_difficulty":7,"share_id":"share-410","share_seq":410,"template_height":800012},{"accepted_at_ms":89974,"credit_policy":null,"job_id":"job-676","job_issued_at_ms":89994,"miner_id":"miner-5","network_difficulty":1000013,"ntime":1700000676,"order_key":"05:miner-5","p2mr_program_hex":"a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4","share_difficulty":1,"share_id":"share-676","share_seq":676,"template_height":800021},{"accepted_at_ms":89996,"credit_policy":"stale-grace","job_id":"job-268","job_issued_at_ms":89986,"miner_id":"miner-4","network_difficulty":1000013,"ntime":1700000268,"order_key":"04:miner-4","p2mr_program_hex":"0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c","share_difficulty":3,"share_id":"share-268","share_seq":268,"template_height":800008},{"accepted_at_ms":89977,"credit_policy":null,"job_id":"job-219","job_issued_at_ms":89993,"miner_id":"miner-10","network_difficulty":1000015,"ntime":1700000219,"order_key":"10:miner-10","p2mr_program_hex":"dbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdb","share_difficulty":1,"share_id":"share-219","share_seq":219,"template_height":800006},{"accepted_at_ms":89987,"credit_policy":null,"job_id":"job-462","job_issued_at_ms":89973,"miner_id":"miner-0","network_difficulty":1000003,"ntime":1700000462,"order_key":"00:miner-0","p2mr_program_hex":"cececececececececececececececececececececececececececececececece","share_difficulty":1,"share_id":"share-462","share_seq":462,"template_height":800014},{"accepted_at_ms":89989,"credit_policy":"stale-grace","job_id":"job-38","job_issued_at_ms":89983,"miner_id":"miner-5","network_difficulty":1000004,"ntime":1700000038,"order_key":"05:miner-5","p2mr_program_hex":"2626262626262626262626262626262626262626262626262626262626262626","share_difficulty":1,"share_id":"share-38","share_seq":38,"template_height":800001},{"accepted_at_ms":89992,"credit_policy":null,"job_id":"job-1329","job_issued_at_ms":89994,"miner_id":"miner-9","network_difficulty":1000003,"ntime":1700001329,"order_key":"09:miner-9","p2mr_program_hex":"3131313131313131313131313131313131313131313131313131313131313131","share_difficulty":1,"share_id":"share-1329","share_seq":1329,"template_height":800041},{"accepted_at_ms":89998,"credit_policy":null,"job_id":"job-18","job_issued_at_ms":89992,"miner_id":"miner-7","network_difficulty":1000001,"ntime":1700000018,"order_key":"07:miner-7","p2mr_program_hex":"1212121212121212121212121212121212121212121212121212121212121212","share_difficulty":7,"share_id":"share-18","share_seq":18,"template_height":800000},{"accepted_at_ms":89994,"credit_policy":null,"job_id":"job-409","job_issued_at_ms":89982,"miner_id":"miner-2","network_difficulty":1000001,"ntime":1700000409,"order_key":"02:miner-2","p2mr_program_hex":"9999999999999999999999999999999999999999999999999999999999999999","share_difficulty":3,"share_id":"share-409","share_seq":409,"template_height":800012},{"accepted_at_ms":89983,"credit_policy":null,"job_id":"job-997","job_issued_at_ms":89981,"miner_id":"miner-7","network_difficulty":1000011,"ntime":1700000997,"order_key":"07:miner-7","p2mr_program_hex":"e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5","share_difficulty":3,"share_id":"share-997","share_seq":997,"template_height":800031},{"accepted_at_ms":89984,"credit_policy":null,"job_id":"job-1602","job_issued_at_ms":90000,"miner_id":"miner-7","network_difficulty":1000004,"ntime":1700001602,"order_key":"07:miner-7","p2mr_program_hex":"4242424242424242424242424242424242424242424242424242424242424242","share_difficulty":3,"share_id":"share-1602","share_seq":1602,"template_height":800050},{"accepted_at_ms":89975,"credit_policy":"stale-grace","job_id":"job-897","job_issued_at_ms":89982,"miner_id":"miner-6","network_difficulty":1000013,"ntime":1700000897,"order_key":"06:miner-6","p2mr_program_hex":"8181818181818181818181818181818181818181818181818181818181818181","share_difficulty":7,"share_id":"share-897","share_seq":897,"template_height":800028},{"accepted_at_ms":89981,"credit_policy":null,"job_id":"job-1382","job_issued_at_ms":89976,"miner_id":"miner-7","network_difficulty":1000005,"ntime":1700001382,"order_key":"07:miner-7","p2mr_program_hex":"6666666666666666666666666666666666666666666666666666666666666666","share_difficulty":1,"share_id":"share-1382","share_seq":1382,"template_height":800043},{"accepted_at_ms":89975,"credit_policy":null,"job_id":"job-300","job_issued_at_ms":89989,"miner_id":"miner-3","network_difficulty":1000011,"ntime":1700000300,"order_key":"03:miner-3","p2mr_program_hex":"2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c","share_difficulty":19,"share_id":"share-300","share_seq":300,"template_height":800009},{"accepted_at_ms":89993,"credit_policy":null,"job_id":"job-1076","job_issued_at_ms":89970,"miner_id":"miner-9","network_difficulty":1000005,"ntime":1700001076,"order_key":"09:miner-9","p2mr_program_hex":"3434343434343434343434343434343434343434343434343434343434343434","share_difficulty":7,"share_id":"share-1076","share_seq":1076,"template_height":800033},{"accepted_at_ms":89970,"credit_policy":"stale-grace","job_id":"job-933","job_issued_at_ms":89993,"miner_id":"miner-9","network_difficulty":1000015,"ntime":1700000933,"order_key":"09:miner-9","p2mr_program_hex":"a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5","share_difficulty":2,"share_id":"share-933","share_seq":933,"template_height":800029},{"accepted_at_ms":89980,"credit_policy":"stale-grace","job_id":"job-583","job_issued_at_ms":89970,"miner_id":"miner-0","network_difficulty":1000005,"ntime":1700000583,"order_key":"00:miner-0","p2mr_program_hex":"4747474747474747474747474747474747474747474747474747474747474747","share_difficulty":7,"share_id":"share-583","share_seq":583,"template_height":800018},{"accepted_at_ms":89992,"credit_policy":null,"job_id":"job-608","job_issued_at_ms":89999,"miner_id":"miner-3","network_difficulty":1000013,"ntime":1700000608,"order_key":"03:miner-3","p2mr_program_hex":"6060606060606060606060606060606060606060606060606060606060606060","share_difficulty":1,"share_id":"share-608","share_seq":608,"template_height":800019},{"accepted_at_ms":89997,"credit_policy":"stale-grace","job_id":"job-891","job_issued_at_ms":89983,"miner_id":"miner-0","network_difficulty":1000007,"ntime":1700000891,"order_key":"00:miner-0","p2mr_program_hex":"7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b","share_difficulty":3,"share_id":"share-891","share_seq":891,"template_height":800027},{"accepted_at_ms":89987,"credit_policy":null,"job_id":"job-1135","job_issued_at_ms":89979,"miner_id":"miner-2","network_difficulty":1000013,"ntime":1700001135,"order_key":"02:miner-2","p2mr_program_hex":"6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f","share_difficulty":7,"share_id":"share-1135","share_seq":1135,"template_height":800035},{"accepted_at_ms":89993,"credit_policy":null,"job_id":"job-1709","job_issued_at_ms":89999,"miner_id":"miner-4","network_difficulty":1000009,"ntime":1700001709,"order_key":"04:miner-4","p2mr_program_hex":"adadadadadadadadadadadadadadadadadadadadadadadadadadadadadadadad","share_difficulty":1,"share_id":"share-1709","share_seq":1709,"template_height":800053},{"accepted_at_ms":89990,"credit_policy":null,"job_id":"job-704","job_issued_at_ms":90000,"miner_id":"miner-0","network_difficulty":1000007,"ntime":1700000704,"order_key":"00:miner-0","p2mr_program_hex":"c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0","share_difficulty":1,"share_id":"share-704","share_seq":704,"template_height":800022},{"accepted_at_ms":89979,"credit_policy":null,"job_id":"job-1467","job_issued_at_ms":89979,"miner_id":"miner-4","network_difficulty":1000005,"ntime":1700001467,"order_key":"04:miner-4","p2mr_program_hex":"bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb","share_difficulty":2,"share_id":"share-1467","share_seq":1467,"template_height":800045},{"accepted_at_ms":89970,"credit_policy":null,"job_id":"job-59","job_issued_at_ms":89973,"miner_id":"miner-4","network_difficulty":1000008,"ntime":1700000059,"order_key":"04:miner-4","p2mr_program_hex":"3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b","share_difficulty":7,"share_id":"share-59","share_seq":59,"template_height":800001},{"accepted_at_ms":89991,"credit_policy":null,"job_id":"job-455","job_issued_at_ms":89976,"miner_id":"miner-4","network_difficulty":1000013,"ntime":1700000455,"order_key":"04:miner-4","p2mr_program_hex":"c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7","share_difficulty":19,"share_id":"share-455","share_seq":455,"template_height":800014},{"accepted_at_ms":89981,"credit_policy":"stale-grace","job_id":"job-487","job_issued_at_ms":89985,"miner_id":"miner-3","network_difficulty":1000011,"ntime":1700000487,"order_key":"03:miner-3","p2mr_program_hex":"e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7","share_difficulty":1,"share_id":"share-487","share_seq":487,"template_height":800015},{"accepted_at_ms":89993,"credit_policy":null,"job_id":"job-1491","job_issued_at_ms":89989,"miner_id":"miner-6","network_difficulty":1000012,"ntime":1700001491,"order_key":"06:miner-6","p2mr_program_hex":"d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3","share_difficulty":1,"share_id":"share-1491","share_seq":1491,"template_height":800046},{"accepted_at_ms":89974,"credit_policy":"stale-grace","job_id":"job-141","job_issued_at_ms":89998,"miner_id":"miner-9","network_difficulty":1000005,"ntime":1700000141,"order_key":"09:miner-9","p2mr_program_hex":"8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d","share_difficulty":19,"share_id":"share-141","share_seq":141,"template_height":800004},{"accepted_at_ms":89987,"credit_policy":"stale-grace","job_id":"job-1739","job_issued_at_ms":89997,"miner_id":"miner-1","network_difficulty":1000005,"ntime":1700001739,"order_key":"01:miner-1","p2mr_program_hex":"cbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcb","share_difficulty":7,"share_id":"share-1739","share_seq":1739,"template_height":800054},{"accepted_at_ms":89992,"credit_policy":null,"job_id":"job-1286","job_issued_at_ms":89981,"miner_id":"miner-10","network_difficulty":1000011,"ntime":1700001286,"order_key":"10:miner-10","p2mr_program_hex":"0606060606060606060606060606060606060606060606060606060606060606","share_difficulty":7,"share_id":"share-1286","share_seq":1286,"template_height":800040},{"accepted_at_ms":89976,"credit_policy":null,"job_id":"job-1488","job_issued_at_ms":89982,"miner_id":"miner-3","network_difficulty":1000009,"ntime":1700001488,"order_key":"03:miner-3","p2mr_program_hex":"d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0","share_difficulty":1,"share_id":"share-1488","share_seq":1488,"template_height":800046},{"accepted_at_ms":89991,"credit_policy":null,"job_id":"job-383","job_issued_at_ms":89993,"miner_id":"miner-9","network_difficulty":1000009,"ntime":1700000383,"order_key":"09:miner-9","p2mr_program_hex":"7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f","share_difficulty":2,"share_id":"share-383","share_seq":383,"template_height":800011},{"accepted_at_ms":89970,"credit_policy":null,"job_id":"job-926","job_issued_at_ms":89983,"miner_id":"miner-2","network_difficulty":1000008,"ntime":1700000926,"order_key":"02:miner-2","p2mr_program_hex":"9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e","share_difficulty":1,"share_id":"share-926","share_seq":926,"template_height":800028},{"accepted_at_ms":90000,"credit_policy":"stale-grace","job_id":"job-1266","job_issued_at_ms":89978,"miner_id":"miner-1","network_difficulty":1000008,"ntime":1700001266,"order_key":"01:miner-1","p2mr_program_hex":"f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2","share_difficulty":7,"share_id":"share-1266","share_seq":1266,"template_height":800039},{"accepted_at_ms":90000,"credit_policy":"stale-grace","job_id":"job-1328","job_issued_at_ms":89987,"miner_id":"miner-8","network_difficulty":1000002,"ntime":1700001328,"order_key":"08:miner-8","p2mr_program_hex":"3030303030303030303030303030303030303030303030303030303030303030","share_difficulty":2,"share_id":"share-1328","share_seq":1328,"template_height":800041},{"accepted_at_ms":89987,"credit_policy":null,"job_id":"job-1115","job_issued_at_ms":89977,"miner_id":"miner-4","network_difficulty":1000010,"ntime":1700001115,"order_key":"04:miner-4","p2mr_program_hex":"5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b","share_difficulty":7,"share_id":"share-1115","share_seq":1115,"template_height":800034},{"accepted_at_ms":89983,"credit_policy":"stale-grace","job_id":"job-862","job_issued_at_ms":89982,"miner_id":"miner-4","network_difficulty":1000012,"ntime":1700000862,"order_key":"04:miner-4","p2mr_program_hex":"5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e","share_difficulty":1,"share_id":"share-862","share_seq":862,"template_height":800026},{"accepted_at_ms":89972,"credit_policy":null,"job_id":"job-916","job_issued_at_ms":89972,"miner_id":"miner-3","network_difficulty":1000015,"ntime":1700000916,"order_key":"03:miner-3","p2mr_program_hex":"9494949494949494949494949494949494949494949494949494949494949494","share_difficulty":1,"share_id":"share-916","share_seq":916,"template_height":800028},{"accepted_at_ms":89979,"credit_policy":null,"job_id":"job-181","job_issued_at_ms":89988,"miner_id":"miner-5","network_difficulty":1000011,"ntime":1700000181,"order_key":"05:miner-5","p2mr_program_hex":"b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5","share_difficulty":2,"share_id":"share-181","share_seq":181,"template_height":800005},{"accepted_at_ms":89973,"credit_policy":null,"job_id":"job-321","job_issued_at_ms":89993,"miner_id":"miner-2","network_difficulty":1000015,"ntime":1700000321,"order_key":"02:miner-2","p2mr_program_hex":"4141414141414141414141414141414141414141414141414141414141414141","share_difficulty":19,"share_id":"share-321","share_seq":321,"template_height":800010},{"accepted_at_ms":89995,"credit_policy":"stale-grace","job_id":"job-570","job_issued_at_ms":89972,"miner_id":"miner-9","network_difficulty":1000009,"ntime":1700000570,"order_key":"09:miner-9","p2mr_program_hex":"3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a","share_difficulty":1,"share_id":"share-570","share_seq":570,"template_height":800017},{"accepted_at_ms":89980,"credit_policy":null,"job_id":"job-1527","job_issued_at_ms":89999,"miner_id":"miner-9","network_difficulty":1000014,"ntime":1700001527,"order_key":"09:miner-9","p2mr_program_hex":"f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7","share_difficulty":7,"share_id":"share-1527","share_seq":1527,"template_height":800047},{"accepted_at_ms":89985,"credit_policy":"stale-grace","job_id":"job-44","job_issued_at_ms":89992,"miner_id":"miner-0","network_difficulty":1000010,"ntime":1700000044,"order_key":"00:miner-0","p2mr_program_hex":"2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c","share_difficulty":19,"share_id":"share-44","share_seq":44,"template_height":800001},{"accepted_at_ms":89990,"credit_policy":"stale-grace","job_id":"job-110","job_issued_at_ms":89997,"miner_id":"miner-0","network_difficulty":1000008,"ntime":1700000110,"order_key":"00:miner-0","p2mr_program_hex":"6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e","share_difficulty":19,"share_id":"share-110","share_seq":110,"template_height":800003},{"accepted_at_ms":89976,"credit_policy":"stale-grace","job_id":"job-1322","job_issued_at_ms":89974,"miner_id":"miner-2","network_difficulty":1000013,"ntime":1700001322,"order_key":"02:miner-2","p2mr_program_hex":"2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a","share_difficulty":3,"share_id":"share-1322","share_seq":1322,"template_height":800041},{"accepted_at_ms":89998,"credit_policy":null,"job_id":"job-376","job_issued_at_ms":89979,"miner_id":"miner-2","network_difficulty":1000002,"ntime":1700000376,"order_key":"02:miner-2","p2mr_program_hex":"7878787878787878787878787878787878787878787878787878787878787878","share_difficulty":2,"share_id":"share-376","share_seq":376,"template_height":800011},{"accepted_at_ms":89970,"credit_policy":"stale-grace","job_id":"job-1660","job_issued_at_ms":89974,"miner_id":"miner-10","network_difficulty":1000011,"ntime":1700001660,"order_key":"10:miner-10","p2mr_program_hex":"7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c","share_difficulty":3,"share_id":"share-1660","share_seq":1660,"template_height":800051},{"accepted_at_ms":89972,"credit_policy":null,"job_id":"job-904","job_issued_at_ms":89970,"miner_id":"miner-2","network_difficulty":1000003,"ntime":1700000904,"order_key":"02:miner-2","p2mr_program_hex":"8888888888888888888888888888888888888888888888888888888888888888","share_difficulty":19,"share_id":"share-904","share_seq":904,"template_height":800028},{"accepted_at_ms":89992,"credit_policy":null,"job_id":"job-928","job_issued_at_ms":89993,"miner_id":"miner-4","network_difficulty":1000010,"ntime":1700000928,"order_key":"04:miner-4","p2mr_program_hex":"a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0","share_difficulty":19,"share_id":"share-928","share_seq":928,"template_height":800029},{"accepted_at_ms":89988,"credit_policy":null,"job_id":"job-801","job_issued_at_ms":89977,"miner_id":"miner-9","network_difficulty":1000002,"ntime":1700000801,"order_key":"09:miner-9","p2mr_program_hex":"2121212121212121212121212121212121212121212121212121212121212121","share_difficulty":1,"share_id":"share-801","share_seq":801,"template_height":800025},{"accepted_at_ms":89982,"credit_policy":null,"job_id":"job-808","job_issued_at_ms":89986,"miner_id":"miner-5","network_difficulty":1000009,"ntime":1700000808,"order_key":"05:miner-5","p2mr_program_hex":"2828282828282828282828282828282828282828282828282828282828282828","share_difficulty":19,"share_id":"share-808","share_seq":808,"template_height":800025},{"accepted_at_ms":89988,"credit_policy":null,"job_id":"job-1396","job_issued_at_ms":89996,"miner_id":"miner-10","network_difficulty":1000002,"ntime":1700001396,"order_key":"10:miner-10","p2mr_program_hex":"7474747474747474747474747474747474747474747474747474747474747474","share_difficulty":2,"share_id":"share-1396","share_seq":1396,"template_height":800043},{"accepted_at_ms":89978,"credit_policy":null,"job_id":"job-1212","job_issued_at_ms":89978,"miner_id":"miner-2","network_difficulty":1000005,"ntime":1700001212,"order_key":"02:miner-2","p2mr_program_hex":"bcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbc","share_difficulty":3,"share_id":"share-1212","share_seq":1212,"template_height":800037},{"accepted_at_ms":89997,"credit_policy":null,"job_id":"job-1377","job_issued_at_ms":89973,"miner_id":"miner-2","network_difficulty":1000000,"ntime":1700001377,"order_key":"02:miner-2","p2mr_program_hex":"6161616161616161616161616161616161616161616161616161616161616161","share_difficulty":3,"share_id":"share-1377","share_seq":1377,"template_height":800043},{"accepted_at_ms":89996,"credit_policy":"stale-grace","job_id":"job-407","job_issued_at_ms":89998,"miner_id":"miner-0","network_difficulty":1000016,"ntime":1700000407,"order_key":"00:miner-0","p2mr_program_hex":"9797979797979797979797979797979797979797979797979797979797979797","share_difficulty":7,"share_id":"share-407","share_seq":407,"template_height":800012},{"accepted_at_ms":89989,"credit_policy":null,"job_id":"job-1673","job_issued_at_ms":89979,"miner_id":"miner-1","network_difficulty":1000007,"ntime":1700001673,"order_key":"01:miner-1","p2mr_program_hex":"8989898989898989898989898989898989898989898989898989898989898989","share_difficulty":2,"share_id":"share-1673","share_seq":1673,"template_height":800052},{"accepted_at_ms":89992,"credit_policy":null,"job_id":"job-459","job_issued_at_ms":89997,"miner_id":"miner-8","network_difficulty":1000000,"ntime":1700000459,"order_key":"08:miner-8","p2mr_program_hex":"cbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcb","share_difficulty":7,"share_id":"share-459","share_seq":459,"template_height":800014},{"accepted_at_ms":89972,"credit_policy":null,"job_id":"job-1460","job_issued_at_ms":89989,"miner_id":"miner-8","network_difficulty":1000015,"ntime":1700001460,"order_key":"08:miner-8","p2mr_program_hex":"b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4","share_difficulty":2,"share_id":"share-1460","share_seq":1460,"template_height":800045},{"accepted_at_ms":89992,"credit_policy":null,"job_id":"job-194","job_issued_at_ms":89989,"miner_id":"miner-7","network_difficulty":1000007,"ntime":1700000194,"order_key":"07:miner-7","p2mr_program_hex":"c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2","share_difficulty":3,"share_id":"share-194","share_seq":194,"template_height":800006},{"accepted_at_ms":89996,"credit_policy":"stale-grace","job_id":"job-1608","job_issued_at_ms":89976,"miner_id":"miner-2","network_difficulty":1000010,"ntime":1700001608,"order_key":"02:miner-2","p2mr_program_hex":"4848484848484848484848484848484848484848484848484848484848484848","share_difficulty":2,"share_id":"share-1608","share_seq":1608,"template_height":800050},{"accepted_at_ms":89995,"credit_policy":"stale-grace","job_id":"job-450","job_issued_at_ms":89974,"miner_id":"miner-10","network_difficulty":1000008,"ntime":1700000450,"order_key":"10:miner-10","p2mr_program_hex":"c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2","share_difficulty":19,"share_id":"share-450","share_seq":450,"template_height":800014},{"accepted_at_ms":89983,"credit_policy":null,"job_id":"job-841","job_issued_at_ms":89995,"miner_id":"miner-5","network_difficulty":1000008,"ntime":1700000841,"order_key":"05:miner-5","p2mr_program_hex":"4949494949494949494949494949494949494949494949494949494949494949","share_difficulty":3,"share_id":"share-841","share_seq":841,"template_height":800026},{"accepted_at_ms":89985,"credit_policy":null,"job_id":"job-278","job_issued_at_ms":89972,"miner_id":"miner-3","network_difficulty":1000006,"ntime":1700000278,"order_key":"03:miner-3","p2mr_program_hex":"1616161616161616161616161616161616161616161616161616161616161616","share_difficulty":19,"share_id":"share-278","share_seq":278,"template_height":800008},{"accepted_at_ms":89985,"credit_policy":null,"job_id":"job-1423","job_issued_at_ms":89990,"miner_id":"miner-4","network_difficulty":1000012,"ntime":1700001423,"order_key":"04:miner-4","p2mr_program_hex":"8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f","share_difficulty":1,"share_id":"share-1423","share_seq":1423,"template_height":800044},{"accepted_at_ms":89980,"credit_policy":"stale-grace","job_id":"job-314","job_issued_at_ms":90000,"miner_id":"miner-6","network_difficulty":1000008,"ntime":1700000314,"order_key":"06:miner-6","p2mr_program_hex":"3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a","share_difficulty":19,"share_id":"share-314","share_seq":314,"template_height":800009},{"accepted_at_ms":89972,"credit_policy":null,"job_id":"job-834","job_issued_at_ms":89998,"miner_id":"miner-9","network_difficulty":1000001,"ntime":1700000834,"order_key":"09:miner-9","p2mr_program_hex":"4242424242424242424242424242424242424242424242424242424242424242","share_difficulty":19,"share_id":"share-834","share_seq":834,"template_height":800026},{"accepted_at_ms":89970,"credit_policy":"stale-grace","job_id":"job-1397","job_issued_at_ms":89972,"miner_id":"miner-0","network_difficulty":1000003,"ntime":1700001397,"order_key":"00:miner-0","p2mr_program_hex":"7575757575757575757575757575757575757575757575757575757575757575","share_difficulty":2,"share_id":"share-1397","share_seq":1397,"template_height":800043},{"accepted_at_ms":89992,"credit_policy":"stale-grace","job_id":"job-1238","job_issued_at_ms":89987,"miner_id":"miner-6","network_difficulty":1000014,"ntime":1700001238,"order_key":"06:miner-6","p2mr_program_hex":"d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6","share_difficulty":19,"share_id":"share-1238","share_seq":1238,"template_height":800038},{"accepted_at_ms":89984,"credit_policy":null,"job_id":"job-1345","job_issued_at_ms":89998,"miner_id":"miner-3","network_difficulty":1000002,"ntime":1700001345,"order_key":"03:miner-3","p2mr_program_hex":"4141414141414141414141414141414141414141414141414141414141414141","share_difficulty":2,"share_id":"share-1345","share_seq":1345,"template_height":800042},{"accepted_at_ms":89982,"credit_policy":null,"job_id":"job-733","job_issued_at_ms":89991,"miner_id":"miner-7","network_difficulty":1000002,"ntime":1700000733,"order_key":"07:miner-7","p2mr_program_hex":"dddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddd","share_difficulty":7,"share_id":"share-733","share_seq":733,"template_height":800022},{"accepted_at_ms":89980,"credit_policy":"stale-grace","job_id":"job-633","job_issued_at_ms":89991,"miner_id":"miner-6","network_difficulty":1000004,"ntime":1700000633,"order_key":"06:miner-6","p2mr_program_hex":"7979797979797979797979797979797979797979797979797979797979797979","share_difficulty":3,"share_id":"share-633","share_seq":633,"template_height":800019},{"accepted_at_ms":89976,"credit_policy":null,"job_id":"job-815","job_issued_at_ms":89996,"miner_id":"miner-1","network_difficulty":1000016,"ntime":1700000815,"order_key":"01:miner-1","p2mr_program_hex":"2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f","share_difficulty":2,"share_id":"share-815","share_seq":815,"template_height":800025},{"accepted_at_ms":89974,"credit_policy":null,"job_id":"job-1455","job_issued_at_ms":89981,"miner_id":"miner-3","network_difficulty":1000010,"ntime":1700001455,"order_key":"03:miner-3","p2mr_program_hex":"afafafafafafafafafafafafafafafafafafafafafafafafafafafafafafafaf","share_difficulty":2,"share_id":"share-1455","share_seq":1455,"template_height":800045},{"accepted_at_ms":89996,"credit_policy":null,"job_id":"job-252","job_issued_at_ms":89985,"miner_id":"miner-10","network_difficulty":1000014,"ntime":1700000252,"order_key":"10:miner-10","p2mr_program_hex":"fcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfc","share_difficulty":2,"share_id":"share-252","share_seq":252,"template_height":800007},{"accepted_at_ms":89977,"credit_policy":null,"job_id":"job-11","job_issued_at_ms":89988,"miner_id":"miner-0","network_difficulty":1000011,"ntime":1700000011,"order_key":"00:miner-0","p2mr_program_hex":"0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b","share_difficulty":2,"share_id":"share-11","share_seq":11,"template_height":800000},{"accepted_at_ms":89994,"credit_policy":null,"job_id":"job-101","job_issued_at_ms":89977,"miner_id":"miner-2","network_difficulty":1000016,"ntime":1700000101,"order_key":"02:miner-2","p2mr_program_hex":"6565656565656565656565656565656565656565656565656565656565656565","share_difficulty":3,"share_id":"share-101","share_seq":101,"template_height":800003},{"accepted_at_ms":89985,"credit_policy":"stale-grace","job_id":"job-618","job_issued_at_ms":89970,"miner_id":"miner-2","network_difficulty":1000006,"ntime":1700000618,"order_key":"02:miner-2","p2mr_program_hex":"6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a","share_difficulty":2,"share_id":"share-618","share_seq":618,"template_height":800019},{"accepted_at_ms":89999,"credit_policy":null,"job_id":"job-1024","job_issued_at_ms":89972,"miner_id":"miner-1","network_difficulty":1000004,"ntime":1700001024,"order_key":"01:miner-1","p2mr_program_hex":"0000000000000000000000000000000000000000000000000000000000000000","share_difficulty":7,"share_id":"share-1024","share_seq":1024,"template_height":800032},{"accepted_at_ms":89985,"credit_policy":null,"job_id":"job-1679","job_issued_at_ms":89980,"miner_id":"miner-7","network_difficulty":1000013,"ntime":1700001679,"order_key":"07:miner-7","p2mr_program_hex":"8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f","share_difficulty":7,"share_id":"share-1679","share_seq":1679,"template_height":800052},{"accepted_at_ms":89971,"credit_policy":"stale-grace","job_id":"job-188","job_issued_at_ms":89995,"miner_id":"miner-1","network_difficulty":1000001,"ntime":1700000188,"order_key":"01:miner-1","p2mr_program_hex":"bcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbc","share_difficulty":2,"share_id":"share-188","share_seq":188,"template_height":800005},{"accepted_at_ms":89980,"credit_policy":"stale-grace","job_id":"job-1343","job_issued_at_ms":89988,"miner_id":"miner-1","network_difficulty":1000000,"ntime":1700001343,"order_key":"01:miner-1","p2mr_program_hex":"3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f","share_difficulty":7,"share_id":"share-1343","share_seq":1343,"template_height":800041},{"accepted_at_ms":89981,"credit_policy":"stale-grace","job_id":"job-431","job_issued_at_ms":89987,"miner_id":"miner-2","network_difficulty":1000006,"ntime":1700000431,"order_key":"02:miner-2","p2mr_program_hex":"afafafafafafafafafafafafafafafafafafafafafafafafafafafafafafafaf","share_difficulty":1,"share_id":"share-431","share_seq":431,"template_height":800013},{"accepted_at_ms":89995,"credit_policy":null,"job_id":"job-1758","job_issued_at_ms":89990,"miner_id":"miner-9","network_difficulty":1000007,"ntime":1700001758,"order_key":"09:miner-9","p2mr_program_hex":"dededededededededededededededededededededededededededededededede","share_difficulty":1,"share_id":"share-1758","share_seq":1758,"template_height":800054},{"accepted_at_ms":89997,"credit_policy":null,"job_id":"job-1446","job_issued_at_ms":89987,"miner_id":"miner-5","network_difficulty":1000001,"ntime":1700001446,"order_key":"05:miner-5","p2mr_program_hex":"a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6","share_difficulty":1,"share_id":"share-1446","share_seq":1446,"template_height":800045},{"accepted_at_ms":89979,"credit_policy":null,"job_id":"job-218","job_issued_at_ms":89976,"miner_id":"miner-9","network_difficulty":1000014,"ntime":1700000218,"order_key":"09:miner-9","p2mr_program_hex":"dadadadadadadadadadadadadadadadadadadadadadadadadadadadadadadada","share_difficulty":3,"share_id":"share-218","share_seq":218,"template_height":800006},{"accepted_at_ms":89987,"credit_policy":null,"job_id":"job-206","job_issued_at_ms":89973,"miner_id":"miner-8","network_difficulty":1000002,"ntime":1700000206,"order_key":"08:miner-8","p2mr_program_hex":"cececececececececececececececececececececececececececececececece","share_difficulty":2,"share_id":"share-206","share_seq":206,"template_height":800006},{"accepted_at_ms":89986,"credit_policy":null,"job_id":"job-332","job_issued_at_ms":89976,"miner_id":"miner-2","network_difficulty":1000009,"ntime":1700000332,"order_key":"02:miner-2","p2mr_program_hex":"4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c","share_difficulty":19,"share_id":"share-332","share_seq":332,"template_height":800010},{"accepted_at_ms":89995,"credit_policy":"stale-grace","job_id":"job-511","job_issued_at_ms":89978,"miner_id":"miner-5","network_difficulty":1000001,"ntime":1700000511,"order_key":"05:miner-5","p2mr_program_hex":"ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff","share_difficulty":2,"share_id":"share-511","share_seq":511,"template_height":800015},{"accepted_at_ms":89993,"credit_policy":null,"job_id":"job-1381","job_issued_at_ms":89973,"miner_id":"miner-6","network_difficulty":1000004,"ntime":1700001381,"order_key":"06:miner-6","p2mr_program_hex":"6565656565656565656565656565656565656565656565656565656565656565","share_difficulty":19,"share_id":"share-1381","share_seq":1381,"template_height":800043},{"accepted_at_ms":89978,"credit_policy":"stale-grace","job_id":"job-1545","job_issued_at_ms":89979,"miner_id":"miner-5","network_difficulty":1000015,"ntime":1700001545,"order_key":"05:miner-5","p2mr_program_hex":"0909090909090909090909090909090909090909090909090909090909090909","share_difficulty":7,"share_id":"share-1545","share_seq":1545,"template_height":800048},{"accepted_at_ms":89970,"credit_policy":"stale-grace","job_id":"job-88","job_issued_at_ms":89975,"miner_id":"miner-0","network_difficulty":1000003,"ntime":1700000088,"order_key":"00:miner-0","p2mr_program_hex":"5858585858585858585858585858585858585858585858585858585858585858","share_difficulty":7,"share_id":"share-88","share_seq":88,"template_height":800002},{"accepted_at_ms":89976,"credit_policy":"stale-grace","job_id":"job-1001","job_issued_at_ms":89978,"miner_id":"miner-0","network_difficulty":1000015,"ntime":1700001001,"order_key":"00:miner-0","p2mr_program_hex":"e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9","share_difficulty":3,"share_id":"share-1001","share_seq":1001,"template_height":800031},{"accepted_at_ms":89986,"credit_policy":"stale-grace","job_id":"job-1052","job_issued_at_ms":89972,"miner_id":"miner-7","network_difficulty":1000015,"ntime":1700001052,"order_key":"07:miner-7","p2mr_program_hex":"1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c","share_difficulty":2,"share_id":"share-1052","share_seq":1052,"template_height":800032},{"accepted_at_ms":89981,"credit_policy":"stale-grace","job_id":"job-62","job_issued_at_ms":89988,"miner_id":"miner-7","network_difficulty":1000011,"ntime":1700000062,"order_key":"07:miner-7","p2mr_program_hex":"3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e","share_difficulty":3,"share_id":"share-62","share_seq":62,"template_height":800001},{"accepted_at_ms":89998,"credit_policy":null,"job_id":"job-635","job_issued_at_ms":89975,"miner_id":"miner-8","network_difficulty":1000006,"ntime":1700000635,"order_key":"08:miner-8","p2mr_program_hex":"7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b","share_difficulty":7,"share_id":"share-635","share_seq":635,"template_height":800019},{"accepted_at_ms":89978,"credit_policy":null,"job_id":"job-155","job_issued_at_ms":89971,"miner_id":"miner-1","network_difficulty":1000002,"ntime":1700000155,"order_key":"01:miner-1","p2mr_program_hex":"9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b","share_difficulty":2,"share_id":"share-155","share_seq":155,"template_height":800004},{"accepted_at_ms":89975,"credit_policy":null,"job_id":"job-1100","job_issued_at_ms":89989,"miner_id":"miner-0","network_difficulty":1000012,"ntime":1700001100,"order_key":"00:miner-0","p2mr_program_hex":"4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c","share_difficulty":19,"share_id":"share-1100","share_seq":1100,"template_height":800034},{"accepted_at_ms":89999,"credit_policy":"stale-grace","job_id":"job-1550","job_issued_at_ms":89972,"miner_id":"miner-10","network_difficulty":1000003,"ntime":1700001550,"order_key":"10:miner-10","p2mr_program_hex":"0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e","share_difficulty":19,"share_id":"share-1550","share_seq":1550,"template_height":800048},{"accepted_at_ms":89990,"credit_policy":null,"job_id":"job-558","job_issued_at_ms":89972,"miner_id":"miner-8","network_difficulty":1000014,"ntime":1700000558,"order_key":"08:miner-8","p2mr_program_hex":"2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e","share_difficulty":7,"share_id":"share-558","share_seq":558,"template_height":800017},{"accepted_at_ms":89970,"credit_policy":null,"job_id":"job-1170","job_issued_at_ms":89982,"miner_id":"miner-4","network_difficulty":1000014,"ntime":1700001170,"order_key":"04:miner-4","p2mr_program_hex":"9292929292929292929292929292929292929292929292929292929292929292","share_difficulty":19,"share_id":"share-1170","share_seq":1170,"template_height":800036},{"accepted_at_ms":89998,"credit_policy":"stale-grace","job_id":"job-94","job_issued_at_ms":89989,"miner_id":"miner-6","network_difficulty":1000009,"ntime":1700000094,"order_key":"06:miner-6","p2mr_program_hex":"5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e","share_difficulty":2,"share_id":"share-94","share_seq":94,"template_height":800002},{"accepted_at_ms":89971,"credit_policy":"stale-grace","job_id":"job-1412","job_issued_at_ms":89971,"miner_id":"miner-4","network_difficulty":1000001,"ntime":1700001412,"order_key":"04:miner-4","p2mr_program_hex":"8484848484848484848484848484848484848484848484848484848484848484","share_difficulty":2,"share_id":"share-1412","share_seq":1412,"template_height":800044},{"accepted_at_ms":89983,"credit_policy":"stale-grace","job_id":"job-699","job_issued_at_ms":89983,"miner_id":"miner-6","network_difficulty":1000002,"ntime":1700000699,"order_key":"06:miner-6","p2mr_program_hex":"bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb","share_difficulty":19,"share_id":"share-699","share_seq":699,"template_height":800021},{"accepted_at_ms":89997,"credit_policy":"stale-grace","job_id":"job-978","job_issued_at_ms":89986,"miner_id":"miner-10","network_difficulty":1000009,"ntime":1700000978,"order_key":"10:miner-10","p2mr_program_hex":"d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2","share_difficulty":3,"share_id":"share-978","share_seq":978,"template_height":800030},{"accepted_at_ms":89976,"credit_policy":null,"job_id":"job-789","job_issued_at_ms":89994,"miner_id":"miner-8","network_difficulty":1000007,"ntime":1700000789,"order_key":"08:miner-8","p2mr_program_hex":"1515151515151515151515151515151515151515151515151515151515151515","share_difficulty":3,"share_id":"share-789","share_seq":789,"template_height":800024},{"accepted_at_ms":89988,"credit_policy":null,"job_id":"job-1470","job_issued_at_ms":89982,"miner_id":"miner-7","network_difficulty":1000008,"ntime":1700001470,"order_key":"07:miner-7","p2mr_program_hex":"bebebebebebebebebebebebebebebebebebebebebebebebebebebebebebebebe","share_difficulty":2,"share_id":"share-1470","share_seq":1470,"template_height":800045},{"accepted_at_ms":89973,"credit_policy":"stale-grace","job_id":"job-728","job_issued_at_ms":89999,"miner_id":"miner-2","network_difficulty":1000014,"ntime":1700000728,"order_key":"02:miner-2","p2mr_program_hex":"d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8","share_difficulty":7,"share_id":"share-728","share_seq":728,"template_height":800022},{"accepted_at_ms":89993,"credit_policy":null,"job_id":"job-955","job_issued_at_ms":89972,"miner_id":"miner-9","network_difficulty":1000003,"ntime":1700000955,"order_key":"09:miner-9","p2mr_program_hex":"bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb","share_difficulty":3,"share_id":"share-955","share_seq":955,"template_height":800029},{"accepted_at_ms":89999,"credit_policy":null,"job_id":"job-1695","job_issued_at_ms":89972,"miner_id":"miner-1","network_difficulty":1000012,"ntime":1700001695,"order_key":"01:miner-1","p2mr_program_hex":"9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f","share_difficulty":3,"share_id":"share-1695","share_seq":1695,"template_height":800052},{"accepted_at_ms":89992,"credit_policy":null,"job_id":"job-1791","job_issued_at_ms":89982,"miner_id":"miner-9","network_difficulty":1000006,"ntime":1700001791,"order_key":"09:miner-9","p2mr_program_hex":"ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff","share_difficulty":3,"share_id":"share-1791","share_seq":1791,"template_height":800055},{"accepted_at_ms":89980,"credit_policy":null,"job_id":"job-1081","job_issued_at_ms":89974,"miner_id":"miner-3","network_difficulty":1000010,"ntime":1700001081,"order_key":"03:miner-3","p2mr_program_hex":"3939393939393939393939393939393939393939393939393939393939393939","share_difficulty":7,"share_id":"share-1081","share_seq":1081,"template_height":800033},{"accepted_at_ms":89980,"credit_policy":null,"job_id":"job-310","job_issued_at_ms":89995,"miner_id":"miner-2","network_difficulty":1000004,"ntime":1700000310,"order_key":"02:miner-2","p2mr_program_hex":"3636363636363636363636363636363636363636363636363636363636363636","share_difficulty":3,"share_id":"share-310","share_seq":310,"template_height":800009},{"accepted_at_ms":89978,"credit_policy":"stale-grace","job_id":"job-1107","job_issued_at_ms":89975,"miner_id":"miner-7","network_difficulty":1000002,"ntime":1700001107,"order_key":"07:miner-7","p2mr_program_hex":"5353535353535353535353535353535353535353535353535353535353535353","share_difficulty":7,"share_id":"share-1107","share_seq":1107,"template_height":800034},{"accepted_at_ms":89993,"credit_policy":null,"job_id":"job-629","job_issued_at_ms":89972,"miner_id":"miner-2","network_difficulty":1000000,"ntime":1700000629,"order_key":"02:miner-2","p2mr_program_hex":"7575757575757575757575757575757575757575757575757575757575757575","share_difficulty":19,"share_id":"share-629","share_seq":629,"template_height":800019},{"accepted_at_ms":89992,"credit_policy":null,"job_id":"job-443","job_issued_at_ms":89990,"miner_id":"miner-3","network_difficulty":1000001,"ntime":1700000443,"order_key":"03:miner-3","p2mr_program_hex":"bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb","share_difficulty":1,"share_id":"share-443","share_seq":443,"template_height":800013},{"accepted_at_ms":89975,"credit_policy":null,"job_id":"job-656","job_issued_at_ms":90000,"miner_id":"miner-7","network_difficulty":1000010,"ntime":1700000656,"order_key":"07:miner-7","p2mr_program_hex":"9090909090909090909090909090909090909090909090909090909090909090","share_difficulty":1,"share_id":"share-656","share_seq":656,"template_height":800020},{"accepted_at_ms":89998,"credit_policy":null,"job_id":"job-1745","job_issued_at_ms":89982,"miner_id":"miner-7","network_difficulty":1000011,"ntime":1700001745,"order_key":"07:miner-7","p2mr_program_hex":"d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1","share_difficulty":2,"share_id":"share-1745","share_seq":1745,"template_height":800054},{"accepted_at_ms":89980,"credit_policy":null,"job_id":"job-898","job_issued_at_ms":90000,"miner_id":"miner-7","network_difficulty":1000014,"ntime":1700000898,"order_key":"07:miner-7","p2mr_program_hex":"8282828282828282828282828282828282828282828282828282828282828282","share_difficulty":7,"share_id":"share-898","share_seq":898,"template_height":800028},{"accepted_at_ms":89985,"credit_policy":null,"job_id":"job-331","job_issued_at_ms":89985,"miner_id":"miner-1","network_difficulty":1000008,"ntime":1700000331,"order_key":"01:miner-1","p2mr_program_hex":"4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b","share_difficulty":1,"share_id":"share-331","share_seq":331,"template_height":800010},{"accepted_at_ms":89998,"credit_policy":null,"job_id":"job-910","job_issued_at_ms":89999,"miner_id":"miner-8","network_difficulty":1000009,"ntime":1700000910,"order_key":"08:miner-8","p2mr_program_hex":"8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e","share_difficulty":2,"share_id":"share-910","share_seq":910,"template_height":800028},{"accepted_at_ms":89983,"credit_policy":null,"job_id":"job-1625","job_issued_at_ms":89976,"miner_id":"miner-8","network_difficulty":1000010,"ntime":1700001625,"order_key":"08:miner-8","p2mr_program_hex":"5959595959595959595959595959595959595959595959595959595959595959","share_difficulty":1,"share_id":"share-1625","share_seq":1625,"template_height":800050},{"accepted_at_ms":89978,"credit_policy":null,"job_id":"job-893","job_issued_at_ms":89986,"miner_id":"miner-2","network_difficulty":1000009,"ntime":1700000893,"order_key":"02:miner-2","p2mr_program_hex":"7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d","share_difficulty":19,"share_id":"share-893","share_seq":893,"template_height":800027},{"accepted_at_ms":89978,"credit_policy":null,"job_id":"job-1274","job_issued_at_ms":89982,"miner_id":"miner-9","network_difficulty":1000016,"ntime":1700001274,"order_key":"09:miner-9","p2mr_program_hex":"fafafafafafafafafafafafafafafafafafafafafafafafafafafafafafafafa","share_difficulty":3,"share_id":"share-1274","share_seq":1274,"template_height":800039},{"accepted_at_ms":90000,"credit_policy":null,"job_id":"job-1566","job_issued_at_ms":89999,"miner_id":"miner-4","network_difficulty":1000002,"ntime":1700001566,"order_key":"04:miner-4","p2mr_program_hex":"1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e","share_difficulty":1,"share_id":"share-1566","share_seq":1566,"template_height":800048},{"accepted_at_ms":89997,"credit_policy":"stale-grace","job_id":"job-1085","job_issued_at_ms":89980,"miner_id":"miner-7","network_difficulty":1000014,"ntime":1700001085,"order_key":"07:miner-7","p2mr_program_hex":"3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d","share_difficulty":1,"share_id":"share-1085","share_seq":1085,"template_height":800033},{"accepted_at_ms":89979,"credit_policy":null,"job_id":"job-195","job_issued_at_ms":89988,"miner_id":"miner-8","network_difficulty":1000008,"ntime":1700000195,"order_key":"08:miner-8","p2mr_program_hex":"c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3","share_difficulty":3,"share_id":"share-195","share_seq":195,"template_height":800006},{"accepted_at_ms":90000,"credit_policy":null,"job_id":"job-1169","job_issued_at_ms":89991,"miner_id":"miner-3","network_difficulty":1000013,"ntime":1700001169,"order_key":"03:miner-3","p2mr_program_hex":"9191919191919191919191919191919191919191919191919191919191919191","share_difficulty":1,"share_id":"share-1169","share_seq":1169,"template_height":800036},{"accepted_at_ms":90000,"credit_policy":null,"job_id":"job-8","job_issued_at_ms":89985,"miner_id":"miner-8","network_difficulty":1000008,"ntime":1700000008,"order_key":"08:miner-8","p2mr_program_hex":"0808080808080808080808080808080808080808080808080808080808080808","share_difficulty":2,"share_id":"share-8","share_seq":8,"template_height":800000},{"accepted_at_ms":89988,"credit_policy":null,"job_id":"job-361","job_issued_at_ms":89974,"miner_id":"miner-9","network_difficulty":1000004,"ntime":1700000361,"order_key":"09:miner-9","p2mr_program_hex":"6969696969696969696969696969696969696969696969696969696969696969","share_difficulty":7,"share_id":"share-361","share_seq":361,"template_height":800011},{"accepted_at_ms":89984,"credit_policy":null,"job_id":"job-1634","job_issued_at_ms":89987,"miner_id":"miner-6","network_difficulty":1000002,"ntime":1700001634,"order_key":"06:miner-6","p2mr_program_hex":"6262626262626262626262626262626262626262626262626262626262626262","share_difficulty":3,"share_id":"share-1634","share_seq":1634,"template_height":800051},{"accepted_at_ms":89996,"credit_policy":"stale-grace","job_id":"job-289","job_issued_at_ms":90000,"miner_id":"miner-3","network_difficulty":1000000,"ntime":1700000289,"order_key":"03:miner-3","p2mr_program_hex":"2121212121212121212121212121212121212121212121212121212121212121","share_difficulty":19,"share_id":"share-289","share_seq":289,"template_height":800009},{"accepted_at_ms":89975,"credit_policy":null,"job_id":"job-270","job_issued_at_ms":89998,"miner_id":"miner-6","network_difficulty":1000015,"ntime":1700000270,"order_key":"06:miner-6","p2mr_program_hex":"0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e","share_difficulty":2,"share_id":"share-270","share_seq":270,"template_height":800008},{"accepted_at_ms":89993,"credit_policy":null,"job_id":"job-757","job_issued_at_ms":89998,"miner_id":"miner-9","network_difficulty":1000009,"ntime":1700000757,"order_key":"09:miner-9","p2mr_program_hex":"f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5","share_difficulty":2,"share_id":"share-757","share_seq":757,"template_height":800023},{"accepted_at_ms":89976,"credit_policy":null,"job_id":"job-1536","job_issued_at_ms":89998,"miner_id":"miner-7","network_difficulty":1000006,"ntime":1700001536,"order_key":"07:miner-7","p2mr_program_hex":"0000000000000000000000000000000000000000000000000000000000000000","share_difficulty":1,"share_id":"share-1536","share_seq":1536,"template_height":800048},{"accepted_at_ms":89976,"credit_policy":"stale-grace","job_id":"job-1006","job_issued_at_ms":89999,"miner_id":"miner-5","network_difficulty":1000003,"ntime":1700001006,"order_key":"05:miner-5","p2mr_program_hex":"eeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeee","share_difficulty":3,"share_id":"share-1006","share_seq":1006,"template_height":800031},{"accepted_at_ms":89975,"credit_policy":"stale-grace","job_id":"job-1184","job_issued_at_ms":89974,"miner_id":"miner-7","network_difficulty":1000011,"ntime":1700001184,"order_key":"07:miner-7","p2mr_program_hex":"a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0","share_difficulty":7,"share_id":"share-1184","share_seq":1184,"template_height":800037},{"accepted_at_ms":89995,"credit_policy":null,"job_id":"job-1080","job_issued_at_ms":89978,"miner_id":"miner-2","network_difficulty":1000009,"ntime":1700001080,"order_key":"02:miner-2","p2mr_program_hex":"3838383838383838383838383838383838383838383838383838383838383838","share_difficulty":3,"share_id":"share-1080","share_seq":1080,"template_height":800033},{"accepted_at_ms":89995,"credit_policy":null,"job_id":"job-1572","job_issued_at_ms":89972,"miner_id":"miner-10","network_difficulty":1000008,"ntime":1700001572,"order_key":"10:miner-10","p2mr_program_hex":"2424242424242424242424242424242424242424242424242424242424242424","share_difficulty":1,"share_id":"share-1572","share_seq":1572,"template_height":800049},{"accepted_at_ms":89992,"credit_policy":null,"job_id":"job-1429","job_issued_at_ms":89986,"miner_id":"miner-10","network_difficulty":1000001,"ntime":1700001429,"order_key":"10:miner-10","p2mr_program_hex":"9595959595959595959595959595959595959595959595959595959595959595","share_difficulty":3,"share_id":"share-1429","share_seq":1429,"template_height":800044},{"accepted_at_ms":89972,"credit_policy":null,"job_id":"job-1097","job_issued_at_ms":89972,"miner_id":"miner-8","network_difficulty":1000009,"ntime":1700001097,"order_key":"08:miner-8","p2mr_program_hex":"4949494949494949494949494949494949494949494949494949494949494949","share_difficulty":19,"share_id":"share-1097","share_seq":1097,"template_height":800034},{"accepted_at_ms":89974,"credit_policy":null,"job_id":"job-722","job_issued_at_ms":89995,"miner_id":"miner-7","network_difficulty":1000008,"ntime":1700000722,"order_key":"07:miner-7","p2mr_program_hex":"d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2","share_difficulty":19,"share_id":"share-722","share_seq":722,"template_height":800022},{"accepted_at_ms":89980,"credit_policy":"stale-grace","job_id":"job-729","job_issued_at_ms":89995,"miner_id":"miner-3","network_difficulty":1000015,"ntime":1700000729,"order_key":"03:miner-3","p2mr_program_hex":"d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9","share_difficulty":7,"share_id":"share-729","share_seq":729,"template_height":800022},{"accepted_at_ms":89977,"credit_policy":null,"job_id":"job-565","job_issued_at_ms":89999,"miner_id":"miner-4","network_difficulty":1000004,"ntime":1700000565,"order_key":"04:miner-4","p2mr_program_hex":"3535353535353535353535353535353535353535353535353535353535353535","share_difficulty":19,"share_id":"share-565","share_seq":565,"template_height":800017},{"accepted_at_ms":89988,"credit_policy":null,"job_id":"job-1176","job_issued_at_ms":89999,"miner_id":"miner-10","network_difficulty":1000003,"ntime":1700001176,"order_key":"10:miner-10","p2mr_program_hex":"9898989898989898989898989898989898989898989898989898989898989898","share_difficulty":2,"share_id":"share-1176","share_seq":1176,"template_height":800036},{"accepted_at_ms":89997,"credit_policy":"stale-grace","job_id":"job-817","job_issued_at_ms":89999,"miner_id":"miner-3","network_difficulty":1000001,"ntime":1700000817,"order_key":"03:miner-3","p2mr_program_hex":"3131313131313131313131313131313131313131313131313131313131313131","share_difficulty":1,"share_id":"share-817","share_seq":817,"template_height":800025},{"accepted_at_ms":89995,"credit_policy":null,"job_id":"job-1113","job_issued_at_ms":89997,"miner_id":"miner-2","network_difficulty":1000008,"ntime":1700001113,"order_key":"02:miner-2","p2mr_program_hex":"5959595959595959595959595959595959595959595959595959595959595959","share_difficulty":7,"share_id":"share-1113","share_seq":1113,"template_height":800034},{"accepted_at_ms":89994,"credit_policy":"stale-grace","job_id":"job-1718","job_issued_at_ms":89991,"miner_id":"miner-2","network_difficulty":1000001,"ntime":1700001718,"order_key":"02:miner-2","p2mr_program_hex":"b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6","share_difficulty":3,"share_id":"share-1718","share_seq":1718,"template_height":800053},{"accepted_at_ms":89995,"credit_policy":null,"job_id":"job-643","job_issued_at_ms":89990,"miner_id":"miner-5","network_difficulty":1000014,"ntime":1700000643,"order_key":"05:miner-5","p2mr_program_hex":"8383838383838383838383838383838383838383838383838383838383838383","share_difficulty":19,"share_id":"share-643","share_seq":643,"template_height":800020},{"accepted_at_ms":89979,"credit_policy":null,"job_id":"job-1165","job_issued_at_ms":89987,"miner_id":"miner-10","network_difficulty":1000009,"ntime":1700001165,"order_key":"10:miner-10","p2mr_program_hex":"8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d","share_difficulty":1,"share_id":"share-1165","share_seq":1165,"template_height":800036},{"accepted_at_ms":89989,"credit_policy":null,"job_id":"job-449","job_issued_at_ms":89988,"miner_id":"miner-9","network_difficulty":1000007,"ntime":1700000449,"order_key":"09:miner-9","p2mr_program_hex":"c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1","share_difficulty":3,"share_id":"share-449","share_seq":449,"template_height":800014},{"accepted_at_ms":89977,"credit_policy":null,"job_id":"job-1087","job_issued_at_ms":89999,"miner_id":"miner-9","network_difficulty":1000016,"ntime":1700001087,"order_key":"09:miner-9","p2mr_program_hex":"3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f","share_difficulty":2,"share_id":"share-1087","share_seq":1087,"template_height":800033},{"accepted_at_ms":89975,"credit_policy":null,"job_id":"job-1734","job_issued_at_ms":89973,"miner_id":"miner-7","network_difficulty":1000000,"ntime":1700001734,"order_key":"07:miner-7","p2mr_program_hex":"c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6","share_difficulty":7,"share_id":"share-1734","share_seq":1734,"template_height":800054},{"accepted_at_ms":89980,"credit_policy":"stale-grace","job_id":"job-342","job_issued_at_ms":89985,"miner_id":"miner-1","network_difficulty":1000002,"ntime":1700000342,"order_key":"01:miner-1","p2mr_program_hex":"5656565656565656565656565656565656565656565656565656565656565656","share_difficulty":7,"share_id":"share-342","share_seq":342,"template_height":800010},{"accepted_at_ms":89988,"credit_policy":null,"job_id":"job-139","job_issued_at_ms":89986,"miner_id":"miner-7","network_difficulty":1000003,"ntime":1700000139,"order_key":"07:miner-7","p2mr_program_hex":"8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b","share_difficulty":1,"share_id":"share-139","share_seq":139,"template_height":800004},{"accepted_at_ms":89995,"credit_policy":null,"job_id":"job-936","job_issued_at_ms":89981,"miner_id":"miner-1","network_difficulty":1000001,"ntime":1700000936,"order_key":"01:miner-1","p2mr_program_hex":"a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8","share_difficulty":1,"share_id":"share-936","share_seq":936,"template_height":800029},{"accepted_at_ms":89989,"credit_policy":"stale-grace","job_id":"job-391","job_issued_at_ms":89972,"miner_id":"miner-6","network_difficulty":1000000,"ntime":1700000391,"order_key":"06:miner-6","p2mr_program_hex":"8787878787878787878787878787878787878787878787878787878787878787","share_difficulty":3,"share_id":"share-391","share_seq":391,"template_height":800012},{"accepted_at_ms":90000,"credit_policy":null,"job_id":"job-1124","job_issued_at_ms":89991,"miner_id":"miner-2","network_difficulty":1000002,"ntime":1700001124,"order_key":"02:miner-2","p2mr_program_hex":"6464646464646464646464646464646464646464646464646464646464646464","share_difficulty":19,"share_id":"share-1124","share_seq":1124,"template_height":800035},{"accepted_at_ms":89973,"credit_policy":null,"job_id":"job-400","job_issued_at_ms":89981,"miner_id":"miner-4","network_difficulty":1000009,"ntime":1700000400,"order_key":"04:miner-4","p2mr_program_hex":"9090909090909090909090909090909090909090909090909090909090909090","share_difficulty":2,"share_id":"share-400","share_seq":400,"template_height":800012},{"accepted_at_ms":89989,"credit_policy":"stale-grace","job_id":"job-1216","job_issued_at_ms":89999,"miner_id":"miner-6","network_difficulty":1000009,"ntime":1700001216,"order_key":"06:miner-6","p2mr_program_hex":"c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0","share_difficulty":2,"share_id":"share-1216","share_seq":1216,"template_height":800038},{"accepted_at_ms":89980,"credit_policy":null,"job_id":"job-827","job_issued_at_ms":89978,"miner_id":"miner-2","network_difficulty":1000011,"ntime":1700000827,"order_key":"02:miner-2","p2mr_program_hex":"3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b","share_difficulty":1,"share_id":"share-827","share_seq":827,"template_height":800025},{"accepted_at_ms":89993,"credit_policy":null,"job_id":"job-78","job_issued_at_ms":90000,"miner_id":"miner-1","network_difficulty":1000010,"ntime":1700000078,"order_key":"01:miner-1","p2mr_program_hex":"4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e","share_difficulty":3,"share_id":"share-78","share_seq":78,"template_height":800002},{"accepted_at_ms":89978,"credit_policy":"stale-grace","job_id":"job-1729","job_issued_at_ms":89994,"miner_id":"miner-2","network_difficulty":1000012,"ntime":1700001729,"order_key":"02:miner-2","p2mr_program_hex":"c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1","share_difficulty":3,"share_id":"share-1729","share_seq":1729,"template_height":800054},{"accepted_at_ms":89989,"credit_policy":null,"job_id":"job-874","job_issued_at_ms":89970,"miner_id":"miner-5","network_difficulty":1000007,"ntime":1700000874,"order_key":"05:miner-5","p2mr_program_hex":"6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a","share_difficulty":2,"share_id":"share-874","share_seq":874,"template_height":800027},{"accepted_at_ms":89989,"credit_policy":null,"job_id":"job-291","job_issued_at_ms":89984,"miner_id":"miner-5","network_difficulty":1000002,"ntime":1700000291,"order_key":"05:miner-5","p2mr_program_hex":"2323232323232323232323232323232323232323232323232323232323232323","share_difficulty":1,"share_id":"share-291","share_seq":291,"template_height":800009},{"accepted_at_ms":89981,"credit_policy":null,"job_id":"job-190","job_issued_at_ms":89979,"miner_id":"miner-3","network_difficulty":1000003,"ntime":1700000190,"order_key":"03:miner-3","p2mr_program_hex":"bebebebebebebebebebebebebebebebebebebebebebebebebebebebebebebebe","share_difficulty":2,"share_id":"share-190","share_seq":190,"template_height":800005},{"accepted_at_ms":89978,"credit_policy":null,"job_id":"job-787","job_issued_at_ms":89993,"miner_id":"miner-6","network_difficulty":1000005,"ntime":1700000787,"order_key":"06:miner-6","p2mr_program_hex":"1313131313131313131313131313131313131313131313131313131313131313","share_difficulty":1,"share_id":"share-787","share_seq":787,"template_height":800024},{"accepted_at_ms":89973,"credit_policy":null,"job_id":"job-286","job_issued_at_ms":89985,"miner_id":"miner-0","network_difficulty":1000014,"ntime":1700000286,"order_key":"00:miner-0","p2mr_program_hex":"1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e","share_difficulty":19,"share_id":"share-286","share_seq":286,"template_height":800008},{"accepted_at_ms":89998,"credit_policy":"stale-grace","job_id":"job-277","job_issued_at_ms":89984,"miner_id":"miner-2","network_difficulty":1000005,"ntime":1700000277,"order_key":"02:miner-2","p2mr_program_hex":"1515151515151515151515151515151515151515151515151515151515151515","share_difficulty":19,"share_id":"share-277","share_seq":277,"template_height":800008},{"accepted_at_ms":89993,"credit_policy":null,"job_id":"job-1320","job_issued_at_ms":89974,"miner_id":"miner-0","network_difficulty":1000011,"ntime":1700001320,"order_key":"00:miner-0","p2mr_program_hex":"2828282828282828282828282828282828282828282828282828282828282828","share_difficulty":3,"share_id":"share-1320","share_seq":1320,"template_height":800041},{"accepted_at_ms":89989,"credit_policy":null,"job_id":"job-1315","job_issued_at_ms":89981,"miner_id":"miner-6","network_difficulty":1000006,"ntime":1700001315,"order_key":"06:miner-6","p2mr_program_hex":"2323232323232323232323232323232323232323232323232323232323232323","share_difficulty":19,"share_id":"share-1315","share_seq":1315,"template_height":800041},{"accepted_at_ms":89976,"credit_policy":null,"job_id":"job-995","job_issued_at_ms":89994,"miner_id":"miner-5","network_difficulty":1000009,"ntime":1700000995,"order_key":"05:miner-5","p2mr_program_hex":"e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3","share_difficulty":1,"share_id":"share-995","share_seq":995,"template_height":800031},{"accepted_at_ms":89982,"credit_policy":null,"job_id":"job-1686","job_issued_at_ms":89982,"miner_id":"miner-3","network_difficulty":1000003,"ntime":1700001686,"order_key":"03:miner-3","p2mr_program_hex":"9696969696969696969696969696969696969696969696969696969696969696","share_difficulty":7,"share_id":"share-1686","share_seq":1686,"template_height":800052},{"accepted_at_ms":89987,"credit_policy":null,"job_id":"job-703","job_issued_at_ms":90000,"miner_id":"miner-10","network_difficulty":1000006,"ntime":1700000703,"order_key":"10:miner-10","p2mr_program_hex":"bfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbf","share_difficulty":19,"share_id":"share-703","share_seq":703,"template_height":800021},{"accepted_at_ms":89989,"credit_policy":null,"job_id":"job-107","job_issued_at_ms":89974,"miner_id":"miner-8","network_difficulty":1000005,"ntime":1700000107,"order_key":"08:miner-8","p2mr_program_hex":"6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b","share_difficulty":1,"share_id":"share-107","share_seq":107,"template_height":800003},{"accepted_at_ms":89973,"credit_policy":null,"job_id":"job-180","job_issued_at_ms":89971,"miner_id":"miner-4","network_difficulty":1000010,"ntime":1700000180,"order_key":"04:miner-4","p2mr_program_hex":"b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4","share_difficulty":2,"share_id":"share-180","share_seq":180,"template_height":800005},{"accepted_at_ms":89999,"credit_policy":null,"job_id":"job-241","job_issued_at_ms":89982,"miner_id":"miner-10","network_difficulty":1000003,"ntime":1700000241,"order_key":"10:miner-10","p2mr_program_hex":"f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1","share_difficulty":2,"share_id":"share-241","share_seq":241,"template_height":800007},{"accepted_at_ms":89977,"credit_policy":null,"job_id":"job-1149","job_issued_at_ms":89985,"miner_id":"miner-5","network_difficulty":1000010,"ntime":1700001149,"order_key":"05:miner-5","p2mr_program_hex":"7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d","share_difficulty":2,"share_id":"share-1149","share_seq":1149,"template_height":800035},{"accepted_at_ms":89987,"credit_policy":null,"job_id":"job-1079","job_issued_at_ms":89986,"miner_id":"miner-1","network_difficulty":1000008,"ntime":1700001079,"order_key":"01:miner-1","p2mr_program_hex":"3737373737373737373737373737373737373737373737373737373737373737","share_difficulty":1,"share_id":"share-1079","share_seq":1079,"template_height":800033},{"accepted_at_ms":89991,"credit_policy":"stale-grace","job_id":"job-1293","job_issued_at_ms":89987,"miner_id":"miner-6","network_difficulty":1000001,"ntime":1700001293,"order_key":"06:miner-6","p2mr_program_hex":"0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d","share_difficulty":1,"share_id":"share-1293","share_seq":1293,"template_height":800040},{"accepted_at_ms":89992,"credit_policy":"stale-grace","job_id":"job-25","job_issued_at_ms":89981,"miner_id":"miner-3","network_difficulty":1000008,"ntime":1700000025,"order_key":"03:miner-3","p2mr_program_hex":"1919191919191919191919191919191919191919191919191919191919191919","share_difficulty":3,"share_id":"share-25","share_seq":25,"template_height":800000},{"accepted_at_ms":89995,"credit_policy":null,"job_id":"job-298","job_issued_at_ms":89979,"miner_id":"miner-1","network_difficulty":1000009,"ntime":1700000298,"order_key":"01:miner-1","p2mr_program_hex":"2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a","share_difficulty":1,"share_id":"share-298","share_seq":298,"template_height":800009},{"accepted_at_ms":89988,"credit_policy":null,"job_id":"job-499","job_issued_at_ms":89994,"miner_id":"miner-4","network_difficulty":1000006,"ntime":1700000499,"order_key":"04:miner-4","p2mr_program_hex":"f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3","share_difficulty":2,"share_id":"share-499","share_seq":499,"template_height":800015},{"accepted_at_ms":89973,"credit_policy":null,"job_id":"job-748","job_issued_at_ms":89971,"miner_id":"miner-0","network_difficulty":1000000,"ntime":1700000748,"order_key":"00:miner-0","p2mr_program_hex":"ecececececececececececececececececececececececececececececececec","share_difficulty":2,"share_id":"share-748","share_seq":748,"template_height":800023},{"accepted_at_ms":89990,"credit_policy":null,"job_id":"job-843","job_issued_at_ms":89996,"miner_id":"miner-7","network_difficulty":1000010,"ntime":1700000843,"order_key":"07:miner-7","p2mr_program_hex":"4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b","share_difficulty":2,"share_id":"share-843","share_seq":843,"template_height":800026},{"accepted_at_ms":89977,"credit_policy":null,"job_id":"job-814","job_issued_at_ms":89992,"miner_id":"miner-0","network_difficulty":1000015,"ntime":1700000814,"order_key":"00:miner-0","p2mr_program_hex":"2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e","share_difficulty":2,"share_id":"share-814","share_seq":814,"template_height":800025},{"accepted_at_ms":89982,"credit_policy":null,"job_id":"job-521","job_issued_at_ms":89970,"miner_id":"miner-4","network_difficulty":1000011,"ntime":1700000521,"order_key":"04:miner-4","p2mr_program_hex":"0909090909090909090909090909090909090909090909090909090909090909","share_difficulty":19,"share_id":"share-521","share_seq":521,"template_height":800016},{"accepted_at_ms":89973,"credit_policy":null,"job_id":"job-994","job_issued_at_ms":89980,"miner_id":"miner-4","network_difficulty":1000008,"ntime":1700000994,"order_key":"04:miner-4","p2mr_program_hex":"e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2","share_difficulty":3,"share_id":"share-994","share_seq":994,"template_height":800031},{"accepted_at_ms":89984,"credit_policy":null,"job_id":"job-1008","job_issued_at_ms":89994,"miner_id":"miner-7","network_difficulty":1000005,"ntime":1700001008,"order_key":"07:miner-7","p2mr_program_hex":"f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0","share_difficulty":3,"share_id":"share-1008","share_seq":1008,"template_height":800031},{"accepted_at_ms":89982,"credit_policy":null,"job_id":"job-21","job_issued_at_ms":89982,"miner_id":"miner-10","network_difficulty":1000004,"ntime":1700000021,"order_key":"10:miner-10","p2mr_program_hex":"1515151515151515151515151515151515151515151515151515151515151515","share_difficulty":3,"share_id":"share-21","share_seq":21,"template_height":800000},{"accepted_at_ms":89971,"credit_policy":"stale-grace","job_id":"job-1687","job_issued_at_ms":89993,"miner_id":"miner-4","network_difficulty":1000004,"ntime":1700001687,"order_key":"04:miner-4","p2mr_program_hex":"9797979797979797979797979797979797979797979797979797979797979797","share_difficulty":19,"share_id":"share-1687","share_seq":1687,"template_height":800052},{"accepted_at_ms":89979,"credit_policy":null,"job_id":"job-1750","job_issued_at_ms":89978,"miner_id":"miner-1","network_difficulty":1000016,"ntime":1700001750,"order_key":"01:miner-1","p2mr_program_hex":"d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6","share_difficulty":3,"share_id":"share-1750","share_seq":1750,"template_height":800054},{"accepted_at_ms":89983,"credit_policy":"stale-grace","job_id":"job-922","job_issued_at_ms":89998,"miner_id":"miner-9","network_difficulty":1000004,"ntime":1700000922,"order_key":"09:miner-9","p2mr_program_hex":"9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a","share_difficulty":19,"share_id":"share-922","share_seq":922,"template_height":800028},{"accepted_at_ms":89983,"credit_policy":null,"job_id":"job-1139","job_issued_at_ms":89998,"miner_id":"miner-6","network_difficulty":1000000,"ntime":1700001139,"order_key":"06:miner-6","p2mr_program_hex":"7373737373737373737373737373737373737373737373737373737373737373","share_difficulty":3,"share_id":"share-1139","share_seq":1139,"template_height":800035},{"accepted_at_ms":89995,"credit_policy":null,"job_id":"job-103","job_issued_at_ms":89997,"miner_id":"miner-4","network_difficulty":1000001,"ntime":1700000103,"order_key":"04:miner-4","p2mr_program_hex":"6767676767676767676767676767676767676767676767676767676767676767","share_difficulty":2,"share_id":"share-103","share_seq":103,"template_height":800003},{"accepted_at_ms":89978,"credit_policy":null,"job_id":"job-497","job_issued_at_ms":89987,"miner_id":"miner-2","network_difficulty":1000004,"ntime":1700000497,"order_key":"02:miner-2","p2mr_program_hex":"f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1","share_difficulty":2,"share_id":"share-497","share_seq":497,"template_height":800015},{"accepted_at_ms":89982,"credit_policy":"stale-grace","job_id":"job-1321","job_issued_at_ms":89974,"miner_id":"miner-1","network_difficulty":1000012,"ntime":1700001321,"order_key":"01:miner-1","p2mr_program_hex":"2929292929292929292929292929292929292929292929292929292929292929","share_difficulty":1,"share_id":"share-1321","share_seq":1321,"template_height":800041},{"accepted_at_ms":89981,"credit_policy":null,"job_id":"job-1664","job_issued_at_ms":89995,"miner_id":"miner-3","network_difficulty":1000015,"ntime":1700001664,"order_key":"03:miner-3","p2mr_program_hex":"8080808080808080808080808080808080808080808080808080808080808080","share_difficulty":3,"share_id":"share-1664","share_seq":1664,"template_height":800052},{"accepted_at_ms":89987,"credit_policy":"stale-grace","job_id":"job-610","job_issued_at_ms":89982,"miner_id":"miner-5","network_difficulty":1000015,"ntime":1700000610,"order_key":"05:miner-5","p2mr_program_hex":"6262626262626262626262626262626262626262626262626262626262626262","share_difficulty":3,"share_id":"share-610","share_seq":610,"template_height":800019},{"accepted_at_ms":89976,"credit_policy":null,"job_id":"job-768","job_issued_at_ms":89980,"miner_id":"miner-9","network_difficulty":1000003,"ntime":1700000768,"order_key":"09:miner-9","p2mr_program_hex":"0000000000000000000000000000000000000000000000000000000000000000","share_difficulty":7,"share_id":"share-768","share_seq":768,"template_height":800024},{"accepted_at_ms":89975,"credit_policy":null,"job_id":"job-1027","job_issued_at_ms":89986,"miner_id":"miner-4","network_difficulty":1000007,"ntime":1700001027,"order_key":"04:miner-4","p2mr_program_hex":"0303030303030303030303030303030303030303030303030303030303030303","share_difficulty":3,"share_id":"share-1027","share_seq":1027,"template_height":800032},{"accepted_at_ms":89987,"credit_policy":null,"job_id":"job-549","job_issued_at_ms":89997,"miner_id":"miner-10","network_difficulty":1000005,"ntime":1700000549,"order_key":"10:miner-10","p2mr_program_hex":"2525252525252525252525252525252525252525252525252525252525252525","share_difficulty":3,"share_id":"share-549","share_seq":549,"template_height":800017},{"accepted_at_ms":89991,"credit_policy":"stale-grace","job_id":"job-569","job_issued_at_ms":89983,"miner_id":"miner-8","network_difficulty":1000008,"ntime":1700000569,"order_key":"08:miner-8","p2mr_program_hex":"3939393939393939393939393939393939393939393939393939393939393939","share_difficulty":1,"share_id":"share-569","share_seq":569,"template_height":800017},{"accepted_at_ms":89990,"credit_policy":"stale-grace","job_id":"job-1137","job_issued_at_ms":89979,"miner_id":"miner-4","network_difficulty":1000015,"ntime":1700001137,"order_key":"04:miner-4","p2mr_program_hex":"7171717171717171717171717171717171717171717171717171717171717171","share_difficulty":1,"share_id":"share-1137","share_seq":1137,"template_height":800035},{"accepted_at_ms":89977,"credit_policy":null,"job_id":"job-1800","job_issued_at_ms":89983,"miner_id":"miner-7","network_difficulty":1000015,"ntime":1700001800,"order_key":"07:miner-7","p2mr_program_hex":"0808080808080808080808080808080808080808080808080808080808080808","share_difficulty":7,"share_id":"share-1800","share_seq":1800,"template_height":800056},{"accepted_at_ms":89976,"credit_policy":null,"job_id":"job-40","job_issued_at_ms":89973,"miner_id":"miner-7","network_difficulty":1000006,"ntime":1700000040,"order_key":"07:miner-7","p2mr_program_hex":"2828282828282828282828282828282828282828282828282828282828282828","share_difficulty":19,"share_id":"share-40","share_seq":40,"template_height":800001},{"accepted_at_ms":89989,"credit_policy":"stale-grace","job_id":"job-1659","job_issued_at_ms":89972,"miner_id":"miner-9","network_difficulty":1000010,"ntime":1700001659,"order_key":"09:miner-9","p2mr_program_hex":"7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b","share_difficulty":7,"share_id":"share-1659","share_seq":1659,"template_height":800051},{"accepted_at_ms":89990,"credit_policy":null,"job_id":"job-1522","job_issued_at_ms":89989,"miner_id":"miner-4","network_difficulty":1000009,"ntime":1700001522,"order_key":"04:miner-4","p2mr_program_hex":"f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2","share_difficulty":7,"share_id":"share-1522","share_seq":1522,"template_height":800047},{"accepted_at_ms":89995,"credit_policy":null,"job_id":"job-1603","job_issued_at_ms":89979,"miner_id":"miner-8","network_difficulty":1000005,"ntime":1700001603,"order_key":"08:miner-8","p2mr_program_hex":"4343434343434343434343434343434343434343434343434343434343434343","share_difficulty":1,"share_id":"share-1603","share_seq":1603,"template_height":800050},{"accepted_at_ms":89995,"credit_policy":null,"job_id":"job-732","job_issued_at_ms":89989,"miner_id":"miner-6","network_difficulty":1000001,"ntime":1700000732,"order_key":"06:miner-6","p2mr_program_hex":"dcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdc","share_difficulty":1,"share_id":"share-732","share_seq":732,"template_height":800022},{"accepted_at_ms":89985,"credit_policy":null,"job_id":"job-1738","job_issued_at_ms":89990,"miner_id":"miner-0","network_difficulty":1000004,"ntime":1700001738,"order_key":"00:miner-0","p2mr_program_hex":"cacacacacacacacacacacacacacacacacacacacacacacacacacacacacacacaca","share_difficulty":3,"share_id":"share-1738","share_seq":1738,"template_height":800054},{"accepted_at_ms":89977,"credit_policy":"stale-grace","job_id":"job-1186","job_issued_at_ms":89983,"miner_id":"miner-9","network_difficulty":1000013,"ntime":1700001186,"order_key":"09:miner-9","p2mr_program_hex":"a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2","share_difficulty":19,"share_id":"share-1186","share_seq":1186,"template_height":800037},{"accepted_at_ms":89983,"credit_policy":null,"job_id":"job-1227","job_issued_at_ms":89975,"miner_id":"miner-6","network_difficulty":1000003,"ntime":1700001227,"order_key":"06:miner-6","p2mr_program_hex":"cbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcb","share_difficulty":2,"share_id":"share-1227","share_seq":1227,"template_height":800038},{"accepted_at_ms":89975,"credit_policy":"stale-grace","job_id":"job-1771","job_issued_at_ms":89992,"miner_id":"miner-0","network_difficulty":1000003,"ntime":1700001771,"order_key":"00:miner-0","p2mr_program_hex":"ebebebebebebebebebebebebebebebebebebebebebebebebebebebebebebebeb","share_difficulty":1,"share_id":"share-1771","share_seq":1771,"template_height":800055},{"accepted_at_ms":89995,"credit_policy":null,"job_id":"job-1383","job_issued_at_ms":89975,"miner_id":"miner-8","network_difficulty":1000006,"ntime":1700001383,"order_key":"08:miner-8","p2mr_program_hex":"6767676767676767676767676767676767676767676767676767676767676767","share_difficulty":1,"share_id":"share-1383","share_seq":1383,"template_height":800043},{"accepted_at_ms":89987,"credit_policy":"stale-grace","job_id":"job-63","job_issued_at_ms":89995,"miner_id":"miner-8","network_difficulty":1000012,"ntime":1700000063,"order_key":"08:miner-8","p2mr_program_hex":"3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f","share_difficulty":7,"share_id":"share-63","share_seq":63,"template_height":800001},{"accepted_at_ms":89976,"credit_policy":null,"job_id":"job-965","job_issued_at_ms":89983,"miner_id":"miner-8","network_difficulty":1000013,"ntime":1700000965,"order_key":"08:miner-8","p2mr_program_hex":"c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5","share_difficulty":7,"share_id":"share-965","share_seq":965,"template_height":800030},{"accepted_at_ms":89986,"credit_policy":null,"job_id":"job-861","job_issued_at_ms":89981,"miner_id":"miner-3","network_difficulty":1000011,"ntime":1700000861,"order_key":"03:miner-3","p2mr_program_hex":"5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d","share_difficulty":2,"share_id":"share-861","share_seq":861,"template_height":800026},{"accepted_at_ms":89996,"credit_policy":null,"job_id":"job-823","job_issued_at_ms":89976,"miner_id":"miner-9","network_difficulty":1000007,"ntime":1700000823,"order_key":"09:miner-9","p2mr_program_hex":"3737373737373737373737373737373737373737373737373737373737373737","share_difficulty":3,"share_id":"share-823","share_seq":823,"template_height":800025},{"accepted_at_ms":89984,"credit_policy":null,"job_id":"job-1565","job_issued_at_ms":89983,"miner_id":"miner-3","network_difficulty":1000001,"ntime":1700001565,"order_key":"03:miner-3","p2mr_program_hex":"1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d","share_difficulty":2,"share_id":"share-1565","share_seq":1565,"template_height":800048},{"accepted_at_ms":89993,"credit_policy":null,"job_id":"job-1091","job_issued_at_ms":89989,"miner_id":"miner-2","network_difficulty":1000003,"ntime":1700001091,"order_key":"02:miner-2","p2mr_program_hex":"4343434343434343434343434343434343434343434343434343434343434343","share_difficulty":7,"share_id":"share-1091","share_seq":1091,"template_height":800034},{"accepted_at_ms":89991,"credit_policy":"stale-grace","job_id":"job-329","job_issued_at_ms":89984,"miner_id":"miner-10","network_difficulty":1000006,"ntime":1700000329,"order_key":"10:miner-10","p2mr_program_hex":"4949494949494949494949494949494949494949494949494949494949494949","share_difficulty":19,"share_id":"share-329","share_seq":329,"template_height":800010},{"accepted_at_ms":89981,"credit_policy":null,"job_id":"job-344","job_issued_at_ms":89996,"miner_id":"miner-3","network_difficulty":1000004,"ntime":1700000344,"order_key":"03:miner-3","p2mr_program_hex":"5858585858585858585858585858585858585858585858585858585858585858","share_difficulty":1,"share_id":"share-344","share_seq":344,"template_height":800010},{"accepted_at_ms":89997,"credit_policy":null,"job_id":"job-1783","job_issued_at_ms":89990,"miner_id":"miner-1","network_difficulty":1000015,"ntime":1700001783,"order_key":"01:miner-1","p2mr_program_hex":"f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7","share_difficulty":2,"share_id":"share-1783","share_seq":1783,"template_height":800055},{"accepted_at_ms":89987,"credit_policy":null,"job_id":"job-1692","job_issued_at_ms":89997,"miner_id":"miner-9","network_difficulty":1000009,"ntime":1700001692,"order_key":"09:miner-9","p2mr_program_hex":"9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c","share_difficulty":19,"share_id":"share-1692","share_seq":1692,"template_height":800052},{"accepted_at_ms":89973,"credit_policy":null,"job_id":"job-710","job_issued_at_ms":89981,"miner_id":"miner-6","network_difficulty":1000013,"ntime":1700000710,"order_key":"06:miner-6","p2mr_program_hex":"c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6","share_difficulty":1,"share_id":"share-710","share_seq":710,"template_height":800022},{"accepted_at_ms":89980,"credit_policy":null,"job_id":"job-1307","job_issued_at_ms":89971,"miner_id":"miner-9","network_difficulty":1000015,"ntime":1700001307,"order_key":"09:miner-9","p2mr_program_hex":"1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b","share_difficulty":3,"share_id":"share-1307","share_seq":1307,"template_height":800040},{"accepted_at_ms":89982,"credit_policy":null,"job_id":"job-119","job_issued_at_ms":89980,"miner_id":"miner-9","network_difficulty":1000000,"ntime":1700000119,"order_key":"09:miner-9","p2mr_program_hex":"7777777777777777777777777777777777777777777777777777777777777777","share_difficulty":7,"share_id":"share-119","share_seq":119,"template_height":800003},{"accepted_at_ms":89976,"credit_policy":"stale-grace","job_id":"job-253","job_issued_at_ms":89982,"miner_id":"miner-0","network_difficulty":1000015,"ntime":1700000253,"order_key":"00:miner-0","p2mr_program_hex":"fdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfd","share_difficulty":7,"share_id":"share-253","share_seq":253,"template_height":800007},{"accepted_at_ms":89983,"credit_policy":null,"job_id":"job-988","job_issued_at_ms":89987,"miner_id":"miner-9","network_difficulty":1000002,"ntime":1700000988,"order_key":"09:miner-9","p2mr_program_hex":"dcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdc","share_difficulty":2,"share_id":"share-988","share_seq":988,"template_height":800030},{"accepted_at_ms":89982,"credit_policy":"stale-grace","job_id":"job-1268","job_issued_at_ms":89980,"miner_id":"miner-3","network_difficulty":1000010,"ntime":1700001268,"order_key":"03:miner-3","p2mr_program_hex":"f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4","share_difficulty":19,"share_id":"share-1268","share_seq":1268,"template_height":800039},{"accepted_at_ms":89992,"credit_policy":"stale-grace","job_id":"job-1083","job_issued_at_ms":89984,"miner_id":"miner-5","network_difficulty":1000012,"ntime":1700001083,"order_key":"05:miner-5","p2mr_program_hex":"3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b","share_difficulty":7,"share_id":"share-1083","share_seq":1083,"template_height":800033},{"accepted_at_ms":89987,"credit_policy":null,"job_id":"job-1121","job_issued_at_ms":89977,"miner_id":"miner-10","network_difficulty":1000016,"ntime":1700001121,"order_key":"10:miner-10","p2mr_program_hex":"6161616161616161616161616161616161616161616161616161616161616161","share_difficulty":3,"share_id":"share-1121","share_seq":1121,"template_height":800035},{"accepted_at_ms":89994,"credit_policy":null,"job_id":"job-1041","job_issued_at_ms":89977,"miner_id":"miner-7","network_difficulty":1000004,"ntime":1700001041,"order_key":"07:miner-7","p2mr_program_hex":"1111111111111111111111111111111111111111111111111111111111111111","share_difficulty":7,"share_id":"share-1041","share_seq":1041,"template_height":800032},{"accepted_at_ms":89974,"credit_policy":null,"job_id":"job-1480","job_issued_at_ms":89971,"miner_id":"miner-6","network_difficulty":1000001,"ntime":1700001480,"order_key":"06:miner-6","p2mr_program_hex":"c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8","share_difficulty":2,"share_id":"share-1480","share_seq":1480,"template_height":800046},{"accepted_at_ms":89983,"credit_policy":null,"job_id":"job-1535","job_issued_at_ms":89986,"miner_id":"miner-6","network_difficulty":1000005,"ntime":1700001535,"order_key":"06:miner-6","p2mr_program_hex":"ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff","share_difficulty":3,"share_id":"share-1535","share_seq":1535,"template_height":800047},{"accepted_at_ms":89997,"credit_policy":null,"job_id":"job-912","job_issued_at_ms":89987,"miner_id":"miner-10","network_difficulty":1000011,"ntime":1700000912,"order_key":"10:miner-10","p2mr_program_hex":"9090909090909090909090909090909090909090909090909090909090909090","share_difficulty":3,"share_id":"share-912","share_seq":912,"template_height":800028},{"accepted_at_ms":89986,"credit_policy":null,"job_id":"job-941","job_issued_at_ms":89980,"miner_id":"miner-6","network_difficulty":1000006,"ntime":1700000941,"order_key":"06:miner-6","p2mr_program_hex":"adadadadadadadadadadadadadadadadadadadadadadadadadadadadadadadad","share_difficulty":1,"share_id":"share-941","share_seq":941,"template_height":800029},{"accepted_at_ms":89976,"credit_policy":null,"job_id":"job-523","job_issued_at_ms":89982,"miner_id":"miner-6","network_difficulty":1000013,"ntime":1700000523,"order_key":"06:miner-6","p2mr_program_hex":"0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b","share_difficulty":3,"share_id":"share-523","share_seq":523,"template_height":800016},{"accepted_at_ms":89976,"credit_policy":null,"job_id":"job-1078","job_issued_at_ms":89983,"miner_id":"miner-0","network_difficulty":1000007,"ntime":1700001078,"order_key":"00:miner-0","p2mr_program_hex":"3636363636363636363636363636363636363636363636363636363636363636","share_difficulty":19,"share_id":"share-1078","share_seq":1078,"template_height":800033},{"accepted_at_ms":89977,"credit_policy":null,"job_id":"job-1584","job_issued_at_ms":89992,"miner_id":"miner-0","network_difficulty":1000003,"ntime":1700001584,"order_key":"00:miner-0","p2mr_program_hex":"3030303030303030303030303030303030303030303030303030303030303030","share_difficulty":3,"share_id":"share-1584","share_seq":1584,"template_height":800049},{"accepted_at_ms":89992,"credit_policy":"stale-grace","job_id":"job-1291","job_issued_at_ms":89995,"miner_id":"miner-4","network_difficulty":1000016,"ntime":1700001291,"order_key":"04:miner-4","p2mr_program_hex":"0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b","share_difficulty":2,"share_id":"share-1291","share_seq":1291,"template_height":800040},{"accepted_at_ms":89972,"credit_policy":"stale-grace","job_id":"job-518","job_issued_at_ms":89982,"miner_id":"miner-1","network_difficulty":1000008,"ntime":1700000518,"order_key":"01:miner-1","p2mr_program_hex":"0606060606060606060606060606060606060606060606060606060606060606","share_difficulty":3,"share_id":"share-518","share_seq":518,"template_height":800016},{"accepted_at_ms":89976,"credit_policy":null,"job_id":"job-1700","job_issued_at_ms":89985,"miner_id":"miner-6","network_difficulty":1000000,"ntime":1700001700,"order_key":"06:miner-6","p2mr_program_hex":"a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4","share_difficulty":1,"share_id":"share-1700","share_seq":1700,"template_height":800053},{"accepted_at_ms":89992,"credit_policy":"stale-grace","job_id":"job-1353","job_issued_at_ms":89978,"miner_id":"miner-0","network_difficulty":1000010,"ntime":1700001353,"order_key":"00:miner-0","p2mr_program_hex":"4949494949494949494949494949494949494949494949494949494949494949","share_difficulty":7,"share_id":"share-1353","share_seq":1353,"template_height":800042},{"accepted_at_ms":89981,"credit_policy":null,"job_id":"job-1202","job_issued_at_ms":89975,"miner_id":"miner-3","network_difficulty":1000012,"ntime":1700001202,"order_key":"03:miner-3","p2mr_program_hex":"b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2","share_difficulty":19,"share_id":"share-1202","share_seq":1202,"template_height":800037},{"accepted_at_ms":89996,"credit_policy":"stale-grace","job_id":"job-61","job_issued_at_ms":89984,"miner_id":"miner-6","network_difficulty":1000010,"ntime":1700000061,"order_key":"06:miner-6","p2mr_program_hex":"3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d","share_difficulty":19,"share_id":"share-61","share_seq":61,"template_height":800001},{"accepted_at_ms":89970,"credit_policy":null,"job_id":"job-1388","job_issued_at_ms":89985,"miner_id":"miner-2","network_difficulty":1000011,"ntime":1700001388,"order_key":"02:miner-2","p2mr_program_hex":"6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c","share_difficulty":3,"share_id":"share-1388","share_seq":1388,"template_height":800043},{"accepted_at_ms":89971,"credit_policy":null,"job_id":"job-566","job_issued_at_ms":89984,"miner_id":"miner-5","network_difficulty":1000005,"ntime":1700000566,"order_key":"05:miner-5","p2mr_program_hex":"3636363636363636363636363636363636363636363636363636363636363636","share_difficulty":1,"share_id":"share-566","share_seq":566,"template_height":800017},{"accepted_at_ms":89994,"credit_policy":null,"job_id":"job-765","job_issued_at_ms":89982,"miner_id":"miner-6","network_difficulty":1000000,"ntime":1700000765,"order_key":"06:miner-6","p2mr_program_hex":"fdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfd","share_difficulty":3,"share_id":"share-765","share_seq":765,"template_height":800023},{"accepted_at_ms":89975,"credit_policy":null,"job_id":"job-1773","job_issued_at_ms":89987,"miner_id":"miner-2","network_difficulty":1000005,"ntime":1700001773,"order_key":"02:miner-2","p2mr_program_hex":"edededededededededededededededededededededededededededededededed","share_difficulty":1,"share_id":"share-1773","share_seq":1773,"template_height":800055},{"accepted_at_ms":89994,"credit_policy":null,"job_id":"job-559","job_issued_at_ms":89993,"miner_id":"miner-9","network_difficulty":1000015,"ntime":1700000559,"order_key":"09:miner-9","p2mr_program_hex":"2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f","share_difficulty":2,"share_id":"share-559","share_seq":559,"template_height":800017},{"accepted_at_ms":89982,"credit_policy":"stale-grace","job_id":"job-1554","job_issued_at_ms":89977,"miner_id":"miner-3","network_difficulty":1000007,"ntime":1700001554,"order_key":"03:miner-3","p2mr_program_hex":"1212121212121212121212121212121212121212121212121212121212121212","share_difficulty":7,"share_id":"share-1554","share_seq":1554,"template_height":800048},{"accepted_at_ms":89972,"credit_policy":null,"job_id":"job-612","job_issued_at_ms":89997,"miner_id":"miner-7","network_difficulty":1000000,"ntime":1700000612,"order_key":"07:miner-7","p2mr_program_hex":"6464646464646464646464646464646464646464646464646464646464646464","share_difficulty":19,"share_id":"share-612","share_seq":612,"template_height":800019},{"accepted_at_ms":89985,"credit_policy":null,"job_id":"job-430","job_issued_at_ms":89991,"miner_id":"miner-1","network_difficulty":1000005,"ntime":1700000430,"order_key":"01:miner-1","p2mr_program_hex":"aeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeae","share_difficulty":7,"share_id":"share-430","share_seq":430,"template_height":800013},{"accepted_at_ms":89980,"credit_policy":null,"job_id":"job-1366","job_issued_at_ms":89991,"miner_id":"miner-2","network_difficulty":1000006,"ntime":1700001366,"order_key":"02:miner-2","p2mr_program_hex":"5656565656565656565656565656565656565656565656565656565656565656","share_difficulty":19,"share_id":"share-1366","share_seq":1366,"template_height":800042},{"accepted_at_ms":89986,"credit_policy":null,"job_id":"job-1067","job_issued_at_ms":89987,"miner_id":"miner-0","network_difficulty":1000013,"ntime":1700001067,"order_key":"00:miner-0","p2mr_program_hex":"2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b","share_difficulty":19,"share_id":"share-1067","share_seq":1067,"template_height":800033},{"accepted_at_ms":89992,"credit_policy":null,"job_id":"job-1237","job_issued_at_ms":89978,"miner_id":"miner-5","network_difficulty":1000013,"ntime":1700001237,"order_key":"05:miner-5","p2mr_program_hex":"d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5","share_difficulty":1,"share_id":"share-1237","share_seq":1237,"template_height":800038},{"accepted_at_ms":89971,"credit_policy":"stale-grace","job_id":"job-486","job_issued_at_ms":89981,"miner_id":"miner-2","network_difficulty":1000010,"ntime":1700000486,"order_key":"02:miner-2","p2mr_program_hex":"e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6","share_difficulty":1,"share_id":"share-486","share_seq":486,"template_height":800015},{"accepted_at_ms":89978,"credit_policy":"stale-grace","job_id":"job-372","job_issued_at_ms":89972,"miner_id":"miner-9","network_difficulty":1000015,"ntime":1700000372,"order_key":"09:miner-9","p2mr_program_hex":"7474747474747474747474747474747474747474747474747474747474747474","share_difficulty":19,"share_id":"share-372","share_seq":372,"template_height":800011},{"accepted_at_ms":90000,"credit_policy":null,"job_id":"job-341","job_issued_at_ms":89982,"miner_id":"miner-0","network_difficulty":1000001,"ntime":1700000341,"order_key":"00:miner-0","p2mr_program_hex":"5555555555555555555555555555555555555555555555555555555555555555","share_difficulty":1,"share_id":"share-341","share_seq":341,"template_height":800010},{"accepted_at_ms":89992,"credit_policy":null,"job_id":"job-1031","job_issued_at_ms":89977,"miner_id":"miner-8","network_difficulty":1000011,"ntime":1700001031,"order_key":"08:miner-8","p2mr_program_hex":"0707070707070707070707070707070707070707070707070707070707070707","share_difficulty":19,"share_id":"share-1031","share_seq":1031,"template_height":800032},{"accepted_at_ms":89997,"credit_policy":"stale-grace","job_id":"job-1280","job_issued_at_ms":89984,"miner_id":"miner-4","network_difficulty":1000005,"ntime":1700001280,"order_key":"04:miner-4","p2mr_program_hex":"0000000000000000000000000000000000000000000000000000000000000000","share_difficulty":1,"share_id":"share-1280","share_seq":1280,"template_height":800040},{"accepted_at_ms":89987,"credit_policy":null,"job_id":"job-528","job_issued_at_ms":89976,"miner_id":"miner-0","network_difficulty":1000001,"ntime":1700000528,"order_key":"00:miner-0","p2mr_program_hex":"1010101010101010101010101010101010101010101010101010101010101010","share_difficulty":19,"share_id":"share-528","share_seq":528,"template_height":800016},{"accepted_at_ms":90000,"credit_policy":"stale-grace","job_id":"job-369","job_issued_at_ms":89980,"miner_id":"miner-6","network_difficulty":1000012,"ntime":1700000369,"order_key":"06:miner-6","p2mr_program_hex":"7171717171717171717171717171717171717171717171717171717171717171","share_difficulty":2,"share_id":"share-369","share_seq":369,"template_height":800011},{"accepted_at_ms":89973,"credit_policy":"stale-grace","job_id":"job-1415","job_issued_at_ms":89982,"miner_id":"miner-7","network_difficulty":1000004,"ntime":1700001415,"order_key":"07:miner-7","p2mr_program_hex":"8787878787878787878787878787878787878787878787878787878787878787","share_difficulty":7,"share_id":"share-1415","share_seq":1415,"template_height":800044},{"accepted_at_ms":89983,"credit_policy":"stale-grace","job_id":"job-453","job_issued_at_ms":89977,"miner_id":"miner-2","network_difficulty":1000011,"ntime":1700000453,"order_key":"02:miner-2","p2mr_program_hex":"c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5","share_difficulty":7,"share_id":"share-453","share_seq":453,"template_height":800014},{"accepted_at_ms":89989,"credit_policy":null,"job_id":"job-1219","job_issued_at_ms":89971,"miner_id":"miner-9","network_difficulty":1000012,"ntime":1700001219,"order_key":"09:miner-9","p2mr_program_hex":"c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3","share_difficulty":19,"share_id":"share-1219","share_seq":1219,"template_height":800038},{"accepted_at_ms":89977,"credit_policy":null,"job_id":"job-204","job_issued_at_ms":89973,"miner_id":"miner-6","network_difficulty":1000000,"ntime":1700000204,"order_key":"06:miner-6","p2mr_program_hex":"cccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccc","share_difficulty":19,"share_id":"share-204","share_seq":204,"template_height":800006},{"accepted_at_ms":89983,"credit_policy":"stale-grace","job_id":"job-1351","job_issued_at_ms":89990,"miner_id":"miner-9","network_difficulty":1000008,"ntime":1700001351,"order_key":"09:miner-9","p2mr_program_hex":"4747474747474747474747474747474747474747474747474747474747474747","share_difficulty":3,"share_id":"share-1351","share_seq":1351,"template_height":800042},{"accepted_at_ms":89996,"credit_policy":null,"job_id":"job-1306","job_issued_at_ms":89986,"miner_id":"miner-8","network_difficulty":1000014,"ntime":1700001306,"order_key":"08:miner-8","p2mr_program_hex":"1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a","share_difficulty":3,"share_id":"share-1306","share_seq":1306,"template_height":800040},{"accepted_at_ms":89970,"credit_policy":null,"job_id":"job-819","job_issued_at_ms":89978,"miner_id":"miner-5","network_difficulty":1000003,"ntime":1700000819,"order_key":"05:miner-5","p2mr_program_hex":"3333333333333333333333333333333333333333333333333333333333333333","share_difficulty":3,"share_id":"share-819","share_seq":819,"template_height":800025},{"accepted_at_ms":89976,"credit_policy":null,"job_id":"job-1548","job_issued_at_ms":89978,"miner_id":"miner-8","network_difficulty":1000001,"ntime":1700001548,"order_key":"08:miner-8","p2mr_program_hex":"0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c","share_difficulty":2,"share_id":"share-1548","share_seq":1548,"template_height":800048},{"accepted_at_ms":89988,"credit_policy":null,"job_id":"job-266","job_issued_at_ms":89989,"miner_id":"miner-2","network_difficulty":1000011,"ntime":1700000266,"order_key":"02:miner-2","p2mr_program_hex":"0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a","share_difficulty":3,"share_id":"share-266","share_seq":266,"template_height":800008},{"accepted_at_ms":89983,"credit_policy":null,"job_id":"job-1663","job_issued_at_ms":89977,"miner_id":"miner-2","network_difficulty":1000014,"ntime":1700001663,"order_key":"02:miner-2","p2mr_program_hex":"7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f","share_difficulty":7,"share_id":"share-1663","share_seq":1663,"template_height":800051},{"accepted_at_ms":89998,"credit_policy":null,"job_id":"job-1481","job_issued_at_ms":89974,"miner_id":"miner-7","network_difficulty":1000002,"ntime":1700001481,"order_key":"07:miner-7","p2mr_program_hex":"c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9","share_difficulty":2,"share_id":"share-1481","share_seq":1481,"template_height":800046},{"accepted_at_ms":89981,"credit_policy":null,"job_id":"job-524","job_issued_at_ms":89990,"miner_id":"miner-7","network_difficulty":1000014,"ntime":1700000524,"order_key":"07:miner-7","p2mr_program_hex":"0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c","share_difficulty":1,"share_id":"share-524","share_seq":524,"template_height":800016},{"accepted_at_ms":89978,"credit_policy":null,"job_id":"job-1763","job_issued_at_ms":89981,"miner_id":"miner-3","network_difficulty":1000012,"ntime":1700001763,"order_key":"03:miner-3","p2mr_program_hex":"e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3","share_difficulty":3,"share_id":"share-1763","share_seq":1763,"template_height":800055},{"accepted_at_ms":89970,"credit_policy":"stale-grace","job_id":"job-468","job_issued_at_ms":89980,"miner_id":"miner-6","network_difficulty":1000009,"ntime":1700000468,"order_key":"06:miner-6","p2mr_program_hex":"d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4","share_difficulty":1,"share_id":"share-468","share_seq":468,"template_height":800014},{"accepted_at_ms":89986,"credit_policy":"stale-grace","job_id":"job-839","job_issued_at_ms":89994,"miner_id":"miner-3","network_difficulty":1000006,"ntime":1700000839,"order_key":"03:miner-3","p2mr_program_hex":"4747474747474747474747474747474747474747474747474747474747474747","share_difficulty":2,"share_id":"share-839","share_seq":839,"template_height":800026},{"accepted_at_ms":89971,"credit_policy":null,"job_id":"job-106","job_issued_at_ms":89972,"miner_id":"miner-7","network_difficulty":1000004,"ntime":1700000106,"order_key":"07:miner-7","p2mr_program_hex":"6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a","share_difficulty":3,"share_id":"share-106","share_seq":106,"template_height":800003},{"accepted_at_ms":89993,"credit_policy":null,"job_id":"job-1595","job_issued_at_ms":89995,"miner_id":"miner-0","network_difficulty":1000014,"ntime":1700001595,"order_key":"00:miner-0","p2mr_program_hex":"3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b","share_difficulty":2,"share_id":"share-1595","share_seq":1595,"template_height":800049},{"accepted_at_ms":89987,"credit_policy":null,"job_id":"job-193","job_issued_at_ms":89990,"miner_id":"miner-6","network_difficulty":1000006,"ntime":1700000193,"order_key":"06:miner-6","p2mr_program_hex":"c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1","share_difficulty":19,"share_id":"share-193","share_seq":193,"template_height":800006},{"accepted_at_ms":89974,"credit_policy":null,"job_id":"job-1475","job_issued_at_ms":89972,"miner_id":"miner-1","network_difficulty":1000013,"ntime":1700001475,"order_key":"01:miner-1","p2mr_program_hex":"c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3","share_difficulty":7,"share_id":"share-1475","share_seq":1475,"template_height":800046},{"accepted_at_ms":89970,"credit_policy":null,"job_id":"job-1370","job_issued_at_ms":90000,"miner_id":"miner-6","network_difficulty":1000010,"ntime":1700001370,"order_key":"06:miner-6","p2mr_program_hex":"5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a","share_difficulty":2,"share_id":"share-1370","share_seq":1370,"template_height":800042},{"accepted_at_ms":89970,"credit_policy":null,"job_id":"job-868","job_issued_at_ms":89991,"miner_id":"miner-10","network_difficulty":1000001,"ntime":1700000868,"order_key":"10:miner-10","p2mr_program_hex":"6464646464646464646464646464646464646464646464646464646464646464","share_difficulty":19,"share_id":"share-868","share_seq":868,"template_height":800027},{"accepted_at_ms":89981,"credit_policy":"stale-grace","job_id":"job-1014","job_issued_at_ms":89992,"miner_id":"miner-2","network_difficulty":1000011,"ntime":1700001014,"order_key":"02:miner-2","p2mr_program_hex":"f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6","share_difficulty":7,"share_id":"share-1014","share_seq":1014,"template_height":800031},{"accepted_at_ms":89990,"credit_policy":"stale-grace","job_id":"job-1277","job_issued_at_ms":89988,"miner_id":"miner-1","network_difficulty":1000002,"ntime":1700001277,"order_key":"01:miner-1","p2mr_program_hex":"fdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfd","share_difficulty":2,"share_id":"share-1277","share_seq":1277,"template_height":800039},{"accepted_at_ms":89985,"credit_policy":null,"job_id":"job-693","job_issued_at_ms":89979,"miner_id":"miner-0","network_difficulty":1000013,"ntime":1700000693,"order_key":"00:miner-0","p2mr_program_hex":"b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5","share_difficulty":2,"share_id":"share-693","share_seq":693,"template_height":800021},{"accepted_at_ms":89994,"credit_policy":null,"job_id":"job-1726","job_issued_at_ms":89986,"miner_id":"miner-10","network_difficulty":1000009,"ntime":1700001726,"order_key":"10:miner-10","p2mr_program_hex":"bebebebebebebebebebebebebebebebebebebebebebebebebebebebebebebebe","share_difficulty":2,"share_id":"share-1726","share_seq":1726,"template_height":800053},{"accepted_at_ms":89981,"credit_policy":"stale-grace","job_id":"job-1185","job_issued_at_ms":89977,"miner_id":"miner-8","network_difficulty":1000012,"ntime":1700001185,"order_key":"08:miner-8","p2mr_program_hex":"a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1","share_difficulty":1,"share_id":"share-1185","share_seq":1185,"template_height":800037},{"accepted_at_ms":89986,"credit_policy":null,"job_id":"job-1187","job_issued_at_ms":89997,"miner_id":"miner-10","network_difficulty":1000014,"ntime":1700001187,"order_key":"10:miner-10","p2mr_program_hex":"a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3","share_difficulty":3,"share_id":"share-1187","share_seq":1187,"template_height":800037},{"accepted_at_ms":89982,"credit_policy":null,"job_id":"job-160","job_issued_at_ms":89976,"miner_id":"miner-6","network_difficulty":1000007,"ntime":1700000160,"order_key":"06:miner-6","p2mr_program_hex":"a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0","share_difficulty":2,"share_id":"share-160","share_seq":160,"template_height":800005},{"accepted_at_ms":89971,"credit_policy":null,"job_id":"job-663","job_issued_at_ms":89980,"miner_id":"miner-3","network_difficulty":1000000,"ntime":1700000663,"order_key":"03:miner-3","p2mr_program_hex":"9797979797979797979797979797979797979797979797979797979797979797","share_difficulty":3,"share_id":"share-663","share_seq":663,"template_height":800020},{"accepted_at_ms":89972,"credit_policy":null,"job_id":"job-1195","job_issued_at_ms":89982,"miner_id":"miner-7","network_difficulty":1000005,"ntime":1700001195,"order_key":"07:miner-7","p2mr_program_hex":"abababababababababababababababababababababababababababababababab","share_difficulty":2,"share_id":"share-1195","share_seq":1195,"template_height":800037},{"accepted_at_ms":89986,"credit_policy":null,"job_id":"job-597","job_issued_at_ms":89987,"miner_id":"miner-3","network_difficulty":1000002,"ntime":1700000597,"order_key":"03:miner-3","p2mr_program_hex":"5555555555555555555555555555555555555555555555555555555555555555","share_difficulty":3,"share_id":"share-597","share_seq":597,"template_height":800018},{"accepted_at_ms":89997,"credit_policy":null,"job_id":"job-327","job_issued_at_ms":89992,"miner_id":"miner-8","network_difficulty":1000004,"ntime":1700000327,"order_key":"08:miner-8","p2mr_program_hex":"4747474747474747474747474747474747474747474747474747474747474747","share_difficulty":1,"share_id":"share-327","share_seq":327,"template_height":800010},{"accepted_at_ms":89995,"credit_policy":null,"job_id":"job-1215","job_issued_at_ms":89996,"miner_id":"miner-5","network_difficulty":1000008,"ntime":1700001215,"order_key":"05:miner-5","p2mr_program_hex":"bfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbf","share_difficulty":1,"share_id":"share-1215","share_seq":1215,"template_height":800037},{"accepted_at_ms":89975,"credit_policy":null,"job_id":"job-1049","job_issued_at_ms":89977,"miner_id":"miner-4","network_difficulty":1000012,"ntime":1700001049,"order_key":"04:miner-4","p2mr_program_hex":"1919191919191919191919191919191919191919191919191919191919191919","share_difficulty":2,"share_id":"share-1049","share_seq":1049,"template_height":800032},{"accepted_at_ms":89971,"credit_policy":null,"job_id":"job-894","job_issued_at_ms":89971,"miner_id":"miner-3","network_difficulty":1000010,"ntime":1700000894,"order_key":"03:miner-3","p2mr_program_hex":"7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e","share_difficulty":7,"share_id":"share-894","share_seq":894,"template_height":800027},{"accepted_at_ms":89982,"credit_policy":"stale-grace","job_id":"job-530","job_issued_at_ms":89972,"miner_id":"miner-2","network_difficulty":1000003,"ntime":1700000530,"order_key":"02:miner-2","p2mr_program_hex":"1212121212121212121212121212121212121212121212121212121212121212","share_difficulty":1,"share_id":"share-530","share_seq":530,"template_height":800016},{"accepted_at_ms":89998,"credit_policy":"stale-grace","job_id":"job-469","job_issued_at_ms":89987,"miner_id":"miner-7","network_difficulty":1000010,"ntime":1700000469,"order_key":"07:miner-7","p2mr_program_hex":"d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5","share_difficulty":3,"share_id":"share-469","share_seq":469,"template_height":800014},{"accepted_at_ms":89988,"credit_policy":"stale-grace","job_id":"job-579","job_issued_at_ms":89990,"miner_id":"miner-7","network_difficulty":1000001,"ntime":1700000579,"order_key":"07:miner-7","p2mr_program_hex":"4343434343434343434343434343434343434343434343434343434343434343","share_difficulty":2,"share_id":"share-579","share_seq":579,"template_height":800018},{"accepted_at_ms":89991,"credit_policy":"stale-grace","job_id":"job-1109","job_issued_at_ms":90000,"miner_id":"miner-9","network_difficulty":1000004,"ntime":1700001109,"order_key":"09:miner-9","p2mr_program_hex":"5555555555555555555555555555555555555555555555555555555555555555","share_difficulty":2,"share_id":"share-1109","share_seq":1109,"template_height":800034},{"accepted_at_ms":89989,"credit_policy":null,"job_id":"job-708","job_issued_at_ms":89990,"miner_id":"miner-4","network_difficulty":1000011,"ntime":1700000708,"order_key":"04:miner-4","p2mr_program_hex":"c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4","share_difficulty":19,"share_id":"share-708","share_seq":708,"template_height":800022},{"accepted_at_ms":89988,"credit_policy":null,"job_id":"job-93","job_issued_at_ms":89980,"miner_id":"miner-5","network_difficulty":1000008,"ntime":1700000093,"order_key":"05:miner-5","p2mr_program_hex":"5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d","share_difficulty":19,"share_id":"share-93","share_seq":93,"template_height":800002},{"accepted_at_ms":89985,"credit_policy":null,"job_id":"job-840","job_issued_at_ms":89988,"miner_id":"miner-4","network_difficulty":1000007,"ntime":1700000840,"order_key":"04:miner-4","p2mr_program_hex":"4848484848484848484848484848484848484848484848484848484848484848","share_difficulty":7,"share_id":"share-840","share_seq":840,"template_height":800026},{"accepted_at_ms":89983,"credit_policy":null,"job_id":"job-1209","job_issued_at_ms":89992,"miner_id":"miner-10","network_difficulty":1000002,"ntime":1700001209,"order_key":"10:miner-10","p2mr_program_hex":"b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9","share_difficulty":2,"share_id":"share-1209","share_seq":1209,"template_height":800037},{"accepted_at_ms":89992,"credit_policy":null,"job_id":"job-1363","job_issued_at_ms":89986,"miner_id":"miner-10","network_difficulty":1000003,"ntime":1700001363,"order_key":"10:miner-10","p2mr_program_hex":"5353535353535353535353535353535353535353535353535353535353535353","share_difficulty":3,"share_id":"share-1363","share_seq":1363,"template_height":800042},{"accepted_at_ms":89995,"credit_policy":null,"job_id":"job-263","job_issued_at_ms":89994,"miner_id":"miner-10","network_difficulty":1000008,"ntime":1700000263,"order_key":"10:miner-10","p2mr_program_hex":"0707070707070707070707070707070707070707070707070707070707070707","share_difficulty":2,"share_id":"share-263","share_seq":263,"template_height":800008},{"accepted_at_ms":89990,"credit_policy":null,"job_id":"job-1643","job_issued_at_ms":89985,"miner_id":"miner-4","network_difficulty":1000011,"ntime":1700001643,"order_key":"04:miner-4","p2mr_program_hex":"6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b","share_difficulty":19,"share_id":"share-1643","share_seq":1643,"template_height":800051},{"accepted_at_ms":89973,"credit_policy":null,"job_id":"job-611","job_issued_at_ms":89984,"miner_id":"miner-6","network_difficulty":1000016,"ntime":1700000611,"order_key":"06:miner-6","p2mr_program_hex":"6363636363636363636363636363636363636363636363636363636363636363","share_difficulty":2,"share_id":"share-611","share_seq":611,"template_height":800019},{"accepted_at_ms":89993,"credit_policy":null,"job_id":"job-1688","job_issued_at_ms":89989,"miner_id":"miner-5","network_difficulty":1000005,"ntime":1700001688,"order_key":"05:miner-5","p2mr_program_hex":"9898989898989898989898989898989898989898989898989898989898989898","share_difficulty":7,"share_id":"share-1688","share_seq":1688,"template_height":800052},{"accepted_at_ms":89988,"credit_policy":"stale-grace","job_id":"job-1154","job_issued_at_ms":89973,"miner_id":"miner-10","network_difficulty":1000015,"ntime":1700001154,"order_key":"10:miner-10","p2mr_program_hex":"8282828282828282828282828282828282828282828282828282828282828282","share_difficulty":1,"share_id":"share-1154","share_seq":1154,"template_height":800036},{"accepted_at_ms":89997,"credit_policy":null,"job_id":"job-1086","job_issued_at_ms":89978,"miner_id":"miner-8","network_difficulty":1000015,"ntime":1700001086,"order_key":"08:miner-8","p2mr_program_hex":"3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e","share_difficulty":19,"share_id":"share-1086","share_seq":1086,"template_height":800033},{"accepted_at_ms":89976,"credit_policy":null,"job_id":"job-1778","job_issued_at_ms":89984,"miner_id":"miner-7","network_difficulty":1000010,"ntime":1700001778,"order_key":"07:miner-7","p2mr_program_hex":"f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2","share_difficulty":19,"share_id":"share-1778","share_seq":1778,"template_height":800055},{"accepted_at_ms":89976,"credit_policy":null,"job_id":"job-303","job_issued_at_ms":89982,"miner_id":"miner-6","network_difficulty":1000014,"ntime":1700000303,"order_key":"06:miner-6","p2mr_program_hex":"2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f","share_difficulty":3,"share_id":"share-303","share_seq":303,"template_height":800009},{"accepted_at_ms":89987,"credit_policy":null,"job_id":"job-1075","job_issued_at_ms":89977,"miner_id":"miner-8","network_difficulty":1000004,"ntime":1700001075,"order_key":"08:miner-8","p2mr_program_hex":"3333333333333333333333333333333333333333333333333333333333333333","share_difficulty":7,"share_id":"share-1075","share_seq":1075,"template_height":800033},{"accepted_at_ms":89993,"credit_policy":null,"job_id":"job-149","job_issued_at_ms":89975,"miner_id":"miner-6","network_difficulty":1000013,"ntime":1700000149,"order_key":"06:miner-6","p2mr_program_hex":"9595959595959595959595959595959595959595959595959595959595959595","share_difficulty":1,"share_id":"share-149","share_seq":149,"template_height":800004},{"accepted_at_ms":89994,"credit_policy":null,"job_id":"job-951","job_issued_at_ms":90000,"miner_id":"miner-5","network_difficulty":1000016,"ntime":1700000951,"order_key":"05:miner-5","p2mr_program_hex":"b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7","share_difficulty":3,"share_id":"share-951","share_seq":951,"template_height":800029},{"accepted_at_ms":89994,"credit_policy":null,"job_id":"job-1301","job_issued_at_ms":90000,"miner_id":"miner-3","network_difficulty":1000009,"ntime":1700001301,"order_key":"03:miner-3","p2mr_program_hex":"1515151515151515151515151515151515151515151515151515151515151515","share_difficulty":19,"share_id":"share-1301","share_seq":1301,"template_height":800040},{"accepted_at_ms":89974,"credit_policy":null,"job_id":"job-39","job_issued_at_ms":89976,"miner_id":"miner-6","network_difficulty":1000005,"ntime":1700000039,"order_key":"06:miner-6","p2mr_program_hex":"2727272727272727272727272727272727272727272727272727272727272727","share_difficulty":2,"share_id":"share-39","share_seq":39,"template_height":800001},{"accepted_at_ms":89990,"credit_policy":null,"job_id":"job-452","job_issued_at_ms":89974,"miner_id":"miner-1","network_difficulty":1000010,"ntime":1700000452,"order_key":"01:miner-1","p2mr_program_hex":"c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4","share_difficulty":19,"share_id":"share-452","share_seq":452,"template_height":800014},{"accepted_at_ms":89970,"credit_policy":null,"job_id":"job-845","job_issued_at_ms":89971,"miner_id":"miner-9","network_difficulty":1000012,"ntime":1700000845,"order_key":"09:miner-9","p2mr_program_hex":"4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d","share_difficulty":7,"share_id":"share-845","share_seq":845,"template_height":800026},{"accepted_at_ms":89981,"credit_policy":"stale-grace","job_id":"job-1402","job_issued_at_ms":89991,"miner_id":"miner-5","network_difficulty":1000008,"ntime":1700001402,"order_key":"05:miner-5","p2mr_program_hex":"7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a","share_difficulty":7,"share_id":"share-1402","share_seq":1402,"template_height":800043},{"accepted_at_ms":89976,"credit_policy":null,"job_id":"job-1564","job_issued_at_ms":89983,"miner_id":"miner-2","network_difficulty":1000000,"ntime":1700001564,"order_key":"02:miner-2","p2mr_program_hex":"1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c","share_difficulty":19,"share_id":"share-1564","share_seq":1564,"template_height":800048},{"accepted_at_ms":89990,"credit_policy":null,"job_id":"job-333","job_issued_at_ms":89978,"miner_id":"miner-3","network_difficulty":1000010,"ntime":1700000333,"order_key":"03:miner-3","p2mr_program_hex":"4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d","share_difficulty":3,"share_id":"share-333","share_seq":333,"template_height":800010},{"accepted_at_ms":89992,"credit_policy":null,"job_id":"job-505","job_issued_at_ms":89973,"miner_id":"miner-10","network_difficulty":1000012,"ntime":1700000505,"order_key":"10:miner-10","p2mr_program_hex":"f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9","share_difficulty":19,"share_id":"share-505","share_seq":505,"template_height":800015},{"accepted_at_ms":89995,"credit_policy":null,"job_id":"job-685","job_issued_at_ms":89971,"miner_id":"miner-3","network_difficulty":1000005,"ntime":1700000685,"order_key":"03:miner-3","p2mr_program_hex":"adadadadadadadadadadadadadadadadadadadadadadadadadadadadadadadad","share_difficulty":1,"share_id":"share-685","share_seq":685,"template_height":800021},{"accepted_at_ms":89981,"credit_policy":null,"job_id":"job-647","job_issued_at_ms":89981,"miner_id":"miner-9","network_difficulty":1000001,"ntime":1700000647,"order_key":"09:miner-9","p2mr_program_hex":"8787878787878787878787878787878787878787878787878787878787878787","share_difficulty":2,"share_id":"share-647","share_seq":647,"template_height":800020},{"accepted_at_ms":89976,"credit_policy":null,"job_id":"job-416","job_issued_at_ms":89986,"miner_id":"miner-9","network_difficulty":1000008,"ntime":1700000416,"order_key":"09:miner-9","p2mr_program_hex":"a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0","share_difficulty":3,"share_id":"share-416","share_seq":416,"template_height":800013},{"accepted_at_ms":89990,"credit_policy":null,"job_id":"job-627","job_issued_at_ms":89974,"miner_id":"miner-0","network_difficulty":1000015,"ntime":1700000627,"order_key":"00:miner-0","p2mr_program_hex":"7373737373737373737373737373737373737373737373737373737373737373","share_difficulty":1,"share_id":"share-627","share_seq":627,"template_height":800019},{"accepted_at_ms":89975,"credit_policy":null,"job_id":"job-284","job_issued_at_ms":89978,"miner_id":"miner-9","network_difficulty":1000012,"ntime":1700000284,"order_key":"09:miner-9","p2mr_program_hex":"1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c","share_difficulty":19,"share_id":"share-284","share_seq":284,"template_height":800008},{"accepted_at_ms":89988,"credit_policy":null,"job_id":"job-1094","job_issued_at_ms":89977,"miner_id":"miner-5","network_difficulty":1000006,"ntime":1700001094,"order_key":"05:miner-5","p2mr_program_hex":"4646464646464646464646464646464646464646464646464646464646464646","share_difficulty":3,"share_id":"share-1094","share_seq":1094,"template_height":800034},{"accepted_at_ms":89974,"credit_policy":"stale-grace","job_id":"job-230","job_issued_at_ms":89972,"miner_id":"miner-10","network_difficulty":1000009,"ntime":1700000230,"order_key":"10:miner-10","p2mr_program_hex":"e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6","share_difficulty":19,"share_id":"share-230","share_seq":230,"template_height":800007},{"accepted_at_ms":89989,"credit_policy":null,"job_id":"job-1633","job_issued_at_ms":89981,"miner_id":"miner-5","network_difficulty":1000001,"ntime":1700001633,"order_key":"05:miner-5","p2mr_program_hex":"6161616161616161616161616161616161616161616161616161616161616161","share_difficulty":7,"share_id":"share-1633","share_seq":1633,"template_height":800051},{"accepted_at_ms":89998,"credit_policy":null,"job_id":"job-1299","job_issued_at_ms":89998,"miner_id":"miner-1","network_difficulty":1000007,"ntime":1700001299,"order_key":"01:miner-1","p2mr_program_hex":"1313131313131313131313131313131313131313131313131313131313131313","share_difficulty":7,"share_id":"share-1299","share_seq":1299,"template_height":800040},{"accepted_at_ms":89972,"credit_policy":null,"job_id":"job-1755","job_issued_at_ms":89976,"miner_id":"miner-6","network_difficulty":1000004,"ntime":1700001755,"order_key":"06:miner-6","p2mr_program_hex":"dbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdb","share_difficulty":1,"share_id":"share-1755","share_seq":1755,"template_height":800054},{"accepted_at_ms":89997,"credit_policy":null,"job_id":"job-250","job_issued_at_ms":89989,"miner_id":"miner-8","network_difficulty":1000012,"ntime":1700000250,"order_key":"08:miner-8","p2mr_program_hex":"fafafafafafafafafafafafafafafafafafafafafafafafafafafafafafafafa","share_difficulty":19,"share_id":"share-250","share_seq":250,"template_height":800007},{"accepted_at_ms":89985,"credit_policy":"stale-grace","job_id":"job-563","job_issued_at_ms":89982,"miner_id":"miner-2","network_difficulty":1000002,"ntime":1700000563,"order_key":"02:miner-2","p2mr_program_hex":"3333333333333333333333333333333333333333333333333333333333333333","share_difficulty":7,"share_id":"share-563","share_seq":563,"template_height":800017},{"accepted_at_ms":89990,"credit_policy":null,"job_id":"job-81","job_issued_at_ms":90000,"miner_id":"miner-4","network_difficulty":1000013,"ntime":1700000081,"order_key":"04:miner-4","p2mr_program_hex":"5151515151515151515151515151515151515151515151515151515151515151","share_difficulty":7,"share_id":"share-81","share_seq":81,"template_height":800002},{"accepted_at_ms":90000,"credit_policy":null,"job_id":"job-863","job_issued_at_ms":89988,"miner_id":"miner-5","network_difficulty":1000013,"ntime":1700000863,"order_key":"05:miner-5","p2mr_program_hex":"5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f","share_difficulty":19,"share_id":"share-863","share_seq":863,"template_height":800026},{"accepted_at_ms":89993,"credit_policy":null,"job_id":"job-594","job_issued_at_ms":89985,"miner_id":"miner-0","network_difficulty":1000016,"ntime":1700000594,"order_key":"00:miner-0","p2mr_program_hex":"5252525252525252525252525252525252525252525252525252525252525252","share_difficulty":7,"share_id":"share-594","share_seq":594,"template_height":800018},{"accepted_at_ms":89984,"credit_policy":null,"job_id":"job-1028","job_issued_at_ms":89987,"miner_id":"miner-5","network_difficulty":1000008,"ntime":1700001028,"order_key":"05:miner-5","p2mr_program_hex":"0404040404040404040404040404040404040404040404040404040404040404","share_difficulty":2,"share_id":"share-1028","share_seq":1028,"template_height":800032},{"accepted_at_ms":89970,"credit_policy":"stale-grace","job_id":"job-323","job_issued_at_ms":89993,"miner_id":"miner-4","network_difficulty":1000000,"ntime":1700000323,"order_key":"04:miner-4","p2mr_program_hex":"4343434343434343434343434343434343434343434343434343434343434343","share_difficulty":7,"share_id":"share-323","share_seq":323,"template_height":800010},{"accepted_at_ms":89990,"credit_policy":null,"job_id":"job-987","job_issued_at_ms":89975,"miner_id":"miner-8","network_difficulty":1000001,"ntime":1700000987,"order_key":"08:miner-8","p2mr_program_hex":"dbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdb","share_difficulty":2,"share_id":"share-987","share_seq":987,"template_height":800030},{"accepted_at_ms":89996,"credit_policy":null,"job_id":"job-577","job_issued_at_ms":89972,"miner_id":"miner-5","network_difficulty":1000016,"ntime":1700000577,"order_key":"05:miner-5","p2mr_program_hex":"4141414141414141414141414141414141414141414141414141414141414141","share_difficulty":19,"share_id":"share-577","share_seq":577,"template_height":800018},{"accepted_at_ms":90000,"credit_policy":null,"job_id":"job-1465","job_issued_at_ms":89979,"miner_id":"miner-2","network_difficulty":1000003,"ntime":1700001465,"order_key":"02:miner-2","p2mr_program_hex":"b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9","share_difficulty":7,"share_id":"share-1465","share_seq":1465,"template_height":800045},{"accepted_at_ms":89976,"credit_policy":null,"job_id":"job-1389","job_issued_at_ms":89994,"miner_id":"miner-3","network_difficulty":1000012,"ntime":1700001389,"order_key":"03:miner-3","p2mr_program_hex":"6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d","share_difficulty":1,"share_id":"share-1389","share_seq":1389,"template_height":800043},{"accepted_at_ms":89974,"credit_policy":null,"job_id":"job-365","job_issued_at_ms":89973,"miner_id":"miner-2","network_difficulty":1000008,"ntime":1700000365,"order_key":"02:miner-2","p2mr_program_hex":"6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d","share_difficulty":2,"share_id":"share-365","share_seq":365,"template_height":800011},{"accepted_at_ms":89994,"credit_policy":null,"job_id":"job-752","job_issued_at_ms":89970,"miner_id":"miner-4","network_difficulty":1000004,"ntime":1700000752,"order_key":"04:miner-4","p2mr_program_hex":"f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0","share_difficulty":19,"share_id":"share-752","share_seq":752,"template_height":800023},{"accepted_at_ms":89974,"credit_policy":null,"job_id":"job-1283","job_issued_at_ms":89995,"miner_id":"miner-7","network_difficulty":1000008,"ntime":1700001283,"order_key":"07:miner-7","p2mr_program_hex":"0303030303030303030303030303030303030303030303030303030303030303","share_difficulty":19,"share_id":"share-1283","share_seq":1283,"template_height":800040},{"accepted_at_ms":89975,"credit_policy":null,"job_id":"job-833","job_issued_at_ms":89977,"miner_id":"miner-8","network_difficulty":1000000,"ntime":1700000833,"order_key":"08:miner-8","p2mr_program_hex":"4141414141414141414141414141414141414141414141414141414141414141","share_difficulty":1,"share_id":"share-833","share_seq":833,"template_height":800026},{"accepted_at_ms":89990,"credit_policy":null,"job_id":"job-436","job_issued_at_ms":89975,"miner_id":"miner-7","network_difficulty":1000011,"ntime":1700000436,"order_key":"07:miner-7","p2mr_program_hex":"b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4","share_difficulty":19,"share_id":"share-436","share_seq":436,"template_height":800013},{"accepted_at_ms":89972,"credit_policy":null,"job_id":"job-1444","job_issued_at_ms":89978,"miner_id":"miner-3","network_difficulty":1000016,"ntime":1700001444,"order_key":"03:miner-3","p2mr_program_hex":"a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4","share_difficulty":3,"share_id":"share-1444","share_seq":1444,"template_height":800045},{"accepted_at_ms":89994,"credit_policy":"stale-grace","job_id":"job-463","job_issued_at_ms":90000,"miner_id":"miner-1","network_difficulty":1000004,"ntime":1700000463,"order_key":"01:miner-1","p2mr_program_hex":"cfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcf","share_difficulty":19,"share_id":"share-463","share_seq":463,"template_height":800014},{"accepted_at_ms":89987,"credit_policy":"stale-grace","job_id":"job-364","job_issued_at_ms":89980,"miner_id":"miner-1","network_difficulty":1000007,"ntime":1700000364,"order_key":"01:miner-1","p2mr_program_hex":"6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c","share_difficulty":3,"share_id":"share-364","share_seq":364,"template_height":800011},{"accepted_at_ms":89982,"credit_policy":"stale-grace","job_id":"job-960","job_issued_at_ms":89991,"miner_id":"miner-3","network_difficulty":1000008,"ntime":1700000960,"order_key":"03:miner-3","p2mr_program_hex":"c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0","share_difficulty":19,"share_id":"share-960","share_seq":960,"template_height":800030},{"accepted_at_ms":89975,"credit_policy":"stale-grace","job_id":"job-700","job_issued_at_ms":89999,"miner_id":"miner-7","network_difficulty":1000003,"ntime":1700000700,"order_key":"07:miner-7","p2mr_program_hex":"bcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbc","share_difficulty":1,"share_id":"share-700","share_seq":700,"template_height":800021},{"accepted_at_ms":89991,"credit_policy":null,"job_id":"job-544","job_issued_at_ms":89971,"miner_id":"miner-5","network_difficulty":1000000,"ntime":1700000544,"order_key":"05:miner-5","p2mr_program_hex":"2020202020202020202020202020202020202020202020202020202020202020","share_difficulty":2,"share_id":"share-544","share_seq":544,"template_height":800017},{"accepted_at_ms":89994,"credit_policy":"stale-grace","job_id":"job-1287","job_issued_at_ms":89990,"miner_id":"miner-0","network_difficulty":1000012,"ntime":1700001287,"order_key":"00:miner-0","p2mr_program_hex":"0707070707070707070707070707070707070707070707070707070707070707","share_difficulty":2,"share_id":"share-1287","share_seq":1287,"template_height":800040},{"accepted_at_ms":89993,"credit_policy":"stale-grace","job_id":"job-950","job_issued_at_ms":89987,"miner_id":"miner-4","network_difficulty":1000015,"ntime":1700000950,"order_key":"04:miner-4","p2mr_program_hex":"b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6","share_difficulty":1,"share_id":"share-950","share_seq":950,"template_height":800029},{"accepted_at_ms":89979,"credit_policy":null,"job_id":"job-358","job_issued_at_ms":89995,"miner_id":"miner-6","network_difficulty":1000001,"ntime":1700000358,"order_key":"06:miner-6","p2mr_program_hex":"6666666666666666666666666666666666666666666666666666666666666666","share_difficulty":1,"share_id":"share-358","share_seq":358,"template_height":800011},{"accepted_at_ms":89990,"credit_policy":null,"job_id":"job-681","job_issued_at_ms":89981,"miner_id":"miner-10","network_difficulty":1000001,"ntime":1700000681,"order_key":"10:miner-10","p2mr_program_hex":"a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9","share_difficulty":19,"share_id":"share-681","share_seq":681,"template_height":800021},{"accepted_at_ms":89992,"credit_policy":null,"job_id":"job-1486","job_issued_at_ms":89990,"miner_id":"miner-1","network_difficulty":1000007,"ntime":1700001486,"order_key":"01:miner-1","p2mr_program_hex":"cececececececececececececececececececececececececececececececece","share_difficulty":3,"share_id":"share-1486","share_seq":1486,"template_height":800046},{"accepted_at_ms":89981,"credit_policy":"stale-grace","job_id":"job-1171","job_issued_at_ms":89977,"miner_id":"miner-5","network_difficulty":1000015,"ntime":1700001171,"order_key":"05:miner-5","p2mr_program_hex":"9393939393939393939393939393939393939393939393939393939393939393","share_difficulty":2,"share_id":"share-1171","share_seq":1171,"template_height":800036},{"accepted_at_ms":89979,"credit_policy":null,"job_id":"job-484","job_issued_at_ms":89974,"miner_id":"miner-0","network_difficulty":1000008,"ntime":1700000484,"order_key":"00:miner-0","p2mr_program_hex":"e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4","share_difficulty":19,"share_id":"share-484","share_seq":484,"template_height":800015},{"accepted_at_ms":89991,"credit_policy":null,"job_id":"job-3","job_issued_at_ms":89976,"miner_id":"miner-3","network_difficulty":1000003,"ntime":1700000003,"order_key":"03:miner-3","p2mr_program_hex":"0303030303030303030303030303030303030303030303030303030303030303","share_difficulty":19,"share_id":"share-3","share_seq":3,"template_height":800000},{"accepted_at_ms":90000,"credit_policy":null,"job_id":"job-403","job_issued_at_ms":89988,"miner_id":"miner-7","network_difficulty":1000012,"ntime":1700000403,"order_key":"07:miner-7","p2mr_program_hex":"9393939393939393939393939393939393939393939393939393939393939393","share_difficulty":7,"share_id":"share-403","share_seq":403,"template_height":800012},{"accepted_at_ms":89974,"credit_policy":"stale-grace","job_id":"job-1510","job_issued_at_ms":89983,"miner_id":"miner-3","network_difficulty":1000014,"ntime":1700001510,"order_key":"03:miner-3","p2mr_program_hex":"e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6","share_difficulty":1,"share_id":"share-1510","share_seq":1510,"template_height":800047},{"accepted_at_ms":89998,"credit_policy":null,"job_id":"job-709","job_issued_at_ms":89976,"miner_id":"miner-5","network_difficulty":1000012,"ntime":1700000709,"order_key":"05:miner-5","p2mr_program_hex":"c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5","share_difficulty":1,"share_id":"share-709","share_seq":709,"template_height":800022},{"accepted_at_ms":89986,"credit_policy":"stale-grace","job_id":"job-1166","job_issued_at_ms":89981,"miner_id":"miner-0","network_difficulty":1000010,"ntime":1700001166,"order_key":"00:miner-0","p2mr_program_hex":"8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e","share_difficulty":2,"share_id":"share-1166","share_seq":1166,"template_height":800036},{"accepted_at_ms":90000,"credit_policy":null,"job_id":"job-216","job_issued_at_ms":89976,"miner_id":"miner-7","network_difficulty":1000012,"ntime":1700000216,"order_key":"07:miner-7","p2mr_program_hex":"d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8","share_difficulty":3,"share_id":"share-216","share_seq":216,"template_height":800006},{"accepted_at_ms":89992,"credit_policy":"stale-grace","job_id":"job-23","job_issued_at_ms":89983,"miner_id":"miner-1","network_difficulty":1000006,"ntime":1700000023,"order_key":"01:miner-1","p2mr_program_hex":"1717171717171717171717171717171717171717171717171717171717171717","share_difficulty":1,"share_id":"share-23","share_seq":23,"template_height":800000},{"accepted_at_ms":90000,"credit_policy":"stale-grace","job_id":"job-1175","job_issued_at_ms":89983,"miner_id":"miner-9","network_difficulty":1000002,"ntime":1700001175,"order_key":"09:miner-9","p2mr_program_hex":"9797979797979797979797979797979797979797979797979797979797979797","share_difficulty":3,"share_id":"share-1175","share_seq":1175,"template_height":800036},{"accepted_at_ms":89974,"credit_policy":null,"job_id":"job-1071","job_issued_at_ms":89987,"miner_id":"miner-4","network_difficulty":1000000,"ntime":1700001071,"order_key":"04:miner-4","p2mr_program_hex":"2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f","share_difficulty":3,"share_id":"share-1071","share_seq":1071,"template_height":800033},{"accepted_at_ms":89981,"credit_policy":null,"job_id":"job-1205","job_issued_at_ms":89996,"miner_id":"miner-6","network_difficulty":1000015,"ntime":1700001205,"order_key":"06:miner-6","p2mr_program_hex":"b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5","share_difficulty":7,"share_id":"share-1205","share_seq":1205,"template_height":800037},{"accepted_at_ms":89972,"credit_policy":"stale-grace","job_id":"job-1262","job_issued_at_ms":89971,"miner_id":"miner-8","network_difficulty":1000004,"ntime":1700001262,"order_key":"08:miner-8","p2mr_program_hex":"eeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeee","share_difficulty":19,"share_id":"share-1262","share_seq":1262,"template_height":800039},{"accepted_at_ms":89997,"credit_policy":null,"job_id":"job-424","job_issued_at_ms":89971,"miner_id":"miner-6","network_difficulty":1000016,"ntime":1700000424,"order_key":"06:miner-6","p2mr_program_hex":"a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8","share_difficulty":1,"share_id":"share-424","share_seq":424,"template_height":800013},{"accepted_at_ms":89976,"credit_policy":null,"job_id":"job-1114","job_issued_at_ms":89985,"miner_id":"miner-3","network_difficulty":1000009,"ntime":1700001114,"order_key":"03:miner-3","p2mr_program_hex":"5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a","share_difficulty":2,"share_id":"share-1114","share_seq":1114,"template_height":800034},{"accepted_at_ms":89989,"credit_policy":null,"job_id":"job-1323","job_issued_at_ms":89991,"miner_id":"miner-3","network_difficulty":1000014,"ntime":1700001323,"order_key":"03:miner-3","p2mr_program_hex":"2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b","share_difficulty":2,"share_id":"share-1323","share_seq":1323,"template_height":800041},{"accepted_at_ms":89983,"credit_policy":"stale-grace","job_id":"job-687","job_issued_at_ms":89987,"miner_id":"miner-5","network_difficulty":1000007,"ntime":1700000687,"order_key":"05:miner-5","p2mr_program_hex":"afafafafafafafafafafafafafafafafafafafafafafafafafafafafafafafaf","share_difficulty":19,"share_id":"share-687","share_seq":687,"template_height":800021},{"accepted_at_ms":89981,"credit_policy":null,"job_id":"job-1003","job_issued_at_ms":89978,"miner_id":"miner-2","network_difficulty":1000000,"ntime":1700001003,"order_key":"02:miner-2","p2mr_program_hex":"ebebebebebebebebebebebebebebebebebebebebebebebebebebebebebebebeb","share_difficulty":7,"share_id":"share-1003","share_seq":1003,"template_height":800031},{"accepted_at_ms":89983,"credit_policy":"stale-grace","job_id":"job-454","job_issued_at_ms":89976,"miner_id":"miner-3","network_difficulty":1000012,"ntime":1700000454,"order_key":"03:miner-3","p2mr_program_hex":"c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6","share_difficulty":19,"share_id":"share-454","share_seq":454,"template_height":800014},{"accepted_at_ms":89998,"credit_policy":"stale-grace","job_id":"job-235","job_issued_at_ms":89995,"miner_id":"miner-4","network_difficulty":1000014,"ntime":1700000235,"order_key":"04:miner-4","p2mr_program_hex":"ebebebebebebebebebebebebebebebebebebebebebebebebebebebebebebebeb","share_difficulty":1,"share_id":"share-235","share_seq":235,"template_height":800007},{"accepted_at_ms":90000,"credit_policy":null,"job_id":"job-283","job_issued_at_ms":89987,"miner_id":"miner-8","network_difficulty":1000011,"ntime":1700000283,"order_key":"08:miner-8","p2mr_program_hex":"1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b","share_difficulty":2,"share_id":"share-283","share_seq":283,"template_height":800008},{"accepted_at_ms":89970,"credit_policy":null,"job_id":"job-1304","job_issued_at_ms":89974,"miner_id":"miner-6","network_difficulty":1000012,"ntime":1700001304,"order_key":"06:miner-6","p2mr_program_hex":"1818181818181818181818181818181818181818181818181818181818181818","share_difficulty":1,"share_id":"share-1304","share_seq":1304,"template_height":800040},{"accepted_at_ms":89988,"credit_policy":null,"job_id":"job-1111","job_issued_at_ms":89996,"miner_id":"miner-0","network_difficulty":1000006,"ntime":1700001111,"order_key":"00:miner-0","p2mr_program_hex":"5757575757575757575757575757575757575757575757575757575757575757","share_difficulty":7,"share_id":"share-1111","share_seq":1111,"template_height":800034},{"accepted_at_ms":89994,"credit_policy":"stale-grace","job_id":"job-939","job_issued_at_ms":89992,"miner_id":"miner-4","network_difficulty":1000004,"ntime":1700000939,"order_key":"04:miner-4","p2mr_program_hex":"abababababababababababababababababababababababababababababababab","share_difficulty":3,"share_id":"share-939","share_seq":939,"template_height":800029},{"accepted_at_ms":89984,"credit_policy":"stale-grace","job_id":"job-641","job_issued_at_ms":89996,"miner_id":"miner-3","network_difficulty":1000012,"ntime":1700000641,"order_key":"03:miner-3","p2mr_program_hex":"8181818181818181818181818181818181818181818181818181818181818181","share_difficulty":7,"share_id":"share-641","share_seq":641,"template_height":800020},{"accepted_at_ms":89974,"credit_policy":null,"job_id":"job-543","job_issued_at_ms":89983,"miner_id":"miner-4","network_difficulty":1000016,"ntime":1700000543,"order_key":"04:miner-4","p2mr_program_hex":"1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f","share_difficulty":7,"share_id":"share-543","share_seq":543,"template_height":800016},{"accepted_at_ms":89993,"credit_policy":null,"job_id":"job-49","job_issued_at_ms":89983,"miner_id":"miner-5","network_difficulty":1000015,"ntime":1700000049,"order_key":"05:miner-5","p2mr_program_hex":"3131313131313131313131313131313131313131313131313131313131313131","share_difficulty":3,"share_id":"share-49","share_seq":49,"template_height":800001},{"accepted_at_ms":89971,"credit_policy":"stale-grace","job_id":"job-73","job_issued_at_ms":89982,"miner_id":"miner-7","network_difficulty":1000005,"ntime":1700000073,"order_key":"07:miner-7","p2mr_program_hex":"4949494949494949494949494949494949494949494949494949494949494949","share_difficulty":19,"share_id":"share-73","share_seq":73,"template_height":800002},{"accepted_at_ms":89973,"credit_policy":null,"job_id":"job-72","job_issued_at_ms":89988,"miner_id":"miner-6","network_difficulty":1000004,"ntime":1700000072,"order_key":"06:miner-6","p2mr_program_hex":"4848484848484848484848484848484848484848484848484848484848484848","share_difficulty":2,"share_id":"share-72","share_seq":72,"template_height":800002},{"accepted_at_ms":89970,"credit_policy":"stale-grace","job_id":"job-1710","job_issued_at_ms":89974,"miner_id":"miner-5","network_difficulty":1000010,"ntime":1700001710,"order_key":"05:miner-5","p2mr_program_hex":"aeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeae","share_difficulty":19,"share_id":"share-1710","share_seq":1710,"template_height":800053},{"accepted_at_ms":90000,"credit_policy":null,"job_id":"job-872","job_issued_at_ms":90000,"miner_id":"miner-3","network_difficulty":1000005,"ntime":1700000872,"order_key":"03:miner-3","p2mr_program_hex":"6868686868686868686868686868686868686868686868686868686868686868","share_difficulty":19,"share_id":"share-872","share_seq":872,"template_height":800027},{"accepted_at_ms":89998,"credit_policy":null,"job_id":"job-140","job_issued_at_ms":89999,"miner_id":"miner-8","network_difficulty":1000004,"ntime":1700000140,"order_key":"08:miner-8","p2mr_program_hex":"8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c","share_difficulty":2,"share_id":"share-140","share_seq":140,"template_height":800004},{"accepted_at_ms":89999,"credit_policy":null,"job_id":"job-1601","job_issued_at_ms":89993,"miner_id":"miner-6","network_difficulty":1000003,"ntime":1700001601,"order_key":"06:miner-6","p2mr_program_hex":"4141414141414141414141414141414141414141414141414141414141414141","share_difficulty":7,"share_id":"share-1601","share_seq":1601,"template_height":800050},{"accepted_at_ms":90000,"credit_policy":"stale-grace","job_id":"job-385","job_issued_at_ms":89999,"miner_id":"miner-0","network_difficulty":1000011,"ntime":1700000385,"order_key":"00:miner-0","p2mr_program_hex":"8181818181818181818181818181818181818181818181818181818181818181","share_difficulty":2,"share_id":"share-385","share_seq":385,"template_height":800012},{"accepted_at_ms":89999,"credit_policy":null,"job_id":"job-1250","job_issued_at_ms":89997,"miner_id":"miner-7","network_difficulty":1000009,"ntime":1700001250,"order_key":"07:miner-7","p2mr_program_hex":"e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2","share_difficulty":1,"share_id":"share-1250","share_seq":1250,"template_height":800039},{"accepted_at_ms":89983,"credit_policy":null,"job_id":"job-1372","job_issued_at_ms":89976,"miner_id":"miner-8","network_difficulty":1000012,"ntime":1700001372,"order_key":"08:miner-8","p2mr_program_hex":"5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c","share_difficulty":3,"share_id":"share-1372","share_seq":1372,"template_height":800042},{"accepted_at_ms":89999,"credit_policy":null,"job_id":"job-114","job_issued_at_ms":89973,"miner_id":"miner-4","network_difficulty":1000012,"ntime":1700000114,"order_key":"04:miner-4","p2mr_program_hex":"7272727272727272727272727272727272727272727272727272727272727272","share_difficulty":19,"share_id":"share-114","share_seq":114,"template_height":800003},{"accepted_at_ms":89998,"credit_policy":null,"job_id":"job-720","job_issued_at_ms":89985,"miner_id":"miner-5","network_difficulty":1000006,"ntime":1700000720,"order_key":"05:miner-5","p2mr_program_hex":"d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0","share_difficulty":19,"share_id":"share-720","share_seq":720,"template_height":800022},{"accepted_at_ms":89974,"credit_policy":null,"job_id":"job-503","job_issued_at_ms":89982,"miner_id":"miner-8","network_difficulty":1000010,"ntime":1700000503,"order_key":"08:miner-8","p2mr_program_hex":"f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7","share_difficulty":2,"share_id":"share-503","share_seq":503,"template_height":800015},{"accepted_at_ms":89975,"credit_policy":null,"job_id":"job-1636","job_issued_at_ms":89996,"miner_id":"miner-8","network_difficulty":1000004,"ntime":1700001636,"order_key":"08:miner-8","p2mr_program_hex":"6464646464646464646464646464646464646464646464646464646464646464","share_difficulty":2,"share_id":"share-1636","share_seq":1636,"template_height":800051},{"accepted_at_ms":89978,"credit_policy":"stale-grace","job_id":"job-580","job_issued_at_ms":90000,"miner_id":"miner-8","network_difficulty":1000002,"ntime":1700000580,"order_key":"08:miner-8","p2mr_program_hex":"4444444444444444444444444444444444444444444444444444444444444444","share_difficulty":2,"share_id":"share-580","share_seq":580,"template_height":800018},{"accepted_at_ms":89981,"credit_policy":"stale-grace","job_id":"job-1575","job_issued_at_ms":89999,"miner_id":"miner-2","network_difficulty":1000011,"ntime":1700001575,"order_key":"02:miner-2","p2mr_program_hex":"2727272727272727272727272727272727272727272727272727272727272727","share_difficulty":1,"share_id":"share-1575","share_seq":1575,"template_height":800049},{"accepted_at_ms":89989,"credit_policy":null,"job_id":"job-447","job_issued_at_ms":89980,"miner_id":"miner-7","network_difficulty":1000005,"ntime":1700000447,"order_key":"07:miner-7","p2mr_program_hex":"bfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbf","share_difficulty":3,"share_id":"share-447","share_seq":447,"template_height":800013},{"accepted_at_ms":89975,"credit_policy":"stale-grace","job_id":"job-1707","job_issued_at_ms":89999,"miner_id":"miner-2","network_difficulty":1000007,"ntime":1700001707,"order_key":"02:miner-2","p2mr_program_hex":"abababababababababababababababababababababababababababababababab","share_difficulty":19,"share_id":"share-1707","share_seq":1707,"template_height":800053},{"accepted_at_ms":89990,"credit_policy":null,"job_id":"job-1295","job_issued_at_ms":89988,"miner_id":"miner-8","network_difficulty":1000003,"ntime":1700001295,"order_key":"08:miner-8","p2mr_program_hex":"0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f","share_difficulty":1,"share_id":"share-1295","share_seq":1295,"template_height":800040},{"accepted_at_ms":90000,"credit_policy":null,"job_id":"job-869","job_issued_at_ms":89986,"miner_id":"miner-0","network_difficulty":1000002,"ntime":1700000869,"order_key":"00:miner-0","p2mr_program_hex":"6565656565656565656565656565656565656565656565656565656565656565","share_difficulty":1,"share_id":"share-869","share_seq":869,"template_height":800027},{"accepted_at_ms":89977,"credit_policy":"stale-grace","job_id":"job-70","job_issued_at_ms":89996,"miner_id":"miner-4","network_difficulty":1000002,"ntime":1700000070,"order_key":"04:miner-4","p2mr_program_hex":"4646464646464646464646464646464646464646464646464646464646464646","share_difficulty":19,"share_id":"share-70","share_seq":70,"template_height":800002},{"accepted_at_ms":89971,"credit_policy":null,"job_id":"job-1131","job_issued_at_ms":89999,"miner_id":"miner-9","network_difficulty":1000009,"ntime":1700001131,"order_key":"09:miner-9","p2mr_program_hex":"6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b","share_difficulty":3,"share_id":"share-1131","share_seq":1131,"template_height":800035},{"accepted_at_ms":89977,"credit_policy":null,"job_id":"job-1090","job_issued_at_ms":89992,"miner_id":"miner-1","network_difficulty":1000002,"ntime":1700001090,"order_key":"01:miner-1","p2mr_program_hex":"4242424242424242424242424242424242424242424242424242424242424242","share_difficulty":1,"share_id":"share-1090","share_seq":1090,"template_height":800034},{"accepted_at_ms":89977,"credit_policy":"stale-grace","job_id":"job-964","job_issued_at_ms":89996,"miner_id":"miner-7","network_difficulty":1000012,"ntime":1700000964,"order_key":"07:miner-7","p2mr_program_hex":"c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4","share_difficulty":1,"share_id":"share-964","share_seq":964,"template_height":800030},{"accepted_at_ms":89979,"credit_policy":null,"job_id":"job-1023","job_issued_at_ms":89971,"miner_id":"miner-0","network_difficulty":1000003,"ntime":1700001023,"order_key":"00:miner-0","p2mr_program_hex":"ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff","share_difficulty":3,"share_id":"share-1023","share_seq":1023,"template_height":800031},{"accepted_at_ms":89998,"credit_policy":null,"job_id":"job-381","job_issued_at_ms":89976,"miner_id":"miner-7","network_difficulty":1000007,"ntime":1700000381,"order_key":"07:miner-7","p2mr_program_hex":"7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d","share_difficulty":3,"share_id":"share-381","share_seq":381,"template_height":800011},{"accepted_at_ms":89978,"credit_policy":null,"job_id":"job-1494","job_issued_at_ms":89991,"miner_id":"miner-9","network_difficulty":1000015,"ntime":1700001494,"order_key":"09:miner-9","p2mr_program_hex":"d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6","share_difficulty":2,"share_id":"share-1494","share_seq":1494,"template_height":800046},{"accepted_at_ms":89992,"credit_policy":null,"job_id":"job-281","job_issued_at_ms":89981,"miner_id":"miner-6","network_difficulty":1000009,"ntime":1700000281,"order_key":"06:miner-6","p2mr_program_hex":"1919191919191919191919191919191919191919191919191919191919191919","share_difficulty":3,"share_id":"share-281","share_seq":281,"template_height":800008},{"accepted_at_ms":89987,"credit_policy":null,"job_id":"job-1767","job_issued_at_ms":89978,"miner_id":"miner-7","network_difficulty":1000016,"ntime":1700001767,"order_key":"07:miner-7","p2mr_program_hex":"e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7","share_difficulty":1,"share_id":"share-1767","share_seq":1767,"template_height":800055},{"accepted_at_ms":89990,"credit_policy":"stale-grace","job_id":"job-1533","job_issued_at_ms":89992,"miner_id":"miner-4","network_difficulty":1000003,"ntime":1700001533,"order_key":"04:miner-4","p2mr_program_hex":"fdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfd","share_difficulty":3,"share_id":"share-1533","share_seq":1533,"template_height":800047},{"accepted_at_ms":89988,"credit_policy":null,"job_id":"job-968","job_issued_at_ms":89996,"miner_id":"miner-0","network_difficulty":1000016,"ntime":1700000968,"order_key":"00:miner-0","p2mr_program_hex":"c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8","share_difficulty":19,"share_id":"share-968","share_seq":968,"template_height":800030},{"accepted_at_ms":89983,"credit_policy":null,"job_id":"job-1569","job_issued_at_ms":89985,"miner_id":"miner-7","network_difficulty":1000005,"ntime":1700001569,"order_key":"07:miner-7","p2mr_program_hex":"2121212121212121212121212121212121212121212121212121212121212121","share_difficulty":1,"share_id":"share-1569","share_seq":1569,"template_height":800049},{"accepted_at_ms":89999,"credit_policy":null,"job_id":"job-930","job_issued_at_ms":89985,"miner_id":"miner-6","network_difficulty":1000012,"ntime":1700000930,"order_key":"06:miner-6","p2mr_program_hex":"a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2","share_difficulty":1,"share_id":"share-930","share_seq":930,"template_height":800029},{"accepted_at_ms":89981,"credit_policy":null,"job_id":"job-1577","job_issued_at_ms":89999,"miner_id":"miner-4","network_difficulty":1000013,"ntime":1700001577,"order_key":"04:miner-4","p2mr_program_hex":"2929292929292929292929292929292929292929292929292929292929292929","share_difficulty":3,"share_id":"share-1577","share_seq":1577,"template_height":800049},{"accepted_at_ms":89972,"credit_policy":null,"job_id":"job-504","job_issued_at_ms":89976,"miner_id":"miner-9","network_difficulty":1000011,"ntime":1700000504,"order_key":"09:miner-9","p2mr_program_hex":"f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8","share_difficulty":19,"share_id":"share-504","share_seq":504,"template_height":800015},{"accepted_at_ms":89971,"credit_policy":null,"job_id":"job-22","job_issued_at_ms":89970,"miner_id":"miner-0","network_difficulty":1000005,"ntime":1700000022,"order_key":"00:miner-0","p2mr_program_hex":"1616161616161616161616161616161616161616161616161616161616161616","share_difficulty":3,"share_id":"share-22","share_seq":22,"template_height":800000},{"accepted_at_ms":89973,"credit_policy":null,"job_id":"job-1457","job_issued_at_ms":89973,"miner_id":"miner-5","network_difficulty":1000012,"ntime":1700001457,"order_key":"05:miner-5","p2mr_program_hex":"b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1","share_difficulty":7,"share_id":"share-1457","share_seq":1457,"template_height":800045},{"accepted_at_ms":89974,"credit_policy":null,"job_id":"job-1285","job_issued_at_ms":89973,"miner_id":"miner-9","network_difficulty":1000010,"ntime":1700001285,"order_key":"09:miner-9","p2mr_program_hex":"0505050505050505050505050505050505050505050505050505050505050505","share_difficulty":3,"share_id":"share-1285","share_seq":1285,"template_height":800040},{"accepted_at_ms":89981,"credit_policy":null,"job_id":"job-1792","job_issued_at_ms":89992,"miner_id":"miner-10","network_difficulty":1000007,"ntime":1700001792,"order_key":"10:miner-10","p2mr_program_hex":"0000000000000000000000000000000000000000000000000000000000000000","share_difficulty":1,"share_id":"share-1792","share_seq":1792,"template_height":800056},{"accepted_at_ms":89982,"credit_policy":null,"job_id":"job-1418","job_issued_at_ms":89984,"miner_id":"miner-10","network_difficulty":1000007,"ntime":1700001418,"order_key":"10:miner-10","p2mr_program_hex":"8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a","share_difficulty":2,"share_id":"share-1418","share_seq":1418,"template_height":800044},{"accepted_at_ms":90000,"credit_policy":"stale-grace","job_id":"job-821","job_issued_at_ms":89999,"miner_id":"miner-7","network_difficulty":1000005,"ntime":1700000821,"order_key":"07:miner-7","p2mr_program_hex":"3535353535353535353535353535353535353535353535353535353535353535","share_difficulty":3,"share_id":"share-821","share_seq":821,"template_height":800025},{"accepted_at_ms":89998,"credit_policy":null,"job_id":"job-1275","job_issued_at_ms":89973,"miner_id":"miner-10","network_difficulty":1000000,"ntime":1700001275,"order_key":"10:miner-10","p2mr_program_hex":"fbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfb","share_difficulty":19,"share_id":"share-1275","share_seq":1275,"template_height":800039},{"accepted_at_ms":89974,"credit_policy":null,"job_id":"job-209","job_issued_at_ms":89972,"miner_id":"miner-0","network_difficulty":1000005,"ntime":1700000209,"order_key":"00:miner-0","p2mr_program_hex":"d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1","share_difficulty":1,"share_id":"share-209","share_seq":209,"template_height":800006},{"accepted_at_ms":89990,"credit_policy":null,"job_id":"job-1631","job_issued_at_ms":89988,"miner_id":"miner-3","network_difficulty":1000016,"ntime":1700001631,"order_key":"03:miner-3","p2mr_program_hex":"5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f","share_difficulty":19,"share_id":"share-1631","share_seq":1631,"template_height":800050},{"accepted_at_ms":89981,"credit_policy":null,"job_id":"job-932","job_issued_at_ms":89982,"miner_id":"miner-8","network_difficulty":1000014,"ntime":1700000932,"order_key":"08:miner-8","p2mr_program_hex":"a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4","share_difficulty":19,"share_id":"share-932","share_seq":932,"template_height":800029},{"accepted_at_ms":89970,"credit_policy":null,"job_id":"job-1357","job_issued_at_ms":89986,"miner_id":"miner-4","network_difficulty":1000014,"ntime":1700001357,"order_key":"04:miner-4","p2mr_program_hex":"4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d","share_difficulty":1,"share_id":"share-1357","share_seq":1357,"template_height":800042},{"accepted_at_ms":90000,"credit_policy":null,"job_id":"job-243","job_issued_at_ms":89986,"miner_id":"miner-1","network_difficulty":1000005,"ntime":1700000243,"order_key":"01:miner-1","p2mr_program_hex":"f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3","share_difficulty":3,"share_id":"share-243","share_seq":243,"template_height":800007},{"accepted_at_ms":89987,"credit_policy":"stale-grace","job_id":"job-465","job_issued_at_ms":89998,"miner_id":"miner-3","network_difficulty":1000006,"ntime":1700000465,"order_key":"03:miner-3","p2mr_program_hex":"d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1","share_difficulty":7,"share_id":"share-465","share_seq":465,"template_height":800014},{"accepted_at_ms":89990,"credit_policy":null,"job_id":"job-883","job_issued_at_ms":89991,"miner_id":"miner-3","network_difficulty":1000016,"ntime":1700000883,"order_key":"03:miner-3","p2mr_program_hex":"7373737373737373737373737373737373737373737373737373737373737373","share_difficulty":1,"share_id":"share-883","share_seq":883,"template_height":800027},{"accepted_at_ms":89974,"credit_policy":null,"job_id":"job-985","job_issued_at_ms":89981,"miner_id":"miner-6","network_difficulty":1000016,"ntime":1700000985,"order_key":"06:miner-6","p2mr_program_hex":"d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9","share_difficulty":3,"share_id":"share-985","share_seq":985,"template_height":800030},{"accepted_at_ms":89978,"credit_policy":null,"job_id":"job-1298","job_issued_at_ms":89995,"miner_id":"miner-0","network_difficulty":1000006,"ntime":1700001298,"order_key":"00:miner-0","p2mr_program_hex":"1212121212121212121212121212121212121212121212121212121212121212","share_difficulty":2,"share_id":"share-1298","share_seq":1298,"template_height":800040},{"accepted_at_ms":89983,"credit_policy":null,"job_id":"job-357","job_issued_at_ms":89986,"miner_id":"miner-5","network_difficulty":1000000,"ntime":1700000357,"order_key":"05:miner-5","p2mr_program_hex":"6565656565656565656565656565656565656565656565656565656565656565","share_difficulty":19,"share_id":"share-357","share_seq":357,"template_height":800011},{"accepted_at_ms":89991,"credit_policy":null,"job_id":"job-1288","job_issued_at_ms":89973,"miner_id":"miner-1","network_difficulty":1000013,"ntime":1700001288,"order_key":"01:miner-1","p2mr_program_hex":"0808080808080808080808080808080808080808080808080808080808080808","share_difficulty":2,"share_id":"share-1288","share_seq":1288,"template_height":800040},{"accepted_at_ms":89994,"credit_policy":null,"job_id":"job-1410","job_issued_at_ms":89999,"miner_id":"miner-2","network_difficulty":1000016,"ntime":1700001410,"order_key":"02:miner-2","p2mr_program_hex":"8282828282828282828282828282828282828282828282828282828282828282","share_difficulty":19,"share_id":"share-1410","share_seq":1410,"template_height":800044},{"accepted_at_ms":89976,"credit_policy":null,"job_id":"job-848","job_issued_at_ms":89979,"miner_id":"miner-1","network_difficulty":1000015,"ntime":1700000848,"order_key":"01:miner-1","p2mr_program_hex":"5050505050505050505050505050505050505050505050505050505050505050","share_difficulty":3,"share_id":"share-848","share_seq":848,"template_height":800026},{"accepted_at_ms":89984,"credit_policy":"stale-grace","job_id":"job-983","job_issued_at_ms":89995,"miner_id":"miner-4","network_difficulty":1000014,"ntime":1700000983,"order_key":"04:miner-4","p2mr_program_hex":"d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7","share_difficulty":3,"share_id":"share-983","share_seq":983,"template_height":800030},{"accepted_at_ms":89977,"credit_policy":"stale-grace","job_id":"job-359","job_issued_at_ms":89983,"miner_id":"miner-7","network_difficulty":1000002,"ntime":1700000359,"order_key":"07:miner-7","p2mr_program_hex":"6767676767676767676767676767676767676767676767676767676767676767","share_difficulty":2,"share_id":"share-359","share_seq":359,"template_height":800011},{"accepted_at_ms":89978,"credit_policy":null,"job_id":"job-370","job_issued_at_ms":89996,"miner_id":"miner-7","network_difficulty":1000013,"ntime":1700000370,"order_key":"07:miner-7","p2mr_program_hex":"7272727272727272727272727272727272727272727272727272727272727272","share_difficulty":1,"share_id":"share-370","share_seq":370,"template_height":800011},{"accepted_at_ms":89984,"credit_policy":null,"job_id":"job-411","job_issued_at_ms":89998,"miner_id":"miner-4","network_difficulty":1000003,"ntime":1700000411,"order_key":"04:miner-4","p2mr_program_hex":"9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b","share_difficulty":2,"share_id":"share-411","share_seq":411,"template_height":800012},{"accepted_at_ms":90000,"credit_policy":null,"job_id":"job-1182","job_issued_at_ms":89995,"miner_id":"miner-5","network_difficulty":1000009,"ntime":1700001182,"order_key":"05:miner-5","p2mr_program_hex":"9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e","share_difficulty":1,"share_id":"share-1182","share_seq":1182,"template_height":800036},{"accepted_at_ms":89992,"credit_policy":"stale-grace","job_id":"job-855","job_issued_at_ms":89994,"miner_id":"miner-8","network_difficulty":1000005,"ntime":1700000855,"order_key":"08:miner-8","p2mr_program_hex":"5757575757575757575757575757575757575757575757575757575757575757","share_difficulty":3,"share_id":"share-855","share_seq":855,"template_height":800026},{"accepted_at_ms":89990,"credit_policy":null,"job_id":"job-258","job_issued_at_ms":89997,"miner_id":"miner-5","network_difficulty":1000003,"ntime":1700000258,"order_key":"05:miner-5","p2mr_program_hex":"0202020202020202020202020202020202020202020202020202020202020202","share_difficulty":19,"share_id":"share-258","share_seq":258,"template_height":800008},{"accepted_at_ms":89982,"credit_policy":null,"job_id":"job-989","job_issued_at_ms":89986,"miner_id":"miner-10","network_difficulty":1000003,"ntime":1700000989,"order_key":"10:miner-10","p2mr_program_hex":"dddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddd","share_difficulty":2,"share_id":"share-989","share_seq":989,"template_height":800030},{"accepted_at_ms":89982,"credit_policy":"stale-grace","job_id":"job-1617","job_issued_at_ms":89975,"miner_id":"miner-0","network_difficulty":1000002,"ntime":1700001617,"order_key":"00:miner-0","p2mr_program_hex":"5151515151515151515151515151515151515151515151515151515151515151","share_difficulty":19,"share_id":"share-1617","share_seq":1617,"template_height":800050},{"accepted_at_ms":89988,"credit_policy":"stale-grace","job_id":"job-684","job_issued_at_ms":90000,"miner_id":"miner-2","network_difficulty":1000004,"ntime":1700000684,"order_key":"02:miner-2","p2mr_program_hex":"acacacacacacacacacacacacacacacacacacacacacacacacacacacacacacacac","share_difficulty":3,"share_id":"share-684","share_seq":684,"template_height":800021},{"accepted_at_ms":89983,"credit_policy":"stale-grace","job_id":"job-1140","job_issued_at_ms":89975,"miner_id":"miner-7","network_difficulty":1000001,"ntime":1700001140,"order_key":"07:miner-7","p2mr_program_hex":"7474747474747474747474747474747474747474747474747474747474747474","share_difficulty":2,"share_id":"share-1140","share_seq":1140,"template_height":800035},{"accepted_at_ms":89980,"credit_policy":null,"job_id":"job-958","job_issued_at_ms":89979,"miner_id":"miner-1","network_difficulty":1000006,"ntime":1700000958,"order_key":"01:miner-1","p2mr_program_hex":"bebebebebebebebebebebebebebebebebebebebebebebebebebebebebebebebe","share_difficulty":7,"share_id":"share-958","share_seq":958,"template_height":800029},{"accepted_at_ms":89979,"credit_policy":null,"job_id":"job-767","job_issued_at_ms":89970,"miner_id":"miner-8","network_difficulty":1000002,"ntime":1700000767,"order_key":"08:miner-8","p2mr_program_hex":"ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff","share_difficulty":2,"share_id":"share-767","share_seq":767,"template_height":800023},{"accepted_at_ms":89988,"credit_policy":null,"job_id":"job-1361","job_issued_at_ms":89973,"miner_id":"miner-8","network_difficulty":1000001,"ntime":1700001361,"order_key":"08:miner-8","p2mr_program_hex":"5151515151515151515151515151515151515151515151515151515151515151","share_difficulty":3,"share_id":"share-1361","share_seq":1361,"template_height":800042},{"accepted_at_ms":89993,"credit_policy":null,"job_id":"job-800","job_issued_at_ms":89988,"miner_id":"miner-8","network_difficulty":1000001,"ntime":1700000800,"order_key":"08:miner-8","p2mr_program_hex":"2020202020202020202020202020202020202020202020202020202020202020","share_difficulty":19,"share_id":"share-800","share_seq":800,"template_height":800025},{"accepted_at_ms":89978,"credit_policy":null,"job_id":"job-116","job_issued_at_ms":89990,"miner_id":"miner-6","network_difficulty":1000014,"ntime":1700000116,"order_key":"06:miner-6","p2mr_program_hex":"7474747474747474747474747474747474747474747474747474747474747474","share_difficulty":7,"share_id":"share-116","share_seq":116,"template_height":800003},{"accepted_at_ms":89975,"credit_policy":null,"job_id":"job-490","job_issued_at_ms":90000,"miner_id":"miner-6","network_difficulty":1000014,"ntime":1700000490,"order_key":"06:miner-6","p2mr_program_hex":"eaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaea","share_difficulty":2,"share_id":"share-490","share_seq":490,"template_height":800015},{"accepted_at_ms":89985,"credit_policy":"stale-grace","job_id":"job-514","job_issued_at_ms":89973,"miner_id":"miner-8","network_difficulty":1000004,"ntime":1700000514,"order_key":"08:miner-8","p2mr_program_hex":"0202020202020202020202020202020202020202020202020202020202020202","share_difficulty":7,"share_id":"share-514","share_seq":514,"template_height":800016},{"accepted_at_ms":89988,"credit_policy":"stale-grace","job_id":"job-692","job_issued_at_ms":89998,"miner_id":"miner-10","network_difficulty":1000012,"ntime":1700000692,"order_key":"10:miner-10","p2mr_program_hex":"b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4","share_difficulty":1,"share_id":"share-692","share_seq":692,"template_height":800021},{"accepted_at_ms":89999,"credit_policy":null,"job_id":"job-345","job_issued_at_ms":89999,"miner_id":"miner-4","network_difficulty":1000005,"ntime":1700000345,"order_key":"04:miner-4","p2mr_program_hex":"5959595959595959595959595959595959595959595959595959595959595959","share_difficulty":7,"share_id":"share-345","share_seq":345,"template_height":800010},{"accepted_at_ms":89972,"credit_policy":"stale-grace","job_id":"job-1270","job_issued_at_ms":89994,"miner_id":"miner-5","network_difficulty":1000012,"ntime":1700001270,"order_key":"05:miner-5","p2mr_program_hex":"f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6","share_difficulty":19,"share_id":"share-1270","share_seq":1270,"template_height":800039},{"accepted_at_ms":89992,"credit_policy":null,"job_id":"job-628","job_issued_at_ms":89988,"miner_id":"miner-1","network_difficulty":1000016,"ntime":1700000628,"order_key":"01:miner-1","p2mr_program_hex":"7474747474747474747474747474747474747474747474747474747474747474","share_difficulty":19,"share_id":"share-628","share_seq":628,"template_height":800019},{"accepted_at_ms":89988,"credit_policy":null,"job_id":"job-510","job_issued_at_ms":89971,"miner_id":"miner-4","network_difficulty":1000000,"ntime":1700000510,"order_key":"04:miner-4","p2mr_program_hex":"fefefefefefefefefefefefefefefefefefefefefefefefefefefefefefefefe","share_difficulty":1,"share_id":"share-510","share_seq":510,"template_height":800015},{"accepted_at_ms":89978,"credit_policy":"stale-grace","job_id":"job-1126","job_issued_at_ms":89970,"miner_id":"miner-4","network_difficulty":1000004,"ntime":1700001126,"order_key":"04:miner-4","p2mr_program_hex":"6666666666666666666666666666666666666666666666666666666666666666","share_difficulty":7,"share_id":"share-1126","share_seq":1126,"template_height":800035},{"accepted_at_ms":89981,"credit_policy":"stale-grace","job_id":"job-1373","job_issued_at_ms":89997,"miner_id":"miner-9","network_difficulty":1000013,"ntime":1700001373,"order_key":"09:miner-9","p2mr_program_hex":"5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d","share_difficulty":7,"share_id":"share-1373","share_seq":1373,"template_height":800042},{"accepted_at_ms":89999,"credit_policy":"stale-grace","job_id":"job-923","job_issued_at_ms":89981,"miner_id":"miner-10","network_difficulty":1000005,"ntime":1700000923,"order_key":"10:miner-10","p2mr_program_hex":"9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b","share_difficulty":3,"share_id":"share-923","share_seq":923,"template_height":800028},{"accepted_at_ms":89997,"credit_policy":null,"job_id":"job-489","job_issued_at_ms":89973,"miner_id":"miner-5","network_difficulty":1000013,"ntime":1700000489,"order_key":"05:miner-5","p2mr_program_hex":"e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9","share_difficulty":2,"share_id":"share-489","share_seq":489,"template_height":800015},{"accepted_at_ms":89976,"credit_policy":null,"job_id":"job-802","job_issued_at_ms":89973,"miner_id":"miner-10","network_difficulty":1000003,"ntime":1700000802,"order_key":"10:miner-10","p2mr_program_hex":"2222222222222222222222222222222222222222222222222222222222222222","share_difficulty":2,"share_id":"share-802","share_seq":802,"template_height":800025},{"accepted_at_ms":89984,"credit_policy":null,"job_id":"job-313","job_issued_at_ms":89994,"miner_id":"miner-5","network_difficulty":1000007,"ntime":1700000313,"order_key":"05:miner-5","p2mr_program_hex":"3939393939393939393939393939393939393939393939393939393939393939","share_difficulty":7,"share_id":"share-313","share_seq":313,"template_height":800009},{"accepted_at_ms":89977,"credit_policy":null,"job_id":"job-120","job_issued_at_ms":89976,"miner_id":"miner-10","network_difficulty":1000001,"ntime":1700000120,"order_key":"10:miner-10","p2mr_program_hex":"7878787878787878787878787878787878787878787878787878787878787878","share_difficulty":3,"share_id":"share-120","share_seq":120,"template_height":800003},{"accepted_at_ms":89999,"credit_policy":null,"job_id":"job-601","job_issued_at_ms":89980,"miner_id":"miner-7","network_difficulty":1000006,"ntime":1700000601,"order_key":"07:miner-7","p2mr_program_hex":"5959595959595959595959595959595959595959595959595959595959595959","share_difficulty":19,"share_id":"share-601","share_seq":601,"template_height":800018},{"accepted_at_ms":89995,"credit_policy":"stale-grace","job_id":"job-77","job_issued_at_ms":89994,"miner_id":"miner-0","network_difficulty":1000009,"ntime":1700000077,"order_key":"00:miner-0","p2mr_program_hex":"4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d","share_difficulty":3,"share_id":"share-77","share_seq":77,"template_height":800002},{"accepted_at_ms":89981,"credit_policy":null,"job_id":"job-262","job_issued_at_ms":89978,"miner_id":"miner-9","network_difficulty":1000007,"ntime":1700000262,"order_key":"09:miner-9","p2mr_program_hex":"0606060606060606060606060606060606060606060606060606060606060606","share_difficulty":1,"share_id":"share-262","share_seq":262,"template_height":800008},{"accepted_at_ms":89973,"credit_policy":null,"job_id":"job-624","job_issued_at_ms":89988,"miner_id":"miner-8","network_difficulty":1000012,"ntime":1700000624,"order_key":"08:miner-8","p2mr_program_hex":"7070707070707070707070707070707070707070707070707070707070707070","share_difficulty":3,"share_id":"share-624","share_seq":624,"template_height":800019},{"accepted_at_ms":89973,"credit_policy":null,"job_id":"job-1413","job_issued_at_ms":89981,"miner_id":"miner-5","network_difficulty":1000002,"ntime":1700001413,"order_key":"05:miner-5","p2mr_program_hex":"8585858585858585858585858585858585858585858585858585858585858585","share_difficulty":1,"share_id":"share-1413","share_seq":1413,"template_height":800044},{"accepted_at_ms":89978,"credit_policy":null,"job_id":"job-1401","job_issued_at_ms":89976,"miner_id":"miner-4","network_difficulty":1000007,"ntime":1700001401,"order_key":"04:miner-4","p2mr_program_hex":"7979797979797979797979797979797979797979797979797979797979797979","share_difficulty":19,"share_id":"share-1401","share_seq":1401,"template_height":800043},{"accepted_at_ms":89997,"credit_policy":null,"job_id":"job-929","job_issued_at_ms":89987,"miner_id":"miner-5","network_difficulty":1000011,"ntime":1700000929,"order_key":"05:miner-5","p2mr_program_hex":"a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1","share_difficulty":3,"share_id":"share-929","share_seq":929,"template_height":800029},{"accepted_at_ms":89989,"credit_policy":null,"job_id":"job-1360","job_issued_at_ms":89996,"miner_id":"miner-7","network_difficulty":1000000,"ntime":1700001360,"order_key":"07:miner-7","p2mr_program_hex":"5050505050505050505050505050505050505050505050505050505050505050","share_difficulty":19,"share_id":"share-1360","share_seq":1360,"template_height":800042},{"accepted_at_ms":89989,"credit_policy":null,"job_id":"job-541","job_issued_at_ms":89973,"miner_id":"miner-2","network_difficulty":1000014,"ntime":1700000541,"order_key":"02:miner-2","p2mr_program_hex":"1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d","share_difficulty":1,"share_id":"share-541","share_seq":541,"template_height":800016},{"accepted_at_ms":89994,"credit_policy":"stale-grace","job_id":"job-1074","job_issued_at_ms":89988,"miner_id":"miner-7","network_difficulty":1000003,"ntime":1700001074,"order_key":"07:miner-7","p2mr_program_hex":"3232323232323232323232323232323232323232323232323232323232323232","share_difficulty":7,"share_id":"share-1074","share_seq":1074,"template_height":800033},{"accepted_at_ms":89987,"credit_policy":null,"job_id":"job-880","job_issued_at_ms":89997,"miner_id":"miner-0","network_difficulty":1000013,"ntime":1700000880,"order_key":"00:miner-0","p2mr_program_hex":"7070707070707070707070707070707070707070707070707070707070707070","share_difficulty":1,"share_id":"share-880","share_seq":880,"template_height":800027},{"accepted_at_ms":89984,"credit_policy":null,"job_id":"job-1582","job_issued_at_ms":89995,"miner_id":"miner-9","network_difficulty":1000001,"ntime":1700001582,"order_key":"09:miner-9","p2mr_program_hex":"2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e","share_difficulty":7,"share_id":"share-1582","share_seq":1582,"template_height":800049},{"accepted_at_ms":89994,"credit_policy":"stale-grace","job_id":"job-831","job_issued_at_ms":89981,"miner_id":"miner-6","network_difficulty":1000015,"ntime":1700000831,"order_key":"06:miner-6","p2mr_program_hex":"3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f","share_difficulty":1,"share_id":"share-831","share_seq":831,"template_height":800025},{"accepted_at_ms":89989,"credit_policy":"stale-grace","job_id":"job-1635","job_issued_at_ms":89973,"miner_id":"miner-7","network_difficulty":1000003,"ntime":1700001635,"order_key":"07:miner-7","p2mr_program_hex":"6363636363636363636363636363636363636363636363636363636363636363","share_difficulty":19,"share_id":"share-1635","share_seq":1635,"template_height":800051},{"accepted_at_ms":89987,"credit_policy":null,"job_id":"job-1784","job_issued_at_ms":89975,"miner_id":"miner-2","network_difficulty":1000016,"ntime":1700001784,"order_key":"02:miner-2","p2mr_program_hex":"f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8","share_difficulty":19,"share_id":"share-1784","share_seq":1784,"template_height":800055},{"accepted_at_ms":89987,"credit_policy":null,"job_id":"job-619","job_issued_at_ms":89992,"miner_id":"miner-3","network_difficulty":1000007,"ntime":1700000619,"order_key":"03:miner-3","p2mr_program_hex":"6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b","share_difficulty":19,"share_id":"share-619","share_seq":619,"template_height":800019},{"accepted_at_ms":89986,"credit_policy":"stale-grace","job_id":"job-1780","job_issued_at_ms":89991,"miner_id":"miner-9","network_difficulty":1000012,"ntime":1700001780,"order_key":"09:miner-9","p2mr_program_hex":"f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4","share_difficulty":3,"share_id":"share-1780","share_seq":1780,"template_height":800055},{"accepted_at_ms":89975,"credit_policy":null,"job_id":"job-304","job_issued_at_ms":89971,"miner_id":"miner-7","network_difficulty":1000015,"ntime":1700000304,"order_key":"07:miner-7","p2mr_program_hex":"3030303030303030303030303030303030303030303030303030303030303030","share_difficulty":7,"share_id":"share-304","share_seq":304,"template_height":800009},{"accepted_at_ms":89994,"credit_policy":null,"job_id":"job-1523","job_issued_at_ms":89992,"miner_id":"miner-5","network_difficulty":1000010,"ntime":1700001523,"order_key":"05:miner-5","p2mr_program_hex":"f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3","share_difficulty":7,"share_id":"share-1523","share_seq":1523,"template_height":800047},{"accepted_at_ms":89977,"credit_policy":null,"job_id":"job-337","job_issued_at_ms":89993,"miner_id":"miner-7","network_difficulty":1000014,"ntime":1700000337,"order_key":"07:miner-7","p2mr_program_hex":"5151515151515151515151515151515151515151515151515151515151515151","share_difficulty":1,"share_id":"share-337","share_seq":337,"template_height":800010},{"accepted_at_ms":89991,"credit_policy":"stale-grace","job_id":"job-614","job_issued_at_ms":89981,"miner_id":"miner-9","network_difficulty":1000002,"ntime":1700000614,"order_key":"09:miner-9","p2mr_program_hex":"6666666666666666666666666666666666666666666666666666666666666666","share_difficulty":1,"share_id":"share-614","share_seq":614,"template_height":800019},{"accepted_at_ms":89983,"credit_policy":null,"job_id":"job-900","job_issued_at_ms":89990,"miner_id":"miner-9","network_difficulty":1000016,"ntime":1700000900,"order_key":"09:miner-9","p2mr_program_hex":"8484848484848484848484848484848484848484848484848484848484848484","share_difficulty":19,"share_id":"share-900","share_seq":900,"template_height":800028},{"accepted_at_ms":89991,"credit_policy":null,"job_id":"job-379","job_issued_at_ms":89971,"miner_id":"miner-5","network_difficulty":1000005,"ntime":1700000379,"order_key":"05:miner-5","p2mr_program_hex":"7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b","share_difficulty":1,"share_id":"share-379","share_seq":379,"template_height":800011},{"accepted_at_ms":89988,"credit_policy":"stale-grace","job_id":"job-1490","job_issued_at_ms":89983,"miner_id":"miner-5","network_difficulty":1000011,"ntime":1700001490,"order_key":"05:miner-5","p2mr_program_hex":"d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2","share_difficulty":19,"share_id":"share-1490","share_seq":1490,"template_height":800046},{"accepted_at_ms":89983,"credit_policy":null,"job_id":"job-877","job_issued_at_ms":89991,"miner_id":"miner-8","network_difficulty":1000010,"ntime":1700000877,"order_key":"08:miner-8","p2mr_program_hex":"6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d","share_difficulty":2,"share_id":"share-877","share_seq":877,"template_height":800027},{"accepted_at_ms":89992,"credit_policy":null,"job_id":"job-609","job_issued_at_ms":89986,"miner_id":"miner-4","network_difficulty":1000014,"ntime":1700000609,"order_key":"04:miner-4","p2mr_program_hex":"6161616161616161616161616161616161616161616161616161616161616161","share_difficulty":2,"share_id":"share-609","share_seq":609,"template_height":800019},{"accepted_at_ms":89999,"credit_policy":"stale-grace","job_id":"job-494","job_issued_at_ms":89975,"miner_id":"miner-10","network_difficulty":1000001,"ntime":1700000494,"order_key":"10:miner-10","p2mr_program_hex":"eeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeee","share_difficulty":1,"share_id":"share-494","share_seq":494,"template_height":800015},{"accepted_at_ms":89990,"credit_policy":"stale-grace","job_id":"job-824","job_issued_at_ms":89977,"miner_id":"miner-10","network_difficulty":1000008,"ntime":1700000824,"order_key":"10:miner-10","p2mr_program_hex":"3838383838383838383838383838383838383838383838383838383838383838","share_difficulty":2,"share_id":"share-824","share_seq":824,"template_height":800025},{"accepted_at_ms":89995,"credit_policy":null,"job_id":"job-1512","job_issued_at_ms":89998,"miner_id":"miner-5","network_difficulty":1000016,"ntime":1700001512,"order_key":"05:miner-5","p2mr_program_hex":"e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8","share_difficulty":2,"share_id":"share-1512","share_seq":1512,"template_height":800047},{"accepted_at_ms":89994,"credit_policy":null,"job_id":"job-1062","job_issued_at_ms":89995,"miner_id":"miner-6","network_difficulty":1000008,"ntime":1700001062,"order_key":"06:miner-6","p2mr_program_hex":"2626262626262626262626262626262626262626262626262626262626262626","share_difficulty":1,"share_id":"share-1062","share_seq":1062,"template_height":800033},{"accepted_at_ms":89978,"credit_policy":"stale-grace","job_id":"job-1400","job_issued_at_ms":89993,"miner_id":"miner-3","network_difficulty":1000006,"ntime":1700001400,"order_key":"03:miner-3","p2mr_program_hex":"7878787878787878787878787878787878787878787878787878787878787878","share_difficulty":3,"share_id":"share-1400","share_seq":1400,"template_height":800043},{"accepted_at_ms":89996,"credit_policy":null,"job_id":"job-1509","job_issued_at_ms":89984,"miner_id":"miner-2","network_difficulty":1000013,"ntime":1700001509,"order_key":"02:miner-2","p2mr_program_hex":"e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5","share_difficulty":19,"share_id":"share-1509","share_seq":1509,"template_height":800047},{"accepted_at_ms":89973,"credit_policy":"stale-grace","job_id":"job-1483","job_issued_at_ms":89999,"miner_id":"miner-9","network_difficulty":1000004,"ntime":1700001483,"order_key":"09:miner-9","p2mr_program_hex":"cbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcb","share_difficulty":2,"share_id":"share-1483","share_seq":1483,"template_height":800046},{"accepted_at_ms":89995,"credit_policy":"stale-grace","job_id":"job-100","job_issued_at_ms":89972,"miner_id":"miner-1","network_difficulty":1000015,"ntime":1700000100,"order_key":"01:miner-1","p2mr_program_hex":"6464646464646464646464646464646464646464646464646464646464646464","share_difficulty":2,"share_id":"share-100","share_seq":100,"template_height":800003},{"accepted_at_ms":89993,"credit_policy":"stale-grace","job_id":"job-804","job_issued_at_ms":89983,"miner_id":"miner-1","network_difficulty":1000005,"ntime":1700000804,"order_key":"01:miner-1","p2mr_program_hex":"2424242424242424242424242424242424242424242424242424242424242424","share_difficulty":3,"share_id":"share-804","share_seq":804,"template_height":800025},{"accepted_at_ms":89977,"credit_policy":"stale-grace","job_id":"job-1168","job_issued_at_ms":89978,"miner_id":"miner-2","network_difficulty":1000012,"ntime":1700001168,"order_key":"02:miner-2","p2mr_program_hex":"9090909090909090909090909090909090909090909090909090909090909090","share_difficulty":2,"share_id":"share-1168","share_seq":1168,"template_height":800036},{"accepted_at_ms":89998,"credit_policy":null,"job_id":"job-297","job_issued_at_ms":89971,"miner_id":"miner-0","network_difficulty":1000008,"ntime":1700000297,"order_key":"00:miner-0","p2mr_program_hex":"2929292929292929292929292929292929292929292929292929292929292929","share_difficulty":3,"share_id":"share-297","share_seq":297,"template_height":800009},{"accepted_at_ms":89999,"credit_policy":null,"job_id":"job-1240","job_issued_at_ms":90000,"miner_id":"miner-8","network_difficulty":1000016,"ntime":1700001240,"order_key":"08:miner-8","p2mr_program_hex":"d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8","share_difficulty":7,"share_id":"share-1240","share_seq":1240,"template_height":800038},{"accepted_at_ms":89983,"credit_policy":"stale-grace","job_id":"job-233","job_issued_at_ms":89999,"miner_id":"miner-2","network_difficulty":1000012,"ntime":1700000233,"order_key":"02:miner-2","p2mr_program_hex":"e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9","share_difficulty":3,"share_id":"share-233","share_seq":233,"template_height":800007},{"accepted_at_ms":89981,"credit_policy":null,"job_id":"job-1344","job_issued_at_ms":89990,"miner_id":"miner-2","network_difficulty":1000001,"ntime":1700001344,"order_key":"02:miner-2","p2mr_program_hex":"4040404040404040404040404040404040404040404040404040404040404040","share_difficulty":1,"share_id":"share-1344","share_seq":1344,"template_height":800042},{"accepted_at_ms":89998,"credit_policy":null,"job_id":"job-1645","job_issued_at_ms":89974,"miner_id":"miner-6","network_difficulty":1000013,"ntime":1700001645,"order_key":"06:miner-6","p2mr_program_hex":"6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d","share_difficulty":1,"share_id":"share-1645","share_seq":1645,"template_height":800051},{"accepted_at_ms":89995,"credit_policy":"stale-grace","job_id":"job-1646","job_issued_at_ms":89982,"miner_id":"miner-7","network_difficulty":1000014,"ntime":1700001646,"order_key":"07:miner-7","p2mr_program_hex":"6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e","share_difficulty":7,"share_id":"share-1646","share_seq":1646,"template_height":800051},{"accepted_at_ms":89978,"credit_policy":null,"job_id":"job-764","job_issued_at_ms":89986,"miner_id":"miner-5","network_difficulty":1000016,"ntime":1700000764,"order_key":"05:miner-5","p2mr_program_hex":"fcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfc","share_difficulty":3,"share_id":"share-764","share_seq":764,"template_height":800023},{"accepted_at_ms":89996,"credit_policy":null,"job_id":"job-274","job_issued_at_ms":89984,"miner_id":"miner-10","network_difficulty":1000002,"ntime":1700000274,"order_key":"10:miner-10","p2mr_program_hex":"1212121212121212121212121212121212121212121212121212121212121212","share_difficulty":3,"share_id":"share-274","share_seq":274,"template_height":800008},{"accepted_at_ms":89998,"credit_policy":null,"job_id":"job-1089","job_issued_at_ms":89998,"miner_id":"miner-0","network_difficulty":1000001,"ntime":1700001089,"order_key":"00:miner-0","p2mr_program_hex":"4141414141414141414141414141414141414141414141414141414141414141","share_difficulty":3,"share_id":"share-1089","share_seq":1089,"template_height":800034},{"accepted_at_ms":89973,"credit_policy":"stale-grace","job_id":"job-1665","job_issued_at_ms":89971,"miner_id":"miner-4","network_difficulty":1000016,"ntime":1700001665,"order_key":"04:miner-4","p2mr_program_hex":"8181818181818181818181818181818181818181818181818181818181818181","share_difficulty":19,"share_id":"share-1665","share_seq":1665,"template_height":800052},{"accepted_at_ms":89975,"credit_policy":"stale-grace","job_id":"job-1724","job_issued_at_ms":89976,"miner_id":"miner-8","network_difficulty":1000007,"ntime":1700001724,"order_key":"08:miner-8","p2mr_program_hex":"bcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbc","share_difficulty":19,"share_id":"share-1724","share_seq":1724,"template_height":800053},{"accepted_at_ms":89973,"credit_policy":"stale-grace","job_id":"job-1422","job_issued_at_ms":89989,"miner_id":"miner-3","network_difficulty":1000011,"ntime":1700001422,"order_key":"03:miner-3","p2mr_program_hex":"8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e","share_difficulty":7,"share_id":"share-1422","share_seq":1422,"template_height":800044},{"accepted_at_ms":89989,"credit_policy":"stale-grace","job_id":"job-650","job_issued_at_ms":89983,"miner_id":"miner-1","network_difficulty":1000004,"ntime":1700000650,"order_key":"01:miner-1","p2mr_program_hex":"8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a","share_difficulty":3,"share_id":"share-650","share_seq":650,"template_height":800020},{"accepted_at_ms":89990,"credit_policy":null,"job_id":"job-1684","job_issued_at_ms":89994,"miner_id":"miner-1","network_difficulty":1000001,"ntime":1700001684,"order_key":"01:miner-1","p2mr_program_hex":"9494949494949494949494949494949494949494949494949494949494949494","share_difficulty":7,"share_id":"share-1684","share_seq":1684,"template_height":800052},{"accepted_at_ms":89998,"credit_policy":null,"job_id":"job-1531","job_issued_at_ms":89985,"miner_id":"miner-2","network_difficulty":1000001,"ntime":1700001531,"order_key":"02:miner-2","p2mr_program_hex":"fbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfb","share_difficulty":3,"share_id":"share-1531","share_seq":1531,"template_height":800047},{"accepted_at_ms":89985,"credit_policy":null,"job_id":"job-68","job_issued_at_ms":89977,"miner_id":"miner-2","network_difficulty":1000000,"ntime":1700000068,"order_key":"02:miner-2","p2mr_program_hex":"4444444444444444444444444444444444444444444444444444444444444444","share_difficulty":7,"share_id":"share-68","share_seq":68,"template_height":800002},{"accepted_at_ms":89992,"credit_policy":null,"job_id":"job-351","job_issued_at_ms":89988,"miner_id":"miner-10","network_difficulty":1000011,"ntime":1700000351,"order_key":"10:miner-10","p2mr_program_hex":"5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f","share_difficulty":19,"share_id":"share-351","share_seq":351,"template_height":800010},{"accepted_at_ms":89987,"credit_policy":null,"job_id":"job-198","job_issued_at_ms":89982,"miner_id":"miner-0","network_difficulty":1000011,"ntime":1700000198,"order_key":"00:miner-0","p2mr_program_hex":"c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6","share_difficulty":7,"share_id":"share-198","share_seq":198,"template_height":800006},{"accepted_at_ms":89971,"credit_policy":null,"job_id":"job-1585","job_issued_at_ms":89974,"miner_id":"miner-1","network_difficulty":1000004,"ntime":1700001585,"order_key":"01:miner-1","p2mr_program_hex":"3131313131313131313131313131313131313131313131313131313131313131","share_difficulty":1,"share_id":"share-1585","share_seq":1585,"template_height":800049},{"accepted_at_ms":89981,"credit_policy":null,"job_id":"job-946","job_issued_at_ms":89976,"miner_id":"miner-0","network_difficulty":1000011,"ntime":1700000946,"order_key":"00:miner-0","p2mr_program_hex":"b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2","share_difficulty":7,"share_id":"share-946","share_seq":946,"template_height":800029},{"accepted_at_ms":89973,"credit_policy":null,"job_id":"job-956","job_issued_at_ms":89979,"miner_id":"miner-10","network_difficulty":1000004,"ntime":1700000956,"order_key":"10:miner-10","p2mr_program_hex":"bcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbc","share_difficulty":7,"share_id":"share-956","share_seq":956,"template_height":800029},{"accepted_at_ms":89984,"credit_policy":null,"job_id":"job-888","job_issued_at_ms":89984,"miner_id":"miner-8","network_difficulty":1000004,"ntime":1700000888,"order_key":"08:miner-8","p2mr_program_hex":"7878787878787878787878787878787878787878787878787878787878787878","share_difficulty":19,"share_id":"share-888","share_seq":888,"template_height":800027},{"accepted_at_ms":89999,"credit_policy":"stale-grace","job_id":"job-1546","job_issued_at_ms":89988,"miner_id":"miner-6","network_difficulty":1000016,"ntime":1700001546,"order_key":"06:miner-6","p2mr_program_hex":"0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a","share_difficulty":2,"share_id":"share-1546","share_seq":1546,"template_height":800048},{"accepted_at_ms":89992,"credit_policy":null,"job_id":"job-287","job_issued_at_ms":89974,"miner_id":"miner-1","network_difficulty":1000015,"ntime":1700000287,"order_key":"01:miner-1","p2mr_program_hex":"1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f","share_difficulty":2,"share_id":"share-287","share_seq":287,"template_height":800008},{"accepted_at_ms":89995,"credit_policy":null,"job_id":"job-237","job_issued_at_ms":89988,"miner_id":"miner-6","network_difficulty":1000016,"ntime":1700000237,"order_key":"06:miner-6","p2mr_program_hex":"edededededededededededededededededededededededededededededededed","share_difficulty":2,"share_id":"share-237","share_seq":237,"template_height":800007},{"accepted_at_ms":89977,"credit_policy":null,"job_id":"job-1059","job_issued_at_ms":89989,"miner_id":"miner-3","network_difficulty":1000005,"ntime":1700001059,"order_key":"03:miner-3","p2mr_program_hex":"2323232323232323232323232323232323232323232323232323232323232323","share_difficulty":7,"share_id":"share-1059","share_seq":1059,"template_height":800033},{"accepted_at_ms":89974,"credit_policy":null,"job_id":"job-984","job_issued_at_ms":89972,"miner_id":"miner-5","network_difficulty":1000015,"ntime":1700000984,"order_key":"05:miner-5","p2mr_program_hex":"d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8","share_difficulty":7,"share_id":"share-984","share_seq":984,"template_height":800030},{"accepted_at_ms":89977,"credit_policy":null,"job_id":"job-782","job_issued_at_ms":89971,"miner_id":"miner-1","network_difficulty":1000000,"ntime":1700000782,"order_key":"01:miner-1","p2mr_program_hex":"0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e","share_difficulty":3,"share_id":"share-782","share_seq":782,"template_height":800024},{"accepted_at_ms":89976,"credit_policy":null,"job_id":"job-1043","job_issued_at_ms":89987,"miner_id":"miner-9","network_difficulty":1000006,"ntime":1700001043,"order_key":"09:miner-9","p2mr_program_hex":"1313131313131313131313131313131313131313131313131313131313131313","share_difficulty":19,"share_id":"share-1043","share_seq":1043,"template_height":800032},{"accepted_at_ms":89972,"credit_policy":null,"job_id":"job-1179","job_issued_at_ms":89976,"miner_id":"miner-2","network_difficulty":1000006,"ntime":1700001179,"order_key":"02:miner-2","p2mr_program_hex":"9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b","share_difficulty":19,"share_id":"share-1179","share_seq":1179,"template_height":800036},{"accepted_at_ms":89981,"credit_policy":null,"job_id":"job-1207","job_issued_at_ms":89987,"miner_id":"miner-8","network_difficulty":1000000,"ntime":1700001207,"order_key":"08:miner-8","p2mr_program_hex":"b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7","share_difficulty":2,"share_id":"share-1207","share_seq":1207,"template_height":800037},{"accepted_at_ms":89982,"credit_policy":"stale-grace","job_id":"job-747","job_issued_at_ms":89996,"miner_id":"miner-10","network_difficulty":1000016,"ntime":1700000747,"order_key":"10:miner-10","p2mr_program_hex":"ebebebebebebebebebebebebebebebebebebebebebebebebebebebebebebebeb","share_difficulty":3,"share_id":"share-747","share_seq":747,"template_height":800023},{"accepted_at_ms":89978,"credit_policy":null,"job_id":"job-721","job_issued_at_ms":89972,"miner_id":"miner-6","network_difficulty":1000007,"ntime":1700000721,"order_key":"06:miner-6","p2mr_program_hex":"d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1","share_difficulty":2,"share_id":"share-721","share_seq":721,"template_height":800022},{"accepted_at_ms":89992,"credit_policy":null,"job_id":"job-813","job_issued_at_ms":89996,"miner_id":"miner-10","network_difficulty":1000014,"ntime":1700000813,"order_key":"10:miner-10","p2mr_program_hex":"2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d","share_difficulty":3,"share_id":"share-813","share_seq":813,"template_height":800025},{"accepted_at_ms":89980,"credit_policy":null,"job_id":"job-1652","job_issued_at_ms":89997,"miner_id":"miner-2","network_difficulty":1000003,"ntime":1700001652,"order_key":"02:miner-2","p2mr_program_hex":"7474747474747474747474747474747474747474747474747474747474747474","share_difficulty":7,"share_id":"share-1652","share_seq":1652,"template_height":800051},{"accepted_at_ms":89988,"credit_policy":null,"job_id":"job-876","job_issued_at_ms":89977,"miner_id":"miner-7","network_difficulty":1000009,"ntime":1700000876,"order_key":"07:miner-7","p2mr_program_hex":"6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c","share_difficulty":2,"share_id":"share-876","share_seq":876,"template_height":800027},{"accepted_at_ms":89994,"credit_policy":null,"job_id":"job-1267","job_issued_at_ms":89991,"miner_id":"miner-2","network_difficulty":1000009,"ntime":1700001267,"order_key":"02:miner-2","p2mr_program_hex":"f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3","share_difficulty":2,"share_id":"share-1267","share_seq":1267,"template_height":800039},{"accepted_at_ms":89979,"credit_policy":null,"job_id":"job-1133","job_issued_at_ms":89990,"miner_id":"miner-0","network_difficulty":1000011,"ntime":1700001133,"order_key":"00:miner-0","p2mr_program_hex":"6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d","share_difficulty":19,"share_id":"share-1133","share_seq":1133,"template_height":800035},{"accepted_at_ms":89972,"credit_policy":"stale-grace","job_id":"job-724","job_issued_at_ms":89974,"miner_id":"miner-9","network_difficulty":1000010,"ntime":1700000724,"order_key":"09:miner-9","p2mr_program_hex":"d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4","share_difficulty":3,"share_id":"share-724","share_seq":724,"template_height":800022},{"accepted_at_ms":89992,"credit_policy":null,"job_id":"job-1453","job_issued_at_ms":89973,"miner_id":"miner-1","network_difficulty":1000008,"ntime":1700001453,"order_key":"01:miner-1","p2mr_program_hex":"adadadadadadadadadadadadadadadadadadadadadadadadadadadadadadadad","share_difficulty":2,"share_id":"share-1453","share_seq":1453,"template_height":800045},{"accepted_at_ms":90000,"credit_policy":null,"job_id":"job-1508","job_issued_at_ms":89992,"miner_id":"miner-1","network_difficulty":1000012,"ntime":1700001508,"order_key":"01:miner-1","p2mr_program_hex":"e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4","share_difficulty":1,"share_id":"share-1508","share_seq":1508,"template_height":800047},{"accepted_at_ms":89970,"credit_policy":"stale-grace","job_id":"job-1621","job_issued_at_ms":89991,"miner_id":"miner-4","network_difficulty":1000006,"ntime":1700001621,"order_key":"04:miner-4","p2mr_program_hex":"5555555555555555555555555555555555555555555555555555555555555555","share_difficulty":3,"share_id":"share-1621","share_seq":1621,"template_height":800050},{"accepted_at_ms":89978,"credit_policy":"stale-grace","job_id":"job-1736","job_issued_at_ms":89977,"miner_id":"miner-9","network_difficulty":1000002,"ntime":1700001736,"order_key":"09:miner-9","p2mr_program_hex":"c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8","share_difficulty":7,"share_id":"share-1736","share_seq":1736,"template_height":800054},{"accepted_at_ms":89999,"credit_policy":null,"job_id":"job-412","job_issued_at_ms":89998,"miner_id":"miner-5","network_difficulty":1000004,"ntime":1700000412,"order_key":"05:miner-5","p2mr_program_hex":"9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c","share_difficulty":2,"share_id":"share-412","share_seq":412,"template_height":800012},{"accepted_at_ms":89977,"credit_policy":null,"job_id":"job-772","job_issued_at_ms":89984,"miner_id":"miner-2","network_difficulty":1000007,"ntime":1700000772,"order_key":"02:miner-2","p2mr_program_hex":"0404040404040404040404040404040404040404040404040404040404040404","share_difficulty":19,"share_id":"share-772","share_seq":772,"template_height":800024},{"accepted_at_ms":89981,"credit_policy":null,"job_id":"job-585","job_issued_at_ms":89985,"miner_id":"miner-2","network_difficulty":1000007,"ntime":1700000585,"order_key":"02:miner-2","p2mr_program_hex":"4949494949494949494949494949494949494949494949494949494949494949","share_difficulty":19,"share_id":"share-585","share_seq":585,"template_height":800018},{"accepted_at_ms":89980,"credit_policy":null,"job_id":"job-584","job_issued_at_ms":89992,"miner_id":"miner-1","network_difficulty":1000006,"ntime":1700000584,"order_key":"01:miner-1","p2mr_program_hex":"4848484848484848484848484848484848484848484848484848484848484848","share_difficulty":19,"share_id":"share-584","share_seq":584,"template_height":800018},{"accepted_at_ms":89973,"credit_policy":null,"job_id":"job-688","job_issued_at_ms":89985,"miner_id":"miner-6","network_difficulty":1000008,"ntime":1700000688,"order_key":"06:miner-6","p2mr_program_hex":"b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0","share_difficulty":3,"share_id":"share-688","share_seq":688,"template_height":800021},{"accepted_at_ms":89986,"credit_policy":null,"job_id":"job-438","job_issued_at_ms":89989,"miner_id":"miner-9","network_difficulty":1000013,"ntime":1700000438,"order_key":"09:miner-9","p2mr_program_hex":"b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6","share_difficulty":19,"share_id":"share-438","share_seq":438,"template_height":800013},{"accepted_at_ms":89974,"credit_policy":"stale-grace","job_id":"job-1188","job_issued_at_ms":89994,"miner_id":"miner-0","network_difficulty":1000015,"ntime":1700001188,"order_key":"00:miner-0","p2mr_program_hex":"a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4","share_difficulty":1,"share_id":"share-1188","share_seq":1188,"template_height":800037},{"accepted_at_ms":89985,"credit_policy":null,"job_id":"job-537","job_issued_at_ms":89982,"miner_id":"miner-9","network_difficulty":1000010,"ntime":1700000537,"order_key":"09:miner-9","p2mr_program_hex":"1919191919191919191919191919191919191919191919191919191919191919","share_difficulty":1,"share_id":"share-537","share_seq":537,"template_height":800016},{"accepted_at_ms":89984,"credit_policy":"stale-grace","job_id":"job-1752","job_issued_at_ms":90000,"miner_id":"miner-3","network_difficulty":1000001,"ntime":1700001752,"order_key":"03:miner-3","p2mr_program_hex":"d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8","share_difficulty":1,"share_id":"share-1752","share_seq":1752,"template_height":800054},{"accepted_at_ms":89986,"credit_policy":null,"job_id":"job-640","job_issued_at_ms":89996,"miner_id":"miner-2","network_difficulty":1000011,"ntime":1700000640,"order_key":"02:miner-2","p2mr_program_hex":"8080808080808080808080808080808080808080808080808080808080808080","share_difficulty":19,"share_id":"share-640","share_seq":640,"template_height":800020},{"accepted_at_ms":89998,"credit_policy":null,"job_id":"job-1300","job_issued_at_ms":89983,"miner_id":"miner-2","network_difficulty":1000008,"ntime":1700001300,"order_key":"02:miner-2","p2mr_program_hex":"1414141414141414141414141414141414141414141414141414141414141414","share_difficulty":19,"share_id":"share-1300","share_seq":1300,"template_height":800040},{"accepted_at_ms":89970,"credit_policy":"stale-grace","job_id":"job-784","job_issued_at_ms":89986,"miner_id":"miner-3","network_difficulty":1000002,"ntime":1700000784,"order_key":"03:miner-3","p2mr_program_hex":"1010101010101010101010101010101010101010101010101010101010101010","share_difficulty":2,"share_id":"share-784","share_seq":784,"template_height":800024},{"accepted_at_ms":89997,"credit_policy":null,"job_id":"job-177","job_issued_at_ms":89980,"miner_id":"miner-1","network_difficulty":1000007,"ntime":1700000177,"order_key":"01:miner-1","p2mr_program_hex":"b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1","share_difficulty":2,"share_id":"share-177","share_seq":177,"template_height":800005},{"accepted_at_ms":89994,"credit_policy":null,"job_id":"job-259","job_issued_at_ms":89999,"miner_id":"miner-6","network_difficulty":1000004,"ntime":1700000259,"order_key":"06:miner-6","p2mr_program_hex":"0303030303030303030303030303030303030303030303030303030303030303","share_difficulty":19,"share_id":"share-259","share_seq":259,"template_height":800008},{"accepted_at_ms":89970,"credit_policy":null,"job_id":"job-590","job_issued_at_ms":89973,"miner_id":"miner-7","network_difficulty":1000012,"ntime":1700000590,"order_key":"07:miner-7","p2mr_program_hex":"4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e","share_difficulty":2,"share_id":"share-590","share_seq":590,"template_height":800018},{"accepted_at_ms":89972,"credit_policy":null,"job_id":"job-1787","job_issued_at_ms":89995,"miner_id":"miner-5","network_difficulty":1000002,"ntime":1700001787,"order_key":"05:miner-5","p2mr_program_hex":"fbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfb","share_difficulty":3,"share_id":"share-1787","share_seq":1787,"template_height":800055},{"accepted_at_ms":89976,"credit_policy":null,"job_id":"job-228","job_issued_at_ms":89988,"miner_id":"miner-8","network_difficulty":1000007,"ntime":1700000228,"order_key":"08:miner-8","p2mr_program_hex":"e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4","share_difficulty":3,"share_id":"share-228","share_seq":228,"template_height":800007},{"accepted_at_ms":89983,"credit_policy":"stale-grace","job_id":"job-889","job_issued_at_ms":89977,"miner_id":"miner-9","network_difficulty":1000005,"ntime":1700000889,"order_key":"09:miner-9","p2mr_program_hex":"7979797979797979797979797979797979797979797979797979797979797979","share_difficulty":7,"share_id":"share-889","share_seq":889,"template_height":800027},{"accepted_at_ms":89994,"credit_policy":"stale-grace","job_id":"job-1657","job_issued_at_ms":89996,"miner_id":"miner-7","network_difficulty":1000008,"ntime":1700001657,"order_key":"07:miner-7","p2mr_program_hex":"7979797979797979797979797979797979797979797979797979797979797979","share_difficulty":19,"share_id":"share-1657","share_seq":1657,"template_height":800051},{"accepted_at_ms":89994,"credit_policy":null,"job_id":"job-129","job_issued_at_ms":89974,"miner_id":"miner-8","network_difficulty":1000010,"ntime":1700000129,"order_key":"08:miner-8","p2mr_program_hex":"8181818181818181818181818181818181818181818181818181818181818181","share_difficulty":19,"share_id":"share-129","share_seq":129,"template_height":800004},{"accepted_at_ms":89981,"credit_policy":null,"job_id":"job-1727","job_issued_at_ms":89997,"miner_id":"miner-0","network_difficulty":1000010,"ntime":1700001727,"order_key":"00:miner-0","p2mr_program_hex":"bfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbf","share_difficulty":1,"share_id":"share-1727","share_seq":1727,"template_height":800053},{"accepted_at_ms":89971,"credit_policy":null,"job_id":"job-835","job_issued_at_ms":89984,"miner_id":"miner-10","network_difficulty":1000002,"ntime":1700000835,"order_key":"10:miner-10","p2mr_program_hex":"4343434343434343434343434343434343434343434343434343434343434343","share_difficulty":2,"share_id":"share-835","share_seq":835,"template_height":800026},{"accepted_at_ms":90000,"credit_policy":null,"job_id":"job-527","job_issued_at_ms":89988,"miner_id":"miner-10","network_difficulty":1000000,"ntime":1700000527,"order_key":"10:miner-10","p2mr_program_hex":"0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f","share_difficulty":1,"share_id":"share-527","share_seq":527,"template_height":800016},{"accepted_at_ms":89977,"credit_policy":null,"job_id":"job-306","job_issued_at_ms":89983,"miner_id":"miner-9","network_difficulty":1000000,"ntime":1700000306,"order_key":"09:miner-9","p2mr_program_hex":"3232323232323232323232323232323232323232323232323232323232323232","share_difficulty":1,"share_id":"share-306","share_seq":306,"template_height":800009},{"accepted_at_ms":89975,"credit_policy":null,"job_id":"job-305","job_issued_at_ms":90000,"miner_id":"miner-8","network_difficulty":1000016,"ntime":1700000305,"order_key":"08:miner-8","p2mr_program_hex":"3131313131313131313131313131313131313131313131313131313131313131","share_difficulty":7,"share_id":"share-305","share_seq":305,"template_height":800009},{"accepted_at_ms":89979,"credit_policy":null,"job_id":"job-1297","job_issued_at_ms":89970,"miner_id":"miner-10","network_difficulty":1000005,"ntime":1700001297,"order_key":"10:miner-10","p2mr_program_hex":"1111111111111111111111111111111111111111111111111111111111111111","share_difficulty":19,"share_id":"share-1297","share_seq":1297,"template_height":800040},{"accepted_at_ms":89980,"credit_policy":"stale-grace","job_id":"job-1524","job_issued_at_ms":89994,"miner_id":"miner-6","network_difficulty":1000011,"ntime":1700001524,"order_key":"06:miner-6","p2mr_program_hex":"f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4","share_difficulty":1,"share_id":"share-1524","share_seq":1524,"template_height":800047},{"accepted_at_ms":89982,"credit_policy":null,"job_id":"job-13","job_issued_at_ms":89993,"miner_id":"miner-2","network_difficulty":1000013,"ntime":1700000013,"order_key":"02:miner-2","p2mr_program_hex":"0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d","share_difficulty":1,"share_id":"share-13","share_seq":13,"template_height":800000},{"accepted_at_ms":89986,"credit_policy":null,"job_id":"job-1331","job_issued_at_ms":89980,"miner_id":"miner-0","network_difficulty":1000005,"ntime":1700001331,"order_key":"00:miner-0","p2mr_program_hex":"3333333333333333333333333333333333333333333333333333333333333333","share_difficulty":2,"share_id":"share-1331","share_seq":1331,"template_height":800041},{"accepted_at_ms":89970,"credit_policy":null,"job_id":"job-921","job_issued_at_ms":89977,"miner_id":"miner-8","network_difficulty":1000003,"ntime":1700000921,"order_key":"08:miner-8","p2mr_program_hex":"9999999999999999999999999999999999999999999999999999999999999999","share_difficulty":19,"share_id":"share-921","share_seq":921,"template_height":800028},{"accepted_at_ms":89997,"credit_policy":null,"job_id":"job-535","job_issued_at_ms":89993,"miner_id":"miner-7","network_difficulty":1000008,"ntime":1700000535,"order_key":"07:miner-7","p2mr_program_hex":"1717171717171717171717171717171717171717171717171717171717171717","share_difficulty":7,"share_id":"share-535","share_seq":535,"template_height":800016},{"accepted_at_ms":89983,"credit_policy":null,"job_id":"job-1120","job_issued_at_ms":89978,"miner_id":"miner-9","network_difficulty":1000015,"ntime":1700001120,"order_key":"09:miner-9","p2mr_program_hex":"6060606060606060606060606060606060606060606060606060606060606060","share_difficulty":3,"share_id":"share-1120","share_seq":1120,"template_height":800035},{"accepted_at_ms":89983,"credit_policy":"stale-grace","job_id":"job-1099","job_issued_at_ms":89988,"miner_id":"miner-10","network_difficulty":1000011,"ntime":1700001099,"order_key":"10:miner-10","p2mr_program_hex":"4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b","share_difficulty":19,"share_id":"share-1099","share_seq":1099,"template_height":800034},{"accepted_at_ms":89981,"credit_policy":"stale-grace","job_id":"job-1484","job_issued_at_ms":89981,"miner_id":"miner-10","network_difficulty":1000005,"ntime":1700001484,"order_key":"10:miner-10","p2mr_program_hex":"cccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccc","share_difficulty":7,"share_id":"share-1484","share_seq":1484,"template_height":800046},{"accepted_at_ms":89984,"credit_policy":null,"job_id":"job-467","job_issued_at_ms":89982,"miner_id":"miner-5","network_difficulty":1000008,"ntime":1700000467,"order_key":"05:miner-5","p2mr_program_hex":"d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3","share_difficulty":1,"share_id":"share-467","share_seq":467,"template_height":800014},{"accepted_at_ms":89975,"credit_policy":null,"job_id":"job-355","job_issued_at_ms":89984,"miner_id":"miner-3","network_difficulty":1000015,"ntime":1700000355,"order_key":"03:miner-3","p2mr_program_hex":"6363636363636363636363636363636363636363636363636363636363636363","share_difficulty":19,"share_id":"share-355","share_seq":355,"template_height":800011},{"accepted_at_ms":89980,"credit_policy":null,"job_id":"job-1263","job_issued_at_ms":89980,"miner_id":"miner-9","network_difficulty":1000005,"ntime":1700001263,"order_key":"09:miner-9","p2mr_program_hex":"efefefefefefefefefefefefefefefefefefefefefefefefefefefefefefefef","share_difficulty":19,"share_id":"share-1263","share_seq":1263,"template_height":800039},{"accepted_at_ms":89975,"credit_policy":null,"job_id":"job-102","job_issued_at_ms":89975,"miner_id":"miner-3","network_difficulty":1000000,"ntime":1700000102,"order_key":"03:miner-3","p2mr_program_hex":"6666666666666666666666666666666666666666666666666666666666666666","share_difficulty":19,"share_id":"share-102","share_seq":102,"template_height":800003},{"accepted_at_ms":89997,"credit_policy":"stale-grace","job_id":"job-1042","job_issued_at_ms":89980,"miner_id":"miner-8","network_difficulty":1000005,"ntime":1700001042,"order_key":"08:miner-8","p2mr_program_hex":"1212121212121212121212121212121212121212121212121212121212121212","share_difficulty":19,"share_id":"share-1042","share_seq":1042,"template_height":800032},{"accepted_at_ms":89990,"credit_policy":null,"job_id":"job-1468","job_issued_at_ms":89977,"miner_id":"miner-5","network_difficulty":1000006,"ntime":1700001468,"order_key":"05:miner-5","p2mr_program_hex":"bcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbc","share_difficulty":1,"share_id":"share-1468","share_seq":1468,"template_height":800045},{"accepted_at_ms":89981,"credit_policy":null,"job_id":"job-1264","job_issued_at_ms":89992,"miner_id":"miner-10","network_difficulty":1000006,"ntime":1700001264,"order_key":"10:miner-10","p2mr_program_hex":"f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0","share_difficulty":2,"share_id":"share-1264","share_seq":1264,"template_height":800039},{"accepted_at_ms":89995,"credit_policy":null,"job_id":"job-1312","job_issued_at_ms":89998,"miner_id":"miner-3","network_difficulty":1000003,"ntime":1700001312,"order_key":"03:miner-3","p2mr_program_hex":"2020202020202020202020202020202020202020202020202020202020202020","share_difficulty":3,"share_id":"share-1312","share_seq":1312,"template_height":800041},{"accepted_at_ms":89992,"credit_policy":"stale-grace","job_id":"job-1489","job_issued_at_ms":89970,"miner_id":"miner-4","network_difficulty":1000010,"ntime":1700001489,"order_key":"04:miner-4","p2mr_program_hex":"d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1","share_difficulty":1,"share_id":"share-1489","share_seq":1489,"template_height":800046},{"accepted_at_ms":89974,"credit_policy":null,"job_id":"job-311","job_issued_at_ms":89990,"miner_id":"miner-3","network_difficulty":1000005,"ntime":1700000311,"order_key":"03:miner-3","p2mr_program_hex":"3737373737373737373737373737373737373737373737373737373737373737","share_difficulty":1,"share_id":"share-311","share_seq":311,"template_height":800009},{"accepted_at_ms":89984,"credit_policy":null,"job_id":"job-1661","job_issued_at_ms":89986,"miner_id":"miner-0","network_difficulty":1000012,"ntime":1700001661,"order_key":"00:miner-0","p2mr_program_hex":"7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d","share_difficulty":1,"share_id":"share-1661","share_seq":1661,"template_height":800051},{"accepted_at_ms":89984,"credit_policy":"stale-grace","job_id":"job-1324","job_issued_at_ms":90000,"miner_id":"miner-4","network_difficulty":1000015,"ntime":1700001324,"order_key":"04:miner-4","p2mr_program_hex":"2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c","share_difficulty":3,"share_id":"share-1324","share_seq":1324,"template_height":800041},{"accepted_at_ms":89994,"credit_policy":null,"job_id":"job-1682","job_issued_at_ms":89975,"miner_id":"miner-10","network_difficulty":1000016,"ntime":1700001682,"order_key":"10:miner-10","p2mr_program_hex":"9292929292929292929292929292929292929292929292929292929292929292","share_difficulty":1,"share_id":"share-1682","share_seq":1682,"template_height":800052},{"accepted_at_ms":89997,"credit_policy":"stale-grace","job_id":"job-587","job_issued_at_ms":89984,"miner_id":"miner-4","network_difficulty":1000009,"ntime":1700000587,"order_key":"04:miner-4","p2mr_program_hex":"4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b","share_difficulty":3,"share_id":"share-587","share_seq":587,"template_height":800018},{"accepted_at_ms":89983,"credit_policy":null,"job_id":"job-1439","job_issued_at_ms":89983,"miner_id":"miner-9","network_difficulty":1000011,"ntime":1700001439,"order_key":"09:miner-9","p2mr_program_hex":"9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f","share_difficulty":2,"share_id":"share-1439","share_seq":1439,"template_height":800044},{"accepted_at_ms":89996,"credit_policy":null,"job_id":"job-1116","job_issued_at_ms":89991,"miner_id":"miner-5","network_difficulty":1000011,"ntime":1700001116,"order_key":"05:miner-5","p2mr_program_hex":"5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c","share_difficulty":2,"share_id":"share-1116","share_seq":1116,"template_height":800034},{"accepted_at_ms":89970,"credit_policy":null,"job_id":"job-1281","job_issued_at_ms":89989,"miner_id":"miner-5","network_difficulty":1000006,"ntime":1700001281,"order_key":"05:miner-5","p2mr_program_hex":"0101010101010101010101010101010101010101010101010101010101010101","share_difficulty":2,"share_id":"share-1281","share_seq":1281,"template_height":800040},{"accepted_at_ms":90000,"credit_policy":null,"job_id":"job-806","job_issued_at_ms":89982,"miner_id":"miner-3","network_difficulty":1000007,"ntime":1700000806,"order_key":"03:miner-3","p2mr_program_hex":"2626262626262626262626262626262626262626262626262626262626262626","share_difficulty":2,"share_id":"share-806","share_seq":806,"template_height":800025},{"accepted_at_ms":89982,"credit_policy":"stale-grace","job_id":"job-1265","job_issued_at_ms":89978,"miner_id":"miner-0","network_difficulty":1000007,"ntime":1700001265,"order_key":"00:miner-0","p2mr_program_hex":"f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1","share_difficulty":2,"share_id":"share-1265","share_seq":1265,"template_height":800039},{"accepted_at_ms":89999,"credit_policy":null,"job_id":"job-197","job_issued_at_ms":89993,"miner_id":"miner-10","network_difficulty":1000010,"ntime":1700000197,"order_key":"10:miner-10","p2mr_program_hex":"c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5","share_difficulty":1,"share_id":"share-197","share_seq":197,"template_height":800006},{"accepted_at_ms":89994,"credit_policy":null,"job_id":"job-404","job_issued_at_ms":89987,"miner_id":"miner-8","network_difficulty":1000013,"ntime":1700000404,"order_key":"08:miner-8","p2mr_program_hex":"9494949494949494949494949494949494949494949494949494949494949494","share_difficulty":2,"share_id":"share-404","share_seq":404,"template_height":800012},{"accepted_at_ms":89987,"credit_policy":null,"job_id":"job-255","job_issued_at_ms":89999,"miner_id":"miner-2","network_difficulty":1000000,"ntime":1700000255,"order_key":"02:miner-2","p2mr_program_hex":"ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff","share_difficulty":7,"share_id":"share-255","share_seq":255,"template_height":800007},{"accepted_at_ms":89972,"credit_policy":null,"job_id":"job-1469","job_issued_at_ms":89992,"miner_id":"miner-6","network_difficulty":1000007,"ntime":1700001469,"order_key":"06:miner-6","p2mr_program_hex":"bdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbd","share_difficulty":3,"share_id":"share-1469","share_seq":1469,"template_height":800045},{"accepted_at_ms":89990,"credit_policy":null,"job_id":"job-1069","job_issued_at_ms":89981,"miner_id":"miner-2","network_difficulty":1000015,"ntime":1700001069,"order_key":"02:miner-2","p2mr_program_hex":"2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d","share_difficulty":2,"share_id":"share-1069","share_seq":1069,"template_height":800033},{"accepted_at_ms":89997,"credit_policy":null,"job_id":"job-750","job_issued_at_ms":89999,"miner_id":"miner-2","network_difficulty":1000002,"ntime":1700000750,"order_key":"02:miner-2","p2mr_program_hex":"eeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeee","share_difficulty":2,"share_id":"share-750","share_seq":750,"template_height":800023},{"accepted_at_ms":89979,"credit_policy":null,"job_id":"job-669","job_issued_at_ms":89989,"miner_id":"miner-9","network_difficulty":1000006,"ntime":1700000669,"order_key":"09:miner-9","p2mr_program_hex":"9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d","share_difficulty":2,"share_id":"share-669","share_seq":669,"template_height":800020},{"accepted_at_ms":89997,"credit_policy":null,"job_id":"job-981","job_issued_at_ms":89988,"miner_id":"miner-2","network_difficulty":1000012,"ntime":1700000981,"order_key":"02:miner-2","p2mr_program_hex":"d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5","share_difficulty":1,"share_id":"share-981","share_seq":981,"template_height":800030},{"accepted_at_ms":90000,"credit_policy":null,"job_id":"job-1630","job_issued_at_ms":90000,"miner_id":"miner-2","network_difficulty":1000015,"ntime":1700001630,"order_key":"02:miner-2","p2mr_program_hex":"5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e","share_difficulty":3,"share_id":"share-1630","share_seq":1630,"template_height":800050},{"accepted_at_ms":89990,"credit_policy":null,"job_id":"job-1669","job_issued_at_ms":89984,"miner_id":"miner-8","network_difficulty":1000003,"ntime":1700001669,"order_key":"08:miner-8","p2mr_program_hex":"8585858585858585858585858585858585858585858585858585858585858585","share_difficulty":3,"share_id":"share-1669","share_seq":1669,"template_height":800052},{"accepted_at_ms":90000,"credit_policy":"stale-grace","job_id":"job-1464","job_issued_at_ms":89977,"miner_id":"miner-1","network_difficulty":1000002,"ntime":1700001464,"order_key":"01:miner-1","p2mr_program_hex":"b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8","share_difficulty":19,"share_id":"share-1464","share_seq":1464,"template_height":800045},{"accepted_at_ms":89973,"credit_policy":null,"job_id":"job-1342","job_issued_at_ms":89992,"miner_id":"miner-0","network_difficulty":1000016,"ntime":1700001342,"order_key":"00:miner-0","p2mr_program_hex":"3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e","share_difficulty":2,"share_id":"share-1342","share_seq":1342,"template_height":800041},{"accepted_at_ms":89997,"credit_policy":null,"job_id":"job-1379","job_issued_at_ms":89991,"miner_id":"miner-4","network_difficulty":1000002,"ntime":1700001379,"order_key":"04:miner-4","p2mr_program_hex":"6363636363636363636363636363636363636363636363636363636363636363","share_difficulty":19,"share_id":"share-1379","share_seq":1379,"template_height":800043},{"accepted_at_ms":89979,"credit_policy":"stale-grace","job_id":"job-1130","job_issued_at_ms":89987,"miner_id":"miner-8","network_difficulty":1000008,"ntime":1700001130,"order_key":"08:miner-8","p2mr_program_hex":"6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a","share_difficulty":7,"share_id":"share-1130","share_seq":1130,"template_height":800035},{"accepted_at_ms":89978,"credit_policy":"stale-grace","job_id":"job-271","job_issued_at_ms":89976,"miner_id":"miner-7","network_difficulty":1000016,"ntime":1700000271,"order_key":"07:miner-7","p2mr_program_hex":"0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f","share_difficulty":3,"share_id":"share-271","share_seq":271,"template_height":800008},{"accepted_at_ms":89977,"credit_policy":null,"job_id":"job-506","job_issued_at_ms":89998,"miner_id":"miner-0","network_difficulty":1000013,"ntime":1700000506,"order_key":"00:miner-0","p2mr_program_hex":"fafafafafafafafafafafafafafafafafafafafafafafafafafafafafafafafa","share_difficulty":3,"share_id":"share-506","share_seq":506,"template_height":800015},{"accepted_at_ms":89980,"credit_policy":null,"job_id":"job-616","job_issued_at_ms":89976,"miner_id":"miner-0","network_difficulty":1000004,"ntime":1700000616,"order_key":"00:miner-0","p2mr_program_hex":"6868686868686868686868686868686868686868686868686868686868686868","share_difficulty":19,"share_id":"share-616","share_seq":616,"template_height":800019},{"accepted_at_ms":89978,"credit_policy":null,"job_id":"job-1183","job_issued_at_ms":89988,"miner_id":"miner-6","network_difficulty":1000010,"ntime":1700001183,"order_key":"06:miner-6","p2mr_program_hex":"9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f","share_difficulty":3,"share_id":"share-1183","share_seq":1183,"template_height":800036},{"accepted_at_ms":89978,"credit_policy":null,"job_id":"job-1334","job_issued_at_ms":89973,"miner_id":"miner-3","network_difficulty":1000008,"ntime":1700001334,"order_key":"03:miner-3","p2mr_program_hex":"3636363636363636363636363636363636363636363636363636363636363636","share_difficulty":1,"share_id":"share-1334","share_seq":1334,"template_height":800041},{"accepted_at_ms":89980,"credit_policy":null,"job_id":"job-938","job_issued_at_ms":89984,"miner_id":"miner-3","network_difficulty":1000003,"ntime":1700000938,"order_key":"03:miner-3","p2mr_program_hex":"aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa","share_difficulty":2,"share_id":"share-938","share_seq":938,"template_height":800029},{"accepted_at_ms":89980,"credit_policy":null,"job_id":"job-766","job_issued_at_ms":89975,"miner_id":"miner-7","network_difficulty":1000001,"ntime":1700000766,"order_key":"07:miner-7","p2mr_program_hex":"fefefefefefefefefefefefefefefefefefefefefefefefefefefefefefefefe","share_difficulty":2,"share_id":"share-766","share_seq":766,"template_height":800023},{"accepted_at_ms":89999,"credit_policy":"stale-grace","job_id":"job-338","job_issued_at_ms":89996,"miner_id":"miner-8","network_difficulty":1000015,"ntime":1700000338,"order_key":"08:miner-8","p2mr_program_hex":"5252525252525252525252525252525252525252525252525252525252525252","share_difficulty":3,"share_id":"share-338","share_seq":338,"template_height":800010},{"accepted_at_ms":89970,"credit_policy":"stale-grace","job_id":"job-288","job_issued_at_ms":89990,"miner_id":"miner-2","network_difficulty":1000016,"ntime":1700000288,"order_key":"02:miner-2","p2mr_program_hex":"2020202020202020202020202020202020202020202020202020202020202020","share_difficulty":19,"share_id":"share-288","share_seq":288,"template_height":800009},{"accepted_at_ms":89998,"credit_policy":"stale-grace","job_id":"job-420","job_issued_at_ms":89972,"miner_id":"miner-2","network_difficulty":1000012,"ntime":1700000420,"order_key":"02:miner-2","p2mr_program_hex":"a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4","share_difficulty":19,"share_id":"share-420","share_seq":420,"template_height":800013},{"accepted_at_ms":89974,"credit_policy":null,"job_id":"job-1717","job_issued_at_ms":89970,"miner_id":"miner-1","network_difficulty":1000000,"ntime":1700001717,"order_key":"01:miner-1","p2mr_program_hex":"b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5","share_difficulty":2,"share_id":"share-1717","share_seq":1717,"template_height":800053},{"accepted_at_ms":89990,"credit_policy":null,"job_id":"job-440","job_issued_at_ms":89970,"miner_id":"miner-0","network_difficulty":1000015,"ntime":1700000440,"order_key":"00:miner-0","p2mr_program_hex":"b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8","share_difficulty":7,"share_id":"share-440","share_seq":440,"template_height":800013},{"accepted_at_ms":89973,"credit_policy":"stale-grace","job_id":"job-1296","job_issued_at_ms":89982,"miner_id":"miner-9","network_difficulty":1000004,"ntime":1700001296,"order_key":"09:miner-9","p2mr_program_hex":"1010101010101010101010101010101010101010101010101010101010101010","share_difficulty":19,"share_id":"share-1296","share_seq":1296,"template_height":800040},{"accepted_at_ms":89983,"credit_policy":null,"job_id":"job-1362","job_issued_at_ms":89989,"miner_id":"miner-9","network_difficulty":1000002,"ntime":1700001362,"order_key":"09:miner-9","p2mr_program_hex":"5252525252525252525252525252525252525252525252525252525252525252","share_difficulty":3,"share_id":"share-1362","share_seq":1362,"template_height":800042},{"accepted_at_ms":89988,"credit_policy":null,"job_id":"job-909","job_issued_at_ms":89977,"miner_id":"miner-7","network_difficulty":1000008,"ntime":1700000909,"order_key":"07:miner-7","p2mr_program_hex":"8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d","share_difficulty":2,"share_id":"share-909","share_seq":909,"template_height":800028},{"accepted_at_ms":89986,"credit_policy":"stale-grace","job_id":"job-1230","job_issued_at_ms":89980,"miner_id":"miner-9","network_difficulty":1000006,"ntime":1700001230,"order_key":"09:miner-9","p2mr_program_hex":"cececececececececececececececececececececececececececececececece","share_difficulty":3,"share_id":"share-1230","share_seq":1230,"template_height":800038},{"accepted_at_ms":89984,"credit_policy":"stale-grace","job_id":"job-388","job_issued_at_ms":89980,"miner_id":"miner-3","network_difficulty":1000014,"ntime":1700000388,"order_key":"03:miner-3","p2mr_program_hex":"8484848484848484848484848484848484848484848484848484848484848484","share_difficulty":3,"share_id":"share-388","share_seq":388,"template_height":800012},{"accepted_at_ms":89978,"credit_policy":null,"job_id":"job-1029","job_issued_at_ms":89985,"miner_id":"miner-6","network_difficulty":1000009,"ntime":1700001029,"order_key":"06:miner-6","p2mr_program_hex":"0505050505050505050505050505050505050505050505050505050505050505","share_difficulty":1,"share_id":"share-1029","share_seq":1029,"template_height":800032},{"accepted_at_ms":89995,"credit_policy":"stale-grace","job_id":"job-661","job_issued_at_ms":89977,"miner_id":"miner-1","network_difficulty":1000015,"ntime":1700000661,"order_key":"01:miner-1","p2mr_program_hex":"9595959595959595959595959595959595959595959595959595959595959595","share_difficulty":1,"share_id":"share-661","share_seq":661,"template_height":800020},{"accepted_at_ms":89974,"credit_policy":"stale-grace","job_id":"job-561","job_issued_at_ms":89987,"miner_id":"miner-0","network_difficulty":1000000,"ntime":1700000561,"order_key":"00:miner-0","p2mr_program_hex":"3131313131313131313131313131313131313131313131313131313131313131","share_difficulty":3,"share_id":"share-561","share_seq":561,"template_height":800017},{"accepted_at_ms":89980,"credit_policy":"stale-grace","job_id":"job-433","job_issued_at_ms":89990,"miner_id":"miner-4","network_difficulty":1000008,"ntime":1700000433,"order_key":"04:miner-4","p2mr_program_hex":"b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1","share_difficulty":2,"share_id":"share-433","share_seq":433,"template_height":800013},{"accepted_at_ms":89978,"credit_policy":"stale-grace","job_id":"job-1409","job_issued_at_ms":89975,"miner_id":"miner-1","network_difficulty":1000015,"ntime":1700001409,"order_key":"01:miner-1","p2mr_program_hex":"8181818181818181818181818181818181818181818181818181818181818181","share_difficulty":3,"share_id":"share-1409","share_seq":1409,"template_height":800044},{"accepted_at_ms":89976,"credit_policy":null,"job_id":"job-591","job_issued_at_ms":89978,"miner_id":"miner-8","network_difficulty":1000013,"ntime":1700000591,"order_key":"08:miner-8","p2mr_program_hex":"4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f","share_difficulty":1,"share_id":"share-591","share_seq":591,"template_height":800018},{"accepted_at_ms":89991,"credit_policy":null,"job_id":"job-598","job_issued_at_ms":89974,"miner_id":"miner-4","network_difficulty":1000003,"ntime":1700000598,"order_key":"04:miner-4","p2mr_program_hex":"5656565656565656565656565656565656565656565656565656565656565656","share_difficulty":19,"share_id":"share-598","share_seq":598,"template_height":800018},{"accepted_at_ms":89983,"credit_policy":"stale-grace","job_id":"job-1192","job_issued_at_ms":89997,"miner_id":"miner-4","network_difficulty":1000002,"ntime":1700001192,"order_key":"04:miner-4","p2mr_program_hex":"a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8","share_difficulty":1,"share_id":"share-1192","share_seq":1192,"template_height":800037},{"accepted_at_ms":89976,"credit_policy":"stale-grace","job_id":"job-1317","job_issued_at_ms":89983,"miner_id":"miner-8","network_difficulty":1000008,"ntime":1700001317,"order_key":"08:miner-8","p2mr_program_hex":"2525252525252525252525252525252525252525252525252525252525252525","share_difficulty":2,"share_id":"share-1317","share_seq":1317,"template_height":800041},{"accepted_at_ms":89980,"credit_policy":null,"job_id":"job-1391","job_issued_at_ms":89973,"miner_id":"miner-5","network_difficulty":1000014,"ntime":1700001391,"order_key":"05:miner-5","p2mr_program_hex":"6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f","share_difficulty":3,"share_id":"share-1391","share_seq":1391,"template_height":800043},{"accepted_at_ms":89991,"credit_policy":null,"job_id":"job-69","job_issued_at_ms":89973,"miner_id":"miner-3","network_difficulty":1000001,"ntime":1700000069,"order_key":"03:miner-3","p2mr_program_hex":"4545454545454545454545454545454545454545454545454545454545454545","share_difficulty":19,"share_id":"share-69","share_seq":69,"template_height":800002},{"accepted_at_ms":89982,"credit_policy":null,"job_id":"job-1129","job_issued_at_ms":89995,"miner_id":"miner-7","network_difficulty":1000007,"ntime":1700001129,"order_key":"07:miner-7","p2mr_program_hex":"6969696969696969696969696969696969696969696969696969696969696969","share_difficulty":7,"share_id":"share-1129","share_seq":1129,"template_height":800035},{"accepted_at_ms":89984,"credit_policy":null,"job_id":"job-269","job_issued_at_ms":89974,"miner_id":"miner-5","network_difficulty":1000014,"ntime":1700000269,"order_key":"05:miner-5","p2mr_program_hex":"0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d","share_difficulty":2,"share_id":"share-269","share_seq":269,"template_height":800008},{"accepted_at_ms":89986,"credit_policy":null,"job_id":"job-686","job_issued_at_ms":89991,"miner_id":"miner-4","network_difficulty":1000006,"ntime":1700000686,"order_key":"04:miner-4","p2mr_program_hex":"aeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeae","share_difficulty":19,"share_id":"share-686","share_seq":686,"template_height":800021},{"accepted_at_ms":89992,"credit_policy":null,"job_id":"job-1454","job_issued_at_ms":89983,"miner_id":"miner-2","network_difficulty":1000009,"ntime":1700001454,"order_key":"02:miner-2","p2mr_program_hex":"aeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeae","share_difficulty":7,"share_id":"share-1454","share_seq":1454,"template_height":800045},{"accepted_at_ms":89986,"credit_policy":null,"job_id":"job-1134","job_issued_at_ms":89996,"miner_id":"miner-1","network_difficulty":1000012,"ntime":1700001134,"order_key":"01:miner-1","p2mr_program_hex":"6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e","share_difficulty":2,"share_id":"share-1134","share_seq":1134,"template_height":800035},{"accepted_at_ms":90000,"credit_policy":"stale-grace","job_id":"job-1068","job_issued_at_ms":89970,"miner_id":"miner-1","network_difficulty":1000014,"ntime":1700001068,"order_key":"01:miner-1","p2mr_program_hex":"2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c","share_difficulty":1,"share_id":"share-1068","share_seq":1068,"template_height":800033},{"accepted_at_ms":89980,"credit_policy":"stale-grace","job_id":"job-1153","job_issued_at_ms":89987,"miner_id":"miner-9","network_difficulty":1000014,"ntime":1700001153,"order_key":"09:miner-9","p2mr_program_hex":"8181818181818181818181818181818181818181818181818181818181818181","share_difficulty":19,"share_id":"share-1153","share_seq":1153,"template_height":800036},{"accepted_at_ms":89974,"credit_policy":null,"job_id":"job-1354","job_issued_at_ms":89978,"miner_id":"miner-1","network_difficulty":1000011,"ntime":1700001354,"order_key":"01:miner-1","p2mr_program_hex":"4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a","share_difficulty":19,"share_id":"share-1354","share_seq":1354,"template_height":800042},{"accepted_at_ms":89992,"credit_policy":null,"job_id":"job-343","job_issued_at_ms":89993,"miner_id":"miner-2","network_difficulty":1000003,"ntime":1700000343,"order_key":"02:miner-2","p2mr_program_hex":"5757575757575757575757575757575757575757575757575757575757575757","share_difficulty":19,"share_id":"share-343","share_seq":343,"template_height":800010},{"accepted_at_ms":89988,"credit_policy":null,"job_id":"job-532","job_issued_at_ms":89990,"miner_id":"miner-4","network_difficulty":1000005,"ntime":1700000532,"order_key":"04:miner-4","p2mr_program_hex":"1414141414141414141414141414141414141414141414141414141414141414","share_difficulty":1,"share_id":"share-532","share_seq":532,"template_height":800016},{"accepted_at_ms":89978,"credit_policy":null,"job_id":"job-1181","job_issued_at_ms":89994,"miner_id":"miner-4","network_difficulty":1000008,"ntime":1700001181,"order_key":"04:miner-4","p2mr_program_hex":"9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d","share_difficulty":1,"share_id":"share-1181","share_seq":1181,"template_height":800036},{"accepted_at_ms":89986,"credit_policy":null,"job_id":"job-1050","job_issued_at_ms":89999,"miner_id":"miner-5","network_difficulty":1000013,"ntime":1700001050,"order_key":"05:miner-5","p2mr_program_hex":"1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a","share_difficulty":2,"share_id":"share-1050","share_seq":1050,"template_height":800032},{"accepted_at_ms":90000,"credit_policy":"stale-grace","job_id":"job-847","job_issued_at_ms":89981,"miner_id":"miner-0","network_difficulty":1000014,"ntime":1700000847,"order_key":"00:miner-0","p2mr_program_hex":"4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f","share_difficulty":1,"share_id":"share-847","share_seq":847,"template_height":800026},{"accepted_at_ms":89990,"credit_policy":null,"job_id":"job-539","job_issued_at_ms":89978,"miner_id":"miner-0","network_difficulty":1000012,"ntime":1700000539,"order_key":"00:miner-0","p2mr_program_hex":"1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b","share_difficulty":19,"share_id":"share-539","share_seq":539,"template_height":800016},{"accepted_at_ms":90000,"credit_policy":"stale-grace","job_id":"job-1721","job_issued_at_ms":89982,"miner_id":"miner-5","network_difficulty":1000004,"ntime":1700001721,"order_key":"05:miner-5","p2mr_program_hex":"b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9","share_difficulty":2,"share_id":"share-1721","share_seq":1721,"template_height":800053},{"accepted_at_ms":89998,"credit_policy":null,"job_id":"job-137","job_issued_at_ms":89995,"miner_id":"miner-5","network_difficulty":1000001,"ntime":1700000137,"order_key":"05:miner-5","p2mr_program_hex":"8989898989898989898989898989898989898989898989898989898989898989","share_difficulty":1,"share_id":"share-137","share_seq":137,"template_height":800004},{"accepted_at_ms":89990,"credit_policy":"stale-grace","job_id":"job-551","job_issued_at_ms":89979,"miner_id":"miner-1","network_difficulty":1000007,"ntime":1700000551,"order_key":"01:miner-1","p2mr_program_hex":"2727272727272727272727272727272727272727272727272727272727272727","share_difficulty":2,"share_id":"share-551","share_seq":551,"template_height":800017},{"accepted_at_ms":89993,"credit_policy":"stale-grace","job_id":"job-1239","job_issued_at_ms":89989,"miner_id":"miner-7","network_difficulty":1000015,"ntime":1700001239,"order_key":"07:miner-7","p2mr_program_hex":"d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7","share_difficulty":19,"share_id":"share-1239","share_seq":1239,"template_height":800038},{"accepted_at_ms":89971,"credit_policy":null,"job_id":"job-971","job_issued_at_ms":89996,"miner_id":"miner-3","network_difficulty":1000002,"ntime":1700000971,"order_key":"03:miner-3","p2mr_program_hex":"cbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcb","share_difficulty":3,"share_id":"share-971","share_seq":971,"template_height":800030},{"accepted_at_ms":90000,"credit_policy":null,"job_id":"job-439","job_issued_at_ms":89989,"miner_id":"miner-10","network_difficulty":1000014,"ntime":1700000439,"order_key":"10:miner-10","p2mr_program_hex":"b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7","share_difficulty":3,"share_id":"share-439","share_seq":439,"template_height":800013},{"accepted_at_ms":89995,"credit_policy":"stale-grace","job_id":"job-604","job_issued_at_ms":89994,"miner_id":"miner-10","network_difficulty":1000009,"ntime":1700000604,"order_key":"10:miner-10","p2mr_program_hex":"5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c","share_difficulty":1,"share_id":"share-604","share_seq":604,"template_height":800018},{"accepted_at_ms":89987,"credit_policy":null,"job_id":"job-1218","job_issued_at_ms":89997,"miner_id":"miner-8","network_difficulty":1000011,"ntime":1700001218,"order_key":"08:miner-8","p2mr_program_hex":"c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2","share_difficulty":3,"share_id":"share-1218","share_seq":1218,"template_height":800038},{"accepted_at_ms":89987,"credit_policy":"stale-grace","job_id":"job-1191","job_issued_at_ms":89998,"miner_id":"miner-3","network_difficulty":1000001,"ntime":1700001191,"order_key":"03:miner-3","p2mr_program_hex":"a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7","share_difficulty":2,"share_id":"share-1191","share_seq":1191,"template_height":800037},{"accepted_at_ms":89996,"credit_policy":null,"job_id":"job-844","job_issued_at_ms":89972,"miner_id":"miner-8","network_difficulty":1000011,"ntime":1700000844,"order_key":"08:miner-8","p2mr_program_hex":"4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c","share_difficulty":3,"share_id":"share-844","share_seq":844,"template_height":800026},{"accepted_at_ms":89972,"credit_policy":"stale-grace","job_id":"job-1248","job_issued_at_ms":89996,"miner_id":"miner-5","network_difficulty":1000007,"ntime":1700001248,"order_key":"05:miner-5","p2mr_program_hex":"e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0","share_difficulty":2,"share_id":"share-1248","share_seq":1248,"template_height":800039},{"accepted_at_ms":89997,"credit_policy":null,"job_id":"job-576","job_issued_at_ms":89990,"miner_id":"miner-4","network_difficulty":1000015,"ntime":1700000576,"order_key":"04:miner-4","p2mr_program_hex":"4040404040404040404040404040404040404040404040404040404040404040","share_difficulty":7,"share_id":"share-576","share_seq":576,"template_height":800018},{"accepted_at_ms":89991,"credit_policy":null,"job_id":"job-701","job_issued_at_ms":89990,"miner_id":"miner-8","network_difficulty":1000004,"ntime":1700000701,"order_key":"08:miner-8","p2mr_program_hex":"bdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbd","share_difficulty":1,"share_id":"share-701","share_seq":701,"template_height":800021},{"accepted_at_ms":90000,"credit_policy":null,"job_id":"job-133","job_issued_at_ms":89989,"miner_id":"miner-1","network_difficulty":1000014,"ntime":1700000133,"order_key":"01:miner-1","p2mr_program_hex":"8585858585858585858585858585858585858585858585858585858585858585","share_difficulty":19,"share_id":"share-133","share_seq":133,"template_height":800004},{"accepted_at_ms":89976,"credit_policy":null,"job_id":"job-460","job_issued_at_ms":90000,"miner_id":"miner-9","network_difficulty":1000001,"ntime":1700000460,"order_key":"09:miner-9","p2mr_program_hex":"cccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccc","share_difficulty":1,"share_id":"share-460","share_seq":460,"template_height":800014},{"accepted_at_ms":89973,"credit_policy":null,"job_id":"job-837","job_issued_at_ms":89986,"miner_id":"miner-1","network_difficulty":1000004,"ntime":1700000837,"order_key":"01:miner-1","p2mr_program_hex":"4545454545454545454545454545454545454545454545454545454545454545","share_difficulty":7,"share_id":"share-837","share_seq":837,"template_height":800026},{"accepted_at_ms":89972,"credit_policy":null,"job_id":"job-934","job_issued_at_ms":89990,"miner_id":"miner-10","network_difficulty":1000016,"ntime":1700000934,"order_key":"10:miner-10","p2mr_program_hex":"a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6","share_difficulty":19,"share_id":"share-934","share_seq":934,"template_height":800029},{"accepted_at_ms":89989,"credit_policy":"stale-grace","job_id":"job-28","job_issued_at_ms":89995,"miner_id":"miner-6","network_difficulty":1000011,"ntime":1700000028,"order_key":"06:miner-6","p2mr_program_hex":"1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c","share_difficulty":3,"share_id":"share-28","share_seq":28,"template_height":800000},{"accepted_at_ms":89975,"credit_policy":"stale-grace","job_id":"job-1180","job_issued_at_ms":89972,"miner_id":"miner-3","network_difficulty":1000007,"ntime":1700001180,"order_key":"03:miner-3","p2mr_program_hex":"9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c","share_difficulty":1,"share_id":"share-1180","share_seq":1180,"template_height":800036},{"accepted_at_ms":89984,"credit_policy":"stale-grace","job_id":"job-1211","job_issued_at_ms":89978,"miner_id":"miner-1","network_difficulty":1000004,"ntime":1700001211,"order_key":"01:miner-1","p2mr_program_hex":"bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb","share_difficulty":19,"share_id":"share-1211","share_seq":1211,"template_height":800037},{"accepted_at_ms":89972,"credit_policy":null,"job_id":"job-1537","job_issued_at_ms":89991,"miner_id":"miner-8","network_difficulty":1000007,"ntime":1700001537,"order_key":"08:miner-8","p2mr_program_hex":"0101010101010101010101010101010101010101010101010101010101010101","share_difficulty":19,"share_id":"share-1537","share_seq":1537,"template_height":800048},{"accepted_at_ms":89992,"credit_policy":null,"job_id":"job-788","job_issued_at_ms":89985,"miner_id":"miner-7","network_difficulty":1000006,"ntime":1700000788,"order_key":"07:miner-7","p2mr_program_hex":"1414141414141414141414141414141414141414141414141414141414141414","share_difficulty":19,"share_id":"share-788","share_seq":788,"template_height":800024},{"accepted_at_ms":89970,"credit_policy":null,"job_id":"job-1493","job_issued_at_ms":89973,"miner_id":"miner-8","network_difficulty":1000014,"ntime":1700001493,"order_key":"08:miner-8","p2mr_program_hex":"d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5","share_difficulty":7,"share_id":"share-1493","share_seq":1493,"template_height":800046},{"accepted_at_ms":89993,"credit_policy":null,"job_id":"job-1473","job_issued_at_ms":89973,"miner_id":"miner-10","network_difficulty":1000011,"ntime":1700001473,"order_key":"10:miner-10","p2mr_program_hex":"c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1","share_difficulty":2,"share_id":"share-1473","share_seq":1473,"template_height":800046},{"accepted_at_ms":89981,"credit_policy":null,"job_id":"job-1538","job_issued_at_ms":89997,"miner_id":"miner-9","network_difficulty":1000008,"ntime":1700001538,"order_key":"09:miner-9","p2mr_program_hex":"0202020202020202020202020202020202020202020202020202020202020202","share_difficulty":7,"share_id":"share-1538","share_seq":1538,"template_height":800048},{"accepted_at_ms":89979,"credit_policy":null,"job_id":"job-913","job_issued_at_ms":89978,"miner_id":"miner-0","network_difficulty":1000012,"ntime":1700000913,"order_key":"00:miner-0","p2mr_program_hex":"9191919191919191919191919191919191919191919191919191919191919191","share_difficulty":2,"share_id":"share-913","share_seq":913,"template_height":800028},{"accepted_at_ms":89985,"credit_policy":"stale-grace","job_id":"job-1055","job_issued_at_ms":89999,"miner_id":"miner-10","network_difficulty":1000001,"ntime":1700001055,"order_key":"10:miner-10","p2mr_program_hex":"1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f","share_difficulty":3,"share_id":"share-1055","share_seq":1055,"template_height":800032},{"accepted_at_ms":89992,"credit_policy":null,"job_id":"job-753","job_issued_at_ms":89982,"miner_id":"miner-5","network_difficulty":1000005,"ntime":1700000753,"order_key":"05:miner-5","p2mr_program_hex":"f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1","share_difficulty":19,"share_id":"share-753","share_seq":753,"template_height":800023},{"accepted_at_ms":89981,"credit_policy":"stale-grace","job_id":"job-378","job_issued_at_ms":89976,"miner_id":"miner-4","network_difficulty":1000004,"ntime":1700000378,"order_key":"04:miner-4","p2mr_program_hex":"7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a","share_difficulty":2,"share_id":"share-378","share_seq":378,"template_height":800011},{"accepted_at_ms":89995,"credit_policy":"stale-grace","job_id":"job-1082","job_issued_at_ms":89976,"miner_id":"miner-4","network_difficulty":1000011,"ntime":1700001082,"order_key":"04:miner-4","p2mr_program_hex":"3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a","share_difficulty":1,"share_id":"share-1082","share_seq":1082,"template_height":800033},{"accepted_at_ms":89978,"credit_policy":null,"job_id":"job-639","job_issued_at_ms":89985,"miner_id":"miner-1","network_difficulty":1000010,"ntime":1700000639,"order_key":"01:miner-1","p2mr_program_hex":"7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f","share_difficulty":2,"share_id":"share-639","share_seq":639,"template_height":800019},{"accepted_at_ms":89972,"credit_policy":null,"job_id":"job-1458","job_issued_at_ms":89989,"miner_id":"miner-6","network_difficulty":1000013,"ntime":1700001458,"order_key":"06:miner-6","p2mr_program_hex":"b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2","share_difficulty":7,"share_id":"share-1458","share_seq":1458,"template_height":800045},{"accepted_at_ms":89976,"credit_policy":null,"job_id":"job-225","job_issued_at_ms":89972,"miner_id":"miner-5","network_difficulty":1000004,"ntime":1700000225,"order_key":"05:miner-5","p2mr_program_hex":"e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1","share_difficulty":19,"share_id":"share-225","share_seq":225,"template_height":800007},{"accepted_at_ms":89983,"credit_policy":null,"job_id":"job-389","job_issued_at_ms":89984,"miner_id":"miner-4","network_difficulty":1000015,"ntime":1700000389,"order_key":"04:miner-4","p2mr_program_hex":"8585858585858585858585858585858585858585858585858585858585858585","share_difficulty":1,"share_id":"share-389","share_seq":389,"template_height":800012},{"accepted_at_ms":89989,"credit_policy":null,"job_id":"job-169","job_issued_at_ms":89990,"miner_id":"miner-4","network_difficulty":1000016,"ntime":1700000169,"order_key":"04:miner-4","p2mr_program_hex":"a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9","share_difficulty":1,"share_id":"share-169","share_seq":169,"template_height":800005},{"accepted_at_ms":89993,"credit_policy":null,"job_id":"job-606","job_issued_at_ms":89977,"miner_id":"miner-1","network_difficulty":1000011,"ntime":1700000606,"order_key":"01:miner-1","p2mr_program_hex":"5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e","share_difficulty":19,"share_id":"share-606","share_seq":606,"template_height":800018},{"accepted_at_ms":89978,"credit_policy":null,"job_id":"job-456","job_issued_at_ms":89970,"miner_id":"miner-5","network_difficulty":1000014,"ntime":1700000456,"order_key":"05:miner-5","p2mr_program_hex":"c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8","share_difficulty":19,"share_id":"share-456","share_seq":456,"template_height":800014},{"accepted_at_ms":90000,"credit_policy":null,"job_id":"job-662","job_issued_at_ms":89976,"miner_id":"miner-2","network_difficulty":1000016,"ntime":1700000662,"order_key":"02:miner-2","p2mr_program_hex":"9696969696969696969696969696969696969696969696969696969696969696","share_difficulty":3,"share_id":"share-662","share_seq":662,"template_height":800020},{"accepted_at_ms":89990,"credit_policy":null,"job_id":"job-1138","job_issued_at_ms":89980,"miner_id":"miner-5","network_difficulty":1000016,"ntime":1700001138,"order_key":"05:miner-5","p2mr_program_hex":"7272727272727272727272727272727272727272727272727272727272727272","share_difficulty":1,"share_id":"share-1138","share_seq":1138,"template_height":800035},{"accepted_at_ms":89983,"credit_policy":null,"job_id":"job-1337","job_issued_at_ms":89991,"miner_id":"miner-6","network_difficulty":1000011,"ntime":1700001337,"order_key":"06:miner-6","p2mr_program_hex":"3939393939393939393939393939393939393939393939393939393939393939","share_difficulty":3,"share_id":"share-1337","share_seq":1337,"template_height":800041},{"accepted_at_ms":89975,"credit_policy":null,"job_id":"job-738","job_issued_at_ms":89975,"miner_id":"miner-1","network_difficulty":1000007,"ntime":1700000738,"order_key":"01:miner-1","p2mr_program_hex":"e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2","share_difficulty":19,"share_id":"share-738","share_seq":738,"template_height":800023},{"accepted_at_ms":89983,"credit_policy":null,"job_id":"job-1624","job_issued_at_ms":89978,"miner_id":"miner-7","network_difficulty":1000009,"ntime":1700001624,"order_key":"07:miner-7","p2mr_program_hex":"5858585858585858585858585858585858585858585858585858585858585858","share_difficulty":3,"share_id":"share-1624","share_seq":1624,"template_height":800050},{"accepted_at_ms":90000,"credit_policy":null,"job_id":"job-1759","job_issued_at_ms":89989,"miner_id":"miner-10","network_difficulty":1000008,"ntime":1700001759,"order_key":"10:miner-10","p2mr_program_hex":"dfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdf","share_difficulty":19,"share_id":"share-1759","share_seq":1759,"template_height":800054},{"accepted_at_ms":89987,"credit_policy":null,"job_id":"job-134","job_issued_at_ms":89977,"miner_id":"miner-2","network_difficulty":1000015,"ntime":1700000134,"order_key":"02:miner-2","p2mr_program_hex":"8686868686868686868686868686868686868686868686868686868686868686","share_difficulty":19,"share_id":"share-134","share_seq":134,"template_height":800004},{"accepted_at_ms":89991,"credit_policy":null,"job_id":"job-972","job_issued_at_ms":89974,"miner_id":"miner-4","network_difficulty":1000003,"ntime":1700000972,"order_key":"04:miner-4","p2mr_program_hex":"cccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccc","share_difficulty":1,"share_id":"share-972","share_seq":972,"template_height":800030},{"accepted_at_ms":89988,"credit_policy":null,"job_id":"job-6","job_issued_at_ms":89985,"miner_id":"miner-6","network_difficulty":1000006,"ntime":1700000006,"order_key":"06:miner-6","p2mr_program_hex":"0606060606060606060606060606060606060606060606060606060606060606","share_difficulty":1,"share_id":"share-6","share_seq":6,"template_height":800000},{"accepted_at_ms":89994,"credit_policy":"stale-grace","job_id":"job-1675","job_issued_at_ms":89990,"miner_id":"miner-3","network_difficulty":1000009,"ntime":1700001675,"order_key":"03:miner-3","p2mr_program_hex":"8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b","share_difficulty":2,"share_id":"share-1675","share_seq":1675,"template_height":800052},{"accepted_at_ms":89994,"credit_policy":"stale-grace","job_id":"job-805","job_issued_at_ms":89992,"miner_id":"miner-2","network_difficulty":1000006,"ntime":1700000805,"order_key":"02:miner-2","p2mr_program_hex":"2525252525252525252525252525252525252525252525252525252525252525","share_difficulty":19,"share_id":"share-805","share_seq":805,"template_height":800025},{"accepted_at_ms":89975,"credit_policy":null,"job_id":"job-1119","job_issued_at_ms":89976,"miner_id":"miner-8","network_difficulty":1000014,"ntime":1700001119,"order_key":"08:miner-8","p2mr_program_hex":"5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f","share_difficulty":7,"share_id":"share-1119","share_seq":1119,"template_height":800034},{"accepted_at_ms":89994,"credit_policy":"stale-grace","job_id":"job-1612","job_issued_at_ms":89975,"miner_id":"miner-6","network_difficulty":1000014,"ntime":1700001612,"order_key":"06:miner-6","p2mr_program_hex":"4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c","share_difficulty":2,"share_id":"share-1612","share_seq":1612,"template_height":800050},{"accepted_at_ms":89991,"credit_policy":null,"job_id":"job-1122","job_issued_at_ms":89992,"miner_id":"miner-0","network_difficulty":1000000,"ntime":1700001122,"order_key":"00:miner-0","p2mr_program_hex":"6262626262626262626262626262626262626262626262626262626262626262","share_difficulty":19,"share_id":"share-1122","share_seq":1122,"template_height":800035},{"accepted_at_ms":89998,"credit_policy":null,"job_id":"job-1228","job_issued_at_ms":89971,"miner_id":"miner-7","network_difficulty":1000004,"ntime":1700001228,"order_key":"07:miner-7","p2mr_program_hex":"cccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccc","share_difficulty":1,"share_id":"share-1228","share_seq":1228,"template_height":800038},{"accepted_at_ms":89981,"credit_policy":null,"job_id":"job-118","job_issued_at_ms":89995,"miner_id":"miner-8","network_difficulty":1000016,"ntime":1700000118,"order_key":"08:miner-8","p2mr_program_hex":"7676767676767676767676767676767676767676767676767676767676767676","share_difficulty":2,"share_id":"share-118","share_seq":118,"template_height":800003},{"accepted_at_ms":89999,"credit_policy":null,"job_id":"job-1093","job_issued_at_ms":89974,"miner_id":"miner-4","network_difficulty":1000005,"ntime":1700001093,"order_key":"04:miner-4","p2mr_program_hex":"4545454545454545454545454545454545454545454545454545454545454545","share_difficulty":1,"share_id":"share-1093","share_seq":1093,"template_height":800034},{"accepted_at_ms":89971,"credit_policy":null,"job_id":"job-128","job_issued_at_ms":89991,"miner_id":"miner-7","network_difficulty":1000009,"ntime":1700000128,"order_key":"07:miner-7","p2mr_program_hex":"8080808080808080808080808080808080808080808080808080808080808080","share_difficulty":3,"share_id":"share-128","share_seq":128,"template_height":800004},{"accepted_at_ms":89989,"credit_policy":null,"job_id":"job-1506","job_issued_at_ms":90000,"miner_id":"miner-10","network_difficulty":1000010,"ntime":1700001506,"order_key":"10:miner-10","p2mr_program_hex":"e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2","share_difficulty":19,"share_id":"share-1506","share_seq":1506,"template_height":800047},{"accepted_at_ms":89983,"credit_policy":"stale-grace","job_id":"job-779","job_issued_at_ms":89984,"miner_id":"miner-9","network_difficulty":1000014,"ntime":1700000779,"order_key":"09:miner-9","p2mr_program_hex":"0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b","share_difficulty":3,"share_id":"share-779","share_seq":779,"template_height":800024},{"accepted_at_ms":89970,"credit_policy":null,"job_id":"job-1541","job_issued_at_ms":89991,"miner_id":"miner-1","network_difficulty":1000011,"ntime":1700001541,"order_key":"01:miner-1","p2mr_program_hex":"0505050505050505050505050505050505050505050505050505050505050505","share_difficulty":2,"share_id":"share-1541","share_seq":1541,"template_height":800048},{"accepted_at_ms":90000,"credit_policy":null,"job_id":"job-749","job_issued_at_ms":89980,"miner_id":"miner-1","network_difficulty":1000001,"ntime":1700000749,"order_key":"01:miner-1","p2mr_program_hex":"edededededededededededededededededededededededededededededededed","share_difficulty":7,"share_id":"share-749","share_seq":749,"template_height":800023},{"accepted_at_ms":89970,"credit_policy":"stale-grace","job_id":"job-1045","job_issued_at_ms":89975,"miner_id":"miner-0","network_difficulty":1000008,"ntime":1700001045,"order_key":"00:miner-0","p2mr_program_hex":"1515151515151515151515151515151515151515151515151515151515151515","share_difficulty":3,"share_id":"share-1045","share_seq":1045,"template_height":800032},{"accepted_at_ms":89983,"credit_policy":null,"job_id":"job-667","job_issued_at_ms":89990,"miner_id":"miner-7","network_difficulty":1000004,"ntime":1700000667,"order_key":"07:miner-7","p2mr_program_hex":"9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b","share_difficulty":1,"share_id":"share-667","share_seq":667,"template_height":800020},{"accepted_at_ms":89999,"credit_policy":null,"job_id":"job-1406","job_issued_at_ms":89978,"miner_id":"miner-9","network_difficulty":1000012,"ntime":1700001406,"order_key":"09:miner-9","p2mr_program_hex":"7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e","share_difficulty":3,"share_id":"share-1406","share_seq":1406,"template_height":800043},{"accepted_at_ms":89974,"credit_policy":null,"job_id":"job-135","job_issued_at_ms":89991,"miner_id":"miner-3","network_difficulty":1000016,"ntime":1700000135,"order_key":"03:miner-3","p2mr_program_hex":"8787878787878787878787878787878787878787878787878787878787878787","share_difficulty":2,"share_id":"share-135","share_seq":135,"template_height":800004},{"accepted_at_ms":89971,"credit_policy":null,"job_id":"job-205","job_issued_at_ms":89984,"miner_id":"miner-7","network_difficulty":1000001,"ntime":1700000205,"order_key":"07:miner-7","p2mr_program_hex":"cdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcd","share_difficulty":2,"share_id":"share-205","share_seq":205,"template_height":800006},{"accepted_at_ms":89980,"credit_policy":null,"job_id":"job-429","job_issued_at_ms":89986,"miner_id":"miner-0","network_difficulty":1000004,"ntime":1700000429,"order_key":"00:miner-0","p2mr_program_hex":"adadadadadadadadadadadadadadadadadadadadadadadadadadadadadadadad","share_difficulty":19,"share_id":"share-429","share_seq":429,"template_height":800013},{"accepted_at_ms":89998,"credit_policy":"stale-grace","job_id":"job-4","job_issued_at_ms":89975,"miner_id":"miner-4","network_difficulty":1000004,"ntime":1700000004,"order_key":"04:miner-4","p2mr_program_hex":"0404040404040404040404040404040404040404040404040404040404040404","share_difficulty":19,"share_id":"share-4","share_seq":4,"template_height":800000},{"accepted_at_ms":89986,"credit_policy":"stale-grace","job_id":"job-471","job_issued_at_ms":89980,"miner_id":"miner-9","network_difficulty":1000012,"ntime":1700000471,"order_key":"09:miner-9","p2mr_program_hex":"d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7","share_difficulty":19,"share_id":"share-471","share_seq":471,"template_height":800014},{"accepted_at_ms":89980,"credit_policy":"stale-grace","job_id":"job-743","job_issued_at_ms":89992,"miner_id":"miner-6","network_difficulty":1000012,"ntime":1700000743,"order_key":"06:miner-6","p2mr_program_hex":"e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7","share_difficulty":2,"share_id":"share-743","share_seq":743,"template_height":800023},{"accepted_at_ms":89980,"credit_policy":null,"job_id":"job-1501","job_issued_at_ms":89972,"miner_id":"miner-5","network_difficulty":1000005,"ntime":1700001501,"order_key":"05:miner-5","p2mr_program_hex":"dddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddd","share_difficulty":1,"share_id":"share-1501","share_seq":1501,"template_height":800046},{"accepted_at_ms":89998,"credit_policy":"stale-grace","job_id":"job-1256","job_issued_at_ms":89973,"miner_id":"miner-2","network_difficulty":1000015,"ntime":1700001256,"order_key":"02:miner-2","p2mr_program_hex":"e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8","share_difficulty":1,"share_id":"share-1256","share_seq":1256,"template_height":800039},{"accepted_at_ms":89994,"credit_policy":null,"job_id":"job-1193","job_issued_at_ms":89972,"miner_id":"miner-5","network_difficulty":1000003,"ntime":1700001193,"order_key":"05:miner-5","p2mr_program_hex":"a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9","share_difficulty":19,"share_id":"share-1193","share_seq":1193,"template_height":800037},{"accepted_at_ms":89979,"credit_policy":null,"job_id":"job-1273","job_issued_at_ms":89994,"miner_id":"miner-8","network_difficulty":1000015,"ntime":1700001273,"order_key":"08:miner-8","p2mr_program_hex":"f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9","share_difficulty":7,"share_id":"share-1273","share_seq":1273,"template_height":800039},{"accepted_at_ms":90000,"credit_policy":null,"job_id":"job-774","job_issued_at_ms":89974,"miner_id":"miner-4","network_difficulty":1000009,"ntime":1700000774,"order_key":"04:miner-4","p2mr_program_hex":"0606060606060606060606060606060606060606060606060606060606060606","share_difficulty":1,"share_id":"share-774","share_seq":774,"template_height":800024},{"accepted_at_ms":89985,"credit_policy":null,"job_id":"job-191","job_issued_at_ms":89984,"miner_id":"miner-4","network_difficulty":1000004,"ntime":1700000191,"order_key":"04:miner-4","p2mr_program_hex":"bfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbf","share_difficulty":19,"share_id":"share-191","share_seq":191,"template_height":800005},{"accepted_at_ms":89979,"credit_policy":null,"job_id":"job-1795","job_issued_at_ms":89974,"miner_id":"miner-2","network_difficulty":1000010,"ntime":1700001795,"order_key":"02:miner-2","p2mr_program_hex":"0303030303030303030303030303030303030303030303030303030303030303","share_difficulty":7,"share_id":"share-1795","share_seq":1795,"template_height":800056},{"accepted_at_ms":89983,"credit_policy":null,"job_id":"job-1173","job_issued_at_ms":89979,"miner_id":"miner-7","network_difficulty":1000000,"ntime":1700001173,"order_key":"07:miner-7","p2mr_program_hex":"9595959595959595959595959595959595959595959595959595959595959595","share_difficulty":19,"share_id":"share-1173","share_seq":1173,"template_height":800036},{"accepted_at_ms":89971,"credit_policy":null,"job_id":"job-1706","job_issued_at_ms":89990,"miner_id":"miner-1","network_difficulty":1000006,"ntime":1700001706,"order_key":"01:miner-1","p2mr_program_hex":"aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa","share_difficulty":3,"share_id":"share-1706","share_seq":1706,"template_height":800053},{"accepted_at_ms":89974,"credit_policy":"stale-grace","job_id":"job-603","job_issued_at_ms":89998,"miner_id":"miner-9","network_difficulty":1000008,"ntime":1700000603,"order_key":"09:miner-9","p2mr_program_hex":"5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b","share_difficulty":19,"share_id":"share-603","share_seq":603,"template_height":800018},{"accepted_at_ms":89986,"credit_policy":"stale-grace","job_id":"job-1579","job_issued_at_ms":89975,"miner_id":"miner-6","network_difficulty":1000015,"ntime":1700001579,"order_key":"06:miner-6","p2mr_program_hex":"2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b","share_difficulty":7,"share_id":"share-1579","share_seq":1579,"template_height":800049},{"accepted_at_ms":89974,"credit_policy":null,"job_id":"job-215","job_issued_at_ms":89975,"miner_id":"miner-6","network_difficulty":1000011,"ntime":1700000215,"order_key":"06:miner-6","p2mr_program_hex":"d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7","share_difficulty":1,"share_id":"share-215","share_seq":215,"template_height":800006},{"accepted_at_ms":89975,"credit_policy":null,"job_id":"job-1689","job_issued_at_ms":89995,"miner_id":"miner-6","network_difficulty":1000006,"ntime":1700001689,"order_key":"06:miner-6","p2mr_program_hex":"9999999999999999999999999999999999999999999999999999999999999999","share_difficulty":7,"share_id":"share-1689","share_seq":1689,"template_height":800052},{"accepted_at_ms":89989,"credit_policy":null,"job_id":"job-360","job_issued_at_ms":89990,"miner_id":"miner-8","network_difficulty":1000003,"ntime":1700000360,"order_key":"08:miner-8","p2mr_program_hex":"6868686868686868686868686868686868686868686868686868686868686868","share_difficulty":3,"share_id":"share-360","share_seq":360,"template_height":800011},{"accepted_at_ms":89976,"credit_policy":null,"job_id":"job-434","job_issued_at_ms":89990,"miner_id":"miner-5","network_difficulty":1000009,"ntime":1700000434,"order_key":"05:miner-5","p2mr_program_hex":"b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2","share_difficulty":1,"share_id":"share-434","share_seq":434,"template_height":800013},{"accepted_at_ms":89981,"credit_policy":null,"job_id":"job-260","job_issued_at_ms":89977,"miner_id":"miner-7","network_difficulty":1000005,"ntime":1700000260,"order_key":"07:miner-7","p2mr_program_hex":"0404040404040404040404040404040404040404040404040404040404040404","share_difficulty":3,"share_id":"share-260","share_seq":260,"template_height":800008},{"accepted_at_ms":89990,"credit_policy":"stale-grace","job_id":"job-36","job_issued_at_ms":89997,"miner_id":"miner-3","network_difficulty":1000002,"ntime":1700000036,"order_key":"03:miner-3","p2mr_program_hex":"2424242424242424242424242424242424242424242424242424242424242424","share_difficulty":7,"share_id":"share-36","share_seq":36,"template_height":800001},{"accepted_at_ms":89994,"credit_policy":null,"job_id":"job-1399","job_issued_at_ms":89988,"miner_id":"miner-2","network_difficulty":1000005,"ntime":1700001399,"order_key":"02:miner-2","p2mr_program_hex":"7777777777777777777777777777777777777777777777777777777777777777","share_difficulty":2,"share_id":"share-1399","share_seq":1399,"template_height":800043},{"accepted_at_ms":89989,"credit_policy":null,"job_id":"job-1574","job_issued_at_ms":89987,"miner_id":"miner-1","network_difficulty":1000010,"ntime":1700001574,"order_key":"01:miner-1","p2mr_program_hex":"2626262626262626262626262626262626262626262626262626262626262626","share_difficulty":2,"share_id":"share-1574","share_seq":1574,"template_height":800049},{"accepted_at_ms":89984,"credit_policy":null,"job_id":"job-1747","job_issued_at_ms":89988,"miner_id":"miner-9","network_difficulty":1000013,"ntime":1700001747,"order_key":"09:miner-9","p2mr_program_hex":"d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3","share_difficulty":19,"share_id":"share-1747","share_seq":1747,"template_height":800054},{"accepted_at_ms":89987,"credit_policy":null,"job_id":"job-1053","job_issued_at_ms":89997,"miner_id":"miner-8","network_difficulty":1000016,"ntime":1700001053,"order_key":"08:miner-8","p2mr_program_hex":"1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d","share_difficulty":2,"share_id":"share-1053","share_seq":1053,"template_height":800032},{"accepted_at_ms":89997,"credit_policy":null,"job_id":"job-1346","job_issued_at_ms":89988,"miner_id":"miner-4","network_difficulty":1000003,"ntime":1700001346,"order_key":"04:miner-4","p2mr_program_hex":"4242424242424242424242424242424242424242424242424242424242424242","share_difficulty":3,"share_id":"share-1346","share_seq":1346,"template_height":800042},{"accepted_at_ms":89980,"credit_policy":null,"job_id":"job-1452","job_issued_at_ms":89975,"miner_id":"miner-0","network_difficulty":1000007,"ntime":1700001452,"order_key":"00:miner-0","p2mr_program_hex":"acacacacacacacacacacacacacacacacacacacacacacacacacacacacacacacac","share_difficulty":3,"share_id":"share-1452","share_seq":1452,"template_height":800045},{"accepted_at_ms":89981,"credit_policy":null,"job_id":"job-1233","job_issued_at_ms":90000,"miner_id":"miner-1","network_difficulty":1000009,"ntime":1700001233,"order_key":"01:miner-1","p2mr_program_hex":"d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1","share_difficulty":7,"share_id":"share-1233","share_seq":1233,"template_height":800038},{"accepted_at_ms":89994,"credit_policy":null,"job_id":"job-648","job_issued_at_ms":89988,"miner_id":"miner-10","network_difficulty":1000002,"ntime":1700000648,"order_key":"10:miner-10","p2mr_program_hex":"8888888888888888888888888888888888888888888888888888888888888888","share_difficulty":1,"share_id":"share-648","share_seq":648,"template_height":800020},{"accepted_at_ms":90000,"credit_policy":null,"job_id":"job-1292","job_issued_at_ms":89999,"miner_id":"miner-5","network_difficulty":1000000,"ntime":1700001292,"order_key":"05:miner-5","p2mr_program_hex":"0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c","share_difficulty":19,"share_id":"share-1292","share_seq":1292,"template_height":800040},{"accepted_at_ms":89995,"credit_policy":"stale-grace","job_id":"job-773","job_issued_at_ms":89997,"miner_id":"miner-3","network_difficulty":1000008,"ntime":1700000773,"order_key":"03:miner-3","p2mr_program_hex":"0505050505050505050505050505050505050505050505050505050505050505","share_difficulty":1,"share_id":"share-773","share_seq":773,"template_height":800024},{"accepted_at_ms":89989,"credit_policy":null,"job_id":"job-111","job_issued_at_ms":89970,"miner_id":"miner-1","network_difficulty":1000009,"ntime":1700000111,"order_key":"01:miner-1","p2mr_program_hex":"6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f","share_difficulty":3,"share_id":"share-111","share_seq":111,"template_height":800003},{"accepted_at_ms":89971,"credit_policy":null,"job_id":"job-547","job_issued_at_ms":89979,"miner_id":"miner-8","network_difficulty":1000003,"ntime":1700000547,"order_key":"08:miner-8","p2mr_program_hex":"2323232323232323232323232323232323232323232323232323232323232323","share_difficulty":1,"share_id":"share-547","share_seq":547,"template_height":800017},{"accepted_at_ms":89992,"credit_policy":null,"job_id":"job-901","job_issued_at_ms":90000,"miner_id":"miner-10","network_difficulty":1000000,"ntime":1700000901,"order_key":"10:miner-10","p2mr_program_hex":"8585858585858585858585858585858585858585858585858585858585858585","share_difficulty":3,"share_id":"share-901","share_seq":901,"template_height":800028},{"accepted_at_ms":89996,"credit_policy":null,"job_id":"job-852","job_issued_at_ms":89973,"miner_id":"miner-5","network_difficulty":1000002,"ntime":1700000852,"order_key":"05:miner-5","p2mr_program_hex":"5454545454545454545454545454545454545454545454545454545454545454","share_difficulty":2,"share_id":"share-852","share_seq":852,"template_height":800026},{"accepted_at_ms":89973,"credit_policy":null,"job_id":"job-854","job_issued_at_ms":89974,"miner_id":"miner-7","network_difficulty":1000004,"ntime":1700000854,"order_key":"07:miner-7","p2mr_program_hex":"5656565656565656565656565656565656565656565656565656565656565656","share_difficulty":3,"share_id":"share-854","share_seq":854,"template_height":800026},{"accepted_at_ms":89981,"credit_policy":"stale-grace","job_id":"job-435","job_issued_at_ms":89995,"miner_id":"miner-6","network_difficulty":1000010,"ntime":1700000435,"order_key":"06:miner-6","p2mr_program_hex":"b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3","share_difficulty":2,"share_id":"share-435","share_seq":435,"template_height":800013},{"accepted_at_ms":89992,"credit_policy":null,"job_id":"job-1474","job_issued_at_ms":89998,"miner_id":"miner-0","network_difficulty":1000012,"ntime":1700001474,"order_key":"00:miner-0","p2mr_program_hex":"c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2","share_difficulty":3,"share_id":"share-1474","share_seq":1474,"template_height":800046},{"accepted_at_ms":89990,"credit_policy":"stale-grace","job_id":"job-1387","job_issued_at_ms":89983,"miner_id":"miner-1","network_difficulty":1000010,"ntime":1700001387,"order_key":"01:miner-1","p2mr_program_hex":"6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b","share_difficulty":2,"share_id":"share-1387","share_seq":1387,"template_height":800043},{"accepted_at_ms":89997,"credit_policy":null,"job_id":"job-458","job_issued_at_ms":89984,"miner_id":"miner-7","network_difficulty":1000016,"ntime":1700000458,"order_key":"07:miner-7","p2mr_program_hex":"cacacacacacacacacacacacacacacacacacacacacacacacacacacacacacacaca","share_difficulty":19,"share_id":"share-458","share_seq":458,"template_height":800014},{"accepted_at_ms":89971,"credit_policy":null,"job_id":"job-1224","job_issued_at_ms":89999,"miner_id":"miner-3","network_difficulty":1000000,"ntime":1700001224,"order_key":"03:miner-3","p2mr_program_hex":"c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8","share_difficulty":19,"share_id":"share-1224","share_seq":1224,"template_height":800038},{"accepted_at_ms":89996,"credit_policy":null,"job_id":"job-43","job_issued_at_ms":89981,"miner_id":"miner-10","network_difficulty":1000009,"ntime":1700000043,"order_key":"10:miner-10","p2mr_program_hex":"2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b","share_difficulty":3,"share_id":"share-43","share_seq":43,"template_height":800001},{"accepted_at_ms":89988,"credit_policy":"stale-grace","job_id":"job-969","job_issued_at_ms":89979,"miner_id":"miner-1","network_difficulty":1000000,"ntime":1700000969,"order_key":"01:miner-1","p2mr_program_hex":"c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9","share_difficulty":19,"share_id":"share-969","share_seq":969,"template_height":800030},{"accepted_at_ms":89975,"credit_policy":null,"job_id":"job-727","job_issued_at_ms":89987,"miner_id":"miner-1","network_difficulty":1000013,"ntime":1700000727,"order_key":"01:miner-1","p2mr_program_hex":"d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7","share_difficulty":3,"share_id":"share-727","share_seq":727,"template_height":800022},{"accepted_at_ms":90000,"credit_policy":null,"job_id":"job-293","job_issued_at_ms":89997,"miner_id":"miner-7","network_difficulty":1000004,"ntime":1700000293,"order_key":"07:miner-7","p2mr_program_hex":"2525252525252525252525252525252525252525252525252525252525252525","share_difficulty":19,"share_id":"share-293","share_seq":293,"template_height":800009},{"accepted_at_ms":89985,"credit_policy":"stale-grace","job_id":"job-1261","job_issued_at_ms":89977,"miner_id":"miner-7","network_difficulty":1000003,"ntime":1700001261,"order_key":"07:miner-7","p2mr_program_hex":"edededededededededededededededededededededededededededededededed","share_difficulty":1,"share_id":"share-1261","share_seq":1261,"template_height":800039},{"accepted_at_ms":89986,"credit_policy":null,"job_id":"job-999","job_issued_at_ms":89983,"miner_id":"miner-9","network_difficulty":1000013,"ntime":1700000999,"order_key":"09:miner-9","p2mr_program_hex":"e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7","share_difficulty":3,"share_id":"share-999","share_seq":999,"template_height":800031},{"accepted_at_ms":89988,"credit_policy":null,"job_id":"job-1516","job_issued_at_ms":89990,"miner_id":"miner-9","network_difficulty":1000003,"ntime":1700001516,"order_key":"09:miner-9","p2mr_program_hex":"ecececececececececececececececececececececececececececececececec","share_difficulty":2,"share_id":"share-1516","share_seq":1516,"template_height":800047},{"accepted_at_ms":89988,"credit_policy":null,"job_id":"job-1063","job_issued_at_ms":89981,"miner_id":"miner-7","network_difficulty":1000009,"ntime":1700001063,"order_key":"07:miner-7","p2mr_program_hex":"2727272727272727272727272727272727272727272727272727272727272727","share_difficulty":1,"share_id":"share-1063","share_seq":1063,"template_height":800033},{"accepted_at_ms":89974,"credit_policy":null,"job_id":"job-1419","job_issued_at_ms":89990,"miner_id":"miner-0","network_difficulty":1000008,"ntime":1700001419,"order_key":"00:miner-0","p2mr_program_hex":"8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b","share_difficulty":7,"share_id":"share-1419","share_seq":1419,"template_height":800044},{"accepted_at_ms":89981,"credit_policy":"stale-grace","job_id":"job-316","job_issued_at_ms":89975,"miner_id":"miner-8","network_difficulty":1000010,"ntime":1700000316,"order_key":"08:miner-8","p2mr_program_hex":"3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c","share_difficulty":19,"share_id":"share-316","share_seq":316,"template_height":800009},{"accepted_at_ms":89976,"credit_policy":null,"job_id":"job-807","job_issued_at_ms":89980,"miner_id":"miner-4","network_difficulty":1000008,"ntime":1700000807,"order_key":"04:miner-4","p2mr_program_hex":"2727272727272727272727272727272727272727272727272727272727272727","share_difficulty":3,"share_id":"share-807","share_seq":807,"template_height":800025},{"accepted_at_ms":89989,"credit_policy":"stale-grace","job_id":"job-1347","job_issued_at_ms":89996,"miner_id":"miner-5","network_difficulty":1000004,"ntime":1700001347,"order_key":"05:miner-5","p2mr_program_hex":"4343434343434343434343434343434343434343434343434343434343434343","share_difficulty":2,"share_id":"share-1347","share_seq":1347,"template_height":800042},{"accepted_at_ms":89971,"credit_policy":"stale-grace","job_id":"job-1350","job_issued_at_ms":89970,"miner_id":"miner-8","network_difficulty":1000007,"ntime":1700001350,"order_key":"08:miner-8","p2mr_program_hex":"4646464646464646464646464646464646464646464646464646464646464646","share_difficulty":1,"share_id":"share-1350","share_seq":1350,"template_height":800042},{"accepted_at_ms":89987,"credit_policy":null,"job_id":"job-509","job_issued_at_ms":89992,"miner_id":"miner-3","network_difficulty":1000016,"ntime":1700000509,"order_key":"03:miner-3","p2mr_program_hex":"fdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfd","share_difficulty":7,"share_id":"share-509","share_seq":509,"template_height":800015},{"accepted_at_ms":89985,"credit_policy":null,"job_id":"job-395","job_issued_at_ms":89991,"miner_id":"miner-10","network_difficulty":1000004,"ntime":1700000395,"order_key":"10:miner-10","p2mr_program_hex":"8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b","share_difficulty":19,"share_id":"share-395","share_seq":395,"template_height":800012},{"accepted_at_ms":89984,"credit_policy":null,"job_id":"job-903","job_issued_at_ms":89975,"miner_id":"miner-1","network_difficulty":1000002,"ntime":1700000903,"order_key":"01:miner-1","p2mr_program_hex":"8787878787878787878787878787878787878787878787878787878787878787","share_difficulty":2,"share_id":"share-903","share_seq":903,"template_height":800028},{"accepted_at_ms":89984,"credit_policy":"stale-grace","job_id":"job-1223","job_issued_at_ms":89973,"miner_id":"miner-2","network_difficulty":1000016,"ntime":1700001223,"order_key":"02:miner-2","p2mr_program_hex":"c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7","share_difficulty":7,"share_id":"share-1223","share_seq":1223,"template_height":800038},{"accepted_at_ms":89996,"credit_policy":null,"job_id":"job-299","job_issued_at_ms":89978,"miner_id":"miner-2","network_difficulty":1000010,"ntime":1700000299,"order_key":"02:miner-2","p2mr_program_hex":"2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b","share_difficulty":2,"share_id":"share-299","share_seq":299,"template_height":800009},{"accepted_at_ms":89976,"credit_policy":null,"job_id":"job-402","job_issued_at_ms":89974,"miner_id":"miner-6","network_difficulty":1000011,"ntime":1700000402,"order_key":"06:miner-6","p2mr_program_hex":"9292929292929292929292929292929292929292929292929292929292929292","share_difficulty":2,"share_id":"share-402","share_seq":402,"template_height":800012},{"accepted_at_ms":89989,"credit_policy":null,"job_id":"job-1605","job_issued_at_ms":89994,"miner_id":"miner-10","network_difficulty":1000007,"ntime":1700001605,"order_key":"10:miner-10","p2mr_program_hex":"4545454545454545454545454545454545454545454545454545454545454545","share_difficulty":1,"share_id":"share-1605","share_seq":1605,"template_height":800050},{"accepted_at_ms":89990,"credit_policy":null,"job_id":"job-53","job_issued_at_ms":89973,"miner_id":"miner-9","network_difficulty":1000002,"ntime":1700000053,"order_key":"09:miner-9","p2mr_program_hex":"3535353535353535353535353535353535353535353535353535353535353535","share_difficulty":2,"share_id":"share-53","share_seq":53,"template_height":800001},{"accepted_at_ms":89996,"credit_policy":null,"job_id":"job-770","job_issued_at_ms":89970,"miner_id":"miner-0","network_difficulty":1000005,"ntime":1700000770,"order_key":"00:miner-0","p2mr_program_hex":"0202020202020202020202020202020202020202020202020202020202020202","share_difficulty":3,"share_id":"share-770","share_seq":770,"template_height":800024},{"accepted_at_ms":89979,"credit_policy":null,"job_id":"job-826","job_issued_at_ms":89977,"miner_id":"miner-1","network_difficulty":1000010,"ntime":1700000826,"order_key":"01:miner-1","p2mr_program_hex":"3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a","share_difficulty":3,"share_id":"share-826","share_seq":826,"template_height":800025},{"accepted_at_ms":89972,"credit_policy":null,"job_id":"job-1427","job_issued_at_ms":89976,"miner_id":"miner-8","network_difficulty":1000016,"ntime":1700001427,"order_key":"08:miner-8","p2mr_program_hex":"9393939393939393939393939393939393939393939393939393939393939393","share_difficulty":19,"share_id":"share-1427","share_seq":1427,"template_height":800044},{"accepted_at_ms":89989,"credit_policy":null,"job_id":"job-740","job_issued_at_ms":89982,"miner_id":"miner-3","network_difficulty":1000009,"ntime":1700000740,"order_key":"03:miner-3","p2mr_program_hex":"e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4","share_difficulty":2,"share_id":"share-740","share_seq":740,"template_height":800023},{"accepted_at_ms":89997,"credit_policy":null,"job_id":"job-159","job_issued_at_ms":89977,"miner_id":"miner-5","network_difficulty":1000006,"ntime":1700000159,"order_key":"05:miner-5","p2mr_program_hex":"9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f","share_difficulty":3,"share_id":"share-159","share_seq":159,"template_height":800004},{"accepted_at_ms":89974,"credit_policy":null,"job_id":"job-948","job_issued_at_ms":89975,"miner_id":"miner-2","network_difficulty":1000013,"ntime":1700000948,"order_key":"02:miner-2","p2mr_program_hex":"b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4","share_difficulty":3,"share_id":"share-948","share_seq":948,"template_height":800029},{"accepted_at_ms":90000,"credit_policy":null,"job_id":"job-20","job_issued_at_ms":89990,"miner_id":"miner-9","network_difficulty":1000003,"ntime":1700000020,"order_key":"09:miner-9","p2mr_program_hex":"1414141414141414141414141414141414141414141414141414141414141414","share_difficulty":1,"share_id":"share-20","share_seq":20,"template_height":800000},{"accepted_at_ms":89982,"credit_policy":null,"job_id":"job-777","job_issued_at_ms":89980,"miner_id":"miner-7","network_difficulty":1000012,"ntime":1700000777,"order_key":"07:miner-7","p2mr_program_hex":"0909090909090909090909090909090909090909090909090909090909090909","share_difficulty":7,"share_id":"share-777","share_seq":777,"template_height":800024},{"accepted_at_ms":89971,"credit_policy":null,"job_id":"job-1030","job_issued_at_ms":89984,"miner_id":"miner-7","network_difficulty":1000010,"ntime":1700001030,"order_key":"07:miner-7","p2mr_program_hex":"0606060606060606060606060606060606060606060606060606060606060606","share_difficulty":2,"share_id":"share-1030","share_seq":1030,"template_height":800032},{"accepted_at_ms":89985,"credit_policy":null,"job_id":"job-1349","job_issued_at_ms":89970,"miner_id":"miner-7","network_difficulty":1000006,"ntime":1700001349,"order_key":"07:miner-7","p2mr_program_hex":"4545454545454545454545454545454545454545454545454545454545454545","share_difficulty":19,"share_id":"share-1349","share_seq":1349,"template_height":800042},{"accepted_at_ms":89989,"credit_policy":null,"job_id":"job-668","job_issued_at_ms":89985,"miner_id":"miner-8","network_difficulty":1000005,"ntime":1700000668,"order_key":"08:miner-8","p2mr_program_hex":"9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c","share_difficulty":2,"share_id":"share-668","share_seq":668,"template_height":800020},{"accepted_at_ms":89993,"credit_policy":null,"job_id":"job-957","job_issued_at_ms":89987,"miner_id":"miner-0","network_difficulty":1000005,"ntime":1700000957,"order_key":"00:miner-0","p2mr_program_hex":"bdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbd","share_difficulty":19,"share_id":"share-957","share_seq":957,"template_height":800029},{"accepted_at_ms":89982,"credit_policy":null,"job_id":"job-1735","job_issued_at_ms":89977,"miner_id":"miner-8","network_difficulty":1000001,"ntime":1700001735,"order_key":"08:miner-8","p2mr_program_hex":"c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7","share_difficulty":7,"share_id":"share-1735","share_seq":1735,"template_height":800054},{"accepted_at_ms":89986,"credit_policy":"stale-grace","job_id":"job-1095","job_issued_at_ms":89991,"miner_id":"miner-6","network_difficulty":1000007,"ntime":1700001095,"order_key":"06:miner-6","p2mr_program_hex":"4747474747474747474747474747474747474747474747474747474747474747","share_difficulty":7,"share_id":"share-1095","share_seq":1095,"template_height":800034},{"accepted_at_ms":89970,"credit_policy":"stale-grace","job_id":"job-1110","job_issued_at_ms":89987,"miner_id":"miner-10","network_difficulty":1000005,"ntime":1700001110,"order_key":"10:miner-10","p2mr_program_hex":"5656565656565656565656565656565656565656565656565656565656565656","share_difficulty":7,"share_id":"share-1110","share_seq":1110,"template_height":800034},{"accepted_at_ms":89993,"credit_policy":null,"job_id":"job-1442","job_issued_at_ms":89990,"miner_id":"miner-1","network_difficulty":1000014,"ntime":1700001442,"order_key":"01:miner-1","p2mr_program_hex":"a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2","share_difficulty":19,"share_id":"share-1442","share_seq":1442,"template_height":800045},{"accepted_at_ms":89979,"credit_policy":null,"job_id":"job-818","job_issued_at_ms":89977,"miner_id":"miner-4","network_difficulty":1000002,"ntime":1700000818,"order_key":"04:miner-4","p2mr_program_hex":"3232323232323232323232323232323232323232323232323232323232323232","share_difficulty":19,"share_id":"share-818","share_seq":818,"template_height":800025},{"accepted_at_ms":89994,"credit_policy":null,"job_id":"job-677","job_issued_at_ms":89981,"miner_id":"miner-6","network_difficulty":1000014,"ntime":1700000677,"order_key":"06:miner-6","p2mr_program_hex":"a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5","share_difficulty":1,"share_id":"share-677","share_seq":677,"template_height":800021},{"accepted_at_ms":89977,"credit_policy":null,"job_id":"job-1730","job_issued_at_ms":89992,"miner_id":"miner-3","network_difficulty":1000013,"ntime":1700001730,"order_key":"03:miner-3","p2mr_program_hex":"c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2","share_difficulty":1,"share_id":"share-1730","share_seq":1730,"template_height":800054},{"accepted_at_ms":89982,"credit_policy":"stale-grace","job_id":"job-1637","job_issued_at_ms":89980,"miner_id":"miner-9","network_difficulty":1000005,"ntime":1700001637,"order_key":"09:miner-9","p2mr_program_hex":"6565656565656565656565656565656565656565656565656565656565656565","share_difficulty":1,"share_id":"share-1637","share_seq":1637,"template_height":800051},{"accepted_at_ms":89989,"credit_policy":null,"job_id":"job-1159","job_issued_at_ms":89998,"miner_id":"miner-4","network_difficulty":1000003,"ntime":1700001159,"order_key":"04:miner-4","p2mr_program_hex":"8787878787878787878787878787878787878787878787878787878787878787","share_difficulty":2,"share_id":"share-1159","share_seq":1159,"template_height":800036},{"accepted_at_ms":89979,"credit_policy":null,"job_id":"job-1096","job_issued_at_ms":89985,"miner_id":"miner-7","network_difficulty":1000008,"ntime":1700001096,"order_key":"07:miner-7","p2mr_program_hex":"4848484848484848484848484848484848484848484848484848484848484848","share_difficulty":1,"share_id":"share-1096","share_seq":1096,"template_height":800034},{"accepted_at_ms":89981,"credit_policy":null,"job_id":"job-1781","job_issued_at_ms":89976,"miner_id":"miner-10","network_difficulty":1000013,"ntime":1700001781,"order_key":"10:miner-10","p2mr_program_hex":"f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5","share_difficulty":1,"share_id":"share-1781","share_seq":1781,"template_height":800055},{"accepted_at_ms":89986,"credit_policy":null,"job_id":"job-1335","job_issued_at_ms":89994,"miner_id":"miner-4","network_difficulty":1000009,"ntime":1700001335,"order_key":"04:miner-4","p2mr_program_hex":"3737373737373737373737373737373737373737373737373737373737373737","share_difficulty":2,"share_id":"share-1335","share_seq":1335,"template_height":800041},{"accepted_at_ms":89974,"credit_policy":null,"job_id":"job-1054","job_issued_at_ms":89974,"miner_id":"miner-9","network_difficulty":1000000,"ntime":1700001054,"order_key":"09:miner-9","p2mr_program_hex":"1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e","share_difficulty":19,"share_id":"share-1054","share_seq":1054,"template_height":800032},{"accepted_at_ms":89995,"credit_policy":"stale-grace","job_id":"job-132","job_issued_at_ms":89988,"miner_id":"miner-0","network_difficulty":1000013,"ntime":1700000132,"order_key":"00:miner-0","p2mr_program_hex":"8484848484848484848484848484848484848484848484848484848484848484","share_difficulty":19,"share_id":"share-132","share_seq":132,"template_height":800004},{"accepted_at_ms":89979,"credit_policy":"stale-grace","job_id":"job-1757","job_issued_at_ms":89989,"miner_id":"miner-8","network_difficulty":1000006,"ntime":1700001757,"order_key":"08:miner-8","p2mr_program_hex":"dddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddd","share_difficulty":2,"share_id":"share-1757","share_seq":1757,"template_height":800054},{"accepted_at_ms":89971,"credit_policy":null,"job_id":"job-775","job_issued_at_ms":89974,"miner_id":"miner-5","network_difficulty":1000010,"ntime":1700000775,"order_key":"05:miner-5","p2mr_program_hex":"0707070707070707070707070707070707070707070707070707070707070707","share_difficulty":3,"share_id":"share-775","share_seq":775,"template_height":800024},{"accepted_at_ms":89985,"credit_policy":null,"job_id":"job-401","job_issued_at_ms":89980,"miner_id":"miner-5","network_difficulty":1000010,"ntime":1700000401,"order_key":"05:miner-5","p2mr_program_hex":"9191919191919191919191919191919191919191919191919191919191919191","share_difficulty":3,"share_id":"share-401","share_seq":401,"template_height":800012},{"accepted_at_ms":89985,"credit_policy":"stale-grace","job_id":"job-1004","job_issued_at_ms":89974,"miner_id":"miner-3","network_difficulty":1000001,"ntime":1700001004,"order_key":"03:miner-3","p2mr_program_hex":"ecececececececececececececececececececececececececececececececec","share_difficulty":2,"share_id":"share-1004","share_seq":1004,"template_height":800031},{"accepted_at_ms":89984,"credit_policy":null,"job_id":"job-1364","job_issued_at_ms":89984,"miner_id":"miner-0","network_difficulty":1000004,"ntime":1700001364,"order_key":"00:miner-0","p2mr_program_hex":"5454545454545454545454545454545454545454545454545454545454545454","share_difficulty":2,"share_id":"share-1364","share_seq":1364,"template_height":800042},{"accepted_at_ms":89977,"credit_policy":null,"job_id":"job-1101","job_issued_at_ms":89997,"miner_id":"miner-1","network_difficulty":1000013,"ntime":1700001101,"order_key":"01:miner-1","p2mr_program_hex":"4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d","share_difficulty":7,"share_id":"share-1101","share_seq":1101,"template_height":800034},{"accepted_at_ms":90000,"credit_policy":null,"job_id":"job-108","job_issued_at_ms":89974,"miner_id":"miner-9","network_difficulty":1000006,"ntime":1700000108,"order_key":"09:miner-9","p2mr_program_hex":"6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c","share_difficulty":1,"share_id":"share-108","share_seq":108,"template_height":800003},{"accepted_at_ms":89970,"credit_policy":null,"job_id":"job-1765","job_issued_at_ms":89994,"miner_id":"miner-5","network_difficulty":1000014,"ntime":1700001765,"order_key":"05:miner-5","p2mr_program_hex":"e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5","share_difficulty":2,"share_id":"share-1765","share_seq":1765,"template_height":800055},{"accepted_at_ms":89977,"credit_policy":null,"job_id":"job-166","job_issued_at_ms":89982,"miner_id":"miner-1","network_difficulty":1000013,"ntime":1700000166,"order_key":"01:miner-1","p2mr_program_hex":"a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6","share_difficulty":1,"share_id":"share-166","share_seq":166,"template_height":800005},{"accepted_at_ms":89986,"credit_policy":null,"job_id":"job-1143","job_issued_at_ms":89981,"miner_id":"miner-10","network_difficulty":1000004,"ntime":1700001143,"order_key":"10:miner-10","p2mr_program_hex":"7777777777777777777777777777777777777777777777777777777777777777","share_difficulty":19,"share_id":"share-1143","share_seq":1143,"template_height":800035},{"accepted_at_ms":89978,"credit_policy":null,"job_id":"job-1356","job_issued_at_ms":89985,"miner_id":"miner-3","network_difficulty":1000013,"ntime":1700001356,"order_key":"03:miner-3","p2mr_program_hex":"4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c","share_difficulty":1,"share_id":"share-1356","share_seq":1356,"template_height":800042},{"accepted_at_ms":89976,"credit_policy":"stale-grace","job_id":"job-1520","job_issued_at_ms":89990,"miner_id":"miner-2","network_difficulty":1000007,"ntime":1700001520,"order_key":"02:miner-2","p2mr_program_hex":"f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0","share_difficulty":2,"share_id":"share-1520","share_seq":1520,"template_height":800047},{"accepted_at_ms":90000,"credit_policy":null,"job_id":"job-54","job_issued_at_ms":89974,"miner_id":"miner-10","network_difficulty":1000003,"ntime":1700000054,"order_key":"10:miner-10","p2mr_program_hex":"3636363636363636363636363636363636363636363636363636363636363636","share_difficulty":2,"share_id":"share-54","share_seq":54,"template_height":800001},{"accepted_at_ms":89987,"credit_policy":"stale-grace","job_id":"job-634","job_issued_at_ms":89974,"miner_id":"miner-7","network_difficulty":1000005,"ntime":1700000634,"order_key":"07:miner-7","p2mr_program_hex":"7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a","share_difficulty":19,"share_id":"share-634","share_seq":634,"template_height":800019},{"accepted_at_ms":89989,"credit_policy":null,"job_id":"job-1197","job_issued_at_ms":89989,"miner_id":"miner-9","network_difficulty":1000007,"ntime":1700001197,"order_key":"09:miner-9","p2mr_program_hex":"adadadadadadadadadadadadadadadadadadadadadadadadadadadadadadadad","share_difficulty":19,"share_id":"share-1197","share_seq":1197,"template_height":800037},{"accepted_at_ms":89990,"credit_policy":"stale-grace","job_id":"job-1549","job_issued_at_ms":89974,"miner_id":"miner-9","network_difficulty":1000002,"ntime":1700001549,"order_key":"09:miner-9","p2mr_program_hex":"0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d","share_difficulty":2,"share_id":"share-1549","share_seq":1549,"template_height":800048},{"accepted_at_ms":89980,"credit_policy":null,"job_id":"job-652","job_issued_at_ms":89985,"miner_id":"miner-3","network_difficulty":1000006,"ntime":1700000652,"order_key":"03:miner-3","p2mr_program_hex":"8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c","share_difficulty":2,"share_id":"share-652","share_seq":652,"template_height":800020},{"accepted_at_ms":89970,"credit_policy":null,"job_id":"job-501","job_issued_at_ms":89998,"miner_id":"miner-6","network_difficulty":1000008,"ntime":1700000501,"order_key":"06:miner-6","p2mr_program_hex":"f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5","share_difficulty":19,"share_id":"share-501","share_seq":501,"template_height":800015},{"accepted_at_ms":89995,"credit_policy":null,"job_id":"job-1583","job_issued_at_ms":89972,"miner_id":"miner-10","network_difficulty":1000002,"ntime":1700001583,"order_key":"10:miner-10","p2mr_program_hex":"2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f","share_difficulty":19,"share_id":"share-1583","share_seq":1583,"template_height":800049},{"accepted_at_ms":89975,"credit_policy":"stale-grace","job_id":"job-1128","job_issued_at_ms":89976,"miner_id":"miner-6","network_difficulty":1000006,"ntime":1700001128,"order_key":"06:miner-6","p2mr_program_hex":"6868686868686868686868686868686868686868686868686868686868686868","share_difficulty":19,"share_id":"share-1128","share_seq":1128,"template_height":800035},{"accepted_at_ms":89991,"credit_policy":"stale-grace","job_id":"job-769","job_issued_at_ms":89981,"miner_id":"miner-10","network_difficulty":1000004,"ntime":1700000769,"order_key":"10:miner-10","p2mr_program_hex":"0101010101010101010101010101010101010101010101010101010101010101","share_difficulty":7,"share_id":"share-769","share_seq":769,"template_height":800024},{"accepted_at_ms":89995,"credit_policy":null,"job_id":"job-276","job_issued_at_ms":90000,"miner_id":"miner-1","network_difficulty":1000004,"ntime":1700000276,"order_key":"01:miner-1","p2mr_program_hex":"1414141414141414141414141414141414141414141414141414141414141414","share_difficulty":1,"share_id":"share-276","share_seq":276,"template_height":800008},{"accepted_at_ms":89975,"credit_policy":null,"job_id":"job-444","job_issued_at_ms":89997,"miner_id":"miner-4","network_difficulty":1000002,"ntime":1700000444,"order_key":"04:miner-4","p2mr_program_hex":"bcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbc","share_difficulty":7,"share_id":"share-444","share_seq":444,"template_height":800013},{"accepted_at_ms":90000,"credit_policy":"stale-grace","job_id":"job-91","job_issued_at_ms":89974,"miner_id":"miner-3","network_difficulty":1000006,"ntime":1700000091,"order_key":"03:miner-3","p2mr_program_hex":"5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b","share_difficulty":3,"share_id":"share-91","share_seq":91,"template_height":800002},{"accepted_at_ms":89977,"credit_policy":null,"job_id":"job-475","job_issued_at_ms":89986,"miner_id":"miner-2","network_difficulty":1000016,"ntime":1700000475,"order_key":"02:miner-2","p2mr_program_hex":"dbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdb","share_difficulty":7,"share_id":"share-475","share_seq":475,"template_height":800014},{"accepted_at_ms":89998,"credit_policy":null,"job_id":"job-1217","job_issued_at_ms":89983,"miner_id":"miner-7","network_difficulty":1000010,"ntime":1700001217,"order_key":"07:miner-7","p2mr_program_hex":"c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1","share_difficulty":7,"share_id":"share-1217","share_seq":1217,"template_height":800038},{"accepted_at_ms":90000,"credit_policy":null,"job_id":"job-1681","job_issued_at_ms":89974,"miner_id":"miner-9","network_difficulty":1000015,"ntime":1700001681,"order_key":"09:miner-9","p2mr_program_hex":"9191919191919191919191919191919191919191919191919191919191919191","share_difficulty":1,"share_id":"share-1681","share_seq":1681,"template_height":800052},{"accepted_at_ms":89990,"credit_policy":null,"job_id":"job-534","job_issued_at_ms":89976,"miner_id":"miner-6","network_difficulty":1000007,"ntime":1700000534,"order_key":"06:miner-6","p2mr_program_hex":"1616161616161616161616161616161616161616161616161616161616161616","share_difficulty":2,"share_id":"share-534","share_seq":534,"template_height":800016},{"accepted_at_ms":89979,"credit_policy":null,"job_id":"job-1034","job_issued_at_ms":89987,"miner_id":"miner-0","network_difficulty":1000014,"ntime":1700001034,"order_key":"00:miner-0","p2mr_program_hex":"0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a","share_difficulty":2,"share_id":"share-1034","share_seq":1034,"template_height":800032},{"accepted_at_ms":89976,"credit_policy":null,"job_id":"job-1392","job_issued_at_ms":89990,"miner_id":"miner-6","network_difficulty":1000015,"ntime":1700001392,"order_key":"06:miner-6","p2mr_program_hex":"7070707070707070707070707070707070707070707070707070707070707070","share_difficulty":3,"share_id":"share-1392","share_seq":1392,"template_height":800043},{"accepted_at_ms":89987,"credit_policy":"stale-grace","job_id":"job-474","job_issued_at_ms":89975,"miner_id":"miner-1","network_difficulty":1000015,"ntime":1700000474,"order_key":"01:miner-1","p2mr_program_hex":"dadadadadadadadadadadadadadadadadadadadadadadadadadadadadadadada","share_difficulty":2,"share_id":"share-474","share_seq":474,"template_height":800014},{"accepted_at_ms":89995,"credit_policy":"stale-grace","job_id":"job-1685","job_issued_at_ms":89995,"miner_id":"miner-2","network_difficulty":1000002,"ntime":1700001685,"order_key":"02:miner-2","p2mr_program_hex":"9595959595959595959595959595959595959595959595959595959595959595","share_difficulty":3,"share_id":"share-1685","share_seq":1685,"template_height":800052},{"accepted_at_ms":89978,"credit_policy":null,"job_id":"job-1746","job_issued_at_ms":89992,"miner_id":"miner-8","network_difficulty":1000012,"ntime":1700001746,"order_key":"08:miner-8","p2mr_program_hex":"d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2","share_difficulty":3,"share_id":"share-1746","share_seq":1746,"template_height":800054},{"accepted_at_ms":89985,"credit_policy":"stale-grace","job_id":"job-674","job_issued_at_ms":89971,"miner_id":"miner-3","network_difficulty":1000011,"ntime":1700000674,"order_key":"03:miner-3","p2mr_program_hex":"a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2","share_difficulty":2,"share_id":"share-674","share_seq":674,"template_height":800021},{"accepted_at_ms":89976,"credit_policy":null,"job_id":"job-95","job_issued_at_ms":89985,"miner_id":"miner-7","network_difficulty":1000010,"ntime":1700000095,"order_key":"07:miner-7","p2mr_program_hex":"5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f","share_difficulty":3,"share_id":"share-95","share_seq":95,"template_height":800002},{"accepted_at_ms":89994,"credit_policy":null,"job_id":"job-328","job_issued_at_ms":89975,"miner_id":"miner-9","network_difficulty":1000005,"ntime":1700000328,"order_key":"09:miner-9","p2mr_program_hex":"4848484848484848484848484848484848484848484848484848484848484848","share_difficulty":2,"share_id":"share-328","share_seq":328,"template_height":800010},{"accepted_at_ms":89996,"credit_policy":"stale-grace","job_id":"job-161","job_issued_at_ms":89975,"miner_id":"miner-7","network_difficulty":1000008,"ntime":1700000161,"order_key":"07:miner-7","p2mr_program_hex":"a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1","share_difficulty":2,"share_id":"share-161","share_seq":161,"template_height":800005},{"accepted_at_ms":89997,"credit_policy":null,"job_id":"job-875","job_issued_at_ms":89989,"miner_id":"miner-6","network_difficulty":1000008,"ntime":1700000875,"order_key":"06:miner-6","p2mr_program_hex":"6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b","share_difficulty":7,"share_id":"share-875","share_seq":875,"template_height":800027},{"accepted_at_ms":89976,"credit_policy":"stale-grace","job_id":"job-417","job_issued_at_ms":89982,"miner_id":"miner-10","network_difficulty":1000009,"ntime":1700000417,"order_key":"10:miner-10","p2mr_program_hex":"a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1","share_difficulty":2,"share_id":"share-417","share_seq":417,"template_height":800013},{"accepted_at_ms":89971,"credit_policy":null,"job_id":"job-943","job_issued_at_ms":89993,"miner_id":"miner-8","network_difficulty":1000008,"ntime":1700000943,"order_key":"08:miner-8","p2mr_program_hex":"afafafafafafafafafafafafafafafafafafafafafafafafafafafafafafafaf","share_difficulty":3,"share_id":"share-943","share_seq":943,"template_height":800029},{"accepted_at_ms":89997,"credit_policy":null,"job_id":"job-1105","job_issued_at_ms":89984,"miner_id":"miner-5","network_difficulty":1000000,"ntime":1700001105,"order_key":"05:miner-5","p2mr_program_hex":"5151515151515151515151515151515151515151515151515151515151515151","share_difficulty":1,"share_id":"share-1105","share_seq":1105,"template_height":800034},{"accepted_at_ms":89984,"credit_policy":null,"job_id":"job-1341","job_issued_at_ms":89981,"miner_id":"miner-10","network_difficulty":1000015,"ntime":1700001341,"order_key":"10:miner-10","p2mr_program_hex":"3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d","share_difficulty":7,"share_id":"share-1341","share_seq":1341,"template_height":800041},{"accepted_at_ms":89993,"credit_policy":null,"job_id":"job-461","job_issued_at_ms":89992,"miner_id":"miner-10","network_difficulty":1000002,"ntime":1700000461,"order_key":"10:miner-10","p2mr_program_hex":"cdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcd","share_difficulty":2,"share_id":"share-461","share_seq":461,"template_height":800014},{"accepted_at_ms":89995,"credit_policy":"stale-grace","job_id":"job-963","job_issued_at_ms":89985,"miner_id":"miner-6","network_difficulty":1000011,"ntime":1700000963,"order_key":"06:miner-6","p2mr_program_hex":"c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3","share_difficulty":1,"share_id":"share-963","share_seq":963,"template_height":800030},{"accepted_at_ms":89987,"credit_policy":"stale-grace","job_id":"job-1611","job_issued_at_ms":89975,"miner_id":"miner-5","network_difficulty":1000013,"ntime":1700001611,"order_key":"05:miner-5","p2mr_program_hex":"4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b","share_difficulty":19,"share_id":"share-1611","share_seq":1611,"template_height":800050},{"accepted_at_ms":89983,"credit_policy":"stale-grace","job_id":"job-319","job_issued_at_ms":89997,"miner_id":"miner-0","network_difficulty":1000013,"ntime":1700000319,"order_key":"00:miner-0","p2mr_program_hex":"3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f","share_difficulty":2,"share_id":"share-319","share_seq":319,"template_height":800009},{"accepted_at_ms":89981,"credit_policy":"stale-grace","job_id":"job-540","job_issued_at_ms":89995,"miner_id":"miner-1","network_difficulty":1000013,"ntime":1700000540,"order_key":"01:miner-1","p2mr_program_hex":"1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c","share_difficulty":3,"share_id":"share-540","share_seq":540,"template_height":800016},{"accepted_at_ms":89995,"credit_policy":null,"job_id":"job-14","job_issued_at_ms":89987,"miner_id":"miner-3","network_difficulty":1000014,"ntime":1700000014,"order_key":"03:miner-3","p2mr_program_hex":"0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e","share_difficulty":3,"share_id":"share-14","share_seq":14,"template_height":800000},{"accepted_at_ms":89994,"credit_policy":null,"job_id":"job-208","job_issued_at_ms":89981,"miner_id":"miner-10","network_difficulty":1000004,"ntime":1700000208,"order_key":"10:miner-10","p2mr_program_hex":"d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0","share_difficulty":1,"share_id":"share-208","share_seq":208,"template_height":800006},{"accepted_at_ms":89993,"credit_policy":null,"job_id":"job-30","job_issued_at_ms":89998,"miner_id":"miner-8","network_difficulty":1000013,"ntime":1700000030,"order_key":"08:miner-8","p2mr_program_hex":"1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e","share_difficulty":19,"share_id":"share-30","share_seq":30,"template_height":800000},{"accepted_at_ms":89986,"credit_policy":null,"job_id":"job-396","job_issued_at_ms":89993,"miner_id":"miner-0","network_difficulty":1000005,"ntime":1700000396,"order_key":"00:miner-0","p2mr_program_hex":"8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c","share_difficulty":3,"share_id":"share-396","share_seq":396,"template_height":800012},{"accepted_at_ms":89978,"credit_policy":null,"job_id":"job-671","job_issued_at_ms":89988,"miner_id":"miner-0","network_difficulty":1000008,"ntime":1700000671,"order_key":"00:miner-0","p2mr_program_hex":"9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f","share_difficulty":1,"share_id":"share-671","share_seq":671,"template_height":800020},{"accepted_at_ms":89979,"credit_policy":null,"job_id":"job-473","job_issued_at_ms":89993,"miner_id":"miner-0","network_difficulty":1000014,"ntime":1700000473,"order_key":"00:miner-0","p2mr_program_hex":"d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9","share_difficulty":3,"share_id":"share-473","share_seq":473,"template_height":800014},{"accepted_at_ms":89975,"credit_policy":null,"job_id":"job-1244","job_issued_at_ms":89990,"miner_id":"miner-1","network_difficulty":1000003,"ntime":1700001244,"order_key":"01:miner-1","p2mr_program_hex":"dcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdc","share_difficulty":19,"share_id":"share-1244","share_seq":1244,"template_height":800038},{"accepted_at_ms":89999,"credit_policy":null,"job_id":"job-562","job_issued_at_ms":89983,"miner_id":"miner-1","network_difficulty":1000001,"ntime":1700000562,"order_key":"01:miner-1","p2mr_program_hex":"3232323232323232323232323232323232323232323232323232323232323232","share_difficulty":3,"share_id":"share-562","share_seq":562,"template_height":800017},{"accepted_at_ms":89972,"credit_policy":null,"job_id":"job-213","job_issued_at_ms":89971,"miner_id":"miner-4","network_difficulty":1000009,"ntime":1700000213,"order_key":"04:miner-4","p2mr_program_hex":"d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5","share_difficulty":19,"share_id":"share-213","share_seq":213,"template_height":800006},{"accepted_at_ms":89978,"credit_policy":null,"job_id":"job-762","job_issued_at_ms":89990,"miner_id":"miner-3","network_difficulty":1000014,"ntime":1700000762,"order_key":"03:miner-3","p2mr_program_hex":"fafafafafafafafafafafafafafafafafafafafafafafafafafafafafafafafa","share_difficulty":2,"share_id":"share-762","share_seq":762,"template_height":800023},{"accepted_at_ms":89976,"credit_policy":null,"job_id":"job-1245","job_issued_at_ms":89999,"miner_id":"miner-2","network_difficulty":1000004,"ntime":1700001245,"order_key":"02:miner-2","p2mr_program_hex":"dddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddd","share_difficulty":3,"share_id":"share-1245","share_seq":1245,"template_height":800038},{"accepted_at_ms":89990,"credit_policy":null,"job_id":"job-257","job_issued_at_ms":89995,"miner_id":"miner-4","network_difficulty":1000002,"ntime":1700000257,"order_key":"04:miner-4","p2mr_program_hex":"0101010101010101010101010101010101010101010101010101010101010101","share_difficulty":7,"share_id":"share-257","share_seq":257,"template_height":800008},{"accepted_at_ms":89999,"credit_policy":null,"job_id":"job-382","job_issued_at_ms":89972,"miner_id":"miner-8","network_difficulty":1000008,"ntime":1700000382,"order_key":"08:miner-8","p2mr_program_hex":"7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e","share_difficulty":2,"share_id":"share-382","share_seq":382,"template_height":800011},{"accepted_at_ms":89994,"credit_policy":"stale-grace","job_id":"job-548","job_issued_at_ms":89998,"miner_id":"miner-9","network_difficulty":1000004,"ntime":1700000548,"order_key":"09:miner-9","p2mr_program_hex":"2424242424242424242424242424242424242424242424242424242424242424","share_difficulty":1,"share_id":"share-548","share_seq":548,"template_height":800017},{"accepted_at_ms":89970,"credit_policy":null,"job_id":"job-1369","job_issued_at_ms":89992,"miner_id":"miner-5","network_difficulty":1000009,"ntime":1700001369,"order_key":"05:miner-5","p2mr_program_hex":"5959595959595959595959595959595959595959595959595959595959595959","share_difficulty":2,"share_id":"share-1369","share_seq":1369,"template_height":800042},{"accepted_at_ms":89974,"credit_policy":"stale-grace","job_id":"job-1558","job_issued_at_ms":89998,"miner_id":"miner-7","network_difficulty":1000011,"ntime":1700001558,"order_key":"07:miner-7","p2mr_program_hex":"1616161616161616161616161616161616161616161616161616161616161616","share_difficulty":19,"share_id":"share-1558","share_seq":1558,"template_height":800048},{"accepted_at_ms":89990,"credit_policy":"stale-grace","job_id":"job-973","job_issued_at_ms":89975,"miner_id":"miner-5","network_difficulty":1000004,"ntime":1700000973,"order_key":"05:miner-5","p2mr_program_hex":"cdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcd","share_difficulty":7,"share_id":"share-973","share_seq":973,"template_height":800030},{"accepted_at_ms":89970,"credit_policy":"stale-grace","job_id":"job-1019","job_issued_at_ms":89995,"miner_id":"miner-7","network_difficulty":1000016,"ntime":1700001019,"order_key":"07:miner-7","p2mr_program_hex":"fbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfb","share_difficulty":1,"share_id":"share-1019","share_seq":1019,"template_height":800031},{"accepted_at_ms":90000,"credit_policy":null,"job_id":"job-1698","job_issued_at_ms":89991,"miner_id":"miner-4","network_difficulty":1000015,"ntime":1700001698,"order_key":"04:miner-4","p2mr_program_hex":"a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2","share_difficulty":19,"share_id":"share-1698","share_seq":1698,"template_height":800053},{"accepted_at_ms":89991,"credit_policy":null,"job_id":"job-1147","job_issued_at_ms":89982,"miner_id":"miner-3","network_difficulty":1000008,"ntime":1700001147,"order_key":"03:miner-3","p2mr_program_hex":"7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b","share_difficulty":3,"share_id":"share-1147","share_seq":1147,"template_height":800035},{"accepted_at_ms":89991,"credit_policy":null,"job_id":"job-574","job_issued_at_ms":89987,"miner_id":"miner-2","network_difficulty":1000013,"ntime":1700000574,"order_key":"02:miner-2","p2mr_program_hex":"3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e","share_difficulty":7,"share_id":"share-574","share_seq":574,"template_height":800017},{"accepted_at_ms":89996,"credit_policy":null,"job_id":"job-1662","job_issued_at_ms":89978,"miner_id":"miner-1","network_difficulty":1000013,"ntime":1700001662,"order_key":"01:miner-1","p2mr_program_hex":"7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e","share_difficulty":3,"share_id":"share-1662","share_seq":1662,"template_height":800051},{"accepted_at_ms":89994,"credit_policy":"stale-grace","job_id":"job-1796","job_issued_at_ms":89976,"miner_id":"miner-3","network_difficulty":1000011,"ntime":1700001796,"order_key":"03:miner-3","p2mr_program_hex":"0404040404040404040404040404040404040404040404040404040404040404","share_difficulty":3,"share_id":"share-1796","share_seq":1796,"template_height":800056},{"accepted_at_ms":89986,"credit_policy":null,"job_id":"job-763","job_issued_at_ms":89974,"miner_id":"miner-4","network_difficulty":1000015,"ntime":1700000763,"order_key":"04:miner-4","p2mr_program_hex":"fbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfb","share_difficulty":2,"share_id":"share-763","share_seq":763,"template_height":800023},{"accepted_at_ms":89998,"credit_policy":null,"job_id":"job-1316","job_issued_at_ms":89978,"miner_id":"miner-7","network_difficulty":1000007,"ntime":1700001316,"order_key":"07:miner-7","p2mr_program_hex":"2424242424242424242424242424242424242424242424242424242424242424","share_difficulty":1,"share_id":"share-1316","share_seq":1316,"template_height":800041},{"accepted_at_ms":89988,"credit_policy":null,"job_id":"job-717","job_issued_at_ms":89999,"miner_id":"miner-2","network_difficulty":1000003,"ntime":1700000717,"order_key":"02:miner-2","p2mr_program_hex":"cdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcd","share_difficulty":19,"share_id":"share-717","share_seq":717,"template_height":800022},{"accepted_at_ms":89976,"credit_policy":null,"job_id":"job-1232","job_issued_at_ms":89992,"miner_id":"miner-0","network_difficulty":1000008,"ntime":1700001232,"order_key":"00:miner-0","p2mr_program_hex":"d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0","share_difficulty":3,"share_id":"share-1232","share_seq":1232,"template_height":800038},{"accepted_at_ms":89978,"credit_policy":null,"job_id":"job-1066","job_issued_at_ms":89972,"miner_id":"miner-10","network_difficulty":1000012,"ntime":1700001066,"order_key":"10:miner-10","p2mr_program_hex":"2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a","share_difficulty":3,"share_id":"share-1066","share_seq":1066,"template_height":800033},{"accepted_at_ms":89981,"credit_policy":null,"job_id":"job-226","job_issued_at_ms":89985,"miner_id":"miner-6","network_difficulty":1000005,"ntime":1700000226,"order_key":"06:miner-6","p2mr_program_hex":"e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2","share_difficulty":19,"share_id":"share-226","share_seq":226,"template_height":800007},{"accepted_at_ms":89986,"credit_policy":null,"job_id":"job-520","job_issued_at_ms":89982,"miner_id":"miner-3","network_difficulty":1000010,"ntime":1700000520,"order_key":"03:miner-3","p2mr_program_hex":"0808080808080808080808080808080808080808080808080808080808080808","share_difficulty":19,"share_id":"share-520","share_seq":520,"template_height":800016},{"accepted_at_ms":89999,"credit_policy":null,"job_id":"job-660","job_issued_at_ms":89978,"miner_id":"miner-0","network_difficulty":1000014,"ntime":1700000660,"order_key":"00:miner-0","p2mr_program_hex":"9494949494949494949494949494949494949494949494949494949494949494","share_difficulty":2,"share_id":"share-660","share_seq":660,"template_height":800020},{"accepted_at_ms":89970,"credit_policy":null,"job_id":"job-1395","job_issued_at_ms":89977,"miner_id":"miner-9","network_difficulty":1000001,"ntime":1700001395,"order_key":"09:miner-9","p2mr_program_hex":"7373737373737373737373737373737373737373737373737373737373737373","share_difficulty":1,"share_id":"share-1395","share_seq":1395,"template_height":800043},{"accepted_at_ms":89979,"credit_policy":null,"job_id":"job-595","job_issued_at_ms":89977,"miner_id":"miner-1","network_difficulty":1000000,"ntime":1700000595,"order_key":"01:miner-1","p2mr_program_hex":"5353535353535353535353535353535353535353535353535353535353535353","share_difficulty":3,"share_id":"share-595","share_seq":595,"template_height":800018},{"accepted_at_ms":89971,"credit_policy":"stale-grace","job_id":"job-795","job_issued_at_ms":89979,"miner_id":"miner-3","network_difficulty":1000013,"ntime":1700000795,"order_key":"03:miner-3","p2mr_program_hex":"1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b","share_difficulty":7,"share_id":"share-795","share_seq":795,"template_height":800024},{"accepted_at_ms":89973,"credit_policy":"stale-grace","job_id":"job-822","job_issued_at_ms":89994,"miner_id":"miner-8","network_difficulty":1000006,"ntime":1700000822,"order_key":"08:miner-8","p2mr_program_hex":"3636363636363636363636363636363636363636363636363636363636363636","share_difficulty":1,"share_id":"share-822","share_seq":822,"template_height":800025},{"accepted_at_ms":89985,"credit_policy":null,"job_id":"job-1199","job_issued_at_ms":89974,"miner_id":"miner-0","network_difficulty":1000009,"ntime":1700001199,"order_key":"00:miner-0","p2mr_program_hex":"afafafafafafafafafafafafafafafafafafafafafafafafafafafafafafafaf","share_difficulty":2,"share_id":"share-1199","share_seq":1199,"template_height":800037},{"accepted_at_ms":89972,"credit_policy":null,"job_id":"job-944","job_issued_at_ms":89997,"miner_id":"miner-9","network_difficulty":1000009,"ntime":1700000944,"order_key":"09:miner-9","p2mr_program_hex":"b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0","share_difficulty":19,"share_id":"share-944","share_seq":944,"template_height":800029},{"accepted_at_ms":89987,"credit_policy":null,"job_id":"job-48","job_issued_at_ms":89982,"miner_id":"miner-4","network_difficulty":1000014,"ntime":1700000048,"order_key":"04:miner-4","p2mr_program_hex":"3030303030303030303030303030303030303030303030303030303030303030","share_difficulty":1,"share_id":"share-48","share_seq":48,"template_height":800001},{"accepted_at_ms":89992,"credit_policy":"stale-grace","job_id":"job-1713","job_issued_at_ms":89998,"miner_id":"miner-8","network_difficulty":1000013,"ntime":1700001713,"order_key":"08:miner-8","p2mr_program_hex":"b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1","share_difficulty":3,"share_id":"share-1713","share_seq":1713,"template_height":800053},{"accepted_at_ms":89993,"credit_policy":null,"job_id":"job-1308","job_issued_at_ms":89972,"miner_id":"miner-10","network_difficulty":1000016,"ntime":1700001308,"order_key":"10:miner-10","p2mr_program_hex":"1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c","share_difficulty":3,"share_id":"share-1308","share_seq":1308,"template_height":800040},{"accepted_at_ms":89981,"credit_policy":"stale-grace","job_id":"job-154","job_issued_at_ms":89994,"miner_id":"miner-0","network_difficulty":1000001,"ntime":1700000154,"order_key":"00:miner-0","p2mr_program_hex":"9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a","share_difficulty":2,"share_id":"share-154","share_seq":154,"template_height":800004},{"accepted_at_ms":89972,"credit_policy":"stale-grace","job_id":"job-990","job_issued_at_ms":89980,"miner_id":"miner-0","network_difficulty":1000004,"ntime":1700000990,"order_key":"00:miner-0","p2mr_program_hex":"dededededededededededededededededededededededededededededededede","share_difficulty":3,"share_id":"share-990","share_seq":990,"template_height":800030},{"accepted_at_ms":90000,"credit_policy":"stale-grace","job_id":"job-1036","job_issued_at_ms":89988,"miner_id":"miner-2","network_difficulty":1000016,"ntime":1700001036,"order_key":"02:miner-2","p2mr_program_hex":"0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c","share_difficulty":3,"share_id":"share-1036","share_seq":1036,"template_height":800032},{"accepted_at_ms":89993,"credit_policy":"stale-grace","job_id":"job-307","job_issued_at_ms":89992,"miner_id":"miner-10","network_difficulty":1000001,"ntime":1700000307,"order_key":"10:miner-10","p2mr_program_hex":"3333333333333333333333333333333333333333333333333333333333333333","share_difficulty":7,"share_id":"share-307","share_seq":307,"template_height":800009},{"accepted_at_ms":89994,"credit_policy":"stale-grace","job_id":"job-1276","job_issued_at_ms":89975,"miner_id":"miner-0","network_difficulty":1000001,"ntime":1700001276,"order_key":"00:miner-0","p2mr_program_hex":"fcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfc","share_difficulty":7,"share_id":"share-1276","share_seq":1276,"template_height":800039},{"accepted_at_ms":89982,"credit_policy":null,"job_id":"job-318","job_issued_at_ms":89992,"miner_id":"miner-10","network_difficulty":1000012,"ntime":1700000318,"order_key":"10:miner-10","p2mr_program_hex":"3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e","share_difficulty":3,"share_id":"share-318","share_seq":318,"template_height":800009},{"accepted_at_ms":89983,"credit_policy":"stale-grace","job_id":"job-871","job_issued_at_ms":89970,"miner_id":"miner-2","network_difficulty":1000004,"ntime":1700000871,"order_key":"02:miner-2","p2mr_program_hex":"6767676767676767676767676767676767676767676767676767676767676767","share_difficulty":19,"share_id":"share-871","share_seq":871,"template_height":800027},{"accepted_at_ms":89979,"credit_policy":null,"job_id":"job-758","job_issued_at_ms":89988,"miner_id":"miner-10","network_difficulty":1000010,"ntime":1700000758,"order_key":"10:miner-10","p2mr_program_hex":"f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6","share_difficulty":7,"share_id":"share-758","share_seq":758,"template_height":800023},{"accepted_at_ms":89994,"credit_policy":null,"job_id":"job-1785","job_issued_at_ms":89974,"miner_id":"miner-3","network_difficulty":1000000,"ntime":1700001785,"order_key":"03:miner-3","p2mr_program_hex":"f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9","share_difficulty":1,"share_id":"share-1785","share_seq":1785,"template_height":800055},{"accepted_at_ms":89998,"credit_policy":null,"job_id":"job-810","job_issued_at_ms":89972,"miner_id":"miner-7","network_difficulty":1000011,"ntime":1700000810,"order_key":"07:miner-7","p2mr_program_hex":"2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a","share_difficulty":1,"share_id":"share-810","share_seq":810,"template_height":800025},{"accepted_at_ms":89980,"credit_policy":null,"job_id":"job-1562","job_issued_at_ms":89996,"miner_id":"miner-0","network_difficulty":1000015,"ntime":1700001562,"order_key":"00:miner-0","p2mr_program_hex":"1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a","share_difficulty":7,"share_id":"share-1562","share_seq":1562,"template_height":800048},{"accepted_at_ms":89979,"credit_policy":null,"job_id":"job-421","job_issued_at_ms":89992,"miner_id":"miner-3","network_difficulty":1000013,"ntime":1700000421,"order_key":"03:miner-3","p2mr_program_hex":"a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5","share_difficulty":1,"share_id":"share-421","share_seq":421,"template_height":800013},{"accepted_at_ms":89998,"credit_policy":null,"job_id":"job-654","job_issued_at_ms":89986,"miner_id":"miner-5","network_difficulty":1000008,"ntime":1700000654,"order_key":"05:miner-5","p2mr_program_hex":"8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e","share_difficulty":3,"share_id":"share-654","share_seq":654,"template_height":800020},{"accepted_at_ms":89974,"credit_policy":"stale-grace","job_id":"job-705","job_issued_at_ms":89985,"miner_id":"miner-1","network_difficulty":1000008,"ntime":1700000705,"order_key":"01:miner-1","p2mr_program_hex":"c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1","share_difficulty":3,"share_id":"share-705","share_seq":705,"template_height":800022},{"accepted_at_ms":89972,"credit_policy":"stale-grace","job_id":"job-162","job_issued_at_ms":89996,"miner_id":"miner-8","network_difficulty":1000009,"ntime":1700000162,"order_key":"08:miner-8","p2mr_program_hex":"a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2","share_difficulty":2,"share_id":"share-162","share_seq":162,"template_height":800005},{"accepted_at_ms":89984,"credit_policy":null,"job_id":"job-658","job_issued_at_ms":89976,"miner_id":"miner-9","network_difficulty":1000012,"ntime":1700000658,"order_key":"09:miner-9","p2mr_program_hex":"9292929292929292929292929292929292929292929292929292929292929292","share_difficulty":19,"share_id":"share-658","share_seq":658,"template_height":800020},{"accepted_at_ms":89972,"credit_policy":null,"job_id":"job-1756","job_issued_at_ms":89991,"miner_id":"miner-7","network_difficulty":1000005,"ntime":1700001756,"order_key":"07:miner-7","p2mr_program_hex":"dcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdc","share_difficulty":3,"share_id":"share-1756","share_seq":1756,"template_height":800054},{"accepted_at_ms":89975,"credit_policy":null,"job_id":"job-1123","job_issued_at_ms":89971,"miner_id":"miner-1","network_difficulty":1000001,"ntime":1700001123,"order_key":"01:miner-1","p2mr_program_hex":"6363636363636363636363636363636363636363636363636363636363636363","share_difficulty":3,"share_id":"share-1123","share_seq":1123,"template_height":800035},{"accepted_at_ms":89991,"credit_policy":null,"job_id":"job-1559","job_issued_at_ms":89995,"miner_id":"miner-8","network_difficulty":1000012,"ntime":1700001559,"order_key":"08:miner-8","p2mr_program_hex":"1717171717171717171717171717171717171717171717171717171717171717","share_difficulty":7,"share_id":"share-1559","share_seq":1559,"template_height":800048},{"accepted_at_ms":89982,"credit_policy":"stale-grace","job_id":"job-783","job_issued_at_ms":89995,"miner_id":"miner-2","network_difficulty":1000001,"ntime":1700000783,"order_key":"02:miner-2","p2mr_program_hex":"0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f","share_difficulty":19,"share_id":"share-783","share_seq":783,"template_height":800024},{"accepted_at_ms":89980,"credit_policy":"stale-grace","job_id":"job-508","job_issued_at_ms":89994,"miner_id":"miner-2","network_difficulty":1000015,"ntime":1700000508,"order_key":"02:miner-2","p2mr_program_hex":"fcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfc","share_difficulty":7,"share_id":"share-508","share_seq":508,"template_height":800015},{"accepted_at_ms":89977,"credit_policy":null,"job_id":"job-279","job_issued_at_ms":89975,"miner_id":"miner-4","network_difficulty":1000007,"ntime":1700000279,"order_key":"04:miner-4","p2mr_program_hex":"1717171717171717171717171717171717171717171717171717171717171717","share_difficulty":3,"share_id":"share-279","share_seq":279,"template_height":800008},{"accepted_at_ms":90000,"credit_policy":null,"job_id":"job-87","job_issued_at_ms":89980,"miner_id":"miner-10","network_difficulty":1000002,"ntime":1700000087,"order_key":"10:miner-10","p2mr_program_hex":"5757575757575757575757575757575757575757575757575757575757575757","share_difficulty":19,"share_id":"share-87","share_seq":87,"template_height":800002},{"accepted_at_ms":89979,"credit_policy":"stale-grace","job_id":"job-622","job_issued_at_ms":89971,"miner_id":"miner-6","network_difficulty":1000010,"ntime":1700000622,"order_key":"06:miner-6","p2mr_program_hex":"6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e","share_difficulty":3,"share_id":"share-622","share_seq":622,"template_height":800019},{"accepted_at_ms":89986,"credit_policy":null,"job_id":"job-1443","job_issued_at_ms":89974,"miner_id":"miner-2","network_difficulty":1000015,"ntime":1700001443,"order_key":"02:miner-2","p2mr_program_hex":"a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3","share_difficulty":3,"share_id":"share-1443","share_seq":1443,"template_height":800045},{"accepted_at_ms":89995,"credit_policy":null,"job_id":"job-60","job_issued_at_ms":89998,"miner_id":"miner-5","network_difficulty":1000009,"ntime":1700000060,"order_key":"05:miner-5","p2mr_program_hex":"3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c","share_difficulty":3,"share_id":"share-60","share_seq":60,"template_height":800001},{"accepted_at_ms":89978,"credit_policy":"stale-grace","job_id":"job-55","job_issued_at_ms":89983,"miner_id":"miner-0","network_difficulty":1000004,"ntime":1700000055,"order_key":"00:miner-0","p2mr_program_hex":"3737373737373737373737373737373737373737373737373737373737373737","share_difficulty":19,"share_id":"share-55","share_seq":55,"template_height":800001},{"accepted_at_ms":89998,"credit_policy":null,"job_id":"job-636","job_issued_at_ms":89992,"miner_id":"miner-9","network_difficulty":1000007,"ntime":1700000636,"order_key":"09:miner-9","p2mr_program_hex":"7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c","share_difficulty":3,"share_id":"share-636","share_seq":636,"template_height":800019},{"accepted_at_ms":89996,"credit_policy":null,"job_id":"job-1098","job_issued_at_ms":89988,"miner_id":"miner-9","network_difficulty":1000010,"ntime":1700001098,"order_key":"09:miner-9","p2mr_program_hex":"4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a","share_difficulty":3,"share_id":"share-1098","share_seq":1098,"template_height":800034},{"accepted_at_ms":89974,"credit_policy":null,"job_id":"job-1529","job_issued_at_ms":89995,"miner_id":"miner-0","network_difficulty":1000016,"ntime":1700001529,"order_key":"00:miner-0","p2mr_program_hex":"f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9","share_difficulty":2,"share_id":"share-1529","share_seq":1529,"template_height":800047},{"accepted_at_ms":89986,"credit_policy":"stale-grace","job_id":"job-1117","job_issued_at_ms":89989,"miner_id":"miner-6","network_difficulty":1000012,"ntime":1700001117,"order_key":"06:miner-6","p2mr_program_hex":"5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d","share_difficulty":19,"share_id":"share-1117","share_seq":1117,"template_height":800034},{"accepted_at_ms":89971,"credit_policy":null,"job_id":"job-1772","job_issued_at_ms":89979,"miner_id":"miner-1","network_difficulty":1000004,"ntime":1700001772,"order_key":"01:miner-1","p2mr_program_hex":"ecececececececececececececececececececececececececececececececec","share_difficulty":3,"share_id":"share-1772","share_seq":1772,"template_height":800055},{"accepted_at_ms":89984,"credit_policy":null,"job_id":"job-907","job_issued_at_ms":89973,"miner_id":"miner-5","network_difficulty":1000006,"ntime":1700000907,"order_key":"05:miner-5","p2mr_program_hex":"8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b","share_difficulty":7,"share_id":"share-907","share_seq":907,"template_height":800028},{"accepted_at_ms":89985,"credit_policy":null,"job_id":"job-560","job_issued_at_ms":89979,"miner_id":"miner-10","network_difficulty":1000016,"ntime":1700000560,"order_key":"10:miner-10","p2mr_program_hex":"3030303030303030303030303030303030303030303030303030303030303030","share_difficulty":2,"share_id":"share-560","share_seq":560,"template_height":800017},{"accepted_at_ms":89996,"credit_policy":null,"job_id":"job-272","job_issued_at_ms":89976,"miner_id":"miner-8","network_difficulty":1000000,"ntime":1700000272,"order_key":"08:miner-8","p2mr_program_hex":"1010101010101010101010101010101010101010101010101010101010101010","share_difficulty":3,"share_id":"share-272","share_seq":272,"template_height":800008},{"accepted_at_ms":89979,"credit_policy":null,"job_id":"job-986","job_issued_at_ms":89989,"miner_id":"miner-7","network_difficulty":1000000,"ntime":1700000986,"order_key":"07:miner-7","p2mr_program_hex":"dadadadadadadadadadadadadadadadadadadadadadadadadadadadadadadada","share_difficulty":3,"share_id":"share-986","share_seq":986,"template_height":800030},{"accepted_at_ms":89998,"credit_policy":null,"job_id":"job-1598","job_issued_at_ms":89993,"miner_id":"miner-3","network_difficulty":1000000,"ntime":1700001598,"order_key":"03:miner-3","p2mr_program_hex":"3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e","share_difficulty":2,"share_id":"share-1598","share_seq":1598,"template_height":800049},{"accepted_at_ms":89994,"credit_policy":null,"job_id":"job-146","job_issued_at_ms":89981,"miner_id":"miner-3","network_difficulty":1000010,"ntime":1700000146,"order_key":"03:miner-3","p2mr_program_hex":"9292929292929292929292929292929292929292929292929292929292929292","share_difficulty":3,"share_id":"share-146","share_seq":146,"template_height":800004},{"accepted_at_ms":89983,"credit_policy":null,"job_id":"job-1552","job_issued_at_ms":89996,"miner_id":"miner-1","network_difficulty":1000005,"ntime":1700001552,"order_key":"01:miner-1","p2mr_program_hex":"1010101010101010101010101010101010101010101010101010101010101010","share_difficulty":19,"share_id":"share-1552","share_seq":1552,"template_height":800048},{"accepted_at_ms":89987,"credit_policy":"stale-grace","job_id":"job-1788","job_issued_at_ms":89998,"miner_id":"miner-6","network_difficulty":1000003,"ntime":1700001788,"order_key":"06:miner-6","p2mr_program_hex":"fcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfc","share_difficulty":2,"share_id":"share-1788","share_seq":1788,"template_height":800055},{"accepted_at_ms":89974,"credit_policy":"stale-grace","job_id":"job-655","job_issued_at_ms":89978,"miner_id":"miner-6","network_difficulty":1000009,"ntime":1700000655,"order_key":"06:miner-6","p2mr_program_hex":"8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f","share_difficulty":1,"share_id":"share-655","share_seq":655,"template_height":800020},{"accepted_at_ms":89981,"credit_policy":null,"job_id":"job-322","job_issued_at_ms":89984,"miner_id":"miner-3","network_difficulty":1000016,"ntime":1700000322,"order_key":"03:miner-3","p2mr_program_hex":"4242424242424242424242424242424242424242424242424242424242424242","share_difficulty":19,"share_id":"share-322","share_seq":322,"template_height":800010},{"accepted_at_ms":89992,"credit_policy":null,"job_id":"job-605","job_issued_at_ms":89988,"miner_id":"miner-0","network_difficulty":1000010,"ntime":1700000605,"order_key":"00:miner-0","p2mr_program_hex":"5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d","share_difficulty":3,"share_id":"share-605","share_seq":605,"template_height":800018},{"accepted_at_ms":89985,"credit_policy":null,"job_id":"job-1084","job_issued_at_ms":89993,"miner_id":"miner-6","network_difficulty":1000013,"ntime":1700001084,"order_key":"06:miner-6","p2mr_program_hex":"3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c","share_difficulty":2,"share_id":"share-1084","share_seq":1084,"template_height":800033},{"accepted_at_ms":89994,"credit_policy":null,"job_id":"job-1289","job_issued_at_ms":89995,"miner_id":"miner-2","network_difficulty":1000014,"ntime":1700001289,"order_key":"02:miner-2","p2mr_program_hex":"0909090909090909090909090909090909090909090909090909090909090909","share_difficulty":3,"share_id":"share-1289","share_seq":1289,"template_height":800040},{"accepted_at_ms":90000,"credit_policy":"stale-grace","job_id":"job-746","job_issued_at_ms":89976,"miner_id":"miner-9","network_difficulty":1000015,"ntime":1700000746,"order_key":"09:miner-9","p2mr_program_hex":"eaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaea","share_difficulty":7,"share_id":"share-746","share_seq":746,"template_height":800023},{"accepted_at_ms":89976,"credit_policy":null,"job_id":"job-949","job_issued_at_ms":89972,"miner_id":"miner-3","network_difficulty":1000014,"ntime":1700000949,"order_key":"03:miner-3","p2mr_program_hex":"b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5","share_difficulty":1,"share_id":"share-949","share_seq":949,"template_height":800029},{"accepted_at_ms":89999,"credit_policy":null,"job_id":"job-1177","job_issued_at_ms":89974,"miner_id":"miner-0","network_difficulty":1000004,"ntime":1700001177,"order_key":"00:miner-0","p2mr_program_hex":"9999999999999999999999999999999999999999999999999999999999999999","share_difficulty":19,"share_id":"share-1177","share_seq":1177,"template_height":800036},{"accepted_at_ms":89971,"credit_policy":null,"job_id":"job-398","job_issued_at_ms":89993,"miner_id":"miner-2","network_difficulty":1000007,"ntime":1700000398,"order_key":"02:miner-2","p2mr_program_hex":"8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e","share_difficulty":19,"share_id":"share-398","share_seq":398,"template_height":800012},{"accepted_at_ms":89992,"credit_policy":null,"job_id":"job-573","job_issued_at_ms":89998,"miner_id":"miner-1","network_difficulty":1000012,"ntime":1700000573,"order_key":"01:miner-1","p2mr_program_hex":"3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d","share_difficulty":2,"share_id":"share-573","share_seq":573,"template_height":800017},{"accepted_at_ms":89973,"credit_policy":null,"job_id":"job-301","job_issued_at_ms":90000,"miner_id":"miner-4","network_difficulty":1000012,"ntime":1700000301,"order_key":"04:miner-4","p2mr_program_hex":"2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d","share_difficulty":7,"share_id":"share-301","share_seq":301,"template_height":800009},{"accepted_at_ms":89973,"credit_policy":null,"job_id":"job-1164","job_issued_at_ms":89995,"miner_id":"miner-9","network_difficulty":1000008,"ntime":1700001164,"order_key":"09:miner-9","p2mr_program_hex":"8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c","share_difficulty":2,"share_id":"share-1164","share_seq":1164,"template_height":800036},{"accepted_at_ms":89970,"credit_policy":null,"job_id":"job-571","job_issued_at_ms":89991,"miner_id":"miner-10","network_difficulty":1000010,"ntime":1700000571,"order_key":"10:miner-10","p2mr_program_hex":"3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b","share_difficulty":2,"share_id":"share-571","share_seq":571,"template_height":800017},{"accepted_at_ms":89999,"credit_policy":null,"job_id":"job-1414","job_issued_at_ms":89999,"miner_id":"miner-6","network_difficulty":1000003,"ntime":1700001414,"order_key":"06:miner-6","p2mr_program_hex":"8686868686868686868686868686868686868686868686868686868686868686","share_difficulty":3,"share_id":"share-1414","share_seq":1414,"template_height":800044},{"accepted_at_ms":89997,"credit_policy":"stale-grace","job_id":"job-89","job_issued_at_ms":89983,"miner_id":"miner-1","network_difficulty":1000004,"ntime":1700000089,"order_key":"01:miner-1","p2mr_program_hex":"5959595959595959595959595959595959595959595959595959595959595959","share_difficulty":19,"share_id":"share-89","share_seq":89,"template_height":800002},{"accepted_at_ms":89984,"credit_policy":"stale-grace","job_id":"job-1252","job_issued_at_ms":90000,"miner_id":"miner-9","network_difficulty":1000011,"ntime":1700001252,"order_key":"09:miner-9","p2mr_program_hex":"e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4","share_difficulty":7,"share_id":"share-1252","share_seq":1252,"template_height":800039},{"accepted_at_ms":89978,"credit_policy":null,"job_id":"job-920","job_issued_at_ms":89985,"miner_id":"miner-7","network_difficulty":1000002,"ntime":1700000920,"order_key":"07:miner-7","p2mr_program_hex":"9898989898989898989898989898989898989898989898989898989898989898","share_difficulty":1,"share_id":"share-920","share_seq":920,"template_height":800028},{"accepted_at_ms":89993,"credit_policy":"stale-grace","job_id":"job-1035","job_issued_at_ms":89998,"miner_id":"miner-1","network_difficulty":1000015,"ntime":1700001035,"order_key":"01:miner-1","p2mr_program_hex":"0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b","share_difficulty":2,"share_id":"share-1035","share_seq":1035,"template_height":800032},{"accepted_at_ms":89988,"credit_policy":null,"job_id":"job-1231","job_issued_at_ms":89987,"miner_id":"miner-10","network_difficulty":1000007,"ntime":1700001231,"order_key":"10:miner-10","p2mr_program_hex":"cfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcf","share_difficulty":1,"share_id":"share-1231","share_seq":1231,"template_height":800038},{"accepted_at_ms":89972,"credit_policy":null,"job_id":"job-201","job_issued_at_ms":89985,"miner_id":"miner-3","network_difficulty":1000014,"ntime":1700000201,"order_key":"03:miner-3","p2mr_program_hex":"c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9","share_difficulty":7,"share_id":"share-201","share_seq":201,"template_height":800006},{"accepted_at_ms":89998,"credit_policy":null,"job_id":"job-1647","job_issued_at_ms":89987,"miner_id":"miner-8","network_difficulty":1000015,"ntime":1700001647,"order_key":"08:miner-8","p2mr_program_hex":"6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f","share_difficulty":19,"share_id":"share-1647","share_seq":1647,"template_height":800051},{"accepted_at_ms":89984,"credit_policy":null,"job_id":"job-1703","job_issued_at_ms":89981,"miner_id":"miner-9","network_difficulty":1000003,"ntime":1700001703,"order_key":"09:miner-9","p2mr_program_hex":"a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7","share_difficulty":2,"share_id":"share-1703","share_seq":1703,"template_height":800053},{"accepted_at_ms":89975,"credit_policy":null,"job_id":"job-1728","job_issued_at_ms":89973,"miner_id":"miner-1","network_difficulty":1000011,"ntime":1700001728,"order_key":"01:miner-1","p2mr_program_hex":"c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0","share_difficulty":3,"share_id":"share-1728","share_seq":1728,"template_height":800054},{"accepted_at_ms":89979,"credit_policy":null,"job_id":"job-653","job_issued_at_ms":89971,"miner_id":"miner-4","network_difficulty":1000007,"ntime":1700000653,"order_key":"04:miner-4","p2mr_program_hex":"8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d","share_difficulty":7,"share_id":"share-653","share_seq":653,"template_height":800020},{"accepted_at_ms":89986,"credit_policy":null,"job_id":"job-1774","job_issued_at_ms":89980,"miner_id":"miner-3","network_difficulty":1000006,"ntime":1700001774,"order_key":"03:miner-3","p2mr_program_hex":"eeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeee","share_difficulty":19,"share_id":"share-1774","share_seq":1774,"template_height":800055},{"accepted_at_ms":89972,"credit_policy":"stale-grace","job_id":"job-638","job_issued_at_ms":89973,"miner_id":"miner-0","network_difficulty":1000009,"ntime":1700000638,"order_key":"00:miner-0","p2mr_program_hex":"7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e","share_difficulty":2,"share_id":"share-638","share_seq":638,"template_height":800019},{"accepted_at_ms":89979,"credit_policy":"stale-grace","job_id":"job-56","job_issued_at_ms":89972,"miner_id":"miner-1","network_difficulty":1000005,"ntime":1700000056,"order_key":"01:miner-1","p2mr_program_hex":"3838383838383838383838383838383838383838383838383838383838383838","share_difficulty":2,"share_id":"share-56","share_seq":56,"template_height":800001},{"accepted_at_ms":89990,"credit_policy":null,"job_id":"job-1448","job_issued_at_ms":89982,"miner_id":"miner-7","network_difficulty":1000003,"ntime":1700001448,"order_key":"07:miner-7","p2mr_program_hex":"a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8","share_difficulty":1,"share_id":"share-1448","share_seq":1448,"template_height":800045},{"accepted_at_ms":89994,"credit_policy":null,"job_id":"job-1754","job_issued_at_ms":89998,"miner_id":"miner-5","network_difficulty":1000003,"ntime":1700001754,"order_key":"05:miner-5","p2mr_program_hex":"dadadadadadadadadadadadadadadadadadadadadadadadadadadadadadadada","share_difficulty":19,"share_id":"share-1754","share_seq":1754,"template_height":800054},{"accepted_at_ms":89996,"credit_policy":null,"job_id":"job-413","job_issued_at_ms":89983,"miner_id":"miner-6","network_difficulty":1000005,"ntime":1700000413,"order_key":"06:miner-6","p2mr_program_hex":"9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d","share_difficulty":19,"share_id":"share-413","share_seq":413,"template_height":800012},{"accepted_at_ms":89999,"credit_policy":null,"job_id":"job-1102","job_issued_at_ms":89978,"miner_id":"miner-2","network_difficulty":1000014,"ntime":1700001102,"order_key":"02:miner-2","p2mr_program_hex":"4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e","share_difficulty":1,"share_id":"share-1102","share_seq":1102,"template_height":800034},{"accepted_at_ms":89982,"credit_policy":null,"job_id":"job-1309","job_issued_at_ms":89991,"miner_id":"miner-0","network_difficulty":1000000,"ntime":1700001309,"order_key":"00:miner-0","p2mr_program_hex":"1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d","share_difficulty":1,"share_id":"share-1309","share_seq":1309,"template_height":800040},{"accepted_at_ms":89984,"credit_policy":null,"job_id":"job-879","job_issued_at_ms":89996,"miner_id":"miner-10","network_difficulty":1000012,"ntime":1700000879,"order_key":"10:miner-10","p2mr_program_hex":"6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f","share_difficulty":2,"share_id":"share-879","share_seq":879,"template_height":800027},{"accepted_at_ms":89977,"credit_policy":null,"job_id":"job-1690","job_issued_at_ms":89971,"miner_id":"miner-7","network_difficulty":1000007,"ntime":1700001690,"order_key":"07:miner-7","p2mr_program_hex":"9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a","share_difficulty":7,"share_id":"share-1690","share_seq":1690,"template_height":800052},{"accepted_at_ms":89976,"credit_policy":null,"job_id":"job-428","job_issued_at_ms":89977,"miner_id":"miner-10","network_difficulty":1000003,"ntime":1700000428,"order_key":"10:miner-10","p2mr_program_hex":"acacacacacacacacacacacacacacacacacacacacacacacacacacacacacacacac","share_difficulty":19,"share_id":"share-428","share_seq":428,"template_height":800013},{"accepted_at_ms":89974,"credit_policy":"stale-grace","job_id":"job-860","job_issued_at_ms":89998,"miner_id":"miner-2","network_difficulty":1000010,"ntime":1700000860,"order_key":"02:miner-2","p2mr_program_hex":"5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c","share_difficulty":3,"share_id":"share-860","share_seq":860,"template_height":800026},{"accepted_at_ms":89998,"credit_policy":null,"job_id":"job-210","job_issued_at_ms":89989,"miner_id":"miner-1","network_difficulty":1000006,"ntime":1700000210,"order_key":"01:miner-1","p2mr_program_hex":"d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2","share_difficulty":3,"share_id":"share-210","share_seq":210,"template_height":800006},{"accepted_at_ms":89981,"credit_policy":null,"job_id":"job-723","job_issued_at_ms":89988,"miner_id":"miner-8","network_difficulty":1000009,"ntime":1700000723,"order_key":"08:miner-8","p2mr_program_hex":"d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3","share_difficulty":3,"share_id":"share-723","share_seq":723,"template_height":800022},{"accepted_at_ms":89970,"credit_policy":null,"job_id":"job-1332","job_issued_at_ms":89996,"miner_id":"miner-1","network_difficulty":1000006,"ntime":1700001332,"order_key":"01:miner-1","p2mr_program_hex":"3434343434343434343434343434343434343434343434343434343434343434","share_difficulty":3,"share_id":"share-1332","share_seq":1332,"template_height":800041},{"accepted_at_ms":89972,"credit_policy":null,"job_id":"job-1528","job_issued_at_ms":89975,"miner_id":"miner-10","network_difficulty":1000015,"ntime":1700001528,"order_key":"10:miner-10","p2mr_program_hex":"f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8","share_difficulty":7,"share_id":"share-1528","share_seq":1528,"template_height":800047},{"accepted_at_ms":89984,"credit_policy":null,"job_id":"job-1711","job_issued_at_ms":89999,"miner_id":"miner-6","network_difficulty":1000011,"ntime":1700001711,"order_key":"06:miner-6","p2mr_program_hex":"afafafafafafafafafafafafafafafafafafafafafafafafafafafafafafafaf","share_difficulty":1,"share_id":"share-1711","share_seq":1711,"template_height":800053},{"accepted_at_ms":89992,"credit_policy":null,"job_id":"job-336","job_issued_at_ms":89991,"miner_id":"miner-6","network_difficulty":1000013,"ntime":1700000336,"order_key":"06:miner-6","p2mr_program_hex":"5050505050505050505050505050505050505050505050505050505050505050","share_difficulty":1,"share_id":"share-336","share_seq":336,"template_height":800010},{"accepted_at_ms":89990,"credit_policy":null,"job_id":"job-842","job_issued_at_ms":89974,"miner_id":"miner-6","network_difficulty":1000009,"ntime":1700000842,"order_key":"06:miner-6","p2mr_program_hex":"4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a","share_difficulty":19,"share_id":"share-842","share_seq":842,"template_height":800026},{"accepted_at_ms":90000,"credit_policy":null,"job_id":"job-126","job_issued_at_ms":89989,"miner_id":"miner-5","network_difficulty":1000007,"ntime":1700000126,"order_key":"05:miner-5","p2mr_program_hex":"7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e","share_difficulty":19,"share_id":"share-126","share_seq":126,"template_height":800003},{"accepted_at_ms":89993,"credit_policy":null,"job_id":"job-212","job_issued_at_ms":89982,"miner_id":"miner-3","network_difficulty":1000008,"ntime":1700000212,"order_key":"03:miner-3","p2mr_program_hex":"d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4","share_difficulty":2,"share_id":"share-212","share_seq":212,"template_height":800006},{"accepted_at_ms":89987,"credit_policy":null,"job_id":"job-174","job_issued_at_ms":89977,"miner_id":"miner-9","network_difficulty":1000004,"ntime":1700000174,"order_key":"09:miner-9","p2mr_program_hex":"aeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeae","share_difficulty":19,"share_id":"share-174","share_seq":174,"template_height":800005},{"accepted_at_ms":89977,"credit_policy":null,"job_id":"job-164","job_issued_at_ms":89974,"miner_id":"miner-10","network_difficulty":1000011,"ntime":1700000164,"order_key":"10:miner-10","p2mr_program_hex":"a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4","share_difficulty":7,"share_id":"share-164","share_seq":164,"template_height":800005},{"accepted_at_ms":89985,"credit_policy":null,"job_id":"job-51","job_issued_at_ms":89975,"miner_id":"miner-7","network_difficulty":1000000,"ntime":1700000051,"order_key":"07:miner-7","p2mr_program_hex":"3333333333333333333333333333333333333333333333333333333333333333","share_difficulty":19,"share_id":"share-51","share_seq":51,"template_height":800001},{"accepted_at_ms":89998,"credit_policy":null,"job_id":"job-1236","job_issued_at_ms":89985,"miner_id":"miner-4","network_difficulty":1000012,"ntime":1700001236,"order_key":"04:miner-4","p2mr_program_hex":"d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4","share_difficulty":19,"share_id":"share-1236","share_seq":1236,"template_height":800038},{"accepted_at_ms":89984,"credit_policy":null,"job_id":"job-878","job_issued_at_ms":89987,"miner_id":"miner-9","network_difficulty":1000011,"ntime":1700000878,"order_key":"09:miner-9","p2mr_program_hex":"6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e","share_difficulty":19,"share_id":"share-878","share_seq":878,"template_height":800027},{"accepted_at_ms":89994,"credit_policy":"stale-grace","job_id":"job-79","job_issued_at_ms":89997,"miner_id":"miner-2","network_difficulty":1000011,"ntime":1700000079,"order_key":"02:miner-2","p2mr_program_hex":"4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f","share_difficulty":19,"share_id":"share-79","share_seq":79,"template_height":800002},{"accepted_at_ms":89975,"credit_policy":null,"job_id":"job-92","job_issued_at_ms":89990,"miner_id":"miner-4","network_difficulty":1000007,"ntime":1700000092,"order_key":"04:miner-4","p2mr_program_hex":"5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c","share_difficulty":2,"share_id":"share-92","share_seq":92,"template_height":800002},{"accepted_at_ms":89981,"credit_policy":null,"job_id":"job-426","job_issued_at_ms":89997,"miner_id":"miner-8","network_difficulty":1000001,"ntime":1700000426,"order_key":"08:miner-8","p2mr_program_hex":"aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa","share_difficulty":1,"share_id":"share-426","share_seq":426,"template_height":800013},{"accepted_at_ms":89974,"credit_policy":"stale-grace","job_id":"job-1770","job_issued_at_ms":89979,"miner_id":"miner-10","network_difficulty":1000002,"ntime":1700001770,"order_key":"10:miner-10","p2mr_program_hex":"eaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaea","share_difficulty":1,"share_id":"share-1770","share_seq":1770,"template_height":800055},{"accepted_at_ms":89984,"credit_policy":"stale-grace","job_id":"job-97","job_issued_at_ms":89987,"miner_id":"miner-9","network_difficulty":1000012,"ntime":1700000097,"order_key":"09:miner-9","p2mr_program_hex":"6161616161616161616161616161616161616161616161616161616161616161","share_difficulty":3,"share_id":"share-97","share_seq":97,"template_height":800003},{"accepted_at_ms":89971,"credit_policy":null,"job_id":"job-214","job_issued_at_ms":89971,"miner_id":"miner-5","network_difficulty":1000010,"ntime":1700000214,"order_key":"05:miner-5","p2mr_program_hex":"d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6","share_difficulty":1,"share_id":"share-214","share_seq":214,"template_height":800006},{"accepted_at_ms":89995,"credit_policy":"stale-grace","job_id":"job-836","job_issued_at_ms":89992,"miner_id":"miner-0","network_difficulty":1000003,"ntime":1700000836,"order_key":"00:miner-0","p2mr_program_hex":"4444444444444444444444444444444444444444444444444444444444444444","share_difficulty":2,"share_id":"share-836","share_seq":836,"template_height":800026},{"accepted_at_ms":89995,"credit_policy":null,"job_id":"job-516","job_issued_at_ms":89974,"miner_id":"miner-10","network_difficulty":1000006,"ntime":1700000516,"order_key":"10:miner-10","p2mr_program_hex":"0404040404040404040404040404040404040404040404040404040404040404","share_difficulty":2,"share_id":"share-516","share_seq":516,"template_height":800016},{"accepted_at_ms":89973,"credit_policy":null,"job_id":"job-630","job_issued_at_ms":89999,"miner_id":"miner-3","network_difficulty":1000001,"ntime":1700000630,"order_key":"03:miner-3","p2mr_program_hex":"7676767676767676767676767676767676767676767676767676767676767676","share_difficulty":2,"share_id":"share-630","share_seq":630,"template_height":800019},{"accepted_at_ms":89996,"credit_policy":null,"job_id":"job-1435","job_issued_at_ms":90000,"miner_id":"miner-5","network_difficulty":1000007,"ntime":1700001435,"order_key":"05:miner-5","p2mr_program_hex":"9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b","share_difficulty":7,"share_id":"share-1435","share_seq":1435,"template_height":800044},{"accepted_at_ms":89973,"credit_policy":null,"job_id":"job-794","job_issued_at_ms":89988,"miner_id":"miner-2","network_difficulty":1000012,"ntime":1700000794,"order_key":"02:miner-2","p2mr_program_hex":"1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a","share_difficulty":1,"share_id":"share-794","share_seq":794,"template_height":800024},{"accepted_at_ms":89994,"credit_policy":null,"job_id":"job-512","job_issued_at_ms":89976,"miner_id":"miner-6","network_difficulty":1000002,"ntime":1700000512,"order_key":"06:miner-6","p2mr_program_hex":"0000000000000000000000000000000000000000000000000000000000000000","share_difficulty":1,"share_id":"share-512","share_seq":512,"template_height":800016},{"accepted_at_ms":90000,"credit_policy":null,"job_id":"job-1511","job_issued_at_ms":89993,"miner_id":"miner-4","network_difficulty":1000015,"ntime":1700001511,"order_key":"04:miner-4","p2mr_program_hex":"e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7","share_difficulty":2,"share_id":"share-1511","share_seq":1511,"template_height":800047},{"accepted_at_ms":89983,"credit_policy":null,"job_id":"job-1433","job_issued_at_ms":89974,"miner_id":"miner-3","network_difficulty":1000005,"ntime":1700001433,"order_key":"03:miner-3","p2mr_program_hex":"9999999999999999999999999999999999999999999999999999999999999999","share_difficulty":2,"share_id":"share-1433","share_seq":1433,"template_height":800044},{"accepted_at_ms":89987,"credit_policy":null,"job_id":"job-588","job_issued_at_ms":89973,"miner_id":"miner-5","network_difficulty":1000010,"ntime":1700000588,"order_key":"05:miner-5","p2mr_program_hex":"4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c","share_difficulty":7,"share_id":"share-588","share_seq":588,"template_height":800018},{"accepted_at_ms":89994,"credit_policy":null,"job_id":"job-607","job_issued_at_ms":89978,"miner_id":"miner-2","network_difficulty":1000012,"ntime":1700000607,"order_key":"02:miner-2","p2mr_program_hex":"5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f","share_difficulty":7,"share_id":"share-607","share_seq":607,"template_height":800018},{"accepted_at_ms":89976,"credit_policy":null,"job_id":"job-1591","job_issued_at_ms":89970,"miner_id":"miner-7","network_difficulty":1000010,"ntime":1700001591,"order_key":"07:miner-7","p2mr_program_hex":"3737373737373737373737373737373737373737373737373737373737373737","share_difficulty":1,"share_id":"share-1591","share_seq":1591,"template_height":800049},{"accepted_at_ms":89977,"credit_policy":null,"job_id":"job-832","job_issued_at_ms":89998,"miner_id":"miner-7","network_difficulty":1000016,"ntime":1700000832,"order_key":"07:miner-7","p2mr_program_hex":"4040404040404040404040404040404040404040404040404040404040404040","share_difficulty":1,"share_id":"share-832","share_seq":832,"template_height":800026},{"accepted_at_ms":89977,"credit_policy":null,"job_id":"job-7","job_issued_at_ms":89998,"miner_id":"miner-7","network_difficulty":1000007,"ntime":1700000007,"order_key":"07:miner-7","p2mr_program_hex":"0707070707070707070707070707070707070707070707070707070707070707","share_difficulty":3,"share_id":"share-7","share_seq":7,"template_height":800000},{"accepted_at_ms":89985,"credit_policy":null,"job_id":"job-1374","job_issued_at_ms":89976,"miner_id":"miner-10","network_difficulty":1000014,"ntime":1700001374,"order_key":"10:miner-10","p2mr_program_hex":"5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e","share_difficulty":3,"share_id":"share-1374","share_seq":1374,"template_height":800042},{"accepted_at_ms":89981,"credit_policy":"stale-grace","job_id":"job-1051","job_issued_at_ms":89994,"miner_id":"miner-6","network_difficulty":1000014,"ntime":1700001051,"order_key":"06:miner-6","p2mr_program_hex":"1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b","share_difficulty":7,"share_id":"share-1051","share_seq":1051,"template_height":800032},{"accepted_at_ms":89976,"credit_policy":null,"job_id":"job-1667","job_issued_at_ms":89987,"miner_id":"miner-6","network_difficulty":1000001,"ntime":1700001667,"order_key":"06:miner-6","p2mr_program_hex":"8383838383838383838383838383838383838383838383838383838383838383","share_difficulty":7,"share_id":"share-1667","share_seq":1667,"template_height":800052},{"accepted_at_ms":89991,"credit_policy":"stale-grace","job_id":"job-730","job_issued_at_ms":89999,"miner_id":"miner-4","network_difficulty":1000016,"ntime":1700000730,"order_key":"04:miner-4","p2mr_program_hex":"dadadadadadadadadadadadadadadadadadadadadadadadadadadadadadadada","share_difficulty":3,"share_id":"share-730","share_seq":730,"template_height":800022},{"accepted_at_ms":89977,"credit_policy":null,"job_id":"job-373","job_issued_at_ms":89990,"miner_id":"miner-10","network_difficulty":1000016,"ntime":1700000373,"order_key":"10:miner-10","p2mr_program_hex":"7575757575757575757575757575757575757575757575757575757575757575","share_difficulty":3,"share_id":"share-373","share_seq":373,"template_height":800011},{"accepted_at_ms":89988,"credit_policy":null,"job_id":"job-666","job_issued_at_ms":89995,"miner_id":"miner-6","network_difficulty":1000003,"ntime":1700000666,"order_key":"06:miner-6","p2mr_program_hex":"9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a","share_difficulty":1,"share_id":"share-666","share_seq":666,"template_height":800020},{"accepted_at_ms":89998,"credit_policy":null,"job_id":"job-229","job_issued_at_ms":89999,"miner_id":"miner-9","network_difficulty":1000008,"ntime":1700000229,"order_key":"09:miner-9","p2mr_program_hex":"e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5","share_difficulty":19,"share_id":"share-229","share_seq":229,"template_height":800007},{"accepted_at_ms":89983,"credit_policy":null,"job_id":"job-1768","job_issued_at_ms":89996,"miner_id":"miner-8","network_difficulty":1000000,"ntime":1700001768,"order_key":"08:miner-8","p2mr_program_hex":"e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8","share_difficulty":19,"share_id":"share-1768","share_seq":1768,"template_height":800055},{"accepted_at_ms":90000,"credit_policy":null,"job_id":"job-596","job_issued_at_ms":89978,"miner_id":"miner-2","network_difficulty":1000001,"ntime":1700000596,"order_key":"02:miner-2","p2mr_program_hex":"5454545454545454545454545454545454545454545454545454545454545454","share_difficulty":3,"share_id":"share-596","share_seq":596,"template_height":800018},{"accepted_at_ms":89978,"credit_policy":"stale-grace","job_id":"job-414","job_issued_at_ms":89976,"miner_id":"miner-7","network_difficulty":1000006,"ntime":1700000414,"order_key":"07:miner-7","p2mr_program_hex":"9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e","share_difficulty":3,"share_id":"share-414","share_seq":414,"template_height":800012},{"accepted_at_ms":89980,"credit_policy":"stale-grace","job_id":"job-1021","job_issued_at_ms":89982,"miner_id":"miner-9","network_difficulty":1000001,"ntime":1700001021,"order_key":"09:miner-9","p2mr_program_hex":"fdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfd","share_difficulty":2,"share_id":"share-1021","share_seq":1021,"template_height":800031},{"accepted_at_ms":89994,"credit_policy":null,"job_id":"job-1618","job_issued_at_ms":89993,"miner_id":"miner-1","network_difficulty":1000003,"ntime":1700001618,"order_key":"01:miner-1","p2mr_program_hex":"5252525252525252525252525252525252525252525252525252525252525252","share_difficulty":19,"share_id":"share-1618","share_seq":1618,"template_height":800050},{"accepted_at_ms":89985,"credit_policy":null,"job_id":"job-33","job_issued_at_ms":89994,"miner_id":"miner-0","network_difficulty":1000016,"ntime":1700000033,"order_key":"00:miner-0","p2mr_program_hex":"2121212121212121212121212121212121212121212121212121212121212121","share_difficulty":2,"share_id":"share-33","share_seq":33,"template_height":800001},{"accepted_at_ms":89975,"credit_policy":null,"job_id":"job-104","job_issued_at_ms":89996,"miner_id":"miner-5","network_difficulty":1000002,"ntime":1700000104,"order_key":"05:miner-5","p2mr_program_hex":"6868686868686868686868686868686868686868686868686868686868686868","share_difficulty":3,"share_id":"share-104","share_seq":104,"template_height":800003},{"accepted_at_ms":89992,"credit_policy":null,"job_id":"job-1573","job_issued_at_ms":89970,"miner_id":"miner-0","network_difficulty":1000009,"ntime":1700001573,"order_key":"00:miner-0","p2mr_program_hex":"2525252525252525252525252525252525252525252525252525252525252525","share_difficulty":19,"share_id":"share-1573","share_seq":1573,"template_height":800049},{"accepted_at_ms":89989,"credit_policy":null,"job_id":"job-707","job_issued_at_ms":89991,"miner_id":"miner-3","network_difficulty":1000010,"ntime":1700000707,"order_key":"03:miner-3","p2mr_program_hex":"c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3","share_difficulty":3,"share_id":"share-707","share_seq":707,"template_height":800022},{"accepted_at_ms":89999,"credit_policy":"stale-grace","job_id":"job-302","job_issued_at_ms":89988,"miner_id":"miner-5","network_difficulty":1000013,"ntime":1700000302,"order_key":"05:miner-5","p2mr_program_hex":"2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e","share_difficulty":7,"share_id":"share-302","share_seq":302,"template_height":800009},{"accepted_at_ms":89971,"credit_policy":"stale-grace","job_id":"job-47","job_issued_at_ms":89989,"miner_id":"miner-3","network_difficulty":1000013,"ntime":1700000047,"order_key":"03:miner-3","p2mr_program_hex":"2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f","share_difficulty":1,"share_id":"share-47","share_seq":47,"template_height":800001},{"accepted_at_ms":89971,"credit_policy":"stale-grace","job_id":"job-1560","job_issued_at_ms":89972,"miner_id":"miner-9","network_difficulty":1000013,"ntime":1700001560,"order_key":"09:miner-9","p2mr_program_hex":"1818181818181818181818181818181818181818181818181818181818181818","share_difficulty":2,"share_id":"share-1560","share_seq":1560,"template_height":800048},{"accepted_at_ms":89992,"credit_policy":null,"job_id":"job-125","job_issued_at_ms":89999,"miner_id":"miner-4","network_difficulty":1000006,"ntime":1700000125,"order_key":"04:miner-4","p2mr_program_hex":"7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d","share_difficulty":19,"share_id":"share-125","share_seq":125,"template_height":800003},{"accepted_at_ms":89988,"credit_policy":null,"job_id":"job-353","job_issued_at_ms":89972,"miner_id":"miner-1","network_difficulty":1000013,"ntime":1700000353,"order_key":"01:miner-1","p2mr_program_hex":"6161616161616161616161616161616161616161616161616161616161616161","share_difficulty":7,"share_id":"share-353","share_seq":353,"template_height":800011},{"accepted_at_ms":89993,"credit_policy":null,"job_id":"job-1701","job_issued_at_ms":89995,"miner_id":"miner-7","network_difficulty":1000001,"ntime":1700001701,"order_key":"07:miner-7","p2mr_program_hex":"a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5","share_difficulty":3,"share_id":"share-1701","share_seq":1701,"template_height":800053},{"accepted_at_ms":89988,"credit_policy":null,"job_id":"job-492","job_issued_at_ms":89978,"miner_id":"miner-8","network_difficulty":1000016,"ntime":1700000492,"order_key":"08:miner-8","p2mr_program_hex":"ecececececececececececececececececececececececececececececececec","share_difficulty":2,"share_id":"share-492","share_seq":492,"template_height":800015},{"accepted_at_ms":89998,"credit_policy":null,"job_id":"job-1016","job_issued_at_ms":89975,"miner_id":"miner-4","network_difficulty":1000013,"ntime":1700001016,"order_key":"04:miner-4","p2mr_program_hex":"f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8","share_difficulty":7,"share_id":"share-1016","share_seq":1016,"template_height":800031},{"accepted_at_ms":89973,"credit_policy":null,"job_id":"job-1194","job_issued_at_ms":89984,"miner_id":"miner-6","network_difficulty":1000004,"ntime":1700001194,"order_key":"06:miner-6","p2mr_program_hex":"aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa","share_difficulty":2,"share_id":"share-1194","share_seq":1194,"template_height":800037},{"accepted_at_ms":89988,"credit_policy":null,"job_id":"job-1699","job_issued_at_ms":89978,"miner_id":"miner-5","network_difficulty":1000016,"ntime":1700001699,"order_key":"05:miner-5","p2mr_program_hex":"a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3","share_difficulty":3,"share_id":"share-1699","share_seq":1699,"template_height":800053},{"accepted_at_ms":89976,"credit_policy":null,"job_id":"job-713","job_issued_at_ms":89998,"miner_id":"miner-9","network_difficulty":1000016,"ntime":1700000713,"order_key":"09:miner-9","p2mr_program_hex":"c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9","share_difficulty":19,"share_id":"share-713","share_seq":713,"template_height":800022},{"accepted_at_ms":89990,"credit_policy":null,"job_id":"job-1629","job_issued_at_ms":89977,"miner_id":"miner-1","network_difficulty":1000014,"ntime":1700001629,"order_key":"01:miner-1","p2mr_program_hex":"5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d","share_difficulty":7,"share_id":"share-1629","share_seq":1629,"template_height":800050},{"accepted_at_ms":89981,"credit_policy":null,"job_id":"job-217","job_issued_at_ms":89985,"miner_id":"miner-8","network_difficulty":1000013,"ntime":1700000217,"order_key":"08:miner-8","p2mr_program_hex":"d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9","share_difficulty":7,"share_id":"share-217","share_seq":217,"template_height":800006},{"accepted_at_ms":89998,"credit_policy":null,"job_id":"job-1513","job_issued_at_ms":89996,"miner_id":"miner-6","network_difficulty":1000000,"ntime":1700001513,"order_key":"06:miner-6","p2mr_program_hex":"e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9","share_difficulty":1,"share_id":"share-1513","share_seq":1513,"template_height":800047},{"accepted_at_ms":89990,"credit_policy":null,"job_id":"job-182","job_issued_at_ms":89980,"miner_id":"miner-6","network_difficulty":1000012,"ntime":1700000182,"order_key":"06:miner-6","p2mr_program_hex":"b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6","share_difficulty":19,"share_id":"share-182","share_seq":182,"template_height":800005},{"accepted_at_ms":89991,"credit_policy":null,"job_id":"job-754","job_issued_at_ms":89978,"miner_id":"miner-6","network_difficulty":1000006,"ntime":1700000754,"order_key":"06:miner-6","p2mr_program_hex":"f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2","share_difficulty":7,"share_id":"share-754","share_seq":754,"template_height":800023},{"accepted_at_ms":89975,"credit_policy":null,"job_id":"job-1367","job_issued_at_ms":89984,"miner_id":"miner-3","network_difficulty":1000007,"ntime":1700001367,"order_key":"03:miner-3","p2mr_program_hex":"5757575757575757575757575757575757575757575757575757575757575757","share_difficulty":19,"share_id":"share-1367","share_seq":1367,"template_height":800042},{"accepted_at_ms":89978,"credit_policy":null,"job_id":"job-744","job_issued_at_ms":89995,"miner_id":"miner-7","network_difficulty":1000013,"ntime":1700000744,"order_key":"07:miner-7","p2mr_program_hex":"e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8","share_difficulty":7,"share_id":"share-744","share_seq":744,"template_height":800023},{"accepted_at_ms":89982,"credit_policy":null,"job_id":"job-1557","job_issued_at_ms":89970,"miner_id":"miner-6","network_difficulty":1000010,"ntime":1700001557,"order_key":"06:miner-6","p2mr_program_hex":"1515151515151515151515151515151515151515151515151515151515151515","share_difficulty":3,"share_id":"share-1557","share_seq":1557,"template_height":800048},{"accepted_at_ms":89971,"credit_policy":null,"job_id":"job-525","job_issued_at_ms":89998,"miner_id":"miner-8","network_difficulty":1000015,"ntime":1700000525,"order_key":"08:miner-8","p2mr_program_hex":"0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d","share_difficulty":7,"share_id":"share-525","share_seq":525,"template_height":800016},{"accepted_at_ms":89986,"credit_policy":"stale-grace","job_id":"job-1622","job_issued_at_ms":89979,"miner_id":"miner-5","network_difficulty":1000007,"ntime":1700001622,"order_key":"05:miner-5","p2mr_program_hex":"5656565656565656565656565656565656565656565656565656565656565656","share_difficulty":19,"share_id":"share-1622","share_seq":1622,"template_height":800050},{"accepted_at_ms":89998,"credit_policy":null,"job_id":"job-1040","job_issued_at_ms":89980,"miner_id":"miner-6","network_difficulty":1000003,"ntime":1700001040,"order_key":"06:miner-6","p2mr_program_hex":"1010101010101010101010101010101010101010101010101010101010101010","share_difficulty":3,"share_id":"share-1040","share_seq":1040,"template_height":800032},{"accepted_at_ms":89974,"credit_policy":null,"job_id":"job-1530","job_issued_at_ms":90000,"miner_id":"miner-1","network_difficulty":1000000,"ntime":1700001530,"order_key":"01:miner-1","p2mr_program_hex":"fafafafafafafafafafafafafafafafafafafafafafafafafafafafafafafafa","share_difficulty":19,"share_id":"share-1530","share_seq":1530,"template_height":800047},{"accepted_at_ms":89975,"credit_policy":null,"job_id":"job-1594","job_issued_at_ms":89973,"miner_id":"miner-10","network_difficulty":1000013,"ntime":1700001594,"order_key":"10:miner-10","p2mr_program_hex":"3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a","share_difficulty":1,"share_id":"share-1594","share_seq":1594,"template_height":800049},{"accepted_at_ms":89974,"credit_policy":"stale-grace","job_id":"job-34","job_issued_at_ms":89982,"miner_id":"miner-1","network_difficulty":1000000,"ntime":1700000034,"order_key":"01:miner-1","p2mr_program_hex":"2222222222222222222222222222222222222222222222222222222222222222","share_difficulty":3,"share_id":"share-34","share_seq":34,"template_height":800001},{"accepted_at_ms":89987,"credit_policy":null,"job_id":"job-557","job_issued_at_ms":89992,"miner_id":"miner-7","network_difficulty":1000013,"ntime":1700000557,"order_key":"07:miner-7","p2mr_program_hex":"2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d","share_difficulty":3,"share_id":"share-557","share_seq":557,"template_height":800017},{"accepted_at_ms":89975,"credit_policy":"stale-grace","job_id":"job-1398","job_issued_at_ms":89982,"miner_id":"miner-1","network_difficulty":1000004,"ntime":1700001398,"order_key":"01:miner-1","p2mr_program_hex":"7676767676767676767676767676767676767676767676767676767676767676","share_difficulty":19,"share_id":"share-1398","share_seq":1398,"template_height":800043},{"accepted_at_ms":89979,"credit_policy":"stale-grace","job_id":"job-1393","job_issued_at_ms":89992,"miner_id":"miner-7","network_difficulty":1000016,"ntime":1700001393,"order_key":"07:miner-7","p2mr_program_hex":"7171717171717171717171717171717171717171717171717171717171717171","share_difficulty":1,"share_id":"share-1393","share_seq":1393,"template_height":800043},{"accepted_at_ms":89972,"credit_policy":null,"job_id":"job-295","job_issued_at_ms":89983,"miner_id":"miner-9","network_difficulty":1000006,"ntime":1700000295,"order_key":"09:miner-9","p2mr_program_hex":"2727272727272727272727272727272727272727272727272727272727272727","share_difficulty":3,"share_id":"share-295","share_seq":295,"template_height":800009},{"accepted_at_ms":90000,"credit_policy":null,"job_id":"job-1039","job_issued_at_ms":89986,"miner_id":"miner-5","network_difficulty":1000002,"ntime":1700001039,"order_key":"05:miner-5","p2mr_program_hex":"0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f","share_difficulty":1,"share_id":"share-1039","share_seq":1039,"template_height":800032},{"accepted_at_ms":89982,"credit_policy":null,"job_id":"job-1425","job_issued_at_ms":89998,"miner_id":"miner-6","network_difficulty":1000014,"ntime":1700001425,"order_key":"06:miner-6","p2mr_program_hex":"9191919191919191919191919191919191919191919191919191919191919191","share_difficulty":3,"share_id":"share-1425","share_seq":1425,"template_height":800044},{"accepted_at_ms":89976,"credit_policy":null,"job_id":"job-1587","job_issued_at_ms":89981,"miner_id":"miner-3","network_difficulty":1000006,"ntime":1700001587,"order_key":"03:miner-3","p2mr_program_hex":"3333333333333333333333333333333333333333333333333333333333333333","share_difficulty":1,"share_id":"share-1587","share_seq":1587,"template_height":800049},{"accepted_at_ms":89983,"credit_policy":"stale-grace","job_id":"job-1156","job_issued_at_ms":89975,"miner_id":"miner-1","network_difficulty":1000000,"ntime":1700001156,"order_key":"01:miner-1","p2mr_program_hex":"8484848484848484848484848484848484848484848484848484848484848484","share_difficulty":1,"share_id":"share-1156","share_seq":1156,"template_height":800036},{"accepted_at_ms":89979,"credit_policy":null,"job_id":"job-816","job_issued_at_ms":89998,"miner_id":"miner-2","network_difficulty":1000000,"ntime":1700000816,"order_key":"02:miner-2","p2mr_program_hex":"3030303030303030303030303030303030303030303030303030303030303030","share_difficulty":7,"share_id":"share-816","share_seq":816,"template_height":800025},{"accepted_at_ms":89972,"credit_policy":"stale-grace","job_id":"job-1683","job_issued_at_ms":89994,"miner_id":"miner-0","network_difficulty":1000000,"ntime":1700001683,"order_key":"00:miner-0","p2mr_program_hex":"9393939393939393939393939393939393939393939393939393939393939393","share_difficulty":1,"share_id":"share-1683","share_seq":1683,"template_height":800052},{"accepted_at_ms":89971,"credit_policy":null,"job_id":"job-41","job_issued_at_ms":89978,"miner_id":"miner-8","network_difficulty":1000007,"ntime":1700000041,"order_key":"08:miner-8","p2mr_program_hex":"2929292929292929292929292929292929292929292929292929292929292929","share_difficulty":19,"share_id":"share-41","share_seq":41,"template_height":800001},{"accepted_at_ms":89974,"credit_policy":null,"job_id":"job-481","job_issued_at_ms":89974,"miner_id":"miner-8","network_difficulty":1000005,"ntime":1700000481,"order_key":"08:miner-8","p2mr_program_hex":"e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1","share_difficulty":3,"share_id":"share-481","share_seq":481,"template_height":800015},{"accepted_at_ms":89971,"credit_policy":null,"job_id":"job-1592","job_issued_at_ms":89976,"miner_id":"miner-8","network_difficulty":1000011,"ntime":1700001592,"order_key":"08:miner-8","p2mr_program_hex":"3838383838383838383838383838383838383838383838383838383838383838","share_difficulty":3,"share_id":"share-1592","share_seq":1592,"template_height":800049},{"accepted_at_ms":89981,"credit_policy":null,"job_id":"job-121","job_issued_at_ms":89981,"miner_id":"miner-0","network_difficulty":1000002,"ntime":1700000121,"order_key":"00:miner-0","p2mr_program_hex":"7979797979797979797979797979797979797979797979797979797979797979","share_difficulty":1,"share_id":"share-121","share_seq":121,"template_height":800003},{"accepted_at_ms":89990,"credit_policy":null,"job_id":"job-1339","job_issued_at_ms":90000,"miner_id":"miner-8","network_difficulty":1000013,"ntime":1700001339,"order_key":"08:miner-8","p2mr_program_hex":"3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b","share_difficulty":1,"share_id":"share-1339","share_seq":1339,"template_height":800041},{"accepted_at_ms":89973,"credit_policy":null,"job_id":"job-138","job_issued_at_ms":89979,"miner_id":"miner-6","network_difficulty":1000002,"ntime":1700000138,"order_key":"06:miner-6","p2mr_program_hex":"8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a","share_difficulty":3,"share_id":"share-138","share_seq":138,"template_height":800004},{"accepted_at_ms":89976,"credit_policy":null,"job_id":"job-1144","job_issued_at_ms":89975,"miner_id":"miner-0","network_difficulty":1000005,"ntime":1700001144,"order_key":"00:miner-0","p2mr_program_hex":"7878787878787878787878787878787878787878787878787878787878787878","share_difficulty":2,"share_id":"share-1144","share_seq":1144,"template_height":800035},{"accepted_at_ms":89981,"credit_policy":null,"job_id":"job-918","job_issued_at_ms":89983,"miner_id":"miner-5","network_difficulty":1000000,"ntime":1700000918,"order_key":"05:miner-5","p2mr_program_hex":"9696969696969696969696969696969696969696969696969696969696969696","share_difficulty":7,"share_id":"share-918","share_seq":918,"template_height":800028},{"accepted_at_ms":89977,"credit_policy":null,"job_id":"job-694","job_issued_at_ms":89992,"miner_id":"miner-1","network_difficulty":1000014,"ntime":1700000694,"order_key":"01:miner-1","p2mr_program_hex":"b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6","share_difficulty":2,"share_id":"share-694","share_seq":694,"template_height":800021},{"accepted_at_ms":90000,"credit_policy":"stale-grace","job_id":"job-493","job_issued_at_ms":89974,"miner_id":"miner-9","network_difficulty":1000000,"ntime":1700000493,"order_key":"09:miner-9","p2mr_program_hex":"edededededededededededededededededededededededededededededededed","share_difficulty":19,"share_id":"share-493","share_seq":493,"template_height":800015},{"accepted_at_ms":89973,"credit_policy":null,"job_id":"job-496","job_issued_at_ms":89986,"miner_id":"miner-1","network_difficulty":1000003,"ntime":1700000496,"order_key":"01:miner-1","p2mr_program_hex":"f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0","share_difficulty":2,"share_id":"share-496","share_seq":496,"template_height":800015},{"accepted_at_ms":90000,"credit_policy":null,"job_id":"job-1407","job_issued_at_ms":89985,"miner_id":"miner-10","network_difficulty":1000013,"ntime":1700001407,"order_key":"10:miner-10","p2mr_program_hex":"7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f","share_difficulty":3,"share_id":"share-1407","share_seq":1407,"template_height":800043},{"accepted_at_ms":89985,"credit_policy":null,"job_id":"job-715","job_issued_at_ms":89984,"miner_id":"miner-0","network_difficulty":1000001,"ntime":1700000715,"order_key":"00:miner-0","p2mr_program_hex":"cbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcb","share_difficulty":19,"share_id":"share-715","share_seq":715,"template_height":800022},{"accepted_at_ms":89993,"credit_policy":null,"job_id":"job-165","job_issued_at_ms":89986,"miner_id":"miner-0","network_difficulty":1000012,"ntime":1700000165,"order_key":"00:miner-0","p2mr_program_hex":"a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5","share_difficulty":1,"share_id":"share-165","share_seq":165,"template_height":800005},{"accepted_at_ms":89984,"credit_policy":null,"job_id":"job-1472","job_issued_at_ms":89975,"miner_id":"miner-9","network_difficulty":1000010,"ntime":1700001472,"order_key":"09:miner-9","p2mr_program_hex":"c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0","share_difficulty":3,"share_id":"share-1472","share_seq":1472,"template_height":800046},{"accepted_at_ms":90000,"credit_policy":null,"job_id":"job-1623","job_issued_at_ms":89983,"miner_id":"miner-6","network_difficulty":1000008,"ntime":1700001623,"order_key":"06:miner-6","p2mr_program_hex":"5757575757575757575757575757575757575757575757575757575757575757","share_difficulty":2,"share_id":"share-1623","share_seq":1623,"template_height":800050},{"accepted_at_ms":89995,"credit_policy":null,"job_id":"job-442","job_issued_at_ms":89975,"miner_id":"miner-2","network_difficulty":1000000,"ntime":1700000442,"order_key":"02:miner-2","p2mr_program_hex":"babababababababababababababababababababababababababababababababa","share_difficulty":19,"share_id":"share-442","share_seq":442,"template_height":800013},{"accepted_at_ms":89998,"credit_policy":null,"job_id":"job-477","job_issued_at_ms":89991,"miner_id":"miner-4","network_difficulty":1000001,"ntime":1700000477,"order_key":"04:miner-4","p2mr_program_hex":"dddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddd","share_difficulty":7,"share_id":"share-477","share_seq":477,"template_height":800014},{"accepted_at_ms":89971,"credit_policy":null,"job_id":"job-1167","job_issued_at_ms":89979,"miner_id":"miner-1","network_difficulty":1000011,"ntime":1700001167,"order_key":"01:miner-1","p2mr_program_hex":"8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f","share_difficulty":19,"share_id":"share-1167","share_seq":1167,"template_height":800036},{"accepted_at_ms":89988,"credit_policy":null,"job_id":"job-1694","job_issued_at_ms":89984,"miner_id":"miner-0","network_difficulty":1000011,"ntime":1700001694,"order_key":"00:miner-0","p2mr_program_hex":"9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e","share_difficulty":2,"share_id":"share-1694","share_seq":1694,"template_height":800052},{"accepted_at_ms":89980,"credit_policy":null,"job_id":"job-1449","job_issued_at_ms":89985,"miner_id":"miner-8","network_difficulty":1000004,"ntime":1700001449,"order_key":"08:miner-8","p2mr_program_hex":"a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9","share_difficulty":1,"share_id":"share-1449","share_seq":1449,"template_height":800045},{"accepted_at_ms":90000,"credit_policy":null,"job_id":"job-390","job_issued_at_ms":89996,"miner_id":"miner-5","network_difficulty":1000016,"ntime":1700000390,"order_key":"05:miner-5","p2mr_program_hex":"8686868686868686868686868686868686868686868686868686868686868686","share_difficulty":19,"share_id":"share-390","share_seq":390,"template_height":800012},{"accepted_at_ms":89973,"credit_policy":null,"job_id":"job-1056","job_issued_at_ms":89979,"miner_id":"miner-0","network_difficulty":1000002,"ntime":1700001056,"order_key":"00:miner-0","p2mr_program_hex":"2020202020202020202020202020202020202020202020202020202020202020","share_difficulty":3,"share_id":"share-1056","share_seq":1056,"template_height":800033},{"accepted_at_ms":89985,"credit_policy":null,"job_id":"job-1437","job_issued_at_ms":89985,"miner_id":"miner-7","network_difficulty":1000009,"ntime":1700001437,"order_key":"07:miner-7","p2mr_program_hex":"9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d","share_difficulty":3,"share_id":"share-1437","share_seq":1437,"template_height":800044},{"accepted_at_ms":89974,"credit_policy":"stale-grace","job_id":"job-239","job_issued_at_ms":89995,"miner_id":"miner-8","network_difficulty":1000001,"ntime":1700000239,"order_key":"08:miner-8","p2mr_program_hex":"efefefefefefefefefefefefefefefefefefefefefefefefefefefefefefefef","share_difficulty":1,"share_id":"share-239","share_seq":239,"template_height":800007},{"accepted_at_ms":89984,"credit_policy":null,"job_id":"job-846","job_issued_at_ms":89982,"miner_id":"miner-10","network_difficulty":1000013,"ntime":1700000846,"order_key":"10:miner-10","p2mr_program_hex":"4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e","share_difficulty":7,"share_id":"share-846","share_seq":846,"template_height":800026},{"accepted_at_ms":89983,"credit_policy":"stale-grace","job_id":"job-1229","job_issued_at_ms":89982,"miner_id":"miner-8","network_difficulty":1000005,"ntime":1700001229,"order_key":"08:miner-8","p2mr_program_hex":"cdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcd","share_difficulty":2,"share_id":"share-1229","share_seq":1229,"template_height":800038},{"accepted_at_ms":89990,"credit_policy":null,"job_id":"job-1798","job_issued_at_ms":89979,"miner_id":"miner-5","network_difficulty":1000013,"ntime":1700001798,"order_key":"05:miner-5","p2mr_program_hex":"0606060606060606060606060606060606060606060606060606060606060606","share_difficulty":19,"share_id":"share-1798","share_seq":1798,"template_height":800056},{"accepted_at_ms":89999,"credit_policy":null,"job_id":"job-17","job_issued_at_ms":89977,"miner_id":"miner-6","network_difficulty":1000000,"ntime":1700000017,"order_key":"06:miner-6","p2mr_program_hex":"1111111111111111111111111111111111111111111111111111111111111111","share_difficulty":2,"share_id":"share-17","share_seq":17,"template_height":800000},{"accepted_at_ms":89990,"credit_policy":null,"job_id":"job-1447","job_issued_at_ms":89993,"miner_id":"miner-6","network_difficulty":1000002,"ntime":1700001447,"order_key":"06:miner-6","p2mr_program_hex":"a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7","share_difficulty":2,"share_id":"share-1447","share_seq":1447,"template_height":800045},{"accepted_at_ms":89975,"credit_policy":null,"job_id":"job-498","job_issued_at_ms":89980,"miner_id":"miner-3","network_difficulty":1000005,"ntime":1700000498,"order_key":"03:miner-3","p2mr_program_hex":"f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2","share_difficulty":1,"share_id":"share-498","share_seq":498,"template_height":800015},{"accepted_at_ms":89978,"credit_policy":null,"job_id":"job-1378","job_issued_at_ms":89997,"miner_id":"miner-3","network_difficulty":1000001,"ntime":1700001378,"order_key":"03:miner-3","p2mr_program_hex":"6262626262626262626262626262626262626262626262626262626262626262","share_difficulty":2,"share_id":"share-1378","share_seq":1378,"template_height":800043},{"accepted_at_ms":89987,"credit_policy":"stale-grace","job_id":"job-1776","job_issued_at_ms":89988,"miner_id":"miner-5","network_difficulty":1000008,"ntime":1700001776,"order_key":"05:miner-5","p2mr_program_hex":"f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0","share_difficulty":7,"share_id":"share-1776","share_seq":1776,"template_height":800055},{"accepted_at_ms":89986,"credit_policy":null,"job_id":"job-1744","job_issued_at_ms":89986,"miner_id":"miner-6","network_difficulty":1000010,"ntime":1700001744,"order_key":"06:miner-6","p2mr_program_hex":"d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0","share_difficulty":19,"share_id":"share-1744","share_seq":1744,"template_height":800054},{"accepted_at_ms":89977,"credit_policy":null,"job_id":"job-386","job_issued_at_ms":89970,"miner_id":"miner-1","network_difficulty":1000012,"ntime":1700000386,"order_key":"01:miner-1","p2mr_program_hex":"8282828282828282828282828282828282828282828282828282828282828282","share_difficulty":1,"share_id":"share-386","share_seq":386,"template_height":800012},{"accepted_at_ms":89972,"credit_policy":"stale-grace","job_id":"job-67","job_issued_at_ms":89989,"miner_id":"miner-1","network_difficulty":1000016,"ntime":1700000067,"order_key":"01:miner-1","p2mr_program_hex":"4343434343434343434343434343434343434343434343434343434343434343","share_difficulty":19,"share_id":"share-67","share_seq":67,"template_height":800002},{"accepted_at_ms":89983,"credit_policy":null,"job_id":"job-682","job_issued_at_ms":89994,"miner_id":"miner-0","network_difficulty":1000002,"ntime":1700000682,"order_key":"00:miner-0","p2mr_program_hex":"aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa","share_difficulty":7,"share_id":"share-682","share_seq":682,"template_height":800021},{"accepted_at_ms":89979,"credit_policy":"stale-grace","job_id":"job-224","job_issued_at_ms":89975,"miner_id":"miner-4","network_difficulty":1000003,"ntime":1700000224,"order_key":"04:miner-4","p2mr_program_hex":"e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0","share_difficulty":7,"share_id":"share-224","share_seq":224,"template_height":800007},{"accepted_at_ms":89973,"credit_policy":null,"job_id":"job-1502","job_issued_at_ms":89970,"miner_id":"miner-6","network_difficulty":1000006,"ntime":1700001502,"order_key":"06:miner-6","p2mr_program_hex":"dededededededededededededededededededededededededededededededede","share_difficulty":19,"share_id":"share-1502","share_seq":1502,"template_height":800046},{"accepted_at_ms":89997,"credit_policy":null,"job_id":"job-1279","job_issued_at_ms":89986,"miner_id":"miner-3","network_difficulty":1000004,"ntime":1700001279,"order_key":"03:miner-3","p2mr_program_hex":"ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff","share_difficulty":7,"share_id":"share-1279","share_seq":1279,"template_height":800039},{"accepted_at_ms":89988,"credit_policy":null,"job_id":"job-702","job_issued_at_ms":89998,"miner_id":"miner-9","network_difficulty":1000005,"ntime":1700000702,"order_key":"09:miner-9","p2mr_program_hex":"bebebebebebebebebebebebebebebebebebebebebebebebebebebebebebebebe","share_difficulty":19,"share_id":"share-702","share_seq":702,"template_height":800021},{"accepted_at_ms":89994,"credit_policy":"stale-grace","job_id":"job-1704","job_issued_at_ms":89997,"miner_id":"miner-10","network_difficulty":1000004,"ntime":1700001704,"order_key":"10:miner-10","p2mr_program_hex":"a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8","share_difficulty":2,"share_id":"share-1704","share_seq":1704,"template_height":800053},{"accepted_at_ms":89981,"credit_policy":null,"job_id":"job-592","job_issued_at_ms":89997,"miner_id":"miner-9","network_difficulty":1000014,"ntime":1700000592,"order_key":"09:miner-9","p2mr_program_hex":"5050505050505050505050505050505050505050505050505050505050505050","share_difficulty":3,"share_id":"share-592","share_seq":592,"template_height":800018},{"accepted_at_ms":89996,"credit_policy":"stale-grace","job_id":"job-1163","job_issued_at_ms":89972,"miner_id":"miner-8","network_difficulty":1000007,"ntime":1700001163,"order_key":"08:miner-8","p2mr_program_hex":"8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b","share_difficulty":1,"share_id":"share-1163","share_seq":1163,"template_height":800036},{"accepted_at_ms":89986,"credit_policy":"stale-grace","job_id":"job-1150","job_issued_at_ms":89984,"miner_id":"miner-6","network_difficulty":1000011,"ntime":1700001150,"order_key":"06:miner-6","p2mr_program_hex":"7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e","share_difficulty":7,"share_id":"share-1150","share_seq":1150,"template_height":800035},{"accepted_at_ms":89989,"credit_policy":null,"job_id":"job-1463","job_issued_at_ms":89989,"miner_id":"miner-0","network_difficulty":1000001,"ntime":1700001463,"order_key":"00:miner-0","p2mr_program_hex":"b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7","share_difficulty":19,"share_id":"share-1463","share_seq":1463,"template_height":800045},{"accepted_at_ms":89977,"credit_policy":null,"job_id":"job-397","job_issued_at_ms":89970,"miner_id":"miner-1","network_difficulty":1000006,"ntime":1700000397,"order_key":"01:miner-1","p2mr_program_hex":"8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d","share_difficulty":2,"share_id":"share-397","share_seq":397,"template_height":800012},{"accepted_at_ms":89996,"credit_policy":null,"job_id":"job-522","job_issued_at_ms":89992,"miner_id":"miner-5","network_difficulty":1000012,"ntime":1700000522,"order_key":"05:miner-5","p2mr_program_hex":"0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a","share_difficulty":7,"share_id":"share-522","share_seq":522,"template_height":800016},{"accepted_at_ms":89974,"credit_policy":"stale-grace","job_id":"job-1358","job_issued_at_ms":90000,"miner_id":"miner-5","network_difficulty":1000015,"ntime":1700001358,"order_key":"05:miner-5","p2mr_program_hex":"4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e","share_difficulty":1,"share_id":"share-1358","share_seq":1358,"template_height":800042},{"accepted_at_ms":89991,"credit_policy":null,"job_id":"job-1020","job_issued_at_ms":89991,"miner_id":"miner-8","network_difficulty":1000000,"ntime":1700001020,"order_key":"08:miner-8","p2mr_program_hex":"fcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfc","share_difficulty":3,"share_id":"share-1020","share_seq":1020,"template_height":800031},{"accepted_at_ms":89986,"credit_policy":null,"job_id":"job-451","job_issued_at_ms":89999,"miner_id":"miner-0","network_difficulty":1000009,"ntime":1700000451,"order_key":"00:miner-0","p2mr_program_hex":"c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3","share_difficulty":3,"share_id":"share-451","share_seq":451,"template_height":800014},{"accepted_at_ms":89981,"credit_policy":null,"job_id":"job-64","job_issued_at_ms":89984,"miner_id":"miner-9","network_difficulty":1000013,"ntime":1700000064,"order_key":"09:miner-9","p2mr_program_hex":"4040404040404040404040404040404040404040404040404040404040404040","share_difficulty":7,"share_id":"share-64","share_seq":64,"template_height":800002},{"accepted_at_ms":89978,"credit_policy":null,"job_id":"job-1403","job_issued_at_ms":89977,"miner_id":"miner-6","network_difficulty":1000009,"ntime":1700001403,"order_key":"06:miner-6","p2mr_program_hex":"7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b","share_difficulty":1,"share_id":"share-1403","share_seq":1403,"template_height":800043},{"accepted_at_ms":89984,"credit_policy":null,"job_id":"job-1777","job_issued_at_ms":89993,"miner_id":"miner-6","network_difficulty":1000009,"ntime":1700001777,"order_key":"06:miner-6","p2mr_program_hex":"f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1","share_difficulty":1,"share_id":"share-1777","share_seq":1777,"template_height":800055},{"accepted_at_ms":89991,"credit_policy":null,"job_id":"job-1103","job_issued_at_ms":89998,"miner_id":"miner-3","network_difficulty":1000015,"ntime":1700001103,"order_key":"03:miner-3","p2mr_program_hex":"4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f","share_difficulty":1,"share_id":"share-1103","share_seq":1103,"template_height":800034},{"accepted_at_ms":89986,"credit_policy":"stale-grace","job_id":"job-1720","job_issued_at_ms":89975,"miner_id":"miner-4","network_difficulty":1000003,"ntime":1700001720,"order_key":"04:miner-4","p2mr_program_hex":"b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8","share_difficulty":3,"share_id":"share-1720","share_seq":1720,"template_height":800053},{"accepted_at_ms":89998,"credit_policy":null,"job_id":"job-1390","job_issued_at_ms":89984,"miner_id":"miner-4","network_difficulty":1000013,"ntime":1700001390,"order_key":"04:miner-4","p2mr_program_hex":"6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e","share_difficulty":2,"share_id":"share-1390","share_seq":1390,"template_height":800043},{"accepted_at_ms":89972,"credit_policy":null,"job_id":"job-1737","job_issued_at_ms":89980,"miner_id":"miner-10","network_difficulty":1000003,"ntime":1700001737,"order_key":"10:miner-10","p2mr_program_hex":"c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9","share_difficulty":2,"share_id":"share-1737","share_seq":1737,"template_height":800054},{"accepted_at_ms":89994,"credit_policy":null,"job_id":"job-1012","job_issued_at_ms":89974,"miner_id":"miner-0","network_difficulty":1000009,"ntime":1700001012,"order_key":"00:miner-0","p2mr_program_hex":"f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4","share_difficulty":3,"share_id":"share-1012","share_seq":1012,"template_height":800031},{"accepted_at_ms":89990,"credit_policy":null,"job_id":"job-1070","job_issued_at_ms":89990,"miner_id":"miner-3","network_difficulty":1000016,"ntime":1700001070,"order_key":"03:miner-3","p2mr_program_hex":"2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e","share_difficulty":7,"share_id":"share-1070","share_seq":1070,"template_height":800033},{"accepted_at_ms":89982,"credit_policy":null,"job_id":"job-1009","job_issued_at_ms":89983,"miner_id":"miner-8","network_difficulty":1000006,"ntime":1700001009,"order_key":"08:miner-8","p2mr_program_hex":"f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1","share_difficulty":3,"share_id":"share-1009","share_seq":1009,"template_height":800031},{"accepted_at_ms":89993,"credit_policy":null,"job_id":"job-348","job_issued_at_ms":89981,"miner_id":"miner-7","network_difficulty":1000008,"ntime":1700000348,"order_key":"07:miner-7","p2mr_program_hex":"5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c","share_difficulty":2,"share_id":"share-348","share_seq":348,"template_height":800010},{"accepted_at_ms":89981,"credit_policy":null,"job_id":"job-2","job_issued_at_ms":89976,"miner_id":"miner-2","network_difficulty":1000002,"ntime":1700000002,"order_key":"02:miner-2","p2mr_program_hex":"0202020202020202020202020202020202020202020202020202020202020202","share_difficulty":1,"share_id":"share-2","share_seq":2,"template_height":800000},{"accepted_at_ms":89972,"credit_policy":"stale-grace","job_id":"job-1581","job_issued_at_ms":89987,"miner_id":"miner-8","network_difficulty":1000000,"ntime":1700001581,"order_key":"08:miner-8","p2mr_program_hex":"2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d","share_difficulty":3,"share_id":"share-1581","share_seq":1581,"template_height":800049},{"accepted_at_ms":89973,"credit_policy":null,"job_id":"job-80","job_issued_at_ms":89984,"miner_id":"miner-3","network_difficulty":1000012,"ntime":1700000080,"order_key":"03:miner-3","p2mr_program_hex":"5050505050505050505050505050505050505050505050505050505050505050","share_difficulty":3,"share_id":"share-80","share_seq":80,"template_height":800002},{"accepted_at_ms":89981,"credit_policy":null,"job_id":"job-1172","job_issued_at_ms":90000,"miner_id":"miner-6","network_difficulty":1000016,"ntime":1700001172,"order_key":"06:miner-6","p2mr_program_hex":"9494949494949494949494949494949494949494949494949494949494949494","share_difficulty":1,"share_id":"share-1172","share_seq":1172,"template_height":800036},{"accepted_at_ms":89981,"credit_policy":"stale-grace","job_id":"job-1551","job_issued_at_ms":89971,"miner_id":"miner-0","network_difficulty":1000004,"ntime":1700001551,"order_key":"00:miner-0","p2mr_program_hex":"0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f","share_difficulty":19,"share_id":"share-1551","share_seq":1551,"template_height":800048},{"accepted_at_ms":89973,"credit_policy":null,"job_id":"job-966","job_issued_at_ms":89995,"miner_id":"miner-9","network_difficulty":1000014,"ntime":1700000966,"order_key":"09:miner-9","p2mr_program_hex":"c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6","share_difficulty":19,"share_id":"share-966","share_seq":966,"template_height":800030},{"accepted_at_ms":89978,"credit_policy":null,"job_id":"job-589","job_issued_at_ms":89974,"miner_id":"miner-6","network_difficulty":1000011,"ntime":1700000589,"order_key":"06:miner-6","p2mr_program_hex":"4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d","share_difficulty":7,"share_id":"share-589","share_seq":589,"template_height":800018},{"accepted_at_ms":89997,"credit_policy":null,"job_id":"job-145","job_issued_at_ms":89994,"miner_id":"miner-2","network_difficulty":1000009,"ntime":1700000145,"order_key":"02:miner-2","p2mr_program_hex":"9191919191919191919191919191919191919191919191919191919191919191","share_difficulty":1,"share_id":"share-145","share_seq":145,"template_height":800004},{"accepted_at_ms":89971,"credit_policy":"stale-grace","job_id":"job-170","job_issued_at_ms":89985,"miner_id":"miner-5","network_difficulty":1000000,"ntime":1700000170,"order_key":"05:miner-5","p2mr_program_hex":"aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa","share_difficulty":7,"share_id":"share-170","share_seq":170,"template_height":800005},{"accepted_at_ms":89993,"credit_policy":"stale-grace","job_id":"job-186","job_issued_at_ms":89997,"miner_id":"miner-10","network_difficulty":1000016,"ntime":1700000186,"order_key":"10:miner-10","p2mr_program_hex":"babababababababababababababababababababababababababababababababa","share_difficulty":2,"share_id":"share-186","share_seq":186,"template_height":800005},{"accepted_at_ms":89977,"credit_policy":null,"job_id":"job-1496","job_issued_at_ms":89985,"miner_id":"miner-0","network_difficulty":1000000,"ntime":1700001496,"order_key":"00:miner-0","p2mr_program_hex":"d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8","share_difficulty":1,"share_id":"share-1496","share_seq":1496,"template_height":800046},{"accepted_at_ms":89978,"credit_policy":null,"job_id":"job-96","job_issued_at_ms":89995,"miner_id":"miner-8","network_difficulty":1000011,"ntime":1700000096,"order_key":"08:miner-8","p2mr_program_hex":"6060606060606060606060606060606060606060606060606060606060606060","share_difficulty":7,"share_id":"share-96","share_seq":96,"template_height":800003},{"accepted_at_ms":89983,"credit_policy":null,"job_id":"job-1432","job_issued_at_ms":89975,"miner_id":"miner-2","network_difficulty":1000004,"ntime":1700001432,"order_key":"02:miner-2","p2mr_program_hex":"9898989898989898989898989898989898989898989898989898989898989898","share_difficulty":19,"share_id":"share-1432","share_seq":1432,"template_height":800044},{"accepted_at_ms":90000,"credit_policy":"stale-grace","job_id":"job-1671","job_issued_at_ms":89985,"miner_id":"miner-10","network_difficulty":1000005,"ntime":1700001671,"order_key":"10:miner-10","p2mr_program_hex":"8787878787878787878787878787878787878787878787878787878787878787","share_difficulty":7,"share_id":"share-1671","share_seq":1671,"template_height":800052},{"accepted_at_ms":89977,"credit_policy":null,"job_id":"job-976","job_issued_at_ms":89982,"miner_id":"miner-8","network_difficulty":1000007,"ntime":1700000976,"order_key":"08:miner-8","p2mr_program_hex":"d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0","share_difficulty":2,"share_id":"share-976","share_seq":976,"template_height":800030},{"accepted_at_ms":89988,"credit_policy":null,"job_id":"job-602","job_issued_at_ms":89981,"miner_id":"miner-8","network_difficulty":1000007,"ntime":1700000602,"order_key":"08:miner-8","p2mr_program_hex":"5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a","share_difficulty":19,"share_id":"share-602","share_seq":602,"template_height":800018},{"accepted_at_ms":89997,"credit_policy":null,"job_id":"job-1553","job_issued_at_ms":89972,"miner_id":"miner-2","network_difficulty":1000006,"ntime":1700001553,"order_key":"02:miner-2","p2mr_program_hex":"1111111111111111111111111111111111111111111111111111111111111111","share_difficulty":3,"share_id":"share-1553","share_seq":1553,"template_height":800048},{"accepted_at_ms":89979,"credit_policy":null,"job_id":"job-1271","job_issued_at_ms":89970,"miner_id":"miner-6","network_difficulty":1000013,"ntime":1700001271,"order_key":"06:miner-6","p2mr_program_hex":"f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7","share_difficulty":3,"share_id":"share-1271","share_seq":1271,"template_height":800039},{"accepted_at_ms":89980,"credit_policy":null,"job_id":"job-915","job_issued_at_ms":89980,"miner_id":"miner-2","network_difficulty":1000014,"ntime":1700000915,"order_key":"02:miner-2","p2mr_program_hex":"9393939393939393939393939393939393939393939393939393939393939393","share_difficulty":19,"share_id":"share-915","share_seq":915,"template_height":800028},{"accepted_at_ms":89970,"credit_policy":"stale-grace","job_id":"job-1214","job_issued_at_ms":89980,"miner_id":"miner-4","network_difficulty":1000007,"ntime":1700001214,"order_key":"04:miner-4","p2mr_program_hex":"bebebebebebebebebebebebebebebebebebebebebebebebebebebebebebebebe","share_difficulty":19,"share_id":"share-1214","share_seq":1214,"template_height":800037},{"accepted_at_ms":89979,"credit_policy":null,"job_id":"job-797","job_issued_at_ms":89983,"miner_id":"miner-5","network_difficulty":1000015,"ntime":1700000797,"order_key":"05:miner-5","p2mr_program_hex":"1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d","share_difficulty":7,"share_id":"share-797","share_seq":797,"template_height":800024},{"accepted_at_ms":89990,"credit_policy":null,"job_id":"job-1151","job_issued_at_ms":89983,"miner_id":"miner-7","network_difficulty":1000012,"ntime":1700001151,"order_key":"07:miner-7","p2mr_program_hex":"7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f","share_difficulty":1,"share_id":"share-1151","share_seq":1151,"template_height":800035},{"accepted_at_ms":89982,"credit_policy":null,"job_id":"job-1255","job_issued_at_ms":89971,"miner_id":"miner-1","network_difficulty":1000014,"ntime":1700001255,"order_key":"01:miner-1","p2mr_program_hex":"e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7","share_difficulty":1,"share_id":"share-1255","share_seq":1255,"template_height":800039},{"accepted_at_ms":89979,"credit_policy":"stale-grace","job_id":"job-683","job_issued_at_ms":89973,"miner_id":"miner-1","network_difficulty":1000003,"ntime":1700000683,"order_key":"01:miner-1","p2mr_program_hex":"abababababababababababababababababababababababababababababababab","share_difficulty":3,"share_id":"share-683","share_seq":683,"template_height":800021},{"accepted_at_ms":89984,"credit_policy":null,"job_id":"job-202","job_issued_at_ms":89976,"miner_id":"miner-4","network_difficulty":1000015,"ntime":1700000202,"order_key":"04:miner-4","p2mr_program_hex":"cacacacacacacacacacacacacacacacacacacacacacacacacacacacacacacaca","share_difficulty":7,"share_id":"share-202","share_seq":202,"template_height":800006},{"accepted_at_ms":89970,"credit_policy":null,"job_id":"job-84","job_issued_at_ms":90000,"miner_id":"miner-7","network_difficulty":1000016,"ntime":1700000084,"order_key":"07:miner-7","p2mr_program_hex":"5454545454545454545454545454545454545454545454545454545454545454","share_difficulty":7,"share_id":"share-84","share_seq":84,"template_height":800002},{"accepted_at_ms":89973,"credit_policy":null,"job_id":"job-1436","job_issued_at_ms":89985,"miner_id":"miner-6","network_difficulty":1000008,"ntime":1700001436,"order_key":"06:miner-6","p2mr_program_hex":"9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c","share_difficulty":3,"share_id":"share-1436","share_seq":1436,"template_height":800044},{"accepted_at_ms":89976,"credit_policy":null,"job_id":"job-1203","job_issued_at_ms":89993,"miner_id":"miner-4","network_difficulty":1000013,"ntime":1700001203,"order_key":"04:miner-4","p2mr_program_hex":"b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3","share_difficulty":2,"share_id":"share-1203","share_seq":1203,"template_height":800037},{"accepted_at_ms":89992,"credit_policy":null,"job_id":"job-58","job_issued_at_ms":89978,"miner_id":"miner-3","network_difficulty":1000007,"ntime":1700000058,"order_key":"03:miner-3","p2mr_program_hex":"3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a","share_difficulty":1,"share_id":"share-58","share_seq":58,"template_height":800001},{"accepted_at_ms":89991,"credit_policy":null,"job_id":"job-185","job_issued_at_ms":89986,"miner_id":"miner-9","network_difficulty":1000015,"ntime":1700000185,"order_key":"09:miner-9","p2mr_program_hex":"b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9","share_difficulty":1,"share_id":"share-185","share_seq":185,"template_height":800005},{"accepted_at_ms":89990,"credit_policy":null,"job_id":"job-1152","job_issued_at_ms":89994,"miner_id":"miner-8","network_difficulty":1000013,"ntime":1700001152,"order_key":"08:miner-8","p2mr_program_hex":"8080808080808080808080808080808080808080808080808080808080808080","share_difficulty":19,"share_id":"share-1152","share_seq":1152,"template_height":800036},{"accepted_at_ms":89987,"credit_policy":"stale-grace","job_id":"job-1174","job_issued_at_ms":89995,"miner_id":"miner-8","network_difficulty":1000001,"ntime":1700001174,"order_key":"08:miner-8","p2mr_program_hex":"9696969696969696969696969696969696969696969696969696969696969696","share_difficulty":19,"share_id":"share-1174","share_seq":1174,"template_height":800036},{"accepted_at_ms":89982,"credit_policy":null,"job_id":"job-1243","job_issued_at_ms":89970,"miner_id":"miner-0","network_difficulty":1000002,"ntime":1700001243,"order_key":"00:miner-0","p2mr_program_hex":"dbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdb","share_difficulty":1,"share_id":"share-1243","share_seq":1243,"template_height":800038},{"accepted_at_ms":89998,"credit_policy":null,"job_id":"job-726","job_issued_at_ms":89981,"miner_id":"miner-0","network_difficulty":1000012,"ntime":1700000726,"order_key":"00:miner-0","p2mr_program_hex":"d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6","share_difficulty":1,"share_id":"share-726","share_seq":726,"template_height":800022},{"accepted_at_ms":89988,"credit_policy":null,"job_id":"job-507","job_issued_at_ms":89991,"miner_id":"miner-1","network_difficulty":1000014,"ntime":1700000507,"order_key":"01:miner-1","p2mr_program_hex":"fbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfb","share_difficulty":3,"share_id":"share-507","share_seq":507,"template_height":800015},{"accepted_at_ms":89994,"credit_policy":"stale-grace","job_id":"job-1628","job_issued_at_ms":89972,"miner_id":"miner-0","network_difficulty":1000013,"ntime":1700001628,"order_key":"00:miner-0","p2mr_program_hex":"5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c","share_difficulty":2,"share_id":"share-1628","share_seq":1628,"template_height":800050},{"accepted_at_ms":89976,"credit_policy":null,"job_id":"job-1782","job_issued_at_ms":89972,"miner_id":"miner-0","network_difficulty":1000014,"ntime":1700001782,"order_key":"00:miner-0","p2mr_program_hex":"f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6","share_difficulty":7,"share_id":"share-1782","share_seq":1782,"template_height":800055},{"accepted_at_ms":89972,"credit_policy":"stale-grace","job_id":"job-1642","job_issued_at_ms":89991,"miner_id":"miner-3","network_difficulty":1000010,"ntime":1700001642,"order_key":"03:miner-3","p2mr_program_hex":"6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a","share_difficulty":7,"share_id":"share-1642","share_seq":1642,"template_height":800051},{"accepted_at_ms":89981,"credit_policy":"stale-grace","job_id":"job-425","job_issued_at_ms":89986,"miner_id":"miner-7","network_difficulty":1000000,"ntime":1700000425,"order_key":"07:miner-7","p2mr_program_hex":"a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9","share_difficulty":1,"share_id":"share-425","share_seq":425,"template_height":800013},{"accepted_at_ms":89997,"credit_policy":null,"job_id":"job-1775","job_issued_at_ms":89985,"miner_id":"miner-4","network_difficulty":1000007,"ntime":1700001775,"order_key":"04:miner-4","p2mr_program_hex":"efefefefefefefefefefefefefefefefefefefefefefefefefefefefefefefef","share_difficulty":2,"share_id":"share-1775","share_seq":1775,"template_height":800055},{"accepted_at_ms":89972,"credit_policy":"stale-grace","job_id":"job-1002","job_issued_at_ms":89996,"miner_id":"miner-1","network_difficulty":1000016,"ntime":1700001002,"order_key":"01:miner-1","p2mr_program_hex":"eaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaea","share_difficulty":1,"share_id":"share-1002","share_seq":1002,"template_height":800031},{"accepted_at_ms":89996,"credit_policy":null,"job_id":"job-887","job_issued_at_ms":89996,"miner_id":"miner-7","network_difficulty":1000003,"ntime":1700000887,"order_key":"07:miner-7","p2mr_program_hex":"7777777777777777777777777777777777777777777777777777777777777777","share_difficulty":19,"share_id":"share-887","share_seq":887,"template_height":800027},{"accepted_at_ms":89993,"credit_policy":"stale-grace","job_id":"job-150","job_issued_at_ms":89994,"miner_id":"miner-7","network_difficulty":1000014,"ntime":1700000150,"order_key":"07:miner-7","p2mr_program_hex":"9696969696969696969696969696969696969696969696969696969696969696","share_difficulty":7,"share_id":"share-150","share_seq":150,"template_height":800004},{"accepted_at_ms":89972,"credit_policy":null,"job_id":"job-586","job_issued_at_ms":90000,"miner_id":"miner-3","network_difficulty":1000008,"ntime":1700000586,"order_key":"03:miner-3","p2mr_program_hex":"4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a","share_difficulty":3,"share_id":"share-586","share_seq":586,"template_height":800018},{"accepted_at_ms":89992,"credit_policy":null,"job_id":"job-86","job_issued_at_ms":89979,"miner_id":"miner-9","network_difficulty":1000001,"ntime":1700000086,"order_key":"09:miner-9","p2mr_program_hex":"5656565656565656565656565656565656565656565656565656565656565656","share_difficulty":1,"share_id":"share-86","share_seq":86,"template_height":800002},{"accepted_at_ms":89990,"credit_policy":"stale-grace","job_id":"job-1748","job_issued_at_ms":89987,"miner_id":"miner-10","network_difficulty":1000014,"ntime":1700001748,"order_key":"10:miner-10","p2mr_program_hex":"d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4","share_difficulty":7,"share_id":"share-1748","share_seq":1748,"template_height":800054},{"accepted_at_ms":89983,"credit_policy":"stale-grace","job_id":"job-1561","job_issued_at_ms":89983,"miner_id":"miner-10","network_difficulty":1000014,"ntime":1700001561,"order_key":"10:miner-10","p2mr_program_hex":"1919191919191919191919191919191919191919191919191919191919191919","share_difficulty":1,"share_id":"share-1561","share_seq":1561,"template_height":800048},{"accepted_at_ms":89972,"credit_policy":null,"job_id":"job-1723","job_issued_at_ms":89993,"miner_id":"miner-7","network_difficulty":1000006,"ntime":1700001723,"order_key":"07:miner-7","p2mr_program_hex":"bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb","share_difficulty":3,"share_id":"share-1723","share_seq":1723,"template_height":800053},{"accepted_at_ms":89974,"credit_policy":null,"job_id":"job-947","job_issued_at_ms":89989,"miner_id":"miner-1","network_difficulty":1000012,"ntime":1700000947,"order_key":"01:miner-1","p2mr_program_hex":"b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3","share_difficulty":2,"share_id":"share-947","share_seq":947,"template_height":800029},{"accepted_at_ms":89999,"credit_policy":null,"job_id":"job-1416","job_issued_at_ms":89999,"miner_id":"miner-8","network_difficulty":1000005,"ntime":1700001416,"order_key":"08:miner-8","p2mr_program_hex":"8888888888888888888888888888888888888888888888888888888888888888","share_difficulty":19,"share_id":"share-1416","share_seq":1416,"template_height":800044},{"accepted_at_ms":89998,"credit_policy":null,"job_id":"job-1319","job_issued_at_ms":89981,"miner_id":"miner-10","network_difficulty":1000010,"ntime":1700001319,"order_key":"10:miner-10","p2mr_program_hex":"2727272727272727272727272727272727272727272727272727272727272727","share_difficulty":2,"share_id":"share-1319","share_seq":1319,"template_height":800041},{"accepted_at_ms":89996,"credit_policy":null,"job_id":"job-124","job_issued_at_ms":89997,"miner_id":"miner-3","network_difficulty":1000005,"ntime":1700000124,"order_key":"03:miner-3","p2mr_program_hex":"7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c","share_difficulty":1,"share_id":"share-124","share_seq":124,"template_height":800003},{"accepted_at_ms":89978,"credit_policy":null,"job_id":"job-866","job_issued_at_ms":89982,"miner_id":"miner-8","network_difficulty":1000016,"ntime":1700000866,"order_key":"08:miner-8","p2mr_program_hex":"6262626262626262626262626262626262626262626262626262626262626262","share_difficulty":3,"share_id":"share-866","share_seq":866,"template_height":800027},{"accepted_at_ms":89995,"credit_policy":null,"job_id":"job-374","job_issued_at_ms":89974,"miner_id":"miner-0","network_difficulty":1000000,"ntime":1700000374,"order_key":"00:miner-0","p2mr_program_hex":"7676767676767676767676767676767676767676767676767676767676767676","share_difficulty":7,"share_id":"share-374","share_seq":374,"template_height":800011},{"accepted_at_ms":89993,"credit_policy":null,"job_id":"job-1208","job_issued_at_ms":89996,"miner_id":"miner-9","network_difficulty":1000001,"ntime":1700001208,"order_key":"09:miner-9","p2mr_program_hex":"b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8","share_difficulty":2,"share_id":"share-1208","share_seq":1208,"template_height":800037},{"accepted_at_ms":89976,"credit_policy":"stale-grace","job_id":"job-665","job_issued_at_ms":89970,"miner_id":"miner-5","network_difficulty":1000002,"ntime":1700000665,"order_key":"05:miner-5","p2mr_program_hex":"9999999999999999999999999999999999999999999999999999999999999999","share_difficulty":7,"share_id":"share-665","share_seq":665,"template_height":800020},{"accepted_at_ms":89987,"credit_policy":null,"job_id":"job-1459","job_issued_at_ms":90000,"miner_id":"miner-7","network_difficulty":1000014,"ntime":1700001459,"order_key":"07:miner-7","p2mr_program_hex":"b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3","share_difficulty":1,"share_id":"share-1459","share_seq":1459,"template_height":800045},{"accepted_at_ms":89993,"credit_policy":null,"job_id":"job-690","job_issued_at_ms":89986,"miner_id":"miner-8","network_difficulty":1000010,"ntime":1700000690,"order_key":"08:miner-8","p2mr_program_hex":"b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2","share_difficulty":3,"share_id":"share-690","share_seq":690,"template_height":800021},{"accepted_at_ms":89976,"credit_policy":"stale-grace","job_id":"job-354","job_issued_at_ms":89983,"miner_id":"miner-2","network_difficulty":1000014,"ntime":1700000354,"order_key":"02:miner-2","p2mr_program_hex":"6262626262626262626262626262626262626262626262626262626262626262","share_difficulty":7,"share_id":"share-354","share_seq":354,"template_height":800011},{"accepted_at_ms":89979,"credit_policy":"stale-grace","job_id":"job-1789","job_issued_at_ms":89982,"miner_id":"miner-7","network_difficulty":1000004,"ntime":1700001789,"order_key":"07:miner-7","p2mr_program_hex":"fdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfd","share_difficulty":2,"share_id":"share-1789","share_seq":1789,"template_height":800055},{"accepted_at_ms":89970,"credit_policy":"stale-grace","job_id":"job-294","job_issued_at_ms":89980,"miner_id":"miner-8","network_difficulty":1000005,"ntime":1700000294,"order_key":"08:miner-8","p2mr_program_hex":"2626262626262626262626262626262626262626262626262626262626262626","share_difficulty":19,"share_id":"share-294","share_seq":294,"template_height":800009},{"accepted_at_ms":89994,"credit_policy":"stale-grace","job_id":"job-157","job_issued_at_ms":89978,"miner_id":"miner-3","network_difficulty":1000004,"ntime":1700000157,"order_key":"03:miner-3","p2mr_program_hex":"9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d","share_difficulty":19,"share_id":"share-157","share_seq":157,"template_height":800004},{"accepted_at_ms":89998,"credit_policy":null,"job_id":"job-1064","job_issued_at_ms":89998,"miner_id":"miner-8","network_difficulty":1000010,"ntime":1700001064,"order_key":"08:miner-8","p2mr_program_hex":"2828282828282828282828282828282828282828282828282828282828282828","share_difficulty":7,"share_id":"share-1064","share_seq":1064,"template_height":800033},{"accepted_at_ms":89996,"credit_policy":"stale-grace","job_id":"job-1760","job_issued_at_ms":89991,"miner_id":"miner-0","network_difficulty":1000009,"ntime":1700001760,"order_key":"00:miner-0","p2mr_program_hex":"e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0","share_difficulty":1,"share_id":"share-1760","share_seq":1760,"template_height":800055},{"accepted_at_ms":89972,"credit_policy":null,"job_id":"job-952","job_issued_at_ms":89997,"miner_id":"miner-6","network_difficulty":1000000,"ntime":1700000952,"order_key":"06:miner-6","p2mr_program_hex":"b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8","share_difficulty":3,"share_id":"share-952","share_seq":952,"template_height":800029},{"accepted_at_ms":89991,"credit_policy":null,"job_id":"job-515","job_issued_at_ms":89975,"miner_id":"miner-9","network_difficulty":1000005,"ntime":1700000515,"order_key":"09:miner-9","p2mr_program_hex":"0303030303030303030303030303030303030303030303030303030303030303","share_difficulty":7,"share_id":"share-515","share_seq":515,"template_height":800016},{"accepted_at_ms":89979,"credit_policy":null,"job_id":"job-1272","job_issued_at_ms":90000,"miner_id":"miner-7","network_difficulty":1000014,"ntime":1700001272,"order_key":"07:miner-7","p2mr_program_hex":"f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8","share_difficulty":3,"share_id":"share-1272","share_seq":1272,"template_height":800039},{"accepted_at_ms":89970,"credit_policy":null,"job_id":"job-902","job_issued_at_ms":89998,"miner_id":"miner-0","network_difficulty":1000001,"ntime":1700000902,"order_key":"00:miner-0","p2mr_program_hex":"8686868686868686868686868686868686868686868686868686868686868686","share_difficulty":2,"share_id":"share-902","share_seq":902,"template_height":800028},{"accepted_at_ms":89986,"credit_policy":null,"job_id":"job-1127","job_issued_at_ms":89980,"miner_id":"miner-5","network_difficulty":1000005,"ntime":1700001127,"order_key":"05:miner-5","p2mr_program_hex":"6767676767676767676767676767676767676767676767676767676767676767","share_difficulty":7,"share_id":"share-1127","share_seq":1127,"template_height":800035},{"accepted_at_ms":89990,"credit_policy":null,"job_id":"job-309","job_issued_at_ms":89977,"miner_id":"miner-1","network_difficulty":1000003,"ntime":1700000309,"order_key":"01:miner-1","p2mr_program_hex":"3535353535353535353535353535353535353535353535353535353535353535","share_difficulty":19,"share_id":"share-309","share_seq":309,"template_height":800009},{"accepted_at_ms":89974,"credit_policy":null,"job_id":"job-1534","job_issued_at_ms":89976,"miner_id":"miner-5","network_difficulty":1000004,"ntime":1700001534,"order_key":"05:miner-5","p2mr_program_hex":"fefefefefefefefefefefefefefefefefefefefefefefefefefefefefefefefe","share_difficulty":1,"share_id":"share-1534","share_seq":1534,"template_height":800047},{"accepted_at_ms":89970,"credit_policy":"stale-grace","job_id":"job-222","job_issued_at_ms":89985,"miner_id":"miner-2","network_difficulty":1000001,"ntime":1700000222,"order_key":"02:miner-2","p2mr_program_hex":"dededededededededededededededededededededededededededededededede","share_difficulty":7,"share_id":"share-222","share_seq":222,"template_height":800006},{"accepted_at_ms":89999,"credit_policy":"stale-grace","job_id":"job-109","job_issued_at_ms":89972,"miner_id":"miner-10","network_difficulty":1000007,"ntime":1700000109,"order_key":"10:miner-10","p2mr_program_hex":"6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d","share_difficulty":2,"share_id":"share-109","share_seq":109,"template_height":800003},{"accepted_at_ms":89981,"credit_policy":null,"job_id":"job-859","job_issued_at_ms":89982,"miner_id":"miner-1","network_difficulty":1000009,"ntime":1700000859,"order_key":"01:miner-1","p2mr_program_hex":"5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b","share_difficulty":19,"share_id":"share-859","share_seq":859,"template_height":800026},{"accepted_at_ms":89999,"credit_policy":null,"job_id":"job-542","job_issued_at_ms":89998,"miner_id":"miner-3","network_difficulty":1000015,"ntime":1700000542,"order_key":"03:miner-3","p2mr_program_hex":"1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e","share_difficulty":3,"share_id":"share-542","share_seq":542,"template_height":800016},{"accepted_at_ms":89997,"credit_policy":"stale-grace","job_id":"job-564","job_issued_at_ms":89978,"miner_id":"miner-3","network_difficulty":1000003,"ntime":1700000564,"order_key":"03:miner-3","p2mr_program_hex":"3434343434343434343434343434343434343434343434343434343434343434","share_difficulty":3,"share_id":"share-564","share_seq":564,"template_height":800017},{"accepted_at_ms":89994,"credit_policy":null,"job_id":"job-393","job_issued_at_ms":89975,"miner_id":"miner-8","network_difficulty":1000002,"ntime":1700000393,"order_key":"08:miner-8","p2mr_program_hex":"8989898989898989898989898989898989898989898989898989898989898989","share_difficulty":2,"share_id":"share-393","share_seq":393,"template_height":800012},{"accepted_at_ms":89996,"credit_policy":"stale-grace","job_id":"job-1146","job_issued_at_ms":89992,"miner_id":"miner-2","network_difficulty":1000007,"ntime":1700001146,"order_key":"02:miner-2","p2mr_program_hex":"7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a","share_difficulty":1,"share_id":"share-1146","share_seq":1146,"template_height":800035},{"accepted_at_ms":89995,"credit_policy":null,"job_id":"job-803","job_issued_at_ms":89983,"miner_id":"miner-0","network_difficulty":1000004,"ntime":1700000803,"order_key":"00:miner-0","p2mr_program_hex":"2323232323232323232323232323232323232323232323232323232323232323","share_difficulty":19,"share_id":"share-803","share_seq":803,"template_height":800025},{"accepted_at_ms":89998,"credit_policy":null,"job_id":"job-221","job_issued_at_ms":89970,"miner_id":"miner-1","network_difficulty":1000000,"ntime":1700000221,"order_key":"01:miner-1","p2mr_program_hex":"dddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddd","share_difficulty":7,"share_id":"share-221","share_seq":221,"template_height":800006},{"accepted_at_ms":89973,"credit_policy":null,"job_id":"job-152","job_issued_at_ms":89990,"miner_id":"miner-9","network_difficulty":1000016,"ntime":1700000152,"order_key":"09:miner-9","p2mr_program_hex":"9898989898989898989898989898989898989898989898989898989898989898","share_difficulty":19,"share_id":"share-152","share_seq":152,"template_height":800004},{"accepted_at_ms":89972,"credit_policy":"stale-grace","job_id":"job-873","job_issued_at_ms":89996,"miner_id":"miner-4","network_difficulty":1000006,"ntime":1700000873,"order_key":"04:miner-4","p2mr_program_hex":"6969696969696969696969696969696969696969696969696969696969696969","share_difficulty":7,"share_id":"share-873","share_seq":873,"template_height":800027},{"accepted_at_ms":89976,"credit_policy":"stale-grace","job_id":"job-1077","job_issued_at_ms":89986,"miner_id":"miner-10","network_difficulty":1000006,"ntime":1700001077,"order_key":"10:miner-10","p2mr_program_hex":"3535353535353535353535353535353535353535353535353535353535353535","share_difficulty":1,"share_id":"share-1077","share_seq":1077,"template_height":800033},{"accepted_at_ms":89981,"credit_policy":null,"job_id":"job-457","job_issued_at_ms":89980,"miner_id":"miner-6","network_difficulty":1000015,"ntime":1700000457,"order_key":"06:miner-6","p2mr_program_hex":"c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9","share_difficulty":2,"share_id":"share-457","share_seq":457,"template_height":800014},{"accepted_at_ms":89978,"credit_policy":null,"job_id":"job-16","job_issued_at_ms":89988,"miner_id":"miner-5","network_difficulty":1000016,"ntime":1700000016,"order_key":"05:miner-5","p2mr_program_hex":"1010101010101010101010101010101010101010101010101010101010101010","share_difficulty":7,"share_id":"share-16","share_seq":16,"template_height":800000},{"accepted_at_ms":89980,"credit_policy":null,"job_id":"job-979","job_issued_at_ms":89988,"miner_id":"miner-0","network_difficulty":1000010,"ntime":1700000979,"order_key":"00:miner-0","p2mr_program_hex":"d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3","share_difficulty":7,"share_id":"share-979","share_seq":979,"template_height":800030},{"accepted_at_ms":89993,"credit_policy":null,"job_id":"job-500","job_issued_at_ms":89974,"miner_id":"miner-5","network_difficulty":1000007,"ntime":1700000500,"order_key":"05:miner-5","p2mr_program_hex":"f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4","share_difficulty":2,"share_id":"share-500","share_seq":500,"template_height":800015},{"accepted_at_ms":89977,"credit_policy":null,"job_id":"job-1668","job_issued_at_ms":89995,"miner_id":"miner-7","network_difficulty":1000002,"ntime":1700001668,"order_key":"07:miner-7","p2mr_program_hex":"8484848484848484848484848484848484848484848484848484848484848484","share_difficulty":3,"share_id":"share-1668","share_seq":1668,"template_height":800052},{"accepted_at_ms":89986,"credit_policy":null,"job_id":"job-649","job_issued_at_ms":90000,"miner_id":"miner-0","network_difficulty":1000003,"ntime":1700000649,"order_key":"00:miner-0","p2mr_program_hex":"8989898989898989898989898989898989898989898989898989898989898989","share_difficulty":3,"share_id":"share-649","share_seq":649,"template_height":800020},{"accepted_at_ms":89970,"credit_policy":null,"job_id":"job-156","job_issued_at_ms":89978,"miner_id":"miner-2","network_difficulty":1000003,"ntime":1700000156,"order_key":"02:miner-2","p2mr_program_hex":"9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c","share_difficulty":2,"share_id":"share-156","share_seq":156,"template_height":800004},{"accepted_at_ms":89987,"credit_policy":null,"job_id":"job-711","job_issued_at_ms":89982,"miner_id":"miner-7","network_difficulty":1000014,"ntime":1700000711,"order_key":"07:miner-7","p2mr_program_hex":"c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7","share_difficulty":1,"share_id":"share-711","share_seq":711,"template_height":800022},{"accepted_at_ms":89998,"credit_policy":"stale-grace","job_id":"job-1761","job_issued_at_ms":89970,"miner_id":"miner-1","network_difficulty":1000010,"ntime":1700001761,"order_key":"01:miner-1","p2mr_program_hex":"e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1","share_difficulty":2,"share_id":"share-1761","share_seq":1761,"template_height":800055},{"accepted_at_ms":89986,"credit_policy":null,"job_id":"job-207","job_issued_at_ms":89984,"miner_id":"miner-9","network_difficulty":1000003,"ntime":1700000207,"order_key":"09:miner-9","p2mr_program_hex":"cfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcf","share_difficulty":1,"share_id":"share-207","share_seq":207,"template_height":800006},{"accepted_at_ms":89972,"credit_policy":"stale-grace","job_id":"job-991","job_issued_at_ms":89977,"miner_id":"miner-1","network_difficulty":1000005,"ntime":1700000991,"order_key":"01:miner-1","p2mr_program_hex":"dfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdf","share_difficulty":1,"share_id":"share-991","share_seq":991,"template_height":800030},{"accepted_at_ms":89994,"credit_policy":"stale-grace","job_id":"job-1376","job_issued_at_ms":89978,"miner_id":"miner-1","network_difficulty":1000016,"ntime":1700001376,"order_key":"01:miner-1","p2mr_program_hex":"6060606060606060606060606060606060606060606060606060606060606060","share_difficulty":3,"share_id":"share-1376","share_seq":1376,"template_height":800043},{"accepted_at_ms":89995,"credit_policy":"stale-grace","job_id":"job-1015","job_issued_at_ms":89986,"miner_id":"miner-3","network_difficulty":1000012,"ntime":1700001015,"order_key":"03:miner-3","p2mr_program_hex":"f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7","share_difficulty":3,"share_id":"share-1015","share_seq":1015,"template_height":800031},{"accepted_at_ms":89999,"credit_policy":null,"job_id":"job-183","job_issued_at_ms":90000,"miner_id":"miner-7","network_difficulty":1000013,"ntime":1700000183,"order_key":"07:miner-7","p2mr_program_hex":"b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7","share_difficulty":3,"share_id":"share-183","share_seq":183,"template_height":800005},{"accepted_at_ms":89997,"credit_policy":null,"job_id":"job-308","job_issued_at_ms":89986,"miner_id":"miner-0","network_difficulty":1000002,"ntime":1700000308,"order_key":"00:miner-0","p2mr_program_hex":"3434343434343434343434343434343434343434343434343434343434343434","share_difficulty":19,"share_id":"share-308","share_seq":308,"template_height":800009},{"accepted_at_ms":89995,"credit_policy":"stale-grace","job_id":"job-825","job_issued_at_ms":89980,"miner_id":"miner-0","network_difficulty":1000009,"ntime":1700000825,"order_key":"00:miner-0","p2mr_program_hex":"3939393939393939393939393939393939393939393939393939393939393939","share_difficulty":7,"share_id":"share-825","share_seq":825,"template_height":800025},{"accepted_at_ms":89989,"credit_policy":null,"job_id":"job-857","job_issued_at_ms":89976,"miner_id":"miner-10","network_difficulty":1000007,"ntime":1700000857,"order_key":"10:miner-10","p2mr_program_hex":"5959595959595959595959595959595959595959595959595959595959595959","share_difficulty":2,"share_id":"share-857","share_seq":857,"template_height":800026}]},"window_weight":10758},"page-boundary-511":{"advances":[],"name":"page-boundary-511","page_size":512,"schema":"qbit-prism-window-pipeline-parity-input/v1","snapshot":{"anchor_job_issued_at_ms":70511,"records":[{"accepted_at_ms":70496,"credit_policy":null,"job_id":"job-1","job_issued_at_ms":70501,"miner_id":"miner-1","network_difficulty":1000001,"ntime":1700000001,"order_key":"01:miner-1","p2mr_program_hex":"0101010101010101010101010101010101010101010101010101010101010101","share_difficulty":7,"share_id":"share-1","share_seq":1,"template_height":800000},{"accepted_at_ms":70486,"credit_policy":"stale-grace","job_id":"job-2","job_issued_at_ms":70485,"miner_id":"miner-2","network_difficulty":1000002,"ntime":1700000002,"order_key":"02:miner-2","p2mr_program_hex":"0202020202020202020202020202020202020202020202020202020202020202","share_difficulty":7,"share_id":"share-2","share_seq":2,"template_height":800000},{"accepted_at_ms":70489,"credit_policy":null,"job_id":"job-3","job_issued_at_ms":70511,"miner_id":"miner-3","network_difficulty":1000003,"ntime":1700000003,"order_key":"03:miner-3","p2mr_program_hex":"0303030303030303030303030303030303030303030303030303030303030303","share_difficulty":19,"share_id":"share-3","share_seq":3,"template_height":800000},{"accepted_at_ms":70485,"credit_policy":null,"job_id":"job-4","job_issued_at_ms":70489,"miner_id":"miner-4","network_difficulty":1000004,"ntime":1700000004,"order_key":"04:miner-4","p2mr_program_hex":"0404040404040404040404040404040404040404040404040404040404040404","share_difficulty":19,"share_id":"share-4","share_seq":4,"template_height":800000},{"accepted_at_ms":70483,"credit_policy":null,"job_id":"job-5","job_issued_at_ms":70492,"miner_id":"miner-5","network_difficulty":1000005,"ntime":1700000005,"order_key":"05:miner-5","p2mr_program_hex":"0505050505050505050505050505050505050505050505050505050505050505","share_difficulty":1,"share_id":"share-5","share_seq":5,"template_height":800000},{"accepted_at_ms":70483,"credit_policy":null,"job_id":"job-6","job_issued_at_ms":70497,"miner_id":"miner-6","network_difficulty":1000006,"ntime":1700000006,"order_key":"06:miner-6","p2mr_program_hex":"0606060606060606060606060606060606060606060606060606060606060606","share_difficulty":1,"share_id":"share-6","share_seq":6,"template_height":800000},{"accepted_at_ms":70481,"credit_policy":null,"job_id":"job-7","job_issued_at_ms":70505,"miner_id":"miner-7","network_difficulty":1000007,"ntime":1700000007,"order_key":"07:miner-7","p2mr_program_hex":"0707070707070707070707070707070707070707070707070707070707070707","share_difficulty":19,"share_id":"share-7","share_seq":7,"template_height":800000},{"accepted_at_ms":70490,"credit_policy":null,"job_id":"job-8","job_issued_at_ms":70491,"miner_id":"miner-8","network_difficulty":1000008,"ntime":1700000008,"order_key":"08:miner-8","p2mr_program_hex":"0808080808080808080808080808080808080808080808080808080808080808","share_difficulty":3,"share_id":"share-8","share_seq":8,"template_height":800000},{"accepted_at_ms":70504,"credit_policy":"stale-grace","job_id":"job-9","job_issued_at_ms":70496,"miner_id":"miner-9","network_difficulty":1000009,"ntime":1700000009,"order_key":"09:miner-9","p2mr_program_hex":"0909090909090909090909090909090909090909090909090909090909090909","share_difficulty":1,"share_id":"share-9","share_seq":9,"template_height":800000},{"accepted_at_ms":70499,"credit_policy":null,"job_id":"job-10","job_issued_at_ms":70505,"miner_id":"miner-10","network_difficulty":1000010,"ntime":1700000010,"order_key":"10:miner-10","p2mr_program_hex":"0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a","share_difficulty":3,"share_id":"share-10","share_seq":10,"template_height":800000},{"accepted_at_ms":70495,"credit_policy":null,"job_id":"job-11","job_issued_at_ms":70510,"miner_id":"miner-0","network_difficulty":1000011,"ntime":1700000011,"order_key":"00:miner-0","p2mr_program_hex":"0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b","share_difficulty":19,"share_id":"share-11","share_seq":11,"template_height":800000},{"accepted_at_ms":70506,"credit_policy":null,"job_id":"job-12","job_issued_at_ms":70497,"miner_id":"miner-1","network_difficulty":1000012,"ntime":1700000012,"order_key":"01:miner-1","p2mr_program_hex":"0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c","share_difficulty":3,"share_id":"share-12","share_seq":12,"template_height":800000},{"accepted_at_ms":70507,"credit_policy":null,"job_id":"job-13","job_issued_at_ms":70488,"miner_id":"miner-2","network_difficulty":1000013,"ntime":1700000013,"order_key":"02:miner-2","p2mr_program_hex":"0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d","share_difficulty":1,"share_id":"share-13","share_seq":13,"template_height":800000},{"accepted_at_ms":70502,"credit_policy":null,"job_id":"job-14","job_issued_at_ms":70495,"miner_id":"miner-3","network_difficulty":1000014,"ntime":1700000014,"order_key":"03:miner-3","p2mr_program_hex":"0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e","share_difficulty":7,"share_id":"share-14","share_seq":14,"template_height":800000},{"accepted_at_ms":70482,"credit_policy":null,"job_id":"job-15","job_issued_at_ms":70497,"miner_id":"miner-4","network_difficulty":1000015,"ntime":1700000015,"order_key":"04:miner-4","p2mr_program_hex":"0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f","share_difficulty":1,"share_id":"share-15","share_seq":15,"template_height":800000},{"accepted_at_ms":70495,"credit_policy":"stale-grace","job_id":"job-16","job_issued_at_ms":70484,"miner_id":"miner-5","network_difficulty":1000016,"ntime":1700000016,"order_key":"05:miner-5","p2mr_program_hex":"1010101010101010101010101010101010101010101010101010101010101010","share_difficulty":2,"share_id":"share-16","share_seq":16,"template_height":800000},{"accepted_at_ms":70482,"credit_policy":null,"job_id":"job-17","job_issued_at_ms":70504,"miner_id":"miner-6","network_difficulty":1000000,"ntime":1700000017,"order_key":"06:miner-6","p2mr_program_hex":"1111111111111111111111111111111111111111111111111111111111111111","share_difficulty":2,"share_id":"share-17","share_seq":17,"template_height":800000},{"accepted_at_ms":70508,"credit_policy":null,"job_id":"job-18","job_issued_at_ms":70481,"miner_id":"miner-7","network_difficulty":1000001,"ntime":1700000018,"order_key":"07:miner-7","p2mr_program_hex":"1212121212121212121212121212121212121212121212121212121212121212","share_difficulty":1,"share_id":"share-18","share_seq":18,"template_height":800000},{"accepted_at_ms":70489,"credit_policy":null,"job_id":"job-19","job_issued_at_ms":70501,"miner_id":"miner-8","network_difficulty":1000002,"ntime":1700000019,"order_key":"08:miner-8","p2mr_program_hex":"1313131313131313131313131313131313131313131313131313131313131313","share_difficulty":1,"share_id":"share-19","share_seq":19,"template_height":800000},{"accepted_at_ms":70503,"credit_policy":null,"job_id":"job-20","job_issued_at_ms":70509,"miner_id":"miner-9","network_difficulty":1000003,"ntime":1700000020,"order_key":"09:miner-9","p2mr_program_hex":"1414141414141414141414141414141414141414141414141414141414141414","share_difficulty":1,"share_id":"share-20","share_seq":20,"template_height":800000},{"accepted_at_ms":70507,"credit_policy":null,"job_id":"job-21","job_issued_at_ms":70485,"miner_id":"miner-10","network_difficulty":1000004,"ntime":1700000021,"order_key":"10:miner-10","p2mr_program_hex":"1515151515151515151515151515151515151515151515151515151515151515","share_difficulty":2,"share_id":"share-21","share_seq":21,"template_height":800000},{"accepted_at_ms":70496,"credit_policy":null,"job_id":"job-22","job_issued_at_ms":70500,"miner_id":"miner-0","network_difficulty":1000005,"ntime":1700000022,"order_key":"00:miner-0","p2mr_program_hex":"1616161616161616161616161616161616161616161616161616161616161616","share_difficulty":1,"share_id":"share-22","share_seq":22,"template_height":800000},{"accepted_at_ms":70493,"credit_policy":null,"job_id":"job-23","job_issued_at_ms":70504,"miner_id":"miner-1","network_difficulty":1000006,"ntime":1700000023,"order_key":"01:miner-1","p2mr_program_hex":"1717171717171717171717171717171717171717171717171717171717171717","share_difficulty":2,"share_id":"share-23","share_seq":23,"template_height":800000},{"accepted_at_ms":70485,"credit_policy":null,"job_id":"job-24","job_issued_at_ms":70510,"miner_id":"miner-2","network_difficulty":1000007,"ntime":1700000024,"order_key":"02:miner-2","p2mr_program_hex":"1818181818181818181818181818181818181818181818181818181818181818","share_difficulty":7,"share_id":"share-24","share_seq":24,"template_height":800000},{"accepted_at_ms":70511,"credit_policy":null,"job_id":"job-25","job_issued_at_ms":70482,"miner_id":"miner-3","network_difficulty":1000008,"ntime":1700000025,"order_key":"03:miner-3","p2mr_program_hex":"1919191919191919191919191919191919191919191919191919191919191919","share_difficulty":1,"share_id":"share-25","share_seq":25,"template_height":800000},{"accepted_at_ms":70499,"credit_policy":"stale-grace","job_id":"job-26","job_issued_at_ms":70488,"miner_id":"miner-4","network_difficulty":1000009,"ntime":1700000026,"order_key":"04:miner-4","p2mr_program_hex":"1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a","share_difficulty":1,"share_id":"share-26","share_seq":26,"template_height":800000},{"accepted_at_ms":70491,"credit_policy":"stale-grace","job_id":"job-27","job_issued_at_ms":70483,"miner_id":"miner-5","network_difficulty":1000010,"ntime":1700000027,"order_key":"05:miner-5","p2mr_program_hex":"1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b","share_difficulty":3,"share_id":"share-27","share_seq":27,"template_height":800000},{"accepted_at_ms":70485,"credit_policy":null,"job_id":"job-28","job_issued_at_ms":70495,"miner_id":"miner-6","network_difficulty":1000011,"ntime":1700000028,"order_key":"06:miner-6","p2mr_program_hex":"1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c","share_difficulty":1,"share_id":"share-28","share_seq":28,"template_height":800000},{"accepted_at_ms":70505,"credit_policy":null,"job_id":"job-29","job_issued_at_ms":70485,"miner_id":"miner-7","network_difficulty":1000012,"ntime":1700000029,"order_key":"07:miner-7","p2mr_program_hex":"1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d","share_difficulty":3,"share_id":"share-29","share_seq":29,"template_height":800000},{"accepted_at_ms":70498,"credit_policy":null,"job_id":"job-30","job_issued_at_ms":70501,"miner_id":"miner-8","network_difficulty":1000013,"ntime":1700000030,"order_key":"08:miner-8","p2mr_program_hex":"1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e","share_difficulty":3,"share_id":"share-30","share_seq":30,"template_height":800000},{"accepted_at_ms":70511,"credit_policy":null,"job_id":"job-31","job_issued_at_ms":70500,"miner_id":"miner-9","network_difficulty":1000014,"ntime":1700000031,"order_key":"09:miner-9","p2mr_program_hex":"1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f","share_difficulty":1,"share_id":"share-31","share_seq":31,"template_height":800000},{"accepted_at_ms":70501,"credit_policy":null,"job_id":"job-32","job_issued_at_ms":70507,"miner_id":"miner-10","network_difficulty":1000015,"ntime":1700000032,"order_key":"10:miner-10","p2mr_program_hex":"2020202020202020202020202020202020202020202020202020202020202020","share_difficulty":2,"share_id":"share-32","share_seq":32,"template_height":800001},{"accepted_at_ms":70492,"credit_policy":null,"job_id":"job-33","job_issued_at_ms":70487,"miner_id":"miner-0","network_difficulty":1000016,"ntime":1700000033,"order_key":"00:miner-0","p2mr_program_hex":"2121212121212121212121212121212121212121212121212121212121212121","share_difficulty":7,"share_id":"share-33","share_seq":33,"template_height":800001},{"accepted_at_ms":70504,"credit_policy":null,"job_id":"job-34","job_issued_at_ms":70486,"miner_id":"miner-1","network_difficulty":1000000,"ntime":1700000034,"order_key":"01:miner-1","p2mr_program_hex":"2222222222222222222222222222222222222222222222222222222222222222","share_difficulty":3,"share_id":"share-34","share_seq":34,"template_height":800001},{"accepted_at_ms":70502,"credit_policy":"stale-grace","job_id":"job-35","job_issued_at_ms":70500,"miner_id":"miner-2","network_difficulty":1000001,"ntime":1700000035,"order_key":"02:miner-2","p2mr_program_hex":"2323232323232323232323232323232323232323232323232323232323232323","share_difficulty":2,"share_id":"share-35","share_seq":35,"template_height":800001},{"accepted_at_ms":70485,"credit_policy":null,"job_id":"job-36","job_issued_at_ms":70504,"miner_id":"miner-3","network_difficulty":1000002,"ntime":1700000036,"order_key":"03:miner-3","p2mr_program_hex":"2424242424242424242424242424242424242424242424242424242424242424","share_difficulty":7,"share_id":"share-36","share_seq":36,"template_height":800001},{"accepted_at_ms":70507,"credit_policy":null,"job_id":"job-37","job_issued_at_ms":70509,"miner_id":"miner-4","network_difficulty":1000003,"ntime":1700000037,"order_key":"04:miner-4","p2mr_program_hex":"2525252525252525252525252525252525252525252525252525252525252525","share_difficulty":2,"share_id":"share-37","share_seq":37,"template_height":800001},{"accepted_at_ms":70496,"credit_policy":null,"job_id":"job-38","job_issued_at_ms":70493,"miner_id":"miner-5","network_difficulty":1000004,"ntime":1700000038,"order_key":"05:miner-5","p2mr_program_hex":"2626262626262626262626262626262626262626262626262626262626262626","share_difficulty":3,"share_id":"share-38","share_seq":38,"template_height":800001},{"accepted_at_ms":70510,"credit_policy":null,"job_id":"job-39","job_issued_at_ms":70497,"miner_id":"miner-6","network_difficulty":1000005,"ntime":1700000039,"order_key":"06:miner-6","p2mr_program_hex":"2727272727272727272727272727272727272727272727272727272727272727","share_difficulty":3,"share_id":"share-39","share_seq":39,"template_height":800001},{"accepted_at_ms":70509,"credit_policy":"stale-grace","job_id":"job-40","job_issued_at_ms":70489,"miner_id":"miner-7","network_difficulty":1000006,"ntime":1700000040,"order_key":"07:miner-7","p2mr_program_hex":"2828282828282828282828282828282828282828282828282828282828282828","share_difficulty":2,"share_id":"share-40","share_seq":40,"template_height":800001},{"accepted_at_ms":70481,"credit_policy":null,"job_id":"job-41","job_issued_at_ms":70495,"miner_id":"miner-8","network_difficulty":1000007,"ntime":1700000041,"order_key":"08:miner-8","p2mr_program_hex":"2929292929292929292929292929292929292929292929292929292929292929","share_difficulty":7,"share_id":"share-41","share_seq":41,"template_height":800001},{"accepted_at_ms":70506,"credit_policy":null,"job_id":"job-42","job_issued_at_ms":70487,"miner_id":"miner-9","network_difficulty":1000008,"ntime":1700000042,"order_key":"09:miner-9","p2mr_program_hex":"2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a","share_difficulty":19,"share_id":"share-42","share_seq":42,"template_height":800001},{"accepted_at_ms":70493,"credit_policy":null,"job_id":"job-43","job_issued_at_ms":70489,"miner_id":"miner-10","network_difficulty":1000009,"ntime":1700000043,"order_key":"10:miner-10","p2mr_program_hex":"2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b","share_difficulty":1,"share_id":"share-43","share_seq":43,"template_height":800001},{"accepted_at_ms":70495,"credit_policy":null,"job_id":"job-44","job_issued_at_ms":70507,"miner_id":"miner-0","network_difficulty":1000010,"ntime":1700000044,"order_key":"00:miner-0","p2mr_program_hex":"2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c","share_difficulty":7,"share_id":"share-44","share_seq":44,"template_height":800001},{"accepted_at_ms":70506,"credit_policy":null,"job_id":"job-45","job_issued_at_ms":70497,"miner_id":"miner-1","network_difficulty":1000011,"ntime":1700000045,"order_key":"01:miner-1","p2mr_program_hex":"2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d","share_difficulty":3,"share_id":"share-45","share_seq":45,"template_height":800001},{"accepted_at_ms":70498,"credit_policy":null,"job_id":"job-46","job_issued_at_ms":70511,"miner_id":"miner-2","network_difficulty":1000012,"ntime":1700000046,"order_key":"02:miner-2","p2mr_program_hex":"2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e","share_difficulty":3,"share_id":"share-46","share_seq":46,"template_height":800001},{"accepted_at_ms":70482,"credit_policy":null,"job_id":"job-47","job_issued_at_ms":70501,"miner_id":"miner-3","network_difficulty":1000013,"ntime":1700000047,"order_key":"03:miner-3","p2mr_program_hex":"2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f","share_difficulty":2,"share_id":"share-47","share_seq":47,"template_height":800001},{"accepted_at_ms":70482,"credit_policy":null,"job_id":"job-48","job_issued_at_ms":70511,"miner_id":"miner-4","network_difficulty":1000014,"ntime":1700000048,"order_key":"04:miner-4","p2mr_program_hex":"3030303030303030303030303030303030303030303030303030303030303030","share_difficulty":19,"share_id":"share-48","share_seq":48,"template_height":800001},{"accepted_at_ms":70505,"credit_policy":null,"job_id":"job-49","job_issued_at_ms":70483,"miner_id":"miner-5","network_difficulty":1000015,"ntime":1700000049,"order_key":"05:miner-5","p2mr_program_hex":"3131313131313131313131313131313131313131313131313131313131313131","share_difficulty":19,"share_id":"share-49","share_seq":49,"template_height":800001},{"accepted_at_ms":70498,"credit_policy":null,"job_id":"job-50","job_issued_at_ms":70492,"miner_id":"miner-6","network_difficulty":1000016,"ntime":1700000050,"order_key":"06:miner-6","p2mr_program_hex":"3232323232323232323232323232323232323232323232323232323232323232","share_difficulty":1,"share_id":"share-50","share_seq":50,"template_height":800001},{"accepted_at_ms":70502,"credit_policy":null,"job_id":"job-51","job_issued_at_ms":70487,"miner_id":"miner-7","network_difficulty":1000000,"ntime":1700000051,"order_key":"07:miner-7","p2mr_program_hex":"3333333333333333333333333333333333333333333333333333333333333333","share_difficulty":19,"share_id":"share-51","share_seq":51,"template_height":800001},{"accepted_at_ms":70502,"credit_policy":"stale-grace","job_id":"job-52","job_issued_at_ms":70485,"miner_id":"miner-8","network_difficulty":1000001,"ntime":1700000052,"order_key":"08:miner-8","p2mr_program_hex":"3434343434343434343434343434343434343434343434343434343434343434","share_difficulty":7,"share_id":"share-52","share_seq":52,"template_height":800001},{"accepted_at_ms":70481,"credit_policy":null,"job_id":"job-53","job_issued_at_ms":70490,"miner_id":"miner-9","network_difficulty":1000002,"ntime":1700000053,"order_key":"09:miner-9","p2mr_program_hex":"3535353535353535353535353535353535353535353535353535353535353535","share_difficulty":2,"share_id":"share-53","share_seq":53,"template_height":800001},{"accepted_at_ms":70481,"credit_policy":null,"job_id":"job-54","job_issued_at_ms":70497,"miner_id":"miner-10","network_difficulty":1000003,"ntime":1700000054,"order_key":"10:miner-10","p2mr_program_hex":"3636363636363636363636363636363636363636363636363636363636363636","share_difficulty":19,"share_id":"share-54","share_seq":54,"template_height":800001},{"accepted_at_ms":70499,"credit_policy":null,"job_id":"job-55","job_issued_at_ms":70487,"miner_id":"miner-0","network_difficulty":1000004,"ntime":1700000055,"order_key":"00:miner-0","p2mr_program_hex":"3737373737373737373737373737373737373737373737373737373737373737","share_difficulty":2,"share_id":"share-55","share_seq":55,"template_height":800001},{"accepted_at_ms":70501,"credit_policy":null,"job_id":"job-56","job_issued_at_ms":70499,"miner_id":"miner-1","network_difficulty":1000005,"ntime":1700000056,"order_key":"01:miner-1","p2mr_program_hex":"3838383838383838383838383838383838383838383838383838383838383838","share_difficulty":3,"share_id":"share-56","share_seq":56,"template_height":800001},{"accepted_at_ms":70492,"credit_policy":null,"job_id":"job-57","job_issued_at_ms":70483,"miner_id":"miner-2","network_difficulty":1000006,"ntime":1700000057,"order_key":"02:miner-2","p2mr_program_hex":"3939393939393939393939393939393939393939393939393939393939393939","share_difficulty":1,"share_id":"share-57","share_seq":57,"template_height":800001},{"accepted_at_ms":70484,"credit_policy":null,"job_id":"job-58","job_issued_at_ms":70491,"miner_id":"miner-3","network_difficulty":1000007,"ntime":1700000058,"order_key":"03:miner-3","p2mr_program_hex":"3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a","share_difficulty":19,"share_id":"share-58","share_seq":58,"template_height":800001},{"accepted_at_ms":70511,"credit_policy":"stale-grace","job_id":"job-59","job_issued_at_ms":70485,"miner_id":"miner-4","network_difficulty":1000008,"ntime":1700000059,"order_key":"04:miner-4","p2mr_program_hex":"3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b","share_difficulty":7,"share_id":"share-59","share_seq":59,"template_height":800001},{"accepted_at_ms":70505,"credit_policy":null,"job_id":"job-60","job_issued_at_ms":70502,"miner_id":"miner-5","network_difficulty":1000009,"ntime":1700000060,"order_key":"05:miner-5","p2mr_program_hex":"3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c","share_difficulty":19,"share_id":"share-60","share_seq":60,"template_height":800001},{"accepted_at_ms":70483,"credit_policy":"stale-grace","job_id":"job-61","job_issued_at_ms":70483,"miner_id":"miner-6","network_difficulty":1000010,"ntime":1700000061,"order_key":"06:miner-6","p2mr_program_hex":"3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d","share_difficulty":2,"share_id":"share-61","share_seq":61,"template_height":800001},{"accepted_at_ms":70489,"credit_policy":null,"job_id":"job-62","job_issued_at_ms":70499,"miner_id":"miner-7","network_difficulty":1000011,"ntime":1700000062,"order_key":"07:miner-7","p2mr_program_hex":"3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e","share_difficulty":19,"share_id":"share-62","share_seq":62,"template_height":800001},{"accepted_at_ms":70508,"credit_policy":null,"job_id":"job-63","job_issued_at_ms":70491,"miner_id":"miner-8","network_difficulty":1000012,"ntime":1700000063,"order_key":"08:miner-8","p2mr_program_hex":"3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f","share_difficulty":1,"share_id":"share-63","share_seq":63,"template_height":800001},{"accepted_at_ms":70502,"credit_policy":null,"job_id":"job-64","job_issued_at_ms":70485,"miner_id":"miner-9","network_difficulty":1000013,"ntime":1700000064,"order_key":"09:miner-9","p2mr_program_hex":"4040404040404040404040404040404040404040404040404040404040404040","share_difficulty":19,"share_id":"share-64","share_seq":64,"template_height":800002},{"accepted_at_ms":70482,"credit_policy":null,"job_id":"job-65","job_issued_at_ms":70484,"miner_id":"miner-10","network_difficulty":1000014,"ntime":1700000065,"order_key":"10:miner-10","p2mr_program_hex":"4141414141414141414141414141414141414141414141414141414141414141","share_difficulty":7,"share_id":"share-65","share_seq":65,"template_height":800002},{"accepted_at_ms":70507,"credit_policy":"stale-grace","job_id":"job-66","job_issued_at_ms":70501,"miner_id":"miner-0","network_difficulty":1000015,"ntime":1700000066,"order_key":"00:miner-0","p2mr_program_hex":"4242424242424242424242424242424242424242424242424242424242424242","share_difficulty":2,"share_id":"share-66","share_seq":66,"template_height":800002},{"accepted_at_ms":70483,"credit_policy":null,"job_id":"job-67","job_issued_at_ms":70486,"miner_id":"miner-1","network_difficulty":1000016,"ntime":1700000067,"order_key":"01:miner-1","p2mr_program_hex":"4343434343434343434343434343434343434343434343434343434343434343","share_difficulty":3,"share_id":"share-67","share_seq":67,"template_height":800002},{"accepted_at_ms":70486,"credit_policy":null,"job_id":"job-68","job_issued_at_ms":70509,"miner_id":"miner-2","network_difficulty":1000000,"ntime":1700000068,"order_key":"02:miner-2","p2mr_program_hex":"4444444444444444444444444444444444444444444444444444444444444444","share_difficulty":2,"share_id":"share-68","share_seq":68,"template_height":800002},{"accepted_at_ms":70497,"credit_policy":"stale-grace","job_id":"job-69","job_issued_at_ms":70504,"miner_id":"miner-3","network_difficulty":1000001,"ntime":1700000069,"order_key":"03:miner-3","p2mr_program_hex":"4545454545454545454545454545454545454545454545454545454545454545","share_difficulty":1,"share_id":"share-69","share_seq":69,"template_height":800002},{"accepted_at_ms":70491,"credit_policy":"stale-grace","job_id":"job-70","job_issued_at_ms":70491,"miner_id":"miner-4","network_difficulty":1000002,"ntime":1700000070,"order_key":"04:miner-4","p2mr_program_hex":"4646464646464646464646464646464646464646464646464646464646464646","share_difficulty":1,"share_id":"share-70","share_seq":70,"template_height":800002},{"accepted_at_ms":70501,"credit_policy":null,"job_id":"job-71","job_issued_at_ms":70481,"miner_id":"miner-5","network_difficulty":1000003,"ntime":1700000071,"order_key":"05:miner-5","p2mr_program_hex":"4747474747474747474747474747474747474747474747474747474747474747","share_difficulty":2,"share_id":"share-71","share_seq":71,"template_height":800002},{"accepted_at_ms":70488,"credit_policy":null,"job_id":"job-72","job_issued_at_ms":70505,"miner_id":"miner-6","network_difficulty":1000004,"ntime":1700000072,"order_key":"06:miner-6","p2mr_program_hex":"4848484848484848484848484848484848484848484848484848484848484848","share_difficulty":19,"share_id":"share-72","share_seq":72,"template_height":800002},{"accepted_at_ms":70495,"credit_policy":null,"job_id":"job-73","job_issued_at_ms":70510,"miner_id":"miner-7","network_difficulty":1000005,"ntime":1700000073,"order_key":"07:miner-7","p2mr_program_hex":"4949494949494949494949494949494949494949494949494949494949494949","share_difficulty":1,"share_id":"share-73","share_seq":73,"template_height":800002},{"accepted_at_ms":70481,"credit_policy":null,"job_id":"job-74","job_issued_at_ms":70494,"miner_id":"miner-8","network_difficulty":1000006,"ntime":1700000074,"order_key":"08:miner-8","p2mr_program_hex":"4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a","share_difficulty":19,"share_id":"share-74","share_seq":74,"template_height":800002},{"accepted_at_ms":70483,"credit_policy":null,"job_id":"job-75","job_issued_at_ms":70501,"miner_id":"miner-9","network_difficulty":1000007,"ntime":1700000075,"order_key":"09:miner-9","p2mr_program_hex":"4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b","share_difficulty":1,"share_id":"share-75","share_seq":75,"template_height":800002},{"accepted_at_ms":70502,"credit_policy":null,"job_id":"job-76","job_issued_at_ms":70503,"miner_id":"miner-10","network_difficulty":1000008,"ntime":1700000076,"order_key":"10:miner-10","p2mr_program_hex":"4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c","share_difficulty":7,"share_id":"share-76","share_seq":76,"template_height":800002},{"accepted_at_ms":70509,"credit_policy":null,"job_id":"job-77","job_issued_at_ms":70504,"miner_id":"miner-0","network_difficulty":1000009,"ntime":1700000077,"order_key":"00:miner-0","p2mr_program_hex":"4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d","share_difficulty":2,"share_id":"share-77","share_seq":77,"template_height":800002},{"accepted_at_ms":70501,"credit_policy":null,"job_id":"job-78","job_issued_at_ms":70502,"miner_id":"miner-1","network_difficulty":1000010,"ntime":1700000078,"order_key":"01:miner-1","p2mr_program_hex":"4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e","share_difficulty":3,"share_id":"share-78","share_seq":78,"template_height":800002},{"accepted_at_ms":70503,"credit_policy":null,"job_id":"job-79","job_issued_at_ms":70504,"miner_id":"miner-2","network_difficulty":1000011,"ntime":1700000079,"order_key":"02:miner-2","p2mr_program_hex":"4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f","share_difficulty":2,"share_id":"share-79","share_seq":79,"template_height":800002},{"accepted_at_ms":70501,"credit_policy":"stale-grace","job_id":"job-80","job_issued_at_ms":70500,"miner_id":"miner-3","network_difficulty":1000012,"ntime":1700000080,"order_key":"03:miner-3","p2mr_program_hex":"5050505050505050505050505050505050505050505050505050505050505050","share_difficulty":3,"share_id":"share-80","share_seq":80,"template_height":800002},{"accepted_at_ms":70487,"credit_policy":null,"job_id":"job-81","job_issued_at_ms":70502,"miner_id":"miner-4","network_difficulty":1000013,"ntime":1700000081,"order_key":"04:miner-4","p2mr_program_hex":"5151515151515151515151515151515151515151515151515151515151515151","share_difficulty":2,"share_id":"share-81","share_seq":81,"template_height":800002},{"accepted_at_ms":70507,"credit_policy":null,"job_id":"job-82","job_issued_at_ms":70511,"miner_id":"miner-5","network_difficulty":1000014,"ntime":1700000082,"order_key":"05:miner-5","p2mr_program_hex":"5252525252525252525252525252525252525252525252525252525252525252","share_difficulty":2,"share_id":"share-82","share_seq":82,"template_height":800002},{"accepted_at_ms":70493,"credit_policy":null,"job_id":"job-83","job_issued_at_ms":70490,"miner_id":"miner-6","network_difficulty":1000015,"ntime":1700000083,"order_key":"06:miner-6","p2mr_program_hex":"5353535353535353535353535353535353535353535353535353535353535353","share_difficulty":2,"share_id":"share-83","share_seq":83,"template_height":800002},{"accepted_at_ms":70510,"credit_policy":"stale-grace","job_id":"job-84","job_issued_at_ms":70487,"miner_id":"miner-7","network_difficulty":1000016,"ntime":1700000084,"order_key":"07:miner-7","p2mr_program_hex":"5454545454545454545454545454545454545454545454545454545454545454","share_difficulty":3,"share_id":"share-84","share_seq":84,"template_height":800002},{"accepted_at_ms":70483,"credit_policy":null,"job_id":"job-85","job_issued_at_ms":70492,"miner_id":"miner-8","network_difficulty":1000000,"ntime":1700000085,"order_key":"08:miner-8","p2mr_program_hex":"5555555555555555555555555555555555555555555555555555555555555555","share_difficulty":19,"share_id":"share-85","share_seq":85,"template_height":800002},{"accepted_at_ms":70490,"credit_policy":null,"job_id":"job-86","job_issued_at_ms":70488,"miner_id":"miner-9","network_difficulty":1000001,"ntime":1700000086,"order_key":"09:miner-9","p2mr_program_hex":"5656565656565656565656565656565656565656565656565656565656565656","share_difficulty":1,"share_id":"share-86","share_seq":86,"template_height":800002},{"accepted_at_ms":70507,"credit_policy":"stale-grace","job_id":"job-87","job_issued_at_ms":70506,"miner_id":"miner-10","network_difficulty":1000002,"ntime":1700000087,"order_key":"10:miner-10","p2mr_program_hex":"5757575757575757575757575757575757575757575757575757575757575757","share_difficulty":1,"share_id":"share-87","share_seq":87,"template_height":800002},{"accepted_at_ms":70491,"credit_policy":null,"job_id":"job-88","job_issued_at_ms":70500,"miner_id":"miner-0","network_difficulty":1000003,"ntime":1700000088,"order_key":"00:miner-0","p2mr_program_hex":"5858585858585858585858585858585858585858585858585858585858585858","share_difficulty":7,"share_id":"share-88","share_seq":88,"template_height":800002},{"accepted_at_ms":70497,"credit_policy":null,"job_id":"job-89","job_issued_at_ms":70505,"miner_id":"miner-1","network_difficulty":1000004,"ntime":1700000089,"order_key":"01:miner-1","p2mr_program_hex":"5959595959595959595959595959595959595959595959595959595959595959","share_difficulty":7,"share_id":"share-89","share_seq":89,"template_height":800002},{"accepted_at_ms":70487,"credit_policy":"stale-grace","job_id":"job-90","job_issued_at_ms":70500,"miner_id":"miner-2","network_difficulty":1000005,"ntime":1700000090,"order_key":"02:miner-2","p2mr_program_hex":"5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a","share_difficulty":7,"share_id":"share-90","share_seq":90,"template_height":800002},{"accepted_at_ms":70508,"credit_policy":null,"job_id":"job-91","job_issued_at_ms":70492,"miner_id":"miner-3","network_difficulty":1000006,"ntime":1700000091,"order_key":"03:miner-3","p2mr_program_hex":"5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b","share_difficulty":19,"share_id":"share-91","share_seq":91,"template_height":800002},{"accepted_at_ms":70490,"credit_policy":"stale-grace","job_id":"job-92","job_issued_at_ms":70492,"miner_id":"miner-4","network_difficulty":1000007,"ntime":1700000092,"order_key":"04:miner-4","p2mr_program_hex":"5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c","share_difficulty":7,"share_id":"share-92","share_seq":92,"template_height":800002},{"accepted_at_ms":70494,"credit_policy":"stale-grace","job_id":"job-93","job_issued_at_ms":70492,"miner_id":"miner-5","network_difficulty":1000008,"ntime":1700000093,"order_key":"05:miner-5","p2mr_program_hex":"5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d","share_difficulty":1,"share_id":"share-93","share_seq":93,"template_height":800002},{"accepted_at_ms":70510,"credit_policy":null,"job_id":"job-94","job_issued_at_ms":70508,"miner_id":"miner-6","network_difficulty":1000009,"ntime":1700000094,"order_key":"06:miner-6","p2mr_program_hex":"5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e","share_difficulty":3,"share_id":"share-94","share_seq":94,"template_height":800002},{"accepted_at_ms":70484,"credit_policy":null,"job_id":"job-95","job_issued_at_ms":70495,"miner_id":"miner-7","network_difficulty":1000010,"ntime":1700000095,"order_key":"07:miner-7","p2mr_program_hex":"5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f","share_difficulty":19,"share_id":"share-95","share_seq":95,"template_height":800002},{"accepted_at_ms":70505,"credit_policy":null,"job_id":"job-96","job_issued_at_ms":70489,"miner_id":"miner-8","network_difficulty":1000011,"ntime":1700000096,"order_key":"08:miner-8","p2mr_program_hex":"6060606060606060606060606060606060606060606060606060606060606060","share_difficulty":3,"share_id":"share-96","share_seq":96,"template_height":800003},{"accepted_at_ms":70505,"credit_policy":null,"job_id":"job-97","job_issued_at_ms":70487,"miner_id":"miner-9","network_difficulty":1000012,"ntime":1700000097,"order_key":"09:miner-9","p2mr_program_hex":"6161616161616161616161616161616161616161616161616161616161616161","share_difficulty":2,"share_id":"share-97","share_seq":97,"template_height":800003},{"accepted_at_ms":70485,"credit_policy":null,"job_id":"job-98","job_issued_at_ms":70490,"miner_id":"miner-10","network_difficulty":1000013,"ntime":1700000098,"order_key":"10:miner-10","p2mr_program_hex":"6262626262626262626262626262626262626262626262626262626262626262","share_difficulty":3,"share_id":"share-98","share_seq":98,"template_height":800003},{"accepted_at_ms":70492,"credit_policy":"stale-grace","job_id":"job-99","job_issued_at_ms":70484,"miner_id":"miner-0","network_difficulty":1000014,"ntime":1700000099,"order_key":"00:miner-0","p2mr_program_hex":"6363636363636363636363636363636363636363636363636363636363636363","share_difficulty":7,"share_id":"share-99","share_seq":99,"template_height":800003},{"accepted_at_ms":70483,"credit_policy":null,"job_id":"job-100","job_issued_at_ms":70499,"miner_id":"miner-1","network_difficulty":1000015,"ntime":1700000100,"order_key":"01:miner-1","p2mr_program_hex":"6464646464646464646464646464646464646464646464646464646464646464","share_difficulty":19,"share_id":"share-100","share_seq":100,"template_height":800003},{"accepted_at_ms":70491,"credit_policy":null,"job_id":"job-101","job_issued_at_ms":70507,"miner_id":"miner-2","network_difficulty":1000016,"ntime":1700000101,"order_key":"02:miner-2","p2mr_program_hex":"6565656565656565656565656565656565656565656565656565656565656565","share_difficulty":2,"share_id":"share-101","share_seq":101,"template_height":800003},{"accepted_at_ms":70491,"credit_policy":null,"job_id":"job-102","job_issued_at_ms":70505,"miner_id":"miner-3","network_difficulty":1000000,"ntime":1700000102,"order_key":"03:miner-3","p2mr_program_hex":"6666666666666666666666666666666666666666666666666666666666666666","share_difficulty":19,"share_id":"share-102","share_seq":102,"template_height":800003},{"accepted_at_ms":70511,"credit_policy":"stale-grace","job_id":"job-103","job_issued_at_ms":70500,"miner_id":"miner-4","network_difficulty":1000001,"ntime":1700000103,"order_key":"04:miner-4","p2mr_program_hex":"6767676767676767676767676767676767676767676767676767676767676767","share_difficulty":1,"share_id":"share-103","share_seq":103,"template_height":800003},{"accepted_at_ms":70506,"credit_policy":null,"job_id":"job-104","job_issued_at_ms":70506,"miner_id":"miner-5","network_difficulty":1000002,"ntime":1700000104,"order_key":"05:miner-5","p2mr_program_hex":"6868686868686868686868686868686868686868686868686868686868686868","share_difficulty":1,"share_id":"share-104","share_seq":104,"template_height":800003},{"accepted_at_ms":70482,"credit_policy":null,"job_id":"job-105","job_issued_at_ms":70505,"miner_id":"miner-6","network_difficulty":1000003,"ntime":1700000105,"order_key":"06:miner-6","p2mr_program_hex":"6969696969696969696969696969696969696969696969696969696969696969","share_difficulty":2,"share_id":"share-105","share_seq":105,"template_height":800003},{"accepted_at_ms":70509,"credit_policy":null,"job_id":"job-106","job_issued_at_ms":70505,"miner_id":"miner-7","network_difficulty":1000004,"ntime":1700000106,"order_key":"07:miner-7","p2mr_program_hex":"6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a","share_difficulty":3,"share_id":"share-106","share_seq":106,"template_height":800003},{"accepted_at_ms":70510,"credit_policy":null,"job_id":"job-107","job_issued_at_ms":70511,"miner_id":"miner-8","network_difficulty":1000005,"ntime":1700000107,"order_key":"08:miner-8","p2mr_program_hex":"6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b","share_difficulty":7,"share_id":"share-107","share_seq":107,"template_height":800003},{"accepted_at_ms":70508,"credit_policy":null,"job_id":"job-108","job_issued_at_ms":70504,"miner_id":"miner-9","network_difficulty":1000006,"ntime":1700000108,"order_key":"09:miner-9","p2mr_program_hex":"6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c","share_difficulty":7,"share_id":"share-108","share_seq":108,"template_height":800003},{"accepted_at_ms":70496,"credit_policy":"stale-grace","job_id":"job-109","job_issued_at_ms":70495,"miner_id":"miner-10","network_difficulty":1000007,"ntime":1700000109,"order_key":"10:miner-10","p2mr_program_hex":"6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d","share_difficulty":7,"share_id":"share-109","share_seq":109,"template_height":800003},{"accepted_at_ms":70503,"credit_policy":null,"job_id":"job-110","job_issued_at_ms":70496,"miner_id":"miner-0","network_difficulty":1000008,"ntime":1700000110,"order_key":"00:miner-0","p2mr_program_hex":"6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e","share_difficulty":7,"share_id":"share-110","share_seq":110,"template_height":800003},{"accepted_at_ms":70508,"credit_policy":null,"job_id":"job-111","job_issued_at_ms":70509,"miner_id":"miner-1","network_difficulty":1000009,"ntime":1700000111,"order_key":"01:miner-1","p2mr_program_hex":"6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f","share_difficulty":3,"share_id":"share-111","share_seq":111,"template_height":800003},{"accepted_at_ms":70491,"credit_policy":null,"job_id":"job-112","job_issued_at_ms":70483,"miner_id":"miner-2","network_difficulty":1000010,"ntime":1700000112,"order_key":"02:miner-2","p2mr_program_hex":"7070707070707070707070707070707070707070707070707070707070707070","share_difficulty":19,"share_id":"share-112","share_seq":112,"template_height":800003},{"accepted_at_ms":70488,"credit_policy":null,"job_id":"job-113","job_issued_at_ms":70492,"miner_id":"miner-3","network_difficulty":1000011,"ntime":1700000113,"order_key":"03:miner-3","p2mr_program_hex":"7171717171717171717171717171717171717171717171717171717171717171","share_difficulty":1,"share_id":"share-113","share_seq":113,"template_height":800003},{"accepted_at_ms":70487,"credit_policy":null,"job_id":"job-114","job_issued_at_ms":70505,"miner_id":"miner-4","network_difficulty":1000012,"ntime":1700000114,"order_key":"04:miner-4","p2mr_program_hex":"7272727272727272727272727272727272727272727272727272727272727272","share_difficulty":19,"share_id":"share-114","share_seq":114,"template_height":800003},{"accepted_at_ms":70482,"credit_policy":"stale-grace","job_id":"job-115","job_issued_at_ms":70505,"miner_id":"miner-5","network_difficulty":1000013,"ntime":1700000115,"order_key":"05:miner-5","p2mr_program_hex":"7373737373737373737373737373737373737373737373737373737373737373","share_difficulty":7,"share_id":"share-115","share_seq":115,"template_height":800003},{"accepted_at_ms":70493,"credit_policy":null,"job_id":"job-116","job_issued_at_ms":70493,"miner_id":"miner-6","network_difficulty":1000014,"ntime":1700000116,"order_key":"06:miner-6","p2mr_program_hex":"7474747474747474747474747474747474747474747474747474747474747474","share_difficulty":3,"share_id":"share-116","share_seq":116,"template_height":800003},{"accepted_at_ms":70488,"credit_policy":"stale-grace","job_id":"job-117","job_issued_at_ms":70494,"miner_id":"miner-7","network_difficulty":1000015,"ntime":1700000117,"order_key":"07:miner-7","p2mr_program_hex":"7575757575757575757575757575757575757575757575757575757575757575","share_difficulty":3,"share_id":"share-117","share_seq":117,"template_height":800003},{"accepted_at_ms":70493,"credit_policy":"stale-grace","job_id":"job-118","job_issued_at_ms":70508,"miner_id":"miner-8","network_difficulty":1000016,"ntime":1700000118,"order_key":"08:miner-8","p2mr_program_hex":"7676767676767676767676767676767676767676767676767676767676767676","share_difficulty":7,"share_id":"share-118","share_seq":118,"template_height":800003},{"accepted_at_ms":70508,"credit_policy":null,"job_id":"job-119","job_issued_at_ms":70510,"miner_id":"miner-9","network_difficulty":1000000,"ntime":1700000119,"order_key":"09:miner-9","p2mr_program_hex":"7777777777777777777777777777777777777777777777777777777777777777","share_difficulty":19,"share_id":"share-119","share_seq":119,"template_height":800003},{"accepted_at_ms":70483,"credit_policy":null,"job_id":"job-120","job_issued_at_ms":70483,"miner_id":"miner-10","network_difficulty":1000001,"ntime":1700000120,"order_key":"10:miner-10","p2mr_program_hex":"7878787878787878787878787878787878787878787878787878787878787878","share_difficulty":19,"share_id":"share-120","share_seq":120,"template_height":800003},{"accepted_at_ms":70490,"credit_policy":null,"job_id":"job-121","job_issued_at_ms":70509,"miner_id":"miner-0","network_difficulty":1000002,"ntime":1700000121,"order_key":"00:miner-0","p2mr_program_hex":"7979797979797979797979797979797979797979797979797979797979797979","share_difficulty":1,"share_id":"share-121","share_seq":121,"template_height":800003},{"accepted_at_ms":70503,"credit_policy":null,"job_id":"job-122","job_issued_at_ms":70494,"miner_id":"miner-1","network_difficulty":1000003,"ntime":1700000122,"order_key":"01:miner-1","p2mr_program_hex":"7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a","share_difficulty":7,"share_id":"share-122","share_seq":122,"template_height":800003},{"accepted_at_ms":70508,"credit_policy":null,"job_id":"job-123","job_issued_at_ms":70499,"miner_id":"miner-2","network_difficulty":1000004,"ntime":1700000123,"order_key":"02:miner-2","p2mr_program_hex":"7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b","share_difficulty":19,"share_id":"share-123","share_seq":123,"template_height":800003},{"accepted_at_ms":70489,"credit_policy":null,"job_id":"job-124","job_issued_at_ms":70502,"miner_id":"miner-3","network_difficulty":1000005,"ntime":1700000124,"order_key":"03:miner-3","p2mr_program_hex":"7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c","share_difficulty":7,"share_id":"share-124","share_seq":124,"template_height":800003},{"accepted_at_ms":70493,"credit_policy":"stale-grace","job_id":"job-125","job_issued_at_ms":70497,"miner_id":"miner-4","network_difficulty":1000006,"ntime":1700000125,"order_key":"04:miner-4","p2mr_program_hex":"7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d","share_difficulty":2,"share_id":"share-125","share_seq":125,"template_height":800003},{"accepted_at_ms":70482,"credit_policy":null,"job_id":"job-126","job_issued_at_ms":70485,"miner_id":"miner-5","network_difficulty":1000007,"ntime":1700000126,"order_key":"05:miner-5","p2mr_program_hex":"7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e","share_difficulty":7,"share_id":"share-126","share_seq":126,"template_height":800003},{"accepted_at_ms":70481,"credit_policy":"stale-grace","job_id":"job-127","job_issued_at_ms":70498,"miner_id":"miner-6","network_difficulty":1000008,"ntime":1700000127,"order_key":"06:miner-6","p2mr_program_hex":"7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f","share_difficulty":1,"share_id":"share-127","share_seq":127,"template_height":800003},{"accepted_at_ms":70498,"credit_policy":"stale-grace","job_id":"job-128","job_issued_at_ms":70484,"miner_id":"miner-7","network_difficulty":1000009,"ntime":1700000128,"order_key":"07:miner-7","p2mr_program_hex":"8080808080808080808080808080808080808080808080808080808080808080","share_difficulty":19,"share_id":"share-128","share_seq":128,"template_height":800004},{"accepted_at_ms":70497,"credit_policy":null,"job_id":"job-129","job_issued_at_ms":70498,"miner_id":"miner-8","network_difficulty":1000010,"ntime":1700000129,"order_key":"08:miner-8","p2mr_program_hex":"8181818181818181818181818181818181818181818181818181818181818181","share_difficulty":2,"share_id":"share-129","share_seq":129,"template_height":800004},{"accepted_at_ms":70486,"credit_policy":null,"job_id":"job-130","job_issued_at_ms":70493,"miner_id":"miner-9","network_difficulty":1000011,"ntime":1700000130,"order_key":"09:miner-9","p2mr_program_hex":"8282828282828282828282828282828282828282828282828282828282828282","share_difficulty":7,"share_id":"share-130","share_seq":130,"template_height":800004},{"accepted_at_ms":70495,"credit_policy":null,"job_id":"job-131","job_issued_at_ms":70508,"miner_id":"miner-10","network_difficulty":1000012,"ntime":1700000131,"order_key":"10:miner-10","p2mr_program_hex":"8383838383838383838383838383838383838383838383838383838383838383","share_difficulty":2,"share_id":"share-131","share_seq":131,"template_height":800004},{"accepted_at_ms":70503,"credit_policy":null,"job_id":"job-132","job_issued_at_ms":70497,"miner_id":"miner-0","network_difficulty":1000013,"ntime":1700000132,"order_key":"00:miner-0","p2mr_program_hex":"8484848484848484848484848484848484848484848484848484848484848484","share_difficulty":1,"share_id":"share-132","share_seq":132,"template_height":800004},{"accepted_at_ms":70495,"credit_policy":null,"job_id":"job-133","job_issued_at_ms":70501,"miner_id":"miner-1","network_difficulty":1000014,"ntime":1700000133,"order_key":"01:miner-1","p2mr_program_hex":"8585858585858585858585858585858585858585858585858585858585858585","share_difficulty":19,"share_id":"share-133","share_seq":133,"template_height":800004},{"accepted_at_ms":70506,"credit_policy":null,"job_id":"job-134","job_issued_at_ms":70499,"miner_id":"miner-2","network_difficulty":1000015,"ntime":1700000134,"order_key":"02:miner-2","p2mr_program_hex":"8686868686868686868686868686868686868686868686868686868686868686","share_difficulty":3,"share_id":"share-134","share_seq":134,"template_height":800004},{"accepted_at_ms":70510,"credit_policy":"stale-grace","job_id":"job-135","job_issued_at_ms":70499,"miner_id":"miner-3","network_difficulty":1000016,"ntime":1700000135,"order_key":"03:miner-3","p2mr_program_hex":"8787878787878787878787878787878787878787878787878787878787878787","share_difficulty":19,"share_id":"share-135","share_seq":135,"template_height":800004},{"accepted_at_ms":70482,"credit_policy":"stale-grace","job_id":"job-136","job_issued_at_ms":70508,"miner_id":"miner-4","network_difficulty":1000000,"ntime":1700000136,"order_key":"04:miner-4","p2mr_program_hex":"8888888888888888888888888888888888888888888888888888888888888888","share_difficulty":1,"share_id":"share-136","share_seq":136,"template_height":800004},{"accepted_at_ms":70497,"credit_policy":null,"job_id":"job-137","job_issued_at_ms":70486,"miner_id":"miner-5","network_difficulty":1000001,"ntime":1700000137,"order_key":"05:miner-5","p2mr_program_hex":"8989898989898989898989898989898989898989898989898989898989898989","share_difficulty":1,"share_id":"share-137","share_seq":137,"template_height":800004},{"accepted_at_ms":70494,"credit_policy":null,"job_id":"job-138","job_issued_at_ms":70491,"miner_id":"miner-6","network_difficulty":1000002,"ntime":1700000138,"order_key":"06:miner-6","p2mr_program_hex":"8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a","share_difficulty":3,"share_id":"share-138","share_seq":138,"template_height":800004},{"accepted_at_ms":70501,"credit_policy":null,"job_id":"job-139","job_issued_at_ms":70493,"miner_id":"miner-7","network_difficulty":1000003,"ntime":1700000139,"order_key":"07:miner-7","p2mr_program_hex":"8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b","share_difficulty":3,"share_id":"share-139","share_seq":139,"template_height":800004},{"accepted_at_ms":70504,"credit_policy":null,"job_id":"job-140","job_issued_at_ms":70510,"miner_id":"miner-8","network_difficulty":1000004,"ntime":1700000140,"order_key":"08:miner-8","p2mr_program_hex":"8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c","share_difficulty":1,"share_id":"share-140","share_seq":140,"template_height":800004},{"accepted_at_ms":70501,"credit_policy":null,"job_id":"job-141","job_issued_at_ms":70488,"miner_id":"miner-9","network_difficulty":1000005,"ntime":1700000141,"order_key":"09:miner-9","p2mr_program_hex":"8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d","share_difficulty":2,"share_id":"share-141","share_seq":141,"template_height":800004},{"accepted_at_ms":70487,"credit_policy":null,"job_id":"job-142","job_issued_at_ms":70499,"miner_id":"miner-10","network_difficulty":1000006,"ntime":1700000142,"order_key":"10:miner-10","p2mr_program_hex":"8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e","share_difficulty":19,"share_id":"share-142","share_seq":142,"template_height":800004},{"accepted_at_ms":70503,"credit_policy":null,"job_id":"job-143","job_issued_at_ms":70507,"miner_id":"miner-0","network_difficulty":1000007,"ntime":1700000143,"order_key":"00:miner-0","p2mr_program_hex":"8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f","share_difficulty":1,"share_id":"share-143","share_seq":143,"template_height":800004},{"accepted_at_ms":70502,"credit_policy":null,"job_id":"job-144","job_issued_at_ms":70486,"miner_id":"miner-1","network_difficulty":1000008,"ntime":1700000144,"order_key":"01:miner-1","p2mr_program_hex":"9090909090909090909090909090909090909090909090909090909090909090","share_difficulty":2,"share_id":"share-144","share_seq":144,"template_height":800004},{"accepted_at_ms":70505,"credit_policy":null,"job_id":"job-145","job_issued_at_ms":70489,"miner_id":"miner-2","network_difficulty":1000009,"ntime":1700000145,"order_key":"02:miner-2","p2mr_program_hex":"9191919191919191919191919191919191919191919191919191919191919191","share_difficulty":1,"share_id":"share-145","share_seq":145,"template_height":800004},{"accepted_at_ms":70511,"credit_policy":"stale-grace","job_id":"job-146","job_issued_at_ms":70503,"miner_id":"miner-3","network_difficulty":1000010,"ntime":1700000146,"order_key":"03:miner-3","p2mr_program_hex":"9292929292929292929292929292929292929292929292929292929292929292","share_difficulty":3,"share_id":"share-146","share_seq":146,"template_height":800004},{"accepted_at_ms":70509,"credit_policy":null,"job_id":"job-147","job_issued_at_ms":70497,"miner_id":"miner-4","network_difficulty":1000011,"ntime":1700000147,"order_key":"04:miner-4","p2mr_program_hex":"9393939393939393939393939393939393939393939393939393939393939393","share_difficulty":3,"share_id":"share-147","share_seq":147,"template_height":800004},{"accepted_at_ms":70501,"credit_policy":"stale-grace","job_id":"job-148","job_issued_at_ms":70491,"miner_id":"miner-5","network_difficulty":1000012,"ntime":1700000148,"order_key":"05:miner-5","p2mr_program_hex":"9494949494949494949494949494949494949494949494949494949494949494","share_difficulty":3,"share_id":"share-148","share_seq":148,"template_height":800004},{"accepted_at_ms":70485,"credit_policy":null,"job_id":"job-149","job_issued_at_ms":70511,"miner_id":"miner-6","network_difficulty":1000013,"ntime":1700000149,"order_key":"06:miner-6","p2mr_program_hex":"9595959595959595959595959595959595959595959595959595959595959595","share_difficulty":1,"share_id":"share-149","share_seq":149,"template_height":800004},{"accepted_at_ms":70506,"credit_policy":null,"job_id":"job-150","job_issued_at_ms":70501,"miner_id":"miner-7","network_difficulty":1000014,"ntime":1700000150,"order_key":"07:miner-7","p2mr_program_hex":"9696969696969696969696969696969696969696969696969696969696969696","share_difficulty":2,"share_id":"share-150","share_seq":150,"template_height":800004},{"accepted_at_ms":70485,"credit_policy":null,"job_id":"job-151","job_issued_at_ms":70506,"miner_id":"miner-8","network_difficulty":1000015,"ntime":1700000151,"order_key":"08:miner-8","p2mr_program_hex":"9797979797979797979797979797979797979797979797979797979797979797","share_difficulty":7,"share_id":"share-151","share_seq":151,"template_height":800004},{"accepted_at_ms":70508,"credit_policy":"stale-grace","job_id":"job-152","job_issued_at_ms":70482,"miner_id":"miner-9","network_difficulty":1000016,"ntime":1700000152,"order_key":"09:miner-9","p2mr_program_hex":"9898989898989898989898989898989898989898989898989898989898989898","share_difficulty":7,"share_id":"share-152","share_seq":152,"template_height":800004},{"accepted_at_ms":70509,"credit_policy":null,"job_id":"job-153","job_issued_at_ms":70502,"miner_id":"miner-10","network_difficulty":1000000,"ntime":1700000153,"order_key":"10:miner-10","p2mr_program_hex":"9999999999999999999999999999999999999999999999999999999999999999","share_difficulty":3,"share_id":"share-153","share_seq":153,"template_height":800004},{"accepted_at_ms":70504,"credit_policy":null,"job_id":"job-154","job_issued_at_ms":70486,"miner_id":"miner-0","network_difficulty":1000001,"ntime":1700000154,"order_key":"00:miner-0","p2mr_program_hex":"9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a","share_difficulty":2,"share_id":"share-154","share_seq":154,"template_height":800004},{"accepted_at_ms":70481,"credit_policy":null,"job_id":"job-155","job_issued_at_ms":70508,"miner_id":"miner-1","network_difficulty":1000002,"ntime":1700000155,"order_key":"01:miner-1","p2mr_program_hex":"9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b","share_difficulty":3,"share_id":"share-155","share_seq":155,"template_height":800004},{"accepted_at_ms":70493,"credit_policy":"stale-grace","job_id":"job-156","job_issued_at_ms":70507,"miner_id":"miner-2","network_difficulty":1000003,"ntime":1700000156,"order_key":"02:miner-2","p2mr_program_hex":"9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c","share_difficulty":19,"share_id":"share-156","share_seq":156,"template_height":800004},{"accepted_at_ms":70486,"credit_policy":null,"job_id":"job-157","job_issued_at_ms":70504,"miner_id":"miner-3","network_difficulty":1000004,"ntime":1700000157,"order_key":"03:miner-3","p2mr_program_hex":"9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d","share_difficulty":1,"share_id":"share-157","share_seq":157,"template_height":800004},{"accepted_at_ms":70498,"credit_policy":null,"job_id":"job-158","job_issued_at_ms":70489,"miner_id":"miner-4","network_difficulty":1000005,"ntime":1700000158,"order_key":"04:miner-4","p2mr_program_hex":"9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e","share_difficulty":2,"share_id":"share-158","share_seq":158,"template_height":800004},{"accepted_at_ms":70508,"credit_policy":null,"job_id":"job-159","job_issued_at_ms":70510,"miner_id":"miner-5","network_difficulty":1000006,"ntime":1700000159,"order_key":"05:miner-5","p2mr_program_hex":"9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f","share_difficulty":3,"share_id":"share-159","share_seq":159,"template_height":800004},{"accepted_at_ms":70491,"credit_policy":"stale-grace","job_id":"job-160","job_issued_at_ms":70508,"miner_id":"miner-6","network_difficulty":1000007,"ntime":1700000160,"order_key":"06:miner-6","p2mr_program_hex":"a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0","share_difficulty":3,"share_id":"share-160","share_seq":160,"template_height":800005},{"accepted_at_ms":70493,"credit_policy":null,"job_id":"job-161","job_issued_at_ms":70506,"miner_id":"miner-7","network_difficulty":1000008,"ntime":1700000161,"order_key":"07:miner-7","p2mr_program_hex":"a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1","share_difficulty":7,"share_id":"share-161","share_seq":161,"template_height":800005},{"accepted_at_ms":70491,"credit_policy":null,"job_id":"job-162","job_issued_at_ms":70485,"miner_id":"miner-8","network_difficulty":1000009,"ntime":1700000162,"order_key":"08:miner-8","p2mr_program_hex":"a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2","share_difficulty":2,"share_id":"share-162","share_seq":162,"template_height":800005},{"accepted_at_ms":70498,"credit_policy":null,"job_id":"job-163","job_issued_at_ms":70507,"miner_id":"miner-9","network_difficulty":1000010,"ntime":1700000163,"order_key":"09:miner-9","p2mr_program_hex":"a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3","share_difficulty":7,"share_id":"share-163","share_seq":163,"template_height":800005},{"accepted_at_ms":70488,"credit_policy":null,"job_id":"job-164","job_issued_at_ms":70482,"miner_id":"miner-10","network_difficulty":1000011,"ntime":1700000164,"order_key":"10:miner-10","p2mr_program_hex":"a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4","share_difficulty":2,"share_id":"share-164","share_seq":164,"template_height":800005},{"accepted_at_ms":70492,"credit_policy":null,"job_id":"job-165","job_issued_at_ms":70510,"miner_id":"miner-0","network_difficulty":1000012,"ntime":1700000165,"order_key":"00:miner-0","p2mr_program_hex":"a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5","share_difficulty":1,"share_id":"share-165","share_seq":165,"template_height":800005},{"accepted_at_ms":70488,"credit_policy":null,"job_id":"job-166","job_issued_at_ms":70496,"miner_id":"miner-1","network_difficulty":1000013,"ntime":1700000166,"order_key":"01:miner-1","p2mr_program_hex":"a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6","share_difficulty":1,"share_id":"share-166","share_seq":166,"template_height":800005},{"accepted_at_ms":70500,"credit_policy":null,"job_id":"job-167","job_issued_at_ms":70487,"miner_id":"miner-2","network_difficulty":1000014,"ntime":1700000167,"order_key":"02:miner-2","p2mr_program_hex":"a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7","share_difficulty":7,"share_id":"share-167","share_seq":167,"template_height":800005},{"accepted_at_ms":70482,"credit_policy":null,"job_id":"job-168","job_issued_at_ms":70503,"miner_id":"miner-3","network_difficulty":1000015,"ntime":1700000168,"order_key":"03:miner-3","p2mr_program_hex":"a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8","share_difficulty":1,"share_id":"share-168","share_seq":168,"template_height":800005},{"accepted_at_ms":70509,"credit_policy":null,"job_id":"job-169","job_issued_at_ms":70488,"miner_id":"miner-4","network_difficulty":1000016,"ntime":1700000169,"order_key":"04:miner-4","p2mr_program_hex":"a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9","share_difficulty":2,"share_id":"share-169","share_seq":169,"template_height":800005},{"accepted_at_ms":70511,"credit_policy":null,"job_id":"job-170","job_issued_at_ms":70487,"miner_id":"miner-5","network_difficulty":1000000,"ntime":1700000170,"order_key":"05:miner-5","p2mr_program_hex":"aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa","share_difficulty":2,"share_id":"share-170","share_seq":170,"template_height":800005},{"accepted_at_ms":70490,"credit_policy":null,"job_id":"job-171","job_issued_at_ms":70485,"miner_id":"miner-6","network_difficulty":1000001,"ntime":1700000171,"order_key":"06:miner-6","p2mr_program_hex":"abababababababababababababababababababababababababababababababab","share_difficulty":19,"share_id":"share-171","share_seq":171,"template_height":800005},{"accepted_at_ms":70494,"credit_policy":null,"job_id":"job-172","job_issued_at_ms":70498,"miner_id":"miner-7","network_difficulty":1000002,"ntime":1700000172,"order_key":"07:miner-7","p2mr_program_hex":"acacacacacacacacacacacacacacacacacacacacacacacacacacacacacacacac","share_difficulty":3,"share_id":"share-172","share_seq":172,"template_height":800005},{"accepted_at_ms":70481,"credit_policy":null,"job_id":"job-173","job_issued_at_ms":70491,"miner_id":"miner-8","network_difficulty":1000003,"ntime":1700000173,"order_key":"08:miner-8","p2mr_program_hex":"adadadadadadadadadadadadadadadadadadadadadadadadadadadadadadadad","share_difficulty":2,"share_id":"share-173","share_seq":173,"template_height":800005},{"accepted_at_ms":70490,"credit_policy":"stale-grace","job_id":"job-174","job_issued_at_ms":70509,"miner_id":"miner-9","network_difficulty":1000004,"ntime":1700000174,"order_key":"09:miner-9","p2mr_program_hex":"aeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeae","share_difficulty":2,"share_id":"share-174","share_seq":174,"template_height":800005},{"accepted_at_ms":70504,"credit_policy":"stale-grace","job_id":"job-175","job_issued_at_ms":70505,"miner_id":"miner-10","network_difficulty":1000005,"ntime":1700000175,"order_key":"10:miner-10","p2mr_program_hex":"afafafafafafafafafafafafafafafafafafafafafafafafafafafafafafafaf","share_difficulty":3,"share_id":"share-175","share_seq":175,"template_height":800005},{"accepted_at_ms":70500,"credit_policy":null,"job_id":"job-176","job_issued_at_ms":70510,"miner_id":"miner-0","network_difficulty":1000006,"ntime":1700000176,"order_key":"00:miner-0","p2mr_program_hex":"b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0","share_difficulty":2,"share_id":"share-176","share_seq":176,"template_height":800005},{"accepted_at_ms":70489,"credit_policy":null,"job_id":"job-177","job_issued_at_ms":70495,"miner_id":"miner-1","network_difficulty":1000007,"ntime":1700000177,"order_key":"01:miner-1","p2mr_program_hex":"b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1","share_difficulty":2,"share_id":"share-177","share_seq":177,"template_height":800005},{"accepted_at_ms":70481,"credit_policy":null,"job_id":"job-178","job_issued_at_ms":70494,"miner_id":"miner-2","network_difficulty":1000008,"ntime":1700000178,"order_key":"02:miner-2","p2mr_program_hex":"b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2","share_difficulty":3,"share_id":"share-178","share_seq":178,"template_height":800005},{"accepted_at_ms":70495,"credit_policy":"stale-grace","job_id":"job-179","job_issued_at_ms":70499,"miner_id":"miner-3","network_difficulty":1000009,"ntime":1700000179,"order_key":"03:miner-3","p2mr_program_hex":"b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3","share_difficulty":7,"share_id":"share-179","share_seq":179,"template_height":800005},{"accepted_at_ms":70497,"credit_policy":"stale-grace","job_id":"job-180","job_issued_at_ms":70487,"miner_id":"miner-4","network_difficulty":1000010,"ntime":1700000180,"order_key":"04:miner-4","p2mr_program_hex":"b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4","share_difficulty":2,"share_id":"share-180","share_seq":180,"template_height":800005},{"accepted_at_ms":70494,"credit_policy":null,"job_id":"job-181","job_issued_at_ms":70487,"miner_id":"miner-5","network_difficulty":1000011,"ntime":1700000181,"order_key":"05:miner-5","p2mr_program_hex":"b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5","share_difficulty":1,"share_id":"share-181","share_seq":181,"template_height":800005},{"accepted_at_ms":70492,"credit_policy":"stale-grace","job_id":"job-182","job_issued_at_ms":70491,"miner_id":"miner-6","network_difficulty":1000012,"ntime":1700000182,"order_key":"06:miner-6","p2mr_program_hex":"b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6","share_difficulty":7,"share_id":"share-182","share_seq":182,"template_height":800005},{"accepted_at_ms":70483,"credit_policy":null,"job_id":"job-183","job_issued_at_ms":70490,"miner_id":"miner-7","network_difficulty":1000013,"ntime":1700000183,"order_key":"07:miner-7","p2mr_program_hex":"b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7","share_difficulty":1,"share_id":"share-183","share_seq":183,"template_height":800005},{"accepted_at_ms":70486,"credit_policy":null,"job_id":"job-184","job_issued_at_ms":70481,"miner_id":"miner-8","network_difficulty":1000014,"ntime":1700000184,"order_key":"08:miner-8","p2mr_program_hex":"b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8","share_difficulty":7,"share_id":"share-184","share_seq":184,"template_height":800005},{"accepted_at_ms":70506,"credit_policy":"stale-grace","job_id":"job-185","job_issued_at_ms":70494,"miner_id":"miner-9","network_difficulty":1000015,"ntime":1700000185,"order_key":"09:miner-9","p2mr_program_hex":"b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9","share_difficulty":7,"share_id":"share-185","share_seq":185,"template_height":800005},{"accepted_at_ms":70509,"credit_policy":null,"job_id":"job-186","job_issued_at_ms":70485,"miner_id":"miner-10","network_difficulty":1000016,"ntime":1700000186,"order_key":"10:miner-10","p2mr_program_hex":"babababababababababababababababababababababababababababababababa","share_difficulty":7,"share_id":"share-186","share_seq":186,"template_height":800005},{"accepted_at_ms":70488,"credit_policy":null,"job_id":"job-187","job_issued_at_ms":70492,"miner_id":"miner-0","network_difficulty":1000000,"ntime":1700000187,"order_key":"00:miner-0","p2mr_program_hex":"bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb","share_difficulty":19,"share_id":"share-187","share_seq":187,"template_height":800005},{"accepted_at_ms":70511,"credit_policy":null,"job_id":"job-188","job_issued_at_ms":70486,"miner_id":"miner-1","network_difficulty":1000001,"ntime":1700000188,"order_key":"01:miner-1","p2mr_program_hex":"bcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbc","share_difficulty":7,"share_id":"share-188","share_seq":188,"template_height":800005},{"accepted_at_ms":70495,"credit_policy":null,"job_id":"job-189","job_issued_at_ms":70508,"miner_id":"miner-2","network_difficulty":1000002,"ntime":1700000189,"order_key":"02:miner-2","p2mr_program_hex":"bdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbd","share_difficulty":2,"share_id":"share-189","share_seq":189,"template_height":800005},{"accepted_at_ms":70484,"credit_policy":null,"job_id":"job-190","job_issued_at_ms":70485,"miner_id":"miner-3","network_difficulty":1000003,"ntime":1700000190,"order_key":"03:miner-3","p2mr_program_hex":"bebebebebebebebebebebebebebebebebebebebebebebebebebebebebebebebe","share_difficulty":19,"share_id":"share-190","share_seq":190,"template_height":800005},{"accepted_at_ms":70497,"credit_policy":null,"job_id":"job-191","job_issued_at_ms":70505,"miner_id":"miner-4","network_difficulty":1000004,"ntime":1700000191,"order_key":"04:miner-4","p2mr_program_hex":"bfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbf","share_difficulty":1,"share_id":"share-191","share_seq":191,"template_height":800005},{"accepted_at_ms":70505,"credit_policy":"stale-grace","job_id":"job-192","job_issued_at_ms":70505,"miner_id":"miner-5","network_difficulty":1000005,"ntime":1700000192,"order_key":"05:miner-5","p2mr_program_hex":"c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0","share_difficulty":2,"share_id":"share-192","share_seq":192,"template_height":800006},{"accepted_at_ms":70510,"credit_policy":"stale-grace","job_id":"job-193","job_issued_at_ms":70483,"miner_id":"miner-6","network_difficulty":1000006,"ntime":1700000193,"order_key":"06:miner-6","p2mr_program_hex":"c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1","share_difficulty":2,"share_id":"share-193","share_seq":193,"template_height":800006},{"accepted_at_ms":70508,"credit_policy":"stale-grace","job_id":"job-194","job_issued_at_ms":70502,"miner_id":"miner-7","network_difficulty":1000007,"ntime":1700000194,"order_key":"07:miner-7","p2mr_program_hex":"c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2","share_difficulty":19,"share_id":"share-194","share_seq":194,"template_height":800006},{"accepted_at_ms":70489,"credit_policy":null,"job_id":"job-195","job_issued_at_ms":70493,"miner_id":"miner-8","network_difficulty":1000008,"ntime":1700000195,"order_key":"08:miner-8","p2mr_program_hex":"c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3","share_difficulty":2,"share_id":"share-195","share_seq":195,"template_height":800006},{"accepted_at_ms":70507,"credit_policy":"stale-grace","job_id":"job-196","job_issued_at_ms":70500,"miner_id":"miner-9","network_difficulty":1000009,"ntime":1700000196,"order_key":"09:miner-9","p2mr_program_hex":"c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4","share_difficulty":19,"share_id":"share-196","share_seq":196,"template_height":800006},{"accepted_at_ms":70489,"credit_policy":"stale-grace","job_id":"job-197","job_issued_at_ms":70485,"miner_id":"miner-10","network_difficulty":1000010,"ntime":1700000197,"order_key":"10:miner-10","p2mr_program_hex":"c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5","share_difficulty":1,"share_id":"share-197","share_seq":197,"template_height":800006},{"accepted_at_ms":70485,"credit_policy":null,"job_id":"job-198","job_issued_at_ms":70482,"miner_id":"miner-0","network_difficulty":1000011,"ntime":1700000198,"order_key":"00:miner-0","p2mr_program_hex":"c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6","share_difficulty":19,"share_id":"share-198","share_seq":198,"template_height":800006},{"accepted_at_ms":70492,"credit_policy":"stale-grace","job_id":"job-199","job_issued_at_ms":70489,"miner_id":"miner-1","network_difficulty":1000012,"ntime":1700000199,"order_key":"01:miner-1","p2mr_program_hex":"c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7","share_difficulty":3,"share_id":"share-199","share_seq":199,"template_height":800006},{"accepted_at_ms":70486,"credit_policy":null,"job_id":"job-200","job_issued_at_ms":70499,"miner_id":"miner-2","network_difficulty":1000013,"ntime":1700000200,"order_key":"02:miner-2","p2mr_program_hex":"c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8","share_difficulty":7,"share_id":"share-200","share_seq":200,"template_height":800006},{"accepted_at_ms":70507,"credit_policy":null,"job_id":"job-201","job_issued_at_ms":70495,"miner_id":"miner-3","network_difficulty":1000014,"ntime":1700000201,"order_key":"03:miner-3","p2mr_program_hex":"c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9","share_difficulty":2,"share_id":"share-201","share_seq":201,"template_height":800006},{"accepted_at_ms":70484,"credit_policy":null,"job_id":"job-202","job_issued_at_ms":70488,"miner_id":"miner-4","network_difficulty":1000015,"ntime":1700000202,"order_key":"04:miner-4","p2mr_program_hex":"cacacacacacacacacacacacacacacacacacacacacacacacacacacacacacacaca","share_difficulty":7,"share_id":"share-202","share_seq":202,"template_height":800006},{"accepted_at_ms":70509,"credit_policy":null,"job_id":"job-203","job_issued_at_ms":70495,"miner_id":"miner-5","network_difficulty":1000016,"ntime":1700000203,"order_key":"05:miner-5","p2mr_program_hex":"cbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcb","share_difficulty":7,"share_id":"share-203","share_seq":203,"template_height":800006},{"accepted_at_ms":70503,"credit_policy":null,"job_id":"job-204","job_issued_at_ms":70500,"miner_id":"miner-6","network_difficulty":1000000,"ntime":1700000204,"order_key":"06:miner-6","p2mr_program_hex":"cccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccc","share_difficulty":1,"share_id":"share-204","share_seq":204,"template_height":800006},{"accepted_at_ms":70511,"credit_policy":null,"job_id":"job-205","job_issued_at_ms":70508,"miner_id":"miner-7","network_difficulty":1000001,"ntime":1700000205,"order_key":"07:miner-7","p2mr_program_hex":"cdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcd","share_difficulty":3,"share_id":"share-205","share_seq":205,"template_height":800006},{"accepted_at_ms":70499,"credit_policy":null,"job_id":"job-206","job_issued_at_ms":70482,"miner_id":"miner-8","network_difficulty":1000002,"ntime":1700000206,"order_key":"08:miner-8","p2mr_program_hex":"cececececececececececececececececececececececececececececececece","share_difficulty":19,"share_id":"share-206","share_seq":206,"template_height":800006},{"accepted_at_ms":70496,"credit_policy":null,"job_id":"job-207","job_issued_at_ms":70505,"miner_id":"miner-9","network_difficulty":1000003,"ntime":1700000207,"order_key":"09:miner-9","p2mr_program_hex":"cfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcf","share_difficulty":19,"share_id":"share-207","share_seq":207,"template_height":800006},{"accepted_at_ms":70509,"credit_policy":null,"job_id":"job-208","job_issued_at_ms":70493,"miner_id":"miner-10","network_difficulty":1000004,"ntime":1700000208,"order_key":"10:miner-10","p2mr_program_hex":"d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0","share_difficulty":3,"share_id":"share-208","share_seq":208,"template_height":800006},{"accepted_at_ms":70506,"credit_policy":null,"job_id":"job-209","job_issued_at_ms":70483,"miner_id":"miner-0","network_difficulty":1000005,"ntime":1700000209,"order_key":"00:miner-0","p2mr_program_hex":"d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1","share_difficulty":1,"share_id":"share-209","share_seq":209,"template_height":800006},{"accepted_at_ms":70486,"credit_policy":null,"job_id":"job-210","job_issued_at_ms":70487,"miner_id":"miner-1","network_difficulty":1000006,"ntime":1700000210,"order_key":"01:miner-1","p2mr_program_hex":"d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2","share_difficulty":19,"share_id":"share-210","share_seq":210,"template_height":800006},{"accepted_at_ms":70505,"credit_policy":"stale-grace","job_id":"job-211","job_issued_at_ms":70495,"miner_id":"miner-2","network_difficulty":1000007,"ntime":1700000211,"order_key":"02:miner-2","p2mr_program_hex":"d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3","share_difficulty":19,"share_id":"share-211","share_seq":211,"template_height":800006},{"accepted_at_ms":70484,"credit_policy":null,"job_id":"job-212","job_issued_at_ms":70486,"miner_id":"miner-3","network_difficulty":1000008,"ntime":1700000212,"order_key":"03:miner-3","p2mr_program_hex":"d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4","share_difficulty":3,"share_id":"share-212","share_seq":212,"template_height":800006},{"accepted_at_ms":70502,"credit_policy":"stale-grace","job_id":"job-213","job_issued_at_ms":70511,"miner_id":"miner-4","network_difficulty":1000009,"ntime":1700000213,"order_key":"04:miner-4","p2mr_program_hex":"d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5","share_difficulty":7,"share_id":"share-213","share_seq":213,"template_height":800006},{"accepted_at_ms":70493,"credit_policy":null,"job_id":"job-214","job_issued_at_ms":70501,"miner_id":"miner-5","network_difficulty":1000010,"ntime":1700000214,"order_key":"05:miner-5","p2mr_program_hex":"d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6","share_difficulty":7,"share_id":"share-214","share_seq":214,"template_height":800006},{"accepted_at_ms":70492,"credit_policy":null,"job_id":"job-215","job_issued_at_ms":70491,"miner_id":"miner-6","network_difficulty":1000011,"ntime":1700000215,"order_key":"06:miner-6","p2mr_program_hex":"d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7","share_difficulty":7,"share_id":"share-215","share_seq":215,"template_height":800006},{"accepted_at_ms":70505,"credit_policy":null,"job_id":"job-216","job_issued_at_ms":70504,"miner_id":"miner-7","network_difficulty":1000012,"ntime":1700000216,"order_key":"07:miner-7","p2mr_program_hex":"d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8","share_difficulty":19,"share_id":"share-216","share_seq":216,"template_height":800006},{"accepted_at_ms":70487,"credit_policy":null,"job_id":"job-217","job_issued_at_ms":70504,"miner_id":"miner-8","network_difficulty":1000013,"ntime":1700000217,"order_key":"08:miner-8","p2mr_program_hex":"d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9","share_difficulty":19,"share_id":"share-217","share_seq":217,"template_height":800006},{"accepted_at_ms":70494,"credit_policy":null,"job_id":"job-218","job_issued_at_ms":70504,"miner_id":"miner-9","network_difficulty":1000014,"ntime":1700000218,"order_key":"09:miner-9","p2mr_program_hex":"dadadadadadadadadadadadadadadadadadadadadadadadadadadadadadadada","share_difficulty":1,"share_id":"share-218","share_seq":218,"template_height":800006},{"accepted_at_ms":70497,"credit_policy":null,"job_id":"job-219","job_issued_at_ms":70494,"miner_id":"miner-10","network_difficulty":1000015,"ntime":1700000219,"order_key":"10:miner-10","p2mr_program_hex":"dbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdb","share_difficulty":3,"share_id":"share-219","share_seq":219,"template_height":800006},{"accepted_at_ms":70504,"credit_policy":"stale-grace","job_id":"job-220","job_issued_at_ms":70501,"miner_id":"miner-0","network_difficulty":1000016,"ntime":1700000220,"order_key":"00:miner-0","p2mr_program_hex":"dcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdc","share_difficulty":19,"share_id":"share-220","share_seq":220,"template_height":800006},{"accepted_at_ms":70509,"credit_policy":null,"job_id":"job-221","job_issued_at_ms":70509,"miner_id":"miner-1","network_difficulty":1000000,"ntime":1700000221,"order_key":"01:miner-1","p2mr_program_hex":"dddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddd","share_difficulty":3,"share_id":"share-221","share_seq":221,"template_height":800006},{"accepted_at_ms":70481,"credit_policy":null,"job_id":"job-222","job_issued_at_ms":70490,"miner_id":"miner-2","network_difficulty":1000001,"ntime":1700000222,"order_key":"02:miner-2","p2mr_program_hex":"dededededededededededededededededededededededededededededededede","share_difficulty":7,"share_id":"share-222","share_seq":222,"template_height":800006},{"accepted_at_ms":70500,"credit_policy":null,"job_id":"job-223","job_issued_at_ms":70498,"miner_id":"miner-3","network_difficulty":1000002,"ntime":1700000223,"order_key":"03:miner-3","p2mr_program_hex":"dfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdf","share_difficulty":19,"share_id":"share-223","share_seq":223,"template_height":800006},{"accepted_at_ms":70497,"credit_policy":"stale-grace","job_id":"job-224","job_issued_at_ms":70493,"miner_id":"miner-4","network_difficulty":1000003,"ntime":1700000224,"order_key":"04:miner-4","p2mr_program_hex":"e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0","share_difficulty":1,"share_id":"share-224","share_seq":224,"template_height":800007},{"accepted_at_ms":70493,"credit_policy":null,"job_id":"job-225","job_issued_at_ms":70497,"miner_id":"miner-5","network_difficulty":1000004,"ntime":1700000225,"order_key":"05:miner-5","p2mr_program_hex":"e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1","share_difficulty":2,"share_id":"share-225","share_seq":225,"template_height":800007},{"accepted_at_ms":70486,"credit_policy":null,"job_id":"job-226","job_issued_at_ms":70500,"miner_id":"miner-6","network_difficulty":1000005,"ntime":1700000226,"order_key":"06:miner-6","p2mr_program_hex":"e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2","share_difficulty":7,"share_id":"share-226","share_seq":226,"template_height":800007},{"accepted_at_ms":70500,"credit_policy":"stale-grace","job_id":"job-227","job_issued_at_ms":70499,"miner_id":"miner-7","network_difficulty":1000006,"ntime":1700000227,"order_key":"07:miner-7","p2mr_program_hex":"e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3","share_difficulty":2,"share_id":"share-227","share_seq":227,"template_height":800007},{"accepted_at_ms":70505,"credit_policy":"stale-grace","job_id":"job-228","job_issued_at_ms":70503,"miner_id":"miner-8","network_difficulty":1000007,"ntime":1700000228,"order_key":"08:miner-8","p2mr_program_hex":"e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4","share_difficulty":1,"share_id":"share-228","share_seq":228,"template_height":800007},{"accepted_at_ms":70500,"credit_policy":null,"job_id":"job-229","job_issued_at_ms":70484,"miner_id":"miner-9","network_difficulty":1000008,"ntime":1700000229,"order_key":"09:miner-9","p2mr_program_hex":"e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5","share_difficulty":3,"share_id":"share-229","share_seq":229,"template_height":800007},{"accepted_at_ms":70487,"credit_policy":null,"job_id":"job-230","job_issued_at_ms":70490,"miner_id":"miner-10","network_difficulty":1000009,"ntime":1700000230,"order_key":"10:miner-10","p2mr_program_hex":"e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6","share_difficulty":19,"share_id":"share-230","share_seq":230,"template_height":800007},{"accepted_at_ms":70490,"credit_policy":null,"job_id":"job-231","job_issued_at_ms":70509,"miner_id":"miner-0","network_difficulty":1000010,"ntime":1700000231,"order_key":"00:miner-0","p2mr_program_hex":"e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7","share_difficulty":1,"share_id":"share-231","share_seq":231,"template_height":800007},{"accepted_at_ms":70499,"credit_policy":"stale-grace","job_id":"job-232","job_issued_at_ms":70485,"miner_id":"miner-1","network_difficulty":1000011,"ntime":1700000232,"order_key":"01:miner-1","p2mr_program_hex":"e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8","share_difficulty":1,"share_id":"share-232","share_seq":232,"template_height":800007},{"accepted_at_ms":70511,"credit_policy":"stale-grace","job_id":"job-233","job_issued_at_ms":70506,"miner_id":"miner-2","network_difficulty":1000012,"ntime":1700000233,"order_key":"02:miner-2","p2mr_program_hex":"e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9","share_difficulty":7,"share_id":"share-233","share_seq":233,"template_height":800007},{"accepted_at_ms":70493,"credit_policy":null,"job_id":"job-234","job_issued_at_ms":70483,"miner_id":"miner-3","network_difficulty":1000013,"ntime":1700000234,"order_key":"03:miner-3","p2mr_program_hex":"eaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaea","share_difficulty":2,"share_id":"share-234","share_seq":234,"template_height":800007},{"accepted_at_ms":70491,"credit_policy":null,"job_id":"job-235","job_issued_at_ms":70505,"miner_id":"miner-4","network_difficulty":1000014,"ntime":1700000235,"order_key":"04:miner-4","p2mr_program_hex":"ebebebebebebebebebebebebebebebebebebebebebebebebebebebebebebebeb","share_difficulty":19,"share_id":"share-235","share_seq":235,"template_height":800007},{"accepted_at_ms":70490,"credit_policy":null,"job_id":"job-236","job_issued_at_ms":70508,"miner_id":"miner-5","network_difficulty":1000015,"ntime":1700000236,"order_key":"05:miner-5","p2mr_program_hex":"ecececececececececececececececececececececececececececececececec","share_difficulty":2,"share_id":"share-236","share_seq":236,"template_height":800007},{"accepted_at_ms":70489,"credit_policy":null,"job_id":"job-237","job_issued_at_ms":70481,"miner_id":"miner-6","network_difficulty":1000016,"ntime":1700000237,"order_key":"06:miner-6","p2mr_program_hex":"edededededededededededededededededededededededededededededededed","share_difficulty":2,"share_id":"share-237","share_seq":237,"template_height":800007},{"accepted_at_ms":70501,"credit_policy":"stale-grace","job_id":"job-238","job_issued_at_ms":70506,"miner_id":"miner-7","network_difficulty":1000000,"ntime":1700000238,"order_key":"07:miner-7","p2mr_program_hex":"eeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeee","share_difficulty":2,"share_id":"share-238","share_seq":238,"template_height":800007},{"accepted_at_ms":70497,"credit_policy":null,"job_id":"job-239","job_issued_at_ms":70494,"miner_id":"miner-8","network_difficulty":1000001,"ntime":1700000239,"order_key":"08:miner-8","p2mr_program_hex":"efefefefefefefefefefefefefefefefefefefefefefefefefefefefefefefef","share_difficulty":7,"share_id":"share-239","share_seq":239,"template_height":800007},{"accepted_at_ms":70492,"credit_policy":null,"job_id":"job-240","job_issued_at_ms":70481,"miner_id":"miner-9","network_difficulty":1000002,"ntime":1700000240,"order_key":"09:miner-9","p2mr_program_hex":"f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0","share_difficulty":3,"share_id":"share-240","share_seq":240,"template_height":800007},{"accepted_at_ms":70488,"credit_policy":null,"job_id":"job-241","job_issued_at_ms":70483,"miner_id":"miner-10","network_difficulty":1000003,"ntime":1700000241,"order_key":"10:miner-10","p2mr_program_hex":"f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1","share_difficulty":19,"share_id":"share-241","share_seq":241,"template_height":800007},{"accepted_at_ms":70509,"credit_policy":null,"job_id":"job-242","job_issued_at_ms":70485,"miner_id":"miner-0","network_difficulty":1000004,"ntime":1700000242,"order_key":"00:miner-0","p2mr_program_hex":"f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2","share_difficulty":1,"share_id":"share-242","share_seq":242,"template_height":800007},{"accepted_at_ms":70505,"credit_policy":"stale-grace","job_id":"job-243","job_issued_at_ms":70501,"miner_id":"miner-1","network_difficulty":1000005,"ntime":1700000243,"order_key":"01:miner-1","p2mr_program_hex":"f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3","share_difficulty":19,"share_id":"share-243","share_seq":243,"template_height":800007},{"accepted_at_ms":70501,"credit_policy":"stale-grace","job_id":"job-244","job_issued_at_ms":70492,"miner_id":"miner-2","network_difficulty":1000006,"ntime":1700000244,"order_key":"02:miner-2","p2mr_program_hex":"f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4","share_difficulty":3,"share_id":"share-244","share_seq":244,"template_height":800007},{"accepted_at_ms":70489,"credit_policy":null,"job_id":"job-245","job_issued_at_ms":70503,"miner_id":"miner-3","network_difficulty":1000007,"ntime":1700000245,"order_key":"03:miner-3","p2mr_program_hex":"f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5","share_difficulty":7,"share_id":"share-245","share_seq":245,"template_height":800007},{"accepted_at_ms":70500,"credit_policy":"stale-grace","job_id":"job-246","job_issued_at_ms":70481,"miner_id":"miner-4","network_difficulty":1000008,"ntime":1700000246,"order_key":"04:miner-4","p2mr_program_hex":"f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6","share_difficulty":3,"share_id":"share-246","share_seq":246,"template_height":800007},{"accepted_at_ms":70494,"credit_policy":null,"job_id":"job-247","job_issued_at_ms":70484,"miner_id":"miner-5","network_difficulty":1000009,"ntime":1700000247,"order_key":"05:miner-5","p2mr_program_hex":"f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7","share_difficulty":19,"share_id":"share-247","share_seq":247,"template_height":800007},{"accepted_at_ms":70489,"credit_policy":null,"job_id":"job-248","job_issued_at_ms":70492,"miner_id":"miner-6","network_difficulty":1000010,"ntime":1700000248,"order_key":"06:miner-6","p2mr_program_hex":"f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8","share_difficulty":19,"share_id":"share-248","share_seq":248,"template_height":800007},{"accepted_at_ms":70505,"credit_policy":null,"job_id":"job-249","job_issued_at_ms":70482,"miner_id":"miner-7","network_difficulty":1000011,"ntime":1700000249,"order_key":"07:miner-7","p2mr_program_hex":"f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9","share_difficulty":1,"share_id":"share-249","share_seq":249,"template_height":800007},{"accepted_at_ms":70511,"credit_policy":null,"job_id":"job-250","job_issued_at_ms":70501,"miner_id":"miner-8","network_difficulty":1000012,"ntime":1700000250,"order_key":"08:miner-8","p2mr_program_hex":"fafafafafafafafafafafafafafafafafafafafafafafafafafafafafafafafa","share_difficulty":1,"share_id":"share-250","share_seq":250,"template_height":800007},{"accepted_at_ms":70500,"credit_policy":null,"job_id":"job-251","job_issued_at_ms":70481,"miner_id":"miner-9","network_difficulty":1000013,"ntime":1700000251,"order_key":"09:miner-9","p2mr_program_hex":"fbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfb","share_difficulty":7,"share_id":"share-251","share_seq":251,"template_height":800007},{"accepted_at_ms":70485,"credit_policy":null,"job_id":"job-252","job_issued_at_ms":70495,"miner_id":"miner-10","network_difficulty":1000014,"ntime":1700000252,"order_key":"10:miner-10","p2mr_program_hex":"fcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfc","share_difficulty":19,"share_id":"share-252","share_seq":252,"template_height":800007},{"accepted_at_ms":70504,"credit_policy":"stale-grace","job_id":"job-253","job_issued_at_ms":70496,"miner_id":"miner-0","network_difficulty":1000015,"ntime":1700000253,"order_key":"00:miner-0","p2mr_program_hex":"fdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfd","share_difficulty":7,"share_id":"share-253","share_seq":253,"template_height":800007},{"accepted_at_ms":70505,"credit_policy":"stale-grace","job_id":"job-254","job_issued_at_ms":70492,"miner_id":"miner-1","network_difficulty":1000016,"ntime":1700000254,"order_key":"01:miner-1","p2mr_program_hex":"fefefefefefefefefefefefefefefefefefefefefefefefefefefefefefefefe","share_difficulty":1,"share_id":"share-254","share_seq":254,"template_height":800007},{"accepted_at_ms":70496,"credit_policy":null,"job_id":"job-255","job_issued_at_ms":70510,"miner_id":"miner-2","network_difficulty":1000000,"ntime":1700000255,"order_key":"02:miner-2","p2mr_program_hex":"ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff","share_difficulty":1,"share_id":"share-255","share_seq":255,"template_height":800007},{"accepted_at_ms":70486,"credit_policy":null,"job_id":"job-256","job_issued_at_ms":70501,"miner_id":"miner-3","network_difficulty":1000001,"ntime":1700000256,"order_key":"03:miner-3","p2mr_program_hex":"0000000000000000000000000000000000000000000000000000000000000000","share_difficulty":3,"share_id":"share-256","share_seq":256,"template_height":800008},{"accepted_at_ms":70484,"credit_policy":null,"job_id":"job-257","job_issued_at_ms":70491,"miner_id":"miner-4","network_difficulty":1000002,"ntime":1700000257,"order_key":"04:miner-4","p2mr_program_hex":"0101010101010101010101010101010101010101010101010101010101010101","share_difficulty":19,"share_id":"share-257","share_seq":257,"template_height":800008},{"accepted_at_ms":70492,"credit_policy":"stale-grace","job_id":"job-258","job_issued_at_ms":70511,"miner_id":"miner-5","network_difficulty":1000003,"ntime":1700000258,"order_key":"05:miner-5","p2mr_program_hex":"0202020202020202020202020202020202020202020202020202020202020202","share_difficulty":1,"share_id":"share-258","share_seq":258,"template_height":800008},{"accepted_at_ms":70511,"credit_policy":null,"job_id":"job-259","job_issued_at_ms":70481,"miner_id":"miner-6","network_difficulty":1000004,"ntime":1700000259,"order_key":"06:miner-6","p2mr_program_hex":"0303030303030303030303030303030303030303030303030303030303030303","share_difficulty":7,"share_id":"share-259","share_seq":259,"template_height":800008},{"accepted_at_ms":70483,"credit_policy":null,"job_id":"job-260","job_issued_at_ms":70494,"miner_id":"miner-7","network_difficulty":1000005,"ntime":1700000260,"order_key":"07:miner-7","p2mr_program_hex":"0404040404040404040404040404040404040404040404040404040404040404","share_difficulty":2,"share_id":"share-260","share_seq":260,"template_height":800008},{"accepted_at_ms":70493,"credit_policy":null,"job_id":"job-261","job_issued_at_ms":70490,"miner_id":"miner-8","network_difficulty":1000006,"ntime":1700000261,"order_key":"08:miner-8","p2mr_program_hex":"0505050505050505050505050505050505050505050505050505050505050505","share_difficulty":1,"share_id":"share-261","share_seq":261,"template_height":800008},{"accepted_at_ms":70504,"credit_policy":null,"job_id":"job-262","job_issued_at_ms":70491,"miner_id":"miner-9","network_difficulty":1000007,"ntime":1700000262,"order_key":"09:miner-9","p2mr_program_hex":"0606060606060606060606060606060606060606060606060606060606060606","share_difficulty":2,"share_id":"share-262","share_seq":262,"template_height":800008},{"accepted_at_ms":70498,"credit_policy":null,"job_id":"job-263","job_issued_at_ms":70485,"miner_id":"miner-10","network_difficulty":1000008,"ntime":1700000263,"order_key":"10:miner-10","p2mr_program_hex":"0707070707070707070707070707070707070707070707070707070707070707","share_difficulty":7,"share_id":"share-263","share_seq":263,"template_height":800008},{"accepted_at_ms":70496,"credit_policy":null,"job_id":"job-264","job_issued_at_ms":70488,"miner_id":"miner-0","network_difficulty":1000009,"ntime":1700000264,"order_key":"00:miner-0","p2mr_program_hex":"0808080808080808080808080808080808080808080808080808080808080808","share_difficulty":1,"share_id":"share-264","share_seq":264,"template_height":800008},{"accepted_at_ms":70484,"credit_policy":"stale-grace","job_id":"job-265","job_issued_at_ms":70484,"miner_id":"miner-1","network_difficulty":1000010,"ntime":1700000265,"order_key":"01:miner-1","p2mr_program_hex":"0909090909090909090909090909090909090909090909090909090909090909","share_difficulty":3,"share_id":"share-265","share_seq":265,"template_height":800008},{"accepted_at_ms":70491,"credit_policy":null,"job_id":"job-266","job_issued_at_ms":70511,"miner_id":"miner-2","network_difficulty":1000011,"ntime":1700000266,"order_key":"02:miner-2","p2mr_program_hex":"0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a","share_difficulty":2,"share_id":"share-266","share_seq":266,"template_height":800008},{"accepted_at_ms":70506,"credit_policy":"stale-grace","job_id":"job-267","job_issued_at_ms":70485,"miner_id":"miner-3","network_difficulty":1000012,"ntime":1700000267,"order_key":"03:miner-3","p2mr_program_hex":"0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b","share_difficulty":3,"share_id":"share-267","share_seq":267,"template_height":800008},{"accepted_at_ms":70495,"credit_policy":null,"job_id":"job-268","job_issued_at_ms":70495,"miner_id":"miner-4","network_difficulty":1000013,"ntime":1700000268,"order_key":"04:miner-4","p2mr_program_hex":"0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c","share_difficulty":7,"share_id":"share-268","share_seq":268,"template_height":800008},{"accepted_at_ms":70499,"credit_policy":null,"job_id":"job-269","job_issued_at_ms":70485,"miner_id":"miner-5","network_difficulty":1000014,"ntime":1700000269,"order_key":"05:miner-5","p2mr_program_hex":"0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d","share_difficulty":19,"share_id":"share-269","share_seq":269,"template_height":800008},{"accepted_at_ms":70497,"credit_policy":null,"job_id":"job-270","job_issued_at_ms":70498,"miner_id":"miner-6","network_difficulty":1000015,"ntime":1700000270,"order_key":"06:miner-6","p2mr_program_hex":"0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e","share_difficulty":2,"share_id":"share-270","share_seq":270,"template_height":800008},{"accepted_at_ms":70503,"credit_policy":null,"job_id":"job-271","job_issued_at_ms":70496,"miner_id":"miner-7","network_difficulty":1000016,"ntime":1700000271,"order_key":"07:miner-7","p2mr_program_hex":"0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f","share_difficulty":3,"share_id":"share-271","share_seq":271,"template_height":800008},{"accepted_at_ms":70505,"credit_policy":null,"job_id":"job-272","job_issued_at_ms":70505,"miner_id":"miner-8","network_difficulty":1000000,"ntime":1700000272,"order_key":"08:miner-8","p2mr_program_hex":"1010101010101010101010101010101010101010101010101010101010101010","share_difficulty":19,"share_id":"share-272","share_seq":272,"template_height":800008},{"accepted_at_ms":70505,"credit_policy":null,"job_id":"job-273","job_issued_at_ms":70506,"miner_id":"miner-9","network_difficulty":1000001,"ntime":1700000273,"order_key":"09:miner-9","p2mr_program_hex":"1111111111111111111111111111111111111111111111111111111111111111","share_difficulty":3,"share_id":"share-273","share_seq":273,"template_height":800008},{"accepted_at_ms":70499,"credit_policy":null,"job_id":"job-274","job_issued_at_ms":70501,"miner_id":"miner-10","network_difficulty":1000002,"ntime":1700000274,"order_key":"10:miner-10","p2mr_program_hex":"1212121212121212121212121212121212121212121212121212121212121212","share_difficulty":7,"share_id":"share-274","share_seq":274,"template_height":800008},{"accepted_at_ms":70484,"credit_policy":null,"job_id":"job-275","job_issued_at_ms":70488,"miner_id":"miner-0","network_difficulty":1000003,"ntime":1700000275,"order_key":"00:miner-0","p2mr_program_hex":"1313131313131313131313131313131313131313131313131313131313131313","share_difficulty":3,"share_id":"share-275","share_seq":275,"template_height":800008},{"accepted_at_ms":70508,"credit_policy":"stale-grace","job_id":"job-276","job_issued_at_ms":70504,"miner_id":"miner-1","network_difficulty":1000004,"ntime":1700000276,"order_key":"01:miner-1","p2mr_program_hex":"1414141414141414141414141414141414141414141414141414141414141414","share_difficulty":19,"share_id":"share-276","share_seq":276,"template_height":800008},{"accepted_at_ms":70511,"credit_policy":null,"job_id":"job-277","job_issued_at_ms":70482,"miner_id":"miner-2","network_difficulty":1000005,"ntime":1700000277,"order_key":"02:miner-2","p2mr_program_hex":"1515151515151515151515151515151515151515151515151515151515151515","share_difficulty":2,"share_id":"share-277","share_seq":277,"template_height":800008},{"accepted_at_ms":70502,"credit_policy":null,"job_id":"job-278","job_issued_at_ms":70511,"miner_id":"miner-3","network_difficulty":1000006,"ntime":1700000278,"order_key":"03:miner-3","p2mr_program_hex":"1616161616161616161616161616161616161616161616161616161616161616","share_difficulty":7,"share_id":"share-278","share_seq":278,"template_height":800008},{"accepted_at_ms":70486,"credit_policy":null,"job_id":"job-279","job_issued_at_ms":70493,"miner_id":"miner-4","network_difficulty":1000007,"ntime":1700000279,"order_key":"04:miner-4","p2mr_program_hex":"1717171717171717171717171717171717171717171717171717171717171717","share_difficulty":1,"share_id":"share-279","share_seq":279,"template_height":800008},{"accepted_at_ms":70484,"credit_policy":"stale-grace","job_id":"job-280","job_issued_at_ms":70498,"miner_id":"miner-5","network_difficulty":1000008,"ntime":1700000280,"order_key":"05:miner-5","p2mr_program_hex":"1818181818181818181818181818181818181818181818181818181818181818","share_difficulty":7,"share_id":"share-280","share_seq":280,"template_height":800008},{"accepted_at_ms":70490,"credit_policy":null,"job_id":"job-281","job_issued_at_ms":70503,"miner_id":"miner-6","network_difficulty":1000009,"ntime":1700000281,"order_key":"06:miner-6","p2mr_program_hex":"1919191919191919191919191919191919191919191919191919191919191919","share_difficulty":2,"share_id":"share-281","share_seq":281,"template_height":800008},{"accepted_at_ms":70495,"credit_policy":"stale-grace","job_id":"job-282","job_issued_at_ms":70497,"miner_id":"miner-7","network_difficulty":1000010,"ntime":1700000282,"order_key":"07:miner-7","p2mr_program_hex":"1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a","share_difficulty":7,"share_id":"share-282","share_seq":282,"template_height":800008},{"accepted_at_ms":70506,"credit_policy":"stale-grace","job_id":"job-283","job_issued_at_ms":70502,"miner_id":"miner-8","network_difficulty":1000011,"ntime":1700000283,"order_key":"08:miner-8","p2mr_program_hex":"1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b","share_difficulty":3,"share_id":"share-283","share_seq":283,"template_height":800008},{"accepted_at_ms":70503,"credit_policy":"stale-grace","job_id":"job-284","job_issued_at_ms":70501,"miner_id":"miner-9","network_difficulty":1000012,"ntime":1700000284,"order_key":"09:miner-9","p2mr_program_hex":"1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c","share_difficulty":3,"share_id":"share-284","share_seq":284,"template_height":800008},{"accepted_at_ms":70498,"credit_policy":null,"job_id":"job-285","job_issued_at_ms":70491,"miner_id":"miner-10","network_difficulty":1000013,"ntime":1700000285,"order_key":"10:miner-10","p2mr_program_hex":"1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d","share_difficulty":3,"share_id":"share-285","share_seq":285,"template_height":800008},{"accepted_at_ms":70484,"credit_policy":"stale-grace","job_id":"job-286","job_issued_at_ms":70482,"miner_id":"miner-0","network_difficulty":1000014,"ntime":1700000286,"order_key":"00:miner-0","p2mr_program_hex":"1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e","share_difficulty":3,"share_id":"share-286","share_seq":286,"template_height":800008},{"accepted_at_ms":70509,"credit_policy":null,"job_id":"job-287","job_issued_at_ms":70495,"miner_id":"miner-1","network_difficulty":1000015,"ntime":1700000287,"order_key":"01:miner-1","p2mr_program_hex":"1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f","share_difficulty":19,"share_id":"share-287","share_seq":287,"template_height":800008},{"accepted_at_ms":70492,"credit_policy":null,"job_id":"job-288","job_issued_at_ms":70503,"miner_id":"miner-2","network_difficulty":1000016,"ntime":1700000288,"order_key":"02:miner-2","p2mr_program_hex":"2020202020202020202020202020202020202020202020202020202020202020","share_difficulty":2,"share_id":"share-288","share_seq":288,"template_height":800009},{"accepted_at_ms":70508,"credit_policy":null,"job_id":"job-289","job_issued_at_ms":70497,"miner_id":"miner-3","network_difficulty":1000000,"ntime":1700000289,"order_key":"03:miner-3","p2mr_program_hex":"2121212121212121212121212121212121212121212121212121212121212121","share_difficulty":7,"share_id":"share-289","share_seq":289,"template_height":800009},{"accepted_at_ms":70506,"credit_policy":null,"job_id":"job-290","job_issued_at_ms":70482,"miner_id":"miner-4","network_difficulty":1000001,"ntime":1700000290,"order_key":"04:miner-4","p2mr_program_hex":"2222222222222222222222222222222222222222222222222222222222222222","share_difficulty":7,"share_id":"share-290","share_seq":290,"template_height":800009},{"accepted_at_ms":70486,"credit_policy":null,"job_id":"job-291","job_issued_at_ms":70495,"miner_id":"miner-5","network_difficulty":1000002,"ntime":1700000291,"order_key":"05:miner-5","p2mr_program_hex":"2323232323232323232323232323232323232323232323232323232323232323","share_difficulty":19,"share_id":"share-291","share_seq":291,"template_height":800009},{"accepted_at_ms":70492,"credit_policy":"stale-grace","job_id":"job-292","job_issued_at_ms":70487,"miner_id":"miner-6","network_difficulty":1000003,"ntime":1700000292,"order_key":"06:miner-6","p2mr_program_hex":"2424242424242424242424242424242424242424242424242424242424242424","share_difficulty":7,"share_id":"share-292","share_seq":292,"template_height":800009},{"accepted_at_ms":70497,"credit_policy":null,"job_id":"job-293","job_issued_at_ms":70490,"miner_id":"miner-7","network_difficulty":1000004,"ntime":1700000293,"order_key":"07:miner-7","p2mr_program_hex":"2525252525252525252525252525252525252525252525252525252525252525","share_difficulty":1,"share_id":"share-293","share_seq":293,"template_height":800009},{"accepted_at_ms":70487,"credit_policy":null,"job_id":"job-294","job_issued_at_ms":70506,"miner_id":"miner-8","network_difficulty":1000005,"ntime":1700000294,"order_key":"08:miner-8","p2mr_program_hex":"2626262626262626262626262626262626262626262626262626262626262626","share_difficulty":3,"share_id":"share-294","share_seq":294,"template_height":800009},{"accepted_at_ms":70482,"credit_policy":"stale-grace","job_id":"job-295","job_issued_at_ms":70496,"miner_id":"miner-9","network_difficulty":1000006,"ntime":1700000295,"order_key":"09:miner-9","p2mr_program_hex":"2727272727272727272727272727272727272727272727272727272727272727","share_difficulty":2,"share_id":"share-295","share_seq":295,"template_height":800009},{"accepted_at_ms":70511,"credit_policy":"stale-grace","job_id":"job-296","job_issued_at_ms":70487,"miner_id":"miner-10","network_difficulty":1000007,"ntime":1700000296,"order_key":"10:miner-10","p2mr_program_hex":"2828282828282828282828282828282828282828282828282828282828282828","share_difficulty":19,"share_id":"share-296","share_seq":296,"template_height":800009},{"accepted_at_ms":70490,"credit_policy":"stale-grace","job_id":"job-297","job_issued_at_ms":70499,"miner_id":"miner-0","network_difficulty":1000008,"ntime":1700000297,"order_key":"00:miner-0","p2mr_program_hex":"2929292929292929292929292929292929292929292929292929292929292929","share_difficulty":1,"share_id":"share-297","share_seq":297,"template_height":800009},{"accepted_at_ms":70493,"credit_policy":null,"job_id":"job-298","job_issued_at_ms":70501,"miner_id":"miner-1","network_difficulty":1000009,"ntime":1700000298,"order_key":"01:miner-1","p2mr_program_hex":"2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a","share_difficulty":1,"share_id":"share-298","share_seq":298,"template_height":800009},{"accepted_at_ms":70490,"credit_policy":null,"job_id":"job-299","job_issued_at_ms":70499,"miner_id":"miner-2","network_difficulty":1000010,"ntime":1700000299,"order_key":"02:miner-2","p2mr_program_hex":"2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b","share_difficulty":3,"share_id":"share-299","share_seq":299,"template_height":800009},{"accepted_at_ms":70504,"credit_policy":"stale-grace","job_id":"job-300","job_issued_at_ms":70497,"miner_id":"miner-3","network_difficulty":1000011,"ntime":1700000300,"order_key":"03:miner-3","p2mr_program_hex":"2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c","share_difficulty":7,"share_id":"share-300","share_seq":300,"template_height":800009},{"accepted_at_ms":70490,"credit_policy":"stale-grace","job_id":"job-301","job_issued_at_ms":70501,"miner_id":"miner-4","network_difficulty":1000012,"ntime":1700000301,"order_key":"04:miner-4","p2mr_program_hex":"2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d","share_difficulty":19,"share_id":"share-301","share_seq":301,"template_height":800009},{"accepted_at_ms":70511,"credit_policy":null,"job_id":"job-302","job_issued_at_ms":70510,"miner_id":"miner-5","network_difficulty":1000013,"ntime":1700000302,"order_key":"05:miner-5","p2mr_program_hex":"2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e","share_difficulty":7,"share_id":"share-302","share_seq":302,"template_height":800009},{"accepted_at_ms":70488,"credit_policy":null,"job_id":"job-303","job_issued_at_ms":70507,"miner_id":"miner-6","network_difficulty":1000014,"ntime":1700000303,"order_key":"06:miner-6","p2mr_program_hex":"2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f","share_difficulty":19,"share_id":"share-303","share_seq":303,"template_height":800009},{"accepted_at_ms":70505,"credit_policy":"stale-grace","job_id":"job-304","job_issued_at_ms":70508,"miner_id":"miner-7","network_difficulty":1000015,"ntime":1700000304,"order_key":"07:miner-7","p2mr_program_hex":"3030303030303030303030303030303030303030303030303030303030303030","share_difficulty":1,"share_id":"share-304","share_seq":304,"template_height":800009},{"accepted_at_ms":70489,"credit_policy":null,"job_id":"job-305","job_issued_at_ms":70497,"miner_id":"miner-8","network_difficulty":1000016,"ntime":1700000305,"order_key":"08:miner-8","p2mr_program_hex":"3131313131313131313131313131313131313131313131313131313131313131","share_difficulty":19,"share_id":"share-305","share_seq":305,"template_height":800009},{"accepted_at_ms":70495,"credit_policy":null,"job_id":"job-306","job_issued_at_ms":70507,"miner_id":"miner-9","network_difficulty":1000000,"ntime":1700000306,"order_key":"09:miner-9","p2mr_program_hex":"3232323232323232323232323232323232323232323232323232323232323232","share_difficulty":19,"share_id":"share-306","share_seq":306,"template_height":800009},{"accepted_at_ms":70491,"credit_policy":null,"job_id":"job-307","job_issued_at_ms":70511,"miner_id":"miner-10","network_difficulty":1000001,"ntime":1700000307,"order_key":"10:miner-10","p2mr_program_hex":"3333333333333333333333333333333333333333333333333333333333333333","share_difficulty":2,"share_id":"share-307","share_seq":307,"template_height":800009},{"accepted_at_ms":70497,"credit_policy":"stale-grace","job_id":"job-308","job_issued_at_ms":70506,"miner_id":"miner-0","network_difficulty":1000002,"ntime":1700000308,"order_key":"00:miner-0","p2mr_program_hex":"3434343434343434343434343434343434343434343434343434343434343434","share_difficulty":3,"share_id":"share-308","share_seq":308,"template_height":800009},{"accepted_at_ms":70497,"credit_policy":null,"job_id":"job-309","job_issued_at_ms":70483,"miner_id":"miner-1","network_difficulty":1000003,"ntime":1700000309,"order_key":"01:miner-1","p2mr_program_hex":"3535353535353535353535353535353535353535353535353535353535353535","share_difficulty":1,"share_id":"share-309","share_seq":309,"template_height":800009},{"accepted_at_ms":70485,"credit_policy":null,"job_id":"job-310","job_issued_at_ms":70509,"miner_id":"miner-2","network_difficulty":1000004,"ntime":1700000310,"order_key":"02:miner-2","p2mr_program_hex":"3636363636363636363636363636363636363636363636363636363636363636","share_difficulty":7,"share_id":"share-310","share_seq":310,"template_height":800009},{"accepted_at_ms":70503,"credit_policy":null,"job_id":"job-311","job_issued_at_ms":70483,"miner_id":"miner-3","network_difficulty":1000005,"ntime":1700000311,"order_key":"03:miner-3","p2mr_program_hex":"3737373737373737373737373737373737373737373737373737373737373737","share_difficulty":7,"share_id":"share-311","share_seq":311,"template_height":800009},{"accepted_at_ms":70507,"credit_policy":null,"job_id":"job-312","job_issued_at_ms":70493,"miner_id":"miner-4","network_difficulty":1000006,"ntime":1700000312,"order_key":"04:miner-4","p2mr_program_hex":"3838383838383838383838383838383838383838383838383838383838383838","share_difficulty":7,"share_id":"share-312","share_seq":312,"template_height":800009},{"accepted_at_ms":70498,"credit_policy":null,"job_id":"job-313","job_issued_at_ms":70483,"miner_id":"miner-5","network_difficulty":1000007,"ntime":1700000313,"order_key":"05:miner-5","p2mr_program_hex":"3939393939393939393939393939393939393939393939393939393939393939","share_difficulty":3,"share_id":"share-313","share_seq":313,"template_height":800009},{"accepted_at_ms":70492,"credit_policy":null,"job_id":"job-314","job_issued_at_ms":70489,"miner_id":"miner-6","network_difficulty":1000008,"ntime":1700000314,"order_key":"06:miner-6","p2mr_program_hex":"3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a","share_difficulty":2,"share_id":"share-314","share_seq":314,"template_height":800009},{"accepted_at_ms":70510,"credit_policy":null,"job_id":"job-315","job_issued_at_ms":70505,"miner_id":"miner-7","network_difficulty":1000009,"ntime":1700000315,"order_key":"07:miner-7","p2mr_program_hex":"3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b","share_difficulty":2,"share_id":"share-315","share_seq":315,"template_height":800009},{"accepted_at_ms":70497,"credit_policy":null,"job_id":"job-316","job_issued_at_ms":70482,"miner_id":"miner-8","network_difficulty":1000010,"ntime":1700000316,"order_key":"08:miner-8","p2mr_program_hex":"3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c","share_difficulty":19,"share_id":"share-316","share_seq":316,"template_height":800009},{"accepted_at_ms":70489,"credit_policy":"stale-grace","job_id":"job-317","job_issued_at_ms":70490,"miner_id":"miner-9","network_difficulty":1000011,"ntime":1700000317,"order_key":"09:miner-9","p2mr_program_hex":"3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d","share_difficulty":19,"share_id":"share-317","share_seq":317,"template_height":800009},{"accepted_at_ms":70494,"credit_policy":null,"job_id":"job-318","job_issued_at_ms":70497,"miner_id":"miner-10","network_difficulty":1000012,"ntime":1700000318,"order_key":"10:miner-10","p2mr_program_hex":"3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e","share_difficulty":3,"share_id":"share-318","share_seq":318,"template_height":800009},{"accepted_at_ms":70500,"credit_policy":null,"job_id":"job-319","job_issued_at_ms":70494,"miner_id":"miner-0","network_difficulty":1000013,"ntime":1700000319,"order_key":"00:miner-0","p2mr_program_hex":"3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f","share_difficulty":1,"share_id":"share-319","share_seq":319,"template_height":800009},{"accepted_at_ms":70509,"credit_policy":null,"job_id":"job-320","job_issued_at_ms":70500,"miner_id":"miner-1","network_difficulty":1000014,"ntime":1700000320,"order_key":"01:miner-1","p2mr_program_hex":"4040404040404040404040404040404040404040404040404040404040404040","share_difficulty":3,"share_id":"share-320","share_seq":320,"template_height":800010},{"accepted_at_ms":70486,"credit_policy":"stale-grace","job_id":"job-321","job_issued_at_ms":70486,"miner_id":"miner-2","network_difficulty":1000015,"ntime":1700000321,"order_key":"02:miner-2","p2mr_program_hex":"4141414141414141414141414141414141414141414141414141414141414141","share_difficulty":19,"share_id":"share-321","share_seq":321,"template_height":800010},{"accepted_at_ms":70495,"credit_policy":null,"job_id":"job-322","job_issued_at_ms":70485,"miner_id":"miner-3","network_difficulty":1000016,"ntime":1700000322,"order_key":"03:miner-3","p2mr_program_hex":"4242424242424242424242424242424242424242424242424242424242424242","share_difficulty":2,"share_id":"share-322","share_seq":322,"template_height":800010},{"accepted_at_ms":70484,"credit_policy":null,"job_id":"job-323","job_issued_at_ms":70498,"miner_id":"miner-4","network_difficulty":1000000,"ntime":1700000323,"order_key":"04:miner-4","p2mr_program_hex":"4343434343434343434343434343434343434343434343434343434343434343","share_difficulty":2,"share_id":"share-323","share_seq":323,"template_height":800010},{"accepted_at_ms":70510,"credit_policy":null,"job_id":"job-324","job_issued_at_ms":70491,"miner_id":"miner-5","network_difficulty":1000001,"ntime":1700000324,"order_key":"05:miner-5","p2mr_program_hex":"4444444444444444444444444444444444444444444444444444444444444444","share_difficulty":2,"share_id":"share-324","share_seq":324,"template_height":800010},{"accepted_at_ms":70505,"credit_policy":"stale-grace","job_id":"job-325","job_issued_at_ms":70497,"miner_id":"miner-6","network_difficulty":1000002,"ntime":1700000325,"order_key":"06:miner-6","p2mr_program_hex":"4545454545454545454545454545454545454545454545454545454545454545","share_difficulty":19,"share_id":"share-325","share_seq":325,"template_height":800010},{"accepted_at_ms":70504,"credit_policy":"stale-grace","job_id":"job-326","job_issued_at_ms":70505,"miner_id":"miner-7","network_difficulty":1000003,"ntime":1700000326,"order_key":"07:miner-7","p2mr_program_hex":"4646464646464646464646464646464646464646464646464646464646464646","share_difficulty":19,"share_id":"share-326","share_seq":326,"template_height":800010},{"accepted_at_ms":70500,"credit_policy":null,"job_id":"job-327","job_issued_at_ms":70500,"miner_id":"miner-8","network_difficulty":1000004,"ntime":1700000327,"order_key":"08:miner-8","p2mr_program_hex":"4747474747474747474747474747474747474747474747474747474747474747","share_difficulty":1,"share_id":"share-327","share_seq":327,"template_height":800010},{"accepted_at_ms":70482,"credit_policy":null,"job_id":"job-328","job_issued_at_ms":70501,"miner_id":"miner-9","network_difficulty":1000005,"ntime":1700000328,"order_key":"09:miner-9","p2mr_program_hex":"4848484848484848484848484848484848484848484848484848484848484848","share_difficulty":2,"share_id":"share-328","share_seq":328,"template_height":800010},{"accepted_at_ms":70509,"credit_policy":null,"job_id":"job-329","job_issued_at_ms":70497,"miner_id":"miner-10","network_difficulty":1000006,"ntime":1700000329,"order_key":"10:miner-10","p2mr_program_hex":"4949494949494949494949494949494949494949494949494949494949494949","share_difficulty":2,"share_id":"share-329","share_seq":329,"template_height":800010},{"accepted_at_ms":70493,"credit_policy":null,"job_id":"job-330","job_issued_at_ms":70484,"miner_id":"miner-0","network_difficulty":1000007,"ntime":1700000330,"order_key":"00:miner-0","p2mr_program_hex":"4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a","share_difficulty":2,"share_id":"share-330","share_seq":330,"template_height":800010},{"accepted_at_ms":70505,"credit_policy":null,"job_id":"job-331","job_issued_at_ms":70506,"miner_id":"miner-1","network_difficulty":1000008,"ntime":1700000331,"order_key":"01:miner-1","p2mr_program_hex":"4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b","share_difficulty":1,"share_id":"share-331","share_seq":331,"template_height":800010},{"accepted_at_ms":70496,"credit_policy":null,"job_id":"job-332","job_issued_at_ms":70490,"miner_id":"miner-2","network_difficulty":1000009,"ntime":1700000332,"order_key":"02:miner-2","p2mr_program_hex":"4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c","share_difficulty":19,"share_id":"share-332","share_seq":332,"template_height":800010},{"accepted_at_ms":70481,"credit_policy":null,"job_id":"job-333","job_issued_at_ms":70507,"miner_id":"miner-3","network_difficulty":1000010,"ntime":1700000333,"order_key":"03:miner-3","p2mr_program_hex":"4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d","share_difficulty":7,"share_id":"share-333","share_seq":333,"template_height":800010},{"accepted_at_ms":70499,"credit_policy":null,"job_id":"job-334","job_issued_at_ms":70493,"miner_id":"miner-4","network_difficulty":1000011,"ntime":1700000334,"order_key":"04:miner-4","p2mr_program_hex":"4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e","share_difficulty":7,"share_id":"share-334","share_seq":334,"template_height":800010},{"accepted_at_ms":70507,"credit_policy":null,"job_id":"job-335","job_issued_at_ms":70497,"miner_id":"miner-5","network_difficulty":1000012,"ntime":1700000335,"order_key":"05:miner-5","p2mr_program_hex":"4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f","share_difficulty":3,"share_id":"share-335","share_seq":335,"template_height":800010},{"accepted_at_ms":70506,"credit_policy":null,"job_id":"job-336","job_issued_at_ms":70494,"miner_id":"miner-6","network_difficulty":1000013,"ntime":1700000336,"order_key":"06:miner-6","p2mr_program_hex":"5050505050505050505050505050505050505050505050505050505050505050","share_difficulty":1,"share_id":"share-336","share_seq":336,"template_height":800010},{"accepted_at_ms":70500,"credit_policy":null,"job_id":"job-337","job_issued_at_ms":70490,"miner_id":"miner-7","network_difficulty":1000014,"ntime":1700000337,"order_key":"07:miner-7","p2mr_program_hex":"5151515151515151515151515151515151515151515151515151515151515151","share_difficulty":3,"share_id":"share-337","share_seq":337,"template_height":800010},{"accepted_at_ms":70492,"credit_policy":null,"job_id":"job-338","job_issued_at_ms":70499,"miner_id":"miner-8","network_difficulty":1000015,"ntime":1700000338,"order_key":"08:miner-8","p2mr_program_hex":"5252525252525252525252525252525252525252525252525252525252525252","share_difficulty":1,"share_id":"share-338","share_seq":338,"template_height":800010},{"accepted_at_ms":70481,"credit_policy":"stale-grace","job_id":"job-339","job_issued_at_ms":70496,"miner_id":"miner-9","network_difficulty":1000016,"ntime":1700000339,"order_key":"09:miner-9","p2mr_program_hex":"5353535353535353535353535353535353535353535353535353535353535353","share_difficulty":2,"share_id":"share-339","share_seq":339,"template_height":800010},{"accepted_at_ms":70511,"credit_policy":"stale-grace","job_id":"job-340","job_issued_at_ms":70510,"miner_id":"miner-10","network_difficulty":1000000,"ntime":1700000340,"order_key":"10:miner-10","p2mr_program_hex":"5454545454545454545454545454545454545454545454545454545454545454","share_difficulty":2,"share_id":"share-340","share_seq":340,"template_height":800010},{"accepted_at_ms":70503,"credit_policy":null,"job_id":"job-341","job_issued_at_ms":70500,"miner_id":"miner-0","network_difficulty":1000001,"ntime":1700000341,"order_key":"00:miner-0","p2mr_program_hex":"5555555555555555555555555555555555555555555555555555555555555555","share_difficulty":19,"share_id":"share-341","share_seq":341,"template_height":800010},{"accepted_at_ms":70491,"credit_policy":null,"job_id":"job-342","job_issued_at_ms":70503,"miner_id":"miner-1","network_difficulty":1000002,"ntime":1700000342,"order_key":"01:miner-1","p2mr_program_hex":"5656565656565656565656565656565656565656565656565656565656565656","share_difficulty":1,"share_id":"share-342","share_seq":342,"template_height":800010},{"accepted_at_ms":70484,"credit_policy":null,"job_id":"job-343","job_issued_at_ms":70488,"miner_id":"miner-2","network_difficulty":1000003,"ntime":1700000343,"order_key":"02:miner-2","p2mr_program_hex":"5757575757575757575757575757575757575757575757575757575757575757","share_difficulty":7,"share_id":"share-343","share_seq":343,"template_height":800010},{"accepted_at_ms":70502,"credit_policy":"stale-grace","job_id":"job-344","job_issued_at_ms":70501,"miner_id":"miner-3","network_difficulty":1000004,"ntime":1700000344,"order_key":"03:miner-3","p2mr_program_hex":"5858585858585858585858585858585858585858585858585858585858585858","share_difficulty":3,"share_id":"share-344","share_seq":344,"template_height":800010},{"accepted_at_ms":70487,"credit_policy":null,"job_id":"job-345","job_issued_at_ms":70490,"miner_id":"miner-4","network_difficulty":1000005,"ntime":1700000345,"order_key":"04:miner-4","p2mr_program_hex":"5959595959595959595959595959595959595959595959595959595959595959","share_difficulty":7,"share_id":"share-345","share_seq":345,"template_height":800010},{"accepted_at_ms":70508,"credit_policy":null,"job_id":"job-346","job_issued_at_ms":70505,"miner_id":"miner-5","network_difficulty":1000006,"ntime":1700000346,"order_key":"05:miner-5","p2mr_program_hex":"5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a","share_difficulty":1,"share_id":"share-346","share_seq":346,"template_height":800010},{"accepted_at_ms":70483,"credit_policy":"stale-grace","job_id":"job-347","job_issued_at_ms":70499,"miner_id":"miner-6","network_difficulty":1000007,"ntime":1700000347,"order_key":"06:miner-6","p2mr_program_hex":"5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b","share_difficulty":1,"share_id":"share-347","share_seq":347,"template_height":800010},{"accepted_at_ms":70481,"credit_policy":"stale-grace","job_id":"job-348","job_issued_at_ms":70490,"miner_id":"miner-7","network_difficulty":1000008,"ntime":1700000348,"order_key":"07:miner-7","p2mr_program_hex":"5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c","share_difficulty":3,"share_id":"share-348","share_seq":348,"template_height":800010},{"accepted_at_ms":70494,"credit_policy":null,"job_id":"job-349","job_issued_at_ms":70485,"miner_id":"miner-8","network_difficulty":1000009,"ntime":1700000349,"order_key":"08:miner-8","p2mr_program_hex":"5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d","share_difficulty":3,"share_id":"share-349","share_seq":349,"template_height":800010},{"accepted_at_ms":70483,"credit_policy":null,"job_id":"job-350","job_issued_at_ms":70487,"miner_id":"miner-9","network_difficulty":1000010,"ntime":1700000350,"order_key":"09:miner-9","p2mr_program_hex":"5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e","share_difficulty":7,"share_id":"share-350","share_seq":350,"template_height":800010},{"accepted_at_ms":70507,"credit_policy":null,"job_id":"job-351","job_issued_at_ms":70483,"miner_id":"miner-10","network_difficulty":1000011,"ntime":1700000351,"order_key":"10:miner-10","p2mr_program_hex":"5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f","share_difficulty":3,"share_id":"share-351","share_seq":351,"template_height":800010},{"accepted_at_ms":70494,"credit_policy":null,"job_id":"job-352","job_issued_at_ms":70495,"miner_id":"miner-0","network_difficulty":1000012,"ntime":1700000352,"order_key":"00:miner-0","p2mr_program_hex":"6060606060606060606060606060606060606060606060606060606060606060","share_difficulty":1,"share_id":"share-352","share_seq":352,"template_height":800011},{"accepted_at_ms":70507,"credit_policy":null,"job_id":"job-353","job_issued_at_ms":70486,"miner_id":"miner-1","network_difficulty":1000013,"ntime":1700000353,"order_key":"01:miner-1","p2mr_program_hex":"6161616161616161616161616161616161616161616161616161616161616161","share_difficulty":3,"share_id":"share-353","share_seq":353,"template_height":800011},{"accepted_at_ms":70497,"credit_policy":"stale-grace","job_id":"job-354","job_issued_at_ms":70503,"miner_id":"miner-2","network_difficulty":1000014,"ntime":1700000354,"order_key":"02:miner-2","p2mr_program_hex":"6262626262626262626262626262626262626262626262626262626262626262","share_difficulty":7,"share_id":"share-354","share_seq":354,"template_height":800011},{"accepted_at_ms":70490,"credit_policy":null,"job_id":"job-355","job_issued_at_ms":70500,"miner_id":"miner-3","network_difficulty":1000015,"ntime":1700000355,"order_key":"03:miner-3","p2mr_program_hex":"6363636363636363636363636363636363636363636363636363636363636363","share_difficulty":3,"share_id":"share-355","share_seq":355,"template_height":800011},{"accepted_at_ms":70491,"credit_policy":null,"job_id":"job-356","job_issued_at_ms":70500,"miner_id":"miner-4","network_difficulty":1000016,"ntime":1700000356,"order_key":"04:miner-4","p2mr_program_hex":"6464646464646464646464646464646464646464646464646464646464646464","share_difficulty":3,"share_id":"share-356","share_seq":356,"template_height":800011},{"accepted_at_ms":70503,"credit_policy":null,"job_id":"job-357","job_issued_at_ms":70504,"miner_id":"miner-5","network_difficulty":1000000,"ntime":1700000357,"order_key":"05:miner-5","p2mr_program_hex":"6565656565656565656565656565656565656565656565656565656565656565","share_difficulty":1,"share_id":"share-357","share_seq":357,"template_height":800011},{"accepted_at_ms":70486,"credit_policy":null,"job_id":"job-358","job_issued_at_ms":70493,"miner_id":"miner-6","network_difficulty":1000001,"ntime":1700000358,"order_key":"06:miner-6","p2mr_program_hex":"6666666666666666666666666666666666666666666666666666666666666666","share_difficulty":3,"share_id":"share-358","share_seq":358,"template_height":800011},{"accepted_at_ms":70503,"credit_policy":null,"job_id":"job-359","job_issued_at_ms":70488,"miner_id":"miner-7","network_difficulty":1000002,"ntime":1700000359,"order_key":"07:miner-7","p2mr_program_hex":"6767676767676767676767676767676767676767676767676767676767676767","share_difficulty":2,"share_id":"share-359","share_seq":359,"template_height":800011},{"accepted_at_ms":70496,"credit_policy":null,"job_id":"job-360","job_issued_at_ms":70497,"miner_id":"miner-8","network_difficulty":1000003,"ntime":1700000360,"order_key":"08:miner-8","p2mr_program_hex":"6868686868686868686868686868686868686868686868686868686868686868","share_difficulty":1,"share_id":"share-360","share_seq":360,"template_height":800011},{"accepted_at_ms":70501,"credit_policy":null,"job_id":"job-361","job_issued_at_ms":70489,"miner_id":"miner-9","network_difficulty":1000004,"ntime":1700000361,"order_key":"09:miner-9","p2mr_program_hex":"6969696969696969696969696969696969696969696969696969696969696969","share_difficulty":2,"share_id":"share-361","share_seq":361,"template_height":800011},{"accepted_at_ms":70502,"credit_policy":null,"job_id":"job-362","job_issued_at_ms":70501,"miner_id":"miner-10","network_difficulty":1000005,"ntime":1700000362,"order_key":"10:miner-10","p2mr_program_hex":"6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a","share_difficulty":2,"share_id":"share-362","share_seq":362,"template_height":800011},{"accepted_at_ms":70490,"credit_policy":null,"job_id":"job-363","job_issued_at_ms":70497,"miner_id":"miner-0","network_difficulty":1000006,"ntime":1700000363,"order_key":"00:miner-0","p2mr_program_hex":"6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b","share_difficulty":2,"share_id":"share-363","share_seq":363,"template_height":800011},{"accepted_at_ms":70506,"credit_policy":null,"job_id":"job-364","job_issued_at_ms":70496,"miner_id":"miner-1","network_difficulty":1000007,"ntime":1700000364,"order_key":"01:miner-1","p2mr_program_hex":"6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c","share_difficulty":19,"share_id":"share-364","share_seq":364,"template_height":800011},{"accepted_at_ms":70499,"credit_policy":null,"job_id":"job-365","job_issued_at_ms":70502,"miner_id":"miner-2","network_difficulty":1000008,"ntime":1700000365,"order_key":"02:miner-2","p2mr_program_hex":"6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d","share_difficulty":1,"share_id":"share-365","share_seq":365,"template_height":800011},{"accepted_at_ms":70489,"credit_policy":null,"job_id":"job-366","job_issued_at_ms":70485,"miner_id":"miner-3","network_difficulty":1000009,"ntime":1700000366,"order_key":"03:miner-3","p2mr_program_hex":"6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e","share_difficulty":3,"share_id":"share-366","share_seq":366,"template_height":800011},{"accepted_at_ms":70494,"credit_policy":"stale-grace","job_id":"job-367","job_issued_at_ms":70506,"miner_id":"miner-4","network_difficulty":1000010,"ntime":1700000367,"order_key":"04:miner-4","p2mr_program_hex":"6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f","share_difficulty":3,"share_id":"share-367","share_seq":367,"template_height":800011},{"accepted_at_ms":70484,"credit_policy":null,"job_id":"job-368","job_issued_at_ms":70489,"miner_id":"miner-5","network_difficulty":1000011,"ntime":1700000368,"order_key":"05:miner-5","p2mr_program_hex":"7070707070707070707070707070707070707070707070707070707070707070","share_difficulty":3,"share_id":"share-368","share_seq":368,"template_height":800011},{"accepted_at_ms":70506,"credit_policy":"stale-grace","job_id":"job-369","job_issued_at_ms":70493,"miner_id":"miner-6","network_difficulty":1000012,"ntime":1700000369,"order_key":"06:miner-6","p2mr_program_hex":"7171717171717171717171717171717171717171717171717171717171717171","share_difficulty":19,"share_id":"share-369","share_seq":369,"template_height":800011},{"accepted_at_ms":70506,"credit_policy":null,"job_id":"job-370","job_issued_at_ms":70505,"miner_id":"miner-7","network_difficulty":1000013,"ntime":1700000370,"order_key":"07:miner-7","p2mr_program_hex":"7272727272727272727272727272727272727272727272727272727272727272","share_difficulty":19,"share_id":"share-370","share_seq":370,"template_height":800011},{"accepted_at_ms":70485,"credit_policy":"stale-grace","job_id":"job-371","job_issued_at_ms":70487,"miner_id":"miner-8","network_difficulty":1000014,"ntime":1700000371,"order_key":"08:miner-8","p2mr_program_hex":"7373737373737373737373737373737373737373737373737373737373737373","share_difficulty":3,"share_id":"share-371","share_seq":371,"template_height":800011},{"accepted_at_ms":70505,"credit_policy":null,"job_id":"job-372","job_issued_at_ms":70483,"miner_id":"miner-9","network_difficulty":1000015,"ntime":1700000372,"order_key":"09:miner-9","p2mr_program_hex":"7474747474747474747474747474747474747474747474747474747474747474","share_difficulty":1,"share_id":"share-372","share_seq":372,"template_height":800011},{"accepted_at_ms":70497,"credit_policy":null,"job_id":"job-373","job_issued_at_ms":70497,"miner_id":"miner-10","network_difficulty":1000016,"ntime":1700000373,"order_key":"10:miner-10","p2mr_program_hex":"7575757575757575757575757575757575757575757575757575757575757575","share_difficulty":1,"share_id":"share-373","share_seq":373,"template_height":800011},{"accepted_at_ms":70503,"credit_policy":null,"job_id":"job-374","job_issued_at_ms":70492,"miner_id":"miner-0","network_difficulty":1000000,"ntime":1700000374,"order_key":"00:miner-0","p2mr_program_hex":"7676767676767676767676767676767676767676767676767676767676767676","share_difficulty":19,"share_id":"share-374","share_seq":374,"template_height":800011},{"accepted_at_ms":70483,"credit_policy":"stale-grace","job_id":"job-375","job_issued_at_ms":70491,"miner_id":"miner-1","network_difficulty":1000001,"ntime":1700000375,"order_key":"01:miner-1","p2mr_program_hex":"7777777777777777777777777777777777777777777777777777777777777777","share_difficulty":7,"share_id":"share-375","share_seq":375,"template_height":800011},{"accepted_at_ms":70499,"credit_policy":null,"job_id":"job-376","job_issued_at_ms":70496,"miner_id":"miner-2","network_difficulty":1000002,"ntime":1700000376,"order_key":"02:miner-2","p2mr_program_hex":"7878787878787878787878787878787878787878787878787878787878787878","share_difficulty":1,"share_id":"share-376","share_seq":376,"template_height":800011},{"accepted_at_ms":70486,"credit_policy":null,"job_id":"job-377","job_issued_at_ms":70494,"miner_id":"miner-3","network_difficulty":1000003,"ntime":1700000377,"order_key":"03:miner-3","p2mr_program_hex":"7979797979797979797979797979797979797979797979797979797979797979","share_difficulty":3,"share_id":"share-377","share_seq":377,"template_height":800011},{"accepted_at_ms":70487,"credit_policy":"stale-grace","job_id":"job-378","job_issued_at_ms":70489,"miner_id":"miner-4","network_difficulty":1000004,"ntime":1700000378,"order_key":"04:miner-4","p2mr_program_hex":"7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a","share_difficulty":1,"share_id":"share-378","share_seq":378,"template_height":800011},{"accepted_at_ms":70504,"credit_policy":null,"job_id":"job-379","job_issued_at_ms":70493,"miner_id":"miner-5","network_difficulty":1000005,"ntime":1700000379,"order_key":"05:miner-5","p2mr_program_hex":"7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b","share_difficulty":7,"share_id":"share-379","share_seq":379,"template_height":800011},{"accepted_at_ms":70490,"credit_policy":"stale-grace","job_id":"job-380","job_issued_at_ms":70495,"miner_id":"miner-6","network_difficulty":1000006,"ntime":1700000380,"order_key":"06:miner-6","p2mr_program_hex":"7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c","share_difficulty":1,"share_id":"share-380","share_seq":380,"template_height":800011},{"accepted_at_ms":70490,"credit_policy":"stale-grace","job_id":"job-381","job_issued_at_ms":70481,"miner_id":"miner-7","network_difficulty":1000007,"ntime":1700000381,"order_key":"07:miner-7","p2mr_program_hex":"7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d","share_difficulty":1,"share_id":"share-381","share_seq":381,"template_height":800011},{"accepted_at_ms":70507,"credit_policy":null,"job_id":"job-382","job_issued_at_ms":70510,"miner_id":"miner-8","network_difficulty":1000008,"ntime":1700000382,"order_key":"08:miner-8","p2mr_program_hex":"7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e","share_difficulty":7,"share_id":"share-382","share_seq":382,"template_height":800011},{"accepted_at_ms":70484,"credit_policy":"stale-grace","job_id":"job-383","job_issued_at_ms":70495,"miner_id":"miner-9","network_difficulty":1000009,"ntime":1700000383,"order_key":"09:miner-9","p2mr_program_hex":"7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f","share_difficulty":2,"share_id":"share-383","share_seq":383,"template_height":800011},{"accepted_at_ms":70511,"credit_policy":null,"job_id":"job-384","job_issued_at_ms":70483,"miner_id":"miner-10","network_difficulty":1000010,"ntime":1700000384,"order_key":"10:miner-10","p2mr_program_hex":"8080808080808080808080808080808080808080808080808080808080808080","share_difficulty":7,"share_id":"share-384","share_seq":384,"template_height":800012},{"accepted_at_ms":70491,"credit_policy":null,"job_id":"job-385","job_issued_at_ms":70508,"miner_id":"miner-0","network_difficulty":1000011,"ntime":1700000385,"order_key":"00:miner-0","p2mr_program_hex":"8181818181818181818181818181818181818181818181818181818181818181","share_difficulty":19,"share_id":"share-385","share_seq":385,"template_height":800012},{"accepted_at_ms":70494,"credit_policy":null,"job_id":"job-386","job_issued_at_ms":70506,"miner_id":"miner-1","network_difficulty":1000012,"ntime":1700000386,"order_key":"01:miner-1","p2mr_program_hex":"8282828282828282828282828282828282828282828282828282828282828282","share_difficulty":7,"share_id":"share-386","share_seq":386,"template_height":800012},{"accepted_at_ms":70487,"credit_policy":null,"job_id":"job-387","job_issued_at_ms":70488,"miner_id":"miner-2","network_difficulty":1000013,"ntime":1700000387,"order_key":"02:miner-2","p2mr_program_hex":"8383838383838383838383838383838383838383838383838383838383838383","share_difficulty":1,"share_id":"share-387","share_seq":387,"template_height":800012},{"accepted_at_ms":70511,"credit_policy":null,"job_id":"job-388","job_issued_at_ms":70511,"miner_id":"miner-3","network_difficulty":1000014,"ntime":1700000388,"order_key":"03:miner-3","p2mr_program_hex":"8484848484848484848484848484848484848484848484848484848484848484","share_difficulty":19,"share_id":"share-388","share_seq":388,"template_height":800012},{"accepted_at_ms":70483,"credit_policy":null,"job_id":"job-389","job_issued_at_ms":70505,"miner_id":"miner-4","network_difficulty":1000015,"ntime":1700000389,"order_key":"04:miner-4","p2mr_program_hex":"8585858585858585858585858585858585858585858585858585858585858585","share_difficulty":7,"share_id":"share-389","share_seq":389,"template_height":800012},{"accepted_at_ms":70504,"credit_policy":null,"job_id":"job-390","job_issued_at_ms":70502,"miner_id":"miner-5","network_difficulty":1000016,"ntime":1700000390,"order_key":"05:miner-5","p2mr_program_hex":"8686868686868686868686868686868686868686868686868686868686868686","share_difficulty":19,"share_id":"share-390","share_seq":390,"template_height":800012},{"accepted_at_ms":70481,"credit_policy":null,"job_id":"job-391","job_issued_at_ms":70510,"miner_id":"miner-6","network_difficulty":1000000,"ntime":1700000391,"order_key":"06:miner-6","p2mr_program_hex":"8787878787878787878787878787878787878787878787878787878787878787","share_difficulty":19,"share_id":"share-391","share_seq":391,"template_height":800012},{"accepted_at_ms":70486,"credit_policy":null,"job_id":"job-392","job_issued_at_ms":70489,"miner_id":"miner-7","network_difficulty":1000001,"ntime":1700000392,"order_key":"07:miner-7","p2mr_program_hex":"8888888888888888888888888888888888888888888888888888888888888888","share_difficulty":1,"share_id":"share-392","share_seq":392,"template_height":800012},{"accepted_at_ms":70506,"credit_policy":null,"job_id":"job-393","job_issued_at_ms":70485,"miner_id":"miner-8","network_difficulty":1000002,"ntime":1700000393,"order_key":"08:miner-8","p2mr_program_hex":"8989898989898989898989898989898989898989898989898989898989898989","share_difficulty":19,"share_id":"share-393","share_seq":393,"template_height":800012},{"accepted_at_ms":70486,"credit_policy":"stale-grace","job_id":"job-394","job_issued_at_ms":70507,"miner_id":"miner-9","network_difficulty":1000003,"ntime":1700000394,"order_key":"09:miner-9","p2mr_program_hex":"8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a","share_difficulty":1,"share_id":"share-394","share_seq":394,"template_height":800012},{"accepted_at_ms":70493,"credit_policy":null,"job_id":"job-395","job_issued_at_ms":70509,"miner_id":"miner-10","network_difficulty":1000004,"ntime":1700000395,"order_key":"10:miner-10","p2mr_program_hex":"8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b","share_difficulty":7,"share_id":"share-395","share_seq":395,"template_height":800012},{"accepted_at_ms":70498,"credit_policy":null,"job_id":"job-396","job_issued_at_ms":70497,"miner_id":"miner-0","network_difficulty":1000005,"ntime":1700000396,"order_key":"00:miner-0","p2mr_program_hex":"8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c","share_difficulty":19,"share_id":"share-396","share_seq":396,"template_height":800012},{"accepted_at_ms":70495,"credit_policy":null,"job_id":"job-397","job_issued_at_ms":70498,"miner_id":"miner-1","network_difficulty":1000006,"ntime":1700000397,"order_key":"01:miner-1","p2mr_program_hex":"8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d","share_difficulty":3,"share_id":"share-397","share_seq":397,"template_height":800012},{"accepted_at_ms":70490,"credit_policy":null,"job_id":"job-398","job_issued_at_ms":70482,"miner_id":"miner-2","network_difficulty":1000007,"ntime":1700000398,"order_key":"02:miner-2","p2mr_program_hex":"8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e","share_difficulty":19,"share_id":"share-398","share_seq":398,"template_height":800012},{"accepted_at_ms":70502,"credit_policy":null,"job_id":"job-399","job_issued_at_ms":70496,"miner_id":"miner-3","network_difficulty":1000008,"ntime":1700000399,"order_key":"03:miner-3","p2mr_program_hex":"8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f","share_difficulty":19,"share_id":"share-399","share_seq":399,"template_height":800012},{"accepted_at_ms":70508,"credit_policy":null,"job_id":"job-400","job_issued_at_ms":70499,"miner_id":"miner-4","network_difficulty":1000009,"ntime":1700000400,"order_key":"04:miner-4","p2mr_program_hex":"9090909090909090909090909090909090909090909090909090909090909090","share_difficulty":1,"share_id":"share-400","share_seq":400,"template_height":800012},{"accepted_at_ms":70510,"credit_policy":null,"job_id":"job-401","job_issued_at_ms":70501,"miner_id":"miner-5","network_difficulty":1000010,"ntime":1700000401,"order_key":"05:miner-5","p2mr_program_hex":"9191919191919191919191919191919191919191919191919191919191919191","share_difficulty":7,"share_id":"share-401","share_seq":401,"template_height":800012},{"accepted_at_ms":70491,"credit_policy":null,"job_id":"job-402","job_issued_at_ms":70498,"miner_id":"miner-6","network_difficulty":1000011,"ntime":1700000402,"order_key":"06:miner-6","p2mr_program_hex":"9292929292929292929292929292929292929292929292929292929292929292","share_difficulty":3,"share_id":"share-402","share_seq":402,"template_height":800012},{"accepted_at_ms":70488,"credit_policy":"stale-grace","job_id":"job-403","job_issued_at_ms":70491,"miner_id":"miner-7","network_difficulty":1000012,"ntime":1700000403,"order_key":"07:miner-7","p2mr_program_hex":"9393939393939393939393939393939393939393939393939393939393939393","share_difficulty":3,"share_id":"share-403","share_seq":403,"template_height":800012},{"accepted_at_ms":70500,"credit_policy":null,"job_id":"job-404","job_issued_at_ms":70511,"miner_id":"miner-8","network_difficulty":1000013,"ntime":1700000404,"order_key":"08:miner-8","p2mr_program_hex":"9494949494949494949494949494949494949494949494949494949494949494","share_difficulty":1,"share_id":"share-404","share_seq":404,"template_height":800012},{"accepted_at_ms":70503,"credit_policy":null,"job_id":"job-405","job_issued_at_ms":70508,"miner_id":"miner-9","network_difficulty":1000014,"ntime":1700000405,"order_key":"09:miner-9","p2mr_program_hex":"9595959595959595959595959595959595959595959595959595959595959595","share_difficulty":2,"share_id":"share-405","share_seq":405,"template_height":800012},{"accepted_at_ms":70482,"credit_policy":null,"job_id":"job-406","job_issued_at_ms":70506,"miner_id":"miner-10","network_difficulty":1000015,"ntime":1700000406,"order_key":"10:miner-10","p2mr_program_hex":"9696969696969696969696969696969696969696969696969696969696969696","share_difficulty":3,"share_id":"share-406","share_seq":406,"template_height":800012},{"accepted_at_ms":70508,"credit_policy":"stale-grace","job_id":"job-407","job_issued_at_ms":70489,"miner_id":"miner-0","network_difficulty":1000016,"ntime":1700000407,"order_key":"00:miner-0","p2mr_program_hex":"9797979797979797979797979797979797979797979797979797979797979797","share_difficulty":2,"share_id":"share-407","share_seq":407,"template_height":800012},{"accepted_at_ms":70487,"credit_policy":"stale-grace","job_id":"job-408","job_issued_at_ms":70490,"miner_id":"miner-1","network_difficulty":1000000,"ntime":1700000408,"order_key":"01:miner-1","p2mr_program_hex":"9898989898989898989898989898989898989898989898989898989898989898","share_difficulty":3,"share_id":"share-408","share_seq":408,"template_height":800012},{"accepted_at_ms":70499,"credit_policy":"stale-grace","job_id":"job-409","job_issued_at_ms":70498,"miner_id":"miner-2","network_difficulty":1000001,"ntime":1700000409,"order_key":"02:miner-2","p2mr_program_hex":"9999999999999999999999999999999999999999999999999999999999999999","share_difficulty":1,"share_id":"share-409","share_seq":409,"template_height":800012},{"accepted_at_ms":70510,"credit_policy":null,"job_id":"job-410","job_issued_at_ms":70482,"miner_id":"miner-3","network_difficulty":1000002,"ntime":1700000410,"order_key":"03:miner-3","p2mr_program_hex":"9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a","share_difficulty":7,"share_id":"share-410","share_seq":410,"template_height":800012},{"accepted_at_ms":70510,"credit_policy":null,"job_id":"job-411","job_issued_at_ms":70499,"miner_id":"miner-4","network_difficulty":1000003,"ntime":1700000411,"order_key":"04:miner-4","p2mr_program_hex":"9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b","share_difficulty":3,"share_id":"share-411","share_seq":411,"template_height":800012},{"accepted_at_ms":70503,"credit_policy":null,"job_id":"job-412","job_issued_at_ms":70487,"miner_id":"miner-5","network_difficulty":1000004,"ntime":1700000412,"order_key":"05:miner-5","p2mr_program_hex":"9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c","share_difficulty":1,"share_id":"share-412","share_seq":412,"template_height":800012},{"accepted_at_ms":70506,"credit_policy":null,"job_id":"job-413","job_issued_at_ms":70495,"miner_id":"miner-6","network_difficulty":1000005,"ntime":1700000413,"order_key":"06:miner-6","p2mr_program_hex":"9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d","share_difficulty":7,"share_id":"share-413","share_seq":413,"template_height":800012},{"accepted_at_ms":70498,"credit_policy":null,"job_id":"job-414","job_issued_at_ms":70482,"miner_id":"miner-7","network_difficulty":1000006,"ntime":1700000414,"order_key":"07:miner-7","p2mr_program_hex":"9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e","share_difficulty":3,"share_id":"share-414","share_seq":414,"template_height":800012},{"accepted_at_ms":70511,"credit_policy":"stale-grace","job_id":"job-415","job_issued_at_ms":70511,"miner_id":"miner-8","network_difficulty":1000007,"ntime":1700000415,"order_key":"08:miner-8","p2mr_program_hex":"9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f","share_difficulty":1,"share_id":"share-415","share_seq":415,"template_height":800012},{"accepted_at_ms":70491,"credit_policy":null,"job_id":"job-416","job_issued_at_ms":70509,"miner_id":"miner-9","network_difficulty":1000008,"ntime":1700000416,"order_key":"09:miner-9","p2mr_program_hex":"a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0","share_difficulty":19,"share_id":"share-416","share_seq":416,"template_height":800013},{"accepted_at_ms":70506,"credit_policy":null,"job_id":"job-417","job_issued_at_ms":70503,"miner_id":"miner-10","network_difficulty":1000009,"ntime":1700000417,"order_key":"10:miner-10","p2mr_program_hex":"a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1","share_difficulty":3,"share_id":"share-417","share_seq":417,"template_height":800013},{"accepted_at_ms":70501,"credit_policy":null,"job_id":"job-418","job_issued_at_ms":70490,"miner_id":"miner-0","network_difficulty":1000010,"ntime":1700000418,"order_key":"00:miner-0","p2mr_program_hex":"a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2","share_difficulty":7,"share_id":"share-418","share_seq":418,"template_height":800013},{"accepted_at_ms":70503,"credit_policy":"stale-grace","job_id":"job-419","job_issued_at_ms":70503,"miner_id":"miner-1","network_difficulty":1000011,"ntime":1700000419,"order_key":"01:miner-1","p2mr_program_hex":"a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3","share_difficulty":1,"share_id":"share-419","share_seq":419,"template_height":800013},{"accepted_at_ms":70492,"credit_policy":null,"job_id":"job-420","job_issued_at_ms":70510,"miner_id":"miner-2","network_difficulty":1000012,"ntime":1700000420,"order_key":"02:miner-2","p2mr_program_hex":"a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4","share_difficulty":1,"share_id":"share-420","share_seq":420,"template_height":800013},{"accepted_at_ms":70483,"credit_policy":null,"job_id":"job-421","job_issued_at_ms":70489,"miner_id":"miner-3","network_difficulty":1000013,"ntime":1700000421,"order_key":"03:miner-3","p2mr_program_hex":"a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5","share_difficulty":7,"share_id":"share-421","share_seq":421,"template_height":800013},{"accepted_at_ms":70498,"credit_policy":"stale-grace","job_id":"job-422","job_issued_at_ms":70485,"miner_id":"miner-4","network_difficulty":1000014,"ntime":1700000422,"order_key":"04:miner-4","p2mr_program_hex":"a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6","share_difficulty":1,"share_id":"share-422","share_seq":422,"template_height":800013},{"accepted_at_ms":70483,"credit_policy":"stale-grace","job_id":"job-423","job_issued_at_ms":70485,"miner_id":"miner-5","network_difficulty":1000015,"ntime":1700000423,"order_key":"05:miner-5","p2mr_program_hex":"a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7","share_difficulty":1,"share_id":"share-423","share_seq":423,"template_height":800013},{"accepted_at_ms":70500,"credit_policy":null,"job_id":"job-424","job_issued_at_ms":70503,"miner_id":"miner-6","network_difficulty":1000016,"ntime":1700000424,"order_key":"06:miner-6","p2mr_program_hex":"a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8","share_difficulty":7,"share_id":"share-424","share_seq":424,"template_height":800013},{"accepted_at_ms":70490,"credit_policy":"stale-grace","job_id":"job-425","job_issued_at_ms":70505,"miner_id":"miner-7","network_difficulty":1000000,"ntime":1700000425,"order_key":"07:miner-7","p2mr_program_hex":"a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9","share_difficulty":3,"share_id":"share-425","share_seq":425,"template_height":800013},{"accepted_at_ms":70501,"credit_policy":null,"job_id":"job-426","job_issued_at_ms":70498,"miner_id":"miner-8","network_difficulty":1000001,"ntime":1700000426,"order_key":"08:miner-8","p2mr_program_hex":"aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa","share_difficulty":19,"share_id":"share-426","share_seq":426,"template_height":800013},{"accepted_at_ms":70496,"credit_policy":"stale-grace","job_id":"job-427","job_issued_at_ms":70482,"miner_id":"miner-9","network_difficulty":1000002,"ntime":1700000427,"order_key":"09:miner-9","p2mr_program_hex":"abababababababababababababababababababababababababababababababab","share_difficulty":2,"share_id":"share-427","share_seq":427,"template_height":800013},{"accepted_at_ms":70484,"credit_policy":"stale-grace","job_id":"job-428","job_issued_at_ms":70502,"miner_id":"miner-10","network_difficulty":1000003,"ntime":1700000428,"order_key":"10:miner-10","p2mr_program_hex":"acacacacacacacacacacacacacacacacacacacacacacacacacacacacacacacac","share_difficulty":1,"share_id":"share-428","share_seq":428,"template_height":800013},{"accepted_at_ms":70486,"credit_policy":"stale-grace","job_id":"job-429","job_issued_at_ms":70499,"miner_id":"miner-0","network_difficulty":1000004,"ntime":1700000429,"order_key":"00:miner-0","p2mr_program_hex":"adadadadadadadadadadadadadadadadadadadadadadadadadadadadadadadad","share_difficulty":19,"share_id":"share-429","share_seq":429,"template_height":800013},{"accepted_at_ms":70511,"credit_policy":null,"job_id":"job-430","job_issued_at_ms":70491,"miner_id":"miner-1","network_difficulty":1000005,"ntime":1700000430,"order_key":"01:miner-1","p2mr_program_hex":"aeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeae","share_difficulty":2,"share_id":"share-430","share_seq":430,"template_height":800013},{"accepted_at_ms":70489,"credit_policy":null,"job_id":"job-431","job_issued_at_ms":70497,"miner_id":"miner-2","network_difficulty":1000006,"ntime":1700000431,"order_key":"02:miner-2","p2mr_program_hex":"afafafafafafafafafafafafafafafafafafafafafafafafafafafafafafafaf","share_difficulty":7,"share_id":"share-431","share_seq":431,"template_height":800013},{"accepted_at_ms":70491,"credit_policy":null,"job_id":"job-432","job_issued_at_ms":70495,"miner_id":"miner-3","network_difficulty":1000007,"ntime":1700000432,"order_key":"03:miner-3","p2mr_program_hex":"b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0","share_difficulty":1,"share_id":"share-432","share_seq":432,"template_height":800013},{"accepted_at_ms":70487,"credit_policy":"stale-grace","job_id":"job-433","job_issued_at_ms":70486,"miner_id":"miner-4","network_difficulty":1000008,"ntime":1700000433,"order_key":"04:miner-4","p2mr_program_hex":"b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1","share_difficulty":7,"share_id":"share-433","share_seq":433,"template_height":800013},{"accepted_at_ms":70489,"credit_policy":null,"job_id":"job-434","job_issued_at_ms":70502,"miner_id":"miner-5","network_difficulty":1000009,"ntime":1700000434,"order_key":"05:miner-5","p2mr_program_hex":"b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2","share_difficulty":19,"share_id":"share-434","share_seq":434,"template_height":800013},{"accepted_at_ms":70483,"credit_policy":"stale-grace","job_id":"job-435","job_issued_at_ms":70498,"miner_id":"miner-6","network_difficulty":1000010,"ntime":1700000435,"order_key":"06:miner-6","p2mr_program_hex":"b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3","share_difficulty":19,"share_id":"share-435","share_seq":435,"template_height":800013},{"accepted_at_ms":70489,"credit_policy":"stale-grace","job_id":"job-436","job_issued_at_ms":70494,"miner_id":"miner-7","network_difficulty":1000011,"ntime":1700000436,"order_key":"07:miner-7","p2mr_program_hex":"b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4","share_difficulty":2,"share_id":"share-436","share_seq":436,"template_height":800013},{"accepted_at_ms":70491,"credit_policy":"stale-grace","job_id":"job-437","job_issued_at_ms":70497,"miner_id":"miner-8","network_difficulty":1000012,"ntime":1700000437,"order_key":"08:miner-8","p2mr_program_hex":"b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5","share_difficulty":1,"share_id":"share-437","share_seq":437,"template_height":800013},{"accepted_at_ms":70509,"credit_policy":null,"job_id":"job-438","job_issued_at_ms":70487,"miner_id":"miner-9","network_difficulty":1000013,"ntime":1700000438,"order_key":"09:miner-9","p2mr_program_hex":"b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6","share_difficulty":2,"share_id":"share-438","share_seq":438,"template_height":800013},{"accepted_at_ms":70511,"credit_policy":null,"job_id":"job-439","job_issued_at_ms":70486,"miner_id":"miner-10","network_difficulty":1000014,"ntime":1700000439,"order_key":"10:miner-10","p2mr_program_hex":"b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7","share_difficulty":19,"share_id":"share-439","share_seq":439,"template_height":800013},{"accepted_at_ms":70506,"credit_policy":"stale-grace","job_id":"job-440","job_issued_at_ms":70489,"miner_id":"miner-0","network_difficulty":1000015,"ntime":1700000440,"order_key":"00:miner-0","p2mr_program_hex":"b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8","share_difficulty":3,"share_id":"share-440","share_seq":440,"template_height":800013},{"accepted_at_ms":70488,"credit_policy":null,"job_id":"job-441","job_issued_at_ms":70507,"miner_id":"miner-1","network_difficulty":1000016,"ntime":1700000441,"order_key":"01:miner-1","p2mr_program_hex":"b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9","share_difficulty":1,"share_id":"share-441","share_seq":441,"template_height":800013},{"accepted_at_ms":70497,"credit_policy":null,"job_id":"job-442","job_issued_at_ms":70501,"miner_id":"miner-2","network_difficulty":1000000,"ntime":1700000442,"order_key":"02:miner-2","p2mr_program_hex":"babababababababababababababababababababababababababababababababa","share_difficulty":7,"share_id":"share-442","share_seq":442,"template_height":800013},{"accepted_at_ms":70510,"credit_policy":null,"job_id":"job-443","job_issued_at_ms":70482,"miner_id":"miner-3","network_difficulty":1000001,"ntime":1700000443,"order_key":"03:miner-3","p2mr_program_hex":"bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb","share_difficulty":3,"share_id":"share-443","share_seq":443,"template_height":800013},{"accepted_at_ms":70501,"credit_policy":"stale-grace","job_id":"job-444","job_issued_at_ms":70501,"miner_id":"miner-4","network_difficulty":1000002,"ntime":1700000444,"order_key":"04:miner-4","p2mr_program_hex":"bcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbc","share_difficulty":1,"share_id":"share-444","share_seq":444,"template_height":800013},{"accepted_at_ms":70481,"credit_policy":null,"job_id":"job-445","job_issued_at_ms":70500,"miner_id":"miner-5","network_difficulty":1000003,"ntime":1700000445,"order_key":"05:miner-5","p2mr_program_hex":"bdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbd","share_difficulty":19,"share_id":"share-445","share_seq":445,"template_height":800013},{"accepted_at_ms":70490,"credit_policy":"stale-grace","job_id":"job-446","job_issued_at_ms":70482,"miner_id":"miner-6","network_difficulty":1000004,"ntime":1700000446,"order_key":"06:miner-6","p2mr_program_hex":"bebebebebebebebebebebebebebebebebebebebebebebebebebebebebebebebe","share_difficulty":2,"share_id":"share-446","share_seq":446,"template_height":800013},{"accepted_at_ms":70481,"credit_policy":null,"job_id":"job-447","job_issued_at_ms":70487,"miner_id":"miner-7","network_difficulty":1000005,"ntime":1700000447,"order_key":"07:miner-7","p2mr_program_hex":"bfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbf","share_difficulty":1,"share_id":"share-447","share_seq":447,"template_height":800013},{"accepted_at_ms":70498,"credit_policy":"stale-grace","job_id":"job-448","job_issued_at_ms":70510,"miner_id":"miner-8","network_difficulty":1000006,"ntime":1700000448,"order_key":"08:miner-8","p2mr_program_hex":"c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0","share_difficulty":7,"share_id":"share-448","share_seq":448,"template_height":800014},{"accepted_at_ms":70484,"credit_policy":"stale-grace","job_id":"job-449","job_issued_at_ms":70504,"miner_id":"miner-9","network_difficulty":1000007,"ntime":1700000449,"order_key":"09:miner-9","p2mr_program_hex":"c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1","share_difficulty":7,"share_id":"share-449","share_seq":449,"template_height":800014},{"accepted_at_ms":70496,"credit_policy":"stale-grace","job_id":"job-450","job_issued_at_ms":70482,"miner_id":"miner-10","network_difficulty":1000008,"ntime":1700000450,"order_key":"10:miner-10","p2mr_program_hex":"c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2","share_difficulty":3,"share_id":"share-450","share_seq":450,"template_height":800014},{"accepted_at_ms":70500,"credit_policy":"stale-grace","job_id":"job-451","job_issued_at_ms":70492,"miner_id":"miner-0","network_difficulty":1000009,"ntime":1700000451,"order_key":"00:miner-0","p2mr_program_hex":"c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3","share_difficulty":7,"share_id":"share-451","share_seq":451,"template_height":800014},{"accepted_at_ms":70489,"credit_policy":null,"job_id":"job-452","job_issued_at_ms":70481,"miner_id":"miner-1","network_difficulty":1000010,"ntime":1700000452,"order_key":"01:miner-1","p2mr_program_hex":"c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4","share_difficulty":19,"share_id":"share-452","share_seq":452,"template_height":800014},{"accepted_at_ms":70486,"credit_policy":null,"job_id":"job-453","job_issued_at_ms":70506,"miner_id":"miner-2","network_difficulty":1000011,"ntime":1700000453,"order_key":"02:miner-2","p2mr_program_hex":"c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5","share_difficulty":1,"share_id":"share-453","share_seq":453,"template_height":800014},{"accepted_at_ms":70495,"credit_policy":null,"job_id":"job-454","job_issued_at_ms":70486,"miner_id":"miner-3","network_difficulty":1000012,"ntime":1700000454,"order_key":"03:miner-3","p2mr_program_hex":"c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6","share_difficulty":3,"share_id":"share-454","share_seq":454,"template_height":800014},{"accepted_at_ms":70494,"credit_policy":null,"job_id":"job-455","job_issued_at_ms":70492,"miner_id":"miner-4","network_difficulty":1000013,"ntime":1700000455,"order_key":"04:miner-4","p2mr_program_hex":"c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7","share_difficulty":7,"share_id":"share-455","share_seq":455,"template_height":800014},{"accepted_at_ms":70511,"credit_policy":null,"job_id":"job-456","job_issued_at_ms":70482,"miner_id":"miner-5","network_difficulty":1000014,"ntime":1700000456,"order_key":"05:miner-5","p2mr_program_hex":"c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8","share_difficulty":7,"share_id":"share-456","share_seq":456,"template_height":800014},{"accepted_at_ms":70511,"credit_policy":null,"job_id":"job-457","job_issued_at_ms":70496,"miner_id":"miner-6","network_difficulty":1000015,"ntime":1700000457,"order_key":"06:miner-6","p2mr_program_hex":"c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9","share_difficulty":3,"share_id":"share-457","share_seq":457,"template_height":800014},{"accepted_at_ms":70494,"credit_policy":null,"job_id":"job-458","job_issued_at_ms":70488,"miner_id":"miner-7","network_difficulty":1000016,"ntime":1700000458,"order_key":"07:miner-7","p2mr_program_hex":"cacacacacacacacacacacacacacacacacacacacacacacacacacacacacacacaca","share_difficulty":1,"share_id":"share-458","share_seq":458,"template_height":800014},{"accepted_at_ms":70511,"credit_policy":null,"job_id":"job-459","job_issued_at_ms":70496,"miner_id":"miner-8","network_difficulty":1000000,"ntime":1700000459,"order_key":"08:miner-8","p2mr_program_hex":"cbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcb","share_difficulty":7,"share_id":"share-459","share_seq":459,"template_height":800014},{"accepted_at_ms":70504,"credit_policy":"stale-grace","job_id":"job-460","job_issued_at_ms":70486,"miner_id":"miner-9","network_difficulty":1000001,"ntime":1700000460,"order_key":"09:miner-9","p2mr_program_hex":"cccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccc","share_difficulty":3,"share_id":"share-460","share_seq":460,"template_height":800014},{"accepted_at_ms":70502,"credit_policy":null,"job_id":"job-461","job_issued_at_ms":70508,"miner_id":"miner-10","network_difficulty":1000002,"ntime":1700000461,"order_key":"10:miner-10","p2mr_program_hex":"cdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcd","share_difficulty":1,"share_id":"share-461","share_seq":461,"template_height":800014},{"accepted_at_ms":70507,"credit_policy":null,"job_id":"job-462","job_issued_at_ms":70483,"miner_id":"miner-0","network_difficulty":1000003,"ntime":1700000462,"order_key":"00:miner-0","p2mr_program_hex":"cececececececececececececececececececececececececececececececece","share_difficulty":7,"share_id":"share-462","share_seq":462,"template_height":800014},{"accepted_at_ms":70503,"credit_policy":null,"job_id":"job-463","job_issued_at_ms":70493,"miner_id":"miner-1","network_difficulty":1000004,"ntime":1700000463,"order_key":"01:miner-1","p2mr_program_hex":"cfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcf","share_difficulty":1,"share_id":"share-463","share_seq":463,"template_height":800014},{"accepted_at_ms":70494,"credit_policy":null,"job_id":"job-464","job_issued_at_ms":70503,"miner_id":"miner-2","network_difficulty":1000005,"ntime":1700000464,"order_key":"02:miner-2","p2mr_program_hex":"d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0","share_difficulty":3,"share_id":"share-464","share_seq":464,"template_height":800014},{"accepted_at_ms":70507,"credit_policy":null,"job_id":"job-465","job_issued_at_ms":70491,"miner_id":"miner-3","network_difficulty":1000006,"ntime":1700000465,"order_key":"03:miner-3","p2mr_program_hex":"d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1","share_difficulty":19,"share_id":"share-465","share_seq":465,"template_height":800014},{"accepted_at_ms":70488,"credit_policy":null,"job_id":"job-466","job_issued_at_ms":70507,"miner_id":"miner-4","network_difficulty":1000007,"ntime":1700000466,"order_key":"04:miner-4","p2mr_program_hex":"d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2","share_difficulty":3,"share_id":"share-466","share_seq":466,"template_height":800014},{"accepted_at_ms":70500,"credit_policy":null,"job_id":"job-467","job_issued_at_ms":70496,"miner_id":"miner-5","network_difficulty":1000008,"ntime":1700000467,"order_key":"05:miner-5","p2mr_program_hex":"d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3","share_difficulty":3,"share_id":"share-467","share_seq":467,"template_height":800014},{"accepted_at_ms":70496,"credit_policy":null,"job_id":"job-468","job_issued_at_ms":70484,"miner_id":"miner-6","network_difficulty":1000009,"ntime":1700000468,"order_key":"06:miner-6","p2mr_program_hex":"d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4","share_difficulty":7,"share_id":"share-468","share_seq":468,"template_height":800014},{"accepted_at_ms":70492,"credit_policy":null,"job_id":"job-469","job_issued_at_ms":70489,"miner_id":"miner-7","network_difficulty":1000010,"ntime":1700000469,"order_key":"07:miner-7","p2mr_program_hex":"d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5","share_difficulty":7,"share_id":"share-469","share_seq":469,"template_height":800014},{"accepted_at_ms":70499,"credit_policy":null,"job_id":"job-470","job_issued_at_ms":70493,"miner_id":"miner-8","network_difficulty":1000011,"ntime":1700000470,"order_key":"08:miner-8","p2mr_program_hex":"d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6","share_difficulty":7,"share_id":"share-470","share_seq":470,"template_height":800014},{"accepted_at_ms":70507,"credit_policy":null,"job_id":"job-471","job_issued_at_ms":70491,"miner_id":"miner-9","network_difficulty":1000012,"ntime":1700000471,"order_key":"09:miner-9","p2mr_program_hex":"d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7","share_difficulty":7,"share_id":"share-471","share_seq":471,"template_height":800014},{"accepted_at_ms":70482,"credit_policy":null,"job_id":"job-472","job_issued_at_ms":70485,"miner_id":"miner-10","network_difficulty":1000013,"ntime":1700000472,"order_key":"10:miner-10","p2mr_program_hex":"d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8","share_difficulty":3,"share_id":"share-472","share_seq":472,"template_height":800014},{"accepted_at_ms":70485,"credit_policy":null,"job_id":"job-473","job_issued_at_ms":70487,"miner_id":"miner-0","network_difficulty":1000014,"ntime":1700000473,"order_key":"00:miner-0","p2mr_program_hex":"d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9","share_difficulty":2,"share_id":"share-473","share_seq":473,"template_height":800014},{"accepted_at_ms":70509,"credit_policy":null,"job_id":"job-474","job_issued_at_ms":70505,"miner_id":"miner-1","network_difficulty":1000015,"ntime":1700000474,"order_key":"01:miner-1","p2mr_program_hex":"dadadadadadadadadadadadadadadadadadadadadadadadadadadadadadadada","share_difficulty":19,"share_id":"share-474","share_seq":474,"template_height":800014},{"accepted_at_ms":70505,"credit_policy":null,"job_id":"job-475","job_issued_at_ms":70509,"miner_id":"miner-2","network_difficulty":1000016,"ntime":1700000475,"order_key":"02:miner-2","p2mr_program_hex":"dbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdb","share_difficulty":2,"share_id":"share-475","share_seq":475,"template_height":800014},{"accepted_at_ms":70488,"credit_policy":"stale-grace","job_id":"job-476","job_issued_at_ms":70487,"miner_id":"miner-3","network_difficulty":1000000,"ntime":1700000476,"order_key":"03:miner-3","p2mr_program_hex":"dcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdc","share_difficulty":1,"share_id":"share-476","share_seq":476,"template_height":800014},{"accepted_at_ms":70498,"credit_policy":null,"job_id":"job-477","job_issued_at_ms":70507,"miner_id":"miner-4","network_difficulty":1000001,"ntime":1700000477,"order_key":"04:miner-4","p2mr_program_hex":"dddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddd","share_difficulty":7,"share_id":"share-477","share_seq":477,"template_height":800014},{"accepted_at_ms":70500,"credit_policy":null,"job_id":"job-478","job_issued_at_ms":70481,"miner_id":"miner-5","network_difficulty":1000002,"ntime":1700000478,"order_key":"05:miner-5","p2mr_program_hex":"dededededededededededededededededededededededededededededededede","share_difficulty":1,"share_id":"share-478","share_seq":478,"template_height":800014},{"accepted_at_ms":70488,"credit_policy":null,"job_id":"job-479","job_issued_at_ms":70493,"miner_id":"miner-6","network_difficulty":1000003,"ntime":1700000479,"order_key":"06:miner-6","p2mr_program_hex":"dfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdf","share_difficulty":3,"share_id":"share-479","share_seq":479,"template_height":800014},{"accepted_at_ms":70494,"credit_policy":null,"job_id":"job-480","job_issued_at_ms":70499,"miner_id":"miner-7","network_difficulty":1000004,"ntime":1700000480,"order_key":"07:miner-7","p2mr_program_hex":"e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0","share_difficulty":3,"share_id":"share-480","share_seq":480,"template_height":800015},{"accepted_at_ms":70511,"credit_policy":null,"job_id":"job-481","job_issued_at_ms":70502,"miner_id":"miner-8","network_difficulty":1000005,"ntime":1700000481,"order_key":"08:miner-8","p2mr_program_hex":"e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1","share_difficulty":3,"share_id":"share-481","share_seq":481,"template_height":800015},{"accepted_at_ms":70503,"credit_policy":null,"job_id":"job-482","job_issued_at_ms":70506,"miner_id":"miner-9","network_difficulty":1000006,"ntime":1700000482,"order_key":"09:miner-9","p2mr_program_hex":"e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2","share_difficulty":7,"share_id":"share-482","share_seq":482,"template_height":800015},{"accepted_at_ms":70499,"credit_policy":"stale-grace","job_id":"job-483","job_issued_at_ms":70503,"miner_id":"miner-10","network_difficulty":1000007,"ntime":1700000483,"order_key":"10:miner-10","p2mr_program_hex":"e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3","share_difficulty":2,"share_id":"share-483","share_seq":483,"template_height":800015},{"accepted_at_ms":70502,"credit_policy":null,"job_id":"job-484","job_issued_at_ms":70506,"miner_id":"miner-0","network_difficulty":1000008,"ntime":1700000484,"order_key":"00:miner-0","p2mr_program_hex":"e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4","share_difficulty":2,"share_id":"share-484","share_seq":484,"template_height":800015},{"accepted_at_ms":70502,"credit_policy":"stale-grace","job_id":"job-485","job_issued_at_ms":70491,"miner_id":"miner-1","network_difficulty":1000009,"ntime":1700000485,"order_key":"01:miner-1","p2mr_program_hex":"e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5","share_difficulty":1,"share_id":"share-485","share_seq":485,"template_height":800015},{"accepted_at_ms":70488,"credit_policy":null,"job_id":"job-486","job_issued_at_ms":70482,"miner_id":"miner-2","network_difficulty":1000010,"ntime":1700000486,"order_key":"02:miner-2","p2mr_program_hex":"e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6","share_difficulty":1,"share_id":"share-486","share_seq":486,"template_height":800015},{"accepted_at_ms":70491,"credit_policy":null,"job_id":"job-487","job_issued_at_ms":70485,"miner_id":"miner-3","network_difficulty":1000011,"ntime":1700000487,"order_key":"03:miner-3","p2mr_program_hex":"e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7","share_difficulty":7,"share_id":"share-487","share_seq":487,"template_height":800015},{"accepted_at_ms":70486,"credit_policy":null,"job_id":"job-488","job_issued_at_ms":70508,"miner_id":"miner-4","network_difficulty":1000012,"ntime":1700000488,"order_key":"04:miner-4","p2mr_program_hex":"e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8","share_difficulty":7,"share_id":"share-488","share_seq":488,"template_height":800015},{"accepted_at_ms":70497,"credit_policy":null,"job_id":"job-489","job_issued_at_ms":70502,"miner_id":"miner-5","network_difficulty":1000013,"ntime":1700000489,"order_key":"05:miner-5","p2mr_program_hex":"e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9","share_difficulty":7,"share_id":"share-489","share_seq":489,"template_height":800015},{"accepted_at_ms":70482,"credit_policy":null,"job_id":"job-490","job_issued_at_ms":70489,"miner_id":"miner-6","network_difficulty":1000014,"ntime":1700000490,"order_key":"06:miner-6","p2mr_program_hex":"eaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaea","share_difficulty":7,"share_id":"share-490","share_seq":490,"template_height":800015},{"accepted_at_ms":70483,"credit_policy":null,"job_id":"job-491","job_issued_at_ms":70489,"miner_id":"miner-7","network_difficulty":1000015,"ntime":1700000491,"order_key":"07:miner-7","p2mr_program_hex":"ebebebebebebebebebebebebebebebebebebebebebebebebebebebebebebebeb","share_difficulty":1,"share_id":"share-491","share_seq":491,"template_height":800015},{"accepted_at_ms":70484,"credit_policy":null,"job_id":"job-492","job_issued_at_ms":70510,"miner_id":"miner-8","network_difficulty":1000016,"ntime":1700000492,"order_key":"08:miner-8","p2mr_program_hex":"ecececececececececececececececececececececececececececececececec","share_difficulty":2,"share_id":"share-492","share_seq":492,"template_height":800015},{"accepted_at_ms":70511,"credit_policy":null,"job_id":"job-493","job_issued_at_ms":70511,"miner_id":"miner-9","network_difficulty":1000000,"ntime":1700000493,"order_key":"09:miner-9","p2mr_program_hex":"edededededededededededededededededededededededededededededededed","share_difficulty":7,"share_id":"share-493","share_seq":493,"template_height":800015},{"accepted_at_ms":70491,"credit_policy":null,"job_id":"job-494","job_issued_at_ms":70504,"miner_id":"miner-10","network_difficulty":1000001,"ntime":1700000494,"order_key":"10:miner-10","p2mr_program_hex":"eeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeee","share_difficulty":2,"share_id":"share-494","share_seq":494,"template_height":800015},{"accepted_at_ms":70487,"credit_policy":null,"job_id":"job-495","job_issued_at_ms":70493,"miner_id":"miner-0","network_difficulty":1000002,"ntime":1700000495,"order_key":"00:miner-0","p2mr_program_hex":"efefefefefefefefefefefefefefefefefefefefefefefefefefefefefefefef","share_difficulty":19,"share_id":"share-495","share_seq":495,"template_height":800015},{"accepted_at_ms":70503,"credit_policy":"stale-grace","job_id":"job-496","job_issued_at_ms":70487,"miner_id":"miner-1","network_difficulty":1000003,"ntime":1700000496,"order_key":"01:miner-1","p2mr_program_hex":"f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0","share_difficulty":2,"share_id":"share-496","share_seq":496,"template_height":800015},{"accepted_at_ms":70506,"credit_policy":null,"job_id":"job-497","job_issued_at_ms":70498,"miner_id":"miner-2","network_difficulty":1000004,"ntime":1700000497,"order_key":"02:miner-2","p2mr_program_hex":"f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1","share_difficulty":1,"share_id":"share-497","share_seq":497,"template_height":800015},{"accepted_at_ms":70507,"credit_policy":null,"job_id":"job-498","job_issued_at_ms":70489,"miner_id":"miner-3","network_difficulty":1000005,"ntime":1700000498,"order_key":"03:miner-3","p2mr_program_hex":"f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2","share_difficulty":1,"share_id":"share-498","share_seq":498,"template_height":800015},{"accepted_at_ms":70490,"credit_policy":null,"job_id":"job-499","job_issued_at_ms":70506,"miner_id":"miner-4","network_difficulty":1000006,"ntime":1700000499,"order_key":"04:miner-4","p2mr_program_hex":"f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3","share_difficulty":2,"share_id":"share-499","share_seq":499,"template_height":800015},{"accepted_at_ms":70482,"credit_policy":null,"job_id":"job-500","job_issued_at_ms":70511,"miner_id":"miner-5","network_difficulty":1000007,"ntime":1700000500,"order_key":"05:miner-5","p2mr_program_hex":"f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4","share_difficulty":19,"share_id":"share-500","share_seq":500,"template_height":800015},{"accepted_at_ms":70484,"credit_policy":null,"job_id":"job-501","job_issued_at_ms":70482,"miner_id":"miner-6","network_difficulty":1000008,"ntime":1700000501,"order_key":"06:miner-6","p2mr_program_hex":"f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5","share_difficulty":2,"share_id":"share-501","share_seq":501,"template_height":800015},{"accepted_at_ms":70495,"credit_policy":"stale-grace","job_id":"job-502","job_issued_at_ms":70506,"miner_id":"miner-7","network_difficulty":1000009,"ntime":1700000502,"order_key":"07:miner-7","p2mr_program_hex":"f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6","share_difficulty":2,"share_id":"share-502","share_seq":502,"template_height":800015},{"accepted_at_ms":70481,"credit_policy":null,"job_id":"job-503","job_issued_at_ms":70498,"miner_id":"miner-8","network_difficulty":1000010,"ntime":1700000503,"order_key":"08:miner-8","p2mr_program_hex":"f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7","share_difficulty":2,"share_id":"share-503","share_seq":503,"template_height":800015},{"accepted_at_ms":70492,"credit_policy":"stale-grace","job_id":"job-504","job_issued_at_ms":70490,"miner_id":"miner-9","network_difficulty":1000011,"ntime":1700000504,"order_key":"09:miner-9","p2mr_program_hex":"f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8","share_difficulty":3,"share_id":"share-504","share_seq":504,"template_height":800015},{"accepted_at_ms":70509,"credit_policy":"stale-grace","job_id":"job-505","job_issued_at_ms":70496,"miner_id":"miner-10","network_difficulty":1000012,"ntime":1700000505,"order_key":"10:miner-10","p2mr_program_hex":"f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9","share_difficulty":3,"share_id":"share-505","share_seq":505,"template_height":800015},{"accepted_at_ms":70498,"credit_policy":null,"job_id":"job-506","job_issued_at_ms":70481,"miner_id":"miner-0","network_difficulty":1000013,"ntime":1700000506,"order_key":"00:miner-0","p2mr_program_hex":"fafafafafafafafafafafafafafafafafafafafafafafafafafafafafafafafa","share_difficulty":7,"share_id":"share-506","share_seq":506,"template_height":800015},{"accepted_at_ms":70500,"credit_policy":"stale-grace","job_id":"job-507","job_issued_at_ms":70501,"miner_id":"miner-1","network_difficulty":1000014,"ntime":1700000507,"order_key":"01:miner-1","p2mr_program_hex":"fbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfb","share_difficulty":19,"share_id":"share-507","share_seq":507,"template_height":800015},{"accepted_at_ms":70496,"credit_policy":null,"job_id":"job-508","job_issued_at_ms":70495,"miner_id":"miner-2","network_difficulty":1000015,"ntime":1700000508,"order_key":"02:miner-2","p2mr_program_hex":"fcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfc","share_difficulty":1,"share_id":"share-508","share_seq":508,"template_height":800015},{"accepted_at_ms":70504,"credit_policy":null,"job_id":"job-509","job_issued_at_ms":70483,"miner_id":"miner-3","network_difficulty":1000016,"ntime":1700000509,"order_key":"03:miner-3","p2mr_program_hex":"fdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfd","share_difficulty":1,"share_id":"share-509","share_seq":509,"template_height":800015},{"accepted_at_ms":70497,"credit_policy":null,"job_id":"job-510","job_issued_at_ms":70510,"miner_id":"miner-4","network_difficulty":1000000,"ntime":1700000510,"order_key":"04:miner-4","p2mr_program_hex":"fefefefefefefefefefefefefefefefefefefefefefefefefefefefefefefefe","share_difficulty":19,"share_id":"share-510","share_seq":510,"template_height":800015},{"accepted_at_ms":70481,"credit_policy":"stale-grace","job_id":"job-511","job_issued_at_ms":70504,"miner_id":"miner-5","network_difficulty":1000001,"ntime":1700000511,"order_key":"05:miner-5","p2mr_program_hex":"ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff","share_difficulty":3,"share_id":"share-511","share_seq":511,"template_height":800015}]},"window_weight":1000000000},"page-boundary-512":{"advances":[],"name":"page-boundary-512","page_size":512,"schema":"qbit-prism-window-pipeline-parity-input/v1","snapshot":{"anchor_job_issued_at_ms":70512,"records":[{"accepted_at_ms":70511,"credit_policy":null,"job_id":"job-1","job_issued_at_ms":70488,"miner_id":"miner-1","network_difficulty":1000001,"ntime":1700000001,"order_key":"01:miner-1","p2mr_program_hex":"0101010101010101010101010101010101010101010101010101010101010101","share_difficulty":2,"share_id":"share-1","share_seq":1,"template_height":800000},{"accepted_at_ms":70493,"credit_policy":null,"job_id":"job-2","job_issued_at_ms":70494,"miner_id":"miner-2","network_difficulty":1000002,"ntime":1700000002,"order_key":"02:miner-2","p2mr_program_hex":"0202020202020202020202020202020202020202020202020202020202020202","share_difficulty":3,"share_id":"share-2","share_seq":2,"template_height":800000},{"accepted_at_ms":70503,"credit_policy":null,"job_id":"job-3","job_issued_at_ms":70485,"miner_id":"miner-3","network_difficulty":1000003,"ntime":1700000003,"order_key":"03:miner-3","p2mr_program_hex":"0303030303030303030303030303030303030303030303030303030303030303","share_difficulty":1,"share_id":"share-3","share_seq":3,"template_height":800000},{"accepted_at_ms":70502,"credit_policy":null,"job_id":"job-4","job_issued_at_ms":70497,"miner_id":"miner-4","network_difficulty":1000004,"ntime":1700000004,"order_key":"04:miner-4","p2mr_program_hex":"0404040404040404040404040404040404040404040404040404040404040404","share_difficulty":2,"share_id":"share-4","share_seq":4,"template_height":800000},{"accepted_at_ms":70492,"credit_policy":null,"job_id":"job-5","job_issued_at_ms":70505,"miner_id":"miner-5","network_difficulty":1000005,"ntime":1700000005,"order_key":"05:miner-5","p2mr_program_hex":"0505050505050505050505050505050505050505050505050505050505050505","share_difficulty":1,"share_id":"share-5","share_seq":5,"template_height":800000},{"accepted_at_ms":70499,"credit_policy":null,"job_id":"job-6","job_issued_at_ms":70508,"miner_id":"miner-6","network_difficulty":1000006,"ntime":1700000006,"order_key":"06:miner-6","p2mr_program_hex":"0606060606060606060606060606060606060606060606060606060606060606","share_difficulty":1,"share_id":"share-6","share_seq":6,"template_height":800000},{"accepted_at_ms":70501,"credit_policy":null,"job_id":"job-7","job_issued_at_ms":70485,"miner_id":"miner-7","network_difficulty":1000007,"ntime":1700000007,"order_key":"07:miner-7","p2mr_program_hex":"0707070707070707070707070707070707070707070707070707070707070707","share_difficulty":2,"share_id":"share-7","share_seq":7,"template_height":800000},{"accepted_at_ms":70511,"credit_policy":null,"job_id":"job-8","job_issued_at_ms":70498,"miner_id":"miner-8","network_difficulty":1000008,"ntime":1700000008,"order_key":"08:miner-8","p2mr_program_hex":"0808080808080808080808080808080808080808080808080808080808080808","share_difficulty":7,"share_id":"share-8","share_seq":8,"template_height":800000},{"accepted_at_ms":70483,"credit_policy":null,"job_id":"job-9","job_issued_at_ms":70504,"miner_id":"miner-9","network_difficulty":1000009,"ntime":1700000009,"order_key":"09:miner-9","p2mr_program_hex":"0909090909090909090909090909090909090909090909090909090909090909","share_difficulty":3,"share_id":"share-9","share_seq":9,"template_height":800000},{"accepted_at_ms":70489,"credit_policy":"stale-grace","job_id":"job-10","job_issued_at_ms":70483,"miner_id":"miner-10","network_difficulty":1000010,"ntime":1700000010,"order_key":"10:miner-10","p2mr_program_hex":"0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a","share_difficulty":2,"share_id":"share-10","share_seq":10,"template_height":800000},{"accepted_at_ms":70504,"credit_policy":null,"job_id":"job-11","job_issued_at_ms":70485,"miner_id":"miner-0","network_difficulty":1000011,"ntime":1700000011,"order_key":"00:miner-0","p2mr_program_hex":"0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b","share_difficulty":1,"share_id":"share-11","share_seq":11,"template_height":800000},{"accepted_at_ms":70482,"credit_policy":null,"job_id":"job-12","job_issued_at_ms":70506,"miner_id":"miner-1","network_difficulty":1000012,"ntime":1700000012,"order_key":"01:miner-1","p2mr_program_hex":"0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c","share_difficulty":2,"share_id":"share-12","share_seq":12,"template_height":800000},{"accepted_at_ms":70494,"credit_policy":null,"job_id":"job-13","job_issued_at_ms":70495,"miner_id":"miner-2","network_difficulty":1000013,"ntime":1700000013,"order_key":"02:miner-2","p2mr_program_hex":"0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d","share_difficulty":2,"share_id":"share-13","share_seq":13,"template_height":800000},{"accepted_at_ms":70491,"credit_policy":"stale-grace","job_id":"job-14","job_issued_at_ms":70491,"miner_id":"miner-3","network_difficulty":1000014,"ntime":1700000014,"order_key":"03:miner-3","p2mr_program_hex":"0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e","share_difficulty":1,"share_id":"share-14","share_seq":14,"template_height":800000},{"accepted_at_ms":70487,"credit_policy":null,"job_id":"job-15","job_issued_at_ms":70497,"miner_id":"miner-4","network_difficulty":1000015,"ntime":1700000015,"order_key":"04:miner-4","p2mr_program_hex":"0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f","share_difficulty":19,"share_id":"share-15","share_seq":15,"template_height":800000},{"accepted_at_ms":70501,"credit_policy":null,"job_id":"job-16","job_issued_at_ms":70499,"miner_id":"miner-5","network_difficulty":1000016,"ntime":1700000016,"order_key":"05:miner-5","p2mr_program_hex":"1010101010101010101010101010101010101010101010101010101010101010","share_difficulty":7,"share_id":"share-16","share_seq":16,"template_height":800000},{"accepted_at_ms":70494,"credit_policy":"stale-grace","job_id":"job-17","job_issued_at_ms":70509,"miner_id":"miner-6","network_difficulty":1000000,"ntime":1700000017,"order_key":"06:miner-6","p2mr_program_hex":"1111111111111111111111111111111111111111111111111111111111111111","share_difficulty":7,"share_id":"share-17","share_seq":17,"template_height":800000},{"accepted_at_ms":70482,"credit_policy":null,"job_id":"job-18","job_issued_at_ms":70490,"miner_id":"miner-7","network_difficulty":1000001,"ntime":1700000018,"order_key":"07:miner-7","p2mr_program_hex":"1212121212121212121212121212121212121212121212121212121212121212","share_difficulty":19,"share_id":"share-18","share_seq":18,"template_height":800000},{"accepted_at_ms":70507,"credit_policy":null,"job_id":"job-19","job_issued_at_ms":70488,"miner_id":"miner-8","network_difficulty":1000002,"ntime":1700000019,"order_key":"08:miner-8","p2mr_program_hex":"1313131313131313131313131313131313131313131313131313131313131313","share_difficulty":2,"share_id":"share-19","share_seq":19,"template_height":800000},{"accepted_at_ms":70491,"credit_policy":null,"job_id":"job-20","job_issued_at_ms":70482,"miner_id":"miner-9","network_difficulty":1000003,"ntime":1700000020,"order_key":"09:miner-9","p2mr_program_hex":"1414141414141414141414141414141414141414141414141414141414141414","share_difficulty":19,"share_id":"share-20","share_seq":20,"template_height":800000},{"accepted_at_ms":70482,"credit_policy":null,"job_id":"job-21","job_issued_at_ms":70498,"miner_id":"miner-10","network_difficulty":1000004,"ntime":1700000021,"order_key":"10:miner-10","p2mr_program_hex":"1515151515151515151515151515151515151515151515151515151515151515","share_difficulty":2,"share_id":"share-21","share_seq":21,"template_height":800000},{"accepted_at_ms":70504,"credit_policy":null,"job_id":"job-22","job_issued_at_ms":70508,"miner_id":"miner-0","network_difficulty":1000005,"ntime":1700000022,"order_key":"00:miner-0","p2mr_program_hex":"1616161616161616161616161616161616161616161616161616161616161616","share_difficulty":19,"share_id":"share-22","share_seq":22,"template_height":800000},{"accepted_at_ms":70497,"credit_policy":null,"job_id":"job-23","job_issued_at_ms":70498,"miner_id":"miner-1","network_difficulty":1000006,"ntime":1700000023,"order_key":"01:miner-1","p2mr_program_hex":"1717171717171717171717171717171717171717171717171717171717171717","share_difficulty":7,"share_id":"share-23","share_seq":23,"template_height":800000},{"accepted_at_ms":70507,"credit_policy":null,"job_id":"job-24","job_issued_at_ms":70494,"miner_id":"miner-2","network_difficulty":1000007,"ntime":1700000024,"order_key":"02:miner-2","p2mr_program_hex":"1818181818181818181818181818181818181818181818181818181818181818","share_difficulty":19,"share_id":"share-24","share_seq":24,"template_height":800000},{"accepted_at_ms":70484,"credit_policy":"stale-grace","job_id":"job-25","job_issued_at_ms":70488,"miner_id":"miner-3","network_difficulty":1000008,"ntime":1700000025,"order_key":"03:miner-3","p2mr_program_hex":"1919191919191919191919191919191919191919191919191919191919191919","share_difficulty":2,"share_id":"share-25","share_seq":25,"template_height":800000},{"accepted_at_ms":70488,"credit_policy":null,"job_id":"job-26","job_issued_at_ms":70506,"miner_id":"miner-4","network_difficulty":1000009,"ntime":1700000026,"order_key":"04:miner-4","p2mr_program_hex":"1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a","share_difficulty":2,"share_id":"share-26","share_seq":26,"template_height":800000},{"accepted_at_ms":70491,"credit_policy":"stale-grace","job_id":"job-27","job_issued_at_ms":70483,"miner_id":"miner-5","network_difficulty":1000010,"ntime":1700000027,"order_key":"05:miner-5","p2mr_program_hex":"1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b","share_difficulty":1,"share_id":"share-27","share_seq":27,"template_height":800000},{"accepted_at_ms":70499,"credit_policy":null,"job_id":"job-28","job_issued_at_ms":70494,"miner_id":"miner-6","network_difficulty":1000011,"ntime":1700000028,"order_key":"06:miner-6","p2mr_program_hex":"1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c","share_difficulty":19,"share_id":"share-28","share_seq":28,"template_height":800000},{"accepted_at_ms":70492,"credit_policy":"stale-grace","job_id":"job-29","job_issued_at_ms":70507,"miner_id":"miner-7","network_difficulty":1000012,"ntime":1700000029,"order_key":"07:miner-7","p2mr_program_hex":"1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d","share_difficulty":19,"share_id":"share-29","share_seq":29,"template_height":800000},{"accepted_at_ms":70496,"credit_policy":null,"job_id":"job-30","job_issued_at_ms":70484,"miner_id":"miner-8","network_difficulty":1000013,"ntime":1700000030,"order_key":"08:miner-8","p2mr_program_hex":"1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e","share_difficulty":7,"share_id":"share-30","share_seq":30,"template_height":800000},{"accepted_at_ms":70489,"credit_policy":null,"job_id":"job-31","job_issued_at_ms":70507,"miner_id":"miner-9","network_difficulty":1000014,"ntime":1700000031,"order_key":"09:miner-9","p2mr_program_hex":"1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f","share_difficulty":1,"share_id":"share-31","share_seq":31,"template_height":800000},{"accepted_at_ms":70495,"credit_policy":null,"job_id":"job-32","job_issued_at_ms":70484,"miner_id":"miner-10","network_difficulty":1000015,"ntime":1700000032,"order_key":"10:miner-10","p2mr_program_hex":"2020202020202020202020202020202020202020202020202020202020202020","share_difficulty":1,"share_id":"share-32","share_seq":32,"template_height":800001},{"accepted_at_ms":70503,"credit_policy":null,"job_id":"job-33","job_issued_at_ms":70501,"miner_id":"miner-0","network_difficulty":1000016,"ntime":1700000033,"order_key":"00:miner-0","p2mr_program_hex":"2121212121212121212121212121212121212121212121212121212121212121","share_difficulty":7,"share_id":"share-33","share_seq":33,"template_height":800001},{"accepted_at_ms":70512,"credit_policy":null,"job_id":"job-34","job_issued_at_ms":70500,"miner_id":"miner-1","network_difficulty":1000000,"ntime":1700000034,"order_key":"01:miner-1","p2mr_program_hex":"2222222222222222222222222222222222222222222222222222222222222222","share_difficulty":2,"share_id":"share-34","share_seq":34,"template_height":800001},{"accepted_at_ms":70509,"credit_policy":null,"job_id":"job-35","job_issued_at_ms":70491,"miner_id":"miner-2","network_difficulty":1000001,"ntime":1700000035,"order_key":"02:miner-2","p2mr_program_hex":"2323232323232323232323232323232323232323232323232323232323232323","share_difficulty":1,"share_id":"share-35","share_seq":35,"template_height":800001},{"accepted_at_ms":70495,"credit_policy":null,"job_id":"job-36","job_issued_at_ms":70504,"miner_id":"miner-3","network_difficulty":1000002,"ntime":1700000036,"order_key":"03:miner-3","p2mr_program_hex":"2424242424242424242424242424242424242424242424242424242424242424","share_difficulty":3,"share_id":"share-36","share_seq":36,"template_height":800001},{"accepted_at_ms":70509,"credit_policy":null,"job_id":"job-37","job_issued_at_ms":70493,"miner_id":"miner-4","network_difficulty":1000003,"ntime":1700000037,"order_key":"04:miner-4","p2mr_program_hex":"2525252525252525252525252525252525252525252525252525252525252525","share_difficulty":19,"share_id":"share-37","share_seq":37,"template_height":800001},{"accepted_at_ms":70493,"credit_policy":null,"job_id":"job-38","job_issued_at_ms":70495,"miner_id":"miner-5","network_difficulty":1000004,"ntime":1700000038,"order_key":"05:miner-5","p2mr_program_hex":"2626262626262626262626262626262626262626262626262626262626262626","share_difficulty":1,"share_id":"share-38","share_seq":38,"template_height":800001},{"accepted_at_ms":70498,"credit_policy":null,"job_id":"job-39","job_issued_at_ms":70512,"miner_id":"miner-6","network_difficulty":1000005,"ntime":1700000039,"order_key":"06:miner-6","p2mr_program_hex":"2727272727272727272727272727272727272727272727272727272727272727","share_difficulty":1,"share_id":"share-39","share_seq":39,"template_height":800001},{"accepted_at_ms":70483,"credit_policy":null,"job_id":"job-40","job_issued_at_ms":70505,"miner_id":"miner-7","network_difficulty":1000006,"ntime":1700000040,"order_key":"07:miner-7","p2mr_program_hex":"2828282828282828282828282828282828282828282828282828282828282828","share_difficulty":19,"share_id":"share-40","share_seq":40,"template_height":800001},{"accepted_at_ms":70495,"credit_policy":null,"job_id":"job-41","job_issued_at_ms":70487,"miner_id":"miner-8","network_difficulty":1000007,"ntime":1700000041,"order_key":"08:miner-8","p2mr_program_hex":"2929292929292929292929292929292929292929292929292929292929292929","share_difficulty":3,"share_id":"share-41","share_seq":41,"template_height":800001},{"accepted_at_ms":70509,"credit_policy":null,"job_id":"job-42","job_issued_at_ms":70488,"miner_id":"miner-9","network_difficulty":1000008,"ntime":1700000042,"order_key":"09:miner-9","p2mr_program_hex":"2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a","share_difficulty":2,"share_id":"share-42","share_seq":42,"template_height":800001},{"accepted_at_ms":70484,"credit_policy":"stale-grace","job_id":"job-43","job_issued_at_ms":70509,"miner_id":"miner-10","network_difficulty":1000009,"ntime":1700000043,"order_key":"10:miner-10","p2mr_program_hex":"2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b","share_difficulty":3,"share_id":"share-43","share_seq":43,"template_height":800001},{"accepted_at_ms":70498,"credit_policy":null,"job_id":"job-44","job_issued_at_ms":70488,"miner_id":"miner-0","network_difficulty":1000010,"ntime":1700000044,"order_key":"00:miner-0","p2mr_program_hex":"2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c","share_difficulty":7,"share_id":"share-44","share_seq":44,"template_height":800001},{"accepted_at_ms":70507,"credit_policy":null,"job_id":"job-45","job_issued_at_ms":70506,"miner_id":"miner-1","network_difficulty":1000011,"ntime":1700000045,"order_key":"01:miner-1","p2mr_program_hex":"2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d","share_difficulty":2,"share_id":"share-45","share_seq":45,"template_height":800001},{"accepted_at_ms":70494,"credit_policy":null,"job_id":"job-46","job_issued_at_ms":70507,"miner_id":"miner-2","network_difficulty":1000012,"ntime":1700000046,"order_key":"02:miner-2","p2mr_program_hex":"2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e","share_difficulty":19,"share_id":"share-46","share_seq":46,"template_height":800001},{"accepted_at_ms":70494,"credit_policy":"stale-grace","job_id":"job-47","job_issued_at_ms":70494,"miner_id":"miner-3","network_difficulty":1000013,"ntime":1700000047,"order_key":"03:miner-3","p2mr_program_hex":"2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f","share_difficulty":7,"share_id":"share-47","share_seq":47,"template_height":800001},{"accepted_at_ms":70507,"credit_policy":null,"job_id":"job-48","job_issued_at_ms":70493,"miner_id":"miner-4","network_difficulty":1000014,"ntime":1700000048,"order_key":"04:miner-4","p2mr_program_hex":"3030303030303030303030303030303030303030303030303030303030303030","share_difficulty":2,"share_id":"share-48","share_seq":48,"template_height":800001},{"accepted_at_ms":70489,"credit_policy":null,"job_id":"job-49","job_issued_at_ms":70509,"miner_id":"miner-5","network_difficulty":1000015,"ntime":1700000049,"order_key":"05:miner-5","p2mr_program_hex":"3131313131313131313131313131313131313131313131313131313131313131","share_difficulty":2,"share_id":"share-49","share_seq":49,"template_height":800001},{"accepted_at_ms":70487,"credit_policy":"stale-grace","job_id":"job-50","job_issued_at_ms":70512,"miner_id":"miner-6","network_difficulty":1000016,"ntime":1700000050,"order_key":"06:miner-6","p2mr_program_hex":"3232323232323232323232323232323232323232323232323232323232323232","share_difficulty":3,"share_id":"share-50","share_seq":50,"template_height":800001},{"accepted_at_ms":70491,"credit_policy":"stale-grace","job_id":"job-51","job_issued_at_ms":70491,"miner_id":"miner-7","network_difficulty":1000000,"ntime":1700000051,"order_key":"07:miner-7","p2mr_program_hex":"3333333333333333333333333333333333333333333333333333333333333333","share_difficulty":2,"share_id":"share-51","share_seq":51,"template_height":800001},{"accepted_at_ms":70488,"credit_policy":null,"job_id":"job-52","job_issued_at_ms":70485,"miner_id":"miner-8","network_difficulty":1000001,"ntime":1700000052,"order_key":"08:miner-8","p2mr_program_hex":"3434343434343434343434343434343434343434343434343434343434343434","share_difficulty":19,"share_id":"share-52","share_seq":52,"template_height":800001},{"accepted_at_ms":70482,"credit_policy":null,"job_id":"job-53","job_issued_at_ms":70507,"miner_id":"miner-9","network_difficulty":1000002,"ntime":1700000053,"order_key":"09:miner-9","p2mr_program_hex":"3535353535353535353535353535353535353535353535353535353535353535","share_difficulty":1,"share_id":"share-53","share_seq":53,"template_height":800001},{"accepted_at_ms":70486,"credit_policy":null,"job_id":"job-54","job_issued_at_ms":70488,"miner_id":"miner-10","network_difficulty":1000003,"ntime":1700000054,"order_key":"10:miner-10","p2mr_program_hex":"3636363636363636363636363636363636363636363636363636363636363636","share_difficulty":2,"share_id":"share-54","share_seq":54,"template_height":800001},{"accepted_at_ms":70487,"credit_policy":null,"job_id":"job-55","job_issued_at_ms":70506,"miner_id":"miner-0","network_difficulty":1000004,"ntime":1700000055,"order_key":"00:miner-0","p2mr_program_hex":"3737373737373737373737373737373737373737373737373737373737373737","share_difficulty":7,"share_id":"share-55","share_seq":55,"template_height":800001},{"accepted_at_ms":70508,"credit_policy":null,"job_id":"job-56","job_issued_at_ms":70482,"miner_id":"miner-1","network_difficulty":1000005,"ntime":1700000056,"order_key":"01:miner-1","p2mr_program_hex":"3838383838383838383838383838383838383838383838383838383838383838","share_difficulty":2,"share_id":"share-56","share_seq":56,"template_height":800001},{"accepted_at_ms":70492,"credit_policy":null,"job_id":"job-57","job_issued_at_ms":70483,"miner_id":"miner-2","network_difficulty":1000006,"ntime":1700000057,"order_key":"02:miner-2","p2mr_program_hex":"3939393939393939393939393939393939393939393939393939393939393939","share_difficulty":19,"share_id":"share-57","share_seq":57,"template_height":800001},{"accepted_at_ms":70495,"credit_policy":"stale-grace","job_id":"job-58","job_issued_at_ms":70496,"miner_id":"miner-3","network_difficulty":1000007,"ntime":1700000058,"order_key":"03:miner-3","p2mr_program_hex":"3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a","share_difficulty":7,"share_id":"share-58","share_seq":58,"template_height":800001},{"accepted_at_ms":70508,"credit_policy":"stale-grace","job_id":"job-59","job_issued_at_ms":70512,"miner_id":"miner-4","network_difficulty":1000008,"ntime":1700000059,"order_key":"04:miner-4","p2mr_program_hex":"3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b","share_difficulty":3,"share_id":"share-59","share_seq":59,"template_height":800001},{"accepted_at_ms":70512,"credit_policy":null,"job_id":"job-60","job_issued_at_ms":70485,"miner_id":"miner-5","network_difficulty":1000009,"ntime":1700000060,"order_key":"05:miner-5","p2mr_program_hex":"3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c","share_difficulty":19,"share_id":"share-60","share_seq":60,"template_height":800001},{"accepted_at_ms":70484,"credit_policy":null,"job_id":"job-61","job_issued_at_ms":70486,"miner_id":"miner-6","network_difficulty":1000010,"ntime":1700000061,"order_key":"06:miner-6","p2mr_program_hex":"3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d","share_difficulty":7,"share_id":"share-61","share_seq":61,"template_height":800001},{"accepted_at_ms":70508,"credit_policy":"stale-grace","job_id":"job-62","job_issued_at_ms":70501,"miner_id":"miner-7","network_difficulty":1000011,"ntime":1700000062,"order_key":"07:miner-7","p2mr_program_hex":"3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e","share_difficulty":7,"share_id":"share-62","share_seq":62,"template_height":800001},{"accepted_at_ms":70499,"credit_policy":"stale-grace","job_id":"job-63","job_issued_at_ms":70496,"miner_id":"miner-8","network_difficulty":1000012,"ntime":1700000063,"order_key":"08:miner-8","p2mr_program_hex":"3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f","share_difficulty":2,"share_id":"share-63","share_seq":63,"template_height":800001},{"accepted_at_ms":70494,"credit_policy":null,"job_id":"job-64","job_issued_at_ms":70511,"miner_id":"miner-9","network_difficulty":1000013,"ntime":1700000064,"order_key":"09:miner-9","p2mr_program_hex":"4040404040404040404040404040404040404040404040404040404040404040","share_difficulty":19,"share_id":"share-64","share_seq":64,"template_height":800002},{"accepted_at_ms":70482,"credit_policy":null,"job_id":"job-65","job_issued_at_ms":70503,"miner_id":"miner-10","network_difficulty":1000014,"ntime":1700000065,"order_key":"10:miner-10","p2mr_program_hex":"4141414141414141414141414141414141414141414141414141414141414141","share_difficulty":3,"share_id":"share-65","share_seq":65,"template_height":800002},{"accepted_at_ms":70511,"credit_policy":null,"job_id":"job-66","job_issued_at_ms":70503,"miner_id":"miner-0","network_difficulty":1000015,"ntime":1700000066,"order_key":"00:miner-0","p2mr_program_hex":"4242424242424242424242424242424242424242424242424242424242424242","share_difficulty":1,"share_id":"share-66","share_seq":66,"template_height":800002},{"accepted_at_ms":70511,"credit_policy":null,"job_id":"job-67","job_issued_at_ms":70510,"miner_id":"miner-1","network_difficulty":1000016,"ntime":1700000067,"order_key":"01:miner-1","p2mr_program_hex":"4343434343434343434343434343434343434343434343434343434343434343","share_difficulty":3,"share_id":"share-67","share_seq":67,"template_height":800002},{"accepted_at_ms":70491,"credit_policy":null,"job_id":"job-68","job_issued_at_ms":70504,"miner_id":"miner-2","network_difficulty":1000000,"ntime":1700000068,"order_key":"02:miner-2","p2mr_program_hex":"4444444444444444444444444444444444444444444444444444444444444444","share_difficulty":7,"share_id":"share-68","share_seq":68,"template_height":800002},{"accepted_at_ms":70490,"credit_policy":null,"job_id":"job-69","job_issued_at_ms":70505,"miner_id":"miner-3","network_difficulty":1000001,"ntime":1700000069,"order_key":"03:miner-3","p2mr_program_hex":"4545454545454545454545454545454545454545454545454545454545454545","share_difficulty":1,"share_id":"share-69","share_seq":69,"template_height":800002},{"accepted_at_ms":70505,"credit_policy":"stale-grace","job_id":"job-70","job_issued_at_ms":70499,"miner_id":"miner-4","network_difficulty":1000002,"ntime":1700000070,"order_key":"04:miner-4","p2mr_program_hex":"4646464646464646464646464646464646464646464646464646464646464646","share_difficulty":2,"share_id":"share-70","share_seq":70,"template_height":800002},{"accepted_at_ms":70511,"credit_policy":null,"job_id":"job-71","job_issued_at_ms":70504,"miner_id":"miner-5","network_difficulty":1000003,"ntime":1700000071,"order_key":"05:miner-5","p2mr_program_hex":"4747474747474747474747474747474747474747474747474747474747474747","share_difficulty":3,"share_id":"share-71","share_seq":71,"template_height":800002},{"accepted_at_ms":70490,"credit_policy":null,"job_id":"job-72","job_issued_at_ms":70487,"miner_id":"miner-6","network_difficulty":1000004,"ntime":1700000072,"order_key":"06:miner-6","p2mr_program_hex":"4848484848484848484848484848484848484848484848484848484848484848","share_difficulty":2,"share_id":"share-72","share_seq":72,"template_height":800002},{"accepted_at_ms":70492,"credit_policy":null,"job_id":"job-73","job_issued_at_ms":70489,"miner_id":"miner-7","network_difficulty":1000005,"ntime":1700000073,"order_key":"07:miner-7","p2mr_program_hex":"4949494949494949494949494949494949494949494949494949494949494949","share_difficulty":2,"share_id":"share-73","share_seq":73,"template_height":800002},{"accepted_at_ms":70505,"credit_policy":null,"job_id":"job-74","job_issued_at_ms":70492,"miner_id":"miner-8","network_difficulty":1000006,"ntime":1700000074,"order_key":"08:miner-8","p2mr_program_hex":"4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a","share_difficulty":7,"share_id":"share-74","share_seq":74,"template_height":800002},{"accepted_at_ms":70503,"credit_policy":null,"job_id":"job-75","job_issued_at_ms":70511,"miner_id":"miner-9","network_difficulty":1000007,"ntime":1700000075,"order_key":"09:miner-9","p2mr_program_hex":"4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b","share_difficulty":19,"share_id":"share-75","share_seq":75,"template_height":800002},{"accepted_at_ms":70498,"credit_policy":null,"job_id":"job-76","job_issued_at_ms":70495,"miner_id":"miner-10","network_difficulty":1000008,"ntime":1700000076,"order_key":"10:miner-10","p2mr_program_hex":"4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c","share_difficulty":1,"share_id":"share-76","share_seq":76,"template_height":800002},{"accepted_at_ms":70498,"credit_policy":null,"job_id":"job-77","job_issued_at_ms":70489,"miner_id":"miner-0","network_difficulty":1000009,"ntime":1700000077,"order_key":"00:miner-0","p2mr_program_hex":"4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d","share_difficulty":1,"share_id":"share-77","share_seq":77,"template_height":800002},{"accepted_at_ms":70506,"credit_policy":null,"job_id":"job-78","job_issued_at_ms":70497,"miner_id":"miner-1","network_difficulty":1000010,"ntime":1700000078,"order_key":"01:miner-1","p2mr_program_hex":"4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e","share_difficulty":19,"share_id":"share-78","share_seq":78,"template_height":800002},{"accepted_at_ms":70484,"credit_policy":null,"job_id":"job-79","job_issued_at_ms":70512,"miner_id":"miner-2","network_difficulty":1000011,"ntime":1700000079,"order_key":"02:miner-2","p2mr_program_hex":"4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f","share_difficulty":7,"share_id":"share-79","share_seq":79,"template_height":800002},{"accepted_at_ms":70497,"credit_policy":null,"job_id":"job-80","job_issued_at_ms":70512,"miner_id":"miner-3","network_difficulty":1000012,"ntime":1700000080,"order_key":"03:miner-3","p2mr_program_hex":"5050505050505050505050505050505050505050505050505050505050505050","share_difficulty":1,"share_id":"share-80","share_seq":80,"template_height":800002},{"accepted_at_ms":70499,"credit_policy":null,"job_id":"job-81","job_issued_at_ms":70496,"miner_id":"miner-4","network_difficulty":1000013,"ntime":1700000081,"order_key":"04:miner-4","p2mr_program_hex":"5151515151515151515151515151515151515151515151515151515151515151","share_difficulty":2,"share_id":"share-81","share_seq":81,"template_height":800002},{"accepted_at_ms":70512,"credit_policy":null,"job_id":"job-82","job_issued_at_ms":70510,"miner_id":"miner-5","network_difficulty":1000014,"ntime":1700000082,"order_key":"05:miner-5","p2mr_program_hex":"5252525252525252525252525252525252525252525252525252525252525252","share_difficulty":1,"share_id":"share-82","share_seq":82,"template_height":800002},{"accepted_at_ms":70483,"credit_policy":null,"job_id":"job-83","job_issued_at_ms":70512,"miner_id":"miner-6","network_difficulty":1000015,"ntime":1700000083,"order_key":"06:miner-6","p2mr_program_hex":"5353535353535353535353535353535353535353535353535353535353535353","share_difficulty":1,"share_id":"share-83","share_seq":83,"template_height":800002},{"accepted_at_ms":70483,"credit_policy":null,"job_id":"job-84","job_issued_at_ms":70506,"miner_id":"miner-7","network_difficulty":1000016,"ntime":1700000084,"order_key":"07:miner-7","p2mr_program_hex":"5454545454545454545454545454545454545454545454545454545454545454","share_difficulty":3,"share_id":"share-84","share_seq":84,"template_height":800002},{"accepted_at_ms":70508,"credit_policy":null,"job_id":"job-85","job_issued_at_ms":70485,"miner_id":"miner-8","network_difficulty":1000000,"ntime":1700000085,"order_key":"08:miner-8","p2mr_program_hex":"5555555555555555555555555555555555555555555555555555555555555555","share_difficulty":3,"share_id":"share-85","share_seq":85,"template_height":800002},{"accepted_at_ms":70507,"credit_policy":"stale-grace","job_id":"job-86","job_issued_at_ms":70495,"miner_id":"miner-9","network_difficulty":1000001,"ntime":1700000086,"order_key":"09:miner-9","p2mr_program_hex":"5656565656565656565656565656565656565656565656565656565656565656","share_difficulty":7,"share_id":"share-86","share_seq":86,"template_height":800002},{"accepted_at_ms":70483,"credit_policy":null,"job_id":"job-87","job_issued_at_ms":70488,"miner_id":"miner-10","network_difficulty":1000002,"ntime":1700000087,"order_key":"10:miner-10","p2mr_program_hex":"5757575757575757575757575757575757575757575757575757575757575757","share_difficulty":7,"share_id":"share-87","share_seq":87,"template_height":800002},{"accepted_at_ms":70511,"credit_policy":null,"job_id":"job-88","job_issued_at_ms":70502,"miner_id":"miner-0","network_difficulty":1000003,"ntime":1700000088,"order_key":"00:miner-0","p2mr_program_hex":"5858585858585858585858585858585858585858585858585858585858585858","share_difficulty":7,"share_id":"share-88","share_seq":88,"template_height":800002},{"accepted_at_ms":70505,"credit_policy":null,"job_id":"job-89","job_issued_at_ms":70497,"miner_id":"miner-1","network_difficulty":1000004,"ntime":1700000089,"order_key":"01:miner-1","p2mr_program_hex":"5959595959595959595959595959595959595959595959595959595959595959","share_difficulty":2,"share_id":"share-89","share_seq":89,"template_height":800002},{"accepted_at_ms":70506,"credit_policy":null,"job_id":"job-90","job_issued_at_ms":70489,"miner_id":"miner-2","network_difficulty":1000005,"ntime":1700000090,"order_key":"02:miner-2","p2mr_program_hex":"5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a","share_difficulty":7,"share_id":"share-90","share_seq":90,"template_height":800002},{"accepted_at_ms":70489,"credit_policy":null,"job_id":"job-91","job_issued_at_ms":70494,"miner_id":"miner-3","network_difficulty":1000006,"ntime":1700000091,"order_key":"03:miner-3","p2mr_program_hex":"5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b","share_difficulty":3,"share_id":"share-91","share_seq":91,"template_height":800002},{"accepted_at_ms":70498,"credit_policy":null,"job_id":"job-92","job_issued_at_ms":70501,"miner_id":"miner-4","network_difficulty":1000007,"ntime":1700000092,"order_key":"04:miner-4","p2mr_program_hex":"5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c","share_difficulty":1,"share_id":"share-92","share_seq":92,"template_height":800002},{"accepted_at_ms":70501,"credit_policy":null,"job_id":"job-93","job_issued_at_ms":70494,"miner_id":"miner-5","network_difficulty":1000008,"ntime":1700000093,"order_key":"05:miner-5","p2mr_program_hex":"5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d","share_difficulty":19,"share_id":"share-93","share_seq":93,"template_height":800002},{"accepted_at_ms":70494,"credit_policy":null,"job_id":"job-94","job_issued_at_ms":70487,"miner_id":"miner-6","network_difficulty":1000009,"ntime":1700000094,"order_key":"06:miner-6","p2mr_program_hex":"5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e","share_difficulty":19,"share_id":"share-94","share_seq":94,"template_height":800002},{"accepted_at_ms":70491,"credit_policy":null,"job_id":"job-95","job_issued_at_ms":70499,"miner_id":"miner-7","network_difficulty":1000010,"ntime":1700000095,"order_key":"07:miner-7","p2mr_program_hex":"5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f","share_difficulty":2,"share_id":"share-95","share_seq":95,"template_height":800002},{"accepted_at_ms":70512,"credit_policy":null,"job_id":"job-96","job_issued_at_ms":70512,"miner_id":"miner-8","network_difficulty":1000011,"ntime":1700000096,"order_key":"08:miner-8","p2mr_program_hex":"6060606060606060606060606060606060606060606060606060606060606060","share_difficulty":3,"share_id":"share-96","share_seq":96,"template_height":800003},{"accepted_at_ms":70504,"credit_policy":"stale-grace","job_id":"job-97","job_issued_at_ms":70504,"miner_id":"miner-9","network_difficulty":1000012,"ntime":1700000097,"order_key":"09:miner-9","p2mr_program_hex":"6161616161616161616161616161616161616161616161616161616161616161","share_difficulty":7,"share_id":"share-97","share_seq":97,"template_height":800003},{"accepted_at_ms":70507,"credit_policy":null,"job_id":"job-98","job_issued_at_ms":70493,"miner_id":"miner-10","network_difficulty":1000013,"ntime":1700000098,"order_key":"10:miner-10","p2mr_program_hex":"6262626262626262626262626262626262626262626262626262626262626262","share_difficulty":19,"share_id":"share-98","share_seq":98,"template_height":800003},{"accepted_at_ms":70486,"credit_policy":null,"job_id":"job-99","job_issued_at_ms":70504,"miner_id":"miner-0","network_difficulty":1000014,"ntime":1700000099,"order_key":"00:miner-0","p2mr_program_hex":"6363636363636363636363636363636363636363636363636363636363636363","share_difficulty":1,"share_id":"share-99","share_seq":99,"template_height":800003},{"accepted_at_ms":70510,"credit_policy":null,"job_id":"job-100","job_issued_at_ms":70482,"miner_id":"miner-1","network_difficulty":1000015,"ntime":1700000100,"order_key":"01:miner-1","p2mr_program_hex":"6464646464646464646464646464646464646464646464646464646464646464","share_difficulty":1,"share_id":"share-100","share_seq":100,"template_height":800003},{"accepted_at_ms":70507,"credit_policy":null,"job_id":"job-101","job_issued_at_ms":70509,"miner_id":"miner-2","network_difficulty":1000016,"ntime":1700000101,"order_key":"02:miner-2","p2mr_program_hex":"6565656565656565656565656565656565656565656565656565656565656565","share_difficulty":3,"share_id":"share-101","share_seq":101,"template_height":800003},{"accepted_at_ms":70503,"credit_policy":"stale-grace","job_id":"job-102","job_issued_at_ms":70498,"miner_id":"miner-3","network_difficulty":1000000,"ntime":1700000102,"order_key":"03:miner-3","p2mr_program_hex":"6666666666666666666666666666666666666666666666666666666666666666","share_difficulty":3,"share_id":"share-102","share_seq":102,"template_height":800003},{"accepted_at_ms":70494,"credit_policy":null,"job_id":"job-103","job_issued_at_ms":70494,"miner_id":"miner-4","network_difficulty":1000001,"ntime":1700000103,"order_key":"04:miner-4","p2mr_program_hex":"6767676767676767676767676767676767676767676767676767676767676767","share_difficulty":3,"share_id":"share-103","share_seq":103,"template_height":800003},{"accepted_at_ms":70509,"credit_policy":null,"job_id":"job-104","job_issued_at_ms":70500,"miner_id":"miner-5","network_difficulty":1000002,"ntime":1700000104,"order_key":"05:miner-5","p2mr_program_hex":"6868686868686868686868686868686868686868686868686868686868686868","share_difficulty":3,"share_id":"share-104","share_seq":104,"template_height":800003},{"accepted_at_ms":70501,"credit_policy":null,"job_id":"job-105","job_issued_at_ms":70507,"miner_id":"miner-6","network_difficulty":1000003,"ntime":1700000105,"order_key":"06:miner-6","p2mr_program_hex":"6969696969696969696969696969696969696969696969696969696969696969","share_difficulty":3,"share_id":"share-105","share_seq":105,"template_height":800003},{"accepted_at_ms":70510,"credit_policy":null,"job_id":"job-106","job_issued_at_ms":70510,"miner_id":"miner-7","network_difficulty":1000004,"ntime":1700000106,"order_key":"07:miner-7","p2mr_program_hex":"6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a","share_difficulty":19,"share_id":"share-106","share_seq":106,"template_height":800003},{"accepted_at_ms":70483,"credit_policy":"stale-grace","job_id":"job-107","job_issued_at_ms":70485,"miner_id":"miner-8","network_difficulty":1000005,"ntime":1700000107,"order_key":"08:miner-8","p2mr_program_hex":"6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b","share_difficulty":2,"share_id":"share-107","share_seq":107,"template_height":800003},{"accepted_at_ms":70489,"credit_policy":"stale-grace","job_id":"job-108","job_issued_at_ms":70509,"miner_id":"miner-9","network_difficulty":1000006,"ntime":1700000108,"order_key":"09:miner-9","p2mr_program_hex":"6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c","share_difficulty":7,"share_id":"share-108","share_seq":108,"template_height":800003},{"accepted_at_ms":70485,"credit_policy":null,"job_id":"job-109","job_issued_at_ms":70490,"miner_id":"miner-10","network_difficulty":1000007,"ntime":1700000109,"order_key":"10:miner-10","p2mr_program_hex":"6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d","share_difficulty":3,"share_id":"share-109","share_seq":109,"template_height":800003},{"accepted_at_ms":70504,"credit_policy":"stale-grace","job_id":"job-110","job_issued_at_ms":70491,"miner_id":"miner-0","network_difficulty":1000008,"ntime":1700000110,"order_key":"00:miner-0","p2mr_program_hex":"6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e","share_difficulty":7,"share_id":"share-110","share_seq":110,"template_height":800003},{"accepted_at_ms":70507,"credit_policy":"stale-grace","job_id":"job-111","job_issued_at_ms":70486,"miner_id":"miner-1","network_difficulty":1000009,"ntime":1700000111,"order_key":"01:miner-1","p2mr_program_hex":"6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f","share_difficulty":3,"share_id":"share-111","share_seq":111,"template_height":800003},{"accepted_at_ms":70501,"credit_policy":null,"job_id":"job-112","job_issued_at_ms":70494,"miner_id":"miner-2","network_difficulty":1000010,"ntime":1700000112,"order_key":"02:miner-2","p2mr_program_hex":"7070707070707070707070707070707070707070707070707070707070707070","share_difficulty":3,"share_id":"share-112","share_seq":112,"template_height":800003},{"accepted_at_ms":70497,"credit_policy":null,"job_id":"job-113","job_issued_at_ms":70490,"miner_id":"miner-3","network_difficulty":1000011,"ntime":1700000113,"order_key":"03:miner-3","p2mr_program_hex":"7171717171717171717171717171717171717171717171717171717171717171","share_difficulty":7,"share_id":"share-113","share_seq":113,"template_height":800003},{"accepted_at_ms":70500,"credit_policy":null,"job_id":"job-114","job_issued_at_ms":70486,"miner_id":"miner-4","network_difficulty":1000012,"ntime":1700000114,"order_key":"04:miner-4","p2mr_program_hex":"7272727272727272727272727272727272727272727272727272727272727272","share_difficulty":3,"share_id":"share-114","share_seq":114,"template_height":800003},{"accepted_at_ms":70501,"credit_policy":"stale-grace","job_id":"job-115","job_issued_at_ms":70511,"miner_id":"miner-5","network_difficulty":1000013,"ntime":1700000115,"order_key":"05:miner-5","p2mr_program_hex":"7373737373737373737373737373737373737373737373737373737373737373","share_difficulty":7,"share_id":"share-115","share_seq":115,"template_height":800003},{"accepted_at_ms":70502,"credit_policy":"stale-grace","job_id":"job-116","job_issued_at_ms":70488,"miner_id":"miner-6","network_difficulty":1000014,"ntime":1700000116,"order_key":"06:miner-6","p2mr_program_hex":"7474747474747474747474747474747474747474747474747474747474747474","share_difficulty":7,"share_id":"share-116","share_seq":116,"template_height":800003},{"accepted_at_ms":70484,"credit_policy":null,"job_id":"job-117","job_issued_at_ms":70509,"miner_id":"miner-7","network_difficulty":1000015,"ntime":1700000117,"order_key":"07:miner-7","p2mr_program_hex":"7575757575757575757575757575757575757575757575757575757575757575","share_difficulty":2,"share_id":"share-117","share_seq":117,"template_height":800003},{"accepted_at_ms":70490,"credit_policy":"stale-grace","job_id":"job-118","job_issued_at_ms":70492,"miner_id":"miner-8","network_difficulty":1000016,"ntime":1700000118,"order_key":"08:miner-8","p2mr_program_hex":"7676767676767676767676767676767676767676767676767676767676767676","share_difficulty":7,"share_id":"share-118","share_seq":118,"template_height":800003},{"accepted_at_ms":70484,"credit_policy":null,"job_id":"job-119","job_issued_at_ms":70510,"miner_id":"miner-9","network_difficulty":1000000,"ntime":1700000119,"order_key":"09:miner-9","p2mr_program_hex":"7777777777777777777777777777777777777777777777777777777777777777","share_difficulty":19,"share_id":"share-119","share_seq":119,"template_height":800003},{"accepted_at_ms":70494,"credit_policy":"stale-grace","job_id":"job-120","job_issued_at_ms":70512,"miner_id":"miner-10","network_difficulty":1000001,"ntime":1700000120,"order_key":"10:miner-10","p2mr_program_hex":"7878787878787878787878787878787878787878787878787878787878787878","share_difficulty":1,"share_id":"share-120","share_seq":120,"template_height":800003},{"accepted_at_ms":70510,"credit_policy":null,"job_id":"job-121","job_issued_at_ms":70497,"miner_id":"miner-0","network_difficulty":1000002,"ntime":1700000121,"order_key":"00:miner-0","p2mr_program_hex":"7979797979797979797979797979797979797979797979797979797979797979","share_difficulty":2,"share_id":"share-121","share_seq":121,"template_height":800003},{"accepted_at_ms":70492,"credit_policy":"stale-grace","job_id":"job-122","job_issued_at_ms":70495,"miner_id":"miner-1","network_difficulty":1000003,"ntime":1700000122,"order_key":"01:miner-1","p2mr_program_hex":"7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a","share_difficulty":2,"share_id":"share-122","share_seq":122,"template_height":800003},{"accepted_at_ms":70509,"credit_policy":"stale-grace","job_id":"job-123","job_issued_at_ms":70504,"miner_id":"miner-2","network_difficulty":1000004,"ntime":1700000123,"order_key":"02:miner-2","p2mr_program_hex":"7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b","share_difficulty":1,"share_id":"share-123","share_seq":123,"template_height":800003},{"accepted_at_ms":70488,"credit_policy":null,"job_id":"job-124","job_issued_at_ms":70487,"miner_id":"miner-3","network_difficulty":1000005,"ntime":1700000124,"order_key":"03:miner-3","p2mr_program_hex":"7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c","share_difficulty":2,"share_id":"share-124","share_seq":124,"template_height":800003},{"accepted_at_ms":70503,"credit_policy":null,"job_id":"job-125","job_issued_at_ms":70491,"miner_id":"miner-4","network_difficulty":1000006,"ntime":1700000125,"order_key":"04:miner-4","p2mr_program_hex":"7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d","share_difficulty":2,"share_id":"share-125","share_seq":125,"template_height":800003},{"accepted_at_ms":70504,"credit_policy":null,"job_id":"job-126","job_issued_at_ms":70489,"miner_id":"miner-5","network_difficulty":1000007,"ntime":1700000126,"order_key":"05:miner-5","p2mr_program_hex":"7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e","share_difficulty":2,"share_id":"share-126","share_seq":126,"template_height":800003},{"accepted_at_ms":70504,"credit_policy":null,"job_id":"job-127","job_issued_at_ms":70505,"miner_id":"miner-6","network_difficulty":1000008,"ntime":1700000127,"order_key":"06:miner-6","p2mr_program_hex":"7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f","share_difficulty":19,"share_id":"share-127","share_seq":127,"template_height":800003},{"accepted_at_ms":70495,"credit_policy":"stale-grace","job_id":"job-128","job_issued_at_ms":70505,"miner_id":"miner-7","network_difficulty":1000009,"ntime":1700000128,"order_key":"07:miner-7","p2mr_program_hex":"8080808080808080808080808080808080808080808080808080808080808080","share_difficulty":1,"share_id":"share-128","share_seq":128,"template_height":800004},{"accepted_at_ms":70496,"credit_policy":null,"job_id":"job-129","job_issued_at_ms":70497,"miner_id":"miner-8","network_difficulty":1000010,"ntime":1700000129,"order_key":"08:miner-8","p2mr_program_hex":"8181818181818181818181818181818181818181818181818181818181818181","share_difficulty":19,"share_id":"share-129","share_seq":129,"template_height":800004},{"accepted_at_ms":70492,"credit_policy":"stale-grace","job_id":"job-130","job_issued_at_ms":70506,"miner_id":"miner-9","network_difficulty":1000011,"ntime":1700000130,"order_key":"09:miner-9","p2mr_program_hex":"8282828282828282828282828282828282828282828282828282828282828282","share_difficulty":19,"share_id":"share-130","share_seq":130,"template_height":800004},{"accepted_at_ms":70495,"credit_policy":null,"job_id":"job-131","job_issued_at_ms":70496,"miner_id":"miner-10","network_difficulty":1000012,"ntime":1700000131,"order_key":"10:miner-10","p2mr_program_hex":"8383838383838383838383838383838383838383838383838383838383838383","share_difficulty":19,"share_id":"share-131","share_seq":131,"template_height":800004},{"accepted_at_ms":70512,"credit_policy":null,"job_id":"job-132","job_issued_at_ms":70491,"miner_id":"miner-0","network_difficulty":1000013,"ntime":1700000132,"order_key":"00:miner-0","p2mr_program_hex":"8484848484848484848484848484848484848484848484848484848484848484","share_difficulty":2,"share_id":"share-132","share_seq":132,"template_height":800004},{"accepted_at_ms":70498,"credit_policy":"stale-grace","job_id":"job-133","job_issued_at_ms":70493,"miner_id":"miner-1","network_difficulty":1000014,"ntime":1700000133,"order_key":"01:miner-1","p2mr_program_hex":"8585858585858585858585858585858585858585858585858585858585858585","share_difficulty":3,"share_id":"share-133","share_seq":133,"template_height":800004},{"accepted_at_ms":70512,"credit_policy":null,"job_id":"job-134","job_issued_at_ms":70496,"miner_id":"miner-2","network_difficulty":1000015,"ntime":1700000134,"order_key":"02:miner-2","p2mr_program_hex":"8686868686868686868686868686868686868686868686868686868686868686","share_difficulty":1,"share_id":"share-134","share_seq":134,"template_height":800004},{"accepted_at_ms":70505,"credit_policy":null,"job_id":"job-135","job_issued_at_ms":70482,"miner_id":"miner-3","network_difficulty":1000016,"ntime":1700000135,"order_key":"03:miner-3","p2mr_program_hex":"8787878787878787878787878787878787878787878787878787878787878787","share_difficulty":19,"share_id":"share-135","share_seq":135,"template_height":800004},{"accepted_at_ms":70511,"credit_policy":null,"job_id":"job-136","job_issued_at_ms":70498,"miner_id":"miner-4","network_difficulty":1000000,"ntime":1700000136,"order_key":"04:miner-4","p2mr_program_hex":"8888888888888888888888888888888888888888888888888888888888888888","share_difficulty":1,"share_id":"share-136","share_seq":136,"template_height":800004},{"accepted_at_ms":70488,"credit_policy":null,"job_id":"job-137","job_issued_at_ms":70506,"miner_id":"miner-5","network_difficulty":1000001,"ntime":1700000137,"order_key":"05:miner-5","p2mr_program_hex":"8989898989898989898989898989898989898989898989898989898989898989","share_difficulty":3,"share_id":"share-137","share_seq":137,"template_height":800004},{"accepted_at_ms":70485,"credit_policy":"stale-grace","job_id":"job-138","job_issued_at_ms":70491,"miner_id":"miner-6","network_difficulty":1000002,"ntime":1700000138,"order_key":"06:miner-6","p2mr_program_hex":"8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a","share_difficulty":7,"share_id":"share-138","share_seq":138,"template_height":800004},{"accepted_at_ms":70512,"credit_policy":"stale-grace","job_id":"job-139","job_issued_at_ms":70503,"miner_id":"miner-7","network_difficulty":1000003,"ntime":1700000139,"order_key":"07:miner-7","p2mr_program_hex":"8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b","share_difficulty":1,"share_id":"share-139","share_seq":139,"template_height":800004},{"accepted_at_ms":70500,"credit_policy":null,"job_id":"job-140","job_issued_at_ms":70495,"miner_id":"miner-8","network_difficulty":1000004,"ntime":1700000140,"order_key":"08:miner-8","p2mr_program_hex":"8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c","share_difficulty":7,"share_id":"share-140","share_seq":140,"template_height":800004},{"accepted_at_ms":70502,"credit_policy":null,"job_id":"job-141","job_issued_at_ms":70484,"miner_id":"miner-9","network_difficulty":1000005,"ntime":1700000141,"order_key":"09:miner-9","p2mr_program_hex":"8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d","share_difficulty":3,"share_id":"share-141","share_seq":141,"template_height":800004},{"accepted_at_ms":70492,"credit_policy":null,"job_id":"job-142","job_issued_at_ms":70504,"miner_id":"miner-10","network_difficulty":1000006,"ntime":1700000142,"order_key":"10:miner-10","p2mr_program_hex":"8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e","share_difficulty":3,"share_id":"share-142","share_seq":142,"template_height":800004},{"accepted_at_ms":70492,"credit_policy":null,"job_id":"job-143","job_issued_at_ms":70499,"miner_id":"miner-0","network_difficulty":1000007,"ntime":1700000143,"order_key":"00:miner-0","p2mr_program_hex":"8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f","share_difficulty":3,"share_id":"share-143","share_seq":143,"template_height":800004},{"accepted_at_ms":70499,"credit_policy":null,"job_id":"job-144","job_issued_at_ms":70483,"miner_id":"miner-1","network_difficulty":1000008,"ntime":1700000144,"order_key":"01:miner-1","p2mr_program_hex":"9090909090909090909090909090909090909090909090909090909090909090","share_difficulty":3,"share_id":"share-144","share_seq":144,"template_height":800004},{"accepted_at_ms":70504,"credit_policy":"stale-grace","job_id":"job-145","job_issued_at_ms":70496,"miner_id":"miner-2","network_difficulty":1000009,"ntime":1700000145,"order_key":"02:miner-2","p2mr_program_hex":"9191919191919191919191919191919191919191919191919191919191919191","share_difficulty":2,"share_id":"share-145","share_seq":145,"template_height":800004},{"accepted_at_ms":70501,"credit_policy":null,"job_id":"job-146","job_issued_at_ms":70492,"miner_id":"miner-3","network_difficulty":1000010,"ntime":1700000146,"order_key":"03:miner-3","p2mr_program_hex":"9292929292929292929292929292929292929292929292929292929292929292","share_difficulty":2,"share_id":"share-146","share_seq":146,"template_height":800004},{"accepted_at_ms":70497,"credit_policy":"stale-grace","job_id":"job-147","job_issued_at_ms":70499,"miner_id":"miner-4","network_difficulty":1000011,"ntime":1700000147,"order_key":"04:miner-4","p2mr_program_hex":"9393939393939393939393939393939393939393939393939393939393939393","share_difficulty":7,"share_id":"share-147","share_seq":147,"template_height":800004},{"accepted_at_ms":70512,"credit_policy":null,"job_id":"job-148","job_issued_at_ms":70491,"miner_id":"miner-5","network_difficulty":1000012,"ntime":1700000148,"order_key":"05:miner-5","p2mr_program_hex":"9494949494949494949494949494949494949494949494949494949494949494","share_difficulty":19,"share_id":"share-148","share_seq":148,"template_height":800004},{"accepted_at_ms":70507,"credit_policy":null,"job_id":"job-149","job_issued_at_ms":70493,"miner_id":"miner-6","network_difficulty":1000013,"ntime":1700000149,"order_key":"06:miner-6","p2mr_program_hex":"9595959595959595959595959595959595959595959595959595959595959595","share_difficulty":1,"share_id":"share-149","share_seq":149,"template_height":800004},{"accepted_at_ms":70493,"credit_policy":null,"job_id":"job-150","job_issued_at_ms":70510,"miner_id":"miner-7","network_difficulty":1000014,"ntime":1700000150,"order_key":"07:miner-7","p2mr_program_hex":"9696969696969696969696969696969696969696969696969696969696969696","share_difficulty":1,"share_id":"share-150","share_seq":150,"template_height":800004},{"accepted_at_ms":70499,"credit_policy":null,"job_id":"job-151","job_issued_at_ms":70510,"miner_id":"miner-8","network_difficulty":1000015,"ntime":1700000151,"order_key":"08:miner-8","p2mr_program_hex":"9797979797979797979797979797979797979797979797979797979797979797","share_difficulty":3,"share_id":"share-151","share_seq":151,"template_height":800004},{"accepted_at_ms":70511,"credit_policy":null,"job_id":"job-152","job_issued_at_ms":70498,"miner_id":"miner-9","network_difficulty":1000016,"ntime":1700000152,"order_key":"09:miner-9","p2mr_program_hex":"9898989898989898989898989898989898989898989898989898989898989898","share_difficulty":3,"share_id":"share-152","share_seq":152,"template_height":800004},{"accepted_at_ms":70496,"credit_policy":"stale-grace","job_id":"job-153","job_issued_at_ms":70502,"miner_id":"miner-10","network_difficulty":1000000,"ntime":1700000153,"order_key":"10:miner-10","p2mr_program_hex":"9999999999999999999999999999999999999999999999999999999999999999","share_difficulty":1,"share_id":"share-153","share_seq":153,"template_height":800004},{"accepted_at_ms":70493,"credit_policy":null,"job_id":"job-154","job_issued_at_ms":70494,"miner_id":"miner-0","network_difficulty":1000001,"ntime":1700000154,"order_key":"00:miner-0","p2mr_program_hex":"9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a","share_difficulty":3,"share_id":"share-154","share_seq":154,"template_height":800004},{"accepted_at_ms":70494,"credit_policy":null,"job_id":"job-155","job_issued_at_ms":70499,"miner_id":"miner-1","network_difficulty":1000002,"ntime":1700000155,"order_key":"01:miner-1","p2mr_program_hex":"9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b","share_difficulty":3,"share_id":"share-155","share_seq":155,"template_height":800004},{"accepted_at_ms":70489,"credit_policy":null,"job_id":"job-156","job_issued_at_ms":70496,"miner_id":"miner-2","network_difficulty":1000003,"ntime":1700000156,"order_key":"02:miner-2","p2mr_program_hex":"9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c","share_difficulty":3,"share_id":"share-156","share_seq":156,"template_height":800004},{"accepted_at_ms":70483,"credit_policy":null,"job_id":"job-157","job_issued_at_ms":70495,"miner_id":"miner-3","network_difficulty":1000004,"ntime":1700000157,"order_key":"03:miner-3","p2mr_program_hex":"9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d","share_difficulty":7,"share_id":"share-157","share_seq":157,"template_height":800004},{"accepted_at_ms":70489,"credit_policy":null,"job_id":"job-158","job_issued_at_ms":70511,"miner_id":"miner-4","network_difficulty":1000005,"ntime":1700000158,"order_key":"04:miner-4","p2mr_program_hex":"9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e","share_difficulty":1,"share_id":"share-158","share_seq":158,"template_height":800004},{"accepted_at_ms":70505,"credit_policy":null,"job_id":"job-159","job_issued_at_ms":70494,"miner_id":"miner-5","network_difficulty":1000006,"ntime":1700000159,"order_key":"05:miner-5","p2mr_program_hex":"9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f","share_difficulty":2,"share_id":"share-159","share_seq":159,"template_height":800004},{"accepted_at_ms":70497,"credit_policy":null,"job_id":"job-160","job_issued_at_ms":70508,"miner_id":"miner-6","network_difficulty":1000007,"ntime":1700000160,"order_key":"06:miner-6","p2mr_program_hex":"a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0","share_difficulty":1,"share_id":"share-160","share_seq":160,"template_height":800005},{"accepted_at_ms":70510,"credit_policy":null,"job_id":"job-161","job_issued_at_ms":70493,"miner_id":"miner-7","network_difficulty":1000008,"ntime":1700000161,"order_key":"07:miner-7","p2mr_program_hex":"a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1","share_difficulty":3,"share_id":"share-161","share_seq":161,"template_height":800005},{"accepted_at_ms":70503,"credit_policy":null,"job_id":"job-162","job_issued_at_ms":70494,"miner_id":"miner-8","network_difficulty":1000009,"ntime":1700000162,"order_key":"08:miner-8","p2mr_program_hex":"a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2","share_difficulty":1,"share_id":"share-162","share_seq":162,"template_height":800005},{"accepted_at_ms":70510,"credit_policy":null,"job_id":"job-163","job_issued_at_ms":70488,"miner_id":"miner-9","network_difficulty":1000010,"ntime":1700000163,"order_key":"09:miner-9","p2mr_program_hex":"a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3","share_difficulty":7,"share_id":"share-163","share_seq":163,"template_height":800005},{"accepted_at_ms":70482,"credit_policy":"stale-grace","job_id":"job-164","job_issued_at_ms":70488,"miner_id":"miner-10","network_difficulty":1000011,"ntime":1700000164,"order_key":"10:miner-10","p2mr_program_hex":"a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4","share_difficulty":19,"share_id":"share-164","share_seq":164,"template_height":800005},{"accepted_at_ms":70506,"credit_policy":null,"job_id":"job-165","job_issued_at_ms":70493,"miner_id":"miner-0","network_difficulty":1000012,"ntime":1700000165,"order_key":"00:miner-0","p2mr_program_hex":"a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5","share_difficulty":3,"share_id":"share-165","share_seq":165,"template_height":800005},{"accepted_at_ms":70483,"credit_policy":null,"job_id":"job-166","job_issued_at_ms":70499,"miner_id":"miner-1","network_difficulty":1000013,"ntime":1700000166,"order_key":"01:miner-1","p2mr_program_hex":"a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6","share_difficulty":7,"share_id":"share-166","share_seq":166,"template_height":800005},{"accepted_at_ms":70486,"credit_policy":null,"job_id":"job-167","job_issued_at_ms":70483,"miner_id":"miner-2","network_difficulty":1000014,"ntime":1700000167,"order_key":"02:miner-2","p2mr_program_hex":"a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7","share_difficulty":19,"share_id":"share-167","share_seq":167,"template_height":800005},{"accepted_at_ms":70487,"credit_policy":null,"job_id":"job-168","job_issued_at_ms":70503,"miner_id":"miner-3","network_difficulty":1000015,"ntime":1700000168,"order_key":"03:miner-3","p2mr_program_hex":"a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8","share_difficulty":2,"share_id":"share-168","share_seq":168,"template_height":800005},{"accepted_at_ms":70501,"credit_policy":null,"job_id":"job-169","job_issued_at_ms":70503,"miner_id":"miner-4","network_difficulty":1000016,"ntime":1700000169,"order_key":"04:miner-4","p2mr_program_hex":"a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9","share_difficulty":19,"share_id":"share-169","share_seq":169,"template_height":800005},{"accepted_at_ms":70484,"credit_policy":null,"job_id":"job-170","job_issued_at_ms":70482,"miner_id":"miner-5","network_difficulty":1000000,"ntime":1700000170,"order_key":"05:miner-5","p2mr_program_hex":"aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa","share_difficulty":3,"share_id":"share-170","share_seq":170,"template_height":800005},{"accepted_at_ms":70509,"credit_policy":null,"job_id":"job-171","job_issued_at_ms":70495,"miner_id":"miner-6","network_difficulty":1000001,"ntime":1700000171,"order_key":"06:miner-6","p2mr_program_hex":"abababababababababababababababababababababababababababababababab","share_difficulty":7,"share_id":"share-171","share_seq":171,"template_height":800005},{"accepted_at_ms":70487,"credit_policy":"stale-grace","job_id":"job-172","job_issued_at_ms":70511,"miner_id":"miner-7","network_difficulty":1000002,"ntime":1700000172,"order_key":"07:miner-7","p2mr_program_hex":"acacacacacacacacacacacacacacacacacacacacacacacacacacacacacacacac","share_difficulty":7,"share_id":"share-172","share_seq":172,"template_height":800005},{"accepted_at_ms":70510,"credit_policy":null,"job_id":"job-173","job_issued_at_ms":70487,"miner_id":"miner-8","network_difficulty":1000003,"ntime":1700000173,"order_key":"08:miner-8","p2mr_program_hex":"adadadadadadadadadadadadadadadadadadadadadadadadadadadadadadadad","share_difficulty":19,"share_id":"share-173","share_seq":173,"template_height":800005},{"accepted_at_ms":70510,"credit_policy":null,"job_id":"job-174","job_issued_at_ms":70500,"miner_id":"miner-9","network_difficulty":1000004,"ntime":1700000174,"order_key":"09:miner-9","p2mr_program_hex":"aeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeae","share_difficulty":2,"share_id":"share-174","share_seq":174,"template_height":800005},{"accepted_at_ms":70497,"credit_policy":null,"job_id":"job-175","job_issued_at_ms":70509,"miner_id":"miner-10","network_difficulty":1000005,"ntime":1700000175,"order_key":"10:miner-10","p2mr_program_hex":"afafafafafafafafafafafafafafafafafafafafafafafafafafafafafafafaf","share_difficulty":7,"share_id":"share-175","share_seq":175,"template_height":800005},{"accepted_at_ms":70496,"credit_policy":"stale-grace","job_id":"job-176","job_issued_at_ms":70500,"miner_id":"miner-0","network_difficulty":1000006,"ntime":1700000176,"order_key":"00:miner-0","p2mr_program_hex":"b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0","share_difficulty":19,"share_id":"share-176","share_seq":176,"template_height":800005},{"accepted_at_ms":70496,"credit_policy":null,"job_id":"job-177","job_issued_at_ms":70512,"miner_id":"miner-1","network_difficulty":1000007,"ntime":1700000177,"order_key":"01:miner-1","p2mr_program_hex":"b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1","share_difficulty":7,"share_id":"share-177","share_seq":177,"template_height":800005},{"accepted_at_ms":70495,"credit_policy":null,"job_id":"job-178","job_issued_at_ms":70505,"miner_id":"miner-2","network_difficulty":1000008,"ntime":1700000178,"order_key":"02:miner-2","p2mr_program_hex":"b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2","share_difficulty":2,"share_id":"share-178","share_seq":178,"template_height":800005},{"accepted_at_ms":70487,"credit_policy":null,"job_id":"job-179","job_issued_at_ms":70493,"miner_id":"miner-3","network_difficulty":1000009,"ntime":1700000179,"order_key":"03:miner-3","p2mr_program_hex":"b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3","share_difficulty":2,"share_id":"share-179","share_seq":179,"template_height":800005},{"accepted_at_ms":70506,"credit_policy":null,"job_id":"job-180","job_issued_at_ms":70509,"miner_id":"miner-4","network_difficulty":1000010,"ntime":1700000180,"order_key":"04:miner-4","p2mr_program_hex":"b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4","share_difficulty":19,"share_id":"share-180","share_seq":180,"template_height":800005},{"accepted_at_ms":70484,"credit_policy":null,"job_id":"job-181","job_issued_at_ms":70496,"miner_id":"miner-5","network_difficulty":1000011,"ntime":1700000181,"order_key":"05:miner-5","p2mr_program_hex":"b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5","share_difficulty":2,"share_id":"share-181","share_seq":181,"template_height":800005},{"accepted_at_ms":70496,"credit_policy":null,"job_id":"job-182","job_issued_at_ms":70506,"miner_id":"miner-6","network_difficulty":1000012,"ntime":1700000182,"order_key":"06:miner-6","p2mr_program_hex":"b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6","share_difficulty":3,"share_id":"share-182","share_seq":182,"template_height":800005},{"accepted_at_ms":70489,"credit_policy":null,"job_id":"job-183","job_issued_at_ms":70507,"miner_id":"miner-7","network_difficulty":1000013,"ntime":1700000183,"order_key":"07:miner-7","p2mr_program_hex":"b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7","share_difficulty":3,"share_id":"share-183","share_seq":183,"template_height":800005},{"accepted_at_ms":70503,"credit_policy":null,"job_id":"job-184","job_issued_at_ms":70512,"miner_id":"miner-8","network_difficulty":1000014,"ntime":1700000184,"order_key":"08:miner-8","p2mr_program_hex":"b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8","share_difficulty":2,"share_id":"share-184","share_seq":184,"template_height":800005},{"accepted_at_ms":70505,"credit_policy":null,"job_id":"job-185","job_issued_at_ms":70482,"miner_id":"miner-9","network_difficulty":1000015,"ntime":1700000185,"order_key":"09:miner-9","p2mr_program_hex":"b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9","share_difficulty":2,"share_id":"share-185","share_seq":185,"template_height":800005},{"accepted_at_ms":70512,"credit_policy":"stale-grace","job_id":"job-186","job_issued_at_ms":70488,"miner_id":"miner-10","network_difficulty":1000016,"ntime":1700000186,"order_key":"10:miner-10","p2mr_program_hex":"babababababababababababababababababababababababababababababababa","share_difficulty":2,"share_id":"share-186","share_seq":186,"template_height":800005},{"accepted_at_ms":70487,"credit_policy":"stale-grace","job_id":"job-187","job_issued_at_ms":70507,"miner_id":"miner-0","network_difficulty":1000000,"ntime":1700000187,"order_key":"00:miner-0","p2mr_program_hex":"bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb","share_difficulty":19,"share_id":"share-187","share_seq":187,"template_height":800005},{"accepted_at_ms":70492,"credit_policy":null,"job_id":"job-188","job_issued_at_ms":70512,"miner_id":"miner-1","network_difficulty":1000001,"ntime":1700000188,"order_key":"01:miner-1","p2mr_program_hex":"bcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbc","share_difficulty":1,"share_id":"share-188","share_seq":188,"template_height":800005},{"accepted_at_ms":70500,"credit_policy":null,"job_id":"job-189","job_issued_at_ms":70504,"miner_id":"miner-2","network_difficulty":1000002,"ntime":1700000189,"order_key":"02:miner-2","p2mr_program_hex":"bdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbd","share_difficulty":19,"share_id":"share-189","share_seq":189,"template_height":800005},{"accepted_at_ms":70485,"credit_policy":"stale-grace","job_id":"job-190","job_issued_at_ms":70486,"miner_id":"miner-3","network_difficulty":1000003,"ntime":1700000190,"order_key":"03:miner-3","p2mr_program_hex":"bebebebebebebebebebebebebebebebebebebebebebebebebebebebebebebebe","share_difficulty":2,"share_id":"share-190","share_seq":190,"template_height":800005},{"accepted_at_ms":70485,"credit_policy":null,"job_id":"job-191","job_issued_at_ms":70506,"miner_id":"miner-4","network_difficulty":1000004,"ntime":1700000191,"order_key":"04:miner-4","p2mr_program_hex":"bfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbf","share_difficulty":19,"share_id":"share-191","share_seq":191,"template_height":800005},{"accepted_at_ms":70487,"credit_policy":null,"job_id":"job-192","job_issued_at_ms":70485,"miner_id":"miner-5","network_difficulty":1000005,"ntime":1700000192,"order_key":"05:miner-5","p2mr_program_hex":"c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0","share_difficulty":7,"share_id":"share-192","share_seq":192,"template_height":800006},{"accepted_at_ms":70512,"credit_policy":null,"job_id":"job-193","job_issued_at_ms":70509,"miner_id":"miner-6","network_difficulty":1000006,"ntime":1700000193,"order_key":"06:miner-6","p2mr_program_hex":"c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1","share_difficulty":2,"share_id":"share-193","share_seq":193,"template_height":800006},{"accepted_at_ms":70488,"credit_policy":null,"job_id":"job-194","job_issued_at_ms":70510,"miner_id":"miner-7","network_difficulty":1000007,"ntime":1700000194,"order_key":"07:miner-7","p2mr_program_hex":"c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2","share_difficulty":1,"share_id":"share-194","share_seq":194,"template_height":800006},{"accepted_at_ms":70482,"credit_policy":null,"job_id":"job-195","job_issued_at_ms":70500,"miner_id":"miner-8","network_difficulty":1000008,"ntime":1700000195,"order_key":"08:miner-8","p2mr_program_hex":"c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3","share_difficulty":7,"share_id":"share-195","share_seq":195,"template_height":800006},{"accepted_at_ms":70482,"credit_policy":null,"job_id":"job-196","job_issued_at_ms":70512,"miner_id":"miner-9","network_difficulty":1000009,"ntime":1700000196,"order_key":"09:miner-9","p2mr_program_hex":"c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4","share_difficulty":1,"share_id":"share-196","share_seq":196,"template_height":800006},{"accepted_at_ms":70496,"credit_policy":null,"job_id":"job-197","job_issued_at_ms":70496,"miner_id":"miner-10","network_difficulty":1000010,"ntime":1700000197,"order_key":"10:miner-10","p2mr_program_hex":"c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5","share_difficulty":3,"share_id":"share-197","share_seq":197,"template_height":800006},{"accepted_at_ms":70487,"credit_policy":null,"job_id":"job-198","job_issued_at_ms":70503,"miner_id":"miner-0","network_difficulty":1000011,"ntime":1700000198,"order_key":"00:miner-0","p2mr_program_hex":"c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6","share_difficulty":2,"share_id":"share-198","share_seq":198,"template_height":800006},{"accepted_at_ms":70508,"credit_policy":null,"job_id":"job-199","job_issued_at_ms":70486,"miner_id":"miner-1","network_difficulty":1000012,"ntime":1700000199,"order_key":"01:miner-1","p2mr_program_hex":"c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7","share_difficulty":7,"share_id":"share-199","share_seq":199,"template_height":800006},{"accepted_at_ms":70507,"credit_policy":null,"job_id":"job-200","job_issued_at_ms":70491,"miner_id":"miner-2","network_difficulty":1000013,"ntime":1700000200,"order_key":"02:miner-2","p2mr_program_hex":"c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8","share_difficulty":7,"share_id":"share-200","share_seq":200,"template_height":800006},{"accepted_at_ms":70512,"credit_policy":null,"job_id":"job-201","job_issued_at_ms":70491,"miner_id":"miner-3","network_difficulty":1000014,"ntime":1700000201,"order_key":"03:miner-3","p2mr_program_hex":"c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9","share_difficulty":19,"share_id":"share-201","share_seq":201,"template_height":800006},{"accepted_at_ms":70484,"credit_policy":"stale-grace","job_id":"job-202","job_issued_at_ms":70504,"miner_id":"miner-4","network_difficulty":1000015,"ntime":1700000202,"order_key":"04:miner-4","p2mr_program_hex":"cacacacacacacacacacacacacacacacacacacacacacacacacacacacacacacaca","share_difficulty":19,"share_id":"share-202","share_seq":202,"template_height":800006},{"accepted_at_ms":70486,"credit_policy":null,"job_id":"job-203","job_issued_at_ms":70485,"miner_id":"miner-5","network_difficulty":1000016,"ntime":1700000203,"order_key":"05:miner-5","p2mr_program_hex":"cbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcb","share_difficulty":19,"share_id":"share-203","share_seq":203,"template_height":800006},{"accepted_at_ms":70489,"credit_policy":null,"job_id":"job-204","job_issued_at_ms":70492,"miner_id":"miner-6","network_difficulty":1000000,"ntime":1700000204,"order_key":"06:miner-6","p2mr_program_hex":"cccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccc","share_difficulty":7,"share_id":"share-204","share_seq":204,"template_height":800006},{"accepted_at_ms":70493,"credit_policy":null,"job_id":"job-205","job_issued_at_ms":70495,"miner_id":"miner-7","network_difficulty":1000001,"ntime":1700000205,"order_key":"07:miner-7","p2mr_program_hex":"cdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcd","share_difficulty":19,"share_id":"share-205","share_seq":205,"template_height":800006},{"accepted_at_ms":70506,"credit_policy":"stale-grace","job_id":"job-206","job_issued_at_ms":70497,"miner_id":"miner-8","network_difficulty":1000002,"ntime":1700000206,"order_key":"08:miner-8","p2mr_program_hex":"cececececececececececececececececececececececececececececececece","share_difficulty":2,"share_id":"share-206","share_seq":206,"template_height":800006},{"accepted_at_ms":70505,"credit_policy":"stale-grace","job_id":"job-207","job_issued_at_ms":70509,"miner_id":"miner-9","network_difficulty":1000003,"ntime":1700000207,"order_key":"09:miner-9","p2mr_program_hex":"cfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcf","share_difficulty":3,"share_id":"share-207","share_seq":207,"template_height":800006},{"accepted_at_ms":70489,"credit_policy":null,"job_id":"job-208","job_issued_at_ms":70501,"miner_id":"miner-10","network_difficulty":1000004,"ntime":1700000208,"order_key":"10:miner-10","p2mr_program_hex":"d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0","share_difficulty":1,"share_id":"share-208","share_seq":208,"template_height":800006},{"accepted_at_ms":70501,"credit_policy":null,"job_id":"job-209","job_issued_at_ms":70509,"miner_id":"miner-0","network_difficulty":1000005,"ntime":1700000209,"order_key":"00:miner-0","p2mr_program_hex":"d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1","share_difficulty":19,"share_id":"share-209","share_seq":209,"template_height":800006},{"accepted_at_ms":70495,"credit_policy":null,"job_id":"job-210","job_issued_at_ms":70483,"miner_id":"miner-1","network_difficulty":1000006,"ntime":1700000210,"order_key":"01:miner-1","p2mr_program_hex":"d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2","share_difficulty":19,"share_id":"share-210","share_seq":210,"template_height":800006},{"accepted_at_ms":70495,"credit_policy":"stale-grace","job_id":"job-211","job_issued_at_ms":70510,"miner_id":"miner-2","network_difficulty":1000007,"ntime":1700000211,"order_key":"02:miner-2","p2mr_program_hex":"d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3","share_difficulty":1,"share_id":"share-211","share_seq":211,"template_height":800006},{"accepted_at_ms":70507,"credit_policy":null,"job_id":"job-212","job_issued_at_ms":70510,"miner_id":"miner-3","network_difficulty":1000008,"ntime":1700000212,"order_key":"03:miner-3","p2mr_program_hex":"d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4","share_difficulty":2,"share_id":"share-212","share_seq":212,"template_height":800006},{"accepted_at_ms":70496,"credit_policy":null,"job_id":"job-213","job_issued_at_ms":70495,"miner_id":"miner-4","network_difficulty":1000009,"ntime":1700000213,"order_key":"04:miner-4","p2mr_program_hex":"d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5","share_difficulty":1,"share_id":"share-213","share_seq":213,"template_height":800006},{"accepted_at_ms":70506,"credit_policy":"stale-grace","job_id":"job-214","job_issued_at_ms":70499,"miner_id":"miner-5","network_difficulty":1000010,"ntime":1700000214,"order_key":"05:miner-5","p2mr_program_hex":"d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6","share_difficulty":3,"share_id":"share-214","share_seq":214,"template_height":800006},{"accepted_at_ms":70504,"credit_policy":null,"job_id":"job-215","job_issued_at_ms":70504,"miner_id":"miner-6","network_difficulty":1000011,"ntime":1700000215,"order_key":"06:miner-6","p2mr_program_hex":"d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7","share_difficulty":3,"share_id":"share-215","share_seq":215,"template_height":800006},{"accepted_at_ms":70510,"credit_policy":null,"job_id":"job-216","job_issued_at_ms":70496,"miner_id":"miner-7","network_difficulty":1000012,"ntime":1700000216,"order_key":"07:miner-7","p2mr_program_hex":"d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8","share_difficulty":3,"share_id":"share-216","share_seq":216,"template_height":800006},{"accepted_at_ms":70495,"credit_policy":null,"job_id":"job-217","job_issued_at_ms":70483,"miner_id":"miner-8","network_difficulty":1000013,"ntime":1700000217,"order_key":"08:miner-8","p2mr_program_hex":"d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9","share_difficulty":2,"share_id":"share-217","share_seq":217,"template_height":800006},{"accepted_at_ms":70510,"credit_policy":null,"job_id":"job-218","job_issued_at_ms":70503,"miner_id":"miner-9","network_difficulty":1000014,"ntime":1700000218,"order_key":"09:miner-9","p2mr_program_hex":"dadadadadadadadadadadadadadadadadadadadadadadadadadadadadadadada","share_difficulty":7,"share_id":"share-218","share_seq":218,"template_height":800006},{"accepted_at_ms":70510,"credit_policy":null,"job_id":"job-219","job_issued_at_ms":70508,"miner_id":"miner-10","network_difficulty":1000015,"ntime":1700000219,"order_key":"10:miner-10","p2mr_program_hex":"dbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdb","share_difficulty":2,"share_id":"share-219","share_seq":219,"template_height":800006},{"accepted_at_ms":70490,"credit_policy":null,"job_id":"job-220","job_issued_at_ms":70495,"miner_id":"miner-0","network_difficulty":1000016,"ntime":1700000220,"order_key":"00:miner-0","p2mr_program_hex":"dcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdc","share_difficulty":1,"share_id":"share-220","share_seq":220,"template_height":800006},{"accepted_at_ms":70485,"credit_policy":"stale-grace","job_id":"job-221","job_issued_at_ms":70498,"miner_id":"miner-1","network_difficulty":1000000,"ntime":1700000221,"order_key":"01:miner-1","p2mr_program_hex":"dddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddd","share_difficulty":19,"share_id":"share-221","share_seq":221,"template_height":800006},{"accepted_at_ms":70506,"credit_policy":"stale-grace","job_id":"job-222","job_issued_at_ms":70504,"miner_id":"miner-2","network_difficulty":1000001,"ntime":1700000222,"order_key":"02:miner-2","p2mr_program_hex":"dededededededededededededededededededededededededededededededede","share_difficulty":2,"share_id":"share-222","share_seq":222,"template_height":800006},{"accepted_at_ms":70502,"credit_policy":null,"job_id":"job-223","job_issued_at_ms":70497,"miner_id":"miner-3","network_difficulty":1000002,"ntime":1700000223,"order_key":"03:miner-3","p2mr_program_hex":"dfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdf","share_difficulty":2,"share_id":"share-223","share_seq":223,"template_height":800006},{"accepted_at_ms":70491,"credit_policy":"stale-grace","job_id":"job-224","job_issued_at_ms":70503,"miner_id":"miner-4","network_difficulty":1000003,"ntime":1700000224,"order_key":"04:miner-4","p2mr_program_hex":"e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0","share_difficulty":2,"share_id":"share-224","share_seq":224,"template_height":800007},{"accepted_at_ms":70512,"credit_policy":null,"job_id":"job-225","job_issued_at_ms":70509,"miner_id":"miner-5","network_difficulty":1000004,"ntime":1700000225,"order_key":"05:miner-5","p2mr_program_hex":"e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1","share_difficulty":19,"share_id":"share-225","share_seq":225,"template_height":800007},{"accepted_at_ms":70498,"credit_policy":null,"job_id":"job-226","job_issued_at_ms":70511,"miner_id":"miner-6","network_difficulty":1000005,"ntime":1700000226,"order_key":"06:miner-6","p2mr_program_hex":"e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2","share_difficulty":3,"share_id":"share-226","share_seq":226,"template_height":800007},{"accepted_at_ms":70510,"credit_policy":"stale-grace","job_id":"job-227","job_issued_at_ms":70483,"miner_id":"miner-7","network_difficulty":1000006,"ntime":1700000227,"order_key":"07:miner-7","p2mr_program_hex":"e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3","share_difficulty":7,"share_id":"share-227","share_seq":227,"template_height":800007},{"accepted_at_ms":70497,"credit_policy":null,"job_id":"job-228","job_issued_at_ms":70506,"miner_id":"miner-8","network_difficulty":1000007,"ntime":1700000228,"order_key":"08:miner-8","p2mr_program_hex":"e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4","share_difficulty":19,"share_id":"share-228","share_seq":228,"template_height":800007},{"accepted_at_ms":70493,"credit_policy":null,"job_id":"job-229","job_issued_at_ms":70488,"miner_id":"miner-9","network_difficulty":1000008,"ntime":1700000229,"order_key":"09:miner-9","p2mr_program_hex":"e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5","share_difficulty":1,"share_id":"share-229","share_seq":229,"template_height":800007},{"accepted_at_ms":70496,"credit_policy":null,"job_id":"job-230","job_issued_at_ms":70494,"miner_id":"miner-10","network_difficulty":1000009,"ntime":1700000230,"order_key":"10:miner-10","p2mr_program_hex":"e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6","share_difficulty":7,"share_id":"share-230","share_seq":230,"template_height":800007},{"accepted_at_ms":70508,"credit_policy":"stale-grace","job_id":"job-231","job_issued_at_ms":70490,"miner_id":"miner-0","network_difficulty":1000010,"ntime":1700000231,"order_key":"00:miner-0","p2mr_program_hex":"e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7","share_difficulty":7,"share_id":"share-231","share_seq":231,"template_height":800007},{"accepted_at_ms":70486,"credit_policy":null,"job_id":"job-232","job_issued_at_ms":70496,"miner_id":"miner-1","network_difficulty":1000011,"ntime":1700000232,"order_key":"01:miner-1","p2mr_program_hex":"e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8","share_difficulty":2,"share_id":"share-232","share_seq":232,"template_height":800007},{"accepted_at_ms":70500,"credit_policy":null,"job_id":"job-233","job_issued_at_ms":70500,"miner_id":"miner-2","network_difficulty":1000012,"ntime":1700000233,"order_key":"02:miner-2","p2mr_program_hex":"e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9","share_difficulty":1,"share_id":"share-233","share_seq":233,"template_height":800007},{"accepted_at_ms":70487,"credit_policy":null,"job_id":"job-234","job_issued_at_ms":70511,"miner_id":"miner-3","network_difficulty":1000013,"ntime":1700000234,"order_key":"03:miner-3","p2mr_program_hex":"eaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaea","share_difficulty":7,"share_id":"share-234","share_seq":234,"template_height":800007},{"accepted_at_ms":70511,"credit_policy":null,"job_id":"job-235","job_issued_at_ms":70495,"miner_id":"miner-4","network_difficulty":1000014,"ntime":1700000235,"order_key":"04:miner-4","p2mr_program_hex":"ebebebebebebebebebebebebebebebebebebebebebebebebebebebebebebebeb","share_difficulty":1,"share_id":"share-235","share_seq":235,"template_height":800007},{"accepted_at_ms":70510,"credit_policy":null,"job_id":"job-236","job_issued_at_ms":70501,"miner_id":"miner-5","network_difficulty":1000015,"ntime":1700000236,"order_key":"05:miner-5","p2mr_program_hex":"ecececececececececececececececececececececececececececececececec","share_difficulty":1,"share_id":"share-236","share_seq":236,"template_height":800007},{"accepted_at_ms":70510,"credit_policy":null,"job_id":"job-237","job_issued_at_ms":70501,"miner_id":"miner-6","network_difficulty":1000016,"ntime":1700000237,"order_key":"06:miner-6","p2mr_program_hex":"edededededededededededededededededededededededededededededededed","share_difficulty":1,"share_id":"share-237","share_seq":237,"template_height":800007},{"accepted_at_ms":70488,"credit_policy":"stale-grace","job_id":"job-238","job_issued_at_ms":70487,"miner_id":"miner-7","network_difficulty":1000000,"ntime":1700000238,"order_key":"07:miner-7","p2mr_program_hex":"eeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeee","share_difficulty":7,"share_id":"share-238","share_seq":238,"template_height":800007},{"accepted_at_ms":70502,"credit_policy":null,"job_id":"job-239","job_issued_at_ms":70498,"miner_id":"miner-8","network_difficulty":1000001,"ntime":1700000239,"order_key":"08:miner-8","p2mr_program_hex":"efefefefefefefefefefefefefefefefefefefefefefefefefefefefefefefef","share_difficulty":7,"share_id":"share-239","share_seq":239,"template_height":800007},{"accepted_at_ms":70483,"credit_policy":null,"job_id":"job-240","job_issued_at_ms":70508,"miner_id":"miner-9","network_difficulty":1000002,"ntime":1700000240,"order_key":"09:miner-9","p2mr_program_hex":"f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0","share_difficulty":7,"share_id":"share-240","share_seq":240,"template_height":800007},{"accepted_at_ms":70512,"credit_policy":"stale-grace","job_id":"job-241","job_issued_at_ms":70499,"miner_id":"miner-10","network_difficulty":1000003,"ntime":1700000241,"order_key":"10:miner-10","p2mr_program_hex":"f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1","share_difficulty":19,"share_id":"share-241","share_seq":241,"template_height":800007},{"accepted_at_ms":70495,"credit_policy":"stale-grace","job_id":"job-242","job_issued_at_ms":70511,"miner_id":"miner-0","network_difficulty":1000004,"ntime":1700000242,"order_key":"00:miner-0","p2mr_program_hex":"f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2","share_difficulty":1,"share_id":"share-242","share_seq":242,"template_height":800007},{"accepted_at_ms":70488,"credit_policy":"stale-grace","job_id":"job-243","job_issued_at_ms":70501,"miner_id":"miner-1","network_difficulty":1000005,"ntime":1700000243,"order_key":"01:miner-1","p2mr_program_hex":"f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3","share_difficulty":3,"share_id":"share-243","share_seq":243,"template_height":800007},{"accepted_at_ms":70494,"credit_policy":null,"job_id":"job-244","job_issued_at_ms":70492,"miner_id":"miner-2","network_difficulty":1000006,"ntime":1700000244,"order_key":"02:miner-2","p2mr_program_hex":"f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4","share_difficulty":2,"share_id":"share-244","share_seq":244,"template_height":800007},{"accepted_at_ms":70503,"credit_policy":null,"job_id":"job-245","job_issued_at_ms":70499,"miner_id":"miner-3","network_difficulty":1000007,"ntime":1700000245,"order_key":"03:miner-3","p2mr_program_hex":"f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5","share_difficulty":7,"share_id":"share-245","share_seq":245,"template_height":800007},{"accepted_at_ms":70487,"credit_policy":null,"job_id":"job-246","job_issued_at_ms":70506,"miner_id":"miner-4","network_difficulty":1000008,"ntime":1700000246,"order_key":"04:miner-4","p2mr_program_hex":"f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6","share_difficulty":1,"share_id":"share-246","share_seq":246,"template_height":800007},{"accepted_at_ms":70484,"credit_policy":null,"job_id":"job-247","job_issued_at_ms":70489,"miner_id":"miner-5","network_difficulty":1000009,"ntime":1700000247,"order_key":"05:miner-5","p2mr_program_hex":"f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7","share_difficulty":1,"share_id":"share-247","share_seq":247,"template_height":800007},{"accepted_at_ms":70512,"credit_policy":null,"job_id":"job-248","job_issued_at_ms":70484,"miner_id":"miner-6","network_difficulty":1000010,"ntime":1700000248,"order_key":"06:miner-6","p2mr_program_hex":"f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8","share_difficulty":1,"share_id":"share-248","share_seq":248,"template_height":800007},{"accepted_at_ms":70507,"credit_policy":"stale-grace","job_id":"job-249","job_issued_at_ms":70487,"miner_id":"miner-7","network_difficulty":1000011,"ntime":1700000249,"order_key":"07:miner-7","p2mr_program_hex":"f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9","share_difficulty":19,"share_id":"share-249","share_seq":249,"template_height":800007},{"accepted_at_ms":70511,"credit_policy":null,"job_id":"job-250","job_issued_at_ms":70499,"miner_id":"miner-8","network_difficulty":1000012,"ntime":1700000250,"order_key":"08:miner-8","p2mr_program_hex":"fafafafafafafafafafafafafafafafafafafafafafafafafafafafafafafafa","share_difficulty":7,"share_id":"share-250","share_seq":250,"template_height":800007},{"accepted_at_ms":70492,"credit_policy":null,"job_id":"job-251","job_issued_at_ms":70483,"miner_id":"miner-9","network_difficulty":1000013,"ntime":1700000251,"order_key":"09:miner-9","p2mr_program_hex":"fbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfb","share_difficulty":7,"share_id":"share-251","share_seq":251,"template_height":800007},{"accepted_at_ms":70495,"credit_policy":null,"job_id":"job-252","job_issued_at_ms":70503,"miner_id":"miner-10","network_difficulty":1000014,"ntime":1700000252,"order_key":"10:miner-10","p2mr_program_hex":"fcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfc","share_difficulty":7,"share_id":"share-252","share_seq":252,"template_height":800007},{"accepted_at_ms":70489,"credit_policy":"stale-grace","job_id":"job-253","job_issued_at_ms":70511,"miner_id":"miner-0","network_difficulty":1000015,"ntime":1700000253,"order_key":"00:miner-0","p2mr_program_hex":"fdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfd","share_difficulty":7,"share_id":"share-253","share_seq":253,"template_height":800007},{"accepted_at_ms":70506,"credit_policy":null,"job_id":"job-254","job_issued_at_ms":70483,"miner_id":"miner-1","network_difficulty":1000016,"ntime":1700000254,"order_key":"01:miner-1","p2mr_program_hex":"fefefefefefefefefefefefefefefefefefefefefefefefefefefefefefefefe","share_difficulty":19,"share_id":"share-254","share_seq":254,"template_height":800007},{"accepted_at_ms":70503,"credit_policy":null,"job_id":"job-255","job_issued_at_ms":70510,"miner_id":"miner-2","network_difficulty":1000000,"ntime":1700000255,"order_key":"02:miner-2","p2mr_program_hex":"ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff","share_difficulty":2,"share_id":"share-255","share_seq":255,"template_height":800007},{"accepted_at_ms":70510,"credit_policy":"stale-grace","job_id":"job-256","job_issued_at_ms":70501,"miner_id":"miner-3","network_difficulty":1000001,"ntime":1700000256,"order_key":"03:miner-3","p2mr_program_hex":"0000000000000000000000000000000000000000000000000000000000000000","share_difficulty":19,"share_id":"share-256","share_seq":256,"template_height":800008},{"accepted_at_ms":70484,"credit_policy":null,"job_id":"job-257","job_issued_at_ms":70483,"miner_id":"miner-4","network_difficulty":1000002,"ntime":1700000257,"order_key":"04:miner-4","p2mr_program_hex":"0101010101010101010101010101010101010101010101010101010101010101","share_difficulty":7,"share_id":"share-257","share_seq":257,"template_height":800008},{"accepted_at_ms":70509,"credit_policy":null,"job_id":"job-258","job_issued_at_ms":70491,"miner_id":"miner-5","network_difficulty":1000003,"ntime":1700000258,"order_key":"05:miner-5","p2mr_program_hex":"0202020202020202020202020202020202020202020202020202020202020202","share_difficulty":1,"share_id":"share-258","share_seq":258,"template_height":800008},{"accepted_at_ms":70504,"credit_policy":null,"job_id":"job-259","job_issued_at_ms":70497,"miner_id":"miner-6","network_difficulty":1000004,"ntime":1700000259,"order_key":"06:miner-6","p2mr_program_hex":"0303030303030303030303030303030303030303030303030303030303030303","share_difficulty":2,"share_id":"share-259","share_seq":259,"template_height":800008},{"accepted_at_ms":70508,"credit_policy":null,"job_id":"job-260","job_issued_at_ms":70501,"miner_id":"miner-7","network_difficulty":1000005,"ntime":1700000260,"order_key":"07:miner-7","p2mr_program_hex":"0404040404040404040404040404040404040404040404040404040404040404","share_difficulty":2,"share_id":"share-260","share_seq":260,"template_height":800008},{"accepted_at_ms":70512,"credit_policy":null,"job_id":"job-261","job_issued_at_ms":70499,"miner_id":"miner-8","network_difficulty":1000006,"ntime":1700000261,"order_key":"08:miner-8","p2mr_program_hex":"0505050505050505050505050505050505050505050505050505050505050505","share_difficulty":2,"share_id":"share-261","share_seq":261,"template_height":800008},{"accepted_at_ms":70498,"credit_policy":null,"job_id":"job-262","job_issued_at_ms":70490,"miner_id":"miner-9","network_difficulty":1000007,"ntime":1700000262,"order_key":"09:miner-9","p2mr_program_hex":"0606060606060606060606060606060606060606060606060606060606060606","share_difficulty":1,"share_id":"share-262","share_seq":262,"template_height":800008},{"accepted_at_ms":70505,"credit_policy":null,"job_id":"job-263","job_issued_at_ms":70485,"miner_id":"miner-10","network_difficulty":1000008,"ntime":1700000263,"order_key":"10:miner-10","p2mr_program_hex":"0707070707070707070707070707070707070707070707070707070707070707","share_difficulty":3,"share_id":"share-263","share_seq":263,"template_height":800008},{"accepted_at_ms":70511,"credit_policy":null,"job_id":"job-264","job_issued_at_ms":70510,"miner_id":"miner-0","network_difficulty":1000009,"ntime":1700000264,"order_key":"00:miner-0","p2mr_program_hex":"0808080808080808080808080808080808080808080808080808080808080808","share_difficulty":19,"share_id":"share-264","share_seq":264,"template_height":800008},{"accepted_at_ms":70500,"credit_policy":"stale-grace","job_id":"job-265","job_issued_at_ms":70499,"miner_id":"miner-1","network_difficulty":1000010,"ntime":1700000265,"order_key":"01:miner-1","p2mr_program_hex":"0909090909090909090909090909090909090909090909090909090909090909","share_difficulty":19,"share_id":"share-265","share_seq":265,"template_height":800008},{"accepted_at_ms":70511,"credit_policy":"stale-grace","job_id":"job-266","job_issued_at_ms":70488,"miner_id":"miner-2","network_difficulty":1000011,"ntime":1700000266,"order_key":"02:miner-2","p2mr_program_hex":"0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a","share_difficulty":19,"share_id":"share-266","share_seq":266,"template_height":800008},{"accepted_at_ms":70512,"credit_policy":"stale-grace","job_id":"job-267","job_issued_at_ms":70482,"miner_id":"miner-3","network_difficulty":1000012,"ntime":1700000267,"order_key":"03:miner-3","p2mr_program_hex":"0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b","share_difficulty":3,"share_id":"share-267","share_seq":267,"template_height":800008},{"accepted_at_ms":70486,"credit_policy":null,"job_id":"job-268","job_issued_at_ms":70497,"miner_id":"miner-4","network_difficulty":1000013,"ntime":1700000268,"order_key":"04:miner-4","p2mr_program_hex":"0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c","share_difficulty":1,"share_id":"share-268","share_seq":268,"template_height":800008},{"accepted_at_ms":70489,"credit_policy":"stale-grace","job_id":"job-269","job_issued_at_ms":70504,"miner_id":"miner-5","network_difficulty":1000014,"ntime":1700000269,"order_key":"05:miner-5","p2mr_program_hex":"0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d","share_difficulty":1,"share_id":"share-269","share_seq":269,"template_height":800008},{"accepted_at_ms":70489,"credit_policy":null,"job_id":"job-270","job_issued_at_ms":70511,"miner_id":"miner-6","network_difficulty":1000015,"ntime":1700000270,"order_key":"06:miner-6","p2mr_program_hex":"0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e","share_difficulty":3,"share_id":"share-270","share_seq":270,"template_height":800008},{"accepted_at_ms":70486,"credit_policy":null,"job_id":"job-271","job_issued_at_ms":70499,"miner_id":"miner-7","network_difficulty":1000016,"ntime":1700000271,"order_key":"07:miner-7","p2mr_program_hex":"0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f","share_difficulty":2,"share_id":"share-271","share_seq":271,"template_height":800008},{"accepted_at_ms":70510,"credit_policy":null,"job_id":"job-272","job_issued_at_ms":70512,"miner_id":"miner-8","network_difficulty":1000000,"ntime":1700000272,"order_key":"08:miner-8","p2mr_program_hex":"1010101010101010101010101010101010101010101010101010101010101010","share_difficulty":1,"share_id":"share-272","share_seq":272,"template_height":800008},{"accepted_at_ms":70482,"credit_policy":null,"job_id":"job-273","job_issued_at_ms":70508,"miner_id":"miner-9","network_difficulty":1000001,"ntime":1700000273,"order_key":"09:miner-9","p2mr_program_hex":"1111111111111111111111111111111111111111111111111111111111111111","share_difficulty":7,"share_id":"share-273","share_seq":273,"template_height":800008},{"accepted_at_ms":70483,"credit_policy":null,"job_id":"job-274","job_issued_at_ms":70486,"miner_id":"miner-10","network_difficulty":1000002,"ntime":1700000274,"order_key":"10:miner-10","p2mr_program_hex":"1212121212121212121212121212121212121212121212121212121212121212","share_difficulty":2,"share_id":"share-274","share_seq":274,"template_height":800008},{"accepted_at_ms":70484,"credit_policy":null,"job_id":"job-275","job_issued_at_ms":70506,"miner_id":"miner-0","network_difficulty":1000003,"ntime":1700000275,"order_key":"00:miner-0","p2mr_program_hex":"1313131313131313131313131313131313131313131313131313131313131313","share_difficulty":19,"share_id":"share-275","share_seq":275,"template_height":800008},{"accepted_at_ms":70511,"credit_policy":null,"job_id":"job-276","job_issued_at_ms":70495,"miner_id":"miner-1","network_difficulty":1000004,"ntime":1700000276,"order_key":"01:miner-1","p2mr_program_hex":"1414141414141414141414141414141414141414141414141414141414141414","share_difficulty":3,"share_id":"share-276","share_seq":276,"template_height":800008},{"accepted_at_ms":70509,"credit_policy":null,"job_id":"job-277","job_issued_at_ms":70487,"miner_id":"miner-2","network_difficulty":1000005,"ntime":1700000277,"order_key":"02:miner-2","p2mr_program_hex":"1515151515151515151515151515151515151515151515151515151515151515","share_difficulty":1,"share_id":"share-277","share_seq":277,"template_height":800008},{"accepted_at_ms":70489,"credit_policy":null,"job_id":"job-278","job_issued_at_ms":70509,"miner_id":"miner-3","network_difficulty":1000006,"ntime":1700000278,"order_key":"03:miner-3","p2mr_program_hex":"1616161616161616161616161616161616161616161616161616161616161616","share_difficulty":2,"share_id":"share-278","share_seq":278,"template_height":800008},{"accepted_at_ms":70491,"credit_policy":null,"job_id":"job-279","job_issued_at_ms":70497,"miner_id":"miner-4","network_difficulty":1000007,"ntime":1700000279,"order_key":"04:miner-4","p2mr_program_hex":"1717171717171717171717171717171717171717171717171717171717171717","share_difficulty":3,"share_id":"share-279","share_seq":279,"template_height":800008},{"accepted_at_ms":70483,"credit_policy":null,"job_id":"job-280","job_issued_at_ms":70482,"miner_id":"miner-5","network_difficulty":1000008,"ntime":1700000280,"order_key":"05:miner-5","p2mr_program_hex":"1818181818181818181818181818181818181818181818181818181818181818","share_difficulty":2,"share_id":"share-280","share_seq":280,"template_height":800008},{"accepted_at_ms":70482,"credit_policy":null,"job_id":"job-281","job_issued_at_ms":70511,"miner_id":"miner-6","network_difficulty":1000009,"ntime":1700000281,"order_key":"06:miner-6","p2mr_program_hex":"1919191919191919191919191919191919191919191919191919191919191919","share_difficulty":7,"share_id":"share-281","share_seq":281,"template_height":800008},{"accepted_at_ms":70498,"credit_policy":null,"job_id":"job-282","job_issued_at_ms":70500,"miner_id":"miner-7","network_difficulty":1000010,"ntime":1700000282,"order_key":"07:miner-7","p2mr_program_hex":"1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a","share_difficulty":19,"share_id":"share-282","share_seq":282,"template_height":800008},{"accepted_at_ms":70504,"credit_policy":null,"job_id":"job-283","job_issued_at_ms":70506,"miner_id":"miner-8","network_difficulty":1000011,"ntime":1700000283,"order_key":"08:miner-8","p2mr_program_hex":"1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b","share_difficulty":1,"share_id":"share-283","share_seq":283,"template_height":800008},{"accepted_at_ms":70484,"credit_policy":null,"job_id":"job-284","job_issued_at_ms":70500,"miner_id":"miner-9","network_difficulty":1000012,"ntime":1700000284,"order_key":"09:miner-9","p2mr_program_hex":"1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c","share_difficulty":2,"share_id":"share-284","share_seq":284,"template_height":800008},{"accepted_at_ms":70510,"credit_policy":"stale-grace","job_id":"job-285","job_issued_at_ms":70509,"miner_id":"miner-10","network_difficulty":1000013,"ntime":1700000285,"order_key":"10:miner-10","p2mr_program_hex":"1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d","share_difficulty":2,"share_id":"share-285","share_seq":285,"template_height":800008},{"accepted_at_ms":70499,"credit_policy":null,"job_id":"job-286","job_issued_at_ms":70490,"miner_id":"miner-0","network_difficulty":1000014,"ntime":1700000286,"order_key":"00:miner-0","p2mr_program_hex":"1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e","share_difficulty":2,"share_id":"share-286","share_seq":286,"template_height":800008},{"accepted_at_ms":70486,"credit_policy":null,"job_id":"job-287","job_issued_at_ms":70484,"miner_id":"miner-1","network_difficulty":1000015,"ntime":1700000287,"order_key":"01:miner-1","p2mr_program_hex":"1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f","share_difficulty":7,"share_id":"share-287","share_seq":287,"template_height":800008},{"accepted_at_ms":70500,"credit_policy":null,"job_id":"job-288","job_issued_at_ms":70492,"miner_id":"miner-2","network_difficulty":1000016,"ntime":1700000288,"order_key":"02:miner-2","p2mr_program_hex":"2020202020202020202020202020202020202020202020202020202020202020","share_difficulty":7,"share_id":"share-288","share_seq":288,"template_height":800009},{"accepted_at_ms":70482,"credit_policy":null,"job_id":"job-289","job_issued_at_ms":70498,"miner_id":"miner-3","network_difficulty":1000000,"ntime":1700000289,"order_key":"03:miner-3","p2mr_program_hex":"2121212121212121212121212121212121212121212121212121212121212121","share_difficulty":1,"share_id":"share-289","share_seq":289,"template_height":800009},{"accepted_at_ms":70506,"credit_policy":"stale-grace","job_id":"job-290","job_issued_at_ms":70506,"miner_id":"miner-4","network_difficulty":1000001,"ntime":1700000290,"order_key":"04:miner-4","p2mr_program_hex":"2222222222222222222222222222222222222222222222222222222222222222","share_difficulty":19,"share_id":"share-290","share_seq":290,"template_height":800009},{"accepted_at_ms":70496,"credit_policy":null,"job_id":"job-291","job_issued_at_ms":70509,"miner_id":"miner-5","network_difficulty":1000002,"ntime":1700000291,"order_key":"05:miner-5","p2mr_program_hex":"2323232323232323232323232323232323232323232323232323232323232323","share_difficulty":7,"share_id":"share-291","share_seq":291,"template_height":800009},{"accepted_at_ms":70492,"credit_policy":null,"job_id":"job-292","job_issued_at_ms":70511,"miner_id":"miner-6","network_difficulty":1000003,"ntime":1700000292,"order_key":"06:miner-6","p2mr_program_hex":"2424242424242424242424242424242424242424242424242424242424242424","share_difficulty":19,"share_id":"share-292","share_seq":292,"template_height":800009},{"accepted_at_ms":70507,"credit_policy":null,"job_id":"job-293","job_issued_at_ms":70498,"miner_id":"miner-7","network_difficulty":1000004,"ntime":1700000293,"order_key":"07:miner-7","p2mr_program_hex":"2525252525252525252525252525252525252525252525252525252525252525","share_difficulty":3,"share_id":"share-293","share_seq":293,"template_height":800009},{"accepted_at_ms":70502,"credit_policy":null,"job_id":"job-294","job_issued_at_ms":70491,"miner_id":"miner-8","network_difficulty":1000005,"ntime":1700000294,"order_key":"08:miner-8","p2mr_program_hex":"2626262626262626262626262626262626262626262626262626262626262626","share_difficulty":1,"share_id":"share-294","share_seq":294,"template_height":800009},{"accepted_at_ms":70511,"credit_policy":null,"job_id":"job-295","job_issued_at_ms":70487,"miner_id":"miner-9","network_difficulty":1000006,"ntime":1700000295,"order_key":"09:miner-9","p2mr_program_hex":"2727272727272727272727272727272727272727272727272727272727272727","share_difficulty":7,"share_id":"share-295","share_seq":295,"template_height":800009},{"accepted_at_ms":70509,"credit_policy":null,"job_id":"job-296","job_issued_at_ms":70497,"miner_id":"miner-10","network_difficulty":1000007,"ntime":1700000296,"order_key":"10:miner-10","p2mr_program_hex":"2828282828282828282828282828282828282828282828282828282828282828","share_difficulty":2,"share_id":"share-296","share_seq":296,"template_height":800009},{"accepted_at_ms":70500,"credit_policy":null,"job_id":"job-297","job_issued_at_ms":70493,"miner_id":"miner-0","network_difficulty":1000008,"ntime":1700000297,"order_key":"00:miner-0","p2mr_program_hex":"2929292929292929292929292929292929292929292929292929292929292929","share_difficulty":3,"share_id":"share-297","share_seq":297,"template_height":800009},{"accepted_at_ms":70483,"credit_policy":null,"job_id":"job-298","job_issued_at_ms":70509,"miner_id":"miner-1","network_difficulty":1000009,"ntime":1700000298,"order_key":"01:miner-1","p2mr_program_hex":"2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a","share_difficulty":3,"share_id":"share-298","share_seq":298,"template_height":800009},{"accepted_at_ms":70511,"credit_policy":null,"job_id":"job-299","job_issued_at_ms":70482,"miner_id":"miner-2","network_difficulty":1000010,"ntime":1700000299,"order_key":"02:miner-2","p2mr_program_hex":"2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b","share_difficulty":3,"share_id":"share-299","share_seq":299,"template_height":800009},{"accepted_at_ms":70511,"credit_policy":"stale-grace","job_id":"job-300","job_issued_at_ms":70488,"miner_id":"miner-3","network_difficulty":1000011,"ntime":1700000300,"order_key":"03:miner-3","p2mr_program_hex":"2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c","share_difficulty":2,"share_id":"share-300","share_seq":300,"template_height":800009},{"accepted_at_ms":70511,"credit_policy":"stale-grace","job_id":"job-301","job_issued_at_ms":70493,"miner_id":"miner-4","network_difficulty":1000012,"ntime":1700000301,"order_key":"04:miner-4","p2mr_program_hex":"2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d","share_difficulty":19,"share_id":"share-301","share_seq":301,"template_height":800009},{"accepted_at_ms":70493,"credit_policy":"stale-grace","job_id":"job-302","job_issued_at_ms":70486,"miner_id":"miner-5","network_difficulty":1000013,"ntime":1700000302,"order_key":"05:miner-5","p2mr_program_hex":"2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e","share_difficulty":7,"share_id":"share-302","share_seq":302,"template_height":800009},{"accepted_at_ms":70511,"credit_policy":null,"job_id":"job-303","job_issued_at_ms":70504,"miner_id":"miner-6","network_difficulty":1000014,"ntime":1700000303,"order_key":"06:miner-6","p2mr_program_hex":"2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f","share_difficulty":1,"share_id":"share-303","share_seq":303,"template_height":800009},{"accepted_at_ms":70493,"credit_policy":"stale-grace","job_id":"job-304","job_issued_at_ms":70510,"miner_id":"miner-7","network_difficulty":1000015,"ntime":1700000304,"order_key":"07:miner-7","p2mr_program_hex":"3030303030303030303030303030303030303030303030303030303030303030","share_difficulty":1,"share_id":"share-304","share_seq":304,"template_height":800009},{"accepted_at_ms":70501,"credit_policy":null,"job_id":"job-305","job_issued_at_ms":70484,"miner_id":"miner-8","network_difficulty":1000016,"ntime":1700000305,"order_key":"08:miner-8","p2mr_program_hex":"3131313131313131313131313131313131313131313131313131313131313131","share_difficulty":19,"share_id":"share-305","share_seq":305,"template_height":800009},{"accepted_at_ms":70490,"credit_policy":"stale-grace","job_id":"job-306","job_issued_at_ms":70505,"miner_id":"miner-9","network_difficulty":1000000,"ntime":1700000306,"order_key":"09:miner-9","p2mr_program_hex":"3232323232323232323232323232323232323232323232323232323232323232","share_difficulty":2,"share_id":"share-306","share_seq":306,"template_height":800009},{"accepted_at_ms":70510,"credit_policy":null,"job_id":"job-307","job_issued_at_ms":70485,"miner_id":"miner-10","network_difficulty":1000001,"ntime":1700000307,"order_key":"10:miner-10","p2mr_program_hex":"3333333333333333333333333333333333333333333333333333333333333333","share_difficulty":3,"share_id":"share-307","share_seq":307,"template_height":800009},{"accepted_at_ms":70505,"credit_policy":null,"job_id":"job-308","job_issued_at_ms":70509,"miner_id":"miner-0","network_difficulty":1000002,"ntime":1700000308,"order_key":"00:miner-0","p2mr_program_hex":"3434343434343434343434343434343434343434343434343434343434343434","share_difficulty":19,"share_id":"share-308","share_seq":308,"template_height":800009},{"accepted_at_ms":70509,"credit_policy":null,"job_id":"job-309","job_issued_at_ms":70496,"miner_id":"miner-1","network_difficulty":1000003,"ntime":1700000309,"order_key":"01:miner-1","p2mr_program_hex":"3535353535353535353535353535353535353535353535353535353535353535","share_difficulty":2,"share_id":"share-309","share_seq":309,"template_height":800009},{"accepted_at_ms":70503,"credit_policy":null,"job_id":"job-310","job_issued_at_ms":70482,"miner_id":"miner-2","network_difficulty":1000004,"ntime":1700000310,"order_key":"02:miner-2","p2mr_program_hex":"3636363636363636363636363636363636363636363636363636363636363636","share_difficulty":1,"share_id":"share-310","share_seq":310,"template_height":800009},{"accepted_at_ms":70512,"credit_policy":null,"job_id":"job-311","job_issued_at_ms":70487,"miner_id":"miner-3","network_difficulty":1000005,"ntime":1700000311,"order_key":"03:miner-3","p2mr_program_hex":"3737373737373737373737373737373737373737373737373737373737373737","share_difficulty":19,"share_id":"share-311","share_seq":311,"template_height":800009},{"accepted_at_ms":70491,"credit_policy":null,"job_id":"job-312","job_issued_at_ms":70488,"miner_id":"miner-4","network_difficulty":1000006,"ntime":1700000312,"order_key":"04:miner-4","p2mr_program_hex":"3838383838383838383838383838383838383838383838383838383838383838","share_difficulty":1,"share_id":"share-312","share_seq":312,"template_height":800009},{"accepted_at_ms":70496,"credit_policy":null,"job_id":"job-313","job_issued_at_ms":70482,"miner_id":"miner-5","network_difficulty":1000007,"ntime":1700000313,"order_key":"05:miner-5","p2mr_program_hex":"3939393939393939393939393939393939393939393939393939393939393939","share_difficulty":2,"share_id":"share-313","share_seq":313,"template_height":800009},{"accepted_at_ms":70483,"credit_policy":null,"job_id":"job-314","job_issued_at_ms":70482,"miner_id":"miner-6","network_difficulty":1000008,"ntime":1700000314,"order_key":"06:miner-6","p2mr_program_hex":"3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a","share_difficulty":7,"share_id":"share-314","share_seq":314,"template_height":800009},{"accepted_at_ms":70488,"credit_policy":null,"job_id":"job-315","job_issued_at_ms":70494,"miner_id":"miner-7","network_difficulty":1000009,"ntime":1700000315,"order_key":"07:miner-7","p2mr_program_hex":"3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b","share_difficulty":19,"share_id":"share-315","share_seq":315,"template_height":800009},{"accepted_at_ms":70489,"credit_policy":null,"job_id":"job-316","job_issued_at_ms":70488,"miner_id":"miner-8","network_difficulty":1000010,"ntime":1700000316,"order_key":"08:miner-8","p2mr_program_hex":"3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c","share_difficulty":7,"share_id":"share-316","share_seq":316,"template_height":800009},{"accepted_at_ms":70505,"credit_policy":null,"job_id":"job-317","job_issued_at_ms":70485,"miner_id":"miner-9","network_difficulty":1000011,"ntime":1700000317,"order_key":"09:miner-9","p2mr_program_hex":"3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d","share_difficulty":1,"share_id":"share-317","share_seq":317,"template_height":800009},{"accepted_at_ms":70501,"credit_policy":null,"job_id":"job-318","job_issued_at_ms":70512,"miner_id":"miner-10","network_difficulty":1000012,"ntime":1700000318,"order_key":"10:miner-10","p2mr_program_hex":"3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e","share_difficulty":2,"share_id":"share-318","share_seq":318,"template_height":800009},{"accepted_at_ms":70489,"credit_policy":null,"job_id":"job-319","job_issued_at_ms":70495,"miner_id":"miner-0","network_difficulty":1000013,"ntime":1700000319,"order_key":"00:miner-0","p2mr_program_hex":"3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f","share_difficulty":1,"share_id":"share-319","share_seq":319,"template_height":800009},{"accepted_at_ms":70511,"credit_policy":null,"job_id":"job-320","job_issued_at_ms":70502,"miner_id":"miner-1","network_difficulty":1000014,"ntime":1700000320,"order_key":"01:miner-1","p2mr_program_hex":"4040404040404040404040404040404040404040404040404040404040404040","share_difficulty":7,"share_id":"share-320","share_seq":320,"template_height":800010},{"accepted_at_ms":70502,"credit_policy":null,"job_id":"job-321","job_issued_at_ms":70510,"miner_id":"miner-2","network_difficulty":1000015,"ntime":1700000321,"order_key":"02:miner-2","p2mr_program_hex":"4141414141414141414141414141414141414141414141414141414141414141","share_difficulty":2,"share_id":"share-321","share_seq":321,"template_height":800010},{"accepted_at_ms":70512,"credit_policy":null,"job_id":"job-322","job_issued_at_ms":70509,"miner_id":"miner-3","network_difficulty":1000016,"ntime":1700000322,"order_key":"03:miner-3","p2mr_program_hex":"4242424242424242424242424242424242424242424242424242424242424242","share_difficulty":2,"share_id":"share-322","share_seq":322,"template_height":800010},{"accepted_at_ms":70488,"credit_policy":"stale-grace","job_id":"job-323","job_issued_at_ms":70483,"miner_id":"miner-4","network_difficulty":1000000,"ntime":1700000323,"order_key":"04:miner-4","p2mr_program_hex":"4343434343434343434343434343434343434343434343434343434343434343","share_difficulty":1,"share_id":"share-323","share_seq":323,"template_height":800010},{"accepted_at_ms":70492,"credit_policy":null,"job_id":"job-324","job_issued_at_ms":70489,"miner_id":"miner-5","network_difficulty":1000001,"ntime":1700000324,"order_key":"05:miner-5","p2mr_program_hex":"4444444444444444444444444444444444444444444444444444444444444444","share_difficulty":7,"share_id":"share-324","share_seq":324,"template_height":800010},{"accepted_at_ms":70493,"credit_policy":null,"job_id":"job-325","job_issued_at_ms":70507,"miner_id":"miner-6","network_difficulty":1000002,"ntime":1700000325,"order_key":"06:miner-6","p2mr_program_hex":"4545454545454545454545454545454545454545454545454545454545454545","share_difficulty":19,"share_id":"share-325","share_seq":325,"template_height":800010},{"accepted_at_ms":70512,"credit_policy":"stale-grace","job_id":"job-326","job_issued_at_ms":70487,"miner_id":"miner-7","network_difficulty":1000003,"ntime":1700000326,"order_key":"07:miner-7","p2mr_program_hex":"4646464646464646464646464646464646464646464646464646464646464646","share_difficulty":2,"share_id":"share-326","share_seq":326,"template_height":800010},{"accepted_at_ms":70482,"credit_policy":null,"job_id":"job-327","job_issued_at_ms":70507,"miner_id":"miner-8","network_difficulty":1000004,"ntime":1700000327,"order_key":"08:miner-8","p2mr_program_hex":"4747474747474747474747474747474747474747474747474747474747474747","share_difficulty":19,"share_id":"share-327","share_seq":327,"template_height":800010},{"accepted_at_ms":70512,"credit_policy":null,"job_id":"job-328","job_issued_at_ms":70500,"miner_id":"miner-9","network_difficulty":1000005,"ntime":1700000328,"order_key":"09:miner-9","p2mr_program_hex":"4848484848484848484848484848484848484848484848484848484848484848","share_difficulty":1,"share_id":"share-328","share_seq":328,"template_height":800010},{"accepted_at_ms":70504,"credit_policy":"stale-grace","job_id":"job-329","job_issued_at_ms":70499,"miner_id":"miner-10","network_difficulty":1000006,"ntime":1700000329,"order_key":"10:miner-10","p2mr_program_hex":"4949494949494949494949494949494949494949494949494949494949494949","share_difficulty":1,"share_id":"share-329","share_seq":329,"template_height":800010},{"accepted_at_ms":70490,"credit_policy":null,"job_id":"job-330","job_issued_at_ms":70490,"miner_id":"miner-0","network_difficulty":1000007,"ntime":1700000330,"order_key":"00:miner-0","p2mr_program_hex":"4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a","share_difficulty":19,"share_id":"share-330","share_seq":330,"template_height":800010},{"accepted_at_ms":70510,"credit_policy":null,"job_id":"job-331","job_issued_at_ms":70498,"miner_id":"miner-1","network_difficulty":1000008,"ntime":1700000331,"order_key":"01:miner-1","p2mr_program_hex":"4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b","share_difficulty":2,"share_id":"share-331","share_seq":331,"template_height":800010},{"accepted_at_ms":70497,"credit_policy":null,"job_id":"job-332","job_issued_at_ms":70483,"miner_id":"miner-2","network_difficulty":1000009,"ntime":1700000332,"order_key":"02:miner-2","p2mr_program_hex":"4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c","share_difficulty":3,"share_id":"share-332","share_seq":332,"template_height":800010},{"accepted_at_ms":70498,"credit_policy":"stale-grace","job_id":"job-333","job_issued_at_ms":70496,"miner_id":"miner-3","network_difficulty":1000010,"ntime":1700000333,"order_key":"03:miner-3","p2mr_program_hex":"4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d","share_difficulty":7,"share_id":"share-333","share_seq":333,"template_height":800010},{"accepted_at_ms":70484,"credit_policy":null,"job_id":"job-334","job_issued_at_ms":70494,"miner_id":"miner-4","network_difficulty":1000011,"ntime":1700000334,"order_key":"04:miner-4","p2mr_program_hex":"4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e","share_difficulty":7,"share_id":"share-334","share_seq":334,"template_height":800010},{"accepted_at_ms":70487,"credit_policy":null,"job_id":"job-335","job_issued_at_ms":70506,"miner_id":"miner-5","network_difficulty":1000012,"ntime":1700000335,"order_key":"05:miner-5","p2mr_program_hex":"4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f","share_difficulty":3,"share_id":"share-335","share_seq":335,"template_height":800010},{"accepted_at_ms":70509,"credit_policy":null,"job_id":"job-336","job_issued_at_ms":70484,"miner_id":"miner-6","network_difficulty":1000013,"ntime":1700000336,"order_key":"06:miner-6","p2mr_program_hex":"5050505050505050505050505050505050505050505050505050505050505050","share_difficulty":19,"share_id":"share-336","share_seq":336,"template_height":800010},{"accepted_at_ms":70512,"credit_policy":null,"job_id":"job-337","job_issued_at_ms":70488,"miner_id":"miner-7","network_difficulty":1000014,"ntime":1700000337,"order_key":"07:miner-7","p2mr_program_hex":"5151515151515151515151515151515151515151515151515151515151515151","share_difficulty":19,"share_id":"share-337","share_seq":337,"template_height":800010},{"accepted_at_ms":70484,"credit_policy":null,"job_id":"job-338","job_issued_at_ms":70501,"miner_id":"miner-8","network_difficulty":1000015,"ntime":1700000338,"order_key":"08:miner-8","p2mr_program_hex":"5252525252525252525252525252525252525252525252525252525252525252","share_difficulty":3,"share_id":"share-338","share_seq":338,"template_height":800010},{"accepted_at_ms":70482,"credit_policy":null,"job_id":"job-339","job_issued_at_ms":70508,"miner_id":"miner-9","network_difficulty":1000016,"ntime":1700000339,"order_key":"09:miner-9","p2mr_program_hex":"5353535353535353535353535353535353535353535353535353535353535353","share_difficulty":3,"share_id":"share-339","share_seq":339,"template_height":800010},{"accepted_at_ms":70504,"credit_policy":null,"job_id":"job-340","job_issued_at_ms":70483,"miner_id":"miner-10","network_difficulty":1000000,"ntime":1700000340,"order_key":"10:miner-10","p2mr_program_hex":"5454545454545454545454545454545454545454545454545454545454545454","share_difficulty":1,"share_id":"share-340","share_seq":340,"template_height":800010},{"accepted_at_ms":70504,"credit_policy":null,"job_id":"job-341","job_issued_at_ms":70503,"miner_id":"miner-0","network_difficulty":1000001,"ntime":1700000341,"order_key":"00:miner-0","p2mr_program_hex":"5555555555555555555555555555555555555555555555555555555555555555","share_difficulty":1,"share_id":"share-341","share_seq":341,"template_height":800010},{"accepted_at_ms":70506,"credit_policy":null,"job_id":"job-342","job_issued_at_ms":70505,"miner_id":"miner-1","network_difficulty":1000002,"ntime":1700000342,"order_key":"01:miner-1","p2mr_program_hex":"5656565656565656565656565656565656565656565656565656565656565656","share_difficulty":3,"share_id":"share-342","share_seq":342,"template_height":800010},{"accepted_at_ms":70489,"credit_policy":null,"job_id":"job-343","job_issued_at_ms":70500,"miner_id":"miner-2","network_difficulty":1000003,"ntime":1700000343,"order_key":"02:miner-2","p2mr_program_hex":"5757575757575757575757575757575757575757575757575757575757575757","share_difficulty":7,"share_id":"share-343","share_seq":343,"template_height":800010},{"accepted_at_ms":70493,"credit_policy":null,"job_id":"job-344","job_issued_at_ms":70507,"miner_id":"miner-3","network_difficulty":1000004,"ntime":1700000344,"order_key":"03:miner-3","p2mr_program_hex":"5858585858585858585858585858585858585858585858585858585858585858","share_difficulty":2,"share_id":"share-344","share_seq":344,"template_height":800010},{"accepted_at_ms":70496,"credit_policy":null,"job_id":"job-345","job_issued_at_ms":70486,"miner_id":"miner-4","network_difficulty":1000005,"ntime":1700000345,"order_key":"04:miner-4","p2mr_program_hex":"5959595959595959595959595959595959595959595959595959595959595959","share_difficulty":19,"share_id":"share-345","share_seq":345,"template_height":800010},{"accepted_at_ms":70493,"credit_policy":null,"job_id":"job-346","job_issued_at_ms":70492,"miner_id":"miner-5","network_difficulty":1000006,"ntime":1700000346,"order_key":"05:miner-5","p2mr_program_hex":"5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a","share_difficulty":1,"share_id":"share-346","share_seq":346,"template_height":800010},{"accepted_at_ms":70494,"credit_policy":null,"job_id":"job-347","job_issued_at_ms":70486,"miner_id":"miner-6","network_difficulty":1000007,"ntime":1700000347,"order_key":"06:miner-6","p2mr_program_hex":"5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b","share_difficulty":1,"share_id":"share-347","share_seq":347,"template_height":800010},{"accepted_at_ms":70511,"credit_policy":null,"job_id":"job-348","job_issued_at_ms":70507,"miner_id":"miner-7","network_difficulty":1000008,"ntime":1700000348,"order_key":"07:miner-7","p2mr_program_hex":"5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c","share_difficulty":19,"share_id":"share-348","share_seq":348,"template_height":800010},{"accepted_at_ms":70510,"credit_policy":null,"job_id":"job-349","job_issued_at_ms":70491,"miner_id":"miner-8","network_difficulty":1000009,"ntime":1700000349,"order_key":"08:miner-8","p2mr_program_hex":"5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d","share_difficulty":19,"share_id":"share-349","share_seq":349,"template_height":800010},{"accepted_at_ms":70494,"credit_policy":null,"job_id":"job-350","job_issued_at_ms":70483,"miner_id":"miner-9","network_difficulty":1000010,"ntime":1700000350,"order_key":"09:miner-9","p2mr_program_hex":"5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e","share_difficulty":1,"share_id":"share-350","share_seq":350,"template_height":800010},{"accepted_at_ms":70512,"credit_policy":"stale-grace","job_id":"job-351","job_issued_at_ms":70510,"miner_id":"miner-10","network_difficulty":1000011,"ntime":1700000351,"order_key":"10:miner-10","p2mr_program_hex":"5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f","share_difficulty":1,"share_id":"share-351","share_seq":351,"template_height":800010},{"accepted_at_ms":70508,"credit_policy":null,"job_id":"job-352","job_issued_at_ms":70493,"miner_id":"miner-0","network_difficulty":1000012,"ntime":1700000352,"order_key":"00:miner-0","p2mr_program_hex":"6060606060606060606060606060606060606060606060606060606060606060","share_difficulty":3,"share_id":"share-352","share_seq":352,"template_height":800011},{"accepted_at_ms":70496,"credit_policy":null,"job_id":"job-353","job_issued_at_ms":70489,"miner_id":"miner-1","network_difficulty":1000013,"ntime":1700000353,"order_key":"01:miner-1","p2mr_program_hex":"6161616161616161616161616161616161616161616161616161616161616161","share_difficulty":2,"share_id":"share-353","share_seq":353,"template_height":800011},{"accepted_at_ms":70497,"credit_policy":null,"job_id":"job-354","job_issued_at_ms":70502,"miner_id":"miner-2","network_difficulty":1000014,"ntime":1700000354,"order_key":"02:miner-2","p2mr_program_hex":"6262626262626262626262626262626262626262626262626262626262626262","share_difficulty":3,"share_id":"share-354","share_seq":354,"template_height":800011},{"accepted_at_ms":70496,"credit_policy":null,"job_id":"job-355","job_issued_at_ms":70506,"miner_id":"miner-3","network_difficulty":1000015,"ntime":1700000355,"order_key":"03:miner-3","p2mr_program_hex":"6363636363636363636363636363636363636363636363636363636363636363","share_difficulty":1,"share_id":"share-355","share_seq":355,"template_height":800011},{"accepted_at_ms":70494,"credit_policy":null,"job_id":"job-356","job_issued_at_ms":70502,"miner_id":"miner-4","network_difficulty":1000016,"ntime":1700000356,"order_key":"04:miner-4","p2mr_program_hex":"6464646464646464646464646464646464646464646464646464646464646464","share_difficulty":19,"share_id":"share-356","share_seq":356,"template_height":800011},{"accepted_at_ms":70498,"credit_policy":null,"job_id":"job-357","job_issued_at_ms":70488,"miner_id":"miner-5","network_difficulty":1000000,"ntime":1700000357,"order_key":"05:miner-5","p2mr_program_hex":"6565656565656565656565656565656565656565656565656565656565656565","share_difficulty":3,"share_id":"share-357","share_seq":357,"template_height":800011},{"accepted_at_ms":70501,"credit_policy":null,"job_id":"job-358","job_issued_at_ms":70484,"miner_id":"miner-6","network_difficulty":1000001,"ntime":1700000358,"order_key":"06:miner-6","p2mr_program_hex":"6666666666666666666666666666666666666666666666666666666666666666","share_difficulty":3,"share_id":"share-358","share_seq":358,"template_height":800011},{"accepted_at_ms":70485,"credit_policy":null,"job_id":"job-359","job_issued_at_ms":70504,"miner_id":"miner-7","network_difficulty":1000002,"ntime":1700000359,"order_key":"07:miner-7","p2mr_program_hex":"6767676767676767676767676767676767676767676767676767676767676767","share_difficulty":3,"share_id":"share-359","share_seq":359,"template_height":800011},{"accepted_at_ms":70502,"credit_policy":null,"job_id":"job-360","job_issued_at_ms":70489,"miner_id":"miner-8","network_difficulty":1000003,"ntime":1700000360,"order_key":"08:miner-8","p2mr_program_hex":"6868686868686868686868686868686868686868686868686868686868686868","share_difficulty":7,"share_id":"share-360","share_seq":360,"template_height":800011},{"accepted_at_ms":70495,"credit_policy":null,"job_id":"job-361","job_issued_at_ms":70491,"miner_id":"miner-9","network_difficulty":1000004,"ntime":1700000361,"order_key":"09:miner-9","p2mr_program_hex":"6969696969696969696969696969696969696969696969696969696969696969","share_difficulty":1,"share_id":"share-361","share_seq":361,"template_height":800011},{"accepted_at_ms":70492,"credit_policy":null,"job_id":"job-362","job_issued_at_ms":70512,"miner_id":"miner-10","network_difficulty":1000005,"ntime":1700000362,"order_key":"10:miner-10","p2mr_program_hex":"6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a","share_difficulty":1,"share_id":"share-362","share_seq":362,"template_height":800011},{"accepted_at_ms":70501,"credit_policy":null,"job_id":"job-363","job_issued_at_ms":70506,"miner_id":"miner-0","network_difficulty":1000006,"ntime":1700000363,"order_key":"00:miner-0","p2mr_program_hex":"6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b","share_difficulty":1,"share_id":"share-363","share_seq":363,"template_height":800011},{"accepted_at_ms":70499,"credit_policy":null,"job_id":"job-364","job_issued_at_ms":70500,"miner_id":"miner-1","network_difficulty":1000007,"ntime":1700000364,"order_key":"01:miner-1","p2mr_program_hex":"6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c","share_difficulty":19,"share_id":"share-364","share_seq":364,"template_height":800011},{"accepted_at_ms":70504,"credit_policy":"stale-grace","job_id":"job-365","job_issued_at_ms":70488,"miner_id":"miner-2","network_difficulty":1000008,"ntime":1700000365,"order_key":"02:miner-2","p2mr_program_hex":"6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d","share_difficulty":1,"share_id":"share-365","share_seq":365,"template_height":800011},{"accepted_at_ms":70492,"credit_policy":"stale-grace","job_id":"job-366","job_issued_at_ms":70495,"miner_id":"miner-3","network_difficulty":1000009,"ntime":1700000366,"order_key":"03:miner-3","p2mr_program_hex":"6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e","share_difficulty":2,"share_id":"share-366","share_seq":366,"template_height":800011},{"accepted_at_ms":70491,"credit_policy":null,"job_id":"job-367","job_issued_at_ms":70494,"miner_id":"miner-4","network_difficulty":1000010,"ntime":1700000367,"order_key":"04:miner-4","p2mr_program_hex":"6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f","share_difficulty":19,"share_id":"share-367","share_seq":367,"template_height":800011},{"accepted_at_ms":70486,"credit_policy":"stale-grace","job_id":"job-368","job_issued_at_ms":70496,"miner_id":"miner-5","network_difficulty":1000011,"ntime":1700000368,"order_key":"05:miner-5","p2mr_program_hex":"7070707070707070707070707070707070707070707070707070707070707070","share_difficulty":7,"share_id":"share-368","share_seq":368,"template_height":800011},{"accepted_at_ms":70508,"credit_policy":null,"job_id":"job-369","job_issued_at_ms":70484,"miner_id":"miner-6","network_difficulty":1000012,"ntime":1700000369,"order_key":"06:miner-6","p2mr_program_hex":"7171717171717171717171717171717171717171717171717171717171717171","share_difficulty":3,"share_id":"share-369","share_seq":369,"template_height":800011},{"accepted_at_ms":70497,"credit_policy":"stale-grace","job_id":"job-370","job_issued_at_ms":70512,"miner_id":"miner-7","network_difficulty":1000013,"ntime":1700000370,"order_key":"07:miner-7","p2mr_program_hex":"7272727272727272727272727272727272727272727272727272727272727272","share_difficulty":2,"share_id":"share-370","share_seq":370,"template_height":800011},{"accepted_at_ms":70510,"credit_policy":null,"job_id":"job-371","job_issued_at_ms":70509,"miner_id":"miner-8","network_difficulty":1000014,"ntime":1700000371,"order_key":"08:miner-8","p2mr_program_hex":"7373737373737373737373737373737373737373737373737373737373737373","share_difficulty":3,"share_id":"share-371","share_seq":371,"template_height":800011},{"accepted_at_ms":70486,"credit_policy":"stale-grace","job_id":"job-372","job_issued_at_ms":70496,"miner_id":"miner-9","network_difficulty":1000015,"ntime":1700000372,"order_key":"09:miner-9","p2mr_program_hex":"7474747474747474747474747474747474747474747474747474747474747474","share_difficulty":1,"share_id":"share-372","share_seq":372,"template_height":800011},{"accepted_at_ms":70508,"credit_policy":null,"job_id":"job-373","job_issued_at_ms":70497,"miner_id":"miner-10","network_difficulty":1000016,"ntime":1700000373,"order_key":"10:miner-10","p2mr_program_hex":"7575757575757575757575757575757575757575757575757575757575757575","share_difficulty":7,"share_id":"share-373","share_seq":373,"template_height":800011},{"accepted_at_ms":70486,"credit_policy":"stale-grace","job_id":"job-374","job_issued_at_ms":70500,"miner_id":"miner-0","network_difficulty":1000000,"ntime":1700000374,"order_key":"00:miner-0","p2mr_program_hex":"7676767676767676767676767676767676767676767676767676767676767676","share_difficulty":3,"share_id":"share-374","share_seq":374,"template_height":800011},{"accepted_at_ms":70505,"credit_policy":"stale-grace","job_id":"job-375","job_issued_at_ms":70498,"miner_id":"miner-1","network_difficulty":1000001,"ntime":1700000375,"order_key":"01:miner-1","p2mr_program_hex":"7777777777777777777777777777777777777777777777777777777777777777","share_difficulty":1,"share_id":"share-375","share_seq":375,"template_height":800011},{"accepted_at_ms":70496,"credit_policy":null,"job_id":"job-376","job_issued_at_ms":70497,"miner_id":"miner-2","network_difficulty":1000002,"ntime":1700000376,"order_key":"02:miner-2","p2mr_program_hex":"7878787878787878787878787878787878787878787878787878787878787878","share_difficulty":2,"share_id":"share-376","share_seq":376,"template_height":800011},{"accepted_at_ms":70511,"credit_policy":null,"job_id":"job-377","job_issued_at_ms":70511,"miner_id":"miner-3","network_difficulty":1000003,"ntime":1700000377,"order_key":"03:miner-3","p2mr_program_hex":"7979797979797979797979797979797979797979797979797979797979797979","share_difficulty":7,"share_id":"share-377","share_seq":377,"template_height":800011},{"accepted_at_ms":70495,"credit_policy":null,"job_id":"job-378","job_issued_at_ms":70487,"miner_id":"miner-4","network_difficulty":1000004,"ntime":1700000378,"order_key":"04:miner-4","p2mr_program_hex":"7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a","share_difficulty":7,"share_id":"share-378","share_seq":378,"template_height":800011},{"accepted_at_ms":70495,"credit_policy":null,"job_id":"job-379","job_issued_at_ms":70490,"miner_id":"miner-5","network_difficulty":1000005,"ntime":1700000379,"order_key":"05:miner-5","p2mr_program_hex":"7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b","share_difficulty":3,"share_id":"share-379","share_seq":379,"template_height":800011},{"accepted_at_ms":70509,"credit_policy":"stale-grace","job_id":"job-380","job_issued_at_ms":70511,"miner_id":"miner-6","network_difficulty":1000006,"ntime":1700000380,"order_key":"06:miner-6","p2mr_program_hex":"7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c","share_difficulty":1,"share_id":"share-380","share_seq":380,"template_height":800011},{"accepted_at_ms":70500,"credit_policy":null,"job_id":"job-381","job_issued_at_ms":70497,"miner_id":"miner-7","network_difficulty":1000007,"ntime":1700000381,"order_key":"07:miner-7","p2mr_program_hex":"7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d","share_difficulty":7,"share_id":"share-381","share_seq":381,"template_height":800011},{"accepted_at_ms":70511,"credit_policy":null,"job_id":"job-382","job_issued_at_ms":70498,"miner_id":"miner-8","network_difficulty":1000008,"ntime":1700000382,"order_key":"08:miner-8","p2mr_program_hex":"7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e","share_difficulty":2,"share_id":"share-382","share_seq":382,"template_height":800011},{"accepted_at_ms":70507,"credit_policy":"stale-grace","job_id":"job-383","job_issued_at_ms":70489,"miner_id":"miner-9","network_difficulty":1000009,"ntime":1700000383,"order_key":"09:miner-9","p2mr_program_hex":"7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f","share_difficulty":2,"share_id":"share-383","share_seq":383,"template_height":800011},{"accepted_at_ms":70490,"credit_policy":"stale-grace","job_id":"job-384","job_issued_at_ms":70504,"miner_id":"miner-10","network_difficulty":1000010,"ntime":1700000384,"order_key":"10:miner-10","p2mr_program_hex":"8080808080808080808080808080808080808080808080808080808080808080","share_difficulty":7,"share_id":"share-384","share_seq":384,"template_height":800012},{"accepted_at_ms":70495,"credit_policy":null,"job_id":"job-385","job_issued_at_ms":70491,"miner_id":"miner-0","network_difficulty":1000011,"ntime":1700000385,"order_key":"00:miner-0","p2mr_program_hex":"8181818181818181818181818181818181818181818181818181818181818181","share_difficulty":19,"share_id":"share-385","share_seq":385,"template_height":800012},{"accepted_at_ms":70493,"credit_policy":null,"job_id":"job-386","job_issued_at_ms":70489,"miner_id":"miner-1","network_difficulty":1000012,"ntime":1700000386,"order_key":"01:miner-1","p2mr_program_hex":"8282828282828282828282828282828282828282828282828282828282828282","share_difficulty":19,"share_id":"share-386","share_seq":386,"template_height":800012},{"accepted_at_ms":70494,"credit_policy":"stale-grace","job_id":"job-387","job_issued_at_ms":70500,"miner_id":"miner-2","network_difficulty":1000013,"ntime":1700000387,"order_key":"02:miner-2","p2mr_program_hex":"8383838383838383838383838383838383838383838383838383838383838383","share_difficulty":1,"share_id":"share-387","share_seq":387,"template_height":800012},{"accepted_at_ms":70502,"credit_policy":null,"job_id":"job-388","job_issued_at_ms":70488,"miner_id":"miner-3","network_difficulty":1000014,"ntime":1700000388,"order_key":"03:miner-3","p2mr_program_hex":"8484848484848484848484848484848484848484848484848484848484848484","share_difficulty":1,"share_id":"share-388","share_seq":388,"template_height":800012},{"accepted_at_ms":70489,"credit_policy":null,"job_id":"job-389","job_issued_at_ms":70493,"miner_id":"miner-4","network_difficulty":1000015,"ntime":1700000389,"order_key":"04:miner-4","p2mr_program_hex":"8585858585858585858585858585858585858585858585858585858585858585","share_difficulty":7,"share_id":"share-389","share_seq":389,"template_height":800012},{"accepted_at_ms":70499,"credit_policy":"stale-grace","job_id":"job-390","job_issued_at_ms":70499,"miner_id":"miner-5","network_difficulty":1000016,"ntime":1700000390,"order_key":"05:miner-5","p2mr_program_hex":"8686868686868686868686868686868686868686868686868686868686868686","share_difficulty":19,"share_id":"share-390","share_seq":390,"template_height":800012},{"accepted_at_ms":70485,"credit_policy":null,"job_id":"job-391","job_issued_at_ms":70490,"miner_id":"miner-6","network_difficulty":1000000,"ntime":1700000391,"order_key":"06:miner-6","p2mr_program_hex":"8787878787878787878787878787878787878787878787878787878787878787","share_difficulty":2,"share_id":"share-391","share_seq":391,"template_height":800012},{"accepted_at_ms":70483,"credit_policy":null,"job_id":"job-392","job_issued_at_ms":70494,"miner_id":"miner-7","network_difficulty":1000001,"ntime":1700000392,"order_key":"07:miner-7","p2mr_program_hex":"8888888888888888888888888888888888888888888888888888888888888888","share_difficulty":19,"share_id":"share-392","share_seq":392,"template_height":800012},{"accepted_at_ms":70501,"credit_policy":null,"job_id":"job-393","job_issued_at_ms":70485,"miner_id":"miner-8","network_difficulty":1000002,"ntime":1700000393,"order_key":"08:miner-8","p2mr_program_hex":"8989898989898989898989898989898989898989898989898989898989898989","share_difficulty":19,"share_id":"share-393","share_seq":393,"template_height":800012},{"accepted_at_ms":70495,"credit_policy":null,"job_id":"job-394","job_issued_at_ms":70508,"miner_id":"miner-9","network_difficulty":1000003,"ntime":1700000394,"order_key":"09:miner-9","p2mr_program_hex":"8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a","share_difficulty":2,"share_id":"share-394","share_seq":394,"template_height":800012},{"accepted_at_ms":70495,"credit_policy":"stale-grace","job_id":"job-395","job_issued_at_ms":70500,"miner_id":"miner-10","network_difficulty":1000004,"ntime":1700000395,"order_key":"10:miner-10","p2mr_program_hex":"8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b","share_difficulty":19,"share_id":"share-395","share_seq":395,"template_height":800012},{"accepted_at_ms":70483,"credit_policy":null,"job_id":"job-396","job_issued_at_ms":70499,"miner_id":"miner-0","network_difficulty":1000005,"ntime":1700000396,"order_key":"00:miner-0","p2mr_program_hex":"8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c","share_difficulty":1,"share_id":"share-396","share_seq":396,"template_height":800012},{"accepted_at_ms":70491,"credit_policy":null,"job_id":"job-397","job_issued_at_ms":70486,"miner_id":"miner-1","network_difficulty":1000006,"ntime":1700000397,"order_key":"01:miner-1","p2mr_program_hex":"8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d","share_difficulty":2,"share_id":"share-397","share_seq":397,"template_height":800012},{"accepted_at_ms":70505,"credit_policy":null,"job_id":"job-398","job_issued_at_ms":70503,"miner_id":"miner-2","network_difficulty":1000007,"ntime":1700000398,"order_key":"02:miner-2","p2mr_program_hex":"8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e","share_difficulty":1,"share_id":"share-398","share_seq":398,"template_height":800012},{"accepted_at_ms":70490,"credit_policy":null,"job_id":"job-399","job_issued_at_ms":70487,"miner_id":"miner-3","network_difficulty":1000008,"ntime":1700000399,"order_key":"03:miner-3","p2mr_program_hex":"8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f","share_difficulty":3,"share_id":"share-399","share_seq":399,"template_height":800012},{"accepted_at_ms":70494,"credit_policy":"stale-grace","job_id":"job-400","job_issued_at_ms":70512,"miner_id":"miner-4","network_difficulty":1000009,"ntime":1700000400,"order_key":"04:miner-4","p2mr_program_hex":"9090909090909090909090909090909090909090909090909090909090909090","share_difficulty":7,"share_id":"share-400","share_seq":400,"template_height":800012},{"accepted_at_ms":70503,"credit_policy":null,"job_id":"job-401","job_issued_at_ms":70487,"miner_id":"miner-5","network_difficulty":1000010,"ntime":1700000401,"order_key":"05:miner-5","p2mr_program_hex":"9191919191919191919191919191919191919191919191919191919191919191","share_difficulty":7,"share_id":"share-401","share_seq":401,"template_height":800012},{"accepted_at_ms":70493,"credit_policy":null,"job_id":"job-402","job_issued_at_ms":70507,"miner_id":"miner-6","network_difficulty":1000011,"ntime":1700000402,"order_key":"06:miner-6","p2mr_program_hex":"9292929292929292929292929292929292929292929292929292929292929292","share_difficulty":19,"share_id":"share-402","share_seq":402,"template_height":800012},{"accepted_at_ms":70487,"credit_policy":"stale-grace","job_id":"job-403","job_issued_at_ms":70482,"miner_id":"miner-7","network_difficulty":1000012,"ntime":1700000403,"order_key":"07:miner-7","p2mr_program_hex":"9393939393939393939393939393939393939393939393939393939393939393","share_difficulty":3,"share_id":"share-403","share_seq":403,"template_height":800012},{"accepted_at_ms":70505,"credit_policy":"stale-grace","job_id":"job-404","job_issued_at_ms":70489,"miner_id":"miner-8","network_difficulty":1000013,"ntime":1700000404,"order_key":"08:miner-8","p2mr_program_hex":"9494949494949494949494949494949494949494949494949494949494949494","share_difficulty":7,"share_id":"share-404","share_seq":404,"template_height":800012},{"accepted_at_ms":70483,"credit_policy":null,"job_id":"job-405","job_issued_at_ms":70495,"miner_id":"miner-9","network_difficulty":1000014,"ntime":1700000405,"order_key":"09:miner-9","p2mr_program_hex":"9595959595959595959595959595959595959595959595959595959595959595","share_difficulty":1,"share_id":"share-405","share_seq":405,"template_height":800012},{"accepted_at_ms":70510,"credit_policy":null,"job_id":"job-406","job_issued_at_ms":70488,"miner_id":"miner-10","network_difficulty":1000015,"ntime":1700000406,"order_key":"10:miner-10","p2mr_program_hex":"9696969696969696969696969696969696969696969696969696969696969696","share_difficulty":1,"share_id":"share-406","share_seq":406,"template_height":800012},{"accepted_at_ms":70487,"credit_policy":null,"job_id":"job-407","job_issued_at_ms":70509,"miner_id":"miner-0","network_difficulty":1000016,"ntime":1700000407,"order_key":"00:miner-0","p2mr_program_hex":"9797979797979797979797979797979797979797979797979797979797979797","share_difficulty":7,"share_id":"share-407","share_seq":407,"template_height":800012},{"accepted_at_ms":70487,"credit_policy":"stale-grace","job_id":"job-408","job_issued_at_ms":70505,"miner_id":"miner-1","network_difficulty":1000000,"ntime":1700000408,"order_key":"01:miner-1","p2mr_program_hex":"9898989898989898989898989898989898989898989898989898989898989898","share_difficulty":2,"share_id":"share-408","share_seq":408,"template_height":800012},{"accepted_at_ms":70488,"credit_policy":"stale-grace","job_id":"job-409","job_issued_at_ms":70487,"miner_id":"miner-2","network_difficulty":1000001,"ntime":1700000409,"order_key":"02:miner-2","p2mr_program_hex":"9999999999999999999999999999999999999999999999999999999999999999","share_difficulty":7,"share_id":"share-409","share_seq":409,"template_height":800012},{"accepted_at_ms":70484,"credit_policy":null,"job_id":"job-410","job_issued_at_ms":70501,"miner_id":"miner-3","network_difficulty":1000002,"ntime":1700000410,"order_key":"03:miner-3","p2mr_program_hex":"9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a","share_difficulty":2,"share_id":"share-410","share_seq":410,"template_height":800012},{"accepted_at_ms":70511,"credit_policy":"stale-grace","job_id":"job-411","job_issued_at_ms":70500,"miner_id":"miner-4","network_difficulty":1000003,"ntime":1700000411,"order_key":"04:miner-4","p2mr_program_hex":"9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b","share_difficulty":2,"share_id":"share-411","share_seq":411,"template_height":800012},{"accepted_at_ms":70501,"credit_policy":"stale-grace","job_id":"job-412","job_issued_at_ms":70507,"miner_id":"miner-5","network_difficulty":1000004,"ntime":1700000412,"order_key":"05:miner-5","p2mr_program_hex":"9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c","share_difficulty":3,"share_id":"share-412","share_seq":412,"template_height":800012},{"accepted_at_ms":70486,"credit_policy":null,"job_id":"job-413","job_issued_at_ms":70483,"miner_id":"miner-6","network_difficulty":1000005,"ntime":1700000413,"order_key":"06:miner-6","p2mr_program_hex":"9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d","share_difficulty":7,"share_id":"share-413","share_seq":413,"template_height":800012},{"accepted_at_ms":70506,"credit_policy":"stale-grace","job_id":"job-414","job_issued_at_ms":70496,"miner_id":"miner-7","network_difficulty":1000006,"ntime":1700000414,"order_key":"07:miner-7","p2mr_program_hex":"9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e","share_difficulty":2,"share_id":"share-414","share_seq":414,"template_height":800012},{"accepted_at_ms":70488,"credit_policy":null,"job_id":"job-415","job_issued_at_ms":70486,"miner_id":"miner-8","network_difficulty":1000007,"ntime":1700000415,"order_key":"08:miner-8","p2mr_program_hex":"9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f","share_difficulty":2,"share_id":"share-415","share_seq":415,"template_height":800012},{"accepted_at_ms":70503,"credit_policy":null,"job_id":"job-416","job_issued_at_ms":70487,"miner_id":"miner-9","network_difficulty":1000008,"ntime":1700000416,"order_key":"09:miner-9","p2mr_program_hex":"a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0","share_difficulty":7,"share_id":"share-416","share_seq":416,"template_height":800013},{"accepted_at_ms":70498,"credit_policy":null,"job_id":"job-417","job_issued_at_ms":70494,"miner_id":"miner-10","network_difficulty":1000009,"ntime":1700000417,"order_key":"10:miner-10","p2mr_program_hex":"a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1","share_difficulty":19,"share_id":"share-417","share_seq":417,"template_height":800013},{"accepted_at_ms":70503,"credit_policy":null,"job_id":"job-418","job_issued_at_ms":70504,"miner_id":"miner-0","network_difficulty":1000010,"ntime":1700000418,"order_key":"00:miner-0","p2mr_program_hex":"a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2","share_difficulty":3,"share_id":"share-418","share_seq":418,"template_height":800013},{"accepted_at_ms":70502,"credit_policy":"stale-grace","job_id":"job-419","job_issued_at_ms":70485,"miner_id":"miner-1","network_difficulty":1000011,"ntime":1700000419,"order_key":"01:miner-1","p2mr_program_hex":"a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3","share_difficulty":3,"share_id":"share-419","share_seq":419,"template_height":800013},{"accepted_at_ms":70487,"credit_policy":null,"job_id":"job-420","job_issued_at_ms":70487,"miner_id":"miner-2","network_difficulty":1000012,"ntime":1700000420,"order_key":"02:miner-2","p2mr_program_hex":"a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4","share_difficulty":7,"share_id":"share-420","share_seq":420,"template_height":800013},{"accepted_at_ms":70507,"credit_policy":"stale-grace","job_id":"job-421","job_issued_at_ms":70512,"miner_id":"miner-3","network_difficulty":1000013,"ntime":1700000421,"order_key":"03:miner-3","p2mr_program_hex":"a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5","share_difficulty":2,"share_id":"share-421","share_seq":421,"template_height":800013},{"accepted_at_ms":70499,"credit_policy":"stale-grace","job_id":"job-422","job_issued_at_ms":70503,"miner_id":"miner-4","network_difficulty":1000014,"ntime":1700000422,"order_key":"04:miner-4","p2mr_program_hex":"a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6","share_difficulty":19,"share_id":"share-422","share_seq":422,"template_height":800013},{"accepted_at_ms":70485,"credit_policy":null,"job_id":"job-423","job_issued_at_ms":70497,"miner_id":"miner-5","network_difficulty":1000015,"ntime":1700000423,"order_key":"05:miner-5","p2mr_program_hex":"a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7","share_difficulty":3,"share_id":"share-423","share_seq":423,"template_height":800013},{"accepted_at_ms":70500,"credit_policy":"stale-grace","job_id":"job-424","job_issued_at_ms":70508,"miner_id":"miner-6","network_difficulty":1000016,"ntime":1700000424,"order_key":"06:miner-6","p2mr_program_hex":"a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8","share_difficulty":7,"share_id":"share-424","share_seq":424,"template_height":800013},{"accepted_at_ms":70508,"credit_policy":"stale-grace","job_id":"job-425","job_issued_at_ms":70496,"miner_id":"miner-7","network_difficulty":1000000,"ntime":1700000425,"order_key":"07:miner-7","p2mr_program_hex":"a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9","share_difficulty":1,"share_id":"share-425","share_seq":425,"template_height":800013},{"accepted_at_ms":70482,"credit_policy":null,"job_id":"job-426","job_issued_at_ms":70491,"miner_id":"miner-8","network_difficulty":1000001,"ntime":1700000426,"order_key":"08:miner-8","p2mr_program_hex":"aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa","share_difficulty":2,"share_id":"share-426","share_seq":426,"template_height":800013},{"accepted_at_ms":70482,"credit_policy":null,"job_id":"job-427","job_issued_at_ms":70508,"miner_id":"miner-9","network_difficulty":1000002,"ntime":1700000427,"order_key":"09:miner-9","p2mr_program_hex":"abababababababababababababababababababababababababababababababab","share_difficulty":3,"share_id":"share-427","share_seq":427,"template_height":800013},{"accepted_at_ms":70507,"credit_policy":null,"job_id":"job-428","job_issued_at_ms":70498,"miner_id":"miner-10","network_difficulty":1000003,"ntime":1700000428,"order_key":"10:miner-10","p2mr_program_hex":"acacacacacacacacacacacacacacacacacacacacacacacacacacacacacacacac","share_difficulty":2,"share_id":"share-428","share_seq":428,"template_height":800013},{"accepted_at_ms":70485,"credit_policy":"stale-grace","job_id":"job-429","job_issued_at_ms":70505,"miner_id":"miner-0","network_difficulty":1000004,"ntime":1700000429,"order_key":"00:miner-0","p2mr_program_hex":"adadadadadadadadadadadadadadadadadadadadadadadadadadadadadadadad","share_difficulty":3,"share_id":"share-429","share_seq":429,"template_height":800013},{"accepted_at_ms":70488,"credit_policy":"stale-grace","job_id":"job-430","job_issued_at_ms":70483,"miner_id":"miner-1","network_difficulty":1000005,"ntime":1700000430,"order_key":"01:miner-1","p2mr_program_hex":"aeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeae","share_difficulty":3,"share_id":"share-430","share_seq":430,"template_height":800013},{"accepted_at_ms":70509,"credit_policy":"stale-grace","job_id":"job-431","job_issued_at_ms":70503,"miner_id":"miner-2","network_difficulty":1000006,"ntime":1700000431,"order_key":"02:miner-2","p2mr_program_hex":"afafafafafafafafafafafafafafafafafafafafafafafafafafafafafafafaf","share_difficulty":1,"share_id":"share-431","share_seq":431,"template_height":800013},{"accepted_at_ms":70485,"credit_policy":null,"job_id":"job-432","job_issued_at_ms":70489,"miner_id":"miner-3","network_difficulty":1000007,"ntime":1700000432,"order_key":"03:miner-3","p2mr_program_hex":"b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0","share_difficulty":2,"share_id":"share-432","share_seq":432,"template_height":800013},{"accepted_at_ms":70491,"credit_policy":null,"job_id":"job-433","job_issued_at_ms":70487,"miner_id":"miner-4","network_difficulty":1000008,"ntime":1700000433,"order_key":"04:miner-4","p2mr_program_hex":"b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1","share_difficulty":7,"share_id":"share-433","share_seq":433,"template_height":800013},{"accepted_at_ms":70498,"credit_policy":"stale-grace","job_id":"job-434","job_issued_at_ms":70507,"miner_id":"miner-5","network_difficulty":1000009,"ntime":1700000434,"order_key":"05:miner-5","p2mr_program_hex":"b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2","share_difficulty":7,"share_id":"share-434","share_seq":434,"template_height":800013},{"accepted_at_ms":70490,"credit_policy":"stale-grace","job_id":"job-435","job_issued_at_ms":70501,"miner_id":"miner-6","network_difficulty":1000010,"ntime":1700000435,"order_key":"06:miner-6","p2mr_program_hex":"b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3","share_difficulty":19,"share_id":"share-435","share_seq":435,"template_height":800013},{"accepted_at_ms":70498,"credit_policy":null,"job_id":"job-436","job_issued_at_ms":70504,"miner_id":"miner-7","network_difficulty":1000011,"ntime":1700000436,"order_key":"07:miner-7","p2mr_program_hex":"b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4","share_difficulty":3,"share_id":"share-436","share_seq":436,"template_height":800013},{"accepted_at_ms":70498,"credit_policy":null,"job_id":"job-437","job_issued_at_ms":70497,"miner_id":"miner-8","network_difficulty":1000012,"ntime":1700000437,"order_key":"08:miner-8","p2mr_program_hex":"b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5","share_difficulty":3,"share_id":"share-437","share_seq":437,"template_height":800013},{"accepted_at_ms":70491,"credit_policy":null,"job_id":"job-438","job_issued_at_ms":70499,"miner_id":"miner-9","network_difficulty":1000013,"ntime":1700000438,"order_key":"09:miner-9","p2mr_program_hex":"b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6","share_difficulty":7,"share_id":"share-438","share_seq":438,"template_height":800013},{"accepted_at_ms":70501,"credit_policy":null,"job_id":"job-439","job_issued_at_ms":70486,"miner_id":"miner-10","network_difficulty":1000014,"ntime":1700000439,"order_key":"10:miner-10","p2mr_program_hex":"b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7","share_difficulty":7,"share_id":"share-439","share_seq":439,"template_height":800013},{"accepted_at_ms":70492,"credit_policy":"stale-grace","job_id":"job-440","job_issued_at_ms":70505,"miner_id":"miner-0","network_difficulty":1000015,"ntime":1700000440,"order_key":"00:miner-0","p2mr_program_hex":"b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8","share_difficulty":19,"share_id":"share-440","share_seq":440,"template_height":800013},{"accepted_at_ms":70484,"credit_policy":null,"job_id":"job-441","job_issued_at_ms":70512,"miner_id":"miner-1","network_difficulty":1000016,"ntime":1700000441,"order_key":"01:miner-1","p2mr_program_hex":"b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9","share_difficulty":19,"share_id":"share-441","share_seq":441,"template_height":800013},{"accepted_at_ms":70484,"credit_policy":null,"job_id":"job-442","job_issued_at_ms":70490,"miner_id":"miner-2","network_difficulty":1000000,"ntime":1700000442,"order_key":"02:miner-2","p2mr_program_hex":"babababababababababababababababababababababababababababababababa","share_difficulty":1,"share_id":"share-442","share_seq":442,"template_height":800013},{"accepted_at_ms":70495,"credit_policy":null,"job_id":"job-443","job_issued_at_ms":70491,"miner_id":"miner-3","network_difficulty":1000001,"ntime":1700000443,"order_key":"03:miner-3","p2mr_program_hex":"bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb","share_difficulty":3,"share_id":"share-443","share_seq":443,"template_height":800013},{"accepted_at_ms":70511,"credit_policy":"stale-grace","job_id":"job-444","job_issued_at_ms":70482,"miner_id":"miner-4","network_difficulty":1000002,"ntime":1700000444,"order_key":"04:miner-4","p2mr_program_hex":"bcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbc","share_difficulty":7,"share_id":"share-444","share_seq":444,"template_height":800013},{"accepted_at_ms":70499,"credit_policy":null,"job_id":"job-445","job_issued_at_ms":70490,"miner_id":"miner-5","network_difficulty":1000003,"ntime":1700000445,"order_key":"05:miner-5","p2mr_program_hex":"bdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbd","share_difficulty":7,"share_id":"share-445","share_seq":445,"template_height":800013},{"accepted_at_ms":70502,"credit_policy":null,"job_id":"job-446","job_issued_at_ms":70505,"miner_id":"miner-6","network_difficulty":1000004,"ntime":1700000446,"order_key":"06:miner-6","p2mr_program_hex":"bebebebebebebebebebebebebebebebebebebebebebebebebebebebebebebebe","share_difficulty":2,"share_id":"share-446","share_seq":446,"template_height":800013},{"accepted_at_ms":70509,"credit_policy":null,"job_id":"job-447","job_issued_at_ms":70507,"miner_id":"miner-7","network_difficulty":1000005,"ntime":1700000447,"order_key":"07:miner-7","p2mr_program_hex":"bfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbf","share_difficulty":3,"share_id":"share-447","share_seq":447,"template_height":800013},{"accepted_at_ms":70497,"credit_policy":null,"job_id":"job-448","job_issued_at_ms":70492,"miner_id":"miner-8","network_difficulty":1000006,"ntime":1700000448,"order_key":"08:miner-8","p2mr_program_hex":"c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0","share_difficulty":19,"share_id":"share-448","share_seq":448,"template_height":800014},{"accepted_at_ms":70509,"credit_policy":"stale-grace","job_id":"job-449","job_issued_at_ms":70504,"miner_id":"miner-9","network_difficulty":1000007,"ntime":1700000449,"order_key":"09:miner-9","p2mr_program_hex":"c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1","share_difficulty":1,"share_id":"share-449","share_seq":449,"template_height":800014},{"accepted_at_ms":70486,"credit_policy":null,"job_id":"job-450","job_issued_at_ms":70498,"miner_id":"miner-10","network_difficulty":1000008,"ntime":1700000450,"order_key":"10:miner-10","p2mr_program_hex":"c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2","share_difficulty":2,"share_id":"share-450","share_seq":450,"template_height":800014},{"accepted_at_ms":70490,"credit_policy":null,"job_id":"job-451","job_issued_at_ms":70510,"miner_id":"miner-0","network_difficulty":1000009,"ntime":1700000451,"order_key":"00:miner-0","p2mr_program_hex":"c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3","share_difficulty":1,"share_id":"share-451","share_seq":451,"template_height":800014},{"accepted_at_ms":70490,"credit_policy":null,"job_id":"job-452","job_issued_at_ms":70490,"miner_id":"miner-1","network_difficulty":1000010,"ntime":1700000452,"order_key":"01:miner-1","p2mr_program_hex":"c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4","share_difficulty":3,"share_id":"share-452","share_seq":452,"template_height":800014},{"accepted_at_ms":70491,"credit_policy":null,"job_id":"job-453","job_issued_at_ms":70495,"miner_id":"miner-2","network_difficulty":1000011,"ntime":1700000453,"order_key":"02:miner-2","p2mr_program_hex":"c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5","share_difficulty":19,"share_id":"share-453","share_seq":453,"template_height":800014},{"accepted_at_ms":70496,"credit_policy":null,"job_id":"job-454","job_issued_at_ms":70489,"miner_id":"miner-3","network_difficulty":1000012,"ntime":1700000454,"order_key":"03:miner-3","p2mr_program_hex":"c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6","share_difficulty":7,"share_id":"share-454","share_seq":454,"template_height":800014},{"accepted_at_ms":70511,"credit_policy":null,"job_id":"job-455","job_issued_at_ms":70502,"miner_id":"miner-4","network_difficulty":1000013,"ntime":1700000455,"order_key":"04:miner-4","p2mr_program_hex":"c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7","share_difficulty":3,"share_id":"share-455","share_seq":455,"template_height":800014},{"accepted_at_ms":70507,"credit_policy":null,"job_id":"job-456","job_issued_at_ms":70507,"miner_id":"miner-5","network_difficulty":1000014,"ntime":1700000456,"order_key":"05:miner-5","p2mr_program_hex":"c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8","share_difficulty":19,"share_id":"share-456","share_seq":456,"template_height":800014},{"accepted_at_ms":70489,"credit_policy":null,"job_id":"job-457","job_issued_at_ms":70482,"miner_id":"miner-6","network_difficulty":1000015,"ntime":1700000457,"order_key":"06:miner-6","p2mr_program_hex":"c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9","share_difficulty":2,"share_id":"share-457","share_seq":457,"template_height":800014},{"accepted_at_ms":70482,"credit_policy":null,"job_id":"job-458","job_issued_at_ms":70483,"miner_id":"miner-7","network_difficulty":1000016,"ntime":1700000458,"order_key":"07:miner-7","p2mr_program_hex":"cacacacacacacacacacacacacacacacacacacacacacacacacacacacacacacaca","share_difficulty":1,"share_id":"share-458","share_seq":458,"template_height":800014},{"accepted_at_ms":70493,"credit_policy":null,"job_id":"job-459","job_issued_at_ms":70483,"miner_id":"miner-8","network_difficulty":1000000,"ntime":1700000459,"order_key":"08:miner-8","p2mr_program_hex":"cbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcb","share_difficulty":3,"share_id":"share-459","share_seq":459,"template_height":800014},{"accepted_at_ms":70504,"credit_policy":null,"job_id":"job-460","job_issued_at_ms":70495,"miner_id":"miner-9","network_difficulty":1000001,"ntime":1700000460,"order_key":"09:miner-9","p2mr_program_hex":"cccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccc","share_difficulty":19,"share_id":"share-460","share_seq":460,"template_height":800014},{"accepted_at_ms":70501,"credit_policy":null,"job_id":"job-461","job_issued_at_ms":70500,"miner_id":"miner-10","network_difficulty":1000002,"ntime":1700000461,"order_key":"10:miner-10","p2mr_program_hex":"cdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcd","share_difficulty":7,"share_id":"share-461","share_seq":461,"template_height":800014},{"accepted_at_ms":70504,"credit_policy":null,"job_id":"job-462","job_issued_at_ms":70509,"miner_id":"miner-0","network_difficulty":1000003,"ntime":1700000462,"order_key":"00:miner-0","p2mr_program_hex":"cececececececececececececececececececececececececececececececece","share_difficulty":19,"share_id":"share-462","share_seq":462,"template_height":800014},{"accepted_at_ms":70506,"credit_policy":null,"job_id":"job-463","job_issued_at_ms":70498,"miner_id":"miner-1","network_difficulty":1000004,"ntime":1700000463,"order_key":"01:miner-1","p2mr_program_hex":"cfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcf","share_difficulty":2,"share_id":"share-463","share_seq":463,"template_height":800014},{"accepted_at_ms":70504,"credit_policy":null,"job_id":"job-464","job_issued_at_ms":70485,"miner_id":"miner-2","network_difficulty":1000005,"ntime":1700000464,"order_key":"02:miner-2","p2mr_program_hex":"d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0","share_difficulty":19,"share_id":"share-464","share_seq":464,"template_height":800014},{"accepted_at_ms":70503,"credit_policy":"stale-grace","job_id":"job-465","job_issued_at_ms":70491,"miner_id":"miner-3","network_difficulty":1000006,"ntime":1700000465,"order_key":"03:miner-3","p2mr_program_hex":"d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1","share_difficulty":3,"share_id":"share-465","share_seq":465,"template_height":800014},{"accepted_at_ms":70505,"credit_policy":null,"job_id":"job-466","job_issued_at_ms":70485,"miner_id":"miner-4","network_difficulty":1000007,"ntime":1700000466,"order_key":"04:miner-4","p2mr_program_hex":"d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2","share_difficulty":2,"share_id":"share-466","share_seq":466,"template_height":800014},{"accepted_at_ms":70491,"credit_policy":"stale-grace","job_id":"job-467","job_issued_at_ms":70510,"miner_id":"miner-5","network_difficulty":1000008,"ntime":1700000467,"order_key":"05:miner-5","p2mr_program_hex":"d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3","share_difficulty":7,"share_id":"share-467","share_seq":467,"template_height":800014},{"accepted_at_ms":70486,"credit_policy":null,"job_id":"job-468","job_issued_at_ms":70482,"miner_id":"miner-6","network_difficulty":1000009,"ntime":1700000468,"order_key":"06:miner-6","p2mr_program_hex":"d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4","share_difficulty":2,"share_id":"share-468","share_seq":468,"template_height":800014},{"accepted_at_ms":70504,"credit_policy":null,"job_id":"job-469","job_issued_at_ms":70503,"miner_id":"miner-7","network_difficulty":1000010,"ntime":1700000469,"order_key":"07:miner-7","p2mr_program_hex":"d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5","share_difficulty":19,"share_id":"share-469","share_seq":469,"template_height":800014},{"accepted_at_ms":70504,"credit_policy":null,"job_id":"job-470","job_issued_at_ms":70512,"miner_id":"miner-8","network_difficulty":1000011,"ntime":1700000470,"order_key":"08:miner-8","p2mr_program_hex":"d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6","share_difficulty":3,"share_id":"share-470","share_seq":470,"template_height":800014},{"accepted_at_ms":70487,"credit_policy":"stale-grace","job_id":"job-471","job_issued_at_ms":70507,"miner_id":"miner-9","network_difficulty":1000012,"ntime":1700000471,"order_key":"09:miner-9","p2mr_program_hex":"d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7","share_difficulty":7,"share_id":"share-471","share_seq":471,"template_height":800014},{"accepted_at_ms":70492,"credit_policy":"stale-grace","job_id":"job-472","job_issued_at_ms":70492,"miner_id":"miner-10","network_difficulty":1000013,"ntime":1700000472,"order_key":"10:miner-10","p2mr_program_hex":"d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8","share_difficulty":1,"share_id":"share-472","share_seq":472,"template_height":800014},{"accepted_at_ms":70506,"credit_policy":null,"job_id":"job-473","job_issued_at_ms":70490,"miner_id":"miner-0","network_difficulty":1000014,"ntime":1700000473,"order_key":"00:miner-0","p2mr_program_hex":"d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9","share_difficulty":19,"share_id":"share-473","share_seq":473,"template_height":800014},{"accepted_at_ms":70483,"credit_policy":null,"job_id":"job-474","job_issued_at_ms":70487,"miner_id":"miner-1","network_difficulty":1000015,"ntime":1700000474,"order_key":"01:miner-1","p2mr_program_hex":"dadadadadadadadadadadadadadadadadadadadadadadadadadadadadadadada","share_difficulty":1,"share_id":"share-474","share_seq":474,"template_height":800014},{"accepted_at_ms":70485,"credit_policy":"stale-grace","job_id":"job-475","job_issued_at_ms":70499,"miner_id":"miner-2","network_difficulty":1000016,"ntime":1700000475,"order_key":"02:miner-2","p2mr_program_hex":"dbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdb","share_difficulty":19,"share_id":"share-475","share_seq":475,"template_height":800014},{"accepted_at_ms":70503,"credit_policy":"stale-grace","job_id":"job-476","job_issued_at_ms":70502,"miner_id":"miner-3","network_difficulty":1000000,"ntime":1700000476,"order_key":"03:miner-3","p2mr_program_hex":"dcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdc","share_difficulty":1,"share_id":"share-476","share_seq":476,"template_height":800014},{"accepted_at_ms":70508,"credit_policy":null,"job_id":"job-477","job_issued_at_ms":70509,"miner_id":"miner-4","network_difficulty":1000001,"ntime":1700000477,"order_key":"04:miner-4","p2mr_program_hex":"dddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddd","share_difficulty":7,"share_id":"share-477","share_seq":477,"template_height":800014},{"accepted_at_ms":70510,"credit_policy":null,"job_id":"job-478","job_issued_at_ms":70495,"miner_id":"miner-5","network_difficulty":1000002,"ntime":1700000478,"order_key":"05:miner-5","p2mr_program_hex":"dededededededededededededededededededededededededededededededede","share_difficulty":7,"share_id":"share-478","share_seq":478,"template_height":800014},{"accepted_at_ms":70509,"credit_policy":null,"job_id":"job-479","job_issued_at_ms":70502,"miner_id":"miner-6","network_difficulty":1000003,"ntime":1700000479,"order_key":"06:miner-6","p2mr_program_hex":"dfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdf","share_difficulty":2,"share_id":"share-479","share_seq":479,"template_height":800014},{"accepted_at_ms":70490,"credit_policy":"stale-grace","job_id":"job-480","job_issued_at_ms":70503,"miner_id":"miner-7","network_difficulty":1000004,"ntime":1700000480,"order_key":"07:miner-7","p2mr_program_hex":"e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0","share_difficulty":1,"share_id":"share-480","share_seq":480,"template_height":800015},{"accepted_at_ms":70507,"credit_policy":null,"job_id":"job-481","job_issued_at_ms":70511,"miner_id":"miner-8","network_difficulty":1000005,"ntime":1700000481,"order_key":"08:miner-8","p2mr_program_hex":"e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1","share_difficulty":7,"share_id":"share-481","share_seq":481,"template_height":800015},{"accepted_at_ms":70507,"credit_policy":null,"job_id":"job-482","job_issued_at_ms":70502,"miner_id":"miner-9","network_difficulty":1000006,"ntime":1700000482,"order_key":"09:miner-9","p2mr_program_hex":"e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2","share_difficulty":7,"share_id":"share-482","share_seq":482,"template_height":800015},{"accepted_at_ms":70485,"credit_policy":null,"job_id":"job-483","job_issued_at_ms":70493,"miner_id":"miner-10","network_difficulty":1000007,"ntime":1700000483,"order_key":"10:miner-10","p2mr_program_hex":"e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3","share_difficulty":7,"share_id":"share-483","share_seq":483,"template_height":800015},{"accepted_at_ms":70495,"credit_policy":null,"job_id":"job-484","job_issued_at_ms":70489,"miner_id":"miner-0","network_difficulty":1000008,"ntime":1700000484,"order_key":"00:miner-0","p2mr_program_hex":"e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4","share_difficulty":19,"share_id":"share-484","share_seq":484,"template_height":800015},{"accepted_at_ms":70501,"credit_policy":null,"job_id":"job-485","job_issued_at_ms":70483,"miner_id":"miner-1","network_difficulty":1000009,"ntime":1700000485,"order_key":"01:miner-1","p2mr_program_hex":"e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5","share_difficulty":19,"share_id":"share-485","share_seq":485,"template_height":800015},{"accepted_at_ms":70487,"credit_policy":null,"job_id":"job-486","job_issued_at_ms":70499,"miner_id":"miner-2","network_difficulty":1000010,"ntime":1700000486,"order_key":"02:miner-2","p2mr_program_hex":"e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6","share_difficulty":3,"share_id":"share-486","share_seq":486,"template_height":800015},{"accepted_at_ms":70504,"credit_policy":null,"job_id":"job-487","job_issued_at_ms":70507,"miner_id":"miner-3","network_difficulty":1000011,"ntime":1700000487,"order_key":"03:miner-3","p2mr_program_hex":"e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7","share_difficulty":19,"share_id":"share-487","share_seq":487,"template_height":800015},{"accepted_at_ms":70485,"credit_policy":null,"job_id":"job-488","job_issued_at_ms":70483,"miner_id":"miner-4","network_difficulty":1000012,"ntime":1700000488,"order_key":"04:miner-4","p2mr_program_hex":"e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8","share_difficulty":7,"share_id":"share-488","share_seq":488,"template_height":800015},{"accepted_at_ms":70497,"credit_policy":"stale-grace","job_id":"job-489","job_issued_at_ms":70501,"miner_id":"miner-5","network_difficulty":1000013,"ntime":1700000489,"order_key":"05:miner-5","p2mr_program_hex":"e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9","share_difficulty":2,"share_id":"share-489","share_seq":489,"template_height":800015},{"accepted_at_ms":70504,"credit_policy":null,"job_id":"job-490","job_issued_at_ms":70504,"miner_id":"miner-6","network_difficulty":1000014,"ntime":1700000490,"order_key":"06:miner-6","p2mr_program_hex":"eaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaea","share_difficulty":3,"share_id":"share-490","share_seq":490,"template_height":800015},{"accepted_at_ms":70511,"credit_policy":null,"job_id":"job-491","job_issued_at_ms":70506,"miner_id":"miner-7","network_difficulty":1000015,"ntime":1700000491,"order_key":"07:miner-7","p2mr_program_hex":"ebebebebebebebebebebebebebebebebebebebebebebebebebebebebebebebeb","share_difficulty":7,"share_id":"share-491","share_seq":491,"template_height":800015},{"accepted_at_ms":70509,"credit_policy":null,"job_id":"job-492","job_issued_at_ms":70493,"miner_id":"miner-8","network_difficulty":1000016,"ntime":1700000492,"order_key":"08:miner-8","p2mr_program_hex":"ecececececececececececececececececececececececececececececececec","share_difficulty":1,"share_id":"share-492","share_seq":492,"template_height":800015},{"accepted_at_ms":70482,"credit_policy":null,"job_id":"job-493","job_issued_at_ms":70483,"miner_id":"miner-9","network_difficulty":1000000,"ntime":1700000493,"order_key":"09:miner-9","p2mr_program_hex":"edededededededededededededededededededededededededededededededed","share_difficulty":7,"share_id":"share-493","share_seq":493,"template_height":800015},{"accepted_at_ms":70487,"credit_policy":null,"job_id":"job-494","job_issued_at_ms":70510,"miner_id":"miner-10","network_difficulty":1000001,"ntime":1700000494,"order_key":"10:miner-10","p2mr_program_hex":"eeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeee","share_difficulty":7,"share_id":"share-494","share_seq":494,"template_height":800015},{"accepted_at_ms":70498,"credit_policy":null,"job_id":"job-495","job_issued_at_ms":70496,"miner_id":"miner-0","network_difficulty":1000002,"ntime":1700000495,"order_key":"00:miner-0","p2mr_program_hex":"efefefefefefefefefefefefefefefefefefefefefefefefefefefefefefefef","share_difficulty":1,"share_id":"share-495","share_seq":495,"template_height":800015},{"accepted_at_ms":70506,"credit_policy":null,"job_id":"job-496","job_issued_at_ms":70486,"miner_id":"miner-1","network_difficulty":1000003,"ntime":1700000496,"order_key":"01:miner-1","p2mr_program_hex":"f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0","share_difficulty":2,"share_id":"share-496","share_seq":496,"template_height":800015},{"accepted_at_ms":70482,"credit_policy":null,"job_id":"job-497","job_issued_at_ms":70485,"miner_id":"miner-2","network_difficulty":1000004,"ntime":1700000497,"order_key":"02:miner-2","p2mr_program_hex":"f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1","share_difficulty":19,"share_id":"share-497","share_seq":497,"template_height":800015},{"accepted_at_ms":70509,"credit_policy":null,"job_id":"job-498","job_issued_at_ms":70491,"miner_id":"miner-3","network_difficulty":1000005,"ntime":1700000498,"order_key":"03:miner-3","p2mr_program_hex":"f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2","share_difficulty":3,"share_id":"share-498","share_seq":498,"template_height":800015},{"accepted_at_ms":70488,"credit_policy":null,"job_id":"job-499","job_issued_at_ms":70501,"miner_id":"miner-4","network_difficulty":1000006,"ntime":1700000499,"order_key":"04:miner-4","p2mr_program_hex":"f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3","share_difficulty":1,"share_id":"share-499","share_seq":499,"template_height":800015},{"accepted_at_ms":70505,"credit_policy":null,"job_id":"job-500","job_issued_at_ms":70486,"miner_id":"miner-5","network_difficulty":1000007,"ntime":1700000500,"order_key":"05:miner-5","p2mr_program_hex":"f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4","share_difficulty":1,"share_id":"share-500","share_seq":500,"template_height":800015},{"accepted_at_ms":70486,"credit_policy":"stale-grace","job_id":"job-501","job_issued_at_ms":70502,"miner_id":"miner-6","network_difficulty":1000008,"ntime":1700000501,"order_key":"06:miner-6","p2mr_program_hex":"f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5","share_difficulty":7,"share_id":"share-501","share_seq":501,"template_height":800015},{"accepted_at_ms":70493,"credit_policy":null,"job_id":"job-502","job_issued_at_ms":70486,"miner_id":"miner-7","network_difficulty":1000009,"ntime":1700000502,"order_key":"07:miner-7","p2mr_program_hex":"f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6","share_difficulty":19,"share_id":"share-502","share_seq":502,"template_height":800015},{"accepted_at_ms":70496,"credit_policy":"stale-grace","job_id":"job-503","job_issued_at_ms":70490,"miner_id":"miner-8","network_difficulty":1000010,"ntime":1700000503,"order_key":"08:miner-8","p2mr_program_hex":"f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7","share_difficulty":7,"share_id":"share-503","share_seq":503,"template_height":800015},{"accepted_at_ms":70506,"credit_policy":"stale-grace","job_id":"job-504","job_issued_at_ms":70511,"miner_id":"miner-9","network_difficulty":1000011,"ntime":1700000504,"order_key":"09:miner-9","p2mr_program_hex":"f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8","share_difficulty":2,"share_id":"share-504","share_seq":504,"template_height":800015},{"accepted_at_ms":70493,"credit_policy":null,"job_id":"job-505","job_issued_at_ms":70498,"miner_id":"miner-10","network_difficulty":1000012,"ntime":1700000505,"order_key":"10:miner-10","p2mr_program_hex":"f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9","share_difficulty":1,"share_id":"share-505","share_seq":505,"template_height":800015},{"accepted_at_ms":70493,"credit_policy":null,"job_id":"job-506","job_issued_at_ms":70484,"miner_id":"miner-0","network_difficulty":1000013,"ntime":1700000506,"order_key":"00:miner-0","p2mr_program_hex":"fafafafafafafafafafafafafafafafafafafafafafafafafafafafafafafafa","share_difficulty":1,"share_id":"share-506","share_seq":506,"template_height":800015},{"accepted_at_ms":70512,"credit_policy":null,"job_id":"job-507","job_issued_at_ms":70505,"miner_id":"miner-1","network_difficulty":1000014,"ntime":1700000507,"order_key":"01:miner-1","p2mr_program_hex":"fbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfb","share_difficulty":2,"share_id":"share-507","share_seq":507,"template_height":800015},{"accepted_at_ms":70501,"credit_policy":null,"job_id":"job-508","job_issued_at_ms":70482,"miner_id":"miner-2","network_difficulty":1000015,"ntime":1700000508,"order_key":"02:miner-2","p2mr_program_hex":"fcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfc","share_difficulty":7,"share_id":"share-508","share_seq":508,"template_height":800015},{"accepted_at_ms":70512,"credit_policy":"stale-grace","job_id":"job-509","job_issued_at_ms":70487,"miner_id":"miner-3","network_difficulty":1000016,"ntime":1700000509,"order_key":"03:miner-3","p2mr_program_hex":"fdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfd","share_difficulty":1,"share_id":"share-509","share_seq":509,"template_height":800015},{"accepted_at_ms":70497,"credit_policy":null,"job_id":"job-510","job_issued_at_ms":70496,"miner_id":"miner-4","network_difficulty":1000000,"ntime":1700000510,"order_key":"04:miner-4","p2mr_program_hex":"fefefefefefefefefefefefefefefefefefefefefefefefefefefefefefefefe","share_difficulty":3,"share_id":"share-510","share_seq":510,"template_height":800015},{"accepted_at_ms":70503,"credit_policy":"stale-grace","job_id":"job-511","job_issued_at_ms":70495,"miner_id":"miner-5","network_difficulty":1000001,"ntime":1700000511,"order_key":"05:miner-5","p2mr_program_hex":"ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff","share_difficulty":2,"share_id":"share-511","share_seq":511,"template_height":800015},{"accepted_at_ms":70484,"credit_policy":null,"job_id":"job-512","job_issued_at_ms":70485,"miner_id":"miner-6","network_difficulty":1000002,"ntime":1700000512,"order_key":"06:miner-6","p2mr_program_hex":"0000000000000000000000000000000000000000000000000000000000000000","share_difficulty":2,"share_id":"share-512","share_seq":512,"template_height":800016}]},"window_weight":1000000000},"page-boundary-513":{"advances":[],"name":"page-boundary-513","page_size":512,"schema":"qbit-prism-window-pipeline-parity-input/v1","snapshot":{"anchor_job_issued_at_ms":70513,"records":[{"accepted_at_ms":70511,"credit_policy":"stale-grace","job_id":"job-1","job_issued_at_ms":70513,"miner_id":"miner-1","network_difficulty":1000001,"ntime":1700000001,"order_key":"01:miner-1","p2mr_program_hex":"0101010101010101010101010101010101010101010101010101010101010101","share_difficulty":3,"share_id":"share-1","share_seq":1,"template_height":800000},{"accepted_at_ms":70499,"credit_policy":null,"job_id":"job-2","job_issued_at_ms":70496,"miner_id":"miner-2","network_difficulty":1000002,"ntime":1700000002,"order_key":"02:miner-2","p2mr_program_hex":"0202020202020202020202020202020202020202020202020202020202020202","share_difficulty":7,"share_id":"share-2","share_seq":2,"template_height":800000},{"accepted_at_ms":70497,"credit_policy":null,"job_id":"job-3","job_issued_at_ms":70498,"miner_id":"miner-3","network_difficulty":1000003,"ntime":1700000003,"order_key":"03:miner-3","p2mr_program_hex":"0303030303030303030303030303030303030303030303030303030303030303","share_difficulty":7,"share_id":"share-3","share_seq":3,"template_height":800000},{"accepted_at_ms":70503,"credit_policy":null,"job_id":"job-4","job_issued_at_ms":70486,"miner_id":"miner-4","network_difficulty":1000004,"ntime":1700000004,"order_key":"04:miner-4","p2mr_program_hex":"0404040404040404040404040404040404040404040404040404040404040404","share_difficulty":19,"share_id":"share-4","share_seq":4,"template_height":800000},{"accepted_at_ms":70491,"credit_policy":null,"job_id":"job-5","job_issued_at_ms":70486,"miner_id":"miner-5","network_difficulty":1000005,"ntime":1700000005,"order_key":"05:miner-5","p2mr_program_hex":"0505050505050505050505050505050505050505050505050505050505050505","share_difficulty":19,"share_id":"share-5","share_seq":5,"template_height":800000},{"accepted_at_ms":70487,"credit_policy":null,"job_id":"job-6","job_issued_at_ms":70491,"miner_id":"miner-6","network_difficulty":1000006,"ntime":1700000006,"order_key":"06:miner-6","p2mr_program_hex":"0606060606060606060606060606060606060606060606060606060606060606","share_difficulty":3,"share_id":"share-6","share_seq":6,"template_height":800000},{"accepted_at_ms":70505,"credit_policy":null,"job_id":"job-7","job_issued_at_ms":70495,"miner_id":"miner-7","network_difficulty":1000007,"ntime":1700000007,"order_key":"07:miner-7","p2mr_program_hex":"0707070707070707070707070707070707070707070707070707070707070707","share_difficulty":1,"share_id":"share-7","share_seq":7,"template_height":800000},{"accepted_at_ms":70501,"credit_policy":null,"job_id":"job-8","job_issued_at_ms":70504,"miner_id":"miner-8","network_difficulty":1000008,"ntime":1700000008,"order_key":"08:miner-8","p2mr_program_hex":"0808080808080808080808080808080808080808080808080808080808080808","share_difficulty":7,"share_id":"share-8","share_seq":8,"template_height":800000},{"accepted_at_ms":70498,"credit_policy":null,"job_id":"job-9","job_issued_at_ms":70489,"miner_id":"miner-9","network_difficulty":1000009,"ntime":1700000009,"order_key":"09:miner-9","p2mr_program_hex":"0909090909090909090909090909090909090909090909090909090909090909","share_difficulty":3,"share_id":"share-9","share_seq":9,"template_height":800000},{"accepted_at_ms":70512,"credit_policy":null,"job_id":"job-10","job_issued_at_ms":70510,"miner_id":"miner-10","network_difficulty":1000010,"ntime":1700000010,"order_key":"10:miner-10","p2mr_program_hex":"0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a","share_difficulty":1,"share_id":"share-10","share_seq":10,"template_height":800000},{"accepted_at_ms":70498,"credit_policy":null,"job_id":"job-11","job_issued_at_ms":70497,"miner_id":"miner-0","network_difficulty":1000011,"ntime":1700000011,"order_key":"00:miner-0","p2mr_program_hex":"0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b","share_difficulty":1,"share_id":"share-11","share_seq":11,"template_height":800000},{"accepted_at_ms":70503,"credit_policy":"stale-grace","job_id":"job-12","job_issued_at_ms":70491,"miner_id":"miner-1","network_difficulty":1000012,"ntime":1700000012,"order_key":"01:miner-1","p2mr_program_hex":"0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c","share_difficulty":1,"share_id":"share-12","share_seq":12,"template_height":800000},{"accepted_at_ms":70487,"credit_policy":null,"job_id":"job-13","job_issued_at_ms":70493,"miner_id":"miner-2","network_difficulty":1000013,"ntime":1700000013,"order_key":"02:miner-2","p2mr_program_hex":"0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d","share_difficulty":1,"share_id":"share-13","share_seq":13,"template_height":800000},{"accepted_at_ms":70497,"credit_policy":null,"job_id":"job-14","job_issued_at_ms":70501,"miner_id":"miner-3","network_difficulty":1000014,"ntime":1700000014,"order_key":"03:miner-3","p2mr_program_hex":"0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e","share_difficulty":7,"share_id":"share-14","share_seq":14,"template_height":800000},{"accepted_at_ms":70491,"credit_policy":"stale-grace","job_id":"job-15","job_issued_at_ms":70502,"miner_id":"miner-4","network_difficulty":1000015,"ntime":1700000015,"order_key":"04:miner-4","p2mr_program_hex":"0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f","share_difficulty":19,"share_id":"share-15","share_seq":15,"template_height":800000},{"accepted_at_ms":70491,"credit_policy":"stale-grace","job_id":"job-16","job_issued_at_ms":70509,"miner_id":"miner-5","network_difficulty":1000016,"ntime":1700000016,"order_key":"05:miner-5","p2mr_program_hex":"1010101010101010101010101010101010101010101010101010101010101010","share_difficulty":3,"share_id":"share-16","share_seq":16,"template_height":800000},{"accepted_at_ms":70503,"credit_policy":null,"job_id":"job-17","job_issued_at_ms":70490,"miner_id":"miner-6","network_difficulty":1000000,"ntime":1700000017,"order_key":"06:miner-6","p2mr_program_hex":"1111111111111111111111111111111111111111111111111111111111111111","share_difficulty":1,"share_id":"share-17","share_seq":17,"template_height":800000},{"accepted_at_ms":70500,"credit_policy":null,"job_id":"job-18","job_issued_at_ms":70506,"miner_id":"miner-7","network_difficulty":1000001,"ntime":1700000018,"order_key":"07:miner-7","p2mr_program_hex":"1212121212121212121212121212121212121212121212121212121212121212","share_difficulty":3,"share_id":"share-18","share_seq":18,"template_height":800000},{"accepted_at_ms":70507,"credit_policy":null,"job_id":"job-19","job_issued_at_ms":70503,"miner_id":"miner-8","network_difficulty":1000002,"ntime":1700000019,"order_key":"08:miner-8","p2mr_program_hex":"1313131313131313131313131313131313131313131313131313131313131313","share_difficulty":19,"share_id":"share-19","share_seq":19,"template_height":800000},{"accepted_at_ms":70486,"credit_policy":null,"job_id":"job-20","job_issued_at_ms":70487,"miner_id":"miner-9","network_difficulty":1000003,"ntime":1700000020,"order_key":"09:miner-9","p2mr_program_hex":"1414141414141414141414141414141414141414141414141414141414141414","share_difficulty":3,"share_id":"share-20","share_seq":20,"template_height":800000},{"accepted_at_ms":70506,"credit_policy":null,"job_id":"job-21","job_issued_at_ms":70509,"miner_id":"miner-10","network_difficulty":1000004,"ntime":1700000021,"order_key":"10:miner-10","p2mr_program_hex":"1515151515151515151515151515151515151515151515151515151515151515","share_difficulty":3,"share_id":"share-21","share_seq":21,"template_height":800000},{"accepted_at_ms":70510,"credit_policy":"stale-grace","job_id":"job-22","job_issued_at_ms":70509,"miner_id":"miner-0","network_difficulty":1000005,"ntime":1700000022,"order_key":"00:miner-0","p2mr_program_hex":"1616161616161616161616161616161616161616161616161616161616161616","share_difficulty":7,"share_id":"share-22","share_seq":22,"template_height":800000},{"accepted_at_ms":70507,"credit_policy":null,"job_id":"job-23","job_issued_at_ms":70503,"miner_id":"miner-1","network_difficulty":1000006,"ntime":1700000023,"order_key":"01:miner-1","p2mr_program_hex":"1717171717171717171717171717171717171717171717171717171717171717","share_difficulty":3,"share_id":"share-23","share_seq":23,"template_height":800000},{"accepted_at_ms":70493,"credit_policy":null,"job_id":"job-24","job_issued_at_ms":70484,"miner_id":"miner-2","network_difficulty":1000007,"ntime":1700000024,"order_key":"02:miner-2","p2mr_program_hex":"1818181818181818181818181818181818181818181818181818181818181818","share_difficulty":19,"share_id":"share-24","share_seq":24,"template_height":800000},{"accepted_at_ms":70510,"credit_policy":null,"job_id":"job-25","job_issued_at_ms":70510,"miner_id":"miner-3","network_difficulty":1000008,"ntime":1700000025,"order_key":"03:miner-3","p2mr_program_hex":"1919191919191919191919191919191919191919191919191919191919191919","share_difficulty":7,"share_id":"share-25","share_seq":25,"template_height":800000},{"accepted_at_ms":70489,"credit_policy":null,"job_id":"job-26","job_issued_at_ms":70502,"miner_id":"miner-4","network_difficulty":1000009,"ntime":1700000026,"order_key":"04:miner-4","p2mr_program_hex":"1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a","share_difficulty":3,"share_id":"share-26","share_seq":26,"template_height":800000},{"accepted_at_ms":70492,"credit_policy":"stale-grace","job_id":"job-27","job_issued_at_ms":70486,"miner_id":"miner-5","network_difficulty":1000010,"ntime":1700000027,"order_key":"05:miner-5","p2mr_program_hex":"1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b","share_difficulty":3,"share_id":"share-27","share_seq":27,"template_height":800000},{"accepted_at_ms":70493,"credit_policy":null,"job_id":"job-28","job_issued_at_ms":70494,"miner_id":"miner-6","network_difficulty":1000011,"ntime":1700000028,"order_key":"06:miner-6","p2mr_program_hex":"1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c","share_difficulty":19,"share_id":"share-28","share_seq":28,"template_height":800000},{"accepted_at_ms":70489,"credit_policy":null,"job_id":"job-29","job_issued_at_ms":70497,"miner_id":"miner-7","network_difficulty":1000012,"ntime":1700000029,"order_key":"07:miner-7","p2mr_program_hex":"1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d","share_difficulty":19,"share_id":"share-29","share_seq":29,"template_height":800000},{"accepted_at_ms":70489,"credit_policy":null,"job_id":"job-30","job_issued_at_ms":70505,"miner_id":"miner-8","network_difficulty":1000013,"ntime":1700000030,"order_key":"08:miner-8","p2mr_program_hex":"1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e","share_difficulty":3,"share_id":"share-30","share_seq":30,"template_height":800000},{"accepted_at_ms":70512,"credit_policy":"stale-grace","job_id":"job-31","job_issued_at_ms":70507,"miner_id":"miner-9","network_difficulty":1000014,"ntime":1700000031,"order_key":"09:miner-9","p2mr_program_hex":"1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f","share_difficulty":19,"share_id":"share-31","share_seq":31,"template_height":800000},{"accepted_at_ms":70510,"credit_policy":"stale-grace","job_id":"job-32","job_issued_at_ms":70486,"miner_id":"miner-10","network_difficulty":1000015,"ntime":1700000032,"order_key":"10:miner-10","p2mr_program_hex":"2020202020202020202020202020202020202020202020202020202020202020","share_difficulty":3,"share_id":"share-32","share_seq":32,"template_height":800001},{"accepted_at_ms":70512,"credit_policy":"stale-grace","job_id":"job-33","job_issued_at_ms":70489,"miner_id":"miner-0","network_difficulty":1000016,"ntime":1700000033,"order_key":"00:miner-0","p2mr_program_hex":"2121212121212121212121212121212121212121212121212121212121212121","share_difficulty":3,"share_id":"share-33","share_seq":33,"template_height":800001},{"accepted_at_ms":70502,"credit_policy":null,"job_id":"job-34","job_issued_at_ms":70501,"miner_id":"miner-1","network_difficulty":1000000,"ntime":1700000034,"order_key":"01:miner-1","p2mr_program_hex":"2222222222222222222222222222222222222222222222222222222222222222","share_difficulty":1,"share_id":"share-34","share_seq":34,"template_height":800001},{"accepted_at_ms":70508,"credit_policy":null,"job_id":"job-35","job_issued_at_ms":70503,"miner_id":"miner-2","network_difficulty":1000001,"ntime":1700000035,"order_key":"02:miner-2","p2mr_program_hex":"2323232323232323232323232323232323232323232323232323232323232323","share_difficulty":2,"share_id":"share-35","share_seq":35,"template_height":800001},{"accepted_at_ms":70513,"credit_policy":"stale-grace","job_id":"job-36","job_issued_at_ms":70506,"miner_id":"miner-3","network_difficulty":1000002,"ntime":1700000036,"order_key":"03:miner-3","p2mr_program_hex":"2424242424242424242424242424242424242424242424242424242424242424","share_difficulty":3,"share_id":"share-36","share_seq":36,"template_height":800001},{"accepted_at_ms":70505,"credit_policy":null,"job_id":"job-37","job_issued_at_ms":70508,"miner_id":"miner-4","network_difficulty":1000003,"ntime":1700000037,"order_key":"04:miner-4","p2mr_program_hex":"2525252525252525252525252525252525252525252525252525252525252525","share_difficulty":1,"share_id":"share-37","share_seq":37,"template_height":800001},{"accepted_at_ms":70504,"credit_policy":null,"job_id":"job-38","job_issued_at_ms":70488,"miner_id":"miner-5","network_difficulty":1000004,"ntime":1700000038,"order_key":"05:miner-5","p2mr_program_hex":"2626262626262626262626262626262626262626262626262626262626262626","share_difficulty":19,"share_id":"share-38","share_seq":38,"template_height":800001},{"accepted_at_ms":70498,"credit_policy":"stale-grace","job_id":"job-39","job_issued_at_ms":70507,"miner_id":"miner-6","network_difficulty":1000005,"ntime":1700000039,"order_key":"06:miner-6","p2mr_program_hex":"2727272727272727272727272727272727272727272727272727272727272727","share_difficulty":19,"share_id":"share-39","share_seq":39,"template_height":800001},{"accepted_at_ms":70503,"credit_policy":"stale-grace","job_id":"job-40","job_issued_at_ms":70497,"miner_id":"miner-7","network_difficulty":1000006,"ntime":1700000040,"order_key":"07:miner-7","p2mr_program_hex":"2828282828282828282828282828282828282828282828282828282828282828","share_difficulty":3,"share_id":"share-40","share_seq":40,"template_height":800001},{"accepted_at_ms":70484,"credit_policy":"stale-grace","job_id":"job-41","job_issued_at_ms":70505,"miner_id":"miner-8","network_difficulty":1000007,"ntime":1700000041,"order_key":"08:miner-8","p2mr_program_hex":"2929292929292929292929292929292929292929292929292929292929292929","share_difficulty":19,"share_id":"share-41","share_seq":41,"template_height":800001},{"accepted_at_ms":70497,"credit_policy":null,"job_id":"job-42","job_issued_at_ms":70506,"miner_id":"miner-9","network_difficulty":1000008,"ntime":1700000042,"order_key":"09:miner-9","p2mr_program_hex":"2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a","share_difficulty":19,"share_id":"share-42","share_seq":42,"template_height":800001},{"accepted_at_ms":70484,"credit_policy":null,"job_id":"job-43","job_issued_at_ms":70498,"miner_id":"miner-10","network_difficulty":1000009,"ntime":1700000043,"order_key":"10:miner-10","p2mr_program_hex":"2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b","share_difficulty":7,"share_id":"share-43","share_seq":43,"template_height":800001},{"accepted_at_ms":70493,"credit_policy":"stale-grace","job_id":"job-44","job_issued_at_ms":70485,"miner_id":"miner-0","network_difficulty":1000010,"ntime":1700000044,"order_key":"00:miner-0","p2mr_program_hex":"2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c","share_difficulty":2,"share_id":"share-44","share_seq":44,"template_height":800001},{"accepted_at_ms":70484,"credit_policy":null,"job_id":"job-45","job_issued_at_ms":70499,"miner_id":"miner-1","network_difficulty":1000011,"ntime":1700000045,"order_key":"01:miner-1","p2mr_program_hex":"2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d","share_difficulty":3,"share_id":"share-45","share_seq":45,"template_height":800001},{"accepted_at_ms":70493,"credit_policy":null,"job_id":"job-46","job_issued_at_ms":70508,"miner_id":"miner-2","network_difficulty":1000012,"ntime":1700000046,"order_key":"02:miner-2","p2mr_program_hex":"2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e","share_difficulty":1,"share_id":"share-46","share_seq":46,"template_height":800001},{"accepted_at_ms":70484,"credit_policy":null,"job_id":"job-47","job_issued_at_ms":70485,"miner_id":"miner-3","network_difficulty":1000013,"ntime":1700000047,"order_key":"03:miner-3","p2mr_program_hex":"2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f","share_difficulty":19,"share_id":"share-47","share_seq":47,"template_height":800001},{"accepted_at_ms":70496,"credit_policy":null,"job_id":"job-48","job_issued_at_ms":70503,"miner_id":"miner-4","network_difficulty":1000014,"ntime":1700000048,"order_key":"04:miner-4","p2mr_program_hex":"3030303030303030303030303030303030303030303030303030303030303030","share_difficulty":2,"share_id":"share-48","share_seq":48,"template_height":800001},{"accepted_at_ms":70483,"credit_policy":null,"job_id":"job-49","job_issued_at_ms":70497,"miner_id":"miner-5","network_difficulty":1000015,"ntime":1700000049,"order_key":"05:miner-5","p2mr_program_hex":"3131313131313131313131313131313131313131313131313131313131313131","share_difficulty":7,"share_id":"share-49","share_seq":49,"template_height":800001},{"accepted_at_ms":70496,"credit_policy":null,"job_id":"job-50","job_issued_at_ms":70508,"miner_id":"miner-6","network_difficulty":1000016,"ntime":1700000050,"order_key":"06:miner-6","p2mr_program_hex":"3232323232323232323232323232323232323232323232323232323232323232","share_difficulty":3,"share_id":"share-50","share_seq":50,"template_height":800001},{"accepted_at_ms":70506,"credit_policy":null,"job_id":"job-51","job_issued_at_ms":70494,"miner_id":"miner-7","network_difficulty":1000000,"ntime":1700000051,"order_key":"07:miner-7","p2mr_program_hex":"3333333333333333333333333333333333333333333333333333333333333333","share_difficulty":7,"share_id":"share-51","share_seq":51,"template_height":800001},{"accepted_at_ms":70494,"credit_policy":null,"job_id":"job-52","job_issued_at_ms":70505,"miner_id":"miner-8","network_difficulty":1000001,"ntime":1700000052,"order_key":"08:miner-8","p2mr_program_hex":"3434343434343434343434343434343434343434343434343434343434343434","share_difficulty":7,"share_id":"share-52","share_seq":52,"template_height":800001},{"accepted_at_ms":70493,"credit_policy":"stale-grace","job_id":"job-53","job_issued_at_ms":70494,"miner_id":"miner-9","network_difficulty":1000002,"ntime":1700000053,"order_key":"09:miner-9","p2mr_program_hex":"3535353535353535353535353535353535353535353535353535353535353535","share_difficulty":2,"share_id":"share-53","share_seq":53,"template_height":800001},{"accepted_at_ms":70483,"credit_policy":null,"job_id":"job-54","job_issued_at_ms":70499,"miner_id":"miner-10","network_difficulty":1000003,"ntime":1700000054,"order_key":"10:miner-10","p2mr_program_hex":"3636363636363636363636363636363636363636363636363636363636363636","share_difficulty":1,"share_id":"share-54","share_seq":54,"template_height":800001},{"accepted_at_ms":70510,"credit_policy":"stale-grace","job_id":"job-55","job_issued_at_ms":70511,"miner_id":"miner-0","network_difficulty":1000004,"ntime":1700000055,"order_key":"00:miner-0","p2mr_program_hex":"3737373737373737373737373737373737373737373737373737373737373737","share_difficulty":7,"share_id":"share-55","share_seq":55,"template_height":800001},{"accepted_at_ms":70495,"credit_policy":"stale-grace","job_id":"job-56","job_issued_at_ms":70499,"miner_id":"miner-1","network_difficulty":1000005,"ntime":1700000056,"order_key":"01:miner-1","p2mr_program_hex":"3838383838383838383838383838383838383838383838383838383838383838","share_difficulty":19,"share_id":"share-56","share_seq":56,"template_height":800001},{"accepted_at_ms":70498,"credit_policy":null,"job_id":"job-57","job_issued_at_ms":70512,"miner_id":"miner-2","network_difficulty":1000006,"ntime":1700000057,"order_key":"02:miner-2","p2mr_program_hex":"3939393939393939393939393939393939393939393939393939393939393939","share_difficulty":19,"share_id":"share-57","share_seq":57,"template_height":800001},{"accepted_at_ms":70506,"credit_policy":null,"job_id":"job-58","job_issued_at_ms":70483,"miner_id":"miner-3","network_difficulty":1000007,"ntime":1700000058,"order_key":"03:miner-3","p2mr_program_hex":"3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a","share_difficulty":19,"share_id":"share-58","share_seq":58,"template_height":800001},{"accepted_at_ms":70496,"credit_policy":"stale-grace","job_id":"job-59","job_issued_at_ms":70511,"miner_id":"miner-4","network_difficulty":1000008,"ntime":1700000059,"order_key":"04:miner-4","p2mr_program_hex":"3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b","share_difficulty":1,"share_id":"share-59","share_seq":59,"template_height":800001},{"accepted_at_ms":70505,"credit_policy":null,"job_id":"job-60","job_issued_at_ms":70495,"miner_id":"miner-5","network_difficulty":1000009,"ntime":1700000060,"order_key":"05:miner-5","p2mr_program_hex":"3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c","share_difficulty":7,"share_id":"share-60","share_seq":60,"template_height":800001},{"accepted_at_ms":70495,"credit_policy":null,"job_id":"job-61","job_issued_at_ms":70484,"miner_id":"miner-6","network_difficulty":1000010,"ntime":1700000061,"order_key":"06:miner-6","p2mr_program_hex":"3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d","share_difficulty":3,"share_id":"share-61","share_seq":61,"template_height":800001},{"accepted_at_ms":70511,"credit_policy":null,"job_id":"job-62","job_issued_at_ms":70484,"miner_id":"miner-7","network_difficulty":1000011,"ntime":1700000062,"order_key":"07:miner-7","p2mr_program_hex":"3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e","share_difficulty":1,"share_id":"share-62","share_seq":62,"template_height":800001},{"accepted_at_ms":70488,"credit_policy":"stale-grace","job_id":"job-63","job_issued_at_ms":70494,"miner_id":"miner-8","network_difficulty":1000012,"ntime":1700000063,"order_key":"08:miner-8","p2mr_program_hex":"3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f","share_difficulty":3,"share_id":"share-63","share_seq":63,"template_height":800001},{"accepted_at_ms":70494,"credit_policy":null,"job_id":"job-64","job_issued_at_ms":70490,"miner_id":"miner-9","network_difficulty":1000013,"ntime":1700000064,"order_key":"09:miner-9","p2mr_program_hex":"4040404040404040404040404040404040404040404040404040404040404040","share_difficulty":1,"share_id":"share-64","share_seq":64,"template_height":800002},{"accepted_at_ms":70487,"credit_policy":null,"job_id":"job-65","job_issued_at_ms":70496,"miner_id":"miner-10","network_difficulty":1000014,"ntime":1700000065,"order_key":"10:miner-10","p2mr_program_hex":"4141414141414141414141414141414141414141414141414141414141414141","share_difficulty":1,"share_id":"share-65","share_seq":65,"template_height":800002},{"accepted_at_ms":70484,"credit_policy":null,"job_id":"job-66","job_issued_at_ms":70502,"miner_id":"miner-0","network_difficulty":1000015,"ntime":1700000066,"order_key":"00:miner-0","p2mr_program_hex":"4242424242424242424242424242424242424242424242424242424242424242","share_difficulty":1,"share_id":"share-66","share_seq":66,"template_height":800002},{"accepted_at_ms":70512,"credit_policy":null,"job_id":"job-67","job_issued_at_ms":70504,"miner_id":"miner-1","network_difficulty":1000016,"ntime":1700000067,"order_key":"01:miner-1","p2mr_program_hex":"4343434343434343434343434343434343434343434343434343434343434343","share_difficulty":1,"share_id":"share-67","share_seq":67,"template_height":800002},{"accepted_at_ms":70503,"credit_policy":"stale-grace","job_id":"job-68","job_issued_at_ms":70510,"miner_id":"miner-2","network_difficulty":1000000,"ntime":1700000068,"order_key":"02:miner-2","p2mr_program_hex":"4444444444444444444444444444444444444444444444444444444444444444","share_difficulty":1,"share_id":"share-68","share_seq":68,"template_height":800002},{"accepted_at_ms":70500,"credit_policy":null,"job_id":"job-69","job_issued_at_ms":70497,"miner_id":"miner-3","network_difficulty":1000001,"ntime":1700000069,"order_key":"03:miner-3","p2mr_program_hex":"4545454545454545454545454545454545454545454545454545454545454545","share_difficulty":3,"share_id":"share-69","share_seq":69,"template_height":800002},{"accepted_at_ms":70483,"credit_policy":"stale-grace","job_id":"job-70","job_issued_at_ms":70500,"miner_id":"miner-4","network_difficulty":1000002,"ntime":1700000070,"order_key":"04:miner-4","p2mr_program_hex":"4646464646464646464646464646464646464646464646464646464646464646","share_difficulty":2,"share_id":"share-70","share_seq":70,"template_height":800002},{"accepted_at_ms":70492,"credit_policy":null,"job_id":"job-71","job_issued_at_ms":70491,"miner_id":"miner-5","network_difficulty":1000003,"ntime":1700000071,"order_key":"05:miner-5","p2mr_program_hex":"4747474747474747474747474747474747474747474747474747474747474747","share_difficulty":19,"share_id":"share-71","share_seq":71,"template_height":800002},{"accepted_at_ms":70494,"credit_policy":null,"job_id":"job-72","job_issued_at_ms":70494,"miner_id":"miner-6","network_difficulty":1000004,"ntime":1700000072,"order_key":"06:miner-6","p2mr_program_hex":"4848484848484848484848484848484848484848484848484848484848484848","share_difficulty":19,"share_id":"share-72","share_seq":72,"template_height":800002},{"accepted_at_ms":70499,"credit_policy":null,"job_id":"job-73","job_issued_at_ms":70499,"miner_id":"miner-7","network_difficulty":1000005,"ntime":1700000073,"order_key":"07:miner-7","p2mr_program_hex":"4949494949494949494949494949494949494949494949494949494949494949","share_difficulty":7,"share_id":"share-73","share_seq":73,"template_height":800002},{"accepted_at_ms":70510,"credit_policy":"stale-grace","job_id":"job-74","job_issued_at_ms":70500,"miner_id":"miner-8","network_difficulty":1000006,"ntime":1700000074,"order_key":"08:miner-8","p2mr_program_hex":"4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a","share_difficulty":19,"share_id":"share-74","share_seq":74,"template_height":800002},{"accepted_at_ms":70495,"credit_policy":"stale-grace","job_id":"job-75","job_issued_at_ms":70510,"miner_id":"miner-9","network_difficulty":1000007,"ntime":1700000075,"order_key":"09:miner-9","p2mr_program_hex":"4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b","share_difficulty":1,"share_id":"share-75","share_seq":75,"template_height":800002},{"accepted_at_ms":70500,"credit_policy":null,"job_id":"job-76","job_issued_at_ms":70507,"miner_id":"miner-10","network_difficulty":1000008,"ntime":1700000076,"order_key":"10:miner-10","p2mr_program_hex":"4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c","share_difficulty":1,"share_id":"share-76","share_seq":76,"template_height":800002},{"accepted_at_ms":70493,"credit_policy":null,"job_id":"job-77","job_issued_at_ms":70500,"miner_id":"miner-0","network_difficulty":1000009,"ntime":1700000077,"order_key":"00:miner-0","p2mr_program_hex":"4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d","share_difficulty":1,"share_id":"share-77","share_seq":77,"template_height":800002},{"accepted_at_ms":70510,"credit_policy":null,"job_id":"job-78","job_issued_at_ms":70507,"miner_id":"miner-1","network_difficulty":1000010,"ntime":1700000078,"order_key":"01:miner-1","p2mr_program_hex":"4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e","share_difficulty":2,"share_id":"share-78","share_seq":78,"template_height":800002},{"accepted_at_ms":70498,"credit_policy":"stale-grace","job_id":"job-79","job_issued_at_ms":70488,"miner_id":"miner-2","network_difficulty":1000011,"ntime":1700000079,"order_key":"02:miner-2","p2mr_program_hex":"4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f","share_difficulty":19,"share_id":"share-79","share_seq":79,"template_height":800002},{"accepted_at_ms":70483,"credit_policy":null,"job_id":"job-80","job_issued_at_ms":70495,"miner_id":"miner-3","network_difficulty":1000012,"ntime":1700000080,"order_key":"03:miner-3","p2mr_program_hex":"5050505050505050505050505050505050505050505050505050505050505050","share_difficulty":19,"share_id":"share-80","share_seq":80,"template_height":800002},{"accepted_at_ms":70509,"credit_policy":null,"job_id":"job-81","job_issued_at_ms":70486,"miner_id":"miner-4","network_difficulty":1000013,"ntime":1700000081,"order_key":"04:miner-4","p2mr_program_hex":"5151515151515151515151515151515151515151515151515151515151515151","share_difficulty":3,"share_id":"share-81","share_seq":81,"template_height":800002},{"accepted_at_ms":70502,"credit_policy":null,"job_id":"job-82","job_issued_at_ms":70504,"miner_id":"miner-5","network_difficulty":1000014,"ntime":1700000082,"order_key":"05:miner-5","p2mr_program_hex":"5252525252525252525252525252525252525252525252525252525252525252","share_difficulty":19,"share_id":"share-82","share_seq":82,"template_height":800002},{"accepted_at_ms":70494,"credit_policy":null,"job_id":"job-83","job_issued_at_ms":70496,"miner_id":"miner-6","network_difficulty":1000015,"ntime":1700000083,"order_key":"06:miner-6","p2mr_program_hex":"5353535353535353535353535353535353535353535353535353535353535353","share_difficulty":2,"share_id":"share-83","share_seq":83,"template_height":800002},{"accepted_at_ms":70501,"credit_policy":null,"job_id":"job-84","job_issued_at_ms":70491,"miner_id":"miner-7","network_difficulty":1000016,"ntime":1700000084,"order_key":"07:miner-7","p2mr_program_hex":"5454545454545454545454545454545454545454545454545454545454545454","share_difficulty":19,"share_id":"share-84","share_seq":84,"template_height":800002},{"accepted_at_ms":70488,"credit_policy":"stale-grace","job_id":"job-85","job_issued_at_ms":70512,"miner_id":"miner-8","network_difficulty":1000000,"ntime":1700000085,"order_key":"08:miner-8","p2mr_program_hex":"5555555555555555555555555555555555555555555555555555555555555555","share_difficulty":1,"share_id":"share-85","share_seq":85,"template_height":800002},{"accepted_at_ms":70491,"credit_policy":null,"job_id":"job-86","job_issued_at_ms":70490,"miner_id":"miner-9","network_difficulty":1000001,"ntime":1700000086,"order_key":"09:miner-9","p2mr_program_hex":"5656565656565656565656565656565656565656565656565656565656565656","share_difficulty":7,"share_id":"share-86","share_seq":86,"template_height":800002},{"accepted_at_ms":70510,"credit_policy":null,"job_id":"job-87","job_issued_at_ms":70496,"miner_id":"miner-10","network_difficulty":1000002,"ntime":1700000087,"order_key":"10:miner-10","p2mr_program_hex":"5757575757575757575757575757575757575757575757575757575757575757","share_difficulty":7,"share_id":"share-87","share_seq":87,"template_height":800002},{"accepted_at_ms":70510,"credit_policy":"stale-grace","job_id":"job-88","job_issued_at_ms":70489,"miner_id":"miner-0","network_difficulty":1000003,"ntime":1700000088,"order_key":"00:miner-0","p2mr_program_hex":"5858585858585858585858585858585858585858585858585858585858585858","share_difficulty":19,"share_id":"share-88","share_seq":88,"template_height":800002},{"accepted_at_ms":70503,"credit_policy":"stale-grace","job_id":"job-89","job_issued_at_ms":70493,"miner_id":"miner-1","network_difficulty":1000004,"ntime":1700000089,"order_key":"01:miner-1","p2mr_program_hex":"5959595959595959595959595959595959595959595959595959595959595959","share_difficulty":1,"share_id":"share-89","share_seq":89,"template_height":800002},{"accepted_at_ms":70505,"credit_policy":"stale-grace","job_id":"job-90","job_issued_at_ms":70498,"miner_id":"miner-2","network_difficulty":1000005,"ntime":1700000090,"order_key":"02:miner-2","p2mr_program_hex":"5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a","share_difficulty":19,"share_id":"share-90","share_seq":90,"template_height":800002},{"accepted_at_ms":70491,"credit_policy":null,"job_id":"job-91","job_issued_at_ms":70513,"miner_id":"miner-3","network_difficulty":1000006,"ntime":1700000091,"order_key":"03:miner-3","p2mr_program_hex":"5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b","share_difficulty":3,"share_id":"share-91","share_seq":91,"template_height":800002},{"accepted_at_ms":70506,"credit_policy":null,"job_id":"job-92","job_issued_at_ms":70491,"miner_id":"miner-4","network_difficulty":1000007,"ntime":1700000092,"order_key":"04:miner-4","p2mr_program_hex":"5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c","share_difficulty":1,"share_id":"share-92","share_seq":92,"template_height":800002},{"accepted_at_ms":70493,"credit_policy":null,"job_id":"job-93","job_issued_at_ms":70503,"miner_id":"miner-5","network_difficulty":1000008,"ntime":1700000093,"order_key":"05:miner-5","p2mr_program_hex":"5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d","share_difficulty":2,"share_id":"share-93","share_seq":93,"template_height":800002},{"accepted_at_ms":70499,"credit_policy":"stale-grace","job_id":"job-94","job_issued_at_ms":70497,"miner_id":"miner-6","network_difficulty":1000009,"ntime":1700000094,"order_key":"06:miner-6","p2mr_program_hex":"5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e","share_difficulty":19,"share_id":"share-94","share_seq":94,"template_height":800002},{"accepted_at_ms":70501,"credit_policy":null,"job_id":"job-95","job_issued_at_ms":70492,"miner_id":"miner-7","network_difficulty":1000010,"ntime":1700000095,"order_key":"07:miner-7","p2mr_program_hex":"5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f","share_difficulty":7,"share_id":"share-95","share_seq":95,"template_height":800002},{"accepted_at_ms":70488,"credit_policy":null,"job_id":"job-96","job_issued_at_ms":70491,"miner_id":"miner-8","network_difficulty":1000011,"ntime":1700000096,"order_key":"08:miner-8","p2mr_program_hex":"6060606060606060606060606060606060606060606060606060606060606060","share_difficulty":2,"share_id":"share-96","share_seq":96,"template_height":800003},{"accepted_at_ms":70487,"credit_policy":null,"job_id":"job-97","job_issued_at_ms":70496,"miner_id":"miner-9","network_difficulty":1000012,"ntime":1700000097,"order_key":"09:miner-9","p2mr_program_hex":"6161616161616161616161616161616161616161616161616161616161616161","share_difficulty":19,"share_id":"share-97","share_seq":97,"template_height":800003},{"accepted_at_ms":70497,"credit_policy":null,"job_id":"job-98","job_issued_at_ms":70492,"miner_id":"miner-10","network_difficulty":1000013,"ntime":1700000098,"order_key":"10:miner-10","p2mr_program_hex":"6262626262626262626262626262626262626262626262626262626262626262","share_difficulty":1,"share_id":"share-98","share_seq":98,"template_height":800003},{"accepted_at_ms":70506,"credit_policy":null,"job_id":"job-99","job_issued_at_ms":70500,"miner_id":"miner-0","network_difficulty":1000014,"ntime":1700000099,"order_key":"00:miner-0","p2mr_program_hex":"6363636363636363636363636363636363636363636363636363636363636363","share_difficulty":19,"share_id":"share-99","share_seq":99,"template_height":800003},{"accepted_at_ms":70490,"credit_policy":null,"job_id":"job-100","job_issued_at_ms":70491,"miner_id":"miner-1","network_difficulty":1000015,"ntime":1700000100,"order_key":"01:miner-1","p2mr_program_hex":"6464646464646464646464646464646464646464646464646464646464646464","share_difficulty":2,"share_id":"share-100","share_seq":100,"template_height":800003},{"accepted_at_ms":70505,"credit_policy":null,"job_id":"job-101","job_issued_at_ms":70506,"miner_id":"miner-2","network_difficulty":1000016,"ntime":1700000101,"order_key":"02:miner-2","p2mr_program_hex":"6565656565656565656565656565656565656565656565656565656565656565","share_difficulty":3,"share_id":"share-101","share_seq":101,"template_height":800003},{"accepted_at_ms":70500,"credit_policy":"stale-grace","job_id":"job-102","job_issued_at_ms":70505,"miner_id":"miner-3","network_difficulty":1000000,"ntime":1700000102,"order_key":"03:miner-3","p2mr_program_hex":"6666666666666666666666666666666666666666666666666666666666666666","share_difficulty":3,"share_id":"share-102","share_seq":102,"template_height":800003},{"accepted_at_ms":70498,"credit_policy":null,"job_id":"job-103","job_issued_at_ms":70503,"miner_id":"miner-4","network_difficulty":1000001,"ntime":1700000103,"order_key":"04:miner-4","p2mr_program_hex":"6767676767676767676767676767676767676767676767676767676767676767","share_difficulty":19,"share_id":"share-103","share_seq":103,"template_height":800003},{"accepted_at_ms":70489,"credit_policy":null,"job_id":"job-104","job_issued_at_ms":70501,"miner_id":"miner-5","network_difficulty":1000002,"ntime":1700000104,"order_key":"05:miner-5","p2mr_program_hex":"6868686868686868686868686868686868686868686868686868686868686868","share_difficulty":3,"share_id":"share-104","share_seq":104,"template_height":800003},{"accepted_at_ms":70499,"credit_policy":"stale-grace","job_id":"job-105","job_issued_at_ms":70492,"miner_id":"miner-6","network_difficulty":1000003,"ntime":1700000105,"order_key":"06:miner-6","p2mr_program_hex":"6969696969696969696969696969696969696969696969696969696969696969","share_difficulty":19,"share_id":"share-105","share_seq":105,"template_height":800003},{"accepted_at_ms":70485,"credit_policy":null,"job_id":"job-106","job_issued_at_ms":70487,"miner_id":"miner-7","network_difficulty":1000004,"ntime":1700000106,"order_key":"07:miner-7","p2mr_program_hex":"6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a","share_difficulty":19,"share_id":"share-106","share_seq":106,"template_height":800003},{"accepted_at_ms":70496,"credit_policy":null,"job_id":"job-107","job_issued_at_ms":70509,"miner_id":"miner-8","network_difficulty":1000005,"ntime":1700000107,"order_key":"08:miner-8","p2mr_program_hex":"6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b","share_difficulty":1,"share_id":"share-107","share_seq":107,"template_height":800003},{"accepted_at_ms":70484,"credit_policy":null,"job_id":"job-108","job_issued_at_ms":70511,"miner_id":"miner-9","network_difficulty":1000006,"ntime":1700000108,"order_key":"09:miner-9","p2mr_program_hex":"6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c","share_difficulty":3,"share_id":"share-108","share_seq":108,"template_height":800003},{"accepted_at_ms":70508,"credit_policy":null,"job_id":"job-109","job_issued_at_ms":70490,"miner_id":"miner-10","network_difficulty":1000007,"ntime":1700000109,"order_key":"10:miner-10","p2mr_program_hex":"6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d","share_difficulty":19,"share_id":"share-109","share_seq":109,"template_height":800003},{"accepted_at_ms":70501,"credit_policy":null,"job_id":"job-110","job_issued_at_ms":70513,"miner_id":"miner-0","network_difficulty":1000008,"ntime":1700000110,"order_key":"00:miner-0","p2mr_program_hex":"6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e","share_difficulty":19,"share_id":"share-110","share_seq":110,"template_height":800003},{"accepted_at_ms":70511,"credit_policy":null,"job_id":"job-111","job_issued_at_ms":70507,"miner_id":"miner-1","network_difficulty":1000009,"ntime":1700000111,"order_key":"01:miner-1","p2mr_program_hex":"6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f","share_difficulty":19,"share_id":"share-111","share_seq":111,"template_height":800003},{"accepted_at_ms":70497,"credit_policy":null,"job_id":"job-112","job_issued_at_ms":70492,"miner_id":"miner-2","network_difficulty":1000010,"ntime":1700000112,"order_key":"02:miner-2","p2mr_program_hex":"7070707070707070707070707070707070707070707070707070707070707070","share_difficulty":1,"share_id":"share-112","share_seq":112,"template_height":800003},{"accepted_at_ms":70492,"credit_policy":null,"job_id":"job-113","job_issued_at_ms":70509,"miner_id":"miner-3","network_difficulty":1000011,"ntime":1700000113,"order_key":"03:miner-3","p2mr_program_hex":"7171717171717171717171717171717171717171717171717171717171717171","share_difficulty":7,"share_id":"share-113","share_seq":113,"template_height":800003},{"accepted_at_ms":70484,"credit_policy":"stale-grace","job_id":"job-114","job_issued_at_ms":70498,"miner_id":"miner-4","network_difficulty":1000012,"ntime":1700000114,"order_key":"04:miner-4","p2mr_program_hex":"7272727272727272727272727272727272727272727272727272727272727272","share_difficulty":19,"share_id":"share-114","share_seq":114,"template_height":800003},{"accepted_at_ms":70508,"credit_policy":null,"job_id":"job-115","job_issued_at_ms":70504,"miner_id":"miner-5","network_difficulty":1000013,"ntime":1700000115,"order_key":"05:miner-5","p2mr_program_hex":"7373737373737373737373737373737373737373737373737373737373737373","share_difficulty":19,"share_id":"share-115","share_seq":115,"template_height":800003},{"accepted_at_ms":70507,"credit_policy":"stale-grace","job_id":"job-116","job_issued_at_ms":70484,"miner_id":"miner-6","network_difficulty":1000014,"ntime":1700000116,"order_key":"06:miner-6","p2mr_program_hex":"7474747474747474747474747474747474747474747474747474747474747474","share_difficulty":19,"share_id":"share-116","share_seq":116,"template_height":800003},{"accepted_at_ms":70486,"credit_policy":null,"job_id":"job-117","job_issued_at_ms":70507,"miner_id":"miner-7","network_difficulty":1000015,"ntime":1700000117,"order_key":"07:miner-7","p2mr_program_hex":"7575757575757575757575757575757575757575757575757575757575757575","share_difficulty":19,"share_id":"share-117","share_seq":117,"template_height":800003},{"accepted_at_ms":70495,"credit_policy":null,"job_id":"job-118","job_issued_at_ms":70506,"miner_id":"miner-8","network_difficulty":1000016,"ntime":1700000118,"order_key":"08:miner-8","p2mr_program_hex":"7676767676767676767676767676767676767676767676767676767676767676","share_difficulty":2,"share_id":"share-118","share_seq":118,"template_height":800003},{"accepted_at_ms":70484,"credit_policy":null,"job_id":"job-119","job_issued_at_ms":70488,"miner_id":"miner-9","network_difficulty":1000000,"ntime":1700000119,"order_key":"09:miner-9","p2mr_program_hex":"7777777777777777777777777777777777777777777777777777777777777777","share_difficulty":2,"share_id":"share-119","share_seq":119,"template_height":800003},{"accepted_at_ms":70486,"credit_policy":null,"job_id":"job-120","job_issued_at_ms":70513,"miner_id":"miner-10","network_difficulty":1000001,"ntime":1700000120,"order_key":"10:miner-10","p2mr_program_hex":"7878787878787878787878787878787878787878787878787878787878787878","share_difficulty":7,"share_id":"share-120","share_seq":120,"template_height":800003},{"accepted_at_ms":70488,"credit_policy":null,"job_id":"job-121","job_issued_at_ms":70509,"miner_id":"miner-0","network_difficulty":1000002,"ntime":1700000121,"order_key":"00:miner-0","p2mr_program_hex":"7979797979797979797979797979797979797979797979797979797979797979","share_difficulty":1,"share_id":"share-121","share_seq":121,"template_height":800003},{"accepted_at_ms":70492,"credit_policy":null,"job_id":"job-122","job_issued_at_ms":70510,"miner_id":"miner-1","network_difficulty":1000003,"ntime":1700000122,"order_key":"01:miner-1","p2mr_program_hex":"7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a","share_difficulty":19,"share_id":"share-122","share_seq":122,"template_height":800003},{"accepted_at_ms":70495,"credit_policy":null,"job_id":"job-123","job_issued_at_ms":70507,"miner_id":"miner-2","network_difficulty":1000004,"ntime":1700000123,"order_key":"02:miner-2","p2mr_program_hex":"7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b","share_difficulty":2,"share_id":"share-123","share_seq":123,"template_height":800003},{"accepted_at_ms":70507,"credit_policy":null,"job_id":"job-124","job_issued_at_ms":70510,"miner_id":"miner-3","network_difficulty":1000005,"ntime":1700000124,"order_key":"03:miner-3","p2mr_program_hex":"7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c","share_difficulty":7,"share_id":"share-124","share_seq":124,"template_height":800003},{"accepted_at_ms":70496,"credit_policy":null,"job_id":"job-125","job_issued_at_ms":70503,"miner_id":"miner-4","network_difficulty":1000006,"ntime":1700000125,"order_key":"04:miner-4","p2mr_program_hex":"7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d","share_difficulty":1,"share_id":"share-125","share_seq":125,"template_height":800003},{"accepted_at_ms":70511,"credit_policy":null,"job_id":"job-126","job_issued_at_ms":70506,"miner_id":"miner-5","network_difficulty":1000007,"ntime":1700000126,"order_key":"05:miner-5","p2mr_program_hex":"7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e","share_difficulty":7,"share_id":"share-126","share_seq":126,"template_height":800003},{"accepted_at_ms":70491,"credit_policy":"stale-grace","job_id":"job-127","job_issued_at_ms":70484,"miner_id":"miner-6","network_difficulty":1000008,"ntime":1700000127,"order_key":"06:miner-6","p2mr_program_hex":"7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f","share_difficulty":3,"share_id":"share-127","share_seq":127,"template_height":800003},{"accepted_at_ms":70489,"credit_policy":null,"job_id":"job-128","job_issued_at_ms":70511,"miner_id":"miner-7","network_difficulty":1000009,"ntime":1700000128,"order_key":"07:miner-7","p2mr_program_hex":"8080808080808080808080808080808080808080808080808080808080808080","share_difficulty":3,"share_id":"share-128","share_seq":128,"template_height":800004},{"accepted_at_ms":70513,"credit_policy":"stale-grace","job_id":"job-129","job_issued_at_ms":70502,"miner_id":"miner-8","network_difficulty":1000010,"ntime":1700000129,"order_key":"08:miner-8","p2mr_program_hex":"8181818181818181818181818181818181818181818181818181818181818181","share_difficulty":7,"share_id":"share-129","share_seq":129,"template_height":800004},{"accepted_at_ms":70483,"credit_policy":null,"job_id":"job-130","job_issued_at_ms":70510,"miner_id":"miner-9","network_difficulty":1000011,"ntime":1700000130,"order_key":"09:miner-9","p2mr_program_hex":"8282828282828282828282828282828282828282828282828282828282828282","share_difficulty":1,"share_id":"share-130","share_seq":130,"template_height":800004},{"accepted_at_ms":70488,"credit_policy":null,"job_id":"job-131","job_issued_at_ms":70496,"miner_id":"miner-10","network_difficulty":1000012,"ntime":1700000131,"order_key":"10:miner-10","p2mr_program_hex":"8383838383838383838383838383838383838383838383838383838383838383","share_difficulty":3,"share_id":"share-131","share_seq":131,"template_height":800004},{"accepted_at_ms":70492,"credit_policy":null,"job_id":"job-132","job_issued_at_ms":70502,"miner_id":"miner-0","network_difficulty":1000013,"ntime":1700000132,"order_key":"00:miner-0","p2mr_program_hex":"8484848484848484848484848484848484848484848484848484848484848484","share_difficulty":2,"share_id":"share-132","share_seq":132,"template_height":800004},{"accepted_at_ms":70507,"credit_policy":"stale-grace","job_id":"job-133","job_issued_at_ms":70503,"miner_id":"miner-1","network_difficulty":1000014,"ntime":1700000133,"order_key":"01:miner-1","p2mr_program_hex":"8585858585858585858585858585858585858585858585858585858585858585","share_difficulty":1,"share_id":"share-133","share_seq":133,"template_height":800004},{"accepted_at_ms":70499,"credit_policy":null,"job_id":"job-134","job_issued_at_ms":70484,"miner_id":"miner-2","network_difficulty":1000015,"ntime":1700000134,"order_key":"02:miner-2","p2mr_program_hex":"8686868686868686868686868686868686868686868686868686868686868686","share_difficulty":3,"share_id":"share-134","share_seq":134,"template_height":800004},{"accepted_at_ms":70507,"credit_policy":null,"job_id":"job-135","job_issued_at_ms":70486,"miner_id":"miner-3","network_difficulty":1000016,"ntime":1700000135,"order_key":"03:miner-3","p2mr_program_hex":"8787878787878787878787878787878787878787878787878787878787878787","share_difficulty":7,"share_id":"share-135","share_seq":135,"template_height":800004},{"accepted_at_ms":70504,"credit_policy":null,"job_id":"job-136","job_issued_at_ms":70489,"miner_id":"miner-4","network_difficulty":1000000,"ntime":1700000136,"order_key":"04:miner-4","p2mr_program_hex":"8888888888888888888888888888888888888888888888888888888888888888","share_difficulty":3,"share_id":"share-136","share_seq":136,"template_height":800004},{"accepted_at_ms":70511,"credit_policy":null,"job_id":"job-137","job_issued_at_ms":70491,"miner_id":"miner-5","network_difficulty":1000001,"ntime":1700000137,"order_key":"05:miner-5","p2mr_program_hex":"8989898989898989898989898989898989898989898989898989898989898989","share_difficulty":19,"share_id":"share-137","share_seq":137,"template_height":800004},{"accepted_at_ms":70500,"credit_policy":null,"job_id":"job-138","job_issued_at_ms":70501,"miner_id":"miner-6","network_difficulty":1000002,"ntime":1700000138,"order_key":"06:miner-6","p2mr_program_hex":"8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a","share_difficulty":1,"share_id":"share-138","share_seq":138,"template_height":800004},{"accepted_at_ms":70488,"credit_policy":"stale-grace","job_id":"job-139","job_issued_at_ms":70510,"miner_id":"miner-7","network_difficulty":1000003,"ntime":1700000139,"order_key":"07:miner-7","p2mr_program_hex":"8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b","share_difficulty":3,"share_id":"share-139","share_seq":139,"template_height":800004},{"accepted_at_ms":70505,"credit_policy":"stale-grace","job_id":"job-140","job_issued_at_ms":70504,"miner_id":"miner-8","network_difficulty":1000004,"ntime":1700000140,"order_key":"08:miner-8","p2mr_program_hex":"8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c","share_difficulty":1,"share_id":"share-140","share_seq":140,"template_height":800004},{"accepted_at_ms":70500,"credit_policy":null,"job_id":"job-141","job_issued_at_ms":70486,"miner_id":"miner-9","network_difficulty":1000005,"ntime":1700000141,"order_key":"09:miner-9","p2mr_program_hex":"8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d","share_difficulty":19,"share_id":"share-141","share_seq":141,"template_height":800004},{"accepted_at_ms":70511,"credit_policy":null,"job_id":"job-142","job_issued_at_ms":70491,"miner_id":"miner-10","network_difficulty":1000006,"ntime":1700000142,"order_key":"10:miner-10","p2mr_program_hex":"8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e","share_difficulty":19,"share_id":"share-142","share_seq":142,"template_height":800004},{"accepted_at_ms":70513,"credit_policy":null,"job_id":"job-143","job_issued_at_ms":70494,"miner_id":"miner-0","network_difficulty":1000007,"ntime":1700000143,"order_key":"00:miner-0","p2mr_program_hex":"8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f","share_difficulty":19,"share_id":"share-143","share_seq":143,"template_height":800004},{"accepted_at_ms":70502,"credit_policy":null,"job_id":"job-144","job_issued_at_ms":70498,"miner_id":"miner-1","network_difficulty":1000008,"ntime":1700000144,"order_key":"01:miner-1","p2mr_program_hex":"9090909090909090909090909090909090909090909090909090909090909090","share_difficulty":7,"share_id":"share-144","share_seq":144,"template_height":800004},{"accepted_at_ms":70499,"credit_policy":null,"job_id":"job-145","job_issued_at_ms":70509,"miner_id":"miner-2","network_difficulty":1000009,"ntime":1700000145,"order_key":"02:miner-2","p2mr_program_hex":"9191919191919191919191919191919191919191919191919191919191919191","share_difficulty":3,"share_id":"share-145","share_seq":145,"template_height":800004},{"accepted_at_ms":70500,"credit_policy":null,"job_id":"job-146","job_issued_at_ms":70503,"miner_id":"miner-3","network_difficulty":1000010,"ntime":1700000146,"order_key":"03:miner-3","p2mr_program_hex":"9292929292929292929292929292929292929292929292929292929292929292","share_difficulty":1,"share_id":"share-146","share_seq":146,"template_height":800004},{"accepted_at_ms":70508,"credit_policy":null,"job_id":"job-147","job_issued_at_ms":70509,"miner_id":"miner-4","network_difficulty":1000011,"ntime":1700000147,"order_key":"04:miner-4","p2mr_program_hex":"9393939393939393939393939393939393939393939393939393939393939393","share_difficulty":1,"share_id":"share-147","share_seq":147,"template_height":800004},{"accepted_at_ms":70507,"credit_policy":null,"job_id":"job-148","job_issued_at_ms":70507,"miner_id":"miner-5","network_difficulty":1000012,"ntime":1700000148,"order_key":"05:miner-5","p2mr_program_hex":"9494949494949494949494949494949494949494949494949494949494949494","share_difficulty":3,"share_id":"share-148","share_seq":148,"template_height":800004},{"accepted_at_ms":70483,"credit_policy":null,"job_id":"job-149","job_issued_at_ms":70506,"miner_id":"miner-6","network_difficulty":1000013,"ntime":1700000149,"order_key":"06:miner-6","p2mr_program_hex":"9595959595959595959595959595959595959595959595959595959595959595","share_difficulty":19,"share_id":"share-149","share_seq":149,"template_height":800004},{"accepted_at_ms":70510,"credit_policy":null,"job_id":"job-150","job_issued_at_ms":70484,"miner_id":"miner-7","network_difficulty":1000014,"ntime":1700000150,"order_key":"07:miner-7","p2mr_program_hex":"9696969696969696969696969696969696969696969696969696969696969696","share_difficulty":7,"share_id":"share-150","share_seq":150,"template_height":800004},{"accepted_at_ms":70500,"credit_policy":null,"job_id":"job-151","job_issued_at_ms":70500,"miner_id":"miner-8","network_difficulty":1000015,"ntime":1700000151,"order_key":"08:miner-8","p2mr_program_hex":"9797979797979797979797979797979797979797979797979797979797979797","share_difficulty":3,"share_id":"share-151","share_seq":151,"template_height":800004},{"accepted_at_ms":70483,"credit_policy":null,"job_id":"job-152","job_issued_at_ms":70512,"miner_id":"miner-9","network_difficulty":1000016,"ntime":1700000152,"order_key":"09:miner-9","p2mr_program_hex":"9898989898989898989898989898989898989898989898989898989898989898","share_difficulty":19,"share_id":"share-152","share_seq":152,"template_height":800004},{"accepted_at_ms":70484,"credit_policy":null,"job_id":"job-153","job_issued_at_ms":70510,"miner_id":"miner-10","network_difficulty":1000000,"ntime":1700000153,"order_key":"10:miner-10","p2mr_program_hex":"9999999999999999999999999999999999999999999999999999999999999999","share_difficulty":3,"share_id":"share-153","share_seq":153,"template_height":800004},{"accepted_at_ms":70500,"credit_policy":null,"job_id":"job-154","job_issued_at_ms":70509,"miner_id":"miner-0","network_difficulty":1000001,"ntime":1700000154,"order_key":"00:miner-0","p2mr_program_hex":"9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a","share_difficulty":19,"share_id":"share-154","share_seq":154,"template_height":800004},{"accepted_at_ms":70505,"credit_policy":null,"job_id":"job-155","job_issued_at_ms":70511,"miner_id":"miner-1","network_difficulty":1000002,"ntime":1700000155,"order_key":"01:miner-1","p2mr_program_hex":"9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b","share_difficulty":3,"share_id":"share-155","share_seq":155,"template_height":800004},{"accepted_at_ms":70511,"credit_policy":null,"job_id":"job-156","job_issued_at_ms":70487,"miner_id":"miner-2","network_difficulty":1000003,"ntime":1700000156,"order_key":"02:miner-2","p2mr_program_hex":"9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c","share_difficulty":19,"share_id":"share-156","share_seq":156,"template_height":800004},{"accepted_at_ms":70505,"credit_policy":null,"job_id":"job-157","job_issued_at_ms":70492,"miner_id":"miner-3","network_difficulty":1000004,"ntime":1700000157,"order_key":"03:miner-3","p2mr_program_hex":"9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d","share_difficulty":19,"share_id":"share-157","share_seq":157,"template_height":800004},{"accepted_at_ms":70494,"credit_policy":null,"job_id":"job-158","job_issued_at_ms":70498,"miner_id":"miner-4","network_difficulty":1000005,"ntime":1700000158,"order_key":"04:miner-4","p2mr_program_hex":"9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e","share_difficulty":3,"share_id":"share-158","share_seq":158,"template_height":800004},{"accepted_at_ms":70507,"credit_policy":null,"job_id":"job-159","job_issued_at_ms":70513,"miner_id":"miner-5","network_difficulty":1000006,"ntime":1700000159,"order_key":"05:miner-5","p2mr_program_hex":"9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f","share_difficulty":1,"share_id":"share-159","share_seq":159,"template_height":800004},{"accepted_at_ms":70505,"credit_policy":null,"job_id":"job-160","job_issued_at_ms":70507,"miner_id":"miner-6","network_difficulty":1000007,"ntime":1700000160,"order_key":"06:miner-6","p2mr_program_hex":"a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0","share_difficulty":1,"share_id":"share-160","share_seq":160,"template_height":800005},{"accepted_at_ms":70484,"credit_policy":null,"job_id":"job-161","job_issued_at_ms":70506,"miner_id":"miner-7","network_difficulty":1000008,"ntime":1700000161,"order_key":"07:miner-7","p2mr_program_hex":"a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1","share_difficulty":3,"share_id":"share-161","share_seq":161,"template_height":800005},{"accepted_at_ms":70494,"credit_policy":"stale-grace","job_id":"job-162","job_issued_at_ms":70489,"miner_id":"miner-8","network_difficulty":1000009,"ntime":1700000162,"order_key":"08:miner-8","p2mr_program_hex":"a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2","share_difficulty":19,"share_id":"share-162","share_seq":162,"template_height":800005},{"accepted_at_ms":70498,"credit_policy":null,"job_id":"job-163","job_issued_at_ms":70512,"miner_id":"miner-9","network_difficulty":1000010,"ntime":1700000163,"order_key":"09:miner-9","p2mr_program_hex":"a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3","share_difficulty":1,"share_id":"share-163","share_seq":163,"template_height":800005},{"accepted_at_ms":70502,"credit_policy":null,"job_id":"job-164","job_issued_at_ms":70512,"miner_id":"miner-10","network_difficulty":1000011,"ntime":1700000164,"order_key":"10:miner-10","p2mr_program_hex":"a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4","share_difficulty":7,"share_id":"share-164","share_seq":164,"template_height":800005},{"accepted_at_ms":70491,"credit_policy":null,"job_id":"job-165","job_issued_at_ms":70507,"miner_id":"miner-0","network_difficulty":1000012,"ntime":1700000165,"order_key":"00:miner-0","p2mr_program_hex":"a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5","share_difficulty":19,"share_id":"share-165","share_seq":165,"template_height":800005},{"accepted_at_ms":70498,"credit_policy":null,"job_id":"job-166","job_issued_at_ms":70483,"miner_id":"miner-1","network_difficulty":1000013,"ntime":1700000166,"order_key":"01:miner-1","p2mr_program_hex":"a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6","share_difficulty":7,"share_id":"share-166","share_seq":166,"template_height":800005},{"accepted_at_ms":70507,"credit_policy":null,"job_id":"job-167","job_issued_at_ms":70502,"miner_id":"miner-2","network_difficulty":1000014,"ntime":1700000167,"order_key":"02:miner-2","p2mr_program_hex":"a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7","share_difficulty":19,"share_id":"share-167","share_seq":167,"template_height":800005},{"accepted_at_ms":70504,"credit_policy":"stale-grace","job_id":"job-168","job_issued_at_ms":70512,"miner_id":"miner-3","network_difficulty":1000015,"ntime":1700000168,"order_key":"03:miner-3","p2mr_program_hex":"a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8","share_difficulty":7,"share_id":"share-168","share_seq":168,"template_height":800005},{"accepted_at_ms":70488,"credit_policy":"stale-grace","job_id":"job-169","job_issued_at_ms":70486,"miner_id":"miner-4","network_difficulty":1000016,"ntime":1700000169,"order_key":"04:miner-4","p2mr_program_hex":"a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9","share_difficulty":19,"share_id":"share-169","share_seq":169,"template_height":800005},{"accepted_at_ms":70507,"credit_policy":null,"job_id":"job-170","job_issued_at_ms":70508,"miner_id":"miner-5","network_difficulty":1000000,"ntime":1700000170,"order_key":"05:miner-5","p2mr_program_hex":"aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa","share_difficulty":3,"share_id":"share-170","share_seq":170,"template_height":800005},{"accepted_at_ms":70493,"credit_policy":null,"job_id":"job-171","job_issued_at_ms":70504,"miner_id":"miner-6","network_difficulty":1000001,"ntime":1700000171,"order_key":"06:miner-6","p2mr_program_hex":"abababababababababababababababababababababababababababababababab","share_difficulty":19,"share_id":"share-171","share_seq":171,"template_height":800005},{"accepted_at_ms":70490,"credit_policy":null,"job_id":"job-172","job_issued_at_ms":70484,"miner_id":"miner-7","network_difficulty":1000002,"ntime":1700000172,"order_key":"07:miner-7","p2mr_program_hex":"acacacacacacacacacacacacacacacacacacacacacacacacacacacacacacacac","share_difficulty":1,"share_id":"share-172","share_seq":172,"template_height":800005},{"accepted_at_ms":70511,"credit_policy":null,"job_id":"job-173","job_issued_at_ms":70486,"miner_id":"miner-8","network_difficulty":1000003,"ntime":1700000173,"order_key":"08:miner-8","p2mr_program_hex":"adadadadadadadadadadadadadadadadadadadadadadadadadadadadadadadad","share_difficulty":3,"share_id":"share-173","share_seq":173,"template_height":800005},{"accepted_at_ms":70501,"credit_policy":null,"job_id":"job-174","job_issued_at_ms":70483,"miner_id":"miner-9","network_difficulty":1000004,"ntime":1700000174,"order_key":"09:miner-9","p2mr_program_hex":"aeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeae","share_difficulty":1,"share_id":"share-174","share_seq":174,"template_height":800005},{"accepted_at_ms":70491,"credit_policy":null,"job_id":"job-175","job_issued_at_ms":70493,"miner_id":"miner-10","network_difficulty":1000005,"ntime":1700000175,"order_key":"10:miner-10","p2mr_program_hex":"afafafafafafafafafafafafafafafafafafafafafafafafafafafafafafafaf","share_difficulty":19,"share_id":"share-175","share_seq":175,"template_height":800005},{"accepted_at_ms":70497,"credit_policy":null,"job_id":"job-176","job_issued_at_ms":70489,"miner_id":"miner-0","network_difficulty":1000006,"ntime":1700000176,"order_key":"00:miner-0","p2mr_program_hex":"b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0","share_difficulty":7,"share_id":"share-176","share_seq":176,"template_height":800005},{"accepted_at_ms":70510,"credit_policy":null,"job_id":"job-177","job_issued_at_ms":70504,"miner_id":"miner-1","network_difficulty":1000007,"ntime":1700000177,"order_key":"01:miner-1","p2mr_program_hex":"b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1","share_difficulty":2,"share_id":"share-177","share_seq":177,"template_height":800005},{"accepted_at_ms":70483,"credit_policy":"stale-grace","job_id":"job-178","job_issued_at_ms":70489,"miner_id":"miner-2","network_difficulty":1000008,"ntime":1700000178,"order_key":"02:miner-2","p2mr_program_hex":"b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2","share_difficulty":1,"share_id":"share-178","share_seq":178,"template_height":800005},{"accepted_at_ms":70488,"credit_policy":null,"job_id":"job-179","job_issued_at_ms":70489,"miner_id":"miner-3","network_difficulty":1000009,"ntime":1700000179,"order_key":"03:miner-3","p2mr_program_hex":"b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3","share_difficulty":19,"share_id":"share-179","share_seq":179,"template_height":800005},{"accepted_at_ms":70492,"credit_policy":"stale-grace","job_id":"job-180","job_issued_at_ms":70492,"miner_id":"miner-4","network_difficulty":1000010,"ntime":1700000180,"order_key":"04:miner-4","p2mr_program_hex":"b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4","share_difficulty":1,"share_id":"share-180","share_seq":180,"template_height":800005},{"accepted_at_ms":70512,"credit_policy":null,"job_id":"job-181","job_issued_at_ms":70488,"miner_id":"miner-5","network_difficulty":1000011,"ntime":1700000181,"order_key":"05:miner-5","p2mr_program_hex":"b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5","share_difficulty":2,"share_id":"share-181","share_seq":181,"template_height":800005},{"accepted_at_ms":70511,"credit_policy":null,"job_id":"job-182","job_issued_at_ms":70501,"miner_id":"miner-6","network_difficulty":1000012,"ntime":1700000182,"order_key":"06:miner-6","p2mr_program_hex":"b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6","share_difficulty":2,"share_id":"share-182","share_seq":182,"template_height":800005},{"accepted_at_ms":70510,"credit_policy":null,"job_id":"job-183","job_issued_at_ms":70509,"miner_id":"miner-7","network_difficulty":1000013,"ntime":1700000183,"order_key":"07:miner-7","p2mr_program_hex":"b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7","share_difficulty":2,"share_id":"share-183","share_seq":183,"template_height":800005},{"accepted_at_ms":70489,"credit_policy":null,"job_id":"job-184","job_issued_at_ms":70489,"miner_id":"miner-8","network_difficulty":1000014,"ntime":1700000184,"order_key":"08:miner-8","p2mr_program_hex":"b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8","share_difficulty":3,"share_id":"share-184","share_seq":184,"template_height":800005},{"accepted_at_ms":70501,"credit_policy":null,"job_id":"job-185","job_issued_at_ms":70494,"miner_id":"miner-9","network_difficulty":1000015,"ntime":1700000185,"order_key":"09:miner-9","p2mr_program_hex":"b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9","share_difficulty":3,"share_id":"share-185","share_seq":185,"template_height":800005},{"accepted_at_ms":70492,"credit_policy":"stale-grace","job_id":"job-186","job_issued_at_ms":70497,"miner_id":"miner-10","network_difficulty":1000016,"ntime":1700000186,"order_key":"10:miner-10","p2mr_program_hex":"babababababababababababababababababababababababababababababababa","share_difficulty":19,"share_id":"share-186","share_seq":186,"template_height":800005},{"accepted_at_ms":70492,"credit_policy":null,"job_id":"job-187","job_issued_at_ms":70485,"miner_id":"miner-0","network_difficulty":1000000,"ntime":1700000187,"order_key":"00:miner-0","p2mr_program_hex":"bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb","share_difficulty":2,"share_id":"share-187","share_seq":187,"template_height":800005},{"accepted_at_ms":70500,"credit_policy":null,"job_id":"job-188","job_issued_at_ms":70502,"miner_id":"miner-1","network_difficulty":1000001,"ntime":1700000188,"order_key":"01:miner-1","p2mr_program_hex":"bcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbc","share_difficulty":3,"share_id":"share-188","share_seq":188,"template_height":800005},{"accepted_at_ms":70505,"credit_policy":"stale-grace","job_id":"job-189","job_issued_at_ms":70497,"miner_id":"miner-2","network_difficulty":1000002,"ntime":1700000189,"order_key":"02:miner-2","p2mr_program_hex":"bdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbd","share_difficulty":7,"share_id":"share-189","share_seq":189,"template_height":800005},{"accepted_at_ms":70513,"credit_policy":null,"job_id":"job-190","job_issued_at_ms":70486,"miner_id":"miner-3","network_difficulty":1000003,"ntime":1700000190,"order_key":"03:miner-3","p2mr_program_hex":"bebebebebebebebebebebebebebebebebebebebebebebebebebebebebebebebe","share_difficulty":1,"share_id":"share-190","share_seq":190,"template_height":800005},{"accepted_at_ms":70509,"credit_policy":null,"job_id":"job-191","job_issued_at_ms":70492,"miner_id":"miner-4","network_difficulty":1000004,"ntime":1700000191,"order_key":"04:miner-4","p2mr_program_hex":"bfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbf","share_difficulty":1,"share_id":"share-191","share_seq":191,"template_height":800005},{"accepted_at_ms":70491,"credit_policy":null,"job_id":"job-192","job_issued_at_ms":70492,"miner_id":"miner-5","network_difficulty":1000005,"ntime":1700000192,"order_key":"05:miner-5","p2mr_program_hex":"c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0","share_difficulty":19,"share_id":"share-192","share_seq":192,"template_height":800006},{"accepted_at_ms":70484,"credit_policy":null,"job_id":"job-193","job_issued_at_ms":70489,"miner_id":"miner-6","network_difficulty":1000006,"ntime":1700000193,"order_key":"06:miner-6","p2mr_program_hex":"c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1","share_difficulty":7,"share_id":"share-193","share_seq":193,"template_height":800006},{"accepted_at_ms":70511,"credit_policy":"stale-grace","job_id":"job-194","job_issued_at_ms":70495,"miner_id":"miner-7","network_difficulty":1000007,"ntime":1700000194,"order_key":"07:miner-7","p2mr_program_hex":"c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2","share_difficulty":1,"share_id":"share-194","share_seq":194,"template_height":800006},{"accepted_at_ms":70508,"credit_policy":null,"job_id":"job-195","job_issued_at_ms":70513,"miner_id":"miner-8","network_difficulty":1000008,"ntime":1700000195,"order_key":"08:miner-8","p2mr_program_hex":"c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3","share_difficulty":7,"share_id":"share-195","share_seq":195,"template_height":800006},{"accepted_at_ms":70503,"credit_policy":"stale-grace","job_id":"job-196","job_issued_at_ms":70486,"miner_id":"miner-9","network_difficulty":1000009,"ntime":1700000196,"order_key":"09:miner-9","p2mr_program_hex":"c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4","share_difficulty":1,"share_id":"share-196","share_seq":196,"template_height":800006},{"accepted_at_ms":70499,"credit_policy":"stale-grace","job_id":"job-197","job_issued_at_ms":70484,"miner_id":"miner-10","network_difficulty":1000010,"ntime":1700000197,"order_key":"10:miner-10","p2mr_program_hex":"c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5","share_difficulty":1,"share_id":"share-197","share_seq":197,"template_height":800006},{"accepted_at_ms":70509,"credit_policy":"stale-grace","job_id":"job-198","job_issued_at_ms":70511,"miner_id":"miner-0","network_difficulty":1000011,"ntime":1700000198,"order_key":"00:miner-0","p2mr_program_hex":"c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6","share_difficulty":19,"share_id":"share-198","share_seq":198,"template_height":800006},{"accepted_at_ms":70503,"credit_policy":null,"job_id":"job-199","job_issued_at_ms":70500,"miner_id":"miner-1","network_difficulty":1000012,"ntime":1700000199,"order_key":"01:miner-1","p2mr_program_hex":"c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7","share_difficulty":1,"share_id":"share-199","share_seq":199,"template_height":800006},{"accepted_at_ms":70483,"credit_policy":null,"job_id":"job-200","job_issued_at_ms":70486,"miner_id":"miner-2","network_difficulty":1000013,"ntime":1700000200,"order_key":"02:miner-2","p2mr_program_hex":"c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8","share_difficulty":19,"share_id":"share-200","share_seq":200,"template_height":800006},{"accepted_at_ms":70483,"credit_policy":"stale-grace","job_id":"job-201","job_issued_at_ms":70502,"miner_id":"miner-3","network_difficulty":1000014,"ntime":1700000201,"order_key":"03:miner-3","p2mr_program_hex":"c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9","share_difficulty":19,"share_id":"share-201","share_seq":201,"template_height":800006},{"accepted_at_ms":70509,"credit_policy":"stale-grace","job_id":"job-202","job_issued_at_ms":70504,"miner_id":"miner-4","network_difficulty":1000015,"ntime":1700000202,"order_key":"04:miner-4","p2mr_program_hex":"cacacacacacacacacacacacacacacacacacacacacacacacacacacacacacacaca","share_difficulty":7,"share_id":"share-202","share_seq":202,"template_height":800006},{"accepted_at_ms":70486,"credit_policy":null,"job_id":"job-203","job_issued_at_ms":70490,"miner_id":"miner-5","network_difficulty":1000016,"ntime":1700000203,"order_key":"05:miner-5","p2mr_program_hex":"cbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcb","share_difficulty":3,"share_id":"share-203","share_seq":203,"template_height":800006},{"accepted_at_ms":70498,"credit_policy":null,"job_id":"job-204","job_issued_at_ms":70491,"miner_id":"miner-6","network_difficulty":1000000,"ntime":1700000204,"order_key":"06:miner-6","p2mr_program_hex":"cccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccc","share_difficulty":2,"share_id":"share-204","share_seq":204,"template_height":800006},{"accepted_at_ms":70495,"credit_policy":null,"job_id":"job-205","job_issued_at_ms":70511,"miner_id":"miner-7","network_difficulty":1000001,"ntime":1700000205,"order_key":"07:miner-7","p2mr_program_hex":"cdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcd","share_difficulty":7,"share_id":"share-205","share_seq":205,"template_height":800006},{"accepted_at_ms":70487,"credit_policy":null,"job_id":"job-206","job_issued_at_ms":70504,"miner_id":"miner-8","network_difficulty":1000002,"ntime":1700000206,"order_key":"08:miner-8","p2mr_program_hex":"cececececececececececececececececececececececececececececececece","share_difficulty":2,"share_id":"share-206","share_seq":206,"template_height":800006},{"accepted_at_ms":70512,"credit_policy":null,"job_id":"job-207","job_issued_at_ms":70510,"miner_id":"miner-9","network_difficulty":1000003,"ntime":1700000207,"order_key":"09:miner-9","p2mr_program_hex":"cfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcf","share_difficulty":3,"share_id":"share-207","share_seq":207,"template_height":800006},{"accepted_at_ms":70504,"credit_policy":null,"job_id":"job-208","job_issued_at_ms":70487,"miner_id":"miner-10","network_difficulty":1000004,"ntime":1700000208,"order_key":"10:miner-10","p2mr_program_hex":"d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0","share_difficulty":7,"share_id":"share-208","share_seq":208,"template_height":800006},{"accepted_at_ms":70495,"credit_policy":null,"job_id":"job-209","job_issued_at_ms":70511,"miner_id":"miner-0","network_difficulty":1000005,"ntime":1700000209,"order_key":"00:miner-0","p2mr_program_hex":"d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1","share_difficulty":7,"share_id":"share-209","share_seq":209,"template_height":800006},{"accepted_at_ms":70505,"credit_policy":null,"job_id":"job-210","job_issued_at_ms":70509,"miner_id":"miner-1","network_difficulty":1000006,"ntime":1700000210,"order_key":"01:miner-1","p2mr_program_hex":"d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2","share_difficulty":3,"share_id":"share-210","share_seq":210,"template_height":800006},{"accepted_at_ms":70507,"credit_policy":"stale-grace","job_id":"job-211","job_issued_at_ms":70503,"miner_id":"miner-2","network_difficulty":1000007,"ntime":1700000211,"order_key":"02:miner-2","p2mr_program_hex":"d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3","share_difficulty":3,"share_id":"share-211","share_seq":211,"template_height":800006},{"accepted_at_ms":70501,"credit_policy":"stale-grace","job_id":"job-212","job_issued_at_ms":70487,"miner_id":"miner-3","network_difficulty":1000008,"ntime":1700000212,"order_key":"03:miner-3","p2mr_program_hex":"d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4","share_difficulty":1,"share_id":"share-212","share_seq":212,"template_height":800006},{"accepted_at_ms":70485,"credit_policy":"stale-grace","job_id":"job-213","job_issued_at_ms":70487,"miner_id":"miner-4","network_difficulty":1000009,"ntime":1700000213,"order_key":"04:miner-4","p2mr_program_hex":"d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5","share_difficulty":7,"share_id":"share-213","share_seq":213,"template_height":800006},{"accepted_at_ms":70499,"credit_policy":null,"job_id":"job-214","job_issued_at_ms":70496,"miner_id":"miner-5","network_difficulty":1000010,"ntime":1700000214,"order_key":"05:miner-5","p2mr_program_hex":"d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6","share_difficulty":1,"share_id":"share-214","share_seq":214,"template_height":800006},{"accepted_at_ms":70498,"credit_policy":null,"job_id":"job-215","job_issued_at_ms":70487,"miner_id":"miner-6","network_difficulty":1000011,"ntime":1700000215,"order_key":"06:miner-6","p2mr_program_hex":"d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7","share_difficulty":2,"share_id":"share-215","share_seq":215,"template_height":800006},{"accepted_at_ms":70490,"credit_policy":null,"job_id":"job-216","job_issued_at_ms":70488,"miner_id":"miner-7","network_difficulty":1000012,"ntime":1700000216,"order_key":"07:miner-7","p2mr_program_hex":"d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8","share_difficulty":1,"share_id":"share-216","share_seq":216,"template_height":800006},{"accepted_at_ms":70508,"credit_policy":null,"job_id":"job-217","job_issued_at_ms":70495,"miner_id":"miner-8","network_difficulty":1000013,"ntime":1700000217,"order_key":"08:miner-8","p2mr_program_hex":"d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9","share_difficulty":19,"share_id":"share-217","share_seq":217,"template_height":800006},{"accepted_at_ms":70509,"credit_policy":"stale-grace","job_id":"job-218","job_issued_at_ms":70511,"miner_id":"miner-9","network_difficulty":1000014,"ntime":1700000218,"order_key":"09:miner-9","p2mr_program_hex":"dadadadadadadadadadadadadadadadadadadadadadadadadadadadadadadada","share_difficulty":2,"share_id":"share-218","share_seq":218,"template_height":800006},{"accepted_at_ms":70502,"credit_policy":null,"job_id":"job-219","job_issued_at_ms":70483,"miner_id":"miner-10","network_difficulty":1000015,"ntime":1700000219,"order_key":"10:miner-10","p2mr_program_hex":"dbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdb","share_difficulty":3,"share_id":"share-219","share_seq":219,"template_height":800006},{"accepted_at_ms":70498,"credit_policy":"stale-grace","job_id":"job-220","job_issued_at_ms":70489,"miner_id":"miner-0","network_difficulty":1000016,"ntime":1700000220,"order_key":"00:miner-0","p2mr_program_hex":"dcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdc","share_difficulty":1,"share_id":"share-220","share_seq":220,"template_height":800006},{"accepted_at_ms":70495,"credit_policy":null,"job_id":"job-221","job_issued_at_ms":70507,"miner_id":"miner-1","network_difficulty":1000000,"ntime":1700000221,"order_key":"01:miner-1","p2mr_program_hex":"dddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddd","share_difficulty":1,"share_id":"share-221","share_seq":221,"template_height":800006},{"accepted_at_ms":70497,"credit_policy":null,"job_id":"job-222","job_issued_at_ms":70511,"miner_id":"miner-2","network_difficulty":1000001,"ntime":1700000222,"order_key":"02:miner-2","p2mr_program_hex":"dededededededededededededededededededededededededededededededede","share_difficulty":3,"share_id":"share-222","share_seq":222,"template_height":800006},{"accepted_at_ms":70489,"credit_policy":"stale-grace","job_id":"job-223","job_issued_at_ms":70497,"miner_id":"miner-3","network_difficulty":1000002,"ntime":1700000223,"order_key":"03:miner-3","p2mr_program_hex":"dfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdf","share_difficulty":1,"share_id":"share-223","share_seq":223,"template_height":800006},{"accepted_at_ms":70505,"credit_policy":"stale-grace","job_id":"job-224","job_issued_at_ms":70508,"miner_id":"miner-4","network_difficulty":1000003,"ntime":1700000224,"order_key":"04:miner-4","p2mr_program_hex":"e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0","share_difficulty":19,"share_id":"share-224","share_seq":224,"template_height":800007},{"accepted_at_ms":70503,"credit_policy":null,"job_id":"job-225","job_issued_at_ms":70492,"miner_id":"miner-5","network_difficulty":1000004,"ntime":1700000225,"order_key":"05:miner-5","p2mr_program_hex":"e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1","share_difficulty":7,"share_id":"share-225","share_seq":225,"template_height":800007},{"accepted_at_ms":70496,"credit_policy":"stale-grace","job_id":"job-226","job_issued_at_ms":70487,"miner_id":"miner-6","network_difficulty":1000005,"ntime":1700000226,"order_key":"06:miner-6","p2mr_program_hex":"e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2","share_difficulty":2,"share_id":"share-226","share_seq":226,"template_height":800007},{"accepted_at_ms":70488,"credit_policy":null,"job_id":"job-227","job_issued_at_ms":70483,"miner_id":"miner-7","network_difficulty":1000006,"ntime":1700000227,"order_key":"07:miner-7","p2mr_program_hex":"e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3","share_difficulty":7,"share_id":"share-227","share_seq":227,"template_height":800007},{"accepted_at_ms":70486,"credit_policy":null,"job_id":"job-228","job_issued_at_ms":70496,"miner_id":"miner-8","network_difficulty":1000007,"ntime":1700000228,"order_key":"08:miner-8","p2mr_program_hex":"e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4","share_difficulty":19,"share_id":"share-228","share_seq":228,"template_height":800007},{"accepted_at_ms":70488,"credit_policy":null,"job_id":"job-229","job_issued_at_ms":70488,"miner_id":"miner-9","network_difficulty":1000008,"ntime":1700000229,"order_key":"09:miner-9","p2mr_program_hex":"e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5","share_difficulty":1,"share_id":"share-229","share_seq":229,"template_height":800007},{"accepted_at_ms":70489,"credit_policy":"stale-grace","job_id":"job-230","job_issued_at_ms":70484,"miner_id":"miner-10","network_difficulty":1000009,"ntime":1700000230,"order_key":"10:miner-10","p2mr_program_hex":"e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6","share_difficulty":19,"share_id":"share-230","share_seq":230,"template_height":800007},{"accepted_at_ms":70512,"credit_policy":null,"job_id":"job-231","job_issued_at_ms":70489,"miner_id":"miner-0","network_difficulty":1000010,"ntime":1700000231,"order_key":"00:miner-0","p2mr_program_hex":"e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7","share_difficulty":19,"share_id":"share-231","share_seq":231,"template_height":800007},{"accepted_at_ms":70494,"credit_policy":null,"job_id":"job-232","job_issued_at_ms":70484,"miner_id":"miner-1","network_difficulty":1000011,"ntime":1700000232,"order_key":"01:miner-1","p2mr_program_hex":"e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8","share_difficulty":3,"share_id":"share-232","share_seq":232,"template_height":800007},{"accepted_at_ms":70483,"credit_policy":null,"job_id":"job-233","job_issued_at_ms":70508,"miner_id":"miner-2","network_difficulty":1000012,"ntime":1700000233,"order_key":"02:miner-2","p2mr_program_hex":"e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9","share_difficulty":3,"share_id":"share-233","share_seq":233,"template_height":800007},{"accepted_at_ms":70483,"credit_policy":null,"job_id":"job-234","job_issued_at_ms":70493,"miner_id":"miner-3","network_difficulty":1000013,"ntime":1700000234,"order_key":"03:miner-3","p2mr_program_hex":"eaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaea","share_difficulty":7,"share_id":"share-234","share_seq":234,"template_height":800007},{"accepted_at_ms":70493,"credit_policy":null,"job_id":"job-235","job_issued_at_ms":70500,"miner_id":"miner-4","network_difficulty":1000014,"ntime":1700000235,"order_key":"04:miner-4","p2mr_program_hex":"ebebebebebebebebebebebebebebebebebebebebebebebebebebebebebebebeb","share_difficulty":2,"share_id":"share-235","share_seq":235,"template_height":800007},{"accepted_at_ms":70500,"credit_policy":null,"job_id":"job-236","job_issued_at_ms":70503,"miner_id":"miner-5","network_difficulty":1000015,"ntime":1700000236,"order_key":"05:miner-5","p2mr_program_hex":"ecececececececececececececececececececececececececececececececec","share_difficulty":3,"share_id":"share-236","share_seq":236,"template_height":800007},{"accepted_at_ms":70504,"credit_policy":null,"job_id":"job-237","job_issued_at_ms":70509,"miner_id":"miner-6","network_difficulty":1000016,"ntime":1700000237,"order_key":"06:miner-6","p2mr_program_hex":"edededededededededededededededededededededededededededededededed","share_difficulty":3,"share_id":"share-237","share_seq":237,"template_height":800007},{"accepted_at_ms":70513,"credit_policy":null,"job_id":"job-238","job_issued_at_ms":70507,"miner_id":"miner-7","network_difficulty":1000000,"ntime":1700000238,"order_key":"07:miner-7","p2mr_program_hex":"eeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeee","share_difficulty":3,"share_id":"share-238","share_seq":238,"template_height":800007},{"accepted_at_ms":70490,"credit_policy":"stale-grace","job_id":"job-239","job_issued_at_ms":70489,"miner_id":"miner-8","network_difficulty":1000001,"ntime":1700000239,"order_key":"08:miner-8","p2mr_program_hex":"efefefefefefefefefefefefefefefefefefefefefefefefefefefefefefefef","share_difficulty":2,"share_id":"share-239","share_seq":239,"template_height":800007},{"accepted_at_ms":70512,"credit_policy":"stale-grace","job_id":"job-240","job_issued_at_ms":70492,"miner_id":"miner-9","network_difficulty":1000002,"ntime":1700000240,"order_key":"09:miner-9","p2mr_program_hex":"f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0","share_difficulty":19,"share_id":"share-240","share_seq":240,"template_height":800007},{"accepted_at_ms":70502,"credit_policy":null,"job_id":"job-241","job_issued_at_ms":70486,"miner_id":"miner-10","network_difficulty":1000003,"ntime":1700000241,"order_key":"10:miner-10","p2mr_program_hex":"f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1","share_difficulty":2,"share_id":"share-241","share_seq":241,"template_height":800007},{"accepted_at_ms":70509,"credit_policy":null,"job_id":"job-242","job_issued_at_ms":70487,"miner_id":"miner-0","network_difficulty":1000004,"ntime":1700000242,"order_key":"00:miner-0","p2mr_program_hex":"f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2","share_difficulty":2,"share_id":"share-242","share_seq":242,"template_height":800007},{"accepted_at_ms":70497,"credit_policy":null,"job_id":"job-243","job_issued_at_ms":70509,"miner_id":"miner-1","network_difficulty":1000005,"ntime":1700000243,"order_key":"01:miner-1","p2mr_program_hex":"f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3","share_difficulty":19,"share_id":"share-243","share_seq":243,"template_height":800007},{"accepted_at_ms":70494,"credit_policy":null,"job_id":"job-244","job_issued_at_ms":70509,"miner_id":"miner-2","network_difficulty":1000006,"ntime":1700000244,"order_key":"02:miner-2","p2mr_program_hex":"f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4","share_difficulty":19,"share_id":"share-244","share_seq":244,"template_height":800007},{"accepted_at_ms":70506,"credit_policy":"stale-grace","job_id":"job-245","job_issued_at_ms":70484,"miner_id":"miner-3","network_difficulty":1000007,"ntime":1700000245,"order_key":"03:miner-3","p2mr_program_hex":"f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5","share_difficulty":19,"share_id":"share-245","share_seq":245,"template_height":800007},{"accepted_at_ms":70483,"credit_policy":null,"job_id":"job-246","job_issued_at_ms":70501,"miner_id":"miner-4","network_difficulty":1000008,"ntime":1700000246,"order_key":"04:miner-4","p2mr_program_hex":"f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6","share_difficulty":3,"share_id":"share-246","share_seq":246,"template_height":800007},{"accepted_at_ms":70513,"credit_policy":"stale-grace","job_id":"job-247","job_issued_at_ms":70506,"miner_id":"miner-5","network_difficulty":1000009,"ntime":1700000247,"order_key":"05:miner-5","p2mr_program_hex":"f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7","share_difficulty":3,"share_id":"share-247","share_seq":247,"template_height":800007},{"accepted_at_ms":70512,"credit_policy":null,"job_id":"job-248","job_issued_at_ms":70489,"miner_id":"miner-6","network_difficulty":1000010,"ntime":1700000248,"order_key":"06:miner-6","p2mr_program_hex":"f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8","share_difficulty":3,"share_id":"share-248","share_seq":248,"template_height":800007},{"accepted_at_ms":70494,"credit_policy":null,"job_id":"job-249","job_issued_at_ms":70487,"miner_id":"miner-7","network_difficulty":1000011,"ntime":1700000249,"order_key":"07:miner-7","p2mr_program_hex":"f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9","share_difficulty":3,"share_id":"share-249","share_seq":249,"template_height":800007},{"accepted_at_ms":70505,"credit_policy":"stale-grace","job_id":"job-250","job_issued_at_ms":70497,"miner_id":"miner-8","network_difficulty":1000012,"ntime":1700000250,"order_key":"08:miner-8","p2mr_program_hex":"fafafafafafafafafafafafafafafafafafafafafafafafafafafafafafafafa","share_difficulty":7,"share_id":"share-250","share_seq":250,"template_height":800007},{"accepted_at_ms":70486,"credit_policy":null,"job_id":"job-251","job_issued_at_ms":70494,"miner_id":"miner-9","network_difficulty":1000013,"ntime":1700000251,"order_key":"09:miner-9","p2mr_program_hex":"fbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfb","share_difficulty":19,"share_id":"share-251","share_seq":251,"template_height":800007},{"accepted_at_ms":70508,"credit_policy":null,"job_id":"job-252","job_issued_at_ms":70500,"miner_id":"miner-10","network_difficulty":1000014,"ntime":1700000252,"order_key":"10:miner-10","p2mr_program_hex":"fcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfc","share_difficulty":19,"share_id":"share-252","share_seq":252,"template_height":800007},{"accepted_at_ms":70510,"credit_policy":null,"job_id":"job-253","job_issued_at_ms":70507,"miner_id":"miner-0","network_difficulty":1000015,"ntime":1700000253,"order_key":"00:miner-0","p2mr_program_hex":"fdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfd","share_difficulty":2,"share_id":"share-253","share_seq":253,"template_height":800007},{"accepted_at_ms":70510,"credit_policy":null,"job_id":"job-254","job_issued_at_ms":70503,"miner_id":"miner-1","network_difficulty":1000016,"ntime":1700000254,"order_key":"01:miner-1","p2mr_program_hex":"fefefefefefefefefefefefefefefefefefefefefefefefefefefefefefefefe","share_difficulty":3,"share_id":"share-254","share_seq":254,"template_height":800007},{"accepted_at_ms":70491,"credit_policy":null,"job_id":"job-255","job_issued_at_ms":70487,"miner_id":"miner-2","network_difficulty":1000000,"ntime":1700000255,"order_key":"02:miner-2","p2mr_program_hex":"ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff","share_difficulty":7,"share_id":"share-255","share_seq":255,"template_height":800007},{"accepted_at_ms":70510,"credit_policy":null,"job_id":"job-256","job_issued_at_ms":70485,"miner_id":"miner-3","network_difficulty":1000001,"ntime":1700000256,"order_key":"03:miner-3","p2mr_program_hex":"0000000000000000000000000000000000000000000000000000000000000000","share_difficulty":1,"share_id":"share-256","share_seq":256,"template_height":800008},{"accepted_at_ms":70505,"credit_policy":null,"job_id":"job-257","job_issued_at_ms":70512,"miner_id":"miner-4","network_difficulty":1000002,"ntime":1700000257,"order_key":"04:miner-4","p2mr_program_hex":"0101010101010101010101010101010101010101010101010101010101010101","share_difficulty":3,"share_id":"share-257","share_seq":257,"template_height":800008},{"accepted_at_ms":70493,"credit_policy":null,"job_id":"job-258","job_issued_at_ms":70507,"miner_id":"miner-5","network_difficulty":1000003,"ntime":1700000258,"order_key":"05:miner-5","p2mr_program_hex":"0202020202020202020202020202020202020202020202020202020202020202","share_difficulty":1,"share_id":"share-258","share_seq":258,"template_height":800008},{"accepted_at_ms":70486,"credit_policy":null,"job_id":"job-259","job_issued_at_ms":70495,"miner_id":"miner-6","network_difficulty":1000004,"ntime":1700000259,"order_key":"06:miner-6","p2mr_program_hex":"0303030303030303030303030303030303030303030303030303030303030303","share_difficulty":2,"share_id":"share-259","share_seq":259,"template_height":800008},{"accepted_at_ms":70499,"credit_policy":null,"job_id":"job-260","job_issued_at_ms":70500,"miner_id":"miner-7","network_difficulty":1000005,"ntime":1700000260,"order_key":"07:miner-7","p2mr_program_hex":"0404040404040404040404040404040404040404040404040404040404040404","share_difficulty":19,"share_id":"share-260","share_seq":260,"template_height":800008},{"accepted_at_ms":70507,"credit_policy":null,"job_id":"job-261","job_issued_at_ms":70503,"miner_id":"miner-8","network_difficulty":1000006,"ntime":1700000261,"order_key":"08:miner-8","p2mr_program_hex":"0505050505050505050505050505050505050505050505050505050505050505","share_difficulty":19,"share_id":"share-261","share_seq":261,"template_height":800008},{"accepted_at_ms":70499,"credit_policy":null,"job_id":"job-262","job_issued_at_ms":70510,"miner_id":"miner-9","network_difficulty":1000007,"ntime":1700000262,"order_key":"09:miner-9","p2mr_program_hex":"0606060606060606060606060606060606060606060606060606060606060606","share_difficulty":7,"share_id":"share-262","share_seq":262,"template_height":800008},{"accepted_at_ms":70493,"credit_policy":null,"job_id":"job-263","job_issued_at_ms":70501,"miner_id":"miner-10","network_difficulty":1000008,"ntime":1700000263,"order_key":"10:miner-10","p2mr_program_hex":"0707070707070707070707070707070707070707070707070707070707070707","share_difficulty":1,"share_id":"share-263","share_seq":263,"template_height":800008},{"accepted_at_ms":70491,"credit_policy":null,"job_id":"job-264","job_issued_at_ms":70503,"miner_id":"miner-0","network_difficulty":1000009,"ntime":1700000264,"order_key":"00:miner-0","p2mr_program_hex":"0808080808080808080808080808080808080808080808080808080808080808","share_difficulty":2,"share_id":"share-264","share_seq":264,"template_height":800008},{"accepted_at_ms":70493,"credit_policy":null,"job_id":"job-265","job_issued_at_ms":70509,"miner_id":"miner-1","network_difficulty":1000010,"ntime":1700000265,"order_key":"01:miner-1","p2mr_program_hex":"0909090909090909090909090909090909090909090909090909090909090909","share_difficulty":19,"share_id":"share-265","share_seq":265,"template_height":800008},{"accepted_at_ms":70496,"credit_policy":null,"job_id":"job-266","job_issued_at_ms":70488,"miner_id":"miner-2","network_difficulty":1000011,"ntime":1700000266,"order_key":"02:miner-2","p2mr_program_hex":"0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a","share_difficulty":19,"share_id":"share-266","share_seq":266,"template_height":800008},{"accepted_at_ms":70493,"credit_policy":null,"job_id":"job-267","job_issued_at_ms":70506,"miner_id":"miner-3","network_difficulty":1000012,"ntime":1700000267,"order_key":"03:miner-3","p2mr_program_hex":"0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b","share_difficulty":7,"share_id":"share-267","share_seq":267,"template_height":800008},{"accepted_at_ms":70498,"credit_policy":null,"job_id":"job-268","job_issued_at_ms":70490,"miner_id":"miner-4","network_difficulty":1000013,"ntime":1700000268,"order_key":"04:miner-4","p2mr_program_hex":"0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c","share_difficulty":3,"share_id":"share-268","share_seq":268,"template_height":800008},{"accepted_at_ms":70489,"credit_policy":null,"job_id":"job-269","job_issued_at_ms":70497,"miner_id":"miner-5","network_difficulty":1000014,"ntime":1700000269,"order_key":"05:miner-5","p2mr_program_hex":"0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d","share_difficulty":7,"share_id":"share-269","share_seq":269,"template_height":800008},{"accepted_at_ms":70501,"credit_policy":"stale-grace","job_id":"job-270","job_issued_at_ms":70502,"miner_id":"miner-6","network_difficulty":1000015,"ntime":1700000270,"order_key":"06:miner-6","p2mr_program_hex":"0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e","share_difficulty":7,"share_id":"share-270","share_seq":270,"template_height":800008},{"accepted_at_ms":70487,"credit_policy":null,"job_id":"job-271","job_issued_at_ms":70483,"miner_id":"miner-7","network_difficulty":1000016,"ntime":1700000271,"order_key":"07:miner-7","p2mr_program_hex":"0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f","share_difficulty":3,"share_id":"share-271","share_seq":271,"template_height":800008},{"accepted_at_ms":70509,"credit_policy":"stale-grace","job_id":"job-272","job_issued_at_ms":70488,"miner_id":"miner-8","network_difficulty":1000000,"ntime":1700000272,"order_key":"08:miner-8","p2mr_program_hex":"1010101010101010101010101010101010101010101010101010101010101010","share_difficulty":19,"share_id":"share-272","share_seq":272,"template_height":800008},{"accepted_at_ms":70506,"credit_policy":null,"job_id":"job-273","job_issued_at_ms":70504,"miner_id":"miner-9","network_difficulty":1000001,"ntime":1700000273,"order_key":"09:miner-9","p2mr_program_hex":"1111111111111111111111111111111111111111111111111111111111111111","share_difficulty":2,"share_id":"share-273","share_seq":273,"template_height":800008},{"accepted_at_ms":70498,"credit_policy":null,"job_id":"job-274","job_issued_at_ms":70497,"miner_id":"miner-10","network_difficulty":1000002,"ntime":1700000274,"order_key":"10:miner-10","p2mr_program_hex":"1212121212121212121212121212121212121212121212121212121212121212","share_difficulty":19,"share_id":"share-274","share_seq":274,"template_height":800008},{"accepted_at_ms":70505,"credit_policy":null,"job_id":"job-275","job_issued_at_ms":70508,"miner_id":"miner-0","network_difficulty":1000003,"ntime":1700000275,"order_key":"00:miner-0","p2mr_program_hex":"1313131313131313131313131313131313131313131313131313131313131313","share_difficulty":3,"share_id":"share-275","share_seq":275,"template_height":800008},{"accepted_at_ms":70504,"credit_policy":null,"job_id":"job-276","job_issued_at_ms":70484,"miner_id":"miner-1","network_difficulty":1000004,"ntime":1700000276,"order_key":"01:miner-1","p2mr_program_hex":"1414141414141414141414141414141414141414141414141414141414141414","share_difficulty":3,"share_id":"share-276","share_seq":276,"template_height":800008},{"accepted_at_ms":70501,"credit_policy":null,"job_id":"job-277","job_issued_at_ms":70503,"miner_id":"miner-2","network_difficulty":1000005,"ntime":1700000277,"order_key":"02:miner-2","p2mr_program_hex":"1515151515151515151515151515151515151515151515151515151515151515","share_difficulty":3,"share_id":"share-277","share_seq":277,"template_height":800008},{"accepted_at_ms":70502,"credit_policy":null,"job_id":"job-278","job_issued_at_ms":70507,"miner_id":"miner-3","network_difficulty":1000006,"ntime":1700000278,"order_key":"03:miner-3","p2mr_program_hex":"1616161616161616161616161616161616161616161616161616161616161616","share_difficulty":3,"share_id":"share-278","share_seq":278,"template_height":800008},{"accepted_at_ms":70490,"credit_policy":null,"job_id":"job-279","job_issued_at_ms":70500,"miner_id":"miner-4","network_difficulty":1000007,"ntime":1700000279,"order_key":"04:miner-4","p2mr_program_hex":"1717171717171717171717171717171717171717171717171717171717171717","share_difficulty":7,"share_id":"share-279","share_seq":279,"template_height":800008},{"accepted_at_ms":70505,"credit_policy":null,"job_id":"job-280","job_issued_at_ms":70494,"miner_id":"miner-5","network_difficulty":1000008,"ntime":1700000280,"order_key":"05:miner-5","p2mr_program_hex":"1818181818181818181818181818181818181818181818181818181818181818","share_difficulty":7,"share_id":"share-280","share_seq":280,"template_height":800008},{"accepted_at_ms":70507,"credit_policy":null,"job_id":"job-281","job_issued_at_ms":70486,"miner_id":"miner-6","network_difficulty":1000009,"ntime":1700000281,"order_key":"06:miner-6","p2mr_program_hex":"1919191919191919191919191919191919191919191919191919191919191919","share_difficulty":3,"share_id":"share-281","share_seq":281,"template_height":800008},{"accepted_at_ms":70494,"credit_policy":"stale-grace","job_id":"job-282","job_issued_at_ms":70504,"miner_id":"miner-7","network_difficulty":1000010,"ntime":1700000282,"order_key":"07:miner-7","p2mr_program_hex":"1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a","share_difficulty":1,"share_id":"share-282","share_seq":282,"template_height":800008},{"accepted_at_ms":70487,"credit_policy":null,"job_id":"job-283","job_issued_at_ms":70492,"miner_id":"miner-8","network_difficulty":1000011,"ntime":1700000283,"order_key":"08:miner-8","p2mr_program_hex":"1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b","share_difficulty":2,"share_id":"share-283","share_seq":283,"template_height":800008},{"accepted_at_ms":70484,"credit_policy":null,"job_id":"job-284","job_issued_at_ms":70510,"miner_id":"miner-9","network_difficulty":1000012,"ntime":1700000284,"order_key":"09:miner-9","p2mr_program_hex":"1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c","share_difficulty":1,"share_id":"share-284","share_seq":284,"template_height":800008},{"accepted_at_ms":70493,"credit_policy":null,"job_id":"job-285","job_issued_at_ms":70487,"miner_id":"miner-10","network_difficulty":1000013,"ntime":1700000285,"order_key":"10:miner-10","p2mr_program_hex":"1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d","share_difficulty":19,"share_id":"share-285","share_seq":285,"template_height":800008},{"accepted_at_ms":70486,"credit_policy":"stale-grace","job_id":"job-286","job_issued_at_ms":70504,"miner_id":"miner-0","network_difficulty":1000014,"ntime":1700000286,"order_key":"00:miner-0","p2mr_program_hex":"1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e","share_difficulty":19,"share_id":"share-286","share_seq":286,"template_height":800008},{"accepted_at_ms":70511,"credit_policy":"stale-grace","job_id":"job-287","job_issued_at_ms":70486,"miner_id":"miner-1","network_difficulty":1000015,"ntime":1700000287,"order_key":"01:miner-1","p2mr_program_hex":"1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f","share_difficulty":7,"share_id":"share-287","share_seq":287,"template_height":800008},{"accepted_at_ms":70489,"credit_policy":"stale-grace","job_id":"job-288","job_issued_at_ms":70483,"miner_id":"miner-2","network_difficulty":1000016,"ntime":1700000288,"order_key":"02:miner-2","p2mr_program_hex":"2020202020202020202020202020202020202020202020202020202020202020","share_difficulty":1,"share_id":"share-288","share_seq":288,"template_height":800009},{"accepted_at_ms":70511,"credit_policy":null,"job_id":"job-289","job_issued_at_ms":70500,"miner_id":"miner-3","network_difficulty":1000000,"ntime":1700000289,"order_key":"03:miner-3","p2mr_program_hex":"2121212121212121212121212121212121212121212121212121212121212121","share_difficulty":2,"share_id":"share-289","share_seq":289,"template_height":800009},{"accepted_at_ms":70499,"credit_policy":null,"job_id":"job-290","job_issued_at_ms":70489,"miner_id":"miner-4","network_difficulty":1000001,"ntime":1700000290,"order_key":"04:miner-4","p2mr_program_hex":"2222222222222222222222222222222222222222222222222222222222222222","share_difficulty":19,"share_id":"share-290","share_seq":290,"template_height":800009},{"accepted_at_ms":70492,"credit_policy":"stale-grace","job_id":"job-291","job_issued_at_ms":70509,"miner_id":"miner-5","network_difficulty":1000002,"ntime":1700000291,"order_key":"05:miner-5","p2mr_program_hex":"2323232323232323232323232323232323232323232323232323232323232323","share_difficulty":7,"share_id":"share-291","share_seq":291,"template_height":800009},{"accepted_at_ms":70501,"credit_policy":null,"job_id":"job-292","job_issued_at_ms":70489,"miner_id":"miner-6","network_difficulty":1000003,"ntime":1700000292,"order_key":"06:miner-6","p2mr_program_hex":"2424242424242424242424242424242424242424242424242424242424242424","share_difficulty":19,"share_id":"share-292","share_seq":292,"template_height":800009},{"accepted_at_ms":70495,"credit_policy":null,"job_id":"job-293","job_issued_at_ms":70492,"miner_id":"miner-7","network_difficulty":1000004,"ntime":1700000293,"order_key":"07:miner-7","p2mr_program_hex":"2525252525252525252525252525252525252525252525252525252525252525","share_difficulty":7,"share_id":"share-293","share_seq":293,"template_height":800009},{"accepted_at_ms":70500,"credit_policy":"stale-grace","job_id":"job-294","job_issued_at_ms":70493,"miner_id":"miner-8","network_difficulty":1000005,"ntime":1700000294,"order_key":"08:miner-8","p2mr_program_hex":"2626262626262626262626262626262626262626262626262626262626262626","share_difficulty":2,"share_id":"share-294","share_seq":294,"template_height":800009},{"accepted_at_ms":70492,"credit_policy":null,"job_id":"job-295","job_issued_at_ms":70490,"miner_id":"miner-9","network_difficulty":1000006,"ntime":1700000295,"order_key":"09:miner-9","p2mr_program_hex":"2727272727272727272727272727272727272727272727272727272727272727","share_difficulty":7,"share_id":"share-295","share_seq":295,"template_height":800009},{"accepted_at_ms":70512,"credit_policy":null,"job_id":"job-296","job_issued_at_ms":70505,"miner_id":"miner-10","network_difficulty":1000007,"ntime":1700000296,"order_key":"10:miner-10","p2mr_program_hex":"2828282828282828282828282828282828282828282828282828282828282828","share_difficulty":2,"share_id":"share-296","share_seq":296,"template_height":800009},{"accepted_at_ms":70500,"credit_policy":null,"job_id":"job-297","job_issued_at_ms":70505,"miner_id":"miner-0","network_difficulty":1000008,"ntime":1700000297,"order_key":"00:miner-0","p2mr_program_hex":"2929292929292929292929292929292929292929292929292929292929292929","share_difficulty":1,"share_id":"share-297","share_seq":297,"template_height":800009},{"accepted_at_ms":70506,"credit_policy":null,"job_id":"job-298","job_issued_at_ms":70488,"miner_id":"miner-1","network_difficulty":1000009,"ntime":1700000298,"order_key":"01:miner-1","p2mr_program_hex":"2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a","share_difficulty":3,"share_id":"share-298","share_seq":298,"template_height":800009},{"accepted_at_ms":70493,"credit_policy":null,"job_id":"job-299","job_issued_at_ms":70505,"miner_id":"miner-2","network_difficulty":1000010,"ntime":1700000299,"order_key":"02:miner-2","p2mr_program_hex":"2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b","share_difficulty":2,"share_id":"share-299","share_seq":299,"template_height":800009},{"accepted_at_ms":70508,"credit_policy":null,"job_id":"job-300","job_issued_at_ms":70511,"miner_id":"miner-3","network_difficulty":1000011,"ntime":1700000300,"order_key":"03:miner-3","p2mr_program_hex":"2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c","share_difficulty":1,"share_id":"share-300","share_seq":300,"template_height":800009},{"accepted_at_ms":70505,"credit_policy":null,"job_id":"job-301","job_issued_at_ms":70497,"miner_id":"miner-4","network_difficulty":1000012,"ntime":1700000301,"order_key":"04:miner-4","p2mr_program_hex":"2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d","share_difficulty":2,"share_id":"share-301","share_seq":301,"template_height":800009},{"accepted_at_ms":70506,"credit_policy":null,"job_id":"job-302","job_issued_at_ms":70501,"miner_id":"miner-5","network_difficulty":1000013,"ntime":1700000302,"order_key":"05:miner-5","p2mr_program_hex":"2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e","share_difficulty":3,"share_id":"share-302","share_seq":302,"template_height":800009},{"accepted_at_ms":70511,"credit_policy":null,"job_id":"job-303","job_issued_at_ms":70509,"miner_id":"miner-6","network_difficulty":1000014,"ntime":1700000303,"order_key":"06:miner-6","p2mr_program_hex":"2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f","share_difficulty":1,"share_id":"share-303","share_seq":303,"template_height":800009},{"accepted_at_ms":70488,"credit_policy":null,"job_id":"job-304","job_issued_at_ms":70489,"miner_id":"miner-7","network_difficulty":1000015,"ntime":1700000304,"order_key":"07:miner-7","p2mr_program_hex":"3030303030303030303030303030303030303030303030303030303030303030","share_difficulty":3,"share_id":"share-304","share_seq":304,"template_height":800009},{"accepted_at_ms":70498,"credit_policy":null,"job_id":"job-305","job_issued_at_ms":70500,"miner_id":"miner-8","network_difficulty":1000016,"ntime":1700000305,"order_key":"08:miner-8","p2mr_program_hex":"3131313131313131313131313131313131313131313131313131313131313131","share_difficulty":1,"share_id":"share-305","share_seq":305,"template_height":800009},{"accepted_at_ms":70499,"credit_policy":null,"job_id":"job-306","job_issued_at_ms":70501,"miner_id":"miner-9","network_difficulty":1000000,"ntime":1700000306,"order_key":"09:miner-9","p2mr_program_hex":"3232323232323232323232323232323232323232323232323232323232323232","share_difficulty":7,"share_id":"share-306","share_seq":306,"template_height":800009},{"accepted_at_ms":70502,"credit_policy":null,"job_id":"job-307","job_issued_at_ms":70498,"miner_id":"miner-10","network_difficulty":1000001,"ntime":1700000307,"order_key":"10:miner-10","p2mr_program_hex":"3333333333333333333333333333333333333333333333333333333333333333","share_difficulty":3,"share_id":"share-307","share_seq":307,"template_height":800009},{"accepted_at_ms":70492,"credit_policy":"stale-grace","job_id":"job-308","job_issued_at_ms":70509,"miner_id":"miner-0","network_difficulty":1000002,"ntime":1700000308,"order_key":"00:miner-0","p2mr_program_hex":"3434343434343434343434343434343434343434343434343434343434343434","share_difficulty":2,"share_id":"share-308","share_seq":308,"template_height":800009},{"accepted_at_ms":70502,"credit_policy":"stale-grace","job_id":"job-309","job_issued_at_ms":70486,"miner_id":"miner-1","network_difficulty":1000003,"ntime":1700000309,"order_key":"01:miner-1","p2mr_program_hex":"3535353535353535353535353535353535353535353535353535353535353535","share_difficulty":19,"share_id":"share-309","share_seq":309,"template_height":800009},{"accepted_at_ms":70495,"credit_policy":null,"job_id":"job-310","job_issued_at_ms":70493,"miner_id":"miner-2","network_difficulty":1000004,"ntime":1700000310,"order_key":"02:miner-2","p2mr_program_hex":"3636363636363636363636363636363636363636363636363636363636363636","share_difficulty":7,"share_id":"share-310","share_seq":310,"template_height":800009},{"accepted_at_ms":70501,"credit_policy":null,"job_id":"job-311","job_issued_at_ms":70486,"miner_id":"miner-3","network_difficulty":1000005,"ntime":1700000311,"order_key":"03:miner-3","p2mr_program_hex":"3737373737373737373737373737373737373737373737373737373737373737","share_difficulty":3,"share_id":"share-311","share_seq":311,"template_height":800009},{"accepted_at_ms":70507,"credit_policy":"stale-grace","job_id":"job-312","job_issued_at_ms":70489,"miner_id":"miner-4","network_difficulty":1000006,"ntime":1700000312,"order_key":"04:miner-4","p2mr_program_hex":"3838383838383838383838383838383838383838383838383838383838383838","share_difficulty":1,"share_id":"share-312","share_seq":312,"template_height":800009},{"accepted_at_ms":70501,"credit_policy":null,"job_id":"job-313","job_issued_at_ms":70493,"miner_id":"miner-5","network_difficulty":1000007,"ntime":1700000313,"order_key":"05:miner-5","p2mr_program_hex":"3939393939393939393939393939393939393939393939393939393939393939","share_difficulty":1,"share_id":"share-313","share_seq":313,"template_height":800009},{"accepted_at_ms":70491,"credit_policy":null,"job_id":"job-314","job_issued_at_ms":70487,"miner_id":"miner-6","network_difficulty":1000008,"ntime":1700000314,"order_key":"06:miner-6","p2mr_program_hex":"3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a","share_difficulty":2,"share_id":"share-314","share_seq":314,"template_height":800009},{"accepted_at_ms":70489,"credit_policy":null,"job_id":"job-315","job_issued_at_ms":70495,"miner_id":"miner-7","network_difficulty":1000009,"ntime":1700000315,"order_key":"07:miner-7","p2mr_program_hex":"3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b","share_difficulty":19,"share_id":"share-315","share_seq":315,"template_height":800009},{"accepted_at_ms":70508,"credit_policy":null,"job_id":"job-316","job_issued_at_ms":70495,"miner_id":"miner-8","network_difficulty":1000010,"ntime":1700000316,"order_key":"08:miner-8","p2mr_program_hex":"3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c","share_difficulty":19,"share_id":"share-316","share_seq":316,"template_height":800009},{"accepted_at_ms":70511,"credit_policy":"stale-grace","job_id":"job-317","job_issued_at_ms":70505,"miner_id":"miner-9","network_difficulty":1000011,"ntime":1700000317,"order_key":"09:miner-9","p2mr_program_hex":"3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d","share_difficulty":7,"share_id":"share-317","share_seq":317,"template_height":800009},{"accepted_at_ms":70489,"credit_policy":null,"job_id":"job-318","job_issued_at_ms":70510,"miner_id":"miner-10","network_difficulty":1000012,"ntime":1700000318,"order_key":"10:miner-10","p2mr_program_hex":"3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e","share_difficulty":19,"share_id":"share-318","share_seq":318,"template_height":800009},{"accepted_at_ms":70505,"credit_policy":null,"job_id":"job-319","job_issued_at_ms":70497,"miner_id":"miner-0","network_difficulty":1000013,"ntime":1700000319,"order_key":"00:miner-0","p2mr_program_hex":"3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f","share_difficulty":19,"share_id":"share-319","share_seq":319,"template_height":800009},{"accepted_at_ms":70495,"credit_policy":"stale-grace","job_id":"job-320","job_issued_at_ms":70508,"miner_id":"miner-1","network_difficulty":1000014,"ntime":1700000320,"order_key":"01:miner-1","p2mr_program_hex":"4040404040404040404040404040404040404040404040404040404040404040","share_difficulty":7,"share_id":"share-320","share_seq":320,"template_height":800010},{"accepted_at_ms":70504,"credit_policy":null,"job_id":"job-321","job_issued_at_ms":70484,"miner_id":"miner-2","network_difficulty":1000015,"ntime":1700000321,"order_key":"02:miner-2","p2mr_program_hex":"4141414141414141414141414141414141414141414141414141414141414141","share_difficulty":7,"share_id":"share-321","share_seq":321,"template_height":800010},{"accepted_at_ms":70506,"credit_policy":null,"job_id":"job-322","job_issued_at_ms":70500,"miner_id":"miner-3","network_difficulty":1000016,"ntime":1700000322,"order_key":"03:miner-3","p2mr_program_hex":"4242424242424242424242424242424242424242424242424242424242424242","share_difficulty":7,"share_id":"share-322","share_seq":322,"template_height":800010},{"accepted_at_ms":70510,"credit_policy":"stale-grace","job_id":"job-323","job_issued_at_ms":70499,"miner_id":"miner-4","network_difficulty":1000000,"ntime":1700000323,"order_key":"04:miner-4","p2mr_program_hex":"4343434343434343434343434343434343434343434343434343434343434343","share_difficulty":2,"share_id":"share-323","share_seq":323,"template_height":800010},{"accepted_at_ms":70506,"credit_policy":"stale-grace","job_id":"job-324","job_issued_at_ms":70511,"miner_id":"miner-5","network_difficulty":1000001,"ntime":1700000324,"order_key":"05:miner-5","p2mr_program_hex":"4444444444444444444444444444444444444444444444444444444444444444","share_difficulty":19,"share_id":"share-324","share_seq":324,"template_height":800010},{"accepted_at_ms":70512,"credit_policy":null,"job_id":"job-325","job_issued_at_ms":70496,"miner_id":"miner-6","network_difficulty":1000002,"ntime":1700000325,"order_key":"06:miner-6","p2mr_program_hex":"4545454545454545454545454545454545454545454545454545454545454545","share_difficulty":7,"share_id":"share-325","share_seq":325,"template_height":800010},{"accepted_at_ms":70502,"credit_policy":"stale-grace","job_id":"job-326","job_issued_at_ms":70495,"miner_id":"miner-7","network_difficulty":1000003,"ntime":1700000326,"order_key":"07:miner-7","p2mr_program_hex":"4646464646464646464646464646464646464646464646464646464646464646","share_difficulty":19,"share_id":"share-326","share_seq":326,"template_height":800010},{"accepted_at_ms":70497,"credit_policy":"stale-grace","job_id":"job-327","job_issued_at_ms":70503,"miner_id":"miner-8","network_difficulty":1000004,"ntime":1700000327,"order_key":"08:miner-8","p2mr_program_hex":"4747474747474747474747474747474747474747474747474747474747474747","share_difficulty":19,"share_id":"share-327","share_seq":327,"template_height":800010},{"accepted_at_ms":70512,"credit_policy":null,"job_id":"job-328","job_issued_at_ms":70498,"miner_id":"miner-9","network_difficulty":1000005,"ntime":1700000328,"order_key":"09:miner-9","p2mr_program_hex":"4848484848484848484848484848484848484848484848484848484848484848","share_difficulty":1,"share_id":"share-328","share_seq":328,"template_height":800010},{"accepted_at_ms":70484,"credit_policy":"stale-grace","job_id":"job-329","job_issued_at_ms":70512,"miner_id":"miner-10","network_difficulty":1000006,"ntime":1700000329,"order_key":"10:miner-10","p2mr_program_hex":"4949494949494949494949494949494949494949494949494949494949494949","share_difficulty":7,"share_id":"share-329","share_seq":329,"template_height":800010},{"accepted_at_ms":70508,"credit_policy":"stale-grace","job_id":"job-330","job_issued_at_ms":70501,"miner_id":"miner-0","network_difficulty":1000007,"ntime":1700000330,"order_key":"00:miner-0","p2mr_program_hex":"4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a","share_difficulty":19,"share_id":"share-330","share_seq":330,"template_height":800010},{"accepted_at_ms":70484,"credit_policy":null,"job_id":"job-331","job_issued_at_ms":70511,"miner_id":"miner-1","network_difficulty":1000008,"ntime":1700000331,"order_key":"01:miner-1","p2mr_program_hex":"4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b","share_difficulty":3,"share_id":"share-331","share_seq":331,"template_height":800010},{"accepted_at_ms":70491,"credit_policy":null,"job_id":"job-332","job_issued_at_ms":70491,"miner_id":"miner-2","network_difficulty":1000009,"ntime":1700000332,"order_key":"02:miner-2","p2mr_program_hex":"4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c","share_difficulty":3,"share_id":"share-332","share_seq":332,"template_height":800010},{"accepted_at_ms":70496,"credit_policy":null,"job_id":"job-333","job_issued_at_ms":70513,"miner_id":"miner-3","network_difficulty":1000010,"ntime":1700000333,"order_key":"03:miner-3","p2mr_program_hex":"4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d","share_difficulty":3,"share_id":"share-333","share_seq":333,"template_height":800010},{"accepted_at_ms":70511,"credit_policy":null,"job_id":"job-334","job_issued_at_ms":70499,"miner_id":"miner-4","network_difficulty":1000011,"ntime":1700000334,"order_key":"04:miner-4","p2mr_program_hex":"4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e","share_difficulty":2,"share_id":"share-334","share_seq":334,"template_height":800010},{"accepted_at_ms":70493,"credit_policy":null,"job_id":"job-335","job_issued_at_ms":70503,"miner_id":"miner-5","network_difficulty":1000012,"ntime":1700000335,"order_key":"05:miner-5","p2mr_program_hex":"4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f","share_difficulty":3,"share_id":"share-335","share_seq":335,"template_height":800010},{"accepted_at_ms":70497,"credit_policy":null,"job_id":"job-336","job_issued_at_ms":70492,"miner_id":"miner-6","network_difficulty":1000013,"ntime":1700000336,"order_key":"06:miner-6","p2mr_program_hex":"5050505050505050505050505050505050505050505050505050505050505050","share_difficulty":19,"share_id":"share-336","share_seq":336,"template_height":800010},{"accepted_at_ms":70506,"credit_policy":null,"job_id":"job-337","job_issued_at_ms":70486,"miner_id":"miner-7","network_difficulty":1000014,"ntime":1700000337,"order_key":"07:miner-7","p2mr_program_hex":"5151515151515151515151515151515151515151515151515151515151515151","share_difficulty":19,"share_id":"share-337","share_seq":337,"template_height":800010},{"accepted_at_ms":70496,"credit_policy":null,"job_id":"job-338","job_issued_at_ms":70513,"miner_id":"miner-8","network_difficulty":1000015,"ntime":1700000338,"order_key":"08:miner-8","p2mr_program_hex":"5252525252525252525252525252525252525252525252525252525252525252","share_difficulty":19,"share_id":"share-338","share_seq":338,"template_height":800010},{"accepted_at_ms":70483,"credit_policy":"stale-grace","job_id":"job-339","job_issued_at_ms":70484,"miner_id":"miner-9","network_difficulty":1000016,"ntime":1700000339,"order_key":"09:miner-9","p2mr_program_hex":"5353535353535353535353535353535353535353535353535353535353535353","share_difficulty":3,"share_id":"share-339","share_seq":339,"template_height":800010},{"accepted_at_ms":70499,"credit_policy":null,"job_id":"job-340","job_issued_at_ms":70510,"miner_id":"miner-10","network_difficulty":1000000,"ntime":1700000340,"order_key":"10:miner-10","p2mr_program_hex":"5454545454545454545454545454545454545454545454545454545454545454","share_difficulty":1,"share_id":"share-340","share_seq":340,"template_height":800010},{"accepted_at_ms":70491,"credit_policy":null,"job_id":"job-341","job_issued_at_ms":70511,"miner_id":"miner-0","network_difficulty":1000001,"ntime":1700000341,"order_key":"00:miner-0","p2mr_program_hex":"5555555555555555555555555555555555555555555555555555555555555555","share_difficulty":2,"share_id":"share-341","share_seq":341,"template_height":800010},{"accepted_at_ms":70494,"credit_policy":null,"job_id":"job-342","job_issued_at_ms":70488,"miner_id":"miner-1","network_difficulty":1000002,"ntime":1700000342,"order_key":"01:miner-1","p2mr_program_hex":"5656565656565656565656565656565656565656565656565656565656565656","share_difficulty":2,"share_id":"share-342","share_seq":342,"template_height":800010},{"accepted_at_ms":70492,"credit_policy":"stale-grace","job_id":"job-343","job_issued_at_ms":70490,"miner_id":"miner-2","network_difficulty":1000003,"ntime":1700000343,"order_key":"02:miner-2","p2mr_program_hex":"5757575757575757575757575757575757575757575757575757575757575757","share_difficulty":3,"share_id":"share-343","share_seq":343,"template_height":800010},{"accepted_at_ms":70486,"credit_policy":"stale-grace","job_id":"job-344","job_issued_at_ms":70508,"miner_id":"miner-3","network_difficulty":1000004,"ntime":1700000344,"order_key":"03:miner-3","p2mr_program_hex":"5858585858585858585858585858585858585858585858585858585858585858","share_difficulty":1,"share_id":"share-344","share_seq":344,"template_height":800010},{"accepted_at_ms":70501,"credit_policy":null,"job_id":"job-345","job_issued_at_ms":70498,"miner_id":"miner-4","network_difficulty":1000005,"ntime":1700000345,"order_key":"04:miner-4","p2mr_program_hex":"5959595959595959595959595959595959595959595959595959595959595959","share_difficulty":7,"share_id":"share-345","share_seq":345,"template_height":800010},{"accepted_at_ms":70507,"credit_policy":"stale-grace","job_id":"job-346","job_issued_at_ms":70488,"miner_id":"miner-5","network_difficulty":1000006,"ntime":1700000346,"order_key":"05:miner-5","p2mr_program_hex":"5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a","share_difficulty":7,"share_id":"share-346","share_seq":346,"template_height":800010},{"accepted_at_ms":70512,"credit_policy":null,"job_id":"job-347","job_issued_at_ms":70510,"miner_id":"miner-6","network_difficulty":1000007,"ntime":1700000347,"order_key":"06:miner-6","p2mr_program_hex":"5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b","share_difficulty":7,"share_id":"share-347","share_seq":347,"template_height":800010},{"accepted_at_ms":70501,"credit_policy":null,"job_id":"job-348","job_issued_at_ms":70496,"miner_id":"miner-7","network_difficulty":1000008,"ntime":1700000348,"order_key":"07:miner-7","p2mr_program_hex":"5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c","share_difficulty":3,"share_id":"share-348","share_seq":348,"template_height":800010},{"accepted_at_ms":70501,"credit_policy":null,"job_id":"job-349","job_issued_at_ms":70488,"miner_id":"miner-8","network_difficulty":1000009,"ntime":1700000349,"order_key":"08:miner-8","p2mr_program_hex":"5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d","share_difficulty":3,"share_id":"share-349","share_seq":349,"template_height":800010},{"accepted_at_ms":70511,"credit_policy":null,"job_id":"job-350","job_issued_at_ms":70490,"miner_id":"miner-9","network_difficulty":1000010,"ntime":1700000350,"order_key":"09:miner-9","p2mr_program_hex":"5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e","share_difficulty":3,"share_id":"share-350","share_seq":350,"template_height":800010},{"accepted_at_ms":70502,"credit_policy":null,"job_id":"job-351","job_issued_at_ms":70492,"miner_id":"miner-10","network_difficulty":1000011,"ntime":1700000351,"order_key":"10:miner-10","p2mr_program_hex":"5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f","share_difficulty":19,"share_id":"share-351","share_seq":351,"template_height":800010},{"accepted_at_ms":70501,"credit_policy":null,"job_id":"job-352","job_issued_at_ms":70492,"miner_id":"miner-0","network_difficulty":1000012,"ntime":1700000352,"order_key":"00:miner-0","p2mr_program_hex":"6060606060606060606060606060606060606060606060606060606060606060","share_difficulty":19,"share_id":"share-352","share_seq":352,"template_height":800011},{"accepted_at_ms":70485,"credit_policy":null,"job_id":"job-353","job_issued_at_ms":70496,"miner_id":"miner-1","network_difficulty":1000013,"ntime":1700000353,"order_key":"01:miner-1","p2mr_program_hex":"6161616161616161616161616161616161616161616161616161616161616161","share_difficulty":2,"share_id":"share-353","share_seq":353,"template_height":800011},{"accepted_at_ms":70486,"credit_policy":null,"job_id":"job-354","job_issued_at_ms":70495,"miner_id":"miner-2","network_difficulty":1000014,"ntime":1700000354,"order_key":"02:miner-2","p2mr_program_hex":"6262626262626262626262626262626262626262626262626262626262626262","share_difficulty":19,"share_id":"share-354","share_seq":354,"template_height":800011},{"accepted_at_ms":70512,"credit_policy":"stale-grace","job_id":"job-355","job_issued_at_ms":70487,"miner_id":"miner-3","network_difficulty":1000015,"ntime":1700000355,"order_key":"03:miner-3","p2mr_program_hex":"6363636363636363636363636363636363636363636363636363636363636363","share_difficulty":3,"share_id":"share-355","share_seq":355,"template_height":800011},{"accepted_at_ms":70509,"credit_policy":null,"job_id":"job-356","job_issued_at_ms":70492,"miner_id":"miner-4","network_difficulty":1000016,"ntime":1700000356,"order_key":"04:miner-4","p2mr_program_hex":"6464646464646464646464646464646464646464646464646464646464646464","share_difficulty":3,"share_id":"share-356","share_seq":356,"template_height":800011},{"accepted_at_ms":70510,"credit_policy":null,"job_id":"job-357","job_issued_at_ms":70502,"miner_id":"miner-5","network_difficulty":1000000,"ntime":1700000357,"order_key":"05:miner-5","p2mr_program_hex":"6565656565656565656565656565656565656565656565656565656565656565","share_difficulty":19,"share_id":"share-357","share_seq":357,"template_height":800011},{"accepted_at_ms":70487,"credit_policy":null,"job_id":"job-358","job_issued_at_ms":70493,"miner_id":"miner-6","network_difficulty":1000001,"ntime":1700000358,"order_key":"06:miner-6","p2mr_program_hex":"6666666666666666666666666666666666666666666666666666666666666666","share_difficulty":2,"share_id":"share-358","share_seq":358,"template_height":800011},{"accepted_at_ms":70504,"credit_policy":null,"job_id":"job-359","job_issued_at_ms":70501,"miner_id":"miner-7","network_difficulty":1000002,"ntime":1700000359,"order_key":"07:miner-7","p2mr_program_hex":"6767676767676767676767676767676767676767676767676767676767676767","share_difficulty":7,"share_id":"share-359","share_seq":359,"template_height":800011},{"accepted_at_ms":70489,"credit_policy":"stale-grace","job_id":"job-360","job_issued_at_ms":70495,"miner_id":"miner-8","network_difficulty":1000003,"ntime":1700000360,"order_key":"08:miner-8","p2mr_program_hex":"6868686868686868686868686868686868686868686868686868686868686868","share_difficulty":2,"share_id":"share-360","share_seq":360,"template_height":800011},{"accepted_at_ms":70490,"credit_policy":null,"job_id":"job-361","job_issued_at_ms":70499,"miner_id":"miner-9","network_difficulty":1000004,"ntime":1700000361,"order_key":"09:miner-9","p2mr_program_hex":"6969696969696969696969696969696969696969696969696969696969696969","share_difficulty":7,"share_id":"share-361","share_seq":361,"template_height":800011},{"accepted_at_ms":70500,"credit_policy":null,"job_id":"job-362","job_issued_at_ms":70492,"miner_id":"miner-10","network_difficulty":1000005,"ntime":1700000362,"order_key":"10:miner-10","p2mr_program_hex":"6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a","share_difficulty":19,"share_id":"share-362","share_seq":362,"template_height":800011},{"accepted_at_ms":70505,"credit_policy":null,"job_id":"job-363","job_issued_at_ms":70490,"miner_id":"miner-0","network_difficulty":1000006,"ntime":1700000363,"order_key":"00:miner-0","p2mr_program_hex":"6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b","share_difficulty":1,"share_id":"share-363","share_seq":363,"template_height":800011},{"accepted_at_ms":70485,"credit_policy":null,"job_id":"job-364","job_issued_at_ms":70486,"miner_id":"miner-1","network_difficulty":1000007,"ntime":1700000364,"order_key":"01:miner-1","p2mr_program_hex":"6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c","share_difficulty":1,"share_id":"share-364","share_seq":364,"template_height":800011},{"accepted_at_ms":70485,"credit_policy":null,"job_id":"job-365","job_issued_at_ms":70495,"miner_id":"miner-2","network_difficulty":1000008,"ntime":1700000365,"order_key":"02:miner-2","p2mr_program_hex":"6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d","share_difficulty":7,"share_id":"share-365","share_seq":365,"template_height":800011},{"accepted_at_ms":70492,"credit_policy":"stale-grace","job_id":"job-366","job_issued_at_ms":70507,"miner_id":"miner-3","network_difficulty":1000009,"ntime":1700000366,"order_key":"03:miner-3","p2mr_program_hex":"6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e","share_difficulty":7,"share_id":"share-366","share_seq":366,"template_height":800011},{"accepted_at_ms":70496,"credit_policy":null,"job_id":"job-367","job_issued_at_ms":70507,"miner_id":"miner-4","network_difficulty":1000010,"ntime":1700000367,"order_key":"04:miner-4","p2mr_program_hex":"6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f","share_difficulty":2,"share_id":"share-367","share_seq":367,"template_height":800011},{"accepted_at_ms":70506,"credit_policy":null,"job_id":"job-368","job_issued_at_ms":70484,"miner_id":"miner-5","network_difficulty":1000011,"ntime":1700000368,"order_key":"05:miner-5","p2mr_program_hex":"7070707070707070707070707070707070707070707070707070707070707070","share_difficulty":3,"share_id":"share-368","share_seq":368,"template_height":800011},{"accepted_at_ms":70501,"credit_policy":"stale-grace","job_id":"job-369","job_issued_at_ms":70492,"miner_id":"miner-6","network_difficulty":1000012,"ntime":1700000369,"order_key":"06:miner-6","p2mr_program_hex":"7171717171717171717171717171717171717171717171717171717171717171","share_difficulty":3,"share_id":"share-369","share_seq":369,"template_height":800011},{"accepted_at_ms":70498,"credit_policy":null,"job_id":"job-370","job_issued_at_ms":70483,"miner_id":"miner-7","network_difficulty":1000013,"ntime":1700000370,"order_key":"07:miner-7","p2mr_program_hex":"7272727272727272727272727272727272727272727272727272727272727272","share_difficulty":3,"share_id":"share-370","share_seq":370,"template_height":800011},{"accepted_at_ms":70510,"credit_policy":null,"job_id":"job-371","job_issued_at_ms":70487,"miner_id":"miner-8","network_difficulty":1000014,"ntime":1700000371,"order_key":"08:miner-8","p2mr_program_hex":"7373737373737373737373737373737373737373737373737373737373737373","share_difficulty":1,"share_id":"share-371","share_seq":371,"template_height":800011},{"accepted_at_ms":70496,"credit_policy":"stale-grace","job_id":"job-372","job_issued_at_ms":70483,"miner_id":"miner-9","network_difficulty":1000015,"ntime":1700000372,"order_key":"09:miner-9","p2mr_program_hex":"7474747474747474747474747474747474747474747474747474747474747474","share_difficulty":2,"share_id":"share-372","share_seq":372,"template_height":800011},{"accepted_at_ms":70485,"credit_policy":"stale-grace","job_id":"job-373","job_issued_at_ms":70503,"miner_id":"miner-10","network_difficulty":1000016,"ntime":1700000373,"order_key":"10:miner-10","p2mr_program_hex":"7575757575757575757575757575757575757575757575757575757575757575","share_difficulty":1,"share_id":"share-373","share_seq":373,"template_height":800011},{"accepted_at_ms":70489,"credit_policy":null,"job_id":"job-374","job_issued_at_ms":70499,"miner_id":"miner-0","network_difficulty":1000000,"ntime":1700000374,"order_key":"00:miner-0","p2mr_program_hex":"7676767676767676767676767676767676767676767676767676767676767676","share_difficulty":7,"share_id":"share-374","share_seq":374,"template_height":800011},{"accepted_at_ms":70483,"credit_policy":null,"job_id":"job-375","job_issued_at_ms":70486,"miner_id":"miner-1","network_difficulty":1000001,"ntime":1700000375,"order_key":"01:miner-1","p2mr_program_hex":"7777777777777777777777777777777777777777777777777777777777777777","share_difficulty":19,"share_id":"share-375","share_seq":375,"template_height":800011},{"accepted_at_ms":70496,"credit_policy":null,"job_id":"job-376","job_issued_at_ms":70505,"miner_id":"miner-2","network_difficulty":1000002,"ntime":1700000376,"order_key":"02:miner-2","p2mr_program_hex":"7878787878787878787878787878787878787878787878787878787878787878","share_difficulty":2,"share_id":"share-376","share_seq":376,"template_height":800011},{"accepted_at_ms":70512,"credit_policy":null,"job_id":"job-377","job_issued_at_ms":70493,"miner_id":"miner-3","network_difficulty":1000003,"ntime":1700000377,"order_key":"03:miner-3","p2mr_program_hex":"7979797979797979797979797979797979797979797979797979797979797979","share_difficulty":3,"share_id":"share-377","share_seq":377,"template_height":800011},{"accepted_at_ms":70501,"credit_policy":null,"job_id":"job-378","job_issued_at_ms":70492,"miner_id":"miner-4","network_difficulty":1000004,"ntime":1700000378,"order_key":"04:miner-4","p2mr_program_hex":"7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a","share_difficulty":3,"share_id":"share-378","share_seq":378,"template_height":800011},{"accepted_at_ms":70497,"credit_policy":"stale-grace","job_id":"job-379","job_issued_at_ms":70498,"miner_id":"miner-5","network_difficulty":1000005,"ntime":1700000379,"order_key":"05:miner-5","p2mr_program_hex":"7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b","share_difficulty":7,"share_id":"share-379","share_seq":379,"template_height":800011},{"accepted_at_ms":70487,"credit_policy":"stale-grace","job_id":"job-380","job_issued_at_ms":70489,"miner_id":"miner-6","network_difficulty":1000006,"ntime":1700000380,"order_key":"06:miner-6","p2mr_program_hex":"7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c","share_difficulty":1,"share_id":"share-380","share_seq":380,"template_height":800011},{"accepted_at_ms":70507,"credit_policy":"stale-grace","job_id":"job-381","job_issued_at_ms":70494,"miner_id":"miner-7","network_difficulty":1000007,"ntime":1700000381,"order_key":"07:miner-7","p2mr_program_hex":"7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d","share_difficulty":7,"share_id":"share-381","share_seq":381,"template_height":800011},{"accepted_at_ms":70495,"credit_policy":null,"job_id":"job-382","job_issued_at_ms":70508,"miner_id":"miner-8","network_difficulty":1000008,"ntime":1700000382,"order_key":"08:miner-8","p2mr_program_hex":"7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e","share_difficulty":19,"share_id":"share-382","share_seq":382,"template_height":800011},{"accepted_at_ms":70494,"credit_policy":"stale-grace","job_id":"job-383","job_issued_at_ms":70487,"miner_id":"miner-9","network_difficulty":1000009,"ntime":1700000383,"order_key":"09:miner-9","p2mr_program_hex":"7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f","share_difficulty":19,"share_id":"share-383","share_seq":383,"template_height":800011},{"accepted_at_ms":70488,"credit_policy":null,"job_id":"job-384","job_issued_at_ms":70505,"miner_id":"miner-10","network_difficulty":1000010,"ntime":1700000384,"order_key":"10:miner-10","p2mr_program_hex":"8080808080808080808080808080808080808080808080808080808080808080","share_difficulty":3,"share_id":"share-384","share_seq":384,"template_height":800012},{"accepted_at_ms":70502,"credit_policy":"stale-grace","job_id":"job-385","job_issued_at_ms":70490,"miner_id":"miner-0","network_difficulty":1000011,"ntime":1700000385,"order_key":"00:miner-0","p2mr_program_hex":"8181818181818181818181818181818181818181818181818181818181818181","share_difficulty":1,"share_id":"share-385","share_seq":385,"template_height":800012},{"accepted_at_ms":70501,"credit_policy":"stale-grace","job_id":"job-386","job_issued_at_ms":70505,"miner_id":"miner-1","network_difficulty":1000012,"ntime":1700000386,"order_key":"01:miner-1","p2mr_program_hex":"8282828282828282828282828282828282828282828282828282828282828282","share_difficulty":3,"share_id":"share-386","share_seq":386,"template_height":800012},{"accepted_at_ms":70486,"credit_policy":"stale-grace","job_id":"job-387","job_issued_at_ms":70507,"miner_id":"miner-2","network_difficulty":1000013,"ntime":1700000387,"order_key":"02:miner-2","p2mr_program_hex":"8383838383838383838383838383838383838383838383838383838383838383","share_difficulty":2,"share_id":"share-387","share_seq":387,"template_height":800012},{"accepted_at_ms":70505,"credit_policy":null,"job_id":"job-388","job_issued_at_ms":70513,"miner_id":"miner-3","network_difficulty":1000014,"ntime":1700000388,"order_key":"03:miner-3","p2mr_program_hex":"8484848484848484848484848484848484848484848484848484848484848484","share_difficulty":7,"share_id":"share-388","share_seq":388,"template_height":800012},{"accepted_at_ms":70485,"credit_policy":null,"job_id":"job-389","job_issued_at_ms":70487,"miner_id":"miner-4","network_difficulty":1000015,"ntime":1700000389,"order_key":"04:miner-4","p2mr_program_hex":"8585858585858585858585858585858585858585858585858585858585858585","share_difficulty":7,"share_id":"share-389","share_seq":389,"template_height":800012},{"accepted_at_ms":70490,"credit_policy":null,"job_id":"job-390","job_issued_at_ms":70490,"miner_id":"miner-5","network_difficulty":1000016,"ntime":1700000390,"order_key":"05:miner-5","p2mr_program_hex":"8686868686868686868686868686868686868686868686868686868686868686","share_difficulty":7,"share_id":"share-390","share_seq":390,"template_height":800012},{"accepted_at_ms":70505,"credit_policy":null,"job_id":"job-391","job_issued_at_ms":70495,"miner_id":"miner-6","network_difficulty":1000000,"ntime":1700000391,"order_key":"06:miner-6","p2mr_program_hex":"8787878787878787878787878787878787878787878787878787878787878787","share_difficulty":3,"share_id":"share-391","share_seq":391,"template_height":800012},{"accepted_at_ms":70502,"credit_policy":null,"job_id":"job-392","job_issued_at_ms":70499,"miner_id":"miner-7","network_difficulty":1000001,"ntime":1700000392,"order_key":"07:miner-7","p2mr_program_hex":"8888888888888888888888888888888888888888888888888888888888888888","share_difficulty":1,"share_id":"share-392","share_seq":392,"template_height":800012},{"accepted_at_ms":70495,"credit_policy":null,"job_id":"job-393","job_issued_at_ms":70513,"miner_id":"miner-8","network_difficulty":1000002,"ntime":1700000393,"order_key":"08:miner-8","p2mr_program_hex":"8989898989898989898989898989898989898989898989898989898989898989","share_difficulty":7,"share_id":"share-393","share_seq":393,"template_height":800012},{"accepted_at_ms":70507,"credit_policy":null,"job_id":"job-394","job_issued_at_ms":70487,"miner_id":"miner-9","network_difficulty":1000003,"ntime":1700000394,"order_key":"09:miner-9","p2mr_program_hex":"8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a","share_difficulty":19,"share_id":"share-394","share_seq":394,"template_height":800012},{"accepted_at_ms":70486,"credit_policy":null,"job_id":"job-395","job_issued_at_ms":70497,"miner_id":"miner-10","network_difficulty":1000004,"ntime":1700000395,"order_key":"10:miner-10","p2mr_program_hex":"8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b","share_difficulty":19,"share_id":"share-395","share_seq":395,"template_height":800012},{"accepted_at_ms":70498,"credit_policy":null,"job_id":"job-396","job_issued_at_ms":70497,"miner_id":"miner-0","network_difficulty":1000005,"ntime":1700000396,"order_key":"00:miner-0","p2mr_program_hex":"8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c","share_difficulty":7,"share_id":"share-396","share_seq":396,"template_height":800012},{"accepted_at_ms":70485,"credit_policy":null,"job_id":"job-397","job_issued_at_ms":70513,"miner_id":"miner-1","network_difficulty":1000006,"ntime":1700000397,"order_key":"01:miner-1","p2mr_program_hex":"8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d","share_difficulty":3,"share_id":"share-397","share_seq":397,"template_height":800012},{"accepted_at_ms":70486,"credit_policy":null,"job_id":"job-398","job_issued_at_ms":70512,"miner_id":"miner-2","network_difficulty":1000007,"ntime":1700000398,"order_key":"02:miner-2","p2mr_program_hex":"8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e","share_difficulty":2,"share_id":"share-398","share_seq":398,"template_height":800012},{"accepted_at_ms":70498,"credit_policy":"stale-grace","job_id":"job-399","job_issued_at_ms":70507,"miner_id":"miner-3","network_difficulty":1000008,"ntime":1700000399,"order_key":"03:miner-3","p2mr_program_hex":"8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f","share_difficulty":7,"share_id":"share-399","share_seq":399,"template_height":800012},{"accepted_at_ms":70486,"credit_policy":null,"job_id":"job-400","job_issued_at_ms":70509,"miner_id":"miner-4","network_difficulty":1000009,"ntime":1700000400,"order_key":"04:miner-4","p2mr_program_hex":"9090909090909090909090909090909090909090909090909090909090909090","share_difficulty":3,"share_id":"share-400","share_seq":400,"template_height":800012},{"accepted_at_ms":70493,"credit_policy":null,"job_id":"job-401","job_issued_at_ms":70512,"miner_id":"miner-5","network_difficulty":1000010,"ntime":1700000401,"order_key":"05:miner-5","p2mr_program_hex":"9191919191919191919191919191919191919191919191919191919191919191","share_difficulty":3,"share_id":"share-401","share_seq":401,"template_height":800012},{"accepted_at_ms":70488,"credit_policy":"stale-grace","job_id":"job-402","job_issued_at_ms":70490,"miner_id":"miner-6","network_difficulty":1000011,"ntime":1700000402,"order_key":"06:miner-6","p2mr_program_hex":"9292929292929292929292929292929292929292929292929292929292929292","share_difficulty":1,"share_id":"share-402","share_seq":402,"template_height":800012},{"accepted_at_ms":70486,"credit_policy":null,"job_id":"job-403","job_issued_at_ms":70511,"miner_id":"miner-7","network_difficulty":1000012,"ntime":1700000403,"order_key":"07:miner-7","p2mr_program_hex":"9393939393939393939393939393939393939393939393939393939393939393","share_difficulty":1,"share_id":"share-403","share_seq":403,"template_height":800012},{"accepted_at_ms":70493,"credit_policy":null,"job_id":"job-404","job_issued_at_ms":70491,"miner_id":"miner-8","network_difficulty":1000013,"ntime":1700000404,"order_key":"08:miner-8","p2mr_program_hex":"9494949494949494949494949494949494949494949494949494949494949494","share_difficulty":7,"share_id":"share-404","share_seq":404,"template_height":800012},{"accepted_at_ms":70509,"credit_policy":null,"job_id":"job-405","job_issued_at_ms":70507,"miner_id":"miner-9","network_difficulty":1000014,"ntime":1700000405,"order_key":"09:miner-9","p2mr_program_hex":"9595959595959595959595959595959595959595959595959595959595959595","share_difficulty":3,"share_id":"share-405","share_seq":405,"template_height":800012},{"accepted_at_ms":70485,"credit_policy":null,"job_id":"job-406","job_issued_at_ms":70511,"miner_id":"miner-10","network_difficulty":1000015,"ntime":1700000406,"order_key":"10:miner-10","p2mr_program_hex":"9696969696969696969696969696969696969696969696969696969696969696","share_difficulty":19,"share_id":"share-406","share_seq":406,"template_height":800012},{"accepted_at_ms":70487,"credit_policy":"stale-grace","job_id":"job-407","job_issued_at_ms":70493,"miner_id":"miner-0","network_difficulty":1000016,"ntime":1700000407,"order_key":"00:miner-0","p2mr_program_hex":"9797979797979797979797979797979797979797979797979797979797979797","share_difficulty":19,"share_id":"share-407","share_seq":407,"template_height":800012},{"accepted_at_ms":70512,"credit_policy":"stale-grace","job_id":"job-408","job_issued_at_ms":70504,"miner_id":"miner-1","network_difficulty":1000000,"ntime":1700000408,"order_key":"01:miner-1","p2mr_program_hex":"9898989898989898989898989898989898989898989898989898989898989898","share_difficulty":7,"share_id":"share-408","share_seq":408,"template_height":800012},{"accepted_at_ms":70492,"credit_policy":null,"job_id":"job-409","job_issued_at_ms":70503,"miner_id":"miner-2","network_difficulty":1000001,"ntime":1700000409,"order_key":"02:miner-2","p2mr_program_hex":"9999999999999999999999999999999999999999999999999999999999999999","share_difficulty":19,"share_id":"share-409","share_seq":409,"template_height":800012},{"accepted_at_ms":70486,"credit_policy":null,"job_id":"job-410","job_issued_at_ms":70487,"miner_id":"miner-3","network_difficulty":1000002,"ntime":1700000410,"order_key":"03:miner-3","p2mr_program_hex":"9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a","share_difficulty":7,"share_id":"share-410","share_seq":410,"template_height":800012},{"accepted_at_ms":70493,"credit_policy":null,"job_id":"job-411","job_issued_at_ms":70489,"miner_id":"miner-4","network_difficulty":1000003,"ntime":1700000411,"order_key":"04:miner-4","p2mr_program_hex":"9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b","share_difficulty":2,"share_id":"share-411","share_seq":411,"template_height":800012},{"accepted_at_ms":70483,"credit_policy":null,"job_id":"job-412","job_issued_at_ms":70503,"miner_id":"miner-5","network_difficulty":1000004,"ntime":1700000412,"order_key":"05:miner-5","p2mr_program_hex":"9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c","share_difficulty":1,"share_id":"share-412","share_seq":412,"template_height":800012},{"accepted_at_ms":70491,"credit_policy":"stale-grace","job_id":"job-413","job_issued_at_ms":70500,"miner_id":"miner-6","network_difficulty":1000005,"ntime":1700000413,"order_key":"06:miner-6","p2mr_program_hex":"9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d","share_difficulty":19,"share_id":"share-413","share_seq":413,"template_height":800012},{"accepted_at_ms":70492,"credit_policy":"stale-grace","job_id":"job-414","job_issued_at_ms":70510,"miner_id":"miner-7","network_difficulty":1000006,"ntime":1700000414,"order_key":"07:miner-7","p2mr_program_hex":"9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e","share_difficulty":1,"share_id":"share-414","share_seq":414,"template_height":800012},{"accepted_at_ms":70513,"credit_policy":"stale-grace","job_id":"job-415","job_issued_at_ms":70495,"miner_id":"miner-8","network_difficulty":1000007,"ntime":1700000415,"order_key":"08:miner-8","p2mr_program_hex":"9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f","share_difficulty":3,"share_id":"share-415","share_seq":415,"template_height":800012},{"accepted_at_ms":70501,"credit_policy":null,"job_id":"job-416","job_issued_at_ms":70486,"miner_id":"miner-9","network_difficulty":1000008,"ntime":1700000416,"order_key":"09:miner-9","p2mr_program_hex":"a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0","share_difficulty":7,"share_id":"share-416","share_seq":416,"template_height":800013},{"accepted_at_ms":70500,"credit_policy":null,"job_id":"job-417","job_issued_at_ms":70490,"miner_id":"miner-10","network_difficulty":1000009,"ntime":1700000417,"order_key":"10:miner-10","p2mr_program_hex":"a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1","share_difficulty":19,"share_id":"share-417","share_seq":417,"template_height":800013},{"accepted_at_ms":70511,"credit_policy":"stale-grace","job_id":"job-418","job_issued_at_ms":70484,"miner_id":"miner-0","network_difficulty":1000010,"ntime":1700000418,"order_key":"00:miner-0","p2mr_program_hex":"a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2","share_difficulty":7,"share_id":"share-418","share_seq":418,"template_height":800013},{"accepted_at_ms":70500,"credit_policy":null,"job_id":"job-419","job_issued_at_ms":70497,"miner_id":"miner-1","network_difficulty":1000011,"ntime":1700000419,"order_key":"01:miner-1","p2mr_program_hex":"a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3","share_difficulty":7,"share_id":"share-419","share_seq":419,"template_height":800013},{"accepted_at_ms":70486,"credit_policy":null,"job_id":"job-420","job_issued_at_ms":70503,"miner_id":"miner-2","network_difficulty":1000012,"ntime":1700000420,"order_key":"02:miner-2","p2mr_program_hex":"a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4","share_difficulty":2,"share_id":"share-420","share_seq":420,"template_height":800013},{"accepted_at_ms":70510,"credit_policy":null,"job_id":"job-421","job_issued_at_ms":70492,"miner_id":"miner-3","network_difficulty":1000013,"ntime":1700000421,"order_key":"03:miner-3","p2mr_program_hex":"a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5","share_difficulty":7,"share_id":"share-421","share_seq":421,"template_height":800013},{"accepted_at_ms":70502,"credit_policy":null,"job_id":"job-422","job_issued_at_ms":70485,"miner_id":"miner-4","network_difficulty":1000014,"ntime":1700000422,"order_key":"04:miner-4","p2mr_program_hex":"a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6a6","share_difficulty":7,"share_id":"share-422","share_seq":422,"template_height":800013},{"accepted_at_ms":70509,"credit_policy":null,"job_id":"job-423","job_issued_at_ms":70485,"miner_id":"miner-5","network_difficulty":1000015,"ntime":1700000423,"order_key":"05:miner-5","p2mr_program_hex":"a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7","share_difficulty":3,"share_id":"share-423","share_seq":423,"template_height":800013},{"accepted_at_ms":70507,"credit_policy":"stale-grace","job_id":"job-424","job_issued_at_ms":70506,"miner_id":"miner-6","network_difficulty":1000016,"ntime":1700000424,"order_key":"06:miner-6","p2mr_program_hex":"a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8","share_difficulty":2,"share_id":"share-424","share_seq":424,"template_height":800013},{"accepted_at_ms":70507,"credit_policy":null,"job_id":"job-425","job_issued_at_ms":70512,"miner_id":"miner-7","network_difficulty":1000000,"ntime":1700000425,"order_key":"07:miner-7","p2mr_program_hex":"a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9","share_difficulty":1,"share_id":"share-425","share_seq":425,"template_height":800013},{"accepted_at_ms":70510,"credit_policy":null,"job_id":"job-426","job_issued_at_ms":70484,"miner_id":"miner-8","network_difficulty":1000001,"ntime":1700000426,"order_key":"08:miner-8","p2mr_program_hex":"aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa","share_difficulty":19,"share_id":"share-426","share_seq":426,"template_height":800013},{"accepted_at_ms":70507,"credit_policy":null,"job_id":"job-427","job_issued_at_ms":70504,"miner_id":"miner-9","network_difficulty":1000002,"ntime":1700000427,"order_key":"09:miner-9","p2mr_program_hex":"abababababababababababababababababababababababababababababababab","share_difficulty":3,"share_id":"share-427","share_seq":427,"template_height":800013},{"accepted_at_ms":70501,"credit_policy":null,"job_id":"job-428","job_issued_at_ms":70489,"miner_id":"miner-10","network_difficulty":1000003,"ntime":1700000428,"order_key":"10:miner-10","p2mr_program_hex":"acacacacacacacacacacacacacacacacacacacacacacacacacacacacacacacac","share_difficulty":3,"share_id":"share-428","share_seq":428,"template_height":800013},{"accepted_at_ms":70512,"credit_policy":"stale-grace","job_id":"job-429","job_issued_at_ms":70488,"miner_id":"miner-0","network_difficulty":1000004,"ntime":1700000429,"order_key":"00:miner-0","p2mr_program_hex":"adadadadadadadadadadadadadadadadadadadadadadadadadadadadadadadad","share_difficulty":2,"share_id":"share-429","share_seq":429,"template_height":800013},{"accepted_at_ms":70494,"credit_policy":null,"job_id":"job-430","job_issued_at_ms":70507,"miner_id":"miner-1","network_difficulty":1000005,"ntime":1700000430,"order_key":"01:miner-1","p2mr_program_hex":"aeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeae","share_difficulty":2,"share_id":"share-430","share_seq":430,"template_height":800013},{"accepted_at_ms":70513,"credit_policy":null,"job_id":"job-431","job_issued_at_ms":70507,"miner_id":"miner-2","network_difficulty":1000006,"ntime":1700000431,"order_key":"02:miner-2","p2mr_program_hex":"afafafafafafafafafafafafafafafafafafafafafafafafafafafafafafafaf","share_difficulty":7,"share_id":"share-431","share_seq":431,"template_height":800013},{"accepted_at_ms":70488,"credit_policy":null,"job_id":"job-432","job_issued_at_ms":70491,"miner_id":"miner-3","network_difficulty":1000007,"ntime":1700000432,"order_key":"03:miner-3","p2mr_program_hex":"b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0","share_difficulty":19,"share_id":"share-432","share_seq":432,"template_height":800013},{"accepted_at_ms":70487,"credit_policy":null,"job_id":"job-433","job_issued_at_ms":70508,"miner_id":"miner-4","network_difficulty":1000008,"ntime":1700000433,"order_key":"04:miner-4","p2mr_program_hex":"b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1","share_difficulty":3,"share_id":"share-433","share_seq":433,"template_height":800013},{"accepted_at_ms":70483,"credit_policy":"stale-grace","job_id":"job-434","job_issued_at_ms":70483,"miner_id":"miner-5","network_difficulty":1000009,"ntime":1700000434,"order_key":"05:miner-5","p2mr_program_hex":"b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2","share_difficulty":1,"share_id":"share-434","share_seq":434,"template_height":800013},{"accepted_at_ms":70497,"credit_policy":null,"job_id":"job-435","job_issued_at_ms":70502,"miner_id":"miner-6","network_difficulty":1000010,"ntime":1700000435,"order_key":"06:miner-6","p2mr_program_hex":"b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3","share_difficulty":7,"share_id":"share-435","share_seq":435,"template_height":800013},{"accepted_at_ms":70492,"credit_policy":null,"job_id":"job-436","job_issued_at_ms":70493,"miner_id":"miner-7","network_difficulty":1000011,"ntime":1700000436,"order_key":"07:miner-7","p2mr_program_hex":"b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4","share_difficulty":7,"share_id":"share-436","share_seq":436,"template_height":800013},{"accepted_at_ms":70490,"credit_policy":null,"job_id":"job-437","job_issued_at_ms":70492,"miner_id":"miner-8","network_difficulty":1000012,"ntime":1700000437,"order_key":"08:miner-8","p2mr_program_hex":"b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5","share_difficulty":1,"share_id":"share-437","share_seq":437,"template_height":800013},{"accepted_at_ms":70485,"credit_policy":null,"job_id":"job-438","job_issued_at_ms":70491,"miner_id":"miner-9","network_difficulty":1000013,"ntime":1700000438,"order_key":"09:miner-9","p2mr_program_hex":"b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6b6","share_difficulty":3,"share_id":"share-438","share_seq":438,"template_height":800013},{"accepted_at_ms":70496,"credit_policy":null,"job_id":"job-439","job_issued_at_ms":70511,"miner_id":"miner-10","network_difficulty":1000014,"ntime":1700000439,"order_key":"10:miner-10","p2mr_program_hex":"b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7b7","share_difficulty":7,"share_id":"share-439","share_seq":439,"template_height":800013},{"accepted_at_ms":70488,"credit_policy":null,"job_id":"job-440","job_issued_at_ms":70510,"miner_id":"miner-0","network_difficulty":1000015,"ntime":1700000440,"order_key":"00:miner-0","p2mr_program_hex":"b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8b8","share_difficulty":7,"share_id":"share-440","share_seq":440,"template_height":800013},{"accepted_at_ms":70505,"credit_policy":null,"job_id":"job-441","job_issued_at_ms":70499,"miner_id":"miner-1","network_difficulty":1000016,"ntime":1700000441,"order_key":"01:miner-1","p2mr_program_hex":"b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9b9","share_difficulty":1,"share_id":"share-441","share_seq":441,"template_height":800013},{"accepted_at_ms":70499,"credit_policy":null,"job_id":"job-442","job_issued_at_ms":70504,"miner_id":"miner-2","network_difficulty":1000000,"ntime":1700000442,"order_key":"02:miner-2","p2mr_program_hex":"babababababababababababababababababababababababababababababababa","share_difficulty":3,"share_id":"share-442","share_seq":442,"template_height":800013},{"accepted_at_ms":70492,"credit_policy":null,"job_id":"job-443","job_issued_at_ms":70499,"miner_id":"miner-3","network_difficulty":1000001,"ntime":1700000443,"order_key":"03:miner-3","p2mr_program_hex":"bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb","share_difficulty":7,"share_id":"share-443","share_seq":443,"template_height":800013},{"accepted_at_ms":70510,"credit_policy":null,"job_id":"job-444","job_issued_at_ms":70500,"miner_id":"miner-4","network_difficulty":1000002,"ntime":1700000444,"order_key":"04:miner-4","p2mr_program_hex":"bcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbc","share_difficulty":3,"share_id":"share-444","share_seq":444,"template_height":800013},{"accepted_at_ms":70483,"credit_policy":"stale-grace","job_id":"job-445","job_issued_at_ms":70497,"miner_id":"miner-5","network_difficulty":1000003,"ntime":1700000445,"order_key":"05:miner-5","p2mr_program_hex":"bdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbd","share_difficulty":1,"share_id":"share-445","share_seq":445,"template_height":800013},{"accepted_at_ms":70489,"credit_policy":null,"job_id":"job-446","job_issued_at_ms":70497,"miner_id":"miner-6","network_difficulty":1000004,"ntime":1700000446,"order_key":"06:miner-6","p2mr_program_hex":"bebebebebebebebebebebebebebebebebebebebebebebebebebebebebebebebe","share_difficulty":2,"share_id":"share-446","share_seq":446,"template_height":800013},{"accepted_at_ms":70497,"credit_policy":"stale-grace","job_id":"job-447","job_issued_at_ms":70499,"miner_id":"miner-7","network_difficulty":1000005,"ntime":1700000447,"order_key":"07:miner-7","p2mr_program_hex":"bfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbf","share_difficulty":3,"share_id":"share-447","share_seq":447,"template_height":800013},{"accepted_at_ms":70492,"credit_policy":null,"job_id":"job-448","job_issued_at_ms":70486,"miner_id":"miner-8","network_difficulty":1000006,"ntime":1700000448,"order_key":"08:miner-8","p2mr_program_hex":"c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0","share_difficulty":1,"share_id":"share-448","share_seq":448,"template_height":800014},{"accepted_at_ms":70512,"credit_policy":null,"job_id":"job-449","job_issued_at_ms":70492,"miner_id":"miner-9","network_difficulty":1000007,"ntime":1700000449,"order_key":"09:miner-9","p2mr_program_hex":"c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1","share_difficulty":7,"share_id":"share-449","share_seq":449,"template_height":800014},{"accepted_at_ms":70507,"credit_policy":null,"job_id":"job-450","job_issued_at_ms":70499,"miner_id":"miner-10","network_difficulty":1000008,"ntime":1700000450,"order_key":"10:miner-10","p2mr_program_hex":"c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2","share_difficulty":7,"share_id":"share-450","share_seq":450,"template_height":800014},{"accepted_at_ms":70504,"credit_policy":"stale-grace","job_id":"job-451","job_issued_at_ms":70498,"miner_id":"miner-0","network_difficulty":1000009,"ntime":1700000451,"order_key":"00:miner-0","p2mr_program_hex":"c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3","share_difficulty":3,"share_id":"share-451","share_seq":451,"template_height":800014},{"accepted_at_ms":70485,"credit_policy":null,"job_id":"job-452","job_issued_at_ms":70505,"miner_id":"miner-1","network_difficulty":1000010,"ntime":1700000452,"order_key":"01:miner-1","p2mr_program_hex":"c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4","share_difficulty":3,"share_id":"share-452","share_seq":452,"template_height":800014},{"accepted_at_ms":70506,"credit_policy":null,"job_id":"job-453","job_issued_at_ms":70484,"miner_id":"miner-2","network_difficulty":1000011,"ntime":1700000453,"order_key":"02:miner-2","p2mr_program_hex":"c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5","share_difficulty":7,"share_id":"share-453","share_seq":453,"template_height":800014},{"accepted_at_ms":70509,"credit_policy":"stale-grace","job_id":"job-454","job_issued_at_ms":70490,"miner_id":"miner-3","network_difficulty":1000012,"ntime":1700000454,"order_key":"03:miner-3","p2mr_program_hex":"c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6c6","share_difficulty":7,"share_id":"share-454","share_seq":454,"template_height":800014},{"accepted_at_ms":70505,"credit_policy":"stale-grace","job_id":"job-455","job_issued_at_ms":70497,"miner_id":"miner-4","network_difficulty":1000013,"ntime":1700000455,"order_key":"04:miner-4","p2mr_program_hex":"c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7c7","share_difficulty":2,"share_id":"share-455","share_seq":455,"template_height":800014},{"accepted_at_ms":70507,"credit_policy":null,"job_id":"job-456","job_issued_at_ms":70491,"miner_id":"miner-5","network_difficulty":1000014,"ntime":1700000456,"order_key":"05:miner-5","p2mr_program_hex":"c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8c8","share_difficulty":19,"share_id":"share-456","share_seq":456,"template_height":800014},{"accepted_at_ms":70510,"credit_policy":null,"job_id":"job-457","job_issued_at_ms":70509,"miner_id":"miner-6","network_difficulty":1000015,"ntime":1700000457,"order_key":"06:miner-6","p2mr_program_hex":"c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9","share_difficulty":19,"share_id":"share-457","share_seq":457,"template_height":800014},{"accepted_at_ms":70493,"credit_policy":null,"job_id":"job-458","job_issued_at_ms":70504,"miner_id":"miner-7","network_difficulty":1000016,"ntime":1700000458,"order_key":"07:miner-7","p2mr_program_hex":"cacacacacacacacacacacacacacacacacacacacacacacacacacacacacacacaca","share_difficulty":2,"share_id":"share-458","share_seq":458,"template_height":800014},{"accepted_at_ms":70491,"credit_policy":null,"job_id":"job-459","job_issued_at_ms":70486,"miner_id":"miner-8","network_difficulty":1000000,"ntime":1700000459,"order_key":"08:miner-8","p2mr_program_hex":"cbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcb","share_difficulty":1,"share_id":"share-459","share_seq":459,"template_height":800014},{"accepted_at_ms":70508,"credit_policy":null,"job_id":"job-460","job_issued_at_ms":70491,"miner_id":"miner-9","network_difficulty":1000001,"ntime":1700000460,"order_key":"09:miner-9","p2mr_program_hex":"cccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccc","share_difficulty":7,"share_id":"share-460","share_seq":460,"template_height":800014},{"accepted_at_ms":70492,"credit_policy":"stale-grace","job_id":"job-461","job_issued_at_ms":70503,"miner_id":"miner-10","network_difficulty":1000002,"ntime":1700000461,"order_key":"10:miner-10","p2mr_program_hex":"cdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcd","share_difficulty":7,"share_id":"share-461","share_seq":461,"template_height":800014},{"accepted_at_ms":70508,"credit_policy":"stale-grace","job_id":"job-462","job_issued_at_ms":70510,"miner_id":"miner-0","network_difficulty":1000003,"ntime":1700000462,"order_key":"00:miner-0","p2mr_program_hex":"cececececececececececececececececececececececececececececececece","share_difficulty":3,"share_id":"share-462","share_seq":462,"template_height":800014},{"accepted_at_ms":70483,"credit_policy":null,"job_id":"job-463","job_issued_at_ms":70490,"miner_id":"miner-1","network_difficulty":1000004,"ntime":1700000463,"order_key":"01:miner-1","p2mr_program_hex":"cfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcf","share_difficulty":1,"share_id":"share-463","share_seq":463,"template_height":800014},{"accepted_at_ms":70503,"credit_policy":"stale-grace","job_id":"job-464","job_issued_at_ms":70512,"miner_id":"miner-2","network_difficulty":1000005,"ntime":1700000464,"order_key":"02:miner-2","p2mr_program_hex":"d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0","share_difficulty":2,"share_id":"share-464","share_seq":464,"template_height":800014},{"accepted_at_ms":70484,"credit_policy":"stale-grace","job_id":"job-465","job_issued_at_ms":70484,"miner_id":"miner-3","network_difficulty":1000006,"ntime":1700000465,"order_key":"03:miner-3","p2mr_program_hex":"d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1","share_difficulty":1,"share_id":"share-465","share_seq":465,"template_height":800014},{"accepted_at_ms":70488,"credit_policy":null,"job_id":"job-466","job_issued_at_ms":70504,"miner_id":"miner-4","network_difficulty":1000007,"ntime":1700000466,"order_key":"04:miner-4","p2mr_program_hex":"d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2","share_difficulty":7,"share_id":"share-466","share_seq":466,"template_height":800014},{"accepted_at_ms":70484,"credit_policy":null,"job_id":"job-467","job_issued_at_ms":70507,"miner_id":"miner-5","network_difficulty":1000008,"ntime":1700000467,"order_key":"05:miner-5","p2mr_program_hex":"d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3","share_difficulty":1,"share_id":"share-467","share_seq":467,"template_height":800014},{"accepted_at_ms":70510,"credit_policy":"stale-grace","job_id":"job-468","job_issued_at_ms":70488,"miner_id":"miner-6","network_difficulty":1000009,"ntime":1700000468,"order_key":"06:miner-6","p2mr_program_hex":"d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4","share_difficulty":1,"share_id":"share-468","share_seq":468,"template_height":800014},{"accepted_at_ms":70490,"credit_policy":null,"job_id":"job-469","job_issued_at_ms":70513,"miner_id":"miner-7","network_difficulty":1000010,"ntime":1700000469,"order_key":"07:miner-7","p2mr_program_hex":"d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5","share_difficulty":3,"share_id":"share-469","share_seq":469,"template_height":800014},{"accepted_at_ms":70502,"credit_policy":null,"job_id":"job-470","job_issued_at_ms":70501,"miner_id":"miner-8","network_difficulty":1000011,"ntime":1700000470,"order_key":"08:miner-8","p2mr_program_hex":"d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6","share_difficulty":19,"share_id":"share-470","share_seq":470,"template_height":800014},{"accepted_at_ms":70496,"credit_policy":null,"job_id":"job-471","job_issued_at_ms":70507,"miner_id":"miner-9","network_difficulty":1000012,"ntime":1700000471,"order_key":"09:miner-9","p2mr_program_hex":"d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7d7","share_difficulty":19,"share_id":"share-471","share_seq":471,"template_height":800014},{"accepted_at_ms":70510,"credit_policy":null,"job_id":"job-472","job_issued_at_ms":70491,"miner_id":"miner-10","network_difficulty":1000013,"ntime":1700000472,"order_key":"10:miner-10","p2mr_program_hex":"d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8","share_difficulty":2,"share_id":"share-472","share_seq":472,"template_height":800014},{"accepted_at_ms":70502,"credit_policy":null,"job_id":"job-473","job_issued_at_ms":70513,"miner_id":"miner-0","network_difficulty":1000014,"ntime":1700000473,"order_key":"00:miner-0","p2mr_program_hex":"d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9d9","share_difficulty":7,"share_id":"share-473","share_seq":473,"template_height":800014},{"accepted_at_ms":70491,"credit_policy":null,"job_id":"job-474","job_issued_at_ms":70483,"miner_id":"miner-1","network_difficulty":1000015,"ntime":1700000474,"order_key":"01:miner-1","p2mr_program_hex":"dadadadadadadadadadadadadadadadadadadadadadadadadadadadadadadada","share_difficulty":3,"share_id":"share-474","share_seq":474,"template_height":800014},{"accepted_at_ms":70511,"credit_policy":"stale-grace","job_id":"job-475","job_issued_at_ms":70489,"miner_id":"miner-2","network_difficulty":1000016,"ntime":1700000475,"order_key":"02:miner-2","p2mr_program_hex":"dbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdbdb","share_difficulty":19,"share_id":"share-475","share_seq":475,"template_height":800014},{"accepted_at_ms":70511,"credit_policy":null,"job_id":"job-476","job_issued_at_ms":70508,"miner_id":"miner-3","network_difficulty":1000000,"ntime":1700000476,"order_key":"03:miner-3","p2mr_program_hex":"dcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdc","share_difficulty":3,"share_id":"share-476","share_seq":476,"template_height":800014},{"accepted_at_ms":70489,"credit_policy":null,"job_id":"job-477","job_issued_at_ms":70513,"miner_id":"miner-4","network_difficulty":1000001,"ntime":1700000477,"order_key":"04:miner-4","p2mr_program_hex":"dddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddd","share_difficulty":2,"share_id":"share-477","share_seq":477,"template_height":800014},{"accepted_at_ms":70512,"credit_policy":null,"job_id":"job-478","job_issued_at_ms":70504,"miner_id":"miner-5","network_difficulty":1000002,"ntime":1700000478,"order_key":"05:miner-5","p2mr_program_hex":"dededededededededededededededededededededededededededededededede","share_difficulty":19,"share_id":"share-478","share_seq":478,"template_height":800014},{"accepted_at_ms":70488,"credit_policy":null,"job_id":"job-479","job_issued_at_ms":70493,"miner_id":"miner-6","network_difficulty":1000003,"ntime":1700000479,"order_key":"06:miner-6","p2mr_program_hex":"dfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdf","share_difficulty":1,"share_id":"share-479","share_seq":479,"template_height":800014},{"accepted_at_ms":70484,"credit_policy":"stale-grace","job_id":"job-480","job_issued_at_ms":70508,"miner_id":"miner-7","network_difficulty":1000004,"ntime":1700000480,"order_key":"07:miner-7","p2mr_program_hex":"e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0","share_difficulty":19,"share_id":"share-480","share_seq":480,"template_height":800015},{"accepted_at_ms":70507,"credit_policy":null,"job_id":"job-481","job_issued_at_ms":70511,"miner_id":"miner-8","network_difficulty":1000005,"ntime":1700000481,"order_key":"08:miner-8","p2mr_program_hex":"e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1","share_difficulty":19,"share_id":"share-481","share_seq":481,"template_height":800015},{"accepted_at_ms":70502,"credit_policy":null,"job_id":"job-482","job_issued_at_ms":70505,"miner_id":"miner-9","network_difficulty":1000006,"ntime":1700000482,"order_key":"09:miner-9","p2mr_program_hex":"e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2","share_difficulty":1,"share_id":"share-482","share_seq":482,"template_height":800015},{"accepted_at_ms":70483,"credit_policy":null,"job_id":"job-483","job_issued_at_ms":70510,"miner_id":"miner-10","network_difficulty":1000007,"ntime":1700000483,"order_key":"10:miner-10","p2mr_program_hex":"e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3","share_difficulty":3,"share_id":"share-483","share_seq":483,"template_height":800015},{"accepted_at_ms":70483,"credit_policy":null,"job_id":"job-484","job_issued_at_ms":70513,"miner_id":"miner-0","network_difficulty":1000008,"ntime":1700000484,"order_key":"00:miner-0","p2mr_program_hex":"e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4","share_difficulty":7,"share_id":"share-484","share_seq":484,"template_height":800015},{"accepted_at_ms":70487,"credit_policy":null,"job_id":"job-485","job_issued_at_ms":70485,"miner_id":"miner-1","network_difficulty":1000009,"ntime":1700000485,"order_key":"01:miner-1","p2mr_program_hex":"e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5","share_difficulty":1,"share_id":"share-485","share_seq":485,"template_height":800015},{"accepted_at_ms":70502,"credit_policy":null,"job_id":"job-486","job_issued_at_ms":70494,"miner_id":"miner-2","network_difficulty":1000010,"ntime":1700000486,"order_key":"02:miner-2","p2mr_program_hex":"e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6","share_difficulty":3,"share_id":"share-486","share_seq":486,"template_height":800015},{"accepted_at_ms":70501,"credit_policy":null,"job_id":"job-487","job_issued_at_ms":70490,"miner_id":"miner-3","network_difficulty":1000011,"ntime":1700000487,"order_key":"03:miner-3","p2mr_program_hex":"e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7","share_difficulty":7,"share_id":"share-487","share_seq":487,"template_height":800015},{"accepted_at_ms":70500,"credit_policy":null,"job_id":"job-488","job_issued_at_ms":70501,"miner_id":"miner-4","network_difficulty":1000012,"ntime":1700000488,"order_key":"04:miner-4","p2mr_program_hex":"e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8","share_difficulty":3,"share_id":"share-488","share_seq":488,"template_height":800015},{"accepted_at_ms":70488,"credit_policy":"stale-grace","job_id":"job-489","job_issued_at_ms":70497,"miner_id":"miner-5","network_difficulty":1000013,"ntime":1700000489,"order_key":"05:miner-5","p2mr_program_hex":"e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9","share_difficulty":19,"share_id":"share-489","share_seq":489,"template_height":800015},{"accepted_at_ms":70509,"credit_policy":"stale-grace","job_id":"job-490","job_issued_at_ms":70507,"miner_id":"miner-6","network_difficulty":1000014,"ntime":1700000490,"order_key":"06:miner-6","p2mr_program_hex":"eaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaea","share_difficulty":19,"share_id":"share-490","share_seq":490,"template_height":800015},{"accepted_at_ms":70500,"credit_policy":null,"job_id":"job-491","job_issued_at_ms":70483,"miner_id":"miner-7","network_difficulty":1000015,"ntime":1700000491,"order_key":"07:miner-7","p2mr_program_hex":"ebebebebebebebebebebebebebebebebebebebebebebebebebebebebebebebeb","share_difficulty":19,"share_id":"share-491","share_seq":491,"template_height":800015},{"accepted_at_ms":70510,"credit_policy":"stale-grace","job_id":"job-492","job_issued_at_ms":70510,"miner_id":"miner-8","network_difficulty":1000016,"ntime":1700000492,"order_key":"08:miner-8","p2mr_program_hex":"ecececececececececececececececececececececececececececececececec","share_difficulty":1,"share_id":"share-492","share_seq":492,"template_height":800015},{"accepted_at_ms":70489,"credit_policy":null,"job_id":"job-493","job_issued_at_ms":70485,"miner_id":"miner-9","network_difficulty":1000000,"ntime":1700000493,"order_key":"09:miner-9","p2mr_program_hex":"edededededededededededededededededededededededededededededededed","share_difficulty":2,"share_id":"share-493","share_seq":493,"template_height":800015},{"accepted_at_ms":70504,"credit_policy":null,"job_id":"job-494","job_issued_at_ms":70483,"miner_id":"miner-10","network_difficulty":1000001,"ntime":1700000494,"order_key":"10:miner-10","p2mr_program_hex":"eeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeee","share_difficulty":1,"share_id":"share-494","share_seq":494,"template_height":800015},{"accepted_at_ms":70511,"credit_policy":null,"job_id":"job-495","job_issued_at_ms":70505,"miner_id":"miner-0","network_difficulty":1000002,"ntime":1700000495,"order_key":"00:miner-0","p2mr_program_hex":"efefefefefefefefefefefefefefefefefefefefefefefefefefefefefefefef","share_difficulty":19,"share_id":"share-495","share_seq":495,"template_height":800015},{"accepted_at_ms":70499,"credit_policy":null,"job_id":"job-496","job_issued_at_ms":70494,"miner_id":"miner-1","network_difficulty":1000003,"ntime":1700000496,"order_key":"01:miner-1","p2mr_program_hex":"f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0","share_difficulty":1,"share_id":"share-496","share_seq":496,"template_height":800015},{"accepted_at_ms":70495,"credit_policy":null,"job_id":"job-497","job_issued_at_ms":70493,"miner_id":"miner-2","network_difficulty":1000004,"ntime":1700000497,"order_key":"02:miner-2","p2mr_program_hex":"f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1","share_difficulty":3,"share_id":"share-497","share_seq":497,"template_height":800015},{"accepted_at_ms":70493,"credit_policy":null,"job_id":"job-498","job_issued_at_ms":70503,"miner_id":"miner-3","network_difficulty":1000005,"ntime":1700000498,"order_key":"03:miner-3","p2mr_program_hex":"f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2","share_difficulty":3,"share_id":"share-498","share_seq":498,"template_height":800015},{"accepted_at_ms":70511,"credit_policy":null,"job_id":"job-499","job_issued_at_ms":70485,"miner_id":"miner-4","network_difficulty":1000006,"ntime":1700000499,"order_key":"04:miner-4","p2mr_program_hex":"f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3f3","share_difficulty":2,"share_id":"share-499","share_seq":499,"template_height":800015},{"accepted_at_ms":70483,"credit_policy":null,"job_id":"job-500","job_issued_at_ms":70510,"miner_id":"miner-5","network_difficulty":1000007,"ntime":1700000500,"order_key":"05:miner-5","p2mr_program_hex":"f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4","share_difficulty":7,"share_id":"share-500","share_seq":500,"template_height":800015},{"accepted_at_ms":70501,"credit_policy":"stale-grace","job_id":"job-501","job_issued_at_ms":70513,"miner_id":"miner-6","network_difficulty":1000008,"ntime":1700000501,"order_key":"06:miner-6","p2mr_program_hex":"f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5","share_difficulty":1,"share_id":"share-501","share_seq":501,"template_height":800015},{"accepted_at_ms":70492,"credit_policy":null,"job_id":"job-502","job_issued_at_ms":70485,"miner_id":"miner-7","network_difficulty":1000009,"ntime":1700000502,"order_key":"07:miner-7","p2mr_program_hex":"f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6","share_difficulty":2,"share_id":"share-502","share_seq":502,"template_height":800015},{"accepted_at_ms":70511,"credit_policy":"stale-grace","job_id":"job-503","job_issued_at_ms":70497,"miner_id":"miner-8","network_difficulty":1000010,"ntime":1700000503,"order_key":"08:miner-8","p2mr_program_hex":"f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7","share_difficulty":2,"share_id":"share-503","share_seq":503,"template_height":800015},{"accepted_at_ms":70506,"credit_policy":null,"job_id":"job-504","job_issued_at_ms":70488,"miner_id":"miner-9","network_difficulty":1000011,"ntime":1700000504,"order_key":"09:miner-9","p2mr_program_hex":"f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8f8","share_difficulty":1,"share_id":"share-504","share_seq":504,"template_height":800015},{"accepted_at_ms":70511,"credit_policy":"stale-grace","job_id":"job-505","job_issued_at_ms":70510,"miner_id":"miner-10","network_difficulty":1000012,"ntime":1700000505,"order_key":"10:miner-10","p2mr_program_hex":"f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9","share_difficulty":3,"share_id":"share-505","share_seq":505,"template_height":800015},{"accepted_at_ms":70494,"credit_policy":null,"job_id":"job-506","job_issued_at_ms":70501,"miner_id":"miner-0","network_difficulty":1000013,"ntime":1700000506,"order_key":"00:miner-0","p2mr_program_hex":"fafafafafafafafafafafafafafafafafafafafafafafafafafafafafafafafa","share_difficulty":7,"share_id":"share-506","share_seq":506,"template_height":800015},{"accepted_at_ms":70510,"credit_policy":null,"job_id":"job-507","job_issued_at_ms":70502,"miner_id":"miner-1","network_difficulty":1000014,"ntime":1700000507,"order_key":"01:miner-1","p2mr_program_hex":"fbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfb","share_difficulty":19,"share_id":"share-507","share_seq":507,"template_height":800015},{"accepted_at_ms":70509,"credit_policy":"stale-grace","job_id":"job-508","job_issued_at_ms":70508,"miner_id":"miner-2","network_difficulty":1000015,"ntime":1700000508,"order_key":"02:miner-2","p2mr_program_hex":"fcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfcfc","share_difficulty":1,"share_id":"share-508","share_seq":508,"template_height":800015},{"accepted_at_ms":70487,"credit_policy":null,"job_id":"job-509","job_issued_at_ms":70493,"miner_id":"miner-3","network_difficulty":1000016,"ntime":1700000509,"order_key":"03:miner-3","p2mr_program_hex":"fdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfd","share_difficulty":3,"share_id":"share-509","share_seq":509,"template_height":800015},{"accepted_at_ms":70490,"credit_policy":null,"job_id":"job-510","job_issued_at_ms":70508,"miner_id":"miner-4","network_difficulty":1000000,"ntime":1700000510,"order_key":"04:miner-4","p2mr_program_hex":"fefefefefefefefefefefefefefefefefefefefefefefefefefefefefefefefe","share_difficulty":1,"share_id":"share-510","share_seq":510,"template_height":800015},{"accepted_at_ms":70495,"credit_policy":null,"job_id":"job-511","job_issued_at_ms":70495,"miner_id":"miner-5","network_difficulty":1000001,"ntime":1700000511,"order_key":"05:miner-5","p2mr_program_hex":"ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff","share_difficulty":7,"share_id":"share-511","share_seq":511,"template_height":800015},{"accepted_at_ms":70487,"credit_policy":null,"job_id":"job-512","job_issued_at_ms":70486,"miner_id":"miner-6","network_difficulty":1000002,"ntime":1700000512,"order_key":"06:miner-6","p2mr_program_hex":"0000000000000000000000000000000000000000000000000000000000000000","share_difficulty":3,"share_id":"share-512","share_seq":512,"template_height":800016},{"accepted_at_ms":70505,"credit_policy":null,"job_id":"job-513","job_issued_at_ms":70502,"miner_id":"miner-7","network_difficulty":1000003,"ntime":1700000513,"order_key":"07:miner-7","p2mr_program_hex":"0101010101010101010101010101010101010101010101010101010101010101","share_difficulty":2,"share_id":"share-513","share_seq":513,"template_height":800016}]},"window_weight":1000000000}},"command":"PYTHONDONTWRITEBYTECODE=1 python3 -c 'import hashlib, json, shlex, sys; from tests.window_pipeline_parity import build_corpus, canonical_json_bytes, case_input_document; names = (\"bulk-seeded\", \"multi-page-interior-cutoff\", \"page-boundary-511\", \"page-boundary-512\", \"page-boundary-513\"); frozen = json.load(open(\"tests/fixtures/window_pipeline_parity/reference.json\"))[\"cases\"]; corpus = {case.name: case for case in build_corpus()}; cases = {name: case_input_document(corpus[name]) for name in names}; bad = [name for name in names if hashlib.sha256(canonical_json_bytes(cases[name])).hexdigest() != frozen[name][\"input_sha256\"]]; bad and sys.exit(\"input_sha256 mismatch: \" + \", \".join(bad)); command = \"PYTHONDONTWRITEBYTECODE=1 python3 -c \" + shlex.quote(sys.orig_argv[-1]) + \" > inputs-unpinned.json\"; sys.stdout.write(json.dumps({\"schema\": \"qbit-prism-window-pipeline-parity-unpinned-inputs/v1\", \"source_commit\": \"504846cc0b72e8f86ed17f896d4ccbbe196a31dc\", \"command\": command, \"cases\": cases}, sort_keys=True, separators=(\",\", \":\")) + \"\\n\")' > inputs-unpinned.json","schema":"qbit-prism-window-pipeline-parity-unpinned-inputs/v1","source_commit":"504846cc0b72e8f86ed17f896d4ccbbe196a31dc"} diff --git a/tests/fixtures/window_pipeline_parity/supplementary.json b/tests/fixtures/window_pipeline_parity/supplementary.json new file mode 100644 index 000000000..338daf625 --- /dev/null +++ b/tests/fixtures/window_pipeline_parity/supplementary.json @@ -0,0 +1,610 @@ +{ + "cases": { + "advance-page-expiry-at-weight": { + "advance_stats": [ + { + "added_rows": 2, + "expired_rows": 2, + "touched_pages": 0 + } + ], + "canonical_bytes_len": 661, + "canonical_bytes_sha256": "82a04cf66a192bb2401a881bfc28bdbc2bd293cb0e807c62ed5ead692275b5ed", + "canonical_digest": "82a04cf66a192bb2401a881bfc28bdbc2bd293cb0e807c62ed5ead692275b5ed", + "input_sha256": "1c31011179d010d61ea826516d9ffd4ce09ed3c9293ff1f23538fcd8efe42577", + "pinned_literals": { + "canonical_bytes": "[{\"accepted_at_ms\":300002,\"job_id\":\"job-5\",\"job_issued_at_ms\":300001,\"miner_id\":\"miner-5\",\"network_difficulty\":1000005,\"ntime\":1700000005,\"order_key\":\"05:miner-5\",\"p2mr_program_hex\":\"0505050505050505050505050505050505050505050505050505050505050505\",\"share_difficulty\":1,\"share_id\":\"share-5\",\"share_seq\":5,\"template_height\":800000},{\"accepted_at_ms\":300002,\"job_id\":\"job-6\",\"job_issued_at_ms\":300001,\"miner_id\":\"miner-6\",\"network_difficulty\":1000006,\"ntime\":1700000006,\"order_key\":\"06:miner-6\",\"p2mr_program_hex\":\"0606060606060606060606060606060606060606060606060606060606060606\",\"share_difficulty\":1,\"share_id\":\"share-6\",\"share_seq\":6,\"template_height\":800000}]", + "input": { + "advances": [ + { + "anchor_job_issued_at_ms": 300002, + "records": [ + { + "accepted_at_ms": 300002, + "credit_policy": null, + "job_id": "job-5", + "job_issued_at_ms": 300001, + "miner_id": "miner-5", + "network_difficulty": 1000005, + "ntime": 1700000005, + "order_key": "05:miner-5", + "p2mr_program_hex": "0505050505050505050505050505050505050505050505050505050505050505", + "share_difficulty": 1, + "share_id": "share-5", + "share_seq": 5, + "template_height": 800000 + }, + { + "accepted_at_ms": 300002, + "credit_policy": null, + "job_id": "job-6", + "job_issued_at_ms": 300001, + "miner_id": "miner-6", + "network_difficulty": 1000006, + "ntime": 1700000006, + "order_key": "06:miner-6", + "p2mr_program_hex": "0606060606060606060606060606060606060606060606060606060606060606", + "share_difficulty": 1, + "share_id": "share-6", + "share_seq": 6, + "template_height": 800000 + } + ] + } + ], + "name": "advance-page-expiry-at-weight", + "page_size": 2, + "schema": "qbit-prism-window-pipeline-parity-input/v1", + "snapshot": { + "anchor_job_issued_at_ms": 300000, + "records": [ + { + "accepted_at_ms": 299991, + "credit_policy": null, + "job_id": "job-1", + "job_issued_at_ms": 299981, + "miner_id": "miner-1", + "network_difficulty": 1000001, + "ntime": 1700000001, + "order_key": "01:miner-1", + "p2mr_program_hex": "0101010101010101010101010101010101010101010101010101010101010101", + "share_difficulty": 1, + "share_id": "share-1", + "share_seq": 1, + "template_height": 800000 + }, + { + "accepted_at_ms": 299992, + "credit_policy": null, + "job_id": "job-2", + "job_issued_at_ms": 299982, + "miner_id": "miner-2", + "network_difficulty": 1000002, + "ntime": 1700000002, + "order_key": "02:miner-2", + "p2mr_program_hex": "0202020202020202020202020202020202020202020202020202020202020202", + "share_difficulty": 1, + "share_id": "share-2", + "share_seq": 2, + "template_height": 800000 + }, + { + "accepted_at_ms": 299993, + "credit_policy": null, + "job_id": "job-3", + "job_issued_at_ms": 299983, + "miner_id": "miner-3", + "network_difficulty": 1000003, + "ntime": 1700000003, + "order_key": "03:miner-3", + "p2mr_program_hex": "0303030303030303030303030303030303030303030303030303030303030303", + "share_difficulty": 1, + "share_id": "share-3", + "share_seq": 3, + "template_height": 800000 + }, + { + "accepted_at_ms": 299994, + "credit_policy": null, + "job_id": "job-4", + "job_issued_at_ms": 299984, + "miner_id": "miner-4", + "network_difficulty": 1000004, + "ntime": 1700000004, + "order_key": "04:miner-4", + "p2mr_program_hex": "0404040404040404040404040404040404040404040404040404040404040404", + "share_difficulty": 1, + "share_id": "share-4", + "share_seq": 4, + "template_height": 800000 + } + ] + }, + "window_weight": 2 + }, + "record_jsons": [ + "{\"accepted_at_ms\":300002,\"job_id\":\"job-5\",\"job_issued_at_ms\":300001,\"miner_id\":\"miner-5\",\"network_difficulty\":1000005,\"ntime\":1700000005,\"order_key\":\"05:miner-5\",\"p2mr_program_hex\":\"0505050505050505050505050505050505050505050505050505050505050505\",\"share_difficulty\":1,\"share_id\":\"share-5\",\"share_seq\":5,\"template_height\":800000}", + "{\"accepted_at_ms\":300002,\"job_id\":\"job-6\",\"job_issued_at_ms\":300001,\"miner_id\":\"miner-6\",\"network_difficulty\":1000006,\"ntime\":1700000006,\"order_key\":\"06:miner-6\",\"p2mr_program_hex\":\"0606060606060606060606060606060606060606060606060606060606060606\",\"share_difficulty\":1,\"share_id\":\"share-6\",\"share_seq\":6,\"template_height\":800000}" + ], + "spool_tail": ",\"compact_share_identities\":[[\"miner-5\",\"05:miner-5\",\"0505050505050505050505050505050505050505050505050505050505050505\"],[\"miner-6\",\"06:miner-6\",\"0606060606060606060606060606060606060606060606060606060606060606\"]],\"compact_shares\":[[5,\"share-5\",0,1,300001,300002,null],[6,\"share-6\",1,1,300001,300002,null]]}" + }, + "record_count": 2, + "record_stream_len": 659, + "record_stream_sha256": "53f3d5bc9fa4dd43cc9c6766969aaf2a886f19f922d9b555592b1d96586eba7a", + "spool_tail_len": 307, + "spool_tail_sha256": "67885cf4dc3a110d6a9c4014a3f83f15b6b107cb846d15f9cac884b18759863b", + "why": "page_size 2, weight 2, four difficulty-1 records: the snapshot retains the one full page [3,4] at total 2; the advance appends the fresh page [5,6] (total 4) and 4-2 = 2 >= 2 expires the old page wholesale exactly at the weight, touched_pages 0. A whole-page expiry using > instead of >= keeps the page, empties it in the partial loop and counts it touched" + }, + "advance-page-expiry-at-weight-second-page": { + "advance_stats": [ + { + "added_rows": 4, + "expired_rows": 4, + "touched_pages": 0 + } + ], + "canonical_bytes_len": 1326, + "canonical_bytes_sha256": "108bcacec300122f4b5ddd0f5ef8e007f10c1735e566adb4866ba2b32b0dae5f", + "canonical_digest": "108bcacec300122f4b5ddd0f5ef8e007f10c1735e566adb4866ba2b32b0dae5f", + "input_sha256": "d7c2f76002324b57bc958d6c6fe66a4ef0be414902ae45e83aa0ea0420f8a2a1", + "pinned_literals": { + "canonical_bytes": "[{\"accepted_at_ms\":310002,\"job_id\":\"job-7\",\"job_issued_at_ms\":310001,\"miner_id\":\"miner-7\",\"network_difficulty\":1000007,\"ntime\":1700000007,\"order_key\":\"07:miner-7\",\"p2mr_program_hex\":\"0707070707070707070707070707070707070707070707070707070707070707\",\"share_difficulty\":1,\"share_id\":\"share-7\",\"share_seq\":7,\"template_height\":800000},{\"accepted_at_ms\":310002,\"job_id\":\"job-8\",\"job_issued_at_ms\":310001,\"miner_id\":\"miner-8\",\"network_difficulty\":1000008,\"ntime\":1700000008,\"order_key\":\"08:miner-8\",\"p2mr_program_hex\":\"0808080808080808080808080808080808080808080808080808080808080808\",\"share_difficulty\":1,\"share_id\":\"share-8\",\"share_seq\":8,\"template_height\":800000},{\"accepted_at_ms\":310002,\"job_id\":\"job-9\",\"job_issued_at_ms\":310001,\"miner_id\":\"miner-9\",\"network_difficulty\":1000009,\"ntime\":1700000009,\"order_key\":\"09:miner-9\",\"p2mr_program_hex\":\"0909090909090909090909090909090909090909090909090909090909090909\",\"share_difficulty\":1,\"share_id\":\"share-9\",\"share_seq\":9,\"template_height\":800000},{\"accepted_at_ms\":310002,\"job_id\":\"job-10\",\"job_issued_at_ms\":310001,\"miner_id\":\"miner-10\",\"network_difficulty\":1000010,\"ntime\":1700000010,\"order_key\":\"10:miner-10\",\"p2mr_program_hex\":\"0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a\",\"share_difficulty\":1,\"share_id\":\"share-10\",\"share_seq\":10,\"template_height\":800000}]", + "input": { + "advances": [ + { + "anchor_job_issued_at_ms": 310002, + "records": [ + { + "accepted_at_ms": 310002, + "credit_policy": null, + "job_id": "job-7", + "job_issued_at_ms": 310001, + "miner_id": "miner-7", + "network_difficulty": 1000007, + "ntime": 1700000007, + "order_key": "07:miner-7", + "p2mr_program_hex": "0707070707070707070707070707070707070707070707070707070707070707", + "share_difficulty": 1, + "share_id": "share-7", + "share_seq": 7, + "template_height": 800000 + }, + { + "accepted_at_ms": 310002, + "credit_policy": null, + "job_id": "job-8", + "job_issued_at_ms": 310001, + "miner_id": "miner-8", + "network_difficulty": 1000008, + "ntime": 1700000008, + "order_key": "08:miner-8", + "p2mr_program_hex": "0808080808080808080808080808080808080808080808080808080808080808", + "share_difficulty": 1, + "share_id": "share-8", + "share_seq": 8, + "template_height": 800000 + }, + { + "accepted_at_ms": 310002, + "credit_policy": null, + "job_id": "job-9", + "job_issued_at_ms": 310001, + "miner_id": "miner-9", + "network_difficulty": 1000009, + "ntime": 1700000009, + "order_key": "09:miner-9", + "p2mr_program_hex": "0909090909090909090909090909090909090909090909090909090909090909", + "share_difficulty": 1, + "share_id": "share-9", + "share_seq": 9, + "template_height": 800000 + }, + { + "accepted_at_ms": 310002, + "credit_policy": null, + "job_id": "job-10", + "job_issued_at_ms": 310001, + "miner_id": "miner-10", + "network_difficulty": 1000010, + "ntime": 1700000010, + "order_key": "10:miner-10", + "p2mr_program_hex": "0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a", + "share_difficulty": 1, + "share_id": "share-10", + "share_seq": 10, + "template_height": 800000 + } + ] + } + ], + "name": "advance-page-expiry-at-weight-second-page", + "page_size": 2, + "schema": "qbit-prism-window-pipeline-parity-input/v1", + "snapshot": { + "anchor_job_issued_at_ms": 310000, + "records": [ + { + "accepted_at_ms": 309991, + "credit_policy": null, + "job_id": "job-1", + "job_issued_at_ms": 309981, + "miner_id": "miner-1", + "network_difficulty": 1000001, + "ntime": 1700000001, + "order_key": "01:miner-1", + "p2mr_program_hex": "0101010101010101010101010101010101010101010101010101010101010101", + "share_difficulty": 1, + "share_id": "share-1", + "share_seq": 1, + "template_height": 800000 + }, + { + "accepted_at_ms": 309992, + "credit_policy": null, + "job_id": "job-2", + "job_issued_at_ms": 309982, + "miner_id": "miner-2", + "network_difficulty": 1000002, + "ntime": 1700000002, + "order_key": "02:miner-2", + "p2mr_program_hex": "0202020202020202020202020202020202020202020202020202020202020202", + "share_difficulty": 1, + "share_id": "share-2", + "share_seq": 2, + "template_height": 800000 + }, + { + "accepted_at_ms": 309993, + "credit_policy": null, + "job_id": "job-3", + "job_issued_at_ms": 309983, + "miner_id": "miner-3", + "network_difficulty": 1000003, + "ntime": 1700000003, + "order_key": "03:miner-3", + "p2mr_program_hex": "0303030303030303030303030303030303030303030303030303030303030303", + "share_difficulty": 1, + "share_id": "share-3", + "share_seq": 3, + "template_height": 800000 + }, + { + "accepted_at_ms": 309994, + "credit_policy": null, + "job_id": "job-4", + "job_issued_at_ms": 309984, + "miner_id": "miner-4", + "network_difficulty": 1000004, + "ntime": 1700000004, + "order_key": "04:miner-4", + "p2mr_program_hex": "0404040404040404040404040404040404040404040404040404040404040404", + "share_difficulty": 1, + "share_id": "share-4", + "share_seq": 4, + "template_height": 800000 + }, + { + "accepted_at_ms": 309995, + "credit_policy": null, + "job_id": "job-5", + "job_issued_at_ms": 309985, + "miner_id": "miner-5", + "network_difficulty": 1000005, + "ntime": 1700000005, + "order_key": "05:miner-5", + "p2mr_program_hex": "0505050505050505050505050505050505050505050505050505050505050505", + "share_difficulty": 1, + "share_id": "share-5", + "share_seq": 5, + "template_height": 800000 + }, + { + "accepted_at_ms": 309996, + "credit_policy": null, + "job_id": "job-6", + "job_issued_at_ms": 309986, + "miner_id": "miner-6", + "network_difficulty": 1000006, + "ntime": 1700000006, + "order_key": "06:miner-6", + "p2mr_program_hex": "0606060606060606060606060606060606060606060606060606060606060606", + "share_difficulty": 1, + "share_id": "share-6", + "share_seq": 6, + "template_height": 800000 + } + ] + }, + "window_weight": 4 + }, + "record_jsons": [ + "{\"accepted_at_ms\":310002,\"job_id\":\"job-7\",\"job_issued_at_ms\":310001,\"miner_id\":\"miner-7\",\"network_difficulty\":1000007,\"ntime\":1700000007,\"order_key\":\"07:miner-7\",\"p2mr_program_hex\":\"0707070707070707070707070707070707070707070707070707070707070707\",\"share_difficulty\":1,\"share_id\":\"share-7\",\"share_seq\":7,\"template_height\":800000}", + "{\"accepted_at_ms\":310002,\"job_id\":\"job-8\",\"job_issued_at_ms\":310001,\"miner_id\":\"miner-8\",\"network_difficulty\":1000008,\"ntime\":1700000008,\"order_key\":\"08:miner-8\",\"p2mr_program_hex\":\"0808080808080808080808080808080808080808080808080808080808080808\",\"share_difficulty\":1,\"share_id\":\"share-8\",\"share_seq\":8,\"template_height\":800000}", + "{\"accepted_at_ms\":310002,\"job_id\":\"job-9\",\"job_issued_at_ms\":310001,\"miner_id\":\"miner-9\",\"network_difficulty\":1000009,\"ntime\":1700000009,\"order_key\":\"09:miner-9\",\"p2mr_program_hex\":\"0909090909090909090909090909090909090909090909090909090909090909\",\"share_difficulty\":1,\"share_id\":\"share-9\",\"share_seq\":9,\"template_height\":800000}", + "{\"accepted_at_ms\":310002,\"job_id\":\"job-10\",\"job_issued_at_ms\":310001,\"miner_id\":\"miner-10\",\"network_difficulty\":1000010,\"ntime\":1700000010,\"order_key\":\"10:miner-10\",\"p2mr_program_hex\":\"0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a\",\"share_difficulty\":1,\"share_id\":\"share-10\",\"share_seq\":10,\"template_height\":800000}" + ], + "spool_tail": ",\"compact_share_identities\":[[\"miner-7\",\"07:miner-7\",\"0707070707070707070707070707070707070707070707070707070707070707\"],[\"miner-8\",\"08:miner-8\",\"0808080808080808080808080808080808080808080808080808080808080808\"],[\"miner-9\",\"09:miner-9\",\"0909090909090909090909090909090909090909090909090909090909090909\"],[\"miner-10\",\"10:miner-10\",\"0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a\"]],\"compact_shares\":[[7,\"share-7\",0,1,310001,310002,null],[8,\"share-8\",1,1,310001,310002,null],[9,\"share-9\",2,1,310001,310002,null],[10,\"share-10\",3,1,310001,310002,null]]}" + }, + "record_count": 4, + "record_stream_len": 1324, + "record_stream_sha256": "388d090b2388249aefb5582b08833ce98fb9152f0e8f9c045591bdf8d5467976", + "spool_tail_len": 569, + "spool_tail_sha256": "8342a226773986bc571f2f655b54348f79dc385935e3fc7785a770c93860bddf", + "why": "page_size 2, weight 4, six difficulty-1 records: the snapshot retains pages [3,4],[5,6] at total 4; the advance appends [7,8],[9,10] (total 8); 8-2 = 6 expires [3,4] above the weight, then 6-2 = 4 >= 4 expires [5,6] exactly at the weight, retaining [7,8],[9,10] with touched_pages 0. The exact-weight page is the second retained page, so the whole-page loop must keep going after its first expiry" + }, + "advance-page-expiry-one-above-weight": { + "advance_stats": [ + { + "added_rows": 2, + "expired_rows": 2, + "touched_pages": 0 + } + ], + "canonical_bytes_len": 661, + "canonical_bytes_sha256": "f1e28c7eb1642cc46855542061403d80db320b8207b82fd37b194e462e3d17c9", + "canonical_digest": "f1e28c7eb1642cc46855542061403d80db320b8207b82fd37b194e462e3d17c9", + "input_sha256": "b447bfdc7826b099c9968d0db6349cebf32fe7464c4313cf92bef0db37f90115", + "pinned_literals": { + "canonical_bytes": "[{\"accepted_at_ms\":330002,\"job_id\":\"job-5\",\"job_issued_at_ms\":330001,\"miner_id\":\"miner-5\",\"network_difficulty\":1000005,\"ntime\":1700000005,\"order_key\":\"05:miner-5\",\"p2mr_program_hex\":\"0505050505050505050505050505050505050505050505050505050505050505\",\"share_difficulty\":2,\"share_id\":\"share-5\",\"share_seq\":5,\"template_height\":800000},{\"accepted_at_ms\":330002,\"job_id\":\"job-6\",\"job_issued_at_ms\":330001,\"miner_id\":\"miner-6\",\"network_difficulty\":1000006,\"ntime\":1700000006,\"order_key\":\"06:miner-6\",\"p2mr_program_hex\":\"0606060606060606060606060606060606060606060606060606060606060606\",\"share_difficulty\":2,\"share_id\":\"share-6\",\"share_seq\":6,\"template_height\":800000}]", + "input": { + "advances": [ + { + "anchor_job_issued_at_ms": 330002, + "records": [ + { + "accepted_at_ms": 330002, + "credit_policy": null, + "job_id": "job-5", + "job_issued_at_ms": 330001, + "miner_id": "miner-5", + "network_difficulty": 1000005, + "ntime": 1700000005, + "order_key": "05:miner-5", + "p2mr_program_hex": "0505050505050505050505050505050505050505050505050505050505050505", + "share_difficulty": 2, + "share_id": "share-5", + "share_seq": 5, + "template_height": 800000 + }, + { + "accepted_at_ms": 330002, + "credit_policy": null, + "job_id": "job-6", + "job_issued_at_ms": 330001, + "miner_id": "miner-6", + "network_difficulty": 1000006, + "ntime": 1700000006, + "order_key": "06:miner-6", + "p2mr_program_hex": "0606060606060606060606060606060606060606060606060606060606060606", + "share_difficulty": 2, + "share_id": "share-6", + "share_seq": 6, + "template_height": 800000 + } + ] + } + ], + "name": "advance-page-expiry-one-above-weight", + "page_size": 2, + "schema": "qbit-prism-window-pipeline-parity-input/v1", + "snapshot": { + "anchor_job_issued_at_ms": 330000, + "records": [ + { + "accepted_at_ms": 329991, + "credit_policy": null, + "job_id": "job-1", + "job_issued_at_ms": 329981, + "miner_id": "miner-1", + "network_difficulty": 1000001, + "ntime": 1700000001, + "order_key": "01:miner-1", + "p2mr_program_hex": "0101010101010101010101010101010101010101010101010101010101010101", + "share_difficulty": 1, + "share_id": "share-1", + "share_seq": 1, + "template_height": 800000 + }, + { + "accepted_at_ms": 329992, + "credit_policy": null, + "job_id": "job-2", + "job_issued_at_ms": 329982, + "miner_id": "miner-2", + "network_difficulty": 1000002, + "ntime": 1700000002, + "order_key": "02:miner-2", + "p2mr_program_hex": "0202020202020202020202020202020202020202020202020202020202020202", + "share_difficulty": 1, + "share_id": "share-2", + "share_seq": 2, + "template_height": 800000 + }, + { + "accepted_at_ms": 329993, + "credit_policy": null, + "job_id": "job-3", + "job_issued_at_ms": 329983, + "miner_id": "miner-3", + "network_difficulty": 1000003, + "ntime": 1700000003, + "order_key": "03:miner-3", + "p2mr_program_hex": "0303030303030303030303030303030303030303030303030303030303030303", + "share_difficulty": 2, + "share_id": "share-3", + "share_seq": 3, + "template_height": 800000 + }, + { + "accepted_at_ms": 329994, + "credit_policy": null, + "job_id": "job-4", + "job_issued_at_ms": 329984, + "miner_id": "miner-4", + "network_difficulty": 1000004, + "ntime": 1700000004, + "order_key": "04:miner-4", + "p2mr_program_hex": "0404040404040404040404040404040404040404040404040404040404040404", + "share_difficulty": 1, + "share_id": "share-4", + "share_seq": 4, + "template_height": 800000 + } + ] + }, + "window_weight": 3 + }, + "record_jsons": [ + "{\"accepted_at_ms\":330002,\"job_id\":\"job-5\",\"job_issued_at_ms\":330001,\"miner_id\":\"miner-5\",\"network_difficulty\":1000005,\"ntime\":1700000005,\"order_key\":\"05:miner-5\",\"p2mr_program_hex\":\"0505050505050505050505050505050505050505050505050505050505050505\",\"share_difficulty\":2,\"share_id\":\"share-5\",\"share_seq\":5,\"template_height\":800000}", + "{\"accepted_at_ms\":330002,\"job_id\":\"job-6\",\"job_issued_at_ms\":330001,\"miner_id\":\"miner-6\",\"network_difficulty\":1000006,\"ntime\":1700000006,\"order_key\":\"06:miner-6\",\"p2mr_program_hex\":\"0606060606060606060606060606060606060606060606060606060606060606\",\"share_difficulty\":2,\"share_id\":\"share-6\",\"share_seq\":6,\"template_height\":800000}" + ], + "spool_tail": ",\"compact_share_identities\":[[\"miner-5\",\"05:miner-5\",\"0505050505050505050505050505050505050505050505050505050505050505\"],[\"miner-6\",\"06:miner-6\",\"0606060606060606060606060606060606060606060606060606060606060606\"]],\"compact_shares\":[[5,\"share-5\",0,2,330001,330002,null],[6,\"share-6\",1,2,330001,330002,null]]}" + }, + "record_count": 2, + "record_stream_len": 659, + "record_stream_sha256": "7437fbce0a011de987f98327a714f0b770899b8772bfb21e6006cf59c4aa60f0", + "spool_tail_len": 307, + "spool_tail_sha256": "52da41ec514a86bc2ddfed1dcb153ae9ec39c825a2d59c20d0053bfd3a2b85e7", + "why": "page_size 2, weight 3, difficulties (1,1,2,1): the snapshot retains [3,4] at total 3; the advance appends [5,6] of difficulty 2 each (total 7); 7-3 = 4, one above the weight, expires the old page wholesale, and the partial loop keeps both new rows (4-2 = 2 < 3), touched_pages 0" + }, + "advance-page-expiry-one-below-weight": { + "advance_stats": [ + { + "added_rows": 2, + "expired_rows": 1, + "touched_pages": 1 + } + ], + "canonical_bytes_len": 991, + "canonical_bytes_sha256": "5648decf18cbee474703e97c9e955d1aef7615152310671b8404abf96d1a2705", + "canonical_digest": "5648decf18cbee474703e97c9e955d1aef7615152310671b8404abf96d1a2705", + "input_sha256": "a6d72bd900dbd394969df4f28f979ae4dea30a9c65397fe64c69cbf9be9cee36", + "pinned_literals": { + "canonical_bytes": "[{\"accepted_at_ms\":319994,\"job_id\":\"job-4\",\"job_issued_at_ms\":319984,\"miner_id\":\"miner-4\",\"network_difficulty\":1000004,\"ntime\":1700000004,\"order_key\":\"04:miner-4\",\"p2mr_program_hex\":\"0404040404040404040404040404040404040404040404040404040404040404\",\"share_difficulty\":1,\"share_id\":\"share-4\",\"share_seq\":4,\"template_height\":800000},{\"accepted_at_ms\":320002,\"job_id\":\"job-5\",\"job_issued_at_ms\":320001,\"miner_id\":\"miner-5\",\"network_difficulty\":1000005,\"ntime\":1700000005,\"order_key\":\"05:miner-5\",\"p2mr_program_hex\":\"0505050505050505050505050505050505050505050505050505050505050505\",\"share_difficulty\":1,\"share_id\":\"share-5\",\"share_seq\":5,\"template_height\":800000},{\"accepted_at_ms\":320002,\"job_id\":\"job-6\",\"job_issued_at_ms\":320001,\"miner_id\":\"miner-6\",\"network_difficulty\":1000006,\"ntime\":1700000006,\"order_key\":\"06:miner-6\",\"p2mr_program_hex\":\"0606060606060606060606060606060606060606060606060606060606060606\",\"share_difficulty\":1,\"share_id\":\"share-6\",\"share_seq\":6,\"template_height\":800000}]", + "input": { + "advances": [ + { + "anchor_job_issued_at_ms": 320002, + "records": [ + { + "accepted_at_ms": 320002, + "credit_policy": null, + "job_id": "job-5", + "job_issued_at_ms": 320001, + "miner_id": "miner-5", + "network_difficulty": 1000005, + "ntime": 1700000005, + "order_key": "05:miner-5", + "p2mr_program_hex": "0505050505050505050505050505050505050505050505050505050505050505", + "share_difficulty": 1, + "share_id": "share-5", + "share_seq": 5, + "template_height": 800000 + }, + { + "accepted_at_ms": 320002, + "credit_policy": null, + "job_id": "job-6", + "job_issued_at_ms": 320001, + "miner_id": "miner-6", + "network_difficulty": 1000006, + "ntime": 1700000006, + "order_key": "06:miner-6", + "p2mr_program_hex": "0606060606060606060606060606060606060606060606060606060606060606", + "share_difficulty": 1, + "share_id": "share-6", + "share_seq": 6, + "template_height": 800000 + } + ] + } + ], + "name": "advance-page-expiry-one-below-weight", + "page_size": 2, + "schema": "qbit-prism-window-pipeline-parity-input/v1", + "snapshot": { + "anchor_job_issued_at_ms": 320000, + "records": [ + { + "accepted_at_ms": 319991, + "credit_policy": null, + "job_id": "job-1", + "job_issued_at_ms": 319981, + "miner_id": "miner-1", + "network_difficulty": 1000001, + "ntime": 1700000001, + "order_key": "01:miner-1", + "p2mr_program_hex": "0101010101010101010101010101010101010101010101010101010101010101", + "share_difficulty": 1, + "share_id": "share-1", + "share_seq": 1, + "template_height": 800000 + }, + { + "accepted_at_ms": 319992, + "credit_policy": null, + "job_id": "job-2", + "job_issued_at_ms": 319982, + "miner_id": "miner-2", + "network_difficulty": 1000002, + "ntime": 1700000002, + "order_key": "02:miner-2", + "p2mr_program_hex": "0202020202020202020202020202020202020202020202020202020202020202", + "share_difficulty": 1, + "share_id": "share-2", + "share_seq": 2, + "template_height": 800000 + }, + { + "accepted_at_ms": 319993, + "credit_policy": null, + "job_id": "job-3", + "job_issued_at_ms": 319983, + "miner_id": "miner-3", + "network_difficulty": 1000003, + "ntime": 1700000003, + "order_key": "03:miner-3", + "p2mr_program_hex": "0303030303030303030303030303030303030303030303030303030303030303", + "share_difficulty": 2, + "share_id": "share-3", + "share_seq": 3, + "template_height": 800000 + }, + { + "accepted_at_ms": 319994, + "credit_policy": null, + "job_id": "job-4", + "job_issued_at_ms": 319984, + "miner_id": "miner-4", + "network_difficulty": 1000004, + "ntime": 1700000004, + "order_key": "04:miner-4", + "p2mr_program_hex": "0404040404040404040404040404040404040404040404040404040404040404", + "share_difficulty": 1, + "share_id": "share-4", + "share_seq": 4, + "template_height": 800000 + } + ] + }, + "window_weight": 3 + }, + "record_jsons": [ + "{\"accepted_at_ms\":319994,\"job_id\":\"job-4\",\"job_issued_at_ms\":319984,\"miner_id\":\"miner-4\",\"network_difficulty\":1000004,\"ntime\":1700000004,\"order_key\":\"04:miner-4\",\"p2mr_program_hex\":\"0404040404040404040404040404040404040404040404040404040404040404\",\"share_difficulty\":1,\"share_id\":\"share-4\",\"share_seq\":4,\"template_height\":800000}", + "{\"accepted_at_ms\":320002,\"job_id\":\"job-5\",\"job_issued_at_ms\":320001,\"miner_id\":\"miner-5\",\"network_difficulty\":1000005,\"ntime\":1700000005,\"order_key\":\"05:miner-5\",\"p2mr_program_hex\":\"0505050505050505050505050505050505050505050505050505050505050505\",\"share_difficulty\":1,\"share_id\":\"share-5\",\"share_seq\":5,\"template_height\":800000}", + "{\"accepted_at_ms\":320002,\"job_id\":\"job-6\",\"job_issued_at_ms\":320001,\"miner_id\":\"miner-6\",\"network_difficulty\":1000006,\"ntime\":1700000006,\"order_key\":\"06:miner-6\",\"p2mr_program_hex\":\"0606060606060606060606060606060606060606060606060606060606060606\",\"share_difficulty\":1,\"share_id\":\"share-6\",\"share_seq\":6,\"template_height\":800000}" + ], + "spool_tail": ",\"compact_share_identities\":[[\"miner-4\",\"04:miner-4\",\"0404040404040404040404040404040404040404040404040404040404040404\"],[\"miner-5\",\"05:miner-5\",\"0505050505050505050505050505050505050505050505050505050505050505\"],[\"miner-6\",\"06:miner-6\",\"0606060606060606060606060606060606060606060606060606060606060606\"]],\"compact_shares\":[[4,\"share-4\",0,1,319984,319994,null],[5,\"share-5\",1,1,320001,320002,null],[6,\"share-6\",2,1,320001,320002,null]]}" + }, + "record_count": 3, + "record_stream_len": 989, + "record_stream_sha256": "d92a5230d0b1788862ad11cba97c0b9371791c6450450b0daf7aaca069a6bd9b", + "spool_tail_len": 436, + "spool_tail_sha256": "2a0827b55d1e9557f5b3ad76e2d94253712556660ed35c36604f777ba9a00f35", + "why": "page_size 2, weight 3, difficulties (1,1,2,1): the snapshot retains [3,4] at total 3; the advance appends [5,6] of difficulty 1 each (total 5); 5-3 = 2, one below the weight, keeps the old page, and the partial loop expires seq 3 alone (5-2 = 3 >= 3, then 3-1 = 2 < 3), touched_pages 1" + } + }, + "command": "PYTHONDONTWRITEBYTECODE=1 python3 -c 'import json, shlex, sys\nfrom tests import window_pipeline_parity as w\nrec = w.corpus_record\nCase = w.WindowPipelineCase\nAdvance = w.WindowPipelineAdvance\ndef snap(anchor, difficulties):\n return tuple(rec(seq, share_difficulty=d, job_issued_at_ms=anchor - 20 + seq, accepted_at_ms=anchor - 10 + seq) for seq, d in enumerate(difficulties, 1))\ndef delta(anchor, first, difficulties):\n return tuple(rec(seq, share_difficulty=d, job_issued_at_ms=anchor + 1, accepted_at_ms=anchor + 2) for seq, d in enumerate(difficulties, first))\ncases = (\n Case(name=\"advance-page-expiry-at-weight\", why=\"page_size 2, weight 2, four difficulty-1 records: the snapshot retains the one full page [3,4] at total 2; the advance appends the fresh page [5,6] (total 4) and 4-2 = 2 >= 2 expires the old page wholesale exactly at the weight, touched_pages 0. A whole-page expiry using > instead of >= keeps the page, empties it in the partial loop and counts it touched\", anchor_job_issued_at_ms=300_000, window_weight=2, page_size=2, snapshot_records=snap(300_000, (1, 1, 1, 1)), advances=(Advance(anchor_job_issued_at_ms=300_002, delta_records=delta(300_000, 5, (1, 1))),)),\n Case(name=\"advance-page-expiry-at-weight-second-page\", why=\"page_size 2, weight 4, six difficulty-1 records: the snapshot retains pages [3,4],[5,6] at total 4; the advance appends [7,8],[9,10] (total 8); 8-2 = 6 expires [3,4] above the weight, then 6-2 = 4 >= 4 expires [5,6] exactly at the weight, retaining [7,8],[9,10] with touched_pages 0. The exact-weight page is the second retained page, so the whole-page loop must keep going after its first expiry\", anchor_job_issued_at_ms=310_000, window_weight=4, page_size=2, snapshot_records=snap(310_000, (1, 1, 1, 1, 1, 1)), advances=(Advance(anchor_job_issued_at_ms=310_002, delta_records=delta(310_000, 7, (1, 1, 1, 1))),)),\n Case(name=\"advance-page-expiry-one-below-weight\", why=\"page_size 2, weight 3, difficulties (1,1,2,1): the snapshot retains [3,4] at total 3; the advance appends [5,6] of difficulty 1 each (total 5); 5-3 = 2, one below the weight, keeps the old page, and the partial loop expires seq 3 alone (5-2 = 3 >= 3, then 3-1 = 2 < 3), touched_pages 1\", anchor_job_issued_at_ms=320_000, window_weight=3, page_size=2, snapshot_records=snap(320_000, (1, 1, 2, 1)), advances=(Advance(anchor_job_issued_at_ms=320_002, delta_records=delta(320_000, 5, (1, 1))),)),\n Case(name=\"advance-page-expiry-one-above-weight\", why=\"page_size 2, weight 3, difficulties (1,1,2,1): the snapshot retains [3,4] at total 3; the advance appends [5,6] of difficulty 2 each (total 7); 7-3 = 4, one above the weight, expires the old page wholesale, and the partial loop keeps both new rows (4-2 = 2 < 3), touched_pages 0\", anchor_job_issued_at_ms=330_000, window_weight=3, page_size=2, snapshot_records=snap(330_000, (1, 1, 2, 1)), advances=(Advance(anchor_job_issued_at_ms=330_002, delta_records=delta(330_000, 5, (2, 2))),)),\n)\nadapter = w.PythonWindowPipelineAdapter()\nentries = {case.name: w.reference_case_entry(case, adapter.run(case)) for case in cases}\nrefused = sorted(name for name, entry in entries.items() if \"rejected\" in entry)\nrefused and sys.exit(\"supplementary case rejected by the oracle: \" + \", \".join(refused))\ncommand = \"PYTHONDONTWRITEBYTECODE=1 python3 -c \" + shlex.quote(sys.orig_argv[-1]) + \" > supplementary.json\"\ndocument = {\"schema\": \"qbit-prism-window-pipeline-parity-supplementary/v1\", \"source_commit\": \"504846cc0b72e8f86ed17f896d4ccbbe196a31dc\", \"command\": command, \"cases\": entries}\nsys.stdout.write(json.dumps(document, indent=2, sort_keys=True) + \"\\n\")' > supplementary.json", + "schema": "qbit-prism-window-pipeline-parity-supplementary/v1", + "source_commit": "504846cc0b72e8f86ed17f896d4ccbbe196a31dc" +} diff --git a/tests/perf/b275_delta_read.md b/tests/perf/b275_delta_read.md new file mode 100644 index 000000000..a90ef61b5 --- /dev/null +++ b/tests/perf/b275_delta_read.md @@ -0,0 +1,210 @@ +# Bounded delta acquisition qualification (#275) + +## Timeline repair: 2026-09-21, pool 16 + +The timeline-fenced candidate passes the predeclared usefulness gate: median +paired last-client improvement **937.311ms (34.66%)**, exceeding both **100ms +and 5%** and the observed A/A noise. **Every 400k case still misses one second; +#275 remains open.** This supersedes the original candidate's performance and +correctness claim below. The original leaf/count proof had a confirmed physical +failover defect; its 30.11% result is not evidence for this repaired binary. + +The repair reads current WAL insertion timeline in both existing leaf witnesses. +Real PostgreSQL 16 tests retain the same frontend pool through suffix and interior +history replacement, then verify rewind or base-backup rejoin and re-promotion. +They authenticate full-fallback Snapshot bytes and WindowRef against the full +reader. Healthy pooled sessions and reconnects retain eligibility, including the +witness query under a non-superuser read role. The 10 focused tests and 27 existing +coordinator tests pass without weakening old assertions. The existing +[D3/D5 boundaries and timeline limitation](../../docs/refresh-window-split.md#bounded-delta-acquisition-275) +remain explicit; this does not claim a globally unique identity for sibling copies. + +Both binaries were newly built from frozen source with the same measurement +harness, changed only to use the normal **16 database connections** per frontend +instead of the test helper's 4. Runtime base is +`d78659de608ad30645ba630f15eb8be998b3e42a`; candidate is `230e536` plus this timeline +repair. Release optimization 3, thin LTO, one codegen unit and no debug assertions +were verified. ELF SHA-256: + +- Base: `71916e14d14d2a67db856c15e76df816c10bccbc7348ad3ddfce1353aefeb2cc` +- Candidate: `8d7c48b7c6386de48964de6fcd0785469cd637f15f9af547fb136839038e2930` + +An initial candidate build reused the base executable from the shared Cargo +cache. Equal ELF hashes exposed it before measurement; it was rejected, the +server release package was cleaned, and the candidate rebuilt. Both accepted +artifacts report fresh compilation; the timeline SQL exists only in the candidate. + +Each case used one frontend, 2000 actual sessions, exactly 2 Tokio workers and a +fresh 400000-share realistic WindowPlan schema on durable local PostgreSQL 16. +The timing boundary and reconciliation below are unchanged. Heavy work serialized +under one host lock. Small 16-share and 400k warm AB pairs preceded two A/A controls +using the same base executable, then four matched AB/BA/AB/BA pairs. + +| Pair, order | Base last client, s | Candidate last client, s | Gain, ms | Paired reduction | +| --- | ---: | ---: | ---: | ---: | +| 1, AB | 2.691942386 | 1.775046123 | 916.896 | 34.06% | +| 2, BA | 2.736155785 | 1.781973938 | 954.182 | 34.87% | +| 3, AB | 2.710566999 | 1.767791274 | 942.776 | 34.78% | +| 4, BA | 2.697804295 | 1.765957482 | 931.847 | 34.54% | + +The separate base/candidate last-client medians are **2.704185647s** and +**1.771418699s**. The median of the four within-pair gains is 937.311ms and the +median of their percentage reductions is 34.66%. A/A pairs measured +2.718429582/2.704936774s and 2.730774289/2.698644977s; maximum within-pair variation +was **32.129ms (1.18%)**. The acceptance gate requires both absolute and relative +improvement beyond that observed noise, in addition to 100ms/5%. + +All 16 cases reconcile 32000 received identities with durable children, with +exact receipt quantiles, no unmatched rows, no delivery errors, no supervisor +stops and no residual schemas. Small 16-share results are 0.552970908/0.551639463s; +excluded 400k warm results are 2.772694633/1.787759003s. Measured primary-pair process +peak RSS ranges 809115648–809922560 bytes for base and 804626432–805101568 for +candidate, excluding PostgreSQL. Minimum available memory across the 16 cases +was 31060910080 bytes, with no additional swap; sampled process-envelope aggregate +CPU idle was above 92.5% for the 400k cases, which cannot exclude brief interference. + +This is one local synthetic-node frontend, not a production eligibility estimate, +500k or two-frontend qualification, real-network result, or a 24-hour memory soak. +The timeline candidate's fallback correctness is tested; its fallback latency +was not separately requalified. The historical split-leaf control below belongs +to the original binary. Raw receipts, source manifests and failed-build evidence +remain private verification artifacts. Independent delta review remains required. + +## Historical original candidate: 2026-09-18, pool4 (superseded) + +The candidate passed its predeclared **15% median improvement screen**, with +**30.11%** lower end-to-end last-client latency across four balanced 400k pairs. +**Every 400k case still missed one second.** This is a bounded acquisition +candidate; #275 remains open. The [acquisition proof and fallback rules](../../docs/refresh-window-split.md#bounded-delta-acquisition-275) +explain its deliberately restricted applicability. + +## Source and method + +Exact base: `d78659de608ad30645ba630f15eb8be998b3e42a`. The candidate is the +original implementation at `782caf2aa24bf9dbc1ce7673b9f3a6d23a2817a8`. Production source hashes +were frozen before building and remained unchanged through qualification; +subsequent edits extended tests and documentation only. Both used Rust 1.98.1, +x86_64 Linux, release optimization 3, thin LTO and one codegen unit, with at most +two Cargo build jobs. Primary ELF SHA-256 values: + +- Base: `7d052237f6343cbec2a6ccd9a966fbd73753772db423bd5b339ffa7e892c924d` +- Candidate: `036da06c4d93d53464913e803a0ef0a2e472184ec7218be01d0570572a575406` + +Runs on 2026-09-18 used the existing `b275_persistence_measure` harness, +**2,000 sessions on one frontend and exactly two Tokio workers**. Each case had a +fresh matched PostgreSQL schema, the same five-recipient `WindowPlan`, 400,000 +preloaded shares, checked first/middle/last rows and exact prepared-window size. +Initial refresh and client login were outside timing. The timed synthetic parent +change advances the payout revision, so it invalidates as-is reuse and exercises +fresh acquisition even with zero newly appended shares. Existing native hashes, +prepared storage, batch persistence and client delivery all remain in the clock. + +Timing begins immediately before concurrent refresh polling and ends at the last +client's decoded fresh `mining.notify` with the expected parent and clean flag. +The unchanged 120-second receive and 240-second body deadlines are not the +one-second target. Source fixtures still use loopback sockets, a synthetic node, +a small empty-transaction template, deterministic old shares and no automatic +refresh polling. All runtime, frontend, admission and collector settings were +identical between base and candidate; no runtime-worker experiment is included. + +The host was a KVM x86_64 guest exposing 22 CPUs on an AMD Ryzen 9 9955HX. +A dedicated disposable PostgreSQL 16.15 primary used `fsync=on`, +`full_page_writes=on`, `synchronous_commit=on`, `wal_sync_method=fdatasync`, +128 MB shared buffers, 100 connections, 4 MB work memory, 1 GB maximum WAL, +300-second checkpoint timeout and zero parallel workers per gather. Heavy work +was serialized by one host lock. Other IDE/OS activity remained; samples about +0.5 seconds apart cannot rule out brief interference. Minimum available host +memory across all cases was 23,788,679,168 bytes. RSS below is the whole test +process high-water mark, excluding PostgreSQL, not a phase allocation counter. + +## Complete observed results + +Small functional cases preceded 400k warm AB, then measured AB / BA / AB / BA. +A means base and B means candidate. Last-client seconds are rounded to nine +places. Only the four measured pairs enter the primary median. + +| Case | Base seconds | Candidate seconds | Reduction | Base peak RSS bytes | Candidate peak RSS bytes | +| --- | ---: | ---: | ---: | ---: | ---: | +| 16 shares, functional | 0.798950340 | 0.804595039 | — | 99,598,336 | 100,769,792 | +| 400k warm | 2.854928298 | 1.957876016 | — | 807,112,704 | 803,917,824 | +| 400k pair 1, AB | 2.879785824 | 2.076545193 | 27.89% | 807,530,496 | 802,754,560 | +| 400k pair 2, BA | 2.912904581 | 2.048605249 | 29.67% | 807,841,792 | 802,942,976 | +| 400k pair 3, AB | 2.914530941 | 2.001175847 | 31.34% | 807,755,776 | 803,192,832 | +| 400k pair 4, BA | 2.881689705 | 1.993829557 | 30.81% | 808,058,880 | 803,479,552 | +| **Measured median** | **2.897297143** | **2.024890548** | **30.11%** | — | — | +| 400k rollback-gap control, AB | 2.895233627 | 2.077325020 | 28.25% | 808,415,232 | 802,656,256 | +| 400k split-leaf fallback control, BA | 2.969597360 | 3.011332568 | -1.41% | 806,576,128 | 807,710,720 | + +All 16 cases completed and reconciled **32,000 unique timed notifications** to +32,000 durable child identities, with no unmatched issued rows. The fixture +checks parent, payout revision, prepared key, frontend, authenticated username, +extranonce1/extranonce2 size and live expiry against stored payloads. An external +artifact checker independently rejoined every received ID and payload identity +and recomputed receipt quantiles. All fixture schemas were removed; no process +supervisor limit fired. The supervisor bounded each process to 600 seconds, +8 GiB sampled RSS, at least 6 GiB available memory and no more than 512 MiB +additional swap. Those limits supplement the unchanged harness deadlines. + +The primary candidate's observed RSS was slightly lower in every measured pair. +The implementation retains one moved window plus one SQL page, trims before +append and releases excess vector capacity after a large retirement. This is a +structural bound plus short observations, not a 24-hour memory soak. + +## History controls and applicability + +The final count proof permits ordinary sequence holes. The gap control invokes +the real revision-gated append with a false commit gate, verifies the rollback, +then appends one valid share and verifies the two-slot sequence advance. It does +this before initial acquisition, retaining a 400k suffix. Thus a persistent +normal rollback gap remains eligible on the timed revision-only refresh; +focused differential tests separately cover appended-delta acquisition itself. + +The split control seeds the same 400k history across two native attached leaves +instead of one. Its candidate falls back to the full read. The single pair +observed 1.41% extra latency and 0.14% extra RSS; fallback overhead exists, and +this control establishes neither general non-regression nor a speedup. + +These controls were external test-only copies of the same harness. The base and +candidate received identical fixture patches; production source stayed unchanged. +Their ELF SHA-256 values were respectively +`8702381e403f4bee676921ffa35141e4e64927b8030963bcfbff3225847aa2c5` and +`ed6bdc748bbfbabe4e46198585b0616fb14aa1aeddadab9ef1e0e34e262a56c9`. +An initial candidate control build incorrectly reused the base artifact through +a shared Cargo cache; identical hashes exposed it before any control ran. +That artifact was rejected, the server package was explicitly cleaned, and the +candidate was rebuilt from its own recorded source path before measurement. + +Single-leaf proof and the final eligible-row count prevent endpoint-only reuse +across an interior partition hole or newly eligible/retroactively inserted row. +The count is still O(window) metadata work. Cross-partition windows, detach or +restore, absent evidence, shared cancelled ownership, short history, retargets, +regressed anchors/cutoffs and more than 4,096 delta sequence slots use full reads. +A default leaf spans 16,777,216 slots, but actual production eligibility rates +were not measured. No broad deployment policy or new database schema is added. + +## Reproduction and limits + +Build the exact base and candidate into separate output directories, then use a +dedicated durable PostgreSQL 16 database and run each binary serially: + +```sh +CARGO_BUILD_JOBS=2 cargo test -p qbit-prism-server --release \ + --test b275_persistence_measure --no-run +PRISM_TEST_DATABASE_URL="$TEST_DATABASE_URL" \ +PRISM_TEST_REQUIRE_INTEGRATION=1 \ +PRISM_B275_WINDOW_SHARES=400000 PRISM_B275_FRONTEND_ORDER=1 \ + "$MEASUREMENT_BINARY" measure_2000_sessions_one_and_two_frontends \ + --ignored --exact --test-threads=1 --nocapture +``` + +Keep fixed settings and fresh schemas; warm AB, then alternate AB/BA across at +least three pairs. Archive binaries, source hashes, raw received/durable payloads, +process resource samples and every failed attempt. Compare last-client medians, +not summed overlapping histograms. The two history controls require the fixture +modifications described above; they are not new runtime environment controls. + +This screen does not establish the one-second goal, 500k performance, two-frontend +non-regression, multiple processes/hosts, AArch64 results, real node/network cost, +share submission throughput, arbitrary recipient cardinality, cold start or +production retention frequency. Authority/settlement lock policy is unchanged. +Independent Tier 3 review remains required before merge. diff --git a/tests/perf/b275_persistence_measure.md b/tests/perf/b275_persistence_measure.md new file mode 100644 index 000000000..f7294b105 --- /dev/null +++ b/tests/perf/b275_persistence_measure.md @@ -0,0 +1,162 @@ +# Job persistence measurements (#275) + +**The one-second target was missed in all six 2,000-session cases.** +Two-frontend non-regression and production-shaped qualification are not established. +Issue #275 remains open; this PR adds measurement and regressions, not runtime +optimization. + +## Tested revision and environment + +- Measured harness: `3d4775cacc8c6f8376df0ce028d1083bda738746`. + Runtime baseline: frozen PR #397, `1b0d409344c1b99f5f4bd04970890b9b034a9eeb`. +- Three paired runs on 2026-09-15, 22:03:44–22:04:16 UTC, using the existing + compiled Cargo test binary: unoptimized with debug information, Rust 1.98.1. +- Apple M4 Max (Mac16,6), 14 logical CPUs, 36 GiB, macOS 26.5.1. +- Dedicated disposable PostgreSQL 16.14 primary: `fsync=on`, + `full_page_writes=on`, `synchronous_commit=on`, checked before and during + each case. No production or testnet endpoint was used. +- Two Tokio workers for all frontends and clients in one process. Each frontend + had two build workers, four database connections and its own 128-permit + job-build admission semaphore. Build and persistence each had a 30 s timeout; + writes had a 20 s timeout. Thus two frontends had 256 aggregate admission + permits and eight database connections. Vardiff was disabled. +- Process descriptor limit: 16,384; kernel per-process ceiling: 138,240. + Roughly 4,000 TCP descriptors plus headroom are needed. +- Runtime/refresh authors coordinated the window; a separate compiler was + allowed to finish. No compiler or test process was found before any repetition. + Pre-run CPU idle was 74.15%, 75.66%, 77.68%. Background apps remained; memory + pressure reported 39–46% free with about 17.4 GiB of existing swap use. + This was a shared development Mac, not an isolated benchmark machine. +- Sixteen deterministic accepted shares, five payout recipients, empty transaction + template and a synthetic local node. Coordinators, public TCP Stratum, + prepared records and durable issued-job persistence were the real runtime. + +## Boundary and results + +Every client subscribed, authorized and received initial work before timing. +The common monotonic clock started immediately before concurrent +`refresh_once` polling after changing the synthetic parent, and stopped for +each client when it decoded a fresh new-parent `mining.notify` with +`clean_jobs=true`. The bracket includes refresh, construction, persistence, +scheduling and TCP delivery; it excludes login and automatic polling delay. +All clients shared one 120 s delivery deadline; each topology had a 240 s +case budget. Client decoding ran through one test task and is included. + +Each pair ran 2,000 sessions on one frontend first, then 1,000 on each of two +frontends, with fresh schemas and full cleanup between topologies. + +| Pair | Frontends | Lower median (s) | Maximum (s) | Refresh returns (s) | Received / durable | +| --- | ---: | ---: | ---: | --- | --- | +| 1 | 1 | 0.829421 | 1.499653 | 0.020061 | 2,000 / 2,000 | +| 1 | 2 | 1.187969 | 1.710496 | 0.039934 / 0.047239 | 2,000 / 2,000 | +| 2 | 1 | 0.815554 | 1.428409 | 0.020274 | 2,000 / 2,000 | +| 2 | 2 | 0.881717 | 1.358960 | 0.023731 / 0.024136 | 2,000 / 2,000 | +| 3 | 1 | 1.029605 | 1.648645 | 0.023670 | 2,000 / 2,000 | +| 3 | 2 | 0.942327 | 1.412927 | 0.025471 / 0.029112 | 2,000 / 2,000 | + +All three invocations exited zero, emitted both required-scale measurements and +durability confirmations, and recorded the explicit integration gate. There were +no incomplete cases or test failures. Cleanup left no measurement schemas and the +disposable primary was stopped. Test-process peak RSS per pair was 104,742,912, +104,579,072 and 103,120,896 bytes, excluding PostgreSQL. + +Two/one ratios of maximum delivery were 1.141, 0.951 and 0.857; median maxima were +1.500 s and 1.413 s. Mixed observations, fixed ordering, three pairs and no +predeclared tolerance do not establish a speedup or non-regression. No target +or deadline was relaxed. + +## Limits + +This is a **debug-profile, 16-share synthetic-node fanout baseline**. +Full payout windows, transaction-heavy templates, miner submissions, real-node +latency, remote networks, separate frontend hosts/processes and automatic tip +polling delay remain unmeasured. + +Frozen v1 recorded eventual delivery but did not record internal failed/retried +attempts or committed rows never delivered. Both remain **unknown** for these +six observations; later harness fixes do not retroactively establish zero. +Commit duration is also unknown (`commit_seconds=null`). Pool and advisory-lock +histogram sums overlap across concurrent work and are not additive wall time +or a way to infer commit cost. A separate three-second lock stub measured +3.003617 s persistence on the original smoke run; it does not establish +lock-free persistence or simulate actual landing work. + +## Reproduction + +Use a clean checkout of the measured harness SHA. Set `PRISM_TEST_DATABASE_URL` to a dedicated +disposable PG16 primary with the three durability settings above; coordinate +other builds/tests and inspect resource limits first. For each of three +repetitions, set `manifest` to a fresh output file, then run: + +```sh +ulimit -n 16384 +PRISM_TEST_REQUIRE_INTEGRATION=1 \ +PRISM_TEST_GATE_MANIFEST="$manifest" \ +cargo test --locked -p qbit-prism-server --test b275_persistence_measure \ + measure_2000_sessions_one_and_two_frontends \ + -- --ignored --exact --test-threads=2 --nocapture +``` + +The original runner directly invoked the existing test-profile binary +`target/debug/deps/b275_persistence_measure-696abc204732aa70` with the same test +arguments and inherited disposable database URL and descriptor limit. That +hash-suffixed path is evidence of the original invocation, not a portable path. +Preserve stdout JSON, exit status, manifests, tested SHA and host/configuration +for any new measurement. Original evidence is in +[PR #402's commit history](https://github.com/Qbit-Org/qbit-mining-bootstrap/pull/402/commits), +at `32f13fe766542155aa4ae7722d6f7ce34ddff4b8`; the current tree contains this +standalone report and reusable tests. + +## Integration after the historical run + +The branch incorporates PR #397 through +`67e24b0dc26e8982b6d8d0b4925d8b82436701bd`, including explicit `ANALYZE` after +bulk fixture loading for the separate 500k qualification. Its 25 s resume budget +and correctness assertions remain unchanged. The six measurements above still +belong only to `3d4775c`; they were not rerun or reattributed to this integration. +That integrated harness used schema `b275.delivery.v3` and named the old reference +`historical_baseline_sha`, so it cannot be mistaken for the tested runtime. +Record the actual build revision in each new run's manifest. Timing boundaries, +session counts, deadlines and acceptance checks were unchanged by that integration. +The current `b275.delivery.v4` controls, p95 definition, run-scope/child-receipt +records and 400k/500k reproduction commands are documented in the +[post-batching qualification](b275_post_batch_qualification.md). Its later +observations do not replace or reattribute the historical rows above. + +On 2026-09-16 the integrated source passed 23 reconciliation tests (all 11 +database gates executed) and seven harness tests (all four gates executed) on +disposable PG16. The retry control recovered eight clients after two failed +attempts and reported `complete=false`. The release-profile 500k qualification +also passed: refresh 5.251 s, shared resume 4.848 s, four waiters reading exactly +500,000 rows across 123 pages, against the unchanged 25 s resume budget. +These are separate integration checks, not new 2,000-session measurements or +production qualification. + +## Revised-harness validation + +Four independent source-review lanes identified reporting and test-strength +gaps. Follow-up harness changes report effective configuration and listener +attempt counters, require no failed attempts for a clean result, count unmatched +issued rows, exercise the compact-issued revision fence during its own lock wait, +scope waiter attribution, share the case deadline and unify fixture cleanup. +These are separate from the frozen v1 scale results above. + +Before that upstream integration, the revised disposable-PG16 suite passed +**7 tests, 0 failures, 1 intentionally +ignored scale test**, including **4/4 required integration gates**. A transient +issued-row rejection regression recovered all eight clients after eight failed +attempts and correctly emitted `complete=false`. Ordinary small deliveries had +zero failed attempts and unmatched rows. Formatting and target-scoped Clippy +with warnings denied passed. + +Final CI exposed an unchanged commit-reconciliation test clock anchored before +proof construction/submission. A controlled 250 ms setup delay reproduced the +same late-confirmation count failure (zero instead of one). Anchoring the same +350 ms hold after observed append entry passed all **12 reconciliation tests**, +with the 200 ms share bound, 1,000 ms candidate bound, late-ACK and candidate +retention assertions unchanged. This is a test-only correction; payout/runtime +behavior is unchanged. Independent delta review also passed all 23 tests matching +`commit_reconcile`, reproduced the failing old-clock control, and verified that +the six historical rows and unknown outcomes survived report pruning. +Current CI and sanitized review dispositions are tracked +on [PR #402](https://github.com/Qbit-Org/qbit-mining-bootstrap/pull/402). diff --git a/tests/perf/b275_post_batch_qualification.md b/tests/perf/b275_post_batch_qualification.md new file mode 100644 index 000000000..713a2efb8 --- /dev/null +++ b/tests/perf/b275_post_batch_qualification.md @@ -0,0 +1,256 @@ +# Post-batching large-window delivery (#275) + +**All 20 cases missed the unchanged one-second target.** Every case delivered +and reconciled exactly 2,000 timed jobs: **40,000 durable matches**, zero failed +delivery attempts, zero refresh/receive errors and zero unmatched issued rows. +Four cases are warmups; the other 16 are four balanced pairs at each window size. +Two-frontend non-regression remains unestablished, and #275 remains open. + +## Source, clock and fixture + +Measured source: `e4b3c9c9bac8e6c86323c85d71dee423a529db4a`, a measurement-only +change over runtime `3bb2a2dec917a46ea6e90be8e2ee55078bbb4d5c` (#435), including +`58b004fc` (#436) and `30f42150` (#437). All 20 cases used the same source and +release binary, SHA-256 +`3da16fe72e08f53c805acc0dabd417c16eb35937bd98afd65160f0c21dec8269`. +Rust/Cargo 1.98.1, aarch64-apple-darwin, optimization 3, no debug assertions or +overflow checks, thin LTO, one codegen unit; build concurrency was two. + +The existing harness used only 16 shares. These runs use its existing +`WindowPlan` at 400,000 and 500,000 shares, first/middle/last row round trips, +`ANALYZE`, and exact prepared-window share-count checks before and after refresh. +Modeled native serialization totals are 234,288,895 and 292,888,895 bytes +(about 586 bytes/share), **not measured SQL storage bytes**. Five recipients, +one payout order key, scaled share difficulty, old deterministic accepted +shares, an empty transaction template and a synthetic local node remain. +Bulk fixture loading bypasses share submission: this is not D1 throughput. + +Each case gets a fresh schema, initial full-window refresh, then all client +logins and initial work before timing. No new shares or balances arrive before +the timed parent change, but that change itself advances `payout_revision` in +the [frozen chain-observation path](https://github.com/Qbit-Org/qbit-mining-bootstrap/blob/e4b3c9c9/crates/qbit-prism-server/src/ledger/window.rs#L489). +The [frozen reuse predicate](https://github.com/Qbit-Org/qbit-mining-bootstrap/blob/e4b3c9c9/crates/qbit-prism-server/src/coordinator/refresh_window.rs#L20) +requires revision equality, so the 600-second snapshot interval cannot preserve +the old retained window across this transition. The refresh therefore captures +a fresh snapshot, including its SQL scan and native share digest, before +building the new work. This corrects the earlier retained-window-reuse inference; +the frozen runs exported no cache-hit or phase measurements, and later diagnostic +traces are not measurements of these 20 cases. +This qualifies the **window size of a preloaded-window refresh/rebuild and +delivery workload**, with initial work already issued. It does not qualify +cold-database startup, changing-share windows, recipient cardinality, large +templates, real-node/network latency, separate frontend hosts, full HA or #291. + +The original common Tokio monotonic `Instant` starts immediately before +concurrent `refresh_once` polling after the synthetic parent switch and ends +per client at decoded fresh new-parent `mining.notify` with `clean_jobs=true`. +It includes construction, persistence, scheduling and client decoding; login +and automatic polling delay are excluded. The shared 120-second delivery +bound and 240-second body deadline are unchanged; neither replaces the +**one-second target**. Setup remains outside that body deadline. + +Every row below has 2,000 total sessions: 2,000 on one frontend (A), or 1,000 +each on two (B). Both topologies share one process and two Tokio workers. +Per frontend: two build workers, four PG connections, 128 job admission +permits, connection capacity 2,000, build/persist phase timeout 30s, socket +write timeout 20s, job retention 300s and vardiff disabled. Thus B doubles +aggregate connections/admission. Template/submit freshness, snapshot interval +and health timeout are 600s; no automatic refresh loop runs. The landed +collector settings are unchanged: 128 admitted entries, at most 64 children +per batch, one active batch through cleanup and 1ms collection dwell. + +## Every observation + +Cases ran serially on 2026-09-17, 15:25:41–15:50:15 UTC. Order is shown below: +400k warm AB then AB/BA/BA/AB; 500k warm BA then BA/AB/AB/BA. +This timestamp span includes supervisor preflight and 308.957s of gaps between +case envelopes, including 111.575s before 500k-p1-B. It is not the sum of test +process runtimes: `/usr/bin/time` recorded 19.08–76.05s per process, while the +supervisor's launch-to-observed-exit `elapsed_seconds` ranged 19.508–76.403s. +The JSON `start_utc` precedes preflight snapshots; `elapsed_seconds` starts +immediately before subprocess launch. All timings come from the retained logs +and case metadata, without another measurement run. +All rows completed and passed durability checks; **every row failed 1s**. +Seconds are rounded to six decimals. p50 preserves the historical lower +median; p95 is nearest rank (receipt index 1,899 of 2,000). + +| Window | Run | FEs | p50 s | p95 s | Maximum s | Refresh returns s | +| --- | --- | ---: | ---: | ---: | ---: | --- | +| 400k | warm-A | 1 | 3.230278 | 3.322110 | 3.326659 | 2.991360 | +| 400k | warm-B | 2 | 3.365928 | 3.426246 | 3.430160 | 3.163825 / 3.143013 | +| 400k | p1-A | 1 | 3.278070 | 3.362290 | 3.366513 | 3.041561 | +| 400k | p1-B | 2 | 3.440843 | 3.501393 | 3.505879 | 3.191590 / 3.192437 | +| 400k | p2-B | 2 | 3.514601 | 3.593961 | 3.597561 | 3.236020 / 3.239192 | +| 400k | p2-A | 1 | 3.356623 | 3.444127 | 3.448638 | 3.130142 | +| 400k | p3-B | 2 | 3.711224 | 3.769376 | 3.774574 | 3.453061 / 3.445235 | +| 400k | p3-A | 1 | 3.570553 | 3.673386 | 3.680419 | 3.315210 | +| 400k | p4-A | 1 | 3.395292 | 3.489196 | 3.492650 | 3.150861 | +| 400k | p4-B | 2 | 3.900819 | 3.962439 | 3.966664 | 3.618343 / 3.617821 | +| 500k | warm-B | 2 | 4.611609 | 4.699999 | 4.706349 | 4.375152 / 4.350348 | +| 500k | warm-A | 1 | 4.235811 | 4.364833 | 4.371359 | 3.968625 | +| 500k | p1-B | 2 | 4.940862 | 5.015358 | 5.018577 | 4.610626 / 4.611636 | +| 500k | p1-A † | 1 | 4.003933 | 4.105645 | 4.109549 | 3.758087 | +| 500k | p2-A | 1 | 3.974786 | 4.066470 | 4.070260 | 3.716437 | +| 500k | p2-B | 2 | 5.303104 | 5.421429 | 5.422711 | 4.990267 / 4.989880 | +| 500k | p3-A | 1 | 4.636992 | 4.738320 | 4.744466 | 4.344059 | +| 500k | p3-B | 2 | 4.551504 | 4.622688 | 4.627072 | 4.305201 / 4.306776 | +| 500k | p4-B | 2 | 4.613439 | 4.672259 | 4.675732 | 4.373455 / 4.374039 | +| 500k | p4-A | 1 | 4.214351 | 4.335089 | 4.340323 | 3.947809 | + +Two/one maximum ratios for pairs 1–4 were **1.041, 1.043, 1.026, 1.136** at +400k and **1.221, 1.332, 0.975, 1.077** at 500k. These are observations from +colocated frontends on a shared host, with no predeclared non-regression +tolerance. They establish neither a general speedup nor non-regression. +The earlier 340/297ms small-fixture pair and historical debug/release reports +are different workloads/revisions and are not controlled comparison baselines. + +After timing, each unique delivered ID matched a durable child with the +expected parent, payout revision, prepared key, frontend instance, authenticated +username, subscribed extranonce1/extranonce2 size and equal live SQL/payload +expiry. Issued-row counts equaled timed deliveries, and the decoded prepared +record remained unchanged. This checks identity/storage fidelity and liveness, +not TTL-policy correctness or cross-process search-space uniqueness. Raw +receipts retain client identities and durable payloads. A separate artifact +check recomputed every quantile and matched all 40,000 client/payload identities; +the SQL predicates were checked inside the test before schema cleanup. + +## Host, storage and contention + +Apple M4 Max (14 logical CPUs), 36 GiB RAM, macOS 26.5.1 (25F80), descriptor +limit 16,384. Dedicated disposable PG 16.14 at `127.0.0.1:55475/postgres`: +`fsync=on`, `full_page_writes=on`, `synchronous_commit=on`, +`wal_sync_method=open_datasync`, `shared_buffers=128MB`, `max_connections=100`, +`work_mem=4MB`, `maintenance_work_mem=64MB`, `wal_compression=off`, +`max_wal_size=1GB`, checkpoint timeout 300s. PG durability settings were verified +by every fixture. RPC and Stratum used loopback ephemeral ports; raw fixture +records contain RPC URLs, while exact Stratum ports were not retained. + +The coordinator held other heavy validation. This was still a **shared host**: +GUI applications, static review/artifact readers and idle unrelated Rust +services remained. Before-case one-minute load ranged 4.57–12.66 and the second +one-second CPU sample was 52.85–78.77% idle. During-case memory-free observations +were 32–51%; reported swap ranged 15,856–17,752 MiB, mostly decreasing. +OS maximum test RSS was 794–1,746 MiB at 400k and 975–2,026 MiB at 500k, +excluding PostgreSQL. Full process/memory/swap snapshots were taken about every +0.5s plus command overhead; subsecond interference is not excluded. + +**† 500k-p1-A has observed formatter activity.** One process sample at +15:43:10.065 UTC saw Cargo with parent `cargo-fmt`, at 1.9%/0.9% sampled CPU. +It occurred 35.773s after subprocess launch, following the case's preflight +(metadata start 15:42:31.232 UTC, launch clock about 15:42:34.292 UTC). +The fixture load alone took 53.093s, so the formatter observation +preceded delivery timing. The case is retained and labeled; there is no claim +of guaranteed isolation or a quiet-host speedup. No rustc or other test binary +was found in these snapshots. Six unrelated `ord`/`qord` services were present +with sampled CPU 0.0%; that does not prove zero unsampled activity. + +Repeated full fixture loading caused frequent WAL-triggered checkpoints, +including PostgreSQL warnings about 21–24s checkpoint spacing. This is real +own-database storage load under the recorded default WAL budget. No checkpoint +was hidden or configuration changed mid-series. Exact checkpoint overlap with +the monotonic delivery bracket was not observed. + +Successful-call histogram ranges below cover the four measured repetitions +per topology and sum across frontends. These overlapping concurrent call-seconds +are **not additive wall time, commit duration, lock hold time or per-job latency**. + +| Window / FEs | Pool wait sum s (calls) | Settlement wait sum s (calls) | +| --- | --- | --- | +| 400k / 1 | 63.940–72.316 (8,046–8,047) | 0.001331–0.001506 (39–40) | +| 400k / 2 | 116.481–128.786 (8,062) | 0.009387–0.017946 (48) | +| 500k / 1 | 70.877–80.756 (8,046–8,047) | 0.001441–0.001796 (39–40) | +| 500k / 2 | 110.648–151.449 (8,062–8,063) | 0.009555–0.032117 (48–49) | + +Across the eight measured 400k repetitions, excluding the two warmups, the +latest refresh return accounts for 90.0–91.5% of final-client elapsed time; +the residual is 0.313–0.365s, not an exclusive fanout phase. +The ordinary observed settlement acquisition waits do not explain the seconds +before refresh returns. Snapshot scanning, native share digesting and full-window +build/hash work are source-supported candidates, not measured phase attribution. +Those phases, executor scheduling, prepared SQL and commit remain unsplit in the +frozen data; `commit_seconds` is explicitly null. +No lock-removal optimization follows from these observations alone. + +A separate supervisor stopped on 8 GiB sampled test RSS, less than 15% host +memory free, more than 512 MiB additional swap, observation failure or 600s +whole-process time. No limit fired and no scale was reduced. This cap does +not replace the original deadlines; standalone harness setup remains unbounded. +A supervisor stop requires checking retained schemas and stopping the owned +disposable primary, since killed processes cannot guarantee Rust teardown. +Every timed case confirmed zero remaining measurement schemas. +Final regression cleanup again found zero schemas and stopped the owned primary. + +## Acceptance and review + +| #275 requirement | Status from this task | +| --- | --- | +| 2,000 sessions within 1s; second frontend does not increase time | Not met / non-regression unestablished; every sample retained above | +| Persistence avoids a 3s settlement-lock stub | Not met: unchanged release control waited 3.003873s with an observed waiter | +| Original payout-revision fence survives lock wait | Existing direct compact-issued regression passed; named cross-frontend readiness test not rerun here | +| Concurrent new row-lock policy interleaving | No row-lock policy change authorized or implemented; not newly qualified | +| Lock topology documented | Existing topology retained; no lock independence claimed | +| Durable timed-notification reconciliation | 40,000 exact child matches, zero failed attempts/unmatched rows | + +Frozen-source smoke passed 7 tests and 4/4 DB gates. Its intentional retry +control recovered 8 clients after 8 failed attempts and correctly retained +`complete=false`; their durable identities also matched. The two ordinary +8-session smoke maxima were 0.019162s and 0.024091s, not scale evidence. + +Independent medium and thermo Opus, Fable adversarial and Sol adversarial +reviews were static during timing. Required reporting fixes landed separately +at `8fb465f4`: when validation finishes within the unchanged 240s body deadline, +incomplete delivery remains the primary error even when validation fails, +other frontends are still checked, received identities survive failed SQL +validation, and delivered-subset verification cannot masquerade as complete +success. Run/progress envelopes explicitly name requested/completed topologies, +position and control sources without renaming gates; non-Unicode controls fail. +Received-subset quantiles remain labeled and include population counts. +If the body deadline expires during evidence reporting or validation, the +terminal error is the outer timeout and the final durability line may be absent. +Any already-emitted incomplete measurement remains evidence; absent validation +does not establish complete durable success. + +These reporting fixes do not reattribute the 20 frozen measurements to a later +SHA. The follow-up release smoke passed 9 tests (4/4 DB gates), including the +failure-precedence controls; formatting, diff check and target release Clippy +with warnings denied passed. Seven real-entry invalid-control cases rejected +empty, zero, unsupported, NaN, bad-order and non-Unicode inputs before fixture +setup. Post-fix retry/ordinary 8-session maxima were 1.009587 / 0.017658 / +0.021085s; the retry remains incomplete despite durable recovered children. +Final review dispositions and CI are recorded in the PR description. + +## Reproduction + +Use the measured SHA above to reproduce these observations, or explicitly name +any later source. Build before timing, with a dedicated disposable durable PG16 +primary and a coordinated interval. Select one topology per process to retain +an independent result even if its paired case fails; `1,2` and `2,1` also work. +The allowed window sizes are `16` (historical default), `400000`, `500000`; +frontend-order values are `1`, `2`, `1,2` (default), `2,1`. Session count is fixed. + +```sh +CARGO_BUILD_JOBS=2 cargo test --release --locked -p qbit-prism-server \ + --test b275_persistence_measure --no-run +ulimit -n 16384 +PRISM_TEST_DATABASE_URL="$disposable_pg16_url" \ +PRISM_TEST_REQUIRE_INTEGRATION=1 PRISM_TEST_GATE_MANIFEST="$unique_manifest" \ +PRISM_B275_WINDOW_SHARES=400000 PRISM_B275_FRONTEND_ORDER=1 \ +CARGO_BUILD_JOBS=2 cargo test --release --locked -p qbit-prism-server \ + --test b275_persistence_measure measure_2000_sessions_one_and_two_frontends \ + -- --ignored --exact --test-threads=1 --nocapture +``` + +The observed supervisor invoked the freshly built binary directly, with those +arguments/environment, through `/usr/bin/time -l`; no compilation ran inside a +sample. Repeat the exact table order, use a fresh manifest for every invocation, +retain every exit/failure, and enforce/report the separate resource watchdog. +Test `executed` gates mean a test entered, not that both topologies ran or 1s +passed. On current source, read `B275_RUN`/`B275_RUN_PROGRESS` scope, +`B275_FIXTURE`, v4 `B275_MEASUREMENT`, receipts, issued-row counts and structured +`B275_DURABILITY` together; missing validation is unknown, never zero/success. +Raw JSON, child receipts, host logs, gate manifests, scripts and build metadata +remain outside Git in the coordinator's issue275-post-batch-20260917 artifact +bundle. No production/testnet endpoint was accessed and PR441 proxy code was +neither changed nor exercised by this harness. +The 101-file raw artifact manifest has SHA-256 +`60e8dc788f943e4ae338e779f33d2ae97da40dd5261c8165f30c2a278c409b14`. diff --git a/tests/perf/b275_release_profile.md b/tests/perf/b275_release_profile.md new file mode 100644 index 000000000..bea03809d --- /dev/null +++ b/tests/perf/b275_release_profile.md @@ -0,0 +1,164 @@ +# Release-profile public Stratum delivery (#275) + +**Five of six 2,000-session cases missed the unchanged one-second target.** +All 12,000 timed deliveries matched durable issued rows, with zero failed +delivery attempts and zero unmatched rows. Two-frontend non-regression and +production-shaped qualification remain unestablished; #275 remains open. +This report adds release evidence to the [historical debug measurements](b275_persistence_measure.md). + +## Revision and conditions + +- Actual tested source: `d5d14ce5e828375c8964db21b8c756dcf2f41d9f`, the fetched + `3.x.x` tip including #402. No source edits preceded the measurements. +- Fresh `cargo test --release --locked --no-run` build, Rust/Cargo 1.98.1, + `aarch64-apple-darwin`: optimization level 3, debug information 0, debug + assertions and overflow checks disabled, thin LTO, one codegen unit. + Cargo reported the test artifact as newly built, not fresh from cache. + Test binary SHA-256: + `47ae2c1e5d98f4ad43992795b7d91d8495f25c573aeee3ee655123b272c172c9`. +- Paired runs: 2026-09-16, 17:41:40–17:42:11 UTC. Apple M4 Max (Mac16,6), + 14 logical CPUs, 36 GiB RAM, macOS 26.5.1 (25F80). +- Fresh dedicated disposable PostgreSQL 16.14 primary on the same host: + `fsync=on`, `full_page_writes=on`, `synchronous_commit=on`, + `wal_sync_method=open_datasync`, `shared_buffers=128MB`, + `max_connections=100`; no recovery/replica. The harness verifies PG16 and + all three durability settings for each topology. +- Other workers held new heavy tests after their active suite exited. + Process checks before/after each pair and every 0.5 s during execution found + no competing Cargo, rustc, Clippy, pytest, nextest or test-binary process. + This does **not** establish a quiet host: background applications remained, + pre-pair one-minute load was 12.29/13.64/12.46, and the second one-second + CPU samples were only 48.28%/55.83%/45.58% idle. Memory free was 40–42%, + existing swap use was 18,295.69 MiB, and disk free was 5.3 GiB. Subsecond + interference and background activity are not excluded. +- Descriptor limit 16,384 (kernel process ceiling 138,240). Test-process + peak RSS per pair: 96,010,240 / 97,206,272 / 97,304,576 bytes, excluding PG. + +## Effective configuration and boundary + +The fixture sets configuration directly; shell production settings do not +override it. Both topologies share two Tokio workers and one test process. +Each frontend has two build workers, four database connections, a connection +limit of 2,000 and 128 job-build admission permits. Build and persistence each +have a 30 s phase timeout, writes 20 s, and issued-job retention 300 s; +vardiff is disabled. Two frontends therefore provide eight aggregate database +connections and 256 admission permits. Template/submit freshness, snapshot +interval and health timeout are 600 s; automatic background polling is absent. + +The fixture contains **16 deterministic accepted shares, five payout recipients, +an empty transaction template and a synthetic local node**. Public TCP Stratum, +coordinators, prepared records and issued-job persistence use the real runtime. +This is a synthetic fanout measurement, not a production-sized payout-window test. + +Every session subscribes, authorizes and receives initial work before timing. +The common monotonic interval starts immediately before concurrent +`refresh_once` polling after changing the synthetic parent, and ends per client +when it decodes fresh new-parent `mining.notify` with `clean_jobs=true`. +The boundary includes refresh, construction, persistence, scheduling and TCP +delivery, including the single test task's client decoding. It excludes login +and automatic tip polling delay. All clients share a 120 s delivery deadline; +each topology has the original 240 s body budget. These are completion bounds, +not substitutes for the **one-second performance target**. + +## Three paired observations + +Each pair runs one frontend with 2,000 sessions, then two with 1,000 each, +using fresh schemas and closed listeners/pools between topologies. + +| Pair | Frontends | Lower median (s) | Maximum (s) | Refresh returns (s) | Within 1 s | +| --- | ---: | ---: | ---: | --- | --- | +| 1 | 1 | 0.639068 | 1.198136 | 0.009681 | No | +| 1 | 2 | 0.906132 | 1.374454 | 0.014736 / 0.012828 | No | +| 2 | 1 | 0.695557 | 1.160832 | 0.010847 | No | +| 2 | 2 | 0.874736 | 1.481149 | 0.007875 / 0.009870 | No | +| 3 | 1 | 0.765952 | 1.301847 | 0.009791 | No | +| 3 | 2 | 0.645584 | 0.970648 | 0.013121 / 0.014339 | Yes | + +Every row received and durably verified exactly 2,000 jobs, reported +`complete=true`, zero refresh/delivery errors, zero failed delivery attempts +and zero unmatched issued rows. Success counters were 2,000 for one frontend +or 1,000 each for two. Durable validation also checked prepared identity, +payout revision, expiry and unchanged prepared bytes. The three invocations +exited zero and each recorded its required integration gate. + +Two/one maximum-delivery ratios were **1.147, 1.276 and 0.746**. Fixed ordering, +three repetitions, mixed direction, background load and no predeclared +non-regression tolerance prevent a speedup/non-regression conclusion. +The older debug results used a different revision and measurement window; +this is not a controlled debug-versus-release speed comparison. + +## Where the observed delay accumulates + +These are successful-call histogram deltas over refresh plus fanout, summed +across frontends. Counts are 10,013 pool checkouts and 2,007 settlement-lock +acquisitions for one frontend, or 10,026 and 2,014 for two. + +| Pair | Frontends | Pool wait sum (s) | Pool mean (ms) | Settlement wait sum (s) | Settlement mean (ms) | +| --- | ---: | ---: | ---: | ---: | ---: | +| 1 | 1 | 333.524 | 33.309 | 2.299 | 1.145 | +| 1 | 2 | 720.512 | 71.864 | 7.503 | 3.726 | +| 2 | 1 | 331.274 | 33.084 | 2.156 | 1.074 | +| 2 | 2 | 746.210 | 74.428 | 8.246 | 4.094 | +| 3 | 1 | 371.612 | 37.113 | 2.480 | 1.236 | +| 3 | 2 | 497.757 | 49.647 | 5.049 | 2.507 | + +Refresh returned within 15 ms, while delivery continued for roughly 1 s or more. +The measured queueing is on the database checkout/settlement path during fanout; +doubling frontend resources did not consistently reduce delivery time. **These +overlapping sums are not additive wall time, per-job latency, lock hold time or +commit duration.** Build, query execution, commit/WAL, scheduling and socket +costs remain unsplit. In particular, the data does not establish that removing +the settlement lock would meet the target. + +The next bounded diagnostic is per-job build/persist timing in the test harness, +paired with a separate PG transaction/commit-wait observation. That should +separate admission/pool queueing from time executing and committing each issued +row before choosing a runtime change. No production lock or batching change +is proposed for landing by this report. + +## Regression evidence and remaining limits + +The same release binary then passed **7 tests, 0 failures, 1 intentionally +ignored scale test**, with **4/4 database gates executed**. The retry injection +recovered all eight clients after six failed attempts and correctly emitted +`complete=false`; eventual delivery did not masquerade as failure-free delivery. +Its durable-row check is not reached on that intentional incomplete result, +so unmatched rows for this retry control remain unknown. Both ordinary smoke +topologies reported zero failed attempts and unmatched rows. The original +revision-fence regression passed. The separate three-second settlement-lock +stub observed a waiter and 3.004581 s persistence; it is not actual landing work +and does not establish lock-free persistence. Commit timing remains unknown. + +Cleanup found zero measurement schemas and stopped the disposable primary. +No production or testnet service was used. Full payout windows, transaction-heavy +templates, miner submissions, real-node latency, remote networks, separate +frontend hosts/processes and automatic polling delay remain unmeasured. +The prior independent 500k-window qualification is separate evidence, not a +qualification of this 2,000-session workload. + +## Reproduction and publication revision + +Use a clean checkout of the actual tested SHA above, coordinate heavy work, +and set `PRISM_TEST_DATABASE_URL` to a disposable durable PG16 primary. +Build before timing. Run this command three times with a fresh gate-manifest +path each time, recording effective configuration and machine conditions: + +```sh +ulimit -n 16384 +PRISM_TEST_REQUIRE_INTEGRATION=1 \ +PRISM_TEST_GATE_MANIFEST="$manifest" \ +cargo test --release --locked -p qbit-prism-server \ + --test b275_persistence_measure \ + measure_2000_sessions_one_and_two_frontends \ + -- --ignored --exact --test-threads=2 --nocapture +``` + +The observed runner invoked the freshly compiled release test binary directly +with those test arguments; compilation was outside the measured window. +For the seven regressions, omit the test name, `--ignored` and `--exact`. + +Publication re-fetched `3.x.x` at `a828c21cb9cb8c854d5148ceb6b1678debf90d22` +(#422) and advanced this documentation branch. That delta changes a separate +recovery fixture, its regression and the gate inventory; it changes none of +the measured runtime or harness/support files. The six observations remain +attributed solely to **`d5d14ce`**, not to the later documentation commit. diff --git a/tests/perf/candidate_storage.py b/tests/perf/candidate_storage.py deleted file mode 100644 index e62f5ae02..000000000 --- a/tests/perf/candidate_storage.py +++ /dev/null @@ -1,162 +0,0 @@ -"""Disposable native candidate storage/ownership probe, not a mined-block fixture. - -Runs actual prepare, append, header replay, hydration, exact retry and cleanup -with a real writer-session proof actor. Creates and drops a private schema. -Run serially in Docker; ARM timings do not establish a production root cause. -""" -from __future__ import annotations - -import argparse -from collections.abc import Sequence -import gc -import hashlib -import json -import platform -import sys -import threading -import time -import uuid -import weakref -from pathlib import Path - -sys.path.insert(0, str(Path(__file__).resolve().parents[2])) - -import psycopg -from psycopg.conninfo import make_conninfo -from lab.prism.candidate_codec import prepare_candidate_intent -from lab.prism.share_ledger import PsqlShareLedger, WRITER_LEASE_HEARTBEAT_SESSION_PREFIX -from tests.perf.candidate_window_membership import _gc_callback, measure -from tests.prism_postgres_candidate_gate import intent, pending_share_for, share - - -class Shares(Sequence): - def __init__(self, count): - self.count = count - - def __len__(self): - return self.count - - def __getitem__(self, index): - if not 0 <= index < self.count: - raise IndexError(index) - row = share(index) - row["miner_id"] += "x" * 240 - return row - - -def main(): - parser = argparse.ArgumentParser() - parser.add_argument("--database-url", required=True) - parser.add_argument("--shares", type=int, default=375000) - parser.add_argument("--candidates", type=int, default=3) - parser.add_argument("--generations", type=int, default=1) - parser.add_argument("--gc", choices=("on", "off"), default="off") - args = parser.parse_args() - if min(args.shares, args.candidates, args.generations) < 1: - parser.error("counts must be positive") - schema = "candidate_perf_" + uuid.uuid4().hex - with psycopg.connect(args.database_url, autocommit=True) as admin: - admin.execute(f'CREATE SCHEMA "{schema}"') - url = make_conninfo(args.database_url, options=f"-csearch_path={schema}") - ledger = None - stop = threading.Event() - proof_thread = None - proofs, wakes, errors = [], [], [] - was_enabled = gc.isenabled() - gc.callbacks.append(_gc_callback) - try: - ledger = PsqlShareLedger( - psql_command="psql", database_url=url, native_client_mode="native", - writer_id="candidate-storage-probe", initialize_schema=True, - writer_session_token=WRITER_LEASE_HEARTBEAT_SESSION_PREFIX + uuid.uuid4().hex, - ) - (gc.enable if args.gc == "on" else gc.disable)() - - def heartbeat(): - due = time.monotonic() - while not stop.is_set(): - started = time.monotonic() - wakes.append(max(0, started - due)) - try: - ledger.prove_writer_lease_guard_session() - proofs.append(time.monotonic() - started) - except Exception as exc: - errors.append(str(exc)) - due = time.monotonic() + 0.05 - stop.wait(max(0, due - time.monotonic())) - - proof_thread = threading.Thread(target=heartbeat) - proof_thread.start() - - def phase(name, function): - before = len(proofs), len(wakes), len(errors) - value = measure(name, function) - print(json.dumps({ - "phase": name, "proof_count": len(proofs) - before[0], - "max_proof_seconds": max(proofs[before[0]:], default=0), - "max_heartbeat_wake_lateness_seconds": max(wakes[before[1]:], default=0), - "proof_errors": errors[before[2]:], - "spool": ledger._candidate_spool.snapshot(), - }), flush=True) - return value - - print(json.dumps({"python": sys.version, "architecture": platform.machine(), - "shares": args.shares, "candidates": args.candidates, - "gc": args.gc, "generations": args.generations}), flush=True) - source = Shares(args.shares) - for index in range(args.candidates): - block_hash = hashlib.sha256(f"{schema}-{index}".encode()).hexdigest() - fields = intent(block_hash, 0, credit=False) - fields["shares_json"] = source - prepared = phase(f"prepare-{index}", lambda: prepare_candidate_intent(fields)) - print(json.dumps({"candidate": index, "bytes": prepared.manifest.byte_count, - "chunks": prepared.manifest.chunk_count, - "sha256": prepared.candidate_sha256}), flush=True) - pending = pending_share_for(fields) - phase(f"append-{index}", lambda: ledger.append_batch([(pending, prepared)])) - phase(f"exact-retry-{index}", lambda: ledger.append_batch([(pending, prepared)])) - del prepared, pending, fields - page = phase("replay-headers", lambda: ledger.pending_block_candidate_headers(limit=args.candidates)) - assert page.exhausted and len(page.rows) == args.candidates - for generation in range(args.generations): - for index, row in enumerate(page.rows): - hydrated = phase(f"hydrate-{generation}-{index}", lambda: ledger.hydrate_block_candidate_intent(row)) - sequence = hydrated["shares_json"] - def walk(): - count = 0 - for count, record in enumerate(sequence, 1): - assert record["share_id"] == source[count - 1]["share_id"] - assert count == args.shares - phase(f"walk-{generation}-{index}", walk) - observed = weakref.ref(hydrated.body) - phase(f"close-{generation}-{index}", hydrated.body.close) - del sequence, hydrated - assert observed() is None, "body survives refcount retirement" - assert ledger._candidate_spool.snapshot()["reserved_bytes"] == 0 - for row in page.rows: - phase("terminalize", lambda: ledger.mark_block_candidate_submitted(block_hash=row["block_hash"])) - - def drain(): - steps = 0 - while ledger.reap_retired_candidate_bodies()["body_id"] is not None: - steps += 1 - if steps > 100000: - raise AssertionError("janitor did not drain") - return steps - phase("janitor", drain) - assert not errors, errors - print(json.dumps({"result": "PASS", "proof_errors": errors}), flush=True) - finally: - stop.set() - if proof_thread is not None: - proof_thread.join() - gc.callbacks.remove(_gc_callback) - (gc.enable if was_enabled else gc.disable)() - if ledger is not None: - ledger.release_writer_lease() - ledger.close() - admin.execute(f'DROP SCHEMA "{schema}" CASCADE') - - -if __name__ == "__main__": - main() diff --git a/tests/perf/candidate_window_membership.py b/tests/perf/candidate_window_membership.py deleted file mode 100644 index 30c25a0e6..000000000 --- a/tests/perf/candidate_window_membership.py +++ /dev/null @@ -1,223 +0,0 @@ -"""Disposable PostgreSQL comparison of bounded candidate membership and the old aggregate. - -Example (after applying the ledger schema to a disposable database): - python tests/perf/candidate_window_membership.py --database-url "$TEST_DATABASE_URL" \ - --shares 375000 --gc off --prepare-disposable-fixture - -This replaces share rows in the supplied disposable database. It is a membership -and codec experiment, not a valid mined-block fixture or production timing proof. -The lease-guard actor is real; it records failures rather than terminating the -harness on a late wake. It reports scheduling delay separately from round-trip -duration; a short round trip alone does not imply a timely wake. Compare these -measurements with the integrated lease and -regtest gates before deployment. Run measurements serially in Docker Python3.14.7. -""" -import argparse -import gc -import json -import os -import platform -import sys -import threading -import time -import uuid -from pathlib import Path - -# Executable directly from a repository checkout. -sys.path.insert(0, str(Path(__file__).resolve().parents[2])) - -import psycopg -from lab.prism.share_ledger import PsqlShareLedger, WRITER_LEASE_HEARTBEAT_SESSION_PREFIX - -GC_PAUSES = {"active": {}, "max": {0: 0.0, 1: 0.0, 2: 0.0}, "count": {0: 0, 1: 0, 2: 0}} - - -def _gc_callback(phase: str, info: dict) -> None: - gen = int(info.get("generation", 0)) - now = time.monotonic() - if phase == "start": - GC_PAUSES["active"][gen] = now - return - start = GC_PAUSES["active"].pop(gen, None) - if start is None: - return - pause = now - start - GC_PAUSES["max"][gen] = max(GC_PAUSES["max"][gen], pause) - GC_PAUSES["count"][gen] += 1 - - -def _reset_gc_pauses() -> None: - GC_PAUSES["active"].clear() - for gen in (0, 1, 2): - GC_PAUSES["max"][gen] = 0.0 - GC_PAUSES["count"][gen] = 0 - - -def _rss() -> dict[str, int]: - out = {} - try: - with open("/proc/self/status") as handle: - for line in handle: - if line.startswith(("VmRSS:", "VmHWM:")): - key, value = line.split(":", 1) - out[key] = int(value.strip().split()[0]) // 1024 - except OSError: - pass - return out - - -def measure(label: str, function, *, note: str | None = None): - stop, ready = threading.Event(), threading.Event() - samples: list[float] = [] - - def observer() -> None: - ready.set() - while not stop.is_set(): - start = time.monotonic() - stop.wait(0.005) - samples.append(max(0.0, time.monotonic() - start - 0.005)) - - thread = threading.Thread(target=observer, name="observer") - thread.start() - ready.wait() - time.sleep(0.025) - _reset_gc_pauses() - before = [s["collections"] for s in gc.get_stats()] - start = time.monotonic() - try: - result = function() - elapsed = time.monotonic() - start - time.sleep(0.025) - finally: - stop.set() - thread.join() - record = dict( - operation=label, - wall_seconds=round(elapsed, 6), - max_observer_lateness_seconds=round(max(samples), 6), - wakes_over_100ms=sum(x >= 0.1 for x in samples), - wakes_over_400ms=sum(x >= 0.4 for x in samples), - gc_collection_delta=[s["collections"] - n for s, n in zip(gc.get_stats(), before)], - gc_max_pause_seconds={g: round(GC_PAUSES["max"][g], 6) for g in (0, 1, 2)}, - gc_pause_count={g: GC_PAUSES["count"][g] for g in (0, 1, 2)}, - rss_mb=_rss(), - ) - if note: - record["note"] = note - print(json.dumps(record), flush=True) - return result - - -def main() -> None: - parser = argparse.ArgumentParser() - parser.add_argument("--shares", type=int, default=375000) - parser.add_argument("--gc", choices=("off", "on"), default="off") - parser.add_argument("--database-url", required=True) - parser.add_argument("--prepare-disposable-fixture", action="store_true", required=True) - args = parser.parse_args() - if args.shares < 1: - parser.error("--shares must be positive") - - with psycopg.connect(args.database_url, autocommit=True) as connection: - connection.execute("TRUNCATE qbit_share_ledger CASCADE") - connection.execute(""" - INSERT INTO qbit_share_ledger ( - share_id, miner_id, payout_order_key, p2mr_program, - share_difficulty, network_difficulty, template_height, job_id, - job_issued_at, ntime, accepted_at, credit_policy, accepted, - writer_id, writer_epoch - ) SELECT - 'share-' || n, 'miner', 'miner', decode(repeat('aa', 32), 'hex'), - 1, 1000000, 1, 'job', to_timestamp(0), 0, to_timestamp(1), - NULL, true, 'fixture', 1 - FROM generate_series(0, %s) AS n - """, (args.shares - 1,)) - ledger = PsqlShareLedger( - psql_command="psql", database_url=args.database_url, - native_client_mode="native", writer_id="candidate-window-probe", - writer_session_token=WRITER_LEASE_HEARTBEAT_SESSION_PREFIX + uuid.uuid4().hex, - ) - - class Rows: - def __iter__(self): - for index in range(args.shares): - yield {"share_id": f"share-{index}"} - - stop = threading.Event() - heartbeats: list[float] = [] - heartbeat_wakes: list[float] = [] - failures: list[str] = [] - - def heartbeat() -> None: - due = time.monotonic() - while not stop.is_set(): - started = time.monotonic() - heartbeat_wakes.append(max(0.0, started - due)) - try: - ledger.prove_writer_lease_guard_session() - heartbeats.append(time.monotonic() - started) - except Exception as exc: - failures.append(str(exc)) - due = time.monotonic() + 0.05 - stop.wait(max(0.0, due - time.monotonic())) - - def report_proofs(phase: str, before: tuple[int, int, int]) -> None: - proof_start, wake_start, error_start = before - print(json.dumps({ - "phase": phase, - "proof_count": len(heartbeats) - proof_start, - "proof_errors": failures[error_start:], - "max_proof_seconds": max(heartbeats[proof_start:], default=0), - "max_heartbeat_wake_lateness_seconds": max(heartbeat_wakes[wake_start:], default=0), - }), flush=True) - - thread = threading.Thread(target=heartbeat, name="lease-proof") - print(json.dumps({ - "python": sys.version, "machine": platform.machine(), - "shares": args.shares, "gc": args.gc, - "psycopg": psycopg.__version__, - "libpq": psycopg.pq.version(), - "switch_interval_seconds": sys.getswitchinterval(), - "gc_thresholds": gc.get_threshold(), - "allocator": {name: os.environ.get(name) for name in ( - "MALLOC_ARENA_MAX", "PYTHONMALLOC", "GLIBC_TUNABLES", - )}, - }), flush=True) - gc.collect() - if args.gc == "off": - gc.disable() - gc.callbacks.append(_gc_callback) - thread.start() - try: - before_proofs = len(heartbeats), len(heartbeat_wakes), len(failures) - result = measure("bounded-window-membership", lambda: ledger.candidate_window_covers( - Rows(), anchor_job_issued_at_ms=2000, network_difficulty=1000000, - )) - assert result is True - report_proofs("bounded-window-membership", before_proofs) - - before_proofs = len(heartbeats), len(heartbeat_wakes), len(failures) - rows = measure("baseline-audit-window-aggregate", lambda: ledger.audit_share_window( - anchor_job_issued_at_ms=2000, network_difficulty=1000000, - )) - assert len(rows) == args.shares - report_proofs("baseline-audit-window-aggregate", before_proofs) - measure("baseline-reference-count-retirement", rows.clear) - finally: - stop.set() - thread.join() - gc.callbacks.remove(_gc_callback) - try: - ledger.release_writer_lease() - finally: - ledger.close() - print(json.dumps({ - "heartbeat_count": len(heartbeats), - "heartbeat_max_seconds": max(heartbeats, default=0), - "heartbeat_max_wake_lateness_seconds": max(heartbeat_wakes, default=0), - "heartbeat_errors": failures, - }), flush=True) - - -if __name__ == "__main__": - main() diff --git a/tests/perf/canonical_audit_retirement.py b/tests/perf/canonical_audit_retirement.py deleted file mode 100644 index e544dfff9..000000000 --- a/tests/perf/canonical_audit_retirement.py +++ /dev/null @@ -1,172 +0,0 @@ -"""Isolated audit codec/ownership experiment; this is not a mined-block fixture. - -Run serially in Docker with the coordinator's Python and allocator settings. -Each artifact has 375,000 shares by default (about 232 MB). The ordinary -phases run without allocation tracing; --trace measures allocations separately -because tracing substantially changes elapsed time and scheduling. -""" -from __future__ import annotations - -import argparse -import gc -import hashlib -import json -import os -from pathlib import Path -import platform -import shutil -import sys -import tempfile -import threading -import time -import tracemalloc -import weakref - -sys.path.insert(0, str(Path(__file__).resolve().parents[2])) - -from lab.prism.audit_bundle_view import CanonicalAuditBundleView, iter_json_byte_chunks -from tests.test_prism_audit_bundle_view import make_share - - -def fixture(path: Path, count: int) -> tuple[int, str]: - digest = hashlib.sha256() - with path.open("wb") as output: - def emit(value: str) -> None: - encoded = value.encode("utf-8") - output.write(encoded) - digest.update(encoded) - - emit('{"schema":"qbit.prism.audit-bundle.v1","shares":[') - for index in range(count): - row = make_share(index) - row["share_id"] += "x" * 147 - row["credit_policy"] = None - emit(("," if index else "") + json.dumps(row, separators=(",", ":"))) - emit('],"found_block":{"block_height":10},"prior_balances":[],') - emit('"reward_manifest":{"shares":[') - for index in range(count): - emit(("," if index else "") + json.dumps( - {"share_seq": index + 1, "counted_difficulty": 5}, separators=(",", ":"), - )) - emit('],"entitlements":[]},"payout_policy_manifest":{"accounts":[]},') - emit('"signed_coinbase_manifest":{"manifest":{"coinbase_tx_hex":"00"}}}') - return path.stat().st_size, digest.hexdigest() - - -def memory() -> dict[str, int]: - result = {} - for line in Path("/proc/self/status").read_text().splitlines(): - key, _, value = line.partition(":") - if key in ("VmRSS", "VmHWM"): - result[key + "_bytes"] = int(value.strip().split()[0]) * 1024 - return result - - -def measure(label: str, operation, *, trace: bool): - stop, ready = threading.Event(), threading.Event() - wakes, pauses = [], [] - gc_started = {} - - def gc_event(phase, info): - generation = info["generation"] - if phase == "start": - gc_started[generation] = time.monotonic() - else: - started = gc_started.pop(generation, None) - if started is not None: - pauses.append((generation, time.monotonic() - started)) - - def observer(): - ready.set() - while not stop.is_set(): - started = time.monotonic() - stop.wait(0.005) - wakes.append(max(0.0, time.monotonic() - started - 0.005)) - - thread = threading.Thread(target=observer) - thread.start() - ready.wait() - time.sleep(0.025) - gc.callbacks.append(gc_event) - if trace: - tracemalloc.start() - try: - started = time.monotonic() - result = operation() - elapsed = time.monotonic() - started - peak = tracemalloc.get_traced_memory()[1] if trace else None - time.sleep(0.025) - finally: - stop.set() - thread.join() - gc.callbacks.remove(gc_event) - if trace: - tracemalloc.stop() - ordered = sorted(wakes) - print(json.dumps({ - "phase": label, "seconds": elapsed, - "observer_max_seconds": max(wakes, default=0), - "observer_p99_seconds": ordered[int((len(ordered) - 1) * 0.99)], - "wakes_over_100ms": sum(value >= 0.1 for value in wakes), - "gc_pause_count": len(pauses), - "gc_max_pause_seconds": max((value for _, value in pauses), default=0), - "peak_traced_bytes": peak, **memory(), - }), flush=True) - return result - - -def main() -> None: - parser = argparse.ArgumentParser(description=__doc__) - parser.add_argument("--shares", type=int, default=375000) - parser.add_argument("--candidates", type=int, default=1) - parser.add_argument("--gc", choices=("off", "on"), default="off") - parser.add_argument("--trace", action="store_true") - parser.add_argument("--baseline", action="store_true") - args = parser.parse_args() - if args.shares < 1 or args.candidates < 1: - parser.error("share and candidate counts must be positive") - print(json.dumps({ - "python": sys.version, "machine": platform.machine(), - "gc": args.gc, "trace": args.trace, "candidates": args.candidates, - "switch_interval": sys.getswitchinterval(), "gc_thresholds": gc.get_threshold(), - "allocator": {name: os.environ.get(name) for name in ( - "MALLOC_ARENA_MAX", "PYTHONMALLOC", "GLIBC_TUNABLES", - )}, - }), flush=True) - with tempfile.TemporaryDirectory(prefix="audit-codec-probe-") as directory: - path = Path(directory) / "audit.json" - size, digest = fixture(path, args.shares) - print(json.dumps({"shares": args.shares, "artifact_bytes": size, "sha256": digest}), flush=True) - gc.collect() - if args.gc == "off": - gc.disable() - for index in range(args.candidates): - owner = {} - owner["view"] = measure(f"scan-{index}", lambda: CanonicalAuditBundleView.scan_path(path), trace=args.trace) - assert owner["view"].sha256_hex == digest - view_ref = weakref.ref(owner["view"]) - source_ref = weakref.ref(owner["view"].source) - count = measure(f"lazy-walk-{index}", lambda: sum(1 for _ in owner["view"]["shares"]), trace=args.trace) - assert count == args.shares - encoded = measure(f"encode-{index}", lambda: sum(len(chunk) for chunk in iter_json_byte_chunks(owner["view"])), trace=args.trace) - assert encoded == size - measure(f"retire-{index}", lambda: owner.pop("view").close(), trace=args.trace) - assert view_ref() is None and source_ref() is None - if args.baseline: - aggregate = Path(directory) / "aggregate.json" - with aggregate.open("wb") as destination: - destination.write(b"[") - for index in range(args.candidates): - if index: - destination.write(b",") - with path.open("rb") as source: - shutil.copyfileobj(source, destination, length=256 * 1024) - destination.write(b"]") - rows = measure("whole-decode-control", lambda: json.loads(aggregate.read_bytes()), trace=args.trace) - assert all(len(row["shares"]) == args.shares for row in rows) - measure("whole-reference-count-retirement", rows.clear, trace=args.trace) - measure("cyclic-gc-control", gc.collect, trace=args.trace) - - -if __name__ == "__main__": - main() diff --git a/tests/perf/serialize_block_share_cost.py b/tests/perf/serialize_block_share_cost.py deleted file mode 100644 index b4a9d1342..000000000 --- a/tests/perf/serialize_block_share_cost.py +++ /dev/null @@ -1,1338 +0,0 @@ -#!/usr/bin/env python3 -"""Per-share interpreter-CPU cost of ``direct_stratum.serialize_block``. - -Measures the block-serialization term on the PRISM share-accept path across -template size, so the "``serialize_block`` is O(block bytes) per accepted -share" gap left open by the issue #143 findings comment can be closed with a -number instead of an estimate. - -Methodology mirrors section 1 of that comment so the results are comparable: - -* every timing is **min-of-7 repetitions**, each repetition an auto-sized - batch of calls, reported per call; -* the headline clock is **interpreter CPU**, not wall clock -- see - ``CLOCK_RATIONALE`` below; -* single-threaded: the findings comment established this path is GIL-bound - and does not scale with threads, so what is wanted is the per-share cost, - not concurrency behaviour. - -Three measurement groups: - -1. **Anchors.** The comment's two published numbers -- a 12.4 us primitives - floor per accepted share and a 9.0 us 0-transaction-template composite - ``assemble_submission`` -- reproduced on the host running this script, so - every sweep number below can be read against the same scale. -2. **Axis 1**, total block bytes from the 0-transaction baseline to the qbit - consensus ceiling of 2,000,000 bytes (``MAX_BLOCK_WEIGHT`` with - ``WITNESS_SCALE_FACTOR = 1``, so weight units are bytes). -3. **Axis 2**, transaction count at fixed total block bytes, which separates - the per-transaction constant from the per-byte memcpy/hex cost. - -Plus a decomposition of the serialization term into its constituent buffer -operations, and a restatement of single-core share capacity at each point. - -Self-contained and re-runnable: no database, no network, no coordinator, no -third-party packages. ``python3 tests/perf/serialize_block_share_cost.py`` -prints the tables; ``--json`` emits the same data as JSON. - -Nothing under ``lab/`` is imported for anything but read-only measurement. -""" - -from __future__ import annotations - -import argparse -import json -import os -import platform -import sys -import sysconfig -import time -from dataclasses import dataclass -from decimal import Decimal -from pathlib import Path -from typing import Any, Callable, Sequence - -if not __package__: - sys.path.insert(0, str(Path(__file__).resolve().parents[2])) - -from lab.auxpow import stratum_codec -from lab.prism import direct_stratum -from lab.prism.share_submission import ( - RecentShareIndex, - parse_submit_request, - validate_submit_request, -) - - -CLOCK_RATIONALE = ( - "time.thread_time_ns() is the headline clock: the driver is single " - "threaded, so per-thread CPU is exactly the interpreter CPU this path " - "burns, it excludes time the OS deschedules us (which perf_counter would " - "charge us for on a non-idle host), and on this platform its resolution " - "is far finer than process_time's. process_time and perf_counter are " - "captured over the same batch and reported alongside so any divergence " - "-- e.g. kernel time for multi-megabyte allocations -- is visible." -) - -# Published anchors from the issue #143 findings comment, section 1. -PUBLISHED_PRIMITIVES_FLOOR_US = 12.4 -PUBLISHED_ZERO_TX_COMPOSITE_US = 9.0 -# Same comment: "~15.7 us for the composite submission path (primitives-ratio -# estimate)" and "64k-81k shares/s of single-core capacity". -PUBLISHED_FULL_PATH_ESTIMATE_US = 15.7 -PUBLISHED_CAPACITY_LOW_PER_S = 64_000 -PUBLISHED_CAPACITY_HIGH_PER_S = 81_000 -# Deployed design point named by the issue: ~4,096 connections at -# PRISM_STRATUM_VARDIFF_TARGET_SECONDS=15 -> 4096/15 shares/s steady state. -DESIGN_POINT_SHARES_PER_S = 4096 / 15 - -# qbit consensus ceiling. crates/qbit-pool-builder/src/lib.rs:16-17 sets -# MAX_BLOCK_WEIGHT = 2_000_000 with WITNESS_SCALE_FACTOR = 1, so weight units -# are bytes; compose.yaml and the ckpool preflight both hard-pin the same -# 2000000 against live getblocktemplate.weightlimit. -MAX_BLOCK_BYTES = 2_000_000 - -EXTRANONCE1_HEX = "deadbeef" -EXTRANONCE2_SIZE = 8 -NTIME_HEX = "68a4c1a0" -NONCE_HEX = "12345678" - -# Header (80) + the transaction-count compact size + the coinbase. -HEADER_BYTES = 80 - -AXIS1_TARGET_BLOCK_BYTES = (0, 50_000, 100_000, 250_000, 500_000, 1_000_000, 1_500_000, 2_000_000) -AXIS1_NOMINAL_TX_BYTES = 500 -AXIS2_TOTAL_BLOCK_BYTES = 1_000_000 -AXIS2_TX_COUNTS = (10, 100, 500, 1_000, 2_000, 5_000) - - -# -------------------------------------------------------------------------- -# timing harness -# -------------------------------------------------------------------------- - - -@dataclass(frozen=True) -class Timing: - """Per-call cost from the best of ``reps`` batched repetitions.""" - - thread_cpu_us: float - process_cpu_us: float - wall_us: float - batch: int - reps: int - - def as_json(self) -> dict[str, Any]: - return { - "thread_cpu_us": round(self.thread_cpu_us, 6), - "process_cpu_us": round(self.process_cpu_us, 6), - "wall_us": round(self.wall_us, 6), - "batch": self.batch, - "reps": self.reps, - } - - -def measure( - fn: Callable[[], Any], - *, - reps: int = 7, - min_batch_seconds: float = 0.05, - max_batch: int = 1 << 22, -) -> Timing: - """Min-of-``reps`` batched timing of ``fn``, reported per call. - - The batch is auto-sized so one repetition spans at least - ``min_batch_seconds``, keeping sub-microsecond primitives well clear of - clock granularity. The minimum is taken on thread CPU; the process-CPU and - wall figures reported are the ones from that same winning repetition, so - the three are internally consistent. - """ - - batch = 1 - while True: - start = time.perf_counter_ns() - for _ in range(batch): - fn() - elapsed = time.perf_counter_ns() - start - if elapsed >= min_batch_seconds * 1e9 or batch >= max_batch: - break - growth = max(2, min(16, int(min_batch_seconds * 1e9 / max(elapsed, 1)) + 1)) - batch = min(max_batch, batch * growth) - - best: tuple[float, float, float] | None = None - for _ in range(reps): - thread_start = time.thread_time_ns() - process_start = time.process_time_ns() - wall_start = time.perf_counter_ns() - for _ in range(batch): - fn() - wall_end = time.perf_counter_ns() - process_end = time.process_time_ns() - thread_end = time.thread_time_ns() - candidate = ( - (thread_end - thread_start) / batch / 1000.0, - (process_end - process_start) / batch / 1000.0, - (wall_end - wall_start) / batch / 1000.0, - ) - if best is None or candidate[0] < best[0]: - best = candidate - - assert best is not None - return Timing(best[0], best[1], best[2], batch=batch, reps=reps) - - -# -------------------------------------------------------------------------- -# synthetic template material -# -------------------------------------------------------------------------- - - -def _compact_size_hex(value: int) -> str: - return direct_stratum.compact_size(value).hex() - - -def coinbase_transaction_hex(placeholder_hex: str, *, outputs: int = 1) -> str: - """A structurally valid non-witness coinbase ending in the placeholder. - - Same shape as ``lab/prism/job_build_benchmark.py``'s - ``synthetic_manifest_coinbase_hex``: a height push followed by the - extranonce placeholder at the tail of the coinbase scriptSig, which is - where ``split_coinbase_extranonce`` requires it. - """ - - script_sig = "03aabbcc" + placeholder_hex - output = (50_00000000).to_bytes(8, "little").hex() + "0151" - return ( - "01000000" - + "01" - + "00" * 32 - + "ffffffff" - + _compact_size_hex(len(script_sig) // 2) - + script_sig - + "ffffffff" - + _compact_size_hex(outputs) - + output * outputs - + "00000000" - ) - - -def synthetic_transaction_hex(size_bytes: int, seed: int) -> str: - """A non-witness 1-in/1-out transaction of exactly ``size_bytes`` bytes. - - Padding lives in the output scriptPubKey, with 0-3 slack bytes available - in the input scriptSig so every target size is reachable across compact - size width boundaries. ``seed`` varies the prevout so txids differ and the - merkle branch is not degenerate. - - Structurally valid enough for the code under measurement: it round-trips - through ``strip_witness_transaction``, which is what the job build runs on - every template transaction. - """ - - for cs_width in (1, 3, 5): - for script_sig_len in range(0, 4): - script_pubkey_len = size_bytes - 59 - cs_width - script_sig_len - if script_pubkey_len < 0: - continue - if len(direct_stratum.compact_size(script_pubkey_len)) != cs_width: - continue - prevout = (seed & 0xFFFFFFFF).to_bytes(4, "little").hex() + "11" * 28 - return ( - "02000000" - + "01" - + prevout - + "03000000" - + _compact_size_hex(script_sig_len) - + "ab" * script_sig_len - + "ffffffff" - + "01" - + (1234).to_bytes(8, "little").hex() - + _compact_size_hex(script_pubkey_len) - + "51" * script_pubkey_len - + "00000000" - ) - raise ValueError(f"no transaction encoding reaches {size_bytes} bytes") - - -def base_template() -> dict[str, Any]: - """The repo's own 0-transaction baseline template. - - Mirrors ``lab/prism/job_build_benchmark.py``'s ``base_template``; the - transaction list is supplied separately to - ``make_job_from_builder_manifest``. - """ - - return { - "height": 1000, - "previousblockhash": "11" * 32, - "bits": "1b00ffff", - "version": 0x20000000, - "curtime": 1_755_000_000, - "coinbasevalue": 50_00000000, - "transactions": [], - } - - -@dataclass -class Scenario: - """One template size, built as a real job the accept path would receive.""" - - label: str - job: direct_stratum.DirectQbitStratumJob - transaction_count: int - block_bytes: int - coinbase_bytes: int - transaction_bytes: int - merkle_branch_levels: int - construction: str - - -def build_scenario( - *, - label: str, - transaction_count: int, - target_block_bytes: int | None, - coinbase_outputs: int = 1, - construction: str = "", -) -> Scenario: - """Build a real ``DirectQbitStratumJob`` of the requested size. - - Route 1 of the two acceptable routes: transactions go through - ``make_job_from_builder_manifest``, so ``merkle_branch_for_coinbase`` runs - ``strip_witness_transaction`` over every one of them and the resulting job - carries a realistic merkle branch. That matters because branch depth is - itself a per-share cost inside ``assemble_submission``; a hand-frozen job - with an empty branch would understate the composite at large tx counts. - """ - - placeholder_hex = EXTRANONCE1_HEX + "00" * EXTRANONCE2_SIZE - coinbase_hex = coinbase_transaction_hex(placeholder_hex, outputs=coinbase_outputs) - coinbase_bytes = len(coinbase_hex) // 2 - - if transaction_count == 0: - transaction_hexes: tuple[str, ...] = () - else: - count_prefix = len(direct_stratum.compact_size(transaction_count + 1)) - assert target_block_bytes is not None - payload = target_block_bytes - HEADER_BYTES - count_prefix - coinbase_bytes - base_size, remainder = divmod(payload, transaction_count) - if base_size < 60: - raise ValueError( - f"{transaction_count} transactions cannot fit in " - f"{target_block_bytes} bytes (min 60 bytes each)" - ) - transaction_hexes = tuple( - synthetic_transaction_hex(base_size + (1 if index < remainder else 0), index) - for index in range(transaction_count) - ) - - job = direct_stratum.make_job_from_builder_manifest( - job_id=f"perf-{label}", - template=base_template(), - manifest={"coinbase_tx_hex": coinbase_hex}, - extranonce1_hex=EXTRANONCE1_HEX, - extranonce2_size=EXTRANONCE2_SIZE, - desired_share_difficulty=Decimal("1"), - transaction_hexes=transaction_hexes, - ) - - transaction_bytes = sum(len(tx) // 2 for tx in transaction_hexes) - block_bytes = ( - HEADER_BYTES - + len(direct_stratum.compact_size(transaction_count + 1)) - + coinbase_bytes - + transaction_bytes - ) - return Scenario( - label=label, - job=job, - transaction_count=transaction_count, - block_bytes=block_bytes, - coinbase_bytes=coinbase_bytes, - transaction_bytes=transaction_bytes, - merkle_branch_levels=len(job.merkle_branch), - construction=construction, - ) - - -# -------------------------------------------------------------------------- -# the measured terms -# -------------------------------------------------------------------------- - - -@dataclass -class ScenarioTiming: - scenario: Scenario - composite: Timing - serialize_term: Timing - serialize_block_only: Timing - hex_encode_only: Timing - - # False once the share path stops serializing the block: the term is then - # measured in isolation and is no longer a component of the composite, so - # a ratio between them is not a share of anything. Set from the assembled - # submission rather than inferred from the numbers, because at the 0-tx - # point the isolated term is still smaller than the composite and the - # ratio would look plausible while meaning nothing. - composite_includes_term: bool = True - - @property - def term_share_percent(self) -> float: - if not self.composite_includes_term: - return float("nan") - if self.composite.thread_cpu_us <= 0: - return float("nan") - return 100.0 * self.serialize_term.thread_cpu_us / self.composite.thread_cpu_us - - @property - def composite_less_term_us(self) -> float: - if not self.composite_includes_term: - return float("nan") - return self.composite.thread_cpu_us - self.serialize_term.thread_cpu_us - - -def submission_inputs(job: direct_stratum.DirectQbitStratumJob) -> dict[str, Any]: - """Recompute the intermediates ``assemble_submission`` derives per share.""" - - extranonce2_hex = "00" * EXTRANONCE2_SIZE - version_hex = stratum_codec.apply_version_bits(job.version, None, 0) - coinbase_without_witness, header = stratum_codec.assemble_header_from_notify_submit( - coinb1_hex=job.coinb1, - extranonce1_hex=job.extranonce1_hex, - extranonce2_hex=extranonce2_hex, - coinb2_hex=job.coinb2, - merkle_branch_hex=list(job.merkle_branch), - version_hex=version_hex, - prevhash_hex=job.prevhash, - ntime_hex=NTIME_HEX, - nbits_hex=job.nbits, - nonce_hex=NONCE_HEX, - ) - full_coinbase_hex = ( - job.full_coinbase_prefix - + job.extranonce1_hex - + extranonce2_hex - + job.full_coinbase_suffix - ) - return { - "extranonce2_hex": extranonce2_hex, - "version_hex": version_hex, - "coinbase_without_witness": coinbase_without_witness, - "header": header, - "full_coinbase_hex": full_coinbase_hex, - } - - -def time_scenario(scenario: Scenario, *, reps: int, min_batch_seconds: float) -> ScenarioTiming: - job = scenario.job - inputs = submission_inputs(job) - header = inputs["header"] - full_coinbase_hex = inputs["full_coinbase_hex"] - extranonce2_hex = inputs["extranonce2_hex"] - transaction_hexes = job.transaction_hexes - serialize_block = direct_stratum.serialize_block - assemble = direct_stratum.assemble_submission - block_bytes = serialize_block(header, full_coinbase_hex, transaction_hexes) - - kwargs = {"reps": reps, "min_batch_seconds": min_batch_seconds} - # Does the composite still contain the term on the tree under measurement? - # The driver's jobs use difficulty-65,536 bits, so their submissions are - # ordinary shares -- which is the path worth measuring, and also the path - # that stops serializing the block once the share-path gate is in. - composite_includes_term = bool( - assemble( - job, - extranonce2_hex=extranonce2_hex, - ntime_hex=NTIME_HEX, - nonce_hex=NONCE_HEX, - ).block_hex - ) - return ScenarioTiming( - scenario=scenario, - composite_includes_term=composite_includes_term, - # The whole per-share call as the coordinator invokes it. - composite=measure( - lambda: assemble( - job, - extranonce2_hex=extranonce2_hex, - ntime_hex=NTIME_HEX, - nonce_hex=NONCE_HEX, - ), - **kwargs, - ), - # direct_stratum.py:500 verbatim, the term under measurement. - serialize_term=measure( - lambda: serialize_block(header, full_coinbase_hex, transaction_hexes).hex(), - **kwargs, - ), - serialize_block_only=measure( - lambda: serialize_block(header, full_coinbase_hex, transaction_hexes), - **kwargs, - ), - hex_encode_only=measure(block_bytes.hex, **kwargs), - ) - - -def decompose_term(scenario: Scenario, *, reps: int, min_batch_seconds: float) -> dict[str, Timing]: - """Split the serialization term into the buffer operations it performs. - - ``serialize_block`` runs ``validate_hex`` (which builds an f-string field - name, length-checks, ``bytes.fromhex``-probes and discards, then returns - ``value.lower()`` -- a full fresh copy of the hex string) over the - coinbase and every transaction, then ``bytes.fromhex`` decodes the - validated result for real, joins, and the caller re-encodes with - ``.hex()``. Each of those is timed here over the same material. - """ - - inputs = submission_inputs(scenario.job) - hexes: tuple[str, ...] = (inputs["full_coinbase_hex"],) + scenario.job.transaction_hexes - decoded = [bytes.fromhex(value) for value in hexes] - joined = b"".join(decoded) - validate_hex = stratum_codec.validate_hex - from_hex = bytes.fromhex - kwargs = {"reps": reps, "min_batch_seconds": min_batch_seconds} - - def fromhex_probe() -> None: - for value in hexes: - from_hex(value) - - def lower_copy() -> None: - for value in hexes: - value.lower() - - def validate_hex_calls() -> None: - # Full validate_hex as serialize_block calls it, f-string field name - # included -- the f-string is built per transaction even though it is - # only ever used on the raising path. - for index, value in enumerate(hexes): - validate_hex(value, field_name=f"transaction_hexes[{index}]") - - def real_decode() -> None: - # The list/generator build that produces the transaction bytes. - [from_hex(value) for value in hexes] - - def join_only() -> None: - b"".join(decoded) - - def hex_encode() -> None: - joined.hex() - - def loop_floor() -> None: - for _value in hexes: - pass - - header = inputs["header"] - count_prefix = direct_stratum.compact_size(len(hexes)) - - def floor_single_decode_bytes_out() -> None: - # Counterfactual A: hex inputs kept, but decoded once (no validate_hex - # probe, no .lower() copy) and the block returned as bytes with no - # caller-side .hex() re-encode. - header + count_prefix + b"".join([from_hex(value) for value in hexes]) - - def floor_bytes_in_bytes_out() -> None: - # Counterfactual B: transactions already held as bytes. Pure memcpy - # floor for producing the same block. - header + count_prefix + b"".join(decoded) - - return { - "validate_hex_fromhex_probe": measure(fromhex_probe, **kwargs), - "validate_hex_lower_copy": measure(lower_copy, **kwargs), - "validate_hex_full_calls": measure(validate_hex_calls, **kwargs), - "real_fromhex_decode": measure(real_decode, **kwargs), - "join_concatenation": measure(join_only, **kwargs), - "caller_hex_reencode": measure(hex_encode, **kwargs), - "bare_iteration_floor": measure(loop_floor, **kwargs), - "floor_single_decode_bytes_out": measure(floor_single_decode_bytes_out, **kwargs), - "floor_bytes_in_bytes_out": measure(floor_bytes_in_bytes_out, **kwargs), - } - - -# -------------------------------------------------------------------------- -# anchors -# -------------------------------------------------------------------------- - - -@dataclass -class PrimitiveResult: - name: str - group: str - timing: Timing - note: str = "" - - -def measure_primitives( - scenario: Scenario, *, reps: int, min_batch_seconds: float -) -> list[PrimitiveResult]: - """Time every per-accepted-share primitive on the accept path in isolation. - - The findings comment reports a 12.4 us "primitives, min-of-7" floor but - does not enumerate its primitive set, so this is a reconstruction from the - code path rather than a replay of the original list. Two groups: - ``assemble`` covers the primitives ``assemble_submission`` composes, and - ``accept_path`` covers the per-accepted-share work around it that the - coordinator does for every share (wire decode, request parse/validate, - duplicate reservation, ack encode). - """ - - job = scenario.job - inputs = submission_inputs(job) - header: bytes = inputs["header"] - coinbase_without_witness: bytes = inputs["coinbase_without_witness"] - full_coinbase_hex: str = inputs["full_coinbase_hex"] - extranonce2_hex: str = inputs["extranonce2_hex"] - version_hex: str = inputs["version_hex"] - merkle_branch = list(job.merkle_branch) - transaction_hexes = job.transaction_hexes - - validate_hex = stratum_codec.validate_hex - apply_version_bits = stratum_codec.apply_version_bits - assemble_coinbase = stratum_codec.assemble_coinbase - merkle_from_branch = stratum_codec.compute_merkle_root_from_branch_hex - serialize_header = stratum_codec.serialize_header_from_stratum_fields - strip_witness = direct_stratum.strip_witness_transaction - header_hash_int = stratum_codec.header_hash_int - header_hash_hex = stratum_codec.header_hash_hex - serialize_block = direct_stratum.serialize_block - submission_cls = direct_stratum.DirectQbitSubmission - - coinbase_bytes = assemble_coinbase( - job.coinb1, job.extranonce1_hex, extranonce2_hex, job.coinb2 - ) - merkle_root = merkle_from_branch(coinbase_bytes, merkle_branch) - block_hex = serialize_block(header, full_coinbase_hex, transaction_hexes).hex() - header_hex = header.hex() - - username = "qbit1qperfworker.rig0" - submit_line = json.dumps( - { - "id": 4171, - "method": "mining.submit", - "params": [username, job.job_id, extranonce2_hex, NTIME_HEX, NONCE_HEX], - } - ) - submit_params: list[object] = [ - username, - job.job_id, - extranonce2_hex, - NTIME_HEX, - NONCE_HEX, - ] - request = parse_submit_request(submit_params) - ack = {"id": 4171, "result": True, "error": None} - recent_shares = RecentShareIndex(capacity=50_000) - duplicate_counter = [0] - - def reserve_fresh() -> None: - # A fresh key every call: reserve() on a repeated key short-circuits, - # so replaying one key would measure the duplicate path, not the - # accepted-share path. - duplicate_counter[0] += 1 - recent_shares.reserve((username, f"{header_hex}{duplicate_counter[0]}")) - - kwargs = {"reps": reps, "min_batch_seconds": min_batch_seconds} - specs: list[tuple[str, str, Callable[[], Any], str]] = [ - ( - "validate_hex(extranonce2)", - "assemble", - lambda: validate_hex(extranonce2_hex, field_name="extranonce2"), - "", - ), - ( - "apply_version_bits", - "assemble", - lambda: apply_version_bits(job.version, None, 0), - "no version rolling on this share", - ), - ( - "list(job.merkle_branch)", - "assemble", - lambda: list(job.merkle_branch), - f"{len(merkle_branch)} branch levels", - ), - ( - "assemble_coinbase", - "assemble", - lambda: assemble_coinbase( - job.coinb1, job.extranonce1_hex, extranonce2_hex, job.coinb2 - ), - "", - ), - ( - "compute_merkle_root_from_branch_hex", - "assemble", - lambda: merkle_from_branch(coinbase_bytes, merkle_branch), - f"{len(merkle_branch)} double_sha256 fold levels", - ), - ( - "serialize_header_from_stratum_fields", - "assemble", - lambda: serialize_header( - version_hex=version_hex, - prevhash_hex=job.prevhash, - merkle_root_serialized=merkle_root, - ntime_hex=NTIME_HEX, - nbits_hex=job.nbits, - nonce_hex=NONCE_HEX, - ), - "", - ), - ( - "full_coinbase_hex concat", - "assemble", - lambda: job.full_coinbase_prefix - + job.extranonce1_hex - + extranonce2_hex - + job.full_coinbase_suffix, - "", - ), - ( - "strip_witness_transaction(coinbase) + compare", - "assemble", - lambda: strip_witness(full_coinbase_hex) != coinbase_without_witness, - "", - ), - ("header_hash_int", "assemble", lambda: header_hash_int(header), ""), - ("header_hash_hex", "assemble", lambda: header_hash_hex(header), ""), - ( - "coinbase_without_witness.hex()", - "assemble", - coinbase_without_witness.hex, - "", - ), - ("header.hex()", "assemble", header.hex, ""), - ( - "serialize_block(...).hex()", - "assemble", - lambda: serialize_block(header, full_coinbase_hex, transaction_hexes).hex(), - f"{scenario.transaction_count} tx, {scenario.block_bytes} block bytes", - ), - ( - "DirectQbitSubmission(...)", - "assemble", - lambda: submission_cls( - coinbase_tx_hex=full_coinbase_hex, - coinbase_txid_preimage_hex=header_hex, - header_hex=header_hex, - block_hex=block_hex, - block_hash_hex=header_hex, - block_hash_int=1, - share_pass=True, - block_pass=False, - applied_version_hex=version_hex, - ), - "", - ), - ( - "json.loads(mining.submit line)", - "accept_path", - lambda: json.loads(submit_line), - "", - ), - ( - "parse_submit_request", - "accept_path", - lambda: parse_submit_request(submit_params), - "", - ), - ( - "validate_submit_request", - "accept_path", - lambda: validate_submit_request( - request, - authorized_username=username, - pool_open=True, - extranonce2_size=EXTRANONCE2_SIZE, - ), - "", - ), - ( - "RecentShareIndex.reserve", - "accept_path", - reserve_fresh, - "fresh key per call, includes its own lock", - ), - ("json.dumps(submit ack)", "accept_path", lambda: json.dumps(ack), ""), - ] - - results = [ - PrimitiveResult(name=name, group=group, timing=measure(fn, **kwargs), note=note) - for name, group, fn, note in specs - ] - results.append( - PrimitiveResult( - name="harness floor (lambda: None)", - group="harness", - timing=measure(lambda: None, **kwargs), - note="per-call cost of the measurement loop itself, included in every row above", - ) - ) - return results - - -# -------------------------------------------------------------------------- -# environment -# -------------------------------------------------------------------------- - - -def describe_environment() -> dict[str, Any]: - try: - load = os.getloadavg() - except (OSError, AttributeError): - load = None - gil_disabled = bool(sysconfig.get_config_var("Py_GIL_DISABLED")) - try: - gil_enabled: bool | None = sys._is_gil_enabled() # type: ignore[attr-defined] - except AttributeError: - gil_enabled = None - cpu_brand = platform.processor() - if sys.platform == "darwin": - # platform.processor() is just "arm" on Apple silicon; the sysctl name - # is the informative one and needs no third-party package. - try: - import subprocess - - brand = subprocess.run( - ["sysctl", "-n", "machdep.cpu.brand_string"], - capture_output=True, - text=True, - timeout=5, - ).stdout.strip() - if brand: - cpu_brand = brand - except Exception: # pragma: no cover - informational only - pass - return { - "cpu": cpu_brand, - "machine": platform.machine(), - "platform": platform.platform(), - "cpu_count": os.cpu_count(), - "python_version": sys.version, - "python_implementation": platform.python_implementation(), - "gil_disabled_build": gil_disabled, - "gil_enabled_at_runtime": gil_enabled, - "loadavg_1_5_15": load, - "clock_rationale": CLOCK_RATIONALE, - "clocks": { - name: { - "implementation": info.implementation, - "resolution_s": info.resolution, - } - for name, info in ( - (name, time.get_clock_info(name)) - for name in ("thread_time", "process_time", "perf_counter") - ) - }, - } - - -# -------------------------------------------------------------------------- -# reporting -# -------------------------------------------------------------------------- - - -def _term_scope_note(rows: list[dict[str, Any]]) -> str | None: - """Explain an ``n/a`` term column instead of leaving the reader to guess.""" - - if all(row.get("composite_includes_term", True) for row in rows): - return None - return ( - "`term % of composite` and `composite - term` read **n/a**: on this " - "tree `assemble_submission` serializes the block only for a hash that " - "solved one, so an ordinary share's composite does not contain the " - "term and a ratio between the two is not a share of anything. The " - "term column is still the isolated cost of serializing a block of " - "that size, which is what a block landing pays. For an after-fix " - "delta compare `serialize_term_us` and the composite separately." - ) - - -def _json_number(value: float) -> float | None: - return None if value != value else value - - -def _fmt(value: float | None, digits: int = 2) -> str: - # None/NaN marks a quantity that is undefined on this tree rather than - # merely unmeasured -- printing "nan" invites the reader to treat it as a - # number. Undefined cells render as "n/a" and carry no unit suffix. - if value is None or value != value: - return "n/a" - return f"{value:,.{digits}f}" - - -def _fmt_percent(value: float | None, digits: int = 1) -> str: - text = _fmt(value, digits) - return text if text == "n/a" else f"{text}%" - - -def render_report(results: dict[str, Any]) -> str: - env = results["environment"] - out: list[str] = [] - add = out.append - - add("# serialize_block per-share CPU cost") - add("") - add("## Machine") - add("") - add(f"- CPU: {env['cpu']} ({env['cpu_count']} logical cores)") - add(f"- Platform: {env['platform']}") - add(f"- Python: {env['python_version'].splitlines()[0]} ({env['python_implementation']})") - add( - f"- GIL: build Py_GIL_DISABLED={env['gil_disabled_build']}, " - f"runtime enabled={env['gil_enabled_at_runtime']}" - ) - load = env["loadavg_1_5_15"] - if load: - add(f"- Load average at start (1/5/15 min): {load[0]:.2f} / {load[1]:.2f} / {load[2]:.2f}") - add(f"- Load average at end: {results['loadavg_end']}") - add(f"- Clock: {env['clock_rationale']}") - for name, info in env["clocks"].items(): - add(f" - `{name}`: {info['implementation']}, resolution {info['resolution_s']:.3g} s") - add("") - - anchors = results["anchors"] - add("## Anchors against the issue #143 findings comment") - add("") - add("| anchor | published | this machine | ratio (mine/published) |") - add("|---|---|---|---|") - add( - f"| primitives floor, per accepted share | {PUBLISHED_PRIMITIVES_FLOOR_US} us | " - f"{_fmt(anchors['primitives_floor_us'])} us | " - f"{_fmt(anchors['primitives_floor_ratio'], 3)} |" - ) - add( - f"| 0-tx-template composite `assemble_submission` | {PUBLISHED_ZERO_TX_COMPOSITE_US} us | " - f"{_fmt(anchors['zero_tx_composite_us'])} us | " - f"{_fmt(anchors['zero_tx_composite_ratio'], 3)} |" - ) - add( - f"| composite / primitives inflation ratio | " - f"{_fmt(PUBLISHED_FULL_PATH_ESTIMATE_US / PUBLISHED_PRIMITIVES_FLOOR_US, 3)} (implied " - f"by 15.7/12.4) | {_fmt(anchors['composite_over_assemble_primitives'], 3)} | " - f"{_fmt(anchors['composite_over_assemble_primitives'] / (PUBLISHED_FULL_PATH_ESTIMATE_US / PUBLISHED_PRIMITIVES_FLOOR_US), 3)} |" - ) - add("") - add("Primitive breakdown at the 0-transaction baseline (min-of-7, interpreter CPU):") - add("") - add("| primitive | group | thread CPU us | note |") - add("|---|---|---|---|") - for row in anchors["primitives"]: - add( - f"| `{row['name']}` | {row['group']} | {_fmt(row['timing']['thread_cpu_us'], 3)} | " - f"{row['note']} |" - ) - add( - f"| **sum (assemble group)** | assemble | " - f"**{_fmt(anchors['assemble_primitives_us'], 3)}** | |" - ) - add( - f"| **sum (accept-path group)** | accept_path | " - f"**{_fmt(anchors['accept_path_primitives_us'], 3)}** | |" - ) - add( - f"| **primitives floor (both groups)** | | " - f"**{_fmt(anchors['primitives_floor_us'], 3)}** | |" - ) - add("") - - add("## Axis 1 - total block bytes") - add("") - add( - "| block bytes | tx count | tx size (B) | merkle levels | composite " - "`assemble_submission` us/share | `serialize_block(...).hex()` us/share | " - "term % of composite | composite - term us | composite wall us |" - ) - add("|---|---|---|---|---|---|---|---|---|") - for row in results["axis1"]: - add( - f"| {row['block_bytes']:,} | {row['transaction_count']:,} | " - f"{row['nominal_tx_bytes']} | {row['merkle_branch_levels']} | " - f"{_fmt(row['composite_us'])} | {_fmt(row['serialize_term_us'])} | " - f"{_fmt_percent(row['term_percent'])} | {_fmt(row['composite_less_term_us'])} | " - f"{_fmt(row['composite_wall_us'])} |" - ) - add("") - axis1_note = _term_scope_note(results["axis1"]) - if axis1_note: - add(axis1_note) - add("") - add("Term split: `serialize_block` (bytes out) vs the caller's `.hex()` re-encode.") - add("") - add("| block bytes | `serialize_block` us | `.hex()` re-encode us | sum us | term measured us |") - add("|---|---|---|---|---|") - for row in results["axis1"]: - add( - f"| {row['block_bytes']:,} | {_fmt(row['serialize_block_only_us'])} | " - f"{_fmt(row['hex_encode_only_us'])} | " - f"{_fmt(row['serialize_block_only_us'] + row['hex_encode_only_us'])} | " - f"{_fmt(row['serialize_term_us'])} |" - ) - add("") - - add("## Axis 2 - transaction count at fixed total block bytes") - add("") - add(f"Total block bytes held at ~{AXIS2_TOTAL_BLOCK_BYTES:,}.") - add("") - add( - "| tx count | block bytes | tx size (B) | merkle levels | composite us/share | " - "term us/share | term % of composite |" - ) - add("|---|---|---|---|---|---|---|") - for row in results["axis2"]: - add( - f"| {row['transaction_count']:,} | {row['block_bytes']:,} | " - f"{row['nominal_tx_bytes']} | {row['merkle_branch_levels']} | " - f"{_fmt(row['composite_us'])} | {_fmt(row['serialize_term_us'])} | " - f"{_fmt_percent(row['term_percent'])} |" - ) - add("") - axis2_note = _term_scope_note(results["axis2"]) - if axis2_note: - add(axis2_note) - add("") - fit = results["axis2_fit"] - add( - f"Least-squares fit of the term against transaction count at fixed bytes: " - f"**{_fmt(fit['per_tx_us'], 4)} us/tx** plus a " - f"{_fmt(fit['intercept_us'])} us fixed term (R^2 = {_fmt(fit['r_squared'], 4)}). " - f"Endpoint slope over {fit['endpoint_span'][0]}->{fit['endpoint_span'][1]} tx: " - f"**{_fmt(fit['endpoint_per_tx_us'], 4)} us/tx**." - ) - add("") - - add("## Decomposition of the term") - add("") - for block_label, parts in results["decomposition"].items(): - add(f"### {block_label}") - add("") - add("| part | thread CPU us | % of term |") - add("|---|---|---|") - term = parts["term_us"] - for name, value in parts["parts_us"].items(): - share = 100.0 * value / term if term else float("nan") - add(f"| {name} | {_fmt(value, 3)} | {_fmt(share, 1)}% |") - add(f"| **measured term** | **{_fmt(term, 3)}** | 100% |") - add(f"| residual (term - accounted parts) | {_fmt(parts['residual_us'], 3)} | " - f"{_fmt(100.0 * parts['residual_us'] / term if term else float('nan'), 1)}% |") - add("") - add("Memoranda for the same material (not part of the sum above):") - add("") - add("| memo | thread CPU us | % of term |") - add("|---|---|---|") - for label, value in ( - ( - "`validate_hex` full calls incl. per-tx f-string field name", - parts["validate_hex_full_calls_us"], - ), - ( - "counterfactual A: one decode, bytes out, no re-encode", - parts["floor_single_decode_bytes_out_us"], - ), - ( - "counterfactual B: transactions already bytes, bytes out", - parts["floor_bytes_in_bytes_out_us"], - ), - ("bare iteration over the same list", parts["bare_iteration_floor_us"]), - ): - add( - f"| {label} | {_fmt(value, 3)} | " - f"{_fmt(100.0 * value / term if term else float('nan'), 1)}% |" - ) - add("") - - add("## Capacity") - add("") - add( - "Single-core capacity restated at each Axis 1 point. `mine` is this machine's " - "measured accept-path budget; `published-scale` rescales the findings comment's " - "64k-81k band by the same relative collapse, so it is directly comparable to the " - "number already in the report." - ) - add("") - add( - "| block bytes | accept-path us/share (mine) | shares/s (mine) | " - "published-scale shares/s | % of one core at ~273 shares/s |" - ) - add("|---|---|---|---|---|") - for row in results["capacity"]: - add( - f"| {row['block_bytes']:,} | {_fmt(row['accept_path_us'])} | " - f"{row['shares_per_second_mine']:,.0f} | " - f"{row['published_scale_low']:,.0f}-{row['published_scale_high']:,.0f} | " - f"{_fmt(row['design_point_core_percent'], 2)}% |" - ) - add("") - return "\n".join(out) - - -# -------------------------------------------------------------------------- -# driver -# -------------------------------------------------------------------------- - - -def _linear_fit(xs: Sequence[float], ys: Sequence[float]) -> tuple[float, float, float]: - n = len(xs) - mean_x = sum(xs) / n - mean_y = sum(ys) / n - sxx = sum((x - mean_x) ** 2 for x in xs) - sxy = sum((x - mean_x) * (y - mean_y) for x, y in zip(xs, ys)) - slope = sxy / sxx if sxx else float("nan") - intercept = mean_y - slope * mean_x - ss_tot = sum((y - mean_y) ** 2 for y in ys) - ss_res = sum((y - (intercept + slope * x)) ** 2 for x, y in zip(xs, ys)) - r_squared = 1.0 - ss_res / ss_tot if ss_tot else float("nan") - return slope, intercept, r_squared - - -def run(args: argparse.Namespace) -> dict[str, Any]: - environment = describe_environment() - reps = args.reps - min_batch_seconds = args.min_batch_seconds - kwargs = {"reps": reps, "min_batch_seconds": min_batch_seconds} - - baseline = build_scenario( - label="0tx", - transaction_count=0, - target_block_bytes=None, - construction="repo baseline: job_build_benchmark.base_template() with transactions=[]", - ) - - primitives = measure_primitives(baseline, **kwargs) - assemble_primitives_us = sum( - p.timing.thread_cpu_us for p in primitives if p.group == "assemble" - ) - accept_path_primitives_us = sum( - p.timing.thread_cpu_us for p in primitives if p.group == "accept_path" - ) - primitives_floor_us = assemble_primitives_us + accept_path_primitives_us - - axis1_scenarios: list[Scenario] = [baseline] - for target in AXIS1_TARGET_BLOCK_BYTES: - if target == 0: - continue - payload_estimate = target - HEADER_BYTES - 3 - baseline.coinbase_bytes - count = max(1, round(payload_estimate / AXIS1_NOMINAL_TX_BYTES)) - axis1_scenarios.append( - build_scenario( - label=f"{target}B", - transaction_count=count, - target_block_bytes=target, - construction=( - f"{count} synthetic 1-in/1-out non-witness transactions of " - f"~{AXIS1_NOMINAL_TX_BYTES} B, sized to hit {target} total block bytes exactly" - ), - ) - ) - - axis1_timings = [ - time_scenario(scenario, **kwargs) for scenario in axis1_scenarios - ] - baseline_timing = axis1_timings[0] - - axis2_scenarios = [ - build_scenario( - label=f"1MB-{count}tx", - transaction_count=count, - target_block_bytes=AXIS2_TOTAL_BLOCK_BYTES, - construction=( - f"{count} equal-sized synthetic transactions filling " - f"{AXIS2_TOTAL_BLOCK_BYTES} total block bytes" - ), - ) - for count in AXIS2_TX_COUNTS - ] - axis2_timings = [time_scenario(scenario, **kwargs) for scenario in axis2_scenarios] - - slope, intercept, r_squared = _linear_fit( - [float(t.scenario.transaction_count) for t in axis2_timings], - [t.serialize_term.thread_cpu_us for t in axis2_timings], - ) - first, last = axis2_timings[0], axis2_timings[-1] - endpoint_slope = ( - last.serialize_term.thread_cpu_us - first.serialize_term.thread_cpu_us - ) / (last.scenario.transaction_count - first.scenario.transaction_count) - - decomposition: dict[str, Any] = {} - for timing in axis1_timings: - scenario = timing.scenario - if scenario.transaction_count and scenario.block_bytes not in ( - 1_000_000, - 2_000_000, - ): - continue - parts = decompose_term(scenario, **kwargs) - accounted = { - "validate_hex: bytes.fromhex probe (discarded)": parts[ - "validate_hex_fromhex_probe" - ].thread_cpu_us, - "validate_hex: value.lower() copy": parts["validate_hex_lower_copy"].thread_cpu_us, - "real bytes.fromhex decode + list build": parts["real_fromhex_decode"].thread_cpu_us, - "b''.join concatenation": parts["join_concatenation"].thread_cpu_us, - "caller .hex() re-encode": parts["caller_hex_reencode"].thread_cpu_us, - } - term_us = timing.serialize_term.thread_cpu_us - decomposition[ - f"{scenario.block_bytes:,} block bytes / {scenario.transaction_count:,} tx" - ] = { - "term_us": term_us, - "parts_us": accounted, - "residual_us": term_us - sum(accounted.values()), - "validate_hex_full_calls_us": parts["validate_hex_full_calls"].thread_cpu_us, - "bare_iteration_floor_us": parts["bare_iteration_floor"].thread_cpu_us, - "floor_single_decode_bytes_out_us": parts[ - "floor_single_decode_bytes_out" - ].thread_cpu_us, - "floor_bytes_in_bytes_out_us": parts["floor_bytes_in_bytes_out"].thread_cpu_us, - "raw": {name: timing_.as_json() for name, timing_ in parts.items()}, - } - - # Accept-path budget: the composite at this template size plus the - # per-share accept-path primitives measured around it. The accept-path - # primitives are inflated by the same composite/primitives ratio the - # findings comment used to turn its 12.4 us floor into 15.7 us. - ratio = ( - baseline_timing.composite.thread_cpu_us / assemble_primitives_us - if assemble_primitives_us - else float("nan") - ) - baseline_accept_path_us = ( - baseline_timing.composite.thread_cpu_us + accept_path_primitives_us * ratio - ) - capacity: list[dict[str, Any]] = [] - for timing in axis1_timings: - accept_path_us = timing.composite.thread_cpu_us + accept_path_primitives_us * ratio - collapse = baseline_accept_path_us / accept_path_us if accept_path_us else float("nan") - capacity.append( - { - "block_bytes": timing.scenario.block_bytes, - "transaction_count": timing.scenario.transaction_count, - "accept_path_us": accept_path_us, - "shares_per_second_mine": 1e6 / accept_path_us, - "relative_to_baseline": collapse, - "published_scale_low": PUBLISHED_CAPACITY_LOW_PER_S * collapse, - "published_scale_high": PUBLISHED_CAPACITY_HIGH_PER_S * collapse, - "design_point_core_percent": ( - 100.0 * DESIGN_POINT_SHARES_PER_S * accept_path_us / 1e6 - ), - } - ) - - def scenario_json(timing: ScenarioTiming) -> dict[str, Any]: - scenario = timing.scenario - nominal = ( - scenario.transaction_bytes // scenario.transaction_count - if scenario.transaction_count - else 0 - ) - return { - "label": scenario.label, - "block_bytes": scenario.block_bytes, - "transaction_count": scenario.transaction_count, - "transaction_bytes": scenario.transaction_bytes, - "coinbase_bytes": scenario.coinbase_bytes, - "nominal_tx_bytes": nominal, - "merkle_branch_levels": scenario.merkle_branch_levels, - "construction": scenario.construction, - "composite_us": timing.composite.thread_cpu_us, - "composite_wall_us": timing.composite.wall_us, - "composite_process_us": timing.composite.process_cpu_us, - "serialize_term_us": timing.serialize_term.thread_cpu_us, - "serialize_term_wall_us": timing.serialize_term.wall_us, - "serialize_block_only_us": timing.serialize_block_only.thread_cpu_us, - "hex_encode_only_us": timing.hex_encode_only.thread_cpu_us, - # null rather than NaN: NaN is not valid strict JSON, and the - # quantity is undefined -- not merely unmeasured -- once the share - # path no longer serializes the block. - "composite_includes_term": timing.composite_includes_term, - "term_percent": _json_number(timing.term_share_percent), - "composite_less_term_us": _json_number(timing.composite_less_term_us), - "raw": { - "composite": timing.composite.as_json(), - "serialize_term": timing.serialize_term.as_json(), - "serialize_block_only": timing.serialize_block_only.as_json(), - "hex_encode_only": timing.hex_encode_only.as_json(), - }, - } - - try: - loadavg_end = "%.2f / %.2f / %.2f" % os.getloadavg() - except (OSError, AttributeError): - loadavg_end = "unavailable" - - return { - "environment": environment, - "loadavg_end": loadavg_end, - "methodology": { - "repetitions": reps, - "statistic": "min-of-N over auto-sized batches, reported per call", - "min_batch_seconds": min_batch_seconds, - "clock": "time.thread_time_ns (headline); process_time_ns and perf_counter_ns alongside", - "threads": 1, - "job_construction": ( - "route 1: real DirectQbitStratumJob via " - "direct_stratum.make_job_from_builder_manifest, so " - "merkle_branch_for_coinbase runs strip_witness_transaction over every " - "transaction and the job carries a realistic merkle branch" - ), - "consensus_ceiling_bytes": MAX_BLOCK_BYTES, - }, - "anchors": { - "published_primitives_floor_us": PUBLISHED_PRIMITIVES_FLOOR_US, - "published_zero_tx_composite_us": PUBLISHED_ZERO_TX_COMPOSITE_US, - "primitives_floor_us": primitives_floor_us, - "primitives_floor_ratio": primitives_floor_us / PUBLISHED_PRIMITIVES_FLOOR_US, - "assemble_primitives_us": assemble_primitives_us, - "accept_path_primitives_us": accept_path_primitives_us, - "zero_tx_composite_us": baseline_timing.composite.thread_cpu_us, - "zero_tx_composite_ratio": ( - baseline_timing.composite.thread_cpu_us / PUBLISHED_ZERO_TX_COMPOSITE_US - ), - "composite_over_assemble_primitives": ratio, - "primitives": [ - { - "name": p.name, - "group": p.group, - "note": p.note, - "timing": p.timing.as_json(), - } - for p in primitives - ], - }, - "axis1": [scenario_json(t) for t in axis1_timings], - "axis2": [scenario_json(t) for t in axis2_timings], - "axis2_fit": { - "per_tx_us": slope, - "intercept_us": intercept, - "r_squared": r_squared, - "endpoint_per_tx_us": endpoint_slope, - "endpoint_span": [first.scenario.transaction_count, last.scenario.transaction_count], - "fixed_total_block_bytes": AXIS2_TOTAL_BLOCK_BYTES, - }, - "decomposition": decomposition, - "capacity": capacity, - "design_point_shares_per_second": DESIGN_POINT_SHARES_PER_S, - } - - -def main(argv: list[str] | None = None) -> int: - parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) - parser.add_argument("--json", action="store_true", help="emit the full result set as JSON") - parser.add_argument("--reps", type=int, default=7, help="repetitions per timing (default 7)") - parser.add_argument( - "--min-batch-seconds", - type=float, - default=0.05, - help="target duration of one timing repetition (default 0.05)", - ) - args = parser.parse_args(argv) - - results = run(args) - if args.json: - json.dump(results, sys.stdout, indent=2, default=str) - sys.stdout.write("\n") - else: - sys.stdout.write(render_report(results)) - sys.stdout.write("\n") - return 0 - - -if __name__ == "__main__": - raise SystemExit(main()) diff --git a/tests/perf/stratum_session_concurrency_load.py b/tests/perf/stratum_session_concurrency_load.py deleted file mode 100644 index f0515610d..000000000 --- a/tests/perf/stratum_session_concurrency_load.py +++ /dev/null @@ -1,2028 +0,0 @@ -#!/usr/bin/env python3 -"""Real-thread concurrency load driver for the PRISM Stratum session path. - -Issue #143 asked what the coordinator actually does when many miners are -connected at once. The deterministic baton harness in -``tests/prism_concurrency_harness.py`` cannot answer that: it exists to make -one interleaving reproducible, and it replaces the scheduler, the clock and -the lock to do it. Those are exactly the three things this question is about. -This driver is the other instrument -- **real OS threads, real sockets, real -wall clock, no baton** -- and the two are deliberately kept separate. - -What is under test is the shipped session stack, driven through a real -``PrismCoordinator``: - -=============================== ============================================ -service reached through -=============================== ============================================ -``StratumSessionService`` ``coordinator.accept_loop`` / ``handle_client`` -``VardiffService`` difficulty assignment, retarget and reconnect - resume, via the real submit path -``ShareSubmissionService`` ``mining.submit`` handling end to end -=============================== ============================================ - -Nothing in ``lab/`` is modified, monkeypatched for behaviour, or -reimplemented. The node RPC is a fake (this measures the coordinator, not -qbitd) and the audit-bundle subprocess is the repository's existing counting -fake; every other object on the path is the shipped one. - -Three modes, which answer different questions: - -``paced`` - The deployed design point: each connection submits one share every - ``--share-interval-seconds`` (default 15, matching - ``PRISM_STRATUM_VARDIFF_TARGET_SECONDS``). This is a steady-state - measurement -- what the coordinator costs when miners behave. -``saturating`` - Every connection submits as fast as it is answered. This finds the - ceiling and, more usefully, shows whether sessions can even be - *established* while the share path is busy. -``herd`` - All sessions are dropped at once and immediately reconnect. This is the - reconnect storm. The headline is **not** throughput: it is a per-session - re-establishment census, including the exact number that never came back - inside the window. - -Metrics, per run: cores-used, shares/s, CPU microseconds per share, ack -latency p50/p95/p99/max, coordinator-lock acquisitions and contentions with -hold attribution by call site, watchdog misses, rejections by reason, and a -fixed-work GIL-wait probe. - -``--json`` captures a run; ``--render PATH`` re-renders a captured run without -measuring, so a review never needs to put two load generators on the same -cores to get both output formats. - -READ THE OUTPUT THIS WAY (the human report repeats all of this): - -* **Driver overhead is included.** The miner threads, their sockets and the - lock-attribution wrapper all run in this process and on these cores. This - measures a host running the coordinator *and* its load, not a coordinator - alone. -* **Absolute values are host-dependent.** Core count, load average and the - Python build change every number here. -* **In-memory ledger results omit database wait.** ``--ledger memory`` exists - to isolate interpreter cost. Only ``--ledger postgres`` includes the real - write path. -* **This is evidence, not a CI threshold.** Nothing here should be asserted - against in an automated test. - -On demand only. The filename does not begin with ``test_``, so unittest -discovery never collects it, and it is never imported by the suite. -""" - -from __future__ import annotations - -import argparse -import hashlib -import json -import os -import platform -import socket -import statistics -import sys -import sysconfig -import threading -import time -import traceback -from dataclasses import dataclass, field -from decimal import Decimal -from pathlib import Path -from typing import Any, Callable, Sequence - -if not __package__: - sys.path.insert(0, str(Path(__file__).resolve().parents[2])) - -from lab.auxpow import stratum_codec, vardiff -from lab.prism import direct_stratum -from lab.prism.prism_coordinator import StratumListenerProfile, _ObservedRLock -from lab.prism.share_ledger import SingleWriterShareLedger -from tests import prism_coordinator_test_support as support - - -# -------------------------------------------------------------------------- -# constants and disclosures -# -------------------------------------------------------------------------- - -#: The deployed design point named by issue #143: one share per connection per -#: vardiff target interval. -DEFAULT_SHARE_INTERVAL_SECONDS = 15.0 - -#: Historical calibration from the original #143 run. These pin what the -#: numbers *mean*; they are explicitly NOT host-independent thresholds and -#: nothing in this driver asserts against them. -CALIBRATION = { - "saturating_n1024_settled": "0/5 settled; only 889-944/1024 established within a 50s settle", - "herd_n1024_unreestablished": "660-768/1024 sessions never re-established, with zero rejections and no watchdog miss", - "lock_contention": "0/~48k settled acquisitions rose to 30-82% contended during herd", - "long_holders": "note_tip_work_delivered, accept_loop, schedule_initial_job", -} - -INTERPRETATION_NOTES = ( - "Driver overhead is included: the miner threads, their sockets and the " - "lock-attribution wrapper run in this process and compete for these " - "cores. This is a host running the coordinator and its load, not a " - "coordinator alone.", - "Absolute values are host-dependent. Core count, load average and the " - "Python build change every number here; compare runs on one host, not " - "across hosts.", - "An in-memory ledger omits database wait entirely. --ledger memory " - "isolates interpreter cost; only --ledger postgres exercises the real " - "durable write path.", - "This is evidence, not a CI threshold. Do not assert against these " - "numbers in an automated test.", -) - -#: A share target below this is reached by essentially every nonce, so the -#: driver spends no measurable CPU searching for one. difficulty_target() -#: clamps at 2**256-1, and DIFF1_TARGET / 1e-12 is past that clamp. -FLOOR_DIFFICULTY = Decimal("1e-12") - -CLOCK_RATIONALE = ( - "cores-used = process CPU / wall, both measured across the same window: " - "time.process_time() sums CPU over every thread in this process, so " - "dividing by time.perf_counter() elapsed gives cores actually kept busy. " - "1.00 means one core's worth of work regardless of how many threads ran, " - "which is the GIL-bound signature; values above 1.00 mean real overlap. " - "Wall clock is the headline for latency because a miner waits in wall " - "time, not in CPU time." -) - - -class DriverAbort(RuntimeError): - """The run could not establish a trustworthy measurement environment.""" - - -class LedgerUnavailable(RuntimeError): - """A requested ledger backend is not reachable on this host.""" - - -# -------------------------------------------------------------------------- -# environment -# -------------------------------------------------------------------------- - - -def _cpu_brand() -> str: - brand = platform.processor() - if sys.platform == "darwin": - try: - import subprocess - - probed = subprocess.run( - ["sysctl", "-n", "machdep.cpu.brand_string"], - capture_output=True, - text=True, - timeout=5, - ).stdout.strip() - if probed: - brand = probed - except Exception: # pragma: no cover - informational only - pass - return brand - - -def _loadavg() -> tuple[float, float, float] | None: - try: - return os.getloadavg() - except (OSError, AttributeError): - return None - - -def _peak_rss_mb() -> float | None: - try: - import resource - except ImportError: # pragma: no cover - non-POSIX - return None - usage = resource.getrusage(resource.RUSAGE_SELF).ru_maxrss - # Linux reports kilobytes, macOS reports bytes. - return usage / (1024 * 1024) if sys.platform == "darwin" else usage / 1024 - - -def raise_descriptor_limit(wanted: int) -> dict[str, Any]: - """Raise RLIMIT_NOFILE toward ``wanted``, reporting what was achieved. - - Each connection costs two descriptors in this process (the miner's socket - and the coordinator's accepted socket), so an N=1024 run needs well over - 2,048. A run that silently hits the limit reads as coordinator - backpressure when it is really the harness, so the achieved limit is - always reported and checked against the connection count. - """ - detail: dict[str, Any] = {"requested": wanted} - try: - import resource - except ImportError: # pragma: no cover - non-POSIX - detail["status"] = "unavailable" - return detail - soft, hard = resource.getrlimit(resource.RLIMIT_NOFILE) - detail["soft_before"] = soft - detail["hard"] = hard - target = min(wanted, hard) if hard != resource.RLIM_INFINITY else wanted - if soft < target: - try: - resource.setrlimit(resource.RLIMIT_NOFILE, (target, hard)) - except (ValueError, OSError) as exc: - detail["status"] = f"raise-failed: {exc}" - detail["soft_after"] = soft - return detail - detail["soft_after"] = resource.getrlimit(resource.RLIMIT_NOFILE)[0] - detail["status"] = "ok" - return detail - - -def describe_environment() -> dict[str, Any]: - try: - gil_enabled: bool | None = sys._is_gil_enabled() # type: ignore[attr-defined] - except AttributeError: - gil_enabled = None - return { - "cpu": _cpu_brand(), - "machine": platform.machine(), - "platform": platform.platform(), - "cpu_count": os.cpu_count(), - "python_version": sys.version, - "python_implementation": platform.python_implementation(), - "gil_disabled_build": bool(sysconfig.get_config_var("Py_GIL_DISABLED")), - "gil_enabled_at_runtime": gil_enabled, - "loadavg_1_5_15": _loadavg(), - "clock_rationale": CLOCK_RATIONALE, - "clocks": { - name: { - "implementation": time.get_clock_info(name).implementation, - "resolution_s": time.get_clock_info(name).resolution, - } - for name in ("process_time", "perf_counter", "monotonic") - }, - } - - -# -------------------------------------------------------------------------- -# statistics -# -------------------------------------------------------------------------- - - -def percentile(values: Sequence[float], fraction: float) -> float: - """Nearest-rank percentile. Empty input is undefined, not zero.""" - if not values: - return float("nan") - ordered = sorted(values) - if fraction <= 0: - return ordered[0] - rank = max(1, min(len(ordered), int(round(fraction * len(ordered) + 0.5)))) - return ordered[rank - 1] - - -def latency_summary(values: Sequence[float]) -> dict[str, Any]: - if not values: - return {"count": 0, "p50_ms": None, "p95_ms": None, "p99_ms": None, "max_ms": None, "mean_ms": None} - return { - "count": len(values), - "p50_ms": percentile(values, 0.50) * 1000.0, - "p95_ms": percentile(values, 0.95) * 1000.0, - "p99_ms": percentile(values, 0.99) * 1000.0, - "max_ms": max(values) * 1000.0, - "mean_ms": statistics.fmean(values) * 1000.0, - } - - -# -------------------------------------------------------------------------- -# coordinator-lock hold attribution -# -------------------------------------------------------------------------- - - -class AttributingLock: - """Wrap the shipped ``_ObservedRLock`` and attribute holds to call sites. - - Production deliberately does not instrument hold *duration*: the class - docstring in ``prism_coordinator`` explains that a clock read on every - acquire and release is exactly the fast-path cost that lock exists to - avoid, and that a re-entrant lock needs per-thread depth tracking to - avoid double counting nested holds. Both objections are correct, and this - PR changes no production code -- so the attribution lives here, in the - driver, and is paid for only while measuring. - - Two consequences the report states rather than hides: - - * This wrapper adds a clock read and a frame inspection per acquisition. - That cost lands inside the measurement. ``--no-lock-attribution`` - removes it, so the overhead can be quantified on the same host instead - of argued about. - * Re-entrant acquisitions are tracked per thread by depth, and only the - outermost hold is timed. A nested acquire is counted as an acquisition - but contributes no second hold interval, so hold seconds cannot exceed - wall time per thread. - - The delegate's own ``contention_snapshot`` remains the source of truth for - acquisitions and contentions; this class never recomputes them. - """ - - def __init__(self, delegate: _ObservedRLock, *, attribute: bool = True) -> None: - self._delegate = delegate - self._attribute = attribute - self._state = threading.local() - self._stats_lock = threading.Lock() - self.holds: dict[str, dict[str, float]] = {} - - # -- call-site identification ----------------------------------------- - - @staticmethod - def _call_site(depth: int) -> str: - """Name the shipped function that took the lock. - - Frames inside this module are skipped so the attribution names a - coordinator call site (``accept_loop``, ``note_tip_work_delivered``, - ``schedule_initial_job``, ...) rather than the wrapper. - """ - frame = sys._getframe(depth) - this_file = __file__ - while frame is not None and frame.f_code.co_filename == this_file: - frame = frame.f_back - if frame is None: - return "<unknown>" - module = Path(frame.f_code.co_filename).stem - return f"{module}.{frame.f_code.co_name}" - - def _enter_hold(self, site_depth: int) -> None: - depth = getattr(self._state, "depth", 0) - self._state.depth = depth + 1 - if depth == 0: - self._state.site = self._call_site(site_depth + 1) - self._state.started = time.perf_counter() - - def _exit_hold(self) -> None: - depth = getattr(self._state, "depth", 0) - if depth <= 0: - return - self._state.depth = depth - 1 - if depth != 1: - return - held = time.perf_counter() - getattr(self._state, "started", time.perf_counter()) - site = getattr(self._state, "site", "<unknown>") - with self._stats_lock: - entry = self.holds.get(site) - if entry is None: - entry = {"holds": 0.0, "seconds": 0.0, "max_seconds": 0.0} - self.holds[site] = entry - entry["holds"] += 1 - entry["seconds"] += held - entry["max_seconds"] = max(entry["max_seconds"], held) - - # -- lock protocol ----------------------------------------------------- - - def acquire(self, blocking: bool = True, timeout: float = -1) -> bool: - acquired = self._delegate.acquire(blocking, timeout) - if acquired and self._attribute: - self._enter_hold(1) - return acquired - - def release(self) -> None: - if self._attribute: - self._exit_hold() - self._delegate.release() - - def __enter__(self) -> "AttributingLock": - self._delegate.acquire() - if self._attribute: - self._enter_hold(1) - return self - - def __exit__(self, *_args: object) -> None: - self.release() - - def _is_owned(self) -> bool: - return self._delegate._is_owned() - - def contention_snapshot(self) -> tuple[int, int, float, float]: - return self._delegate.contention_snapshot() - - # -- reporting --------------------------------------------------------- - - def snapshot(self) -> dict[str, Any]: - acquisitions, contentions, wait_sum, wait_max = self._delegate.contention_snapshot() - with self._stats_lock: - sites = [ - { - "site": site, - "holds": int(entry["holds"]), - "hold_seconds": entry["seconds"], - "max_hold_ms": entry["max_seconds"] * 1000.0, - "mean_hold_us": (entry["seconds"] / entry["holds"]) * 1e6 if entry["holds"] else None, - } - for site, entry in self.holds.items() - ] - sites.sort(key=lambda row: row["hold_seconds"], reverse=True) - return { - "acquisitions": acquisitions, - "contentions": contentions, - "contended_percent": (contentions / acquisitions * 100.0) if acquisitions else None, - "wait_seconds_sum": wait_sum, - "wait_seconds_max": wait_max, - "attribution_enabled": self._attribute, - "sites": sites, - } - - -def diff_lock_snapshot(before: dict[str, Any], after: dict[str, Any]) -> dict[str, Any]: - """Attribute only what happened between two snapshots. - - Startup and settle take the lock too; charging their acquisitions to the - measurement window would understate the contended percentage during it. - """ - acquisitions = after["acquisitions"] - before["acquisitions"] - contentions = after["contentions"] - before["contentions"] - prior = {row["site"]: row for row in before["sites"]} - sites = [] - for row in after["sites"]: - was = prior.get(row["site"]) - holds = row["holds"] - (was["holds"] if was else 0) - seconds = row["hold_seconds"] - (was["hold_seconds"] if was else 0.0) - if holds <= 0 and seconds <= 0: - continue - sites.append( - { - "site": row["site"], - "holds": holds, - "hold_seconds": seconds, - "max_hold_ms": row["max_hold_ms"], - "mean_hold_us": (seconds / holds) * 1e6 if holds else None, - } - ) - sites.sort(key=lambda row: row["hold_seconds"], reverse=True) - return { - "acquisitions": acquisitions, - "contentions": contentions, - "contended_percent": (contentions / acquisitions * 100.0) if acquisitions else None, - "wait_seconds_sum": after["wait_seconds_sum"] - before["wait_seconds_sum"], - "wait_seconds_max": after["wait_seconds_max"], - "attribution_enabled": after["attribution_enabled"], - "sites": sites, - } - - -# -------------------------------------------------------------------------- -# controls and probes -# -------------------------------------------------------------------------- - - -def _sha256_work(payload: bytes, rounds: int) -> None: - for _ in range(rounds): - hashlib.sha256(payload).digest() - - -def _python_work(iterations: int) -> int: - total = 0 - for index in range(iterations): - total = (total + index * 7919) & 0xFFFFFFFF - return total - - -def _measure_threaded(work: Callable[[], None], threads: int) -> dict[str, float]: - started = threading.Barrier(threads + 1) - workers = [threading.Thread(target=lambda: (started.wait(), work())) for _ in range(threads)] - for worker in workers: - worker.start() - started.wait() - cpu0, wall0 = time.process_time(), time.perf_counter() - for worker in workers: - worker.join() - cpu = time.process_time() - cpu0 - wall = time.perf_counter() - wall0 - return {"cores_used": cpu / wall if wall > 0 else float("nan"), "wall_s": wall, "cpu_s": cpu} - - -def run_controls(*, threads: int, min_wall_seconds: float) -> dict[str, Any]: - """Positive and negative controls, grown until they clear the wall floor. - - The positive control is a 1 MiB ``hashlib.sha256`` loop, which releases - the GIL and therefore scales; the negative control is a small-buffer - pure-Python loop, which cannot. Together they say whether this host is - capable of showing parallelism at all right now -- a host already - saturated by other work reports a low positive control, and every stage - number measured beside it would be a statement about the host rather than - about the coordinator. - - Thread start/join is charged against the measurement, so a configuration - that finishes too fast reads low for reasons that have nothing to do with - the GIL. Both controls are therefore grown until they clear - ``min_wall_seconds``. - """ - payload = b"\x5a" * (1024 * 1024) - rounds = 64 - while True: - positive = _measure_threaded(lambda: _sha256_work(payload, rounds), threads) - if positive["wall_s"] >= min_wall_seconds or rounds >= 1 << 20: - break - rounds = max(rounds * 2, int(rounds * min_wall_seconds / max(positive["wall_s"], 1e-6)) + 1) - iterations = 200_000 - while True: - negative = _measure_threaded(lambda: _python_work(iterations), threads) - if negative["wall_s"] >= min_wall_seconds or iterations >= 1 << 30: - break - iterations = max(iterations * 2, int(iterations * min_wall_seconds / max(negative["wall_s"], 1e-6)) + 1) - return { - "threads": threads, - "positive_sha256_1mib": positive | {"rounds_per_thread": rounds}, - "negative_pure_python": negative | {"iterations_per_thread": iterations}, - "min_wall_seconds": min_wall_seconds, - } - - -class GilWaitProbe: - """Fixed-work probe: how long does a known unit of Python take under load? - - A single thread repeatedly runs an identical pure-Python work unit. The - unit never changes, so any growth in its wall time is time the thread - spent not holding the GIL. Calibrated on an idle process first, the ratio - ``loaded / idle`` is a direct read of GIL pressure -- and unlike - cores-used it does not need the load to be uniform, because each sample - is independently comparable to the calibration. - """ - - def __init__(self, iterations: int = 40_000) -> None: - self.iterations = iterations - self.samples: list[float] = [] - self.baseline_s: float | None = None - # The probe is a busy loop: it burns most of a core for the whole - # measurement window, and time.process_time() sums every thread. Left - # unaccounted it would land in cores-used and CPU-per-share as if the - # coordinator had spent it. The probe therefore charges itself, using - # per-thread CPU, and the load metrics subtract it. - self.cpu_seconds = 0.0 - self._stop = threading.Event() - self._thread: threading.Thread | None = None - - def calibrate(self, repeats: int = 25) -> float: - timings = [] - for _ in range(repeats): - started = time.perf_counter() - _python_work(self.iterations) - timings.append(time.perf_counter() - started) - self.baseline_s = min(timings) - return self.baseline_s - - def _loop(self) -> None: - cpu0 = time.thread_time() - while not self._stop.is_set(): - started = time.perf_counter() - _python_work(self.iterations) - self.samples.append(time.perf_counter() - started) - # Published every iteration, not just at exit: the load window is - # only part of the probe's lifetime, so callers sample this at the - # window boundaries and subtract the difference. Charging the - # probe's whole run against one window over-subtracts and can - # drive the coordinator's measured CPU to zero. - self.cpu_seconds = time.thread_time() - cpu0 - - def start(self) -> None: - self.samples.clear() - self._stop.clear() - self._thread = threading.Thread(target=self._loop, name="gil-wait-probe", daemon=True) - self._thread.start() - - def stop(self) -> None: - self._stop.set() - if self._thread is not None: - self._thread.join(timeout=10.0) - - def result(self) -> dict[str, Any]: - if not self.samples or self.baseline_s is None: - return {"samples": 0, "baseline_ms": None, "note": "probe did not run"} - median = statistics.median(self.samples) - return { - "samples": len(self.samples), - "iterations_per_sample": self.iterations, - "baseline_ms": self.baseline_s * 1000.0, - "median_ms": median * 1000.0, - "p95_ms": percentile(self.samples, 0.95) * 1000.0, - "max_ms": max(self.samples) * 1000.0, - "stretch_median": median / self.baseline_s, - "stretch_p95": percentile(self.samples, 0.95) / self.baseline_s, - "gil_wait_fraction_median": max(0.0, 1.0 - self.baseline_s / median), - "probe_cpu_seconds": self.cpu_seconds, - } - - -# -------------------------------------------------------------------------- -# ledger selection -# -------------------------------------------------------------------------- - -#: The repository's real-server gate, in the order test/test-prism-postgres-*.sh -#: consult it. QBIT_PRISM_EXTERNAL_PSQL_COMMAND points at an already-running -#: server; the PRISM_* names are the coordinator's own runtime contract. -POSTGRES_GATE_ENV = ( - "QBIT_PRISM_EXTERNAL_PSQL_COMMAND", - "PRISM_POSTGRES_PSQL_COMMAND", - "PRISM_DATABASE_URL", -) - - -def postgres_command_from_gate() -> tuple[str, str] | None: - """Return ``(env_name, psql_command)`` if the real-server gate is set.""" - for name in POSTGRES_GATE_ENV: - value = os.environ.get(name, "").strip() - if not value: - continue - if name == "PRISM_DATABASE_URL": - import shlex - - return name, f"psql {shlex.quote(value)}" - return name, value - return None - - -def build_ledger(mode: str) -> tuple[Any, dict[str, Any]]: - """Build the requested ledger, or explain exactly what is missing. - - Every output identifies the ledger mode, because a shares/s figure means - something entirely different with and without a durable write behind it. - """ - if mode == "memory": - ledger = SingleWriterShareLedger() - return ledger, { - "mode": "memory", - "backend_name": ledger.backend_name, - "includes_database_wait": False, - "detail": "shipped in-memory ledger; isolates interpreter cost and omits all database wait", - } - gate = postgres_command_from_gate() - if gate is None: - raise LedgerUnavailable( - "PostgreSQL mode requires the repository's real-server gate. None of " - + ", ".join(POSTGRES_GATE_ENV) - + " is set. Start a server the way test/test-prism-postgres-ledger.sh does:\n" - " docker run --rm -d --name qbit-prism-perf-postgres " - "-e POSTGRES_USER=qbit -e POSTGRES_PASSWORD=qbit -e POSTGRES_DB=qbit postgres:16-alpine\n" - "then re-run with:\n" - " QBIT_PRISM_EXTERNAL_PSQL_COMMAND='docker exec -i qbit-prism-perf-postgres psql -U qbit -d qbit'" - ) - env_name, psql_command = gate - from lab.prism.share_ledger import ( - WRITER_LEASE_HEARTBEAT_SESSION_PREFIX, - PsqlShareLedger, - ) - import uuid - - ledger = PsqlShareLedger( - psql_command=psql_command, - database_url=os.environ.get("PRISM_DATABASE_URL") or None, - writer_id="prism-perf-driver", - writer_epoch=1, - writer_session_token=f"{WRITER_LEASE_HEARTBEAT_SESSION_PREFIX}{uuid.uuid4().hex}", - initialize_schema=True, - ) - return ledger, { - "mode": "postgres", - "backend_name": ledger.backend_name, - "includes_database_wait": True, - "gate_env": env_name, - "detail": "real PsqlShareLedger through the repository's real-server gate; includes durable write wait", - } - - -# -------------------------------------------------------------------------- -# template sizing -# -------------------------------------------------------------------------- - - -def synthetic_transaction_hex(index: int, payload_bytes: int) -> str: - """A structurally valid non-witness transaction of roughly ``payload_bytes``. - - Template size is an input to this driver because merkle-branch depth and - per-job work both grow with it, and a 0-transaction template flatters the - job-build path in a way no deployed pool would see. - """ - script = (f"{index:08x}" * max(1, payload_bytes // 4))[: max(2, payload_bytes * 2)] - if len(script) % 2: - script += "0" - script_bytes = bytes.fromhex(script) - output_script = "51" - return ( - "01000000" - + "01" - + f"{index + 1:064x}" - + "00000000" - + direct_stratum.compact_size(len(script_bytes)).hex() - + script - + "ffffffff" - + "01" - + (10_000).to_bytes(8, "little").hex() - + direct_stratum.compact_size(len(bytes.fromhex(output_script))).hex() - + output_script - + "00000000" - ) - - -def build_template(transactions: int, transaction_bytes: int) -> dict[str, object]: - template = support.base_template() - template["transactions"] = [ - {"data": synthetic_transaction_hex(index, transaction_bytes)} - for index in range(transactions) - ] - return template - - -# -------------------------------------------------------------------------- -# the rig: one real coordinator behind one real listening socket -# -------------------------------------------------------------------------- - - -@dataclass -class DriverConfig: - connections: int = 64 - mode: str = "paced" - share_interval_seconds: float = DEFAULT_SHARE_INTERVAL_SECONDS - settle_seconds: float = 10.0 - measure_seconds: float = 15.0 - herd_return_seconds: float = 50.0 - ledger_mode: str = "memory" - difficulty_mode: str = "resume" - template_transactions: int = 0 - template_transaction_bytes: int = 250 - lock_attribution: bool = True - control_threads: int = 8 - control_floor_fraction: float = 0.5 - control_min_wall_seconds: float = 0.5 - min_wall_seconds: float = 5.0 - max_rss_mb: float = 8192.0 - max_loadavg_per_core: float = 2.0 - watchdog_timeout_seconds: float = 30.0 - connect_timeout_seconds: float = 20.0 - skip_controls: bool = False - - -class Rig: - """A real ``PrismCoordinator`` on a real socket, with real accept threads. - - The coordinator is assembled by the repository's own - ``prism_coordinator_test_support.coordinator`` scaffold, which is how - every unit test in this tree obtains one without a live qbitd. The - difference here is what is layered on top: the shipped ``_ObservedRLock`` - (the scaffold installs a plain RLock, which would report no contention at - all), a real listening socket, a real accept loop on its own thread, and - real per-connection handler threads spawned by the shipped session - service. - """ - - def __init__(self, config: DriverConfig) -> None: - self.config = config - self.template = build_template( - config.template_transactions, config.template_transaction_bytes - ) - self.server, self.rpc = support.coordinator(template=self.template) - support.install_fake_bundle_builder(self.server) - self.ledger, self.ledger_detail = build_ledger(config.ledger_mode) - self.server.ledger = self.ledger - - base_lock = _ObservedRLock() - self.lock = AttributingLock(base_lock, attribute=config.lock_attribution) - self.server.lock = self.lock - - self._install_rpc_shim() - self._install_difficulty_policy() - - self.server.watchdog_timeout_seconds = config.watchdog_timeout_seconds - self.server.stratum_max_connections = 0 # unbounded; admission is what we measure - self.server.vardiff_resume_enabled = config.difficulty_mode == "resume" - - self.listener_socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM) - self.listener_socket.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) - self.listener_socket.bind(("127.0.0.1", 0)) - self.listener_socket.listen(max(128, min(config.connections * 2, 4096))) - self.listener_socket.settimeout(0.25) - self.port = self.listener_socket.getsockname()[1] - self.profile = StratumListenerProfile( - name="default", - bind="127.0.0.1", - port=self.port, - vardiff_config=self.server.vardiff_config, - share_difficulty=self.server.share_difficulty, - minimum_advertised_difficulty=FLOOR_DIFFICULTY, - heartbeat_name="stratum_accept", - ) - self.accept_errors: list[str] = [] - self.watchdog_misses: dict[str, int] = {} - self._accept_thread: threading.Thread | None = None - self._watchdog_thread: threading.Thread | None = None - self._watchdog_stop = threading.Event() - - # -- collaborators ----------------------------------------------------- - - def _install_rpc_shim(self) -> None: - """Answer the one RPC the session path needs that FakeRpc lacks. - - ``P2mrAddressValidator`` calls ``validateaddress`` for every distinct - payout identity. The shipped validator, its LRU and its singleflight - all still run; only the node answer is synthesized. - """ - inner = self.rpc.call - - def call(method: str, params: list[object] | None = None) -> object: - if method == "validateaddress": - address = str((params or [""])[0]) - digest = hashlib.sha256(address.encode()).hexdigest()[:64] - return {"isvalid": True, "scriptPubKey": "5220" + digest} - return inner(method, params) - - self.rpc.call = call # type: ignore[method-assign] - - def _install_difficulty_policy(self) -> None: - """Set the vardiff band this run measures under. - - ``pinned`` clamps min == max so no nonce search is ever needed; it - isolates coordinator cost but is explicitly not representative, and - the report says so. ``resume`` and ``climb`` both leave a real band - open so ``VardiffService`` retargets for real -- they differ in - whether a reconnecting session may adopt its retained difficulty. - """ - config = self.config - floor = FLOOR_DIFFICULTY - ceiling = floor if config.difficulty_mode == "pinned" else Decimal("1e-6") - self.server.vardiff_config = vardiff.VardiffConfig( - enabled=True, - target_share_interval_seconds=Decimal(str(config.share_interval_seconds)), - min_difficulty=floor, - max_difficulty=ceiling, - retarget_interval_seconds=Decimal("90"), - max_step_factor=Decimal("4"), - startup_difficulty=floor, - max_step_down_factor=Decimal("4"), - ewma_alpha=Decimal("0.4"), - retarget_tolerance=Decimal("0.25"), - ) - self.server.share_difficulty = floor - - # -- lifecycle --------------------------------------------------------- - - def start(self) -> None: - def accept() -> None: - try: - self.server.accept_loop(self.listener_socket, self.profile) - except BaseException: # noqa: BLE001 - recorded, not swallowed - self.accept_errors.append(traceback.format_exc()) - - self._accept_thread = threading.Thread(target=accept, name="prism-accept", daemon=True) - self._accept_thread.start() - - def watch() -> None: - while not self._watchdog_stop.wait(0.5): - for name in self.server._overdue_heartbeats(time.monotonic()): - self.watchdog_misses[name] = self.watchdog_misses.get(name, 0) + 1 - - self._watchdog_thread = threading.Thread(target=watch, name="prism-watchdog-probe", daemon=True) - self._watchdog_thread.start() - - def stop(self) -> None: - self._watchdog_stop.set() - self.server.stop_event.set() - if self._watchdog_thread is not None: - self._watchdog_thread.join(timeout=3.0) - if self._accept_thread is not None: - self._accept_thread.join(timeout=5.0) - try: - self.listener_socket.close() - except OSError: - pass - for client in list(getattr(self.server, "clients", []) or []): - try: - client.sock.close() - except OSError: - pass - - # -- observation ------------------------------------------------------- - - def rejection_counts(self) -> dict[str, int]: - counts = { - reason: int(count) - for reason, count in dict(getattr(self.server, "rejection_counts_by_reason", {})).items() - if count - } - limits = dict(getattr(self.server, "connection_limit_rejection_counts", {}) or {}) - for name, count in limits.items(): - if count: - counts[f"connection-limit:{name}"] = int(count) - return counts - - def vardiff_resume_outcomes(self) -> dict[str, int]: - service = getattr(self.server, "_vardiff_service", None) - counts = getattr(service, "vardiff_resume_outcome_counts", None) - return {name: int(count) for name, count in dict(counts or {}).items()} - - -# -------------------------------------------------------------------------- -# the miner side: one real socket and one real thread per connection -# -------------------------------------------------------------------------- - - -@dataclass -class SessionOutcome: - """What one connection did, kept per session for the herd census.""" - - index: int - established: bool = False - establish_seconds: float | None = None - failure: str | None = None - accepted: int = 0 - rejected: int = 0 - reject_reasons: dict[str, int] = field(default_factory=dict) - ack_latencies: list[float] = field(default_factory=list) - nonce_hashes: int = 0 - - -class MinerSession: - """One Stratum client: real socket, real handshake, real submits. - - Deliberately hand-rolled rather than reusing ``tests/stratum_client.py``: - that client is a diagnostic CLI with per-message reporting, and its - formatting cost would land inside every ack latency measured here. - """ - - def __init__(self, index: int, port: int, config: DriverConfig) -> None: - self.index = index - self.port = port - self.config = config - self.username = f"{support.PAYOUT_ADDRESS}.perf{index}" - self.sock: socket.socket | None = None - self.stream: Any = None - self.extranonce1 = "" - self.extranonce2_size = 8 - self.difficulty = FLOOR_DIFFICULTY - self.job: list[Any] | None = None - self.outcome = SessionOutcome(index=index) - self._request_id = 0 - self._nonce = (index * 0x9E3779B1) & 0xFFFFFFFF - - # -- wire -------------------------------------------------------------- - - def _send(self, message: dict[str, Any]) -> None: - assert self.stream is not None - self.stream.write(json.dumps(message).encode() + b"\n") - self.stream.flush() - - def _read(self) -> dict[str, Any] | None: - assert self.stream is not None - line = self.stream.readline() - if not line: - return None - return json.loads(line) - - def _absorb(self, message: dict[str, Any]) -> None: - """Apply a server notification. Both are load-bearing for submits.""" - method = message.get("method") - if method == "mining.set_difficulty": - params = message.get("params") or [None] - if params[0] is not None: - self.difficulty = Decimal(str(params[0])) - elif method == "mining.notify": - self.job = list(message.get("params") or []) - - def _await_response(self, request_id: int, deadline: float) -> dict[str, Any]: - while True: - if time.monotonic() > deadline: - raise TimeoutError(f"no response to request {request_id}") - message = self._read() - if message is None: - raise ConnectionError("server closed the connection") - if message.get("id") == request_id: - return message - self._absorb(message) - - def _next_id(self) -> int: - self._request_id += 1 - return self._request_id - - # -- lifecycle --------------------------------------------------------- - - def connect(self) -> None: - self.sock = socket.create_connection( - ("127.0.0.1", self.port), timeout=self.config.connect_timeout_seconds - ) - self.sock.setsockopt(socket.IPPROTO_TCP, socket.TCP_NODELAY, 1) - self.sock.settimeout(self.config.connect_timeout_seconds) - self.stream = self.sock.makefile("rwb") - - def establish(self) -> None: - """Subscribe, authorize and wait for the first job. - - "Established" means the coordinator delivered work -- not merely that - the TCP connection was accepted. A session that is accepted but never - receives a job cannot mine, so counting it as established would hide - exactly the failure the herd census exists to find. - """ - started = time.monotonic() - deadline = started + self.config.connect_timeout_seconds - self.connect() - - request_id = self._next_id() - self._send({"id": request_id, "method": "mining.subscribe", "params": ["prism-perf-driver/1"]}) - response = self._await_response(request_id, deadline) - result = response.get("result") or [] - if len(result) >= 3: - self.extranonce1 = str(result[1]) - self.extranonce2_size = int(result[2]) - - request_id = self._next_id() - self._send({"id": request_id, "method": "mining.authorize", "params": [self.username, "x"]}) - response = self._await_response(request_id, deadline) - if response.get("result") is not True: - raise ConnectionError(f"authorize refused: {response.get('error')}") - - while self.job is None: - if time.monotonic() > deadline: - raise TimeoutError("no job delivered") - message = self._read() - if message is None: - raise ConnectionError("server closed before delivering work") - self._absorb(message) - - self.outcome.established = True - self.outcome.establish_seconds = time.monotonic() - started - - def close(self) -> None: - for closer in (self.stream, self.sock): - try: - if closer is not None: - closer.close() - except OSError: - pass - self.stream = None - self.sock = None - - def drop(self) -> None: - """Abandon the connection the way a herd does: close, no goodbye.""" - self.job = None - self.outcome.established = False - self.close() - - # -- work -------------------------------------------------------------- - - def _find_nonce(self) -> tuple[str, str]: - """Return an ``(extranonce2, nonce)`` that clears the share target. - - A real miner searches, so this driver searches too, using the shipped - header assembly and hash. At the floor difficulty the target is the - 2**256-1 clamp and the first candidate always passes, which is the - point: nonce search is not what this instrument is trying to measure, - and the hash count is reported so its cost stays visible. - """ - assert self.job is not None - job_id, prevhash, coinb1, coinb2, branch, version, nbits, ntime = self.job[:8] - target = direct_stratum.difficulty_target(self.difficulty) - extranonce2 = f"{self.index & ((1 << (self.extranonce2_size * 8)) - 1):0{self.extranonce2_size * 2}x}" - for _ in range(1 << 20): - self._nonce = (self._nonce + 1) & 0xFFFFFFFF - nonce_hex = f"{self._nonce:08x}" - _coinbase, header = stratum_codec.assemble_header_from_notify_submit( - coinb1_hex=coinb1, - extranonce1_hex=self.extranonce1, - extranonce2_hex=extranonce2, - coinb2_hex=coinb2, - merkle_branch_hex=list(branch), - version_hex=version, - prevhash_hex=prevhash, - ntime_hex=ntime, - nbits_hex=nbits, - nonce_hex=nonce_hex, - ) - self.outcome.nonce_hashes += 1 - if stratum_codec.header_hash_int(header) <= target: - return extranonce2, nonce_hex - raise RuntimeError("no nonce cleared the share target") - - def submit_share(self) -> None: - """Submit one share and time the acknowledgement.""" - assert self.job is not None - job_id = self.job[0] - ntime = self.job[7] - extranonce2, nonce_hex = self._find_nonce() - request_id = self._next_id() - started = time.perf_counter() - self._send( - { - "id": request_id, - "method": "mining.submit", - "params": [self.username, job_id, extranonce2, ntime, nonce_hex], - } - ) - response = self._await_response(request_id, time.monotonic() + self.config.connect_timeout_seconds) - self.outcome.ack_latencies.append(time.perf_counter() - started) - if response.get("result") is True: - self.outcome.accepted += 1 - return - self.outcome.rejected += 1 - error = response.get("error") or [] - reason = "unknown" - if isinstance(error, list) and len(error) >= 3 and isinstance(error[2], dict): - reason = str(error[2].get("reason_id", "unknown")) - elif isinstance(error, list) and len(error) >= 2: - reason = str(error[1]) - self.outcome.reject_reasons[reason] = self.outcome.reject_reasons.get(reason, 0) + 1 - - -# -------------------------------------------------------------------------- -# run orchestration -# -------------------------------------------------------------------------- - - -def _establish_all( - sessions: Sequence[MinerSession], *, window_seconds: float -) -> dict[str, Any]: - """Bring every session up concurrently and census the result. - - One thread per session, started together, is the point: a serial ramp - would never produce the admission contention this driver exists to - measure. - """ - started = threading.Barrier(len(sessions) + 1) - deadline_wall = time.monotonic() + window_seconds - - def bring_up(session: MinerSession) -> None: - started.wait() - try: - session.establish() - except BaseException as exc: # noqa: BLE001 - the census wants the reason - session.outcome.established = False - session.outcome.failure = f"{type(exc).__name__}: {exc}" - session.close() - - threads = [ - threading.Thread(target=bring_up, args=(session,), name=f"miner-{session.index}", daemon=True) - for session in sessions - ] - for thread in threads: - thread.start() - began = time.perf_counter() - started.wait() - for thread in threads: - thread.join(timeout=max(0.0, deadline_wall - time.monotonic()) + 5.0) - elapsed = time.perf_counter() - began - - established = [s for s in sessions if s.outcome.established] - failures: dict[str, int] = {} - for session in sessions: - if session.outcome.failure: - key = session.outcome.failure.split(":")[0] - failures[key] = failures.get(key, 0) + 1 - times = [s.outcome.establish_seconds for s in established if s.outcome.establish_seconds is not None] - return { - "requested": len(sessions), - "established": len(established), - "not_established": len(sessions) - len(established), - "window_seconds": window_seconds, - "wall_seconds": elapsed, - "failure_kinds": failures, - "establish_latency": latency_summary(times), - } - - -def _run_share_load( - sessions: Sequence[MinerSession], - *, - mode: str, - duration_seconds: float, - share_interval_seconds: float, - probe: "GilWaitProbe | None" = None, -) -> dict[str, Any]: - """Drive shares for ``duration_seconds`` and return window totals.""" - live = [s for s in sessions if s.outcome.established] - if not live: - return {"threads": 0, "note": "no established session could submit"} - stop = threading.Event() - barrier = threading.Barrier(len(live) + 1) - errors: dict[str, int] = {} - errors_lock = threading.Lock() - - def pump(session: MinerSession) -> None: - barrier.wait() - # Stagger paced miners across the interval so they do not all submit - # on the same tick, which would manufacture a convoy that no real - # pool sees. - if mode == "paced": - time.sleep(share_interval_seconds * (session.index % 100) / 100.0) - while not stop.is_set(): - try: - session.submit_share() - except BaseException as exc: # noqa: BLE001 - counted, not swallowed - with errors_lock: - key = type(exc).__name__ - errors[key] = errors.get(key, 0) + 1 - session.outcome.failure = session.outcome.failure or f"{type(exc).__name__}: {exc}" - return - if mode == "paced" and stop.wait(share_interval_seconds): - return - - threads = [ - threading.Thread(target=pump, args=(session,), name=f"pump-{session.index}", daemon=True) - for session in live - ] - for thread in threads: - thread.start() - barrier.wait() - # The probe is sampled on exactly these two lines, so its discount covers - # the measured window and nothing else. Sampling it around this call - # instead would also charge thread creation, which can exceed the window's - # own CPU and zero out the result. - probe_cpu0 = probe.cpu_seconds if probe is not None else 0.0 - cpu0, wall0 = time.process_time(), time.perf_counter() - time.sleep(duration_seconds) - stop.set() - for thread in threads: - thread.join(timeout=30.0) - cpu = time.process_time() - cpu0 - wall = time.perf_counter() - wall0 - probe_cpu = (probe.cpu_seconds - probe_cpu0) if probe is not None else 0.0 - - accepted = sum(s.outcome.accepted for s in live) - rejected = sum(s.outcome.rejected for s in live) - latencies: list[float] = [] - for session in live: - latencies.extend(session.outcome.ack_latencies) - reject_reasons: dict[str, int] = {} - for session in live: - for reason, count in session.outcome.reject_reasons.items(): - reject_reasons[reason] = reject_reasons.get(reason, 0) + count - return { - "threads": len(live), - "wall_seconds": wall, - "process_cpu_seconds": cpu, - "cores_used": cpu / wall if wall > 0 else float("nan"), - "accepted_shares": accepted, - "rejected_shares": rejected, - "shares_per_second": accepted / wall if wall > 0 else float("nan"), - "cpu_us_per_share": (cpu * 1e6 / accepted) if accepted else None, - "ack_latency": latency_summary(latencies), - "reject_reasons": reject_reasons, - "pump_errors": errors, - "nonce_hashes": sum(s.outcome.nonce_hashes for s in live), - "probe_cpu_seconds_window": probe_cpu, - } - - -def _run_herd(rig: Rig, sessions: Sequence[MinerSession], config: DriverConfig) -> dict[str, Any]: - """Drop every session at once, reconnect, and census what came back. - - The headline is the census, not throughput. A reconnect storm that - settles at high shares/s while a third of the pool never gets work back - is a failure, and a throughput-only report would call it a success. - """ - before_lock = rig.lock.snapshot() - live_before = sum(1 for s in sessions if s.outcome.established) - dropped_at = time.perf_counter() - for session in sessions: - session.drop() - drop_seconds = time.perf_counter() - dropped_at - - # Give the coordinator a moment to observe the disconnects before the - # herd returns; a reconnect that races its own teardown measures cleanup, - # not admission. - time.sleep(0.5) - - returning = [MinerSession(s.index, rig.port, config) for s in sessions] - census = _establish_all(returning, window_seconds=config.herd_return_seconds) - after_lock = rig.lock.snapshot() - - per_session = [ - { - "index": session.index, - "established": session.outcome.established, - "establish_seconds": session.outcome.establish_seconds, - "failure": session.outcome.failure, - } - for session in returning - ] - return { - "dropped": live_before, - "drop_seconds": drop_seconds, - "return_window_seconds": config.herd_return_seconds, - "census": census, - "re_established": census["established"], - "never_re_established": census["not_established"], - "lock": diff_lock_snapshot(before_lock, after_lock), - "vardiff_resume_outcomes": rig.vardiff_resume_outcomes(), - "per_session": per_session, - "sessions": returning, - } - - -def _health_abort(config: DriverConfig, controls: dict[str, Any] | None) -> dict[str, Any] | None: - """Decide whether this host can support a trustworthy measurement. - - Everything below is a statement about the *environment*, checked before - any conclusion is drawn from it. If one fails the run reports the abort - and no stage or herd numbers at all -- a number measured on a host that - cannot show parallelism is not a weaker result, it is a different - quantity. - """ - load = _loadavg() - cores = os.cpu_count() or 1 - if load is not None and load[0] / cores > config.max_loadavg_per_core: - return { - "reason": "load-average", - "detail": ( - f"1-minute load average {load[0]:.2f} over {cores} cores is " - f"{load[0] / cores:.2f} per core, above the " - f"{config.max_loadavg_per_core:.2f} guard. Another workload owns " - "this host; every number here would be about that workload." - ), - } - if controls is None: - return None - positive = controls["positive_sha256_1mib"] - negative = controls["negative_pure_python"] - floor = config.control_floor_fraction * controls["threads"] - if positive["cores_used"] < floor: - return { - "reason": "control-floor", - "detail": ( - f"the 1 MiB sha256 positive control reached only " - f"{positive['cores_used']:.2f} cores across {controls['threads']} " - f"threads, below the {floor:.2f} floor " - f"({config.control_floor_fraction:.0%} of thread count). This host " - "cannot currently demonstrate parallelism, so a low cores-used " - "reading from the coordinator would not be evidence about the " - "coordinator." - ), - } - if negative["cores_used"] > 1.5: - return { - "reason": "negative-control", - "detail": ( - f"the pure-Python negative control reached " - f"{negative['cores_used']:.2f} cores, which should pin near 1.00 " - "under a GIL. The measurement model does not describe this " - "interpreter." - ), - } - return None - - -def _discount_probe_cpu(load: dict[str, Any] | None, probe_cpu_seconds: float) -> None: - """Re-derive the load metrics with the probe's own CPU removed. - - The raw figures are kept beside the adjusted ones rather than replaced: a - reader who wants to know what this whole process cost should be able to - see it, and silently rewriting a measured number is how a driver starts - lying. The adjusted values are the headline because the probe is - measurement apparatus, not coordinator work. - """ - if not load or not load.get("threads"): - return - wall = float(load.get("wall_seconds") or 0.0) - raw_cpu = float(load.get("process_cpu_seconds") or 0.0) - load["probe_cpu_seconds"] = probe_cpu_seconds - load["process_cpu_seconds_raw"] = raw_cpu - load["cores_used_raw"] = load.get("cores_used") - adjusted = raw_cpu - probe_cpu_seconds - if adjusted < 0.0: - # Never silently report zero work: a negative discount means the - # probe accounting is wrong, and that is a defect in this driver - # rather than a coordinator that used no CPU. - load["probe_discount_error"] = ( - f"probe CPU {probe_cpu_seconds:.3f}s exceeded window CPU " - f"{raw_cpu:.3f}s; reporting the undiscounted figures" - ) - load["probe_cpu_seconds"] = probe_cpu_seconds - return - adjusted = max(0.0, adjusted) - load["process_cpu_seconds"] = adjusted - load["cores_used"] = adjusted / wall if wall > 0 else float("nan") - accepted = int(load.get("accepted_shares") or 0) - load["cpu_us_per_share"] = (adjusted * 1e6 / accepted) if accepted else None - - -def run(config: DriverConfig) -> dict[str, Any]: - """Execute one measurement and return the full captured result.""" - environment = describe_environment() - descriptors = raise_descriptor_limit(max(4096, config.connections * 4)) - results: dict[str, Any] = { - "environment": environment, - "descriptors": descriptors, - "config": { - "connections": config.connections, - "mode": config.mode, - "share_interval_seconds": config.share_interval_seconds, - "settle_seconds": config.settle_seconds, - "measure_seconds": config.measure_seconds, - "herd_return_seconds": config.herd_return_seconds, - "ledger_mode": config.ledger_mode, - "difficulty_mode": config.difficulty_mode, - "template_transactions": config.template_transactions, - "template_transaction_bytes": config.template_transaction_bytes, - "lock_attribution": config.lock_attribution, - "control_floor_fraction": config.control_floor_fraction, - "min_wall_seconds": config.min_wall_seconds, - }, - "calibration": CALIBRATION, - "interpretation": list(INTERPRETATION_NOTES), - "aborted": None, - } - - controls = ( - None - if config.skip_controls - else run_controls( - threads=config.control_threads, min_wall_seconds=config.control_min_wall_seconds - ) - ) - results["controls"] = controls - - abort = _health_abort(config, controls) - if abort is not None: - results["aborted"] = abort - results["loadavg_end"] = _loadavg() - results["peak_rss_mb"] = _peak_rss_mb() - return results - - soft = descriptors.get("soft_after") or 0 - if soft and soft < config.connections * 2 + 64: - results["aborted"] = { - "reason": "descriptor-limit", - "detail": ( - f"RLIMIT_NOFILE is {soft}, below the {config.connections * 2 + 64} " - f"this run needs for {config.connections} connections. Sessions " - "would fail to establish for harness reasons and read as " - "coordinator backpressure." - ), - } - results["loadavg_end"] = _loadavg() - results["peak_rss_mb"] = _peak_rss_mb() - return results - - probe = GilWaitProbe() - probe.calibrate() - - rig = Rig(config) - results["ledger"] = rig.ledger_detail - rig.start() - sessions = [MinerSession(index, rig.port, config) for index in range(config.connections)] - try: - settle = _establish_all(sessions, window_seconds=max(config.settle_seconds, 1.0)) - results["settle"] = settle - - lock_before = rig.lock.snapshot() - probe.start() - if config.mode == "herd": - herd = _run_herd(rig, sessions, config) - returning = herd.pop("sessions") - load = _run_share_load( - returning, - mode="paced", - duration_seconds=config.measure_seconds, - share_interval_seconds=config.share_interval_seconds, - probe=probe, - ) - results["herd"] = herd - results["load"] = load - else: - results["load"] = _run_share_load( - sessions, - mode=config.mode, - duration_seconds=config.measure_seconds, - share_interval_seconds=config.share_interval_seconds, - probe=probe, - ) - probe.stop() - lock_after = rig.lock.snapshot() - - _discount_probe_cpu( - results.get("load"), - float((results.get("load") or {}).get("probe_cpu_seconds_window") or 0.0), - ) - results["gil_wait_probe"] = probe.result() - results["lock"] = diff_lock_snapshot(lock_before, lock_after) - results["lock_total"] = lock_after - results["rejections"] = rig.rejection_counts() - results["watchdog_misses"] = dict(rig.watchdog_misses) - results["accept_errors"] = rig.accept_errors[:3] - results["vardiff_resume_outcomes"] = rig.vardiff_resume_outcomes() - finally: - probe.stop() - for session in sessions: - session.close() - rig.stop() - - peak_rss = _peak_rss_mb() - results["peak_rss_mb"] = peak_rss - results["loadavg_end"] = _loadavg() - if peak_rss is not None and peak_rss > config.max_rss_mb: - results["memory_guard"] = { - "exceeded": True, - "peak_rss_mb": peak_rss, - "limit_mb": config.max_rss_mb, - "detail": "peak RSS passed the guard; results may include swap wait", - } - else: - results["memory_guard"] = {"exceeded": False, "peak_rss_mb": peak_rss, "limit_mb": config.max_rss_mb} - - measured_wall = float(results.get("load", {}).get("wall_seconds") or 0.0) - if measured_wall and measured_wall < config.min_wall_seconds: - results["wall_floor_warning"] = ( - f"measurement window was {measured_wall:.2f}s, below the " - f"{config.min_wall_seconds:.2f}s floor; thread start/join is charged " - "against it and cores-used reads low for that reason alone" - ) - return results - - -# -------------------------------------------------------------------------- -# reporting -# -------------------------------------------------------------------------- - - -def _fmt(value: Any, digits: int = 2, suffix: str = "") -> str: - if value is None: - return "n/a" - if isinstance(value, float) and value != value: - return "n/a" - if isinstance(value, (int, float)): - return f"{value:,.{digits}f}{suffix}" - return str(value) - - -def _rule(char: str = "=") -> str: - return char * 78 - - -def render_report(results: dict[str, Any]) -> str: - out: list[str] = [] - add = out.append - env = results["environment"] - config = results["config"] - - add(_rule()) - add("PRISM Stratum session concurrency load (real threads, real sockets)") - add(_rule()) - add("") - add(f"CPU {env['cpu']} x{env['cpu_count']} logical") - add(f"Platform {env['platform']}") - add(f"Python {env['python_version'].splitlines()[0]}") - add(f"GIL build Py_GIL_DISABLED={env['gil_disabled_build']}, runtime enabled={env['gil_enabled_at_runtime']}") - load = env["loadavg_1_5_15"] - if load: - add(f"Load at start {load[0]:.2f} / {load[1]:.2f} / {load[2]:.2f}") - if results.get("loadavg_end"): - end = results["loadavg_end"] - add(f"Load at end {end[0]:.2f} / {end[1]:.2f} / {end[2]:.2f}") - add(f"Peak RSS {_fmt(results.get('peak_rss_mb'), 1, ' MB')}") - descriptors = results.get("descriptors", {}) - add(f"Descriptors soft={descriptors.get('soft_after')} (was {descriptors.get('soft_before')}), status={descriptors.get('status')}") - add("") - - ledger = results.get("ledger") - add("RUN") - add(f" mode {config['mode']}") - add(f" connections {config['connections']:,}") - if ledger: - add(f" LEDGER MODE {ledger['mode']} ({ledger['backend_name']}) -- database wait included: {ledger['includes_database_wait']}") - else: - add(f" LEDGER MODE {config['ledger_mode']} (not constructed; run aborted before the rig was built)") - add(f" difficulty mode {config['difficulty_mode']}") - add(f" share interval {config['share_interval_seconds']:g}s") - add(f" template {config['template_transactions']:,} transactions x {config['template_transaction_bytes']:,} bytes") - add(f" settle / measure {config['settle_seconds']:g}s / {config['measure_seconds']:g}s") - add(f" lock attribution {'on' if config['lock_attribution'] else 'off'}") - add("") - add(f"Clock {env['clock_rationale']}") - add("") - - controls = results.get("controls") - add(_rule("-")) - add("CONTROLS (this host, this process, immediately before the run)") - add(_rule("-")) - if controls is None: - add(" skipped (--skip-controls): no evidence this host can show parallelism") - else: - positive = controls["positive_sha256_1mib"] - negative = controls["negative_pure_python"] - add(f" positive: 1 MiB sha256 x{controls['threads']} threads {positive['cores_used']:.2f} cores over {positive['wall_s']:.2f}s") - add(f" negative: pure-Python x{controls['threads']} threads {negative['cores_used']:.2f} cores over {negative['wall_s']:.2f}s") - add(f" floor required: {config['control_floor_fraction'] * controls['threads']:.2f} cores") - add("") - - if results.get("aborted"): - abort = results["aborted"] - add(_rule()) - add(f"ABORTED: {abort['reason']}") - add(_rule()) - add("") - for line in _wrap(abort["detail"]): - add(f" {line}") - add("") - add(" No stage, load or herd conclusions are reported: the run could not") - add(" establish a trustworthy measurement environment, so there is nothing") - add(" here that would be evidence about the coordinator.") - add("") - return "\n".join(out) - - settle = results.get("settle") - if settle: - add(_rule("-")) - add("SETTLE: session establishment") - add(_rule("-")) - add(f" requested {settle['requested']:,}") - add(f" established {settle['established']:,}") - add(f" NOT established {settle['not_established']:,}") - add(f" wall {settle['wall_seconds']:.2f}s (window {settle['window_seconds']:g}s)") - latency = settle["establish_latency"] - if latency["count"]: - add(f" establish latency p50 {_fmt(latency['p50_ms'])}ms p95 {_fmt(latency['p95_ms'])}ms p99 {_fmt(latency['p99_ms'])}ms max {_fmt(latency['max_ms'])}ms") - if settle["failure_kinds"]: - add(f" failure kinds {settle['failure_kinds']}") - add("") - - herd = results.get("herd") - if herd: - add(_rule("-")) - add("HERD: reconnect census (the headline, not throughput)") - add(_rule("-")) - census = herd["census"] - add(f" dropped together {herd['dropped']:,} in {herd['drop_seconds'] * 1000:.0f}ms") - add(f" re-established {herd['re_established']:,} / {census['requested']:,}") - add(f" NEVER re-established {herd['never_re_established']:,} (within {herd['return_window_seconds']:g}s)") - latency = census["establish_latency"] - if latency["count"]: - add(f" return latency p50 {_fmt(latency['p50_ms'])}ms p95 {_fmt(latency['p95_ms'])}ms p99 {_fmt(latency['p99_ms'])}ms max {_fmt(latency['max_ms'])}ms") - if census["failure_kinds"]: - add(f" failure kinds {census['failure_kinds']}") - if herd.get("vardiff_resume_outcomes"): - add(f" vardiff resume {herd['vardiff_resume_outcomes']}") - herd_lock = herd.get("lock") or {} - add(f" lock during herd {herd_lock.get('acquisitions', 0):,} acquisitions, {_fmt(herd_lock.get('contended_percent'), 1)}% contended") - add(f" lock wait total / max {_fmt(herd_lock.get('wait_seconds_sum'), 3)}s / {_fmt((herd_lock.get('wait_seconds_max') or 0) * 1000)}ms") - for line in _render_hold_sites(herd_lock, indent=" "): - add(line) - add("") - - load_result = results.get("load") or {} - if load_result.get("threads"): - add(_rule("-")) - add(f"LOAD: {config['mode']} share submission") - add(_rule("-")) - add(f" submitting sessions {load_result['threads']:,}") - add(f" wall {load_result['wall_seconds']:.2f}s") - add(f" cores-used {_fmt(load_result['cores_used'])} (of {env['cpu_count']} logical, GIL probe discounted)") - if load_result.get("cores_used_raw") is not None: - add( - f" whole process {_fmt(load_result['cores_used_raw'])} " - f"including {_fmt(load_result.get('probe_cpu_seconds'), 2, 's')} of GIL-probe CPU" - ) - add(f" accepted shares {load_result['accepted_shares']:,}") - add(f" rejected shares {load_result['rejected_shares']:,}") - add(f" shares/s {_fmt(load_result['shares_per_second'])}") - add(f" CPU us/share {_fmt(load_result['cpu_us_per_share'])} (GIL probe discounted)") - ack = load_result["ack_latency"] - add(f" ack latency p50 {_fmt(ack['p50_ms'])}ms p95 {_fmt(ack['p95_ms'])}ms p99 {_fmt(ack['p99_ms'])}ms max {_fmt(ack['max_ms'])}ms") - add(f" nonce hashes {load_result['nonce_hashes']:,} (driver-side search cost, included above)") - if load_result["reject_reasons"]: - add(f" reject reasons {load_result['reject_reasons']}") - if load_result["pump_errors"]: - add(f" submit errors {load_result['pump_errors']}") - add("") - elif load_result: - add(_rule("-")) - add(f"LOAD: {config['mode']} share submission") - add(_rule("-")) - add(f" {load_result.get('note', 'no load ran')}") - add("") - - lock = results.get("lock") - if lock: - add(_rule("-")) - add("COORDINATOR LOCK (measurement window only)") - add(_rule("-")) - add(f" acquisitions {lock['acquisitions']:,}") - add(f" contentions {lock['contentions']:,} ({_fmt(lock['contended_percent'], 1)}%)") - add(f" wait total / max {_fmt(lock['wait_seconds_sum'], 3)}s / {_fmt(lock['wait_seconds_max'] * 1000)}ms") - for line in _render_hold_sites(lock, indent=""): - add(line) - add("") - total = results.get("lock_total") - if total: - add(" Whole run, including settle and any herd -- this is where the") - add(" establishment path (accept_loop, schedule_initial_job) shows up:") - add(f" acquisitions {total['acquisitions']:,}") - add(f" contentions {total['contentions']:,} ({_fmt(total['contended_percent'], 1)}%)") - for line in _render_hold_sites(total, indent=" "): - add(line) - add("") - - probe = results.get("gil_wait_probe") - if probe and probe.get("samples"): - add(_rule("-")) - add("GIL-WAIT PROBE (fixed work unit, one dedicated thread)") - add(_rule("-")) - add(f" samples {probe['samples']:,} x {probe['iterations_per_sample']:,} iterations") - add(f" idle baseline {_fmt(probe['baseline_ms'], 3)}ms") - add(f" under load median {_fmt(probe['median_ms'], 3)}ms p95 {_fmt(probe['p95_ms'], 3)}ms max {_fmt(probe['max_ms'], 3)}ms") - add(f" stretch median {_fmt(probe['stretch_median'])}x p95 {_fmt(probe['stretch_p95'])}x") - add(f" GIL wait (median) {_fmt(probe['gil_wait_fraction_median'] * 100, 1)}% of the unit's wall time") - add("") - - add(_rule("-")) - add("HEALTH") - add(_rule("-")) - add(f" watchdog misses {results.get('watchdog_misses') or 'none'}") - add(f" rejections {results.get('rejections') or 'none'}") - if results.get("vardiff_resume_outcomes"): - add(f" vardiff resume {results['vardiff_resume_outcomes']}") - guard = results.get("memory_guard") or {} - add(f" peak RSS {_fmt(guard.get('peak_rss_mb'), 1, ' MB')} (guard {_fmt(guard.get('limit_mb'), 0, ' MB')}){' EXCEEDED' if guard.get('exceeded') else ''}") - if results.get("wall_floor_warning"): - add(f" wall floor {results['wall_floor_warning']}") - if results.get("accept_errors"): - add(f" accept errors {len(results['accept_errors'])} (first shown in JSON)") - add("") - - add(_rule("-")) - add("HOW TO READ THIS") - add(_rule("-")) - for note in results.get("interpretation", INTERPRETATION_NOTES): - for index, line in enumerate(_wrap(note)): - add(f" {'* ' if index == 0 else ' '}{line}") - add("") - add(" Historical calibration from the original #143 run, which pins what") - add(" these numbers mean and is NOT a threshold for this host:") - for key, value in results.get("calibration", CALIBRATION).items(): - for index, line in enumerate(_wrap(f"{key}: {value}", width=68)): - add(f" {'- ' if index == 0 else ' '}{line}") - add("") - return "\n".join(out) - - -def _render_hold_sites(lock: dict[str, Any], *, indent: str = "") -> list[str]: - """Render the per-call-site hold table, or say why there isn't one.""" - if not lock: - return [] - if not lock.get("attribution_enabled", True): - return [f"{indent} hold attribution off (--no-lock-attribution)"] - sites = lock.get("sites") or [] - if not sites: - return [f"{indent} hold attribution no holds recorded in this window"] - lines = [ - "", - f"{indent} hold attribution by call site (top 8 by total hold):", - f"{indent} {'call site':<54}{'holds':>9}{'total s':>10}{'mean us':>10}{'max ms':>9}", - ] - for row in sites[:8]: - lines.append( - f"{indent} {row['site'][:53]:<54}{row['holds']:>9,}" - f"{row['hold_seconds']:>10.3f}{_fmt(row['mean_hold_us'], 1):>10}" - f"{_fmt(row['max_hold_ms'], 1):>9}" - ) - return lines - - -def _wrap(text: str, width: int = 72) -> list[str]: - words = text.split() - lines: list[str] = [] - current: list[str] = [] - for word in words: - if current and sum(len(w) + 1 for w in current) + len(word) > width: - lines.append(" ".join(current)) - current = [word] - else: - current.append(word) - if current: - lines.append(" ".join(current)) - return lines or [""] - - -# -------------------------------------------------------------------------- -# self-test -# -------------------------------------------------------------------------- - - -def _selftest(argv_json_path: str | None = None) -> int: - """Smoke the driver itself at a size that fits any host. - - Not a unit test and not collected by discovery: this proves the - instrument runs, captures, replays, and refuses to report conclusions - after an abort. It is the check to run before trusting a real sweep. - """ - failures: list[str] = [] - - def check(name: str, condition: bool, detail: str = "") -> None: - status = "PASS" if condition else "FAIL" - print(f" [{status}] {name}{'' if condition else f' -- {detail}'}", flush=True) - if not condition: - failures.append(name) - - print("selftest: discovery safety", flush=True) - check( - "filename is not collected by unittest discovery", - not Path(__file__).name.startswith("test"), - f"{Path(__file__).name} would match the default test*.py pattern", - ) - - base = dict( - connections=4, - settle_seconds=10.0, - measure_seconds=2.0, - herd_return_seconds=10.0, - min_wall_seconds=0.0, - control_threads=2, - control_min_wall_seconds=0.2, - control_floor_fraction=0.0, - max_loadavg_per_core=1e9, - connect_timeout_seconds=20.0, - ) - - captured: dict[str, Any] | None = None - for mode in ("paced", "saturating", "herd"): - print(f"selftest: {mode} mode at N=4", flush=True) - config = DriverConfig(mode=mode, share_interval_seconds=0.5, **base) - try: - result = run(config) - except BaseException as exc: # noqa: BLE001 - the smoke wants the reason - check(f"{mode} run completes", False, f"{type(exc).__name__}: {exc}") - continue - check(f"{mode} run completes", True) - check(f"{mode} was not aborted", not result.get("aborted"), str(result.get("aborted"))) - check(f"{mode} established every session", result["settle"]["established"] == 4, str(result["settle"])) - check(f"{mode} identifies the ledger mode", bool(result.get("ledger", {}).get("mode"))) - check(f"{mode} recorded lock acquisitions", (result.get("lock") or {}).get("acquisitions", 0) > 0) - if mode == "herd": - check("herd reports a per-session census", len(result["herd"]["per_session"]) == 4) - check( - "herd reports the exact count that did not return", - isinstance(result["herd"]["never_re_established"], int), - ) - else: - check(f"{mode} accepted shares", result["load"]["accepted_shares"] > 0, str(result["load"])) - check(f"{mode} renders", bool(render_report(result))) - captured = result - - print("selftest: JSON capture and render replay", flush=True) - if captured is not None: - path = argv_json_path or os.path.join( - os.environ.get("TMPDIR", "/tmp"), "prism_session_load_selftest.json" - ) - with open(path, "w", encoding="utf-8") as handle: - json.dump(captured, handle, indent=2, default=str) - with open(path, encoding="utf-8") as handle: - replayed = json.load(handle) - first = render_report(replayed) - second = render_report(json.loads(json.dumps(replayed, default=str))) - check("render replay is byte-identical across reloads", first == second) - check("render replay names the ledger mode", "LEDGER MODE" in first) - print(f" capture written to {path}", flush=True) - - print("selftest: control-floor abort", flush=True) - abort_config = DriverConfig( - connections=2, - measure_seconds=1.0, - control_threads=2, - control_min_wall_seconds=0.2, - # Arithmetically unreachable rather than merely demanding: 2 threads - # cannot exceed 2.00 cores-used, so a floor of 5x thread count can - # never be met on any host. A floor that a fast idle machine could - # occasionally clear would make this smoke pass or fail with load. - control_floor_fraction=5.0, - max_loadavg_per_core=1e9, - ) - aborted = run(abort_config) - check("aborts when the control floor is not met", bool(aborted.get("aborted")), str(aborted.get("aborted"))) - if aborted.get("aborted"): - check("abort reason is the control floor", aborted["aborted"]["reason"] == "control-floor") - report = render_report(aborted) - check("abort report says ABORTED", "ABORTED" in report) - for forbidden in ("SETTLE:", "LOAD:", "HERD:", "COORDINATOR LOCK"): - check(f"abort report omits {forbidden!r}", forbidden not in report) - - print("", flush=True) - if failures: - print(f"selftest FAILED: {len(failures)} check(s): {failures}", flush=True) - return 1 - print("selftest PASSED", flush=True) - return 0 - - -# -------------------------------------------------------------------------- -# entry point -# -------------------------------------------------------------------------- - - -def build_parser() -> argparse.ArgumentParser: - parser = argparse.ArgumentParser( - description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter - ) - parser.add_argument("--json", action="store_true", help="emit the full result set as JSON") - parser.add_argument( - "--render", - metavar="PATH", - default=None, - help=( - "render a previously captured --json file instead of measuring. " - "Running the sweep twice to get both formats would put two load " - "generators on the same cores and corrupt both." - ), - ) - parser.add_argument("--selftest", action="store_true", help="smoke the driver itself and exit") - parser.add_argument("--connections", type=int, default=64, help="connection count (default 64; supports 1024+)") - parser.add_argument( - "--mode", - choices=("paced", "saturating", "herd"), - default="paced", - help="paced: one share per interval per connection (the design point); " - "saturating: submit as fast as answered; herd: drop all and census the return", - ) - parser.add_argument( - "--share-interval-seconds", - type=float, - default=DEFAULT_SHARE_INTERVAL_SECONDS, - help=f"paced submit interval (default {DEFAULT_SHARE_INTERVAL_SECONDS:g}, the vardiff target)", - ) - parser.add_argument("--settle-seconds", type=float, default=50.0, help="session establishment window (default 50)") - parser.add_argument("--measure-seconds", type=float, default=30.0, help="measurement window (default 30)") - parser.add_argument("--herd-return-seconds", type=float, default=50.0, help="herd re-establishment window (default 50)") - parser.add_argument( - "--ledger", - dest="ledger_mode", - choices=("memory", "postgres"), - default="memory", - help="memory isolates interpreter cost; postgres uses the repository's real-server gate", - ) - parser.add_argument( - "--difficulty-mode", - choices=("resume", "climb", "pinned"), - default="resume", - help="resume: reconnects may adopt their retained difficulty (shipped default); " - "climb: every reconnect retargets from the floor; " - "pinned: min==max, isolation only and not representative", - ) - parser.add_argument("--template-transactions", type=int, default=0, help="synthetic transactions in the template (default 0)") - parser.add_argument("--template-transaction-bytes", type=int, default=250, help="approximate bytes per synthetic transaction") - parser.add_argument( - "--no-lock-attribution", - dest="lock_attribution", - action="store_false", - help="skip per-call-site hold timing, so its own overhead can be quantified", - ) - parser.add_argument("--control-threads", type=int, default=8, help="threads for the positive/negative controls") - parser.add_argument( - "--control-floor-fraction", - type=float, - default=0.5, - help="the positive control must reach this fraction of control-thread count in cores-used, else abort", - ) - parser.add_argument("--control-min-wall-seconds", type=float, default=0.5, help="wall floor per control measurement") - parser.add_argument("--min-wall-seconds", type=float, default=5.0, help="warn below this measurement wall clock") - parser.add_argument("--max-rss-mb", type=float, default=8192.0, help="peak RSS guard") - parser.add_argument( - "--max-loadavg-per-core", - type=float, - default=2.0, - help="abort if the 1-minute load average per core exceeds this before measuring", - ) - parser.add_argument("--connect-timeout-seconds", type=float, default=20.0, help="per-connection socket timeout") - parser.add_argument("--skip-controls", action="store_true", help="skip controls (removes the parallelism evidence)") - return parser - - -def main(argv: list[str] | None = None) -> int: - args = build_parser().parse_args(argv) - - if args.selftest: - return _selftest() - - if args.render: - with open(args.render, encoding="utf-8") as handle: - sys.stdout.write(render_report(json.load(handle))) - sys.stdout.write("\n") - return 0 - - config = DriverConfig( - connections=args.connections, - mode=args.mode, - share_interval_seconds=args.share_interval_seconds, - settle_seconds=args.settle_seconds, - measure_seconds=args.measure_seconds, - herd_return_seconds=args.herd_return_seconds, - ledger_mode=args.ledger_mode, - difficulty_mode=args.difficulty_mode, - template_transactions=args.template_transactions, - template_transaction_bytes=args.template_transaction_bytes, - lock_attribution=args.lock_attribution, - control_threads=args.control_threads, - control_floor_fraction=args.control_floor_fraction, - control_min_wall_seconds=args.control_min_wall_seconds, - min_wall_seconds=args.min_wall_seconds, - max_rss_mb=args.max_rss_mb, - max_loadavg_per_core=args.max_loadavg_per_core, - connect_timeout_seconds=args.connect_timeout_seconds, - skip_controls=args.skip_controls, - ) - try: - results = run(config) - except LedgerUnavailable as exc: - sys.stderr.write(f"{exc}\n") - return 2 - - if args.json: - json.dump(results, sys.stdout, indent=2, default=str) - sys.stdout.write("\n") - else: - sys.stdout.write(render_report(results)) - sys.stdout.write("\n") - return 0 - - -if __name__ == "__main__": - raise SystemExit(main()) diff --git a/tests/perf/window_builder_pipe_latency.py b/tests/perf/window_builder_pipe_latency.py deleted file mode 100644 index d6c0ade6c..000000000 --- a/tests/perf/window_builder_pipe_latency.py +++ /dev/null @@ -1,951 +0,0 @@ -#!/usr/bin/env python3 -"""Builder pipe transport latency: fixed polling sleeps vs readiness waits. - -Issue #236's cold window builds spent 47.5-48.8 s of a 54-56 s materialization -inside ``daemon_prepare``. That timer covers lock admission, request encoding, -the pipe transport in both directions, the Rust fold, and response handling; -it does not say how much of it is transport. The shipped transport helpers in -``lab/prism/bundle_compiler.py`` slept a fixed 20 ms after every EAGAIN, once -per drained pipe capacity, so the suspicion was that a multi-megabyte -exchange paid far more in idle sleeps than in copying. A synthetic macOS probe -(146 ms vs 2 ms for a 4 MiB read) motivated the change; it is not production -evidence, and this driver exists to produce the Linux numbers. - -Two sections, both running the **shipped callables** of two revisions of the -transport module side by side in one process: - -1. **Raw transport** over real pipes against a Python peer subprocess: - ``_serve_builder_read_exact``, ``_serve_builder_read_line``, - ``_serve_builder_write`` and (Linux) ``_serve_builder_splice_spool``. - Reported per helper and payload size: wall seconds (median / min over - ``--reps``), throughput, **blocked waits** (readiness waits for the - patched module, 20 ms sleeps for the baseline), bytes, and a byte-exact - digest check against the peer. -2. **Real daemon** (``qbit-prism-build-audit-bundle --serve``, prebuilt via - ``--tool-bin-dir`` / ``PRISM_TOOL_BIN_DIR``): one cold ``prepare_window`` - ``full`` round trip through ``BundleCompiler.prepare_payout_window`` at - each ``--daemon-sizes`` window, attributed by wrapping the compiler's own - transport methods: - - =================== ======================================================= - phase what the wall time contains - =================== ======================================================= - ``spawn`` daemon spawn + protocol handshake (line read) - ``request_encode`` total minus every wrapped phase: unattributed - coordinator time outside the transport calls, - dominated by (not isolated to) the whole-request - ``json.dumps`` PR 2 targets - ``request_write`` ``_serve_builder_write`` of the request line: elapsed - until the pipe accepted every byte -- not proof the - daemon consumed them; up to one pipe capacity may - still be unread - ``response_wait`` ``_serve_builder_read_line`` of the envelope: elapsed - from write completion to envelope receipt -- daemon - scheduling and processing (remaining parse, fold, - canonical serialization) plus response readiness and - receipt; wall time, NOT measured Rust CPU. Splitting - it needs Rust-side profiling. - ``response_read`` ``_serve_builder_read_exact`` of the raw canonical - items section + its newline: elapsed receipt of the - bytes the daemon produces after the envelope - =================== ======================================================= - - Both revisions must return the same ``share_snapshot_sha256`` and the same - canonical item bytes; a mismatch is reported as a correctness failure. - -The baseline module is extracted with ``git show REV:lab/prism/bundle_compiler.py`` -and imported under a private name; it resolves its own imports from the -current tree, which is fine for the transport code because the helpers only -depend on the module-level constants beside them. Wait counting hooks the -patched module's ``_PipeReadinessWaiter`` and the baseline's ``time.sleep``; -every peer is a subprocess so nothing else in the process sleeps during a -measurement. - -Self-contained: no database, no network, no coordinator, standard library -only. ``python3 tests/perf/window_builder_pipe_latency.py --baseline-rev -e59bda34016f6dc0a6bc39f9b879b8df64f51f60`` prints the tables; ``--json FILE`` -also captures the structured results. - -This is an **on-demand instrument, not a test**: it asserts no thresholds and -is deliberately not named ``test_*`` so the discovery run never executes it. -Nothing under ``lab/`` is modified. -""" - -from __future__ import annotations - -import argparse -import contextlib -import fcntl -import hashlib -import importlib.util -import json -import os -import platform -import statistics -import subprocess -import sys -import tempfile -import threading -import time -from dataclasses import asdict, dataclass, field -from pathlib import Path -from types import ModuleType -from typing import Any, Callable, Iterator - -if not __package__: - sys.path.insert(0, str(Path(__file__).resolve().parents[2])) - -from lab.prism.share_ledger import ( # noqa: E402 - path fix-up above - DEFAULT_INCREMENTAL_SHARE_WINDOW_PAGE_SIZE, - AcceptedShareRecord, -) - - -REPO_ROOT = Path(__file__).resolve().parents[2] -COMPILER_RELATIVE_PATH = "lab/prism/bundle_compiler.py" -DEFAULT_TOOL_BIN_DIR = REPO_ROOT / "target" / "release" -DAEMON_BINARY = "qbit-prism-build-audit-bundle" -FIXED_NOW_MS = 1_760_000_000_000 -F_GETPIPE_SZ = 1032 - - -# --------------------------------------------------------------------------- -# module loading and wait counting -# --------------------------------------------------------------------------- - - -def load_compiler(label: str, rev: str | None) -> tuple[ModuleType, str]: - """The transport module for one revision (None = the working tree).""" - if rev is None: - import lab.prism.bundle_compiler as module - - head = subprocess.run( - ["git", "-C", str(REPO_ROOT), "rev-parse", "HEAD"], - check=False, - stdout=subprocess.PIPE, - stderr=subprocess.DEVNULL, - text=True, - ).stdout.strip() - dirty = subprocess.run( - ["git", "-C", str(REPO_ROOT), "status", "--porcelain", COMPILER_RELATIVE_PATH], - check=False, - stdout=subprocess.PIPE, - stderr=subprocess.DEVNULL, - text=True, - ).stdout.strip() - return module, f"{head or 'unknown'}{' (working tree, modified)' if dirty else ''}" - resolved = subprocess.run( - ["git", "-C", str(REPO_ROOT), "rev-parse", rev], - check=True, - stdout=subprocess.PIPE, - text=True, - ).stdout.strip() - source = subprocess.run( - ["git", "-C", str(REPO_ROOT), "show", f"{resolved}:{COMPILER_RELATIVE_PATH}"], - check=True, - stdout=subprocess.PIPE, - ).stdout - directory = tempfile.mkdtemp(prefix="prism-236-transport-") - path = Path(directory) / f"bundle_compiler_{label}.py" - path.write_bytes(source) - name = f"prism_bundle_compiler_{label}" - spec = importlib.util.spec_from_file_location(name, path) - assert spec is not None and spec.loader is not None - module = importlib.util.module_from_spec(spec) - sys.modules[name] = module - spec.loader.exec_module(module) - return module, resolved - - -class WaitCounter: - def __init__(self) -> None: - self.count = 0 - - -@contextlib.contextmanager -def counting_blocked_waits(module: ModuleType) -> Iterator[WaitCounter]: - """Count blocked waits taken by ``module``'s transport loops. - - The patched module waits through ``_PipeReadinessWaiter.wait``; the - baseline sleeps through ``time.sleep``, which its loops resolve at call - time, so a process-global hook counts exactly those sleeps as long as - nothing else in the process sleeps meanwhile (every peer is a - subprocess). - """ - counter = WaitCounter() - waiter_class = getattr(module, "_PipeReadinessWaiter", None) - if waiter_class is not None: - - class Counting(waiter_class): # type: ignore[misc,valid-type] - def wait(self, deadline: float) -> bool: - counter.count += 1 - return super().wait(deadline) - - module._PipeReadinessWaiter = Counting - try: - yield counter - finally: - module._PipeReadinessWaiter = waiter_class - return - real_sleep = time.sleep - - def sleep(seconds: float) -> None: - counter.count += 1 - real_sleep(seconds) - - time.sleep = sleep - try: - yield counter - finally: - time.sleep = real_sleep - - -class BenchRuntime: - """The BundleCompilerRuntime port, reduced to counters and no-ops.""" - - signing_seed_hex = "42" * 32 - ledger_attestation_signing_seed_hex = "43" * 32 - - def __init__(self, timeout_seconds: float) -> None: - self.bundle_build_timeout_seconds = timeout_seconds - self._job_build_scheduler_lock = threading.Lock() - self._tip_refresh_metrics_lock = threading.Lock() - self.job_build_worker_counts = { - "starts": 0, - "restarts": 0, - "crashes": 0, - "terminations": 0, - } - self._job_build_worker_restart_pending = False - self.tip_refresh_worker_restarts = 0 - self.tip_refresh_worker_failures = 0 - self._phases: dict[str, float] = {} - - def _ensure_job_cache_state(self) -> None: - pass - - def _ensure_tip_refresh_state(self) -> None: - pass - - def _job_build_checkpoint(self, phase: str, cancellation: Any) -> None: - pass - - def _job_build_phases(self) -> dict[str, float]: - return self._phases - - def prism_payout_policy(self) -> dict[str, object]: - return {} - - def prism_ctv_settlement_config(self, **_kwargs: Any) -> None: - return None - - def _observe_tip_refresh_build_phase(self, name: str, elapsed: float) -> None: - pass - - def _record_tip_refresh_ipc_bytes(self, direction: str, byte_count: int) -> None: - pass - - -class _Control: - """Never instantiated; only its type is consulted.""" - - -class _Serialization: - def __init__(self) -> None: - self.spool_failures = 0 - - def mark_spool_failed(self) -> None: - self.spool_failures += 1 - - -def make_compiler(module: ModuleType, runtime: BenchRuntime, tool_bin_dir: Path | None) -> Any: - tool_command = None - if tool_bin_dir is not None: - - def tool_command(bin_name: str) -> list[str]: - return [str(tool_bin_dir / bin_name)] - - return module.BundleCompiler( - runtime, - superseded_error=RuntimeError, - cancellation_error_types=(RuntimeError,), - build_control_type=_Control, - tool_command=tool_command, - ) - - -# --------------------------------------------------------------------------- -# section 1: raw transport against a peer subprocess -# --------------------------------------------------------------------------- - - -# Writes `total` bytes to stdout in `chunk`-sized blocks (a single line when -# the mode is "line"), pausing `pause` seconds after each block, after a go -# byte arrives on stdin; the digest of what was written goes to stderr. -PEER_WRITE_STDOUT = r""" -import hashlib, os, sys, time -total = int(sys.argv[1]); chunk = int(sys.argv[2]); pause = float(sys.argv[3]) -line = sys.argv[4] == "line" -os.read(0, 1) -digest = hashlib.sha256() -remaining = total -while remaining: - n = min(chunk, remaining) - block = os.urandom(n) - if line: - block = block.replace(b"\n", b"\x00") - digest.update(block) - view = memoryview(block) - while view: - written = os.write(1, view) - view = view[written:] - remaining -= n - if pause: - time.sleep(pause) -if line: - os.write(1, b"\n") -sys.stderr.write(digest.hexdigest() + "\n") -sys.stderr.flush() -""" - -# Reads stdin to EOF in `chunk`-sized reads, pausing `pause` seconds after -# each, then reports "<bytes> <digest>" on stdout. -PEER_READ_STDIN = r""" -import hashlib, os, sys, time -chunk = int(sys.argv[1]); pause = float(sys.argv[2]) -digest = hashlib.sha256(); total = 0 -while True: - data = os.read(0, chunk) - if not data: - break - digest.update(data); total += len(data) - if pause: - time.sleep(pause) -sys.stdout.write(f"{total} {digest.hexdigest()}\n") -sys.stdout.flush() -""" - - -@dataclass -class TransportSample: - seconds: float - waits: int - bytes: int - ok: bool - - -def _far_deadline() -> float: - return time.monotonic() + 600.0 - - -def run_read_exact(module: ModuleType, size: int, peer_chunk: int, peer_pause: float) -> TransportSample: - peer = subprocess.Popen( - [sys.executable, "-c", PEER_WRITE_STDOUT, str(size), str(peer_chunk), str(peer_pause), "raw"], - stdin=subprocess.PIPE, - stdout=subprocess.PIPE, - stderr=subprocess.PIPE, - ) - assert peer.stdin is not None and peer.stdout is not None - os.set_blocking(peer.stdout.fileno(), False) - client = module._ServeBuilderClient(process=peer) - compiler = make_compiler(module, BenchRuntime(600.0), None) - with counting_blocked_waits(module) as counter: - peer.stdin.write(b"g") - peer.stdin.flush() - started = time.perf_counter() - data = compiler._serve_builder_read_exact(client, size, _far_deadline(), None, None) - elapsed = time.perf_counter() - started - ok = len(data) == size and hashlib.sha256(data).hexdigest() == _writer_digest(peer) - return TransportSample(elapsed, counter.count, size, ok) - - -def run_read_line(module: ModuleType, size: int, peer_chunk: int, peer_pause: float) -> TransportSample: - peer = subprocess.Popen( - [sys.executable, "-c", PEER_WRITE_STDOUT, str(size), str(peer_chunk), str(peer_pause), "line"], - stdin=subprocess.PIPE, - stdout=subprocess.PIPE, - stderr=subprocess.PIPE, - ) - assert peer.stdin is not None and peer.stdout is not None - os.set_blocking(peer.stdout.fileno(), False) - client = module._ServeBuilderClient(process=peer) - compiler = make_compiler(module, BenchRuntime(600.0), None) - with counting_blocked_waits(module) as counter: - peer.stdin.write(b"g") - peer.stdin.flush() - started = time.perf_counter() - line = compiler._serve_builder_read_line(client, _far_deadline(), None, None) - elapsed = time.perf_counter() - started - ok = len(line) == size and hashlib.sha256(line).hexdigest() == _writer_digest(peer) - return TransportSample(elapsed, counter.count, size + 1, ok) - - -def _spawn_reader(peer_chunk: int, peer_pause: float) -> subprocess.Popen[bytes]: - return subprocess.Popen( - [sys.executable, "-c", PEER_READ_STDIN, str(peer_chunk), str(peer_pause)], - stdin=subprocess.PIPE, - stdout=subprocess.PIPE, - stderr=subprocess.PIPE, - ) - - -def _reader_result(peer: subprocess.Popen[bytes]) -> tuple[int, str]: - """The reader peer's "<bytes> <digest>" report, after its stdin closed.""" - assert peer.stdin is not None and peer.stdout is not None and peer.stderr is not None - peer.stdin.close() - stdout = peer.stdout.read() - stderr = peer.stderr.read() - peer.wait(timeout=120) - if peer.returncode != 0: - raise RuntimeError(f"reader peer failed: {stderr.decode(errors='replace')}") - total, digest = stdout.decode().split() - return int(total), digest - - -def _writer_digest(peer: subprocess.Popen[bytes]) -> str: - """The writer peer's digest line, after its payload was fully read.""" - assert peer.stdin is not None and peer.stderr is not None - peer.stdin.close() - stderr = peer.stderr.read() - peer.wait(timeout=120) - if peer.returncode != 0: - raise RuntimeError(f"writer peer failed: {stderr.decode(errors='replace')}") - return stderr.decode().strip() - - -def run_write(module: ModuleType, size: int, peer_chunk: int, peer_pause: float) -> TransportSample: - payload = os.urandom(size) - expected = hashlib.sha256(payload).hexdigest() - peer = _spawn_reader(peer_chunk, peer_pause) - assert peer.stdin is not None - os.set_blocking(peer.stdin.fileno(), False) - client = module._ServeBuilderClient(process=peer) - compiler = make_compiler(module, BenchRuntime(600.0), None) - with counting_blocked_waits(module) as counter: - started = time.perf_counter() - written = compiler._serve_builder_write(client, payload, _far_deadline(), None, None) - elapsed = time.perf_counter() - started - total, digest = _reader_result(peer) - ok = written == size and total == size and digest == expected - return TransportSample(elapsed, counter.count, size, ok) - - -def run_splice(module: ModuleType, size: int, peer_chunk: int, peer_pause: float) -> TransportSample: - payload = os.urandom(size) - expected = hashlib.sha256(payload).hexdigest() - spool = tempfile.TemporaryFile() - spool.write(payload) - spool.flush() - peer = _spawn_reader(peer_chunk, peer_pause) - assert peer.stdin is not None - os.set_blocking(peer.stdin.fileno(), False) - client = module._ServeBuilderClient(process=peer) - compiler = make_compiler(module, BenchRuntime(600.0), None) - serialization = _Serialization() - try: - with counting_blocked_waits(module) as counter: - started = time.perf_counter() - moved = compiler._serve_builder_splice_spool( - client, serialization, spool, size, _far_deadline(), None, None - ) - elapsed = time.perf_counter() - started - finally: - spool.close() - total, digest = _reader_result(peer) - ok = moved == size and total == size and digest == expected and serialization.spool_failures == 0 - return TransportSample(elapsed, counter.count, size, ok) - - -RAW_HELPERS: dict[str, Callable[[ModuleType, int, int, float], TransportSample]] = { - "read_exact": run_read_exact, - "read_line": run_read_line, - "write": run_write, -} -if hasattr(os, "splice"): - RAW_HELPERS["splice"] = run_splice - - -@dataclass -class RawResult: - helper: str - variant: str - bytes: int - reps: int - median_seconds: float - min_seconds: float - max_seconds: float - median_waits: float - mib_per_second: float - ok: bool - samples: list[dict[str, Any]] = field(default_factory=list) - - -def run_raw_section( - variants: list[tuple[str, ModuleType]], - sizes: list[int], - reps: int, - peer_chunk: int, - peer_pause: float, - log: Callable[[str], None], -) -> list[RawResult]: - results: list[RawResult] = [] - for helper, runner in RAW_HELPERS.items(): - for size in sizes: - samples: dict[str, list[TransportSample]] = {label: [] for label, _ in variants} - # Alternate variants per repetition so host drift lands on both. - for rep in range(reps): - for label, module in variants: - sample = runner(module, size, peer_chunk, peer_pause) - samples[label].append(sample) - log( - f" {helper:10s} {size / (1 << 20):8.2f} MiB {label:9s} rep {rep + 1}:" - f" {sample.seconds * 1000:9.2f} ms, {sample.waits:6d} waits," - f" {'ok' if sample.ok else 'MISMATCH'}" - ) - for label, _module in variants: - seconds = [s.seconds for s in samples[label]] - median = statistics.median(seconds) - results.append( - RawResult( - helper=helper, - variant=label, - bytes=size, - reps=reps, - median_seconds=median, - min_seconds=min(seconds), - max_seconds=max(seconds), - median_waits=statistics.median(s.waits for s in samples[label]), - mib_per_second=(size / (1 << 20)) / median if median > 0 else float("inf"), - ok=all(s.ok for s in samples[label]), - samples=[asdict(s) for s in samples[label]], - ) - ) - return results - - -# --------------------------------------------------------------------------- -# section 2: the real daemon's cold prepare_window round trip -# --------------------------------------------------------------------------- - - -def build_records(count: int, *, miners: int, rigs_per_miner: int) -> list[AcceptedShareRecord]: - """Records shaped like the ledger's rows (production share_id form).""" - programs = [hashlib.sha256(f"bench-miner-{index}".encode()).hexdigest() for index in range(miners)] - records: list[AcceptedShareRecord] = [] - for index in range(count): - miner = index % miners - block_hash = hashlib.sha256(f"{index}".encode()).hexdigest() - records.append( - AcceptedShareRecord( - share_seq=index + 1, - share_id=f"bench-miner-{miner}.rig{index % rigs_per_miner}:{block_hash}", - miner_id=f"bench-miner-{miner}", - order_key=f"bench-miner-{miner:06d}", - p2mr_program_hex=programs[miner], - share_difficulty=16384, - network_difficulty=226646186, - template_height=9, - job_id=f"bench-job-{index + 1}", - job_issued_at_ms=FIXED_NOW_MS - (count - index) * 1_000, - accepted_at_ms=FIXED_NOW_MS - (count - index) * 1_000, - ntime=FIXED_NOW_MS // 1000 - (count - index), - ) - ) - return records - - -@dataclass -class PhaseSample: - seconds: float = 0.0 - bytes: int = 0 - waits: int = 0 - calls: int = 0 - - -@dataclass -class DaemonSample: - variant: str - records: int - status: str - total_seconds: float - phases: dict[str, PhaseSample] - request_bytes: int - response_bytes: int - digest: str | None - items_sha256: str | None - record_count: int - - -def run_daemon_prepare( - module: ModuleType, - label: str, - records_json: list[dict[str, object]], - *, - anchor_job_issued_at_ms: int, - window_weight: int, - tool_bin_dir: Path, - timeout_seconds: float, -) -> DaemonSample: - runtime = BenchRuntime(timeout_seconds) - compiler = make_compiler(module, runtime, tool_bin_dir) - phases: dict[str, PhaseSample] = { - name: PhaseSample() - for name in ("spawn", "request_encode", "request_write", "response_wait", "response_read") - } - counter_holder: list[WaitCounter] = [] - state = {"in_spawn": False} - - def timed(name: str, method: Callable[..., Any], byte_count: Callable[[tuple, Any], int]) -> Callable[..., Any]: - def wrapper(*args: Any, **kwargs: Any) -> Any: - if state["in_spawn"] and name != "spawn": - # The handshake line read runs inside the spawn wrapper and - # is already attributed to spawn. - return method(*args, **kwargs) - counter = counter_holder[0] - before = counter.count - started = time.perf_counter() - try: - result = method(*args, **kwargs) - finally: - phase = phases[name] - phase.seconds += time.perf_counter() - started - phase.waits += counter.count - before - phase.calls += 1 - phase.bytes += byte_count(args, result) - return result - - return wrapper - - def spawn_wrapper(*args: Any, **kwargs: Any) -> Any: - state["in_spawn"] = True - try: - return timed_spawn(*args, **kwargs) - finally: - state["in_spawn"] = False - - timed_spawn = timed("spawn", compiler._spawn_serve_builder_locked, lambda args, result: 0) - compiler._spawn_serve_builder_locked = spawn_wrapper - compiler._serve_builder_write = timed( - "request_write", compiler._serve_builder_write, lambda args, result: int(result) - ) - compiler._serve_builder_read_line = timed( - "response_wait", compiler._serve_builder_read_line, lambda args, result: len(result) + 1 - ) - compiler._serve_builder_read_exact = timed( - "response_read", compiler._serve_builder_read_exact, lambda args, result: len(result) - ) - outcome = None - error: str | None = None - with counting_blocked_waits(module) as counter: - counter_holder.append(counter) - started = time.perf_counter() - try: - outcome = compiler.prepare_payout_window( - mode="full", - records_json=records_json, - anchor_job_issued_at_ms=anchor_job_issued_at_ms, - append_invalidation_epoch=0, - window_weight=window_weight, - page_size=DEFAULT_INCREMENTAL_SHARE_WINDOW_PAGE_SIZE, - ) - except Exception as exc: # noqa: BLE001 - reported, not raised - error = f"{type(exc).__name__}: {exc}" - total = time.perf_counter() - started - compiler.shutdown_serve_builder() - measured = sum( - phases[name].seconds for name in ("spawn", "request_write", "response_wait", "response_read") - ) - phases["request_encode"].seconds = max(0.0, total - measured) - phases["request_encode"].calls = 1 - if error is not None: - status = f"error: {error}" - elif outcome is None: - status = "none (daemon anomaly, fell back)" - else: - status = str(outcome.status) - items = getattr(outcome, "window_items", None) if outcome is not None else None - return DaemonSample( - variant=label, - records=len(records_json), - status=status, - total_seconds=total, - phases=phases, - request_bytes=phases["request_write"].bytes, - response_bytes=phases["response_wait"].bytes + phases["response_read"].bytes, - digest=getattr(outcome, "share_snapshot_sha256", None) if outcome is not None else None, - items_sha256=hashlib.sha256(items).hexdigest() if items is not None else None, - record_count=int(getattr(outcome, "record_count", 0) or 0) if outcome is not None else 0, - ) - - -def run_daemon_section( - variants: list[tuple[str, ModuleType]], - sizes: list[int], - reps: int, - *, - miners: int, - rigs_per_miner: int, - tool_bin_dir: Path, - timeout_seconds: float, - log: Callable[[str], None], -) -> list[dict[str, Any]]: - results: list[dict[str, Any]] = [] - for size in sizes: - log(f" building {size} records ...") - records = build_records(size, miners=miners, rigs_per_miner=rigs_per_miner) - anchor = int(records[-1].job_issued_at_ms) - weight = sum(int(record.share_difficulty) for record in records) - records_json = [record.to_prism_json() for record in records] - samples: dict[str, list[DaemonSample]] = {label: [] for label, _ in variants} - for rep in range(reps): - for label, module in variants: - sample = run_daemon_prepare( - module, - label, - records_json, - anchor_job_issued_at_ms=anchor, - window_weight=weight, - tool_bin_dir=tool_bin_dir, - timeout_seconds=timeout_seconds, - ) - samples[label].append(sample) - phase_text = ", ".join( - f"{name} {phase.seconds:.3f}s/{phase.waits}w" - for name, phase in sample.phases.items() - ) - log( - f" daemon {size:7d} {label:9s} rep {rep + 1}: {sample.status}," - f" total {sample.total_seconds:.3f}s [{phase_text}]" - ) - digests = {s.digest for label in samples for s in samples[label]} - items = {s.items_sha256 for label in samples for s in samples[label]} - parity = len(digests) == 1 and None not in digests and len(items) == 1 and None not in items - for label, _module in variants: - runs = samples[label] - by_total = sorted(runs, key=lambda s: s.total_seconds) - median_run = by_total[len(by_total) // 2] - results.append( - { - "records": size, - "variant": label, - "reps": reps, - "statuses": [s.status for s in runs], - "parity_across_variants": parity, - "digest": median_run.digest, - "record_count": median_run.record_count, - "request_bytes": median_run.request_bytes, - "response_bytes": median_run.response_bytes, - "median_total_seconds": statistics.median(s.total_seconds for s in runs), - "min_total_seconds": min(s.total_seconds for s in runs), - "median_run_phases": { - name: asdict(phase) for name, phase in median_run.phases.items() - }, - "runs": [ - { - "status": s.status, - "total_seconds": s.total_seconds, - "phases": {name: asdict(phase) for name, phase in s.phases.items()}, - } - for s in runs - ], - } - ) - return results - - -# --------------------------------------------------------------------------- -# reporting -# --------------------------------------------------------------------------- - - -def machine_info() -> dict[str, Any]: - read_end, write_end = os.pipe() - try: - pipe_capacity = fcntl.fcntl(read_end, F_GETPIPE_SZ) - except OSError: - pipe_capacity = None - finally: - os.close(read_end) - os.close(write_end) - try: - load = os.getloadavg() - except OSError: - load = None - return { - "platform": platform.platform(), - "python": sys.version.split()[0], - "implementation": platform.python_implementation(), - "cpu_count": os.cpu_count(), - "pipe_capacity_bytes": pipe_capacity, - "load_average": load, - "splice": hasattr(os, "splice"), - } - - -def render_raw(results: list[RawResult]) -> str: - lines = [ - "| helper | bytes | variant | median ms | min ms | max ms | median waits | MiB/s | exact |", - "|---|---:|---|---:|---:|---:|---:|---:|---|", - ] - for result in results: - lines.append( - f"| {result.helper} | {result.bytes:,} | {result.variant} |" - f" {result.median_seconds * 1000:.2f} | {result.min_seconds * 1000:.2f} |" - f" {result.max_seconds * 1000:.2f} | {result.median_waits:.0f} |" - f" {result.mib_per_second:.1f} | {'yes' if result.ok else 'NO'} |" - ) - return "\n".join(lines) - - -def render_daemon(results: list[dict[str, Any]]) -> str: - lines = [ - "| records | variant | status | total s | spawn s | encode s | write s (waits) | wait s (waits) | read s (waits) | req MiB | resp MiB | parity |", - "|---:|---|---|---:|---:|---:|---:|---:|---:|---:|---:|---|", - ] - for result in results: - phases = result["median_run_phases"] - - def cell(name: str) -> str: - phase = phases[name] - return f"{phase['seconds']:.3f} ({phase['waits']})" - - lines.append( - f"| {result['records']:,} | {result['variant']} | {result['statuses'][-1]} |" - f" {result['median_total_seconds']:.3f} | {phases['spawn']['seconds']:.3f} |" - f" {phases['request_encode']['seconds']:.3f} | {cell('request_write')} |" - f" {cell('response_wait')} | {cell('response_read')} |" - f" {result['request_bytes'] / (1 << 20):.1f} | {result['response_bytes'] / (1 << 20):.1f} |" - f" {'yes' if result['parity_across_variants'] else 'NO'} |" - ) - return "\n".join(lines) - - -def parse_sizes(text: str) -> list[int]: - return [int(float(item)) for item in text.split(",") if item.strip()] - - -def main(argv: list[str] | None = None) -> int: - parser = argparse.ArgumentParser( - description=__doc__, - formatter_class=argparse.RawDescriptionHelpFormatter, - ) - parser.add_argument( - "--baseline-rev", - default=None, - help="git revision whose lab/prism/bundle_compiler.py is measured beside the working tree", - ) - parser.add_argument( - "--sizes-mib", - default="1,4,16", - help="raw transport payload sizes in MiB (default 1,4,16)", - ) - parser.add_argument("--reps", type=int, default=5, help="raw transport repetitions (default 5)") - parser.add_argument( - "--peer-chunk", - type=int, - default=1 << 16, - help="peer subprocess write/read chunk in bytes (default 65536)", - ) - parser.add_argument( - "--peer-pause", - type=float, - default=0.0, - help="seconds the peer pauses after each chunk (default 0: pipe-bound)", - ) - parser.add_argument( - "--daemon-sizes", - default="52000,210000", - help="window sizes for the real-daemon cold prepare (default 52000,210000)", - ) - parser.add_argument("--daemon-reps", type=int, default=2, help="cold prepares per size (default 2)") - parser.add_argument("--miners", type=int, default=64, help="distinct miners in the window (default 64)") - parser.add_argument("--rigs-per-miner", type=int, default=8, help="rig suffixes per miner (default 8)") - parser.add_argument( - "--tool-bin-dir", - default=os.environ.get("PRISM_TOOL_BIN_DIR") or str(DEFAULT_TOOL_BIN_DIR), - help="directory holding a prebuilt qbit-prism-build-audit-bundle (default $PRISM_TOOL_BIN_DIR or target/release)", - ) - parser.add_argument("--daemon-timeout", type=float, default=900.0, help="per-prepare budget in seconds") - parser.add_argument("--skip-raw", action="store_true", help="skip the raw transport section") - parser.add_argument("--skip-daemon", action="store_true", help="skip the real-daemon section") - parser.add_argument("--json", default=None, help="write structured results to this file") - parser.add_argument("--quiet", action="store_true", help="suppress per-repetition progress lines") - args = parser.parse_args(argv) - - def log(text: str) -> None: - if not args.quiet: - print(text, flush=True) - - variants: list[tuple[str, ModuleType]] = [] - revisions: dict[str, str] = {} - if args.baseline_rev: - module, resolved = load_compiler("baseline", args.baseline_rev) - variants.append(("baseline", module)) - revisions["baseline"] = resolved - module, resolved = load_compiler("patched", None) - variants.append(("patched", module)) - revisions["patched"] = resolved - - info = machine_info() - print("# window_builder_pipe_latency") - print() - for key, value in info.items(): - print(f"- {key}: {value}") - for label, resolved in revisions.items(): - print(f"- {label}: {resolved}") - print() - - output: dict[str, Any] = { - "machine": info, - "revisions": revisions, - "arguments": vars(args), - } - - if not args.skip_raw: - sizes = [int(size * (1 << 20)) for size in (float(s) for s in args.sizes_mib.split(","))] - print(f"## Raw transport (peer chunk {args.peer_chunk} B, peer pause {args.peer_pause} s, {args.reps} reps)") - print() - raw = run_raw_section(variants, sizes, args.reps, args.peer_chunk, args.peer_pause, log) - print() - print(render_raw(raw)) - print() - output["raw"] = [asdict(result) for result in raw] - - if not args.skip_daemon: - tool_bin_dir = Path(args.tool_bin_dir) - binary = tool_bin_dir / DAEMON_BINARY - if not (binary.is_file() and os.access(binary, os.X_OK)): - message = ( - f"real-daemon section skipped: {binary} is not an executable;" - " build it with `cargo build --release -p qbit-prism --bin" - f" {DAEMON_BINARY}` or point --tool-bin-dir / PRISM_TOOL_BIN_DIR at a build" - ) - print(f"## Real daemon\n\n{message}\n") - output["daemon_skipped"] = message - else: - sizes = parse_sizes(args.daemon_sizes) - print(f"## Real daemon cold prepare_window ({binary}, {args.daemon_reps} reps)") - print() - daemon = run_daemon_section( - variants, - sizes, - args.daemon_reps, - miners=args.miners, - rigs_per_miner=args.rigs_per_miner, - tool_bin_dir=tool_bin_dir, - timeout_seconds=args.daemon_timeout, - log=log, - ) - print() - print(render_daemon(daemon)) - print() - output["daemon"] = daemon - - if args.json: - Path(args.json).write_text(json.dumps(output, indent=2, default=str) + "\n") - print(f"wrote {args.json}") - return 0 - - -if __name__ == "__main__": - sys.exit(main()) diff --git a/tests/perf/window_ledger_decode_latency.py b/tests/perf/window_ledger_decode_latency.py deleted file mode 100644 index 617e04e40..000000000 --- a/tests/perf/window_ledger_decode_latency.py +++ /dev/null @@ -1,593 +0,0 @@ -#!/usr/bin/env python3 -"""Payout-window database decode latency and monitor responsiveness (#236). - -The 2026-09 union-mainnet incident (issue #236) attributes coordinator lease -exits to the coordinator's own scheduling stalls while a ~210k-share payout -window was being decoded: ``snapshot_at_job_issue`` returned one ``json_agg`` -value the size of the whole window, and decoding it was one C call that never -released the GIL, so the lease monitor thread went unscheduled for as long as -the decode took. PR 1 of the fix plan turns both payout-window reads into one -JSON object per row, decoded in bounded batches on both database backends. - -This driver measures that database/decode phase alone -- ledger admission, -statement execution, result receipt, row decoding and record conversion -- -for the three read shapes the coordinator issues, on both backends, at the -incident window size and the stress size, with a monitor thread beside it: - -================= ========================================================== -read call -================= ========================================================== -``bounded`` ``snapshot_at_job_issue(anchor, window_weight=<all rows>)`` -``unbounded`` ``snapshot_at_job_issue(anchor)`` (full history) -``delta`` ``snapshot_between_job_issues(prev, anchor)`` (all rows) -================= ========================================================== - -Per call it records wall time, the calling thread's CPU time, whole-process -CPU time, RSS before/after/high-water, the record count, and the ledger's -own admission/execution attribution. Beside every call a **monitor thread** -sleeps 10 ms in a loop and records how late each wake-up was: the maximum -lateness, and how many wake-ups were more than 100, 250 and 500 ms late. It -is a proxy for the lease monitor's wake lateness, not the lease monitor -itself: the real heartbeat needs a running coordinator and a guard session, -and the quantity that decides whether it can run is exactly this one -- -whether another Python thread can get the interpreter while the read is in -progress. Server-side time is measured separately with ``EXPLAIN (ANALYZE)`` -of the statement the ledger actually sent, so the client-side share of the -wall time (transfer + decode + convert) is attributable as ``wall - server`` -without labelling any of it as PostgreSQL time. - -**Baseline versus patch is the same script on two checkouts**: ``--lab-root`` -selects the tree ``lab.prism.share_ledger`` is imported from, and the result -records that tree's commit. The statements are captured from whatever that -checkout sends (``json_agg`` at the baseline, one object per row after), so -the server-side comparison is of the real statements, not a re-typed copy. - -The fixture is generated, production-shaped and disposable: ``username.rig: -block_hash_hex`` share ids (``share_writer.py``'s form), 62-character payout -addresses as miner id and order key, 32-byte P2MR programs, the live-host -difficulty values ``window_pipeline_gil_scaling.py`` uses, a distinct job id -per 64 shares, a ``stale-grace`` credit policy on ~1% of rows, and miner skew -in the shape ``test-prism-postgres-scale.sh`` seeds. Nothing is exported from -a production ledger. Point it at a disposable database; it truncates the -share ledger between sizes. - -This is an **on-demand instrument, not a test**: it asserts no thresholds and -is not named ``test_*``, so the discovery run never executes it. Example:: - - python3 tests/perf/window_ledger_decode_latency.py \\ - --psql-command "psql -p 55436 -d qbit_bench" \\ - --database-url "postgresql://ubuntu@/qbit_bench?host=/var/run/postgresql&port=55436" \\ - --shares 210000 400000 --reps 3 --json patched.json - python3 tests/perf/window_ledger_decode_latency.py ... --lab-root /path/to/baseline-checkout --json baseline.json -""" - -from __future__ import annotations - -import argparse -import hashlib -import json -import os -import platform -import resource -import statistics -import subprocess -import sys -import threading -import time -from pathlib import Path -from typing import Any, Callable - -REPO_ROOT = Path(__file__).resolve().parents[2] - - -def _install_lab_root(lab_root: Path) -> None: - """Make ``lab`` import from the chosen checkout, before anything imports it.""" - for name in list(sys.modules): - if name == "lab" or name.startswith("lab."): - raise SystemExit("lab.* was imported before --lab-root was applied") - sys.path.insert(0, str(lab_root)) - - -def _git_head(root: Path) -> str | None: - try: - return subprocess.run( - ["git", "-C", str(root), "rev-parse", "HEAD"], - capture_output=True, - text=True, - check=True, - ).stdout.strip() - except (OSError, subprocess.CalledProcessError): - return None - - -# -------------------------------------------------------------------------- -# fixture -# -------------------------------------------------------------------------- - -SHARE_DIFFICULTY = 16384 -NETWORK_DIFFICULTY = 226646186 -EPOCH_SECONDS = 1_700_000_000 -SHARE_SPACING_MS = 250 -ACCEPT_LAG_MS = 400 - - -def seed_sql(share_rows: int, miners: int) -> str: - miners = max(4, int(miners)) - return f""" -TRUNCATE qbit_share_ledger CASCADE; -ALTER SEQUENCE qbit_share_ledger_share_seq_seq RESTART WITH 1; -INSERT INTO qbit_share_ledger ( - share_id, miner_id, payout_order_key, p2mr_program, - share_difficulty, network_difficulty, template_height, job_id, - job_issued_at, ntime, accepted_at, accepted, writer_id, writer_epoch, - credit_policy -) -SELECT - 'bench-miner-' || miner_idx || '.rig' || (g % 512) || ':' - || encode(sha256(('bench-share:' || g)::bytea), 'hex'), - 'qbit1' || substr(encode(sha256(('bench-miner:' || miner_idx)::bytea), 'hex'), 1, 58), - 'qbit1' || substr(encode(sha256(('bench-miner:' || miner_idx)::bytea), 'hex'), 1, 58), - sha256(('bench-program:' || miner_idx)::bytea), - {SHARE_DIFFICULTY}, - {NETWORK_DIFFICULTY}, - 1000000 + (g / 64), - 'bench-job-' || lpad((g / 64)::text, 12, '0'), - to_timestamp({EPOCH_SECONDS}) + (g * interval '{SHARE_SPACING_MS} milliseconds'), - {EPOCH_SECONDS} + (g / 4), - to_timestamp({EPOCH_SECONDS}) + (g * interval '{SHARE_SPACING_MS} milliseconds') - + interval '{ACCEPT_LAG_MS} milliseconds', - TRUE, - 'bench-seed', - 1, - CASE WHEN g % 97 = 0 THEN 'stale-grace' ELSE NULL END -FROM ( - SELECT - gs AS g, - CASE - WHEN mod(gs, 10) < 3 THEN 1 - WHEN mod(gs, 10) < 5 THEN 2 - WHEN mod(gs, 10) < 6 THEN 3 - ELSE 4 + mod(gs, {miners} - 3) - END AS miner_idx - FROM generate_series(1, {int(share_rows)}) AS gs -) AS seeded; -ANALYZE qbit_share_ledger; -""" - - -def anchors_for(share_rows: int) -> tuple[int, int, int]: - """(anchor_ms, previous_anchor_ms, window_weight covering every row).""" - last_accepted_ms = ( - EPOCH_SECONDS * 1000 + share_rows * SHARE_SPACING_MS + ACCEPT_LAG_MS - ) - anchor_ms = last_accepted_ms + 10_000 - previous_anchor_ms = EPOCH_SECONDS * 1000 - 1 - return anchor_ms, previous_anchor_ms, share_rows * SHARE_DIFFICULTY - - -# -------------------------------------------------------------------------- -# measurement -# -------------------------------------------------------------------------- - - -class WakeMonitor(threading.Thread): - """Sleep 10 ms in a loop and record how late each wake-up was.""" - - def __init__(self, interval: float = 0.010) -> None: - super().__init__(daemon=True, name="wake-monitor") - self.interval = interval - self._stop_event = threading.Event() - self.samples = 0 - self.max_late = 0.0 - self.late_100 = 0 - self.late_250 = 0 - self.late_500 = 0 - self.late_total = 0.0 - - def run(self) -> None: - while not self._stop_event.is_set(): - started = time.perf_counter() - time.sleep(self.interval) - late = max(0.0, time.perf_counter() - started - self.interval) - self.samples += 1 - self.late_total += late - if late > self.max_late: - self.max_late = late - if late > 0.100: - self.late_100 += 1 - if late > 0.250: - self.late_250 += 1 - if late > 0.500: - self.late_500 += 1 - - def stop(self) -> dict[str, Any]: - self._stop_event.set() - self.join(timeout=5) - return { - "samples": self.samples, - "max_late_ms": round(self.max_late * 1000, 1), - "mean_late_ms": round( - (self.late_total / self.samples * 1000) if self.samples else 0.0, 2 - ), - "late_over_100ms": self.late_100, - "late_over_250ms": self.late_250, - "late_over_500ms": self.late_500, - } - - -def _rss_kb() -> tuple[int, int]: - rss = hwm = 0 - try: - for line in Path("/proc/self/status").read_text().splitlines(): - if line.startswith("VmRSS:"): - rss = int(line.split()[1]) - elif line.startswith("VmHWM:"): - hwm = int(line.split()[1]) - except OSError: - pass - return rss, hwm - - -def capture_statement(ledger: Any, call: Callable[[], Any]) -> tuple[str, Any]: - """Run ``call`` and return the statement the ledger sent for it. - - Works on both checkouts: the row-result read helper after #236, the - single-value read helper before it. - """ - statements: list[str] = [] - for name in ("_run_retry_safe_read_json_rows", "_run_retry_safe_read_json"): - original = getattr(ledger, name, None) - if original is None: - continue - - def recording(sql: str, *args: Any, _original=original, **kwargs: Any) -> Any: - statements.append(sql) - return _original(sql, *args, **kwargs) - - setattr(ledger, name, recording) - try: - result = call() - finally: - delattr(ledger, name) - if len(statements) != 1: - raise SystemExit(f"expected one statement, captured {len(statements)}") - return statements[0], result - raise SystemExit("no read helper to capture on this ledger") - - -def timed_call(ledger: Any, call: Callable[[], Any]) -> dict[str, Any]: - stats_before = ledger.ledger_read_gate_stats() - rss_before, _ = _rss_kb() - monitor = WakeMonitor() - monitor.start() - time.sleep(0.05) # let the monitor establish its cadence - process_cpu = time.process_time() - thread_cpu = time.thread_time() - started = time.perf_counter() - result = call() - wall = time.perf_counter() - started - thread_cpu = time.thread_time() - thread_cpu - process_cpu = time.process_time() - process_cpu - monitor_stats = monitor.stop() - rss_after, hwm_after = _rss_kb() - stats_after = ledger.ledger_read_gate_stats() - attribution: dict[str, float] = {} - for operation, after in stats_after.items(): - before = stats_before.get(operation, {}) - delta = float(after.get("execute_seconds_total", 0.0)) - float( - before.get("execute_seconds_total", 0.0) - ) - gate_delta = float(after.get("gate_wait_seconds_total", 0.0)) - float( - before.get("gate_wait_seconds_total", 0.0) - ) - if delta or gate_delta: - attribution[operation] = { - "execute_ms": round(delta * 1000, 1), - "gate_wait_ms": round(gate_delta * 1000, 1), - } - return { - "records": len(result), - "wall_ms": round(wall * 1000, 1), - "thread_cpu_ms": round(thread_cpu * 1000, 1), - "process_cpu_ms": round(process_cpu * 1000, 1), - "rss_before_mb": round(rss_before / 1024, 1), - "rss_after_mb": round(rss_after / 1024, 1), - "rss_hwm_mb": round(hwm_after / 1024, 1), - "monitor": monitor_stats, - "ledger_attribution": attribution, - } - - -def explain(psql_ledger: Any, sql: str) -> dict[str, Any]: - """Server-side planning + execution time and the plan's node summary.""" - output = psql_ledger._run_sql("EXPLAIN (ANALYZE, FORMAT JSON) " + sql) - plan = json.loads(output)[0] - nodes: list[str] = [] - - def walk(node: dict[str, Any], depth: int) -> None: - label = node.get("Node Type", "?") - if node.get("Strategy"): - label += f"/{node['Strategy']}" - if node.get("Index Name"): - label += f"[{node['Index Name']}]" - nodes.append(f"{' ' * depth}{label} rows={node.get('Actual Rows')}") - for child in node.get("Plans", []) or []: - walk(child, depth + 1) - - walk(plan["Plan"], 0) - return { - "planning_ms": round(float(plan.get("Planning Time", 0.0)), 1), - "execution_ms": round(float(plan["Execution Time"]), 1), - "plan": nodes, - } - - -# -------------------------------------------------------------------------- -# driver -# -------------------------------------------------------------------------- - - -def run(args: argparse.Namespace) -> dict[str, Any]: - lab_root = Path(args.lab_root).resolve() - _install_lab_root(lab_root) - from lab.prism.share_ledger import PsqlShareLedger # noqa: E402 - - results: dict[str, Any] = { - "lab_root": str(lab_root), - "lab_commit": _git_head(lab_root), - "driver_commit": _git_head(REPO_ROOT), - "python": platform.python_version(), - "platform": platform.platform(), - "cpu_count": os.cpu_count(), - "switch_interval_ms": sys.getswitchinterval() * 1000, - "miners": args.miners, - "reps": args.reps, - "backends": list(args.backends), - "sizes": [], - } - sizes_for_json = results["sizes"] - - seed_ledger = PsqlShareLedger( - psql_command=args.psql_command, - native_client_mode="psql", - writer_id="bench-seed", - writer_epoch=1, - initialize_schema=True, - ) - try: - server_version = seed_ledger._run_json( - "SELECT json_build_object('v', version());" - )["v"] - finally: - pass - results["postgres"] = server_version - - for share_rows in args.shares: - print(f"== {share_rows} shares: seeding", flush=True) - seed_started = time.perf_counter() - seed_ledger._run_script(seed_sql(share_rows, args.miners)) - seed_seconds = time.perf_counter() - seed_started - count = int( - seed_ledger._run_json( - "SELECT json_build_object('n', count(*)) FROM qbit_share_ledger WHERE accepted;" - )["n"] - ) - if count != share_rows: - raise SystemExit(f"seeded {count} rows, expected {share_rows}") - anchor_ms, previous_anchor_ms, window_weight = anchors_for(share_rows) - size_result: dict[str, Any] = { - "shares": share_rows, - "seed_seconds": round(seed_seconds, 1), - "anchor_ms": anchor_ms, - "window_weight": window_weight, - "server": {}, - "backends": {}, - } - sizes_for_json.append(size_result) - - read_shapes: dict[str, Callable[[Any], Callable[[], Any]]] = { - "bounded": lambda ledger: lambda: ledger.snapshot_at_job_issue( - anchor_ms, window_weight=window_weight - ), - "unbounded": lambda ledger: lambda: ledger.snapshot_at_job_issue(anchor_ms), - "delta": lambda ledger: lambda: ledger.snapshot_between_job_issues( - previous_anchor_ms, anchor_ms - ), - } - - for backend in args.backends: - if backend == "native": - ledger = PsqlShareLedger( - psql_command=args.psql_command, - database_url=args.database_url, - native_client_mode="1", - writer_id="bench-read", - writer_epoch=1, - read_only=True, - ) - else: - ledger = PsqlShareLedger( - psql_command=args.psql_command, - native_client_mode="psql", - writer_id="bench-read", - writer_epoch=1, - read_only=True, - ) - backend_result: dict[str, Any] = { - "execution_backend": ledger.execution_backend, - "reads": {}, - } - size_result["backends"][backend] = backend_result - try: - for shape, make_call in read_shapes.items(): - call = make_call(ledger) - # One warm-up per shape captures the statement (and warms - # the buffer cache); measured reps follow. - statement, warm = capture_statement(ledger, call) - if shape not in size_result["server"] and args.explain: - size_result["server"][shape] = { - "statement_uses_json_agg": "json_agg" in statement, - "statement_sha256_prefix": hashlib.sha256( - statement.encode() - ).hexdigest()[:12], - **explain(seed_ledger, statement), - } - reps = [timed_call(ledger, call) for _ in range(args.reps)] - walls = [rep["wall_ms"] for rep in reps] - backend_result["reads"][shape] = { - "records": len(warm), - "wall_ms_median": round(statistics.median(walls), 1), - "wall_ms_max": round(max(walls), 1), - "monitor_max_late_ms_max": max( - rep["monitor"]["max_late_ms"] for rep in reps - ), - "monitor_late_over_100ms_total": sum( - rep["monitor"]["late_over_100ms"] for rep in reps - ), - "monitor_late_over_250ms_total": sum( - rep["monitor"]["late_over_250ms"] for rep in reps - ), - "monitor_late_over_500ms_total": sum( - rep["monitor"]["late_over_500ms"] for rep in reps - ), - "reps": reps, - } - print( - f" {backend:>6} {shape:>9}: records={len(warm)} " - f"wall median={statistics.median(walls):.0f}ms " - f"max={max(walls):.0f}ms " - f"monitor max-late={backend_result['reads'][shape]['monitor_max_late_ms_max']:.0f}ms " - f">100ms={backend_result['reads'][shape]['monitor_late_over_100ms_total']} " - f">250ms={backend_result['reads'][shape]['monitor_late_over_250ms_total']}", - flush=True, - ) - finally: - ledger.close() - for shape, server in size_result["server"].items(): - print( - f" server {shape:>9}: planning={server['planning_ms']}ms " - f"execution={server['execution_ms']}ms json_agg={server['statement_uses_json_agg']}", - flush=True, - ) - seed_ledger.close() - results["peak_rss_mb"] = round(resource.getrusage(resource.RUSAGE_SELF).ru_maxrss / 1024, 1) - return results - - -def render_comparison(baseline: dict[str, Any], patched: dict[str, Any]) -> str: - """Markdown tables comparing two result files, size by size.""" - lines: list[str] = [] - lines.append( - f"Baseline `{(baseline.get('lab_commit') or '?')[:12]}` vs patched " - f"`{(patched.get('lab_commit') or '?')[:12]}` (driver " - f"`{(patched.get('driver_commit') or '?')[:12]}`); reps={patched.get('reps')}; " - f"miners={patched.get('miners')}; {patched.get('postgres', '?')}; " - f"Python {patched.get('python')}; {patched.get('cpu_count')} CPUs." - ) - lines.append("") - by_size_base = {size["shares"]: size for size in baseline["sizes"]} - for size in patched["sizes"]: - shares = size["shares"] - base = by_size_base.get(shares) - lines.append(f"### {shares:,} shares") - lines.append("") - lines.append( - "| backend | read | records | wall median (ms) base -> patch | wall max (ms) base -> patch " - "| monitor max lateness (ms) base -> patch | wake-ups > 100 ms base -> patch " - "| wake-ups > 250 ms base -> patch | wake-ups > 500 ms base -> patch |" - ) - lines.append("|---|---|---:|---:|---:|---:|---:|---:|---:|") - for backend, patched_backend in size["backends"].items(): - base_backend = (base or {}).get("backends", {}).get(backend, {}) - for shape, after in patched_backend["reads"].items(): - before = base_backend.get("reads", {}).get(shape) - - def pair(key: str, fmt: str = "{:.0f}") -> str: - if before is None: - return "n/a -> " + fmt.format(after[key]) - return fmt.format(before[key]) + " -> " + fmt.format(after[key]) - - lines.append( - f"| {backend} | {shape} | {after['records']:,} | {pair('wall_ms_median')} " - f"| {pair('wall_ms_max')} | {pair('monitor_max_late_ms_max')} " - f"| {pair('monitor_late_over_100ms_total', '{}')} " - f"| {pair('monitor_late_over_250ms_total', '{}')} " - f"| {pair('monitor_late_over_500ms_total', '{}')} |" - ) - lines.append("") - lines.append("| read | server planning+execution (ms) base -> patch | statement uses json_agg base -> patch |") - lines.append("|---|---:|---|") - for shape, after in size.get("server", {}).items(): - before = (base or {}).get("server", {}).get(shape) - after_ms = after["planning_ms"] + after["execution_ms"] - if before is None: - lines.append(f"| {shape} | n/a -> {after_ms:.0f} | n/a -> {after['statement_uses_json_agg']} |") - else: - before_ms = before["planning_ms"] + before["execution_ms"] - lines.append( - f"| {shape} | {before_ms:.0f} -> {after_ms:.0f} " - f"| {before['statement_uses_json_agg']} -> {after['statement_uses_json_agg']} |" - ) - lines.append("") - rss_rows = [] - for backend, patched_backend in size["backends"].items(): - base_backend = (base or {}).get("backends", {}).get(backend, {}) - for shape, after in patched_backend["reads"].items(): - before = base_backend.get("reads", {}).get(shape) - after_rss = max(rep["rss_hwm_mb"] for rep in after["reps"]) - after_cpu = statistics.median(rep["thread_cpu_ms"] for rep in after["reps"]) - if before is None: - rss_rows.append(f"| {backend} | {shape} | n/a -> {after_cpu:.0f} | n/a -> {after_rss:.0f} |") - else: - before_rss = max(rep["rss_hwm_mb"] for rep in before["reps"]) - before_cpu = statistics.median(rep["thread_cpu_ms"] for rep in before["reps"]) - rss_rows.append( - f"| {backend} | {shape} | {before_cpu:.0f} -> {after_cpu:.0f} " - f"| {before_rss:.0f} -> {after_rss:.0f} |" - ) - lines.append("| backend | read | caller-thread CPU median (ms) base -> patch | process RSS high-water (MB) base -> patch |") - lines.append("|---|---|---:|---:|") - lines.extend(rss_rows) - lines.append("") - return "\n".join(lines) - - -def main(argv: list[str] | None = None) -> int: - parser = argparse.ArgumentParser( - description=__doc__, - formatter_class=argparse.RawDescriptionHelpFormatter, - ) - parser.add_argument( - "--compare", - nargs=2, - metavar=("BASELINE_JSON", "PATCHED_JSON"), - help="render a markdown comparison of two --json results and exit (no database needed)", - ) - parser.add_argument("--psql-command", help="psql invocation for the disposable database") - parser.add_argument("--database-url", help="DSN for the native (psycopg) backend") - parser.add_argument("--shares", type=int, nargs="+", default=[210_000, 400_000]) - parser.add_argument("--miners", type=int, default=8, help="distinct miners (skewed like the scale gate)") - parser.add_argument("--reps", type=int, default=3) - parser.add_argument("--backends", nargs="+", default=["native", "psql"], choices=["native", "psql"]) - parser.add_argument("--lab-root", default=str(REPO_ROOT), help="checkout to import lab.prism.share_ledger from") - parser.add_argument("--no-explain", dest="explain", action="store_false") - parser.add_argument("--json", help="write structured results here") - args = parser.parse_args(argv) - if args.compare: - baseline = json.loads(Path(args.compare[0]).read_text()) - patched = json.loads(Path(args.compare[1]).read_text()) - print(render_comparison(baseline, patched)) - return 0 - if not args.psql_command: - parser.error("--psql-command is required unless --compare is given") - if "native" in args.backends and not args.database_url: - parser.error("--database-url is required for the native backend") - results = run(args) - if args.json: - Path(args.json).write_text(json.dumps(results, indent=2, sort_keys=True)) - print(f"wrote {args.json}") - return 0 - - -if __name__ == "__main__": - sys.exit(main()) diff --git a/tests/perf/window_paging_allocation.py b/tests/perf/window_paging_allocation.py deleted file mode 100644 index 8c3106c9c..000000000 --- a/tests/perf/window_paging_allocation.py +++ /dev/null @@ -1,1236 +0,0 @@ -#!/usr/bin/env python3 -"""Rust payout-window paging allocation: builder RSS and timing (#240). - -``PayoutWindow::from_full_snapshot`` (``crates/qbit-prism/src/window.rs``) -paged the retained records with ``remaining.split_off(page_size)`` in a -loop. ``Vec::split_off`` keeps the original vector's allocation, so every -512-record page retained a backing allocation sized for every record still -unpaged when it was cut, and the tail was copied again on each iteration: -O(N^2/P) retained memory and copying. A prior synthetic reproduction for -#236 measured 9,613 MiB builder RSS and a 12 s response at 210k shares, and -a SIGKILL at 400k. This driver measures that path against the real -``--serve`` daemon binary, before and after the fix, with every experiment -bounded so it cannot exhaust the host. - -Per binary and window size it runs one daemon through the coordinator's own -``prepare_window`` protocol over blocking pipes: - -===================== ====================================================== -phase what it exercises -===================== ====================================================== -``full`` cold full-snapshot preparation of the whole fixture -``advance_small`` ``advance`` with a handful of appended records -``advance_large`` ``advance`` with a multi-page delta (the delta paging - loop had the same ``split_off`` shape) -``recenter`` a second full preparation on the same daemon at a - smaller weight -- what the coordinator's self-check - re-center sends while the previous window is held -``recenter_advance`` ``advance`` against the re-centered digest -===================== ====================================================== - -and reports, per phase, wall time split into ``request_write`` (until the -pipe accepted the last request byte), ``response_wait`` (from then until the -envelope line arrived) and ``response_read`` (the raw canonical-items -section), the daemon's own ``metrics`` (JSON parse, fold/advance, canonical -serialization -- Rust processing measured inside the process, so transport -and processing are separated rather than inferred), the daemon's resident -set after the phase, and the observed peak with source provenance. All peak -observations precede process exit and are labelled lower bounds on the final -lifetime peak, including kernel ``VmHWM`` reads. The derived -``residual`` column is ``response_wait`` minus the daemon's three timers; the -client's wall interval and the daemon's internal timers are independent -intervals on either side of a pipe, so it is an approximation of scheduling -and envelope handling, not a measured transport term. Every digest the -daemon returns is checked against its own items bytes, the coordinator's -mirror surgery (drop prefix, append suffix) is replayed and re-hashed, and -unless ``--skip-oracle`` is given the shipped Python fold -(``IncrementalShareWindow``) is run on the same inputs as an independent -oracle for every digest. - -Bounding: the daemon runs under ``RLIMIT_AS`` (``--daemon-memory-limit-mb``, -default 6144), so a quadratic binary aborts on allocation failure instead of -taking the host down; a size is skipped, and the skip recorded, when -``MemAvailable`` is below the limit plus ``--memory-margin-mb``; and every -exchange -- the handshake included -- runs under a wall-clock deadline -(``--exchange-timeout``) after which the daemon is killed, reaped and the -phase recorded as a failure, so a stalled daemon cannot hang the run. The -known-quadratic baseline should only be run at sizes whose expected footprint -fits the limit (52k and 105k here); the 210k/400k baseline numbers are on -record in ``window_pipeline_gil_scaling.md`` section 9 and are not repeated. - -Fixture: ``build_records`` from ``window_pipeline_gil_scaling.py`` -- the -same production-shaped ``share_id`` fixture (200 identities, difficulty -16384) the #236 probe and the cited baseline used -- with ``window_weight`` -equal to the fixture's total difficulty so every record is retained. - -Self-contained: standard library plus read-only imports from ``lab/``; no -database, network or coordinator. Example:: - - cargo build --release --locked -p qbit-prism --bin qbit-prism-build-audit-bundle - python3 tests/perf/window_paging_allocation.py \ - --daemon-binary target/release/qbit-prism-build-audit-bundle \ - --baseline-binary /path/to/old/qbit-prism-build-audit-bundle \ - --sizes 52000,105000,210000,400000 --baseline-sizes 52000,105000 \ - --json window_paging_allocation.json - -This is an **on-demand instrument, not a test**: it asserts no thresholds -and is deliberately not named ``test_*`` so discovery never runs it (#160). -Nothing under ``lab/`` is modified. - -Memory measurements use MiB (2**20 bytes). Existing ``_mb`` result keys and -``*-mb`` CLI option names are retained and use the same binary units. -""" - -from __future__ import annotations - -import argparse -import hashlib -import json -import os -import platform -import resource -import subprocess -import sys -import threading -import time -from dataclasses import asdict, dataclass, field -from pathlib import Path -from typing import Any, Callable - -if not __package__: - sys.path.insert(0, str(Path(__file__).resolve().parents[2])) - -from lab.prism.bundle_compiler import ( # noqa: E402 - path fix-up above - PRISM_SERVE_BUILDER_PROTOCOL_VERSION, -) -from lab.prism.share_ledger import ( # noqa: E402 - DEFAULT_INCREMENTAL_SHARE_WINDOW_PAGE_SIZE, - AcceptedShareRecord, - IncrementalShareWindow, -) -from tests.perf.window_pipeline_gil_scaling import ( # noqa: E402 - benchmark_miner_programs, - build_records, -) - - -REPO_ROOT = Path(__file__).resolve().parents[2] -DAEMON_BINARY = "qbit-prism-build-audit-bundle" -DEFAULT_TOOL_BIN_DIR = REPO_ROOT / "target" / "release" -DEFAULT_SIZES = "52000,105000,210000,400000" -DEFAULT_BASELINE_SIZES = "52000,105000" -DEFAULT_MINERS = 200 -DEFAULT_SMALL_DELTA = 16 -DEFAULT_LARGE_DELTA = 2 * DEFAULT_INCREMENTAL_SHARE_WINDOW_PAGE_SIZE + 19 -WRITE_CHUNK = 1 << 20 -DEFAULT_EXCHANGE_TIMEOUT = 600.0 -DEFAULT_SHUTDOWN_TIMEOUT = 30.0 -PHASES = ("full", "advance_small", "advance_large", "recenter", "recenter_advance") - - -# --------------------------------------------------------------------------- -# host observation -# --------------------------------------------------------------------------- - - -def read_meminfo_mb(key: str) -> float | None: - try: - with open("/proc/meminfo", encoding="ascii") as handle: - for line in handle: - if line.startswith(key + ":"): - return int(line.split()[1]) / 1024.0 - except OSError: - return None - return None - - -def read_proc_status_mb(pid: int) -> dict[str, float]: - """``VmRSS``/``VmHWM``/``VmSize``/``VmPeak`` of a live process, in MiB.""" - out: dict[str, float] = {} - try: - with open(f"/proc/{pid}/status", encoding="ascii") as handle: - for line in handle: - key, _, rest = line.partition(":") - if key in ("VmRSS", "VmHWM", "VmSize", "VmPeak"): - out[key] = int(rest.split()[0]) / 1024.0 - except (OSError, ValueError): - pass - return out - - -def machine_info() -> dict[str, Any]: - try: - load = os.getloadavg() - except OSError: - load = None - return { - "platform": platform.platform(), - "python": sys.version.split()[0], - "implementation": platform.python_implementation(), - "cpu_count": os.cpu_count(), - "mem_total_mb": read_meminfo_mb("MemTotal"), - "mem_available_mb_at_start": read_meminfo_mb("MemAvailable"), - "load_average_at_start": load, - "malloc_arena_max": os.environ.get("MALLOC_ARENA_MAX"), - } - - -def git_describe(path: Path) -> dict[str, Any]: - """The source revision this driver runs from, and whether it is clean.""" - def run(*args: str) -> str | None: - try: - return subprocess.run( - ["git", *args], cwd=path, capture_output=True, text=True, check=True - ).stdout.strip() - except (OSError, subprocess.CalledProcessError): - return None - - return { - "commit": run("rev-parse", "HEAD"), - "branch": run("rev-parse", "--abbrev-ref", "HEAD"), - "dirty": bool(run("status", "--porcelain", "--untracked-files=no")), - } - - -def binary_identity(binary: Path) -> dict[str, Any]: - digest = hashlib.sha256() - with open(binary, "rb") as handle: - for chunk in iter(lambda: handle.read(1 << 20), b""): - digest.update(chunk) - stat = binary.stat() - return { - "path": str(binary), - "sha256": digest.hexdigest(), - "size_bytes": stat.st_size, - "mtime": time.strftime("%Y-%m-%dT%H:%M:%S", time.localtime(stat.st_mtime)), - } - - -# --------------------------------------------------------------------------- -# RSS sampling -# --------------------------------------------------------------------------- - - -class RssSampler: - """Samples ``VmRSS`` (and the running ``VmHWM``) of one pid on a thread. - - Kernel high-water marks and timeline maxima are observations made while - the child can still allocate, so both are lower bounds on its final - lifetime peak (see :func:`resolve_peak_rss`). - """ - - def __init__(self, pid: int, interval: float) -> None: - self.pid = pid - self.interval = interval - self.samples: list[tuple[float, float]] = [] - self.max_sampled_mb = 0.0 - self.max_sampled_hwm_mb = 0.0 - self._stop = threading.Event() - self._thread = threading.Thread(target=self._run, name="rss-sampler", daemon=True) - - def _run(self) -> None: - while not self._stop.is_set(): - status = read_proc_status_mb(self.pid) - rss = status.get("VmRSS") - if rss is not None: - self.samples.append((time.perf_counter(), rss)) - self.max_sampled_mb = max(self.max_sampled_mb, rss) - hwm = status.get("VmHWM") - if hwm is not None: - self.max_sampled_hwm_mb = max(self.max_sampled_hwm_mb, hwm) - self._stop.wait(self.interval) - - def __enter__(self) -> RssSampler: - self._thread.start() - return self - - def __exit__(self, *_exc: object) -> None: - self._stop.set() - self._thread.join() - - -# --------------------------------------------------------------------------- -# peak provenance -# --------------------------------------------------------------------------- - - -PEAK_SOURCE_LABELS = { - "vmhwm": "lower bound (kernel VmHWM at close)", - "vmhwm_before_kill": "lower bound (kernel VmHWM before watchdog kill)", - "lower_bound": "lower bound (highest earlier VmHWM/RSS observation)", - "unavailable": "unavailable", -} - - -def resolve_peak_rss(outcome: dict[str, Any], observations: list[float | None]) -> dict[str, Any]: - """Settle ``peak_rss_mb`` and its provenance in a daemon outcome. - - Keep the numeric observation and its source. A live kernel ``VmHWM`` - read at close or before SIGKILL is a lower bound: the daemon can still - allocate before it exits or the signal is delivered. Earlier phase and - sampler maxima are also lower bounds. None of these observations proves - the final lifetime peak, even if shutdown later exits cleanly. - """ - if outcome.get("peak_rss_mb") is not None: - outcome.setdefault("peak_rss_source", "vmhwm") - return outcome - candidates = [value for value in observations if value] - if candidates: - outcome["peak_rss_mb"] = max(candidates) - outcome["peak_rss_source"] = "lower_bound" - else: - outcome["peak_rss_mb"] = None - outcome["peak_rss_source"] = "unavailable" - return outcome - - -def format_peak(value: float | None, source: str | None, digits: int = 0) -> str: - """Render a peak figure, prefixing a lower bound with ``≥``.""" - if value is None: - return "-" - text = f"{value:.{digits}f}" - return f"≥ {text}" - - -def peak_source_label(source: str | None) -> str: - return PEAK_SOURCE_LABELS.get(source or "unavailable", str(source)) - - -# --------------------------------------------------------------------------- -# the daemon exchange -# --------------------------------------------------------------------------- - - -@dataclass -class Exchange: - request_bytes: int - request_write_seconds: float - response_wait_seconds: float - response_read_seconds: float - envelope: dict[str, Any] | None - payload: bytes | None - error: str | None = None - - -class DaemonTimeout(RuntimeError): - """An exchange (or the handshake) exceeded its wall-clock deadline.""" - - -class Daemon: - """One ``--serve`` daemon over blocking pipes, memory- and time-bounded. - - Every exchange, the handshake included, runs under ``exchange_timeout``: - a watchdog kills the daemon when it expires, which unblocks the pending - pipe read (EOF) and the pending write (EPIPE), and the exchange is - reported as a timeout failure. ``close`` is idempotent and always reaps - the child and closes every pipe and the stderr capture; the constructor - calls it before re-raising if the handshake fails. - """ - - def __init__( - self, - binary: Path, - *, - memory_limit_mb: int | None, - stderr_path: Path, - exchange_timeout: float = DEFAULT_EXCHANGE_TIMEOUT, - shutdown_timeout: float = DEFAULT_SHUTDOWN_TIMEOUT, - ) -> None: - self.binary = binary - self.stderr_path = stderr_path - self.exchange_timeout = exchange_timeout - self.shutdown_timeout = shutdown_timeout - self.exit_code: int | None = None - self.timed_out: str | None = None - self.status_at_kill: dict[str, float] = {} - self._outcome: dict[str, Any] | None = None - self._lock = threading.Lock() - limit_bytes = None if not memory_limit_mb else memory_limit_mb * 1024 * 1024 - - def preexec() -> None: - if limit_bytes is not None: - resource.setrlimit(resource.RLIMIT_AS, (limit_bytes, limit_bytes)) - - self._stderr = open(stderr_path, "wb") - try: - self.process = subprocess.Popen( - [ - str(binary), - "--serve", - "--signing-key-seed-hex", - "42" * 32, - "--ledger-signing-key-seed-hex", - "43" * 32, - ], - stdin=subprocess.PIPE, - stdout=subprocess.PIPE, - stderr=self._stderr, - preexec_fn=preexec, - ) - except Exception: - self._stderr.close() - raise - try: - assert self.process.stdout is not None - with self._watchdog("handshake"): - handshake_line = self.process.stdout.readline() - if self.timed_out: - raise DaemonTimeout( - f"{binary} sent no handshake within {exchange_timeout:g}s; killed" - ) - if not handshake_line: - raise RuntimeError(f"{binary} produced no handshake (see {stderr_path})") - handshake = json.loads(handshake_line) - expected = { - "event": "handshake", - "tool": DAEMON_BINARY, - "protocol": PRISM_SERVE_BUILDER_PROTOCOL_VERSION, - } - announced = {key: handshake.get(key) for key in expected} - if announced != expected: - raise RuntimeError(f"daemon announced {announced!r}, expected {expected!r}") - except BaseException as error: - outcome = self.close() - if isinstance(error, Exception): - # Let a caller (or a test) see how the child was reaped. - error.daemon_outcome = outcome # type: ignore[attr-defined] - raise - - @property - def pid(self) -> int: - return self.process.pid - - def alive(self) -> bool: - return self.process.poll() is None - - def status_mb(self) -> dict[str, float]: - return read_proc_status_mb(self.process.pid) - - def _kill_on_timeout(self, label: str) -> None: - with self._lock: - if self.timed_out is None: - self.timed_out = label - # The high-water mark is lost once the child is reaped, so - # sample it before the kill. - self.status_at_kill = self.status_mb() - if self.process.poll() is None: - self.process.kill() - - class _Watchdog: - def __init__(self, daemon: Daemon, label: str) -> None: - self.timer = threading.Timer(daemon.exchange_timeout, daemon._kill_on_timeout, (label,)) - self.timer.daemon = True - - def __enter__(self) -> Daemon._Watchdog: - self.timer.start() - return self - - def __exit__(self, *_exc: object) -> None: - self.timer.cancel() - - def _watchdog(self, label: str) -> Daemon._Watchdog: - return Daemon._Watchdog(self, label) - - def exchange(self, request: bytes, label: str = "exchange") -> Exchange: - """Write one request line, read the envelope and its raw section. - - Bounded by ``exchange_timeout`` end to end; a timeout kills the daemon - and is reported in ``Exchange.error``. - """ - process = self.process - assert process.stdin is not None and process.stdout is not None - if self.timed_out or process.poll() is not None: - return Exchange(len(request), 0.0, 0.0, 0.0, None, None, error="daemon is not running") - stdin_fd = process.stdin.fileno() - stdout = process.stdout - write_done: list[float] = [] - write_error: list[str] = [] - - def writer() -> None: - view = memoryview(request) - offset = 0 - try: - while offset < len(view): - offset += os.write(stdin_fd, view[offset : offset + WRITE_CHUNK]) - except OSError as exc: - write_error.append(f"{type(exc).__name__}: {exc}") - finally: - write_done.append(time.perf_counter()) - - def timeout_error(detail: str) -> str: - return f"{label} exceeded {self.exchange_timeout:g}s; daemon killed ({detail})" - - with self._watchdog(label): - started = time.perf_counter() - thread = threading.Thread(target=writer, name="request-writer", daemon=True) - thread.start() - envelope_line = stdout.readline() - envelope_at = time.perf_counter() - # The write either finished, failed with EPIPE once the daemon - # died, or is stuck behind a daemon that stopped reading, which - # the watchdog kill also unblocks; the join is bounded regardless. - thread.join(timeout=max(1.0, self.exchange_timeout)) - if thread.is_alive(): - write_done.append(time.perf_counter()) - write_error.append("request writer still blocked after the exchange") - write_seconds = write_done[0] - started - wait_seconds = envelope_at - write_done[0] - if not envelope_line: - detail = "closed stdout without an envelope" + ( - f"; write: {write_error[0]}" if write_error else "" - ) - error = timeout_error(detail) if self.timed_out else f"daemon {detail}" - return Exchange(len(request), write_seconds, wait_seconds, 0.0, None, None, error=error) - envelope = json.loads(envelope_line) - payload: bytes | None = None - read_seconds = 0.0 - if envelope.get("ok") is True: - length_key = "window_items_len" if "window_items_len" in envelope else "appended_items_len" - length = int(envelope[length_key]) - chunks: list[bytes] = [] - remaining = length - while remaining > 0: - chunk = stdout.read(min(remaining, 1 << 22)) - if not chunk: - break - chunks.append(chunk) - remaining -= len(chunk) - newline = stdout.read(1) - read_seconds = time.perf_counter() - envelope_at - payload = b"".join(chunks) - if remaining or newline != b"\n": - detail = f"short raw section: {len(payload)} of {length} bytes" - error = timeout_error(detail) if self.timed_out else detail - return Exchange(len(request), write_seconds, wait_seconds, read_seconds, envelope, payload, error=error) - return Exchange(len(request), write_seconds, wait_seconds, read_seconds, envelope, payload) - - def close(self) -> dict[str, Any]: - """Reap the daemon and close every handle; idempotent. - - Closing stdin asks a healthy daemon to exit; one that has not exited - within ``shutdown_timeout`` is killed. ``VmHWM`` is read before the - child is reaped (it is gone from ``/proc`` afterwards); the outcome's - ``peak_rss_source`` records whether the figure is that live read - (``vmhwm``), the watchdog's read before a kill - (``vmhwm_before_kill``), or ``unavailable`` because the daemon had - already exited -- :func:`resolve_peak_rss` may then substitute a - labelled lower bound. All live reads are lower bounds too: memory - may still grow before normal exit or before SIGKILL is delivered. - """ - if self._outcome is not None: - return self._outcome - process = self.process - final_status = self.status_mb() if process.poll() is None else {} - if process.stdin is not None: - try: - process.stdin.close() - except OSError: - pass - try: - self.exit_code = process.wait(timeout=self.shutdown_timeout) - except subprocess.TimeoutExpired: - process.kill() - self.exit_code = process.wait(timeout=10.0) - if process.stdout is not None: - process.stdout.close() - self._stderr.close() - stderr_tail = "" - try: - stderr_tail = self.stderr_path.read_text(errors="replace")[-2000:] - except OSError: - pass - if final_status.get("VmHWM") is not None: - peak, peak_source = final_status["VmHWM"], "vmhwm" - elif self.status_at_kill.get("VmHWM") is not None: - peak, peak_source = self.status_at_kill["VmHWM"], "vmhwm_before_kill" - else: - peak, peak_source = None, "unavailable" - peak_vsize = final_status.get("VmPeak", self.status_at_kill.get("VmPeak")) - self._outcome = { - "pid": process.pid, - "exit_code": self.exit_code, - "signal": -self.exit_code if self.exit_code is not None and self.exit_code < 0 else None, - "timed_out": self.timed_out, - "peak_rss_mb": peak, - "peak_rss_source": peak_source, - "peak_vsize_mb": peak_vsize, - "stderr_tail": stderr_tail, - } - return self._outcome - - -# --------------------------------------------------------------------------- -# fixture and requests -# --------------------------------------------------------------------------- - - -def delta_records(last: AcceptedShareRecord, count: int, *, miners: int, offset: int) -> list[AcceptedShareRecord]: - """``count`` appended records after ``last``, shaped like the probe's delta.""" - programs = benchmark_miner_programs(miners) - out: list[AcceptedShareRecord] = [] - for index in range(count): - seq = int(last.share_seq) + offset + index + 1 - miner = seq % miners - out.append( - AcceptedShareRecord( - share_seq=seq, - share_id=f"bench-miner-{miner}.rig{seq % 512}:{'ab' * 32}", - miner_id=f"bench-miner-{miner}", - order_key=f"bench-miner-{miner:06d}", - p2mr_program_hex=programs[miner], - share_difficulty=int(last.share_difficulty), - network_difficulty=int(last.network_difficulty), - template_height=int(last.template_height), - job_id=f"bench-job-{seq}", - job_issued_at_ms=int(last.job_issued_at_ms) + offset + index + 1, - accepted_at_ms=int(last.accepted_at_ms) + offset + index + 1, - ntime=int(last.ntime) + offset + index + 1, - ) - ) - return out - - -def encode_request(fields: dict[str, Any]) -> bytes: - """The bytes the coordinator sends: one compact JSON line.""" - return json.dumps(fields, separators=(",", ":")).encode("ascii") + b"\n" - - -def full_request(records_json: list[dict[str, object]], *, anchor: int, weight: int, page_size: int) -> bytes: - return encode_request( - { - "request": "prepare_window", - "mode": "full", - "append_invalidation_epoch": 0, - "anchor_job_issued_at_ms": anchor, - "records": records_json, - "window_weight": weight, - "page_size": page_size, - } - ) - - -def advance_request(records_json: list[dict[str, object]], *, anchor: int, base_digest: str) -> bytes: - return encode_request( - { - "request": "prepare_window", - "mode": "advance", - "append_invalidation_epoch": 0, - "anchor_job_issued_at_ms": anchor, - "records": records_json, - "base_digest": base_digest, - } - ) - - -def items_digest(items: bytes) -> str: - return hashlib.sha256(b"[" + items + b"]").hexdigest() - - -@dataclass -class Fixture: - size: int - page_size: int - records: list[AcceptedShareRecord] - records_json: list[dict[str, object]] - anchor: int - weight: int - recenter_weight: int - small_delta: list[AcceptedShareRecord] - small_anchor: int - large_delta: list[AcceptedShareRecord] - large_anchor: int - recenter_delta: list[AcceptedShareRecord] - recenter_delta_anchor: int - build_seconds: float - - -def build_fixture(size: int, *, miners: int, page_size: int, small: int, large: int) -> Fixture: - started = time.perf_counter() - records = build_records(size, miners=miners, share_id_shape="production") - anchor = int(records[-1].job_issued_at_ms) - weight = sum(int(record.share_difficulty) for record in records) - last = records[-1] - small_delta = delta_records(last, small, miners=miners, offset=0) - small_anchor = int(last.job_issued_at_ms) + small + 1 - large_delta = delta_records(last, large, miners=miners, offset=small + 1) - large_anchor = int(last.job_issued_at_ms) + small + 1 + large + 1 - # The re-center starts again from the original snapshot at a smaller - # live weight; its advance appends after the original last record. - recenter_delta = delta_records(last, small, miners=miners, offset=0) - return Fixture( - size=size, - page_size=page_size, - records=records, - records_json=[record.to_prism_json() for record in records], - anchor=anchor, - weight=weight, - recenter_weight=(weight * 3) // 4, - small_delta=small_delta, - small_anchor=small_anchor, - large_delta=large_delta, - large_anchor=large_anchor, - recenter_delta=recenter_delta, - recenter_delta_anchor=small_anchor, - build_seconds=time.perf_counter() - started, - ) - - -# --------------------------------------------------------------------------- -# the Python oracle -# --------------------------------------------------------------------------- - - -def python_oracle(fixture: Fixture) -> dict[str, Any]: - """Every expected digest from the shipped in-process fold.""" - timings: dict[str, float] = {} - digests: dict[str, str] = {} - counts: dict[str, int] = {} - - def timed(name: str, run: Callable[[], IncrementalShareWindow]) -> IncrementalShareWindow: - started = time.perf_counter() - window = run() - timings[name] = time.perf_counter() - started - digests[name] = window.json_records().canonical_json_sha256() - counts[name] = int(window.record_count) - return window - - full = timed( - "full", - lambda: IncrementalShareWindow.from_full_snapshot( - fixture.records, - anchor_job_issued_at_ms=fixture.anchor, - window_weight=fixture.weight, - page_size=fixture.page_size, - ), - ) - small = timed( - "advance_small", - lambda: full.advance(fixture.small_delta, anchor_job_issued_at_ms=fixture.small_anchor)[0], - ) - timed( - "advance_large", - lambda: small.advance(fixture.large_delta, anchor_job_issued_at_ms=fixture.large_anchor)[0], - ) - recentered = timed( - "recenter", - lambda: IncrementalShareWindow.from_full_snapshot( - fixture.records, - anchor_job_issued_at_ms=fixture.anchor, - window_weight=fixture.recenter_weight, - page_size=fixture.page_size, - ), - ) - timed( - "recenter_advance", - lambda: recentered.advance( - fixture.recenter_delta, anchor_job_issued_at_ms=fixture.recenter_delta_anchor - )[0], - ) - return {"digests": digests, "record_counts": counts, "seconds": timings} - - -# --------------------------------------------------------------------------- -# one daemon run -# --------------------------------------------------------------------------- - - -@dataclass -class PhaseResult: - phase: str - records_sent: int - request_bytes: int - request_write_seconds: float - response_wait_seconds: float - response_read_seconds: float - daemon_metrics: dict[str, float] = field(default_factory=dict) - status: str = "" - digest: str | None = None - record_count: int | None = None - items_bytes: int | None = None - self_consistent: bool | None = None - oracle_match: bool | None = None - rss_after_mb: float | None = None - hwm_after_mb: float | None = None - stats: dict[str, int] = field(default_factory=dict) - error: str | None = None - - @property - def residual_wait_seconds(self) -> float | None: - """``response_wait`` minus the daemon's own timers -- an approximation. - - The client's wall interval (from the last request byte accepted to - the envelope line received) and the daemon's ``Instant`` timers are - independent intervals measured on either side of a pipe: the daemon - may still be reading and parsing while the client clock starts, and - the envelope write and scheduling are on neither timer. So this is - roughly scheduling plus envelope handling, not a measured transport - term, and it is clamped at zero. None when the daemon reported no - metrics. - """ - if not self.daemon_metrics: - return None - return max(0.0, self.response_wait_seconds - sum(self.daemon_metrics.values())) - - -@dataclass -class RunResult: - variant: str - binary: str - size: int - page_size: int - mem_available_before_mb: float | None - load_before: tuple[float, float, float] | None - memory_limit_mb: int | None - phases: list[PhaseResult] - outcome: dict[str, Any] - max_sampled_rss_mb: float - rss_timeline: list[tuple[float, float]] - skipped: str | None = None - error: str | None = None - - -def run_variant( - label: str, - binary: Path, - fixture: Fixture, - *, - oracle: dict[str, Any] | None, - memory_limit_mb: int | None, - memory_margin_mb: int, - sample_interval: float, - exchange_timeout: float, - stderr_dir: Path, - log: Callable[[str], None], -) -> RunResult: - available = read_meminfo_mb("MemAvailable") - try: - load = os.getloadavg() - except OSError: - load = None - required = (memory_limit_mb or 0) + memory_margin_mb - if available is not None and available < required: - reason = f"MemAvailable {available:.0f} MiB below required {required} MiB" - log(f" {label} @ {fixture.size}: skipped, {reason}") - return RunResult( - label, str(binary), fixture.size, fixture.page_size, available, load, - memory_limit_mb, [], {}, 0.0, [], skipped=reason, - ) - - stderr_path = stderr_dir / f"{label}-{fixture.size}.stderr" - try: - daemon = Daemon( - binary, - memory_limit_mb=memory_limit_mb, - stderr_path=stderr_path, - exchange_timeout=exchange_timeout, - ) - except Exception as exc: # noqa: BLE001 - reported per run, not raised - error = f"daemon failed to start: {type(exc).__name__}: {exc}" - log(f" {label} @ {fixture.size}: {error}") - return RunResult( - label, str(binary), fixture.size, fixture.page_size, available, load, - memory_limit_mb, [], getattr(exc, "daemon_outcome", {}), 0.0, [], error=error, - ) - phases: list[PhaseResult] = [] - expected = (oracle or {}).get("digests", {}) - - def run_phase(name: str, request: bytes, records_sent: int, *, base_items: bytes | None) -> tuple[PhaseResult, bytes | None]: - log(f" {label} @ {fixture.size}: {name} ({records_sent} records, {len(request) / 2**20:.1f} MiB request)") - exchange = daemon.exchange(request, label=name) - status_after = daemon.status_mb() if daemon.alive() else {} - result = PhaseResult( - phase=name, - records_sent=records_sent, - request_bytes=exchange.request_bytes, - request_write_seconds=exchange.request_write_seconds, - response_wait_seconds=exchange.response_wait_seconds, - response_read_seconds=exchange.response_read_seconds, - rss_after_mb=status_after.get("VmRSS"), - hwm_after_mb=status_after.get("VmHWM"), - error=exchange.error, - ) - envelope = exchange.envelope - if envelope is None: - result.status = "timeout" if daemon.timed_out else "no_response" - return result, None - metrics = envelope.get("metrics") - if isinstance(metrics, dict): - result.daemon_metrics = {key: float(value) for key, value in metrics.items()} - if envelope.get("ok") is not True: - for key in ("needs_full", "fallback", "fold_invalid", "out_of_range"): - if envelope.get(key): - result.status = key - break - else: - result.status = "error" - result.error = str(envelope.get("error")) - return result, None - if exchange.error: - result.status = "timeout" if daemon.timed_out else "short_response" - return result, None - result.status = "prepared" - result.digest = str(envelope["share_snapshot_sha256"]) - result.record_count = int(envelope["record_count"]) - payload = exchange.payload or b"" - if "window_items_len" in envelope: - items = payload - else: - assert base_items is not None - items = base_items[int(envelope["retained_drop_bytes"]) :] + payload - result.stats = { - key: int(envelope[key]) for key in ("added_rows", "expired_rows", "touched_pages") - } - result.items_bytes = len(items) - result.self_consistent = items_digest(items) == result.digest - if name in expected: - result.oracle_match = expected[name] == result.digest - return result, items - - run_error: str | None = None - max_sampled = 0.0 - max_sampled_hwm = 0.0 - timeline: list[tuple[float, float]] = [] - try: - with RssSampler(daemon.pid, sample_interval) as sampler: - try: - full, items = run_phase( - "full", - full_request(fixture.records_json, anchor=fixture.anchor, weight=fixture.weight, page_size=fixture.page_size), - fixture.size, - base_items=None, - ) - phases.append(full) - if full.status == "prepared" and items is not None and daemon.alive(): - small, items_small = run_phase( - "advance_small", - advance_request([r.to_prism_json() for r in fixture.small_delta], anchor=fixture.small_anchor, base_digest=full.digest or ""), - len(fixture.small_delta), - base_items=items, - ) - phases.append(small) - if small.status == "prepared" and items_small is not None and daemon.alive(): - large, _ = run_phase( - "advance_large", - advance_request([r.to_prism_json() for r in fixture.large_delta], anchor=fixture.large_anchor, base_digest=small.digest or ""), - len(fixture.large_delta), - base_items=items_small, - ) - phases.append(large) - if daemon.alive() and full.status == "prepared": - recenter, items_recenter = run_phase( - "recenter", - full_request(fixture.records_json, anchor=fixture.anchor, weight=fixture.recenter_weight, page_size=fixture.page_size), - fixture.size, - base_items=None, - ) - phases.append(recenter) - if recenter.status == "prepared" and items_recenter is not None and daemon.alive(): - recenter_advance, _ = run_phase( - "recenter_advance", - advance_request([r.to_prism_json() for r in fixture.recenter_delta], anchor=fixture.recenter_delta_anchor, base_digest=recenter.digest or ""), - len(fixture.recenter_delta), - base_items=items_recenter, - ) - phases.append(recenter_advance) - except Exception as exc: # noqa: BLE001 - reported per run, not raised - run_error = f"{type(exc).__name__}: {exc}" - log(f" {label} @ {fixture.size}: aborted: {run_error}") - finally: - max_sampled = sampler.max_sampled_mb - max_sampled_hwm = sampler.max_sampled_hwm_mb - timeline = sampler.samples - finally: - # Always reaps the child and closes every pipe and the stderr - # capture, whatever happened above. - outcome = daemon.close() - # A daemon that exited on its own took its VmHWM with it; what remains - # is a labelled lower bound from the observations made while it lived. - resolve_peak_rss( - outcome, - [phase.hwm_after_mb for phase in phases] + [max_sampled_hwm, max_sampled], - ) - return RunResult( - label, - str(binary), - fixture.size, - fixture.page_size, - available, - load, - memory_limit_mb, - phases, - outcome, - max_sampled, - timeline, - error=run_error, - ) - - -# --------------------------------------------------------------------------- -# rendering -# --------------------------------------------------------------------------- - - -def _fmt(value: float | None, digits: int = 2) -> str: - return "-" if value is None else f"{value:.{digits}f}" - - -def _flag(value: bool | None) -> str: - if value is None: - return "n/a" - return "yes" if value else "NO" - - -def render_runs(runs: list[RunResult]) -> str: - lines = [ - "| binary | shares | phase | records | request MiB | write s | wait s | read s | parse s | fold s | serialize s | residual s (approx.) | RSS after MiB | status | self-check | oracle |", - "|---|---:|---|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---|---|---|", - ] - for run in runs: - if run.skipped or (run.error and not run.phases): - lines.append(f"| {run.variant} | {run.size:,} | {'skipped' if run.skipped else 'failed'} | | | | | | | | | | | {run.skipped or run.error} | | |") - continue - for phase in run.phases: - metrics = phase.daemon_metrics - lines.append( - f"| {run.variant} | {run.size:,} | {phase.phase} | {phase.records_sent:,} |" - f" {phase.request_bytes / 2**20:.1f} | {_fmt(phase.request_write_seconds)} |" - f" {_fmt(phase.response_wait_seconds)} | {_fmt(phase.response_read_seconds)} |" - f" {_fmt(metrics.get('input_deserialization_seconds'), 3)} |" - f" {_fmt(metrics.get('fold_seconds'), 3)} |" - f" {_fmt(metrics.get('output_serialization_seconds'), 3)} |" - f" {_fmt(phase.residual_wait_seconds, 3)} |" - f" {_fmt(phase.rss_after_mb, 0)} | {phase.status}" - f"{' (' + phase.error + ')' if phase.error else ''} |" - f" {_flag(phase.self_consistent)} | {_flag(phase.oracle_match)} |" - ) - return "\n".join(lines) - - -def render_summary(runs: list[RunResult]) -> str: - lines = [ - "| binary | shares | peak RSS MiB | peak source | peak VSZ MiB | MiB per 1k shares | full wait s | full fold s | daemon exit | available MiB before |", - "|---|---:|---:|---|---:|---:|---:|---:|---|---:|", - ] - for run in runs: - if run.skipped: - lines.append(f"| {run.variant} | {run.size:,} | skipped: {run.skipped} | | | | | | | {_fmt(run.mem_available_before_mb, 0)} |") - continue - full = next((phase for phase in run.phases if phase.phase == "full"), None) - peak = run.outcome.get("peak_rss_mb") - peak_source = run.outcome.get("peak_rss_source") - exit_code = run.outcome.get("exit_code") - signal = run.outcome.get("signal") - ended = "clean (0)" if exit_code == 0 else (f"signal {signal}" if signal else f"exit {exit_code}") - if exit_code is None: - ended = "unavailable" - if run.outcome.get("timed_out"): - ended += f", killed on {run.outcome['timed_out']} timeout" - if run.error and not run.phases: - ended += f", failed: {run.error}" - lines.append( - f"| {run.variant} | {run.size:,} | {format_peak(peak, peak_source)} | {peak_source_label(peak_source)} |" - f" {format_peak(run.outcome.get('peak_vsize_mb'), peak_source)} |" - f" {format_peak(None if peak is None else peak / (run.size / 1000.0), peak_source, 2)} |" - f" {_fmt(full.response_wait_seconds if full else None)} |" - f" {_fmt(full.daemon_metrics.get('fold_seconds') if full else None, 3)} |" - f" {ended} | {_fmt(run.mem_available_before_mb, 0)} |" - ) - return "\n".join(lines) - - -def parse_sizes(text: str) -> list[int]: - return [int(float(item)) for item in text.split(",") if item.strip()] - - -def resolve_binary(text: str | None, *, default: Path | None) -> Path | None: - if text: - candidate = Path(text) - elif default is not None: - candidate = default - else: - return None - if candidate.is_dir(): - candidate = candidate / DAEMON_BINARY - if not (candidate.is_file() and os.access(candidate, os.X_OK)): - raise SystemExit(f"{candidate} is not an executable daemon binary") - return candidate.resolve() - - -def main(argv: list[str] | None = None) -> int: - parser = argparse.ArgumentParser( - description=__doc__, - formatter_class=argparse.RawDescriptionHelpFormatter, - ) - parser.add_argument( - "--daemon-binary", - default=os.environ.get("PRISM_TOOL_BIN_DIR") or str(DEFAULT_TOOL_BIN_DIR), - help="the fixed qbit-prism-build-audit-bundle (a file, or a directory holding it; default $PRISM_TOOL_BIN_DIR or target/release)", - ) - parser.add_argument("--daemon-label", default="fixed", help="row label for --daemon-binary (default fixed)") - parser.add_argument("--baseline-binary", default=None, help="an unfixed daemon binary to measure at --baseline-sizes only") - parser.add_argument("--baseline-label", default="baseline", help="row label for --baseline-binary") - parser.add_argument("--sizes", default=DEFAULT_SIZES, help=f"window sizes for the fixed binary (default {DEFAULT_SIZES})") - parser.add_argument("--baseline-sizes", default=DEFAULT_BASELINE_SIZES, help=f"window sizes for the baseline binary (default {DEFAULT_BASELINE_SIZES}; keep these inside the memory limit)") - parser.add_argument("--page-size", type=int, default=DEFAULT_INCREMENTAL_SHARE_WINDOW_PAGE_SIZE) - parser.add_argument("--miners", type=int, default=DEFAULT_MINERS, help=f"distinct identities in the fixture (default {DEFAULT_MINERS})") - parser.add_argument("--small-delta", type=int, default=DEFAULT_SMALL_DELTA, help=f"records in the small advance (default {DEFAULT_SMALL_DELTA})") - parser.add_argument("--large-delta", type=int, default=DEFAULT_LARGE_DELTA, help=f"records in the multi-page advance (default {DEFAULT_LARGE_DELTA})") - parser.add_argument("--daemon-memory-limit-mb", type=int, default=6144, help="RLIMIT_AS for every daemon in MiB; 0 disables (default 6144)") - parser.add_argument("--memory-margin-mb", type=int, default=2048, help="MemAvailable headroom required above the limit before a run (default 2048)") - parser.add_argument("--sample-interval", type=float, default=0.02, help="RSS sampling interval in seconds (default 0.02)") - parser.add_argument("--exchange-timeout", type=float, default=DEFAULT_EXCHANGE_TIMEOUT, help=f"wall-clock deadline per exchange, handshake included; the daemon is killed and the phase recorded as a timeout (default {DEFAULT_EXCHANGE_TIMEOUT:g})") - parser.add_argument("--skip-oracle", action="store_true", help="do not run the Python fold as a digest oracle") - parser.add_argument("--stderr-dir", default=None, help="directory for daemon stderr captures (default: a temporary directory)") - parser.add_argument("--json", default=None, help="write structured results to this file") - parser.add_argument("--quiet", action="store_true") - args = parser.parse_args(argv) - - def log(text: str) -> None: - if not args.quiet: - print(text, flush=True) - - fixed = resolve_binary(args.daemon_binary, default=DEFAULT_TOOL_BIN_DIR) - baseline = resolve_binary(args.baseline_binary, default=None) - assert fixed is not None - memory_limit = args.daemon_memory_limit_mb or None - stderr_dir = Path(args.stderr_dir) if args.stderr_dir else Path( - os.environ.get("TMPDIR", "/tmp") - ) / f"window_paging_allocation-{os.getpid()}" - stderr_dir.mkdir(parents=True, exist_ok=True) - - info = machine_info() - source = git_describe(REPO_ROOT) - binaries = {args.daemon_label: binary_identity(fixed)} - if baseline is not None: - binaries[args.baseline_label] = binary_identity(baseline) - - print("# window_paging_allocation") - print() - for key, value in info.items(): - print(f"- {key}: {value}") - print(f"- source: {source}") - for label, identity in binaries.items(): - print(f"- {label}: {identity['path']} (sha256 {identity['sha256'][:16]}..., {identity['mtime']})") - print(f"- page_size: {args.page_size}; miners: {args.miners}; deltas: {args.small_delta} / {args.large_delta}") - print(f"- daemon RLIMIT_AS: {memory_limit or 'none'} MiB; required MemAvailable: {(memory_limit or 0) + args.memory_margin_mb} MiB; exchange timeout: {args.exchange_timeout:g} s") - print() - - plan: list[tuple[str, Path, int]] = [] - sizes = parse_sizes(args.sizes) - baseline_sizes = parse_sizes(args.baseline_sizes) if baseline is not None else [] - for size in sorted(set(sizes) | set(baseline_sizes)): - if baseline is not None and size in baseline_sizes: - plan.append((args.baseline_label, baseline, size)) - if size in sizes: - plan.append((args.daemon_label, fixed, size)) - - runs: list[RunResult] = [] - oracles: dict[int, dict[str, Any]] = {} - fixtures_seconds: dict[int, float] = {} - current_size: int | None = None - fixture: Fixture | None = None - oracle: dict[str, Any] | None = None - - def skip_parent_work_if_low_memory(label: str, binary: Path, size: int, stage: str) -> bool: - available = read_meminfo_mb("MemAvailable") - required = (memory_limit or 0) + args.memory_margin_mb - if available is None or available >= required: - return False - reason = f"MemAvailable {available:.0f} MiB below required {required} MiB before {stage}" - log(f" {label} @ {size}: skipped, {reason}") - runs.append(RunResult( - label, str(binary), size, args.page_size, available, None, - memory_limit, [], {}, 0.0, [], skipped=reason, - )) - return True - - for label, binary, size in plan: - if size != current_size: - # Release the preceding fixture before checking headroom for - # the next parent-side allocation. The daemon admission check - # alone is too late to protect fixture/oracle construction. - fixture = None - oracle = None - if skip_parent_work_if_low_memory(label, binary, size, "fixture construction"): - continue - log(f" building {size:,}-record fixture ...") - fixture = build_fixture(size, miners=args.miners, page_size=args.page_size, small=args.small_delta, large=args.large_delta) - fixtures_seconds[size] = fixture.build_seconds - if not args.skip_oracle: - if skip_parent_work_if_low_memory(label, binary, size, "Python oracle"): - fixture = None - continue - log(f" running the Python fold oracle at {size:,} ...") - oracle = python_oracle(fixture) - oracles[size] = oracle - current_size = size - assert fixture is not None - runs.append( - run_variant( - label, - binary, - fixture, - oracle=oracle, - memory_limit_mb=memory_limit, - memory_margin_mb=args.memory_margin_mb, - sample_interval=args.sample_interval, - exchange_timeout=args.exchange_timeout, - stderr_dir=stderr_dir, - log=log, - ) - ) - last = runs[-1] - if not last.skipped and last.phases: - log( - f" -> peak RSS {format_peak(last.outcome.get('peak_rss_mb'), last.outcome.get('peak_rss_source'))} MiB" - f" ({peak_source_label(last.outcome.get('peak_rss_source'))}), exit {last.outcome.get('exit_code')}," - f" phases {[(p.phase, p.status) for p in last.phases]}" - ) - - print("## Summary per daemon") - print() - print(render_summary(runs)) - print() - print("## Phases") - print() - print(render_runs(runs)) - print() - if oracles: - print("## Python oracle (in-process fold on the same inputs)") - print() - print("| shares | phase | seconds | records | digest |") - print("|---:|---|---:|---:|---|") - for size, oracle in sorted(oracles.items()): - for phase in PHASES: - print(f"| {size:,} | {phase} | {oracle['seconds'][phase]:.2f} | {oracle['record_counts'][phase]:,} | {oracle['digests'][phase][:16]}... |") - print() - - if args.json: - output = { - "machine": info, - "source": source, - "binaries": binaries, - "arguments": vars(args), - "fixture_build_seconds": fixtures_seconds, - "oracles": oracles, - "runs": [ - { - **{key: value for key, value in asdict(run).items() if key != "rss_timeline"}, - "phases": [ - {**asdict(phase), "residual_wait_seconds": phase.residual_wait_seconds} - for phase in run.phases - ], - "rss_timeline_mb": [ - [round(t - run.rss_timeline[0][0], 3), round(rss, 1)] for t, rss in run.rss_timeline - ] if run.rss_timeline else [], - } - for run in runs - ], - } - Path(args.json).write_text(json.dumps(output, indent=2, default=str) + "\n") - print(f"wrote {args.json}") - return 0 - - -if __name__ == "__main__": - sys.exit(main()) diff --git a/tests/perf/window_pipeline_gil_scaling.py b/tests/perf/window_pipeline_gil_scaling.py deleted file mode 100644 index e5d23ab27..000000000 --- a/tests/perf/window_pipeline_gil_scaling.py +++ /dev/null @@ -1,2390 +0,0 @@ -#!/usr/bin/env python3 -"""GIL scaling of the PRISM payout-window materialization pipeline. - -Issue #131 profiles one window materialization as ~135 ms of **GIL-held** -Python at today's 21,868-share window (~555 ms at 100k, ~2.2 s at 400k), split -across four terms: window fold, canonical-JSON digest, spool serialization, -and record->JSON conversion. "GIL-held" was *asserted* for this pipeline and -never measured on it -- the status comment #162 left on #143 names exactly -this gap: "GIL behaviour at megabyte buffer sizes is unmeasured." - -It matters because #143 section 1 bracketed CPython's GIL-release threshold at -**1-2 KiB** and reached **7.89 cores across 8 threads** on a 1 MiB -``hashlib.sha256`` positive control. Every buffer in this pipeline is far past -that threshold. A term built out of GIL-releasing primitives is already -running in parallel and is not evidence for a migration. - -This driver answers, per stage: **how much of the profiled time is actually -GIL-held, and how much is already parallel?** - -Method mirrors #143 section 1 so the numbers are comparable: - -* **cores-used = (process CPU time) / (wall time)** with the stage running in - N = 1, 2, 4, 8 threads, each thread driving its **own independent input**. - 1.00 flat from 1->8 threads means fully GIL-held; scaling toward N means the - stage releases the GIL. -* a **1 MiB ``hashlib.sha256`` positive control** and a **small-buffer - pure-Python negative control** are measured in the same process, at the same - thread counts, in the same alternation as the stages, and are reported beside - every table. The positive control sets this host's achievable ceiling; the - negative control must pin at ~1.00. -* configurations are visited **alternating** (outer loop = repetition, inner - loop = configuration) rather than in blocks, so host drift lands on every - configuration equally rather than on whichever ran last. - -**Thread-start amortization is load-bearing and was measured, not assumed.** -On this rig the same 1 MiB sha256 positive control reads 5.33 cores at 0.26 s -of wall and 7.42 cores at 0.90 s -- the shortfall is thread create/join -overhead charged against too short a measurement, not a GIL effect. Every -configuration is therefore grown until its wall clock clears -``--min-wall-seconds`` (default 1.0 s). Skipping this understates cores-used -on *every* stage, which would have manufactured a false confirmation of the -"GIL-held" premise. See ``AMORTIZATION_RATIONALE``. - -Stages drive the **shipped** callables -- nothing is reimplemented: - -=========================== ================================================== -stage callable -=========================== ================================================== -``fold`` ``IncrementalShareWindow.from_full_snapshot`` -``fold_pages`` ``_IncrementalShareWindowPage.from_records`` (all) -``digest`` ``IncrementalShareJsonSequence.canonical_json_sha256`` -``to_prism_json`` ``AcceptedShareRecord.to_prism_json`` (all records) -``spool_acquire`` ``_ShareWindowSerialization.acquire_spooled_tail`` -``spool_compact`` ``_ShareWindowSerialization.compact_tail_chunks`` -``spool_encode`` ``str.encode("utf-8")`` of both fragments -``spool_write`` ``TemporaryFile`` write/flush/seek of the payload -=========================== ================================================== - -``fold_pages``, ``spool_compact``, ``spool_encode`` and ``spool_write`` are -sub-terms, reported to decompose the two stages that are internally mixed; -they are not additional profile rows. - -Records are shaped like ``lab/prism/job_build_benchmark.py``'s defaults -(``--shares 21868 --miners 2``), whose values encode a live-host measurement. -``--share-id-shape production`` re-runs the byte accounting with the -``username:block_hash_hex`` share_id that ``lab/prism/share_writer.py`` builds, -because the benchmark's short synthetic ``share_id`` materially changes payload -bytes (but not, as measured, any GIL verdict). - -Self-contained and re-runnable: no database, no network, no coordinator, no -third-party packages. ``python3 tests/perf/window_pipeline_gil_scaling.py`` -prints the tables; ``--json`` emits the same data structured. - -This is an **on-demand instrument, not a test**: it asserts no thresholds and -is deliberately not named ``test_*`` so the discovery run never executes it -(#160 -- a threshold assertion on a shared runner is a flaky test in waiting). -Nothing under ``lab/`` is imported for anything but read-only measurement. - -``--latency-probe`` (#236) is a second mode over the same fixture: instead of -cores-used across threads it runs each serialization phase once at the -incident's window sizes (210k and 400k shares by default) in the main thread -while a monitor thread wakes on the writer-lease monitor's cadence -(``WRITER_LEASE_HEARTBEAT_MONITOR_SECONDS``) and records how late every wake -was. A GIL-held C call delays the monitor for its whole duration; a Python -loop of bounded C calls lets the interpreter switch at its normal interval. -Each bounded phase is paired with the historical whole-window call it -replaced, and ``--daemon-binary`` adds the real Rust builder's prepare -round trip split into request write, response wait and response read. See -``LATENCY_PROBE_RATIONALE``. -""" - -from __future__ import annotations - -import argparse -import contextlib -import gc -import hashlib -import json -import os -import platform -import sys -import sysconfig -import tempfile -import threading -import time -from dataclasses import dataclass -from pathlib import Path -from typing import Any, Callable, Iterator, Sequence - -if not __package__: - sys.path.insert(0, str(Path(__file__).resolve().parents[2])) - -from lab.prism.bundle_compiler import ( - BundleCompiler, - _compact_share_payload, - _compact_share_tail_chunks, - _iter_prepare_window_request_chunks, - _ShareWindowSerialization, -) -from lab.prism.share_json_stream import ( - canonical_share_array_sha256, - iter_canonical_share_item_chunks, - iter_json_object_text_chunks, -) -from lab.prism.share_ledger import ( - DEFAULT_INCREMENTAL_SHARE_WINDOW_PAGE_SIZE, - AcceptedShareRecord, - DaemonShareJsonSequence, - DaemonShareWindowMirror, - IncrementalShareWindow, - _IncrementalShareWindowPage, - block_candidate_identity, - block_candidate_identity_sha256, - sha256_json_hex, -) -from lab.prism.writer_lease_timing import ( - WRITER_LEASE_HEARTBEAT_MONITOR_SECONDS, - WRITER_LEASE_HEARTBEAT_SCHEDULER_SLACK_SECONDS, -) - - -CLOCK_RATIONALE = ( - "cores-used = process CPU / wall. time.process_time_ns() is the numerator " - "because it sums CPU across every thread in the process, which is exactly " - "the quantity that separates 'N threads made progress together' from 'N " - "threads took turns under the GIL'. time.perf_counter_ns() is the " - "denominator. thread_time is deliberately NOT used here: it is per-thread " - "and cannot see the concurrency that is the whole question." -) - -AMORTIZATION_RATIONALE = ( - "Thread create/join is charged to wall clock but contributes little " - "process CPU, so a measurement whose wall is close to the thread-start " - "cost reads LOW on cores-used regardless of GIL behaviour. Measured on " - "this rig with the 1 MiB sha256 positive control: 5.33 cores at 0.26 s " - "wall, 6.43 at 0.25 s, 7.42 at 0.90 s, 7.42 at 3.71 s. Every " - "configuration is therefore grown until wall >= --min-wall-seconds. " - "Without this the bias is one-sided: it pushes every stage toward 1.00, " - "i.e. toward falsely confirming 'GIL-held'." -) - -# --------------------------------------------------------------------------- -# published anchors -# --------------------------------------------------------------------------- - -# #143 section 1: 1 MiB hashlib.sha256, 8 threads. -PUBLISHED_POSITIVE_CONTROL_CORES = 7.89 -# #143 section 1 bracketed CPython's GIL-release threshold here. -PUBLISHED_RELEASE_THRESHOLD_BYTES = (1024, 2048) - -# #131's profile table, per materialization, developer workstation. Keyed by -# window size; values are milliseconds attributed to each term. -PUBLISHED_PROFILE_MS: dict[int, dict[str, float]] = { - 21_868: { - "fold": 71.0, - "digest": 31.0, - "spool_acquire": 28.0, - "to_prism_json": 5.0, - }, - 100_000: { - "fold": 271.0, - "digest": 139.0, - "spool_acquire": 122.0, - "to_prism_json": 23.0, - }, - 400_000: { - "fold": 1_057.0, - "digest": 530.0, - "spool_acquire": 476.0, - "to_prism_json": 95.0, - }, -} - -# #131 annotates the spool row with a byte size at each window size. -PUBLISHED_SPOOL_ROW_BYTES: dict[int, int] = { - 21_868: 8_000_000, - 100_000: 37_000_000, - 400_000: 151_000_000, -} - -# job_build_benchmark.py defaults: --shares 21868, --miners 2. -BENCHMARK_SHARES = 21_868 -BENCHMARK_MINERS = 2 - -DEFAULT_SIZES = (21_868, 100_000, 400_000) -DEFAULT_THREADS = (1, 2, 4, 8) -# Above this window size an 8-thread sweep holds ~0.7 GB of independent input -# per thread; the contract for this measurement allows the largest size to be -# reported at the endpoints only. Overridable with --large-size-threads. -LARGE_SIZE_BYTES_THRESHOLD = 200_000 -LARGE_SIZE_THREADS = (1, 8) - -POSITIVE_CONTROL_BYTES = 1 << 20 -NEGATIVE_CONTROL_ITERATIONS = 20_000 - -# Fixed epoch so runs are reproducible; the pipeline only compares these. -FIXED_NOW_MS = 1_760_000_000_000 - - -# --------------------------------------------------------------------------- -# measurement harness -# --------------------------------------------------------------------------- - - -@dataclass(frozen=True) -class CoresResult: - """One (stage, thread-count) measurement.""" - - threads: int - cores_used: float - wall_seconds: float - process_cpu_seconds: float - iterations: int - cpu_ms_per_call: float - wall_ms_per_call: float - loadavg_1: float | None - grew: int - - def as_json(self) -> dict[str, Any]: - return { - "threads": self.threads, - "cores_used": round(self.cores_used, 4), - "wall_seconds": round(self.wall_seconds, 4), - "process_cpu_seconds": round(self.process_cpu_seconds, 4), - "iterations": self.iterations, - "cpu_ms_per_call": round(self.cpu_ms_per_call, 4), - "wall_ms_per_call": round(self.wall_ms_per_call, 4), - "loadavg_1": self.loadavg_1, - "grew": self.grew, - } - - -def _loadavg_1() -> float | None: - try: - return os.getloadavg()[0] - except (OSError, AttributeError): # pragma: no cover - platform dependent - return None - - -def measure_cores( - make_callable: Callable[[int], Callable[[], Any]], - threads: int, - *, - min_wall_seconds: float, - max_wall_seconds: float, - max_growths: int = 4, -) -> CoresResult: - """Cores-used for ``threads`` copies of a stage on independent inputs. - - ``make_callable(i)`` returns the zero-argument callable thread ``i`` will - drive; each index must own its own input so the only thing shared between - threads is the interpreter itself. - - The iteration count is calibrated from one untimed warm call and then - **grown until the wall clock clears ``min_wall_seconds``** -- see - ``AMORTIZATION_RATIONALE``; an under-length measurement is biased toward - 1.00 and would read as "GIL-held" whatever the truth is. ``max_wall_seconds`` - bounds the opposite end: a fully GIL-held stage serializes, so its wall grows - ~linearly in ``threads`` and would otherwise run for minutes at 400k shares. - """ - - callables = [make_callable(index) for index in range(threads)] - - # Warm one copy (imports, allocator arenas, branch predictors) and use it - # to size the batch. The warm call is not part of any reported number. - warm_start = time.perf_counter() - callables[0]() - per_call = max(time.perf_counter() - warm_start, 1e-9) - - iterations = max(1, int(min_wall_seconds / per_call)) - serialized_wall = per_call * iterations * threads - if serialized_wall > max_wall_seconds: - iterations = max(1, int(max_wall_seconds / (per_call * threads))) - - grew = 0 - while True: - gate = threading.Barrier(threads + 1) - - def worker(fn: Callable[[], Any], count: int) -> None: - gate.wait() - for _ in range(count): - fn() - - workers = [ - threading.Thread(target=worker, args=(fn, iterations), daemon=True) - for fn in callables - ] - for thread in workers: - thread.start() - # Threads exist and are parked on the barrier before the clocks start, - # so thread *creation* is outside the measured window. Start/join - # scheduling still is not, which is what min_wall_seconds covers. - gate.wait() - load = _loadavg_1() - cpu_start = time.process_time_ns() - wall_start = time.perf_counter_ns() - for thread in workers: - thread.join() - wall_ns = time.perf_counter_ns() - wall_start - cpu_ns = time.process_time_ns() - cpu_start - - wall_seconds = wall_ns / 1e9 - if wall_seconds >= min_wall_seconds or grew >= max_growths: - break - # Under-length: grow toward the floor and re-measure from scratch. - # Wall scales with the iteration count whether the stage is serialized - # or parallel, so the projection needs no thread-count factor. - scale = max(2, int(min_wall_seconds / max(wall_seconds, 1e-6)) + 1) - if wall_seconds * scale > max_wall_seconds: - break - iterations = max(iterations + 1, iterations * scale) - grew += 1 - - calls = iterations * threads - return CoresResult( - threads=threads, - cores_used=cpu_ns / wall_ns if wall_ns else float("nan"), - wall_seconds=wall_ns / 1e9, - process_cpu_seconds=cpu_ns / 1e9, - iterations=iterations, - cpu_ms_per_call=cpu_ns / 1e6 / calls, - wall_ms_per_call=wall_ns / 1e6 / iterations, - loadavg_1=load, - grew=grew, - ) - - -# --------------------------------------------------------------------------- -# controls -# --------------------------------------------------------------------------- - - -def positive_control(index: int) -> Callable[[], Any]: - """#143's control: 1 MiB hashlib.sha256, which must scale with threads. - - Each thread hashes a distinct buffer so nothing is shared but the - interpreter. - """ - # bytes([n]) -- a one-element list. bytes(n) would build n zero bytes and - # give every thread a differently sized buffer, which desynchronizes them - # and reads as a GIL effect it is not. - buffer = bytes([(index * 37 + 11) % 251]) * POSITIVE_CONTROL_BYTES - - def run() -> bytes: - return hashlib.sha256(buffer).digest() - - return run - - -def negative_control(index: int) -> Callable[[], Any]: - """Small-buffer pure-Python CPU, which must pin at ~1.00 cores.""" - seed = index + 1 - - def run() -> int: - total = 0 - for value in range(NEGATIVE_CONTROL_ITERATIONS): - total += value * seed - return total - - return run - - -# --------------------------------------------------------------------------- -# inputs -# --------------------------------------------------------------------------- - - -def benchmark_miner_programs(count: int) -> tuple[str, ...]: - """Same construction as ``lab/prism/job_build_benchmark.py``.""" - return tuple( - hashlib.sha256(f"bench-miner-{index}".encode()).hexdigest() - for index in range(count) - ) - - -def build_records( - count: int, - *, - miners: int = BENCHMARK_MINERS, - share_id_shape: str = "benchmark", - salt: int = 0, -) -> list[AcceptedShareRecord]: - """Records shaped like ``job_build_benchmark.BenchLedger``'s window. - - ``share_id_shape="production"`` swaps the benchmark's short synthetic - ``share_id`` for the ``username:block_hash_hex`` form that - ``lab/prism/share_writer.py`` actually builds (the ledger schema's - ``length(share_id) >= 65`` index predicate corroborates the length). Only - the byte accounting moves; the GIL verdicts do not. - """ - - programs = benchmark_miner_programs(miners) - records: list[AcceptedShareRecord] = [] - for index in range(count): - miner = index % miners - if share_id_shape == "production": - block_hash = hashlib.sha256( - f"{salt}:{index}".encode() - ).hexdigest() - share_id = f"bench-miner-{miner}.rig{index % 512}:{block_hash}" - else: - share_id = f"bench-share-{index + 1}" - records.append( - AcceptedShareRecord( - share_seq=index + 1, - share_id=share_id, - miner_id=f"bench-miner-{miner}", - order_key=f"bench-miner-{miner:06d}", - p2mr_program_hex=programs[miner], - share_difficulty=16384, - network_difficulty=226646186, - template_height=9, - job_id=f"bench-job-{index + 1}", - job_issued_at_ms=FIXED_NOW_MS - (count - index) * 1_000, - accepted_at_ms=FIXED_NOW_MS - (count - index) * 1_000, - ntime=FIXED_NOW_MS // 1000 - (count - index), - ) - ) - return records - - -@dataclass -class StageInputs: - """One thread's private copy of every input the stages need.""" - - records: list[AcceptedShareRecord] - anchor_job_issued_at_ms: int - window_weight: int - window: IncrementalShareWindow - page_records: tuple[tuple[AcceptedShareRecord, ...], ...] - json_sequence: Any - shares: list[dict[str, object]] - snapshot_sha256: str - identities_json: str - compact_shares_json: str - identities_bytes: bytes - compact_shares_bytes: bytes - canonical_json_bytes: int - spool_bytes: int - - -def build_inputs( - count: int, - *, - miners: int, - share_id_shape: str, - salt: int, -) -> StageInputs: - records = build_records( - count, miners=miners, share_id_shape=share_id_shape, salt=salt - ) - anchor = int(records[-1].job_issued_at_ms) - # Retain every record: the window-weight cutoff is not what is under test, - # and a partial retention would silently shrink the measured window. - weight = sum(int(record.share_difficulty) for record in records) - window = IncrementalShareWindow.from_full_snapshot( - records, - anchor_job_issued_at_ms=anchor, - window_weight=weight, - ) - sequence = window.json_records() - shares = list(sequence) - snapshot = sequence.canonical_json_sha256() - identities, compact_shares = _compact_share_payload(shares) - identities_json = json.dumps(identities, separators=(",", ":")) - compact_shares_json = json.dumps(compact_shares, separators=(",", ":")) - identities_bytes = identities_json.encode("utf-8") - compact_shares_bytes = compact_shares_json.encode("utf-8") - canonical_bytes = sum(len(page.canonical_json_items) for page in window.pages) - # Exactly the byte layout acquire_spooled_tail writes. - spool_bytes = ( - len(b',"compact_share_identities":') - + len(identities_bytes) - + len(b',"compact_shares":') - + len(compact_shares_bytes) - + len(b"}") - ) - return StageInputs( - records=records, - anchor_job_issued_at_ms=anchor, - window_weight=weight, - window=window, - page_records=tuple(page.records for page in window.pages), - json_sequence=sequence, - shares=shares, - snapshot_sha256=snapshot, - identities_json=identities_json, - compact_shares_json=compact_shares_json, - identities_bytes=identities_bytes, - compact_shares_bytes=compact_shares_bytes, - canonical_json_bytes=canonical_bytes, - spool_bytes=spool_bytes, - ) - - -# --------------------------------------------------------------------------- -# stages -# --------------------------------------------------------------------------- - - -@dataclass(frozen=True) -class Stage: - key: str - callable_name: str - profile_row: str | None - parent: str | None - note: str - factory: Callable[[StageInputs], Callable[[], Any]] - - -def _fold(inputs: StageInputs) -> Callable[[], Any]: - records = inputs.records - anchor = inputs.anchor_job_issued_at_ms - weight = inputs.window_weight - - def run() -> IncrementalShareWindow: - return IncrementalShareWindow.from_full_snapshot( - records, - anchor_job_issued_at_ms=anchor, - window_weight=weight, - ) - - return run - - -def _fold_pages(inputs: StageInputs) -> Callable[[], Any]: - page_records = inputs.page_records - - def run() -> tuple[Any, ...]: - return tuple( - _IncrementalShareWindowPage.from_records(records) - for records in page_records - ) - - return run - - -def _digest(inputs: StageInputs) -> Callable[[], Any]: - sequence = inputs.json_sequence - - def run() -> str: - return sequence.canonical_json_sha256() - - return run - - -def _to_prism_json(inputs: StageInputs) -> Callable[[], Any]: - records = inputs.records - - def run() -> list[dict[str, object]]: - return [record.to_prism_json() for record in records] - - return run - - -def _new_serialization(inputs: StageInputs) -> _ShareWindowSerialization: - return _ShareWindowSerialization( - key=(inputs.snapshot_sha256, len(inputs.records), inputs.window_weight), - share_count=len(inputs.records), - share_snapshot_sha256=inputs.snapshot_sha256, - ) - - -def _spool_acquire(inputs: StageInputs) -> Callable[[], Any]: - shares = inputs.shares - - def run() -> None: - # A fresh instance every call: acquire_spooled_tail memoizes, so a - # reused instance would measure a dict lookup after the first call. - serialization = _new_serialization(inputs) - serialization.acquire_spooled_tail(shares) - # Retire then release so the leased descriptor is closed by the - # shipped teardown path rather than left to the GC. - serialization.retire_spool() - serialization.release_spooled_tail() - - return run - - -def _spool_compact(inputs: StageInputs) -> Callable[[], Any]: - shares = inputs.shares - - def run() -> tuple[str, ...]: - return _new_serialization(inputs).compact_tail_chunks(shares) - - return run - - -def _spool_encode(inputs: StageInputs) -> Callable[[], Any]: - identities_json = inputs.identities_json - compact_shares_json = inputs.compact_shares_json - - def run() -> tuple[bytes, bytes]: - return ( - identities_json.encode("utf-8"), - compact_shares_json.encode("utf-8"), - ) - - return run - - -def _spool_write(inputs: StageInputs) -> Callable[[], Any]: - identities_bytes = inputs.identities_bytes - compact_shares_bytes = inputs.compact_shares_bytes - - def run() -> None: - spool = tempfile.TemporaryFile() - try: - spool.write(b',"compact_share_identities":') - spool.write(identities_bytes) - spool.write(b',"compact_shares":') - spool.write(compact_shares_bytes) - spool.write(b"}") - spool.flush() - spool.seek(0, os.SEEK_END) - finally: - spool.close() - - return run - - -STAGES: tuple[Stage, ...] = ( - Stage( - "fold", - "IncrementalShareWindow.from_full_snapshot", - "Window fold", - None, - "sorted() + per-record validation + paging; paging calls to_prism_json " - "and json.dumps per record", - _fold, - ), - Stage( - "fold_pages", - "_IncrementalShareWindowPage.from_records (every page)", - None, - "fold", - "the paging half of the fold: to_prism_json, json.dumps, b','.join", - _fold_pages, - ), - Stage( - "digest", - "IncrementalShareJsonSequence.canonical_json_sha256", - "Canonical-JSON digest", - None, - "one hashlib.sha256 fed pre-encoded per-page buffers", - _digest, - ), - Stage( - "to_prism_json", - "AcceptedShareRecord.to_prism_json (every record)", - "Record->JSON conversion", - None, - "measured standalone; note the fold already calls it internally", - _to_prism_json, - ), - Stage( - "spool_acquire", - "_ShareWindowSerialization.acquire_spooled_tail (cold)", - "Spool serialization", - None, - "compact_fragments + utf-8 encode + TemporaryFile writes", - _spool_acquire, - ), - Stage( - "spool_compact", - "_ShareWindowSerialization.compact_tail_chunks (cold)", - None, - "spool_acquire", - "identity dedup + batched json.dumps into bounded chunks (#236)", - _spool_compact, - ), - Stage( - "spool_encode", - 'str.encode("utf-8") of both fragments', - None, - "spool_acquire", - "str -> bytes for the two fragment strings", - _spool_encode, - ), - Stage( - "spool_write", - "TemporaryFile write/flush/seek of the spool payload", - None, - "spool_acquire", - "the os.write half of the spool term", - _spool_write, - ), -) - -STAGES_BY_KEY = {stage.key: stage for stage in STAGES} -PROFILE_ROW_STAGES = tuple(stage for stage in STAGES if stage.profile_row) - - -# --------------------------------------------------------------------------- -# environment -# --------------------------------------------------------------------------- - - -def _peak_rss_mb() -> float | None: - try: - import resource - - peak = resource.getrusage(resource.RUSAGE_SELF).ru_maxrss - except Exception: # pragma: no cover - platform dependent - return None - # Linux reports KiB, macOS bytes. - return peak / 1024.0 if sys.platform != "darwin" else peak / (1024.0 * 1024.0) - - -def _current_rss_mb() -> float | None: - """Resident set size *right now*. - - Deliberately not ru_maxrss: that is a lifetime high-water mark that never - falls, so after the smaller window sizes have run it would over-estimate - the live footprint by several hundred MB and skip the large size for a - shortage that does not exist. - """ - try: - with open("/proc/self/statm", encoding="ascii") as handle: - resident_pages = int(handle.read().split()[1]) - except (OSError, IndexError, ValueError): # pragma: no cover - platform - return None - return resident_pages * os.sysconf("SC_PAGE_SIZE") / (1024.0 * 1024.0) - - -def _available_memory_mb() -> float | None: - try: - with open("/proc/meminfo", encoding="ascii") as handle: - for line in handle: - if line.startswith("MemAvailable:"): - return int(line.split()[1]) / 1024.0 - except OSError: # pragma: no cover - platform dependent - return None - return None - - -def describe_environment() -> dict[str, Any]: - try: - load = os.getloadavg() - except (OSError, AttributeError): # pragma: no cover - platform dependent - load = None - try: - gil_enabled: bool | None = sys._is_gil_enabled() # type: ignore[attr-defined] - except AttributeError: - gil_enabled = None - physical = None - try: - with open("/proc/cpuinfo", encoding="ascii") as handle: - text = handle.read() - cores = { - line.split(":", 1)[1].strip() - for line in text.splitlines() - if line.startswith("cpu cores") - } - if cores: - physical = sorted(cores)[0] - except OSError: # pragma: no cover - platform dependent - pass - return { - "cpu": platform.processor() or platform.machine(), - "machine": platform.machine(), - "platform": platform.platform(), - "cpu_count": os.cpu_count(), - "cpu_cores_per_socket": physical, - "python_version": sys.version, - "python_implementation": platform.python_implementation(), - "gil_disabled_build": bool(sysconfig.get_config_var("Py_GIL_DISABLED")), - "gil_enabled_at_runtime": gil_enabled, - "loadavg_1_5_15": load, - "available_memory_mb": _available_memory_mb(), - "page_size_records": DEFAULT_INCREMENTAL_SHARE_WINDOW_PAGE_SIZE, - "clock_rationale": CLOCK_RATIONALE, - "amortization_rationale": AMORTIZATION_RATIONALE, - } - - -# --------------------------------------------------------------------------- -# analysis -# --------------------------------------------------------------------------- - - -def released_fraction(cores: float, ceiling: float) -> float | None: - """Fraction of a stage's CPU that ran with the GIL released. - - Normalized against the positive control measured **on this host at the - same thread count**, not against the nominal thread count: an 8-vCPU guest - cannot reach 8.00 even on a perfectly parallel workload, so dividing by 8 - would understate every stage. 1.00 cores maps to 0.0, the control's - ceiling maps to 1.0. Returns None when the ceiling offers no headroom. - """ - headroom = ceiling - 1.0 - if headroom <= 0.05: - return None - return max(0.0, min(1.0, (cores - 1.0) / headroom)) - - -def _fmt(value: float | None, digits: int = 2) -> str: - if value is None: - return "n/a" - if value != value: # NaN - return "n/a" - return f"{value:.{digits}f}" - - -def _fmt_mb(value: float | None) -> str: - return "n/a" if value is None else f"{value / 1e6:.2f}" - - -def _median(values: Sequence[float]) -> float: - ordered = sorted(values) - count = len(ordered) - if not count: - return float("nan") - mid = count // 2 - if count % 2: - return ordered[mid] - return (ordered[mid - 1] + ordered[mid]) / 2.0 - - -# --------------------------------------------------------------------------- -# runner -# --------------------------------------------------------------------------- - - -def thread_counts_for(size: int, threads: Sequence[int], large: Sequence[int]) -> tuple[int, ...]: - if size > LARGE_SIZE_BYTES_THRESHOLD: - return tuple(t for t in threads if t in set(large)) - return tuple(threads) - - -def run(args: argparse.Namespace) -> dict[str, Any]: - sizes = tuple(args.sizes) - threads = tuple(args.threads) - large_threads = tuple(args.large_size_threads) - reps = args.reps - - environment = describe_environment() - results: dict[str, Any] = { - "environment": environment, - "methodology": { - "metric": "cores-used = process CPU / wall", - "repetitions": reps, - "statistic": "median of N alternating repetitions", - "alternation": ( - "outer loop repetition, inner loop configuration, so host " - "drift lands on every configuration equally" - ), - "min_wall_seconds": args.min_wall_seconds, - "max_wall_seconds": args.max_wall_seconds, - "independent_inputs_per_thread": True, - "share_id_shape": args.share_id_shape, - "miners": args.miners, - "page_size": DEFAULT_INCREMENTAL_SHARE_WINDOW_PAGE_SIZE, - "published_positive_control_cores": PUBLISHED_POSITIVE_CONTROL_CORES, - }, - "sizes": [], - "controls": {}, - "peak_rss_mb": None, - "aborted": None, - } - - control_specs = ( - ("positive", positive_control, "1 MiB hashlib.sha256"), - ("negative", negative_control, f"{NEGATIVE_CONTROL_ITERATIONS}-iteration pure-Python loop"), - ) - - # ---- controls, alternating across thread counts and repetitions -------- - control_samples: dict[str, dict[int, list[CoresResult]]] = { - name: {count: [] for count in threads} for name, _, _ in control_specs - } - for _ in range(reps): - for count in threads: - for name, factory, _ in control_specs: - control_samples[name][count].append( - measure_cores( - factory, - count, - min_wall_seconds=args.min_wall_seconds, - max_wall_seconds=args.max_wall_seconds, - ) - ) - - ceilings: dict[int, float] = {} - for name, _, description in control_specs: - per_thread = [] - for count in threads: - samples = control_samples[name][count] - cores = _median([s.cores_used for s in samples]) - if name == "positive": - ceilings[count] = cores - per_thread.append( - { - "threads": count, - "cores_used_median": cores, - "cores_used_min": min(s.cores_used for s in samples), - "cores_used_max": max(s.cores_used for s in samples), - "wall_seconds_median": _median([s.wall_seconds for s in samples]), - "loadavg_1_median": _median( - [s.loadavg_1 for s in samples if s.loadavg_1 is not None] - ) - if any(s.loadavg_1 is not None for s in samples) - else None, - "samples": [s.as_json() for s in samples], - } - ) - results["controls"][name] = { - "description": description, - "by_threads": per_thread, - } - - positive_at_max = ceilings.get(max(threads)) - results["controls"]["positive_control_reproduces"] = ( - None - if positive_at_max is None - else positive_at_max / PUBLISHED_POSITIVE_CONTROL_CORES - ) - - # The contract for this measurement: if the positive control does not - # reproduce, the rig is wrong and the stage numbers must not be reported. - # The floor is a fraction of the *nominal* thread count rather than an - # absolute core count, so the gate stays meaningful when the sweep is run - # at fewer threads: a 4-thread sweep can never reach 6 cores, and an - # absolute floor would abort a perfectly healthy rig. #143 published - # 7.89/8 = 0.986 of nominal. - control_floor = args.control_floor_fraction * max(threads) - results["methodology"]["control_floor_cores"] = control_floor - results["methodology"]["control_floor_fraction"] = args.control_floor_fraction - if positive_at_max is not None and positive_at_max < control_floor: - results["aborted"] = ( - f"positive control reached {positive_at_max:.2f} cores at " - f"{max(threads)} threads, below the floor of " - f"{control_floor:.2f} (= {args.control_floor_fraction:g} x " - f"{max(threads)} threads; #143 published " - f"{PUBLISHED_POSITIVE_CONTROL_CORES} at 8). The rig is not " - f"measuring parallelism correctly; stage numbers are withheld." - ) - results["peak_rss_mb"] = _peak_rss_mb() - return results - - # ---- stages ----------------------------------------------------------- - for size in sizes: - size_threads = thread_counts_for(size, threads, large_threads) - if not size_threads: - continue - max_threads = max(size_threads) - - rss_before = _current_rss_mb() or 0.0 - probe = build_inputs( - size, - miners=args.miners, - share_id_shape=args.share_id_shape, - salt=0, - ) - rss_after = _current_rss_mb() or 0.0 - available = _available_memory_mb() - # Footprint of exactly one thread's inputs, measured rather than - # guessed, with 1.6x headroom for the transient window a running fold - # allocates before the previous one is collected. - per_input_mb = max(rss_after - rss_before, 1.0) - estimate_mb = per_input_mb * max_threads * 1.6 - if available is not None and estimate_mb > available * 0.8: - results["sizes"].append( - { - "size": size, - "skipped": ( - f"estimated {estimate_mb:.0f} MB for {max_threads} " - f"independent inputs ({per_input_mb:.0f} MB each) " - f"exceeds 80% of {available:.0f} MB available" - ), - } - ) - del probe - continue - - inputs = [probe] + [ - build_inputs( - size, - miners=args.miners, - share_id_shape=args.share_id_shape, - salt=index, - ) - for index in range(1, max_threads) - ] - - samples: dict[str, dict[int, list[CoresResult]]] = { - stage.key: {count: [] for count in size_threads} for stage in STAGES - } - for _ in range(reps): - for count in size_threads: - for stage in STAGES: - samples[stage.key][count].append( - measure_cores( - # noqa is for `inputs`: it is bound at the top of - # this loop body and freed by `del inputs` at the - # end of it, which is enough for Ruff to read the - # closure as unbound. measure_cores joins every - # thread before returning, so no invocation of this - # lambda outlives the binding. - lambda index, stage=stage: stage.factory(inputs[index]), # noqa: F821 - count, - min_wall_seconds=args.min_wall_seconds, - max_wall_seconds=args.max_wall_seconds, - ) - ) - - stage_rows = [] - for stage in STAGES: - by_threads = [] - for count in size_threads: - stage_samples = samples[stage.key][count] - cores = _median([s.cores_used for s in stage_samples]) - by_threads.append( - { - "threads": count, - "cores_used_median": cores, - "cores_used_min": min(s.cores_used for s in stage_samples), - "cores_used_max": max(s.cores_used for s in stage_samples), - "cpu_ms_per_call_median": _median( - [s.cpu_ms_per_call for s in stage_samples] - ), - "wall_ms_per_call_median": _median( - [s.wall_ms_per_call for s in stage_samples] - ), - "released_fraction": released_fraction( - cores, ceilings.get(count, float("nan")) - ) - if count > 1 - else None, - "loadavg_1_median": _median( - [s.loadavg_1 for s in stage_samples if s.loadavg_1 is not None] - ) - if any(s.loadavg_1 is not None for s in stage_samples) - else None, - "samples": [s.as_json() for s in stage_samples], - } - ) - stage_rows.append( - { - "key": stage.key, - "callable": stage.callable_name, - "profile_row": stage.profile_row, - "parent": stage.parent, - "note": stage.note, - "by_threads": by_threads, - } - ) - - results["sizes"].append( - { - "size": size, - "thread_counts": list(size_threads), - "input_mb_per_thread": round(per_input_mb, 1), - "bytes": { - "canonical_json_bytes": probe.canonical_json_bytes, - "spool_payload_bytes": probe.spool_bytes, - "canonical_bytes_per_record": probe.canonical_json_bytes / size, - "spool_bytes_per_record": probe.spool_bytes / size, - "page_count": len(probe.window.pages), - "canonical_bytes_per_page": ( - probe.canonical_json_bytes / max(len(probe.window.pages), 1) - ), - "published_spool_row_bytes": PUBLISHED_SPOOL_ROW_BYTES.get(size), - }, - "stages": stage_rows, - } - ) - - del inputs - del probe - gc.collect() - - results["peak_rss_mb"] = _peak_rss_mb() - try: - results["loadavg_end"] = list(os.getloadavg()) - except (OSError, AttributeError): # pragma: no cover - platform dependent - results["loadavg_end"] = None - return results - - -# --------------------------------------------------------------------------- -# reporting -# --------------------------------------------------------------------------- - - -def render_report(results: dict[str, Any]) -> str: - out: list[str] = [] - env = results["environment"] - method = results["methodology"] - - out.append("=" * 78) - out.append("GIL scaling of the PRISM payout-window materialization pipeline") - out.append("=" * 78) - out.append("") - out.append(f"CPU {env['cpu']} x{env['cpu_count']} logical") - out.append(f"Platform {env['platform']}") - out.append(f"Python {env['python_version'].splitlines()[0]}") - out.append( - f"GIL build Py_GIL_DISABLED={env['gil_disabled_build']}, " - f"runtime enabled={env['gil_enabled_at_runtime']}" - ) - load = env["loadavg_1_5_15"] - out.append( - "Load at start " - + ("n/a" if load is None else " / ".join(f"{v:.2f}" for v in load)) - ) - out.append(f"Page size {env['page_size_records']} records") - out.append( - f"Statistic median of {method['repetitions']} alternating repetitions; " - f"wall floor {method['min_wall_seconds']}s" - ) - out.append("") - out.append("Metric: " + method["metric"]) - out.append("") - - # ---- controls ---------------------------------------------------------- - out.append("-" * 78) - out.append("CONTROLS (measured in this process, same alternation as the stages)") - out.append("-" * 78) - out.append("") - header = f"{'control':<34}" + "".join( - f"{'N=' + str(row['threads']):>10}" - for row in results["controls"]["positive"]["by_threads"] - ) - out.append(header) - out.append("-" * len(header)) - for name in ("positive", "negative"): - control = results["controls"][name] - label = f"{name}: {control['description']}" - out.append( - f"{label[:33]:<34}" - + "".join( - f"{_fmt(row['cores_used_median']):>10}" - for row in control["by_threads"] - ) - ) - out.append("") - loads = [ - row["loadavg_1_median"] - for row in results["controls"]["positive"]["by_threads"] - if row["loadavg_1_median"] is not None - ] - if loads: - out.append(f"load average (1m) during controls: {min(loads):.2f}-{max(loads):.2f}") - reproduces = results["controls"].get("positive_control_reproduces") - top_threads = results["controls"]["positive"]["by_threads"][-1]["threads"] - if reproduces is not None and top_threads == 8: - out.append( - f"positive control vs #143's published {PUBLISHED_POSITIVE_CONTROL_CORES} " - f"cores at 8 threads: ratio {reproduces:.3f}" - ) - elif reproduces is not None: - # #143's anchor is an 8-thread number; comparing a shorter sweep's top - # thread count against it would read as a failure to reproduce when it - # is only a different thread count. - out.append( - f"positive control top sweep point is {top_threads} threads; " - f"#143's {PUBLISHED_POSITIVE_CONTROL_CORES}-core anchor is an " - f"8-thread figure and is not comparable here" - ) - out.append("") - - if results.get("aborted"): - out.append("!" * 78) - out.append("ABORTED: " + results["aborted"]) - out.append("!" * 78) - return "\n".join(out) - - # ---- per size ---------------------------------------------------------- - for entry in results["sizes"]: - size = entry["size"] - out.append("=" * 78) - out.append(f"WINDOW SIZE: {size:,} shares") - out.append("=" * 78) - if entry.get("skipped"): - out.append(f" SKIPPED: {entry['skipped']}") - out.append("") - continue - - counts = entry["thread_counts"] - byte_info = entry["bytes"] - out.append("") - out.append( - f" canonical JSON {_fmt_mb(byte_info['canonical_json_bytes'])} MB " - f"({byte_info['canonical_bytes_per_record']:.0f} B/record, " - f"{byte_info['page_count']} pages, " - f"{byte_info['canonical_bytes_per_page'] / 1024:.0f} KiB/page)" - ) - published_bytes = byte_info.get("published_spool_row_bytes") - out.append( - f" spool payload {_fmt_mb(byte_info['spool_payload_bytes'])} MB " - f"({byte_info['spool_bytes_per_record']:.0f} B/record)" - + ( - f" [#131 spool row says {published_bytes / 1e6:.0f} MB]" - if published_bytes - else "" - ) - ) - out.append("") - - head = f" {'stage':<30}" + "".join(f"{'N=' + str(c):>9}" for c in counts) - head += f"{'CPU ms':>10}{'released':>10}" - out.append(head) - out.append(" " + "-" * (len(head) - 2)) - for stage_row in entry["stages"]: - stage = STAGES_BY_KEY[stage_row["key"]] - label = (" " if stage.parent else "") + stage.key - cells = "" - for count in counts: - match = next( - r for r in stage_row["by_threads"] if r["threads"] == count - ) - cells += f"{_fmt(match['cores_used_median']):>9}" - single = next( - r for r in stage_row["by_threads"] if r["threads"] == counts[0] - ) - top = next( - r for r in stage_row["by_threads"] if r["threads"] == counts[-1] - ) - released = top["released_fraction"] - out.append( - f" {label:<30}{cells}" - f"{single['cpu_ms_per_call_median']:>10.1f}" - f"{(_fmt(released * 100, 0) + '%') if released is not None else 'n/a':>10}" - ) - out.append("") - loads = [ - r["loadavg_1_median"] - for s in entry["stages"] - for r in s["by_threads"] - if r["loadavg_1_median"] is not None - ] - if loads: - out.append(f" load average (1m) during this table: {min(loads):.2f}-{max(loads):.2f}") - out.append("") - - # ---- split of the published profile -------------------------------- - published = PUBLISHED_PROFILE_MS.get(size) - if published: - out.append(f" #131 profile split for {size:,} shares") - out.append( - f" {'profile row':<26}{'#131 ms':>9}{'released':>10}" - f"{'GIL-held':>10}{'parallel':>10}" - ) - out.append(" " + "-" * 65) - held_total = 0.0 - parallel_total = 0.0 - for stage in PROFILE_ROW_STAGES: - stage_row = next( - s for s in entry["stages"] if s["key"] == stage.key - ) - top = next( - r for r in stage_row["by_threads"] if r["threads"] == counts[-1] - ) - fraction = top["released_fraction"] or 0.0 - ms = published[stage.key] - parallel_ms = ms * fraction - held_ms = ms - parallel_ms - held_total += held_ms - parallel_total += parallel_ms - out.append( - f" {stage.profile_row:<26}{ms:>9.0f}" - f"{fraction * 100:>9.0f}%{held_ms:>10.0f}{parallel_ms:>10.0f}" - ) - total = held_total + parallel_total - out.append(" " + "-" * 65) - out.append( - f" {'TOTAL':<26}{total:>9.0f}{'':>10}" - f"{held_total:>10.0f}{parallel_total:>10.0f}" - ) - if total: - out.append( - f" => {parallel_total / total * 100:.0f}% of the profiled " - f"time is already running in parallel" - ) - out.append("") - - peak = results.get("peak_rss_mb") - if peak: - out.append(f"peak RSS: {peak:.0f} MB") - end = results.get("loadavg_end") - if end: - out.append("load average at end: " + " / ".join(f"{v:.2f}" for v in end)) - return "\n".join(out) - - -# --------------------------------------------------------------------------- -# #236 monitor-lateness probe -# --------------------------------------------------------------------------- - -LATENCY_PROBE_RATIONALE = ( - "The lease monitor thread must wake on its cadence while the payout " - "window is being serialized in another thread of the same process. A " - "single C call (json.dumps/json.loads over a whole 200k-share window) " - "holds the GIL until it returns, so the monitor's wake is late by the " - "call's duration; a Python loop of bounded C calls lets the interpreter " - "switch threads at its normal interval. The probe therefore measures, " - "for each phase, the maximum and p99 lateness of a thread that sleeps " - "on WRITER_LEASE_HEARTBEAT_MONITOR_SECONDS and records wake - due. This " - "is scheduling evidence on one host, not a proof of hard real-time " - "behaviour, and not a production measurement." -) - -PROBE_DEFAULT_SIZES = (210_000, 400_000) -# Distinct payout identities in the probe fixture: the compact payload's -# identity table and the fold's per-identity work depend on it. -PROBE_DEFAULT_MINERS = 200 -# The plan's stricter engineering target for monitor wake lateness, giving -# headroom against the configured scheduler slack. -PROBE_STRICT_LATENESS_SECONDS = 0.25 -PROBE_CONTROL_SECONDS = 1.0 -# Records appended by the advance step of the daemon round trip. -PROBE_ADVANCE_RECORDS = 16 - - -class MonitorLatenessProbe: - """A thread waking on the lease monitor's cadence, recording lateness. - - Each wake is scheduled ``interval`` after the previous wake (a late - wake does not owe a burst of catch-up wakes, exactly like a loop that - sleeps for its interval), and its lateness is ``wake - due``. Samples - are attributed to whichever phase the main thread has declared current; - wakes between phases are discarded. - """ - - def __init__(self, interval: float) -> None: - self._interval = float(interval) - self._lock = threading.Lock() - self._phase: str | None = None - self._samples: dict[str, list[float]] = {} - # Cyclic-GC pauses attributed to the current phase: (generation, - # seconds) per collection, from gc.callbacks. A full collection over - # a large live heap is itself one GIL-held stretch, and is the first - # suspect whenever a bounded phase still shows a late wake. - self._gc_pauses: dict[str, list[tuple[int, float]]] = {} - self._gc_started: float | None = None - self._stop = threading.Event() - self._thread = threading.Thread( - target=self._run, - name="lease-monitor-lateness-probe", - daemon=True, - ) - - def start(self) -> None: - gc.callbacks.append(self._gc_callback) - self._thread.start() - - def stop(self) -> None: - self._stop.set() - self._thread.join() - with contextlib.suppress(ValueError): - gc.callbacks.remove(self._gc_callback) - - def _gc_callback(self, event: str, info: dict[str, Any]) -> None: - # Runs on whichever thread triggered the collection, under the GIL. - if event == "start": - self._gc_started = time.perf_counter() - return - started = self._gc_started - self._gc_started = None - if started is None: - return - pause = time.perf_counter() - started - with self._lock: - phase = self._phase - if phase is not None: - self._gc_pauses[phase].append((int(info.get("generation", -1)), pause)) - - def _run(self) -> None: - due = time.monotonic() + self._interval - while not self._stop.is_set(): - delay = due - time.monotonic() - if delay > 0: - time.sleep(delay) - now = time.monotonic() - late = now - due - with self._lock: - phase = self._phase - if phase is not None: - self._samples[phase].append(late) - due = now + self._interval - - @contextlib.contextmanager - def phase(self, name: str) -> Iterator[None]: - with self._lock: - self._phase = name - self._samples[name] = [] - self._gc_pauses[name] = [] - try: - yield - finally: - with self._lock: - self._phase = None - - def summary(self, name: str) -> dict[str, Any]: - with self._lock: - samples = list(self._samples.get(name, ())) - pauses = list(self._gc_pauses.get(name, ())) - gc_summary = { - "gc_collections": len(pauses), - "gc_full_collections": sum(1 for gen, _ in pauses if gen >= 2), - "gc_max_pause_ms": max((pause for _, pause in pauses), default=0.0) * 1e3, - "gc_total_pause_ms": sum(pause for _, pause in pauses) * 1e3, - } - if not samples: - return { - "wakes": 0, - "max_late_ms": None, - "p99_late_ms": None, - "mean_late_ms": None, - "over_strict": 0, - "over_slack": 0, - **gc_summary, - } - ordered = sorted(samples) - p99_index = min(len(ordered) - 1, int(round(0.99 * (len(ordered) - 1)))) - return { - "wakes": len(samples), - "max_late_ms": ordered[-1] * 1e3, - "p99_late_ms": ordered[p99_index] * 1e3, - "mean_late_ms": sum(samples) / len(samples) * 1e3, - "over_strict": sum( - 1 for late in samples if late > PROBE_STRICT_LATENESS_SECONDS - ), - "over_slack": sum( - 1 - for late in samples - if late > WRITER_LEASE_HEARTBEAT_SCHEDULER_SLACK_SECONDS - ), - **gc_summary, - } - - -@dataclass(frozen=True) -class ProbePhaseResult: - key: str - kind: str - note: str - wall_seconds: float - cpu_seconds: float - bytes: int | None - lateness: dict[str, Any] - detail: dict[str, float] | None = None - - def as_json(self) -> dict[str, Any]: - return { - "key": self.key, - "kind": self.kind, - "note": self.note, - "wall_ms": round(self.wall_seconds * 1e3, 3), - "cpu_ms": round(self.cpu_seconds * 1e3, 3), - "bytes": self.bytes, - "lateness": { - key: (round(value, 3) if isinstance(value, float) else value) - for key, value in self.lateness.items() - }, - "detail_ms": ( - {key: round(value * 1e3, 3) for key, value in self.detail.items()} - if self.detail - else None - ), - } - - -class _ProbeRuntime: - """The slice of the coordinator runtime port prepare_payout_window uses.""" - - def __init__(self) -> None: - self.signing_seed_hex = "42" * 32 - self.ledger_attestation_signing_seed_hex = "43" * 32 - self.bundle_build_timeout_seconds = 600.0 - self._job_build_scheduler_lock = threading.Lock() - self.job_build_worker_counts = { - "starts": 0, - "restarts": 0, - "crashes": 0, - "terminations": 0, - } - self._job_build_worker_restart_pending = False - self._tip_refresh_metrics_lock = threading.Lock() - self.tip_refresh_worker_restarts = 0 - - def _ensure_job_cache_state(self) -> None: - return None - - -class _ProbeBuildControl: - """Placeholder build-control type; the probe never registers one.""" - - -def _count_bytes(chunks: Iterator[bytes | str]) -> int: - total = 0 - for chunk in chunks: - total += len(chunk) - return total - - -def _probe_candidate(shares: list[dict[str, object]]) -> dict[str, Any]: - """A durable block-candidate intent shaped like block_candidate_intent.""" - return { - "schema": "qbit.prism.block-candidate-intent.v1", - "block_hash_hex": "ab" * 32, - "block_hex": "00" * 256, - "coinbase_tx_hex": "01" * 128, - "parent_hash": "cd" * 32, - "expected_height": 800_001, - "template": { - "previousblockhash": "cd" * 32, - "height": 800_001, - "coinbasevalue": 50_00000000, - }, - "shares_json": shares, - "prior_balances": [], - "found_block": { - "block_height": 800_001, - "coinbase_value_sats": 50_00000000, - "network_difficulty": 226646186, - "anchor_job_issued_at_ms": FIXED_NOW_MS, - }, - "prospective_prior_balances": None, - "witness_merkle_leaves_hex": [], - "pending_share": {"share_id": "s", "accepted_at_ms": FIXED_NOW_MS}, - "username": "bench-miner-0", - } - - -def _instrument_transport( - compiler: BundleCompiler, - timers: dict[str, float], -) -> None: - """Time the compiler's transport helpers on this instance only.""" - for name in ( - "_serve_builder_write", - "_serve_builder_read_line", - "_serve_builder_read_exact", - ): - original = getattr(compiler, name) - - def wrapper( - *args: Any, - _original: Callable[..., Any] = original, - _name: str = name, - **kwargs: Any, - ) -> Any: - started = time.perf_counter() - try: - return _original(*args, **kwargs) - finally: - timers[_name] = timers.get(_name, 0.0) + ( - time.perf_counter() - started - ) - - setattr(compiler, name, wrapper) - - -def run_latency_probe(args: argparse.Namespace) -> dict[str, Any]: - daemon_binary = args.daemon_binary - if daemon_binary is not None: - daemon_path = Path(daemon_binary) - if not (daemon_path.is_file() and os.access(daemon_path, os.X_OK)): - raise SystemExit(f"--daemon-binary {daemon_binary} is not executable") - results: dict[str, Any] = { - "environment": describe_environment(), - "latency_probe": { - "monitor_interval_seconds": WRITER_LEASE_HEARTBEAT_MONITOR_SECONDS, - "scheduler_slack_seconds": WRITER_LEASE_HEARTBEAT_SCHEDULER_SLACK_SECONDS, - "strict_lateness_seconds": PROBE_STRICT_LATENESS_SECONDS, - "miners": args.probe_miners, - "share_id_shape": "production", - "daemon_binary": daemon_binary, - "rationale": LATENCY_PROBE_RATIONALE, - "sizes": [], - }, - } - probe = MonitorLatenessProbe(WRITER_LEASE_HEARTBEAT_MONITOR_SECONDS) - probe.start() - try: - for size in args.probe_sizes: - results["latency_probe"]["sizes"].append( - _probe_one_size( - size, - probe=probe, - miners=args.probe_miners, - daemon_binary=daemon_binary, - reps=args.probe_reps, - ) - ) - gc.collect() - finally: - probe.stop() - results["peak_rss_mb"] = _peak_rss_mb() - return results - - -def _probe_one_size( - size: int, - *, - probe: MonitorLatenessProbe, - miners: int, - daemon_binary: str | None, - reps: int, -) -> dict[str, Any]: - records = build_records(size, miners=miners, share_id_shape="production") - anchor = int(records[-1].job_issued_at_ms) - weight = sum(int(record.share_difficulty) for record in records) - state: dict[str, Any] = {} - phases: list[ProbePhaseResult] = [] - - def measure( - key: str, - kind: str, - note: str, - run: Callable[[], int | None], - *, - detail: dict[str, float] | None = None, - ) -> None: - best: ProbePhaseResult | None = None - for _ in range(max(1, reps)): - gc.collect() - with probe.phase(key): - cpu_started = time.process_time() - started = time.perf_counter() - byte_count = run() - wall = time.perf_counter() - started - cpu = time.process_time() - cpu_started - lateness = probe.summary(key) - result = ProbePhaseResult( - key=key, - kind=kind, - note=note, - wall_seconds=wall, - cpu_seconds=cpu, - bytes=byte_count, - lateness=lateness, - detail=dict(detail) if detail else None, - ) - # Keep the repetition with the worst lateness: the question is - # whether the monitor can be late, not how fast the best run was. - if best is None or ( - (result.lateness["max_late_ms"] or 0.0) - > (best.lateness["max_late_ms"] or 0.0) - ): - best = result - assert best is not None - phases.append(best) - - # ---- controls --------------------------------------------------------- - measure( - "idle_control", - "control", - "main thread sleeps; the monitor's own wake jitter on this host", - lambda: time.sleep(PROBE_CONTROL_SECONDS), - ) - - def busy_python() -> None: - deadline = time.perf_counter() + PROBE_CONTROL_SECONDS - total = 0 - while time.perf_counter() < deadline: - for value in range(10_000): - total += value - state["busy_total"] = total - - measure( - "busy_python_control", - "control", - "pure-Python loop; lateness is the interpreter's switch interval", - busy_python, - ) - - # ---- conversion and fold --------------------------------------------- - def convert() -> None: - state["shares"] = [record.to_prism_json() for record in records] - - measure( - "record_conversion", - "bounded", - "AcceptedShareRecord.to_prism_json per record (Python loop)", - convert, - ) - shares: list[dict[str, object]] = state["shares"] - - def fold() -> None: - state["window"] = IncrementalShareWindow.from_full_snapshot( - records, - anchor_job_issued_at_ms=anchor, - window_weight=weight, - ) - - measure( - "fold_in_process", - "bounded", - "IncrementalShareWindow.from_full_snapshot: the in-process fallback " - "fold, one bounded json.dumps per 512-record page", - fold, - ) - window: IncrementalShareWindow = state["window"] - sequence = window.json_records() - items = b",".join( - page.canonical_json_items for page in window.pages if page.canonical_json_items - ) - digest = sequence.canonical_json_sha256() - count = window.record_count - - def fold_again_and_release() -> None: - # A second fold whose result is dropped at once: the release of a - # whole window's pages (records, dicts, bytes) is what a full rescan - # or rotation pays when the previous window goes away. - second = IncrementalShareWindow.from_full_snapshot( - records, - anchor_job_issued_at_ms=anchor, - window_weight=weight, - ) - state["second_window"] = second - - measure( - "fold_in_process_keep", - "bounded", - "from_full_snapshot with the result retained (no release inside)", - fold_again_and_release, - ) - - def release_second_window() -> None: - del state["second_window"] - - measure( - "fold_release", - "residual", - "del of a whole in-process window (pages, records, dicts, bytes)", - release_second_window, - ) - - # ---- canonical encoding and digest ----------------------------------- - measure( - "canonical_encode_stream", - "bounded", - "iter_canonical_share_item_chunks over the share dicts", - lambda: _count_bytes(iter_canonical_share_item_chunks(shares)), - ) - measure( - "canonical_encode_whole", - "historical", - "json.dumps(shares, sort_keys=True, ...) in one call", - lambda: len( - json.dumps( - shares, sort_keys=True, separators=(",", ":"), default=str - ).encode("utf-8") - ), - ) - measure( - "array_digest_stream", - "bounded", - "canonical_share_array_sha256 over a plain list (legacy digest path)", - lambda: (canonical_share_array_sha256(shares), None)[1], - ) - measure( - "array_digest_whole", - "historical", - "sha256(json.dumps(shares, sort_keys=True, ...)) in one call", - lambda: ( - hashlib.sha256( - json.dumps( - shares, sort_keys=True, separators=(",", ":"), default=str - ).encode("utf-8") - ).hexdigest(), - None, - )[1], - ) - - # ---- compact payload and spool --------------------------------------- - measure( - "compact_tail_stream", - "bounded", - "_compact_share_tail_chunks: identities + compact rows in batches", - lambda: _count_bytes(iter(_compact_share_tail_chunks(shares))), - ) - - def compact_whole() -> int: - identities, compact_shares = _compact_share_payload(shares) - return len(json.dumps(identities, separators=(",", ":"))) + len( - json.dumps(compact_shares, separators=(",", ":")) - ) - - measure( - "compact_tail_whole", - "historical", - "_compact_share_payload + two whole json.dumps (old compact_fragments)", - compact_whole, - ) - - def spool_cold() -> int: - serialization = _ShareWindowSerialization( - key=(digest, count, weight), - share_count=count, - share_snapshot_sha256=digest, - ) - lease = serialization.acquire_spooled_tail(shares) - try: - return int(lease[1]) if lease is not None else None - finally: - serialization.retire_spool() - if lease is not None: - serialization.release_spooled_tail() - - measure( - "spool_write_stream", - "bounded", - "_ShareWindowSerialization.acquire_spooled_tail cold (encode + write)", - spool_cold, - ) - - # ---- daemon prepare request ------------------------------------------ - request_fields: dict[str, object] = { - "request": "prepare_window", - "mode": "full", - "append_invalidation_epoch": 0, - "anchor_job_issued_at_ms": anchor, - "records": shares, - "window_weight": weight, - "page_size": DEFAULT_INCREMENTAL_SHARE_WINDOW_PAGE_SIZE, - } - measure( - "prepare_request_stream", - "bounded", - "_iter_prepare_window_request_chunks (envelope + record batches)", - lambda: _count_bytes(_iter_prepare_window_request_chunks(request_fields)), - ) - measure( - "prepare_request_whole", - "historical", - "json.dumps(request_fields) in one call (old prepare_payout_window)", - lambda: len( - json.dumps(request_fields, separators=(",", ":")).encode("utf-8") - ), - ) - - # ---- mirror validation and lazy parse -------------------------------- - measure( - "mirror_validate", - "bounded", - "DaemonShareWindowMirror.from_full_items: digest + record walk", - lambda: ( - DaemonShareWindowMirror.from_full_items( - anchor_job_issued_at_ms=anchor, - window_weight=weight, - page_size=DEFAULT_INCREMENTAL_SHARE_WINDOW_PAGE_SIZE, - record_count=count, - canonical_items=items, - share_snapshot_sha256=digest, - ), - len(items), - )[1], - ) - def parse_stream() -> int: - state["parsed_stream"] = DaemonShareJsonSequence(items, count)._records() - return len(items) - - measure( - "mirror_parse_stream", - "bounded", - "DaemonShareJsonSequence._records: raw_decode per record, chunked decode", - parse_stream, - ) - - def release_parsed_stream() -> None: - # Dropping the parsed tuple frees every dict and string it holds in - # one refcount cascade -- a single C call, measured on its own. - del state["parsed_stream"] - - measure( - "mirror_parse_release", - "residual", - "del of the parsed record tuple (one deallocation cascade)", - release_parsed_stream, - ) - - def parse_whole() -> int: - state["parsed_whole"] = json.loads(b"[" + items + b"]") - return len(items) - - measure( - "mirror_parse_whole", - "historical", - 'json.loads(b"[" + items + b"]") in one call (old _records)', - parse_whole, - ) - - def release_parsed_whole() -> None: - del state["parsed_whole"] - - measure( - "mirror_parse_whole_release", - "residual", - "del of json.loads' record list (one deallocation cascade)", - release_parsed_whole, - ) - - # ---- durable candidate identity -------------------------------------- - candidate = _probe_candidate(shares) - measure( - "candidate_identity_stream", - "bounded", - "block_candidate_identity_sha256 (shares_json streamed)", - lambda: (block_candidate_identity_sha256(candidate), None)[1], - ) - measure( - "candidate_identity_whole", - "historical", - "sha256_json_hex(block_candidate_identity(candidate)) in one call", - lambda: (sha256_json_hex(block_candidate_identity(candidate)), None)[1], - ) - - # ---- one-shot canonical build input ---------------------------------- - payload = { - "found_block": candidate["found_block"], - "prior_balances": [], - "payout_policy": {"policy": "day-one"}, - "coinbase_script_sig_suffix_hex": "00", - "witness_merkle_leaves_hex": [], - "shares": shares, - } - measure( - "oneshot_payload_stream", - "bounded", - "iter_json_object_text_chunks(payload, array_keys=('shares',))", - lambda: _count_bytes( - iter_json_object_text_chunks(payload, array_keys=("shares",)) - ), - ) - measure( - "oneshot_payload_whole", - "historical", - "json.dumps(payload) in one call (the canonical build's share array)", - lambda: len(json.dumps(payload, separators=(",", ":"))), - ) - - # ---- real daemon round trip ------------------------------------------ - daemon: dict[str, Any] | None = None - if daemon_binary is not None: - daemon = _probe_daemon( - probe=probe, - measure=measure, - records=records, - shares=shares, - anchor=anchor, - weight=weight, - miners=miners, - daemon_binary=daemon_binary, - ) - - return { - "size": size, - "record_count": count, - "bytes": { - "canonical_items": len(items), - "compact_tail": sum( - len(chunk) for chunk in _compact_share_tail_chunks(shares) - ), - }, - "phases": [phase.as_json() for phase in phases], - "daemon": daemon, - } - - -def _probe_daemon( - *, - probe: MonitorLatenessProbe, - measure: Callable[..., None], - records: list[AcceptedShareRecord], - shares: list[dict[str, object]], - anchor: int, - weight: int, - miners: int, - daemon_binary: str, -) -> dict[str, Any]: - """The real --serve builder's prepare_window round trip, attributed.""" - runtime = _ProbeRuntime() - compiler = BundleCompiler( - runtime, # type: ignore[arg-type] - superseded_error=RuntimeError, - cancellation_error_types=(), - build_control_type=_ProbeBuildControl, - tool_command=lambda _name: [daemon_binary], - ) - timers: dict[str, float] = {} - _instrument_transport(compiler, timers) - outcomes: dict[str, Any] = {} - try: - # Spawn and handshake outside the measured phases, with a - # one-record window that does not collide with the real one. - warm = compiler.prepare_payout_window( - mode="full", - records_json=[records[0].to_prism_json()], - anchor_job_issued_at_ms=anchor, - append_invalidation_epoch=0, - window_weight=int(records[0].share_difficulty), - page_size=DEFAULT_INCREMENTAL_SHARE_WINDOW_PAGE_SIZE, - ) - outcomes["warm_status"] = getattr(warm, "status", None) - timers.clear() - - def full() -> int: - outcome = compiler.prepare_payout_window( - mode="full", - records_json=shares, - anchor_job_issued_at_ms=anchor, - append_invalidation_epoch=0, - window_weight=weight, - page_size=DEFAULT_INCREMENTAL_SHARE_WINDOW_PAGE_SIZE, - ) - outcomes["full"] = outcome - return len(outcome.window_items or b"") if outcome is not None else None - - measure( - "daemon_prepare_full", - "daemon", - "BundleCompiler.prepare_payout_window(mode='full'): streamed " - "request, Rust fold, raw response", - full, - detail=timers, - ) - full_detail = dict(timers) - timers.clear() - full_outcome = outcomes.get("full") - status = getattr(full_outcome, "status", None) - mirror_bytes = None - if status == "prepared": - measure( - "daemon_mirror_validate", - "bounded", - "DaemonShareWindowMirror.from_full_items on the daemon's bytes", - lambda: len( - DaemonShareWindowMirror.from_full_items( - anchor_job_issued_at_ms=anchor, - window_weight=weight, - page_size=DEFAULT_INCREMENTAL_SHARE_WINDOW_PAGE_SIZE, - record_count=full_outcome.record_count, - canonical_items=full_outcome.window_items or b"", - share_snapshot_sha256=full_outcome.share_snapshot_sha256 or "", - ).canonical_items - ), - ) - mirror_bytes = len(full_outcome.window_items or b"") - programs = benchmark_miner_programs(miners) - last = records[-1] - delta = [] - for index in range(PROBE_ADVANCE_RECORDS): - seq = int(last.share_seq) + index + 1 - miner = seq % miners - delta.append( - AcceptedShareRecord( - share_seq=seq, - share_id=f"bench-miner-{miner}.rig{seq % 512}:{'ab' * 32}", - miner_id=f"bench-miner-{miner}", - order_key=f"bench-miner-{miner:06d}", - p2mr_program_hex=programs[miner], - share_difficulty=int(last.share_difficulty), - network_difficulty=int(last.network_difficulty), - template_height=int(last.template_height), - job_id=f"bench-job-{seq}", - job_issued_at_ms=int(last.job_issued_at_ms) + index + 1, - accepted_at_ms=int(last.accepted_at_ms) + index + 1, - ntime=int(last.ntime) + index + 1, - ).to_prism_json() - ) - new_anchor = int(last.job_issued_at_ms) + PROBE_ADVANCE_RECORDS + 1 - - def advance() -> int: - outcome = compiler.prepare_payout_window( - mode="advance", - records_json=delta, - anchor_job_issued_at_ms=new_anchor, - append_invalidation_epoch=0, - base_digest=full_outcome.share_snapshot_sha256, - ) - outcomes["advance"] = outcome - return len(outcome.appended_items) if outcome is not None else None - - measure( - "daemon_prepare_advance", - "daemon", - f"prepare_payout_window(mode='advance') with " - f"{PROBE_ADVANCE_RECORDS} appended records", - advance, - detail=timers, - ) - finally: - compiler.shutdown_serve_builder() - return { - "binary": daemon_binary, - "warm_status": outcomes.get("warm_status"), - "full_status": getattr(outcomes.get("full"), "status", None), - "full_error": getattr(outcomes.get("full"), "error", None), - "full_transport_ms": { - key: round(value * 1e3, 3) for key, value in full_detail.items() - }, - "advance_status": getattr(outcomes.get("advance"), "status", None), - "advance_stats": ( - { - "added_rows": outcomes["advance"].added_rows, - "expired_rows": outcomes["advance"].expired_rows, - "touched_pages": outcomes["advance"].touched_pages, - } - if getattr(outcomes.get("advance"), "status", None) == "prepared" - else None - ), - "mirror_bytes": mirror_bytes, - "worker_counts": dict(runtime.job_build_worker_counts), - "serve_counts": dict(compiler.serve_builder_counts), - } - - -def render_latency_probe(results: dict[str, Any]) -> str: - out: list[str] = [] - env = results["environment"] - probe = results["latency_probe"] - out.append("=" * 78) - out.append("#236 monitor-lateness probe: PRISM payout-window serialization phases") - out.append("=" * 78) - out.append("") - out.append(f"CPU {env['cpu']} x{env['cpu_count']} logical") - out.append(f"Platform {env['platform']}") - out.append(f"Python {env['python_version'].splitlines()[0]}") - load = env["loadavg_1_5_15"] - out.append( - "Load at start " - + ("n/a" if load is None else " / ".join(f"{v:.2f}" for v in load)) - ) - out.append( - f"Monitor cadence {probe['monitor_interval_seconds'] * 1e3:.0f} ms " - f"(WRITER_LEASE_HEARTBEAT_MONITOR_SECONDS); scheduler slack " - f"{probe['scheduler_slack_seconds'] * 1e3:.0f} ms; strict target " - f"{probe['strict_lateness_seconds'] * 1e3:.0f} ms" - ) - out.append( - f"Fixture production share_id shape, {probe['miners']} identities" - ) - out.append( - "Daemon " - + (probe["daemon_binary"] or "not measured (pass --daemon-binary)") - ) - out.append("") - out.append("Metric: per phase, wall time of the phase in the main thread and the") - out.append("lateness (wake - due) of a monitor thread sleeping on the lease") - out.append("monitor's cadence. 'historical' rows are the whole-window calls the") - out.append("'bounded' rows replaced; both run the same input in the same process.") - out.append("'gc full' counts generation-2 cyclic collections during the phase and") - out.append("'gc max' is the longest single collection pause (ms), from gc.callbacks.") - out.append("Lateness is what the monitor thread observed on this host, whatever the") - out.append("cause (a GIL-held C call, a GC pause, or other load on a shared host).") - out.append("") - for entry in probe["sizes"]: - out.append("=" * 78) - out.append( - f"WINDOW SIZE: {entry['size']:,} shares " - f"({entry['record_count']:,} retained)" - ) - out.append("=" * 78) - out.append( - f" canonical items {entry['bytes']['canonical_items'] / 1e6:.1f} MB, " - f"compact tail {entry['bytes']['compact_tail'] / 1e6:.1f} MB" - ) - out.append("") - head = ( - f" {'phase':<28}{'kind':<11}{'wall ms':>9}{'cpu ms':>9}{'MB':>8}" - f"{'wakes':>7}{'max late':>10}{'p99 late':>10}{'>250ms':>8}{'>500ms':>8}" - f"{'gc full':>8}{'gc max':>8}" - ) - out.append(head) - out.append(" " + "-" * (len(head) - 2)) - for phase in entry["phases"]: - late = phase["lateness"] - out.append( - f" {phase['key']:<28}{phase['kind']:<11}" - f"{phase['wall_ms']:>9.1f}{phase['cpu_ms']:>9.1f}" - f"{(phase['bytes'] or 0) / 1e6 if phase['bytes'] else 0:>8.1f}" - f"{late['wakes']:>7}" - f"{_fmt(late['max_late_ms'], 1):>10}" - f"{_fmt(late['p99_late_ms'], 1):>10}" - f"{late['over_strict']:>8}{late['over_slack']:>8}" - f"{late.get('gc_full_collections', 0):>8}" - f"{_fmt(late.get('gc_max_pause_ms'), 1):>8}" - ) - if phase.get("detail_ms"): - detail = ", ".join( - f"{key.removeprefix('_serve_builder_')}={value:.1f} ms" - for key, value in phase["detail_ms"].items() - ) - out.append(f" {'':<28}transport: {detail}") - out.append("") - daemon = entry.get("daemon") - if daemon: - out.append( - f" daemon: full={daemon['full_status']} advance={daemon['advance_status']}" - f" mirror_bytes={daemon['mirror_bytes']}" - f" advance_stats={daemon['advance_stats']}" - f" worker_counts={daemon['worker_counts']}" - ) - if daemon.get("full_error"): - out.append(f" daemon error: {daemon['full_error']}") - out.append("") - peak = results.get("peak_rss_mb") - if peak: - out.append(f"peak RSS: {peak:.0f} MB") - return "\n".join(out) - - -# --------------------------------------------------------------------------- -# CLI -# --------------------------------------------------------------------------- - - -def _int_list(text: str) -> list[int]: - return [int(part) for part in text.replace(",", " ").split()] - - -def main(argv: list[str] | None = None) -> int: - parser = argparse.ArgumentParser( - description=__doc__, - formatter_class=argparse.RawDescriptionHelpFormatter, - ) - parser.add_argument("--json", action="store_true", help="emit results as JSON") - parser.add_argument( - "--render", - metavar="PATH", - default=None, - help=( - "render the tables from a previously saved --json file instead of " - "measuring. Running the sweep twice to get both output formats " - "would put two 8-thread processes on the same cores and corrupt " - "both, so capture --json once and render from it." - ), - ) - parser.add_argument( - "--sizes", - type=_int_list, - default=list(DEFAULT_SIZES), - help="window sizes in shares (default: 21868,100000,400000)", - ) - parser.add_argument( - "--threads", - type=_int_list, - default=list(DEFAULT_THREADS), - help="thread counts to sweep (default: 1,2,4,8)", - ) - parser.add_argument( - "--large-size-threads", - type=_int_list, - default=list(LARGE_SIZE_THREADS), - help=( - f"thread counts used for sizes above {LARGE_SIZE_BYTES_THRESHOLD:,} " - f"shares, where 8 independent inputs cost ~0.7 GB each " - f"(default: 1,8)" - ), - ) - parser.add_argument("--reps", type=int, default=3, help="repetitions (default 3)") - parser.add_argument( - "--min-wall-seconds", - type=float, - default=1.0, - help=( - "wall-clock floor per configuration; below this, thread start/join " - "biases cores-used downward (default 1.0)" - ), - ) - parser.add_argument( - "--max-wall-seconds", - type=float, - default=12.0, - help=( - "bound on a single configuration's wall clock; a fully GIL-held " - "stage serializes, so its wall grows with thread count (default 12)" - ), - ) - parser.add_argument( - "--control-floor-fraction", - type=float, - default=0.75, - help=( - "the positive control must reach this fraction of the highest " - "thread count in cores-used, else the run aborts without " - "reporting stages (default 0.75, i.e. 6.0 cores on an 8-thread " - "sweep; #143 published 0.986)" - ), - ) - parser.add_argument( - "--miners", - type=int, - default=BENCHMARK_MINERS, - help=f"distinct payout identities (default {BENCHMARK_MINERS})", - ) - parser.add_argument( - "--share-id-shape", - choices=("benchmark", "production"), - default="benchmark", - help=( - "benchmark: job_build_benchmark's short synthetic share_id; " - "production: the username:block_hash_hex form share_writer builds" - ), - ) - parser.add_argument( - "--latency-probe", - action="store_true", - help=( - "#236 mode: run each serialization phase once per size in the " - "main thread while a monitor thread on the lease monitor's " - "cadence records its wake lateness; see LATENCY_PROBE_RATIONALE" - ), - ) - parser.add_argument( - "--probe-sizes", - type=_int_list, - default=list(PROBE_DEFAULT_SIZES), - help="--latency-probe window sizes in shares (default: 210000,400000)", - ) - parser.add_argument( - "--probe-reps", - type=int, - default=1, - help=( - "--latency-probe repetitions per phase; the repetition with the " - "worst monitor lateness is reported (default 1)" - ), - ) - parser.add_argument( - "--probe-miners", - type=int, - default=PROBE_DEFAULT_MINERS, - help=f"--latency-probe distinct payout identities (default {PROBE_DEFAULT_MINERS})", - ) - parser.add_argument( - "--daemon-binary", - default=None, - help=( - "--latency-probe: path to a prebuilt qbit-prism-build-audit-bundle; " - "adds the real --serve daemon's prepare round trip" - ), - ) - args = parser.parse_args(argv) - - if args.render: - with open(args.render, encoding="utf-8") as handle: - loaded = json.load(handle) - sys.stdout.write( - render_latency_probe(loaded) - if "latency_probe" in loaded - else render_report(loaded) - ) - sys.stdout.write("\n") - return 0 - - if args.latency_probe: - results = run_latency_probe(args) - if args.json: - json.dump(results, sys.stdout, indent=2, default=str) - else: - sys.stdout.write(render_latency_probe(results)) - sys.stdout.write("\n") - return 0 - - results = run(args) - if args.json: - json.dump(results, sys.stdout, indent=2, default=str) - sys.stdout.write("\n") - else: - sys.stdout.write(render_report(results)) - sys.stdout.write("\n") - return 0 - - -if __name__ == "__main__": - raise SystemExit(main()) diff --git a/tests/prism_candidate_storm.py b/tests/prism_candidate_storm.py deleted file mode 100644 index 2de773d60..000000000 --- a/tests/prism_candidate_storm.py +++ /dev/null @@ -1,2508 +0,0 @@ -#!/usr/bin/env python3 -"""Candidate-storm characterization instrument for PRISM. - -The 2026-08-20 testnet4 incident produced 3,120 durable block candidates, -well above both the 32-entry live queue and the 1,024-row replay-enumeration -page. This instrument reproduces that cardinality against the shipped -in-memory ledger and coordinator paths, then reports the component counts on -both sides of a process restart. - -The ownership counts are deliberately first-class output. Live admission -marks a candidate outstanding *before* attempting the bounded queue, so the -marker covers coalesced wakeups as well as queued ones. Restart adoption -marks every restored row replay-inflight before it is processed. Those -markers mean "this process still owns eventual disposition", not "qbitd has -already been offered this candidate". Any maintenance selector that treats -either whole set as active node-offer evidence will exclude the whole storm. -A wakeup parked in the retry holder is the same kind of marker for the same -reason: ``_retain_block_candidate_for_retry`` also runs when the fast-lane -reservation declines, before any submitblock. So is a claimed disposition -lease: ``submit_next`` takes it on the dequeued hash *before* the terminal, -capacity and fast-lane checks, each of which releases it again having -offered nothing. So, less obviously, is a recorded terminal outcome: when -pool capacity is already closed the deferred split builds a node submission -with ``attempted=False`` and lets accounting terminalize the durable row -without calling qbitd once, and the durable ``attempt_count`` is marked on -that same zero-RPC path. The rig therefore reports the attested offers -(``node_offer_evidence``) apart from the conservative must-not-abandon union -(``selector_evidence``) that a selector owes the #183 contract, so neither a -retry holder, nor a held lease, nor a terminal seal is ever mistaken for a -node call. - -This is a component instrument, not a performance threshold. It uses the -real coordinator admission, durable codec, pagination, payout barriers, and -``SingleWriterShareLedger``; it does not include PostgreSQL wait or qbitd. -Issues #187 and #185 can extend the same rig with wall-clock restart/drain -assertions and component-memory gauges without rebuilding the storm fixture. - -The decided-height extension models the instant after the storm height is -settled: ``decide_height`` makes one candidate the active block at that -height (``DecidedHeightRpc``), ``ownership`` reads every candidate's -ownership and node-offer-evidence facts from the shipped state so a -maintenance selector can be evaluated against the real population, -``perturb`` installs one selector-evidence shape at a time through the -shipped entry points, and ``drain_per_row`` drives the shipped per-row -disposition path in the production split -- ``submit_next`` with -``defer_accounting=True`` handing every node offer to the real accounting -queue and task runner -- reporting the exact hash sets it terminalized and -what they cost. The per-row drain is the oracle any set-oriented selector -has to agree with; the instrument does not contain a selector itself, and -suppresses the shipped #183 collapse for the duration of the drain so the -oracle stays the per-row path even where the drain re-enumerates. - -Driving both lanes from one thread is what makes the drain deterministic, -and it costs nothing in fidelity: the two lanes are already serialized per -hash by the disposition lease, and the drain stamps the accounting-thread -identity the shipped loop stamps, so retry pacing records a not-before -deadline exactly as it does on the real accounting thread. Splitting the -drive this way also matters for correctness, not just fidelity: -``submit_next`` claims the disposition lease *blocking* unless accounting -is deferred, and ``_restore_replayed_candidate_acceptance_evidence`` runs -only inside the accounting task. - -The cleanup-backlog extension (issue #198) measures the one collapse state -with no durable replay source: the in-memory retry record a won row leaves -behind when its post-write cleanup fails. ``CleanupFaultInjector`` breaks -exactly one cleanup dependency (``CLEANUP_FAULTS``) at the shipped seam -while counting every seam per hash, and ``measure_cleanup_backlog`` drives -the shipped collapse walk under that fault, reports the backlog it produces --- depth, oldest age, retained pending-share holders, terminal-outcome pins, -deep and traced bytes per record, and the rows the admission bound -preserved -- holds the fault for a bounded run of failing retry passes, -heals it, and then proves recovery: the backlog drains through the shipped -retry pass at a measured throughput, admission resumes, every collapsed -hash runs each cleanup seam exactly once more (no double cleanup), no -terminal row is replay-adopted or re-offered to the node, and the decided -winner keeps its pending-share floor authority. -""" - -from __future__ import annotations - -import argparse -import gc -from collections.abc import Mapping -import json -import linecache -import os -import platform -import queue -import random -import subprocess -import sys -import threading -import time -import tracemalloc -from contextlib import ExitStack, nullcontext, redirect_stderr, redirect_stdout -from dataclasses import asdict, dataclass -from io import StringIO -from pathlib import Path -from types import SimpleNamespace -from typing import Any, ContextManager, Iterable, Sequence - -if not __package__: - sys.path.insert(0, str(Path(__file__).resolve().parents[1])) - -from lab.prism.block_candidates import ( # noqa: E402 - BLOCK_CANDIDATE_COLLAPSE_CLEANUP_PAYOUT_STEP, - BLOCK_CANDIDATE_COLLAPSE_CLEANUP_STEPS, - COLLAPSE_DIFFICULTY_SCALE, - COLLAPSE_POW_LIMIT_BITS, - MAX_BLOCK_REPLAY_ENUMERATION_ROWS, - MAX_PENDING_BLOCK_CANDIDATES, - _BlockCandidateNodeSubmission, -) -from lab.prism.coordinator_config import ( # noqa: E402 - DEFAULT_BLOCK_CANDIDATE_CLEANUP_RETRY_BACKLOG_MAX, -) -from lab.prism.process_telemetry import ( # noqa: E402 - MallocTrimmer, - ProcessHeapTelemetry, - read_anonymous_map_shape, - read_malloc_arena_count, - read_malloc_info_summary, - read_resident_memory_bytes, -) -from lab.prism.share_ledger import ( # noqa: E402 - PendingShare, - SingleWriterShareLedger, -) -from lab.prism.template_artifacts import scaled_network_difficulty # noqa: E402 -from tests.prism_vardiff_test_support import ( # noqa: E402 - FakeRpc, - block_candidate, - submit_coordinator, -) - - -OBSERVED_TESTNET_CANDIDATE_STORM = 3_120 -STORM_PARENT_HASH = "11" * 32 - -# The selector-evidence shapes record_abandoned and submit_next consult, -# each installable on one seeded candidate through CandidateStormRig.perturb. -PERTURBATIONS = ( - "lease_held", - "retained_success", - "tip_observed", - "retry_slot", - "prepared_pool_block", - "terminal", -) - -# Issue #198: every in-memory dependency one won row's terminal cleanup runs -# through, each injectable as a fault through CleanupFaultInjector. The -# first seven are the shipped cleanup steps, named exactly as the retry -# registry names what a hash still owes; ``floor-index`` is the page-scope -# pending-share holder scan that precedes them, whose failure aborts the -# whole apply's cleanup and is the path the retry re-indexes for. -CLEANUP_FAULTS = tuple(BLOCK_CANDIDATE_COLLAPSE_CLEANUP_STEPS) + ("floor-index",) -# Install the seam counters without breaking anything: the clean baseline -# every marginal per-record figure is taken against. -CLEANUP_FAULT_NONE = "none" -# seam -> (owner, attribute, successful calls one collapsed hash is owed). -# The payout withdrawal is two calls (invalidate, then drop the tombstone); -# the floor release is once per retained holder and the holder index once -# per page, so neither has a per-hash expectation. -_CLEANUP_SEAMS: dict[str, tuple[str, str, int | None]] = { - BLOCK_CANDIDATE_COLLAPSE_CLEANUP_PAYOUT_STEP: ( - "server", - "_clear_accepted_block_payout_preview", - 2, - ), - "finalize-retry": ("service", "finalize_retries", 1), - "retry-state": ("server", "_clear_block_candidate_retry_state", 1), - "outstanding-and-tip-observation": ( - "server", - "_discard_outstanding_block_candidate", - 1, - ), - "terminal-outcome": ("server", "_record_block_candidate_terminal_outcome", 1), - "abandonment-accounting": ( - "server", - "_record_committed_block_candidate_abandonment", - 1, - ), - "pending-share-floor": ("server", "_finish_pending_share_commit", None), - "floor-index": ("service", "_collapsed_candidate_floor_holders", None), -} -assert set(_CLEANUP_SEAMS) == set(CLEANUP_FAULTS) - - -@dataclass(frozen=True) -class CandidateStormSnapshot: - """Bounded component gauges after one phase of the storm.""" - - durable_pending: int - live_queue_capacity: int - live_queued: int - live_coalesced: int - outstanding_marked: int - outstanding_covers_all_durable: bool - replay_queued: int - replay_inflight_marked: int - replay_inflight_covers_all_durable: bool - accepted_parent_previews: int - replay_enumeration_owed: bool - # Candidates the #183 must-not-abandon contract covers, i.e. carrying - # any evidence that a node offer happened, is in flight, or cannot be - # ruled out (see CandidateOwnership.selector_evidence). The - # outstanding and replay-inflight markers above are deliberately NOT - # part of it. - selector_evidence_marked: int = 0 - # How that union splits: rows something actually attests a node offer - # for, and the three widenings that attest nothing -- a wakeup parked in - # a retry holder, a claimed disposition lease, and a recorded terminal - # outcome. None is by itself evidence qbitd was ever called; production - # reaches all three without a submitblock. They can coincide on one row, - # so they are counted separately rather than folded into one residue. - node_offer_evidence_marked: int = 0 - unoffered_retry_marked: int = 0 - unoffered_lease_marked: int = 0 - unoffered_terminal_marked: int = 0 - - -@dataclass(frozen=True) -class CandidateOwnership: - """Ownership and evidence facts for one durable candidate at one instant. - - Every field is read from the shipped coordinator/ledger state, so a - maintenance selector can be evaluated against the real population - instead of a hand-built corpus. ``outstanding`` and ``replay_inflight`` - are process-ownership markers; the fields folded into - ``node_offer_evidence`` are the ones that mean a node offer happened, - is happening, or left acceptance evidence behind. ``retry_held``, - ``disposition_held`` and ``terminal_outcome`` are none of those: they - are a third category, covered by the conservative ``selector_evidence`` - union without attesting an offer. - """ - - block_hash: str - expected_height: int - parent_hash: str - network_difficulty: float | None - outbox_state: str | None - attempt_count: int - live_queued: bool - replay_queued: bool - outstanding: bool - replay_inflight: bool - preview_barrier: bool - disposition_held: bool - retry_held: bool - node_acceptance_retained: bool - tip_observed: bool - accounted_accepted: bool - terminal_outcome: bool | None - pool_block_chain_state: str | None - - @property - def node_offer_evidence(self) -> bool: - """Whether anything attests that qbitd was offered this hash. - - Every fact here is written by the offer or downstream of it: a - retained submission records an offer that already returned, a tip - observation and an accounted acceptance are readings of an accepted - offer, and a pool-block row is what an accepted offer's payout - preparation left behind. ``retry_held``, ``disposition_held`` and - ``terminal_outcome`` are deliberately absent; see - :attr:`retry_held_without_offer`, :attr:`lease_held_without_offer` - and :attr:`terminal_without_offer`. - """ - return bool( - self.node_acceptance_retained - or self.tip_observed - or self.accounted_accepted - or self.pool_block_chain_state is not None - ) - - @property - def retry_held_without_offer(self) -> bool: - """A wakeup parked in a retry holder with no offer attested for it. - - ``_retain_block_candidate_for_retry`` also runs on paths that never - reached qbitd -- most plainly when ``_reserve_block_fast_lane_slot`` - declines and ``submit_next`` parks the candidate *before* any - submitblock. Retry retention on its own therefore says only "this - process still owes this row a disposition", exactly like - ``outstanding`` and ``replay_inflight``, and reading it as an offer - would credit the rig with a node call that never happened. - """ - return bool(self.retry_held) and not self.node_offer_evidence - - @property - def lease_held_without_offer(self) -> bool: - """A claimed disposition lease with no offer attested for it. - - ``submit_next`` claims the lease on the dequeued hash *before* it - consults the terminal outcome, before the accounted/capacity close, - and before ``_reserve_block_fast_lane_slot``. Each of those paths - releases the lease again without ever reaching submitblock, and the - deferred split hands the still-held lease to accounting even when the - node submission is ``attempted=False``. The registry this field is - read from is weaker still: ``_claim_block_candidate_disposition`` - registers a user on the flight *before* it takes the flight's lock, - so the key is present for a blocked waiter as well as for the holder. - A held lease therefore says only "some pass owns this row's eventual - disposition", exactly like ``outstanding`` and ``replay_inflight``, - and reading it as an offer would credit the rig with a node call that - never happened. - """ - return bool(self.disposition_held) and not self.node_offer_evidence - - @property - def terminal_without_offer(self) -> bool: - """A recorded terminal outcome with no offer attested for it. - - ``_record_block_candidate_terminal_outcome`` means "this process - finished deciding this row", not "this process offered it". The - deferred submitter split reaches it having called nothing: when pool - capacity is already closed -- ``accepted_block_count`` has reached - ``max_blocks``, or ``stop_after_block`` is set and one block landed - -- ``submit_next`` builds a ``_BlockCandidateNodeSubmission( - attempted=False)`` and hands it straight to accounting, which - terminalizes the durable outbox row without a single ``submitblock`` - (the already-accounted and finalize-only replay paths do the same). - - ``attempt_count`` is no substitute for the missing RPC either: - ``submit_writer`` calls ``_mark_block_candidate_attempted`` *before* - the offer branch, and the ledger documents the mark as "admission to - a real processing phase", so the durable row records an attempt on - exactly the same zero-RPC path. - - Excluding the terminal registry from :attr:`node_offer_evidence` - costs that gauge no attestation it legitimately had: finalization - adds an accepted hash to ``_accounted_accepted_block_hashes`` and - persists its pool-block row before the terminal outcome is recorded, - so a candidate that really was offered and accepted still attests it - in its own right. - """ - return self.terminal_outcome is not None and not self.node_offer_evidence - - @property - def selector_evidence(self) -> bool: - """The conservative must-not-abandon union the #183 contract needs. - - This is :attr:`node_offer_evidence` widened by ``retry_held``, by - ``disposition_held``, and by a recorded ``terminal_outcome``. Each - widening covers an ambiguity rather than an attestation: a parked - wakeup may equally have come from a path that *did* offer and is - retrying its tail, a claimed lease may equally span an offer already - in flight, and a terminal outcome may equally be the seal on an offer - that returned. Nothing the selector can read tells the two apart, so - it has to leave the row to the per-row path either way -- and a row - whose lease it cannot claim, or whose disposition this process has - already fixed, is not one it could dispose of anyway. Because the - widening is deliberately conservative, a true value here is not a - claim that qbitd was offered the hash -- read - :attr:`node_offer_evidence` for that, and - :attr:`retry_held_without_offer` / :attr:`lease_held_without_offer` / - :attr:`terminal_without_offer` for the rows the widenings add. - """ - return ( - self.node_offer_evidence - or bool(self.retry_held) - or bool(self.disposition_held) - or self.terminal_outcome is not None - ) - - -@dataclass(frozen=True) -class PerRowDrainReport: - """What the shipped per-row disposition path did, and cost, over a storm. - - Every field reports *one* invocation of :meth:`CandidateStormRig.drain_per_row`. - ``abandoned_hashes`` is the oracle a set-oriented selector has to agree - with: the exact set of durable rows that call terminalized as abandoned, - not a count, and not every abandoned row standing in the rig. It and - ``submitted_hashes`` are transitions -- a row already terminal when the - call started is left out -- so successive bounded drains (``max_rounds``) - partition the work between them instead of each claiming the whole - history. ``pending_hashes`` is the complementary residue: the rows still - awaiting disposition when the call returned, which is a state rather than - a transition because nothing ever transitions *into* pending. - - ``deferred_hashes`` and ``lease_blocked_hashes`` are the oracle's - counterweight -- rows the drain reached and the shipped path deliberately - refused to terminalize -- so a selector that claims one of them can be - caught rather than silently believed. - - ``rounds`` is the number of node offers the call actually performed -- - the ``submitblock`` delta in the rig's own RPC log -- not the number of - wakeups it consumed. ``submit_next_block_candidate`` returns True on - several paths that never reach qbitd: a disposition lease held elsewhere - parks the wakeup in ``_block_disposition_waiting_retries``, a same-hash - duplicate whose disposition already landed is dropped, a declined - fast-lane reservation retains the candidate for retry, and the - capacity-closed split hands accounting an ``attempted=False`` submission - that terminalizes the durable row without calling qbitd once. Counting - those would make ``rounds`` disagree with this report's own ``rpc_calls``, - ``accounting_tasks`` and ``ledger_attempt_marks``, and would let a - ``max_rounds`` budget be spent without a single offer being made. - """ - - rounds: int - accounting_tasks: int - replay_enumerations: int - abandoned_hashes: frozenset[str] - submitted_hashes: frozenset[str] - pending_hashes: frozenset[str] - withheld_hashes: frozenset[str] - lease_blocked_hashes: frozenset[str] - deferred_hashes: frozenset[str] - rpc_calls: dict[str, int] - ledger_attempt_marks: int - wall_seconds: float - - @property - def abandoned_rows(self) -> int: - return len(self.abandoned_hashes) - - @property - def submitted_rows(self) -> int: - return len(self.submitted_hashes) - - @property - def pending_rows(self) -> int: - return len(self.pending_hashes) - - -class _FaultableRegistry(dict): - """``finalize_retries`` with a breakable ``pop``. - - The finalize-retry cleanup step reaches no coordinator seam -- it pops - the service's own registry under the lock -- so the fault has to live - on the container. Membership reads stay plain dict reads. - """ - - def __init__(self, injector: "CleanupFaultInjector", seam: str, contents: Any) -> None: - super().__init__(contents) - self._injector = injector - self._seam = seam - - def pop(self, key: Any, *default: Any) -> Any: # type: ignore[override] - self._injector._guard(self._seam) - result = super().pop(key, *default) - self._injector._count(self._seam, str(key).lower()) - return result - - -class CleanupFaultInjector: - """Break one cleanup dependency until healed; count every seam meanwhile. - - Every seam the terminal cleanup runs through is wrapped so a *successful* - call is counted per hash (per holder for the floor release, per page for - the holder index), and exactly one of them -- ``fault`` -- raises while - ``on`` is true. Nothing else is replaced: the collapse, the fence, the - retry registry and the retry pass all stay as shipped, so the backlog - this produces is the backlog production would produce under the same - fault. ``CLEANUP_FAULT_NONE`` installs the counters alone, which is the - clean baseline the marginal memory figure is taken against. - - The counters are what make recovery provable rather than believed: after - healing, every collapsed hash must show exactly the calls one terminal - cleanup is owed at every seam -- one more than that is a double cleanup, - one fewer is a step the retry never finished -- and a floor holder must - be released at most once by identity. - """ - - def __init__(self, server: Any, fault: str) -> None: - if fault != CLEANUP_FAULT_NONE and fault not in CLEANUP_FAULTS: - raise ValueError(f"unknown cleanup fault: {fault}") - self.server = server - self.service = server._ensure_block_candidate_service() - self.fault = fault - self.on = fault != CLEANUP_FAULT_NONE - self.failed_calls = 0 - # seam -> key -> successful calls. - self.calls: dict[str, dict[str, int]] = {seam: {} for seam in _CLEANUP_SEAMS} - # Floor releases by holder identity. - self.floor_releases: dict[int, int] = {} - self._originals: dict[str, tuple[Any, str, Any]] = {} - - def install(self) -> None: - if self._originals: - raise RuntimeError("cleanup faults are already installed") - for seam, (owner_name, attribute, _expected) in _CLEANUP_SEAMS.items(): - owner = self.server if owner_name == "server" else self.service - original = getattr(owner, attribute) - self._originals[seam] = (owner, attribute, original) - if seam == "finalize-retry": - setattr(owner, attribute, _FaultableRegistry(self, seam, original)) - continue - setattr(owner, attribute, self._wrap(seam, original)) - - def uninstall(self) -> None: - for seam, (owner, attribute, original) in self._originals.items(): - if seam == "finalize-retry": - # Hand the live contents back to the shipped container. - current = getattr(owner, attribute) - original.clear() - original.update(current) - setattr(owner, attribute, original) - self._originals.clear() - - def heal(self) -> None: - self.on = False - - def _wrap(self, seam: str, original: Any) -> Any: - def wrapped(*args: Any, **kwargs: Any) -> Any: - self._guard(seam) - result = original(*args, **kwargs) - self._count(seam, self._key(seam, args, kwargs)) - return result - - return wrapped - - def _guard(self, seam: str) -> None: - if self.on and seam == self.fault: - self.failed_calls += 1 - raise RuntimeError(f"injected cleanup fault: {seam}") - - def _key(self, seam: str, args: tuple[Any, ...], kwargs: dict[str, Any]) -> str: - if seam == "pending-share-floor": - share = args[0] if args else kwargs["pending_share"] - self.floor_releases[id(share)] = self.floor_releases.get(id(share), 0) + 1 - # The rig stamps ``share_id`` as ``miner:<hash>``. - return str(getattr(share, "share_id", "")).rsplit(":", 1)[-1].lower() - if seam == "floor-index": - return "page" - value = args[0] if args else kwargs.get("block_hash", "") - return str(value).lower() - - def _count(self, seam: str, key: str) -> None: - counts = self.calls[seam] - counts[key] = counts.get(key, 0) + 1 - - def snapshot_calls(self) -> dict[str, dict[str, int]]: - return {seam: dict(counts) for seam, counts in self.calls.items()} - - def _expected(self, seam: str, expected: int) -> int: - # An aborted apply -- the holder index raising -- has already run the - # payout withdrawal for the whole page before it stops, and the - # retry then re-runs every step because the abort proved nothing - # about which ones completed. That documented repeat of an - # idempotent step is the shipped abort contract, not a double - # cleanup, so the expectation carries it. - if self.fault == "floor-index" and seam == BLOCK_CANDIDATE_COLLAPSE_CLEANUP_PAYOUT_STEP: - return 2 * expected - return expected - - def excess_calls(self, hashes: Iterable[str]) -> dict[str, int]: - """Per seam, how many of ``hashes`` were cleaned up more than once.""" - wanted = {str(value).lower() for value in hashes} - return { - seam: sum( - 1 - for key in wanted - if self.calls[seam].get(key, 0) > self._expected(seam, expected) - ) - for seam, (_owner, _attribute, expected) in _CLEANUP_SEAMS.items() - if expected is not None - } - - def calls_since( - self, - earlier: dict[str, dict[str, int]], - hashes: Iterable[str], - ) -> int: - """Successful seam calls for ``hashes`` made after ``earlier`` was taken.""" - wanted = {str(value).lower() for value in hashes} - return sum( - counts.get(key, 0) - earlier.get(seam, {}).get(key, 0) - for seam, counts in self.calls.items() - for key in wanted - ) - - def missing_calls(self, hashes: Iterable[str]) -> dict[str, int]: - """Per seam, how many of ``hashes`` still lack their one cleanup.""" - wanted = {str(value).lower() for value in hashes} - return { - seam: sum(1 for key in wanted if self.calls[seam].get(key, 0) < expected) - for seam, (_owner, _attribute, expected) in _CLEANUP_SEAMS.items() - if expected is not None - } - - @property - def floor_double_releases(self) -> int: - return sum(1 for count in self.floor_releases.values() if count > 1) - - -def _deep_size(value: Any, seen: set[int]) -> int: - """Bytes reachable from ``value`` through containers and instance dicts. - - Each object is charged once per ``seen`` set, so a string or holder - shared by two records is counted for the first and free for the second - -- the same accounting the process pays. - """ - if id(value) in seen: - return 0 - seen.add(id(value)) - size = sys.getsizeof(value) - if isinstance(value, dict): - for key, item in value.items(): - size += _deep_size(key, seen) + _deep_size(item, seen) - elif isinstance(value, (tuple, list, set, frozenset)): - for item in value: - size += _deep_size(item, seen) - elif hasattr(value, "__dict__"): - size += _deep_size(vars(value), seen) - slots = getattr(type(value), "__slots__", ()) - if isinstance(slots, str): - slots = (slots,) - for name in slots: - if hasattr(value, name): - size += _deep_size(getattr(value, name), seen) - return size - - -@dataclass(frozen=True) -class CleanupBacklogReport: - """One faulted collapse walk, its backlog, and the recovery that drained it. - - Every figure is from one :meth:`CandidateStormRig.measure_cleanup_backlog` - call. The memory figures are two independent readings of the same - state: ``registry_bytes`` walks the shipped registry with - ``sys.getsizeof`` (records, their step sets, hash keys, and the exact - holder objects they retain, each object charged once), while - ``traced_walk_bytes`` is ``tracemalloc``'s change in live traced bytes - across the faulted walk -- which also carries everything else the walk - allocated (fences, counters, the adopted remainder), so the per-record - marginal figure is taken against a ``CLEANUP_FAULT_NONE`` walk of the - same storm by :func:`cleanup_backlog_marginal_bytes_per_record`. - - The recovery block is the contract this measurement exists to prove: - ``excess_cleanup_calls`` and ``floor_double_releases`` must be zero (no - double cleanup), ``missing_cleanup_calls`` must be zero (every owed - step ran), ``terminal_rows_adopted`` must be zero (no replay adoption of - a terminal row), ``node_offers`` and ``drain_rounds`` must be zero (no - re-offer), and ``winner_floor_holder_retained`` must hold when the rig - seeded credit shares (no loss of payout-floor authority). - """ - - fault: str - view: str - candidates: int - credit_shares: bool - backlog_max: int - # -- the faulted walk -------------------------------------------------- - walk_seconds: float - collapsed_rows: int - deferred_records: int - preserved_rows: int - backpressure_engagements: int - backpressure_active: bool - pending_share_holders: int - terminal_outcome_pins: int - oldest_age_seconds: float - registry_bytes: int - holder_bytes: int - registry_bytes_per_record: float - holder_bytes_per_holder: float - traced_walk_bytes: int - traced_owner_bytes: int - traced_peak_bytes: int - # -- sustained failure ------------------------------------------------- - sustained_passes: int - sustained_failed_passes: int - depth_after_sustained: int - holders_after_sustained: int - pins_after_sustained: int - # -- recovery ---------------------------------------------------------- - recovery_seconds: float - recovery_passes: int - recovered_records: int - retry_records_per_second: float - depth_after_recovery: int - second_walk_seconds: float - collapsed_rows_after_recovery: int - excess_cleanup_calls: dict[str, int] - missing_cleanup_calls: dict[str, int] - post_recovery_cleanup_calls: int - floor_double_releases: int - terminal_rows_adopted: int - node_offers: int - drain_rounds: int - pending_rows_after_drain: int - floor_holders_before: int - floor_holders_after: int - winner_floor_holder_retained: bool - - @property - def recovered_cleanly(self) -> bool: - return ( - self.depth_after_recovery == 0 - and not any(self.excess_cleanup_calls.values()) - and not any(self.missing_cleanup_calls.values()) - and self.post_recovery_cleanup_calls == 0 - and self.floor_double_releases == 0 - and self.terminal_rows_adopted == 0 - and self.node_offers == 0 - and self.drain_rounds == 0 - ) - - -def cleanup_backlog_marginal_bytes_per_record( - faulted: CleanupBacklogReport, - clean: CleanupBacklogReport, - *, - owner_only: bool = False, -) -> float: - """Traced bytes one retry record cost over a clean walk of the same storm. - - ``owner_only`` restricts both readings to allocations made by the - block-candidate owner module, which is where the record, its step set - and its registry entry are allocated. - """ - if faulted.deferred_records <= 0: - return 0.0 - if owner_only: - return (faulted.traced_owner_bytes - clean.traced_owner_bytes) / faulted.deferred_records - return (faulted.traced_walk_bytes - clean.traced_walk_bytes) / faulted.deferred_records - - -def _warm_traceback_caches() -> None: - """Load the source lines a faulted cleanup's tracebacks will print. - - ``traceback.print_exc`` fills ``linecache`` with the whole source of - every frame's file the first time it runs, hundreds of kilobytes that - would otherwise land inside the first faulted walk's traced delta. - """ - import lab.prism.block_candidates as block_candidates - - for path in (block_candidates.__file__, __file__): - linecache.getlines(path) - - -def _owner_traced_bytes(snapshot: Any) -> int: - """Live traced bytes allocated by the block-candidate owner module.""" - import lab.prism.block_candidates as block_candidates - - filtered = snapshot.filter_traces( - (tracemalloc.Filter(True, block_candidates.__file__),) - ) - return sum(statistic.size for statistic in filtered.statistics("filename")) - - -class DecidedHeightRpc(FakeRpc): - """A chain whose best tip is one storm candidate at the storm height. - - Models the instant after a height is decided: ``getbestblockhash`` and - ``getblockhash(height)`` name the winner, the winner's header is active - with one confirmation, every other candidate header is unknown, and a - further ``submitblock`` answers ``inconclusive`` (a valid block that did - not become the tip). Calls are counted per method so a drain's RPC cost - is measurable; the counter is thread-safe because the shipped fast lane - runs ``submitblock`` on a bounded worker thread. - """ - - def __init__( - self, - *, - winner_hash: str, - height: int, - bits: str = COLLAPSE_POW_LIMIT_BITS, - ) -> None: - self.winner_hash = winner_hash.lower() - self.height = int(height) - # The decided block's compact bits as qbitd would report them in - # getblockheader. Work is stated in bits, not in the raw difficulty - # float, because a candidate row carries scaled difficulty - # (raw * COLLAPSE_DIFFICULTY_SCALE) and only a bits-derived value is - # in the same units. The default is the qbit powLimit, at or below - # every sibling in the storm, so the decided block never reads as - # the weaker chain. - self.bits = str(bits).lower() - self.difficulty = ( - scaled_network_difficulty(self.bits) / COLLAPSE_DIFFICULTY_SCALE - ) - self.calls: dict[str, int] = {} - self._lock = threading.Lock() - - def _count(self, method: str) -> None: - with self._lock: - self.calls[method] = self.calls.get(method, 0) + 1 - - def call(self, method: str, params: list[object] | None = None) -> object: - self._count(method) - if method == "getbestblockhash": - return self.winner_hash - if method == "getblockcount": - return self.height - if method == "getblockhash": - height = int((params or [0])[0]) - if height == self.height: - return self.winner_hash - raise RuntimeError(f"qbit RPC getblockhash failed: -8 unknown height {height}") - if method == "getblockheader": - block_hash = str((params or [""])[0]).lower() - if block_hash == self.winner_hash: - return { - "hash": self.winner_hash, - "height": self.height, - "confirmations": 1, - "bits": self.bits, - "difficulty": self.difficulty, - } - raise RuntimeError("qbit RPC getblockheader failed: -5 Block not found") - if method == "submitblock": - return "inconclusive" - return super().call(method, params) - - -class CandidateStormRig: - """Seed one same-parent storm and observe live admission plus restart.""" - - def __init__( - self, - *, - candidates: int = OBSERVED_TESTNET_CANDIDATE_STORM, - queue_depth: int = MAX_PENDING_BLOCK_CANDIDATES, - quiet: bool = True, - credit_shares: bool = False, - backlog_max: int | None = None, - ) -> None: - if candidates <= 0: - raise ValueError("candidates must be positive") - if queue_depth <= 0: - raise ValueError("queue_depth must be positive") - if backlog_max is not None and backlog_max <= 0: - raise ValueError("backlog_max must be positive") - self.candidates = candidates - self.queue_depth = queue_depth - self.quiet = quiet - # Issue #198. ``credit_shares`` seeds every candidate as credit-bearing - # and registers its pending share on the writer's floor, so the - # collapse cleanup has real floor authority to release and the - # decided winner has real authority to keep; the restart view adopts - # holders through the shipped decode exactly as production does. - # ``backlog_max`` pins the cleanup-retry admission bound on every - # coordinator the rig owns (the shipped default otherwise). - self.credit_shares = bool(credit_shares) - self.backlog_max = backlog_max - self.ledger = SingleWriterShareLedger() - self.live_server: Any | None = None - self.restarted_server: Any | None = None - self.block_hashes: list[str] = [] - self.decided_rpc: DecidedHeightRpc | None = None - - def _coordinator(self) -> tuple[Any, Any]: - server, state, _recording = submit_coordinator(tip=STORM_PARENT_HASH) - server.max_blocks = self.candidates + 1 - server.stop_after_block = False - server.block_candidate_queue = queue.Queue(maxsize=self.queue_depth) - server.ledger = self.ledger - # A shipped tunable, set to zero so a parked or retained retry neither - # sleeps on the submitter lane nor parks behind a not-before deadline. - # This is a component oracle, not a pacing test: every wakeup must be - # re-offerable in the same round it was parked. - server.block_candidate_retry_initial_seconds = 0.0 - if self.backlog_max is not None: - server.block_candidate_cleanup_retry_backlog_max = int(self.backlog_max) - if self.decided_rpc is not None: - server.rpc = self.decided_rpc - return server, state - - def _silence(self) -> ContextManager[Any]: - if not self.quiet: - return nullcontext() - return redirect_stdout(StringIO()) - - def _silence_all(self) -> ContextManager[Any]: - """Silence both streams: a faulted cleanup prints tracebacks to stderr.""" - if not self.quiet: - return nullcontext() - stack = ExitStack() - stack.enter_context(redirect_stdout(StringIO())) - stack.enter_context(redirect_stderr(StringIO())) - return stack - - def _candidate(self, server: Any, state: Any, index: int) -> tuple[Any, Any]: - block_hash = f"{index:064x}" - pending = PendingShare( - share_id=f"miner-a:{block_hash}", - miner_id="miner-a", - order_key="miner-a", - p2mr_program_hex="11" * 32, - share_difficulty=1, - network_difficulty=1, - template_height=9, - job_id="job-1", - job_issued_at_ms=1, - accepted_at_ms=index, - ntime=1, - ) - candidate = block_candidate( - server, - state, - SimpleNamespace( - coinbase_tx_hex="00", - block_hash_hex=block_hash, - block_hex="00", - share_pass=True, - block_pass=True, - ), - pending_share=pending, - credit_share_on_accept=self.credit_shares, - ) - return candidate, (pending, server.block_candidate_intent(candidate)) - - def seed_live(self) -> CandidateStormSnapshot: - """Persist the storm, then admit every live candidate wakeup.""" - - if self.live_server is not None: - raise RuntimeError("live storm has already been seeded") - server, state = self._coordinator() - candidates = [] - durable_records = [] - for index in range(1, self.candidates + 1): - candidate, durable_record = self._candidate(server, state, index) - candidates.append(candidate) - durable_records.append(durable_record) - self.block_hashes = [ - str(candidate.submission.block_hash_hex).lower() - for candidate in candidates - ] - self.ledger.append_batch(durable_records) - if self.credit_shares: - # The live stamp path registers a credit-bearing share on the - # floor before its candidate is admitted; the rig appends the - # batch directly, so it registers the same holder here. - writer = server._ensure_share_writer_service() - for candidate in candidates: - writer.adopt_pending_share(candidate.pending_share) - with self._silence(): - for candidate in candidates: - server.enqueue_block_candidate(candidate) - self.live_server = server - return self.snapshot(server) - - def restart_and_enumerate(self) -> CandidateStormSnapshot: - """Create a fresh coordinator and enumerate every durable row.""" - - if self.live_server is None: - raise RuntimeError("seed_live must run before restart") - if self.restarted_server is not None: - raise RuntimeError("storm has already been restarted") - server, _state = self._coordinator() - # The restart view exists to hand a set-oriented selector the full - # replay-adopted population, so the shipped #183 collapse is - # suppressed for this enumeration exactly as it is for the per-row - # drain. Without that, deciding the height before the restart would - # let the enumeration dispose of the storm itself and leave the - # instrument with nothing to characterize. - server._ensure_block_candidate_service()._collapse_superseded_block_candidates = ( - lambda durable_rows, **_kwargs: durable_rows - ) - server._note_block_replay_enumeration_owed() - try: - with self._silence(): - restored = server.replay_pending_block_candidates() - finally: - server._ensure_block_candidate_service().__dict__.pop( - "_collapse_superseded_block_candidates", - None, - ) - if restored != self.candidates: - raise AssertionError( - f"restart restored {restored} of {self.candidates} candidates" - ) - self.restarted_server = server - return self.snapshot(server) - - def snapshot(self, server: Any) -> CandidateStormSnapshot: - """Capture fixed-cardinality gauges without retaining payload copies.""" - - service = server._ensure_block_candidate_service() - service._ensure_block_replay_state() - durable_rows = self.ledger.pending_block_candidate_rows( - limit=self.candidates + 1 - ) - durable_hashes = { - str(row["block_hash"]).lower() - for row in durable_rows - } - outstanding = set(service._outstanding_block_candidate_hashes) - replay_inflight = set(service._block_replay_inflight_hashes) - ownership = self.ownership(server) - return CandidateStormSnapshot( - durable_pending=len(durable_hashes), - live_queue_capacity=service.candidate_queue.maxsize, - live_queued=service.candidate_queue.qsize(), - live_coalesced=int(service.wakeups_coalesced), - outstanding_marked=len(outstanding), - outstanding_covers_all_durable=durable_hashes <= outstanding, - replay_queued=service._block_replay_candidate_queue.qsize(), - replay_inflight_marked=len(replay_inflight), - replay_inflight_covers_all_durable=durable_hashes <= replay_inflight, - accepted_parent_previews=len(server._accepted_block_payout_previews), - replay_enumeration_owed=server._block_replay_enumeration_owed(), - selector_evidence_marked=sum( - 1 for record in ownership if record.selector_evidence - ), - node_offer_evidence_marked=sum( - 1 for record in ownership if record.node_offer_evidence - ), - unoffered_retry_marked=sum( - 1 for record in ownership if record.retry_held_without_offer - ), - unoffered_lease_marked=sum( - 1 for record in ownership if record.lease_held_without_offer - ), - unoffered_terminal_marked=sum( - 1 for record in ownership if record.terminal_without_offer - ), - ) - - # -- decided-height extension ------------------------------------------ - - @property - def storm_height(self) -> int: - """The template height every storm candidate was built for.""" - server = self.live_server or self.restarted_server - if server is None: - raise RuntimeError("seed_live must run before the height is known") - return int(server.jobs["job-1"].template["height"]) - - def decide_height(self, *, winner_index: int | None = None) -> str: - """Make one storm candidate the active block at the storm height. - - Installs :class:`DecidedHeightRpc` on every coordinator this rig - owns (and on ones created later), so both the live and the restart - views observe the same decided chain. The default winner is the - LAST seeded candidate: it is a coalesced wakeup in the live view (it - never entered the bounded queue) and the last row of the restart - replay order, so a per-row drain reaches every sibling before it - would reach the winner. - """ - if not self.block_hashes: - raise RuntimeError("seed_live must run before a height can be decided") - if winner_index is None: - winner_index = self.candidates - if not 1 <= winner_index <= self.candidates: - raise ValueError("winner_index must name a seeded candidate") - winner_hash = self.block_hashes[winner_index - 1] - self.decided_rpc = DecidedHeightRpc( - winner_hash=winner_hash, - height=self.storm_height, - ) - for server in (self.live_server, self.restarted_server): - if server is not None: - server.rpc = self.decided_rpc - return winner_hash - - def ownership(self, server: Any) -> list[CandidateOwnership]: - """Read every seeded candidate's ownership/evidence facts at once.""" - service = server._ensure_block_candidate_service() - service._ensure_block_replay_state() - service._ensure_block_candidate_disposition_state() - outbox = self.ledger._block_candidate_outbox - live_queued = { - str(item.submission.block_hash_hex).lower() - for item in list(service.candidate_queue.queue) - } - replay_queued = { - str(item.submission.block_hash_hex).lower() - for item in list(service._block_replay_candidate_queue.queue) - } - with server.lock: - outstanding = set(service._outstanding_block_candidate_hashes) - replay_inflight = set(service._block_replay_inflight_hashes) - tip_observed = set(service._tip_observed_accepted_block_hashes) - retained = set( - getattr(service, "_block_candidate_retained_node_submissions", {}) - ) - accounted = set(server._accounted_accepted_block_hashes) - terminal = dict(service._block_candidate_terminal_outcomes) - retry_held: set[str] = set() - for holder in ( - service.retry_candidate, - getattr(service, "_block_accounting_deferred_retry_candidate", None), - ): - if holder is not None: - retry_held.add(str(holder.submission.block_hash_hex).lower()) - retry_held.update(service._block_disposition_waiting_retries) - retry_held.update(service.finalize_retries) - with service._block_candidate_disposition_registry_lock: - disposition_held = set(service._block_candidate_disposition_flights) - with server._accepted_block_payout_preview_condition: - previews = set(server._accepted_block_payout_previews) - records = [] - for block_hash in self.block_hashes: - row = outbox.get(block_hash) - intent = row["candidate"] if row is not None else None - pool_block = self.ledger.pool_block_state(block_hash=block_hash) - records.append( - CandidateOwnership( - block_hash=block_hash, - expected_height=( - int(intent["expected_height"]) - if isinstance(intent, Mapping) - else self.storm_height - ), - parent_hash=( - str(intent["parent_hash"]).lower() - if isinstance(intent, Mapping) - else STORM_PARENT_HASH - ), - network_difficulty=( - float(intent["found_block"]["network_difficulty"]) - if isinstance(intent, Mapping) - and isinstance(intent.get("found_block"), dict) - and intent["found_block"].get("network_difficulty") is not None - else None - ), - outbox_state=str(row["state"]) if row is not None else None, - attempt_count=int(row["attempt_count"]) if row is not None else 0, - live_queued=block_hash in live_queued, - replay_queued=block_hash in replay_queued, - outstanding=block_hash in outstanding, - replay_inflight=block_hash in replay_inflight, - preview_barrier=block_hash in previews, - disposition_held=block_hash in disposition_held, - retry_held=block_hash in retry_held, - node_acceptance_retained=block_hash in retained, - tip_observed=block_hash in tip_observed, - accounted_accepted=block_hash in accounted, - terminal_outcome=terminal.get(block_hash), - pool_block_chain_state=( - str(pool_block["chain_state"]) - if pool_block is not None - else None - ), - ) - ) - return records - - # -- perturbation ------------------------------------------------------- - - def _take_queued_candidate(self, server: Any, block_hash: str) -> Any: - """Remove one queued wakeup from whichever lane currently holds it.""" - service = server._ensure_block_candidate_service() - service._ensure_block_replay_state() - for queue_obj in ( - service.candidate_queue, - service._block_replay_candidate_queue, - ): - for item in list(queue_obj.queue): - if str(item.submission.block_hash_hex).lower() == block_hash: - # Single-threaded instrument: removing in place from the - # backing deque keeps both queue order and qsize() exact. - queue_obj.queue.remove(item) - return item - raise KeyError(f"no queued wakeup for {block_hash}") - - def perturb(self, server: Any, kind: str, block_hash: str) -> Any: - """Install one selector-evidence shape on a seeded candidate. - - Each kind reproduces one of the facts ``submit_next`` and - ``record_abandoned`` consult, through the shipped entry point rather - than a hand-written marker, and each shows up in :meth:`ownership` as - ``selector_evidence``. Three kinds show up there *without* - ``node_offer_evidence``, because none reaches qbitd: ``retry_slot`` - moves an already-queued, never-offered wakeup into the retry holder, - which is exactly what the shipped path does when the fast-lane - reservation declines; ``lease_held`` claims the disposition lease - by itself, which is all ``submit_next`` holds while it runs the - terminal, capacity and fast-lane checks that precede any submitblock; - and ``terminal`` records the disposition seal that the capacity-closed - path installs with ``attempted=False`` and no submitblock at all. - Returns a callable that undoes the perturbation where undoing is - meaningful (the held disposition lease), otherwise ``None``. - """ - service = server._ensure_block_candidate_service() - service._ensure_block_replay_state() - service._ensure_block_candidate_disposition_state() - key = str(block_hash).lower() - if kind == "lease_held": - lease = server._claim_block_candidate_disposition(key, blocking=False) - if lease is None: - raise RuntimeError(f"disposition lease already held for {key}") - return lambda: server._release_block_candidate_disposition(lease) - if kind == "retained_success": - service._stash_retained_block_candidate_node_submission( - key, - _BlockCandidateNodeSubmission(attempted=True, result=None), - ) - return None - if kind == "tip_observed": - # Only an outstanding hash can register a tip observation, and - # replay adoption alone never makes one outstanding -- the node - # offer does (see test_replay_adoption_alone_drops_a_tip_observation). - # Model "offered, then seen as the tip" through both shipped entry - # points so the shape holds in the restart view too. - server._register_outstanding_block_candidate(key) - server._note_tip_observation_for_candidates(key) - return None - if kind == "retry_slot": - candidate = self._take_queued_candidate(server, key) - with server.lock: - service.retry_candidate = candidate - return None - if kind == "prepared_pool_block": - self.ledger.persist_accepted_block( - block_hash=key, - block_height=self.storm_height, - parent_hash=STORM_PARENT_HASH, - final_bundle={}, - audit_report={}, - ) - return None - if kind == "terminal": - service._record_block_candidate_terminal_outcome(key, accepted=False) - return None - raise ValueError(f"unknown perturbation: {kind}") - - # -- per-row drain ------------------------------------------------------ - - # A head that neither terminalizes a row nor parks on a held lease is - # being deferred by the shipped path (acceptance evidence). Offer it - # once more, then withhold it: re-offering forever is what the shipped - # loop does behind its backoff, and this drain has no backoff. - _DRAIN_STALL_LIMIT = 2 - - def _dequeue_head(self, server: Any) -> tuple[str | None, str | None]: - """Peek the hash and lane ``submit_next`` would take next, in order.""" - service = server._ensure_block_candidate_service() - service._ensure_block_replay_state() - service._ensure_block_candidate_disposition_state() - with server.lock: - retry = service.retry_candidate - if retry is not None and service._block_candidate_retry_ready_locked( - retry - ): - return str(retry.submission.block_hash_hex).lower(), "retry" - for lane, queue_obj in ( - ("live", service.candidate_queue), - ("replay", service._block_replay_candidate_queue), - ): - items = list(queue_obj.queue) - if items: - return str(items[0].submission.block_hash_hex).lower(), lane - with server.lock: - waiting = service._block_disposition_waiting_retries - ready = [ - key - for key in waiting - if service._block_candidate_retry_ready_locked(waiting[key]) - ] - if ready: - return ( - min( - ready, - key=lambda key: int( - waiting[key].context.template["height"] - ), - ), - "waiting", - ) - return None, None - - def _withhold_head( - self, - server: Any, - block_hash: str, - lane: str, - ) -> Any | None: - """Lift one withheld hash out of the lane that would dequeue it next. - - Returns the exact wakeup object removed, so the drain can put it back - before it returns. Discarding it instead would strand the row for - good: replay adoption keeps the hash in - ``_block_replay_inflight_hashes`` until a terminal outcome is - recorded, and ``_enqueue_replayed_block_candidate`` drops any - re-adoption of an in-flight hash as a duplicate, so no later - enumeration can rebuild the wakeup this call threw away. - """ - service = server._ensure_block_candidate_service() - if lane == "retry": - with server.lock: - candidate = service.retry_candidate - service.retry_candidate = None - return candidate - if lane == "live": - return service.candidate_queue.get_nowait() - if lane == "replay": - return service._block_replay_candidate_queue.get_nowait() - if lane == "waiting": - with server.lock: - return service._block_disposition_waiting_retries.pop( - block_hash, - None, - ) - return None - - def _restore_withheld_wakeup( - self, - server: Any, - block_hash: str, - lane: str, - wakeup: Any, - ) -> bool: - """Put one lifted wakeup back where it was lifted from. - - Returns False when an equivalent wakeup already stands for the hash, - which makes this object a dropped same-hash duplicate for the caller - to release. - - Queue lanes are restored by pushing onto the front of the backing - deque rather than through ``put_nowait``. The wakeup was the head - when it was lifted and everything ahead of it has since been - consumed, so the front is its exact position; and because - ``get_nowait`` never touched ``unfinished_tasks``, putting it back in - place leaves that counter matching the item's original ``put`` - instead of double-counting it. This is the same in-place idiom - :meth:`_take_queued_candidate` uses, for the same reason. - """ - service = server._ensure_block_candidate_service() - service._ensure_block_replay_state() - service._ensure_block_candidate_disposition_state() - if lane in ("retry", "waiting"): - with server.lock: - if lane == "retry" and service.retry_candidate is None: - service.retry_candidate = wakeup - return True - # Either this is a waiting-registry entry, or the single - # retry head slot has been taken since. The shipped merge - # parks a displaced hash in the waiting registry for exactly - # this reason, so park it there rather than drop it. - waiting = service._block_disposition_waiting_retries - if block_hash in waiting: - return False - waiting[block_hash] = wakeup - return True - queue_obj = ( - service._block_replay_candidate_queue - if lane == "replay" - else service.candidate_queue - ) - queue_obj.queue.appendleft(wakeup) - return True - - def _next_drainable( - self, - server: Any, - withheld: set[str], - held: list[tuple[str, str, Any]] | None = None, - ) -> tuple[str | None, str | None]: - """Return the next hash to drive, lifting withheld ones out on the way. - - Every wakeup lifted out is appended to ``held`` in lift order so the - drain can restore it; see :meth:`_restore_withheld_wakeup`. - """ - while True: - head, lane = self._dequeue_head(server) - if head is None or head not in withheld: - return head, lane - wakeup = self._withhold_head(server, head, str(lane)) - if wakeup is not None and held is not None: - held.append((str(lane), head, wakeup)) - - @staticmethod - def _node_offers_made(rpc: Any) -> int: - """Total ``submitblock`` calls the rig's RPC log has counted so far. - - :class:`DecidedHeightRpc` counts per method under a lock, and the - shipped fast lane runs ``submitblock`` on a bounded worker thread - whose completion ``submit_next`` waits for, so a delta taken across - one ``submit_next`` names exactly the offers that call performed. - - The counter is required rather than optional: a drain that could not - read it would report zero offers forever, which would silently make - ``max_rounds`` unbounded instead of stopping the drain. Every drain - runs against a decided height, so the counter is always installed. - """ - calls = getattr(rpc, "calls", None) - if not isinstance(calls, dict): - raise TypeError( - "drain_per_row needs the per-method RPC counter installed by " - "decide_height; rounds and max_rounds are measured from its " - "submitblock count" - ) - return int(calls.get("submitblock", 0)) - - def _run_block_accounting_tasks(self, server: Any) -> int: - """Run the shipped accounting lane to quiescence, on this thread. - - Mirrors ``BlockCandidateService.block_accounting_loop``: the - definitive-acceptance lane first while its dispatch quota lasts, then - the primary queue, then the unbounded spillover, then the - invalid-intent quarantine, with the loop's own retain-on-failure - behaviour. The quota is read from the service rather than assumed so - a scenario that pins it also pins what this drain does. - """ - service = server._ensure_block_candidate_service() - service._ensure_block_accounting_state() - # block_accounting_loop stamps this on entry, and - # _pace_block_candidate_retry reads it to record a not-before deadline - # instead of sleeping while the lease and writer admission are held. - service._block_accounting_thread_ident = threading.get_ident() - ran = 0 - accepted_run = 0 - while True: - source_queue = None - if accepted_run < service._block_accounting_accepted_dispatch_quota(): - try: - _priority, _sequence, task = ( - service._block_accounting_accepted_queue.get_nowait() - ) - source_queue = service._block_accounting_accepted_queue - accepted_run += 1 - except queue.Empty: - pass - if source_queue is None: - try: - _priority, _sequence, task = ( - service._block_accounting_queue.get_nowait() - ) - source_queue = service._block_accounting_queue - except queue.Empty: - try: - _priority, _sequence, task = ( - service._block_accounting_overflow_queue.get_nowait() - ) - source_queue = service._block_accounting_overflow_queue - except queue.Empty: - pass - if source_queue is not None: - accepted_run = 0 - if source_queue is None: - try: - _priority, _sequence, task = ( - service._block_accounting_accepted_queue.get_nowait() - ) - source_queue = service._block_accounting_accepted_queue - accepted_run = 0 - except queue.Empty: - while server._run_one_invalid_block_candidate_quarantine(): - pass - return ran - try: - server._run_block_accounting_task(task) - except Exception: - server._retain_block_candidate_for_retry(task.candidate) - finally: - source_queue.task_done() - ran += 1 - - def drain_per_row( - self, - server: Any, - *, - stop_before_hash: str | None = None, - preserve_hashes: Iterable[str] | None = None, - max_rounds: int | None = None, - dequeue_skip: bool = False, - ) -> PerRowDrainReport: - """Drive the shipped per-row path in the production submitter split. - - Each round runs ``submit_next_block_candidate(defer_accounting=True)`` - -- the call ``BlockCandidateService.run`` makes -- and then drains the - real accounting queue through ``_run_block_accounting_task``, so the - node offer, the fast-lane reservation, the replayed-candidate - acceptance-evidence restore and the durable finalization all run on - their shipped code paths. When both lanes are empty the outbox replay - query recovers coalesced wakeups, exactly as the loop does. - - ``defer_accounting=True`` is not a detail: with accounting inline - ``submit_next`` claims each disposition lease *blocking*, so a single - candidate whose lease is held elsewhere stops the drain forever, and - ``_restore_replayed_candidate_acceptance_evidence`` -- which is what - keeps a replayed candidate with a prepared pool-block row from being - abandoned -- never runs at all. - - ``stop_before_hash`` ends the drain when that hash reaches the head - (used for the decided winner, whose finalization tail is outside this - instrument). ``preserve_hashes`` instead withholds hashes for the - whole drain: they are lifted out of the dequeue order without being - offered or disposed, so their durable row and evidence markers stay - exactly as the caller arranged them while every other row drains. - Any head the shipped path parks on a held lease, or repeatedly - declines to terminalize, is withheld the same way and reported, so - the drain is bounded no matter what evidence the caller installed. - - Withholding is scoped to the invocation, so every wakeup this call - lifted out is put back before it returns, on every exit -- exhausted, - ``max_rounds``, ``stop_before_hash``, or an exception. Restoring is - not a courtesy: a withheld row keeps its ownership markers, and a - hash marked replay-inflight is one no enumeration will re-adopt, so - a discarded wakeup leaves the row pending with nothing left to - dispose of it and no later drain able to reach it. - - ``max_rounds`` stops the drain after that many *node offers* -- the - ``submitblock`` calls this invocation actually made, measured across - each ``submit_next`` from the rig's own RPC log -- so a caller can - step the oracle one offered row at a time. It is deliberately not - the count of consumed wakeups: ``submit_next`` also returns True for - a row it parked on a lease held elsewhere, a same-hash duplicate it - dropped, a fast-lane reservation it declined, and the capacity-closed - ``attempted=False`` split that terminalizes a durable row without - calling qbitd, and spending the budget on those would stop a bounded - drain before it performed the offers it promised. Boundedness for - those rows comes from the withholding above, not from the budget. - The report then covers only the rows *this* call moved: it is - compared against the durable state captured on entry, so a second - bounded call never re-reports the first one's abandonment. - - ``dequeue_skip`` is off by default, and off is what keeps this the - per-row oracle. Issue #181 item 2 added a second set-oriented - disposition inside ``submit_next`` itself -- the dequeue-time stale - sibling skip -- which would otherwise dispose of rows this drain is - meant to measure the per-row path against, exactly as #183's - replay-adoption collapse would. Suppressed the same way and for the - same reason. Turning it on drives the shipped skip instead, so the - same report shape (abandoned hashes, per-method RPC counts, attempt - marks) can be compared against the oracle the default produces. - """ - service = server._ensure_block_candidate_service() - service._ensure_block_candidate_disposition_state() - # Issue #183 landed a set-oriented collapse inside ``replay_pending``. - # This drain exists to be the *per-row* oracle that selector is - # measured against, and it re-enumerates the outbox whenever both - # lanes empty, so the collapse is suppressed for its duration: - # otherwise the enumeration would dispose of rows itself and the - # comparison would be the selector against the selector. - service._collapse_superseded_block_candidates = ( - lambda durable_rows, **_kwargs: durable_rows - ) - if not dequeue_skip: - # Issue #181 item 2's dequeue-time skip is a second shipped - # disposition inside ``submit_next``. The oracle is the per-row - # path, so it is suppressed exactly as the #183 collapse is. - service._skip_superseded_block_candidate_at_dequeue = ( - lambda _candidate, **_kwargs: False - ) - rpc = server.rpc - calls_before = dict(getattr(rpc, "calls", {})) - outbox = self.ledger._block_candidate_outbox - attempts_before = sum(int(row["attempt_count"]) for row in outbox.values()) - # The durable state of every row on entry. A row is only ever - # finished out of "pending" once, so comparing against this is what - # makes the report name this call's transitions rather than every - # terminal row the rig has accumulated across earlier drains. - states_before = { - str(block_hash).lower(): str(row["state"]) - for block_hash, row in outbox.items() - } - withheld = {str(item).lower() for item in (preserve_hashes or ())} - caller_withheld = frozenset(withheld) - stop_hash = None if stop_before_hash is None else str(stop_before_hash).lower() - lease_blocked: set[str] = set() - deferred: set[str] = set() - stalls: dict[str, int] = {} - # Node offers this call performed. The rig's RPC log is the only - # authoritative source: ``submit_next``'s boolean is True for - # no-offer paths too (see PerRowDrainReport). - rounds = 0 - accounting_tasks = 0 - enumerations = 0 - # Every wakeup withholding lifted out, in lift order, so the finally - # below can put each one back exactly where it came from. - held_wakeups: list[tuple[str, str, Any]] = [] - started = time.monotonic() - try: - with self._silence(): - while True: - if max_rounds is not None and rounds >= max_rounds: - break - head, _lane = self._next_drainable(server, withheld, held_wakeups) - if head is None: - enumerations += 1 - if server.replay_pending_block_candidates() == 0: - break - if self._next_drainable( - server, - withheld, - held_wakeups, - )[0] is None: - # The enumeration restored only withheld rows; the - # shipped loop would re-adopt and re-park them for as - # long as their evidence stands. - break - continue - if stop_hash is not None and head == stop_hash: - break - with server.lock: - terminal_before = len(service._block_candidate_terminal_outcomes) - offers_before = self._node_offers_made(rpc) - server.submit_next_block_candidate(defer_accounting=True) - # submit_next waits for its own submitblock worker before - # returning, so this delta is complete and is either 0 or - # 1 for the row just driven. - rounds += self._node_offers_made(rpc) - offers_before - accounting_tasks += self._run_block_accounting_tasks(server) - with server.lock: - progressed = ( - len(service._block_candidate_terminal_outcomes) - != terminal_before - ) - parked = head in service._block_disposition_waiting_retries - if progressed: - stalls.pop(head, None) - continue - if parked: - # submit_next could not claim this hash's lease without - # blocking and parked the wakeup, exactly as the shipped - # loop does. Withhold it instead of spinning behind a - # lease this drain does not own. - withheld.add(head) - lease_blocked.add(head) - continue - stalls[head] = stalls.get(head, 0) + 1 - if stalls[head] >= self._DRAIN_STALL_LIMIT: - withheld.add(head) - deferred.add(head) - finally: - try: - with self._silence(): - # Newest lift first: a mid-drain enumeration can re-adopt a - # row this call already lifted, and the re-adopted object is - # the one the shipped markers now describe. Restoring - # newest-first also rebuilds the original lane order, - # because each lifted wakeup goes back to its lane's front. - restored: set[str] = set() - for lane, key, wakeup in reversed(held_wakeups): - if key in restored or not self._restore_withheld_wakeup( - server, - key, - lane, - wakeup, - ): - # A wakeup for this hash already stands, so this - # object is a dropped same-hash duplicate and must - # release the credit floor it carries -- exactly - # what the shipped duplicate paths do. - service._release_dropped_duplicate_candidate_floor( - wakeup - ) - continue - restored.add(key) - finally: - service.__dict__.pop( - "_collapse_superseded_block_candidates", - None, - ) - service.__dict__.pop( - "_skip_superseded_block_candidate_at_dequeue", - None, - ) - terminalized: dict[str, set[str]] = {"submitted": set(), "abandoned": set()} - still_pending: set[str] = set() - for block_hash, row in outbox.items(): - key = str(block_hash).lower() - state = str(row["state"]) - if state == "pending": - still_pending.add(key) - elif states_before.get(key) != state: - terminalized.setdefault(state, set()).add(key) - calls_after = dict(getattr(rpc, "calls", {})) - return PerRowDrainReport( - rounds=rounds, - accounting_tasks=accounting_tasks, - replay_enumerations=enumerations, - abandoned_hashes=frozenset(terminalized["abandoned"]), - submitted_hashes=frozenset(terminalized["submitted"]), - pending_hashes=frozenset(still_pending), - withheld_hashes=( - caller_withheld - | frozenset(lease_blocked) - | frozenset(deferred) - ), - lease_blocked_hashes=frozenset(lease_blocked), - deferred_hashes=frozenset(deferred), - rpc_calls={ - method: calls_after.get(method, 0) - calls_before.get(method, 0) - for method in sorted(set(calls_before) | set(calls_after)) - }, - ledger_attempt_marks=( - sum(int(row["attempt_count"]) for row in outbox.values()) - - attempts_before - ), - wall_seconds=time.monotonic() - started, - ) - - # -- cleanup-retry backlog (issue #198) --------------------------------- - - def _pending_hashes(self) -> set[str]: - return { - str(row["block_hash"]).lower() - for row in self.ledger.pending_block_candidate_rows( - limit=self.candidates + 1 - ) - } - - def _registry_bytes(self, server: Any) -> tuple[int, int, int]: - """(registry bytes, holder bytes within them, retained holders). - - Read under the coordinator lock like every other registry reader. - Holders are charged inside the registry total first, then measured - again on their own so the per-holder figure can be reported apart - from the per-record one. - """ - service = server._ensure_block_candidate_service() - with server.lock: - registry = service._block_candidate_collapse_cleanup_retries - total = _deep_size(registry, set()) - holders: dict[int, Any] = {} - for record in registry.values(): - for share in record.shares: - holders[id(share)] = share - holder_seen: set[int] = set() - holder_bytes = sum(_deep_size(share, holder_seen) for share in holders.values()) - return total, holder_bytes, len(holders) - - def _collapse_walk(self, server: Any) -> tuple[set[str], float, int]: - """Run the shipped enumeration walk; return (rows it won, seconds, adoptions).""" - before = self._pending_hashes() - started = time.monotonic() - with self._silence_all(): - server._note_block_replay_enumeration_owed() - adopted = int(server.replay_pending_block_candidates()) - elapsed = time.monotonic() - started - return before - self._pending_hashes(), elapsed, adopted - - def _drain_backpressured_replay_page( - self, - server: Any, - *, - stop_before_hash: str, - ) -> set[str]: - """Drive the bounded spill page through the shipped dequeue collapse.""" - service = server._ensure_block_candidate_service() - if not service._block_replay_backpressure_drain_pending: - return set() - terminalized: set[str] = set() - replay_queue = service._block_replay_candidate_queue - # The queue cannot grow in this synchronous rig. Bound the driver by - # the captured page size so a failed disposition cannot spin. - rounds = replay_queue.qsize() - with self._silence_all(): - for _ in range(rounds): - if replay_queue.empty(): - break - candidate = replay_queue.queue[0] - block_hash = str(candidate.submission.block_hash_hex).lower() - if block_hash == stop_before_hash: - break - row = self.ledger._block_candidate_outbox[block_hash] - before = str(row["state"]) - if not server.submit_next_block_candidate(defer_accounting=True): - break - self._run_block_accounting_tasks(server) - after = str(row["state"]) - if before == "pending" and after != "pending": - terminalized.add(block_hash) - return terminalized - - def measure_cleanup_backlog( - self, - fault: str, - *, - view: str = "restart", - sustained_passes: int = 64, - trace_memory: bool = True, - ) -> CleanupBacklogReport: - """Drive the shipped collapse under one cleanup fault, then recover it. - - Phases, each on the shipped code path: - - 1. Seed the storm (and restart, for the restart view), decide the - height, and install ``fault`` at its seam. - 2. Run the enumeration walk. Every row the fenced write wins whose - cleanup the fault breaks becomes one retry record; rows the - admission bound declines stay durable and are counted as - preserved. The registry is measured in place. - 3. With the fault still on, run up to ``sustained_passes`` retry - passes: each must fail, re-register its record, and leave depth, - holders and pins unchanged -- the backlog is never shed. - 4. Heal the fault and drain the backlog through the shipped retry - pass until it reports nothing due, timing the drain. - 5. Drain the one replay page adopted at backpressure through the - shipped dequeue collapse, run the walk again so admission can - resume over whatever remains durable, then drive ``drain_per_row`` - over every queued wakeup to prove no terminal row reaches the - node. - - ``view`` selects the coordinator: ``live`` holds the storm's first - ``queue_depth`` wakeups in the bounded queue with the rest coalesced, - so only that many holders can be retained; ``restart`` holds every - row in the replay queue, so every record retains a holder -- the - heavier and more instructive case. - """ - if fault != CLEANUP_FAULT_NONE and fault not in CLEANUP_FAULTS: - raise ValueError(f"unknown cleanup fault: {fault}") - if view not in ("live", "restart"): - raise ValueError("view must be 'live' or 'restart'") - if self.live_server is None: - self.seed_live() - if view == "restart" and self.restarted_server is None: - self.restart_and_enumerate() - server = self.restarted_server if view == "restart" else self.live_server - assert server is not None - winner = ( - self.decide_height() - if self.decided_rpc is None - else self.decided_rpc.winner_hash - ) - service = server._ensure_block_candidate_service() - service._ensure_block_replay_state() - service._ensure_block_candidate_disposition_state() - writer = server._ensure_share_writer_service() - floor_before = len(writer._pending_share_commit_floor) - injector = CleanupFaultInjector(server, fault) - injector.install() - traced_walk = -1 - traced_owner = -1 - traced_peak = -1 - try: - counts_before = service.block_candidate_collapse_snapshot() - with server.lock: - inflight_before = set(service._block_replay_inflight_hashes) - if trace_memory: - _warm_traceback_caches() - gc.collect() - tracemalloc.start() - tracemalloc.reset_peak() - traced_before = tracemalloc.get_traced_memory()[0] - owner_before = _owner_traced_bytes(tracemalloc.take_snapshot()) - collapsed, walk_seconds, _adopted = self._collapse_walk(server) - if trace_memory: - gc.collect() - traced_after, traced_peak_after = tracemalloc.get_traced_memory() - owner_after = _owner_traced_bytes(tracemalloc.take_snapshot()) - tracemalloc.stop() - traced_walk = traced_after - traced_before - traced_owner = owner_after - owner_before - traced_peak = traced_peak_after - traced_before - counts_after = service.block_candidate_collapse_snapshot() - snapshot = service.collapsed_candidate_cleanup_backlog_snapshot() - registry_bytes, holder_bytes, holders = self._registry_bytes(server) - records = int(snapshot["depth"]) - # -- sustained failure ------------------------------------------ - sustained = 0 - failed_before = counts_after["cleanup_retry_failed"] - with self._silence_all(): - for _ in range(min(int(sustained_passes), records)): - if not service._run_one_collapsed_block_candidate_cleanup_retry(): - break - sustained += 1 - sustained_failed = ( - service.block_candidate_collapse_snapshot()["cleanup_retry_failed"] - - failed_before - ) - after_sustained = service.collapsed_candidate_cleanup_backlog_snapshot() - # -- recovery --------------------------------------------------- - injector.heal() - recovered_before = service.block_candidate_collapse_snapshot()[ - "cleanup_recovered" - ] - passes = 0 - started = time.monotonic() - with self._silence_all(): - while service._run_one_collapsed_block_candidate_cleanup_retry(): - passes += 1 - recovery_seconds = time.monotonic() - started - recovered = ( - service.block_candidate_collapse_snapshot()["cleanup_recovered"] - - recovered_before - ) - after_recovery = service.collapsed_candidate_cleanup_backlog_snapshot() - recovered_calls = injector.snapshot_calls() - resumed_at_dequeue = self._drain_backpressured_replay_page( - server, - stop_before_hash=winner, - ) - collapsed_after, second_walk_seconds, _adopted = self._collapse_walk(server) - with server.lock: - inflight_after = set(service._block_replay_inflight_hashes) - every_collapsed = collapsed | resumed_at_dequeue | collapsed_after - excess = injector.excess_calls(every_collapsed) - missing = injector.missing_calls(every_collapsed) - terminal_adopted = len((inflight_after - inflight_before) & every_collapsed) - drain = self.drain_per_row(server, stop_before_hash=winner) - # A hash whose record was discharged must never be cleaned up - # again: not by the second walk, not by the queued duplicates - # the drain drops. - post_recovery_calls = injector.calls_since(recovered_calls, collapsed) - finally: - if trace_memory and tracemalloc.is_tracing(): - tracemalloc.stop() - injector.uninstall() - # The winner's holder is the one the shipped path still owns: in the - # restart view the object adopted at restart, in the live view the - # object the first walk's recovery enumeration adopted (the seeded - # live wakeup coalesced and was never queued). Looked up after the - # walks for exactly that reason. - winner_share = next( - ( - item.pending_share - for queue_obj in ( - service.candidate_queue, - service._block_replay_candidate_queue, - ) - for item in list(queue_obj.queue) - if str(item.submission.block_hash_hex).lower() == winner - ), - None, - ) - floor_after = len(writer._pending_share_commit_floor) - return CleanupBacklogReport( - fault=fault, - view=view, - candidates=self.candidates, - credit_shares=self.credit_shares, - backlog_max=int(snapshot["backlog_max"]), - walk_seconds=walk_seconds, - collapsed_rows=len(collapsed), - deferred_records=records, - preserved_rows=( - counts_after["backlog_deferred"] - counts_before["backlog_deferred"] - ), - backpressure_engagements=int(snapshot["backpressure_engagements"]), - backpressure_active=bool(snapshot["backpressure_active"]), - pending_share_holders=int(snapshot["pending_share_holders"]), - terminal_outcome_pins=int(snapshot["terminal_outcome_pins"]), - oldest_age_seconds=float(snapshot["oldest_age_seconds"]), - registry_bytes=registry_bytes, - holder_bytes=holder_bytes, - registry_bytes_per_record=(registry_bytes / records) if records else 0.0, - holder_bytes_per_holder=(holder_bytes / holders) if holders else 0.0, - traced_walk_bytes=traced_walk, - traced_owner_bytes=traced_owner, - traced_peak_bytes=traced_peak, - sustained_passes=sustained, - sustained_failed_passes=int(sustained_failed), - depth_after_sustained=int(after_sustained["depth"]), - holders_after_sustained=int(after_sustained["pending_share_holders"]), - pins_after_sustained=int(after_sustained["terminal_outcome_pins"]), - recovery_seconds=recovery_seconds, - recovery_passes=passes, - recovered_records=int(recovered), - retry_records_per_second=( - (recovered / recovery_seconds) if recovery_seconds > 0 else 0.0 - ), - depth_after_recovery=int(after_recovery["depth"]), - second_walk_seconds=second_walk_seconds, - collapsed_rows_after_recovery=len( - resumed_at_dequeue | collapsed_after - ), - excess_cleanup_calls=excess, - missing_cleanup_calls=missing, - post_recovery_cleanup_calls=post_recovery_calls, - floor_double_releases=injector.floor_double_releases, - terminal_rows_adopted=terminal_adopted, - node_offers=int(self.decided_rpc.calls.get("submitblock", 0)), - drain_rounds=drain.rounds, - pending_rows_after_drain=drain.pending_rows, - floor_holders_before=floor_before, - floor_holders_after=floor_after, - winner_floor_holder_retained=( - winner_share is not None - and id(winner_share) in writer._pending_share_commit_floor - ), - ) - - -# -- heap and allocator readings (issue #226) ------------------------------ -# -# #185 measured the coordinator at 125 MiB before a storm and 613 MiB after -# the drain, and #226 found the mainnet resident set growing 390 MB/h with a -# memory map shaped like glibc per-thread arenas. These readings put the same -# instrument around the storm so both can be re-run on any host: the resident -# set, the interpreter's live-block count, glibc's arena/in-use/free bytes and -# arena count, and the anonymous-map band histogram, at every phase boundary, -# then after the storm's objects are released and collected, then after one -# malloc_trim. Every number is host-specific -- absolutes vary by up to an -# order of magnitude across hosts -- and the instrument reports the platform -# beside them; what transfers is the mechanism: does the live-block count -# return, does the free-bytes reading grow, does the trim return it. - -ARENA_PROBE_MIN_BYTES = 4 * 1024 -# Under glibc's initial 128 KiB mmap threshold, so the buffers come from the -# arenas rather than from direct mmap; over pymalloc's 512-byte ceiling, so -# they come from glibc at all. -ARENA_PROBE_MAX_BYTES = 120 * 1024 -ARENA_PROBE_LIVE_BUFFERS = 64 -# A worker's wait at the barrier, and half the join ceiling. 64 threads x -# 2,000 rounds take about a second on a development host; a slow host has -# an order of magnitude of headroom before the probe reports a failure. -ARENA_PROBE_BARRIER_TIMEOUT_SECONDS = 60.0 - - -@dataclass(frozen=True) -class HeapReading: - """One phase boundary's resident-set and allocator readings.""" - - phase: str - elapsed_seconds: float - resident_bytes: int - allocated_blocks: int - threads: int - malloc_info_available: bool - malloc_arena_bytes: int - malloc_in_use_bytes: int - malloc_free_bytes: int - malloc_mmapped_bytes: int - malloc_arena_count: int - # Free bytes split by where they sit: interior bins, which malloc_trim - # returns in every arena, and per-thread heap tops, which it cannot. - malloc_free_top_bytes: int - malloc_free_bin_bytes: int - anonymous_regions: int - anonymous_bytes: int - anonymous_regions_4mib_to_64mib: int - heap_segment_bytes: int - - -def take_heap_reading( - phase: str, - telemetry: ProcessHeapTelemetry, - started_monotonic: float, -) -> HeapReading: - sample = telemetry.sample() - shape = read_anonymous_map_shape() - malloc_info = read_malloc_info_summary() - return HeapReading( - phase=phase, - elapsed_seconds=round(time.monotonic() - started_monotonic, 3), - resident_bytes=read_resident_memory_bytes(), - allocated_blocks=sample.allocated_blocks, - threads=sample.threads, - malloc_info_available=sample.malloc_info_available, - malloc_arena_bytes=sample.malloc_arena_bytes, - malloc_in_use_bytes=sample.malloc_in_use_bytes, - malloc_free_bytes=sample.malloc_free_bytes, - malloc_mmapped_bytes=sample.malloc_mmapped_bytes, - malloc_arena_count=malloc_info.arenas, - malloc_free_top_bytes=malloc_info.top_bytes, - malloc_free_bin_bytes=malloc_info.bin_bytes, - anonymous_regions=shape.regions, - anonymous_bytes=shape.total_bytes, - anonymous_regions_4mib_to_64mib=shape.band_regions.get("4mib_to_64mib", -1), - heap_segment_bytes=shape.heap_segment_bytes, - ) - - -def allocator_environment() -> dict[str, str]: - """The glibc and CPython allocator variables this process started with.""" - return { - name: value - for name, value in sorted(os.environ.items()) - if name.startswith("MALLOC_") or name in ("GLIBC_TUNABLES", "PYTHONMALLOC") - } - - -def run_arena_probe( - threads: int, - rounds: int, - *, - seed: int = 226, - telemetry: ProcessHeapTelemetry | None = None, - allocate: Any = bytes, -) -> dict[str, Any]: - """Reproduce the per-thread arena shape and report what it costs. - - ``threads`` Python threads each allocate ``rounds`` medium buffers - (4-120 KiB: over pymalloc's ceiling, under glibc's initial mmap - threshold) into a small ring so a bounded number stay live at once, then - drop the ring and exit. glibc assigns a thread its own arena on its first - allocation (up to ``MALLOC_ARENA_MAX``) only while no exited thread's - arena is waiting on the free list, so the threads meet at a barrier - before any exits -- the coordinator's 64 threads are long-lived, and - that is the shape under test. The arena count after the probe is the - mechanism: it should approach ``threads + 1`` at glibc's default and stop - at the configured cap otherwise. The rings' freed buffers stay on each - arena's free lists after the threads exit -- arenas outlive threads -- - which is the retained-free-bytes reading the trim that follows is judged - by. ``allocate`` is the buffer constructor (``bytes``); a test injects a - failing one to prove a broken worker cannot wedge the probe. - """ - telemetry = telemetry if telemetry is not None else ProcessHeapTelemetry() - before = telemetry.sample() - resident_before = read_resident_memory_bytes() - arenas_before = read_malloc_arena_count() - started = time.monotonic() - # Bounded: a worker that fails breaks the barrier for everyone, a worker - # that stalls trips the barrier timeout, and the join below has its own - # ceiling, so one bad thread cannot wedge the instrument. Every such - # event is reported in ``failures`` rather than raised mid-probe. - all_allocated = threading.Barrier(threads) - failures: list[str] = [] - - def worker(index: int) -> None: - rng = random.Random(seed + index) - ring: list[bytes | None] = [None] * ARENA_PROBE_LIVE_BUFFERS - try: - for round_index in range(rounds): - size = rng.randint(ARENA_PROBE_MIN_BYTES, ARENA_PROBE_MAX_BYTES) - ring[round_index % ARENA_PROBE_LIVE_BUFFERS] = allocate(size) - all_allocated.wait(timeout=ARENA_PROBE_BARRIER_TIMEOUT_SECONDS) - except threading.BrokenBarrierError: - failures.append(f"worker {index}: barrier broken or timed out") - except Exception as exc: - all_allocated.abort() - failures.append(f"worker {index}: {exc!r}") - finally: - ring.clear() - - workers = [ - threading.Thread(target=worker, args=(index,), name=f"arena-probe-{index}") - for index in range(threads) - ] - for thread in workers: - thread.start() - join_deadline = time.monotonic() + ARENA_PROBE_BARRIER_TIMEOUT_SECONDS * 2 - for thread in workers: - thread.join(timeout=max(0.0, join_deadline - time.monotonic())) - if thread.is_alive(): - all_allocated.abort() - failures.append(f"{thread.name}: did not finish before the join ceiling") - seconds = time.monotonic() - started - after = telemetry.sample() - return { - "threads": threads, - "rounds": rounds, - "buffer_bytes": [ARENA_PROBE_MIN_BYTES, ARENA_PROBE_MAX_BYTES], - "live_buffers_per_thread": ARENA_PROBE_LIVE_BUFFERS, - "failures": failures, - "seconds": round(seconds, 6), - "arena_count_before": arenas_before, - "arena_count_after": read_malloc_arena_count(), - "resident_before": resident_before, - "resident_after": read_resident_memory_bytes(), - "malloc_info_available": before.malloc_info_available and after.malloc_info_available, - "malloc_arena_before": before.malloc_arena_bytes, - "malloc_arena_after": after.malloc_arena_bytes, - "malloc_free_before": before.malloc_free_bytes, - "malloc_free_after": after.malloc_free_bytes, - "malloc_in_use_before": before.malloc_in_use_bytes, - "malloc_in_use_after": after.malloc_in_use_bytes, - } - - -def allocator_experiment_child_args(argv: Sequence[str]) -> list[str]: - """The child invocation: this run's arguments minus the experiment options. - - ``--verbose`` is dropped too: a child's coordinator logs would precede - the JSON document on the same stdout the parent parses, and the parse - would fail only after the whole storm had run. - """ - child: list[str] = [] - skip_next = False - for token in argv: - if skip_next: - skip_next = False - continue - if token in ("--allocator-experiment", "--allocator-mmap-threshold"): - skip_next = True - continue - if token.startswith("--allocator-experiment=") or token.startswith( - "--allocator-mmap-threshold=" - ): - continue - if token == "--verbose": - continue - child.append(token) - if "--heap-report" not in child: - child.append("--heap-report") - return child - - -def run_allocator_experiment( - settings: Sequence[str], - child_args: Sequence[str], - *, - mmap_threshold: int | None = None, -) -> list[dict[str, Any]]: - """Run this instrument once per ``MALLOC_ARENA_MAX`` setting, in a child each. - - glibc reads its environment once at process start, so every setting - needs its own process; ``default`` runs with the variable unset. The - children inherit everything else, and ``mmap_threshold`` (when given) - pins ``MALLOC_MMAP_THRESHOLD_`` in each of them, which also switches - glibc's dynamic threshold off. - """ - results: list[dict[str, Any]] = [] - for setting in settings: - env = dict(os.environ) - env.pop("MALLOC_ARENA_MAX", None) - if setting != "default": - env["MALLOC_ARENA_MAX"] = str(int(setting)) - if mmap_threshold is not None: - env["MALLOC_MMAP_THRESHOLD_"] = str(int(mmap_threshold)) - completed = subprocess.run( - [sys.executable, str(Path(__file__).resolve()), *child_args], - env=env, - capture_output=True, - text=True, - check=True, - ) - child = json.loads(completed.stdout) - decided = child.get("decided") or {} - results.append( - { - "malloc_arena_max": setting, - "malloc_mmap_threshold": mmap_threshold, - "heap": child["heap"], - "per_row_drain": decided.get("per_row_drain"), - } - ) - return results - - -def _run_storm_phases( - args: argparse.Namespace, - read_heap: Any, - phase_seconds: dict[str, float], -) -> dict[str, Any]: - """Seed, restart, optionally decide/drain, optionally fault; return the report. - - Everything the storm allocates is referenced only from this frame, so a - caller that wants the post-storm resident set reads it after this - returns. ``read_heap`` is called at every phase boundary and - ``phase_seconds`` receives each phase's wall-clock. - """ - rig = CandidateStormRig( - candidates=args.candidates, - queue_depth=args.queue_depth, - quiet=not args.verbose, - ) - phase_started = time.monotonic() - live = rig.seed_live() - phase_seconds["seed_live"] = round(time.monotonic() - phase_started, 6) - read_heap("after_seed_live") - phase_started = time.monotonic() - restarted = rig.restart_and_enumerate() - phase_seconds["restart_enumerate"] = round(time.monotonic() - phase_started, 6) - read_heap("after_restart_enumerate") - report: dict[str, Any] = { - "bounds": { - "live_queue": MAX_PENDING_BLOCK_CANDIDATES, - "replay_page": MAX_BLOCK_REPLAY_ENUMERATION_ROWS, - }, - "input": { - "candidates": args.candidates, - "queue_depth": args.queue_depth, - }, - "live": asdict(live), - "restart": asdict(restarted), - } - if args.decide or args.drain_per_row: - winner = rig.decide_height() - decided = rig.snapshot(rig.restarted_server) - report["decided"] = { - "winner_hash": winner, - "restart": asdict(decided), - } - read_heap("after_decide") - if args.drain_per_row: - phase_started = time.monotonic() - drain = rig.drain_per_row( - rig.restarted_server, - stop_before_hash=winner, - ) - phase_seconds["drain_per_row"] = round(time.monotonic() - phase_started, 6) - read_heap("after_drain") - # The abandoned/submitted hash sets are the programmatic output - # and are storm-sized; only the bounded sets are printed. - report["decided"]["per_row_drain"] = { - "rounds": drain.rounds, - "accounting_tasks": drain.accounting_tasks, - "replay_enumerations": drain.replay_enumerations, - "abandoned_rows": drain.abandoned_rows, - "submitted_rows": drain.submitted_rows, - "pending_rows": drain.pending_rows, - "pending_hashes": sorted(drain.pending_hashes), - "withheld_hashes": sorted(drain.withheld_hashes), - "lease_blocked_hashes": sorted(drain.lease_blocked_hashes), - "deferred_hashes": sorted(drain.deferred_hashes), - "rpc_calls": drain.rpc_calls, - "ledger_attempt_marks": drain.ledger_attempt_marks, - "wall_seconds": drain.wall_seconds, - } - if args.cleanup_fault: - faults: list[str] = [] - for fault in args.cleanup_fault: - if fault == "all": - faults.extend([CLEANUP_FAULT_NONE, *CLEANUP_FAULTS]) - else: - faults.append(fault) - runs: list[dict[str, Any]] = [] - clean: CleanupBacklogReport | None = None - for fault in dict.fromkeys(faults): - # A fresh storm per run: the backlog and its memory are only - # comparable across runs that started from the same state. - fault_rig = CandidateStormRig( - candidates=args.candidates, - queue_depth=args.queue_depth, - quiet=not args.verbose, - credit_shares=args.credit_shares, - backlog_max=args.backlog_max, - ) - measured = fault_rig.measure_cleanup_backlog( - fault, - view=args.cleanup_view, - trace_memory=not args.no_tracemalloc, - ) - if fault == CLEANUP_FAULT_NONE: - clean = measured - entry = asdict(measured) - entry["recovered_cleanly"] = measured.recovered_cleanly - entry["traced_marginal_bytes_per_record"] = ( - cleanup_backlog_marginal_bytes_per_record(measured, clean) - if clean is not None and fault != CLEANUP_FAULT_NONE - else None - ) - entry["traced_owner_marginal_bytes_per_record"] = ( - cleanup_backlog_marginal_bytes_per_record( - measured, - clean, - owner_only=True, - ) - if clean is not None and fault != CLEANUP_FAULT_NONE - else None - ) - runs.append(entry) - report["cleanup_backlog"] = { - "python": sys.version.split()[0], - "platform": sys.platform, - "runs": runs, - } - read_heap("after_cleanup_faults") - return report - - -def main() -> int: - parser = argparse.ArgumentParser(description=__doc__) - parser.add_argument( - "--candidates", - type=int, - default=OBSERVED_TESTNET_CANDIDATE_STORM, - ) - parser.add_argument( - "--queue-depth", - type=int, - default=MAX_PENDING_BLOCK_CANDIDATES, - ) - parser.add_argument( - "--verbose", - action="store_true", - help="show coordinator coalescing and replay logs before the JSON report", - ) - parser.add_argument( - "--decide", - action="store_true", - help="after restart enumeration, make the last candidate the active " - "block at the storm height and report ownership/evidence gauges", - ) - parser.add_argument( - "--drain-per-row", - action="store_true", - help="with --decide, also drive the shipped per-row disposition path " - "over every sibling on the restarted coordinator and report its cost", - ) - parser.add_argument( - "--cleanup-fault", - action="append", - choices=(CLEANUP_FAULT_NONE, "all", *CLEANUP_FAULTS), - help="issue #198: run the shipped collapse walk under this injected " - "cleanup fault on a fresh storm, measure the retry backlog, and prove " - "its recovery; repeatable, 'all' runs the clean baseline and every fault", - ) - parser.add_argument( - "--cleanup-view", - choices=("live", "restart"), - default="restart", - help="coordinator view the cleanup-fault runs use (default: restart)", - ) - parser.add_argument( - "--backlog-max", - type=int, - default=None, - help="pin the cleanup-retry admission bound for the cleanup-fault runs " - f"(default: the shipped {DEFAULT_BLOCK_CANDIDATE_CLEANUP_RETRY_BACKLOG_MAX})", - ) - parser.add_argument( - "--credit-shares", - action="store_true", - help="seed credit-bearing candidates so the cleanup-fault runs exercise " - "real pending-share floor authority", - ) - parser.add_argument( - "--no-tracemalloc", - action="store_true", - help="skip the tracemalloc reading in the cleanup-fault runs", - ) - parser.add_argument( - "--heap-report", - action="store_true", - help="issue #226: report resident set, glibc arena, and anonymous-map " - "readings at every phase, after the storm is released and collected, " - "and after one malloc_trim (host-specific numbers; the platform is " - "reported beside them)", - ) - parser.add_argument( - "--arena-probe-threads", - type=int, - default=0, - help="with --heap-report, run this many allocating threads after the " - "storm to reproduce the per-thread arena shape (0 skips the probe)", - ) - parser.add_argument( - "--arena-probe-rounds", - type=int, - default=2_000, - help="medium-buffer allocations per arena-probe thread (default 2000)", - ) - parser.add_argument( - "--allocator-experiment", - default=None, - help="comma-separated MALLOC_ARENA_MAX settings ('default', '2', '4'); " - "runs this instrument once per setting in a child process with " - "--heap-report and reports every child's heap section side by side", - ) - parser.add_argument( - "--allocator-mmap-threshold", - type=int, - default=None, - help="with --allocator-experiment, also pin MALLOC_MMAP_THRESHOLD_ in " - "every child (which switches glibc's dynamic threshold off)", - ) - args = parser.parse_args() - if args.allocator_experiment: - settings = [ - token.strip() - for token in args.allocator_experiment.split(",") - if token.strip() - ] - runs = run_allocator_experiment( - settings, - allocator_experiment_child_args(sys.argv[1:]), - mmap_threshold=args.allocator_mmap_threshold, - ) - print( - json.dumps( - { - "allocator_experiment": { - "python": sys.version.split()[0], - "platform": sys.platform, - "libc": " ".join(platform.libc_ver()), - "machine": platform.machine(), - "cpu_count": os.cpu_count(), - "runs": runs, - } - }, - indent=2, - sort_keys=True, - ) - ) - return 0 - telemetry = ProcessHeapTelemetry() if args.heap_report else None - readings: list[HeapReading] = [] - phase_seconds: dict[str, float] = {} - instrument_started = time.monotonic() - - def read_heap(phase: str) -> None: - if telemetry is not None: - readings.append(take_heap_reading(phase, telemetry, instrument_started)) - - read_heap("start") - # The storm runs in its own frame so that, when it returns, nothing in - # this function still references the rig, the drain, or a fault run. - report = _run_storm_phases(args, read_heap, phase_seconds) - if telemetry is not None: - probe: dict[str, Any] | None = None - if args.arena_probe_threads > 0: - probe = run_arena_probe( - args.arena_probe_threads, - args.arena_probe_rounds, - telemetry=telemetry, - ) - read_heap("after_arena_probe") - # #185's question: once the storm's own objects are gone, does the - # process return to its pre-storm size? Every reference to the storm - # died with _run_storm_phases' frame; collect, and read. The gap - # between "after_release_gc" and "start" that survives is what the - # interpreter still holds (allocated_blocks) plus what glibc has not - # returned (malloc_free_bytes and the resident set); the trim that - # follows shows how much of the latter one malloc_trim recovers. - gc.collect() - read_heap("after_release_gc") - trim = MallocTrimmer(telemetry, log=lambda _line: None).trim_once("storm") - read_heap("after_malloc_trim") - report["heap"] = { - "python": sys.version.split()[0], - "platform": sys.platform, - "libc": " ".join(platform.libc_ver()), - "machine": platform.machine(), - "cpu_count": os.cpu_count(), - "allocator_env": allocator_environment(), - "phase_seconds": phase_seconds, - "readings": [asdict(reading) for reading in readings], - "arena_probe": probe, - "malloc_trim": trim.as_dict() if trim is not None else None, - } - print(json.dumps(report, indent=2, sort_keys=True)) - return 0 - - -if __name__ == "__main__": - raise SystemExit(main()) diff --git a/tests/prism_concurrency_harness.py b/tests/prism_concurrency_harness.py deleted file mode 100644 index f5dc91e8d..000000000 --- a/tests/prism_concurrency_harness.py +++ /dev/null @@ -1,4175 +0,0 @@ -#!/usr/bin/env python3 -"""Deterministic concurrency harness for PRISM's stateful owners. - -Why this exists ---------------- -The concurrency defects this line keeps shipping (tip-refresh livelock, -publication starvation, lease heartbeat self-fencing, the found-block landing -livelock, the orphaned lease-tuple lock in #123) share a shape: two owners -interleave around a PostgreSQL row, and the bad schedule is rare enough that -running the coordinator and hoping never produces it. Reviews caught them; -tests could not express them. - -This module makes the schedule an input rather than an accident. Four pieces: - -``DeterministicScheduler`` - Real threads, one baton. Exactly one actor runs at a time, and it only - stops at named checkpoints the harness places at PostgreSQL statement - boundaries. The controlling test decides who runs next, so an - interleaving is *reproducible and order-stable*, not merely likely. The - scheduler records a trace of every checkpoint, which is what determinism - is asserted against. - -``VirtualClock`` - All time the lease lifecycle measures about itself — adoption silence - from advisory-guard acquisition, caller deadlines, TTL expiry, retry - sleeps — reads this clock. Nothing sleeps in wall-clock time. Tests - advance it explicitly, or call ``advance_to_next_deadline`` to jump to - the earliest pending timeout. - -``FakePostgres`` - An in-memory model of the tables PRISM's owners contend on, with the - PostgreSQL semantics those owners actually depend on: exclusive tuple - locks held for the lifetime of a transaction, ``FOR NO KEY UPDATE SKIP - LOCKED`` declining to queue, per-statement READ COMMITTED snapshots, - ``lock_timeout``/``statement_timeout``, session-scoped advisory locks, - and ``pg_stat_activity`` locker attribution against a snapshot-visible - ``xmax``. Explicit transactions are modelled as BEGIN / statement / - COMMIT with a checkpoint in between, because that gap is the whole of - #123. - -``HarnessLock`` / ``HarnessRLock`` / ``HarnessCondition`` - Process-local synchronisation the scheduler can see. A contended acquire - parks at a named checkpoint instead of wedging the baton, and a timed - wait measures the virtual clock. The lease scenarios never needed these: - they interleave two *processes* around one row, so no two actors ever - contended a lock inside one coordinator. The landing topology does — - the payout-balance serializer, the publication order guard and the - per-hash disposition lease are exactly what its two tails share. - -Substitution goes through ports, not monkeypatching ---------------------------------------------------- -``PsqlShareLedger`` takes ``sql_backend_factory``, ``lease_guard_factory``, -``monotonic`` and ``lease_retry_sleep``. The fakes here implement -``LedgerSqlPort`` and ``LeaseGuardPort``. No bound method is reassigned and -no ``time`` module is patched, so the code under test is the shipped code: -the real retry loop, the real CAS SQL, the real adoption arithmetic, the real -fail-closed branches. - -The synchronisation primitives are the one deliberate exception, and they are -a different kind of substitution: a harness lock preserves the semantics of -the lock it replaces exactly and changes only how a waiter blocks, so the -decision logic under test is untouched. Landing scenarios install them on the -coordinator's lock attributes at construction, the same way the coordinator -itself installs a ``threading.RLock`` there. - -What is modelled and what is not --------------------------------- -Modelled: the writer-lease statements ``PsqlShareLedger`` emits — startup -acquisition, same-identity adoption, renewal, release, and the guard-session -verification probe — plus the landing statements it emits against -``qbit_pool_blocks`` and ``qbit_block_candidate_outbox``: prepared-row -persistence, the ``qbit_confirm_pool_block`` publication-ordinal allocator -and its read-back, the durable ordinal floor, pool-block state, prepared -rejection and the outbox terminal update. - -Not modelled: everything else. ``FakePostgres`` classifies each statement and -raises ``UnsupportedStatement`` on anything it does not recognise, including -the share-ledger statements, which belong to the follow-up state machines -named in issue #128. That is deliberate: a fake that silently answers an -unrecognised statement drifts away from production without anyone noticing, -so this one fails loudly instead. The classifier is anchored on fragments of -the production SQL, so a change to that SQL breaks classification rather than -quietly changing meaning. - -Where a modelled statement is a large CTE — ``persist_accepted_block`` writes -pool-block, bundle, payout-entry and carry-forward rows in one statement — -the model reproduces the effects the landing decisions read back, not the -whole statement. The classifier still pins the statement's identity on its -production fragments, so the day persist grows an effect a landing consults, -that fragment set is where the model is extended. -""" - -from __future__ import annotations - -import base64 -import contextlib -import hashlib -import io -import json -import os -import re -import sys -import threading -import uuid -from collections.abc import Callable -from dataclasses import dataclass, field, replace -from datetime import datetime, timedelta, timezone -from enum import Enum -from pathlib import Path -from typing import TYPE_CHECKING, Any - -if TYPE_CHECKING: - from typing import Self - -sys.path.insert(0, str(Path(__file__).resolve().parents[1])) - -from lab.prism.candidate_codec import ( # noqa: E402 - INDEX_PAGE_ENTRIES, - INDEX_SPAN_ROWS, - replay_header_from_fields, -) -from lab.prism.candidate_store import BODY_READ_MAX_CHUNKS # noqa: E402 -from lab.prism.share_ledger import ( # noqa: E402 - PAYOUT_WINDOW_ROW_BATCH_SIZE, - WRITER_LEASE_HEARTBEAT_SESSION_PREFIX, - LedgerOperationTimeout, - PsqlShareLedger, - parse_single_json_value, -) -from lab.prism.writer_lease_timing import ( # noqa: E402 - WRITER_LEASE_GUARD_STATEMENT_TIMEOUT_SECONDS, -) - -# Real seconds the controller will wait for an actor to hand the baton back -# before declaring the harness itself broken. It is a failure detector, not -# an ordering input: no schedule depends on it, so it cannot change what a -# run does. An actor that blocks on a resource the harness does not model (a -# real threading.Lock, a real socket) trips it instead of hanging the suite. -# It is not free of consequence — an actor that genuinely took this long to -# reach its next checkpoint would fail a run that ought to pass — but the -# margin against microseconds of work is enormous. The env override exists -# for heavily oversubscribed CI hosts where 20s of scheduler stall is a real -# false red; it widens the failure detector only and never affects ordering. -BATON_TIMEOUT_SECONDS = float( - os.environ.get("PRISM_HARNESS_BATON_TIMEOUT_SECONDS", "20.0") -) - -# Virtual epoch origin for clock_timestamp(). Fixed so timestamp text is -# byte-identical across runs, which matters: the adoption CAS compares -# updated_at as text. -CLOCK_ORIGIN = datetime(2026, 6, 26, 19, 49, 22, 233718, tzinfo=timezone.utc) - -# The guard session's server-side statement_timeout, taken from the shipped -# timing policy rather than repeated, so a change to the bound the heartbeat -# envelope is sized against also changes what this model enforces. -GUARD_STATEMENT_TIMEOUT_SECONDS = WRITER_LEASE_GUARD_STATEMENT_TIMEOUT_SECONDS - - -class HarnessError(AssertionError): - """The harness could not model what the code under test asked for.""" - - -class UnsupportedStatement(HarnessError): - """FakePostgres was handed SQL it does not model.""" - - -class SchedulerStall(HarnessError): - """An actor did not reach a checkpoint within the baton timeout.""" - - -class NotRunnable(HarnessError): - """The controller tried to step an actor that cannot make progress.""" - - -class _ActorAbort(BaseException): - """Unwinds a parked actor thread at harness teardown.""" - - -# -------------------------------------------------------------------------- -# Virtual clock -# -------------------------------------------------------------------------- - - -class VirtualClock: - """Monotonic and wall clock that only moves when a test says so.""" - - def __init__(self, *, monotonic_origin: float = 10_000.0) -> None: - self._monotonic = float(monotonic_origin) - self._origin_monotonic = float(monotonic_origin) - - def monotonic(self) -> float: - return self._monotonic - - def now(self) -> datetime: - """``clock_timestamp()``: wall time, advancing with the monotonic clock.""" - return CLOCK_ORIGIN + timedelta( - seconds=self._monotonic - self._origin_monotonic - ) - - def advance(self, seconds: float) -> float: - seconds = float(seconds) - if seconds < 0: - raise HarnessError("virtual time cannot move backwards") - self._monotonic += seconds - return self._monotonic - - def advance_to(self, monotonic_target: float) -> float: - if monotonic_target < self._monotonic: - raise HarnessError("virtual time cannot move backwards") - self._monotonic = float(monotonic_target) - return self._monotonic - - @staticmethod - def timestamp_text(moment: datetime) -> str: - """Render a timestamptz the way psql/psycopg hand it back as text.""" - return f"{moment:%Y-%m-%d %H:%M:%S}.{moment.microsecond:06d}+00" - - -# -------------------------------------------------------------------------- -# Scheduler -# -------------------------------------------------------------------------- - - -@dataclass -class Call: - """One unit of work handed to an actor.""" - - label: str - fn: Callable[[], Any] - done: bool = False - result: Any = None - error: BaseException | None = None - - def value(self) -> Any: - """Return the result, re-raising whatever the actor raised.""" - if not self.done: - raise HarnessError(f"call {self.label!r} has not finished") - if self.error is not None: - raise self.error - return self.result - - -class Actor: - """One logical thread of the system under test. - - An actor owns a work queue. It parks at the ``idle`` checkpoint between - items, so a scenario reads as a sequence of ``call``/``start`` steps - rather than as thread plumbing. - """ - - def __init__(self, scheduler: DeterministicScheduler, name: str) -> None: - self.scheduler = scheduler - self.name = name - self.queue: list[Call] = [] - self.calls: list[Call] = [] - self.stop: str | None = None - self.finished = False - self.parked_forever = False - self._ready: Callable[[], bool] | None = None - self._wake_at: float | None = None - self._blocked = False - self._thread = threading.Thread( - target=self._run, - name=f"harness-{name}", - daemon=True, - ) - self._started = False - - # -- controller side --------------------------------------------------- - - def submit(self, fn: Callable[[], Any], *, label: str | None = None) -> Call: - call = Call(label=label or getattr(fn, "__name__", "call"), fn=fn) - self.queue.append(call) - self.calls.append(call) - return call - - @property - def blocked(self) -> bool: - return self._blocked - - @property - def block_reason(self) -> str | None: - return self.stop if self._blocked else None - - @property - def wake_at(self) -> float | None: - return self._wake_at if self._blocked else None - - def runnable(self) -> bool: - if self.finished or self.parked_forever: - return False - if not self._blocked: - return True - if self._ready is not None and self._ready(): - return True - return ( - self._wake_at is not None - and self.scheduler.clock.monotonic() >= self._wake_at - ) - - # -- actor side -------------------------------------------------------- - - def _run(self) -> None: - try: - self.scheduler._await_baton(self) - while True: - while not self.queue: - # Blocked rather than checkpointed: an actor with nothing - # to do is not runnable, so it neither consumes a drain - # round nor masks the fact that the system has quiesced. - self.scheduler.block("idle", ready=lambda: bool(self.queue)) - call = self.queue.pop(0) - try: - call.result = call.fn() - except BaseException as exc: # recorded, not swallowed - if isinstance(exc, _ActorAbort): - raise - call.error = exc - finally: - call.done = True - self.scheduler.checkpoint(f"done:{call.label}") - except _ActorAbort: - pass - finally: - self.finished = True - self.stop = "finished" - self.scheduler._release_baton(self) - - -class DeterministicScheduler: - """Baton-passing scheduler: one runnable actor, chosen by the test.""" - - def __init__( - self, - clock: VirtualClock, - *, - baton_timeout_seconds: float = BATON_TIMEOUT_SECONDS, - ) -> None: - self.clock = clock - self.baton_timeout_seconds = float(baton_timeout_seconds) - self.actors: dict[str, Actor] = {} - self.trace: list[str] = [] - self._condition = threading.Condition() - self._current: Actor | None = None - self._local = threading.local() - self._closed = False - - # -- construction ------------------------------------------------------ - - def actor(self, name: str) -> Actor: - if name in self.actors: - raise HarnessError(f"actor {name!r} already exists") - actor = Actor(self, name) - self.actors[name] = actor - return actor - - # -- actor-side primitives -------------------------------------------- - - def current(self) -> Actor: - actor = getattr(self._local, "actor", None) - if actor is None: - raise HarnessError( - "this call must run on an actor thread; drive it through " - "Actor.submit/LeaseHarness.run rather than from the test body" - ) - return actor - - def checkpoint(self, label: str) -> None: - """Announce a stop and hand the baton back to the controller.""" - actor = self.current() - actor.stop = label - actor._blocked = False - actor._ready = None - actor._wake_at = None - self.trace.append(f"{actor.name}@{label}") - self._yield(actor) - - def block( - self, - label: str, - *, - ready: Callable[[], bool] | None = None, - wake_at: float | None = None, - ) -> None: - """Park at ``label`` until ``ready()`` holds or the clock reaches ``wake_at``. - - Returns once the controller schedules this actor again; the caller - re-checks its own condition, so a spurious resume is harmless. - """ - actor = self.current() - actor.stop = label - actor._blocked = True - actor._ready = ready - actor._wake_at = wake_at - self.trace.append(f"{actor.name}@{label}") - self._yield(actor) - actor._blocked = False - actor._ready = None - actor._wake_at = None - - def park_forever(self, label: str) -> None: - """Abandon this actor at ``label``; it never runs again. - - Models a client that vanished: the process is gone, but nothing it - held server-side has been released. - """ - actor = self.current() - actor.stop = label - actor.parked_forever = True - actor._blocked = True - actor._ready = lambda: False - actor._wake_at = None - self.trace.append(f"{actor.name}@{label}") - self._yield(actor) - raise HarnessError("a parked actor must never be resumed") - - # -- baton mechanics --------------------------------------------------- - - def _yield(self, actor: Actor) -> None: - with self._condition: - self._current = None - self._condition.notify_all() - while self._current is not actor: - if self._closed: - raise _ActorAbort() - self._condition.wait(0.05) - - def _await_baton(self, actor: Actor) -> None: - self._local.actor = actor - with self._condition: - while self._current is not actor: - if self._closed: - raise _ActorAbort() - self._condition.wait(0.05) - - def _release_baton(self, actor: Actor) -> None: - with self._condition: - if self._current is actor: - self._current = None - self._condition.notify_all() - - # -- controller-side stepping ----------------------------------------- - - def step(self, actor: Actor) -> str: - """Run ``actor`` until its next checkpoint. Returns the stop label.""" - if actor.finished: - raise NotRunnable(f"actor {actor.name!r} has finished") - if not actor.runnable(): - raise NotRunnable( - f"actor {actor.name!r} is blocked at {actor.stop!r} and cannot " - "make progress; advance the clock or let another actor commit" - ) - if not actor._started: - actor._started = True - actor._thread.start() - with self._condition: - self._current = actor - self._condition.notify_all() - waited = self._condition.wait_for( - lambda: self._current is not actor, - timeout=self.baton_timeout_seconds, - ) - if not waited: - raise SchedulerStall( - f"actor {actor.name!r} did not reach a checkpoint within " - f"{self.baton_timeout_seconds:g}s (last stop {actor.stop!r}). It is " - "probably blocked on a resource the harness does not model." - ) - return actor.stop or "finished" - - def run_until(self, actor: Actor, label: str, *, limit: int = 200) -> str: - """Step ``actor`` until it stops at ``label``.""" - for _ in range(limit): - stop = self.step(actor) - if stop == label: - return stop - raise HarnessError( - f"actor {actor.name!r} did not reach {label!r} within {limit} steps; " - f"last stop {actor.stop!r}" - ) - - def runnable_actors(self) -> list[Actor]: - return [actor for actor in self.actors.values() if actor.runnable()] - - def next_deadline(self) -> float | None: - """Earliest virtual monotonic time at which a blocked actor wakes.""" - deadlines = [ - actor.wake_at - for actor in self.actors.values() - if actor.blocked - and not actor.parked_forever - and actor.wake_at is not None - ] - return min(deadlines) if deadlines else None - - def close(self) -> None: - with self._condition: - self._closed = True - self._current = None - self._condition.notify_all() - for actor in self.actors.values(): - if actor._started: - actor._thread.join(timeout=2.0) - - -# -------------------------------------------------------------------------- -# Scheduler-aware synchronisation -# -------------------------------------------------------------------------- - - -class HarnessLock: - """A mutex the scheduler can see, drop-in for ``threading.Lock``. - - The code under test acquires and releases exactly as it would a real - lock; the only difference is what happens when the lock is already held. - A real lock would block the actor thread outside the baton, so the - controller would time out and report a stall on a wait that is in fact - legitimate. This one parks the waiter at ``lock:<name>`` and lets the - controller schedule the holder, which is what makes a contended - process-local lock an *interleaving point* rather than a harness limit. - - Ownership is per actor, not per OS thread: actors are threads here, but - naming the actor is what lets the trace read as a schedule. - - Grants are FIFO. PostgreSQL's tuple-lock queue is modelled that way for - the same reason (see ``_await_lease_lock``): without a queue the model - could express a grant order that neither a real futex-backed lock nor - the scheduler would produce, and the schedule is an input here. - """ - - reentrant = False - - def __init__(self, scheduler: DeterministicScheduler, name: str) -> None: - self.scheduler = scheduler - self.name = name - self._owner: Actor | None = None - self._depth = 0 - self._waiters: list[Actor] = [] - - # -- observation ------------------------------------------------------- - - @property - def owner(self) -> Actor | None: - return self._owner - - def locked(self) -> bool: - return self._owner is not None - - def held_by_current(self) -> bool: - return self._owner is self.scheduler.current() - - # -- acquisition ------------------------------------------------------- - - def acquire(self, blocking: bool = True, timeout: float = -1) -> bool: - actor = self.scheduler.current() - if self._owner is actor: - if not self.reentrant: - raise HarnessError( - f"actor {actor.name!r} re-acquired non-reentrant lock " - f"{self.name!r}; production would deadlock here" - ) - self._depth += 1 - return True - deadline = ( - None - if timeout is None or timeout < 0 - else self.scheduler.clock.monotonic() + max(0.0, float(timeout)) - ) - - def granted() -> bool: - if self._owner is not None: - return False - return not self._waiters or self._waiters[0] is actor - - if granted(): - self._take(actor) - return True - if not blocking: - return False - self._waiters.append(actor) - try: - while not granted(): - if ( - deadline is not None - and self.scheduler.clock.monotonic() >= deadline - ): - return False - self.scheduler.block( - f"lock:{self.name}", - ready=granted, - wake_at=deadline, - ) - finally: - self._waiters.remove(actor) - self._take(actor) - return True - - def _take(self, actor: Actor) -> None: - self._owner = actor - self._depth = 1 - - def release(self) -> None: - actor = self.scheduler.current() - if self._owner is None: - raise HarnessError(f"lock {self.name!r} released while unheld") - if self._owner is not actor: - raise HarnessError( - f"lock {self.name!r} released by {actor.name!r} but held by " - f"{self._owner.name!r}" - ) - self._depth -= 1 - if self._depth == 0: - self._owner = None - - def __enter__(self) -> bool: - return self.acquire() - - def __exit__(self, *exc_info: object) -> None: - self.release() - - -class HarnessRLock(HarnessLock): - """``threading.RLock`` for actors: the owner may re-enter.""" - - reentrant = True - - -class HarnessSemaphore: - """``threading.BoundedSemaphore`` for actors. - - The ledger's read slots are a semaphore, and ``_operation_gate`` treats - it exactly like the writer lock — same ``acquire(timeout=)`` contract, - same ``LedgerOperationTimeout`` on expiry. Modelling it means a scenario - can exhaust read concurrency on purpose rather than discovering that the - harness quietly never blocks there. - """ - - def __init__( - self, - scheduler: DeterministicScheduler, - name: str, - *, - value: int = 1, - ) -> None: - self.scheduler = scheduler - self.name = name - self.initial = int(value) - self._value = int(value) - self._waiters: list[Actor] = [] - - @property - def value(self) -> int: - return self._value - - def acquire(self, blocking: bool = True, timeout: float = -1) -> bool: - actor = self.scheduler.current() - deadline = ( - None - if timeout is None or timeout < 0 - else self.scheduler.clock.monotonic() + max(0.0, float(timeout)) - ) - - def granted() -> bool: - if self._value <= 0: - return False - return not self._waiters or self._waiters[0] is actor - - if granted(): - self._value -= 1 - return True - if not blocking: - return False - self._waiters.append(actor) - try: - while not granted(): - if ( - deadline is not None - and self.scheduler.clock.monotonic() >= deadline - ): - return False - self.scheduler.block( - f"semaphore:{self.name}", - ready=granted, - wake_at=deadline, - ) - finally: - self._waiters.remove(actor) - self._value -= 1 - return True - - def release(self) -> None: - if self._value >= self.initial: - raise HarnessError(f"semaphore {self.name!r} released too many times") - self._value += 1 - - def __enter__(self) -> bool: - return self.acquire() - - def __exit__(self, *exc_info: object) -> None: - self.release() - - -class HarnessCondition: - """``threading.Condition`` over a harness lock and the virtual clock. - - ``wait`` releases the lock, parks, and re-acquires before returning, so a - predicate loop written against a real condition behaves identically. A - timed wait measures the virtual clock, which is what lets a scenario put - a heartbeat-slice wait (``_await_unfenced_appends_predating_anchor``, the - ledger admission slices) under a watchdog without anything sleeping. - """ - - def __init__( - self, - lock: HarnessLock | None = None, - *, - scheduler: DeterministicScheduler | None = None, - name: str = "condition", - ) -> None: - if lock is None: - if scheduler is None: - raise HarnessError("a condition needs a lock or a scheduler") - lock = HarnessRLock(scheduler, name) - self._lock = lock - self.scheduler = lock.scheduler - self.name = name - # Monotonic notify counter. A waiter records the value it parked on, - # so a notify that arrives while the controller has the baton is - # never lost the way a missed edge would be. - self._generation = 0 - - def __enter__(self) -> bool: - return self._lock.acquire() - - def __exit__(self, *exc_info: object) -> None: - self._lock.release() - - def acquire(self, *args: Any, **kwargs: Any) -> bool: - return self._lock.acquire(*args, **kwargs) - - def release(self) -> None: - self._lock.release() - - def wait(self, timeout: float | None = None) -> bool: - if not self._lock.held_by_current(): - raise HarnessError( - f"cannot wait on condition {self.name!r} without holding its lock" - ) - parked_generation = self._generation - depth = self._lock._depth - self._lock._depth = 1 - self._lock.release() - clock = self.scheduler.clock - wake_at = ( - None if timeout is None else clock.monotonic() + max(0.0, float(timeout)) - ) - notified = False - while True: - if self._generation != parked_generation: - notified = True - break - if wake_at is not None and clock.monotonic() >= wake_at: - break - self.scheduler.block( - f"cond:{self.name}", - ready=lambda: self._generation != parked_generation, - wake_at=wake_at, - ) - self._lock.acquire() - self._lock._depth = depth - return notified - - def wait_for( - self, - predicate: Callable[[], bool], - timeout: float | None = None, - ) -> bool: - clock = self.scheduler.clock - deadline = ( - None if timeout is None else clock.monotonic() + max(0.0, float(timeout)) - ) - while not predicate(): - if deadline is None: - self.wait() - continue - remaining = deadline - clock.monotonic() - if remaining <= 0: - break - self.wait(remaining) - return predicate() - - def notify(self, n: int = 1) -> None: - self.notify_all() - - def notify_all(self) -> None: - self._generation += 1 - - -class HarnessEvent: - """``threading.Event`` for actors, with a clock-backed ``wait``.""" - - def __init__(self, scheduler: DeterministicScheduler, name: str = "event") -> None: - self.scheduler = scheduler - self.name = name - self._set = False - - def is_set(self) -> bool: - return self._set - - def set(self) -> None: - self._set = True - - def clear(self) -> None: - self._set = False - - def wait(self, timeout: float | None = None) -> bool: - clock = self.scheduler.clock - wake_at = ( - None if timeout is None else clock.monotonic() + max(0.0, float(timeout)) - ) - while not self._set: - if wake_at is not None and clock.monotonic() >= wake_at: - break - self.scheduler.block( - f"event:{self.name}", - ready=lambda: self._set, - wake_at=wake_at, - ) - return self._set - - -# -------------------------------------------------------------------------- -# Statement classification -# -------------------------------------------------------------------------- - - -class LeaseOp(str, Enum): - ACQUIRE = "acquire" - ADOPT = "adopt" - RENEW = "renew" - RELEASE = "release" - VERIFY = "verify" - # The cheap read-only ownership proof the heartbeat runs on most beats - # (issue #212). Deliberately a distinct trace step from VERIFY: a - # scenario asserting the split has to be able to see which question the - # coordinator actually asked PostgreSQL on each beat. - PROVE = "prove" - - -class LandingOp(str, Enum): - """Statements the block-candidate landing tails emit. - - Names follow the ``PsqlShareLedger`` method that emits them rather than - the SQL verb, because a scenario reads the trace as landing steps. - """ - - PERSIST_BLOCK = "persist_block" - CONFIRM_BLOCK = "confirm_block" - CONFIRMED_SEQUENCE = "confirmed_sequence" - PUBLICATION_FLOOR = "publication_floor" - POOL_BLOCK_STATE = "pool_block_state" - REJECT_PREPARED = "reject_prepared" - OUTBOX_RECORD = "outbox_record" - OUTBOX_ATTEMPT = "outbox_attempt" - OUTBOX_FINISH = "outbox_finish" - OUTBOX_PENDING_PAGE = "outbox_pending_page" - OUTBOX_BATCH_ABANDON = "outbox_batch_abandon" - # Issue #255: chunked candidate bodies. Reads first, then the staging - # writes (no lease CTE), then the fenced orphan retirement and the - # lease-free chunk reaping. - OUTBOX_OBSERVE = "outbox_observe" - OUTBOX_HEADER_PAGE = "outbox_header_page" - BODY_MANIFEST = "body_manifest" - BODY_SPANS = "body_spans" - BODY_PAGES = "body_pages" - BODY_PAGE = "body_page" - BODY_STAGE = "body_stage" - BODY_CHUNK = "body_chunk" - BODY_INDEX = "body_index" - BODY_SEAL = "body_seal" - BODY_RETIRE_STAGING = "body_retire_staging" - BODY_RETIRE_ORPHANS = "body_retire_orphans" - BODY_REAP = "body_reap" - SCHEMA_PRESENCE = "schema_presence" - SCHEMA_CAPABILITY = "schema_capability" - ALL_SHARES = "all_shares" - PRIOR_BALANCES = "prior_balances" - PRIOR_BALANCES_AS_OF = "prior_balances_as_of" - SHARE_STATS = "share_stats" - - -StatementKind = LeaseOp | LandingOp - - -@dataclass(frozen=True) -class Statement: - kind: StatementKind - payload: dict[str, Any] - lease_ttl_seconds: float | None - authority_margin_seconds: float | None - advisory_lock: tuple[int, int] | None - sql: str - - @property - def is_landing(self) -> bool: - return isinstance(self.kind, LandingOp) - - -_PAYLOAD_RE = re.compile( - r"\$(qbit_prism_json(?:_x)*)\$(.*?)\$\1\$::jsonb", - re.DOTALL, -) -_INTERVAL_RE = re.compile(r"make_interval\(secs => ([0-9eE\.\+\-]+)\)") -_CLASSID_RE = re.compile(r"classid = (\d+)::oid") -_OBJID_RE = re.compile(r"objid = (\d+)::oid") - -# Each entry is (op, required fragments). Fragments are lifted verbatim from -# the production SQL in lab/prism/share_ledger.py, so a change there breaks -# classification loudly instead of silently changing what the fake models. -_SIGNATURES: tuple[tuple[LeaseOp, tuple[str, ...]], ...] = ( - ( - LeaseOp.PROVE, - ( - "'guard_advisory_lock_held'", - "'writer_session_token_current'", - "'lease_renewal_due'", - ), - ), - ( - LeaseOp.VERIFY, - ( - "FOR NO KEY UPDATE SKIP LOCKED", - "'guard_advisory_lock_held'", - "'lease_renewed_count'", - "'lease_locked_by_this_process'", - ), - ), - ( - LeaseOp.ACQUIRE, - ( - "INSERT INTO qbit_ledger_writer_lease", - "ON CONFLICT (singleton) DO UPDATE", - "qbit_ledger_writer_lease.lease_expires_at <= clock_timestamp()", - ), - ), - ( - LeaseOp.ADOPT, - ( - "UPDATE qbit_ledger_writer_lease", - "data->>'observed_writer_session_token'", - "data->>'observed_lease_updated_at'", - ), - ), - ( - LeaseOp.RELEASE, - ( - "UPDATE qbit_ledger_writer_lease", - "lease_expires_at = clock_timestamp() - interval '1 second'", - "json_build_object('released'", - ), - ), - ( - LeaseOp.RENEW, - ( - "UPDATE qbit_ledger_writer_lease", - "'renewed_count', (SELECT count(*) FROM lease)", - ), - ), -) - -# The landing statements, same discipline: every fragment is lifted verbatim -# from lab/prism/share_ledger.py. Ordered most specific first, because two -# pool-block reads differ only by which columns they project. -_LANDING_SIGNATURES: tuple[tuple[LandingOp, tuple[str, ...]], ...] = ( - ( - LandingOp.PERSIST_BLOCK, - ( - "INSERT INTO qbit_pool_blocks (", - "existing_block AS (", - "INSERT INTO qbit_pool_audit_bundles (", - ), - ), - ( - LandingOp.CONFIRM_BLOCK, - ("'confirmed_count', qbit_confirm_pool_block(",), - ), - ( - LandingOp.REJECT_PREPARED, - ("'rejected_count', qbit_reject_prepared_pool_block(",), - ), - ( - LandingOp.POOL_BLOCK_STATE, - ( - "'state', (", - "'maturity_state', maturity_state,", - "FROM qbit_pool_blocks", - ), - ), - ( - LandingOp.CONFIRMED_SEQUENCE, - ( - "'audit_publication_sequence', (", - "FROM qbit_pool_blocks", - "AND chain_state = 'confirmed'", - ), - ), - ( - LandingOp.PUBLICATION_FLOOR, - ( - "'audit_publication_sequence_floor',", - "COALESCE(MAX(audit_publication_sequence), 0)", - ), - ), - ( - # Issue #255: the metadata-first page. Listed before the legacy page - # because it carries every fragment of that page plus its own. - LandingOp.OUTBOX_HEADER_PAGE, - ("'header_bytes', octet_length(page.header::text)",), - ), - ( - LandingOp.OUTBOX_PENDING_PAGE, - ( - # The read-only replay enumeration (issue #211). Its fragments - # name the three facts one page must answer from one snapshot: - # the row, whether the candidate already landed, and the keyset - # cursor the next page resumes from. - "'pool_block_exists', EXISTS (", - "FROM qbit_pool_blocks pool", - "FROM qbit_block_candidate_outbox", - "WHERE state = 'pending'", - ), - ), - (LandingOp.OUTBOX_OBSERVE, ("'body_state', (",)), - (LandingOp.SCHEMA_PRESENCE, ("to_regclass('qbit_block_candidate_body')",)), - (LandingOp.SCHEMA_CAPABILITY, ("FROM qbit_prism_schema_capabilities",)), - (LandingOp.BODY_STAGE, ("INSERT INTO qbit_block_candidate_body (",)), - (LandingOp.BODY_CHUNK, ("INSERT INTO qbit_block_candidate_body_chunk (",)), - (LandingOp.BODY_INDEX, ("INSERT INTO qbit_block_candidate_body_span (",)), - (LandingOp.BODY_SEAL, ("SET state = 'sealed', sealed_at = clock_timestamp()",)), - ( - LandingOp.BODY_RETIRE_STAGING, - ("SET state = 'retired', retired_at = clock_timestamp()", "AND state = 'staging'"), - ), - ( - LandingOp.BODY_RETIRE_ORPHANS, - ("SET state = 'retired', retired_at = clock_timestamp()", "FOR UPDATE OF body"), - ), - (LandingOp.BODY_REAP, ("deleted_chunks AS (",)), - (LandingOp.BODY_MANIFEST, ("'manifest', (", "'referenced', EXISTS (")), - (LandingOp.BODY_SPANS, ("'spans', (",)), - (LandingOp.BODY_PAGES, ("'pages', (",)), - (LandingOp.BODY_PAGE, ("encode(page.chunk, 'base64')",)), - ( - LandingOp.OUTBOX_RECORD, - ( - "INSERT INTO qbit_block_candidate_outbox (", - "'block candidate payload mismatch'", - # Both fragments above also occur in the share-submission append, - # which opens the same outbox insert and raises the same mismatch - # error, so on their own they name a share append as the outbox - # state machine. Only `persist_block_candidate_intent` inserts the - # candidate with no share to attach it to and therefore leaves a - # colliding row alone; the share append claims that row with - # `ON CONFLICT (block_hash) DO UPDATE`. That difference in conflict - # action is the discriminator, and it is load-bearing rather than - # incidental: it is the SQL-level statement of which caller owns - # the row. - "ON CONFLICT (block_hash) DO NOTHING", - ), - ), - ( - LandingOp.OUTBOX_BATCH_ABANDON, - ( - # The page-oriented terminal write (issue #196), listed before - # OUTBOX_FINISH because it satisfies that signature too: the - # discriminators are the set predicate and the pool-block - # re-check, which is the clause that makes acting on a stale - # enumeration safe. - "UPDATE qbit_block_candidate_outbox", - "SET state = 'abandoned',", - "WHERE block_hash = ANY(ARRAY[", - "FROM qbit_pool_blocks pool", - ), - ), - ( - LandingOp.OUTBOX_FINISH, - ( - "UPDATE qbit_block_candidate_outbox", - "completed_at = clock_timestamp(),", - "candidate = NULL", - ), - ), - ( - LandingOp.OUTBOX_ATTEMPT, - ( - "UPDATE qbit_block_candidate_outbox", - "SET attempt_count = attempt_count + 1,", - ), - ), - ( - LandingOp.ALL_SHARES, - ( - "'share_seq', share_seq,", - "FROM qbit_share_ledger", - "WHERE accepted;", - ), - ), - ( - LandingOp.SHARE_STATS, - ( - "'accepted_share_count', count(*),", - "FROM qbit_share_ledger", - ), - ), - ( - LandingOp.PRIOR_BALANCES_AS_OF, - ( - "'recipient_id', miner_id,", - "FROM qbit_payout_carry_forward carry", - "AND block.block_height <= target.block_height", - ), - ), - ( - LandingOp.PRIOR_BALANCES, - ( - "'recipient_id', miner_id,", - "FROM qbit_current_carry_forward_balances();", - ), - ), -) - -# Statements that belong to the state machines issue #128 still defers. -# Recognised only so the failure names the right follow-up instead of reading -# as an unexplained harness gap. -_DEFERRED_SIGNATURES: tuple[tuple[str, str], ...] = ( - ("INSERT INTO qbit_share_ledger", "share submission and dedupe"), - ("qbit_ctv_fanouts", "CTV fanout broadcasting"), -) - -# Landing statement arguments. The ledger inlines them as SQL literals, so -# the model reads them back out of the statement rather than being told. -_POOL_FUNCTION_RE = re.compile( - r"qbit_(?:confirm|reject_prepared)_pool_block\(\s*" - r"'([0-9a-f]+)',\s*(-?\d+),\s*" - r"'((?:[^']|'')*)',\s*(\d+),\s*'((?:[^']|'')*)'", - re.DOTALL, -) -# The lease CTE every outbox statement opens with, which inlines the writer -# identity rather than carrying it in a jsonb payload. -_LEASE_IDENTITY_RE = re.compile( - r"writer_id = '((?:[^']|'')*)'\s*" - r"AND writer_epoch = (\d+)\s*" - r"AND writer_session_token = '((?:[^']|'')*)'", - re.DOTALL, -) -_POOL_BLOCK_HASH_RE = re.compile(r"WHERE block_hash = '([0-9a-f]+)'") -# The pending page's own two arguments: the bounded window and the optional -# keyset cursor. Both are inlined as literals by pending_block_candidate_rows. -_PENDING_PAGE_LIMIT_RE = re.compile(r"LIMIT (\d+)\n\) pending;") -_PENDING_PAGE_CURSOR_RE = re.compile( - r"AND \(created_at, block_hash\) > " - r"\('((?:[^']|'')*)'::timestamptz, '((?:[^']|'')*)'\)" -) -_OUTBOX_STATE_RE = re.compile(r"SET state = '(submitted|abandoned)'") -_OUTBOX_BATCH_HASHES_RE = re.compile( - r"WHERE block_hash = ANY\(ARRAY\[(.*?)\]::text\[\]\)", - re.DOTALL, -) -_OUTBOX_ERROR_RE = re.compile(r"last_error = (NULL|'(?:[^']|'')*')") -_OUTBOX_SHA_RE = re.compile(r"candidate_sha256 <> '([0-9a-f]+)'") -# Issue #255 literals: the body id a statement addresses, the chunk bytes a -# chunk upload inlines, and the arguments of the index/page reads. -_BODY_ID_RE = re.compile(r"body_id = '([0-9a-f]{32})'") -_CHUNK_HEX_RE = re.compile(r"decode\(\$qbit_prism_chunk\$([0-9a-f]*)\$qbit_prism_chunk\$, 'hex'\)") -_SPANS_AFTER_RE = re.compile(r"AND field > '((?:[^']|'')*)'") -_PAGES_FIELD_RE = re.compile(r"AND field = '((?:[^']|'')*)'\s*AND page_ordinal >= (\d+)") -_BODY_PAGE_ARGS_RE = re.compile(r"AND ordinal >= (\d+)\s*ORDER BY ordinal\s*LIMIT (\d+)") -_HEADER_PAGE_LIMIT_RE = re.compile(r"LIMIT (\d+)\n\),\nmeasured AS") -_HEADER_PAGE_BYTES_RE = re.compile(r"running_bytes <= (\d+)") - - -#: Terminal-arm keys that name a lease column. Nothing is visible to report a -#: holder from -- the arm exists precisely for that case -- so they answer -#: null. -_ACQUIRE_TERMINAL_NULL_KEYS = frozenset( - { - "writer_id", - "writer_epoch", - "writer_session_token", - "lease_expires_at", - "lease_updated_at", - "lease_age_seconds", - "lease_wait_seconds", - } -) - -#: Terminal-arm keys whose value is a SQL literal, taken from the statement -#: rather than restated here. ``acquired`` is false on this arm; -#: ``lease_snapshot_retry`` is the flag ``_try_acquire_writer_lease`` retries -#: on; ``lease`` and ``retry_reason`` carry the operator-facing subject and -#: explanation that replace the old "postgres query returned no JSON". -#: -#: Reading the value out of the SQL matters more than it looks. An earlier -#: draft keyed on the *name* -- anything containing "retry" answered true -- -#: which got ``lease_snapshot_retry`` right by luck and would have answered -#: ``retry_reason`` with a boolean where production emits a sentence. The -#: model now answers what the statement says, so it cannot drift from the -#: wording the ledger PR chose. -_ACQUIRE_TERMINAL_LITERAL_KEYS = frozenset( - { - "acquired", - "lease_snapshot_retry", - "lease", - "retry_reason", - } -) - - -def _split_sql_arguments(body: str) -> list[str]: - """Split a call's argument text on its top-level commas. - - Parenthesised sub-expressions and single-quoted literals are held - together, because both occur in this statement: the holder arm nests - ``GREATEST(0, EXTRACT(...))``, and the terminal arm's retry reason is a - sentence that must survive as one argument. - """ - parts: list[str] = [] - current: list[str] = [] - depth = 0 - quoted = False - index = 0 - while index < len(body): - char = body[index] - if quoted: - current.append(char) - if char == "'": - if index + 1 < len(body) and body[index + 1] == "'": - current.append("'") - index += 1 - else: - quoted = False - elif char == "'": - quoted = True - current.append(char) - elif char in "([": - depth += 1 - current.append(char) - elif char in ")]": - depth -= 1 - current.append(char) - elif char == "," and depth == 0: - parts.append("".join(current).strip()) - current = [] - else: - current.append(char) - index += 1 - remainder = "".join(current).strip() - if remainder: - parts.append(remainder) - return parts - - -def _balanced_call_body(text: str, open_index: int) -> str: - """Return the argument text of a call whose ``(`` is at ``open_index``.""" - depth = 0 - for index in range(open_index, len(text)): - char = text[index] - if char == "(": - depth += 1 - elif char == ")": - depth -= 1 - if depth == 0: - return text[open_index + 1 : index] - raise UnsupportedStatement("acquire statement has unbalanced parentheses") - - -def acquire_result_arms(sql: str) -> int: - """Count the result arms of the acquire statement's final COALESCE. - - The acquire statement ends in ``SELECT COALESCE(<arm>, <arm>, ...)``. At - this baseline there are exactly two: the ``RETURNING`` row of the upsert - (acquired) and the committed holder row (not acquired). Both can be empty - at once -- two coordinators racing the very first acquire -- and the - statement then evaluates to SQL NULL, which the ports turn into - production's "postgres query returned no JSON". - - Issue #140 adds an explicit not-acquired/retry arm so that race reports a - lease outcome instead of what reads as a driver fault. That is a change to - the shipped statement, so this model keys off the statement rather than - off a version flag or a local constant: a statement carrying a terminal - arm cannot evaluate to NULL, and continuing to model it as if it could - would let a scenario assert an outcome the server can no longer produce -- - the exact class of harness lie this suite exists to prevent. - - Counting arms rather than matching a chosen literal is deliberate: the arm - has to exist for the fix to work, whatever its author names its keys. - """ - _head, separator, tail = sql.partition("SELECT COALESCE(") - if not separator: - raise UnsupportedStatement( - "acquire statement does not end in the SELECT COALESCE this model " - "recognises; its result shape changed and the lease evaluator has " - "to be re-read against it rather than assumed:\n" + sql.strip()[:400] - ) - arms = tail.count("'acquired',") - if arms < 2: - raise UnsupportedStatement( - f"acquire statement's COALESCE names {arms} result arm(s); this " - "model knows the two-arm shape (acquired / holder) and the " - "three-or-more-arm shape issue #140 adds. Fewer than two is a " - "shape it has never seen." - ) - return arms - - -def acquire_terminal_arm_fields(sql: str) -> tuple[tuple[str, str], ...]: - """Return ``(key, raw value text)`` for the last ``json_build_object`` arm. - - ``json_build_object`` takes alternating key and value arguments, so the - pairing is the call's own structure rather than a convention this model - imposes. - """ - _head, _separator, tail = sql.partition("SELECT COALESCE(") - marker = "json_build_object(" - index = tail.rfind(marker) - if index < 0: - raise UnsupportedStatement("acquire statement's terminal arm builds no JSON object") - body = _balanced_call_body(tail, index + len(marker) - 1) - arguments = _split_sql_arguments(body) - if len(arguments) % 2: - raise UnsupportedStatement( - "the acquire statement's terminal arm names an odd number of " - f"json_build_object arguments ({len(arguments)}); it cannot be read " - "as key/value pairs" - ) - fields: list[tuple[str, str]] = [] - for key_text, value_text in zip(arguments[::2], arguments[1::2], strict=True): - if len(key_text) < 2 or not key_text.startswith("'") or not key_text.endswith("'"): - raise UnsupportedStatement( - f"the acquire statement's terminal arm names {key_text!r} where a " - "quoted key was expected" - ) - fields.append((_unquote(key_text[1:-1]), value_text)) - return tuple(fields) - - -def acquire_terminal_arm_keys(sql: str) -> tuple[str, ...]: - """Return the keys named by the last ``json_build_object`` arm.""" - return tuple(key for key, _value in acquire_terminal_arm_fields(sql)) - - -def classify(sql: str) -> Statement: - """Map one production statement onto the operation it performs.""" - for op, fragments in _SIGNATURES: - if all(fragment in sql for fragment in fragments): - if op is LeaseOp.PROVE: - # The proof renews nothing, so its only interval literal is - # the renewal-due (own-write authority) margin. - lease_ttl_seconds = None - authority_margin_seconds = _extract_interval(sql, first=True) - else: - lease_ttl_seconds = _extract_interval(sql, first=True) - authority_margin_seconds = ( - _extract_interval(sql, first=False) - if op is LeaseOp.VERIFY - else None - ) - return Statement( - kind=op, - payload=_extract_payload(sql), - lease_ttl_seconds=lease_ttl_seconds, - authority_margin_seconds=authority_margin_seconds, - advisory_lock=_extract_advisory_lock(sql), - sql=sql, - ) - for landing_op, fragments in _LANDING_SIGNATURES: - if all(fragment in sql for fragment in fragments): - return Statement( - kind=landing_op, - payload=_landing_payload(landing_op, sql), - lease_ttl_seconds=_extract_interval(sql, first=True), - authority_margin_seconds=None, - advisory_lock=None, - sql=sql, - ) - for marker, state_machine in _DEFERRED_SIGNATURES: - if marker in sql: - raise UnsupportedStatement( - f"{marker} belongs to the {state_machine} state machine, which " - "issue #128 defers to a follow-up PR; this harness models the " - "writer-lease lifecycle and block candidate landing" - ) - raise UnsupportedStatement( - "FakePostgres does not model this statement. Either the production " - "SQL changed shape (update _SIGNATURES or _LANDING_SIGNATURES) or the " - "scenario reached a state machine this harness does not cover:\n" - f"{sql.strip()[:400]}" - ) - - -def _unquote(literal: str) -> str: - """Undo ``PsqlShareLedger._text_literal`` quoting.""" - return literal.replace("''", "'") - - -def _lease_identity(op: LandingOp, sql: str) -> dict[str, Any]: - match = _LEASE_IDENTITY_RE.search(sql) - if match is None: - raise UnsupportedStatement( - f"{op.value} statement carried no writer lease identity" - ) - return { - "writer_id": _unquote(match.group(1)), - "writer_epoch": int(match.group(2)), - "writer_session_token": _unquote(match.group(3)), - } - - -def _landing_payload(op: LandingOp, sql: str) -> dict[str, Any]: - """Recover the arguments a landing statement inlined as SQL literals.""" - if op is LandingOp.PERSIST_BLOCK: - return _extract_payload(sql) - if op in {LandingOp.CONFIRM_BLOCK, LandingOp.REJECT_PREPARED}: - match = _POOL_FUNCTION_RE.search(sql) - if match is None: - raise UnsupportedStatement( - f"{op.value} statement did not name a block hash and height" - ) - return { - "block_hash": match.group(1), - "active_tip_height": int(match.group(2)), - "writer_id": _unquote(match.group(3)), - "writer_epoch": int(match.group(4)), - "writer_session_token": _unquote(match.group(5)), - } - if op in {LandingOp.POOL_BLOCK_STATE, LandingOp.CONFIRMED_SEQUENCE}: - match = _POOL_BLOCK_HASH_RE.search(sql) - if match is None: - raise UnsupportedStatement(f"{op.value} statement named no block hash") - return {"block_hash": match.group(1)} - if op is LandingOp.OUTBOX_RECORD: - hash_match = _POOL_BLOCK_HASH_RE.search(sql) - sha_match = _OUTBOX_SHA_RE.search(sql) - if hash_match is None or sha_match is None: - raise UnsupportedStatement( - "outbox record statement named no block hash or payload digest" - ) - body_match = _BODY_ID_RE.search(sql) - header_match = _PAYLOAD_RE.search(sql) - return { - "block_hash": hash_match.group(1), - "candidate_sha256": sha_match.group(1), - "body_id": None if body_match is None else body_match.group(1), - "header": ( - None if header_match is None else json.loads(header_match.group(2)) - ), - **_lease_identity(op, sql), - } - if op in { - LandingOp.BODY_STAGE, - LandingOp.BODY_INDEX, - LandingOp.BODY_SEAL, - LandingOp.BODY_RETIRE_ORPHANS, - LandingOp.BODY_REAP, - }: - return _extract_payload(sql) - if op is LandingOp.BODY_CHUNK: - chunk_match = _CHUNK_HEX_RE.search(sql) - if chunk_match is None: - raise UnsupportedStatement("chunk upload statement carried no chunk bytes") - return {**_extract_payload(sql), "chunk": bytes.fromhex(chunk_match.group(1))} - if op in { - LandingOp.OUTBOX_OBSERVE, - }: - hash_match = _POOL_BLOCK_HASH_RE.search(sql) - if hash_match is None: - raise UnsupportedStatement(f"{op.value} statement named no block hash") - return {"block_hash": hash_match.group(1)} - if op in {LandingOp.BODY_MANIFEST, LandingOp.BODY_RETIRE_STAGING}: - body_match = _BODY_ID_RE.search(sql) - if body_match is None: - raise UnsupportedStatement(f"{op.value} statement named no body id") - return {"body_id": body_match.group(1)} - if op is LandingOp.BODY_SPANS: - body_match = _BODY_ID_RE.search(sql) - after_match = _SPANS_AFTER_RE.search(sql) - if body_match is None: - raise UnsupportedStatement("span page statement named no body id") - return { - "body_id": body_match.group(1), - "after_field": None if after_match is None else _unquote(after_match.group(1)), - } - if op is LandingOp.BODY_PAGES: - body_match = _BODY_ID_RE.search(sql) - args_match = _PAGES_FIELD_RE.search(sql) - if body_match is None or args_match is None: - raise UnsupportedStatement("page-index statement named no body, field or ordinal") - return { - "body_id": body_match.group(1), - "field": _unquote(args_match.group(1)), - "from_ordinal": int(args_match.group(2)), - } - if op is LandingOp.BODY_PAGE: - body_match = _BODY_ID_RE.search(sql) - args_match = _BODY_PAGE_ARGS_RE.search(sql) - if body_match is None or args_match is None: - raise UnsupportedStatement("chunk page statement named no body, ordinal or limit") - return { - "body_id": body_match.group(1), - "from_ordinal": int(args_match.group(1)), - "max_chunks": int(args_match.group(2)), - } - if op is LandingOp.OUTBOX_HEADER_PAGE: - limit_match = _HEADER_PAGE_LIMIT_RE.search(sql) - bytes_match = _HEADER_PAGE_BYTES_RE.search(sql) - if limit_match is None or bytes_match is None: - raise UnsupportedStatement("header page statement carried no bounded LIMIT or byte cap") - cursor_match = _PENDING_PAGE_CURSOR_RE.search(sql) - return { - # The statement fetches one row beyond the cap it returns. - "limit": int(limit_match.group(1)) - 1, - "max_bytes": int(bytes_match.group(1)), - "after_cursor": ( - None - if cursor_match is None - else (_unquote(cursor_match.group(1)), _unquote(cursor_match.group(2))) - ), - } - if op in {LandingOp.SCHEMA_PRESENCE, LandingOp.SCHEMA_CAPABILITY}: - return {} - if op is LandingOp.OUTBOX_BATCH_ABANDON: - hashes_match = _OUTBOX_BATCH_HASHES_RE.search(sql) - error_match = _OUTBOX_ERROR_RE.search(sql) - if hashes_match is None: - raise UnsupportedStatement( - "outbox batch abandon statement named no block hash set" - ) - raw_error = None if error_match is None else error_match.group(1) - return { - "block_hashes": tuple( - _unquote(element.strip()[1:-1]) - for element in _split_sql_arguments(hashes_match.group(1)) - ), - "last_error": ( - None - if raw_error in (None, "NULL") - else str(raw_error)[1:-1].replace("''", "'") - ), - **_lease_identity(op, sql), - } - if op is LandingOp.OUTBOX_PENDING_PAGE: - limit_match = _PENDING_PAGE_LIMIT_RE.search(sql) - if limit_match is None: - raise UnsupportedStatement( - "pending candidate page statement carried no bounded LIMIT" - ) - cursor_match = _PENDING_PAGE_CURSOR_RE.search(sql) - return { - "limit": int(limit_match.group(1)), - "after_cursor": ( - None - if cursor_match is None - else ( - _unquote(cursor_match.group(1)), - _unquote(cursor_match.group(2)), - ) - ), - } - if op in {LandingOp.OUTBOX_ATTEMPT, LandingOp.OUTBOX_FINISH}: - hash_match = _POOL_BLOCK_HASH_RE.search(sql) - if hash_match is None: - raise UnsupportedStatement(f"{op.value} statement named no block hash") - payload: dict[str, Any] = { - "block_hash": hash_match.group(1), - **_lease_identity(op, sql), - } - if op is LandingOp.OUTBOX_FINISH: - state_match = _OUTBOX_STATE_RE.search(sql) - if state_match is None: - raise UnsupportedStatement( - "outbox finish statement named no terminal state" - ) - payload["state"] = state_match.group(1) - error_match = _OUTBOX_ERROR_RE.search(sql) - raw_error = None if error_match is None else error_match.group(1) - payload["last_error"] = ( - None - if raw_error in (None, "NULL") - else str(raw_error)[1:-1].replace("''", "'") - ) - return payload - return {} - - -def _extract_payload(sql: str) -> dict[str, Any]: - match = _PAYLOAD_RE.search(sql) - if match is None: - raise UnsupportedStatement("statement carried no jsonb payload") - payload = json.loads(match.group(2)) - if not isinstance(payload, dict): - raise UnsupportedStatement("statement payload was not an object") - return payload - - -def _extract_interval(sql: str, *, first: bool) -> float | None: - matches = _INTERVAL_RE.findall(sql) - if not matches: - return None - if first: - return float(matches[0]) - return float(matches[1]) if len(matches) > 1 else None - - -def _extract_advisory_lock(sql: str) -> tuple[int, int] | None: - classid = _CLASSID_RE.search(sql) - objid = _OBJID_RE.search(sql) - if classid is None or objid is None: - return None - return (int(classid.group(1)), int(objid.group(1))) - - -def advisory_lock_pair(key: int) -> tuple[int, int]: - """Split a 64-bit advisory key the way pg_locks reports it.""" - return ((key >> 32) & 0xFFFFFFFF, key & 0xFFFFFFFF) - - -# -------------------------------------------------------------------------- -# The PostgreSQL model -# -------------------------------------------------------------------------- - - -@dataclass -class LeaseRow: - writer_id: str - writer_epoch: int - writer_session_token: str - lease_expires_at: datetime - updated_at: datetime - # The locking transaction's xid, exactly as PostgreSQL stamps xmax on a - # row-locked tuple. Visible to a snapshot taken while the lock is held. - xmax: str | None = None - - -@dataclass -class PoolBlockRow: - """One ``qbit_pool_blocks`` row. - - ``audit_publication_sequence`` is the durable publication ordinal the - audit store gates on. It is assigned once, by the confirmation that first - moves the row to ``confirmed``, and survives every later disposition — - which is why an exact replay must not burn a second ordinal and why the - floor is a MAX over this column rather than a counter. - """ - - block_hash: str - block_height: int - parent_hash: str - chain_state: str = "prepared" - maturity_state: str = "immature" - audit_publication_sequence: int | None = None - - -@dataclass -class OutboxRow: - """One ``qbit_block_candidate_outbox`` row: the durable replay source.""" - - block_hash: str - candidate_sha256: str - # ``created_at DEFAULT clock_timestamp()``. The pending page orders and - # keysets on ``(created_at, block_hash)``, so the model has to carry the - # stamp the real column would: an ordering that came from insertion order - # would answer the cursor questions the enumeration actually asks with a - # property production does not have. - created_at: datetime = CLOCK_ORIGIN - state: str = "pending" - attempt_count: int = 0 - last_error: str | None = None - # Issue #255: a version-2 row references a sealed body and carries the - # bounded replay header; terminalization detaches the body. - storage_version: int = 1 - body_id: str | None = None - retired_body_id: str | None = None - header: dict[str, Any] | None = None - - -@dataclass -class BodyRow: - """One ``qbit_block_candidate_body`` manifest with its parts (#255). - - Chunks, spans and pages live on the manifest here for simplicity; the - model enforces the same rules the database triggers do: parts are - accepted only while ``staging``, never updated, and deleted only once - ``retired``. - """ - - body_id: str - block_hash: str - candidate_sha256: str - byte_count: int - chunk_count: int - chunk_bytes: int - share_count: int - shares_offset: int - shares_end: int - span_count: int - page_count: int - writer_id: str - writer_epoch: int - writer_session_token: str - created_at: datetime = CLOCK_ORIGIN - state: str = "staging" - sealed_at: datetime | None = None - retired_at: datetime | None = None - chunks: dict[int, tuple[bytes, str]] = field(default_factory=dict) - spans: dict[str, dict[str, Any]] = field(default_factory=dict) - pages: dict[tuple[str, int], tuple[int, int]] = field(default_factory=dict) - - def copy(self) -> "BodyRow": - return replace( - self, - chunks=dict(self.chunks), - spans={key: dict(value) for key, value in self.spans.items()}, - pages=dict(self.pages), - ) - - -@dataclass -class Transaction: - xid: str - backend: Backend - explicit: bool - holds_lease_lock: bool = False - staged_lease: LeaseRow | None = None - # Landing writes are staged for the same reason lease writes are: a - # deadline-scoped statement sends COMMIT as a separate message, so there - # is a window in which this transaction can see its own pool-block row - # and nobody else can. #133 lives inside exactly such a window, so a - # model that published the write at statement time could hide it. - staged_pool_blocks: dict[str, PoolBlockRow] = field(default_factory=dict) - staged_outbox: dict[str, OutboxRow] = field(default_factory=dict) - staged_bodies: dict[str, BodyRow] = field(default_factory=dict) - deleted_bodies: set[str] = field(default_factory=set) - orphaned: bool = False - - -@dataclass -class Backend: - """One server-side session.""" - - pid: str - application_name: str - server: FakePostgres - transaction: Transaction | None = None - closed: bool = False - advisory_locks: set[tuple[int, int]] = field(default_factory=set) - # Guard sessions carry a server-side statement_timeout; pool sessions - # take their deadline from the caller instead. - statement_timeout_seconds: float | None = None - - @property - def backend_xid(self) -> str | None: - if self.closed or self.transaction is None: - return None - return self.transaction.xid - - -class LockTimeout(RuntimeError): - """SQLSTATE 55P03 / 57014 as the ledger's callers observe them.""" - - -class FakePostgres: - """In-memory model of the writer-lease table and its locking behaviour.""" - - def __init__(self, scheduler: DeterministicScheduler, clock: VirtualClock) -> None: - self.scheduler = scheduler - self.clock = clock - self.lease: LeaseRow | None = None - self.statements: list[Statement] = [] - # Optional per-statement server time, in virtual seconds. Scenarios - # that model database latency (the rapid-block tail behind issue - # #212) install a callable here; the default None keeps every - # existing scenario's statements instantaneous. - self.statement_latency_seconds: ( - Callable[[Statement], float] | None - ) = None - # Server time charged to the statement currently being evaluated; - # the guard statements report it as server_execution_seconds. - self._last_statement_server_seconds = 0.0 - # Landing tables. Insertion-ordered dicts, because the trace and every - # assertion made against them has to be order-stable across runs. - self.pool_blocks: dict[str, PoolBlockRow] = {} - self.outbox: dict[str, OutboxRow] = {} - # Issue #255: chunked candidate bodies, keyed by body id. - self.bodies: dict[str, BodyRow] = {} - self.shares: list[dict[str, Any]] = [] - self.carry_forward_balances: list[dict[str, Any]] = [] - self._lease_lock_holder: Transaction | None = None - self._lease_lock_waiters: list[Transaction] = [] - self._backends: list[Backend] = [] - self._advisory_owners: dict[tuple[int, int], Backend] = {} - self._next_pid = 1 - self._next_xid = 1 - # qbit_audit_publication_sequence_seq. A sequence, not MAX + 1: the - # allocator burns a value per fresh confirmation and never reuses - # one, which is why the floor read below is a separate MAX. - self._audit_publication_sequence = 0 - - # -- sessions ---------------------------------------------------------- - - def connect( - self, - *, - application_name: str, - statement_timeout_seconds: float | None = None, - ) -> Backend: - backend = Backend( - pid=f"pid-{self._next_pid}", - application_name=application_name, - server=self, - statement_timeout_seconds=statement_timeout_seconds, - ) - self._next_pid += 1 - self._backends.append(backend) - return backend - - def disconnect(self, backend: Backend) -> None: - """Close a session cleanly: PostgreSQL rolls back and releases everything.""" - if backend.closed: - return - backend.closed = True - if backend.transaction is not None: - self._rollback(backend.transaction) - for key in list(backend.advisory_locks): - if self._advisory_owners.get(key) is backend: - del self._advisory_owners[key] - backend.advisory_locks.clear() - - def try_advisory_lock(self, backend: Backend, key: int) -> bool: - pair = advisory_lock_pair(key) - owner = self._advisory_owners.get(pair) - if owner is not None and not owner.closed: - return owner is backend - self._advisory_owners[pair] = backend - backend.advisory_locks.add(pair) - return True - - def live_backends(self) -> list[Backend]: - return [backend for backend in self._backends if not backend.closed] - - # -- transactions ------------------------------------------------------ - - def _begin(self, backend: Backend, *, explicit: bool) -> Transaction: - if backend.transaction is not None: - raise HarnessError( - f"backend {backend.pid} already has an open transaction" - ) - transaction = Transaction( - xid=f"xid-{self._next_xid}", - backend=backend, - explicit=explicit, - ) - self._next_xid += 1 - backend.transaction = transaction - return transaction - - def _commit(self, transaction: Transaction) -> None: - if transaction.staged_lease is not None: - self.lease = replace(transaction.staged_lease, xmax=None) - transaction.staged_lease = None - self.pool_blocks.update(transaction.staged_pool_blocks) - self.outbox.update(transaction.staged_outbox) - self.bodies.update(transaction.staged_bodies) - for body_id in transaction.deleted_bodies: - self.bodies.pop(body_id, None) - transaction.staged_pool_blocks.clear() - transaction.staged_outbox.clear() - transaction.staged_bodies.clear() - transaction.deleted_bodies.clear() - self._release_lease_lock(transaction) - transaction.backend.transaction = None - - def _rollback(self, transaction: Transaction) -> None: - transaction.staged_lease = None - # The publication ordinal is deliberately not rolled back: nextval is - # non-transactional, so an aborted confirmation burns its value and - # leaves a gap. The floor read is a MAX precisely so a gap is - # harmless. - transaction.staged_pool_blocks.clear() - transaction.staged_outbox.clear() - transaction.staged_bodies.clear() - transaction.deleted_bodies.clear() - self._release_lease_lock(transaction) - transaction.backend.transaction = None - - def _release_lease_lock(self, transaction: Transaction) -> None: - if self._lease_lock_holder is transaction: - self._lease_lock_holder = None - if self.lease is not None and self.lease.xmax == transaction.xid: - self.lease = replace(self.lease, xmax=None) - - def orphan(self, transaction: Transaction) -> None: - """The client vanished without RST: the transaction stays open. - - PostgreSQL finished executing the statement and is waiting for a - COMMIT that will never arrive, so it keeps the tuple lock. - - This model does not implement a reaper. Production now sets - ``idle_in_transaction_session_timeout`` - (``PRISM_POSTGRES_IDLE_IN_TRANSACTION_TIMEOUT_SECONDS``, 15s by - default) on every coordinator session, so a real orphan of *this* - deployment's making is aborted and its locks released. An orphan here - is therefore the worst case rather than the only case: the foreign - session the guard cannot reach, or the window before it fires. - Scenarios that want the reap must model it explicitly; nothing in the - harness ends an orphan on its own. - """ - transaction.orphaned = True - - @property - def lease_lock_holder(self) -> Transaction | None: - return self._lease_lock_holder - - def lease_lock_held_by_other(self, transaction: Transaction | None) -> bool: - holder = self._lease_lock_holder - return holder is not None and holder is not transaction - - # -- statement execution ---------------------------------------------- - - def execute( - self, - backend: Backend, - sql: str, - *, - transaction: Transaction | None, - timeout_seconds: float | None, - tag: str, - ) -> Any: - """Run one statement, stopping at every interleaving point it offers.""" - if backend.closed: - raise RuntimeError("connection is closed") - statement = classify(sql) - self.statements.append(statement) - autocommit = transaction is None - if autocommit: - transaction = self._begin(backend, explicit=False) - - self.scheduler.checkpoint(f"{tag}.begin:{statement.kind.value}") - - # A caller deadline arrives as SET LOCAL lock_timeout; the guard - # session's bound is a connection-level statement_timeout. PostgreSQL - # reports them with different messages and SQLSTATEs, and the whole - # point of the SKIP LOCKED heartbeat was that the guard's bound is the - # fatal one, so the model must not blur them. - effective_timeout = timeout_seconds - timeout_kind = "lock" - session_bound = backend.statement_timeout_seconds - if session_bound is not None and ( - effective_timeout is None or session_bound <= effective_timeout - ): - effective_timeout = session_bound - timeout_kind = "statement" - try: - # The modelled server time is also what the guard statements - # report about themselves (issue #227): production computes - # clock_timestamp() - statement_timestamp() inside the - # statement, and here that is exactly the latency just charged. - self._last_statement_server_seconds = self._charge_execution_latency( - statement, - effective_timeout, - timeout_kind, - ) - result = self._evaluate( - statement, - transaction, - effective_timeout, - tag, - timeout_kind, - ) - except BaseException: - if autocommit: - self._rollback(transaction) - raise - - self.scheduler.checkpoint(f"{tag}.done:{statement.kind.value}") - if autocommit: - self._commit(transaction) - return result - - def _charge_execution_latency( - self, - statement: Statement, - timeout_seconds: float | None, - timeout_kind: str, - ) -> float: - """Spend modelled server time for this statement, honouring its bound. - - Returns the seconds charged, which the guard statements report back - as their own ``server_execution_seconds`` (issue #227). - - Lock waits were already modelled (``_await_lease_lock``); this is the - other half — a statement that is simply *slow*, which is what the - rapid-block bursts behind issue #212 produce and what the heartbeat's - staleness envelope has to absorb. Advancing the virtual clock inside - the statement is exact under the baton scheduler: no other actor is - running, so the time is charged to this statement alone and every - other actor's deadline sees it on its next turn. - - A statement whose modelled cost exceeds the session bound is - cancelled at the bound, exactly as PostgreSQL's statement_timeout - cancels it — the guard session's 0.5s ceiling is the term the - heartbeat policy is sized against, so a scenario must not be able to - model a guarded round trip that quietly outlives it. - """ - latency_for = self.statement_latency_seconds - if latency_for is None: - return 0.0 - latency = float(latency_for(statement) or 0.0) - if latency <= 0.0: - return 0.0 - if timeout_seconds is not None and latency > timeout_seconds: - self.clock.advance(max(0.0, timeout_seconds)) - raise LockTimeout( - f"canceling statement due to {timeout_kind} timeout" - ) - self.clock.advance(latency) - return latency - - def _evaluate( - self, - statement: Statement, - transaction: Transaction, - timeout_seconds: float | None, - tag: str, - timeout_kind: str = "lock", - ) -> Any: - if statement.is_landing: - return self._evaluate_landing( - statement, - transaction, - timeout_seconds, - tag, - timeout_kind, - ) - if statement.kind is LeaseOp.VERIFY: - return self._evaluate_verify(statement, transaction, tag) - if statement.kind is LeaseOp.PROVE: - return self._evaluate_prove(statement, transaction) - - # READ COMMITTED takes one snapshot per command, *before* any lock - # wait. This is the distinction that makes a queued statement report - # stale data: after the wait, only the conflicting tuple is re-read - # (EvalPlanQual) to re-evaluate the write predicate, while every other - # scan in the statement — including the arm that builds the whole - # not-acquired result — still reads `snapshot`. A model that re-read - # committed state everywhere would report a fresh row and a small - # lease_age_seconds where the real server reports the pre-wait row and - # an age that has silently absorbed the entire wait. - snapshot = self._visible_lease(transaction) - - if statement.kind is LeaseOp.ACQUIRE: - # INSERT ... ON CONFLICT DO UPDATE locks the conflicting tuple - # before evaluating its WHERE, so it queues even when the - # predicate would exclude the row. - self._await_lease_lock( - statement, - transaction, - timeout_seconds, - tag, - timeout_kind, - ) - return self._evaluate_acquire(statement, transaction, snapshot) - - # A plain UPDATE evaluates its WHERE against the snapshot version - # first and never locks a row the predicate already excludes: it - # returns zero rows immediately rather than queueing. Getting this - # wrong matters to any fencing argument, where "the predecessor is - # blocked" and "the predecessor already knows it lost" are different - # states. - evaluators = { - LeaseOp.ADOPT: (self._adopt_matches, self._evaluate_adopt), - LeaseOp.RENEW: (self._renew_matches, self._evaluate_renew), - LeaseOp.RELEASE: (self._release_matches, self._evaluate_release), - } - try: - matches, evaluate = evaluators[statement.kind] - except KeyError: - raise UnsupportedStatement( - f"unhandled lease operation {statement.kind}" - ) from None - if snapshot is None or not matches(statement, snapshot): - return evaluate(statement, transaction, None) - self._await_lease_lock( - statement, - transaction, - timeout_seconds, - tag, - timeout_kind, - ) - # EvalPlanQual: the qual is re-checked against the version this - # statement just locked, which a concurrent commit may have moved. - current = self._visible_lease(transaction) - if current is None or not matches(statement, current): - return evaluate(statement, transaction, None) - return evaluate(statement, transaction, current) - - def _await_lease_lock( - self, - statement: Statement, - transaction: Transaction, - timeout_seconds: float | None, - tag: str, - timeout_kind: str = "lock", - ) -> None: - # Waiting is not conditional on a *committed* row existing. Two - # concurrent INSERTs conflict on the singleton unique index, and the - # second one queues behind the first exactly as an ON CONFLICT - # DO UPDATE queues behind an uncommitted update. - deadline = ( - None - if timeout_seconds is None - else self.clock.monotonic() + max(0.0, timeout_seconds) - ) - - def granted() -> bool: - # PostgreSQL serializes tuple-lock waiters through a heavyweight - # lock and grants in arrival order, so a later arrival cannot - # overtake a queued waiter. Without the queue the model could - # express a grant order the server never produces — which matters - # precisely because the schedule is an input here. - if self.lease_lock_held_by_other(transaction): - return False - return ( - not self._lease_lock_waiters - or self._lease_lock_waiters[0] is transaction - ) - - if granted(): - self._take_lease_lock(transaction) - return - self._lease_lock_waiters.append(transaction) - try: - while not granted(): - if deadline is not None and self.clock.monotonic() >= deadline: - raise LockTimeout( - f"canceling statement due to {timeout_kind} timeout\n" - "CONTEXT: while updating tuple (0,1) in relation " - '"qbit_ledger_writer_lease"' - ) - self.scheduler.block( - f"{tag}.lockwait:{statement.kind.value}", - ready=granted, - wake_at=deadline, - ) - finally: - self._lease_lock_waiters.remove(transaction) - self._take_lease_lock(transaction) - - def _take_lease_lock(self, transaction: Transaction) -> None: - if self.lease_lock_held_by_other(transaction): - raise HarnessError( - "the lease tuple lock was taken while another transaction " - "held it; the model must queue, never steal" - ) - self._lease_lock_holder = transaction - transaction.holds_lease_lock = True - if self.lease is not None: - self.lease = replace(self.lease, xmax=transaction.xid) - - def _visible_lease(self, transaction: Transaction) -> LeaseRow | None: - """READ COMMITTED: the statement sees committed state plus its own writes.""" - if transaction.staged_lease is not None: - return transaction.staged_lease - return self.lease - - def _stage(self, transaction: Transaction, row: LeaseRow) -> None: - transaction.staged_lease = row - if not transaction.explicit: - return - # An explicit transaction's write is invisible to everyone else until - # COMMIT, but the tuple lock is already taken; reflect the locker in - # the committed row's xmax so pg_stat_activity attribution works. - if self.lease is not None: - self.lease = replace(self.lease, xmax=transaction.xid) - - # -- individual operations -------------------------------------------- - - def _evaluate_acquire( - self, - statement: Statement, - transaction: Transaction, - snapshot: LeaseRow | None, - ) -> dict[str, Any] | None: - """INSERT ... ON CONFLICT DO UPDATE, with EPQ on the locked tuple. - - Returns None when the statement evaluates to SQL NULL, which the - ports turn into production's "postgres query returned no JSON" the - same way `parse_single_json_value` does. That is a reachable - production outcome, not a harness edge case: when two coordinators - race the very first acquire, the loser's DO UPDATE affects zero rows - and its COALESCE fallback reads a snapshot in which no row exists - yet, so the whole statement returns NULL. - """ - payload = statement.payload - now = self.clock.now() - ttl = self._ttl(statement) - acquired = { - "acquired": True, - "writer_id": payload["writer_id"], - "writer_epoch": int(payload["writer_epoch"]), - "writer_session_token": payload["writer_session_token"], - } - - def claim() -> None: - self._stage( - transaction, - LeaseRow( - writer_id=str(payload["writer_id"]), - writer_epoch=int(payload["writer_epoch"]), - writer_session_token=str(payload["writer_session_token"]), - lease_expires_at=now + timedelta(seconds=ttl), - updated_at=now, - ), - ) - - # The conflict is resolved against the tuple this statement locked, - # not against its snapshot. - current = self._visible_lease(transaction) - if current is None: - claim() - return acquired - if self._is_exact_session(current, payload) or ( - current.lease_expires_at <= now - ): - claim() - return acquired - if snapshot is None: - return self._unobserved_not_acquired(statement) - return self._observed(snapshot, acquired=False) - - def _unobserved_not_acquired(self, statement: Statement) -> dict[str, Any] | None: - """Answer a race whose loser can see neither the upsert nor a holder. - - Which answer is correct is a property of the shipped statement, not of - this harness, so it is read off the statement every time. The two-arm - shape evaluates to SQL NULL here; the shape issue #140 adds carries a - terminal arm and therefore cannot. - """ - if acquire_result_arms(statement.sql) == 2: - return None - payload: dict[str, Any] = {} - for key, value_text in acquire_terminal_arm_fields(statement.sql): - payload[key] = self._terminal_arm_value(key, value_text) - if "acquired" not in payload: - raise UnsupportedStatement( - "the acquire statement's terminal arm does not name 'acquired'; " - "the lease evaluator cannot report an outcome from it" - ) - return payload - - @staticmethod - def _terminal_arm_value(key: str, value_text: str) -> Any: - """Answer one terminal-arm key from what the statement says it emits. - - The key set stays an allowlist -- an unknown key is refused even when - its value is a perfectly readable literal, because a key this model - has never seen is a shape nobody has reasoned about. What is *not* - guessed is the value: for the literal keys it is parsed out of the - SQL, so the model reports the wording the ledger PR chose rather than - a restatement of it that can drift. - """ - text = value_text.strip() - if key in _ACQUIRE_TERMINAL_LITERAL_KEYS: - lowered = text.lower() - if lowered == "true": - return True - if lowered == "false": - return False - if lowered == "null": - return None - if len(text) >= 2 and text.startswith("'") and text.endswith("'"): - return _unquote(text[1:-1]) - raise UnsupportedStatement( - f"the acquire statement's terminal arm gives {key!r} the " - f"non-literal value {text!r}; this model can only report a " - "value the statement states outright" - ) - if key in _ACQUIRE_TERMINAL_NULL_KEYS: - # A lease column with no visible row behind it. - return None - raise UnsupportedStatement( - f"the acquire statement's terminal arm names {key!r}, which this " - "model does not know how to answer. Teach it that key rather than " - "letting a scenario assert against a guess." - ) - - @staticmethod - def _adopt_matches(statement: Statement, row: LeaseRow) -> bool: - payload = statement.payload - return ( - row.writer_id == payload["writer_id"] - and row.writer_epoch == int(payload["writer_epoch"]) - and row.writer_session_token - == payload["observed_writer_session_token"] - and VirtualClock.timestamp_text(row.updated_at) - == payload["observed_lease_updated_at"] - ) - - @staticmethod - def _renew_matches(statement: Statement, row: LeaseRow) -> bool: - return FakePostgres._is_exact_session(row, statement.payload) - - _release_matches = _renew_matches - - def _evaluate_adopt( - self, - statement: Statement, - transaction: Transaction, - matched: LeaseRow | None, - ) -> dict[str, Any] | None: - payload = statement.payload - now = self.clock.now() - ttl = self._ttl(statement) - # `lease_expires_at > clock_timestamp()` is evaluated with the rest of - # the predicate but depends on live time rather than on the row - # version, so it is checked here rather than in _adopt_matches. - if matched is not None and matched.lease_expires_at > now: - self._stage( - transaction, - replace( - matched, - writer_session_token=str(payload["writer_session_token"]), - lease_expires_at=now + timedelta(seconds=ttl), - updated_at=now, - xmax=None, - ), - ) - return { - "acquired": True, - "adopted": True, - "writer_id": payload["writer_id"], - "writer_epoch": int(payload["writer_epoch"]), - "writer_session_token": payload["writer_session_token"], - } - # The CAS affected no rows, so the COALESCE fallback reports whatever - # this statement's snapshot holds. - snapshot = self._visible_lease(transaction) - if snapshot is None: - return None - observed = self._observed(snapshot, acquired=False) - observed["adopted"] = False - return observed - - def _evaluate_renew( - self, - statement: Statement, - transaction: Transaction, - matched: LeaseRow | None, - ) -> dict[str, Any]: - now = self.clock.now() - ttl = self._ttl(statement) - if matched is None: - return {"error": "writer lease is not active"} - self._stage( - transaction, - replace( - matched, - lease_expires_at=now + timedelta(seconds=ttl), - updated_at=now, - xmax=None, - ), - ) - return {"backend": "postgres-psql", "renewed_count": 1} - - def _evaluate_release( - self, - statement: Statement, - transaction: Transaction, - matched: LeaseRow | None, - ) -> dict[str, Any]: - now = self.clock.now() - if matched is None: - return {"released": 0} - self._stage( - transaction, - replace( - matched, - lease_expires_at=now - timedelta(seconds=1), - updated_at=now, - xmax=None, - ), - ) - return {"released": 1} - - def _evaluate_prove( - self, - statement: Statement, - transaction: Transaction, - ) -> dict[str, Any]: - """The read-only ownership proof: no row lock, no locker attribution. - - Everything it reads comes from one MVCC snapshot plus this backend's - own advisory-lock state, so unlike VERIFY there is no - snapshot-versus-live-state window and no interleaving point inside - the statement. That is the whole point of the split: the frequent - statement cannot queue, cannot misattribute, and cannot need a - second round trip. - """ - payload = statement.payload - now = self.clock.now() - margin = statement.authority_margin_seconds or 0.0 - snapshot = self._visible_lease(transaction) - exact = snapshot is not None and self._is_exact_session(snapshot, payload) - guard_backend = transaction.backend - advisory_held = ( - statement.advisory_lock is not None - and statement.advisory_lock in guard_backend.advisory_locks - and self._advisory_owners.get(statement.advisory_lock) is guard_backend - and not guard_backend.closed - ) - return { - "backend": "postgres-psql", - "guard_advisory_lock_held": bool(advisory_held), - "writer_session_token_current": bool(exact), - "lease_expired": bool( - exact and snapshot.lease_expires_at <= now - ), - "lease_renewal_due": bool( - exact - and snapshot.lease_expires_at <= now + timedelta(seconds=margin) - ), - "server_execution_seconds": float( - self._last_statement_server_seconds - ), - } - - def _evaluate_verify( - self, - statement: Statement, - transaction: Transaction, - tag: str, - ) -> dict[str, Any]: - payload = statement.payload - now = self.clock.now() - ttl = self._ttl(statement) - margin = statement.authority_margin_seconds or 0.0 - # One MVCC snapshot for every lease-row read in this statement. - snapshot = self._visible_lease(transaction) - exact = snapshot is not None and self._is_exact_session(snapshot, payload) - - # `renewable ... FOR NO KEY UPDATE SKIP LOCKED` never queues. - renewed = 0 - if exact and not self.lease_lock_held_by_other(transaction): - assert snapshot is not None - self._take_lease_lock(transaction) - self._stage( - transaction, - replace( - snapshot, - lease_expires_at=now + timedelta(seconds=ttl), - updated_at=now, - xmax=None, - ), - ) - renewed = 1 - - # The documented misfire window: the statement's row reads come from - # `snapshot`, while pg_stat_activity is read live. A checkpoint here - # lets a scenario commit an own fenced write between the two. - self.scheduler.checkpoint(f"{tag}.snapshot:verify") - - guard_backend = transaction.backend - advisory_held = ( - statement.advisory_lock is not None - and statement.advisory_lock in guard_backend.advisory_locks - and self._advisory_owners.get(statement.advisory_lock) is guard_backend - and not guard_backend.closed - ) - locked_by_this_process = any( - backend.application_name == payload.get("pool_application_name") - and backend.backend_xid is not None - and snapshot is not None - and backend.backend_xid == snapshot.xmax - for backend in self.live_backends() - ) - return { - "backend": "postgres-psql", - "guard_advisory_lock_held": bool(advisory_held), - "writer_session_token_current": bool(exact), - "lease_renewed_count": renewed, - "lease_expired": bool(exact and snapshot.lease_expires_at <= now), - "lease_expiring_within_authority_margin": bool( - exact - and snapshot.lease_expires_at <= now + timedelta(seconds=margin) - ), - "lease_locked_by_this_process": bool(locked_by_this_process), - "server_execution_seconds": float( - self._last_statement_server_seconds - ), - } - - # -- block candidate landing ------------------------------------------- - - def _visible_pool_block( - self, - transaction: Transaction, - block_hash: str, - ) -> PoolBlockRow | None: - """READ COMMITTED: committed rows plus this transaction's own writes.""" - staged = transaction.staged_pool_blocks.get(block_hash) - if staged is not None: - return staged - return self.pool_blocks.get(block_hash) - - def _visible_pool_blocks( - self, - transaction: Transaction, - ) -> dict[str, PoolBlockRow]: - merged = dict(self.pool_blocks) - merged.update(transaction.staged_pool_blocks) - return merged - - def _stage_pool_block( - self, - transaction: Transaction, - row: PoolBlockRow, - ) -> PoolBlockRow: - staged = replace(row) - transaction.staged_pool_blocks[staged.block_hash] = staged - return staged - - def _visible_outbox_row( - self, - transaction: Transaction, - block_hash: str, - ) -> OutboxRow | None: - staged = transaction.staged_outbox.get(block_hash) - if staged is not None: - return staged - return self.outbox.get(block_hash) - - def _visible_body(self, transaction: Transaction, body_id: str) -> BodyRow | None: - if body_id in transaction.deleted_bodies: - return None - staged = transaction.staged_bodies.get(body_id) - if staged is not None: - return staged - return self.bodies.get(body_id) - - def _stage_body(self, transaction: Transaction, body: BodyRow) -> BodyRow: - staged = transaction.staged_bodies.get(body.body_id) - if staged is None: - staged = body.copy() - transaction.staged_bodies[body.body_id] = staged - return staged - - def _body_referenced(self, transaction: Transaction, body_id: str, *, pending_only: bool) -> bool: - visible = dict(self.outbox) - visible.update(transaction.staged_outbox) - return any( - row.body_id == body_id and (not pending_only or row.state == "pending") - for row in visible.values() - ) - - def _renew_lease_for_landing( - self, - statement: Statement, - transaction: Transaction, - timeout_seconds: float | None, - tag: str, - timeout_kind: str, - ) -> bool: - """The ``lease AS (UPDATE ...)`` CTE every landing statement opens with. - - Every landing statement fences itself on the writer lease and renews - it in the same round trip. Modelling that is not decoration: a - deposed writer's landing statement fails here, which is the only - thing standing between a fenced-out process and a durable pool-block - write. The identity comes out of the statement itself — inlined as - literals by the ledger — so the model never has to be told which - writer a session belongs to. - - The renewal is an ordinary UPDATE on the lease tuple, so it obeys the - same rules as every other one: a predicate the snapshot already - excludes returns zero rows without queueing, and a matching predicate - takes the tuple lock and waits behind whoever holds it. A landing - stalled behind an orphaned lease transaction is a real outage shape, - and the model has to be able to express it. - """ - payload = statement.payload - writer_id = payload.get("writer_id") - writer_epoch = payload.get("writer_epoch") - session_token = payload.get("writer_session_token") - if writer_id is None or session_token is None: - # A statement whose identity the classifier could not recover is - # a model gap, not a lease failure; fail loudly rather than - # inventing a fence outcome. - raise UnsupportedStatement( - f"{statement.kind.value} statement carried no writer identity" - ) - identity = { - "writer_id": writer_id, - "writer_epoch": writer_epoch, - "writer_session_token": session_token, - } - - def matches(row: LeaseRow | None) -> bool: - return row is not None and self._is_exact_session(row, identity) - - if not matches(self._visible_lease(transaction)): - return False - self._await_lease_lock( - statement, - transaction, - timeout_seconds, - tag, - timeout_kind, - ) - current = self._visible_lease(transaction) - if not matches(current): - return False - assert current is not None - now = self.clock.now() - ttl = statement.lease_ttl_seconds - if ttl is not None: - self._stage( - transaction, - replace( - current, - lease_expires_at=now + timedelta(seconds=ttl), - updated_at=now, - xmax=None, - ), - ) - return True - - def _evaluate_landing( - self, - statement: Statement, - transaction: Transaction, - timeout_seconds: float | None, - tag: str, - timeout_kind: str, - ) -> Any: - kind = statement.kind - assert isinstance(kind, LandingOp) - payload = statement.payload - # Reads that open no lease CTE at all. - if kind is LandingOp.PUBLICATION_FLOOR: - return { - "audit_publication_sequence_floor": max( - [ - row.audit_publication_sequence or 0 - for row in self._visible_pool_blocks(transaction).values() - ], - default=0, - ) - } - if kind is LandingOp.ALL_SHARES: - return [dict(share) for share in self.shares] - if kind is LandingOp.SHARE_STATS: - # The evidence annotations, not an accounting input: the audit - # envelope carries them and the publication replay path reuses - # the originally durable values rather than re-reading. - return { - "accepted_share_count": len(self.shares), - "distinct_miner_count": len( - {str(share.get("miner_id")) for share in self.shares} - ), - "max_share_seq": max( - (int(share.get("share_seq", 0)) for share in self.shares), - default=0, - ), - } - if kind in {LandingOp.PRIOR_BALANCES, LandingOp.PRIOR_BALANCES_AS_OF}: - # The durable carry-forward view. A landing reads it to prove the - # payout base under its coinbase has not moved since the job was - # issued, so a scenario that wants to fail that check moves this - # list rather than reaching into the coordinator. The as-of - # variant restricts the same view to blocks at or below one - # confirmed height; with no payout entries modelled the two - # coincide, and a scenario that needs them to differ is asking - # for the settlement machine, which #128 lists separately. - return [dict(balance) for balance in self.carry_forward_balances] - if kind is LandingOp.POOL_BLOCK_STATE: - row = self._visible_pool_block(transaction, str(payload["block_hash"])) - return { - "state": None - if row is None - else { - "block_hash": row.block_hash, - "block_height": row.block_height, - "parent_hash": row.parent_hash, - "chain_state": row.chain_state, - "maturity_state": row.maturity_state, - "audit_publication_sequence": row.audit_publication_sequence, - } - } - if kind is LandingOp.OUTBOX_PENDING_PAGE: - # Deliberately grouped with the reads above, before - # _renew_lease_for_landing: the production statement opens no - # lease CTE, so a model that renewed here would report a - # lease-touching read the ledger does not perform -- and would - # hide the very property issue #211 turns on, that enumeration - # neither takes the writer gate nor touches the lease row. - return self._pending_candidate_page(transaction, payload) - if kind is LandingOp.OUTBOX_HEADER_PAGE: - return self._header_page(transaction, payload) - if kind is LandingOp.OUTBOX_OBSERVE: - row = self._visible_outbox_row(transaction, str(payload["block_hash"])) - if row is None: - return {"found": False} - body = None if row.body_id is None else self._visible_body(transaction, row.body_id) - return { - "found": True, - "state": row.state, - "share_id": None, - "candidate_sha256": row.candidate_sha256, - "storage_version": row.storage_version, - "body_id": row.body_id, - "body_state": None if body is None else body.state, - } - if kind is LandingOp.SCHEMA_PRESENCE: - return {"has_body_table": True, "has_capabilities": True} - if kind is LandingOp.SCHEMA_CAPABILITY: - return {"declared": 2} - if kind in { - LandingOp.BODY_MANIFEST, - LandingOp.BODY_SPANS, - LandingOp.BODY_PAGES, - LandingOp.BODY_PAGE, - }: - return self._body_read(kind, transaction, payload) - if kind in { - LandingOp.BODY_STAGE, - LandingOp.BODY_CHUNK, - LandingOp.BODY_INDEX, - LandingOp.BODY_SEAL, - LandingOp.BODY_RETIRE_STAGING, - LandingOp.BODY_REAP, - }: - # Staging, sealing and reaping open no lease CTE: they are private - # preparation (or reclamation of a permanently retired body) and - # a model that renewed here would misreport a lease touch. - return self._body_write(kind, transaction, payload) - if kind is LandingOp.CONFIRMED_SEQUENCE: - row = self._visible_pool_block(transaction, str(payload["block_hash"])) - confirmed = ( - row is not None - and row.chain_state == "confirmed" - and row.maturity_state != "reversed" - ) - return { - "audit_publication_sequence": ( - row.audit_publication_sequence if confirmed else None - ) - } - - lease_held = self._renew_lease_for_landing( - statement, - transaction, - timeout_seconds, - tag, - timeout_kind, - ) - if kind in {LandingOp.CONFIRM_BLOCK, LandingOp.REJECT_PREPARED}: - if not lease_held: - # The PL/pgSQL function RAISEs rather than returning a value, - # so the caller sees an error, not a zero count. - raise RuntimeError("writer lease is not active") - if kind is LandingOp.CONFIRM_BLOCK: - return { - "backend": "postgres-psql", - "confirmed_count": self._confirm_pool_block( - transaction, - str(payload["block_hash"]), - int(payload["active_tip_height"]), - ), - } - return { - "backend": "postgres-psql", - "rejected_count": self._reject_prepared_pool_block( - transaction, - str(payload["block_hash"]), - ), - } - - if not lease_held: - return {"error": "writer lease is not active"} - if kind is LandingOp.PERSIST_BLOCK: - return self._persist_pool_block(statement, transaction) - if kind is LandingOp.OUTBOX_RECORD: - return self._record_outbox_row(statement, transaction) - if kind is LandingOp.BODY_RETIRE_ORPHANS: - return self._retire_orphan_bodies(transaction, payload) - if kind is LandingOp.OUTBOX_BATCH_ABANDON: - # The pool-block clause lives *inside* this fenced UPDATE for the - # reason issue #211 depends on: the caller's page fact is advisory - # by the time it gets here, and a candidate that landed in between - # must be absent from the returned set rather than abandoned. - pool_blocks = self._visible_pool_blocks(transaction) - abandoned: list[str] = [] - for block_hash in payload["block_hashes"]: - row = self._visible_outbox_row(transaction, str(block_hash)) - if row is None or row.state != "pending": - continue - if row.block_hash in pool_blocks: - continue - staged = replace(row) - staged.state = "abandoned" - staged.last_error = payload.get("last_error") - self._detach_body(transaction, staged) - transaction.staged_outbox[staged.block_hash] = staged - abandoned.append(staged.block_hash) - return {"abandoned": sorted(abandoned)} - if kind in {LandingOp.OUTBOX_ATTEMPT, LandingOp.OUTBOX_FINISH}: - row = self._visible_outbox_row(transaction, str(payload["block_hash"])) - if row is None or row.state != "pending": - return {"updated": 0} - staged = replace(row) - if kind is LandingOp.OUTBOX_ATTEMPT: - staged.attempt_count += 1 - else: - staged.state = str(payload["state"]) - staged.last_error = payload.get("last_error") - self._detach_body(transaction, staged) - transaction.staged_outbox[staged.block_hash] = staged - return {"updated": 1} - raise UnsupportedStatement(f"unhandled landing operation {kind}") - - # -- chunked candidate bodies (#255) ------------------------------------ - - def _detach_body(self, transaction: Transaction, row: OutboxRow) -> None: - """The terminal write detaches and retires the body; chunks stay.""" - if row.body_id is None: - return - body = self._visible_body(transaction, row.body_id) - row.retired_body_id = row.body_id - row.body_id = None - if body is not None and body.state != "retired": - staged = self._stage_body(transaction, body) - staged.state = "retired" - staged.retired_at = self.clock.now() - - def _body_owner_matches(self, body: BodyRow, payload: dict[str, Any]) -> bool: - return ( - body.state == "staging" - and body.writer_id == payload.get("writer_id") - and body.writer_epoch == int(payload.get("writer_epoch", -1)) - and body.writer_session_token == payload.get("writer_session_token") - ) - - def _body_write( - self, - kind: LandingOp, - transaction: Transaction, - payload: dict[str, Any], - ) -> Any: - if kind is LandingOp.BODY_STAGE: - body_id = str(payload["body_id"]) - existing = self._visible_body(transaction, body_id) - if existing is not None: - return {"staged": 0, "state": existing.state} - transaction.staged_bodies[body_id] = BodyRow( - body_id=body_id, - block_hash=str(payload["block_hash"]), - candidate_sha256=str(payload["candidate_sha256"]), - byte_count=int(payload["byte_count"]), - chunk_count=int(payload["chunk_count"]), - chunk_bytes=int(payload["chunk_bytes"]), - share_count=int(payload["share_count"]), - shares_offset=int(payload["shares_offset"]), - shares_end=int(payload["shares_end"]), - span_count=int(payload.get("span_count", 0)), - page_count=int(payload.get("page_count", 0)), - writer_id=str(payload["writer_id"]), - writer_epoch=int(payload["writer_epoch"]), - writer_session_token=str(payload["writer_session_token"]), - created_at=self.clock.now(), - ) - return {"staged": 1, "state": "staging"} - if kind in {LandingOp.BODY_CHUNK, LandingOp.BODY_INDEX}: - body = self._visible_body(transaction, str(payload["body_id"])) - if body is None or not self._body_owner_matches(body, payload): - return {"owned": False, "inserted": 0, "spans": 0, "pages": 0} - staged = self._stage_body(transaction, body) - if kind is LandingOp.BODY_CHUNK: - ordinal = int(payload["ordinal"]) - if ordinal in staged.chunks: - return {"owned": True, "inserted": 0} - staged.chunks[ordinal] = (bytes(payload["chunk"]), str(payload["chunk_sha256"])) - return {"owned": True, "inserted": 1} - spans = 0 - pages = 0 - for span in payload.get("spans") or []: - if span["field"] not in staged.spans: - staged.spans[str(span["field"])] = dict(span) - spans += 1 - for page in payload.get("pages") or []: - key = (str(page[0]), int(page[1])) - if key not in staged.pages: - staged.pages[key] = (int(page[2]), int(page[3])) - pages += 1 - return {"owned": True, "spans": spans, "pages": pages} - if kind is LandingOp.BODY_SEAL: - body = self._visible_body(transaction, str(payload["body_id"])) - if body is None: - return {"sealed": 0, "state": None} - ordinals = sorted(body.chunks) - byte_count = sum(len(data) for data, _ in body.chunks.values()) - digests_ok = all( - hashlib.sha256(data).hexdigest() == digest for data, digest in body.chunks.values() - ) - lengths_ok = all( - len(body.chunks[ordinal][0]) == body.chunk_bytes or ordinal == body.chunk_count - 1 - for ordinal in ordinals - ) - complete = ( - self._body_owner_matches(body, payload) - and body.candidate_sha256 == payload.get("candidate_sha256") - and len(ordinals) == body.chunk_count - and byte_count == body.byte_count - and (body.chunk_count == 0 or ordinals[0] == 0) - and (ordinals[-1] if ordinals else -1) == body.chunk_count - 1 - and digests_ok - and lengths_ok - and len(body.spans) == body.span_count - and len(body.pages) == body.page_count - ) - if not complete: - return { - "sealed": 0, - "state": body.state, - "observed_chunk_count": len(ordinals), - "observed_byte_count": byte_count, - "observed_span_count": len(body.spans), - "observed_page_count": len(body.pages), - "digests_ok": digests_ok, - "lengths_ok": lengths_ok, - } - staged = self._stage_body(transaction, body) - staged.state = "sealed" - staged.sealed_at = self.clock.now() - return {"sealed": 1, "state": "staging"} - if kind is LandingOp.BODY_RETIRE_STAGING: - body = self._visible_body(transaction, str(payload["body_id"])) - if body is None or body.state != "staging": - return {"retired": 0} - staged = self._stage_body(transaction, body) - staged.state = "retired" - staged.retired_at = self.clock.now() - return {"retired": 1} - if kind is LandingOp.BODY_REAP: - visible = { - body_id: body - for body_id, body in {**self.bodies, **transaction.staged_bodies}.items() - if body_id not in transaction.deleted_bodies - } - retired = sorted( - (body for body in visible.values() if body.state == "retired"), - key=lambda body: (body.retired_at or CLOCK_ORIGIN, body.body_id), - ) - if not retired: - return {"body_id": None, "deleted_chunks": 0, "deleted_bodies": 0} - target = self._stage_body(transaction, retired[0]) - max_chunks = max(1, int(payload.get("max_chunks", 1))) - # The production statement's snapshot decides which table this - # step works on: chunks first, then pages once no chunk was - # visible, then spans, then the manifest on a later step. - before = { - "chunks": bool(target.chunks), - "pages": bool(target.pages), - "spans": bool(target.spans), - } - chunk_victims = sorted(target.chunks)[:max_chunks] - for ordinal in chunk_victims: - del target.chunks[ordinal] - page_victims = [] if before["chunks"] else sorted(target.pages)[:max_chunks] - for key in page_victims: - del target.pages[key] - span_victims = [] if before["chunks"] or before["pages"] else sorted(target.spans)[:max_chunks] - for key in span_victims: - del target.spans[key] - deleted_bodies = 0 - if not any(before.values()): - transaction.deleted_bodies.add(target.body_id) - transaction.staged_bodies.pop(target.body_id, None) - deleted_bodies = 1 - return { - "body_id": target.body_id, - "deleted_chunks": len(chunk_victims), - "deleted_pages": len(page_victims), - "deleted_spans": len(span_victims), - "deleted_bodies": deleted_bodies, - "remaining": before, - } - raise UnsupportedStatement(f"unhandled body write {kind}") - - def _retire_orphan_bodies( - self, - transaction: Transaction, - payload: dict[str, Any], - ) -> Any: - stale = timedelta(seconds=float(payload.get("stale_staging_seconds", 0.0))) - now = self.clock.now() - visible = { - body_id: body - for body_id, body in {**self.bodies, **transaction.staged_bodies}.items() - if body_id not in transaction.deleted_bodies - } - candidates = [] - for body in sorted(visible.values(), key=lambda body: (body.created_at, body.body_id)): - if self._body_referenced(transaction, body.body_id, pending_only=False): - continue - foreign = not ( - body.writer_id == payload.get("writer_id") - and body.writer_session_token == payload.get("writer_session_token") - ) - if body.state == "staging" and foreign and body.created_at < now - stale: - candidates.append(body) - elif body.state == "sealed" and body.sealed_at is not None and body.sealed_at < now - stale: - candidates.append(body) - retired: list[str] = [] - for body in candidates[:1]: - # The manifest trigger's invariant: a referenced body is never - # retired by orphan cleanup. - if self._body_referenced(transaction, body.body_id, pending_only=False): - raise RuntimeError( - f"candidate body {body.body_id} is referenced by a pending outbox row" - ) - staged = self._stage_body(transaction, body) - staged.state = "retired" - staged.retired_at = now - retired.append(body.body_id) - return {"retired": retired} - - def _body_read( - self, - kind: LandingOp, - transaction: Transaction, - payload: dict[str, Any], - ) -> Any: - body = self._visible_body(transaction, str(payload["body_id"])) - if kind is LandingOp.BODY_MANIFEST: - if body is None: - return {"found": False, "manifest": None} - return { - "found": True, - "manifest": { - "body_id": body.body_id, - "storage_version": 2, - "block_hash": body.block_hash, - "candidate_sha256": body.candidate_sha256, - "byte_count": body.byte_count, - "chunk_count": body.chunk_count, - "chunk_bytes": body.chunk_bytes, - "share_count": body.share_count, - "shares_offset": body.shares_offset, - "shares_end": body.shares_end, - "span_count": body.span_count, - "page_count": body.page_count, - "state": body.state, - "referenced": self._body_referenced(transaction, body.body_id, pending_only=True), - }, - } - state = None if body is None else body.state - if kind is LandingOp.BODY_SPANS: - after = payload.get("after_field") - spans = [] if body is None else sorted(body.spans.values(), key=lambda span: span["field"]) - spans = [span for span in spans if after is None or span["field"] > after][:INDEX_SPAN_ROWS] - return {"state": state, "spans": spans} - if kind is LandingOp.BODY_PAGES: - field_name = str(payload["field"]) - from_ordinal = int(payload["from_ordinal"]) - entries = [] if body is None else sorted( - ( - [ordinal, offset, record_index] - for (name, ordinal), (offset, record_index) in body.pages.items() - if name == field_name and ordinal >= from_ordinal - ), - ) - return {"state": state, "pages": entries[:INDEX_PAGE_ENTRIES]} - if kind is LandingOp.BODY_PAGE: - from_ordinal = int(payload["from_ordinal"]) - max_chunks = max(1, min(int(payload["max_chunks"]), BODY_READ_MAX_CHUNKS)) - chunks = [] if body is None else [ - { - "ordinal": ordinal, - "sha256": body.chunks[ordinal][1], - # PostgreSQL wraps base64 at 76 characters with newlines. - "base64": base64.encodebytes(body.chunks[ordinal][0]).decode("ascii").rstrip("\n"), - } - for ordinal in sorted(body.chunks) - if ordinal >= from_ordinal - ][:max_chunks] - return { - "state": state, - "referenced": body is not None - and self._body_referenced(transaction, body.body_id, pending_only=True), - "chunks": chunks, - } - raise UnsupportedStatement(f"unhandled body read {kind}") - - def _header_page(self, transaction: Transaction, payload: dict[str, Any]) -> Any: - """The metadata-first page (#255): bounded headers, explicit exhaustion.""" - visible = dict(self.outbox) - visible.update(transaction.staged_outbox) - pool_blocks = self._visible_pool_blocks(transaction) - after = payload.get("after_cursor") - limit = max(1, int(payload["limit"])) - max_bytes = max(1, int(payload["max_bytes"])) - rows = sorted( - (row for row in visible.values() if row.state == "pending"), - key=lambda row: (row.created_at, row.block_hash), - ) - fetched: list[OutboxRow] = [] - for row in rows: - cursor_stamp = self._pending_cursor_stamp(row.created_at) - if after is not None and (cursor_stamp, row.block_hash) <= (str(after[0]), str(after[1])): - continue - fetched.append(row) - if len(fetched) > limit: - break - page: list[dict[str, Any]] = [] - running = 0 - for row in fetched[:limit]: - header = replay_header_from_fields(row.header or {"block_hash_hex": row.block_hash}) - header_bytes = len(json.dumps(header, separators=(",", ":"))) - if page and running + header_bytes > max_bytes: - break - running += header_bytes - body = None if row.body_id is None else self._visible_body(transaction, row.body_id) - page.append( - { - "block_hash": row.block_hash, - "storage_version": row.storage_version, - "candidate_sha256": row.candidate_sha256, - "header": header, - "header_bytes": header_bytes, - "body": None if body is None else { - "body_id": body.body_id, - "storage_version": 2, - "candidate_sha256": body.candidate_sha256, - "byte_count": body.byte_count, - "chunk_count": body.chunk_count, - "chunk_bytes": body.chunk_bytes, - "share_count": body.share_count, - "state": body.state, - }, - "pool_block_exists": row.block_hash in pool_blocks, - "cursor": [self._pending_cursor_stamp(row.created_at), row.block_hash], - } - ) - return {"fetched": len(fetched), "returned": len(page), "rows": page} - - def _confirm_pool_block( - self, - transaction: Transaction, - block_hash: str, - active_tip_height: int, - ) -> int: - """``qbit_confirm_pool_block``: allocate the publication ordinal. - - The ordinal comes from a sequence, not from ``MAX + 1``. That - distinction is the whole of the allocator's contract: an exact replay - of an already-confirmed row matches zero rows in the UPDATE and so - burns nothing, while a terminally disposed row reports the superseded - disposition (-1) instead of a plain miss. The durable floor read is a - separate MAX, which is what lets a *later* ordinal publish first and - leave an earlier one behind the floor — the state #133 turns on. - """ - row = self._visible_pool_block(transaction, block_hash) - if ( - row is not None - and row.block_height == active_tip_height - and row.chain_state == "prepared" - and row.maturity_state == "immature" - ): - self._audit_publication_sequence += 1 - staged = self._stage_pool_block(transaction, row) - staged.chain_state = "confirmed" - staged.audit_publication_sequence = self._audit_publication_sequence - return 1 - if ( - row is not None - and row.block_height == active_tip_height - and row.chain_state == "confirmed" - and row.maturity_state != "reversed" - ): - return 1 - if row is not None and ( - row.chain_state in {"inactive", "rejected", "reversed"} - or row.maturity_state == "reversed" - ): - return -1 - return 0 - - def _reject_prepared_pool_block( - self, - transaction: Transaction, - block_hash: str, - ) -> int: - row = self._visible_pool_block(transaction, block_hash) - if ( - row is None - or row.chain_state != "prepared" - or row.maturity_state != "immature" - ): - return 0 - staged = self._stage_pool_block(transaction, row) - staged.chain_state = "rejected" - staged.maturity_state = "reversed" - return 1 - - def _persist_pool_block( - self, - statement: Statement, - transaction: Transaction, - ) -> dict[str, Any]: - """``persist_accepted_block``, reduced to what a landing reads back. - - The production statement writes the pool block, its audit bundle, the - payout entries and the carry-forward deltas in one CTE. The landing - decisions downstream read the prepared row (through - ``pool_block_state`` and the confirmation) and the returned counts; - the payout rows are the reorg reconciler's and the settlement - machine's business, neither of which this harness drives yet. The - counts below are therefore reported from what is modelled, and the - row shape stays honest: a re-persist of an existing block reports the - existing row rather than inserting a second one. - """ - payload = statement.payload - block_hash = str(payload["block_hash"]) - existing = self._visible_pool_block(transaction, block_hash) - if existing is None: - transaction.staged_pool_blocks[block_hash] = PoolBlockRow( - block_hash=block_hash, - block_height=int(payload["block_height"]), - parent_hash=str(payload["parent_hash"]), - ) - elif existing.block_height != int(payload["block_height"]): - raise RuntimeError("pool block height conflicts") - accounts = payload.get("accounts") or [] - return { - "backend": "postgres-psql", - "share_count": len(self.shares), - "block_count": 1, - "bundle_count": 1, - "payout_entry_count": len(accounts), - "carry_forward_count": 0, - "onchain_output_count": 0, - } - - def _pending_candidate_page( - self, - transaction: Transaction, - payload: dict[str, Any], - ) -> list[dict[str, Any]]: - """One snapshot answering row, landed-block fact and cursor together. - - The three facts come from one visible set here for the same reason - they come from one statement in production: a caller that read the - rows and their pool-block facts at different instants could see a - candidate as pending *and* unlanded when it was neither. - - The order and the keyset predicate are ``(created_at, block_hash)``, - the total order the production index provides. The block-hash - tiebreak is not decoration -- creation stamps collide under one - transaction's ``clock_timestamp()``, and a cursor on the stamp alone - would replay or skip a whole colliding group. - """ - visible = dict(self.outbox) - visible.update(transaction.staged_outbox) - pool_blocks = self._visible_pool_blocks(transaction) - after = payload.get("after_cursor") - rows = sorted( - ( - row - for row in visible.values() - if row.state == "pending" - ), - key=lambda row: (row.created_at, row.block_hash), - ) - page: list[dict[str, Any]] = [] - for row in rows: - cursor_stamp = self._pending_cursor_stamp(row.created_at) - if after is not None and (cursor_stamp, row.block_hash) <= ( - str(after[0]), - str(after[1]), - ): - continue - page.append( - { - "block_hash": row.block_hash, - "candidate": {"block_hash_hex": row.block_hash}, - "storage_version": row.storage_version, - "candidate_sha256": row.candidate_sha256, - "pool_block_exists": row.block_hash in pool_blocks, - "cursor": [cursor_stamp, row.block_hash], - } - ) - if len(page) >= int(payload["limit"]): - break - return page - - @staticmethod - def _pending_cursor_stamp(moment: datetime) -> str: - """``to_char(created_at AT TIME ZONE 'UTC', ...US"Z"')``, exactly. - - Microsecond precision with an explicit UTC marker, because the cursor - is compared as text and a truncated stamp would re-emit or skip whole - sub-second groups. - """ - return ( - f"{moment.astimezone(timezone.utc):%Y-%m-%dT%H:%M:%S}" - f".{moment.microsecond:06d}Z" - ) - - def _record_outbox_row( - self, - statement: Statement, - transaction: Transaction, - ) -> dict[str, Any]: - payload = statement.payload - block_hash = str(payload["block_hash"]) - digest = str(payload["candidate_sha256"]) - existing = self._visible_outbox_row(transaction, block_hash) - if existing is None: - body_id = payload.get("body_id") - if body_id is not None: - # A version-2 reference may point only at a sealed body with - # the same digest and block hash (#255). - body = self._visible_body(transaction, str(body_id)) - if ( - body is None - or body.state != "sealed" - or body.candidate_sha256 != digest - or body.block_hash != block_hash - ): - return {"error": "block candidate body is not sealed"} - transaction.staged_outbox[block_hash] = OutboxRow( - block_hash=block_hash, - candidate_sha256=digest, - created_at=self.clock.now(), - storage_version=1 if body_id is None else 2, - body_id=None if body_id is None else str(body_id), - header=payload.get("header"), - ) - return {"inserted": 1, "state": "pending"} - if existing.candidate_sha256 != digest: - return {"error": "block candidate payload mismatch"} - return {"inserted": 0, "state": existing.state} - - # -- helpers ----------------------------------------------------------- - - @staticmethod - def _is_exact_session(row: LeaseRow, payload: dict[str, Any]) -> bool: - return ( - row.writer_id == payload["writer_id"] - and row.writer_epoch == int(payload["writer_epoch"]) - and row.writer_session_token == payload["writer_session_token"] - ) - - @staticmethod - def _ttl(statement: Statement) -> float: - if statement.lease_ttl_seconds is None: - raise UnsupportedStatement( - f"{statement.kind.value} statement carried no make_interval TTL" - ) - return statement.lease_ttl_seconds - - def _observed(self, row: LeaseRow, *, acquired: bool) -> dict[str, Any]: - now = self.clock.now() - return { - "acquired": acquired, - "writer_id": row.writer_id, - "writer_epoch": row.writer_epoch, - "writer_session_token": row.writer_session_token, - "lease_expires_at": VirtualClock.timestamp_text(row.lease_expires_at), - "lease_updated_at": VirtualClock.timestamp_text(row.updated_at), - "lease_age_seconds": max( - 0.0, (now - row.updated_at).total_seconds() - ), - "lease_wait_seconds": max( - 0.0, (row.lease_expires_at - now).total_seconds() - ), - } - - -# -------------------------------------------------------------------------- -# Ports -# -------------------------------------------------------------------------- - - -class FakeSqlBackend: - """``LedgerSqlPort`` over ``FakePostgres``. - - Mirrors ``_NativePostgresClient.run_json``'s transaction shape exactly, - because that shape is what #123 turns on: with no caller deadline the - statement runs in autocommit and commits with it, while a deadline-scoped - statement runs BEGIN / SET LOCAL / statement / COMMIT, leaving a window in - which the server holds the tuple lock and is waiting for a COMMIT the - client may never send. - """ - - def __init__( - self, - server: FakePostgres, - *, - pool_size: int, - application_name: str, - tag: str, - ) -> None: - self.server = server - self._pool_size = max(1, int(pool_size)) - self.application_name = application_name - self.tag = tag - self._free: list[Backend] = [ - server.connect(application_name=application_name) - for _ in range(self._pool_size) - ] - self._in_use: list[Backend] = [] - self.closed = False - - @property - def pool_size(self) -> int: - return self._pool_size - - def _acquire(self, timeout_seconds: float | None = None) -> Backend: - deadline = ( - None - if timeout_seconds is None - else self.server.clock.monotonic() + max(0.0, timeout_seconds) - ) - while not self._free: - if deadline is not None and self.server.clock.monotonic() >= deadline: - # _NativePostgresClient.connection raises exactly this rather - # than waiting past the caller's budget. - raise LedgerOperationTimeout( - "timed out waiting for a postgres pool slot" - ) - self.server.scheduler.block( - f"{self.tag}.pool:exhausted", - ready=lambda: bool(self._free), - wake_at=deadline, - ) - backend = self._free.pop(0) - self._in_use.append(backend) - return backend - - def _release(self, backend: Backend) -> None: - if backend in self._in_use: - self._in_use.remove(backend) - if not backend.closed and backend.transaction is None: - self._free.append(backend) - - def run_json( - self, - sql: str, - *, - retry_safe: bool = False, - timeout_seconds: float | None = None, - on_statement_start: Callable[[], None] | None = None, - ) -> Any: - if self.closed: - raise RuntimeError("connection pool is closed") - backend = self._acquire(timeout_seconds) - transaction: Transaction | None = None - try: - if on_statement_start is not None: - on_statement_start() - if timeout_seconds is not None: - # `with conn.transaction():` — SET LOCAL statement_timeout and - # lock_timeout, run the statement, then send COMMIT. - transaction = self.server._begin(backend, explicit=True) - result = self.server.execute( - backend, - sql, - transaction=transaction, - timeout_seconds=timeout_seconds, - tag=self.tag, - ) - if transaction is not None: - self.server.scheduler.checkpoint(f"{self.tag}.precommit") - self.server._commit(transaction) - # Production's own NULL handling, called rather than re-described: - # a statement that evaluates to SQL NULL raises here, exactly as - # it does for a real client. - return parse_single_json_value(result) - except LockTimeout as exc: - if transaction is not None and backend.transaction is transaction: - self.server._rollback(transaction) - if timeout_seconds is None: - raise - # Both shipped `LedgerSqlPort` implementations translate a - # cancellation caused by the caller's own deadline into - # `LedgerOperationTimeout`, and only when that deadline was armed: - # `_NativePostgresClient.run_json` gates on - # `timeout_seconds is not None and _is_postgres_deadline_error`, - # and the psql backend applies the same test to stderr. That - # translation is the contract callers retry against, so a model of - # this seam that skipped it would let a caller-side retry loop look - # dead here while working in production. Nothing exercised it - # before a deadline reached a pooled lease statement. - raise LedgerOperationTimeout( - f"postgres operation exceeded {timeout_seconds:g}s" - ) from exc - except BaseException: - if transaction is not None and backend.transaction is transaction: - self.server._rollback(transaction) - raise - finally: - self._release(backend) - - def run_json_rows( - self, - sql: str, - *, - retry_safe: bool = False, - timeout_seconds: float | None = None, - on_statement_start: Callable[[], None] | None = None, - row_converter: Callable[[Any], Any] | None = None, - batch_size: int = PAYOUT_WINDOW_ROW_BATCH_SIZE, - on_batch: Callable[[], None] | None = None, - ) -> list[Any]: - """``LedgerSqlPort.run_json_rows`` over the same modelled server. - - The row-result statements (#236: the payout-window snapshot and - delta) take the identical connection, transaction and deadline - shape as ``run_json``; only the result differs, and the model - answers one with the list of row values. ``FakePostgres`` models no - such statement today, so ``classify`` rejects the SQL with - ``UnsupportedStatement`` exactly as it rejects any other statement - outside the lease vocabulary. Kept here so the fake satisfies the - port it stands in for, and so a scenario that does reach the - payout-window read fails on the model's vocabulary rather than on - a missing method. - """ - rows = self.run_json( - sql, - retry_safe=retry_safe, - timeout_seconds=timeout_seconds, - on_statement_start=on_statement_start, - ) - if not isinstance(rows, list): - raise UnsupportedStatement( - "row-result statement must be modelled as a list of row values" - ) - batch_size = max(1, int(batch_size)) - results: list[Any] = [] - for start in range(0, len(rows), batch_size): - if start and on_batch is not None: - on_batch() - batch = rows[start : start + batch_size] - results.extend( - batch if row_converter is None else [row_converter(v) for v in batch] - ) - return results - - def run_script(self, sql: str) -> None: - raise UnsupportedStatement( - "schema initialization is not modelled; construct the ledger with " - "initialize_schema=False" - ) - - def close(self) -> None: - self.closed = True - for backend in [*self._free, *self._in_use]: - if backend.transaction is not None and backend.transaction.orphaned: - # A vanished client's backend outlives the object that owned - # it; closing the pool locally does not reach the server. - continue - self.server.disconnect(backend) - self._free.clear() - - -class FakeLeaseGuard: - """``LeaseGuardPort`` over ``FakePostgres``. - - The guard session is autocommit, carries a server-side - ``statement_timeout``, and serializes callers through one query slot. - That slot is modelled with a scheduler-aware lock so a second caller - parks at a named checkpoint instead of deadlocking the baton. - """ - - def __init__( - self, - server: FakePostgres, - *, - advisory_lock_key: int, - application_name: str, - tag: str, - ) -> None: - self.server = server - self.advisory_lock_key = advisory_lock_key - self.tag = tag - self.backend = server.connect( - application_name=application_name, - statement_timeout_seconds=GUARD_STATEMENT_TIMEOUT_SECONDS, - ) - self._held = False - self._closed = False - self._slot_owner: Actor | None = None - - def try_acquire(self) -> bool: - self._held = self.server.try_advisory_lock( - self.backend, - self.advisory_lock_key, - ) - return self._held - - @property - def held(self) -> bool: - return self._held and not self._closed and not self.backend.closed - - def run_json( - self, - sql: str, - *, - on_query_start: Callable[[], None] | None = None, - on_statement_end: Callable[[], None] | None = None, - on_statement_result: Callable[[Any], None] | None = None, - followup: Callable[[Any], str | None] | None = None, - ) -> Any: - actor = self.server.scheduler.current() - while self._slot_owner is not None and self._slot_owner is not actor: - self.server.scheduler.block( - f"{self.tag}.slot:wait", - ready=lambda: self._slot_owner is None, - ) - self._slot_owner = actor - try: - if on_query_start is not None: - on_query_start() - if not self.held: - raise RuntimeError("postgres writer lease guard is not held") - result = parse_single_json_value( - self.server.execute( - self.backend, - sql, - transaction=None, - timeout_seconds=None, - tag=self.tag, - ) - ) - # Same order as the native guard: the client-side end mark, - # then the parsed result (which carries the statement's own - # server execution time, issue #227). - if on_statement_end is not None: - on_statement_end() - if on_statement_result is not None: - on_statement_result(result) - while followup is not None: - next_sql = followup(result) - if next_sql is None: - break - result = parse_single_json_value( - self.server.execute( - self.backend, - next_sql, - transaction=None, - timeout_seconds=None, - tag=self.tag, - ) - ) - if on_statement_end is not None: - on_statement_end() - if on_statement_result is not None: - on_statement_result(result) - return result - finally: - self._slot_owner = None - - def close(self) -> None: - if self._closed: - return - self._closed = True - self._held = False - self.server.disconnect(self.backend) - - -# -------------------------------------------------------------------------- -# Harness front end -# -------------------------------------------------------------------------- - - -class Coordinator: - """One PRISM coordinator process: an actor plus the ledger it builds.""" - - def __init__( - self, - harness: LeaseHarness, - name: str, - *, - writer_id: str, - writer_epoch: int, - session_token: str, - ledger_kwargs: dict[str, Any], - ) -> None: - self.harness = harness - self.name = name - self.writer_id = writer_id - self.writer_epoch = writer_epoch - self.session_token = session_token - self.actor = harness.scheduler.actor(name) - self._ledger: PsqlShareLedger | None = None - self._ledger_kwargs = ledger_kwargs - self.sql_backend: FakeSqlBackend | None = None - self.guard: FakeLeaseGuard | None = None - - # -- construction ------------------------------------------------------ - - def _sql_backend_factory( - self, - conninfo: str, - *, - pool_size: int, - application_name: str, - ) -> FakeSqlBackend: - self.sql_backend = FakeSqlBackend( - self.harness.server, - pool_size=pool_size, - application_name=application_name, - tag=self.name, - ) - return self.sql_backend - - def _lease_guard_factory( - self, - conninfo: str, - *, - advisory_lock_key: int, - ) -> FakeLeaseGuard: - self.guard = FakeLeaseGuard( - self.harness.server, - advisory_lock_key=advisory_lock_key, - application_name=f"{self.name}-guard", - tag=f"{self.name}.guard", - ) - return self.guard - - def _build(self) -> PsqlShareLedger: - ledger = PsqlShareLedger( - psql_command="psql postgres://harness/qbit", - database_url="postgres://harness/qbit", - writer_id=self.writer_id, - writer_epoch=self.writer_epoch, - writer_session_token=self.session_token, - initialize_schema=False, - monotonic=self.harness.clock.monotonic, - lease_retry_sleep=self.harness.sleep, - sql_backend_factory=self._sql_backend_factory, - lease_guard_factory=self._lease_guard_factory, - **self._ledger_kwargs, - ) - self._ledger = ledger - return ledger - - def start(self) -> Call: - """Queue this coordinator's startup (construction acquires the lease).""" - return self.actor.submit(self._build, label="startup") - - @property - def ledger(self) -> PsqlShareLedger: - if self._ledger is None: - raise HarnessError( - f"coordinator {self.name!r} has not finished startup yet" - ) - return self._ledger - - @property - def started(self) -> bool: - return self._ledger is not None - - # -- driving ----------------------------------------------------------- - - def submit(self, fn: Callable[[], Any], *, label: str | None = None) -> Call: - return self.actor.submit(fn, label=label) - - def call(self, fn: Callable[[], Any], *, label: str | None = None) -> Any: - """Run ``fn`` on this actor to completion and return its result.""" - call = self.actor.submit(fn, label=label) - self.harness.run_until(self, f"done:{call.label}") - return call.value() - - def vanish(self) -> None: - """Abandon this coordinator where it stands, without releasing anything. - - A partition, SIGSTOP or VM pause: the process stops responding and no - RST reaches the server. - - What survives server-side depends on the transaction shape, and the - distinction is the whole of #123 rather than a detail. A - deadline-scoped statement runs inside an explicit transaction whose - COMMIT is a separate client message, so the server is left holding an - open transaction and its locks indefinitely. A plain autocommit - statement has no such message: PostgreSQL finishes executing it and - commits it on its own, and the vanished client merely never reads the - answer. Modelling both as orphans would let a scenario construct the - outage shape on a code path that provably cannot produce it — #123 is - explicit that the shape did not exist before the deadline plumbing - arrived. - """ - actor = self.actor - if not actor.blocked and actor.stop is None: - raise HarnessError(f"coordinator {self.name!r} has not run yet") - for backend in [ - *(self.sql_backend._in_use if self.sql_backend else []), - ]: - transaction = backend.transaction - if transaction is None: - continue - if transaction.explicit: - self.harness.server.orphan(transaction) - else: - self.harness.server._commit(transaction) - actor.parked_forever = True - actor._blocked = True - actor._ready = lambda: False - actor._wake_at = None - self.harness.scheduler.trace.append(f"{self.name}@vanished") - - -class HarnessBase: - """What every state machine's harness owns, whichever one it drives. - - Issue #128 names four owners and this covers the second of them, so the - boundary matters more than it would for a one-off: the clock, the - scheduler, the PostgreSQL model, the stepping vocabulary and the output - capture are the same for landing as for the lease lifecycle, and will be - the same again for tip refresh and share submission. What differs is only - what a harness *builds* — coordinators here, a pool with two landing - tails there — so that is all a subclass supplies. - - ``run_scenario`` and ``assert_deterministic`` are typed against this - class rather than against any one harness, which is what stops the - determinism check from being re-derived per state machine. - """ - - def __init__( - self, - *, - capture_output: bool = True, - baton_timeout_seconds: float = BATON_TIMEOUT_SECONDS, - ) -> None: - self.clock = VirtualClock() - self.scheduler = DeterministicScheduler( - self.clock, - baton_timeout_seconds=baton_timeout_seconds, - ) - self.server = FakePostgres(self.scheduler, self.clock) - self.sleeps: list[tuple[str, float]] = [] - self._output = io.StringIO() - self._errors = io.StringIO() - self._redirects: list[contextlib.AbstractContextManager[Any]] = [] - if capture_output: - # Kept apart rather than merged. The coordinator reserves stdout - # for a strict JSON-lines protocol in worker processes and routes - # audit-path diagnostics to stderr, so which stream a message - # arrives on is itself an assertion a scenario may want to make. - self._redirects = [ - contextlib.redirect_stdout(self._output), - contextlib.redirect_stderr(self._errors), - ] - for redirect in self._redirects: - redirect.__enter__() - - # -- lifecycle --------------------------------------------------------- - - def __enter__(self) -> Self: - return self - - def __exit__(self, *exc_info: object) -> None: - self.close() - - def close(self) -> None: - self.scheduler.close() - for redirect in reversed(self._redirects): - redirect.__exit__(None, None, None) - self._redirects = [] - - @property - def output(self) -> str: - return self._output.getvalue() - - @property - def errors(self) -> str: - return self._errors.getvalue() - - @property - def trace(self) -> list[str]: - return list(self.scheduler.trace) - - # -- synchronisation --------------------------------------------------- - - def lock(self, name: str) -> HarnessLock: - return HarnessLock(self.scheduler, name) - - def rlock(self, name: str) -> HarnessRLock: - return HarnessRLock(self.scheduler, name) - - def condition(self, name: str) -> HarnessCondition: - return HarnessCondition(self.rlock(name), name=name) - - def event(self, name: str) -> HarnessEvent: - return HarnessEvent(self.scheduler, name) - - def semaphore(self, name: str, value: int) -> HarnessSemaphore: - return HarnessSemaphore(self.scheduler, name, value=value) - - # -- clock ------------------------------------------------------------- - - def sleep(self, seconds: float) -> None: - """Park until the virtual clock reaches the wake time.""" - actor = self.scheduler.current() - self.sleeps.append((actor.name, float(seconds))) - wake_at = self.clock.monotonic() + float(seconds) - while self.clock.monotonic() < wake_at: - self.scheduler.block(f"sleep:{seconds:g}", wake_at=wake_at) - - def advance(self, seconds: float) -> float: - return self.clock.advance(seconds) - - def advance_to_next_deadline(self) -> float | None: - """Jump to the earliest pending sleep or timeout. - - Returns the new virtual monotonic time, or None when no actor is - waiting on time at all — which is itself a finding: an actor blocked - with no deadline can only be released by another actor, and if none - can run, nothing bounds the wait. - """ - deadline = self.scheduler.next_deadline() - if deadline is None: - return None - return self.clock.advance_to(max(deadline, self.clock.monotonic())) - - # -- stepping ---------------------------------------------------------- - - def _actor(self, who: Any) -> Actor: - if isinstance(who, Actor): - return who - actor = getattr(who, "actor", None) - if isinstance(actor, Actor): - return actor - raise HarnessError(f"{who!r} is not an actor and does not own one") - - def step(self, who: Any) -> str: - return self.scheduler.step(self._actor(who)) - - def run_until(self, who: Any, label: str, *, limit: int = 200) -> str: - return self.scheduler.run_until(self._actor(who), label, limit=limit) - - def run_until_blocked(self, who: Any, label: str, *, limit: int = 200) -> str: - """Step until the actor parks at ``label`` and is genuinely stuck there.""" - actor = self._actor(who) - stop = self.scheduler.run_until(actor, label, limit=limit) - if not actor.blocked: - raise HarnessError( - f"actor {actor.name!r} reached {label!r} but is not blocked" - ) - return stop - - def drain(self, order: list[Any] | None = None, *, limit: int = 500) -> None: - """Step runnable actors in a fixed order until none can progress. - - Order is explicit and never depends on thread timing, so a drained - run is as reproducible as a hand-written schedule. - """ - actors = ( - [self._actor(who) for who in order] - if order is not None - else list(self.scheduler.actors.values()) - ) - for _ in range(limit): - progressed = False - for actor in actors: - if actor.runnable(): - self.scheduler.step(actor) - progressed = True - if not progressed: - return - raise HarnessError(f"drain did not quiesce within {limit} rounds") - - # -- observation ------------------------------------------------------- - - def statement_kinds(self) -> list[str]: - return [statement.kind.value for statement in self.server.statements] - - -class LeaseHarness(HarnessBase): - """Front end: build coordinators, step them, advance time, read the trace.""" - - def __init__( - self, - *, - lease_ttl_seconds: float = 60.0, - capture_output: bool = True, - baton_timeout_seconds: float = BATON_TIMEOUT_SECONDS, - **default_ledger_kwargs: Any, - ) -> None: - super().__init__( - capture_output=capture_output, - baton_timeout_seconds=baton_timeout_seconds, - ) - self.coordinators: dict[str, Coordinator] = {} - self._default_ledger_kwargs = { - "lease_ttl_seconds": lease_ttl_seconds, - **default_ledger_kwargs, - } - self._session_counter = 0 - - # -- construction ------------------------------------------------------ - - def coordinator( - self, - name: str, - *, - writer_id: str = "prism-coordinator", - writer_epoch: int = 1, - session_token: str | None = None, - heartbeat_capable: bool = True, - **ledger_kwargs: Any, - ) -> Coordinator: - self._session_counter += 1 - if session_token is None: - suffix = f"{name}-{self._session_counter}" - session_token = ( - f"{WRITER_LEASE_HEARTBEAT_SESSION_PREFIX}{suffix}" - if heartbeat_capable - else uuid.uuid5(uuid.NAMESPACE_OID, suffix).hex - ) - coordinator = Coordinator( - self, - name, - writer_id=writer_id, - writer_epoch=writer_epoch, - session_token=session_token, - ledger_kwargs={**self._default_ledger_kwargs, **ledger_kwargs}, - ) - self.coordinators[name] = coordinator - return coordinator - - # -- observation ------------------------------------------------------- - - def lease_row(self) -> LeaseRow | None: - return self.server.lease - - def lease_holder_session(self) -> str | None: - row = self.server.lease - return None if row is None else row.writer_session_token - - -# -------------------------------------------------------------------------- -# Determinism checking -# -------------------------------------------------------------------------- - - -@dataclass(frozen=True) -class ScenarioRun: - """What one execution of a scenario produced.""" - - trace: tuple[str, ...] - outcome: Any - - def digest(self) -> str: - return json.dumps( - {"trace": list(self.trace), "outcome": self.outcome}, - sort_keys=True, - default=repr, - ) - - -def run_scenario( - scenario: Callable[[Any], Any], - *, - harness_factory: Callable[..., HarnessBase] = LeaseHarness, - **harness_kwargs: Any, -) -> ScenarioRun: - """Execute ``scenario`` against a fresh harness and record what happened. - - ``harness_factory`` names which state machine's harness to build. It - defaults to the lease lifecycle only because that is what landed first; - nothing here knows anything about leases, and a scenario for any owner - deriving from ``HarnessBase`` runs unchanged. - """ - harness = harness_factory(**harness_kwargs) - try: - outcome = scenario(harness) - return ScenarioRun(trace=tuple(harness.trace), outcome=outcome) - finally: - harness.close() - - -def assert_deterministic( - test: Any, - scenario: Callable[[Any], Any], - *, - repeats: int = 25, - harness_factory: Callable[..., HarnessBase] = LeaseHarness, - **harness_kwargs: Any, -) -> ScenarioRun: - """Run ``scenario`` repeatedly and require an identical trace and outcome. - - A concurrency test that is merely usually right is worse than no test, so - every scenario in this suite is required to prove order-stability rather - than to assert it once. The checkpoint trace is the stronger half of the - check: two runs can agree on the answer while disagreeing about the - schedule that produced it, and that is exactly the flakiness this harness - exists to rule out. - """ - if repeats < 2: - raise HarnessError("determinism needs at least two runs to mean anything") - runs = [ - run_scenario( - scenario, - harness_factory=harness_factory, - **harness_kwargs, - ) - for _ in range(repeats) - ] - first = runs[0] - for index, run in enumerate(runs[1:], start=2): - if run.trace != first.trace: - test.fail( - f"run {index} produced a different schedule than run 1.\n" - f"run 1: {list(first.trace)}\n" - f"run {index}: {list(run.trace)}" - ) - if run.digest() != first.digest(): - test.fail( - f"run {index} produced a different outcome than run 1.\n" - f"run 1: {first.outcome!r}\n" - f"run {index}: {run.outcome!r}" - ) - return first - - -__all__ = [ - "Actor", - "Backend", - "Call", - "Coordinator", - "DeterministicScheduler", - "FakeLeaseGuard", - "FakePostgres", - "FakeSqlBackend", - "HarnessBase", - "HarnessCondition", - "HarnessError", - "HarnessEvent", - "HarnessLock", - "HarnessRLock", - "HarnessSemaphore", - "LandingOp", - "LeaseHarness", - "LeaseOp", - "LeaseRow", - "LockTimeout", - "NotRunnable", - "OutboxRow", - "PoolBlockRow", - "ScenarioRun", - "SchedulerStall", - "Statement", - "StatementKind", - "Transaction", - "UnsupportedStatement", - "VirtualClock", - "acquire_result_arms", - "acquire_terminal_arm_fields", - "acquire_terminal_arm_keys", - "advisory_lock_pair", - "assert_deterministic", - "classify", - "run_scenario", -] diff --git a/tests/prism_coordinator_test_support.py b/tests/prism_coordinator_test_support.py deleted file mode 100644 index 4a6cb9660..000000000 --- a/tests/prism_coordinator_test_support.py +++ /dev/null @@ -1,500 +0,0 @@ -#!/usr/bin/env python3 -"""Shared no-environment fixtures for coordinator job and delivery tests.""" -# ruff: noqa: F401 - -from __future__ import annotations - -import contextlib -import io -import os -import queue -import socket -import subprocess -import sys -import threading -import time -import unittest -from pathlib import Path -from concurrent.futures import Future -from contextlib import contextmanager -from dataclasses import dataclass, replace as dataclass_replace -from decimal import Decimal -from types import SimpleNamespace -from unittest.mock import patch - -from lab.auxpow import vardiff -from lab.prism import direct_stratum -from lab.prism import prism_coordinator as prism_coordinator_module -from lab.prism.coordinator_shutdown import ShutdownInProgress -from lab.prism.prism_coordinator import ( - ClientState, - JobBuildAdmissionDeadlineExceeded, - JobBuildSuperseded, - MAX_PRISM_JOB_BUNDLE_CACHE_ENTRIES, - PRISM_JOB_EXTRANONCE1_PLACEHOLDER_HEX, - PRISM_REJECTION_REASON_IDS, - PendingShareAppend, - PayoutLedgerArtifact, - PrismCoordinator, - TemplateRefreshBlocked, - WorkerIdentity, - _compact_share_payload, - canonical_json_sha256, - canonical_json_text, - now_ms, - qbit_template_fingerprint, -) -from lab.prism.coordinator_config import default_prism_coinbase_tag_hex -from lab.prism.payout_state import ( - PayoutStatePublicationBlocked as _PayoutStatePublicationBlocked, -) -from lab.prism.share_ledger import ( - LedgerOperationTimeout, - PendingShare, - SingleWriterShareLedger, -) - - -PAYOUT_ADDRESS = "tq1z70ukpvs96kye6jmgvl3nttevtkrq8uu89snkpm6m8gwqukw8u5dsz32kwa" - - -EXTRANONCE2_SIZE = 8 - - -@dataclass(frozen=True) -class FakeShare: - miner_id: str - share_seq: int - - def to_prism_json(self) -> dict[str, object]: - return {"share_seq": self.share_seq, "miner_id": self.miner_id} - - -class FakeLedger: - backend_name = "fake" - - def __init__(self, miners: list[str] | None = None) -> None: - self.miners = miners if miners is not None else ["miner-a", "miner-b", "miner-c"] - self.snapshot_calls = 0 - self.stats_calls = 0 - - def accepted_share_stats(self) -> dict[str, int]: - self.stats_calls += 1 - return { - "accepted_share_count": len(self.miners), - "distinct_miner_count": len(set(self.miners)), - } - - def all_shares(self) -> list[FakeShare]: - raise AssertionError("all_shares must not be called when accepted_share_stats exists") - - def snapshot_at_job_issue(self, anchor_job_issued_at_ms: int, *, window_weight: int | None = None) -> list[FakeShare]: - self.snapshot_calls += 1 - return [FakeShare(miner_id=miner, share_seq=seq + 1) for seq, miner in enumerate(self.miners)] - - def current_prior_balances(self) -> list[dict[str, object]]: - return [] - - def metrics(self) -> dict[str, int]: - return {"blocks": 0, "owed_accounts": 0} - - -class ReadyLedgerWithEmptyFirstSnapshot(FakeLedger): - def __init__(self) -> None: - super().__init__(miners=["miner-a", "miner-b", "miner-c"]) - - def snapshot_at_job_issue(self, anchor_job_issued_at_ms: int, *, window_weight: int | None = None) -> list[FakeShare]: - self.snapshot_calls += 1 - if self.snapshot_calls == 1: - return [] - return [FakeShare(miner_id=miner, share_seq=seq + 1) for seq, miner in enumerate(self.miners)] - - -class FakeRpc: - def __init__(self, template: dict[str, object], tip: str) -> None: - self.template = template - self.tip = tip - self.blockchain_info: dict[str, object] = { - "initialblockdownload": False, - "blocks": 100, - "headers": 100, - } - self.calls: list[str] = [] - - def call(self, method: str, params: list[object] | None = None) -> object: - self.calls.append(method) - if method == "getblocktemplate": - return dict(self.template) - if method == "getbestblockhash": - return self.tip - if method == "getblockchaininfo": - return dict(self.blockchain_info) - if method == "getblockcount": - return int(self.blockchain_info["blocks"]) - raise AssertionError(f"unexpected RPC {method}") - - def count(self, method: str) -> int: - return sum(1 for name in self.calls if name == method) - - -def synthetic_manifest_coinbase_hex(suffix_hex: str) -> str: - """A structurally valid non-witness coinbase whose scriptSig ends with the - extranonce placeholder suffix, as the audit bundle builder produces.""" - height_push = "03aabbcc" - script_sig = height_push + suffix_hex - script_sig_bytes = bytes.fromhex(script_sig) - output = (50_00000000).to_bytes(8, "little").hex() + "0151" - return ( - "01000000" - + "01" - + "00" * 32 - + "ffffffff" - + direct_stratum.compact_size(len(script_sig_bytes)).hex() - + script_sig - + "ffffffff" - + "01" - + output - + "00000000" - ) - - -def base_template(height: int = 10, prevhash: str = "11" * 32) -> dict[str, object]: - # Realistic (non-regtest) bits: the network target must be harder than the - # vardiff range for per-client share targets to differ, as on testnet4. - return { - "height": height, - "previousblockhash": prevhash, - "bits": "1b00ffff", - "version": 0x20000000, - "curtime": 1_700_000_000, - "coinbasevalue": 50_00000000, - "transactions": [], - } - - -def durable_candidate_row(index: int) -> dict[str, object]: - """A minimal valid outbox row whose intent survives block_candidate_from_intent.""" - block_hash = f"{index + 1:064x}" - return { - "block_hash": block_hash, - "candidate": { - "schema": "qbit.prism.block-candidate-intent.v1", - "block_hash_hex": block_hash, - "block_hex": "00", - "coinbase_tx_hex": "00", - "parent_hash": "11" * 32, - "expected_height": 10, - "template": { - "previousblockhash": "11" * 32, - "height": 10, - "coinbasevalue": 50_00000000, - }, - "shares_json": [], - "prior_balances": [], - "found_block": {}, - "prospective_prior_balances": None, - "witness_merkle_leaves_hex": [], - "extranonce1_hex": "00000001", - "extranonce2_hex": "00", - "username": "miner-a", - "pending_share": { - "share_id": f"miner-a:{block_hash}", - "miner_id": "miner-a", - "order_key": "miner-a", - "p2mr_program_hex": "22" * 32, - "share_difficulty": 1, - "network_difficulty": 1, - "template_height": 10, - "job_id": "job-1", - "job_issued_at_ms": 1, - "accepted_at_ms": 1, - "ntime": 1, - }, - "credit_share_on_accept": False, - "collection_only": False, - }, - } - - -def worker(payout: str = PAYOUT_ADDRESS, username: str | None = None) -> WorkerIdentity: - return WorkerIdentity( - username=username or payout, - payout_address=payout, - worker_name=None, - script_pubkey_hex="5220" + "22" * 32, - p2mr_program_hex="22" * 32, - ) - - -def client(connection_id: int, identity: WorkerIdentity | None = None) -> ClientState: - state = ClientState.__new__(ClientState) - state.sock = None - state.address = ("127.0.0.1", 40_000 + connection_id) - state.connection_id = connection_id - state.extranonce1_hex = f"{connection_id:08x}" - state.subscribed = True - state.authorized = True - identity = identity or worker() - state.username = identity.username - state.worker = identity - state.version_mask = 0 - state.active_job = None - state.share_difficulty = Decimal("1") - state.pending_share_difficulty = None - state.active_job_ids = set() - state.post_accept_refresh_block = None - state.tip_work_delivered = None - state.closing = False - state.job_update_lock = threading.RLock() - state.send_lock = threading.Lock() - return state - - -class ObservedRLock: - """RLock test double that exposes a contending acquire without sleeps.""" - - def __init__(self) -> None: - self._lock = threading.RLock() - self.observe_acquires = False - self.acquire_attempted = threading.Event() - - def acquire(self, blocking: bool = True, timeout: float = -1) -> bool: - if self.observe_acquires: - self.acquire_attempted.set() - return self._lock.acquire(blocking, timeout) - - def release(self) -> None: - self._lock.release() - - def __enter__(self) -> ObservedRLock: - self.acquire() - return self - - def __exit__(self, *_args: object) -> None: - self.release() - - -def coordinator(*, ledger: object | None = None, template: dict[str, object] | None = None) -> tuple[PrismCoordinator, FakeRpc]: - server = PrismCoordinator.__new__(PrismCoordinator) - template = template or base_template() - rpc = FakeRpc(template, tip=str(template["previousblockhash"])) - server.rpc = rpc - server.qbit_chain = "regtest" - server.lock = threading.RLock() - server.stop_event = threading.Event() - server.clients = set() - server.jobs = {} - server.job_counter = 0 - server.connection_counter = 0 - server.accepted_block_count = 0 - server.max_blocks = 1_000 - server.started_monotonic = time.monotonic() - server.submitted_share_count = 0 - server.stale_share_count = 0 - server.duplicate_share_count = 0 - server.low_difficulty_share_count = 0 - server.rejection_counts_by_reason = {reason: 0 for reason in PRISM_REJECTION_REASON_IDS} - server.job_build_failure_count = 0 - server.tip_refresh_job_count = 0 - server.post_accept_refresh_failure_count = 0 - server.reorg_reconciler_enabled = False - server.reorg_inactive_block_count = 0 - server.reorg_reactivated_block_count = 0 - server.reorg_reconcile_skip_count = 0 - server.reorg_reconcile_error_count = 0 - server.matured_payout_count = 0 - server.last_reorg_reconciled_tip_hash = None - server.last_reorg_reconciled_trusted = False - server.last_reorg_reconciled_monotonic = None - server.latest_evidence = None - server.latest_coinbase_size_bytes = None - server.tip_template_snapshot = None - server.extranonce2_size = EXTRANONCE2_SIZE - server.coinbase_tag_hex = default_prism_coinbase_tag_hex() - server.share_difficulty = Decimal("1") - server.vardiff_config = vardiff.VardiffConfig( - enabled=True, - target_share_interval_seconds=Decimal("15"), - min_difficulty=Decimal("0.000000001"), - max_difficulty=Decimal("1024"), - retarget_interval_seconds=Decimal("90"), - max_step_factor=Decimal("4"), - startup_difficulty=Decimal("1"), - max_step_down_factor=Decimal("4"), - ewma_alpha=Decimal("0.4"), - retarget_tolerance=Decimal("0.25"), - ) - server.default_share_weight = 1 - server.share_weights_by_username = {} - server.min_ready_miners = 3 - server.ledger = ledger if ledger is not None else FakeLedger() - server.blockpoll_seconds = 2.0 - # Failed-refresh spacing is opt-in per test: its holdoff waits on real - # time, which deadlocks tests that freeze time.monotonic around failing - # polls. Pacing behavior is covered by test_prism_refresh_retry_pacing. - server.tip_refresh_failure_holdoff_seconds = 0.0 - server.job_bundle_cache_seconds = 10.0 - server.template_cache_seconds = 2.0 - server.reorg_reconcile_cache_seconds = 5.0 - server.health_refresh_seconds = 5.0 - server.stratum_send_timeout_seconds = 20.0 - server._ensure_job_cache_state() - return server, rpc - - -def install_fake_bundle_builder(server: PrismCoordinator) -> dict[str, object]: - """Replace the audit bundle subprocess with a counting fake whose manifest - coinbase embeds exactly the suffix the coordinator asked for.""" - recorded: dict[str, object] = {"calls": 0, "suffixes": []} - - def fake_build_audit_bundle(**kwargs: object) -> dict[str, object]: - recorded["calls"] = int(recorded["calls"]) + 1 - suffix_hex = str(kwargs["coinbase_script_sig_suffix_hex"]) - recorded["suffixes"].append(suffix_hex) - recorded["last_kwargs"] = kwargs - return { - "found_block": dict(kwargs["found_block"]), - "payout_policy_manifest": {"accounts": []}, - "signed_coinbase_manifest": { - "manifest": { - "coinbase_tx_hex": synthetic_manifest_coinbase_hex(suffix_hex), - } - }, - } - - server.build_audit_bundle = fake_build_audit_bundle # type: ignore[method-assign] - server._ensure_bundle_compiler().build_audit_bundle = ( # type: ignore[method-assign] - fake_build_audit_bundle - ) - return recorded - - -def stamped_pending_share(accepted_at_ms: int) -> PendingShare: - return PendingShare( - share_id=f"miner-a:{accepted_at_ms}", - miner_id="miner-a", - order_key="miner-a", - p2mr_program_hex="22" * 32, - share_difficulty=1, - network_difficulty=1, - template_height=9, - job_id="job-1", - job_issued_at_ms=accepted_at_ms - 1, - accepted_at_ms=accepted_at_ms, - ntime=1_700_000_000, - ) - - -class AnchorRecordingLedger(FakeLedger): - def __init__(self) -> None: - super().__init__() - self.anchors: list[int] = [] - - def snapshot_at_job_issue( - self, anchor_job_issued_at_ms: int, *, window_weight: int | None = None - ) -> list[FakeShare]: - self.anchors.append(int(anchor_job_issued_at_ms)) - return super().snapshot_at_job_issue( - anchor_job_issued_at_ms, window_weight=window_weight - ) - - -class IncrementalRecordingLedger(SingleWriterShareLedger): - def __init__(self) -> None: - super().__init__() - self.full_snapshot_calls = 0 - self.delta_snapshot_calls = 0 - - def snapshot_at_job_issue( - self, - anchor_job_issued_at_ms: int, - *, - window_weight: int | None = None, - ) -> list[object]: - self.full_snapshot_calls += 1 - return super().snapshot_at_job_issue( - anchor_job_issued_at_ms, - window_weight=window_weight, - ) - - def snapshot_between_job_issues( - self, - previous_anchor_job_issued_at_ms: int, - anchor_job_issued_at_ms: int, - ) -> list[object]: - self.delta_snapshot_calls += 1 - return super().snapshot_between_job_issues( - previous_anchor_job_issued_at_ms, - anchor_job_issued_at_ms, - ) - - -def append_incremental_share( - ledger: SingleWriterShareLedger, - *, - share_seq: int, - accepted_at_ms: int, -) -> None: - ledger.append( - PendingShare( - share_id=f"incremental-{share_seq}", - miner_id=f"miner-{share_seq % 3}", - order_key=f"miner-{share_seq % 3}", - p2mr_program_hex=f"{share_seq % 256:02x}" * 32, - share_difficulty=1, - network_difficulty=1, - template_height=9, - job_id=f"job-{share_seq}", - job_issued_at_ms=accepted_at_ms - 1, - accepted_at_ms=accepted_at_ms, - ntime=1_700_000_000 + share_seq, - ) - ) - - -def mark_progress_healthy(server: PrismCoordinator) -> None: - snapshot = server.fetch_qbit_tip_template_snapshot() - server._record_progress_tip_poll(snapshot) - server._record_progress_publication( - snapshot, - int(getattr(server, "_payout_state_generation", 0)), - ) - - -def spool_share(seq: int) -> dict[str, object]: - return { - "share_seq": seq, - "share_id": f"share-{seq}", - "miner_id": "miner-a", - "order_key": "miner-a", - "p2mr_program_hex": "22" * 32, - "share_difficulty": 1, - "job_issued_at_ms": 1_700_000_000_000 + seq, - "accepted_at_ms": 1_700_000_000_100 + seq, - "credit_policy": None, - } - - -ECHO_BUILDER_COMMAND = [ - sys.executable, - "-c", - "import json,sys; json.dump({'received': json.load(sys.stdin)}, sys.stdout)", -] - - -FAKE_SERVE_BUILDER_COMMAND = [ - sys.executable, - str(Path(__file__).resolve().parent / "fixtures" / "fake_serve_builder.py"), -] - - -def build_job_cache_test_coordinator( - *, - ledger: object | None = None, - template: dict[str, object] | None = None, -) -> tuple[PrismCoordinator, FakeRpc]: - """Build a coordinator fixture without production init or worker startup.""" - return coordinator(ledger=ledger, template=template) - - -__all__ = [name for name in globals() if not name.startswith("__")] diff --git a/tests/prism_landing_harness.py b/tests/prism_landing_harness.py deleted file mode 100644 index 5d12fb4f3..000000000 --- a/tests/prism_landing_harness.py +++ /dev/null @@ -1,951 +0,0 @@ -#!/usr/bin/env python3 -"""Deterministic harness for PRISM's block candidate landing state machine. - -Issue #128 ranks landing first by incident history, and its shape is the -reason: a found block is real money, and the landing is the one owner where -two threads of the *same* process reach the same durable rows. The lease -harness in ``prism_concurrency_harness`` interleaves two coordinators around -one row; this one interleaves the two tails one coordinator lands on. - -The two tails -------------- -``lab/prism/block_candidates.py`` runs landings down two routes: - -``_submit_synchronous_block_candidate`` - The miner connection thread. A share that is block-worthy at the - assigned target lands inline on the client's own thread, which is - normal while the listener floor sits above network difficulty. - -``block_accounting_loop`` / ``_run_block_accounting_task`` - The dedicated accounting actor, which lands everything else. - -The disposition lease that serialises them is **per hash** -(``block_candidates.py``), so it provides no mutual exclusion between two -*distinct* found blocks. That is the topology #133 turns on, and expressing -it is what this harness exists for. - -What is real here and what is modelled --------------------------------------- -Real: ``BlockCandidateService``, ``BlockFinalizationService``, -``AuditArtifactStore`` over a genuine audit root, ``PsqlShareLedger`` -emitting its shipped SQL, and the coordinator's own landing arithmetic — -budgets, watchdog ceiling, epoch fences, publication ordinal handling. - -Modelled: PostgreSQL (``FakePostgres``, extended with ``qbit_pool_blocks`` -and ``qbit_block_candidate_outbox``), qbitd (``LandingRpc``), the audit -bundle build and its verification, and every lock the two tails share — -installed as harness locks so a contended acquire is an interleaving point -rather than a wedged baton. - -The audit root is a real directory on purpose. What #133 loses is a *file* -that is never written, and no in-memory double can fail to write it. -""" - -from __future__ import annotations - -import json -import shutil -import sys -import tempfile -import threading -from collections.abc import Callable -from dataclasses import dataclass -from pathlib import Path -from types import SimpleNamespace -from typing import Any - -sys.path.insert(0, str(Path(__file__).resolve().parents[1])) - -from tests.prism_concurrency_harness import ( # noqa: E402 - BATON_TIMEOUT_SECONDS, - Actor, - Call, - FakeLeaseGuard, - FakePostgres, - FakeSqlBackend, - HarnessBase, - HarnessCondition, - HarnessError, - PoolBlockRow, -) -from lab.prism.block_candidates import ( # noqa: E402 - PrismBlockCandidate, - _BlockCandidateAccountingTask, - _BlockCandidateNodeSubmission, -) -from lab.prism.share_ledger import ( # noqa: E402 - WRITER_LEASE_HEARTBEAT_SESSION_PREFIX, - PendingShare, - PsqlShareLedger, -) - -PARENT_HASH = "00" * 32 -DEFAULT_HEIGHT = 10 - - -# -------------------------------------------------------------------------- -# qbitd -# -------------------------------------------------------------------------- - - -class LandingRpc: - """The node, as the landing path sees it. - - ``submitblock`` accepts by default and the chain tip follows the last - accepted block, because a landing that is *rejected* never reaches the - interleavings this harness is for. A scenario overrides ``submit_result`` - to make one landing fail. - """ - - def __init__(self, *, tip: str, height: int) -> None: - self.tip = tip - self.height = height - self.calls: list[tuple[str, tuple[Any, ...]]] = [] - self.submitted: list[str] = [] - self.submit_result: Callable[[str], object] | None = None - self.blocks_by_height: dict[int, str] = {} - self._registered: dict[str, tuple[str, int]] = {} - - def register(self, *, block_hex: str, block_hash: str, height: int) -> None: - """Teach the node which block a submitted payload is.""" - self._registered[block_hex] = (block_hash, int(height)) - - def call(self, method: str, params: list[object] | None = None) -> object: - args = tuple(params or ()) - self.calls.append((method, args)) - if method == "getbestblockhash": - return self.tip - if method == "getblockcount": - return self.height - if method == "getblockhash": - requested = int(args[0]) - return self.blocks_by_height.get(requested, self.tip) - if method == "submitblock": - return self._submit(str(args[0]) if args else "") - if method == "getblockchaininfo": - return {"initialblockdownload": False} - if method == "getnetworkinfo": - return {"connections": 4} - if method == "getblockheader": - raise RuntimeError("qbit RPC getblockheader failed: -5 Block not found") - raise RuntimeError(f"unmodelled qbit RPC method {method}") - - def _submit(self, block_hex: str) -> object: - self.submitted.append(block_hex) - result = None if self.submit_result is None else self.submit_result(block_hex) - if result in (None, "duplicate"): - known = self._registered.get(block_hex) - if known is not None: - self.accept(block_hash=known[0], height=known[1]) - return result - - def accept(self, *, block_hash: str, height: int) -> None: - """Make ``block_hash`` the active block at ``height``.""" - self.blocks_by_height[height] = block_hash - self.tip = block_hash - self.height = height - - -# -------------------------------------------------------------------------- -# Candidates -# -------------------------------------------------------------------------- - - -@dataclass(frozen=True) -class LandingCandidate: - """One found block, plus everything a scenario needs to talk about it.""" - - block_hash: str - block_height: int - job_id: str - candidate: PrismBlockCandidate - - @property - def envelope_name(self) -> str: - return f"{self.block_height}-{self.block_hash}" - - -# -------------------------------------------------------------------------- -# The harness -# -------------------------------------------------------------------------- - - -class LandingHarness(HarnessBase): - """Front end: build found blocks, land them on either tail, read the trace.""" - - def __init__( - self, - *, - parent_hash: str = PARENT_HASH, - block_height: int = DEFAULT_HEIGHT, - lease_ttl_seconds: float = 60.0, - watchdog_timeout_seconds: float = 120.0, - block_landing_db_timeout_seconds: float = 30.0, - read_concurrency: int = 4, - capture_output: bool = True, - baton_timeout_seconds: float = BATON_TIMEOUT_SECONDS, - ) -> None: - super().__init__( - capture_output=capture_output, - baton_timeout_seconds=baton_timeout_seconds, - ) - self.parent_hash = parent_hash - self.block_height = int(block_height) - self.lease_ttl_seconds = float(lease_ttl_seconds) - self.read_concurrency = int(read_concurrency) - self._audit_root = Path(tempfile.mkdtemp(prefix="prism-landing-harness-")) - self._job_counter = 0 - self.sql_backend: FakeSqlBackend | None = None - self.guard: FakeLeaseGuard | None = None - self.session_token = f"{WRITER_LEASE_HEARTBEAT_SESSION_PREFIX}landing" - # Which external-side-effect components this run refuses authority - # for, and which ones actually asked. A scenario sets the first to - # drive the degraded branch; the second is evidence the gate ran. - self.withheld_lease_components: set[str] | None = None - self.lease_fence_components: list[str] = [] - # Production phase names a scenario wants to stop at (see - # _install_phase_breakpoints). - self.breakpoints: set[str] = set() - - self.rpc = LandingRpc(tip=parent_hash, height=self.block_height - 1) - self.pool = self._build_pool( - watchdog_timeout_seconds=watchdog_timeout_seconds, - block_landing_db_timeout_seconds=block_landing_db_timeout_seconds, - ) - - # The tails. Named for the production threads they stand in for, so a - # checkpoint trace reads as "which tail was running". - self.client = self.scheduler.actor("client") - self.accounting = self.scheduler.actor("accounting") - # Startup is its own actor: the ledger constructor acquires the - # writer lease, which is a modelled statement and so has to run on an - # actor thread like everything else. - self.startup = self.scheduler.actor("startup") - # The share writer's side of the epoch machinery. A landing's fences - # exist to order against it, so it has to be a third schedulable - # thread rather than something the test body does between steps. - self.appender = self.scheduler.actor("appender") - - # -- lifecycle --------------------------------------------------------- - - def close(self) -> None: - super().close() - store = self.pool.__dict__.get("_audit_artifact_store") - if store is not None: - # Teardown runs on the controller thread, which is not an actor, - # so the store's locks go back to ordinary ones before its own - # close path takes them. The scheduler is already shut down by - # then; nothing is left to schedule. - store._lock = threading.RLock() - store._publication_lifecycle_lock = threading.RLock() - store.close() - shutil.rmtree(self._audit_root, ignore_errors=True) - - # -- construction ------------------------------------------------------ - - def _build_pool( - self, - *, - watchdog_timeout_seconds: float, - block_landing_db_timeout_seconds: float, - ) -> Any: - # Imported here rather than at module scope: the support module pulls - # in the whole coordinator surface, and only a landing scenario needs - # it. - from tests.prism_vardiff_test_support import submit_coordinator - - pool, client_state, _ledger = submit_coordinator(tip=self.parent_hash) - self.client_state = client_state - pool.max_blocks = 1_000 - pool.stop_after_block = False - pool.rpc = self.rpc - pool.watchdog_timeout_seconds = watchdog_timeout_seconds - pool.block_landing_db_timeout_seconds = block_landing_db_timeout_seconds - pool.block_submitter_db_timeout_seconds = block_landing_db_timeout_seconds - pool.audit_dir = self._audit_root / "audit" - pool.evidence_path = self._audit_root / "state" / "evidence.json" - pool.ledger_writer_public_key_hex = "aa" * 32 - # The bundle build and its verification are the audit machine, not - # the landing machine. #128 scopes this harness to landing, so both - # are answered from a fixed report: a landing that turns on bundle - # content would be testing the wrong owner. - pool.build_audit_bundle = lambda **_kwargs: _verified_block_bundle() - pool.verify_bundle = ( - lambda *_args, expected_block_height=DEFAULT_HEIGHT, **_kwargs: ( - _verified_audit_report(block_height=expected_block_height) - ) - ) - # Reconciliation walks the chain through the node one block at a - # time. It is the reorg reconciler's state machine, and #128 lists it - # separately; a landing only needs its verdict. - pool.ensure_reorg_reconciled_for_tip = lambda _tip, **_kwargs: True - pool.reorg_reconciler_enabled = False - self._install_harness_locks(pool) - self._install_inline_outbox_worker(pool) - self._install_inline_external_lease_fence(pool) - self._install_phase_breakpoints(pool) - return pool - - def _install_phase_breakpoints(self, pool: Any) -> None: - """Let a scenario stop a tail at a phase the landing already names. - - SQL statement boundaries are the harness's default interleaving - points, and for the lease lifecycle they were enough: every state - transition there *is* a statement. Landing is not like that. Its - interesting boundaries are the gaps *between* statements — the - confirm→publish window is bounded on one side by a durable - confirmation and on the other by a lock the publish step has not - taken yet, and no statement runs at the instant that matters. - - The landing already names those boundaries for its own watchdog, in - ``_record_block_submitter_phase`` and - ``_record_block_candidate_progress``. Reusing that vocabulary means a - scenario says where it wants to interleave in the code's own words — - ``progress:evidence-write`` is the top of the publish step — instead - of counting statements and hoping the count survives a refactor. An - armed phase that never fires is a loud failure, because the scenario - then waits at a checkpoint that never arrives. - """ - record_progress = pool._record_block_candidate_progress - record_phase = pool._record_block_submitter_phase - - def progress(phase: str = "accounting-progress") -> None: - record_progress(phase) - self._maybe_break(f"progress:{phase}") - - def submitter_phase(phase: str) -> None: - record_phase(phase) - self._maybe_break(f"phase:{phase}") - - pool._record_block_candidate_progress = progress - pool._record_block_submitter_phase = submitter_phase - - def _maybe_break(self, label: str) -> None: - if label in self.breakpoints: - self.scheduler.checkpoint(label) - - def break_at(self, *labels: str) -> None: - """Arm one or more phase breakpoints for every tail.""" - self.breakpoints.update(labels) - - def _install_inline_outbox_worker(self, pool: Any) -> None: - """Run the durable outbox calls on the caller's own actor. - - This is the harness's one substantive model boundary, so it is worth - being exact about. - - The *landing-class* ledger steps — persist, confirm, the ordinal - read-back, the pool-block reads — already run inline on the - block-work thread in production, so they arrive here unchanged, with - their real budgets, their real admission gates and their real - statements. The **outbox** marks do not: they run on a spawned - bounded worker thread (``_run_block_submitter_ledger_call``) so a - driver that ignores its statement deadline cannot wedge the - submitter, and the caller waits on a ``threading.Event`` in slices. - - A baton scheduler cannot schedule that worker. It is created inside - the wrapper with no factory seam, and the caller blocks on a real - event while holding the baton, so the worker could only ever run - unscheduled — which would put its statements outside the schedule - the harness exists to control. Running the operation inline instead - keeps everything the landing state machine depends on (the operation - itself, its budget scope, its error propagation and the per-class - latency metric) and drops only the liveness wrapper around it. - - What that costs is honest to state: this harness cannot express a - *stuck* outbox driver, so the restart escalations in - ``_maybe_restart_for_stuck_block_call`` are out of its reach. Those - are covered example-wise in the block-candidate suite. Giving the - harness the wrapper too needs a thread-factory seam in production, - which belongs to whichever issue wants that interleaving. - """ - service = pool._ensure_block_candidate_service() - - def run_inline( - key: tuple[object, ...], - phase: str, - operation: Callable[[], Any], - *, - timeout_seconds: float | None = None, - call_class: str = "fast", - ) -> Any: - budget = ( - service._block_submitter_db_timeout() - if timeout_seconds is None - else max(0.001, float(timeout_seconds)) - ) - started = self.clock.monotonic() - - def observe(timed_out: bool) -> None: - pool._record_block_ledger_call( - call_class=call_class, - budget_seconds=budget, - duration_seconds=max(0.0, self.clock.monotonic() - started), - timed_out=timed_out, - ) - - try: - with service._block_submitter_ledger_timeout_scope(budget): - result = operation() - except BaseException as exc: - observe(isinstance(exc, TimeoutError)) - raise - observe(False) - return result - - pool._run_block_submitter_ledger_call = run_inline - - def _install_inline_external_lease_fence(self, pool: Any) -> None: - """Prove the guarded session on the caller's own actor. - - ``_require_fresh_ledger_lease_for_external_side_effect`` gates two - landing steps that reach outside PostgreSQL: the fallback lane's - ``submitblock``, and the restore of a superseded audit envelope. Both - gates matter here — the second is the authority check #133's fix - depends on — so the harness keeps them, and keeps the real - verification statement behind them. - - What it cannot keep is the shape of the wait. The production fence - runs the verification on a spawned thread and joins it in wall-clock - time, so that it can hard-exit a process whose guard session has - genuinely stopped answering. The baton cannot schedule that thread - (see ``_install_inline_outbox_worker``), and a verification that runs - unscheduled would take its statement outside the schedule. Running it - inline preserves every outcome the landing branches on: the - verification's own SQL, the deferred-renewal refusal, and a refusal - that withholds restore authority. - - Out of reach, and named rather than implied: the fence's *timeout* - arm — a guard session that answers too slowly, or not at all. - """ - from lab.prism.background_services import guard_session_verifier - from lab.prism.coordinator_shutdown import ShutdownInProgress - from lab.prism.share_ledger import WriterLeaseRenewalDeferred - - def require_fresh_lease(component: str) -> None: - if self.withheld_lease_components is not None and ( - component in self.withheld_lease_components - ): - raise ShutdownInProgress( - f"writer lease guard verification failed before {component}" - ) - ledger = pool.ledger - if not bool(getattr(ledger, "writer_lease_guard_required", False)): - return - verify = guard_session_verifier(ledger) - if verify is None: - raise ShutdownInProgress( - f"writer lease guard verification is unavailable before {component}" - ) - self.lease_fence_components.append(component) - try: - result = verify() - except BaseException as exc: - raise ShutdownInProgress( - f"writer lease guard verification failed before {component}" - ) from exc - if isinstance(result, dict) and result.get( - "renewal_deferred_to_own_write" - ): - raise WriterLeaseRenewalDeferred( - f"withholding {component}: writer lease renewal is deferred " - "behind this coordinator's own in-flight fenced write" - ) - - pool._require_fresh_ledger_lease_for_external_side_effect = ( - require_fresh_lease - ) - - def _install_harness_locks(self, pool: Any) -> None: - """Replace every lock the two tails share with a schedulable one. - - Semantics are preserved exactly — a mutex stays a mutex, a - reentrant lock stays reentrant, the condition keeps its lock — and - only the blocking mechanism changes. Nothing in the code under test - can tell the difference except that a contended wait now parks at a - named checkpoint instead of outside the baton. - - This list is deliberately explicit rather than a sweep over - ``threading`` objects: a lock that appears in a later slice and is - not named here will stall the scheduler at a checkpoint that says - exactly which wait was unmodelled, which is the failure mode worth - having. - """ - pool._ensure_job_cache_state() - pool.lock = self.rlock("coordinator") - pool._job_cache_lock = self.lock("job-cache") - pool._payout_balance_mutation_lock = self.rlock("payout-balance-mutation") - pool._payout_append_landing_fence_lock = self.lock("payout-append-landing-fence") - pool._payout_state_prepare_lock = self.rlock("payout-state-prepare") - # Rebuilt rather than reused: the condition is defined over the job - # cache lock, and it has just been replaced. - pool._payout_unfenced_append_drained = HarnessCondition( - pool._job_cache_lock, - name="unfenced-append-drained", - ) - pool._accepted_block_payout_preview_condition = self.condition( - "accepted-block-payout-preview" - ) - pool._tip_refresh_lock = self.lock("tip-refresh") - - def _sql_backend_factory( - self, - conninfo: str, - *, - pool_size: int, - application_name: str, - ) -> FakeSqlBackend: - self.sql_backend = FakeSqlBackend( - self.server, - pool_size=pool_size, - application_name=application_name, - tag="ledger", - ) - return self.sql_backend - - def _lease_guard_factory( - self, - conninfo: str, - *, - advisory_lock_key: int, - ) -> FakeLeaseGuard: - self.guard = FakeLeaseGuard( - self.server, - advisory_lock_key=advisory_lock_key, - application_name="landing-guard", - tag="ledger.guard", - ) - return self.guard - - def _build_ledger(self) -> PsqlShareLedger: - ledger = PsqlShareLedger( - psql_command="psql postgres://harness/qbit", - database_url="postgres://harness/qbit", - writer_id="prism-coordinator", - writer_epoch=1, - writer_session_token=self.session_token, - initialize_schema=False, - lease_ttl_seconds=self.lease_ttl_seconds, - read_concurrency=self.read_concurrency, - monotonic=self.clock.monotonic, - lease_retry_sleep=self.sleep, - sql_backend_factory=self._sql_backend_factory, - lease_guard_factory=self._lease_guard_factory, - # Canonicalization is a Rust binary that validates audit bundle - # schema. It is the audit machine, not the landing machine, and - # the bundle here is a fixed stand-in; the constructor already - # takes the seam. - audit_bundle_canonicalizer=_canonicalize_bundle, - ) - # The ledger's own admission gates. Both tails queue on them, and - # #125 is precisely about how long a caller waits here, so they have - # to be visible to the scheduler and to the virtual clock. - ledger._lock = self.lock("ledger-writer") - ledger._read_semaphore = self.semaphore( - "ledger-read-slot", - self.read_concurrency, - ) - # The accepted-share-stats single-flight. Both tails read the stats - # while building their evidence, so the second one queues here behind - # the first — which is exactly the confirm→publish gap #133 needs to - # be *open*, not blocked. - ledger._stats_refresh_lock = self.lock("ledger-stats-refresh") - ledger._stats_lock = self.lock("ledger-stats") - return ledger - - def boot(self) -> PsqlShareLedger: - """Acquire the writer lease and attach the ledger to the pool. - - Runs on its own actor because lease acquisition is modelled SQL. - Returns once the lease is held, so a scenario that only cares about - landing never has to step through startup. - """ - call = self.startup.submit(self._build_ledger, label="ledger-startup") - self.run_until(self.startup, "done:ledger-startup") - ledger = call.value() - self.pool.ledger = ledger - self._install_audit_store_locks() - return ledger - - def _install_audit_store_locks(self) -> None: - """Make the publication order guard schedulable. - - The guard is what serialises confirm against publish inside one - landing, and both tails take it. #133 is a *released*-guard defect — - the interleaving happens in the gap where neither tail holds it — so - the point of installing a harness lock here is the opposite of - enabling the bug: it lets a scenario prove the guard is genuinely - released in that gap, by having the other tail take it and proceed. - """ - store = self.pool._ensure_audit_artifact_store() - store._lock = self.rlock("audit-store") - store._publication_lifecycle_lock = self.rlock("audit-publication-lifecycle") - - # -- found blocks ------------------------------------------------------ - - def found_block( - self, - block_hash: str, - *, - height: int | None = None, - parent_hash: str | None = None, - anchor_job_issued_at_ms: int = 12_000, - append_epoch: int | None = None, - collection_only: bool = False, - ) -> LandingCandidate: - """Register a job and mint the found-block candidate it produced. - - ``append_epoch`` stamps the candidate's build key with a payout - append-invalidation epoch. It defaults to the live epoch, which is - what an ordinary job carries; a scenario that wants the landing to - see a *moved* epoch bumps the live one afterwards rather than lying - about what the job was built from. - """ - self._job_counter += 1 - job_id = f"job-{self._job_counter}" - block_height = self.block_height if height is None else int(height) - parent = self.parent_hash if parent_hash is None else str(parent_hash) - if append_epoch is None: - append_epoch = int(self.pool._payout_ledger_append_invalidation_epoch) - context = SimpleNamespace( - job=SimpleNamespace( - job_id=job_id, - share_target=(1 << 256) - 1, - share_difficulty=1, - transaction_hexes=(), - ), - template={ - "previousblockhash": parent, - "height": block_height, - "coinbasevalue": 50_00000000, - }, - found_block={ - "network_difficulty": 1, - "anchor_job_issued_at_ms": int(anchor_job_issued_at_ms), - }, - issued_at_ms=int(anchor_job_issued_at_ms), - collection_only=collection_only, - worker=self.client_state.worker, - shares_json=[], - prior_balances=[], - payout_append_invalidation_epoch=int(append_epoch), - ) - self.pool.jobs[job_id] = context - self.client_state.active_job_ids.add(job_id) - block_hex = f"block:{block_hash}" - self.rpc.register( - block_hex=block_hex, - block_hash=block_hash, - height=block_height, - ) - submission = SimpleNamespace( - coinbase_tx_hex="c0ffee", - block_hash_hex=block_hash, - block_hex=block_hex, - share_pass=False, - block_pass=True, - ) - candidate = PrismBlockCandidate( - context=context, - submission=submission, - extranonce1_hex=self.client_state.extranonce1_hex, - extranonce2_hex="00" * 8, - # A real PendingShare, not a stand-in: the durable outbox intent - # encodes it, so the row the landing replays from is only as - # honest as this is. - pending_share=PendingShare( - share_id=f"miner-a:{block_hash}", - miner_id="miner-a", - order_key="miner-a", - p2mr_program_hex="11" * 32, - share_difficulty=1, - network_difficulty=1, - template_height=block_height - 1, - job_id=job_id, - job_issued_at_ms=int(anchor_job_issued_at_ms), - accepted_at_ms=int(anchor_job_issued_at_ms), - ntime=1_700_000_000, - ), - client=self.client_state, - credit_share_on_accept=False, - ) - return LandingCandidate( - block_hash=block_hash, - block_height=block_height, - job_id=job_id, - candidate=candidate, - ) - - # -- driving the tails ------------------------------------------------- - - def _persist_intent(self, found: LandingCandidate) -> None: - """Write the durable outbox row this candidate replays from. - - Production writes it at the pre-submit boundary, before either tail - runs, and refuses to hold a retry slot until it is durable. The tails - below start after that boundary, so each one writes its own row first - — otherwise the terminal outbox update at the end of a landing would - find nothing to mark and the durable replay source would be missing - from every scenario. - """ - persist_intent = getattr(self.pool.ledger, "persist_block_candidate_intent", None) - if not callable(persist_intent): - return - persist_intent(self.pool.block_candidate_intent(found.candidate)) - - def land_on_client_tail(self, found: LandingCandidate) -> Call: - """Queue ``found`` down the synchronous miner-connection-thread route.""" - - def run() -> bool: - self._persist_intent(found) - return self.pool._submit_synchronous_block_candidate(found.candidate) - - return self.client.submit( - run, - label=f"client-land:{found.block_hash[:4]}", - ) - - def land_on_accounting_tail( - self, - found: LandingCandidate, - *, - node_submission: _BlockCandidateNodeSubmission | None = None, - ) -> Call: - """Queue ``found`` down the accounting-actor route. - - The accounting actor's task carries the fast lane's node-offer - evidence, so the default here is an unattempted offer: the tail then - performs the submit itself, which is the shape both #133 and #126 - need. - """ - submission = node_submission or _BlockCandidateNodeSubmission(attempted=False) - - def run() -> None: - self._persist_intent(found) - # The task the accounting loop dequeues, built the way - # ``submit_next`` builds it: the disposition lease for this hash - # is already claimed, and the loop hands it back afterwards. The - # loop itself is a queue drain, so the harness supplies the task - # rather than the queue. - lease = self.pool._claim_block_candidate_disposition( - found.block_hash, - blocking=True, - ) - if lease is None: - raise HarnessError( - f"disposition for {found.block_hash} was already claimed" - ) - self.pool._run_block_accounting_task( - _BlockCandidateAccountingTask( - candidate=found.candidate, - node_submission=submission, - disposition_lease=lease, - ) - ) - - return self.accounting.submit( - run, - label=f"accounting-land:{found.block_hash[:4]}", - ) - - # -- the append side --------------------------------------------------- - - def publish_window_anchor(self, anchor_ms: int) -> None: - """Expose a newer published job window's anchor to the append side. - - A seedless build hands its declared anchor to this watermark at the - publication fence, so jobs still serving that window keep an anchor - visible to the append-side ``predates`` checks after the build's own - exposure retires. It is the cheapest way for a scenario to say "a - newer window than this candidate's is live", which is the precondition - for an append that invalidates *something* without invalidating the - candidate under test. - """ - anchor = int(anchor_ms) - current = getattr(self.pool, "_payout_published_job_window_anchor_ms", None) - self.pool._payout_published_job_window_anchor_ms = max(anchor, current or 0) - - def append_late_visible_share( - self, - *, - stamp_ms: int, - share_id: str | None = None, - accepted_at_ms: int | None = None, - ) -> Call: - """Make one share durable and let it advance the invalidation epoch. - - Queued on its own actor, because the append side is a third thread in - production (the share writer) and because the bump takes the landing - fence, which a landing may be holding. ``stamp_ms`` sets the job - issue stamp and, unless ``accepted_at_ms`` says otherwise, the - acceptance stamp too. With both stamps equal it is exactly the - quantity the append-side ``predates`` test compares against an - anchor: the row invalidates a window whose declared anchor is at or - after it, and no other. A distinct ``accepted_at_ms`` stages a - straddling row -- issued at or before an anchor, accepted after -- - which predates only anchors at or after *both* stamps, and whose - recorded bump stamp is their max. - - Returns the queued call; its value is the advanced epoch, or None when - no live anchor predated the row. - """ - stamp = int(stamp_ms) - accepted = stamp if accepted_at_ms is None else int(accepted_at_ms) - share = PendingShare( - share_id=share_id or f"appender:{stamp}", - miner_id="miner-b", - order_key="miner-b", - p2mr_program_hex="22" * 32, - share_difficulty=1, - network_difficulty=1, - template_height=self.block_height - 1, - job_id="append-job", - job_issued_at_ms=stamp, - accepted_at_ms=accepted, - ntime=1_700_000_000, - ) - - def run() -> int | None: - self.server.shares.append( - {"share_seq": len(self.server.shares) + 1, "miner_id": share.miner_id} - ) - return self.pool._record_late_visible_payout_append(share) - - label = f"append:{stamp}" if accepted == stamp else f"append:{stamp}:{accepted}" - return self.appender.submit(run, label=label) - - def live_append_epoch(self) -> int: - return int(self.pool._payout_ledger_append_invalidation_epoch) - - # -- observation ------------------------------------------------------- - - @property - def audit_root(self) -> Path: - return self._audit_root - - def live_envelope_path(self, found: LandingCandidate) -> Path: - store = self.pool._ensure_audit_artifact_store() - return store.live_envelope_path( - block_height=found.block_height, - block_hash=found.block_hash, - ) - - def envelope_written(self, found: LandingCandidate) -> bool: - """Whether this block's published evidence pointer exists on disk. - - This is #133's whole assertion. The block is paid and audited - either way; what the defect destroyed was the file. - """ - return self.live_envelope_path(found).exists() - - def pool_block(self, block_hash: str) -> PoolBlockRow | None: - return self.server.pool_blocks.get(block_hash) - - def publication_sequence(self, found: LandingCandidate) -> int | None: - row = self.pool_block(found.block_hash) - return None if row is None else row.audit_publication_sequence - - def publication_floor(self) -> int: - return max( - [ - row.audit_publication_sequence or 0 - for row in self.server.pool_blocks.values() - ], - default=0, - ) - - def outbox_state(self, found: LandingCandidate) -> str | None: - row = self.server.outbox.get(found.block_hash) - return None if row is None else row.state - - def landing_summary(self) -> dict[str, Any]: - """A compact, order-stable digest for ``assert_deterministic``. - - Deliberately not the whole world: it names the durable facts a - landing scenario asserts about, so two runs that agree here and on - the checkpoint trace agree on everything a scenario claims. - """ - return { - "pool_blocks": [ - { - "block_hash": row.block_hash, - "block_height": row.block_height, - "chain_state": row.chain_state, - "maturity_state": row.maturity_state, - "audit_publication_sequence": row.audit_publication_sequence, - } - for row in self.server.pool_blocks.values() - ], - "outbox": [ - { - "block_hash": row.block_hash, - "state": row.state, - "attempt_count": row.attempt_count, - "last_error": row.last_error, - } - for row in self.server.outbox.values() - ], - "statements": self.statement_kinds(), - "submitted_blocks": list(self.rpc.submitted), - "envelopes": sorted( - path.name - for path in (self._audit_root / "audit").rglob("*.json") - if path.is_file() - ), - } - - -# -------------------------------------------------------------------------- -# Fixed audit answers -# -------------------------------------------------------------------------- - - -def _canonicalize_bundle(bundle: dict[str, Any]) -> bytes: - return json.dumps(bundle, sort_keys=True, separators=(",", ":")).encode("utf-8") - - -def _verified_block_bundle(coinbase_tx_hex: str = "c0ffee") -> dict[str, object]: - return { - "found_block": {"coinbase_value_sats": 50_00000000}, - "ledger_window_attestation": {"signature": {"public_key_hex": "aa" * 32}}, - "payout_policy_manifest": {"accounts": []}, - "signed_coinbase_manifest": { - "manifest": { - "coinbase_tx_hex": coinbase_tx_hex, - "payout_count": 1, - } - }, - } - - -def _verified_audit_report( - coinbase_tx_hex: str = "c0ffee", - *, - block_height: int = DEFAULT_HEIGHT, -) -> dict[str, object]: - return { - "schema": "qbit.prism.audit-verification-report.v1", - "block_height": int(block_height), - "coinbase_value_sats": 50_00000000, - "reward_manifest_sha256_hex": "44" * 32, - "payout_policy_manifest_sha256_hex": "55" * 32, - "prism_audit_commitment_leaf_hex": "66" * 32, - "audit_commitment_root_hex": "77" * 32, - "coinbase_txid": "11" * 32, - "coinbase_wtxid": "88" * 32, - "coinbase_manifest_sha256_hex": "22" * 32, - "audit_bundle_sha256_hex": "33" * 32, - "coinbase_tx_hex": coinbase_tx_hex, - "min_output_sats": 1, - "onchain_output_count": 0, - "accrued_account_count": 0, - } - - -__all__ = [ - "DEFAULT_HEIGHT", - "PARENT_HASH", - "LandingCandidate", - "LandingHarness", - "LandingRpc", -] diff --git a/tests/prism_postgres_a1_gate.py b/tests/prism_postgres_a1_gate.py deleted file mode 100644 index d40c7caa6..000000000 --- a/tests/prism_postgres_a1_gate.py +++ /dev/null @@ -1,2306 +0,0 @@ -"""Non-discovered PostgreSQL/A1 integration gate. - -This helper is invoked by ``test/test-prism-postgres-ledger.sh``. It lives -outside unittest discovery because every scenario requires an explicitly -provisioned PostgreSQL target. -""" - -from __future__ import annotations - -import atexit -import hashlib -import json -import os -import re -import shlex -import signal -import subprocess -import tempfile -import threading -import time -from datetime import datetime, timezone -from typing import Any - -from lab.prism import public_api -from lab.prism.share_ledger import PsqlShareLedger, SingleWriterShareLedger - - -PSQL_TIMEOUT_SECONDS = 30.0 -PSQL_OUTPUT_LIMIT_BYTES = 1 << 20 -SCHEMA_PATTERN = re.compile(r"qbit_a1_[a-z0-9_]+") -BASE_PSQL_COMMAND = os.environ.get("PRISM_PSQL_COMMAND", "") -if not BASE_PSQL_COMMAND: - raise SystemExit("PRISM_PSQL_COMMAND is required") -BASE_PSQL_ARGV = shlex.split(BASE_PSQL_COMMAND) -if not BASE_PSQL_ARGV: - raise SystemExit("PRISM_PSQL_COMMAND is empty") - -RUN_TOKEN = os.urandom(16).hex() -RUN_MARKER = f"qbit-a1-test:{RUN_TOKEN}" -OWNED_SCHEMAS: list[tuple[str, str]] = [] -ACTIVE_CHILDREN: set[subprocess.Popen[str]] = set() -ACTIVE_CHILDREN_LOCK = threading.Lock() - - -def fake_bundle_bytes(payload: dict[str, object]) -> bytes: - return json.dumps(payload, separators=(",", ":")).encode() - - -PERSIST_BUNDLE: dict[str, object] = { - "signed_coinbase_manifest": {"manifest": {"payout_count": 1}}, - "payout_policy_manifest": { - "accounts": [ - { - "recipient_id": "a1-parity-miner", - "order_key": "a1-parity-order", - "p2mr_program_hex": "42" * 32, - "gross_amount_sats": 1000, - "prior_balance_sats": 0, - "candidate_balance_sats": 1000, - "onchain_amount_sats": 0, - "carry_forward_balance_sats": 1000, - "action": "accrued", - } - ] - }, -} -PERSIST_BUNDLE_SHA256 = hashlib.sha256( - fake_bundle_bytes(PERSIST_BUNDLE) -).hexdigest() -PERSIST_REPORT: dict[str, object] = { - "coinbase_txid": "20" * 32, - "coinbase_manifest_sha256_hex": "30" * 32, - "audit_bundle_sha256_hex": PERSIST_BUNDLE_SHA256, - "coinbase_tx_hex": "00", -} - - -class GateFailure(RuntimeError): - pass - - -def assert_equal(actual: object, expected: object, message: str) -> None: - if actual != expected: - raise GateFailure(f"{message}: expected {expected!r}, got {actual!r}") - - -def _terminate_and_reap(process: subprocess.Popen[str]) -> None: - if process.poll() is not None: - process.wait() - return - try: - os.killpg(process.pid, signal.SIGTERM) - except ProcessLookupError: - pass - try: - process.wait(timeout=2.0) - return - except subprocess.TimeoutExpired: - pass - try: - os.killpg(process.pid, signal.SIGKILL) - except ProcessLookupError: - pass - process.wait(timeout=2.0) - - -def cleanup_active_children() -> None: - with ACTIVE_CHILDREN_LOCK: - children = list(ACTIVE_CHILDREN) - for process in children: - _terminate_and_reap(process) - for stream in (process.stdin, process.stdout, process.stderr): - if stream is not None and not stream.closed: - stream.close() - with ACTIVE_CHILDREN_LOCK: - ACTIVE_CHILDREN.discard(process) - - -def run_psql(sql: str, *, schema: str | None = None) -> str: - if schema is not None and SCHEMA_PATTERN.fullmatch(schema) is None: - raise GateFailure(f"invalid scoped schema: {schema!r}") - scoped_sql = sql - if schema is not None: - scoped_sql = ( - "SET statement_timeout = '20s';\n" - "SET lock_timeout = '10s';\n" - f'SET search_path TO "{schema}", pg_catalog;\n' - + sql - ) - command = [ - *BASE_PSQL_ARGV, - "--no-psqlrc", - "--set", - "ON_ERROR_STOP=1", - "--tuples-only", - "--no-align", - "--quiet", - ] - with ( - tempfile.TemporaryFile(mode="w+t", encoding="utf-8") as stdout_file, - tempfile.TemporaryFile(mode="w+t", encoding="utf-8") as stderr_file, - ): - process = subprocess.Popen( - command, - stdin=subprocess.PIPE, - stdout=stdout_file, - stderr=stderr_file, - text=True, - start_new_session=True, - ) - with ACTIVE_CHILDREN_LOCK: - ACTIVE_CHILDREN.add(process) - try: - process.communicate( - scoped_sql, - timeout=PSQL_TIMEOUT_SECONDS, - ) - except BaseException: - _terminate_and_reap(process) - raise - finally: - if process.poll() is None: - _terminate_and_reap(process) - if process.stdin is not None and not process.stdin.closed: - process.stdin.close() - with ACTIVE_CHILDREN_LOCK: - ACTIVE_CHILDREN.discard(process) - stdout_file.seek(0) - stderr_file.seek(0) - stdout = stdout_file.read(PSQL_OUTPUT_LIMIT_BYTES + 1) - stderr = stderr_file.read(PSQL_OUTPUT_LIMIT_BYTES + 1) - if ( - len(stdout.encode("utf-8")) > PSQL_OUTPUT_LIMIT_BYTES - or len(stderr.encode("utf-8")) > PSQL_OUTPUT_LIMIT_BYTES - ): - raise GateFailure("psql output exceeded 1 MiB limit") - if process.returncode != 0: - raise GateFailure( - f"psql failed with exit {process.returncode}: {stderr.strip()}" - ) - return stdout - - -def run_json(sql: str, *, schema: str | None = None) -> Any: - output = run_psql(sql, schema=schema).strip() - if not output: - raise GateFailure("psql query returned no JSON") - return json.loads(output.splitlines()[-1]) - - -def _schema_registered(schema: str) -> bool: - return any(name == schema for name, _marker in OWNED_SCHEMAS) - - -def create_owned_schema(label: str) -> str: - if re.fullmatch(r"[a-z0-9_]+", label) is None: - raise GateFailure(f"invalid schema label: {label!r}") - schema = f"qbit_a1_{label}_{os.getpid()}_{os.urandom(5).hex()}" - if SCHEMA_PATTERN.fullmatch(schema) is None or len(schema) > 63: - raise GateFailure(f"invalid generated schema: {schema!r}") - # Register authority before the transaction starts. If this call is - # interrupted, run_psql kills/reaps its child before atexit examines the - # durable marker. - OWNED_SCHEMAS.append((schema, RUN_MARKER)) - run_psql( - f""" -BEGIN; -CREATE SCHEMA "{schema}"; -COMMENT ON SCHEMA "{schema}" IS '{RUN_MARKER}'; -COMMIT; -""" - ) - return schema - - -def cleanup_owned_schemas() -> None: - while OWNED_SCHEMAS: - schema, expected_marker = OWNED_SCHEMAS[-1] - if SCHEMA_PATTERN.fullmatch(schema) is None: - raise GateFailure(f"refusing to drop invalid schema: {schema!r}") - marker = run_json( - f""" -SELECT json_build_object( - 'marker', ( - SELECT obj_description(oid, 'pg_namespace') - FROM pg_namespace - WHERE nspname = '{schema}' - ) -); -""" - )["marker"] - if marker is None: - OWNED_SCHEMAS.pop() - continue - if marker != expected_marker: - raise GateFailure( - f"refusing to drop schema with wrong marker: {schema!r}" - ) - run_psql(f'DROP SCHEMA "{schema}" CASCADE;') - OWNED_SCHEMAS.pop() - - -def _signal_exit(signum: int, _frame: object) -> None: - # Raising first lets the active run_psql frame kill and reap its exact - # process group. Registered atexit cleanup runs only after that unwind. - raise SystemExit(128 + signum) - - -signal.signal(signal.SIGTERM, _signal_exit) -signal.signal(signal.SIGINT, _signal_exit) -atexit.register(cleanup_owned_schemas) -atexit.register(cleanup_active_children) - - -class ScopedPsqlLedger(PsqlShareLedger): - def __init__(self, *, test_schema: str, **kwargs: object) -> None: - if not _schema_registered(test_schema): - raise GateFailure("ledger requires a registered owned schema") - self._test_schema = test_schema - kwargs["psql_command"] = BASE_PSQL_COMMAND - kwargs["native_client_mode"] = "psql" - super().__init__(**kwargs) # type: ignore[arg-type] - - def _run_sql(self, sql: str) -> str: - if not _schema_registered(self._test_schema): - raise GateFailure("ledger schema authority was revoked") - return run_psql(sql, schema=self._test_schema) - - -def public_sentinel() -> dict[str, object]: - return run_json( - """ -SELECT json_build_object( - 'pool_oid', 'public.qbit_pool_blocks'::regclass::oid, - 'pool_rows', ( - SELECT md5(COALESCE( - string_agg(row_to_json(row_value)::text, ',' ORDER BY block_hash), - '' - )) - FROM public.qbit_pool_blocks row_value - ), - 'sequence_oid', - 'public.qbit_audit_publication_sequence_seq'::regclass::oid, - 'sequence_definition', ( - SELECT json_build_object( - 'type_oid', sequence.seqtypid, - 'start', sequence.seqstart, - 'increment', sequence.seqincrement, - 'max', sequence.seqmax, - 'min', sequence.seqmin, - 'cache', sequence.seqcache, - 'cycle', sequence.seqcycle, - 'persistence', relation.relpersistence, - 'owner', relation.relowner - ) - FROM pg_catalog.pg_sequence sequence - JOIN pg_catalog.pg_class relation - ON relation.oid = sequence.seqrelid - WHERE sequence.seqrelid = - 'public.qbit_audit_publication_sequence_seq'::regclass - ), - 'sequence_state', ( - SELECT row_to_json(state) - FROM ( - SELECT last_value, is_called - FROM public.qbit_audit_publication_sequence_seq - ) state - ), - 'index_oid', - 'public.qbit_pool_blocks_audit_publication_sequence_idx'::regclass::oid, - 'index_filenode', pg_relation_filenode( - 'public.qbit_pool_blocks_audit_publication_sequence_idx'::regclass - ), - 'index_definition', pg_get_indexdef( - 'public.qbit_pool_blocks_audit_publication_sequence_idx'::regclass - ), - 'index_catalog', ( - SELECT json_build_object( - 'kind', relation.relkind, - 'persistence', relation.relpersistence, - 'owner', relation.relowner, - 'same_owner_as_table', relation.relowner = ( - SELECT pool_blocks.relowner - FROM pg_catalog.pg_class pool_blocks - JOIN pg_catalog.pg_namespace pool_namespace - ON pool_namespace.oid = pool_blocks.relnamespace - WHERE pool_namespace.nspname = 'public' - AND pool_blocks.relname = 'qbit_pool_blocks' - ), - 'unique', index_definition.indisunique, - 'valid', index_definition.indisvalid, - 'ready', index_definition.indisready, - 'live', index_definition.indislive, - 'immediate', index_definition.indimmediate, - 'primary', index_definition.indisprimary, - 'exclusion', index_definition.indisexclusion, - 'clustered', index_definition.indisclustered, - 'replica_identity', index_definition.indisreplident, - 'nulls_not_distinct', index_definition.indnullsnotdistinct, - 'key_count', index_definition.indnkeyatts, - 'attribute_count', index_definition.indnatts, - 'keys', index_definition.indkey::text, - 'classes', index_definition.indclass::text, - 'collations', index_definition.indcollation::text, - 'options', index_definition.indoption::text, - 'expressions', index_definition.indexprs::text, - 'predicate', index_definition.indpred::text - ) - FROM pg_catalog.pg_index index_definition - JOIN pg_catalog.pg_class relation - ON relation.oid = index_definition.indexrelid - WHERE index_definition.indexrelid = - 'public.qbit_pool_blocks_audit_publication_sequence_idx'::regclass - ), - 'constraint', ( - SELECT json_build_object( - 'oid', oid, - 'conbin', conbin::text, - 'definition', pg_get_constraintdef(oid, true), - 'validated', convalidated, - 'name', conname, - 'type', contype, - 'keys', conkey::text, - 'local', conislocal, - 'inherited_count', coninhcount, - 'no_inherit', connoinherit, - 'deferrable', condeferrable, - 'deferred', condeferred - ) - FROM pg_constraint - WHERE conrelid = 'public.qbit_pool_blocks'::regclass - AND conname = - 'qbit_pool_blocks_audit_publication_sequence_check' - ) -); -""" - ) - - -def marker_schema_count() -> int: - value = run_json( - f""" -SELECT json_build_object( - 'count', count(*) -) -FROM pg_namespace -WHERE obj_description(oid, 'pg_namespace') = '{RUN_MARKER}'; -""" - )["count"] - return int(value) - - -def allocator_state(ledger: ScopedPsqlLedger) -> dict[str, object]: - return ledger._run_json( - """ -SELECT json_build_object( - 'last_value', last_value, - 'is_called', is_called -) -FROM qbit_audit_publication_sequence_seq; -""" - ) - - -def seed_prepared_direct( - postgres: ScopedPsqlLedger, - memory: SingleWriterShareLedger, - *, - block_hash: str, - block_height: int, -) -> None: - postgres._run_sql( - f""" -INSERT INTO qbit_pool_blocks ( - block_hash, - block_height, - parent_hash, - coinbase_txid, - payout_manifest_sha256, - chain_state, - maturity_state -) VALUES ( - '{block_hash}', - {block_height}, - '{'10' * 32}', - '{'20' * 32}', - '{'30' * 32}', - 'prepared', - 'immature' -) -ON CONFLICT (block_hash) DO NOTHING; -""" - ) - memory.persist_accepted_block( - block_hash=block_hash, - block_height=block_height, - parent_hash="10" * 32, - final_bundle={}, - audit_report={}, - ) - - -def normalized_response(payload: dict[str, int | str]) -> dict[str, int | str]: - return {**payload, "backend": "ledger"} - - -def assert_states_equal( - postgres: ScopedPsqlLedger, - memory: SingleWriterShareLedger, - block_hashes: list[str], - message: str, -) -> None: - for block_hash in block_hashes: - assert_equal( - postgres.pool_block_state(block_hash=block_hash), - memory.pool_block_state(block_hash=block_hash), - f"{message} state {block_hash}", - ) - assert_equal( - postgres.audit_publication_sequence_floor(), - memory.audit_publication_sequence_floor(), - f"{message} durable floor", - ) - - -def call_both( - postgres: ScopedPsqlLedger, - memory: SingleWriterShareLedger, - method: str, - *, - block_hash: str, - active_tip_height: int, - known_hashes: list[str], - message: str, -) -> dict[str, int | str]: - postgres_result = getattr(postgres, method)( - block_hash=block_hash, - active_tip_height=active_tip_height, - ) - memory_result = getattr(memory, method)( - block_hash=block_hash, - active_tip_height=active_tip_height, - ) - assert_equal( - normalized_response(postgres_result), - normalized_response(memory_result), - f"{message} response", - ) - assert_states_equal(postgres, memory, known_hashes, message) - return postgres_result - - -def test_exact_transition_parity() -> None: - schema = create_owned_schema("parity") - postgres = ScopedPsqlLedger( - test_schema=schema, - writer_id="a1-parity", - writer_epoch=1, - initialize_schema=True, - audit_bundle_canonicalizer=fake_bundle_bytes, - ) - memory = SingleWriterShareLedger() - try: - assert_equal( - postgres._run_json( - "SELECT json_build_object('schema', current_schema());" - )["schema"], - schema, - "parity current schema", - ) - assert_equal(allocator_state(postgres), {"last_value": 1, "is_called": False}, "parity fresh allocator") - block_a = "a1" * 32 - block_b = "b1" * 32 - block_c = "c1" * 32 - known = [block_a] - seed_prepared_direct(postgres, memory, block_hash=block_a, block_height=10) - seed_prepared_direct(postgres, memory, block_hash=block_a, block_height=10) - assert_states_equal(postgres, memory, known, "duplicate direct prepared seed") - - api_hash = "d1" * 32 - expected_inline_body_bytes = fake_bundle_bytes(PERSIST_BUNDLE) - expected_inline_body_byte_len = len(expected_inline_body_bytes) - postgres_first_persist = postgres.persist_accepted_block( - block_hash=api_hash, - block_height=30, - parent_hash="10" * 32, - final_bundle=PERSIST_BUNDLE, - audit_report=PERSIST_REPORT, - ) - memory_first_persist = memory.persist_accepted_block( - block_hash=api_hash, - block_height=30, - parent_hash="10" * 32, - final_bundle=PERSIST_BUNDLE, - audit_report=PERSIST_REPORT, - ) - assert_equal( - postgres_first_persist, - { - "backend": "postgres-psql", - "share_count": 0, - "block_count": 1, - "bundle_count": 1, - "payout_entry_count": 1, - "carry_forward_count": 1, - "onchain_output_count": 1, - "audit_bundle_sha256": PERSIST_BUNDLE_SHA256, - "body_uri": "", - "audit_body_byte_len": expected_inline_body_byte_len, - }, - "PostgreSQL first persist API response", - ) - persisted_inline_body = postgres._run_json( - f""" -SELECT json_build_object( - 'audit_bundle_present', audit_bundle IS NOT NULL, - 'body_uri_is_null', body_uri IS NULL, - 'audit_body_byte_len', audit_body_byte_len, - 'audit_bundle_sha256', audit_bundle_sha256, - 'audit_bundle', audit_bundle -) -FROM qbit_pool_audit_bundles -WHERE block_hash = '{api_hash}'; -""" - ) - assert_equal( - persisted_inline_body, - { - "audit_bundle_present": True, - "body_uri_is_null": True, - "audit_body_byte_len": expected_inline_body_byte_len, - "audit_bundle_sha256": hashlib.sha256( - expected_inline_body_bytes - ).hexdigest(), - "audit_bundle": PERSIST_BUNDLE, - }, - "PostgreSQL inline audit body row", - ) - assert_equal( - memory_first_persist, - { - "backend": "memory", - "share_count": 0, - "block_count": 0, - "payout_entry_count": 0, - "carry_forward_count": 0, - }, - "memory first persist API response", - ) - postgres_duplicate_persist = postgres.persist_accepted_block( - block_hash=api_hash, - block_height=30, - parent_hash="10" * 32, - final_bundle=PERSIST_BUNDLE, - audit_report=PERSIST_REPORT, - ) - memory_duplicate_persist = memory.persist_accepted_block( - block_hash=api_hash, - block_height=30, - parent_hash="10" * 32, - final_bundle=PERSIST_BUNDLE, - audit_report=PERSIST_REPORT, - ) - assert_equal( - postgres_duplicate_persist, - postgres_first_persist, - "PostgreSQL duplicate persist exact response", - ) - assert_equal( - memory_duplicate_persist, - memory_first_persist, - "memory duplicate persist exact response", - ) - known.append(api_hash) - assert_states_equal(postgres, memory, known, "public persist API parity") - - # The confirm disposition matrix. Every value is asserted on both - # backends through call_both, which compares the full responses, so - # the memory ledger stays a parity implementation of the SQL function: - # 1 fresh flip out of 'prepared', allocates a publication ordinal - # 2 idempotent replay of an already-confirmed row, allocates none - # 0 no row, or a live row this confirmation does not match - # -1 terminally disposed before the confirmation landed - # 1 and 2 are the split issue #61 needs: only a fresh confirmation has - # payout state to publish. - before = allocator_state(postgres) - unmatched = call_both(postgres, memory, "confirm_accepted_block", block_hash=block_a, active_tip_height=9, known_hashes=known, message="wrong-height confirmation") - assert_equal(unmatched["confirmed_count"], 0, "wrong-height confirmation disposition") - assert_equal(allocator_state(postgres), before, "wrong-height confirmation allocator immobility") - confirmed = call_both(postgres, memory, "confirm_accepted_block", block_hash=block_a, active_tip_height=10, known_hashes=known, message="first confirmation") - assert_equal(confirmed["confirmed_count"], 1, "first confirmation disposition") - assert_equal(confirmed["audit_publication_sequence"], 1, "first event ordinal") - before = allocator_state(postgres) - replay = call_both(postgres, memory, "confirm_accepted_block", block_hash=block_a, active_tip_height=10, known_hashes=known, message="exact confirmation replay") - assert_equal(replay["confirmed_count"], 2, "exact confirmation replay disposition") - assert_equal(replay["audit_publication_sequence"], 1, "replay ordinal") - for replay_round in range(2, 4): - replay = call_both( - postgres, - memory, - "confirm_accepted_block", - block_hash=block_a, - active_tip_height=10, - known_hashes=known, - message=f"exact confirmation replay round {replay_round}", - ) - assert_equal( - replay["confirmed_count"], - 2, - f"replay round {replay_round} disposition", - ) - assert_equal( - replay["audit_publication_sequence"], - 1, - f"replay round {replay_round} ordinal", - ) - assert_equal(allocator_state(postgres), before, "exact replay allocator immobility") - before = allocator_state(postgres) - call_both(postgres, memory, "reactivate_pool_block", block_hash=block_a, active_tip_height=10, known_hashes=known, message="confirmed count-zero reactivation") - assert_equal(allocator_state(postgres), before, "count-zero reactivation allocator immobility") - call_both(postgres, memory, "mark_pool_block_inactive", block_hash=block_a, active_tip_height=10, known_hashes=known, message="inactive transition") - before = allocator_state(postgres) - inactive_confirmation = call_both( - postgres, - memory, - "confirm_accepted_block", - block_hash=block_a, - active_tip_height=10, - known_hashes=known, - message="inactive confirmation", - ) - assert_equal( - inactive_confirmation["confirmed_count"], - -1, - "inactive confirmation disposition", - ) - assert_equal( - allocator_state(postgres), - before, - "inactive confirmation allocator immobility", - ) - call_both(postgres, memory, "reactivate_pool_block", block_hash=block_a, active_tip_height=9, known_hashes=known, message="wrong-height reactivation") - assert_equal(allocator_state(postgres), before, "wrong-height reactivation allocator immobility") - postgres.release_writer_lease() - postgres.close() - postgres = ScopedPsqlLedger( - test_schema=schema, - writer_id="a1-parity-reactivation-restart", - writer_epoch=1, - audit_bundle_canonicalizer=fake_bundle_bytes, - ) - assert_states_equal( - postgres, - memory, - known, - "inactive restart serializer", - ) - assert_equal( - allocator_state(postgres), - before, - "inactive restart allocator preservation", - ) - reactivated = call_both(postgres, memory, "reactivate_pool_block", block_hash=block_a, active_tip_height=10, known_hashes=known, message="reactivation") - assert_equal(reactivated["audit_publication_sequence"], 1, "reactivation preserves published ordinal") - assert_equal(allocator_state(postgres), before, "reactivation allocator immobility") - before = allocator_state(postgres) - reactivated_confirmation = call_both( - postgres, - memory, - "confirm_accepted_block", - block_hash=block_a, - active_tip_height=10, - known_hashes=known, - message="reactivated confirmation replay", - ) - # Reactivation restores 'confirmed' without returning the row to - # 'prepared', so a confirmation after it is an idempotent replay too. - assert_equal( - reactivated_confirmation["confirmed_count"], - 2, - "reactivated confirmation replay disposition", - ) - assert_equal( - reactivated_confirmation["audit_publication_sequence"], - 1, - "reactivated confirmation replay ordinal", - ) - call_both(postgres, memory, "reactivate_pool_block", block_hash=block_a, active_tip_height=10, known_hashes=known, message="reactivation replay") - assert_equal(allocator_state(postgres), before, "reactivation replay allocator immobility") - call_both(postgres, memory, "mark_pool_block_inactive", block_hash=block_a, active_tip_height=10, known_hashes=known, message="second inactive transition") - call_both(postgres, memory, "reverse_immature_block", block_hash=block_a, active_tip_height=10, known_hashes=known, message="terminal reversal") - before = allocator_state(postgres) - postgres.release_writer_lease() - postgres.close() - postgres = ScopedPsqlLedger( - test_schema=schema, - writer_id="a1-parity-reversed-restart", - writer_epoch=1, - audit_bundle_canonicalizer=fake_bundle_bytes, - ) - assert_states_equal(postgres, memory, known, "reversed restart serializer") - assert_equal( - allocator_state(postgres), - before, - "reversed restart allocator preservation", - ) - reversed_confirmation = call_both(postgres, memory, "confirm_accepted_block", block_hash=block_a, active_tip_height=10, known_hashes=known, message="reversed confirmation") - assert_equal( - reversed_confirmation["confirmed_count"], - -1, - "reversed confirmation disposition", - ) - call_both(postgres, memory, "reactivate_pool_block", block_hash=block_a, active_tip_height=10, known_hashes=known, message="reversed reactivation") - assert_equal(allocator_state(postgres), before, "terminal transition allocator immobility") - - known.append(block_b) - seed_prepared_direct(postgres, memory, block_hash=block_b, block_height=1) - lower = call_both(postgres, memory, "confirm_accepted_block", block_hash=block_b, active_tip_height=1, known_hashes=known, message="later lower-height confirmation") - assert_equal(lower["audit_publication_sequence"], 2, "event order dominates block height") - - known.append(block_c) - seed_prepared_direct(postgres, memory, block_hash=block_c, block_height=20) - before = allocator_state(postgres) - call_both(postgres, memory, "reject_prepared_block", block_hash=block_c, active_tip_height=20, known_hashes=known, message="prepared rejection") - call_both(postgres, memory, "reject_prepared_block", block_hash=block_c, active_tip_height=20, known_hashes=known, message="rejection replay") - assert_equal(allocator_state(postgres), before, "rejection allocator immobility") - block_d = "e1" * 32 - known.append(block_d) - seed_prepared_direct(postgres, memory, block_hash=block_d, block_height=0) - final_confirmation = call_both( - postgres, - memory, - "confirm_accepted_block", - block_hash=block_d, - active_tip_height=0, - known_hashes=known, - message="post-replay fresh confirmation", - ) - assert_equal( - final_confirmation["confirmed_count"], - 1, - "post-replay fresh confirmation disposition", - ) - assert_equal( - final_confirmation["audit_publication_sequence"], - 3, - "post-replay exact next ordinal", - ) - assert_equal(postgres.audit_publication_sequence_floor(), 3, "parity final durable floor") - finally: - postgres.release_writer_lease() - postgres.close() - - -LEGACY_POOL_BLOCKS_SQL = """ -CREATE TABLE qbit_pool_blocks ( - block_hash text PRIMARY KEY, - block_height bigint NOT NULL CHECK (block_height >= 0), - parent_hash text NOT NULL, - coinbase_txid text NOT NULL, - payout_manifest_sha256 text NOT NULL, - found_at timestamptz NOT NULL DEFAULT clock_timestamp(), - chain_state text NOT NULL DEFAULT 'prepared' - CHECK (chain_state IN ('prepared', 'confirmed', 'inactive', 'rejected', 'reversed')), - maturity_state text NOT NULL DEFAULT 'immature' - CHECK (maturity_state IN ('immature', 'mature', 'reversed')), - matured_at timestamptz, - disconnected_at timestamptz, - CHECK ((maturity_state = 'mature') = (matured_at IS NOT NULL)), - CHECK ((maturity_state = 'reversed') = (disconnected_at IS NOT NULL)) -); -""" - - -def assert_empty_serializer_case(*, legacy: bool) -> None: - label = "empty_legacy" if legacy else "empty_fresh" - schema = create_owned_schema(label) - if legacy: - run_psql(LEGACY_POOL_BLOCKS_SQL, schema=schema) - first = ScopedPsqlLedger( - test_schema=schema, - writer_id=f"a1-{label}-first", - writer_epoch=1, - initialize_schema=True, - ) - try: - assert_equal(first.audit_publication_sequence_floor(), 0, f"{label} floor") - assert_equal(allocator_state(first), {"last_value": 1, "is_called": False}, f"{label} allocator") - finally: - first.release_writer_lease() - first.close() - second = ScopedPsqlLedger( - test_schema=schema, - writer_id=f"a1-{label}-second", - writer_epoch=1, - initialize_schema=True, - ) - try: - assert_equal(second.audit_publication_sequence_floor(), 0, f"{label} rerun floor") - assert_equal(allocator_state(second), {"last_value": 1, "is_called": False}, f"{label} rerun allocator") - block_hash = hashlib.sha256(label.encode()).hexdigest() - memory = SingleWriterShareLedger() - before = allocator_state(second) - for method, operation in ( - ("confirm_accepted_block", "confirmation"), - ("reactivate_pool_block", "reactivation"), - ("mark_pool_block_inactive", "inactive transition"), - ("reverse_immature_block", "reversal"), - ("reject_prepared_block", "rejection"), - ): - call_both( - second, - memory, - method, - block_hash=block_hash, - active_tip_height=1, - known_hashes=[block_hash], - message=f"{label} missing {operation}", - ) - assert_equal( - second.pool_block_state(block_hash=block_hash), - None, - f"{label} missing state serializer", - ) - assert_equal( - allocator_state(second), - before, - f"{label} missing-row allocator immobility", - ) - seed_prepared_direct(second, memory, block_hash=block_hash, block_height=1) - seed_prepared_direct(second, memory, block_hash=block_hash, block_height=1) - assert_states_equal( - second, - memory, - [block_hash], - f"{label} nullable prepared serializer", - ) - assert_equal( - second.pool_block_state(block_hash=block_hash)[ - "audit_publication_sequence" - ], - None, - f"{label} explicit prepared nullable ordinal", - ) - before = allocator_state(second) - call_both( - second, - memory, - "confirm_accepted_block", - block_hash=block_hash, - active_tip_height=0, - known_hashes=[block_hash], - message=f"{label} wrong-height confirmation", - ) - call_both( - second, - memory, - "reactivate_pool_block", - block_hash=block_hash, - active_tip_height=1, - known_hashes=[block_hash], - message=f"{label} prepared count-zero reactivation", - ) - assert_equal( - allocator_state(second), - before, - f"{label} nullable zero-op allocator immobility", - ) - result = call_both( - second, - memory, - "confirm_accepted_block", - block_hash=block_hash, - active_tip_height=1, - known_hashes=[block_hash], - message=f"{label} first confirmation", - ) - assert_equal(result["audit_publication_sequence"], 1, f"{label} first ordinal") - assert_equal(second.audit_publication_sequence_floor(), 1, f"{label} confirmed floor") - finally: - second.release_writer_lease() - second.close() - - -def transition_binding_state_sql( - *, - target_schema: str, - decoy_schema: str, -) -> str: - return f""" -SELECT json_build_object( - 'target_sequence', ( - SELECT json_build_object('last_value', last_value, 'is_called', is_called) - FROM "{target_schema}".qbit_audit_publication_sequence_seq - ), - 'decoy_sequence', ( - SELECT json_build_object('last_value', last_value, 'is_called', is_called) - FROM {decoy_schema}.qbit_audit_publication_sequence_seq - ), - 'target_rows', ( - SELECT json_object_agg( - block_hash, - json_build_object( - 'chain_state', chain_state, - 'audit_publication_sequence', audit_publication_sequence - ) - ) - FROM "{target_schema}".qbit_pool_blocks - WHERE block_hash IN ('{'71' * 32}', '{'72' * 32}') - ), - 'decoy_rows', ( - SELECT json_object_agg( - block_hash, - json_build_object( - 'chain_state', chain_state, - 'audit_publication_sequence', audit_publication_sequence - ) - ) - FROM {decoy_schema}.qbit_pool_blocks - ), - 'target_lease_changed', ( - SELECT before.value <> - row_to_json(current_lease)::text - FROM a1_target_lease_before before, - "{target_schema}".qbit_ledger_writer_lease current_lease - ), - 'target_lease', ( - SELECT json_build_object( - 'writer_id', writer_id, - 'writer_epoch', writer_epoch, - 'writer_session_token', writer_session_token, - 'active', lease_expires_at > clock_timestamp(), - 'expiry_after_update', lease_expires_at > updated_at - ) - FROM "{target_schema}".qbit_ledger_writer_lease - ), - 'decoy_lease_unchanged', ( - SELECT before.value = row_to_json(current_lease)::text - FROM a1_decoy_lease_before before, - {decoy_schema}.qbit_ledger_writer_lease current_lease - ), - 'decoy_lease', ( - SELECT json_build_object( - 'writer_id', writer_id, - 'writer_epoch', writer_epoch, - 'writer_session_token', writer_session_token - ) - FROM {decoy_schema}.qbit_ledger_writer_lease - ), - 'proconfig', ( - SELECT json_object_agg(procedure.proname, procedure.proconfig[1]) - FROM pg_catalog.pg_proc procedure - JOIN pg_catalog.pg_namespace namespace - ON namespace.oid = procedure.pronamespace - WHERE procedure.oid IN ( - pg_catalog.to_regprocedure( - '"{target_schema}".qbit_confirm_pool_block(' - 'text,bigint,text,bigint,text,interval)' - ), - pg_catalog.to_regprocedure( - '"{target_schema}".qbit_reactivate_pool_block(' - 'text,bigint,text,bigint,text,interval)' - ) - ) - ) -); -""" - - -def seed_transition_binding_target(ledger: ScopedPsqlLedger) -> None: - ledger._run_sql( - f""" -INSERT INTO qbit_pool_blocks ( - block_hash, - audit_publication_sequence, - block_height, - parent_hash, - coinbase_txid, - payout_manifest_sha256, - chain_state, - maturity_state -) VALUES - ('{'71' * 32}', NULL, 10, '{'10' * 32}', '{'20' * 32}', - '{'30' * 32}', 'prepared', 'immature'), - ('{'72' * 32}', NULL, 20, '{'10' * 32}', '{'20' * 32}', - '{'30' * 32}', 'prepared', 'immature'); -""" - ) - confirmed = ledger.confirm_accepted_block( - block_hash="72" * 32, - active_tip_height=20, - ) - assert_equal( - confirmed["audit_publication_sequence"], - 1, - "binding seed confirmation ordinal", - ) - assert_equal( - ledger.mark_pool_block_inactive( - block_hash="72" * 32, - active_tip_height=20, - )["inactive_count"], - 1, - "binding seed inactive transition", - ) - - -def decoy_objects_sql(*, temporary: bool) -> str: - temporary_keyword = "TEMPORARY " if temporary else "" - return f""" -CREATE {temporary_keyword}TABLE qbit_ledger_writer_lease ( - singleton boolean PRIMARY KEY, - writer_id text NOT NULL, - writer_epoch bigint NOT NULL, - writer_session_token text NOT NULL, - lease_expires_at timestamptz NOT NULL, - updated_at timestamptz NOT NULL -); -CREATE {temporary_keyword}TABLE qbit_pool_blocks ( - block_hash text PRIMARY KEY, - audit_publication_sequence bigint, - block_height bigint NOT NULL, - chain_state text NOT NULL, - maturity_state text NOT NULL -); -CREATE {temporary_keyword}SEQUENCE qbit_audit_publication_sequence_seq - AS bigint START WITH 700; -INSERT INTO qbit_ledger_writer_lease VALUES ( - true, - 'a1-binding', - 1, - 'a1-binding-token', - '2040-01-01T00:00:00Z', - '2000-01-01T00:00:00Z' -); -INSERT INTO qbit_pool_blocks VALUES - ('{'71' * 32}', NULL, 10, 'prepared', 'immature'), - ('{'72' * 32}', 700, 20, 'inactive', 'immature'); -""" - - -def assert_transition_binding_case(*, temporary: bool) -> None: - label = "binding_temp" if temporary else "binding_ordinary" - target_schema = create_owned_schema(label) - target = ScopedPsqlLedger( - test_schema=target_schema, - writer_id="a1-binding", - writer_epoch=1, - writer_session_token="a1-binding-token", - initialize_schema=True, - ) - try: - seed_transition_binding_target(target) - if temporary: - decoy_schema = "pg_temp" - caller_setup = decoy_objects_sql(temporary=True) - caller_path = f'pg_temp, "{target_schema}", pg_catalog' - else: - ordinary_decoy = create_owned_schema("binding_decoy") - run_psql(decoy_objects_sql(temporary=False), schema=ordinary_decoy) - decoy_schema = f'"{ordinary_decoy}"' - caller_setup = "" - caller_path = f'"{ordinary_decoy}", "{target_schema}", pg_catalog' - expected_config = f"search_path=pg_catalog, {target_schema}, pg_temp" - result = run_json( - f""" -SET statement_timeout = '20s'; -{caller_setup} -SET search_path TO {caller_path}; -CREATE TEMPORARY TABLE a1_target_lease_before (value text NOT NULL); -INSERT INTO a1_target_lease_before -SELECT row_to_json(target_lease)::text -FROM "{target_schema}".qbit_ledger_writer_lease target_lease; -CREATE TEMPORARY TABLE a1_decoy_lease_before (value text NOT NULL); -INSERT INTO a1_decoy_lease_before -SELECT row_to_json(decoy_lease)::text -FROM {decoy_schema}.qbit_ledger_writer_lease decoy_lease; -SELECT "{target_schema}".qbit_confirm_pool_block( - '{'71' * 32}', 10, 'a1-binding', 1, 'a1-binding-token', interval '5 minutes' -); -SELECT "{target_schema}".qbit_reactivate_pool_block( - '{'72' * 32}', 20, 'a1-binding', 1, 'a1-binding-token', interval '5 minutes' -); -{transition_binding_state_sql(target_schema=target_schema, decoy_schema=decoy_schema)} -""" - ) - assert_equal( - result["proconfig"], - { - "qbit_confirm_pool_block": expected_config, - "qbit_reactivate_pool_block": expected_config, - }, - f"{label} exact transition proconfig", - ) - assert_equal( - result["target_sequence"], - {"last_value": 2, "is_called": True}, - f"{label} target allocator", - ) - assert_equal( - result["decoy_sequence"], - {"last_value": 700, "is_called": False}, - f"{label} decoy allocator preservation", - ) - assert_equal( - result["target_rows"], - { - "71" * 32: { - "chain_state": "confirmed", - "audit_publication_sequence": 2, - }, - "72" * 32: { - "chain_state": "confirmed", - "audit_publication_sequence": 1, - }, - }, - f"{label} target row transitions", - ) - assert_equal( - result["decoy_rows"], - { - "71" * 32: { - "chain_state": "prepared", - "audit_publication_sequence": None, - }, - "72" * 32: { - "chain_state": "inactive", - "audit_publication_sequence": 700, - }, - }, - f"{label} decoy row preservation", - ) - assert_equal( - result["target_lease_changed"], - True, - f"{label} target lease renewal", - ) - assert_equal( - result["target_lease"], - { - "writer_id": "a1-binding", - "writer_epoch": 1, - "writer_session_token": "a1-binding-token", - "active": True, - "expiry_after_update": True, - }, - f"{label} target lease identity", - ) - assert_equal( - result["decoy_lease_unchanged"], - True, - f"{label} decoy lease byte preservation", - ) - assert_equal( - result["decoy_lease"], - { - "writer_id": "a1-binding", - "writer_epoch": 1, - "writer_session_token": "a1-binding-token", - }, - f"{label} decoy lease identity", - ) - finally: - target.release_writer_lease() - target.close() - - -def test_durable_floor_ignores_allocator_gaps() -> None: - schema = create_owned_schema("durable_floor") - ledger = ScopedPsqlLedger( - test_schema=schema, - writer_id="a1-durable-floor", - writer_epoch=1, - initialize_schema=True, - ) - memory_seed = SingleWriterShareLedger() - block_a = "61" * 32 - block_b = "62" * 32 - rejected = "63" * 32 - try: - assert_equal(ledger.audit_publication_sequence_floor(), 0, "empty floor") - seed_prepared_direct( - ledger, - memory_seed, - block_hash=block_a, - block_height=10, - ) - assert_equal( - ledger.pool_block_state(block_hash=block_a)[ - "audit_publication_sequence" - ], - None, - "prepared floor row is nullable", - ) - committed_gap = ledger._run_json( - """ -SELECT json_build_object( - 'value', nextval('qbit_audit_publication_sequence_seq') -); -""" - )["value"] - assert_equal(committed_gap, 1, "committed unattached allocator gap") - assert_equal(ledger.audit_publication_sequence_floor(), 0, "gap floor") - ledger._run_sql( - """ -BEGIN; -SELECT nextval('qbit_audit_publication_sequence_seq'); -ROLLBACK; -""" - ) - assert_equal( - allocator_state(ledger), - {"last_value": 2, "is_called": True}, - "rolled-back allocator gap persists", - ) - assert_equal( - ledger.audit_publication_sequence_floor(), - 0, - "rolled-back gap is not authority", - ) - confirmed = ledger.confirm_accepted_block( - block_hash=block_a, - active_tip_height=10, - ) - assert_equal( - confirmed["audit_publication_sequence"], - 3, - "confirmation follows both allocator gaps", - ) - before = allocator_state(ledger) - for _round in range(3): - replay = ledger.confirm_accepted_block( - block_hash=block_a, - active_tip_height=10, - ) - assert_equal( - replay["confirmed_count"], - 2, - "gap-vector confirmation replay disposition", - ) - assert_equal( - replay["audit_publication_sequence"], - 3, - "gap-vector confirmation replay", - ) - assert_equal(allocator_state(ledger), before, "gap replay immobility") - ledger.mark_pool_block_inactive( - block_hash=block_a, - active_tip_height=10, - ) - assert_equal(ledger.audit_publication_sequence_floor(), 3, "inactive floor") - reactivated = ledger.reactivate_pool_block( - block_hash=block_a, - active_tip_height=10, - ) - assert_equal( - reactivated["audit_publication_sequence"], - 3, - "reactivation preserves ordinal after gaps", - ) - assert_equal( - allocator_state(ledger), - before, - "gap-vector reactivation allocator immobility", - ) - before = allocator_state(ledger) - assert_equal( - ledger.reactivate_pool_block( - block_hash=block_a, - active_tip_height=10, - ), - {"backend": "postgres-psql", "reactivated_count": 0}, - "gap-vector reactivation replay", - ) - assert_equal(allocator_state(ledger), before, "reactivation replay immobility") - ledger.mark_pool_block_inactive( - block_hash=block_a, - active_tip_height=10, - ) - ledger.reverse_immature_block( - block_hash=block_a, - active_tip_height=10, - ) - assert_equal(ledger.audit_publication_sequence_floor(), 3, "reversed floor") - seed_prepared_direct( - ledger, - memory_seed, - block_hash=rejected, - block_height=20, - ) - before = allocator_state(ledger) - assert_equal( - ledger.reject_prepared_block( - block_hash=rejected, - active_tip_height=20, - )["rejected_count"], - 1, - "rejected nullable floor row", - ) - assert_equal(allocator_state(ledger), before, "rejection allocator immobility") - assert_equal(ledger.audit_publication_sequence_floor(), 3, "rejected floor") - assert_equal( - ledger._run_json( - """ -SELECT json_build_object( - 'value', nextval('qbit_audit_publication_sequence_seq') -); -""" - )["value"], - 4, - "second unattached allocator gap", - ) - seed_prepared_direct( - ledger, - memory_seed, - block_hash=block_b, - block_height=1, - ) - assert_equal( - ledger.confirm_accepted_block( - block_hash=block_b, - active_tip_height=1, - )["audit_publication_sequence"], - 5, - "post-gap event ordinal", - ) - ledger._run_sql( - f""" -INSERT INTO qbit_pool_blocks ( - block_hash, audit_publication_sequence, block_height, parent_hash, - coinbase_txid, payout_manifest_sha256, chain_state, maturity_state -) VALUES - ('{'64' * 32}', 9000000000, 30, '{'10' * 32}', '{'20' * 32}', - '{'30' * 32}', 'prepared', 'immature'), - ('{'65' * 32}', 9000000001, 31, '{'10' * 32}', '{'20' * 32}', - '{'30' * 32}', 'rejected', 'immature'); -""" - ) - assert_equal( - ledger.audit_publication_sequence_floor(), - 9_000_000_001, - "partial-state bigint durable floor", - ) - finally: - ledger.release_writer_lease() - ledger.close() - restart = ScopedPsqlLedger( - test_schema=schema, - writer_id="a1-durable-floor-restart", - writer_epoch=1, - initialize_schema=True, - ) - try: - assert_equal( - restart.audit_publication_sequence_floor(), - 9_000_000_001, - "restart bigint durable floor", - ) - assert_equal( - restart.pool_block_state(block_hash=block_a), - { - "block_hash": block_a, - "block_height": 10, - "parent_hash": "10" * 32, - "chain_state": "reversed", - "maturity_state": "reversed", - "audit_publication_sequence": 3, - }, - "restart exact reversed serializer", - ) - assert_equal( - restart.pool_block_state(block_hash=rejected)[ - "audit_publication_sequence" - ], - None, - "restart exact rejected nullable serializer", - ) - finally: - restart.release_writer_lease() - restart.close() - - -HASHRATE_SERIES_SUBJECTS: list[tuple[str, str | None]] = [ - ("pool", None), - ("miner", "m1"), - ("miner", "m2"), - # An unknown miner keeps the empty-series ('[]') path in the sweep. - ("miner", "m3"), -] -# (range_id, bucket, lookback_seconds); the internal "window" range id rides -# the same statement as the public ranges and must stay working. -HASHRATE_SERIES_RANGES: list[tuple[str, str, int]] = [ - ("all", "5m", 0), - ("all", "1h", 0), - ("all", "1d", 0), - ("1w", "5m", 900), - ("1w", "1h", 0), - ("1m", "1h", 3600), - ("6m", "1d", 0), - ("window", "5m", 900), -] - - -def seed_hashrate_shares( - ledger: ScopedPsqlLedger, - shares: list[dict[str, object]], -) -> None: - """Insert shares in list order so share_seq mirrors the list index + 1.""" - values = [] - for index, share in enumerate(shares, start=1): - accepted = bool(share.get("accepted", True)) - values.append( - "(" - f"'hashrate-share-{index}', " - f"'{share['miner_id']}', " - f"'{index:04d}', " - f"decode('{index:02x}' || repeat('00', 31), 'hex'), " - f"{int(share['share_difficulty'])}, " # type: ignore[call-overload] - "1000, 10, " - f"'hashrate-job-{index}', " - f"to_timestamp({int(share['accepted_epoch']) - 1}), " # type: ignore[call-overload] - f"{1_700_000_000 + index}, " - f"to_timestamp({int(share['accepted_epoch'])}), " # type: ignore[call-overload] - f"{'true' if accepted else 'false'}, " - f"{'NULL' if accepted else chr(39) + 'low-difficulty' + chr(39)}, " - "'a1-hashrate', 1" - ")" - ) - ledger._run_sql( - "INSERT INTO qbit_share_ledger (\n" - " share_id, miner_id, payout_order_key, p2mr_program,\n" - " share_difficulty, network_difficulty, template_height, job_id,\n" - " job_issued_at, ntime, accepted_at, accepted, reject_reason,\n" - " writer_id, writer_epoch\n" - ") VALUES\n " - + ",\n ".join(values) - + ";" - ) - - -def hashrate_series_matrix( - ledger: ScopedPsqlLedger, - *, - anchor: int, - forced_raw: bool, -) -> dict[tuple[str, str | None, str, str], list[dict[str, object]]]: - """Fetch every subject x range x bucket combination in one sweep. - - ``forced_raw`` shadows the schema probe so the historical raw-scan - statement runs against the same database; its output is the parity - ground truth every rollup-backed sweep is compared against byte for - byte. - """ - if forced_raw: - ledger.__dict__["_hashrate_rollup_schema_present"] = lambda: False - try: - results = {} - for subject_type, subject_id in HASHRATE_SERIES_SUBJECTS: - for range_id, bucket, lookback_seconds in HASHRATE_SERIES_RANGES: - results[(subject_type, subject_id, range_id, bucket)] = ( - ledger.dashboard_hashrate_series( - subject_type=subject_type, - subject_id=subject_id, - range_id=range_id, - bucket=bucket, - lookback_seconds=lookback_seconds, - range_anchor_epoch=None if range_id == "all" else anchor, - ) - ) - return results - finally: - ledger.__dict__.pop("_hashrate_rollup_schema_present", None) - - -def assert_hashrate_matrices_equal( - actual: dict[tuple[str, str | None, str, str], list[dict[str, object]]], - expected: dict[tuple[str, str | None, str, str], list[dict[str, object]]], - message: str, -) -> None: - assert_equal(sorted(actual), sorted(expected), f"{message} combination set") - for key in expected: - assert_equal(actual[key], expected[key], f"{message} {key}") - - -def hashrate_rollup_progress_row(ledger: ScopedPsqlLedger) -> object: - return ledger._run_json( - """ -SELECT json_build_object( - 'last_share_seq', ( - SELECT last_share_seq FROM qbit_hashrate_rollup_progress WHERE singleton - ) -); -""" - )["last_share_seq"] - - -def hashrate_rollup_snapshot(ledger: ScopedPsqlLedger) -> dict[str, object]: - return ledger._run_json( - """ -SELECT json_build_object( - 'pool', ( - SELECT COALESCE(json_agg(json_build_object( - 'grain', grain_seconds, - 'bucket', bucket_epoch, - 'count', accepted_share_count, - 'difficulty', accepted_share_difficulty::text - ) ORDER BY grain_seconds, bucket_epoch), '[]'::json) - FROM qbit_hashrate_rollup_pool - ), - 'miner', ( - SELECT COALESCE(json_agg(json_build_object( - 'grain', grain_seconds, - 'bucket', bucket_epoch, - 'miner', miner_id, - 'count', accepted_share_count, - 'difficulty', accepted_share_difficulty::text - ) ORDER BY grain_seconds, bucket_epoch, miner_id), '[]'::json) - FROM qbit_hashrate_rollup_miner - ) -); -""" - ) - - -def test_hashrate_rollup_series_parity() -> None: - """Rollup-served hashrate series must be byte-identical to the raw scan. - - Covers the acceptance bar for every serving state: before any rollup - pass (absent progress row), mid-backfill (watermark inside history, so - the live tail must cover the un-rolled remainder), fully caught up, - after the progress row is deleted with stale rollup rows left behind, - and with the rollup tables missing entirely (the pre-migration raw - statement). The range anchor is deliberately not bucket-aligned so the - raw scan emits a partial bucket straddling the range lower bound, which - the rollup path must reproduce from the ledger rather than from full - rollup buckets. A future-dated share (a coordinator clock running ahead - of the database clock) is folded into the rollups and must stay excluded - from every serving state, exactly as the raw scan's - accepted_at <= ended_at excludes it. - """ - schema = create_owned_schema("hashrate") - ledger = ScopedPsqlLedger( - test_schema=schema, - writer_id="a1-hashrate", - writer_epoch=1, - initialize_schema=True, - ) - try: - anchor = int(time.time()) - anchor -= anchor % 300 - anchor += 7 # unaligned modulo every grain, so boundary buckets exist - lookback_1w = 900 - range_start_1w = anchor - 7 * 86400 - lookback_1w - bucket_base = anchor - 7 - 86400 # aligned 5m bucket, one day back - seed_hashrate_shares( - ledger, - [ - # seq 1-6: one aligned 5m bucket split across two maintenance - # passes below, so the same rollup rows must accumulate. - {"miner_id": "m1", "accepted_epoch": bucket_base + 10, "share_difficulty": 100}, - {"miner_id": "m2", "accepted_epoch": bucket_base + 20, "share_difficulty": 200}, - {"miner_id": "m1", "accepted_epoch": bucket_base + 30, "share_difficulty": 300}, - {"miner_id": "m1", "accepted_epoch": bucket_base + 40, "share_difficulty": 400}, - {"miner_id": "m2", "accepted_epoch": bucket_base + 50, "share_difficulty": 500}, - {"miner_id": "m1", "accepted_epoch": bucket_base + 60, "share_difficulty": 600}, - # seq 7: rejected rows advance the watermark without counting. - {"miner_id": "m1", "accepted_epoch": bucket_base + 70, "share_difficulty": 700, "accepted": False}, - # seq 8: outside the 1w window, inside 1m/6m/all. - {"miner_id": "m2", "accepted_epoch": anchor - 9 * 86400, "share_difficulty": 800}, - # seq 9-12: around the 1w lower bound. seq 9 sits inside the - # straddling 5m bucket but before the bound, so it must stay - # excluded; 10-12 form the raw scan's partial leading bucket. - {"miner_id": "m1", "accepted_epoch": range_start_1w - 5, "share_difficulty": 900}, - {"miner_id": "m1", "accepted_epoch": range_start_1w, "share_difficulty": 1000}, - {"miner_id": "m2", "accepted_epoch": range_start_1w + 10, "share_difficulty": 1100}, - {"miner_id": "m1", "accepted_epoch": range_start_1w + 290, "share_difficulty": 1200}, - # seq 13-14: recent tail-of-history shares. - {"miner_id": "m2", "accepted_epoch": anchor - 50, "share_difficulty": 1300, "accepted": False}, - {"miner_id": "m1", "accepted_epoch": anchor - 30, "share_difficulty": 1400}, - # seq 15: future-dated relative to the database clock (a - # coordinator clock running ahead). It is folded into the - # rollups like any other share, and every serving state must - # exclude it exactly as the raw scan's - # accepted_at <= ended_at does. - {"miner_id": "m2", "accepted_epoch": anchor + 3600, "share_difficulty": 1500}, - ], - ) - raw_truth = hashrate_series_matrix(ledger, anchor=anchor, forced_raw=True) - straddle_epoch = range_start_1w - range_start_1w % 300 - straddle_stamp = datetime.fromtimestamp( - straddle_epoch, timezone.utc - ).strftime("%Y-%m-%dT%H:%M:%SZ") - partial_bucket = [ - point - for point in raw_truth[("pool", None, "1w", "5m")] - if point["timestamp"] == straddle_stamp - ] - assert_equal( - [ - (point["accepted_share_count"], point["accepted_share_difficulty"]) - for point in partial_bucket - ], - [(3, "3300")], - "raw ground truth contains the partial range-boundary bucket", - ) - - assert_hashrate_matrices_equal( - hashrate_series_matrix(ledger, anchor=anchor, forced_raw=False), - raw_truth, - "absent progress row serves the raw scan", - ) - - assert_equal( - ledger.advance_hashrate_rollups(batch_limit=5), - {"scanned": 5, "last_share_seq": 5, "caught_up": False}, - "first rollup pass bootstraps the watermark", - ) - assert_hashrate_matrices_equal( - hashrate_series_matrix(ledger, anchor=anchor, forced_raw=False), - raw_truth, - "mid-backfill rollup + tail merge", - ) - - assert_equal( - ledger.advance_hashrate_rollups(batch_limit=4), - {"scanned": 4, "last_share_seq": 9, "caught_up": False}, - "second rollup pass advances the watermark", - ) - # seq 1-5 landed in the first pass and seq 6 in the second, all in - # the same aligned 5m bucket: the upsert must accumulate, and the - # rejected seq 7 must advance the watermark without counting. - assert_equal( - ledger._run_json( - f""" -SELECT json_build_object( - 'count', accepted_share_count, - 'difficulty', accepted_share_difficulty::text -) -FROM qbit_hashrate_rollup_pool -WHERE grain_seconds = 300 AND bucket_epoch = {bucket_base}; -""" - ), - {"count": 6, "difficulty": "2100"}, - "adjacent passes accumulate into the same pool bucket", - ) - assert_equal( - ledger._run_json( - f""" -SELECT json_build_object( - 'count', accepted_share_count, - 'difficulty', accepted_share_difficulty::text -) -FROM qbit_hashrate_rollup_miner -WHERE grain_seconds = 300 AND bucket_epoch = {bucket_base} AND miner_id = 'm1'; -""" - ), - {"count": 4, "difficulty": "1400"}, - "adjacent passes accumulate into the same miner bucket", - ) - - assert_equal( - ledger.advance_hashrate_rollups(batch_limit=50000), - {"scanned": 6, "last_share_seq": 15, "caught_up": True}, - "final rollup pass reports caught up", - ) - assert_hashrate_matrices_equal( - hashrate_series_matrix(ledger, anchor=anchor, forced_raw=False), - raw_truth, - "caught-up rollup serving", - ) - assert_equal( - hashrate_rollup_progress_row(ledger), - 15, - "watermark covers rejected rows", - ) - assert_equal( - ledger._run_json( - """ -SELECT json_build_object( - 'count', ( - SELECT COALESCE(sum(accepted_share_count), 0) - FROM qbit_hashrate_rollup_pool - WHERE grain_seconds = 300 - ) -); -""" - )["count"], - 13, - "rejected shares never reach the rollup rows; the future-dated" - " share does", - ) - assert_equal( - ledger.advance_hashrate_rollups(batch_limit=50000), - {"scanned": 0, "last_share_seq": 15, "caught_up": True}, - "caught-up pass is an idempotent no-op", - ) - - # Deleting only the progress row leaves fully populated rollup - # tables behind; the gate on the watermark must exclude them and the - # widened tail alone must reproduce the raw scan -- if the gating - # were broken every bucket would double. - ledger._run_sql("DELETE FROM qbit_hashrate_rollup_progress;") - assert_hashrate_matrices_equal( - hashrate_series_matrix(ledger, anchor=anchor, forced_raw=False), - raw_truth, - "deleted progress row falls back to the raw scan", - ) - - # A database that predates the rollup DDL cannot even parse the - # rollup statement; the probe must route to the unchanged raw - # statement. - ledger._run_sql( - "DROP TABLE qbit_hashrate_rollup_miner;\n" - "DROP TABLE qbit_hashrate_rollup_pool;\n" - "DROP TABLE qbit_hashrate_rollup_progress;" - ) - ledger.__dict__.pop("_hashrate_rollup_schema_ready", None) - assert_hashrate_matrices_equal( - hashrate_series_matrix(ledger, anchor=anchor, forced_raw=False), - raw_truth, - "missing rollup tables run the raw statement", - ) - finally: - ledger.release_writer_lease() - ledger.close() - - -def test_hashrate_rollup_watermark_guard() -> None: - """A concurrent watermark advance must fail the pass without writes. - - The guard can only trip when the progress row changes between the - statement's snapshot and its guarded upsert, so the race is staged for - real: a second session updates the row inside an open transaction, the - ledger pass is started and provably blocks on that row lock, and only - then does the second session commit. The blocked pass must resume, - lose the guard, write no rollup rows at all, and raise. - """ - schema = create_owned_schema("hashrate_guard") - ledger = ScopedPsqlLedger( - test_schema=schema, - writer_id="a1-hashrate-guard", - writer_epoch=1, - initialize_schema=True, - ) - locker: subprocess.Popen[str] | None = None - try: - now_epoch = int(time.time()) - seed_hashrate_shares( - ledger, - [ - {"miner_id": "m1", "accepted_epoch": now_epoch - 500, "share_difficulty": 100}, - {"miner_id": "m2", "accepted_epoch": now_epoch - 400, "share_difficulty": 200}, - {"miner_id": "m1", "accepted_epoch": now_epoch - 300, "share_difficulty": 300}, - {"miner_id": "m2", "accepted_epoch": now_epoch - 200, "share_difficulty": 400}, - ], - ) - assert_equal( - ledger.advance_hashrate_rollups(batch_limit=100), - {"scanned": 4, "last_share_seq": 4, "caught_up": True}, - "guard scenario bootstrap pass", - ) - before = hashrate_rollup_snapshot(ledger) - # Give the raced pass real work so a broken guard would visibly - # double-count rather than trivially writing nothing. - seed_hashrate_shares_offset = [ - {"miner_id": "m1", "accepted_epoch": now_epoch - 100, "share_difficulty": 500}, - {"miner_id": "m2", "accepted_epoch": now_epoch - 90, "share_difficulty": 600}, - ] - ledger._run_sql( - "INSERT INTO qbit_share_ledger (\n" - " share_id, miner_id, payout_order_key, p2mr_program,\n" - " share_difficulty, network_difficulty, template_height, job_id,\n" - " job_issued_at, ntime, accepted_at, accepted, reject_reason,\n" - " writer_id, writer_epoch\n" - ") VALUES\n" - + ",\n".join( - "(" - f"'hashrate-guard-share-{index}', " - f"'{share['miner_id']}', " - f"'9{index:03d}', " - f"decode('9{index:01d}' || repeat('00', 31), 'hex'), " - f"{share['share_difficulty']}, " - "1000, 10, " - f"'hashrate-guard-job-{index}', " - f"to_timestamp({int(share['accepted_epoch']) - 1}), " # type: ignore[call-overload] - f"{1_700_009_000 + index}, " - f"to_timestamp({int(share['accepted_epoch'])}), " # type: ignore[call-overload] - "true, NULL, 'a1-hashrate-guard', 1)" - for index, share in enumerate(seed_hashrate_shares_offset, start=1) - ) - + ";" - ) - - locker = subprocess.Popen( - [ - *BASE_PSQL_ARGV, - "--no-psqlrc", - "--set", - "ON_ERROR_STOP=1", - "--quiet", - ], - stdin=subprocess.PIPE, - stdout=subprocess.DEVNULL, - stderr=subprocess.DEVNULL, - text=True, - start_new_session=True, - ) - with ACTIVE_CHILDREN_LOCK: - ACTIVE_CHILDREN.add(locker) - assert locker.stdin is not None - locker.stdin.write( - f'SET search_path TO "{schema}", pg_catalog;\n' - "BEGIN;\n" - "UPDATE qbit_hashrate_rollup_progress\n" - "SET last_share_seq = last_share_seq + 100,\n" - " updated_at = clock_timestamp()\n" - "WHERE singleton;\n" - ) - locker.stdin.flush() - deadline = time.monotonic() + 20.0 - while True: - holding = run_json( - """ -SELECT json_build_object('holding', ( - SELECT count(*) - FROM pg_stat_activity - WHERE state = 'idle in transaction' - AND query LIKE '%last_share_seq = last_share_seq + 100%' -)); -""" - )["holding"] - if int(holding) > 0: - break - if time.monotonic() > deadline: - raise GateFailure("watermark lock holder never reached its open transaction") - time.sleep(0.1) - - outcome: dict[str, object] = {} - - def run_raced_advance() -> None: - try: - outcome["result"] = ledger.advance_hashrate_rollups(batch_limit=100) - except BaseException as error: # noqa: BLE001 - reported below - outcome["error"] = error - - raced = threading.Thread(target=run_raced_advance, daemon=True) - raced.start() - deadline = time.monotonic() + 20.0 - while True: - waiting = run_json( - """ -SELECT json_build_object('waiting', ( - SELECT count(*) - FROM pg_stat_activity - WHERE state = 'active' - AND wait_event_type = 'Lock' - AND pid <> pg_backend_pid() - AND query LIKE '%qbit_hashrate_rollup_progress%' -)); -""" - )["waiting"] - if int(waiting) > 0: - break - if time.monotonic() > deadline: - raise GateFailure("raced rollup advance never blocked on the progress row") - time.sleep(0.1) - locker.stdin.write("COMMIT;\n") - locker.stdin.flush() - locker.stdin.close() - raced.join(timeout=25.0) - if raced.is_alive(): - raise GateFailure("raced rollup advance did not finish after the commit") - error = outcome.get("error") - if not isinstance(error, RuntimeError) or "advanced concurrently" not in str(error): - raise GateFailure( - f"raced rollup advance outcome was {outcome!r}; expected the loud watermark failure" - ) - assert_equal( - hashrate_rollup_snapshot(ledger), - before, - "lost watermark race writes no rollup rows", - ) - assert_equal( - hashrate_rollup_progress_row(ledger), - 104, - "only the concurrent watermark advance landed", - ) - finally: - if locker is not None: - _terminate_and_reap(locker) - if locker.stdin is not None and not locker.stdin.closed: - locker.stdin.close() - with ACTIVE_CHILDREN_LOCK: - ACTIVE_CHILDREN.discard(locker) - ledger.release_writer_lease() - ledger.close() - - -def test_hashrate_rollup_difficulty_overflow_headroom() -> None: - """Bucket totals may exceed one share's numeric(78, 0) precision. - - Individually valid 78-digit share difficulties land in one bucket, so - the total needs 79 digits. The aggregate columns are unconstrained - numeric precisely for this: a constrained column would make the - maintenance upsert overflow on every retry and wedge the watermark. - Both the insert fold and the accumulate (ON CONFLICT) fold must carry - the oversized total exactly, matching the raw scan's unconstrained - sum(numeric). The dashboard's TH/s shaping is not exercised here: its - Decimal rendering cannot represent such magnitudes on the raw path - either, and this scenario pins the storage layer, not that shared - shaping. - """ - schema = create_owned_schema("hashrate_overflow") - ledger = ScopedPsqlLedger( - test_schema=schema, - writer_id="a1-hashrate-overflow", - writer_epoch=1, - initialize_schema=True, - ) - try: - anchor = int(time.time()) - # All three shares inside one 5m bucket, so every grain folds them - # into a single row whose total needs 79 digits. - bucket_epoch = anchor // 300 * 300 - 600 - huge = 6 * 10**77 - seed_hashrate_shares( - ledger, - [ - {"miner_id": "m1", "accepted_epoch": bucket_epoch + 10, "share_difficulty": huge}, - {"miner_id": "m1", "accepted_epoch": bucket_epoch + 20, "share_difficulty": huge}, - {"miner_id": "m1", "accepted_epoch": bucket_epoch + 30, "share_difficulty": huge}, - ], - ) - # First pass exercises the oversized insert fold... - assert_equal( - ledger.advance_hashrate_rollups(batch_limit=2), - {"scanned": 2, "last_share_seq": 2, "caught_up": False}, - "overflow insert fold advances", - ) - total = str(2 * huge) - snapshot = hashrate_rollup_snapshot(ledger) - assert_equal( - [(row["grain"], row["count"], row["difficulty"]) for row in snapshot["pool"]], - [(300, 2, total), (3600, 2, total), (86400, 2, total)], - "pool rollup carries the 79-digit total", - ) - assert_equal( - [(row["grain"], row["miner"], row["difficulty"]) for row in snapshot["miner"]], - [(300, "m1", total), (3600, "m1", total), (86400, "m1", total)], - "miner rollup carries the 79-digit total", - ) - # ...and the second pass exercises the oversized accumulate fold, - # the exact statement a constrained column would fail on every retry. - assert_equal( - ledger.advance_hashrate_rollups(batch_limit=100), - {"scanned": 1, "last_share_seq": 3, "caught_up": True}, - "overflow accumulate fold advances", - ) - accumulated = str(3 * huge) - snapshot = hashrate_rollup_snapshot(ledger) - assert_equal( - [(row["grain"], row["count"], row["difficulty"]) for row in snapshot["pool"]], - [(300, 3, accumulated), (3600, 3, accumulated), (86400, 3, accumulated)], - "pool rollup accumulates past one share's precision", - ) - assert_equal( - [(row["grain"], row["miner"], row["difficulty"]) for row in snapshot["miner"]], - [(300, "m1", accumulated), (3600, "m1", accumulated), (86400, "m1", accumulated)], - "miner rollup accumulates past one share's precision", - ) - finally: - ledger.release_writer_lease() - ledger.close() - - -def test_hashrate_rollup_lease_fence() -> None: - """A fenced-out coordinator's rollup advance writes nothing and raises. - - The maintenance statement renews the exact writer lease tuple and gates - the watermark advance on that renewal matching, so a coordinator whose - lease another session took over must fail loudly without touching the - watermark or either rollup table -- its process-local writer lock alone - does not authorize mutating ledger-derived state. - """ - schema = create_owned_schema("hashrate_lease") - ledger = ScopedPsqlLedger( - test_schema=schema, - writer_id="a1-hashrate-lease", - writer_epoch=1, - initialize_schema=True, - ) - try: - now_epoch = int(time.time()) - seed_hashrate_shares( - ledger, - [ - {"miner_id": "m1", "accepted_epoch": now_epoch - 500, "share_difficulty": 100}, - {"miner_id": "m2", "accepted_epoch": now_epoch - 400, "share_difficulty": 200}, - {"miner_id": "m1", "accepted_epoch": now_epoch - 300, "share_difficulty": 300}, - ], - ) - # Bootstrap with one share left behind the watermark so a broken - # fence would visibly fold it rather than trivially writing nothing. - assert_equal( - ledger.advance_hashrate_rollups(batch_limit=2), - {"scanned": 2, "last_share_seq": 2, "caught_up": False}, - "lease scenario bootstrap pass", - ) - before_rows = hashrate_rollup_snapshot(ledger) - original_lease = ledger._run_json( - """ -SELECT json_build_object( - 'writer_id', writer_id, - 'writer_epoch', writer_epoch, - 'writer_session_token', writer_session_token -) -FROM qbit_ledger_writer_lease -WHERE singleton; -""" - ) - ledger._run_sql( - "UPDATE qbit_ledger_writer_lease\n" - "SET writer_id = 'a1-hashrate-usurper',\n" - " writer_epoch = 2,\n" - " writer_session_token = 'a1-hashrate-usurper-session',\n" - " lease_expires_at = clock_timestamp() + interval '60 seconds',\n" - " updated_at = clock_timestamp()\n" - "WHERE singleton;" - ) - try: - ledger.advance_hashrate_rollups(batch_limit=100) - except RuntimeError as error: - if "writer lease is not active" not in str(error): - raise GateFailure( - f"fenced-out advance raised {error!r}; expected the lease refusal" - ) from error - else: - raise GateFailure("fenced-out advance did not raise") - assert_equal( - hashrate_rollup_progress_row(ledger), - 2, - "fenced-out advance leaves the watermark alone", - ) - assert_equal( - hashrate_rollup_snapshot(ledger), - before_rows, - "fenced-out advance writes no rollup rows", - ) - # Hand the lease back so release and cleanup run as the owner. - ledger._run_sql( - "UPDATE qbit_ledger_writer_lease\n" - f"SET writer_id = '{original_lease['writer_id']}',\n" - f" writer_epoch = {int(original_lease['writer_epoch'])},\n" - f" writer_session_token = '{original_lease['writer_session_token']}',\n" - " lease_expires_at = clock_timestamp() + interval '60 seconds',\n" - " updated_at = clock_timestamp()\n" - "WHERE singleton;" - ) - assert_equal( - ledger.advance_hashrate_rollups(batch_limit=100), - {"scanned": 1, "last_share_seq": 3, "caught_up": True}, - "restored lease folds the held-back share", - ) - finally: - ledger.release_writer_lease() - ledger.close() - - -def _marker_iso(epoch: int) -> str: - return datetime.fromtimestamp(epoch, timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ") - - -def test_dashboard_block_markers_read_model() -> None: - """The block-marker SQL over real qbit_pool_blocks rows. - - Seeds found blocks across two 1h buckets plus one block just below the 1w - range's lower bound, one reversed block, and one future-dated block, then - asserts the read model's counts, per-bucket top-3 ordering (found_at - DESC, height DESC), range filtering against the caller-provided anchor, - the unbounded ``all`` range, and the upper cutoff at the statement clock - that keeps a future-dated block off the chart -- the same bound the - hashrate series applies. - """ - schema = create_owned_schema("block_markers") - ledger = ScopedPsqlLedger( - test_schema=schema, - writer_id="a1-block-markers", - writer_epoch=1, - initialize_schema=True, - ) - anchor_epoch = 1_750_000_000 - bucket_a = public_api.hashrate_series_min_epoch(anchor_epoch, 7 * 86400, 3600) - bucket_b = anchor_epoch // 3600 * 3600 - 7200 - hash_in_a = "71" * 32 - hash_below_range = "72" * 32 - hashes_in_b = {201: "73" * 32, 202: "74" * 32, 203: "75" * 32, 204: "76" * 32} - hash_reversed = "77" * 32 - hash_future = "78" * 32 - found_in_a = bucket_a + 10 - found_in_b = {201: bucket_b + 100, 202: bucket_b + 200, 203: bucket_b + 300, 204: bucket_b + 300} - try: - rows = [ - (hash_in_a, 100, found_in_a, "confirmed"), - # One second below the range lower bound: dropped from 1w with the - # whole straddling bucket, present in the unbounded all range. - (hash_below_range, 99, bucket_a - 1, "confirmed"), - (hashes_in_b[201], 201, found_in_b[201], "confirmed"), - (hashes_in_b[202], 202, found_in_b[202], "confirmed"), - # Two blocks tied on found_at so the height tiebreak is observable. - (hashes_in_b[203], 203, found_in_b[203], "confirmed"), - (hashes_in_b[204], 204, found_in_b[204], "confirmed"), - # Future-dated relative to the database clock (a clock jump or a - # bad import): capped out by found_at <= clock_timestamp() so it - # can never render beyond the chart's current endpoint. - (hash_future, 300, int(time.time()) + 3600, "confirmed"), - ] - values = ",\n ".join( - f"('{block_hash}', {height}, '{'10' * 32}', '{'20' * 32}', " - f"'{'30' * 32}', to_timestamp({epoch}), '{chain_state}', 'immature', NULL)" - for block_hash, height, epoch, chain_state in rows - ) - ledger._run_sql( - f""" -INSERT INTO qbit_pool_blocks ( - block_hash, block_height, parent_hash, coinbase_txid, - payout_manifest_sha256, found_at, chain_state, maturity_state, - disconnected_at -) VALUES - {values}, - ('{hash_reversed}', 205, '{'10' * 32}', '{'20' * 32}', '{'30' * 32}', - to_timestamp({bucket_b + 400}), 'reversed', 'reversed', - to_timestamp({bucket_b + 500})); -""" - ) - - week = ledger.dashboard_block_markers( - range_id="1w", - bucket="1h", - range_anchor_epoch=anchor_epoch, - ) - assert_equal( - week, - { - "total_blocks": 5, - "points": [ - { - "timestamp": _marker_iso(bucket_a), - "block_count": 1, - "blocks": [ - { - "height": 100, - "hash": hash_in_a, - "found_at": _marker_iso(found_in_a), - } - ], - }, - { - "timestamp": _marker_iso(bucket_b), - "block_count": 4, - "blocks": [ - { - "height": 204, - "hash": hashes_in_b[204], - "found_at": _marker_iso(found_in_b[204]), - }, - { - "height": 203, - "hash": hashes_in_b[203], - "found_at": _marker_iso(found_in_b[203]), - }, - { - "height": 202, - "hash": hashes_in_b[202], - "found_at": _marker_iso(found_in_b[202]), - }, - ], - }, - ], - }, - "1w block markers", - ) - - everything = ledger.dashboard_block_markers( - range_id="all", - bucket="1d", - range_anchor_epoch=None, - ) - assert_equal(everything["total_blocks"], 6, "all-range total excludes reversed only") - assert_equal( - sum(int(point["block_count"]) for point in everything["points"]), - 6, - "all-range bucket counts sum to the total", - ) - listed_hashes = { - block["hash"] - for point in everything["points"] - for block in point["blocks"] - } - if hash_below_range not in listed_hashes: - raise GateFailure("all range must include the below-1w-range block") - if hash_reversed in listed_hashes: - raise GateFailure("reversed block leaked into the marker series") - if hash_future in listed_hashes: - raise GateFailure("future-dated block leaked into the marker series") - epochs = [ - int( - datetime.strptime(str(point["timestamp"]), "%Y-%m-%dT%H:%M:%SZ") - .replace(tzinfo=timezone.utc) - .timestamp() - ) - for point in everything["points"] - ] - assert_equal(epochs, sorted(epochs), "all-range points ascend") - for epoch in epochs: - assert_equal(epoch % 86400, 0, "all-range 1d bucket alignment") - finally: - ledger.release_writer_lease() - ledger.close() - - -def main() -> None: - public_before = public_sentinel() - failure: BaseException | None = None - try: - test_exact_transition_parity() - assert_empty_serializer_case(legacy=False) - assert_empty_serializer_case(legacy=True) - assert_transition_binding_case(temporary=False) - assert_transition_binding_case(temporary=True) - test_durable_floor_ignores_allocator_gaps() - test_hashrate_rollup_series_parity() - test_hashrate_rollup_watermark_guard() - test_hashrate_rollup_difficulty_overflow_headroom() - test_hashrate_rollup_lease_fence() - test_dashboard_block_markers_read_model() - except BaseException as error: - failure = error - try: - cleanup_active_children() - cleanup_owned_schemas() - assert_equal(marker_schema_count(), 0, "owned schema marker cleanup") - assert_equal(public_sentinel(), public_before, "public sentinel preservation") - except BaseException as cleanup_error: - if failure is None: - raise - raise GateFailure( - f"scenario failed with {failure!r}; cleanup also failed with " - f"{cleanup_error!r}" - ) from cleanup_error - else: - atexit.unregister(cleanup_active_children) - atexit.unregister(cleanup_owned_schemas) - if failure is not None: - raise failure - print( - "prism postgres A1 gate PASS " - "exact-transition-parity empty-fresh empty-legacy " - "ordinary-decoy-binding temporary-decoy-binding durable-floor-gaps " - "hashrate-rollup-parity hashrate-rollup-guard " - "hashrate-rollup-overflow hashrate-rollup-lease-fence " - "block-markers" - ) - - -if __name__ == "__main__": - main() diff --git a/tests/prism_postgres_a1_migration_gate.py b/tests/prism_postgres_a1_migration_gate.py deleted file mode 100644 index 949e75f8b..000000000 --- a/tests/prism_postgres_a1_migration_gate.py +++ /dev/null @@ -1,1782 +0,0 @@ -"""Non-discovered PostgreSQL ordinal migration integration gate.""" - -from __future__ import annotations - -import hashlib -import subprocess -import tempfile -import time -from pathlib import Path -from typing import Any - -from tests import prism_postgres_a1_gate as support -from tests.prism_postgres_a1_gate import ( - ACTIVE_CHILDREN, - ACTIVE_CHILDREN_LOCK, - BASE_PSQL_ARGV, - GateFailure, - LEGACY_POOL_BLOCKS_SQL, - PSQL_OUTPUT_LIMIT_BYTES, - PSQL_TIMEOUT_SECONDS, - RUN_TOKEN, - SCHEMA_PATTERN, - ScopedPsqlLedger, - _schema_registered, - _terminate_and_reap, - assert_equal, - create_owned_schema, - run_json, - run_psql, -) - -MIGRATION_HASH_A = "81" * 32 -MIGRATION_HASH_B = "82" * 32 -MIGRATION_HASH_C = "83" * 32 -MIGRATION_HASH_OTHER = "84" * 32 -BIGINT_MAX = 9_223_372_036_854_775_807 - - -def add_ordinal_column(schema: str) -> None: - run_psql( - "ALTER TABLE qbit_pool_blocks " - "ADD COLUMN audit_publication_sequence bigint;", - schema=schema, - ) - - -def create_ordinal_sequence( - schema: str, - *, - last_value: int = 1, - is_called: bool = False, -) -> None: - run_psql( - f""" -CREATE SEQUENCE qbit_audit_publication_sequence_seq AS bigint; -SELECT setval( - 'qbit_audit_publication_sequence_seq'::regclass, - {last_value}, - {'true' if is_called else 'false'} -); -""", - schema=schema, - ) - - -def seed_migration_rows( - schema: str, - *, - ordinals: tuple[int | None, int | None, int | None] | None, - include_other_states: bool = True, -) -> None: - ordinal_column = ( - ", audit_publication_sequence" if ordinals is not None else "" - ) - ordinal_values = ( - ["NULL" if value is None else str(value) for value in ordinals] - if ordinals is not None - else [None, None, None] - ) - - def row( - block_hash: str, - found_at: str, - ordinal: str | None, - state: str, - maturity: str = "immature", - ) -> str: - ordinal_sql = f", {ordinal}" if ordinal is not None else "" - disconnected = ( - ", '2020-01-02T00:00:00Z'" if maturity == "reversed" else ", NULL" - ) - return ( - f"('{block_hash}', 10, '{'10' * 32}', '{'20' * 32}', " - f"'{'30' * 32}', '{found_at}', '{state}', '{maturity}', " - f"NULL{disconnected}{ordinal_sql})" - ) - - rows = [ - # Insertion order intentionally differs from deterministic migration - # order. A/B share a timestamp and sort by block hash. - row( - MIGRATION_HASH_C, - "2020-01-01T00:00:02Z", - ordinal_values[2], - "confirmed", - ), - row( - MIGRATION_HASH_B, - "2020-01-01T00:00:01Z", - ordinal_values[1], - "confirmed", - ), - row( - MIGRATION_HASH_A, - "2020-01-01T00:00:01Z", - ordinal_values[0], - "confirmed", - ), - ] - if include_other_states: - rows.extend( - [ - row( - MIGRATION_HASH_OTHER, - "2020-01-01T00:00:03Z", - "NULL" if ordinals is not None else None, - "prepared", - ), - row( - "85" * 32, - "2020-01-01T00:00:04Z", - "NULL" if ordinals is not None else None, - "inactive", - ), - row( - "86" * 32, - "2020-01-01T00:00:05Z", - "NULL" if ordinals is not None else None, - "rejected", - "reversed", - ), - row( - "87" * 32, - "2020-01-01T00:00:06Z", - "NULL" if ordinals is not None else None, - "reversed", - "reversed", - ), - ] - ) - run_psql( - f""" -INSERT INTO qbit_pool_blocks ( - block_hash, - block_height, - parent_hash, - coinbase_txid, - payout_manifest_sha256, - found_at, - chain_state, - maturity_state, - matured_at, - disconnected_at - {ordinal_column} -) VALUES -{', '.join(rows)}; -""", - schema=schema, - ) - - -def migration_snapshot(schema: str) -> dict[str, object]: - if SCHEMA_PATTERN.fullmatch(schema) is None: - raise GateFailure(f"invalid migration snapshot schema: {schema!r}") - snapshot = run_json( - f""" -SELECT json_build_object( - 'rows', ( - SELECT COALESCE( - json_agg(to_jsonb(block) ORDER BY block_hash), - '[]'::json - ) - FROM "{schema}".qbit_pool_blocks block - ), - 'column', ( - SELECT json_build_object( - 'attnum', attribute.attnum, - 'type_oid', attribute.atttypid::pg_catalog.int8, - 'nullable', NOT attribute.attnotnull, - 'has_default', attribute.atthasdef, - 'identity', attribute.attidentity, - 'generated', attribute.attgenerated, - 'collation', attribute.attcollation::pg_catalog.int8 - ) - FROM pg_catalog.pg_attribute attribute - JOIN pg_catalog.pg_class relation - ON relation.oid = attribute.attrelid - JOIN pg_catalog.pg_namespace namespace - ON namespace.oid = relation.relnamespace - WHERE namespace.nspname = '{schema}' - AND relation.relname = 'qbit_pool_blocks' - AND attribute.attname = 'audit_publication_sequence' - AND attribute.attnum > 0 - AND NOT attribute.attisdropped - ), - 'sequence_catalog', ( - SELECT json_build_object( - 'oid', relation.oid, - 'kind', relation.relkind, - 'persistence', relation.relpersistence, - 'owner', relation.relowner, - 'type_oid', sequence.seqtypid::pg_catalog.int8, - 'start', sequence.seqstart, - 'increment', sequence.seqincrement, - 'max', sequence.seqmax, - 'min', sequence.seqmin, - 'cache', sequence.seqcache, - 'cycle', sequence.seqcycle, - 'owned_dependencies', ( - SELECT count(*) - FROM pg_catalog.pg_depend dependency - WHERE dependency.classid = 'pg_catalog.pg_class'::regclass - AND dependency.objid = relation.oid - AND dependency.refclassid = 'pg_catalog.pg_class'::regclass - AND dependency.refobjsubid > 0 - AND dependency.deptype IN ('a', 'i') - ), - 'same_owner_as_table', relation.relowner = ( - SELECT pool_blocks.relowner - FROM pg_catalog.pg_class pool_blocks - JOIN pg_catalog.pg_namespace pool_namespace - ON pool_namespace.oid = pool_blocks.relnamespace - WHERE pool_namespace.nspname = '{schema}' - AND pool_blocks.relname = 'qbit_pool_blocks' - ) - ) - FROM pg_catalog.pg_class relation - JOIN pg_catalog.pg_namespace namespace - ON namespace.oid = relation.relnamespace - LEFT JOIN pg_catalog.pg_sequence sequence - ON sequence.seqrelid = relation.oid - WHERE namespace.nspname = '{schema}' - AND relation.relname = 'qbit_audit_publication_sequence_seq' - ), - 'indexes', ( - SELECT COALESCE(json_agg(index_row ORDER BY index_row.name), '[]'::json) - FROM ( - SELECT - index_relation.relname AS name, - index_relation.oid, - pg_catalog.pg_relation_filenode(index_relation.oid) AS filenode, - index_relation.relkind AS kind, - index_relation.relpersistence AS persistence, - index_relation.relowner AS owner, - pg_catalog.pg_get_indexdef(index_relation.oid) AS definition, - index_definition.indisunique AS unique, - index_definition.indisvalid AS valid, - index_definition.indisready AS ready, - index_definition.indislive AS live, - index_definition.indimmediate AS immediate, - index_definition.indisprimary AS primary, - index_definition.indisexclusion AS exclusion, - index_definition.indisclustered AS clustered, - index_definition.indisreplident AS replica_identity, - index_definition.indnullsnotdistinct AS nulls_not_distinct, - index_definition.indnkeyatts AS key_count, - index_definition.indnatts AS attribute_count, - index_definition.indkey::text AS keys, - index_definition.indclass::text AS classes, - index_definition.indcollation::text AS collations, - index_definition.indoption::text AS options, - index_definition.indexprs::text AS expressions, - index_definition.indpred::text AS predicate - FROM pg_catalog.pg_index index_definition - JOIN pg_catalog.pg_class index_relation - ON index_relation.oid = index_definition.indexrelid - JOIN pg_catalog.pg_class table_relation - ON table_relation.oid = index_definition.indrelid - JOIN pg_catalog.pg_namespace namespace - ON namespace.oid = table_relation.relnamespace - WHERE namespace.nspname = '{schema}' - AND table_relation.relname = 'qbit_pool_blocks' - AND EXISTS ( - SELECT 1 - FROM unnest(index_definition.indkey) AS key(attnum) - JOIN pg_catalog.pg_attribute attribute - ON attribute.attrelid = table_relation.oid - AND attribute.attnum = key.attnum - WHERE attribute.attname = 'audit_publication_sequence' - ) - ) index_row - ), - 'named_index_relation', ( - SELECT json_build_object( - 'oid', relation.oid, - 'kind', relation.relkind, - 'persistence', relation.relpersistence, - 'owner', relation.relowner, - 'filenode', pg_catalog.pg_relation_filenode(relation.oid) - ) - FROM pg_catalog.pg_class relation - JOIN pg_catalog.pg_namespace namespace - ON namespace.oid = relation.relnamespace - WHERE namespace.nspname = '{schema}' - AND relation.relname = - 'qbit_pool_blocks_audit_publication_sequence_idx' - ), - 'constraints', ( - SELECT COALESCE( - json_agg(constraint_row ORDER BY constraint_row.name), - '[]'::json - ) - FROM ( - SELECT - constraint_definition.conname AS name, - constraint_definition.oid, - constraint_definition.contype AS type, - constraint_definition.conbin::text AS expression_tree, - pg_catalog.pg_get_constraintdef( - constraint_definition.oid, - true - ) AS definition, - constraint_definition.convalidated AS validated, - constraint_definition.conkey::text AS keys, - constraint_definition.conislocal AS local, - constraint_definition.coninhcount AS inherited_count, - constraint_definition.connoinherit AS no_inherit, - constraint_definition.condeferrable AS deferrable, - constraint_definition.condeferred AS deferred - FROM pg_catalog.pg_constraint constraint_definition - JOIN pg_catalog.pg_class table_relation - ON table_relation.oid = constraint_definition.conrelid - JOIN pg_catalog.pg_namespace namespace - ON namespace.oid = table_relation.relnamespace - WHERE namespace.nspname = '{schema}' - AND table_relation.relname = 'qbit_pool_blocks' - AND ( - constraint_definition.conname = - 'qbit_pool_blocks_audit_publication_sequence_check' - OR pg_catalog.pg_get_constraintdef( - constraint_definition.oid, - true - ) LIKE '%audit_publication_sequence%' - ) - ) constraint_row - ) -); -""" - ) - sequence_catalog = snapshot["sequence_catalog"] - if sequence_catalog is None or sequence_catalog["kind"] != "S": - snapshot["sequence_state"] = None - else: - snapshot["sequence_state"] = run_json( - f""" -SELECT json_build_object('last_value', last_value, 'is_called', is_called) -FROM "{schema}".qbit_audit_publication_sequence_seq; -""" - ) - return snapshot - - -def assert_advisory_lock_available(message: str) -> None: - result = run_json( - """ -WITH acquired AS ( - SELECT pg_catalog.pg_try_advisory_lock( - pg_catalog.hashtext('qbit_audit_publication_sequence_migration') - ) AS ok -) -SELECT json_build_object( - 'acquired', (SELECT ok FROM acquired), - 'released', CASE - WHEN (SELECT ok FROM acquired) THEN pg_catalog.pg_advisory_unlock( - pg_catalog.hashtext('qbit_audit_publication_sequence_migration') - ) - ELSE false - END -); -""" - ) - assert_equal( - result, - {"acquired": True, "released": True}, - message, - ) - - -def initialize_migration_schema( - schema: str, - *, - writer_id: str, -) -> ScopedPsqlLedger: - return ScopedPsqlLedger( - test_schema=schema, - writer_id=writer_id, - writer_epoch=1, - initialize_schema=True, - ) - - -def assert_valid_migration_case( - *, - schema: str, - label: str, - expected_ordinals: dict[str, int | None], - expected_next_ordinal: int | None, - expected_sequence_state: dict[str, object] | None = None, -) -> dict[str, object]: - if not _schema_registered(schema): - raise GateFailure(f"{label} requires an explicit registered schema") - ledger = initialize_migration_schema(schema, writer_id=f"a1-{label}-first") - try: - first = migration_snapshot(schema) - rows = { - str(row["block_hash"]): row.get("audit_publication_sequence") - for row in first["rows"] # type: ignore[union-attr] - } - for block_hash, expected in expected_ordinals.items(): - assert_equal(rows[block_hash], expected, f"{label} ordinal {block_hash}") - if expected_sequence_state is not None: - assert_equal( - first["sequence_state"], - expected_sequence_state, - f"{label} exact allocator state", - ) - assert_equal(len(first["indexes"]), 1, f"{label} canonical index count") - assert_equal(len(first["constraints"]), 1, f"{label} canonical constraint count") - assert_equal( - first["column"], - { - "attnum": first["column"]["attnum"], # type: ignore[index] - "type_oid": 20, - "nullable": True, - "has_default": False, - "identity": "", - "generated": "", - "collation": 0, - }, - f"{label} exact ordinal column catalog", - ) - sequence_catalog = first["sequence_catalog"] - assert sequence_catalog is not None - assert_equal( - { - key: sequence_catalog[key] # type: ignore[index] - for key in ( - "kind", - "persistence", - "type_oid", - "start", - "increment", - "max", - "min", - "cache", - "cycle", - "owned_dependencies", - "same_owner_as_table", - ) - }, - { - "kind": "S", - "persistence": "p", - "type_oid": 20, - "start": 1, - "increment": 1, - "max": BIGINT_MAX, - "min": 1, - "cache": 1, - "cycle": False, - "owned_dependencies": 0, - "same_owner_as_table": True, - }, - f"{label} exact ordinal sequence catalog", - ) - index = first["indexes"][0] # type: ignore[index] - assert_equal( - { - key: index[key] - for key in ( - "name", - "kind", - "persistence", - "unique", - "valid", - "ready", - "live", - "immediate", - "primary", - "exclusion", - "clustered", - "replica_identity", - "nulls_not_distinct", - "key_count", - "attribute_count", - "collations", - "options", - "expressions", - "predicate", - ) - }, - { - "name": "qbit_pool_blocks_audit_publication_sequence_idx", - "kind": "i", - "persistence": "p", - "unique": True, - "valid": True, - "ready": True, - "live": True, - "immediate": True, - "primary": False, - "exclusion": False, - "clustered": False, - "replica_identity": False, - "nulls_not_distinct": False, - "key_count": 1, - "attribute_count": 1, - "collations": "0", - "options": "0", - "expressions": None, - "predicate": None, - }, - f"{label} exact ordinal index catalog", - ) - assert_equal( - index["definition"], - "CREATE UNIQUE INDEX " - "qbit_pool_blocks_audit_publication_sequence_idx ON " - f"{schema}.qbit_pool_blocks USING btree " - "(audit_publication_sequence)", - f"{label} exact ordinal index definition", - ) - constraint = first["constraints"][0] # type: ignore[index] - assert_equal( - { - key: constraint[key] - for key in ( - "name", - "type", - "definition", - "validated", - "local", - "inherited_count", - "no_inherit", - "deferrable", - "deferred", - ) - }, - { - "name": "qbit_pool_blocks_audit_publication_sequence_check", - "type": "c", - "definition": "CHECK ((audit_publication_sequence IS NULL OR " - "audit_publication_sequence > 0) AND (chain_state <> " - "'confirmed'::text OR audit_publication_sequence IS NOT NULL))", - "validated": True, - "local": True, - "inherited_count": 0, - "no_inherit": False, - "deferrable": False, - "deferred": False, - }, - f"{label} exact ordinal constraint catalog", - ) - finally: - ledger.release_writer_lease() - ledger.close() - for rerun_round in (1, 2): - rerun = initialize_migration_schema( - schema, - writer_id=f"a1-{label}-rerun-{rerun_round}", - ) - try: - assert_equal( - migration_snapshot(schema), - first, - f"{label} idempotent catalog rerun {rerun_round}", - ) - finally: - rerun.release_writer_lease() - rerun.close() - legacy_inactive_hash = "85" * 32 - legacy_inactive_sequence = expected_ordinals.get(legacy_inactive_hash) - if legacy_inactive_sequence is not None: - reactivator = ScopedPsqlLedger( - test_schema=schema, - writer_id=f"a1-{label}-legacy-reactivation", - writer_epoch=1, - ) - try: - reactivated = reactivator.reactivate_pool_block( - block_hash=legacy_inactive_hash, - active_tip_height=10, - ) - assert_equal( - reactivated["audit_publication_sequence"], - legacy_inactive_sequence, - f"{label} legacy inactive reactivation preserves backfilled ordinal", - ) - finally: - reactivator.release_writer_lease() - reactivator.close() - if expected_next_ordinal is not None: - final = ScopedPsqlLedger( - test_schema=schema, - writer_id=f"a1-{label}-next", - writer_epoch=1, - ) - try: - block_hash = hashlib.sha256(f"{label}-next".encode()).hexdigest() - final._run_sql( - f""" -INSERT INTO qbit_pool_blocks ( - block_hash, block_height, parent_hash, coinbase_txid, - payout_manifest_sha256, chain_state, maturity_state -) VALUES ( - '{block_hash}', 99, '{'10' * 32}', '{'20' * 32}', - '{'30' * 32}', 'prepared', 'immature' -); -""" - ) - confirmation = final.confirm_accepted_block( - block_hash=block_hash, - active_tip_height=99, - ) - assert_equal( - confirmation["audit_publication_sequence"], - expected_next_ordinal, - f"{label} next production ordinal", - ) - finally: - final.release_writer_lease() - final.close() - assert_advisory_lock_available(f"{label} advisory lock released") - return first - - -def insert_migration_row( - schema: str, - *, - block_hash: str, - ordinal: int | None, - found_at: str, - chain_state: str = "confirmed", -) -> None: - ordinal_sql = "NULL" if ordinal is None else str(ordinal) - run_psql( - f""" -INSERT INTO qbit_pool_blocks ( - block_hash, - audit_publication_sequence, - block_height, - parent_hash, - coinbase_txid, - payout_manifest_sha256, - found_at, - chain_state, - maturity_state -) VALUES ( - '{block_hash}', - {ordinal_sql}, - 10, - '{'10' * 32}', - '{'20' * 32}', - '{'30' * 32}', - '{found_at}', - '{chain_state}', - 'immature' -); -""", - schema=schema, - ) - - -def assert_initializer_rejects_unchanged( - schema: str, - *, - label: str, - error_fragment: str, -) -> dict[str, object]: - before = migration_snapshot(schema) - try: - initialize_migration_schema(schema, writer_id=f"a1-{label}-reject") - except GateFailure as error: - if error_fragment not in str(error): - raise GateFailure( - f"{label} wrong failure: expected {error_fragment!r}, got {error!r}" - ) from error - else: - raise GateFailure(f"{label} initializer unexpectedly succeeded") - assert_equal( - migration_snapshot(schema), - before, - f"{label} transactional rollback snapshot", - ) - assert_advisory_lock_available(f"{label} rejecting advisory lock released") - return before - - -def create_canonical_ordinal_constraint(schema: str, *, not_valid: bool = False) -> None: - run_psql( - """ -ALTER TABLE qbit_pool_blocks -ADD CONSTRAINT qbit_pool_blocks_audit_publication_sequence_check -CHECK ( - (audit_publication_sequence IS NULL OR audit_publication_sequence > 0) - AND (chain_state <> 'confirmed' OR audit_publication_sequence IS NOT NULL) -) -""" - + (" NOT VALID;" if not_valid else ";"), - schema=schema, - ) - - -def test_m0_m11_migration_matrix() -> None: - other_null_expected = { - MIGRATION_HASH_OTHER: None, - "85" * 32: 4, - "86" * 32: None, - "87" * 32: None, - } - common_null_expected = { - MIGRATION_HASH_A: 1, - MIGRATION_HASH_B: 2, - MIGRATION_HASH_C: 3, - **other_null_expected, - } - - m0 = create_owned_schema("m0") - run_psql(LEGACY_POOL_BLOCKS_SQL, schema=m0) - seed_migration_rows(m0, ordinals=None) - assert_valid_migration_case( - schema=m0, - label="m0", - expected_ordinals=common_null_expected, - expected_next_ordinal=5, - ) - - m1 = create_owned_schema("m1") - run_psql(LEGACY_POOL_BLOCKS_SQL, schema=m1) - add_ordinal_column(m1) - seed_migration_rows(m1, ordinals=(None, None, None)) - assert_valid_migration_case( - schema=m1, - label="m1", - expected_ordinals=common_null_expected, - expected_next_ordinal=5, - ) - - m2 = create_owned_schema("m2") - run_psql(LEGACY_POOL_BLOCKS_SQL, schema=m2) - create_ordinal_sequence(m2) - seed_migration_rows(m2, ordinals=None) - assert_valid_migration_case( - schema=m2, - label="m2", - expected_ordinals=common_null_expected, - expected_next_ordinal=5, - ) - - m3 = create_owned_schema("m3") - run_psql(LEGACY_POOL_BLOCKS_SQL, schema=m3) - add_ordinal_column(m3) - create_ordinal_sequence(m3) - seed_migration_rows(m3, ordinals=(None, None, None)) - assert_valid_migration_case( - schema=m3, - label="m3", - expected_ordinals=common_null_expected, - expected_next_ordinal=5, - ) - - m4 = create_owned_schema("m4") - run_psql(LEGACY_POOL_BLOCKS_SQL, schema=m4) - add_ordinal_column(m4) - create_ordinal_sequence(m4) - seed_migration_rows(m4, ordinals=(None, None, None)) - run_psql( - "CREATE UNIQUE INDEX " - "qbit_pool_blocks_audit_publication_sequence_idx " - "ON qbit_pool_blocks (audit_publication_sequence);", - schema=m4, - ) - m4_before = migration_snapshot(m4) - m4_first = assert_valid_migration_case( - schema=m4, - label="m4", - expected_ordinals=common_null_expected, - expected_next_ordinal=5, - ) - assert_equal( - m4_first["indexes"], - m4_before["indexes"], - "M4 first migration preserves correct index OID/filenode/catalog", - ) - assert_equal( - m4_first["named_index_relation"], - m4_before["named_index_relation"], - "M4 first migration preserves correct named index relation", - ) - - m5 = create_owned_schema("m5") - run_psql(LEGACY_POOL_BLOCKS_SQL, schema=m5) - add_ordinal_column(m5) - create_ordinal_sequence(m5, last_value=3, is_called=True) - seed_migration_rows(m5, ordinals=(1, 2, 3)) - create_canonical_ordinal_constraint(m5) - m5_before = migration_snapshot(m5) - m5_first = assert_valid_migration_case( - schema=m5, - label="m5", - expected_ordinals={ - MIGRATION_HASH_A: 1, - MIGRATION_HASH_B: 2, - MIGRATION_HASH_C: 3, - **other_null_expected, - }, - expected_next_ordinal=5, - ) - assert_equal( - m5_first["constraints"], - m5_before["constraints"], - "M5 first migration preserves correct constraint OID/conbin/catalog", - ) - - m6 = create_owned_schema("m6") - run_psql(LEGACY_POOL_BLOCKS_SQL, schema=m6) - add_ordinal_column(m6) - create_ordinal_sequence(m6) - seed_migration_rows(m6, ordinals=(9002, None, None)) - assert_valid_migration_case( - schema=m6, - label="m6", - expected_ordinals={ - MIGRATION_HASH_A: 9002, - MIGRATION_HASH_B: 9003, - MIGRATION_HASH_C: 9004, - MIGRATION_HASH_OTHER: None, - "85" * 32: 9005, - "86" * 32: None, - "87" * 32: None, - }, - expected_next_ordinal=9006, - expected_sequence_state={"last_value": 9005, "is_called": True}, - ) - - for label, called, inactive_ordinal, expected_next in ( - ("m7a", True, 12, 13), - ("m8a", False, 11, 12), - ): - schema = create_owned_schema(label) - run_psql(LEGACY_POOL_BLOCKS_SQL, schema=schema) - add_ordinal_column(schema) - create_ordinal_sequence(schema, last_value=11, is_called=called) - seed_migration_rows(schema, ordinals=(5, 6, 7)) - assert_valid_migration_case( - schema=schema, - label=label, - expected_ordinals={ - MIGRATION_HASH_A: 5, - MIGRATION_HASH_B: 6, - MIGRATION_HASH_C: 7, - MIGRATION_HASH_OTHER: None, - "85" * 32: inactive_ordinal, - "86" * 32: None, - "87" * 32: None, - }, - expected_next_ordinal=expected_next, - expected_sequence_state={ - "last_value": inactive_ordinal, - "is_called": True, - }, - ) - - for label, called, expected, inactive_ordinal, expected_next in ( - ("m7b", True, (12, 13, 14), 15, 16), - ("m8b", False, (11, 12, 13), 14, 15), - ): - schema = create_owned_schema(label) - run_psql(LEGACY_POOL_BLOCKS_SQL, schema=schema) - add_ordinal_column(schema) - create_ordinal_sequence(schema, last_value=11, is_called=called) - seed_migration_rows(schema, ordinals=(None, None, None)) - run_psql( - f""" -UPDATE qbit_pool_blocks -SET audit_publication_sequence = 7 -WHERE block_hash = '{MIGRATION_HASH_OTHER}'; -""", - schema=schema, - ) - assert_valid_migration_case( - schema=schema, - label=label, - expected_ordinals={ - MIGRATION_HASH_A: expected[0], - MIGRATION_HASH_B: expected[1], - MIGRATION_HASH_C: expected[2], - MIGRATION_HASH_OTHER: 7, - "85" * 32: inactive_ordinal, - "86" * 32: None, - "87" * 32: None, - }, - expected_next_ordinal=expected_next, - expected_sequence_state={ - "last_value": inactive_ordinal, - "is_called": True, - }, - ) - - m9 = create_owned_schema("m9") - run_psql(LEGACY_POOL_BLOCKS_SQL, schema=m9) - add_ordinal_column(m9) - seed_migration_rows(m9, ordinals=(0, None, None)) - assert_initializer_rejects_unchanged( - m9, - label="m9", - error_fragment="invalid non-positive audit publication sequence", - ) - - m10 = create_owned_schema("m10") - run_psql(LEGACY_POOL_BLOCKS_SQL, schema=m10) - add_ordinal_column(m10) - seed_migration_rows(m10, ordinals=(4, 4, None)) - assert_initializer_rejects_unchanged( - m10, - label="m10", - error_fragment="duplicate audit publication sequence", - ) - - m11 = create_owned_schema("m11") - run_psql(LEGACY_POOL_BLOCKS_SQL, schema=m11) - create_ordinal_sequence(m11, last_value=19, is_called=True) - seed_migration_rows(m11, ordinals=None) - run_psql( - """ -ALTER TABLE qbit_pool_blocks -ADD CONSTRAINT qbit_pool_blocks_audit_publication_sequence_check -CHECK (block_height < 1000); -""", - schema=m11, - ) - assert_initializer_rejects_unchanged( - m11, - label="m11_wrong_constraint", - error_fragment="invalid audit publication sequence constraint definition", - ) - run_psql( - "ALTER TABLE qbit_pool_blocks DROP CONSTRAINT " - "qbit_pool_blocks_audit_publication_sequence_check;", - schema=m11, - ) - assert_valid_migration_case( - schema=m11, - label="m11", - expected_ordinals={ - MIGRATION_HASH_A: 20, - MIGRATION_HASH_B: 21, - MIGRATION_HASH_C: 22, - "85" * 32: 23, - }, - expected_next_ordinal=24, - ) - - m11_alt = create_owned_schema("m11_alt") - run_psql(LEGACY_POOL_BLOCKS_SQL, schema=m11_alt) - add_ordinal_column(m11_alt) - create_ordinal_sequence(m11_alt, last_value=3, is_called=True) - seed_migration_rows(m11_alt, ordinals=(1, 2, 3)) - run_psql( - "CREATE UNIQUE INDEX qbit_audit_publication_sequence_alternate " - "ON qbit_pool_blocks (audit_publication_sequence);", - schema=m11_alt, - ) - assert_initializer_rejects_unchanged( - m11_alt, - label="m11_alternate_index", - error_fragment="duplicate audit publication sequence index definition", - ) - run_psql( - "DROP INDEX qbit_audit_publication_sequence_alternate;", - schema=m11_alt, - ) - assert_valid_migration_case( - schema=m11_alt, - label="m11_alt", - expected_ordinals={ - MIGRATION_HASH_A: 1, - MIGRATION_HASH_B: 2, - MIGRATION_HASH_C: 3, - "85" * 32: 4, - }, - expected_next_ordinal=5, - ) - - m11_named = create_owned_schema("m11_named") - run_psql(LEGACY_POOL_BLOCKS_SQL, schema=m11_named) - add_ordinal_column(m11_named) - create_ordinal_sequence(m11_named, last_value=3, is_called=True) - seed_migration_rows(m11_named, ordinals=(1, 2, 3)) - run_psql( - "CREATE INDEX qbit_pool_blocks_audit_publication_sequence_idx " - "ON qbit_pool_blocks (audit_publication_sequence);", - schema=m11_named, - ) - assert_initializer_rejects_unchanged( - m11_named, - label="m11_wrong_named_index", - error_fragment="invalid audit publication sequence index definition", - ) - run_psql( - "DROP INDEX qbit_pool_blocks_audit_publication_sequence_idx;", - schema=m11_named, - ) - assert_valid_migration_case( - schema=m11_named, - label="m11_named", - expected_ordinals={ - MIGRATION_HASH_A: 1, - MIGRATION_HASH_B: 2, - MIGRATION_HASH_C: 3, - "85" * 32: 4, - }, - expected_next_ordinal=5, - ) - - m11_sequence = create_owned_schema("m11_sequence") - run_psql(LEGACY_POOL_BLOCKS_SQL, schema=m11_sequence) - add_ordinal_column(m11_sequence) - run_psql( - "CREATE SEQUENCE qbit_audit_publication_sequence_seq AS integer;", - schema=m11_sequence, - ) - seed_migration_rows(m11_sequence, ordinals=(None, None, None)) - assert_initializer_rejects_unchanged( - m11_sequence, - label="m11_wrong_sequence", - error_fragment="invalid audit publication sequence definition", - ) - run_psql( - "DROP SEQUENCE qbit_audit_publication_sequence_seq;", - schema=m11_sequence, - ) - create_ordinal_sequence(m11_sequence) - assert_valid_migration_case( - schema=m11_sequence, - label="m11_sequence", - expected_ordinals=common_null_expected, - expected_next_ordinal=5, - ) - - m11_column = create_owned_schema("m11_column") - run_psql(LEGACY_POOL_BLOCKS_SQL, schema=m11_column) - run_psql( - "ALTER TABLE qbit_pool_blocks " - "ADD COLUMN audit_publication_sequence integer;", - schema=m11_column, - ) - seed_migration_rows(m11_column, ordinals=(None, None, None)) - assert_initializer_rejects_unchanged( - m11_column, - label="m11_wrong_column", - error_fragment="invalid audit publication sequence column definition", - ) - run_psql( - "ALTER TABLE qbit_pool_blocks ALTER COLUMN " - "audit_publication_sequence TYPE bigint;", - schema=m11_column, - ) - assert_valid_migration_case( - schema=m11_column, - label="m11_column", - expected_ordinals=common_null_expected, - expected_next_ordinal=5, - ) - - m11_not_valid = create_owned_schema("m11_not_valid") - run_psql(LEGACY_POOL_BLOCKS_SQL, schema=m11_not_valid) - add_ordinal_column(m11_not_valid) - create_ordinal_sequence(m11_not_valid, last_value=3, is_called=True) - seed_migration_rows(m11_not_valid, ordinals=(1, 2, 3)) - create_canonical_ordinal_constraint(m11_not_valid, not_valid=True) - before_not_valid = migration_snapshot(m11_not_valid) - not_valid_constraint = before_not_valid["constraints"][0] # type: ignore[index] - assert_equal( - not_valid_constraint["validated"], - False, - "M11 preexisting canonical constraint starts NOT VALID", - ) - normalized = assert_valid_migration_case( - schema=m11_not_valid, - label="m11_not_valid", - expected_ordinals={ - MIGRATION_HASH_A: 1, - MIGRATION_HASH_B: 2, - MIGRATION_HASH_C: 3, - **other_null_expected, - }, - expected_next_ordinal=5, - ) - assert_equal( - normalized["constraints"][0]["oid"], # type: ignore[index] - not_valid_constraint["oid"], - "M11 NOT VALID normalization preserves constraint OID", - ) - - m11_alt_constraint = create_owned_schema("m11_alt_constraint") - run_psql(LEGACY_POOL_BLOCKS_SQL, schema=m11_alt_constraint) - add_ordinal_column(m11_alt_constraint) - create_ordinal_sequence(m11_alt_constraint, last_value=3, is_called=True) - seed_migration_rows(m11_alt_constraint, ordinals=(1, 2, 3)) - run_psql( - """ -ALTER TABLE qbit_pool_blocks -ADD CONSTRAINT qbit_audit_publication_sequence_alternate_check -CHECK ( - (audit_publication_sequence IS NULL OR audit_publication_sequence > 0) - AND (chain_state <> 'confirmed' OR audit_publication_sequence IS NOT NULL) -); -""", - schema=m11_alt_constraint, - ) - assert_initializer_rejects_unchanged( - m11_alt_constraint, - label="m11_alternate_constraint", - error_fragment="duplicate audit publication sequence constraint definition", - ) - run_psql( - "ALTER TABLE qbit_pool_blocks DROP CONSTRAINT " - "qbit_audit_publication_sequence_alternate_check;", - schema=m11_alt_constraint, - ) - assert_valid_migration_case( - schema=m11_alt_constraint, - label="m11_alt_constraint", - expected_ordinals={ - MIGRATION_HASH_A: 1, - MIGRATION_HASH_B: 2, - MIGRATION_HASH_C: 3, - **other_null_expected, - }, - expected_next_ordinal=5, - ) - - m11_relkind = create_owned_schema("m11_relkind") - run_psql(LEGACY_POOL_BLOCKS_SQL, schema=m11_relkind) - add_ordinal_column(m11_relkind) - create_ordinal_sequence(m11_relkind, last_value=3, is_called=True) - seed_migration_rows(m11_relkind, ordinals=(1, 2, 3)) - run_psql( - "CREATE TABLE qbit_pool_blocks_audit_publication_sequence_idx " - "(sentinel integer);", - schema=m11_relkind, - ) - assert_initializer_rejects_unchanged( - m11_relkind, - label="m11_wrong_index_relkind", - error_fragment="invalid audit publication sequence index definition", - ) - run_psql( - "DROP TABLE qbit_pool_blocks_audit_publication_sequence_idx;", - schema=m11_relkind, - ) - assert_valid_migration_case( - schema=m11_relkind, - label="m11_relkind", - expected_ordinals={ - MIGRATION_HASH_A: 1, - MIGRATION_HASH_B: 2, - MIGRATION_HASH_C: 3, - **other_null_expected, - }, - expected_next_ordinal=5, - ) - - -def prepare_ordinal_migration_schema( - label: str, - *, - sequence_last: int, - sequence_called: bool, -) -> str: - schema = create_owned_schema(label) - run_psql(LEGACY_POOL_BLOCKS_SQL, schema=schema) - add_ordinal_column(schema) - create_ordinal_sequence( - schema, - last_value=sequence_last, - is_called=sequence_called, - ) - return schema - - -def assert_next_allocation_exhausted(schema: str, *, label: str) -> None: - block_hash = hashlib.sha256(f"{label}-exhausted".encode()).hexdigest() - run_psql( - f""" -INSERT INTO qbit_pool_blocks ( - block_hash, block_height, parent_hash, coinbase_txid, - payout_manifest_sha256, chain_state, maturity_state -) VALUES ( - '{block_hash}', 101, '{'10' * 32}', '{'20' * 32}', - '{'30' * 32}', 'prepared', 'immature' -); -""", - schema=schema, - ) - ledger = ScopedPsqlLedger( - test_schema=schema, - writer_id=f"a1-{label}-exhausted", - writer_epoch=1, - ) - try: - before = migration_snapshot(schema) - try: - ledger.confirm_accepted_block( - block_hash=block_hash, - active_tip_height=101, - ) - except GateFailure as error: - if "reached maximum value" not in str(error): - raise GateFailure( - f"{label} wrong allocator exhaustion: {error!r}" - ) from error - else: - raise GateFailure(f"{label} exhausted allocator unexpectedly advanced") - assert_equal( - migration_snapshot(schema), - before, - f"{label} exhaustion leaves rows/catalog/allocator unchanged", - ) - finally: - ledger.release_writer_lease() - ledger.close() - - -def test_migration_bigint_boundaries() -> None: - max_called = prepare_ordinal_migration_schema( - "bound_max_called", - sequence_last=BIGINT_MAX, - sequence_called=True, - ) - for offset, block_hash in zip( - (-2, -1, 0), - (MIGRATION_HASH_A, MIGRATION_HASH_B, MIGRATION_HASH_C), - strict=True, - ): - insert_migration_row( - max_called, - block_hash=block_hash, - ordinal=BIGINT_MAX + offset, - found_at=f"2020-01-01T00:00:0{offset + 3}Z", - ) - assert_valid_migration_case( - schema=max_called, - label="bound_max_called", - expected_ordinals={ - MIGRATION_HASH_A: BIGINT_MAX - 2, - MIGRATION_HASH_B: BIGINT_MAX - 1, - MIGRATION_HASH_C: BIGINT_MAX, - }, - expected_next_ordinal=None, - expected_sequence_state={"last_value": BIGINT_MAX, "is_called": True}, - ) - assert_next_allocation_exhausted( - max_called, - label="bound_max_called", - ) - - max_uncalled_one = prepare_ordinal_migration_schema( - "bound_max_one", - sequence_last=BIGINT_MAX, - sequence_called=False, - ) - insert_migration_row( - max_uncalled_one, - block_hash=MIGRATION_HASH_A, - ordinal=None, - found_at="2020-01-01T00:00:01Z", - ) - assert_valid_migration_case( - schema=max_uncalled_one, - label="bound_max_one", - expected_ordinals={MIGRATION_HASH_A: BIGINT_MAX}, - expected_next_ordinal=None, - expected_sequence_state={"last_value": BIGINT_MAX, "is_called": True}, - ) - assert_next_allocation_exhausted( - max_uncalled_one, - label="bound_max_one", - ) - - max_uncalled_two = prepare_ordinal_migration_schema( - "bound_max_two", - sequence_last=BIGINT_MAX, - sequence_called=False, - ) - for block_hash, found_at in ( - (MIGRATION_HASH_A, "2020-01-01T00:00:01Z"), - (MIGRATION_HASH_B, "2020-01-01T00:00:02Z"), - ): - insert_migration_row( - max_uncalled_two, - block_hash=block_hash, - ordinal=None, - found_at=found_at, - ) - assert_initializer_rejects_unchanged( - max_uncalled_two, - label="bound_max_two", - error_fragment="audit publication sequence exhausted", - ) - - durable_max_pending = prepare_ordinal_migration_schema( - "bound_durable_max", - sequence_last=1, - sequence_called=False, - ) - insert_migration_row( - durable_max_pending, - block_hash=MIGRATION_HASH_A, - ordinal=BIGINT_MAX, - found_at="2020-01-01T00:00:01Z", - ) - insert_migration_row( - durable_max_pending, - block_hash=MIGRATION_HASH_B, - ordinal=None, - found_at="2020-01-01T00:00:02Z", - ) - assert_initializer_rejects_unchanged( - durable_max_pending, - label="bound_durable_max", - error_fragment="audit publication sequence exhausted", - ) - - near_max_three = prepare_ordinal_migration_schema( - "bound_near_three", - sequence_last=BIGINT_MAX - 2, - sequence_called=False, - ) - for block_hash, second in ( - (MIGRATION_HASH_C, 2), - (MIGRATION_HASH_B, 1), - (MIGRATION_HASH_A, 1), - ): - insert_migration_row( - near_max_three, - block_hash=block_hash, - ordinal=None, - found_at=f"2020-01-01T00:00:0{second}Z", - ) - assert_valid_migration_case( - schema=near_max_three, - label="bound_near_three", - expected_ordinals={ - MIGRATION_HASH_A: BIGINT_MAX - 2, - MIGRATION_HASH_B: BIGINT_MAX - 1, - MIGRATION_HASH_C: BIGINT_MAX, - }, - expected_next_ordinal=None, - expected_sequence_state={"last_value": BIGINT_MAX, "is_called": True}, - ) - - near_max_four = prepare_ordinal_migration_schema( - "bound_near_four", - sequence_last=BIGINT_MAX - 2, - sequence_called=False, - ) - for index, block_hash in enumerate( - (MIGRATION_HASH_A, MIGRATION_HASH_B, MIGRATION_HASH_C, "88" * 32), - start=1, - ): - insert_migration_row( - near_max_four, - block_hash=block_hash, - ordinal=None, - found_at=f"2020-01-01T00:00:0{index}Z", - ) - assert_initializer_rejects_unchanged( - near_max_four, - label="bound_near_four", - error_fragment="audit publication sequence exhausted", - ) - - -def test_invalid_sequence_and_column_definitions() -> None: - sequence_definitions = { - "seq_start": "CREATE SEQUENCE qbit_audit_publication_sequence_seq AS bigint START WITH 2", - "seq_increment": "CREATE SEQUENCE qbit_audit_publication_sequence_seq AS bigint INCREMENT BY 2", - "seq_min": "CREATE SEQUENCE qbit_audit_publication_sequence_seq AS bigint MINVALUE 2 START WITH 2", - "seq_max": "CREATE SEQUENCE qbit_audit_publication_sequence_seq AS bigint MAXVALUE 100", - "seq_cache": "CREATE SEQUENCE qbit_audit_publication_sequence_seq AS bigint CACHE 2", - "seq_cycle": "CREATE SEQUENCE qbit_audit_publication_sequence_seq AS bigint CYCLE", - "seq_unlogged": "CREATE UNLOGGED SEQUENCE qbit_audit_publication_sequence_seq AS bigint", - } - for label, definition in sequence_definitions.items(): - schema = create_owned_schema(label) - run_psql(LEGACY_POOL_BLOCKS_SQL, schema=schema) - add_ordinal_column(schema) - run_psql(definition + ";", schema=schema) - seed_migration_rows( - schema, - ordinals=(1, 2, 3), - include_other_states=False, - ) - assert_initializer_rejects_unchanged( - schema, - label=label, - error_fragment="invalid audit publication sequence definition", - ) - - owned = create_owned_schema("seq_owned") - run_psql(LEGACY_POOL_BLOCKS_SQL, schema=owned) - add_ordinal_column(owned) - create_ordinal_sequence(owned) - run_psql( - "ALTER SEQUENCE qbit_audit_publication_sequence_seq " - "OWNED BY qbit_pool_blocks.audit_publication_sequence;", - schema=owned, - ) - seed_migration_rows(owned, ordinals=(1, 2, 3)) - assert_initializer_rejects_unchanged( - owned, - label="seq_owned", - error_fragment="invalid audit publication sequence definition", - ) - - wrong_kind = create_owned_schema("seq_wrong_kind") - run_psql(LEGACY_POOL_BLOCKS_SQL, schema=wrong_kind) - add_ordinal_column(wrong_kind) - run_psql( - "CREATE TABLE qbit_audit_publication_sequence_seq (sentinel integer);", - schema=wrong_kind, - ) - seed_migration_rows(wrong_kind, ordinals=(1, 2, 3)) - assert_initializer_rejects_unchanged( - wrong_kind, - label="seq_wrong_kind", - error_fragment="invalid audit publication sequence definition", - ) - - column_definitions = { - "column_not_null": "bigint NOT NULL", - "column_default": "bigint DEFAULT 1", - "column_identity": "bigint GENERATED BY DEFAULT AS IDENTITY", - } - for label, definition in column_definitions.items(): - schema = create_owned_schema(label) - run_psql(LEGACY_POOL_BLOCKS_SQL, schema=schema) - run_psql( - "ALTER TABLE qbit_pool_blocks ADD COLUMN " - f"audit_publication_sequence {definition};", - schema=schema, - ) - seed_migration_rows( - schema, - ordinals=(1, 2, 3), - include_other_states=False, - ) - assert_initializer_rejects_unchanged( - schema, - label=label, - error_fragment="invalid audit publication sequence column definition", - ) - - generated = create_owned_schema("column_generated") - run_psql(LEGACY_POOL_BLOCKS_SQL, schema=generated) - seed_migration_rows(generated, ordinals=None) - run_psql( - "ALTER TABLE qbit_pool_blocks ADD COLUMN " - "audit_publication_sequence bigint " - "GENERATED ALWAYS AS (block_height + 1) STORED;", - schema=generated, - ) - assert_initializer_rejects_unchanged( - generated, - label="column_generated", - error_fragment="invalid audit publication sequence column definition", - ) - # Owner mismatch needs CREATE/SET ROLE authority that the supported - # external-psql mode does not promise. Production still validates exact - # same-owner identity, and every portable sequence property is exercised. - - -def _psql_process_command() -> list[str]: - return [ - *BASE_PSQL_ARGV, - "--no-psqlrc", - "--set", - "ON_ERROR_STOP=1", - "--tuples-only", - "--no-align", - "--quiet", - ] - - -def _register_process(process: subprocess.Popen[str]) -> None: - with ACTIVE_CHILDREN_LOCK: - ACTIVE_CHILDREN.add(process) - - -def _forget_process(process: subprocess.Popen[str]) -> None: - with ACTIVE_CHILDREN_LOCK: - ACTIVE_CHILDREN.discard(process) - - -def _wait_for_advisory_holder( - process: subprocess.Popen[str], - *, - application_name: str, - deadline: float, -) -> None: - while time.monotonic() < deadline: - state = run_json( - f""" -SELECT json_build_object( - 'ready', EXISTS ( - SELECT 1 - FROM pg_catalog.pg_stat_activity activity - JOIN pg_catalog.pg_locks lock_state - ON lock_state.pid = activity.pid - WHERE activity.application_name = '{application_name}' - AND lock_state.locktype = 'advisory' - AND lock_state.granted - ) -); -""" - ) - if state["ready"]: - return - if process.poll() is not None: - break - time.sleep(0.05) - raise GateFailure( - f"advisory holder {application_name!r} not observed; exit={process.poll()}" - ) - - -def _wait_file_process( - process: subprocess.Popen[str], - *, - stdout_file: Any, - stderr_file: Any, - expected_success: bool, - error_fragment: str | None, -) -> None: - try: - process.wait(timeout=PSQL_TIMEOUT_SECONDS) - except subprocess.TimeoutExpired as error: - _terminate_and_reap(process) - raise GateFailure("tagged psql worker timed out") from error - finally: - _forget_process(process) - stdout_file.seek(0) - stderr_file.seek(0) - stdout = stdout_file.read(PSQL_OUTPUT_LIMIT_BYTES + 1) - stderr = stderr_file.read(PSQL_OUTPUT_LIMIT_BYTES + 1) - if ( - len(stdout.encode("utf-8")) > PSQL_OUTPUT_LIMIT_BYTES - or len(stderr.encode("utf-8")) > PSQL_OUTPUT_LIMIT_BYTES - ): - raise GateFailure("tagged psql worker output exceeded 1 MiB") - if expected_success and process.returncode != 0: - raise GateFailure( - f"tagged psql worker failed with {process.returncode}: {stderr.strip()}" - ) - if not expected_success: - if process.returncode == 0: - raise GateFailure("rejecting tagged psql worker unexpectedly succeeded") - if error_fragment is not None and error_fragment not in stderr: - raise GateFailure( - f"tagged psql worker wrong failure: {stderr.strip()}" - ) - - -def test_migration_advisory_waiter() -> None: - schema_sql = ( - Path(__file__).resolve().parents[1] - / "crates/qbit-prism/sql/001_share_ledger.sql" - ).read_text(encoding="utf-8") - for outcome, rejecting in (("commit", False), ("rollback", True)): - schema = create_owned_schema(f"migration_wait_{outcome}") - rejection_before: dict[str, object] | None = None - if rejecting: - run_psql(LEGACY_POOL_BLOCKS_SQL, schema=schema) - add_ordinal_column(schema) - seed_migration_rows(schema, ordinals=(0, None, None)) - rejection_before = migration_snapshot(schema) - # PostgreSQL truncates application_name to NAMEDATALEN - 1 bytes. - # Keep the tags below that limit so catalog coordination uses the - # exact values the clients set. - holder_tag = f"a1_mig_holder_{RUN_TOKEN}_{outcome}" - waiter_tag = f"a1_mig_waiter_{RUN_TOKEN}_{outcome}" - holder_input = tempfile.TemporaryFile(mode="w+t", encoding="utf-8") - holder_stderr = tempfile.TemporaryFile(mode="w+t", encoding="utf-8") - waiter_input = tempfile.TemporaryFile(mode="w+t", encoding="utf-8") - waiter_stdout = tempfile.TemporaryFile(mode="w+t", encoding="utf-8") - waiter_stderr = tempfile.TemporaryFile(mode="w+t", encoding="utf-8") - holder: subprocess.Popen[str] | None = None - waiter: subprocess.Popen[str] | None = None - try: - holder_input.write( - f""" -SET application_name = '{holder_tag}'; -BEGIN; -SELECT pg_catalog.pg_advisory_xact_lock( - pg_catalog.hashtext('qbit_audit_publication_sequence_migration') -); -DO $qbit_a1_wait$ -DECLARE - waiter_observed boolean := false; -BEGIN - FOR attempt IN 1..600 LOOP - PERFORM pg_catalog.pg_stat_clear_snapshot(); - SELECT EXISTS ( - SELECT 1 - FROM pg_catalog.pg_stat_activity waiter - JOIN pg_catalog.pg_locks lock_state - ON lock_state.pid = waiter.pid - WHERE waiter.application_name = '{waiter_tag}' - AND lock_state.locktype = 'advisory' - AND NOT lock_state.granted - AND pg_catalog.pg_backend_pid() = ANY( - pg_catalog.pg_blocking_pids(waiter.pid) - ) - ) - INTO waiter_observed; - EXIT WHEN waiter_observed; - PERFORM pg_catalog.pg_sleep(0.05); - END LOOP; - IF NOT waiter_observed THEN - RAISE EXCEPTION 'migration advisory waiter not observed'; - END IF; -END; -$qbit_a1_wait$; -{outcome.upper()}; -""" - ) - holder_input.seek(0) - holder = subprocess.Popen( - _psql_process_command(), - stdin=holder_input, - stdout=subprocess.DEVNULL, - stderr=holder_stderr, - text=True, - bufsize=1, - start_new_session=True, - ) - _register_process(holder) - _wait_for_advisory_holder( - holder, - application_name=holder_tag, - deadline=time.monotonic() + PSQL_TIMEOUT_SECONDS, - ) - - waiter_input.write( - f""" -SET application_name = '{waiter_tag}'; -SET statement_timeout = '20s'; -SET lock_timeout = '20s'; -SET search_path TO "{schema}", pg_catalog; -{schema_sql} -""" - ) - waiter_input.seek(0) - waiter = subprocess.Popen( - _psql_process_command(), - stdin=waiter_input, - stdout=waiter_stdout, - stderr=waiter_stderr, - text=True, - start_new_session=True, - ) - _register_process(waiter) - try: - holder.wait(timeout=PSQL_TIMEOUT_SECONDS) - except subprocess.TimeoutExpired as error: - _terminate_and_reap(holder) - raise GateFailure("migration advisory holder timed out") from error - finally: - _forget_process(holder) - if holder.returncode != 0: - holder_stderr.seek(0) - raise GateFailure( - "migration advisory holder failed: " - + holder_stderr.read(PSQL_OUTPUT_LIMIT_BYTES + 1).strip() - ) - - _wait_file_process( - waiter, - stdout_file=waiter_stdout, - stderr_file=waiter_stderr, - expected_success=not rejecting, - error_fragment=( - "invalid non-positive audit publication sequence" - if rejecting - else None - ), - ) - if rejecting: - assert rejection_before is not None - assert_equal( - migration_snapshot(schema), - rejection_before, - "rejecting advisory waiter exact rollback", - ) - else: - success_snapshot = migration_snapshot(schema) - assert_equal( - success_snapshot["sequence_state"], - {"last_value": 1, "is_called": False}, - "successful advisory waiter empty allocator", - ) - assert_advisory_lock_available( - f"migration {outcome} advisory lock no leak" - ) - remaining = run_json( - f""" -SELECT json_build_object('count', count(*)) -FROM pg_catalog.pg_stat_activity -WHERE application_name IN ('{holder_tag}', '{waiter_tag}'); -""" - )["count"] - assert_equal( - remaining, - 0, - f"migration {outcome} tagged backend cleanup", - ) - finally: - for process in (waiter, holder): - if process is not None: - if process.poll() is None: - _terminate_and_reap(process) - _forget_process(process) - for stream in (process.stdin, process.stdout, process.stderr): - if stream is not None and not stream.closed: - stream.close() - for stream in ( - holder_input, - holder_stderr, - waiter_input, - waiter_stdout, - waiter_stderr, - ): - if not stream.closed: - stream.close() - - -def main() -> None: - public_before = support.public_sentinel() - failure: BaseException | None = None - try: - test_m0_m11_migration_matrix() - test_migration_bigint_boundaries() - test_invalid_sequence_and_column_definitions() - test_migration_advisory_waiter() - except BaseException as error: - failure = error - try: - support.cleanup_active_children() - support.cleanup_owned_schemas() - support.assert_equal(support.marker_schema_count(), 0, "migration marker cleanup") - support.assert_equal(support.public_sentinel(), public_before, "migration public preservation") - except BaseException as cleanup_error: - if failure is None: - raise - raise GateFailure( - f"migration scenario failed with {failure!r}; cleanup also failed with {cleanup_error!r}" - ) from cleanup_error - else: - support.atexit.unregister(support.cleanup_active_children) - support.atexit.unregister(support.cleanup_owned_schemas) - if failure is not None: - raise failure - print( - "prism postgres A1 migration gate PASS " - "M0-M11 bigint-bounds invalid-definitions migration-advisory-waiter" - ) - - -if __name__ == "__main__": - main() diff --git a/tests/prism_postgres_a1_process_gate.py b/tests/prism_postgres_a1_process_gate.py deleted file mode 100644 index 9ff853653..000000000 --- a/tests/prism_postgres_a1_process_gate.py +++ /dev/null @@ -1,2624 +0,0 @@ -"""Non-discovered PostgreSQL/A1 cross-process integration gate.""" - -from __future__ import annotations - -from contextlib import ExitStack -import fcntl -import hashlib -import json -import os -from pathlib import Path -import re -import select -import subprocess -import sys -import tempfile -import time -import traceback -from typing import Any - -from lab.prism import audit_artifacts as audit_artifacts_module -from lab.prism.audit_artifacts import ( - AuditArtifactConfig, - AuditArtifactStore, - AuditPublicationIdentity, -) -from lab.prism.share_ledger import PsqlShareLedger -from tests import prism_postgres_a1_gate as support -from tests import prism_postgres_a1_migration_gate as migration - - -_PROCESS_GATE_MODULE = "tests.prism_postgres_a1_process_gate" -_C2_DIGEST = "11" * 32 - - -class _WorkerScopedPsqlLedger(PsqlShareLedger): - """Child-process ledger that consumes, but never owns, the parent schema.""" - - def __init__( - self, - *, - test_schema: str, - application_name: str, - **kwargs: object, - ) -> None: - if support.SCHEMA_PATTERN.fullmatch(test_schema) is None: - raise support.GateFailure(f"invalid worker schema: {test_schema!r}") - if ( - re.fullmatch(r"qbit_a1_c[23]_[a-z0-9_]+", application_name) is None - or len(application_name) > 63 - ): - raise support.GateFailure( - f"invalid process worker application name: {application_name!r}" - ) - self._worker_schema = test_schema - self._application_name = application_name - kwargs["psql_command"] = support.BASE_PSQL_COMMAND - kwargs["native_client_mode"] = "psql" - super().__init__(**kwargs) # type: ignore[arg-type] - - def _run_sql(self, sql: str) -> str: - return support.run_psql( - f"SET application_name = '{self._application_name}';\n" + sql, - schema=self._worker_schema, - ) - - -def _c2_report(*, block_height: int) -> dict[str, object]: - return { - "schema": "qbit.prism.audit-verification-report.v1", - "block_height": block_height, - "audit_bundle_sha256_hex": _C2_DIGEST, - "reward_manifest_sha256_hex": "44" * 32, - "payout_policy_manifest_sha256_hex": "55" * 32, - "prism_audit_commitment_leaf_hex": "66" * 32, - "audit_commitment_root_hex": "77" * 32, - "coinbase_txid": "22" * 32, - "coinbase_wtxid": "88" * 32, - "coinbase_manifest_sha256_hex": "33" * 32, - "coinbase_tx_hex": "00", - "coinbase_value_sats": 1, - "min_output_sats": 1, - "onchain_output_count": 0, - "accrued_account_count": 0, - } - - -def _c2_store(root: Path, evidence_path: Path) -> AuditArtifactStore: - return AuditArtifactStore( - AuditArtifactConfig( - root=root, - evidence_path=evidence_path, - live_bundle_retention=1, - candidate_retention_seconds=60, - share_segment_size=0, - ) - ) - - -def _c2_publish( - store: AuditArtifactStore, - *, - identity: AuditPublicationIdentity, - publication_floor_sequence: int, - created_at: str, -) -> object: - report = _c2_report(block_height=identity.block_height) - verification_identity = store.build_verification_identity( - trust_source="configured", - trusted_writer_public_key_hex="44" * 32, - literal_sha256=_C2_DIGEST, - literal_byte_len=123, - report=report, - ) - return store.publish_success( - identity=identity, - publication_floor_sequence=publication_floor_sequence, - report=report, - persistence={ - "audit_bundle_sha256": _C2_DIGEST, - "body_uri": "", - }, - evidence={ - "accepted_share_count": 0, - "distinct_miner_count": 0, - }, - verification_identity=verification_identity, - created_at=created_at, - ) - - -def _c2_authority(store: AuditArtifactStore) -> dict[str, list[int]]: - root_value = os.fstat(store._root_fd) - lock_value = os.fstat(store._publication_lock_fd) - return { - "root": [root_value.st_dev, root_value.st_ino], - "lock": [lock_value.st_dev, lock_value.st_ino], - } - - -def _c2_path_snapshot(path: Path) -> dict[str, object]: - try: - value = path.lstat() - except FileNotFoundError: - return {"exists": False} - payload: dict[str, object] = { - "exists": True, - "device": value.st_dev, - "inode": value.st_ino, - "mode": value.st_mode, - "size": value.st_size, - "mtime_ns": value.st_mtime_ns, - } - if path.is_file() and not path.is_symlink(): - payload["sha256"] = hashlib.sha256(path.read_bytes()).hexdigest() - return payload - - -def _c2_filesystem_snapshot( - *, - root: Path, - evidence_path: Path, -) -> dict[str, object]: - return { - "root": _c2_path_snapshot(root), - "entries": { - path.name: _c2_path_snapshot(path) - for path in sorted(root.iterdir(), key=lambda value: value.name) - }, - "evidence": _c2_path_snapshot(evidence_path), - } - - -def _c2_database_snapshot( - ledger: PsqlShareLedger, - *, - block_hashes: tuple[str, ...], -) -> dict[str, object]: - quoted = ", ".join(f"'{block_hash}'" for block_hash in block_hashes) - return ledger._run_json( - f""" -SELECT json_build_object( - 'rows', COALESCE(( - SELECT json_agg(json_build_object( - 'block_hash', block_hash, - 'block_height', block_height, - 'chain_state', chain_state, - 'maturity_state', maturity_state, - 'audit_publication_sequence', audit_publication_sequence - ) ORDER BY block_hash) - FROM qbit_pool_blocks - WHERE block_hash IN ({quoted}) - ), '[]'::json), - 'floor', COALESCE(( - SELECT MAX(audit_publication_sequence) - FROM qbit_pool_blocks - ), 0), - 'allocator', ( - SELECT json_build_object( - 'last_value', last_value, - 'is_called', is_called - ) - FROM qbit_audit_publication_sequence_seq - ) -); -""" - ) - - -def _emit_worker_event(event: str, **payload: object) -> None: - print( - json.dumps( - {"event": event, **payload}, - separators=(",", ":"), - sort_keys=True, - ), - flush=True, - ) - - -def _read_worker_command(expected: str) -> dict[str, object]: - line = sys.stdin.readline(support.PSQL_OUTPUT_LIMIT_BYTES + 2) - if not line: - raise support.GateFailure(f"worker command {expected!r} was not received") - if len(line.encode("utf-8")) > support.PSQL_OUTPUT_LIMIT_BYTES: - raise support.GateFailure("worker command exceeded 1 MiB") - try: - payload = json.loads(line) - except json.JSONDecodeError as error: - raise support.GateFailure("worker command is not JSON") from error - if not isinstance(payload, dict) or payload.get("command") != expected: - raise support.GateFailure( - f"expected worker command {expected!r}, got {payload!r}" - ) - return payload - - -class _JsonWorker: - def __init__(self, config_path: Path) -> None: - self._stderr = tempfile.TemporaryFile(mode="w+t", encoding="utf-8") - self._captured = 0 - self._event_buffer = bytearray() - self.process = subprocess.Popen( - [ - sys.executable, - "-m", - _PROCESS_GATE_MODULE, - "--c2-worker", - str(config_path), - ], - stdin=subprocess.PIPE, - stdout=subprocess.PIPE, - stderr=self._stderr, - text=True, - bufsize=1, - start_new_session=True, - ) - migration._register_process(self.process) - - def _stderr_text(self) -> str: - raw = os.pread( - self._stderr.fileno(), - support.PSQL_OUTPUT_LIMIT_BYTES + 1, - 0, - ) - if len(raw) > support.PSQL_OUTPUT_LIMIT_BYTES: - return "worker stderr exceeded 1 MiB" - return raw.decode("utf-8", errors="replace").strip() - - def read_event(self, expected: str) -> dict[str, object]: - if self.process.stdout is None: - raise support.GateFailure("JSON worker has no stdout") - deadline = time.monotonic() + support.PSQL_TIMEOUT_SECONDS - while time.monotonic() < deadline: - newline = self._event_buffer.find(b"\n") - if newline >= 0: - raw_line = bytes(self._event_buffer[:newline]) - del self._event_buffer[: newline + 1] - try: - line = raw_line.decode("utf-8") - except UnicodeDecodeError as error: - raise support.GateFailure( - "JSON worker emitted non-UTF-8 protocol bytes" - ) from error - try: - payload = json.loads(line) - except json.JSONDecodeError as error: - raise support.GateFailure( - f"JSON worker emitted invalid protocol line: {line!r}" - ) from error - if not isinstance(payload, dict): - raise support.GateFailure("JSON worker event is not an object") - if payload.get("event") == "error": - raise support.GateFailure( - f"JSON worker failed: {payload.get('message')}; " - f"stderr={self._stderr_text()}" - ) - if payload.get("event") != expected: - raise support.GateFailure( - f"expected JSON worker event {expected!r}, got {payload!r}" - ) - return payload - remaining = max(0.0, deadline - time.monotonic()) - ready, _, _ = select.select([self.process.stdout], [], [], remaining) - if not ready: - break - chunk = os.read(self.process.stdout.fileno(), 64 * 1024) - if not chunk: - break - self._captured += len(chunk) - if self._captured > support.PSQL_OUTPUT_LIMIT_BYTES: - raise support.GateFailure("JSON worker stdout exceeded 1 MiB") - self._event_buffer.extend(chunk) - if ( - b"\n" not in self._event_buffer - and len(self._event_buffer) > support.PSQL_OUTPUT_LIMIT_BYTES - ): - raise support.GateFailure( - "JSON worker event line exceeded 1 MiB" - ) - raise support.GateFailure( - f"JSON worker event {expected!r} timed out; " - f"exit={self.process.poll()} stderr={self._stderr_text()}" - ) - - def send(self, command: str) -> None: - if self.process.stdin is None or self.process.stdin.closed: - raise support.GateFailure("JSON worker has no writable stdin") - self.process.stdin.write( - json.dumps( - {"command": command}, - separators=(",", ":"), - sort_keys=True, - ) - + "\n" - ) - self.process.stdin.flush() - - def wait_success(self) -> None: - if self.process.stdin is not None and not self.process.stdin.closed: - self.process.stdin.close() - try: - self.process.wait(timeout=support.PSQL_TIMEOUT_SECONDS) - except subprocess.TimeoutExpired as error: - support._terminate_and_reap(self.process) - raise support.GateFailure("JSON worker timed out during exit") from error - finally: - migration._forget_process(self.process) - residual = bytes(self._event_buffer) - self._event_buffer.clear() - if self.process.stdout is not None: - while True: - chunk = os.read(self.process.stdout.fileno(), 64 * 1024) - if not chunk: - break - residual += chunk - self._captured += len(chunk) - if self._captured > support.PSQL_OUTPUT_LIMIT_BYTES: - raise support.GateFailure("JSON worker stdout exceeded 1 MiB") - stderr = self._stderr_text() - if self.process.returncode != 0: - raise support.GateFailure( - f"JSON worker exited {self.process.returncode}: {stderr}" - ) - if residual.strip(): - raise support.GateFailure( - "JSON worker emitted unexpected residual output: " - + residual.decode("utf-8", errors="replace").strip() - ) - if stderr: - raise support.GateFailure( - f"JSON worker emitted unexpected stderr: {stderr}" - ) - - def close(self) -> None: - if self.process.poll() is None: - support._terminate_and_reap(self.process) - migration._forget_process(self.process) - for stream in (self.process.stdin, self.process.stdout, self.process.stderr): - if stream is not None and not stream.closed: - stream.close() - if not self._stderr.closed: - self._stderr.close() - - -def _c2_worker_ledger(config: dict[str, object]) -> _WorkerScopedPsqlLedger: - writer = config.get("writer") - if not isinstance(writer, dict): - raise support.GateFailure("process worker writer configuration is invalid") - return _WorkerScopedPsqlLedger( - test_schema=str(config["schema"]), - application_name=str(config["application_name"]), - writer_id=str(writer["id"]), - writer_epoch=int(writer["epoch"]), - writer_session_token=str(writer["token"]), - initialize_schema=False, - ) - - -def _c2_worker_store(config: dict[str, object]) -> AuditArtifactStore: - return _c2_store( - Path(str(config["root"])), - Path(str(config["evidence_path"])), - ) - - -def _c2_worker_identity( - config: dict[str, object], - key: str, - *, - sequence: int, -) -> AuditPublicationIdentity: - value = config.get(key) - if not isinstance(value, dict): - raise support.GateFailure(f"process worker {key} identity is invalid") - return AuditPublicationIdentity( - sequence, - int(value["height"]), - str(value["hash"]), - ) - - -def _run_c2_a_worker(config: dict[str, object]) -> None: - ledger: _WorkerScopedPsqlLedger | None = None - store: AuditArtifactStore | None = None - try: - ledger = _c2_worker_ledger(config) - store = _c2_worker_store(config) - initial = _c2_worker_identity( - config, - "initial", - sequence=int(config["initial_sequence"]), - ) - with store.publication_order_guard(): - floor = ledger.audit_publication_sequence_floor() - _emit_worker_event( - "guard-acquired", - authority=_c2_authority(store), - floor=floor, - pid=os.getpid(), - ) - _read_worker_command("repair") - publication = _c2_publish( - store, - identity=initial, - publication_floor_sequence=floor, - created_at=f"c2-{config['variant']}-a-repair", - ) - _emit_worker_event( - "repaired", - published=publication.published, # type: ignore[attr-defined] - identity=publication.identity.to_json(), # type: ignore[attr-defined] - latest=store.latest_evidence(), - authority=_c2_authority(store), - ) - _emit_worker_event("guard-released", authority=_c2_authority(store)) - _read_worker_command("stale-check") - stale_hashes = tuple( - str(config[key]["hash"]) # type: ignore[index] - for key in ("stale_confirmation", "stale_reactivation") - ) - before = _c2_database_snapshot(ledger, block_hashes=stale_hashes) - errors: dict[str, str] = {} - stale_confirmation = config["stale_confirmation"] - stale_reactivation = config["stale_reactivation"] - assert isinstance(stale_confirmation, dict) - assert isinstance(stale_reactivation, dict) - operations = ( - ( - "confirmation", - ledger.confirm_accepted_block, - stale_confirmation, - ), - ( - "reactivation", - ledger.reactivate_pool_block, - stale_reactivation, - ), - ) - for label, operation, target in operations: - try: - operation( - block_hash=str(target["hash"]), - active_tip_height=int(target["height"]), - ) - except Exception as error: - message = str(error) - if "writer lease is not active" not in message: - raise support.GateFailure( - f"stale A {label} wrong failure: {message}" - ) from error - errors[label] = message - else: - raise support.GateFailure( - f"stale A {label} unexpectedly succeeded" - ) - after = _c2_database_snapshot(ledger, block_hashes=stale_hashes) - _emit_worker_event( - "stale-result", - errors=errors, - before=before, - after=after, - authority=_c2_authority(store), - ) - _read_worker_command("finish") - store.close() - store = None - ledger.close() - ledger = None - _emit_worker_event("closed") - finally: - if store is not None: - store.close() - if ledger is not None: - ledger.close() - - -def _run_c2_b_worker(config: dict[str, object]) -> None: - ledger: _WorkerScopedPsqlLedger | None = None - store: AuditArtifactStore | None = None - original_flock = audit_artifacts_module.fcntl.flock - try: - ledger = _c2_worker_ledger(config) - store = _c2_worker_store(config) - writer = config["writer"] - assert isinstance(writer, dict) - ledger._run_sql( - f""" -INSERT INTO qbit_a1_c2_attempts (variant, transition, writer_id) -VALUES ( - '{config['variant']}', - '{config['transition']}', - '{writer['id']}' -); -""" - ) - attempted = False - - def observed_flock(fd: int, operation: int) -> object: - nonlocal attempted - if ( - not attempted - and fd == store._publication_lock_fd # type: ignore[union-attr] - and operation & fcntl.LOCK_EX - ): - attempted = True - _emit_worker_event( - "flock-attempt", - authority=_c2_authority(store), # type: ignore[arg-type] - pid=os.getpid(), - ) - return original_flock(fd, operation) - - audit_artifacts_module.fcntl.flock = observed_flock - try: - with store.publication_order_guard(): - if str(config["transition"]) == "confirmation": - target = config["target"] - assert isinstance(target, dict) - result = ledger.confirm_accepted_block( - block_hash=str(target["hash"]), - active_tip_height=int(target["height"]), - ) - count = int(result["confirmed_count"]) - elif str(config["transition"]) == "reactivation": - target = config["target"] - assert isinstance(target, dict) - result = ledger.reactivate_pool_block( - block_hash=str(target["hash"]), - active_tip_height=int(target["height"]), - ) - count = int(result["reactivated_count"]) - else: - raise support.GateFailure("invalid C2 B transition") - support.assert_equal(count, 1, "C2 B transition count") - sequence = int(result["audit_publication_sequence"]) - identity = _c2_worker_identity( - config, - "target", - sequence=sequence, - ) - floor = ledger.audit_publication_sequence_floor() - publication = _c2_publish( - store, - identity=identity, - publication_floor_sequence=floor, - created_at=f"c2-{config['variant']}-b-publication", - ) - _emit_worker_event( - "transition-published", - transition_result=result, - floor=floor, - published=publication.published, # type: ignore[attr-defined] - identity=identity.to_json(), - latest=store.latest_evidence(), - authority=_c2_authority(store), - ) - finally: - audit_artifacts_module.fcntl.flock = original_flock - if not attempted: - raise support.GateFailure("C2 B did not attempt the publication flock") - _read_worker_command("finish") - released = ledger.release_writer_lease() - support.assert_equal(released, True, "C2 B writer lease release") - store.close() - store = None - ledger.close() - ledger = None - _emit_worker_event("closed", lease_released=released) - finally: - audit_artifacts_module.fcntl.flock = original_flock - if store is not None: - store.close() - if ledger is not None: - try: - ledger.release_writer_lease() - except BaseException: - pass - ledger.close() - - -def _c3_stale_publication_attempt( - *, - ledger: _WorkerScopedPsqlLedger, - store: AuditArtifactStore, - config: dict[str, object], -) -> tuple[int, dict[str, object]]: - floor = ledger.audit_publication_sequence_floor() - identity = _c2_worker_identity( - config, - "initial", - sequence=int(config["initial_sequence"]), - ) - try: - publication = _c2_publish( - store, - identity=identity, - publication_floor_sequence=floor, - created_at=f"c3-{config['variant']}-late-a", - ) - except RuntimeError as error: - message = str(error) - if "behind" not in message: - raise support.GateFailure( - f"C3 stale A publication wrong failure: {message}" - ) from error - outcome = {"kind": "behind-error", "message": message} - else: - if publication.published: - raise support.GateFailure("C3 stale A publication regressed evidence") - outcome = { - "kind": "returned", - "published": publication.published, - "identity": publication.identity.to_json(), - } - return floor, outcome - - -def _c3_stale_transition_check( - ledger: _WorkerScopedPsqlLedger, - config: dict[str, object], -) -> dict[str, object]: - stale_hashes = tuple( - str(config[key]["hash"]) # type: ignore[index] - for key in ("stale_confirmation", "stale_reactivation") - ) - before = _c2_database_snapshot(ledger, block_hashes=stale_hashes) - errors: dict[str, str] = {} - stale_confirmation = config["stale_confirmation"] - stale_reactivation = config["stale_reactivation"] - assert isinstance(stale_confirmation, dict) - assert isinstance(stale_reactivation, dict) - operations = ( - ("confirmation", ledger.confirm_accepted_block, stale_confirmation), - ("reactivation", ledger.reactivate_pool_block, stale_reactivation), - ) - for label, operation, target in operations: - try: - operation( - block_hash=str(target["hash"]), - active_tip_height=int(target["height"]), - ) - except Exception as error: - message = str(error) - if "writer lease is not active" not in message: - raise support.GateFailure( - f"C3 stale A {label} wrong failure: {message}" - ) from error - errors[label] = message - else: - raise support.GateFailure(f"C3 stale A {label} unexpectedly succeeded") - after = _c2_database_snapshot(ledger, block_hashes=stale_hashes) - return {"errors": errors, "before": before, "after": after} - - -def _run_c3_primary_a_worker(config: dict[str, object]) -> None: - ledger: _WorkerScopedPsqlLedger | None = None - store: AuditArtifactStore | None = None - try: - ledger = _c2_worker_ledger(config) - store = _c2_worker_store(config) - _emit_worker_event( - "parked", - authority=_c2_authority(store), - pid=os.getpid(), - guard="outside", - ) - _read_worker_command("attempt") - with store.publication_order_guard(): - floor, outcome = _c3_stale_publication_attempt( - ledger=ledger, - store=store, - config=config, - ) - _emit_worker_event( - "stale-publication", - authority=_c2_authority(store), - floor=floor, - outcome=outcome, - latest=store.latest_evidence(), - ) - transition_check = _c3_stale_transition_check(ledger, config) - retention: list[dict[str, int]] = [] - for live_retention in (0, 1): - store.reconfigure(live_bundle_retention=live_retention) - result = store.prune_best_effort() - retention.append( - { - "retention": live_retention, - "live_removed": result.live_removed, - "candidate_removed": result.candidate_removed, - "errors": result.errors, - } - ) - _emit_worker_event( - "post-check", - authority=_c2_authority(store), - transition_check=transition_check, - retention=retention, - latest=store.latest_evidence(), - ) - _read_worker_command("finish") - store.close() - store = None - ledger.close() - ledger = None - _emit_worker_event("closed") - finally: - if store is not None: - store.close() - if ledger is not None: - ledger.close() - - -def _run_c3_late_a_worker(config: dict[str, object]) -> None: - ledger: _WorkerScopedPsqlLedger | None = None - store: AuditArtifactStore | None = None - try: - ledger = _c2_worker_ledger(config) - store = _c2_worker_store(config) - initial = config["initial"] - assert isinstance(initial, dict) - with store.publication_order_guard(): - result = ledger.confirm_accepted_block( - block_hash=str(initial["hash"]), - active_tip_height=int(initial["height"]), - ) - sequence = int(result["audit_publication_sequence"]) - support.assert_equal(sequence, 1, "C3 late A confirmation ordinal N") - floor = ledger.audit_publication_sequence_floor() - support.assert_equal(floor, sequence, "C3 late A confirmed floor N") - _emit_worker_event( - "confirmed", - authority=_c2_authority(store), - pid=os.getpid(), - identity=_c2_worker_identity( - config, - "initial", - sequence=sequence, - ).to_json(), - floor=floor, - ) - _emit_worker_event( - "parked", - authority=_c2_authority(store), - pid=os.getpid(), - guard="outside", - ) - _read_worker_command("attempt") - with store.publication_order_guard(): - floor, outcome = _c3_stale_publication_attempt( - ledger=ledger, - store=store, - config=config, - ) - _emit_worker_event( - "stale-publication", - authority=_c2_authority(store), - floor=floor, - outcome=outcome, - latest=store.latest_evidence(), - ) - _read_worker_command("finish") - store.close() - store = None - ledger.close() - ledger = None - _emit_worker_event("closed") - finally: - if store is not None: - store.close() - if ledger is not None: - ledger.close() - - -def _run_c3_b_worker(config: dict[str, object]) -> None: - ledger: _WorkerScopedPsqlLedger | None = None - store: AuditArtifactStore | None = None - original_flock = audit_artifacts_module.fcntl.flock - try: - ledger = _c2_worker_ledger(config) - store = _c2_worker_store(config) - writer = config["writer"] - assert isinstance(writer, dict) - ledger._run_sql( - f""" -INSERT INTO qbit_a1_c3_attempts (variant, writer_id) -VALUES ('{config['variant']}', '{writer['id']}'); -""" - ) - attempted = False - - def observed_flock(fd: int, operation: int) -> object: - nonlocal attempted - if ( - not attempted - and fd == store._publication_lock_fd # type: ignore[union-attr] - and operation & fcntl.LOCK_EX - ): - attempted = True - _emit_worker_event( - "flock-attempt", - authority=_c2_authority(store), # type: ignore[arg-type] - pid=os.getpid(), - ) - return original_flock(fd, operation) - - audit_artifacts_module.fcntl.flock = observed_flock - try: - with store.publication_order_guard(): - target = config["target"] - assert isinstance(target, dict) - result = ledger.confirm_accepted_block( - block_hash=str(target["hash"]), - active_tip_height=int(target["height"]), - ) - support.assert_equal( - result["confirmed_count"], - 1, - "C3 B confirmation count", - ) - sequence = int(result["audit_publication_sequence"]) - identity = _c2_worker_identity( - config, - "target", - sequence=sequence, - ) - floor = ledger.audit_publication_sequence_floor() - publication = _c2_publish( - store, - identity=identity, - publication_floor_sequence=floor, - created_at=f"c3-{config['variant']}-b-publication", - ) - _emit_worker_event( - "transition-published", - authority=_c2_authority(store), - transition_result=result, - floor=floor, - identity=identity.to_json(), - published=publication.published, - latest=store.latest_evidence(), - ) - finally: - audit_artifacts_module.fcntl.flock = original_flock - if not attempted: - raise support.GateFailure("C3 B did not attempt the publication flock") - _read_worker_command("finish") - released = ledger.release_writer_lease() - support.assert_equal(released, True, "C3 B writer lease release") - store.close() - store = None - ledger.close() - ledger = None - _emit_worker_event("closed", lease_released=released) - finally: - audit_artifacts_module.fcntl.flock = original_flock - if store is not None: - store.close() - if ledger is not None: - try: - ledger.release_writer_lease() - except BaseException: - pass - ledger.close() - - -def _run_c2_worker(config_path: Path) -> int: - try: - raw = config_path.read_bytes() - if len(raw) > support.PSQL_OUTPUT_LIMIT_BYTES: - raise support.GateFailure("process worker configuration exceeded 1 MiB") - config = json.loads(raw) - if not isinstance(config, dict): - raise support.GateFailure("process worker configuration is not an object") - role = config.get("role") - if role == "a": - _run_c2_a_worker(config) - elif role == "b": - _run_c2_b_worker(config) - elif role == "c3-primary-a": - _run_c3_primary_a_worker(config) - elif role == "c3-late-a": - _run_c3_late_a_worker(config) - elif role == "c3-b": - _run_c3_b_worker(config) - else: - raise support.GateFailure(f"invalid C2 worker role: {role!r}") - except BaseException as error: - _emit_worker_event( - "error", - error_type=type(error).__name__, - message=str(error), - ) - traceback.print_exc(file=sys.stderr) - return 1 - return 0 - - -def _start_file_worker( - sql: str, -) -> tuple[subprocess.Popen[str], Any, Any, Any]: - input_file = tempfile.TemporaryFile(mode="w+t", encoding="utf-8") - stdout_file = tempfile.TemporaryFile(mode="w+t", encoding="utf-8") - stderr_file = tempfile.TemporaryFile(mode="w+t", encoding="utf-8") - input_file.write(sql) - input_file.seek(0) - process = subprocess.Popen( - migration._psql_process_command(), - stdin=input_file, - stdout=stdout_file, - stderr=stderr_file, - text=True, - start_new_session=True, - ) - migration._register_process(process) - return process, input_file, stdout_file, stderr_file - - -def _close_file_worker( - worker: tuple[subprocess.Popen[str], Any, Any, Any] | None, -) -> None: - if worker is None: - return - process, input_file, stdout_file, stderr_file = worker - if process.poll() is None: - support._terminate_and_reap(process) - migration._forget_process(process) - for stream in ( - process.stdin, - process.stdout, - process.stderr, - input_file, - stdout_file, - stderr_file, - ): - if stream is not None and not stream.closed: - stream.close() - - -def _wait_success( - worker: tuple[subprocess.Popen[str], Any, Any, Any], -) -> None: - process, _input_file, stdout_file, stderr_file = worker - migration._wait_file_process( - process, - stdout_file=stdout_file, - stderr_file=stderr_file, - expected_success=True, - error_fragment=None, - ) - - -def _wait_holder_success( - holder: subprocess.Popen[str], - *, - stderr_file: Any, - label: str, -) -> None: - try: - holder.wait(timeout=support.PSQL_TIMEOUT_SECONDS) - except subprocess.TimeoutExpired as error: - support._terminate_and_reap(holder) - raise support.GateFailure(f"{label} timed out") from error - finally: - migration._forget_process(holder) - stdout = "" - if holder.stdout is not None: - stdout = holder.stdout.read(support.PSQL_OUTPUT_LIMIT_BYTES + 1) - stderr_file.seek(0) - stderr = stderr_file.read(support.PSQL_OUTPUT_LIMIT_BYTES + 1) - if ( - len(stdout.encode("utf-8")) > support.PSQL_OUTPUT_LIMIT_BYTES - or len(stderr.encode("utf-8")) > support.PSQL_OUTPUT_LIMIT_BYTES - ): - raise support.GateFailure(f"{label} output exceeded 1 MiB") - if holder.returncode != 0: - diagnostics = stderr.strip() or stdout.strip() or "no diagnostic output" - raise support.GateFailure( - f"{label} failed with {holder.returncode}: {diagnostics}" - ) - - -def _prepared_rows_sql(block_hashes: tuple[str, ...]) -> str: - values = ",\n".join( - ( - f"('{block_hash}', {height}, '{'10' * 32}', '{'20' * 32}', " - f"'{'30' * 32}', 'prepared', 'immature')" - ) - for height, block_hash in enumerate(block_hashes, start=10) - ) - return f""" -INSERT INTO qbit_pool_blocks ( - block_hash, block_height, parent_hash, coinbase_txid, - payout_manifest_sha256, chain_state, maturity_state -) VALUES -{values}; -""" - - -def _confirmation_worker_sql( - *, - schema: str, - application_name: str, - round_name: str, - block_hash: str, - height: int, - writer_id: str, - writer_epoch: int, - writer_token: str, - gate_a: int, - gate_b: int, -) -> str: - return f""" -SET application_name = '{application_name}'; -SET statement_timeout = '20s'; -SET lock_timeout = '20s'; -SET search_path TO "{schema}", pg_catalog; -BEGIN; -SELECT pg_catalog.pg_advisory_xact_lock_shared({gate_a}, {gate_b}); -DO $qbit_a1_confirm$ -DECLARE - confirmed_count integer; -BEGIN - SELECT "{schema}".qbit_confirm_pool_block( - '{block_hash}', {height}, '{writer_id}', {writer_epoch}, - '{writer_token}', interval '5 minutes' - ) - INTO confirmed_count; - IF confirmed_count <> 1 THEN - RAISE EXCEPTION 'expected one confirmed pool block, got %', confirmed_count; - END IF; -END; -$qbit_a1_confirm$; -INSERT INTO "{schema}".qbit_a1_confirmation_results ( - round_name, - block_hash, - audit_publication_sequence -) -SELECT - '{round_name}', - block_hash, - audit_publication_sequence -FROM "{schema}".qbit_pool_blocks -WHERE block_hash = '{block_hash}' - AND chain_state = 'confirmed' - AND audit_publication_sequence IS NOT NULL; -COMMIT; -""" - - -def _assert_no_tagged_backends(tags: tuple[str, ...], message: str) -> None: - quoted = ", ".join(f"'{tag}'" for tag in tags) - count = int( - support.run_json( - f""" -SELECT json_build_object('count', count(*)) -FROM pg_catalog.pg_stat_activity -WHERE application_name IN ({quoted}); -""" - )["count"] - ) - support.assert_equal(count, 0, message) - - -def test_database_observed_confirmation_order() -> None: - for round_index, reverse_launch in enumerate((False, True), start=1): - round_name = f"c1-round-{round_index}" - schema = support.create_owned_schema(f"c1_round_{round_index}") - writer_id = f"a1-c1-writer-{round_index}" - writer_token = f"a1-c1-token-{round_index}" - ledger = support.ScopedPsqlLedger( - test_schema=schema, - writer_id=writer_id, - writer_epoch=1, - writer_session_token=writer_token, - initialize_schema=True, - ) - block_hashes = ("91" * 32, "92" * 32) - heights = {block_hashes[0]: 10, block_hashes[1]: 11} - gate_a = 41_000 + round_index - gate_b = 51_000 + round_index - holder_tag = f"qbit_a1_c1_holder_{support.RUN_TOKEN}_{round_index}" - worker_tags = ( - f"qbit_a1_c1_worker_a_{support.RUN_TOKEN}_{round_index}", - f"qbit_a1_c1_worker_b_{support.RUN_TOKEN}_{round_index}", - ) - holder_input = tempfile.TemporaryFile(mode="w+t", encoding="utf-8") - holder_stderr = tempfile.TemporaryFile(mode="w+t", encoding="utf-8") - holder: subprocess.Popen[str] | None = None - workers: list[tuple[subprocess.Popen[str], Any, Any, Any]] = [] - try: - ledger._run_sql( - """ -CREATE TABLE qbit_a1_confirmation_results ( - completion_id bigserial PRIMARY KEY, - round_name text NOT NULL, - block_hash text NOT NULL UNIQUE, - audit_publication_sequence bigint NOT NULL -); -""" - + _prepared_rows_sql(block_hashes) - ) - holder_input.write( - f""" -SET application_name = '{holder_tag}'; -BEGIN; -SELECT pg_catalog.pg_advisory_xact_lock({gate_a}, {gate_b}); -DO $qbit_a1_wait$ -DECLARE - both_waiting boolean := false; -BEGIN - FOR attempt IN 1..600 LOOP - PERFORM pg_catalog.pg_stat_clear_snapshot(); - SELECT count(DISTINCT worker.pid) = 2 - INTO both_waiting - FROM pg_catalog.pg_stat_activity worker - WHERE worker.application_name IN ( - '{worker_tags[0]}', - '{worker_tags[1]}' - ) - AND pg_catalog.pg_backend_pid() = ANY( - pg_catalog.pg_blocking_pids(worker.pid) - ) - AND EXISTS ( - SELECT 1 - FROM pg_catalog.pg_locks lock_state - WHERE lock_state.pid = worker.pid - AND lock_state.locktype = 'advisory' - AND NOT lock_state.granted - ); - EXIT WHEN both_waiting; - PERFORM pg_catalog.pg_sleep(0.05); - END LOOP; - IF NOT both_waiting THEN - RAISE EXCEPTION 'two confirmation advisory waiters were not observed'; - END IF; -END; -$qbit_a1_wait$; -COMMIT; -""" - ) - holder_input.seek(0) - holder = subprocess.Popen( - migration._psql_process_command(), - stdin=holder_input, - stdout=subprocess.DEVNULL, - stderr=holder_stderr, - text=True, - bufsize=1, - start_new_session=True, - ) - migration._register_process(holder) - migration._wait_for_advisory_holder( - holder, - application_name=holder_tag, - deadline=time.monotonic() + support.PSQL_TIMEOUT_SECONDS, - ) - launch = [0, 1] - if reverse_launch: - launch.reverse() - for worker_index in launch: - block_hash = block_hashes[worker_index] - workers.append( - _start_file_worker( - _confirmation_worker_sql( - schema=schema, - application_name=worker_tags[worker_index], - round_name=round_name, - block_hash=block_hash, - height=heights[block_hash], - writer_id=writer_id, - writer_epoch=1, - writer_token=writer_token, - gate_a=gate_a, - gate_b=gate_b, - ) - ) - ) - _wait_holder_success( - holder, - stderr_file=holder_stderr, - label="C1 holder", - ) - for worker in workers: - _wait_success(worker) - results = ledger._run_json( - f""" -SELECT json_build_object( - 'rows', COALESCE(json_agg(json_build_object( - 'completion_id', completion_id, - 'block_hash', block_hash, - 'audit_publication_sequence', audit_publication_sequence - ) ORDER BY completion_id), '[]'::json) -) -FROM qbit_a1_confirmation_results -WHERE round_name = '{round_name}'; -""" - )["rows"] - support.assert_equal(len(results), 2, f"{round_name} completion count") - support.assert_equal( - [row["completion_id"] for row in results], - [1, 2], - f"{round_name} exact completion identifiers", - ) - support.assert_equal( - [row["audit_publication_sequence"] for row in results], - [1, 2], - f"{round_name} completion order matches ordinal order", - ) - support.assert_equal( - {row["block_hash"] for row in results}, - set(block_hashes), - f"{round_name} distinct completed blocks", - ) - before_replay = support.allocator_state(ledger) - for _replay_round in range(3): - for block_hash in block_hashes: - state = ledger.pool_block_state(block_hash=block_hash) - replay = ledger.confirm_accepted_block( - block_hash=block_hash, - active_tip_height=heights[block_hash], - ) - support.assert_equal( - replay["audit_publication_sequence"], - state["audit_publication_sequence"], # type: ignore[index] - f"{round_name} exact replay ordinal", - ) - support.assert_equal( - support.allocator_state(ledger), - before_replay, - f"{round_name} replay allocator immobility", - ) - fresh_hash = "93" * 32 - ledger._run_sql(_prepared_rows_sql((fresh_hash,))) - support.assert_equal( - ledger.confirm_accepted_block( - block_hash=fresh_hash, - active_tip_height=10, - )["audit_publication_sequence"], - 3, - f"{round_name} exact next ordinal", - ) - _assert_no_tagged_backends( - (holder_tag, *worker_tags), - f"{round_name} tagged backend cleanup", - ) - finally: - for worker in workers: - _close_file_worker(worker) - if holder is not None: - if holder.poll() is None: - support._terminate_and_reap(holder) - migration._forget_process(holder) - for stream in (holder.stdin, holder.stdout, holder.stderr): - if stream is not None and not stream.closed: - stream.close() - if not holder_stderr.closed: - holder_stderr.close() - if not holder_input.closed: - holder_input.close() - ledger.release_writer_lease() - ledger.close() - - -def _c2_seed_rows( - ledger: support.ScopedPsqlLedger, - *, - initial: dict[str, object], - target: dict[str, object], - stale_confirmation: dict[str, object], - stale_reactivation: dict[str, object], -) -> None: - rows = ( - (initial, "prepared"), - (target, "prepared"), - (stale_confirmation, "prepared"), - (stale_reactivation, "inactive"), - ) - values = ",\n".join( - ( - f"('{row['hash']}', {int(row['height'])}, '{'10' * 32}', " - f"'{'20' * 32}', '{'30' * 32}', '{chain_state}', 'immature')" - ) - for row, chain_state in rows - ) - ledger._run_sql( - f""" -CREATE TABLE qbit_a1_c2_attempts ( - attempt_id bigserial PRIMARY KEY, - variant text NOT NULL, - transition text NOT NULL, - writer_id text NOT NULL, - attempted_at timestamptz NOT NULL DEFAULT clock_timestamp() -); - -INSERT INTO qbit_pool_blocks ( - block_hash, - block_height, - parent_hash, - coinbase_txid, - payout_manifest_sha256, - chain_state, - maturity_state -) VALUES -{values}; -""" - ) - - -def _c2_attempt_and_lease( - ledger: support.ScopedPsqlLedger, - *, - variant: str, -) -> dict[str, object]: - return ledger._run_json( - f""" -SELECT json_build_object( - 'attempts', COALESCE(( - SELECT json_agg(json_build_object( - 'variant', variant, - 'transition', transition, - 'writer_id', writer_id - ) ORDER BY attempt_id) - FROM qbit_a1_c2_attempts - WHERE variant = '{variant}' - ), '[]'::json), - 'lease', ( - SELECT json_build_object( - 'writer_id', writer_id, - 'writer_epoch', writer_epoch, - 'writer_session_token', writer_session_token - ) - FROM qbit_ledger_writer_lease - WHERE singleton - ) -); -""" - ) - - -def _c2_expire_writer( - ledger: support.ScopedPsqlLedger, - *, - writer_id: str, - writer_epoch: int, - writer_token: str, -) -> None: - expired = ledger._run_json( - f""" -WITH expired AS ( - UPDATE qbit_ledger_writer_lease - SET updated_at = clock_timestamp() - interval '6 minutes', - lease_expires_at = clock_timestamp() - interval '1 minute' - WHERE singleton - AND writer_id = '{writer_id}' - AND writer_epoch = {writer_epoch} - AND writer_session_token = '{writer_token}' - RETURNING writer_id -) -SELECT json_build_object('count', count(*)) -FROM expired; -""" - )["count"] - support.assert_equal(expired, 1, "C2 exact A lease expiry") - - -def _c2_write_config(path: Path, config: dict[str, object]) -> None: - path.write_text( - json.dumps(config, separators=(",", ":"), sort_keys=True), - encoding="utf-8", - ) - - -def _c2_expected_authority(root: Path) -> dict[str, list[int]]: - root_value = root.lstat() - lock_value = (root / ".prism-audit-publication.lock").lstat() - return { - "root": [root_value.st_dev, root_value.st_ino], - "lock": [lock_value.st_dev, lock_value.st_ino], - } - - -def _assert_c2_final_database( - snapshot: dict[str, object], - *, - initial: dict[str, object], - target: dict[str, object], - stale_confirmation: dict[str, object], - stale_reactivation: dict[str, object], - initial_sequence: int, - target_sequence: int, - variant: str, -) -> None: - rows = snapshot["rows"] - assert isinstance(rows, list) - by_hash = {str(row["block_hash"]): row for row in rows} - support.assert_equal( - by_hash[str(initial["hash"])], - { - "block_hash": initial["hash"], - "block_height": initial["height"], - "chain_state": "confirmed", - "maturity_state": "immature", - "audit_publication_sequence": initial_sequence, - }, - f"C2 {variant} initial exact row", - ) - support.assert_equal( - by_hash[str(target["hash"])], - { - "block_hash": target["hash"], - "block_height": target["height"], - "chain_state": "confirmed", - "maturity_state": "immature", - "audit_publication_sequence": target_sequence, - }, - f"C2 {variant} B exact row", - ) - support.assert_equal( - by_hash[str(stale_confirmation["hash"])], - { - "block_hash": stale_confirmation["hash"], - "block_height": stale_confirmation["height"], - "chain_state": "prepared", - "maturity_state": "immature", - "audit_publication_sequence": None, - }, - f"C2 {variant} stale prepared C row", - ) - support.assert_equal( - by_hash[str(stale_reactivation["hash"])], - { - "block_hash": stale_reactivation["hash"], - "block_height": stale_reactivation["height"], - "chain_state": "inactive", - "maturity_state": "immature", - "audit_publication_sequence": None, - }, - f"C2 {variant} stale inactive R row", - ) - support.assert_equal( - snapshot["floor"], - target_sequence, - f"C2 {variant} final floor", - ) - support.assert_equal( - snapshot["allocator"], - {"last_value": target_sequence, "is_called": True}, - f"C2 {variant} exact allocator", - ) - ordinals = sorted( - int(row["audit_publication_sequence"]) - for row in rows - if row["audit_publication_sequence"] is not None - ) - support.assert_equal( - len(ordinals), - len(set(ordinals)), - f"C2 {variant} unique durable ordinals", - ) - - -def _test_c2_a_wins(transition: str) -> None: - if transition not in {"confirmation", "reactivation"}: - raise support.GateFailure(f"invalid C2 transition: {transition!r}") - variant_index = 1 if transition == "confirmation" else 2 - variant = transition - schema = support.create_owned_schema(f"c2_{variant}") - writer_a = { - "id": f"a1-c2-{variant}-writer-a", - "epoch": 1, - "token": f"a1-c2-{variant}-token-a", - } - writer_b = { - "id": f"a1-c2-{variant}-writer-b", - "epoch": 2, - "token": f"a1-c2-{variant}-token-b", - } - initial = {"hash": f"a{variant_index}" * 32, "height": 10} - target = {"hash": f"b{variant_index}" * 32, "height": 20} - stale_confirmation = {"hash": f"c{variant_index}" * 32, "height": 30} - stale_reactivation = {"hash": f"d{variant_index}" * 32, "height": 31} - all_hashes = tuple( - str(value["hash"]) - for value in ( - initial, - target, - stale_confirmation, - stale_reactivation, - ) - ) - ledger = support.ScopedPsqlLedger( - test_schema=schema, - writer_id=str(writer_a["id"]), - writer_epoch=int(writer_a["epoch"]), - writer_session_token=str(writer_a["token"]), - initialize_schema=True, - ) - worker_a: _JsonWorker | None = None - worker_b: _JsonWorker | None = None - setup_store: AuditArtifactStore | None = None - try: - _c2_seed_rows( - ledger, - initial=initial, - target=target, - stale_confirmation=stale_confirmation, - stale_reactivation=stale_reactivation, - ) - initial_sequence = 1 - if transition == "reactivation": - target_prior = ledger.confirm_accepted_block( - block_hash=str(target["hash"]), - active_tip_height=int(target["height"]), - ) - support.assert_equal( - target_prior["audit_publication_sequence"], - 1, - "C2 reactivation target prior confirmation ordinal", - ) - support.assert_equal( - ledger.mark_pool_block_inactive( - block_hash=str(target["hash"]), - active_tip_height=int(target["height"]), - )["inactive_count"], - 1, - "C2 reactivation target prior inactive transition", - ) - initial_sequence = 2 - initial_result = ledger.confirm_accepted_block( - block_hash=str(initial["hash"]), - active_tip_height=int(initial["height"]), - ) - support.assert_equal( - initial_result["audit_publication_sequence"], - initial_sequence, - f"C2 {variant} initial ordinal", - ) - support.assert_equal( - ledger.audit_publication_sequence_floor(), - initial_sequence, - f"C2 {variant} initial floor", - ) - support.assert_equal( - support.allocator_state(ledger), - {"last_value": initial_sequence, "is_called": True}, - f"C2 {variant} initial allocator", - ) - target_sequence = initial_sequence + 1 - with tempfile.TemporaryDirectory() as tmp, ExitStack() as temp_cleanup: - base = Path(tmp) - root = base / "audit" - evidence_path = base / "state" / "evidence.json" - setup_store = _c2_store(root, evidence_path) - temp_cleanup.callback(setup_store.close) - initial_identity = AuditPublicationIdentity( - initial_sequence, - int(initial["height"]), - str(initial["hash"]), - ) - with setup_store.publication_order_guard(): - initial_publication = _c2_publish( - setup_store, - identity=initial_identity, - publication_floor_sequence=initial_sequence, - created_at=f"c2-{variant}-initial", - ) - support.assert_equal( - initial_publication.published, # type: ignore[attr-defined] - True, - f"C2 {variant} initial publication", - ) - initial_envelope = setup_store.live_envelope_path( - block_height=int(initial["height"]), - block_hash=str(initial["hash"]), - ) - setup_store.close() - setup_store = None - evidence_path.write_bytes(b"{c2-damaged-evidence") - expected_authority = _c2_expected_authority(root) - common_config: dict[str, object] = { - "schema": schema, - "root": str(root), - "evidence_path": str(evidence_path), - "variant": variant, - "transition": transition, - "initial_sequence": initial_sequence, - "initial": initial, - "target": target, - "stale_confirmation": stale_confirmation, - "stale_reactivation": stale_reactivation, - } - config_a = base / "worker-a.json" - config_b = base / "worker-b.json" - tag_suffix = f"{variant[0]}_{support.RUN_TOKEN[:8]}" - tag_a = f"qbit_a1_c2_a_{tag_suffix}" - tag_b = f"qbit_a1_c2_b_{tag_suffix}" - _c2_write_config( - config_a, - { - **common_config, - "role": "a", - "writer": writer_a, - "application_name": tag_a, - }, - ) - _c2_write_config( - config_b, - { - **common_config, - "role": "b", - "writer": writer_b, - "application_name": tag_b, - }, - ) - worker_a = _JsonWorker(config_a) - temp_cleanup.callback(worker_a.close) - a_guard = worker_a.read_event("guard-acquired") - support.assert_equal( - a_guard["authority"], - expected_authority, - f"C2 {variant} A exact root/lock inode", - ) - support.assert_equal( - a_guard["floor"], - initial_sequence, - f"C2 {variant} A floor N", - ) - blocked_filesystem = _c2_filesystem_snapshot( - root=root, - evidence_path=evidence_path, - ) - blocked_database = _c2_database_snapshot( - ledger, - block_hashes=all_hashes, - ) - _c2_expire_writer( - ledger, - writer_id=str(writer_a["id"]), - writer_epoch=int(writer_a["epoch"]), - writer_token=str(writer_a["token"]), - ) - worker_b = _JsonWorker(config_b) - temp_cleanup.callback(worker_b.close) - b_attempt = worker_b.read_event("flock-attempt") - support.assert_equal( - b_attempt["authority"], - expected_authority, - f"C2 {variant} B exact root/lock inode", - ) - support.assert_equal( - worker_b.process.poll(), - None, - f"C2 {variant} B remains alive while flock-blocked", - ) - child_pids = {int(a_guard["pid"]), int(b_attempt["pid"])} - support.assert_equal( - len(child_pids), - 2, - f"C2 {variant} distinct A/B OS processes", - ) - if os.getpid() in child_pids: - raise support.GateFailure( - f"C2 {variant} worker reused the parent process" - ) - support.assert_equal( - _c2_attempt_and_lease(ledger, variant=variant), - { - "attempts": [ - { - "variant": variant, - "transition": transition, - "writer_id": writer_b["id"], - } - ], - "lease": { - "writer_id": writer_b["id"], - "writer_epoch": writer_b["epoch"], - "writer_session_token": writer_b["token"], - }, - }, - f"C2 {variant} DB-visible B attempt and replacement lease", - ) - support.assert_equal( - _c2_database_snapshot(ledger, block_hashes=all_hashes), - blocked_database, - f"C2 {variant} B row/floor/allocator unchanged while blocked", - ) - support.assert_equal( - _c2_filesystem_snapshot(root=root, evidence_path=evidence_path), - blocked_filesystem, - f"C2 {variant} filesystem unchanged while B blocked", - ) - worker_a.send("repair") - repaired = worker_a.read_event("repaired") - support.assert_equal( - repaired["published"], - True, - f"C2 {variant} A exact repair published", - ) - support.assert_equal( - repaired["identity"], - initial_identity.to_json(), - f"C2 {variant} A repair identity N", - ) - support.assert_equal( - repaired["authority"], - expected_authority, - f"C2 {variant} A repair authority", - ) - released = worker_a.read_event("guard-released") - support.assert_equal( - released["authority"], - expected_authority, - f"C2 {variant} A guard release authority", - ) - b_done = worker_b.read_event("transition-published") - expected_b_identity = AuditPublicationIdentity( - target_sequence, - int(target["height"]), - str(target["hash"]), - ) - support.assert_equal( - b_done["identity"], - expected_b_identity.to_json(), - f"C2 {variant} B identity N+1", - ) - support.assert_equal( - b_done["floor"], - target_sequence, - f"C2 {variant} B fresh floor", - ) - support.assert_equal( - b_done["published"], - True, - f"C2 {variant} B publication", - ) - support.assert_equal( - b_done["authority"], - expected_authority, - f"C2 {variant} B publication authority", - ) - final_database = _c2_database_snapshot( - ledger, - block_hashes=all_hashes, - ) - _assert_c2_final_database( - final_database, - initial=initial, - target=target, - stale_confirmation=stale_confirmation, - stale_reactivation=stale_reactivation, - initial_sequence=initial_sequence, - target_sequence=target_sequence, - variant=variant, - ) - latest = json.loads(evidence_path.read_text(encoding="utf-8")) - support.assert_equal( - latest["audit_publication_identity"], - expected_b_identity.to_json(), - f"C2 {variant} final evidence B/N+1", - ) - support.assert_equal( - latest["block_hash"], - target["hash"], - f"C2 {variant} final evidence B hash", - ) - target_envelope = root / ( - f"prism-live-audit-bundle-{target['height']}-{target['hash']}.json" - ) - support.assert_equal( - initial_envelope.exists(), - False, - f"C2 {variant} retention removes unpinned A envelope", - ) - support.assert_equal( - target_envelope.exists(), - True, - f"C2 {variant} retention keeps B envelope", - ) - support.assert_equal( - sorted(path.name for path in root.iterdir()), - [ - ".prism-audit-publication.lock", - target_envelope.name, - ], - f"C2 {variant} exact retained root entries", - ) - before_stale_database = _c2_database_snapshot( - ledger, - block_hashes=all_hashes, - ) - before_stale_filesystem = _c2_filesystem_snapshot( - root=root, - evidence_path=evidence_path, - ) - worker_a.send("stale-check") - stale_result = worker_a.read_event("stale-result") - stale_errors = stale_result["errors"] - assert isinstance(stale_errors, dict) - support.assert_equal( - set(stale_errors), - {"confirmation", "reactivation"}, - f"C2 {variant} exact stale A failure labels", - ) - for label in ("confirmation", "reactivation"): - if "writer lease is not active" not in str(stale_errors[label]): - raise support.GateFailure( - f"C2 {variant} stale A {label} wrong error: " - f"{stale_errors[label]!r}" - ) - support.assert_equal( - stale_result["before"], - stale_result["after"], - f"C2 {variant} child-observed stale transition immobility", - ) - support.assert_equal( - stale_result["authority"], - expected_authority, - f"C2 {variant} stale A authority", - ) - support.assert_equal( - _c2_database_snapshot(ledger, block_hashes=all_hashes), - before_stale_database, - f"C2 {variant} stale A row/floor/allocator immobility", - ) - support.assert_equal( - _c2_filesystem_snapshot(root=root, evidence_path=evidence_path), - before_stale_filesystem, - f"C2 {variant} stale A filesystem immobility", - ) - worker_a.send("finish") - worker_a.read_event("closed") - worker_a.wait_success() - worker_b.send("finish") - b_closed = worker_b.read_event("closed") - support.assert_equal( - b_closed["lease_released"], - True, - f"C2 {variant} B lease released", - ) - worker_b.wait_success() - _assert_no_tagged_backends( - (tag_a, tag_b), - f"C2 {variant} tagged child backend cleanup", - ) - finally: - if setup_store is not None: - setup_store.close() - if worker_a is not None: - worker_a.close() - if worker_b is not None: - worker_b.close() - ledger.close() - - -def test_c2_a_wins_confirmation_publication() -> None: - _test_c2_a_wins("confirmation") - - -def _c3_seed_rows( - ledger: support.ScopedPsqlLedger, - *, - initial: dict[str, object], - target: dict[str, object], - stale_confirmation: dict[str, object], - stale_reactivation: dict[str, object], -) -> None: - rows = ( - (initial, "prepared"), - (target, "prepared"), - (stale_confirmation, "prepared"), - (stale_reactivation, "inactive"), - ) - values = ",\n".join( - ( - f"('{row['hash']}', {int(row['height'])}, '{'10' * 32}', " - f"'{'20' * 32}', '{'30' * 32}', '{chain_state}', 'immature')" - ) - for row, chain_state in rows - ) - ledger._run_sql( - f""" -CREATE TABLE qbit_a1_c3_attempts ( - attempt_id bigserial PRIMARY KEY, - variant text NOT NULL, - writer_id text NOT NULL, - attempted_at timestamptz NOT NULL DEFAULT clock_timestamp() -); - -INSERT INTO qbit_pool_blocks ( - block_hash, - block_height, - parent_hash, - coinbase_txid, - payout_manifest_sha256, - chain_state, - maturity_state -) VALUES -{values}; -""" - ) - - -def _c3_attempt_and_lease( - ledger: support.ScopedPsqlLedger, - *, - variant: str, -) -> dict[str, object]: - return ledger._run_json( - f""" -SELECT json_build_object( - 'attempts', COALESCE(( - SELECT json_agg(json_build_object( - 'variant', variant, - 'writer_id', writer_id - ) ORDER BY attempt_id) - FROM qbit_a1_c3_attempts - WHERE variant = '{variant}' - ), '[]'::json), - 'lease', ( - SELECT json_build_object( - 'writer_id', writer_id, - 'writer_epoch', writer_epoch, - 'writer_session_token', writer_session_token - ) - FROM qbit_ledger_writer_lease - WHERE singleton - ) -); -""" - ) - - -def _c3_expected_database( - *, - initial: dict[str, object], - target: dict[str, object], - stale_confirmation: dict[str, object], - stale_reactivation: dict[str, object], -) -> dict[str, object]: - return { - "rows": [ - { - "block_hash": initial["hash"], - "block_height": initial["height"], - "chain_state": "confirmed", - "maturity_state": "immature", - "audit_publication_sequence": 1, - }, - { - "block_hash": target["hash"], - "block_height": target["height"], - "chain_state": "confirmed", - "maturity_state": "immature", - "audit_publication_sequence": 2, - }, - { - "block_hash": stale_confirmation["hash"], - "block_height": stale_confirmation["height"], - "chain_state": "prepared", - "maturity_state": "immature", - "audit_publication_sequence": None, - }, - { - "block_hash": stale_reactivation["hash"], - "block_height": stale_reactivation["height"], - "chain_state": "inactive", - "maturity_state": "immature", - "audit_publication_sequence": None, - }, - ], - "floor": 2, - "allocator": {"last_value": 2, "is_called": True}, - } - - -def _assert_c3_stale_outcome(outcome: object, *, label: str) -> None: - if not isinstance(outcome, dict): - raise support.GateFailure(f"{label} outcome is not an object") - if outcome.get("kind") == "returned": - support.assert_equal( - outcome.get("published"), - False, - f"{label} returns published false", - ) - return - if outcome.get("kind") == "behind-error" and "behind" in str( - outcome.get("message") - ): - return - raise support.GateFailure(f"{label} unexpected outcome: {outcome!r}") - - -def _test_c3_interleaving(variant: str) -> None: - if variant not in {"primary", "late"}: - raise support.GateFailure(f"invalid C3 variant: {variant!r}") - variant_index = 3 if variant == "primary" else 4 - schema = support.create_owned_schema(f"c3_{variant}") - writer_a = { - "id": f"a1-c3-{variant}-writer-a", - "epoch": 1, - "token": f"a1-c3-{variant}-token-a", - } - writer_b = { - "id": f"a1-c3-{variant}-writer-b", - "epoch": 2, - "token": f"a1-c3-{variant}-token-b", - } - initial = {"hash": f"a{variant_index}" * 32, "height": 10} - target = {"hash": f"b{variant_index}" * 32, "height": 20} - stale_confirmation = {"hash": f"c{variant_index}" * 32, "height": 30} - stale_reactivation = {"hash": f"d{variant_index}" * 32, "height": 31} - all_hashes = tuple( - str(value["hash"]) - for value in ( - initial, - target, - stale_confirmation, - stale_reactivation, - ) - ) - ledger = support.ScopedPsqlLedger( - test_schema=schema, - writer_id=str(writer_a["id"]), - writer_epoch=int(writer_a["epoch"]), - writer_session_token=str(writer_a["token"]), - initialize_schema=True, - ) - worker_a: _JsonWorker | None = None - worker_b: _JsonWorker | None = None - setup_store: AuditArtifactStore | None = None - try: - _c3_seed_rows( - ledger, - initial=initial, - target=target, - stale_confirmation=stale_confirmation, - stale_reactivation=stale_reactivation, - ) - if variant == "primary": - initial_result = ledger.confirm_accepted_block( - block_hash=str(initial["hash"]), - active_tip_height=int(initial["height"]), - ) - support.assert_equal( - initial_result["audit_publication_sequence"], - 1, - "C3 primary initial ordinal N", - ) - with tempfile.TemporaryDirectory() as tmp, ExitStack() as temp_cleanup: - base = Path(tmp) - root = base / "audit" - evidence_path = base / "state" / "evidence.json" - setup_store = _c2_store(root, evidence_path) - temp_cleanup.callback(setup_store.close) - initial_identity = AuditPublicationIdentity( - 1, - int(initial["height"]), - str(initial["hash"]), - ) - initial_envelope = setup_store.live_envelope_path( - block_height=int(initial["height"]), - block_hash=str(initial["hash"]), - ) - if variant == "primary": - with setup_store.publication_order_guard(): - initial_publication = _c2_publish( - setup_store, - identity=initial_identity, - publication_floor_sequence=1, - created_at="c3-primary-initial", - ) - support.assert_equal( - initial_publication.published, - True, - "C3 primary initial publication N", - ) - setup_store.close() - setup_store = None - if variant == "primary": - initial_envelope.unlink() - support.assert_equal( - evidence_path.exists(), - True, - "C3 primary damaged N retains evidence", - ) - support.assert_equal( - initial_envelope.exists(), - False, - f"C3 {variant} N envelope starts absent", - ) - expected_authority = _c2_expected_authority(root) - tag_suffix = f"{variant[0]}_{support.RUN_TOKEN[:8]}" - tag_a = f"qbit_a1_c3_a_{tag_suffix}" - tag_b = f"qbit_a1_c3_b_{tag_suffix}" - common_config: dict[str, object] = { - "schema": schema, - "root": str(root), - "evidence_path": str(evidence_path), - "variant": variant, - "transition": "confirmation", - "initial_sequence": 1, - "initial": initial, - "target": target, - "stale_confirmation": stale_confirmation, - "stale_reactivation": stale_reactivation, - } - config_a = base / "worker-a.json" - config_b = base / "worker-b.json" - _c2_write_config( - config_a, - { - **common_config, - "role": ( - "c3-primary-a" if variant == "primary" else "c3-late-a" - ), - "writer": writer_a, - "application_name": tag_a, - }, - ) - _c2_write_config( - config_b, - { - **common_config, - "role": "c3-b", - "writer": writer_b, - "application_name": tag_b, - }, - ) - worker_a = _JsonWorker(config_a) - temp_cleanup.callback(worker_a.close) - if variant == "late": - confirmed = worker_a.read_event("confirmed") - support.assert_equal( - confirmed["authority"], - expected_authority, - "C3 late A confirmation guard authority", - ) - support.assert_equal( - confirmed["identity"], - initial_identity.to_json(), - "C3 late A confirmed identity N", - ) - support.assert_equal( - confirmed["floor"], - 1, - "C3 late A confirmed floor N", - ) - parked = worker_a.read_event("parked") - support.assert_equal( - parked["authority"], - expected_authority, - f"C3 {variant} A exact root/lock inode", - ) - support.assert_equal( - parked["guard"], - "outside", - f"C3 {variant} A parks outside guard", - ) - if variant == "late": - support.assert_equal( - parked["pid"], - confirmed["pid"], - "C3 late A exits guard in same child", - ) - support.assert_equal( - worker_a.process.poll(), - None, - f"C3 {variant} parked A remains alive", - ) - initial_database = _c2_database_snapshot( - ledger, - block_hashes=all_hashes, - ) - support.assert_equal( - initial_database["floor"], - 1, - f"C3 {variant} durable floor N before replacement", - ) - support.assert_equal( - initial_database["allocator"], - {"last_value": 1, "is_called": True}, - f"C3 {variant} allocator N before replacement", - ) - support.assert_equal( - initial_database["rows"], - [ - { - "block_hash": initial["hash"], - "block_height": initial["height"], - "chain_state": "confirmed", - "maturity_state": "immature", - "audit_publication_sequence": 1, - }, - { - "block_hash": target["hash"], - "block_height": target["height"], - "chain_state": "prepared", - "maturity_state": "immature", - "audit_publication_sequence": None, - }, - { - "block_hash": stale_confirmation["hash"], - "block_height": stale_confirmation["height"], - "chain_state": "prepared", - "maturity_state": "immature", - "audit_publication_sequence": None, - }, - { - "block_hash": stale_reactivation["hash"], - "block_height": stale_reactivation["height"], - "chain_state": "inactive", - "maturity_state": "immature", - "audit_publication_sequence": None, - }, - ], - f"C3 {variant} exact rows before replacement", - ) - damaged_snapshot = _c2_filesystem_snapshot( - root=root, - evidence_path=evidence_path, - ) - if variant == "primary": - damaged_evidence = json.loads( - evidence_path.read_text(encoding="utf-8") - ) - support.assert_equal( - damaged_evidence["audit_publication_identity"], - initial_identity.to_json(), - "C3 primary damaged evidence still names N", - ) - _c2_expire_writer( - ledger, - writer_id=str(writer_a["id"]), - writer_epoch=int(writer_a["epoch"]), - writer_token=str(writer_a["token"]), - ) - worker_b = _JsonWorker(config_b) - temp_cleanup.callback(worker_b.close) - b_attempt = worker_b.read_event("flock-attempt") - support.assert_equal( - b_attempt["authority"], - expected_authority, - f"C3 {variant} B exact root/lock inode", - ) - child_pids = {int(parked["pid"]), int(b_attempt["pid"])} - support.assert_equal( - len(child_pids), - 2, - f"C3 {variant} distinct A/B OS processes", - ) - if os.getpid() in child_pids: - raise support.GateFailure( - f"C3 {variant} worker reused the parent process" - ) - b_done = worker_b.read_event("transition-published") - expected_b_identity = AuditPublicationIdentity( - 2, - int(target["height"]), - str(target["hash"]), - ) - support.assert_equal( - b_done["authority"], - expected_authority, - f"C3 {variant} B publication authority", - ) - support.assert_equal( - b_done["identity"], - expected_b_identity.to_json(), - f"C3 {variant} B identity N+1", - ) - support.assert_equal(b_done["floor"], 2, f"C3 {variant} B floor N+1") - support.assert_equal( - b_done["published"], - True, - f"C3 {variant} B publishes N+1", - ) - support.assert_equal( - _c3_attempt_and_lease(ledger, variant=variant), - { - "attempts": [ - {"variant": variant, "writer_id": writer_b["id"]} - ], - "lease": { - "writer_id": writer_b["id"], - "writer_epoch": writer_b["epoch"], - "writer_session_token": writer_b["token"], - }, - }, - f"C3 {variant} B attempt and active replacement lease", - ) - expected_database = _c3_expected_database( - initial=initial, - target=target, - stale_confirmation=stale_confirmation, - stale_reactivation=stale_reactivation, - ) - support.assert_equal( - _c2_database_snapshot(ledger, block_hashes=all_hashes), - expected_database, - f"C3 {variant} exact database after B", - ) - target_envelope = root / ( - f"prism-live-audit-bundle-{target['height']}-{target['hash']}.json" - ) - b_evidence_bytes = evidence_path.read_bytes() - b_envelope_bytes = target_envelope.read_bytes() - durable_b_evidence = json.loads(b_evidence_bytes) - durable_b_envelope = json.loads(b_envelope_bytes) - support.assert_equal( - durable_b_evidence["audit_publication_identity"], - expected_b_identity.to_json(), - f"C3 {variant} durable B evidence identity N+1", - ) - support.assert_equal( - durable_b_evidence["block_hash"], - target["hash"], - f"C3 {variant} durable B evidence hash", - ) - support.assert_equal( - { - "block_hash": durable_b_envelope["block_hash"], - "block_height": durable_b_envelope["block_height"], - "audit_bundle_sha256": durable_b_envelope[ - "audit_bundle_sha256" - ], - }, - { - "block_hash": target["hash"], - "block_height": target["height"], - "audit_bundle_sha256": _C2_DIGEST, - }, - f"C3 {variant} durable B envelope identity", - ) - b_evidence_stat = evidence_path.lstat() - b_envelope_stat = target_envelope.lstat() - b_filesystem = _c2_filesystem_snapshot( - root=root, - evidence_path=evidence_path, - ) - support.assert_equal( - sorted(path.name for path in root.iterdir()), - [".prism-audit-publication.lock", target_envelope.name], - f"C3 {variant} exact B root entries", - ) - support.assert_equal( - initial_envelope.exists(), - False, - f"C3 {variant} N remains absent after B", - ) - worker_a.send("attempt") - stale_publication = worker_a.read_event("stale-publication") - support.assert_equal( - stale_publication["authority"], - expected_authority, - f"C3 {variant} late A same lock inode", - ) - support.assert_equal( - stale_publication["floor"], - 2, - f"C3 {variant} late A reads fresh N+1 floor", - ) - _assert_c3_stale_outcome( - stale_publication["outcome"], - label=f"C3 {variant} stale N publication", - ) - stale_latest = stale_publication["latest"] - assert isinstance(stale_latest, dict) - support.assert_equal( - stale_latest["audit_publication_identity"], - expected_b_identity.to_json(), - f"C3 {variant} stale A reconciles B evidence", - ) - if variant == "primary": - post_check = worker_a.read_event("post-check") - transition_check = post_check["transition_check"] - assert isinstance(transition_check, dict) - errors = transition_check["errors"] - assert isinstance(errors, dict) - support.assert_equal( - set(errors), - {"confirmation", "reactivation"}, - "C3 primary exact stale lease-failure labels", - ) - for label in ("confirmation", "reactivation"): - if "writer lease is not active" not in str(errors[label]): - raise support.GateFailure( - f"C3 primary stale {label} wrong error: {errors[label]!r}" - ) - support.assert_equal( - transition_check["before"], - transition_check["after"], - "C3 primary child-observed stale transition immobility", - ) - support.assert_equal( - post_check["retention"], - [ - { - "retention": 0, - "live_removed": 0, - "candidate_removed": 0, - "errors": 0, - }, - { - "retention": 1, - "live_removed": 0, - "candidate_removed": 0, - "errors": 0, - }, - ], - "C3 primary stale A retention 0/1 pins B", - ) - support.assert_equal( - post_check["authority"], - expected_authority, - "C3 primary post-check authority", - ) - support.assert_equal( - _c2_database_snapshot(ledger, block_hashes=all_hashes), - expected_database, - f"C3 {variant} late A row/floor/allocator immobility", - ) - support.assert_equal( - _c3_attempt_and_lease(ledger, variant=variant)["lease"], - { - "writer_id": writer_b["id"], - "writer_epoch": writer_b["epoch"], - "writer_session_token": writer_b["token"], - }, - f"C3 {variant} B lease remains active through A checks", - ) - support.assert_equal( - _c2_filesystem_snapshot(root=root, evidence_path=evidence_path), - b_filesystem, - f"C3 {variant} exact B filesystem survives late A", - ) - support.assert_equal( - evidence_path.read_bytes(), - b_evidence_bytes, - f"C3 {variant} B evidence bytes unchanged", - ) - support.assert_equal( - target_envelope.read_bytes(), - b_envelope_bytes, - f"C3 {variant} B envelope bytes unchanged", - ) - evidence_after = evidence_path.lstat() - envelope_after = target_envelope.lstat() - support.assert_equal( - (evidence_after.st_dev, evidence_after.st_ino), - (b_evidence_stat.st_dev, b_evidence_stat.st_ino), - f"C3 {variant} B evidence inode unchanged", - ) - support.assert_equal( - (envelope_after.st_dev, envelope_after.st_ino), - (b_envelope_stat.st_dev, b_envelope_stat.st_ino), - f"C3 {variant} B envelope inode unchanged", - ) - support.assert_equal( - initial_envelope.exists(), - False, - f"C3 {variant} stale A never repairs N", - ) - if variant == "primary": - damaged_entries = damaged_snapshot["entries"] - assert isinstance(damaged_entries, dict) - support.assert_equal( - sorted(damaged_entries), - [".prism-audit-publication.lock"], - "C3 primary damaged snapshot has no N envelope", - ) - worker_a.send("finish") - worker_a.read_event("closed") - worker_a.wait_success() - worker_b.send("finish") - b_closed = worker_b.read_event("closed") - support.assert_equal( - b_closed["lease_released"], - True, - f"C3 {variant} B lease release", - ) - worker_b.wait_success() - _assert_no_tagged_backends( - (tag_a, tag_b), - f"C3 {variant} tagged child backend cleanup", - ) - with support.ACTIVE_CHILDREN_LOCK: - support.assert_equal( - len(support.ACTIVE_CHILDREN), - 0, - f"C3 {variant} active child registry cleanup", - ) - finally: - if setup_store is not None: - setup_store.close() - if worker_a is not None: - worker_a.close() - if worker_b is not None: - worker_b.close() - ledger.close() - - -def test_c3_b_wins_and_late_a_interleavings() -> None: - for variant in ("primary", "late"): - _test_c3_interleaving(variant) - - -def server_evidence() -> dict[str, object]: - evidence = support.run_json( - """ -SELECT json_build_object( - 'server_version', current_setting('server_version'), - 'server_version_num', current_setting('server_version_num') -); -""" - ) - configured_image = os.environ.get("QBIT_PRISM_GATE_IMAGE", "").strip() - provisioned_image_digest = os.environ.get( - "QBIT_PRISM_GATE_IMAGE_DIGEST", - "", - ).strip() - if not configured_image or configured_image.casefold() == "unreported": - raise support.GateFailure("configured PostgreSQL image evidence is required") - if ( - not provisioned_image_digest - or provisioned_image_digest.casefold() == "unreported" - ): - raise support.GateFailure( - "provisioned PostgreSQL image digest evidence is required" - ) - evidence["configured_image"] = configured_image - evidence["provisioned_image_digest"] = provisioned_image_digest - return evidence - - -def main() -> None: - public_before = support.public_sentinel() - failure: BaseException | None = None - try: - test_database_observed_confirmation_order() - test_c2_a_wins_confirmation_publication() - test_c3_b_wins_and_late_a_interleavings() - except BaseException as error: - failure = error - try: - support.cleanup_active_children() - support.cleanup_owned_schemas() - support.assert_equal( - support.marker_schema_count(), - 0, - "process gate marker cleanup", - ) - support.assert_equal( - support.public_sentinel(), - public_before, - "process gate public preservation", - ) - except BaseException as cleanup_error: - if failure is None: - raise - raise support.GateFailure( - f"process scenario failed with {failure!r}; cleanup also failed " - f"with {cleanup_error!r}" - ) from cleanup_error - else: - support.atexit.unregister(support.cleanup_active_children) - support.atexit.unregister(support.cleanup_owned_schemas) - if failure is not None: - raise failure - print("prism postgres A1 process gate evidence " + json.dumps(server_evidence())) - print( - "prism postgres A1 process gate PASS " - "C1-confirmation-order C2-A-wins-confirmation-publication " - "C3-B-wins-late-A" - ) - - -if __name__ == "__main__": - if len(sys.argv) == 3 and sys.argv[1] == "--c2-worker": - raise SystemExit(_run_c2_worker(Path(sys.argv[2]))) - if len(sys.argv) != 1: - raise SystemExit("unexpected process-gate arguments") - main() diff --git a/tests/prism_postgres_a1_revert_gate.py b/tests/prism_postgres_a1_revert_gate.py deleted file mode 100644 index 8a220d747..000000000 --- a/tests/prism_postgres_a1_revert_gate.py +++ /dev/null @@ -1,548 +0,0 @@ -"""Non-discovered PostgreSQL ordinal schema-revert integration gate. - -Round-trips crates/qbit-prism/sql/001_share_ledger_revert_audit_publication_sequence.sql: -forward-migrate a schema holding confirmed rows, revert it to its -pre-ordinal shape, confirm blocks pre-ordinal-style against the reverted -schema, then re-apply the forward migration and require a correct backfill -with no double-assigned ordinals. Invoked by -``test/test-prism-postgres-ledger.sh``. -""" - -from __future__ import annotations - -from pathlib import Path - -from tests import prism_postgres_a1_gate as support -from tests.prism_postgres_a1_gate import ( - GateFailure, - LEGACY_POOL_BLOCKS_SQL, - ScopedPsqlLedger, - assert_equal, - create_owned_schema, - run_json, - run_psql, -) - -REVERT_SCRIPT_PATH = ( - Path(__file__).resolve().parents[1] - / "crates/qbit-prism/sql/001_share_ledger_revert_audit_publication_sequence.sql" -) -REVERT_SCRIPT_SQL = REVERT_SCRIPT_PATH.read_text(encoding="utf-8") - -REVERT_HASH_A = "91" * 32 -REVERT_HASH_B = "92" * 32 -REVERT_HASH_C = "93" * 32 -REVERT_HASH_INACTIVE = "94" * 32 -REVERT_HASH_REVERSED = "95" * 32 -REVERT_HASH_PREPARED = "96" * 32 -REVERT_HASH_LEGACY_UPDATE = "97" * 32 -REVERT_HASH_POST_REAPPLY = "98" * 32 - - -def apply_revert(schema: str) -> None: - run_psql(REVERT_SCRIPT_SQL, schema=schema) - - -def seed_legacy_pool_blocks(schema: str) -> None: - run_psql(LEGACY_POOL_BLOCKS_SQL, schema=schema) - run_psql( - f""" -INSERT INTO qbit_pool_blocks ( - block_hash, block_height, parent_hash, coinbase_txid, - payout_manifest_sha256, found_at, chain_state, maturity_state, - matured_at, disconnected_at -) VALUES - -- Insertion order intentionally differs from deterministic backfill - -- order. A/B share a timestamp and sort by block hash. - ('{REVERT_HASH_C}', 10, '{'10' * 32}', '{'20' * 32}', '{'30' * 32}', - '2020-01-01T00:00:02Z', 'confirmed', 'immature', NULL, NULL), - ('{REVERT_HASH_B}', 10, '{'10' * 32}', '{'20' * 32}', '{'30' * 32}', - '2020-01-01T00:00:01Z', 'confirmed', 'immature', NULL, NULL), - ('{REVERT_HASH_A}', 10, '{'10' * 32}', '{'20' * 32}', '{'30' * 32}', - '2020-01-01T00:00:01Z', 'confirmed', 'immature', NULL, NULL), - ('{REVERT_HASH_INACTIVE}', 10, '{'10' * 32}', '{'20' * 32}', '{'30' * 32}', - '2020-01-01T00:00:03Z', 'inactive', 'immature', NULL, NULL), - ('{REVERT_HASH_REVERSED}', 10, '{'10' * 32}', '{'20' * 32}', '{'30' * 32}', - '2020-01-01T00:00:03Z', 'reversed', 'reversed', NULL, - '2020-01-02T00:00:00Z'), - ('{REVERT_HASH_PREPARED}', 10, '{'10' * 32}', '{'20' * 32}', '{'30' * 32}', - '2020-01-01T00:00:04Z', 'prepared', 'immature', NULL, NULL); -""", - schema=schema, - ) - - -def insert_prepared_block(schema: str, block_hash: str, found_at: str) -> None: - run_psql( - f""" -INSERT INTO qbit_pool_blocks ( - block_hash, block_height, parent_hash, coinbase_txid, - payout_manifest_sha256, found_at, chain_state, maturity_state -) VALUES ( - '{block_hash}', 10, '{'10' * 32}', '{'20' * 32}', '{'30' * 32}', - '{found_at}', 'prepared', 'immature' -); -""", - schema=schema, - ) - - -def ordinal_catalog(schema: str) -> dict[str, object]: - return run_json( - """ -SELECT json_build_object( - 'column_present', EXISTS ( - SELECT 1 - FROM pg_attribute - WHERE attrelid = to_regclass('qbit_pool_blocks') - AND attname = 'audit_publication_sequence' - AND attnum > 0 - AND NOT attisdropped - ), - 'sequence_present', - to_regclass('qbit_audit_publication_sequence_seq') IS NOT NULL, - 'index_present', - to_regclass('qbit_pool_blocks_audit_publication_sequence_idx') - IS NOT NULL, - 'constraint_present', EXISTS ( - SELECT 1 - FROM pg_constraint - WHERE conrelid = to_regclass('qbit_pool_blocks') - AND conname = 'qbit_pool_blocks_audit_publication_sequence_check' - ), - 'trigger_present', EXISTS ( - SELECT 1 - FROM pg_trigger - WHERE tgrelid = to_regclass('qbit_pool_blocks') - AND tgname = 'qbit_pool_blocks_assign_publication_ordinal' - AND NOT tgisinternal - ), - 'trigger_function_present', EXISTS ( - SELECT 1 - FROM pg_proc function_definition - JOIN pg_namespace namespace - ON namespace.oid = function_definition.pronamespace - WHERE namespace.nspname = current_schema() - AND function_definition.proname = - 'qbit_pool_blocks_assign_publication_ordinal' - ), - 'confirm_mentions_ordinal', COALESCE(( - SELECT function_definition.prosrc LIKE '%audit_publication_sequence%' - FROM pg_proc function_definition - JOIN pg_namespace namespace - ON namespace.oid = function_definition.pronamespace - WHERE namespace.nspname = current_schema() - AND function_definition.proname = 'qbit_confirm_pool_block' - ), false), - 'confirm_search_path_pinned', COALESCE(( - SELECT function_definition.proconfig IS NOT NULL - FROM pg_proc function_definition - JOIN pg_namespace namespace - ON namespace.oid = function_definition.pronamespace - WHERE namespace.nspname = current_schema() - AND function_definition.proname = 'qbit_confirm_pool_block' - ), false), - 'reactivate_search_path_pinned', COALESCE(( - SELECT function_definition.proconfig IS NOT NULL - FROM pg_proc function_definition - JOIN pg_namespace namespace - ON namespace.oid = function_definition.pronamespace - WHERE namespace.nspname = current_schema() - AND function_definition.proname = 'qbit_reactivate_pool_block' - ), false) -); -""", - schema=schema, - ) - - -MIGRATED_CATALOG = { - "column_present": True, - "sequence_present": True, - "index_present": True, - "constraint_present": True, - "trigger_present": True, - "trigger_function_present": True, - "confirm_mentions_ordinal": True, - "confirm_search_path_pinned": True, - "reactivate_search_path_pinned": True, -} -REVERTED_CATALOG = { - "column_present": False, - "sequence_present": False, - "index_present": False, - "constraint_present": False, - "trigger_present": False, - "trigger_function_present": False, - "confirm_mentions_ordinal": False, - "confirm_search_path_pinned": False, - "reactivate_search_path_pinned": False, -} - - -def ordinal_rows(schema: str) -> dict[str, int | None]: - rows = run_json( - """ -SELECT COALESCE(json_agg(json_build_object( - 'block_hash', block_hash, - 'audit_publication_sequence', audit_publication_sequence -) ORDER BY block_hash), '[]'::json) -FROM qbit_pool_blocks; -""", - schema=schema, - ) - return { - str(row["block_hash"]): row["audit_publication_sequence"] - for row in rows - } - - -def duplicate_ordinal_count(schema: str) -> int: - return int( - run_json( - """ -SELECT json_build_object('duplicates', count(*)) -FROM ( - SELECT audit_publication_sequence - FROM qbit_pool_blocks - WHERE audit_publication_sequence IS NOT NULL - GROUP BY audit_publication_sequence - HAVING count(*) > 1 -) duplicated; -""", - schema=schema, - )["duplicates"] - ) - - -def allocator_state(schema: str) -> dict[str, object]: - return run_json( - """ -SELECT json_build_object( - 'last_value', last_value, - 'is_called', is_called -) -FROM qbit_audit_publication_sequence_seq; -""", - schema=schema, - ) - - -def install_writer_lease(schema: str, *, writer_id: str, token: str) -> None: - run_psql( - f""" -DELETE FROM qbit_ledger_writer_lease; -INSERT INTO qbit_ledger_writer_lease ( - writer_id, writer_epoch, writer_session_token, lease_expires_at -) VALUES ( - '{writer_id}', 1, '{token}', clock_timestamp() + interval '5 minutes' -); -""", - schema=schema, - ) - - -def expire_writer_lease(schema: str) -> None: - run_psql( - """ -UPDATE qbit_ledger_writer_lease -SET updated_at = clock_timestamp() - interval '6 minutes', - lease_expires_at = clock_timestamp() - interval '1 minute'; -""", - schema=schema, - ) - - -def initialize_forward_schema(schema: str, *, writer_id: str) -> None: - ledger = ScopedPsqlLedger( - test_schema=schema, - writer_id=writer_id, - writer_epoch=1, - initialize_schema=True, - ) - try: - pass - finally: - ledger.release_writer_lease() - ledger.close() - - -def test_revert_round_trip() -> None: - schema = create_owned_schema("revert_round_trip") - seed_legacy_pool_blocks(schema) - initialize_forward_schema(schema, writer_id="a1-revert-forward") - - assert_equal( - ordinal_catalog(schema), MIGRATED_CATALOG, "round-trip migrated catalog" - ) - assert_equal( - ordinal_rows(schema), - { - REVERT_HASH_A: 1, - REVERT_HASH_B: 2, - REVERT_HASH_C: 3, - REVERT_HASH_INACTIVE: 4, - REVERT_HASH_REVERSED: None, - REVERT_HASH_PREPARED: None, - }, - "round-trip forward backfill ordinals", - ) - - apply_revert(schema) - assert_equal( - ordinal_catalog(schema), REVERTED_CATALOG, "round-trip reverted catalog" - ) - assert_equal( - run_json( - """ -SELECT json_build_object( - 'confirmed', ( - SELECT count(*) FROM qbit_pool_blocks WHERE chain_state = 'confirmed' - ), - 'total', (SELECT count(*) FROM qbit_pool_blocks) -); -""", - schema=schema, - ), - {"confirmed": 3, "total": 6}, - "revert keeps every pool-block row", - ) - - # Pre-ordinal code confirms through qbit_confirm_pool_block's plain - # chain_state UPDATE; the restored function must work with no ordinal - # machinery present. - install_writer_lease( - schema, writer_id="revert-legacy-writer", token="revert-legacy-session" - ) - confirm_sql = ( - "SELECT json_build_object('confirmed', qbit_confirm_pool_block(" - f"'{REVERT_HASH_PREPARED}', 10, 'revert-legacy-writer', 1, " - "'revert-legacy-session'));" - ) - assert_equal( - int(run_json(confirm_sql, schema=schema)["confirmed"]), - 1, - "restored confirm function confirms against reverted schema", - ) - assert_equal( - int(run_json(confirm_sql, schema=schema)["confirmed"]), - 1, - "restored confirm function replays an exact confirmation", - ) - # The rawest pre-ordinal shape: a direct chain_state UPDATE. - insert_prepared_block( - schema, REVERT_HASH_LEGACY_UPDATE, "2020-01-01T00:00:05Z" - ) - run_psql( - f""" -UPDATE qbit_pool_blocks -SET chain_state = 'confirmed' -WHERE block_hash = '{REVERT_HASH_LEGACY_UPDATE}'; -""", - schema=schema, - ) - assert_equal( - run_json( - f""" -SELECT json_build_object( - 'chain_state', ( - SELECT chain_state FROM qbit_pool_blocks - WHERE block_hash = '{REVERT_HASH_LEGACY_UPDATE}' - ) -); -""", - schema=schema, - )["chain_state"], - "confirmed", - "plain chain_state UPDATE confirms against reverted schema", - ) - - # Idempotent style: a second run of the revert is a no-op success. - apply_revert(schema) - assert_equal( - ordinal_catalog(schema), - REVERTED_CATALOG, - "second revert run is a no-op", - ) - - # Re-apply the forward migration: the backfill must renumber every - # confirmed and inactive row deterministically by (found_at, block_hash), - # including the two rows confirmed pre-ordinal-style after the revert. - expire_writer_lease(schema) - initialize_forward_schema(schema, writer_id="a1-revert-reapply") - assert_equal( - ordinal_catalog(schema), MIGRATED_CATALOG, "re-applied catalog" - ) - reapplied = ordinal_rows(schema) - assert_equal( - reapplied, - { - REVERT_HASH_A: 1, - REVERT_HASH_B: 2, - REVERT_HASH_C: 3, - REVERT_HASH_INACTIVE: 4, - REVERT_HASH_PREPARED: 5, - REVERT_HASH_REVERSED: None, - REVERT_HASH_LEGACY_UPDATE: 6, - }, - "re-applied forward backfill ordinals", - ) - assert_equal(duplicate_ordinal_count(schema), 0, "re-applied uniqueness") - assert_equal( - allocator_state(schema), - {"last_value": 6, "is_called": True}, - "re-applied allocator floor", - ) - - # The assignment trigger continues past the backfilled floor for a - # pre-ordinal-style confirmation and does not disturb existing ordinals. - insert_prepared_block( - schema, REVERT_HASH_POST_REAPPLY, "2020-01-01T00:00:06Z" - ) - run_psql( - f""" -UPDATE qbit_pool_blocks -SET chain_state = 'confirmed' -WHERE block_hash = '{REVERT_HASH_POST_REAPPLY}'; -""", - schema=schema, - ) - after_trigger = ordinal_rows(schema) - assert_equal( - after_trigger, - {**reapplied, REVERT_HASH_POST_REAPPLY: 7}, - "trigger assigns the next ordinal after re-apply", - ) - assert_equal(duplicate_ordinal_count(schema), 0, "post-trigger uniqueness") - - # A further schema apply must not re-assign or double-assign anything. - expire_writer_lease(schema) - initialize_forward_schema(schema, writer_id="a1-revert-stability") - assert_equal( - ordinal_rows(schema), - after_trigger, - "third schema apply leaves every ordinal unchanged", - ) - assert_equal( - allocator_state(schema), - {"last_value": 7, "is_called": True}, - "third schema apply leaves the allocator unchanged", - ) - - -def test_revert_refuses_unknown_dependents() -> None: - schema = create_owned_schema("revert_dependent") - seed_legacy_pool_blocks(schema) - initialize_forward_schema(schema, writer_id="a1-revert-dependent") - migrated_rows = ordinal_rows(schema) - run_psql( - """ -CREATE VIEW qbit_pool_blocks_ordinal_view AS -SELECT block_hash, audit_publication_sequence -FROM qbit_pool_blocks; -""", - schema=schema, - ) - try: - apply_revert(schema) - except GateFailure as failure: - message = str(failure) - if "audit_publication_sequence" not in message or "depend" not in message: - raise - else: - raise GateFailure( - "revert with a dependent view unexpectedly succeeded" - ) - # The failed revert must roll back completely: nothing half-reverted. - assert_equal( - ordinal_catalog(schema), - MIGRATED_CATALOG, - "failed revert leaves the migrated catalog intact", - ) - assert_equal( - ordinal_rows(schema), - migrated_rows, - "failed revert leaves every ordinal intact", - ) - run_psql("DROP VIEW qbit_pool_blocks_ordinal_view;", schema=schema) - apply_revert(schema) - assert_equal( - ordinal_catalog(schema), - REVERTED_CATALOG, - "revert succeeds once the dependent view is gone", - ) - - -def test_revert_noop_and_missing_table() -> None: - schema = create_owned_schema("revert_legacy_noop") - seed_legacy_pool_blocks(schema) - # Never migrated: the revert must be a no-op that still converges on the - # pre-ordinal shape (including the restored confirmation function). - apply_revert(schema) - assert_equal( - ordinal_catalog(schema), - REVERTED_CATALOG, - "revert of a never-migrated schema is a no-op", - ) - assert_equal( - run_json( - "SELECT json_build_object(" - "'total', (SELECT count(*) FROM qbit_pool_blocks));", - schema=schema, - )["total"], - 6, - "no-op revert keeps every legacy row", - ) - - empty_schema = create_owned_schema("revert_missing_table") - try: - apply_revert(empty_schema) - except GateFailure as failure: - if "missing qbit_pool_blocks in current schema" not in str(failure): - raise - else: - raise GateFailure( - "revert without qbit_pool_blocks unexpectedly succeeded" - ) - - -def main() -> None: - public_before = support.public_sentinel() - failure: BaseException | None = None - try: - test_revert_round_trip() - test_revert_refuses_unknown_dependents() - test_revert_noop_and_missing_table() - except BaseException as error: - failure = error - try: - support.cleanup_active_children() - support.cleanup_owned_schemas() - support.assert_equal( - support.marker_schema_count(), 0, "revert marker cleanup" - ) - support.assert_equal( - support.public_sentinel(), - public_before, - "revert public preservation", - ) - except BaseException as cleanup_error: - if failure is None: - raise - raise GateFailure( - f"revert scenario failed with {failure!r}; " - f"cleanup also failed with {cleanup_error!r}" - ) from cleanup_error - else: - support.atexit.unregister(support.cleanup_active_children) - support.atexit.unregister(support.cleanup_owned_schemas) - if failure is not None: - raise failure - print( - "prism postgres A1 revert gate PASS " - "round-trip legacy-confirm loud-failure noop-idempotent" - ) - - -if __name__ == "__main__": - main() diff --git a/tests/prism_postgres_candidate_gate.py b/tests/prism_postgres_candidate_gate.py deleted file mode 100644 index 2bcd8f0af..000000000 --- a/tests/prism_postgres_candidate_gate.py +++ /dev/null @@ -1,281 +0,0 @@ -#!/usr/bin/env python3 -"""Real-PostgreSQL gate for chunked candidate bodies (issue #255). - -Runs against a disposable database through the transport the environment -offers: ``PRISM_TEST_PSQL_COMMAND`` (the psql subprocess backend) and, -when ``PRISM_TEST_DATABASE_URL`` is set and psycopg imports, the native -pooled client as well. Every check is a semantic gate on the shipped -``PsqlShareLedger``: the additive migration applies twice, both durable -write routes stage outside the writer gate and publish under the fence, -the metadata-first page answers exhaustion, hydration reproduces the -share array from bounded chunk pages, terminalization detaches the body, -the janitor reclaims it in bounded steps, the database's own triggers -refuse every mutation of a sealed body, and a body an outbox row -references cannot be retired. - -Usage: ``python3 -m tests.prism_postgres_candidate_gate``; the wrapper -``test/test-prism-postgres-candidate-storage.sh`` provisions the container. -""" - -from __future__ import annotations - -import hashlib -import json -import os -import shlex -import subprocess -import sys -import time -import uuid -from pathlib import Path - -sys.path.insert(0, str(Path(__file__).resolve().parents[1])) - -from lab.prism.share_ledger import ( # noqa: E402 - PendingShare, - PsqlShareLedger, - block_candidate_identity_sha256, -) - -CHECKS: list[str] = [] - - -def check(condition: object, label: str) -> None: - if not condition: - raise SystemExit(f"prism postgres candidate gate: FAIL {label}") - CHECKS.append(label) - print(f"prism postgres candidate gate: ok {label}", flush=True) - - -def psql_json(command: str, sql: str) -> object: - completed = subprocess.run( - [*shlex.split(command), "--tuples-only", "--no-align", "--quiet"], - input=sql, - text=True, - stdout=subprocess.PIPE, - stderr=subprocess.PIPE, - check=False, - ) - if completed.returncode != 0: - raise RuntimeError(completed.stderr.strip()) - output = completed.stdout.strip() - return json.loads(output.splitlines()[-1]) if output else None - - -def psql_expect_error(command: str, sql: str, fragment: str, label: str) -> None: - completed = subprocess.run( - [*shlex.split(command), "--quiet", "--set", "ON_ERROR_STOP=1"], - input=sql, - text=True, - stdout=subprocess.PIPE, - stderr=subprocess.PIPE, - check=False, - ) - refused = completed.returncode != 0 and fragment in completed.stderr - if not refused: - print( - f"prism postgres candidate gate: unexpected outcome for {label}: " - f"exit={completed.returncode} stderr={completed.stderr.strip()[:400]!r} " - f"stdout={completed.stdout.strip()[:200]!r}", - flush=True, - ) - check(refused, f"{label} ({fragment})") - - -def share(index: int) -> dict[str, object]: - return { - "share_seq": index, - "share_id": f"miner-{index % 5}:é{index}", - "miner_id": f"m{index % 5}", - "order_key": "k", - "p2mr_program_hex": "ab" * 32, - "share_difficulty": 10**20 + index, - "network_difficulty": 1000, - "template_height": 9, - "job_id": "job-a", - "job_issued_at_ms": 1_700_000_000_000 + index, - "accepted_at_ms": 1_700_000_000_001 + index, - "ntime": 1_700_000_000, - "credit_policy": None, - } - - -def intent(block_hash: str, share_count: int, *, credit: bool) -> dict[str, object]: - pending = { - "share_id": f"miner-0:{block_hash}", - "miner_id": "m0", - "order_key": "m0", - "p2mr_program_hex": "ab" * 32, - "share_difficulty": 100, - "network_difficulty": 1000, - "template_height": 9, - "job_id": "job-a", - "job_issued_at_ms": 1_700_000_000_000, - "accepted_at_ms": 1_700_000_000_500, - "ntime": 1_700_000_000, - "credit_policy": None, - } - return { - "schema": "qbit.prism.block-candidate-intent.v1", - "block_hash_hex": block_hash, - "block_hex": "00" * 200, - "coinbase_tx_hex": "01", - "parent_hash": "11" * 32, - "expected_height": 10, - "template": {"previousblockhash": "11" * 32, "height": 10, "coinbasevalue": 5}, - "shares_json": [share(index) for index in range(share_count)], - "prior_balances": [{"recipient_id": f"r{index}", "balance_sats": index} for index in range(3000)], - "found_block": {"network_difficulty": 1000}, - "prospective_prior_balances": None, - "witness_merkle_leaves_hex": ["cc" * 32] * 5, - "extranonce1_hex": "00", - "extranonce2_hex": "01", - "username": "miner-0", - "pending_share": pending, - "credit_share_on_accept": credit, - "collection_only": False, - } - - -def pending_share_for(fields: dict[str, object]) -> PendingShare: - return PendingShare(**dict(fields["pending_share"])) # type: ignore[arg-type] - - -def run_gate(psql_command: str, database_url: str | None, *, native: bool) -> None: - label = "native" if native else "psql" - ledger = PsqlShareLedger( - psql_command=psql_command, - database_url=database_url if native else None, - native_client_mode="1" if native else "0", - writer_id=f"writer-{label}", - writer_epoch=1, - initialize_schema=True, - candidate_spool_dir=os.environ.get("PRISM_TEST_SPOOL_DIR"), - ) - try: - # The migration is idempotent: apply it a second time through psql. - root = Path(__file__).resolve().parents[1] - psql_json(psql_command, (root / "crates/qbit-prism/sql/002_candidate_bodies.sql").read_text() + "\nSELECT json_build_object('ok', true);") - check(ledger.verify_candidate_schema() == {"declared": 2, "has_body_table": True}, f"{label}: schema capability declared") - - block_a = hashlib.sha256(f"{label}-a-{uuid.uuid4().hex}".encode()).hexdigest() - fields = intent(block_a, 3000, credit=True) - oracle_digest = block_candidate_identity_sha256(fields) - started = time.monotonic() - result = ledger.persist_block_candidate_intent(fields) - check(result.inserted and result.state == "pending", f"{label}: standalone persist publishes a chunked body") - again = ledger.persist_block_candidate_intent(intent(block_a, 3000, credit=True)) - check(not again.inserted and again.state == "pending", f"{label}: exact standalone replay is idempotent") - try: - ledger.persist_block_candidate_intent({**intent(block_a, 3000, credit=True), "username": "other"}) - check(False, f"{label}: differing payload is refused") - except RuntimeError as exc: - check("payload mismatch" in str(exc), f"{label}: differing payload is refused") - print(f"prism postgres candidate gate: staged 3000-share body in {time.monotonic() - started:.2f}s", flush=True) - - row = psql_json( - psql_command, - f"SELECT json_build_object('storage_version', storage_version, 'candidate_sha256', candidate_sha256, 'body_id', body_id, 'state', state, 'candidate_is_null', candidate IS NULL) FROM qbit_block_candidate_outbox WHERE block_hash = '{block_a}';", - ) - check(row["storage_version"] == 2 and row["candidate_is_null"] and row["candidate_sha256"] == oracle_digest, f"{label}: outbox row is version 2 with the historical digest") - body_id = row["body_id"] - manifest = psql_json(psql_command, f"SELECT json_build_object('state', state, 'chunk_count', chunk_count, 'span_count', span_count, 'page_count', page_count) FROM qbit_block_candidate_body WHERE body_id = '{body_id}';") - check(manifest["state"] == "sealed" and manifest["chunk_count"] >= 1 and manifest["span_count"] >= 2 and manifest["page_count"] >= 2, f"{label}: manifest is sealed with spans and pages") - - page = ledger.pending_block_candidate_headers(limit=32) - check(len(page.rows) == 1 and page.exhausted and page.rows[0]["block_hash"] == block_a, f"{label}: header page enumerates the row and answers exhaustion") - header = page.rows[0]["header"] - check(header["parent_hash"] == "11" * 32 and header["expected_height"] == 10 and header["pending_share"]["accepted_at_ms"] == 1_700_000_000_500 and header["oversized"] is False, f"{label}: bounded header carries the typed facts") - check(page.rows[0]["body"]["state"] == "sealed" and page.rows[0]["body"]["body_id"] == body_id, f"{label}: header carries the sealed body reference") - - hydrated = ledger.hydrate_block_candidate_intent(page.rows[0]) - check(len(hydrated["shares_json"]) == 3000 and hydrated["shares_json"][2999] == share(2999) and hydrated["shares_json"][0] == share(0), f"{label}: hydration reproduces the share array") - check(hydrated["pending_share"]["accepted_at_ms"] == 1_700_000_000_500 and hydrated.candidate_sha256 == oracle_digest, f"{label}: hydration restores the stamp and identity") - check(list(hydrated["prior_balances"]) == fields["prior_balances"], f"{label}: large balances decode through the spool adapter") - spool_path = hydrated.body.path - hydrated.body.close() - check(not os.path.exists(spool_path), f"{label}: spool is unlinked on close") - - # Immutability at the database boundary. - psql_expect_error(psql_command, f"INSERT INTO qbit_block_candidate_body_chunk (body_id, ordinal, chunk, chunk_sha256) VALUES ('{body_id}', 9999, '\\x00', '{'0' * 64}');", "accepts no parts", f"{label}: chunk insert after seal is refused") - psql_expect_error(psql_command, f"UPDATE qbit_block_candidate_body_chunk SET chunk = '\\x00' WHERE body_id = '{body_id}' AND ordinal = 0;", "immutable", f"{label}: chunk update is refused") - psql_expect_error(psql_command, f"DELETE FROM qbit_block_candidate_body_chunk WHERE body_id = '{body_id}' AND ordinal = 0;", "cannot be deleted", f"{label}: sealed chunk delete is refused") - psql_expect_error(psql_command, f"UPDATE qbit_block_candidate_body SET byte_count = byte_count + 1 WHERE body_id = '{body_id}';", "is sealed", f"{label}: sealed manifest scalars are immutable") - psql_expect_error(psql_command, f"UPDATE qbit_block_candidate_body SET state = 'retired', retired_at = clock_timestamp() WHERE body_id = '{body_id}';", "referenced by a pending outbox row", f"{label}: a referenced body cannot be retired") - psql_expect_error(psql_command, f"INSERT INTO qbit_block_candidate_body_page (body_id, field, page_ordinal, body_offset, record_index) VALUES ('{body_id}', 'x', 99999, 0, 0);", "accepts no parts", f"{label}: page insert after seal is refused") - check(ledger.retire_orphan_candidate_bodies() == (), f"{label}: orphan retirement leaves the referenced body alone") - - # Credit-on-accept: the share append joins the published body without re-uploading. - pending = pending_share_for(fields) - records = ledger.append_batch([(pending, intent(block_a, 3000, credit=True))]) - check(len(records) == 1 and records[0].candidate_outbox_state == "pending", f"{label}: credit append links the existing body") - linked = psql_json(psql_command, f"SELECT json_build_object('share_id', share_id, 'body_id', body_id) FROM qbit_block_candidate_outbox WHERE block_hash = '{block_a}';") - check(linked["share_id"] == pending.share_id and linked["body_id"] == body_id, f"{label}: outbox row carries the share and the same body") - replay = ledger.append_batch([(pending, intent(block_a, 3000, credit=True))]) - check(replay[0].share_seq == records[0].share_seq, f"{label}: exact replay of the credit append is idempotent") - try: - ledger.append_batch([(pending, {**intent(block_a, 3000, credit=True), "username": "other"})]) - check(False, f"{label}: differing credit payload is refused") - except RuntimeError as exc: - check("payload mismatch" in str(exc), f"{label}: differing credit payload is refused") - - # Async route: a fresh share with a new candidate in one fenced statement. - block_b = hashlib.sha256(f"{label}-b-{uuid.uuid4().hex}".encode()).hexdigest() - fields_b = intent(block_b, 700, credit=False) - records_b = ledger.append_batch([(pending_share_for(fields_b), fields_b)]) - check(records_b[0].candidate_outbox_state == "pending", f"{label}: share append publishes a new chunked candidate") - page = ledger.pending_block_candidate_headers(limit=1) - check(len(page.rows) == 1 and not page.exhausted, f"{label}: a full page is not exhaustion") - page2 = ledger.pending_block_candidate_headers(limit=1, after_cursor=page.next_cursor) - check(len(page2.rows) == 1 and page2.exhausted, f"{label}: the cursor walks to the last row and proves exhaustion") - - # Terminalization detaches the body; the janitor reclaims it in bounded steps. - check(ledger.mark_block_candidate_submitted(block_hash=block_a), f"{label}: terminal write succeeds") - detached = psql_json(psql_command, f"SELECT json_build_object('body_id', body_id, 'retired_body_id', retired_body_id, 'state', (SELECT state FROM qbit_block_candidate_body WHERE body_id = '{body_id}')) FROM qbit_block_candidate_outbox WHERE block_hash = '{block_a}';") - check(detached["body_id"] is None and detached["retired_body_id"] == body_id and detached["state"] == "retired", f"{label}: terminal write detaches and retires the body") - steps = 0 - reclaimed = False - while steps < 500: - outcome = ledger.reap_retired_candidate_bodies(max_chunks=2) - steps += 1 - if outcome["deleted_bodies"]: - reclaimed = True - break - if outcome["body_id"] is None: - break - remaining = psql_json(psql_command, f"SELECT json_build_object('body', EXISTS (SELECT 1 FROM qbit_block_candidate_body WHERE body_id = '{body_id}'), 'chunks', EXISTS (SELECT 1 FROM qbit_block_candidate_body_chunk WHERE body_id = '{body_id}'), 'pages', EXISTS (SELECT 1 FROM qbit_block_candidate_body_page WHERE body_id = '{body_id}'));") - check(reclaimed and not any(remaining.values()) and steps > 3, f"{label}: janitor reclaims the retired body in {steps} bounded steps") - - # A pre-#255 terminal write cannot touch a version-2 row (rollback floor). - psql_expect_error(psql_command, f"UPDATE qbit_block_candidate_outbox SET state = 'abandoned', completed_at = clock_timestamp(), candidate = NULL WHERE block_hash = '{block_b}';", "dual_format_check", f"{label}: legacy terminal write is refused by the schema") - check(ledger.mark_block_candidate_abandoned(block_hash=block_b, error="gate"), f"{label}: fenced abandon detaches the second body") - # Drain the second body too, then a step with nothing to do. - for _ in range(500): - if not ledger.reap_retired_candidate_bodies(max_chunks=64)["body_id"]: - break - check(ledger.reap_retired_candidate_bodies()["body_id"] is None, f"{label}: janitor idles with nothing retired") - finally: - ledger.release_writer_lease() - ledger.close() - - -def main() -> int: - psql_command = os.environ["PRISM_TEST_PSQL_COMMAND"] - database_url = os.environ.get("PRISM_TEST_DATABASE_URL") - run_gate(psql_command, database_url, native=False) - native_available = False - if database_url: - try: - import psycopg # noqa: F401 - - native_available = True - except ImportError: - print("prism postgres candidate gate: psycopg unavailable; native transport skipped", flush=True) - if native_available: - run_gate(psql_command, database_url, native=True) - print(f"prism postgres candidate gate: PASS ({len(CHECKS)} checks)", flush=True) - return 0 - - -if __name__ == "__main__": - raise SystemExit(main()) diff --git a/tests/prism_public_read_replica_gate.py b/tests/prism_public_read_replica_gate.py deleted file mode 100644 index 6b4133e0c..000000000 --- a/tests/prism_public_read_replica_gate.py +++ /dev/null @@ -1,288 +0,0 @@ -#!/usr/bin/env python3 -"""Non-discovered public read replica integration gate (issue #145). - -Invoked by ``test/test-prism-public-read-replica.sh``. Lives outside unittest -discovery because it requires an explicitly provisioned primary plus streaming -hot-standby replica. Proves, against real containers (the #134 pattern): - -- the extracted public read service serves a moved endpoint with data that - reached it by streaming replication, and publishes the observed lag; -- the replica contract is enforced rather than described: a replication stream - silent past the configured bound answers 503, and a backing server that is - not in recovery is refused outright; -- the contract is scoped by what a route actually reads -- the environment-only - mining-configuration route keeps serving 200 while replica-backed routes - refuse. - -The unit-level route split and the freshness rules under a fake clock live in -tests/test_prism_public_read_service.py; this gate proves the container-side -serving contract those tests model. -""" - -from __future__ import annotations - -import contextlib -import json -import os -import shlex -import subprocess -import tempfile -import threading -import time -import urllib.error -import urllib.request -from http.server import ThreadingHTTPServer -from pathlib import Path - -from lab.prism import public_read_service - -PRIMARY_PSQL_COMMAND = os.environ.get("GATE_PRIMARY_PSQL_COMMAND", "") -REPLICA_PSQL_COMMAND = os.environ.get("GATE_REPLICA_PSQL_COMMAND", "") -if not PRIMARY_PSQL_COMMAND or not REPLICA_PSQL_COMMAND: - raise SystemExit( - "GATE_PRIMARY_PSQL_COMMAND and GATE_REPLICA_PSQL_COMMAND are required" - ) - -BLOCK_HASH = "aa" * 32 - - -class GateFailure(RuntimeError): - pass - - -def assert_equal(actual: object, expected: object, message: str) -> None: - if actual != expected: - raise GateFailure(f"{message}: expected {expected!r}, got {actual!r}") - - -def http_get(url: str) -> tuple[int, object, dict[str, str]]: - try: - with urllib.request.urlopen(url, timeout=10) as response: - body = response.read() - return ( - response.status, - json.loads(body) if body else None, - {k.lower(): v for k, v in response.headers.items()}, - ) - except urllib.error.HTTPError as exc: - body = exc.read() - return ( - exc.code, - json.loads(body) if body else None, - {k.lower(): v for k, v in exc.headers.items()}, - ) - - -def http_get_text(url: str) -> str: - with urllib.request.urlopen(url, timeout=10) as response: - return response.read().decode() - - -def run_psql(command: str, sql: str) -> None: - process = subprocess.run( - shlex.split(command) + ["-v", "ON_ERROR_STOP=1", "-c", sql], - capture_output=True, - text=True, - timeout=60, - ) - if process.returncode != 0: - raise GateFailure( - f"psql failed ({process.returncode}): {process.stderr.strip()}" - ) - - -def psql_scalar(command: str, sql: str) -> str | None: - """Scalar result, or None when the query could not run. - - None rather than an exception because callers poll for state that is still - replicating: before the DDL replays, the query fails with "relation does - not exist", which is a "not yet", not a gate failure. - """ - process = subprocess.run( - shlex.split(command) + ["-tA", "-c", sql], - capture_output=True, - text=True, - timeout=60, - ) - if process.returncode != 0: - return None - return process.stdout.strip() - - -def wait_for_replica_row(deadline_seconds: float = 90.0) -> None: - """Poll the standby until the row arrives, tolerating the schema's absence. - - Both the schema and the row reach the standby by streaming replication -- - the shell wrapper starts the replica before this gate runs, so the standby - was cloned from a primary that had neither. Until the CREATE TABLE replays, - the count query fails rather than returning 0, so a failed query has to be - treated as "not yet" inside the deadline instead of aborting the gate. - """ - deadline = time.monotonic() + deadline_seconds - last_seen: str | None = None - while time.monotonic() < deadline: - last_seen = psql_scalar( - REPLICA_PSQL_COMMAND, - "SELECT count(*) FROM qbit_pool_blocks " - f"WHERE block_hash = '{BLOCK_HASH}'", - ) - if last_seen == "1": - return - time.sleep(0.5) - raise GateFailure( - f"replica did not replay the qbit_pool_blocks row within " - f"{deadline_seconds}s (last count query returned {last_seen!r})" - ) - - -GATE_ENVIRONMENT = { - "QBIT_RPC_HOST": "127.0.0.1", - "QBIT_RPC_PORT": "18452", - "QBIT_RPC_USER": "gate", - "QBIT_RPC_PASSWORD": "gate", - # Required at startup: this process runs no Stratum listener. - "PRISM_PUBLIC_STRATUM_URL": "stratum+tcp://gate.invalid:3340", - # psql transport, because the DSN is a `docker exec` into the container. - "PRISM_POSTGRES_NATIVE_CLIENT": "0", - "PRISM_DATABASE_URL": "", - "PRISM_PUBLIC_REPLICA_MODE": "require", -} - - -@contextlib.contextmanager -def serving(**overrides: str): - """Build the real service from the environment and serve it on a free port. - - Goes through build_service() rather than assembling the collaborators by - hand: the point of a container gate is that the process an operator starts - is the process under test, including every refusal build_service makes. - """ - audit_dir = Path(tempfile.mkdtemp(prefix="prism-public-read-gate-")) - environment = dict(GATE_ENVIRONMENT) - environment["PRISM_AUDIT_DIR"] = str(audit_dir) - environment["PRISM_POSTGRES_PSQL_COMMAND"] = REPLICA_PSQL_COMMAND - environment.update(overrides) - - previous = dict(os.environ) - os.environ.update(environment) - try: - coordinator, readiness, metrics, replica = public_read_service.build_service() - finally: - os.environ.clear() - os.environ.update(previous) - - readiness.start() - service = public_read_service.PublicReadService( - coordinator, - metrics=metrics, - readiness=readiness, - replica=replica, - ) - server = ThreadingHTTPServer( - ("127.0.0.1", 0), public_read_service.make_handler(service) - ) - server.daemon_threads = True - thread = threading.Thread(target=server.serve_forever, daemon=True) - thread.start() - try: - yield f"http://127.0.0.1:{server.server_address[1]}" - finally: - server.shutdown() - server.server_close() - readiness.stop() - thread.join(timeout=5) - - -def check_serves_replicated_data() -> None: - with serving() as base_url: - status, payload, _ = http_get(f"{base_url}/healthz") - assert_equal(status, 200, "health status") - assert_equal(payload["ok"], True, "health ok") - assert_equal(payload["replica"]["in_recovery"], True, "in_recovery") - if payload["replica"]["receiver_heartbeat_age_seconds"] is None: - raise GateFailure("health reported no walreceiver heartbeat") - - status, payload, headers = http_get(f"{base_url}/public/v1/blocks") - assert_equal(status, 200, "blocks status") - assert_equal(payload["schema"], "prism.dashboard.blocks.v1", "blocks schema") - assert_equal(payload["pagination"]["total_count"], 1, "blocks total_count") - assert_equal(payload["rows"][0]["hash"], BLOCK_HASH, "blocks row hash") - if public_read_service.REPLICA_LAG_HEADER.lower() not in headers: - raise GateFailure( - f"public response missing {public_read_service.REPLICA_LAG_HEADER}" - ) - - metrics = http_get_text(f"{base_url}/metrics") - for gauge in ( - "qbit_prism_public_replica_in_recovery 1", - "qbit_prism_public_replica_heartbeat_age_seconds", - "qbit_prism_public_replica_replay_lag_seconds", - "qbit_prism_public_replica_apply_backlog_bytes", - ): - if gauge not in metrics: - raise GateFailure(f"metrics missing {gauge!r}") - - -def check_refuses_past_the_bound() -> None: - """Any live heartbeat is older than this bound, so every probe fails it.""" - - with serving(PRISM_PUBLIC_REPLICA_MAX_LAG_SECONDS="0.0001") as base_url: - status, payload, _ = http_get(f"{base_url}/public/v1/blocks") - assert_equal(status, 503, "degraded blocks status") - assert_equal(payload["error"]["code"], "upstream_unavailable", "degraded code") - if "silent for" not in payload["error"]["message"]: - raise GateFailure(f"unexpected degraded message: {payload}") - - status, _, _ = http_get(f"{base_url}/healthz") - assert_equal(status, 503, "degraded health status") - - # Scoped by what the route reads: mining-configuration is assembled - # from the environment and touches no read slot, so the replica being - # stale is not a reason to refuse it. - status, payload, _ = http_get(f"{base_url}/public/v1/mining-configuration") - assert_equal(status, 200, "mining-configuration under a stale replica") - - -def check_refuses_a_writable_primary() -> None: - with serving(PRISM_POSTGRES_PSQL_COMMAND=PRIMARY_PSQL_COMMAND) as base_url: - status, payload, _ = http_get(f"{base_url}/public/v1/blocks") - assert_equal(status, 503, "primary refusal status") - if "not in recovery" not in payload["error"]["message"]: - raise GateFailure(f"unexpected primary refusal message: {payload}") - - status, _, _ = http_get(f"{base_url}/healthz") - assert_equal(status, 503, "primary refusal health status") - - -def main() -> int: - # The schema is created on the primary *after* the standby was cloned -- - # the shell wrapper brings the replica up before invoking this gate -- so - # both the DDL and the row below reach the standby by streaming, which is - # what wait_for_replica_row polls for. - run_psql( - PRIMARY_PSQL_COMMAND, - Path("crates/qbit-prism/sql/001_share_ledger.sql").read_text(encoding="utf-8"), - ) - run_psql( - PRIMARY_PSQL_COMMAND, - f""" -INSERT INTO qbit_pool_blocks - (block_hash, block_height, parent_hash, coinbase_txid, - payout_manifest_sha256, chain_state) -VALUES ('{BLOCK_HASH}', 101, '{"bb" * 32}', '{"cc" * 32}', - '{"dd" * 32}', 'confirmed') -ON CONFLICT (block_hash) DO NOTHING; -""", - ) - wait_for_replica_row() - - check_serves_replicated_data() - check_refuses_past_the_bound() - check_refuses_a_writable_primary() - - print("public read replica gate: OK") - return 0 - - -if __name__ == "__main__": - raise SystemExit(main()) diff --git a/tests/prism_regtest_crash_coordinator.py b/tests/prism_regtest_crash_coordinator.py deleted file mode 100644 index d013491ce..000000000 --- a/tests/prism_regtest_crash_coordinator.py +++ /dev/null @@ -1,73 +0,0 @@ -#!/usr/bin/env python3 -"""Crash once after durable candidate publication and before its first node offer. - -Only the disposable regtest harness launches this entry point. The successor -uses the normal coordinator entry point and discovers the candidate from SQL. -""" - -from __future__ import annotations - -import json -import os -from pathlib import Path -import sys - -sys.path.insert(0, str(Path(__file__).resolve().parents[1])) - -import psycopg - -from lab.prism.block_candidates import BlockCandidateService -from lab.prism.prism_coordinator import main - - -def crash_before_offer(self, candidate): - block_hash = candidate.submission.block_hash_hex.lower() - with psycopg.connect(os.environ["PRISM_DATABASE_URL"]) as connection: - row = connection.execute(""" - SELECT o.candidate_sha256, o.share_id, s.writer_epoch, - COALESCE((to_jsonb(o)->>'storage_version')::integer, 1), - to_jsonb(o)->>'body_id' - FROM qbit_block_candidate_outbox o - JOIN qbit_share_ledger s ON s.share_id = o.share_id - WHERE o.block_hash = %s AND o.state = 'pending' AND s.accepted - """, (block_hash,)).fetchone() - if row is None: - raise RuntimeError("crash fixture reached node offer before atomic share/outbox durability") - requested_version = os.environ.get("PRISM_CANDIDATE_STORAGE_VERSION") - if requested_version is not None and row[3] != int(requested_version): - raise RuntimeError("crash fixture persisted a different candidate storage version") - if row[3] == 2: - body = connection.execute(""" - SELECT b.state, b.candidate_sha256, b.chunk_count, b.byte_count, - count(c.ordinal), COALESCE(sum(octet_length(c.chunk)), 0), - min(c.ordinal), max(c.ordinal), - bool_and(sha256(c.chunk) = decode(c.chunk_sha256, 'hex')) - FROM qbit_block_candidate_body b - LEFT JOIN qbit_block_candidate_body_chunk c USING (body_id) - WHERE b.body_id = %s - GROUP BY b.body_id - """, (row[4],)).fetchone() - if body is None or not ( - body[0] == "sealed" and body[1] == row[0] - and body[2] == body[4] and body[3] == body[5] - and body[6] == 0 and body[7] == body[2] - 1 and body[8] is True - ): - raise RuntimeError("crash fixture found an incomplete published candidate body") - marker = Path(sys.argv[1]) - with marker.open("x", encoding="utf-8") as handle: - json.dump({ - "block_hash": block_hash, "candidate_sha256": row[0], - "share_id": row[1], "writer_epoch": row[2], - "storage_version": row[3], "body_id": row[4], - }, handle) - handle.flush() - os.fsync(handle.fileno()) - print(f"regtest intentional exit after durable candidate {block_hash}", flush=True) - os._exit(75) - - -if __name__ == "__main__": - if len(sys.argv) != 2: - raise SystemExit("requires the disposable crash-marker path") - BlockCandidateService._submit_block_candidate_to_node = crash_before_offer - raise SystemExit(main()) diff --git a/tests/prism_regtest_share_miner.py b/tests/prism_regtest_share_miner.py deleted file mode 100644 index f9e37e353..000000000 --- a/tests/prism_regtest_share_miner.py +++ /dev/null @@ -1,72 +0,0 @@ -#!/usr/bin/env python3 -"""Seed a real regtest payout window with one acknowledged non-block share. - -The live harness subsequently runs the ordinary block-solving miner. Keeping -this fixture separate leaves the deployed miner simulator's behavior intact. -""" - -from __future__ import annotations - -import importlib.util -from pathlib import Path -import sys -import time - - -def load_miner(): - path = Path(__file__).resolve().parents[1] / "lab/miner-sim/miner_sim.py" - spec = importlib.util.spec_from_file_location("prism_fixture_miner", path) - assert spec is not None and spec.loader is not None - module = importlib.util.module_from_spec(spec) - sys.modules[spec.name] = module - spec.loader.exec_module(module) - return module - - -def solve_share(miner, job, extranonce1, extranonce2_size, share_target, deadline): - block_target = miner.target_from_compact(job.nbits) - if share_target <= block_target: - raise RuntimeError("primed regtest requires a share target easier than the block target") - for extranonce2 in range(1 << (8 * extranonce2_size)): - extranonce2_hex = f"{extranonce2:0{extranonce2_size * 2}x}" - _, raw = miner.assemble_header(job, extranonce1, extranonce2_hex, "00000000") - header = bytearray(raw) - for nonce in range(0x100000000): - if nonce % 4096 == 0 and time.time() >= deadline: - raise miner.RecoverableError("ordinary-share fixture timed out") - header[76:80] = nonce.to_bytes(4, "little") - value = int.from_bytes(miner.double_sha256(header), "little") - if block_target < value <= share_target: - return extranonce2_hex, f"{nonce:08x}" - raise RuntimeError("ordinary-share fixture exhausted the nonce space") - - -def main() -> int: - miner = load_miner() - username = miner.resolve_miner_username() - deadline = time.time() + miner.MINER_TIMEOUT_SECONDS - while time.time() < deadline: - client = miner.StratumClient(miner.STRATUM_HOST, miner.STRATUM_PORT, deadline) - try: - extranonce1, size, job, target, _ = miner.receive_handshake_and_job(client, username) - extranonce2, nonce = solve_share(miner, job, extranonce1, size, target, deadline) - request_id = client.send("mining.submit", [ - username, job.job_id, extranonce2, job.ntime, nonce, - ]) - while time.time() < deadline: - message = client.recv() - if message.get("id") != request_id: - continue - if message.get("result") is True: - print("ordinary share durably accepted", flush=True) - return 0 - raise miner.RecoverableError(str(message.get("error") or "share rejected")) - except miner.RecoverableError as exc: - print(f"ordinary-share fixture retry: {exc}", flush=True) - finally: - client.close() - raise SystemExit("ordinary-share fixture timed out without a durable acknowledgment") - - -if __name__ == "__main__": - raise SystemExit(main()) diff --git a/tests/prism_vardiff_test_support.py b/tests/prism_vardiff_test_support.py deleted file mode 100644 index 806a191fe..000000000 --- a/tests/prism_vardiff_test_support.py +++ /dev/null @@ -1,1053 +0,0 @@ -#!/usr/bin/env python3 -# ruff: noqa: F401 -"""Shared no-environment fixtures for coordinator domain tests.""" - -from __future__ import annotations - -import errno -import hashlib -import inspect -import json -import os -import queue -import socket -import subprocess -import tempfile -import contextlib -import threading -import time -import unittest -from concurrent.futures import Future -from dataclasses import replace as dataclass_replace -from decimal import Decimal -from pathlib import Path -from types import SimpleNamespace -from unittest.mock import patch - -from lab.auxpow import vardiff -from lab.prism import direct_stratum -from lab.prism.share_ledger import ( - LedgerOperationTimeout, - PendingShare, - PsqlShareLedger, - ShareReplayConflict, - ShareReplayResult, - SingleWriterShareLedger, - WRITER_LEASE_HEARTBEAT_SESSION_PREFIX, -) -from lab.prism.prism_coordinator import ( - CachedJobBundle, - CachedTemplateArtifacts, - ClientState, - MAX_PENDING_SHARE_APPENDS, - PRISM_CREDIT_POLICY_STALE_GRACE, - PRISM_REJECTION_BACKEND_RPC_UNAVAILABLE, - PRISM_REJECTION_BLOCK_ACCEPT_PENDING, - PRISM_REJECTION_CANDIDATE_AUDIT_MISMATCH, - PRISM_REJECTION_DUPLICATE_SHARE, - PRISM_REJECTION_INVALID_NTIME_OR_NONCE, - PRISM_REJECTION_LEDGER_CONFIRMATION_FAILED, - PRISM_REJECTION_LEDGER_CONFIRMATION_SUPERSEDED, - PRISM_REJECTION_LOW_DIFFICULTY, - PendingShareAppend, - PayoutStateArtifact, - PrismBlockCandidate, - PRISM_REJECTION_POOL_CLOSED, - PRISM_REJECTION_REASON_IDS, - PRISM_REJECTION_STALE_JOB, - PRISM_REJECTION_SUBMITBLOCK_REJECTED, - PRISM_REJECTION_UNAUTHORIZED_WORKER, - PRISM_REJECTION_UNKNOWN_JOB, - PRISM_WORKER_METRICS_OVERFLOW_LABEL, - QbitTipTemplateSnapshot, - StratumError, - StratumListenerProfile, - JobBuildSuperseded, - TemplateRefreshBlocked, - TemplateRefreshSuperseded, - PrismCoordinator, - ShutdownInProgress, - WorkerIdentity, - WriterLeaseRenewalDeferred, - _FanoutCancellation, - _ObservedRLock, - _JobBuildCancellation, - qbit_template_fingerprint, - qbit_gbt_rules, - scaled_target_difficulty, - target_from_compact, -) -from lab.prism.reorg_reconciler import _ReconcileFlight -from lab.prism.coordinator_config import ( - DEFAULT_TESTNET_USERNAME_FALLBACK_ADDRESS, - default_prism_coinbase_tag_hex, - default_prism_username_fallback_address, - env_positive_float, - load_prism_highdiff_listener, - load_prism_vardiff_config, - validate_prism_production_gate, - validate_same_tip_job_retention_limits, -) -from lab.prism.job_bundle import JobBuildCancelled -from lab.prism.job_delivery import MAX_ACTIVE_PRISM_JOBS_PER_CLIENT -from lab.prism.payout_state import ( - PayoutStatePublicationBlocked as _PayoutStatePublicationBlocked, -) -from lab.prism.stratum_session import parse_stratum_password_options - - -PAYOUT_ADDRESS = "tq1z70ukpvs96kye6jmgvl3nttevtkrq8uu89snkpm6m8gwqukw8u5dsz32kwa" - - -def fake_audit_bundle_popen( - captured: dict[str, object], - *, - output_text: str = '{"ok":true}', - returncode: int = 0, - stderr_text: str = "", -) -> type: - class FakeStdin: - def __init__(self) -> None: - self.parts: list[str] = [] - - def write(self, value: str) -> int: - self.parts.append(value) - return len(value) - - def close(self) -> None: - return None - - class FakePopen: - def __init__(self, cmd: list[str], **kwargs: object) -> None: - captured["cmd"] = cmd - self.stdin = FakeStdin() - self.stdout = kwargs["stdout"] - self.stderr = kwargs["stderr"] - - def wait(self, timeout: float | None = None) -> int: - captured["timeout"] = timeout - captured["payload"] = json.loads("".join(self.stdin.parts)) - self.stdout.write(output_text) - self.stderr.write(stderr_text) - return returncode - - return FakePopen - - -def tx_output(value_sats: int, script_hex: str) -> str: - return value_sats.to_bytes(8, "little").hex() + direct_stratum.compact_size(len(bytes.fromhex(script_hex))).hex() + script_hex - - -def synthetic_witness_transaction(seed: str) -> str: - script = seed * 3 - witness_item = seed * 5 - return ( - "01000000" - + "0001" - + "01" - + (seed * 32) - + "00000000" - + direct_stratum.compact_size(len(bytes.fromhex(script))).hex() - + script - + "ffffffff" - + "01" - + tx_output(1, "51") - + "01" - + direct_stratum.compact_size(len(bytes.fromhex(witness_item))).hex() - + witness_item - + "00000000" - ) - - -class FakeJob: - def __init__(self, difficulty: Decimal) -> None: - self.share_difficulty = difficulty - - -class FakeLedger: - backend_name = "fake" - - def __init__(self, shares: int = 0, prior_balances: list[dict[str, object]] | None = None) -> None: - self.shares = shares - self.prior_balances = prior_balances or [] - - def all_shares(self) -> list[object]: - return [object()] * self.shares - - def current_prior_balances(self) -> list[dict[str, object]]: - return [dict(balance) for balance in self.prior_balances] - - def metrics(self) -> dict[str, int]: - return {"blocks": 2, "owed_accounts": 3} - - def reorg_watch_blocks(self, *, active_tip_height: int) -> list[dict[str, object]]: - return [] - - def mark_pool_block_inactive(self, *, block_hash: str, active_tip_height: int) -> dict[str, object]: - return {"backend": "fake", "inactive_count": 0} - - def reject_prepared_block(self, *, block_hash: str, active_tip_height: int) -> dict[str, object]: - return {"backend": "fake", "rejected_count": 0} - - def reactivate_pool_block(self, *, block_hash: str, active_tip_height: int) -> dict[str, object]: - return {"backend": "fake", "reactivated_count": 0} - - def mark_mature_pool_payouts(self, *, active_tip_height: int) -> dict[str, object]: - return {"backend": "fake", "matured_count": 0} - - -class RecordingLedger(FakeLedger): - def __init__(self) -> None: - super().__init__(shares=0) - self.pending: list[object] = [] - self.persisted: list[dict[str, object]] = [] - self.confirmed: list[dict[str, object]] = [] - self.reversed: list[dict[str, object]] = [] - self.rejected: list[dict[str, object]] = [] - self.submit_seen = False - self._audit_publication_sequences: dict[str, int] = {} - - def append(self, pending: object) -> object: - self.pending.append(pending) - self.shares += 1 - return SimpleNamespace(share_seq=self.shares, miner_id=getattr(pending, "miner_id", "miner-a")) - - def append_recovered_share(self, pending: PendingShare) -> ShareReplayResult: - for index, existing in enumerate(self.pending, start=1): - if getattr(existing, "share_id", None) != pending.share_id: - continue - if existing != pending: - raise ShareReplayConflict(pending.share_id) - return ShareReplayResult( - "exact_existing", - SimpleNamespace(share_seq=index, miner_id=pending.miner_id), - ) - return ShareReplayResult("inserted", self.append(pending)) - - def persist_accepted_block(self, **kwargs: object) -> dict[str, object]: - self.persisted.append({**kwargs, "submit_seen_at_persist": self.submit_seen}) - return { - "backend": "fake", - "share_count": self.shares, - "block_count": 1, - "bundle_count": 1, - "payout_entry_count": 0, - "carry_forward_count": 0, - "onchain_output_count": 1, - } - - def reverse_immature_block(self, **kwargs: object) -> dict[str, object]: - self.reversed.append(kwargs) - return {"backend": "fake", "reversed_count": 1} - - def reject_prepared_block(self, **kwargs: object) -> dict[str, object]: - self.rejected.append(kwargs) - return {"backend": "fake", "rejected_count": 1} - - def confirm_accepted_block(self, **kwargs: object) -> dict[str, object]: - self.confirmed.append({**kwargs, "submit_seen_at_confirm": self.submit_seen}) - block_hash = str(kwargs.get("block_hash") or "") - sequence = self._audit_publication_sequences.get(block_hash) - already_confirmed = sequence is not None - if sequence is None: - sequence = len(self._audit_publication_sequences) + 1 - self._audit_publication_sequences[block_hash] = sequence - return { - "backend": "fake", - # Mirror the production disposition split: 1 is the fresh flip - # that allocates the ordinal, 2 the idempotent replay of a row - # this writer already confirmed, which allocates none. - "confirmed_count": 2 if already_confirmed else 1, - "audit_publication_sequence": sequence, - } - - def audit_publication_sequence_floor(self) -> int: - return max(self._audit_publication_sequences.values(), default=0) - - def all_shares(self) -> list[object]: - return [ - SimpleNamespace(miner_id=getattr(pending, "miner_id", "miner-a")) - for pending in self.pending - ] - - -class FakeRpc: - def call(self, method: str, params: list[object] | None = None) -> object: - if method == "getblockchaininfo": - return {"initialblockdownload": False} - if method == "getnetworkinfo": - return {"connections": 4} - raise RuntimeError(method) - - -class FeeEstimateRpc(FakeRpc): - def __init__(self, estimate: object) -> None: - self.estimate = estimate - self.calls: list[tuple[str, list[object] | None]] = [] - - def call(self, method: str, params: list[object] | None = None) -> object: - self.calls.append((method, params)) - if method == "estimatesmartfee": - return self.estimate - return super().call(method, params) - - -class TemplateRpc(FakeRpc): - def __init__(self, template: object) -> None: - self.template = template - self.calls: list[tuple[str, list[object] | None]] = [] - - def call(self, method: str, params: list[object] | None = None) -> object: - self.calls.append((method, params)) - if method == "getblocktemplate": - return self.template - return super().call(method, params) - - -class AddressValidationRpc(FakeRpc): - def __init__( - self, - *, - valid_address: str = PAYOUT_ADDRESS, - script_byte: str = "11", - p2mr: bool = True, - ) -> None: - self.valid_address = valid_address - self.script_byte = script_byte - self.p2mr = p2mr - self.validated: list[str] = [] - - def call(self, method: str, params: list[object] | None = None) -> object: - if method == "validateaddress": - address = str((params or [""])[0]) - self.validated.append(address) - script = "5220" + self.script_byte * 32 if self.p2mr else "51" - return {"isvalid": address == self.valid_address, "scriptPubKey": script} - return super().call(method, params) - - -AddressRpc = AddressValidationRpc - - -class TipRpc(FakeRpc): - def __init__(self, tip: str) -> None: - self.tip = tip - - def call(self, method: str, params: list[object] | None = None) -> object: - if method == "getbestblockhash": - return self.tip - if method == "getblockheader": - raise RuntimeError("qbit RPC getblockheader failed: -5 Block not found") - return super().call(method, params) - - -class RejectingSubmitTipRpc(TipRpc): - def call(self, method: str, params: list[object] | None = None) -> object: - if method == "submitblock": - return "bad-prevblk" - return super().call(method, params) - - -class ParentTipRpc(TipRpc): - def __init__(self, *, tip: str, parent: str) -> None: - super().__init__(tip) - self.parent = parent - self.submitblock_calls = 0 - - def call(self, method: str, params: list[object] | None = None) -> object: - if method == "getblock": - self.assert_tip_param(params) - return {"hash": self.tip, "previousblockhash": self.parent} - if method == "submitblock": - self.submitblock_calls += 1 - return None - return super().call(method, params) - - def assert_tip_param(self, params: list[object] | None) -> None: - if not params or str(params[0]) != self.tip: - raise AssertionError(f"expected getblock current tip {self.tip}, got {params!r}") - - -class UnsupportedBlockwaitRpc(TipRpc): - def call( - self, - method: str, - params: list[object] | None = None, - *, - timeout: float | None = None, - ) -> object: - if method == "waitfornewblock": - raise RuntimeError("Method not found") - return super().call(method, params) - - -class TipTemplateRpc(FakeRpc): - def __init__(self, *, tip: str, template: dict[str, object]) -> None: - self.tip = tip - self.template = template - self.calls: list[tuple[str, list[object] | None]] = [] - - def call(self, method: str, params: list[object] | None = None) -> object: - self.calls.append((method, params)) - if method == "getbestblockhash": - return self.tip - if method == "getblocktemplate": - return self.template - return super().call(method, params) - - -class ReorgRpc(TipTemplateRpc): - def __init__( - self, - *, - tip: str, - template: dict[str, object], - height: int, - block_hashes: dict[int, str], - initialblockdownload: bool = False, - headers: int | None = None, - ) -> None: - super().__init__(tip=tip, template=template) - self.height = height - self.block_hashes = block_hashes - self.initialblockdownload = initialblockdownload - self.headers = headers if headers is not None else height - - def call(self, method: str, params: list[object] | None = None) -> object: - if method == "getblockchaininfo": - return { - "initialblockdownload": self.initialblockdownload, - "blocks": self.height, - "headers": self.headers, - } - if method == "getblockcount": - return self.height - if method == "getblockhash": - height = int((params or [0])[0]) - try: - return self.block_hashes[height] - except KeyError as exc: - raise RuntimeError(f"unknown height {height}") from exc - return super().call(method, params) - - -class ReorgLedger(FakeLedger): - def __init__(self, rows: list[dict[str, object]]) -> None: - super().__init__(shares=0) - self.rows = [dict(row) for row in rows] - self.events: list[object] = [] - - def reorg_watch_blocks(self, *, active_tip_height: int) -> list[dict[str, object]]: - self.events.append(("watch", active_tip_height)) - return [dict(row) for row in self.rows] - - def mark_pool_block_inactive(self, *, block_hash: str, active_tip_height: int) -> dict[str, object]: - self.events.append(("inactive", block_hash, active_tip_height)) - for row in self.rows: - if str(row.get("block_hash", "")).lower() == block_hash.lower(): - row["chain_state"] = "inactive" - return {"backend": "fake", "inactive_count": 1} - return {"backend": "fake", "inactive_count": 0} - - def reactivate_pool_block(self, *, block_hash: str, active_tip_height: int) -> dict[str, object]: - self.events.append(("reactivate", block_hash, active_tip_height)) - for row in self.rows: - if str(row.get("block_hash", "")).lower() == block_hash.lower(): - row["chain_state"] = "confirmed" - return {"backend": "fake", "reactivated_count": 1} - return {"backend": "fake", "reactivated_count": 0} - - def mark_mature_pool_payouts(self, *, active_tip_height: int) -> dict[str, object]: - self.events.append(("mature", active_tip_height)) - return {"backend": "fake", "matured_count": 0} - - -class SubmitRpc(FakeRpc): - def __init__( - self, - *, - tip: str, - block_hash: str, - submit_result: object = None, - ledger: RecordingLedger | None = None, - ) -> None: - self.tip = tip - self.block_hash = block_hash - self.submit_result = submit_result - self.ledger = ledger - self.height = 9 - self.submitted = False - - def call(self, method: str, params: list[object] | None = None) -> object: - if method == "getbestblockhash": - return self.tip - if method == "getblockcount": - return self.height - if method == "submitblock": - self.submitted = True - if self.ledger is not None: - self.ledger.submit_seen = True - if self.submit_result is None: - self.height += 1 - return self.submit_result - if method == "getblockhash": - return self.block_hash - return super().call(method, params) - - -class SubmitAcceptingTemplateRpc(FakeRpc): - def __init__( - self, - *, - old_tip: str, - block_hash: str, - fail_template_after_submit: bool = False, - ledger: RecordingLedger | None = None, - ) -> None: - self.old_tip = old_tip - self.block_hash = block_hash - self.fail_template_after_submit = fail_template_after_submit - self.ledger = ledger - self.height = 9 - self.submitted = False - - def call(self, method: str, params: list[object] | None = None) -> object: - if method == "getbestblockhash": - return self.block_hash if self.submitted else self.old_tip - if method == "getblockcount": - return self.height - if method == "submitblock": - self.submitted = True - self.height += 1 - if self.ledger is not None: - self.ledger.submit_seen = True - return None - if method == "getblockhash": - return self.block_hash - if method == "getblocktemplate": - if self.submitted and self.fail_template_after_submit: - raise RuntimeError("transient getblocktemplate failure after submitblock") - previousblockhash = self.block_hash if self.submitted else self.old_tip - return gbt_template(previousblockhash, height=self.height + 1) - return super().call(method, params) - - -def client() -> ClientState: - state = ClientState(sock=object(), address=("127.0.0.1", 1), connection_id=1, extranonce1_hex="00000001") - state.subscribed = True - state.authorized = True - return state - - -def gbt_template( - previousblockhash: str, - *, - height: int = 10, - coinbasevalue: int = 50_00000000, - curtime: int = 1_700_000_000, - transactions: list[str] | None = None, -) -> dict[str, object]: - return { - "previousblockhash": previousblockhash, - "version": 0x20000000, - "bits": "207fffff", - "curtime": curtime, - "height": height, - "coinbasevalue": coinbasevalue, - "transactions": [{"data": tx_hex} for tx_hex in transactions or []], - } - - -def worker_identity(username: str = "miner-a") -> WorkerIdentity: - return WorkerIdentity( - username=username, - payout_address=username, - worker_name=None, - script_pubkey_hex="5220" + "11" * 32, - p2mr_program_hex="11" * 32, - ) - - -def stratum_job( - job_id: str, - *, - difficulty: Decimal = Decimal("1"), - clean_jobs: bool = True, -) -> SimpleNamespace: - return SimpleNamespace( - job_id=job_id, - share_difficulty=difficulty, - share_target=target_from_compact("207fffff"), - prevhash="00" * 32, - coinb1="", - coinb2="", - merkle_branch=(), - version="20000000", - nbits="207fffff", - ntime="6553f100", - clean_jobs=clean_jobs, - transaction_hexes=(), - ) - - -def prism_context( - job_id: str, - previousblockhash: str, - *, - worker: WorkerIdentity | None = None, - difficulty: Decimal = Decimal("1"), - clean_jobs: bool = True, -) -> SimpleNamespace: - return SimpleNamespace( - job=stratum_job(job_id, difficulty=difficulty, clean_jobs=clean_jobs), - template=gbt_template(previousblockhash), - found_block={"network_difficulty": 1}, - issued_at_ms=12345, - collection_only=False, - worker=worker or worker_identity(), - shares_json=[], - prior_balances=[], - ) - - -def prepare_idle_client( - server: PrismCoordinator, - state: ClientState, - *, - connection_id: int = 1, - difficulty: Decimal = Decimal("16"), - tip: str = "00" * 32, -) -> None: - if not hasattr(server, "jobs"): - server.jobs = {} - state.connection_id = connection_id - state.extranonce1_hex = f"{connection_id:08x}" - state.username = f"miner-{connection_id}" - state.worker = worker_identity(state.username) - job_id = f"job-{connection_id}" - state.active_job = prism_context( - job_id, - tip, - worker=state.worker, - difficulty=difficulty, - ) - state.active_job_ids = {job_id} - state.share_difficulty = difficulty - state.pending_share_difficulty = None - state.vardiff_window_started_monotonic = time.monotonic() - 2 - state.vardiff_window_accepted = 0 - state.vardiff_window_submitted = 0 - state.vardiff_window_work = Decimal("0") - server.clients.add(state) - server.jobs[job_id] = state.active_job - - -def install_idle_job_cache( - server: PrismCoordinator, - *, - tip: str = "00" * 32, -) -> CachedJobBundle: - server._pool_ready_latched = True - server.job_bundle_cache_seconds = 60.0 - server.job_counter = 0 - server.share_weights_by_username = {} - server.default_share_weight = 1 - server._ensure_job_cache_state() - template = gbt_template(tip) - fingerprint = qbit_template_fingerprint(template) - artifacts = CachedTemplateArtifacts( - template=template, - fingerprint=fingerprint, - previousblockhash=tip, - transaction_hexes=(), - witness_merkle_leaves_hex=(), - network_difficulty=1, - fetched_monotonic=time.monotonic(), - generation=1, - ) - key = server._job_bundle_key( - artifacts, - mode="ready", - payout_state_generation=server._payout_state_generation, - payout_artifact_generation=0, - worker=None, - ) - payout_artifact_sha256 = "aa" * 32 - qbit_target = direct_stratum.difficulty_target(Decimal("1024")) - base_job = direct_stratum.DirectQbitStratumJob( - job_id="prism-template-base", - previousblockhash_display=tip, - prevhash=tip, - coinb1="", - coinb2="", - full_coinbase_prefix="", - full_coinbase_suffix="", - merkle_branch=(), - transaction_hexes=(), - version="20000000", - nbits="207fffff", - ntime="6553f100", - qbit_target=qbit_target, - share_target=qbit_target, - share_difficulty=Decimal("1"), - extranonce1_hex="ffffffff", - extranonce2_size=8, - clean_jobs=True, - ) - bundle = CachedJobBundle( - key=key, - template=template, - template_fingerprint=fingerprint, - coinbase_manifest={}, - shares_json=[], - prior_balances=[], - found_block={"network_difficulty": 1}, - collection_only=False, - issued_at_ms=12345, - base_job=base_job, - built_monotonic=time.monotonic(), - template_generation=1, - payout_state_generation=0, - build_key=SimpleNamespace( - payout_artifact_sha256=payout_artifact_sha256, - payout_append_invalidation_epoch=( - server._payout_ledger_append_invalidation_epoch - ), - ), - ) - with server._job_cache_lock: - server._template_artifacts = artifacts - server._published_payout_state = dataclass_replace( - server._published_payout_state, - artifact=PayoutStateArtifact( - generation=server._payout_state_generation, - source_generation=0, - prior_balances_json="[]", - prior_balances_sha256=payout_artifact_sha256, - prepared_monotonic=time.monotonic(), - ), - ) - server._job_bundle_cache.clear() - server._job_bundle_cache[bundle.key] = bundle - return bundle - - -def verified_block_bundle(coinbase_tx_hex: str = "c0ffee") -> dict[str, object]: - return { - "found_block": {"coinbase_value_sats": 50_00000000}, - "ledger_window_attestation": {"signature": {"public_key_hex": "aa" * 32}}, - "payout_policy_manifest": {"accounts": []}, - "signed_coinbase_manifest": { - "manifest": { - "coinbase_tx_hex": coinbase_tx_hex, - "payout_count": 1, - } - }, - } - - -def verified_audit_report(coinbase_tx_hex: str = "c0ffee") -> dict[str, object]: - return { - "schema": "qbit.prism.audit-verification-report.v1", - "block_height": 10, - "coinbase_value_sats": 50_00000000, - "reward_manifest_sha256_hex": "44" * 32, - "payout_policy_manifest_sha256_hex": "55" * 32, - "prism_audit_commitment_leaf_hex": "66" * 32, - "audit_commitment_root_hex": "77" * 32, - "coinbase_txid": "11" * 32, - "coinbase_wtxid": "88" * 32, - "coinbase_manifest_sha256_hex": "22" * 32, - "audit_bundle_sha256_hex": "33" * 32, - "coinbase_tx_hex": coinbase_tx_hex, - "min_output_sats": 1, - "onchain_output_count": 0, - "accrued_account_count": 0, - } - - -def coordinator() -> PrismCoordinator: - server = PrismCoordinator.__new__(PrismCoordinator) - server.vardiff_config = vardiff.VardiffConfig( - enabled=True, - target_share_interval_seconds=Decimal("15"), - min_difficulty=Decimal("0.000000001"), - max_difficulty=Decimal("1024"), - retarget_interval_seconds=Decimal("1"), - max_step_factor=Decimal("4"), - startup_difficulty=Decimal("0.000000001"), - max_step_down_factor=Decimal("4"), - ewma_alpha=Decimal("1"), - retarget_tolerance=Decimal("0"), - ) - server.share_difficulty = Decimal("0.000000001") - server.lock = threading.RLock() - server.stop_event = threading.Event() - server.clients = set() - server.submitted_share_count = 0 - server.stale_share_count = 0 - server.duplicate_share_count = 0 - server.low_difficulty_share_count = 0 - server.collection_block_submission_count = 0 - server._pool_ready_latched = False - server.grace_credited_share_count = 0 - server.idle_retarget_count = 0 - server.rejection_counts_by_reason = {reason: 0 for reason in PRISM_REJECTION_REASON_IDS} - server.worker_metrics_limit = 100 - server.worker_metrics_lock = threading.Lock() - server.worker_share_counts = {} - server.worker_rejection_counts = {} - server.evicted_job_graveyard = {} - server.block_candidate_queue = queue.Queue(maxsize=8) - server.block_candidates_dropped = 0 - server.block_candidate_abandoned_counts = {} - server.share_append_queue = queue.Queue(maxsize=8) - server.share_writer_active = False - server.share_append_failure_count = 0 - server.share_recovery_path = None - server.share_recovery_lock = threading.Lock() - server.shares_recovered_to_disk = 0 - server.shares_replayed = 0 - server.current_tip_first_seen = None - server.current_tip_parent = None - server.stale_grace_seconds = 3.0 - server.blockwait_enabled = True - server.blockwait_timeout_seconds = 5.0 - server.vardiff_idle_sweep_seconds = 15.0 - server.job_build_failure_count = 0 - server.tip_refresh_job_count = 0 - server.post_accept_refresh_failure_count = 0 - server.reorg_reconciler_enabled = False - server.reorg_inactive_block_count = 0 - server.reorg_reactivated_block_count = 0 - server.reorg_reconcile_skip_count = 0 - server.reorg_reconcile_error_count = 0 - server.matured_payout_count = 0 - server.last_reorg_reconciled_tip_hash = None - server.last_reorg_reconciled_trusted = False - server.accepted_block_count = 1 - server.started_monotonic = time.monotonic() - 10 - server.ledger = FakeLedger(shares=5) - server.latest_coinbase_size_bytes = 250 - server.rpc = FakeRpc() - server.qbit_chain = "regtest" - server.blockpoll_seconds = 2.0 - # Failed-refresh spacing is opt-in per test: its holdoff waits on real - # time, which deadlocks tests that freeze time.monotonic around failing - # polls. Pacing behavior is covered by test_prism_refresh_retry_pacing. - server.tip_refresh_failure_holdoff_seconds = 0.0 - server.ctv_broadcaster_enabled = False - server.ctv_broadcaster_wallet = None - server.ctv_broadcaster_fee_sats = 0 - server.ctv_broadcaster_limit = 100 - server.ctv_broadcaster_interval_seconds = 30.0 - server.ctv_fanout_broadcast_daemon = None - server._ctv_fanout_market_fee_rate_cache = {} - server.tip_template_snapshot = None - server._tip_refresh_lock = threading.Lock() - server.extranonce2_size = 8 - server.coinbase_tag_hex = default_prism_coinbase_tag_hex() - server.version_mask = direct_stratum.QBIT_VERSION_ROLLING_MASK - server.version_mask_selection = direct_stratum.VersionRollingMaskSelection( - direct_stratum.QBIT_VERSION_ROLLING_MASK, - "fallback", - "test", - ) - return server - - -def submit_coordinator(tip: str = "00" * 32) -> tuple[PrismCoordinator, ClientState, RecordingLedger]: - server = coordinator() - # This suite runs on a synthetic millisecond timeline (declared anchors - # like 12000, share stamps of 1-2) while _expose_inflight_scan_anchor - # prunes exposures older than a real-wall-clock ceiling. Every synthetic - # anchor is decades past that ceiling, so the next exposure (a landing - # publishing its declared anchor) silently pruned a test's standing - # anchor and turned epoch-bump assertions into thread races against the - # landing's own anchor retirement. Pin the ceiling out of reach so - # exposed anchors live exactly as long as each test intends. - server.payout_artifact_max_anchor_age_seconds = float("inf") - server.vardiff_config = SimpleNamespace(enabled=False) - server.rpc = TipRpc(tip) - server.jobs = {} - server.recent_share_keys = set() - server.accepted_block_count = 0 - server.max_blocks = 1 - server.stop_after_block = True - server.extranonce2_size = 8 - server.share_weights_by_username = {"miner-a": 7} - ledger = RecordingLedger() - server.ledger = ledger - worker = WorkerIdentity( - username="miner-a", - payout_address="miner-a", - worker_name=None, - script_pubkey_hex="5220" + "11" * 32, - p2mr_program_hex="11" * 32, - ) - context = SimpleNamespace( - job=SimpleNamespace( - job_id="job-1", - share_target=target_from_compact("207fffff"), - share_difficulty=Decimal("1"), - transaction_hexes=(), - ), - template={"previousblockhash": tip, "height": 10, "coinbasevalue": 50_00000000}, - found_block={"network_difficulty": 1}, - issued_at_ms=12345, - collection_only=False, - worker=worker, - shares_json=[], - prior_balances=[], - ) - state = client() - state.username = "miner-a" - state.worker = worker - state.active_job_ids = {"job-1"} - server.jobs["job-1"] = context - return server, state, ledger - - -def block_candidate( - server: PrismCoordinator, - state: ClientState, - submission: object, - *, - job_id: str = "job-1", - pending_share: object | None = None, - credit_share_on_accept: bool = False, -) -> PrismBlockCandidate: - return PrismBlockCandidate( - context=server.jobs[job_id], - submission=submission, - extranonce1_hex=state.extranonce1_hex, - extranonce2_hex="00" * 8, - pending_share=pending_share - or SimpleNamespace( - share_id="miner-a:" + submission.block_hash_hex, - job_issued_at_ms=0, - accepted_at_ms=0, - ), - client=state, - credit_share_on_accept=credit_share_on_accept, - ) - - -def highdiff_vardiff_config(**overrides: object) -> vardiff.VardiffConfig: - values: dict[str, object] = dict( - enabled=True, - target_share_interval_seconds=Decimal("15"), - min_difficulty=Decimal("500000"), - max_difficulty=Decimal("4294967296"), - retarget_interval_seconds=Decimal("1"), - max_step_factor=Decimal("4"), - startup_difficulty=Decimal("500000"), - max_step_down_factor=Decimal("4"), - ewma_alpha=Decimal("1"), - retarget_tolerance=Decimal("0"), - ) - values.update(overrides) - return vardiff.VardiffConfig(**values) # type: ignore[arg-type] - - -def clear_stratum_diff_env() -> None: - for name in [ - name - for name in os.environ - if name.startswith("PRISM_STRATUM_HIGHDIFF") or name.startswith("PRISM_STRATUM_VARDIFF") - ]: - os.environ.pop(name, None) - - -class LostAckSubmitRpc(FakeRpc): - """qbitd accepts submitted blocks while every submitblock ack is lost. - - Models the mainnet 2026-07-25 interleaving: the block connects (the tip - flips to the candidate hash, so blockwait reports it) but the RPC - response never reaches the coordinator, and the retry disposition can - then read a transient racing snapshot in which the tip has moved on and - the header probe cannot prove the candidate active. - """ - - def __init__(self, *, start_tip: str, hash_by_hex: dict[str, str]) -> None: - self.tip = start_tip - self.height = 9 - self.hash_by_hex = dict(hash_by_hex) - self.active: dict[str, int] = {} - self.racing_tip: str | None = None - self.getblockhash_override: str | None = None - self.lose_acks = True - self.submitblock_calls = 0 - - def call(self, method: str, params: list[object] | None = None) -> object: - if method == "getbestblockhash": - return self.racing_tip or self.tip - if method == "getblockcount": - return self.height - if method == "submitblock": - self.submitblock_calls += 1 - block_hash = self.hash_by_hex[str((params or [""])[0])] - if block_hash in self.active: - return "duplicate" - if self.racing_tip is not None: - return "bad-prevblk" - self.height += 1 - self.active[block_hash] = self.height - self.tip = block_hash - if self.lose_acks: - raise OSError("connection reset by peer before submitblock ack") - return None - if method == "getblockheader": - requested = str((params or [""])[0]).lower() - if self.racing_tip is None and requested in self.active: - block_height = self.active[requested] - return { - "height": block_height, - "confirmations": self.height - block_height + 1, - } - raise RuntimeError("qbit RPC getblockheader failed: -5 Block not found") - if method == "getblockhash": - if self.getblockhash_override is not None: - return self.getblockhash_override - requested_height = int((params or [0])[0]) - for block_hash, block_height in self.active.items(): - if block_height == requested_height: - return block_hash - raise RuntimeError(f"unknown height {requested_height}") - return super().call(method, params) - - -class AcceptanceProbeRpc(FakeRpc): - def __init__( - self, - *, - tip: str, - header: dict[str, object] | None = None, - fail: bool = False, - fail_tip: bool = False, - ) -> None: - self.tip = tip - self.header = header - self.fail = fail - self.fail_tip = fail_tip - - def call(self, method: str, params: list[object] | None = None) -> object: - if self.fail: - raise RuntimeError("qbit RPC unavailable") - if method == "getbestblockhash": - if self.fail_tip: - raise RuntimeError("qbit RPC getbestblockhash unavailable") - return self.tip - if method == "getblockheader": - if self.header is None: - raise RuntimeError("qbit RPC getblockheader failed: -5 Block not found") - return self.header - return super().call(method, params) - - -def build_vardiff_test_coordinator() -> PrismCoordinator: - """Build a coordinator fixture without production init or worker startup.""" - return coordinator() - - -__all__ = [name for name in globals() if not name.startswith("__")] diff --git a/tests/prism_window_rows_reference.py b/tests/prism_window_rows_reference.py deleted file mode 100644 index a52a868e0..000000000 --- a/tests/prism_window_rows_reference.py +++ /dev/null @@ -1,97 +0,0 @@ -"""Reference check for the per-row payout-window reads (issue #236). - -The payout-window snapshot and delta project one JSON object per row and -decode it in bounded batches. Their selection CTEs did not change, so the -pre-#236 statement -- the same CTE with the rows folded into one ``json_agg`` -value -- is a byte-exact oracle for membership, order and field values. This -module rebuilds that oracle from the statement the ledger actually sends and -compares the two on a live PostgreSQL, whichever backend the ledger runs. - -Used by ``test/test-prism-postgres-ledger.sh`` (psql subprocess backend) and -``test/test-prism-postgres-native-ledger.sh`` (pooled psycopg backend). -""" - -from __future__ import annotations - -import re -from typing import Any, Callable - -from lab.prism.share_ledger import AcceptedShareRecord, PsqlShareLedger - -_ROW_PROJECTION = re.compile( - r"SELECT (?P<object>json_build_object\(.*?\))\nFROM rows\nORDER BY share_seq ASC;", - re.S, -) - - -def aggregate_reference_sql(row_sql: str) -> str: - """Turn the per-row statement back into its pre-#236 ``json_agg`` form.""" - rewritten, count = _ROW_PROJECTION.subn( - lambda match: ( - "SELECT COALESCE(json_agg(" - + match.group("object") - + " ORDER BY share_seq ASC), '[]'::json)\nFROM rows;" - ), - row_sql, - ) - if count != 1: - raise AssertionError( - "expected exactly one per-row projection in the window statement, " - f"found {count}" - ) - if "json_agg" in row_sql: - raise AssertionError("the per-row statement still carries a json_agg") - return rewritten - - -def capture_window_sql( - ledger: PsqlShareLedger, - call: Callable[[], list[AcceptedShareRecord]], -) -> tuple[str, list[AcceptedShareRecord]]: - """Run ``call`` on ``ledger`` and return the row statement it sent.""" - statements: list[str] = [] - original = ledger._run_retry_safe_read_json_rows - - def recording(sql: str, **kwargs: Any) -> list[Any]: - statements.append(sql) - return original(sql, **kwargs) - - ledger._run_retry_safe_read_json_rows = recording # type: ignore[method-assign] - try: - records = call() - finally: - del ledger._run_retry_safe_read_json_rows - if len(statements) != 1: - raise AssertionError( - f"expected one row statement per window read, saw {len(statements)}" - ) - return statements[0], records - - -def assert_rows_match_aggregate( - ledger: PsqlShareLedger, - call: Callable[[], list[AcceptedShareRecord]], - *, - label: str, - expected_len: int | None = None, -) -> list[AcceptedShareRecord]: - """Assert the per-row read equals the ``json_agg`` oracle, record for record.""" - row_sql, records = capture_window_sql(ledger, call) - reference_sql = aggregate_reference_sql(row_sql) - reference = [ - ledger._record_from_json(item) for item in ledger._run_json(reference_sql) - ] - if records != reference: - detail = ( - f"rows={len(records)} reference={len(reference)}" - if len(records) != len(reference) - else next( - f"first mismatch at index {index}: {left!r} != {right!r}" - for index, (left, right) in enumerate(zip(records, reference)) - if left != right - ) - ) - raise SystemExit(f"{label}: per-row window read differs from json_agg oracle ({detail})") - if expected_len is not None and len(records) != expected_len: - raise SystemExit(f"{label}: expected {expected_len} records, got {len(records)}") - return records diff --git a/tests/prism_worker_difficulty_store_gate.py b/tests/prism_worker_difficulty_store_gate.py deleted file mode 100644 index 5498b7660..000000000 --- a/tests/prism_worker_difficulty_store_gate.py +++ /dev/null @@ -1,427 +0,0 @@ -"""Non-discovered PostgreSQL parity gate for the worker-difficulty store. - -Drives :class:`MemoryWorkerDifficultyStore` and -:class:`PostgresWorkerDifficultyStore` through the same operation sequence and -requires identical results after every step, then checks the PostgreSQL-only -properties (schema reapply safety, CHECK constraint enforcement, index-backed -plans). It lives outside unittest discovery because it needs an explicitly -provisioned PostgreSQL target; run it from the repository root: - - PRISM_PSQL_COMMAND="psql -p 5432 -d some_scratch_db" \\ - python3 -m tests.prism_worker_difficulty_store_gate - -The target database is written to (the monolithic ledger schema is applied -into the default schema); use a scratch database. -""" - -from __future__ import annotations - -import json -import os -import shlex -import subprocess -import sys -from decimal import Decimal -from pathlib import Path - -from lab.prism.worker_difficulty_store import ( - MemoryWorkerDifficultyStore, - PostgresWorkerDifficultyStore, - WorkerDifficultyRecord, -) - -ROOT = Path(__file__).resolve().parents[1] -LEDGER_SCHEMA = ROOT / "crates" / "qbit-prism" / "sql" / "001_share_ledger.sql" -PSQL_TIMEOUT_SECONDS = 60.0 - -BASE_PSQL_COMMAND = os.environ.get("PRISM_PSQL_COMMAND", "") -if not BASE_PSQL_COMMAND: - raise SystemExit("PRISM_PSQL_COMMAND is required") -BASE_PSQL_ARGV = shlex.split(BASE_PSQL_COMMAND) -if not BASE_PSQL_ARGV: - raise SystemExit("PRISM_PSQL_COMMAND is empty") - -BASE_MS = 1_756_000_000_000 - - -class GateFailure(RuntimeError): - pass - - -def assert_equal(actual: object, expected: object, message: str) -> None: - if actual != expected: - raise GateFailure(f"{message}: expected {expected!r}, got {actual!r}") - - -def assert_true(condition: bool, message: str) -> None: - if not condition: - raise GateFailure(message) - - -def run_psql(sql: str, *, expect_failure: bool = False) -> str: - command = [ - *BASE_PSQL_ARGV, - "--no-psqlrc", - "--set", - "ON_ERROR_STOP=1", - "--tuples-only", - "--no-align", - "--quiet", - ] - completed = subprocess.run( - command, - input=sql, - text=True, - stdout=subprocess.PIPE, - stderr=subprocess.PIPE, - timeout=PSQL_TIMEOUT_SECONDS, - check=False, - ) - if expect_failure: - if completed.returncode == 0: - raise GateFailure(f"psql unexpectedly succeeded for: {sql.strip()[:200]}") - return completed.stderr - if completed.returncode != 0: - raise GateFailure( - f"psql failed with exit {completed.returncode}: {completed.stderr.strip()}" - ) - return completed.stdout - - -def run_json(sql: str) -> object: - output = run_psql(sql).strip() - if not output: - raise GateFailure(f"psql query returned no JSON: {sql.strip()[:200]}") - return json.loads(output.splitlines()[-1]) - - -def run_script(sql: str) -> None: - run_psql(sql) - - -def record_key(record: WorkerDifficultyRecord) -> tuple[str, str]: - return (record.listener, record.worker_username) - - -def check_parity( - memory: MemoryWorkerDifficultyStore, - postgres: PostgresWorkerDifficultyStore, - message: str, -) -> None: - """Full-table equality via the bounded load with a generous limit.""" - memory_rows = memory.load_recent(evidence_after_ms=0, limit=10_000) - postgres_rows = postgres.load_recent(evidence_after_ms=0, limit=10_000) - assert_equal(postgres_rows, memory_rows, message) - - -def main() -> None: - schema_sql = LEDGER_SCHEMA.read_text(encoding="utf-8") - - # Idempotent schema application: the monolithic file applies cleanly - # twice, and the focused module DDL reapplies over it a third time. - run_psql(schema_sql) - run_psql(schema_sql) - - memory = MemoryWorkerDifficultyStore() - postgres = PostgresWorkerDifficultyStore(run_json, run_script=run_script) - postgres.ensure_schema() - postgres.ensure_schema() - - # Clean slate in case the scratch database was reused. - run_psql("DELETE FROM qbit_worker_difficulty;") - - def upsert_both(**kwargs: object) -> None: - memory_outcome = memory.upsert(**kwargs) # type: ignore[arg-type] - postgres_outcome = postgres.upsert(**kwargs) # type: ignore[arg-type] - assert_equal( - postgres_outcome, - memory_outcome, - f"upsert outcome parity for {kwargs!r}", - ) - - # Insert across two lanes, same username in both; lookalike usernames in - # one lane; a quote-bearing username to exercise literal quoting. - upsert_both( - listener="stratum-public", - worker_username="miner.worker1", - difficulty=Decimal("16"), - evidence_at_ms=BASE_MS + 1_000, - now_ms=BASE_MS + 1_000, - ) - upsert_both( - listener="stratum-highdiff", - worker_username="miner.worker1", - difficulty=Decimal("65536"), - evidence_at_ms=BASE_MS + 2_000, - now_ms=BASE_MS + 2_000, - ) - upsert_both( - listener="stratum-public", - worker_username="Miner.worker1", - difficulty=Decimal("32"), - evidence_at_ms=BASE_MS + 3_000, - now_ms=BASE_MS + 3_000, - ) - upsert_both( - listener="stratum-public", - worker_username=" miner.worker1", - difficulty=Decimal("64"), - evidence_at_ms=BASE_MS + 4_000, - now_ms=BASE_MS + 4_000, - ) - upsert_both( - listener="stratum-public", - worker_username="o'brien.worker1", - difficulty=Decimal("128"), - evidence_at_ms=BASE_MS + 5_000, - now_ms=BASE_MS + 5_000, - ) - check_parity(memory, postgres, "parity after inserts") - - # Update with newer evidence applies; the lane sibling is untouched. - upsert_both( - listener="stratum-public", - worker_username="miner.worker1", - difficulty=Decimal("24"), - evidence_at_ms=BASE_MS + 6_000, - now_ms=BASE_MS + 6_500, - ) - # Older evidence is refused even with a later wall-clock write time. - upsert_both( - listener="stratum-public", - worker_username="miner.worker1", - difficulty=Decimal("1"), - evidence_at_ms=BASE_MS + 5_999, - now_ms=BASE_MS + 9_000, - ) - # Equal evidence is deterministic last-write-wins. - upsert_both( - listener="stratum-public", - worker_username="miner.worker1", - difficulty=Decimal("48"), - evidence_at_ms=BASE_MS + 6_000, - now_ms=BASE_MS + 9_500, - ) - check_parity(memory, postgres, "parity after evidence-guarded updates") - - def lower_both(**kwargs: object) -> None: - memory_outcome = memory.apply_downward(**kwargs) # type: ignore[arg-type] - postgres_outcome = postgres.apply_downward(**kwargs) # type: ignore[arg-type] - assert_equal( - postgres_outcome, - memory_outcome, - f"apply_downward outcome parity for {kwargs!r}", - ) - - # The lower-only correction: reduces a higher row, refuses to raise one, - # is a no-op on an equal value or a missing key, and never touches - # evidence_at -- so a later share-backed write whose evidence predates the - # correction still applies. - lower_both( - listener="stratum-public", - worker_username="miner.worker1", - difficulty=Decimal("8"), - now_ms=BASE_MS + 20_000, - ) - lower_both( - listener="stratum-public", - worker_username="miner.worker1", - difficulty=Decimal("4096"), - now_ms=BASE_MS + 20_100, - ) - lower_both( - listener="stratum-public", - worker_username="miner.worker1", - difficulty=Decimal("8.000"), - now_ms=BASE_MS + 20_200, - ) - lower_both( - listener="stratum-public", - worker_username="never.seen", - difficulty=Decimal("8"), - now_ms=BASE_MS + 20_300, - ) - check_parity(memory, postgres, "parity after lower-only corrections") - assert_true( - not any( - record.worker_username == "never.seen" - for record in postgres.load_recent(evidence_after_ms=0, limit=10_000) - ), - "apply_downward must never insert a row for an unknown worker", - ) - # Evidence older than the correction's wall clock, newer than the row's: - # the correction wrote no evidence, so this must still apply. - upsert_both( - listener="stratum-public", - worker_username="miner.worker1", - difficulty=Decimal("512"), - evidence_at_ms=BASE_MS + 6_001, - now_ms=BASE_MS + 21_000, - ) - check_parity( - memory, - postgres, - "a lower-only correction must not suppress later share evidence", - ) - - # Decimal exactness through the durable round trip. - exact_values = [ - Decimal("0.000030517578125"), # 2**-15 - Decimal("123456789012345678901234567890.5"), - Decimal("1E-12"), - Decimal("8192.000"), - ] - for index, difficulty in enumerate(exact_values): - upsert_both( - listener="decimal-lane", - worker_username=f"miner.exact{index}", - difficulty=difficulty, - evidence_at_ms=BASE_MS + 10_000 + index, - now_ms=BASE_MS + 10_000 + index, - ) - loaded = postgres.load_recent(evidence_after_ms=BASE_MS + 9_999, limit=10) - by_user = {entry.worker_username: entry.difficulty for entry in loaded} - for index, difficulty in enumerate(exact_values): - assert_equal( - by_user[f"miner.exact{index}"], - difficulty, - f"decimal round trip for {difficulty}", - ) - check_parity(memory, postgres, "parity after decimal round-trip rows") - - # Bounded newest-first load: limit and deterministic ordering pinned, - # cutoff strictly exclusive. Both stores must agree element-for-element. - for cutoff, limit in [ - (0, 3), - (BASE_MS + 4_000, 2), - (BASE_MS + 6_000, 10), - (BASE_MS + 10_003, 10), - ]: - assert_equal( - postgres.load_recent(evidence_after_ms=cutoff, limit=limit), - memory.load_recent(evidence_after_ms=cutoff, limit=limit), - f"bounded load parity (cutoff={cutoff}, limit={limit})", - ) - top = postgres.load_recent(evidence_after_ms=0, limit=3) - assert_equal( - [record_key(entry) for entry in top], - [ - ("decimal-lane", "miner.exact3"), - ("decimal-lane", "miner.exact2"), - ("decimal-lane", "miner.exact1"), - ], - "newest-first pinned ordering", - ) - boundary = postgres.load_recent(evidence_after_ms=BASE_MS + 10_003, limit=10) - assert_equal(boundary, [], "strict cutoff excludes the boundary row") - - # Bounded prune removes the deterministic oldest set; both back ends - # agree on the count and on every survivor. - memory_deleted = memory.prune(evidence_cutoff_ms=BASE_MS + 5_000, limit=2) - postgres_deleted = postgres.prune(evidence_cutoff_ms=BASE_MS + 5_000, limit=2) - assert_equal(postgres_deleted, memory_deleted, "bounded prune count parity") - assert_equal(postgres_deleted, 2, "bounded prune removes exactly the limit") - check_parity(memory, postgres, "parity after bounded prune") - - # Unbounded prune clears the rest of the old range store-side. - memory_deleted = memory.prune(evidence_cutoff_ms=BASE_MS + 6_000) - postgres_deleted = postgres.prune(evidence_cutoff_ms=BASE_MS + 6_000) - assert_equal(postgres_deleted, memory_deleted, "prune count parity") - check_parity(memory, postgres, "parity after prune") - - # A no-op prune below every remaining row deletes nothing. - assert_equal(postgres.prune(evidence_cutoff_ms=1), memory.prune(evidence_cutoff_ms=1), "no-op prune parity") - - # Ordering ties: several keys share one evidence timestamp, so the - # (listener, worker_username) byte-order tie-break itself is what both - # back ends must agree on (COLLATE "C" pins the PostgreSQL side to byte - # order regardless of the database's locale collation). - tie_usernames = ["zeta", "Alpha", " alpha", "alpha", "ALPHA.9"] - for tie_index, username in enumerate(tie_usernames): - upsert_both( - listener="tie-lane", - worker_username=username, - difficulty=Decimal(tie_index + 1), - evidence_at_ms=BASE_MS + 20_000, - now_ms=BASE_MS + 20_000, - ) - tie_memory = memory.load_recent(evidence_after_ms=BASE_MS + 19_999, limit=10) - tie_postgres = postgres.load_recent(evidence_after_ms=BASE_MS + 19_999, limit=10) - assert_equal(tie_postgres, tie_memory, "tie-break ordering parity") - assert_equal( - [entry.worker_username for entry in tie_postgres], - sorted(tie_usernames), - "tie-break is byte order", - ) - check_parity(memory, postgres, "parity with the tie group present") - - # Schema-level validation: the CHECK constraints refuse what the Python - # validation refuses, so a writer bypassing this module cannot poison the - # preload path with NaN/Infinity/non-positive difficulties or empty keys. - for bad_row in ( - "('lane', 'user-nan', 'NaN'::numeric, now(), now())", - "('lane', 'user-inf', 'Infinity'::numeric, now(), now())", - "('lane', 'user-zero', 0, now(), now())", - "('lane', 'user-negative', -5, now(), now())", - "('', 'user-empty-lane', 1, now(), now())", - "('lane', '', 1, now(), now())", - ): - stderr = run_psql( - "INSERT INTO qbit_worker_difficulty " - "(listener, worker_username, difficulty, evidence_at, updated_at) " - f"VALUES {bad_row};", - expect_failure=True, - ) - assert_true( - "violates check constraint" in stderr or "not-null constraint" in stderr, - f"unexpected rejection reason for {bad_row}: {stderr.strip()}", - ) - - # The bounded newest-first load plans through the evidence index rather - # than a sequential scan once the planner is told scans are expensive - # (the table is tiny here, so seqscan is otherwise legitimately cheaper). - plan = run_psql( - """ -BEGIN; -SET LOCAL enable_seqscan = off; -EXPLAIN (COSTS OFF) -SELECT listener, worker_username, difficulty -FROM qbit_worker_difficulty -WHERE evidence_at > now() - interval '1 hour' -ORDER BY evidence_at DESC, listener, worker_username -LIMIT 100; -ROLLBACK; -""" - ) - assert_true( - "qbit_worker_difficulty_evidence_idx" in plan, - f"bounded load did not use the evidence index:\n{plan}", - ) - assert_true("Sort" not in plan, f"bounded load required an explicit sort:\n{plan}") - - # Canonical share/payout tables are untouched by this gate's traffic. - counts = run_json( - """ -SELECT json_build_object( - 'shares', (SELECT count(*) FROM qbit_share_ledger), - 'blocks', (SELECT count(*) FROM qbit_pool_blocks), - 'payouts', (SELECT count(*) FROM qbit_pool_payout_entries), - 'carry', (SELECT count(*) FROM qbit_payout_carry_forward) -); -""" - ) - assert_equal( - counts, - {"shares": 0, "blocks": 0, "payouts": 0, "carry": 0}, - "canonical tables must stay empty", - ) - - print("prism worker-difficulty store gate: all checks passed") - - -if __name__ == "__main__": - try: - main() - except GateFailure as failure: - print(f"GATE FAILURE: {failure}", file=sys.stderr) - raise SystemExit(1) diff --git a/tests/test_check_deleted_test_map.py b/tests/test_check_deleted_test_map.py new file mode 100644 index 000000000..ca303fc96 --- /dev/null +++ b/tests/test_check_deleted_test_map.py @@ -0,0 +1,1264 @@ +#!/usr/bin/env python3 + +from __future__ import annotations + +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer +import json +import os +import re +import subprocess +import sys +import tempfile +import threading +import time +import unittest +from pathlib import Path + + +ROOT = Path(__file__).resolve().parents[1] +SCRIPT = ROOT / "scripts" / "check_deleted_test_map.py" +CI_WORKFLOW = ROOT / ".github" / "workflows" / "ci.yml" +ISSUES = "https://github.com/Qbit-Org/qbit-mining-bootstrap/issues" + +FILE_MANIFEST = """# Deleted files, independent of the map under test. +tests/test_a.py +tests/test_b.py +tests/test_c.py +tests/test_d.py +tests/test_e.py +""" + +CASE_MANIFEST = """# Deleted cases, independent of the map under test. +test_case_one +""" + +MAP = f"""# Deleted 2.x.x tests and their 3.x.x replacements + +Regenerate the file list with: + +``` +| this fenced line | is not | a table | +``` + +## Summary + +| Rows | Full | Partial | Open gap | Needs triage | Retired | +| --- | --- | --- | --- | --- | --- | +| 5 | 1 | 1 | 1 | 1 | 1 | + +Sections and row counts: + +- Ledger (3) +- Stratum (2) + +## Ledger + +`server::run` and `#[tokio::test]` are code, not test references. [#5]({ISSUES}/5) +shipped the ledger; prose may link a closed issue as history. + +| 2.x.x file | Covered | 3.x.x replacement or status | +| --- | --- | --- | +| `tests/test_a.py` | full | `crates/demo/tests/ledger.rs::lands_one_block` (asserts `a \\| b`), `crates/demo/tests/ledger.rs::lands_two_blocks` | +| `tests/test_b.py` | partial | partial: covered by `tests/test_kept.py::test_kept`, written when [#6]({ISSUES}/6) landed; not covered: the rest (→ [#7]({ISSUES}/7)) | <!-- retired-setting: PRISM_DEMO --> +| `tests/test_c.py` | retired | retired — Python only | + +| 2.x.x case | Covered | 3.x.x replacement or status | +| --- | --- | --- | +| `test_case_one` | full | `crates/demo/tests/ledger.rs::lands_one_block` | + +## Stratum + +| 2.x.x file | Covered | 3.x.x replacement or status | +| --- | --- | --- | +| `tests/test_d.py` | open gap | open gap → [#8]({ISSUES}/8) | +| `tests/test_e.py` | needs triage | needs triage — nobody owns it, see [#9]({ISSUES}/9) | + +## Needs triage + +| 2.x.x file | Area | Reason | +| --- | --- | --- | +| `tests/test_e.py` | Stratum | nobody owns it | +""" + +RUST = """use support::fixture; + +fn helper_that_is_not_a_test() {} + +#[test] +fn lands_one_block() {} + +// A comment between the attribute and the function is fine. +#[tokio::test( + flavor = "multi_thread", + worker_threads = 2 +)] +// Another comment. +async fn lands_two_blocks() {} +""" + +PYTHON = """import unittest + + +def helper(): + pass + + +class KeptTests(unittest.TestCase): + def test_kept(self): + pass +""" + + +def write_cargo_package(root: Path, package: str) -> Path: + directory = root / "crates" / package + directory.mkdir(parents=True, exist_ok=True) + (directory / "Cargo.toml").write_text( + f'[package]\nname = "{package}"\nversion = "0.1.0"\nedition = "2021"\n', encoding="utf-8" + ) + return directory + + +def write_tree( + root: Path, text: str = MAP, *, manifest: str | None = FILE_MANIFEST, case_manifest: str | None = CASE_MANIFEST +) -> None: + (root / "docs").mkdir() + (root / "docs" / "prism-deleted-test-map.md").write_text(text, encoding="utf-8") + if manifest is not None: + (root / "docs" / "prism-deleted-test-files.txt").write_text(manifest, encoding="utf-8") + if case_manifest is not None: + (root / "docs" / "prism-deleted-test-cases.txt").write_text(case_manifest, encoding="utf-8") + (root / "crates" / "demo" / "tests").mkdir(parents=True) + (root / "crates" / "demo" / "tests" / "ledger.rs").write_text(RUST, encoding="utf-8") + (root / "tests").mkdir() + (root / "tests" / "test_kept.py").write_text(PYTHON, encoding="utf-8") + (root / "Cargo.toml").write_text('[workspace]\nmembers = ["crates/*"]\nresolver = "2"\n', encoding="utf-8") + write_cargo_package(root, "demo") + + +def edited(old: str, new: str) -> str: + # A fixture edit that matches nothing would turn a failing-case test into a passing one. + assert MAP.count(old) == 1, old + return MAP.replace(old, new) + + +def line_of(fragment: str, text: str = MAP) -> int: + lines = [number for number, line in enumerate(text.splitlines(), start=1) if fragment in line] + assert len(lines) == 1, (fragment, lines) + return lines[0] + + +def run_check(root: Path, *args: str, env: dict[str, str] | None = None) -> subprocess.CompletedProcess[str]: + merged = {key: value for key, value in os.environ.items() if key not in ("GITHUB_TOKEN", "GH_TOKEN", "GITHUB_API_URL")} + merged.update(env or {}) + return subprocess.run( + [sys.executable, str(SCRIPT), "--root", str(root), *args], + cwd=ROOT, + env=merged, + text=True, + capture_output=True, + check=False, + ) + + +class GitHubStub: + """A local stand-in for the GitHub issues API with scripted replies per issue.""" + + def __init__(self, replies: dict[int, list[tuple[int, object, float]]]) -> None: + self.replies = replies + self.requests: list[tuple[int, str | None]] = [] + stub = self + + class Handler(BaseHTTPRequestHandler): + def do_GET(self) -> None: # noqa: N802 - the base class names it + found = re.fullmatch(r"/repos/Qbit-Org/qbit-mining-bootstrap/issues/([0-9]+)", self.path) + number = int(found.group(1)) if found else -1 + stub.requests.append((number, self.headers.get("Authorization"))) + scripted = stub.replies.get(number, [(404, {"message": "Not Found"}, 0.0)]) + status, body, delay = scripted.pop(0) if len(scripted) > 1 else scripted[0] + time.sleep(delay) + payload = body if isinstance(body, bytes) else json.dumps(body).encode("utf-8") + try: + self.send_response(status) + self.send_header("Content-Type", "application/json") + self.send_header("Content-Length", str(len(payload))) + self.end_headers() + self.wfile.write(payload) + except OSError: + pass # the client gave up first, which is what the deadline test wants + + def log_message(self, *args: object) -> None: + pass + + self.server = ThreadingHTTPServer(("127.0.0.1", 0), Handler) + self.server.daemon_threads = True + self.url = f"http://127.0.0.1:{self.server.server_address[1]}" + self.thread = threading.Thread(target=self.server.serve_forever, daemon=True) + + def __enter__(self) -> GitHubStub: + self.thread.start() + return self + + def __exit__(self, *exc: object) -> None: + self.server.shutdown() + self.server.server_close() + + def asked(self) -> list[int]: + return [number for number, _ in self.requests] + + +def state(value: str) -> list[tuple[int, object, float]]: + return [(200, {"state": value}, 0.0)] + + +ALL_OPEN = {6: state("open"), 7: state("open"), 8: state("open"), 9: state("open")} + + +class OfflineTests(unittest.TestCase): + def check( + self, text: str = MAP, *, manifest: str | None = FILE_MANIFEST, case_manifest: str | None = CASE_MANIFEST + ) -> subprocess.CompletedProcess[str]: + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + write_tree(root, text, manifest=manifest, case_manifest=case_manifest) + return run_check(root) + + def assert_fails(self, text: str, *expected: str) -> None: + result = self.check(text) + self.assertEqual(result.returncode, 1, result.stderr) + for fragment in expected: + self.assertIn(fragment, result.stderr) + + def test_the_repository_map_holds(self) -> None: + result = subprocess.run([sys.executable, str(SCRIPT)], cwd=ROOT, text=True, capture_output=True, check=False) + self.assertEqual(result.returncode, 0, result.stderr) + self.assertIn("142 file rows", result.stdout) + self.assertIn("17 case rows", result.stdout) + + def test_a_consistent_map_passes_with_escaped_pipes_markers_and_fences(self) -> None: + result = self.check() + self.assertEqual(result.returncode, 0, result.stderr) + self.assertIn("ok (5 file rows, 1 case rows)", result.stdout) + + def test_a_renamed_file_fails_even_when_counts_are_unchanged(self) -> None: + self.assert_fails( + edited("`tests/test_c.py`", "`tests/test_c_typo.py`"), + f":{line_of('`tests/test_c.py`')}: file `tests/test_c_typo.py` is not in the deleted-file manifest", + "deleted file `tests/test_c.py` is missing from the file tables", + ) + + def test_a_missing_file_fails_even_when_counts_are_reconciled(self) -> None: + text = edited("| `tests/test_c.py` | retired | retired — Python only |\n", "") + text = text.replace("| 5 | 1 | 1 | 1 | 1 | 1 |", "| 4 | 1 | 1 | 1 | 1 | 0 |") + text = text.replace("- Ledger (3)", "- Ledger (2)") + self.assert_fails(text, "deleted file `tests/test_c.py` is missing from the file tables") + + def test_an_extra_file_fails_even_when_counts_are_reconciled(self) -> None: + text = edited( + "| `tests/test_c.py` | retired | retired — Python only |", + "| `tests/test_c.py` | retired | retired — Python only |\n| `tests/test_invented.py` | retired | retired — invented |", + ) + text = text.replace("| 5 | 1 | 1 | 1 | 1 | 1 |", "| 6 | 1 | 1 | 1 | 1 | 2 |") + text = text.replace("- Ledger (3)", "- Ledger (4)") + self.assert_fails(text, "file `tests/test_invented.py` is not in the deleted-file manifest") + + def test_a_case_row_cannot_replace_a_file_row(self) -> None: + text = edited("| `tests/test_c.py` | retired | retired — Python only |\n", "") + text = text.replace("| 5 | 1 | 1 | 1 | 1 | 1 |", "| 4 | 1 | 1 | 1 | 1 | 0 |") + text = text.replace("- Ledger (3)", "- Ledger (2)") + text = text.replace("`test_case_one`", "`tests/test_c.py`") + self.assert_fails(text, "deleted file `tests/test_c.py` is missing from the file tables") + + def test_an_unavailable_or_empty_manifest_fails_closed(self) -> None: + for manifest, message in ((None, "cannot read"), ("# No entries\n\n", "manifest has no deleted files")): + with self.subTest(manifest=manifest): + result = self.check(manifest=manifest) + self.assertEqual(result.returncode, 1, result.stderr) + self.assertIn("docs/prism-deleted-test-files.txt", result.stderr) + self.assertIn(message, result.stderr) + + def test_duplicate_manifest_entries_fail(self) -> None: + result = self.check(manifest=FILE_MANIFEST + "tests/test_c.py\n") + self.assertEqual(result.returncode, 1, result.stderr) + self.assertIn("duplicate deleted file `tests/test_c.py`", result.stderr) + + def test_a_missing_case_fails_even_when_file_counts_are_unchanged(self) -> None: + self.assert_fails( + edited("| `test_case_one` | full | `crates/demo/tests/ledger.rs::lands_one_block` |\n", ""), + "deleted case `test_case_one` is missing from the case tables", + ) + + def test_a_renamed_case_fails_even_when_counts_are_unchanged(self) -> None: + self.assert_fails( + edited("`test_case_one`", "`test_case_typo`"), + f":{line_of('`test_case_one`')}: case `test_case_typo` is not in the deleted-case manifest", + "deleted case `test_case_one` is missing from the case tables", + ) + + def test_an_extra_case_fails(self) -> None: + self.assert_fails( + edited( + "| `test_case_one` | full |", + "| `test_case_one` | full | `crates/demo/tests/ledger.rs::lands_one_block` |\n| `test_case_two` | full |", + ), + "case `test_case_two` is not in the deleted-case manifest", + ) + + def test_a_file_row_cannot_replace_a_case_row(self) -> None: + text = edited("| 2.x.x case |", "| 2.x.x file |") + text = text.replace("| 5 | 1 | 1 | 1 | 1 | 1 |", "| 6 | 2 | 1 | 1 | 1 | 1 |") + text = text.replace("- Ledger (3)", "- Ledger (4)") + result = self.check(text, manifest=FILE_MANIFEST + "test_case_one\n") + self.assertEqual(result.returncode, 1, result.stderr) + self.assertIn("deleted case `test_case_one` is missing from the case tables", result.stderr) + + def test_an_unavailable_or_empty_case_manifest_fails_closed(self) -> None: + for manifest, message in ((None, "cannot read"), ("# No entries\n\n", "manifest has no deleted cases")): + with self.subTest(manifest=manifest): + result = self.check(case_manifest=manifest) + self.assertEqual(result.returncode, 1, result.stderr) + self.assertIn("docs/prism-deleted-test-cases.txt", result.stderr) + self.assertIn(message, result.stderr) + + def test_duplicate_case_manifest_entries_fail(self) -> None: + result = self.check(case_manifest=CASE_MANIFEST + "test_case_one\n") + self.assertEqual(result.returncode, 1, result.stderr) + self.assertIn("duplicate deleted case `test_case_one`", result.stderr) + + def test_duplicate_case_rows_fail(self) -> None: + self.assert_fails( + edited( + "| `test_case_one` | full |", + "| `test_case_one` | full | `crates/demo/tests/ledger.rs::lands_one_block` |\n| `test_case_one` | full |", + ), + f"case `test_case_one` already has a row at line {line_of('`test_case_one`')}", + ) + + def test_a_missing_map_fails(self) -> None: + with tempfile.TemporaryDirectory() as directory: + result = run_check(Path(directory)) + self.assertEqual(result.returncode, 1) + self.assertIn("cannot read", result.stderr) + + def test_reference_to_a_missing_file_fails(self) -> None: + self.assert_fails( + edited("`tests/test_kept.py::test_kept`", "`tests/test_gone.py::test_kept`"), + f"docs/prism-deleted-test-map.md:{line_of('`tests/test_b.py`')}: `tests/test_gone.py::test_kept`: no such file", + ) + + def test_reference_to_a_missing_function_fails(self) -> None: + self.assert_fails( + edited("ledger.rs::lands_two_blocks`", "ledger.rs::lands_three_blocks`"), + "no `fn lands_three_blocks` in that file", + ) + self.assert_fails(edited("test_kept.py::test_kept`", "test_kept.py::test_renamed`"), "no `def test_renamed`") + + def test_reference_to_a_function_that_is_not_a_test_fails(self) -> None: + self.assert_fails( + edited("ledger.rs::lands_two_blocks`", "ledger.rs::helper_that_is_not_a_test`"), + "`fn helper_that_is_not_a_test` exists but is not a test function", + ) + self.assert_fails( + edited("test_kept.py::test_kept`", "test_kept.py::helper`"), + "`def helper` exists but is not a test function", + ) + + def test_rust_citations_reject_ignored_tests_without_a_ci_selection(self) -> None: + attributes = ( + "#[test]\n#[ignore]", + '#[ignore = "manual measurement"]\n#[test]', + '#[tokio::test]\n#[ignore = "manual measurement"]', + '#[test]\n#[ignore\n = "multiline reason"\n]', + '#[test]\n# [ /* comment */ ignore ]', + ) + for attribute in attributes: + with self.subTest(attribute=attribute), tempfile.TemporaryDirectory() as directory: + root = Path(directory) + write_tree(root) + source = RUST.replace("#[test]", attribute) + (root / "crates/demo/tests/ledger.rs").write_text(source, encoding="utf-8") + result = run_check(root) + self.assertEqual(result.returncode, 1, result.stderr) + self.assertIn("ledger.rs::lands_one_block", result.stderr) + self.assertIn("#[ignore]", result.stderr) + self.assertIn("scripts/run_rust_test_shard.py", result.stderr) + + def test_rust_citations_require_workspace_target_reachability(self) -> None: + for label, workspace, package, source in ( + ("not a member", '[workspace]\nmembers = []\n', "", RUST), + ("excluded member", '[workspace]\nmembers = ["crates/*"]\nexclude = ["crates/demo"]\n', "", RUST), + ("autotests disabled", None, "autotests = false\n", RUST), + ("custom harness", None, '\n[[test]]\nname = "ledger"\nharness = false\n', RUST), + ("renamed custom harness", None, '\n[[test]]\nname = "custom"\npath = "tests/ledger.rs"\nharness = false\n', RUST), + ("normalized target path", None, '\n[[test]]\nname = "custom"\npath = "tests/../tests/ledger.rs"\nharness = false\n', RUST), + ("missing feature", None, '\n[[test]]\nname = "ledger"\nrequired-features = ["manual"]\n[features]\nmanual = []\n', RUST), + ("disabled file", None, "", "#![cfg(any())]\n" + RUST), + ("disabled module", None, "", "#[cfg(any())]\nmod disabled {\n" + RUST + "}\n"), + ("disabled inline body", None, "", "mod disabled {\n#![cfg(any())]\n" + RUST + "}\n"), + ("disabled function", None, "", RUST.replace("#[test]", "#[cfg(any())]\n#[test]")), + ("disabled function body", None, "", RUST.replace("fn lands_one_block() {}", "fn lands_one_block() { #![cfg(any())] }")), + ("same-line live helper", None, "", RUST.replace("#[test]\nfn lands_one_block() {}", "#[test]\n#[cfg(any())]\nfn lands_one_block() {} fn live_helper() {}")), + ("not test", None, "", "#[cfg(not(test))]\nmod disabled {\n" + RUST + "}\n"), + ("unknown condition", None, "", "#[cfg(not(unknown_flag))]\nmod maybe {\n" + RUST + "}\n"), + ("conditional module rewrite", None, "", "#[cfg_attr(test, cfg(any()))]\nmod disabled {\n" + RUST + "}\n"), + ("nested function", None, "", "fn helper() {\n" + RUST + "}\n"), + ("macro definition", None, "", "macro_rules! unused { () => {\n" + RUST + "}; }\n"), + ): + with self.subTest(case=label), tempfile.TemporaryDirectory() as directory: + root = Path(directory) + write_tree(root) + if workspace is not None: + (root / "Cargo.toml").write_text(workspace, encoding="utf-8") + with (root / "crates/demo/Cargo.toml").open("a") as manifest: + manifest.write(package) + (root / "crates/demo/tests/ledger.rs").write_text(source, encoding="utf-8") + result = run_check(root) + self.assertEqual(result.returncode, 1, result.stderr) + self.assertIn("Cargo target", result.stderr) + + def test_rust_citations_require_real_module_declarations(self) -> None: + for declaration, accepted in ( + ("", False), + ("// mod child;", False), + ('const TEXT: &str = "mod child;";', False), + ("#[cfg(any())]\nmod child;", False), + ("mod child;", True), + ("#[cfg(test)]\nmod child;", True), + ("#[cfg(all(test, not(any()), any(test, all())))]\nmod child;", True), + ('#[path = "child.rs"]\nmod renamed;', True), + ('#[cfg_attr(test, path = "other.rs")]\nmod child;', False), + ('#[path = r"other.rs"]\nmod child;', False), + ): + with self.subTest(declaration=declaration), tempfile.TemporaryDirectory() as directory: + root = Path(directory) + write_tree(root, MAP.replace("crates/demo/tests/ledger.rs::", "crates/demo/src/child.rs::")) + source = root / "crates/demo/src" + source.mkdir() + (source / "lib.rs").write_text(declaration, encoding="utf-8") + (source / "child.rs").write_text(RUST, encoding="utf-8") + result = run_check(root) + self.assertEqual(result.returncode, 0 if accepted else 1, result.stderr) + + def test_rust_citations_follow_nested_module_paths(self) -> None: + cases = ( + ({"src/lib.rs": "mod outer;", "src/outer.rs": "mod child;"}, "src/outer/child.rs"), + ({"src/lib.rs": "mod outer;", "src/outer/mod.rs": "mod child;"}, "src/outer/child.rs"), + ({"src/lib.rs": "mod outer { mod child; }"}, "src/outer/child.rs"), + ({"src/lib.rs": 'mod outer { #[path = "renamed.rs"] mod child; }'}, "src/outer/renamed.rs"), + ({"src/lib.rs": "mod parent;", "src/parent.rs": 'mod outer { #[path = "renamed.rs"] mod child; }'}, "src/parent/outer/renamed.rs"), + ({"src/lib.rs": '#[path = "elsewhere/outer.rs"] mod outer;', "src/elsewhere/outer.rs": "mod child;"}, "src/elsewhere/child.rs"), + ({"src/lib.rs": '#[path = "elsewhere"] mod outer { mod child; }'}, "src/elsewhere/child.rs"), + ) + for modules, citation in cases: + with self.subTest(modules=modules), tempfile.TemporaryDirectory() as directory: + root = Path(directory) + write_tree(root, MAP.replace("crates/demo/tests/ledger.rs::", f"crates/demo/{citation}::")) + for path, source in {**modules, citation: RUST}.items(): + target = root / "crates/demo" / path + target.parent.mkdir(parents=True, exist_ok=True) + target.write_text(source, encoding="utf-8") + result = run_check(root) + self.assertEqual(result.returncode, 0, result.stderr) + + def test_rust_library_target_override_does_not_keep_the_old_source(self) -> None: + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + write_tree(root, MAP.replace("crates/demo/tests/ledger.rs::", "crates/demo/src/lib.rs::")) + (root / "crates/demo/src").mkdir() + (root / "crates/demo/src/lib.rs").write_text(RUST, encoding="utf-8") + (root / "crates/demo/src/alternate.rs").write_text(RUST, encoding="utf-8") + with (root / "crates/demo/Cargo.toml").open("a") as manifest: + manifest.write('[lib]\nname = "custom"\npath = "src/alternate.rs"\n') + result = run_check(root) + self.assertEqual(result.returncode, 1, result.stderr) + self.assertIn("Cargo target", result.stderr) + + def test_rust_cargo_manifests_fail_closed(self) -> None: + for manifest in ("", "[broken", "workspace = 42\n", '[workspace]\nmembers = ["crates/*"]\n'): + with self.subTest(manifest=manifest), tempfile.TemporaryDirectory() as directory: + root = Path(directory) + write_tree(root) + (root / "Cargo.toml").write_text(manifest, encoding="utf-8") + if manifest.startswith("[workspace]"): + (root / "crates/demo/Cargo.toml").rename(root / "crates/demo/unused.toml") + result = run_check(root) + self.assertEqual(result.returncode, 1, result.stderr) + self.assertIn("Cargo target", result.stderr) + self.assertNotIn("Traceback", result.stderr) + + def test_rust_citations_accept_explicit_targets_with_autodiscovery_disabled(self) -> None: + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + write_tree(root) + with (root / "crates/demo/Cargo.toml").open("a") as manifest: + manifest.write('autotests = false\n[[test]]\nname = "custom"\npath = "tests/ledger.rs"\n') + result = run_check(root) + self.assertEqual(result.returncode, 0, result.stderr) + result = run_check(root, env={"PATH": ""}) + self.assertEqual(result.returncode, 0, result.stderr) + + def test_rust_ignored_citations_follow_ci_target_and_exact_test_selection(self) -> None: + exact = "ten_thousand_unsubscribed_connections_do_not_advance_postgres_sequence" + cases = ( + ("qbit-prism-server", "observability_database", "contract", False, True), + ("qbit-prism-server", "compact_runtime_scale", "contract", False, True), + ("qbit-prism-server", "issued_job_dependency", "contract", True, True), + ("qbit-prism-server", "stratum_admission_postgres", exact, False, True), + ("qbit-prism-server", "stratum_admission_postgres", "other_contract", False, False), + ("qbit-prism-server", "stratum_admission_postgres", exact + "_extra", False, False), + ("qbit-prism-server", "stratum_admission_postgres", exact, True, False), + ("qbit-prism-server", "audit_body_normalization", "contract", False, False), + ("another-package", "observability_database", "contract", False, False), + ) + for package, target, name, nested, selected in cases: + with self.subTest(package=package, target=target, name=name, nested=nested): + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + citation = f"crates/{package}/tests/{target}.rs" + # Exercise partial rows as well as the full file/case rows above. + write_tree(root, edited("tests/test_kept.py::test_kept", f"{citation}::{name}")) + write_cargo_package(root, package) + source = f'#[test]\n#[ignore = "explicit integration run"]\nfn {name}() {{}}\n' + if nested: + source = "mod nested {\n" + source + "}\n" + source_path = root / citation + source_path.parent.mkdir(parents=True, exist_ok=True) + source_path.write_text(source, encoding="utf-8") + result = run_check(root) + self.assertEqual(result.returncode, 0 if selected else 1, result.stderr) + if not selected: + self.assertIn("#[ignore]", result.stderr) + + def test_rust_ignored_citations_require_a_real_unfiltered_module_inclusion(self) -> None: + declaration = '#[path = "observability/database_privacy.rs"]\nmod database_privacy;\n' + cases = ( + (declaration, "observability_database", True), + ("// comment\n" + declaration, "observability_database", True), + ("/*" + declaration + "*/", "observability_database", False), + ('const TEXT: &str = r###"' + declaration + '"###;', "observability_database", False), + ("#[cfg(any())]\n" + declaration, "observability_database", False), + ("#[cfg(test)]\n" + declaration, "observability_database", True), + # The path in an inline module resolves under its own directory. + ("mod nested {\n" + declaration + "}", "observability_database", False), + (declaration, "stratum_admission_postgres", False), + ("", "observability_database", False), + ) + name = "ten_thousand_unsubscribed_connections_do_not_advance_postgres_sequence" + citation = "crates/qbit-prism-server/tests/observability/database_privacy.rs" + for source, target, selected in cases: + with self.subTest(source=source, target=target), tempfile.TemporaryDirectory() as directory: + root = Path(directory) + write_tree(root, edited("tests/test_kept.py::test_kept", f"{citation}::{name}")) + write_cargo_package(root, "qbit-prism-server") + source_path = root / citation + source_path.parent.mkdir(parents=True) + source_path.write_text(f"#[test]\n#[ignore]\nfn {name}() {{}}\n", encoding="utf-8") + target_path = root / f"crates/qbit-prism-server/tests/{target}.rs" + target_path.write_text(source, encoding="utf-8") + result = run_check(root) + self.assertEqual(result.returncode, 0 if selected else 1, result.stderr) + + def test_rust_ignored_citations_use_declared_cargo_names(self) -> None: + for source_name, target_name, package_name, accepted in ( + ("observability_database", "manual", "qbit-prism-server", False), + ("manual", "observability_database", "qbit-prism-server", True), + ("observability_database", "observability_database", "another-package", False), + ): + with self.subTest(source=source_name, target=target_name, package=package_name), tempfile.TemporaryDirectory() as directory: + root = Path(directory) + citation = f"crates/qbit-prism-server/tests/{source_name}.rs" + write_tree(root, edited("tests/test_kept.py::test_kept", f"{citation}::contract")) + package = write_cargo_package(root, "qbit-prism-server") + (package / "Cargo.toml").write_text( + f'[package]\nname = "{package_name}"\nversion = "0.1.0"\nedition = "2021"\n' + f'[[test]]\nname = "{target_name}"\npath = "tests/{source_name}.rs"\n', encoding="utf-8" + ) + (package / "tests").mkdir() + (root / citation).write_text("#[test]\n#[ignore]\nfn contract() {}\n", encoding="utf-8") + result = run_check(root) + self.assertEqual(result.returncode, 0 if accepted else 1, result.stderr) + + def test_rust_ignore_text_does_not_skip_real_tests_or_leak_to_the_next_test(self) -> None: + for attribute in ( + '// #[ignore]', + '/* #[ignore] */', + '#[doc = "#[ignore]"]', + '#[doc = r###"\n#[ignore]\n"###]', + ): + with self.subTest(attribute=attribute), tempfile.TemporaryDirectory() as directory: + root = Path(directory) + write_tree(root) + source = '#[test]\n#[ignore]\nfn manual_only() {}\n' + RUST.replace( + "#[test]", f"#[test]\n{attribute}" + ) + (root / "crates/demo/tests/ledger.rs").write_text(source, encoding="utf-8") + result = run_check(root) + self.assertEqual(result.returncode, 0, result.stderr) + + def test_rust_citations_ignore_functions_inside_literals_and_comments(self) -> None: + phantom = "\n#[test]\nfn phantom() {}\n" + sources = { + "block comment": "/*" + phantom + "*/", + "nested block comment": "/* outer /* inner */" + phantom + "*/", + "doc comment": "/**" + phantom + "*/\nstruct Fixture;", + "line comments": "// #[test]\n// fn phantom() {}\n", + "string": 'const FIXTURE: &str = "' + phantom + '";', + "escaped quote": 'const FIXTURE: &str = "\\\"' + phantom + '";', + "byte string": 'const FIXTURE: &[u8] = b"' + phantom + '";', + "C string": 'const FIXTURE: &std::ffi::CStr = c"' + phantom + '";', + "doc attribute": '#[doc = "' + phantom + '"]\nstruct Fixture;', + } + for prefix, rust_type in (("r", "str"), ("br", "[u8]"), ("cr", "std::ffi::CStr")): + for hashes in ("", "#", "###"): + # Quotes and too few hashes do not close the raw string. + interior = '"' + hashes[:-1] if hashes else "" + sources[f"{prefix} with {len(hashes)} hashes"] = ( + f'const FIXTURE: &{rust_type} = {prefix}{hashes}"{interior}{phantom}"{hashes};' + ) + for label, source in sources.items(): + with self.subTest(label=label), tempfile.TemporaryDirectory() as directory: + root = Path(directory) + write_tree(root, edited("ledger.rs::lands_two_blocks`", "ledger.rs::phantom`")) + (root / "crates/demo/tests/ledger.rs").write_text(RUST + source, encoding="utf-8") + result = run_check(root) + self.assertEqual(result.returncode, 1, result.stderr) + self.assertIn("no `fn phantom` in that file", result.stderr) + + def test_rust_citations_ignore_test_attributes_inside_literals_and_comments(self) -> None: + for attribute in ( + "/* #[test] */", + "/* outer /* inner */\n#[test]\n*/", + '#[doc = "#[test]"]', + '#[doc = r###"\n#[test]\n"###]', + "// #[test]", + ): + with self.subTest(attribute=attribute), tempfile.TemporaryDirectory() as directory: + root = Path(directory) + write_tree(root, edited("ledger.rs::lands_two_blocks`", "ledger.rs::phantom`")) + source = RUST + f"\n{attribute}\nfn phantom() {{}}\n" + (root / "crates/demo/tests/ledger.rs").write_text(source, encoding="utf-8") + result = run_check(root) + self.assertEqual(result.returncode, 1, result.stderr) + self.assertIn("`fn phantom` exists but is not a test function", result.stderr) + + def test_rust_citations_preserve_real_tests_around_literals_and_comments(self) -> None: + source = r''' +fn character_and_lifetime_fixture<'a>(value: &'a str) -> &'a str { + let _ = ('"', '\'', '\\', '\u{22}', b'"', b'\x22'); + let _ = ("/*", "//", "\\\"", r###"/* "## //"###); + 'label: loop { break 'label; } + value +} + +#[test] +/* outer comment + /* nested comment */ + "quotes and // do not end a block comment" +*/ + +fn lands_one_block() {} + +#[test] // comment after a real attribute +#[should_panic(expected = "escaped quote: \" /*")] +/// Documentation between the attribute and the test. +fn /* comment between tokens */ lands_two_blocks() { panic!("escaped quote: \" /*"); } +''' + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + write_tree(root) + (root / "crates/demo/tests/ledger.rs").write_text(source, encoding="utf-8") + result = run_check(root) + self.assertEqual(result.returncode, 0, result.stderr) + + def test_python_citations_must_be_in_the_discovered_suite(self) -> None: + sources = { + "plain class": PYTHON.replace("(unittest.TestCase)", ""), + "module function": "def test_kept():\n pass\n", + "nested function": "def helper():\n def test_kept():\n pass\n", + "nested class": "import unittest\nclass Outer:\n class KeptTests(unittest.TestCase):\n def test_kept(self):\n pass\n", + "unreachable class": "if False:\n" + "\n".join(" " + line for line in PYTHON.splitlines()), + "replaced method": PYTHON + "\nKeptTests.test_kept = None\n", + "load_tests filters it out": PYTHON + "\ndef load_tests(loader, tests, pattern):\n return unittest.TestSuite()\n", + } + for label, source in sources.items(): + with self.subTest(label=label), tempfile.TemporaryDirectory() as directory: + root = Path(directory) + write_tree(root) + (root / "tests" / "test_kept.py").write_text(source, encoding="utf-8") + result = run_check(root) + self.assertEqual(result.returncode, 1, result.stderr) + self.assertIn("`def test_kept` exists but is not a test function", result.stderr) + + def test_python_citations_accept_discovered_aliases_and_inherited_methods(self) -> None: + sources = ( + PYTHON.replace("import unittest", "import unittest as ut").replace("unittest.TestCase", "ut.TestCase"), + PYTHON.replace("import unittest", "from unittest import TestCase as Case").replace("unittest.TestCase", "Case"), + PYTHON.replace("class KeptTests(unittest.TestCase):", "class Mixin:") + + "\nclass KeptTests(Mixin, unittest.TestCase):\n pass\n", + PYTHON.replace("unittest.TestCase", "unittest.IsolatedAsyncioTestCase").replace("def test_kept", "async def test_kept"), + ) + for source in sources: + with self.subTest(source=source), tempfile.TemporaryDirectory() as directory: + root = Path(directory) + write_tree(root) + (root / "tests" / "test_kept.py").write_text(source, encoding="utf-8") + result = run_check(root) + self.assertEqual(result.returncode, 0, result.stderr) + + def test_python_citations_reject_effectively_skipped_cases(self) -> None: + for decorator in ( + '@unittest.skip("disabled")', + '@unittest.skipIf(True, "disabled")', + '@unittest.skipUnless(False, "disabled")', + ): + sources = { + "class": PYTHON.replace("class KeptTests", decorator + "\nclass KeptTests"), + "method": PYTHON.replace(" def test_kept", " " + decorator + "\n def test_kept"), + "inherited class flag": PYTHON.replace("class KeptTests", decorator + "\nclass BaseTests") + + "\nclass KeptTests(BaseTests):\n pass\n", + "inherited method": PYTHON.replace(" def test_kept", " " + decorator + "\n def test_kept") + .replace("class KeptTests(unittest.TestCase):", "class Mixin:") + + "\nclass KeptTests(Mixin, unittest.TestCase):\n pass\n", + } + for label, source in sources.items(): + with self.subTest(decorator=decorator, case=label), tempfile.TemporaryDirectory() as directory: + root = Path(directory) + write_tree(root) + (root / "tests/test_kept.py").write_text(source, encoding="utf-8") + result = run_check(root) + self.assertEqual(result.returncode, 1, result.stderr) + self.assertIn("`def test_kept` exists but is not a test function", result.stderr) + + def test_python_citations_accept_inactive_skip_decorators_and_live_inherited_cases(self) -> None: + for decorator in ('@unittest.skipIf(False, "enabled")', '@unittest.skipUnless(True, "enabled")'): + for anchor in ("class KeptTests", " def test_kept"): + with self.subTest(decorator=decorator, anchor=anchor), tempfile.TemporaryDirectory() as directory: + root = Path(directory) + write_tree(root) + indent = " " if anchor.startswith(" ") else "" + source = PYTHON.replace(anchor, indent + decorator + "\n" + anchor) + (root / "tests/test_kept.py").write_text(source, encoding="utf-8") + result = run_check(root) + self.assertEqual(result.returncode, 0, result.stderr) + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + write_tree(root) + source = PYTHON + '\n@unittest.skip("disabled subclass")\nclass SkippedTests(KeptTests):\n pass\n' + (root / "tests/test_kept.py").write_text(source, encoding="utf-8") + result = run_check(root) + self.assertEqual(result.returncode, 0, result.stderr) + + def test_python_citations_reject_files_outside_the_discovery_pattern(self) -> None: + for path in ("tests/kept.py", "tests/nested/test_kept.py", "scripts/test_kept.py"): + with self.subTest(path=path), tempfile.TemporaryDirectory() as directory: + root = Path(directory) + write_tree(root, edited("`tests/test_kept.py::test_kept`", f"`{path}::test_kept`")) + target = root / path + target.parent.mkdir(exist_ok=True) + target.write_text(PYTHON, encoding="utf-8") + result = run_check(root) + self.assertEqual(result.returncode, 1, result.stderr) + self.assertIn("`def test_kept` exists but is not a test function", result.stderr) + + def test_python_discovery_failures_are_reported_without_running_test_bodies(self) -> None: + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + write_tree(root) + source = PYTHON.replace(" pass", " raise RuntimeError('test body ran')") + (root / "tests" / "test_kept.py").write_text(source, encoding="utf-8") + result = run_check(root) + self.assertEqual(result.returncode, 0, result.stderr) + (root / "tests" / "test_broken.py").write_text("raise RuntimeError('discovery failed')\n", encoding="utf-8") + result = run_check(root) + self.assertEqual(result.returncode, 1, result.stderr) + self.assertIn("Python unittest discovery failed", result.stderr) + self.assertIn("discovery failed", result.stderr) + + def test_a_reference_in_prose_outside_any_table_is_checked_too(self) -> None: + self.assert_fails( + edited("are code, not test references.", "are code; `crates/demo/tests/ledger.rs::gone` is not."), + f":{line_of('are code, not test references.')}: `crates/demo/tests/ledger.rs::gone`: no `fn gone`", + ) + + def test_a_malformed_reference_is_an_error_not_a_skipped_one(self) -> None: + self.assert_fails( + edited("`crates/demo/tests/ledger.rs::lands_two_blocks`", "`crates/demo/tests/ledger::lands_two_blocks`"), + "looks like a test reference but is not", + ) + + def test_a_reference_may_not_leave_the_repository(self) -> None: + self.assert_fails( + edited("`tests/test_kept.py::test_kept`", "`tests/../../outside/test_kept.py::test_kept`"), + "path leaves the repository", + ) + + def test_summary_total_and_status_counts_must_equal_the_rows(self) -> None: + self.assert_fails(edited("| 5 | 1 | 1 | 1 | 1 | 1 |", "| 6 | 1 | 1 | 1 | 1 | 1 |"), "summary says 6 rows, the file tables hold 5") + self.assert_fails( + edited("| 5 | 1 | 1 | 1 | 1 | 1 |", "| 5 | 2 | 1 | 1 | 1 | 0 |"), + "summary says 2 full, the rows say 1", + "summary says 0 retired, the rows say 1", + ) + self.assert_fails(edited("| 5 | 1 | 1 | 1 | 1 | 1 |", "| 5 | one | 1 | 1 | 1 | 1 |"), "must be 6 whole numbers") + + def test_a_status_change_without_a_summary_change_fails(self) -> None: + self.assert_fails( + edited("| `tests/test_c.py` | retired | retired — Python only |", "| `tests/test_c.py` | partial | partial: some |"), + "summary says 1 partial, the rows say 2", + "summary says 1 retired, the rows say 0", + ) + + def test_case_rows_are_not_counted_as_files(self) -> None: + text = edited("| `test_case_one` | full |", "| `test_case_one` | full | `crates/demo/tests/ledger.rs::lands_one_block` |\n| `test_case_two` | full |") + result = self.check(text, case_manifest=CASE_MANIFEST + "test_case_two\n") + self.assertEqual(result.returncode, 0, result.stderr) + self.assertIn("ok (5 file rows, 2 case rows)", result.stdout) + + def test_section_list_must_match_the_sections(self) -> None: + self.assert_fails(edited("- Ledger (3)", "- Ledger (4)"), "section list says 4 rows in 'Ledger', the section holds 3") + self.assert_fails(edited("- Stratum (2)\n", ""), "section 'Stratum' holds 2 rows and is missing from the section list") + self.assert_fails(edited("- Stratum (2)", "- Stratum (2)\n- Vardiff (1)"), "section list names 'Vardiff'") + self.assert_fails(edited("- Stratum (2)", "- Stratum: 2"), "section list item must read") + + def test_row_shape_is_enforced(self) -> None: + self.assert_fails(edited("(asserts `a \\| b`)", "(asserts `a | b`)"), "row has 4 cells, expected 3") + self.assert_fails(edited("| `tests/test_c.py` | retired |", "| tests/test_c.py | retired |"), "first cell must be one `code` name") + self.assert_fails(edited("| retired — Python only |", "| retired — Python only"), f":{line_of('`tests/test_c.py`')}: table line does not end with an unescaped pipe") + + def test_one_unreadable_line_does_not_hide_the_rest_of_its_table(self) -> None: + result = self.check(edited("| open gap → [#8]", "| open gap → [#8] \\").replace("| `tests/test_e.py` | needs triage |", "| `tests/test_e.py` | someday |")) + self.assertEqual(result.returncode, 1) + self.assertIn("unknown status 'someday'", result.stderr) + + def test_an_unknown_table_is_an_error(self) -> None: + self.assert_fails( + edited("| 2.x.x case | Covered | 3.x.x replacement or status |", "| 2.x.x method | Covered | 3.x.x replacement or status |"), + "unrecognised table header", + ) + + def test_status_rules(self) -> None: + self.assert_fails(edited("| `tests/test_c.py` | retired | retired —", "| `tests/test_c.py` | obsolete | retired —"), "unknown status 'obsolete'") + self.assert_fails( + edited("| `tests/test_c.py` | retired | retired — Python only |", "| `tests/test_c.py` | retired | partial: some |"), + "status is 'retired' but the text does not lead with it", + ) + self.assert_fails( + edited("| `test_case_one` | full | `crates/demo/tests/ledger.rs::lands_one_block` |", "| `test_case_one` | full | covered somewhere |"), + "a full row must cite at least one `path::name` test", + ) + self.assert_fails( + edited(f"open gap → [#8]({ISSUES}/8)", "open gap, someone should look"), + "an open gap row must link the issue that closes the gap", + ) + + def test_a_partial_file_row_must_cite_a_test(self) -> None: + for citation in ("", "the native suite", "`server::run`", "`tests/test_kept::test_kept`"): + with self.subTest(citation=citation): + self.assert_fails( + edited("`tests/test_kept.py::test_kept`", citation), + f":{line_of('`tests/test_b.py`')}: a partial row must cite at least one `path::name` test", + ) + + def test_a_partial_case_row_must_cite_a_test(self) -> None: + self.assert_fails( + edited( + "| `test_case_one` | full | `crates/demo/tests/ledger.rs::lands_one_block` |", + "| `test_case_one` | partial | partial: covered by the native suite |", + ), + f":{line_of('`test_case_one`')}: a partial row must cite at least one `path::name` test", + ) + + def test_a_partial_case_row_with_a_test_citation_passes(self) -> None: + result = self.check(edited("| `test_case_one` | full |", "| `test_case_one` | partial | partial: covered by")) + self.assertEqual(result.returncode, 0, result.stderr) + + def test_duplicate_rows_fail(self) -> None: + self.assert_fails( + edited("| `tests/test_c.py` | retired | retired — Python only |", "| `tests/test_a.py` | retired | retired — Python only |"), + f":{line_of('`tests/test_c.py`')}: file `tests/test_a.py` already has a row at line {line_of('`tests/test_a.py`')}", + ) + + def test_needs_triage_index_must_list_exactly_the_needs_triage_rows(self) -> None: + self.assert_fails( + edited("| `tests/test_e.py` | Stratum | nobody owns it |\n", ""), + "`tests/test_e.py` is needs triage but missing from the needs triage index", + ) + self.assert_fails( + edited("| `tests/test_e.py` | Stratum | nobody owns it |", "| `tests/test_e.py` | Stratum | nobody |\n| `tests/test_c.py` | Ledger | stale |"), + "`tests/test_c.py` is in the needs triage index but has no needs triage row", + ) + self.assert_fails(edited("| `tests/test_e.py` | Stratum |", "| `tests/test_e.py` | Ledger |"), "area 'Ledger' is not the section the row sits in, 'Stratum'") + + def test_needs_triage_case_requires_an_index_entry(self) -> None: + text = edited( + "| `test_case_one` | full | `crates/demo/tests/ledger.rs::lands_one_block` |", + "| `test_case_one` | needs triage | needs triage — no native test or owner |", + ) + self.assert_fails( + text, + f":{line_of('`test_case_one`')}: `test_case_one` is needs triage but missing from the needs triage index", + ) + + def test_needs_triage_case_index_entry_must_be_unique_and_match_its_section(self) -> None: + text = edited( + "| `test_case_one` | full | `crates/demo/tests/ledger.rs::lands_one_block` |", + "| `test_case_one` | needs triage | needs triage — no native test or owner |", + ) + entry = "| `test_case_one` | Ledger | no native test or owner |\n" + result = self.check(text + entry) + self.assertEqual(result.returncode, 0, result.stderr) + self.assert_fails( + text + entry.replace("| Ledger |", "| Stratum |"), + "area 'Stratum' is not the section the row sits in, 'Ledger'", + ) + self.assert_fails(text + entry + entry, "`test_case_one` is listed twice in the needs triage index") + self.assert_fails(MAP + entry, "`test_case_one` is in the needs triage index but has no needs triage row") + + def test_issue_links_must_point_at_the_issue_they_name(self) -> None: + self.assert_fails(edited(f"[#7]({ISSUES}/7)", f"[#7]({ISSUES}/70)"), f"[#7] links '{ISSUES}/70'") + self.assert_fails(edited(f"[#7]({ISSUES}/7)", "[#7](https://github.com/other/repo/issues/7)"), "[#7] links") + + def test_an_owner_arrow_must_be_a_link(self) -> None: + for arrow in ("→", "->"): + with self.subTest(arrow=arrow): + self.assert_fails(edited(f"(→ [#7]({ISSUES}/7))", f"({arrow} #7)"), "an owner arrow must be followed by an issue link") + + def test_malformed_owner_links_fail_even_beside_valid_links(self) -> None: + for arrow in ("→", "->"): + for reference in ( + "[#7]", + "[#7", + "[ #7]", + "[[#7]]", + "[#7][owner]", + f"[#7] ({ISSUES}/7)", + f"[#7]({ISSUES}/7", + f"[#7]( {ISSUES}/7)", + f"[#7]({ISSUES}/7 )", + f'[#7]({ISSUES}/7 "owner")', + f"#7]({ISSUES}/7)", + ): + with self.subTest(arrow=arrow, reference=reference): + text = edited(f"(→ [#7]({ISSUES}/7))", f"{arrow} {reference}") + self.assert_fails( + text, + f"docs/prism-deleted-test-map.md:{line_of('`tests/test_b.py`')}: an owner arrow must be followed by an issue link", + ) + + def test_malformed_owner_links_fail_in_prose_cases_and_triage(self) -> None: + for arrow in ("→", "->"): + for original in ( + "prose may link a closed issue as history.", + "| `test_case_one` | full | `crates/demo/tests/ledger.rs::lands_one_block`", + "| Stratum | nobody owns it", + ): + with self.subTest(arrow=arrow, original=original): + text = edited(original, f"{original} {arrow} [#11]") + self.assert_fails( + text, + f"docs/prism-deleted-test-map.md:{line_of(original)}: an owner arrow must be followed by an issue link", + ) + + def test_summary_lists_counts_and_open_rows(self) -> None: + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + write_tree(root) + result = run_check(root, "--summary") + self.assertEqual(result.returncode, 0, result.stderr) + self.assertIn("5 rows: 1 full, 1 partial, 1 retired, 1 open gap, 1 needs triage.", result.stdout) + self.assertIn("- `tests/test_d.py` (links #8)", result.stdout) + self.assertIn("- `tests/test_e.py` (links #9)", result.stdout) + self.assertIn("- `tests/test_b.py` (partial, owner #6, #7)", result.stdout) + + def test_summary_lists_open_case_rows_with_and_without_owners(self) -> None: + for status, detail, suffix in ( + ("open gap", f"open gap → [#10]({ISSUES}/10)", " (links #10)"), + ("needs triage", "needs triage — no native test or owner", ""), + ("needs triage", f"needs triage — see [#10]({ISSUES}/10)", " (links #10)"), + ): + with self.subTest(detail=detail), tempfile.TemporaryDirectory() as directory: + text = edited( + "| `test_case_one` | full | `crates/demo/tests/ledger.rs::lands_one_block` |", + f"| `test_case_one` | {status} | {detail} |", + ) + if status == "needs triage": + text += "| `test_case_one` | Ledger | no native test or owner |\n" + root = Path(directory) + write_tree(root, text) + result = run_check(root, "--summary") + self.assertEqual(result.returncode, 0, result.stderr) + heading = f"{status.capitalize()} (2):\n" + self.assertIn(heading, result.stdout) + section = result.stdout.split(heading, 1)[1].split("\n\n", 1)[0] + self.assertIn(f"- `test_case_one`{suffix}", section) + self.assertEqual(result.stdout.count("- `test_case_one`"), 1) + + def test_summary_lists_unowned_partial_file_and_case_gaps(self) -> None: + text = edited( + f", written when [#6]({ISSUES}/6) landed; not covered: the rest (→ [#7]({ISSUES}/7))", + "; not covered, no open issue: the rest (#7 closed)", + ).replace( + "| `test_case_one` | full | `crates/demo/tests/ledger.rs::lands_one_block` |", + "| `test_case_one` | partial | partial: `crates/demo/tests/ledger.rs::lands_one_block`; not covered: recovery |", + ) + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + write_tree(root, text) + result = run_check(root, "--summary") + self.assertEqual(result.returncode, 0, result.stderr) + self.assertIn("Partial rows with an uncovered remainder and no owner issue (2):", result.stdout) + self.assertIn("- `tests/test_b.py` (partial, no owner issue)", result.stdout) + self.assertIn("- `test_case_one` (partial, no owner issue)", result.stdout) + + def test_summary_detects_unowned_remainders_without_semicolon_separators(self) -> None: + for separator in ("; ", ", ", ". ", " — ", ": ", " "): + for label in ("not covered", "not covered, no open issue", "Not covered"): + for remainder, expected in (("recovery", True), ("none with a native subject", False)): + with self.subTest(separator=separator, label=label, remainder=remainder), tempfile.TemporaryDirectory() as directory: + detail = f"{separator}{label}: {remainder}; retired: Python helpers" + text = edited( + f", written when [#6]({ISSUES}/6) landed; not covered: the rest (→ [#7]({ISSUES}/7))", + detail, + ).replace( + "| `test_case_one` | full | `crates/demo/tests/ledger.rs::lands_one_block` |", + "| `test_case_one` | partial | partial: `crates/demo/tests/ledger.rs::lands_one_block`" + f"{detail} |", + ) + root = Path(directory) + write_tree(root, text) + result = run_check(root, "--summary") + self.assertEqual(result.returncode, 0, result.stderr) + self.assertEqual("- `tests/test_b.py` (partial, no owner issue)" in result.stdout, expected) + self.assertEqual("- `test_case_one` (partial, no owner issue)" in result.stdout, expected) + if expected: + self.assertIn("Partial rows with an uncovered remainder and no owner issue (2):", result.stdout) + else: + self.assertNotIn("no owner issue", result.stdout) + + def test_summary_does_not_report_retired_remainders_or_duplicate_owned_rows(self) -> None: + text = edited( + "| `test_case_one` | full | `crates/demo/tests/ledger.rs::lands_one_block` |", + "| `test_case_one` | partial | partial: `crates/demo/tests/ledger.rs::lands_one_block`; retired: Python helpers |", + ) + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + write_tree(root, text) + result = run_check(root, "--summary") + self.assertEqual(result.returncode, 0, result.stderr) + self.assertEqual(result.stdout.count("- `tests/test_b.py`"), 1) + self.assertNotIn("- `test_case_one`", result.stdout) + self.assertNotIn("no owner issue", result.stdout) + + def test_summary_excludes_explicitly_empty_native_remainders(self) -> None: + for label in ("not covered", "not covered, no open issue"): + with self.subTest(label=label), tempfile.TemporaryDirectory() as directory: + text = edited( + f", written when [#6]({ISSUES}/6) landed; not covered: the rest (→ [#7]({ISSUES}/7))", + f"; {label}: none with a native subject; the Python queue mechanics are retired", + ).replace( + "| `test_case_one` | full | `crates/demo/tests/ledger.rs::lands_one_block` |", + "| `test_case_one` | partial | partial: `crates/demo/tests/ledger.rs::lands_one_block`; " + f"{label}: none with a native subject; the Python epoch fanout is retired |", + ) + root = Path(directory) + write_tree(root, text) + result = run_check(root, "--summary") + self.assertEqual(result.returncode, 0, result.stderr) + self.assertNotIn("- `tests/test_b.py`", result.stdout) + self.assertNotIn("- `test_case_one`", result.stdout) + self.assertNotIn("no owner issue", result.stdout) + + def test_summary_keeps_real_gaps_beside_retired_or_empty_remainders(self) -> None: + for remainder in ( + "not covered: recovery; retired: Python helpers", + "not covered: none with a native subject; retired: Python helpers; not covered: recovery", + ): + with self.subTest(remainder=remainder), tempfile.TemporaryDirectory() as directory: + text = edited( + "| `test_case_one` | full | `crates/demo/tests/ledger.rs::lands_one_block` |", + "| `test_case_one` | partial | partial: `crates/demo/tests/ledger.rs::lands_one_block`; " + f"{remainder} |", + ) + root = Path(directory) + write_tree(root, text) + result = run_check(root, "--summary") + self.assertEqual(result.returncode, 0, result.stderr) + self.assertIn("Partial rows with an uncovered remainder and no owner issue (1):", result.stdout) + self.assertIn("- `test_case_one` (partial, no owner issue)", result.stdout) + + +class IssueStateTests(unittest.TestCase): + def check( + self, replies: dict[int, list[tuple[int, object, float]]], *args: str, text: str = MAP, env: dict[str, str] | None = None + ) -> tuple[subprocess.CompletedProcess[str], GitHubStub]: + with tempfile.TemporaryDirectory() as directory, GitHubStub(replies) as stub: + root = Path(directory) + write_tree(root, text) + return run_check(root, "--check-issues", "--api-url", stub.url, *args, env=env), stub + + def test_open_owners_pass_and_only_owners_are_asked_about(self) -> None: + result, stub = self.check(dict(ALL_OPEN)) + self.assertEqual(result.returncode, 0, result.stderr) + self.assertIn("4 owner issue(s) open", result.stdout) + # #5 is history in prose. #6 and #9 carry no arrow but sit in rows, where any link is an owner. + self.assertEqual(sorted(stub.asked()), [6, 7, 8, 9]) + + def test_a_closed_owner_fails_and_names_the_lines(self) -> None: + for number, line in ((6, line_of("`tests/test_b.py`")), (8, line_of("`tests/test_d.py`")), (9, line_of("needs triage — nobody"))): + with self.subTest(issue=number): + result, _ = self.check({**ALL_OPEN, number: state("closed")}) + self.assertEqual(result.returncode, 1, result.stderr) + self.assertIn(f"#{number} is closed but the map names it as the owner", result.stderr) + self.assertIn(f"lines {line};", result.stderr) + + def test_an_arrow_link_in_prose_is_an_owner_in_either_spelling(self) -> None: + for arrow in ("→", "->"): + with self.subTest(arrow=arrow): + text = edited("prose may link a closed issue as history.", f"the rest is {arrow} [#11]({ISSUES}/11).") + result, _ = self.check({**ALL_OPEN, 11: state("closed")}, text=text) + self.assertEqual(result.returncode, 1, result.stderr) + self.assertIn("#11 is closed", result.stderr) + + def test_each_valid_owner_arrow_is_checked_on_the_same_line(self) -> None: + for arrow in ("→", "->"): + for spacing in ("", " ", "\t"): + with self.subTest(arrow=arrow, spacing=spacing): + text = edited( + f"→ [#7]({ISSUES}/7)", + f"{arrow}{spacing}[#7]({ISSUES}/7), {arrow}{spacing}[#11]({ISSUES}/11)", + ) + result, stub = self.check({**ALL_OPEN, 11: state("open")}, text=text) + self.assertEqual(result.returncode, 0, result.stderr) + self.assertEqual(sorted(stub.asked()), [6, 7, 8, 9, 11]) + + def test_malformed_owner_links_fail_when_known_owners_are_open(self) -> None: + for arrow in ("→", "->"): + with self.subTest(arrow=arrow): + text = edited(f"→ [#7]({ISSUES}/7)", f"→ [#7]({ISSUES}/7), {arrow} [#11]") + result, _ = self.check(dict(ALL_OPEN), text=text) + self.assertEqual(result.returncode, 1, result.stderr) + self.assertIn("an owner arrow must be followed by an issue link", result.stderr) + + def test_a_link_in_the_needs_triage_index_is_an_owner(self) -> None: + text = edited("| Stratum | nobody owns it |", f"| Stratum | nobody owns it, [#10]({ISSUES}/10) might |") + result, stub = self.check({**ALL_OPEN, 10: state("closed")}, text=text) + self.assertEqual(result.returncode, 1, result.stderr) + self.assertIn("#10 is closed", result.stderr) + self.assertIn(10, stub.asked()) + + def test_a_server_error_is_retried_and_then_reported_as_unknown(self) -> None: + result, stub = self.check({**ALL_OPEN, 8: [(500, {"message": "boom"}, 0.0)]}) + self.assertEqual(result.returncode, 2, result.stderr) + self.assertIn(f"#8 (map lines {line_of('`tests/test_d.py`')}): state unknown: HTTP 500", result.stderr) + self.assertIn("nothing is assumed open", result.stderr) + self.assertEqual(stub.asked().count(8), 3) + + def test_a_transient_server_error_recovers(self) -> None: + result, stub = self.check({**ALL_OPEN, 8: [(502, {}, 0.0), (200, {"state": "open"}, 0.0)]}) + self.assertEqual(result.returncode, 0, result.stderr) + self.assertEqual(stub.asked().count(8), 2) + + def test_refusals_are_unknown_without_retry(self) -> None: + for status, fragment in ((403, "HTTP 403"), (429, "HTTP 429"), (404, "HTTP 404: no such issue")): + with self.subTest(status=status): + result, stub = self.check({**ALL_OPEN, 7: [(status, {"message": "no"}, 0.0)]}) + self.assertEqual(result.returncode, 2, result.stderr) + self.assertIn(fragment, result.stderr) + self.assertEqual(stub.asked().count(7), 1) + + def test_an_unexpected_reply_is_unknown_not_open(self) -> None: + for body, fragment in ( + ({"state": "reopened"}, "state 'reopened'"), + ({"title": "no state"}, "state None"), + ([], "state None"), + (b"<html>gateway</html>", "reply was not JSON"), + ): + with self.subTest(body=body): + result, _ = self.check({**ALL_OPEN, 9: [(200, body, 0.0)]}) + self.assertEqual(result.returncode, 2, result.stderr) + self.assertIn(fragment, result.stderr) + + def test_a_closed_owner_outranks_an_unknown_one_and_both_are_reported(self) -> None: + result, _ = self.check({**ALL_OPEN, 7: state("closed"), 8: [(403, {}, 0.0)]}) + self.assertEqual(result.returncode, 1, result.stderr) + self.assertIn("#7 is closed", result.stderr) + self.assertIn(f"#8 (map lines {line_of('`tests/test_d.py`')}): state unknown", result.stderr) + + def test_the_total_deadline_bounds_a_slow_api(self) -> None: + started = time.monotonic() + result, _ = self.check({number: [(200, {"state": "open"}, 1.5)] for number in ALL_OPEN}, "--timeout-seconds", "0.4") + self.assertEqual(result.returncode, 2, result.stderr) + self.assertIn("ran out of time", result.stderr) + self.assertLess(time.monotonic() - started, 5.0) + + def test_an_unreachable_api_is_unknown(self) -> None: + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + write_tree(root) + result = run_check(root, "--check-issues", "--api-url", "http://127.0.0.1:1", "--timeout-seconds", "5") + self.assertEqual(result.returncode, 2, result.stderr) + self.assertIn("state unknown: request failed", result.stderr) + + def test_offline_problems_still_fail_when_every_owner_is_open(self) -> None: + result, _ = self.check(dict(ALL_OPEN), text=edited("| 5 | 1 | 1 | 1 | 1 | 1 |", "| 9 | 1 | 1 | 1 | 1 | 1 |")) + self.assertEqual(result.returncode, 1, result.stderr) + self.assertIn("summary says 9 rows", result.stderr) + + def test_the_token_is_sent_only_when_one_is_set(self) -> None: + _, anonymous = self.check(dict(ALL_OPEN)) + self.assertEqual({header for _, header in anonymous.requests}, {None}) + for variable in ("GITHUB_TOKEN", "GH_TOKEN"): + with self.subTest(variable=variable): + _, stub = self.check(dict(ALL_OPEN), env={variable: "fixture-token"}) + self.assertEqual({header for _, header in stub.requests}, {"Bearer fixture-token"}) + + def test_arguments_are_validated_at_the_boundary(self) -> None: + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + write_tree(root) + for args, fragment in ( + (("--timeout-seconds", "nan"), "finite number of seconds above zero"), + (("--timeout-seconds", "inf"), "finite number of seconds above zero"), + (("--timeout-seconds", "0"), "finite number of seconds above zero"), + (("--timeout-seconds", "-3"), "finite number of seconds above zero"), + (("--timeout-seconds", "soon"), "is not a number"), + (("--api-url", "file:///etc/passwd"), "must be an http(s) URL"), + (("--api-url", "https://"), "must be an http(s) URL"), + (("--repository", "just-a-name"), "must be <owner>/<name>"), + ): + with self.subTest(args=args): + result = run_check(root, "--check-issues", *args) + self.assertNotEqual(result.returncode, 0) + self.assertIn(fragment, result.stderr) + result = run_check(root, env={"GITHUB_API_URL": "ftp://example.invalid"}) + self.assertNotEqual(result.returncode, 0) + self.assertIn("must be an http(s) URL", result.stderr) + + +class WorkflowTests(unittest.TestCase): + def jobs(self) -> dict[str, str]: + workflow = CI_WORKFLOW.read_text(encoding="utf-8") + body = workflow.split("\njobs:\n", 1)[1] + names = re.findall(r"^ ([A-Za-z0-9_-]+):$", body, flags=re.MULTILINE) + parts = re.split(r"^ [A-Za-z0-9_-]+:$", body, flags=re.MULTILINE)[1:] + return dict(zip(names, parts, strict=True)) + + def test_the_offline_check_is_part_of_the_required_checks(self) -> None: + jobs = self.jobs() + self.assertIn("run: python3 scripts/check_deleted_test_map.py\n", jobs["lint-compile-compose"]) + needs = re.search(r"^ needs: \[(.*)\]$", jobs["checks"], flags=re.MULTILINE) + self.assertIsNotNone(needs) + self.assertIn("lint-compile-compose", [name.strip() for name in needs.group(1).split(",")]) + + def test_the_issue_state_check_runs_can_fail_and_does_not_gate_merges(self) -> None: + jobs = self.jobs() + job = jobs["deleted-test-map-owner-issues"] + self.assertIn("run: python3 scripts/check_deleted_test_map.py --check-issues\n", job) + self.assertIn("GITHUB_TOKEN: ${{ github.token }}", job) + self.assertIn("issues: read", job) + self.assertIsNone(re.search(r"^\s*continue-on-error:", job, flags=re.MULTILINE)) + needs = re.search(r"^ needs: \[(.*)\]$", jobs["checks"], flags=re.MULTILINE) + self.assertNotIn("deleted-test-map-owner-issues", needs.group(1)) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_check_e2e_scenarios.py b/tests/test_check_e2e_scenarios.py new file mode 100644 index 000000000..0c555c223 --- /dev/null +++ b/tests/test_check_e2e_scenarios.py @@ -0,0 +1,455 @@ +#!/usr/bin/env python3 +"""Tests for scripts/check_e2e_scenarios.py (#543).""" + +from __future__ import annotations + +import json +from pathlib import Path +import subprocess +import sys +import tempfile +import textwrap +import unittest + + +ROOT = Path(__file__).resolve().parents[1] +SCRIPT = ROOT / "scripts" / "check_e2e_scenarios.py" +sys.path.insert(0, str(ROOT / "scripts")) + +import check_e2e_scenarios as scenarios # noqa: E402 + + +LIVE = "qbit-prism-server::live_regtest::two_node_tests::lost_race" +SMOKE = "qbit-prism-load::load_smoke::smoke_reconciles" +COMPONENT = "qbit-prism-server::ledger_postgres::alpha" +NIGHTLY = "qbit-prism-server::live_regtest::node_outage_tests::reindex" +WEEKLY = "qbit-prism-server::live_regtest::chain_events_load_tests::weekly_soak" +UNIT = "crates/qbit-prism-server/tests/stratum_protocol.rs::oversized_frames_close" + +LANES = """ +schema = "qbit.e2e-scenarios.v1" + +[lanes.pr] +title = "PR" +runs = true +workflow = ".github/workflows/ci.yml" + +[lanes.nightly] +title = "Nightly" +runs = true +workflow = ".github/workflows/prism-load-nightly.yml" + +[lanes.dispatch] +title = "Dispatch" +runs = true +workflow = ".github/workflows/prism-load-nightly.yml" + +[lanes.weekly] +title = "Weekly" +runs = true +workflow = ".github/workflows/prism-load-nightly.yml" + +[lanes.L4] +title = "Real node under load" +runs = false +owner = "#553" + +[lanes.L6] +title = "Shipped images" +runs = true +workflow = ".github/workflows/prism-load-nightly.yml" +""" + +SCENARIOS = f""" +[[scenario]] +id = "lost-race" +title = "Lost race" +owner = "#521" +lanes = ["pr"] +criteria = "Credited once." +runs = true +tests = ["{LIVE}"] + +[[scenario]] +id = "smoke" +title = "Smoke" +owner = "#521" +lanes = ["pr"] +criteria = "Reconciles." +runs = true +tests = ["{SMOKE}"] +presets = ["smoke"] + +[[scenario]] +id = "reindex" +title = "Reindex" +owner = "#521" +lanes = ["nightly"] +criteria = "Lands once." +runs = true +tests = ["{NIGHTLY}"] + +[[scenario]] +id = "soak-under-load" +title = "Soak under load" +owner = "#553" +lanes = ["weekly"] +criteria = "Lands every block." +runs = true +tests = ["{WEEKLY}"] + +[[scenario]] +id = "d1" +title = "D1" +owner = "#521" +lanes = ["nightly"] +criteria = "Gates." +runs = true +presets = ["d1-20k"] + +[[scenario]] +id = "cells" +title = "Cells" +owner = "#521" +lanes = ["dispatch"] +criteria = "Gates." +runs = true +presets = ["d1-500k"] + +[[scenario]] +id = "frames" +title = "Frames" +owner = "#468" +lanes = ["pr"] +criteria = "Closes." +runs = true +unit_tests = ["{UNIT}"] + +[[scenario]] +id = "images" +title = "Images" +owner = "#544" +lanes = ["L6"] +criteria = "Mines." +runs = true +lane_checks = ["found-block", "resume"] + +[[scenario]] +id = "under-load" +title = "Under load" +owner = "#553" +lanes = ["L4"] +criteria = "Set by #553." +runs = false +reason = "Needs L4." +""" + +RUST = """\ +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn oversized_frames_close() { +} + +#[test] +#[ignore = "opt-in"] +fn ignored_case() {} + +fn helper_not_a_test() {} +""" + + +WORKFLOW = ( + 'on:\n schedule:\n - cron: x\n - cron: "43 3 * * 0"\n - cron: "41 5 * * 6"\n' + " pull_request:\n workflow_dispatch:\n" + "scripts/prism_load_matrix.py\n--expected test/prism-nightly-gated-tests.txt\n" + "SELECTION: ${{ inputs.preset || (github.event.schedule == '41 5 * * 6' && 'weekly') }}\n" + "--expected test/prism-weekly-gated-tests.txt\n" + "scripts/prism_shipped_image_lane.py run\n" +) + + +class Fixture: + """A throwaway repository root holding just what the check reads.""" + + def __init__(self, root: Path) -> None: + self.root = root + self.write("test/prism-gated-tests.txt", "\n".join(sorted([LIVE, SMOKE, COMPONENT])) + "\n") + self.write("test/prism-nightly-gated-tests.txt", f"# opt-in\n{NIGHTLY}\n") + self.write("test/prism-weekly-gated-tests.txt", f"# opt-in weekly\n{WEEKLY}\n") + for name, schedule in (("d1-20k", "nightly"), ("d1-500k", "manual"), ("smoke", "smoke")): + self.write( + f"crates/qbit-prism-load/presets/{name}.json", + json.dumps({"schema": "qbit.prism.load-preset.v1", "name": name, "schedule": schedule}), + ) + self.write("crates/qbit-prism-server/tests/stratum_protocol.rs", RUST) + self.write( + ".github/workflows/ci.yml", + "run: python3 scripts/run_rust_test_shard.py\n--expected test/prism-gated-tests.txt\n", + ) + self.write( + ".github/workflows/prism-load-nightly.yml", + WORKFLOW, + ) + self.write("scripts/prism_shipped_image_lane.py", 'CHECKS = ("found-block", "resume")\n') + self.manifest = LANES + SCENARIOS + + def write(self, relative: str, text: str) -> None: + path = self.root / relative + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(text, encoding="utf-8") + + def problems(self) -> list[str]: + import tomllib + + return scenarios.check( + tomllib.loads(self.manifest), scenarios.Lanes(self.root), self.root + ) + + +class CheckE2eScenarios(unittest.TestCase): + def setUp(self) -> None: + self.tmp = tempfile.TemporaryDirectory() + self.fixture = Fixture(Path(self.tmp.name)) + + def tearDown(self) -> None: + self.tmp.cleanup() + + def assertProblem(self, needle: str) -> None: + problems = self.fixture.problems() + self.assertTrue(any(needle in p for p in problems), problems) + + def replace(self, old: str, new: str) -> None: + self.assertIn(old, self.fixture.manifest) + self.fixture.manifest = self.fixture.manifest.replace(old, new, 1) + + def test_a_manifest_that_matches_the_lanes_passes(self) -> None: + self.assertEqual(self.fixture.problems(), []) + + def test_a_scenario_test_the_manifest_does_not_name_fails(self) -> None: + added = "qbit-prism-server::live_regtest::zz_new_scenario" + self.fixture.write( + "test/prism-gated-tests.txt", + "\n".join(sorted([LIVE, SMOKE, COMPONENT, added])) + "\n", + ) + self.assertProblem(f"pr runs {added}, which no running scenario names") + + def test_component_gated_tests_need_no_entry(self) -> None: + self.assertFalse(any(COMPONENT in p for p in self.fixture.problems())) + + def test_an_unnamed_nightly_test_or_preset_fails(self) -> None: + self.fixture.write( + "crates/qbit-prism-load/presets/growth.json", + json.dumps({"schema": "qbit.prism.load-preset.v1", "name": "growth", "schedule": "nightly"}), + ) + self.assertProblem("nightly runs preset growth, which no running scenario names") + self.replace(f'tests = ["{NIGHTLY}"]', f'tests = ["{LIVE}"]') + self.assertProblem(f"nightly runs {NIGHTLY}, which no running scenario names") + + def test_a_claim_no_lane_runs_fails(self) -> None: + self.replace(f'tests = ["{LIVE}"]', 'tests = ["qbit-prism-server::live_regtest::gone"]') + self.assertProblem("is in none of") + self.replace('presets = ["d1-20k"]', 'presets = ["no-such-preset"]') + self.assertProblem("preset 'no-such-preset' is unknown or runs in no lane") + + def test_declared_lanes_must_match_where_the_evidence_runs(self) -> None: + self.replace('lanes = ["nightly"]\ncriteria = "Gates."', 'lanes = ["pr"]\ncriteria = "Gates."') + self.assertProblem("claims lane 'pr', but none of its evidence runs there") + self.assertProblem("its evidence runs in 'nightly'; add it to lanes") + + def test_a_running_scenario_on_a_lane_that_does_not_run_fails(self) -> None: + self.replace('lanes = ["pr"]\ncriteria = "Closes."', 'lanes = ["pr", "L4"]\ncriteria = "Closes."') + self.assertProblem("lane 'L4' is marked as not running") + + def test_unit_tests_must_exist_be_tests_and_not_be_ignored(self) -> None: + path = "crates/qbit-prism-server/tests/stratum_protocol.rs" + for name, message in ( + ("ignored_case", "is #[ignore]d"), + ("helper_not_a_test", "is not a #[test]"), + ("missing", "defines no fn missing"), + ): + with self.subTest(name=name): + self.fixture.manifest = LANES + SCENARIOS.replace(UNIT, f"{path}::{name}") + self.assertProblem(message) + self.fixture.manifest = LANES + SCENARIOS.replace(UNIT, "crates/nope.rs::x") + self.assertProblem("crates/nope.rs does not exist") + + def test_an_unexercised_scenario_needs_a_reason_and_no_evidence(self) -> None: + self.replace('reason = "Needs L4."', "") + self.assertProblem("scenario under-load is unexercised and gives no reason") + self.fixture.manifest = LANES + SCENARIOS.replace( + 'reason = "Needs L4."', f'reason = "x"\ntests = ["{LIVE}"]' + ) + self.assertProblem("is unexercised but cites evidence") + + def test_a_running_scenario_needs_evidence(self) -> None: + self.replace(f'tests = ["{NIGHTLY}"]', "") + self.assertProblem("scenario reindex runs but cites no tests, presets, unit_tests, lane_checks or suites") + + def test_malformed_entries_are_refused(self) -> None: + cases = ( + ('owner = "#468"', 'owner = "468"', "owner '468' is not #<issue>"), + ('id = "frames"', 'id = "lost-race"', "scenario id 'lost-race' appears 2 times"), + ('criteria = "Closes."', 'criteria = " "', "criteria is missing or empty"), + ('criteria = "Closes."', 'criteria = "Closes."\ntest = "typo"', "unknown key 'test'"), + ('lanes = ["L4"]', 'lanes = ["L9"]', "lane 'L9' is not defined"), + ('lanes = ["L4"]', "lanes = []", "lanes must be a non-empty list"), + ('runs = false', 'runs = "no"', "runs must be true or false"), + ('schema = "qbit.e2e-scenarios.v1"', 'schema = "v0"', "schema is 'v0'"), + ) + for old, new, message in cases: + with self.subTest(message=message): + self.fixture.manifest = LANES + SCENARIOS + self.replace(old, new) + self.assertProblem(message) + + def test_a_lane_marked_running_needs_its_wiring(self) -> None: + self.fixture.write(".github/workflows/prism-load-nightly.yml", "on:\n workflow_dispatch:\n") + self.assertProblem("prism-load-nightly.yml no longer contains 'cron:'") + self.fixture.manifest = self.fixture.manifest.replace( + 'runs = false\nowner = "#553"', "runs = true" + ) + self.assertProblem("lane L4 is marked running, but this check cannot enumerate it") + + def test_a_lane_not_running_names_its_owner(self) -> None: + self.replace('runs = false\nowner = "#553"', "runs = false") + self.assertProblem("lane L4 does not run yet and names no owner issue") + + def test_a_weekly_preset_runs_in_the_weekly_lane_only_while_it_is_wired(self) -> None: + self.fixture.write( + "crates/qbit-prism-load/presets/soak.json", + json.dumps({"schema": "qbit.prism.load-preset.v1", "name": "soak", "schedule": "weekly"}), + ) + self.assertProblem("weekly runs preset soak, which no running scenario names") + self.fixture.manifest += ( + '\n[[scenario]]\nid = "soak"\ntitle = "Soak"\nowner = "#575"\nlanes = ["weekly"]\n' + 'criteria = "Its gates."\nruns = true\npresets = ["soak"]\n' + ) + self.assertEqual(self.fixture.problems(), []) + self.fixture.write( + ".github/workflows/prism-load-nightly.yml", + WORKFLOW.replace("github.event.schedule == '41 5 * * 6' && 'weekly'", ""), + ) + self.assertProblem("lane weekly: .github/workflows/prism-load-nightly.yml no longer contains") + + def test_a_weekly_test_runs_in_the_weekly_lane_and_each_needs_a_scenario(self) -> None: + added = "qbit-prism-server::live_regtest::chain_events_load_tests::weekly_tips" + self.fixture.write( + "test/prism-weekly-gated-tests.txt", "\n".join(sorted([WEEKLY, added])) + "\n" + ) + self.assertProblem(f"weekly runs {added}, which no running scenario names") + self.replace('lanes = ["weekly"]\ncriteria = "Lands', 'lanes = ["nightly"]\ncriteria = "Lands') + self.assertProblem("scenario soak-under-load claims lane 'nightly', but none of its evidence runs there") + self.assertProblem("scenario soak-under-load: its evidence runs in 'weekly'; add it to lanes") + + def test_the_weekly_lane_needs_the_live_weekly_job_wired(self) -> None: + for needle in ('cron: "43 3 * * 0"', "--expected test/prism-weekly-gated-tests.txt"): + with self.subTest(needle=needle): + self.fixture.write( + ".github/workflows/prism-load-nightly.yml", WORKFLOW.replace(needle, "") + ) + self.assertProblem(f"lane weekly: .github/workflows/prism-load-nightly.yml no longer contains {needle!r}") + + def test_l6_checks_run_in_l6_and_each_needs_a_scenario(self) -> None: + self.fixture.write("scripts/prism_shipped_image_lane.py", 'CHECKS = ("found-block", "resume", "new")\n') + self.assertProblem("L6 runs check new, which no running scenario names") + self.replace('lane_checks = ["found-block", "resume"]', 'lane_checks = ["found-block", "gone"]') + self.assertProblem("defines no check 'gone', so L6 does not run it") + + def test_an_l6_check_claimed_on_another_lane_fails(self) -> None: + self.replace('lanes = ["L6"]\ncriteria = "Mines."', 'lanes = ["nightly"]\ncriteria = "Mines."') + self.assertProblem("claims lane 'nightly', but none of its evidence runs there") + self.assertProblem("its evidence runs in 'L6'; add it to lanes") + + def test_l6_marked_running_needs_its_trigger_and_driver_in_the_workflow(self) -> None: + self.fixture.write( + ".github/workflows/prism-load-nightly.yml", + "on:\n schedule:\n - cron: x\n workflow_dispatch:\n" + "scripts/prism_load_matrix.py\n--expected test/prism-nightly-gated-tests.txt\n", + ) + self.assertProblem("no longer contains 'cron: \"43 3 * * 0\"'") + self.assertProblem("no longer contains 'pull_request:'") + self.assertProblem("no longer contains 'scripts/prism_shipped_image_lane.py run'") + + def test_a_driver_without_checks_is_refused(self) -> None: + self.fixture.write("scripts/prism_shipped_image_lane.py", "OTHER = 1\n") + with self.assertRaisesRegex(ValueError, "defines no CHECKS"): + self.fixture.problems() + + def test_a_suite_runs_in_its_lane_and_is_cited_with_only_its_own_presets(self) -> None: + # #550: a suite (presets/suites.toml) runs in the lane it names. + self.fixture.write( + "crates/qbit-prism-load/presets/suites.toml", + 'schema = "qbit.prism.load-suites.v1"\n[suites.l3]\ndescription = "L3"\n' + 'lane = "L3"\nissues = ["#550"]\nrepeats = 3\npresets = ["d1-500k"]\n', + ) + self.fixture.write( + ".github/workflows/prism-load-l3.yml", + "\n".join(needle for needle in scenarios.LANE_WIRING["L3"][1]) + "\n", + ) + self.fixture.manifest += ( + '\n[lanes.L3]\ntitle = "L3"\nruns = true\nowner = "#550"\n' + 'workflow = ".github/workflows/prism-load-l3.yml"\n' + ) + self.assertProblem("L3 runs suite l3, which no running scenario names") + entry = ( + '\n[[scenario]]\nid = "matrix"\ntitle = "Matrix"\nowner = "#550"\n' + 'lanes = ["dispatch", "L3"]\ncriteria = "Gates."\nruns = true\n' + 'suites = ["l3"]\npresets = ["d1-500k"]\n' + ) + base = self.fixture.manifest + self.fixture.manifest = base + entry + self.assertEqual(self.fixture.problems(), []) + self.fixture.manifest = base + entry.replace('presets = ["d1-500k"]', 'presets = ["d1-20k"]') + self.assertProblem("preset 'd1-20k' is in none of its suites (l3)") + self.fixture.manifest = base + entry.replace('suites = ["l3"]', 'suites = ["l3", "gone"]') + self.assertProblem("suite 'gone' is not in") + # A suite's lane is claimed like a preset's. + self.fixture.manifest = base + entry.replace('lanes = ["dispatch", "L3"]', 'lanes = ["dispatch"]') + self.assertProblem("its evidence runs in 'L3'; add it to lanes") + (self.fixture.root / ".github/workflows/prism-load-l3.yml").write_text("on: push\n") + self.fixture.manifest = base + entry + self.assertProblem("prism-load-l3.yml no longer contains") + + def test_the_smoke_preset_runs_in_pr_only_through_the_load_smoke_test(self) -> None: + self.fixture.write("test/prism-gated-tests.txt", "\n".join(sorted([LIVE, COMPONENT])) + "\n") + self.assertProblem("preset 'smoke' is unknown or runs in no lane") + + def test_a_smoke_preset_also_runs_in_pr_through_the_faults_test(self) -> None: + # #554: `faults` runs its own smoke preset, and every test in that + # binary is an end-to-end scenario the manifest has to name. + faults = "qbit-prism-load::faults::drain_and_lock" + self.fixture.write("test/prism-gated-tests.txt", "\n".join(sorted([LIVE, COMPONENT, faults])) + "\n") + self.replace(f'tests = ["{SMOKE}"]', f'tests = ["{faults}"]') + self.assertEqual(self.fixture.problems(), []) + added = "qbit-prism-load::faults::zz_unnamed" + self.fixture.write( + "test/prism-gated-tests.txt", + "\n".join(sorted([LIVE, COMPONENT, faults, added])) + "\n", + ) + self.assertProblem(f"pr runs {added}, which no running scenario names") + + +class CheckedInManifest(unittest.TestCase): + # This is how the check runs in CI: the required python-tests shards run + # every module under tests/, so a manifest that disagrees with the lanes + # fails the PR. + def test_the_checked_in_manifest_passes(self) -> None: + result = subprocess.run( + [sys.executable, str(SCRIPT)], cwd=ROOT, text=True, capture_output=True, check=False + ) + self.assertEqual(result.returncode, 0, result.stderr) + self.assertIn("each claim holds", result.stdout) + + def test_every_unexercised_scenario_links_an_issue_and_gives_a_reason(self) -> None: + import tomllib + + manifest = tomllib.loads(scenarios.DEFAULT_MANIFEST.read_text(encoding="utf-8")) + for scenario in manifest["scenario"]: + if scenario["runs"]: + continue + with self.subTest(id=scenario["id"]): + self.assertRegex(scenario["owner"], r"^#[1-9][0-9]*$") + self.assertTrue(scenario["reason"].strip()) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_check_env_production_gate.py b/tests/test_check_env_production_gate.py index 498c00960..184b1db0d 100644 --- a/tests/test_check_env_production_gate.py +++ b/tests/test_check_env_production_gate.py @@ -3,7 +3,6 @@ from __future__ import annotations import os -import shlex import shutil import subprocess import tempfile @@ -13,6 +12,12 @@ ROOT_DIR = Path(__file__).resolve().parents[1] CHECK_ENV = ROOT_DIR / "scripts" / "check-env.sh" +FAKE_PUBLIC_VALIDATION = ('if [ "$1" = compose ]; then\n' + # Compose consumes the piped overlay. Exiting before + # reading it can SIGPIPE the producer under pipefail. + ' case " $* " in *" -f - "*) cat >/dev/null ;; esac\n' + ' echo "PRISM public database configuration valid; authentication is checked by readiness"\n' + 'fi\n') class CheckEnvProductionGateTests(unittest.TestCase): def setUp(self) -> None: root = Path(self.enterContext(tempfile.TemporaryDirectory())) @@ -37,17 +42,12 @@ def production_prism_env(self, root: Path) -> dict[str, str]: "PRISM_POSTGRES_IMAGE": "registry.example.invalid/postgres@sha256:" + "e" * 64, "PRISM_POSTGRES_DATA_SOURCE": str(root / "postgres-data"), "PRISM_POSTGRES_WAL_SOURCE": str(root / "postgres-wal"), - "PRISM_POSTGRES_REPLICA_DATA_SOURCE": str(root / "postgres-replica"), "PRISM_AUDIT_DATA_SOURCE": str(root / "prism-audit"), + "PRISM_SECRETS_SOURCE": str(root / "prism-secrets"), "PRISM_DATABASE_URL": "postgresql://example.invalid/qbit", "PRISM_POSTGRES_PASSWORD": "not-default", - "PRISM_MANIFEST_SIGNING_SEED_HEX": "42" * 32, - "PRISM_LEDGER_ATTESTATION_SIGNING_SEED_HEX": "43" * 32, "PRISM_LEDGER_WRITER_PUBLIC_KEY_HEX": "44" * 32, - "PRISM_LEDGER_WRITER_ID": "managed-writer", - "PRISM_LEDGER_WRITER_EPOCH": "7", "PRISM_AUDIT_DIR": "/var/lib/qbit/prism/audit", - "PRISM_EVIDENCE_PATH": "/var/lib/qbit/prism/evidence.json", "PRISM_STRATUM_STALE_GRACE_SECONDS": "3", "PRISM_STRATUM_SHARE_DIFF": "1024", "PRISM_STRATUM_VARDIFF": "1", @@ -60,9 +60,6 @@ def production_prism_env(self, root: Path) -> dict[str, str]: "PRISM_STRATUM_VARDIFF_MAX_STEP_DOWN": "4", "PRISM_STRATUM_VARDIFF_EWMA_ALPHA": "0.4", "PRISM_STRATUM_VARDIFF_RETARGET_TOLERANCE": "0.25", - "PRISM_STRATUM_VARDIFF_IDLE_SWEEP_SECONDS": "15", - "PRISM_SHARE_COMMIT_BATCH_SIZE": "64", - "PRISM_SHARE_COMMIT_LINGER_MILLISECONDS": "5", "PRISM_SHARE_COMMIT_TIMEOUT_SECONDS": "15", "PRISM_STRATUM_SEND_TIMEOUT_SECONDS": "20", } @@ -105,10 +102,30 @@ def write_fake_docker(self, root: Path, *, exit_code: int = 0) -> Path: fake_bin = root / "bin" fake_bin.mkdir() docker = fake_bin / "docker" - docker.write_text(f"#!/bin/sh\nexit {exit_code}\n", encoding="utf-8") + # These tests isolate the surrounding production gates. Credential + # validation itself is qualified through the native CLI/image tests. + docker.write_text( + '#!/bin/sh\n' + FAKE_PUBLIC_VALIDATION + f'exit {exit_code}\n', encoding="utf-8") docker.chmod(0o755) return fake_bin + def test_public_validation_fixture_consumes_compose_stdin(self) -> None: + with tempfile.TemporaryDirectory() as temp_dir: + docker = self.write_fake_docker(Path(temp_dir)) / "docker" + # More than a pipe buffer makes an early reader exit deterministic, + # instead of depending on scheduling of the small real overlay. + result = subprocess.run( + [ + "/bin/bash", "-o", "pipefail", "-c", + 'printf "%1048576s" "" | "$1" compose -f -', + "fixture-overlay", str(docker), + ], + text=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE, + timeout=5, + ) + self.assertEqual(result.returncode, 0, result.stderr) + self.assertIn("PRISM public database configuration valid", result.stdout) + def production_testnet4_env(self, root: Path) -> dict[str, str]: checkout, commit = self.write_pinned_qbit_checkout(root) fake_bin = self.write_fake_docker(root) @@ -138,13 +155,8 @@ def production_testnet4_env(self, root: Path) -> dict[str, str]: "CKPOOL_STARTDIFF": "65536", "PRISM_DATABASE_URL": "postgresql://example.invalid/qbit", "PRISM_POSTGRES_PASSWORD": "not-default", - "PRISM_MANIFEST_SIGNING_SEED_HEX": "42" * 32, - "PRISM_LEDGER_ATTESTATION_SIGNING_SEED_HEX": "43" * 32, "PRISM_LEDGER_WRITER_PUBLIC_KEY_HEX": "44" * 32, - "PRISM_LEDGER_WRITER_ID": "managed-writer", - "PRISM_LEDGER_WRITER_EPOCH": "7", "PRISM_AUDIT_DIR": "/var/lib/qbit/prism/audit", - "PRISM_EVIDENCE_PATH": "/var/lib/qbit/prism/evidence.json", "PRISM_STRATUM_STALE_GRACE_SECONDS": "3", "PRISM_STRATUM_SHARE_DIFF": "1024", "PRISM_STRATUM_VARDIFF_MIN_DIFF": "1024", @@ -162,6 +174,10 @@ def run_check_env( env = os.environ.copy() env.pop("QBIT_GIT_COMMIT", None) env["PATH"] = f"{self.docker_bin}:{env['PATH']}" + # A deployment names its own pool fee recipient: production refuses + # .env.example's lab program without one (#535). An empty override + # leaves the sourced files' value in place, as for every setting. + env["PRISM_POOL_FEE_ADDRESS"] = "qb1doctortestpoolfee" # Tests exercising successful Docker checks or a minimal PATH supply # their own controlled tool fixtures through overrides. env.update(overrides) @@ -183,9 +199,6 @@ def isolated_check_env_root(self, directory: Path) -> tuple[Path, Path]: shutil.copyfile(CHECK_ENV, script) shutil.copyfile(ROOT_DIR / ".env.example", root / ".env.example") shutil.copyfile(ROOT_DIR / "config" / "upstream.env.example", root / "config" / "upstream.env") - entrypoint = Path("docker/qbit/qbit-entrypoint.sh") - (root / entrypoint).parent.mkdir(parents=True) - shutil.copyfile(ROOT_DIR / entrypoint, root / entrypoint) return root, script def test_deploy_env_file_is_loaded_as_the_final_config_layer(self) -> None: @@ -747,103 +760,54 @@ def test_production_rejects_reused_state_paths_before_docker_check(self) -> None self.assertIn("PRISM_POSTGRES_DATA_SOURCE and PRISM_POSTGRES_WAL_SOURCE must be distinct", result.stderr) self.assertNotIn("docker is required", result.stderr) - def test_production_validates_replica_storage_before_docker(self) -> None: - key = "PRISM_POSTGRES_REPLICA_DATA_SOURCE" + def test_production_rejects_direct_signing_seeds_before_docker_check(self) -> None: + for name in ("PRISM_MANIFEST_SIGNING_SEED_HEX", "PRISM_LEDGER_ATTESTATION_SIGNING_SEED_HEX"): + with self.subTest(name=name), tempfile.TemporaryDirectory() as temp_dir: + env = self.production_prism_env(Path(temp_dir)) + env[name] = "42" * 32 + + result = self.run_check_env(**env) + + self.assertNotEqual(result.returncode, 0) + self.assertIn(f"production mode requires mounted {name}_FILE; unset {name}", result.stderr) + self.assertNotIn("42" * 32, result.stderr) + self.assertNotIn("docker is required", result.stderr) + + def test_production_treats_whitespace_signing_seeds_as_unset(self) -> None: with tempfile.TemporaryDirectory() as temp_dir: - root, script = self.isolated_check_env_root(Path(temp_dir)) - env = self.production_prism_env(root) - env[key] = "" - fake_bin = self.write_fake_docker(root) - docker_log = root / "docker-calls.log" - (fake_bin / "docker").write_text( - '#!/bin/sh\nprintf "%s\\n" "$*" >> "$DOCKER_CALL_LOG"\nexit 0\n', - encoding="utf-8", - ) - deploy_env = root / "deployment.env" - env.update({ - "PATH": f"{fake_bin}:{os.environ['PATH']}", - "DOCKER_CALL_LOG": str(docker_log), - "DEPLOY_ENV_FILE": str(deploy_env), - }) - cases = [ - (None, "absolute host path"), - ("", "absolute host path"), - ("replica-volume", "absolute host path"), - ("./replica-data", "absolute host path"), - ] - for other in ( - "QBIT_DATA_SOURCE", - "PRISM_POSTGRES_DATA_SOURCE", - "PRISM_POSTGRES_WAL_SOURCE", - "PRISM_AUDIT_DATA_SOURCE", - ): - cases.append((env[other] + "/", "must be distinct")) - for value, error in cases: - with self.subTest(value=value): - deploy_env.write_text( - "" if value is None else f"{key}={shlex.quote(value)}\n", - encoding="utf-8", - ) - result = self.run_check_env(script=script, cwd=root, **env) - self.assertNotEqual(result.returncode, 0) - self.assertIn(key, result.stderr) - self.assertIn(error, result.stderr) - self.assertFalse(docker_log.exists(), "validation called Docker") + env = self.production_prism_env(Path(temp_dir)) + env["PRISM_MANIFEST_SIGNING_SEED_HEX"] = " \t " - def test_replica_storage_rejects_overlap_and_aliases_before_docker(self) -> None: - key = "PRISM_POSTGRES_REPLICA_DATA_SOURCE" + result = self.run_check_env(**env) + + self.assertNotIn("requires mounted", result.stderr) + + def test_release_provenance_requires_distinct_absolute_prism_secrets_source(self) -> None: with tempfile.TemporaryDirectory() as temp_dir: root = Path(temp_dir) - env = self.production_prism_env(root) - primary = Path(env["PRISM_POSTGRES_DATA_SOURCE"]) - primary.mkdir(parents=True) - marker = primary / "PG_VERSION" - marker.write_text("16") - alias = root / "primary-alias" - alias.symlink_to(primary, target_is_directory=True) - cases = [ - str(primary.parent), str(primary / "replica"), - str(primary) + "/../" + primary.name + "//", - str(alias), str(alias / "new-replica"), - ] - for value in cases: + for value, message in ( + ("", "release provenance requires PRISM_SECRETS_SOURCE as an absolute host path"), + ("relative/secrets", "release provenance requires PRISM_SECRETS_SOURCE as an absolute host path"), + (str(root / "prism-audit"), "PRISM_AUDIT_DATA_SOURCE and PRISM_SECRETS_SOURCE must be distinct"), + ): with self.subTest(value=value): - result = self.run_check_env(**{**env, key: value}) + env = self.production_prism_env(root) + env["PRISM_SECRETS_SOURCE"] = value + + result = self.run_check_env(**env) + self.assertNotEqual(result.returncode, 0) - self.assertIn("non-overlapping", result.stderr) - self.assertNotIn("docker daemon", result.stderr) - self.assertEqual(marker.read_text(), "16") - # Component boundaries matter: primary and primary-copy are siblings. - result = self.run_check_env(**{**env, key: str(primary) + "-copy"}) - self.assertNotIn("non-overlapping", result.stderr) - self.assertIn("docker daemon", result.stderr) - - def test_production_replica_storage_honors_configuration_precedence(self) -> None: - key = "PRISM_POSTGRES_REPLICA_DATA_SOURCE" - with tempfile.TemporaryDirectory() as temp_dir: - root, script = self.isolated_check_env_root(Path(temp_dir)) - checkout, commit = self.write_pinned_qbit_checkout(root) - fake_bin = self.write_fake_docker(root) - env = self.production_prism_env(root) - env.update({ - "PATH": f"{fake_bin}:{os.environ['PATH']}", - "QBIT_SRC_DIR": str(checkout), - "QBIT_SRC_DIR_OVERRIDE": str(checkout), - "QBIT_GIT_COMMIT": commit, - }) - deploy_env = root / "deployment.env" - env["DEPLOY_ENV_FILE"] = str(deploy_env) - for file_value, shell_value in ( - (str(root / "file-replica"), ""), - ("invalid-named-volume", str(root / "shell-replica")), - ): - with self.subTest(file_value=file_value, shell_value=shell_value): - deploy_env.write_text(f"{key}={shlex.quote(file_value)}\n", encoding="utf-8") - result = self.run_check_env( - script=script, cwd=root, **{**env, key: shell_value} - ) - self.assertEqual(result.returncode, 0, result.stderr) - self.assertIn(f"qbit source checkout verified at {commit}", result.stdout) + self.assertIn(message, result.stderr) + + def test_lab_accepts_direct_signing_seeds(self) -> None: + result = self.run_check_env( + MINING_LANES="prism", + PRISM_MANIFEST_SIGNING_SEED_HEX="42" * 32, + PRISM_LEDGER_ATTESTATION_SIGNING_SEED_HEX="43" * 32, + ) + + self.assertNotIn("requires mounted", result.stderr) + self.assertIn("docker daemon is not reachable", result.stderr) def test_qbit_mainnet_auxpow_requires_mainnet_parent(self) -> None: result = self.run_check_env( @@ -868,13 +832,8 @@ def test_production_mode_rejects_prism_test_bypass_before_docker_check(self) -> BITCOIN_RPC_PASSWORD="not-default", PRISM_DATABASE_URL="postgresql://example.invalid/qbit", PRISM_POSTGRES_PASSWORD="not-default", - PRISM_MANIFEST_SIGNING_SEED_HEX="42" * 32, - PRISM_LEDGER_ATTESTATION_SIGNING_SEED_HEX="43" * 32, PRISM_LEDGER_WRITER_PUBLIC_KEY_HEX="44" * 32, - PRISM_LEDGER_WRITER_ID="managed-writer", - PRISM_LEDGER_WRITER_EPOCH="7", PRISM_AUDIT_DIR="/var/lib/qbit/prism/audit", - PRISM_EVIDENCE_PATH="/var/lib/qbit/prism/evidence.json", CKPOOL_MINDIFF="1024", CKPOOL_STARTDIFF="65536", CKPOOL_REQUIRE_P2MR_PAYOUT="1", @@ -897,13 +856,8 @@ def test_production_mode_rejects_default_prism_postgres_password(self) -> None: BITCOIN_RPC_PASSWORD="not-default", PRISM_DATABASE_URL="postgresql://example.invalid/qbit", PRISM_POSTGRES_PASSWORD="change-this", - PRISM_MANIFEST_SIGNING_SEED_HEX="42" * 32, - PRISM_LEDGER_ATTESTATION_SIGNING_SEED_HEX="43" * 32, PRISM_LEDGER_WRITER_PUBLIC_KEY_HEX="44" * 32, - PRISM_LEDGER_WRITER_ID="managed-writer", - PRISM_LEDGER_WRITER_EPOCH="7", PRISM_AUDIT_DIR="/var/lib/qbit/prism/audit", - PRISM_EVIDENCE_PATH="/var/lib/qbit/prism/evidence.json", CKPOOL_MINDIFF="1024", CKPOOL_STARTDIFF="65536", CKPOOL_REQUIRE_P2MR_PAYOUT="1", @@ -925,13 +879,8 @@ def test_production_mode_rejects_default_prism_database_url_password(self) -> No BITCOIN_RPC_PASSWORD="not-default", PRISM_DATABASE_URL="postgresql://qbit:change-this@prism-postgres:5432/qbit", PRISM_POSTGRES_PASSWORD="not-default", - PRISM_MANIFEST_SIGNING_SEED_HEX="42" * 32, - PRISM_LEDGER_ATTESTATION_SIGNING_SEED_HEX="43" * 32, PRISM_LEDGER_WRITER_PUBLIC_KEY_HEX="44" * 32, - PRISM_LEDGER_WRITER_ID="managed-writer", - PRISM_LEDGER_WRITER_EPOCH="7", PRISM_AUDIT_DIR="/var/lib/qbit/prism/audit", - PRISM_EVIDENCE_PATH="/var/lib/qbit/prism/evidence.json", CKPOOL_MINDIFF="1024", CKPOOL_STARTDIFF="65536", CKPOOL_REQUIRE_P2MR_PAYOUT="1", @@ -1064,13 +1013,8 @@ def test_production_git_provider_requires_exact_commit_pin(self) -> None: "BITCOIN_RPC_PASSWORD": "not-default", "PRISM_DATABASE_URL": "postgresql://example.invalid/qbit", "PRISM_POSTGRES_PASSWORD": "not-default", - "PRISM_MANIFEST_SIGNING_SEED_HEX": "42" * 32, - "PRISM_LEDGER_ATTESTATION_SIGNING_SEED_HEX": "43" * 32, "PRISM_LEDGER_WRITER_PUBLIC_KEY_HEX": "44" * 32, - "PRISM_LEDGER_WRITER_ID": "managed-writer", - "PRISM_LEDGER_WRITER_EPOCH": "7", "PRISM_AUDIT_DIR": "/var/lib/qbit/prism/audit", - "PRISM_EVIDENCE_PATH": "/var/lib/qbit/prism/evidence.json", "PRISM_STRATUM_STALE_GRACE_SECONDS": "3", "CKPOOL_MINDIFF": "1024", "CKPOOL_STARTDIFF": "65536", @@ -1127,20 +1071,23 @@ def test_mainnet_accepts_bounded_stale_grace(self) -> None: self.assertEqual(result.returncode, 0, result.stderr) self.assertNotIn("PRISM_STRATUM_STALE_GRACE_SECONDS", result.stderr) - def test_prism_stale_grace_python3_requirement_matches_prior_prerequisites(self) -> None: - # python3 was already required in production (production difficulty), but - # never for lab bring-up. Validating stale grace must not change that: - # production still fails loudly, lab skips the check instead. + def test_prism_stale_grace_validates_without_python_in_lab_and_production(self) -> None: with tempfile.TemporaryDirectory() as temp_dir: - root = Path(temp_dir) + root, script = self.isolated_check_env_root(Path(temp_dir)) + (root / "docker" / "qbit").mkdir(parents=True) + shutil.copyfile( + ROOT_DIR / "docker" / "qbit" / "qbit-entrypoint.sh", + root / "docker" / "qbit" / "qbit-entrypoint.sh", + ) + checkout, commit = self.write_pinned_qbit_checkout(root) minimal_bin = root / "no-python-bin" minimal_bin.mkdir() for tool in ( + "awk", "bash", "sh", "cat", "dirname", - "docker", "env", "git", "rm", @@ -1153,43 +1100,65 @@ def test_prism_stale_grace_python3_requirement_matches_prior_prerequisites(self) if tool == "docker" else shutil.which(tool) ) - if resolved is not None: - (minimal_bin / tool).symlink_to(resolved) + self.assertIsNotNone(resolved, f"test needs {tool}") + (minimal_bin / tool).symlink_to(resolved) + docker_calls = root / "docker-calls" + docker = minimal_bin / "docker" + docker.write_text( + '#!/bin/sh\nprintf "called\\n" >> "$FAKE_DOCKER_CALLS"\n' + FAKE_PUBLIC_VALIDATION, + encoding="utf-8", + ) + docker.chmod(0o755) self.assertIsNone( shutil.which("python3", path=str(minimal_bin)), "the minimal PATH must not expose python3", ) - - lab = self.run_check_env( - PATH=str(minimal_bin), - MINING_LANES="prism", - PRISM_STRATUM_STALE_GRACE_SECONDS="not-a-number", - ) - self.assertNotIn("PRISM_STRATUM_STALE_GRACE_SECONDS", lab.stderr) - self.assertIn( - "python3 not found; skipping PRISM_STRATUM_STALE_GRACE_SECONDS validation", - lab.stdout, - ) - - production = self.run_check_env( - PATH=str(minimal_bin), - MINING_LANES="prism", - QBIT_PRODUCTION="1", - QBIT_CHAIN="signet", - QBIT_CHAIN_FLAG="-signet", - PRISM_STRATUM_STALE_GRACE_SECONDS="not-a-number", - ) - self.assertNotEqual(production.returncode, 0) - self.assertIn( - "python3 is required to validate PRISM_STRATUM_STALE_GRACE_SECONDS", - production.stderr, - ) + common = self.production_prism_env(root) + common.update({ + "PATH": str(minimal_bin), + "FAKE_DOCKER_CALLS": str(docker_calls), + "QBIT_GIT_COMMIT": commit, + "QBIT_SRC_DIR": str(checkout), + "QBIT_SRC_DIR_OVERRIDE": str(checkout), + }) + for production in (False, True): + mode = {} if production else { + "QBIT_PRODUCTION": "0", + "QBIT_TOOLS_PRODUCTION": "0", + "QBIT_REQUIRE_RELEASE_PROVENANCE": "0", + "QBIT_CHAIN": "regtest", + "QBIT_CHAIN_FLAG": "-regtest", + } + for grace, valid in ( + ("3", True), ("0", True), ("+.5", True), + ("18446744073709549568", True), + ("not-a-number", False), ("-1", False), + ("18446744073709551616", False), (r"\063", False), + ): + with self.subTest(production=production, grace=grace): + docker_calls.unlink(missing_ok=True) + result = self.run_check_env( + script=script, cwd=root, + **{**common, **mode, "PRISM_STRATUM_STALE_GRACE_SECONDS": grace}, + ) + if valid: + self.assertEqual(result.returncode, 0, result.stderr) + self.assertTrue(docker_calls.exists(), result.stdout) + else: + self.assertNotEqual(result.returncode, 0) + self.assertIn("PRISM_STRATUM_STALE_GRACE_SECONDS", result.stderr) + self.assertFalse(docker_calls.exists(), "invalid grace reached Docker") + self.assertNotIn("python", result.stderr + result.stdout) def test_prism_rejects_invalid_stale_grace_before_docker_check(self) -> None: # The exported value is what Compose hands the coordinator, so the # doctor validates it (not the .env.example default) in every mode, # with the coordinator's own float syntax and range. - for grace in ("-1", "not-a-number", "nan", "inf", "1" + "0" * 400): + for grace in ( + "-1", "not-a-number", "nan", "inf", "1" + "0" * 400, + "1e20", "18446744073709551615", "18446744073709550592", + " 3 ", "1_0", "0x3", "3\\n", "1,5", + ): with self.subTest(grace=grace): result = self.run_check_env( MINING_LANES="prism", @@ -1204,7 +1173,10 @@ def test_prism_rejects_invalid_stale_grace_before_docker_check(self) -> None: self.assertNotIn("docker is required", result.stderr) def test_prism_accepts_runtime_float_stale_grace_syntax(self) -> None: - for grace in ("0", "3", "0.5", "1e-1", ".5", "3."): + for grace in ( + "0", "-0", "+0", "3", "0.5", "1e-1", ".5", "3.", + "+.5", "3E+0", "1e-999", "-1e-999", "18446744073709550591", " \t ", + ): with self.subTest(grace=grace): result = self.run_check_env( MINING_LANES="prism", @@ -1268,7 +1240,7 @@ def test_mainnet_provenance_passes_without_capacity_evidence_and_verifies_pin(se fake_bin = root / "bin" fake_bin.mkdir() docker = fake_bin / "docker" - docker.write_text("#!/bin/sh\nexit 0\n", encoding="utf-8") + docker.write_text("#!/bin/sh\n" + FAKE_PUBLIC_VALIDATION + "exit 0\n", encoding="utf-8") docker.chmod(0o755) common = self.production_prism_env(root) common.update({ @@ -1359,6 +1331,43 @@ def test_release_provenance_opt_in_applies_without_production_mode(self) -> None ) self.assertNotIn("docker is required", result.stderr) + def test_production_rejects_literal_difficulty_escapes_before_docker(self) -> None: + names = ( + "PRISM_STRATUM_SHARE_DIFF", + "PRISM_STRATUM_VARDIFF_MIN_DIFF", + "PRISM_STRATUM_VARDIFF_START_DIFF", + "PRISM_STRATUM_VARDIFF_MAX_DIFF", + ) + with tempfile.TemporaryDirectory() as temp_dir: + root = Path(temp_dir) + checkout, commit = self.write_pinned_qbit_checkout(root) + fake_bin = self.write_fake_docker(root) + docker_calls = root / "docker-calls" + (fake_bin / "docker").write_text( + '#!/bin/sh\nprintf "called\\n" >> "$FAKE_DOCKER_CALLS"\n' + FAKE_PUBLIC_VALIDATION, + encoding="utf-8", + ) + common = self.production_prism_env(root) + common.update({ + "PATH": f"{fake_bin}:{os.environ['PATH']}", + "FAKE_DOCKER_CALLS": str(docker_calls), + "QBIT_GIT_COMMIT": commit, + "QBIT_SRC_DIR": str(checkout), + "QBIT_SRC_DIR_OVERRIDE": str(checkout), + **dict.fromkeys(names, "1"), + }) + valid = self.run_check_env(**common) + self.assertEqual(valid.returncode, 0, valid.stderr) + self.assertTrue(docker_calls.exists()) + for name in names: + for value in (r"\x31", r"\061"): + with self.subTest(name=name, value=value): + docker_calls.unlink(missing_ok=True) + result = self.run_check_env(**{**common, name: value}) + self.assertNotEqual(result.returncode, 0) + self.assertIn(f"{name} must be a decimal number", result.stderr) + self.assertFalse(docker_calls.exists(), "invalid difficulty reached Docker") + def test_production_rejects_unsafe_prism_difficulty_profiles(self) -> None: with tempfile.TemporaryDirectory() as temp_dir: root = Path(temp_dir) @@ -1428,13 +1437,8 @@ def test_production_requires_commit_pinned_mining_sources(self) -> None: "BITCOIN_RPC_PASSWORD": "not-default", "PRISM_DATABASE_URL": "postgresql://example.invalid/qbit", "PRISM_POSTGRES_PASSWORD": "not-default", - "PRISM_MANIFEST_SIGNING_SEED_HEX": "42" * 32, - "PRISM_LEDGER_ATTESTATION_SIGNING_SEED_HEX": "43" * 32, "PRISM_LEDGER_WRITER_PUBLIC_KEY_HEX": "44" * 32, - "PRISM_LEDGER_WRITER_ID": "managed-writer", - "PRISM_LEDGER_WRITER_EPOCH": "7", "PRISM_AUDIT_DIR": "/var/lib/qbit/prism/audit", - "PRISM_EVIDENCE_PATH": "/var/lib/qbit/prism/evidence.json", "PRISM_STRATUM_STALE_GRACE_SECONDS": "3", "CKPOOL_MINDIFF": "1024", "CKPOOL_STARTDIFF": "65536", @@ -1481,5 +1485,115 @@ def test_ctv_rejects_non_positive_explicit_fee_rate_before_docker_check(self) -> self.assertNotIn("docker is required", result.stderr) + def test_settlement_requires_a_pool_fee_in_every_mode_before_docker_check(self) -> None: + # #525, #535: mirrors the server's startup refusal in both settlement + # modes, including an explicit process-environment disable over the + # sourced defaults. + for ctv, mode in ( + ("1", "PRISM_CTV_SETTLEMENT_ENABLED=1"), + ("0", "direct settlement (PRISM_CTV_SETTLEMENT_ENABLED=0)"), + ): + for fee_enabled in ("0", "false", "OFF"): + with self.subTest(ctv=ctv, fee_enabled=fee_enabled): + result = self.run_check_env( + MINING_LANES="prism", + PRISM_CTV_SETTLEMENT_ENABLED=ctv, + PRISM_POOL_FEE_ENABLED=fee_enabled, + ) + + self.assertNotEqual(result.returncode, 0) + self.assertIn( + f"{mode} requires PRISM_POOL_FEE_ENABLED=1 (PRISM_POOL_FEE_BPS=0 is allowed)", + result.stderr, + ) + self.assertNotIn("docker is required", result.stderr) + + def test_env_example_defaults_enable_a_pool_fee_in_every_mode(self) -> None: + # #535: a fresh setup copied from .env.example passes the fee check + # in either mode and stops only at the Docker boundary. + for ctv in ("0", "1"): + with self.subTest(ctv=ctv): + result = self.run_check_env( + MINING_LANES="prism", PRISM_CTV_SETTLEMENT_ENABLED=ctv + ) + + self.assertNotIn("requires PRISM_POOL_FEE_ENABLED=1", result.stderr) + self.assertIn("docker", result.stderr) + + def test_process_environment_disables_a_pool_fee_the_env_file_enables(self) -> None: + # Compose prefers the shell value over the env file; so must the doctor. + with tempfile.TemporaryDirectory() as temp_dir: + deploy_env = Path(temp_dir) / "deploy.env" + deploy_env.write_text( + "MINING_LANES=prism\nPRISM_CTV_SETTLEMENT_ENABLED=1\n" + "PRISM_POOL_FEE_ENABLED=1\nPRISM_POOL_FEE_BPS=0\n", + encoding="utf-8", + ) + accepted = self.run_check_env(DEPLOY_ENV_FILE=str(deploy_env)) + refused = self.run_check_env( + DEPLOY_ENV_FILE=str(deploy_env), PRISM_POOL_FEE_ENABLED="0" + ) + + self.assertNotIn("requires PRISM_POOL_FEE_ENABLED=1", accepted.stderr) + self.assertNotEqual(refused.returncode, 0) + self.assertIn( + "PRISM_CTV_SETTLEMENT_ENABLED=1 requires PRISM_POOL_FEE_ENABLED=1", refused.stderr + ) + + def test_an_enabled_pool_fee_needs_a_recipient(self) -> None: + # The lab program is sourced from .env.example; clearing it with no + # address leaves no recipient, which the server refuses. + with tempfile.TemporaryDirectory() as temp_dir: + deploy_env = Path(temp_dir) / "deploy.env" + deploy_env.write_text( + "MINING_LANES=prism\nPRISM_POOL_FEE_ENABLED=1\n" + "PRISM_POOL_FEE_P2MR_PROGRAM_HEX=\n", + encoding="utf-8", + ) + refused = self.run_check_env( + DEPLOY_ENV_FILE=str(deploy_env), PRISM_POOL_FEE_ADDRESS="" + ) + accepted = self.run_check_env(DEPLOY_ENV_FILE=str(deploy_env)) + + self.assertNotEqual(refused.returncode, 0) + self.assertIn("PRISM_POOL_FEE_ENABLED=1 needs a pool fee recipient", refused.stderr) + self.assertNotIn("docker is required", refused.stderr) + self.assertNotIn("needs a pool fee recipient", accepted.stderr) + + def test_production_rejects_the_lab_pool_fee_recipient(self) -> None: + # #535: .env.example's program is refused in production unless an + # address replaces it; lab mode keeps it. + with tempfile.TemporaryDirectory() as temp_dir: + env = self.production_prism_env(Path(temp_dir)) + refused = self.run_check_env(**env, PRISM_POOL_FEE_ADDRESS="") + with_address = self.run_check_env(**env) + own_program = self.run_check_env( + **env, + PRISM_POOL_FEE_ADDRESS="", + PRISM_POOL_FEE_P2MR_PROGRAM_HEX="ab" * 32, + ) + lab = self.run_check_env(MINING_LANES="prism", PRISM_POOL_FEE_ADDRESS="") + + message = "production rejects the development pool fee recipient" + self.assertNotEqual(refused.returncode, 0) + self.assertIn(message, refused.stderr) + self.assertNotIn("docker is required", refused.stderr) + for result in (with_address, own_program, lab): + self.assertNotIn(message, result.stderr) + + def test_settlement_accepts_any_enabled_pool_fee_in_every_mode(self) -> None: + for ctv in ("0", "1"): + for fee_enabled in ("1", "True", "on"): + with self.subTest(ctv=ctv, fee_enabled=fee_enabled): + result = self.run_check_env( + MINING_LANES="prism", + PRISM_CTV_SETTLEMENT_ENABLED=ctv, + PRISM_POOL_FEE_ENABLED=fee_enabled, + PRISM_POOL_FEE_BPS="0", + ) + + self.assertNotIn("requires PRISM_POOL_FEE_ENABLED=1", result.stderr) + + if __name__ == "__main__": unittest.main() diff --git a/tests/test_check_env_public_credentials.py b/tests/test_check_env_public_credentials.py new file mode 100644 index 000000000..de7b74353 --- /dev/null +++ b/tests/test_check_env_public_credentials.py @@ -0,0 +1,212 @@ +"""Shell launcher boundary; real SQLx/Compose checks live in the image suite.""" +import json +import itertools +import os +from pathlib import Path +import subprocess +import shutil +import sys +import tempfile +import unittest + +ROOT = Path(__file__).resolve().parents[1] +VALID = "PRISM public database configuration valid; authentication is checked by readiness" + + +class CheckEnvPublicCredentialLauncherTests(unittest.TestCase): + def run_preflight(self, docker_body, *, settings=None, deploy=None, args=()): + with tempfile.TemporaryDirectory(prefix="public-preflight-launcher-") as directory: + root = Path(directory) + (root / "scripts").mkdir() + (root / "config").mkdir() + shutil.copyfile(ROOT / "scripts/check-env.sh", root / "scripts/check-env.sh") + shutil.copyfile(ROOT / ".env.example", root / ".env.example") + shutil.copyfile(ROOT / "config/upstream.env.example", root / "config/upstream.env.example") + docker = root / "docker" + docker.write_text(f"#!{sys.executable}\nimport os, sys\n{docker_body}\n", encoding="utf-8") + docker.chmod(0o755) + env = {"PATH": f"{root}:{os.environ['PATH']}", **(settings or {})} + if deploy is not None: + env_file = root / "deployment.env" + env_file.write_text(deploy, encoding="utf-8") + env["DEPLOY_ENV_FILE"] = str(env_file) + result = subprocess.run( + ["/bin/bash", str(root / "scripts/check-env.sh"), "--public-reader-only", *args], + cwd=root, env=env, capture_output=True, text=True, timeout=10, + ) + self.assertFalse("fixture-secret" in result.stdout + result.stderr, + "launcher diagnostics exposed credential data") + return result + + def test_original_exported_inputs_and_explicit_empty_overrides_reach_compose(self): + for value in ("", "postgres://reader:fixture-secret@db/reader"): + with self.subTest(empty=not value): + result = self.run_preflight( + 'assert os.environ["PRISM_PUBLIC_DATABASE_URL"] == os.environ["EXPECTED_DSN"]\n' + 'assert os.environ["PRISM_PUBLIC_POSTGRES_PASSWORD"] == ""\n' + 'assert os.environ["QBIT_PRODUCTION"] == "1"\n' + 'assert "PRISM_MANIFEST_SIGNING_SEED_HEX" not in os.environ\n' + 'assert not any("fixture-secret" in arg for arg in sys.argv)\n' + 'assert sys.argv[-2:] == ["qbit-prism-server", "check-public-database-config"]\n' + 'assert all(arg in sys.argv for arg in ["--rm", "--no-deps", "--pull", "never", "-T"])\n' + f'print({VALID!r})', + settings={"PRISM_PUBLIC_DATABASE_URL": value, "EXPECTED_DSN": value, + "PRISM_PUBLIC_POSTGRES_PASSWORD": "", "QBIT_PRODUCTION": "1"}, + deploy="PRISM_PUBLIC_DATABASE_URL=postgres://file-reader:fixture-secret@db/file\n" + "PRISM_PUBLIC_POSTGRES_PASSWORD=fixture-secret-file\nQBIT_PRODUCTION=0\n", + ) + self.assertEqual(result.returncode, 0, result.stderr) + + def test_env_file_only_inputs_stay_in_file_and_overlay_order_is_preserved(self): + result = self.run_preflight( + 'assert "PRISM_PUBLIC_DATABASE_URL" not in os.environ\n' + 'assert "PRISM_PUBLIC_POSTGRES_PASSWORD" not in os.environ\n' + 'assert sys.argv[sys.argv.index("--env-file") + 1].endswith("upstream.env.example")\n' + 'assert sys.argv.index("compose.production.yaml") < sys.argv.index("compose.prism-external-db.yaml")\n' + 'assert sys.argv.index("compose.prism-external-db.yaml") < sys.argv.index("compose.prism-ha.yaml")\n' + f'print({VALID!r})', + deploy="PRISM_PUBLIC_DATABASE_URL=postgres://reader:fixture-secret@db/reader\n", + args=("--compose-file", "compose.production.yaml", "--compose-file", "compose.prism-external-db.yaml", + "--compose-file", "compose.prism-ha.yaml"), + ) + self.assertEqual(result.returncode, 0, result.stderr) + + def test_arbitrary_interpolation_inputs_keep_caller_precedence(self): + result = self.run_preflight( + 'assert os.environ["READER_SECRET"] == "fixture-secret-caller"\n' + f'print({VALID!r})', + settings={"READER_SECRET": "fixture-secret-caller"}, + deploy="READER_SECRET=fixture-secret-file\n" + "PRISM_PUBLIC_POSTGRES_PASSWORD=${READER_SECRET}\n", + ) + self.assertEqual(result.returncode, 0, result.stderr) + + def test_unknown_launcher_failures_and_empty_results_fail_without_raw_diagnostics(self): + for body in ('print("postgres://reader:fixture-secret@db/reader", file=sys.stderr)\nsys.exit(1)', + 'print("fixture-secret")'): + result = self.run_preflight(body) + self.assertNotEqual(result.returncode, 0) + self.assertIn("public reader validation", result.stderr) + + def test_config_errors_remain_distinct_from_launcher_failures(self): + result = self.run_preflight( + 'print("Error: invalid public PRISM_DATABASE_URL: fixture-secret", file=sys.stderr)\nsys.exit(1)') + self.assertNotEqual(result.returncode, 0) + self.assertIn("public reader: invalid public PRISM_DATABASE_URL", result.stderr) + + def test_connection_options_keep_the_specific_value_free_diagnostic(self): + result = self.run_preflight( + 'print("Error: invalid public PRISM_DATABASE_URL connection options: fixture-secret", file=sys.stderr)\nsys.exit(1)') + self.assertNotEqual(result.returncode, 0) + self.assertIn("public reader: invalid public PRISM_DATABASE_URL connection options", result.stderr) + + def test_make_lab_prepares_image_before_mandatory_validation(self): + result = self.run_preflight( + 'from pathlib import Path\n' + 'prepared = Path("prepared")\n' + 'if "build" in sys.argv:\n' + ' assert sys.argv[-1] == "prism-public-api"\n' + ' prepared.touch()\n' + 'else:\n' + ' assert prepared.exists()\n' + ' assert "compose.prism-external-db.yaml" in sys.argv\n' + f' print({VALID!r})', + args=("--make-deployment", "--compose-file", "compose.prism-external-db.yaml"), + ) + self.assertEqual(result.returncode, 0, result.stderr) + + def test_production_selectors_never_build_and_apply_production_before_overlays(self): + for settings, deploy in (({"QBIT_PRODUCTION": "1"}, None), + ({"QBIT_TOOLS_PRODUCTION": "1"}, None), + ({"QBIT_CHAIN": "mainnet"}, None), + ({}, "QBIT_TOOLS_PRODUCTION=1\n")): + with self.subTest(settings=settings, env_file=deploy is not None): + result = self.run_preflight( + 'assert "build" not in sys.argv and "pull" not in sys.argv\n' + 'production = next(i for i, arg in enumerate(sys.argv) if arg.endswith("/compose.production.yaml"))\n' + 'assert production < sys.argv.index("compose.prism-external-db.yaml")\n' + f'print({VALID!r})', + settings=settings, deploy=deploy, + args=("--make-deployment", "--compose-file", "compose.prism-external-db.yaml"), + ) + self.assertEqual(result.returncode, 0, result.stderr) + + def test_lab_build_failure_is_not_skipped_and_never_exposes_build_output(self): + result = self.run_preflight( + 'assert "build" in sys.argv\n' + 'print("fixture-secret", file=sys.stderr)\nsys.exit(1)', args=("--make-deployment",), + ) + self.assertNotEqual(result.returncode, 0) + self.assertIn("public reader image preparation failed", result.stderr) + + def test_digest_pinned_lab_artifacts_are_validated_without_build_or_pull(self): + result = self.run_preflight( + 'assert "build" not in sys.argv and "pull" not in sys.argv\n' + f'print({VALID!r})', + settings={"PRISM_COORDINATOR_IMAGE": "fixture/prism@sha256:" + "a" * 64}, + args=("--make-deployment",), + ) + self.assertEqual(result.returncode, 0, result.stderr) + + def test_lab_preparation_still_rejects_invalid_qbit_chain_selection(self): + result = self.run_preflight('raise AssertionError("Docker must not run")', + settings={"QBIT_CHAIN": "regtest", "QBIT_CHAIN_FLAG": "-chain=main"}, + args=("--make-deployment",)) + self.assertNotEqual(result.returncode, 0) + self.assertIn("QBIT_CHAIN=regtest requires QBIT_CHAIN_FLAG=-regtest", result.stderr) + + def test_real_make_doctor_and_launch_share_overlays_and_prepare_fresh_image(self): + for target, bitcoin_chain in itertools.product(("doctor", "up-prism-pool"), ("regtest", "mainnet", "testnet4")): + with self.subTest(target=target, bitcoin_chain=bitcoin_chain), tempfile.TemporaryDirectory(prefix="public-reader-make-") as directory: + root = Path(directory) + for name in ("Makefile", "scripts/check-env.sh", ".env.example", "config/upstream.env.example", + "docker/qbit/qbit-entrypoint.sh"): + destination = root / name + destination.parent.mkdir(parents=True, exist_ok=True) + shutil.copyfile(ROOT / name, destination) + for name in ("CMakeLists.txt", "src/CMakeLists.txt", "test/functional/test_framework/auxpow.py"): + path = root / "qbit" / name + path.parent.mkdir(parents=True, exist_ok=True) + path.touch() + (root / "scripts/prepare-qbit-source.sh").write_text( + '#!/bin/sh\nprintf "%s/qbit\\n" "$PWD"\n', encoding="utf-8") + (root / "bin").mkdir() + docker = root / "bin/docker" + docker.write_text( + f'#!{sys.executable}\nimport json, sys\nfrom pathlib import Path\n' + 'with Path("calls.jsonl").open("a") as log: log.write(json.dumps(sys.argv[1:]) + "\\n")\n' + 'if "build" in sys.argv: Path("prepared").touch()\n' + 'if "run" in sys.argv:\n' + ' assert Path("prepared").exists()\n' + f' print({VALID!r})\n' + 'if "--environment" in sys.argv:\n' + ' print("PRISM_LEDGER_WRITER_PUBLIC_KEY_HEX=fixture-public-key")\n' + ' print("PRISM_MANIFEST_SIGNING_SEED_HEX=fixture-test-seed")\n' + ' print("PRISM_LEDGER_ATTESTATION_SIGNING_SEED_HEX=fixture-test-seed")\n', encoding="utf-8") + docker.chmod(0o755) + overlays = ["compose.prism-external-db.yaml", "compose.prism-ha.yaml"] + result = subprocess.run( + ["make", "--no-print-directory", target, "MINING_LANES=prism", "QBIT_PROVIDER=source", + f"COMPOSE_OVERLAY_FILES={' '.join(overlays)}"], cwd=root, + # Disabled Bitcoin-lane settings must not gate a PRISM lab. + env={"PATH": f"{root / 'bin'}:{os.environ['PATH']}", "BITCOIN_CHAIN": bitcoin_chain, + "BITCOIN_CHAIN_FLAG": "-regtest"}, + text=True, capture_output=True, timeout=15, + ) + self.assertEqual(result.returncode, 0, result.stderr) + calls = [json.loads(line) for line in (root / "calls.jsonl").read_text().splitlines()] + build = next(i for i, args in enumerate(calls) if "build" in args) + validation = next(i for i, args in enumerate(calls) if "run" in args) + self.assertLess(build, validation) + for args in calls: + if args[0] != "compose": + continue + self.assertLess(args.index(overlays[0]), args.index(overlays[1])) + if target == "up-prism-pool": + launch = next(i for i, args in enumerate(calls) if "up" in args) + self.assertLess(validation, launch) + self.assertNotIn("prism-postgres", calls[launch]) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_check_gate_env_reads.py b/tests/test_check_gate_env_reads.py new file mode 100644 index 000000000..fe10f6781 --- /dev/null +++ b/tests/test_check_gate_env_reads.py @@ -0,0 +1,111 @@ +#!/usr/bin/env python3 + +from __future__ import annotations + +import subprocess +import sys +import tempfile +import unittest +from pathlib import Path + + +ROOT = Path(__file__).resolve().parents[1] +SCRIPT = ROOT / "scripts" / "check_gate_env_reads.py" +GATE_CRATE = ROOT / "crates" / "qbit-prism-test-gate" + + +def write(root: Path, relative: str, text: str) -> None: + path = root / relative + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(text, encoding="utf-8") + + +def run_check(root: Path) -> subprocess.CompletedProcess[str]: + return subprocess.run( + [sys.executable, str(SCRIPT), "--root", str(root)], + cwd=ROOT, + text=True, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + check=False, + ) + + +class CheckGateEnvReadsTests(unittest.TestCase): + def test_a_direct_read_outside_the_gate_crate_fails_and_is_named(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + write( + root, + "crates/server/tests/db.rs", + "fn f() {\n" + ' let _ = std::env::var("PRISM_TEST_DATABASE_URL");\n' + ' let _ = std::env::var_os("QBITD_BIN");\n' + "}\n", + ) + write(root, "crates/server/src/lib.rs", 'const J: &str = "GITHUB_JOB";\n') + result = run_check(root) + self.assertEqual(result.returncode, 1, result.stdout) + self.assertIn("crates/server/tests/db.rs:2: PRISM_TEST_DATABASE_URL", result.stderr) + self.assertIn("crates/server/tests/db.rs:3: QBITD_BIN", result.stderr) + self.assertIn("crates/server/src/lib.rs:1: GITHUB_JOB", result.stderr) + self.assertIn("3 direct read(s)", result.stderr) + + def test_the_gate_crate_itself_may_name_the_variables(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + write( + root, + "crates/qbit-prism-test-gate/src/lib.rs", + 'pub const NAME: &str = "PRISM_TEST_DATABASE_URL";\n' + 'pub const SWITCH: &str = "PRISM_TEST_REQUIRE_INTEGRATION";\n', + ) + write(root, "crates/server/src/lib.rs", "pub fn nothing() {}\n") + result = run_check(root) + self.assertEqual(result.returncode, 0, result.stderr) + self.assertIn("no direct reads", result.stdout) + + def test_doc_comments_and_longer_literals_are_not_reads(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + write( + root, + "crates/server/tests/db.rs", + "//! PRISM_TEST_DATABASE_URL=postgres://x cargo test\n" + '#[ignore = "requires disposable PRISM_TEST_DATABASE_URL; run in database CI"]\n' + "fn f() {\n" + ' assert!(message.contains("PRISM_TEST_DATABASE_URL is set but not readable"));\n' + ' let _ = std::env::var("PRISM_DATABASE_URL");\n' + "}\n", + ) + result = run_check(root) + self.assertEqual(result.returncode, 0, result.stderr) + + def test_target_directories_are_skipped(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + write(root, "crates/server/src/lib.rs", "pub fn nothing() {}\n") + write(root, "target/debug/build/x.rs", 'let _ = env::var("QBITD_BIN");\n') + write(root, "crates/target/x.rs", 'let _ = env::var("QBITD_BIN");\n') + result = run_check(root) + self.assertEqual(result.returncode, 0, result.stderr) + + def test_a_checkout_without_crates_is_an_error_not_a_pass(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + result = run_check(Path(tmp)) + self.assertEqual(result.returncode, 1, result.stdout) + self.assertIn("is not a directory", result.stderr) + + def test_the_checked_in_tree_reads_gate_variables_only_through_the_gate(self) -> None: + result = run_check(ROOT) + self.assertEqual(result.returncode, 0, result.stderr) + self.assertIn("outside crates/qbit-prism-test-gate", result.stdout) + self.assertTrue((GATE_CRATE / "src" / "lib.rs").is_file()) + + def test_ci_runs_the_check(self) -> None: + workflow = (ROOT / ".github" / "workflows" / "ci.yml").read_text(encoding="utf-8") + self.assertIn("python3 scripts/check_gate_env_reads.py", workflow) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_check_gate_manifest.py b/tests/test_check_gate_manifest.py new file mode 100644 index 000000000..6106db81e --- /dev/null +++ b/tests/test_check_gate_manifest.py @@ -0,0 +1,209 @@ +#!/usr/bin/env python3 + +from __future__ import annotations + +import os +import subprocess +import sys +import tempfile +import unittest +from pathlib import Path + + +ROOT = Path(__file__).resolve().parents[1] +SCRIPT = ROOT / "scripts" / "check_gate_manifest.py" +EXPECTED_LIST = ROOT / "test" / "prism-gated-tests.txt" +CI_WORKFLOW = ROOT / ".github" / "workflows" / "ci.yml" +GATE_LIB = ROOT / "crates" / "qbit-prism-test-gate" / "src" / "lib.rs" + +A = "qbit-prism-server::ledger_postgres::alpha" +B = "qbit-prism-server::ledger_postgres::two_x::beta" +C = "qbit-prism::window_daemon_gate::gamma" + + +def run_check( + manifest: str | None, + expected: str | None, + *, + log: str | None = None, + extra: list[str] | None = None, + env: dict[str, str] | None = None, +) -> subprocess.CompletedProcess[str]: + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + args = [sys.executable, str(SCRIPT)] + manifest_path = root / "manifest.txt" + if manifest is not None: + manifest_path.write_text(manifest, encoding="utf-8") + args += ["--manifest", str(manifest_path)] + expected_path = root / "expected.txt" + if expected is not None: + expected_path.write_text(expected, encoding="utf-8") + args += ["--expected", str(expected_path)] + if log is not None: + log_path = root / "tests.log" + log_path.write_text(log, encoding="utf-8") + args += ["--log", str(log_path)] + args += extra or [] + merged = os.environ.copy() + merged.pop("GITHUB_STEP_SUMMARY", None) + merged.update(env or {}) + return subprocess.run( + args, cwd=ROOT, env=merged, text=True, capture_output=True, check=False + ) + + +class CheckGateManifestTests(unittest.TestCase): + def test_a_complete_manifest_passes_and_is_printed(self) -> None: + result = run_check(f"executed {B}\nexecuted {A}\nexecuted {C}\n", f"{A}\n{B}\n{C}\n") + self.assertEqual(result.returncode, 0, result.stderr) + self.assertIn("all 3 expected gated tests executed", result.stdout) + self.assertIn("## Gated test execution manifest", result.stdout) + self.assertIn(f"executed {A}\nexecuted {B}\nexecuted {C}", result.stdout) + self.assertEqual(result.stderr, "") + + def test_a_test_that_did_not_execute_fails(self) -> None: + result = run_check(f"executed {A}\n", f"{A}\n{B}\n") + self.assertEqual(result.returncode, 1, result.stdout) + self.assertIn(f"{B} did not execute", result.stderr) + self.assertIn("1 distinct gated tests executed, below the minimum of 2", result.stderr) + + def test_a_skipped_or_failed_line_fails_even_when_every_test_is_listed(self) -> None: + for kind in ("skipped", "failed"): + with self.subTest(kind=kind): + result = run_check(f"executed {A}\n{kind} {B}\n", f"{A}\n{B}\n") + self.assertEqual(result.returncode, 1, result.stdout) + self.assertIn(f"{B} was {kind}", result.stderr) + + def test_an_unlisted_executed_test_fails_so_the_list_must_grow(self) -> None: + result = run_check(f"executed {A}\nexecuted {C}\n", f"{A}\n") + self.assertEqual(result.returncode, 1, result.stdout) + self.assertIn(f"{C} executed but is not in the expected list", result.stderr) + self.assertIn("test/prism-gated-tests.txt", result.stderr) + + def test_repeated_executions_count_once_and_are_noted(self) -> None: + result = run_check(f"executed {A}\nexecuted {A}\nexecuted {B}\n", f"{A}\n{B}\n") + self.assertEqual(result.returncode, 0, result.stderr) + self.assertIn(f"{A} executed 2 times; counted once", result.stdout) + + def test_a_malformed_manifest_line_fails(self) -> None: + for line in ("ran x::y", f"executed {A}", "executed", "executed not-an-id"): + with self.subTest(line=line): + result = run_check(f"{line}\n", f"{A}\n") + self.assertEqual(result.returncode, 1, result.stdout) + self.assertIn("is not a line the gate writes", result.stderr) + + def test_a_missing_or_empty_manifest_fails(self) -> None: + result = run_check(None, f"{A}\n") + self.assertEqual(result.returncode, 1, result.stdout) + self.assertIn("manifest", result.stderr) + self.assertIn("is missing", result.stderr) + result = run_check("", f"{A}\n") + self.assertEqual(result.returncode, 1, result.stdout) + self.assertIn("is empty", result.stderr) + + def test_the_expected_list_must_be_sorted_unique_and_well_formed(self) -> None: + cases = { + f"{B}\n{A}\n": "must be sorted", + f"{A}\n{A}\n": "repeats", + "": "names no tests", + "not an id\n": "is not a <package>::<binary>::<test> id", + } + for content, message in cases.items(): + with self.subTest(content=content): + result = run_check(f"executed {A}\n", content) + self.assertEqual(result.returncode, 1, result.stdout) + self.assertIn(message, result.stderr) + result = run_check(f"executed {A}\n", f"# comment\n\n{A} # trailing\n") + self.assertEqual(result.returncode, 0, result.stderr) + + def test_the_log_is_scanned_for_skip_lines(self) -> None: + skip_lines = [ + f"[prism-test-gate] skipped {A}: PRISM_TEST_DATABASE_URL is unset or empty", + "skipping PostgreSQL integration test; set PRISM_TEST_DATABASE_URL", + "PRISM_TEST_DATABASE_URL not set; database contract test skipped", + "set PRISM_TEST_DATABASE_URL for real rollup transaction tests", + "SKIPPED: jsonb_ceiling_gate did not run.", + ] + for line in skip_lines: + with self.subTest(line=line): + result = run_check(f"executed {A}\n", f"{A}\n", log=f"test alpha ... ok\n{line}\n") + self.assertEqual(result.returncode, 1, result.stdout) + self.assertIn(f"log shows a skipped gated test: {line}", result.stderr) + benign = ( + "test alpha ... ok\n" + "test beta ... ignored, requires disposable PRISM_TEST_DATABASE_URL; run explicitly\n" + "test result: ok. 3 passed; 0 failed; 2 ignored\n" + ) + result = run_check(f"executed {A}\n", f"{A}\n", log=benign) + self.assertEqual(result.returncode, 0, result.stderr) + + def test_the_report_is_appended_to_the_step_summary(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + summary = Path(tmp) / "summary.md" + summary.write_text("# before\n", encoding="utf-8") + result = run_check(f"executed {A}\n", f"{A}\n", env={"GITHUB_STEP_SUMMARY": str(summary)}) + self.assertEqual(result.returncode, 0, result.stderr) + text = summary.read_text(encoding="utf-8") + self.assertTrue(text.startswith("# before\n## Gated test execution manifest"), text) + self.assertIn(f"executed {A}", text) + with tempfile.TemporaryDirectory() as tmp: + summary = Path(tmp) / "explicit.md" + result = run_check(f"failed {A}\n", f"{A}\n", extra=["--summary", str(summary)]) + self.assertEqual(result.returncode, 1, result.stdout) + self.assertIn("**FAILED**", summary.read_text(encoding="utf-8")) + + def test_the_skip_prefix_matches_the_gate_crate(self) -> None: + lib = GATE_LIB.read_text(encoding="utf-8") + script = SCRIPT.read_text(encoding="utf-8") + self.assertIn('pub const SKIP_PREFIX: &str = "[prism-test-gate] skipped";', lib) + self.assertIn('SKIP_PREFIX = "[prism-test-gate] skipped"', script) + + def test_the_checked_in_expected_list_is_valid_and_names_the_ignored_runs(self) -> None: + result = run_check( + "".join(f"executed {line}\n" for line in EXPECTED_LIST.read_text(encoding="utf-8").splitlines() if line and not line.startswith("#")), + EXPECTED_LIST.read_text(encoding="utf-8"), + ) + self.assertEqual(result.returncode, 0, result.stderr) + ids = [ + line.split("#", 1)[0].strip() + for line in EXPECTED_LIST.read_text(encoding="utf-8").splitlines() + ] + ids = [i for i in ids if i] + self.assertIn( + "qbit-prism-server::stratum_admission_postgres::" + "ten_thousand_unsubscribed_connections_do_not_advance_postgres_sequence", + ids, + ) + self.assertIn( + "qbit-prism-server::observability_database::" + "collector_uses_real_schema_pending_rows_and_failed_read_semantics", + ids, + ) + oracle = [i for i in ids if i.startswith("qbit-prism-server::window_read_oracle::")] + self.assertEqual(len(oracle), 8, oracle) + dependency = [i for i in ids if i.startswith("qbit-prism-server::issued_job_dependency::")] + self.assertEqual(len(dependency), 7, dependency) + + def test_ci_requires_shards_and_combined_proof_and_uploads_the_manifest(self) -> None: + workflow = CI_WORKFLOW.read_text(encoding="utf-8") + native = workflow.split("\n prism-native-postgres:", 1)[1].split("\n docker-builds:", 1)[0] + self.assertIn('PRISM_TEST_REQUIRE_INTEGRATION: "1"', native) + self.assertIn("PRISM_TEST_GATE_MANIFEST=", native) + self.assertIn('>> "${GITHUB_ENV}"', native) + self.assertIn("scripts/check_gate_manifest.py", native) + self.assertIn("--expected test/prism-gated-tests.txt", native) + self.assertIn("scripts/run_rust_test_shard.py", native) + self.assertIn("fail-fast: false", native) + self.assertIn("SHARD_COUNT: ${{ strategy.job-total }}", native) + self.assertIn("name: prism-gate-shard-${{ matrix.shard }}", native) + self.assertIn("needs: [prism-native-postgres]", native) + self.assertIn('run: test "$SHARD_RESULT" = success', native) + self.assertIn("actions/download-artifact@", native) + checks = workflow.split("\n checks:", 1)[1].split("\n ckpool-rejects", 1)[0] + self.assertIn("prism-native-postgres, prism-integration-proof]", checks) + self.assertIn("actions/upload-artifact@", native) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_check_prism_pool_acquires.py b/tests/test_check_prism_pool_acquires.py new file mode 100644 index 000000000..3fae425be --- /dev/null +++ b/tests/test_check_prism_pool_acquires.py @@ -0,0 +1,183 @@ +"""Negative controls for the real production checkout census.""" +from collections import Counter +import json +from pathlib import Path +import shutil +import tempfile +import unittest + +from scripts import check_prism_pool_acquires as guard + + +class PoolAcquireGuardTests(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls.inventory = json.loads((guard.ROOT / guard.INVENTORY).read_text()) + + def setUp(self): + self.directory = tempfile.TemporaryDirectory() + self.addCleanup(self.directory.cleanup) + self.root = Path(self.directory.name) + shutil.copytree(guard.ROOT / guard.SOURCE, self.root / guard.SOURCE) + + def append(self, path, text): + target = self.root / guard.SOURCE / path + target.write_text(target.read_text() + text) + + def errors(self): + sites, _, _ = guard.census(self.root) + return guard.check(sites, self.inventory) + + def test_existing_classifications_are_current(self): + self.assertEqual(self.errors(), []) + + def test_new_direct_implicit_pool_executor_is_rejected(self): + self.append("ledger/window.rs", ''' +impl Ledger { + async fn negative_implicit(&self) -> sqlx::Result<i32> { + sqlx::query_scalar("SELECT 1").fetch_one(&self.pool).await + } +} +''') + self.assertTrue(any("unclassified" in e and "negative_implicit" in e for e in self.errors())) + + def test_new_explicit_pool_acquire_is_rejected(self): + self.append("ledger/window.rs", ''' +impl Ledger { + async fn negative_explicit(&self) { + let connection = self.pool.acquire().await; + } +} +''') + self.assertTrue(any("unclassified" in e and "negative_explicit" in e for e in self.errors())) + + def test_alias_and_multiline_raw_sql_executors_are_rejected(self): + self.append("ledger/window.rs", ''' +impl Ledger { + async fn negative_alias(&self) { + let database = self.pool.clone(); + sqlx::raw_sql(r#"SELECT '/* not a comment */'"#) + .execute( + &database + ).await; + database.begin().await; + sqlx::Executor::fetch_one(&database, "SELECT 1").await; + } +} +''') + errors = [e for e in self.errors() if "negative_alias" in e] + self.assertEqual(len(errors), 3, errors) + + def test_generic_helper_and_imported_alias_callers_are_rejected(self): + self.append("ledger/window.rs", ''' +use crate::ledger::audit_completeness as renamed_completeness; +impl Ledger { + async fn negative_helper(&self) { + let database = &self.pool; + crate::ledger::audit_completeness(database).await; + renamed_completeness(database).await; + } +} +''') + errors = [e for e in self.errors() if "negative_helper" in e] + self.assertEqual(len(errors), 2, errors) + + def test_new_helper_module_is_scanned(self): + self.append("lib.rs", "\nmod negative_module;\n") + (self.root / guard.SOURCE / "negative_module.rs").write_text(''' +async fn helper(database: &sqlx::PgPool) { + database.acquire().await; +} +''') + self.assertTrue(any("negative_module.rs" in e for e in self.errors())) + + def test_borrowed_connection_is_not_counted_as_checkout(self): + before = guard.census(self.root) + self.append("ledger/window.rs", ''' +async fn negative_borrowed(connection: &mut sqlx::PgConnection) { + sqlx::query("SELECT 1").fetch_one(&mut *connection).await; +} +''') + after = guard.census(self.root) + self.assertEqual(after[0], before[0]) + self.assertEqual(after[2], before[2] + 1) + + def test_reader_helper_without_metrics_cannot_be_added_silently(self): + self.append("ledger/audit.rs", ''' +async fn negative_reader(reader: &mut AuditReader<'_>) { + sqlx::query("SELECT 1").fetch_one(&mut *reader.connection(None).await?).await; +} +''') + self.assertTrue(any("unclassified" in e and "negative_reader" in e for e in self.errors())) + + def test_pool_type_alias_enrolls_helper_callers(self): + self.append("ledger/window.rs", ''' +use sqlx::PgPool as ImportedDatabase; +type DatabaseAlias = ImportedDatabase; +async fn aliased_helper(database: &DatabaseAlias) { database.acquire().await; } +async fn negative_type_alias(database: &DatabaseAlias) { aliased_helper(database).await; } +''') + errors = self.errors() + self.assertTrue(any("negative_type_alias" in e and "aliased_helper" in e for e in errors)) + + def test_generated_rust_requires_explicit_scanner_support(self): + self.append("lib.rs", '\ninclude!("generated.rs");\n') + with self.assertRaisesRegex(ValueError, "include! needs explicit source-census support"): + guard.census(self.root) + + def test_comments_strings_and_test_only_modules_are_opaque(self): + before = guard.census(self.root)[0] + self.append("ledger/window.rs", ''' +// pool.acquire().await; +/* outer /* pool.begin().await */ nested */ +const NEGATIVE_SQL: &str = r##"pool.acquire(); .fetch_one(&pool)"##; +#[cfg(test)] +mod negative_tests { async fn fixture() { pool.acquire().await; } } +#[cfg(test)] +#[path = "missing_test_file.rs"] +mod only_for_tests; +#[cfg(all(test, target_os = "linux"))] +mod platform_tests { async fn fixture() { pool.acquire().await; } } +''') + self.assertEqual(guard.census(self.root)[0], before) + self.append("ledger/window.rs", ''' +#[cfg(all(unix, any(test, feature = "production")))] +mod negative_conditional { + async fn production_path() { pool.acquire().await; } +} +''') + self.assertTrue(any("unclassified" in e and "production_path" in e for e in self.errors())) + + def test_added_occurrence_and_stale_entries_are_rejected(self): + sites = guard.census(self.root)[0] + site = next(iter(sites)) + added = sites.copy() + added[site] += 1 + self.assertTrue(any("changed count" in e and site in e for e in guard.check(added, self.inventory))) + removed = sites.copy() + del removed[site] + self.assertTrue(any("stale classification" in e and site in e for e in guard.check(removed, self.inventory))) + + def test_removing_or_disabling_timer_changes_classification(self): + target = self.root / guard.SOURCE / "ledger/connect/acquire.rs" + original = target.read_text() + for replacement in ["self.pool.acquire()", "time_pool_acquire(None, self.pool.acquire())"]: + with self.subTest(replacement=replacement): + target.write_text(original.replace("time_pool_acquire(self.metrics.as_deref(), self.pool.acquire())", replacement)) + errors = self.errors() + self.assertTrue(any("unclassified" in e and "connect/acquire.rs" in e for e in errors)) + self.assertTrue(any("stale classification" in e and "connect/acquire.rs" in e for e in errors)) + + def test_allowlist_requires_owner_reason_and_current_policy(self): + site = "fixture::query::fetch_one(database)" + sites = Counter({site: 1}) + inventory = {"sites": {site: [1, "excluded"]}, "policies": {"excluded": {"class": "operator"}}} + self.assertTrue(any("owner and reason" in e for e in guard.check(sites, inventory))) + inventory["policies"]["excluded"].update(owner="tools", reason="operator-only diagnostic") + self.assertEqual(guard.check(sites, inventory), []) + inventory["policies"]["unused"] = inventory["policies"]["excluded"].copy() + self.assertIn("stale policy: unused", guard.check(sites, inventory)) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_check_prism_settings.py b/tests/test_check_prism_settings.py new file mode 100644 index 000000000..751f1bf16 --- /dev/null +++ b/tests/test_check_prism_settings.py @@ -0,0 +1,97 @@ +"""Regression cases for retired operator configuration guidance.""" +from pathlib import Path +import shutil +import subprocess +import tempfile +import unittest + +from scripts.check_prism_settings import check + + +class PrismSettingsGuardTests(unittest.TestCase): + def setUp(self): + self.directory = tempfile.TemporaryDirectory() + self.addCleanup(self.directory.cleanup) + self.root = Path(self.directory.name) + source = self.root / 'crates/qbit-prism-server/src' + (source / 'config').mkdir(parents=True) + (source / 'config/native-settings.txt').write_text('PRISM_ACTIVE\n') + (source / 'config/retired-settings.txt').write_text('PRISM_RETIRED\n') + (source / 'config.rs').write_text('let setting = optional("PRISM_ACTIVE");') + (self.root / 'scripts').mkdir() + (self.root / 'scripts/check-env.sh').write_text('echo ready\n') + (self.root / 'docs').mkdir() + + def test_live_retired_guidance_is_rejected_with_location(self): + (self.root / 'docs/operator.md').write_text('Set `PRISM_RETIRED=1` to tune work.\n') + errors = check(self.root) + self.assertEqual(len(errors), 1) + self.assertIn('docs/operator.md:1:', errors[0]) + self.assertIn('PRISM_RETIRED', errors[0]) + + def test_historical_marker_is_local_and_cannot_exempt_shell(self): + (self.root / 'docs/operator.md').write_text( + 'Removed `PRISM_RETIRED`. <!-- retired-setting: PRISM_RETIRED -->\n') + self.assertEqual(check(self.root), []) + (self.root / 'scripts/check-env.sh').write_text( + 'echo "$PRISM_RETIRED" # <!-- retired-setting: PRISM_RETIRED -->\n') + self.assertTrue(any('scripts/check-env.sh:1:' in error for error in check(self.root))) + + def test_inventory_tracks_added_and_removed_runtime_readers(self): + source = self.root / 'crates/qbit-prism-server/src/config.rs' + source.write_text('let setting = optional("PRISM_NEW");') + errors = check(self.root) + self.assertTrue(any('missing from inventory: PRISM_NEW' in error for error in errors)) + self.assertTrue(any('without a native reader: PRISM_ACTIVE' in error for error in errors)) + + def test_patch_deletions_are_historical_but_additions_are_checked(self): + patch = self.root / 'docs/migration.patch' + patch.write_text('-PRISM_RETIRED=1\n') + self.assertEqual(check(self.root), []) + patch.write_text('+PRISM_RETIRED=1\n') + self.assertTrue(any('PRISM_RETIRED' in error for error in check(self.root))) + + def test_operator_readmes_env_example_and_doc_tree_are_checked(self): + guidance = [ + 'doc/mining/tuning.md', + 'PRISM.md', + 'README.md', + 'crates/qbit-prism-server/README.md', + '.env.example', + ] + for name in guidance: + path = self.root / name + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text('intro\nPRISM_RETIRED=1\n') + errors = check(self.root) + self.assertEqual(sorted(errors), sorted( + f'{name}:2: retired setting presented as live: PRISM_RETIRED' for name in guidance)) + for name in guidance: + (self.root / name).write_text( + 'intro\nPRISM_RETIRED was removed. <!-- retired-setting: PRISM_RETIRED -->\n') + # Only Markdown can carry a historical marker; an environment template is live config. + self.assertEqual(check(self.root), [ + '.env.example:2: retired setting presented as live: PRISM_RETIRED']) + + def test_marker_exempts_only_the_named_setting(self): + (self.root / 'crates/qbit-prism-server/src/config/retired-settings.txt').write_text( + 'PRISM_RETIRED\nPRISM_OTHER_RETIRED\n') + (self.root / 'docs/operator.md').write_text( + 'Set PRISM_RETIRED and PRISM_OTHER_RETIRED. <!-- retired-setting: PRISM_RETIRED -->\n') + self.assertEqual(check(self.root), [ + 'docs/operator.md:1: retired setting presented as live: PRISM_OTHER_RETIRED']) + + @unittest.skipUnless(shutil.which('git'), 'git is required to evaluate ignore rules') + def test_gitignored_generated_guidance_is_not_scanned(self): + subprocess.run(['git', 'init', '-q', str(self.root)], check=True) + (self.root / '.gitignore').write_text('doc/generated/\n') + generated = self.root / 'doc/generated/report.md' + generated.parent.mkdir(parents=True) + generated.write_text('PRISM_RETIRED=1\n') + (self.root / 'doc/operator.md').write_text('PRISM_RETIRED=1\n') + self.assertEqual(check(self.root), [ + 'doc/operator.md:1: retired setting presented as live: PRISM_RETIRED']) + + +if __name__ == '__main__': + unittest.main() diff --git a/tests/test_check_toolchain_pin.py b/tests/test_check_toolchain_pin.py new file mode 100644 index 000000000..2f9f94f71 --- /dev/null +++ b/tests/test_check_toolchain_pin.py @@ -0,0 +1,189 @@ +#!/usr/bin/env python3 + +from __future__ import annotations + +import os +import subprocess +import sys +import tempfile +import unittest +from pathlib import Path + + +ROOT = Path(__file__).resolve().parents[1] +SCRIPT = ROOT / "scripts" / "check_toolchain_pin.py" +CI_WORKFLOW = ROOT / ".github" / "workflows" / "ci.yml" +PIN = "1.98.1" +DIGEST = "9a73a5088750b4c95158ab26629c854c3d6fc4b173cb7bc8079ad252d8ed7bfa" + + +def write_tree( + root: Path, + *, + channel: str = PIN, + profile: str = "minimal", + components: str = '["rustfmt", "clippy"]', + rust_version: str | None = PIN, + crates: dict[str, str] | None = None, + builder: str | None = None, +) -> None: + (root / "rust-toolchain.toml").write_text( + "[toolchain]\n" + f'channel = "{channel}"\n' + f'profile = "{profile}"\n' + f"components = {components}\n", + encoding="utf-8", + ) + crates = {"alpha": "rust-version.workspace = true"} if crates is None else crates + members = "".join(f' "crates/{name}",\n' for name in crates) + version_line = "" if rust_version is None else f'rust-version = "{rust_version}"\n' + (root / "Cargo.toml").write_text( + f"[workspace]\nmembers = [\n{members}]\n\n[workspace.package]\nversion = \"3.0.0\"\n{version_line}", + encoding="utf-8", + ) + for name, line in crates.items(): + crate = root / "crates" / name + crate.mkdir(parents=True) + (crate / "Cargo.toml").write_text( + f'[package]\nname = "{name}"\nversion.workspace = true\n{line}\n', encoding="utf-8" + ) + dockerfile = root / "lab" / "prism" / "Dockerfile" + dockerfile.parent.mkdir(parents=True) + builder = f"FROM rust:{PIN}-bookworm@sha256:{DIGEST} AS build" if builder is None else builder + dockerfile.write_text( + f"# syntax=docker/dockerfile:1.7\n\n{builder}\nWORKDIR /build\n\nFROM debian:bookworm-slim\n", + encoding="utf-8", + ) + + +def run_check(root: Path, *args: str, env: dict[str, str] | None = None) -> subprocess.CompletedProcess[str]: + merged = os.environ.copy() + merged.update(env or {}) + return subprocess.run( + [sys.executable, str(SCRIPT), "--root", str(root), *args], + cwd=ROOT, + env=merged, + text=True, + capture_output=True, + check=False, + ) + + +def fake_tool(directory: Path, name: str, version_line: str) -> None: + path = directory / name + path.write_text(f"#!/bin/sh\nprintf '%s\\n' '{version_line}'\n", encoding="utf-8") + path.chmod(0o755) + + +class CheckToolchainPinTests(unittest.TestCase): + def test_an_agreeing_tree_passes(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + write_tree(root) + result = run_check(root) + self.assertEqual(result.returncode, 0, result.stderr) + self.assertIn(f"Rust {PIN} is pinned consistently", result.stdout) + + def test_print_pin_prints_only_the_version(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + write_tree(root, rust_version="9.9.9") + result = run_check(root, "--print-pin") + self.assertEqual(result.returncode, 0, result.stderr) + self.assertEqual(result.stdout, f"{PIN}\n") + + def test_rust_version_drift_fails(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + write_tree(root, rust_version="1.97.0") + result = run_check(root) + self.assertEqual(result.returncode, 1, result.stdout) + self.assertIn(f"rust-version is '1.97.0' but rust-toolchain.toml pins {PIN}", result.stderr) + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + write_tree(root, rust_version=None) + result = run_check(root) + self.assertEqual(result.returncode, 1, result.stdout) + self.assertIn("rust-version is None", result.stderr) + + def test_a_crate_that_does_not_inherit_rust_version_fails(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + write_tree(root, crates={"alpha": "rust-version.workspace = true", "beta": f'rust-version = "{PIN}"', "gamma": ""}) + result = run_check(root) + self.assertEqual(result.returncode, 1, result.stdout) + self.assertIn("crates/beta/Cargo.toml: rust-version must be inherited", result.stderr) + self.assertIn("crates/gamma/Cargo.toml: rust-version must be inherited", result.stderr) + self.assertNotIn("crates/alpha", result.stderr) + + def test_dockerfile_builder_drift_fails(self) -> None: + cases = { + f"FROM rust:1.97.0-bookworm@sha256:{DIGEST} AS build": f"rust:1.97.0-bookworm but rust-toolchain.toml pins {PIN}", + f"FROM rust:{PIN}-bookworm AS build": "expected exactly one", + f"FROM rust:{PIN}-bookworm@sha256:abc AS build": "digest must be 64 hex characters", + "FROM rust:1-bookworm AS build": "expected exactly one", + } + for builder, message in cases.items(): + with self.subTest(builder=builder), tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + write_tree(root, builder=builder) + result = run_check(root) + self.assertEqual(result.returncode, 1, result.stdout) + self.assertIn(message, result.stderr) + + def test_the_toolchain_file_must_pin_an_exact_stable_with_the_components(self) -> None: + cases = [ + dict(channel="stable"), + dict(channel="1.98"), + dict(profile="default"), + dict(components='["rustfmt"]'), + ] + for case in cases: + with self.subTest(case=case), tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + write_tree(root, **case) + result = run_check(root) + self.assertEqual(result.returncode, 1, result.stdout) + self.assertIn("rust-toolchain.toml", result.stderr) + with tempfile.TemporaryDirectory() as tmp: + result = run_check(Path(tmp)) + self.assertEqual(result.returncode, 1, result.stdout) + self.assertIn("rust-toolchain.toml is missing", result.stderr) + + def test_installed_compares_rustc_and_cargo_against_the_pin(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + write_tree(root) + bin_dir = root / "bin" + bin_dir.mkdir() + fake_tool(bin_dir, "rustc", f"rustc {PIN} (48a229cea 2026-09-01)") + fake_tool(bin_dir, "cargo", f"cargo {PIN} (797e8a9bc 2026-08-05)") + path = f"{bin_dir}{os.pathsep}{os.environ.get('PATH', '')}" + result = run_check(root, "--installed", env={"PATH": path}) + self.assertEqual(result.returncode, 0, result.stderr) + self.assertIn("installed rustc and cargo", result.stdout) + fake_tool(bin_dir, "cargo", "cargo 1.99.0 (deadbeef 2026-10-01)") + result = run_check(root, "--installed", env={"PATH": path}) + self.assertEqual(result.returncode, 1, result.stdout) + self.assertIn(f"cargo reports 'cargo 1.99.0 (deadbeef 2026-10-01)' but rust-toolchain.toml pins {PIN}", result.stderr) + self.assertNotIn("rustc reports", result.stderr) + result = run_check(root, "--installed", env={"PATH": str(root / "nowhere")}) + self.assertEqual(result.returncode, 1, result.stdout) + self.assertIn("rustc is not on PATH", result.stderr) + + def test_the_checked_in_tree_holds_one_pin(self) -> None: + result = run_check(ROOT) + self.assertEqual(result.returncode, 0, result.stderr) + self.assertIn("lab/prism/Dockerfile", result.stdout) + + def test_ci_uses_the_pin_and_asserts_the_installed_toolchain(self) -> None: + workflow = CI_WORKFLOW.read_text(encoding="utf-8") + self.assertNotIn("dtolnay/rust-toolchain@stable", workflow) + self.assertIn("uses: ./.github/actions/rust-toolchain", workflow) + action = (ROOT / ".github" / "actions" / "rust-toolchain" / "action.yml").read_text(encoding="utf-8") + self.assertIn("scripts/check_toolchain_pin.py --installed", action) + self.assertIn("python3 scripts/check_toolchain_pin.py", workflow) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_check_version_skew.py b/tests/test_check_version_skew.py new file mode 100644 index 000000000..269e44b28 --- /dev/null +++ b/tests/test_check_version_skew.py @@ -0,0 +1,177 @@ +#!/usr/bin/env python3 + +from __future__ import annotations + +import os +import subprocess +import sys +import tempfile +import unittest +from pathlib import Path + + +ROOT = Path(__file__).resolve().parents[1] +SCRIPT = ROOT / "scripts" / "check_version_skew.py" +CI_WORKFLOW = ROOT / ".github" / "workflows" / "ci.yml" +MAKEFILE = ROOT / "Makefile" + + +def write_workspace( + root: Path, + *, + workspace_version: str, + version_file: str | None, + crates: dict[str, str | None], +) -> None: + """Lay out a Cargo workspace; a crate inherits the workspace version unless given a literal.""" + members = "".join(f' "crates/{name}",\n' for name in crates) + (root / "Cargo.toml").write_text( + "[workspace]\n" + 'resolver = "2"\n' + f"members = [\n{members}]\n" + "\n" + "[workspace.package]\n" + f'version = "{workspace_version}"\n', + encoding="utf-8", + ) + for name, literal in crates.items(): + crate = root / "crates" / name + (crate / "src").mkdir(parents=True) + (crate / "src" / "lib.rs").write_text("", encoding="utf-8") + version_line = ( + "version.workspace = true" if literal is None else f'version = "{literal}"' + ) + (crate / "Cargo.toml").write_text( + f'[package]\nname = "{name}"\n{version_line}\nedition = "2021"\n', + encoding="utf-8", + ) + if version_file is not None: + (root / "VERSION").write_text(version_file, encoding="utf-8") + + +def run_check(root: Path, **env_overrides: str) -> subprocess.CompletedProcess[str]: + env = os.environ.copy() + env.update(env_overrides) + return subprocess.run( + [sys.executable, str(SCRIPT), "--root", str(root)], + cwd=ROOT, + env=env, + text=True, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + check=False, + ) + + +class CheckVersionSkewTests(unittest.TestCase): + def test_matching_version_file_and_crates_pass(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + write_workspace( + root, + workspace_version="3.0.0", + version_file="3.0.0\n", + crates={"alpha": None, "beta": None}, + ) + result = run_check(root) + self.assertEqual(result.returncode, 0, result.stderr) + self.assertIn( + "VERSION 3.0.0 matches every workspace crate: alpha, beta", result.stdout + ) + self.assertEqual(result.stderr, "") + + def test_version_file_behind_the_workspace_names_every_crate(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + write_workspace( + root, + workspace_version="3.0.0", + version_file="2.0.2\n", + crates={"alpha": None, "beta": None}, + ) + result = run_check(root) + self.assertEqual(result.returncode, 1, result.stdout) + self.assertIn("alpha is 3.0.0 but VERSION is 2.0.2", result.stderr) + self.assertIn("beta is 3.0.0 but VERSION is 2.0.2", result.stderr) + self.assertIn("2 of 2 workspace crates differ from VERSION 2.0.2", result.stderr) + + def test_crate_with_its_own_literal_version_is_named_alone(self) -> None: + # A crate that drops `version.workspace = true` for a literal is the + # skew this check exists for; the manifests are never grepped, so the + # literal is caught through Cargo's own resolution. + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + write_workspace( + root, + workspace_version="3.0.0", + version_file="3.0.0\n", + crates={"alpha": None, "beta": "2.0.0"}, + ) + result = run_check(root) + self.assertEqual(result.returncode, 1, result.stdout) + self.assertIn("beta is 2.0.0 but VERSION is 3.0.0", result.stderr) + self.assertNotIn("alpha", result.stderr) + self.assertIn("1 of 2 workspace crates differ from VERSION 3.0.0", result.stderr) + + def test_empty_missing_or_malformed_version_file_fails(self) -> None: + for content in ("", "\n", "3.0.0\n3.0.0\n", "v3.0.0\n", " 3.0.0\n", "three\n"): + with self.subTest(content=content), tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + write_workspace( + root, + workspace_version="3.0.0", + version_file=content, + crates={"alpha": None}, + ) + result = run_check(root) + self.assertEqual(result.returncode, 1, result.stdout) + self.assertIn("VERSION must hold exactly one SemVer line", result.stderr) + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + write_workspace( + root, workspace_version="3.0.0", version_file=None, crates={"alpha": None} + ) + result = run_check(root) + self.assertEqual(result.returncode, 1, result.stdout) + self.assertIn("VERSION is missing", result.stderr) + + def test_unusable_cargo_is_reported_rather_than_passing(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + write_workspace( + root, + workspace_version="3.0.0", + version_file="3.0.0\n", + crates={"alpha": None}, + ) + result = run_check(root, CARGO=str(root / "no-such-cargo")) + self.assertEqual(result.returncode, 1, result.stdout) + self.assertIn("cannot run", result.stderr) + self.assertIn("no-such-cargo", result.stderr) + + def test_checked_in_tree_has_one_release_identity(self) -> None: + result = run_check(ROOT) + self.assertEqual(result.returncode, 0, result.stderr) + self.assertIn( + "matches every workspace crate: qbit-pool-builder, qbit-prism, " + "qbit-prism-load, qbit-prism-server, qbit-prism-test-gate", + result.stdout, + ) + + def test_ci_runs_the_check_in_a_job_the_merge_gate_needs(self) -> None: + workflow = CI_WORKFLOW.read_text(encoding="utf-8") + rust_tests = workflow.split("name: Rust tests", 1)[1].split("\n checks:", 1)[0] + self.assertIn("python3 scripts/check_version_skew.py", rust_tests) + checks = workflow.split("\n checks:", 1)[1] + needs = checks.split("needs: [", 1)[1].split("]", 1)[0] + self.assertIn("rust-tests", needs.split(", ")) + + def test_make_target_runs_the_check(self) -> None: + makefile = MAKEFILE.read_text(encoding="utf-8") + self.assertIn("check-version-skew:\n\tpython3 scripts/check_version_skew.py\n", makefile) + phony = next(line for line in makefile.splitlines() if line.startswith(".PHONY:")) + self.assertIn("check-version-skew", phony.split()) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_ctv_broadcaster.py b/tests/test_ctv_broadcaster.py deleted file mode 100644 index e62d2cdd2..000000000 --- a/tests/test_ctv_broadcaster.py +++ /dev/null @@ -1,436 +0,0 @@ -#!/usr/bin/env python3 - -from __future__ import annotations - -import unittest -from decimal import Decimal - -from lab.prism.ctv_broadcaster import ( - AWAITING_MATURITY, - BROADCAST, - BROADCASTABLE, - CONFIRMED, - CPFP_CHILD_TX_VERSION, - REORGED, - BroadcasterError, - CtvFanoutBroadcaster, - FanoutArtifact, - RpcError, - WriterLeaseRenewalDeferred, - build_unsigned_cpfp_child, -) - -FANOUT_TXID = "ab" * 32 -CHILD_TXID = "cc" * 32 -COINBASE_BLOCK = "0b" * 32 -MATURITY = 10 - - -def artifact() -> FanoutArtifact: - return FanoutArtifact( - fanout_txid=FANOUT_TXID, - fanout_tx_hex="03000000ff", # opaque to the fake node - anchor_vout=8, - coinbase_txid="2c" * 32, - coinbase_block_hash=COINBASE_BLOCK, - coinbase_height=100, - parent_coinbase_vout=3, - ) - - -def no_anchor_artifact() -> FanoutArtifact: - return FanoutArtifact( - fanout_txid=FANOUT_TXID, - fanout_tx_hex="03000000ff", # opaque to the fake node - anchor_vout=None, - coinbase_txid="2c" * 32, - coinbase_block_hash=COINBASE_BLOCK, - coinbase_height=100, - parent_coinbase_vout=3, - ) - - -class FakeRpc: - def __init__( - self, - *, - tip: int = 200, - mempool=(), - confirmed=(), - reorged_blocks=(), - missing_blocks=(), - transient_missing_blocks=(), - parent_spent_on_chain: bool = False, - parent_spender_txid: str = FANOUT_TXID, - ) -> None: - self.tip = tip - self.mempool = set(mempool) - self.confirmed = set(confirmed) - self.reorged_blocks = set(reorged_blocks) - self.missing_blocks = set(missing_blocks) - # Block hashes that raise on the first getblock and succeed afterward, - # modelling a transient node error mid-scan. - self.transient_missing_blocks = set(transient_missing_blocks) - self.parent_spent_on_chain = parent_spent_on_chain - self.parent_spender_txid = parent_spender_txid - self.submitted_packages: list = [] - self.submitted_raw_transactions: list[str] = [] - - def __call__(self, method, params=None, *, wallet=None): - params = params or [] - if method == "getblockchaininfo": - return {"blocks": self.tip} - if method == "getrawmempool": - return list(self.mempool) - if method == "getrawtransaction": - txid = params[0] - if txid in self.confirmed: - return {"confirmations": 6} - if txid in self.mempool: - return {"confirmations": 0} - raise RpcError("getrawtransaction", "No such mempool transaction") - if method == "getblockhash": - return f"{int(params[0]):064x}" - if method == "getblock": - block_hash = params[0] - if block_hash in self.transient_missing_blocks: - self.transient_missing_blocks.discard(block_hash) - raise RpcError("getblock", "Block temporarily unavailable") - if block_hash in self.missing_blocks: - raise RpcError("getblock", "Block not found") - if block_hash in self.reorged_blocks: - return {"confirmations": -1} - if len(params) > 1 and int(params[1]) == 2: - txs = [] - if self.parent_spent_on_chain and block_hash == f"{self.tip:064x}": - txs.append( - { - "txid": self.parent_spender_txid, - "vin": [ - { - "txid": artifact().coinbase_txid, - "vout": artifact().parent_coinbase_vout, - } - ], - } - ) - return {"confirmations": 1, "tx": txs} - return {"confirmations": 10} - if method == "gettxout": - txid, vout, include_mempool = params - if txid == artifact().coinbase_txid and int(vout) == artifact().parent_coinbase_vout: - if self.parent_spent_on_chain: - return None - if bool(include_mempool) and FANOUT_TXID in self.mempool: - return None - return {"bestblock": COINBASE_BLOCK, "confirmations": 10, "value": Decimal("1.0")} - raise RpcError("gettxout", "No such txout") - if method == "listunspent": - return [ - {"txid": "cd" * 32, "vout": 0, "amount": Decimal("1.0"), "spendable": True} - ] - if method == "getnewaddress": - return "qbrt1zchange" - if method == "getaddressinfo": - return {"scriptPubKey": "5220" + "11" * 32} - if method == "signrawtransactionwithwallet": - return {"hex": params[0], "complete": True} - if method == "submitpackage": - self.submitted_packages.append(params[0]) - return { - "package_msg": "success", - "tx-results": { - "wA": {"txid": FANOUT_TXID}, - "wB": {"txid": CHILD_TXID}, - }, - } - if method == "sendrawtransaction": - self.submitted_raw_transactions.append(str(params[0])) - self.mempool.add(FANOUT_TXID) - return FANOUT_TXID - raise RpcError(method, "unexpected method in FakeRpc") - - -def broadcaster(fake: FakeRpc) -> CtvFanoutBroadcaster: - return CtvFanoutBroadcaster(fake, funding_wallet="broadcaster", maturity=MATURITY) - - -def direct_broadcaster(fake: FakeRpc) -> CtvFanoutBroadcaster: - return CtvFanoutBroadcaster(fake, maturity=MATURITY) - - -class BuildChildTests(unittest.TestCase): - def test_structure_and_change(self) -> None: - unsigned, change = build_unsigned_cpfp_child( - FANOUT_TXID, 8, "cd" * 32, 1, 50_000, 12_000, "5220" + "11" * 32 - ) - self.assertEqual(change, 38_000) - # version 3 (TRUC) + exactly two inputs. - self.assertTrue(unsigned.startswith("03000000")) - self.assertEqual(unsigned[8:10], "02") - # both prevouts present (palindromic bytes => reversal is a no-op here). - self.assertIn("ab" * 32, unsigned) - self.assertIn("cd" * 32, unsigned) - self.assertIn("5220" + "11" * 32, unsigned) - self.assertEqual(CPFP_CHILD_TX_VERSION, 3) - - def test_rejects_fee_at_or_above_funding(self) -> None: - with self.assertRaises(BroadcasterError): - build_unsigned_cpfp_child(FANOUT_TXID, 8, "cd" * 32, 1, 50_000, 50_000, "51") - - def test_rejects_zero_fee(self) -> None: - with self.assertRaises(BroadcasterError): - build_unsigned_cpfp_child(FANOUT_TXID, 8, "cd" * 32, 1, 50_000, 0, "51") - - def test_rejects_funding_equal_to_anchor(self) -> None: - with self.assertRaises(BroadcasterError): - build_unsigned_cpfp_child(FANOUT_TXID, 8, FANOUT_TXID, 8, 50_000, 10, "51") - - -class SettlementStatusTests(unittest.TestCase): - def test_awaiting_maturity(self) -> None: - fake = FakeRpc(tip=100 + MATURITY - 1) - self.assertEqual(broadcaster(fake).settlement_status(artifact()), AWAITING_MATURITY) - - def test_broadcastable_when_mature(self) -> None: - fake = FakeRpc(tip=100 + MATURITY) - self.assertEqual(broadcaster(fake).settlement_status(artifact()), BROADCASTABLE) - - def test_broadcast_when_in_mempool(self) -> None: - fake = FakeRpc(tip=200, mempool=[FANOUT_TXID]) - self.assertEqual(broadcaster(fake).settlement_status(artifact()), BROADCAST) - - def test_confirmed(self) -> None: - fake = FakeRpc(tip=200, confirmed=[FANOUT_TXID]) - self.assertEqual(broadcaster(fake).settlement_status(artifact()), CONFIRMED) - - def test_confirmed_without_txindex_when_parent_outpoint_is_spent_on_chain(self) -> None: - fake = FakeRpc(tip=200, parent_spent_on_chain=True) - self.assertEqual(broadcaster(fake).settlement_status(artifact()), CONFIRMED) - - def test_unexpected_parent_spender_is_not_confirmed(self) -> None: - fake = FakeRpc(tip=200, parent_spent_on_chain=True, parent_spender_txid="ef" * 32) - self.assertEqual(broadcaster(fake).settlement_status(artifact()), REORGED) - - def test_spent_parent_with_unknown_spender_is_not_broadcastable(self) -> None: - fake = FakeRpc( - tip=200, - parent_spent_on_chain=True, - missing_blocks=[f"{artifact().coinbase_height + 1:064x}"], - ) - self.assertEqual(broadcaster(fake).settlement_status(artifact()), REORGED) - - def test_transient_scan_failure_does_not_poison_spender_coverage(self) -> None: - # A transient getblock error mid-scan must not mark the unread tail as - # covered: the spender lives in the tip block, but the scan aborts at - # height 150 first. A later call, once the node recovers, must still - # find the spender instead of leaving it hidden until restart. - transient_hash = f"{150:064x}" - fake = FakeRpc( - tip=200, - parent_spent_on_chain=True, - transient_missing_blocks=[transient_hash], - ) - engine = broadcaster(fake) - self.assertEqual(engine.settlement_status(artifact()), REORGED) - self.assertEqual(engine.settlement_status(artifact()), CONFIRMED) - - def test_reorged_when_coinbase_block_disconnected(self) -> None: - fake = FakeRpc(tip=200, reorged_blocks=[COINBASE_BLOCK]) - self.assertEqual(broadcaster(fake).settlement_status(artifact()), REORGED) - - def test_disconnected_coinbase_before_maturity_keeps_waiting(self) -> None: - fake = FakeRpc(tip=100 + MATURITY - 1, reorged_blocks=[COINBASE_BLOCK]) - self.assertEqual(broadcaster(fake).settlement_status(artifact()), AWAITING_MATURITY) - - def test_confirmed_beats_reorg_flag(self) -> None: - # A confirmed fanout is final even if the coinbase block shows disconnected. - fake = FakeRpc(tip=200, confirmed=[FANOUT_TXID], reorged_blocks=[COINBASE_BLOCK]) - self.assertEqual(broadcaster(fake).settlement_status(artifact()), CONFIRMED) - - -class BroadcastTests(unittest.TestCase): - def test_mutating_wallet_and_node_rpcs_are_lease_fenced(self) -> None: - fake = FakeRpc(tip=100 + MATURITY) - fenced_operations: list[str] = [] - engine = CtvFanoutBroadcaster( - fake, - funding_wallet="broadcaster", - maturity=MATURITY, - before_external_side_effect=fenced_operations.append, - ) - - attempt = engine.broadcast(artifact(), fee_sats=50_000) - - self.assertTrue(attempt.submitted) - self.assertEqual( - fenced_operations, - [ - "ctv_wallet_getnewaddress", - "ctv_wallet_sign", - "ctv_submitpackage", - ], - ) - - def test_stale_lease_gate_prevents_ctv_external_side_effect(self) -> None: - fake = FakeRpc(tip=100 + MATURITY) - - def refuse(_operation: str) -> None: - raise RuntimeError("writer guard stale") - - engine = CtvFanoutBroadcaster( - fake, - funding_wallet="broadcaster", - maturity=MATURITY, - before_external_side_effect=refuse, - ) - - attempt = engine.broadcast(artifact(), fee_sats=50_000) - - self.assertFalse(attempt.submitted) - self.assertIn("writer guard stale", attempt.detail) - self.assertEqual(fake.submitted_packages, []) - self.assertEqual(fake.submitted_raw_transactions, []) - - def test_lease_renewal_deferral_propagates_instead_of_becoming_an_attempt( - self, - ) -> None: - """The fence's deferral is a preflight refusal, not a broadcast outcome. - - Converting WriterLeaseRenewalDeferred into an error attempt would - journal a failure and push the payout behind the persistent retry - backoff, while the deferral is transient by design: the daemon's - caller retries on its ordinary pass interval once the deferring - write commits. The exception must pass through the generic - RPC-error conversion, with no RPC sent. - """ - fake = FakeRpc(tip=100 + MATURITY) - - def defer(_operation: str) -> None: - raise WriterLeaseRenewalDeferred("renewal is deferred") - - engine = CtvFanoutBroadcaster( - fake, - funding_wallet="broadcaster", - maturity=MATURITY, - before_external_side_effect=defer, - ) - - with self.assertRaisesRegex( - WriterLeaseRenewalDeferred, - "renewal is deferred", - ): - engine.broadcast(artifact(), fee_sats=50_000) - self.assertEqual(fake.submitted_packages, []) - self.assertEqual(fake.submitted_raw_transactions, []) - - with self.assertRaisesRegex( - WriterLeaseRenewalDeferred, - "renewal is deferred", - ): - engine.broadcast(no_anchor_artifact(), fee_sats=0) - self.assertEqual(fake.submitted_raw_transactions, []) - - def test_broadcasts_when_broadcastable(self) -> None: - fake = FakeRpc(tip=100 + MATURITY) - attempt = broadcaster(fake).broadcast(artifact(), fee_sats=50_000) - self.assertTrue(attempt.submitted) - self.assertEqual(attempt.status, BROADCAST) - self.assertEqual(attempt.child_txid, CHILD_TXID) - self.assertEqual(attempt.fee_sats, 50_000) - self.assertEqual(len(fake.submitted_packages), 1) - self.assertEqual(fake.submitted_raw_transactions, []) - - def test_no_anchor_direct_broadcasts_parent_when_no_cpfp_fee_requested(self) -> None: - fake = FakeRpc(tip=100 + MATURITY) - attempt = direct_broadcaster(fake).broadcast(no_anchor_artifact(), fee_sats=0) - self.assertTrue(attempt.submitted) - self.assertEqual(attempt.status, BROADCAST) - self.assertIsNone(attempt.child_txid) - self.assertEqual(attempt.fee_sats, 0) - self.assertEqual(fake.submitted_raw_transactions, [no_anchor_artifact().fanout_tx_hex]) - self.assertEqual(fake.submitted_packages, []) - - def test_no_anchor_artifact_broadcasts_parent_directly_even_with_cpfp_fee_configured(self) -> None: - fake = FakeRpc(tip=100 + MATURITY) - attempt = broadcaster(fake).broadcast(no_anchor_artifact(), fee_sats=50_000) - self.assertTrue(attempt.submitted) - self.assertEqual(attempt.status, BROADCAST) - self.assertIsNone(attempt.child_txid) - self.assertEqual(fake.submitted_raw_transactions, [no_anchor_artifact().fanout_tx_hex]) - self.assertEqual(fake.submitted_packages, []) - - def test_anchored_fanout_requires_positive_cpfp_fee(self) -> None: - fake = FakeRpc(tip=100 + MATURITY) - attempt = direct_broadcaster(fake).broadcast(artifact(), fee_sats=0) - self.assertFalse(attempt.submitted) - self.assertEqual(attempt.status, BROADCASTABLE) - self.assertEqual(attempt.fee_sats, 0) - self.assertEqual(attempt.package_msg, "invalid_fee") - self.assertIn("CPFP fee", attempt.detail) - self.assertEqual(fake.submitted_raw_transactions, []) - self.assertEqual(fake.submitted_packages, []) - - def test_positive_cpfp_fee_without_wallet_is_rejected_not_raised(self) -> None: - fake = FakeRpc(tip=100 + MATURITY) - attempt = direct_broadcaster(fake).broadcast(artifact(), fee_sats=50_000) - self.assertFalse(attempt.submitted) - self.assertEqual(attempt.status, BROADCASTABLE) - self.assertEqual(attempt.fee_sats, 50_000) - self.assertIn("funding wallet", attempt.detail) - self.assertEqual(fake.submitted_packages, []) - - def test_idempotent_when_already_in_mempool(self) -> None: - fake = FakeRpc(tip=200, mempool=[FANOUT_TXID]) - attempt = broadcaster(fake).broadcast(artifact(), fee_sats=50_000) - self.assertFalse(attempt.submitted) - self.assertEqual(attempt.status, BROADCAST) - self.assertEqual(len(fake.submitted_packages), 0) - - def test_no_broadcast_before_maturity(self) -> None: - fake = FakeRpc(tip=100 + MATURITY - 1) - attempt = direct_broadcaster(fake).broadcast(artifact(), fee_sats=0) - self.assertFalse(attempt.submitted) - self.assertEqual(attempt.status, AWAITING_MATURITY) - self.assertEqual(len(fake.submitted_packages), 0) - self.assertEqual(fake.submitted_raw_transactions, []) - - def test_no_broadcast_when_confirmed(self) -> None: - fake = FakeRpc(tip=200, confirmed=[FANOUT_TXID]) - attempt = broadcaster(fake).broadcast(artifact(), fee_sats=50_000) - self.assertFalse(attempt.submitted) - self.assertEqual(attempt.status, CONFIRMED) - - def test_bump_resubmits_at_higher_fee(self) -> None: - fake = FakeRpc(tip=200, mempool=[FANOUT_TXID]) - attempt = broadcaster(fake).bump(artifact(), higher_fee_sats=90_000) - self.assertTrue(attempt.submitted) - self.assertEqual(attempt.fee_sats, 90_000) - self.assertEqual(len(fake.submitted_packages), 1) - - def test_bump_refuses_when_confirmed(self) -> None: - fake = FakeRpc(tip=200, confirmed=[FANOUT_TXID]) - attempt = broadcaster(fake).bump(artifact(), higher_fee_sats=90_000) - self.assertFalse(attempt.submitted) - self.assertEqual(attempt.status, CONFIRMED) - self.assertEqual(len(fake.submitted_packages), 0) - - def test_bump_refuses_when_reorged(self) -> None: - fake = FakeRpc(tip=200, reorged_blocks=[COINBASE_BLOCK]) - attempt = broadcaster(fake).bump(artifact(), higher_fee_sats=90_000) - self.assertFalse(attempt.submitted) - self.assertEqual(attempt.status, REORGED) - - def test_bump_rejects_negative_fee(self) -> None: - fake = FakeRpc(tip=200, mempool=[FANOUT_TXID]) - attempt = broadcaster(fake).bump(artifact(), higher_fee_sats=-1) - self.assertFalse(attempt.submitted) - self.assertEqual(attempt.status, BROADCAST) - self.assertEqual(attempt.fee_sats, -1) - self.assertEqual(attempt.package_msg, "invalid_fee") - self.assertIn("higher_fee_sats must be non-negative", attempt.detail) - self.assertEqual(len(fake.submitted_packages), 0) - - -if __name__ == "__main__": - unittest.main() diff --git a/tests/test_ctv_broadcaster_daemon.py b/tests/test_ctv_broadcaster_daemon.py deleted file mode 100644 index b0ea8c60b..000000000 --- a/tests/test_ctv_broadcaster_daemon.py +++ /dev/null @@ -1,1101 +0,0 @@ -#!/usr/bin/env python3 - -from __future__ import annotations - -import http.client -import json -from datetime import datetime, timedelta, timezone -import threading -import time -import unittest -from unittest.mock import patch - -from lab.prism.ctv_broadcaster import ( - AWAITING_MATURITY, - BROADCAST, - BROADCASTABLE, - CONFIRMED, - REORGED, - BroadcastAttempt, -) -from lab.prism.ctv_broadcaster_daemon import ( - CtvFanoutBroadcastDaemon, - CtvFanoutChunkResult, - CtvFanoutDaemonResult, - MAX_CTV_FANOUT_BROADCASTER_CHUNK_SIZE, - artifact_from_status_row, -) -from lab.prism.backfill_ctv_fanouts import build_parser, ledger_from_args -from lab.prism.run_ctv_broadcaster_daemon import ( - env_positive_int, - make_daemon_from_env, - make_ledger_from_env, -) -from lab.prism.prism_coordinator import JsonRpc -from lab.prism.share_ledger import PsqlShareLedger, SingleWriterShareLedger - - -def pending_row(fanout_txid: str = "aa" * 32) -> dict[str, object]: - return { - "fanout_txid": fanout_txid, - "fanout_tx_hex": "0300000001", - "anchor_vout": 2, - "parent_coinbase_txid": "bb" * 32, - "parent_coinbase_vout": 1, - "block_hash": "cc" * 32, - "block_height": 100, - "settlement_status": "broadcastable", - } - - -def awaiting_maturity_row(fanout_txid: str = "aa" * 32) -> dict[str, object]: - row = pending_row(fanout_txid) - row["settlement_status"] = "awaiting_maturity" - return row - - -class FakeLedger: - def __init__(self, rows: list[dict[str, object]]) -> None: - self.rows = rows - self.updates: list[dict[str, object]] = [] - self.attempts: list[dict[str, object]] = [] - self.lease_renewals = 0 - - def pending_ctv_fanout_statuses(self, *, limit: int = 100) -> list[dict[str, object]]: - return self.rows[:limit] - - def renew_writer_lease(self) -> dict[str, int | str]: - self.lease_renewals += 1 - return {"backend": "fake", "renewed_count": 1} - - def update_ctv_fanout_status(self, *, fanout_txid: str, settlement_status: str) -> dict[str, int | str]: - self.updates.append({"fanout_txid": fanout_txid, "settlement_status": settlement_status}) - return {"backend": "fake", "updated_count": 1} - - def record_ctv_fanout_broadcast_attempt( - self, - *, - fanout_txid: str, - attempt_status: str, - package_tx_hexes: list[str] | None = None, - package_txids: list[str] | None = None, - submit_result: dict[str, object] | None = None, - error: str | None = None, - ) -> dict[str, int | str]: - self.attempts.append( - { - "fanout_txid": fanout_txid, - "attempt_status": attempt_status, - "package_tx_hexes": package_tx_hexes or [], - "package_txids": package_txids or [], - "submit_result": submit_result or {}, - "error": error, - } - ) - return {"backend": "fake", "attempt_count": len(self.attempts)} - - -class FakeBroadcaster: - def __init__(self, attempts: dict[str, BroadcastAttempt], *, tip: int | None = None) -> None: - self.attempts = attempts - self.fees: list[int] = [] - self.maturity = 1000 - self.tip = tip - self.tip_probes = 0 - - def broadcast(self, artifact: object, fee_sats: int) -> BroadcastAttempt: - self.fees.append(fee_sats) - fanout_txid = getattr(artifact, "fanout_txid") - return self.attempts[fanout_txid] - - def tip_height(self) -> int: - self.tip_probes += 1 - if self.tip is None: - raise AssertionError("unexpected chain-tip probe") - return self.tip - - -class CtvFanoutBroadcastDaemonTests(unittest.TestCase): - def test_artifact_from_status_row_preserves_parent_coinbase_vout(self) -> None: - artifact = artifact_from_status_row(pending_row()) - - self.assertEqual(artifact.parent_coinbase_vout, 1) - self.assertEqual(artifact.anchor_vout, 2) - - def test_artifact_from_status_row_allows_no_anchor_fanout(self) -> None: - row = pending_row() - row["anchor_vout"] = None - artifact = artifact_from_status_row(row) - - self.assertIsNone(artifact.anchor_vout) - - def test_submitted_package_is_journaled(self) -> None: - fanout_txid = "aa" * 32 - ledger = FakeLedger([pending_row(fanout_txid)]) - broadcaster = FakeBroadcaster( - { - fanout_txid: BroadcastAttempt( - fanout_txid=fanout_txid, - status=BROADCAST, - submitted=True, - child_txid="dd" * 32, - fee_sats=750, - package_msg="success", - detail="initial broadcast", - ) - } - ) - - result = CtvFanoutBroadcastDaemon(ledger, broadcaster, fee_sats=750).run_once() - - self.assertEqual(result.submitted_count, 1) - self.assertEqual(result.updated_count, 0) - self.assertEqual(ledger.attempts[0]["attempt_status"], "submitted") - self.assertEqual(ledger.attempts[0]["package_txids"], [fanout_txid, "dd" * 32]) - self.assertEqual(broadcaster.fees, [750]) - - def test_terminal_and_maturity_statuses_update_without_journal(self) -> None: - confirmed_txid = "11" * 32 - reorged_txid = "22" * 32 - broadcast_txid = "33" * 32 - ledger = FakeLedger([pending_row(confirmed_txid), pending_row(reorged_txid), pending_row(broadcast_txid)]) - broadcaster = FakeBroadcaster( - { - confirmed_txid: BroadcastAttempt(confirmed_txid, CONFIRMED, submitted=False), - reorged_txid: BroadcastAttempt(reorged_txid, REORGED, submitted=False), - broadcast_txid: BroadcastAttempt(broadcast_txid, BROADCAST, submitted=False), - } - ) - - result = CtvFanoutBroadcastDaemon(ledger, broadcaster, fee_sats=500).run_once() - - self.assertEqual(result.submitted_count, 0) - self.assertEqual(result.updated_count, 3) - self.assertEqual(result.failed_count, 0) - self.assertEqual( - ledger.updates, - [ - {"fanout_txid": confirmed_txid, "settlement_status": "confirmed"}, - {"fanout_txid": reorged_txid, "settlement_status": "reorged"}, - {"fanout_txid": broadcast_txid, "settlement_status": "broadcast_submitted"}, - ], - ) - self.assertEqual(ledger.attempts, []) - - def test_rejected_package_is_journaled(self) -> None: - fanout_txid = "44" * 32 - ledger = FakeLedger([pending_row(fanout_txid)]) - broadcaster = FakeBroadcaster( - { - fanout_txid: BroadcastAttempt( - fanout_txid=fanout_txid, - status=BROADCASTABLE, - submitted=False, - fee_sats=900, - package_msg="insufficient fee", - detail="initial broadcast: package rejected (insufficient fee)", - ) - } - ) - - result = CtvFanoutBroadcastDaemon(ledger, broadcaster, fee_sats=900).run_once() - - self.assertEqual(result.failed_count, 1) - self.assertEqual(ledger.updates, []) - self.assertEqual(ledger.attempts[0]["attempt_status"], "rejected") - self.assertIn("insufficient fee", str(ledger.attempts[0]["error"])) - - def test_failed_rows_are_not_rebroadcast(self) -> None: - fanout_txid = "44" * 32 - row = pending_row(fanout_txid) - row["settlement_status"] = "failed" - ledger = FakeLedger([row]) - broadcaster = FakeBroadcaster({}) - - result = CtvFanoutBroadcastDaemon(ledger, broadcaster, fee_sats=900).run_once() - - self.assertEqual(result.scanned_count, 1) - self.assertEqual(result.submitted_count, 0) - self.assertEqual(result.updated_count, 0) - self.assertEqual(result.failed_count, 0) - self.assertEqual(ledger.attempts, []) - self.assertEqual(broadcaster.fees, []) - - def test_future_backoff_rows_are_not_rebroadcast(self) -> None: - fanout_txid = "45" * 32 - row = pending_row(fanout_txid) - row["next_broadcast_attempt_at"] = datetime.now(timezone.utc) + timedelta(minutes=5) - ledger = FakeLedger([row]) - broadcaster = FakeBroadcaster({}) - - result = CtvFanoutBroadcastDaemon(ledger, broadcaster, fee_sats=900).run_once() - - self.assertEqual(result.scanned_count, 1) - self.assertEqual(result.submitted_count, 0) - self.assertEqual(result.updated_count, 0) - self.assertEqual(result.failed_count, 0) - self.assertEqual(ledger.attempts, []) - self.assertEqual(broadcaster.fees, []) - - def test_immature_rows_skip_probes_and_writes_and_renew_lease(self) -> None: - txids = [f"{value:02x}" * 32 for value in (1, 2, 3)] - ledger = FakeLedger([awaiting_maturity_row(txid) for txid in txids]) - broadcaster = FakeBroadcaster({}, tip=1099) # rows mature at 100 + 1000 - - result = CtvFanoutBroadcastDaemon(ledger, broadcaster, fee_sats=0).run_once() - - self.assertEqual( - result, - CtvFanoutDaemonResult( - scanned_count=3, - submitted_count=0, - updated_count=0, - failed_count=0, - ), - ) - self.assertEqual(broadcaster.fees, []) # no per-row settlement probes - self.assertEqual(broadcaster.tip_probes, 1) # one tip probe for the pass - self.assertEqual(ledger.updates, []) - self.assertEqual(ledger.attempts, []) - self.assertEqual(ledger.lease_renewals, 1) - - def test_height_mature_row_is_probed_and_real_transition_persists(self) -> None: - fanout_txid = "aa" * 32 - ledger = FakeLedger([awaiting_maturity_row(fanout_txid)]) - broadcaster = FakeBroadcaster( - {fanout_txid: BroadcastAttempt(fanout_txid, BROADCAST, submitted=False)}, - tip=1100, - ) - - result = CtvFanoutBroadcastDaemon(ledger, broadcaster, fee_sats=0).run_once() - - self.assertEqual(result.updated_count, 1) - self.assertEqual(broadcaster.fees, [0]) - self.assertEqual( - ledger.updates, - [{"fanout_txid": fanout_txid, "settlement_status": "broadcast_submitted"}], - ) - self.assertEqual(ledger.lease_renewals, 0) - - def test_unchanged_derived_status_skips_ledger_write(self) -> None: - in_mempool_txid = "aa" * 32 - still_immature_txid = "bb" * 32 - in_mempool = pending_row(in_mempool_txid) - in_mempool["settlement_status"] = "broadcast_submitted" - ledger = FakeLedger([in_mempool, awaiting_maturity_row(still_immature_txid)]) - broadcaster = FakeBroadcaster( - { - in_mempool_txid: BroadcastAttempt(in_mempool_txid, BROADCAST, submitted=False), - still_immature_txid: BroadcastAttempt( - still_immature_txid, AWAITING_MATURITY, submitted=False - ), - }, - # Height-mature by the pass-level tip, so both rows take the full - # probe path; the probe then re-derives each row's stored status. - tip=1100, - ) - - result = CtvFanoutBroadcastDaemon(ledger, broadcaster, fee_sats=0).run_once() - - self.assertEqual(result.scanned_count, 2) - self.assertEqual(result.updated_count, 0) - self.assertEqual(broadcaster.fees, [0, 0]) # probes still happen - self.assertEqual(ledger.updates, []) - self.assertEqual(ledger.attempts, []) - # Each unchanged probe renews the lease in place of the write it no - # longer makes, so the pass-end renewal is not needed. - self.assertEqual(ledger.lease_renewals, 2) - - def test_long_unchanged_probe_batches_renew_lease_per_probed_row(self) -> None: - # A full batch of in-mempool rows whose derived status matches the - # stored one makes no fenced write; each probe must refresh the lease - # in the write's place so slow RPCs cannot outlast the TTL mid-chunk. - txids = [f"{value:02x}" * 32 for value in range(1, 5)] - rows = [] - for txid in txids: - row = pending_row(txid) - row["settlement_status"] = "broadcast_submitted" - rows.append(row) - ledger = FakeLedger(rows) - broadcaster = FakeBroadcaster( - {txid: BroadcastAttempt(txid, BROADCAST, submitted=False) for txid in txids}, - ) - - result = CtvFanoutBroadcastDaemon(ledger, broadcaster, fee_sats=0).run_once(chunk_size=2) - - self.assertEqual(result.scanned_count, 4) - self.assertEqual(result.updated_count, 0) - self.assertEqual(len(broadcaster.fees), 4) - self.assertEqual(ledger.updates, []) - self.assertEqual(ledger.lease_renewals, 4) # one per unchanged probe - - def test_transition_write_needs_no_extra_renewal(self) -> None: - unchanged_txid = "aa" * 32 - transition_txid = "bb" * 32 - unchanged = pending_row(unchanged_txid) - unchanged["settlement_status"] = "broadcast_submitted" - ledger = FakeLedger([unchanged, pending_row(transition_txid)]) - broadcaster = FakeBroadcaster( - { - unchanged_txid: BroadcastAttempt(unchanged_txid, BROADCAST, submitted=False), - transition_txid: BroadcastAttempt(transition_txid, CONFIRMED, submitted=False), - }, - ) - - result = CtvFanoutBroadcastDaemon(ledger, broadcaster, fee_sats=0).run_once(chunk_size=2) - - self.assertEqual(result.updated_count, 1) - self.assertEqual( - ledger.updates, - [{"fanout_txid": transition_txid, "settlement_status": "confirmed"}], - ) - # Only the unchanged probe renews; the transition row's fenced write - # refreshes the lease itself and the pass end adds nothing. - self.assertEqual(ledger.lease_renewals, 1) - - def test_empty_scan_renews_lease_without_probes(self) -> None: - ledger = FakeLedger([]) - broadcaster = FakeBroadcaster({}) - - result = CtvFanoutBroadcastDaemon(ledger, broadcaster, fee_sats=0).run_once() - - self.assertEqual(result.scanned_count, 0) - self.assertEqual(broadcaster.tip_probes, 0) - self.assertEqual(ledger.lease_renewals, 1) - - def test_yielded_pass_without_writes_still_renews_lease(self) -> None: - ledger = FakeLedger([pending_row()]) - daemon = CtvFanoutBroadcastDaemon(ledger, FakeBroadcaster({}), fee_sats=0) - - result = daemon.run_once(tip_refresh_pending=lambda: True) - - self.assertTrue(result.yielded_to_tip_refresh) - self.assertEqual(result.scanned_count, 0) - self.assertEqual(ledger.lease_renewals, 1) - - def test_mid_pass_yield_after_immature_skips_still_renews_lease(self) -> None: - # Repeated tip-refresh yields across write-free passes must not let - # the writer lease lapse now that immature rows skip their no-op - # (lease-refreshing) status rewrites. - txids = ["aa" * 32, "bb" * 32] - ledger = FakeLedger([awaiting_maturity_row(txid) for txid in txids]) - broadcaster = FakeBroadcaster({}, tip=1099) - refresh_pending = False - - def flag_refresh() -> None: - nonlocal refresh_pending - refresh_pending = True - - result = CtvFanoutBroadcastDaemon(ledger, broadcaster, fee_sats=0).run_once( - chunk_size=1, - progress_callback=flag_refresh, - tip_refresh_pending=lambda: refresh_pending, - ) - - self.assertTrue(result.yielded_to_tip_refresh) - self.assertEqual(result.scanned_count, 1) - self.assertEqual(ledger.updates, []) - self.assertEqual(ledger.attempts, []) - self.assertEqual(ledger.lease_renewals, 1) - - def test_ledger_without_lease_renewal_support_is_tolerated(self) -> None: - result = CtvFanoutBroadcastDaemon( - SingleWriterShareLedger(), FakeBroadcaster({}), fee_sats=0 - ).run_once() - - self.assertEqual(result.scanned_count, 0) - - def test_progress_callback_covers_skipped_due_updated_submitted_and_failed_rows(self) -> None: - skipped_txid = "10" * 32 - not_due_txid = "20" * 32 - updated_txid = "30" * 32 - submitted_txid = "40" * 32 - failed_txid = "50" * 32 - skipped = pending_row(skipped_txid) - skipped["settlement_status"] = "failed" - not_due = pending_row(not_due_txid) - not_due["next_broadcast_attempt_at"] = datetime.now(timezone.utc) + timedelta(minutes=5) - ledger = FakeLedger( - [ - skipped, - not_due, - pending_row(updated_txid), - pending_row(submitted_txid), - pending_row(failed_txid), - ] - ) - broadcaster = FakeBroadcaster( - { - updated_txid: BroadcastAttempt(updated_txid, CONFIRMED, submitted=False), - submitted_txid: BroadcastAttempt( - submitted_txid, - BROADCAST, - submitted=True, - fee_sats=900, - package_msg="success", - ), - failed_txid: BroadcastAttempt( - failed_txid, - BROADCASTABLE, - submitted=False, - fee_sats=900, - package_msg="rejected", - ), - } - ) - progress_snapshots: list[tuple[int, int, int]] = [] - - result = CtvFanoutBroadcastDaemon(ledger, broadcaster, fee_sats=900).run_once( - progress_callback=lambda: progress_snapshots.append( - (len(broadcaster.fees), len(ledger.updates), len(ledger.attempts)) - ) - ) - - self.assertEqual( - result, - CtvFanoutDaemonResult( - scanned_count=5, - submitted_count=1, - updated_count=1, - failed_count=1, - ), - ) - self.assertEqual( - progress_snapshots, - [ - (0, 0, 0), - (0, 0, 0), - (1, 1, 0), - (2, 1, 1), - (3, 1, 2), - ], - ) - - def test_yields_between_committed_chunks_and_processes_remaining_rows_next_pass(self) -> None: - txids = [f"{value:02x}" * 32 for value in range(1, 5)] - - class LockTrackingLedger(FakeLedger): - def __init__(self, rows: list[dict[str, object]]) -> None: - super().__init__(rows) - self.write_active = False - - def update_ctv_fanout_status( - self, - *, - fanout_txid: str, - settlement_status: str, - ) -> dict[str, int | str]: - self.write_active = True - try: - return super().update_ctv_fanout_status( - fanout_txid=fanout_txid, - settlement_status=settlement_status, - ) - finally: - self.write_active = False - - def record_ctv_fanout_broadcast_attempt( - self, - *, - fanout_txid: str, - attempt_status: str, - package_tx_hexes: list[str] | None = None, - package_txids: list[str] | None = None, - submit_result: dict[str, object] | None = None, - error: str | None = None, - ) -> dict[str, int | str]: - self.write_active = True - try: - return super().record_ctv_fanout_broadcast_attempt( - fanout_txid=fanout_txid, - attempt_status=attempt_status, - package_tx_hexes=package_tx_hexes, - package_txids=package_txids, - submit_result=submit_result, - error=error, - ) - finally: - self.write_active = False - - class IdempotentBroadcaster: - def __init__(self) -> None: - self.submitted_txids: set[str] = set() - self.calls: list[str] = [] - - def broadcast(self, artifact: object, fee_sats: int) -> BroadcastAttempt: - fanout_txid = str(getattr(artifact, "fanout_txid")) - self.calls.append(fanout_txid) - if fanout_txid in self.submitted_txids: - return BroadcastAttempt(fanout_txid, BROADCAST, submitted=False) - self.submitted_txids.add(fanout_txid) - return BroadcastAttempt( - fanout_txid, - BROADCAST, - submitted=True, - fee_sats=fee_sats, - package_msg="success", - ) - - ledger = LockTrackingLedger([pending_row(txid) for txid in txids]) - broadcaster = IdempotentBroadcaster() - daemon = CtvFanoutBroadcastDaemon(ledger, broadcaster, fee_sats=0) - refresh_pending = threading.Event() - progress_count = 0 - lock_states_at_refresh_checks: list[bool] = [] - chunks: list[CtvFanoutChunkResult] = [] - - def record_progress_and_request_refresh() -> None: - nonlocal progress_count - progress_count += 1 - if progress_count == 2: - refresh_pending.set() - - def tip_refresh_pending() -> bool: - lock_states_at_refresh_checks.append(ledger.write_active) - return refresh_pending.is_set() - - first_result = daemon.run_once( - limit=4, - chunk_size=2, - progress_callback=record_progress_and_request_refresh, - tip_refresh_pending=tip_refresh_pending, - chunk_callback=chunks.append, - ) - - self.assertEqual( - first_result, - CtvFanoutDaemonResult( - scanned_count=2, - submitted_count=2, - updated_count=0, - failed_count=0, - yielded_to_tip_refresh=True, - ), - ) - self.assertEqual([chunk.processed_count for chunk in chunks], [2]) - self.assertEqual([attempt["fanout_txid"] for attempt in ledger.attempts], txids[:2]) - self.assertEqual(lock_states_at_refresh_checks, [False, False]) - - refresh_pending.clear() - second_result = daemon.run_once(limit=4, chunk_size=2) - - self.assertEqual(second_result.scanned_count, 4) - self.assertEqual(second_result.submitted_count, 2) - self.assertEqual(second_result.updated_count, 2) - self.assertFalse(second_result.yielded_to_tip_refresh) - self.assertEqual(broadcaster.calls, txids[:2] + txids) - self.assertEqual(broadcaster.submitted_txids, set(txids)) - self.assertEqual([attempt["fanout_txid"] for attempt in ledger.attempts], txids) - - def test_pending_refresh_can_prepare_while_current_chunk_finishes(self) -> None: - first_txid = "61" * 32 - second_txid = "62" * 32 - ledger = FakeLedger([pending_row(first_txid), pending_row(second_txid)]) - current_row_started = threading.Event() - release_current_row = threading.Event() - refresh_pending = threading.Event() - refresh_prepared = threading.Event() - - class BlockingBroadcaster: - def __init__(self) -> None: - self.calls: list[str] = [] - - def broadcast(self, artifact: object, fee_sats: int) -> BroadcastAttempt: - fanout_txid = str(getattr(artifact, "fanout_txid")) - self.calls.append(fanout_txid) - current_row_started.set() - if not release_current_row.wait(timeout=5): - raise AssertionError("test did not release broadcaster row") - return BroadcastAttempt( - fanout_txid, - BROADCAST, - submitted=True, - fee_sats=fee_sats, - package_msg="success", - ) - - broadcaster = BlockingBroadcaster() - daemon = CtvFanoutBroadcastDaemon(ledger, broadcaster, fee_sats=0) - results: list[CtvFanoutDaemonResult] = [] - failures: list[BaseException] = [] - - def run_broadcaster_pass() -> None: - try: - results.append( - daemon.run_once( - limit=2, - chunk_size=1, - tip_refresh_pending=refresh_pending.is_set, - ) - ) - except BaseException as exc: # pragma: no cover - surfaced below - failures.append(exc) - - broadcaster_thread = threading.Thread(target=run_broadcaster_pass) - broadcaster_thread.start() - self.assertTrue(current_row_started.wait(timeout=5)) - - def prepare_refresh() -> None: - refresh_pending.set() - refresh_prepared.set() - - refresh_thread = threading.Thread(target=prepare_refresh) - refresh_thread.start() - refresh_thread.join(timeout=5) - self.assertFalse(refresh_thread.is_alive()) - self.assertTrue(refresh_prepared.is_set()) - - release_current_row.set() - broadcaster_thread.join(timeout=5) - self.assertFalse(broadcaster_thread.is_alive()) - self.assertEqual(failures, []) - self.assertEqual(broadcaster.calls, [first_txid]) - self.assertEqual(len(results), 1) - self.assertTrue(results[0].yielded_to_tip_refresh) - self.assertEqual(results[0].scanned_count, 1) - - def test_chunk_observations_report_bounded_rows_and_deterministic_durations(self) -> None: - rows = [pending_row(f"{value:02x}" * 32) for value in range(3)] - for row in rows: - row["settlement_status"] = "failed" - chunks: list[CtvFanoutChunkResult] = [] - - with patch( - "lab.prism.ctv_broadcaster_daemon.monotonic", - side_effect=[10.0, 10.25, 20.0, 20.5], - ): - result = CtvFanoutBroadcastDaemon( - FakeLedger(rows), - FakeBroadcaster({}), - fee_sats=0, - ).run_once(chunk_size=2, chunk_callback=chunks.append) - - self.assertEqual(result.scanned_count, 3) - self.assertEqual( - chunks, - [ - CtvFanoutChunkResult(processed_count=2, elapsed_seconds=0.25), - CtvFanoutChunkResult(processed_count=1, elapsed_seconds=0.5), - ], - ) - - def test_rejects_nonpositive_chunk_size_before_querying_ledger(self) -> None: - ledger = FakeLedger([]) - daemon = CtvFanoutBroadcastDaemon(ledger, FakeBroadcaster({}), fee_sats=0) - - for chunk_size in (0, -1): - with self.subTest(chunk_size=chunk_size), self.assertRaisesRegex( - ValueError, - "chunk_size", - ): - daemon.run_once(chunk_size=chunk_size) - - with self.assertRaisesRegex(ValueError, "at most"): - daemon.run_once( - chunk_size=MAX_CTV_FANOUT_BROADCASTER_CHUNK_SIZE + 1 - ) - - def test_direct_parent_failure_is_journaled_without_terminal_rejection(self) -> None: - fanout_txid = "45" * 32 - ledger = FakeLedger([pending_row(fanout_txid)]) - broadcaster = FakeBroadcaster( - { - fanout_txid: BroadcastAttempt( - fanout_txid=fanout_txid, - status=BROADCASTABLE, - submitted=False, - fee_sats=0, - package_msg="error", - detail="initial broadcast: transient RPC error", - ) - } - ) - - result = CtvFanoutBroadcastDaemon(ledger, broadcaster, fee_sats=0).run_once() - - self.assertEqual(result.failed_count, 1) - self.assertEqual(ledger.updates, []) - self.assertEqual(ledger.attempts[0]["attempt_status"], "planned") - self.assertEqual(ledger.attempts[0]["submit_result"]["fee_sats"], 0) - - def test_cpfp_rpc_error_is_journaled_without_terminal_rejection(self) -> None: - fanout_txid = "46" * 32 - ledger = FakeLedger([pending_row(fanout_txid)]) - broadcaster = FakeBroadcaster( - { - fanout_txid: BroadcastAttempt( - fanout_txid=fanout_txid, - status=BROADCASTABLE, - submitted=False, - fee_sats=900, - package_msg="error", - detail="initial broadcast: transient RPC error", - ) - } - ) - - result = CtvFanoutBroadcastDaemon(ledger, broadcaster, fee_sats=900).run_once() - - self.assertEqual(result.failed_count, 1) - self.assertEqual(ledger.updates, []) - self.assertEqual(ledger.attempts[0]["attempt_status"], "planned") - self.assertEqual(ledger.attempts[0]["submit_result"]["fee_sats"], 900) - - def test_direct_txid_mismatch_is_terminal_rejection(self) -> None: - fanout_txid = "47" * 32 - ledger = FakeLedger([pending_row(fanout_txid)]) - broadcaster = FakeBroadcaster( - { - fanout_txid: BroadcastAttempt( - fanout_txid=fanout_txid, - status=BROADCASTABLE, - submitted=False, - fee_sats=0, - package_msg="txid_mismatch", - detail="initial broadcast: submitted txid did not match artifact", - ) - } - ) - - result = CtvFanoutBroadcastDaemon(ledger, broadcaster, fee_sats=0).run_once() - - self.assertEqual(result.failed_count, 1) - self.assertEqual(ledger.attempts[0]["attempt_status"], "rejected") - self.assertEqual(ledger.attempts[0]["submit_result"]["fee_sats"], 0) - - def test_allows_zero_fee_for_direct_parent_broadcast(self) -> None: - fanout_txid = "55" * 32 - ledger = FakeLedger([pending_row(fanout_txid)]) - broadcaster = FakeBroadcaster( - { - fanout_txid: BroadcastAttempt( - fanout_txid=fanout_txid, - status=BROADCAST, - submitted=True, - fee_sats=0, - package_msg="success", - detail="initial broadcast", - ) - } - ) - - result = CtvFanoutBroadcastDaemon(ledger, broadcaster, fee_sats=0).run_once() - - self.assertEqual(result.submitted_count, 1) - self.assertEqual(ledger.attempts[0]["package_txids"], [fanout_txid]) - self.assertEqual(ledger.attempts[0]["submit_result"]["fee_sats"], 0) - - def test_rejects_negative_fee(self) -> None: - with self.assertRaisesRegex(ValueError, "fee_sats"): - CtvFanoutBroadcastDaemon(FakeLedger([]), FakeBroadcaster({}), fee_sats=-1) - - def test_env_positive_int_rejects_missing_or_invalid_values(self) -> None: - with patch.dict("os.environ", {}, clear=True): - self.assertEqual(env_positive_int("MISSING_WITH_DEFAULT", 5), 5) - with self.assertRaisesRegex(SystemExit, "REQUIRED"): - env_positive_int("REQUIRED") - for value in ("0", "-1", "nope"): - with self.subTest(value=value), patch.dict("os.environ", {"FEE": value}, clear=True): - with self.assertRaises(SystemExit): - env_positive_int("FEE") - - def test_make_daemon_from_env_requires_wallet_for_cpfp_fee(self) -> None: - with patch.dict( - "os.environ", - { - "QBIT_RPC_HOST": "127.0.0.1", - "QBIT_RPC_USER": "user", - "QBIT_RPC_PASSWORD": "password", - "PRISM_CTV_BROADCASTER_FEE_BITS": "1", - }, - clear=True, - ): - with self.assertRaisesRegex(SystemExit, "PRISM_CTV_BROADCASTER_WALLET"): - make_daemon_from_env() - - # The lease/session-guard env vars below tune caller-side lease - # acquisition and the server-side session guards (#137). These two - # operator-run scripts construct PsqlShareLedger directly, so the tests - # pin that operator tuning actually reaches the constructed ledger - # instead of silently falling back to the class defaults. Construction - # is kept offline by stubbing the two lease I/O steps at the end of - # PsqlShareLedger.__init__; everything asserted is set before them. - - LEASE_KNOB_ENV = { - "PRISM_LEDGER_LEASE_ACQUIRE_LOCK_TIMEOUT_SECONDS": "2.5", - "PRISM_LEDGER_LEASE_ACQUIRE_ATTEMPTS": "7", - "PRISM_POSTGRES_IDLE_IN_TRANSACTION_TIMEOUT_SECONDS": "33.5", - "PRISM_POSTGRES_TCP_KEEPALIVES_IDLE_SECONDS": "17", - "PRISM_POSTGRES_TCP_KEEPALIVES_INTERVAL_SECONDS": "4", - "PRISM_POSTGRES_TCP_KEEPALIVES_COUNT": "9", - } - - def assert_lease_knobs_applied(self, ledger: object) -> None: - self.assertIsInstance(ledger, PsqlShareLedger) - self.assertEqual(ledger._lease_acquire_lock_timeout_seconds, 2.5) - self.assertEqual(ledger._lease_acquire_attempts, 7) - guards = ledger._session_guards - self.assertEqual(guards.idle_in_transaction_timeout_seconds, 33.5) - self.assertEqual(guards.tcp_keepalives_idle_seconds, 17) - self.assertEqual(guards.tcp_keepalives_interval_seconds, 4) - self.assertEqual(guards.tcp_keepalives_count, 9) - - def test_make_ledger_from_env_honours_lease_and_session_guard_env(self) -> None: - environ = { - "PRISM_POSTGRES_PSQL_COMMAND": "psql postgresql://qbit@127.0.0.1:1/qbit", - **self.LEASE_KNOB_ENV, - } - with patch.dict("os.environ", environ, clear=True), patch.object( - PsqlShareLedger, "_initialize_writer_lease_guard", lambda self, database_url: None - ), patch.object(PsqlShareLedger, "_ensure_writer_lease", lambda self: None): - ledger = make_ledger_from_env() - try: - self.assert_lease_knobs_applied(ledger) - finally: - ledger.close() - - def test_backfill_construction_honours_lease_and_session_guard_env(self) -> None: - environ = { - "PRISM_POSTGRES_PSQL_COMMAND": "psql postgresql://qbit@127.0.0.1:1/qbit", - **self.LEASE_KNOB_ENV, - } - with patch.dict("os.environ", environ, clear=True), patch.object( - PsqlShareLedger, "_initialize_writer_lease_guard", lambda self, database_url: None - ), patch.object(PsqlShareLedger, "_ensure_writer_lease", lambda self: None): - args = build_parser().parse_args(["--no-init-schema"]) - ledger = ledger_from_args(args) - try: - self.assert_lease_knobs_applied(ledger) - finally: - ledger.close() - - def test_json_rpc_wallet_call_uses_wallet_url(self) -> None: - seen: list[tuple[str, str]] = [] - - class FakeResponse: - status = 200 - - def read(self) -> bytes: - return json.dumps({"error": None, "result": {"ok": True}}).encode() - - class FakeConnection: - def __init__(self, host: str, port: int, timeout: float) -> None: - self.host = host - self.port = port - self.timeout = timeout - self.sock = None - - def request(self, method: str, path: str, body: bytes, headers: dict) -> None: - seen.append((method, path)) - - def getresponse(self) -> FakeResponse: - return FakeResponse() - - def close(self) -> None: - return None - - rpc = JsonRpc(host="127.0.0.1", port=18452, user="u", password="p") - with patch("http.client.HTTPConnection", FakeConnection): - self.assertEqual(rpc.call("getwalletinfo", wallet="fee wallet"), {"ok": True}) - - self.assertEqual(seen, [("POST", "/wallet/fee%20wallet")]) - - def test_json_rpc_reuses_connection_and_retries_after_stale_drop(self) -> None: - # First call establishes a keep-alive connection; the second reuses it; - # a mid-call disconnect (idle keep-alive closed by the server) is - # retried once on a fresh connection rather than surfacing an error. - constructed: list[object] = [] - - class FakeResponse: - status = 200 - - def read(self) -> bytes: - return json.dumps({"error": None, "result": "tip"}).encode() - - class FakeConnection: - def __init__(self, host: str, port: int, timeout: float) -> None: - self.sock = None - self.requests = 0 - self.closed = False - self.fail_next = False - constructed.append(self) - - def request(self, method: str, path: str, body: bytes, headers: dict) -> None: - self.requests += 1 - if self.fail_next: - raise ConnectionResetError("stale keep-alive") - - def getresponse(self) -> FakeResponse: - return FakeResponse() - - def close(self) -> None: - self.closed = True - - rpc = JsonRpc(host="127.0.0.1", port=18452, user="u", password="p") - with patch("http.client.HTTPConnection", FakeConnection): - self.assertEqual(rpc.call("getbestblockhash"), "tip") - self.assertEqual(rpc.call("getbestblockhash"), "tip") - self.assertEqual(len(constructed), 1) # one connection reused - self.assertEqual(constructed[0].requests, 2) - - constructed[0].fail_next = True # live connection goes stale mid-request - self.assertEqual(rpc.call("getbestblockhash"), "tip") - self.assertTrue(constructed[0].closed) - self.assertEqual(len(constructed), 2) # reconnected once - self.assertEqual(constructed[1].requests, 1) - - def test_json_rpc_never_transparently_retries_mutating_methods(self) -> None: - class FakeConnection: - def __init__(self, host: str, port: int, timeout: float) -> None: - self.sock = None - self.requests = 0 - self.closed = False - - def request(self, method: str, path: str, body: bytes, headers: dict) -> None: - self.requests += 1 - raise ConnectionResetError("result is uncertain") - - def close(self) -> None: - self.closed = True - - methods = ( - "getnewaddress", - "sendrawtransaction", - "signrawtransactionwithwallet", - "submitblock", - "submitpackage", - ) - for method in methods: - with self.subTest(method=method): - constructed: list[FakeConnection] = [] - - def make_connection( - host: str, - port: int, - timeout: float, - ) -> FakeConnection: - connection = FakeConnection(host, port, timeout) - constructed.append(connection) - return connection - - rpc = JsonRpc( - host="127.0.0.1", - port=18452, - user="u", - password="p", - ) - with patch("http.client.HTTPConnection", make_connection), self.assertRaises( - ConnectionResetError - ): - rpc.call(method) - - self.assertEqual(len(constructed), 1) - self.assertEqual(constructed[0].requests, 1) - self.assertTrue(constructed[0].closed) - - def test_json_rpc_enforces_wall_clock_deadline_on_slow_body(self) -> None: - # The socket timeout bounds each recv, not the call: a body dripping - # one packet per interval keeps every recv in-timeout while the call - # runs unbounded. Lease authority margins treat the timeout as a - # wall-clock cap on fence-guarded effects, so the deadline watchdog - # must sever the connection and surface a TimeoutError instead. - severed = threading.Event() - - class DrippingResponse: - status = 200 - - def read(self) -> bytes: - # Blocks like a slow-dripping body until the watchdog severs - # the connection, then fails like a closed socket would. The - # wait is a backstop only; the assertion below proves the - # watchdog actually severed within its deadline. - severed.wait(5) - raise ConnectionResetError("connection severed") - - class FakeSock: - def shutdown(self, how: int) -> None: - severed.set() - - class FakeConnection: - def __init__(self, host: str, port: int, timeout: float) -> None: - self.sock = FakeSock() - - def request(self, method: str, path: str, body: bytes, headers: dict) -> None: - return None - - def getresponse(self) -> DrippingResponse: - return DrippingResponse() - - def close(self) -> None: - severed.set() - - rpc = JsonRpc(host="127.0.0.1", port=18452, user="u", password="p") - begun = time.monotonic() - with patch("http.client.HTTPConnection", FakeConnection): - with self.assertRaisesRegex(TimeoutError, "timed out"): - rpc.call("getbestblockhash", timeout=0.2) - # The watchdog severed at its ~0.3s allowance, not the 5s backstop: - # anything near the backstop means the sever never reached the sock. - self.assertLess(time.monotonic() - begun, 2.0) - self.assertTrue(severed.is_set()) - - def test_json_rpc_connections_never_auto_reconnect(self) -> None: - # http.client's auto_open lets request() silently re-open a - # connection whose socket was cleared — exactly what the deadline - # watchdog's close() does when it lands between the pre-send check - # and the send. That implicit fresh socket would live until the - # next watchdog sweep, long enough for a short mutating POST to - # reach qbitd after the caller released its ordering locks. The - # connection must refuse to resurrect itself: a severed socket - # makes the send fail, and only call()'s explicit, deadline-checked - # connect may open one. - rpc = JsonRpc(host="127.0.0.1", port=18452, user="u", password="p") - conn = rpc._acquire_connection(1.0) - self.assertEqual(conn.auto_open, 0) - with self.assertRaises(http.client.NotConnected): - conn.send(b"post-deadline byte") - - def test_json_rpc_deadline_stops_late_name_resolution_before_send(self) -> None: - # A stalled name resolution has no socket for the watchdog to sever, - # and once resolution returns the caller thread would resume straight - # into sendall() — a short mutating POST could reach qbitd between - # watchdog sweeps. The explicit connect-then-recheck must make the - # late connect lose deterministically: the request is never sent. - constructed: list[object] = [] - - class FakeSock: - def shutdown(self, how: int) -> None: - return None - - class FakeConnection: - def __init__(self, host: str, port: int, timeout: float) -> None: - self.sock: FakeSock | None = None - self.request_calls = 0 - constructed.append(self) - - def connect(self) -> None: - # Name resolution stalls past the deadline, then the - # connection comes up as if DNS finally answered. - threading.Event().wait(0.45) - self.sock = FakeSock() - - def request(self, method: str, path: str, body: bytes, headers: dict) -> None: - self.request_calls += 1 - - def getresponse(self) -> object: - raise AssertionError("request must never be sent past the deadline") - - def close(self) -> None: - return None - - rpc = JsonRpc(host="127.0.0.1", port=18452, user="u", password="p") - with patch("http.client.HTTPConnection", FakeConnection): - with self.assertRaisesRegex(TimeoutError, "timed out"): - rpc.call("submitblock", timeout=0.2) - # The deadline check fired after the late connect and before any - # request byte went out. - self.assertEqual(constructed[0].request_calls, 0) - - -if __name__ == "__main__": - unittest.main() diff --git a/tests/test_docs_lab_prism_references.py b/tests/test_docs_lab_prism_references.py new file mode 100644 index 000000000..167329380 --- /dev/null +++ b/tests/test_docs_lab_prism_references.py @@ -0,0 +1,3783 @@ +#!/usr/bin/env python3 +"""Guard against docs that name Python coordinator code deleted in #244 (issue #303). + +Two contracts over every tracked file under ``docs/``: + +a. No runnable ``python -m lab.…`` or ``python lab/….py`` command invokes a + module or script absent from the branch, however it is wrapped across shell + backslash continuation lines. A ``-m`` target must be a module file + (``lab/a/b.py``) or a package with ``lab/a/b/__main__.py``: CPython refuses + to run a package that has only ``__init__.py``, and a bare directory is a + namespace package with the same refusal, while a module file inside such a + directory runs. There is no allowlist. The check is lexical: + it reads direct ``python``/``python3``/``python3.N`` invocations, bare or + by path (``/usr/bin/python3``), with their CPython option forms, quoted + interpreter names, option words and targets (plain, ANSI-C ``$'…'`` or + locale ``$"…"`` quotes), shell word concatenation of the interpreter, of + an option word and of the target (``"python"3``, ``'pyth'on3``, ``"-"O``, + ``"-m"lab.prism.deleted``, ``"lab.prism."deleted``) and equivalent POSIX + script paths (``./``, ``..`` and repeated slashes), + and does not follow ``cd``, ``PYTHONPATH`` or other environment + indirection, aliases, shell variables, or backslash escapes. Shell comments, + ordinary arguments and heredoc bodies run no command of their own. Command + positions include shell lists, brace groups and loop conditions/bodies, + the env/sudo/nohup/command/exec/builtin and docker/podman exec wrappers, + and literal sh/bash/dash/ksh/zsh ``-c`` strings up to a ``set -n`` or + ``set -o noexec`` of their own, after which the shell reads without + executing. + Other launcher grammars, shell evaluation of stdin, and expansions inside + quoted arguments or heredocs are outside this lexical check. +b. Every ``lab/prism/…`` path or ``lab.prism.…`` module reference resolves to a + tracked file or directory. GitHub links pinned to a 40-hex commit SHA are + stable history and exempt. Pre-existing residue that #303 declares out of + scope is ratcheted in ``RATCHET``: the count may shrink, never grow. + +A ``.patch`` or ``.diff`` file under ``docs/`` quotes another tree's before-and- +after text and is not this repository's prose, so neither contract reads it; +every other tracked file under ``docs/`` is in scope, whatever its suffix. +""" + +from __future__ import annotations + +import posixpath +import re +import shlex +import string +import subprocess +import unicodedata +import unittest +from collections import Counter +from pathlib import Path + + +ROOT = Path(__file__).resolve().parents[1] + +# Dangling-reference ceilings for residue #303 leaves to its own follow-up. +# `docs/prism-overload-alerts.md` and `docs/prism-coordinator-refactor/` are +# #244 residue with the same problem as capacity-readiness and get the same +# treatment as a separate change; `docs/prism-rust-migration.md` describes the +# 2.x.x lane, where the named module still exists. Each ceiling is the count +# measured when this guard landed. Lower a ceiling when its file improves; +# never raise one. `docs/prism-capacity-readiness.md` is deliberately absent. +RATCHET = { + "docs/prism-overload-alerts.md": 32, + "docs/prism-coordinator-refactor/README.md": 1, + "docs/prism-coordinator-refactor/a1-audit-artifacts.md": 1, + "docs/prism-coordinator-refactor/b3-decision.md": 3, + "docs/prism-coordinator-refactor/validation.md": 1, + "docs/prism-rust-migration.md": 1, +} + +# A unified diff under docs/ is quoted history: its `-` and `+` lines are the +# text of another tree before and after a change (`docs/prism-alert-rules- +# qbit-tools.patch` is a diff of qbit-tools' Ansible role), and editing them +# would break the patch. Neither contract reads such a file. Every other suffix +# stays in scope, so a `.json` or `.yaml` example is still checked. +QUOTED_HISTORY_SUFFIXES = frozenset({".patch", ".diff"}) + + +def prose_reference_pattern(root: str, separator: str) -> re.Pattern[str]: + # Quotes and backticks preserve literal spaces and punctuation. Bare + # references consume whole tokens; only trailing prose punctuation is + # removed below, so Dockerfile!old and tool*old cannot resolve as prefixes. + # A Markdown link's ]( or ][ ends its label, outside quoted literals. + literal = rf"{root}(?:{separator}.*?)?" + bare = rf"{root}(?:{separator}(?:(?!\]\(|\]\[)[^\s`'\"<>])+)?" + return re.compile( + rf"(?P<quote>[`'\"])(?P<literal>{literal})(?P=quote)|(?P<bare>{bare})" + ) + + +# A root starts where no letter or digit precedes it. `\b` would refuse an +# underscore there, since `_` is a word character, and the emphasized +# `_lab/prism/x_` would never match; whether that underscore is emphasis or +# the `my_lab` of another path is settled by the root prefixes below. +ROOT_BOUNDARY = r"(?<![^\W_])" +PATH_REFERENCE = prose_reference_pattern(rf"{ROOT_BOUNDARY}lab/prism", "/") +MODULE_REFERENCE = prose_reference_pattern(rf"{ROOT_BOUNDARY}lab\.prism", r"\.") +# A bare reference loses its trailing prose punctuation. `_` is not listed: a +# file may be named `tool_`, and `lab/prism/tool_` names it. The run that +# closes an emphasis the token's own underscores opened is removed instead, by +# the delimiters the pair spends (``prose_reference``), so `_lab/prism/tool__` +# still names `lab/prism/tool_`. +PROSE_TRAILING_PUNCTUATION = ".,;:!?*()[]{}|" +URL_REFERENCE_PREFIX = re.compile(r"https?://\S+/$") +# A `lab/prism` path names the repository's own tree only where a path token +# starts: at the start of a line or after whitespace, behind any Markdown +# openers `( [ { < * | >` and underscore emphasis that begin that token, or +# after a Markdown link's `](`; then behind any opening quotes or backticks +# and at most one `./`. The `>` is the blockquote marker, which needs no +# space after it: `>lab/prism/x` and the nested `>>lab/prism/x` quote the +# root path as `> lab/prism/x` does. The underscores are emphasis only when +# they close: Markdown renders `_lab/prism/x_`, `__lab/prism/x__` and +# `` _`lab/prism/x`_ `` with the underscores gone and the path emphasized, +# while `_lab/prism/x` with no closing run renders literally and names a +# path in `_lab/`, so the run is captured as `emphasis` and the token is +# read only when ``underscore_emphasis`` finds its closer. Inside a code +# span nothing is emphasis: `` `_lab/prism/x` `` is that literal path, and +# an underscore after the backtick starts no token, since the openers come +# before the quotes here. It also starts directly after the ref of a GitHub +# blob, tree or raw URL. +# `\b` alone matched inside `my-lab/prism`, `my*lab/prism`, `my(lab/prism`, +# `my=lab/prism`, `my>lab/prism`, `vendor/lab/prism`, `/tmp/lab/prism` and +# `https://example.com/lab/prism`, which are other paths, and `my_lab/prism` +# is one more: an opener starts a token only at the start of the line or +# after whitespace. `../lab/prism` and `/lab/prism` are not the root either: +# a scan of doc text does not know which directory the doc sits in, so +# neither spelling can be resolved. Whitespace still starts a token inside a +# code span, so `echo lab/prism/x` is read, and its underscores pair as they +# do in prose, so `echo _lab/prism/x_` is read as `echo *lab/prism/x*` is. +PATH_ROOT_PREFIX = re.compile( + r"(?:(?:^|\s)(?:[(\[{<*|>]|(?P<emphasis>_+))*[`'\"]*(?:\./)?" + r"|\]\([<`'\"]*(?:\./)?" + r"|https?://github\.com/[^/\s]+/[^/\s]+/(?:blob|tree|raw)/[^/\s]+/)$" +) +# A dotted `lab.prism` module is the repository's own only where a token +# starts, as `PATH_ROOT_PREFIX` reads it (the blockquote `>lab.prism.x` and +# `>>lab.prism.x` and the emphasized `_lab.prism.x_` included) but with no +# `./` or URL form: a module name is never a path. A shell `$'` or `$"` may +# open the token, so the prose net still reads `-m $'lab.prism.x"`. `\b` +# alone matched inside `vendor.lab.prism`, `my-lab.prism`, `my>lab.prism`, +# `../lab.prism`, `/tmp/lab.prism` and `https://example.com/lab.prism`, +# which name other modules or paths, and `my_lab.prism.x` and the unclosed +# `_lab.prism.x` name a module of the `my_lab` or `_lab` package. +MODULE_ROOT_PREFIX = re.compile( + r"(?:(?:^|\s)(?:[(\[{<*|>]|(?P<emphasis>_+))*(?:\$['\"])?[`'\"]*|\]\([<`'\"]*)$" +) +# `python3 -OO -X dev -m lab.a.b` and `python3.12 -Werror lab/a/b.py`. Every +# option form `python3 --help` lists may sit between the interpreter and its +# target: clustered flag letters, `-W`/`-X` with an attached or following +# argument, and `--check-hash-based-pycs <mode>` (CPython rejects the `=` +# spelling). `-c cmd` runs its argument and ends the option list, so it is +# deliberately not a prefix option. The interpreter, every option word and +# the target are each read as a whole shell word and unquoted before the +# grammar sees them, see ``command_target`` and the option patterns below. +# The flag letters are every single-letter option `python3 --help` lists on +# CPython 3.14 other than `-c`, `-m`, `-h` with its alias `-?`, and `-V`. +# `-h` and `-?` print the help text and `-V` the version, and CPython then +# exits before it reads the target: on CPython 3.14, `python3 -h -m lab.x`, +# `python3 -V lab/x.py`, `python3 -hm lab.x`, `python3 -Vm lab.x`, `python3 +# -Oh -m lab.x`, `python3 -hW error -m lab.x` and `python3 -VX dev -m lab.x` +# all print and exit 0 without importing or opening anything, while `python3 +# -m lab.x` and `python3 lab/x.py` run it. A command carrying one of them +# ahead of its target therefore runs nothing, so neither letter is a prefix +# option nor may close a `-m` cluster, and the target after one is left to +# the prose contract like the argument of `-c`. +PYTHON_FLAG = r"[bBdEiIOPqRsSuvx]" +# A word the shell hands over as one argument: runs of unquoted characters +# alternating with matching-quoted strings (`'error'::Warning`, `"dev mode"`, +# `"lab.prism."deleted`), non-empty. A quoted string may carry bash's `$` +# prefix: ANSI-C quoting `$'…'` and locale quoting `$"…"` are strings the +# shell strips like plain quotes, so `python3 -m $'json.'tool`, `python3 -m +# $"json.tool"` and `$'python3' -m json.tool` all ran `json.tool` on bash 3.2. +# A `$` directly before a quote therefore belongs to the quoted piece and is +# never an unquoted character, which keeps the split of a word unique; a `$` +# before anything else (`$VAR`, `"$c"`) is an ordinary unquoted character, as +# before. The first piece is one unquoted character or one quoted string, so +# a lone or unterminated opening quote is not a word; after it, unquoted runs +# and quoted strings alternate rather than nest, so the pattern never has two +# ways to split one word and cannot backtrack exponentially. The word ends at +# whitespace, at a bash metacharacter (`|`, `&`, `;`, `(`, `)`, `<`, `>`), at +# a backtick (which closes the inline code a command sits in, and in the +# shell opens a command substitution this check does not follow), or at a +# quote that opens no string: inside `sh -c "python3 -m lab.a.b"` the closing +# `"` belongs to the enclosing string, and the inner command is read as +# before. The quotes the shell strips, with their `$` prefix, are removed by +# ``unquote`` before a word is read as an option or a target. The backslash +# escapes `$'…'` decodes are not: bash runs `python3 -m $'json\x2etool'` as +# `json.tool`, but this check follows no backslash escape (module docstring), +# so ``literal_word`` rejects a target word containing such an escape. +WORD_BREAK = r"\s|&;()<>`" +UNQUOTED_CHARACTER = rf"[^{WORD_BREAK}'\"$]|\$(?!['\"])" +QUOTED_STRING = r"\$?'[^']*'|\$?\"[^\"]*\"" + + +def shell_word(quoted_string: str) -> str: + """The word pattern above, with ``quoted_string`` as its quoted piece.""" + return ( + rf"(?:{UNQUOTED_CHARACTER}|{quoted_string})" + rf"(?:{UNQUOTED_CHARACTER})*(?:(?:{quoted_string})(?:{UNQUOTED_CHARACTER})*)*" + rf"(?![^{WORD_BREAK}'\"])" + ) + + +SHELL_WORD = shell_word(QUOTED_STRING) +# The interpreter word is read with quoted pieces that hold no whitespace. A +# program name is one word, so nothing is lost, and the restriction is what +# lets the word be read from every position of a line: with arbitrary quoted +# pieces, `"$c" sh -c "python3` read from `$c` would be the one word +# `$c" sh -c "python3`, pairing the closing quote of `"$c"` with the opening +# quote of the `sh -c` string and hiding the `python3` inside it. +INTERPRETER_STRING = r"\$?'[^'\s]*'|\$?\"[^\"\s]*\"" +INTERPRETER_WORD = shell_word(INTERPRETER_STRING) +# The words after the interpreter are read one shell word at a time by +# ``command_target``, each stripped of its matching quotes by ``unquote`` +# before the grammar below sees it, so an option word is as runnable quoted, +# in pieces or ANSI-C quoted as bare: bash 3.2 and CPython 3.14 ran `python3 +# "-O" -m json.tool`, `python3 '-OO' -m json.tool`, `python3 $'-OO' -m +# json.tool`, `python3 "-"O -m json.tool`, `python3 '-m' json.tool`, +# `python3 "-m"json.tool`, `python3 -"m" json.tool`, `python3 "-X" dev -m +# json.tool`, `python3 '-X'dev -m json.tool`, `python3 "-X"'dev' -m +# json.tool`, `python3 "-W" error -m json.tool`, `python3 '-uW'error -m +# json.tool` and `python3 "--check-hash-based-pycs" always -m json.tool` +# exactly as their bare spellings, and `python3 '--' lab/gone.py` ran the +# script, since the shell strips the quotes before CPython sees one +# argument. A word with an unterminated or mismatched quote (`"-O'`, `-X +# "dev'`) is no shell word: bash rejects the line with "unexpected EOF while +# looking for matching `"'" before Python starts, so the command ends there +# with no target, as it does at such a target word. The grammar, applied to +# the unquoted word in this order: +# - a flag cluster (`-O`, `-OO`, `-bb`, `-IsE`) is skipped; +# - a `-W`/`-X` cluster takes its argument attached (`-Xdev`, `-uWerror`, +# `-X=dev`, which CPython 3.14 takes as an unknown `-X` value and runs on) +# or, with nothing attached, as the next word whatever it is (`-X dev`, +# `-X "dev mode"`; `python3 -X -m lab.x` hands `-m` to `-X` and then opens +# `lab.x` as a script). It is tried before the `-m` cluster, so `-Wm lab.x` +# and `-Xm lab.x` hand `m` to `-W`/`-X` and open `lab.x` as a script, which +# names no lab script, as CPython does; +# - `--check-hash-based-pycs` takes the next word, which unquoted must be one +# of its three modes: CPython 3.14 rejects `--check-hash-based-pycs +# sometimes -m lab.x` ("must be one of 'default', 'always', or 'never'") +# before it runs anything, so that is no command; +# - `-m` may close a flag cluster and takes its module attached or as the +# next word: CPython 3.12 runs `-mlab.x`, `-Im lab.x`, `-OOm lab.x` and +# `-Imlab.x` alike (`-Rm` on 3.14); +# - `--` ends the options and the next word is the script path: `python3 -- +# s.py` and `python3 -O -- s.py` run the script on CPython 3.14, while +# `python3 -- -m json.tool` fails with "can't open file '.../-m'", so its +# script target is `-m`, which names no lab script; +# - any other word is the script path. `-c…`, `-h…`, `-?`, `-V…` and a +# `--long` option therefore end the command as its target word, which +# names no lab script, and what follows is left to the prose contract. +FLAG_CLUSTER = re.compile(rf"-{PYTHON_FLAG}+") +ARGUMENT_OPTION = re.compile(rf"-{PYTHON_FLAG}*[WX](?P<argument>.*)") +MODULE_OPTION = re.compile(rf"-{PYTHON_FLAG}*m(?P<module>.*)") +PYCS_OPTION = "--check-hash-based-pycs" +PYCS_MODES = frozenset({"always", "default", "never"}) +OPTIONS_END = "--" +# The next shell word after whitespace, where the tokenizer stands after the +# interpreter or the word before; nothing at the end of the line or at a word +# the shell would reject. +NEXT_WORD = re.compile(rf"\s+(?P<word>{SHELL_WORD})") +# A redirection the shell removes before CPython sees its arguments, so it may +# sit anywhere among the options and target: `python3 2>/dev/null -m lab.x`, +# `python3 </dev/null -m lab.x`, `python3 -m 2>&1 lab.x` and `python3 +# -X>log dev lab/x.py` run as if it were absent. A file descriptor number +# counts only as a whole word before `<` or `>`; the operator, including the +# `>&`, `<&`, `&>` and `&>>` duplications, is followed by its one target word. +# An escaped operator is word text, not a redirection, and ends the scan. +REDIRECTION = re.compile( + rf"\s*(?:(?<=\s)[0-9]+(?=[<>]))?(?<!\\)(?:<<<|<<-?|<&|<>|<|>&|>>|>\||>|&>>?)\s*{SHELL_WORD}" +) + + +# The interpreter is read as a whole shell word (`INTERPRETER_WORD`), like +# the option words and the target: bash 3.2 runs `"python"3 -m json.tool`, +# `'pyth'on3 -m json.tool`, `python"3" -m json.tool` and `"/usr/bin/pyth"on3 +# -m json.tool` exactly as `python3`, so a name the shell assembles from +# pieces is as runnable as a bare or a wholly quoted one. ``runs_python`` +# strips the matching quotes, takes the basename after the last `/` +# (`/usr/bin/python3` runs Python as `python3` does) and accepts the word +# only if it is wholly `python`, `python3` or `python3.N` (`INTERPRETER`): +# `mypython3`, `cpython3` and `python2` are other programs, and bash answers +# `mypython3 -m json.tool` with "command not found". A word starts where no +# unquoted word character precedes it, so `env python3 …`, `docker exec "$c" +# python3 …`, `$(python3 …)`, inline-code `` `python3 …` `` and the inner +# command of `sh -c "python3 …"` are read as before, while `mypython3` is +# one word. ``python_commands`` selects executable positions and requires a +# candidate to cover its whole shell word, so the tail `python3` after the +# closing quote of `"my"python3` cannot become another command. The candidate +# remains a lookahead to expose the interpreter's start and end. A word that +# opens a quote and never closes it +# (`"python3' -m lab.a.b`) is a shell syntax error and matches nothing, as +# before. ``command_target`` reads the words after a candidate that names +# Python, and the reported command is the line from the interpreter word to +# the target word. +INTERPRETER = re.compile(r"python(?:3(?:\.\d+)?)?") +INTERPRETER_CANDIDATE = re.compile(rf"(?=(?<![^{WORD_BREAK}'\"])(?P<interpreter>{INTERPRETER_WORD}))") +# The `-m` target and the script path are read as whole shell words, since +# bash hands `-m "lab.prism."deleted`, `-m lab."prism".deleted` and +# `"./lab/prism/"deleted.py` over exactly as their bare spellings (verified +# with `python3 -m "json."tool` on bash 3.2 and CPython 3.14). +# ``command_target`` strips the matching quotes and ``dead_commands`` normalizes +# POSIX script path components without following filesystem links, then reads +# the word against the target grammar: a word that then names no `lab` +# module or script (`$VAR`, `json.tool`, `lab.prism.` with nothing after the +# dot) is not a `lab` command. +# CPython's `-m` resolves filenames, not Python identifiers: hyphens, +# leading digits, Unicode and quoted spaces can all name runnable modules. +# Keep nonempty dotted segments. ``literal_word`` rules out expansions and +# escapes before this grammar sees a module, preserving quoted literal `$` +# and backslash characters. Paths still stay outside this lexical check. +MODULE_TARGET = re.compile(r"lab(?:\.[^./\x00]+)+") +# Script filenames may also contain literal spaces, dollars, backslashes or +# glob characters when quoted. Expansion checks happen before this grammar. +SCRIPT_TARGET = re.compile(r"lab/[^\x00]+\.py") +MATCHING_QUOTES = re.compile(QUOTED_STRING) +WORD_PART = re.compile(rf"{QUOTED_STRING}|(?:{UNQUOTED_CHARACTER})+") +# A `#` opens a shell comment only where a word starts: at the start of the +# line or after whitespace or a metacharacter (`WORD_BREAK`), neither quoted +# nor escaped. bash 5.2 runs nothing from `# echo NO`, `true;#echo NO` or the +# tail of `echo a # echo NO`, while `echo '#'`, `echo "# x"`, `echo \#`, +# `echo a#b`, `echo $#`, `echo ${#a[@]}`, `echo a\ #b`, `echo $'\'#'` and +# `echo "\"#"` each print a `#` or a count and run the command after `;`. +# Inside a fenced code block (`FENCE`) the comment ends as bash ends it: at the +# end of the line, backticks included (`# was `echo NO`` and `echo ok # was +# `echo NO`` ran no `echo NO`), unless it opened inside a backtick command +# substitution, which it ends at the closing backtick (bash printed `RUN` for +# `` out=`true # echo NO`; echo RUN ``). Outside a fence a backtick opens or +# closes Markdown inline code, a shell context of its own: the comment ends at +# the next backtick and no quote carries across one, so a heading, an issue +# `#303`, a `(#anchor)` link or a "don't" hides no `` `python3 …` `` after it. +# A backslash inside a comment is comment text and continues nothing: bash +# ran the line after `# c \`. +COMMENT_BOUNDARY = re.compile(rf"[{WORD_BREAK}]") +# A line opening or closing a fenced code block: three or more backticks, +# which an info string may not contain, or tildes, at any indentation so a +# fence nested in a list item counts. A closing fence repeats the opening +# character at least as many times, with nothing after it. +FENCE = re.compile(r"\s*(`{3,}(?=[^`]*$)|~{3,})") +PINNED_GITHUB_URL = re.compile( + r"github\.com/[^/\s]+/[^/\s]+/(?:blob|tree|raw)/[0-9a-f]{40}/$" +) + + +def tracked_paths() -> frozenset[str]: + """Every tracked file plus every directory prefix of one.""" + listing = subprocess.run( + ["git", "ls-files", "-z"], cwd=ROOT, stdout=subprocess.PIPE, check=True + ).stdout.decode() + paths = set() + for path in filter(None, listing.split("\0")): + parts = path.split("/") + paths.update("/".join(parts[:depth]) for depth in range(1, len(parts) + 1)) + return frozenset(paths) + + +def documented(paths) -> list[str]: + """The tracked paths under ``docs/`` both contracts read, sorted. + + Quoted history (``QUOTED_HISTORY_SUFFIXES``) is left out. ``tracked_paths`` + lists directory prefixes as well, so the caller settles whether a path is + a file against the working tree. + """ + return sorted( + path + for path in paths + if path.startswith("docs/") and Path(path).suffix not in QUOTED_HISTORY_SUFFIXES + ) + + +def module_candidates(module: str) -> tuple[str, ...]: + """Every tracked path a prose ``lab.a.b`` mention may name: file, package or directory.""" + relative = module.replace(".", "/") + return (f"{relative}.py", f"{relative}/__init__.py", relative) + + +def runnable_candidates(module: str) -> tuple[str, str]: + """The paths ``python -m lab.a.b`` can execute: a module file or a package ``__main__``. + + Neither ``__init__.py`` nor a bare directory counts: CPython 3.14 answers + both with "No module named lab.a.b.__main__; 'lab.a.b' is a package and + cannot be directly executed" and exits 1, while ``lab/a/b.py`` runs even + when ``lab/a`` is a namespace package without ``__init__.py``. + """ + relative = module.replace(".", "/") + return (f"{relative}.py", f"{relative}/__main__.py") + + +def unquote(word: str) -> str: + """``word`` as the shell hands it to CPython: matching quotes removed, their contents kept. + + The ``$`` of an ANSI-C ``$'…'`` or locale ``$"…"`` string goes with its + quotes, so ``$'lab.prism.'deleted`` is ``lab.prism.deleted``. + """ + return MATCHING_QUOTES.sub(lambda match: match.group(0).removeprefix("$")[1:-1], word) + + +def literal_word(word: str, *, script: bool = False) -> str | None: + """Unquote a validated shell word only when its parts need no expansion or decoding. + + Single quotes preserve every character; ANSI-C quotes preserve dollars + but require decoding for backslashes. Double quotes still expand dollars + and backticks, and escape `$`, backticks, double quotes and backslashes. + A backslash before any other character in double quotes is literal. + Script paths also exclude unquoted glob and brace syntax, whose expansion + depends on the shell and filesystem; quoting preserves those characters. + """ + parts = [] + for match in WORD_PART.finditer(word): + part = match.group(0) + if part.startswith("'"): + part = part[1:-1] + elif part.startswith("$'"): + part = part[2:-1] + if "\\" in part: + return None + elif part.startswith(('"', '$"')): + part = part.removeprefix("$")[1:-1] + if part.endswith("\\") or re.search(r'[$`]|\\[$`"\\]', part): + return None + elif any(character in part for character in "$\\`"): + return None + elif script and any(character in part for character in "*?[]{}"): + return None + parts.append(part) + return "".join(parts) + + +def runs_python(word: str) -> bool: + """Whether ``word`` names CPython: unquoted, its basename is ``python``, ``python3`` or ``python3.N``.""" + return INTERPRETER.fullmatch(unquote(word).rsplit("/", 1)[-1]) is not None + + +def shell_tokens(line: str): + """``(kind, start, end)`` for words and operators, without entering quoted data. + + This lexer keeps escaped quotes inside their word, even though interpreting + escapes in executable names and targets remains outside the check. An + unmatched prose apostrophe ends at Markdown inline code; a quoted word + with a matching close keeps its literal backticks. + """ + position = 0 + while position < len(line): + if line[position].isspace(): + if line[position] == "\n": + yield "operator", position, position + 1 + position += 1 + continue + start = position + if line[position] in "|&;()<>`": + position += 1 + if line[start] in "<>" and line.startswith(line[start], position): + position += 1 + if line[start] == "<" and position < len(line) and line[position] in "<-": + position += 1 + elif line.startswith((">&", "<&", ">|", "&>"), start): + # One redirection operator: its `&` or `|` separates no command. + position += 1 + if line.startswith("&>>", start): + position += 1 + yield "operator", start, position + continue + while position < len(line) and not re.fullmatch(rf"[{WORD_BREAK}]", line[position]): + character = line[position] + if character == "\\": + position += 2 + elif character in "'\"": + quote_start = position + ansi = character == "'" and position > start and line[position - 1] == "$" + position += 1 + while position < len(line) and line[position] != character: + position += 2 if line[position] == "\\" and (character == '"' or ansi) else 1 + if position >= len(line): + backtick = line.find("`", quote_start) + if backtick != -1: + position = backtick + break + yield "unfinished", start, len(line) + return + position += 1 + else: + position += 1 + yield "word", start, min(position, len(line)) + + +SHELLS = frozenset({"sh", "bash", "dash", "ksh", "zsh"}) +LAUNCHERS = frozenset({"env", "sudo", "nohup", "command", "exec", "builtin", "docker", "podman"}) +# The launchers that are shell builtins, the only ones `builtin` can name. +SHELL_BUILTIN_LAUNCHERS = frozenset({"command", "exec", "builtin"}) +WRAPPER_ARGUMENTS = { + "env": {"-u", "--unset", "-C", "--chdir"}, + "sudo": {"-u", "--user", "-g", "--group", "-h", "--host", "-p", "--prompt", "-C", "--close-from", "-D", "--chdir", "-R", "--chroot", "-r", "--role", "-t", "--type", "-T", "--command-timeout"}, + "exec": {"-a"}, + "docker": {"-e", "--env", "--env-file", "-u", "--user", "-w", "--workdir", "--detach-keys"}, +} +SHELL_ASSIGNMENT = re.compile(r"[A-Za-z_][A-Za-z_0-9]*\+?=") + + +def env_split_words(source: str) -> list[tuple[str, int]] | None: + """Literal env -S arguments and their line offsets, without shell evaluation. + + Quotes group words, and # comments begin only at an unquoted word start. + Shell operators are ordinary argv characters. Expansion and escape syntax + are outside this check, just as for literal shell words. + """ + if "$" in source or "\\" in source: + return None + result = [] + word = [] + quote = "" + started = False + number = 0 + first_line = 0 + for character in source: + if not quote and character in " \t\n\r\v\f": + if started: + result.append(("".join(word), first_line)) + word, started = [], False + elif not quote and not started and character == "#": + break + else: + if not started: + started, first_line = True, number + if quote: + if character == quote: + quote = "" + else: + word.append(character) + elif character in "'\"": + quote = character + else: + word.append(character) + number += character == "\n" + if quote: + return None + if started: + result.append(("".join(word), first_line)) + return result + + +def executable_word(words: list[str], offsets: list[int]) -> int | None: + """Find the executable, expanding literal env -S argv and line offsets in place.""" + index = 0 + # Reserved words are syntax only when unquoted in shell command position; + # after an assignment or launcher they are ordinary executable arguments. + while index < len(words) and words[index] in {"!", "if", "then", "elif", "else", "do", "{", "while", "until", "time", "coproc"}: + prefix = words[index] + index += 1 + # Bash's time prefix accepts one unquoted -p; quoted spellings are + # executable words, just as quoted time is not a reserved word. + if prefix == "time" and index < len(words) and words[index] == "-p": + index += 1 + # A coprocess name precedes a compound command, never a simple one. + # Only unquoted openers handled by this loop introduce such a command. + if prefix == "coproc" and index + 1 < len(words) and words[index + 1] in {"{", "if", "while", "until"}: + index += 1 + # Bash recognizes += only in the leading assignment list, with an + # unquoted name and operator. After a launcher it is an ordinary argument. + while index < len(words) and SHELL_ASSIGNMENT.match(words[index]): + index += 1 + while index < len(words): + word = unquote(words[index]) + program = word.rsplit("/", 1)[-1] + if program not in LAUNCHERS: + return index + index += 1 + if program in {"docker", "podman"}: + if index == len(words) or unquote(words[index]) != "exec": + return None + program = "docker" + index += 1 + if program == "builtin": + # `builtin [shell-builtin [arg ...]]` runs its argument as a shell + # builtin and nothing else, so Python is reached through it only + # by way of a builtin this check reads: `command`, `exec` or + # `builtin` itself. bash 5.2.21 ran `echo hi` for `builtin + # command echo hi`, `builtin exec echo hi`, `builtin builtin + # command echo hi`, `builtin -- command echo hi`, `builtin + # 'command' echo hi`, `command builtin command echo hi`, `command + # -p builtin command echo hi` and `FOO=x builtin command echo hi`, + # and printed `echo` for `builtin command -v echo hi`; it answered + # `builtin python3 -c 1`, `builtin env echo hi`, `builtin sudo + # echo hi`, `builtin nohup echo hi`, `builtin docker exec c echo + # hi`, `builtin bash -c 'echo hi'`, `builtin /usr/bin/env echo + # hi`, `builtin ./command echo hi`, `builtin FOO=x command echo + # hi` and `builtin -- -- command echo hi` with "not a shell + # builtin", ran nothing for `builtin` and `builtin --` alone, + # rejected `-x`, `-p`, `-pv`, `--foo` and `--version` as invalid + # options and printed its help for `--help`, running nothing in + # each case; the next command on the line ran after every + # failure. `--` is its one option word, taken once. `exec` after + # it still looks its own argument up as a program, so `builtin + # exec command echo hi` failed with "exec: command: not found" as + # `exec command echo hi` does. dash 0.5.12 has no `builtin` and + # answered "not found", running nothing. A `builtin` after env, + # sudo, nohup, exec or docker exec is looked up as a program and + # not found, as `command` and `exec` are there; that shared gap + # is outside this check. + if index < len(words) and unquote(words[index]) == "--": + index += 1 + if index == len(words): + return None + argument = unquote(words[index]) + if argument == "set": + # The one other builtin read here (``sets_noexec``): bash ran + # nothing after `builtin set -n; echo NO`, `builtin -- set -n; + # echo NO`, `command builtin set -n; echo NO` and `builtin + # builtin set -n; echo NO`. + return index + if argument not in SHELL_BUILTIN_LAUNCHERS: + return None + continue + terminated = False + while index < len(words): + option = unquote(words[index]) + # S takes the rest of a short cluster, or the next word, as its + # source; only argument-free flags may precede it in the cluster. + cluster = re.fullmatch(r"-[iv]*S(.*)", option, re.DOTALL) if program == "env" else None + if cluster or program == "env" and (option == "--split-string" or option.startswith("--split-string=")): + separate = option == "--split-string" or (cluster is not None and not cluster.group(1)) + argument = index + int(separate) + if argument >= len(words): + return None + source = literal_word(words[argument]) + if source is None: + return None + if not separate: + source = source[cluster.start(1) if cluster else len("--split-string="):] + split = env_split_words(source) + if split is None: + return None + # Quote every argv word so operators and shell reserved words + # remain data when the expanded command is read below. + quoted = ["'" + word.replace("'", "'\"'\"'") + "'" for word, _ in split] + origin = offsets[argument] + words[index:argument + 1] = quoted + offsets[index:argument + 1] = [origin + offset for _, offset in split] + continue + if option in {"--help", "--version"} or (program == "sudo" and option in {"-l", "-ll", "--list"}): + return None + if program == "env" and (option == "--null" or re.match(r"-[iv]*0", option)): + # `-0`/`--null` prints the environment NUL-terminated and takes + # no command, so like `--help` it ends the command: GNU + # coreutils 9.4 answered `env -0 echo hi`, `env --null echo + # hi`, `env -i0 echo hi`, `env -0i echo hi`, `env -v0 echo + # hi`, `env -0u FOO echo hi`, `env -0S 'echo hi'`, `env -0 -S + # 'echo hi'` and `env -S '-0 echo hi'` alike with "cannot + # specify --null (-0) with command" and exit 125, and printed + # `FOO=x` NUL-terminated for `env -i -0 FOO=x`. Only + # argument-free flags may precede the `0` in a cluster, as + # with the sudo modes below: `env -u0 echo hi` unsets the + # variable `0` and ran. An abbreviated `--nul` is outside this + # check, as it is for `--help`. + return None + if program == "sudo" and ( + option in {"--edit", "--remove-timestamp", "--validate"} + or re.match(r"-[ABbEHkNnPSis]*[eKvV]", option) + ): + # Edit, credential-only and version modes run no command. + # Only flags without arguments may precede the mode letter in + # a cluster: -nv validates, while -pv gives p the prompt "v". + # Lowercase -k resets credentials but still permits a command + # to run. sudo 1.9.15p5 printed its usage and ran nothing for + # `sudo -nV id`, `sudo -Vn id`, `sudo -HV id` and `sudo -Vu + # nobody id`, as for `sudo -V id`, while `sudo -uV nobody id` + # read the user `V`. + return None + if program == "sudo" and ( + re.fullmatch(r"-[ABbEHkNnPSis]+h", option) + or option == "-h" and ( + index + 1 == len(words) + or unquote(words[index + 1]).startswith("-") + or SHELL_ASSIGNMENT.match(unquote(words[index + 1])) + ) + ): + # sudo's -h takes an optional argument. Attached, it is the + # host (`-hhost`, `-nhhost`; `-nhu nobody` reads the host `u` + # and the command `nobody`); as the whole word `-h` it takes + # the next word when that is neither an option nor an + # assignment; otherwise it is the help mode, whatever + # follows. sudo 1.9.15p5 went on to the command, as far as + # the sudoers plugin lets a remote host be named, for `sudo + # -h host id`, `sudo -hhost id`, `sudo -nhhost id`, `sudo -n + # -h host id` and `sudo -h host -u nobody id`, and printed + # its usage and ran nothing for `sudo -nh host id`, `sudo -Hh + # host id`, `sudo -ih host id`, `sudo -nh`, `sudo -h`, `sudo + # -h -u nobody id`, `sudo -h -- id` and `sudo -h FOO=x id`. + # Its own assignment test is the wider one noted below. + return None + if program == "command" and re.fullmatch(r"-[pvV]*[vV][pvV]*", option): + # The lookup flags print information and run nothing, alone + # or in a cluster with `-p` or each other: bash 5.2.21 + # printed `echo` and ran nothing for `command -v echo hi`, + # `command -pv echo hi`, `command -vp echo hi`, `command -vv + # echo hi` and `command -pv -- echo hi`, described the + # builtin for `command -pV echo hi` and `command -Vp echo + # hi`, and ran `echo hi` for `command -p echo hi`, `command + # -pp echo hi` and `command -p -- echo hi`. + return None + if option == "--": + index += 1 + terminated = True + break + if not option.startswith("-"): + break + arguments = WRAPPER_ARGUMENTS.get(program, ()) + index += 1 + if option.startswith("--"): + if option in arguments: + index += 1 # `--user nobody`; `--user=nobody` is one word + continue + # A short cluster is read letter by letter, as getopt reads it for + # env, sudo and bash's exec: the first letter that takes an + # argument takes the rest of the cluster (`-unobody`; `-pu nobody` + # gives p the prompt "u" and runs `nobody`) or, when it ends the + # cluster, the next word (`-nu nobody`). sudo 1.9.15p5 ran `id + # -un` as nobody for `sudo -nu nobody`, `sudo -nunobody` and + # `sudo -nHu nobody`, answered `sudo -pu nobody id` with "nobody: + # command not found" and `sudo -un nobody id` with "unknown user + # n"; GNU coreutils 9.4 ran `env -iu FOO echo hi` and `env -iC + # /tmp pwd` but `env -ui FOO echo hi` failed to run `FOO`; bash + # 5.2.21 ran `exec -ca zzz echo hi` and `exec -cazzz echo hi`, + # while `exec -ac zzz echo hi` found no `zzz`. docker was not + # installed on the inspected host; its `-e`, `-u` and `-w` are + # read as the same clusters. + for position, letter in enumerate(option[1:], 1): + if f"-{letter}" in arguments: + if position == len(option) - 1: + index += 1 # nothing attached: the next word is the argument + break + if program == "docker": + index += 1 # container name + elif program == "env" or (program == "sudo" and not terminated): + # Only env and sudo take VAR=value words between their options and + # the command: `env [OPTION]... [-] [NAME=VALUE]... [COMMAND + # [ARG]...]` and `sudo [options] [VAR=value] [-i | -s] [command + # [arg ...]]` in their usage lines. Both accept FOO+=value too, + # setting the literal name FOO+ (sudo 1.9.15p5 exported `FOO+=x` + # for `sudo FOO+=x env`, as GNU coreutils 9.4 env does), and both + # receive a quoted `'FOO=x'` as the same word once the shell + # strips the quotes. After `--` they part: env still exported FOO + # for `env -- FOO=x sh -c 'echo $FOO'`, while sudo answered `sudo + # -- FOO=x sh -c '…'` with "FOO=x: command not found". No other + # launcher takes one: nohup 9.4 answered `nohup FOO=x echo hi` + # with "failed to run command 'FOO=x'", bash 5.2.21 `command + # FOO=x echo hi` with "FOO=x: command not found" and `exec FOO=x + # echo hi` with "exec: FOO=x: not found", each exit 127 and each + # running nothing, and `docker exec [OPTIONS] CONTAINER COMMAND + # [ARG...]` runs the word after the container as the program. A + # `FOO=x` after those is therefore the executable word. sudo's + # own test is wider (it also took `9FOO=x`); such spellings are + # outside this check. + while index < len(words) and SHELL_ASSIGNMENT.match(unquote(words[index])): + index += 1 + return None + + +# The letters bash 5.2's `set` accepts, from its usage line `set +# [-abefhkmnptuvxBCEHPT] [-o option-name] [--] [-] [arg ...]`. +SET_FLAGS = frozenset("abefhkmnptuvxBCEHPTo") + + +def sets_noexec(words: list[str], index: int) -> bool: + """Whether the command whose executable word is ``words[index]`` is the shell's ``set`` leaving noexec on. + + A non-interactive shell that runs `set -n` or `set -o noexec` reads the + rest of its input without executing it, and no later command can turn + the mode off, `set +n` included, since none runs: bash 5.2.21 and dash + 0.5.12 printed nothing for `set -n; echo NO`, `set -o noexec; echo NO`, + `set -n; set +n; echo NO`, `set -en; echo NO`, `set -no noexec; echo + NO`, `set -on noexec; echo NO`, `set -o errexit -n; echo NO`, `set -n + x; echo NO`, `set -n -- x; echo NO`, `set '-n'; echo NO`, `FOO=x set + -n; echo NO`, `command set -n; echo NO`, `{ set -n; }; echo NO`, `if + set -n; then :; fi; echo NO` and `set -n` on a line of its own before + `echo NO`, and bash also for `builtin set -n; echo NO`, `time set -n; + echo NO`, `! set -n; echo NO`, `set + -n; echo NO`, `set -o -n; echo + NO`, `set -no; echo NO` and `set -n 2>/dev/null; echo NO`. The options + of one `set` apply in order, so a `+n` or `+o noexec` later in the same + command leaves execution on: both shells ran `echo YES` for `set -n +n; + echo YES` and `set -o noexec +o noexec; echo YES`, as for `set +n; echo + YES`, `set -- -n; echo YES`, `set - -n; echo YES` and `set x -n; echo + YES`, where `--`, `-` or the first argument ends the options. `-o` + takes the next word as its option name unless that word is missing or + starts with `-` or `+`, when bash prints the settings and reads on: `set + -o -n; echo NO` printed them and ran nothing, `set -o +n; echo YES` ran. + A letter outside ``SET_FLAGS`` makes bash reject the whole command and + apply none of it: it ran `echo RAN` for `set -nz; echo RAN`, `set -n + -z; echo RAN`, `set -z -o noexec; echo RAN` and `set -+n; echo RAN`; + dash exits at such an option, and at `set -o -n`, running nothing + either way, and bash's reading is the one followed. An option name bash + rejects stops the command with the earlier options applied (`set -o + nosuch -n; echo RAN` ran, `set -n -o nosuch; echo RAN` did not); every + name is read here as one the shell accepts, so that stop is outside + this check, as it is for the invocation options. + + Only the shell's own `set` counts. After env, sudo, nohup, exec or + docker exec the word names a program: GNU coreutils 9.4 answered `env + set -n; echo YES` and `nohup set -n; echo YES` with "No such file or + directory" and the shell ran on, and bash found no program for `exec + set -n` and exited 127, an exit that is not followed. `command` and + `builtin` reach the builtin, as above. A `set -n` in a subshell leaves + the parent shell running: both shells ran `echo YES` for `(set -n); + echo YES` and `x=$(set -n); echo YES`, and only B for `(set -n; echo + A); echo B`; ``python_commands`` follows that by parenthesis depth. + Read as run where it stands, and so not followed: a `set -n` in a + pipeline element or a background job (`set -n | cat; echo YES` and `set + -n & wait; echo YES` ran, each in a subshell), in a backtick + substitution, behind a condition (`false && set -n; echo YES` ran, + `true && set -n; echo NO` did not, and a `case` arm is the same), or in + a function body, which runs in the calling shell when the function is + called (`f() { set -n; }; f; echo NO` ran nothing in both shells, `f() + { set -n; }; echo YES` with no call ran); `\\set -n` is an escape, + outside this check as escapes are. + """ + if unquote(words[index]) != "set" or any( + unquote(word).rsplit("/", 1)[-1] in LAUNCHERS - SHELL_BUILTIN_LAUNCHERS for word in words[:index] + ): + return False + noexec = False + arguments = [unquote(word) for word in words[index + 1:]] + position = 0 + while position < len(arguments): + option = arguments[position] + position += 1 + if option in {"-", "--"} or not option.startswith(("-", "+")): + break + if any(flag not in SET_FLAGS for flag in option[1:]): + return False # bash rejects the whole command and applies none of it + for flag in option[1:]: + if flag == "n": + noexec = option[0] == "-" + elif flag == "o" and position < len(arguments) and not arguments[position].startswith(("-", "+")): + if arguments[position] == "noexec": + noexec = option[0] == "-" + position += 1 + return noexec + + +def shell_command_argument(words: list[str], shell: str) -> int | None: + """Locate a literal shell's command string after consuming its option arguments.""" + index = 0 + command_string = False + noexec = False + dump = False + restored = False # the last `n` or `o noexec` setting was a `+` + argument_flags = "oO" if shell in {"bash", "sh"} else "o" + # Bash's dump modes read the command string and run nothing: `-D`, + # `--dump-strings` and `--dump-po-strings` print its `$"…"` strings and + # exit, and bash documents `-D` as implying `-n`. Unlike `-n`, no later + # `+n` or `+o noexec` restores execution, and the `+` sign is ignored as + # it is for `+c`: bash 5.2.21 printed nothing for `bash -D -c 'echo hi'`, + # `bash -Dc 'echo hi'`, `bash -cD 'echo hi'`, `bash +D -c 'echo hi'`, + # `bash -D +n -c 'echo hi'`, `bash +n -D -c 'echo hi'`, `bash -D +o + # noexec -c 'echo hi'`, `bash --dump-strings -c 'echo hi'`, `bash + # --dump-po-strings -c 'echo hi'` and `bash --dump-strings +n -c 'echo + # hi'`, while `bash -n +n -c 'echo hi'` ran, and it rejected the + # abbreviated `--dump-str` and `--dump-strings=x` as invalid options, + # running nothing either. dash 0.5.12 answers `-D` with "Illegal option" + # and runs nothing, so an `sh` that is either shell reads the same. ksh + # has the mode bash took the letter from, with one difference: a `+n` or + # `+o noexec` that stands as the last such setting restores execution, + # before or after the `-D`. ksh93u+m 1.0.8 printed only the strings for + # `ksh -D -c 'echo hi'`, `ksh -Dc 'echo hi'`, `ksh -cD 'echo hi'`, `ksh + # -eD -c 'echo hi'`, `ksh -Do vi -c 'echo hi'`, `ksh --dump-strings -c + # 'echo hi'`, `ksh -D +n -n -c 'echo hi'` and `ksh -D +o noexec -n -c + # 'echo hi'`, printed nothing for `ksh +D -c 'echo hi'` and `ksh -n +D + # -c 'echo hi'`, and ran `echo hi` for `ksh -D +n -c 'echo hi'`, `ksh +n + # -D -c 'echo hi'`, `ksh -D +o noexec -c 'echo hi'`, `ksh -nD +n -c + # 'echo hi'`, `ksh +D +n -c 'echo hi'` and `ksh --dump-strings +n -c + # 'echo hi'`; it rejected `--dump-po-strings` as an unknown option and + # ran nothing, which like every unknown option is outside this check. + # mksh R59 rejects `-D` as an unknown option in every spelling and runs + # nothing, so a `ksh` of either kind runs nothing for `ksh -D -c`. zsh + # 5.9 keeps its own meaning for the letter (`PUSHD_IGNORE_DUPS`) and ran + # `echo hi` for `zsh -D -c 'echo hi'` and `zsh +D -c 'echo hi'`, so its + # strings are read as before. + while index < len(words): + option = unquote(words[index]) + if option in {"--help", "--version"}: + return None + if option == "--": + index += 1 + break + if shell in {"bash", "sh"} and option in {"--rcfile", "--init-file"}: + index += 2 + continue + if shell in {"bash", "sh"} and option in {"--dump-strings", "--dump-po-strings"} or ( + shell == "ksh" and option == "--dump-strings" + ): + dump = True + index += 1 + continue + if option.startswith("--"): + index += 1 + continue + if re.fullmatch(r"[+-][A-Za-z]+", option) is None: + break + index += 1 + # Process flags in order: +n/+o noexec can undo an earlier -n, + # including options between -c and its command-string argument. + for flag in option[1:]: + command_string |= flag == "c" + dump |= flag == "D" and shell in {"bash", "sh", "ksh"} + if flag == "n": + noexec = option[0] == "-" + restored = not noexec + if flag in argument_flags: + if index >= len(words): + return None + if flag == "o" and unquote(words[index]) == "noexec": + noexec = option[0] == "-" + restored = not noexec + index += 1 + if dump and (shell != "ksh" or not restored): + return None + return index if command_string and not noexec and index < len(words) else None + + +def python_commands(line: str, noexec: list[bool] | None = None): + """``(line offset, source, match)`` for Python in executable positions, including shell ``-c``. + + Whole words keep ordinary arguments opaque. Only a supported shell's + literal command-string argument starts another shell context; Python's + ``-c`` argument and strings passed to printf, echo, etc. remain data. + + ``noexec`` is the `set -n` state of a literal shell string (see + ``sets_noexec``): one entry per open subshell, the innermost last, true + once the shell reads without executing. The caller keeps it across the + lines of one string, so a `set -n` ends what runs to the end of that + string, and each string starts its own, so the state reaches neither the + document nor another string. A document line is read with none: its + inline code spans and fenced lines are commands of their own, so a `set + -n` there hides nothing, as a `cd` line changes nothing. + """ + words: list[tuple[int, int]] = [] + redirect = False + for kind, start, end in [*shell_tokens(line), ("operator", len(line), len(line))]: + token = line[start:end] + if kind == "unfinished": + return + if kind == "word": + if not redirect: + words.append((start, end)) + redirect = False + continue + if token.startswith(("<", ">", "&>")): + if token[0] != "&" and words and words[-1][1] == start and re.fullmatch(r"[0-9]+", line[words[-1][0]:start]): + words.pop() # the adjacent number is a file descriptor, not a word + redirect = True + continue + command_words = [line[a:b] for a, b in words] + original_words = command_words.copy() + offsets = [line[:a].count("\n") for a, _ in words] + if noexec is not None and noexec[-1]: + index = None # read by the shell, not executed + else: + index = executable_word(command_words, offsets) + if noexec is not None and index is not None and sets_noexec(command_words, index): + noexec[-1] = True + index = None + if index is not None: + command_line, spans = line, words + if command_words != original_words: + command_line = " ".join(command_words) + spans = [] + position = 0 + for word in command_words: + spans.append((position, position + len(word))) + position += len(word) + 1 + first, last = spans[index] + interpreter = command_line[first:last] + match = INTERPRETER_CANDIDATE.match(command_line, first) + if runs_python(interpreter) and match is not None and match.end("interpreter") == last: + yield offsets[index], command_line, match + elif unquote(interpreter).rsplit("/", 1)[-1] in SHELLS: + argument = shell_command_argument( + command_words[index + 1:], unquote(interpreter).rsplit("/", 1)[-1] + ) + if argument is not None: + argument += index + 1 + source = literal_word(command_words[argument]) + if source is not None: + state = [False] + for number, nested in shell_lines(source, shell_source=True): + for offset, source_line, match in python_commands(nested, state): + yield offsets[argument] + number - 1 + offset, source_line, match + if noexec is not None and token == "(": + noexec.append(noexec[-1]) + elif noexec is not None and token == ")" and len(noexec) > 1: + noexec.pop() + words = [] + redirect = False + + +def shell_words(line: str, position: int): + """Each ``NEXT_WORD`` match on ``line`` from ``position``, until no word follows. + + The words end at the end of the line and at a word with an unterminated or + mismatched quote, which bash rejects before anything runs. Redirections + (``REDIRECTION``) between the words are skipped with their target word. + """ + while True: + while (redirection := REDIRECTION.match(line, position)) is not None: + position = redirection.end() + if (match := NEXT_WORD.match(line, position)) is None: + return + yield match + position = match.end() + + +def command_target(line: str, position: int) -> tuple[str, str, int] | None: + """``(kind, target, end)`` of the command whose interpreter word ends at ``position``. + + Reads the words after the interpreter against the option grammar (see + ``FLAG_CLUSTER``) until one names the target: ``kind`` is ``"module"`` + after ``-m`` and ``"script"`` otherwise, ``target`` is the word as the + shell hands it to CPython (matching quotes stripped, or the remainder of + an attached ``-m``), and ``end`` is where the target word ends on the + line. ``None`` when the words run out, at a word the shell rejects, or at + a ``--check-hash-based-pycs`` mode CPython rejects, or when a + target word requires shell expansion or escape decoding outside this check. + """ + words = shell_words(line, position) + for match in words: + word = unquote(match.group("word")) + if FLAG_CLUSTER.fullmatch(word): + continue + if (option := ARGUMENT_OPTION.fullmatch(word)) is not None: + if not option.group("argument") and next(words, None) is None: + return None + continue + if word == PYCS_OPTION: + mode = next(words, None) + if mode is None or unquote(mode.group("word")) not in PYCS_MODES: + return None + continue + if (option := MODULE_OPTION.fullmatch(word)) is not None: + attached = bool(option.group("module")) + target = match if attached else next(words, None) + if target is None or (module := literal_word(target.group("word"))) is None: + return None + if attached: + module = module[option.start("module"):] + return "module", module, target.end() + if word == OPTIONS_END: + target = next(words, None) + else: + target = match + if target is None or (script := literal_word(target.group("word"), script=True)) is None: + return None + return "script", script, target.end() + return None + + +def blank_comments(line: str, *, fenced: bool) -> str: + """``line`` with each shell comment (see ``COMMENT_BOUNDARY``) turned to spaces. + + ``fenced`` says whether the line sits in a fenced code block. Quotes are + followed as bash reads them: nothing escapes inside ``'…'``, a backslash + escapes the next character inside ``"…"`` and ``$'…'`` and outside + quotes. Columns are kept, so a command before a comment reads and reports + as before. + """ + characters = list(line) + quote = "" + word_start = True + in_backticks = False + index = 0 + while index < len(line): + character = line[index] + if quote and (fenced or character != "`"): + if character == "\\" and quote != "'": + index += 1 + elif character == quote[-1]: + quote = "" + elif character == "#" and word_start: + end = line.find("`", index) if in_backticks or not fenced else -1 + end = len(line) if end == -1 else end + characters[index:end] = " " * (end - index) + index = end + continue + elif character == "\\": + index += 1 + elif character in "'\"": + quote = character + elif character == "$" and line.startswith("'", index + 1): + quote = "$'" + index += 1 + elif character == "`": + quote = "" + in_backticks = not in_backticks + word_start = not quote and COMMENT_BOUNDARY.fullmatch(character) is not None + index += 1 + return "".join(characters) + + +def shell_lines(text: str, *, shell_source: bool = False) -> list[tuple[int, str]]: + """``(first line, text)`` per logical shell line, backslash continuations joined, comments blanked. + + Each joined line is blanked again as a whole, in the fence state of its + first line, since a quote may carry across the continuation (bash printed + ``a # b`` for ``echo "a \\`` followed by ``# b"``). A backslash that ends + a comment is blanked with it, so it joins nothing and the next physical + line is a command of its own. Every physical line, joined or not, may open + or close a fence, and a fence line itself is Markdown, not shell. Literal + shell command strings set ``shell_source``: their comments end at the + shell boundary and their backticks cannot open Markdown inline code. + """ + lines: list[tuple[int, str]] = [] + fence = "" + fence_indent = "" + fenced = False + heredocs: list[tuple[str, bool]] = [] + quoted = False + for number, line in enumerate(text.splitlines(), 1): + if heredocs: + delimiter, strip_tabs = heredocs[0] + body = line.removeprefix(fence_indent) + if (body.lstrip("\t") if strip_tabs else body) == delimiter: + heredocs.pop(0) + lines.append((number, "")) + continue + marker = FENCE.match(line) + on_fence = not shell_source and marker is not None and ( + not fence or (marker.group(1).startswith(fence) and not line[marker.end():].strip()) + ) + if lines and lines[-1][1].endswith("\\"): + first, head = lines[-1] + lines[-1] = (first, blank_comments(head[:-1] + line, fenced=fenced)) + elif quoted and not on_fence: + first, head = lines[-1] + lines[-1] = (first, blank_comments(head + "\n" + line, fenced=fenced)) + else: + fenced = shell_source or (bool(fence) and not on_fence) + lines.append((number, blank_comments(line, fenced=fenced))) + if on_fence: + fence_indent = "" if fence else line[:marker.start(1)] + fence = "" if fence else marker.group(1) + logical = lines[-1][1] + tokens = list(shell_tokens(logical)) + # A prose apostrophe in the first word is not a multiline shell + # argument. Fences and arguments after a command may carry quotes. + quoted = bool(tokens) and tokens[-1][0] == "unfinished" and (fenced or len(tokens) > 1) + if not quoted and not logical.endswith("\\"): + for token_index, (kind, start, end) in enumerate(tokens[:-1]): + operator = logical[start:end] + if kind == "operator" and operator in {"<<", "<<-"}: + kind, start, end = tokens[token_index + 1] + if kind == "word": + # Delimiters undergo POSIX quote removal, not the + # expansions required of ordinary shell arguments. + try: + delimiter = shlex.split(logical[start:end])[0] + except ValueError: + continue # shell-specific quote decoding is outside this check + heredocs.append((delimiter, operator == "<<-")) + return lines + + +def dead_commands(text: str, tracked: frozenset[str]) -> list[tuple[int, str, str]]: + """``(first line, command, missing path)`` for each runnable command with no target. + + The command is the line from the interpreter word to the target word. A + dead ``-m`` command reports both paths that would make it runnable, joined + by ``or``, so the reader is not sent to create ``lab/a/b.py`` beside a + tracked ``lab/a/b/`` package that merely lacks ``__main__.py``. + """ + found = [] + for number, line in shell_lines(text): + for offset, source, match in python_commands(line): + target = command_target(source, match.end("interpreter")) + if target is None: + continue + kind, word, end = target + command = source[match.start("interpreter"):end] + if kind == "module": + if not MODULE_TARGET.fullmatch(word): + continue + candidates = runnable_candidates(word) + if not any(candidate in tracked for candidate in candidates): + found.append((number + offset, command, " or ".join(candidates))) + else: + if not word.endswith(".py"): + continue + script = posixpath.normpath(word) + if SCRIPT_TARGET.fullmatch(script) and script not in tracked: + found.append((number + offset, command, script)) + return found + + +UNDERSCORE_RUN = re.compile(r"_+") + + +def markdown_punctuation(character: str) -> bool: + """Whether ``character`` is punctuation to CommonMark: ASCII punctuation, or Unicode P or S.""" + return character in string.punctuation or unicodedata.category(character)[0] in "PS" + + +def underscore_emphasis(line: str, root: re.Match[str]) -> tuple[re.Match[str], int] | None: + """The run closing the emphasis the underscores of ``root`` open, with the delimiters the pair spends. + + The underscore runs after the opener are paired as CommonMark 0.31 pairs + them. A run can close when it is right-flanking (its previous character + is neither whitespace nor punctuation, or is punctuation followed by + whitespace or punctuation) and, for `_`, not also left-flanking unless + followed by punctuation; it can open under the mirror image. A run + between letters or digits is neither and is text, so + `_lab.prism.process_telemetry_` closes at its end. A closer pairs with + the nearest unpaired opener, spending two delimiters when both runs have + two or more and one otherwise, and spends what it has left on the next + opener down: `___lab/prism/x___` spends three on its opener, + `_lab/prism/x and _y_` spends its closer on `_y` and leaves the opener + literal, and `_lab/prism/x and _y_ z_` closes it at the end. Code spans + are read before emphasis, so a run an odd number of backticks after the + opener is text. ``None`` when no run pairs with the opener, as for + `_lab/prism/x` alone: Markdown renders that underscore literally. + + One line is read, as far as the guard's line scan sees: an emphasis + closed on a later line is missed, the runs before the opener are not + read (a closer pairs with the nearest opener, so they take none from + it), and the rule of three for runs that both open and close, the `*` + delimiters and link and HTML boundaries are not followed. + """ + openers = [len(root.group("emphasis"))] # the token's opener first, nearer openers after it + start = root.end("emphasis") + for run in UNDERSCORE_RUN.finditer(line, start): + if line.count("`", start, run.start()) % 2: + continue + before = line[run.start() - 1] if run.start() else " " + after = line[run.end()] if run.end() < len(line) else " " + before_punctuation = markdown_punctuation(before) + after_punctuation = markdown_punctuation(after) + left = not after.isspace() and (not after_punctuation or before.isspace() or before_punctuation) + right = not before.isspace() and (not before_punctuation or after.isspace() or after_punctuation) + remaining = run.end() - run.start() + spent = 0 + if right and (not left or after_punctuation): + while remaining and openers: + use = 2 if remaining >= 2 and openers[-1] >= 2 else 1 + openers[-1] -= use + remaining -= use + if len(openers) == 1: + spent += use + if not openers[-1]: + openers.pop() + if spent: + return run, spent + if remaining and left and (not right or before_punctuation): + openers.append(remaining) + return None + + +def prose_reference(line: str, match: re.Match[str], root: re.Match[str]) -> str | None: + """The path or module ``match`` names on ``line``, where the token root ``root`` precedes it. + + A quoted literal is kept whole. A bare token loses its trailing prose + punctuation and, when ``root`` opened underscore emphasis, the run that + closes it (``underscore_emphasis``), by the delimiters the pair spends: + `_lab/prism/x_` names `lab/prism/x`, `_lab/prism/tool__` names + `lab/prism/tool_`, and `_lab/prism/x and more_` closes past the token, + which is then read whole. ``None`` when the underscores open no + emphasis: `_lab/prism/x` renders literally and names another path. + """ + reference = match.group("literal") or match.group("bare") + if root.group("emphasis"): + if (closed := underscore_emphasis(line, root)) is None: + return None + closer, spent = closed + if match.group("bare") and closer.start() < match.end("bare"): + # The closer ends the token, behind at most trailing punctuation + # and the closers of emphasis opened further out, as in + # `(_lab/prism/x_)`, `**_lab/prism/x_**` and `_(_lab/prism/x_)_`. + if not line[closer.end() : match.end("bare")].strip(PROSE_TRAILING_PUNCTUATION + "_"): + reference = line[match.start("bare") : closer.start()] + closer.group()[spent:] + if match.group("bare"): + reference = reference.rstrip(PROSE_TRAILING_PUNCTUATION) + return reference + + +def dangling_references(text: str, tracked: frozenset[str]) -> list[tuple[int, str]]: + """``(line, reference)`` for each ``lab/prism`` mention that resolves to nothing.""" + found = [] + for number, line in enumerate(text.splitlines(), 1): + for match in PATH_REFERENCE.finditer(line): + prefix = line[: match.start()] + if PINNED_GITHUB_URL.search(prefix) or (root := PATH_ROOT_PREFIX.search(prefix)) is None: + continue + if (reference := prose_reference(line, match, root)) is None: + continue + if match.group("bare") and URL_REFERENCE_PREFIX.search(prefix): + reference = re.split(r"[?#]", reference, maxsplit=1)[0] + normalized = posixpath.normpath(reference) + if normalized == "lab" or normalized.startswith("lab/"): + if reference.endswith("/"): + if any(path.startswith(normalized + "/") for path in tracked): + continue + elif normalized in tracked: + continue + found.append((number, reference)) + for match in MODULE_REFERENCE.finditer(line): + prefix = line[: match.start("literal" if match.group("literal") else "bare")] + if (root := MODULE_ROOT_PREFIX.search(prefix)) is None: + continue + if (reference := prose_reference(line, match, root)) is None: + continue + if not any(c in tracked for c in module_candidates(reference)): + found.append((number, reference)) + return found + + +class DocsLabPrismReferenceTests(unittest.TestCase): + @classmethod + def setUpClass(cls) -> None: + cls.tracked = tracked_paths() + cls.docs = [path for path in documented(cls.tracked) if (ROOT / path).is_file()] + cls.texts = { + path: (ROOT / path).read_text(encoding="utf-8", errors="replace") + for path in cls.docs + } + + def test_no_doc_command_invokes_a_missing_module_or_script(self) -> None: + dead = [ + f"{path}:{number}: {command} -> {missing}" + for path in self.docs + for number, command, missing in dead_commands(self.texts[path], self.tracked) + ] + self.assertEqual( + dead, + [], + "runnable commands in docs/ invoke modules or scripts absent from the branch:\n" + + "\n".join(dead), + ) + + def test_lab_prism_references_resolve_outside_the_ratchet(self) -> None: + dangling = [ + f"{path}:{number}: {reference}" + for path in self.docs + if path not in RATCHET + for number, reference in dangling_references(self.texts[path], self.tracked) + ] + self.assertEqual( + dangling, + [], + "docs/ name lab/prism paths that do not exist on this branch; describe the " + "retired Python lane in words or link a commit-pinned GitHub URL:\n" + + "\n".join(dangling), + ) + + def test_ratcheted_files_do_not_gain_dangling_references(self) -> None: + for path, ceiling in RATCHET.items(): + with self.subTest(path=path): + if path not in self.texts: + self.fail(f"{path} is no longer tracked; remove its RATCHET entry") + found = dangling_references(self.texts[path], self.tracked) + if len(found) > ceiling: + self.fail( + f"{path} gained dangling lab/prism references ({len(found)} > " + f"{ceiling}); repair them rather than raising the ceiling:\n" + + "\n".join(f"{path}:{n}: {ref}" for n, ref in found) + ) + if len(found) < ceiling: + self.fail( + f"{path} improved ({len(found)} dangling references, ceiling " + f"{ceiling}); lower RATCHET[{path!r}] to {len(found)} in this test" + ) + + def test_capacity_readiness_is_not_ratcheted(self) -> None: + self.assertNotIn("docs/prism-capacity-readiness.md", RATCHET) + + def test_quoted_history_under_docs_is_not_read(self) -> None: + # A patch or diff quotes another tree's lines, so it is left out of + # both contracts; prose and data files of any other suffix stay in. + paths = { + "docs", + "docs/prism-alert-rules-qbit-tools.patch", + "docs/prism-coordinator-refactor/history.diff", + "docs/prism-capacity-readiness.md", + "docs/prism-coordinator-refactor/dashboard.json", + "docs/prism-alerts.yaml", + "lab/prism/Dockerfile", + "tests/history.patch", + } + self.assertEqual( + documented(paths), + [ + "docs/prism-alerts.yaml", + "docs/prism-capacity-readiness.md", + "docs/prism-coordinator-refactor/dashboard.json", + ], + ) + for path in self.docs: + self.assertNotIn(Path(path).suffix, QUOTED_HISTORY_SUFFIXES, path) + + +class ScannerTests(unittest.TestCase): + TRACKED = frozenset({"lab", "lab/prism", "lab/prism/Dockerfile"}) + # What `dead_commands` reports for `-m lab.prism.process_telemetry`: either + # path would make the command runnable, so both are named. + TELEMETRY = "lab/prism/process_telemetry.py or lab/prism/process_telemetry/__main__.py" + + def references(self, text: str) -> list[str]: + return [ref for _, ref in dangling_references(text, self.TRACKED)] + + def commands(self, text: str) -> list[str]: + return [missing for _, _, missing in dead_commands(text, self.TRACKED)] + + def located(self, text: str) -> list[tuple[int, str]]: + return [(number, missing) for number, _, missing in dead_commands(text, self.TRACKED)] + + def test_bare_path_to_deleted_module_is_dangling(self) -> None: + self.assertEqual(self.references("Rendered by `lab/prism/metrics.py`."), ["lab/prism/metrics.py"]) + + def test_commit_pinned_github_url_is_exempt(self) -> None: + sha = "504846cc0b72e8f86ed17f896d4ccbbe196a31dc" + text = f"[2.x](https://github.com/o/r/blob/{sha}/lab/prism/observability.py#L317)." + self.assertEqual(self.references(text), []) + self.assertEqual( + self.references("https://github.com/o/r/blob/main/lab/prism/observability.py"), + ["lab/prism/observability.py"], + ) + + def test_existing_paths_resolve_including_sentence_ending_period(self) -> None: + self.assertEqual(self.references("Only `lab/prism` holds lab/prism/Dockerfile."), []) + self.assertEqual(self.references("The module `lab.prism` remains."), []) + + def test_dotted_module_to_deleted_file_is_dangling(self) -> None: + self.assertEqual(self.references("see lab.prism.process_telemetry"), ["lab.prism.process_telemetry"]) + + def test_prose_references_do_not_resolve_truncated_prefixes(self) -> None: + tracked = self.TRACKED | {"lab/prism/tool.py"} + for suffix in ( + "$old", "-deleted", "+old", "=old", "é", r"\old", "/old", + "!old", "*old", "?old", "#old", ",old", ";old", ":old", "(old", "[old", "{old", "|old", + ): + for reference, path in ( + (f"lab/prism/Dockerfile{suffix}", f"lab/prism/Dockerfile{suffix}"), + (f"lab.prism.tool{suffix}", f"lab/prism/tool{suffix}.py"), + ): + with self.subTest(reference=reference): + text = f"See {reference}." + self.assertEqual(dangling_references(text, tracked), [(1, reference)]) + self.assertEqual(dangling_references(text, tracked | {path}), []) + + def test_quoted_prose_references_preserve_the_complete_literal(self) -> None: + tracked = self.TRACKED | {"lab/prism/tool.py"} + for suffix in (" old", "$old", "-deleted", ".", ",old", "#old", "?old", "[old]", "(old)", "*old", "`old", "](old", "][old"): + for reference, path in ( + (f"lab/prism/Dockerfile{suffix}", f"lab/prism/Dockerfile{suffix}"), + (f"lab.prism.tool{suffix}", f"lab/prism/tool{suffix.replace('.', '/')}.py"), + ): + for quote in ("`", "'", '"'): + if quote in reference: + continue + with self.subTest(reference=reference, quote=quote): + text = f"See {quote}{reference}{quote}." + self.assertEqual(dangling_references(text, tracked), [(1, reference)]) + self.assertEqual(dangling_references(text, tracked | {path}), []) + + def test_bare_prose_references_stop_at_surrounding_punctuation(self) -> None: + tracked = self.TRACKED | {"lab/prism/tool.py"} + for reference in ("lab/prism/Dockerfile", "lab.prism.tool"): + for surrounding in ( + "{}.", "{},", "{};", "{}:", "{}!", "{}?", "({})", "[{}]", "<{}>", "**{}**", + "[{}](https://example.com)", "[{}][build]", + ): + with self.subTest(reference=reference, surrounding=surrounding): + self.assertEqual(dangling_references(surrounding.format(reference), tracked), []) + self.assertEqual( + dangling_references(surrounding.format(reference + "$old"), tracked), + [(1, reference + "$old")], + ) + self.assertEqual( + dangling_references("https://github.com/o/r/blob/main/lab/prism/Dockerfile#L1", tracked), + [], + ) + + def test_quoted_directory_references_require_a_tracked_descendant(self) -> None: + self.assertEqual(self.references("See `lab/prism/`."), []) + self.assertEqual(self.references("See `lab/prism/Dockerfile/`."), ["lab/prism/Dockerfile/"]) + + def test_prose_references_do_not_include_unrelated_namespaces(self) -> None: + for reference in ("lab/prismatic/tool", "lab.prismatic.tool", "lab/prism-old/tool", "lab.prism-old.tool"): + with self.subTest(reference=reference): + self.assertEqual(self.references(f"See `{reference}`."), []) + + def test_prose_reference_roots_require_a_leading_word_boundary(self) -> None: + for root in ("lab/prism/deleted.py", "lab.prism.deleted"): + for prefix in ("col", "my_", "3", "é"): + for quote in ("", "`", "'", '"'): + with self.subTest(root=root, prefix=prefix, quote=quote): + self.assertEqual(self.references(f"See {quote}{prefix}{root}{quote}."), []) + + def test_module_references_start_at_a_token_root(self) -> None: + root = "lab.prism.deleted" + for text in ( + f"See {root}.", f"See `{root}`.", f"See '{root}'.", f'See "{root}".', f"{root} is gone.", + f"[old]({root})", f"[{root}](https://example.com)", f"[{root}][old]", f"<{root}>", f"**{root}**", + f"|{root}|", f"[old](<{root}>)", f"(`{root}`)", f'["{root}"]', f'See `"{root}"`.', + f"Don't lose the students' `{root}`.", f"It's {root}, isn't it?", f"Run `python3 -m {root}`.", + f"(**`{root}`**)", f"from {root} import main", f"Run `python3 -m $'{root}'`.", f'See $"{root}".', + ): + with self.subTest(text=text): + self.assertEqual(self.references(text), [root]) + self.assertEqual(dangling_references(text, self.TRACKED | {"lab/prism/deleted.py"}), []) + for prefix in ( + "vendor.", "a.", "_", "my-", "my*", "my(", "my=", "my[", "my|", "my<", "my>", "--module=", "vendor/", + "/tmp/", "./", "../", "/", "~/", "$HOME/", "@", "+", "C:\\", "https://example.com/", + "https://example.com/?m=", "https://github.com/o/r/blob/main/", + ): + for quote in ("", "`", "'", '"'): + for text in (f"See {quote}{prefix}{root}{quote}.", f"See {prefix}{quote}{root}{quote}."): + with self.subTest(text=text): + self.assertEqual(self.references(text), []) + for text in (f"See ${root}.", f"See `${root}`.", f"See my$'{root}'."): + with self.subTest(text=text): + self.assertEqual(self.references(text), []) + for text in ( + f"See `vendor.{root}` and `{root}`.", f"See https://example.com/{root} and {root}.", + f'See vendor."{root}" and "{root}".', f"[old](https://example.com/{root}) and [{root}](x)", + ): + with self.subTest(text=text): + self.assertEqual(self.references(text), [root]) + + def test_path_references_start_at_the_repository_root(self) -> None: + root = "lab/prism/deleted.py" + for text in ( + f"See {root}.", f"See `{root}`.", f"See '{root}'.", f'See "{root}".', f"See `./{root}`.", + f'See "./{root}".', f"{root} is gone.", f"[old]({root})", f"[old](./{root})", f"<{root}>", + f"**{root}**", f"|{root}|", f"[old](<{root}>)", f"(`{root}`)", f'["{root}"]', f'See `"{root}"`.', + f"https://github.com/o/r/blob/main/{root}#L1", f"https://github.com/o/r/tree/main/{root}", + f"https://github.com/o/r/raw/main/{root}", f"See `https://github.com/o/r/blob/main/{root}`.", + f'"https://github.com/o/r/blob/main/{root}"', f"Don't lose the students' `{root}`.", + f"It's {root}, isn't it?", f"Run `echo {root}`.", f"Run `python3 {root}`.", f"(**`{root}`**)", + ): + with self.subTest(text=text): + self.assertEqual(self.references(text), [root]) + self.assertEqual(dangling_references(text, self.TRACKED | {root}), []) + for prefix in ( + "my-", "my*", "my(", "my=", "my[", "my|", "my<", "my>", "--out=", "vendor/", "/tmp/", "../", "/", "~/", + "a.", "$HOME/", "@", "+", "C:\\", "my-\"", "https://example.com/", + "https://github.com/o/r/blob/main/vendor/", + ): + for quote in ("", "`", "'", '"'): + with self.subTest(prefix=prefix, quote=quote): + self.assertEqual(self.references(f"See {quote}{prefix}{root}{quote}."), []) + for text in (f"See `my*(**{root}`.", f"See `my[`{root}`.", f"See `x](my*{root}`."): + with self.subTest(text=text): + self.assertEqual(self.references(text), []) + self.assertEqual(self.references(f"See `vendor/{root}` and `{root}`."), [root]) + self.assertEqual(self.references(f"The students' `my*{root}` and `{root}` differ."), [root]) + + def test_blockquote_markers_start_reference_tokens(self) -> None: + # A Markdown blockquote marker needs no space after it, so `>lab/prism/x` + # and the nested `>>lab/prism/x` name the root path as `> lab/prism/x` + # does. A `>` inside a token (`my>lab/prism`) opens nothing, as `my<` + # does not: an opener counts only at the start of the line or after + # whitespace. + for root in ("lab/prism/deleted.py", "lab.prism.deleted"): + for surrounding in ( + ">{}", ">>{}", "> >{}", " >{}", ">`{}`", ">'{}'", '>"{}"', ">**{}**", + ">(`{}`)", ">{} is gone.", ">>`{}`.", + ): + text = surrounding.format(root) + with self.subTest(text=text): + self.assertEqual(self.references(text), [root]) + self.assertEqual(dangling_references(text, self.TRACKED | {"lab/prism/deleted.py"}), []) + for surrounding in ("my>{}", "my>>{}", "x>>{}", "`my>{}`", "<b>{}", "->{}", "=>{}"): + text = surrounding.format(root) + with self.subTest(text=text): + self.assertEqual(self.references(text), []) + self.assertEqual(self.references(">./lab/prism/deleted.py"), ["lab/prism/deleted.py"]) + self.assertEqual(self.references("> ./lab/prism/deleted.py"), ["lab/prism/deleted.py"]) + + def test_closed_underscore_emphasis_starts_reference_tokens(self) -> None: + # Markdown renders `_lab/prism/x_`, `__lab/prism/x__` and + # `` _`lab/prism/x`_ `` with the underscores gone and the path + # emphasized, so the doc shows the root path, and the closing run is + # no part of it. An underscore no later run closes renders literally + # and names a path in `_lab/`, `my_lab` is another path, a run inside + # a word is text, and inside a code span nothing is emphasis + # (markdown-it-py 3.0.0 in CommonMark mode agrees on every spelling). + for root in ("lab/prism/deleted.py", "lab.prism.deleted"): + for surrounding in ( + "_{}_", "__{}__", "___{}___", "_{}_.", "See _{}_.", "See _{}_, then", "The old _{}_ file", + "_`{}`_", "__`{}`__", "_'{}'_", '_"{}"_', "(_{}_)", "_({})_", "**_{}_**", "_**{}**_", "[_{}_](x)", + "> _{}_", ">_{}_", "> The old _{}_ file.", "_{} is gone_", "_{} and `more`_", "_{}_ and _more_", + "_a _{}_", "__{}_", "_{} _y_ z_", "_{}_'s", + ): + text = surrounding.format(root) + with self.subTest(text=text): + self.assertEqual(self.references(text), [root]) + self.assertEqual(dangling_references(text, self.TRACKED | {"lab/prism/deleted.py"}), []) + for surrounding in ( + "_{}", "_{}.", "See _{}.", "__{}", "`_{}`", "`_{}_`", "my_{}", "`my_{}`", "_{}_y", "_{} and _y_", + "_{} and `tool_`", "_{} then_more", "[x](_{}_)", + ): + text = surrounding.format(root) + with self.subTest(text=text): + self.assertEqual(self.references(text), []) + self.assertEqual(self.references("_lab.prism.process_telemetry_"), ["lab.prism.process_telemetry"]) + # A path may end in `_`, which no trailing-punctuation rule may trim: + # only the run that closes the emphasis the token opened goes, by the + # delimiters the pair spends, so `_lab/prism/tool__` renders and names + # `lab/prism/tool_` as `lab/prism/tool_` does. + tracked = self.TRACKED | {"lab/prism/tool_"} + for text in ( + "See lab/prism/tool_.", "See `lab/prism/tool_`.", "lab/prism/tool_", "_lab/prism/tool__", + "__lab/prism/tool___", "_`lab/prism/tool_`_", "See lab.prism.tool_.", "_lab.prism.tool__", + ): + with self.subTest(text=text): + self.assertEqual(dangling_references(text, tracked), []) + self.assertEqual(dangling_references("See lab/prism/tool_.", self.TRACKED), [(1, "lab/prism/tool_")]) + self.assertEqual(dangling_references("_lab/prism/tool__", self.TRACKED), [(1, "lab/prism/tool_")]) + + def test_module_commands_with_missing_targets_are_caught(self) -> None: + self.assertEqual( + self.commands("python3 -m lab.prism.process_telemetry rss-bound --samples s.csv \\"), + [self.TELEMETRY], + ) + self.assertEqual( + self.commands('docker exec "$c" python3 -m lab.prism.process_telemetry rss-sample --pid 1'), + [self.TELEMETRY], + ) + self.assertEqual(self.commands("python -u -m lab.tool run"), ["lab/tool.py or lab/tool/__main__.py"]) + self.assertEqual(self.commands("python3 lab/prism/storm.py --decide"), ["lab/prism/storm.py"]) + + def test_commands_with_existing_targets_pass(self) -> None: + tracked = self.TRACKED | {"lab/prism/tool.py", "lab/pkg/__init__.py", "lab/pkg/__main__.py"} + text = "python3 -m lab.prism.tool\npython -m lab.pkg\npython3 lab/prism/tool.py" + self.assertEqual(dead_commands(text, tracked), []) + + def test_non_identifier_module_targets_are_caught(self) -> None: + for module in ( + "lab.prism.deleted-module", + "lab.prism.123module", + "lab.prism.-module", + "lab.prism.café", + "lab.prism.module name", + "lab.prism.module+name", + "lab.prism.pkg-name.deleted-module", + ): + relative = module.replace(".", "/") + missing = f"{relative}.py or {relative}/__main__.py" + for option in (f"-m '{module}'", f'-m "{module}"', f"-OOm'{module}'"): + with self.subTest(module=module, option=option): + command = f"python3 {option}" + self.assertEqual(dead_commands(command, self.TRACKED), [(1, command, missing)]) + self.assertEqual(dead_commands(command, self.TRACKED | {f"{relative}.py"}), []) + self.assertEqual(dead_commands(command, self.TRACKED | {f"{relative}/__main__.py"}), []) + tracked = self.TRACKED | {relative, f"{relative}/__init__.py"} + self.assertEqual(dead_commands(command, tracked), [(1, command, missing)]) + + def test_hyphenated_module_spellings_report_the_complete_target(self) -> None: + missing = "lab/prism/deleted-module.py or lab/prism/deleted-module/__main__.py" + # An existing identifier prefix must not make the full target pass. + tracked = self.TRACKED | {"lab/prism/deleted.py"} + for command in ( + "python3 -m lab.prism.deleted-module", + "python3 -mlab.prism.deleted-module", + 'python3 -m "lab.prism."deleted-module', + "python3 -m $'lab.prism.deleted-module'", + ): + with self.subTest(command=command): + self.assertEqual(dead_commands(command, tracked), [(1, command, missing)]) + text = "```bash\npython3 -m \\\n lab.prism.deleted-module --help\n```" + self.assertEqual(self.located(text), [(2, missing)]) + + def test_hyphenated_command_cannot_hide_within_reference_ratchet(self) -> None: + prose = "The retired module `lab.prism.deleted` is historical." + command = "python3 -m lab.prism.deleted-module" + self.assertEqual(len(self.references(prose)), len(self.references(command))) + self.assertEqual( + self.commands(command), + ["lab/prism/deleted-module.py or lab/prism/deleted-module/__main__.py"], + ) + + def test_single_quoted_literal_module_targets_are_caught(self) -> None: + for name in ("deleted$module", r"deleted\module", "deleted`module"): + module = f"lab.prism.{name}" + relative = f"lab/prism/{name}" + missing = f"{relative}.py or {relative}/__main__.py" + for word in (f"'{module}'", f"lab.prism.'{name}'", f'"lab.prism."\'{name}\''): + for option in (f"-m {word}", f"-OOm{word}", f'"-m"{word}'): + with self.subTest(name=name, option=option): + command = f"python3 {option}" + self.assertEqual(dead_commands(command, self.TRACKED), [(1, command, missing)]) + self.assertEqual(dead_commands(command, self.TRACKED | {f"{relative}.py"}), []) + self.assertEqual(dead_commands(command, self.TRACKED | {f"{relative}/__main__.py"}), []) + tracked = self.TRACKED | {"lab/prism/deleted.py", f"{relative}/__init__.py"} + self.assertEqual(dead_commands(command, tracked), [(1, command, missing)]) + + def test_other_quotes_preserve_only_literal_module_characters(self) -> None: + for word, name in ( + ("$'lab.prism.deleted$module'", "deleted$module"), + ("lab.prism.$'deleted$module'", "deleted$module"), + (r'"lab.prism.deleted\module"', r"deleted\module"), + (r'$"lab.prism.deleted\module"', r"deleted\module"), + ): + for option in (f"-m {word}", f"-OOm{word}"): + with self.subTest(option=option): + command = f"python3 {option}" + relative = f"lab/prism/{name}" + missing = f"{relative}.py or {relative}/__main__.py" + self.assertEqual(dead_commands(command, self.TRACKED), [(1, command, missing)]) + self.assertEqual(dead_commands(command, self.TRACKED | {f"{relative}.py"}), []) + + def test_module_expansions_and_escapes_are_not_literal_targets(self) -> None: + for word in ( + "lab.prism.deleted$module", + '"lab.prism.deleted$module"', + '$"lab.prism.deleted$module"', + r"lab.prism.deleted\module", + r"$'lab.prism.deleted\module'", + r'"lab.prism.deleted\\module"', + r'"lab.prism.deleted\$module"', + r'"lab.prism.deleted\"', + '"lab.prism.deleted`module`"', + "'lab.prism.deleted$module'$SUFFIX", + "'lab.prism.deleted'\"$MODULE\"", + ): + for option in (f"-m {word}", f"-OOm{word}"): + with self.subTest(option=option): + self.assertEqual(self.commands(f"python3 {option}"), []) + + def test_wrapped_literal_module_targets_keep_the_first_line(self) -> None: + for name in ("deleted$module", r"deleted\module"): + relative = f"lab/prism/{name}" + text = f"```bash\npython3 -m \\\n 'lab.prism.{name}' --help\n```" + self.assertEqual(self.located(text), [(2, f"{relative}.py or {relative}/__main__.py")]) + + def test_literal_module_command_cannot_hide_within_reference_ratchet(self) -> None: + prose = "The retired module `lab.prism.deleted` is historical." + for name in ("deleted$module", r"deleted\module"): + command = f"python3 -m 'lab.prism.{name}'" + with self.subTest(command=command): + self.assertEqual(len(self.references(prose)), len(self.references(command))) + relative = f"lab/prism/{name}" + self.assertEqual(self.commands(command), [f"{relative}.py or {relative}/__main__.py"]) + + def test_quoted_literal_script_targets_are_caught(self) -> None: + for name in ("deleted$file", r"deleted\file", "deleted`file", "deleted file", "déleted", "deleted[ab]*?{x,y}"): + path = f"lab/prism/{name}.py" + for word in (f"'{path}'", f"./lab/prism/'{name}'.py", f'"./lab/prism/"\'{name}.py\''): + for options in ("", "-OO ", "-- ", "-O -- "): + with self.subTest(word=word, options=options): + command = f"python3 {options}{word}" + self.assertEqual(dead_commands(command, self.TRACKED), [(1, command, path)]) + self.assertEqual(dead_commands(command, self.TRACKED | {path}), []) + + def test_other_quotes_preserve_only_literal_script_characters(self) -> None: + for word, path in ( + ("$'lab/prism/deleted$file.py'", "lab/prism/deleted$file.py"), + (r'"lab/prism/deleted\file.py"', r"lab/prism/deleted\file.py"), + (r'$"lab/prism/deleted\file.py"', r"lab/prism/deleted\file.py"), + ('"lab/prism/deleted*.py"', "lab/prism/deleted*.py"), + ): + for options in ("", "-- "): + with self.subTest(word=word, options=options): + command = f"python3 {options}{word}" + self.assertEqual(dead_commands(command, self.TRACKED), [(1, command, path)]) + self.assertEqual(dead_commands(command, self.TRACKED | {path}), []) + + def test_script_expansions_and_escapes_are_not_literal_targets(self) -> None: + for word in ( + "lab/prism/deleted$file.py", '"lab/prism/deleted$file.py"', + r"lab/prism/deleted\file.py", r"$'lab/prism/deleted\file.py'", + r'"lab/prism/deleted\\file.py"', r'"lab/prism/deleted\$file.py"', + '"lab/prism/deleted`file`.py"', "lab/prism/deleted*.py", + "lab/prism/deleted?.py", "lab/prism/deleted[ab].py", "lab/prism/deleted{a,b}.py", + "'lab/prism/deleted'$FILE.py", + ): + for options in ("", "-- "): + with self.subTest(word=word, options=options): + self.assertEqual(self.commands(f"python3 {options}{word}"), []) + + def test_wrapped_literal_script_cannot_hide_within_reference_ratchet(self) -> None: + path = "lab/prism/deleted$file.py" + command = f"python3 -- \\\n '{path}'" + prose = "The retired script `lab/prism/deleted.py` is historical." + self.assertEqual(len(self.references(prose)), len(self.references(command))) + self.assertEqual(self.located(f"```sh\n{command}\n```"), [(2, path)]) + + # Verified on CPython 3.14: `python3 -m lab.pkg` with only `lab/pkg/__init__.py`, + # or with a bare `lab/pkg/` directory, prints "No module named + # lab.pkg.__main__; 'lab.pkg' is a package and cannot be directly executed" + # and exits 1; with `lab/pkg/__main__.py` it runs, and `python3 -m + # lab.auxpow.vardiff` runs `lab/auxpow/vardiff.py` with no `lab/auxpow/__init__.py`. + def test_module_command_on_init_only_package_is_dead(self) -> None: + tracked = self.TRACKED | {"lab/pkg", "lab/pkg/__init__.py"} + self.assertEqual( + dead_commands("python3 -m lab.pkg run", tracked), + [(1, "python3 -m lab.pkg", "lab/pkg.py or lab/pkg/__main__.py")], + ) + + def test_module_command_on_namespace_directory_is_dead(self) -> None: + tracked = self.TRACKED | {"lab/auxpow", "lab/auxpow/vardiff.py"} + self.assertEqual( + dead_commands("python3 -m lab.auxpow", tracked), + [(1, "python3 -m lab.auxpow", "lab/auxpow.py or lab/auxpow/__main__.py")], + ) + + def test_module_command_on_package_with_main_passes(self) -> None: + tracked = self.TRACKED | {"lab/pkg", "lab/pkg/__init__.py", "lab/pkg/__main__.py"} + self.assertEqual(dead_commands("python3 -m lab.pkg run", tracked), []) + self.assertEqual(dead_commands("python3 -m lab.pkg", self.TRACKED | {"lab/pkg/__main__.py"}), []) + + def test_module_command_on_file_inside_namespace_package_passes(self) -> None: + tracked = self.TRACKED | {"lab/auxpow", "lab/auxpow/vardiff.py"} + self.assertEqual(dead_commands("python3 -m lab.auxpow.vardiff --help", tracked), []) + + def test_prose_mention_of_init_only_package_is_not_dangling(self) -> None: + # The prose contract asks whether the name exists, not whether it runs: + # `lab.prism.pkg` still resolves through `.py`, `__init__.py` or the directory. + tracked = self.TRACKED | {"lab/prism/pkg", "lab/prism/pkg/__init__.py"} + self.assertEqual(dangling_references("see lab.prism.pkg", tracked), []) + self.assertEqual(dangling_references("see lab.prism.bare", self.TRACKED | {"lab/prism/bare"}), []) + self.assertEqual(dangling_references("see lab.prism.gone", tracked), [(1, "lab.prism.gone")]) + + def test_wrapped_module_commands_with_missing_targets_are_caught(self) -> None: + for wrapped in ( + "python3 -m \\\n lab.prism.process_telemetry rss-bound", + "python3 \\\n -m lab.prism.process_telemetry rss-bound", + "python3 -u \\\n -m lab.prism.process_telemetry \\\n rss-bound", + ): + with self.subTest(wrapped=wrapped): + text = f"```bash\ncd repo\n{wrapped} \\\n --samples s.csv\n```" + self.assertEqual(self.located(text), [(3, self.TELEMETRY)]) + + def test_wrapped_script_command_with_missing_target_is_caught(self) -> None: + text = "python3 \\\n lab/prism/storm.py \\\n --decide" + self.assertEqual(self.located(text), [(1, "lab/prism/storm.py")]) + + def test_wrapped_commands_with_existing_targets_pass(self) -> None: + tracked = self.TRACKED | {"lab/prism/tool.py", "lab/pkg/__init__.py", "lab/pkg/__main__.py"} + text = "python3 -m \\\n lab.prism.tool\npython \\\n -m lab.pkg\npython3 \\\n lab/prism/tool.py" + self.assertEqual(dead_commands(text, tracked), []) + + # Every option form `python3 --help` accepts ahead of `-m` or a script path. + OPTIONS = ( + "-OO", + "-bb -u", + "-IsE", + "-R", + "-ER -X dev", + "-X dev", + "-Xdev", + "-W error", + "-uWerror", + "-X importtime=2", + "--check-hash-based-pycs always", + ) + + def test_commands_with_real_option_forms_and_missing_targets_are_caught(self) -> None: + for options in self.OPTIONS: + with self.subTest(options=options): + self.assertEqual( + self.commands(f"python3 {options} -m lab.prism.process_telemetry rss-bound"), + [self.TELEMETRY], + ) + self.assertEqual( + self.commands(f"python {options} lab/prism/storm.py --decide"), + ["lab/prism/storm.py"], + ) + + def test_commands_with_real_option_forms_and_existing_targets_pass(self) -> None: + tracked = self.TRACKED | {"lab/prism/tool.py", "lab/pkg/__init__.py", "lab/pkg/__main__.py"} + for options in self.OPTIONS: + with self.subTest(options=options): + text = ( + f"python3 {options} -m lab.prism.tool\npython3.12 {options} -m lab.pkg\n" + f"python {options} lab/prism/tool.py" + ) + self.assertEqual(dead_commands(text, tracked), []) + + def test_wrapped_option_forms_with_missing_targets_are_caught(self) -> None: + for options in self.OPTIONS: + wrapped = options.replace(" ", " \\\n ") + with self.subTest(options=options): + text = f"python3 \\\n {wrapped} \\\n -m lab.prism.process_telemetry \\\n rss-bound" + self.assertEqual(self.located(text), [(1, self.TELEMETRY)]) + text = f"python3.12 {wrapped} \\\n lab/prism/storm.py \\\n --decide" + self.assertEqual(self.located(text), [(1, "lab/prism/storm.py")]) + + def test_quoted_targets_with_missing_targets_are_caught(self) -> None: + for quote in ("'", '"'): + with self.subTest(quote=quote): + self.assertEqual( + self.commands(f"python3 -m {quote}lab.prism.process_telemetry{quote} rss-bound"), + [self.TELEMETRY], + ) + self.assertEqual( + self.commands(f"python3 {quote}lab/prism/storm.py{quote} --decide"), + ["lab/prism/storm.py"], + ) + + # Each ran `json.tool` on CPython 3.14 exactly as its bare spelling: the + # shell strips matching quotes, attached or not, before CPython sees them. + QUOTED_OPTIONS = ( + "-X 'dev'", + '-X "dev mode"', + "-X'dev'", + '-X"dev mode"', + "-W 'error'", + '-W"error"', + "-uW 'error'", + "-W 'error'::DeprecationWarning", + "--check-hash-based-pycs 'always'", + '--check-hash-based-pycs "never"', + ) + + # On CPython 3.14, `python3 -- s.py` and `python3 -O -- s.py` run the + # script, while `python3 -- -m json.tool` fails with "can't open file + # '.../-m'": after `--` the `-m` is a script path, not an option. + def test_terminated_script_commands_with_missing_targets_are_caught(self) -> None: + for options in ("--", "-OO --", "-R -X dev --"): + with self.subTest(options=options): + self.assertEqual( + self.commands(f"python3 {options} lab/prism/storm.py --decide"), ["lab/prism/storm.py"] + ) + self.assertEqual( + self.commands(f"python {options} './lab/prism/storm.py'"), ["lab/prism/storm.py"] + ) + + def test_terminated_script_commands_with_existing_targets_pass(self) -> None: + tracked = self.TRACKED | {"lab/prism/tool.py"} + text = ( + "python3 -- lab/prism/tool.py\npython3.12 -OO -- \"./lab/prism/tool.py\"\n" + "python -R -- lab/prism/tool.py" + ) + self.assertEqual(dead_commands(text, tracked), []) + + def test_wrapped_terminated_script_command_with_missing_target_is_caught(self) -> None: + text = "python3 -OO \\\n -- \\\n lab/prism/storm.py \\\n --decide" + self.assertEqual(self.located(text), [(1, "lab/prism/storm.py")]) + + def test_terminator_before_module_option_is_not_a_command(self) -> None: + # CPython treats everything after `--` as the script path and its + # arguments, so `-m` names a file called `-m` that does not exist and + # nothing runs; the prose contract still sees the module reference. + for text in ("python3 -- -m lab.prism.x", "python3 -OO -- -m lab.prism.x"): + with self.subTest(text=text): + self.assertEqual(self.commands(text), []) + self.assertEqual(self.references(text), ["lab.prism.x"]) + + def test_quoted_option_arguments_with_missing_targets_are_caught(self) -> None: + for options in self.QUOTED_OPTIONS: + with self.subTest(options=options): + self.assertEqual( + self.commands(f"python3 {options} -m lab.prism.process_telemetry rss-bound"), + [self.TELEMETRY], + ) + self.assertEqual( + self.commands(f"python {options} lab/prism/storm.py --decide"), + ["lab/prism/storm.py"], + ) + + def test_quoted_option_arguments_with_existing_targets_pass(self) -> None: + tracked = self.TRACKED | {"lab/prism/tool.py", "lab/pkg/__init__.py", "lab/pkg/__main__.py"} + for options in self.QUOTED_OPTIONS: + with self.subTest(options=options): + text = ( + f"python3 {options} -m lab.prism.tool\npython3.12 {options} -m lab.pkg\n" + f"python {options} lab/prism/tool.py" + ) + self.assertEqual(dead_commands(text, tracked), []) + + def test_wrapped_quoted_option_arguments_with_missing_targets_are_caught(self) -> None: + text = "python3 \\\n -X \\\n 'dev mode' \\\n -m lab.prism.process_telemetry \\\n rss-bound" + self.assertEqual(self.located(text), [(1, self.TELEMETRY)]) + text = "python3.12 --check-hash-based-pycs \\\n \"always\" \\\n lab/prism/storm.py \\\n --decide" + self.assertEqual(self.located(text), [(1, "lab/prism/storm.py")]) + + def test_mismatched_option_argument_quotes_are_not_a_command(self) -> None: + # bash rejects `-X 'dev"` with "unexpected EOF while looking for + # matching `''" before Python starts, so the line runs nothing; the + # prose contract still sees the reference. + for options in ("-X 'dev\"", "-X\"dev'", "-W \"error'", "--check-hash-based-pycs 'always\""): + with self.subTest(options=options): + text = f"python3 {options} -m lab.prism.x" + self.assertEqual(self.commands(text), []) + self.assertEqual(self.references(text), ["lab.prism.x"]) + self.assertEqual(self.commands(f"python3 {options} lab/prism/x.py"), []) + + def test_dot_relative_scripts_are_caught_as_the_bare_path(self) -> None: + for script in ("./lab/prism/storm.py", '"./lab/prism/storm.py"', "././lab/prism/storm.py"): + with self.subTest(script=script): + self.assertEqual(self.commands(f"python3 {script} --decide"), ["lab/prism/storm.py"]) + + def test_equivalent_script_paths_resolve_to_tracked_files(self) -> None: + tracked = self.TRACKED | {"lab/prism/tool.py"} + for script in ( + "lab/prism/../prism/tool.py", "lab/./prism/tool.py", "lab//prism/tool.py", + "././lab/prism/../prism//./tool.py", ".//lab/prism/tool.py", + ): + for word in (script, f"'{script}'", f'"{script}"'): + with self.subTest(word=word): + command = f"python3 -O -- {word} --help" + self.assertEqual(dead_commands(command, tracked), []) + self.assertEqual(dangling_references(command, tracked), []) + + def test_equivalent_missing_script_paths_keep_command_and_line(self) -> None: + for script in ( + "lab/prism/../prism/deleted.py", "lab/./prism/deleted.py", + "lab//prism/deleted.py", "./lab/prism/../prism//./deleted.py", + ): + with self.subTest(script=script): + text = f"```sh\npython3 -- \\\n '{script}' --help\n```" + self.assertEqual( + dead_commands(text, self.TRACKED), + [(2, f"python3 -- '{script}'", "lab/prism/deleted.py")], + ) + + def test_normalized_script_paths_outside_lab_are_not_lab_commands(self) -> None: + for script in ( + "lab/../scripts/tool.py", "lab/prism/../../scripts/tool.py", + "lab/../../lab/prism/tool.py", "/lab/prism/tool.py", + "lab/../lab-old/tool.py", + ): + with self.subTest(script=script): + self.assertEqual(self.commands(f"python3 '{script}'"), []) + + def test_equivalent_prose_paths_preserve_directory_and_lab_boundaries(self) -> None: + for reference in ("lab/prism/./Dockerfile", "lab/prism/../prism//Dockerfile", "lab/prism/../prism/"): + with self.subTest(reference=reference): + self.assertEqual(self.references(f"See `{reference}`."), []) + for reference in ( + "lab/prism/../prism/deleted.py", "lab/prism/../prism/Dockerfile/", + "lab/prism/../../scripts/tool.py", "lab/prism/../../../lab/prism/Dockerfile", + ): + with self.subTest(reference=reference): + self.assertEqual( + dangling_references(f"See `{reference}`.", self.TRACKED | {"scripts/tool.py"}), + [(1, reference)], + ) + + # Every `-m` spelling here ran `json.tool` on CPython 3.12 (`-Rm` on 3.14); + # `-Wm json.tool` and `-Xm json.tool` did not, opening `json.tool` as a + # script instead. + ADJACENT_MODULE_OPTIONS = ("-m{}", "-m'{}'", "-Im {}", "-OOm {}", "-Im{}", "-Rm {}") + + def test_adjacent_module_options_with_missing_targets_are_caught(self) -> None: + for option in self.ADJACENT_MODULE_OPTIONS: + with self.subTest(option=option): + text = f"python3 {option.format('lab.prism.process_telemetry')} rss-bound" + self.assertEqual(self.commands(text), [self.TELEMETRY]) + self.assertEqual(self.commands("python3 -Wm lab.prism.process_telemetry"), []) + self.assertEqual(self.commands("python3 -Xm lab.prism.process_telemetry"), []) + + def test_adjacent_module_options_with_existing_targets_pass(self) -> None: + tracked = self.TRACKED | {"lab/prism/tool.py", "lab/pkg/__init__.py", "lab/pkg/__main__.py"} + for option in self.ADJACENT_MODULE_OPTIONS: + with self.subTest(option=option): + text = f"python3 {option.format('lab.prism.tool')}\npython3.12 {option.format('lab.pkg')}" + self.assertEqual(dead_commands(text, tracked), []) + + def test_quoted_and_dot_relative_forms_with_existing_targets_pass(self) -> None: + tracked = self.TRACKED | {"lab/prism/tool.py", "lab/pkg/__init__.py", "lab/pkg/__main__.py"} + text = ( + "python3 -m 'lab.prism.tool'\npython -m \"lab.pkg\"\npython3 'lab/prism/tool.py'\n" + "python3 \"./lab/prism/tool.py\"\npython3 ./lab/prism/tool.py" + ) + self.assertEqual(dead_commands(text, tracked), []) + + def test_wrapped_quoted_targets_with_missing_targets_are_caught(self) -> None: + text = "python3 -m \\\n 'lab.prism.process_telemetry' \\\n rss-bound" + self.assertEqual(self.located(text), [(1, self.TELEMETRY)]) + text = "python3 \\\n \"./lab/prism/storm.py\" \\\n --decide" + self.assertEqual(self.located(text), [(1, "lab/prism/storm.py")]) + + # Each ran `json.tool` on bash 3.2 and CPython 3.14 exactly as the bare + # spelling: the shell joins the quoted and unquoted pieces of a word and + # strips the quotes before CPython sees one argument. + CONCATENATED_MODULES = ('"lab.prism."{}', "'lab.prism.'{}", 'lab."prism".{}', "lab.'prism'.{}", '"lab.prism".{}') + CONCATENATED_SCRIPTS = ( + '"lab/prism/"{}.py', + "'lab/prism/'{}.py", + 'lab/"prism"/{}.py', + '"./lab/prism/"{}.py', + '"./"./lab/prism/{}.py', + "./'lab/prism/{}.py'", + ) + + def test_concatenated_targets_with_missing_targets_are_caught_as_the_bare_target(self) -> None: + for module in self.CONCATENATED_MODULES: + with self.subTest(module=module): + text = f"python3 -m {module.format('process_telemetry')} rss-bound" + self.assertEqual(self.commands(text), [self.TELEMETRY]) + for script in self.CONCATENATED_SCRIPTS: + with self.subTest(script=script): + self.assertEqual(self.commands(f"python3 {script.format('storm')} --decide"), ["lab/prism/storm.py"]) + self.assertEqual( + self.commands('python3 -X "dev" -m "lab.prism."process_telemetry'), [self.TELEMETRY] + ) + + def test_concatenated_targets_with_existing_targets_pass(self) -> None: + tracked = self.TRACKED | {"lab/prism/tool.py", "lab/pkg/__init__.py", "lab/pkg/__main__.py"} + for module in self.CONCATENATED_MODULES: + with self.subTest(module=module): + self.assertEqual(dead_commands(f"python3 -m {module.format('tool')}", tracked), []) + for script in self.CONCATENATED_SCRIPTS: + with self.subTest(script=script): + self.assertEqual(dead_commands(f"python3 {script.format('tool')}", tracked), []) + self.assertEqual(dead_commands('python -m "lab".pkg run', tracked), []) + + def test_wrapped_concatenated_targets_with_missing_targets_are_caught(self) -> None: + text = "```bash\ncd repo\npython3 -m \\\n \"lab.prism.\"process_telemetry \\\n rss-bound\n```" + self.assertEqual(self.located(text), [(3, self.TELEMETRY)]) + text = "python3 \\\n \"./lab/prism/\"storm.py \\\n --decide" + self.assertEqual(self.located(text), [(1, "lab/prism/storm.py")]) + + # Each ran on bash 3.2 and CPython 3.14 exactly as the plain spelling: + # `python3 -m $'json.tool'`, `python3 -m $'json.'tool`, `python3 -m + # $"json.tool"`, `$'python3' -m json.tool`, `$"python3" -m json.tool`, + # `python3 $'lab/prism/'storm.py` and `python3 lab/prism/$'storm'.py`. + ANSI_C_MODULES = ("$'lab.prism.{}'", "$'lab.prism.'{}", '$"lab.prism.{}"', "lab.prism.$'{}'", "lab.$'prism'.{}") + ANSI_C_SCRIPTS = ("$'lab/prism/{}.py'", "$'lab/prism/'{}.py", '$"lab/prism/{}.py"', "lab/prism/$'{}'.py") + ANSI_C_INTERPRETERS = ("$'python3'", '$"python3"', "$'pyth'on3", "$'/usr/bin/'python3", "$'python3.12'") + + def test_ansi_c_quoted_words_with_missing_targets_are_caught_as_the_bare_target(self) -> None: + for module in self.ANSI_C_MODULES: + with self.subTest(module=module): + text = f"python3 -m {module.format('process_telemetry')} rss-bound" + self.assertEqual(self.commands(text), [self.TELEMETRY]) + for script in self.ANSI_C_SCRIPTS: + with self.subTest(script=script): + self.assertEqual(self.commands(f"python3 {script.format('storm')} --decide"), ["lab/prism/storm.py"]) + for interpreter in self.ANSI_C_INTERPRETERS: + with self.subTest(interpreter=interpreter): + self.assertEqual( + self.commands(f"{interpreter} -m lab.prism.process_telemetry rss-bound"), [self.TELEMETRY] + ) + self.assertEqual(self.commands(f"{interpreter} lab/prism/storm.py --decide"), ["lab/prism/storm.py"]) + self.assertEqual( + self.commands("$'python3' -X $'dev' -m $'lab.prism.'process_telemetry"), [self.TELEMETRY] + ) + self.assertEqual( + dead_commands("$'python3' -m $'lab.prism.'process_telemetry", self.TRACKED), + [(1, "$'python3' -m $'lab.prism.'process_telemetry", self.TELEMETRY)], + ) + + def test_ansi_c_quoted_words_with_existing_targets_pass(self) -> None: + tracked = self.TRACKED | {"lab/prism/tool.py", "lab/pkg/__init__.py", "lab/pkg/__main__.py"} + for module in self.ANSI_C_MODULES: + with self.subTest(module=module): + self.assertEqual(dead_commands(f"python3 -m {module.format('tool')}", tracked), []) + for script in self.ANSI_C_SCRIPTS: + with self.subTest(script=script): + self.assertEqual(dead_commands(f"python3 {script.format('tool')}", tracked), []) + for interpreter in self.ANSI_C_INTERPRETERS: + with self.subTest(interpreter=interpreter): + self.assertEqual(dead_commands(f"{interpreter} -m lab.prism.tool", tracked), []) + self.assertEqual(dead_commands(f"{interpreter} lab/prism/tool.py", tracked), []) + self.assertEqual(dead_commands("python -m $'lab'.pkg run", tracked), []) + + def test_ansi_c_escapes_are_not_followed(self) -> None: + # bash decodes `\x2e` to `.` and runs `python3 -m $'lab\x2eprism.x'`, + # but backslash escapes are outside this check (module docstring): the + # backslash stays in the word, which then names no target and no + # interpreter. An unterminated `$'` is a syntax error like a bare one; + # a `$` before anything but a quote is an ordinary character, so a + # variable is still not a command. + for text in ( + "python3 -m $'lab\\x2eprism.process_telemetry' rss-bound", + "python3 $'lab/prism/storm\\x2epy' --decide", + "$'pyth\\x6fn3' -m lab.prism.process_telemetry", + "python3 -m $'lab.prism.process_telemetry\\'' rss-bound", + "python3 -m $'lab.prism.x\"", + "python3 $\"lab/prism/x.py'", + "python3 -m $MODULE rss-bound", + "python3 -m $MODULE'.x'", + "python3 -m lab.prism.$MODULE", + "python3 -m lab.prism.x$", + ): + with self.subTest(text=text): + self.assertEqual(self.commands(text), []) + self.assertEqual(self.references("python3 -m $'lab.prism.x\""), ["lab.prism.x"]) + + def test_wrapped_ansi_c_quoted_words_are_caught_at_the_right_line(self) -> None: + text = "```bash\ncd repo\npython3 -m \\\n $'lab.prism.'process_telemetry \\\n rss-bound\n```" + self.assertEqual(self.located(text), [(3, self.TELEMETRY)]) + text = "$'python3' \\\n $\"lab/prism/storm.py\" \\\n --decide" + self.assertEqual(self.located(text), [(1, "lab/prism/storm.py")]) + + def test_words_that_name_no_lab_target_are_not_a_command(self) -> None: + # `python3 -m "lab.prism."` hands CPython `lab.prism.`, which fails with + # "No module named lab.prism." and runs nothing; a variable, another + # package or a name outside the grammar is not a `lab` command either. + for text in ( + 'python3 -m "lab.prism."', + 'python3 -m "lab.prism." rss-bound', + "python3 -m 'lab.prism.'", + "python3 -m $MODULE rss-bound", + 'python3 -m "$MODULE"', + 'python3 -m "json."tool', + "python3 -m lab.prism..x", + "python3 -m lab.prism.x.", + 'python3 "$SCRIPT" --decide', + 'python3 "lab/prism/"storm --decide', + ): + with self.subTest(text=text): + self.assertEqual(self.commands(text), []) + + def test_target_words_end_at_shell_metacharacters_and_inline_code(self) -> None: + # bash ends a word at `;`, `|`, `&`, `(`, `)`, `<` and `>` (`python3 -m + # json.tool;` ran on bash 3.2), and a backtick closes the inline code a + # command sits in, so each of these still runs the missing target. + for text in ( + "`python3 -m lab.prism.process_telemetry`", + "python3 -m lab.prism.process_telemetry; echo done", + "python3 -m lab.prism.process_telemetry|tee log", + "python3 -m lab.prism.process_telemetry>>log", + "python3 -m lab.prism.process_telemetry&", + "$(python3 -m lab.prism.process_telemetry)", + ): + with self.subTest(text=text): + self.assertEqual(self.commands(text), [self.TELEMETRY]) + self.assertEqual(self.commands("`python3 lab/prism/storm.py`; $(python3 lab/prism/storm.py)"), + ["lab/prism/storm.py", "lab/prism/storm.py"]) + + def test_target_words_end_at_the_close_of_an_enclosing_shell_string(self) -> None: + # Inside `sh -c "..."` the quote after the target closes the enclosing + # string, not a string the target opened, so the inner command is read. + self.assertEqual( + self.commands('docker exec "$c" sh -c "python3 -m lab.prism.process_telemetry"'), + [self.TELEMETRY], + ) + self.assertEqual(self.commands("sh -c 'python3 lab/prism/storm.py'"), ["lab/prism/storm.py"]) + + def test_mismatched_quotes_are_not_a_command(self) -> None: + # The shell rejects an unterminated quote before anything runs, so the + # line is not a runnable command and is left to the prose contract, + # which still sees the reference, as it does inside `-c` strings. + text = "python3 -m 'lab.prism.x\"" + self.assertEqual(self.commands(text), []) + self.assertEqual(self.references(text), ["lab.prism.x"]) + self.assertEqual(self.commands("python3 \"lab/prism/x.py'"), []) + + def test_quoted_interpreters_with_missing_targets_are_caught(self) -> None: + # The shell strips matching quotes around the interpreter name and runs + # Python all the same, so `"python3" -m lab.x` is as dead as the bare form. + for quote in ("'", '"'): + with self.subTest(quote=quote): + for interpreter in ("python3", "python3.12", "python"): + text = f"{quote}{interpreter}{quote} -m lab.prism.process_telemetry rss-bound" + self.assertEqual(self.commands(text), [self.TELEMETRY]) + text = f"{quote}{interpreter}{quote} lab/prism/storm.py --decide" + self.assertEqual(self.commands(text), ["lab/prism/storm.py"]) + self.assertEqual( + self.commands(f"{quote}python{quote} -OO -m {quote}lab.prism.process_telemetry{quote}"), + [self.TELEMETRY], + ) + + def test_quoted_interpreters_with_existing_targets_pass(self) -> None: + tracked = self.TRACKED | {"lab/prism/tool.py", "lab/pkg/__init__.py", "lab/pkg/__main__.py"} + text = ( + "\"python3\" -m lab.prism.tool\n'python3.12' -OO -m lab.pkg\n\"python\" lab/prism/tool.py\n" + "'python3' \"./lab/prism/tool.py\"" + ) + self.assertEqual(dead_commands(text, tracked), []) + + def test_quoted_interpreter_inside_a_shell_string_is_still_caught(self) -> None: + # A quote opens before the interpreter but nothing closes right after + # it, so the interpreter quote is empty and the inner command is read. + self.assertEqual( + self.commands('sh -c "python3 -m lab.prism.process_telemetry rss-bound"'), + [self.TELEMETRY], + ) + self.assertEqual(self.commands("sh -c 'python3.12 lab/prism/storm.py --decide'"), ["lab/prism/storm.py"]) + + def test_mismatched_interpreter_quotes_are_not_a_command(self) -> None: + text = "\"python3' -m lab.prism.x" + self.assertEqual(self.commands(text), []) + self.assertEqual(self.references(text), ["lab.prism.x"]) + self.assertEqual(self.commands("'python3\" lab/prism/x.py"), []) + + def test_wrapped_quoted_interpreter_is_caught_at_the_right_line(self) -> None: + text = "```bash\ncd repo\n\"python3\" \\\n -m lab.prism.process_telemetry \\\n rss-bound\n```" + self.assertEqual(self.located(text), [(3, self.TELEMETRY)]) + text = "'python3.12' -OO \\\n lab/prism/storm.py \\\n --decide" + self.assertEqual(self.located(text), [(1, "lab/prism/storm.py")]) + + # Each ran `json.tool` on bash 3.2 and CPython 3.14 exactly as `python3`: + # the shell joins the pieces of the interpreter word and strips the quotes + # before it looks the program up, so a name spelled in pieces runs Python. + CONCATENATED_INTERPRETERS = ('"python"3', "'pyth'on3", 'python"3"', "python''3", '"pyth"on"3"') + + def test_concatenated_interpreters_with_missing_targets_are_caught(self) -> None: + for interpreter in self.CONCATENATED_INTERPRETERS: + with self.subTest(interpreter=interpreter): + text = f"{interpreter} -m lab.prism.process_telemetry rss-bound" + self.assertEqual(self.commands(text), [self.TELEMETRY]) + self.assertEqual( + self.commands(f"{interpreter} lab/prism/storm.py --decide"), ["lab/prism/storm.py"] + ) + self.assertEqual( + self.commands(f"{interpreter} -OO -m 'lab.prism.'process_telemetry"), [self.TELEMETRY] + ) + self.assertEqual(self.commands("'pyth'on3.12 -m lab.prism.process_telemetry"), [self.TELEMETRY]) + + def test_concatenated_interpreters_with_existing_targets_pass(self) -> None: + tracked = self.TRACKED | {"lab/prism/tool.py", "lab/pkg/__init__.py", "lab/pkg/__main__.py"} + for interpreter in self.CONCATENATED_INTERPRETERS: + with self.subTest(interpreter=interpreter): + text = ( + f"{interpreter} -m lab.prism.tool\n{interpreter} -OO -m lab.pkg\n" + f"{interpreter} lab/prism/tool.py" + ) + self.assertEqual(dead_commands(text, tracked), []) + + def test_wrapped_concatenated_interpreter_is_caught_at_the_right_line(self) -> None: + text = "```bash\ncd repo\n\"python\"3 \\\n -m lab.prism.process_telemetry \\\n rss-bound\n```" + self.assertEqual(self.located(text), [(3, self.TELEMETRY)]) + text = "'pyth'on3 -OO \\\n lab/prism/storm.py \\\n --decide" + self.assertEqual(self.located(text), [(1, "lab/prism/storm.py")]) + + # `/usr/bin/python3 -m json.tool` and `"/usr/bin/pyth"on3 -m json.tool` both + # ran on bash 3.2: the basename after the last `/` is what names Python. + def test_path_interpreters_with_missing_targets_are_caught_once(self) -> None: + for interpreter in ( + "/usr/bin/python3", + "/opt/homebrew/bin/python3.12", + '"/usr/bin/pyth"on3', + '"/usr/bin/"python3', + "'/usr/bin/python'3", + "./venv/bin/python", + ): + with self.subTest(interpreter=interpreter): + self.assertEqual( + self.commands(f"{interpreter} -m lab.prism.process_telemetry rss-bound"), [self.TELEMETRY] + ) + self.assertEqual(self.commands(f"{interpreter} lab/prism/storm.py --decide"), ["lab/prism/storm.py"]) + self.assertEqual( + dead_commands('"/usr/bin/"python3 lab/prism/storm.py', self.TRACKED), + [(1, '"/usr/bin/"python3 lab/prism/storm.py', "lab/prism/storm.py")], + ) + + def test_words_that_name_no_interpreter_are_not_a_command(self) -> None: + # bash answers `mypython3 -m json.tool` with "command not found", and + # `cpython3`, `python2`, `python3x` or a program merely under a `python` + # directory are other programs; the prose contract still sees the + # reference. A word that opens a quote it never closes is a syntax error. + for interpreter in ( + "mypython3", + "cpython3", + "python2", + "python3x", + '"my"python3', + "python3/bin/tool", + "python\"3", + "env", + ): + with self.subTest(interpreter=interpreter): + text = f"{interpreter} -m lab.prism.x" + self.assertEqual(self.commands(text), []) + self.assertEqual(self.references(text), ["lab.prism.x"]) + self.assertEqual(self.commands(f"{interpreter} lab/prism/x.py"), []) + + def test_interpreter_words_start_after_shell_boundaries(self) -> None: + # A launcher word before the interpreter is not consumed with it: each + # of these still runs the missing target through `python3`. + for text in ( + "env python3 lab/prism/storm.py", + "env -i python3 -m lab.prism.process_telemetry", + 'docker exec "$c" python3 lab/prism/storm.py', + "sudo -u prism /usr/bin/python3 lab/prism/storm.py", + "nohup python3 lab/prism/storm.py &", + "(python3 lab/prism/storm.py)", + "$(python3 lab/prism/storm.py)", + "`python3 lab/prism/storm.py`", + "true && python3 lab/prism/storm.py", + "true; python3 lab/prism/storm.py", + 'sh -c "python3 lab/prism/storm.py"', + ): + with self.subTest(text=text): + self.assertEqual(len(self.commands(text)), 1, self.commands(text)) + + def test_version_suffixed_interpreters_are_scanned(self) -> None: + self.assertEqual( + self.commands("python3.12 -m lab.prism.process_telemetry rss-bound"), + [self.TELEMETRY], + ) + self.assertEqual(self.commands("python3.14 -OO lab/prism/storm.py"), ["lab/prism/storm.py"]) + self.assertEqual(self.commands("python2 -m lab.prism.process_telemetry"), []) + + def test_bash_append_assignments_before_commands_are_scanned(self) -> None: + for command, missing in ( + ("python3 -m lab.example.deleted", "lab/example/deleted.py or lab/example/deleted/__main__.py"), + ("python3 lab/example/deleted.py", "lab/example/deleted.py"), + ): + for text in ( + f"FOO+=x {command}", f"FOO+= {command}", + f"_FOO2+='hello world' {command}", f'FOO+="hello world" {command}', + f"FOO=base FOO+=x BAR+=y {command}", + f"FOO+=x 2>/dev/null {command}", f"FOO+=x command -- {command}", + f"if FOO+=x {command}; then true; fi", + f"bash -c 'FOO+=x {command}'", + # env accepts FOO+ as a variable name; it does not append to FOO. + f"env FOO+=x {command}", f"env 'FOO+=x' {command}", + f"env -S 'FOO+=x {command}'", + ): + with self.subTest(text=text): + self.assertEqual(self.located(text), [(1, missing)]) + self.assertEqual(dead_commands(text, {"lab/example/deleted.py"}), []) + text = f"```bash\nFOO+=x \\\n {command}\n```" + self.assertEqual(self.located(text), [(2, missing)]) + + def test_append_assignment_lookalikes_do_not_start_commands(self) -> None: + command = "python3 -m lab.example.deleted" + for prefix in ( + "'FOO+=x'", '"FOO+=x"', 'F"OO"+=x', "FOO'+'=x", "FOO+'='x", + "1FOO+=x", "FOO++=x", "FOO+ =x", "echo FOO+=x", + "command FOO+=x", "command -- FOO+=x", "nohup FOO+=x", + "exec FOO+=x", "docker exec container FOO+=x", + "env command FOO+=x", "FOO+=x command BAR+=y", + ): + with self.subTest(prefix=prefix): + self.assertEqual(self.commands(f"{prefix} {command}"), []) + self.assertEqual( + self.commands(f"{prefix} {command}; python3 lab/prism/storm.py"), + ["lab/prism/storm.py"], + ) + + def test_compound_shell_commands_are_scanned(self) -> None: + for command, missing in ( + ("python3 -m lab.example.deleted", "lab/example/deleted.py or lab/example/deleted/__main__.py"), + ("python3 lab/example/deleted.py", "lab/example/deleted.py"), + ): + for text in ( + f"{{ {command}; }}", + f"{{ {{ env X=1 {command}; }}; }}", + f"while {command}; do break; done", + f"until {command}; do break; done", + f"while true; do {{ {command}; }}; break; done", + f"for item in one two; do {command}; done", + f"if {{ {command}; }}; then true; fi", + f"sh -c '{{ {command}; }}'", + ): + with self.subTest(text=text): + self.assertEqual(self.located(text), [(1, missing)]) + text = f"```sh\nwhile\n {{ {command}; }}\ndo\n break\ndone\n```" + self.assertEqual(self.located(text), [(3, missing)]) + self.assertEqual(list(dead_commands(text, {"lab/example/deleted.py"})), []) + + def test_compound_shell_prefixes_in_data_are_not_commands(self) -> None: + command = "python3 -m lab.example.deleted" + for keyword in ("{", "while", "until"): + for prefix in ( + f"'{keyword}'", f'"{keyword}"', f"echo {keyword}", + f"env {keyword}", f"command -- {keyword}", f"X=1 {keyword}", + ): + with self.subTest(prefix=prefix): + self.assertEqual(self.commands(f"{prefix} {command}"), []) + for text in ( + f"for item in while {command}; do echo done; done", + f"{{ echo {command}; }}", + f"while echo {command}; do break; done", + ): + with self.subTest(text=text): + self.assertEqual(self.commands(text), []) + + def test_time_shell_prefixes_run_the_following_command(self) -> None: + for command, missing in ( + ("python3 -m lab.example.deleted", "lab/example/deleted.py or lab/example/deleted/__main__.py"), + ("python3 lab/example/deleted.py", "lab/example/deleted.py"), + ): + for text in ( + f"time {command}", f"time -p {command}", f"time ! {command}", + f"if time -p {command}; then true; fi", + f"while time {command}; do break; done", + f"{{ time -p {command}; }}", f"time -p {{ {command}; }}", + f"time X=1 {command}", f"time -p env X=1 {command}", + f"time command -- {command}", f"time sudo -u prism {command}", + f"env X=1 bash -c 'time -p {command}'", + ): + with self.subTest(text=text): + self.assertEqual(self.located(text), [(1, missing)]) + self.assertEqual(dead_commands(text, {"lab/example/deleted.py"}), []) + text = f"```bash\ntime -p \\\n {command}\n```" + self.assertEqual(self.located(text), [(2, missing)]) + + def test_time_words_and_options_in_data_are_not_shell_prefixes(self) -> None: + command = "python3 -m lab.example.deleted" + for prefix in ( + "'time'", '"time"', "ti'me'", "$'time'", "'time' -p", + "echo time", "printf '%s\\n' time -p", "env time", "X=1 time", + "command -- time", "time echo", "time -p printf '%s\\n'", + "time '-p'", 'time "-p"', 'time -"p"', "time -p -p", "time --help", + ): + with self.subTest(prefix=prefix): + text = f"{prefix} {command}" + self.assertEqual(self.commands(text), []) + self.assertEqual(self.commands(text + "; python3 lab/prism/storm.py"), ["lab/prism/storm.py"]) + + def test_coproc_prefixes_run_the_following_command(self) -> None: + for command, missing in ( + ("python3 -m lab.example.deleted", "lab/example/deleted.py or lab/example/deleted/__main__.py"), + ("python3 lab/example/deleted.py", "lab/example/deleted.py"), + ): + for text in ( + f"coproc {command}", + f"coproc env X=1 {command}", + f"coproc {{ {command}; }}", + f"coproc worker {{ {command}; }}", + f"coproc python3 {{ {command}; }}", + f"coproc worker ( {command} )", + f"coproc worker if {command}; then true; fi", + f"coproc worker while {command}; do break; done", + f"coproc worker until {command}; do break; done", + f"bash -c 'coproc worker {{ {command}; }}'", + ): + with self.subTest(text=text): + self.assertEqual(self.located(text), [(1, missing)]) + self.assertEqual(dead_commands(text, {"lab/example/deleted.py"}), []) + text = f"```bash\ncoproc worker {{\n {command};\n}}\n```" + self.assertEqual(self.located(text), [(3, missing)]) + text = f"```bash\ncoproc \\\n {command}\n```" + self.assertEqual(self.located(text), [(2, missing)]) + + def test_coproc_words_and_names_in_data_are_not_commands(self) -> None: + command = "python3 -m lab.example.deleted" + for prefix in ( + "'coproc'", '"coproc"', "co'proc'", "$'coproc'", + "echo coproc", "printf '%s\\n' coproc", "env coproc", + "command -- coproc", "X=1 coproc", + "coproc printf", "coproc echo", "coproc worker", + "coproc worker '{'", 'coproc worker "{"', + "coproc worker time", + ): + with self.subTest(prefix=prefix): + text = f"{prefix} {command}" + self.assertEqual(self.commands(text), []) + self.assertEqual(self.commands(text + "; python3 lab/prism/storm.py"), ["lab/prism/storm.py"]) + self.assertEqual(self.commands(f"coproc python3 {{ printf '%s\\n' {command}; }}"), []) + + def test_inline_code_is_neither_a_module_nor_a_script_command(self) -> None: + # `-c cmd` runs its argument and ends the option list; the prose contract + # still sees the module reference inside the code string. + text = "python3 -c 'import lab.prism.x'" + self.assertEqual(self.commands(text), []) + self.assertEqual(self.references(text), ["lab.prism.x"]) + self.assertEqual(self.commands("python3 -OO -c 'import lab.prism.x'"), []) + self.assertEqual(self.commands("python3 -c -m lab.prism.x"), []) + self.assertEqual(self.commands("python3 -c lab/prism/x.py"), []) + + # Verified on CPython 3.14 with a `lab/gone.py` and a `gone.py` that raise + # on import: each of these printed the help text or the version and exited + # 0 without running either, while `python3 -m lab.gone` and `python3 + # gone.py` raised. `-h`, its alias `-?` and `-V` end the run before the + # target is read wherever they sit among the options, so a command + # carrying one runs nothing; the prose contract still sees the reference. + HELP_AND_VERSION_OPTIONS = ( + "-h", + "-?", + "-V", + "-VV", + "-OO -h", + "-Oh", + "-hOO", + "-V -X dev", + "-VX dev", + "-hW error", + ) + + def test_help_and_version_options_are_not_a_command(self) -> None: + for options in self.HELP_AND_VERSION_OPTIONS: + with self.subTest(options=options): + text = f"python3 {options} -m lab.prism.x" + self.assertEqual(self.commands(text), []) + self.assertEqual(self.references(text), ["lab.prism.x"]) + self.assertEqual(self.commands(f"python3 {options} lab/prism/x.py"), []) + self.assertEqual(self.commands(f"python3.12 {options} -m 'lab.prism.'x"), []) + self.assertEqual(self.commands("python3 -V -- lab/prism/x.py"), []) + # A cluster `-h` or `-V` closes with `m` prints and exits the same way. + for option in ("-hm", "-Vm", "-Ohm", "-hIm", "-VVm"): + with self.subTest(option=option): + self.assertEqual(self.commands(f"python3 {option} lab.prism.x"), []) + self.assertEqual(self.commands(f"python3 {option}lab.prism.x"), []) + self.assertEqual(self.references(f"python3 {option} lab.prism.x"), ["lab.prism.x"]) + + # Each ran `json.tool` on bash 3.2 and CPython 3.14 exactly as its bare + # spelling: `python3 "-O" -m json.tool`, `python3 '-OO' -m json.tool`, + # `python3 $'-OO' -m json.tool`, `python3 "-"O -m json.tool`, `python3 + # "-X" dev -m json.tool`, `python3 '-X'dev -m json.tool`, `python3 + # "-X"'dev' -m json.tool`, `python3 "-W" error -m json.tool`, `python3 + # '-uW'error -m json.tool` and `python3 "--check-hash-based-pycs" always + # -m json.tool`. The shell strips the quotes of an option word, attached + # or not, before CPython sees it, so a quoted option is as runnable as a + # bare one and, ahead of a missing target, as dead. + QUOTED_OPTION_WORDS = ( + '"-O"', + "'-OO'", + "$'-OO'", + '"-"O', + '"-X" dev', + "'-X'dev", + "\"-X\"'dev'", + '"-W" error', + "'-uW'error", + '"--check-hash-based-pycs" always', + ) + # `python3 '-m' json.tool`, `python3 "-m" json.tool`, `python3 + # "-m"json.tool` and `python3 -"m" json.tool` each ran `json.tool` the + # same way, and `python3 '--' lab/gone.py` ran the script. + QUOTED_MODULE_OPTIONS = ("'-m' {}", '"-m" {}', '"-m"{}', '-"m" {}') + + def test_quoted_option_words_with_missing_targets_are_caught(self) -> None: + for options in self.QUOTED_OPTION_WORDS: + with self.subTest(options=options): + self.assertEqual( + self.commands(f"python3 {options} -m lab.prism.process_telemetry rss-bound"), + [self.TELEMETRY], + ) + self.assertEqual( + self.commands(f"python {options} lab/prism/storm.py --decide"), + ["lab/prism/storm.py"], + ) + for option in self.QUOTED_MODULE_OPTIONS: + with self.subTest(option=option): + module = option.format("lab.prism.process_telemetry") + self.assertEqual(self.commands(f"python3 {module} rss-bound"), [self.TELEMETRY]) + self.assertEqual(self.commands(f"python3.12 -OO {module}"), [self.TELEMETRY]) + self.assertEqual(self.commands("python3 '--' lab/prism/storm.py --decide"), ["lab/prism/storm.py"]) + self.assertEqual(self.commands("python3 \"-OO\" '--' './lab/prism/storm.py'"), ["lab/prism/storm.py"]) + # The reported command runs from the interpreter word to the target + # word, quotes and all, so the reader finds it in the document. + self.assertEqual( + dead_commands('python3 "-O" -m lab.prism.process_telemetry rss-bound', self.TRACKED), + [(1, 'python3 "-O" -m lab.prism.process_telemetry', self.TELEMETRY)], + ) + self.assertEqual( + dead_commands('python3 "-m"lab.prism.process_telemetry', self.TRACKED), + [(1, 'python3 "-m"lab.prism.process_telemetry', self.TELEMETRY)], + ) + + def test_quoted_option_words_with_existing_targets_pass(self) -> None: + tracked = self.TRACKED | {"lab/prism/tool.py", "lab/pkg/__init__.py", "lab/pkg/__main__.py"} + for options in self.QUOTED_OPTION_WORDS: + with self.subTest(options=options): + text = ( + f"python3 {options} -m lab.prism.tool\npython3.12 {options} -m lab.pkg\n" + f"python {options} lab/prism/tool.py" + ) + self.assertEqual(dead_commands(text, tracked), []) + for option in self.QUOTED_MODULE_OPTIONS: + with self.subTest(option=option): + text = f"python3 {option.format('lab.prism.tool')}\npython3.12 {option.format('lab.pkg')}" + self.assertEqual(dead_commands(text, tracked), []) + text = "python3 '--' lab/prism/tool.py\npython3 \"-O\" \"--\" ./lab/prism/tool.py" + self.assertEqual(dead_commands(text, tracked), []) + + def test_wrapped_quoted_option_words_are_caught_at_the_right_line(self) -> None: + text = "```bash\ncd repo\npython3 \\\n \"-O\" \\\n '-m' lab.prism.process_telemetry \\\n rss-bound\n```" + self.assertEqual(self.located(text), [(3, self.TELEMETRY)]) + text = "python3.12 \"-X\" \\\n dev \\\n '--' \\\n lab/prism/storm.py \\\n --decide" + self.assertEqual(self.located(text), [(1, "lab/prism/storm.py")]) + + # Verified on CPython 3.14 with a `lab/gone.py` that raises + # `SystemExit("RAN")`: none of these printed RAN. `'-h' -m lab.gone` and + # `"-hm" lab.gone` print the help text and `"-V" lab/gone.py` the + # version, and exit 0; `"-Wm" lab.gone` and `"-Xm" lab.gone` hand `m` to + # `-W`/`-X` and fail to open a script called `lab.gone`; + # `"--check-hash-based-pycs" sometimes -m lab.gone` is rejected ("must be + # one of 'default', 'always', or 'never'") before anything runs; `'--' -m + # lab.gone` fails to open a file called `-m`; and `"-c" 'import lab.gone'` + # runs its argument as inline code exactly as bare `-c` does, which the + # command contract leaves alone. Quoting an option the guard leaves out + # therefore changes nothing, and the prose contract still sees the + # reference. + def test_quoted_non_command_options_are_not_a_command(self) -> None: + for text, reference in ( + ("python3 \"-c\" 'import lab.prism.x'", "lab.prism.x"), + ("python3 '-h' -m lab.prism.x", "lab.prism.x"), + ('python3 "-V" lab/prism/x.py', "lab/prism/x.py"), + ('python3 "-hm" lab.prism.x', "lab.prism.x"), + ('python3 "-Wm" lab.prism.x', "lab.prism.x"), + ('python3 "-Xm" lab.prism.x', "lab.prism.x"), + ('python3 "--check-hash-based-pycs" sometimes -m lab.prism.x', "lab.prism.x"), + ("python3 '--' -m lab.prism.x", "lab.prism.x"), + ): + with self.subTest(text=text): + self.assertEqual(self.commands(text), []) + self.assertEqual(self.references(text), [reference]) + + def test_mismatched_option_word_quotes_are_not_a_command(self) -> None: + # bash rejects `python3 "-O' -m lab.gone` and `python3 -X "dev' -m + # lab.gone` with "unexpected EOF while looking for matching `"'" + # before Python starts, so the line runs nothing; the prose contract + # still sees the reference. + for text, reference in ( + ("python3 \"-O' -m lab.prism.x", "lab.prism.x"), + ("python3 -X \"dev' -m lab.prism.x", "lab.prism.x"), + ("python3 '-OO\" lab/prism/x.py", "lab/prism/x.py"), + ("python3 \"-m' lab.prism.x", "lab.prism.x"), + ): + with self.subTest(text=text): + self.assertEqual(self.commands(text), []) + self.assertEqual(self.references(text), [reference]) + + # The mode of `--check-hash-based-pycs` is a shell word like any other: + # bash 3.2 hands `$'always'`, `alwa"ys"`, `de'fault'`, `"al"ways` and + # `$'never'` to CPython 3.14 as the bare mode, and `python3 + # --check-hash-based-pycs $'always' -m json.tool` ran `json.tool`. The + # word is therefore unquoted before it is checked against the three + # modes, so a mode spelled in pieces still runs, and reports, a missing + # target. A word that is no mode once unquoted (`"sometimes"`, + # `$'some'times`) is rejected by CPython before anything runs, and a + # mismatched quote is rejected by bash, so neither is a command. + HASH_MODE_WORDS = ("$'always'", 'alwa"ys"', "'never'", '$"default"', "de'fault'", '"al"ways') + + def test_quoted_hash_mode_arguments_are_read_as_shell_words(self) -> None: + tracked = self.TRACKED | {"lab/prism/tool.py"} + for mode in self.HASH_MODE_WORDS: + with self.subTest(mode=mode): + text = f"python3 --check-hash-based-pycs {mode} -m lab.prism.process_telemetry rss-bound" + self.assertEqual(self.commands(text), [self.TELEMETRY]) + self.assertEqual( + self.commands(f"python --check-hash-based-pycs {mode} lab/prism/storm.py"), ["lab/prism/storm.py"] + ) + self.assertEqual(dead_commands(f"python3 --check-hash-based-pycs {mode} -m lab.prism.tool", tracked), []) + for mode in ('"sometimes"', "$'some'times", "'always\"", "$'alwa'ys\""): + with self.subTest(mode=mode): + text = f"python3 --check-hash-based-pycs {mode} -m lab.prism.x" + self.assertEqual(self.commands(text), []) + self.assertEqual(self.references(text), ["lab.prism.x"]) + + # bash 5.2 ran nothing from a comment line or from the tail of a line + # after its comment opens (see ``COMMENT_BOUNDARY``); the prose contract + # still counts every reference a comment names. + def test_comment_lines_are_not_a_command(self) -> None: + for text in ( + "# python3 -m lab.prism.process_telemetry rss-bound", + "#python3 lab/prism/storm.py", + " # python3 -m lab.prism.process_telemetry", + "```bash\n# python3 -m lab.prism.process_telemetry\n```", + "`# python3 lab/prism/storm.py`", + ): + with self.subTest(text=text): + self.assertEqual(self.commands(text), []) + self.assertEqual(len(self.references(text)), 1) + + def test_trailing_comments_are_not_a_command(self) -> None: + tracked = self.TRACKED | {"lab/prism/tool.py"} + for text in ( + "python3 -m lab.prism.tool # python3 -m lab.prism.process_telemetry", + "echo done # python3 lab/prism/storm.py", + "true;# python3 -m lab.prism.process_telemetry", + "`echo done # python3 lab/prism/storm.py`", + "python3 -m # lab.prism.process_telemetry", + "python3 -X # dev -m lab.prism.process_telemetry", + ): + with self.subTest(text=text): + self.assertEqual(dead_commands(text, tracked), []) + self.assertEqual( + dangling_references("python3 -m lab.prism.tool # was lab/prism/storm.py", tracked), + [(1, "lab/prism/storm.py")], + ) + + def test_commands_before_comments_are_still_caught(self) -> None: + self.assertEqual( + dead_commands("python3 -m lab.prism.process_telemetry # retired in #244", self.TRACKED), + [(1, "python3 -m lab.prism.process_telemetry", self.TELEMETRY)], + ) + # A comment ends at the backtick that closes its inline code, and a + # `#` inside a nested shell string is the inner shell's business. + for text in ( + "python3 lab/prism/storm.py --decide # see the runbook", + "sh -c 'python3 lab/prism/storm.py' # nested", + 'sh -c "python3 lab/prism/storm.py # inner"', + "## Run `python3 lab/prism/storm.py`", + "Issue #303: `python3 lab/prism/storm.py`", + "See [the runbook](#run) and `python3 lab/prism/storm.py`", + "`true # comment` then `python3 lab/prism/storm.py`", + ): + with self.subTest(text=text): + self.assertEqual(self.commands(text), ["lab/prism/storm.py"]) + + def test_quoted_escaped_and_in_word_hashes_open_no_comment(self) -> None: + # Each prefix printed its `#` or a count on bash 5.2 and then ran the + # command after `;`, so the missing target there is still dead. + for prefix in ( + "echo '#'", + 'echo "# x"', + "echo \\#", + "echo a#b", + "echo $#", + "echo ${#a[@]}", + "echo a\\ #b", + "echo $'\\'#'", + 'echo "\\"#"', + "curl https://example.com/runbook#step", + ): + for command, missing in ( + ("python3 -m lab.prism.process_telemetry", self.TELEMETRY), + ("python3 lab/prism/storm.py", "lab/prism/storm.py"), + ): + text = f"{prefix}; {command}" + with self.subTest(text=text): + self.assertEqual(dead_commands(text, self.TRACKED), [(1, command, missing)]) + + def test_backslash_ending_a_comment_continues_nothing(self) -> None: + # bash 5.2 ran the line after `# c \` and after `echo a # c \`. + text = "```bash\n# retired: python3 -m lab.prism.process_telemetry \\\npython3 lab/prism/storm.py\n```" + self.assertEqual( + dead_commands(text, self.TRACKED), [(3, "python3 lab/prism/storm.py", "lab/prism/storm.py")] + ) + self.assertEqual( + dangling_references(text, self.TRACKED), + [(2, "lab.prism.process_telemetry"), (3, "lab/prism/storm.py")], + ) + text = "python3 -m lab.prism.tool # wrapped \\\n python3 -m lab.prism.process_telemetry" + self.assertEqual( + dead_commands(text, self.TRACKED | {"lab/prism/tool.py"}), + [(2, "python3 -m lab.prism.process_telemetry", self.TELEMETRY)], + ) + # A continuation into a comment joins it and the comment ends the + # command there; the next line runs on its own as `-m …`. + self.assertEqual(self.commands("python3 \\\n # retired\n -m lab.prism.process_telemetry"), []) + text = "python3 \\\n -m lab.prism.process_telemetry \\\n rss-bound # python3 lab/prism/storm.py" + self.assertEqual(self.located(text), [(1, self.TELEMETRY)]) + # An escaped, in-word or quoted `#` opens no comment, so the backslash + # after it still continues the line, and a quote carried across a + # continuation keeps its `#` literal: bash printed `a # b` for + # `echo "a \` followed by `# b"` and ran the command after `;`. + for text in ( + "echo \\# \\\n && python3 lab/prism/storm.py", + "echo \\\\#x \\\n && python3 lab/prism/storm.py", + "echo '#' \\\n && python3 lab/prism/storm.py", + 'echo "a \\\n# b"; python3 lab/prism/storm.py', + ): + with self.subTest(text=text): + self.assertEqual(self.located(text), [(1, "lab/prism/storm.py")]) + + def test_fenced_comments_hide_their_backticks(self) -> None: + # bash 5.2 ran no `echo NO` from `# was `echo NO`` or `echo ok # was + # `echo NO``: in a fence the comment runs to the end of the line. + tracked = self.TRACKED | {"lab/prism/tool.py"} + for text in ( + "```sh\n# historical `python3 -m lab.prism.process_telemetry`\n```", + "```bash\npython3 -m lab.prism.tool # was `python3 lab/prism/storm.py`\n```", + "- step\n\n ~~~bash\n # `python3 lab/prism/storm.py` and `python3 -m lab.prism.process_telemetry`\n ~~~", + "````sh\n```\n# `python3 lab/prism/storm.py`\n````", + ): + with self.subTest(text=text): + self.assertEqual(dead_commands(text, tracked), []) + self.assertEqual( + self.references("```sh\n# historical `python3 -m lab.prism.process_telemetry`\n```"), + ["lab.prism.process_telemetry"], + ) + # A comment inside a command substitution ends at its closing backtick: + # bash printed `RUN` for `` out=`true # echo NO`; echo RUN ``. + text = "```sh\nout=`true # python3 lab/prism/storm.py`; python3 -m lab.prism.process_telemetry\n```" + self.assertEqual(self.located(text), [(2, self.TELEMETRY)]) + + def test_prose_after_a_fence_ends_comments_at_inline_code(self) -> None: + text = ( + "```sh\n# `python3 -m lab.prism.process_telemetry`\n```\n" + "## Run `python3 lab/prism/storm.py`\n" + "```python3 lab/prism/storm.py```\n" + "# `python3 -m lab.prism.process_telemetry`" + ) + # The fenced comment on line 2 hides its inline code; the same text as + # a prose heading on line 6 does not. + self.assertEqual( + self.located(text), [(4, "lab/prism/storm.py"), (5, "lab/prism/storm.py"), (6, self.TELEMETRY)] + ) + # No quote carries across inline code, so prose apostrophes neither + # hide a command nor keep a comment open. + self.assertEqual(self.commands("Don't run `true # python3 lab/prism/storm.py`"), []) + self.assertEqual(self.commands("It's `python3 lab/prism/storm.py` # done"), ["lab/prism/storm.py"]) + + def test_ordinary_arguments_are_not_executable_commands(self) -> None: + for text in ( + "printf '%s\\n' 'python3 -m lab.example.deleted'", + 'echo "python3 -m lab.example.deleted"', + "printf '%s\\n' python3 -m lab.example.deleted", + "echo 'python3' -m lab.example.deleted", + "printf '%s\\n' sh -c 'python3 -m lab.example.deleted'", + "python3 -c 'python3 -m lab.example.deleted'", + "cat <<< 'python3 -m lab.example.deleted'", + "echo done > python3 -m lab.example.deleted", + "'python3 -m lab.example.deleted'", + '"python3 -m lab.example.deleted"', + "command -v python3 -m lab.example.deleted", + "command -V python3 -m lab.example.deleted", + "'X=1' python3 -m lab.example.deleted", + "'if' python3 -m lab.example.deleted", + "env --help python3 -m lab.example.deleted", + "sudo -l python3 -m lab.example.deleted", + "nohup --version python3 -m lab.example.deleted", + "docker exec --help container python3 -m lab.example.deleted", + ): + with self.subTest(text=text): + self.assertEqual(self.commands(text), []) + self.assertEqual(self.commands(text + "; python3 lab/prism/storm.py"), ["lab/prism/storm.py"]) + + def test_env_split_string_runs_literal_argv(self) -> None: + for options in ( + "-S 'python3 -m lab.example.deleted'", + "--split-string 'python3 -m lab.example.deleted'", + "--split-string='python3 -m lab.example.deleted'", + "-S'python3 -m lab.example.deleted'", + "-i -S 'python3 -m' lab.example.deleted", + "-S 'python3' -m lab.example.deleted", + "-S 'MODE=test python3 -m lab.example.deleted'", + "-S '-u MODE python3 -m lab.example.deleted'", + "-S 'sudo -u prism python3 -m lab.example.deleted'", + "-S 'nohup python3 -m lab.example.deleted'", + "-S '-S \"python3 -m\" lab.example.deleted'", + "-S 'env -S \"python3 -m\"' lab.example.deleted", + "-S 'python3 -m # ignored' lab.example.deleted", + "-vS 'python3 -m lab.example.deleted'", + "-iS 'python3 -m lab.example.deleted'", + "-ivS'python3 -m lab.example.deleted'", + "-viS 'python3 -m' lab.example.deleted", + "'-vS' 'python3 -m lab.example.deleted'", + ): + with self.subTest(options=options): + text = f"env {options}" + self.assertEqual(self.located(text), [(1, "lab/example/deleted.py or lab/example/deleted/__main__.py")]) + self.assertEqual(dead_commands(text, {"lab/example/deleted.py"}), []) + for text in ( + "env -S 'python3 lab/example/deleted.py'", + "sudo -u prism env -S 'python3' lab/example/deleted.py", + "docker exec container env -S 'python3 lab/example/deleted.py'", + "env -S 'sh -c \"python3 lab/example/deleted.py\"'", + "env -S 'sh -c' 'python3 lab/example/deleted.py'", + ): + with self.subTest(text=text): + self.assertEqual(self.commands(text), ["lab/example/deleted.py"]) + + def test_env_split_string_words_are_argv_not_shell_code(self) -> None: + for text in ( + "env -S 'echo python3 -m lab.example.deleted'", + "env -S 'echo; python3 -m lab.example.deleted'", + "env -S 'echo | python3 -m lab.example.deleted'", + "env -S 'echo\npython3 -m lab.example.deleted'", + "env -S 'true # python3 -m lab.example.deleted'", + "env -S '\"python3 -m lab.example.deleted\"'", + "env -S 'if python3 -m lab.example.deleted'", + "env -S 'sh -nc \"python3 -m lab.example.deleted\"'", + "env -S 'sudo -v python3 -m lab.example.deleted'", + "env -S 'python3 -c \"python3 -m lab.example.deleted\"'", + "printf '%s' env -S 'python3 -m lab.example.deleted'", + "env -S '${INTERPRETER} -m lab.example.deleted'", + "env -S 'python3\\_ -m lab.example.deleted'", + "env -S 'python3 -m \"lab.example.deleted'", + "env -S", + "env -vS", + "env -uS 'python3 -m lab.example.deleted'", + "env -u -vS 'python3 -m lab.example.deleted'", + "env -CS 'python3 -m lab.example.deleted'", + "env -C -iS 'python3 -m lab.example.deleted'", + "env -aS 'python3 -m lab.example.deleted'", + "env -xS 'python3 -m lab.example.deleted'", + "env --vS 'python3 -m lab.example.deleted'", + "env -vS '${INTERPRETER} -m lab.example.deleted'", + ): + with self.subTest(text=text): + self.assertEqual(self.commands(text), []) + self.assertEqual(self.commands(text + "; python3 lab/prism/storm.py"), ["lab/prism/storm.py"]) + self.assertEqual(self.commands("env -S 'python3 lab/prism/deleted;name.py'"), ["lab/prism/deleted;name.py"]) + self.assertEqual(self.commands("env -S 'python3 lab/prism/deleted#name.py'"), ["lab/prism/deleted#name.py"]) + self.assertEqual(self.commands("env -S 'python3 lab/prism/deleted|name.py'"), ["lab/prism/deleted|name.py"]) + self.assertEqual(self.commands("env -S 'python3 \"lab/prism/deleted name.py\"'"), ["lab/prism/deleted name.py"]) + + def test_env_split_string_preserves_empty_words_comments_and_locations(self) -> None: + self.assertEqual(self.commands("env -S '' python3 lab/prism/storm.py"), ["lab/prism/storm.py"]) + self.assertEqual(self.commands("env -S '# ignored' python3 lab/prism/storm.py"), ["lab/prism/storm.py"]) + self.assertEqual(self.commands("env -S '\"\" python3 lab/prism/storm.py'"), []) + self.assertEqual(env_split_words("one#two '#three' \"#four\" # ignored"), [("one#two", 0), ("#three", 0), ("#four", 0)]) + text = "```sh\nenv -S '\npython3 lab/prism/storm.py'\n```" + self.assertEqual(self.located(text), [(3, "lab/prism/storm.py")]) + text = "```sh\nenv -S 'sh -c \"true\npython3 lab/prism/storm.py\"'\n```" + self.assertEqual(self.located(text), [(3, "lab/prism/storm.py")]) + for options in ("-vS '", "-iS'"): + with self.subTest(options=options): + text = f"```sh\nenv {options}\npython3 lab/prism/storm.py'\n```" + self.assertEqual(self.located(text), [(3, "lab/prism/storm.py")]) + text = "```sh\nenv -S 'sh -c'\n```" + self.assertEqual(self.commands(text), []) + + def test_only_env_and_sudo_take_assignment_arguments(self) -> None: + # nohup 9.4 answers `nohup FOO=x echo hi` with "failed to run command + # 'FOO=x'", bash 5.2.21 `command FOO=x echo hi` with "FOO=x: command + # not found" and `exec FOO=x echo hi` with "exec: FOO=x: not found", + # each exit 127, and docker exec runs the word after the container: + # the assignment is the program, and nothing after it runs. sudo + # 1.9.15p5 exported `FOO=x` and `FOO+=x` before a command, but ran + # `FOO=x` as the command after `--`. + command = "python3 -m lab.prism.deleted" + for launcher in ( + "nohup", "command", "command -p", "exec", "exec -a name", "docker exec c", + "podman exec -it c", "env nohup", "sudo nohup", "sudo -u prism command", + "nohup sudo --", "sudo --", "sudo -n --", "env sudo --", + ): + for assignments in ("FOO=x", "FOO=x BAR=y", "FOO+=x", "'FOO=x'", 'FOO="x y"'): + with self.subTest(launcher=launcher, assignments=assignments): + text = f"{launcher} {assignments} {command}" + self.assertEqual(self.commands(text), []) + self.assertEqual(self.references(text), ["lab.prism.deleted"]) + self.assertEqual(self.commands(text + "; python3 lab/prism/storm.py"), ["lab/prism/storm.py"]) + for launcher in ( + "env", "env -i", "env --", "env -", "sudo", "sudo -u prism", "sudo -n", "nohup env", + "sudo env", "env sudo", "nohup sudo -u prism", "command env", "exec env", + "docker exec c env", "sudo FOO=x env", "env FOO=x sudo", + ): + for assignments in ("FOO=x", "FOO=x BAR=y", "FOO+=x", "'FOO=x'", '"FOO=x"', 'FOO="x y"'): + with self.subTest(launcher=launcher, assignments=assignments): + text = f"{launcher} {assignments} python3 lab/prism/storm.py" + self.assertEqual(self.commands(text), ["lab/prism/storm.py"]) + # A leading assignment is shell syntax and precedes any launcher. + for text in ( + "FOO=x nohup python3 lab/prism/storm.py", "FOO=x BAR+=y command python3 lab/prism/storm.py", + "FOO=x exec python3 lab/prism/storm.py", "FOO=x docker exec c python3 lab/prism/storm.py", + "FOO=x sudo BAR=y python3 lab/prism/storm.py", + ): + with self.subTest(text=text): + self.assertEqual(self.commands(text), ["lab/prism/storm.py"]) + + def test_env_null_mode_does_not_run_commands(self) -> None: + # GNU coreutils 9.4 refuses a command after `-0`/`--null` ("cannot + # specify --null (-0) with command", exit 125), whether the `0` stands + # alone, follows other argument-free flags in a cluster, or arrives + # through `-S`, so nothing after it runs. + for options in ( + "-0", "--null", "'-0'", "-i0", "-0i", "-v0", "-iv0", "-0u FOO", "-0 -u FOO", + "-i -0", "-0 -i", "-0S", "-0 -S", "-0 --null", + ): + for argument in ( + "python3 -m lab.prism.deleted", "python3 lab/prism/deleted.py", + "sh -c 'python3 -m lab.prism.deleted'", + ): + for delimiter in ("", "-- "): + with self.subTest(options=options, argument=argument, delimiter=delimiter): + text = f"env {options} {delimiter}{argument}" + self.assertEqual(self.commands(text), []) + self.assertEqual(len(self.references(text)), 1) + self.assertEqual( + self.commands(text + "; python3 lab/prism/storm.py"), + ["lab/prism/storm.py"], + ) + for text in ( + "env -S '-0 python3 -m lab.prism.deleted'", + "env -iS '-0 python3 -m lab.prism.deleted'", + "env -0S 'python3 -m lab.prism.deleted'", + "sudo env -0 python3 -m lab.prism.deleted", + "nohup env --null python3 lab/prism/deleted.py", + ): + with self.subTest(text=text): + self.assertEqual(self.commands(text), []) + self.assertEqual(self.commands(text + "; python3 lab/prism/storm.py"), ["lab/prism/storm.py"]) + text = "```sh\nenv -0 \\\n python3 -m lab.prism.deleted\npython3 lab/prism/storm.py\n```" + self.assertEqual(self.located(text), [(4, "lab/prism/storm.py")]) + # `-u0` and `-u 0` unset the variable `0` and run the command. + for options in ("-u0", "-u 0", "-iu0", "-i -u 0", "--unset=0", "--unset 0", "-S '-u 0'"): + with self.subTest(options=options): + self.assertEqual(self.commands(f"env {options} python3 lab/prism/storm.py"), ["lab/prism/storm.py"]) + + def test_sudo_chdir_arguments_before_commands_are_consumed(self) -> None: + for options in ( + "-D /tmp", "--chdir /tmp", "-D/tmp", "--chdir=/tmp", + "'-D' '/tmp/work dir'", "-u prism -D /tmp --", "-D /tmp -n", + ): + for command, missing in ( + ("python3 -m lab.example.deleted", "lab/example/deleted.py or lab/example/deleted/__main__.py"), + ("python3 lab/example/deleted.py", "lab/example/deleted.py"), + ): + with self.subTest(options=options, command=command): + text = f"sudo {options} {command}" + self.assertEqual(self.located(text), [(1, missing)]) + self.assertEqual(dead_commands(text, {"lab/example/deleted.py"}), []) + text = "```sh\nsudo -D \\\n /tmp \\\n python3 lab/example/deleted.py\n```" + self.assertEqual(self.located(text), [(2, "lab/example/deleted.py")]) + self.assertEqual( + self.commands("env sudo --chdir /tmp sh -c 'python3 lab/prism/storm.py'"), + ["lab/prism/storm.py"], + ) + + def test_sudo_chdir_arguments_are_not_executable_words(self) -> None: + for option in ("-D", "--chdir"): + for arguments in ( + "python3 -m lab.example.deleted", "'python3 -m lab.example.deleted' true", + "", + ): + with self.subTest(option=option, arguments=arguments): + self.assertEqual(self.commands(f"sudo {option} {arguments}"), []) + + def test_sudo_chroot_role_and_type_arguments_before_commands_are_consumed(self) -> None: + for options in ( + "-R /", "--chroot /", "-R/", "--chroot=/", "'-R' '/srv/chroot dir'", + "-r admin", "--role admin", "-radmin", "--role=admin", "\"-r\" 'admin'", + "-t unconfined_t", "--type unconfined_t", "-tunconfined_t", "--type=unconfined_t", + "'--type' unconfined_t", "-u prism -R / -r admin -t unconfined_t --", "-R / -n", + "-D /tmp -R /", "-r python3 -t python3", "-R -e", "-r -v", "-t -K", + ): + for command, missing in ( + ("python3 -m lab.example.deleted", "lab/example/deleted.py or lab/example/deleted/__main__.py"), + ("python3 lab/example/deleted.py", "lab/example/deleted.py"), + ): + with self.subTest(options=options, command=command): + text = f"sudo {options} {command}" + self.assertEqual(self.located(text), [(1, missing)]) + self.assertEqual(dead_commands(text, {"lab/example/deleted.py"}), []) + text = "```sh\nsudo -r \\\n python3 \\\n python3 lab/example/deleted.py\n```" + self.assertEqual(self.located(text), [(2, "lab/example/deleted.py")]) + text = "```sh\nenv -S 'sudo --type\npython3\npython3 lab/prism/storm.py'\n```" + self.assertEqual(self.located(text), [(4, "lab/prism/storm.py")]) + for text in ( + "env sudo --chroot / sh -c 'python3 lab/prism/storm.py'", + "env -S 'sudo -R / python3 lab/prism/storm.py'", + "sudo -r admin env -u MODE python3 lab/prism/storm.py", + "nohup sudo -t unconfined_t command -- python3 lab/prism/storm.py", + ): + with self.subTest(text=text): + self.assertEqual(self.commands(text), ["lab/prism/storm.py"]) + + def test_sudo_chroot_role_and_type_arguments_are_not_executable_words(self) -> None: + for option in ("-R", "--chroot", "-r", "--role", "-t", "--type"): + for arguments in ( + "python3 -m lab.example.deleted", "python3 lab/example/deleted.py", + "'python3 -m lab.example.deleted' true", "", + ): + with self.subTest(option=option, arguments=arguments): + self.assertEqual(self.commands(f"sudo {option} {arguments}"), []) + for options in ("-R / -e", "-r admin --validate", "-t -e -K", "--chroot=/ -nv"): + with self.subTest(options=options): + text = f"sudo {options} -- python3 -m lab.prism.deleted" + self.assertEqual(self.commands(text), []) + self.assertEqual(self.commands(text + "; python3 lab/prism/storm.py"), ["lab/prism/storm.py"]) + + def test_sudo_edit_mode_arguments_are_not_commands(self) -> None: + for options in ( + "-e", "--edit", "'-e'", "-ne", "-en", "-Hne", "-euprism", + "-nepPrompt", "-n -u prism --edit", "-D /tmp -e", + ): + for argument in ( + "python3 -m lab.prism.deleted", "python3 lab/prism/deleted.py", + "sh -c 'python3 -m lab.prism.deleted'", + ): + with self.subTest(options=options, argument=argument): + text = f"sudo {options} -- {argument}" + self.assertEqual(self.commands(text), []) + self.assertEqual(len(self.references(text)), 1) + self.assertEqual( + self.commands(text + "; python3 lab/prism/storm.py"), + ["lab/prism/storm.py"], + ) + self.assertEqual(self.commands("sudo -e python3 -m lab.prism.deleted"), []) + self.assertEqual(self.commands("env sudo -ne -- python3 lab/prism/deleted.py"), []) + text = "```sh\nsudo -e \\\n -- python3 -m lab.prism.deleted\npython3 lab/prism/storm.py\n```" + self.assertEqual(self.located(text), [(4, "lab/prism/storm.py")]) + + def test_sudo_credential_modes_do_not_run_commands(self) -> None: + for options in ( + "-K", "--remove-timestamp", "'-K'", "-nK", "-Kn", "-HnK", + "-v", "--validate", "'-v'", "-nv", "-vn", "-Hnv", "-vuprism", + "-n -u prism --validate", "-p Prompt -v", + ): + for argument in ( + "python3 -m lab.prism.deleted", "python3 lab/prism/deleted.py", + "sh -c 'python3 -m lab.prism.deleted'", + ): + for delimiter in ("", "-- "): + with self.subTest(options=options, argument=argument, delimiter=delimiter): + text = f"sudo {options} {delimiter}{argument}" + self.assertEqual(self.commands(text), []) + self.assertEqual(len(self.references(text)), 1) + self.assertEqual( + self.commands(text + "; python3 lab/prism/storm.py"), + ["lab/prism/storm.py"], + ) + self.assertEqual(self.commands("env sudo -nv -- python3 lab/prism/deleted.py"), []) + text = "```sh\nsudo --validate \\\n -- python3 -m lab.prism.deleted\npython3 lab/prism/storm.py\n```" + self.assertEqual(self.located(text), [(4, "lab/prism/storm.py")]) + + def test_sudo_version_mode_in_a_cluster_does_not_run_commands(self) -> None: + # sudo 1.9.15p5 printed its usage and ran nothing for `sudo -nV id`, + # `sudo -Vn id`, `sudo -HV id`, `sudo -Vu nobody id` and `sudo -nVu + # nobody id`, as for `sudo -V id`; `sudo -uV nobody id` read the + # user `V` and `sudo -pV nobody id` the prompt "V", so the word after + # those is the program. + for options in ("-V", "'-V'", "-nV", "-Vn", "-HV", "-HnV", "-Vu nobody", "-nVu nobody", "-V --", "-n -V"): + for argument in ( + "python3 -m lab.prism.deleted", "python3 lab/prism/deleted.py", "sh -c 'python3 -m lab.prism.deleted'", + ): + with self.subTest(options=options, argument=argument): + text = f"sudo {options} {argument}" + self.assertEqual(self.commands(text), []) + self.assertEqual(len(self.references(text)), 1) + self.assertEqual(self.commands(text + "; python3 lab/prism/storm.py"), ["lab/prism/storm.py"]) + for text in ("sudo -uV nobody", "sudo -pV nobody"): + for argument in ("python3 -m lab.prism.deleted", "python3 lab/prism/deleted.py"): + with self.subTest(text=text, argument=argument): + line = f"{text} {argument}" + self.assertEqual(self.commands(line), []) + self.assertEqual(self.commands(line + "; python3 lab/prism/storm.py"), ["lab/prism/storm.py"]) + self.assertEqual(self.commands("env sudo -nV python3 lab/prism/deleted.py"), []) + text = "```sh\nsudo -nV \\\n python3 -m lab.prism.deleted\npython3 lab/prism/storm.py\n```" + self.assertEqual(self.located(text), [(4, "lab/prism/storm.py")]) + + def test_sudo_timestamp_reset_and_mode_option_arguments_preserve_commands(self) -> None: + for options in ( + "-k", "--reset-timestamp", "-nk", "-kn", "-kuprism", "-k -u prism", + "-p K", "-pK", "-p -K", "--prompt=K", "-p v", "-pv", "-p -v", + "-u v", "-uv", "--user=v", "--prompt=validate", "-u remove-timestamp", + ): + with self.subTest(options=options): + self.assertEqual( + self.commands(f"sudo {options} python3 lab/prism/storm.py"), + ["lab/prism/storm.py"], + ) + + def test_sudo_option_arguments_containing_edit_flags_preserve_commands(self) -> None: + for options in ( + "-p e", "-pe", "-peditor", "--prompt=editor", "-p -e", + "-u editor", "-ueditor", "-geditors", "-D repo", "-Drepo", + "--chdir=repo", "-D -e", + ): + with self.subTest(options=options): + self.assertEqual( + self.commands(f"sudo {options} python3 lab/prism/storm.py"), + ["lab/prism/storm.py"], + ) + + def test_clustered_wrapper_options_consume_their_operands(self) -> None: + # getopt reads a short cluster letter by letter: the first letter + # that takes an argument takes the rest of the cluster or, at its + # end, the next word. sudo 1.9.15p5 ran `id -un` as nobody for + # `sudo -nu nobody`, `sudo -nunobody` and `sudo -nHu nobody`; GNU + # coreutils 9.4 ran `env -iu FOO echo hi` and `env -iC /tmp pwd`; + # bash 5.2.21 ran `exec -ca zzz echo hi` and `exec -cazzz echo hi`. + for text in ( + "sudo -nu nobody {}", "sudo -nunobody {}", "sudo -nHu nobody {}", "sudo -Hnu nobody {}", + "sudo -bnu nobody {}", "sudo -ng wheel {}", "sudo -nhhost {}", "sudo -np Prompt {}", + "sudo -nC 3 {}", "sudo -nD /tmp {}", "sudo -nR / {}", "sudo -nr admin {}", + "sudo -nt unconfined_t {}", "sudo -nT 30 {}", "sudo -Enu nobody -g wheel {}", + "sudo '-nu' nobody {}", "sudo -nu nobody -- {}", "sudo -nu nobody FOO=x {}", + "env -iu FOO {}", "env -vu FOO {}", "env -iC /tmp {}", "env -iuFOO {}", "env -ivu FOO -i {}", + "env -iu FOO BAR=y {}", "exec -ca name {}", "exec -la name {}", "exec -cla name {}", + "exec -caname {}", "docker exec -ite FOO=x c {}", "docker exec -itu root c {}", + "docker exec -itw /tmp c {}", "docker exec -iteFOO=x c {}", "podman exec -du root c {}", + "nohup sudo -nu nobody env -iu FOO {}", + ): + for command, missing in ( + ("python3 -m lab.example.deleted", "lab/example/deleted.py or lab/example/deleted/__main__.py"), + ("python3 lab/example/deleted.py", "lab/example/deleted.py"), + ): + with self.subTest(text=text.format(command)): + self.assertEqual(self.located(text.format(command)), [(1, missing)]) + self.assertEqual(dead_commands(text.format(command), {"lab/example/deleted.py"}), []) + text = "```sh\nsudo -nu \\\n nobody \\\n python3 lab/example/deleted.py\n```" + self.assertEqual(self.located(text), [(2, "lab/example/deleted.py")]) + self.assertEqual(self.commands("sudo -nu nobody sh -c 'python3 lab/prism/storm.py'"), ["lab/prism/storm.py"]) + + def test_clustered_wrapper_operands_end_at_the_first_argument_letter(self) -> None: + # `-pu nobody` gives p the prompt "u" and runs `nobody` (sudo 1.9.15p5: + # "nobody: command not found"), `-un nobody` sets the user `n` + # ("unknown user n"), `env -ui FOO` unsets `i` and runs `FOO`, and + # `exec -ac zzz` names the process `c` and runs `zzz`: the word after + # such a cluster is the program, so the Python after it never runs. + # `-nv` and `-nvu nobody` still validate credentials and run nothing. + for text in ( + "sudo -pu nobody", "sudo -un nobody", "sudo -ug nobody", "sudo -Du nobody", "sudo -nuv nobody", + "sudo -nv", "sudo -nvu nobody", "sudo -nu", + "env -ui FOO", "env -Cu /tmp", "env -iu", + "exec -ac name", "exec -ca", + "docker exec -ei FOO c", "docker exec -itu c", + ): + for argument in ("python3 -m lab.prism.deleted", "python3 lab/prism/deleted.py"): + with self.subTest(text=text, argument=argument): + line = f"{text} {argument}" + self.assertEqual(self.commands(line), []) + self.assertEqual(len(self.references(line)), 1) + self.assertEqual(self.commands(line + "; python3 lab/prism/storm.py"), ["lab/prism/storm.py"]) + + def test_sudo_host_flag_reads_a_host_only_attached_or_as_the_lone_option(self) -> None: + # sudo's -h takes an optional argument, so a cluster ending in `h` + # with nothing attached is the help mode, as is a lone `-h` followed + # by nothing, an option or an assignment: sudo 1.9.15p5 printed its + # usage and ran nothing for `sudo -nh host id`, `sudo -Hh host id`, + # `sudo -ih host id`, `sudo -nh`, `sudo -h -u nobody id`, `sudo -h -- + # id` and `sudo -h FOO=x id`, while `sudo -h host id`, `sudo -hhost + # id`, `sudo -nhhost id` and `sudo -n -h host id` went on to the + # command, and `sudo -nhu nobody id` read the host `u` and the + # command `nobody`. + for options in ( + "-nh host", "-Hh host", "-nHh host", "-kh host", "-bh host", "-Sh host", "-Eh host", "-ih host", "-sh host", + "'-nh' host", "-nh", "-h", "-h -u nobody", "-h -n", "-h --", "-h FOO=x", "-n -h -u nobody", "-nh FOO=x", + "-nh --", + ): + for argument in ( + "python3 -m lab.prism.deleted", "python3 lab/prism/deleted.py", "sh -c 'python3 -m lab.prism.deleted'", + ): + with self.subTest(options=options, argument=argument): + text = f"sudo {options} {argument}" + self.assertEqual(self.commands(text), []) + self.assertEqual(len(self.references(text)), 1) + self.assertEqual(self.commands(text + "; python3 lab/prism/storm.py"), ["lab/prism/storm.py"]) + for text in ( + "sudo -h host {}", "sudo -hhost {}", "sudo -nhhost {}", "sudo -n -h host {}", "sudo -nu nobody -h host {}", + "sudo -h host -u nobody {}", "sudo -h 'host' {}", "sudo '-h' host {}", "sudo -h host -- {}", + "sudo --host host {}", "sudo --host=host {}", "sudo -h =host {}", + ): + for command, missing in ( + ("python3 -m lab.example.deleted", "lab/example/deleted.py or lab/example/deleted/__main__.py"), + ("python3 lab/example/deleted.py", "lab/example/deleted.py"), + ): + with self.subTest(text=text.format(command)): + self.assertEqual(self.located(text.format(command)), [(1, missing)]) + self.assertEqual(dead_commands(text.format(command), {"lab/example/deleted.py"}), []) + # Any letter attached to the `h` is the host, and `-ph` gives p the + # prompt "h": the word after it is the program. + for text in ("sudo -nhu nobody", "sudo -hn host", "sudo -hh host", "sudo -ph host"): + for argument in ("python3 -m lab.prism.deleted", "python3 lab/prism/deleted.py"): + with self.subTest(text=text, argument=argument): + line = f"{text} {argument}" + self.assertEqual(self.commands(line), []) + self.assertEqual(self.commands(line + "; python3 lab/prism/storm.py"), ["lab/prism/storm.py"]) + text = "```sh\nsudo -nh \\\n host \\\n python3 lab/example/deleted.py\npython3 lab/prism/storm.py\n```" + self.assertEqual(self.located(text), [(5, "lab/prism/storm.py")]) + + def test_command_lookup_flags_in_a_cluster_do_not_run_commands(self) -> None: + # bash 5.2.21 printed `echo` and ran nothing for `command -pv echo + # hi`, `command -vp echo hi`, `command -vv echo hi` and `command -pv + # -- echo hi`, described the builtin for `command -pV echo hi` and + # `command -Vp echo hi`, and ran `echo hi` for `command -p echo hi`, + # `command -pp echo hi` and `command -p -- echo hi`. + for options in ("-v", "-V", "-pv", "-vp", "-pV", "-Vp", "-vv", "-pvp", "'-pv'", "-p -v", "-pv -p", "-pv --"): + for argument in ( + "python3 -m lab.prism.deleted", "python3 lab/prism/deleted.py", "sh -c 'python3 -m lab.prism.deleted'", + ): + with self.subTest(options=options, argument=argument): + text = f"command {options} {argument}" + self.assertEqual(self.commands(text), []) + self.assertEqual(len(self.references(text)), 1) + self.assertEqual(self.commands(text + "; python3 lab/prism/storm.py"), ["lab/prism/storm.py"]) + for options in ("-p", "-pp", "-p --", "--", "-p -p"): + with self.subTest(options=options): + self.assertEqual(self.commands(f"command {options} python3 lab/prism/storm.py"), ["lab/prism/storm.py"]) + self.assertEqual(self.commands("sudo command -pv python3 lab/prism/storm.py"), []) + self.assertEqual(self.commands("command -pv sudo python3 lab/prism/storm.py"), []) + text = "```sh\ncommand -pv \\\n python3 lab/prism/deleted.py\npython3 lab/prism/storm.py\n```" + self.assertEqual(self.located(text), [(4, "lab/prism/storm.py")]) + + def test_builtin_runs_only_the_shell_builtins_the_check_reads(self) -> None: + # bash 5.2.21 ran `echo hi` for `builtin command echo hi`, `builtin + # exec echo hi`, `builtin builtin command echo hi`, `builtin -- + # command echo hi`, `builtin 'command' echo hi`, `command builtin + # command echo hi`, `command -p builtin command echo hi` and `FOO=x + # builtin command echo hi`. + for prefix in ( + "builtin command", "builtin exec", "builtin builtin command", "builtin builtin exec", + "builtin -- command", "builtin -- exec", "builtin 'command'", 'builtin com"mand"', + "command builtin command", "command -p builtin command", "builtin command -p", + "builtin command --", "builtin exec -a name", "builtin -- builtin -- command", + "FOO=x builtin command", "time builtin exec", "if builtin command", + ): + for command, missing in ( + ("python3 -m lab.prism.deleted", "lab/prism/deleted.py or lab/prism/deleted/__main__.py"), + ("python3 lab/prism/deleted.py", "lab/prism/deleted.py"), + ("sh -c 'python3 lab/prism/deleted.py'", "lab/prism/deleted.py"), + ): + with self.subTest(prefix=prefix, command=command): + self.assertEqual(self.commands(f"{prefix} {command}"), [missing]) + self.assertEqual(list(dead_commands(f"{prefix} python3 lab/prism/tool.py", {"lab/prism/tool.py"})), []) + text = "```sh\nbuiltin -- command \\\n python3 lab/prism/deleted.py\npython3 lab/prism/storm.py\n```" + self.assertEqual(self.located(text), [(2, "lab/prism/deleted.py"), (4, "lab/prism/storm.py")]) + # bash 5.2.21 answered `builtin python3 -c 1`, `builtin env echo hi`, + # `builtin sudo echo hi`, `builtin nohup echo hi`, `builtin docker + # exec c echo hi`, `builtin bash -c 'echo hi'`, `builtin /usr/bin/env + # echo hi`, `builtin ./command echo hi`, `builtin FOO=x command echo + # hi` and `builtin -- -- command echo hi` with "not a shell builtin", + # ran nothing for `builtin` and `builtin --`, rejected `-x`, `-p`, + # `-pv`, `--foo` and `--version` as invalid options and printed its + # help for `--help`, and ran the next command on the line after each; + # dash 0.5.12 has no `builtin` at all. + for prefix in ( + "builtin", "builtin --", "builtin env", "builtin sudo", "builtin nohup", "builtin -- env", + "builtin docker exec c", "builtin podman exec c", "builtin sh -c", "builtin bash -c", + "builtin /usr/bin/env", "builtin ./command", "builtin /bin/command", "builtin FOO=x command", + "builtin -x command", "builtin -p command", "builtin -pv command", "builtin --foo command", + "builtin --version command", "builtin --help command", "builtin -- -- command", + "builtin -- -x command", "builtin '' command", "builtin command -v", "builtin -- command -V", + "command builtin env", "builtin builtin sudo", "sudo builtin python3", "builtin builtin", + ): + for argument in ( + "python3 -m lab.prism.deleted", "python3 lab/prism/deleted.py", "sh -c 'python3 -m lab.prism.deleted'", + ): + with self.subTest(prefix=prefix, argument=argument): + text = f"{prefix} {argument}" + self.assertEqual(self.commands(text), []) + self.assertEqual(len(self.references(text)), 1) + self.assertEqual(self.commands(text + "; python3 lab/prism/storm.py"), ["lab/prism/storm.py"]) + text = "```sh\nbuiltin python3 \\\n -m lab.prism.deleted\npython3 lab/prism/storm.py\n```" + self.assertEqual(self.located(text), [(4, "lab/prism/storm.py")]) + + def test_multiline_quoted_arguments_remain_data(self) -> None: + for quote in ("'", '"'): + with self.subTest(quote=quote): + text = ( + f"printf '%s\\n' {quote}first line\n" + "python3 -m lab.example.deleted\n" + f"last line{quote}; python3 lab/prism/storm.py" + ) + self.assertEqual(self.located(text), [(3, "lab/prism/storm.py")]) + + def test_heredoc_bodies_are_not_executable_commands(self) -> None: + for delimiter in ("EOF", "'EOF'", '"EOF"', "E'O'F", r"\EOF"): + for operator, indent in (("<<", ""), ("<<-", "\t")): + with self.subTest(delimiter=delimiter, operator=operator): + text = ( + f"cat {operator}{delimiter}\n" + f"{indent}python3 -m lab.example.deleted\n" + f"{indent}python3 lab/example/deleted.py\n" + f"{indent}EOF\n" + "python3 lab/prism/storm.py" + ) + self.assertEqual(self.located(text), [(5, "lab/prism/storm.py")]) + + def test_indented_fenced_heredocs_end_at_the_rendered_delimiter(self) -> None: + for operator, tabs in (("<<", ""), ("<<-", "\t")): + with self.subTest(operator=operator): + text = ( + f"- example\n\n ```sh\n cat {operator}'EOF'\n" + " python3 -m lab.example.deleted\n" + f" {tabs}EOF\n python3 lab/prism/storm.py\n ```" + ) + self.assertEqual(self.located(text), [(7, "lab/prism/storm.py")]) + + def test_file_descriptor_redirections_are_not_executable_words(self) -> None: + for prefix in ( + "2>/dev/null", "2>/dev/null env", "env 2>/dev/null", "command --", + "X='value'", "env 'X=1'", + ): + with self.subTest(prefix=prefix): + self.assertEqual(self.commands(f"{prefix} python3 lab/prism/storm.py"), ["lab/prism/storm.py"]) + self.assertEqual(self.commands("printf '%s\\n' 2>/dev/null python3 -m lab.example.deleted"), []) + + def test_redirections_among_python_options_and_targets_are_removed(self) -> None: + missing = "lab/example/deleted.py or lab/example/deleted/__main__.py" + for redirection in ( + "2>/dev/null", "</dev/null", "2> /dev/null", "> out.log", ">>out.log", "2>&1", ">&2", + "2>&-", "<&0", "&>/dev/null", "&>> out.log", ">|out.log", "<>tty", "<<<'data'", + "2>'quoted file'", '>"$log"', "2>/dev/null </dev/null", + ): + for command in ( + f"python3 {redirection} -m lab.example.deleted", + f"python3 -O {redirection} -m lab.example.deleted", + f"python3 -m {redirection} lab.example.deleted", + f"python3 -X {redirection} dev -m lab.example.deleted", + f"python3 -W {redirection} error -m lab.example.deleted", + f"python3 --check-hash-based-pycs {redirection} always -m lab.example.deleted", + f"{redirection} python3 -m lab.example.deleted", + f"env {redirection} python3 -m lab.example.deleted", + ): + with self.subTest(command=command): + self.assertEqual(self.commands(command), [missing]) + self.assertEqual(list(dead_commands(command, {"lab/example/deleted.py"})), []) + for command in ( + "python3 2>/dev/null lab/example/deleted.py", "python3 -- 2>&1 lab/example/deleted.py", + "python3>/dev/null -OO lab/example/deleted.py", "python3 -O>log lab/example/deleted.py", + "true && python3 </dev/null lab/example/deleted.py", "true & python3 2>&1 lab/example/deleted.py", + ): + with self.subTest(command=command): + self.assertEqual(self.commands(command), ["lab/example/deleted.py"]) + self.assertEqual(list(dead_commands(command, {"lab/example/deleted.py"})), []) + + def test_redirected_commands_keep_their_text_and_first_line(self) -> None: + text = "```sh\ntrue\npython3 2>/dev/null \\\n </dev/null -m lab.example.deleted >out.log\n```" + self.assertEqual( + [(number, command) for number, command, _ in dead_commands(text, self.TRACKED)], + [(3, "python3 2>/dev/null </dev/null -m lab.example.deleted")], + ) + text = "python3 <<'EOF' -m lab.example.deleted\npython3 -m lab.prism.gone\nEOF\npython3 2>&1 lab/prism/storm.py" + self.assertEqual( + self.located(text), + [(1, "lab/example/deleted.py or lab/example/deleted/__main__.py"), (4, "lab/prism/storm.py")], + ) + + def test_redirection_text_in_data_and_escapes_is_not_removed(self) -> None: + for text in ( + "python3 2 >/dev/null -m lab.example.deleted", + "python3 '2'>/dev/null -m lab.example.deleted", + "python3 2&>/dev/null -m lab.example.deleted", + "python3 2&>>/dev/null -m lab.example.deleted", + "python3 ٢>/dev/null -m lab.example.deleted", + "٢>/dev/null python3 -m lab.example.deleted", + "python3 '2>/dev/null' -m lab.example.deleted", + 'python3 "</dev/null" -m lab.example.deleted', + "python3 2'>'/dev/null -m lab.example.deleted", + "python3 \\>out -m lab.example.deleted", + "python3 2\\>out -m lab.example.deleted", + "python3 -c 2>/dev/null -m lab.example.deleted", + "env -S 'python3 2>/dev/null -m lab.example.deleted'", + "python3 2> ; -m lab.example.deleted", + "printf '%s\\n' 2>&1 python3 -m lab.example.deleted", + "echo x >&2 python3 -m lab.example.deleted", + "echo x &>/dev/null python3 -m lab.example.deleted", + "echo x &>> log python3 -m lab.example.deleted", + "echo x >| log python3 -m lab.example.deleted", + "echo x <&0 python3 -m lab.example.deleted", + ): + with self.subTest(text=text): + self.assertEqual(self.commands(text), []) + + def test_heredoc_data_cannot_change_comments_continuations_or_fences(self) -> None: + text = ( + "```sh\ncat <<'EOF'\n" + "# quoted data \\\n" + "```\n" + "python3 -m lab.example.deleted\n" + "EOF\n" + "# python3 -m lab.example.deleted\n" + "python3 lab/prism/storm.py\n```" + ) + self.assertEqual(self.located(text), [(8, "lab/prism/storm.py")]) + # Only tabs are stripped by <<-; whitespace or trailing text cannot + # prematurely close a heredoc and turn its remaining body into code. + text = "cat <<-EOF\n EOF\nEOF extra\npython3 -m lab.example.deleted\n\tEOF\npython3 lab/prism/storm.py" + self.assertEqual(self.located(text), [(6, "lab/prism/storm.py")]) + + def test_multiple_heredocs_and_commands_on_the_opening_line(self) -> None: + text = ( + "cat <<FIRST <<'SECOND'; python3 lab/prism/storm.py\n" + "python3 -m lab.example.deleted\nFIRST\n" + "python3 -m lab.example.deleted\nSECOND\n" + "python3 -m lab.prism.process_telemetry" + ) + self.assertEqual(self.located(text), [(1, "lab/prism/storm.py"), (6, self.TELEMETRY)]) + text = "python3 lab/prism/storm.py <<EOF\npython3 -m lab.example.deleted\nEOF" + self.assertEqual(self.located(text), [(1, "lab/prism/storm.py")]) + text = "printf '%s\\n' 'cat <<EOF'\npython3 lab/prism/storm.py" + self.assertEqual(self.located(text), [(2, "lab/prism/storm.py")]) + + def test_shell_command_strings_are_scanned_in_their_own_context(self) -> None: + for prefix in ("sh -c", "bash -lc", 'docker exec "$c" sh -c'): + with self.subTest(prefix=prefix): + text = f'''{prefix} "printf '%s\\n' 'python3 -m lab.example.deleted'; python3 lab/prism/storm.py"''' + self.assertEqual(self.commands(text), ["lab/prism/storm.py"]) + text = f'''{prefix} 'true # python3 -m lab.example.deleted' ''' + self.assertEqual(self.commands(text), []) + text = f'''{prefix} '# `python3 -m lab.example.deleted`' ''' + self.assertEqual(self.commands(text), []) + text = "sh -c 'true\npython3 lab/prism/storm.py'" + self.assertEqual(self.located(text), [(2, "lab/prism/storm.py")]) + self.assertEqual(self.commands("sh -c python3 -m lab.example.deleted"), []) + + def test_shell_option_arguments_before_command_strings_are_consumed(self) -> None: + for options in ( + "-o pipefail -c", "-O extglob -c", "+o pipefail -c", "+O extglob -c", + "-eo pipefail -c", "-co pipefail", "-oc pipefail", "-cO extglob", + "-o pipefail -O extglob -c", "'-o' 'pipefail' '-c'", + "--rcfile /dev/null -c", "--init-file /dev/null -c", "-c -e", "-c --", + ): + for command, missing in ( + ("python3 -m lab.example.deleted", "lab/example/deleted.py or lab/example/deleted/__main__.py"), + ("python3 lab/example/deleted.py", "lab/example/deleted.py"), + ): + with self.subTest(options=options, command=command): + text = f"bash {options} '{command}'" + self.assertEqual(self.located(text), [(1, missing)]) + self.assertEqual(list(dead_commands(text, {"lab/example/deleted.py"})), []) + text = 'docker exec "$c" bash -o pipefail -c \'true\npython3 lab/example/deleted.py\'' + self.assertEqual(self.located(text), [(2, "lab/example/deleted.py")]) + self.assertEqual(self.commands("sh -o errexit -c 'python3 lab/prism/storm.py'"), ["lab/prism/storm.py"]) + + def test_noexec_shell_command_strings_are_not_runnable(self) -> None: + for shell in ("bash", "sh", "dash", "ksh", "zsh"): + for options in ( + "-n -c", "-nc", "-cn", "-c -n", "-en -c", "'-n' '-c'", + "-o noexec -c", "-co noexec", "-c -o noexec", "-o 'noexec' -c", + "+n -n -c", "+o noexec -n -c", "-n +n -o noexec -c", + "-n -o errexit -c", "-n +o errexit -c", "-n -c --", + ): + with self.subTest(shell=shell, options=options): + text = f"{shell} {options} 'python3 -m lab.prism.deleted'" + self.assertEqual(self.commands(text), []) + self.assertEqual(self.references(text), ["lab.prism.deleted"]) + + def test_disabling_noexec_restores_shell_command_scanning(self) -> None: + for shell in ("bash", "sh", "dash", "ksh", "zsh"): + for options in ( + "+n -c", "+o noexec -c", "-n +n -c", "-n +en -c", + "-n +o noexec -c", "-o noexec +n -c", "-o noexec +o noexec -c", + "-n -c +n", "-c -n +o noexec", "-co noexec +n", + ): + with self.subTest(shell=shell, options=options): + text = f"{shell} {options} 'python3 lab/prism/storm.py'" + self.assertEqual(self.commands(text), ["lab/prism/storm.py"]) + text = f"{shell} -c 'python3 lab/prism/storm.py' -n" + self.assertEqual(self.commands(text), ["lab/prism/storm.py"]) + + def test_noexec_tracking_preserves_shell_option_arguments(self) -> None: + for options in ( + "-o nounset -c", "--rcfile noexec -c", "--init-file noexec -c", + "-on noexec +n -c", "-no noexec +n -c", "-oo noexec errexit +n -c", + ): + with self.subTest(options=options): + text = f"bash {options} 'python3 lab/prism/storm.py'" + self.assertEqual(self.commands(text), ["lab/prism/storm.py"]) + for options in ("-on errexit -c", "-oon errexit noexec -c"): + with self.subTest(options=options): + self.assertEqual(self.commands(f"bash {options} 'python3 lab/prism/storm.py'"), []) + + def test_bash_dump_modes_do_not_run_shell_command_strings(self) -> None: + # bash 5.2.21 printed nothing for `bash -D -c 'echo hi'`, `bash -Dc + # 'echo hi'`, `bash -cD 'echo hi'`, `bash +D -c 'echo hi'`, `bash -D + # +n -c 'echo hi'`, `bash --dump-strings -c 'echo hi'`, `bash + # --dump-po-strings -c 'echo hi'` and `bash --dump-po-strings +o + # noexec -c 'echo hi'`: the dump modes imply -n and no later +n + # restores execution. dash 0.5.12 rejects -D and runs nothing + # either, so `sh` reads the same. + for shell in ("bash", "sh"): + for options in ( + "-D -c", "-Dc", "-cD", "-c -D", "+D -c", "'-D' -c", "-eD -c", "-D +n -c", "+n -D -c", + "-D +o noexec -c", "-nD +n -c", "-D -o pipefail -c", "-Do pipefail -c", + "--dump-strings -c", "--dump-po-strings -c", "--dump-strings +n -c", + "--dump-po-strings +o noexec -c", + ): + with self.subTest(shell=shell, options=options): + text = f"{shell} {options} 'python3 -m lab.prism.deleted'" + self.assertEqual(self.commands(text), []) + self.assertEqual(self.references(text), ["lab.prism.deleted"]) + self.assertEqual(self.commands(text + "; python3 lab/prism/storm.py"), ["lab/prism/storm.py"]) + text = "```sh\nbash -D \\\n -c 'python3 -m lab.prism.deleted'\npython3 lab/prism/storm.py\n```" + self.assertEqual(self.located(text), [(4, "lab/prism/storm.py")]) + self.assertEqual(self.commands("sudo bash -Dc 'python3 lab/prism/deleted.py'"), []) + # ksh93u+m 1.0.8 printed only the strings for `ksh -D -c 'echo hi'`, + # `ksh -Dc 'echo hi'`, `ksh +D -c 'echo hi'`, `ksh --dump-strings -c + # 'echo hi'` and `ksh -D +n -n -c 'echo hi'`, and mksh R59 rejects + # the letter and runs nothing, but a `+n` or `+o noexec` standing as + # the last such setting restores execution in ksh93: it ran `echo + # hi` for `ksh -D +n -c 'echo hi'`, `ksh +n -D -c 'echo hi'`, `ksh + # -D +o noexec -c 'echo hi'` and `ksh --dump-strings +n -c 'echo hi'`. + for options in ( + "-D -c", "-Dc", "-cD", "-c -D", "+D -c", "'-D' -c", "-eD -c", "-D +n -n -c", "-n +D -c", + "-D +o noexec -n -c", "-D -o pipefail -c", "-Do pipefail -c", "--dump-strings -c", "-D +D -c", + ): + with self.subTest(options=options): + text = f"ksh {options} 'python3 -m lab.prism.deleted'" + self.assertEqual(self.commands(text), []) + self.assertEqual(self.references(text), ["lab.prism.deleted"]) + self.assertEqual(self.commands(text + "; python3 lab/prism/storm.py"), ["lab/prism/storm.py"]) + for options in ( + "-D +n -c", "+n -D -c", "-D +o noexec -c", "-nD +n -c", "+D +n -c", "--dump-strings +n -c", + "-D -n +n -c", "+o noexec -D -c", "-c", + ): + with self.subTest(options=options): + text = f"ksh {options} 'python3 lab/prism/storm.py'" + self.assertEqual(self.commands(text), ["lab/prism/storm.py"]) + # dash rejects the letter and zsh 5.9 ran `zsh -D -c 'echo hi'`, so + # the string is still read there; after the string, `-D` is the `$0`. + for shell in ("dash", "zsh"): + for options in ("-D -c", "-Dc", "--dump-strings -c"): + with self.subTest(shell=shell, options=options): + text = f"{shell} {options} 'python3 lab/prism/storm.py'" + self.assertEqual(self.commands(text), ["lab/prism/storm.py"]) + self.assertEqual(self.commands("ksh -c 'python3 lab/prism/storm.py' -D"), ["lab/prism/storm.py"]) + for shell in ("bash", "sh"): + for options in ("-c", "-n +n -c", "-o pipefail -c", "-O extglob -c"): + with self.subTest(shell=shell, options=options): + text = f"{shell} {options} 'python3 lab/prism/storm.py' -D" + self.assertEqual(self.commands(text), ["lab/prism/storm.py"]) + + def test_noexec_set_inside_a_shell_string_ends_what_runs(self) -> None: + # bash 5.2.21 and dash 0.5.12 printed nothing for `set -n; echo NO`, + # `set -o noexec; echo NO`, `set -n; set +n; echo NO` and the other + # spellings ``sets_noexec`` lists, and the outer shell ran `echo YES` + # for `sh -c 'set -n; echo NO'; echo YES`. + for setting in ( + "set -n", "set -o noexec", "set -en", "set -ne", "set -nB", "set -no noexec", "set -on noexec", + "set -oo noexec errexit", "set -oe noexec", "set -o errexit -n", "set +e -n", "set +n -n", + "set +o noexec -o noexec", "set -n +", "set + -n", "set -o -n", "set -n -o", "set -no", "set -n x", + "set -n -- x", "set -n --", "set -n -", "set -o noexec x", "set -n 2>/dev/null", "set >/dev/null -n", + "FOO=x set -n", "command set -n", "command -p set -n", "command -- set -n", "builtin set -n", + "builtin -- set -n", "command builtin set -n", "builtin builtin set -n", "time set -n", "! set -n", + "{ set -n; }", "if set -n; then :; fi", "true && set -n", "set -n; set +n", "set -n; set +o noexec", + "set -o noexec; set +n", "set -n; (true)", "(true); set -n", "set -n -o nosuch", + ): + for shell in ("bash", "sh", "dash", "ksh", "zsh"): + for command in ( + "python3 -m lab.prism.deleted", "python3 lab/prism/deleted.py", 'bash -c "python3 -m lab.prism.deleted"', + "env python3 lab/prism/deleted.py", "builtin command python3 lab/prism/deleted.py", + ): + with self.subTest(setting=setting, shell=shell, command=command): + text = f"{shell} -c '{setting}; {command}'" + self.assertEqual(self.commands(text), []) + self.assertEqual(len(self.references(text)), 1) + self.assertEqual(self.commands(text + "; python3 lab/prism/storm.py"), ["lab/prism/storm.py"]) + for text in ( + "bash -c \"set '-n'; python3 lab/prism/deleted.py\"", + "bash -c \"set '-o' 'noexec'; python3 lab/prism/deleted.py\"", + "sh -c 'set -n\npython3 lab/prism/deleted.py'", + "sh -c 'true\nset -o noexec\ntrue\npython3 -m lab.prism.deleted'", + "sh -c 'set -n; python3 lab/prism/deleted.py\npython3 -m lab.prism.deleted'", + "docker exec \"$c\" bash -o pipefail -c 'set -n\npython3 lab/prism/deleted.py'", + "sudo sh -c 'set -n; sh -c \"python3 lab/prism/deleted.py\"'", + "bash -c 'set -n; bash -c \"set +n; python3 lab/prism/deleted.py\"'", + ): + with self.subTest(text=text): + self.assertEqual(self.commands(text), []) + text = "sh -c 'python3 lab/prism/storm.py\nset -n\npython3 lab/prism/deleted.py'\npython3 lab/prism/wave.py" + self.assertEqual(self.located(text), [(1, "lab/prism/storm.py"), (4, "lab/prism/wave.py")]) + text = "```sh\nbash -c 'set -n\n python3 lab/prism/deleted.py'\npython3 lab/prism/storm.py\n```" + self.assertEqual(self.located(text), [(4, "lab/prism/storm.py")]) + + def test_noexec_set_the_shell_undoes_rejects_or_never_reaches_leaves_the_string_running(self) -> None: + # bash 5.2.21 and dash 0.5.12 ran `echo YES` for `set -n +n; echo YES`, + # `set -o noexec +o noexec; echo YES`, `set +n; echo YES`, `set -- -n; + # echo YES`, `set - -n; echo YES`, `set x -n; echo YES`, `(set -n); + # echo YES` and `x=$(set -n); echo YES`; bash for `set -o +n; echo + # YES`, `set -nz; echo RAN`, `set -n -z; echo RAN`, `set -z -o noexec; + # echo RAN` and `set -+n; echo RAN`; and the shell ran on after GNU + # coreutils 9.4 env and nohup found no `set` program. docker was not + # installed on the inspected host; `docker exec` runs its command as + # a program and is read as env is. + for setting in ( + "set -n +n", "set +n", "set -o noexec +o noexec", "set -o noexec +n", "set -n +o noexec", + "set -o errexit -o noexec +o noexec", "set -oo noexec errexit +n", "set +o noexec", "set -o +n", + "set -- -n", "set - -n", "set x -n", "set -o -- -n", "set -o - -n", "set -o", "set", + "set -nz", "set -z -n", "set -n -z", "set -z -o noexec", "set -+n", "set --noexec", "set -o noexecx", + "set -o pipefail", "set -eu", "set -x", "env set -n", "env -i set -n", "nohup set -n", + "docker exec c set -n", "command -v set -n", "command -V set -n", "(set -n)", "(set -n; true)", + "( set -o noexec )", "(set -n; (true))", "((set -n))", "x=$(set -n)", "(x=$(set -n); true)", + "(set -n\ntrue)", + ): + for shell in ("bash", "sh", "dash", "ksh", "zsh"): + with self.subTest(setting=setting, shell=shell): + text = f"{shell} -c '{setting}; python3 lab/prism/deleted.py'" + self.assertEqual(self.commands(text), ["lab/prism/deleted.py"]) + self.assertEqual(self.commands("bash -c \"set '' -n; python3 lab/prism/deleted.py\""), ["lab/prism/deleted.py"]) + # The subshell's state ends at its parenthesis, and an enclosing + # `set -n` reaches into one. + self.assertEqual(self.commands("bash -c '(set -n; python3 lab/prism/deleted.py); python3 lab/prism/storm.py'"), ["lab/prism/storm.py"]) + self.assertEqual(self.commands("bash -c '(set -n; (python3 lab/prism/deleted.py)); python3 lab/prism/storm.py'"), ["lab/prism/storm.py"]) + self.assertEqual(self.commands("bash -c 'set -n; (python3 lab/prism/deleted.py); python3 lab/prism/storm.py'"), []) + self.assertEqual(self.commands("bash -c '(true); set -n; python3 lab/prism/deleted.py'"), []) + # A pipeline element and a background job run in subshells, which is + # not followed: the `set -n` is read as the shell's own. + self.assertEqual(self.commands("bash -c 'set -n | cat; python3 lab/prism/deleted.py; python3 lab/prism/storm.py'"), []) + # A string's state reaches neither the document nor another string, + # and a document line has none: its code spans are separate commands. + for text, expected in ( + ("bash -c 'set -n'; python3 lab/prism/deleted.py", [(1, "lab/prism/deleted.py")]), + ("sh -c 'set -n' && python3 lab/prism/deleted.py", [(1, "lab/prism/deleted.py")]), + ("bash -c 'sh -c \"set -n\"; python3 lab/prism/deleted.py'", [(1, "lab/prism/deleted.py")]), + ("```sh\nbash -c 'set -n'\npython3 lab/prism/deleted.py\n```", [(3, "lab/prism/deleted.py")]), + ("```sh\nset -n\npython3 lab/prism/deleted.py\n```", [(3, "lab/prism/deleted.py")]), + ("set -n; python3 lab/prism/deleted.py", [(1, "lab/prism/deleted.py")]), + ("Run `set -n` first, then `python3 lab/prism/deleted.py`.", [(1, "lab/prism/deleted.py")]), + ("```sh\nsh -c 'set -n\npython3 lab/prism/deleted.py'\nsh -c 'python3 lab/prism/storm.py'\n```", [(4, "lab/prism/storm.py")]), + ): + with self.subTest(text=text): + self.assertEqual(self.located(text), expected) + + def test_shell_option_arguments_are_not_command_strings(self) -> None: + command = "python3 -m lab.example.deleted" + for text in ( + f"bash -o '{command}' -c true", + f"bash -O '{command}' -c true", + f"bash --rcfile '{command}' -c true", + f"bash -o -c '{command}'", + f"bash -o pipefail -- -c '{command}'", + f"bash -o pipefail script.sh -c '{command}'", + f"bash --help -c '{command}'", + f"bash -o pipefail -c \"echo '{command}'\"", + "bash -o", "bash -O extglob -c", "bash --rcfile", + ): + with self.subTest(text=text): + self.assertEqual(self.commands(text), []) + + def test_shell_data_still_counts_as_dangling_prose(self) -> None: + for text in ( + "printf '%s\\n' 'python3 -m lab.prism.deleted'", + "cat <<'EOF'\npython3 -m lab.prism.deleted\nEOF", + ): + with self.subTest(text=text): + self.assertEqual(self.commands(text), []) + self.assertEqual(self.references(text), ["lab.prism.deleted"]) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_install_prism_qbit.py b/tests/test_install_prism_qbit.py new file mode 100644 index 000000000..4283594a6 --- /dev/null +++ b/tests/test_install_prism_qbit.py @@ -0,0 +1,299 @@ +#!/usr/bin/env python3 +"""Contract tests for the pinned qbit installer the PRISM native CI job runs. + +The installer only ever fetches from GitHub in CI, so these tests drive it +against a loopback HTTP server that plays the release download route and the +release-asset API route, each scripted to answer with the pinned bytes, other +bytes, or a gateway error. That keeps the fallback order, the digest check +before extraction, and the cache handling from regressing without a red test. +""" + +from __future__ import annotations + +import hashlib +import http.server +import io +import os +import re +import subprocess +import tarfile +import tempfile +import threading +import unittest +from pathlib import Path + + +ROOT = Path(__file__).resolve().parents[1] +SCRIPT = ROOT / ".github" / "scripts" / "install-prism-qbit.sh" +CI_WORKFLOW = ROOT / ".github" / "workflows" / "ci.yml" + +VERSION = "1.0.0" +ASSET = f"qbit-{VERSION}-x86_64-linux-gnu.tar.gz" +ASSET_ID = "478569140" +PINNED_SHA256 = "ae121af03263b55d530e3f3e8719a71362d0950cba82f54a2bc2d6c437a029b5" +RELEASE_PATH = f"/release/{ASSET}" +API_PATH = f"/api/assets/{ASSET_ID}" + + +def make_tarball(qbitd_body: str) -> bytes: + """A release-shaped tarball: qbit-<version>/bin/qbitd, executable.""" + payload = qbitd_body.encode("utf-8") + buffer = io.BytesIO() + with tarfile.open(fileobj=buffer, mode="w:gz") as archive: + info = tarfile.TarInfo(f"qbit-{VERSION}/bin/qbitd") + info.size = len(payload) + info.mode = 0o755 + archive.addfile(info, io.BytesIO(payload)) + return buffer.getvalue() + + +class RouteHandler(http.server.BaseHTTPRequestHandler): + def do_GET(self) -> None: # noqa: N802 - http.server API + self.server.requests.append((self.path, self.headers.get("Accept"))) + response = self.server.responses.get(self.path) + if response is None: + body = b"no such route" + self.send_response(404) + elif isinstance(response, int): + body = b"gateway timeout" + self.send_response(response) + else: + body = response + self.send_response(200) + self.send_header("Content-Type", "application/octet-stream") + self.send_header("Content-Length", str(len(body))) + self.end_headers() + self.wfile.write(body) + + def log_message(self, *_args: object) -> None: + pass + + +class RouteServer(http.server.ThreadingHTTPServer): + def __init__(self) -> None: + super().__init__(("127.0.0.1", 0), RouteHandler) + self.responses: dict[str, bytes | int] = {} + self.requests: list[tuple[str, str | None]] = [] + + def url(self, path: str) -> str: + return f"http://127.0.0.1:{self.server_address[1]}{path}" + + def count(self, path: str) -> int: + return sum(1 for seen, _accept in self.requests if seen == path) + + +class InstallPrismQbitTests(unittest.TestCase): + def setUp(self) -> None: + self.good = make_tarball("#!/bin/sh\necho pinned\n") + self.good_sha256 = hashlib.sha256(self.good).hexdigest() + self.other = make_tarball("#!/bin/sh\necho other\n") + self.assertNotEqual(self.good_sha256, hashlib.sha256(self.other).hexdigest()) + self.server = RouteServer() + thread = threading.Thread(target=self.server.serve_forever, daemon=True) + thread.start() + self.addCleanup(self.server.server_close) + self.addCleanup(self.server.shutdown) + self.tmp = Path(tempfile.mkdtemp(prefix="install-prism-qbit-")) + self.addCleanup(lambda: subprocess.run(["rm", "-rf", str(self.tmp)], check=False)) + self.destination = self.tmp / "dest" + self.cache = self.tmp / "cache" + self.github_env = self.tmp / "github.env" + # Exercise the production script with fixture pins in a disposable + # copy; the shipped installer has no environment override for them. + script = SCRIPT.read_text(encoding="utf-8") + substitutions = { + "sha256": self.good_sha256, + "release_url": self.server.url(RELEASE_PATH), + "asset_api_url": self.server.url(API_PATH), + "retries": "1", + "retry_max_time": "5", + } + for name, value in substitutions.items(): + script, count = re.subn(rf"(?m)^{name}=.*$", f'{name}="{value}"', script) + self.assertEqual(count, 1, name) + self.script = self.tmp / "install-prism-qbit.sh" + self.script.write_text(script, encoding="utf-8") + + def run_script(self, *, cache: bool = True) -> subprocess.CompletedProcess[str]: + env = os.environ.copy() + for proxy in ("http_proxy", "HTTP_PROXY", "all_proxy", "ALL_PROXY"): + env.pop(proxy, None) + env.update( + { + "GITHUB_ENV": str(self.github_env), + } + ) + if cache: + env["PRISM_QBIT_CACHE_DIR"] = str(self.cache) + else: + env.pop("PRISM_QBIT_CACHE_DIR", None) + return subprocess.run( + ["bash", str(self.script), str(self.destination)], + env=env, + capture_output=True, + text=True, + check=False, + ) + + @property + def qbitd(self) -> Path: + return self.destination / f"qbit-{VERSION}" / "bin" / "qbitd" + + def assert_installed(self, result: subprocess.CompletedProcess[str]) -> None: + self.assertEqual(result.returncode, 0, result.stderr) + self.assertTrue(os.access(self.qbitd, os.X_OK), self.qbitd) + self.assertEqual(self.qbitd.read_text(encoding="utf-8"), "#!/bin/sh\necho pinned\n") + self.assertEqual( + self.github_env.read_text(encoding="utf-8"), + f"QBITD_BIN={self.qbitd}\n", + ) + + def test_release_route_download_is_verified_extracted_and_cached(self) -> None: + self.server.responses[RELEASE_PATH] = self.good + self.server.responses[API_PATH] = self.good + + result = self.run_script() + + self.assert_installed(result) + self.assertEqual(self.server.count(RELEASE_PATH), 1, self.server.requests) + self.assertEqual(self.server.count(API_PATH), 0, self.server.requests) + self.assertEqual( + hashlib.sha256((self.cache / ASSET).read_bytes()).hexdigest(), + self.good_sha256, + ) + self.assertFalse((self.cache / f"{ASSET}.partial").exists()) + + def test_works_without_a_cache_directory(self) -> None: + self.server.responses[RELEASE_PATH] = self.good + + result = self.run_script(cache=False) + + self.assert_installed(result) + self.assertFalse(self.cache.exists()) + + def test_gateway_error_on_release_route_falls_back_to_the_asset_api(self) -> None: + self.server.responses[RELEASE_PATH] = 504 + self.server.responses[API_PATH] = self.good + + result = self.run_script() + + self.assert_installed(result) + # The original request plus the one configured retry, then the API. + self.assertEqual(self.server.count(RELEASE_PATH), 2, self.server.requests) + self.assertEqual(self.server.count(API_PATH), 1, self.server.requests) + api_accept = [accept for path, accept in self.server.requests if path == API_PATH] + self.assertEqual(api_accept, ["application/octet-stream"]) + self.assertIn("release download route", result.stderr) + self.assertIn("release-asset API route", result.stderr) + + def test_release_route_bytes_with_the_wrong_digest_are_rejected_before_extraction( + self, + ) -> None: + self.server.responses[RELEASE_PATH] = self.other + self.server.responses[API_PATH] = self.good + + result = self.run_script() + + self.assert_installed(result) + self.assertEqual(self.server.count(RELEASE_PATH), 1, self.server.requests) + self.assertEqual(self.server.count(API_PATH), 1, self.server.requests) + self.assertIn("digest mismatch", result.stderr) + self.assertIn(f"expected {self.good_sha256}", result.stderr) + + def test_fails_when_every_route_serves_the_wrong_digest(self) -> None: + self.server.responses[RELEASE_PATH] = self.other + self.server.responses[API_PATH] = self.other + + result = self.run_script() + + self.assertNotEqual(result.returncode, 0) + self.assertFalse(self.qbitd.exists()) + self.assertFalse((self.cache / ASSET).exists()) + self.assertEqual(result.stderr.count("digest mismatch"), 2, result.stderr) + self.assertIn("both failed", result.stderr) + self.assertEqual(self.github_env.exists(), False) + + def test_fails_when_every_route_is_unavailable(self) -> None: + self.server.responses[RELEASE_PATH] = 504 + self.server.responses[API_PATH] = 502 + + result = self.run_script() + + self.assertNotEqual(result.returncode, 0) + self.assertFalse(self.qbitd.exists()) + self.assertFalse(self.cache.exists()) + self.assertIn("the release download route did not deliver", result.stderr) + self.assertIn("the release-asset API route did not deliver", result.stderr) + self.assertIn("both failed", result.stderr) + + def test_verified_cached_tarball_needs_no_download(self) -> None: + self.cache.mkdir() + (self.cache / ASSET).write_bytes(self.good) + self.server.responses[RELEASE_PATH] = 504 + self.server.responses[API_PATH] = 504 + + result = self.run_script() + + self.assert_installed(result) + self.assertEqual(self.server.requests, []) + self.assertIn("using the cached", result.stderr) + + def test_cached_tarball_with_the_wrong_digest_is_replaced_not_extracted(self) -> None: + self.cache.mkdir() + (self.cache / ASSET).write_bytes(self.other) + self.server.responses[RELEASE_PATH] = self.good + + result = self.run_script() + + self.assert_installed(result) + self.assertIn("discarding the cached", result.stderr) + self.assertEqual(self.server.count(RELEASE_PATH), 1, self.server.requests) + self.assertEqual( + hashlib.sha256((self.cache / ASSET).read_bytes()).hexdigest(), + self.good_sha256, + ) + + def test_script_pins_the_v1_release_asset_and_both_github_routes(self) -> None: + script = SCRIPT.read_text(encoding="utf-8") + self.assertRegex(script, rf"(?m)^version={re.escape(VERSION)}$") + self.assertRegex(script, rf"(?m)^asset_id={ASSET_ID}$") + self.assertRegex(script, rf"(?m)^sha256={PINNED_SHA256}$") + self.assertEqual(set(re.findall(r"PRISM_QBIT_\w+", script)), {"PRISM_QBIT_CACHE_DIR"}) + self.assertIn( + "https://github.com/Qbit-Org/qbit/releases/download/v${version}/${asset}", + script, + ) + self.assertIn( + "https://api.github.com/repos/Qbit-Org/qbit/releases/assets/${asset_id}", + script, + ) + self.assertIn("--header 'Accept: application/octet-stream'", script) + self.assertIn("--retry-all-errors", script) + self.assertNotIn("Authorization", script) + self.assertNotIn("GITHUB_TOKEN", script) + self.assertTrue(os.access(SCRIPT, os.X_OK), SCRIPT) + + def test_ci_native_job_restores_installs_and_saves_the_pinned_tarball(self) -> None: + workflow = CI_WORKFLOW.read_text(encoding="utf-8") + native = workflow.split("\n prism-native-postgres:", 1)[1].split("\n docker-builds:", 1)[0] + self.assertIn("uses: actions/cache/restore@", native) + self.assertIn("uses: actions/cache/save@", native) + self.assertEqual( + native.count("hashFiles('.github/scripts/install-prism-qbit.sh')"), 2, native + ) + self.assertIn("if: steps.prism-qbit-cache.outputs.cache-hit != 'true'", native) + self.assertIn( + 'PRISM_QBIT_CACHE_DIR="${HOME}/.cache/prism-qbit" \\\n' + ' bash .github/scripts/install-prism-qbit.sh "${RUNNER_TEMP}/prism-qbit"', + native, + ) + # CI must run the pins as written in the script: only the cache + # directory is configured from the workflow. + self.assertEqual(set(re.findall(r"PRISM_QBIT_\w+", native)), {"PRISM_QBIT_CACHE_DIR"}) + # The inline download the script replaced must not come back. + self.assertNotIn("releases/download/v1.0.0", native) + self.assertNotIn(PINNED_SHA256, native) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_mainnet_compose_contract.py b/tests/test_mainnet_compose_contract.py index cf48dd466..5e580cb07 100644 --- a/tests/test_mainnet_compose_contract.py +++ b/tests/test_mainnet_compose_contract.py @@ -14,6 +14,7 @@ ROOT = Path(__file__).resolve().parents[1] FIXTURE = ROOT / "tests" / "fixtures" / "mainnet-compose.env" +SECRETS_TARGET = "/run/secrets/qbit-prism" OPERATOR_SERVICES = { "qbitd", "ckpool", @@ -90,6 +91,89 @@ def setUpClass(cls) -> None: f"stderr:\n{completed.stderr}" ) cls.config = json.loads(completed.stdout) + cls.compose_env = env + cls.ha_config = cls.render_ha() + + @classmethod + def render_ha(cls, **overrides: str) -> dict[str, Any]: + completed = subprocess.run( + [cls.docker, "compose", "--env-file", str(ROOT / "config/upstream.env.example"), + "--env-file", str(FIXTURE), "-f", str(ROOT / "compose.yaml"), + "-f", str(ROOT / "compose.production.yaml"), + "-f", str(ROOT / "compose.prism-ha.yaml"), + "--project-name", "qbit-mainnet-ha-contract", "--profile", "prism", + "config", "--format", "json"], + cwd=ROOT, env={**cls.compose_env, **overrides}, text=True, + stdout=subprocess.PIPE, stderr=subprocess.PIPE, check=False, + ) + if completed.returncode != 0: + raise AssertionError(f"HA compose render failed: {completed.stderr}") + return json.loads(completed.stdout) + + def test_ha_preserves_global_rpc_and_shared_database(self) -> None: + base = self._environment("prism-coordinator") + for name in ("prism-coordinator", "prism-coordinator-2"): + with self.subTest(service=name): + env = self.ha_config["services"][name]["environment"] + for key in ("QBIT_RPC_HOST", "QBIT_RPC_URL", "PRISM_DATABASE_URL"): + self.assertEqual(env[key], base[key], key) + self.assertEqual(env["PRISM_AUDIT_BIND"], "0.0.0.0") + + def test_ha_frontends_have_distinct_ids_and_published_ports(self) -> None: + pair = [self.ha_config["services"][name] + for name in ("prism-coordinator", "prism-coordinator-2")] + self.assertEqual(len({s["environment"]["PRISM_INSTANCE_ID"] for s in pair}), 2) + ports = [str(p["published"]) for s in pair for p in s["ports"] + if str(p["published"]) != "0"] + self.assertEqual(len(ports), len(set(ports))) + + def test_block_submission_kill_switch_defaults_on_and_reaches_both_frontends(self) -> None: + # #291: production submits; a rehearsal holds every frontend of the pair. + self.assertEqual( + self._environment("prism-coordinator")["PRISM_BLOCK_SUBMIT_ENABLED"], "1" + ) + held = self.render_ha(PRISM_BLOCK_SUBMIT_ENABLED="0") + for name in ("prism-coordinator", "prism-coordinator-2"): + with self.subTest(service=name): + default = self.ha_config["services"][name]["environment"] + self.assertEqual(default["PRISM_BLOCK_SUBMIT_ENABLED"], "1") + env = held["services"][name]["environment"] + self.assertEqual(env["PRISM_BLOCK_SUBMIT_ENABLED"], "0") + + def test_ha_requires_http_health_despite_base_listener_disable(self) -> None: + for port in ("3341", "25351"): + config = self.render_ha(PRISM_AUDIT_PORT="0", PRISM_HA_AUDIT_PORT=port) + for name in ("prism-coordinator", "prism-coordinator-2"): + with self.subTest(service=name, port=port): + service = config["services"][name] + self.assertEqual(service["environment"]["PRISM_AUDIT_PORT"], port) + self.assertIn(int(port), [p["target"] for p in service["ports"]]) + + def test_ha_per_frontend_overrides_win_over_base_values(self) -> None: + config = self.render_ha( + PRISM_HA_RPC_URL_1="https://east.example.invalid/rpc", + PRISM_HA_RPC_URL_2="https://west.example.invalid/rpc", + PRISM_HA_RPC_HOST_1="east.example.invalid", + PRISM_HA_RPC_HOST_2="west.example.invalid", + PRISM_HA_INSTANCE_ID_1="verify-east", PRISM_HA_INSTANCE_ID_2="verify-west", + PRISM_HA_STRATUM_PORT_HOST_1="127.0.0.1:25340", + PRISM_HA_STRATUM_PORT_HOST_2="127.0.0.1:25343", + PRISM_HA_HIGHDIFF_PORT_HOST_1="127.0.0.1:25345", + PRISM_HA_HIGHDIFF_PORT_HOST_2="127.0.0.1:25346", + PRISM_HA_AUDIT_BIND="::", + ) + for name, side, ports in ( + ("prism-coordinator", "east", {"25340", "25345"}), + ("prism-coordinator-2", "west", {"25343", "25346"}), + ): + with self.subTest(service=name): + service = config["services"][name] + env = service["environment"] + self.assertEqual(env["PRISM_INSTANCE_ID"], f"verify-{side}") + self.assertEqual(env["QBIT_RPC_HOST"], f"{side}.example.invalid") + self.assertEqual(env["QBIT_RPC_URL"], f"https://{side}.example.invalid/rpc") + self.assertEqual(env["PRISM_AUDIT_BIND"], "::") + self.assertLessEqual(ports, {str(p["published"]) for p in service["ports"]}) def test_profiles_render_expected_service_graph(self) -> None: services = set(self.config["services"]) @@ -156,6 +240,64 @@ def test_ctv_has_an_explicit_static_fee_rate(self) -> None: self.assertEqual(env["PRISM_CTV_FANOUT_FEE_MARKET_RATE_BITS_PER_1000_WEIGHT"], "1000") self.assertEqual(env["PRISM_CTV_FANOUT_FEE_PREMIUM_BPS"], "12000") + def test_ctv_settlement_runs_with_a_pool_fee(self) -> None: + # #525: without one, the first sub-floor balance stops all work. + env = self._environment("prism-coordinator") + + self.assertEqual(env["PRISM_CTV_SETTLEMENT_ENABLED"], "1") + self.assertEqual(env["PRISM_POOL_FEE_ENABLED"], "1") + self.assertEqual(env["PRISM_POOL_FEE_BPS"], "200") + self.assertEqual(env["PRISM_POOL_FEE_ADDRESS"], "qb1syntheticmainnetpoolfeeaddress") + + def test_role_rendered_compose_names_exactly_one_pool_fee_recipient(self) -> None: + # #535: the deploy role runs compose.yaml alone and renders the fee + # address without a program line. A recipient default in compose.yaml + # would put a program beside that address, and every server refuses + # "configure exactly one pool fee address or P2MR program". + fixture = [ + line + for line in FIXTURE.read_text(encoding="utf-8").splitlines() + if not line.startswith("PRISM_POOL_FEE_P2MR_PROGRAM_HEX=") + ] + with tempfile.TemporaryDirectory() as temp_dir: + deploy_env = Path(temp_dir) / "role.env" + deploy_env.write_text("\n".join(fixture) + "\n", encoding="utf-8") + for ctv in ("1", "0"): + with self.subTest(ctv=ctv): + env = self._render_coordinator( + deploy_env, PRISM_CTV_SETTLEMENT_ENABLED=ctv + ) + self.assertEqual(env["PRISM_CTV_SETTLEMENT_ENABLED"], ctv) + self.assertEqual(env["PRISM_POOL_FEE_ENABLED"], "1") + self.assertEqual( + env["PRISM_POOL_FEE_ADDRESS"], "qb1syntheticmainnetpoolfeeaddress" + ) + self.assertEqual(env["PRISM_POOL_FEE_P2MR_PROGRAM_HEX"], "") + # A deployment that names no fee gets none and no recipient: the + # server refuses to run with the settings to set, while every tool, + # policy-transition's current side included, still reads it. + with tempfile.TemporaryDirectory() as temp_dir: + deploy_env = Path(temp_dir) / "bare.env" + deploy_env.write_text("MINING_LANES=prism\n", encoding="utf-8") + env = self._render_coordinator(deploy_env) + self.assertEqual(env["PRISM_POOL_FEE_ENABLED"], "0") + self.assertEqual(env["PRISM_POOL_FEE_BPS"], "") + self.assertEqual(env["PRISM_POOL_FEE_ADDRESS"], "") + self.assertEqual(env["PRISM_POOL_FEE_P2MR_PROGRAM_HEX"], "") + + def _render_coordinator(self, deploy_env: Path, **overrides: str) -> dict[str, str]: + completed = subprocess.run( + [self.docker, "compose", "--env-file", str(ROOT / "config/upstream.env.example"), + "--env-file", str(deploy_env), "-f", str(ROOT / "compose.yaml"), + "--project-name", "qbit-role-pool-fee-contract", "--profile", "prism", + "config", "--format", "json", "prism-coordinator"], + cwd=ROOT, env={**self.compose_env, **overrides}, text=True, + stdout=subprocess.PIPE, stderr=subprocess.PIPE, check=False, + ) + self.assertEqual(completed.returncode, 0, completed.stderr) + raw_env = json.loads(completed.stdout)["services"]["prism-coordinator"]["environment"] + return {str(key): str(value) for key, value in raw_env.items()} + def test_prism_uses_explicit_non_lab_difficulty_profile(self) -> None: env = self._environment("prism-coordinator") @@ -163,8 +305,35 @@ def test_prism_uses_explicit_non_lab_difficulty_profile(self) -> None: self.assertEqual(env["PRISM_STRATUM_VARDIFF_MIN_DIFF"], "1024") self.assertEqual(env["PRISM_STRATUM_VARDIFF_START_DIFF"], "4096") self.assertEqual(env["PRISM_STRATUM_VARDIFF_MAX_DIFF"], "65536") - self.assertEqual(env["PRISM_TEMPLATE_REFRESH_FAILURE_EXIT_SECONDS"], "120") - self.assertEqual(env["PRISM_COORDINATION_BLOCKED_EXIT_SECONDS"], "900") + self.assertEqual(env["PRISM_DATABASE_MAX_CONNECTIONS"], "16") + self.assertEqual(env["PRISM_HEALTH_TIP_POLL_MAX_AGE_SECONDS"], "15") + + def test_prism_services_receive_the_complete_rpc_endpoint_override(self) -> None: + for service in ("prism-coordinator", "prism-public-api"): + with self.subTest(service=service): + self.assertEqual( + self._environment(service)["QBIT_RPC_URL"], + "https://rpc.example.invalid:8443/qbit/mainnet/rpc", + ) + + def test_prism_services_receive_configured_database_and_read_limits(self) -> None: + writer = self._environment("prism-coordinator") + self.assertEqual(writer["QBIT_TOOLS_PRODUCTION"], "1") + self.assertEqual(writer["PRISM_DATABASE_STATEMENT_TIMEOUT_MS"], "23000") + self.assertEqual(writer["PRISM_DATABASE_LOCK_TIMEOUT_MS"], "7000") + self.assertEqual(writer["PRISM_SHARE_COMMIT_TIMEOUT_SECONDS"], "19") + self.assertEqual(writer["PRISM_RPC_TIMEOUT_SECONDS"], "21") + self.assertEqual(writer["PRISM_BLOCK_SUBMIT_RPC_TIMEOUT_SECONDS"], "2") + self.assertEqual(writer["PRISM_CANDIDATE_ORPHAN_CONFIRMATIONS"], "17") + for service in ("prism-coordinator", "prism-public-api"): + with self.subTest(service=service): + env = self._environment(service) + self.assertEqual(env["PRISM_PUBLIC_READ_STATEMENT_TIMEOUT_SECONDS"], "9") + self.assertEqual(env["PRISM_POSTGRES_READ_CONCURRENCY"], "3") + self.assertEqual(env["PRISM_PUBLIC_AUDIT_REBUILD_CONCURRENCY"], "2") + self.assertEqual(env["PRISM_PUBLIC_AUDIT_ARTIFACT_MAX_IN_FLIGHT"], "11") + self.assertEqual(env["PRISM_PAYOUT_MIN_OUTPUT_BITS"], "24576") + self.assertEqual(env["PRISM_PAYOUT_MIN_OUTPUT_SATS"], "24576") def test_prism_runtime_has_no_capacity_evidence_dependency(self) -> None: env = self._environment("prism-coordinator") @@ -207,9 +376,6 @@ def test_operator_state_uses_explicit_host_bind_mounts(self) -> None: ("prism-postgres-replica", "/var/lib/postgresql/data"): ( "/srv/qbit-mining-bootstrap/mainnet/postgres-replica/data" ), - ("prism-public-api", "/var/lib/qbit-prism/audit"): ( - "/srv/qbit-mining-bootstrap/mainnet/prism/audit" - ), } for (service, target), source in expected.items(): @@ -227,6 +393,10 @@ def test_operator_state_uses_explicit_host_bind_mounts(self) -> None: mounts[0].get("bind", {}).get("create_host_path", False) ) + public = self.config["services"]["prism-public-api"] + self.assertEqual(public["command"], ["qbit-prism-server", "public-api"]) + self.assertFalse(public.get("volumes"), "native public audits come from PostgreSQL") + def test_postgres_wal_has_a_separate_storage_boundary(self) -> None: postgres = self.config["services"]["prism-postgres"] mounts = { @@ -359,13 +529,65 @@ def test_production_secrets_do_not_use_repository_defaults(self) -> None: self.assertNotIn("-rpcpassword=change-this", bitcoin_command) self.assertNotEqual(postgres_env["POSTGRES_PASSWORD"], "change-this") self.assertNotIn("change-this", prism_env["PRISM_DATABASE_URL"]) - self.assertEqual(prism_env["PRISM_LEDGER_WRITER_SESSION_TOKEN"], "") - self.assertEqual(len(prism_env["PRISM_MANIFEST_SIGNING_SEED_HEX"]), 64) - self.assertEqual(len(prism_env["PRISM_LEDGER_ATTESTATION_SIGNING_SEED_HEX"]), 64) - self.assertNotEqual( - prism_env["PRISM_MANIFEST_SIGNING_SEED_HEX"], - prism_env["PRISM_LEDGER_ATTESTATION_SIGNING_SEED_HEX"], + self.assertNotIn("PRISM_LEDGER_WRITER_SESSION_TOKEN", prism_env) + self.assertNotIn("d" * 64, json.dumps(self.config)) + self.assertNotIn("e" * 64, json.dumps(self.config)) + + def test_frontends_read_signing_seeds_only_from_the_read_only_mount(self) -> None: + # Issue #260 D4: the fixture's stale direct seeds must not reach a + # production frontend, including the second HA frontend. + for config, name in ( + (self.config, "prism-coordinator"), + (self.ha_config, "prism-coordinator"), + (self.ha_config, "prism-coordinator-2"), + ): + with self.subTest(service=name, ha=config is self.ha_config): + service = config["services"][name] + env = service["environment"] + self.assertEqual(env["PRISM_MANIFEST_SIGNING_SEED_HEX"], "") + self.assertEqual(env["PRISM_LEDGER_ATTESTATION_SIGNING_SEED_HEX"], "") + self.assertEqual(env["PRISM_ALLOW_TEST_SIGNING_SEEDS"], "0") + self.assertEqual( + env["PRISM_MANIFEST_SIGNING_SEED_HEX_FILE"], + f"{SECRETS_TARGET}/manifest-signing-seed-hex", + ) + self.assertEqual( + env["PRISM_LEDGER_ATTESTATION_SIGNING_SEED_HEX_FILE"], + f"{SECRETS_TARGET}/ledger-attestation-signing-seed-hex", + ) + mounts = [ + volume for volume in service["volumes"] + if isinstance(volume, dict) and volume.get("target") == SECRETS_TARGET + ] + self.assertEqual(len(mounts), 1) + self.assertEqual(mounts[0]["type"], "bind") + self.assertEqual( + mounts[0]["source"], "/srv/qbit-mining-bootstrap/mainnet/prism/secrets" + ) + self.assertTrue(mounts[0].get("read_only")) + self.assertFalse(mounts[0].get("bind", {}).get("create_host_path", False)) + + def test_public_read_tier_is_seedless(self) -> None: + public = self.config["services"]["prism-public-api"] + self.assertFalse( + [name for name in public["environment"] + if "SIGNING_SEED" in name or "OPERATOR_BEARER_TOKEN" in name] ) + self.assertFalse(public.get("volumes")) + + def test_prism_containers_disable_core_dumps(self) -> None: + for config, name in ( + (self.config, "prism-coordinator"), + (self.config, "prism-public-api"), + (self.ha_config, "prism-coordinator-2"), + ): + with self.subTest(service=name): + core = config["services"][name]["ulimits"]["core"] + # The rendered form omits zero-valued soft and hard limits. + limits = ( + (core.get("soft", 0), core.get("hard", 0)) if isinstance(core, dict) else (core, core) + ) + self.assertEqual(limits, (0, 0)) def test_bitcoin_build_uses_architecture_specific_checksums(self) -> None: args = self.config["services"]["bitcoind"]["build"]["args"] @@ -483,6 +705,12 @@ def test_missing_production_inputs_render_only_failure_sentinels(self) -> None: if isinstance(volume, dict) ) ) + secrets = [ + volume for volume in config["services"]["prism-coordinator"]["volumes"] + if isinstance(volume, dict) and volume.get("target") == SECRETS_TARGET + ] + self.assertEqual(len(secrets), 1) + self.assertTrue(secrets[0]["source"].startswith("/production-source-not-configured/")) @staticmethod def _network_selectors(command: list[str]) -> list[str]: diff --git a/tests/test_makefile_lifecycle.py b/tests/test_makefile_lifecycle.py index ef7944fae..f9c80f23f 100644 --- a/tests/test_makefile_lifecycle.py +++ b/tests/test_makefile_lifecycle.py @@ -181,6 +181,77 @@ def test_operator_build_mode_normalizes_production_and_main_chain_case(self) -> expected, ) + def run_prism_signing_preflight(self, **environment: str) -> subprocess.CompletedProcess[str]: + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + compose_env = root / "compose.env" + compose_env.write_text( + "".join(f"{name}={value}\n" for name, value in environment.items()), + encoding="utf-8", + ) + fake_compose = root / "fake-compose" + fake_compose.write_text( + "#!/bin/sh\n" + 'if [ "${1:-}" = config ] && [ "${2:-}" = --environment ]; then\n' + f" exec cat {shlex.quote(str(compose_env))}\n" + "fi\n" + "exit 9\n", + encoding="utf-8", + ) + fake_compose.chmod(0o755) + wrapper = root / "Makefile" + wrapper.write_text( + f"include {ROOT / 'Makefile'}\n\n" + "check-prism-signing:\n" + "\t@$(COMPOSE_ENV_HELPERS) $(PRISM_SIGNING_PREFLIGHT) printf 'preflight passed\\n'\n", + encoding="utf-8", + ) + return subprocess.run( + ["make", "--no-print-directory", "-s", "-f", str(wrapper), + "check-prism-signing", f"COMPOSE={fake_compose}"], + cwd=root, + text=True, + capture_output=True, + check=False, + ) + + def test_prism_signing_preflight_requires_mounted_seeds_only_in_production(self) -> None: + key = {"PRISM_LEDGER_WRITER_PUBLIC_KEY_HEX": "44" * 32} + direct = { + "PRISM_MANIFEST_SIGNING_SEED_HEX": "42" * 32, + "PRISM_LEDGER_ATTESTATION_SIGNING_SEED_HEX": "43" * 32, + } + for environment in ( + {**key, **direct}, + { + **key, + "PRISM_MANIFEST_SIGNING_SEED_HEX_FILE": "/run/secrets/manifest", + "PRISM_LEDGER_ATTESTATION_SIGNING_SEED_HEX_FILE": "/run/secrets/ledger", + }, + {**key, "QBIT_PRODUCTION": "1", "PRISM_SECRETS_SOURCE": "/srv/prism/secrets"}, + ): + with self.subTest(environment=sorted(environment)): + result = self.run_prism_signing_preflight(**environment) + self.assertEqual(result.returncode, 0, result.stderr) + self.assertIn("preflight passed", result.stdout) + + for environment, message in ( + ( + {**key, "PRISM_LEDGER_ATTESTATION_SIGNING_SEED_HEX": "43" * 32}, + "PRISM_MANIFEST_SIGNING_SEED_HEX or PRISM_MANIFEST_SIGNING_SEED_HEX_FILE is required", + ), + ({**key, **direct, "QBIT_PRODUCTION": "1"}, "PRISM_SECRETS_SOURCE is required"), + ( + {"QBIT_CHAIN": "mainnet", "PRISM_SECRETS_SOURCE": "/srv/prism/secrets"}, + "PRISM_LEDGER_WRITER_PUBLIC_KEY_HEX is required", + ), + ): + with self.subTest(message=message): + result = self.run_prism_signing_preflight(**environment) + self.assertNotEqual(result.returncode, 0) + self.assertIn(message, result.stderr) + self.assertNotIn("preflight passed", result.stdout) + def test_host_entrypoint_scripts_avoid_bash_4_case_expansion(self) -> None: for relative_path in ( "scripts/check-env.sh", @@ -211,7 +282,6 @@ def run_target( bitcoin_node_extra_args: str = "", compose_config_fails: bool = False, confirm: bool = False, - prism_public_stratum_url: str | None = None, ) -> tuple[subprocess.CompletedProcess[str], list[str]]: with tempfile.TemporaryDirectory() as tmp: tmp_path = Path(tmp) @@ -230,12 +300,6 @@ def run_target( printf 'QBIT_CHAIN_FLAG=%s\\n' "$FAKE_QBIT_CHAIN_FLAG" printf 'BITCOIN_CHAIN=%s\\n' "$FAKE_BITCOIN_CHAIN" printf 'BITCOIN_CHAIN_FLAG=%s\\n' "$FAKE_BITCOIN_CHAIN_FLAG" - for key in PRISM_MANIFEST_SIGNING_SEED_HEX PRISM_LEDGER_ATTESTATION_SIGNING_SEED_HEX PRISM_LEDGER_WRITER_PUBLIC_KEY_HEX; do - printf '%s=%s\\n' "$key" "$FAKE_PRISM_SIGNING_KEY" - done - if [ "${FAKE_PRISM_PUBLIC_STRATUM_URL+x}" = x ]; then - printf 'PRISM_PUBLIC_STRATUM_URL=%s\\n' "$FAKE_PRISM_PUBLIC_STRATUM_URL" - fi exit 0 fi printf '%s\\n' "$*" >> "$FAKE_COMPOSE_CALL_LOG" @@ -261,12 +325,8 @@ def run_target( "BITCOIN_CHAIN_FLAG": bitcoin_chain_flag, "QBIT_NODE_EXTRA_ARG": qbit_node_extra_arg, "BITCOIN_NODE_EXTRA_ARGS": bitcoin_node_extra_args, - "FAKE_PRISM_SIGNING_KEY": "11" * 32, } ) - env.pop("FAKE_PRISM_PUBLIC_STRATUM_URL", None) - if prism_public_stratum_url is not None: - env["FAKE_PRISM_PUBLIC_STRATUM_URL"] = prism_public_stratum_url if confirm: env["PURGE_CONFIRM"] = "delete-all-local-mining-data" else: @@ -280,10 +340,6 @@ def run_target( target, f"COMPOSE={fake_compose}", f"COMPOSE_ALL_PROFILES={fake_compose}", - f"PRODUCTION_COMPOSE={fake_compose}", - # Source staging is covered separately; exercise the - # target's own preflight before any service can start. - "WITH_RESOLVED_QBIT=set -euo pipefail;", ], cwd=ROOT, env=env, @@ -294,36 +350,6 @@ def run_target( calls = call_log.read_text(encoding="utf-8").splitlines() if call_log.exists() else [] return result, calls - def test_prism_start_rejects_missing_or_blank_public_stratum_url(self) -> None: - for production in ("0", "1"): - for url in (None, "", " \t "): - with self.subTest(production=production, url=url): - result, calls = self.run_target( - "up-prism-pool", - qbit_production=production, - prism_public_stratum_url=url, - ) - self.assertNotEqual(result.returncode, 0) - self.assertIn("PRISM_PUBLIC_STRATUM_URL is required", result.stderr) - self.assertEqual(calls, []) - - def test_prism_start_with_public_stratum_url_launches_both_services(self) -> None: - for production, launch_flags in ( - ("0", "up --build"), - ("1", "up -d --no-build --pull never"), - ): - with self.subTest(production=production): - result, calls = self.run_target( - "up-prism-pool", - qbit_production=production, - prism_public_stratum_url="stratum+tcp://pool.example:3340", - ) - self.assertEqual(result.returncode, 0, result.stderr) - self.assertEqual( - calls, - [f"--profile prism {launch_flags} qbitd prism-postgres prism-coordinator prism-public-api"], - ) - def test_down_stops_all_profiles_without_deleting_volumes(self) -> None: result, calls = self.run_target("down") diff --git a/tests/test_measure_postgres_wal.py b/tests/test_measure_postgres_wal.py new file mode 100644 index 000000000..2d23d7eca --- /dev/null +++ b/tests/test_measure_postgres_wal.py @@ -0,0 +1,49 @@ +import pathlib, subprocess, sys, unittest +from unittest.mock import patch +from scripts.measure_postgres_wal import measure +ROOT = pathlib.Path(__file__).resolve().parents[1] +HELPER = ROOT / "scripts" / "measure_postgres_wal.py" + +class WalHelperTests(unittest.TestCase): + @patch("scripts.measure_postgres_wal.psql", side_effect=RuntimeError("PostgreSQL command failed")) + def test_initial_connection_failure_is_distinct(self, _): + with self.assertRaisesRegex(RuntimeError, "initial connection failed"): + measure("postgresql://127.0.0.1/db", "select 1", "unit", "test") + def test_invalid_dsn_fails_truthfully(self): + p=subprocess.run([sys.executable, str(HELPER),"--dsn","postgresql://invalid.invalid/x","--sql","select 1","--config","test"], text=True, capture_output=True, cwd=ROOT) + self.assertNotEqual(p.returncode, 0) + @patch("scripts.measure_postgres_wal.psql", side_effect=["0/10", "ok", "0/30", "32"]) + def test_success_reports_positive_delta_and_scope(self, _): + result = measure("postgresql://127.0.0.1/db", "select 1", "unit", "test") + self.assertEqual(result["outcome"], "ok"); self.assertEqual(result["wal_bytes"], 32.0); self.assertEqual(result["measurement_scope"], "server-wide") + @patch("scripts.measure_postgres_wal.psql", side_effect=["0/10", RuntimeError("PostgreSQL command failed")]) + def test_operation_failure_is_distinct(self, _): + result = measure("postgresql://127.0.0.1/db", "bad", "unit", "test") + self.assertEqual(result["outcome"], "operation_failed"); self.assertEqual(result["side_effect_status"], "unknown") + @patch("scripts.measure_postgres_wal.psql", side_effect=["0/10", "ok", RuntimeError("PostgreSQL command failed")]) + def test_end_lsn_failure_preserves_success(self, _): + result = measure("postgresql://127.0.0.1/db", "select 1", "unit", "test") + self.assertEqual(result["outcome"], "measurement_failed"); self.assertEqual(result["operation_status"], "succeeded") + @patch("scripts.measure_postgres_wal.psql", side_effect=["0/10", "ok", "0/30", RuntimeError("PostgreSQL command failed")]) + def test_delta_failure_preserves_success(self, _): + result = measure("postgresql://127.0.0.1/db", "select 1", "unit", "test") + self.assertEqual(result["measurement_status"], "failed"); self.assertEqual(result["execution_stage"], "post-operation") + def test_empty_sql_rejected(self): + p = subprocess.run([sys.executable, str(HELPER), "--dsn", "postgresql://127.0.0.1/db", "--sql", " ", "--config", "test"], capture_output=True, text=True, cwd=ROOT) + self.assertEqual(p.returncode, 1) + self.assertIn("invalid_input", p.stdout) + def test_password_and_query_dsn_rejected(self): + for dsn in ("postgresql://u:p@127.0.0.1/db", "postgresql://127.0.0.1/db?hostaddr=evil"): + p = subprocess.run([sys.executable, str(HELPER), "--dsn", dsn, "--sql", "select 1", "--config", "test"], capture_output=True, text=True, cwd=ROOT) + self.assertEqual(p.returncode, 1) + self.assertIn("invalid_input", p.stdout) + @patch("scripts.measure_postgres_wal.psql", side_effect=RuntimeError("PostgreSQL command timed out")) + def test_timeout_is_sanitized(self, _): + with self.assertRaises(RuntimeError) as ctx: measure("postgresql://127.0.0.1/db", "select 1", "u", "t", 1) + self.assertIn("timed out", str(ctx.exception)) + @patch("scripts.measure_postgres_wal.psql", side_effect=RuntimeError("psql executable is unavailable")) + def test_missing_executable_is_sanitized(self, _): + with self.assertRaises(RuntimeError) as ctx: measure("postgresql://127.0.0.1/db", "select 1", "u", "t", 1) + self.assertIn("unavailable", str(ctx.exception)) + +if __name__ == "__main__": unittest.main() diff --git a/tests/test_mining_compose_profiles.py b/tests/test_mining_compose_profiles.py index 72f575bdd..e40c9e232 100644 --- a/tests/test_mining_compose_profiles.py +++ b/tests/test_mining_compose_profiles.py @@ -4,20 +4,11 @@ import json import os -import shlex import shutil import subprocess -import tempfile import unittest -import urllib.parse -from contextlib import nullcontext from pathlib import Path from typing import Any -from unittest.mock import Mock, patch - -from lab.prism import public_api, public_read_service -from lab.prism.coordinator_config import CoordinatorConfig, load_coordinator_config -from lab.prism.prism_coordinator import JsonRpc, PrismCoordinator ROOT = Path(__file__).resolve().parents[1] @@ -42,13 +33,10 @@ def setUpClass(cls) -> None: raise unittest.SkipTest(f"docker compose is unavailable: {version.stderr.strip()}") cls.docker = docker - def render_profile( - self, profile: str, overrides: dict[str, str] | None = None - ) -> dict[str, Any]: + def render_profile(self, profile: str) -> dict[str, Any]: inherited_keys = ("PATH", "HOME", "DOCKER_HOST", "DOCKER_CONTEXT", "XDG_CONFIG_HOME") env = {key: os.environ[key] for key in inherited_keys if key in os.environ} env["QBIT_SRC_DIR"] = str(ROOT) - env.update(overrides or {}) completed = subprocess.run( [ self.docker, @@ -82,254 +70,6 @@ def render_profile( ) return json.loads(completed.stdout) - def test_replica_retry_controls_reach_the_container(self) -> None: - attempts = "PRISM_POSTGRES_REPLICA_BASEBACKUP_ATTEMPTS" - retry_seconds = "PRISM_POSTGRES_REPLICA_BASEBACKUP_RETRY_SECONDS" - for overrides, expected in ( - ({}, {attempts: "60", retry_seconds: "5"}), - ({attempts: "120", retry_seconds: "2"}, {attempts: "120", retry_seconds: "2"}), - ): - with self.subTest(overrides=overrides): - config = self.render_profile("prism", overrides) - env = config["services"]["prism-postgres-replica"]["environment"] - for key, value in expected.items(): - self.assertEqual(env.get(key), value, key) - - def test_public_psql_uses_replica_configuration(self) -> None: - primary_command = "psql --host=primary.internal --dbname=qbit" - replica_url = "postgresql://reader@replica.internal:5432/qbit" - replica_command = "psql --host=replica.internal --dbname=qbit --username=reader" - for public_command in ("", replica_command): - with self.subTest(public_command=public_command): - config = self.render_profile("prism", { - "PRISM_POSTGRES_NATIVE_CLIENT": "psql", - "PRISM_POSTGRES_PSQL_COMMAND": primary_command, - "PRISM_PUBLIC_DATABASE_URL": replica_url, - "PRISM_PUBLIC_PSQL_COMMAND": public_command, - }) - services = config["services"] - self.assertEqual( - services["prism-coordinator"]["environment"]["PRISM_POSTGRES_PSQL_COMMAND"], - primary_command, - ) - env = services["prism-public-api"]["environment"] - self.assertEqual(env["PRISM_DATABASE_URL"], replica_url) - env = {key: value for key, value in env.items() if value is not None} - with patch.dict(os.environ, env, clear=True), patch.object( - public_read_service, "build_audit_artifact_store", return_value=None - ): - ledger = public_read_service.build_ledger_from_env(env) - try: - expected = shlex.split(public_command) if public_command else ["psql", replica_url] - self.assertEqual(ledger._command, expected) - self.assertIsNone(ledger._native) - self.assertTrue(ledger._read_only) - finally: - ledger.close() - - def test_public_database_url_uses_configured_replica_credentials(self) -> None: - for user, password, database in ( - ("pool_reader", "synthetic-rotated-password", "pool_ledger"), - ("reader/@?#%: ü", "synthetic/@?#%:' ü", "ledger/@?#% ü"), - ): - credentials = { - "PRISM_POSTGRES_USER": user, - "PRISM_POSTGRES_PASSWORD": password, - "PRISM_POSTGRES_DB": database, - "PRISM_POSTGRES_NATIVE_CLIENT": "psql", - "PRISM_DATABASE_URL": "postgresql://writer@primary.internal/pool_ledger", - } - quote = lambda value: urllib.parse.quote(value, safe="") - derived_url = (f"postgresql://{quote(user)}:{quote(password)}" - f"@prism-postgres-replica:5432/{quote(database)}") - custom_url = "postgresql://reader@standby.internal/pool_ledger" - for overrides, expected in ( - (credentials, derived_url), - ({**credentials, "PRISM_PUBLIC_DATABASE_URL": ""}, derived_url), - ({**credentials, "PRISM_PUBLIC_DATABASE_URL": custom_url}, custom_url), - ): - with self.subTest(user=user, explicit=overrides.get("PRISM_PUBLIC_DATABASE_URL")): - services = self.render_profile("prism", overrides)["services"] - env = {k: v for k, v in services["prism-public-api"]["environment"].items() - if v is not None} - with patch.dict(os.environ, env, clear=True), patch.object( - public_read_service, "build_audit_artifact_store", return_value=None - ): - ledger = public_read_service.build_ledger_from_env(env) - try: - self.assertEqual(ledger._command, ["psql", expected]) - if expected == derived_url: - parsed = urllib.parse.urlsplit(ledger._command[1]) - self.assertEqual(urllib.parse.unquote(parsed.username), user) - self.assertEqual(urllib.parse.unquote(parsed.password), password) - self.assertEqual(urllib.parse.unquote(parsed.path[1:]), database) - finally: - ledger.close() - self.assertEqual( - services["prism-coordinator"]["environment"]["PRISM_DATABASE_URL"], - credentials["PRISM_DATABASE_URL"], - ) - - def test_public_http_limits_reach_the_running_server(self) -> None: - services = self.render_profile("prism", { - "PRISM_PUBLIC_HTTP_MAX_CONNECTIONS": "7", - "PRISM_PUBLIC_HTTP_TIMEOUT_SECONDS": "1.25", - "PRISM_PUBLIC_API_BIND": "127.0.0.1", - })["services"] - env = {k: v for k, v in services["prism-public-api"]["environment"].items() - if v is not None} - # The test listener uses an ephemeral port; Compose needs a real target. - env["PRISM_PUBLIC_API_PORT"] = "0" - observed = [] - - def inspect_server(server): - self.assertEqual(server.connection_timeout, 1.25) - for _ in range(7): - self.assertTrue(server._connection_slots.acquire(blocking=False)) - self.assertFalse(server._connection_slots.acquire(blocking=False)) - for _ in range(7): - server._connection_slots.release() - observed.append(server) - - with patch.dict(os.environ, env, clear=True), patch.object( - public_read_service, "build_service", - return_value=(object(), Mock(), public_read_service.ServiceMetrics(), None), - ), patch.object(public_read_service.signal, "signal"), patch.object( - public_read_service.BoundedPublicHTTPServer, "serve_forever", inspect_server, - ): - self.assertEqual(public_read_service.main(), 0) - self.assertEqual(len(observed), 1) - - def test_public_minimum_payout_preserves_pool_fallbacks(self) -> None: - for overrides, expected in ( - ({}, 0), - ({"PRISM_PAYOUT_MIN_OUTPUT_SATS": "500"}, 500), - ({"PRISM_PAYOUT_MIN_OUTPUT_BITS": "700", "PRISM_PAYOUT_MIN_OUTPUT_SATS": "500"}, 700), - ({"PRISM_PUBLIC_MINIMUM_PAYOUT_BITS": "900", "PRISM_PAYOUT_MIN_OUTPUT_BITS": "700"}, 900), - ({"PRISM_PUBLIC_MINIMUM_PAYOUT_BITS": "0", "PRISM_PAYOUT_MIN_OUTPUT_SATS": "500"}, 0), - ): - with self.subTest(overrides=overrides): - services = self.render_profile("prism", overrides)["services"] - env = services["prism-public-api"]["environment"] - env = {key: value for key, value in env.items() if value is not None} - with patch.dict(os.environ, env, clear=True): - self.assertEqual(public_api.public_minimum_payout_bits(), expected) - - def coordinator_config(self, overrides: dict[str, str]) -> CoordinatorConfig: - services = self.render_profile("prism", overrides)["services"] - env = services["prism-coordinator"]["environment"] - env = {key: value for key, value in env.items() if value is not None} - # Supply fixture signing material so configuration validation reaches - # the runtime knobs without starting a coordinator or opening storage. - env.update({ - "PRISM_MANIFEST_SIGNING_SEED_HEX": "11" * 32, - "PRISM_LEDGER_ATTESTATION_SIGNING_SEED_HEX": "22" * 32, - "PRISM_LEDGER_WRITER_PUBLIC_KEY_HEX": "33" * 32, - }) - return load_coordinator_config(env) - - def test_vardiff_resume_controls_reach_runtime_config(self) -> None: - for overrides, expected in ( - ({}, (True, 900, 8192, 1024)), - ({ - "PRISM_STRATUM_VARDIFF_RESUME": "0", - "PRISM_STRATUM_VARDIFF_RESUME_TTL_SECONDS": "42.5", - "PRISM_STRATUM_VARDIFF_RESUME_MAX_ENTRIES": "33", - "PRISM_STRATUM_VARDIFF_RESUME_MAX_START_FACTOR": "8", - }, (False, 42.5, 33, 8)), - ): - with self.subTest(overrides=overrides): - config = self.coordinator_config(overrides).stratum - self.assertEqual(( - config.vardiff_resume_enabled, - config.vardiff_resume_ttl_seconds, - config.vardiff_resume_max_entries, - config.vardiff_resume_max_start_factor, - ), expected) - - def test_rust_window_switch_reaches_runtime_config(self) -> None: - for overrides, expected in ( - ({}, False), - ({"PRISM_WINDOW_PIPELINE_RUST": "1"}, True), - ({"PRISM_WINDOW_PIPELINE_RUST": "0"}, False), - ): - with self.subTest(overrides=overrides): - config = self.coordinator_config(overrides) - self.assertEqual(config.jobs.window_pipeline_rust_enabled, expected) - with self.assertRaisesRegex(SystemExit, "PRISM_WINDOW_PIPELINE_RUST"): - self.coordinator_config({"PRISM_WINDOW_PIPELINE_RUST": "invalid"}) - - def test_hashrate_rollup_controls_reach_coordinator(self) -> None: - for overrides, expected in ( - ({}, (True, 15.0, 50000)), - ({ - "PRISM_HASHRATE_ROLLUP_ENABLED": "0", - "PRISM_HASHRATE_ROLLUP_INTERVAL_SECONDS": "42.5", - "PRISM_HASHRATE_ROLLUP_BATCH_SHARES": "1000", - }, (False, 42.5, 1000)), - ): - with self.subTest(overrides=overrides), tempfile.TemporaryDirectory() as root: - services = self.render_profile("prism", overrides)["services"] - env = services["prism-coordinator"]["environment"] - env = {key: value for key, value in env.items() if value is not None} - # The constructor reads these controls from the process - # environment. Keep its unrelated storage and RPC local. - config = load_coordinator_config({ - "QBIT_RPC_HOST": "qbit.example", - "QBIT_RPC_USER": "rpc-user", - "QBIT_RPC_PASSWORD": "rpc-password", - "PRISM_ALLOW_MEMORY_LEDGER": "1", - "PRISM_ALLOW_TEST_SIGNING_SEEDS": "1", - "PRISM_ALLOW_BUNDLE_EMBEDDED_LEDGER_KEY": "1", - "PRISM_AUDIT_DIR": root, - "PRISM_EVIDENCE_PATH": str(Path(root) / "evidence.json"), - }) - with patch.dict(os.environ, env, clear=True), patch.object( - JsonRpc, "call", side_effect=RuntimeError("offline") - ): - coordinator = PrismCoordinator(config) - self.assertEqual(( - coordinator.hashrate_rollup_enabled, - coordinator.hashrate_rollup_interval_seconds, - coordinator.hashrate_rollup_batch_shares, - ), expected) - - def test_public_read_deadline_reaches_origin_dispatch(self) -> None: - for overrides, expected in ( - ({}, 20), - ({"PRISM_PUBLIC_READ_STATEMENT_TIMEOUT_SECONDS": "7"}, 7), - ({"PRISM_PUBLIC_READ_STATEMENT_TIMEOUT_SECONDS": "0"}, 0), - ): - with self.subTest(overrides=overrides): - services = self.render_profile("prism", overrides)["services"] - env = services["prism-public-api"]["environment"] - env = {key: value for key, value in env.items() if value is not None} - coordinator = Mock() - timeout = coordinator.ledger.operation_timeout - timeout.return_value = nullcontext() - with patch.dict(os.environ, env, clear=True), patch.object( - public_api, "dispatch", return_value=(200, {"ok": True}) - ) as dispatch: - result = public_read_service.bounded_public_dispatch( - coordinator, "/public/v1/pool", {}, occupies_read_slot=True - ) - self.assertEqual(result, (200, {"ok": True})) - dispatch.assert_called_once_with(coordinator, "/public/v1/pool", {}) - if expected: - timeout.assert_called_once_with(float(expected)) - else: - timeout.assert_not_called() - - def test_candidate_cleanup_backlog_limit_reaches_runtime_config(self) -> None: - key = "PRISM_BLOCK_CANDIDATE_CLEANUP_RETRY_BACKLOG_MAX" - for overrides, expected in (({}, 4096), ({key: "128"}, 128), ({key: "8192"}, 8192)): - with self.subTest(overrides=overrides): - config = self.coordinator_config(overrides) - self.assertEqual(config.block.candidate_cleanup_retry_backlog_max, expected) - for invalid in ("0", "8193"): - with self.subTest(invalid=invalid), self.assertRaisesRegex(SystemExit, key): - self.coordinator_config({key: invalid}) - def test_each_mining_profile_has_an_exact_service_graph(self) -> None: expected = { "permissionless": {"qbitd", "ckpool", "permissionless-miner"}, diff --git a/tests/test_prism_accepted_block_accounting.py b/tests/test_prism_accepted_block_accounting.py deleted file mode 100644 index 77b262cba..000000000 --- a/tests/test_prism_accepted_block_accounting.py +++ /dev/null @@ -1,656 +0,0 @@ -#!/usr/bin/env python3 -"""Definitive-acceptance priority at the block-accounting handoff (#181). - -The 2026-08-21 testnet4 run left a tail of accepted-parent preview waits -- -the first two at only 0.32 PH/s -- behind stale same-height replay work that -the accounting lane served first. The task that carries the decided block -already knows it is decided: its ``_BlockCandidateNodeSubmission`` is an -attempted offer that raised nothing and returned nothing. - -These tests own that classification and the lane it selects: - -* the discriminator itself, including the two shapes that are easy to - misread as acceptance (a transport failure, which also leaves ``result`` - at ``None``, and a ``"duplicate"`` rejection); -* the issue's own counterexample -- three durable same-height replay tasks - queued before one definitively accepted task -- now draining accepted - first; -* the burst case, where the primary handoff queue is at its bound and - spillover has already begun. This is the case a priority *key* on the - primary queue cannot reach: once the overflow queue is non-empty every - later handoff joins it, and the loop drains the primary in full before it - ever looks at the overflow, so the accepted task would be served last - exactly when it matters most; -* the fairness bound that keeps the accepted lane from becoming a - starvation lane, stated as a bound rather than as an argument; and -* the spillover put rule's own anti-starvation property, which this change - leaves byte-for-byte alone and which is asserted here so it stays that - way. -""" - -from __future__ import annotations - -import itertools -import sys -import threading -import unittest -from contextlib import redirect_stdout -from io import StringIO -from pathlib import Path -from types import SimpleNamespace -from typing import Any, Callable - -if not __package__: - sys.path.insert(0, str(Path(__file__).resolve().parents[1])) - -from lab.prism.block_candidates import ( # noqa: E402 - BLOCK_ACCOUNTING_ACCEPTED_DISPATCH_QUOTA, - DEFAULT_BLOCK_ACCOUNTING_QUEUE_DEPTH, - _BlockCandidateAccountingTask, - _BlockCandidateNodeSubmission, - _is_definitive_node_acceptance, -) -from lab.prism.share_ledger import ( # noqa: E402 - PendingShare, - SingleWriterShareLedger, -) -from tests.prism_vardiff_test_support import ( # noqa: E402 - RejectingSubmitTipRpc, - block_candidate, - submit_coordinator, -) - - -# The parent every seeded job builds on, and the tip the chain moved to -# while the stale candidate sat in the durable outbox. -PARENT_TIP = "00" * 32 -MOVED_TIP = "77" * 32 - - -class AccountingLaneRig: - """Drive the shipped accounting loop over controllable lane traffic. - - Only the three sinks the loop reaches are replaced -- the accounting task - runner, the invalid-candidate quarantine drain and the deferred-cleanup - retry -- so the lane selection under test, the three queues, the handoff - routing and the dispatch quota all stay exactly as shipped. A real - :class:`_BlockCandidateAccountingTask` is still run through the shipped - ``_run_block_accounting_task``; only the synthetic placeholders are - counted and dropped. - - The loop runs on the calling thread, which makes dispatch order a plain - list rather than a race. It is bounded twice over: the quarantine stub - counts the loop's empty passes and stops it after a couple of them, and - a watchdog timer sets the stop event if the loop somehow never reaches - an empty pass. A starved lane therefore ends the test with a report - instead of an unbounded wait. - """ - - def __init__( - self, - *, - quota: int | None = None, - idle_passes_before_stop: int = 2, - ledger: Any = None, - rpc: Any = None, - ) -> None: - server, state, _recording = submit_coordinator() - server.max_blocks = 1 << 30 - server.stop_after_block = False - server.block_candidate_retry_initial_seconds = 0.0 - if quota is not None: - server.block_accounting_accepted_dispatch_quota = int(quota) - if ledger is not None: - server.ledger = ledger - if rpc is not None: - server.rpc = rpc - self.server = server - self.state = state - self.ledger = server.ledger - self.rpc = server.rpc - self.service = server._ensure_block_candidate_service() - self.service._ensure_block_accounting_state() - self.service._ensure_block_replay_state() - self.service._ensure_block_candidate_disposition_state() - # Dispatch order, one label per task the lane selected. - self.dispatched: list[str] = [] - # Called after each dispatch so a scenario can hand off more work - # mid-drain, which is what the submitter thread does in production. - self.on_dispatch: Callable[[Any], None] | None = None - self._shipped_run = self.service._run_block_accounting_task - self._idle_passes = 0 - self._idle_ceiling = int(idle_passes_before_stop) - self._labels = itertools.count() - server._run_block_accounting_task = self._run_task - server._run_one_invalid_block_candidate_quarantine = self._idle_pass - self.service._run_one_collapsed_block_candidate_cleanup_retry = ( - self._no_cleanup - ) - - # -- the replaced sinks ------------------------------------------------ - - def _run_task(self, task: Any) -> None: - self.dispatched.append(self.label(task)) - if isinstance(task, _BlockCandidateAccountingTask): - self._shipped_run(task) - if self.on_dispatch is not None: - self.on_dispatch(task) - - def _idle_pass(self) -> bool: - self._idle_passes += 1 - if self._idle_passes >= self._idle_ceiling: - self.server.stop_event.set() - return False - - @staticmethod - def _no_cleanup() -> bool: - return False - - # -- traffic ----------------------------------------------------------- - - @staticmethod - def label(task: Any) -> str: - return str( - getattr(task, "label", None) - or task.candidate.submission.block_hash_hex - ) - - def enqueue( - self, - label: str, - *, - accepted: bool, - height: int = 10, - ) -> Any: - """Hand one synthetic task to the shipped routing. - - The task carries only what ``_enqueue_block_accounting_task`` reads: - the template height it keys on, the hash its spill log names, and the - node submission whose classification chooses the lane. - """ - node_submission = ( - _BlockCandidateNodeSubmission(attempted=True) - if accepted - else _BlockCandidateNodeSubmission(attempted=False) - ) - task = SimpleNamespace( - label=label, - candidate=SimpleNamespace( - context=SimpleNamespace(template={"height": int(height)}), - submission=SimpleNamespace(block_hash_hex=label), - ), - node_submission=node_submission, - ) - # The spill log is part of the rule this scenario exercises; it is - # captured rather than printed so the suite stays readable. - with redirect_stdout(StringIO()): - assert self.server._enqueue_block_accounting_task(task) - return task - - def accepted_stream(self, *, until: int) -> Callable[[Any], None]: - """Keep the accepted lane non-empty for the whole drain. - - Every accepted dispatch immediately hands off another accepted task, - so the lane is never observed empty while stale work waits -- which - is the only condition under which the fairness bound says anything. - The stream stops after ``until`` total dispatches so the drain ends. - """ - - def rearm(task: Any) -> None: - if len(self.dispatched) >= until: - return - if _is_definitive_node_acceptance(task.node_submission): - self.enqueue( - f"accepted-{next(self._labels)}", - accepted=True, - ) - - return rearm - - # -- the drain --------------------------------------------------------- - - def drain(self, *, timeout: float = 15.0) -> list[str]: - watchdog = threading.Timer(timeout, self.server.stop_event.set) - watchdog.start() - try: - with redirect_stdout(StringIO()): - self.service.block_accounting_loop() - finally: - watchdog.cancel() - return list(self.dispatched) - - # -- queue inspection -------------------------------------------------- - - @property - def accepted_queue(self) -> Any: - return self.service._block_accounting_accepted_queue - - @property - def primary_queue(self) -> Any: - return self.service._block_accounting_queue - - @property - def overflow_queue(self) -> Any: - return self.service._block_accounting_overflow_queue - - -def durable_candidate( - rig: AccountingLaneRig, - block_hash: str, - *, - tag: str = "aa", -) -> Any: - """Seed one durable pending outbox row and return its candidate.""" - pending = PendingShare( - share_id=f"miner-a:{block_hash}", - miner_id="miner-a", - order_key="miner-a", - p2mr_program_hex="11" * 32, - share_difficulty=1, - network_difficulty=1, - template_height=9, - job_id="job-1", - job_issued_at_ms=1, - accepted_at_ms=1, - ntime=1, - ) - candidate = block_candidate( - rig.server, - rig.state, - SimpleNamespace( - coinbase_tx_hex="00", - block_hash_hex=block_hash, - block_hex=tag, - share_pass=True, - block_pass=True, - ), - pending_share=pending, - ) - rig.ledger.append_batch( - [(pending, rig.server.block_candidate_intent(candidate))] - ) - return candidate - - -def durable_accounting_task( - rig: AccountingLaneRig, - candidate: Any, -) -> _BlockCandidateAccountingTask: - """Queue the durable candidate the way the deferred handoff leaves it. - - ``attempted=False`` is the replay shape: the row was adopted from the - outbox, so no offer has been made for it yet and it is not accepted - evidence. The disposition lease travels with the task until the - accounting lane releases it. - """ - block_hash = str(candidate.submission.block_hash_hex) - lease = rig.server._claim_block_candidate_disposition( - block_hash, - blocking=False, - ) - assert lease is not None - task = _BlockCandidateAccountingTask( - candidate=candidate, - node_submission=_BlockCandidateNodeSubmission(attempted=False), - disposition_lease=lease, - ) - assert rig.server._enqueue_block_accounting_task(task) - return task - - -class DefinitiveNodeAcceptanceTests(unittest.TestCase): - """The discriminator, and the two sites that must never disagree.""" - - def test_only_an_attempted_silent_offer_is_definitive_acceptance( - self, - ) -> None: - self.assertTrue( - _is_definitive_node_acceptance( - _BlockCandidateNodeSubmission(attempted=True) - ) - ) - # A transport failure builds attempted=True with error set, which - # leaves result at its None default: a two-clause test that dropped - # `error is None` would read this failed offer as an acceptance. - self.assertFalse( - _is_definitive_node_acceptance( - _BlockCandidateNodeSubmission( - attempted=True, - error=RuntimeError("submitblock timed out"), - ) - ) - ) - # A rejection string is a result, so it is not silent acceptance. - self.assertFalse( - _is_definitive_node_acceptance( - _BlockCandidateNodeSubmission(attempted=True, result="duplicate") - ) - ) - # No offer has been made at all. - self.assertFalse( - _is_definitive_node_acceptance( - _BlockCandidateNodeSubmission(attempted=False) - ) - ) - self.assertFalse(_is_definitive_node_acceptance(None)) - - def test_the_handoff_routes_exactly_the_definitive_acceptances( - self, - ) -> None: - rig = AccountingLaneRig() - rig.enqueue("accepted", accepted=True) - self.assertEqual(rig.accepted_queue.qsize(), 1) - self.assertTrue(rig.primary_queue.empty()) - for label, node_submission in ( - ("errored", _BlockCandidateNodeSubmission( - attempted=True, - error=RuntimeError("boom"), - )), - ("duplicate", _BlockCandidateNodeSubmission( - attempted=True, - result="duplicate", - )), - ("unattempted", _BlockCandidateNodeSubmission(attempted=False)), - ): - task = SimpleNamespace( - label=label, - candidate=SimpleNamespace( - context=SimpleNamespace(template={"height": 10}), - submission=SimpleNamespace(block_hash_hex=label), - ), - node_submission=node_submission, - ) - self.assertTrue(rig.server._enqueue_block_accounting_task(task)) - # None of the three joined the accepted lane. - self.assertEqual(rig.accepted_queue.qsize(), 1) - self.assertEqual(rig.primary_queue.qsize(), 3) - - def test_the_retained_offer_stash_shares_the_one_predicate(self) -> None: - rig = AccountingLaneRig() - service = rig.service - - def retained() -> dict[str, Any]: - return dict( - getattr( - service, - "_block_candidate_retained_node_submissions", - {}, - ) - ) - - for block_hash, node_submission in ( - ("11" * 32, _BlockCandidateNodeSubmission( - attempted=True, - error=RuntimeError("boom"), - )), - ("22" * 32, _BlockCandidateNodeSubmission( - attempted=True, - result="duplicate", - )), - ("33" * 32, _BlockCandidateNodeSubmission(attempted=False)), - ("44" * 32, None), - ): - service._stash_retained_block_candidate_node_submission( - block_hash, - node_submission, - ) - self.assertEqual(retained(), {}) - definitive = _BlockCandidateNodeSubmission(attempted=True) - service._stash_retained_block_candidate_node_submission( - "55" * 32, - definitive, - ) - self.assertEqual(retained(), {"55" * 32: definitive}) - - -class AcceptedAccountingPriorityTests(unittest.TestCase): - """Dispatch order: the issue's counterexample and the burst it hides in.""" - - def test_accepted_evidence_drains_before_queued_same_height_replay( - self, - ) -> None: - """#181's BLOCKED integration review, now passing. - - Three durable same-height replay tasks are queued and a definitively - accepted task is queued afterward. The reported drain order was - replay 1, replay 2, replay 3, accepted -- every one of those replay - tasks costing ~6 RPCs, two ledger writes and a payout-balance - mutation before the decided block's accounting could start. - """ - rig = AccountingLaneRig() - for index in (1, 2, 3): - rig.enqueue(f"replay-{index}", accepted=False, height=10) - rig.enqueue("accepted", accepted=True, height=10) - self.assertEqual( - rig.drain(), - ["accepted", "replay-1", "replay-2", "replay-3"], - ) - - def test_accepted_evidence_overtakes_a_spilled_stale_backlog( - self, - ) -> None: - """The burst case, which a priority key cannot reach. - - The primary queue is at its bound and spillover has begun before the - accepted task is handed off. A key on the primary heap is inert - here: the accepted task would be *put* into the overflow queue by - the unchanged spillover rule, and the loop drains the bounded - primary in full before it looks at the overflow at all. Ordering by - lane rather than by key is what makes the accepted class - structurally un-spillable. - - The accepted task is deliberately given the highest height and the - last arrival sequence in the whole batch, so nothing about its key - can explain it draining first. - """ - rig = AccountingLaneRig() - depth = DEFAULT_BLOCK_ACCOUNTING_QUEUE_DEPTH - self.assertEqual(rig.primary_queue.maxsize, depth) - for index in range(depth): - rig.enqueue(f"stale-{index}", accepted=False, height=10) - self.assertTrue(rig.primary_queue.full()) - # The burst state itself: this handoff finds the primary full and - # spills, and the one after it joins the spill by the put rule. - rig.enqueue("spilled-0", accepted=False, height=10) - self.assertFalse(rig.overflow_queue.empty()) - rig.enqueue("spilled-1", accepted=False, height=10) - self.assertEqual(rig.overflow_queue.qsize(), 2) - rig.enqueue("accepted", accepted=True, height=99) - self.assertEqual(rig.accepted_queue.qsize(), 1) - - order = rig.drain() - self.assertEqual(order[0], "accepted") - self.assertEqual( - order, - ["accepted"] - + [f"stale-{index}" for index in range(depth)] - + ["spilled-0", "spilled-1"], - ) - - def test_a_newer_stale_handoff_never_jumps_an_older_spill_entry( - self, - ) -> None: - """The spillover put rule's anti-starvation property, unchanged. - - Membership is what this change moved, not the rule. Once the - overflow queue is non-empty, a later non-accepted handoff still - joins it rather than refilling the bounded primary in front of the - older spill entries -- even when the primary has since drained and - has room again, which is exactly when refilling would starve them. - """ - rig = AccountingLaneRig(quota=1) - rig.server.block_accounting_queue_depth = 1 - del rig.service._block_accounting_queue - rig.service._ensure_block_accounting_state() - self.assertEqual(rig.primary_queue.maxsize, 1) - - rig.enqueue("primary", accepted=False, height=10) - rig.enqueue("spilled-old", accepted=False, height=10) - self.assertEqual(rig.overflow_queue.qsize(), 1) - - def hand_off_newer(task: Any) -> None: - if AccountingLaneRig.label(task) != "primary": - return - # The primary queue has just been emptied by this dispatch, so - # it has room; the put rule must still send the newer handoff - # behind the older spill entry. - self.assertTrue(rig.primary_queue.empty()) - rig.enqueue("newer", accepted=False, height=10) - self.assertTrue(rig.primary_queue.empty()) - self.assertEqual(rig.overflow_queue.qsize(), 2) - - rig.on_dispatch = hand_off_newer - self.assertEqual(rig.drain(), ["primary", "spilled-old", "newer"]) - - -class AcceptedDispatchQuotaTests(unittest.TestCase): - """Bounded fairness (D3): stale replay is delayed, never starved.""" - - QUOTAS = (1, BLOCK_ACCOUNTING_ACCEPTED_DISPATCH_QUOTA) - - def test_the_default_quota_is_four(self) -> None: - self.assertEqual(BLOCK_ACCOUNTING_ACCEPTED_DISPATCH_QUOTA, 4) - - def test_a_stale_task_waits_behind_at_most_the_quota(self) -> None: - """The acceptance criterion, stated the way D3 states it. - - With a continuous stream of accepted tasks and at least one stale - task pending, the stale task is dispatched after at most ``quota`` - accepted services -- never behind an unbounded stream of them, - whatever the accepted arrival rate. - """ - for quota in self.QUOTAS: - with self.subTest(quota=quota): - rig = AccountingLaneRig(quota=quota) - rig.enqueue("stale", accepted=False, height=10) - rig.on_dispatch = rig.accepted_stream(until=8 * quota) - rig.enqueue("accepted-seed", accepted=True, height=10) - - order = rig.drain() - self.assertIn("stale", order) - waited_behind = order.index("stale") - self.assertLessEqual(waited_behind, quota) - # Every dispatch it waited behind was an accepted one, so - # the count above really is the accepted-service bound. - self.assertTrue( - all( - label.startswith("accepted") - for label in order[:waited_behind] - ), - order, - ) - # And the quota is a bound on delay, not a ban on accepted - # work: the stream kept running afterwards. - self.assertGreater(len(order) - waited_behind, 1) - - def test_the_quota_only_yields_to_work_that_exists(self) -> None: - """An exhausted quota with no stale work still dispatches accepted. - - The bound is defined against *available* non-accepted work. With - the stale lanes empty there is nothing to yield to, so an accepted - backlog keeps draining instead of stalling behind a spent counter. - """ - rig = AccountingLaneRig(quota=1) - for index in range(5): - rig.enqueue(f"accepted-{index}", accepted=True, height=10) - self.assertEqual( - rig.drain(), - [f"accepted-{index}" for index in range(5)], - ) - - def test_a_delayed_stale_task_still_reaches_terminal_disposition( - self, - ) -> None: - """Delay is the fix; starvation would be a different bug. - - A stale accounting task that never runs leaves its durable outbox - row pending, its offer-time accepted-block payout-preview barrier - armed and its disposition lease held. The stale task here is a real - one with a real pending outbox row whose parent tip has since moved, - run through the shipped ``_run_block_accounting_task``, while the - accepted lane is kept continuously non-empty around it. Delayed by - the quota, it still reaches its terminal abandonment. - """ - for quota in self.QUOTAS: - with self.subTest(quota=quota): - block_hash = "e7" * 32 - rig = AccountingLaneRig( - quota=quota, - ledger=SingleWriterShareLedger(), - rpc=RejectingSubmitTipRpc(MOVED_TIP), - ) - candidate = durable_candidate(rig, block_hash, tag="e7") - self.assertEqual( - rig.ledger._block_candidate_outbox[block_hash]["state"], - "pending", - ) - durable_accounting_task(rig, candidate) - rig.on_dispatch = rig.accepted_stream(until=8 * quota) - rig.enqueue("accepted-seed", accepted=True, height=10) - - order = rig.drain() - self.assertIn(block_hash, order) - self.assertLessEqual(order.index(block_hash), quota) - self.assertEqual( - rig.ledger._block_candidate_outbox[block_hash]["state"], - "abandoned", - ) - with rig.server.lock: - self.assertIn( - block_hash, - rig.service._block_candidate_terminal_outcomes, - ) - # The lease travelled with the task and the lane released it. - with rig.service._block_candidate_disposition_registry_lock: - self.assertNotIn( - block_hash, - rig.service._block_candidate_disposition_flights, - ) - - -class AcceptedLaneShapeTests(unittest.TestCase): - """The lane's own registration and unboundedness.""" - - def test_the_accepted_lane_is_unbounded(self) -> None: - """A maxsize here would need a spill target, and any queue-state - dependent spill is the inversion this lane removes. The offer has - already happened, so it may never be converted back into a - raw-submit retry; max-block admission bounds how many unresolved - real offers can exist at once. - """ - rig = AccountingLaneRig() - self.assertEqual(rig.accepted_queue.maxsize, 0) - self.assertEqual(rig.overflow_queue.maxsize, 0) - self.assertEqual( - rig.primary_queue.maxsize, - DEFAULT_BLOCK_ACCOUNTING_QUEUE_DEPTH, - ) - for index in range(4 * DEFAULT_BLOCK_ACCOUNTING_QUEUE_DEPTH): - rig.enqueue(f"accepted-{index}", accepted=True, height=10) - self.assertEqual( - rig.accepted_queue.qsize(), - 4 * DEFAULT_BLOCK_ACCOUNTING_QUEUE_DEPTH, - ) - self.assertTrue(rig.primary_queue.empty()) - self.assertTrue(rig.overflow_queue.empty()) - - def test_the_accepted_lane_is_reachable_through_the_coordinator( - self, - ) -> None: - """The state-field registration, which only a delegate-backed - coordinator exercises: the lane must be readable and writable under - its historical coordinator attribute name like the other two. - """ - rig = AccountingLaneRig() - self.assertIs( - rig.server._block_accounting_accepted_queue, - rig.service._block_accounting_accepted_queue, - ) - rig.server._block_accounting_accepted_queue.put_nowait( - (10, 0, SimpleNamespace()) - ) - self.assertEqual(rig.service._block_accounting_accepted_queue.qsize(), 1) - - -if __name__ == "__main__": - unittest.main() diff --git a/tests/test_prism_accepted_preview_publication_latency.py b/tests/test_prism_accepted_preview_publication_latency.py deleted file mode 100644 index 2a5aad979..000000000 --- a/tests/test_prism_accepted_preview_publication_latency.py +++ /dev/null @@ -1,880 +0,0 @@ -#!/usr/bin/env python3 -"""Acceptance-to-preview-publication latency, driven by accepted-block cadence. - -Issue #181 item 3. The 2026-08-21 testnet4 validation run had #190/#191/#196 -deployed and demonstrably active, and the accepted-parent tail still dominated: -25 accepted-parent preview wait timeouts, the first two at only 0.32 PH/s, -semantic current-work coverage repeatedly at 0.0, and the coordinator using -about 1.26 CPU cores at ~8.5 PH/s. That comment states the distinguishing -input directly: "The distinguishing input should be **accepted-block cadence**, -not just PH/s: this testnet difficulty turns the hashrate into rapid -consecutive accepted tips, which is what exercises the tail." - -So this instrument is parameterised on **accepted tips per minute**, never on -hashrate. It releases one definitively accepted winner per tip, each at the -next height and parented on the previous winner, with a fixed number of -same-height siblings behind it, and measures the shipped -``qbit_prism_accepted_block_preview_publication_seconds`` histogram end to end -through the real coordinator: the real bounded FIFO ``candidate_queue``, the -real submitter split (``submit_next`` with ``defer_accounting=True``, the call -``BlockCandidateService.run`` makes), the real bounded accounting handoff and -its unbounded spillover, the shipped ``block_accounting_loop`` on its own -thread, and the shipped payout-preview publication boundary. - -Why the node latency is modelled rather than left at zero ---------------------------------------------------------- -The quantity under test is a wall-clock queueing latency, and the cost that -fills the accounting lane is node round-trips: the #181 spike measured a stale -sibling at "1 ``submitblock`` + 3 ``getbestblockhash`` + 3 ``getblockheader`` + -2 ledger writes". Against an in-process fake those calls cost microseconds, so -a zero-latency rig reports a p95 near zero for *every* dispatch order and can -therefore distinguish nothing. ``rpc_latency_seconds`` restores the only cost -that makes ordering observable. It is an explicit, reported parameter of the -instrument: the tips-per-minute numbers below are cadences relative to *this* -modelled drain rate, and are not claims about testnet4's block interval. - -What is deliberately not modelled: PostgreSQL wait (the 2026-08-21 run had -Postgres at ~12% CPU and the host at ~1.26 cores, so the tail was not storage -or CPU bound), qbitd itself, and the child job-build side of the wait. This -measures the producer half of the 5 s child wait budget -- -``DEFAULT_ACCEPTED_BLOCK_PAYOUT_PREVIEW_WAIT_SECONDS`` -- which is the half -#181 is about. That budget is not tuned here and must not be. - -Relationship to the shipped instruments ---------------------------------------- -``tests/prism_candidate_storm.py`` is the per-row oracle for a *single* decided -height: it seeds one same-parent storm, and its per-row drain explicitly stops -before the winner because "the decided winner, whose finalization tail is -outside this instrument". The accepted tail *is* what this file measures, and -it needs consecutive heights over a wall clock, so the cadence driver lives -here rather than being bolted onto a single-height characterization rig. It -reuses that rig's proven fixtures directly -- ``SingleWriterShareLedger`` with -durable rows appended before admission, the ``PendingShare``/intent seeding -shape, and a per-method-counting RPC. -""" - -from __future__ import annotations - -import argparse -import json -import math -import sys -import tempfile -import threading -import time -import unittest -from contextlib import nullcontext, redirect_stdout -from dataclasses import asdict, dataclass, field -from io import StringIO -from pathlib import Path -from types import SimpleNamespace -from typing import Any, ContextManager - -if not __package__: - sys.path.insert(0, str(Path(__file__).resolve().parents[1])) - -from lab.prism.block_candidates import ( # noqa: E402 - COLLAPSE_DIFFICULTY_SCALE, - COLLAPSE_POW_LIMIT_BITS, - PRISM_ACCEPTED_BLOCK_PREVIEW_PUBLICATION_RESULTS, - PRISM_ACCEPTED_BLOCK_PREVIEW_PUBLICATION_SECONDS_BUCKETS, -) -from lab.prism.payout_state import ( # noqa: E402 - DEFAULT_ACCEPTED_BLOCK_PAYOUT_PREVIEW_WAIT_SECONDS, -) -from lab.prism.share_ledger import ( # noqa: E402 - PendingShare, - SingleWriterShareLedger, -) -from lab.prism.template_artifacts import scaled_network_difficulty # noqa: E402 -from tests.prism_vardiff_test_support import ( # noqa: E402 - FakeRpc, - block_candidate, - submit_coordinator, - verified_audit_report, - verified_block_bundle, -) - - -CADENCE_PARENT_HASH = "22" * 32 -CADENCE_BASE_HEIGHT = 10 -# One node round-trip. Sized from the #181 spike's per-row measurement (a stale -# sibling costs about six of these) so that a single stale accounting task -# costs roughly a fifth of a second, the order the 2026-08-21 run's 5-12 s -# build_superseded waves imply for a handful of them. -DEFAULT_CADENCE_RPC_LATENCY_SECONDS = 0.03 -# Same-height candidates released behind each accepted winner. The 2026-08-20 -# storm reached 3,120; #196 brought the durable peak to 158. Four per tip is -# well inside what #196 leaves for the per-row lane and keeps the instrument's -# own runtime bounded. -DEFAULT_CADENCE_SIBLINGS_PER_TIP = 4 -# The acceptance bar, restated from the 2026-08-21 comment: "p95 -# acceptance-to-preview publication remains below the 5 s child wait budget at -# the tested accepted-block cadence". -PREVIEW_PUBLICATION_P95_BUDGET_SECONDS = ( - DEFAULT_ACCEPTED_BLOCK_PAYOUT_PREVIEW_WAIT_SECONDS -) - - -def bucket_percentile( - buckets: dict[float, int], - count: int, - quantile: float, -) -> float: - """Smallest histogram boundary whose cumulative count covers ``quantile``. - - Prometheus buckets are cumulative, so a percentile read from them is a - bucket boundary, not an interpolated sample. Returning the boundary is the - honest answer: it means "at least ``quantile`` of observations completed - within this many seconds". ``inf`` means only the ``+Inf`` bucket covers - it, i.e. the percentile ran off the top of the tuple. - """ - if count <= 0: - return 0.0 - target = quantile * count - for boundary in sorted(buckets): - if int(buckets[boundary]) >= target: - return float(boundary) - return math.inf - - -@dataclass(frozen=True) -class CadenceReport: - """One cadence sweep's inputs and measured publication latency.""" - - tips: int - tips_per_minute: float - inter_acceptance_seconds: float - siblings_per_tip: int - rpc_latency_seconds: float - accepted_block_count: int - published_count: int - degraded_count: int - published_p95_seconds: float - published_p50_seconds: float - published_mean_seconds: float - slowest_bucket_seconds: float - durable_pending_rows: int - node_calls: dict[str, int] = field(default_factory=dict) - - @property - def observed_count(self) -> int: - return self.published_count + self.degraded_count - - -class CadenceChainRpc(FakeRpc): - """A node that accepts registered winners and calls every sibling stale. - - Each registered winner's ``submitblock`` returns ``None`` -- the definitive - acceptance the #181 discriminator keys on -- and occupies its height, which - moves the best tip. Every other ``submitblock`` answers ``inconclusive``: a - valid block that did not become the tip, which is what a same-height - sibling of an already-accepted winner gets. Calls are counted per method, - as ``DecidedHeightRpc`` counts them, and every call pays - ``latency_seconds`` so a node round-trip has the cost that makes lane - ordering observable in wall clock. - - ``getblockheader`` reports the occupant's compact ``bits`` (and the raw - ``difficulty`` float qbitd reports beside it) because #181 item 2's - dequeue-time skip reads clause 4b from them. Without ``bits`` the clause - has no work value to compare and every evaluation fails closed, so the - instrument would silently measure a coordinator whose skip never fires - -- and would still report a changed p95, because failing closed is not - free. The default is the qbit powLimit, exactly as ``DecidedHeightRpc`` - defaults, which is at or above the raw ``network_difficulty`` of 1 this - rig stamps its candidates with. - """ - - def __init__( - self, - *, - parent: str, - height: int, - latency_seconds: float = DEFAULT_CADENCE_RPC_LATENCY_SECONDS, - bits: str = COLLAPSE_POW_LIMIT_BITS, - ) -> None: - self.tip = str(parent).lower() - self.height = int(height) - self.bits = str(bits).lower() - self.latency_seconds = max(0.0, float(latency_seconds)) - self.active: dict[int, str] = {} - self.winners: dict[str, int] = {} - self.calls: dict[str, int] = {} - self._lock = threading.Lock() - - def register_winner(self, block_hash: str, height: int) -> None: - with self._lock: - self.winners[str(block_hash).lower()] = int(height) - - def call_count(self) -> dict[str, int]: - with self._lock: - return dict(self.calls) - - def call(self, method: str, params: list[object] | None = None) -> object: - with self._lock: - self.calls[method] = self.calls.get(method, 0) + 1 - if self.latency_seconds: - time.sleep(self.latency_seconds) - if method == "getbestblockhash": - with self._lock: - return self.tip - if method == "getblockcount": - with self._lock: - return self.height - if method == "submitblock": - # The candidate carries its own hash as its block bytes, so one - # fake node can tell the registered winner from its siblings. - block_hex = str((params or [""])[0]).lower() - with self._lock: - height = self.winners.get(block_hex) - if height is None: - return "inconclusive" - self.active[height] = block_hex - self.tip = block_hex - self.height = max(self.height, height) - return None - if method == "getblockhash": - height = int((params or [0])[0]) - with self._lock: - found = self.active.get(height) - if found is None: - raise RuntimeError( - f"qbit RPC getblockhash failed: -8 unknown height {height}" - ) - return found - if method == "getblockheader": - block_hash = str((params or [""])[0]).lower() - with self._lock: - heights = { - active: height for height, active in self.active.items() - } - height = heights.get(block_hash) - top = self.height - if height is None: - raise RuntimeError( - "qbit RPC getblockheader failed: -5 Block not found" - ) - return { - "hash": block_hash, - "height": height, - "confirmations": top - height + 1, - "bits": self.bits, - # Reported beside bits, as qbitd does, and a million times - # smaller than the scaled units a candidate row carries: a - # reader that took this float instead would make an - # equal-work occupant read as weaker. - "difficulty": ( - scaled_network_difficulty(self.bits) - / COLLAPSE_DIFFICULTY_SCALE - ), - } - return super().call(method, params) - - -class AcceptedTipCadenceRig: - """Release accepted tips at a fixed cadence and measure publication. - - The lanes run in the shipped production split: the submitter thread calls - ``submit_next`` with ``defer_accounting=True`` and hands every node offer - to the accounting lane, and ``block_accounting_loop`` drains that lane on - its own thread. Nothing about either lane's ordering is simulated here -- - the queues, the spillover rule and the dequeue discipline are the shipped - ones -- so what the histogram records is the shipped dispatch order. - """ - - def __init__( - self, - *, - tips: int, - tips_per_minute: float, - siblings_per_tip: int = DEFAULT_CADENCE_SIBLINGS_PER_TIP, - rpc_latency_seconds: float = DEFAULT_CADENCE_RPC_LATENCY_SECONDS, - quiet: bool = True, - ) -> None: - if tips <= 0: - raise ValueError("tips must be positive") - if tips_per_minute <= 0: - raise ValueError("tips_per_minute must be positive") - if siblings_per_tip < 0: - raise ValueError("siblings_per_tip must not be negative") - self.tips = int(tips) - self.tips_per_minute = float(tips_per_minute) - self.siblings_per_tip = int(siblings_per_tip) - self.rpc_latency_seconds = float(rpc_latency_seconds) - self.quiet = quiet - self.ledger = SingleWriterShareLedger() - self.rpc = CadenceChainRpc( - parent=CADENCE_PARENT_HASH, - height=CADENCE_BASE_HEIGHT - 1, - latency_seconds=self.rpc_latency_seconds, - ) - self._lane_failures: list[str] = [] - - @property - def inter_acceptance_seconds(self) -> float: - return 60.0 / self.tips_per_minute - - def _silence(self) -> ContextManager[Any]: - if not self.quiet: - return nullcontext() - return redirect_stdout(StringIO()) - - # -- fixture ----------------------------------------------------------- - - def _coordinator(self) -> tuple[Any, Any]: - server, state, _recording = submit_coordinator(tip=CADENCE_PARENT_HASH) - # Every released winner must be admissible: max-block admission is not - # what this instrument is measuring. - server.max_blocks = self.tips + 1 - server.stop_after_block = False - server.accepted_block_count = 0 - server.ledger = self.ledger - server.rpc = self.rpc - # A shipped tunable, set to zero so a wakeup parked by a transient - # failure is re-offerable in the same round rather than parking behind - # a not-before deadline that would show up as publication latency it - # did not cause. - server.block_candidate_retry_initial_seconds = 0.0 - server._ensure_job_cache_state() - server.build_audit_bundle = lambda **_kwargs: verified_block_bundle() - server.verify_bundle = self._verify_bundle - return server, state - - @staticmethod - def _verify_bundle( - *_args: Any, - expected_block_height: int | None = None, - **_kwargs: Any, - ) -> dict[str, object]: - # The shipped verifier report is height-checked against the candidate; - # consecutive tips mean consecutive heights, so the stub answers the - # height it was asked about rather than a fixed one. - report = verified_audit_report() - if expected_block_height is not None: - report["block_height"] = int(expected_block_height) - return report - - def _candidate( - self, - server: Any, - state: Any, - *, - block_hash: str, - height: int, - parent: str, - ordinal: int, - ) -> tuple[Any, tuple[Any, Any]]: - job_id = f"job-{height}-{ordinal}" - context = SimpleNamespace( - job=server.jobs["job-1"].job, - template={ - "previousblockhash": parent, - "height": height, - "coinbasevalue": 50_00000000, - }, - found_block={"network_difficulty": 1}, - issued_at_ms=12345, - collection_only=False, - worker=server.jobs["job-1"].worker, - shares_json=[], - prior_balances=[], - # The compact issued-job preview. Its presence is what makes the - # accepted tail publish a payout preview at all - # (block_finalization.py's issued_preview branch), which is the - # end of the interval under measurement. - prospective_prior_balances=(), - ) - server.jobs[job_id] = context - pending = PendingShare( - share_id=f"miner-a:{block_hash}", - miner_id="miner-a", - order_key="miner-a", - p2mr_program_hex="11" * 32, - share_difficulty=1, - network_difficulty=1, - template_height=height, - job_id=job_id, - job_issued_at_ms=1, - accepted_at_ms=ordinal, - ntime=1, - ) - submission = SimpleNamespace( - coinbase_tx_hex="c0ffee", - block_hash_hex=block_hash, - block_hex=block_hash, - share_pass=True, - block_pass=True, - ) - candidate = block_candidate( - server, - state, - submission, - job_id=job_id, - pending_share=pending, - ) - return candidate, (pending, server.block_candidate_intent(candidate)) - - @staticmethod - def _hash_for(tip_index: int, ordinal: int) -> str: - return f"{((tip_index + 1) << 8) | ordinal:064x}" - - # -- lanes ------------------------------------------------------------- - - def _submitter_loop(self, server: Any) -> None: - service = server._ensure_block_candidate_service() - # block_submit_loop stamps this on entry; the progress boundary stamps - # are gated to it. - service._block_submitter_thread_ident = threading.get_ident() - while not server.stop_event.is_set(): - try: - server.submit_next_block_candidate( - timeout=0.05, - defer_accounting=True, - ) - except Exception as exc: # noqa: BLE001 - # Reported, never swallowed: a submitter that dies silently - # would look like a cadence the instrument simply never - # reached. - self._lane_failures.append(f"submitter: {exc!r}") - return - - # -- driver ------------------------------------------------------------ - - def _build_tips( - self, - server: Any, - state: Any, - ) -> list[tuple[int, list[Any], list[Any]]]: - """Build every tip's candidates before either lane starts. - - Job contexts are registered in ``server.jobs``, which shipped paths - iterate. Building them all up front keeps the driver from mutating - that dictionary underneath a running lane; only the durable append and - the queue admission happen on the cadence, which is what the cadence - is supposed to control. - """ - built: list[tuple[int, list[Any], list[Any]]] = [] - for tip_index in range(self.tips): - height = CADENCE_BASE_HEIGHT + tip_index - parent = ( - CADENCE_PARENT_HASH - if tip_index == 0 - else self._hash_for(tip_index - 1, 0) - ) - candidates = [] - durable_records = [] - for ordinal in range(self.siblings_per_tip + 1): - candidate, durable_record = self._candidate( - server, - state, - block_hash=self._hash_for(tip_index, ordinal), - height=height, - parent=parent, - ordinal=ordinal, - ) - candidates.append(candidate) - durable_records.append(durable_record) - built.append((height, candidates, durable_records)) - return built - - def _release_tip( - self, - server: Any, - tip_index: int, - built: list[tuple[int, list[Any], list[Any]]], - ) -> None: - height, candidates, durable_records = built[tip_index] - # Registered before admission: the winner has to be acceptable the - # instant the submitter offers it. - self.rpc.register_winner(self._hash_for(tip_index, 0), height) - # Durable first, exactly as live admission does: the outbox row is - # what makes a coalesced wakeup replayable. Appending per tip rather - # than up front is also what keeps a replay enumeration from pulling - # a later tip's candidates forward and dissolving the cadence. - self.ledger.append_batch(durable_records) - for candidate in candidates: - server.enqueue_block_candidate(candidate) - - def _drive( - self, - server: Any, - built: list[tuple[int, list[Any], list[Any]]], - ) -> None: - interval = self.inter_acceptance_seconds - started = time.monotonic() - for tip_index in range(self.tips): - delay = (started + tip_index * interval) - time.monotonic() - if delay > 0: - time.sleep(delay) - self._release_tip(server, tip_index, built) - - def _await_publication(self, server: Any, *, timeout: float) -> None: - """Wait until every released winner has published, or time out. - - The deadline is generous on purpose: a cadence that overruns it is a - result, not a flake, and the report carries the shortfall so the test - can say what did not publish. - """ - service = server._ensure_block_candidate_service() - deadline = time.monotonic() + timeout - while time.monotonic() < deadline: - snapshot = service.accepted_block_preview_publication_snapshot() - observed = sum( - int(histogram["count"]) for histogram in snapshot.values() - ) - if observed >= self.tips: - return - time.sleep(0.05) - - def run(self) -> CadenceReport: - server, state = self._coordinator() - service = server._ensure_block_candidate_service() - service._ensure_block_accounting_state() - accounting = threading.Thread( - target=server.block_accounting_loop, - name="cadence-block-accounting", - daemon=True, - ) - submitter = threading.Thread( - target=self._submitter_loop, - args=(server,), - name="cadence-block-submitter", - daemon=True, - ) - with tempfile.TemporaryDirectory() as tempdir: - server.audit_dir = Path(tempdir) - server.evidence_path = Path(tempdir) / "evidence.json" - server.ledger_writer_public_key_hex = "aa" * 32 - with self._silence(): - built = self._build_tips(server, state) - accounting.start() - submitter.start() - try: - self._drive(server, built) - # The drain budget scales with the work released, not with - # the cadence: a saturated lane still has to be allowed to - # finish so its latencies land in the histogram instead of - # being truncated into a flattering p95. - self._await_publication( - server, - timeout=30.0 - + self.tips - * (self.siblings_per_tip + 1) - * 12.0 - * max(self.rpc_latency_seconds, 0.001), - ) - finally: - server.stop_event.set() - submitter.join(timeout=60.0) - accounting.join(timeout=60.0) - snapshot = service.accepted_block_preview_publication_snapshot() - pending = self.ledger.block_candidate_pending_metrics() - published = snapshot["published"] - degraded = snapshot["degraded"] - published_count = int(published["count"]) - return CadenceReport( - tips=self.tips, - tips_per_minute=self.tips_per_minute, - inter_acceptance_seconds=self.inter_acceptance_seconds, - siblings_per_tip=self.siblings_per_tip, - rpc_latency_seconds=self.rpc_latency_seconds, - accepted_block_count=int(server.accepted_block_count), - published_count=published_count, - degraded_count=int(degraded["count"]), - published_p95_seconds=bucket_percentile( - published["buckets"], published_count, 0.95 - ), - published_p50_seconds=bucket_percentile( - published["buckets"], published_count, 0.50 - ), - published_mean_seconds=( - float(published["sum"]) / published_count - if published_count - else 0.0 - ), - slowest_bucket_seconds=bucket_percentile( - published["buckets"], published_count, 1.0 - ), - durable_pending_rows=int(pending.get("pending_count", -1)), - node_calls=self.rpc.call_count(), - ) - - @property - def lane_failures(self) -> tuple[str, ...]: - return tuple(self._lane_failures) - - -def accepted_lane_is_present(server: Any) -> bool: - """Whether item 1's dedicated definitive-acceptance lane has landed. - - On the #181 base (``ce4a1f3``) every accounting handoff shares one bounded - primary queue and its unbounded spillover, and a definitively accepted task - is dispatched strictly behind whatever stale work is already queued. The - high-cadence assertion below is written against whichever of those two - worlds it is running in, so that this file is a truthful statement in both - and becomes the bar the moment the lane exists. - """ - service = server._ensure_block_candidate_service() - service._ensure_block_accounting_state() - return hasattr(service, "_block_accounting_accepted_queue") - - -class AcceptedPreviewPublicationHistogramTests(unittest.TestCase): - """The metric itself: labels, boundaries, and what closes an interval.""" - - def _service(self) -> Any: - server, _state, _ledger = submit_coordinator() - return server, server._ensure_block_candidate_service() - - def test_five_second_child_wait_budget_is_a_bucket_boundary(self) -> None: - # The acceptance criterion is a p95 below the 5 s child wait budget, - # and a Prometheus histogram can only answer that at a boundary it - # carries. The range must also extend past it, or a saturated tail - # folds into +Inf and stops being measurable. - self.assertIn( - PREVIEW_PUBLICATION_P95_BUDGET_SECONDS, - PRISM_ACCEPTED_BLOCK_PREVIEW_PUBLICATION_SECONDS_BUCKETS, - ) - self.assertGreater( - max(PRISM_ACCEPTED_BLOCK_PREVIEW_PUBLICATION_SECONDS_BUCKETS), - PREVIEW_PUBLICATION_P95_BUDGET_SECONDS, - ) - self.assertEqual( - tuple(sorted(PRISM_ACCEPTED_BLOCK_PREVIEW_PUBLICATION_SECONDS_BUCKETS)), - PRISM_ACCEPTED_BLOCK_PREVIEW_PUBLICATION_SECONDS_BUCKETS, - ) - self.assertEqual( - PRISM_ACCEPTED_BLOCK_PREVIEW_PUBLICATION_RESULTS, - ("published", "degraded"), - ) - - def test_publication_without_an_acceptance_stamp_observes_nothing( - self, - ) -> None: - # A replayed candidate confirmed by chain probe never made a - # definitive offer in this process. Reporting a zero for it would - # understate the very percentile the bar is read from. - server, service = self._service() - server._observe_accepted_block_preview_publication( - "ab" * 32, - result="published", - ) - snapshot = service.accepted_block_preview_publication_snapshot() - self.assertEqual(snapshot["published"]["count"], 0) - self.assertEqual(snapshot["degraded"]["count"], 0) - - def test_only_the_first_publication_of_a_hash_is_measured(self) -> None: - server, service = self._service() - service._note_accepted_block_preview_acceptance("AB" * 32) - server._observe_accepted_block_preview_publication( - "ab" * 32, - result="published", - ) - # A matching republication -- the early return with an existing - # published generation, or a deferred retry's republish -- is not a - # second interval. - server._observe_accepted_block_preview_publication( - "ab" * 32, - result="published", - ) - server._observe_accepted_block_preview_publication( - "ab" * 32, - result="degraded", - ) - # Nor does a second definitive offer of the same hash restart it. - service._note_accepted_block_preview_acceptance("ab" * 32) - server._observe_accepted_block_preview_publication( - "ab" * 32, - result="published", - ) - snapshot = service.accepted_block_preview_publication_snapshot() - self.assertEqual(snapshot["published"]["count"], 1) - self.assertEqual(snapshot["degraded"]["count"], 0) - - def test_degraded_publication_is_labelled_apart(self) -> None: - server, service = self._service() - service._note_accepted_block_preview_acceptance("cd" * 32) - server._observe_accepted_block_preview_publication( - "cd" * 32, - result="degraded", - ) - snapshot = service.accepted_block_preview_publication_snapshot() - self.assertEqual(snapshot["degraded"]["count"], 1) - self.assertEqual(snapshot["published"]["count"], 0) - - def test_acceptance_stamps_are_bounded(self) -> None: - from lab.prism.block_candidates import ( - MAX_ACCEPTED_BLOCK_PREVIEW_PUBLICATION_STAMPS, - ) - - _server, service = self._service() - for index in range(MAX_ACCEPTED_BLOCK_PREVIEW_PUBLICATION_STAMPS + 64): - service._note_accepted_block_preview_acceptance(f"{index:064x}") - self.assertLessEqual( - len(service._accepted_block_preview_acceptance_monotonic), - MAX_ACCEPTED_BLOCK_PREVIEW_PUBLICATION_STAMPS, - ) - - def test_failed_offer_is_not_a_definitive_acceptance(self) -> None: - # The submitblock error path also builds ``result=None``; a two-clause - # test would stamp a failed offer as an acceptance and then measure a - # publication that a later, successful offer produced. - from lab.prism.block_candidates import _BlockCandidateNodeSubmission - - failed = _BlockCandidateNodeSubmission( - attempted=True, - error=RuntimeError("submitblock failed"), - ) - self.assertIsNone(failed.result) - self.assertFalse( - failed.attempted - and failed.error is None - and failed.result is None - ) - - -class AcceptedTipCadenceRegressionTests(unittest.TestCase): - """The bar: p95 publication latency at a low and a high accepted cadence.""" - - # A cadence comfortably slower than the modelled per-tip accounting cost, - # so the lane drains between tips. This is the "normal forward progress" - # case from the 2026-08-21 acceptance criteria. - LOW_CADENCE_TIPS_PER_MINUTE = 40.0 - LOW_CADENCE_TIPS = 8 - # A cadence faster than the lane can drain, which is what the run's tip - # bursts produced. On the #181 base this is the inversion itself: the - # accepted task for height H+n sorts behind every stale sibling of heights - # H..H+n-1 already in the accounting queue. - HIGH_CADENCE_TIPS_PER_MINUTE = 150.0 - HIGH_CADENCE_TIPS = 20 - - def _report(self, **kwargs: Any) -> CadenceReport: - rig = AcceptedTipCadenceRig(**kwargs) - report = rig.run() - self.assertEqual( - rig.lane_failures, - (), - f"a coordinator lane died during the sweep: {rig.lane_failures}", - ) - self.assertEqual( - report.published_count + report.degraded_count, - report.tips, - "every released winner must close an acceptance-to-publication " - f"interval; report={asdict(report)}", - ) - return report - - def test_low_cadence_publishes_well_inside_the_child_wait_budget( - self, - ) -> None: - report = self._report( - tips=self.LOW_CADENCE_TIPS, - tips_per_minute=self.LOW_CADENCE_TIPS_PER_MINUTE, - ) - self.assertEqual(report.degraded_count, 0, asdict(report)) - self.assertLessEqual( - report.published_p95_seconds, - PREVIEW_PUBLICATION_P95_BUDGET_SECONDS, - "p95 acceptance-to-preview publication must stay below the 5 s " - f"child wait budget at a low accepted cadence; report={asdict(report)}", - ) - # "Comfortably": at a cadence the lane keeps up with, the accepted tail - # should not be spending a meaningful fraction of the budget at all. - self.assertLessEqual( - report.published_p95_seconds, - 2.0, - f"low-cadence p95 is not comfortable; report={asdict(report)}", - ) - - def test_high_cadence_exercises_the_accepted_parent_tail(self) -> None: - server, _state, _ledger = submit_coordinator() - lane_present = accepted_lane_is_present(server) - report = self._report( - tips=self.HIGH_CADENCE_TIPS, - tips_per_minute=self.HIGH_CADENCE_TIPS_PER_MINUTE, - ) - if lane_present: - # Item 1 has landed: this is the bar #181 closes against. - self.assertLessEqual( - report.published_p95_seconds, - PREVIEW_PUBLICATION_P95_BUDGET_SECONDS, - "p95 acceptance-to-preview publication must stay below the " - "5 s child wait budget at a high accepted-block cadence; " - f"report={asdict(report)}", - ) - return - # On the #181 base there is no accepted lane, and this assertion is - # the honest statement of what the instrument sees: the tail really is - # exercised. If this ever fails on the base, the instrument has - # stopped reproducing the failure mode and the bar above would be one - # items 1 and 2 could pass without fixing anything -- which is exactly - # the outcome the contract says to report rather than paper over. - self.assertGreater( - report.published_p95_seconds, - PREVIEW_PUBLICATION_P95_BUDGET_SECONDS, - "without a dedicated definitive-acceptance lane the high-cadence " - "sweep must drive p95 publication latency past the 5 s child " - "wait budget; if it no longer does, this instrument has stopped " - f"exercising the accepted-parent tail. report={asdict(report)}", - ) - - -def main() -> int: - parser = argparse.ArgumentParser( - description=( - "Report acceptance-to-preview-publication latency at one or more " - "accepted-block cadences." - ) - ) - parser.add_argument( - "--tips-per-minute", - type=float, - action="append", - default=None, - help="accepted tips per minute; repeatable (default: 40 and 150)", - ) - parser.add_argument("--tips", type=int, default=None) - parser.add_argument( - "--siblings-per-tip", - type=int, - default=DEFAULT_CADENCE_SIBLINGS_PER_TIP, - ) - parser.add_argument( - "--rpc-latency-seconds", - type=float, - default=DEFAULT_CADENCE_RPC_LATENCY_SECONDS, - ) - parser.add_argument("--verbose", action="store_true") - args = parser.parse_args() - cadences = args.tips_per_minute or [ - AcceptedTipCadenceRegressionTests.LOW_CADENCE_TIPS_PER_MINUTE, - AcceptedTipCadenceRegressionTests.HIGH_CADENCE_TIPS_PER_MINUTE, - ] - reports = [] - for tips_per_minute in cadences: - if args.tips is not None: - tips = args.tips - elif tips_per_minute >= 100: - tips = AcceptedTipCadenceRegressionTests.HIGH_CADENCE_TIPS - else: - tips = AcceptedTipCadenceRegressionTests.LOW_CADENCE_TIPS - rig = AcceptedTipCadenceRig( - tips=tips, - tips_per_minute=tips_per_minute, - siblings_per_tip=args.siblings_per_tip, - rpc_latency_seconds=args.rpc_latency_seconds, - quiet=not args.verbose, - ) - report = rig.run() - reports.append(asdict(report)) - if rig.lane_failures: - reports[-1]["lane_failures"] = list(rig.lane_failures) - print(json.dumps(reports, indent=2, sort_keys=True)) - return 0 - - -if __name__ == "__main__": - if len(sys.argv) > 1 and sys.argv[1] == "report": - del sys.argv[1] - raise SystemExit(main()) - unittest.main() diff --git a/tests/test_prism_accepted_preview_telemetry.py b/tests/test_prism_accepted_preview_telemetry.py deleted file mode 100644 index 0ff3c37e1..000000000 --- a/tests/test_prism_accepted_preview_telemetry.py +++ /dev/null @@ -1,654 +0,0 @@ -#!/usr/bin/env python3 -"""Issue #224 Wave 0: the shared accepted-preview telemetry contract. - -These tests pin the contract the three Wave 1 lanes build on: every -vocabulary is closed and label-safe, the rescan reasons mirror the -payout_state literals, the recorder refuses labels outside the contract, -normalizes the two runtime strings onto ``other``, keeps its cells -thread-safe, and the renderer exports every cell from zero without ever -carrying a block hash or height into a label. -""" - -from __future__ import annotations - -import ast -import json -import math -from pathlib import Path -import re -import threading -from types import SimpleNamespace -import unittest - -from lab.prism import accepted_preview_telemetry as contract -from lab.prism.accepted_preview_telemetry import ( - AcceptedPreviewPublicationDiagnostic, - AcceptedPreviewTelemetry, - PRISM_ACCEPTED_LANDING_PHASES, - PRISM_ACCEPTED_PREVIEW_DIAGNOSTICS_CAPACITY, - PRISM_LEDGER_READ_OPERATIONS, - PRISM_PAYOUT_WINDOW_FULL_RESCAN_PATHS, - PRISM_PAYOUT_WINDOW_FULL_RESCAN_REASONS, - PRISM_REORG_RECONCILE_CALLERS, - PRISM_REORG_RECONCILE_STEPS, - empty_duration_stats, - ensure_accepted_preview_telemetry, - fold_ledger_read_stats, - normalize_ledger_read_operation, - normalize_payout_window_full_rescan_reason, -) -from lab.prism.block_candidates import ( - PRISM_ACCEPTED_BLOCK_PREVIEW_PUBLICATION_RESULTS, -) -from lab.prism.metrics import MetricsRenderer -from lab.prism.reorg_reconciler import PRISM_REORG_RECONCILE_LOOKUP_PATHS - -REPO_ROOT = Path(__file__).resolve().parents[1] -LABEL_VALUE = re.compile(r"^[a-z][a-z0-9_]*$") -VOCABULARIES = { - "PRISM_ACCEPTED_LANDING_PHASES": PRISM_ACCEPTED_LANDING_PHASES, - "PRISM_REORG_RECONCILE_CALLERS": PRISM_REORG_RECONCILE_CALLERS, - "PRISM_REORG_RECONCILE_STEPS": PRISM_REORG_RECONCILE_STEPS, - "PRISM_PAYOUT_WINDOW_FULL_RESCAN_REASONS": PRISM_PAYOUT_WINDOW_FULL_RESCAN_REASONS, - "PRISM_PAYOUT_WINDOW_FULL_RESCAN_PATHS": PRISM_PAYOUT_WINDOW_FULL_RESCAN_PATHS, - "PRISM_LEDGER_READ_OPERATIONS": PRISM_LEDGER_READ_OPERATIONS, -} - - -def _string_constants(node: ast.AST) -> set[str]: - return { - child.value - for child in ast.walk(node) - if isinstance(child, ast.Constant) and isinstance(child.value, str) - } - - -def payout_state_full_rescan_reason_literals() -> set[str]: - """Every reason literal payout_state can hand to a full rescan. - - The source of truth keeps them as scattered literals rather than a - tuple (and that module is not this wave's to edit), so the contract - tuple is checked against the module text: assignments to - ``full_reason`` / ``check_reason`` / the cached-window invalidation - reason, the first argument of ``_DaemonWindowRebuildRequired``, and - the ``reason=`` keyword of ``_full_payout_window_materialization``. - """ - source = (REPO_ROOT / "lab" / "prism" / "payout_state.py").read_text() - found: set[str] = set() - for node in ast.walk(ast.parse(source)): - if isinstance(node, ast.Assign): - for target in node.targets: - if isinstance(target, ast.Name) and target.id in ( - "full_reason", - "check_reason", - ): - found |= _string_constants(node.value) - elif ( - isinstance(target, ast.Attribute) - and target.attr - == "_incremental_payout_artifact_window_invalidation_reason" - ): - found |= _string_constants(node.value) - elif isinstance(node, ast.Call): - func = node.func - if isinstance(func, ast.Name): - name = func.id - elif isinstance(func, ast.Attribute): - name = func.attr - else: - continue - if name == "_DaemonWindowRebuildRequired" and node.args: - found |= _string_constants(node.args[0]) - elif name == "_full_payout_window_materialization": - for keyword in node.keywords: - if keyword.arg == "reason": - found |= _string_constants(keyword.value) - return found - - -def _clock() -> tuple[list[float], "callable"]: - ticks: list[float] = [100.0] - - def monotonic() -> float: - return ticks[0] - - return ticks, monotonic - - -class VocabularyContractTests(unittest.TestCase): - def test_every_vocabulary_is_closed_unique_and_label_safe(self) -> None: - for name, vocabulary in VOCABULARIES.items(): - with self.subTest(vocabulary=name): - self.assertIsInstance(vocabulary, tuple) - self.assertGreater(len(vocabulary), 0) - self.assertEqual(len(vocabulary), len(set(vocabulary))) - for member in vocabulary: - self.assertRegex(member, LABEL_VALUE) - - def test_runtime_string_vocabularies_carry_the_other_bucket(self) -> None: - self.assertEqual(PRISM_PAYOUT_WINDOW_FULL_RESCAN_REASONS[-1], "other") - self.assertEqual(PRISM_LEDGER_READ_OPERATIONS[-1], "other") - self.assertIn("other", PRISM_REORG_RECONCILE_CALLERS) - # Programmer-chosen vocabularies have no fold-in bucket: a typo is - # a ValueError at the call site, not a silently merged series. - self.assertNotIn("other", PRISM_ACCEPTED_LANDING_PHASES) - self.assertNotIn("other", PRISM_REORG_RECONCILE_STEPS) - self.assertNotIn("other", PRISM_PAYOUT_WINDOW_FULL_RESCAN_PATHS) - - def test_landing_phases_follow_the_landing_order(self) -> None: - self.assertEqual( - PRISM_ACCEPTED_LANDING_PHASES, - ( - "lane_wait", - "balance_lock_wait", - "reconcile", - "prior_balances_check", - "chain_probe", - "preview_prepare", - "preview_publish", - ), - ) - - def test_reconcile_callers_join_the_existing_lookup_paths(self) -> None: - # The demand-accounting family already labels tip_refresh and - # job_build; the caller vocabulary spells them identically so the - # two families join on the label value. - for path in PRISM_REORG_RECONCILE_LOOKUP_PATHS: - self.assertIn(path, PRISM_REORG_RECONCILE_CALLERS) - self.assertEqual( - PRISM_REORG_RECONCILE_STEPS, - ( - "admission_wait", - "watch_query", - "chain_probe", - "mutations", - "candidate_prepare", - "publish", - ), - ) - - def test_full_rescan_reasons_mirror_payout_state_literals(self) -> None: - literals = payout_state_full_rescan_reason_literals() - # The collector must find the reasons the issue was diagnosed on; - # an empty or partial harvest would make the drift check vacuous. - for expected in ( - "reconcile_invalidation", - "cold_start", - "window_daemon_state_lost", - "periodic_self_check", - ): - self.assertIn(expected, literals) - self.assertEqual( - sorted(literals - set(PRISM_PAYOUT_WINDOW_FULL_RESCAN_REASONS)), - [], - "payout_state grew a full-rescan reason the contract does not carry", - ) - self.assertEqual( - sorted(set(PRISM_PAYOUT_WINDOW_FULL_RESCAN_REASONS) - literals), - ["other"], - "the contract carries a reason payout_state no longer produces", - ) - - def test_ledger_read_operations_cover_every_attributed_statement(self) -> None: - source = (REPO_ROOT / "lab" / "prism" / "share_ledger.py").read_text() - literals = set(re.findall(r'operation="([^"]+)"', source)) - self.assertIn("pending_block_candidate_rows", literals) - self.assertEqual( - sorted(literals - set(PRISM_LEDGER_READ_OPERATIONS)), - [], - "share_ledger attributes a read under a name outside the contract", - ) - - def test_normalizers_bound_runtime_strings(self) -> None: - self.assertEqual( - normalize_payout_window_full_rescan_reason("reconcile_invalidation"), - "reconcile_invalidation", - ) - for value in ("hash=" + "ab" * 32, "", None, 7, "Other"): - self.assertEqual(normalize_payout_window_full_rescan_reason(value), "other") - self.assertEqual( - normalize_ledger_read_operation("payout_window_snapshot"), - "payout_window_snapshot", - ) - self.assertEqual(normalize_ledger_read_operation("snapshot_at_job_issue"), "other") - - -class TelemetryRecorderTests(unittest.TestCase): - def test_empty_owner_snapshot_covers_every_closed_product_at_zero(self) -> None: - snapshot = AcceptedPreviewTelemetry().snapshot() - self.assertEqual( - set(snapshot["landing_phases"]), set(PRISM_ACCEPTED_LANDING_PHASES) - ) - self.assertEqual( - set(snapshot["reconcile_passes"]), set(PRISM_REORG_RECONCILE_CALLERS) - ) - self.assertEqual( - set(snapshot["reconcile_steps"]), - { - (caller, step) - for caller in PRISM_REORG_RECONCILE_CALLERS - for step in PRISM_REORG_RECONCILE_STEPS - }, - ) - self.assertEqual( - set(snapshot["full_rescans"]), - { - (reason, path) - for reason in PRISM_PAYOUT_WINDOW_FULL_RESCAN_REASONS - for path in PRISM_PAYOUT_WINDOW_FULL_RESCAN_PATHS - }, - ) - for family in ("landing_phases", "reconcile_passes", "reconcile_steps", "full_rescans"): - for stats in snapshot[family].values(): - self.assertEqual(stats, empty_duration_stats()) - self.assertEqual(snapshot["diagnostics_retained"], 0) - self.assertEqual(snapshot["diagnostics_recorded"], 0) - - def test_programmer_labels_outside_the_contract_raise(self) -> None: - telemetry = AcceptedPreviewTelemetry() - with self.assertRaises(ValueError): - telemetry.observe_landing_phase("reorg-reconcile", 0.1) - with self.assertRaises(ValueError): - telemetry.observe_reconcile_pass("replay", 0.1) - with self.assertRaises(ValueError): - telemetry.observe_reconcile_step("landing", "getblockhash", 0.1) - with self.assertRaises(ValueError): - telemetry.observe_reconcile_step("replay", "watch_query", 0.1) - with self.assertRaises(ValueError): - telemetry.observe_payout_window_full_rescan( - "reconcile_invalidation", "rust", 0.1 - ) - with self.assertRaises(ValueError): - with telemetry.landing_phase("submitblock-rpc"): - pass - with self.assertRaises(ValueError): - with telemetry.reconcile_step("landing", "sweep"): - pass - # Nothing was recorded by the refused calls. - snapshot = telemetry.snapshot() - for family in ("landing_phases", "reconcile_passes", "reconcile_steps", "full_rescans"): - for stats in snapshot[family].values(): - self.assertEqual(int(stats["count"]), 0) - - def test_runtime_rescan_reason_folds_into_other(self) -> None: - telemetry = AcceptedPreviewTelemetry() - telemetry.observe_payout_window_full_rescan("reconcile_invalidation", "daemon", 3.0) - telemetry.observe_payout_window_full_rescan("hash=" + "ab" * 32, "daemon", 1.0) - telemetry.observe_payout_window_full_rescan(None, "in_process", 2.0) - rescans = telemetry.snapshot()["full_rescans"] - self.assertEqual(rescans[("reconcile_invalidation", "daemon")]["count"], 1) - self.assertEqual(rescans[("other", "daemon")]["count"], 1) - self.assertEqual(rescans[("other", "in_process")]["count"], 1) - self.assertEqual( - {key for key, stats in rescans.items() if stats["count"]}, - {("reconcile_invalidation", "daemon"), ("other", "daemon"), ("other", "in_process")}, - ) - - def test_accumulates_sum_count_max_and_clamps_bad_durations(self) -> None: - telemetry = AcceptedPreviewTelemetry() - for seconds in (0.25, 1.5, 0.75): - telemetry.observe_landing_phase("reconcile", seconds) - for bad in (-1.0, math.nan, math.inf): - telemetry.observe_landing_phase("reconcile", bad) - stats = telemetry.snapshot()["landing_phases"]["reconcile"] - self.assertEqual(stats["count"], 6) - self.assertAlmostEqual(stats["sum"], 2.5) - self.assertEqual(stats["max"], 1.5) - self.assertIsInstance(stats["count"], int) - - def test_snapshot_is_a_copy(self) -> None: - telemetry = AcceptedPreviewTelemetry() - first = telemetry.snapshot() - first["landing_phases"]["reconcile"]["count"] = 99 - first["reconcile_steps"][("landing", "publish")]["sum"] = 99.0 - second = telemetry.snapshot() - self.assertEqual(second["landing_phases"]["reconcile"]["count"], 0) - self.assertEqual(second["reconcile_steps"][("landing", "publish")]["sum"], 0.0) - - def test_context_managers_record_elapsed_even_when_raising(self) -> None: - ticks, monotonic = _clock() - telemetry = AcceptedPreviewTelemetry(monotonic=monotonic) - with telemetry.landing_phase("balance_lock_wait"): - ticks[0] += 0.5 - with self.assertRaises(RuntimeError): - with telemetry.reconcile_step("landing", "mutations"): - ticks[0] += 0.25 - raise RuntimeError("fenced mutation failed") - snapshot = telemetry.snapshot() - self.assertEqual( - snapshot["landing_phases"]["balance_lock_wait"], - {"count": 1, "sum": 0.5, "max": 0.5}, - ) - self.assertEqual( - snapshot["reconcile_steps"][("landing", "mutations")], - {"count": 1, "sum": 0.25, "max": 0.25}, - ) - - def test_concurrent_observers_never_lose_a_sample(self) -> None: - telemetry = AcceptedPreviewTelemetry() - per_thread = 500 - threads = 8 - - def worker(index: int) -> None: - for _ in range(per_thread): - telemetry.observe_landing_phase("preview_publish", 0.001) - telemetry.observe_reconcile_step("job_build", "chain_probe", 0.002) - telemetry.observe_payout_window_full_rescan( - "cold_start" if index % 2 else "bogus", "in_process", 0.003 - ) - telemetry.record_publication_diagnostic( - AcceptedPreviewPublicationDiagnostic( - block_hash=f"{index:02x}" * 32, - block_height=index, - result="published", - acceptance_to_publication_seconds=0.5, - ) - ) - - pool = [threading.Thread(target=worker, args=(index,)) for index in range(threads)] - for thread in pool: - thread.start() - for thread in pool: - thread.join() - snapshot = telemetry.snapshot() - total = per_thread * threads - self.assertEqual(snapshot["landing_phases"]["preview_publish"]["count"], total) - self.assertEqual( - snapshot["reconcile_steps"][("job_build", "chain_probe")]["count"], total - ) - rescans = snapshot["full_rescans"] - self.assertEqual( - rescans[("cold_start", "in_process")]["count"] - + rescans[("other", "in_process")]["count"], - total, - ) - self.assertEqual(snapshot["diagnostics_recorded"], total) - self.assertEqual( - snapshot["diagnostics_retained"], PRISM_ACCEPTED_PREVIEW_DIAGNOSTICS_CAPACITY - ) - - def test_diagnostics_ring_is_bounded_oldest_first_and_stamped(self) -> None: - ticks, monotonic = _clock() - telemetry = AcceptedPreviewTelemetry(diagnostics_capacity=3, monotonic=monotonic) - for height in range(5): - ticks[0] += 1.0 - telemetry.record_publication_diagnostic( - AcceptedPreviewPublicationDiagnostic( - block_hash=f"{height:02x}" * 32, - block_height=height, - result="published", - acceptance_to_publication_seconds=float(height), - ) - ) - retained = telemetry.diagnostics_snapshot() - self.assertEqual([record.block_height for record in retained], [2, 3, 4]) - self.assertEqual([record.recorded_monotonic for record in retained], [103.0, 104.0, 105.0]) - snapshot = telemetry.snapshot() - self.assertEqual(snapshot["diagnostics_retained"], 3) - self.assertEqual(snapshot["diagnostics_recorded"], 5) - # A caller's own stamp is kept, and the retained record is what the - # recorder handed back. - stamped = telemetry.record_publication_diagnostic( - AcceptedPreviewPublicationDiagnostic( - block_hash="ff" * 32, - block_height=9, - result="degraded", - acceptance_to_publication_seconds=6.0, - recorded_monotonic=42.0, - ) - ) - self.assertEqual(stamped.recorded_monotonic, 42.0) - self.assertIs(telemetry.diagnostics_snapshot()[-1], stamped) - with self.assertRaises(TypeError): - telemetry.record_publication_diagnostic({"block_hash": "ab" * 32}) # type: ignore[arg-type] - with self.assertRaises(ValueError): - AcceptedPreviewTelemetry(diagnostics_capacity=0) - - def test_ensure_owner_attaches_once_and_is_shared(self) -> None: - owner = SimpleNamespace() - first = ensure_accepted_preview_telemetry(owner) - self.assertIs(ensure_accepted_preview_telemetry(owner), first) - self.assertIs(owner.__dict__["_accepted_preview_telemetry"], first) - # Recording through one handle is visible through the other: the - # renderer and every Wave 1 lane share one owner per coordinator. - first.observe_landing_phase("lane_wait", 0.1) - self.assertEqual( - ensure_accepted_preview_telemetry(owner).snapshot()["landing_phases"]["lane_wait"]["count"], - 1, - ) - self.assertIsNot(ensure_accepted_preview_telemetry(SimpleNamespace()), first) - with self.assertRaises(TypeError): - ensure_accepted_preview_telemetry(object()) - - def test_ensure_owner_is_race_free(self) -> None: - owner = SimpleNamespace() - seen: list[AcceptedPreviewTelemetry] = [] - seen_lock = threading.Lock() - barrier = threading.Barrier(16) - - def worker() -> None: - barrier.wait() - telemetry = ensure_accepted_preview_telemetry(owner) - with seen_lock: - seen.append(telemetry) - - pool = [threading.Thread(target=worker) for _ in range(16)] - for thread in pool: - thread.start() - for thread in pool: - thread.join() - self.assertEqual(len(seen), 16) - self.assertEqual(len({id(telemetry) for telemetry in seen}), 1) - - -class DiagnosticRecordTests(unittest.TestCase): - def test_normalizes_hash_and_bounds_every_field(self) -> None: - record = AcceptedPreviewPublicationDiagnostic( - block_hash=" " + "AB" * 32 + " ", - block_height="12", # type: ignore[arg-type] - result="published", - acceptance_to_publication_seconds=-3.0, - phase_seconds={"reconcile": 1.25, "preview_publish": math.nan}, - reconcile_caller="landing", - full_rescan_reason="not-a-contract-reason", - full_rescan_path="daemon", - full_rescan_seconds=math.inf, - ledger_gate_wait_seconds=0.5, - ledger_execute_seconds=-0.5, - ) - self.assertEqual(record.block_hash, "ab" * 32) - self.assertEqual(record.block_height, 12) - self.assertEqual(record.acceptance_to_publication_seconds, 0.0) - self.assertEqual(tuple(record.phase_seconds), PRISM_ACCEPTED_LANDING_PHASES) - self.assertEqual(record.phase_seconds["reconcile"], 1.25) - self.assertEqual(record.phase_seconds["preview_publish"], 0.0) - self.assertEqual(record.phase_seconds["lane_wait"], 0.0) - self.assertEqual(record.full_rescan_reason, "other") - self.assertEqual(record.full_rescan_seconds, 0.0) - self.assertEqual(record.ledger_execute_seconds, 0.0) - with self.assertRaises(TypeError): - record.phase_seconds["reconcile"] = 9.0 # type: ignore[index] - - def test_refuses_values_outside_the_vocabularies(self) -> None: - base = dict( - block_hash="ab" * 32, - block_height=1, - result="published", - acceptance_to_publication_seconds=1.0, - ) - for result in PRISM_ACCEPTED_BLOCK_PREVIEW_PUBLICATION_RESULTS: - AcceptedPreviewPublicationDiagnostic(**dict(base, result=result)) - with self.assertRaises(ValueError): - AcceptedPreviewPublicationDiagnostic(**dict(base, result="timeout")) - with self.assertRaises(ValueError): - AcceptedPreviewPublicationDiagnostic(**dict(base, block_hash=" ")) - with self.assertRaises(ValueError): - AcceptedPreviewPublicationDiagnostic( - **dict(base, phase_seconds={"submitblock-rpc": 0.1}) - ) - with self.assertRaises(ValueError): - AcceptedPreviewPublicationDiagnostic(**dict(base, reconcile_caller="replay")) - with self.assertRaises(ValueError): - AcceptedPreviewPublicationDiagnostic(**dict(base, full_rescan_path="rust")) - - def test_log_fields_are_flat_json_serializable_and_fixed(self) -> None: - record = AcceptedPreviewPublicationDiagnostic( - block_hash="cd" * 32, - block_height=7, - result="degraded", - acceptance_to_publication_seconds=4.123456789, - phase_seconds={"reconcile": 3.0}, - reconcile_caller="post_confirm", - full_rescan_reason="reconcile_invalidation", - full_rescan_path="daemon", - full_rescan_seconds=2.5, - ) - fields = record.log_fields() - json.dumps(fields, sort_keys=True) - self.assertEqual( - set(fields), - { - "block_hash", - "block_height", - "result", - "acceptance_to_publication_seconds", - "reconcile_caller", - "full_rescan_reason", - "full_rescan_path", - "full_rescan_seconds", - "ledger_gate_wait_seconds", - "ledger_execute_seconds", - *(f"phase_{phase}_seconds" for phase in PRISM_ACCEPTED_LANDING_PHASES), - }, - ) - self.assertEqual(fields["acceptance_to_publication_seconds"], 4.123457) - self.assertEqual(fields["phase_reconcile_seconds"], 3.0) - self.assertEqual(fields["phase_lane_wait_seconds"], 0.0) - summary = record.summary() - self.assertIn("hash=" + "cd" * 32, summary) - self.assertIn("rescan=reconcile_invalidation/daemon=2.500s", summary) - self.assertIn("reconcile=3.000s", summary) - - -class LedgerReadFoldTests(unittest.TestCase): - STATS = { - "calls_total": 2, - "gate_wait_seconds_total": 0.25, - "gate_wait_seconds_max": 0.2, - "gate_timeouts_total": 1, - "execute_seconds_total": 1.0, - "execute_seconds_max": 0.75, - "execute_timeouts_total": 0, - } - - def test_contract_operations_pass_through_as_copies(self) -> None: - source = { - "pending_block_candidate_rows": dict(self.STATS), - "payout_window_snapshot": dict(self.STATS, calls_total=5), - } - folded = fold_ledger_read_stats(source) - self.assertEqual(folded, source) - folded["payout_window_snapshot"]["calls_total"] = 99 - self.assertEqual(source["payout_window_snapshot"]["calls_total"], 5) - self.assertEqual(fold_ledger_read_stats({}), {}) - - def test_out_of_contract_operations_merge_into_other(self) -> None: - folded = fold_ledger_read_stats( - { - "snapshot_at_job_issue": dict(self.STATS), - "pending_block_candidate_rows": dict(self.STATS), - "replay_probe": dict(self.STATS, gate_wait_seconds_max=0.9, execute_timeouts_total=3), - } - ) - self.assertEqual(set(folded), {"pending_block_candidate_rows", "other"}) - other = folded["other"] - self.assertEqual(other["calls_total"], 4) - self.assertIsInstance(other["calls_total"], int) - self.assertAlmostEqual(other["gate_wait_seconds_total"], 0.5) - self.assertEqual(other["gate_wait_seconds_max"], 0.9) - self.assertEqual(other["execute_seconds_max"], 0.75) - self.assertEqual(other["gate_timeouts_total"], 2) - self.assertEqual(other["execute_timeouts_total"], 3) - - -class RendererContractTests(unittest.TestCase): - def _labels(self, line: str) -> dict[str, str]: - return dict(re.findall(r'(\w+)="([^"]*)"', line)) - - def test_empty_owner_renders_deterministically_at_zero(self) -> None: - port = SimpleNamespace() - renderer = MetricsRenderer(port) # type: ignore[arg-type] - first = renderer.accepted_preview_attribution_metrics_lines() - second = renderer.accepted_preview_attribution_metrics_lines() - self.assertEqual(first, second) - families = [ - line.split()[2] for line in first if line.startswith("# TYPE ") - ] - self.assertEqual( - families, - [ - "qbit_prism_accepted_block_landing_phase_seconds", - "qbit_prism_reorg_reconcile_pass_seconds", - "qbit_prism_reorg_reconcile_step_seconds", - "qbit_prism_payout_window_full_rescan_seconds", - ], - ) - samples = [line for line in first if not line.startswith("#")] - for line in samples: - value = line.rsplit(" ", 1)[1] - self.assertIn(value, ("0", "0.000000"), line) - # Sample lines are unique: no cell renders twice. - self.assertEqual(len(samples), len({line.split(" ")[0] for line in samples})) - - def test_rendered_labels_stay_inside_the_vocabularies(self) -> None: - port = SimpleNamespace() - telemetry = ensure_accepted_preview_telemetry(port) - block_hash = "ab" * 32 - telemetry.observe_landing_phase("chain_probe", 0.3) - telemetry.observe_reconcile_pass("other", 0.1) - telemetry.observe_reconcile_step("other", "publish", 0.05) - telemetry.observe_payout_window_full_rescan("hash=" + block_hash, "daemon", 2.0) - telemetry.record_publication_diagnostic( - AcceptedPreviewPublicationDiagnostic( - block_hash=block_hash, - block_height=4242, - result="published", - acceptance_to_publication_seconds=4.5, - ) - ) - lines = MetricsRenderer(port).accepted_preview_attribution_metrics_lines() # type: ignore[arg-type] - payload = "\n".join(lines) - self.assertNotIn(block_hash, payload) - self.assertNotIn("4242", payload) - allowed = { - "phase": set(PRISM_ACCEPTED_LANDING_PHASES), - "caller": set(PRISM_REORG_RECONCILE_CALLERS), - "step": set(PRISM_REORG_RECONCILE_STEPS), - "reason": set(PRISM_PAYOUT_WINDOW_FULL_RESCAN_REASONS), - "path": set(PRISM_PAYOUT_WINDOW_FULL_RESCAN_PATHS), - } - for line in lines: - if line.startswith("#"): - continue - labels = self._labels(line) - self.assertTrue(labels, line) - for name, value in labels.items(): - self.assertIn(value, allowed[name], line) - self.assertIn( - 'qbit_prism_payout_window_full_rescan_seconds_count{reason="other",path="daemon"} 1', - lines, - ) - self.assertIn( - 'qbit_prism_accepted_block_landing_phase_seconds_sum{phase="chain_probe"} 0.300000', - lines, - ) - - def test_public_surface_is_declared(self) -> None: - for name in contract.__all__: - self.assertTrue(hasattr(contract, name), name) - for name in VOCABULARIES: - self.assertIn(name, contract.__all__) - - -if __name__ == "__main__": - unittest.main() diff --git a/tests/test_prism_admission_lock.py b/tests/test_prism_admission_lock.py deleted file mode 100644 index 1214b181c..000000000 --- a/tests/test_prism_admission_lock.py +++ /dev/null @@ -1,651 +0,0 @@ -#!/usr/bin/env python3 -"""Admission-path lock isolation for the reconnect herd (issue #159). - -The coordinator lock used to serialize three unrelated things that a -reconnect herd hits at once: per-connection stale-grace anchoring, first-job -queue bookkeeping, and an O(N) fleet coverage census run once per delivered -job. These tests pin each of them to its own owner, and pin the lock order -that keeps the split deadlock-free. -""" - -from __future__ import annotations - -import threading -import time -import unittest -from decimal import Decimal -from types import SimpleNamespace - -from lab.prism.prism_coordinator import ( - ClientState, - PendingInitialJob, - PrismCoordinator, - WorkerIdentity, -) - - -HERD = 1024 -TIP_A = "aa" * 32 -TIP_B = "bb" * 32 - - -class FakeSocket: - def __init__(self) -> None: - self.closed = False - - def settimeout(self, _timeout: object) -> None: - return - - def setsockopt(self, *_args: object) -> None: - return - - def shutdown(self, _how: object) -> None: - return - - def close(self) -> None: - self.closed = True - - -class TrackedLock: - """Plain lock that remembers its current owner thread.""" - - def __init__(self) -> None: - self._lock = threading.Lock() - self._owner: int | None = None - - def acquire(self, blocking: bool = True, timeout: float = -1) -> bool: - acquired = self._lock.acquire(blocking, timeout) - if acquired: - self._owner = threading.get_ident() - return acquired - - def release(self) -> None: - self._owner = None - self._lock.release() - - def __enter__(self) -> "TrackedLock": - self.acquire() - return self - - def __exit__(self, *_args: object) -> None: - self.release() - - def held_by_current_thread(self) -> bool: - return self._owner == threading.get_ident() - - -class CountingLock: - """Coordinator-lock double that counts acquisitions and pins lock order. - - Acquisitions are counted while the lock is held, so the count is exact - under any amount of contention. ``forbidden`` names a lock that must never - be held when this one is taken; a violation there is the reverse nesting - that would make the admission split deadlock-prone. - """ - - def __init__(self, forbidden: TrackedLock | None = None) -> None: - self._lock = threading.RLock() - self._forbidden = forbidden - self.acquisitions = 0 - self.order_violations = 0 - - def acquire(self, blocking: bool = True, timeout: float = -1) -> bool: - forbidden = self._forbidden - if forbidden is not None and forbidden.held_by_current_thread(): - self.order_violations += 1 - acquired = self._lock.acquire(blocking, timeout) - if acquired: - self.acquisitions += 1 - return acquired - - def release(self) -> None: - self._lock.release() - - def __enter__(self) -> "CountingLock": - self.acquire() - return self - - def __exit__(self, *_args: object) -> None: - self.release() - - -class AnchorRecordingClient(ClientState): - """ClientState that counts every commit of the stale-grace anchor.""" - - @property - def tip_work_delivered(self) -> tuple[str, float] | None: - return self.__dict__.get("_anchor") - - @tip_work_delivered.setter - def tip_work_delivered(self, value: tuple[str, float] | None) -> None: - self.__dict__["_anchor"] = value - if value is not None: - self.__dict__["_anchor_writes"] = ( - int(self.__dict__.get("_anchor_writes", 0)) + 1 - ) - - @property - def anchor_writes(self) -> int: - return int(self.__dict__.get("_anchor_writes", 0)) - - -def worker(name: str) -> WorkerIdentity: - return WorkerIdentity( - username=name, - payout_address=name, - worker_name=None, - script_pubkey_hex="5220" + "11" * 32, - p2mr_program_hex="11" * 32, - ) - - -def coordinator( - *, - connection_limit: int = HERD + 8, - pending_limit: int = HERD, -) -> PrismCoordinator: - server = PrismCoordinator.__new__(PrismCoordinator) - server.stop_event = threading.Event() - server.clients = set() - server.jobs = {} - server.connection_counter = 0 - server.job_counter = 0 - server.stratum_max_connections = connection_limit - server.stratum_max_connections_per_username = 0 - server.stratum_max_pending_initial_jobs = pending_limit - server.stratum_initial_job_timeout_seconds = 30.0 - server.mining_health_startup_grace_seconds = 30.0 - server.initial_job_max_workers = 1 - server.tip_refresh_max_workers = 1 - server.tip_template_snapshot = None - server.started_monotonic = 0.0 - server.submitted_share_count = 0 - server.rejection_counts_by_reason = {} - server._record_heartbeat = lambda _name: None # type: ignore[method-assign] - server.apply_stratum_send_timeout = lambda _sock: None # type: ignore[method-assign] - server.client_startup_difficulty = lambda _profile: Decimal("1") # type: ignore[method-assign] - # The S2 admission lock is a private implementation detail; the lock-order - # assertions below are the reason this test reaches for it. - admission = TrackedLock() - server.lock = CountingLock(forbidden=admission) - server._ensure_job_cache_state() - server._ensure_tip_refresh_state() - server._ensure_initial_job_state() - service = server._ensure_job_delivery_service() - service._initial_job_admission_lock = admission - return server - - -def client( - server: PrismCoordinator, - connection_id: int, - *, - with_job: bool = False, - recording: bool = False, -) -> ClientState: - factory = AnchorRecordingClient if recording else ClientState - state = factory( - sock=FakeSocket(), - address=("127.0.0.1", 40_000 + connection_id), - connection_id=connection_id, - extranonce1_hex=f"{connection_id:08x}", - ) - state.subscribed = True - state.authorized = True - state.authorization_generation = 1 - state.worker = worker(f"miner-{connection_id}") - state.username = state.worker.username - if with_job: - state.active_job = SimpleNamespace( - template={"previousblockhash": TIP_A}, - payout_state_generation=0, - ) - server.clients.add(state) - return state - - -def run_concurrently(targets: list, *, timeout: float = 60.0) -> None: - """Start every target behind one barrier and join them all.""" - barrier = threading.Barrier(len(targets), timeout=timeout) - - def entry(target) -> None: # type: ignore[no-untyped-def] - barrier.wait() - target() - - threads = [threading.Thread(target=entry, args=(target,)) for target in targets] - for thread in threads: - thread.start() - for thread in threads: - thread.join(timeout) - for thread in threads: - assert not thread.is_alive(), "worker thread did not finish" - - -class DeliveryAnchorIsolationTests(unittest.TestCase): - def test_contended_coordinator_lock_cannot_delay_a_delivery_anchor(self) -> None: - server = coordinator() - blocked = client(server, 1) - delivering = client(server, 2) - held = threading.Event() - release = threading.Event() - - def hold_coordinator_lock() -> None: - with server.lock: - held.set() - release.wait(30) - - holder = threading.Thread(target=hold_coordinator_lock) - holder.start() - try: - self.assertTrue(held.wait(10)) - before = server.lock.acquisitions - started = time.monotonic() - server.note_tip_work_delivered(delivering, TIP_A) - elapsed = time.monotonic() - started - finally: - release.set() - holder.join(10) - - # A different connection's first delivery completed while the whole - # control plane was blocked: it neither took nor waited for the - # coordinator lock. - self.assertEqual(server.lock.acquisitions, before) - self.assertLess(elapsed, 5.0) - self.assertIsNotNone(delivering.tip_work_delivered) - assert delivering.tip_work_delivered is not None - self.assertEqual(delivering.tip_work_delivered[0], TIP_A) - self.assertIsNone(blocked.tip_work_delivered) - self.assertEqual(server.lock.order_violations, 0) - - def test_same_tip_refresh_race_cannot_slide_an_existing_anchor(self) -> None: - server = coordinator() - state = client(server, 1) - - server.note_tip_work_delivered(state, TIP_A) - first = state.tip_work_delivered - self.assertIsNotNone(first) - - # However many same-tip refreshes race, the stale-grace anchor stays - # on the first delivery for that tip. - run_concurrently( - [lambda: server.note_tip_work_delivered(state, TIP_A) for _ in range(16)] - ) - self.assertEqual(state.tip_work_delivered, first) - - # A new tip is a new anchor. - server.note_tip_work_delivered(state, TIP_B) - anchored = state.tip_work_delivered - assert anchored is not None and first is not None - self.assertEqual(anchored[0], TIP_B) - self.assertGreaterEqual(anchored[1], first[1]) - - def test_reconnect_herd_first_deliveries_anchor_each_connection_once(self) -> None: - server = coordinator() - clients = [ - client(server, index + 1, recording=True) for index in range(HERD) - ] - before = server.lock.acquisitions - - def deliver(state: ClientState) -> None: - server.note_tip_work_delivered(state, TIP_A) - - # Four threads race the first delivery of every connection at once. - run_concurrently( - [ - (lambda state=state: deliver(state)) - for state in clients - for _ in range(4) - ] - ) - - self.assertEqual(server.lock.acquisitions, before) - self.assertEqual(server.lock.order_violations, 0) - for state in clients: - anchor = state.tip_work_delivered - self.assertIsNotNone(anchor) - assert anchor is not None - self.assertEqual(anchor[0], TIP_A) - self.assertEqual(state.anchor_writes, 1) - - -class InitialJobAdmissionLockTests(unittest.TestCase): - def test_queue_bookkeeping_never_reaches_the_coordinator_lock(self) -> None: - server = coordinator(pending_limit=HERD) - started = threading.Event() - release = threading.Event() - - def blocked(request: PendingInitialJob) -> bool: - started.set() - release.wait(30) - return not server._initial_request_cancelled(request) - - server._run_initial_job = blocked # type: ignore[method-assign] - clients = [client(server, index + 1) for index in range(HERD)] - try: - # Fresh admission asks the coordinator exactly one constant-time - # question -- does this connection already hold current-tip work -- - # and nothing else. - before = server.lock.acquisitions - for state in clients: - self.assertTrue(server.schedule_initial_job(state)) - self.assertTrue(started.wait(10)) - self.assertEqual(server.lock.acquisitions - before, HERD) - self.assertEqual(len(server.pending_initial_jobs), HERD) - - # Coalescing a repeat request is pure queue bookkeeping. - before = server.lock.acquisitions - for state in clients: - self.assertTrue(server.schedule_initial_job(state)) - self.assertEqual(server.lock.acquisitions, before) - self.assertEqual(server.initial_job_coalesced_count, HERD) - self.assertEqual(len(server.pending_initial_jobs), HERD) - - # So is releasing every one of them again, including the physical - # queue reclamation its cancellation callbacks perform. - before = server.lock.acquisitions - for state in clients: - server.cancel_initial_job_delivery(state) - self.assertEqual(server.lock.acquisitions, before) - self.assertEqual(len(server.pending_initial_jobs), 0) - self.assertEqual(server.initial_job_cancelled_count, HERD) - self.assertEqual( - server.initial_job_queue_capacity_reclaimed_count, - HERD - 1, - ) - self.assertEqual(server.lock.order_violations, 0) - finally: - release.set() - server.shutdown_initial_job_executor() - - def test_queue_bookkeeping_runs_while_the_coordinator_lock_is_held(self) -> None: - server = coordinator(pending_limit=8) - submitted: list[PendingInitialJob] = [] - server._submit_initial_job_request = ( # type: ignore[method-assign] - lambda request: bool(submitted.append(request)) or True - ) - state = client(server, 1) - self.assertTrue(server.schedule_initial_job(state)) - self.assertEqual(len(submitted), 1) - - held = threading.Event() - release = threading.Event() - - def hold_coordinator_lock() -> None: - with server.lock: - held.set() - release.wait(30) - - finished = threading.Event() - - def bookkeep() -> None: - # Coalescing and cancellation are admission-domain only, so a - # blocked control plane cannot stall a reconnecting miner's queue - # bookkeeping. - server.schedule_initial_job(state) - server.cancel_initial_job_delivery(state) - finished.set() - - holder = threading.Thread(target=hold_coordinator_lock) - bookkeeper = threading.Thread(target=bookkeep) - holder.start() - try: - self.assertTrue(held.wait(10)) - bookkeeper.start() - self.assertTrue( - finished.wait(5), - "queue bookkeeping waited on the contended coordinator lock", - ) - # Still held: the bookkeeping above ran through, not after, the - # control plane being blocked. - self.assertFalse(release.is_set()) - self.assertEqual(server.initial_job_coalesced_count, 1) - self.assertEqual(len(server.pending_initial_jobs), 0) - self.assertEqual(server.initial_job_cancelled_count, 1) - finally: - release.set() - holder.join(10) - bookkeeper.join(10) - self.assertEqual(server.lock.order_violations, 0) - - def test_concurrent_admission_enforces_the_exact_pending_capacity(self) -> None: - capacity = 64 - server = coordinator(pending_limit=capacity) - submitted: list[PendingInitialJob] = [] - submit_lock = threading.Lock() - - def record(request: PendingInitialJob) -> bool: - with submit_lock: - submitted.append(request) - return True - - server._submit_initial_job_request = record # type: ignore[method-assign] - clients = [client(server, index + 1) for index in range(HERD)] - admitted: list[bool] = [False] * HERD - - def admit(index: int) -> None: - admitted[index] = bool(server.schedule_initial_job(clients[index])) - - run_concurrently( - [(lambda index=index: admit(index)) for index in range(HERD)] - ) - - self.assertEqual(sum(admitted), capacity) - self.assertEqual(len(server.pending_initial_jobs), capacity) - self.assertEqual(len(submitted), capacity) - self.assertEqual(server.initial_job_queue_rejection_count, HERD - capacity) - # Every rejected connection was retired exactly once, and every - # admitted one still owns its slot. - self.assertEqual(len(server.clients), capacity) - for index, state in enumerate(clients): - self.assertEqual(server.pending_initial_jobs.get(state) is not None, - admitted[index]) - self.assertEqual(state.sock.closed, not admitted[index]) - self.assertEqual(server.lock.order_violations, 0) - - -class PendingQueueCompatibilitySurfaceTests(unittest.TestCase): - """The coordinator-facing queue surface is owned by the admission lock.""" - - def test_capacity_read_is_taken_under_the_admission_lock(self) -> None: - server = coordinator(pending_limit=8) - server._submit_initial_job_request = lambda _request: True # type: ignore[method-assign] - state = client(server, 1) - self.assertTrue(server.schedule_initial_job(state)) - - admission = server._ensure_job_delivery_service()._initial_job_admission_lock - started = threading.Event() - finished = threading.Event() - observed: list[object] = [] - - def capacity_read() -> None: - started.set() - observed.append(len(server.pending_initial_jobs)) - observed.append(state in server.pending_initial_jobs) - finished.set() - - reader = threading.Thread(target=capacity_read) - self.assertTrue(admission.acquire(timeout=5)) - try: - reader.start() - self.assertTrue(started.wait(5)) - # The external capacity read is the one the session owner's - # reauthorization pre-check performs. It must serialize with - # admission rather than run beside it. - self.assertFalse( - finished.wait(0.5), - "capacity read completed without the admission lock", - ) - finally: - admission.release() - reader.join(10) - - self.assertTrue(finished.is_set()) - self.assertEqual(observed, [1, True]) - - def test_external_reads_cannot_observe_a_partly_swept_queue(self) -> None: - cohort = 64 - server = coordinator(pending_limit=HERD) - clients = [client(server, index + 1) for index in range(cohort)] - observed: set[int] = set() - failures: list[BaseException] = [] - stop = threading.Event() - empty_observed = threading.Event() - cohort_observed = threading.Event() - - def reader() -> None: - try: - while not stop.is_set(): - size = len(server.pending_initial_jobs) - observed.add(size) - if size == 0: - empty_observed.set() - elif size == cohort: - cohort_observed.set() - except BaseException as exc: # pragma: no cover - failure path - failures.append(exc) - - watcher = threading.Thread(target=reader) - watcher.start() - try: - self.assertTrue( - empty_observed.wait(5), - "reader did not observe the empty queue before seeding", - ) - for _ in range(24): - expired = time.monotonic() - 1.0 - for state in clients: - state.closing = False - server.clients.add(state) - server.pending_initial_jobs = { - state: PendingInitialJob( - client=state, - connection_id=state.connection_id, - authorization_generation=1, - difficulty_generation=0, - worker=state.worker, - requested_monotonic=expired, - deadline_monotonic=expired, - ) - for state in clients - } - self.assertTrue( - cohort_observed.wait(5), - "reader did not observe the fully seeded queue", - ) - self.assertEqual(server.sweep_initial_job_timeouts(), cohort) - finally: - stop.set() - watcher.join(10) - - self.assertEqual(failures, []) - # The sweep releases the whole cohort inside one critical section and - # seeding installs it inside another, so an external reader sees the - # queue wholly pending or wholly released -- never mid-cancellation. - self.assertTrue( - observed.issubset({0, cohort}), - f"external read observed a partly swept queue: {sorted(observed)}", - ) - self.assertIn(cohort, observed) - self.assertIn(0, observed) - self.assertEqual(server.lock.order_violations, 0) - - def test_surface_is_strictly_immutable(self) -> None: - server = coordinator(pending_limit=4) - state = client(server, 1) - view = server.pending_initial_jobs - - # No write reaches admission state through the compatibility surface: - # every mutator is absent, not merely guarded. - for mutator in ( - "__setitem__", - "__delitem__", - "pop", - "popitem", - "clear", - "update", - "setdefault", - ): - self.assertFalse( - hasattr(view, mutator), - f"compatibility surface exposes {mutator}", - ) - request = PendingInitialJob( - client=state, - connection_id=state.connection_id, - authorization_generation=1, - difficulty_generation=0, - worker=state.worker, - requested_monotonic=0.0, - deadline_monotonic=None, - ) - with self.assertRaises(TypeError): - view[state] = request - with self.assertRaises(TypeError): - del view[state] - - # Seeding goes through the setter, which adopts the mapping's contents - # under the admission lock instead of exposing a mutable map. - server.pending_initial_jobs = {state: request} - self.assertEqual(len(server.pending_initial_jobs), 1) - self.assertIs(server.pending_initial_jobs[state], request) - - # The owner's mutable map is private and is never the object handed to - # a compatibility reader. - service = server._ensure_job_delivery_service() - self.assertIsNot(view, service._pending_initial_jobs) - self.assertIsInstance(service._pending_initial_jobs, dict) - - -class CoverageCensusOwnershipTests(unittest.TestCase): - def test_observability_refresh_not_delivery_resets_the_coverage_failure( - self, - ) -> None: - server = coordinator(connection_limit=4) - state = client(server, 1, with_job=True) - server.current_tip_first_seen = (TIP_A, 100.0) - observability = server._ensure_observability_service() - observability.set_delivery_failure_started_monotonic_for_test(50.0) - - # Delivery anchors this connection and does nothing else: no fleet - # census, no coordinator lock, no health-policy write. - before = server.lock.acquisitions - server.note_tip_work_delivered(state, TIP_A) - self.assertEqual(server.lock.acquisitions, before) - self.assertEqual( - observability.state().mining_delivery_failure_started_monotonic, - 50.0, - ) - - # The refresh census -- the one that already walks authorized clients - # and current-work coverage -- performs the reset. - inputs = observability.port.mining_delivery_inputs(120.0) - - self.assertEqual(inputs.authorized_connections, 1) - self.assertEqual(inputs.clients_with_current_tip_jobs, 1) - self.assertIsNone( - observability.state().mining_delivery_failure_started_monotonic - ) - - def test_refresh_below_the_coverage_threshold_leaves_the_failure_running( - self, - ) -> None: - server = coordinator(connection_limit=HERD) - covered = [client(server, index + 1, with_job=True) for index in range(90)] - for index in range(10): - client(server, 1_000 + index) - server.current_tip_first_seen = (TIP_A, 100.0) - observability = server._ensure_observability_service() - observability.set_delivery_failure_started_monotonic_for_test(50.0) - - inputs = observability.port.mining_delivery_inputs(120.0) - - self.assertEqual(inputs.authorized_connections, 100) - self.assertEqual(inputs.clients_with_current_tip_jobs, len(covered)) - self.assertEqual( - observability.state().mining_delivery_failure_started_monotonic, - 50.0, - ) - - -if __name__ == "__main__": - unittest.main() diff --git a/tests/test_prism_allocator_experiment.py b/tests/test_prism_allocator_experiment.py deleted file mode 100644 index 5d9712a45..000000000 --- a/tests/test_prism_allocator_experiment.py +++ /dev/null @@ -1,256 +0,0 @@ -#!/usr/bin/env python3 -"""Issue #226 part 2: the storm instrument's heap and allocator readings. - -These pin the mechanism the allocator experiment relies on, not any byte -count: that a phase reading carries every field the soak runbook reads, -that the arena probe multiplies glibc arenas at the default and stops at a -configured cap, that the experiment driver strips its own options before -re-invoking the instrument, and that ``--heap-report`` produces the section -the runbook compares across settings. -""" - -from __future__ import annotations - -import json -import os -import subprocess -import sys -import time -import unittest -from dataclasses import fields -from pathlib import Path - -from lab.prism.process_telemetry import ProcessHeapTelemetry, read_malloc_arena_count -from tests.prism_candidate_storm import ( - ARENA_PROBE_LIVE_BUFFERS, - ARENA_PROBE_MAX_BYTES, - ARENA_PROBE_MIN_BYTES, - HeapReading, - allocator_environment, - allocator_experiment_child_args, - run_arena_probe, - take_heap_reading, -) - -ROOT = Path(__file__).resolve().parents[1] -STORM = ROOT / "tests" / "prism_candidate_storm.py" - -HEAP_READING_FIELDS = ( - "phase", - "elapsed_seconds", - "resident_bytes", - "allocated_blocks", - "threads", - "malloc_info_available", - "malloc_arena_bytes", - "malloc_in_use_bytes", - "malloc_free_bytes", - "malloc_mmapped_bytes", - "malloc_arena_count", - "malloc_free_top_bytes", - "malloc_free_bin_bytes", - "anonymous_regions", - "anonymous_bytes", - "anonymous_regions_4mib_to_64mib", - "heap_segment_bytes", -) - - -class HeapReadingTests(unittest.TestCase): - def test_reading_carries_every_field_the_runbook_reads(self) -> None: - self.assertEqual(tuple(field.name for field in fields(HeapReading)), HEAP_READING_FIELDS) - reading = take_heap_reading("probe", ProcessHeapTelemetry(), 0.0) - self.assertEqual(reading.phase, "probe") - self.assertGreaterEqual(reading.elapsed_seconds, 0.0) - self.assertGreater(reading.allocated_blocks, 0) - self.assertGreaterEqual(reading.threads, 1) - if Path("/proc/self/statm").exists(): - self.assertGreater(reading.resident_bytes, 0) - self.assertGreaterEqual(reading.anonymous_regions, 1) - self.assertGreaterEqual(reading.anonymous_regions_4mib_to_64mib, 0) - if reading.malloc_info_available: - self.assertGreaterEqual(reading.malloc_arena_count, 1) - self.assertGreaterEqual(reading.malloc_arena_bytes, reading.malloc_in_use_bytes) - - def test_allocator_environment_reports_only_allocator_variables(self) -> None: - with unittest.mock.patch.dict( - os.environ, - {"MALLOC_ARENA_MAX": "2", "PYTHONMALLOC": "malloc", "PRISM_UNRELATED": "x"}, - ): - reported = allocator_environment() - self.assertEqual(reported.get("MALLOC_ARENA_MAX"), "2") - self.assertEqual(reported.get("PYTHONMALLOC"), "malloc") - self.assertNotIn("PRISM_UNRELATED", reported) - - -class ArenaProbeTests(unittest.TestCase): - def test_probe_buffers_land_in_glibc_arenas(self) -> None: - """Over pymalloc's ceiling, under glibc's initial mmap threshold.""" - self.assertGreater(ARENA_PROBE_MIN_BYTES, 512) - self.assertLess(ARENA_PROBE_MAX_BYTES, 128 * 1024) - self.assertGreater(ARENA_PROBE_LIVE_BUFFERS, 0) - - def test_probe_multiplies_arenas_at_default_and_respects_the_cap(self) -> None: - """glibc gives concurrent threads their own arenas until MALLOC_ARENA_MAX.""" - if read_malloc_arena_count() < 0: - self.skipTest("malloc_info unavailable on this platform") - script = ( - "import sys, json\n" - f"sys.path.insert(0, {str(ROOT)!r})\n" - "from tests.prism_candidate_storm import run_arena_probe\n" - "print(json.dumps(run_arena_probe(12, 40)))\n" - ) - uncapped = subprocess.run( - [sys.executable, "-c", script], - env={k: v for k, v in os.environ.items() if k != "MALLOC_ARENA_MAX"}, - capture_output=True, - text=True, - check=True, - ) - capped = subprocess.run( - [sys.executable, "-c", script], - env={**os.environ, "MALLOC_ARENA_MAX": "3"}, - capture_output=True, - text=True, - check=True, - ) - free_run = json.loads(uncapped.stdout.strip().splitlines()[-1]) - cap_run = json.loads(capped.stdout.strip().splitlines()[-1]) - self.assertEqual(free_run["threads"], 12) - self.assertEqual(free_run["rounds"], 40) - self.assertGreater(free_run["arena_count_after"], free_run["arena_count_before"]) - self.assertGreater(free_run["arena_count_after"], 3) - self.assertLessEqual(cap_run["arena_count_after"], 3) - for run in (free_run, cap_run): - self.assertEqual(run["failures"], []) - self.assertGreaterEqual(run["seconds"], 0.0) - self.assertTrue(run["malloc_info_available"]) - # Freed rings stay on the arenas' free lists: the mechanism the - # trim is later judged by. - self.assertGreaterEqual(run["malloc_free_after"], run["malloc_free_before"]) - # In-process, with the cap of this process (whatever it is), the - # probe still returns the full record. - record = run_arena_probe(2, 5) - self.assertEqual(set(record), set(free_run)) - self.assertEqual(record["failures"], []) - - def test_probe_reports_a_failing_worker_instead_of_wedging(self) -> None: - """One worker that raises breaks the barrier for the rest; the probe returns.""" - calls = {"count": 0} - - def failing_allocate(size: int) -> bytes: - calls["count"] += 1 - if calls["count"] == 1: - raise RuntimeError("synthetic allocation failure") - return bytes(size) - - started = time.monotonic() - record = run_arena_probe(4, 3, allocate=failing_allocate) - self.assertLess(time.monotonic() - started, 30.0) - self.assertEqual(record["threads"], 4) - self.assertTrue(record["failures"], record) - self.assertTrue( - any("synthetic allocation failure" in failure for failure in record["failures"]), - record["failures"], - ) - - -class ExperimentDriverTests(unittest.TestCase): - def test_child_args_strip_the_experiment_options_and_add_heap_report(self) -> None: - self.assertEqual( - allocator_experiment_child_args( - ["--decide", "--allocator-experiment", "2,4", "--drain-per-row"] - ), - ["--decide", "--drain-per-row", "--heap-report"], - ) - self.assertEqual( - allocator_experiment_child_args( - [ - "--allocator-experiment=default", - "--allocator-mmap-threshold=131072", - "--heap-report", - "--candidates", - "8", - ] - ), - ["--heap-report", "--candidates", "8"], - ) - self.assertEqual( - allocator_experiment_child_args(["--allocator-mmap-threshold", "131072"]), - ["--heap-report"], - ) - # --verbose never reaches a child: its logs would precede the JSON - # the parent parses and the parse would fail after the whole storm. - self.assertEqual( - allocator_experiment_child_args(["--verbose", "--decide", "--allocator-experiment", "2"]), - ["--decide", "--heap-report"], - ) - - def test_heap_report_section_is_produced_end_to_end(self) -> None: - """A small storm with --heap-report carries the readings the runbook compares.""" - completed = subprocess.run( - [ - sys.executable, - str(STORM), - "--candidates", - "40", - "--queue-depth", - "8", - "--heap-report", - "--arena-probe-threads", - "2", - "--arena-probe-rounds", - "10", - ], - cwd=ROOT, - capture_output=True, - text=True, - check=True, - ) - report = json.loads(completed.stdout) - heap = report["heap"] - self.assertEqual( - set(heap), - { - "python", - "platform", - "libc", - "machine", - "cpu_count", - "allocator_env", - "phase_seconds", - "readings", - "arena_probe", - "malloc_trim", - }, - ) - phases = [reading["phase"] for reading in heap["readings"]] - self.assertEqual( - phases, - [ - "start", - "after_seed_live", - "after_restart_enumerate", - "after_arena_probe", - "after_release_gc", - "after_malloc_trim", - ], - ) - for reading in heap["readings"]: - self.assertEqual(set(reading), set(HEAP_READING_FIELDS)) - self.assertEqual(set(heap["phase_seconds"]), {"seed_live", "restart_enumerate"}) - self.assertEqual(heap["arena_probe"]["threads"], 2) - self.assertEqual(heap["arena_probe"]["failures"], []) - if heap["readings"][0]["malloc_info_available"]: - trim = heap["malloc_trim"] - self.assertIsNotNone(trim) - self.assertEqual(trim["in_use_delta"], 0) - self.assertLessEqual(trim["free_delta"], 0) - # The storm sections the instrument already produced are untouched. - self.assertEqual(report["input"], {"candidates": 40, "queue_depth": 8}) - self.assertIn("live", report) - self.assertIn("restart", report) - - -if __name__ == "__main__": - unittest.main() diff --git a/tests/test_prism_ancestor_redrive.py b/tests/test_prism_ancestor_redrive.py deleted file mode 100644 index 32a9c7d18..000000000 --- a/tests/test_prism_ancestor_redrive.py +++ /dev/null @@ -1,809 +0,0 @@ -#!/usr/bin/env python3 -"""In-process targeted ancestor re-drive for stuck payout transitions (#190). - -The production wedge these tests reproduce (2026-08-24 smoke test): a found -block's finalization deferred on an ancestor whose accepted payout -transition was armed and landed but whose preview never published. The -retained-retry loop could only re-check the transition -- 37 cycles of -``ledger-confirmation-failed`` over 906 seconds -- while the durable-replay -pass, the one path that provably resolves the ancestor (startup replay -resolved the same state in ~19s after the watchdog restart), was -short-circuited by the very retry candidate it needed to unblock. - -The fix under test: a per-ancestor streak of finalization deferrals arms a -targeted re-drive; the next ``replay_pending`` pass bypasses the -retry/live/replay-queue short-circuits, re-enumerates the durable outbox -through the same adoption startup replay uses (same pagination, same -landing-class budget), and either re-queues the ancestor's pending row for -ordinary replay finalization or -- when the completed enumeration proves no -pending row exists, nothing in-process owns the hash, the block is durably -confirmed, and a fresh chain probe proves it active on the node's chain -- -clears the stale transition. The attempt cap bounds the mechanism per -ancestor (each attempt needs a fresh deferral streak); on exhaustion -deferrals fall back to the pre-#190 behavior and the publication-progress -watchdog remains the backstop. -""" -# ruff: noqa: E402 - -from __future__ import annotations - -import dataclasses -import sys -import time -import unittest -from pathlib import Path -from types import SimpleNamespace -from unittest.mock import patch - -if not __package__: - sys.path.insert(0, str(Path(__file__).resolve().parents[1])) - -from lab.prism.share_ledger import PendingShare, SingleWriterShareLedger -from tests.prism_coordinator_test_support import ( - coordinator as light_coordinator, -) -from tests.prism_vardiff_test_support import block_candidate, submit_coordinator - - -ANCESTOR_HASH = "aa" * 32 -CHILD_HASH = "cc" * 32 -PARENT_TIP = "00" * 32 - - -def _pending_share(block_hash: str, stamp: int) -> PendingShare: - return PendingShare( - share_id=f"miner-a:{block_hash}", - miner_id="miner-a", - order_key="miner-a", - p2mr_program_hex="11" * 32, - share_difficulty=1, - network_difficulty=1, - template_height=9, - job_id="job-1", - job_issued_at_ms=1, - accepted_at_ms=stamp, - ntime=1, - ) - - -def _wedged_coordinator(): - """The incident shape: retained child, armed+landed ancestor, no preview. - - The ancestor's durable outbox row stays pending (its replay source), its - in-memory transition is armed and landed with the preview never - published, and the child found on top of it is retained in the retry - slot -- exactly the state the 2026-08-24 run wedged in. - """ - server, state, _recording = submit_coordinator() - ledger = SingleWriterShareLedger() - server.ledger = ledger - server.block_candidate_retry_initial_seconds = 0.0 - server.block_candidate_retry_max_seconds = 0.0 - ancestor = block_candidate( - server, - state, - SimpleNamespace( - coinbase_tx_hex="00", - block_hash_hex=ANCESTOR_HASH, - block_hex="00", - share_pass=False, - block_pass=True, - ), - pending_share=_pending_share(ANCESTOR_HASH, 1), - ) - ledger.persist_block_candidate_intent(server.block_candidate_intent(ancestor)) - server._begin_accepted_block_payout_preview(ANCESTOR_HASH, block_height=10) - server._mark_accepted_block_payout_landed(ANCESTOR_HASH, block_height=10) - child_context = SimpleNamespace( - job=SimpleNamespace( - job_id="job-2", - share_target=server.jobs["job-1"].job.share_target, - share_difficulty=server.jobs["job-1"].job.share_difficulty, - transaction_hexes=(), - ), - template={ - "previousblockhash": ANCESTOR_HASH, - "height": 11, - "coinbasevalue": 50_00000000, - }, - found_block={"network_difficulty": 1}, - issued_at_ms=12346, - collection_only=False, - worker=state.worker, - shares_json=[], - prior_balances=[], - ) - server.jobs["job-2"] = child_context - state.active_job_ids.add("job-2") - child = block_candidate( - server, - state, - SimpleNamespace( - coinbase_tx_hex="00", - block_hash_hex=CHILD_HASH, - block_hex="00", - share_pass=False, - block_pass=True, - ), - job_id="job-2", - pending_share=_pending_share(CHILD_HASH, 2), - ) - # The child's intent is durable too (persisted at admission in the - # incident), so a forced enumeration sees a pending row for a hash the - # retry holder still owns -- adoption must skip it. - ledger.persist_block_candidate_intent(server.block_candidate_intent(child)) - server._retain_block_candidate_for_retry(child) - return server, state, ledger, child - - -def _script_chain_probe(server, result): - """Pin the sweep's node-side probe; returns the recorded calls.""" - service = server._ensure_block_candidate_service() - calls: list[tuple[str, int | None]] = [] - - def probe(block_hash, *, expected_height=None): - calls.append((str(block_hash).lower(), expected_height)) - return result - - service._block_candidate_chain_probe = probe - return calls - - -class SweepLedger(SingleWriterShareLedger): - """Empty outbox plus a caller-scripted durable pool-block verdict.""" - - def __init__(self, block_state: dict[str, object] | None) -> None: - super().__init__() - self.scripted_block_state = block_state - self.pool_block_state_calls: list[str] = [] - - def pool_block_state(self, *, block_hash: str) -> dict[str, object] | None: - self.pool_block_state_calls.append(block_hash) - return self.scripted_block_state - - -class AncestorRedriveWedgeTests(unittest.TestCase): - """Issue #190 finding 1: the wedge, and the re-drive that resolves it.""" - - def _child_defers(self, server) -> bool: - return server._defer_for_pending_parent_payout_transition( - block_hash=CHILD_HASH, - parent_hash=ANCESTOR_HASH, - parent_height=10, - worker=None, - ) - - def test_deferral_streak_redrives_the_stuck_ancestor(self) -> None: - server, _state, _ledger, _child = _wedged_coordinator() - server.accepted_parent_redrive_defer_threshold = 3 - server.accepted_parent_redrive_attempt_max = 3 - service = server._ensure_block_candidate_service() - - # The pre-fix wedge: every retry cycle re-checks the transition and - # defers again, while the durable replay pass -- the only code that - # can resolve the ancestor -- short-circuits on the retained retry - # candidate it exists to unblock. - with patch("builtins.print"): - for _cycle in range(2): - self.assertTrue(self._child_defers(server)) - self.assertEqual(server.replay_pending_block_candidates(), 0) - self.assertFalse(service._ancestor_redrive_owed()) - - # Crossing the deferral threshold arms the targeted re-drive. - self.assertTrue(self._child_defers(server)) - self.assertTrue(service._ancestor_redrive_owed()) - self.assertEqual(int(server.accepted_parent_redrive_attempt_count), 1) - - # The armed pass bypasses the retry short-circuit and re-adopts the - # ancestor's pending row through the same durable-replay code - # startup replay runs. The child's pending row is durable too, but - # its live copy sits in the retry holder, so adoption must skip it - # rather than install a second in-process copy under the holder. - with patch("builtins.print"): - self.assertEqual(server.replay_pending_block_candidates(), 1) - self.assertFalse(service._ancestor_redrive_owed()) - with server.lock: - self.assertIn(ANCESTOR_HASH, server._block_replay_inflight_hashes) - self.assertNotIn(CHILD_HASH, server._block_replay_inflight_hashes) - - # The submitter's next pass runs the re-driven ancestor while the - # child's retry is parked in backoff. The finalization tail itself - # is owned by the block-finalization suites; the double here - # resolves the transition through the same production payout-state - # seam that tail uses on completion. - landed: list[str] = [] - - def finalize(candidate, **_kwargs) -> bool: - block_hash = str(candidate.submission.block_hash_hex).lower() - landed.append(block_hash) - server._clear_accepted_block_payout_preview(block_hash) - return True - - server.submit_block_candidate = finalize - # Post-accept job fanout is the tip-refresh owner's machinery, not - # this wedge's; the replay-reconstructed client is not a full - # ClientState in this fixture. - server.refresh_jobs_after_pending_accepted_block = ( - lambda *_args, **_kwargs: 0 - ) - server._block_candidate_retry_not_before = { - CHILD_HASH: time.monotonic() + 300.0 - } - with patch("builtins.print"): - self.assertTrue(server.submit_next_block_candidate()) - self.assertEqual(landed, [ANCESTOR_HASH]) - with server._accepted_block_payout_preview_condition: - self.assertNotIn( - ANCESTOR_HASH, server._accepted_block_payout_previews - ) - - # With the ancestor resolved, the retained child's next retry passes - # the fence and its finalization completes. - with patch("builtins.print"): - self.assertFalse(self._child_defers(server)) - self.assertEqual(int(server.accepted_parent_redrive_resolved_count), 1) - server._clear_block_candidate_retry_state(CHILD_HASH) - with patch("builtins.print"): - self.assertTrue(server.submit_next_block_candidate()) - self.assertEqual(landed, [ANCESTOR_HASH, CHILD_HASH]) - - def test_streak_below_threshold_never_arms_a_redrive(self) -> None: - server, _state, _ledger, _child = _wedged_coordinator() - server.accepted_parent_redrive_defer_threshold = 5 - service = server._ensure_block_candidate_service() - - with patch("builtins.print"): - for _cycle in range(4): - self.assertTrue(self._child_defers(server)) - self.assertEqual(server.replay_pending_block_candidates(), 0) - self.assertFalse(service._ancestor_redrive_owed()) - self.assertEqual(int(server.accepted_parent_redrive_attempt_count), 0) - - def test_redrive_respects_the_attempt_cap_and_falls_back(self) -> None: - server, _rpc = light_coordinator() - server.accepted_parent_redrive_defer_threshold = 1 - server.accepted_parent_redrive_attempt_max = 2 - service = server._ensure_block_candidate_service() - server._begin_accepted_block_payout_preview(ANCESTOR_HASH, block_height=9) - server._mark_accepted_block_payout_landed(ANCESTOR_HASH, block_height=9) - - def child_defers() -> bool: - return server._defer_for_pending_parent_payout_transition( - block_hash=CHILD_HASH, - parent_hash=ANCESTOR_HASH, - parent_height=9, - worker=None, - ) - - with patch("builtins.print"): - for expected_attempts in (1, 2): - self.assertTrue(child_defers()) - self.assertEqual( - int(server.accepted_parent_redrive_attempt_count), - expected_attempts, - ) - self.assertTrue(service._ancestor_redrive_owed()) - self.assertEqual( - service._consume_ancestor_redrive_requests(), - (ANCESTOR_HASH,), - ) - # The cap is exhausted: the wedge keeps deferring exactly as it - # did before the fix, no further pass is armed, and exhaustion - # is counted once for the ancestor. - for _cycle in range(3): - self.assertTrue(child_defers()) - self.assertFalse(service._ancestor_redrive_owed()) - self.assertEqual(int(server.accepted_parent_redrive_attempt_count), 2) - self.assertEqual(int(server.accepted_parent_redrive_exhausted_count), 1) - server._clear_accepted_block_payout_preview(ANCESTOR_HASH) - - def test_armed_request_does_not_double_arm_before_consumption(self) -> None: - server, _rpc = light_coordinator() - server.accepted_parent_redrive_defer_threshold = 1 - server.accepted_parent_redrive_attempt_max = 5 - service = server._ensure_block_candidate_service() - server._begin_accepted_block_payout_preview(ANCESTOR_HASH, block_height=9) - server._mark_accepted_block_payout_landed(ANCESTOR_HASH, block_height=9) - - with patch("builtins.print"): - for _cycle in range(3): - self.assertTrue( - server._defer_for_pending_parent_payout_transition( - block_hash=CHILD_HASH, - parent_hash=ANCESTOR_HASH, - parent_height=9, - worker=None, - ) - ) - # Deferrals while a pass is already armed count toward nothing new. - self.assertEqual(int(server.accepted_parent_redrive_attempt_count), 1) - self.assertEqual( - service._consume_ancestor_redrive_requests(), (ANCESTOR_HASH,) - ) - server._clear_accepted_block_payout_preview(ANCESTOR_HASH) - - -class AncestorRedriveSweepTests(unittest.TestCase): - """The no-pending-row wedge shape: converge with durable state, or stand - down for the watchdog.""" - - def _defer_once(self, server) -> None: - with patch("builtins.print"): - self.assertTrue( - server._defer_for_pending_parent_payout_transition( - block_hash=CHILD_HASH, - parent_hash=ANCESTOR_HASH, - parent_height=10, - worker=None, - ) - ) - - def _armed_sweep_coordinator(self, block_state, *, chain_probe=True): - server, _state, _recording = submit_coordinator() - ledger = SweepLedger(block_state) - server.ledger = ledger - server.accepted_parent_redrive_defer_threshold = 1 - probe_calls = _script_chain_probe(server, chain_probe) - server._begin_accepted_block_payout_preview( - ANCESTOR_HASH, block_height=10 - ) - server._mark_accepted_block_payout_landed(ANCESTOR_HASH, block_height=10) - return server, ledger, probe_calls - - def test_confirmed_stale_transition_is_cleared(self) -> None: - server, ledger, probe_calls = self._armed_sweep_coordinator( - { - "block_hash": ANCESTOR_HASH, - "block_height": 10, - "parent_hash": PARENT_TIP, - "chain_state": "confirmed", - "maturity_state": "immature", - } - ) - self._defer_once(server) - with patch("builtins.print"): - self.assertEqual(server.replay_pending_block_candidates(), 0) - self.assertEqual(ledger.pool_block_state_calls, [ANCESTOR_HASH]) - # The clear demanded the node-side proof at the transition's height. - self.assertEqual(probe_calls, [(ANCESTOR_HASH, 10)]) - with server._accepted_block_payout_preview_condition: - self.assertNotIn( - ANCESTOR_HASH, server._accepted_block_payout_previews - ) - self.assertNotIn( - ANCESTOR_HASH, - server._invalidated_accepted_block_payout_previews, - ) - with patch("builtins.print"): - self.assertFalse( - server._defer_for_pending_parent_payout_transition( - block_hash=CHILD_HASH, - parent_hash=ANCESTOR_HASH, - parent_height=10, - worker=None, - ) - ) - self.assertEqual(int(server.accepted_parent_redrive_resolved_count), 1) - - def test_unconfirmed_transition_is_left_for_the_watchdog(self) -> None: - server, ledger, _probe_calls = self._armed_sweep_coordinator(None) - self._defer_once(server) - with patch("builtins.print"): - self.assertEqual(server.replay_pending_block_candidates(), 0) - self.assertEqual(ledger.pool_block_state_calls, [ANCESTOR_HASH]) - with server._accepted_block_payout_preview_condition: - self.assertIn(ANCESTOR_HASH, server._accepted_block_payout_previews) - self._defer_once(server) - self.assertEqual(int(server.accepted_parent_redrive_resolved_count), 0) - - def test_reversed_pool_block_transition_is_left_alone(self) -> None: - server, ledger, _probe_calls = self._armed_sweep_coordinator( - { - "block_hash": ANCESTOR_HASH, - "block_height": 10, - "parent_hash": PARENT_TIP, - "chain_state": "confirmed", - "maturity_state": "reversed", - } - ) - self._defer_once(server) - with patch("builtins.print"): - self.assertEqual(server.replay_pending_block_candidates(), 0) - with server._accepted_block_payout_preview_condition: - self.assertIn(ANCESTOR_HASH, server._accepted_block_payout_previews) - - def test_transition_owned_by_an_in_process_lane_is_left_alone(self) -> None: - server, ledger, _probe_calls = self._armed_sweep_coordinator( - { - "block_hash": ANCESTOR_HASH, - "block_height": 10, - "parent_hash": PARENT_TIP, - "chain_state": "confirmed", - "maturity_state": "immature", - } - ) - service = server._ensure_block_candidate_service() - service._ensure_block_replay_state() - # Armed first, then adopted: deferrals against an already-inflight - # ancestor bank nothing (covered in the bookkeeping suite), so the - # adopted short-circuit is reached by arming before adoption. - self._defer_once(server) - with server.lock: - server._block_replay_inflight_hashes.add(ANCESTOR_HASH) - with patch("builtins.print"): - self.assertEqual(server.replay_pending_block_candidates(), 0) - # An inflight hash means ordinary replay finalization still owns the - # resolution; the sweep never consults the ledger for it. - self.assertEqual(ledger.pool_block_state_calls, []) - with server._accepted_block_payout_preview_condition: - self.assertIn(ANCESTOR_HASH, server._accepted_block_payout_previews) - - def test_unprovable_chain_activity_is_left_for_the_watchdog(self) -> None: - """A durably confirmed row alone is not proof against a fresh reorg. - - Reconciliation fails closed while a landed transition exists, so a - block qbitd has already reorged out can still read confirmed in the - durable row. The sweep must demand the node-side probe the landing - demands and stand down on anything short of a definite True. - """ - for probe_verdict in (None, False): - with self.subTest(probe_verdict=probe_verdict): - server, ledger, probe_calls = self._armed_sweep_coordinator( - { - "block_hash": ANCESTOR_HASH, - "block_height": 10, - "parent_hash": PARENT_TIP, - "chain_state": "confirmed", - "maturity_state": "immature", - }, - chain_probe=probe_verdict, - ) - self._defer_once(server) - with patch("builtins.print"): - self.assertEqual( - server.replay_pending_block_candidates(), 0 - ) - self.assertEqual( - ledger.pool_block_state_calls, [ANCESTOR_HASH] - ) - self.assertEqual(probe_calls, [(ANCESTOR_HASH, 10)]) - with server._accepted_block_payout_preview_condition: - self.assertIn( - ANCESTOR_HASH, server._accepted_block_payout_previews - ) - self.assertEqual( - int(server.accepted_parent_redrive_resolved_count), 0 - ) - - def test_retry_held_ancestor_is_left_to_its_owner(self) -> None: - """The in-process-ownership guard itself, reached with nothing adopted. - - The ancestor's own candidate sits in the retry holder while its - outbox row is already terminal, so the forced pass enumerates - nothing, the adopted short-circuit cannot fire, and the sweep runs. - The guard must see the held copy and stand down before the durable - read or the chain probe -- a guard that wrongly answered "not owned" - would clear the transition out from under the live lane and fails - every assertion below. - """ - server, state, _recording = submit_coordinator() - ledger = SweepLedger( - { - "block_hash": ANCESTOR_HASH, - "block_height": 10, - "parent_hash": PARENT_TIP, - "chain_state": "confirmed", - "maturity_state": "immature", - } - ) - server.ledger = ledger - server.accepted_parent_redrive_defer_threshold = 1 - probe_calls = _script_chain_probe(server, True) - ancestor = block_candidate( - server, - state, - SimpleNamespace( - coinbase_tx_hex="00", - block_hash_hex=ANCESTOR_HASH, - block_hex="00", - share_pass=False, - block_pass=True, - ), - pending_share=_pending_share(ANCESTOR_HASH, 1), - ) - server._begin_accepted_block_payout_preview( - ANCESTOR_HASH, block_height=10 - ) - server._mark_accepted_block_payout_landed(ANCESTOR_HASH, block_height=10) - server._retain_block_candidate_for_retry(ancestor) - self._defer_once(server) - with patch("builtins.print"): - self.assertEqual(server.replay_pending_block_candidates(), 0) - with server.lock: - self.assertNotIn( - ANCESTOR_HASH, server._block_replay_inflight_hashes - ) - self.assertEqual(ledger.pool_block_state_calls, []) - self.assertEqual(probe_calls, []) - with server._accepted_block_payout_preview_condition: - self.assertIn(ANCESTOR_HASH, server._accepted_block_payout_previews) - - -class AncestorRedriveBookkeepingTests(unittest.TestCase): - """The per-ancestor record's cap, streak, eviction, and counter rules.""" - - def _tracker(self, *, threshold: int, cap: int): - server, _rpc = light_coordinator() - server.accepted_parent_redrive_defer_threshold = threshold - server.accepted_parent_redrive_attempt_max = cap - return server, server._ensure_block_candidate_service() - - def test_each_attempt_needs_a_fresh_deferral_streak(self) -> None: - """Deferrals behind an armed request must not bank toward the next. - - Otherwise attempts 2..N arm on single deferrals right after - consumption and the cap exhausts before the first pass's adoption - had any chance to resolve the ancestor. - """ - server, service = self._tracker(threshold=3, cap=3) - with patch("builtins.print"): - for _cycle in range(3): - service.note_pending_parent_transition_deferral( - CHILD_HASH, ANCESTOR_HASH - ) - self.assertEqual(int(server.accepted_parent_redrive_attempt_count), 1) - # Deferrals while armed bank nothing. - for _cycle in range(4): - service.note_pending_parent_transition_deferral( - CHILD_HASH, ANCESTOR_HASH - ) - self.assertEqual( - service._consume_ancestor_redrive_requests(), (ANCESTOR_HASH,) - ) - # A fresh threshold of deferrals is required for attempt 2. - for _cycle in range(2): - service.note_pending_parent_transition_deferral( - CHILD_HASH, ANCESTOR_HASH - ) - self.assertEqual( - int(server.accepted_parent_redrive_attempt_count), 1 - ) - service.note_pending_parent_transition_deferral( - CHILD_HASH, ANCESTOR_HASH - ) - self.assertEqual(int(server.accepted_parent_redrive_attempt_count), 2) - - def test_zero_attempt_cap_disables_the_mechanism_silently(self) -> None: - """Cap 0 means off: no arming, no exhaustion signal, no bookkeeping. - - The exhausted counter is the alert-worthy pre-restart signal; an - installation that disabled the feature must not emit it. - """ - server, service = self._tracker(threshold=1, cap=0) - with patch("builtins.print"): - for _cycle in range(5): - service.note_pending_parent_transition_deferral( - CHILD_HASH, ANCESTOR_HASH - ) - self.assertFalse(service._ancestor_redrive_owed()) - self.assertEqual(int(server.accepted_parent_redrive_attempt_count), 0) - self.assertEqual(int(server.accepted_parent_redrive_exhausted_count), 0) - self.assertEqual(service._ancestor_redrive_records, {}) - self.assertEqual(service._ancestor_redrive_last_blocking, {}) - - def test_resolution_without_a_consumed_pass_is_not_counted(self) -> None: - """An armed-but-unconsumed request that resolves on its own is the - mechanism standing by, not succeeding; the resolved counter must not - inflate against attempts.""" - server, service = self._tracker(threshold=1, cap=3) - with patch("builtins.print"): - service.note_pending_parent_transition_deferral( - CHILD_HASH, ANCESTOR_HASH - ) - self.assertTrue(service._ancestor_redrive_owed()) - service.note_pending_parent_transition_resolved(CHILD_HASH) - self.assertEqual(int(server.accepted_parent_redrive_resolved_count), 0) - self.assertEqual(service._ancestor_redrive_records, {}) - self.assertEqual(service._ancestor_redrive_last_blocking, {}) - - def test_deferrals_while_the_ancestor_is_inflight_bank_nothing(self) -> None: - """An adopted ancestor is mid-resolution; a new pass would no-op. - - Burning armed attempts against an adoption already in progress - would exhaust the cap before slow-but-working replay finalization - completes; the streak must resume only once the flight ends. - """ - server, service = self._tracker(threshold=1, cap=3) - service._ensure_block_replay_state() - with server.lock: - server._block_replay_inflight_hashes.add(ANCESTOR_HASH) - with patch("builtins.print"): - for _cycle in range(3): - service.note_pending_parent_transition_deferral( - CHILD_HASH, ANCESTOR_HASH - ) - self.assertFalse(service._ancestor_redrive_owed()) - self.assertEqual(int(server.accepted_parent_redrive_attempt_count), 0) - with server.lock: - server._block_replay_inflight_hashes.discard(ANCESTOR_HASH) - with patch("builtins.print"): - service.note_pending_parent_transition_deferral( - CHILD_HASH, ANCESTOR_HASH - ) - self.assertTrue(service._ancestor_redrive_owed()) - self.assertEqual(int(server.accepted_parent_redrive_attempt_count), 1) - - def test_child_entry_eviction_never_cancels_an_ancestor_record(self) -> None: - """The two registries are different hash spaces; trimming one must - not reach into the other. - - Hash B is both a child (B deferred on A once) and an armed ancestor - (C's deferrals armed B's re-drive). Flooding the child-keyed index - past its bound evicts B's stale child entry; B's armed record must - survive that eviction untouched. - """ - server, service = self._tracker(threshold=3, cap=3) - ancestor_b = "bb" * 32 - with patch("builtins.print"): - service.note_pending_parent_transition_deferral( - ancestor_b, ANCESTOR_HASH - ) - for _cycle in range(3): - service.note_pending_parent_transition_deferral( - CHILD_HASH, ancestor_b - ) - self.assertEqual(int(server.accepted_parent_redrive_attempt_count), 1) - for index in range(70): - service.note_pending_parent_transition_deferral( - "%064x" % (0x1000 + index), ancestor_b - ) - with server.lock: - self.assertNotIn(ancestor_b, service._ancestor_redrive_last_blocking) - self.assertEqual( - service._consume_ancestor_redrive_requests(), (ancestor_b,) - ) - self.assertEqual(int(server.accepted_parent_redrive_attempt_count), 1) - - def test_eviction_never_drops_the_actively_wedged_ancestor(self) -> None: - """The lifetime cap must survive the pathological many-ancestor - stream the eviction bound exists for. - - An exhausted ancestor that keeps deferring re-inserts its whole - record every deferral, so a storm of distinct arming ancestors must - never evict it -- eviction would discard the exhaustion latch and - re-arm re-drives past the cap. - """ - server, service = self._tracker(threshold=1, cap=1) - with patch("builtins.print"): - service.note_pending_parent_transition_deferral( - CHILD_HASH, ANCESTOR_HASH - ) - self.assertEqual( - service._consume_ancestor_redrive_requests(), (ANCESTOR_HASH,) - ) - service.note_pending_parent_transition_deferral( - CHILD_HASH, ANCESTOR_HASH - ) - self.assertEqual( - int(server.accepted_parent_redrive_exhausted_count), 1 - ) - for index in range(70): - service.note_pending_parent_transition_deferral( - "%064x" % (0x2000 + index), - "%064x" % (0x3000 + index), - ) - # The wedge keeps deferring throughout the storm. - service.note_pending_parent_transition_deferral( - CHILD_HASH, ANCESTOR_HASH - ) - attempts_after_storm = int(server.accepted_parent_redrive_attempt_count) - service.note_pending_parent_transition_deferral( - CHILD_HASH, ANCESTOR_HASH - ) - self.assertEqual( - int(server.accepted_parent_redrive_attempt_count), - attempts_after_storm, - ) - self.assertEqual(int(server.accepted_parent_redrive_exhausted_count), 1) - with server.lock: - record = service._ancestor_redrive_records.get(ANCESTOR_HASH) - self.assertIsNotNone(record) - self.assertTrue(record.exhausted) - - -class AncestorRedriveRunLoopTests(unittest.TestCase): - """The submitter loop's fall-through: an armed re-drive must reach the - replay entrypoint even when immediate live work keeps succeeding -- - sustained live traffic taking the ``continue`` every pass is exactly how - the incident starved the one path that resolves a stuck ancestor.""" - - def _run_one_iteration(self, *, arm_redrive: bool): - server, _rpc = light_coordinator() - server.accepted_parent_redrive_defer_threshold = 1 - server.accepted_parent_redrive_attempt_max = 3 - service = server._ensure_block_candidate_service() - if arm_redrive: - with patch("builtins.print"): - service.note_pending_parent_transition_deferral( - CHILD_HASH, ANCESTOR_HASH - ) - self.assertTrue(service._ancestor_redrive_owed()) - server._start_block_accounting_thread = lambda: None - submits: list[float | None] = [] - - def submit_next(timeout=None, *, defer_accounting=False): - submits.append(timeout) - return True - - server.submit_next_block_candidate = submit_next - replays: list[bool] = [] - stop_states = iter([False, True]) - stop = SimpleNamespace(is_set=lambda: next(stop_states)) - service.ports = dataclasses.replace( - service.ports, - stop_event=lambda: stop, - replay_entrypoint=lambda: (replays.append(True), 0)[1], - ) - # Immediate work is available and keeps succeeding. - service.retry_candidate = object() - with patch("builtins.print"): - service.run() - return replays, submits - - def test_armed_redrive_falls_through_to_the_replay_entrypoint(self) -> None: - replays, submits = self._run_one_iteration(arm_redrive=True) - self.assertEqual(replays, [True]) - self.assertEqual(submits, [None, 1.0]) - - def test_without_a_redrive_immediate_success_short_circuits(self) -> None: - replays, submits = self._run_one_iteration(arm_redrive=False) - self.assertEqual(replays, []) - self.assertEqual(submits, [None]) - - -class AncestorRedriveForcedPassTests(unittest.TestCase): - """A forced re-drive pass runs the startup enumeration's code: keyset - pagination when the ledger supports it, on the landing-class budget.""" - - def test_forced_pass_paginates_on_the_landing_budget(self) -> None: - server, _state, _recording = submit_coordinator() - ledger = SweepLedger(None) - server.ledger = ledger - server.accepted_parent_redrive_defer_threshold = 1 - _script_chain_probe(server, True) - server._begin_accepted_block_payout_preview( - ANCESTOR_HASH, block_height=10 - ) - server._mark_accepted_block_payout_landed(ANCESTOR_HASH, block_height=10) - original_call = server._run_block_submitter_ledger_call - outbox_queries: list[tuple[object, object, object]] = [] - - def spying_call(key, label, fn, **kwargs): - if label == "replay-outbox-query": - outbox_queries.append( - (key, kwargs.get("call_class"), kwargs.get("timeout_seconds")) - ) - return original_call(key, label, fn, **kwargs) - - server._run_block_submitter_ledger_call = spying_call - with patch("builtins.print"): - self.assertTrue( - server._defer_for_pending_parent_payout_transition( - block_hash=CHILD_HASH, - parent_hash=ANCESTOR_HASH, - parent_height=10, - worker=None, - ) - ) - self.assertEqual(server.replay_pending_block_candidates(), 0) - self.assertEqual(len(outbox_queries), 1) - key, call_class, timeout_seconds = outbox_queries[0] - # The paginated fetch keys its dedupe on (label, limit, page, - # cursor); the legacy widening window keys on (label, limit) alone. - self.assertEqual(len(key), 4) - self.assertEqual(call_class, "landing") - self.assertIsNotNone(timeout_seconds) - - -if __name__ == "__main__": - unittest.main() diff --git a/tests/test_prism_audit_api.py b/tests/test_prism_audit_api.py deleted file mode 100644 index c8b24ca01..000000000 --- a/tests/test_prism_audit_api.py +++ /dev/null @@ -1,478 +0,0 @@ -#!/usr/bin/env python3 - -from __future__ import annotations - -import json -import tempfile -import threading -import unittest -import urllib.error -import urllib.request -from http.server import ThreadingHTTPServer -from pathlib import Path - -from lab.prism.prism_coordinator import PrismCoordinator, make_audit_handler - - -class FakeLedger: - backend_name = "fake-ledger" - block_hash = "a" * 64 - fanout_txid = "b" * 64 - audit_commitment_leaf_hex = "d" * 64 - - def audit_share_window(self, *, anchor_job_issued_at_ms: int, network_difficulty: int) -> list[dict[str, object]]: - return [ - { - "share_id": "share-2", - "counted_difficulty": network_difficulty, - "job_issued_at_ms": anchor_job_issued_at_ms, - } - ] - - def audit_block_payouts(self, *, block_hash: str) -> list[dict[str, object]]: - if block_hash == "a" * 64: - return [{"block_hash": block_hash, "chain_state": "inactive", "miner_id": "miner-a"}] - return [] - - def audit_bundle(self, *, block_hash: str) -> dict[str, object] | None: - if block_hash == self.block_hash: - return {"block_hash": block_hash, "audit_bundle": {"schema": "qbit.prism.audit-bundle.v1"}} - return None - - def audit_bundle_by_commitment(self, *, commitment_leaf_hex: str) -> dict[str, object] | None: - if commitment_leaf_hex == self.audit_commitment_leaf_hex: - return { - "block_hash": self.block_hash, - "audit_commitment_leaf_hex": commitment_leaf_hex, - "audit_bundle": { - "schema": "qbit.prism.audit-bundle.v1", - "audit_commitment_leaves_hex": [commitment_leaf_hex], - }, - } - return None - - def audit_ctv_fanouts(self, *, block_hash: str) -> list[dict[str, object]]: - if block_hash == self.block_hash: - return [ - { - "block_hash": block_hash, - "fanout_txid": self.fanout_txid, - "fanout_tx_hex": "03", - "anchor_vout": 2, - "settlement_status": "awaiting_maturity", - } - ] - return [] - - def audit_ctv_fanout_manifest_set(self, *, block_hash: str) -> dict[str, object] | None: - if block_hash == self.block_hash: - return { - "schema": "qbit.prism.ctv-fanout-recovery.v1", - "block_hash": block_hash, - "manifest_set_sha256": "c" * 64, - "manifest_set_json": "{}", - "manifest_set": {"schema": "qbit.prism.ctv-fanout-manifest-set.v1"}, - } - return None - - def ctv_fanout_status(self, *, fanout_txid: str) -> dict[str, object] | None: - if fanout_txid == self.fanout_txid: - return { - "schema": "qbit.prism.ctv-fanout-status.v1", - "fanout_txid": fanout_txid, - "settlement_status": "awaiting_maturity", - "broadcast_attempts": [], - } - return None - - def pending_ctv_fanout_statuses(self, *, limit: int = 100) -> list[dict[str, object]]: - return [ - { - "schema": "qbit.prism.ctv-fanout-status.v1", - "fanout_txid": self.fanout_txid, - "block_hash": self.block_hash, - "block_height": 100, - "chunk_index": 0, - "chunk_count": 1, - "settlement_status": "broadcastable", - } - ][:limit] - - def current_owed_balances(self) -> list[dict[str, object]]: - return [ - {"recipient_id": "miner-a", "order_key": "a", "p2mr_program_hex": "11" * 32, "balance_sats": 42}, - {"recipient_id": "miner-a", "order_key": "b", "p2mr_program_hex": "22" * 32, "balance_sats": 8}, - {"recipient_id": "miner-b", "order_key": "c", "p2mr_program_hex": "33" * 32, "balance_sats": 100}, - ] - - def recipient_payout_history(self, *, recipient_id: str, limit: int = 50) -> list[dict[str, object]]: - rows = [ - { - "block_hash": self.block_hash, - "block_height": 100, - "coinbase_txid": "e" * 64, - "payout_manifest_sha256": "f" * 64, - "recipient_id": "miner-a", - "order_key": "a", - "p2mr_program_hex": "11" * 32, - "onchain_amount_sats": 25, - "carry_forward_balance_sats": 17, - "action": "onchain", - "maturity_state": "immature", - }, - { - "block_hash": "9" * 64, - "block_height": 99, - "coinbase_txid": "8" * 64, - "payout_manifest_sha256": "7" * 64, - "recipient_id": "miner-b", - "order_key": "c", - "p2mr_program_hex": "33" * 32, - "onchain_amount_sats": 100, - "carry_forward_balance_sats": 0, - "action": "onchain", - "maturity_state": "mature", - }, - ] - return [row for row in rows if row["recipient_id"] == recipient_id][:limit] - - def carry_forward_integrity_report(self) -> dict[str, object]: - return { - "schema": "qbit.prism.carry-forward-integrity.v1", - "backend": "fake-ledger", - "checked_active_rows": 2, - "audit_chain_version": "qbit.prism.carry-forward-active-delta-chain.v1", - "audit_row_count": 2, - "audit_head_sha256": "ab" * 32, - "mismatch_count": 1, - "mismatches": [ - { - "carry_forward_seq": 2, - "block_hash": self.block_hash, - "block_height": 100, - "recipient_id": "miner-a", - "order_key": "a", - "p2mr_program_hex": "11" * 32, - "prior_balance_sats": 0, - "expected_prior_balance_sats": 17, - "mismatch_reason": "prior_balance", - } - ], - } - - def metrics(self) -> dict[str, int]: - return {"blocks": 1, "owed_accounts": 1} - - def all_shares(self) -> list[object]: - return [object(), object()] - - -class FakeCoordinator: - def __init__(self) -> None: - self.ledger = FakeLedger() - self.accepted_block_count = 1 - - def health_payload(self) -> dict[str, object]: - return {"ok": True, "ledger_backend": self.ledger.backend_name} - - def latest_evidence_payload(self) -> dict[str, object]: - return {"schema": "qbit.prism.live-stratum-evidence.v1", "block_hash": "a" * 64} - - def owed_balances_payload(self) -> dict[str, object]: - return { - "schema": "qbit.prism.owed-balances.v1", - "balances": self.ledger.current_owed_balances(), - } - - def carry_forward_integrity_payload(self) -> dict[str, object]: - report = self.ledger.carry_forward_integrity_report() - report["ledger_backend"] = self.ledger.backend_name - return report - - def miner_status_payload(self, recipient_id: str) -> dict[str, object]: - balances = [ - balance - for balance in self.ledger.current_owed_balances() - if str(balance.get("recipient_id")) == recipient_id - ] - return { - "schema": "qbit.prism.miner-status.v1", - "recipient_id": recipient_id, - "owed_balance_sats": sum(int(balance["balance_sats"]) for balance in balances), - "owed_balances": balances, - "recent_payouts": self.ledger.recipient_payout_history(recipient_id=recipient_id), - } - - def metrics_payload(self) -> str: - return ( - "qbit_prism_accepted_shares_total 2\n" - "qbit_prism_blocks_accepted_total 1\n" - "qbit_prism_ctv_fanouts_pending 1\n" - "qbit_prism_ctv_fanouts_broadcastable 1\n" - "qbit_prism_ctv_fanouts_failed 0\n" - ) - - -class UnhealthyCoordinator(FakeCoordinator): - def cached_health_payload(self) -> tuple[int, dict[str, object]]: - return 503, { - "ok": False, - "schema": "qbit.prism.audit-health.v1", - "reason": "refresh_pending_too_long", - "reasons": [ - "refresh_pending_too_long", - "current_generation_not_delivered", - ], - "pending_refresh_age_seconds": 16.0, - } - - -class CachedMetricsCoordinator(FakeCoordinator): - def cached_metrics_payload(self) -> tuple[int, str]: - return 200, "qbit_prism_cached_snapshot 1\n" - - def metrics_payload(self) -> str: - raise AssertionError("HTTP scrape reached the live metrics renderer") - - -class UninitializedCachedMetricsCoordinator(FakeCoordinator): - def cached_metrics_payload(self) -> tuple[int, str]: - return 503, "qbit_prism_metrics_snapshot_age_seconds -1\n" - - def metrics_payload(self) -> str: - return "qbit_prism_direct_compatibility 1\n" - - -class PrismAuditApiTests(unittest.TestCase): - def setUp(self) -> None: - handler = make_audit_handler(FakeCoordinator()) # type: ignore[arg-type] - self.server = ThreadingHTTPServer(("127.0.0.1", 0), handler) - self.thread = threading.Thread(target=self.server.serve_forever, daemon=True) - self.thread.start() - self.base_url = f"http://127.0.0.1:{self.server.server_port}" - - def tearDown(self) -> None: - self.server.shutdown() - self.server.server_close() - self.thread.join(timeout=5) - - def get_json(self, path: str) -> dict[str, object]: - with urllib.request.urlopen(self.base_url + path, timeout=5) as response: - return json.loads(response.read()) - - def get_text(self, path: str) -> str: - with urllib.request.urlopen(self.base_url + path, timeout=5) as response: - return response.read().decode() - - def test_health_latest_owed_and_metrics_endpoints(self) -> None: - self.assertTrue(self.get_json("/healthz")["ok"]) - self.assertEqual(self.get_json("/audit/latest")["schema"], "qbit.prism.live-stratum-evidence.v1") - self.assertEqual(len(self.get_json("/owed-balances")["balances"]), 3) - miner_status = self.get_json("/miners/miner-a/status") - payout_status = self.get_json("/payouts/miner-b/status") - missing_status = self.get_json("/miners/unknown/status") - self.assertEqual(miner_status["owed_balance_sats"], 50) - self.assertEqual(len(miner_status["owed_balances"]), 2) - self.assertEqual(len(miner_status["recent_payouts"]), 1) - self.assertEqual(miner_status["recent_payouts"][0]["action"], "onchain") - self.assertEqual(miner_status["recent_payouts"][0]["maturity_state"], "immature") - self.assertEqual(payout_status["owed_balance_sats"], 100) - self.assertEqual(payout_status["recent_payouts"][0]["block_height"], 99) - self.assertEqual(missing_status["owed_balance_sats"], 0) - self.assertEqual(missing_status["owed_balances"], []) - self.assertEqual(missing_status["recent_payouts"], []) - - metrics = self.get_text("/metrics") - - self.assertIn("qbit_prism_accepted_shares_total 2", metrics) - self.assertIn("qbit_prism_blocks_accepted_total 1", metrics) - self.assertIn("qbit_prism_ctv_fanouts_pending 1", metrics) - self.assertIn("qbit_prism_ctv_fanouts_broadcastable 1", metrics) - self.assertIn("qbit_prism_ctv_fanouts_failed 0", metrics) - - def test_metrics_endpoint_reads_only_the_cached_complete_snapshot(self) -> None: - handler = make_audit_handler(CachedMetricsCoordinator()) # type: ignore[arg-type] - server = ThreadingHTTPServer(("127.0.0.1", 0), handler) - thread = threading.Thread(target=server.serve_forever, daemon=True) - thread.start() - try: - with urllib.request.urlopen( - f"http://127.0.0.1:{server.server_port}/metrics", - timeout=5, - ) as response: - self.assertEqual(response.status, 200) - self.assertEqual( - response.read().decode(), - "qbit_prism_cached_snapshot 1\n", - ) - finally: - server.shutdown() - server.server_close() - thread.join(timeout=5) - - def test_metrics_endpoint_falls_back_when_compatibility_cache_is_uninitialized( - self, - ) -> None: - handler = make_audit_handler( # type: ignore[arg-type] - UninitializedCachedMetricsCoordinator() - ) - server = ThreadingHTTPServer(("127.0.0.1", 0), handler) - thread = threading.Thread(target=server.serve_forever, daemon=True) - thread.start() - try: - with urllib.request.urlopen( - f"http://127.0.0.1:{server.server_port}/metrics", - timeout=5, - ) as response: - self.assertEqual(response.status, 200) - self.assertEqual( - response.read().decode(), - "qbit_prism_direct_compatibility 1\n", - ) - finally: - server.shutdown() - server.server_close() - thread.join(timeout=5) - - def test_coordinator_latest_audit_endpoint_preserves_none_and_snapshot_payload(self) -> None: - coordinator = PrismCoordinator.__new__(PrismCoordinator) - coordinator.latest_evidence = None - with tempfile.TemporaryDirectory() as audit_root: - coordinator.audit_dir = Path(audit_root) / "artifacts" - coordinator.evidence_path = Path(audit_root) / "evidence.json" - handler = make_audit_handler(coordinator) - server = ThreadingHTTPServer(("127.0.0.1", 0), handler) - thread = threading.Thread(target=server.serve_forever, daemon=True) - thread.start() - base_url = f"http://127.0.0.1:{server.server_port}" - try: - with self.assertRaises(urllib.error.HTTPError) as raised: - urllib.request.urlopen(base_url + "/audit/latest", timeout=5) - self.assertEqual(raised.exception.code, 404) - raised.exception.close() - - seed = {"schema": "test", "nested": {"value": 1}} - coordinator.latest_evidence = seed - seed["nested"]["value"] = 2 - with urllib.request.urlopen( - base_url + "/audit/latest", - timeout=5, - ) as response: - payload = json.loads(response.read()) - self.assertEqual(payload, {"schema": "test", "nested": {"value": 1}}) - payload["nested"]["value"] = 3 - self.assertEqual( - coordinator.latest_evidence_payload(), - {"schema": "test", "nested": {"value": 1}}, - ) - finally: - server.shutdown() - server.server_close() - thread.join(timeout=5) - store = coordinator.__dict__.get("_audit_artifact_store") - if store is not None: - store.close() - - def test_unhealthy_progress_returns_http_503_and_bounded_reason(self) -> None: - handler = make_audit_handler(UnhealthyCoordinator()) # type: ignore[arg-type] - server = ThreadingHTTPServer(("127.0.0.1", 0), handler) - thread = threading.Thread(target=server.serve_forever, daemon=True) - thread.start() - try: - with self.assertRaises(urllib.error.HTTPError) as raised: - urllib.request.urlopen( - f"http://127.0.0.1:{server.server_port}/healthz", - timeout=5, - ) - self.assertEqual(raised.exception.code, 503) - payload = json.loads(raised.exception.read()) - raised.exception.close() - self.assertFalse(payload["ok"]) - self.assertEqual(payload["reason"], "refresh_pending_too_long") - self.assertNotIn("username", payload) - finally: - server.shutdown() - server.server_close() - thread.join(timeout=5) - - def test_audit_endpoints_serialize_rows_and_bundle(self) -> None: - block_hash = "a" * 64 - commitment_leaf = "d" * 64 - - window = self.get_json("/audit/share-window?anchor_job_issued_at_ms=1234&network_difficulty=99") - payouts = self.get_json(f"/audit/blocks/{block_hash}/payouts") - bundle = self.get_json(f"/audit/blocks/{block_hash}/bundle") - commitment_bundle = self.get_json(f"/audit/commitments/{commitment_leaf}/bundle") - - self.assertEqual(window["rows"], [{"share_id": "share-2", "counted_difficulty": 99, "job_issued_at_ms": 1234}]) - self.assertEqual( - payouts["rows"], - [{"block_hash": block_hash, "chain_state": "inactive", "miner_id": "miner-a"}], - ) - self.assertEqual(bundle["audit_bundle"], {"schema": "qbit.prism.audit-bundle.v1"}) - self.assertEqual(commitment_bundle["block_hash"], block_hash) - self.assertEqual(commitment_bundle["audit_commitment_leaf_hex"], commitment_leaf) - self.assertEqual(commitment_bundle["audit_bundle"]["audit_commitment_leaves_hex"], [commitment_leaf]) - - def test_carry_forward_integrity_endpoint(self) -> None: - report = self.get_json("/audit/carry-forward-integrity") - alias = self.get_json("/audit/ledger-integrity") - - self.assertEqual(report["schema"], "qbit.prism.carry-forward-integrity.v1") - self.assertEqual(report["ledger_backend"], "fake-ledger") - self.assertEqual(report["checked_active_rows"], 2) - self.assertEqual(report["audit_chain_version"], "qbit.prism.carry-forward-active-delta-chain.v1") - self.assertEqual(report["audit_row_count"], 2) - self.assertEqual(report["audit_head_sha256"], "ab" * 32) - self.assertEqual(report["mismatch_count"], 1) - self.assertEqual(report["mismatches"][0]["mismatch_reason"], "prior_balance") - self.assertEqual(alias, report) - - def test_ctv_fanout_recovery_and_status_endpoints(self) -> None: - block_hash = "a" * 64 - fanout_txid = "b" * 64 - - fanouts = self.get_json(f"/audit/blocks/{block_hash}/ctv-fanouts") - manifest_set = self.get_json(f"/audit/blocks/{block_hash}/ctv-fanout-manifest-set") - status = self.get_json(f"/audit/fanouts/{fanout_txid}/status") - pending = self.get_json("/audit/fanouts/pending?limit=1") - - self.assertEqual(fanouts["schema"], "qbit.prism.audit-ctv-fanouts.v1") - self.assertEqual(fanouts["rows"][0]["fanout_txid"], fanout_txid) - self.assertEqual(manifest_set["manifest_set_sha256"], "c" * 64) - self.assertEqual(status["settlement_status"], "awaiting_maturity") - self.assertEqual(pending["schema"], "qbit.prism.pending-ctv-fanouts.v1") - self.assertEqual(pending["count"], 1) - self.assertEqual(pending["rows"][0]["settlement_status"], "broadcastable") - - def test_bad_block_hash_returns_400(self) -> None: - with self.assertRaises(urllib.error.HTTPError) as raised: - self.get_json("/audit/blocks/not-a-hash/payouts") - - self.assertEqual(raised.exception.code, 400) - raised.exception.close() - - def test_bad_fanout_txid_returns_400(self) -> None: - with self.assertRaises(urllib.error.HTTPError) as raised: - self.get_json("/audit/fanouts/not-a-txid/status") - - self.assertEqual(raised.exception.code, 400) - raised.exception.close() - - def test_bad_commitment_hash_returns_400(self) -> None: - with self.assertRaises(urllib.error.HTTPError) as raised: - self.get_json("/audit/commitments/not-a-hash/bundle") - - self.assertEqual(raised.exception.code, 400) - raised.exception.close() - - def test_unknown_commitment_hash_returns_404(self) -> None: - with self.assertRaises(urllib.error.HTTPError) as raised: - self.get_json(f"/audit/commitments/{'e' * 64}/bundle") - - self.assertEqual(raised.exception.code, 404) - raised.exception.close() - - -if __name__ == "__main__": - unittest.main() diff --git a/tests/test_prism_audit_artifacts.py b/tests/test_prism_audit_artifacts.py deleted file mode 100644 index 72cf2a5f0..000000000 --- a/tests/test_prism_audit_artifacts.py +++ /dev/null @@ -1,4564 +0,0 @@ -#!/usr/bin/env python3 - -import gzip -import hashlib -import json -import os -from dataclasses import replace as dataclass_replace -from pathlib import Path -import select -import subprocess -import sys -import tempfile -import threading -import time -import unittest -from typing import Any -from unittest import mock - -from lab.prism.audit_artifacts import ( - AuditArtifactConfig, - AuditArtifactStore, - AuditPublicationIdentity, - CanonicalAuditBundleCorrupt, - LIVE_ENVELOPE_SCHEMA, - LIVE_EVIDENCE_SCHEMA, - OwnedCandidateArtifact, - _AuditShareSegmentConflict, - _FileIdentity, - canonical_audit_bundle_bytes, - gzip_canonical_bundle_bytes, -) -from lab.prism.bundle_compiler import canonical_bundle_bytes -from lab.prism.share_ledger import SingleWriterShareLedger - - -BLOCK_A = "aa" * 32 -BLOCK_B = "bb" * 32 -DIGEST = "11" * 32 - - -class _TestAuditArtifactStore(AuditArtifactStore): - """Keep publication fixtures terse while supplying a real verifier identity.""" - - def publish_success(self, **kwargs: object): # type: ignore[no-untyped-def] - report = kwargs.get("report") - identity = kwargs.get("identity") - if "publication_floor_sequence" not in kwargs: - if not isinstance(identity, AuditPublicationIdentity): - raise AssertionError("publication test identity is required") - kwargs["publication_floor_sequence"] = identity.sequence - if "verification_identity" not in kwargs: - if not isinstance(report, dict): - raise AssertionError("publication test report is required") - kwargs["verification_identity"] = self.build_verification_identity( - trust_source="configured", - trusted_writer_public_key_hex="44" * 32, - literal_sha256=DIGEST, - literal_byte_len=123, - report=report, - ) - with self.publication_order_guard(): - return super().publish_success(**kwargs) # type: ignore[arg-type] - - -class AuditArtifactStoreTest(unittest.TestCase): - def make_store(self, root: Path, **kwargs: object) -> AuditArtifactStore: - evidence_path = Path(kwargs.pop("evidence_path", root / "evidence.json")) - return _TestAuditArtifactStore( - AuditArtifactConfig( - root=root, - evidence_path=evidence_path, - live_bundle_retention=int(kwargs.pop("live_bundle_retention", 5)), - candidate_retention_seconds=int( - kwargs.pop("candidate_retention_seconds", 86_400) - ), - share_segment_size=int(kwargs.pop("share_segment_size", 0)), - verifier_timeout_seconds=float( - kwargs.pop("verifier_timeout_seconds", 60.0) - ), - ), - **kwargs, - ) - - @staticmethod - def transfer_candidate( - store: AuditArtifactStore, - candidate: OwnedCandidateArtifact, - ) -> None: - path = candidate.path - with path.open("rb") as handle: - store.adopt_compiler_candidate( - candidate, - path=path, - value=os.fstat(handle.fileno()), - ) - - @staticmethod - def report( - digest: str = DIGEST, - *, - block_height: int = 1, - ) -> dict[str, object]: - return { - "schema": "qbit.prism.audit-verification-report.v1", - "block_height": block_height, - "audit_bundle_sha256_hex": digest, - "reward_manifest_sha256_hex": "44" * 32, - "payout_policy_manifest_sha256_hex": "55" * 32, - "prism_audit_commitment_leaf_hex": "66" * 32, - "audit_commitment_root_hex": "77" * 32, - "coinbase_txid": "22" * 32, - "coinbase_wtxid": "88" * 32, - "coinbase_manifest_sha256_hex": "33" * 32, - "coinbase_tx_hex": "00", - "coinbase_value_sats": 1, - "min_output_sats": 1, - "onchain_output_count": 0, - "accrued_account_count": 0, - } - - @staticmethod - def persistence(digest: str = DIGEST) -> dict[str, object]: - return {"audit_bundle_sha256": digest, "body_uri": ""} - - def test_paths_reject_untrusted_hashes_and_stay_in_resolved_root(self) -> None: - with tempfile.TemporaryDirectory() as tmp: - root = Path(tmp) / "audit" - store = self.make_store(root) - candidate = store.issue_candidate(block_hash=BLOCK_A) - self.assertEqual(candidate.path.parent, root.resolve()) - self.assertEqual( - store.live_envelope_path(block_height=0, block_hash=BLOCK_A).parent, - root.resolve(), - ) - for invalid in ("a", "gg" * 32, "../" + BLOCK_A, BLOCK_A + "/x"): - with self.subTest(invalid=invalid), self.assertRaises(ValueError): - store.issue_candidate(block_hash=invalid) - - def test_publication_identity_and_live_path_reject_lossy_types(self) -> None: - invalid_identities = ( - (True, 1, BLOCK_A), - ("1", 1, BLOCK_A), - (-1, 1, BLOCK_A), - (1, False, BLOCK_A), - (1, "1", BLOCK_A), - (1, -1, BLOCK_A), - (1, 1, BLOCK_A.upper()), - ) - for sequence, height, block_hash in invalid_identities: - with self.subTest( - sequence=sequence, - height=height, - block_hash=block_hash, - ), self.assertRaises(ValueError): - AuditPublicationIdentity(sequence, height, block_hash) # type: ignore[arg-type] - with tempfile.TemporaryDirectory() as tmp: - store = self.make_store(Path(tmp)) - for height in (True, "1"): - with self.subTest(path_height=height), self.assertRaises(ValueError): - store.live_envelope_path( # type: ignore[arg-type] - block_height=height, - block_hash=BLOCK_A, - ) - identity = AuditPublicationIdentity(1, 1, BLOCK_A) - for floor in (True, "1", -1): - with self.subTest(publication_floor=floor), self.assertRaises( - ValueError - ): - store.publish_success( - identity=identity, - publication_floor_sequence=floor, # type: ignore[arg-type] - report=self.report(), - persistence=self.persistence(), - evidence={}, - created_at="now", - ) - with self.assertRaisesRegex(RuntimeError, "exceeds"): - store.publish_success( - identity=identity, - publication_floor_sequence=0, - report=self.report(), - persistence=self.persistence(), - evidence={}, - created_at="now", - ) - with self.assertRaisesRegex(RuntimeError, "behind"): - store.publish_success( - identity=identity, - publication_floor_sequence=2, - report=self.report(), - persistence=self.persistence(), - evidence={}, - created_at="now", - ) - - def test_live_publication_canonicalizes_persistence_digest_for_restart(self) -> None: - with tempfile.TemporaryDirectory() as tmp: - root = Path(tmp) - store = self.make_store(root) - publication = store.publish_success( - identity=AuditPublicationIdentity(1, 1, BLOCK_A), - report=self.report(), - persistence=self.persistence(DIGEST.upper()), - evidence={}, - created_at="now", - ) - self.assertEqual( - publication.evidence["persistence"]["audit_bundle_sha256"], # type: ignore[index] - DIGEST, - ) - store.close() - restarted = self.make_store(root) - latest = restarted.latest_evidence() - self.assertIsNotNone(latest) - assert latest is not None - self.assertEqual(latest["persistence"]["audit_bundle_sha256"], DIGEST) - - def test_root_final_symlink_is_rejected_and_ancestor_is_resolved(self) -> None: - with tempfile.TemporaryDirectory() as tmp: - base = Path(tmp) - real = base / "real" - real.mkdir() - ancestor = base / "ancestor" - ancestor.symlink_to(real, target_is_directory=True) - store = self.make_store(ancestor / "audit") - self.assertEqual(store.root, (real / "audit").resolve()) - target = base / "target" - target.mkdir() - final_link = base / "final-link" - final_link.symlink_to(target, target_is_directory=True) - with self.assertRaisesRegex(RuntimeError, "non-symlink"): - self.make_store(final_link) - - def test_root_directory_authority_swap_matrix_fails_closed_and_recovers(self) -> None: - with tempfile.TemporaryDirectory() as tmp: - base = Path(tmp) - root = base / "audit" - evidence_path = base / "state" / "evidence.json" - store = self.make_store(root, evidence_path=evidence_path) - store.publish_success( - identity=AuditPublicationIdentity(1, 1, BLOCK_A), - report=self.report(), - persistence=self.persistence(), - evidence={}, - created_at="now", - ) - candidate = store.issue_candidate(block_hash=BLOCK_B) - store.write_compatibility_candidate(candidate, {"candidate": True}) - body = store.body_path(BLOCK_A, DIGEST) - store._write_immutable_bytes(body, b"body") - - pinned_root = base / "audit-pinned" - root.rename(pinned_root) - root.mkdir() - sentinel = root / "operator-sentinel" - sentinel.write_bytes(b"operator") - - self.assertIsNone(store.latest_evidence()) - self.assertEqual(store.publication_sequence_floor(), 0) - self.assertEqual(store.metrics_snapshot()["scan_error"], 1) - self.assertEqual(store.prune_best_effort().errors, 1) - with self.assertRaisesRegex(RuntimeError, "root identity"): - store._write_immutable_bytes(body, b"replacement") - with self.assertRaisesRegex(RuntimeError, "root identity"): - store._read_owned_regular_bytes(body) - with self.assertRaisesRegex(RuntimeError, "root identity"): - store.discard_candidate(candidate) - with self.assertRaisesRegex(RuntimeError, "root identity"): - store.publish_success( - identity=AuditPublicationIdentity(2, 2, BLOCK_B), - report=self.report(block_height=2), - persistence=self.persistence(), - evidence={}, - created_at="blocked", - ) - self.assertEqual(sentinel.read_bytes(), b"operator") - self.assertEqual(sorted(path.name for path in root.iterdir()), [sentinel.name]) - - replacement = base / "audit-replacement" - root.rename(replacement) - pinned_root.rename(root) - self.assertEqual(store.publication_sequence_floor(), 1) - self.assertEqual(store.latest_evidence()["block_hash"], BLOCK_A) # type: ignore[index] - self.assertEqual(store._read_owned_regular_bytes(body)[0], b"body") - store.discard_candidate(candidate) - self.assertFalse(candidate.path.exists()) - self.assertEqual((replacement / sentinel.name).read_bytes(), b"operator") - - def test_external_evidence_parent_swap_rolls_back_publication_and_recovers(self) -> None: - with tempfile.TemporaryDirectory() as tmp: - base = Path(tmp) - root = base / "audit" - state = base / "state" - evidence_path = state / "evidence.json" - store = self.make_store(root, evidence_path=evidence_path) - store.publish_success( - identity=AuditPublicationIdentity(1, 1, BLOCK_A), - report=self.report(), - persistence=self.persistence(), - evidence={}, - created_at="now", - ) - original_write = store._write_mutable_json - pinned_state = base / "state-pinned" - replacement_state = base / "state-replacement" - - def swap_before_evidence( - path: Path, - *args: object, - **kwargs: object, - ) -> object: - if path == store.evidence_path: - state.rename(pinned_state) - state.mkdir() - (state / "operator-sentinel").write_bytes(b"operator") - return original_write(path, *args, **kwargs) - - new_envelope = store.live_envelope_path( - block_height=2, - block_hash=BLOCK_B, - ) - with mock.patch.object( - store, - "_write_mutable_json", - side_effect=swap_before_evidence, - ), self.assertRaisesRegex(RuntimeError, "evidence parent identity"): - store.publish_success( - identity=AuditPublicationIdentity(2, 2, BLOCK_B), - report=self.report(block_height=2), - persistence=self.persistence(), - evidence={}, - created_at="blocked", - ) - self.assertFalse(new_envelope.exists()) - self.assertEqual( - (state / "operator-sentinel").read_bytes(), - b"operator", - ) - self.assertIsNone(store.latest_evidence()) - self.assertEqual(store.publication_sequence_floor(), 0) - - state.rename(replacement_state) - pinned_state.rename(state) - self.assertEqual(store.publication_sequence_floor(), 1) - self.assertEqual(store.latest_evidence()["block_hash"], BLOCK_A) # type: ignore[index] - - def test_live_prune_rechecks_evidence_authority_before_removal(self) -> None: - with tempfile.TemporaryDirectory() as tmp: - base = Path(tmp) - root = base / "audit" - state = base / "state" - pinned_state = base / "state-pinned" - store = self.make_store( - root, - evidence_path=state / "evidence.json", - live_bundle_retention=0, - ) - store.publish_success( - identity=AuditPublicationIdentity(1, 1, BLOCK_A), - report=self.report(), - persistence=self.persistence(), - evidence={}, - created_at="now", - ) - stale = store.live_envelope_path(block_height=2, block_hash=BLOCK_B) - stale.write_text("stale", encoding="utf-8") - real_unlink = store._unlink_scanned_owned - swapped = False - - def swap_after_scan( - path: Path, - identity: _FileIdentity, - *, - require_all_authorities: bool = False, - ) -> bool: - nonlocal swapped - if require_all_authorities and not swapped: - swapped = True - state.rename(pinned_state) - state.mkdir() - (state / "operator-sentinel").write_bytes(b"operator") - return real_unlink( - path, - identity, - require_all_authorities=require_all_authorities, - ) - - with mock.patch.object( - store, - "_unlink_scanned_owned", - side_effect=swap_after_scan, - ): - result = store.prune_best_effort() - self.assertTrue(swapped) - self.assertEqual(result.live_removed, 0) - self.assertGreaterEqual(result.errors, 1) - self.assertTrue(stale.exists()) - self.assertEqual( - (state / "operator-sentinel").read_bytes(), - b"operator", - ) - - def test_candidate_prune_rechecks_reservations_after_active_snapshot(self) -> None: - with tempfile.TemporaryDirectory() as tmp: - store = self.make_store( - Path(tmp), - candidate_retention_seconds=0, - ) - scan_waiting = threading.Event() - release_scan = threading.Event() - result: list[object] = [] - errors: list[BaseException] = [] - real_listdir = os.listdir - - def pause_before_scan(fd: int) -> list[str]: - # Scans enumerate through a fresh directory description of - # the pinned root inode (issue #255), never the pinned - # descriptor itself; recognize the scan by that identity. - value = os.fstat(fd) - if (value.st_dev, value.st_ino) == store._root_identity: - scan_waiting.set() - if not release_scan.wait(5): - raise AssertionError("timed out waiting to release prune scan") - return real_listdir(fd) - - def prune() -> None: - try: - result.append(store.prune_best_effort()) - except BaseException as exc: # pragma: no cover - asserted below - errors.append(exc) - - with mock.patch( - "lab.prism.audit_artifacts.os.listdir", - side_effect=pause_before_scan, - ): - thread = threading.Thread(target=prune) - thread.start() - self.assertTrue(scan_waiting.wait(5)) - candidate = store.issue_candidate(block_hash=BLOCK_A) - store.write_compatibility_candidate(candidate, {"reserved": True}) - expected = candidate.path.read_bytes() - release_scan.set() - thread.join(timeout=5) - - self.assertFalse(errors) - self.assertFalse(thread.is_alive()) - self.assertEqual(len(result), 1) - self.assertEqual(result[0].candidate_removed, 0) # type: ignore[union-attr] - self.assertEqual(candidate.path.read_bytes(), expected) - store.discard_candidate(candidate) - - def test_observer_scans_fail_closed_on_midscan_authority_change(self) -> None: - with tempfile.TemporaryDirectory() as tmp: - base = Path(tmp) - root = base / "audit" - store = self.make_store( - root, - candidate_retention_seconds=0, - live_bundle_retention=0, - ) - candidate = root / f"prism-live-audit-bundle-candidate-{BLOCK_A}.json" - candidate.write_bytes(b"candidate") - real_lstat = store._owned_lstat - - def run_with_swap(operation: object, label: str) -> object: - pinned = base / f"{label}-pinned" - replacement = base / f"{label}-replacement" - swapped = False - - def swap_then_reject(path: Path) -> os.stat_result: - nonlocal swapped - if not swapped: - swapped = True - root.rename(pinned) - root.mkdir() - (root / "operator-sentinel").write_bytes(b"operator") - raise RuntimeError("authority changed") - return real_lstat(path) - - with mock.patch.object( - store, - "_owned_lstat", - side_effect=swap_then_reject, - ): - result = operation() # type: ignore[operator] - self.assertTrue(swapped) - self.assertEqual( - (root / "operator-sentinel").read_bytes(), - b"operator", - ) - root.rename(replacement) - pinned.rename(root) - return result - - metrics = run_with_swap(store.metrics_snapshot, "metrics") - self.assertEqual(metrics["scan_error"], 1) # type: ignore[index] - self.assertTrue(candidate.exists()) - retained = run_with_swap(store.prune_best_effort, "prune") - self.assertGreaterEqual(retained.errors, 1) # type: ignore[union-attr] - self.assertEqual(retained.candidate_removed, 0) # type: ignore[union-attr] - self.assertTrue(candidate.exists()) - - def test_directory_authority_reconfigure_and_close_are_atomic_and_fd_clean(self) -> None: - with tempfile.TemporaryDirectory() as tmp: - base = Path(tmp) - old_root = base / "old-root" - old_evidence = base / "old-state" / "evidence.json" - store = self.make_store(old_root, evidence_path=old_evidence) - old_root_fd = store._root_fd - old_publication_lock_fd = store._publication_lock_fd - old_evidence_fd = store._evidence_parent_fd - real_open = store._open_directory_authority - calls = 0 - prepared_root_fd: int | None = None - - def fail_second(path: Path) -> tuple[int, tuple[int, int]]: - nonlocal calls, prepared_root_fd - calls += 1 - if calls == 2: - raise OSError("second authority") - result = real_open(path) - prepared_root_fd = result[0] - return result - - new_root = base / "new-root" - new_evidence = base / "new-state" / "evidence.json" - with mock.patch.object( - store, - "_open_directory_authority", - side_effect=fail_second, - ), self.assertRaisesRegex(OSError, "second authority"): - store.reconfigure(root=new_root, evidence_path=new_evidence) - self.assertEqual(store.root, old_root.resolve()) - self.assertEqual(store.evidence_path, old_evidence.resolve()) - os.fstat(old_root_fd) - os.fstat(old_evidence_fd) - self.assertIsNotNone(prepared_root_fd) - assert prepared_root_fd is not None - with self.assertRaises(OSError): - os.fstat(prepared_root_fd) - - real_os_close = os.close - old_fds = { - old_root_fd, - old_publication_lock_fd, - old_evidence_fd, - } - - def close_then_report_error(fd: int) -> None: - real_os_close(fd) - if fd in old_fds: - raise OSError("close status unavailable") - - with mock.patch( - "lab.prism.audit_artifacts.os.close", - side_effect=close_then_report_error, - ): - store.reconfigure(root=new_root, evidence_path=new_evidence) - new_root_fd = store._root_fd - new_publication_lock_fd = store._publication_lock_fd - new_evidence_fd = store._evidence_parent_fd - for closed_fd in ( - old_root_fd, - old_publication_lock_fd, - old_evidence_fd, - ): - with self.assertRaises(OSError): - os.fstat(closed_fd) - store.close() - store.close() - self.assertIsNone(store.latest_evidence()) - self.assertEqual(store.publication_sequence_floor(), 0) - metrics = store.metrics_snapshot() - self.assertEqual(metrics["scan_error"], 1) - for kind in ( - "body", - "share_segment", - "live_bundle", - "candidate", - "other", - ): - self.assertEqual(metrics[kind], {"files": 0, "bytes": 0}) - retention = store.prune_best_effort() - self.assertEqual(retention.live_removed, 0) - self.assertEqual(retention.candidate_removed, 0) - self.assertEqual(retention.errors, 1) - with self.assertRaisesRegex(RuntimeError, "closed"): - store.issue_candidate(block_hash=BLOCK_A) - for closed_fd in ( - new_root_fd, - new_publication_lock_fd, - new_evidence_fd, - ): - with self.assertRaises(OSError): - os.fstat(closed_fd) - - def test_reconfigure_hides_transient_new_authority_state_from_readers(self) -> None: - with tempfile.TemporaryDirectory() as tmp: - base = Path(tmp) - old_store = self.make_store( - base / "old-root", - evidence_path=base / "old-state" / "evidence.json", - ) - old_store.publish_success( - identity=AuditPublicationIdentity(1, 1, BLOCK_A), - report=self.report(), - persistence=self.persistence(), - evidence={}, - created_at="old", - ) - new_root = base / "new-root" - new_evidence = base / "new-state" / "evidence.json" - prepared = self.make_store(new_root, evidence_path=new_evidence) - prepared.publish_success( - identity=AuditPublicationIdentity(2, 2, BLOCK_B), - report=self.report(block_height=2), - persistence=self.persistence(), - evidence={}, - created_at="new", - ) - prepared.close() - - real_load = old_store._load_current_evidence_locked - load_entered = threading.Event() - release_load = threading.Event() - reader_started = threading.Event() - reader_values: list[dict[str, object] | None] = [] - errors: list[BaseException] = [] - - def blocked_load() -> None: - load_entered.set() - if not release_load.wait(5): - raise AssertionError("timed out waiting to release evidence load") - real_load() - - def reconfigure() -> None: - try: - old_store.reconfigure( - root=new_root, - evidence_path=new_evidence, - ) - except BaseException as exc: # pragma: no cover - asserted below - errors.append(exc) - - def read_latest() -> None: - reader_started.set() - try: - reader_values.append(old_store.latest_evidence()) - except BaseException as exc: # pragma: no cover - asserted below - errors.append(exc) - - with mock.patch.object( - old_store, - "_load_current_evidence_locked", - side_effect=blocked_load, - ): - reconfigure_thread = threading.Thread(target=reconfigure) - reconfigure_thread.start() - self.assertTrue(load_entered.wait(5)) - reader_thread = threading.Thread(target=read_latest) - reader_thread.start() - self.assertTrue(reader_started.wait(5)) - reader_thread.join(timeout=0.05) - self.assertTrue(reader_thread.is_alive()) - release_load.set() - reconfigure_thread.join(timeout=5) - reader_thread.join(timeout=5) - - self.assertFalse(errors) - self.assertFalse(reconfigure_thread.is_alive()) - self.assertFalse(reader_thread.is_alive()) - self.assertEqual(len(reader_values), 1) - assert reader_values[0] is not None - self.assertEqual(reader_values[0]["block_hash"], BLOCK_B) - self.assertEqual(old_store.publication_sequence_floor(), 2) - - def test_publication_guard_is_required_and_internal_file_is_hidden(self) -> None: - with tempfile.TemporaryDirectory() as tmp: - root = Path(tmp) - store = AuditArtifactStore( - AuditArtifactConfig( - root=root, - evidence_path=root / "evidence.json", - ), - canonicalizer=canonical_bundle_bytes, - ) - identity = AuditPublicationIdentity(1, 1, BLOCK_A) - with self.assertRaisesRegex(RuntimeError, "guard is required"): - store.publish_success( - identity=identity, - publication_floor_sequence=1, - report=self.report(), - persistence=self.persistence(), - evidence={}, - verification_identity={}, - created_at="now", - ) - with self.assertRaisesRegex(RuntimeError, "guard is required"): - store.adopt_legacy_publication_identity( - identity, - publication_floor_sequence=1, - ) - self.assertFalse(store.evidence_path.exists()) - metrics = store.metrics_snapshot() - self.assertEqual(metrics["scan_error"], 0) - for kind in ( - "body", - "share_segment", - "live_bundle", - "candidate", - "other", - ): - self.assertEqual(metrics[kind], {"files": 0, "bytes": 0}) - - lock_path = root / ".prism-audit-publication.lock" - parked = root / ".prism-audit-publication.lock.parked" - lock_path.rename(parked) - lock_path.write_bytes(b"replacement") - with self.assertRaisesRegex(RuntimeError, "lock identity changed"): - with store.publication_order_guard(): - pass - self.assertEqual(store.prune_best_effort().errors, 1) - lock_path.unlink() - parked.rename(lock_path) - with store.publication_order_guard(): - with store.publication_order_guard(): - pass - old_authority = ( - store.root, - store.evidence_path, - store._root_fd, - store._publication_lock_fd, - store._evidence_parent_fd, - ) - next_root = root / "nested-reconfigure" - with self.assertRaisesRegex(RuntimeError, "inside publication guard"): - store.reconfigure( - root=next_root, - evidence_path=next_root / "evidence.json", - ) - self.assertEqual( - ( - store.root, - store.evidence_path, - store._root_fd, - store._publication_lock_fd, - store._evidence_parent_fd, - ), - old_authority, - ) - self.assertFalse(next_root.exists()) - for fd in old_authority[2:]: - os.fstat(fd) - - def test_close_waits_for_guard_and_is_rejected_inside_owned_guard(self) -> None: - with tempfile.TemporaryDirectory() as tmp: - root = Path(tmp) - store = self.make_store(root) - guard_entered = threading.Event() - release_guard = threading.Event() - close_attempting = threading.Event() - close_finished = threading.Event() - - def hold_guard() -> None: - with store.publication_order_guard(): - guard_entered.set() - release_guard.wait(5) - - def close_store() -> None: - close_attempting.set() - store.close() - close_finished.set() - - guard_thread = threading.Thread(target=hold_guard) - guard_thread.start() - self.assertTrue(guard_entered.wait(5)) - close_thread = threading.Thread(target=close_store) - close_thread.start() - self.assertTrue(close_attempting.wait(5)) - self.assertFalse(close_finished.wait(0.05)) - release_guard.set() - guard_thread.join(timeout=5) - close_thread.join(timeout=5) - self.assertFalse(guard_thread.is_alive()) - self.assertFalse(close_thread.is_alive()) - self.assertTrue(close_finished.is_set()) - with self.assertRaisesRegex(RuntimeError, "closed"): - with store.publication_order_guard(): - pass - - second = self.make_store(root / "second") - old_fds = ( - second._root_fd, - second._publication_lock_fd, - second._evidence_parent_fd, - ) - with second.publication_order_guard(): - with self.assertRaisesRegex(RuntimeError, "inside publication guard"): - second.close() - for fd in old_fds: - os.fstat(fd) - second.close() - - def test_publication_lock_symlink_is_rejected_on_open_and_reconfigure(self) -> None: - with tempfile.TemporaryDirectory() as tmp: - base = Path(tmp) - outside = base / "outside" - outside.write_bytes(b"outside") - bad_root = base / "bad-root" - bad_root.mkdir() - (bad_root / ".prism-audit-publication.lock").symlink_to(outside) - with self.assertRaisesRegex(RuntimeError, "cannot be opened safely"): - AuditArtifactStore( - AuditArtifactConfig( - root=bad_root, - evidence_path=bad_root / "evidence.json", - ) - ) - - good_root = base / "good-root" - store = self.make_store(good_root) - old_state = ( - store.root, - store._root_fd, - store._publication_lock_fd, - store._evidence_parent_fd, - ) - with self.assertRaisesRegex(RuntimeError, "cannot be opened safely"): - store.reconfigure(root=bad_root) - self.assertEqual(store.root, old_state[0]) - for fd in old_state[1:]: - os.fstat(fd) - self.assertEqual(outside.read_bytes(), b"outside") - - def test_two_store_and_subprocess_publication_guards_serialize(self) -> None: - with tempfile.TemporaryDirectory() as tmp: - root = Path(tmp) - first = self.make_store(root) - second = self.make_store(root) - attempting = threading.Event() - entered = threading.Event() - release = threading.Event() - - def hold_second() -> None: - attempting.set() - with second.publication_order_guard(): - entered.set() - release.wait(5) - - with first.publication_order_guard(): - thread = threading.Thread(target=hold_second) - thread.start() - self.assertTrue(attempting.wait(5)) - self.assertFalse(entered.wait(0.05)) - with self.assertRaisesRegex(RuntimeError, "another store"): - with second.publication_order_guard(): - pass - self.assertTrue(entered.wait(5)) - release.set() - thread.join(timeout=5) - self.assertFalse(thread.is_alive()) - - script = """ -import sys -from pathlib import Path -from lab.prism.audit_artifacts import AuditArtifactConfig, AuditArtifactStore -root = Path(sys.argv[1]) -store = AuditArtifactStore(AuditArtifactConfig(root=root, evidence_path=root / 'evidence.json')) -with store.publication_order_guard(): - print('locked', flush=True) - sys.stdin.readline() -""" - child = subprocess.Popen( - [sys.executable, "-c", script, str(root)], - cwd=Path(__file__).resolve().parents[1], - stdin=subprocess.PIPE, - stdout=subprocess.PIPE, - stderr=subprocess.PIPE, - text=True, - ) - try: - assert child.stdin is not None - assert child.stdout is not None - assert child.stderr is not None - ready, _writable, _exceptional = select.select( - [child.stdout], - [], - [], - 5, - ) - self.assertEqual(ready, [child.stdout], "child guard handshake timed out") - self.assertEqual(child.stdout.readline().strip(), "locked") - parent_attempting = threading.Event() - parent_entered = threading.Event() - - def enter_parent() -> None: - parent_attempting.set() - with first.publication_order_guard(): - parent_entered.set() - - parent_thread = threading.Thread(target=enter_parent) - parent_thread.start() - self.assertTrue(parent_attempting.wait(5)) - self.assertFalse(parent_entered.wait(0.05)) - child.stdin.write("\n") - child.stdin.flush() - child.wait(timeout=5) - parent_thread.join(timeout=5) - self.assertEqual(child.returncode, 0, child.stderr.read()) - self.assertTrue(parent_entered.is_set()) - self.assertFalse(parent_thread.is_alive()) - finally: - if child.poll() is None: - try: - assert child.stdin is not None - child.stdin.write("\n") - child.stdin.flush() - child.wait(timeout=2) - except (BrokenPipeError, subprocess.TimeoutExpired): - child.kill() - child.wait(timeout=2) - for stream in (child.stdin, child.stdout, child.stderr): - if stream is not None: - stream.close() - - def test_peer_publication_reconciles_stale_replay_repair_and_prune(self) -> None: - with tempfile.TemporaryDirectory() as tmp: - root = Path(tmp) - stale = self.make_store(root, live_bundle_retention=2) - first = stale.publish_success( - identity=AuditPublicationIdentity(1, 1, BLOCK_A), - report=self.report(), - persistence=self.persistence(), - evidence={"confirmation": {"confirmed_count": 1}}, - created_at="first", - ) - peer = self.make_store(root, live_bundle_retention=2) - second = peer.publish_success( - identity=AuditPublicationIdentity(2, 2, BLOCK_B), - report=self.report(block_height=2), - persistence=self.persistence(), - evidence={"confirmation": {"confirmed_count": 1}}, - created_at="second", - ) - - replay = stale.publish_success( - identity=AuditPublicationIdentity(1, 1, BLOCK_A), - publication_floor_sequence=2, - report=self.report(), - persistence=self.persistence(), - evidence={"confirmation": {"confirmed_count": 1}}, - created_at="stale", - ) - self.assertFalse(replay.published) - self.assertEqual(stale.latest_evidence()["block_hash"], BLOCK_B) # type: ignore[index] - - stale._invalidated_legacy_identity = AuditPublicationIdentity( - 1, - 1, - BLOCK_A, - ) - with self.assertRaisesRegex(RuntimeError, "identity conflict"): - stale.publish_success( - identity=AuditPublicationIdentity(2, 2, BLOCK_A), - publication_floor_sequence=2, - report=self.report(block_height=2), - persistence=self.persistence(), - evidence={}, - created_at="conflict", - ) - stale.reconfigure(live_bundle_retention=0) - result = stale.prune_best_effort() - self.assertTrue(second.envelope_path.exists()) - self.assertFalse(first.envelope_path.exists()) - self.assertEqual(result.live_removed, 1) - - def test_floor_publication_fence_blocks_allocator_and_stale_inverse_repair( - self, - ) -> None: - with tempfile.TemporaryDirectory() as tmp: - root = Path(tmp) - store = self.make_store(root, live_bundle_retention=2) - ledger = SingleWriterShareLedger() - balance_lock = threading.RLock() - - def persist(block_hash: str, height: int) -> None: - ledger.persist_accepted_block( - block_hash=block_hash, - block_height=height, - parent_hash="00" * 32, - final_bundle={}, - audit_report={}, - ) - - persist(BLOCK_A, 1) - confirmed_a = ledger.confirm_accepted_block( - block_hash=BLOCK_A, - active_tip_height=1, - ) - identity_a = AuditPublicationIdentity( - int(confirmed_a["audit_publication_sequence"]), - 1, - BLOCK_A, - ) - floor_read = threading.Event() - release_publisher = threading.Event() - allocator_attempting = threading.Event() - allocator_allocated = threading.Event() - errors: list[BaseException] = [] - publications: list[object] = [] - - def publish_a() -> None: - try: - with balance_lock: - with store.publication_order_guard(): - floor = ledger.audit_publication_sequence_floor() - floor_read.set() - if not release_publisher.wait(5): - raise AssertionError("publisher release timed out") - publications.append( - store.publish_success( - identity=identity_a, - publication_floor_sequence=floor, - report=self.report(block_height=1), - persistence=self.persistence(), - evidence={"confirmation": confirmed_a}, - created_at="a", - ) - ) - except BaseException as exc: # pragma: no cover - asserted below - errors.append(exc) - - def allocate_and_publish_b() -> None: - try: - allocator_attempting.set() - with balance_lock: - with store.publication_order_guard(): - persist(BLOCK_B, 2) - confirmed_b = ledger.confirm_accepted_block( - block_hash=BLOCK_B, - active_tip_height=2, - ) - allocator_allocated.set() - identity_b = AuditPublicationIdentity( - int(confirmed_b["audit_publication_sequence"]), - 2, - BLOCK_B, - ) - publications.append( - store.publish_success( - identity=identity_b, - publication_floor_sequence=( - ledger.audit_publication_sequence_floor() - ), - report=self.report(block_height=2), - persistence=self.persistence(), - evidence={"confirmation": confirmed_b}, - created_at="b", - ) - ) - except BaseException as exc: # pragma: no cover - asserted below - errors.append(exc) - - publisher = threading.Thread(target=publish_a) - publisher.start() - allocator: threading.Thread | None = None - try: - self.assertTrue(floor_read.wait(5)) - allocator = threading.Thread(target=allocate_and_publish_b) - allocator.start() - self.assertTrue(allocator_attempting.wait(5)) - unexpectedly_acquired = balance_lock.acquire(blocking=False) - if unexpectedly_acquired: - balance_lock.release() - self.assertFalse(unexpectedly_acquired) - self.assertFalse(allocator_allocated.is_set()) - finally: - release_publisher.set() - publisher.join(timeout=5) - if allocator is not None: - allocator.join(timeout=5) - self.assertFalse(publisher.is_alive()) - assert allocator is not None - self.assertFalse(allocator.is_alive()) - self.assertFalse(errors) - self.assertTrue(allocator_allocated.is_set()) - self.assertEqual(len(publications), 2) - self.assertEqual(ledger.audit_publication_sequence_floor(), 2) - latest = store.latest_evidence() - assert latest is not None - self.assertEqual(latest["block_hash"], BLOCK_B) - - stale_envelope = store.live_envelope_path( - block_height=1, - block_hash=BLOCK_A, - ) - stale_envelope.unlink() - durable_before = store.evidence_path.read_bytes() - durable_identity = _FileIdentity.from_stat(store.evidence_path.stat()) - with balance_lock: - with store.publication_order_guard(): - stale = store.publish_success( - identity=identity_a, - publication_floor_sequence=( - ledger.audit_publication_sequence_floor() - ), - report=self.report(block_height=1), - persistence=self.persistence(), - evidence={"confirmation": confirmed_a}, - created_at="stale", - ) - self.assertFalse(stale.published) - self.assertFalse(stale_envelope.exists()) - self.assertEqual(store.evidence_path.read_bytes(), durable_before) - self.assertTrue(durable_identity.matches(store.evidence_path.stat())) - - def test_interleaved_distinct_landings_preserve_the_superseded_envelope( - self, - ) -> None: - """Two distinct blocks land inside one confirm->publish window. - - The synchronous client-thread tail confirms block A under the payout - balance lock and the publication order guard, then releases both before - it publishes. The accounting-actor tail lands block B inside that gap - and publishes first, so A reaches publication already superseded. A - must not become the current evidence reference, but its own height+hash - live envelope is its block's only published evidence and must be - written. - """ - with tempfile.TemporaryDirectory() as tmp: - root = Path(tmp) - store = self.make_store(root, live_bundle_retention=4) - ledger = SingleWriterShareLedger() - balance_lock = threading.RLock() - - def persist(block_hash: str, height: int) -> None: - ledger.persist_accepted_block( - block_hash=block_hash, - block_height=height, - parent_hash="00" * 32, - final_bundle={}, - audit_report={}, - ) - - persist(BLOCK_A, 1) - confirmed_a = ledger.confirm_accepted_block( - block_hash=BLOCK_A, - active_tip_height=1, - ) - sequence_a = int(confirmed_a["audit_publication_sequence"]) - identity_a = AuditPublicationIdentity(sequence_a, 1, BLOCK_A) - envelope_a = store.live_envelope_path(block_height=1, block_hash=BLOCK_A) - - confirm_gap_open = threading.Event() - resume_publish_a = threading.Event() - errors: list[BaseException] = [] - publications: dict[str, object] = {} - - def synchronous_tail_publishes_a() -> None: - try: - # The confirm scope releases the balance lock and the - # publication guard before the publish scope is entered. - # Holding either across the gap would make the interleaving - # unreachable and the regression invisible. - with balance_lock: - with store.publication_order_guard(): - if ledger.audit_publication_sequence_floor() != sequence_a: - raise AssertionError("confirm floor is unexpected") - confirm_gap_open.set() - if not resume_publish_a.wait(5): - raise AssertionError("publish release timed out") - with balance_lock: - with store.publication_order_guard(): - publications["a"] = store.publish_success( - identity=identity_a, - publication_floor_sequence=( - ledger.audit_publication_sequence_floor() - ), - report=self.report(block_height=1), - persistence=self.persistence(), - evidence={"confirmation": confirmed_a}, - created_at="a", - # The live writer finalizing its own landing, - # exactly as the block finalization path does. - restore_superseded_envelope=True, - ) - except BaseException as exc: # pragma: no cover - asserted below - errors.append(exc) - - def actor_tail_lands_b() -> None: - try: - with balance_lock: - with store.publication_order_guard(): - persist(BLOCK_B, 2) - confirmed_b = ledger.confirm_accepted_block( - block_hash=BLOCK_B, - active_tip_height=2, - ) - identity_b = AuditPublicationIdentity( - int(confirmed_b["audit_publication_sequence"]), - 2, - BLOCK_B, - ) - publications["b_identity"] = identity_b - publications["b"] = store.publish_success( - identity=identity_b, - publication_floor_sequence=( - ledger.audit_publication_sequence_floor() - ), - report=self.report(block_height=2), - persistence=self.persistence(), - evidence={"confirmation": confirmed_b}, - created_at="b", - ) - except BaseException as exc: # pragma: no cover - asserted below - errors.append(exc) - - synchronous = threading.Thread(target=synchronous_tail_publishes_a) - synchronous.start() - actor: threading.Thread | None = None - envelope_b_before = b"" - try: - self.assertTrue(confirm_gap_open.wait(5)) - actor = threading.Thread(target=actor_tail_lands_b) - actor.start() - actor.join(timeout=5) - self.assertFalse(actor.is_alive()) - self.assertFalse(errors) - b = publications["b"] - self.assertTrue(b.published) - self.assertEqual( - publications["b_identity"].sequence, - sequence_a + 1, - ) - self.assertEqual(ledger.audit_publication_sequence_floor(), sequence_a + 1) - envelope_b_before = b.envelope_path.read_bytes() - self.assertFalse(envelope_a.exists()) - finally: - resume_publish_a.set() - synchronous.join(timeout=5) - if actor is not None: - actor.join(timeout=5) - self.assertFalse(synchronous.is_alive()) - self.assertFalse(errors) - - a = publications["a"] - self.assertEqual(a.identity.sequence, sequence_a) - self.assertFalse(a.published) - self.assertTrue(a.envelope_path.exists()) - self.assertEqual(a.envelope_path, envelope_a) - landed_a = json.loads(envelope_a.read_text(encoding="utf-8")) - self.assertEqual(landed_a["block_hash"], BLOCK_A) - self.assertEqual(landed_a["block_height"], 1) - - latest = store.latest_evidence() - assert latest is not None - self.assertEqual(latest["block_hash"], BLOCK_B) - b = publications["b"] - self.assertTrue(b.envelope_path.exists()) - self.assertEqual(b.envelope_path.read_bytes(), envelope_b_before) - - def test_superseded_landing_converges_after_the_peer_publishes(self) -> None: - """The second interleaving shape: B is confirmed but not yet published. - - A publishes at its own ordinal against a floor the peer already - advanced, which is retained as retryable. Once B publishes, A's retry - lands its live envelope without disturbing the durable evidence - pointer. - """ - with tempfile.TemporaryDirectory() as tmp: - root = Path(tmp) - store = self.make_store(root, live_bundle_retention=4) - ledger = SingleWriterShareLedger() - - def persist(block_hash: str, height: int) -> None: - ledger.persist_accepted_block( - block_hash=block_hash, - block_height=height, - parent_hash="00" * 32, - final_bundle={}, - audit_report={}, - ) - - persist(BLOCK_A, 1) - confirmed_a = ledger.confirm_accepted_block( - block_hash=BLOCK_A, - active_tip_height=1, - ) - identity_a = AuditPublicationIdentity( - int(confirmed_a["audit_publication_sequence"]), - 1, - BLOCK_A, - ) - persist(BLOCK_B, 2) - confirmed_b = ledger.confirm_accepted_block( - block_hash=BLOCK_B, - active_tip_height=2, - ) - identity_b = AuditPublicationIdentity( - int(confirmed_b["audit_publication_sequence"]), - 2, - BLOCK_B, - ) - self.assertEqual(identity_b.sequence, identity_a.sequence + 1) - floor = ledger.audit_publication_sequence_floor() - self.assertEqual(floor, identity_b.sequence) - envelope_a = store.live_envelope_path(block_height=1, block_hash=BLOCK_A) - - with self.assertRaisesRegex( - RuntimeError, - "behind the durable ledger floor", - ): - store.publish_success( - identity=identity_a, - publication_floor_sequence=floor, - report=self.report(block_height=1), - persistence=self.persistence(), - evidence={"confirmation": confirmed_a}, - created_at="a-early", - ) - self.assertFalse(envelope_a.exists()) - self.assertIsNone(store.latest_evidence()) - - published_b = store.publish_success( - identity=identity_b, - publication_floor_sequence=floor, - report=self.report(block_height=2), - persistence=self.persistence(), - evidence={"confirmation": confirmed_b}, - created_at="b", - ) - self.assertTrue(published_b.published) - durable_before = store.evidence_path.read_bytes() - durable_identity = _FileIdentity.from_stat(store.evidence_path.stat()) - - retried_a = store.publish_success( - identity=identity_a, - publication_floor_sequence=( - ledger.audit_publication_sequence_floor() - ), - report=self.report(block_height=1), - persistence=self.persistence(), - evidence={"confirmation": confirmed_a}, - created_at="a-retry", - restore_superseded_envelope=True, - ) - self.assertFalse(retried_a.published) - self.assertTrue(envelope_a.exists()) - landed_a = json.loads(envelope_a.read_text(encoding="utf-8")) - self.assertEqual(landed_a["block_hash"], BLOCK_A) - self.assertEqual(landed_a["block_height"], 1) - latest = store.latest_evidence() - assert latest is not None - self.assertEqual(latest["block_hash"], BLOCK_B) - self.assertEqual(store.evidence_path.read_bytes(), durable_before) - self.assertTrue(durable_identity.matches(store.evidence_path.stat())) - - def test_reconfigure_busy_and_post_swap_failure_are_atomic_and_fd_clean(self) -> None: - with tempfile.TemporaryDirectory() as tmp: - base = Path(tmp) - old_root = base / "old-root" - old_evidence = base / "old-state" / "evidence.json" - store = self.make_store(old_root, evidence_path=old_evidence) - new_root = base / "new-root" - blocker = self.make_store( - new_root, - evidence_path=base / "new-state" / "evidence.json", - ) - blocker_entered = threading.Event() - release_blocker = threading.Event() - - def hold_new_root() -> None: - with blocker.publication_order_guard(): - blocker_entered.set() - release_blocker.wait(5) - - blocker_thread = threading.Thread(target=hold_new_root) - blocker_thread.start() - self.assertTrue(blocker_entered.wait(5)) - old_fds = ( - store._root_fd, - store._publication_lock_fd, - store._evidence_parent_fd, - ) - with self.assertRaisesRegex(RuntimeError, "guard is busy"): - store.reconfigure( - root=new_root, - evidence_path=base / "new-state" / "evidence.json", - ) - self.assertEqual(store.root, old_root.resolve()) - for fd in old_fds: - os.fstat(fd) - release_blocker.set() - blocker_thread.join(timeout=5) - self.assertFalse(blocker_thread.is_alive()) - - target_root = base / "target-root" - target_evidence = base / "target-state" / "evidence.json" - prepared_fds: list[int] = [] - real_open_directory = store._open_directory_authority - real_open_lock = store._open_publication_lock_authority - - def record_directory(path: Path) -> tuple[int, tuple[int, int]]: - result = real_open_directory(path) - prepared_fds.append(result[0]) - return result - - def record_lock( - root: Path, - root_fd: int, - root_identity: tuple[int, int], - ) -> tuple[int, _FileIdentity]: - result = real_open_lock(root, root_fd, root_identity) - prepared_fds.append(result[0]) - return result - - with mock.patch.object( - store, - "_open_directory_authority", - side_effect=record_directory, - ), mock.patch.object( - store, - "_open_publication_lock_authority", - side_effect=record_lock, - ), mock.patch.object( - store, - "_reload_current_evidence_locked", - side_effect=RuntimeError("injected post-swap load failure"), - ), self.assertRaisesRegex(RuntimeError, "post-swap load"): - store.reconfigure( - root=target_root, - evidence_path=target_evidence, - ) - self.assertEqual(store.root, old_root.resolve()) - self.assertEqual(store.evidence_path, old_evidence.resolve()) - self.assertEqual( - ( - store._root_fd, - store._publication_lock_fd, - store._evidence_parent_fd, - ), - old_fds, - ) - for fd in old_fds: - os.fstat(fd) - for fd in prepared_fds: - with self.assertRaises(OSError): - os.fstat(fd) - - boundary_root = base / "boundary-root" - boundary_evidence = base / "boundary-state" / "evidence.json" - boundary_prepared_fds: list[int] = [] - explicit_old_validations = 0 - real_validate = store._validate_publication_lock_identity - - def record_boundary_directory( - path: Path, - ) -> tuple[int, tuple[int, int]]: - result = real_open_directory(path) - boundary_prepared_fds.append(result[0]) - return result - - def record_boundary_lock( - root: Path, - root_fd: int, - root_identity: tuple[int, int], - ) -> tuple[int, _FileIdentity]: - result = real_open_lock(root, root_fd, root_identity) - boundary_prepared_fds.append(result[0]) - return result - - def fail_final_old_validation(**kwargs: object) -> None: - nonlocal explicit_old_validations - if kwargs.get("root_fd") == old_fds[0]: - explicit_old_validations += 1 - if explicit_old_validations == 2: - raise RuntimeError("injected final old-authority loss") - real_validate(**kwargs) # type: ignore[arg-type] - - with mock.patch.object( - store, - "_open_directory_authority", - side_effect=record_boundary_directory, - ), mock.patch.object( - store, - "_open_publication_lock_authority", - side_effect=record_boundary_lock, - ), mock.patch.object( - store, - "_validate_publication_lock_identity", - side_effect=fail_final_old_validation, - ), self.assertRaisesRegex(RuntimeError, "final old-authority"): - store.reconfigure( - root=boundary_root, - evidence_path=boundary_evidence, - ) - self.assertEqual(store.root, old_root.resolve()) - self.assertEqual(store.evidence_path, old_evidence.resolve()) - self.assertEqual( - ( - store._root_fd, - store._publication_lock_fd, - store._evidence_parent_fd, - ), - old_fds, - ) - for fd in old_fds: - os.fstat(fd) - for fd in boundary_prepared_fds: - with self.assertRaises(OSError): - os.fstat(fd) - - store.reconfigure( - root=boundary_root, - evidence_path=boundary_evidence, - ) - self.assertEqual(store.root, boundary_root.resolve()) - self.assertEqual(store.evidence_path, boundary_evidence.resolve()) - for fd in old_fds: - with self.assertRaises(OSError): - os.fstat(fd) - - def test_strict_artifact_classification_rejects_lookalikes(self) -> None: - self.assertEqual( - AuditArtifactStore.artifact_kind( - f"prism-audit-bundle-body-{BLOCK_A}-{DIGEST}.json" - ), - "body", - ) - self.assertEqual( - AuditArtifactStore.artifact_kind( - f"prism-live-audit-bundle-2-{BLOCK_A}.json" - ), - "live_bundle", - ) - self.assertEqual( - AuditArtifactStore.artifact_kind( - f".prism-live-audit-bundle-candidate-{BLOCK_A}.json.tmp" - ), - "candidate", - ) - for name in ( - f"prism-live-audit-bundle-02-{BLOCK_A}.json", - f"prism-live-audit-bundle-2-{BLOCK_A}.json.bak", - "prism-live-audit-bundle-candidate-operator.json", - ): - self.assertEqual(AuditArtifactStore.artifact_kind(name), "other") - - def test_a1_only_consumes_an_injected_j1_canonical_capability(self) -> None: - with mock.patch( - "lab.prism.audit_artifacts.subprocess.run", - side_effect=AssertionError("A1 must not own canonicalizer subprocesses"), - ): - self.assertEqual( - canonical_audit_bundle_bytes({"x": 1}, lambda _value: b"typed"), - b"typed", - ) - with self.assertRaisesRegex(RuntimeError, "J1 canonical"): - canonical_audit_bundle_bytes({"x": 1}) - - completed = mock.Mock(returncode=0, stdout=b"rust-typed", stderr=b"") - with mock.patch( - "lab.prism.bundle_compiler.subprocess.run", - return_value=completed, - ) as run: - self.assertEqual(canonical_bundle_bytes({"x": "miner-é"}), b"rust-typed") - self.assertIn("qbit-prism-audit-canonicalize", " ".join(run.call_args.args[0])) - - def test_candidate_cleanup_removes_only_its_adopted_inode(self) -> None: - with tempfile.TemporaryDirectory() as tmp: - store = self.make_store(Path(tmp)) - candidate = store.issue_candidate(block_hash=BLOCK_A) - candidate.path.write_bytes(b"first") - self.transfer_candidate(store, candidate) - candidate.path.unlink() - candidate.path.write_bytes(b"replacement") - store.discard_candidate(candidate) - self.assertEqual(candidate.path.read_bytes(), b"replacement") - - def test_unadopted_candidate_collision_is_preserved_and_released(self) -> None: - with tempfile.TemporaryDirectory() as tmp: - root = Path(tmp) - store = self.make_store(root) - candidate = store.issue_candidate(block_hash=BLOCK_A) - candidate.path.write_bytes(b"competitor") - with self.assertRaises(FileExistsError): - store.write_compatibility_candidate(candidate, {"x": 1}) - self.assertEqual(candidate.path.read_bytes(), b"competitor") - store.reconfigure(root=root / "next") - - def test_unadopted_discard_never_deletes_a_created_path(self) -> None: - with tempfile.TemporaryDirectory() as tmp: - store = self.make_store(Path(tmp)) - candidate = store.issue_candidate(block_hash=BLOCK_A) - candidate.path.write_bytes(b"not-transferred") - store.discard_candidate(candidate) - self.assertEqual(candidate.path.read_bytes(), b"not-transferred") - - def test_reconfigure_rejects_active_candidate(self) -> None: - with tempfile.TemporaryDirectory() as tmp: - root = Path(tmp) - store = self.make_store(root / "a") - store.issue_candidate(block_hash=BLOCK_A) - with self.assertRaisesRegex(RuntimeError, "candidates are active"): - store.reconfigure(root=root / "b") - - def test_compatibility_candidate_failure_cleans_owned_temp(self) -> None: - with tempfile.TemporaryDirectory() as tmp: - store = self.make_store(Path(tmp)) - candidate = store.issue_candidate(block_hash=BLOCK_A) - with mock.patch("os.fsync", side_effect=OSError("fsync")): - with self.assertRaises(OSError): - store.write_compatibility_candidate(candidate, {"x": 1}) - self.assertFalse(candidate.path.exists()) - - def test_verify_candidate_binds_literal_identity_and_report(self) -> None: - with tempfile.TemporaryDirectory() as tmp: - store = self.make_store(Path(tmp)) - candidate = store.issue_candidate(block_hash=BLOCK_A) - candidate.path.write_bytes(b"canonical") - self.transfer_candidate(store, candidate) - digest = hashlib.sha256(b"canonical").hexdigest() - verified = store.verify_candidate( - candidate, - coinbase_tx_hex="00", - expected_coinbase_value_sats=1, - trusted_writer_public_key_hex="44" * 32, - verifier=lambda *_args, **_kwargs: self.report(digest), - ) - self.assertTrue(verified.canonical_copy_eligible) - self.assertEqual(verified.literal_sha256, digest) - self.assertEqual( - verified.verification_identity, - AuditArtifactStore.build_verification_identity( - trust_source="configured", - trusted_writer_public_key_hex="44" * 32, - literal_sha256=digest, - literal_byte_len=len(b"canonical"), - report=self.report(digest), - ), - ) - - def test_verifier_retry_identity_never_reuses_prior_attempt(self) -> None: - with tempfile.TemporaryDirectory() as tmp: - store = self.make_store(Path(tmp)) - candidate = store.issue_candidate(block_hash=BLOCK_A) - candidate.path.write_bytes(b"canonical") - self.transfer_candidate(store, candidate) - digest = hashlib.sha256(b"canonical").hexdigest() - - with self.assertRaisesRegex(RuntimeError, "failed attempt"): - store.verify_candidate( - candidate, - coinbase_tx_hex="00", - expected_coinbase_value_sats=1, - trusted_writer_public_key_hex="44" * 32, - verifier=lambda *_args, **_kwargs: (_ for _ in ()).throw( - RuntimeError("failed attempt") - ), - ) - verified = store.verify_candidate( - candidate, - coinbase_tx_hex="00", - expected_coinbase_value_sats=1, - trusted_writer_public_key_hex="44" * 32, - verifier=lambda *_args, **_kwargs: self.report(digest), - ) - store.require_current_verified_candidate(verified, candidate) - - store.discard_candidate(candidate) - replacement = store.issue_candidate(block_hash=BLOCK_A) - replacement.path.write_bytes(b"canonical") - self.transfer_candidate(store, replacement) - with self.assertRaisesRegex(RuntimeError, "another candidate"): - store.require_current_verified_candidate(verified, replacement) - with self.assertRaisesRegex(RuntimeError, "later failure"): - store.verify_candidate( - replacement, - coinbase_tx_hex="00", - expected_coinbase_value_sats=1, - trusted_writer_public_key_hex="44" * 32, - verifier=lambda *_args, **_kwargs: (_ for _ in ()).throw( - RuntimeError("later failure") - ), - ) - - def test_verify_candidate_rejects_post_verifier_swap(self) -> None: - with tempfile.TemporaryDirectory() as tmp: - store = self.make_store(Path(tmp)) - candidate = store.issue_candidate(block_hash=BLOCK_A) - candidate.path.write_bytes(b"canonical") - self.transfer_candidate(store, candidate) - - def verifier(_path: Path, *_args: object, **_kwargs: object) -> dict[str, object]: - candidate.path.unlink() - candidate.path.write_bytes(b"replacement") - return self.report(hashlib.sha256(b"replacement").hexdigest()) - - with self.assertRaisesRegex(RuntimeError, "changed during"): - store.verify_candidate( - candidate, - coinbase_tx_hex="00", - expected_coinbase_value_sats=1, - trusted_writer_public_key_hex="44" * 32, - verifier=verifier, - ) - - def test_verifier_report_requires_complete_coinbase_identity(self) -> None: - with tempfile.TemporaryDirectory() as tmp: - for missing in ( - "audit_bundle_sha256_hex", - "reward_manifest_sha256_hex", - "payout_policy_manifest_sha256_hex", - "prism_audit_commitment_leaf_hex", - "audit_commitment_root_hex", - "coinbase_txid", - "coinbase_wtxid", - "coinbase_manifest_sha256_hex", - "coinbase_tx_hex", - "coinbase_value_sats", - "min_output_sats", - "onchain_output_count", - "accrued_account_count", - ): - with self.subTest(missing=missing): - store = self.make_store(Path(tmp) / missing) - candidate = store.issue_candidate(block_hash=BLOCK_A) - candidate.path.write_bytes(b"canonical") - self.transfer_candidate(store, candidate) - report = self.report(hashlib.sha256(b"canonical").hexdigest()) - report.pop(missing) - with self.assertRaises((RuntimeError, ValueError)): - store.verify_candidate( - candidate, - coinbase_tx_hex="00", - expected_coinbase_value_sats=1, - trusted_writer_public_key_hex="44" * 32, - verifier=lambda *_args, _report=report, **_kwargs: _report, - ) - store.discard_candidate(candidate) - - def test_verifier_report_schema_types_and_height_fail_closed(self) -> None: - cases = ( - ("schema", "wrong"), - ("block_height", "1"), - ("block_height", 2), - ("coinbase_value_sats", 1.5), - ("min_output_sats", True), - ("onchain_output_count", -1), - ("accrued_account_count", 1.5), - ) - for field, value in cases: - with self.subTest(field=field, value=value): - report = self.report() - report[field] = value - with self.assertRaises(RuntimeError): - AuditArtifactStore._validate_verifier_report( - report, - coinbase_tx_hex="00", - expected_coinbase_value_sats=1, - expected_block_height=1, - ) - - def test_trust_precedence_and_embedded_test_key_gate(self) -> None: - bundle = { - "ledger_window_attestation": { - "signature": {"public_key_hex": "55" * 32} - } - } - self.assertEqual( - AuditArtifactStore.trusted_writer_key( - "44" * 32, - bundle, - allow_embedded_test_key=True, - ), - "44" * 32, - ) - with self.assertRaisesRegex(RuntimeError, "configured"): - AuditArtifactStore.trusted_writer_key( - None, - bundle, - allow_embedded_test_key=False, - ) - self.assertEqual( - AuditArtifactStore.trusted_writer_key( - None, - bundle, - allow_embedded_test_key=True, - ), - "55" * 32, - ) - - def test_verifier_reads_unlinked_snapshot_during_candidate_aba(self) -> None: - with tempfile.TemporaryDirectory() as tmp: - store = self.make_store(Path(tmp)) - candidate = store.issue_candidate(block_hash=BLOCK_A) - candidate.path.write_bytes(b"canonical") - self.transfer_candidate(store, candidate) - original = candidate.path.stat() - digest = hashlib.sha256(b"canonical").hexdigest() - - def verifier(path: Path, *_args: object, **_kwargs: object) -> dict[str, object]: - self.assertEqual(path.read_bytes(), b"canonical") - candidate.path.write_bytes(b"replacement") - candidate.path.write_bytes(b"canonical") - os.utime( - candidate.path, - ns=(original.st_atime_ns, original.st_mtime_ns), - ) - return self.report(digest) - - verified = store.verify_candidate( - candidate, - coinbase_tx_hex="00", - expected_coinbase_value_sats=1, - trusted_writer_public_key_hex="44" * 32, - expected_block_height=1, - verifier=verifier, - ) - self.assertEqual(verified.literal_sha256, digest) - - def test_verifier_subprocess_timeout_nonzero_malformed_and_oversize(self) -> None: - with tempfile.TemporaryDirectory() as tmp: - root = Path(tmp) - bundle = root / "bundle.json" - bundle.write_text("{}", encoding="utf-8") - cases = ( - ( - [sys.executable, "-c", "import time; time.sleep(2)"], - "timed out", - 0.05, - ), - ([sys.executable, "-c", "import sys; sys.stderr.write('bad'); sys.exit(2)"], "failed: bad", 1.0), - ([sys.executable, "-c", "print('not-json')"], "invalid JSON", 1.0), - ([sys.executable, "-c", "import sys; sys.stdout.write('x' * 1100000)"], "output exceeded", 1.0), - ) - for command, message, timeout in cases: - with self.subTest(message=message): - store = self.make_store( - root / message.replace(" ", "-"), - verifier_timeout_seconds=timeout, - ) - with mock.patch( - "lab.prism.audit_artifacts.prism_tool_command", - return_value=command, - ), self.assertRaisesRegex(RuntimeError, message): - store.verify_bundle( - bundle, - "00", - "44" * 32, - expected_coinbase_value_sats=1, - ) - - def test_verifier_timeout_kills_descendants_that_inherit_output_pipes(self) -> None: - with tempfile.TemporaryDirectory() as tmp: - root = Path(tmp) - bundle = root / "bundle.json" - marker = root / "descendant-survived" - bundle.write_text("{}", encoding="utf-8") - descendant = ( - "import pathlib,time; time.sleep(0.4); " - f"pathlib.Path({str(marker)!r}).write_text('survived')" - ) - parent = ( - "import subprocess,sys,time; " - f"subprocess.Popen([sys.executable,'-c',{descendant!r}]," - "stdout=sys.stdout,stderr=sys.stderr); time.sleep(5)" - ) - store = self.make_store( - root / "audit", - verifier_timeout_seconds=0.05, - ) - with mock.patch( - "lab.prism.audit_artifacts.prism_tool_command", - return_value=[sys.executable, "-c", parent], - ), self.assertRaisesRegex(RuntimeError, "timed out"): - store.verify_bundle( - bundle, - "00", - "44" * 32, - expected_coinbase_value_sats=1, - ) - time.sleep(0.5) - self.assertFalse(marker.exists()) - - def test_publication_uses_durable_ordinal_not_height(self) -> None: - with tempfile.TemporaryDirectory() as tmp: - store = self.make_store(Path(tmp), live_bundle_retention=0) - newer = store.publish_success( - identity=AuditPublicationIdentity(8, 101, BLOCK_B), - report=self.report(block_height=101), - persistence=self.persistence(), - evidence={"audit_report": self.report(), "persistence": self.persistence()}, - created_at="now", - ) - stale = store.publish_success( - identity=AuditPublicationIdentity(7, 100, BLOCK_A), - report=self.report(block_height=100), - persistence=self.persistence(), - evidence={"audit_report": self.report(), "persistence": self.persistence()}, - created_at="later", - ) - self.assertTrue(newer.published) - self.assertFalse(stale.published) - self.assertEqual(store.latest_evidence()["block_hash"], BLOCK_B) # type: ignore[index] - deep_reorg = store.publish_success( - identity=AuditPublicationIdentity(9, 99, BLOCK_A), - report=self.report(block_height=99), - persistence=self.persistence(), - evidence={"audit_report": self.report(), "persistence": self.persistence()}, - created_at="latest", - ) - self.assertTrue(deep_reorg.published) - self.assertEqual(store.latest_evidence()["block_hash"], BLOCK_A) # type: ignore[index] - - def test_publication_and_startup_bind_report_height(self) -> None: - with tempfile.TemporaryDirectory() as tmp: - root = Path(tmp) - store = self.make_store(root) - with self.assertRaisesRegex(RuntimeError, "block height"): - store.publish_success( - identity=AuditPublicationIdentity(1, 2, BLOCK_A), - report=self.report(block_height=1), - persistence=self.persistence(), - evidence={}, - created_at="now", - ) - store.publish_success( - identity=AuditPublicationIdentity(1, 2, BLOCK_A), - report=self.report(block_height=2), - persistence=self.persistence(), - evidence={}, - created_at="now", - ) - payload = json.loads(store.evidence_path.read_text(encoding="utf-8")) - payload["audit_report"]["block_height"] = 1 - store.evidence_path.write_text(json.dumps(payload), encoding="utf-8") - self.assertIsNone(self.make_store(root).latest_evidence()) - - def test_equal_ordinal_conflict_fails_closed(self) -> None: - with tempfile.TemporaryDirectory() as tmp: - store = self.make_store(Path(tmp)) - store.publish_success( - identity=AuditPublicationIdentity(1, 1, BLOCK_A), - report=self.report(block_height=1), - persistence=self.persistence(), - evidence={"audit_report": self.report(), "persistence": self.persistence()}, - created_at="now", - ) - with self.assertRaisesRegex(RuntimeError, "conflict"): - store.publish_success( - identity=AuditPublicationIdentity(1, 1, BLOCK_B), - report=self.report(block_height=1), - persistence=self.persistence(), - evidence={"audit_report": self.report(), "persistence": self.persistence()}, - created_at="later", - ) - - def test_exact_replay_rejects_changed_coinbase_identity(self) -> None: - with tempfile.TemporaryDirectory() as tmp: - store = self.make_store(Path(tmp)) - identity = AuditPublicationIdentity(1, 1, BLOCK_A) - store.publish_success( - identity=identity, - report=self.report(block_height=1), - persistence=self.persistence(), - evidence={}, - created_at="now", - ) - changed = self.report(block_height=1) - changed["coinbase_txid"] = "99" * 32 - with self.assertRaisesRegex(RuntimeError, "replay payload conflict"): - store.publish_success( - identity=identity, - report=changed, - persistence=self.persistence(), - evidence={}, - created_at="later", - ) - - def test_exact_replay_is_stable_but_changed_evidence_conflicts(self) -> None: - with tempfile.TemporaryDirectory() as tmp: - store = self.make_store(Path(tmp)) - identity = AuditPublicationIdentity(3, 4, BLOCK_A) - first = store.publish_success( - identity=identity, - report=self.report(block_height=4), - persistence=self.persistence(), - evidence={"confirmation": {"confirmed_count": 1}}, - created_at="now", - ) - replay = store.publish_success( - identity=identity, - report=self.report(block_height=4), - persistence=self.persistence(), - evidence={"confirmation": {"confirmed_count": 1}}, - created_at="later", - ) - self.assertTrue(first.published) - self.assertFalse(replay.published) - advanced_floor_replay = store.publish_success( - identity=identity, - publication_floor_sequence=4, - report=self.report(block_height=4), - persistence=self.persistence(), - evidence={"confirmation": {"confirmed_count": 1}}, - created_at="later-still", - ) - self.assertFalse(advanced_floor_replay.published) - # The confirm disposition is observational, like the share - # counters above: the flip that published this evidence reported a - # fresh confirmation (1), and an outbox replay of the same durable - # row reports the idempotent arm (2) -- which is the whole point of - # that split (issue #61). Reuse the durable value rather than - # stranding the replay as a payload conflict. - idempotent_disposition_replay = store.publish_success( - identity=identity, - report=self.report(block_height=4), - persistence=self.persistence(), - evidence={"confirmation": {"confirmed_count": 2}}, - created_at="idempotent", - ) - self.assertFalse(idempotent_disposition_replay.published) - self.assertEqual( - idempotent_disposition_replay.evidence["confirmation"], - {"confirmed_count": 1}, - ) - # Only the disposition is observational. Everything else the - # confirmation carries -- the publication ordinal above all -- is - # block identity and still conflicts when it changes. - with self.assertRaisesRegex(RuntimeError, "replay payload conflict"): - store.publish_success( - identity=identity, - report=self.report(block_height=4), - persistence=self.persistence(), - evidence={ - "confirmation": { - "confirmed_count": 2, - "audit_publication_sequence": 9, - } - }, - created_at="later", - ) - # A changed evidence payload outside the confirmation is unchanged - # in its treatment. - with self.assertRaisesRegex(RuntimeError, "replay payload conflict"): - store.publish_success( - identity=identity, - report=self.report(block_height=4), - persistence=self.persistence(), - evidence={ - "confirmation": {"confirmed_count": 1}, - "job_share_count": 7, - }, - created_at="later", - ) - - def test_exact_replay_repairs_invalid_mutable_evidence_but_rejects_envelope_replacement( - self, - ) -> None: - identity = AuditPublicationIdentity(3, 4, BLOCK_A) - for missing in ("evidence", "envelope", "both"): - with self.subTest(missing=missing), tempfile.TemporaryDirectory() as tmp: - root = Path(tmp) - store = self.make_store(root) - first = store.publish_success( - identity=identity, - report=self.report(block_height=4), - persistence=self.persistence(), - evidence={"confirmation": {"confirmed_count": 1}}, - created_at="now", - ) - if missing in {"evidence", "both"}: - store.evidence_path.unlink() - if missing in {"envelope", "both"}: - first.envelope_path.unlink() - before_advanced_floor_attempt = ( - first.envelope_path.read_bytes() - if first.envelope_path.exists() - else None, - store.evidence_path.read_bytes() - if store.evidence_path.exists() - else None, - ) - with self.assertRaisesRegex(RuntimeError, "behind"): - store.publish_success( - identity=identity, - publication_floor_sequence=4, - report=self.report(block_height=4), - persistence=self.persistence(), - evidence={"confirmation": {"confirmed_count": 1}}, - created_at="advanced-floor", - ) - self.assertEqual( - ( - first.envelope_path.read_bytes() - if first.envelope_path.exists() - else None, - store.evidence_path.read_bytes() - if store.evidence_path.exists() - else None, - ), - before_advanced_floor_attempt, - ) - repaired = store.publish_success( - identity=identity, - report=self.report(block_height=4), - persistence=self.persistence(), - evidence={"confirmation": {"confirmed_count": 1}}, - created_at="later", - ) - self.assertTrue(repaired.published) - self.assertTrue(store.evidence_path.exists()) - self.assertTrue(first.envelope_path.exists()) - store.close() - restarted = self.make_store(root) - self.assertEqual( - restarted.latest_evidence()["block_hash"], # type: ignore[index] - BLOCK_A, - ) - - for replacement_bytes in ( - b"not-json", - b'{"operator":"replacement"}', - ): - for restart in (False, True): - with self.subTest( - evidence_replacement=replacement_bytes, - restart=restart, - ), tempfile.TemporaryDirectory() as tmp: - root = Path(tmp) - store = self.make_store(root) - first = store.publish_success( - identity=identity, - report=self.report(block_height=4), - persistence=self.persistence(), - evidence={"confirmation": {"confirmed_count": 1}}, - created_at="now", - ) - envelope_bytes = first.envelope_path.read_bytes() - store.evidence_path.write_bytes(replacement_bytes) - if restart: - store.close() - store = self.make_store(root) - self.assertIsNone(store.latest_evidence()) - with self.assertRaisesRegex(RuntimeError, "behind"): - store.publish_success( - identity=identity, - publication_floor_sequence=4, - report=self.report(block_height=4), - persistence=self.persistence(), - evidence={"confirmation": {"confirmed_count": 1}}, - created_at="advanced-floor", - ) - self.assertEqual( - store.evidence_path.read_bytes(), - replacement_bytes, - ) - repaired = store.publish_success( - identity=identity, - report=self.report(block_height=4), - persistence=self.persistence(), - evidence={"confirmation": {"confirmed_count": 1}}, - created_at="later", - ) - self.assertTrue(repaired.published) - self.assertEqual(first.envelope_path.read_bytes(), envelope_bytes) - self.assertEqual( - json.loads(store.evidence_path.read_text(encoding="utf-8"))[ - "block_hash" - ], - BLOCK_A, - ) - - for restart in (False, True): - with self.subTest( - envelope_replacement=True, - restart=restart, - ), tempfile.TemporaryDirectory() as tmp: - root = Path(tmp) - store = self.make_store(root) - first = store.publish_success( - identity=identity, - report=self.report(block_height=4), - persistence=self.persistence(), - evidence={"confirmation": {"confirmed_count": 1}}, - created_at="now", - ) - first.envelope_path.write_bytes(b'{"operator":"replacement"}') - replacement_bytes = first.envelope_path.read_bytes() - if restart: - store.close() - store = self.make_store(root) - self.assertIsNone(store.latest_evidence()) - with self.assertRaisesRegex(RuntimeError, "(conflicts|invalid)"): - store.publish_success( - identity=identity, - report=self.report(block_height=4), - persistence=self.persistence(), - evidence={"confirmation": {"confirmed_count": 1}}, - created_at="later", - ) - self.assertEqual(first.envelope_path.read_bytes(), replacement_bytes) - - def test_newer_publication_repairs_invalid_evidence_and_repoints_it(self) -> None: - for next_block_hash, next_height in ((BLOCK_A, 4), (BLOCK_B, 5)): - for restart in (False, True): - with self.subTest( - next_block_hash=next_block_hash, - restart=restart, - ), tempfile.TemporaryDirectory() as tmp: - root = Path(tmp) - store = self.make_store(root) - first = store.publish_success( - identity=AuditPublicationIdentity(3, 4, BLOCK_A), - report=self.report(block_height=4), - persistence=self.persistence(), - evidence={"confirmation": {"confirmed_count": 1}}, - created_at="now", - ) - first_envelope_bytes = first.envelope_path.read_bytes() - store.evidence_path.write_bytes(b'{"operator":"replacement"}') - if restart: - store.close() - store = self.make_store(root) - self.assertIsNone(store.latest_evidence()) - second = store.publish_success( - identity=AuditPublicationIdentity( - 4, - next_height, - next_block_hash, - ), - publication_floor_sequence=4, - report=self.report(block_height=next_height), - persistence=self.persistence(), - evidence={"confirmation": {"confirmed_count": 1}}, - created_at="later", - ) - self.assertTrue(second.published) - self.assertEqual( - first.envelope_path.read_bytes(), - first_envelope_bytes, - ) - self.assertTrue(second.envelope_path.exists()) - self.assertEqual( - json.loads(store.evidence_path.read_text(encoding="utf-8"))[ - "block_hash" - ], - next_block_hash, - ) - - def test_restart_repair_requires_the_fresh_durable_publication_floor(self) -> None: - identity = AuditPublicationIdentity(3, 4, BLOCK_A) - for damage in ("missing_envelope", "malformed_evidence", "both_missing"): - with self.subTest(damage=damage), tempfile.TemporaryDirectory() as tmp: - root = Path(tmp) - store = self.make_store(root) - first = store.publish_success( - identity=identity, - report=self.report(block_height=4), - persistence=self.persistence(), - evidence={"confirmation": {"confirmed_count": 1}}, - created_at="now", - ) - if damage in {"missing_envelope", "both_missing"}: - first.envelope_path.unlink() - if damage == "malformed_evidence": - store.evidence_path.write_bytes(b"not-json") - elif damage == "both_missing": - store.evidence_path.unlink() - store.close() - - restarted = self.make_store(root) - self.assertIsNone(restarted.latest_evidence()) - evidence_before = ( - restarted.evidence_path.read_bytes() - if restarted.evidence_path.exists() - else None - ) - with self.assertRaisesRegex(RuntimeError, "behind"): - restarted.publish_success( - identity=AuditPublicationIdentity(2, 5, BLOCK_B), - publication_floor_sequence=3, - report=self.report(block_height=5), - persistence=self.persistence(), - evidence={"confirmation": {"confirmed_count": 1}}, - created_at="stale", - ) - self.assertFalse( - restarted.live_envelope_path( - block_height=5, - block_hash=BLOCK_B, - ).exists() - ) - self.assertEqual( - restarted.evidence_path.read_bytes() - if restarted.evidence_path.exists() - else None, - evidence_before, - ) - repaired = restarted.publish_success( - identity=identity, - publication_floor_sequence=3, - report=self.report(block_height=4), - persistence=self.persistence(), - evidence={"confirmation": {"confirmed_count": 1}}, - created_at="repair", - ) - self.assertTrue(repaired.published) - self.assertTrue(first.envelope_path.exists()) - self.assertEqual( - json.loads(restarted.evidence_path.read_text(encoding="utf-8"))[ - "block_hash" - ], - BLOCK_A, - ) - - def test_failed_invalid_evidence_repair_preserves_bytes_and_revokes_stale_pin(self) -> None: - with tempfile.TemporaryDirectory() as tmp: - store = self.make_store(Path(tmp)) - identity = AuditPublicationIdentity(3, 4, BLOCK_A) - first = store.publish_success( - identity=identity, - report=self.report(block_height=4), - persistence=self.persistence(), - evidence={"confirmation": {"confirmed_count": 1}}, - created_at="now", - ) - latest_before = store.latest_evidence() - envelope_before = first.envelope_path.read_bytes() - corrupt_evidence = b'{"operator":"replacement"}' - store.evidence_path.write_bytes(corrupt_evidence) - original_fsync = store._fsync_directory - failed = False - - def fail_evidence_parent(parent: Path) -> None: - nonlocal failed - if parent == store.evidence_path.parent and not failed: - failed = True - raise OSError("injected evidence durability failure") - original_fsync(parent) - - with mock.patch.object( - store, - "_fsync_directory", - side_effect=fail_evidence_parent, - ), self.assertRaisesRegex(OSError, "evidence durability"): - store.publish_success( - identity=identity, - publication_floor_sequence=3, - report=self.report(block_height=4), - persistence=self.persistence(), - evidence={"confirmation": {"confirmed_count": 1}}, - created_at="later", - ) - - self.assertTrue(failed) - self.assertEqual(store.evidence_path.read_bytes(), corrupt_evidence) - self.assertEqual(first.envelope_path.read_bytes(), envelope_before) - self.assertIsNotNone(latest_before) - self.assertIsNone(store.latest_evidence()) - self.assertIsNone(store._current_envelope) - self.assertIsNone(store._current_identity) - - def test_exact_replay_reuses_nonidentity_global_stats_after_restart(self) -> None: - with tempfile.TemporaryDirectory() as tmp: - root = Path(tmp) - identity = AuditPublicationIdentity(3, 4, BLOCK_A) - store = self.make_store(root) - store.publish_success( - identity=identity, - report=self.report(block_height=4), - persistence={**self.persistence(), "share_count": 10}, - evidence={ - "confirmation": {"confirmed_count": 1}, - "accepted_share_count": 10, - "distinct_miner_count": 2, - "job_share_count": 3, - }, - created_at="now", - ) - restarted = self.make_store(root) - replay = restarted.publish_success( - identity=identity, - report=self.report(block_height=4), - persistence={**self.persistence(), "share_count": 99}, - evidence={ - "confirmation": {"confirmed_count": 1}, - "accepted_share_count": 99, - "distinct_miner_count": 20, - "job_share_count": 3, - }, - created_at="later", - ) - self.assertFalse(replay.published) - self.assertEqual(replay.evidence["accepted_share_count"], 10) - self.assertEqual(replay.evidence["distinct_miner_count"], 2) - self.assertEqual(replay.evidence["job_share_count"], 3) - self.assertEqual(replay.evidence["persistence"]["share_count"], 10) # type: ignore[index] - with self.assertRaisesRegex(RuntimeError, "replay payload conflict"): - restarted.publish_success( - identity=identity, - report=self.report(block_height=4), - persistence={**self.persistence(), "share_count": 100}, - evidence={ - "confirmation": {"confirmed_count": 1}, - "accepted_share_count": 100, - "distinct_miner_count": 21, - "job_share_count": 4, - }, - created_at="later", - ) - - def test_verification_identity_is_restart_stable_and_tamper_evident(self) -> None: - with tempfile.TemporaryDirectory() as tmp: - root = Path(tmp) - store = self.make_store(root) - report = self.report(block_height=4) - verification = AuditArtifactStore.build_verification_identity( - trust_source="configured", - trusted_writer_public_key_hex="44" * 32, - literal_sha256="99" * 32, - literal_byte_len=456, - report=report, - ) - identity = AuditPublicationIdentity(3, 4, BLOCK_A) - store.publish_success( - identity=identity, - report=report, - persistence=self.persistence(), - evidence={}, - verification_identity=verification, - created_at="now", - ) - restarted = self.make_store(root) - latest = restarted.latest_evidence() - assert latest is not None - self.assertEqual(latest["audit_verification_identity"], verification) - replay = restarted.publish_success( - identity=identity, - report=report, - persistence=self.persistence(), - evidence={}, - verification_identity=verification, - created_at="later", - ) - self.assertFalse(replay.published) - - changed_report = dict(report) - changed_report["coinbase_wtxid"] = "aa" * 32 - variants = ( - ( - report, - AuditArtifactStore.build_verification_identity( - trust_source="configured", - trusted_writer_public_key_hex="55" * 32, - literal_sha256="99" * 32, - literal_byte_len=456, - report=report, - ), - ), - ( - report, - AuditArtifactStore.build_verification_identity( - trust_source="embedded_test_only", - trusted_writer_public_key_hex="44" * 32, - literal_sha256="99" * 32, - literal_byte_len=456, - report=report, - ), - ), - ( - report, - AuditArtifactStore.build_verification_identity( - trust_source="configured", - trusted_writer_public_key_hex="44" * 32, - literal_sha256="aa" * 32, - literal_byte_len=456, - report=report, - ), - ), - ( - report, - AuditArtifactStore.build_verification_identity( - trust_source="configured", - trusted_writer_public_key_hex="44" * 32, - literal_sha256="99" * 32, - literal_byte_len=457, - report=report, - ), - ), - ( - changed_report, - AuditArtifactStore.build_verification_identity( - trust_source="configured", - trusted_writer_public_key_hex="44" * 32, - literal_sha256="99" * 32, - literal_byte_len=456, - report=changed_report, - ), - ), - ) - for variant_report, changed in variants: - with self.subTest(changed=changed), self.assertRaisesRegex( - RuntimeError, - "replay payload conflict", - ): - restarted.publish_success( - identity=identity, - report=variant_report, - persistence=self.persistence(), - evidence={}, - verification_identity=changed, - created_at="later", - ) - - def test_startup_rejects_each_verification_identity_tamper(self) -> None: - cases = ( - ("trust_source", "embedded_test_only"), - ("ledger_writer_public_key_hex", "55" * 32), - ("literal_sha256_hex", "99" * 32), - ("literal_byte_len", 999), - ("identity_sha256_hex", "88" * 32), - ) - for field, value in cases: - with self.subTest(field=field), tempfile.TemporaryDirectory() as tmp: - root = Path(tmp) - store = self.make_store(root) - store.publish_success( - identity=AuditPublicationIdentity(1, 1, BLOCK_A), - report=self.report(), - persistence=self.persistence(), - evidence={}, - created_at="now", - ) - payload = json.loads(store.evidence_path.read_text(encoding="utf-8")) - payload["audit_verification_identity"][field] = value - store.evidence_path.write_text(json.dumps(payload), encoding="utf-8") - self.assertIsNone(self.make_store(root).latest_evidence()) - - with tempfile.TemporaryDirectory() as tmp: - root = Path(tmp) - store = self.make_store(root) - store.publish_success( - identity=AuditPublicationIdentity(1, 1, BLOCK_A), - report=self.report(), - persistence=self.persistence(), - evidence={}, - created_at="now", - ) - payload = json.loads(store.evidence_path.read_text(encoding="utf-8")) - payload["audit_verification_identity"]["report"]["coinbase_txid"] = ( - "99" * 32 - ) - store.evidence_path.write_text(json.dumps(payload), encoding="utf-8") - self.assertIsNone(self.make_store(root).latest_evidence()) - - def test_higher_ordinal_replaces_same_height_block(self) -> None: - with tempfile.TemporaryDirectory() as tmp: - store = self.make_store(Path(tmp)) - store.publish_success( - identity=AuditPublicationIdentity(1, 10, BLOCK_A), - report=self.report(block_height=10), - persistence=self.persistence(), - evidence={}, - created_at="now", - ) - replacement = store.publish_success( - identity=AuditPublicationIdentity(2, 10, BLOCK_B), - report=self.report(block_height=10), - persistence=self.persistence(), - evidence={}, - created_at="later", - ) - self.assertTrue(replacement.published) - self.assertEqual(store.latest_evidence()["block_hash"], BLOCK_B) # type: ignore[index] - - def test_publication_rejects_body_for_wrong_block_or_digest(self) -> None: - with tempfile.TemporaryDirectory() as tmp: - store = self.make_store(Path(tmp)) - identity = AuditPublicationIdentity(1, 1, BLOCK_A) - for body in ( - store.body_path(BLOCK_B, DIGEST), - store.body_path(BLOCK_A, "77" * 32), - ): - with self.subTest(body=body), self.assertRaisesRegex( - RuntimeError, - "body URI does not match", - ): - store.publish_success( - identity=identity, - report=self.report(block_height=1), - persistence={ - "audit_bundle_sha256": DIGEST, - "body_uri": str(body), - }, - evidence={}, - created_at="now", - ) - - def test_restart_preserves_publication_ordinal_fence(self) -> None: - with tempfile.TemporaryDirectory() as tmp: - root = Path(tmp) - store = self.make_store(root) - store.publish_success( - identity=AuditPublicationIdentity(5, 5, BLOCK_B), - report=self.report(block_height=5), - persistence=self.persistence(), - evidence={}, - created_at="now", - ) - restarted = self.make_store(root) - stale = restarted.publish_success( - identity=AuditPublicationIdentity(4, 6, BLOCK_A), - report=self.report(), - persistence=self.persistence(), - evidence={}, - created_at="later", - ) - self.assertFalse(stale.published) - self.assertEqual(restarted.latest_evidence()["block_hash"], BLOCK_B) # type: ignore[index] - - def test_invalid_evidence_can_be_repaired_by_durable_publication(self) -> None: - with tempfile.TemporaryDirectory() as tmp: - root = Path(tmp) - (root / "evidence.json").write_text("broken", encoding="utf-8") - store = self.make_store(root) - publication = store.publish_success( - identity=AuditPublicationIdentity(1, 1, BLOCK_A), - report=self.report(), - persistence=self.persistence(), - evidence={}, - created_at="now", - ) - self.assertTrue(publication.published) - self.assertEqual(self.make_store(root).latest_evidence()["block_hash"], BLOCK_A) # type: ignore[index] - - def test_startup_rejects_coinbase_tampered_evidence(self) -> None: - with tempfile.TemporaryDirectory() as tmp: - root = Path(tmp) - store = self.make_store(root) - store.publish_success( - identity=AuditPublicationIdentity(1, 1, BLOCK_A), - report=self.report(), - persistence=self.persistence(), - evidence={}, - created_at="now", - ) - payload = json.loads(store.evidence_path.read_text(encoding="utf-8")) - payload["coinbase_txid"] = "99" * 32 - store.evidence_path.write_text(json.dumps(payload), encoding="utf-8") - self.assertIsNone(self.make_store(root).latest_evidence()) - - def test_startup_rejects_noncanonical_durable_identity_fields(self) -> None: - canonical_digest = "ab" * 32 - - def publish_fixture(root: Path, *, with_body: bool = False) -> AuditArtifactStore: - store = self.make_store(root) - persistence = self.persistence(canonical_digest) - if with_body: - persistence["body_uri"] = str( - store.body_path(BLOCK_A, canonical_digest) - ) - store.publish_success( - identity=AuditPublicationIdentity(1, 1, BLOCK_A), - report=self.report(canonical_digest), - persistence=persistence, - evidence={}, - created_at="now", - ) - return store - - cases = ( - ("height-string", lambda evidence, _envelope: evidence.__setitem__("block_height", "1")), - ("height-bool", lambda evidence, _envelope: evidence.__setitem__("block_height", True)), - ("sequence-string", lambda evidence, _envelope: evidence["audit_publication_identity"].__setitem__("sequence", "1")), - ("sequence-bool", lambda evidence, _envelope: evidence["audit_publication_identity"].__setitem__("sequence", True)), - ("coinbase-value-string", lambda evidence, _envelope: evidence.__setitem__("coinbase_value_sats", "1")), - ("block-hash-uppercase", lambda evidence, _envelope: evidence.__setitem__("block_hash", BLOCK_A.upper())), - ("persistence-digest-uppercase", lambda evidence, _envelope: evidence["persistence"].__setitem__("audit_bundle_sha256", canonical_digest.upper())), - ("report-hash-uppercase", lambda evidence, _envelope: evidence["audit_report"].__setitem__("audit_bundle_sha256_hex", canonical_digest.upper())), - ("envelope-height-string", lambda _evidence, envelope: envelope.__setitem__("block_height", "1")), - ("envelope-value-bool", lambda _evidence, envelope: envelope.__setitem__("coinbase_value_sats", True)), - ) - for name, mutate in cases: - with self.subTest(name=name), tempfile.TemporaryDirectory() as tmp: - root = Path(tmp) - store = publish_fixture(root) - evidence = json.loads(store.evidence_path.read_text(encoding="utf-8")) - envelope_path = Path(evidence["audit_bundle_path"]) - envelope = json.loads(envelope_path.read_text(encoding="utf-8")) - mutate(evidence, envelope) - store.evidence_path.write_text(json.dumps(evidence), encoding="utf-8") - envelope_path.write_text(json.dumps(envelope), encoding="utf-8") - self.assertIsNone(self.make_store(root).latest_evidence()) - - with tempfile.TemporaryDirectory() as tmp: - root = Path(tmp) - store = publish_fixture(root, with_body=True) - evidence = json.loads(store.evidence_path.read_text(encoding="utf-8")) - envelope_path = Path(evidence["audit_bundle_path"]) - envelope = json.loads(envelope_path.read_text(encoding="utf-8")) - (root / "sub").mkdir() - body_name = Path(evidence["persistence"]["body_uri"]).name - noncanonical = str(root / "sub" / ".." / body_name) - evidence["persistence"]["body_uri"] = noncanonical - envelope["body_uri"] = noncanonical - store.evidence_path.write_text(json.dumps(evidence), encoding="utf-8") - envelope_path.write_text(json.dumps(envelope), encoding="utf-8") - self.assertIsNone(self.make_store(root).latest_evidence()) - - with tempfile.TemporaryDirectory() as tmp: - root = Path(tmp) - store = publish_fixture(root) - evidence = json.loads(store.evidence_path.read_text(encoding="utf-8")) - original_cwd = Path.cwd() - try: - os.chdir(root) - evidence["audit_bundle_path"] = Path( - evidence["audit_bundle_path"] - ).name - store.evidence_path.write_text(json.dumps(evidence), encoding="utf-8") - self.assertIsNone(self.make_store(root).latest_evidence()) - finally: - os.chdir(original_cwd) - - def test_prune_failure_does_not_undo_durable_publication(self) -> None: - with tempfile.TemporaryDirectory() as tmp: - store = self.make_store(Path(tmp)) - with mock.patch.object(store, "prune_best_effort", side_effect=OSError("prune")): - result = store.publish_success( - identity=AuditPublicationIdentity(1, 1, BLOCK_A), - report=self.report(block_height=1), - persistence=self.persistence(), - evidence={}, - created_at="now", - ) - self.assertTrue(result.published) - self.assertEqual(store.latest_evidence()["block_hash"], BLOCK_A) # type: ignore[index] - - def test_evidence_failure_preserves_old_reference_and_skips_prune(self) -> None: - with tempfile.TemporaryDirectory() as tmp: - store = self.make_store(Path(tmp), live_bundle_retention=0) - store.publish_success( - identity=AuditPublicationIdentity(1, 1, BLOCK_A), - report=self.report(), - persistence=self.persistence(), - evidence={"audit_report": self.report(), "persistence": self.persistence()}, - created_at="old", - ) - old = store.evidence_path.read_bytes() - original = store._write_mutable_json - - def fail_evidence(path: Path, *args: object, **kwargs: object) -> object: - if path == store.evidence_path: - raise OSError("evidence") - return original(path, *args, **kwargs) - - with mock.patch.object(store, "_write_mutable_json", side_effect=fail_evidence): - with self.assertRaises(OSError): - store.publish_success( - identity=AuditPublicationIdentity(2, 2, BLOCK_B), - report=self.report(block_height=2), - persistence=self.persistence(), - evidence={"audit_report": self.report(), "persistence": self.persistence()}, - created_at="new", - ) - self.assertEqual(store.evidence_path.read_bytes(), old) - self.assertTrue( - store.live_envelope_path(block_height=1, block_hash=BLOCK_A).exists() - ) - - def test_publication_fsyncs_each_parent_in_commit_order(self) -> None: - with tempfile.TemporaryDirectory() as tmp: - base = Path(tmp) - root = base / "artifacts" - evidence_path = base / "state" / "evidence.json" - store = self.make_store(root, evidence_path=evidence_path) - fsync_parents: list[Path] = [] - real_fsync = store._fsync_directory - - def record(parent: Path) -> None: - fsync_parents.append(parent) - real_fsync(parent) - - with mock.patch.object( - store, - "_fsync_directory", - side_effect=record, - ): - store.publish_success( - identity=AuditPublicationIdentity(1, 1, BLOCK_A), - report=self.report(), - persistence=self.persistence(), - evidence={}, - created_at="now", - ) - - self.assertGreaterEqual(len(fsync_parents), 2) - self.assertEqual(fsync_parents[:2], [root.resolve(), evidence_path.parent.resolve()]) - - def test_second_parent_fsync_failure_preserves_recoverable_state(self) -> None: - with tempfile.TemporaryDirectory() as tmp: - base = Path(tmp) - root = base / "artifacts" - evidence_path = base / "state" / "evidence.json" - store = self.make_store(root, evidence_path=evidence_path) - fsync_parents: list[Path] = [] - real_fsync = store._fsync_directory - failed = False - - def fail_evidence_parent_once(parent: Path) -> None: - nonlocal failed - fsync_parents.append(parent) - if parent == evidence_path.parent.resolve() and not failed: - failed = True - raise OSError("evidence parent fsync") - real_fsync(parent) - - envelope = store.live_envelope_path( - block_height=1, - block_hash=BLOCK_A, - ) - with mock.patch.object( - store, - "_fsync_directory", - side_effect=fail_evidence_parent_once, - ), self.assertRaisesRegex(OSError, "evidence parent fsync"): - store.publish_success( - identity=AuditPublicationIdentity(1, 1, BLOCK_A), - report=self.report(), - persistence=self.persistence(), - evidence={}, - created_at="now", - ) - - self.assertEqual( - fsync_parents[:2], - [root.resolve(), evidence_path.parent.resolve()], - ) - self.assertFalse(envelope.exists()) - self.assertFalse(evidence_path.exists()) - restarted = self.make_store(root, evidence_path=evidence_path) - self.assertIsNone(restarted.latest_evidence()) - - def test_evidence_failure_preserves_competing_envelope_replacement(self) -> None: - with tempfile.TemporaryDirectory() as tmp: - store = self.make_store(Path(tmp), live_bundle_retention=0) - store.publish_success( - identity=AuditPublicationIdentity(1, 1, BLOCK_A), - report=self.report(), - persistence=self.persistence(), - evidence={}, - created_at="old", - ) - competitor_path = store.live_envelope_path( - block_height=2, - block_hash=BLOCK_B, - ) - original = store._write_mutable_json - - def replace_then_fail( - path: Path, - *args: object, - **kwargs: object, - ) -> object: - if path == store.evidence_path: - competitor_path.unlink() - competitor_path.write_bytes(b"competitor") - raise OSError("evidence") - return original(path, *args, **kwargs) - - with mock.patch.object( - store, - "_write_mutable_json", - side_effect=replace_then_fail, - ), self.assertRaises(OSError): - store.publish_success( - identity=AuditPublicationIdentity(2, 2, BLOCK_B), - report=self.report(block_height=2), - persistence=self.persistence(), - evidence={}, - created_at="new", - ) - - self.assertEqual(competitor_path.read_bytes(), b"competitor") - self.assertEqual(store.latest_evidence()["block_hash"], BLOCK_A) # type: ignore[index] - - def test_directory_fsync_failure_rolls_back_previous_bytes(self) -> None: - with tempfile.TemporaryDirectory() as tmp: - store = self.make_store(Path(tmp)) - path = store.root / "operator.json" - path.write_bytes(b"old") - real = store._fsync_directory - calls = 0 - - def fail_once(parent: Path) -> None: - nonlocal calls - calls += 1 - if calls == 1: - raise OSError("dir fsync") - real(parent) - - with mock.patch.object(store, "_fsync_directory", side_effect=fail_once): - with self.assertRaises(OSError): - store._write_mutable_bytes(path, b"new") - self.assertEqual(path.read_bytes(), b"old") - - def test_directory_fsync_rollback_preserves_competing_replacement(self) -> None: - with tempfile.TemporaryDirectory() as tmp: - store = self.make_store(Path(tmp)) - path = store.root / "mutable.json" - path.write_bytes(b"old") - real = store._fsync_directory - calls = 0 - - def swap_then_fail(parent: Path) -> None: - nonlocal calls - calls += 1 - if calls == 1: - path.unlink() - path.write_bytes(b"competitor") - raise OSError("dir fsync") - real(parent) - - with mock.patch.object(store, "_fsync_directory", side_effect=swap_then_fail): - with self.assertRaises(OSError): - store._write_mutable_bytes(path, b"new") - self.assertEqual(path.read_bytes(), b"competitor") - - def test_identity_cleanup_never_moves_an_existing_replacement(self) -> None: - with tempfile.TemporaryDirectory() as tmp: - root = Path(tmp) - store = self.make_store(root) - path = root / "owned.json" - path.write_bytes(b"owned-a") - owned = _FileIdentity.from_stat(path.stat()) - path.unlink() - path.write_bytes(b"competitor-b") - replacement = path.stat() - # Force the Linux ABA case even on filesystems that do not - # immediately reuse the unlinked inode in this test. - owned = dataclass_replace( - owned, - device=replacement.st_dev, - inode=replacement.st_ino, - ) - with mock.patch.object( - store, - "_owned_replace", - side_effect=AssertionError("replacement must not move"), - ): - self.assertFalse(store._remove_identity_safe(path, owned)) - - self.assertEqual(path.read_bytes(), b"competitor-b") - quarantines = list(root.glob(".owned.json.*.cleanup")) - self.assertEqual(quarantines, []) - - def test_identity_cleanup_never_relocates_nonregular_replacement(self) -> None: - with tempfile.TemporaryDirectory() as tmp: - root = Path(tmp) - store = self.make_store(root) - path = root / "owned.json" - path.write_bytes(b"owned") - owned = _FileIdentity.from_stat(path.stat()) - path.unlink() - path.mkdir() - - self.assertFalse(store._remove_identity_safe(path, owned)) - self.assertTrue(path.is_dir()) - self.assertEqual(list(root.glob(".owned.json.*.cleanup")), []) - - path.rmdir() - target = root / "operator" - target.write_bytes(b"operator") - path.symlink_to(target) - self.assertFalse(store._remove_identity_safe(path, owned)) - self.assertTrue(path.is_symlink()) - self.assertEqual(path.read_bytes(), b"operator") - - def test_atomic_stage_faults_preserve_target_and_clean_owned_temps(self) -> None: - with tempfile.TemporaryDirectory() as tmp: - store = self.make_store(Path(tmp)) - path = store.root / "mutable.json" - path.write_bytes(b"old") - with mock.patch("os.fsync", side_effect=OSError("file fsync")): - with self.assertRaises(OSError): - store._write_mutable_bytes(path, b"new") - self.assertEqual(path.read_bytes(), b"old") - self.assertEqual(list(store.root.glob(".*.tmp")), []) - - real_replace = os.replace - - def fail_publish( - source: object, - target: object, - **kwargs: object, - ) -> None: - target_path = Path(target) - if kwargs.get("dst_dir_fd") == store._root_fd: - target_path = store.root / target_path - if target_path == path: - raise OSError("replace") - real_replace(source, target, **kwargs) - - with mock.patch("os.replace", side_effect=fail_publish): - with self.assertRaises(OSError): - store._write_mutable_bytes(path, b"new") - self.assertEqual(path.read_bytes(), b"old") - self.assertEqual(list(store.root.glob(".*.tmp")), []) - - token = "66" * 16 - stage = store.root / f".{path.name}.{token}.tmp" - stage.write_bytes(b"preexisting") - with mock.patch( - "lab.prism.audit_artifacts.uuid.uuid4", - return_value=mock.Mock(hex=token), - ): - with self.assertRaises(FileExistsError): - store._write_mutable_bytes(path, b"new") - self.assertEqual(stage.read_bytes(), b"preexisting") - self.assertEqual(path.read_bytes(), b"old") - - def test_immutable_primitive_rejects_arbitrary_owned_root_targets(self) -> None: - with tempfile.TemporaryDirectory() as tmp: - store = self.make_store(Path(tmp)) - target = store.root / "operator-selected.json" - with self.assertRaisesRegex(RuntimeError, "is not an owned body"): - store._write_immutable_bytes(target, b"payload") - self.assertFalse(target.exists()) - - def test_immutable_publish_fsync_failure_is_recoverable_and_idempotent(self) -> None: - with tempfile.TemporaryDirectory() as tmp: - store = self.make_store(Path(tmp)) - target = store.body_path(BLOCK_A, DIGEST) - real_fsync = store._fsync_directory - calls = 0 - - def fail_once(parent: Path) -> None: - nonlocal calls - calls += 1 - if calls == 1: - raise OSError("dir fsync") - real_fsync(parent) - - with mock.patch.object( - store, - "_fsync_directory", - side_effect=fail_once, - ), self.assertRaisesRegex(OSError, "dir fsync"): - store._write_immutable_bytes(target, b"payload") - self.assertFalse(target.exists()) - self.assertEqual(list(store.root.glob(".*.tmp")), []) - - store._write_immutable_bytes(target, b"payload") - with mock.patch.object( - store, - "_fsync_directory", - wraps=real_fsync, - ) as fsync: - store._write_immutable_bytes(target, b"payload") - fsync.assert_called_once_with(target.parent) - with self.assertRaisesRegex(RuntimeError, "does not match"): - store._write_immutable_bytes(target, b"different") - self.assertEqual(target.read_bytes(), b"payload") - - def test_compatibility_none_override_is_stable_and_defensive(self) -> None: - with tempfile.TemporaryDirectory() as tmp: - root = Path(tmp) - store = self.make_store(root) - store.publish_success( - identity=AuditPublicationIdentity(1, 1, BLOCK_A), - report=self.report(), - persistence=self.persistence(), - evidence={"nested": {"value": 1}}, - created_at="now", - ) - store.set_latest_evidence_for_compatibility(None) - self.assertIsNone(store.latest_evidence()) - seed = {"nested": {"value": 2}} - store.set_latest_evidence_for_compatibility(seed) - seed["nested"]["value"] = 3 - first = store.latest_evidence() - assert first is not None - self.assertEqual(first["nested"]["value"], 2) - first["nested"]["value"] = 4 - self.assertEqual(store.latest_evidence()["nested"]["value"], 2) # type: ignore[index] - - def test_compact_body_and_segment_headers_are_identity_bound(self) -> None: - with tempfile.TemporaryDirectory() as tmp: - store = self.make_store( - Path(tmp), - canonicalizer=lambda value: json.dumps( - value, - separators=(",", ":"), - ).encode(), - share_segment_size=2, - ) - logical = {"schema": "example", "shares": [{"share_seq": 1}]} - digest = hashlib.sha256( - json.dumps(logical, separators=(",", ":")).encode() - ).hexdigest() - body_path = store.body_path(BLOCK_A, digest) - wrapper = { - "schema": "qbit.prism.audit-body-ref.v1", - "block_hash": BLOCK_B, - "audit_bundle_sha256": digest, - "audit_bundle_schema": "example", - "share_count": 1, - "share_segment_size": 2, - "shares_key_index": 1, - "bundle_without_shares": {"schema": "example"}, - "share_parts": [{ - "kind": "inline", - "first_share_seq": 1, - "last_share_seq": 1, - "share_count": 1, - "shares": [{"share_seq": 1}], - }], - } - body_path.write_bytes(store.storage_json_bytes(wrapper)) - with self.assertRaisesRegex(RuntimeError, "identity mismatch"): - store.read_external_body(str(body_path), expected_sha256=digest) - - segment = { - "schema": "qbit.prism.audit-share-segment.v1", - "first_share_seq": 1, - "last_share_seq": 2, - "share_count": 2, - "shares": [{"share_seq": 1}], - } - segment_bytes = store.storage_json_bytes(segment) - segment_digest = hashlib.sha256(segment_bytes).hexdigest() - segment_path = store.root / ( - f"prism-audit-share-segment-1-2-{segment_digest}.json" - ) - segment_path.write_bytes(segment_bytes) - with self.assertRaisesRegex(RuntimeError, "header mismatch"): - store.read_audit_share_segment( - { - "kind": "segment", - "first_share_seq": 1, - "last_share_seq": 2, - "share_count": 2, - "sha256": segment_digest, - "body_uri": str(segment_path), - }, - parent_body_uri=str(body_path), - ) - - def test_memory_ledger_reactivation_preserves_publication_ordinal(self) -> None: - ledger = SingleWriterShareLedger() - self.assertEqual(ledger.audit_publication_sequence_floor(), 0) - - def persist(block_hash: str, height: int) -> None: - ledger.persist_accepted_block( - block_hash=block_hash, - block_height=height, - parent_hash=BLOCK_B, - final_bundle={}, - audit_report={}, - ) - - persist(BLOCK_A, 1) - self.assertEqual( - ledger.pool_block_state(block_hash=BLOCK_A)["chain_state"], # type: ignore[index] - "prepared", - ) - first = ledger.confirm_accepted_block(block_hash=BLOCK_A, active_tip_height=1) - replay = ledger.confirm_accepted_block(block_hash=BLOCK_A, active_tip_height=1) - self.assertEqual(ledger.audit_publication_sequence_floor(), 1) - self.assertEqual( - first["audit_publication_sequence"], - replay["audit_publication_sequence"], - ) - persist(BLOCK_A, 1) - self.assertEqual( - ledger.pool_block_state(block_hash=BLOCK_A)["chain_state"], # type: ignore[index] - "confirmed", - ) - self.assertEqual( - ledger.mark_pool_block_inactive(block_hash=BLOCK_A, active_tip_height=2)[ - "inactive_count" - ], - 1, - ) - persist(BLOCK_A, 1) - inactive_state = ledger.pool_block_state(block_hash=BLOCK_A) - self.assertEqual(inactive_state["chain_state"], "inactive") # type: ignore[index] - self.assertEqual( - inactive_state["audit_publication_sequence"], # type: ignore[index] - first["audit_publication_sequence"], - ) - wrong_height = ledger.reactivate_pool_block( - block_hash=BLOCK_A, - active_tip_height=0, - ) - self.assertEqual(wrong_height["reactivated_count"], 0) - self.assertNotIn("audit_publication_sequence", wrong_height) - reactivated = ledger.reactivate_pool_block( - block_hash=BLOCK_A, - active_tip_height=1, - ) - self.assertEqual( - reactivated["audit_publication_sequence"], - first["audit_publication_sequence"], - ) - self.assertEqual(ledger.audit_publication_sequence_floor(), 1) - replay_reactivation = ledger.reactivate_pool_block( - block_hash=BLOCK_A, - active_tip_height=1, - ) - self.assertEqual(replay_reactivation["reactivated_count"], 0) - self.assertNotIn("audit_publication_sequence", replay_reactivation) - self.assertEqual( - ledger.mark_pool_block_inactive( - block_hash=BLOCK_A, - active_tip_height=2, - )["inactive_count"], - 1, - ) - self.assertEqual( - ledger.reverse_immature_block( - block_hash=BLOCK_A, - active_tip_height=2, - )["reversed_count"], - 1, - ) - persist(BLOCK_A, 1) - reversed_state = ledger.pool_block_state(block_hash=BLOCK_A) - self.assertEqual(reversed_state["chain_state"], "reversed") # type: ignore[index] - self.assertEqual(reversed_state["maturity_state"], "reversed") # type: ignore[index] - self.assertEqual( - reversed_state["audit_publication_sequence"], # type: ignore[index] - reactivated["audit_publication_sequence"], - ) - # A reversed row is terminally disposed: confirmation reports the - # superseded disposition (-1) instead of the unexplained 0. - self.assertEqual( - ledger.confirm_accepted_block( - block_hash=BLOCK_A, - active_tip_height=1, - ), - {"backend": "memory", "confirmed_count": -1}, - ) - self.assertEqual( - ledger.reactivate_pool_block( - block_hash=BLOCK_A, - active_tip_height=1, - ), - {"backend": "memory", "reactivated_count": 0}, - ) - persist(BLOCK_B, 2) - next_publication = ledger.confirm_accepted_block( - block_hash=BLOCK_B, - active_tip_height=2, - ) - self.assertEqual(next_publication["audit_publication_sequence"], 2) - self.assertEqual(ledger.audit_publication_sequence_floor(), 2) - with self.assertRaisesRegex(RuntimeError, "mature pool block"): - ledger.reverse_immature_block( - block_hash=BLOCK_B, - active_tip_height=1002, - ) - rejected_hash = "cc" * 32 - persist(rejected_hash, 3) - self.assertEqual( - ledger.reject_prepared_block( - block_hash=rejected_hash, - active_tip_height=3, - )["rejected_count"], - 1, - ) - self.assertEqual( - ledger.reject_prepared_block( - block_hash=rejected_hash, - active_tip_height=3, - )["rejected_count"], - 0, - ) - rejected_state = ledger.pool_block_state(block_hash=rejected_hash) - self.assertEqual(rejected_state["chain_state"], "rejected") # type: ignore[index] - self.assertEqual(rejected_state["maturity_state"], "reversed") # type: ignore[index] - - def test_memory_ledger_rejects_unknown_wrong_height_and_inactive_confirmation(self) -> None: - ledger = SingleWriterShareLedger() - self.assertEqual( - ledger.confirm_accepted_block( - block_hash=BLOCK_A, - active_tip_height=1, - )["confirmed_count"], - 0, - ) - ledger.persist_accepted_block( - block_hash=BLOCK_A, - block_height=1, - parent_hash=BLOCK_B, - final_bundle={}, - audit_report={}, - ) - self.assertEqual( - ledger.confirm_accepted_block( - block_hash=BLOCK_A, - active_tip_height=2, - )["confirmed_count"], - 0, - ) - ledger.confirm_accepted_block(block_hash=BLOCK_A, active_tip_height=1) - self.assertEqual( - ledger.mark_pool_block_inactive( - block_hash=BLOCK_A, - active_tip_height=2, - )["inactive_count"], - 1, - ) - self.assertEqual( - ledger.mark_pool_block_inactive( - block_hash=BLOCK_A, - active_tip_height=2, - )["inactive_count"], - 0, - ) - # An inactive row is terminally disposed for this confirmation - # attempt: it reports superseded (-1), not the unexplained 0. - inactive_confirmation = ledger.confirm_accepted_block( - block_hash=BLOCK_A, - active_tip_height=1, - ) - self.assertEqual(inactive_confirmation["confirmed_count"], -1) - self.assertNotIn("audit_publication_sequence", inactive_confirmation) - - def test_invalid_evidence_disables_live_prune_but_not_candidate_prune(self) -> None: - with tempfile.TemporaryDirectory() as tmp: - root = Path(tmp) - (root / "evidence.json").write_text("not-json", encoding="utf-8") - live = root / f"prism-live-audit-bundle-1-{BLOCK_A}.json" - live.write_text("{}", encoding="utf-8") - candidate = root / f"prism-live-audit-bundle-candidate-{BLOCK_A}.json" - candidate.write_text("{}", encoding="utf-8") - store = self.make_store( - root, - live_bundle_retention=0, - candidate_retention_seconds=0, - ) - result = store.prune_best_effort() - self.assertTrue(live.exists()) - self.assertFalse(candidate.exists()) - self.assertEqual(result.live_removed, 0) - self.assertEqual(result.candidate_removed, 1) - - def test_legacy_evidence_requires_durable_identity_upgrade(self) -> None: - with tempfile.TemporaryDirectory() as tmp: - root = Path(tmp) - store = self.make_store(root) - envelope = store.live_envelope_path(block_height=2, block_hash=BLOCK_B) - envelope.write_text( - json.dumps( - { - "schema": LIVE_ENVELOPE_SCHEMA, - "block_hash": BLOCK_B, - "block_height": 2, - "audit_bundle_sha256": DIGEST, - "body_uri": "", - "body_filename": None, - "coinbase_txid": "22" * 32, - "coinbase_manifest_sha256": "33" * 32, - "coinbase_tx_hex": "00", - "coinbase_value_sats": 1, - } - ), - encoding="utf-8", - ) - legacy = { - "schema": LIVE_EVIDENCE_SCHEMA, - "block_hash": BLOCK_B, - "block_height": 2, - "audit_bundle_path": str(envelope), - "audit_report": self.report(block_height=2), - "persistence": self.persistence(), - "coinbase_txid": "22" * 32, - "coinbase_manifest_sha256_hex": "33" * 32, - "coinbase_tx_hex": "00", - "coinbase_value_sats": 1, - } - store.evidence_path.write_text(json.dumps(legacy), encoding="utf-8") - restarted = self.make_store(root) - with self.assertRaisesRegex(RuntimeError, "validated publication identity"): - restarted.publish_success( - identity=AuditPublicationIdentity(10, 1, BLOCK_A), - report=self.report(), - persistence=self.persistence(), - evidence={"audit_report": self.report(), "persistence": self.persistence()}, - created_at="later", - ) - with restarted.publication_order_guard(): - restarted.adopt_legacy_publication_identity( - AuditPublicationIdentity(20, 2, BLOCK_B), - publication_floor_sequence=20, - ) - with self.assertRaisesRegex(RuntimeError, "never exact-replay"): - restarted.publish_success( - identity=AuditPublicationIdentity(10, 1, BLOCK_A), - report=self.report(block_height=1), - persistence=self.persistence(), - evidence={ - "audit_report": self.report(), - "persistence": self.persistence(), - }, - created_at="later", - ) - - # The disk marker never grants order or pin authority by itself. - second_process = self.make_store(root) - self.assertIsNone(second_process.latest_evidence()) - self.assertEqual(second_process.publication_sequence_floor(), 0) - self.assertEqual( - second_process.legacy_evidence_identity(), - AuditPublicationIdentity(20, 2, BLOCK_B), - ) - with second_process.publication_order_guard(): - second_process.adopt_legacy_publication_identity( - AuditPublicationIdentity(20, 2, BLOCK_B), - publication_floor_sequence=20, - ) - self.assertEqual(second_process.publication_sequence_floor(), 20) - self.assertEqual( - second_process.latest_evidence()["block_hash"], # type: ignore[index] - BLOCK_B, - ) - for stale_identity in ( - AuditPublicationIdentity(20, 2, BLOCK_B), - AuditPublicationIdentity(19, 3, BLOCK_A), - ): - with self.subTest(stale_identity=stale_identity), self.assertRaisesRegex( - RuntimeError, - "never exact-replay", - ): - second_process.publish_success( - identity=stale_identity, - report=self.report(block_height=stale_identity.block_height), - persistence=self.persistence(), - evidence={}, - created_at="stale", - ) - repair = second_process.publish_success( - identity=AuditPublicationIdentity(21, 3, BLOCK_A), - report=self.report(block_height=3), - persistence=self.persistence(), - evidence={}, - created_at="repair", - ) - self.assertTrue(repair.published) - - def test_legacy_proof_token_is_revoked_by_peer_inode_replacement(self) -> None: - with tempfile.TemporaryDirectory() as tmp: - root = Path(tmp) - seed = self.make_store(root) - envelope = seed.live_envelope_path( - block_height=2, - block_hash=BLOCK_B, - ) - envelope.write_text( - json.dumps( - { - "schema": LIVE_ENVELOPE_SCHEMA, - "block_hash": BLOCK_B, - "block_height": 2, - "audit_bundle_sha256": DIGEST, - "body_uri": "", - "body_filename": None, - "coinbase_txid": "22" * 32, - "coinbase_manifest_sha256": "33" * 32, - "coinbase_tx_hex": "00", - "coinbase_value_sats": 1, - } - ), - encoding="utf-8", - ) - seed.evidence_path.write_text( - json.dumps( - { - "schema": LIVE_EVIDENCE_SCHEMA, - "block_hash": BLOCK_B, - "block_height": 2, - "audit_bundle_path": str(envelope), - "audit_report": self.report(block_height=2), - "persistence": self.persistence(), - "coinbase_txid": "22" * 32, - "coinbase_manifest_sha256_hex": "33" * 32, - "coinbase_tx_hex": "00", - "coinbase_value_sats": 1, - } - ), - encoding="utf-8", - ) - seed.close() - identity = AuditPublicationIdentity(20, 2, BLOCK_B) - proven = self.make_store(root) - with proven.publication_order_guard(): - proven.adopt_legacy_publication_identity( - identity, - publication_floor_sequence=20, - ) - self.assertIsNotNone(proven._legacy_proof_token) - self.assertEqual(proven.publication_sequence_floor(), 20) - original_evidence_inode = _FileIdentity.from_stat( - proven.evidence_path.stat() - ) - - peer = self.make_store(root) - with peer.publication_order_guard(): - peer.adopt_legacy_publication_identity( - identity, - publication_floor_sequence=20, - ) - self.assertFalse( - original_evidence_inode.matches(proven.evidence_path.stat()) - ) - - proven.prune_best_effort() - self.assertIsNone(proven._legacy_proof_token) - self.assertIsNone(proven.latest_evidence()) - self.assertEqual(proven.publication_sequence_floor(), 0) - self.assertEqual(proven.legacy_evidence_identity(), identity) - - def test_unprovable_legacy_evidence_is_repairable(self) -> None: - with tempfile.TemporaryDirectory() as tmp: - store = self.make_store(Path(tmp)) - store._evidence_state = "legacy" - store._latest_evidence = {"block_hash": BLOCK_A, "block_height": 1} - store._current_identity = AuditPublicationIdentity(0, 1, BLOCK_A) - store.invalidate_unprovable_legacy_evidence() - repaired = store.publish_success( - identity=AuditPublicationIdentity(2, 2, BLOCK_B), - report=self.report(block_height=2), - persistence=self.persistence(), - evidence={}, - created_at="now", - ) - self.assertTrue(repaired.published) - - def test_disk_legacy_invalidation_is_sticky_until_durable_repair(self) -> None: - with tempfile.TemporaryDirectory() as tmp: - root = Path(tmp) - store = self.make_store(root, live_bundle_retention=0) - envelope = store.live_envelope_path(block_height=2, block_hash=BLOCK_B) - envelope.write_text( - json.dumps( - { - "schema": LIVE_ENVELOPE_SCHEMA, - "block_hash": BLOCK_B, - "block_height": 2, - "audit_bundle_sha256": DIGEST, - "body_uri": "", - "body_filename": None, - "coinbase_txid": "22" * 32, - "coinbase_manifest_sha256": "33" * 32, - "coinbase_tx_hex": "00", - "coinbase_value_sats": 1, - } - ), - encoding="utf-8", - ) - legacy = { - "schema": LIVE_EVIDENCE_SCHEMA, - "block_hash": BLOCK_B, - "block_height": 2, - "audit_bundle_path": str(envelope), - "audit_report": self.report(block_height=2), - "persistence": self.persistence(), - "coinbase_txid": "22" * 32, - "coinbase_manifest_sha256_hex": "33" * 32, - "coinbase_tx_hex": "00", - "coinbase_value_sats": 1, - } - store.evidence_path.write_text(json.dumps(legacy), encoding="utf-8") - restarted = self.make_store(root, live_bundle_retention=0) - restarted.invalidate_unprovable_legacy_evidence() - self.assertIsNone(restarted.latest_evidence()) - self.assertEqual(restarted.prune_best_effort().live_removed, 0) - repaired = restarted.publish_success( - identity=AuditPublicationIdentity(21, 3, BLOCK_A), - report=self.report(block_height=3), - persistence=self.persistence(), - evidence={}, - created_at="repair", - ) - self.assertTrue(repaired.published) - self.assertEqual(restarted.latest_evidence()["block_hash"], BLOCK_A) # type: ignore[index] - - def test_retention_preserves_current_and_unowned_entries(self) -> None: - with tempfile.TemporaryDirectory() as tmp: - root = Path(tmp) - store = self.make_store(root, live_bundle_retention=0) - publication = store.publish_success( - identity=AuditPublicationIdentity(1, 1, BLOCK_A), - report=self.report(), - persistence=self.persistence(), - evidence={"audit_report": self.report(), "persistence": self.persistence()}, - created_at="now", - ) - lookalike = root / f"prism-live-audit-bundle-1-{BLOCK_B}.json.bak" - lookalike.write_text("operator", encoding="utf-8") - link = root / f"prism-live-audit-bundle-2-{BLOCK_B}.json" - link.symlink_to(lookalike) - store.prune_best_effort() - self.assertTrue(publication.envelope_path.exists()) - self.assertTrue(lookalike.exists()) - self.assertTrue(link.is_symlink()) - - def test_retention_ties_pin_active_candidate_and_never_remove_bodies(self) -> None: - with tempfile.TemporaryDirectory() as tmp: - root = Path(tmp) - store = self.make_store( - root, - live_bundle_retention=2, - candidate_retention_seconds=0, - ) - current = store.publish_success( - identity=AuditPublicationIdentity(1, 0, BLOCK_A), - report=self.report(block_height=0), - persistence=self.persistence(), - evidence={}, - created_at="current", - ) - first = store.live_envelope_path(block_height=1, block_hash=BLOCK_A) - second = store.live_envelope_path(block_height=2, block_hash=BLOCK_B) - first.write_text("{}", encoding="utf-8") - second.write_text("{}", encoding="utf-8") - os.utime(first, ns=(1, 1)) - os.utime(second, ns=(1, 1)) - active = store.issue_candidate(block_hash=BLOCK_A) - active.path.write_bytes(b"active") - self.transfer_candidate(store, active) - body = store.body_path(BLOCK_A, DIGEST) - body.write_bytes(b"body") - result = store.prune_best_effort() - self.assertEqual(result.live_removed, 1) - self.assertTrue(current.envelope_path.exists()) - self.assertTrue(active.path.exists()) - self.assertTrue(body.exists()) - store.discard_candidate(active) - - def test_body_segment_and_evidence_symlinks_fail_no_follow(self) -> None: - with tempfile.TemporaryDirectory() as tmp: - root = Path(tmp) - store = self.make_store(root) - outside = root / "outside" - outside.write_text("{}", encoding="utf-8") - body = store.body_path(BLOCK_A, DIGEST) - body.symlink_to(outside) - with self.assertRaisesRegex(RuntimeError, "not retrievable"): - store.read_external_body(str(body), expected_sha256=DIGEST) - store.evidence_path.symlink_to(outside) - self.assertIsNone(self.make_store(root).latest_evidence()) - - def test_metrics_ignore_symlinks_and_classify_malformed_as_other(self) -> None: - with tempfile.TemporaryDirectory() as tmp: - root = Path(tmp) - store = self.make_store(root) - body = root / f"prism-audit-bundle-body-{BLOCK_A}-{DIGEST}.json" - body.write_bytes(b"body") - malformed = root / "prism-audit-bundle-body-operator.json" - malformed.write_bytes(b"x") - (root / f"prism-live-audit-bundle-1-{BLOCK_A}.json").symlink_to(body) - metrics = store.metrics_snapshot() - self.assertEqual(metrics["body"], {"files": 1, "bytes": 4}) - self.assertEqual(metrics["other"], {"files": 1, "bytes": 1}) - - def test_mutable_share_slot_merge_preserves_prior_range(self) -> None: - with tempfile.TemporaryDirectory() as tmp: - store = self.make_store(Path(tmp), share_segment_size=4) - uri, _digest = store.write_audit_share_segment_range( - segment_first_share_seq=1, - segment_last_share_seq=4, - first_share_seq=1, - last_share_seq=2, - shares=[{"share_seq": 1}, {"share_seq": 2}], - ) - store.write_audit_share_segment_range( - segment_first_share_seq=1, - segment_last_share_seq=4, - first_share_seq=3, - last_share_seq=4, - shares=[{"share_seq": 3}, {"share_seq": 4}], - ) - payload = json.loads(Path(uri).read_text(encoding="utf-8")) - self.assertEqual([row["share_seq"] for row in payload["shares"]], [1, 2, 3, 4]) - - def test_concurrent_mutable_share_slot_merge_is_serialized_and_lossless( - self, - ) -> None: - rows = [ - { - "share_seq": share_seq, - "worker": f"miner-{share_seq}", - "accepted": True, - } - for share_seq in range(1, 5) - ] - scenarios = ( - ("disjoint", rows[:2], rows[2:]), - ("identical_overlap", rows[:3], rows[1:]), - ) - - for scenario, first_shares, second_shares in scenarios: - with self.subTest(scenario=scenario), tempfile.TemporaryDirectory() as tmp: - store = self.make_store(Path(tmp), share_segment_size=4) - first_name = f"audit-slot-{scenario}-first" - second_name = f"audit-slot-{scenario}-second" - first_at_write = threading.Event() - release_first = threading.Event() - second_lock_attempt = threading.Event() - second_lock_acquired = threading.Event() - second_at_write = threading.Event() - write_call_lock = threading.Lock() - write_call_threads: list[str] = [] - results: dict[str, tuple[str, str]] = {} - errors: dict[str, BaseException] = {} - original_lock = store._lock - # The slot writer streams its chunks through the bounded - # mutable primitive (issue #255); gate that primitive. - original_write_mutable_chunks = store._write_mutable_chunks - - class ObservedLock: - def __enter__(self) -> "ObservedLock": - if threading.current_thread().name == second_name: - second_lock_attempt.set() - original_lock.acquire() - if threading.current_thread().name == second_name: - second_lock_acquired.set() - return self - - def __exit__( - self, - _exc_type: object, - _exc_value: object, - _traceback: object, - ) -> None: - original_lock.release() - - def gated_write(path: Path, chunks_factory: object) -> None: - thread_name = threading.current_thread().name - with write_call_lock: - write_call_threads.append(thread_name) - if thread_name == first_name: - first_at_write.set() - if not release_first.wait(timeout=5.0): - raise AssertionError("first share-slot writer was not released") - elif thread_name == second_name: - second_at_write.set() - original_write_mutable_chunks(path, chunks_factory) - - def write_range(label: str, shares: list[dict[str, object]]) -> None: - try: - results[label] = store.write_audit_share_segment_range( - segment_first_share_seq=1, - segment_last_share_seq=4, - first_share_seq=int(shares[0]["share_seq"]), - last_share_seq=int(shares[-1]["share_seq"]), - shares=shares, - ) - except BaseException as exc: - errors[label] = exc - - first_thread = threading.Thread( - target=write_range, - args=("first", first_shares), - name=first_name, - ) - second_thread = threading.Thread( - target=write_range, - args=("second", second_shares), - name=second_name, - ) - second_started = False - with mock.patch.object( - store, - "_lock", - ObservedLock(), - ), mock.patch.object( - store, - "_write_mutable_chunks", - side_effect=gated_write, - ): - first_thread.start() - try: - self.assertTrue( - first_at_write.wait(timeout=5.0), - "first writer did not reach the gated mutable write", - ) - second_thread.start() - second_started = True - self.assertTrue( - second_lock_attempt.wait(timeout=5.0), - "second writer did not attempt the store lock", - ) - self.assertTrue(first_thread.is_alive()) - self.assertTrue(second_thread.is_alive()) - self.assertFalse(second_lock_acquired.is_set()) - self.assertFalse(second_at_write.is_set()) - finally: - release_first.set() - first_thread.join(timeout=5.0) - if second_started: - second_thread.join(timeout=5.0) - - self.assertFalse(first_thread.is_alive()) - self.assertFalse(second_thread.is_alive()) - active_names = {thread.name for thread in threading.enumerate()} - self.assertNotIn(first_name, active_names) - self.assertNotIn(second_name, active_names) - self.assertEqual(errors, {}) - self.assertTrue(second_lock_acquired.is_set()) - self.assertTrue(second_at_write.is_set()) - self.assertEqual(write_call_threads, [first_name, second_name]) - - self.assertEqual(results["first"][0], results["second"][0]) - slot_path = Path(results["first"][0]) - expected_payload = store.audit_share_segment_payload( - first_share_seq=1, - last_share_seq=4, - shares=rows, - ) - expected_bytes = store.storage_json_bytes(expected_payload) - stored_bytes = slot_path.read_bytes() - self.assertEqual(stored_bytes, expected_bytes) - stored = json.loads(stored_bytes) - sequences = [int(row["share_seq"]) for row in stored["shares"]] - self.assertEqual(sequences, [1, 2, 3, 4]) - self.assertEqual(len(sequences), len(set(sequences))) - self.assertEqual(stored["shares"], rows) - - inputs = {"first": first_shares, "second": second_shares} - for label, shares in inputs.items(): - incoming_payload = store.audit_share_segment_payload( - first_share_seq=int(shares[0]["share_seq"]), - last_share_seq=int(shares[-1]["share_seq"]), - shares=shares, - ) - expected_digest = hashlib.sha256( - store.storage_json_bytes(incoming_payload) - ).hexdigest() - self.assertEqual(results[label][1], expected_digest) - - before_retry = slot_path.stat() - for label, shares in inputs.items(): - retry = store.write_audit_share_segment_range( - segment_first_share_seq=1, - segment_last_share_seq=4, - first_share_seq=int(shares[0]["share_seq"]), - last_share_seq=int(shares[-1]["share_seq"]), - shares=shares, - ) - self.assertEqual(retry, results[label]) - after_retry = slot_path.stat() - self.assertEqual(slot_path.read_bytes(), expected_bytes) - self.assertEqual( - (after_retry.st_dev, after_retry.st_ino, after_retry.st_mtime_ns), - (before_retry.st_dev, before_retry.st_ino, before_retry.st_mtime_ns), - ) - - conflicting_share = {**rows[1], "worker": "conflicting-miner"} - with self.assertRaisesRegex( - RuntimeError, - "conflicts at share_seq 2", - ): - store.write_audit_share_segment_range( - segment_first_share_seq=1, - segment_last_share_seq=4, - first_share_seq=2, - last_share_seq=2, - shares=[conflicting_share], - ) - after_conflict = slot_path.stat() - self.assertEqual(slot_path.read_bytes(), expected_bytes) - self.assertEqual( - ( - after_conflict.st_dev, - after_conflict.st_ino, - after_conflict.st_mtime_ns, - ), - (after_retry.st_dev, after_retry.st_ino, after_retry.st_mtime_ns), - ) - - -class AuditShareSegmentRecoveryTests(unittest.TestCase): - """Memo §4 recovery: store-owned merge/conflict/quarantine, ledger callback.""" - - def make_store(self, root: Path) -> AuditArtifactStore: - return AuditArtifactStore( - AuditArtifactConfig( - root=root, - evidence_path=root / "evidence.json", - share_segment_size=4, - ), - canonicalizer=lambda bundle: json.dumps( - bundle, - separators=(",", ":"), - ).encode("utf-8"), - ) - - @staticmethod - def rows(first: int, last: int) -> list[dict[str, object]]: - return [ - {"share_seq": share_seq, "worker": f"miner-{share_seq}"} - for share_seq in range(first, last + 1) - ] - - def write_range( - self, - store: AuditArtifactStore, - first: int, - last: int, - *, - loader: object = None, - shares: list[dict[str, object]] | None = None, - ) -> tuple[str, str]: - return store.write_audit_share_segment_range( - segment_first_share_seq=1, - segment_last_share_seq=4, - first_share_seq=first, - last_share_seq=last, - shares=self.rows(first, last) if shares is None else shares, - load_missing_range=loader, - ) - - def test_slot_gap_is_repaired_from_the_ledger_callback(self) -> None: - with tempfile.TemporaryDirectory() as tmp: - store = self.make_store(Path(tmp)) - uri, _digest = self.write_range(store, 1, 2) - calls: list[tuple[int, int]] = [] - - def loader(*, first_share_seq: int, last_share_seq: int) -> list[dict[str, object]]: - calls.append((first_share_seq, last_share_seq)) - return self.rows(first_share_seq, last_share_seq) - - gap_uri, _range_digest = self.write_range(store, 4, 4, loader=loader) - self.assertEqual(calls, [(3, 3)]) - self.assertEqual(gap_uri, uri) - payload = json.loads(Path(uri).read_text(encoding="utf-8")) - self.assertEqual( - [row["share_seq"] for row in payload["shares"]], - [1, 2, 3, 4], - ) - - def test_unrepairable_gap_preserves_slot_and_quarantines(self) -> None: - for loader in ( - None, - lambda **_kwargs: (_ for _ in ()).throw(RuntimeError("backfill down")), - lambda **_kwargs: [{"share_seq": 3}, {"share_seq": 9}], - ): - with self.subTest(loader=loader), tempfile.TemporaryDirectory() as tmp: - store = self.make_store(Path(tmp)) - uri, _digest = self.write_range(store, 1, 2) - slot_bytes = Path(uri).read_bytes() - incoming = self.rows(4, 4) - expected_digest = hashlib.sha256( - store.storage_json_bytes( - store.audit_share_segment_payload( - first_share_seq=4, - last_share_seq=4, - shares=incoming, - ) - ) - ).hexdigest() - fresh_uri, range_digest = self.write_range( - store, - 4, - 4, - loader=loader, - shares=incoming, - ) - # The stable slot survives byte-for-byte; the incoming range - # gets an immutable content-addressed URI and finalization - # continues. - self.assertEqual(Path(uri).read_bytes(), slot_bytes) - self.assertNotEqual(fresh_uri, uri) - self.assertEqual(range_digest, expected_digest) - self.assertIn(expected_digest, Path(fresh_uri).name) - conflicts = [ - name - for name in os.listdir(tmp) - if ".conflict-" in name - ] - self.assertEqual(len(conflicts), 1) - self.assertEqual( - (Path(tmp) / conflicts[0]).read_bytes(), - slot_bytes, - ) - - def test_conflicting_payload_still_raises(self) -> None: - with tempfile.TemporaryDirectory() as tmp: - store = self.make_store(Path(tmp)) - self.write_range(store, 1, 2) - conflicting = [{"share_seq": 2, "worker": "other-miner"}] - with self.assertRaisesRegex( - _AuditShareSegmentConflict, - "conflicts at share_seq 2", - ): - self.write_range( - store, - 2, - 2, - loader=lambda **kwargs: self.rows( - kwargs["first_share_seq"], - kwargs["last_share_seq"], - ), - shares=conflicting, - ) - - def test_quarantine_snapshot_is_idempotent_for_equal_bytes(self) -> None: - with tempfile.TemporaryDirectory() as tmp: - store = self.make_store(Path(tmp)) - uri, _digest = self.write_range(store, 1, 2) - slot_bytes = Path(uri).read_bytes() - first = store.quarantine_audit_share_segment( - Path(uri), - expected_bytes=slot_bytes, - ) - second = store.quarantine_audit_share_segment( - Path(uri), - expected_bytes=slot_bytes, - ) - self.assertEqual(first, second) - self.assertEqual(first.read_bytes(), slot_bytes) - third = store.quarantine_audit_share_segment( - Path(uri), - expected_bytes=b"different-earlier-bytes", - ) - self.assertNotEqual(third, first) - - def test_range_parts_thread_the_recovery_callback(self) -> None: - with tempfile.TemporaryDirectory() as tmp: - store = self.make_store(Path(tmp)) - self.write_range(store, 1, 2) - calls: list[tuple[int, int]] = [] - - def loader(*, first_share_seq: int, last_share_seq: int) -> list[dict[str, object]]: - calls.append((first_share_seq, last_share_seq)) - return self.rows(first_share_seq, last_share_seq) - - parts = store.audit_share_range_parts( - self.rows(4, 4), - load_missing_range=loader, - ) - self.assertEqual(calls, [(3, 3)]) - assert parts is not None - self.assertEqual(parts[0]["kind"], "segment_range") - self.assertEqual(parts[0]["first_share_seq"], 4) - - -class CanonicalAuditBundleTests(unittest.TestCase): - """Phase 2: the exact bytes behind a published digest stay retrievable.""" - - def make_store( - self, - root: Path, - *, - share_segment_size: int = 2, - counter: list[int] | None = None, - **kwargs: object, - ) -> AuditArtifactStore: - def canonicalizer(bundle: dict[str, object]) -> bytes: - if counter is not None: - counter.append(1) - return json.dumps(bundle, separators=(",", ":")).encode("utf-8") - - return AuditArtifactStore( - AuditArtifactConfig( - root=root, - evidence_path=root / "evidence.json", - share_segment_size=share_segment_size, - **kwargs, # type: ignore[arg-type] - ), - canonicalizer=canonicalizer, - ) - - @staticmethod - def bundle(share_count: int = 4) -> dict[str, object]: - return { - "schema": "example", - "shares": [ - {"share_seq": share_seq, "worker": f"miner-{share_seq}"} - for share_seq in range(1, share_count + 1) - ], - } - - @staticmethod - def canonical(bundle: dict[str, object]) -> bytes: - return json.dumps(bundle, separators=(",", ":")).encode("utf-8") - - def digest(self, bundle: dict[str, object]) -> str: - return hashlib.sha256(self.canonical(bundle)).hexdigest() - - def publish( - self, - store: AuditArtifactStore, - bundle: dict[str, object], - *, - block_hash: str = BLOCK_A, - ) -> tuple[str, str]: - digest = self.digest(bundle) - body_uri = str(store.body_path(block_hash, digest)) - published = store.prepare_external_audit_body( - {"block_hash": block_hash, "audit_bundle_sha256": digest}, - bundle, - body_uri=body_uri, - ) - assert published is not None - return published, digest - - def test_publication_verifies_the_digest_before_any_store_side_effect(self) -> None: - with tempfile.TemporaryDirectory() as tmp: - root = Path(tmp) - store = self.make_store(root) - bundle = self.bundle() - wrong = "cc" * 32 - with self.assertRaisesRegex(RuntimeError, "sha256 mismatch"): - store.write_canonical_audit_bundle( - BLOCK_A, - wrong, - self.canonical(bundle), - ) - self.assertEqual( - list(root.glob("prism-audit-bundle-canonical-*")), - [], - ) - - def test_publication_is_deterministic_reproducible_and_idempotent(self) -> None: - with tempfile.TemporaryDirectory() as tmp: - root = Path(tmp) - store = self.make_store(root) - bundle = self.bundle() - digest = self.digest(bundle) - path = store.write_canonical_audit_bundle( - BLOCK_A, - digest, - self.canonical(bundle), - ) - self.assertEqual( - path.name, - f"prism-audit-bundle-canonical-{BLOCK_A}-{digest}.json.gz", - ) - first = path.read_bytes() - # mtime=0 and no FNAME field keep the header a constant. - self.assertEqual(first[:10].hex(), "1f8b08000000000002ff") - self.assertEqual(gzip.decompress(first), self.canonical(bundle)) - self.assertEqual(first, gzip_canonical_bundle_bytes(self.canonical(bundle))) - before = path.stat() - repeat = store.write_canonical_audit_bundle( - BLOCK_A, - digest, - self.canonical(bundle), - ) - after = repeat.stat() - self.assertEqual(path.read_bytes(), first) - self.assertEqual((before.st_dev, before.st_ino), (after.st_dev, after.st_ino)) - - def test_republication_accepts_equivalent_gzip_from_another_compressor( - self, - ) -> None: - with tempfile.TemporaryDirectory() as tmp: - root = Path(tmp) - store = self.make_store(root) - bundle = self.bundle() - payload = self.canonical(bundle) - digest = self.digest(bundle) - path = store.write_canonical_audit_bundle(BLOCK_A, digest, payload) - - alternative = gzip.compress(payload, compresslevel=1, mtime=0) - self.assertNotEqual(alternative, path.read_bytes()) - self.assertEqual(gzip.decompress(alternative), payload) - path.write_bytes(alternative) - before = path.stat() - - repeat = store.write_canonical_audit_bundle(BLOCK_A, digest, payload) - - after = repeat.stat() - self.assertEqual(repeat.read_bytes(), alternative) - self.assertEqual((before.st_dev, before.st_ino), (after.st_dev, after.st_ino)) - - def test_conflicting_or_corrupt_artifact_fails_loudly(self) -> None: - with tempfile.TemporaryDirectory() as tmp: - root = Path(tmp) - store = self.make_store(root) - bundle = self.bundle() - digest = self.digest(bundle) - path = store.canonical_bundle_path(BLOCK_A, digest) - path.write_bytes(gzip_canonical_bundle_bytes(b"{}")) - with self.assertRaisesRegex(RuntimeError, "does not match payload"): - store.write_canonical_audit_bundle( - BLOCK_A, - digest, - self.canonical(bundle), - ) - path.write_bytes(b"not gzip at all") - with self.assertRaisesRegex(RuntimeError, "does not match payload"): - store.write_canonical_audit_bundle( - BLOCK_A, - digest, - self.canonical(bundle), - ) - - def test_post_publication_verification_catches_a_bad_publish(self) -> None: - with tempfile.TemporaryDirectory() as tmp: - root = Path(tmp) - store = self.make_store(root) - bundle = self.bundle() - digest = self.digest(bundle) - path = store.canonical_bundle_path(BLOCK_A, digest) - - def publish_other_bytes(target: Path, _payload: bytes) -> None: - Path(target).write_bytes(gzip_canonical_bundle_bytes(b'{"other":1}')) - - with mock.patch.object( - store, - "_write_immutable_bytes", - side_effect=publish_other_bytes, - ): - with self.assertRaises(CanonicalAuditBundleCorrupt): - store.write_canonical_audit_bundle( - BLOCK_A, - digest, - self.canonical(bundle), - ) - path.unlink() - with mock.patch.object(store, "_write_immutable_bytes"): - with self.assertRaisesRegex(RuntimeError, "absent after publication"): - store.write_canonical_audit_bundle( - BLOCK_A, - digest, - self.canonical(bundle), - ) - - def test_corruption_is_distinguishable_from_a_clean_miss(self) -> None: - with tempfile.TemporaryDirectory() as tmp: - root = Path(tmp) - store = self.make_store(root) - bundle = self.bundle() - digest = self.digest(bundle) - self.assertIsNone(store.read_canonical_audit_bundle(BLOCK_A, digest)) - path = store.write_canonical_audit_bundle( - BLOCK_A, - digest, - self.canonical(bundle), - ) - self.assertEqual( - store.read_canonical_audit_bundle(BLOCK_A, digest), - self.canonical(bundle), - ) - # A miss for a different block stays a miss, not a failure. - self.assertIsNone(store.read_canonical_audit_bundle(BLOCK_B, digest)) - - path.write_bytes(path.read_bytes()[:12]) - with self.assertRaisesRegex( - CanonicalAuditBundleCorrupt, - "does not decompress", - ): - store.read_canonical_audit_bundle(BLOCK_A, digest) - - path.write_bytes(gzip_canonical_bundle_bytes(b'{"schema":"other"}')) - with self.assertRaisesRegex(CanonicalAuditBundleCorrupt, "hash mismatch"): - store.read_canonical_audit_bundle(BLOCK_A, digest) - - path.unlink() - outside = root / "outside.gz" - outside.write_bytes(gzip_canonical_bundle_bytes(self.canonical(bundle))) - path.symlink_to(outside) - with self.assertRaisesRegex( - CanonicalAuditBundleCorrupt, - "not retrievable", - ): - store.read_canonical_audit_bundle(BLOCK_A, digest) - - def test_root_replacement_revokes_canonical_publication_and_reads(self) -> None: - with tempfile.TemporaryDirectory() as tmp: - base = Path(tmp) - root = base / "audit" - root.mkdir() - store = self.make_store(root) - bundle = self.bundle() - digest = self.digest(bundle) - store.write_canonical_audit_bundle( - BLOCK_A, - digest, - self.canonical(bundle), - ) - - pinned = base / "audit-pinned" - root.rename(pinned) - root.mkdir() - for operation in ( - lambda: store.canonical_bundle_path(BLOCK_A, digest), - lambda: store.write_canonical_audit_bundle( - BLOCK_A, - digest, - self.canonical(bundle), - ), - lambda: store.read_canonical_audit_bundle(BLOCK_A, digest), - ): - with self.assertRaisesRegex(RuntimeError, "root identity"): - operation() - self.assertEqual(list(root.iterdir()), []) - - replacement = base / "audit-replacement" - root.rename(replacement) - pinned.rename(root) - self.assertEqual( - store.read_canonical_audit_bundle(BLOCK_A, digest), - self.canonical(bundle), - ) - - def test_kind_metrics_and_pruning_preserve_canonical_bundles(self) -> None: - with tempfile.TemporaryDirectory() as tmp: - root = Path(tmp) - store = self.make_store( - root, - live_bundle_retention=0, - candidate_retention_seconds=0, - ) - bundle = self.bundle() - digest = self.digest(bundle) - name = f"prism-audit-bundle-canonical-{BLOCK_A}-{digest}.json.gz" - self.assertEqual(AuditArtifactStore.artifact_kind(name), "canonical_bundle") - for invalid in ( - f"prism-audit-bundle-canonical-{BLOCK_A}-{digest}.json", - f"prism-audit-bundle-canonical-{BLOCK_A}.json.gz", - f"prism-audit-bundle-canonical-{BLOCK_A}-{digest.upper()}.json.gz", - ): - with self.subTest(invalid=invalid): - self.assertEqual(AuditArtifactStore.artifact_kind(invalid), "other") - - path = store.write_canonical_audit_bundle( - BLOCK_A, - digest, - self.canonical(bundle), - ) - metrics = store.metrics_snapshot() - self.assertEqual( - metrics["canonical_bundle"], - {"files": 1, "bytes": path.stat().st_size}, - ) - - # Retention only ever unlinks the two expendable kinds, and a - # canonical bundle is classified as neither. - self.assertNotIn( - AuditArtifactStore.artifact_kind(path.name), - {"live_bundle", "candidate"}, - ) - stale = store.issue_candidate(block_hash=BLOCK_B) - stale.path.write_bytes(b"stale") - store.release_candidate(stale) - os.utime(stale.path, ns=(1, 1)) - result = store.prune_best_effort() - self.assertEqual(result.errors, 0) - self.assertEqual(result.candidate_removed, 1) - self.assertFalse(stale.path.exists()) - self.assertTrue(path.exists()) - self.assertEqual( - store.read_canonical_audit_bundle(BLOCK_A, digest), - self.canonical(bundle), - ) - - def test_write_path_persists_the_canonical_bytes_it_already_holds(self) -> None: - with tempfile.TemporaryDirectory() as tmp: - root = Path(tmp) - calls: list[int] = [] - store = self.make_store(root, counter=calls) - bundle = self.bundle() - body_uri, digest = self.publish(store, bundle) - self.assertEqual( - store.read_canonical_audit_bundle(BLOCK_A, digest), - self.canonical(bundle), - ) - # One serialization for the digest check, reused for publication; - # the canonical copy is never re-derived. - self.assertEqual(len(calls), 1) - self.assertTrue(Path(body_uri).exists()) - - # Re-persisting the same block is a no-op on both artifacts. - body_bytes = Path(body_uri).read_bytes() - canonical_path = store.canonical_bundle_path(BLOCK_A, digest) - canonical_bytes = canonical_path.read_bytes() - self.publish(store, bundle) - self.assertEqual(Path(body_uri).read_bytes(), body_bytes) - self.assertEqual(canonical_path.read_bytes(), canonical_bytes) - - def test_write_path_reuses_a_verified_canonical_candidate_verbatim(self) -> None: - with tempfile.TemporaryDirectory() as tmp: - root = Path(tmp) - calls: list[int] = [] - store = self.make_store(root, counter=calls) - bundle = self.bundle() - digest = self.digest(bundle) - candidate = root / "candidate.json" - candidate.write_bytes(self.canonical(bundle)) - calls.clear() - store.prepare_external_audit_body( - {"block_hash": BLOCK_A, "audit_bundle_sha256": digest}, - bundle, - body_uri=str(store.body_path(BLOCK_A, digest)), - canonical_bundle_path=candidate, - ) - # The candidate bytes are taken as-is: nothing is reserialized. - self.assertEqual(calls, []) - self.assertEqual( - store.read_canonical_audit_bundle(BLOCK_A, digest), - self.canonical(bundle), - ) - - def test_backfill_helper_repairs_missing_segments_under_the_body_digest(self) -> None: - with tempfile.TemporaryDirectory() as tmp: - root = Path(tmp) - store = self.make_store(root) - bundle = self.bundle() - body_uri, digest = self.publish(store, bundle) - for segment in root.glob("prism-audit-share-segment-*"): - segment.unlink() - requested: list[tuple[int, int]] = [] - - def loader( - *, - first_share_seq: int, - last_share_seq: int, - ) -> list[dict[str, object]]: - requested.append((first_share_seq, last_share_seq)) - return [ - {"share_seq": share_seq, "worker": f"miner-{share_seq}"} - for share_seq in range(first_share_seq, last_share_seq + 1) - ] - - recovered = store.canonical_bundle_bytes_from_external_body( - block_hash=BLOCK_A, - audit_bundle_sha256=digest, - body_uri=body_uri, - load_missing_range=loader, - ) - self.assertEqual(recovered, self.canonical(bundle)) - self.assertEqual(requested, [(1, 2), (3, 4)]) - - # Without a callback the loss is loud rather than silently patched. - with self.assertRaisesRegex(RuntimeError, "not retrievable"): - store.canonical_bundle_bytes_from_external_body( - block_hash=BLOCK_A, - audit_bundle_sha256=digest, - body_uri=body_uri, - ) - - def test_backfill_helper_rejects_unverified_database_fallback_bytes(self) -> None: - with tempfile.TemporaryDirectory() as tmp: - root = Path(tmp) - store = self.make_store(root) - bundle = self.bundle() - body_uri, digest = self.publish(store, bundle) - for segment in root.glob("prism-audit-share-segment-*"): - segment.unlink() - - def tampered( - *, - first_share_seq: int, - last_share_seq: int, - ) -> list[dict[str, object]]: - return [ - {"share_seq": share_seq, "worker": "attacker"} - for share_seq in range(first_share_seq, last_share_seq + 1) - ] - - def short( - *, - first_share_seq: int, - last_share_seq: int, - ) -> list[dict[str, object]]: - return [{"share_seq": first_share_seq, "worker": f"miner-{first_share_seq}"}] - - def failing(*, first_share_seq: int, last_share_seq: int) -> list[Any]: - raise RuntimeError("ledger unavailable") - - for loader, expected in ( - (tampered, "hash mismatch"), - (short, "not exactly contiguous"), - (failing, "could not be recovered"), - ): - with self.subTest(loader=loader.__name__): - with self.assertRaisesRegex(RuntimeError, expected): - store.canonical_bundle_bytes_from_external_body( - block_hash=BLOCK_A, - audit_bundle_sha256=digest, - body_uri=body_uri, - load_missing_range=loader, - ) - self.assertEqual( - list(root.glob("prism-audit-share-segment-*")), - [], - ) - - def test_backfill_helper_binds_the_body_uri_to_the_advertised_identity(self) -> None: - with tempfile.TemporaryDirectory() as tmp: - root = Path(tmp) - store = self.make_store(root) - bundle = self.bundle() - body_uri, digest = self.publish(store, bundle) - for block_hash, expected_digest, message in ( - (BLOCK_B, digest, "block hash does not match"), - (BLOCK_A, "cc" * 32, "digest does not match"), - ): - with self.subTest(block_hash=block_hash, digest=expected_digest): - with self.assertRaisesRegex(RuntimeError, message): - store.canonical_bundle_bytes_from_external_body( - block_hash=block_hash, - audit_bundle_sha256=expected_digest, - body_uri=body_uri, - ) - - def test_pre_v2_literal_body_is_taken_without_reserialization(self) -> None: - with tempfile.TemporaryDirectory() as tmp: - root = Path(tmp) - calls: list[int] = [] - store = self.make_store(root, share_segment_size=0, counter=calls) - bundle = self.bundle() - body_uri, digest = self.publish(store, bundle) - # A pre-v2 body stores the canonical bundle verbatim. - self.assertEqual(Path(body_uri).read_bytes(), self.canonical(bundle)) - - calls.clear() - self.assertEqual( - store.literal_canonical_bundle_bytes( - body_uri, - expected_sha256=digest, - ), - self.canonical(bundle), - ) - self.assertEqual( - store.canonical_bundle_bytes_from_external_body( - block_hash=BLOCK_A, - audit_bundle_sha256=digest, - body_uri=body_uri, - ), - self.canonical(bundle), - ) - # Neither call parsed and reserialized the bundle. - self.assertEqual(calls, []) - - def test_literal_helper_declines_a_compact_body(self) -> None: - with tempfile.TemporaryDirectory() as tmp: - root = Path(tmp) - store = self.make_store(root) - bundle = self.bundle() - body_uri, digest = self.publish(store, bundle) - self.assertIsNone( - store.literal_canonical_bundle_bytes( - body_uri, - expected_sha256=digest, - ) - ) - with self.assertRaisesRegex(RuntimeError, "digest does not match"): - store.literal_canonical_bundle_bytes( - body_uri, - expected_sha256="cc" * 32, - ) - - -if __name__ == "__main__": - unittest.main() diff --git a/tests/test_prism_audit_bundle_view.py b/tests/test_prism_audit_bundle_view.py deleted file mode 100644 index 3ef4b4c05..000000000 --- a/tests/test_prism_audit_bundle_view.py +++ /dev/null @@ -1,925 +0,0 @@ -#!/usr/bin/env python3 -"""Bounded canonical audit artifact views (issue #255). - -The view must reproduce ``json.load`` of the artifact member for member -while never holding a window-sized Python structure: the share arrays are -replayable lazy sequences, one oversized record only grows its own decode, -UTF-8 and escapes may straddle read windows, the descriptor is released -without cyclic garbage collection, and the streaming encoder reproduces the -compact ``json.dumps`` bytes exactly. -""" - -from __future__ import annotations - -import gc -import hashlib -import io -import json -import os -from pathlib import Path -import tempfile -import time -import tracemalloc -import unittest -from unittest import mock -import weakref - -from lab.prism.audit_bundle_view import ( - SCAN_CHUNK_BYTES, - ArtifactResourcePressure, - ArtifactSource, - CanonicalArtifactError, - CanonicalArtifactSyntaxError, - CanonicalAuditBundleView, - LazyRecordSequence, - LazyRecordSlice, - RawJsonDocument, - RawJsonRecord, - iter_json_byte_chunks, - iter_json_chunks, - json_chunks_sha256_and_size, - materialize_json, - spool_json_document, -) - - -def make_share(index: int, *, oversized_bytes: int = 0) -> dict[str, object]: - share: dict[str, object] = { - "share_seq": index + 1, - "share_id": f"share-{index + 1}", - "miner_id": "mïner-ü" if index % 7 == 0 else f"miner-{index % 5}", - "order_key": f"o{index % 5}", - "p2mr_program_hex": ("%02x" % (index % 5)) * 32, - "share_difficulty": 2**100 + index, - "network_difficulty": 2**127 + 1, - "template_height": 10, - "job_id": "job-1", - "job_issued_at_ms": 1_700_000_000_000 - 1000, - "accepted_at_ms": 1_700_000_000_000 - 500, - "ntime": 1_700_000_000, - } - if index % 3 == 0: - share["credit_policy"] = None - elif index % 3 == 1: - share["credit_policy"] = 'quote"back\\slash\n\ttab ☃' - if oversized_bytes: - share["share_id"] = "Z" * oversized_bytes - return share - - -def make_bundle(count: int, *, oversized_index: int | None = None, oversized_bytes: int = 0) -> dict[str, object]: - shares = [ - make_share( - index, - oversized_bytes=oversized_bytes if index == oversized_index else 0, - ) - for index in range(count) - ] - return { - "schema": "qbit.prism.audit-bundle.v1", - "shares": shares, - "found_block": {"block_height": 10, "network_difficulty": 2**90}, - "prior_balances": [], - "reward_manifest": { - "schema": "qbit.prism.reward-manifest.v1", - "included_share_count": count, - "shares": [ - {"share_seq": index + 1, "counted_difficulty": 5, "credit_policy": None} - for index in range(count) - ], - "entitlements": [{"recipient_id": "a", "weight": 1}], - }, - "payout_policy_manifest": { - "accounts": [{"recipient_id": "a", "nested": {"deep": [1, {"x": [1, 2]}]}}] - }, - "signed_coinbase_manifest": {"manifest": {"coinbase_tx_hex": "00"}}, - } - - -def write_document(root: Path, name: str, raw: bytes) -> Path: - path = root / name - path.write_bytes(raw) - return path - - -def compact(bundle: dict[str, object], *, ensure_ascii: bool = True, separators: tuple[str, str] = (",", ":")) -> bytes: - return json.dumps(bundle, separators=separators, ensure_ascii=ensure_ascii).encode("utf-8") - - -def scan(path: Path, **kwargs: object) -> CanonicalAuditBundleView: - return CanonicalAuditBundleView.scan_path(path, **kwargs) # type: ignore[arg-type] - - -class ViewParityTests(unittest.TestCase): - def test_view_reproduces_json_load_across_row_counts(self) -> None: - with tempfile.TemporaryDirectory() as tmp: - root = Path(tmp) - for count in (0, 1, 255, 256, 257, 511, 512, 513, 1300): - for ensure_ascii, separators in ( - (True, (",", ":")), - (False, (", ", ": ")), - ): - with self.subTest(count=count, ascii=ensure_ascii): - bundle = make_bundle(count) - raw = compact(bundle, ensure_ascii=ensure_ascii, separators=separators) - path = write_document(root, f"bundle-{count}.json", raw) - view = scan(path, chunk_bytes=4096, stride=16) - try: - self.assertEqual(view.sha256_hex, hashlib.sha256(raw).hexdigest()) - self.assertEqual(view.byte_length, len(raw)) - self.assertEqual(list(view), list(bundle)) - self.assertEqual(view.key_index("shares"), 1) - self.assertIsInstance(view["shares"], LazyRecordSequence) - self.assertIsInstance(view["reward_manifest"]["shares"], LazyRecordSequence) - self.assertEqual(len(view["shares"]), count) - # Full logical equality, both operand orders. - self.assertTrue(view == bundle) - self.assertTrue(bundle == view) - self.assertFalse(view != bundle) - self.assertFalse(bundle != view) - # Replayable: two complete iterations agree. - self.assertEqual(list(view["shares"]), bundle["shares"]) - self.assertEqual(list(view["shares"]), bundle["shares"]) - self.assertEqual( - list(view["reward_manifest"]["shares"]), - bundle["reward_manifest"]["shares"], - ) - if count: - shares = view["shares"] - self.assertEqual(shares[0], bundle["shares"][0]) - self.assertEqual(shares[-1], bundle["shares"][-1]) - self.assertEqual(shares[count // 2], bundle["shares"][count // 2]) - self.assertIsInstance(shares[3:9], LazyRecordSlice) - self.assertEqual(shares[3:9], bundle["shares"][3:9]) - self.assertEqual(shares[::7], bundle["shares"][::7]) - self.assertEqual( - [list(batch) for batch in shares.batches(100)], - [bundle["shares"][i : i + 100] for i in range(0, count, 100)], - ) - with self.assertRaises(IndexError): - shares[count] - with self.assertRaises(IndexError): - shares[-count - 1] - finally: - view.close() - - def test_default_windows_and_strides_agree_with_small_ones(self) -> None: - with tempfile.TemporaryDirectory() as tmp: - bundle = make_bundle(2000) - raw = compact(bundle, ensure_ascii=False) - path = write_document(Path(tmp), "bundle.json", raw) - default_view = scan(path) - tiny_view = scan(path, chunk_bytes=7, stride=3) - try: - self.assertEqual(default_view, bundle) - self.assertEqual(tiny_view, bundle) - for index in (0, 1, 2, 3, 999, 1998, 1999): - self.assertEqual(tiny_view["shares"][index], bundle["shares"][index]) - self.assertEqual(default_view["shares"][index], bundle["shares"][index]) - finally: - default_view.close() - tiny_view.close() - - def test_oversized_single_field_is_isolated_to_its_own_decode(self) -> None: - oversized = 3 * SCAN_CHUNK_BYTES + 17 - with tempfile.TemporaryDirectory() as tmp: - bundle = make_bundle(257, oversized_index=128, oversized_bytes=oversized) - raw = compact(bundle) - path = write_document(Path(tmp), "bundle.json", raw) - view = scan(path) - try: - self.assertEqual(view, bundle) - self.assertEqual(view["shares"][128], bundle["shares"][128]) - # Reading an ordinary record never pays for the oversized one. - tracemalloc.start() - try: - tracemalloc.reset_peak() - self.assertEqual(view["shares"][0], bundle["shares"][0]) - _current, peak = tracemalloc.get_traced_memory() - finally: - tracemalloc.stop() - self.assertLess(peak, oversized) - tracemalloc.start() - try: - tracemalloc.reset_peak() - self.assertEqual(view["shares"][128], bundle["shares"][128]) - _current, peak = tracemalloc.get_traced_memory() - finally: - tracemalloc.stop() - # One oversized record costs a small multiple of itself - # (decoded text plus the string), never the window. - self.assertLess(peak, 8 * oversized) - finally: - view.close() - - def test_null_and_absent_members_stay_distinct(self) -> None: - with tempfile.TemporaryDirectory() as tmp: - bundle = make_bundle(6) - raw = compact(bundle) - path = write_document(Path(tmp), "bundle.json", raw) - view = scan(path) - try: - self.assertEqual(view, bundle) - absent = json.loads(raw) - del absent["shares"][0]["credit_policy"] - self.assertNotEqual(view, absent) - nulled = json.loads(raw) - nulled["shares"][2]["credit_policy"] = None - self.assertNotEqual(view, nulled) - self.assertEqual(view["shares"][0]["share_difficulty"], 2**100) - self.assertEqual(view["found_block"]["network_difficulty"], 2**90) - finally: - view.close() - - def test_equality_mismatches_are_detected_streaming(self) -> None: - with tempfile.TemporaryDirectory() as tmp: - bundle = make_bundle(600) - raw = compact(bundle) - path = write_document(Path(tmp), "bundle.json", raw) - view = scan(path, chunk_bytes=1024) - try: - last_record = json.loads(raw) - last_record["shares"][-1]["share_seq"] = 999_999 - self.assertNotEqual(view, last_record) - self.assertNotEqual(last_record, view) - extra_key = {**json.loads(raw), "extra": 1} - self.assertNotEqual(view, extra_key) - missing_key = json.loads(raw) - del missing_key["prior_balances"] - self.assertNotEqual(view, missing_key) - counted = json.loads(raw) - counted["reward_manifest"]["shares"][300]["counted_difficulty"] = 6 - self.assertNotEqual(view, counted) - shorter = json.loads(raw) - shorter["shares"].pop() - self.assertNotEqual(view, shorter) - longer = json.loads(raw) - longer["shares"].append(make_share(600)) - self.assertNotEqual(view, longer) - self.assertNotEqual(view["shares"], "not a sequence") - self.assertEqual(view["shares"][10:20], list(view["shares"])[10:20]) - finally: - view.close() - - -class TokenBoundaryTests(unittest.TestCase): - """Number tokens cut by a read window must never decode as a prefix.""" - - DOCUMENTS = ( - {"x": 1e300, "shares": []}, - {"x": -1.5e-3, "shares": [1.25, -0.5e2, 3, 1e300, -7]}, - {"x": 12345678901234567890, "shares": [0, -0, 10, 100000000000000000000]}, - {"x": 0.000001, "y": 5e-324, "shares": [{"n": 1e+300}, {"n": -2.5E+10}]}, - {"a": True, "b": None, "c": False, "x": -7, "shares": ["1e+", "-", "1."]}, - ) - - def _raw(self, document: dict[str, object]) -> bytes: - text = json.dumps(document, separators=(",", ":")) - # Spell exponents the way serializers may: keep an explicit sign. - return text.replace("1e+300", "1e+300").encode() - - def test_tiny_window_sweep_matches_json_loads(self) -> None: - with tempfile.TemporaryDirectory() as tmp: - for index, document in enumerate(self.DOCUMENTS): - raw = self._raw(document) - # Also exercise an explicit plus sign the stdlib serializer - # never emits but valid documents may carry. - variants = [raw, raw.replace(b"1e+300", b"1E+300").replace(b"-7", b"-7.0e+0")] - for variant_index, variant in enumerate(variants): - expected = json.loads(variant) - path = write_document(Path(tmp), f"num-{index}-{variant_index}.json", variant) - for chunk_bytes in (1, 2, 3, 4, 5, 7, 8, 16, 64): - with self.subTest(document=index, variant=variant_index, chunk_bytes=chunk_bytes): - view = scan(path, chunk_bytes=chunk_bytes, stride=2) - try: - self.assertEqual(view, expected) - self.assertEqual(list(view["shares"]), expected["shares"]) - self.assertEqual(view["shares"][0:2], expected["shares"][0:2]) - if expected["shares"]: - self.assertEqual(view["shares"][-1], expected["shares"][-1]) - finally: - view.close() - - def test_default_window_alignment_sweep(self) -> None: - with tempfile.TemporaryDirectory() as tmp: - for pad in range(SCAN_CHUNK_BYTES - 40, SCAN_CHUNK_BYTES + 8): - raw = b'{"pad":"' + b"a" * pad + b'","x":1e+300,"y":-2.5e-7,"shares":[12,3.5e+1]}' - expected = json.loads(raw) - path = write_document(Path(tmp), "aligned.json", raw) - with self.subTest(pad=pad): - view = scan(path) - try: - self.assertEqual(view, expected) - self.assertEqual(view["shares"][1], 35.0) - finally: - view.close() - - def test_malformed_tokens_are_rejected_at_scan_time(self) -> None: - """Every rejection the standard decoder makes, the scan makes too.""" - with tempfile.TemporaryDirectory() as tmp: - for index, raw in enumerate( - ( - b'{"x":1e+,"shares":[]}', - b'{"x":1.,"shares":[]}', - b'{"x":01,"shares":[]}', - b'{"x":-,"shares":[]}', - b'{"x":1e+300', - b'{"shares":[1.]}', - b'{"shares":[1 2]}', - b'{"shares":[{"a":1,}]}', - b'{"shares":[{"a" 1}]}', - b'{"shares":["a\x01b"]}', - b'{"shares":["\\q"]}', - b'{"shares":["\\u12"]}', - b'{"shares":[{"a":[1,]}]}', - b'{"shares":[tru]}', - b'{"shares":[{"a":-}]}', - b'{"shares":[{"a":1}{"b":2}]}', - b'{"shares":[[1,2}]}', - b'{"shares":[{"a":1,"b":}]}', - b'{"shares":[{,}]}', - b'{"shares":[nul]}', - b'{"shares":[1]', - ) - ): - with self.subTest(document=raw): - self.assertRaises(ValueError, json.loads, raw) - path = write_document(Path(tmp), f"bad-{index}.json", raw) - for chunk_bytes in (1, 3, 8, 4096): - with self.subTest(document=raw, chunk_bytes=chunk_bytes): - with self.assertRaises(json.JSONDecodeError): - scan(path, chunk_bytes=chunk_bytes) - - def test_nested_and_escaped_records_survive_tiny_windows(self) -> None: - document = { - "shares": [ - {"a": [1, {"b": "é\n\"}\\", "c": ""}], "d": " x ", "e": [], "f": {}}, - {"g": [[[]]], "h": "😀", "i": -0.0, "j": [True, False, None]}, - "plain", - 12, - [], - ], - "reward_manifest": {"shares": [{"n": 1}, {"n": [2, {"m": "é"}]}]}, - } - raw = json.dumps(document, indent=2).encode() - with tempfile.TemporaryDirectory() as tmp: - path = write_document(Path(tmp), "nested.json", raw) - for chunk_bytes in (1, 2, 5, 13, 64, SCAN_CHUNK_BYTES): - with self.subTest(chunk_bytes=chunk_bytes): - view = scan(path, chunk_bytes=chunk_bytes, stride=2) - try: - self.assertEqual(view, document) - self.assertEqual(list(view["shares"]), document["shares"]) - self.assertEqual(view["shares"][1]["h"], "😀") - self.assertEqual(view["shares"][2:5], document["shares"][2:5]) - finally: - view.close() - - -class ViewRobustnessTests(unittest.TestCase): - def test_malformed_documents_fail_as_json_decode_errors_and_release_fd(self) -> None: - cases = [ - b"[1,2]", - b'{"a":1', - b'{"a":1}x', - b'{"shares":[1,2,}', - b'{"shares":[{"a":1}', - b'{"a":"\xff"}', - b'{"shares":[{"a":1}]}{', - b'{"shares":[{"a":1}],}', - b'{1:2}', - b"", - b'{"shares":[{"a":"unterminated}]}', - ] - with tempfile.TemporaryDirectory() as tmp: - for index, raw in enumerate(cases): - with self.subTest(document=raw): - path = write_document(Path(tmp), f"bad-{index}.json", raw) - fd = os.open(path, os.O_RDONLY) - with self.assertRaises(json.JSONDecodeError) as caught: - CanonicalAuditBundleView.scan(fd, chunk_bytes=4) - self.assertIsInstance(caught.exception, CanonicalArtifactSyntaxError) - self.assertIsInstance(caught.exception, CanonicalArtifactError) - with self.assertRaises(OSError): - os.fstat(fd) - - def test_not_a_regular_file_is_rejected_and_fd_closed(self) -> None: - with tempfile.TemporaryDirectory() as tmp: - fd = os.open(tmp, os.O_RDONLY) - with self.assertRaises(CanonicalArtifactError): - ArtifactSource(fd) - with self.assertRaises(OSError): - os.fstat(fd) - - def test_identity_change_is_detected_by_lazy_reads(self) -> None: - with tempfile.TemporaryDirectory() as tmp: - bundle = make_bundle(40) - raw = compact(bundle) - path = write_document(Path(tmp), "bundle.json", raw) - view = scan(path) - try: - header = view["found_block"] - with path.open("ab") as handle: - handle.write(b" ") - self.assertEqual(view["found_block"], header) - with self.assertRaises(CanonicalArtifactError) as caught: - list(view["shares"]) - self.assertNotIsInstance(caught.exception, json.JSONDecodeError) - with self.assertRaises(CanonicalArtifactError): - view.verify_identity() - with self.assertRaises(CanonicalArtifactError): - list(view.iter_bytes()) - finally: - view.close() - - def test_descriptor_is_released_without_cyclic_gc(self) -> None: - gc.disable() - try: - with tempfile.TemporaryDirectory() as tmp: - bundle = make_bundle(30) - raw = compact(bundle) - path = write_document(Path(tmp), "bundle.json", raw) - view = scan(path) - fd = view.source.fileno() - shares = view["shares"] - self.assertEqual(shares[3], bundle["shares"][3]) - ref = weakref.ref(view) - source_ref = weakref.ref(view.source) - del view - # The lazy sequence still owns the source; the descriptor - # stays open exactly as long as a legitimate owner exists. - self.assertIsNone(ref()) - self.assertIsNotNone(source_ref()) - self.assertEqual(shares[4], bundle["shares"][4]) - del shares - self.assertIsNone(source_ref()) - with self.assertRaises(OSError): - os.fstat(fd) - # Explicit close is idempotent and fails lazy reads closed. - view = scan(path) - lazy = view["shares"] - view.close() - view.close() - self.assertTrue(view.closed) - self.assertEqual(view["schema"], bundle["schema"]) - with self.assertRaises(CanonicalArtifactError): - list(lazy) - finally: - gc.enable() - - def test_scan_stops_within_one_read_when_cancelled(self) -> None: - with tempfile.TemporaryDirectory() as tmp: - bundle = make_bundle(3000) - raw = compact(bundle) - path = write_document(Path(tmp), "bundle.json", raw) - calls: list[int] = [] - - class Cancelled(RuntimeError): - pass - - def checkpoint() -> None: - calls.append(1) - if len(calls) == 3: - raise Cancelled("cancelled") - - fd = os.open(path, os.O_RDONLY) - with self.assertRaises(Cancelled): - CanonicalAuditBundleView.scan(fd, chunk_bytes=8192, cancellation=checkpoint) - self.assertEqual(len(calls), 3) - with self.assertRaises(OSError): - os.fstat(fd) - - -class StreamingEncoderTests(unittest.TestCase): - def test_encoder_reproduces_compact_json_with_lazy_members(self) -> None: - with tempfile.TemporaryDirectory() as tmp: - bundle = make_bundle(700) - raw = compact(bundle) - path = write_document(Path(tmp), "bundle.json", raw) - view = scan(path, chunk_bytes=2048, stride=8) - try: - storage = { - "schema": "example", - "bundle_without_shares": view.without("shares"), - "count": 700, - "parts": [ - {"kind": "inline", "shares": view["shares"][0:5]}, - {"kind": "inline", "shares": view["shares"][695:700]}, - ], - "plain": {"list": [1, 2.5, None, True, "ü"], "empty": [], "obj": {}}, - "lazy_only": view["shares"], - } - expected_storage = { - "schema": "example", - "bundle_without_shares": {k: v for k, v in bundle.items() if k != "shares"}, - "count": 700, - "parts": [ - {"kind": "inline", "shares": bundle["shares"][0:5]}, - {"kind": "inline", "shares": bundle["shares"][695:700]}, - ], - "plain": {"list": [1, 2.5, None, True, "ü"], "empty": [], "obj": {}}, - "lazy_only": bundle["shares"], - } - expected = json.dumps(expected_storage, separators=(",", ":")) - for batch_records, chunk_chars in ((1, 1), (7, 100), (256, 65536)): - chunks = list( - iter_json_chunks( - storage, - batch_records=batch_records, - chunk_chars=chunk_chars, - ) - ) - self.assertEqual("".join(chunks), expected) - self.assertTrue(all(chunk.isascii() for chunk in chunks)) - digest, size = json_chunks_sha256_and_size(iter_json_byte_chunks(storage)) - self.assertEqual(digest, hashlib.sha256(expected.encode()).hexdigest()) - self.assertEqual(size, len(expected)) - with self.assertRaises(TypeError): - json.dumps(storage) - finally: - view.close() - - def test_encoder_matches_json_dumps_for_plain_values(self) -> None: - values = [ - {}, - [], - {"a": [1, [2, [3, {"b": None}]]], "c": "d"}, - {"k": {1: "int key"}}, - [{"x": 1}, {"y": [1, 2]}, "s", 3], - "solo", - 12, - ] - for value in values: - with self.subTest(value=value): - self.assertEqual( - "".join(iter_json_chunks(value, batch_records=2, chunk_chars=3)), - json.dumps(value, separators=(",", ":")), - ) - - -def _write_synthetic_bundle(path: Path, count: int) -> tuple[bytes, int]: - """Write a count-share bundle by streaming, returning (sha256, size).""" - digest = hashlib.sha256() - size = 0 - with path.open("wb") as handle: - - def emit(text: str) -> None: - nonlocal size - data = text.encode("utf-8") - digest.update(data) - size += len(data) - handle.write(data) - - emit('{"schema":"qbit.prism.audit-bundle.v1","shares":[') - for index in range(count): - if index: - emit(",") - emit(json.dumps(make_share(index), separators=(",", ":"))) - emit('],"found_block":{"block_height":10,"network_difficulty":1000},"prior_balances":[],') - emit('"reward_manifest":{"schema":"qbit.prism.reward-manifest.v1","shares":[') - for index in range(count): - if index: - emit(",") - emit( - json.dumps( - {"share_seq": index + 1, "counted_difficulty": 5}, - separators=(",", ":"), - ) - ) - emit('],"entitlements":[]},"payout_policy_manifest":{"accounts":[]},') - emit('"signed_coinbase_manifest":{"manifest":{"coinbase_tx_hex":"00"}}}') - return digest.digest(), size - - -class StressWindowTests(unittest.TestCase): - """Window-sized artifacts; the 375k case runs when explicitly enabled.""" - - def _exercise(self, count: int, *, peak_limit: int) -> dict[str, float]: - with tempfile.TemporaryDirectory() as tmp: - path = Path(tmp) / "canonical.json" - expected_digest, size = _write_synthetic_bundle(path, count) - tracemalloc.start() - try: - tracemalloc.reset_peak() - started = time.monotonic() - view = scan(path) - scan_seconds = time.monotonic() - started - try: - self.assertEqual(view.sha256_hex, expected_digest.hex()) - self.assertEqual(view.byte_length, size) - self.assertEqual(len(view["shares"]), count) - self.assertEqual(len(view["reward_manifest"]["shares"]), count) - for index in (0, 1, count // 2, count - 2, count - 1): - self.assertEqual(view["shares"][index], make_share(index)) - started = time.monotonic() - walked = 0 - for record in view["shares"]: - walked += 1 - if walked % 50_000 == 0: - self.assertEqual(record["share_seq"], walked) - walk_seconds = time.monotonic() - started - self.assertEqual(walked, count) - encoded = 0 - started = time.monotonic() - for chunk in iter_json_byte_chunks({"shares": view["shares"]}): - encoded += len(chunk) - encode_seconds = time.monotonic() - started - _current, peak = tracemalloc.get_traced_memory() - finally: - view.close() - finally: - tracemalloc.stop() - self.assertGreater(encoded, 0) - self.assertLess(peak, peak_limit, f"peak traced memory {peak} exceeded {peak_limit}") - return { - "count": count, - "bytes": size, - "scan_seconds": scan_seconds, - "walk_seconds": walk_seconds, - "encode_seconds": encode_seconds, - "peak_traced_bytes": peak, - } - - def test_twenty_thousand_shares_stay_bounded(self) -> None: - # ~7.5 MB of JSON; peak traced memory stays a few read windows. - report = self._exercise(20_000, peak_limit=6 * 1024 * 1024) - self.assertGreater(report["bytes"], 5_000_000) - - @unittest.skipUnless( - os.environ.get("PRISM_BOUNDED_AUDIT_STRESS"), - "set PRISM_BOUNDED_AUDIT_STRESS=1 to run the 375k-share artifact case", - ) - def test_375k_shares_across_generations(self) -> None: - reports = [] - for generation in range(int(os.environ.get("PRISM_BOUNDED_AUDIT_GENERATIONS", "2"))): - reports.append(self._exercise(375_000, peak_limit=32 * 1024 * 1024)) - for report in reports: - print(f"bounded-audit-stress {json.dumps(report)}", flush=True) - - -class BoundedScanTests(unittest.TestCase): - """The scan holds at most about two read chunks whatever a value's size.""" - - def test_checkpoint_index_lives_on_disk_and_pressure_is_recoverable(self) -> None: - with tempfile.TemporaryDirectory() as tmp: - bundle = make_bundle(3000) - raw = compact(bundle) - path = write_document(Path(tmp), "bundle.json", raw) - view = scan(path, stride=16) - try: - stats = view.scan_stats - # 3000 shares and 3000 counted shares at a stride of 16, one - # checkpoint per recipient-scaled array (entitlements, - # accounts) and none for the empty prior balances. - self.assertEqual(stats.checkpoint_entries, 2 * 188 + 2) - self.assertEqual(view["shares"][2999], bundle["shares"][2999]) - self.assertEqual(view["reward_manifest"]["shares"][1234], bundle["reward_manifest"]["shares"][1234]) - self.assertIsInstance(view["prior_balances"], LazyRecordSequence) - self.assertIsInstance(view["payout_policy_manifest"]["accounts"], LazyRecordSequence) - self.assertIsInstance(view["reward_manifest"]["entitlements"], LazyRecordSequence) - self.assertEqual(view, bundle) - finally: - view.close() - # Scratch pressure is neither a syntax error nor corruption: the - # artifact is intact and the caller retries later. - fd = os.open(path, os.O_RDONLY) - with mock.patch("lab.prism.audit_bundle_view.tempfile.TemporaryFile", side_effect=OSError("no temp")): - with self.assertRaises(ArtifactResourcePressure) as caught: - CanonicalAuditBundleView.scan(fd, stride=16) - self.assertNotIsInstance(caught.exception, CanonicalArtifactError) - self.assertNotIsInstance(caught.exception, json.JSONDecodeError) - with self.assertRaises(OSError): - os.fstat(fd) - # A closed view fails lazy reads closed rather than reading a - # released index or descriptor. - view = scan(path, stride=16) - lazy = view["shares"] - view.close() - with self.assertRaises(CanonicalArtifactError): - lazy[5] - - def test_oversized_records_are_normalized_by_the_isolated_helper(self) -> None: - limit = 8192 - oversized = 3 * limit - with tempfile.TemporaryDirectory() as tmp: - bundle = make_bundle(40, oversized_index=7, oversized_bytes=oversized) - bundle["shares"][8]["nested"] = {"deep": [1, {"x": "é\"}"}], "big": "Q" * (2 * limit)} - bundle["witness_merkle_leaves_hex"] = ["ab" * 32, "W" * (2 * limit)] - raw = compact(bundle, ensure_ascii=False) - path = write_document(Path(tmp), "bundle.json", raw) - with mock.patch("lab.prism.audit_bundle_view.RECORD_DECODE_SOFT_LIMIT_BYTES", limit): - view = scan(path, chunk_bytes=1024) - try: - records = list(view["shares"]) - self.assertEqual(view.scan_stats.isolated_records, 2) - isolated = records[7] - self.assertIsInstance(isolated, RawJsonRecord) - self.assertTrue(isolated.is_object) - self.assertEqual(isolated["share_seq"], 8) - self.assertEqual(isolated["miner_id"], bundle["shares"][7]["miner_id"]) - self.assertIn("share_id", isolated.omitted_members) - self.assertIn("share_id", isolated) - with self.assertRaises(CanonicalArtifactError): - isolated["share_id"] - with self.assertRaises(KeyError): - isolated["absent"] - self.assertEqual(list(isolated), list(bundle["shares"][7])) - # Equality streams the sorted encodings, both operand - # orders, against dictionaries and other records. - self.assertEqual(isolated, bundle["shares"][7]) - self.assertTrue(bundle["shares"][7] == isolated) - self.assertNotEqual(isolated, bundle["shares"][8]) - self.assertNotEqual(isolated, {**bundle["shares"][7], "extra": 1}) - self.assertEqual(records[8], bundle["shares"][8]) - self.assertEqual(view["shares"][7], view["shares"][7]) - self.assertNotEqual(view["shares"][7], view["shares"][8]) - self.assertEqual(view, bundle) - self.assertEqual(materialize_json(isolated), bundle["shares"][7]) - # The streaming encoder emits the record's own encoding - # verbatim, byte-identical to json.dumps of the value. - self.assertEqual( - "".join(iter_json_chunks({"shares": view["shares"][6:9]})), - json.dumps({"shares": bundle["shares"][6:9]}, separators=(",", ":")), - ) - self.assertEqual( - "".join(iter_json_chunks(isolated, sort_keys=True)), - json.dumps(bundle["shares"][7], sort_keys=True, separators=(",", ":")), - ) - # A non-object oversized value (a leaf string) is - # normalized the same way. - leaves = list(view["witness_merkle_leaves_hex"]) - self.assertEqual(leaves[0], "ab" * 32) - self.assertIsInstance(leaves[1], RawJsonRecord) - self.assertFalse(leaves[1].is_object) - self.assertEqual(leaves[1], "W" * (2 * limit)) - self.assertEqual(materialize_json(view["witness_merkle_leaves_hex"]), bundle["witness_merkle_leaves_hex"]) - finally: - view.close() - - def test_isolated_helper_failures_are_classified(self) -> None: - with tempfile.TemporaryDirectory() as tmp: - raw = b'{"shares":[{"a":"' + b"x" * 20000 + b'"}]}' - path = write_document(Path(tmp), "bundle.json", raw) - with mock.patch("lab.prism.audit_bundle_view.RECORD_DECODE_SOFT_LIMIT_BYTES", 1024): - view = scan(path) - try: - self.assertEqual(view["shares"][0], {"a": "x" * 20000}) - with mock.patch( - "lab.prism.audit_bundle_view.subprocess.Popen", - side_effect=OSError("fork failed"), - ): - with self.assertRaises(ArtifactResourcePressure): - view["shares"][0] - real_temporary_file = tempfile.TemporaryFile - calls = {"count": 0} - - def failing_spool(*args: object, **kwargs: object) -> object: - calls["count"] += 1 - raise OSError("disk full") - - with mock.patch( - "lab.prism.audit_bundle_view.tempfile.TemporaryFile", - side_effect=failing_spool, - ): - with self.assertRaises(ArtifactResourcePressure): - view["shares"][0] - self.assertGreaterEqual(calls["count"], 1) - self.assertIs(tempfile.TemporaryFile, real_temporary_file) - finally: - view.close() - # The scan rejects a token-malformed record before any helper - # runs; the helper classifies malformed input the same way when - # it is handed such bytes directly, never as resource pressure. - bad = b'{"shares":[{"a":tru,"b":"' + b"y" * 20000 + b'"}]}' - bad_path = write_document(Path(tmp), "bad.json", bad) - with self.assertRaises(json.JSONDecodeError): - scan(bad_path) - from lab.prism.audit_bundle_view import normalize_record_isolated - - source = ArtifactSource(os.open(bad_path, os.O_RDONLY), path=bad_path) - try: - with self.assertRaises(json.JSONDecodeError) as caught: - normalize_record_isolated(source, 11, len(bad) - 2) - self.assertIsInstance(caught.exception, CanonicalArtifactSyntaxError) - finally: - source.close() - - def test_raw_document_and_spooled_documents_round_trip(self) -> None: - with tempfile.TemporaryDirectory() as tmp: - bundle = make_bundle(12) - raw = compact(bundle, ensure_ascii=False) - path = write_document(Path(tmp), "bundle.json", raw) - view = scan(path) - try: - document = RawJsonDocument(view.source) - self.assertEqual(document.byte_length, len(raw)) - self.assertEqual(b"".join(document.iter_byte_chunks(chunk_bytes=7)), raw) - self.assertEqual("".join(document.iter_text_chunks(chunk_bytes=7)), raw.decode("utf-8")) - embedded = "".join(iter_json_chunks({"audit_bundle": document, "n": 1})) - self.assertEqual(json.loads(embedded), {"audit_bundle": bundle, "n": 1}) - self.assertEqual(materialize_json(view), bundle) - spooled = spool_json_document( - iter_json_byte_chunks({"share_parts": [{"kind": "x", "n": i} for i in range(700)]}), - lazy_paths=(("share_parts",),), - stride=8, - ) - try: - parts = spooled["share_parts"] - self.assertIsInstance(parts, LazyRecordSequence) - self.assertEqual(len(parts), 700) - self.assertEqual(parts[699], {"kind": "x", "n": 699}) - self.assertEqual(list(parts)[:3], [{"kind": "x", "n": i} for i in range(3)]) - finally: - spooled.close() - finally: - view.close() - - def test_oversized_values_are_skipped_without_being_held(self) -> None: - chunk = 4096 - oversized = 6 * SCAN_CHUNK_BYTES + 11 # far above the soft limit - with tempfile.TemporaryDirectory() as tmp: - bundle = make_bundle(300, oversized_index=150, oversized_bytes=oversized) - bundle["shares"][151]["nested"] = {"deep": [1, {"x": '"}]{[\\'}], "s": "a\\\"b"} - bundle["shares"][152]["credit_policy"] = "quote\"in\\side" + "é" * 3000 - bundle["reward_manifest"]["shares"][150]["blob"] = "Y" * (3 * chunk + 1) - bundle["witness_merkle_leaves_hex"] = ["ab" * 32] * 500 - raw = compact(bundle, ensure_ascii=False) - path = write_document(Path(tmp), "bundle.json", raw) - tracemalloc.start() - try: - tracemalloc.reset_peak() - view = scan(path, chunk_bytes=chunk) - _current, scan_peak = tracemalloc.get_traced_memory() - finally: - tracemalloc.stop() - try: - stats = view.scan_stats - # The oversized record was walked structurally: the largest - # decoded window stayed a couple of chunks, not the record. - self.assertLess(stats.window_high_water_chars, 4 * chunk) - self.assertGreaterEqual(stats.max_value_chars, oversized) - self.assertEqual(stats.oversized_values, 1) - self.assertLess(scan_peak, oversized) - self.assertEqual(view.sha256_hex, hashlib.sha256(raw).hexdigest()) - self.assertEqual(len(view["shares"]), 300) - self.assertEqual(len(view["witness_merkle_leaves_hex"]), 500) - # Consumers that need the record decode it whole; that - # decode is the remaining record-sized allocation. - self.assertEqual(view["shares"][150], bundle["shares"][150]) - self.assertEqual(view["shares"][151], bundle["shares"][151]) - self.assertEqual(view["shares"][152], bundle["shares"][152]) - self.assertEqual(view, bundle) - self.assertEqual(list(view["shares"][149:153]), bundle["shares"][149:153]) - finally: - view.close() - - def test_structural_walk_rejects_unterminated_values(self) -> None: - cases = [ - b'{"shares":[{"a":"' + b"x" * 5000, - b'{"shares":[{"a":[1,2}]}', - b'{"shares":[{"a":"\\' , - b'{"shares":[,]}', - b'{"shares":[1,]}', - b'{"shares":["abc]}', - ] - with tempfile.TemporaryDirectory() as tmp: - for index, raw in enumerate(cases): - with self.subTest(document=raw[:24]): - path = write_document(Path(tmp), f"bad-{index}.json", raw) - with self.assertRaises(json.JSONDecodeError): - scan(path, chunk_bytes=16) - - -class ArtifactSourceTests(unittest.TestCase): - def test_iter_bytes_and_sha256_cover_the_whole_file(self) -> None: - with tempfile.TemporaryDirectory() as tmp: - raw = os.urandom(3 * SCAN_CHUNK_BYTES + 5) - path = write_document(Path(tmp), "blob.bin", raw) - source = ArtifactSource(os.open(path, os.O_RDONLY), path=path) - try: - self.assertEqual(b"".join(source.iter_bytes(chunk_bytes=1000)), raw) - self.assertEqual(source.sha256_hex(), hashlib.sha256(raw).hexdigest()) - self.assertEqual(source.pread(len(raw) - 3, 10), raw[-3:]) - self.assertEqual(source.pread(len(raw), 10), b"") - finally: - source.close() - self.assertTrue(source.closed) - with self.assertRaises(CanonicalArtifactError): - source.fileno() - - def test_chunk_reader_adapter_streams_sequentially(self) -> None: - from lab.prism.audit_artifacts import _ChunkReader - - with tempfile.TemporaryDirectory() as tmp: - raw = bytes(range(256)) * 700 - path = write_document(Path(tmp), "blob.bin", raw) - source = ArtifactSource(os.open(path, os.O_RDONLY), path=path) - try: - reader = io.BufferedReader(_ChunkReader(source), buffer_size=4096) - self.assertEqual(reader.read(), raw) - finally: - source.close() - - -if __name__ == "__main__": - unittest.main() diff --git a/tests/test_prism_audit_http.py b/tests/test_prism_audit_http.py deleted file mode 100644 index 7b44aa45c..000000000 --- a/tests/test_prism_audit_http.py +++ /dev/null @@ -1,555 +0,0 @@ -#!/usr/bin/env python3 -"""Direct routing and bounded lifecycle tests for the PRISM HTTP facade.""" - -from __future__ import annotations - -import json -import threading -import time -import unittest -from unittest.mock import patch -import urllib.error -import urllib.request - -from lab.prism.audit_http import ( - AuditHttpConfig, - AuditHttpFacade, - _BoundedThreadingHttpServer, -) -from lab.prism.observability import ( - METRICS_STALE_WARNING, - METRICS_STATE_FRESH, - METRICS_STATE_HEADER, - METRICS_STATE_STALE, - METRICS_STATE_UNAVAILABLE, - MetricsSnapshotResponse, -) - - -class FakeAuditHttpPort: - def __init__(self) -> None: - self.health_entered: threading.Event | None = None - self.health_release: threading.Event | None = None - self.health_calls = 0 - self.metrics_calls = 0 - self.metrics_response = MetricsSnapshotResponse( - status=200, - body="qbit_prism_cached_fixture 1\n", - state=METRICS_STATE_FRESH, - age_seconds=0, - ) - - def cached_health_payload(self) -> tuple[int, dict[str, object]]: - self.health_calls += 1 - if self.health_entered is not None: - self.health_entered.set() - if self.health_release is not None: - self.health_release.wait(2.0) - return 200, {"ok": True, "schema": "health-fixture"} - - def cached_metrics_payload(self) -> MetricsSnapshotResponse: - self.metrics_calls += 1 - return self.metrics_response - - def latest_evidence_payload(self) -> dict[str, object] | None: - return None - - def owed_balances_payload(self) -> dict[str, object]: - return {"balances": []} - - def carry_forward_integrity_payload(self) -> dict[str, object]: - return {"ok": True} - - def miner_status_payload(self, recipient_id: str) -> dict[str, object]: - return {"recipient_id": recipient_id} - - def public_payload( - self, - path: str, - query: dict[str, list[str]], - ) -> tuple[int, object]: - raise AssertionError((path, query)) - - -class AuditHttpFacadeTests(unittest.TestCase): - def test_start_failure_releases_lifecycle_lock_before_join(self) -> None: - class NeverReadyServer: - def __init__(self, address: tuple[str, int], _handler: object) -> None: - self.server_address = address - self.ready = threading.Event() - self.shutdown_requested = threading.Event() - self.closed = False - self.serve_committed = False - - def serve_unless_startup_cancelled(self, *, poll_interval: float) -> None: - del poll_interval - self.serve_committed = True - self.shutdown_requested.wait(2.0) - - def cancel_startup(self) -> bool: - return self.serve_committed - - def shutdown(self) -> None: - self.shutdown_requested.set() - - def server_close(self) -> None: - self.closed = True - - facade = AuditHttpFacade( - FakeAuditHttpPort(), # type: ignore[arg-type] - AuditHttpConfig( - "127.0.0.1", - 0, - join_timeout_seconds=0.5, - ), - ) - started = time.monotonic() - with patch( - "lab.prism.audit_http._BoundedThreadingHttpServer", - NeverReadyServer, - ), self.assertRaisesRegex(RuntimeError, "did not enter"): - facade.start() - - self.assertLess(time.monotonic() - started, 1.25) - state = facade.state() - self.assertEqual(state.lifecycle, "stopped") - self.assertFalse(state.thread_alive) - self.assertIsNone(state.bound_address) - - def test_start_timeout_cancels_thread_delayed_before_serve_forever(self) -> None: - release_thread = threading.Event() - serve_forever_called = threading.Event() - real_thread = threading.Thread - - class DelayedThread(real_thread): - def run(self) -> None: - release_thread.wait(2.0) - super().run() - - def join(self, timeout: float | None = None) -> None: - release_thread.set() - super().join(timeout) - - class TrackedServer: - instance: TrackedServer | None = None - - def __init__(self, address: tuple[str, int], _handler: object) -> None: - self.server_address = address - self.ready = threading.Event() - self.cancelled = False - self.closed = False - TrackedServer.instance = self - - def serve_unless_startup_cancelled(self, *, poll_interval: float) -> None: - del poll_interval - if not self.cancelled: - serve_forever_called.set() - - def cancel_startup(self) -> bool: - self.cancelled = True - return False - - def shutdown(self) -> None: - raise AssertionError("shutdown called before serve_forever") - - def server_close(self) -> None: - self.closed = True - - facade = AuditHttpFacade( - FakeAuditHttpPort(), # type: ignore[arg-type] - AuditHttpConfig("127.0.0.1", 0, join_timeout_seconds=0.5), - ) - with patch("lab.prism.audit_http.threading.Thread", DelayedThread), patch( - "lab.prism.audit_http._BoundedThreadingHttpServer", - TrackedServer, - ), self.assertRaisesRegex(RuntimeError, "did not enter"): - facade.start() - - self.assertFalse(serve_forever_called.is_set()) - assert TrackedServer.instance is not None - self.assertTrue(TrackedServer.instance.cancelled) - self.assertTrue(TrackedServer.instance.closed) - self.assertEqual(facade.state().lifecycle, "stopped") - - def test_configuration_bounds_listener_and_join_values(self) -> None: - with self.assertRaises(ValueError): - AuditHttpConfig("", 1) - with self.assertRaises(ValueError): - AuditHttpConfig("127.0.0.1", -1) - with self.assertRaises(ValueError): - AuditHttpConfig("127.0.0.1", 65_536) - with self.assertRaises(ValueError): - AuditHttpConfig("127.0.0.1", 1, join_timeout_seconds=-1) - - def test_start_serves_cached_payloads_and_stop_is_idempotent(self) -> None: - port = FakeAuditHttpPort() - facade = AuditHttpFacade( - port, # type: ignore[arg-type] - AuditHttpConfig("127.0.0.1", 0), - ) - state = facade.start() - self.assertEqual(state.lifecycle, "running") - self.assertTrue(state.thread_alive) - self.assertIsNotNone(state.bound_address) - assert state.bound_address is not None - base_url = f"http://127.0.0.1:{state.bound_address[1]}" - - with urllib.request.urlopen(base_url + "/healthz", timeout=2) as response: - self.assertEqual(response.status, 200) - self.assertEqual( - json.loads(response.read()), - {"ok": True, "schema": "health-fixture"}, - ) - self.assertTrue( - response.headers["Server"].startswith( - "QbitPrismAudit/0.1 Python/" - ) - ) - with urllib.request.urlopen(base_url + "/metrics", timeout=2) as response: - self.assertEqual(response.status, 200) - self.assertEqual( - response.headers["Content-Type"], - "text/plain; version=0.0.4", - ) - self.assertEqual(response.read(), b"qbit_prism_cached_fixture 1\n") - - self.assertEqual(port.health_calls, 1) - self.assertEqual(port.metrics_calls, 1) - self.assertTrue(facade.stop()) - self.assertTrue(facade.stop()) - stopped = facade.state() - self.assertEqual(stopped.lifecycle, "stopped") - self.assertFalse(stopped.thread_alive) - self.assertIsNone(stopped.bound_address) - - def test_metrics_publishes_freshness_in_response_metadata(self) -> None: - """A scraper must tell fresh from stale from unavailable (issue #184). - - None of these assertions parse the Prometheus body: the whole point of - the response metadata is that a consumer does not have to. - """ - - port = FakeAuditHttpPort() - facade = AuditHttpFacade( - port, # type: ignore[arg-type] - AuditHttpConfig(bind="127.0.0.1", port=0), - ) - self.addCleanup(facade.stop) - state = facade.start() - assert state.bound_address is not None - url = f"http://127.0.0.1:{state.bound_address[1]}/metrics" - - with urllib.request.urlopen(url, timeout=2) as response: - self.assertEqual(response.status, 200) - self.assertEqual(response.headers[METRICS_STATE_HEADER], "fresh") - self.assertEqual(response.headers["Age"], "0") - self.assertIsNone(response.headers["Warning"]) - self.assertEqual(response.headers["Cache-Control"], "no-store") - self.assertEqual(response.read(), b"qbit_prism_cached_fixture 1\n") - - # A stale complete payload is served, not refused: 200 is what keeps - # Prometheus from discarding the last document that still exists. - port.metrics_response = MetricsSnapshotResponse( - status=200, - body="qbit_prism_cached_fixture 1\n", - state=METRICS_STATE_STALE, - age_seconds=42, - ) - with urllib.request.urlopen(url, timeout=2) as response: - self.assertEqual(response.status, 200) - self.assertEqual(response.headers[METRICS_STATE_HEADER], "stale") - self.assertEqual(response.headers["Age"], "42") - self.assertEqual(response.headers["Warning"], METRICS_STALE_WARNING) - self.assertEqual( - response.headers["Content-Type"], - "text/plain; version=0.0.4", - ) - # The prior complete document, unchanged. - self.assertEqual(response.read(), b"qbit_prism_cached_fixture 1\n") - - # Warm-up still refuses, and says so out of band. No Age and no stale - # warning: there is no served payload for either to describe. - port.metrics_response = MetricsSnapshotResponse( - status=503, - body="qbit_prism_metrics_snapshot_available 0\n", - state=METRICS_STATE_UNAVAILABLE, - age_seconds=None, - ) - with self.assertRaises(urllib.error.HTTPError) as raised: - urllib.request.urlopen(url, timeout=2) - refusal = raised.exception - self.addCleanup(refusal.close) - self.assertEqual(refusal.status, 503) - self.assertEqual(refusal.headers[METRICS_STATE_HEADER], "unavailable") - self.assertIsNone(refusal.headers["Age"]) - self.assertIsNone(refusal.headers["Warning"]) - self.assertEqual( - refusal.read(), - b"qbit_prism_metrics_snapshot_available 0\n", - ) - self.assertEqual(port.metrics_calls, 3) - - def test_unexpected_serve_exit_closes_listener_before_restart(self) -> None: - class ExitServer: - instances: list[ExitServer] = [] - - def __init__(self, address: tuple[str, int], _handler: object) -> None: - if self.instances and not self.instances[-1].closed.is_set(): - raise OSError("previous listener is still open") - self.server_address = address - self.ready = threading.Event() - self.release = threading.Event() - self.closed = threading.Event() - self.instances.append(self) - - def serve_unless_startup_cancelled(self, *, poll_interval: float) -> None: - del poll_interval - self.ready.set() - self.release.wait(2.0) - - def cancel_startup(self) -> bool: - return True - - def shutdown(self) -> None: - self.release.set() - - def server_close(self) -> None: - self.closed.set() - - facade = AuditHttpFacade( - FakeAuditHttpPort(), # type: ignore[arg-type] - AuditHttpConfig("127.0.0.1", 0), - ) - with patch( - "lab.prism.audit_http._BoundedThreadingHttpServer", - ExitServer, - ): - self.assertEqual(facade.start().lifecycle, "running") - first = ExitServer.instances[0] - first.release.set() - self.assertTrue(first.closed.wait(1.0)) - deadline = time.monotonic() + 1.0 - while facade.state().bound_address is not None: - if time.monotonic() >= deadline: - self.fail("exited audit HTTP listener was not retired") - time.sleep(0.01) - - exited = facade.state() - self.assertEqual(exited.lifecycle, "exited") - self.assertFalse(exited.thread_alive) - self.assertEqual(facade.start().lifecycle, "running") - self.assertEqual(len(ExitServer.instances), 2) - self.assertTrue(facade.stop()) - - def test_stop_before_start_is_safe(self) -> None: - facade = AuditHttpFacade(FakeAuditHttpPort()) # type: ignore[arg-type] - - self.assertTrue(facade.stop()) - self.assertEqual(facade.state().lifecycle, "stopped") - - def test_active_request_thread_does_not_delay_listener_shutdown(self) -> None: - port = FakeAuditHttpPort() - port.health_entered = threading.Event() - port.health_release = threading.Event() - facade = AuditHttpFacade( - port, # type: ignore[arg-type] - AuditHttpConfig("127.0.0.1", 0), - ) - state = facade.start() - assert state.bound_address is not None - outcome: list[object] = [] - - def request() -> None: - try: - with urllib.request.urlopen( - f"http://127.0.0.1:{state.bound_address[1]}/healthz", - timeout=2, - ) as response: - outcome.append(response.read()) - except Exception as exc: - outcome.append(exc) - - request_thread = threading.Thread(target=request) - request_thread.start() - self.assertTrue(port.health_entered.wait(1.0)) - - self.assertTrue(facade.stop()) - self.assertFalse(port.health_release.is_set()) - - port.health_release.set() - request_thread.join(2.0) - self.assertFalse(request_thread.is_alive()) - self.assertEqual(outcome, [b'{"ok": true, "schema": "health-fixture"}\n']) - - def test_bind_failure_does_not_publish_a_thread_or_socket(self) -> None: - first = AuditHttpFacade( - FakeAuditHttpPort(), # type: ignore[arg-type] - AuditHttpConfig("127.0.0.1", 0), - ) - first_state = first.start() - assert first_state.bound_address is not None - second = AuditHttpFacade( - FakeAuditHttpPort(), # type: ignore[arg-type] - AuditHttpConfig("127.0.0.1", first_state.bound_address[1]), - ) - try: - with self.assertRaises(OSError): - second.start() - failed = second.state() - self.assertEqual(failed.lifecycle, "new") - self.assertFalse(failed.thread_alive) - self.assertIsNone(failed.bound_address) - finally: - self.assertTrue(second.stop()) - self.assertTrue(first.stop()) - - def test_stop_racing_startup_before_serve_commit_returns_within_bound( - self, - ) -> None: - """The documented old-layer gap: stop() used to call the blocking - BaseServer.shutdown() whenever the serve thread was alive, so a stop - racing startup before serve_forever() committed waited forever on a - loop that never starts. The bounded stop must return within its - configured budget and still release the real listener socket.""" - real_thread = threading.Thread - stalled_started = threading.Event() - release_stall = threading.Event() - - class StalledThread(real_thread): - def run(self) -> None: - stalled_started.set() - release_stall.wait(5.0) - super().run() - - class RecordingBounded(_BoundedThreadingHttpServer): - instances: list[_BoundedThreadingHttpServer] = [] - - def __init__(self, *args: object, **kwargs: object) -> None: - super().__init__(*args, **kwargs) # type: ignore[arg-type] - RecordingBounded.instances.append(self) - - facade = AuditHttpFacade( - FakeAuditHttpPort(), # type: ignore[arg-type] - AuditHttpConfig("127.0.0.1", 0, join_timeout_seconds=0.25), - ) - start_outcome: list[object] = [] - - def run_start() -> None: - try: - start_outcome.append(facade.start()) - except Exception as exc: - start_outcome.append(exc) - - starter = real_thread(target=run_start) - with patch("lab.prism.audit_http.threading.Thread", StalledThread), patch( - "lab.prism.audit_http._BoundedThreadingHttpServer", - RecordingBounded, - ): - starter.start() - self.assertTrue(stalled_started.wait(2.0)) - # The serve thread is alive but has not committed to - # serve_forever(); stop() must still return within its bound. - stop_started = time.monotonic() - stopped = facade.stop() - elapsed = time.monotonic() - stop_started - - self.assertLess(elapsed, 1.5) - self.assertFalse(stopped) - self.assertEqual(facade.state().lifecycle, "stop_timeout") - - release_stall.set() - starter.join(5.0) - self.assertFalse(starter.is_alive()) - self.assertEqual(len(start_outcome), 1) - self.assertIsInstance(start_outcome[0], RuntimeError) - self.assertEqual(len(RecordingBounded.instances), 1) - # The cancelled serve thread never entered serve_forever, and the - # listener socket is closed so a successor can bind the port. - self.assertEqual(RecordingBounded.instances[0].socket.fileno(), -1) - state = facade.state() - self.assertEqual(state.lifecycle, "stopped") - self.assertFalse(state.thread_alive) - self.assertIsNone(state.bound_address) - - def test_stop_reports_timeout_when_committed_loop_refuses_to_exit(self) -> None: - """A committed serve loop that ignores shutdown must not block stop() - past its budget; the facade reports stop_timeout (the state the - coordinator's audit_http_stop shutdown log is keyed on).""" - - class WedgedServer: - instance: WedgedServer | None = None - - def __init__(self, address: tuple[str, int], _handler: object) -> None: - self.server_address = address - self.ready = threading.Event() - self.release = threading.Event() - self.closed = False - WedgedServer.instance = self - - def serve_unless_startup_cancelled(self, *, poll_interval: float) -> None: - del poll_interval - self.ready.set() - self.release.wait(5.0) - - def cancel_startup(self) -> bool: - return True - - def shutdown(self) -> None: - # Deliberately ignore the request: the loop stays wedged. - return None - - def server_close(self) -> None: - self.closed = True - - facade = AuditHttpFacade( - FakeAuditHttpPort(), # type: ignore[arg-type] - AuditHttpConfig("127.0.0.1", 0, join_timeout_seconds=0.25), - ) - with patch( - "lab.prism.audit_http._BoundedThreadingHttpServer", - WedgedServer, - ): - self.assertEqual(facade.start().lifecycle, "running") - stop_started = time.monotonic() - stopped = facade.stop() - elapsed = time.monotonic() - stop_started - - self.assertFalse(stopped) - self.assertLess(elapsed, 1.5) - self.assertEqual(facade.state().lifecycle, "stop_timeout") - assert WedgedServer.instance is not None - self.assertTrue(WedgedServer.instance.closed) - WedgedServer.instance.release.set() - - def test_concurrent_stops_against_a_live_listener_all_return(self) -> None: - facade = AuditHttpFacade( - FakeAuditHttpPort(), # type: ignore[arg-type] - AuditHttpConfig("127.0.0.1", 0), - ) - self.assertEqual(facade.start().lifecycle, "running") - results: list[bool] = [] - errors: list[BaseException] = [] - - def run_stop() -> None: - try: - results.append(facade.stop()) - except BaseException as exc: - errors.append(exc) - - stoppers = [threading.Thread(target=run_stop) for _ in range(3)] - for stopper in stoppers: - stopper.start() - for stopper in stoppers: - stopper.join(5.0) - - self.assertTrue(all(not stopper.is_alive() for stopper in stoppers)) - self.assertEqual(errors, []) - self.assertEqual(len(results), 3) - self.assertTrue(all(results)) - state = facade.state() - self.assertEqual(state.lifecycle, "stopped") - self.assertFalse(state.thread_alive) - - -if __name__ == "__main__": - unittest.main() diff --git a/tests/test_prism_audit_root_scan.py b/tests/test_prism_audit_root_scan.py deleted file mode 100644 index a5539358b..000000000 --- a/tests/test_prism_audit_root_scan.py +++ /dev/null @@ -1,144 +0,0 @@ -#!/usr/bin/env python3 -"""Audit root enumeration through a fresh directory description. - -Some kernels and overlay filesystems serve a stale or truncated ``readdir`` -through a long-lived directory descriptor once the directory has been -mutated after its first enumeration. Observed on an aarch64 Docker host: a -conflict snapshot written moments earlier was absent from the next scan -through the pinned root descriptor, so quarantine deduplication created a -second copy and metrics and retention saw an empty root. The store now -reopens ``.`` relative to the pinned descriptor for every scan, keeping the -inode authority while never trusting the pinned description's own offset -or cache. -""" - -from __future__ import annotations - -import json -import os -from pathlib import Path -import tempfile -import unittest -from unittest import mock -from typing import Any - -from lab.prism.audit_artifacts import AuditArtifactConfig, AuditArtifactStore - - -BLOCK_HASH = "a1" * 32 - - -class FreshDirectoryScanTests(unittest.TestCase): - """Root enumeration never trusts the pinned descriptor's own readdir.""" - - def make_store(self, root: Path) -> AuditArtifactStore: - store = AuditArtifactStore( - AuditArtifactConfig( - root=root, - evidence_path=root / "evidence.json", - share_segment_size=4, - candidate_retention_seconds=0, - ), - canonicalizer=lambda bundle: json.dumps(bundle, separators=(",", ":")).encode(), - ) - self.addCleanup(store.close) - return store - - def test_scans_survive_a_stale_pinned_descriptor_enumeration(self) -> None: - with tempfile.TemporaryDirectory() as tmp: - root = Path(tmp) - store = self.make_store(root) - real_listdir = os.listdir - stale_reads = 0 - - def stale_listdir(target: Any = None) -> list[str]: - nonlocal stale_reads - if isinstance(target, int) and target == store._root_fd: - # The defect this guards against: a long-lived directory - # description that stops reporting entries created after - # its first enumeration. - stale_reads += 1 - return [] - return real_listdir(target) - - rows = [{"share_seq": seq, "worker": f"miner-{seq}"} for seq in (1, 2)] - with mock.patch("lab.prism.audit_artifacts.os.listdir", side_effect=stale_listdir): - uri, _digest = store.write_audit_share_segment_range( - segment_first_share_seq=1, - segment_last_share_seq=4, - first_share_seq=1, - last_share_seq=2, - shares=rows, - ) - slot_bytes = Path(uri).read_bytes() - first = store.quarantine_audit_share_segment(Path(uri), expected_bytes=slot_bytes) - second = store.quarantine_audit_share_segment(Path(uri), expected_bytes=slot_bytes) - self.assertEqual(first, second) - self.assertEqual(len(list(root.glob("*.conflict-*"))), 1) - metrics = store.metrics_snapshot() - self.assertEqual(metrics["scan_error"], 0) - self.assertEqual(metrics["share_segment"]["files"], 1) - stale = store.issue_candidate(block_hash=BLOCK_HASH) - stale.path.write_bytes(b"stale") - store.release_candidate(stale) - os.utime(stale.path, ns=(1, 1)) - result = store.prune_best_effort() - self.assertEqual(result.errors, 0) - self.assertEqual(result.candidate_removed, 1) - self.assertFalse(stale.path.exists()) - self.assertEqual(stale_reads, 0) - - def test_scan_rejects_a_swapped_root_before_enumerating(self) -> None: - with tempfile.TemporaryDirectory() as tmp: - base = Path(tmp) - root = base / "audit" - store = self.make_store(root) - (root / f"prism-audit-bundle-body-{BLOCK_HASH}-{'11' * 32}.json").write_bytes(b"x") - self.assertEqual(store.metrics_snapshot()["body"]["files"], 1) - root.rename(base / "pinned") - root.mkdir() - try: - metrics = store.metrics_snapshot() - self.assertEqual(metrics["scan_error"], 1) - with self.assertRaises(RuntimeError): - store._scan_root_names() - finally: - root.rmdir() - (base / "pinned").rename(root) - self.assertEqual(store.metrics_snapshot()["body"]["files"], 1) - - def test_scan_opens_and_closes_a_fresh_description_each_time(self) -> None: - with tempfile.TemporaryDirectory() as tmp: - root = Path(tmp) - store = self.make_store(root) - (root / "prism-audit-bundle-body-operator.json").write_bytes(b"x") - opened: list[int] = [] - real_open = os.open - real_close = os.close - - def recording_open(*args: Any, **kwargs: Any) -> int: - fd = real_open(*args, **kwargs) - if kwargs.get("dir_fd") == store._root_fd and args[0] == ".": - opened.append(fd) - return fd - - closed: list[int] = [] - - def recording_close(fd: int) -> None: - closed.append(fd) - real_close(fd) - - with mock.patch("lab.prism.audit_artifacts.os.open", side_effect=recording_open), mock.patch( - "lab.prism.audit_artifacts.os.close", - side_effect=recording_close, - ): - names = store._scan_root_names() - store.metrics_snapshot() - self.assertIn("prism-audit-bundle-body-operator.json", names) - self.assertEqual(len(opened), 2) - self.assertTrue(all(fd in closed for fd in opened)) - self.assertNotIn(store._root_fd, closed) - - -if __name__ == "__main__": - unittest.main() diff --git a/tests/test_prism_backfill_audit_bundle_canonical.py b/tests/test_prism_backfill_audit_bundle_canonical.py deleted file mode 100644 index 12f8f50ed..000000000 --- a/tests/test_prism_backfill_audit_bundle_canonical.py +++ /dev/null @@ -1,1215 +0,0 @@ -#!/usr/bin/env python3 - -import copy -import hashlib -import io -import json -import re -import tempfile -import unittest -from dataclasses import dataclass, field -from pathlib import Path -from typing import Any -from unittest import mock - -from lab.prism.audit_artifacts import AuditArtifactConfig, AuditArtifactStore -from lab.prism.backfill_audit_bundle_canonical import ( - LEDGER_PREFLIGHT_CANDIDATES, - AuditBundleLedgerAdapter, - BackfillWriterLeaseUnavailable, - CanonicalArtifactAdapter, - CanonicalBundleBackfill, - DEFAULT_BATCH_SIZE, - MissingCanonicalCapability, - audit_bundle_page_sql, - build_audit_store_from_env, - build_parser, - ledger_from_args, - main, -) -from lab.prism.share_ledger import PsqlShareLedger - - -AUDIT_SHARE_SEGMENT_SCHEMA = "qbit.prism.audit-share-segment.v1" - - -def sha256_hex(payload: bytes) -> str: - return hashlib.sha256(payload).hexdigest() - - -def canonical_bytes(bundle: dict[str, Any]) -> bytes: - """Stand-in for J1's canonicalizer: deterministic and order-independent.""" - - return json.dumps(bundle, sort_keys=True, separators=(",", ":")).encode("utf-8") - - -def digest_of(bundle: dict[str, Any]) -> str: - return sha256_hex(canonical_bytes(bundle)) - - -def block_hash(index: int) -> str: - return f"{index:064x}" - - -def share(seq: int) -> dict[str, Any]: - return { - "share_seq": seq, - "share_id": f"share-{seq}", - "miner_id": f"miner-{seq % 3}", - "share_difficulty": "1024", - } - - -def bundle_for(hash_hex: str, shares: list[dict[str, Any]]) -> dict[str, Any]: - return { - "schema": "qbit.prism.audit-bundle", - "block_hash": hash_hex, - "found_block": {"block_hash": hash_hex, "block_height": 4242}, - "shares": shares, - } - - -@dataclass -class ExternalBody: - """A compact/v2 external body as the artifact store would resolve it.""" - - bundle_without_shares: dict[str, Any] - shares_key: str - parts: list[dict[str, Any]] = field(default_factory=list) - - -class FakeArtifactStore: - """The assumed phase-2 canonical artifact surface, in memory.""" - - def __init__(self, journal: list[tuple[Any, ...]]) -> None: - self.journal = journal - self.canonical: dict[tuple[str, str], bytes] = {} - self.literal_bodies: dict[str, bytes] = {} - self.external_bodies: dict[str, ExternalBody] = {} - self.canonicalized: list[dict[str, Any]] = [] - self.write_failures: set[tuple[str, str]] = set() - - # -- phase-1 surface, unchanged ------------------------------------ - - def canonical_audit_bundle_bytes(self, bundle: dict[str, Any]) -> bytes: - self.canonicalized.append(copy.deepcopy(bundle)) - return canonical_bytes(bundle) - - def storage_json_bytes(self, payload: dict[str, Any]) -> bytes: - return json.dumps(payload, sort_keys=True, separators=(",", ":")).encode("utf-8") - - def audit_share_segment_payload( - self, - *, - first_share_seq: int, - last_share_seq: int, - shares: list[Any], - ) -> dict[str, Any]: - return { - "schema": AUDIT_SHARE_SEGMENT_SCHEMA, - "first_share_seq": first_share_seq, - "last_share_seq": last_share_seq, - "share_count": len(shares), - "shares": list(shares), - } - - # -- assumed phase-2 surface --------------------------------------- - - def read_canonical_audit_bundle(self, block_hash_hex: str, digest: str) -> bytes | None: - self.journal.append(("read_canonical", block_hash_hex, digest)) - return self.canonical.get((block_hash_hex, digest)) - - def write_canonical_audit_bundle( - self, - block_hash_hex: str, - digest: str, - payload: bytes, - ) -> None: - self.journal.append(("write_canonical", block_hash_hex, digest)) - if (block_hash_hex, digest) in self.write_failures: - raise RuntimeError("canonical artifact write refused") - self.canonical[(block_hash_hex, digest)] = bytes(payload) - - def read_literal_audit_body( - self, - body_uri: object, - *, - expected_sha256: str, - ) -> bytes | None: - self.journal.append(("read_literal", str(body_uri))) - payload = self.literal_bodies.get(str(body_uri)) - if payload is None: - return None - # A verified reader declines bytes that are not the advertised ones. - return payload if sha256_hex(payload) == expected_sha256 else None - - def reconstruct_external_audit_body( - self, - body_uri: object, - *, - expected_sha256: str, - load_missing_range: Any, - ) -> dict[str, Any]: - self.journal.append(("reconstruct", str(body_uri))) - body = self.external_bodies.get(str(body_uri)) - if body is None: - raise RuntimeError(f"no external audit body at {body_uri}") - shares: list[Any] = [] - for part in body.parts: - if part.get("on_disk", True): - shares.extend(copy.deepcopy(part["shares"])) - continue - declared = { - key: value - for key, value in part.items() - if key not in {"on_disk", "shares"} - } - shares.extend( - load_missing_range( - first_share_seq=part["first_share_seq"], - last_share_seq=part["last_share_seq"], - part=declared, - ) - ) - assembled = dict(body.bundle_without_shares) - assembled[body.shares_key] = shares - return assembled - - -class FakeLedger: - """The PostgreSQL surface the backfill depends on, in memory.""" - - def __init__( - self, - journal: list[tuple[Any, ...]], - rows: list[dict[str, Any]], - shares: dict[int, dict[str, Any]] | None = None, - ) -> None: - self.journal = journal - self.rows = {str(row["block_hash"]): row for row in rows} - self.shares = shares or {} - self.page_requests: list[tuple[str | None, int]] = [] - self.preflight_failures: set[tuple[str, str]] = set() - self.preflight_results: dict[tuple[str, str], Any] = {} - - @staticmethod - def _text_literal(value: str) -> str: - return "'" + str(value).replace("'", "''") + "'" - - def _run_read_json(self, sql: str) -> list[dict[str, Any]]: - if "FROM qbit_pool_audit_bundles" not in sql: - raise AssertionError("page query does not read the audit bundle table") - if "ORDER BY block_hash ASC" not in sql: - raise AssertionError("page query is not ordered by the primary key") - limit_match = re.search(r"LIMIT (\d+)", sql) - if limit_match is None: - raise AssertionError("page query is not bounded") - limit = int(limit_match.group(1)) - cursor_match = re.search(r"WHERE block_hash > '([^']*)'", sql) - start_after = cursor_match.group(1) if cursor_match else None - self.page_requests.append((start_after, limit)) - keys = sorted(self.rows) - if start_after is not None: - keys = [key for key in keys if key > start_after] - return [copy.deepcopy(self.rows[key]) for key in keys[:limit]] - - def load_audit_share_ledger_range( - self, - *, - first_share_seq: int, - last_share_seq: int, - ) -> list[dict[str, Any]]: - self.journal.append(("load_share_range", first_share_seq, last_share_seq)) - return [ - copy.deepcopy(self.shares[seq]) - for seq in range(first_share_seq, last_share_seq + 1) - if seq in self.shares - ] - - def preflight_audit_bundle_publication( - self, - *, - block_hash: str, - audit_bundle_sha256: str, - ) -> Any: - self.journal.append(("preflight", block_hash, audit_bundle_sha256)) - if (block_hash, audit_bundle_sha256) in self.preflight_failures: - raise RuntimeError("writer lease is not active") - return self.preflight_results.get( - (block_hash, audit_bundle_sha256), - {"confirmed": True}, - ) - - -def inline_row(index: int, *, digest: str | None = None) -> dict[str, Any]: - hash_hex = block_hash(index) - bundle = bundle_for(hash_hex, [share(index)]) - return { - "block_hash": hash_hex, - "audit_bundle_sha256": digest or digest_of(bundle), - "body_uri": None, - "audit_bundle": bundle, - } - - -class BackfillHarness: - """Wire fakes to the backfill under test and expose what each run did.""" - - def __init__( - self, - rows: list[dict[str, Any]], - shares: dict[int, dict[str, Any]] | None = None, - ) -> None: - self.journal: list[tuple[Any, ...]] = [] - self.store = FakeArtifactStore(self.journal) - self.ledger = FakeLedger(self.journal, rows, shares) - self.stderr = io.StringIO() - - def run(self, **kwargs: Any) -> dict[str, Any]: - backfill = CanonicalBundleBackfill( - artifacts=CanonicalArtifactAdapter(self.store), - ledger=AuditBundleLedgerAdapter(self.ledger), - stderr=self.stderr, - **kwargs, - ) - return backfill.run() - - def publication_events(self) -> list[tuple[Any, ...]]: - return [ - event - for event in self.journal - if event[0] in {"preflight", "write_canonical"} - ] - - def errors(self) -> str: - return self.stderr.getvalue() - - -class CanonicalBundleBackfillTest(unittest.TestCase): - def test_inline_history_publishes_through_the_real_phase_2_store(self) -> None: - row = inline_row(1) - journal: list[tuple[Any, ...]] = [] - ledger = FakeLedger(journal, [row]) - with tempfile.TemporaryDirectory() as tmp: - root = Path(tmp) - store = AuditArtifactStore( - AuditArtifactConfig( - root=root, - evidence_path=root / "evidence.json", - ), - canonicalizer=canonical_bytes, - ) - try: - result = CanonicalBundleBackfill( - artifacts=CanonicalArtifactAdapter(store), - ledger=AuditBundleLedgerAdapter(ledger), - ).run() - - self.assertEqual(result["written"], 1) - self.assertEqual( - store.read_canonical_audit_bundle( - str(row["block_hash"]), - str(row["audit_bundle_sha256"]), - ), - canonical_bytes(row["audit_bundle"]), - ) - self.assertEqual( - journal[-1], - ( - "preflight", - row["block_hash"], - row["audit_bundle_sha256"], - ), - ) - finally: - store.close() - - def test_first_run_publishes_every_shape_and_rerun_is_idempotent(self) -> None: - inline = inline_row(1) - - literal_hash = block_hash(2) - # Deliberately not what any canonicalizer here would emit: promotion - # must copy these bytes, not re-derive them. - literal_payload = b'{ "legacy": true, "block_hash": "' + literal_hash.encode() + b'" }\n' - literal = { - "block_hash": literal_hash, - "audit_bundle_sha256": sha256_hex(literal_payload), - "body_uri": f"prism-audit-bundle-body-{literal_hash}.json", - "audit_bundle": None, - } - - compact_hash = block_hash(3) - compact_bundle = bundle_for(compact_hash, [share(7), share(8)]) - compact = { - "block_hash": compact_hash, - "audit_bundle_sha256": digest_of(compact_bundle), - "body_uri": f"prism-audit-bundle-body-{compact_hash}.json", - "audit_bundle": None, - } - - harness = BackfillHarness([inline, literal, compact]) - harness.store.literal_bodies[str(literal["body_uri"])] = literal_payload - harness.store.external_bodies[str(compact["body_uri"])] = ExternalBody( - bundle_without_shares={ - key: value for key, value in compact_bundle.items() if key != "shares" - }, - shares_key="shares", - parts=[ - { - "kind": "segment", - "first_share_seq": 7, - "last_share_seq": 8, - "shares": [share(7), share(8)], - "on_disk": True, - } - ], - ) - - first = harness.run() - self.assertEqual(first["visited"], 3) - self.assertEqual(first["written"], 3) - self.assertEqual(first["already_present"], 0) - self.assertEqual(first["literal_freebie"], 1) - self.assertEqual(first["reconstructed"], 1) - self.assertEqual(first["inline_canonicalized"], 1) - self.assertEqual(first["mismatch"], 0) - self.assertEqual(first["errors"], 0) - self.assertEqual(first["exit_code"], 0) - self.assertEqual(first["last_checkpoint"], compact_hash) - self.assertEqual(harness.errors(), "") - - published = dict(harness.store.canonical) - self.assertEqual(len(published), 3) - for (_hash, digest), payload in published.items(): - self.assertEqual(sha256_hex(payload), digest) - - # An identical rerun against the same store republishes nothing. - rerun_journal_start = len(harness.journal) - second = harness.run() - self.assertEqual(second["visited"], 3) - self.assertEqual(second["already_present"], 3) - self.assertEqual(second["written"], 0) - self.assertEqual(second["mismatch"], 0) - self.assertEqual(second["errors"], 0) - self.assertEqual(second["exit_code"], 0) - self.assertEqual(dict(harness.store.canonical), published) - rerun_events = [ - event - for event in harness.journal[rerun_journal_start:] - if event[0] in {"preflight", "write_canonical"} - ] - self.assertEqual(rerun_events, []) - - def test_pre_v2_literal_body_is_promoted_without_reserialization(self) -> None: - hash_hex = block_hash(11) - # Whitespace and key order a re-serialization would normalize away. - literal_payload = b'{\n "shares": [],\n "block_hash": "' + hash_hex.encode() + b'"\n}\n' - digest = sha256_hex(literal_payload) - row = { - "block_hash": hash_hex, - "audit_bundle_sha256": digest, - "body_uri": f"prism-audit-bundle-body-{hash_hex}.json", - "audit_bundle": None, - } - harness = BackfillHarness([row]) - harness.store.literal_bodies[str(row["body_uri"])] = literal_payload - - summary = harness.run() - - self.assertEqual(summary["written"], 1) - self.assertEqual(summary["literal_freebie"], 1) - self.assertEqual(summary["reconstructed"], 0) - self.assertEqual(summary["inline_canonicalized"], 0) - self.assertEqual(harness.store.canonical[(hash_hex, digest)], literal_payload) - # The canonicalizer is the proof: promoting must not parse or re-emit. - self.assertEqual(harness.store.canonicalized, []) - self.assertNotIn("reconstruct", [event[0] for event in harness.journal]) - - def test_modern_reconstruction_reloads_absent_segment_from_database(self) -> None: - hash_hex = block_hash(21) - on_disk = [share(1), share(2)] - from_db = [share(3), share(4)] - full_bundle = bundle_for(hash_hex, on_disk + from_db) - digest = digest_of(full_bundle) - row = { - "block_hash": hash_hex, - "audit_bundle_sha256": digest, - "body_uri": f"prism-audit-bundle-body-{hash_hex}.json", - "audit_bundle": None, - } - harness = BackfillHarness( - [row], - shares={item["share_seq"]: item for item in from_db}, - ) - missing_part = { - "kind": "segment_range", - "first_share_seq": 3, - "last_share_seq": 4, - "range_sha256": sha256_hex( - harness.store.storage_json_bytes( - harness.store.audit_share_segment_payload( - first_share_seq=3, - last_share_seq=4, - shares=from_db, - ) - ) - ), - "on_disk": False, - } - harness.store.external_bodies[str(row["body_uri"])] = ExternalBody( - bundle_without_shares={ - key: value for key, value in full_bundle.items() if key != "shares" - }, - shares_key="shares", - parts=[ - { - "kind": "segment", - "first_share_seq": 1, - "last_share_seq": 2, - "shares": on_disk, - "on_disk": True, - }, - missing_part, - ], - ) - - summary = harness.run() - - self.assertEqual(summary["written"], 1) - self.assertEqual(summary["reconstructed"], 1) - self.assertEqual(summary["db_segment_reloads"], 1) - self.assertEqual(summary["literal_freebie"], 0) - self.assertEqual(summary["mismatch"], 0) - self.assertEqual(summary["errors"], 0) - self.assertIn(("load_share_range", 3, 4), harness.journal) - self.assertEqual( - harness.store.canonical[(hash_hex, digest)], - canonical_bytes(full_bundle), - ) - - def test_database_range_that_fails_its_segment_digest_publishes_nothing(self) -> None: - hash_hex = block_hash(22) - on_disk = [share(1)] - expected_db = [share(3), share(4)] - full_bundle = bundle_for(hash_hex, on_disk + expected_db) - digest = digest_of(full_bundle) - row = { - "block_hash": hash_hex, - "audit_bundle_sha256": digest, - "body_uri": f"prism-audit-bundle-body-{hash_hex}.json", - "audit_bundle": None, - } - tampered = [dict(expected_db[0], miner_id="someone-else"), expected_db[1]] - harness = BackfillHarness( - [row], - shares={item["share_seq"]: item for item in tampered}, - ) - harness.store.external_bodies[str(row["body_uri"])] = ExternalBody( - bundle_without_shares={ - key: value for key, value in full_bundle.items() if key != "shares" - }, - shares_key="shares", - parts=[ - { - "kind": "segment", - "first_share_seq": 1, - "last_share_seq": 1, - "shares": on_disk, - "on_disk": True, - }, - { - "kind": "segment_range", - "first_share_seq": 3, - "last_share_seq": 4, - "range_sha256": sha256_hex( - harness.store.storage_json_bytes( - harness.store.audit_share_segment_payload( - first_share_seq=3, - last_share_seq=4, - shares=expected_db, - ) - ) - ), - "on_disk": False, - }, - ], - ) - - summary = harness.run() - - self.assertEqual(summary["mismatch"], 1) - self.assertEqual(summary["written"], 0) - self.assertEqual(summary["db_segment_reloads"], 0) - self.assertEqual(summary["exit_code"], 1) - self.assertEqual(harness.store.canonical, {}) - self.assertIn("canonicalizes to", harness.errors()) - - def test_incomplete_database_range_is_refused(self) -> None: - hash_hex = block_hash(23) - full_bundle = bundle_for(hash_hex, [share(3), share(4)]) - row = { - "block_hash": hash_hex, - "audit_bundle_sha256": digest_of(full_bundle), - "body_uri": f"prism-audit-bundle-body-{hash_hex}.json", - "audit_bundle": None, - } - # The ledger can only serve half of the absent range. - harness = BackfillHarness([row], shares={3: share(3)}) - harness.store.external_bodies[str(row["body_uri"])] = ExternalBody( - bundle_without_shares={ - key: value for key, value in full_bundle.items() if key != "shares" - }, - shares_key="shares", - parts=[ - { - "kind": "segment_range", - "first_share_seq": 3, - "last_share_seq": 4, - "on_disk": False, - } - ], - ) - - summary = harness.run() - - self.assertEqual(summary["written"], 0) - self.assertEqual(summary["errors"], 1) - self.assertEqual(summary["exit_code"], 1) - self.assertEqual(harness.store.canonical, {}) - self.assertIn("incomplete or out of order", harness.errors()) - - def test_digest_mismatch_is_loud_and_publishes_nothing(self) -> None: - good = inline_row(31) - bad = inline_row(32, digest="ab" * 32) - harness = BackfillHarness([good, bad]) - - summary = harness.run() - - self.assertEqual(summary["visited"], 2) - self.assertEqual(summary["written"], 1) - self.assertEqual(summary["mismatch"], 1) - self.assertEqual(summary["exit_code"], 1) - self.assertEqual(list(harness.store.canonical), [(good["block_hash"], good["audit_bundle_sha256"])]) - - reported = harness.errors() - self.assertIn(str(bad["block_hash"]), reported) - self.assertIn("ab" * 32, reported) - self.assertIn("inline bundle canonicalizes to", reported) - # Never rewrite the advertised digest to whatever the bytes hash to. - self.assertNotIn(("write_canonical", bad["block_hash"], digest_of(bad["audit_bundle"])), harness.journal) - # The bad row still advances the checkpoint so a resume makes progress. - self.assertEqual(summary["last_checkpoint"], bad["block_hash"]) - self.assertEqual(len(summary["failed_rows"]), 1) - self.assertEqual( - summary["failed_rows"][0]["block_hash"], - bad["block_hash"], - ) - self.assertEqual( - summary["failed_rows"][0]["audit_bundle_sha256"], - bad["audit_bundle_sha256"], - ) - self.assertEqual(summary["failed_rows"][0]["kind"], "mismatch") - - def test_existing_canonical_artifact_with_wrong_bytes_is_reported(self) -> None: - row = inline_row(41) - harness = BackfillHarness([row]) - harness.store.canonical[ - (str(row["block_hash"]), str(row["audit_bundle_sha256"])) - ] = b"not the canonical bytes" - - summary = harness.run() - - self.assertEqual(summary["mismatch"], 1) - self.assertEqual(summary["written"], 0) - self.assertEqual(summary["already_present"], 0) - self.assertEqual(summary["exit_code"], 1) - self.assertIn("existing canonical artifact hashes to", harness.errors()) - # The corrupt artifact is left exactly as found, never overwritten. - self.assertEqual( - harness.store.canonical[ - (str(row["block_hash"]), str(row["audit_bundle_sha256"])) - ], - b"not the canonical bytes", - ) - - def test_dry_run_verifies_without_publishing_or_taking_the_lease(self) -> None: - rows = [inline_row(51), inline_row(52)] - harness = BackfillHarness(rows) - - summary = harness.run(dry_run=True) - - self.assertEqual(summary["visited"], 2) - self.assertEqual(summary["dry_run"], 2) - self.assertEqual(summary["written"], 0) - self.assertEqual(summary["mismatch"], 0) - self.assertEqual(summary["errors"], 0) - self.assertEqual(summary["exit_code"], 0) - self.assertTrue(summary["dry_run_mode"]) - self.assertEqual(harness.store.canonical, {}) - # No filesystem publication means no lease refresh either. - self.assertEqual(harness.publication_events(), []) - - def test_dry_run_still_reports_mismatches(self) -> None: - harness = BackfillHarness([inline_row(53, digest="cd" * 32)]) - - summary = harness.run(dry_run=True) - - self.assertEqual(summary["mismatch"], 1) - self.assertEqual(summary["dry_run"], 0) - self.assertEqual(summary["exit_code"], 1) - self.assertEqual(harness.publication_events(), []) - - def test_batches_page_in_resumable_primary_key_order(self) -> None: - indexes = [61, 62, 63, 64, 65] - rows = [inline_row(index) for index in indexes] - # Insertion order is shuffled: the ordering must come from the query. - harness = BackfillHarness([rows[3], rows[0], rows[4], rows[2], rows[1]]) - - summary = harness.run(batch_size=2) - - self.assertEqual(summary["visited"], 5) - self.assertEqual(summary["written"], 5) - self.assertEqual(summary["last_checkpoint"], block_hash(65)) - self.assertEqual( - harness.ledger.page_requests, - [(None, 2), (block_hash(62), 2), (block_hash(64), 2)], - ) - written = [ - event[1] for event in harness.journal if event[0] == "write_canonical" - ] - self.assertEqual(written, [block_hash(index) for index in indexes]) - - def test_limit_stops_early_and_start_after_resumes_from_the_checkpoint(self) -> None: - indexes = [71, 72, 73, 74, 75] - rows = [inline_row(index) for index in indexes] - - first = BackfillHarness(rows) - first_summary = first.run(batch_size=2, limit=3) - self.assertEqual(first_summary["visited"], 3) - self.assertEqual(first_summary["written"], 3) - self.assertEqual(first_summary["last_checkpoint"], block_hash(73)) - self.assertEqual(first_summary["limit"], 3) - self.assertEqual(first.ledger.page_requests, [(None, 2), (block_hash(72), 1)]) - - resumed = BackfillHarness(rows) - resumed_summary = resumed.run( - batch_size=2, - start_after=first_summary["last_checkpoint"], - ) - self.assertEqual(resumed_summary["visited"], 2) - self.assertEqual(resumed_summary["written"], 2) - self.assertEqual(resumed_summary["start_after"], block_hash(73)) - self.assertEqual(resumed_summary["last_checkpoint"], block_hash(75)) - self.assertEqual( - [event[1] for event in resumed.journal if event[0] == "write_canonical"], - [block_hash(74), block_hash(75)], - ) - self.assertEqual(resumed.ledger.page_requests[0], (block_hash(73), 2)) - - def test_resuming_past_the_last_row_is_a_clean_no_op(self) -> None: - harness = BackfillHarness([inline_row(81)]) - - summary = harness.run(start_after=block_hash(81)) - - self.assertEqual(summary["visited"], 0) - self.assertEqual(summary["written"], 0) - self.assertEqual(summary["exit_code"], 0) - self.assertEqual(summary["last_checkpoint"], block_hash(81)) - - def test_lease_preflight_runs_immediately_before_every_publication(self) -> None: - rows = [inline_row(91), inline_row(92)] - harness = BackfillHarness(rows) - - summary = harness.run() - - self.assertEqual(summary["written"], 2) - self.assertEqual( - harness.publication_events(), - [ - ("preflight", rows[0]["block_hash"], rows[0]["audit_bundle_sha256"]), - ("write_canonical", rows[0]["block_hash"], rows[0]["audit_bundle_sha256"]), - ("preflight", rows[1]["block_hash"], rows[1]["audit_bundle_sha256"]), - ("write_canonical", rows[1]["block_hash"], rows[1]["audit_bundle_sha256"]), - ], - ) - - def test_preflight_refusal_withholds_the_publication(self) -> None: - rows = [inline_row(101), inline_row(102)] - harness = BackfillHarness(rows) - harness.ledger.preflight_failures.add( - (str(rows[0]["block_hash"]), str(rows[0]["audit_bundle_sha256"])) - ) - - summary = harness.run() - - self.assertEqual(summary["written"], 1) - self.assertEqual(summary["errors"], 1) - self.assertEqual(summary["exit_code"], 1) - self.assertNotIn( - ("write_canonical", rows[0]["block_hash"], rows[0]["audit_bundle_sha256"]), - harness.journal, - ) - self.assertIn("writer lease is not active", harness.errors()) - # A refusal on one row does not stop the requested range. - self.assertIn( - ("write_canonical", rows[1]["block_hash"], rows[1]["audit_bundle_sha256"]), - harness.journal, - ) - - def test_preflight_error_result_withholds_the_publication(self) -> None: - row = inline_row(103) - harness = BackfillHarness([row]) - harness.ledger.preflight_results[ - (str(row["block_hash"]), str(row["audit_bundle_sha256"])) - ] = {"error": "existing audit bundle does not match payload"} - - summary = harness.run() - - self.assertEqual(summary["written"], 0) - self.assertEqual(summary["errors"], 1) - self.assertEqual(summary["exit_code"], 1) - self.assertEqual(harness.store.canonical, {}) - self.assertIn("existing audit bundle does not match payload", harness.errors()) - - def test_unconfirmed_preflight_withholds_the_publication(self) -> None: - row = inline_row(104) - harness = BackfillHarness([row]) - harness.ledger.preflight_results[ - (str(row["block_hash"]), str(row["audit_bundle_sha256"])) - ] = {"confirmed": False} - - summary = harness.run() - - self.assertEqual(summary["written"], 0) - self.assertEqual(summary["errors"], 1) - self.assertEqual(harness.store.canonical, {}) - - def test_row_without_any_recoverable_body_is_reported(self) -> None: - hash_hex = block_hash(111) - row = { - "block_hash": hash_hex, - "audit_bundle_sha256": digest_of(bundle_for(hash_hex, [])), - "body_uri": None, - "audit_bundle": None, - } - harness = BackfillHarness([row]) - - summary = harness.run() - - self.assertEqual(summary["errors"], 1) - self.assertEqual(summary["written"], 0) - self.assertEqual(summary["exit_code"], 1) - self.assertIn("neither an inline bundle nor an external body", harness.errors()) - - def test_non_canonical_row_identity_is_reported(self) -> None: - row = inline_row(112) - row["audit_bundle_sha256"] = "not-hex" - harness = BackfillHarness([row]) - - summary = harness.run() - - self.assertEqual(summary["errors"], 1) - self.assertEqual(summary["written"], 0) - self.assertEqual(summary["exit_code"], 1) - self.assertIn("row identity is not canonical", harness.errors()) - - def test_unreadable_external_body_falls_back_to_the_inline_bundle(self) -> None: - hash_hex = block_hash(121) - bundle = bundle_for(hash_hex, [share(5)]) - row = { - "block_hash": hash_hex, - "audit_bundle_sha256": digest_of(bundle), - # Registered nowhere: the literal reader declines and the - # reconstruction helper raises. - "body_uri": f"prism-audit-bundle-body-{hash_hex}.json", - "audit_bundle": bundle, - } - harness = BackfillHarness([row]) - - summary = harness.run() - - self.assertEqual(summary["written"], 1) - self.assertEqual(summary["inline_canonicalized"], 1) - self.assertEqual(summary["mismatch"], 0) - self.assertEqual(summary["errors"], 0) - self.assertEqual( - harness.store.canonical[(hash_hex, str(row["audit_bundle_sha256"]))], - canonical_bytes(bundle), - ) - - def test_store_write_failure_is_counted_and_scanning_continues(self) -> None: - rows = [inline_row(131), inline_row(132)] - harness = BackfillHarness(rows) - harness.store.write_failures.add( - (str(rows[0]["block_hash"]), str(rows[0]["audit_bundle_sha256"])) - ) - - summary = harness.run() - - self.assertEqual(summary["written"], 1) - self.assertEqual(summary["errors"], 1) - self.assertEqual(summary["exit_code"], 1) - self.assertEqual(summary["last_checkpoint"], rows[1]["block_hash"]) - - def test_summary_is_machine_readable_and_names_the_bound_interface(self) -> None: - harness = BackfillHarness([inline_row(141)]) - - summary = harness.run() - - for key in ( - "schema", - "visited", - "written", - "already_present", - "dry_run", - "literal_freebie", - "mismatch", - "errors", - "last_checkpoint", - "exit_code", - "failed_rows", - ): - self.assertIn(key, summary) - self.assertEqual( - summary["schema"], - "qbit.prism.audit-bundle-canonical-backfill.v1", - ) - # The summary must survive a round trip through JSON unchanged. - self.assertEqual(json.loads(json.dumps(summary, sort_keys=True)), summary) - self.assertEqual( - summary["interface"]["artifacts"]["read_canonical"], - "read_canonical_audit_bundle", - ) - self.assertEqual( - summary["interface"]["ledger"]["publication_preflight"], - "preflight_audit_bundle_publication", - ) - self.assertEqual( - summary["interface"]["ledger"]["preflight_digest_keyword"], - "audit_bundle_sha256", - ) - - -class WithoutAttributes: - """A build that simply does not have some of the assumed methods.""" - - def __init__(self, target: Any, *hidden: str) -> None: - self.__dict__["_target"] = target - self.__dict__["_hidden"] = frozenset(hidden) - - def __getattr__(self, name: str) -> Any: - if name in self.__dict__["_hidden"]: - raise AttributeError(name) - return getattr(self.__dict__["_target"], name) - - -class AdapterBindingTest(unittest.TestCase): - def test_phase_2_production_interfaces_bind_and_round_trip(self) -> None: - with tempfile.TemporaryDirectory() as tmp: - root = Path(tmp) - store = AuditArtifactStore( - AuditArtifactConfig( - root=root, - evidence_path=root / "evidence.json", - ), - canonicalizer=canonical_bytes, - ) - try: - artifacts = CanonicalArtifactAdapter(store) - block = "ab" * 32 - payload = b'{"schema":"qbit.prism.audit-bundle.v1"}' - digest = sha256_hex(payload) - - artifacts.write_canonical(block, digest, payload) - - self.assertEqual(artifacts.read_canonical(block, digest), payload) - self.assertEqual( - artifacts.resolved_interface()["literal_body_reader"], - "literal_canonical_bundle_bytes", - ) - self.assertEqual( - artifacts.resolved_interface()["external_body_reconstruction"], - "canonical_bundle_bytes_from_external_body", - ) - finally: - store.close() - - ledger = AuditBundleLedgerAdapter(PsqlShareLedger.__new__(PsqlShareLedger)) - self.assertEqual( - ledger.resolved_interface()["publication_preflight"], - "preflight_canonical_bundle_publication", - ) - - def test_missing_canonical_capability_is_named(self) -> None: - journal: list[tuple[Any, ...]] = [] - store = WithoutAttributes( - FakeArtifactStore(journal), - "read_canonical_audit_bundle", - ) - with self.assertRaises(MissingCanonicalCapability) as caught: - CanonicalArtifactAdapter(store) - self.assertIn("read_canonical_audit_bundle", str(caught.exception)) - - def test_missing_ledger_preflight_is_named(self) -> None: - journal: list[tuple[Any, ...]] = [] - ledger = WithoutAttributes( - FakeLedger(journal, []), - *LEDGER_PREFLIGHT_CANDIDATES, - ) - with self.assertRaises(MissingCanonicalCapability) as caught: - AuditBundleLedgerAdapter(ledger) - self.assertIn("preflight", str(caught.exception)) - - def test_reconstruction_helper_without_callback_still_binds(self) -> None: - """A build without the phase-2 helper falls back to the phase-1 reader.""" - - journal: list[tuple[Any, ...]] = [] - target = FakeArtifactStore(journal) - target.read_external_body = ( # type: ignore[attr-defined] - lambda body_uri, *, expected_sha256: None - ) - store = WithoutAttributes(target, "reconstruct_external_audit_body") - adapter = CanonicalArtifactAdapter(store) - self.assertEqual( - adapter.resolved_interface()["external_body_reconstruction"], - "read_external_body", - ) - # The phase-1 reader takes no missing-range callback, so it must not be - # handed one; binding still succeeds and the row simply cannot recover - # an absent segment. - self.assertIsNone( - adapter.canonical_bytes_from_external_body( - "bb" * 32, - "body", - "aa" * 32, - lambda **_: [], - ) - ) - - -class PageQueryTest(unittest.TestCase): - def test_first_page_has_no_cursor_predicate(self) -> None: - sql = audit_bundle_page_sql(start_after=None, limit=25) - self.assertNotIn("WHERE", sql) - self.assertIn("ORDER BY block_hash ASC", sql) - self.assertIn("LIMIT 25", sql) - self.assertIn("FROM qbit_pool_audit_bundles", sql) - - def test_resumed_page_filters_and_escapes_the_cursor(self) -> None: - sql = audit_bundle_page_sql(start_after="a b'c", limit=5) - self.assertIn("WHERE block_hash > 'a b''c'", sql) - self.assertIn("LIMIT 5", sql) - - def test_page_limit_must_be_positive(self) -> None: - with self.assertRaises(ValueError): - audit_bundle_page_sql(start_after=None, limit=0) - - def test_page_query_selects_only_the_columns_the_backfill_reads(self) -> None: - sql = audit_bundle_page_sql(start_after=None, limit=1) - for column in ("block_hash", "audit_bundle_sha256", "body_uri", "audit_bundle"): - self.assertIn(column, sql) - # A backfill never writes the ledger. - for statement in ("INSERT", "UPDATE", "DELETE"): - self.assertNotIn(statement, sql.upper()) - - -class ConstructionTest(unittest.TestCase): - def _backfill(self, **kwargs: Any) -> CanonicalBundleBackfill: - journal: list[tuple[Any, ...]] = [] - return CanonicalBundleBackfill( - artifacts=CanonicalArtifactAdapter(FakeArtifactStore(journal)), - ledger=AuditBundleLedgerAdapter(FakeLedger(journal, [])), - **kwargs, - ) - - def test_batch_size_must_be_positive(self) -> None: - with self.assertRaises(ValueError): - self._backfill(batch_size=0) - - def test_limit_must_be_positive_when_given(self) -> None: - with self.assertRaises(ValueError): - self._backfill(limit=0) - - def test_dry_run_store_has_no_publication_authority(self) -> None: - with tempfile.TemporaryDirectory() as tmp: - root = Path(tmp) - store = build_audit_store_from_env( - { - "PRISM_AUDIT_DIR": str(root), - "PRISM_EVIDENCE_PATH": str(root / "evidence.json"), - }, - read_only=True, - ) - try: - self.assertFalse((root / ".prism-audit-publication.lock").exists()) - payload = b'{"schema":"qbit.prism.audit-bundle.v1"}' - with self.assertRaisesRegex(RuntimeError, "read-only"): - store.write_canonical_audit_bundle( - "ab" * 32, - sha256_hex(payload), - payload, - ) - finally: - store.close() - - def test_dry_run_ledger_uses_read_only_database_sessions(self) -> None: - args = build_parser().parse_args( - [ - "--dry-run", - "--psql-command", - "psql postgresql://example.invalid/qbit", - ] - ) - with mock.patch( - "lab.prism.backfill_audit_bundle_canonical.PsqlShareLedger" - ) as constructor: - ledger_from_args(args, mock.sentinel.store) - - self.assertTrue(constructor.call_args.kwargs["read_only"]) - self.assertFalse(constructor.call_args.kwargs["initialize_schema"]) - - def test_publish_ledger_refuses_same_identity_lease_waits(self) -> None: - args = build_parser().parse_args( - ["--psql-command", "psql postgresql://example.invalid/qbit"] - ) - with mock.patch( - "lab.prism.backfill_audit_bundle_canonical.PsqlShareLedger" - ) as constructor: - ledger_from_args(args, mock.sentinel.store) - - refuse_wait = constructor.call_args.kwargs["lease_retry_sleep"] - with self.assertRaisesRegex( - BackfillWriterLeaseUnavailable, - "stop the coordinator", - ): - refuse_wait(60) - - -class CommandLifecycleTest(unittest.TestCase): - @staticmethod - def resources(journal: list[str]) -> tuple[mock.Mock, mock.Mock]: - store = mock.Mock() - ledger = mock.Mock() - ledger.release_writer_lease.side_effect = lambda: ( - journal.append("release_writer_lease") or True - ) - ledger.close.side_effect = lambda: journal.append("ledger_close") - store.close.side_effect = lambda: journal.append("store_close") - return store, ledger - - def run_main( - self, - *, - store: mock.Mock, - ledger: mock.Mock, - backfill: mock.Mock, - ) -> int: - with ( - mock.patch( - "lab.prism.backfill_audit_bundle_canonical.build_audit_store_from_env", - return_value=store, - ), - mock.patch( - "lab.prism.backfill_audit_bundle_canonical.CanonicalArtifactAdapter", - return_value=mock.sentinel.artifacts, - ), - mock.patch( - "lab.prism.backfill_audit_bundle_canonical.ledger_from_args", - return_value=ledger, - ), - mock.patch( - "lab.prism.backfill_audit_bundle_canonical.AuditBundleLedgerAdapter", - return_value=mock.sentinel.ledger_adapter, - ), - mock.patch( - "lab.prism.backfill_audit_bundle_canonical.CanonicalBundleBackfill", - return_value=backfill, - ), - mock.patch("sys.stdout", new=io.StringIO()), - ): - return main([]) - - def test_success_releases_lease_then_closes_ledger_and_store(self) -> None: - journal: list[str] = [] - store, ledger = self.resources(journal) - backfill = mock.Mock() - backfill.run.return_value = {"exit_code": 0} - - self.assertEqual( - self.run_main(store=store, ledger=ledger, backfill=backfill), - 0, - ) - self.assertEqual( - journal, - ["release_writer_lease", "ledger_close", "store_close"], - ) - - def test_failure_releases_lease_then_closes_ledger_and_store(self) -> None: - journal: list[str] = [] - store, ledger = self.resources(journal) - backfill = mock.Mock() - backfill.run.side_effect = RuntimeError("backfill failed") - - with self.assertRaisesRegex(RuntimeError, "backfill failed"): - self.run_main(store=store, ledger=ledger, backfill=backfill) - self.assertEqual( - journal, - ["release_writer_lease", "ledger_close", "store_close"], - ) - - def test_conflicting_writer_lease_fails_clearly_and_closes_store(self) -> None: - journal: list[str] = [] - store, _ledger = self.resources(journal) - with ( - mock.patch( - "lab.prism.backfill_audit_bundle_canonical.build_audit_store_from_env", - return_value=store, - ), - mock.patch( - "lab.prism.backfill_audit_bundle_canonical.CanonicalArtifactAdapter", - return_value=mock.sentinel.artifacts, - ), - mock.patch( - "lab.prism.backfill_audit_bundle_canonical.ledger_from_args", - side_effect=RuntimeError("lease held by prism-coordinator"), - ), - ): - with self.assertRaisesRegex(SystemExit, "stop the coordinator"): - main([]) - - self.assertEqual(journal, ["store_close"]) - - -class CommandLineTest(unittest.TestCase): - def test_checkpoint_batch_and_dry_run_options_exist(self) -> None: - args = build_parser().parse_args( - ["--start-after", block_hash(1), "--batch-size", "7", "--limit", "3", "--dry-run"] - ) - self.assertEqual(args.start_after, block_hash(1)) - self.assertEqual(args.batch_size, 7) - self.assertEqual(args.limit, 3) - self.assertTrue(args.dry_run) - - def test_defaults_publish_the_whole_table(self) -> None: - args = build_parser().parse_args([]) - self.assertIsNone(args.start_after) - self.assertIsNone(args.limit) - self.assertFalse(args.dry_run) - self.assertEqual(args.batch_size, DEFAULT_BATCH_SIZE) - - def test_established_postgres_options_are_available(self) -> None: - args = build_parser().parse_args([]) - for attribute in ( - "psql_command", - "writer_id", - "writer_epoch", - "writer_session_token", - "lease_ttl_seconds", - "lease_acquire_attempts", - "postgres_tcp_keepalives_count", - ): - self.assertTrue(hasattr(args, attribute), attribute) - - -if __name__ == "__main__": - unittest.main() diff --git a/tests/test_prism_background_services.py b/tests/test_prism_background_services.py deleted file mode 100644 index 37af1d5db..000000000 --- a/tests/test_prism_background_services.py +++ /dev/null @@ -1,1022 +0,0 @@ -#!/usr/bin/env python3 - -from __future__ import annotations - -import threading -from types import SimpleNamespace -import unittest -from unittest.mock import patch - -from lab.prism.background_services import ( - BackgroundServiceRegistry, - BackgroundServiceSpec, - WatchdogPorts, - WatchdogService, -) -from lab.prism.prism_coordinator import PrismCoordinator - - -def make_watchdog_ports(**overrides: object) -> WatchdogPorts: - """Inert watchdog ports; individual tests override what they exercise.""" - ports: dict[str, object] = { - "wait_for_stop": lambda _timeout: True, - "interval_seconds": lambda: 1.0, - "fatal_exit_requested": lambda: False, - "publication_state": lambda _now: (None, 0.0, 1.0, 1.0), - "hard_exit": lambda _reason: None, - "liveness_enabled": lambda: True, - "overdue_heartbeats": lambda _now: [], - "liveness_timeout_seconds": lambda: 60.0, - "coordination_budget_seconds": lambda: 900.0, - "publication_budget_seconds": lambda: 30.0, - "ensure_job_cache_state": lambda: None, - "publication_failure_expired": lambda _now: False, - "publication_divergence_since": lambda: None, - "lease_release_timeout_seconds": lambda: 0.05, - "shutdown_controller": lambda: None, - "request_shutdown": lambda: None, - "release_ledger_lease": lambda _deadline: True, - "lease_failure_reason": lambda: None, - "exit_process": lambda _code: None, - "log": lambda _message: None, - } - ports.update(overrides) - return WatchdogPorts(**ports) # type: ignore[arg-type] - - -class WatchdogServiceTests(unittest.TestCase): - def test_publication_failure_exits_after_one_bounded_wait(self) -> None: - events: list[object] = [] - - def hard_exit(reason: str) -> None: - events.append(("hard_exit", reason)) - raise SystemExit(1) - - service = WatchdogService( - make_watchdog_ports( - wait_for_stop=lambda timeout: events.append(("wait", timeout)) - or False, - interval_seconds=lambda: 2.0, - publication_state=lambda _now: ("publication", 0.0, 900.0, 30.0), - hard_exit=hard_exit, - ) - ) - - with self.assertRaises(SystemExit): - service.run() - - self.assertEqual(events[0], ("wait", 2.0)) - self.assertEqual(events[1], ("hard_exit", "publication")) - self.assertIn( - "publication-progress watchdog firing", - str(service.failure_detail), - ) - - def test_coordination_classification_precedes_publication(self) -> None: - events: list[object] = [] - - def hard_exit(reason: str) -> None: - events.append(("hard_exit", reason)) - raise SystemExit(1) - - service = WatchdogService( - make_watchdog_ports( - wait_for_stop=lambda _timeout: False, - publication_state=lambda _now: ( - "coordination", - 12.5, - 10.0, - 30.0, - ), - hard_exit=hard_exit, - ) - ) - - with self.assertRaises(SystemExit): - service.run() - - self.assertEqual(events, [("hard_exit", "coordination")]) - self.assertIn("coordination-blocked", str(service.failure_detail)) - self.assertIn("streak_age=12.500s", str(service.failure_detail)) - - def test_requested_stop_does_not_sample_or_exit(self) -> None: - service = WatchdogService( - make_watchdog_ports( - wait_for_stop=lambda _timeout: True, - fatal_exit_requested=lambda: False, - publication_state=lambda _now: self.fail( - "stopped watchdog sampled publication state" - ), - hard_exit=lambda _reason: self.fail("stopped watchdog exited"), - exit_process=lambda _code: self.fail("stopped watchdog exited"), - log=lambda _message: self.fail("stopped watchdog logged"), - ) - ) - - service.run() - - def test_fatal_stop_request_exits_nonzero(self) -> None: - events: list[object] = [] - service = WatchdogService( - make_watchdog_ports( - wait_for_stop=lambda _timeout: True, - fatal_exit_requested=lambda: True, - exit_process=lambda code: events.append(("exit", code)), - log=lambda message: events.append(("log", message)), - ) - ) - - service.run() - - self.assertEqual(events[0][0], "log") - self.assertIn("fatal block-work restart requested", str(events[0][1])) - self.assertEqual(events[1], ("exit", 1)) - - def test_claimed_coordination_exit_is_terminal(self) -> None: - service = WatchdogService( - make_watchdog_ports(coordination_budget_seconds=lambda: 5.0) - ) - service.record_coordination_blocked_refresh(100.0) - state = service.publication_watchdog_state(106.0) - self.assertEqual(state[0], "coordination") - # A refresh completing after the claim cannot cancel the exit. - service.clear_coordination_blocked_streak() - self.assertGreater( - service.coordination_blocked_streak_age_seconds(107.0), - 0.0, - ) - - def test_publication_recheck_uses_divergence_snapshot(self) -> None: - # The cheap preflight fired, but the locked divergence recheck shows - # no divergence old enough: no exit may be claimed. - service = WatchdogService( - make_watchdog_ports( - publication_failure_expired=lambda _now: True, - publication_divergence_since=lambda: None, - ) - ) - state = service.publication_watchdog_state(100.0) - self.assertIsNone(state[0]) - divergent = WatchdogService( - make_watchdog_ports( - publication_budget_seconds=lambda: 30.0, - publication_failure_expired=lambda _now: True, - publication_divergence_since=lambda: 50.0, - ) - ) - self.assertEqual(divergent.publication_watchdog_state(100.0)[0], "publication") - - def test_hard_exit_bounds_release_and_exits_unconditionally(self) -> None: - events: list[object] = [] - controller = SimpleNamespace( - begin_shutdown=lambda label: events.append(("begin", label)) or True, - wait_for_writer_quiescence=lambda budget: (True, 0.0, []), - wait_for_lease_handling=lambda: True, - ) - service = WatchdogService( - make_watchdog_ports( - shutdown_controller=lambda: controller, - request_shutdown=lambda: events.append(("request", None)), - release_ledger_lease=lambda deadline: events.append( - ("release", True) - ) - or True, - exit_process=lambda code: events.append(("exit", code)), - log=lambda message: events.append(("log", message)), - lease_release_timeout_seconds=lambda: 0.2, - ) - ) - - service.hard_exit("liveness") - - self.assertEqual(events[0], ("request", None)) - self.assertEqual(events[1], ("begin", "watchdog_liveness")) - self.assertEqual(events[2], ("release", True)) - self.assertEqual(events[3][0], "log") - self.assertIn("lease_handled=True", str(events[3][1])) - self.assertEqual(events[-1], ("exit", 1)) - - def test_hard_exit_withholds_release_when_quiescence_fails(self) -> None: - events: list[object] = [] - controller = SimpleNamespace( - begin_shutdown=lambda _label: True, - wait_for_writer_quiescence=lambda _budget: (False, 0.05, ["writer"]), - ) - service = WatchdogService( - make_watchdog_ports( - shutdown_controller=lambda: controller, - release_ledger_lease=lambda _deadline: events.append( - ("release", None) - ) - or True, - exit_process=lambda code: events.append(("exit", code)), - log=lambda message: events.append(("log", message)), - ) - ) - - service.hard_exit("publication") - - self.assertNotIn(("release", None), events) - self.assertIn("lease_handled=False", str(events[0][1])) - self.assertEqual(events[-1], ("exit", 1)) - - def test_lease_heartbeat_diagnostic_uses_lease_failure_reason(self) -> None: - events: list[object] = [] - controller = SimpleNamespace( - begin_shutdown=lambda _label: True, - wait_for_writer_quiescence=lambda _budget: (True, 0.0, []), - ) - service = WatchdogService( - make_watchdog_ports( - shutdown_controller=lambda: controller, - lease_failure_reason=lambda: "lease heartbeat stopped", - exit_process=lambda code: events.append(("exit", code)), - log=lambda message: events.append(("log", message)), - ) - ) - - service.hard_exit("lease_heartbeat") - - self.assertIn("lease heartbeat stopped", str(events[0][1])) - self.assertEqual(events[-1], ("exit", 1)) - - -class DormantThread: - def __init__( - self, - *, - target: object, - name: str, - daemon: bool, - ) -> None: - self.target = target - self.name = name - self.daemon = daemon - self.start_count = 0 - self.join_timeouts: list[float | None] = [] - - def start(self) -> None: - self.start_count += 1 - - def join(self, timeout: float | None = None) -> None: - self.join_timeouts.append(timeout) - - -class FlakyThreadFactory: - def __init__(self) -> None: - self.attempts = 0 - self.threads: list[DormantThread] = [] - - def __call__(self, **kwargs: object) -> DormantThread: - factory = self - - class FlakyDormantThread(DormantThread): - def start(self) -> None: - factory.attempts += 1 - if factory.attempts == 1: - raise RuntimeError("thread start failed") - super().start() - - thread = FlakyDormantThread(**kwargs) # type: ignore[arg-type] - self.threads.append(thread) - return thread - - -class ContentionObservedLock: - """Lock that exposes an acquire attempt made while another caller owns it.""" - - def __init__(self) -> None: - self._lock = threading.Lock() - self.contended = threading.Event() - - def __enter__(self) -> "ContentionObservedLock": - if self._lock.locked(): - self.contended.set() - self._lock.acquire() - return self - - def __exit__(self, *_args: object) -> None: - self._lock.release() - - -def specification( - name: str, - *, - join_timeout: float = 1.0, - watchdog_monitored: bool = False, -) -> BackgroundServiceSpec: - return BackgroundServiceSpec( - name=name, - thread_name=f"prism-{name}", - target=lambda: None, - daemon=True, - join_timeout=join_timeout, - watchdog_monitored=watchdog_monitored, - ) - - -class BackgroundServiceRegistryTests(unittest.TestCase): - def test_named_start_is_idempotent_and_retains_exact_thread(self) -> None: - registry = BackgroundServiceRegistry( - [specification("poll", watchdog_monitored=True)], - thread_factory=DormantThread, # type: ignore[arg-type] - ) - - first = registry.start("poll") - second = registry.start("poll") - snapshot = registry.snapshot("poll") - - self.assertIs(first, second) - self.assertIs(snapshot.thread, first) - self.assertTrue(snapshot.started) - self.assertEqual(first.start_count, 1) # type: ignore[attr-defined] - self.assertEqual(first.name, "prism-poll") - self.assertTrue(first.daemon) - - def test_drain_threads_are_only_started_services_in_registration_order(self) -> None: - registry = BackgroundServiceRegistry( - [ - specification("poll", join_timeout=1.0), - specification("writer", join_timeout=5.0), - specification("optional", join_timeout=2.0), - ], - thread_factory=DormantThread, # type: ignore[arg-type] - ) - poll = registry.start("poll") - writer = registry.start("writer") - - self.assertEqual( - registry.threads_to_drain(), - ((poll, 1.0), (writer, 5.0)), - ) - - def test_watchdog_names_derive_from_the_same_service_records(self) -> None: - registry = BackgroundServiceRegistry( - [ - specification("poll", watchdog_monitored=True), - specification("health"), - specification("writer", watchdog_monitored=True), - ], - thread_factory=DormantThread, # type: ignore[arg-type] - ) - - registry.start("writer") - - self.assertEqual(registry.watchdog_service_names(), ("poll", "writer")) - self.assertEqual( - registry.watchdog_service_names(started_only=True), - ("writer",), - ) - - def test_duplicate_service_or_thread_names_are_rejected(self) -> None: - registry = BackgroundServiceRegistry([specification("poll")]) - - with self.assertRaisesRegex(ValueError, "already registered: poll"): - registry.register(specification("poll")) - with self.assertRaisesRegex(ValueError, "thread name is already registered"): - registry.register( - BackgroundServiceSpec( - name="other", - thread_name="prism-poll", - target=lambda: None, - daemon=True, - join_timeout=1.0, - watchdog_monitored=False, - ) - ) - - def test_start_failure_rolls_back_and_retry_runs_start_hook_once(self) -> None: - factory = FlakyThreadFactory() - registry = BackgroundServiceRegistry( - [specification("poll", watchdog_monitored=True)], - thread_factory=factory, - ) - started: list[str] = [] - - with self.assertRaisesRegex(RuntimeError, "thread start failed"): - registry.start( - "poll", - on_started=lambda service: started.append(service.name), - ) - - failed = registry.snapshot("poll") - self.assertFalse(failed.started) - self.assertIsNone(failed.thread) - self.assertEqual(started, []) - - thread = registry.start( - "poll", - on_started=lambda service: started.append(service.name), - ) - - self.assertEqual(factory.attempts, 2) - self.assertIs(thread, factory.threads[1]) - self.assertEqual(started, ["poll"]) - - def test_started_thread_retries_a_failed_start_hook_without_restarting(self) -> None: - registry = BackgroundServiceRegistry( - [specification("poll", watchdog_monitored=True)], - thread_factory=DormantThread, # type: ignore[arg-type] - ) - hook_attempts = 0 - - def flaky_hook(_service: BackgroundServiceSpec) -> None: - nonlocal hook_attempts - hook_attempts += 1 - if hook_attempts == 1: - raise RuntimeError("start hook failed") - - with self.assertRaisesRegex(RuntimeError, "start hook failed"): - registry.start("poll", on_started=flaky_hook) - - started = registry.snapshot("poll") - self.assertTrue(started.started) - self.assertIsNotNone(started.thread) - - retried = registry.start("poll", on_started=flaky_hook) - - self.assertIs(retried, started.thread) - self.assertEqual(retried.start_count, 1) # type: ignore[attr-defined] - self.assertEqual(hook_attempts, 2) - - def test_dynamic_registration_is_equivalent_or_fails_clearly(self) -> None: - def target() -> None: - return None - - registered = specification("dynamic") - registered = BackgroundServiceSpec( - name=registered.name, - thread_name=registered.thread_name, - target=target, - daemon=registered.daemon, - join_timeout=registered.join_timeout, - watchdog_monitored=registered.watchdog_monitored, - registration_identity=("dynamic", 1), - ) - registry = BackgroundServiceRegistry() - - self.assertTrue(registry.register_if_absent(registered)) - self.assertFalse( - registry.register_if_absent( - BackgroundServiceSpec( - name="dynamic", - thread_name="prism-dynamic", - target=lambda: None, - daemon=True, - join_timeout=1.0, - watchdog_monitored=False, - registration_identity=("dynamic", 1), - ) - ) - ) - with self.assertRaisesRegex(ValueError, "incompatible.*dynamic"): - registry.register_if_absent( - BackgroundServiceSpec( - name="dynamic", - thread_name="prism-dynamic", - target=target, - daemon=True, - join_timeout=1.0, - watchdog_monitored=False, - registration_identity=("dynamic", 2), - ) - ) - - def test_post_start_registration_keeps_registration_order_for_drain(self) -> None: - registry = BackgroundServiceRegistry( - [specification("first")], - thread_factory=DormantThread, # type: ignore[arg-type] - ) - first = registry.start("first") - registry.register(specification("second", join_timeout=2.0)) - registry.register(specification("third", join_timeout=3.0)) - third = registry.start("third") - second = registry.start("second") - - self.assertEqual( - registry.threads_to_drain(), - ((first, 1.0), (second, 2.0), (third, 3.0)), - ) - - -class CoordinatorBackgroundServiceIntegrationTests(unittest.TestCase): - @staticmethod - def coordinator_with_optional_services(enabled: bool) -> PrismCoordinator: - server = PrismCoordinator.__new__(PrismCoordinator) - server.blockwait_enabled = enabled - server.vardiff_idle_sweep_seconds = 1.0 if enabled else 0.0 - server.stratum_initial_job_timeout_seconds = 1.0 if enabled else 0.0 - server.ctv_broadcaster_enabled = enabled - server.watchdog_enabled = enabled - server.audit_bind = "127.0.0.1" if enabled else None - server.audit_port = 8080 if enabled else 0 - return server - - def test_every_process_service_is_registered_unconditionally(self) -> None: - # The registry declares every spec regardless of the live enable - # flags; serve() applies the conditional at each named start. A - # disabled coordinator therefore composes the identical registry. - disabled = self.coordinator_with_optional_services(False) - enabled = self.coordinator_with_optional_services(True) - - self.assertEqual( - disabled._make_background_service_registry().service_names(), - enabled._make_background_service_registry().service_names(), - ) - - def test_audit_http_starts_only_after_both_snapshot_refreshers(self) -> None: - # Upstream #120 ordering: both cached-snapshot refreshers are armed - # before the listener facade binds; the warm-up itself runs inside - # their background loops so the bind path never blocks on it. - server = PrismCoordinator.__new__(PrismCoordinator) - server.audit_bind = "127.0.0.1" - server.audit_port = 3341 - events: list[str] = [] - server.start_health_snapshot_refresher = lambda: events.append( # type: ignore[method-assign] - "health" - ) - server.start_metrics_snapshot_refresher = lambda: events.append( # type: ignore[method-assign] - "metrics" - ) - server._ensure_audit_http_facade = lambda: SimpleNamespace( # type: ignore[method-assign] - start=lambda: events.append("http") - ) - - with patch("builtins.print"): - server.start_audit_server() - - self.assertEqual(events, ["health", "metrics", "http"]) - - def test_process_service_specs_preserve_names_and_join_order(self) -> None: - server = self.coordinator_with_optional_services(True) - - registry = server._make_background_service_registry() - - # The watchdog spec is deliberately absent from the composed - # registry: serve() registers it inline at its start boundary so the - # publication-progress watchdog exists before synchronous startup - # prewarm/recovery work. - self.assertEqual( - registry.service_names(), - ( - "qbit_blockpoll", - "block_submitter", - "qbit_blockwait", - "vardiff_idle_sweep", - "initial_job_timeout_sweep", - "share_writer", - "ctv_fanout_broadcaster", - "health_snapshot_refresher", - "metrics_snapshot_refresher", - ), - ) - expected = { - "qbit_blockpoll": ("prism-qbit-block-poll", 1.0, True), - "block_submitter": ("prism-block-submitter", 1.0, True), - "qbit_blockwait": ("prism-qbit-block-wait", 1.0, True), - "vardiff_idle_sweep": ("prism-vardiff-idle-sweep", 1.0, True), - "initial_job_timeout_sweep": ("prism-initial-job-timeouts", 1.0, False), - "share_writer": ("prism-share-writer", 5.0, True), - "ctv_fanout_broadcaster": ( - "prism-ctv-fanout-broadcaster", - 1.0, - True, - ), - "health_snapshot_refresher": ( - "prism-health-snapshot-refresher", - 1.0, - False, - ), - "metrics_snapshot_refresher": ( - "prism-metrics-snapshot-refresher", - 1.0, - False, - ), - } - for name, properties in expected.items(): - service = registry.snapshot(name).specification - self.assertEqual( - (service.thread_name, service.join_timeout, service.watchdog_monitored), - properties, - ) - - def test_monitored_service_start_seeds_its_own_watchdog_key(self) -> None: - server = self.coordinator_with_optional_services(False) - server._heartbeats = {} - server._watchdog_pauses = {} - server._heartbeats_lock = threading.Lock() - server._background_services = BackgroundServiceRegistry( - [specification("tracked", watchdog_monitored=True)], - thread_factory=DormantThread, # type: ignore[arg-type] - ) - - server._start_background_service("tracked") - - self.assertEqual(tuple(server._heartbeats), ("tracked",)) - server._heartbeats["tracked"] = 1.0 - server._start_background_service("tracked") - self.assertEqual(server._heartbeats["tracked"], 1.0) - - def test_concurrent_wrapper_starts_seed_one_heartbeat_and_one_thread(self) -> None: - server = self.coordinator_with_optional_services(False) - server._heartbeats = {} - server._watchdog_pauses = {} - server._heartbeats_lock = threading.Lock() - server._background_services = BackgroundServiceRegistry( - [specification("tracked", watchdog_monitored=True)], - thread_factory=DormantThread, # type: ignore[arg-type] - ) - observed_registry_lock = ContentionObservedLock() - server._background_services._lock = observed_registry_lock # type: ignore[assignment] - heartbeat_entered = threading.Event() - release_heartbeat = threading.Event() - heartbeat_names: list[str] = [] - results: list[object] = [] - errors: list[BaseException] = [] - - def blocked_heartbeat(name: str) -> None: - heartbeat_names.append(name) - heartbeat_entered.set() - if not release_heartbeat.wait(5): - raise AssertionError("heartbeat test interleaving timed out") - - server._record_heartbeat = blocked_heartbeat # type: ignore[method-assign] - - def start() -> None: - try: - results.append(server._start_background_service("tracked")) - except BaseException as exc: - errors.append(exc) - - first = threading.Thread(target=start) - second = threading.Thread(target=start) - first.start() - self.assertTrue(heartbeat_entered.wait(5)) - second.start() - second_contended_inside_start = observed_registry_lock.contended.wait(5) - release_heartbeat.set() - first.join(5) - second.join(5) - - self.assertFalse(first.is_alive()) - self.assertFalse(second.is_alive()) - self.assertTrue(second_contended_inside_start) - self.assertEqual(errors, []) - self.assertEqual(heartbeat_names, ["tracked"]) - self.assertEqual(len(results), 2) - self.assertIs(results[0], results[1]) - self.assertEqual(results[0].start_count, 1) # type: ignore[union-attr] - - def test_wrapper_start_failure_rolls_back_heartbeat_and_retries_cleanly(self) -> None: - server = self.coordinator_with_optional_services(False) - server._heartbeats = {} - server._watchdog_pauses = {} - server._heartbeats_lock = threading.Lock() - factory = FlakyThreadFactory() - server._background_services = BackgroundServiceRegistry( - [specification("tracked", watchdog_monitored=True)], - thread_factory=factory, - ) - - with self.assertRaisesRegex(RuntimeError, "thread start failed"): - server._start_background_service("tracked") - - self.assertEqual(server._heartbeats, {}) - self.assertFalse(server._background_services.snapshot("tracked").started) - - thread = server._start_background_service("tracked") - - self.assertIs(thread, factory.threads[1]) - self.assertEqual(tuple(server._heartbeats), ("tracked",)) - self.assertTrue(server._background_services.snapshot("tracked").started) - - def test_concurrent_secondary_starts_register_and_start_once(self) -> None: - server = self.coordinator_with_optional_services(False) - server._heartbeats = {} - server._watchdog_pauses = {} - server._heartbeats_lock = threading.Lock() - server._background_services = BackgroundServiceRegistry( - thread_factory=DormantThread, # type: ignore[arg-type] - ) - profile = SimpleNamespace( - heartbeat_name="stratum_accept_highdiff", - name="highdiff", - ) - listener = SimpleNamespace() - registration_barrier = threading.Barrier(2) - original_register = server._background_services.register_if_absent - - def synchronized_register(service: BackgroundServiceSpec) -> bool: - registration_barrier.wait(timeout=5) - return original_register(service) - - server._background_services.register_if_absent = synchronized_register # type: ignore[method-assign] - heartbeat_names: list[str] = [] - server._record_heartbeat = heartbeat_names.append # type: ignore[method-assign] - results: list[object] = [] - errors: list[BaseException] = [] - - def start() -> None: - try: - results.append( - server._start_secondary_accept_service( # type: ignore[arg-type] - listener, - profile, - ) - ) - except BaseException as exc: - errors.append(exc) - - callers = [threading.Thread(target=start) for _ in range(2)] - for caller in callers: - caller.start() - for caller in callers: - caller.join(5) - - self.assertTrue(all(not caller.is_alive() for caller in callers)) - self.assertEqual(errors, []) - self.assertEqual(heartbeat_names, ["stratum_accept_highdiff"]) - self.assertEqual(len(results), 2) - self.assertIs(results[0], results[1]) - self.assertEqual(results[0].start_count, 1) # type: ignore[union-attr] - self.assertEqual( - server._background_services.service_names(), - ("stratum_accept_highdiff",), - ) - - def test_secondary_listener_is_named_monitored_and_bounded_for_drain(self) -> None: - server = self.coordinator_with_optional_services(False) - server._heartbeats = {} - server._watchdog_pauses = {} - server._heartbeats_lock = threading.Lock() - server._background_services = BackgroundServiceRegistry( - thread_factory=DormantThread, # type: ignore[arg-type] - ) - profile = SimpleNamespace( - heartbeat_name="stratum_accept_highdiff", - name="highdiff", - ) - - thread = server._start_secondary_accept_service( # type: ignore[arg-type] - SimpleNamespace(), - profile, - ) - - snapshot = server._background_services.snapshot("stratum_accept_highdiff") - self.assertEqual(thread.name, "prism-stratum-accept-highdiff") - self.assertTrue(snapshot.specification.watchdog_monitored) - self.assertEqual(snapshot.specification.join_timeout, 1.0) - self.assertEqual(tuple(server._heartbeats), ("stratum_accept_highdiff",)) - self.assertEqual( - server._background_services.threads_to_drain(), - ((thread, 1.0),), - ) - - def test_health_refresher_start_marks_latch_before_thread_dispatch(self) -> None: - # Upstream #120 contract: the running latch is set before the - # registry can start the refresher thread, and it is a one-shot - # latch (the loop never clears it), so a second start call returns - # without touching the registry again. - server = self.coordinator_with_optional_services(False) - server.stop_event = threading.Event() - server.stop_event.set() - latch_at_dispatch: list[bool] = [] - - class LatchObservingThread(DormantThread): - def start(self) -> None: - latch_at_dispatch.append( - server._ensure_observability_service() - .state() - .health_refresh_loop_running - ) - super().start() - - server._background_services = BackgroundServiceRegistry( - [specification("health_snapshot_refresher")], - thread_factory=LatchObservingThread, # type: ignore[arg-type] - ) - - server.start_health_snapshot_refresher() - - started = server._background_services.snapshot("health_snapshot_refresher") - self.assertEqual(latch_at_dispatch, [True]) - self.assertTrue(started.started) - self.assertTrue( - server._ensure_observability_service() - .state() - .health_refresh_loop_running - ) - - server.start_health_snapshot_refresher() - - self.assertEqual(started.thread.start_count, 1) # type: ignore[union-attr] - self.assertTrue( - server._ensure_observability_service() - .state() - .health_refresh_loop_running - ) - - -class ScriptedStopEvent: - """Stop signal whose transitions the test scripts explicitly.""" - - def __init__(self) -> None: - self.stopped = False - self.waits: list[float] = [] - - def is_set(self) -> bool: - return self.stopped - - def wait(self, timeout: float) -> bool: - self.waits.append(timeout) - return self.stopped - - -class ScriptedRollupLedger: - """Rollup-capable ledger stub yielding one scripted result per pass.""" - - def __init__(self, results: list[object], *, on_exhausted=None) -> None: - self.results = list(results) - self.calls: list[int] = [] - self.on_exhausted = on_exhausted - - def advance_hashrate_rollups(self, *, batch_limit: int) -> dict[str, object]: - self.calls.append(batch_limit) - result = self.results.pop(0) - if not self.results and self.on_exhausted is not None: - self.on_exhausted() - if isinstance(result, BaseException): - raise result - return result # type: ignore[return-value] - - -class HashrateRollupMaintenanceServiceTests(unittest.TestCase): - @staticmethod - def rollup_coordinator(ledger: object) -> PrismCoordinator: - server = PrismCoordinator.__new__(PrismCoordinator) - server.blockwait_enabled = False - server.vardiff_idle_sweep_seconds = 0.0 - server.stratum_initial_job_timeout_seconds = 0.0 - server.ctv_broadcaster_enabled = False - server.watchdog_enabled = False - server.audit_bind = None - server.audit_port = 0 - server.ledger = ledger - server.hashrate_rollup_enabled = True - server.hashrate_rollup_interval_seconds = 15.0 - server.hashrate_rollup_batch_shares = 50000 - return server - - def test_rollup_service_registers_only_for_rollup_capable_ledgers(self) -> None: - # The capability conditional is the one intentional exception to the - # registered-unconditionally rule: a memory ledger never grows - # advance_hashrate_rollups at runtime, so the service either exists - # for the life of the process or never will. - capable = self.rollup_coordinator(ScriptedRollupLedger([])) - registry = capable._make_background_service_registry() - self.assertIn("hashrate-rollup-maintenance", registry.service_names()) - service = registry.snapshot("hashrate-rollup-maintenance").specification - self.assertEqual( - (service.thread_name, service.join_timeout, service.watchdog_monitored), - ("prism-hashrate-rollup-maintenance", 1.0, False), - ) - - memory_backed = self.rollup_coordinator(SimpleNamespace()) - self.assertNotIn( - "hashrate-rollup-maintenance", - memory_backed._make_background_service_registry().service_names(), - ) - - def test_rollup_start_honours_enable_flag_and_ledger_capability(self) -> None: - started = self.rollup_coordinator(ScriptedRollupLedger([])) - started._heartbeats = {} - started._watchdog_pauses = {} - started._heartbeats_lock = threading.Lock() - started._background_services = BackgroundServiceRegistry( - [specification("hashrate-rollup-maintenance")], - thread_factory=DormantThread, # type: ignore[arg-type] - ) - with patch("builtins.print"): - started._start_hashrate_rollup_maintenance_if_enabled() - self.assertTrue( - started._background_services.snapshot("hashrate-rollup-maintenance").started - ) - - disabled = self.rollup_coordinator(ScriptedRollupLedger([])) - disabled.hashrate_rollup_enabled = False - disabled._background_services = BackgroundServiceRegistry( - [specification("hashrate-rollup-maintenance")], - thread_factory=DormantThread, # type: ignore[arg-type] - ) - disabled._start_hashrate_rollup_maintenance_if_enabled() - self.assertFalse( - disabled._background_services.snapshot("hashrate-rollup-maintenance").started - ) - - # A memory-backed coordinator has no registered service to start, so - # the guard must return before asking the registry for the name. - memory_backed = self.rollup_coordinator(SimpleNamespace()) - memory_backed._background_services = BackgroundServiceRegistry([]) - memory_backed._start_hashrate_rollup_maintenance_if_enabled() - - def test_rollup_loop_drains_backlog_then_sleeps_the_interval(self) -> None: - # Catch-up looping is the backfill mechanism: while the ledger - # reports more history behind the watermark the loop advances again - # immediately, and only a caught-up pass reaches the interval wait. - stop = ScriptedStopEvent() - ledger = ScriptedRollupLedger( - [ - {"scanned": 3, "last_share_seq": 3, "caught_up": False}, - {"scanned": 3, "last_share_seq": 6, "caught_up": False}, - {"scanned": 2, "last_share_seq": 8, "caught_up": True}, - ], - on_exhausted=lambda: setattr(stop, "stopped", True), - ) - server = self.rollup_coordinator(ledger) - server.stop_event = stop # type: ignore[assignment] - server.hashrate_rollup_batch_shares = 3 - - with patch("builtins.print"): - server.hashrate_rollup_maintenance_loop() - - self.assertEqual(ledger.calls, [3, 3, 3]) - self.assertEqual(stop.waits, [15.0]) - - def test_rollup_passes_run_inside_the_writer_admission(self) -> None: - """Each pass is a writer operation the shutdown controller counts. - - The maintenance statement renews the writer lease on exact identity, - which an orderly release keeps -- so a pass outside the admission - could slip past the stop check, renew the just-released lease, and - mutate rollups after shutdown reported the writer quiesced. Running - the ledger call inside _writer_operation makes quiescence wait for - an in-flight pass. - """ - stop = ScriptedStopEvent() - server = self.rollup_coordinator(None) - server.stop_event = stop # type: ignore[assignment] - controller = server._ensure_shutdown_controller() - observed: list[int] = [] - - class AdmissionObservingLedger(ScriptedRollupLedger): - def advance_hashrate_rollups(self, *, batch_limit: int) -> dict[str, object]: - observed.append(controller.active_writers.get("hashrate_rollup_maintenance", 0)) - return super().advance_hashrate_rollups(batch_limit=batch_limit) - - server.ledger = AdmissionObservingLedger( - [{"scanned": 0, "last_share_seq": 8, "caught_up": True}], - on_exhausted=lambda: setattr(stop, "stopped", True), - ) - - with patch("builtins.print"): - server.hashrate_rollup_maintenance_loop() - - self.assertEqual([1], observed) - self.assertEqual( - 0, controller.active_writers.get("hashrate_rollup_maintenance", 0) - ) - - def test_rollup_loop_stops_cleanly_when_admission_is_closed(self) -> None: - """A refused admission is the orderly shutdown, not a failure. - - The race this pins: shutdown begins after the loop's stop check but - before the pass enters the admission. The pass must be refused, and - the loop must exit without logging an error or running the ledger. - """ - stop = ScriptedStopEvent() - ledger = ScriptedRollupLedger([]) - server = self.rollup_coordinator(ledger) - server.stop_event = stop # type: ignore[assignment] - server._ensure_shutdown_controller().request_shutdown(None) - - with patch("builtins.print"), patch("traceback.print_exc") as print_exc: - server.hashrate_rollup_maintenance_loop() - - self.assertEqual([], ledger.calls) - self.assertEqual([], stop.waits) - self.assertEqual(0, print_exc.call_count) - - def test_rollup_loop_logs_the_error_and_retries_next_interval(self) -> None: - stop = ScriptedStopEvent() - ledger = ScriptedRollupLedger( - [ - RuntimeError("rollup pass failed"), - {"scanned": 0, "last_share_seq": 8, "caught_up": True}, - ], - on_exhausted=lambda: setattr(stop, "stopped", True), - ) - server = self.rollup_coordinator(ledger) - server.stop_event = stop # type: ignore[assignment] - - with patch("builtins.print"), patch("traceback.print_exc") as print_exc: - server.hashrate_rollup_maintenance_loop() - - # The failed pass reaches the interval wait instead of hot-looping, - # then the next pass succeeds and the scripted stop ends the loop. - self.assertEqual(ledger.calls, [50000, 50000]) - self.assertEqual(stop.waits, [15.0, 15.0]) - self.assertEqual(print_exc.call_count, 1) - - -if __name__ == "__main__": - unittest.main() diff --git a/tests/test_prism_block_candidate_collapse.py b/tests/test_prism_block_candidate_collapse.py deleted file mode 100644 index 0a9352fdf..000000000 --- a/tests/test_prism_block_candidate_collapse.py +++ /dev/null @@ -1,4851 +0,0 @@ -#!/usr/bin/env python3 -"""Decided-height block-candidate collapse (issue #183). - -The 2026-08-20 testnet4 incident left 3,120 durable block candidates behind -one decided height. The shipped per-row disposition path terminalizes them -correctly but at one node offer, one disposition lease, and one durable -write each; ``tests/prism_candidate_storm.py`` measures that cost and, in -``drain_per_row``, is the oracle every set-oriented selector has to agree -with. - -These tests own the selector and its apply: predicate S clause by clause, -its fail-closed behaviour, the bounded chain reads, the non-blocking lease, -the immediate pre-write revalidation, the exact-returned-set cleanup and its -failure modes, the page partition both replay enumeration shapes must -perform, the bounded logs and fixed-label metrics, and the storm comparisons -against the shipped per-row oracle at 100 and 3,120 candidates in both the -live and the restart view. - -The safety property under test is one-directional and absolute: for every -comparison ``S - oracle_abandoned`` must be empty. A conservative -``oracle_abandoned - S`` delta is allowed and reported; an S-only hash is a -bug that would destroy a block the per-row path deliberately kept. -""" - -from __future__ import annotations - -import queue -import sys -import threading -import time -import unittest -from contextlib import redirect_stdout -from io import StringIO -from pathlib import Path -from types import SimpleNamespace -from typing import Any, Iterable - -if not __package__: - sys.path.insert(0, str(Path(__file__).resolve().parents[1])) - -from lab.prism.block_candidates import ( # noqa: E402 - BLOCK_CANDIDATE_COLLAPSE_CLEANUP_FLOOR_STEP, - BLOCK_CANDIDATE_COLLAPSE_CLEANUP_PAYOUT_STEP, - BLOCK_CANDIDATE_COLLAPSE_CLEANUP_STEPS, - BLOCK_CANDIDATE_COLLAPSE_LOG_FAILURES, - BLOCK_CANDIDATE_COLLAPSE_LOG_GROUPS, - BLOCK_CANDIDATE_COLLAPSE_LOG_SAMPLE_HASHES, - BLOCK_CANDIDATE_TERMINAL_OUTCOME_EVICTION_SCAN, - COLLAPSE_DIFFICULTY_SCALE, - COLLAPSE_POW_LIMIT_BITS, - DEFAULT_BLOCK_ACCOUNTING_CLEANUP_RETRY_WORK_ITEMS, - MAX_BLOCK_CANDIDATE_COLLAPSE_HEIGHT_PROBES, - MAX_BLOCK_CANDIDATE_TERMINAL_OUTCOMES, - MAX_BLOCK_REPLAY_ENUMERATION_ROWS, - MAX_PENDING_BLOCK_CANDIDATES, - PRISM_BLOCK_CANDIDATE_COLLAPSE_OUTCOMES, - PRISM_BLOCK_CANDIDATE_COLLAPSE_REASON, - PRISM_BLOCK_CANDIDATE_COLLAPSE_STALE_JOB_CLASS, - PRISM_STALE_JOB_ABANDON_CLASSES, - _BlockCandidateAccountingTask, - _BlockCandidateChainView, - _BlockCandidateNodeSubmission, - _collapse_scaled_difficulty, -) -from lab.prism.metrics import MetricsRenderer # noqa: E402 -from lab.prism.share_ledger import ( # noqa: E402 - PendingShare, - SingleWriterShareLedger, -) -from lab.prism.template_artifacts import scaled_network_difficulty # noqa: E402 -from lab.prism.share_submission import PRISM_REJECTION_STALE_JOB # noqa: E402 -from tests.prism_candidate_storm import ( # noqa: E402 - PERTURBATIONS, - STORM_PARENT_HASH, - CandidateStormRig, -) -from tests.prism_vardiff_test_support import ( # noqa: E402 - FakeRpc, - block_candidate, - submit_coordinator, -) - - -STORM_PARENT = STORM_PARENT_HASH -OTHER_PARENT = "22" * 32 -DECIDED_HEIGHT = 10 -DECIDED_WINNER = "dd" * 32 -# A candidate building on the current best tip: the block waiting to be -# offered, which predicate S must never select and whose submission must not -# sit behind a long enumeration walk. -LIVE_SOLVE_HASH = f"{0xFEED0001:064x}" -# Compact header bits, the only work fact these tests state. A candidate's -# durable ``found_block.network_difficulty`` is -# ``scaled_network_difficulty(template["bits"])`` -- the raw difficulty times -# COLLAPSE_DIFFICULTY_SCALE -- so a chain-side work value is only comparable -# with it once it is re-derived from the occupant's own bits. Spelling the -# fixture in bits is what makes "equal work" mean the same integer on both -# sides instead of two numbers a million apart. -MIN_WORK_BITS = COLLAPSE_POW_LIMIT_BITS -# A retargeted header: raw difficulty ~2.1e9, scaled ~2.1e15. -RETARGETED_BITS = "1d00ffff" -# Scaled work above 2**53, where widening the stored integer to float rounds -# it *up* and an equal-work occupant would read as strictly weaker. -HIGH_WORK_BITS = "1b0404cb" - - -def _hash(index: int) -> str: - return f"{index:064x}" - - -def _scaled(bits: str) -> int: - """The PRISM-scaled difficulty a header with ``bits`` carries.""" - return scaled_network_difficulty(bits) - - -def _raw(bits: str) -> float: - """The raw ``getblockheader.difficulty`` float a node reports for ``bits``. - - A million times smaller than the scaled units every candidate row is - stamped in; the fixture reports it alongside ``bits`` exactly so a - selector that read it again would fail the equal-work tests. - """ - return _scaled(bits) / COLLAPSE_DIFFICULTY_SCALE - - -class CollapseChainRpc(FakeRpc): - """A controllable decided-height chain view with per-method call counts. - - Every read predicate S makes is separately overridable -- absent, raising, - or returning an unusable value -- so each fail-closed branch can be driven - without reaching for a mock framework. - """ - - def __init__( - self, - *, - tip: str = DECIDED_WINNER, - tip_height: int = DECIDED_HEIGHT, - active: dict[int, str] | None = None, - bits: dict[str, str] | None = None, - ) -> None: - self.tip = tip.lower() - self.tip_height = int(tip_height) - self.active = { - int(height): str(value).lower() - for height, value in (active or {DECIDED_HEIGHT: DECIDED_WINNER}).items() - } - # Per-block compact bits; every unlisted block sits at the qbit - # powLimit, the weakest header the chain can carry. - self.bits = { - str(key).lower(): str(value).lower() - for key, value in (bits or {}).items() - } - self.default_bits = MIN_WORK_BITS - self.failures: set[str] = set() - self.results: dict[str, object] = {} - self.calls: list[tuple[str, tuple[object, ...]]] = [] - self.before_call: Any | None = None - - def call(self, method: str, params: list[object] | None = None) -> object: - self.calls.append((method, tuple(params or ()))) - if self.before_call is not None: - self.before_call(self, method, params) - if method in self.failures: - raise RuntimeError(f"qbit RPC {method} failed") - if method in self.results: - return self.results[method] - if method == "getbestblockhash": - return self.tip - if method == "getblockcount": - return self.tip_height - if method == "getblockhash": - height = int((params or [0])[0]) - if height in self.active: - return self.active[height] - raise RuntimeError( - f"qbit RPC getblockhash failed: -8 unknown height {height}" - ) - if method == "getblockheader": - block_hash = str((params or [""])[0]).lower() - if block_hash not in set(self.active.values()) | {self.tip}: - raise RuntimeError( - "qbit RPC getblockheader failed: -5 Block not found" - ) - bits = self.bits.get(block_hash, self.default_bits) - return { - "hash": block_hash, - "height": self.tip_height, - "confirmations": 1, - "bits": bits, - # qbitd reports both; ``difficulty`` is the raw float, a - # million times smaller than the units the candidate rows - # carry, and the selector must not read it. - "difficulty": _raw(bits), - } - return super().call(method, params) - - def counts(self) -> dict[str, int]: - counts: dict[str, int] = {} - for method, _params in self.calls: - counts[method] = counts.get(method, 0) + 1 - return counts - - -class CollapseLedger: - """A real in-memory ledger with controllable page reads and batch writes.""" - - def __init__(self, inner: Any | None = None) -> None: - self.inner = inner if inner is not None else SingleWriterShareLedger() - self.page_hook: Any | None = None - self.abandon_hook: Any | None = None - self.abandon_calls: list[tuple[tuple[str, ...], str]] = [] - - def __getattr__(self, name: str) -> Any: - return getattr(self.inner, name) - - def pending_block_candidate_rows( - self, - *, - limit: int = 32, - after_cursor: object | None = None, - ) -> list[dict[str, Any]]: - rows = self.inner.pending_block_candidate_rows( - limit=limit, - after_cursor=after_cursor, - ) - if self.page_hook is not None: - rows = self.page_hook(rows) - return rows - - def mark_block_candidates_abandoned( - self, - *, - block_hashes: Any, - error: str, - ) -> tuple[str, ...]: - requested = tuple(str(value).lower() for value in block_hashes) - self.abandon_calls.append((requested, error)) - if self.abandon_hook is not None: - return self.abandon_hook(requested, error) - return self.inner.mark_block_candidates_abandoned( - block_hashes=requested, - error=error, - ) - - -class NoBatchCollapseLedger(CollapseLedger): - """A compatibility ledger with no fenced batch abandonment at all.""" - - def __getattr__(self, name: str) -> Any: - if name == "mark_block_candidates_abandoned": - raise AttributeError(name) - return super().__getattr__(name) - - mark_block_candidates_abandoned = None # type: ignore[assignment] - - -class WindowCollapseLedger(CollapseLedger): - """A ledger whose page reader cannot paginate, forcing the legacy window.""" - - def pending_block_candidate_rows( # type: ignore[override] - self, - *, - limit: int = 32, - ) -> list[dict[str, Any]]: - rows = self.inner.pending_block_candidate_rows(limit=limit) - if self.page_hook is not None: - rows = self.page_hook(rows) - return rows - - -class CollapseFixture: - """One coordinator, ledger, and chain view seeded with durable rows.""" - - def __init__( - self, - *, - ledger: CollapseLedger | None = None, - rpc: CollapseChainRpc | None = None, - ) -> None: - server, state, _recording = submit_coordinator(tip=STORM_PARENT) - server.max_blocks = 1 << 30 - server.stop_after_block = False - server.block_candidate_queue = queue.Queue( - maxsize=MAX_PENDING_BLOCK_CANDIDATES - ) - server.block_candidate_retry_initial_seconds = 0.0 - self.ledger = ledger if ledger is not None else CollapseLedger() - server.ledger = self.ledger - self.rpc = rpc if rpc is not None else CollapseChainRpc() - server.rpc = self.rpc - self.server = server - self.state = state - self.service = server._ensure_block_candidate_service() - self.service._ensure_block_replay_state() - self.service._ensure_block_candidate_disposition_state() - self._jobs: set[str] = set() - - # -- seeding ----------------------------------------------------------- - - def job( - self, - *, - parent: str = STORM_PARENT, - height: int = DECIDED_HEIGHT, - network_difficulty: int = 1, - ) -> str: - job_id = f"job-{parent}-{height}-{network_difficulty}" - if job_id in self._jobs: - return job_id - base = self.server.jobs["job-1"] - self.server.jobs[job_id] = SimpleNamespace( - job=SimpleNamespace( - job_id=job_id, - share_target=base.job.share_target, - share_difficulty=base.job.share_difficulty, - transaction_hexes=(), - ), - template={ - "previousblockhash": parent, - "height": int(height), - "coinbasevalue": 50_00000000, - }, - found_block={"network_difficulty": int(network_difficulty)}, - issued_at_ms=12345, - collection_only=False, - worker=base.worker, - shares_json=[], - prior_balances=[], - ) - self._jobs.add(job_id) - return job_id - - def seed( - self, - hashes: list[str], - *, - parent: str = STORM_PARENT, - height: int = DECIDED_HEIGHT, - network_difficulty: int = 1, - credit_share_on_accept: bool = False, - ) -> list[Any]: - job_id = self.job( - parent=parent, - height=height, - network_difficulty=network_difficulty, - ) - candidates = [] - entries = [] - for block_hash in hashes: - pending = PendingShare( - share_id=f"miner-a:{block_hash}", - miner_id="miner-a", - order_key="miner-a", - p2mr_program_hex="11" * 32, - share_difficulty=1, - network_difficulty=int(network_difficulty), - template_height=int(height) - 1, - job_id=job_id, - job_issued_at_ms=1, - accepted_at_ms=1, - ntime=1, - ) - candidate = block_candidate( - self.server, - self.state, - SimpleNamespace( - coinbase_tx_hex="00", - block_hash_hex=block_hash, - block_hex="00", - share_pass=True, - block_pass=True, - ), - job_id=job_id, - pending_share=pending, - credit_share_on_accept=credit_share_on_accept, - ) - candidates.append(candidate) - entries.append((pending, self.server.block_candidate_intent(candidate))) - self.ledger.inner.append_batch(entries) - return candidates - - # -- driving ----------------------------------------------------------- - - def page(self, limit: int = 512) -> list[dict[str, Any]]: - return self.ledger.pending_block_candidate_rows(limit=limit) - - def collapse( - self, - rows: list[Any] | None = None, - *, - call_class: str = "fast", - ) -> tuple[list[Any], str]: - """Run one apply over a page and return (retained rows, log text).""" - if rows is None: - rows = self.page() - buffer = StringIO() - with redirect_stdout(buffer): - retained = self.service._collapse_superseded_block_candidates( - rows, - timeout_seconds=None, - call_class=call_class, - ) - return retained, buffer.getvalue() - - def replay(self, *, enumeration_owed: bool = True) -> tuple[int, str]: - if enumeration_owed: - self.server._note_block_replay_enumeration_owed() - buffer = StringIO() - with redirect_stdout(buffer): - queued = self.server.replay_pending_block_candidates() - return queued, buffer.getvalue() - - def retry_cleanup(self, *, force_due: bool = False) -> tuple[bool, str]: - """Run one deferred cleanup retry pass on the accounting lane's runner. - - ``force_due`` clears the per-hash backoff deadline so a second and - later attempt can be driven without sleeping; a freshly deferred - hash is due immediately and needs it not at all. - """ - service = self.service - if force_due: - with self.server.lock: - for record in ( - service._block_candidate_collapse_cleanup_retries.values() - ): - record.not_before_monotonic = 0.0 - buffer = StringIO() - with redirect_stdout(buffer): - ran = service._run_one_collapsed_block_candidate_cleanup_retry() - return ran, buffer.getvalue() - - # -- observation ------------------------------------------------------- - - def cleanup_backlog(self) -> dict[str, frozenset[str]]: - return self.service.collapsed_candidate_cleanup_backlog() - - def pending(self) -> set[str]: - return { - str(row["block_hash"]).lower() - for row in self.ledger.inner.pending_block_candidate_rows(limit=1 << 16) - } - - def counts(self) -> dict[str, int]: - return self.service.block_candidate_collapse_snapshot() - - def floor(self) -> dict[int, Any]: - writer = self.server._ensure_share_writer_service() - return dict(writer._pending_share_commit_floor) - - -def _selected(fixture: CollapseFixture, rows: list[Any] | None = None) -> set[str]: - """The hashes one apply actually removed from the durable outbox.""" - before = fixture.pending() - fixture.collapse(rows) - return before - fixture.pending() - - -class BlockCandidateCollapsePredicateTests(unittest.TestCase): - """Predicate S, clause by clause, against the shipped durable page.""" - - def test_decided_height_sibling_is_selected(self) -> None: - fixture = CollapseFixture() - fixture.seed([_hash(1), _hash(2)]) - self.assertEqual(_selected(fixture), {_hash(1), _hash(2)}) - self.assertEqual(fixture.counts()["selected"], 2) - self.assertEqual(fixture.counts()["abandoned"], 2) - - def test_ownership_markers_alone_do_not_exclude(self) -> None: - """Outstanding/replay-inflight mean owned, not offered (#183 note).""" - fixture = CollapseFixture() - fixture.seed([_hash(1)]) - fixture.server._register_outstanding_block_candidate(_hash(1)) - with fixture.server.lock: - fixture.service._block_replay_inflight_hashes.add(_hash(1)) - self.assertEqual(_selected(fixture), {_hash(1)}) - - def test_attempt_count_alone_does_not_exclude(self) -> None: - fixture = CollapseFixture() - fixture.seed([_hash(1)]) - fixture.ledger.inner.mark_block_candidate_attempted(block_hash=_hash(1)) - row = fixture.ledger.inner._block_candidate_outbox[_hash(1)] - self.assertEqual(int(row["attempt_count"]), 1) - self.assertEqual(_selected(fixture), {_hash(1)}) - - def test_each_evidence_source_excludes_its_hash(self) -> None: - target = _hash(1) - other = _hash(2) - - def flight(fixture: CollapseFixture) -> None: - lease = fixture.server._claim_block_candidate_disposition( - target, - blocking=False, - ) - self.assertIsNotNone(lease) - self.addCleanup( - fixture.server._release_block_candidate_disposition, - lease, - ) - - def retry_candidate(fixture: CollapseFixture) -> None: - candidate = fixture.seed([target])[0] - with fixture.server.lock: - fixture.service.retry_candidate = candidate - - def waiting_retry(fixture: CollapseFixture) -> None: - candidate = fixture.seed([target])[0] - with fixture.server.lock: - fixture.service._block_disposition_waiting_retries[target] = candidate - - def finalize_retry(fixture: CollapseFixture) -> None: - with fixture.server.lock: - fixture.service.finalize_retries[target] = (False, "boom") - - def deferred_retry(fixture: CollapseFixture) -> None: - candidate = fixture.seed([target])[0] - with fixture.server.lock: - fixture.service._block_accounting_deferred_retry_candidate = candidate - - def retained_submission(fixture: CollapseFixture) -> None: - fixture.service._stash_retained_block_candidate_node_submission( - target, - _BlockCandidateNodeSubmission(attempted=True, result=None), - ) - - def tip_observed(fixture: CollapseFixture) -> None: - fixture.server._register_outstanding_block_candidate(target) - fixture.server._note_tip_observation_for_candidates(target) - - def accounted(fixture: CollapseFixture) -> None: - fixture.server._ensure_job_cache_state() - with fixture.server.lock: - fixture.server._accounted_accepted_block_hashes.add(target) - - def terminal(fixture: CollapseFixture) -> None: - fixture.server._record_block_candidate_terminal_outcome( - target, - accepted=False, - ) - - sources = { - "_block_candidate_disposition_flights": flight, - "retry_candidate": retry_candidate, - "_block_disposition_waiting_retries": waiting_retry, - "finalize_retries": finalize_retry, - "_block_accounting_deferred_retry_candidate": deferred_retry, - "_block_candidate_retained_node_submissions": retained_submission, - "_tip_observed_accepted_block_hashes": tip_observed, - "_accounted_accepted_block_hashes": accounted, - "_block_candidate_terminal_outcomes": terminal, - } - for name, install in sources.items(): - with self.subTest(evidence=name): - fixture = CollapseFixture() - fixture.seed([target, other]) - with redirect_stdout(StringIO()): - install(fixture) - self.assertEqual(_selected(fixture), {other}) - - def test_pool_block_row_excludes_its_hash(self) -> None: - fixture = CollapseFixture() - fixture.seed([_hash(1), _hash(2)]) - fixture.ledger.inner.persist_accepted_block( - block_hash=_hash(1), - block_height=DECIDED_HEIGHT, - parent_hash=STORM_PARENT, - final_bundle={}, - audit_report={}, - ) - self.assertEqual(_selected(fixture), {_hash(2)}) - - def test_missing_pool_block_fact_fails_the_whole_page_closed(self) -> None: - fixture = CollapseFixture() - fixture.seed([_hash(1), _hash(2)]) - fixture.ledger.page_hook = lambda rows: [ - {key: value for key, value in row.items() if key != "pool_block_exists"} - if index == 0 - else row - for index, row in enumerate(rows) - ] - rows = fixture.page() - retained, _log = fixture.collapse(rows) - self.assertEqual(fixture.pending(), {_hash(1), _hash(2)}) - self.assertEqual(len(retained), len(rows)) - self.assertEqual(fixture.counts()["fail_closed"], 2) - self.assertEqual(fixture.counts()["selected"], 0) - - def test_a_page_without_the_pool_block_fact_spends_no_chain_read(self) -> None: - """A compatibility page reader must not cost a tip read per poll.""" - fixture = CollapseFixture() - fixture.seed([_hash(1), _hash(2)]) - fixture.ledger.page_hook = lambda rows: [ - {"block_hash": row["block_hash"], "candidate": row["candidate"]} - for row in rows - ] - fixture.collapse() - self.assertEqual(fixture.rpc.calls, []) - self.assertEqual(fixture.pending(), {_hash(1), _hash(2)}) - self.assertEqual(fixture.counts()["fail_closed"], 2) - - def test_malformed_pool_block_fact_fails_the_whole_page_closed(self) -> None: - fixture = CollapseFixture() - fixture.seed([_hash(1), _hash(2)]) - fixture.ledger.page_hook = lambda rows: [ - {**row, "pool_block_exists": "no"} if index == 1 else row - for index, row in enumerate(rows) - ] - fixture.collapse() - self.assertEqual(fixture.pending(), {_hash(1), _hash(2)}) - self.assertEqual(fixture.counts()["fail_closed"], 2) - - def test_parent_equal_to_best_tip_excludes_its_hash(self) -> None: - fixture = CollapseFixture( - rpc=CollapseChainRpc( - tip=OTHER_PARENT, - tip_height=DECIDED_HEIGHT, - active={DECIDED_HEIGHT: DECIDED_WINNER}, - ) - ) - fixture.seed([_hash(1)], parent=OTHER_PARENT) - fixture.seed([_hash(2)], parent=STORM_PARENT) - self.assertEqual(_selected(fixture), {_hash(2)}) - - def test_height_above_the_tip_excludes_its_hash(self) -> None: - fixture = CollapseFixture( - rpc=CollapseChainRpc( - tip=DECIDED_WINNER, - tip_height=DECIDED_HEIGHT, - active={DECIDED_HEIGHT: DECIDED_WINNER}, - ) - ) - fixture.seed([_hash(1)], height=DECIDED_HEIGHT + 1) - fixture.seed([_hash(2)], height=DECIDED_HEIGHT) - self.assertEqual(_selected(fixture), {_hash(2)}) - # A height above the tip must not even be looked up: getblockhash - # would fail there and fail the whole page closed. - self.assertEqual( - [params for method, params in fixture.rpc.calls if method == "getblockhash"], - [(DECIDED_HEIGHT,), (DECIDED_HEIGHT,)], - ) - - def test_candidate_that_is_the_active_block_is_never_selected(self) -> None: - fixture = CollapseFixture( - rpc=CollapseChainRpc( - tip=_hash(1), - tip_height=DECIDED_HEIGHT, - active={DECIDED_HEIGHT: _hash(1)}, - ) - ) - fixture.seed([_hash(1), _hash(2)]) - self.assertEqual(_selected(fixture), {_hash(2)}) - - def test_lower_work_occupant_never_destroys_a_heavier_sibling(self) -> None: - """Clause 4b: a testnet4 minimum-difficulty occupant is not a decision.""" - fixture = CollapseFixture( - rpc=CollapseChainRpc( - tip=DECIDED_WINNER, - tip_height=DECIDED_HEIGHT, - active={DECIDED_HEIGHT: DECIDED_WINNER}, - bits={DECIDED_WINNER: MIN_WORK_BITS}, - ) - ) - occupant = _scaled(MIN_WORK_BITS) - fixture.seed([_hash(1)], network_difficulty=occupant * 8) - fixture.seed([_hash(2)], network_difficulty=occupant) - self.assertEqual(_selected(fixture), {_hash(2)}) - - def test_equal_work_occupant_still_supersedes(self) -> None: - fixture = CollapseFixture( - rpc=CollapseChainRpc( - tip=DECIDED_WINNER, - tip_height=DECIDED_HEIGHT, - active={DECIDED_HEIGHT: DECIDED_WINNER}, - bits={DECIDED_WINNER: RETARGETED_BITS}, - ) - ) - fixture.seed([_hash(1)], network_difficulty=_scaled(RETARGETED_BITS)) - self.assertEqual(_selected(fixture), {_hash(1)}) - - def test_header_work_is_read_in_the_candidates_scaled_units(self) -> None: - """Regression: the row is scaled, ``getblockheader.difficulty`` is raw. - - A candidate stamped with ``scaled_network_difficulty`` and an - active sibling of exactly equal work must collapse. Reading the - header's raw ``difficulty`` float instead of re-deriving from its - bits makes the occupant look COLLAPSE_DIFFICULTY_SCALE times - weaker, so clause 4b/5 rejects it and a decided height never - collapses at all. - """ - for bits in (MIN_WORK_BITS, RETARGETED_BITS, HIGH_WORK_BITS): - with self.subTest(bits=bits): - fixture = CollapseFixture( - rpc=CollapseChainRpc( - tip=DECIDED_WINNER, - tip_height=DECIDED_HEIGHT, - active={DECIDED_HEIGHT: DECIDED_WINNER}, - bits={DECIDED_WINNER: bits}, - ) - ) - scaled = _scaled(bits) - # The stored fact really is COLLAPSE_DIFFICULTY_SCALE times - # the float the node reports for the very same header -- - # only approximately so, which is itself why bits and not - # the float are authoritative here. - self.assertAlmostEqual( - _raw(bits) * COLLAPSE_DIFFICULTY_SCALE / scaled, - 1.0, - places=9, - ) - fixture.seed([_hash(1)], network_difficulty=scaled) - self.assertEqual(_selected(fixture), {_hash(1)}) - - def test_candidate_heavier_than_the_occupant_is_preserved(self) -> None: - """One scaled unit of extra work is still a heavier sibling.""" - for bits in (RETARGETED_BITS, HIGH_WORK_BITS): - with self.subTest(bits=bits): - fixture = CollapseFixture( - rpc=CollapseChainRpc( - tip=DECIDED_WINNER, - tip_height=DECIDED_HEIGHT, - active={DECIDED_HEIGHT: DECIDED_WINNER}, - bits={DECIDED_WINNER: bits}, - ) - ) - scaled = _scaled(bits) - fixture.seed([_hash(1)], network_difficulty=scaled + 1) - fixture.seed([_hash(2)], network_difficulty=scaled) - self.assertEqual(_selected(fixture), {_hash(2)}) - self.assertEqual(fixture.pending(), {_hash(1)}) - - def test_collapse_reuses_the_production_scaled_difficulty_formula(self) -> None: - """The restated formula must not drift from ``template_artifacts``.""" - for bits in ( - MIN_WORK_BITS, - RETARGETED_BITS, - HIGH_WORK_BITS, - "1f00ffff", - "1e00ffff", - "1c7fffff", - ): - with self.subTest(bits=bits): - self.assertEqual( - _collapse_scaled_difficulty(bits), - scaled_network_difficulty(bits), - ) - - def test_unusable_header_bits_fails_the_page_closed(self) -> None: - for value in ( - None, - "", - "n/a", - "not-hex!", - "1d00ff", - "1d00ffff00", - "00000000", - "00000001", - 0x1D00FFFF, - True, - b"1d00ffff", - ): - with self.subTest(bits=value): - fixture = CollapseFixture() - fixture.seed([_hash(1), _hash(2)]) - fixture.rpc.results["getblockheader"] = { - "hash": DECIDED_WINNER, - "bits": value, - # A usable raw float is present and must not rescue the - # read: the scaled comparison has no raw fallback. - "difficulty": 1.0, - } - fixture.collapse() - self.assertEqual(fixture.pending(), {_hash(1), _hash(2)}) - self.assertEqual(fixture.counts()["fail_closed"], 2) - - def test_header_without_bits_fails_the_page_closed(self) -> None: - fixture = CollapseFixture() - fixture.seed([_hash(1), _hash(2)]) - fixture.rpc.results["getblockheader"] = { - "hash": DECIDED_WINNER, - "height": DECIDED_HEIGHT, - "confirmations": 1, - "difficulty": 1.0, - } - fixture.collapse() - self.assertEqual(fixture.pending(), {_hash(1), _hash(2)}) - self.assertEqual(fixture.counts()["fail_closed"], 2) - self.assertEqual(fixture.ledger.abandon_calls, []) - - def test_each_chain_read_failure_fails_the_page_closed(self) -> None: - for method in ( - "getbestblockhash", - "getblockcount", - "getblockhash", - "getblockheader", - ): - with self.subTest(method=method): - fixture = CollapseFixture() - fixture.seed([_hash(1), _hash(2)]) - fixture.rpc.failures.add(method) - retained, _log = fixture.collapse() - self.assertEqual(fixture.pending(), {_hash(1), _hash(2)}) - self.assertEqual(len(retained), 2) - self.assertEqual(fixture.counts()["fail_closed"], 2) - self.assertEqual(fixture.counts()["selected"], 0) - self.assertEqual(fixture.ledger.abandon_calls, []) - - def test_unknown_chain_read_results_fail_the_page_closed(self) -> None: - for method, value in ( - ("getbestblockhash", None), - ("getbestblockhash", "not-a-hash"), - ("getblockcount", "abc"), - ("getblockcount", True), - ("getblockhash", ""), - ("getblockheader", "not-an-object"), - ): - with self.subTest(method=method, value=value): - fixture = CollapseFixture() - fixture.seed([_hash(1)]) - fixture.rpc.results[method] = value - fixture.collapse() - self.assertEqual(fixture.pending(), {_hash(1)}) - self.assertGreaterEqual(fixture.counts()["fail_closed"], 1) - - def test_malformed_intent_fails_the_whole_page_closed(self) -> None: - fixture = CollapseFixture() - fixture.seed([_hash(1), _hash(2)]) - - def corrupt(rows: list[dict[str, Any]]) -> list[dict[str, Any]]: - patched = [] - for row in rows: - if str(row["block_hash"]).lower() == _hash(1): - intent = dict(row["candidate"]) - intent["expected_height"] = "not-a-height" - row = {**row, "candidate": intent} - patched.append(row) - return patched - - fixture.ledger.page_hook = corrupt - self.assertEqual(_selected(fixture), set()) - self.assertEqual(fixture.pending(), {_hash(1), _hash(2)}) - self.assertEqual(fixture.counts()["fail_closed"], 2) - self.assertEqual(fixture.ledger.abandon_calls, []) - - def test_intent_disagreeing_with_template_fails_the_page_closed(self) -> None: - fixture = CollapseFixture() - fixture.seed([_hash(1), _hash(2)]) - - def corrupt(rows: list[dict[str, Any]]) -> list[dict[str, Any]]: - patched = [] - for row in rows: - if str(row["block_hash"]).lower() == _hash(1): - intent = dict(row["candidate"]) - intent["template"] = { - **intent["template"], - "height": DECIDED_HEIGHT + 5, - } - row = {**row, "candidate": intent} - patched.append(row) - return patched - - fixture.ledger.page_hook = corrupt - self.assertEqual(_selected(fixture), set()) - self.assertEqual(fixture.pending(), {_hash(1), _hash(2)}) - self.assertEqual(fixture.counts()["fail_closed"], 2) - - def test_unusable_candidate_difficulty_fails_the_page_closed(self) -> None: - for value in (None, "n/a", float("nan"), True, float("inf"), 0, -1): - with self.subTest(difficulty=value): - fixture = CollapseFixture() - fixture.seed([_hash(1), _hash(2)]) - - def corrupt(rows: list[dict[str, Any]]) -> list[dict[str, Any]]: - intent = dict(rows[0]["candidate"]) - intent["found_block"] = { - **intent["found_block"], - "network_difficulty": value, - } - rows[0] = {**rows[0], "candidate": intent} - return rows - - fixture.ledger.page_hook = corrupt - self.assertEqual(_selected(fixture), set()) - self.assertEqual(fixture.pending(), {_hash(1), _hash(2)}) - self.assertEqual(fixture.counts()["fail_closed"], 2) - - def test_chain_is_read_once_per_page_and_once_per_distinct_height(self) -> None: - fixture = CollapseFixture( - rpc=CollapseChainRpc( - tip=DECIDED_WINNER, - tip_height=DECIDED_HEIGHT, - active={ - DECIDED_HEIGHT: DECIDED_WINNER, - DECIDED_HEIGHT - 1: "ee" * 32, - }, - ) - ) - fixture.seed([_hash(index) for index in range(1, 51)]) - fixture.seed( - [_hash(index) for index in range(51, 101)], - height=DECIDED_HEIGHT - 1, - ) - fixture.collapse() - counts = fixture.rpc.counts() - # One tip read for the page plus one for the pre-write revalidation; - # one active-block read per distinct height in each pass; the - # occupant header only in the selection pass, because revalidation - # found both occupants unchanged. - self.assertEqual(counts["getbestblockhash"], 2) - self.assertEqual(counts["getblockcount"], 1) - self.assertEqual(counts["getblockhash"], 4) - self.assertEqual(counts["getblockheader"], 2) - self.assertEqual(fixture.counts()["abandoned"], 100) - - -class BlockCandidateCollapseApplyTests(unittest.TestCase): - """Lease claiming, revalidation, the single write, and its result set.""" - - def test_lease_is_claimed_without_blocking(self) -> None: - fixture = CollapseFixture() - fixture.seed([_hash(1), _hash(2)]) - claims: list[bool] = [] - original = fixture.server._claim_block_candidate_disposition - - def record(block_hash: str, *, blocking: bool): - claims.append(blocking) - return original(block_hash, blocking=blocking) - - fixture.server._claim_block_candidate_disposition = record - fixture.collapse() - self.assertTrue(claims) - self.assertEqual(set(claims), {False}) - - def test_unclaimable_lease_is_skipped_not_waited_on(self) -> None: - fixture = CollapseFixture() - fixture.seed([_hash(1), _hash(2)]) - original = fixture.server._claim_block_candidate_disposition - - def refuse(block_hash: str, *, blocking: bool): - if block_hash == _hash(1): - return None - return original(block_hash, blocking=blocking) - - fixture.server._claim_block_candidate_disposition = refuse - self.assertEqual(_selected(fixture), {_hash(2)}) - self.assertEqual(fixture.counts()["lease_skipped"], 1) - self.assertEqual(fixture.ledger.abandon_calls[0][0], (_hash(2),)) - - def test_apply_leases_do_not_disqualify_their_own_rows(self) -> None: - """The apply's own flights must not read back as busy evidence.""" - fixture = CollapseFixture() - fixture.seed([_hash(1), _hash(2)]) - seen: list[frozenset[str]] = [] - original = fixture.service._block_candidate_collapse_evidence - - def record(hashes, *, ignore_leases=frozenset()): - seen.append(frozenset(ignore_leases)) - return original(hashes, ignore_leases=ignore_leases) - - fixture.service._block_candidate_collapse_evidence = record - self.assertEqual(_selected(fixture), {_hash(1), _hash(2)}) - self.assertEqual(seen[0], frozenset()) - self.assertEqual(seen[1], frozenset({_hash(1), _hash(2)})) - - def test_every_lease_is_released_after_the_apply(self) -> None: - fixture = CollapseFixture() - fixture.seed([_hash(1), _hash(2)]) - fixture.collapse() - with fixture.service._block_candidate_disposition_registry_lock: - self.assertEqual(fixture.service._block_candidate_disposition_flights, {}) - - def test_leases_are_released_when_the_write_fails(self) -> None: - fixture = CollapseFixture() - fixture.seed([_hash(1)]) - - def explode(hashes: tuple[str, ...], error: str): - raise RuntimeError("ledger down") - - fixture.ledger.abandon_hook = explode - fixture.collapse() - with fixture.service._block_candidate_disposition_registry_lock: - self.assertEqual(fixture.service._block_candidate_disposition_flights, {}) - self.assertEqual(fixture.pending(), {_hash(1)}) - self.assertEqual(fixture.counts()["fail_closed"], 1) - - def test_leases_are_released_when_cleanup_fails(self) -> None: - fixture = CollapseFixture() - fixture.seed([_hash(1)]) - fixture.server._clear_accepted_block_payout_preview = ( - lambda *args, **kwargs: (_ for _ in ()).throw(RuntimeError("nope")) - ) - fixture.collapse() - with fixture.service._block_candidate_disposition_registry_lock: - self.assertEqual(fixture.service._block_candidate_disposition_flights, {}) - self.assertEqual(fixture.counts()["cleanup_failed"], 1) - - def test_revalidation_drops_a_candidate_that_became_the_active_block(self) -> None: - """The immediate pre-write re-read closes the select-to-write gap.""" - fixture = CollapseFixture() - fixture.seed([_hash(1), _hash(2)]) - state = {"selected": False} - rpc = fixture.rpc - - def move_chain(chain: CollapseChainRpc, method: str, params) -> None: - if method != "getbestblockhash": - return - if state["selected"]: - # Second best-tip read: the revalidation. Candidate 1 was - # offered, accepted, and is now the active block. - chain.tip = _hash(1) - chain.active = {DECIDED_HEIGHT: _hash(1)} - state["selected"] = True - - rpc.before_call = move_chain - self.assertEqual(_selected(fixture), {_hash(2)}) - self.assertEqual(fixture.counts()["revalidation_dropped"], 1) - self.assertEqual(fixture.ledger.abandon_calls[0][0], (_hash(2),)) - - def test_revalidation_drops_a_candidate_whose_parent_became_the_tip(self) -> None: - fixture = CollapseFixture() - fixture.seed([_hash(1)], parent=STORM_PARENT) - fixture.seed([_hash(2)], parent=OTHER_PARENT) - state = {"seen": 0} - - def move_chain(chain: CollapseChainRpc, method: str, params) -> None: - if method != "getbestblockhash": - return - state["seen"] += 1 - if state["seen"] == 2: - chain.tip = OTHER_PARENT - - fixture.rpc.before_call = move_chain - self.assertEqual(_selected(fixture), {_hash(1)}) - self.assertEqual(fixture.counts()["revalidation_dropped"], 1) - - def test_revalidation_reads_a_changed_occupants_work(self) -> None: - """A lighter replacement occupant drops the row, in scaled units.""" - fixture = CollapseFixture() - fixture.seed([_hash(1)], network_difficulty=_scaled(RETARGETED_BITS)) - fixture.rpc.bits[DECIDED_WINNER] = RETARGETED_BITS - fixture.rpc.bits["ee" * 32] = MIN_WORK_BITS - state = {"seen": 0} - - def move_chain(chain: CollapseChainRpc, method: str, params) -> None: - if method != "getbestblockhash": - return - state["seen"] += 1 - if state["seen"] == 2: - chain.tip = "ee" * 32 - chain.active = {DECIDED_HEIGHT: "ee" * 32} - - fixture.rpc.before_call = move_chain - self.assertEqual(_selected(fixture), set()) - self.assertEqual(fixture.counts()["revalidation_dropped"], 1) - headers = [ - params for method, params in fixture.rpc.calls if method == "getblockheader" - ] - self.assertEqual(headers, [(DECIDED_WINNER,), ("ee" * 32,)]) - - def test_revalidation_keeps_an_equal_work_changed_occupant(self) -> None: - """The changed-occupant re-read normalizes exactly as selection does. - - The replacement occupant carries different bits from the one - selection read but exactly the candidate's scaled work, so the row - still collapses. A revalidation that compared the replacement's raw - header float against the row's scaled value would drop it instead. - """ - fixture = CollapseFixture() - fixture.seed([_hash(1)], network_difficulty=_scaled(HIGH_WORK_BITS)) - fixture.rpc.bits[DECIDED_WINNER] = HIGH_WORK_BITS - fixture.rpc.bits["ee" * 32] = HIGH_WORK_BITS - state = {"seen": 0} - - def move_chain(chain: CollapseChainRpc, method: str, params) -> None: - if method != "getbestblockhash": - return - state["seen"] += 1 - if state["seen"] == 2: - chain.active = {DECIDED_HEIGHT: "ee" * 32} - - fixture.rpc.before_call = move_chain - self.assertEqual(_selected(fixture), {_hash(1)}) - self.assertEqual(fixture.counts()["revalidation_dropped"], 0) - headers = [ - params for method, params in fixture.rpc.calls if method == "getblockheader" - ] - self.assertEqual(headers, [(DECIDED_WINNER,), ("ee" * 32,)]) - - def test_revalidation_of_a_changed_occupant_fails_closed_without_bits( - self, - ) -> None: - """The pre-write header re-read has no raw-difficulty fallback either.""" - fixture = CollapseFixture() - fixture.seed([_hash(1)], network_difficulty=_scaled(RETARGETED_BITS)) - fixture.rpc.bits[DECIDED_WINNER] = RETARGETED_BITS - state = {"seen": 0} - - def move_chain(chain: CollapseChainRpc, method: str, params) -> None: - if method != "getbestblockhash": - return - state["seen"] += 1 - if state["seen"] == 2: - chain.active = {DECIDED_HEIGHT: "ee" * 32} - chain.results["getblockheader"] = { - "hash": "ee" * 32, - "difficulty": _raw(RETARGETED_BITS), - } - - fixture.rpc.before_call = move_chain - fixture.collapse() - self.assertEqual(fixture.pending(), {_hash(1)}) - self.assertEqual(fixture.counts()["fail_closed"], 1) - self.assertEqual(fixture.ledger.abandon_calls, []) - - def test_revalidation_failure_fails_the_page_closed(self) -> None: - fixture = CollapseFixture() - fixture.seed([_hash(1), _hash(2)]) - state = {"seen": 0} - - def fail_second(chain: CollapseChainRpc, method: str, params) -> None: - if method != "getbestblockhash": - return - state["seen"] += 1 - if state["seen"] == 2: - chain.failures.add("getbestblockhash") - - fixture.rpc.before_call = fail_second - retained, _log = fixture.collapse() - self.assertEqual(fixture.pending(), {_hash(1), _hash(2)}) - self.assertEqual(len(retained), 2) - self.assertEqual(fixture.ledger.abandon_calls, []) - self.assertEqual(fixture.counts()["fail_closed"], 2) - - def test_one_batch_write_replaces_the_per_row_writes(self) -> None: - fixture = CollapseFixture() - fixture.seed([_hash(index) for index in range(1, 21)]) - fixture.collapse() - self.assertEqual(len(fixture.ledger.abandon_calls), 1) - requested, error = fixture.ledger.abandon_calls[0] - self.assertEqual(len(requested), 20) - self.assertIn("tip moved before submit", error) - self.assertIn(DECIDED_WINNER, error) - - def test_cleanup_follows_the_returned_set_not_the_request(self) -> None: - fixture = CollapseFixture() - fixture.seed([_hash(1), _hash(2), _hash(3)]) - service = fixture.service - - def partial(hashes: tuple[str, ...], error: str) -> tuple[str, ...]: - won = tuple(value for value in hashes if value != _hash(2)) - return fixture.ledger.inner.mark_block_candidates_abandoned( - block_hashes=won, - error=error, - ) - - fixture.ledger.abandon_hook = partial - retained, _log = fixture.collapse() - self.assertEqual(fixture.pending(), {_hash(2)}) - self.assertEqual(fixture.counts()["abandoned"], 2) - self.assertEqual(fixture.counts()["write_lost"], 1) - with fixture.server.lock: - terminal = dict(service._block_candidate_terminal_outcomes) - self.assertEqual(set(terminal), {_hash(1), _hash(3)}) - self.assertNotIn(_hash(2), terminal) - self.assertEqual( - {str(row["block_hash"]).lower() for row in retained}, - {_hash(2)}, - ) - - def test_a_hash_outside_the_request_is_never_cleaned_up(self) -> None: - fixture = CollapseFixture() - fixture.seed([_hash(1)]) - stranger = _hash(99) - - def stray(hashes: tuple[str, ...], error: str) -> tuple[str, ...]: - fixture.ledger.inner.mark_block_candidates_abandoned( - block_hashes=hashes, - error=error, - ) - return hashes + (stranger,) - - fixture.ledger.abandon_hook = stray - fixture.collapse() - with fixture.server.lock: - terminal = dict(fixture.service._block_candidate_terminal_outcomes) - self.assertEqual(set(terminal), {_hash(1)}) - - def test_an_empty_returned_set_preserves_the_whole_page(self) -> None: - fixture = CollapseFixture() - fixture.seed([_hash(1), _hash(2)]) - fixture.ledger.abandon_hook = lambda hashes, error: () - rows = fixture.page() - retained, _log = fixture.collapse(rows) - self.assertEqual(len(retained), 2) - self.assertEqual(fixture.counts()["abandoned"], 0) - self.assertEqual(fixture.counts()["write_lost"], 2) - with fixture.server.lock: - self.assertEqual(fixture.service._block_candidate_terminal_outcomes, {}) - - def test_a_non_hash_set_result_fails_the_page_closed(self) -> None: - """A count or a boolean says nothing about which rows were won.""" - for value in (True, 2, None, "01" * 32): - with self.subTest(returned=value): - fixture = CollapseFixture() - fixture.seed([_hash(1), _hash(2)]) - - def answer(hashes: tuple[str, ...], error: str, value=value): - fixture.ledger.inner.mark_block_candidates_abandoned( - block_hashes=hashes, - error=error, - ) - return value - - fixture.ledger.abandon_hook = answer - rows = fixture.page() - retained, _log = fixture.collapse(rows) - self.assertEqual(len(retained), 2) - self.assertEqual(fixture.counts()["abandoned"], 0) - self.assertEqual(fixture.counts()["fail_closed"], 2) - with fixture.server.lock: - self.assertEqual( - fixture.service._block_candidate_terminal_outcomes, - {}, - ) - - def test_no_write_is_issued_for_an_empty_selection(self) -> None: - fixture = CollapseFixture( - rpc=CollapseChainRpc(tip=STORM_PARENT, tip_height=DECIDED_HEIGHT) - ) - fixture.seed([_hash(1)]) - fixture.collapse() - self.assertEqual(fixture.ledger.abandon_calls, []) - self.assertEqual(fixture.pending(), {_hash(1)}) - - def test_a_ledger_without_the_batch_method_is_left_entirely_alone(self) -> None: - fixture = CollapseFixture(ledger=NoBatchCollapseLedger()) - fixture.seed([_hash(1)]) - rows = fixture.page() - retained, _log = fixture.collapse(rows) - self.assertEqual(retained, rows) - self.assertEqual(fixture.rpc.calls, []) - self.assertEqual(fixture.counts()["considered"], 0) - self.assertEqual(fixture.pending(), {_hash(1)}) - - -class BlockCandidateCollapseCleanupTests(unittest.TestCase): - """Cleanup list M: everything terminal per-row abandonment releases.""" - - def _fixture_with_state(self) -> tuple[CollapseFixture, str]: - fixture = CollapseFixture() - target = _hash(1) - fixture.seed([target]) - service = fixture.service - server = fixture.server - server._register_outstanding_block_candidate(target) - server._begin_accepted_block_payout_preview( - target, - block_height=DECIDED_HEIGHT, - ) - server._mark_accepted_block_payout_landed( - target, - block_height=DECIDED_HEIGHT, - ) - with server.lock: - service._block_fast_lane_reservations.add(target) - service._block_replay_inflight_hashes.add(target) - service.retry_delays[target] = 4.0 - # Both registries are created on first touch by the pacing and - # landing-timeout paths; seed them the way those paths would. - service._block_candidate_retry_not_before = {target: 1.0} - service._block_landing_timeout_counts = {target: 3} - return fixture, target - - def test_cleanup_releases_every_owned_state(self) -> None: - fixture, target = self._fixture_with_state() - service = fixture.service - server = fixture.server - with redirect_stdout(StringIO()): - fixture.collapse() - self.assertEqual(fixture.pending(), set()) - with server.lock: - self.assertIs(service._block_candidate_terminal_outcomes[target], False) - self.assertNotIn(target, service._block_fast_lane_reservations) - self.assertNotIn(target, service._block_replay_inflight_hashes) - self.assertNotIn(target, service._outstanding_block_candidate_hashes) - self.assertNotIn(target, service._tip_observed_accepted_block_hashes) - self.assertNotIn(target, service.retry_delays) - self.assertNotIn(target, service._block_candidate_retry_not_before) - self.assertNotIn(target, service._block_landing_timeout_counts) - self.assertNotIn(target, service.finalize_retries) - with server._accepted_block_payout_preview_condition: - self.assertNotIn(target, server._accepted_block_payout_previews) - self.assertNotIn( - target, - server._invalidated_accepted_block_payout_previews, - ) - self.assertEqual( - server.block_candidate_abandoned_counts, - {PRISM_BLOCK_CANDIDATE_COLLAPSE_REASON: 1}, - ) - self.assertEqual( - server.stale_job_abandon_counts[ - PRISM_BLOCK_CANDIDATE_COLLAPSE_STALE_JOB_CLASS - ], - 1, - ) - - def test_cleanup_leaves_no_payout_tombstone_behind(self) -> None: - fixture, target = self._fixture_with_state() - with redirect_stdout(StringIO()): - fixture.collapse() - with fixture.server._accepted_block_payout_preview_condition: - self.assertEqual( - fixture.server._invalidated_accepted_block_payout_previews, - {}, - ) - - def test_finalize_retry_state_is_removed_by_cleanup(self) -> None: - fixture = CollapseFixture() - target = _hash(1) - fixture.seed([target]) - with fixture.server.lock: - fixture.service.finalize_retries[target] = (False, "stale") - with redirect_stdout(StringIO()): - fixture.service._clean_up_collapsed_block_candidates((target,)) - with fixture.server.lock: - self.assertEqual(fixture.service.finalize_retries, {}) - - def test_abandonment_is_counted_exactly_once(self) -> None: - fixture = CollapseFixture() - target = _hash(1) - fixture.seed([target]) - with redirect_stdout(StringIO()): - fixture.service._clean_up_collapsed_block_candidates((target,)) - fixture.service._clean_up_collapsed_block_candidates((target,)) - self.assertEqual( - fixture.server.block_candidate_abandoned_counts, - {PRISM_BLOCK_CANDIDATE_COLLAPSE_REASON: 1}, - ) - self.assertEqual( - fixture.server.stale_job_abandon_counts[ - PRISM_BLOCK_CANDIDATE_COLLAPSE_STALE_JOB_CLASS - ], - 1, - ) - - def test_cleanup_is_idempotent(self) -> None: - fixture, target = self._fixture_with_state() - with redirect_stdout(StringIO()): - fixture.service._clean_up_collapsed_block_candidates((target,)) - fixture.service._clean_up_collapsed_block_candidates((target,)) - self.assertEqual(fixture.counts()["cleanup_failed"], 0) - - def test_one_failing_cleanup_does_not_strand_later_steps_or_rows(self) -> None: - fixture = CollapseFixture() - hashes = [_hash(1), _hash(2), _hash(3)] - candidates = fixture.seed(hashes, credit_share_on_accept=True) - writer = fixture.server._ensure_share_writer_service() - writer.adopt_pending_share(candidates[1].pending_share) - fixture.service.candidate_queue.put_nowait(candidates[1]) - original = fixture.server._clear_block_candidate_retry_state - - def explode(block_hash: str) -> None: - if block_hash == _hash(2): - raise RuntimeError("cleanup boom") - original(block_hash) - - fixture.server._clear_block_candidate_retry_state = explode - with redirect_stdout(StringIO()): - fixture.collapse() - self.assertEqual(fixture.pending(), set()) - with fixture.server.lock: - terminal = dict(fixture.service._block_candidate_terminal_outcomes) - self.assertEqual(set(terminal), set(hashes)) - self.assertNotIn(id(candidates[1].pending_share), fixture.floor()) - self.assertEqual(fixture.counts()["cleanup_failed"], 1) - self.assertEqual(fixture.counts()["abandoned"], 3) - - def test_systemic_cleanup_failure_logging_stays_bounded(self) -> None: - fixture = CollapseFixture() - hashes = [_hash(index) for index in range(1, 41)] - fixture.seed(hashes) - fixture.server._clear_block_candidate_retry_state = ( - lambda block_hash: (_ for _ in ()).throw(RuntimeError("boom")) - ) - _retained, log = fixture.collapse() - detailed = [ - line - for line in log.splitlines() - if "cleanup failed step=" in line - ] - self.assertEqual(len(detailed), BLOCK_CANDIDATE_COLLAPSE_LOG_FAILURES) - self.assertIn("cleanup failed rows=40 of 40", log) - self.assertEqual(fixture.counts()["cleanup_failed"], 40) - - def test_an_aborted_cleanup_still_partitions_the_terminal_rows(self) -> None: - """A terminal row must never be replay-adopted, cleanup or not.""" - fixture = CollapseFixture() - fixture.seed([_hash(1), _hash(2)]) - fixture.service._collapsed_candidate_floor_holders = ( - lambda abandoned: (_ for _ in ()).throw(RuntimeError("boom")) - ) - rows = fixture.page() - retained, log = fixture.collapse(rows) - self.assertEqual(retained, []) - self.assertEqual(fixture.pending(), set()) - self.assertIn("cleanup aborted rows=2", log) - self.assertEqual(fixture.counts()["cleanup_failed"], 2) - - def test_a_log_failure_does_not_double_count_cleanup_failures(self) -> None: - """cleanup_failed counts affected hashes, not diagnostic failures. - - A contained per-step failure belongs to the one hash it happened to, - while a later summary formatter fault does not change any hash's - cleanup state. Treating both as cleanup reported four affected hashes - for a three-row page even though only one cleanup actually failed. - """ - fixture = CollapseFixture() - hashes = [_hash(1), _hash(2), _hash(3)] - fixture.seed(hashes) - original = fixture.server._clear_block_candidate_retry_state - - def explode(block_hash: str) -> None: - if block_hash == _hash(2): - raise RuntimeError("cleanup boom") - original(block_hash) - - fixture.server._clear_block_candidate_retry_state = explode - # The grouped formatter runs only after every cleanup step. - fixture.service._log_collapsed_block_candidates = ( - lambda *args, **kwargs: (_ for _ in ()).throw(RuntimeError("log boom")) - ) - rows = fixture.page() - retained, log = fixture.collapse(rows) - counts = fixture.counts() - self.assertEqual(counts["abandoned"], 3) - self.assertEqual(counts["cleanup_failed"], 1) - self.assertLessEqual(counts["cleanup_failed"], counts["abandoned"]) - # Both faults are visible, every cleanup attempt still ran, and the - # terminal rows are still partitioned out of the page. - self.assertIn("cleanup failed rows=1 of 3", log) - self.assertIn("logging failed rows=3", log) - self.assertNotIn("cleanup aborted rows=3", log) - self.assertEqual(retained, []) - self.assertEqual(fixture.pending(), set()) - with fixture.server.lock: - self.assertEqual( - set(fixture.service._block_candidate_terminal_outcomes), - set(hashes), - ) - with fixture.service._block_candidate_disposition_registry_lock: - self.assertEqual(fixture.service._block_candidate_disposition_flights, {}) - - def test_a_queued_candidates_floor_holder_is_released(self) -> None: - fixture = CollapseFixture() - target = _hash(1) - candidate = fixture.seed([target], credit_share_on_accept=True)[0] - writer = fixture.server._ensure_share_writer_service() - writer.adopt_pending_share(candidate.pending_share) - fixture.service.candidate_queue.put_nowait(candidate) - self.assertIn(id(candidate.pending_share), fixture.floor()) - with redirect_stdout(StringIO()): - fixture.collapse() - self.assertNotIn(id(candidate.pending_share), fixture.floor()) - - def test_a_replay_queued_candidates_floor_holder_is_released(self) -> None: - fixture = CollapseFixture() - target = _hash(1) - candidate = fixture.seed([target], credit_share_on_accept=True)[0] - writer = fixture.server._ensure_share_writer_service() - writer.adopt_pending_share(candidate.pending_share) - fixture.service._block_replay_candidate_queue.put_nowait(candidate) - with redirect_stdout(StringIO()): - fixture.collapse() - self.assertNotIn(id(candidate.pending_share), fixture.floor()) - - def test_another_candidates_floor_holder_is_never_released(self) -> None: - fixture = CollapseFixture() - collapsed = fixture.seed([_hash(1)], credit_share_on_accept=True)[0] - # A same-hash duplicate object and an unrelated hash both keep their - # own identity-keyed holders; only the collapsed object's is dropped. - duplicate = fixture.seed( - [_hash(1)], - credit_share_on_accept=True, - )[0] - untouched = fixture.seed( - [_hash(2)], - parent=OTHER_PARENT, - credit_share_on_accept=True, - )[0] - writer = fixture.server._ensure_share_writer_service() - for candidate in (collapsed, duplicate, untouched): - writer.adopt_pending_share(candidate.pending_share) - fixture.service.candidate_queue.put_nowait(collapsed) - fixture.rpc.tip = OTHER_PARENT - with redirect_stdout(StringIO()): - fixture.collapse() - floor = fixture.floor() - self.assertNotIn(id(collapsed.pending_share), floor) - self.assertIn(id(duplicate.pending_share), floor) - self.assertIn(id(untouched.pending_share), floor) - - def test_no_floor_holder_is_invented_for_an_unadopted_row(self) -> None: - fixture = CollapseFixture() - fixture.seed([_hash(1)], credit_share_on_accept=True) - before = set(fixture.floor()) - with redirect_stdout(StringIO()): - fixture.collapse() - self.assertEqual(set(fixture.floor()), before) - - def test_cleanup_is_safe_for_an_already_terminal_hash(self) -> None: - fixture = CollapseFixture() - target = _hash(1) - fixture.seed([target]) - fixture.server._record_block_candidate_terminal_outcome( - target, - accepted=True, - ) - with redirect_stdout(StringIO()): - fixture.service._clean_up_collapsed_block_candidates((target,)) - with fixture.server.lock: - self.assertIs( - fixture.service._block_candidate_terminal_outcomes[target], - False, - ) - self.assertEqual(fixture.counts()["cleanup_failed"], 0) - - def test_collapse_reason_matches_the_per_row_stale_job_reason(self) -> None: - self.assertEqual( - PRISM_BLOCK_CANDIDATE_COLLAPSE_REASON, - PRISM_REJECTION_STALE_JOB, - ) - self.assertIn( - PRISM_BLOCK_CANDIDATE_COLLAPSE_STALE_JOB_CLASS, - PRISM_STALE_JOB_ABANDON_CLASSES, - ) - - -def _break_cleanup_step(target: Any, name: str, *, only: str | None = None) -> dict: - """Make one cleanup step raise until the returned switch is flipped.""" - original = getattr(target, name) - broken = {"on": True} - - def failing(block_hash: str, *args: Any, **kwargs: Any) -> Any: - if broken["on"] and (only is None or block_hash == only): - raise RuntimeError(f"{name} boom") - return original(block_hash, *args, **kwargs) - - setattr(target, name, failing) - return broken - - -class AccountingLaneRig: - """Drive the shipped accounting loop against controllable lane traffic. - - Only the two sinks the loop reaches through the coordinator are replaced - -- the ordinary accounting task runner and the invalid-candidate - quarantine drain -- so the loop body, its cleanup-retry cadence, and the - real deferred cleanup all stay exactly as shipped. ``mode`` chooses - which lane is kept permanently busy: - - ``accounting`` - Each finished task re-arms the primary handoff queue before it - returns, so the loop's next queue read can never see an empty lane. - ``quarantine`` - Both handoff queues stay empty and the quarantine drain always - reports an item, so the loop never reaches its idle branch either. - ``both`` - The two alternate: each quarantine item re-arms the accounting - queue, so one work item of each kind completes per pair. - ``idle`` - No traffic at all, so the loop falls straight through to the idle - branch that predates the cadence. - - In every busy mode the idle branch is unreachable by construction, which - is what makes these fixtures a starvation test rather than a race. - """ - - ACCOUNTING = "accounting" - QUARANTINE = "quarantine" - BOTH = "both" - IDLE = "idle" - - def __init__(self, fixture: CollapseFixture, mode: str) -> None: - self.fixture = fixture - self.server = fixture.server - self.service = fixture.service - self.mode = mode - self.accounting_items = 0 - self.quarantine_items = 0 - # (accounting items, quarantine items) completed at each offer. - self.offers: list[tuple[int, int]] = [] - self.attempts = 0 - self.log = "" - self.thread_alive = False - self._sequence = 0 - self.service._ensure_block_accounting_state() - self.service._ensure_block_replay_state() - self.server._run_block_accounting_task = self._run_task - self.server._run_one_invalid_block_candidate_quarantine = self._run_quarantine - shipped = self.service._run_one_collapsed_block_candidate_cleanup_retry - - def offer() -> bool: - self.offers.append((self.accounting_items, self.quarantine_items)) - ran = shipped() - if ran: - self.attempts += 1 - return ran - - self.service._run_one_collapsed_block_candidate_cleanup_retry = offer - if mode in (self.ACCOUNTING, self.BOTH): - self._arm_accounting() - - @property - def work_items(self) -> int: - return self.accounting_items + self.quarantine_items - - def _arm_accounting(self) -> None: - self._sequence += 1 - self.service._block_accounting_queue.put_nowait( - (0, self._sequence, SimpleNamespace()) - ) - - def _run_task(self, task: Any) -> None: - self.accounting_items += 1 - if self.mode == self.ACCOUNTING: - self._arm_accounting() - - def _run_quarantine(self) -> bool: - if self.mode not in (self.QUARANTINE, self.BOTH): - return False - self.quarantine_items += 1 - if self.mode == self.BOTH: - self._arm_accounting() - return True - - def run_until(self, predicate: Any, *, timeout: float = 10.0) -> bool: - """Run the lane until ``predicate`` holds or the deadline expires. - - Bounded on purpose: a starved retry must end the test with a report, - not with an unbounded wait. Counters are only read by callers after - the loop thread has been stopped and joined, so every assertion sees - one consistent snapshot. - """ - thread = threading.Thread( - target=self.service.block_accounting_loop, - daemon=True, - ) - buffer = StringIO() - with redirect_stdout(buffer): - thread.start() - try: - deadline = time.monotonic() + timeout - while not predicate() and time.monotonic() < deadline: - time.sleep(0.002) - finally: - self.server.stop_event.set() - thread.join(timeout=timeout) - self.log = buffer.getvalue() - self.thread_alive = thread.is_alive() - return predicate() - - -class BlockCandidateCollapseCleanupRetryTests(unittest.TestCase): - """A failed cleanup is the only state with no durable replay source. - - Once the fenced batch write returns a hash, that row is terminal: it is - partitioned out of the replay page and no later enumeration can hand it - back. A cleanup step that failed therefore has nothing left to retry - it, and whatever it did not tear down -- a payout preview or its - tombstone, a pending-share floor holder, an outstanding-hash marker -- - would stay installed for the process lifetime. These tests own the - bounded, idempotent retry that closes that gap, and the accounting of - it: the affected-hash series still counts hashes, never attempts. - """ - - @staticmethod - def _breaker(target: Any, name: str, *, only: str | None = None) -> dict: - """Make one cleanup step raise until the returned switch is flipped.""" - return _break_cleanup_step(target, name, only=only) - - # -- payout preview and tombstone -------------------------------------- - - def test_a_failed_payout_withdrawal_is_retried_until_it_recovers(self) -> None: - """The wait storm the collapse exists to end must not be re-armed.""" - fixture = CollapseFixture() - target = _hash(1) - fixture.seed([target]) - server = fixture.server - server._begin_accepted_block_payout_preview( - target, - block_height=DECIDED_HEIGHT, - ) - server._mark_accepted_block_payout_landed( - target, - block_height=DECIDED_HEIGHT, - ) - broken = self._breaker(server, "_clear_accepted_block_payout_preview") - with redirect_stdout(StringIO()): - fixture.collapse() - # The durable row is terminal and gone from every replay source - # while the landed barrier it armed is still installed: descendants - # keep waiting on a transition nothing else will ever clear. - self.assertEqual(fixture.pending(), set()) - self.assertTrue(server._accepted_block_payout_transition_landed(target)) - self.assertEqual(fixture.counts()["cleanup_failed"], 1) - self.assertEqual( - fixture.cleanup_backlog(), - {target: frozenset({BLOCK_CANDIDATE_COLLAPSE_CLEANUP_PAYOUT_STEP})}, - ) - broken["on"] = False - ran, log = fixture.retry_cleanup() - self.assertTrue(ran) - with server._accepted_block_payout_preview_condition: - self.assertNotIn(target, server._accepted_block_payout_previews) - self.assertNotIn( - target, - server._invalidated_accepted_block_payout_previews, - ) - self.assertEqual(fixture.cleanup_backlog(), {}) - self.assertIn("cleanup recovered", log) - counts = fixture.counts() - self.assertEqual(counts["cleanup_recovered"], 1) - # The affected-hash series counts the hash once, whatever it took. - self.assertEqual(counts["cleanup_failed"], 1) - - def test_a_failed_tombstone_drop_is_retried(self) -> None: - """A retained tombstone fails descendants closed just as hard.""" - fixture = CollapseFixture() - target = _hash(1) - fixture.seed([target]) - server = fixture.server - server._begin_accepted_block_payout_preview( - target, - block_height=DECIDED_HEIGHT, - ) - server._mark_accepted_block_payout_landed( - target, - block_height=DECIDED_HEIGHT, - ) - original = server._clear_accepted_block_payout_preview - broken = {"on": True} - - def clear(block_hash: str, *, invalidate_published: bool = False) -> None: - # The withdrawal lands and installs the tombstone; only the - # second half -- the drop -- fails. - if broken["on"] and not invalidate_published: - raise RuntimeError("tombstone drop boom") - original(block_hash, invalidate_published=invalidate_published) - - server._clear_accepted_block_payout_preview = clear - with redirect_stdout(StringIO()): - fixture.collapse() - with server._accepted_block_payout_preview_condition: - self.assertIn( - target, - server._invalidated_accepted_block_payout_previews, - ) - broken["on"] = False - self.assertTrue(fixture.retry_cleanup()[0]) - with server._accepted_block_payout_preview_condition: - self.assertEqual(server._invalidated_accepted_block_payout_previews, {}) - self.assertEqual(fixture.cleanup_backlog(), {}) - - # -- pending-share floor ----------------------------------------------- - - def test_a_failed_floor_release_survives_the_queue_it_was_read_from(self) -> None: - """The floor keys holders by identity, so the retry must carry them.""" - fixture = CollapseFixture() - target = _hash(1) - candidate = fixture.seed([target], credit_share_on_accept=True)[0] - server = fixture.server - server._ensure_share_writer_service().adopt_pending_share( - candidate.pending_share - ) - fixture.service.candidate_queue.put_nowait(candidate) - original = server._finish_pending_share_commit - broken = {"on": True} - - def finish(pending_share: Any) -> None: - if broken["on"]: - raise RuntimeError("floor release boom") - original(pending_share) - - server._finish_pending_share_commit = finish - with redirect_stdout(StringIO()): - fixture.collapse() - self.assertIn(id(candidate.pending_share), fixture.floor()) - self.assertEqual( - fixture.cleanup_backlog(), - {target: frozenset({BLOCK_CANDIDATE_COLLAPSE_CLEANUP_FLOOR_STEP})}, - ) - # The queue the holder was indexed from drains before the retry - # runs, so a fresh scan would find nothing to release. - self.assertIs(fixture.service.candidate_queue.get_nowait(), candidate) - self.assertEqual( - fixture.service._collapsed_candidate_floor_holders((target,)), - {}, - ) - broken["on"] = False - self.assertTrue(fixture.retry_cleanup()[0]) - self.assertNotIn(id(candidate.pending_share), fixture.floor()) - self.assertEqual(fixture.cleanup_backlog(), {}) - self.assertEqual(fixture.counts()["cleanup_recovered"], 1) - - def test_another_candidates_floor_holder_is_never_released_by_a_retry( - self, - ) -> None: - fixture = CollapseFixture() - collapsed = fixture.seed([_hash(1)], credit_share_on_accept=True)[0] - untouched = fixture.seed( - [_hash(2)], - parent=OTHER_PARENT, - credit_share_on_accept=True, - )[0] - server = fixture.server - writer = server._ensure_share_writer_service() - for candidate in (collapsed, untouched): - writer.adopt_pending_share(candidate.pending_share) - fixture.service.candidate_queue.put_nowait(collapsed) - fixture.rpc.tip = OTHER_PARENT - original = server._finish_pending_share_commit - broken = {"on": True} - - def finish(pending_share: Any) -> None: - if broken["on"]: - raise RuntimeError("floor release boom") - original(pending_share) - - server._finish_pending_share_commit = finish - with redirect_stdout(StringIO()): - fixture.collapse() - broken["on"] = False - self.assertTrue(fixture.retry_cleanup()[0]) - floor = fixture.floor() - self.assertNotIn(id(collapsed.pending_share), floor) - self.assertIn(id(untouched.pending_share), floor) - - # -- what a retry repeats ---------------------------------------------- - - def test_a_retry_repeats_only_the_steps_that_are_still_owed(self) -> None: - fixture = CollapseFixture() - target = _hash(1) - fixture.seed([target]) - server = fixture.server - discarded: list[str] = [] - original_discard = server._discard_outstanding_block_candidate - - def discard(block_hash: str) -> None: - discarded.append(block_hash) - original_discard(block_hash) - - server._discard_outstanding_block_candidate = discard - broken = self._breaker(server, "_clear_block_candidate_retry_state") - with redirect_stdout(StringIO()): - fixture.collapse() - self.assertEqual(discarded, [target]) - self.assertEqual( - fixture.cleanup_backlog(), - {target: frozenset({"retry-state"})}, - ) - broken["on"] = False - self.assertTrue(fixture.retry_cleanup()[0]) - # A step that already completed is never run a second time. - self.assertEqual(discarded, [target]) - self.assertEqual(fixture.cleanup_backlog(), {}) - - def test_an_aborted_cleanup_is_deferred_and_retried_in_full(self) -> None: - """An abort proves nothing, so every won hash owes every step again.""" - fixture = CollapseFixture() - target = _hash(1) - candidate = fixture.seed([target], credit_share_on_accept=True)[0] - server = fixture.server - server._ensure_share_writer_service().adopt_pending_share( - candidate.pending_share - ) - fixture.service.candidate_queue.put_nowait(candidate) - scan = fixture.service._collapsed_candidate_floor_holders - fixture.service._collapsed_candidate_floor_holders = ( - lambda abandoned: (_ for _ in ()).throw(RuntimeError("index boom")) - ) - retained, log = fixture.collapse() - self.assertEqual(retained, []) - self.assertEqual(fixture.pending(), set()) - self.assertIn("cleanup aborted rows=1", log) - self.assertEqual(fixture.counts()["cleanup_failed"], 1) - self.assertEqual( - fixture.cleanup_backlog(), - {target: frozenset(BLOCK_CANDIDATE_COLLAPSE_CLEANUP_STEPS)}, - ) - # The abort ran the withdrawal and then stopped: the floor holder - # and every terminal accounting step are still owed. - self.assertIn(id(candidate.pending_share), fixture.floor()) - self.assertIn("terminal-outcome", fixture.cleanup_backlog()[target]) - with server.lock: - # The apply published the terminal fence before cleanup, so the - # durably terminal row is already unofferable even though the - # step that normally publishes it never ran. - self.assertIs( - fixture.service._block_candidate_terminal_outcomes[target], - False, - ) - self.assertEqual(server.block_candidate_abandoned_counts, {}) - fixture.service._collapsed_candidate_floor_holders = scan - self.assertTrue(fixture.retry_cleanup()[0]) - # The retry re-indexes the holders the abort never got to read. - self.assertNotIn(id(candidate.pending_share), fixture.floor()) - with server.lock: - self.assertIs( - fixture.service._block_candidate_terminal_outcomes[target], - False, - ) - self.assertNotIn( - target, - fixture.service._outstanding_block_candidate_hashes, - ) - self.assertEqual( - server.block_candidate_abandoned_counts, - {PRISM_BLOCK_CANDIDATE_COLLAPSE_REASON: 1}, - ) - self.assertEqual(fixture.cleanup_backlog(), {}) - self.assertEqual(fixture.counts()["cleanup_recovered"], 1) - - def test_a_cleanup_retry_never_re_adopts_or_re_offers_the_row(self) -> None: - fixture = CollapseFixture() - target = _hash(1) - fixture.seed([target]) - server = fixture.server - broken = self._breaker(server, "_record_block_candidate_terminal_outcome") - with redirect_stdout(StringIO()): - fixture.collapse() - abandon_calls = len(fixture.ledger.abandon_calls) - rpc_calls = len(fixture.rpc.calls) - broken["on"] = False - self.assertTrue(fixture.retry_cleanup()[0]) - # No durable read or write, no node offer, no queued candidate. - self.assertEqual(fixture.pending(), set()) - self.assertEqual(len(fixture.ledger.abandon_calls), abandon_calls) - self.assertEqual(len(fixture.rpc.calls), rpc_calls) - self.assertTrue(fixture.service.candidate_queue.empty()) - self.assertTrue(fixture.service._block_replay_candidate_queue.empty()) - with server.lock: - self.assertEqual(fixture.service._block_replay_inflight_hashes, set()) - self.assertIsNone(fixture.service.retry_candidate) - self.assertIs( - fixture.service._block_candidate_terminal_outcomes[target], - False, - ) - - # -- bounds, pacing, and accounting ------------------------------------ - - def test_the_backlog_is_bounded_by_the_affected_terminal_hashes(self) -> None: - fixture = CollapseFixture() - hashes = [_hash(1), _hash(2), _hash(3)] - fixture.seed(hashes) - server = fixture.server - original = server._clear_block_candidate_retry_state - broken = {"on": True} - - def retry_state(block_hash: str) -> None: - if broken["on"] and block_hash != _hash(3): - raise RuntimeError("retry state boom") - original(block_hash) - - server._clear_block_candidate_retry_state = retry_state - with redirect_stdout(StringIO()): - fixture.collapse() - # Only the hashes whose cleanup actually failed are retained. - self.assertEqual(set(fixture.cleanup_backlog()), {_hash(1), _hash(2)}) - broken["on"] = False - # One hash per pass, and the lane idles once the backlog drains. - self.assertTrue(fixture.retry_cleanup()[0]) - self.assertEqual(len(fixture.cleanup_backlog()), 1) - self.assertTrue(fixture.retry_cleanup()[0]) - self.assertEqual(fixture.cleanup_backlog(), {}) - self.assertFalse(fixture.retry_cleanup()[0]) - counts = fixture.counts() - self.assertEqual(counts["cleanup_recovered"], 2) - self.assertEqual(counts["cleanup_failed"], 2) - self.assertLessEqual(counts["cleanup_recovered"], counts["cleanup_failed"]) - - def test_a_persistently_failing_retry_backs_off_without_recounting(self) -> None: - fixture = CollapseFixture() - target = _hash(1) - fixture.seed([target]) - fixture.service.retry_initial_seconds = 0.5 - fixture.service.retry_max_seconds = 2.0 - fixture.server._clear_block_candidate_retry_state = ( - lambda block_hash: (_ for _ in ()).throw(RuntimeError("boom")) - ) - with redirect_stdout(StringIO()): - fixture.collapse() - self.assertEqual(fixture.counts()["cleanup_failed"], 1) - for attempt, delay in ((1, 1.0), (2, 2.0), (3, 2.0)): - self.assertTrue(fixture.retry_cleanup(force_due=True)[0]) - self.assertEqual(fixture.counts()["cleanup_retry_failed"], attempt) - record = fixture.service._block_candidate_collapse_cleanup_retries[ - target - ] - self.assertEqual(record.attempts, attempt) - self.assertEqual(record.delay_seconds, delay) - # Attempts never inflate the affected-hash series. - counts = fixture.counts() - self.assertEqual(counts["cleanup_failed"], 1) - self.assertEqual(counts["cleanup_recovered"], 0) - self.assertEqual(set(fixture.cleanup_backlog()), {target}) - # A record parked behind its own backoff is not due. - self.assertFalse(fixture.retry_cleanup()[0]) - - def test_the_backlog_only_ever_holds_fixed_step_labels(self) -> None: - fixture = CollapseFixture() - hashes = [_hash(1), _hash(2)] - fixture.seed(hashes) - fixture.service._collapsed_candidate_floor_holders = ( - lambda abandoned: (_ for _ in ()).throw(RuntimeError("boom")) - ) - with redirect_stdout(StringIO()): - fixture.collapse() - backlog = fixture.cleanup_backlog() - self.assertEqual(set(backlog), set(hashes)) - for steps in backlog.values(): - self.assertLessEqual( - steps, - frozenset(BLOCK_CANDIDATE_COLLAPSE_CLEANUP_STEPS), - ) - # A step name outside the closed set is refused, not stored. - with redirect_stdout(StringIO()): - fixture.service._defer_collapsed_candidate_cleanup( - _hash(9), - ("not-a-cleanup-step",), - ) - self.assertNotIn(_hash(9), fixture.cleanup_backlog()) - self.assertEqual( - set(fixture.counts()), - set(PRISM_BLOCK_CANDIDATE_COLLAPSE_OUTCOMES), - ) - - def test_the_accounting_lane_drives_the_deferred_cleanup(self) -> None: - """The retry must reach a shipped loop, not just a direct call.""" - fixture = CollapseFixture() - target = _hash(1) - fixture.seed([target]) - server = fixture.server - broken = self._breaker(server, "_clear_block_candidate_retry_state") - with redirect_stdout(StringIO()): - fixture.collapse() - self.assertEqual(set(fixture.cleanup_backlog()), {target}) - broken["on"] = False - service = fixture.service - service._ensure_block_accounting_state() - thread = threading.Thread(target=service.block_accounting_loop, daemon=True) - with redirect_stdout(StringIO()): - thread.start() - try: - # Bounded: the lane either drains the backlog quickly or the - # deadline expires and the assertions below report it. - deadline = time.monotonic() + 5.0 - while fixture.cleanup_backlog() and time.monotonic() < deadline: - time.sleep(0.01) - finally: - server.stop_event.set() - thread.join(timeout=5.0) - self.assertFalse(thread.is_alive()) - self.assertEqual(fixture.cleanup_backlog(), {}) - self.assertEqual(fixture.counts()["cleanup_recovered"], 1) - - -class BlockCandidateCollapseCleanupCadenceTests(unittest.TestCase): - """The accounting lane must reach a due cleanup while it is busy. - - A collapsed row whose cleanup failed has no durable replay source, so - the accounting lane is its only driver. Driving it purely from that - lane's idle branch made the retry hostage to the lane going idle: - sustained ordinary accounting traffic, or a continuously replenished - invalid-candidate quarantine queue, starved it for as long as the - traffic lasted. These tests own the explicit work-item cadence that - closes the gap in both directions -- the cleanup reaches its record - within a bounded number of work items, and the cleanup backlog in turn - can spend no more than one attempt per cadence, so it cannot starve - accounting or quarantine in return. - """ - - def _deferred(self, fixture: CollapseFixture, hashes: list[str]) -> dict: - """Collapse ``hashes`` with one cleanup step broken; return the switch.""" - fixture.seed(hashes) - broken = _break_cleanup_step( - fixture.server, - "_clear_block_candidate_retry_state", - ) - with redirect_stdout(StringIO()): - fixture.collapse() - self.assertEqual(set(fixture.cleanup_backlog()), set(hashes)) - return broken - - def test_the_cadence_bound_is_the_shipped_default(self) -> None: - fixture = CollapseFixture() - self.assertEqual( - fixture.service._block_accounting_cleanup_retry_work_items(), - DEFAULT_BLOCK_ACCOUNTING_CLEANUP_RETRY_WORK_ITEMS, - ) - self.assertGreaterEqual(DEFAULT_BLOCK_ACCOUNTING_CLEANUP_RETRY_WORK_ITEMS, 1) - - def test_a_non_positive_cadence_offers_after_every_work_item(self) -> None: - """A misconfigured bound degrades to offering more, never never.""" - fixture = CollapseFixture() - fixture.server.block_accounting_cleanup_retry_work_items = 0 - self.assertEqual( - fixture.service._block_accounting_cleanup_retry_work_items(), - 1, - ) - broken = self._deferred(fixture, [_hash(1)]) - broken["on"] = False - rig = AccountingLaneRig(fixture, AccountingLaneRig.ACCOUNTING) - self.assertTrue(rig.run_until(lambda: not fixture.cleanup_backlog())) - self.assertLessEqual(rig.offers[0][0], 1) - - # -- starvation by ordinary accounting traffic ------------------------- - - def test_a_due_cleanup_runs_while_the_accounting_queues_stay_busy(self) -> None: - """The handoff queues never empty, so only the cadence can run it.""" - fixture = CollapseFixture() - target = _hash(1) - broken = self._deferred(fixture, [target]) - broken["on"] = False - bound = fixture.service._block_accounting_cleanup_retry_work_items() - rig = AccountingLaneRig(fixture, AccountingLaneRig.ACCOUNTING) - self.assertTrue( - rig.run_until(lambda: not fixture.cleanup_backlog()), - f"cleanup starved behind accounting traffic; log={rig.log}", - ) - self.assertFalse(rig.thread_alive) - self.assertEqual(fixture.cleanup_backlog(), {}) - self.assertEqual(fixture.counts()["cleanup_recovered"], 1) - # The lane was never idle: every offer came from the cadence, and - # the first one landed inside the documented bound. - self.assertGreater(rig.accounting_items, 0) - self.assertEqual(rig.quarantine_items, 0) - self.assertLessEqual(rig.offers[0][0], bound) - - # -- starvation by invalid-candidate quarantine traffic ---------------- - - def test_a_due_cleanup_runs_while_quarantine_is_replenished(self) -> None: - """Quarantine always reports work, so the idle branch never runs.""" - fixture = CollapseFixture() - target = _hash(1) - broken = self._deferred(fixture, [target]) - broken["on"] = False - bound = fixture.service._block_accounting_cleanup_retry_work_items() - rig = AccountingLaneRig(fixture, AccountingLaneRig.QUARANTINE) - self.assertTrue( - rig.run_until(lambda: not fixture.cleanup_backlog()), - f"cleanup starved behind quarantine traffic; log={rig.log}", - ) - self.assertFalse(rig.thread_alive) - self.assertEqual(fixture.cleanup_backlog(), {}) - self.assertEqual(fixture.counts()["cleanup_recovered"], 1) - self.assertEqual(rig.accounting_items, 0) - self.assertGreater(rig.quarantine_items, 0) - self.assertLessEqual(rig.offers[0][1], bound) - - # -- the cadence cannot be starved *by* the cleanup -------------------- - - def test_a_cleanup_backlog_spends_one_attempt_per_cadence(self) -> None: - """A permanently failing backlog must not take the lane over.""" - fixture = CollapseFixture() - hashes = [_hash(index) for index in range(1, 6)] - self._deferred(fixture, hashes) - # retry_initial_seconds is 0 in this fixture, so the doubled backoff - # stays 0 and every record in the backlog is due on every offer. - self.assertEqual(fixture.service.retry_initial_seconds, 0.0) - bound = fixture.service._block_accounting_cleanup_retry_work_items() - rig = AccountingLaneRig(fixture, AccountingLaneRig.BOTH) - self.assertTrue(rig.run_until(lambda: rig.work_items >= 20 * bound)) - self.assertFalse(rig.thread_alive) - # One offer per completed cadence, and one attempt per offer at most. - self.assertEqual(len(rig.offers), rig.work_items // bound) - self.assertLessEqual(rig.attempts, rig.work_items // bound) - self.assertEqual(fixture.counts()["cleanup_retry_failed"], rig.attempts) - # Both lanes kept draining throughout; neither was taken over. - self.assertGreaterEqual(rig.accounting_items, 5 * bound) - self.assertGreaterEqual(rig.quarantine_items, 5 * bound) - # Round-robin re-registration keeps the whole backlog owed. - self.assertEqual(set(fixture.cleanup_backlog()), set(hashes)) - self.assertEqual(fixture.counts()["cleanup_recovered"], 0) - - # -- the cadence offers, the record's own deadline decides ------------- - - def test_a_record_behind_its_backoff_is_offered_but_not_run(self) -> None: - fixture = CollapseFixture() - target = _hash(1) - self._deferred(fixture, [target]) - registry = fixture.service._block_candidate_collapse_cleanup_retries - with fixture.server.lock: - registry[target].not_before_monotonic = time.monotonic() + 3600.0 - bound = fixture.service._block_accounting_cleanup_retry_work_items() - rig = AccountingLaneRig(fixture, AccountingLaneRig.ACCOUNTING) - self.assertTrue(rig.run_until(lambda: len(rig.offers) >= 3)) - self.assertFalse(rig.thread_alive) - # Offered on cadence... - self.assertLessEqual(rig.offers[0][0], bound) - # ...and refused by the record's own deadline, not run early. - self.assertEqual(rig.attempts, 0) - self.assertEqual(registry[target].attempts, 0) - self.assertEqual(set(fixture.cleanup_backlog()), {target}) - counts = fixture.counts() - self.assertEqual(counts["cleanup_retry_failed"], 0) - self.assertEqual(counts["cleanup_recovered"], 0) - - # -- the pre-existing idle path is unchanged --------------------------- - - def test_a_truly_idle_lane_still_runs_the_cleanup_immediately(self) -> None: - """The idle branch must not wait for a cadence's worth of items.""" - fixture = CollapseFixture() - target = _hash(1) - broken = self._deferred(fixture, [target]) - broken["on"] = False - rig = AccountingLaneRig(fixture, AccountingLaneRig.IDLE) - self.assertTrue( - rig.run_until(lambda: not fixture.cleanup_backlog()), - f"idle lane did not finish the cleanup; log={rig.log}", - ) - self.assertFalse(rig.thread_alive) - self.assertEqual(fixture.cleanup_backlog(), {}) - self.assertEqual(fixture.counts()["cleanup_recovered"], 1) - # No work item of either kind was needed. - self.assertEqual(rig.accounting_items, 0) - self.assertEqual(rig.quarantine_items, 0) - self.assertEqual(rig.offers[0], (0, 0)) - - # -- regression proof for the cadence itself --------------------------- - - def test_without_the_cadence_both_lanes_starve_the_cleanup(self) -> None: - """Neutralising the cadence must break the two starvation tests. - - Raising the bound out of reach removes every cadence offer, which is - exactly what deleting the cadence calls does to the loop: in both - busy modes the idle branch is unreachable, so nothing is left to - drive the retry. If this test ever stops starving, the two tests - above have stopped proving the cadence. - """ - bound = DEFAULT_BLOCK_ACCOUNTING_CLEANUP_RETRY_WORK_ITEMS - for mode in (AccountingLaneRig.ACCOUNTING, AccountingLaneRig.QUARANTINE): - with self.subTest(mode=mode): - fixture = CollapseFixture() - target = _hash(1) - broken = self._deferred(fixture, [target]) - broken["on"] = False - fixture.server.block_accounting_cleanup_retry_work_items = 1 << 30 - rig = AccountingLaneRig(fixture, mode) - rig.run_until(lambda: rig.work_items >= 50 * bound) - self.assertFalse(rig.thread_alive) - self.assertGreaterEqual(rig.work_items, 50 * bound) - self.assertEqual(rig.offers, []) - self.assertEqual(rig.attempts, 0) - self.assertEqual(set(fixture.cleanup_backlog()), {target}) - self.assertEqual(fixture.counts()["cleanup_recovered"], 0) - - -class BlockCandidateCollapseSubmissionFenceTests(unittest.TestCase): - """A won row is fenced from the node by the write, not by its cleanup. - - The fenced batch write is the moment a row becomes durably terminal, but - the in-memory fence the rest of the submitter honours -- the recorded - terminal outcome -- used to be installed only by the ``terminal-outcome`` - cleanup step. A cleanup that failed at that step, or a page-level abort - that never reached it, therefore left a same-hash candidate in the live, - replay, retry, or waiting lane free to be dequeued the moment the apply - released its disposition lease: ``submit_next`` found no terminal - outcome, consulted nothing else, and offered qbitd a block whose durable - row was already abandoned. - - These tests own the invariant that closes it. The apply publishes the - false terminal outcome for the exact returned hash set while every won - hash's lease is still held and before any cleanup runs, so the fence is - up from the write onwards and no cleanup outcome -- success, contained - per-step failure, page-level abort, or a retry that keeps failing -- can - make a won row offerable again. - """ - - _breaker = staticmethod(BlockCandidateCollapseCleanupRetryTests._breaker) - - # -- harness ----------------------------------------------------------- - - @staticmethod - def _arm_node(fixture: CollapseFixture) -> None: - """Let a node offer succeed, so an escape is loud rather than lost. - - With ``submitblock`` erroring, a fence regression could be mistaken - for an ordinary transport failure; answering it makes the offer land - exactly as it would in production. - """ - fixture.rpc.results["submitblock"] = None - fixture.server.block_submit_rpc_timeout_seconds = 0.5 - - @staticmethod - def _offers(fixture: CollapseFixture) -> int: - return sum(1 for method, _ in fixture.rpc.calls if method == "submitblock") - - @staticmethod - def _queue( - fixture: CollapseFixture, - candidate: Any, - *, - replay: bool = False, - ) -> None: - service = fixture.service - service._ensure_block_replay_state() - if replay: - service._block_replay_candidate_queue.put_nowait(candidate) - else: - service.candidate_queue.put_nowait(candidate) - - @staticmethod - def _submit(fixture: CollapseFixture, *, defer_accounting: bool = True) -> bool: - """Drive the shipped queue-to-node path exactly as the loop does.""" - with redirect_stdout(StringIO()): - return fixture.server.submit_next_block_candidate( - defer_accounting=defer_accounting - ) - - def _seed(self, fixture: CollapseFixture, target: str) -> Any: - """One durable row whose credit-bearing candidate owns a floor holder.""" - candidate = fixture.seed([target], credit_share_on_accept=True)[0] - fixture.server._ensure_share_writer_service().adopt_pending_share( - candidate.pending_share - ) - self._arm_node(fixture) - return candidate - - def _seed_queued( - self, - fixture: CollapseFixture, - target: str, - *, - replay: bool = False, - ) -> Any: - candidate = self._seed(fixture, target) - self._queue(fixture, candidate, replay=replay) - return candidate - - def _assert_unofferable(self, fixture: CollapseFixture, target: str) -> None: - """Nothing was offered, nothing was resurrected, nothing re-adopted.""" - self.assertEqual(self._offers(fixture), 0) - self.assertEqual(fixture.pending(), set()) - self.assertEqual(len(fixture.ledger.abandon_calls), 1) - service = fixture.service - self.assertTrue(service.candidate_queue.empty()) - self.assertTrue(service._block_replay_candidate_queue.empty()) - with fixture.server.lock: - self.assertIsNone(service.retry_candidate) - self.assertNotIn(target, service._block_replay_inflight_hashes) - self.assertNotIn(target, service._block_disposition_waiting_retries) - - # -- the two cleanup outcomes that used to unfence a won row ----------- - - def test_a_failed_terminal_outcome_still_fences_the_queued_candidate( - self, - ) -> None: - """The P1: the step that publishes the fence is the step that failed.""" - fixture = CollapseFixture() - target = _hash(1) - candidate = self._seed_queued(fixture, target) - broken = self._breaker( - fixture.server, - "_record_block_candidate_terminal_outcome", - ) - with redirect_stdout(StringIO()): - fixture.collapse() - # Exactly the failure this fence exists for: the durable row is gone - # and its terminal cleanup is still owed, with the candidate object - # still sitting in the live queue. - self.assertEqual(fixture.pending(), set()) - self.assertEqual( - fixture.cleanup_backlog(), - {target: frozenset({"terminal-outcome"})}, - ) - self.assertFalse(fixture.service.candidate_queue.empty()) - # The submitter reaches the candidate before the accounting lane - # reaches the retry -- the ordering the incident actually produced. - self.assertTrue(self._submit(fixture)) - self._assert_unofferable(fixture, target) - # Only now does the retry finish the step that failed. - broken["on"] = False - self.assertTrue(fixture.retry_cleanup()[0]) - self.assertEqual(fixture.cleanup_backlog(), {}) - self.assertEqual(self._offers(fixture), 0) - self.assertNotIn(id(candidate.pending_share), fixture.floor()) - counts = fixture.counts() - self.assertEqual(counts["cleanup_failed"], 1) - self.assertEqual(counts["cleanup_recovered"], 1) - - def test_an_aborted_cleanup_still_fences_the_queued_candidate(self) -> None: - """A page-level abort never reaches the terminal-outcome step at all.""" - fixture = CollapseFixture() - target = _hash(1) - candidate = self._seed_queued(fixture, target) - scan = fixture.service._collapsed_candidate_floor_holders - fixture.service._collapsed_candidate_floor_holders = ( - lambda abandoned: (_ for _ in ()).throw(RuntimeError("index boom")) - ) - with redirect_stdout(StringIO()): - fixture.collapse() - self.assertEqual( - fixture.cleanup_backlog(), - {target: frozenset(BLOCK_CANDIDATE_COLLAPSE_CLEANUP_STEPS)}, - ) - # The abort ran nothing past the withdrawal, so the queued object - # still owns its identity-keyed floor holder. - self.assertIn(id(candidate.pending_share), fixture.floor()) - self.assertTrue(self._submit(fixture)) - self._assert_unofferable(fixture, target) - # The dropped duplicate released its own holder on the way out, which - # is what lets the retry re-scan the drained queues and lose nothing. - self.assertNotIn(id(candidate.pending_share), fixture.floor()) - fixture.service._collapsed_candidate_floor_holders = scan - self.assertTrue(fixture.retry_cleanup()[0]) - self.assertEqual(fixture.cleanup_backlog(), {}) - self.assertEqual(self._offers(fixture), 0) - - # -- lane ownership ---------------------------------------------------- - - def test_the_replay_lane_is_fenced_by_the_same_published_outcome(self) -> None: - """Durable replay dequeues through the same guard as a live solve.""" - fixture = CollapseFixture() - target = _hash(1) - self._seed_queued(fixture, target, replay=True) - service = fixture.service - with fixture.server.lock: - service._block_replay_inflight_hashes.add(target) - broken = self._breaker( - fixture.server, - "_record_block_candidate_terminal_outcome", - ) - with redirect_stdout(StringIO()): - fixture.collapse() - self.assertFalse(service._block_replay_candidate_queue.empty()) - self.assertTrue(self._submit(fixture)) - self.assertEqual(self._offers(fixture), 0) - self.assertTrue(service._block_replay_candidate_queue.empty()) - self.assertEqual(fixture.pending(), set()) - # A freshly decoded same-hash replay object is refused admission by - # the same published outcome rather than queued behind it. - sibling = fixture.seed([target], credit_share_on_accept=True)[0] - with redirect_stdout(StringIO()): - self.assertFalse(service._enqueue_replayed_block_candidate(sibling)) - self.assertTrue(service._block_replay_candidate_queue.empty()) - self.assertEqual(self._offers(fixture), 0) - broken["on"] = False - self.assertTrue(fixture.retry_cleanup()[0]) - self.assertEqual(fixture.cleanup_backlog(), {}) - - def test_the_waiting_retry_lane_is_fenced_when_the_lease_releases(self) -> None: - """A wakeup parked behind the apply's own lease must not be offered.""" - fixture = CollapseFixture() - target = _hash(1) - candidate = self._seed(fixture, target) - service = fixture.service - inner = fixture.ledger.inner.mark_block_candidates_abandoned - - def park_then_write(requested: tuple[str, ...], error: str) -> Any: - # A parked wakeup is collapse evidence, so the only ordering that - # produces one for a won hash is this: the submitter dequeued the - # hash, could not claim the apply's lease, and parked it after the - # pre-write revalidation re-read the evidence and before the - # fenced write landed. - with fixture.server.lock: - service._block_disposition_waiting_retries[target] = candidate - return inner(block_hashes=requested, error=error) - - fixture.ledger.abandon_hook = park_then_write - broken = self._breaker( - fixture.server, - "_record_block_candidate_terminal_outcome", - ) - with redirect_stdout(StringIO()): - fixture.collapse() - # The failed step is also the one that drops parked wakeups, so the - # candidate is still parked and will be dequeued from there. - with fixture.server.lock: - self.assertIn(target, service._block_disposition_waiting_retries) - self.assertTrue(self._submit(fixture)) - self._assert_unofferable(fixture, target) - broken["on"] = False - self.assertTrue(fixture.retry_cleanup()[0]) - self.assertEqual(fixture.cleanup_backlog(), {}) - - # -- the non-queue seams ------------------------------------------------ - - def test_the_direct_writer_seam_refuses_a_fenced_hash(self) -> None: - """The historical direct entrypoint takes the guard and answers False.""" - fixture = CollapseFixture() - target = _hash(1) - candidate = self._seed_queued(fixture, target) - fixture.service.candidate_queue.get_nowait() - broken = self._breaker( - fixture.server, - "_record_block_candidate_terminal_outcome", - ) - with redirect_stdout(StringIO()): - fixture.collapse() - with redirect_stdout(StringIO()): - landed = fixture.server._submit_next_block_candidate_writer(candidate) - self.assertFalse(landed) - self.assertEqual(self._offers(fixture), 0) - self.assertEqual(fixture.pending(), set()) - self.assertEqual(len(fixture.ledger.abandon_calls), 1) - broken["on"] = False - self.assertTrue(fixture.retry_cleanup()[0]) - - def test_the_synchronous_seam_refuses_a_fenced_hash(self) -> None: - """The miner-facing resubmit joins the same terminal disposition.""" - fixture = CollapseFixture() - target = _hash(1) - candidate = self._seed_queued(fixture, target) - fixture.service.candidate_queue.get_nowait() - broken = self._breaker( - fixture.server, - "_record_block_candidate_terminal_outcome", - ) - with redirect_stdout(StringIO()): - fixture.collapse() - with redirect_stdout(StringIO()): - landed = fixture.server._submit_synchronous_block_candidate(candidate) - self.assertFalse(landed) - self.assertEqual(self._offers(fixture), 0) - self.assertEqual(fixture.pending(), set()) - self.assertNotIn(id(candidate.pending_share), fixture.floor()) - broken["on"] = False - self.assertTrue(fixture.retry_cleanup()[0]) - - # -- durability of the fence across the retry series ------------------- - - def test_the_fence_survives_every_retry_failure_and_its_discharge(self) -> None: - """Repeated failures, then recovery, and never one node offer.""" - fixture = CollapseFixture() - target = _hash(1) - candidate = self._seed(fixture, target) - fixture.service.retry_initial_seconds = 0.5 - fixture.service.retry_max_seconds = 2.0 - broken = self._breaker( - fixture.server, - "_record_block_candidate_terminal_outcome", - ) - with redirect_stdout(StringIO()): - fixture.collapse() - for attempt in (1, 2, 3): - self.assertTrue(fixture.retry_cleanup(force_due=True)[0]) - self.assertEqual(fixture.counts()["cleanup_retry_failed"], attempt) - self.assertEqual(set(fixture.cleanup_backlog()), {target}) - # The fence is unaffected by the attempt that just failed: the - # same-hash wakeup is dropped, not offered, on every round. - self._queue(fixture, candidate) - self.assertTrue(self._submit(fixture)) - self._assert_unofferable(fixture, target) - broken["on"] = False - self.assertTrue(fixture.retry_cleanup(force_due=True)[0]) - self.assertEqual(fixture.cleanup_backlog(), {}) - # Discharging the backlog leaves the ordinary terminal fence behind, - # so the durably terminal row is still never re-offered. - self._queue(fixture, candidate) - self.assertTrue(self._submit(fixture)) - self._assert_unofferable(fixture, target) - with fixture.server.lock: - self.assertIs( - fixture.service._block_candidate_terminal_outcomes[target], - False, - ) - # Accounting stayed bounded by the affected hashes, and the label - # space is still the closed outcome set. - counts = fixture.counts() - self.assertEqual(counts["cleanup_failed"], 1) - self.assertEqual(counts["cleanup_recovered"], 1) - self.assertEqual(counts["cleanup_retry_failed"], 3) - self.assertLessEqual(counts["cleanup_recovered"], counts["cleanup_failed"]) - self.assertEqual( - set(counts), - set(PRISM_BLOCK_CANDIDATE_COLLAPSE_OUTCOMES), - ) - - def test_a_clean_collapse_fences_its_queued_candidate_too(self) -> None: - """The fence is a property of the write, not of a cleanup failure.""" - fixture = CollapseFixture() - target = _hash(1) - self._seed_queued(fixture, target) - with redirect_stdout(StringIO()): - fixture.collapse() - self.assertEqual(fixture.cleanup_backlog(), {}) - self.assertEqual(fixture.counts()["cleanup_failed"], 0) - self.assertTrue(self._submit(fixture)) - self._assert_unofferable(fixture, target) - - def test_a_preserved_row_is_never_fenced_by_another_pages_win(self) -> None: - """Only the exact returned hash set is published, never the request.""" - fixture = CollapseFixture() - won, lost = _hash(1), _hash(2) - fixture.seed([won, lost], credit_share_on_accept=True) - fixture.ledger.abandon_hook = lambda requested, error: (won,) - with redirect_stdout(StringIO()): - fixture.collapse() - service = fixture.service - with fixture.server.lock: - outcomes = dict(service._block_candidate_terminal_outcomes) - self.assertEqual(outcomes, {won: False}) - self.assertEqual(fixture.counts()["write_lost"], 1) - - -class _CountingRegistry(dict): - """A registry that counts membership probes and reports being walked. - - The page-bounded evidence read has to answer from probes alone; a - regression that goes back to copying a registry into a set shows up here - as a walk (optionally a hard failure) rather than as a wall-clock wobble - that a fast enough machine hides. - """ - - def __init__(self, *args: Any, **kwargs: Any) -> None: - super().__init__(*args, **kwargs) - self.probes = 0 - self.walks = 0 - self.forbid_walks = False - - def __contains__(self, key: object) -> bool: - self.probes += 1 - return super().__contains__(key) - - def __iter__(self) -> Any: - self.walks += 1 - if self.forbid_walks: - raise AssertionError("terminal outcome registry was walked") - return super().__iter__() - - -def _history_hashes(count: int, *, start: int = 1 << 20) -> list[str]: - """``count`` distinct hashes standing in for already-disposed blocks.""" - return [f"{index:064x}" for index in range(start, start + count)] - - -class BlockCandidateCollapseEvidenceCostTests(unittest.TestCase): - """One evidence probe per hash the caller is deciding, and no more. - - ``_block_candidate_terminal_outcomes`` is the one registry the collapse - evidence read touches that grows with every block the process has ever - disposed: the 2026-08-20 storm left 312,000 entries behind, and every - poll folded all of them into a set while holding ``coordinator.lock`` - -- about 124 ms per page, linear in history, with every unrelated share - ack queued behind the same lock. The selector never needed the union: - it asks one question, "is this page's hash already offered", for at most - a page of hashes at a time. - - These tests pin that shape with a registry that counts its probes and - refuses to be walked, so the cost claim is a deterministic assertion - about work performed rather than a timing measurement. - """ - - HISTORY = 200_000 - - @staticmethod - def _install_history( - fixture: CollapseFixture, - size: int, - *, - terminal: Iterable[str] = (), - ) -> _CountingRegistry: - registry = _CountingRegistry( - (block_hash, False) for block_hash in _history_hashes(size) - ) - for block_hash in terminal: - registry[block_hash] = False - with fixture.server.lock: - fixture.service._block_candidate_terminal_outcomes = registry - registry.probes = 0 - registry.walks = 0 - return registry - - def test_evidence_answers_a_page_without_walking_the_history(self) -> None: - fixture = CollapseFixture() - page = [_hash(1), _hash(2)] - fixture.seed(page) - registry = self._install_history( - fixture, - self.HISTORY, - terminal=[_hash(2)], - ) - registry.forbid_walks = True - evidence = fixture.service._block_candidate_collapse_evidence(page) - # Semantics unchanged: the historical terminal outcome still speaks - # for its hash, it is simply asked about rather than enumerated. - self.assertEqual(evidence, frozenset({_hash(2)})) - self.assertEqual(registry.walks, 0) - self.assertEqual(registry.probes, len(page)) - - def test_selection_work_does_not_grow_with_the_history(self) -> None: - """The same page costs the same probes at 1,000 and at 200,000.""" - page = [_hash(index) for index in range(1, 6)] - probes: dict[int, int] = {} - for size in (1_000, self.HISTORY): - with self.subTest(history=size): - fixture = CollapseFixture() - fixture.seed(page) - rows = fixture.page() - registry = self._install_history(fixture, size) - registry.forbid_walks = True - chain = _BlockCandidateChainView(fixture.service) - selected = fixture.service._select_superseded_block_candidates( - rows, - chain, - ) - self.assertEqual( - {row.block_hash for row in selected}, - set(page), - ) - self.assertEqual(registry.walks, 0) - probes[size] = registry.probes - self.assertEqual(probes[1_000], probes[self.HISTORY]) - self.assertEqual(probes[self.HISTORY], len(page)) - - def test_revalidation_probes_only_the_leased_subset(self) -> None: - """The pre-write re-read is bounded by the leases, not by the page.""" - fixture = CollapseFixture() - page = [_hash(index) for index in range(1, 6)] - fixture.seed(page) - chain = _BlockCandidateChainView(fixture.service) - selected = fixture.service._select_superseded_block_candidates( - fixture.page(), - chain, - ) - leased = selected[:2] - registry = self._install_history(fixture, 1_000) - registry.forbid_walks = True - qualified, _tip = fixture.service._revalidate_superseded_block_candidates( - leased, - chain, - ) - self.assertEqual( - {row.block_hash for row in qualified}, - {row.block_hash for row in leased}, - ) - self.assertEqual(registry.walks, 0) - self.assertEqual(registry.probes, len(leased)) - - def test_a_collapse_over_a_large_history_bounds_it(self) -> None: - """The apply still wins its page, and leaves the registry bounded.""" - fixture = CollapseFixture() - page = [_hash(index) for index in range(1, 6)] - fixture.seed(page) - with fixture.server.lock: - fixture.service._block_candidate_terminal_outcomes = { - block_hash: False - for block_hash in _history_hashes(self.HISTORY) - } - self.assertEqual(_selected(fixture), set(page)) - outcomes = fixture.service._block_candidate_terminal_outcomes - self.assertEqual(len(outcomes), MAX_BLOCK_CANDIDATE_TERMINAL_OUTCOMES) - for block_hash in page: - self.assertIs(outcomes[block_hash], False) - - -class BlockCandidateTerminalOutcomeBoundTests(unittest.TestCase): - """The terminal-outcome registry is bounded, and never unfences a copy. - - The same-hash disposition guard used to remember every outcome for the - life of the process. That is what made the collapse evidence read - expensive, but it is also unbounded memory in its own right, so the - registry is trimmed oldest-first once it passes - ``MAX_BLOCK_CANDIDATE_TERMINAL_OUTCOMES``. - - Eviction is the dangerous half. A forgotten outcome reads exactly like - an outcome that never happened, so a same-hash candidate still sitting - in the live, replay, retry, parked, or quarantine lane would find no - fence and offer qbitd a block whose durable row is already terminal -- - the very escape ``_publish_collapsed_candidate_terminal_fence`` exists - to close. Every lane that can still be holding such a copy therefore - pins its hash, and these tests own that pin, lane by lane. - """ - - CAP = MAX_BLOCK_CANDIDATE_TERMINAL_OUTCOMES - - # -- harness ----------------------------------------------------------- - - @staticmethod - def _fill(fixture: CollapseFixture, count: int) -> None: - """Accumulate historical outcomes behind whatever is already there.""" - with fixture.server.lock: - outcomes = fixture.service._block_candidate_terminal_outcomes - for block_hash in _history_hashes(count): - outcomes[block_hash] = False - - @staticmethod - def _record(fixture: CollapseFixture, count: int) -> None: - """Drive ``count`` real terminal outcomes, each one a trim occasion.""" - for block_hash in _history_hashes(count, start=1 << 40): - fixture.server._record_block_candidate_terminal_outcome( - block_hash, - accepted=False, - ) - - def _pressure(self, fixture: CollapseFixture) -> None: - """Push the registry past its bound through the shipped writer.""" - self._fill(fixture, self.CAP) - self._record(fixture, 64) - - # -- the bound --------------------------------------------------------- - - def test_the_registry_stops_growing_at_its_bound(self) -> None: - fixture = CollapseFixture() - self._record(fixture, self.CAP + 512) - outcomes = fixture.service._block_candidate_terminal_outcomes - self.assertEqual(len(outcomes), self.CAP) - history = _history_hashes(self.CAP + 512, start=1 << 40) - # Oldest-first: the first outcomes are gone, the newest are kept. - self.assertIsNone( - fixture.server._block_candidate_terminal_outcome(history[0]) - ) - self.assertIs( - fixture.server._block_candidate_terminal_outcome(history[-1]), - False, - ) - - def test_a_re_recorded_outcome_is_treated_as_the_newest(self) -> None: - """Reasserting an outcome must not leave it at the eviction front.""" - fixture = CollapseFixture() - target = _hash(1) - fixture.server._record_block_candidate_terminal_outcome( - target, - accepted=False, - ) - self._fill(fixture, 8) - fixture.server._record_block_candidate_terminal_outcome( - target, - accepted=False, - ) - outcomes = fixture.service._block_candidate_terminal_outcomes - self.assertEqual(list(outcomes)[-1], target) - - def test_pinned_entries_do_not_stall_the_trim(self) -> None: - """A run of pinned hashes longer than the scan window still drains.""" - fixture = CollapseFixture() - pinned = _history_hashes( - BLOCK_CANDIDATE_TERMINAL_OUTCOME_EVICTION_SCAN * 2, - start=1 << 30, - ) - for block_hash in pinned: - fixture.server._record_block_candidate_terminal_outcome( - block_hash, - accepted=False, - ) - with fixture.server.lock: - fixture.service._block_replay_inflight_hashes.update(pinned) - self._pressure(fixture) - outcomes = fixture.service._block_candidate_terminal_outcomes - self.assertEqual(len(outcomes), self.CAP) - for block_hash in pinned: - self.assertIn(block_hash, outcomes) - - def test_a_fence_page_larger_than_the_bound_keeps_its_publication( - self, - ) -> None: - """A storm-sized fence never evicts the outcomes it just published.""" - fixture = CollapseFixture() - self._fill(fixture, self.CAP) - page = tuple(_history_hashes(self.CAP + 100, start=1 << 50)) - with redirect_stdout(StringIO()): - fixture.service._publish_collapsed_candidate_terminal_fence(page) - outcomes = fixture.service._block_candidate_terminal_outcomes - for block_hash in page: - self.assertIs(outcomes[block_hash], False) - # Only history was dropped; the registry stays over its bound - # rather than unfencing a row the apply is still cleaning up. - self.assertEqual(len(outcomes), len(page)) - - def test_the_trim_is_skipped_rather_than_waiting_on_a_lane(self) -> None: - """Nothing is evicted against a lane that could not be read.""" - fixture = CollapseFixture() - self._fill(fixture, self.CAP + 16) - service = fixture.service - with service.candidate_queue.mutex: - with fixture.server.lock: - self.assertEqual( - service._bound_block_candidate_terminal_outcomes(), - 0, - ) - self.assertEqual( - len(service._block_candidate_terminal_outcomes), - self.CAP + 16, - ) - with fixture.server.lock: - self.assertEqual( - service._bound_block_candidate_terminal_outcomes(), - 16, - ) - - # -- the pins ---------------------------------------------------------- - - def test_every_lane_that_holds_a_copy_pins_its_outcome(self) -> None: - """Each lane the fence protects keeps its hash out of the eviction.""" - target = _hash(1) - - def outstanding(fixture: CollapseFixture, candidate: Any) -> None: - fixture.server._register_outstanding_block_candidate(target) - - def replay_inflight(fixture: CollapseFixture, candidate: Any) -> None: - with fixture.server.lock: - fixture.service._block_replay_inflight_hashes.add(target) - - def quarantined(fixture: CollapseFixture, candidate: Any) -> None: - with fixture.server.lock: - fixture.service._block_quarantine_hashes.add(target) - - def fast_lane(fixture: CollapseFixture, candidate: Any) -> None: - with fixture.server.lock: - fixture.service._block_fast_lane_reservations.add(target) - - def waiting_retry(fixture: CollapseFixture, candidate: Any) -> None: - with fixture.server.lock: - fixture.service._block_disposition_waiting_retries[target] = ( - candidate - ) - - def finalize_retry(fixture: CollapseFixture, candidate: Any) -> None: - with fixture.server.lock: - fixture.service.finalize_retries[target] = (False, "boom") - - def retained_submission(fixture: CollapseFixture, candidate: Any) -> None: - fixture.service._stash_retained_block_candidate_node_submission( - target, - _BlockCandidateNodeSubmission(attempted=True, result=None), - ) - - def cleanup_retry(fixture: CollapseFixture, candidate: Any) -> None: - fixture.service._defer_collapsed_candidate_cleanup( - target, - frozenset({"terminal-outcome"}), - ) - - def retry_holder(fixture: CollapseFixture, candidate: Any) -> None: - with fixture.server.lock: - fixture.service.retry_candidate = candidate - - def deferred_retry_holder( - fixture: CollapseFixture, - candidate: Any, - ) -> None: - with fixture.server.lock: - fixture.service._block_accounting_deferred_retry_candidate = ( - candidate - ) - - def live_queue(fixture: CollapseFixture, candidate: Any) -> None: - fixture.service.candidate_queue.put_nowait(candidate) - - def replay_queue(fixture: CollapseFixture, candidate: Any) -> None: - fixture.service._block_replay_candidate_queue.put_nowait(candidate) - - def disposition_flight(fixture: CollapseFixture, candidate: Any) -> None: - lease = fixture.server._claim_block_candidate_disposition( - target, - blocking=False, - ) - self.assertIsNotNone(lease) - - def dequeued_pin(fixture: CollapseFixture, candidate: Any) -> None: - # The instant between leaving a lane and owning a flight. - with fixture.server.lock: - fixture.service._pin_dequeued_block_candidate_locked(candidate) - - lanes = { - "_outstanding_block_candidate_hashes": outstanding, - "_block_replay_inflight_hashes": replay_inflight, - "_block_quarantine_hashes": quarantined, - "_block_fast_lane_reservations": fast_lane, - "_block_disposition_waiting_retries": waiting_retry, - "finalize_retries": finalize_retry, - "_block_candidate_retained_node_submissions": retained_submission, - "_block_candidate_collapse_cleanup_retries": cleanup_retry, - "retry_candidate": retry_holder, - "_block_accounting_deferred_retry_candidate": deferred_retry_holder, - "candidate_queue": live_queue, - "_block_replay_candidate_queue": replay_queue, - "_block_candidate_disposition_flights": disposition_flight, - "_block_candidate_dequeued_hashes": dequeued_pin, - } - for name, install in lanes.items(): - with self.subTest(lane=name): - fixture = CollapseFixture() - candidate = fixture.seed([target])[0] - # The outcome is recorded first, exactly as a terminal - # disposition does, and the lane is populated afterwards: - # recording clears the reservation, replay, and waiting - # markers for its own hash by design. - fixture.server._record_block_candidate_terminal_outcome( - target, - accepted=False, - ) - with redirect_stdout(StringIO()): - install(fixture, candidate) - self._pressure(fixture) - outcomes = fixture.service._block_candidate_terminal_outcomes - self.assertEqual(len(outcomes), self.CAP) - self.assertIs( - fixture.server._block_candidate_terminal_outcome(target), - False, - ) - - def test_a_queued_duplicate_keeps_its_fence_under_eviction_pressure( - self, - ) -> None: - """The escape the fence closes must survive the registry's bound. - - A won row's cleanup clears the ownership markers -- outstanding, - replay-inflight -- for the hash it just terminalized, so a same-hash - candidate still queued behind the apply is named by no registry at - all. Its fence would be the oldest evictable entry in the registry - and the dequeue would then find nothing. The queue itself is read, - so it does not. - """ - fixture = CollapseFixture() - queued, forgotten = _hash(1), _hash(2) - candidate = fixture.seed([queued], credit_share_on_accept=True)[0] - fixture.seed([forgotten]) - fixture.server._ensure_share_writer_service().adopt_pending_share( - candidate.pending_share - ) - # A working node offer, so an escape lands as a real submitblock - # rather than being lost in a transport error. - fixture.rpc.results["submitblock"] = None - fixture.server.block_submit_rpc_timeout_seconds = 0.5 - fixture.service.candidate_queue.put_nowait(candidate) - with redirect_stdout(StringIO()): - fixture.collapse() - self.assertEqual(fixture.pending(), set()) - self.assertEqual(fixture.cleanup_backlog(), {}) - with fixture.server.lock: - service = fixture.service - self.assertNotIn( - queued, - service._outstanding_block_candidate_hashes, - ) - self.assertNotIn(queued, service._block_replay_inflight_hashes) - self._pressure(fixture) - # The unheld hash ages out; the queued copy's fence does not. - self.assertIsNone( - fixture.server._block_candidate_terminal_outcome(forgotten) - ) - self.assertIs( - fixture.server._block_candidate_terminal_outcome(queued), - False, - ) - with redirect_stdout(StringIO()): - self.assertTrue( - fixture.server.submit_next_block_candidate(defer_accounting=True) - ) - offers = [ - method for method, _ in fixture.rpc.calls if method == "submitblock" - ] - self.assertEqual(offers, []) - self.assertTrue(fixture.service.candidate_queue.empty()) - - # -- the dequeue-to-disposition pin ------------------------------------ - - WAIT_SECONDS = 10.0 - - def _park_claim( - self, - fixture: CollapseFixture, - target: str, - ) -> tuple[threading.Event, threading.Event]: - """Park the submitter inside the old dequeue-to-disposition gap. - - Wraps the disposition claim so the pass that dequeued ``target`` is - held after it has left its lane and before any flight exists for - it -- exactly where ``submit_next``'s local variable used to be the - only thing holding the candidate. The park is deterministic: only - the test releases it, after it has applied eviction pressure and - looked at the registry. - """ - service = fixture.service - original = service._claim_block_candidate_disposition - in_gap = threading.Event() - release = threading.Event() - self.addCleanup(release.set) - - def park_then_claim(block_hash: str, *, blocking: bool) -> Any: - if block_hash == target: - in_gap.set() - if not release.wait(self.WAIT_SECONDS): - raise AssertionError("the parked claim was never released") - return original(block_hash, blocking=blocking) - - service._claim_block_candidate_disposition = park_then_claim - return in_gap, release - - @staticmethod - def _submit_in_background( - fixture: CollapseFixture, - ) -> tuple[threading.Thread, list[Any]]: - """Run one submitter pass on its own thread; collect its result.""" - results: list[Any] = [] - - def run() -> None: - try: - results.append( - fixture.server.submit_next_block_candidate( - defer_accounting=True - ) - ) - except BaseException as error: # reported to the test, not raised here - results.append(error) - - thread = threading.Thread(target=run, daemon=True) - thread.start() - return thread, results - - @staticmethod - def _dequeue_lanes_empty(fixture: CollapseFixture, target: str) -> bool: - """Whether no dequeue source still holds a copy of ``target``.""" - service = fixture.service - with fixture.server.lock: - return ( - service.retry_candidate is None - and service.candidate_queue.empty() - and service._block_replay_candidate_queue.empty() - and target not in service._block_disposition_waiting_retries - ) - - def test_a_dequeued_duplicate_keeps_its_fence_across_the_claim_gap( - self, - ) -> None: - """Eviction inside the dequeue-to-disposition gap finds the hash pinned. - - Every lane pins its hash while a candidate sits in it, and the - disposition flight pins it from the claim onward. Between the two - the candidate used to be held only by ``submit_next``'s local - variable: the retry holder cleared, the queue entry consumed, or - the waiting entry popped, and no flight installed yet. An eviction - in that window dropped the oldest unpinned outcome -- this - duplicate's fence -- and the pass then found nothing and offered a - durably terminal block to the node. - - The dequeue now moves the hash into the dequeued-hash pin under the - same lock hold that empties the lane, and hands it to the flight - (or back to the waiting registry) under the lock again, so there is - no instant at which nothing names it. The barrier parks the pass - in the old gap, applies real eviction pressure from the other side, - and proves the fence survives -- for each of the four sources a - pass can dequeue from. - """ - target = _hash(1) - - def retry_holder(fixture: CollapseFixture, candidate: Any) -> None: - with fixture.server.lock: - fixture.service.retry_candidate = candidate - - def live_queue(fixture: CollapseFixture, candidate: Any) -> None: - fixture.service.candidate_queue.put_nowait(candidate) - - def replay_queue(fixture: CollapseFixture, candidate: Any) -> None: - fixture.service._block_replay_candidate_queue.put_nowait(candidate) - - def waiting_retry(fixture: CollapseFixture, candidate: Any) -> None: - with fixture.server.lock: - fixture.service._block_disposition_waiting_retries[target] = ( - candidate - ) - - sources = { - "retry_candidate": retry_holder, - "candidate_queue": live_queue, - "_block_replay_candidate_queue": replay_queue, - "_block_disposition_waiting_retries": waiting_retry, - } - for name, install in sources.items(): - with self.subTest(source=name): - fixture = CollapseFixture() - service = fixture.service - candidate = fixture.seed([target])[0] - # A working node offer, so an escape would land as a real - # submitblock rather than being lost in a transport error. - fixture.rpc.results["submitblock"] = None - fixture.server.block_submit_rpc_timeout_seconds = 0.5 - # Recorded first, exactly as a terminal disposition does; - # the duplicate arrives in its lane afterwards. - fixture.server._record_block_candidate_terminal_outcome( - target, - accepted=False, - ) - install(fixture, candidate) - # The duplicate's fence is now the oldest entry and the - # registry sits one over its bound: the next recorded - # outcome evicts exactly the oldest unpinned hash. - self._fill(fixture, self.CAP) - in_gap, release = self._park_claim(fixture, target) - with redirect_stdout(StringIO()): - thread, results = self._submit_in_background(fixture) - self.assertTrue(in_gap.wait(self.WAIT_SECONDS)) - # This is the old gap: the lane is empty, no flight - # exists, and the candidate object is the submitter's - # local variable. - self.assertTrue(self._dequeue_lanes_empty(fixture, target)) - with service._block_candidate_disposition_registry_lock: - self.assertNotIn( - target, - service._block_candidate_disposition_flights, - ) - self._record(fixture, 64) - # The fence survived the pressure; history did not. - self.assertIs( - fixture.server._block_candidate_terminal_outcome(target), - False, - ) - self.assertEqual( - len(service._block_candidate_terminal_outcomes), - self.CAP, - ) - # ...because the dequeue itself is what names the hash. - with fixture.server.lock: - self.assertEqual( - service._block_candidate_dequeued_hashes, - {target: 1}, - ) - release.set() - thread.join(self.WAIT_SECONDS) - self.assertFalse(thread.is_alive()) - self.assertEqual(results, [True]) - offers = [ - method - for method, _ in fixture.rpc.calls - if method == "submitblock" - ] - self.assertEqual(offers, []) - # The duplicate was dropped against its fence: nothing holds - # it, no pin outlived the pass, and the fence still stands. - self.assertTrue(self._dequeue_lanes_empty(fixture, target)) - with fixture.server.lock: - self.assertEqual(service._block_candidate_dequeued_hashes, {}) - with service._block_candidate_disposition_registry_lock: - self.assertEqual(service._block_candidate_disposition_flights, {}) - self.assertIs( - fixture.server._block_candidate_terminal_outcome(target), - False, - ) - - def test_the_dequeued_pin_never_outlives_the_claim(self) -> None: - """Every way the claim comes back hands the hash on and unpins it. - - A pin that leaked would be a hash the eviction could never reclaim - -- unbounded growth by another name -- and a pin dropped early is - the window the pin exists to close. A lease hands the hash to the - installed flight (covered above); a miss parks the candidate in the - waiting registry in the same critical section; an exception leaves - nothing behind because the candidate object leaves with it; and a - parked retry that is not yet due is never dequeued, so it is never - pinned. - """ - target = _hash(1) - - with self.subTest(outcome="lease-miss"): - fixture = CollapseFixture() - service = fixture.service - candidate = fixture.seed([target])[0] - # Another same-hash pass spans the offer-to-finalize region. - lease = fixture.server._claim_block_candidate_disposition( - target, - blocking=False, - ) - self.assertIsNotNone(lease) - self.addCleanup( - fixture.server._release_block_candidate_disposition, - lease, - ) - service.candidate_queue.put_nowait(candidate) - with redirect_stdout(StringIO()): - self.assertTrue( - fixture.server.submit_next_block_candidate( - defer_accounting=True - ) - ) - with fixture.server.lock: - self.assertIs( - service._block_disposition_waiting_retries.get(target), - candidate, - ) - self.assertEqual(service._block_candidate_dequeued_hashes, {}) - self.assertTrue(service.candidate_queue.empty()) - - with self.subTest(outcome="claim-raises"): - fixture = CollapseFixture() - service = fixture.service - candidate = fixture.seed([target])[0] - - def explode(block_hash: str, *, blocking: bool) -> Any: - raise RuntimeError("claim exploded") - - service._claim_block_candidate_disposition = explode - service.candidate_queue.put_nowait(candidate) - with redirect_stdout(StringIO()): - with self.assertRaises(RuntimeError): - fixture.server.submit_next_block_candidate( - defer_accounting=True - ) - with fixture.server.lock: - self.assertEqual(service._block_candidate_dequeued_hashes, {}) - with service._block_candidate_disposition_registry_lock: - self.assertEqual(service._block_candidate_disposition_flights, {}) - self.assertTrue(self._dequeue_lanes_empty(fixture, target)) - - with self.subTest(outcome="retry-not-due"): - fixture = CollapseFixture() - service = fixture.service - candidate = fixture.seed([target])[0] - with fixture.server.lock: - service.retry_candidate = candidate - service._block_candidate_retry_not_before = { - target: time.monotonic() + 60.0 - } - with redirect_stdout(StringIO()): - self.assertFalse( - fixture.server.submit_next_block_candidate( - defer_accounting=True - ) - ) - with fixture.server.lock: - self.assertIs(service.retry_candidate, candidate) - self.assertEqual(service._block_candidate_dequeued_hashes, {}) - - # -- the accounting lane's exception handoff --------------------------- - - @staticmethod - def _accounting_task( - fixture: CollapseFixture, - target: str, - candidate: Any, - ) -> Any: - """One queued accounting task owning ``target``'s disposition lease. - - Built exactly as the submitter's deferred handoff leaves it: the - node offer is done and the lease travels with the task until the - accounting lane releases it. - """ - server = fixture.server - lease = server._claim_block_candidate_disposition(target, blocking=False) - assert lease is not None - task = _BlockCandidateAccountingTask( - candidate=candidate, - node_submission=_BlockCandidateNodeSubmission( - attempted=True, - result=None, - ), - disposition_lease=lease, - ) - assert server._enqueue_block_accounting_task(task) - return task - - def _drain_accounting_lane( - self, - fixture: CollapseFixture, - *, - before_stop: Any = None, - ) -> None: - """Run the shipped accounting loop until its queues are drained. - - Bounded: the lane either finishes the queued task quickly or the - deadline expires and the assertions that follow report it. All three - handoff lanes are watched, because a task built with definitive node - acceptance is routed to the accepted lane rather than the primary - queue and a wait that only watched the primary would stop the loop - before that task ever ran. - """ - service = fixture.service - server = fixture.server - service._ensure_block_accounting_state() - - def unfinished() -> int: - return ( - service._block_accounting_accepted_queue.unfinished_tasks - + service._block_accounting_queue.unfinished_tasks - + service._block_accounting_overflow_queue.unfinished_tasks - ) - - thread = threading.Thread(target=service.block_accounting_loop, daemon=True) - with redirect_stdout(StringIO()): - thread.start() - try: - if before_stop is not None: - before_stop() - deadline = time.monotonic() + self.WAIT_SECONDS - while unfinished() and time.monotonic() < deadline: - time.sleep(0.01) - finally: - server.stop_event.set() - thread.join(self.WAIT_SECONDS) - self.assertFalse(thread.is_alive()) - self.assertEqual(unfinished(), 0) - - def test_an_accounting_failure_hands_its_candidate_off_without_a_gap( - self, - ) -> None: - """The lane's unexpected-failure retry leaves no unnamed instant. - - ``_run_block_accounting_task`` owns the task's disposition lease and - releases it in its ``finally``; ``block_accounting_loop`` used to - retain the candidate only afterwards, in its own catch. Between - the release and that retain the candidate was the loop's local - variable and nothing named its hash -- the dequeue gap above in its - accounting-lane form. The lane now retains while it still holds - the disposition, which routes the candidate into the accounting - owner's deferred holder; the ``finally`` then releases the lease - and merges that holder into the retry slot under - ``coordinator.lock``, exactly as it always did for a writer-side - retain. The loop's catch sees the task already retained and does - not retain it again. - - The barrier parks the lane the instant its lease release returns, - applies eviction pressure, and proves the fence survives; afterwards - the candidate sits in the retry slot exactly once, with one retry - counted and its floor holder intact. - """ - target = _hash(1) - fixture = CollapseFixture() - service, server = fixture.service, fixture.server - candidate = fixture.seed([target], credit_share_on_accept=True)[0] - server._ensure_share_writer_service().adopt_pending_share( - candidate.pending_share - ) - self.assertEqual(len(fixture.floor()), 1) - # The fence is the oldest entry and the registry sits one over its - # bound: the next recorded outcome evicts the oldest unpinned hash. - server._record_block_candidate_terminal_outcome(target, accepted=False) - self._fill(fixture, self.CAP) - writer_calls = [0] - - def exploding_writer( - candidate_arg: Any, - *, - node_submission: Any, - disposition_held: bool, - ) -> bool: - writer_calls[0] += 1 - raise RuntimeError("accounting writer exploded") - - server._call_block_candidate_writer = exploding_writer - task = self._accounting_task(fixture, target, candidate) - # Park the lane the instant its lease release returns. - in_gap = threading.Event() - release = threading.Event() - self.addCleanup(release.set) - original_release = service._release_block_candidate_disposition - - def release_then_park(lease: Any) -> None: - original_release(lease) - if lease.block_hash == target: - in_gap.set() - if not release.wait(self.WAIT_SECONDS): - raise AssertionError("the parked accounting lane was never released") - - service._release_block_candidate_disposition = release_then_park - - def in_the_gap() -> None: - try: - self.assertTrue(in_gap.wait(self.WAIT_SECONDS)) - # The old gap: the lease is gone and the loop's catch has - # not run; the candidate is a local variable of the lane. - with service._block_candidate_disposition_registry_lock: - self.assertNotIn( - target, - service._block_candidate_disposition_flights, - ) - self._record(fixture, 64) - # The fence survived the pressure; history did not. - self.assertIs( - server._block_candidate_terminal_outcome(target), - False, - ) - self.assertEqual( - len(service._block_candidate_terminal_outcomes), - self.CAP, - ) - # ...because the lane retained into the deferred holder - # before it let go of the lease. - with server.lock: - self.assertIs( - service._block_accounting_deferred_retry_candidate, - candidate, - ) - finally: - # A failed assertion must not leave the lane parked for the - # whole ceiling before the test can report it. - release.set() - - self._drain_accounting_lane(fixture, before_stop=in_the_gap) - self.assertEqual(writer_calls[0], 1) - self.assertTrue(task.retained_for_retry.is_set()) - # Handed off exactly once: in the retry slot, the deferred holder - # cleared, one retry counted, one floor holder, nothing offered. - with server.lock: - self.assertIs(service.retry_candidate, candidate) - self.assertIsNone(service._block_accounting_deferred_retry_candidate) - self.assertFalse( - getattr(service, "_block_accounting_holds_disposition", False) - ) - self.assertEqual(service.retries, 1) - self.assertIn(target, service._outstanding_block_candidate_hashes) - self.assertEqual(len(fixture.floor()), 1) - with service._block_candidate_disposition_registry_lock: - self.assertEqual(service._block_candidate_disposition_flights, {}) - self.assertIs(server._block_candidate_terminal_outcome(target), False) - offers = [ - method for method, _ in fixture.rpc.calls if method == "submitblock" - ] - self.assertEqual(offers, []) - - def test_a_failure_ahead_of_the_accounting_lane_is_still_retained_once( - self, - ) -> None: - """The loop's catch still retains when the lane itself never ran. - - The lane's own exception path marks the task once it has retained. - A failure that fires before the lane can -- here the coordinator - delegate raising ahead of it -- leaves the mark unset, and the loop - retains the candidate exactly as it always did: once. - """ - target = _hash(1) - fixture = CollapseFixture() - service, server = fixture.service, fixture.server - candidate = fixture.seed([target])[0] - task = self._accounting_task(fixture, target, candidate) - # The lane never runs, so nothing releases the task's lease; the - # test owns it from here. - self.addCleanup( - server._release_block_candidate_disposition, - task.disposition_lease, - ) - - def delegate_explodes(task_arg: Any) -> None: - raise RuntimeError("delegate exploded") - - server._run_block_accounting_task = delegate_explodes - self._drain_accounting_lane(fixture) - self.assertFalse(task.retained_for_retry.is_set()) - with server.lock: - self.assertIs(service.retry_candidate, candidate) - self.assertIsNone( - getattr(service, "_block_accounting_deferred_retry_candidate", None) - ) - self.assertEqual(service.retries, 1) - - -class BlockCandidateAbandonmentDedupBoundTests(unittest.TestCase): - """The counted-abandonment dedup set retires with the fence it guards. - - ``_counted_block_candidate_abandonments`` exists because one hash can - reach the terminal accounting path more than once: a collapse whose - cleanup failed retries every owed step, and a finalize failure freezes a - false finalize-only disposition that replays. Abandonment metrics count - candidates, not attempts, so the key deduplicates them. - - Nothing ever retired those keys. One key per candidate the process had - ever abandoned is the same unbounded history the terminal-outcome - registry was just bounded away from, and a collapsed storm writes them - 3,120 at a time. The key is now retired inside the eviction pass, in - the same ``coordinator.lock`` critical section that has already proved - the hash unpinned by every live and cleanup-owing lane, and only for a - hash whose fence that pass actually drops. - - Retiring one early is the dangerous half: a lane still owed its - accounting would count its candidate a second time and the abandoned - series would exceed the candidates it describes. Both writer orderings - have to survive that -- the direct finalize counts before it publishes - its terminal outcome, the collapse publishes first and counts during - cleanup -- so these tests own the bound and the pins in both. - """ - - CAP = MAX_BLOCK_CANDIDATE_TERMINAL_OUTCOMES - REASON = PRISM_BLOCK_CANDIDATE_COLLAPSE_REASON - STALE_CLASS = PRISM_BLOCK_CANDIDATE_COLLAPSE_STALE_JOB_CLASS - STORM = 3_120 - STORMS = 4 - # Well clear of the fixture's own hashes and of the pressure histories. - STORM_START = 1 << 60 - - # -- harness ----------------------------------------------------------- - - def _count(self, fixture: CollapseFixture, block_hash: str) -> None: - """One committed abandonment, with the outcome the collapse builds.""" - fixture.server._record_committed_block_candidate_abandonment( - block_hash, - SimpleNamespace(reason=self.REASON, stale_job_class=self.STALE_CLASS), - ) - - @staticmethod - def _fence(fixture: CollapseFixture, block_hash: str) -> None: - fixture.server._record_block_candidate_terminal_outcome( - block_hash, - accepted=False, - ) - - def _abandon( - self, - fixture: CollapseFixture, - block_hash: str, - *, - terminal_first: bool, - ) -> None: - """Drive one terminal abandonment in one of the two shipped orders. - - ``terminal_first`` is the collapse: the fenced write publishes the - terminal outcome and the cleanup's accounting step follows it. The - other order is ``finalize``, which counts as soon as the durable - outbox mark returns and records the process-local outcome at its - tail. - """ - if terminal_first: - self._fence(fixture, block_hash) - self._count(fixture, block_hash) - else: - self._count(fixture, block_hash) - self._fence(fixture, block_hash) - - @staticmethod - def _counted(fixture: CollapseFixture) -> set[str]: - with fixture.server.lock: - return set(fixture.service._counted_block_candidate_abandonments) - - @staticmethod - def _outcomes(fixture: CollapseFixture) -> set[str]: - with fixture.server.lock: - return set(fixture.service._block_candidate_terminal_outcomes) - - def _pressure(self, fixture: CollapseFixture, *, round_index: int = 0) -> None: - """Push the registry past its bound through the shipped writer. - - The history is uncounted, so its own eviction retires nothing; it is - here purely to make the pass reach whatever the test is protecting. - Each round uses a disjoint range so a second round is real pressure - rather than a re-stamp of the first. - """ - offset = round_index * (1 << 18) - with fixture.server.lock: - outcomes = fixture.service._block_candidate_terminal_outcomes - for block_hash in _history_hashes(self.CAP, start=(1 << 20) + offset): - outcomes[block_hash] = False - for block_hash in _history_hashes(64, start=(1 << 40) + offset): - self._fence(fixture, block_hash) - - # -- the bound --------------------------------------------------------- - - def test_repeated_storms_do_not_grow_the_dedup_history(self) -> None: - """Storm after storm, the dedup set tracks the fences, not the past.""" - for terminal_first in (False, True): - with self.subTest(terminal_first=terminal_first): - fixture = CollapseFixture() - sizes: list[int] = [] - for index in range(self.STORMS): - start = self.STORM_START + index * self.STORM - for block_hash in _history_hashes(self.STORM, start=start): - self._abandon( - fixture, - block_hash, - terminal_first=terminal_first, - ) - sizes.append(len(self._counted(fixture))) - # It stops growing where the fences do, and holds there - # however many further storms arrive. - self.assertEqual(sizes[-1], self.CAP) - self.assertEqual(sizes[-1], sizes[-2]) - self.assertLessEqual(max(sizes), self.CAP) - self.assertEqual(self._counted(fixture), self._outcomes(fixture)) - # Bounding it cost no accuracy: every candidate counted once. - counts = fixture.server.block_candidate_abandoned_counts - self.assertEqual(counts[self.REASON], self.STORM * self.STORMS) - self.assertEqual( - fixture.server.stale_job_abandon_counts[self.STALE_CLASS], - self.STORM * self.STORMS, - ) - - def test_a_collapse_page_retires_its_keys_when_its_fences_age_out( - self, - ) -> None: - """The shipped apply, not a hand-rolled ordering, ends up bounded.""" - fixture = CollapseFixture() - page = [_hash(index) for index in range(1, 5)] - fixture.seed(page) - self.assertEqual(_selected(fixture), set(page)) - self.assertEqual(fixture.cleanup_backlog(), {}) - self.assertEqual( - fixture.server.block_candidate_abandoned_counts[self.REASON], - len(page), - ) - self.assertEqual(self._counted(fixture), set(page)) - self._pressure(fixture) - for block_hash in page: - self.assertIsNone( - fixture.server._block_candidate_terminal_outcome(block_hash) - ) - self.assertEqual(self._counted(fixture), set()) - - def test_the_key_retires_with_its_fence_and_not_before(self) -> None: - """A pass that evicts nothing retires nothing.""" - fixture = CollapseFixture() - target = _hash(1) - self._abandon(fixture, target, terminal_first=True) - with fixture.server.lock: - self.assertEqual( - fixture.service._bound_block_candidate_terminal_outcomes(), - 0, - ) - self.assertEqual(self._counted(fixture), {target}) - self._pressure(fixture) - self.assertIsNone(fixture.server._block_candidate_terminal_outcome(target)) - self.assertEqual(self._counted(fixture), set()) - - # -- the two orderings ------------------------------------------------- - - def test_both_writer_orderings_converge_on_the_same_state(self) -> None: - """Count-then-fence and fence-then-count are indistinguishable.""" - states = [] - for terminal_first in (False, True): - fixture = CollapseFixture() - history = _history_hashes(self.CAP + 64, start=self.STORM_START) - for block_hash in history: - self._abandon(fixture, block_hash, terminal_first=terminal_first) - states.append( - ( - self._counted(fixture), - self._outcomes(fixture), - dict(fixture.server.block_candidate_abandoned_counts), - dict(fixture.server.stale_job_abandon_counts), - ) - ) - self.assertEqual(states[0], states[1]) - counted, outcomes, counts, stale = states[0] - self.assertEqual(counted, outcomes) - self.assertEqual(len(counted), self.CAP) - self.assertEqual(counts[self.REASON], self.CAP + 64) - self.assertEqual(stale[self.STALE_CLASS], self.CAP + 64) - - def test_a_repeated_disposition_is_still_counted_once(self) -> None: - """The dedup the key exists for survives in both orderings.""" - for terminal_first in (False, True): - with self.subTest(terminal_first=terminal_first): - fixture = CollapseFixture() - target = _hash(1) - for _attempt in range(3): - self._abandon(fixture, target, terminal_first=terminal_first) - self.assertEqual( - fixture.server.block_candidate_abandoned_counts[self.REASON], - 1, - ) - self.assertEqual(self._counted(fixture), {target}) - - def test_a_false_finalize_failure_stays_counted_before_any_outcome( - self, - ) -> None: - """The ambiguous branch counts with no fence to retire it by. - - ``finalize`` freezes a false finalize-only disposition when the - durable abandonment mark raises: it counts the candidate and leaves - the hash in ``finalize_retries`` without ever recording a terminal - outcome. There is nothing for the eviction pass to drop, and the - registry that pins it is itself bounded, so the key simply waits for - the replay that publishes its outcome. - """ - fixture = CollapseFixture() - target = _hash(1) - self._count(fixture, target) - with fixture.server.lock: - fixture.service.finalize_retries[target] = (False, "boom") - self._pressure(fixture) - self.assertIsNone(fixture.server._block_candidate_terminal_outcome(target)) - self.assertEqual(self._counted(fixture), {target}) - # The paced finalize-only replay re-enters the same accounting call. - self._count(fixture, target) - self.assertEqual( - fixture.server.block_candidate_abandoned_counts[self.REASON], - 1, - ) - # Its retry succeeds: the tail clears the registry and publishes the - # outcome, and only then can the key age out with it. - with fixture.server.lock: - fixture.service.finalize_retries.pop(target, None) - self._fence(fixture, target) - self._pressure(fixture, round_index=1) - self.assertIsNone(fixture.server._block_candidate_terminal_outcome(target)) - self.assertEqual(self._counted(fixture), set()) - - # -- the pins ---------------------------------------------------------- - - def test_pressure_cannot_forget_a_pinned_hash_and_double_count_it( - self, - ) -> None: - """Every lane that can still run the accounting keeps its key.""" - target = _hash(1) - - def live_queue(fixture: CollapseFixture, candidate: Any) -> None: - fixture.service.candidate_queue.put_nowait(candidate) - - def replay_queue(fixture: CollapseFixture, candidate: Any) -> None: - fixture.service._block_replay_candidate_queue.put_nowait(candidate) - - def cleanup_owed(fixture: CollapseFixture, candidate: Any) -> None: - fixture.service._defer_collapsed_candidate_cleanup( - target, - frozenset({"abandonment-accounting"}), - ) - - def finalize_retry(fixture: CollapseFixture, candidate: Any) -> None: - with fixture.server.lock: - fixture.service.finalize_retries[target] = (False, "boom") - - def disposition_flight(fixture: CollapseFixture, candidate: Any) -> None: - lease = fixture.server._claim_block_candidate_disposition( - target, - blocking=False, - ) - self.assertIsNotNone(lease) - self.addCleanup( - fixture.server._release_block_candidate_disposition, - lease, - ) - - def outstanding(fixture: CollapseFixture, candidate: Any) -> None: - fixture.server._register_outstanding_block_candidate(target) - - def retry_holder(fixture: CollapseFixture, candidate: Any) -> None: - with fixture.server.lock: - fixture.service.retry_candidate = candidate - - lanes = { - "candidate_queue": live_queue, - "_block_replay_candidate_queue": replay_queue, - "_block_candidate_collapse_cleanup_retries": cleanup_owed, - "finalize_retries": finalize_retry, - "_block_candidate_disposition_flights": disposition_flight, - "_outstanding_block_candidate_hashes": outstanding, - "retry_candidate": retry_holder, - } - for terminal_first in (False, True): - for name, install in lanes.items(): - with self.subTest(lane=name, terminal_first=terminal_first): - fixture = CollapseFixture() - candidate = fixture.seed([target])[0] - # The outcome is recorded before the lane is populated: - # recording clears the reservation, replay, and waiting - # markers for its own hash by design. - self._abandon(fixture, target, terminal_first=terminal_first) - with redirect_stdout(StringIO()): - install(fixture, candidate) - self._pressure(fixture) - self.assertEqual( - len(fixture.service._block_candidate_terminal_outcomes), - self.CAP, - ) - self.assertIs( - fixture.server._block_candidate_terminal_outcome(target), - False, - ) - self.assertIn(target, self._counted(fixture)) - # The lane now runs the accounting it was still owed. - # The surviving key is the only thing standing between - # that and a second count of one candidate. - self._count(fixture, target) - self.assertEqual( - fixture.server.block_candidate_abandoned_counts[ - self.REASON - ], - 1, - ) - - -class BlockCandidateCollapseEnumerationTests(unittest.TestCase): - """Both replay enumeration shapes collapse before they adopt.""" - - def _adopted(self, fixture: CollapseFixture) -> set[str]: - service = fixture.service - service._ensure_block_replay_state() - with fixture.server.lock: - return set(service._block_replay_inflight_hashes) - - def test_keyset_pagination_collapses_then_adopts_the_remainder(self) -> None: - fixture = CollapseFixture() - fixture.seed([_hash(index) for index in range(1, 6)]) - fixture.seed([_hash(index) for index in range(6, 9)], parent=OTHER_PARENT) - fixture.rpc.tip = OTHER_PARENT - queued, log = fixture.replay(enumeration_owed=True) - self.assertIn("pending block candidate enumeration page=1", log) - self.assertEqual(fixture.pending(), {_hash(6), _hash(7), _hash(8)}) - self.assertEqual( - self._adopted(fixture), - {_hash(6), _hash(7), _hash(8)}, - ) - self.assertEqual(queued, 3) - - def test_legacy_window_collapses_then_adopts_the_remainder(self) -> None: - fixture = CollapseFixture() - fixture.seed([_hash(index) for index in range(1, 6)]) - fixture.seed([_hash(index) for index in range(6, 9)], parent=OTHER_PARENT) - fixture.rpc.tip = OTHER_PARENT - # Enumeration is not owed, so replay_pending takes the widening - # window even against a cursor-capable ledger. - queued, log = fixture.replay(enumeration_owed=False) - self.assertNotIn("enumeration page=", log) - self.assertEqual(fixture.pending(), {_hash(6), _hash(7), _hash(8)}) - self.assertEqual( - self._adopted(fixture), - {_hash(6), _hash(7), _hash(8)}, - ) - self.assertEqual(queued, 3) - - def test_cursorless_ledger_collapses_then_adopts_the_remainder(self) -> None: - fixture = CollapseFixture(ledger=WindowCollapseLedger()) - fixture.seed([_hash(index) for index in range(1, 6)]) - fixture.seed([_hash(index) for index in range(6, 9)], parent=OTHER_PARENT) - fixture.rpc.tip = OTHER_PARENT - queued, log = fixture.replay(enumeration_owed=True) - self.assertNotIn("enumeration page=", log) - self.assertEqual(fixture.pending(), {_hash(6), _hash(7), _hash(8)}) - self.assertEqual(self._adopted(fixture), {_hash(6), _hash(7), _hash(8)}) - self.assertEqual(queued, 3) - - def test_a_collapsed_row_is_never_adopted_or_re_armed(self) -> None: - """A won row's fetched payload is a stale copy of a terminal row.""" - for owed in (True, False): - with self.subTest(enumeration_owed=owed): - fixture = CollapseFixture() - fixture.seed([_hash(1), _hash(2)]) - decoded: list[str] = [] - original = fixture.server.block_candidate_from_intent - - def record(intent): - decoded.append(str(intent["block_hash_hex"]).lower()) - return original(intent) - - fixture.server.block_candidate_from_intent = record - queued, _log = fixture.replay(enumeration_owed=owed) - self.assertEqual(queued, 0) - self.assertEqual(decoded, []) - self.assertEqual(self._adopted(fixture), set()) - with fixture.server._accepted_block_payout_preview_condition: - self.assertEqual( - fixture.server._accepted_block_payout_previews, - {}, - ) - - def test_a_partly_returned_request_still_adopts_every_other_row(self) -> None: - for owed in (True, False): - with self.subTest(enumeration_owed=owed): - fixture = CollapseFixture() - fixture.seed([_hash(1), _hash(2), _hash(3)]) - - def partial(hashes: tuple[str, ...], error: str) -> tuple[str, ...]: - won = tuple(value for value in hashes if value != _hash(2)) - return fixture.ledger.inner.mark_block_candidates_abandoned( - block_hashes=won, - error=error, - ) - - fixture.ledger.abandon_hook = partial - queued, _log = fixture.replay(enumeration_owed=owed) - self.assertEqual(fixture.pending(), {_hash(2)}) - self.assertEqual(self._adopted(fixture), {_hash(2)}) - self.assertEqual(queued, 1) - - def test_a_fail_closed_page_adopts_every_row(self) -> None: - for owed in (True, False): - with self.subTest(enumeration_owed=owed): - fixture = CollapseFixture() - fixture.seed([_hash(1), _hash(2), _hash(3)]) - fixture.rpc.failures.add("getblockcount") - queued, _log = fixture.replay(enumeration_owed=owed) - self.assertEqual( - fixture.pending(), - {_hash(1), _hash(2), _hash(3)}, - ) - self.assertEqual( - self._adopted(fixture), - {_hash(1), _hash(2), _hash(3)}, - ) - self.assertEqual(queued, 3) - - def test_a_failed_write_adopts_every_row(self) -> None: - for owed in (True, False): - with self.subTest(enumeration_owed=owed): - fixture = CollapseFixture() - fixture.seed([_hash(1), _hash(2)]) - fixture.ledger.abandon_hook = lambda hashes, error: ( - (_ for _ in ()).throw(RuntimeError("ledger down")) - ) - queued, _log = fixture.replay(enumeration_owed=owed) - self.assertEqual(fixture.pending(), {_hash(1), _hash(2)}) - self.assertEqual(self._adopted(fixture), {_hash(1), _hash(2)}) - self.assertEqual(queued, 2) - - def test_a_timed_out_write_that_lands_late_preserves_then_converges(self) -> None: - """The caller's deadline expires while the batch write is still out. - - This drives the real submitter ledger-call seam rather than raising - from the ledger hook: the batch write runs on the bounded worker - ``_run_block_submitter_ledger_call`` spawns, the coordinator-side - wait expires first and raises ``BlockSubmitterDatabaseTimeout``, and - the worker completes the write afterwards. The apply learns nothing - from a call that never answered it, so the whole page fails closed - and stays adoptable, and the write that lands later is still fenced - by the durable pool-block fact rather than by what the selector read - a page earlier. Whatever that write transitions, no cleanup and no - payout authority may follow from a hash the call did not return - synchronously. - """ - wait_seconds = 10.0 - fixture = CollapseFixture() - abandoned_late = _hash(1) - landed_meanwhile = _hash(2) - fixture.seed([abandoned_late, landed_meanwhile], credit_share_on_accept=True) - # Short enough to keep the test quick, wide enough that the page - # read ahead of it is never the call that expires. - fixture.server.block_submit_db_timeout_seconds = 0.25 - started = threading.Event() - release = threading.Event() - self.addCleanup(release.set) - - def stall_then_write(hashes: tuple[str, ...], error: str) -> tuple[str, ...]: - # Deterministic: the write cannot land before the deadline - # because only the test releases it, and it is released only - # after the timed-out apply has returned. - started.set() - if not release.wait(wait_seconds): - raise AssertionError("the stalled ledger call was never released") - return fixture.ledger.inner.mark_block_candidates_abandoned( - block_hashes=hashes, - error=error, - ) - - fixture.ledger.abandon_hook = stall_then_write - queued, log = fixture.replay(enumeration_owed=False) - self.assertTrue(started.is_set()) - self.assertIn("ledger phase timed out phase=collapse-superseded", log) - # Failed closed: every row is preserved, adopted, and left to the - # per-row path exactly as an outright write failure leaves it. - self.assertEqual(queued, 2) - self.assertEqual(fixture.pending(), {abandoned_late, landed_meanwhile}) - self.assertEqual(self._adopted(fixture), {abandoned_late, landed_meanwhile}) - counts = fixture.counts() - self.assertEqual(counts["fail_closed"], 2) - self.assertEqual(counts["abandoned"], 0) - self.assertEqual(counts["write_lost"], 0) - self.assertEqual(counts["cleanup_failed"], 0) - self.assertEqual( - fixture.service.block_ledger_call_class_metrics()["fast"][ - "timeouts_total" - ], - 1, - ) - with fixture.service._block_candidate_disposition_registry_lock: - self.assertEqual(fixture.service._block_candidate_disposition_flights, {}) - adopted_floor = set(fixture.floor()) - self.assertEqual(len(adopted_floor), 2) - with fixture.service._block_submitter_ledger_calls_lock: - stalled = [ - call - for key, call in fixture.service._block_submitter_ledger_calls.items() - if key[0] == "collapse-superseded" - ] - self.assertEqual(len(stalled), 1) - # One of the two rows really did land while the call was out. Only - # the durable pool-block row proves it, and the fenced write re-asks - # for that fact itself. - fixture.ledger.inner.persist_accepted_block( - block_hash=landed_meanwhile, - block_height=DECIDED_HEIGHT, - parent_hash=STORM_PARENT, - final_bundle={}, - audit_report={}, - ) - release.set() - self.assertTrue(stalled[0].done.wait(wait_seconds)) - self.assertEqual(len(fixture.ledger.abandon_calls), 1) - self.assertEqual( - set(fixture.ledger.abandon_calls[0][0]), - {abandoned_late, landed_meanwhile}, - ) - self.assertEqual(set(stalled[0].result), {abandoned_late}) - self.assertEqual(fixture.pending(), {landed_meanwhile}) - # Nothing was cleaned up for the hash the call never returned to - # anybody: its in-memory state still belongs to the per-row path. - server = fixture.server - with server.lock: - self.assertEqual(fixture.service._block_candidate_terminal_outcomes, {}) - self.assertEqual(server.block_candidate_abandoned_counts, {}) - self.assertEqual(set(fixture.floor()), adopted_floor) - # A later enumeration is idempotent: the terminal row is gone from - # the outbox, the landed one is excluded by its pool-block row, and - # no second fenced write is issued. - second_queued, _second_log = fixture.replay(enumeration_owed=True) - self.assertEqual(second_queued, 0) - self.assertEqual(len(fixture.ledger.abandon_calls), 1) - self.assertEqual(fixture.pending(), {landed_meanwhile}) - self.assertEqual(self._adopted(fixture), {abandoned_late, landed_meanwhile}) - counts = fixture.counts() - self.assertEqual(counts["selected"], 2) - self.assertEqual(counts["abandoned"], 0) - self.assertEqual(counts["cleanup_failed"], 0) - # The barrier replay armed is still only a barrier: no landed - # transition and no published preview were invented for a row whose - # fate the coordinator never observed. - with server._accepted_block_payout_preview_condition: - previews = dict(server._accepted_block_payout_previews) - self.assertEqual( - dict(server._invalidated_accepted_block_payout_previews), - {}, - ) - self.assertEqual(set(previews), {abandoned_late, landed_meanwhile}) - self.assertFalse(previews[abandoned_late].landed) - self.assertIsNone(previews[abandoned_late].preview) - self.assertFalse( - server._accepted_block_payout_transition_landed(abandoned_late) - ) - - def test_repeated_late_collapse_writes_leave_no_ledger_calls(self) -> None: - """A page key that never recurs must still leave the registry. - - Each fenced write is keyed by the exact page of hashes it abandons, - so the write that lands after the caller's deadline is the very - thing that empties that page out of the outbox: nothing invokes that - key again to observe the completion and drop the entry. Several such - pages in a row must leave the registry back at empty, not one - completed call per page each pinning its whole hash tuple for the - life of the process. - """ - fixture = CollapseFixture() - # Wide enough that the page read ahead of the write is never the - # call that expires, short enough to keep the test quick. - fixture.server.block_submit_db_timeout_seconds = 0.25 - rounds = 4 - page_keys: list[tuple[object, ...]] = [] - for round_index in range(rounds): - fixture.seed( - [_hash(2 * round_index + 1), _hash(2 * round_index + 2)] - ) - release = threading.Event() - self.addCleanup(release.set) - - def stall_then_write( - hashes: tuple[str, ...], - error: str, - release: threading.Event = release, - ) -> tuple[str, ...]: - if not release.wait(10): - raise AssertionError( - "the stalled ledger call was never released" - ) - return fixture.ledger.inner.mark_block_candidates_abandoned( - block_hashes=hashes, - error=error, - ) - - fixture.ledger.abandon_hook = stall_then_write - _retained, log = fixture.collapse() - self.assertIn( - "ledger phase timed out phase=collapse-superseded", - log, - ) - with fixture.service._block_submitter_ledger_calls_lock: - stalled = [ - key - for key, call in ( - fixture.service._block_submitter_ledger_calls.items() - ) - if not call.done.is_set() - ] - self.assertEqual(len(stalled), 1) - page_keys.append(stalled[0]) - worker = next( - thread - for thread in threading.enumerate() - if thread.name == "prism-block-ledger-collapse-superseded" - ) - release.set() - worker.join(10) - self.assertFalse(worker.is_alive()) - # The write committed on the worker, and took its entry with it. - with fixture.service._block_submitter_ledger_calls_lock: - self.assertEqual( - dict(fixture.service._block_submitter_ledger_calls), - {}, - ) - - # Every page really was a distinct key, and every delayed write - # really did commit. - self.assertEqual(len(fixture.ledger.abandon_calls), rounds) - self.assertEqual(len(set(page_keys)), rounds) - self.assertEqual(fixture.pending(), set()) - - def test_pagination_still_completes_after_a_full_page_collapses(self) -> None: - """Collapse must not break the short-page completeness proof.""" - fixture = CollapseFixture() - total = MAX_BLOCK_REPLAY_ENUMERATION_ROWS + 7 - fixture.seed([_hash(index) for index in range(1, total + 1)]) - queued, log = fixture.replay(enumeration_owed=True) - self.assertEqual(fixture.pending(), set()) - self.assertEqual(queued, 0) - self.assertIn("enumeration page=2", log) - self.assertFalse(fixture.server._block_replay_enumeration_owed()) - self.assertEqual(len(fixture.ledger.abandon_calls), 2) - self.assertEqual( - [len(call[0]) for call in fixture.ledger.abandon_calls], - [MAX_BLOCK_REPLAY_ENUMERATION_ROWS, 7], - ) - - -class BlockCandidateCollapseHeightProbeBoundTests(unittest.TestCase): - """A height-diverse backlog must not turn collapse into a page-sized walk. - - Collapse reads the chain once per *distinct candidate height*. For the - same-height storm this path exists for that is a handful of round trips - however wide the page is. For a page whose rows sit at many different - heights it was one ``getblockhash`` plus one ``getblockheader`` per - height, sequentially, on the sole block-submitter thread: a full - 1,024-row page of distinct heights is ~2,048 node calls, and at a - production RPC deadline that is minutes to hours in which no live solve - reaches ``submitblock``. - - MAX_BLOCK_CANDIDATE_COLLAPSE_HEIGHT_PROBES caps the distinct heights one - enumeration walk reads. These tests own the four properties that make it - safe: the cap actually binds, an unprobed row is preserved rather than - terminalized and collapses on a later walk, the walk hands the submitter - back to a waiting live solve once its budget is gone, and none of that - disturbs the storm, which spends one probe for the whole walk however - many pages it spans. - """ - - # -- harness ----------------------------------------------------------- - - @staticmethod - def _diverse(heights: int) -> tuple[CollapseFixture, list[str]]: - """A backlog of one row at each of ``heights`` distinct decided heights. - - Every candidate height is occupied by a different heavier block and - the tip sits one height above the highest of them, so predicate S - would select every seeded row if the walk could afford to probe every - height. What the walk actually collapses is therefore exactly what - the probe budget bought. - """ - top = heights + 1 - occupied = { - height: f"{0xC0DE0000 + height:064x}" - for height in range(1, top) - } - occupied[top] = DECIDED_WINNER - fixture = CollapseFixture( - rpc=CollapseChainRpc( - tip=DECIDED_WINNER, - tip_height=top, - active=occupied, - ) - ) - seeded = [] - for height in range(1, top): - block_hash = _hash(height) - seeded.append(block_hash) - fixture.seed([block_hash], height=height) - return fixture, seeded - - @staticmethod - def _live_solve(fixture: CollapseFixture, *, height: int) -> Any: - """One durable candidate on the best tip, admitted to the live lane. - - Its parent is the tip, so clause 3 rejects it before any height is - read: it costs the walk no probe and is never a collapse candidate. - It is queued through the shipped admission path so the submitter sees - exactly what a fresh solve leaves behind. - """ - candidate = fixture.seed( - [LIVE_SOLVE_HASH], - parent=DECIDED_WINNER, - height=height, - )[0] - fixture.server.enqueue_block_candidate(candidate) - fixture.rpc.results["submitblock"] = None - fixture.server.block_submit_rpc_timeout_seconds = 0.5 - return candidate - - @staticmethod - def _offers(fixture: CollapseFixture) -> int: - return sum(1 for method, _ in fixture.rpc.calls if method == "submitblock") - - # -- the bound binds --------------------------------------------------- - - def test_a_height_diverse_page_stops_at_the_probe_bound(self) -> None: - """The P1: 1,024 rows at 1,024 heights, still a fixed handful of reads.""" - rows = MAX_BLOCK_REPLAY_ENUMERATION_ROWS - bound = MAX_BLOCK_CANDIDATE_COLLAPSE_HEIGHT_PROBES - fixture, _seeded = self._diverse(rows) - page = fixture.page(limit=rows) - self.assertEqual(len(page), rows) - retained, _log = fixture.collapse(page) - probed = { - params[0] - for method, params in fixture.rpc.calls - if method == "getblockhash" - } - self.assertEqual(len(probed), bound) - counts = fixture.rpc.counts() - # Selection probes the bound; the pre-write revalidation re-reads - # exactly the heights it selected and reuses their cached work, so - # the whole pass is a constant regardless of the page's row count. - self.assertEqual(counts["getblockhash"], 2 * bound) - self.assertEqual(counts["getblockheader"], bound) - self.assertEqual(counts["getbestblockhash"], 2) - self.assertEqual(counts["getblockcount"], 1) - self.assertEqual(fixture.counts()["abandoned"], bound) - self.assertEqual(fixture.counts()["height_deferred"], rows - bound) - # Every unprobed row is handed back, and handed back durable. - self.assertEqual(len(retained), rows - bound) - self.assertEqual( - {str(row["block_hash"]).lower() for row in retained}, - fixture.pending(), - ) - - def test_unprobed_rows_are_retained_and_later_passes_converge(self) -> None: - """Deferral is not loss: each pass buys the bound, and they add up.""" - bound = MAX_BLOCK_CANDIDATE_COLLAPSE_HEIGHT_PROBES - heights = 3 * bound - fixture, seeded = self._diverse(heights) - outstanding = set(seeded) - self.assertEqual(fixture.pending(), outstanding) - collapsed: set[str] = set() - passes = 0 - # Bounded rather than a while-loop: a pass that stopped making - # progress must fail this test, not spin in it. - for _attempt in range(heights + 1): - if not fixture.pending(): - break - before = fixture.pending() - retained, _log = fixture.collapse(fixture.page(limit=heights)) - after = fixture.pending() - passes += 1 - won = before - after - self.assertEqual(len(won), bound) - self.assertEqual(won & collapsed, set()) - self.assertEqual( - {str(row["block_hash"]).lower() for row in retained}, - after, - "a preserved row must still be the durable row it came from", - ) - collapsed |= won - self.assertEqual(collapsed, outstanding) - self.assertEqual(fixture.pending(), set()) - self.assertEqual(passes, heights // bound) - - # -- the walk hands the submitter back -------------------------------- - - def test_the_walk_yields_to_a_waiting_live_solve_between_pages(self) -> None: - """A diverse backlog cannot hold the only thread that reaches qbitd.""" - bound = MAX_BLOCK_CANDIDATE_COLLAPSE_HEIGHT_PROBES - page_rows = MAX_BLOCK_REPLAY_ENUMERATION_ROWS - fixture, _seeded = self._diverse(page_rows + 7) - # Seeded last, so its durable row sits on the page the walk yields - # before ever fetching. - self._live_solve(fixture, height=page_rows + 9) - queued, log = fixture.replay(enumeration_owed=True) - self.assertIn("enumeration page=1", log) - self.assertNotIn("enumeration page=2", log) - self.assertIn("yielded to a waiting live candidate", log) - counts = fixture.rpc.counts() - self.assertEqual(counts["getblockhash"], 2 * bound) - self.assertEqual(counts["getblockheader"], bound) - # An unproven enumeration stays owed: yielding must not be mistaken - # for the completeness proof that unblocks job builds. - self.assertTrue(fixture.server._block_replay_enumeration_owed()) - # Nothing was skipped -- the page it did fetch was adopted in full. - self.assertEqual(queued, page_rows - bound) - service = fixture.service - self.assertEqual(service.candidate_queue.qsize(), 1) - self.assertEqual( - service._block_replay_candidate_queue.qsize(), - page_rows - bound, - ) - # The scheduling contract itself: the very next turn of the submitter - # loop lands the live solve, ahead of the whole adopted backlog. - with redirect_stdout(StringIO()): - self.assertTrue( - fixture.server.submit_next_block_candidate(defer_accounting=True) - ) - self.assertEqual(self._offers(fixture), 1) - self.assertTrue(service.candidate_queue.empty()) - self.assertEqual( - service._block_replay_candidate_queue.qsize(), - page_rows - bound, - ) - - def test_a_walk_with_nothing_waiting_finishes_its_pages(self) -> None: - """The yield is conditional; without a live solve the walk completes.""" - bound = MAX_BLOCK_CANDIDATE_COLLAPSE_HEIGHT_PROBES - rows = MAX_BLOCK_REPLAY_ENUMERATION_ROWS + 7 - fixture, _seeded = self._diverse(rows) - queued, log = fixture.replay(enumeration_owed=True) - self.assertIn("enumeration page=2", log) - self.assertNotIn("yielded", log) - self.assertFalse(fixture.server._block_replay_enumeration_owed()) - self.assertEqual(queued, rows - bound) - # Page two spends no chain read at all. Every row in this backlog - # sits at a height of its own, so page two holds nothing the walk - # already read and a spent budget leaves it nothing to qualify -- - # even the two page-scope tip reads would have been round trips for - # a decided answer. - counts = fixture.rpc.counts() - self.assertEqual(counts["getbestblockhash"], 2) - self.assertEqual(counts["getblockcount"], 1) - self.assertEqual(counts["getblockhash"], 2 * bound) - self.assertEqual(fixture.counts()["height_deferred"], rows - bound) - - def test_a_same_height_storm_walk_is_unchanged(self) -> None: - """The shape collapse exists for pays one probe per page and never yields.""" - fixture = CollapseFixture() - storm = MAX_BLOCK_REPLAY_ENUMERATION_ROWS + 7 - fixture.seed([_hash(index) for index in range(1, storm + 1)]) - self._live_solve(fixture, height=DECIDED_HEIGHT + 1) - queued, log = fixture.replay(enumeration_owed=True) - # A live solve waits for the whole walk, but one decided height costs - # one probe per page, so the budget is never spent and the storm - # still collapses inside a single, complete enumeration. - self.assertNotIn("yielded", log) - self.assertIn("enumeration page=2", log) - self.assertFalse(fixture.server._block_replay_enumeration_owed()) - self.assertEqual(fixture.counts()["abandoned"], storm) - self.assertEqual(fixture.counts()["height_deferred"], 0) - self.assertEqual(fixture.pending(), {LIVE_SOLVE_HASH}) - self.assertEqual(queued, 1) - self.assertEqual( - [len(call[0]) for call in fixture.ledger.abandon_calls], - [MAX_BLOCK_REPLAY_ENUMERATION_ROWS, 7], - ) - - def test_a_storm_deeper_than_the_budget_collapses_in_one_walk(self) -> None: - """A storm spanning more pages than the walk has probes still collapses. - - The budget is per walk but the chain reads it pays for were once per - *page*: every page rebuilt its view, so one decided height was - charged again on every page and the walk ran out after - MAX_BLOCK_CANDIDATE_COLLAPSE_HEIGHT_PROBES pages. Past that point - collapse did nothing at all. Each remaining page was preserved - whole by the exhausted-budget early return and replay-adopted, or -- - as here, with a live candidate waiting -- never fetched at all, - because a spent budget is half of the yield condition. Either way a - backlog deeper than eight pages collapsed only 8,192 rows per walk, - which at the 312,000-row scale this path is sized for defeats it - outright. - - This walk is nine pages of one decided height, so the eight-page - boundary is genuinely crossed, and the whole storm has to go in one - enumeration off a single probe. - """ - bound = MAX_BLOCK_CANDIDATE_COLLAPSE_HEIGHT_PROBES - page_rows = MAX_BLOCK_REPLAY_ENUMERATION_ROWS - # One page past the bound, plus a short final page: nine fetched - # pages against eight affordable probes. - storm = bound * page_rows + 7 - fixture = CollapseFixture() - fixture.seed([_hash(index) for index in range(1, storm + 1)]) - # Seeded last, so it sits on the ninth page -- the page a walk that - # recharged per page never reaches, having yielded to this very - # candidate once its budget was gone. - self._live_solve(fixture, height=DECIDED_HEIGHT + 1) - queued, log = fixture.replay(enumeration_owed=True) - # One complete enumeration: every page fetched, none yielded, and the - # completeness proof the job-build gate waits on delivered. - self.assertIn(f"enumeration page={bound + 1}", log) - self.assertNotIn("yielded", log) - self.assertFalse(fixture.server._block_replay_enumeration_owed()) - # The whole storm collapsed, not the first eight pages of it. - self.assertEqual(fixture.counts()["abandoned"], storm) - self.assertEqual(fixture.counts()["height_deferred"], 0) - self.assertEqual(fixture.pending(), {LIVE_SOLVE_HASH}) - self.assertEqual( - [len(call[0]) for call in fixture.ledger.abandon_calls], - [page_rows] * bound + [7], - ) - # Not one collapsed row was replay-adopted: the single adoption is - # the live solve, which predicate S rejects on clause 3. - self.assertEqual(queued, 1) - service = fixture.service - self.assertEqual(service._block_replay_candidate_queue.qsize(), 1) - adopted = service._block_replay_candidate_queue.get_nowait() - self.assertEqual(adopted.submission.block_hash_hex.lower(), LIVE_SOLVE_HASH) - # The decided height is charged once for the whole walk. Selection - # reads it on page one and every later page answers from the walk's - # cache, so the occupant's header is read exactly once; the nine - # remaining height reads are the per-page pre-write revalidation, - # which keeps a budget -- and therefore a cache -- of its own by - # design and re-reads the chain under the held leases. - probes = [ - params[0] - for method, params in fixture.rpc.calls - if method == "getblockhash" - ] - self.assertEqual(set(probes), {DECIDED_HEIGHT}) - self.assertEqual(len(probes), 1 + (bound + 1)) - counts = fixture.rpc.counts() - self.assertEqual(counts["getblockheader"], 1) - # Page-scope reads stay per page: one selection tip pair and one - # revalidation tip read each. - self.assertEqual(counts["getbestblockhash"], 2 * (bound + 1)) - self.assertEqual(counts["getblockcount"], bound + 1) - - def test_cached_heights_keep_collapsing_after_the_budget_is_spent(self) -> None: - """A spent budget stops new heights, not the ones already read. - - The counter and the caches are shared by the walk, but the page pass - used to return the whole page the moment the counter hit zero. A - first page spread across all eight affordable heights therefore - disarmed collapse for the rest of the walk, even for later siblings - at exactly those heights -- rows the walk could have decided without - one new round trip, because their occupant was already in - ``probe_budget.active``. Every one of them was preserved and - replay-adopted instead, which is the same defeat the shared cache was - added to prevent, just one page later. - - Nine heights are seeded and only eight are ever affordable, so the - walk here has to draw the line in exactly the right place: reuse what - it read, refuse what it did not. - """ - bound = MAX_BLOCK_CANDIDATE_COLLAPSE_HEIGHT_PROBES - page_rows = MAX_BLOCK_REPLAY_ENUMERATION_ROWS - per_height = page_rows // bound - # The ninth height is the negative control: decided, occupied by a - # heavier block, and selected by every clause of S -- so the only - # thing that can defer it is the spent budget. - control_height = bound + 1 - top = control_height + 1 - occupied = { - height: f"{0xC0DE0000 + height:064x}" - for height in range(1, top) - } - occupied[top] = DECIDED_WINNER - fixture = CollapseFixture( - rpc=CollapseChainRpc( - tip=DECIDED_WINNER, - tip_height=top, - active=occupied, - ) - ) - index = 1 - - def seed(height: int, rows: int) -> list[str]: - nonlocal index - hashes = [_hash(value) for value in range(index, index + rows)] - index += rows - fixture.seed(hashes, height=height) - return hashes - - # Page one: a full page spread across all eight affordable heights, - # which spends the walk's budget exactly. - for height in range(1, bound + 1): - seed(height, per_height) - # Page two: siblings at those same eight heights -- free to decide -- - # and, last on the page, the control rows at the height the walk can - # no longer afford to read. - for height in range(1, bound + 1): - seed(height, per_height - 1) - control = set(seed(control_height, bound)) - # A short third page, still at cached heights, proves the reuse - # outlives the page that first ran into the bound and ends the walk. - for height in range(1, bound): - seed(height, 1) - cached_rows = page_rows + (bound * (per_height - 1)) + (bound - 1) - - queued, log = fixture.replay(enumeration_owed=True) - # One complete enumeration, all three pages walked. - self.assertIn("enumeration page=3", log) - self.assertNotIn("yielded", log) - self.assertFalse(fixture.server._block_replay_enumeration_owed()) - # Every sibling at a height this walk had already read collapsed, - # including the two pages that ran entirely on a spent counter. - self.assertEqual(fixture.counts()["abandoned"], cached_rows) - self.assertEqual( - [len(call[0]) for call in fixture.ledger.abandon_calls], - [page_rows, bound * (per_height - 1), bound - 1], - ) - # The control rows are the only deferral, and they are preserved - # durable rather than lost. - self.assertEqual(fixture.counts()["height_deferred"], bound) - self.assertEqual(fixture.pending(), control) - # Nor did the walk leak a collapsed sibling into replay adoption: the - # deferred controls are the entire adopted set. - self.assertEqual(queued, bound) - service = fixture.service - self.assertEqual(service._block_replay_candidate_queue.qsize(), bound) - adopted = set() - while not service._block_replay_candidate_queue.empty(): - candidate = service._block_replay_candidate_queue.get_nowait() - adopted.add(candidate.submission.block_hash_hex.lower()) - self.assertEqual(adopted, control) - # Not one new distinct height was read past the bound. The control - # height is never asked about at all -- an unprobed height is not a - # cheaper probe, it is no probe. - probes = [ - params[0] - for method, params in fixture.rpc.calls - if method == "getblockhash" - ] - self.assertEqual(set(probes), set(range(1, bound + 1))) - self.assertNotIn(control_height, probes) - # Selection reads the eight heights once for the whole walk; the rest - # are the per-page pre-write revalidation, which keeps its own budget - # and re-reads each retained height under the held leases. - self.assertEqual(len(probes), bound + (bound + bound + (bound - 1))) - counts = fixture.rpc.counts() - # One header per occupant for the walk: page two and page three both - # answer clause 4b from the work this walk already derived. - self.assertEqual(counts["getblockheader"], bound) - # Page-scope tip reads stay per page and stay fresh, on the pages a - # spent budget now lets through as much as on the first. - self.assertEqual(counts["getbestblockhash"], 2 * 3) - self.assertEqual(counts["getblockcount"], 3) - - -class BlockCandidateCollapseObservabilityTests(unittest.TestCase): - """Bounded logs and bounded, hash-free metrics.""" - - def test_the_bulk_path_does_not_log_one_line_per_candidate(self) -> None: - fixture = CollapseFixture() - fixture.seed([_hash(index) for index in range(1, 201)]) - _retained, log = fixture.collapse() - lines = [line for line in log.splitlines() if line.strip()] - self.assertLessEqual(len(lines), 4 + BLOCK_CANDIDATE_COLLAPSE_LOG_GROUPS) - self.assertNotIn("block candidate abandoned reason=", log) - summary = [line for line in lines if "collapsed superseded" in line] - self.assertEqual(len(summary), 1) - self.assertIn("considered=200", summary[0]) - self.assertIn("abandoned=200", summary[0]) - - def test_group_lines_and_hash_samples_stay_bounded(self) -> None: - fixture = CollapseFixture() - groups = BLOCK_CANDIDATE_COLLAPSE_LOG_GROUPS + 3 - index = 1 - for group in range(groups): - parent = f"{group:064x}" - fixture.seed( - [_hash(index + offset) for offset in range(5)], - parent=parent, - ) - index += 5 - _retained, log = fixture.collapse() - group_lines = [ - line for line in log.splitlines() if "collapsed candidate group " in line - ] - self.assertEqual(len(group_lines), BLOCK_CANDIDATE_COLLAPSE_LOG_GROUPS) - self.assertIn("collapsed candidate groups not shown count=3", log) - for line in group_lines: - sample = line.split("sample=", 1)[1].strip() - self.assertLessEqual( - len(sample.split(",")), - BLOCK_CANDIDATE_COLLAPSE_LOG_SAMPLE_HASHES, - ) - - def test_fail_closed_logging_is_rate_limited(self) -> None: - fixture = CollapseFixture() - fixture.seed([_hash(1)]) - fixture.rpc.failures.add("getblockcount") - _retained, first = fixture.collapse() - _retained, second = fixture.collapse() - self.assertIn("collapse failed", first) - self.assertNotIn("collapse failed", second) - self.assertEqual(fixture.counts()["fail_closed"], 2) - - def test_the_snapshot_carries_exactly_the_fixed_outcome_keys(self) -> None: - fixture = CollapseFixture() - self.assertEqual( - tuple(fixture.counts()), - PRISM_BLOCK_CANDIDATE_COLLAPSE_OUTCOMES, - ) - - def test_metrics_render_bounded_hash_free_collapse_series(self) -> None: - fixture = CollapseFixture() - fixture.seed([_hash(1), _hash(2)]) - with redirect_stdout(StringIO()): - fixture.collapse() - lines = MetricsRenderer(fixture.server).block_submitter_metrics_lines() - series = [ - line - for line in lines - if line.startswith("qbit_prism_block_candidate_collapse_total") - ] - self.assertEqual(len(series), len(PRISM_BLOCK_CANDIDATE_COLLAPSE_OUTCOMES)) - rendered = "\n".join(lines) - self.assertIn( - 'qbit_prism_block_candidate_collapse_total{outcome="abandoned"} 2', - rendered, - ) - self.assertIn( - 'qbit_prism_block_candidate_collapse_total{outcome="considered"} 2', - rendered, - ) - for forbidden in (_hash(1), _hash(2), STORM_PARENT, DECIDED_WINNER, "job-"): - self.assertNotIn(forbidden, rendered) - - def test_per_row_diagnostics_survive_for_preserved_candidates(self) -> None: - """A candidate the collapse declines still logs its own abandonment.""" - fixture = CollapseFixture() - preserved = _hash(1) - candidate = fixture.seed([preserved], parent=OTHER_PARENT)[0] - fixture.rpc.tip = OTHER_PARENT - fixture.collapse() - self.assertEqual(fixture.pending(), {preserved}) - buffer = StringIO() - with redirect_stdout(buffer): - fixture.service.record_abandoned( - PRISM_REJECTION_STALE_JOB, - "tip moved before submit: deadbeef", - block_hash=preserved, - worker="miner-a", - expected_height=DECIDED_HEIGHT, - stale_job_class="tip_moved", - ) - self.assertIn("block candidate abandoned reason=stale-job", buffer.getvalue()) - self.assertIs(candidate.credit_share_on_accept, False) - - -class BlockCandidateCollapseStormTests(unittest.TestCase): - """Compare the selector against the shipped per-row oracle at storm sizes.""" - - def _oracle(self, candidates: int, view: str) -> tuple[set[str], str]: - rig = CandidateStormRig(candidates=candidates) - rig.seed_live() - if view == "restart": - rig.restart_and_enumerate() - server = rig.restarted_server if view == "restart" else rig.live_server - # ``drain_per_row`` suppresses the collapse itself, so this stays - # the shipped per-row oracle without any patching here. - winner = rig.decide_height() - report = rig.drain_per_row(server, stop_before_hash=winner) - return set(report.abandoned_hashes), winner - - def _selector( - self, - candidates: int, - view: str, - *, - perturbation: str | None = None, - target: str | None = None, - ) -> tuple[set[str], Any, str]: - rig = CandidateStormRig(candidates=candidates) - rig.seed_live() - if view == "restart": - rig.restart_and_enumerate() - server = rig.restarted_server if view == "restart" else rig.live_server - winner = rig.decide_height() - if perturbation is not None: - with redirect_stdout(StringIO()): - rig.perturb(server, perturbation, str(target)) - before = { - str(row["block_hash"]).lower() - for row in rig.ledger.pending_block_candidate_rows(limit=candidates + 1) - } - server._note_block_replay_enumeration_owed() - with redirect_stdout(StringIO()): - server.replay_pending_block_candidates() - after = { - str(row["block_hash"]).lower() - for row in rig.ledger.pending_block_candidate_rows(limit=candidates + 1) - } - return before - after, server, winner - - def _compare(self, candidates: int, view: str) -> set[str]: - oracle, winner = self._oracle(candidates, view) - selected, server, winner_again = self._selector(candidates, view) - self.assertEqual(winner, winner_again) - self.assertEqual( - selected - oracle, - set(), - f"selector abandoned {len(selected - oracle)} hash(es) the shipped " - f"per-row path refused ({candidates} candidates, {view} view)", - ) - # The baseline storm is the exact-agreement case: every sibling of the - # decided block is superseded and carries no offer evidence. - self.assertEqual(oracle - selected, set()) - self.assertNotIn(winner, selected) - self.assertEqual( - server.block_candidate_abandoned_counts, - {PRISM_BLOCK_CANDIDATE_COLLAPSE_REASON: len(selected)}, - ) - return selected - - def test_efficacy_and_agreement_at_one_hundred_candidates(self) -> None: - for view in ("restart", "live"): - with self.subTest(view=view): - selected = self._compare(100, view) - # Every candidate but the decided winner; the storm's whole - # point is that the shipped path pays 99 node offers for it. - self.assertGreaterEqual(len(selected), 90) - self.assertEqual(len(selected), 99) - - def test_agreement_at_the_observed_testnet_storm(self) -> None: - for view in ("restart", "live"): - with self.subTest(view=view): - selected = self._compare(3_120, view) - self.assertEqual(len(selected), 3_119) - - def test_the_storm_costs_a_bounded_number_of_chain_reads(self) -> None: - selected, server, _winner = self._selector(3_120, "restart") - self.assertEqual(len(selected), 3_119) - pages = -(-3_120 // MAX_BLOCK_REPLAY_ENUMERATION_ROWS) - calls = server.rpc.calls - # Tip reads are page-scope: two per page (selection plus - # revalidation) and one tip-height read each. The decided height and - # its occupant are walk-scope: selection reads them once for the - # whole walk and every later page answers from the walk's cache, so - # the only further active-block reads are the one each page's - # pre-write revalidation makes under its own fresh budget. - self.assertEqual(calls["getbestblockhash"], 2 * pages) - self.assertEqual(calls["getblockcount"], pages) - self.assertEqual(calls["getblockhash"], 1 + pages) - self.assertEqual(calls["getblockheader"], 1) - self.assertNotIn("submitblock", calls) - - def test_no_perturbation_ever_produces_a_selector_only_hash(self) -> None: - candidates = 40 - target = _hash(7) - for view in ("restart", "live"): - for kind in PERTURBATIONS: - with self.subTest(view=view, perturbation=kind): - rig = CandidateStormRig(candidates=candidates) - rig.seed_live() - if view == "restart": - rig.restart_and_enumerate() - server = ( - rig.restarted_server - if view == "restart" - else rig.live_server - ) - winner = rig.decide_height() - with redirect_stdout(StringIO()): - undo = rig.perturb(server, kind, target) - report = rig.drain_per_row(server, stop_before_hash=winner) - oracle = set(report.abandoned_hashes) - if undo is not None: - undo() - selected, _server, _winner = self._selector( - candidates, - view, - perturbation=kind, - target=target, - ) - self.assertEqual( - selected - oracle, - set(), - f"{kind} produced a selector-only hash in the {view} view", - ) - self.assertNotIn(target, selected) - self.assertLessEqual(len(oracle - selected), 1) - - -if __name__ == "__main__": - unittest.main() diff --git a/tests/test_prism_block_candidates.py b/tests/test_prism_block_candidates.py deleted file mode 100644 index ef9f6b120..000000000 --- a/tests/test_prism_block_candidates.py +++ /dev/null @@ -1,12464 +0,0 @@ -#!/usr/bin/env python3 -"""Focused PRISM coordinator block candidates tests.""" -# ruff: noqa: F403, F405 - -from __future__ import annotations - -import dataclasses -import io -import unittest -from unittest import mock -from tests.prism_vardiff_test_support import * -from lab.prism.coordinator_config import LifecycleConfig -from lab.prism.audit_artifacts import ( - AuditArtifactStore, - AuditPublicationIdentity, - RetentionResult, -) -from lab.prism.block_candidates import ( - MAX_BLOCK_REPLAY_ENUMERATION_ROWS, - MAX_BLOCK_SUBMITTER_STUCK_CALL_WORKERS, - MAX_PENDING_BLOCK_CANDIDATES, - BlockCandidateAttemptResult, - BlockCandidateRunResult, - _BlockCandidateNodeSubmission, - _BlockSubmitterLedgerCall, - block_candidate_from_intent, - block_candidate_intent, -) -from tests.prism_coordinator_test_support import durable_candidate_row -from lab.prism.bundle_compiler import BundleCompiler -from lab.prism.prism_coordinator import PrismCoordinator - - -_compat_verified_audit_report = verified_audit_report - - -def configure_temporary_audit_root( - test_case: unittest.TestCase, - server: PrismCoordinator, -) -> None: - temporary = tempfile.TemporaryDirectory() - server.audit_dir = Path(temporary.name) / "audit" - server.evidence_path = Path(temporary.name) / "state" / "evidence.json" - - def cleanup() -> None: - store = server.__dict__.get("_audit_artifact_store") - if isinstance(store, AuditArtifactStore): - store.close() - temporary.cleanup() - - test_case.addCleanup(cleanup) - - -def verified_audit_report( - coinbase_tx_hex: str = "c0ffee", - block_height: int = 10, -) -> dict[str, object]: - report = _compat_verified_audit_report(coinbase_tx_hex) - report["schema"] = "qbit.prism.audit-verification-report.v1" - report["block_height"] = block_height - report["coinbase_value_sats"] = 50_00000000 - return report - - -class PrismCoordinatorVardiffTests(unittest.TestCase): - def test_audit_store_lazy_adopts_compatibility_fields_before_and_after_construction(self) -> None: - with tempfile.TemporaryDirectory() as tmp: - base = Path(tmp) - server = PrismCoordinator.__new__(PrismCoordinator) - server.audit_dir = base / "audit-a" - server.evidence_path = base / "state-a" / "evidence.json" - server.audit_live_bundle_retention = 3 - server.audit_candidate_retention_seconds = 7 - server.audit_share_segment_size = 11 - first = server._ensure_audit_artifact_store() - self.assertEqual(first.root, server.audit_dir.resolve()) - self.assertEqual(first.evidence_path, server.evidence_path.resolve()) - self.assertEqual(first.live_bundle_retention, 3) - self.assertEqual(first.candidate_retention_seconds, 7) - self.assertEqual(first.share_segment_size, 11) - - server.audit_dir = base / "audit-b" - server.evidence_path = base / "state-b" / "evidence.json" - server.audit_live_bundle_retention = 5 - server.audit_candidate_retention_seconds = 13 - server.audit_share_segment_size = 17 - second = server._ensure_audit_artifact_store() - self.assertIs(second, first) - self.assertEqual(second.root, server.audit_dir.resolve()) - self.assertEqual(second.evidence_path, server.evidence_path.resolve()) - self.assertEqual(second.live_bundle_retention, 5) - self.assertEqual(second.candidate_retention_seconds, 13) - self.assertEqual(second.share_segment_size, 17) - - candidate = second.issue_candidate(block_hash="aa" * 32) - server.audit_dir = base / "audit-c" - with self.assertRaisesRegex(RuntimeError, "candidates are active"): - server._ensure_audit_artifact_store() - self.assertIs(server.__dict__["_audit_artifact_store"], first) - self.assertEqual(first.root, (base / "audit-b").resolve()) - first.discard_candidate(candidate) - - def test_coordinator_latest_evidence_seed_is_stable_and_defensive_before_and_after_store(self) -> None: - with tempfile.TemporaryDirectory() as tmp: - server = PrismCoordinator.__new__(PrismCoordinator) - seed = {"nested": {"value": 1}} - server.latest_evidence = seed - seed["nested"]["value"] = 2 - before = server.latest_evidence - self.assertEqual(before, {"nested": {"value": 1}}) - assert before is not None - before["nested"]["value"] = 3 - self.assertEqual(server.latest_evidence, {"nested": {"value": 1}}) - - root = Path(tmp) - server.audit_dir = root / "audit" - server.evidence_path = root / "state" / "evidence.json" - store = server._ensure_audit_artifact_store() - self.assertNotIn("_audit_latest_evidence_seed", server.__dict__) - self.assertEqual(server.latest_evidence_payload(), {"nested": {"value": 1}}) - snapshot = server.latest_evidence_payload() - assert snapshot is not None - snapshot["nested"]["value"] = 4 # type: ignore[index] - self.assertEqual(server.latest_evidence_payload(), {"nested": {"value": 1}}) - server.latest_evidence = None - self.assertIsNone(server.latest_evidence_payload()) - server.latest_evidence = {"after": {"value": 5}} - self.assertEqual(server.latest_evidence_payload(), {"after": {"value": 5}}) - self.assertIs(server._ensure_audit_artifact_store(), store) - - def test_coordinator_audit_static_and_instance_facades_preserve_contract(self) -> None: - server = PrismCoordinator.__new__(PrismCoordinator) - sentinel = mock.Mock() - sentinel.metrics_snapshot.return_value = {"scan_error": 0} - sentinel.prune_best_effort.return_value = RetentionResult(live_removed=1) - sentinel.verify_bundle.return_value = {"verified": True} - server._ensure_audit_artifact_store = lambda: sentinel # type: ignore[method-assign] - - self.assertEqual( - PrismCoordinator.audit_artifact_kind( - f"prism-live-audit-bundle-1-{'aa' * 32}.json" - ), - "live_bundle", - ) - self.assertEqual(server.audit_artifact_metrics(), {"scan_error": 0}) - keep = Path("keep.json") - server.prune_audit_artifacts(keep_live_path=keep) - sentinel.prune_best_effort.assert_called_once_with(keep_live_path=keep) - result = PrismCoordinator.verify_bundle( - server, - Path("bundle.json"), - "00", - "11" * 32, - expected_coinbase_value_sats=5, - expected_block_height=6, - ) - self.assertEqual(result, {"verified": True}) - sentinel.verify_bundle.assert_called_once_with( - Path("bundle.json"), - "00", - "11" * 32, - expected_coinbase_value_sats=5, - expected_block_height=6, - ) - server.ledger_writer_public_key_hex = "22" * 32 - self.assertEqual( - server.trusted_ledger_writer_public_key_hex({}), - "22" * 32, - ) - - with tempfile.TemporaryDirectory() as tmp: - candidate = Path(tmp) / "candidate.json" - candidate.write_bytes(b"canonical") - digest = hashlib.sha256(b"canonical").hexdigest() - self.assertEqual( - PrismCoordinator.verified_canonical_bundle_path( - candidate, - {"audit_bundle_sha256_hex": digest.upper()}, - ), - candidate, - ) - - def test_candidate_verifier_override_is_resolved_after_store_construction(self) -> None: - server, state, ledger = submit_coordinator() - server.max_blocks = 10 - server.stop_after_block = False - with tempfile.TemporaryDirectory() as tmp: - server.audit_dir = Path(tmp) - server.evidence_path = Path(tmp) / "evidence.json" - server.ledger_writer_public_key_hex = "aa" * 32 - store = server._ensure_audit_artifact_store() - calls: list[str] = [] - - def verifier_a(*_args: object, **_kwargs: object) -> dict[str, object]: - calls.append("a") - return verified_audit_report() - - def verifier_b(*_args: object, **_kwargs: object) -> dict[str, object]: - calls.append("b") - return verified_audit_report() - - server.verify_bundle = verifier_a # type: ignore[method-assign] - server.verify_bundle = verifier_b # type: ignore[method-assign] - block_hash = "91" * 32 - server.rpc = SubmitRpc( - tip="00" * 32, - block_hash=block_hash, - ledger=ledger, - ) - server.build_audit_bundle = ( # type: ignore[method-assign] - lambda **_kwargs: verified_block_bundle() - ) - submission = SimpleNamespace( - coinbase_tx_hex="c0ffee", - block_hash_hex=block_hash, - block_hex="00", - ) - self.assertTrue( - server.submit_block_candidate( - block_candidate(server, state, submission) - ) - ) - self.assertIs(server._ensure_audit_artifact_store(), store) - self.assertEqual(calls, ["b"]) - - direct_server, direct_state, direct_ledger = submit_coordinator() - direct_server.max_blocks = 10 - direct_server.stop_after_block = False - direct_server.audit_dir = Path(tmp) / "direct-a1" - direct_server.evidence_path = Path(tmp) / "direct-a1-evidence.json" - direct_server.ledger_writer_public_key_hex = "aa" * 32 - direct_store = direct_server._ensure_audit_artifact_store() - direct_calls: list[str] = [] - - def direct_a1_verifier( - *_args: object, - **_kwargs: object, - ) -> dict[str, object]: - direct_calls.append("a1") - return verified_audit_report() - - direct_store.verify_bundle = direct_a1_verifier # type: ignore[method-assign] - self.assertNotIn("verify_bundle", direct_server.__dict__) - direct_hash = "93" * 32 - direct_server.rpc = SubmitRpc( - tip="00" * 32, - block_hash=direct_hash, - ledger=direct_ledger, - ) - direct_server.build_audit_bundle = ( # type: ignore[method-assign] - lambda **_kwargs: verified_block_bundle() - ) - direct_submission = SimpleNamespace( - coinbase_tx_hex="c0ffee", - block_hash_hex=direct_hash, - block_hex="00", - ) - self.assertTrue( - direct_server.submit_block_candidate( - block_candidate( - direct_server, - direct_state, - direct_submission, - ) - ) - ) - self.assertEqual(direct_calls, ["a1"]) - - def test_audit_publication_occurs_after_durable_confirm_and_before_success_tail(self) -> None: - server, state, ledger = submit_coordinator() - events: list[str] = [] - with tempfile.TemporaryDirectory() as tmp: - server.audit_dir = Path(tmp) - server.evidence_path = Path(tmp) / "evidence.json" - server.ledger_writer_public_key_hex = "aa" * 32 - store = server._ensure_audit_artifact_store() - payout_service = server._ensure_payout_state_service() - real_persist = ledger.persist_accepted_block - real_confirm = ledger.confirm_accepted_block - real_floor = ledger.audit_publication_sequence_floor - real_identity = server._audit_publication_identity - real_publish = store.publish_success - real_shutdown = server.request_shutdown - - def verify(*_args: object, **_kwargs: object) -> dict[str, object]: - events.append("verify") - return verified_audit_report() - - def persist(**kwargs: object) -> dict[str, object]: - events.append("persist") - return real_persist(**kwargs) - - def confirm(**kwargs: object) -> dict[str, object]: - self.assertTrue( - payout_service._payout_balance_mutation_lock._is_owned(), # type: ignore[attr-defined] - ) - self.assertEqual(store._publication_guard_owner, threading.get_ident()) - events.append("confirm") - return real_confirm(**kwargs) - - def publication_identity(**kwargs: object) -> AuditPublicationIdentity: - self.assertTrue( - payout_service._payout_balance_mutation_lock._is_owned(), # type: ignore[attr-defined] - ) - self.assertEqual(store._publication_guard_owner, threading.get_ident()) - events.append("publication_identity") - return real_identity(**kwargs) - - def publication_floor() -> int: - self.assertTrue( - payout_service._payout_balance_mutation_lock._is_owned(), # type: ignore[attr-defined] - ) - self.assertEqual(store._publication_guard_owner, threading.get_ident()) - events.append("publication_floor") - return real_floor() - - def publish(**kwargs: object) -> object: - self.assertTrue( - payout_service._payout_balance_mutation_lock._is_owned(), # type: ignore[attr-defined] - ) - self.assertEqual(store._publication_guard_owner, threading.get_ident()) - events.append("publish_success") - return real_publish(**kwargs) - - def terminal_success() -> None: - events.append("terminal_success") - real_shutdown() - - server.verify_bundle = verify # type: ignore[method-assign] - ledger.persist_accepted_block = persist # type: ignore[method-assign] - ledger.confirm_accepted_block = confirm # type: ignore[method-assign] - ledger.audit_publication_sequence_floor = publication_floor # type: ignore[method-assign] - server._audit_publication_identity = publication_identity # type: ignore[method-assign] - store.publish_success = publish # type: ignore[method-assign] - server.request_shutdown = terminal_success # type: ignore[method-assign] - block_hash = "92" * 32 - server.rpc = SubmitRpc( - tip="00" * 32, - block_hash=block_hash, - ledger=ledger, - ) - server.build_audit_bundle = ( # type: ignore[method-assign] - lambda **_kwargs: verified_block_bundle() - ) - submission = SimpleNamespace( - coinbase_tx_hex="c0ffee", - block_hash_hex=block_hash, - block_hex="00", - ) - self.assertTrue( - server.submit_block_candidate( - block_candidate(server, state, submission) - ) - ) - self.assertEqual( - events, - [ - "verify", - "persist", - "confirm", - "publication_identity", - "publication_floor", - "publish_success", - "terminal_success", - ], - ) - - def test_audit_publication_failure_after_confirm_retries_same_ordinal_once(self) -> None: - server, state, ledger = submit_coordinator() - block_hash = "94" * 32 - with tempfile.TemporaryDirectory() as tmp: - server.audit_dir = Path(tmp) - server.evidence_path = Path(tmp) / "evidence.json" - server.ledger_writer_public_key_hex = "aa" * 32 - store = server._ensure_audit_artifact_store() - server.rpc = SubmitRpc( - tip="00" * 32, - block_hash=block_hash, - ledger=ledger, - ) - server.build_audit_bundle = ( # type: ignore[method-assign] - lambda **_kwargs: verified_block_bundle() - ) - server.verify_bundle = ( # type: ignore[method-assign] - lambda *_args, **_kwargs: verified_audit_report() - ) - real_confirm = ledger.confirm_accepted_block - real_publish = store.publish_success - real_shutdown = server.request_shutdown - confirmation_sequences: list[int] = [] - publication_sequences: list[int] = [] - terminal_successes: list[str] = [] - publication_attempts = 0 - - def confirm(**kwargs: object) -> dict[str, object]: - result = real_confirm(**kwargs) - result["audit_publication_sequence"] = 7 - confirmation_sequences.append(7) - return result - - def publish(**kwargs: object) -> object: - nonlocal publication_attempts - publication_attempts += 1 - identity = kwargs["identity"] - assert isinstance(identity, AuditPublicationIdentity) - publication_sequences.append(identity.sequence) - if publication_attempts == 1: - raise RuntimeError("injected evidence publication failure") - return real_publish(**kwargs) - - def terminal_success() -> None: - terminal_successes.append("success") - real_shutdown() - - ledger.confirm_accepted_block = confirm # type: ignore[method-assign] - ledger.audit_publication_sequence_floor = lambda: 7 # type: ignore[method-assign] - store.publish_success = publish # type: ignore[method-assign] - server.request_shutdown = terminal_success # type: ignore[method-assign] - submission = SimpleNamespace( - coinbase_tx_hex="c0ffee", - block_hash_hex=block_hash, - block_hex="00", - ) - candidate = block_candidate(server, state, submission) - - with self.assertRaisesRegex( - RuntimeError, - "injected evidence publication failure", - ): - server.submit_block_candidate(candidate) - self.assertEqual(confirmation_sequences, [7]) - self.assertEqual(publication_sequences, [7]) - self.assertIsNone(store.latest_evidence()) - self.assertEqual(server.accepted_block_count, 0) - self.assertEqual(terminal_successes, []) - - assert isinstance(server.rpc, SubmitRpc) - server.rpc.tip = block_hash - self.assertTrue(server.submit_block_candidate(candidate)) - self.assertEqual(confirmation_sequences, [7, 7]) - self.assertEqual(publication_sequences, [7, 7]) - self.assertEqual(server.accepted_block_count, 1) - self.assertEqual(terminal_successes, ["success"]) - latest = store.latest_evidence() - self.assertIsNotNone(latest) - assert latest is not None - self.assertEqual( - latest["audit_publication_identity"]["sequence"], # type: ignore[index] - 7, - ) - - def test_publication_floor_failure_or_mismatch_never_publishes_evidence(self) -> None: - cases = ( - ("query_failure", RuntimeError("floor query failed"), "floor query failed"), - ("below_identity", 0, "exceeds"), - ("above_identity", 2, "behind"), - ("invalid_bool", True, "floor sequence"), - ) - for name, floor_result, error_pattern in cases: - with self.subTest(name=name), tempfile.TemporaryDirectory() as tmp: - server, state, ledger = submit_coordinator() - block_hash = "95" * 32 - server.audit_dir = Path(tmp) - server.evidence_path = Path(tmp) / "evidence.json" - server.ledger_writer_public_key_hex = "aa" * 32 - store = server._ensure_audit_artifact_store() - server.rpc = SubmitRpc( - tip="00" * 32, - block_hash=block_hash, - ledger=ledger, - ) - server.build_audit_bundle = ( # type: ignore[method-assign] - lambda **_kwargs: verified_block_bundle() - ) - server.verify_bundle = ( # type: ignore[method-assign] - lambda *_args, **_kwargs: verified_audit_report() - ) - publish_calls = 0 - real_publish = store.publish_success - - def publication_floor() -> int: - if isinstance(floor_result, BaseException): - raise floor_result - return floor_result # type: ignore[return-value] - - def publish(**kwargs: object) -> object: - nonlocal publish_calls - publish_calls += 1 - return real_publish(**kwargs) - - ledger.audit_publication_sequence_floor = publication_floor # type: ignore[method-assign] - store.publish_success = publish # type: ignore[method-assign] - submission = SimpleNamespace( - coinbase_tx_hex="c0ffee", - block_hash_hex=block_hash, - block_hex="00", - ) - with self.assertRaisesRegex( - (RuntimeError, ValueError), - error_pattern, - ): - server.submit_block_candidate( - block_candidate(server, state, submission) - ) - self.assertEqual( - publish_calls, - 0 if name == "query_failure" else 1, - ) - self.assertIsNone(store.latest_evidence()) - self.assertEqual(server.accepted_block_count, 0) - - def test_make_ledger_preserves_single_a1_store_identity(self) -> None: - with tempfile.TemporaryDirectory() as tmp: - server = PrismCoordinator.__new__(PrismCoordinator) - server.audit_dir = Path(tmp) / "audit-a" - server.evidence_path = Path(tmp) / "state-a" / "evidence.json" - store = server._ensure_audit_artifact_store() - ledger = SimpleNamespace() - with mock.patch.dict( - os.environ, - {"PRISM_POSTGRES_PSQL_COMMAND": "psql example"}, - clear=True, - ), mock.patch( - "lab.prism.prism_coordinator.PsqlShareLedger", - return_value=ledger, - ) as constructor: - self.assertIs(server.make_ledger(), ledger) - self.assertIs( - constructor.call_args.kwargs["audit_artifact_store"], - store, - ) - server.audit_dir = Path(tmp) / "audit-b" - server.evidence_path = Path(tmp) / "state-b" / "evidence.json" - self.assertIs(server._ensure_audit_artifact_store(), store) - self.assertEqual(store.root, server.audit_dir.resolve()) - - memory_server = PrismCoordinator.__new__(PrismCoordinator) - with mock.patch.dict( - os.environ, - {"PRISM_ALLOW_MEMORY_LEDGER": "1"}, - clear=True, - ): - self.assertIsInstance( - memory_server.make_ledger(), - SingleWriterShareLedger, - ) - self.assertNotIn("_audit_artifact_store", memory_server.__dict__) - - def test_legacy_evidence_upgrade_requires_exact_durable_pool_block_proof(self) -> None: - legacy_identity = AuditPublicationIdentity(0, 10, "aa" * 32) - - for maturity_state in ("immature", "mature"): - with self.subTest(valid_maturity=maturity_state): - server = coordinator() - store = mock.MagicMock() - store.legacy_evidence_identity.return_value = legacy_identity - server._ensure_audit_artifact_store = lambda: store # type: ignore[method-assign] - server.ledger = SimpleNamespace( - audit_publication_sequence_floor=lambda: 7, - pool_block_state=lambda **_kwargs: { - "block_hash": legacy_identity.block_hash, - "block_height": legacy_identity.block_height, - "chain_state": "confirmed", - "maturity_state": maturity_state, - "audit_publication_sequence": 7, - } - ) - server._upgrade_legacy_audit_evidence() - store.adopt_legacy_publication_identity.assert_called_once_with( - AuditPublicationIdentity(7, 10, legacy_identity.block_hash), - publication_floor_sequence=7, - ) - store.invalidate_unprovable_legacy_evidence.assert_not_called() - - invalid_states = ( - None, - { - "block_height": 10, - "chain_state": "confirmed", - "maturity_state": "immature", - "audit_publication_sequence": 7, - }, - { - "block_hash": legacy_identity.block_hash.upper(), - "block_height": 10, - "chain_state": "confirmed", - "maturity_state": "immature", - "audit_publication_sequence": 7, - }, - { - "block_hash": legacy_identity.block_hash, - "block_height": "10", - "chain_state": "confirmed", - "maturity_state": "immature", - "audit_publication_sequence": 7, - }, - { - "block_hash": legacy_identity.block_hash, - "block_height": True, - "chain_state": "confirmed", - "maturity_state": "immature", - "audit_publication_sequence": 7, - }, - { - "block_height": 10, - "chain_state": "inactive", - "maturity_state": "immature", - "audit_publication_sequence": 7, - }, - { - "block_height": 10, - "chain_state": "confirmed", - "maturity_state": "reversed", - "audit_publication_sequence": 7, - }, - { - "block_height": 10, - "chain_state": "confirmed", - "maturity_state": "unknown", - "audit_publication_sequence": 7, - }, - { - "block_height": 11, - "chain_state": "confirmed", - "maturity_state": "immature", - "audit_publication_sequence": 7, - }, - { - "block_height": 10, - "chain_state": "confirmed", - "maturity_state": "immature", - "audit_publication_sequence": True, - }, - ) - for state in invalid_states: - with self.subTest(invalid_state=state): - server = coordinator() - store = mock.MagicMock() - store.legacy_evidence_identity.return_value = legacy_identity - server._ensure_audit_artifact_store = lambda: store # type: ignore[method-assign] - server.ledger = SimpleNamespace( - pool_block_state=lambda **_kwargs: state - ) - server._upgrade_legacy_audit_evidence() - store.invalidate_unprovable_legacy_evidence.assert_called_once_with() - store.adopt_legacy_publication_identity.assert_not_called() - - server = coordinator() - store = mock.MagicMock() - store.legacy_evidence_identity.return_value = legacy_identity - server._ensure_audit_artifact_store = lambda: store # type: ignore[method-assign] - server.ledger = SingleWriterShareLedger() - server._upgrade_legacy_audit_evidence() - store.invalidate_unprovable_legacy_evidence.assert_called_once_with() - - def test_build_audit_bundle_passes_pool_fee_policy_to_cli_payload(self) -> None: - server = coordinator() - server.rpc = AddressRpc(valid_address="tq1fee", script_byte="99") - server.signing_seed_hex = "42" * 32 - server.ledger_attestation_signing_seed_hex = "43" * 32 - captured: dict[str, object] = {} - - with patch.dict( - os.environ, - { - "PRISM_POOL_FEE_ENABLED": "1", - "PRISM_POOL_FEE_BPS": "125", - "PRISM_POOL_FEE_ADDRESS": "tq1fee", - }, - clear=True, - ), patch( - "lab.prism.prism_coordinator.subprocess.Popen", - fake_audit_bundle_popen(captured), - ): - bundle = server.build_audit_bundle( - shares=[], - found_block={"block_height": 10, "coinbase_value_sats": 50_00000000}, - prior_balances=[], - coinbase_script_sig_suffix_hex="00", - ) - - self.assertEqual(bundle, {"ok": True}) - self.assertEqual(captured["payload"]["payout_policy"]["pool_fee_policy"]["fee_bps"], 125) - self.assertEqual( - captured["payload"]["payout_policy"]["pool_fee_policy"]["p2mr_program_hex"], - "99" * 32, - ) - - def test_build_audit_bundle_passes_ctv_settlement_config_to_cli_payload(self) -> None: - server = coordinator() - server.signing_seed_hex = "42" * 32 - server.ledger_attestation_signing_seed_hex = "43" * 32 - captured: dict[str, object] = {} - - with patch.dict( - os.environ, - { - "PRISM_CTV_SETTLEMENT_ENABLED": "1", - "PRISM_DIRECT_COINBASE_PAYOUT_FLOOR_BITS": "10485760", - "PRISM_MAX_COINBASE_SETTLEMENT_OUTPUTS": "16", - "PRISM_MAX_DIRECT_COINBASE_OUTPUTS": "12", - "PRISM_MAX_CTV_FANOUT_RECIPIENTS_PER_TRANSACTION": "1000", - "PRISM_CTV_FANOUT_FEE_MARKET_RATE_BITS_PER_1000_WEIGHT": "25", - "PRISM_CTV_FANOUT_FEE_PREMIUM_BPS": "12000", - }, - clear=True, - ), patch( - "lab.prism.prism_coordinator.subprocess.Popen", - fake_audit_bundle_popen(captured), - ): - bundle = server.build_audit_bundle( - shares=[], - found_block={"block_height": 10, "coinbase_value_sats": 50_00000000}, - prior_balances=[], - coinbase_script_sig_suffix_hex="00", - ) - - self.assertEqual(bundle, {"ok": True}) - self.assertEqual( - captured["payload"]["ctv_settlement"], - { - "direct_floor_sats": 10_485_760, - "config": { - "max_coinbase_settlement_outputs": 16, - "max_direct_coinbase_outputs": 12, - "max_fanout_recipients_per_transaction": 1000, - "reserved_coinbase_outputs": 0, - }, - "fanout_fee_rate_policy": { - "market_fee_rate_sats_per_1000_weight": 25, - "premium_bps": 12_000, - }, - }, - ) - - def test_build_audit_bundle_preserves_exact_canonical_output_file(self) -> None: - server = coordinator() - server.signing_seed_hex = "42" * 32 - server.ledger_attestation_signing_seed_hex = "43" * 32 - canonical_bytes = b'{"ok":true,"nested":[1,2]}' - captured: dict[str, object] = {} - - with tempfile.TemporaryDirectory() as tmp, patch( - "lab.prism.prism_coordinator.subprocess.Popen", - fake_audit_bundle_popen( - captured, - output_text=canonical_bytes.decode("utf-8"), - ), - ): - output_path = Path(tmp) / "candidate.audit.json" - bundle = server.build_audit_bundle( - shares=[], - found_block={"block_height": 10, "coinbase_value_sats": 50_00000000}, - prior_balances=[], - coinbase_script_sig_suffix_hex="00", - canonical_output_path=output_path, - ) - - self.assertEqual(bundle, {"ok": True, "nested": [1, 2]}) - self.assertEqual(output_path.read_bytes(), canonical_bytes) - self.assertIn("--canonical-output", captured["cmd"]) - self.assertEqual(captured["payload"]["shares"], []) - - def test_build_audit_bundle_transfers_the_exact_open_output_inode(self) -> None: - server = coordinator() - server.signing_seed_hex = "42" * 32 - server.ledger_attestation_signing_seed_hex = "43" * 32 - captured: dict[str, object] = {} - adopted: list[tuple[Path, int, int]] = [] - - def adopt(path: Path, value: os.stat_result) -> None: - current = path.stat() - self.assertEqual((value.st_dev, value.st_ino), (current.st_dev, current.st_ino)) - adopted.append((path, value.st_dev, value.st_ino)) - - with tempfile.TemporaryDirectory() as tmp, patch( - "lab.prism.bundle_compiler.subprocess.Popen", - fake_audit_bundle_popen(captured), - ): - output_path = Path(tmp) / "candidate.audit.json" - server.build_audit_bundle( - shares=[], - found_block={"block_height": 10, "coinbase_value_sats": 50_00000000}, - prior_balances=[], - coinbase_script_sig_suffix_hex="00", - canonical_output_path=output_path, - canonical_output_adopter=adopt, - ) - - self.assertEqual(len(adopted), 1) - self.assertEqual(adopted[0][0], output_path) - self.assertEqual(adopted[0][1:], (output_path.stat().st_dev, output_path.stat().st_ino)) - - def test_build_audit_bundle_pinned_parent_never_recreates_stale_path(self) -> None: - server = coordinator() - server.signing_seed_hex = "42" * 32 - server.ledger_attestation_signing_seed_hex = "43" * 32 - captured: dict[str, object] = {} - - with tempfile.TemporaryDirectory() as tmp, patch( - "lab.prism.bundle_compiler.subprocess.Popen", - fake_audit_bundle_popen(captured), - ): - base = Path(tmp) - root = base / "audit" - root.mkdir() - pinned = base / "audit-pinned" - output_path = root / "candidate.audit.json" - parent_fd = os.open( - root, - os.O_RDONLY | getattr(os, "O_DIRECTORY", 0), - ) - root.rename(pinned) - - def reject_stale_authority( - path: Path, - value: os.stat_result, - ) -> None: - self.assertEqual(path, output_path) - self.assertFalse(root.exists()) - pinned_output = pinned / output_path.name - self.assertTrue(pinned_output.exists()) - self.assertEqual( - (value.st_dev, value.st_ino), - (pinned_output.stat().st_dev, pinned_output.stat().st_ino), - ) - raise RuntimeError("stale A1 authority") - - try: - with self.assertRaisesRegex(RuntimeError, "stale A1 authority"): - server.build_audit_bundle( - shares=[], - found_block={ - "block_height": 10, - "coinbase_value_sats": 50_00000000, - }, - prior_balances=[], - coinbase_script_sig_suffix_hex="00", - canonical_output_path=output_path, - canonical_output_parent_fd=parent_fd, - canonical_output_adopter=reject_stale_authority, - ) - finally: - os.close(parent_fd) - - self.assertFalse(root.exists()) - self.assertFalse((pinned / output_path.name).exists()) - - def test_build_audit_bundle_adopter_failure_preserves_path_replacement(self) -> None: - server = coordinator() - server.signing_seed_hex = "42" * 32 - server.ledger_attestation_signing_seed_hex = "43" * 32 - captured: dict[str, object] = {} - - def replace_then_reject(path: Path, _value: os.stat_result) -> None: - path.unlink() - path.write_bytes(b"competitor") - raise RuntimeError("transfer rejected") - - with tempfile.TemporaryDirectory() as tmp, patch( - "lab.prism.bundle_compiler.subprocess.Popen", - fake_audit_bundle_popen(captured), - ): - output_path = Path(tmp) / "candidate.audit.json" - with self.assertRaisesRegex(RuntimeError, "transfer rejected"): - server.build_audit_bundle( - shares=[], - found_block={"block_height": 10, "coinbase_value_sats": 50_00000000}, - prior_balances=[], - coinbase_script_sig_suffix_hex="00", - canonical_output_path=output_path, - canonical_output_adopter=replace_then_reject, - ) - - self.assertEqual(output_path.read_bytes(), b"competitor") - - def test_builder_cleanup_never_moves_an_existing_replacement(self) -> None: - with tempfile.TemporaryDirectory() as tmp: - root = Path(tmp) - path = root / "candidate.audit.json" - path.write_bytes(b"owned-a") - identity = path.stat() - path.unlink() - path.write_bytes(b"competitor-b") - with patch( - "os.replace", - side_effect=AssertionError("replacement must not move"), - ): - BundleCompiler._remove_created_output_if_same(path, identity) - - self.assertEqual(path.read_bytes(), b"competitor-b") - quarantines = list(root.glob(".candidate.audit.json.*.cleanup")) - self.assertEqual(quarantines, []) - - def test_builder_cleanup_never_relocates_nonregular_replacement(self) -> None: - with tempfile.TemporaryDirectory() as tmp: - root = Path(tmp) - path = root / "candidate.audit.json" - path.write_bytes(b"owned") - identity = path.stat() - path.unlink() - path.mkdir() - - BundleCompiler._remove_created_output_if_same(path, identity) - - self.assertTrue(path.is_dir()) - self.assertEqual( - list(root.glob(".candidate.audit.json.*.cleanup")), - [], - ) - - path.rmdir() - target = root / "operator" - target.write_bytes(b"operator") - path.symlink_to(target) - BundleCompiler._remove_created_output_if_same(path, identity) - self.assertTrue(path.is_symlink()) - self.assertEqual(path.read_bytes(), b"operator") - - def test_build_audit_bundle_summary_only_requests_and_parses_job_summary(self) -> None: - server = coordinator() - server.signing_seed_hex = "42" * 32 - server.ledger_attestation_signing_seed_hex = "43" * 32 - summary = { - "found_block": { - "block_height": 10, - "coinbase_value_sats": 50_00000000, - }, - "signed_coinbase_manifest": { - "manifest": {"coinbase_tx_hex": "c0ffee"}, - "signature": {"signature_hex": "11" * 64}, - }, - } - captured: dict[str, object] = {} - - with patch( - "lab.prism.prism_coordinator.subprocess.Popen", - fake_audit_bundle_popen( - captured, - output_text=json.dumps(summary, separators=(",", ":")), - ), - ), patch.object( - Path, - "open", - side_effect=AssertionError("summary-only build must not open an output path"), - ): - bundle = server.build_audit_bundle( - shares=[], - found_block={"block_height": 10, "coinbase_value_sats": 50_00000000}, - prior_balances=[], - coinbase_script_sig_suffix_hex="00", - summary_only=True, - ) - - self.assertEqual(bundle, summary) - self.assertEqual(set(bundle), {"found_block", "signed_coinbase_manifest"}) - self.assertIn("--job-summary-output", captured["cmd"]) - self.assertNotIn("--canonical-output", captured["cmd"]) - - def test_build_audit_bundle_removes_partial_output_after_builder_failure(self) -> None: - server = coordinator() - server.signing_seed_hex = "42" * 32 - server.ledger_attestation_signing_seed_hex = "43" * 32 - - captured: dict[str, object] = {} - with tempfile.TemporaryDirectory() as tmp: - output_path = Path(tmp) / "candidate.audit.json" - with patch( - "lab.prism.prism_coordinator.subprocess.Popen", - fake_audit_bundle_popen( - captured, - output_text='{"partial":', - returncode=9, - stderr_text="synthetic builder failure", - ), - ): - with self.assertRaisesRegex(RuntimeError, "synthetic builder failure"): - server.build_audit_bundle( - shares=[], - found_block={"block_height": 10, "coinbase_value_sats": 50_00000000}, - prior_balances=[], - coinbase_script_sig_suffix_hex="00", - canonical_output_path=output_path, - ) - self.assertFalse(output_path.exists()) - self.assertEqual(server.job_build_worker_counts["crashes"], 1) - - with patch( - "lab.prism.prism_coordinator.subprocess.Popen", - fake_audit_bundle_popen(captured), - ): - recovered = server.build_audit_bundle( - shares=[], - found_block={"block_height": 10, "coinbase_value_sats": 50_00000000}, - prior_balances=[], - coinbase_script_sig_suffix_hex="00", - ) - - self.assertEqual(recovered, {"ok": True}) - self.assertEqual(server.job_build_worker_counts["restarts"], 1) - - - def test_build_audit_bundle_removes_owned_output_after_parse_failure(self) -> None: - server = coordinator() - server.signing_seed_hex = "42" * 32 - server.ledger_attestation_signing_seed_hex = "43" * 32 - captured: dict[str, object] = {} - - with tempfile.TemporaryDirectory() as tmp, patch( - "lab.prism.bundle_compiler.subprocess.Popen", - fake_audit_bundle_popen(captured, output_text='{"partial":'), - ): - output_path = Path(tmp) / "candidate.audit.json" - with self.assertRaises(json.JSONDecodeError): - server.build_audit_bundle( - shares=[], - found_block={"block_height": 10, "coinbase_value_sats": 50_00000000}, - prior_balances=[], - coinbase_script_sig_suffix_hex="00", - canonical_output_path=output_path, - ) - self.assertFalse(output_path.exists()) - - def test_build_audit_bundle_recovers_after_cancelled_worker_timeout(self) -> None: - server = coordinator() - server.signing_seed_hex = "42" * 32 - server.ledger_attestation_signing_seed_hex = "43" * 32 - cancellation = _JobBuildCancellation(timeout_seconds=60) - process_calls = 0 - - class FakeStdin: - def write(self, value: str) -> int: - return len(value) - - def close(self) -> None: - return None - - class HungThenHealthyPopen: - def __init__(self, _cmd: list[str], **kwargs: object) -> None: - nonlocal process_calls - process_calls += 1 - self.healthy = process_calls == 2 - self.stdin = FakeStdin() - self.stdout = kwargs["stdout"] - self.stderr = kwargs["stderr"] - self.returncode: int | None = None - self.output_written = False - - def poll(self) -> int | None: - if self.returncode is not None: - return self.returncode - if self.healthy: - if not self.output_written: - self.stdout.write('{"ok":true}') # type: ignore[union-attr] - self.output_written = True - self.returncode = 0 - return 0 - cancellation.cancel("timeout") - return None - - def terminate(self) -> None: - self.returncode = -15 - - def kill(self) -> None: - self.returncode = -9 - - def wait(self, timeout: float | None = None) -> int: - del timeout - assert self.returncode is not None - return self.returncode - - build_kwargs = { - "shares": [], - "found_block": { - "block_height": 10, - "coinbase_value_sats": 50_00000000, - }, - "prior_balances": [], - "coinbase_script_sig_suffix_hex": "00", - } - with patch( - "lab.prism.prism_coordinator.subprocess.Popen", - HungThenHealthyPopen, - ): - with self.assertRaisesRegex(JobBuildCancelled, "timeout"): - server.build_audit_bundle( - **build_kwargs, - cancellation=cancellation, - ) - recovered = server.build_audit_bundle( - **build_kwargs, - cancellation=_JobBuildCancellation(timeout_seconds=60), - ) - - self.assertEqual(recovered, {"ok": True}) - self.assertEqual(process_calls, 2) - self.assertEqual(server.job_build_worker_counts["terminations"], 1) - self.assertEqual(server.job_build_worker_counts["restarts"], 1) - - def test_build_audit_bundle_does_not_unlink_preexisting_output_path(self) -> None: - server = coordinator() - server.signing_seed_hex = "42" * 32 - server.ledger_attestation_signing_seed_hex = "43" * 32 - - with tempfile.TemporaryDirectory() as tmp, patch( - "lab.prism.prism_coordinator.subprocess.Popen" - ) as popen: - output_path = Path(tmp) / "preexisting-candidate.audit.json" - output_path.write_bytes(b"do-not-clobber") - with self.assertRaises(FileExistsError): - server.build_audit_bundle( - shares=[], - found_block={"block_height": 10, "coinbase_value_sats": 50_00000000}, - prior_balances=[], - coinbase_script_sig_suffix_hex="00", - canonical_output_path=output_path, - ) - - self.assertEqual(output_path.read_bytes(), b"do-not-clobber") - popen.assert_not_called() - - def test_block_submit_rejects_job_when_prior_balances_changed_before_persist(self) -> None: - server, state, _recording = submit_coordinator() - configure_temporary_audit_root(self, server) - ledger = SingleWriterShareLedger() - ledger.durable_payout_state = True # type: ignore[attr-defined] - server.ledger = ledger - server.jobs["job-1"].prior_balances = [ - { - "recipient_id": "miner-a", - "order_key": "miner-a", - "p2mr_program_hex": "11" * 32, - "balance_sats": 25, - } - ] - server.build_audit_bundle = lambda **_kwargs: (_ for _ in ()).throw( # type: ignore[method-assign] - AssertionError("audit bundle should not be rebuilt from stale prior balances") - ) - submission = SimpleNamespace( - coinbase_tx_hex="c0ffee", - block_hash_hex="ef" * 32, - block_hex="00", - ) - pending = self._pending_append("balance-stale").pending_share - candidate = block_candidate( - server, - state, - submission, - pending_share=pending, - ) - ledger.append_batch( - [(pending, server.block_candidate_intent(candidate))] - ) - server.enqueue_block_candidate(candidate) - - self.assertTrue(server.submit_next_block_candidate()) - # The share was already accepted at submit time, so a lost block is a - # block-abandonment, not a stale share rejection. - self.assertEqual(server.stale_share_count, 0) - self.assertEqual(server.block_candidate_abandoned_counts[PRISM_REJECTION_STALE_JOB], 1) - self.assertEqual( - server.stale_job_abandon_counts, - {"tip_moved": 0, "balance_stale": 1, "append_epoch_stale": 0}, - ) - outbox_row = ledger._block_candidate_outbox[submission.block_hash_hex] - self.assertEqual(outbox_row["state"], "abandoned") - self.assertEqual( - outbox_row["last_error"], - "prior balances changed since the job was issued", - ) - metrics = server.metrics_payload() - self.assertIn( - 'qbit_prism_stale_job_abandons_total{class="tip_moved"} 0', - metrics, - ) - self.assertIn( - 'qbit_prism_stale_job_abandons_total{class="balance_stale"} 1', - metrics, - ) - - def test_block_submit_defers_descendant_until_active_ancestor_is_durable( - self, - ) -> None: - parent_hash = "ed" * 32 - ancestor_hash = "ee" * 32 - server, state, ledger = submit_coordinator(tip=parent_hash) - server.max_blocks = 10 - server.stop_after_block = False - ledger.durable_payout_state = True - preview = [ - { - "recipient_id": "miner-a", - "order_key": "miner-a", - "p2mr_program_hex": "11" * 32, - "balance_sats": 25, - } - ] - context = server.jobs["job-1"] - context.template["height"] = 12 - context.prior_balances = preview - original_rpc_call = server.rpc.call - submit_calls: list[str] = [] - - def active_ancestor_call( - method: str, - params: list[object] | None = None, - ) -> object: - if method == "getblockhash": - self.assertEqual(params, [10]) - return ancestor_hash - if method == "submitblock": - submit_calls.append(method) - return original_rpc_call(method, params) - - server.rpc.call = active_ancestor_call # type: ignore[method-assign] - server._begin_accepted_block_payout_preview( - ancestor_hash, - block_height=10, - ) - server._publish_accepted_block_payout_preview(ancestor_hash, preview) - server.build_audit_bundle = lambda **_kwargs: (_ for _ in ()).throw( # type: ignore[method-assign] - AssertionError("descendant must wait for ancestor durability") - ) - submission = SimpleNamespace( - coinbase_tx_hex="c0ffee", - block_hash_hex="ef" * 32, - block_hex="00", - ) - - accepted = server.submit_block_candidate( - block_candidate(server, state, submission) - ) - - self.assertFalse(accepted) - # Node propagation is the fast lane: the durable descendant is offered - # to qbitd before payout/accounting notices the ancestor transition. - # Its accounting still defers until that ancestor is durable. - self.assertEqual(submit_calls, ["submitblock"]) - self.assertEqual(ledger.persisted, []) - self.assertEqual( - server._block_candidate_outcome.reason, - PRISM_REJECTION_LEDGER_CONFIRMATION_FAILED, - ) - self.assertNotIn( - PRISM_REJECTION_STALE_JOB, - server.block_candidate_abandoned_counts, - ) - self.assertFalse(server.stop_event.is_set()) - - def test_active_descendant_replay_stays_landed_until_ancestor_is_durable( - self, - ) -> None: - parent_hash = "ea" * 32 - ancestor_hash = "eb" * 32 - descendant_hash = "ec" * 32 - server, state, ledger = submit_coordinator(tip=parent_hash) - server.max_blocks = 10 - server.stop_after_block = False - ledger.durable_payout_state = True - preview = [ - { - "recipient_id": "miner-a", - "order_key": "miner-a", - "p2mr_program_hex": "11" * 32, - "balance_sats": 25, - } - ] - context = server.jobs["job-1"] - context.template["height"] = 12 - context.prior_balances = preview - original_rpc_call = server.rpc.call - - def active_descendant_call( - method: str, - params: list[object] | None = None, - ) -> object: - if method == "getbestblockhash": - return descendant_hash - if method == "getblockhash": - self.assertEqual(params, [10]) - return ancestor_hash - if method == "submitblock": - raise AssertionError("active descendant must not be resubmitted") - return original_rpc_call(method, params) - - server.rpc.call = active_descendant_call # type: ignore[method-assign] - server._begin_accepted_block_payout_preview( - ancestor_hash, - block_height=10, - ) - server._publish_accepted_block_payout_preview(ancestor_hash, preview) - server.build_audit_bundle = lambda **_kwargs: (_ for _ in ()).throw( # type: ignore[method-assign] - AssertionError("active descendant must wait for ancestor durability") - ) - submission = SimpleNamespace( - coinbase_tx_hex="c0ffee", - block_hash_hex=descendant_hash, - block_hex="00", - ) - - accepted = server.submit_block_candidate( - block_candidate(server, state, submission) - ) - - self.assertFalse(accepted) - self.assertEqual(ledger.persisted, []) - self.assertEqual( - server._block_candidate_outcome.reason, - PRISM_REJECTION_LEDGER_CONFIRMATION_FAILED, - ) - transition = server._accepted_block_payout_previews[descendant_hash] - self.assertTrue(transition.landed) - self.assertIsNone(transition.preview) - self.assertFalse(server.stop_event.is_set()) - - def test_block_submit_reconciliation_error_is_structured_rejection(self) -> None: - tip = "f0" * 32 - server, state, _ledger = submit_coordinator(tip=tip) - server.reorg_reconciler_enabled = True - server.rpc = ReorgRpc( - tip=tip, - template=gbt_template(tip, height=10), - height=9, - block_hashes={9: tip}, - ) - - class FailingSubmitReorgLedger(RecordingLedger): - def reorg_watch_blocks(self, *, active_tip_height: int) -> list[dict[str, object]]: - raise RuntimeError("ledger unavailable") - - ledger = FailingSubmitReorgLedger() - server.ledger = ledger - server.build_audit_bundle = lambda **_kwargs: (_ for _ in ()).throw( # type: ignore[method-assign] - AssertionError("audit bundle should not be rebuilt after reconcile failure") - ) - submission = SimpleNamespace( - coinbase_tx_hex="c0ffee", - block_hash_hex="f1" * 32, - block_hex="00", - ) - - with patch("builtins.print") as printed: - accepted = server.submit_block_candidate(block_candidate(server, state, submission)) - - self.assertFalse(accepted) - self.assertEqual( - server.block_candidate_abandoned_counts.get(PRISM_REJECTION_BACKEND_RPC_UNAVAILABLE, 0), - 0, - ) - messages = [str(call.args[0]) for call in printed.call_args_list if call.args] - self.assertTrue(any("block candidate deferred" in message for message in messages)) - self.assertFalse(any("block candidate abandoned" in message for message in messages)) - self.assertEqual(server.rejection_counts_by_reason[PRISM_REJECTION_BACKEND_RPC_UNAVAILABLE], 0) - self.assertEqual(ledger.persisted, []) - self.assertEqual(ledger.pending, []) - - def test_address_worker_suffixes_share_one_payout_account(self) -> None: - server, state_a, ledger = submit_coordinator() - usernames = [f"{PAYOUT_ADDRESS}.rig-a", f"{PAYOUT_ADDRESS}.rig-b"] - submissions = [ - SimpleNamespace( - header_hex="aa" * 80, - block_hash_hex="bd" * 32, - share_pass=True, - block_pass=False, - ), - SimpleNamespace( - header_hex="bb" * 80, - block_hash_hex="be" * 32, - share_pass=True, - block_pass=False, - ), - ] - states = [state_a, client()] - states[1].active_job_ids = {"job-1"} - for state, username in zip(states, usernames, strict=True): - worker = WorkerIdentity( - username=username, - payout_address=PAYOUT_ADDRESS, - worker_name=username.rsplit(".", 1)[1], - script_pubkey_hex="5220" + "55" * 32, - p2mr_program_hex="55" * 32, - ) - state.username = username - state.worker = worker - state.subscribed = True - state.authorized = True - server.jobs["job-1"].worker = worker - with patch( - "lab.prism.prism_coordinator.direct_stratum.assemble_submission", - return_value=submissions.pop(0), - ): - server.handle_submit( - state, - [username, "job-1", "00" * 8, "00000001", "00000002"], - ) - - self.assertEqual([pending.miner_id for pending in ledger.pending], [PAYOUT_ADDRESS, PAYOUT_ADDRESS]) - self.assertEqual([pending.order_key for pending in ledger.pending], [PAYOUT_ADDRESS, PAYOUT_ADDRESS]) - - def _pending_append(self, tag: str, accepted_at_ms: int = 2) -> PendingShareAppend: - from lab.prism.share_ledger import PendingShare - - return PendingShareAppend( - pending_share=PendingShare( - share_id=f"miner-a:{tag}", - miner_id="miner-a", - order_key="miner-a", - p2mr_program_hex="11" * 32, - share_difficulty=1, - network_difficulty=1, - template_height=10, - job_id="job-1", - job_issued_at_ms=1, - accepted_at_ms=accepted_at_ms, - ntime=1_700_000_000, - ), - username="miner-a", - job_id="job-1", - block_hash_hex=tag * 32, - collection_only=False, - credit_policy=None, - ) - - def test_orphaned_block_candidate_keeps_share_credit(self) -> None: - # Option-A semantics: a share that met its target stays credited even - # when its block candidate loses the tip race in the submitter. - old_tip = "00" * 32 - new_tip = "11" * 32 - server, state, _recording = submit_coordinator(tip=old_tip) - configure_temporary_audit_root(self, server) - ledger = SingleWriterShareLedger() - server.ledger = ledger - submission = SimpleNamespace( - header_hex="aa" * 80, - block_hash_hex="cc" * 32, - block_hex="00", - share_pass=True, - block_pass=True, - ) - - with patch( - "lab.prism.prism_coordinator.direct_stratum.assemble_submission", - return_value=submission, - ): - server.handle_submit( - state, - ["miner-a", "job-1", "00" * 8, "00000001", "00000002"], - ) - - self.assertEqual(len(ledger.all_shares()), 1) - # The tip moves before the submitter drains the candidate. - server.rpc = RejectingSubmitTipRpc(new_tip) - - self.assertTrue(server.submit_next_block_candidate()) - - self.assertEqual(server.accepted_block_count, 0) - # Counted as a block abandonment, not a share rejection. - self.assertEqual(server.block_candidate_abandoned_counts[PRISM_REJECTION_STALE_JOB], 1) - self.assertEqual( - server.stale_job_abandon_counts, - {"tip_moved": 1, "balance_stale": 0, "append_epoch_stale": 0}, - ) - self.assertEqual(server.stale_share_count, 0) - # The credited share survives the lost block race. - self.assertEqual(len(ledger.all_shares()), 1) - outbox_row = ledger._block_candidate_outbox[submission.block_hash_hex] - self.assertEqual(outbox_row["state"], "abandoned") - self.assertEqual( - outbox_row["last_error"], - f"tip moved before submit: {new_tip}", - ) - metrics = server.metrics_payload() - self.assertIn( - 'qbit_prism_stale_job_abandons_total{class="tip_moved"} 1', - metrics, - ) - self.assertIn( - 'qbit_prism_stale_job_abandons_total{class="balance_stale"} 0', - metrics, - ) - - def test_block_candidate_queue_overflow_coalesces_wakeup_without_drop(self) -> None: - server, state, _ledger = submit_coordinator() - configure_temporary_audit_root(self, server) - server.block_candidate_queue = queue.Queue(maxsize=2) - - def candidate(tag: str) -> PrismBlockCandidate: - return block_candidate( - server, - state, - SimpleNamespace(block_hash_hex=tag * 32, share_pass=True, block_pass=True), - ) - - server.enqueue_block_candidate(candidate("aa")) - server.enqueue_block_candidate(candidate("bb")) - server.enqueue_block_candidate(candidate("cc")) - - self.assertEqual(server.block_candidates_dropped, 0) - self.assertEqual(server.block_candidate_queue.qsize(), 2) - remaining = [ - server.block_candidate_queue.get_nowait().submission.block_hash_hex - for _ in range(2) - ] - # Existing wakeups remain ordered; the third candidate remains durable - # in the outbox and will be re-read after the queue drains. - self.assertEqual(remaining, ["aa" * 32, "bb" * 32]) - self.assertIn( - "qbit_prism_block_candidates_dropped_total 0", server.metrics_payload() - ) - self.assertIn( - "qbit_prism_block_candidate_wakeups_coalesced_total 1", - server.metrics_payload(), - ) - - def test_b1_codec_owner_round_trips_without_coordinator_state(self) -> None: - server, state, _ledger = submit_coordinator() - candidate = block_candidate( - server, - state, - SimpleNamespace( - block_hash_hex="ca" * 32, - block_hex="00", - coinbase_tx_hex="11", - share_pass=True, - block_pass=True, - ), - pending_share=PendingShare( - share_id="codec-share", - miner_id="miner-a", - order_key="miner-a", - p2mr_program_hex="11" * 32, - share_difficulty=1, - network_difficulty=1, - template_height=10, - job_id="job-1", - job_issued_at_ms=1, - accepted_at_ms=2, - ntime=3, - ), - ) - - intent = block_candidate_intent(candidate) - replayed = block_candidate_from_intent(intent) - - self.assertEqual(replayed.submission.block_hash_hex, "ca" * 32) - self.assertEqual(replayed.context.template, candidate.context.template) - self.assertEqual(replayed.pending_share, candidate.pending_share) - # The module decode carries the restart sentinel: process-local - # append epochs are meaningless after a restart, so the landing - # fence must revalidate the recorded window instead. - self.assertEqual(replayed.context.payout_append_invalidation_epoch, -1) - - def test_b1_intent_refuses_a_submission_with_no_serialized_block(self) -> None: - """An unmaterialized block must never become a durable intent. - - ``assemble_submission`` builds ``block_hex`` only for a hash that - solved the block, which every candidate route requires. If that - coupling were ever broken the block is already lost; persisting the - empty value would lose it again after every restart, as a candidate - that can never be offered to the node. Fail on the client thread - instead, before the durable write. - """ - server, state, _ledger = submit_coordinator() - candidate = block_candidate( - server, - state, - direct_stratum.DirectQbitSubmission( - coinbase_tx_hex="11", - coinbase_txid_preimage_hex="", - header_hex="", - block_hex="", - block_hash_hex="cb" * 32, - block_hash_int=1, - share_pass=True, - block_pass=True, - applied_version_hex="", - ), - ) - - with self.assertRaisesRegex(ValueError, "carries no serialized block"): - block_candidate_intent(candidate) - - def test_b1_intent_still_tolerates_an_embedder_without_block_bytes(self) -> None: - """The invariant belongs to the real submission type, not to stubs. - - Some integrations retain no block bytes at all; the landing path - already guards them by attribute probe. Tightening the codec must not - turn those into hard failures. - """ - server, state, _ledger = submit_coordinator() - candidate = block_candidate( - server, - state, - SimpleNamespace( - block_hash_hex="cc" * 32, - coinbase_tx_hex="11", - share_pass=True, - block_pass=True, - ), - pending_share=PendingShare( - share_id="embedder-share", - miner_id="miner-a", - order_key="miner-a", - p2mr_program_hex="11" * 32, - share_difficulty=1, - network_difficulty=1, - template_height=10, - job_id="job-1", - job_issued_at_ms=1, - accepted_at_ms=2, - ntime=3, - ), - ) - - intent = block_candidate_intent(candidate) - - self.assertEqual(intent["block_hex"], "") - - def test_b1_replayed_intent_re_encodes_without_tripping_the_guard(self) -> None: - """Decode erases duck-typing, so the guard must exempt a replay. - - An embedder that retains no block bytes writes ``""`` and is tolerated. - ``block_candidate_from_intent`` then rebuilds every replayed candidate - as a real ``DirectQbitSubmission``, so that tolerated value comes back - wearing the strict type. The only caller that re-encodes a replayed - candidate is the credit-on-accept append, which runs after the block - has landed -- raising there would fail a credit for a block already in - the chain. Re-persisting an intent the row already holds is not the - write the guard exists to stop. - """ - server, state, _ledger = submit_coordinator() - pending = PendingShare( - share_id="replay-share", - miner_id="miner-a", - order_key="miner-a", - p2mr_program_hex="11" * 32, - share_difficulty=1, - network_difficulty=1, - template_height=10, - job_id="job-1", - job_issued_at_ms=1, - accepted_at_ms=2, - ntime=3, - ) - written = block_candidate_intent( - block_candidate( - server, - state, - SimpleNamespace( - block_hash_hex="cf" * 32, - coinbase_tx_hex="11", - share_pass=True, - block_pass=True, - ), - pending_share=pending, - credit_share_on_accept=True, - ) - ) - self.assertEqual(written["block_hex"], "") - - replayed = dataclasses.replace( - block_candidate_from_intent(written), - durable_replay=True, - ) - # Decode really does hand back the strict type; without the replay - # exemption this is exactly what would trip the guard. - self.assertIsInstance( - replayed.submission, - direct_stratum.DirectQbitSubmission, - ) - self.assertEqual(replayed.submission.block_hex, "") - - self.assertEqual(block_candidate_intent(replayed)["block_hex"], "") - - def test_b1_replayed_intent_preserves_a_real_serialized_block(self) -> None: - """The ordinary replay: block bytes survive the round trip intact.""" - server, state, _ledger = submit_coordinator() - pending = PendingShare( - share_id="replay-real-share", - miner_id="miner-a", - order_key="miner-a", - p2mr_program_hex="11" * 32, - share_difficulty=1, - network_difficulty=1, - template_height=10, - job_id="job-1", - job_issued_at_ms=1, - accepted_at_ms=2, - ntime=3, - ) - written = block_candidate_intent( - block_candidate( - server, - state, - direct_stratum.DirectQbitSubmission( - coinbase_tx_hex="11", - coinbase_txid_preimage_hex="", - header_hex="", - block_hex="0badc0de", - block_hash_hex="da" * 32, - block_hash_int=1, - share_pass=True, - block_pass=True, - applied_version_hex="", - ), - pending_share=pending, - credit_share_on_accept=True, - ) - ) - - replayed = dataclasses.replace( - block_candidate_from_intent(written), - durable_replay=True, - ) - - self.assertEqual(replayed.submission.block_hex, "0badc0de") - self.assertEqual(block_candidate_intent(replayed)["block_hex"], "0badc0de") - - def test_b1_service_owns_queue_and_structured_attempt_results(self) -> None: - server, state, _ledger = submit_coordinator() - service = server._ensure_block_candidate_service() - self.assertIs(server.block_candidate_queue, service.candidate_queue) - self.assertIsInstance(service.submit_next(), BlockCandidateRunResult) - self.assertFalse(service.submit_next().ran) - - candidate = block_candidate( - server, - state, - SimpleNamespace( - block_hash_hex="cb" * 32, - share_pass=True, - block_pass=True, - ), - ) - - def reject(_candidate: PrismBlockCandidate) -> bool: - server._abandon_block_candidate( - PRISM_REJECTION_STALE_JOB, - "direct owner result", - block_hash="cb" * 32, - worker="miner-a", - ) - return False - - server.submit_block_candidate = reject # type: ignore[method-assign] - with patch("builtins.print"): - result = service.attempt(candidate) - - self.assertIsInstance(result, BlockCandidateAttemptResult) - self.assertFalse(result.accepted) - self.assertEqual(result.reason, PRISM_REJECTION_STALE_JOB) - - def test_b1_service_resolves_replaced_stop_event_at_use_time(self) -> None: - server, _state, _ledger = submit_coordinator() - service = server._ensure_block_candidate_service() - replacement = threading.Event() - server.stop_event = replacement - replacement.set() - - service.run() - - self.assertIs(service.ports.stop_event(), replacement) - - def test_b1_landing_tail_routes_through_the_named_submit_port(self) -> None: - # #127: both landing consumers hand the whole call shape to the named - # submit port instead of asking the runtime seam which tail is live. - server, state, _ledger = submit_coordinator() - service = server._ensure_block_candidate_service() - self.assertFalse(hasattr(server, "_is_production_block_submit")) - calls: list[tuple[str, object, bool]] = [] - - def recording_port( - landing: PrismBlockCandidate, - *, - node_submission: object = None, - disposition_held: bool = False, - ) -> bool: - calls.append( - ( - str(landing.submission.block_hash_hex), - node_submission, - disposition_held, - ) - ) - return True - - service.ports = dataclasses.replace( - service.ports, - submit_candidate=recording_port, - ) - queued = block_candidate( - server, - state, - SimpleNamespace( - coinbase_tx_hex="00", - block_hash_hex="d1" * 32, - block_hex="00", - share_pass=True, - block_pass=True, - ), - ) - server.enqueue_block_candidate(queued) - self.assertTrue(server.submit_next_block_candidate()) - - synchronous = block_candidate( - server, - state, - SimpleNamespace( - coinbase_tx_hex="00", - block_hash_hex="d2" * 32, - block_hex="00", - share_pass=True, - block_pass=True, - ), - ) - self.assertTrue(server._submit_synchronous_block_candidate(synchronous)) - - self.assertEqual([call[0] for call in calls], ["d1" * 32, "d2" * 32]) - # Both consumers already own the same-hash disposition, and both pass - # the node submission they created rather than leaving it to the port. - self.assertEqual([call[2] for call in calls], [True, True]) - for _block_hash, node_submission, _held in calls: - self.assertIsInstance(node_submission, _BlockCandidateNodeSubmission) - - def test_b1_submit_port_honours_replacement_installed_after_construction( - self, - ) -> None: - # The service is built before the double is installed, so the named - # port must resolve the coordinator's callable at call time. - server, state, _ledger = submit_coordinator() - service = server._ensure_block_candidate_service() - landed: list[str] = [] - - def double(landing: PrismBlockCandidate) -> bool: - landed.append(str(landing.submission.block_hash_hex)) - return True - - server.submit_block_candidate = double # type: ignore[method-assign] - candidate = block_candidate( - server, - state, - SimpleNamespace( - coinbase_tx_hex="00", - block_hash_hex="d3" * 32, - block_hex="00", - share_pass=True, - block_pass=True, - ), - ) - server.enqueue_block_candidate(candidate) - self.assertTrue(server.submit_next_block_candidate()) - - self.assertEqual(landed, ["d3" * 32]) - self.assertIs(service, server._ensure_block_candidate_service()) - - def test_b1_submit_port_preserves_every_former_landing_call_shape(self) -> None: - server, state, _ledger = submit_coordinator() - port = server._ensure_block_candidate_service().ports.submit_candidate - candidate = block_candidate( - server, - state, - SimpleNamespace( - coinbase_tx_hex="00", - block_hash_hex="d4" * 32, - block_hex="00", - share_pass=True, - block_pass=True, - ), - ) - node_submission = _BlockCandidateNodeSubmission(attempted=False) - - # Shape 1: production entrypoint plus a held disposition runs the - # serialized inner tail, not the public entrypoint that would take - # that same-hash guard a second time. - serialized: list[object] = [] - server._submit_block_candidate_serialized = ( # type: ignore[method-assign] - lambda _landing, *, node_submission: ( - serialized.append(node_submission) or True - ) - ) - self.assertTrue( - port(candidate, node_submission=node_submission, disposition_held=True) - ) - self.assertEqual(serialized, [node_submission]) - - shapes: list[tuple[str, object]] = [] - - def bare_double(_landing: PrismBlockCandidate) -> bool: - shapes.append(("bare", None)) - return True - - # Shape 2: no node submission was created, so the historical bare - # entrypoint call stands -- including when the disposition is held, - # which is the writer path's middle arm. - server.submit_block_candidate = bare_double # type: ignore[method-assign] - self.assertTrue(port(candidate)) - self.assertTrue( - port(candidate, node_submission=None, disposition_held=True) - ) - - def sniffed_double( - _landing: PrismBlockCandidate, - *, - node_submission: object = None, - ) -> bool: - shapes.append(("node-submission", node_submission)) - return True - - # Shape 3: an already-created node submission is passed through to a - # replacement whose signature can accept it. - server.submit_block_candidate = sniffed_double # type: ignore[method-assign] - self.assertTrue( - port(candidate, node_submission=node_submission, disposition_held=True) - ) - self.assertTrue(port(candidate, node_submission=node_submission)) - - self.assertEqual( - shapes, - [ - ("bare", None), - ("bare", None), - ("node-submission", node_submission), - ("node-submission", node_submission), - ], - ) - - def test_b1_production_landing_still_reaches_the_serialized_tail(self) -> None: - # #127 is structural hygiene: submit-double tests still do not run - # _submit_block_candidate_serialized. This pins the other side -- with - # no double installed, ordinary production operation still does. - old_tip = "00" * 32 - new_tip = "11" * 32 - block_hash = "cd" * 32 - server, state, _recording = submit_coordinator(tip=old_tip) - ledger = SingleWriterShareLedger() - server.ledger = ledger - pending = self._pending_append("production-serialized-tail").pending_share - candidate = block_candidate( - server, - state, - SimpleNamespace( - coinbase_tx_hex="00", - block_hash_hex=block_hash, - block_hex="00", - share_pass=True, - block_pass=True, - ), - pending_share=pending, - ) - ledger.append_batch([(pending, server.block_candidate_intent(candidate))]) - submitted: list[str] = [] - original_submitted = ledger.mark_block_candidate_submitted - - def mark_submitted(*, block_hash: str) -> bool: - submitted.append(block_hash) - return original_submitted(block_hash=block_hash) - - ledger.mark_block_candidate_submitted = mark_submitted # type: ignore[method-assign] - server._begin_accepted_block_payout_preview(block_hash, block_height=10) - server._mark_accepted_block_payout_landed(block_hash, block_height=10) - with server.lock: - server._accounted_accepted_block_hashes.add(block_hash) - server.rpc = TipRpc(new_tip) - serialized: list[object] = [] - real_serialized = server._submit_block_candidate_serialized - - def recording_serialized( - landing: PrismBlockCandidate, - *, - node_submission: object, - ) -> bool: - serialized.append(node_submission) - return real_serialized(landing, node_submission=node_submission) - - server._submit_block_candidate_serialized = ( # type: ignore[method-assign] - recording_serialized - ) - server.enqueue_block_candidate(candidate) - - self.assertTrue(server.submit_next_block_candidate()) - - self.assertEqual(submitted, [block_hash]) - self.assertEqual(len(serialized), 1) - self.assertIsInstance(serialized[0], _BlockCandidateNodeSubmission) - - def test_failed_replay_adoption_does_not_abort_remaining_rows(self) -> None: - server, state, _recording = submit_coordinator() - ledger = SingleWriterShareLedger() - server.ledger = ledger - - def durable_candidate(tag: str, stamp: int) -> PrismBlockCandidate: - pending = PendingShare( - share_id=f"miner-a:{tag * 32}", - miner_id="miner-a", - order_key="miner-a", - p2mr_program_hex="11" * 32, - share_difficulty=1, - network_difficulty=1, - template_height=9, - job_id="job-1", - job_issued_at_ms=1, - accepted_at_ms=stamp, - ntime=1, - ) - value = dataclass_replace( - block_candidate( - server, - state, - SimpleNamespace( - coinbase_tx_hex="00", - block_hash_hex=tag * 32, - block_hex="00", - share_pass=False, - block_pass=True, - ), - pending_share=pending, - ), - credit_share_on_accept=True, - ) - ledger.append_batch([(pending, server.block_candidate_intent(value))]) - return value - - first = durable_candidate("a5", 1) - second = durable_candidate("b5", 2) - service = server._ensure_block_candidate_service() - original_adopt = service.adopt_replayed_candidate - - def fail_first(value: PrismBlockCandidate) -> None: - if value.submission.block_hash_hex == first.submission.block_hash_hex: - raise RuntimeError("share-writer adoption unavailable") - original_adopt(value) - - def unexpected_finish(_pending: PendingShare) -> None: - raise AssertionError("unadopted replay must not be finished") - - service.adopt_replayed_candidate = fail_first # type: ignore[method-assign] - server._finish_pending_share_candidate = ( # type: ignore[method-assign] - unexpected_finish - ) - - with patch("builtins.print"): - self.assertEqual(server.replay_pending_block_candidates(), 1) - replayed = server._block_replay_candidate_queue.get_nowait() - self.assertEqual( - replayed.submission.block_hash_hex, - second.submission.block_hash_hex, - ) - # The unadopted row quarantines off the node-offer lane. Draining it - # terminalizes only that row, and its never-adopted floor holder is - # never finished. - with patch("builtins.print"): - self.assertTrue(server._run_one_invalid_block_candidate_quarantine()) - self.assertEqual( - [intent["block_hash_hex"] for intent in ledger.pending_block_candidates()], - [second.submission.block_hash_hex], - ) - - def test_durable_block_candidates_replay_after_queue_drains(self) -> None: - server, state, _recording = submit_coordinator() - ledger = SingleWriterShareLedger() - server.ledger = ledger - server.block_candidate_queue = queue.Queue(maxsize=2) - - for index, tag in enumerate(("aa", "bb", "cc"), start=1): - pending = PendingShare( - share_id=f"miner-a:{tag * 32}", - miner_id="miner-a", - order_key="miner-a", - p2mr_program_hex="11" * 32, - share_difficulty=1, - network_difficulty=1, - template_height=9, - job_id="job-1", - job_issued_at_ms=1, - accepted_at_ms=index, - ntime=1, - ) - submission = SimpleNamespace( - coinbase_tx_hex="00", - block_hash_hex=tag * 32, - block_hex="00", - share_pass=True, - block_pass=True, - ) - candidate = block_candidate( - server, state, submission, pending_share=pending - ) - ledger.append_batch([(pending, server.block_candidate_intent(candidate))]) - - self.assertEqual(server.replay_pending_block_candidates(), 3) - self.assertTrue(server.block_candidate_queue.empty()) - first = server._block_replay_candidate_queue.get_nowait() - second = server._block_replay_candidate_queue.get_nowait() - third = server._block_replay_candidate_queue.get_nowait() - self.assertEqual( - [ - first.submission.block_hash_hex, - second.submission.block_hash_hex, - third.submission.block_hash_hex, - ], - ["aa" * 32, "bb" * 32, "cc" * 32], - ) - ledger.mark_block_candidate_submitted(block_hash="aa" * 32) - ledger.mark_block_candidate_abandoned(block_hash="bb" * 32, error="stale") - self.assertEqual(third.pending_share.share_id, "miner-a:" + "cc" * 32) - - def test_candidate_intent_avoids_duplicate_template_transaction_bodies(self) -> None: - server, state, _ledger = submit_coordinator() - witness_tx = synthetic_witness_transaction("55") - server.jobs["job-1"].template["transactions"] = [{"data": witness_tx}] - server.jobs["job-1"].job.transaction_hexes = (witness_tx,) - pending = self._pending_append("ca").pending_share - candidate = block_candidate( - server, - state, - SimpleNamespace( - coinbase_tx_hex="00", - block_hash_hex="ca" * 32, - block_hex="00" + witness_tx, - share_pass=True, - block_pass=True, - ), - pending_share=pending, - ) - - intent = server.block_candidate_intent(candidate) - - self.assertEqual( - set(intent["template"]), - {"previousblockhash", "height", "coinbasevalue"}, - ) - self.assertNotIn("transaction_hexes", intent) - self.assertEqual( - intent["witness_merkle_leaves_hex"], - direct_stratum.witness_merkle_leaves_hex((witness_tx,)), - ) - - def test_transient_candidate_failure_remains_pending_for_retry(self) -> None: - server, state, _recording = submit_coordinator() - configure_temporary_audit_root(self, server) - ledger = SingleWriterShareLedger() - server.ledger = ledger - pending = PendingShare( - share_id="miner-a:" + "aa" * 32, - miner_id="miner-a", - order_key="miner-a", - p2mr_program_hex="11" * 32, - share_difficulty=1, - network_difficulty=1, - template_height=9, - job_id="job-1", - job_issued_at_ms=1, - accepted_at_ms=2, - ntime=1, - ) - candidate = block_candidate( - server, - state, - SimpleNamespace( - coinbase_tx_hex="00", - block_hash_hex="aa" * 32, - block_hex="00", - share_pass=True, - block_pass=True, - ), - pending_share=pending, - ) - intent = server.block_candidate_intent(candidate) - ledger.append_batch([(pending, intent)]) - server.enqueue_block_candidate(candidate) - server.submit_block_candidate = ( # type: ignore[method-assign] - lambda _candidate: (_ for _ in ()).throw(RuntimeError("rpc unavailable")) - ) - - self.assertTrue(server.submit_next_block_candidate()) - - self.assertEqual(ledger.pending_block_candidates(), [intent]) - self.assertEqual(server.block_candidate_abandoned_counts, {}) - self.assertIn( - "qbit_prism_block_candidate_retries_total 1", - server.metrics_payload(), - ) - - def test_retryable_parent_stays_ahead_of_queued_child(self) -> None: - server, state, _ledger = submit_coordinator() - server.block_candidate_retry_initial_seconds = 0 - server.block_candidate_retry_max_seconds = 0 - parent = block_candidate( - server, - state, - SimpleNamespace( - coinbase_tx_hex="00", - block_hash_hex="a1" * 32, - block_hex="00", - share_pass=True, - block_pass=True, - ), - ) - child = block_candidate( - server, - state, - SimpleNamespace( - coinbase_tx_hex="00", - block_hash_hex="b1" * 32, - block_hex="01", - share_pass=True, - block_pass=True, - ), - ) - attempts: list[str] = [] - - def submit(candidate: PrismBlockCandidate) -> bool: - block_hash = str(candidate.submission.block_hash_hex) - attempts.append(block_hash) - if block_hash == "a1" * 32 and attempts.count(block_hash) == 1: - server._abandon_block_candidate( - PRISM_REJECTION_BACKEND_RPC_UNAVAILABLE, - "temporary parent finalization failure", - block_hash=block_hash, - worker="miner-a", - ) - return False - return True - - server.submit_block_candidate = submit # type: ignore[method-assign] - self.assertTrue(server.enqueue_block_candidate(parent)) - self.assertTrue(server.enqueue_block_candidate(child)) - - self.assertTrue(server.submit_next_block_candidate()) - self.assertIs(server._retry_block_candidate, parent) - self.assertEqual(server.block_candidate_queue.qsize(), 1) - self.assertTrue(server.submit_next_block_candidate()) - self.assertIsNone(getattr(server, "_retry_block_candidate", None)) - self.assertEqual(server.block_candidate_queue.qsize(), 1) - self.assertTrue(server.submit_next_block_candidate()) - - self.assertEqual(attempts, ["a1" * 32, "a1" * 32, "b1" * 32]) - self.assertTrue(server.block_candidate_queue.empty()) - - def test_candidate_retry_backoff_is_capped_and_cleared_on_success(self) -> None: - server, state, _recording = submit_coordinator() - ledger = SingleWriterShareLedger() - server.ledger = ledger - pending = self._pending_append("retry-success").pending_share - candidate = block_candidate( - server, - state, - SimpleNamespace( - coinbase_tx_hex="00", - block_hash_hex="a1" * 32, - block_hex="00", - share_pass=True, - block_pass=True, - ), - pending_share=pending, - ) - ledger.append_batch([(pending, server.block_candidate_intent(candidate))]) - server.block_candidate_retry_initial_seconds = 0.1 - server.block_candidate_retry_max_seconds = 0.4 - attempts = 0 - - def retry_then_succeed(_candidate: PrismBlockCandidate) -> bool: - nonlocal attempts - attempts += 1 - if attempts <= 4: - server._defer_block_candidate( - PRISM_REJECTION_BACKEND_RPC_UNAVAILABLE, - "temporary RPC outage", - worker="miner-a", - ) - return False - return True - - server.submit_block_candidate = retry_then_succeed # type: ignore[method-assign] - waits: list[float] = [] - with patch.object( - server.stop_event, - "wait", - side_effect=lambda delay: waits.append(delay) or False, - ): - for _attempt in range(5): - server.enqueue_block_candidate(candidate) - self.assertTrue(server.submit_next_block_candidate()) - - self.assertEqual(len(waits), 6) - for observed, expected in zip( - waits, - [0.1, 0.2, 0.25, 0.15, 0.25, 0.15], - strict=True, - ): - self.assertAlmostEqual(observed, expected) - self.assertNotIn(candidate.submission.block_hash_hex, server.block_candidate_retry_delays) - self.assertEqual(server.block_candidate_abandoned_counts, {}) - self.assertEqual(ledger.pending_block_candidates(), []) - - def test_candidate_retry_state_is_cleared_on_terminal_abandonment(self) -> None: - server, state, _recording = submit_coordinator() - ledger = SingleWriterShareLedger() - server.ledger = ledger - pending = self._pending_append("retry-terminal").pending_share - candidate = block_candidate( - server, - state, - SimpleNamespace( - coinbase_tx_hex="00", - block_hash_hex="a2" * 32, - block_hex="00", - share_pass=True, - block_pass=True, - ), - pending_share=pending, - ) - ledger.append_batch([(pending, server.block_candidate_intent(candidate))]) - server.block_candidate_retry_delays = {candidate.submission.block_hash_hex: 2.0} - - def terminal(_candidate: PrismBlockCandidate) -> bool: - block_hash = _candidate.submission.block_hash_hex - block_height = int(_candidate.context.template["height"]) - server._begin_accepted_block_payout_preview( - block_hash, - block_height=block_height, - ) - server._mark_accepted_block_payout_landed( - block_hash, - block_height=block_height, - ) - server._abandon_block_candidate( - PRISM_REJECTION_STALE_JOB, - "tip moved", - block_hash=block_hash, - worker="miner-a", - ) - return False - - server.submit_block_candidate = terminal # type: ignore[method-assign] - server.enqueue_block_candidate(candidate) - - self.assertTrue(server.submit_next_block_candidate()) - - self.assertNotIn(candidate.submission.block_hash_hex, server.block_candidate_retry_delays) - self.assertEqual(server.block_candidate_abandoned_counts[PRISM_REJECTION_STALE_JOB], 1) - self.assertEqual(ledger.pending_block_candidates(), []) - self.assertNotIn( - candidate.submission.block_hash_hex, - server._accepted_block_payout_previews, - ) - self.assertNotIn( - candidate.submission.block_hash_hex, - server._invalidated_accepted_block_payout_previews, - ) - - def test_terminal_abandonment_keeps_tombstone_when_outbox_update_fails( - self, - ) -> None: - server, state, _recording = submit_coordinator() - ledger = SingleWriterShareLedger() - server.ledger = ledger - pending = self._pending_append("terminal-outbox-failure").pending_share - candidate = block_candidate( - server, - state, - SimpleNamespace( - coinbase_tx_hex="00", - block_hash_hex="a3" * 32, - block_hex="00", - share_pass=True, - block_pass=True, - ), - pending_share=pending, - ) - block_hash = candidate.submission.block_hash_hex - block_height = int(candidate.context.template["height"]) - ledger.append_batch([(pending, server.block_candidate_intent(candidate))]) - - def terminal(_candidate: PrismBlockCandidate) -> bool: - server._begin_accepted_block_payout_preview( - block_hash, - block_height=block_height, - ) - server._mark_accepted_block_payout_landed( - block_hash, - block_height=block_height, - ) - server._abandon_block_candidate( - PRISM_REJECTION_STALE_JOB, - "tip moved", - block_hash=block_hash, - worker="miner-a", - ) - return False - - server.submit_block_candidate = terminal # type: ignore[method-assign] - server.enqueue_block_candidate(candidate) - with patch.object( - ledger, - "mark_block_candidate_abandoned", - side_effect=RuntimeError("postgres unavailable"), - ): - self.assertTrue(server.submit_next_block_candidate()) - - self.assertNotIn(block_hash, server._accepted_block_payout_previews) - self.assertIn(block_hash, server._invalidated_accepted_block_payout_previews) - self.assertEqual( - [intent["block_hash_hex"] for intent in ledger.pending_block_candidates()], - [block_hash], - ) - - def test_finalize_failure_replays_with_candidate_backoff(self) -> None: - server, state, _recording = submit_coordinator() - configure_temporary_audit_root(self, server) - ledger = SingleWriterShareLedger() - server.ledger = ledger - pending = self._pending_append("f1").pending_share - candidate = block_candidate( - server, - state, - SimpleNamespace( - coinbase_tx_hex="00", - block_hash_hex="f1" * 32, - block_hex="00", - share_pass=True, - block_pass=True, - ), - pending_share=pending, - ) - candidate_intent = server.block_candidate_intent(candidate) - ledger.append_batch([(pending, candidate_intent)]) - server.block_candidate_retry_initial_seconds = 0.1 - server.block_candidate_retry_max_seconds = 0.4 - # The block itself lands once; only the terminal outbox update fails. - # Replays must pace like any other candidate retry AND run finalize- - # only, never re-entering submission. - submit_calls = 0 - - def accepting_submit(_candidate: PrismBlockCandidate) -> bool: - nonlocal submit_calls - submit_calls += 1 - return True - - server.submit_block_candidate = accepting_submit # type: ignore[method-assign] - original_finish = ledger.mark_block_candidate_submitted - original_mark_attempted = ledger.mark_block_candidate_attempted - finish_attempts = 0 - attempt_marks = 0 - - def mark_attempted(*, block_hash: str) -> bool: - nonlocal attempt_marks - attempt_marks += 1 - return original_mark_attempted(block_hash=block_hash) - - def flaky_finish(*, block_hash: str) -> bool: - nonlocal finish_attempts - finish_attempts += 1 - if finish_attempts <= 4: - raise RuntimeError("ledger unavailable") - return original_finish(block_hash=block_hash) - - ledger.mark_block_candidate_attempted = mark_attempted # type: ignore[method-assign] - ledger.mark_block_candidate_submitted = flaky_finish # type: ignore[method-assign] - waits: list[float] = [] - with patch.object( - server.stop_event, - "wait", - side_effect=lambda delay: waits.append(delay) or False, - ): - for _attempt in range(4): - server.enqueue_block_candidate(candidate) - self.assertTrue(server.submit_next_block_candidate()) - self.assertEqual(ledger.pending_block_candidates(), [candidate_intent]) - server.enqueue_block_candidate(candidate) - self.assertTrue(server.submit_next_block_candidate()) - - # The first accepted finalize failure returns unpaced so the caller - # can refresh the fleet immediately; the ladder starts at the first - # finalize-only replay. - self.assertEqual(len(waits), 4) - for observed, expected in zip( - waits, - [0.1, 0.2, 0.25, 0.15], - strict=True, - ): - self.assertAlmostEqual(observed, expected) - self.assertEqual(finish_attempts, 5) - self.assertEqual(attempt_marks, 1) - self.assertEqual(submit_calls, 1) - self.assertNotIn(candidate.submission.block_hash_hex, server.block_candidate_retry_delays) - self.assertEqual(server.block_candidate_abandoned_counts, {}) - self.assertEqual(ledger.pending_block_candidates(), []) - self.assertEqual(server._block_candidate_finalize_retries, {}) - self.assertIn( - "qbit_prism_block_candidate_retries_total 4", - server.metrics_payload(), - ) - - def test_abandon_finalize_failure_counts_one_abandonment(self) -> None: - server, state, _recording = submit_coordinator() - ledger = SingleWriterShareLedger() - server.ledger = ledger - moved_tip = "11" * 32 - expected_error = f"tip moved before submit: {moved_tip}" - pending = self._pending_append("f2").pending_share - candidate = block_candidate( - server, - state, - SimpleNamespace( - coinbase_tx_hex="00", - block_hash_hex="f2" * 32, - block_hex="00", - share_pass=True, - block_pass=True, - ), - pending_share=pending, - ) - ledger.append_batch([(pending, server.block_candidate_intent(candidate))]) - server.block_candidate_retry_initial_seconds = 0.1 - server.block_candidate_retry_max_seconds = 0.4 - submit_calls = 0 - - def terminal_submit(_candidate: PrismBlockCandidate) -> bool: - nonlocal submit_calls - submit_calls += 1 - server._abandon_block_candidate( - PRISM_REJECTION_STALE_JOB, - expected_error, - block_hash=_candidate.submission.block_hash_hex, - worker="miner-a", - stale_job_class="tip_moved", - ) - return False - - server.submit_block_candidate = terminal_submit # type: ignore[method-assign] - original_finish = ledger.mark_block_candidate_abandoned - finish_attempts = 0 - - def flaky_finish(*, block_hash: str, error: str) -> bool: - nonlocal finish_attempts - finish_attempts += 1 - if finish_attempts <= 2: - raise RuntimeError("ledger unavailable") - return original_finish(block_hash=block_hash, error=error) - - ledger.mark_block_candidate_abandoned = flaky_finish # type: ignore[method-assign] - waits: list[float] = [] - with patch.object( - server.stop_event, - "wait", - side_effect=lambda delay: waits.append(delay) or False, - ): - for _attempt in range(3): - server.enqueue_block_candidate(candidate) - self.assertTrue(server.submit_next_block_candidate()) - - self.assertEqual(waits, [0.1, 0.2]) - self.assertEqual(submit_calls, 1) - self.assertEqual(finish_attempts, 3) - # Finalize-only replays must not recount the terminal abandonment. - self.assertEqual( - server.block_candidate_abandoned_counts, - {PRISM_REJECTION_STALE_JOB: 1}, - ) - self.assertEqual( - server.stale_job_abandon_counts, - {"tip_moved": 1, "balance_stale": 0, "append_epoch_stale": 0}, - ) - self.assertEqual(ledger.pending_block_candidates(), []) - outbox_row = ledger._block_candidate_outbox[ - candidate.submission.block_hash_hex - ] - self.assertEqual(outbox_row["state"], "abandoned") - self.assertEqual(outbox_row["last_error"], expected_error) - self.assertEqual(server._block_candidate_finalize_retries, {}) - - def test_accepted_finalize_failure_still_triggers_post_accept_refresh(self) -> None: - server, state, _recording = submit_coordinator() - ledger = SingleWriterShareLedger() - server.ledger = ledger - pending = self._pending_append("f3").pending_share - candidate = block_candidate( - server, - state, - SimpleNamespace( - coinbase_tx_hex="00", - block_hash_hex="f3" * 32, - block_hex="00", - share_pass=True, - block_pass=True, - ), - pending_share=pending, - ) - ledger.append_batch([(pending, server.block_candidate_intent(candidate))]) - server.block_candidate_retry_initial_seconds = 0.1 - server.block_candidate_retry_max_seconds = 0.4 - server.submit_block_candidate = lambda _candidate: True # type: ignore[method-assign] - - def failing_finish(*, block_hash: str) -> bool: - raise RuntimeError("ledger unavailable") - - ledger.mark_block_candidate_submitted = failing_finish # type: ignore[method-assign] - refreshed_clients: list[object] = [] - server.refresh_jobs_after_pending_accepted_block = ( # type: ignore[method-assign] - lambda client, **_kwargs: refreshed_clients.append(client) or 0 - ) - released_shares: list[object] = [] - original_release = server._finish_pending_share_commit - - def recording_release(pending_share: object) -> None: - released_shares.append(pending_share) - original_release(pending_share) - - server._finish_pending_share_commit = recording_release # type: ignore[method-assign] - waits: list[float] = [] - with patch.object( - server.stop_event, - "wait", - side_effect=lambda delay: waits.append(delay) or False, - ): - server.enqueue_block_candidate(candidate) - self.assertTrue(server.submit_next_block_candidate()) - # The block is active on-chain; the fleet refresh fires on the - # first finalize failure without waiting out a backoff, and the - # snapshot anchor floor is released despite the pending outbox - # mark. - self.assertEqual(refreshed_clients, [state]) - self.assertEqual(waits, []) - self.assertIn(pending, released_shares) - self.assertTrue(server.submit_next_block_candidate()) - self.assertEqual(refreshed_clients, [state]) - self.assertEqual(waits, [0.1]) - - def test_invalid_durable_candidate_is_quarantined_by_outbox_row_key(self) -> None: - for payload_hash in (None, "ff" * 32): - with self.subTest(payload_hash=payload_hash): - server, _state, _recording = submit_coordinator() - configure_temporary_audit_root(self, server) - ledger = SingleWriterShareLedger() - server.ledger = ledger - durable_hash = "de" * 32 - invalid = { - "schema": "unsupported", - "block_hash_hex": durable_hash, - "block_hex": "00", - } - ledger.persist_block_candidate_intent(invalid) - stored = ledger._block_candidate_outbox[durable_hash]["candidate"] - if payload_hash is None: - stored.pop("block_hash_hex") - else: - stored["block_hash_hex"] = payload_hash - server.block_candidate_retry_delays = {durable_hash: 1.0} - - self.assertEqual(server.replay_pending_block_candidates(), 0) - # Malformed-row cleanup is lower-priority maintenance, so it - # cannot form an N x database-timeout convoy ahead of valid - # recovered blocks on the node-offer lane. - self.assertTrue( - server._run_one_invalid_block_candidate_quarantine() - ) - - self.assertEqual(ledger.pending_block_candidates(), []) - self.assertNotIn(durable_hash, server.block_candidate_retry_delays) - self.assertIn( - "qbit_prism_block_candidate_poisoned_total 1", - server.metrics_payload(), - ) - - def test_block_submitter_drops_candidate_when_pool_closed(self) -> None: - server, state, ledger = submit_coordinator() - server.accepted_block_count = 1 - server.max_blocks = 1 - submission = SimpleNamespace( - coinbase_tx_hex="c0ffee", - block_hash_hex="dd" * 32, - block_hex="00", - ) - - accepted = server.submit_block_candidate(block_candidate(server, state, submission)) - - self.assertFalse(accepted) - self.assertEqual(server.block_candidate_abandoned_counts[PRISM_REJECTION_POOL_CLOSED], 1) - self.assertEqual(server.rejection_counts_by_reason[PRISM_REJECTION_POOL_CLOSED], 0) - self.assertEqual(ledger.persisted, []) - - def test_block_worthy_share_is_credited_and_enqueued_before_block_submission(self) -> None: - # The share ack must never wait on the block path: a block-worthy - # share that met its target is credited immediately and the candidate - # is queued for the submitter thread. Nothing submits synchronously - # (the fixture RPC would raise on an unexpected submitblock call). - server, state, ledger = submit_coordinator() - submission = SimpleNamespace( - header_hex="aa" * 80, - block_hash_hex="cc" * 32, - share_pass=True, - block_pass=True, - ) - - with patch( - "lab.prism.prism_coordinator.direct_stratum.assemble_submission", - return_value=submission, - ): - should_close = server.handle_submit( - state, - ["miner-a", "job-1", "00" * 8, "00000001", "00000002"], - ) - - self.assertFalse(should_close) - self.assertEqual(len(ledger.pending), 1) - self.assertIsNone(ledger.pending[0].credit_policy) - self.assertEqual(server.block_candidate_queue.qsize(), 1) - queued = server.block_candidate_queue.get_nowait() - self.assertIs(queued.submission, submission) - self.assertFalse(queued.credit_share_on_accept) - - def test_block_candidate_submits_before_full_audit_persistence(self) -> None: - server, state, ledger = submit_coordinator() - server._ensure_job_cache_state() - server._ensure_tip_refresh_state() - stale_bundle_key = ("stale-payout-bundle",) - server._job_bundle_cache[stale_bundle_key] = object() # type: ignore[assignment] - active_fanout = _FanoutCancellation() - server._active_tip_refresh = ( # type: ignore[assignment] - SimpleNamespace(payout_state_generation=0), - active_fanout, - ) - with tempfile.TemporaryDirectory() as tempdir: - server.audit_dir = Path(tempdir) - server.evidence_path = Path(tempdir) / "evidence.json" - server.ledger_writer_public_key_hex = "aa" * 32 - block_hash = "cc" * 32 - rpc = SubmitRpc(tip="00" * 32, block_hash=block_hash, ledger=ledger) - server.rpc = rpc - witness_tx = synthetic_witness_transaction("44") - server.jobs["job-1"].job.transaction_hexes = (witness_tx,) - build_kwargs: list[dict[str, object]] = [] - # Deliberately use insertion order that differs from this fixture's - # stand-in canonical serialization. The alternate builder creates - # the requested output but writes pretty, noncanonical JSON, so - # existence alone must not make the path eligible for persistence. - alternate_bundle = { - "signed_coinbase_manifest": { - "manifest": { - "coinbase_tx_hex": "c0ffee", - "payout_count": 1, - } - }, - "payout_policy_manifest": {"accounts": []}, - "ledger_window_attestation": {"signature": {"public_key_hex": "aa" * 32}}, - "found_block": {"coinbase_value_sats": 50_00000000}, - } - canonical_bytes = json.dumps( - alternate_bundle, - sort_keys=True, - separators=(",", ":"), - ).encode() - canonical_sha256 = hashlib.sha256(canonical_bytes).hexdigest() - - def fake_build_audit_bundle(**kwargs: object) -> dict[str, object]: - build_kwargs.append(kwargs) - output_path = kwargs["canonical_output_path"] - assert isinstance(output_path, Path) - adopter = kwargs["canonical_output_adopter"] - assert callable(adopter) - with output_path.open("x+", encoding="utf-8") as output: - output.write(json.dumps(alternate_bundle, indent=2)) - output.flush() - os.fsync(output.fileno()) - adopter(output_path, os.fstat(output.fileno())) - return alternate_bundle - - def fake_verify_bundle(bundle_path: Path, *_args: object, **_kwargs: object) -> dict[str, object]: - candidate_bytes = bundle_path.read_bytes() - self.assertEqual(json.loads(candidate_bytes), alternate_bundle) - self.assertNotEqual(candidate_bytes, canonical_bytes) - self.assertNotEqual( - hashlib.sha256(candidate_bytes).hexdigest(), - canonical_sha256, - ) - return { - **verified_audit_report(), - "audit_bundle_sha256_hex": canonical_sha256, - } - - persist_accepted_block = ledger.persist_accepted_block - - def persist_with_canonicalization(**kwargs: object) -> dict[str, object]: - self.assertIsNone(kwargs["canonical_bundle_path"]) - self.assertEqual(kwargs["final_bundle"], alternate_bundle) - report = kwargs["audit_report"] - assert isinstance(report, dict) - self.assertEqual(report["audit_bundle_sha256_hex"], canonical_sha256) - return persist_accepted_block(**kwargs) - - server.build_audit_bundle = fake_build_audit_bundle # type: ignore[method-assign] - server.verify_bundle = fake_verify_bundle # type: ignore[method-assign] - ledger.persist_accepted_block = persist_with_canonicalization # type: ignore[method-assign] - submission = SimpleNamespace( - coinbase_tx_hex="c0ffee", - block_hash_hex=block_hash, - block_hex="00", - ) - pending = SimpleNamespace(share_id="miner-a:" + block_hash) - - accepted = server.submit_block_candidate( - block_candidate(server, state, submission, pending_share=pending) - ) - self.assertTrue(accepted) - self.assertTrue( - server._ensure_shutdown_controller().writer_admission_closed() - ) - - live_files = sorted(Path(tempdir).glob("prism-live-audit-bundle-[0-9]*.json")) - self.assertEqual(len(live_files), 1) - self.assertEqual(list(Path(tempdir).glob("prism-live-audit-bundle-candidate-*.json")), []) - self.assertEqual(list(Path(tempdir).glob(".prism-live-audit-bundle-candidate-*.tmp")), []) - envelope = json.loads(live_files[0].read_text(encoding="utf-8")) - self.assertEqual(envelope["schema"], "qbit.prism.live-audit-bundle-envelope.v1") - self.assertEqual(envelope["block_hash"], block_hash) - self.assertEqual(envelope["block_height"], 10) - self.assertEqual(envelope["audit_bundle_sha256"], canonical_sha256) - self.assertNotIn("signed_coinbase_manifest", envelope) - self.assertEqual( - Path(server.latest_evidence["audit_bundle_path"]), - live_files[0].resolve(), - ) - - self.assertTrue(rpc.submitted) - self.assertEqual( - build_kwargs[0]["witness_merkle_leaves_hex"], - direct_stratum.witness_merkle_leaves_hex((witness_tx,)), - ) - self.assertEqual( - build_kwargs[0]["coinbase_script_sig_suffix_hex"], - server.coinbase_tag_hex + state.extranonce1_hex + "00" * 8, - ) - self.assertEqual(ledger.persisted[0]["block_hash"], block_hash) - self.assertEqual(ledger.persisted[0]["block_height"], 10) - self.assertTrue(ledger.persisted[0]["submit_seen_at_persist"]) - # This alternate test builder wrote noncanonical bytes to the requested - # path. The verified content is valid, but the path is never claimed as - # a byte-canonical artifact for ledger persistence. - self.assertIsNone(ledger.persisted[0]["canonical_bundle_path"]) - self.assertEqual(server._payout_state_generation, 1) - self.assertEqual(server._job_bundle_cache, {}) - self.assertTrue(active_fanout.is_set()) - self.assertTrue(server._tip_refresh_retry.is_set()) - - def test_issued_preview_is_invalidated_when_final_coinbase_mismatches(self) -> None: - server, state, ledger = submit_coordinator() - server._ensure_job_cache_state() - block_hash = "cf" * 32 - server.rpc = SubmitRpc(tip="00" * 32, block_hash=block_hash, ledger=ledger) - server.build_audit_bundle = ( # type: ignore[method-assign] - lambda **_kwargs: verified_block_bundle("deadbeef") - ) - submission = SimpleNamespace( - coinbase_tx_hex="c0ffee", - block_hash_hex=block_hash, - block_hex="00", - ) - candidate = block_candidate(server, state, submission) - candidate.context.prospective_prior_balances = () - - with tempfile.TemporaryDirectory() as tempdir: - server.audit_dir = Path(tempdir) - self.assertFalse(server.submit_block_candidate(candidate)) - - self.assertEqual(server._payout_state_generation, 2) - self.assertEqual(server._accepted_block_payout_previews, {}) - self.assertEqual(ledger.persisted, []) - self.assertEqual( - server._block_candidate_outcome.reason, - PRISM_REJECTION_CANDIDATE_AUDIT_MISMATCH, - ) - - def test_accepted_block_persistence_allows_delivery_but_serializes_reconciliation( - self, - ) -> None: - server, state, ledger = submit_coordinator() - server._ensure_job_cache_state() - persist_started = threading.Event() - persist_finished = threading.Event() - release_persist = threading.Event() - confirm_started = threading.Event() - release_delivery = threading.Event() - reconcile_lock_attempted = threading.Event() - reconcile_mutated = threading.Event() - event_lock = threading.Lock() - events: list[str] = [] - original_persist = ledger.persist_accepted_block - original_confirm = ledger.confirm_accepted_block - - def note(name: str) -> None: - with event_lock: - events.append(name) - - def blocking_persist(**kwargs: object) -> dict[str, object]: - note("persist-start") - persist_started.set() - if not release_persist.wait(15): - raise AssertionError("timed out waiting to release accepted-block persistence") - result = original_persist(**kwargs) - note("persist-end") - persist_finished.set() - return result - - def observed_confirm(**kwargs: object) -> dict[str, object]: - note("confirm") - confirm_started.set() - return original_confirm(**kwargs) - - ledger.reorg_watch_blocks = lambda *, active_tip_height: [ # type: ignore[method-assign] - { - "block_height": active_tip_height - 2, - "block_hash": "aa" * 32, - "chain_state": "confirmed", - } - ] - - def mark_pool_block_inactive(**_kwargs: object) -> dict[str, object]: - note("reconcile-mutation") - reconcile_mutated.set() - return {"backend": "fake", "inactive_count": 1} - - ledger.persist_accepted_block = blocking_persist # type: ignore[method-assign] - ledger.confirm_accepted_block = observed_confirm # type: ignore[method-assign] - ledger.mark_pool_block_inactive = mark_pool_block_inactive # type: ignore[method-assign] - accepted: list[bool] = [] - reconcile_results: list[dict[str, object]] = [] - errors: list[BaseException] = [] - delivery_admitted = threading.Event() - delivery_thread: threading.Thread | None = None - reconcile_thread: threading.Thread | None = None - mutation_lock = server._payout_balance_mutation_lock - - class ObservedBalanceLock: - def __enter__(self) -> ObservedBalanceLock: - if threading.current_thread() is reconcile_thread: - reconcile_lock_attempted.set() - mutation_lock.acquire() - return self - - def __exit__(self, *_args: object) -> None: - mutation_lock.release() - - # The landing's audit-build release window drives the serializer - # through the plain lock interface as well. - def acquire(self, *args: object, **kwargs: object) -> bool: - return mutation_lock.acquire(*args, **kwargs) # type: ignore[arg-type] - - def release(self) -> None: - mutation_lock.release() - - server._payout_balance_mutation_lock = ObservedBalanceLock() # type: ignore[assignment] - - with tempfile.TemporaryDirectory() as tempdir: - server.audit_dir = Path(tempdir) - server.evidence_path = Path(tempdir) / "evidence.json" - server.ledger_writer_public_key_hex = "aa" * 32 - block_hash = "cd" * 32 - server.rpc = SubmitRpc( - tip="00" * 32, - block_hash=block_hash, - ledger=ledger, - ) - server.build_audit_bundle = ( # type: ignore[method-assign] - lambda **_kwargs: verified_block_bundle() - ) - server.verify_bundle = ( # type: ignore[method-assign] - lambda *_args, **_kwargs: verified_audit_report() - ) - submission = SimpleNamespace( - coinbase_tx_hex="c0ffee", - block_hash_hex=block_hash, - block_hex="00", - ) - candidate = block_candidate(server, state, submission) - - def submit() -> None: - try: - accepted.append(server.submit_block_candidate(candidate)) - except BaseException as exc: # noqa: BLE001 - asserted below - errors.append(exc) - - def deliver() -> None: - try: - with server._payout_state_delivery_gate.delivery(): - note("replacement-delivery") - delivery_admitted.set() - if not release_delivery.wait(15): - raise AssertionError( - "timed out waiting to release payout delivery" - ) - except BaseException as exc: # noqa: BLE001 - asserted below - errors.append(exc) - - def reconcile() -> None: - try: - reconcile_results.append( - server.reconcile_prism_pool_blocks_once(tip_hash=block_hash) - ) - except BaseException as exc: # noqa: BLE001 - asserted below - errors.append(exc) - - submit_thread = threading.Thread(target=submit) - submit_thread.start() - confirmation_waited_for_delivery = False - try: - self.assertTrue(persist_started.wait(5)) - server.reorg_reconciler_enabled = True - reconcile_thread = threading.Thread(target=reconcile) - reconcile_thread.start() - self.assertTrue(reconcile_lock_attempted.wait(5)) - self.assertFalse(reconcile_mutated.is_set()) - delivery_thread = threading.Thread(target=deliver) - delivery_thread.start() - admitted_while_persisting = delivery_admitted.wait(5) - if admitted_while_persisting: - self.assertFalse(reconcile_mutated.is_set()) - release_persist.set() - self.assertTrue(persist_finished.wait(5)) - confirmation_waited_for_delivery = not confirm_started.wait(0.1) - finally: - release_persist.set() - release_delivery.set() - submit_thread.join(10) - if delivery_thread is not None: - delivery_thread.join(10) - if reconcile_thread is not None: - reconcile_thread.join(10) - - self.assertFalse(submit_thread.is_alive()) - self.assertIsNotNone(delivery_thread) - self.assertFalse(delivery_thread.is_alive()) - self.assertIsNotNone(reconcile_thread) - self.assertFalse(reconcile_thread.is_alive()) - if errors: - raise errors[0] - self.assertTrue(admitted_while_persisting) - # Confirmation is durable catch-up to the already-published - # prospective state and therefore does not wait on delivery. - self.assertFalse(confirmation_waited_for_delivery) - self.assertTrue(confirm_started.is_set()) - self.assertTrue(reconcile_mutated.is_set()) - self.assertEqual(accepted, [True]) - self.assertEqual(len(reconcile_results), 1) - self.assertEqual(reconcile_results[0]["inactive_blocks"], 1) - self.assertEqual(server._payout_state_generation, 2) - self.assertLess(events.index("persist-start"), events.index("replacement-delivery")) - self.assertLess(events.index("replacement-delivery"), events.index("persist-end")) - self.assertLess(events.index("persist-end"), events.index("confirm")) - self.assertLess(events.index("confirm"), events.index("reconcile-mutation")) - - def test_next_tip_preview_job_lands_after_parent_persistence(self) -> None: - old_tip = "00" * 32 - parent_hash = "cd" * 32 - child_hash = "ce" * 32 - server, state, ledger = submit_coordinator(tip=old_tip) - server._ensure_job_cache_state() - ledger.durable_payout_state = True - server.reorg_reconciler_enabled = False - server.stop_after_block = False - server.max_blocks = 10 - server.clients = {state} - server.refresh_jobs_after_accepted_block = ( # type: ignore[method-assign] - lambda **_kwargs: None - ) - build_started = threading.Event() - release_build = threading.Event() - persist_started = threading.Event() - release_persist = threading.Event() - original_persist = ledger.persist_accepted_block - original_confirm = ledger.confirm_accepted_block - preview = [ - { - "recipient_id": "miner-a", - "order_key": "miner-a", - "p2mr_program_hex": "11" * 32, - "balance_sats": 25, - } - ] - - def payout_bundle_payload() -> dict[str, object]: - return { - "found_block": {"coinbase_value_sats": 50_00000000}, - "ledger_window_attestation": { - "signature": {"public_key_hex": "aa" * 32} - }, - "payout_policy_manifest": { - "accounts": [ - { - "recipient_id": "miner-a", - "order_key": "miner-a", - "p2mr_program_hex": "11" * 32, - "gross_amount_sats": 25, - "prior_balance_sats": 0, - "candidate_balance_sats": 25, - "onchain_amount_sats": 0, - "carry_forward_balance_sats": 25, - "action": "accrued", - } - ] - }, - "signed_coinbase_manifest": { - "manifest": {"coinbase_tx_hex": "c0ffee", "payout_count": 1} - }, - } - - def blocking_payout_bundle(**_kwargs: object) -> dict[str, object]: - if not build_started.is_set(): - build_started.set() - if not release_build.wait(15): - raise AssertionError("timed out waiting to release parent bundle build") - return payout_bundle_payload() - - def blocking_persist(**kwargs: object) -> dict[str, object]: - if str(kwargs["block_hash"]).lower() == parent_hash: - persist_started.set() - if not release_persist.wait(15): - raise AssertionError("timed out waiting to release parent persistence") - return original_persist(**kwargs) - - def expose_confirmed_preview(**kwargs: object) -> dict[str, object]: - persisted = next( - row - for row in reversed(ledger.persisted) - if str(row["block_hash"]).lower() - == str(kwargs["block_hash"]).lower() - ) - ledger.prior_balances = server._accepted_block_payout_preview_from_bundle( - persisted["final_bundle"] # type: ignore[arg-type] - ) - return original_confirm(**kwargs) - - ledger.persist_accepted_block = blocking_persist # type: ignore[method-assign] - ledger.confirm_accepted_block = expose_confirmed_preview # type: ignore[method-assign] - - class TwoBlockRpc: - def __init__(self) -> None: - self.tip = old_tip - self.height = 9 - self.hashes = {9: old_tip} - self.submitted: list[str] = [] - - def call(self, method: str, params: object = None) -> object: - if method == "getbestblockhash": - return self.tip - if method == "getblockcount": - return self.height - if method == "submitblock": - block_hex = str((params or [""])[0]) # type: ignore[index] - block_hash = parent_hash if block_hex == "00" else child_hash - self.height += 1 - self.tip = block_hash - self.hashes[self.height] = block_hash - self.submitted.append(block_hash) - ledger.submit_seen = True - return None - if method == "getblockhash": - return self.hashes[int((params or [0])[0])] # type: ignore[index] - raise RuntimeError(method) - - rpc = TwoBlockRpc() - server.rpc = rpc - server.build_audit_bundle = blocking_payout_bundle # type: ignore[method-assign] - server.verify_bundle = ( # type: ignore[method-assign] - lambda *_args, **kwargs: verified_audit_report( - block_height=int(kwargs["expected_block_height"]) - ) - ) - sent: list[dict[str, object]] = [] - state.send = lambda payload: sent.append(payload) # type: ignore[method-assign] - - def build_child_job(client: ClientState, *, clean_jobs: bool) -> object: - context = prism_context( - "child-job", - parent_hash, - worker=client.worker, - clean_jobs=clean_jobs, - ) - context.template["height"] = 11 - context.prior_balances = server._prior_balances_for_job_parent(parent_hash) - context.payout_state_generation = server._payout_state_generation - return context - - server.build_job_for_client = build_child_job # type: ignore[method-assign] - parent_submission = SimpleNamespace( - coinbase_tx_hex="c0ffee", - block_hash_hex=parent_hash, - block_hex="00", - ) - parent_candidate = block_candidate(server, state, parent_submission) - parent_candidate.context.prospective_prior_balances = ( - server._serialize_prior_balance_preview(preview) - ) - parent_results: list[bool] = [] - parent_errors: list[BaseException] = [] - - def submit_parent() -> None: - try: - parent_results.append(server.submit_block_candidate(parent_candidate)) - except BaseException as exc: # noqa: BLE001 - asserted below - parent_errors.append(exc) - - with tempfile.TemporaryDirectory() as tempdir: - server.audit_dir = Path(tempdir) - server.evidence_path = Path(tempdir) / "evidence.json" - server.ledger_writer_public_key_hex = "aa" * 32 - parent_thread = threading.Thread(target=submit_parent) - parent_thread.start() - try: - self.assertTrue(build_started.wait(5)) - self.assertEqual(ledger.current_prior_balances(), []) - self.assertTrue(server.maybe_send_job(state, clean_jobs=True)) - child_context = state.active_job - self.assertIsNotNone(child_context) - assert child_context is not None - self.assertTrue(parent_thread.is_alive()) - self.assertEqual(child_context.prior_balances, preview) - self.assertEqual( - child_context.payout_state_generation, - server._payout_state_generation, - ) - preview_generation = server._payout_state_generation - self.assertEqual(sent[-1]["method"], "mining.notify") - self.assertTrue(sent[-1]["params"][8]) # type: ignore[index] - - child_submission = SimpleNamespace( - header_hex="bb" * 80, - coinbase_tx_hex="c0ffee", - block_hash_hex=child_hash, - block_hex="01", - share_pass=True, - block_pass=True, - ) - with patch( - "lab.prism.prism_coordinator.direct_stratum.assemble_submission", - return_value=child_submission, - ): - self.assertFalse( - server.handle_submit( - state, - ["miner-a", "child-job", "00" * 8, "00000001", "00000002"], - ) - ) - self.assertEqual(server.block_candidate_queue.qsize(), 1) - release_build.set() - self.assertTrue(persist_started.wait(5)) - self.assertTrue(parent_thread.is_alive()) - finally: - release_build.set() - release_persist.set() - parent_thread.join(10) - - self.assertFalse(parent_thread.is_alive()) - if parent_errors: - raise parent_errors[0] - self.assertEqual(parent_results, [True]) - self.assertEqual(ledger.current_prior_balances(), preview) - self.assertEqual(server._payout_state_generation, preview_generation) - self.assertTrue(server.submit_next_block_candidate()) - - self.assertEqual(rpc.submitted, [parent_hash, child_hash]) - self.assertEqual(server.accepted_block_count, 2) - self.assertEqual(len(ledger.persisted), 2) - self.assertEqual(len(ledger.confirmed), 2) - self.assertEqual( - server.block_candidate_abandoned_counts.get(PRISM_REJECTION_STALE_JOB, 0), - 0, - ) - self.assertEqual(server._accepted_block_payout_previews, {}) - - def test_direct_block_preparation_does_not_hold_delivery_gate(self) -> None: - server, state, ledger = submit_coordinator() - server._ensure_job_cache_state() - entered = threading.Event() - release = threading.Event() - accepted: list[bool] = [] - errors: list[BaseException] = [] - block_hash = "cf" * 32 - original_persist = ledger.persist_accepted_block - - def blocking_persist(**kwargs: object) -> dict[str, object]: - self.assertIsNone( - server._payout_state_delivery_gate._mutation_owner - ) - entered.set() - if not release.wait(5): - raise AssertionError("test did not release direct-block preparation") - return original_persist(**kwargs) - - ledger.persist_accepted_block = blocking_persist # type: ignore[method-assign] - with tempfile.TemporaryDirectory() as tempdir: - server.audit_dir = Path(tempdir) - server.evidence_path = Path(tempdir) / "evidence.json" - server.ledger_writer_public_key_hex = "aa" * 32 - server.rpc = SubmitRpc( - tip="00" * 32, - block_hash=block_hash, - ledger=ledger, - ) - server.build_audit_bundle = ( # type: ignore[method-assign] - lambda **_kwargs: verified_block_bundle() - ) - server.verify_bundle = ( # type: ignore[method-assign] - lambda *_args, **_kwargs: verified_audit_report() - ) - submission = SimpleNamespace( - coinbase_tx_hex="c0ffee", - block_hash_hex=block_hash, - block_hex="00", - ) - - def submit() -> None: - try: - accepted.append( - server.submit_block_candidate( - block_candidate(server, state, submission) - ) - ) - except BaseException as exc: # pragma: no cover - asserted below - errors.append(exc) - - thread = threading.Thread(target=submit) - thread.start() - try: - self.assertTrue(entered.wait(5)) - with server._payout_state_delivery_gate.delivery_cancelable( - lambda: False, - generation=server._payout_state_generation, - priority=True, - ) as admission: - self.assertTrue(admission) - release.set() - thread.join(5) - self.assertFalse(thread.is_alive()) - self.assertTrue( - server._payout_state_prepare_lock.acquire(timeout=1) - ) - server._payout_state_prepare_lock.release() - admission.mark_delivered() - finally: - release.set() - thread.join(5) - - self.assertFalse(thread.is_alive()) - self.assertEqual(errors, []) - self.assertEqual(accepted, [True]) - self.assertEqual(server._payout_state_generation, 1) - - def test_failed_direct_block_audit_does_not_reserve_payout_source(self) -> None: - for failure_phase in ("build", "verify"): - with self.subTest(failure_phase=failure_phase): - server, state, ledger = submit_coordinator() - server._ensure_job_cache_state() - initial_source = server._payout_state_source - initial_published = server._published_payout_state - block_hash = "ce" * 32 - server.rpc = SubmitRpc( - tip="00" * 32, - block_hash=block_hash, - ledger=ledger, - ) - submission = SimpleNamespace( - coinbase_tx_hex="c0ffee", - block_hash_hex=block_hash, - block_hex="00", - ) - - with tempfile.TemporaryDirectory() as tempdir: - server.audit_dir = Path(tempdir) - server.ledger_writer_public_key_hex = "aa" * 32 - if failure_phase == "build": - server.build_audit_bundle = ( # type: ignore[method-assign] - lambda **_kwargs: (_ for _ in ()).throw( - RuntimeError("audit reconstruction failed") - ) - ) - else: - server.build_audit_bundle = ( # type: ignore[method-assign] - lambda **_kwargs: verified_block_bundle() - ) - server.verify_bundle = ( # type: ignore[method-assign] - lambda *_args, **_kwargs: (_ for _ in ()).throw( - RuntimeError("audit verification failed") - ) - ) - - with self.assertRaisesRegex( - RuntimeError, - "audit (reconstruction|verification) failed", - ): - server.submit_block_candidate( - block_candidate(server, state, submission) - ) - - self.assertEqual(server._payout_state_source, initial_source) - self.assertEqual(server._published_payout_state, initial_published) - self.assertEqual(server._payout_state_generation, 0) - self.assertEqual(ledger.persisted, []) - - def test_uncertain_direct_block_ledger_commit_fences_delivery(self) -> None: - server, state, ledger = submit_coordinator() - server._ensure_job_cache_state() - block_hash = "cd" * 32 - server.rpc = SubmitRpc( - tip="00" * 32, - block_hash=block_hash, - ledger=ledger, - ) - server.build_audit_bundle = ( # type: ignore[method-assign] - lambda **_kwargs: verified_block_bundle() - ) - server.verify_bundle = ( # type: ignore[method-assign] - lambda *_args, **_kwargs: verified_audit_report() - ) - ledger.confirm_accepted_block = ( # type: ignore[method-assign] - lambda **_kwargs: (_ for _ in ()).throw( - RuntimeError("ledger confirmation unavailable") - ) - ) - submission = SimpleNamespace( - coinbase_tx_hex="c0ffee", - block_hash_hex=block_hash, - block_hex="00", - ) - - with tempfile.TemporaryDirectory() as tempdir: - server.audit_dir = Path(tempdir) - server.ledger_writer_public_key_hex = "aa" * 32 - with self.assertRaisesRegex( - RuntimeError, - "ledger confirmation unavailable", - ): - server.submit_block_candidate( - block_candidate(server, state, submission) - ) - - self.assertEqual(len(ledger.persisted), 1) - # The prospective preview was source/generation 1; the uncertain - # durable commit supersedes it with fenced source 2. - self.assertEqual(server._payout_state_generation, 1) - self.assertEqual(server._payout_state_source[0], 2) - self.assertEqual(server._published_payout_state.source_generation, 1) - self.assertTrue(server._payout_state_publication_blocked) - self.assertTrue(server._payout_state_delivery_gate._delivery_blocked) - - def test_uncertain_commit_supersedes_concurrently_published_source(self) -> None: - server, state, ledger = submit_coordinator() - server._ensure_job_cache_state() - block_hash = "cb" * 32 - newer_tip = "dc" * 32 - server.rpc = SubmitRpc( - tip="00" * 32, - block_hash=block_hash, - ledger=ledger, - ) - server.build_audit_bundle = ( # type: ignore[method-assign] - lambda **_kwargs: verified_block_bundle() - ) - server.verify_bundle = ( # type: ignore[method-assign] - lambda *_args, **_kwargs: verified_audit_report() - ) - - def publish_newer_source_then_fail(**_kwargs: object) -> dict[str, object]: - server._reserve_payout_state_source( - "external_tip", - tip_hash=newer_tip, - ) - self.assertEqual( - server._publish_payout_state_candidate( - server._current_payout_state_candidate() - ), - 2, - ) - raise RuntimeError("ledger confirmation unavailable") - - ledger.confirm_accepted_block = publish_newer_source_then_fail # type: ignore[method-assign] - submission = SimpleNamespace( - coinbase_tx_hex="c0ffee", - block_hash_hex=block_hash, - block_hex="00", - ) - - with tempfile.TemporaryDirectory() as tempdir: - server.audit_dir = Path(tempdir) - server.ledger_writer_public_key_hex = "aa" * 32 - with self.assertRaisesRegex( - RuntimeError, - "ledger confirmation unavailable", - ): - server.submit_block_candidate( - block_candidate(server, state, submission) - ) - - self.assertEqual(server._payout_state_generation, 2) - self.assertEqual(server._published_payout_state.source_generation, 2) - self.assertEqual(server._payout_state_source[0], 3) - self.assertEqual(server._payout_state_source[1], newer_tip) - self.assertEqual( - server._payout_state_source[2], - "direct_block_uncertain", - ) - self.assertTrue(server._payout_state_publication_blocked) - self.assertTrue(server._payout_state_delivery_gate._delivery_blocked) - - def test_post_confirm_publication_loss_completes_candidate_and_fences(self) -> None: - # Once persist + confirm are durable, losing the forced payout - # publication must not abort the candidate: the outbox row is marked - # submitted and the success tail runs, while delivery stays fenced - # until the scheduled refresh publishes the newest source. - server, state, ledger = submit_coordinator() - server._ensure_job_cache_state() - server.max_blocks = 2 - server.stop_after_block = False - block_hash = "e1" * 32 - server.rpc = SubmitRpc( - tip="00" * 32, - block_hash=block_hash, - ledger=ledger, - ) - server.build_audit_bundle = ( # type: ignore[method-assign] - lambda **_kwargs: verified_block_bundle() - ) - server.verify_bundle = ( # type: ignore[method-assign] - lambda *_args, **_kwargs: verified_audit_report() - ) - submitted: list[dict[str, object]] = [] - ledger.mark_block_candidate_submitted = ( # type: ignore[attr-defined] - lambda **kwargs: submitted.append(kwargs) or True - ) - real_confirm = ledger.confirm_accepted_block - - def confirm_then_reserve_newer_source(**kwargs: object) -> dict[str, object]: - result = real_confirm(**kwargs) - server._reserve_payout_state_source( - "external_tip", - tip_hash="ee" * 32, - ) - return result - - ledger.confirm_accepted_block = confirm_then_reserve_newer_source # type: ignore[method-assign] - server._publish_current_payout_state_with_retry_budget = ( # type: ignore[method-assign] - lambda **_kwargs: None - ) - submission = SimpleNamespace( - coinbase_tx_hex="c0ffee", - block_hash_hex=block_hash, - block_hex="00", - ) - - with tempfile.TemporaryDirectory() as tempdir: - server.audit_dir = Path(tempdir) - server.evidence_path = Path(tempdir) / "evidence.json" - server.ledger_writer_public_key_hex = "aa" * 32 - self.assertTrue( - server._submit_next_block_candidate_writer( - block_candidate(server, state, submission) - ) - ) - - self.assertIsNone(getattr(server, "_retry_block_candidate", None)) - self.assertEqual(len(ledger.confirmed), 1) - self.assertEqual(len(submitted), 1) - self.assertEqual(submitted[0]["block_hash"], block_hash) - self.assertEqual(server.accepted_block_count, 1) - self.assertIn(block_hash, server._accounted_accepted_block_hashes) - self.assertTrue(server._payout_state_publication_blocked) - self.assertTrue(server._payout_state_delivery_gate._delivery_blocked) - self.assertTrue(server.tip_refresh_is_pending()) - - # The scheduled refresh publishes the pending source and reopens - # delivery without any candidate replay. - del server._publish_current_payout_state_with_retry_budget - self.assertIsNotNone(server._publish_current_payout_state_with_retry_budget()) - self.assertFalse(server._payout_state_publication_blocked) - self.assertFalse(server._payout_state_delivery_gate._delivery_blocked) - - def test_idempotent_direct_block_replay_skips_publication(self) -> None: - server, state, ledger = submit_coordinator() - server._ensure_job_cache_state() - server.max_blocks = 2 - server.stop_after_block = False - block_hash = "d0" * 32 - with tempfile.TemporaryDirectory() as tempdir: - server.audit_dir = Path(tempdir) - server.evidence_path = Path(tempdir) / "evidence.json" - server.ledger_writer_public_key_hex = "aa" * 32 - rpc = SubmitRpc( - tip="00" * 32, - block_hash=block_hash, - ledger=ledger, - ) - server.rpc = rpc - server.build_audit_bundle = ( # type: ignore[method-assign] - lambda **_kwargs: verified_block_bundle() - ) - server.verify_bundle = ( # type: ignore[method-assign] - lambda *_args, **_kwargs: verified_audit_report() - ) - submission = SimpleNamespace( - coinbase_tx_hex="c0ffee", - block_hash_hex=block_hash, - block_hex="00", - ) - - self.assertTrue( - server.submit_block_candidate( - block_candidate(server, state, submission) - ) - ) - self.assertEqual(server._payout_state_generation, 1) - self.assertEqual( - server._published_payout_state.source_tip_hash, - block_hash, - ) - self.assertEqual(server._payout_state_source[0], 1) - - # Replay the durable candidate after its block landed, its - # confirmation committed, and the network built on top of it. - # Exact confirmed replay reports confirmed_count=1 at the block's - # own expected height even though the active chain tip is now a - # child. The published payout state already covers the - # candidate, so the replay must not reserve a source, bump the - # generation, wipe the job-bundle cache, or schedule refresh - # churn. - child_tip = "d7" * 32 - - class AncestorReplayRpc: - def call(self, method: str, params: object = None) -> object: - if method == "getbestblockhash": - return child_tip - if method == "getblockheader": - if params != [block_hash]: - raise AssertionError(params) - return {"height": 10, "confirmations": 2} - if method == "getblockhash": - if params != [10]: - raise AssertionError(params) - return block_hash - if method == "getblockcount": - return 11 - if method == "submitblock": - raise AssertionError( - "active ancestor must not be resubmitted" - ) - raise RuntimeError(method) - - server.rpc = AncestorReplayRpc() - ledger.pool_block_state = ( # type: ignore[attr-defined] - lambda **_kwargs: { - "chain_state": "confirmed", - "maturity_state": "immature", - } - ) - def confirm_exact_ancestor(**kwargs: object) -> dict[str, object]: - self.assertEqual(kwargs["block_hash"], block_hash) - self.assertEqual(kwargs["active_tip_height"], 10) - return {"backend": "fake", "confirmed_count": 1} - - ledger.confirm_accepted_block = confirm_exact_ancestor # type: ignore[method-assign] - retry_calls = 0 - - def count_retry() -> None: - nonlocal retry_calls - retry_calls += 1 - - server._schedule_tip_refresh_retry = count_retry # type: ignore[method-assign] - cache_key = ("sentinel",) - server._job_bundle_cache[cache_key] = object() - discarded_before = server.payout_state_candidates_discarded - pending_marks_before = server._tip_refresh_pending_counter - - self.assertTrue( - server.submit_block_candidate( - block_candidate(server, state, submission) - ) - ) - - self.assertEqual(server._payout_state_generation, 1) - self.assertEqual(server._published_payout_state.source_generation, 1) - self.assertEqual(server._published_payout_state.source_tip_hash, block_hash) - self.assertEqual(server._payout_state_source[0], 1) - self.assertIn(cache_key, server._job_bundle_cache) - self.assertEqual(retry_calls, 0) - self.assertEqual( - server._tip_refresh_pending_counter, - pending_marks_before, - ) - self.assertEqual( - server.payout_state_candidates_discarded, - discarded_before, - ) - self.assertFalse(server._payout_state_publication_blocked) - self.assertFalse(server._payout_state_delivery_gate._delivery_blocked) - - def test_at_tip_idempotent_confirm_replay_skips_publication(self) -> None: - """An at-tip replay reports the idempotent disposition, not a flip. - - qbit_confirm_pool_block returns 2 when its UPDATE matched nothing but - the row is already confirmed at that (hash, height): the call changed - nothing and allocated no publication ordinal. Before that split the - same replay reported 1 and was indistinguishable from a fresh flip, - so the window after finding a block -- where the candidate replays - while its own block is still the best tip -- republished payout state - on every pass. The replay must complete the candidate without - reserving a source, bumping the generation, wiping the job-bundle - cache or scheduling refresh churn, and must never be mistaken for a - failed confirmation, which fences delivery, republishes a withdrawal - and halts the pool (issue #61). - """ - server, state, ledger = submit_coordinator() - server._ensure_job_cache_state() - server.max_blocks = 2 - server.stop_after_block = False - block_hash = "da" * 32 - with tempfile.TemporaryDirectory() as tempdir: - server.audit_dir = Path(tempdir) - server.evidence_path = Path(tempdir) / "evidence.json" - server.ledger_writer_public_key_hex = "aa" * 32 - server.rpc = SubmitRpc( - tip="00" * 32, - block_hash=block_hash, - ledger=ledger, - ) - server.build_audit_bundle = ( # type: ignore[method-assign] - lambda **_kwargs: verified_block_bundle() - ) - server.verify_bundle = ( # type: ignore[method-assign] - lambda *_args, **_kwargs: verified_audit_report() - ) - submission = SimpleNamespace( - coinbase_tx_hex="c0ffee", - block_hash_hex=block_hash, - block_hex="00", - ) - - self.assertTrue( - server.submit_block_candidate( - block_candidate(server, state, submission) - ) - ) - self.assertEqual(server._payout_state_generation, 1) - self.assertEqual( - server._published_payout_state.source_tip_hash, - block_hash, - ) - self.assertEqual(server._payout_state_source[0], 1) - flipped_sequence = int( - ledger._audit_publication_sequences[block_hash] - ) - - # Replay the durable candidate while its own block is still the - # best tip -- no successor has arrived, so nothing about the - # active chain distinguishes this pass from the original landing. - - class AtTipReplayRpc: - def call(self, method: str, params: object = None) -> object: - if method == "getbestblockhash": - return block_hash - if method == "getblockheader": - if params != [block_hash]: - raise AssertionError(params) - return {"height": 10, "confirmations": 1} - if method == "getblockhash": - if params != [10]: - raise AssertionError(params) - return block_hash - if method == "getblockcount": - return 10 - if method == "submitblock": - raise AssertionError( - "active tip must not be resubmitted" - ) - raise RuntimeError(method) - - server.rpc = AtTipReplayRpc() - ledger.pool_block_state = ( # type: ignore[attr-defined] - lambda **_kwargs: { - "chain_state": "confirmed", - "maturity_state": "immature", - } - ) - - def confirm_at_tip_replay(**kwargs: object) -> dict[str, object]: - self.assertEqual(kwargs["block_hash"], block_hash) - self.assertEqual(kwargs["active_tip_height"], 10) - # The idempotent arm returns the ordinal the original flip - # allocated and burns no sequence. - return { - "backend": "fake", - "confirmed_count": 2, - "audit_publication_sequence": flipped_sequence, - } - - ledger.confirm_accepted_block = confirm_at_tip_replay # type: ignore[method-assign] - retry_calls = 0 - - def count_retry() -> None: - nonlocal retry_calls - retry_calls += 1 - - server._schedule_tip_refresh_retry = count_retry # type: ignore[method-assign] - cache_key = ("sentinel",) - server._job_bundle_cache[cache_key] = object() - discarded_before = server.payout_state_candidates_discarded - pending_marks_before = server._tip_refresh_pending_counter - - self.assertTrue( - server.submit_block_candidate( - block_candidate(server, state, submission) - ) - ) - latest_evidence = server.latest_evidence - - self.assertEqual(server._payout_state_generation, 1) - self.assertEqual(server._published_payout_state.source_generation, 1) - self.assertEqual(server._published_payout_state.source_tip_hash, block_hash) - self.assertEqual(server._payout_state_source[0], 1) - self.assertIn(cache_key, server._job_bundle_cache) - self.assertEqual(retry_calls, 0) - self.assertEqual( - server._tip_refresh_pending_counter, - pending_marks_before, - ) - self.assertEqual( - server.payout_state_candidates_discarded, - discarded_before, - ) - self.assertFalse(server._payout_state_publication_blocked) - self.assertFalse(server._payout_state_delivery_gate._delivery_blocked) - # A covered replay is not a confirmation failure: no shutdown, and the - # candidate stays accepted rather than being abandoned. - self.assertFalse(server.stop_event.is_set()) - self.assertEqual(server.accepted_block_count, 1) - # The audit publication replays exactly against the ordinal the flip - # allocated. The disposition itself is observational -- reporting 2 - # where the published evidence recorded 1 must not strand the replay - # as a payload conflict. - self.assertIsNotNone(latest_evidence) - assert latest_evidence is not None - self.assertEqual( - latest_evidence["audit_publication_identity"]["sequence"], - flipped_sequence, - ) - self.assertEqual(latest_evidence["confirmation"]["confirmed_count"], 1) - - def test_leaked_publication_fence_replay_republishes(self) -> None: - server, state, ledger = submit_coordinator() - server._ensure_job_cache_state() - server.max_blocks = 2 - server.stop_after_block = False - block_hash = "d8" * 32 - with tempfile.TemporaryDirectory() as tempdir: - server.audit_dir = Path(tempdir) - server.evidence_path = Path(tempdir) / "evidence.json" - server.ledger_writer_public_key_hex = "aa" * 32 - server.rpc = SubmitRpc( - tip="00" * 32, - block_hash=block_hash, - ledger=ledger, - ) - server.build_audit_bundle = ( # type: ignore[method-assign] - lambda **_kwargs: verified_block_bundle() - ) - server.verify_bundle = ( # type: ignore[method-assign] - lambda *_args, **_kwargs: verified_audit_report() - ) - submission = SimpleNamespace( - coinbase_tx_hex="c0ffee", - block_hash_hex=block_hash, - block_hex="00", - ) - - self.assertTrue( - server.submit_block_candidate( - block_candidate(server, state, submission) - ) - ) - self.assertEqual(server._payout_state_generation, 1) - - # Simulate the exception tail a replay must heal: a prior attempt - # force-blocked delivery while its source generation already - # matched the published source, then failed before republishing. - # The leaked fence blocks every job build until a publication - # lands, so an exact confirmed replay must heal the leaked fence - # before taking the covered-state skip. - server._block_payout_state_publication(force=True) - self.assertTrue(server._payout_state_publication_blocked) - self.assertEqual( - server._payout_state_source[0], - server._published_payout_state.source_generation, - ) - child_tip = "d9" * 32 - - class AncestorReplayRpc: - def call(self, method: str, params: object = None) -> object: - if method == "getbestblockhash": - return child_tip - if method == "getblockheader": - if params != [block_hash]: - raise AssertionError(params) - return {"height": 10, "confirmations": 2} - if method == "getblockhash": - if params != [10]: - raise AssertionError(params) - return block_hash - if method == "getblockcount": - return 11 - if method == "submitblock": - raise AssertionError( - "active ancestor must not be resubmitted" - ) - raise RuntimeError(method) - - server.rpc = AncestorReplayRpc() - ledger.pool_block_state = ( # type: ignore[attr-defined] - lambda **_kwargs: { - "chain_state": "confirmed", - "maturity_state": "immature", - } - ) - def confirm_exact_ancestor(**kwargs: object) -> dict[str, object]: - self.assertEqual(kwargs["block_hash"], block_hash) - self.assertEqual(kwargs["active_tip_height"], 10) - return {"backend": "fake", "confirmed_count": 1} - - ledger.confirm_accepted_block = confirm_exact_ancestor # type: ignore[method-assign] - - self.assertTrue( - server.submit_block_candidate( - block_candidate(server, state, submission) - ) - ) - - self.assertEqual(server._payout_state_generation, 2) - # Fence healing republishes identical covered state. It advances the - # delivery generation without inventing a logical invalidation source. - self.assertEqual(server._published_payout_state.source_generation, 1) - self.assertEqual(server._payout_state_source[0], 1) - self.assertFalse(server._payout_state_publication_blocked) - self.assertFalse(server._payout_state_delivery_gate._delivery_blocked) - - def test_direct_block_disabled_reconciler_bounds_publish_supersession(self) -> None: - server, state, ledger = submit_coordinator() - server._ensure_job_cache_state() - server.payout_reconcile_supersession_retries = 2 - block_hash = "d3" * 32 - real_publish = server._publish_payout_state_candidate - publish_attempts = 0 - - def supersede_before_publish(candidate: object) -> int | None: - nonlocal publish_attempts - publish_attempts += 1 - server._reserve_payout_state_source( - "external_tip", - tip_hash=f"{publish_attempts + 20:064x}", - ) - return real_publish(candidate) # type: ignore[arg-type] - - server._publish_payout_state_candidate = supersede_before_publish # type: ignore[method-assign] - with tempfile.TemporaryDirectory() as tempdir: - server.audit_dir = Path(tempdir) - server.evidence_path = Path(tempdir) / "evidence.json" - server.ledger_writer_public_key_hex = "aa" * 32 - server.rpc = SubmitRpc( - tip="00" * 32, - block_hash=block_hash, - ledger=ledger, - ) - server.build_audit_bundle = ( # type: ignore[method-assign] - lambda **_kwargs: verified_block_bundle() - ) - server.verify_bundle = ( # type: ignore[method-assign] - lambda *_args, **_kwargs: verified_audit_report() - ) - submission = SimpleNamespace( - coinbase_tx_hex="c0ffee", - block_hash_hex=block_hash, - block_hex="00", - ) - - accepted = server.submit_block_candidate( - block_candidate(server, state, submission) - ) - - self.assertTrue(accepted) - self.assertEqual(publish_attempts, 3) - self.assertEqual(server._payout_state_generation, 0) - self.assertTrue(server._payout_state_publication_blocked) - self.assertTrue(server._payout_state_delivery_gate._delivery_blocked) - self.assertTrue(server.tip_refresh_is_pending()) - - def test_untrusted_direct_block_reconcile_publishes_newer_source_once(self) -> None: - server, state, ledger = submit_coordinator() - server._ensure_job_cache_state() - server.reorg_reconciler_enabled = True - server.ensure_reorg_reconciled_for_tip = ( # type: ignore[method-assign] - lambda _tip, *, _coalesce_same_tip: not _coalesce_same_tip - ) - server.qbit_chain_view_untrusted = lambda: True # type: ignore[method-assign] - block_hash = "d1" * 32 - newer_tip = "d2" * 32 - real_confirm = ledger.confirm_accepted_block - - def superseding_noop_confirmation(**kwargs: object) -> dict[str, object]: - server._reserve_payout_state_source( - "external_tip", - tip_hash=newer_tip, - ) - return real_confirm(**kwargs) - - ledger.confirm_accepted_block = superseding_noop_confirmation # type: ignore[method-assign] - with tempfile.TemporaryDirectory() as tempdir: - server.audit_dir = Path(tempdir) - server.evidence_path = Path(tempdir) / "evidence.json" - server.ledger_writer_public_key_hex = "aa" * 32 - server.rpc = SubmitRpc( - tip="00" * 32, - block_hash=block_hash, - ledger=ledger, - ) - server.build_audit_bundle = ( # type: ignore[method-assign] - lambda **_kwargs: verified_block_bundle() - ) - server.verify_bundle = ( # type: ignore[method-assign] - lambda *_args, **_kwargs: verified_audit_report() - ) - submission = SimpleNamespace( - coinbase_tx_hex="c0ffee", - block_hash_hex=block_hash, - block_hex="00", - ) - - accepted = server.submit_block_candidate( - block_candidate(server, state, submission) - ) - - self.assertTrue(accepted) - self.assertEqual(server._payout_state_generation, 2) - self.assertEqual(server._published_payout_state.source_tip_hash, newer_tip) - self.assertEqual(server.payout_state_candidates_discarded, 0) - self.assertEqual(server.reorg_reconcile_skip_count, 1) - - def _superseded_confirm_coordinator( - self, - confirmed_count: int, - ) -> tuple[PrismCoordinator, object, object, str, list[object]]: - server, state, ledger = submit_coordinator() - server._ensure_job_cache_state() - block_hash = "d3" * 32 - server.rpc = SubmitRpc( - tip="00" * 32, - block_hash=block_hash, - ledger=ledger, - ) - server.build_audit_bundle = ( # type: ignore[method-assign] - lambda **_kwargs: verified_block_bundle() - ) - server.verify_bundle = ( # type: ignore[method-assign] - lambda *_args, **_kwargs: verified_audit_report() - ) - ledger.confirm_accepted_block = ( # type: ignore[method-assign] - lambda **_kwargs: { - "backend": "fake", - "confirmed_count": confirmed_count, - } - ) - shutdown_requests: list[object] = [] - real_shutdown = server.request_shutdown - - def observed_shutdown(signum: int | None = None) -> None: - shutdown_requests.append(signum) - real_shutdown(signum) - - server.request_shutdown = observed_shutdown # type: ignore[method-assign] - return server, state, ledger, block_hash, shutdown_requests - - def test_superseded_ledger_confirmation_abandons_without_shutdown(self) -> None: - # A confirm disposition of -1 means the row was terminally disposed - # (reorg quarantine, rejection, or reversal) before the confirmation - # landed: terminal for the candidate, benign for the pool. The - # coordinator must abandon the candidate and keep serving instead of - # converting the routine race into a full-pool outage. - server, state, ledger, block_hash, shutdown_requests = ( - self._superseded_confirm_coordinator(-1) - ) - # The definitive submitblock ack normally defers terminal - # abandonment; age that evidence out immediately to model the - # settled post-reorg view in which the chain no longer carries the - # candidate hash. - server.observed_tip_accept_window_seconds = 1e-9 - submission = SimpleNamespace( - coinbase_tx_hex="c0ffee", - block_hash_hex=block_hash, - block_hex="00", - ) - - with tempfile.TemporaryDirectory() as tempdir: - server.audit_dir = Path(tempdir) - server.evidence_path = Path(tempdir) / "evidence.json" - server.ledger_writer_public_key_hex = "aa" * 32 - accepted = server.submit_block_candidate( - block_candidate(server, state, submission) - ) - - self.assertFalse(accepted) - self.assertEqual(len(ledger.persisted), 1) - self.assertEqual( - server._block_candidate_outcome.reason, - PRISM_REJECTION_LEDGER_CONFIRMATION_SUPERSEDED, - ) - self.assertEqual(shutdown_requests, []) - self.assertFalse(server.stop_event.is_set()) - self.assertEqual(server.accepted_block_count, 0) - # The prospective preview was withdrawn with publication - # invalidation, exactly as on the fail-closed path. - transition = server._accepted_block_payout_previews.get(block_hash) - self.assertTrue(transition is None or transition.preview is None) - - def test_superseded_confirmation_with_fresh_acceptance_evidence_defers( - self, - ) -> None: - # While first-party acceptance evidence (the definitive submitblock - # ack) is still fresh, even a superseded disposition must not commit - # a terminal abandonment: the reorg reconciler may still reactivate - # the row. The candidate defers -- and the pool keeps serving. - server, state, _ledger, block_hash, shutdown_requests = ( - self._superseded_confirm_coordinator(-1) - ) - submission = SimpleNamespace( - coinbase_tx_hex="c0ffee", - block_hash_hex=block_hash, - block_hex="00", - ) - - with tempfile.TemporaryDirectory() as tempdir: - server.audit_dir = Path(tempdir) - server.evidence_path = Path(tempdir) / "evidence.json" - server.ledger_writer_public_key_hex = "aa" * 32 - accepted = server.submit_block_candidate( - block_candidate(server, state, submission) - ) - - self.assertFalse(accepted) - self.assertEqual( - server._block_candidate_outcome.reason, - PRISM_REJECTION_BLOCK_ACCEPT_PENDING, - ) - self.assertEqual(shutdown_requests, []) - self.assertFalse(server.stop_event.is_set()) - - def test_unexplained_zero_ledger_confirmation_still_requests_shutdown( - self, - ) -> None: - # A 0 with no superseding row remains unexplained: the fail-closed - # shutdown escalation is preserved exactly as before the disposition - # split. - server, state, _ledger, block_hash, shutdown_requests = ( - self._superseded_confirm_coordinator(0) - ) - submission = SimpleNamespace( - coinbase_tx_hex="c0ffee", - block_hash_hex=block_hash, - block_hex="00", - ) - - with tempfile.TemporaryDirectory() as tempdir: - server.audit_dir = Path(tempdir) - server.evidence_path = Path(tempdir) / "evidence.json" - server.ledger_writer_public_key_hex = "aa" * 32 - accepted = server.submit_block_candidate( - block_candidate(server, state, submission) - ) - - self.assertFalse(accepted) - self.assertEqual(len(shutdown_requests), 1) - self.assertTrue(server.stop_event.is_set()) - self.assertEqual( - server._block_candidate_outcome.reason, - PRISM_REJECTION_LEDGER_CONFIRMATION_FAILED, - ) - - def test_verified_canonical_bundle_path_requires_exact_bytes(self) -> None: - with tempfile.TemporaryDirectory() as tempdir: - candidate_path = Path(tempdir) / "candidate.json" - candidate_bytes = b'{"canonical":true}' - candidate_path.write_bytes(candidate_bytes) - canonical_sha256 = hashlib.sha256(candidate_bytes).hexdigest() - - self.assertEqual( - PrismCoordinator.verified_canonical_bundle_path( - candidate_path, - {"audit_bundle_sha256_hex": canonical_sha256.upper()}, - ), - candidate_path, - ) - self.assertIsNone( - PrismCoordinator.verified_canonical_bundle_path( - candidate_path, - {"audit_bundle_sha256_hex": "00" * 32}, - ) - ) - - def test_active_ancestor_candidate_resumes_full_finalization_without_resubmit(self) -> None: - server, state, ledger = submit_coordinator() - server.max_blocks = 10 - server.stop_after_block = False - refreshes: list[str] = [] - server.refresh_jobs_after_accepted_block = ( # type: ignore[method-assign] - lambda **kwargs: refreshes.append(str(kwargs["block_hash"])) - ) - with tempfile.TemporaryDirectory() as tempdir: - server.audit_dir = Path(tempdir) - server.evidence_path = Path(tempdir) / "evidence.json" - server.ledger_writer_public_key_hex = "aa" * 32 - block_hash = "ac" * 32 - - class ActiveAncestorRpc: - def __init__(self) -> None: - self.revalidated_heights: list[int] = [] - - def call(self, method: str, params: object = None) -> object: - if method == "getbestblockhash": - return "ef" * 32 - if method == "getblockheader": - self.assert_candidate(params) - return {"height": 10, "confirmations": 2} - if method == "getblockcount": - return 11 - if method == "getblockhash": - if params != [10]: - raise AssertionError(params) - self.revalidated_heights.append(10) - return block_hash - if method == "submitblock": - raise AssertionError("active ancestor must not be resubmitted") - raise RuntimeError(method) - - @staticmethod - def assert_candidate(params: object) -> None: - if params != [block_hash]: - raise AssertionError(params) - - active_rpc = ActiveAncestorRpc() - server.rpc = active_rpc - server.ensure_reorg_reconciled_for_tip = ( # type: ignore[method-assign] - lambda _tip, *, _coalesce_same_tip: not _coalesce_same_tip - ) - server.build_audit_bundle = lambda **_kwargs: { # type: ignore[method-assign] - "found_block": {"coinbase_value_sats": 50_00000000}, - "ledger_window_attestation": {"signature": {"public_key_hex": "aa" * 32}}, - "payout_policy_manifest": {"accounts": []}, - "signed_coinbase_manifest": { - "manifest": {"coinbase_tx_hex": "c0ffee", "payout_count": 1} - }, - } - server.verify_bundle = ( # type: ignore[method-assign] - lambda *_args, **_kwargs: verified_audit_report() - ) - submission = SimpleNamespace( - coinbase_tx_hex="c0ffee", - block_hash_hex=block_hash, - block_hex="00", - ) - candidate = block_candidate(server, state, submission) - - self.assertTrue(server.submit_block_candidate(candidate)) - # Direct finalization records the refresh for its caller so slow - # job fanout happens after writer admission has been released. - self.assertEqual( - state.post_accept_refresh_block, - (10, block_hash), - ) - server.refresh_jobs_after_pending_accepted_block( - state, - heartbeat_name="block_submitter", - ) - # A failed outbox terminal update can replay A after later block B - # changes global balances. Validate A against its height-bounded - # active-chain view and suppress duplicate process-local accounting. - ledger.durable_payout_state = True - ledger.prior_balances = [ - { - "recipient_id": "later-miner", - "order_key": "later-miner", - "p2mr_program_hex": "22" * 32, - "balance_sats": 50, - } - ] - ledger.pool_block_state = lambda **_kwargs: { # type: ignore[attr-defined] - "block_hash": block_hash, - "block_height": 10, - "parent_hash": "00" * 32, - "chain_state": "confirmed", - "maturity_state": "immature", - } - historical_reads: list[dict[str, object]] = [] - ledger.prior_balances_after_pool_block = ( # type: ignore[attr-defined] - lambda **kwargs: historical_reads.append(dict(kwargs)) or [] - ) - self.assertTrue(server.submit_block_candidate(candidate)) - # The confirmed-ancestor replay keeps its height-bounded - # active-chain view (issue #209 leaves replay behavior intact). - self.assertEqual(historical_reads, [{"block_hash": block_hash}]) - latest_evidence = server.latest_evidence - - self.assertEqual([row["block_hash"] for row in ledger.persisted], [block_hash] * 2) - # This compatibility builder ignores canonical_output_path, so its - # Python fallback remains verifier-only. - self.assertIsNone(ledger.persisted[0]["canonical_bundle_path"]) - self.assertEqual(ledger.confirmed[0]["active_tip_height"], 10) - self.assertEqual(ledger.confirmed[0]["block_hash"], block_hash) - self.assertEqual(active_rpc.revalidated_heights, [10, 10]) - self.assertFalse(ledger.confirmed[0]["submit_seen_at_confirm"]) - # The share credit happens on the client thread at submit time now; - # the block path itself appends nothing. - self.assertEqual(len(ledger.pending), 0) - self.assertFalse(server.stop_event.is_set()) - self.assertEqual(server.accepted_block_count, 1) - self.assertEqual(refreshes, [block_hash]) - self.assertIsNotNone(latest_evidence) - assert latest_evidence is not None - self.assertEqual(latest_evidence["persistence"]["block_count"], 1) - self.assertEqual(latest_evidence["confirmation"]["confirmed_count"], 1) - # Evidence carries an aggregate miner count, not a materialized list of - # every miner id (which scanned the whole ledger twice under the lock). - self.assertEqual(latest_evidence["accepted_share_count"], 0) - self.assertEqual(latest_evidence["distinct_miner_count"], 0) - self.assertNotIn("distinct_miners", latest_evidence) - - def test_fresh_confirmation_never_reads_historical_balances(self) -> None: - # Issue #209: a fresh confirmation lands as the ledger's newest - # confirmed pool block, so its post-confirm validation reads the - # O(recipients) current-balance summary. The O(history) height-bounded - # reader -- whose carry-table scan once held payout serialization past - # the parent-preview wait during back-to-back blocks -- is replay-only. - server, state, ledger = submit_coordinator() - server._ensure_job_cache_state() - server.max_blocks = 2 - server.stop_after_block = False - ledger.durable_payout_state = True - block_hash = "e3" * 32 - historical_reads: list[dict[str, object]] = [] - # A poisoned historical view: touching it on the fresh path would - # fail the post-confirm comparison and force shutdown. - ledger.prior_balances_after_pool_block = ( # type: ignore[attr-defined] - lambda **kwargs: historical_reads.append(dict(kwargs)) - or [ - { - "recipient_id": "history-miner", - "order_key": "history-miner", - "p2mr_program_hex": "33" * 32, - "balance_sats": 999, - } - ] - ) - current_reads = 0 - original_current = ledger.current_prior_balances - - def counting_current_prior_balances() -> list[dict[str, object]]: - nonlocal current_reads - current_reads += 1 - return original_current() - - ledger.current_prior_balances = counting_current_prior_balances # type: ignore[method-assign] - with tempfile.TemporaryDirectory() as tempdir: - server.audit_dir = Path(tempdir) - server.evidence_path = Path(tempdir) / "evidence.json" - server.ledger_writer_public_key_hex = "aa" * 32 - server.rpc = SubmitRpc( - tip="00" * 32, - block_hash=block_hash, - ledger=ledger, - ) - server.build_audit_bundle = ( # type: ignore[method-assign] - lambda **_kwargs: verified_block_bundle() - ) - server.verify_bundle = ( # type: ignore[method-assign] - lambda *_args, **_kwargs: verified_audit_report() - ) - submission = SimpleNamespace( - coinbase_tx_hex="c0ffee", - block_hash_hex=block_hash, - block_hex="00", - ) - - self.assertTrue( - server.submit_block_candidate( - block_candidate(server, state, submission) - ) - ) - - self.assertEqual(historical_reads, []) - self.assertGreaterEqual(current_reads, 1) - self.assertEqual(len(ledger.confirmed), 1) - self.assertFalse(server.stop_event.is_set()) - self.assertEqual(server.accepted_block_count, 1) - - def test_audit_retention_prunes_only_live_and_candidate_files(self) -> None: - server = coordinator() - with tempfile.TemporaryDirectory() as tempdir: - server.audit_dir = Path(tempdir) - server.audit_live_bundle_retention = 2 - server.audit_candidate_retention_seconds = 0 - for index in range(4): - path = Path(tempdir) / f"prism-live-audit-bundle-{index + 1}-{'aa' * 32}.json" - path.write_text("{}", encoding="utf-8") - os.utime(path, (100 + index, 100 + index)) - candidate = Path(tempdir) / f"prism-live-audit-bundle-candidate-{'bb' * 32}.json" - candidate.write_text("{}", encoding="utf-8") - temp_candidate = Path(tempdir) / f".prism-live-audit-bundle-candidate-{'bb' * 32}.json.tmp" - temp_candidate.write_text("{}", encoding="utf-8") - body = Path(tempdir) / f"prism-audit-bundle-body-{'cc' * 32}-{'dd' * 32}.json" - body.write_text("{}", encoding="utf-8") - segment = Path(tempdir) / f"prism-audit-share-segment-1-2-{'ee' * 32}.json" - segment.write_text("{}", encoding="utf-8") - - # Live deletion is fail-closed unless durable evidence has supplied - # publication authority. This facade test supplies that authority - # explicitly while keeping its hand-written retention fixtures. - store = server._ensure_audit_artifact_store() - store._compatibility_evidence_override = True - store._evidence_state = "valid" - - server.prune_audit_artifacts() - - live_names = sorted(path.name for path in Path(tempdir).glob("prism-live-audit-bundle-[0-9]*.json")) - self.assertEqual( - live_names, - [ - f"prism-live-audit-bundle-3-{'aa' * 32}.json", - f"prism-live-audit-bundle-4-{'aa' * 32}.json", - ], - ) - self.assertFalse(candidate.exists()) - self.assertFalse(temp_candidate.exists()) - self.assertTrue(body.exists()) - self.assertTrue(segment.exists()) - - def test_audit_retention_zero_preserves_current_live_envelope(self) -> None: - server = coordinator() - with tempfile.TemporaryDirectory() as tempdir: - server.audit_dir = Path(tempdir) - server.audit_live_bundle_retention = 0 - old = Path(tempdir) / f"prism-live-audit-bundle-1-{'aa' * 32}.json" - old.write_text("{}", encoding="utf-8") - current = Path(tempdir) / f"prism-live-audit-bundle-2-{'bb' * 32}.json" - current.write_text("{}", encoding="utf-8") - - store = server._ensure_audit_artifact_store() - store._compatibility_evidence_override = True - store._evidence_state = "valid" - store._current_envelope = current.resolve() - - server.prune_audit_artifacts(keep_live_path=current) - - self.assertFalse(old.exists()) - self.assertTrue(current.exists()) - - def test_accepted_direct_block_refreshes_clean_job_after_submit_response(self) -> None: - old_tip = "00" * 32 - block_hash = "ab" * 32 - server, state, ledger = submit_coordinator(tip=old_tip) - server.stop_after_block = False - server.max_blocks = 10 - server.clients = {state} - state.share_difficulty = Decimal("1") - sent: list[dict[str, object]] = [] - state.send = lambda payload: sent.append(payload) # type: ignore[method-assign] - old_context = server.jobs["job-1"] - server.tip_template_snapshot = QbitTipTemplateSnapshot( - bestblockhash=old_tip, - previousblockhash=old_tip, - template_fingerprint=qbit_template_fingerprint(old_context.template), - ) - - def build_fresh_job(client: ClientState, *, clean_jobs: bool) -> object: - self.assertIs(client, state) - self.assertNotIn( - "accepted_block_handling", - server._ensure_shutdown_controller().snapshot()["active_writers"], - ) - return prism_context( - "fresh-job", - block_hash, - worker=state.worker, - difficulty=server.desired_client_share_difficulty(client), - clean_jobs=clean_jobs, - ) - - server.build_job_for_client = build_fresh_job # type: ignore[method-assign] - - with tempfile.TemporaryDirectory() as tempdir: - server.audit_dir = Path(tempdir) - server.evidence_path = Path(tempdir) / "evidence.json" - server.ledger_writer_public_key_hex = "aa" * 32 - server.rpc = SubmitAcceptingTemplateRpc(old_tip=old_tip, block_hash=block_hash, ledger=ledger) - server.build_audit_bundle = lambda **_kwargs: verified_block_bundle() # type: ignore[method-assign] - server.verify_bundle = lambda *_args, **_kwargs: verified_audit_report() # type: ignore[method-assign] - submission = SimpleNamespace( - header_hex="aa" * 80, - share_pass=True, - block_pass=True, - coinbase_tx_hex="c0ffee", - block_hash_hex=block_hash, - block_hex="00", - ) - - with patch( - "lab.prism.prism_coordinator.direct_stratum.assemble_submission", - return_value=submission, - ): - server.handle_request( - state, - { - "id": "submit-1", - "method": "mining.submit", - "params": ["miner-a", "job-1", "00" * 8, "00000001", "00000002"], - }, - ) - # The ack goes out before the block path runs; draining the - # submitter queue lands the block and pushes fresh work. - self.assertEqual(sent, [{"id": "submit-1", "result": True, "error": None}]) - self.assertTrue(server.submit_next_block_candidate()) - self.assertEqual(server.poll_qbit_tip_template_once(), 1) - - self.assertEqual(sent[0], {"id": "submit-1", "result": True, "error": None}) - self.assertEqual([payload.get("method") for payload in sent[1:]], ["mining.set_difficulty", "mining.notify"]) - self.assertEqual(sent[2]["params"][0], "fresh-job") - self.assertTrue(sent[2]["params"][8]) - self.assertEqual(server.tip_refresh_job_count, 1) - self.assertEqual(server.post_accept_refresh_failure_count, 0) - self.assertEqual(server.accepted_block_count, 1) - self.assertNotIn("job-1", server.jobs) - self.assertIn("fresh-job", server.jobs) - self.assertEqual(state.active_job_ids, {"fresh-job"}) - self.assertIn(state, server.clients) - server.stale_grace_seconds = 0 - - with self.assertRaises(StratumError) as raised: - server.handle_submit( - state, - ["miner-a", "job-1", "00" * 8, "00000001", "00000002"], - ) - self.assertEqual(raised.exception.code, 21) - self.assertEqual(raised.exception.reason, PRISM_REJECTION_UNKNOWN_JOB) - - fresh_submission = SimpleNamespace( - header_hex="bb" * 80, - block_hash_hex="bc" * 32, - share_pass=True, - block_pass=False, - ) - with patch( - "lab.prism.prism_coordinator.direct_stratum.assemble_submission", - return_value=fresh_submission, - ): - should_close = server.handle_submit( - state, - ["miner-a", "fresh-job", "00" * 8, "00000001", "00000002"], - ) - - self.assertFalse(should_close) - self.assertEqual(ledger.pending[-1].job_id, "fresh-job") - - def test_post_accept_notification_does_not_run_failing_template_build(self) -> None: - old_tip = "00" * 32 - block_hash = "ad" * 32 - server, state, ledger = submit_coordinator(tip=old_tip) - server.stop_after_block = False - server.max_blocks = 10 - server.clients = {state} - sent: list[dict[str, object]] = [] - state.send = lambda payload: sent.append(payload) # type: ignore[method-assign] - - def unexpected_build(client: ClientState, *, clean_jobs: bool) -> object: - raise AssertionError("job build should not run when template refresh fails") - - server.build_job_for_client = unexpected_build # type: ignore[method-assign] - - with tempfile.TemporaryDirectory() as tempdir: - server.audit_dir = Path(tempdir) - server.evidence_path = Path(tempdir) / "evidence.json" - server.ledger_writer_public_key_hex = "aa" * 32 - server.rpc = SubmitAcceptingTemplateRpc( - old_tip=old_tip, - block_hash=block_hash, - fail_template_after_submit=True, - ledger=ledger, - ) - server.build_audit_bundle = lambda **_kwargs: verified_block_bundle() # type: ignore[method-assign] - server.verify_bundle = lambda *_args, **_kwargs: verified_audit_report() # type: ignore[method-assign] - submission = SimpleNamespace( - header_hex="aa" * 80, - share_pass=True, - block_pass=True, - coinbase_tx_hex="c0ffee", - block_hash_hex=block_hash, - block_hex="00", - ) - - with patch( - "lab.prism.prism_coordinator.direct_stratum.assemble_submission", - return_value=submission, - ): - server.handle_request( - state, - { - "id": "submit-1", - "method": "mining.submit", - "params": ["miner-a", "job-1", "00" * 8, "00000001", "00000002"], - }, - ) - self.assertTrue(server.submit_next_block_candidate()) - - self.assertEqual(sent, [{"id": "submit-1", "result": True, "error": None}]) - self.assertEqual(server.accepted_block_count, 1) - self.assertEqual(len(ledger.persisted), 1) - self.assertEqual(len(ledger.confirmed), 1) - self.assertEqual(len(ledger.pending), 1) - self.assertEqual(server.tip_refresh_job_count, 0) - self.assertEqual(server.post_accept_refresh_failure_count, 0) - self.assertEqual(state.active_job_ids, {"job-1"}) - self.assertIn("job-1", server.jobs) - self.assertTrue(server.tip_refresh_is_pending()) - self.assertTrue(server._tip_refresh_retry.is_set()) - self.assertIn("qbit_prism_post_accept_refresh_failures_total 0", server.metrics_payload()) - - def test_post_accept_refresh_preserves_pending_vardiff_difficulty_pair(self) -> None: - old_tip = "00" * 32 - block_hash = "ae" * 32 - server, state, ledger = submit_coordinator(tip=old_tip) - server.vardiff_config = vardiff.VardiffConfig( - enabled=True, - target_share_interval_seconds=Decimal("15"), - min_difficulty=Decimal("1"), - max_difficulty=Decimal("1024"), - retarget_interval_seconds=Decimal("90"), - max_step_factor=Decimal("4"), - startup_difficulty=Decimal("1"), - max_step_down_factor=Decimal("4"), - ewma_alpha=Decimal("1"), - retarget_tolerance=Decimal("0"), - ) - server.stop_after_block = False - server.max_blocks = 10 - server.clients = {state} - state.share_difficulty = Decimal("1") - state.pending_share_difficulty = Decimal("8") - state.vardiff_window_started_monotonic = time.monotonic() - sent: list[dict[str, object]] = [] - state.send = lambda payload: sent.append(payload) # type: ignore[method-assign] - - def build_fresh_job(client: ClientState, *, clean_jobs: bool) -> object: - return prism_context( - "fresh-vardiff-job", - block_hash, - worker=state.worker, - difficulty=server.desired_client_share_difficulty(client), - clean_jobs=clean_jobs, - ) - - server.build_job_for_client = build_fresh_job # type: ignore[method-assign] - - with tempfile.TemporaryDirectory() as tempdir: - server.audit_dir = Path(tempdir) - server.evidence_path = Path(tempdir) / "evidence.json" - server.ledger_writer_public_key_hex = "aa" * 32 - server.rpc = SubmitAcceptingTemplateRpc(old_tip=old_tip, block_hash=block_hash, ledger=ledger) - server.build_audit_bundle = lambda **_kwargs: verified_block_bundle() # type: ignore[method-assign] - server.verify_bundle = lambda *_args, **_kwargs: verified_audit_report() # type: ignore[method-assign] - submission = SimpleNamespace( - header_hex="aa" * 80, - share_pass=True, - block_pass=True, - coinbase_tx_hex="c0ffee", - block_hash_hex=block_hash, - block_hex="00", - ) - - with patch( - "lab.prism.prism_coordinator.direct_stratum.assemble_submission", - return_value=submission, - ): - server.handle_request( - state, - { - "id": "submit-1", - "method": "mining.submit", - "params": ["miner-a", "job-1", "00" * 8, "00000001", "00000002"], - }, - ) - self.assertTrue(server.submit_next_block_candidate()) - self.assertEqual(server.poll_qbit_tip_template_once(), 1) - - self.assertEqual(sent[0], {"id": "submit-1", "result": True, "error": None}) - self.assertEqual(sent[1]["method"], "mining.set_difficulty") - self.assertEqual(sent[1]["params"], [8.0]) - self.assertEqual(sent[2]["method"], "mining.notify") - self.assertEqual(sent[2]["params"][0], "fresh-vardiff-job") - self.assertTrue(sent[2]["params"][8]) - self.assertEqual(state.share_difficulty, Decimal("8")) - self.assertIsNone(state.pending_share_difficulty) - self.assertEqual(state.vardiff_window_submitted, 1) - self.assertEqual(state.vardiff_window_accepted, 1) - - def test_rejected_candidate_never_creates_prepared_payout_state(self) -> None: - server, state, ledger = submit_coordinator() - with tempfile.TemporaryDirectory() as tempdir: - server.audit_dir = Path(tempdir) - server.evidence_path = Path(tempdir) / "evidence.json" - server.ledger_writer_public_key_hex = "aa" * 32 - block_hash = "dd" * 32 - server.rpc = SubmitRpc(tip="00" * 32, block_hash=block_hash, submit_result="rejected") - server.build_audit_bundle = lambda **_kwargs: { # type: ignore[method-assign] - "found_block": {"coinbase_value_sats": 50_00000000}, - "ledger_window_attestation": {"signature": {"public_key_hex": "aa" * 32}}, - "payout_policy_manifest": {"accounts": []}, - "signed_coinbase_manifest": { - "manifest": { - "coinbase_tx_hex": "c0ffee", - "payout_count": 1, - } - }, - } - server.verify_bundle = ( # type: ignore[method-assign] - lambda *_args, **_kwargs: verified_audit_report() - ) - submission = SimpleNamespace( - coinbase_tx_hex="c0ffee", - block_hash_hex=block_hash, - block_hex="00", - ) - - accepted = server.submit_block_candidate(block_candidate(server, state, submission)) - - self.assertFalse(accepted) - self.assertEqual(ledger.persisted, []) - self.assertEqual(ledger.rejected, []) - self.assertEqual(ledger.reversed, []) - self.assertEqual(len(ledger.pending), 0) - self.assertEqual( - server.block_candidate_abandoned_counts[PRISM_REJECTION_SUBMITBLOCK_REJECTED], 1 - ) - - def test_build_audit_bundle_passes_coinbase_output_policy_to_cli_payload(self) -> None: - server = coordinator() - server.rpc = AddressRpc(valid_address="tq1fee", script_byte="99") - server.signing_seed_hex = "42" * 32 - server.ledger_attestation_signing_seed_hex = "43" * 32 - captured: dict[str, object] = {} - - with patch.dict( - os.environ, - { - "PRISM_COINBASE_OUTPUT_POLICY": "pool-fee-first", - "PRISM_POOL_FEE_ENABLED": "1", - "PRISM_POOL_FEE_BPS": "125", - "PRISM_POOL_FEE_ADDRESS": "tq1fee", - }, - clear=True, - ), patch( - "lab.prism.prism_coordinator.subprocess.Popen", - fake_audit_bundle_popen(captured), - ): - server.build_audit_bundle( - shares=[], - found_block={"block_height": 10, "coinbase_value_sats": 50_00000000}, - prior_balances=[], - coinbase_script_sig_suffix_hex="00", - ) - - payout_policy = captured["payload"]["payout_policy"] - self.assertEqual(payout_policy["coinbase_output_policy"], "pool-fee-first") - self.assertEqual(payout_policy["pool_fee_policy"]["fee_bps"], 125) - - def test_ledger_artifact_not_built_when_publication_not_required(self) -> None: - tip = "5b" * 32 - ledger = ReorgLedger([]) - server, build_calls = self._reconcile_coordinator_with_artifact_counter( - ledger, - ReorgRpc( - tip=tip, - template=gbt_template(tip, height=11), - height=10, - block_hashes={10: tip}, - ), - ) - - first = server.reconcile_prism_pool_blocks_once(tip_hash=tip) - self.assertEqual(first["published_generation"], 1) - self.assertEqual(len(build_calls), 1) - self.assertFalse(build_calls[0][2]) - - second = server.reconcile_prism_pool_blocks_once(tip_hash=tip) - - self.assertIsNone(second["published_generation"]) - self.assertEqual( - len(build_calls), - 1, - "a pass without publication need must not build the ledger artifact", - ) - self.assertEqual( - ledger.events, - [("watch", 10), ("mature", 10), ("watch", 10), ("mature", 10)], - ) - - def test_payout_change_forces_publication_and_rereads_balances(self) -> None: - tip = "5d" * 32 - pool_block_hash = "ae" * 32 - ledger = ReorgLedger([]) - server, build_calls = self._reconcile_coordinator_with_artifact_counter( - ledger, - ReorgRpc( - tip=tip, - template=gbt_template(tip, height=11), - height=10, - block_hashes={10: tip}, - ), - ) - - first = server.reconcile_prism_pool_blocks_once(tip_hash=tip) - self.assertEqual(first["published_generation"], 1) - self.assertEqual(len(build_calls), 1) - - # An orphaned confirmed block appears without any new payout source: - # payout_changed alone must still force a publication. - ledger.rows.append( - { - "block_hash": pool_block_hash, - "block_height": 12, - "chain_state": "confirmed", - "maturity_state": "immature", - } - ) - - second = server.reconcile_prism_pool_blocks_once(tip_hash=tip) - - self.assertEqual(second["inactive_blocks"], 1) - self.assertEqual(second["published_generation"], 2) - self.assertEqual(len(build_calls), 2) - self.assertEqual( - build_calls[1][2:], - (False, True), - "a confirmed payout mutation keeps the exact share window and " - "only rereads current prior balances", - ) - self.assertEqual(ledger.rows[0]["chain_state"], "inactive") - self.assertIn(("inactive", pool_block_hash, 10), ledger.events) - - def test_block_submit_lands_as_issued_after_late_append_epoch_invalidation(self) -> None: - # A late-visible replay append advances the live epoch and schedules - # the refresh wave asynchronously; until that wave retires the job, - # membership admission still lets the pre-append job submit. The - # fast lane offers the solve to qbitd without ledger - # synchronization, so once the node accepts it the landing keeps - # the as-issued payout snapshot for accounting (the replayed share - # rides the next window) instead of terminally abandoning an - # accepted block. - server, state, _recording = submit_coordinator() - ledger = SingleWriterShareLedger() - server.ledger = ledger - server._ensure_job_cache_state() - with server._job_cache_lock: - server._payout_ledger_append_invalidation_epoch += 1 - bundle_builds: list[dict[str, object]] = [] - - def recording_build_audit_bundle(**kwargs: object) -> dict[str, object]: - bundle_builds.append(dict(kwargs)) - return verified_block_bundle() - - server.build_audit_bundle = recording_build_audit_bundle # type: ignore[method-assign] - server.verify_bundle = lambda *_args, **_kwargs: verified_audit_report() # type: ignore[method-assign] - tail_dir = tempfile.TemporaryDirectory() - self.addCleanup(tail_dir.cleanup) - server.audit_dir = Path(tail_dir.name) - server.evidence_path = Path(tail_dir.name) / "evidence.json" - server.ledger_writer_public_key_hex = "aa" * 32 - - submitblock_calls: list[object] = [] - - class RecordingSubmitRpc(TipRpc): - def call( - rpc_self, - method: str, - params: list[object] | None = None, - ) -> object: - if method == "getblockcount": - return 9 - if method == "submitblock": - submitblock_calls.append(params) - return None - if method == "getblockhash": - return "ea" * 32 - return super().call(method, params) - - server.rpc = RecordingSubmitRpc("00" * 32) - submission = SimpleNamespace( - coinbase_tx_hex="c0ffee", - block_hash_hex="ea" * 32, - block_hex="00", - ) - pending = self._pending_append("append-epoch-stale").pending_share - candidate = block_candidate( - server, - state, - submission, - pending_share=pending, - ) - ledger.append_batch( - [(pending, server.block_candidate_intent(candidate))] - ) - server.enqueue_block_candidate(candidate) - - self.assertTrue(server.submit_next_block_candidate()) - # The share was already accepted at submit time, so a lost block is a - # block-abandonment, not a stale share rejection. - self.assertEqual(server.stale_share_count, 0) - # The node accepted the pre-append solve: accounting keeps the - # as-issued snapshot instead of terminally abandoning it. - self.assertEqual(submitblock_calls, [["00"]]) - self.assertEqual(len(bundle_builds), 1) - self.assertNotIn( - PRISM_REJECTION_STALE_JOB, - server.block_candidate_abandoned_counts, - ) - self.assertEqual( - getattr(server, "stale_job_abandon_counts", {}).get( - "append_epoch_stale", 0 - ), - 0, - ) - outbox_row = ledger._block_candidate_outbox[submission.block_hash_hex] - self.assertEqual(outbox_row["state"], "submitted") - self.assertIsNone(outbox_row["last_error"]) - self.assertEqual(server.accepted_block_count, 1) - metrics = server.metrics_payload() - self.assertIn( - 'qbit_prism_stale_job_abandons_total{class="append_epoch_stale"} 0', - metrics, - ) - - def test_replayed_block_candidate_is_exempt_from_append_epoch_fence(self) -> None: - # Epochs are process-local: a candidate reconstructed from durable - # intent carries no meaningful stamp, so an epoch advanced by this - # process's own share replay must not abandon the recovered block. - # Cross-restart payout drift is governed by the durable share-window - # replay and prior-balance fences instead. - server, state, ledger = submit_coordinator() - server.stop_after_block = False - server.max_blocks = 10 - block_hash = "cd" * 32 - pending = self._pending_append("replayed-epoch").pending_share - original = block_candidate( - server, - state, - SimpleNamespace( - coinbase_tx_hex="c0ffee", - block_hash_hex=block_hash, - block_hex="00", - ), - pending_share=pending, - ) - candidate = server.block_candidate_from_intent( - server.block_candidate_intent(original) - ) - self.assertEqual(candidate.context.payout_append_invalidation_epoch, -1) - server._ensure_job_cache_state() - with server._job_cache_lock: - server._payout_ledger_append_invalidation_epoch += 1 - - with tempfile.TemporaryDirectory() as tempdir: - server.audit_dir = Path(tempdir) - server.evidence_path = Path(tempdir) / "evidence.json" - server.ledger_writer_public_key_hex = "aa" * 32 - server.rpc = SubmitRpc( - tip="00" * 32, - block_hash=block_hash, - ledger=ledger, - ) - server.build_audit_bundle = ( # type: ignore[method-assign] - lambda **_kwargs: verified_block_bundle() - ) - server.verify_bundle = ( # type: ignore[method-assign] - lambda *_args, **_kwargs: verified_audit_report() - ) - accepted = server.submit_block_candidate(candidate) - - self.assertTrue(accepted) - self.assertEqual(server.block_candidate_abandoned_counts, {}) - self.assertEqual(len(ledger.persisted), 1) - self.assertEqual(len(ledger.confirmed), 1) - - def test_offered_replay_keeps_as_issued_snapshot_despite_window_omission( - self, - ) -> None: - # Revalidation guards an offer the node has not yet seen. The - # dedicated submitter's fast lane offers the durable bytes before - # the landing runs, so once the node accepts, the recorded coinbase - # is settled reality: the landing must keep the as-issued snapshot - # and skip the audit-window walk entirely instead of terminally - # abandoning payout accounting for an accepted block (or defer- - # looping behind the walk's deadline under saturation). - server, state, _recording = submit_coordinator() - ledger = SingleWriterShareLedger() - ledger.durable_payout_state = True # type: ignore[attr-defined] - window_calls: list[tuple[int, int]] = [] - - def durable_window( - *, - anchor_job_issued_at_ms: int, - network_difficulty: int, - ) -> list[dict[str, object]]: - window_calls.append((anchor_job_issued_at_ms, network_difficulty)) - return [ - {"share_id": "recorded-window-share"}, - {"share_id": "late-appended-share"}, - ] - - ledger.audit_share_window = durable_window # type: ignore[method-assign] - server.ledger = ledger - context = server.jobs["job-1"] - context.shares_json = [{"share_id": "recorded-window-share"}] - context.found_block = { - "network_difficulty": 1, - "anchor_job_issued_at_ms": 12000, - } - bundle_builds: list[dict[str, object]] = [] - - def recording_build_audit_bundle(**kwargs: object) -> dict[str, object]: - bundle_builds.append(dict(kwargs)) - return verified_block_bundle() - - server.build_audit_bundle = recording_build_audit_bundle # type: ignore[method-assign] - server.verify_bundle = lambda *_args, **_kwargs: verified_audit_report() # type: ignore[method-assign] - tail_dir = tempfile.TemporaryDirectory() - self.addCleanup(tail_dir.cleanup) - server.audit_dir = Path(tail_dir.name) - server.evidence_path = Path(tail_dir.name) / "evidence.json" - server.ledger_writer_public_key_hex = "aa" * 32 - - submitblock_calls: list[object] = [] - - class RecordingSubmitRpc(TipRpc): - def call( - rpc_self, - method: str, - params: list[object] | None = None, - ) -> object: - if method == "getblockcount": - return 9 - if method == "submitblock": - submitblock_calls.append(params) - return None - if method == "getblockhash": - return "ec" * 32 - return super().call(method, params) - - server.rpc = RecordingSubmitRpc("00" * 32) - submission = SimpleNamespace( - coinbase_tx_hex="c0ffee", - block_hash_hex="ec" * 32, - block_hex="00", - ) - pending = self._pending_append("replayed-window-omission").pending_share - original = block_candidate( - server, - state, - submission, - pending_share=pending, - ) - intent = server.block_candidate_intent(original) - candidate = server.block_candidate_from_intent(intent) - self.assertEqual(candidate.context.payout_append_invalidation_epoch, -1) - ledger.append_batch([(pending, intent)]) - server.enqueue_block_candidate(candidate) - - self.assertTrue(server.submit_next_block_candidate()) - # The fast lane offered first, so the walk never runs and the - # as-issued snapshot is accounted. - self.assertEqual(submitblock_calls, [["00"]]) - self.assertEqual(window_calls, []) - self.assertEqual(len(bundle_builds), 1) - self.assertEqual(server.stale_share_count, 0) - self.assertNotIn( - PRISM_REJECTION_STALE_JOB, - server.block_candidate_abandoned_counts, - ) - outbox_row = ledger._block_candidate_outbox[submission.block_hash_hex] - self.assertEqual(outbox_row["state"], "submitted") - self.assertIsNone(outbox_row["last_error"]) - self.assertEqual(server.accepted_block_count, 1) - - def test_unoffered_replay_still_rejects_window_omitting_durable_append( - self, - ) -> None: - # The pre-offer half of the same fence: a direct embedder resumes a - # reconstructed candidate with no node offer made (attempted=False, - # as the compatibility probe reports when block bytes are not - # retained), and the durable window surfaces a share the recorded - # coinbase omitted — the landing must still refuse to mint that - # coinbase, and the revalidation walk must run before any offer. - server, state, _recording = submit_coordinator() - ledger = SingleWriterShareLedger() - ledger.durable_payout_state = True # type: ignore[attr-defined] - window_calls: list[tuple[int, int]] = [] - - def durable_window( - *, - anchor_job_issued_at_ms: int, - network_difficulty: int, - ) -> list[dict[str, object]]: - window_calls.append((anchor_job_issued_at_ms, network_difficulty)) - return [ - {"share_id": "recorded-window-share"}, - {"share_id": "late-appended-share"}, - ] - - ledger.audit_share_window = durable_window # type: ignore[method-assign] - server.ledger = ledger - context = server.jobs["job-1"] - context.shares_json = [{"share_id": "recorded-window-share"}] - context.found_block = { - "network_difficulty": 1, - "anchor_job_issued_at_ms": 12000, - } - server.build_audit_bundle = lambda **_kwargs: (_ for _ in ()).throw( # type: ignore[method-assign] - AssertionError( - "audit bundle must not be built from a payout window omitting " - "a durably appended share" - ) - ) - submission = SimpleNamespace( - coinbase_tx_hex="c0ffee", - block_hash_hex="ed" * 32, - block_hex="00", - ) - pending = self._pending_append("unoffered-window-omission").pending_share - original = block_candidate( - server, - state, - submission, - pending_share=pending, - ) - intent = server.block_candidate_intent(original) - candidate = server.block_candidate_from_intent(intent) - self.assertEqual(candidate.context.payout_append_invalidation_epoch, -1) - ledger.append_batch([(pending, intent)]) - - self.assertFalse( - server.submit_block_candidate( - candidate, - node_submission=SimpleNamespace( - attempted=False, - error=None, - result=None, - ), - ) - ) - - self.assertEqual(window_calls, [(12000, 1)]) - self.assertEqual( - server.block_candidate_abandoned_counts[PRISM_REJECTION_STALE_JOB], - 1, - ) - self.assertEqual( - server.stale_job_abandon_counts, - {"tip_moved": 0, "balance_stale": 0, "append_epoch_stale": 1}, - ) - # Direct embedders do not use the outbox-finalization wrapper: - # the terminal outcome is recorded and counted, while the durable - # row stays pending for queue-driven replay to finalize. - outbox_row = ledger._block_candidate_outbox[submission.block_hash_hex] - self.assertEqual(outbox_row["state"], "pending") - - def test_replayed_block_candidate_lands_when_durable_window_replays_intact( - self, - ) -> None: - # The durable revalidation is a fence against omitted appends, not a - # new obstacle to recovery: when the audit window at the declared - # anchor replays exactly the recorded shares, the reconstructed - # candidate still lands -- even while this process's own live epoch - # has advanced past the meaningless replayed stamp. - server, state, ledger = submit_coordinator() - server.stop_after_block = False - server.max_blocks = 10 - ledger.durable_payout_state = True - ledger.audit_share_window = ( # type: ignore[attr-defined] - lambda *, anchor_job_issued_at_ms, network_difficulty: [ - {"share_id": "recorded-window-share"} - ] - ) - context = server.jobs["job-1"] - context.shares_json = [{"share_id": "recorded-window-share"}] - context.found_block = { - "network_difficulty": 1, - "anchor_job_issued_at_ms": 12000, - } - block_hash = "cf" * 32 - pending = self._pending_append("replayed-durable-window").pending_share - original = block_candidate( - server, - state, - SimpleNamespace( - coinbase_tx_hex="c0ffee", - block_hash_hex=block_hash, - block_hex="00", - ), - pending_share=pending, - ) - candidate = server.block_candidate_from_intent( - server.block_candidate_intent(original) - ) - self.assertEqual(candidate.context.payout_append_invalidation_epoch, -1) - server._ensure_job_cache_state() - with server._job_cache_lock: - server._payout_ledger_append_invalidation_epoch += 1 - - with tempfile.TemporaryDirectory() as tempdir: - server.audit_dir = Path(tempdir) - server.evidence_path = Path(tempdir) / "evidence.json" - server.ledger_writer_public_key_hex = "aa" * 32 - server.rpc = SubmitRpc( - tip="00" * 32, - block_hash=block_hash, - ledger=ledger, - ) - server.build_audit_bundle = ( # type: ignore[method-assign] - lambda **_kwargs: verified_block_bundle() - ) - server.verify_bundle = ( # type: ignore[method-assign] - lambda *_args, **_kwargs: verified_audit_report() - ) - accepted = server.submit_block_candidate(candidate) - - self.assertTrue(accepted) - self.assertEqual(server.block_candidate_abandoned_counts, {}) - self.assertEqual(len(ledger.persisted), 1) - self.assertEqual(len(ledger.confirmed), 1) - - def test_block_submit_keeps_as_issued_accounting_when_append_races_the_landing( - self, - ) -> None: - # Node propagation is the fast lane: the block is offered to qbitd - # before payout accounting notices anything, so an append-side - # invalidation racing the landing can no longer block submitblock. - # The landing still observes the bump at its epoch fence, but the - # node already accepted the block, so accounting proceeds with the - # as-issued snapshot instead of terminally abandoning a live - # block's payouts. - # The bump here is driven through the REAL invalidation path from - # the drain hook -- with no armed artifact or in-flight walk in the - # harness, it fires only because the landing exposed its own - # declared anchor. - server, state, _recording = submit_coordinator() - ledger = SingleWriterShareLedger() - server.ledger = ledger - context = server.jobs["job-1"] - context.found_block = { - "network_difficulty": 1, - "anchor_job_issued_at_ms": 12000, - } - bundle_builds: list[dict[str, object]] = [] - - def recording_build_audit_bundle(**kwargs: object) -> dict[str, object]: - bundle_builds.append(dict(kwargs)) - return verified_block_bundle() - - server.build_audit_bundle = recording_build_audit_bundle # type: ignore[method-assign] - server.verify_bundle = lambda *_args, **_kwargs: verified_audit_report() # type: ignore[method-assign] - tail_dir = tempfile.TemporaryDirectory() - self.addCleanup(tail_dir.cleanup) - server.audit_dir = Path(tail_dir.name) - server.evidence_path = Path(tail_dir.name) / "evidence.json" - server.ledger_writer_public_key_hex = "aa" * 32 - late_append = self._pending_append("late-during-landing").pending_share - - race_fired: list[bool] = [] - original_drain = server._await_unfenced_appends_predating_anchor - - def racing_drain(anchor_ms: int) -> None: - original_drain(anchor_ms) - if not race_fired: - race_fired.append(True) - server._invalidate_incremental_payout_window_for_append( - late_append - ) - - server._await_unfenced_appends_predating_anchor = racing_drain # type: ignore[method-assign] - - submitblock_calls: list[object] = [] - - class RecordingSubmitRpc(TipRpc): - def call( - rpc_self, - method: str, - params: list[object] | None = None, - ) -> object: - if method == "getblockcount": - return 9 - if method == "submitblock": - submitblock_calls.append(params) - return None - if method == "getblockhash": - return "da" * 32 - return super().call(method, params) - - server.rpc = RecordingSubmitRpc("00" * 32) - submission = SimpleNamespace( - coinbase_tx_hex="c0ffee", - block_hash_hex="da" * 32, - block_hex="00", - ) - pending = self._pending_append("append-races-landing").pending_share - candidate = block_candidate( - server, - state, - submission, - pending_share=pending, - ) - ledger.append_batch( - [(pending, server.block_candidate_intent(candidate))] - ) - server.enqueue_block_candidate(candidate) - - self.assertTrue(server.submit_next_block_candidate()) - self.assertEqual(race_fired, [True]) - # The fast lane offered the block before the race could be observed. - self.assertEqual(submitblock_calls, [["00"]]) - with server._job_cache_lock: - self.assertEqual(server._payout_ledger_append_invalidation_epoch, 1) - self.assertEqual(server.stale_share_count, 0) - # The landing observed the bump but the node already accepted - # the block: accounting keeps the as-issued snapshot instead of - # terminally abandoning a live block's payouts. - self.assertEqual(len(bundle_builds), 1) - self.assertNotIn( - PRISM_REJECTION_STALE_JOB, - server.block_candidate_abandoned_counts, - ) - self.assertEqual( - getattr(server, "stale_job_abandon_counts", {}).get( - "append_epoch_stale", 0 - ), - 0, - ) - outbox_row = ledger._block_candidate_outbox[submission.block_hash_hex] - self.assertEqual(outbox_row["state"], "submitted") - self.assertIsNone(outbox_row["last_error"]) - self.assertEqual(server.accepted_block_count, 1) - # The landing retires its exposed anchor on the way out. - with server._job_cache_lock: - self.assertEqual(server._payout_window_inflight_scan_anchors, {}) - - def test_offered_replay_lands_as_issued_without_revalidation_walk( - self, - ) -> None: - # The fast lane offers a reconstructed candidate's durable bytes - # before the landing runs, so the slow audit-window walk is skipped - # outright — even while this process's live epoch advances mid- - # flight — and accounting keeps the as-issued snapshot rather than - # abandoning (or re-walking) an accepted block. - server, state, _recording = submit_coordinator() - ledger = SingleWriterShareLedger() - ledger.durable_payout_state = True # type: ignore[attr-defined] - - window_calls: list[tuple[int, int]] = [] - - def recording_audit_share_window( - *, anchor_job_issued_at_ms: int, network_difficulty: int - ) -> list[dict[str, object]]: - window_calls.append((anchor_job_issued_at_ms, network_difficulty)) - return [{"share_id": "recorded-window-share"}] - - ledger.audit_share_window = recording_audit_share_window # type: ignore[method-assign] - server.ledger = ledger - context = server.jobs["job-1"] - context.shares_json = [{"share_id": "recorded-window-share"}] - context.found_block = { - "network_difficulty": 1, - "anchor_job_issued_at_ms": 12000, - } - bundle_builds: list[dict[str, object]] = [] - - def recording_build_audit_bundle(**kwargs: object) -> dict[str, object]: - bundle_builds.append(dict(kwargs)) - return verified_block_bundle() - - server.build_audit_bundle = recording_build_audit_bundle # type: ignore[method-assign] - server.verify_bundle = lambda *_args, **_kwargs: verified_audit_report() # type: ignore[method-assign] - tail_dir = tempfile.TemporaryDirectory() - self.addCleanup(tail_dir.cleanup) - server.audit_dir = Path(tail_dir.name) - server.evidence_path = Path(tail_dir.name) / "evidence.json" - server.ledger_writer_public_key_hex = "aa" * 32 - - submitblock_calls: list[object] = [] - - class RecordingSubmitRpc(TipRpc): - def call( - rpc_self, - method: str, - params: list[object] | None = None, - ) -> object: - if method == "getblockcount": - return 9 - if method == "submitblock": - submitblock_calls.append(params) - return None - if method == "getblockhash": - return "db" * 32 - return super().call(method, params) - - server.rpc = RecordingSubmitRpc("00" * 32) - submission = SimpleNamespace( - coinbase_tx_hex="c0ffee", - block_hash_hex="db" * 32, - block_hex="00", - ) - pending = self._pending_append("replayed-append-race").pending_share - original = block_candidate( - server, - state, - submission, - pending_share=pending, - ) - intent = server.block_candidate_intent(original) - candidate = server.block_candidate_from_intent(intent) - self.assertEqual(candidate.context.payout_append_invalidation_epoch, -1) - ledger.append_batch([(pending, intent)]) - server.enqueue_block_candidate(candidate) - # A live epoch bump mid-flight is irrelevant to a reconstructed - # candidate whose offer is already out: no revalidation rebases it - # and the epoch fences stand down (no effective epoch). - server._ensure_job_cache_state() - with server._job_cache_lock: - server._payout_ledger_append_invalidation_epoch += 1 - - self.assertTrue(server.submit_next_block_candidate()) - # The fast lane offered the replayed block; the walk never ran. - self.assertEqual(submitblock_calls, [["00"]]) - self.assertEqual(window_calls, []) - # The landing observed the bump but the node already accepted - # the block: accounting keeps the as-issued snapshot instead of - # terminally abandoning a live block's payouts. - self.assertEqual(len(bundle_builds), 1) - self.assertNotIn( - PRISM_REJECTION_STALE_JOB, - server.block_candidate_abandoned_counts, - ) - self.assertEqual( - getattr(server, "stale_job_abandon_counts", {}).get( - "append_epoch_stale", 0 - ), - 0, - ) - outbox_row = ledger._block_candidate_outbox[submission.block_hash_hex] - self.assertEqual(outbox_row["state"], "submitted") - self.assertIsNone(outbox_row["last_error"]) - self.assertEqual(server.accepted_block_count, 1) - - def test_landing_blocked_by_fenced_append_lands_as_issued_on_bumped_epoch( - self, - ) -> None: - # A replay-shaped append holds the landing fence across the durable - # commit itself, not only across its epoch bump. The node offer is - # the fast lane and does not wait, but a landing whose offer is - # already in must still wait at the fence before ACCOUNTING; with - # the offer already accepted, the bumped epoch is recorded and - # accounting proceeds with the as-issued coinbase (the durable - # predating share rides the next window) instead of abandoning the - # accepted block. - server, state, _recording = submit_coordinator() - ledger = SingleWriterShareLedger() - server.ledger = ledger - context = server.jobs["job-1"] - context.found_block = { - "network_difficulty": 1, - "anchor_job_issued_at_ms": 12000, - } - bundle_builds: list[dict[str, object]] = [] - - def recording_build_audit_bundle(**kwargs: object) -> dict[str, object]: - bundle_builds.append(dict(kwargs)) - return verified_block_bundle() - - server.build_audit_bundle = recording_build_audit_bundle # type: ignore[method-assign] - server.verify_bundle = lambda *_args, **_kwargs: verified_audit_report() # type: ignore[method-assign] - tail_dir = tempfile.TemporaryDirectory() - self.addCleanup(tail_dir.cleanup) - server.audit_dir = Path(tail_dir.name) - server.evidence_path = Path(tail_dir.name) / "evidence.json" - server.ledger_writer_public_key_hex = "aa" * 32 - - submitblock_calls: list[object] = [] - - class RecordingSubmitRpc(TipRpc): - def call( - rpc_self, - method: str, - params: list[object] | None = None, - ) -> object: - if method == "getblockcount": - return 9 - if method == "submitblock": - submitblock_calls.append(params) - return None - if method == "getblockhash": - return "dc" * 32 - return super().call(method, params) - - server.rpc = RecordingSubmitRpc("00" * 32) - submission = SimpleNamespace( - coinbase_tx_hex="c0ffee", - block_hash_hex="dc" * 32, - block_hex="00", - ) - pending = self._pending_append("landing-vs-fenced-append").pending_share - candidate = block_candidate( - server, - state, - submission, - pending_share=pending, - ) - ledger.append_batch( - [(pending, server.block_candidate_intent(candidate))] - ) - server.enqueue_block_candidate(candidate) - - # Expose a live anchor the late row predates, as an armed window - # would: the writer's pre-commit peek must fire before any landing - # is in flight to declare its own anchor. - anchor_token = server._expose_inflight_scan_anchor(12000) - self.addCleanup(server._retire_inflight_scan_anchor, anchor_token) - - commit_entered = threading.Event() - release_commit = threading.Event() - self.addCleanup(release_commit.set) - original_append_batch = ledger.append_batch - - def gated_append_batch(entries: list[object]) -> list[object]: - commit_entered.set() - release_commit.wait(timeout=10.0) - return original_append_batch(entries) - - ledger.append_batch = gated_append_batch # type: ignore[method-assign] - late_entry = self._pending_append("late-fenced-append") - writer = threading.Thread( - target=server._append_share_batch, - args=([late_entry],), - daemon=True, - ) - writer.start() - self.assertTrue(commit_entered.wait(timeout=10.0)) - # The fence is held while the predating row's commit is in flight. - self.assertTrue(server._payout_append_landing_fence_lock.locked()) - - landing = threading.Thread( - target=server.submit_next_block_candidate, - daemon=True, - ) - landing.start() - # Give the landing time to run up against the fence: the fast-lane - # node offer already went out, but with the commit still gated the - # landing must not have reached a terminal accounting decision. - time.sleep(0.05) - self.assertEqual(submitblock_calls, [["00"]]) - self.assertEqual(server.block_candidate_abandoned_counts, {}) - - release_commit.set() - writer.join(timeout=10.0) - self.assertFalse(writer.is_alive()) - landing.join(timeout=10.0) - self.assertFalse(landing.is_alive()) - - # The bump landed together with the durable append; the landing - # observed it, and with the offer already accepted it kept the - # as-issued snapshot for accounting. - self.assertEqual(submitblock_calls, [["00"]]) - self.assertIn(late_entry.pending_share.share_id, ledger._share_ids) - with server._job_cache_lock: - self.assertEqual(server._payout_ledger_append_invalidation_epoch, 1) - # The landing observed the bump but the node already accepted - # the block: accounting keeps the as-issued snapshot instead of - # terminally abandoning a live block's payouts. - self.assertEqual(len(bundle_builds), 1) - self.assertNotIn( - PRISM_REJECTION_STALE_JOB, - server.block_candidate_abandoned_counts, - ) - self.assertEqual( - getattr(server, "stale_job_abandon_counts", {}).get( - "append_epoch_stale", 0 - ), - 0, - ) - outbox_row = ledger._block_candidate_outbox[submission.block_hash_hex] - self.assertEqual(outbox_row["state"], "submitted") - self.assertIsNone(outbox_row["last_error"]) - self.assertEqual(server.accepted_block_count, 1) - - def test_lease_deferral_unwinds_landed_bar_before_submitblock(self) -> None: - # The landed bar is armed before the lease fence to keep a transport - # failure's uncertain submitblock outcome conservative. A - # WriterLeaseRenewalDeferred refusal fires before the RPC, so the - # outcome is not uncertain: qbitd provably never saw the block, and - # leaving the bar armed in the (now surviving) process would bar - # reconciliation and payout-state publication for as long as the - # writer's own fenced write keeps the renewal deferred. The pending - # preview must survive, though: child work keeps waiting for the - # retry instead of snapshotting pre-accept balances. - server, state, _recording = submit_coordinator() - ledger = SingleWriterShareLedger() - server.ledger = ledger - server.block_candidate_retry_initial_seconds = 0.0 - - class NoSubmitRpc(TipRpc): - def call( - rpc_self, - method: str, - params: list[object] | None = None, - ) -> object: - if method == "getblockcount": - return 9 - if method == "submitblock": - raise AssertionError( - "submitblock must not run after a deferral refusal" - ) - return super().call(method, params) - - server.rpc = NoSubmitRpc("00" * 32) - # Force the fenced fallback lane: with no fast-lane node offer - # (attempted=False), the disposition itself must clear the lease - # fence before submitblock — the branch the deferral exits from. - server._node_submission_for_candidate = ( # type: ignore[method-assign] - lambda _candidate: SimpleNamespace( - attempted=False, - result=None, - error=None, - ) - ) - submission = SimpleNamespace( - coinbase_tx_hex="c0ffee", - block_hash_hex="de" * 32, - block_hex="00", - ) - pending = self._pending_append("deferral-unwind").pending_share - candidate = block_candidate( - server, - state, - submission, - pending_share=pending, - ) - ledger.append_batch( - [(pending, server.block_candidate_intent(candidate))] - ) - server.enqueue_block_candidate(candidate) - - with patch.object( - server, - "_require_fresh_ledger_lease_for_external_side_effect", - side_effect=WriterLeaseRenewalDeferred("renewal is deferred"), - ) as fence: - self.assertTrue(server.submit_next_block_candidate()) - - fence.assert_called_once_with("submitblock") - block_hash = str(submission.block_hash_hex).lower() - self.assertFalse( - server._accepted_block_payout_transition_landed(block_hash) - ) - with server._accepted_block_payout_preview_condition: - self.assertIn(block_hash, server._accepted_block_payout_previews) - - def test_lease_deferral_keeps_prior_attempts_landed_bar(self) -> None: - # A transition landed by an earlier attempt that DID reach - # submitblock records a genuinely uncertain outcome (a lost RPC - # ack); a later attempt's pre-RPC deferral must not withdraw that - # bar, or payout state could publish without the possibly-active - # coinbase's base preserved. - server, state, _recording = submit_coordinator() - ledger = SingleWriterShareLedger() - server.ledger = ledger - server.block_candidate_retry_initial_seconds = 0.0 - - class NoSubmitRpc(TipRpc): - def call( - rpc_self, - method: str, - params: list[object] | None = None, - ) -> object: - if method == "getblockcount": - return 9 - if method == "submitblock": - raise AssertionError( - "submitblock must not run after a deferral refusal" - ) - return super().call(method, params) - - server.rpc = NoSubmitRpc("00" * 32) - # Force the fenced fallback lane, as above. - server._node_submission_for_candidate = ( # type: ignore[method-assign] - lambda _candidate: SimpleNamespace( - attempted=False, - result=None, - error=None, - ) - ) - submission = SimpleNamespace( - coinbase_tx_hex="c0ffee", - block_hash_hex="df" * 32, - block_hex="00", - ) - pending = self._pending_append("deferral-keeps-bar").pending_share - candidate = block_candidate( - server, - state, - submission, - pending_share=pending, - ) - ledger.append_batch( - [(pending, server.block_candidate_intent(candidate))] - ) - block_hash = str(submission.block_hash_hex).lower() - server._begin_accepted_block_payout_preview(block_hash, block_height=10) - server._mark_accepted_block_payout_landed(block_hash, block_height=10) - server.enqueue_block_candidate(candidate) - - with patch.object( - server, - "_require_fresh_ledger_lease_for_external_side_effect", - side_effect=WriterLeaseRenewalDeferred("renewal is deferred"), - ) as fence: - self.assertTrue(server.submit_next_block_candidate()) - - fence.assert_called_once_with("submitblock") - self.assertTrue( - server._accepted_block_payout_transition_landed(block_hash) - ) - - def test_landing_drains_unfenced_inflight_append_before_epoch_fences( - self, - ) -> None: - # The unfenced classification is a one-time predicate: an append - # checked while no anchor is exposed commits outside the landing - # fence, so a landing that exposes its declared anchor afterwards - # could hold the fence across submitblock while the row becomes - # durable mid-RPC and its epoch bump queues behind that same - # fence -- arriving too late to reject the block. The landing must - # instead drain such in-flight commits right after exposing its - # anchor, so the bump lands before its epoch fences run. - server, state, _recording = submit_coordinator() - ledger = SingleWriterShareLedger() - server.ledger = ledger - bundle_builds: list[dict[str, object]] = [] - - def recording_build_audit_bundle(**kwargs: object) -> dict[str, object]: - bundle_builds.append(dict(kwargs)) - return verified_block_bundle() - - server.build_audit_bundle = recording_build_audit_bundle # type: ignore[method-assign] - server.verify_bundle = lambda *_args, **_kwargs: verified_audit_report() # type: ignore[method-assign] - tail_dir = tempfile.TemporaryDirectory() - self.addCleanup(tail_dir.cleanup) - server.audit_dir = Path(tail_dir.name) - server.evidence_path = Path(tail_dir.name) / "evidence.json" - server.ledger_writer_public_key_hex = "aa" * 32 - context = server.jobs["job-1"] - context.found_block = { - "network_difficulty": 1, - "anchor_job_issued_at_ms": 12000, - } - submission = SimpleNamespace( - coinbase_tx_hex="c0ffee", - block_hash_hex="dc" * 32, - block_hex="00", - ) - pending = self._pending_append("landing-drains").pending_share - candidate = block_candidate( - server, - state, - submission, - pending_share=pending, - ) - ledger.append_batch( - [(pending, server.block_candidate_intent(candidate))] - ) - server.enqueue_block_candidate(candidate) - - late_entry = self._pending_append("unfenced-inflight") - commit_entered = threading.Event() - commit_release = threading.Event() - original_append_batch = ledger.append_batch - - def gated_append_batch(batch: list[object]) -> list[object]: - commit_entered.set() - if not commit_release.wait(timeout=10.0): - raise AssertionError("gated ledger commit was never released") - return original_append_batch(batch) - - ledger.append_batch = gated_append_batch # type: ignore[method-assign] - - # Release the gated commit exactly when the landing starts - # draining, so the test exercises the wait itself rather than a - # lucky ordering. - drained_anchors: list[int] = [] - original_drain = server._await_unfenced_appends_predating_anchor - - def recording_drain(anchor_ms: int) -> None: - drained_anchors.append(int(anchor_ms)) - commit_release.set() - original_drain(anchor_ms) - - server._await_unfenced_appends_predating_anchor = recording_drain # type: ignore[method-assign] - - submitblock_calls: list[object] = [] - - class RecordingSubmitRpc(TipRpc): - def call( - rpc_self, - method: str, - params: list[object] | None = None, - ) -> object: - if method == "getblockcount": - return 9 - if method == "submitblock": - submitblock_calls.append(params) - return None - if method == "getblockhash": - return "dc" * 32 - return super().call(method, params) - - server.rpc = RecordingSubmitRpc("00" * 32) - - append_thread = threading.Thread( - target=server._append_share_entry, - args=(late_entry,), - daemon=True, - ) - append_thread.start() - # The append was classified with no anchor exposed and is now - # committing outside the fence. - self.assertTrue(commit_entered.wait(timeout=10.0)) - - self.assertTrue(server.submit_next_block_candidate()) - - append_thread.join(timeout=10.0) - self.assertFalse(append_thread.is_alive()) - self.assertEqual(drained_anchors, [12000]) - self.assertIn(late_entry.pending_share.share_id, ledger._share_ids) - with server._job_cache_lock: - self.assertEqual(server._payout_ledger_append_invalidation_epoch, 1) - self.assertEqual( - server._payout_unfenced_append_inflight_stamps, {} - ) - # The drained append's bump landed before the landing's epoch - # fences; the landing observed it and, with the node offer already - # accepted, accounted the as-issued window (the durable predating - # share rides the next window). - self.assertEqual(submitblock_calls, [["00"]]) - # The landing observed the bump but the node already accepted - # the block: accounting keeps the as-issued snapshot instead of - # terminally abandoning a live block's payouts. - self.assertEqual(len(bundle_builds), 1) - self.assertNotIn( - PRISM_REJECTION_STALE_JOB, - server.block_candidate_abandoned_counts, - ) - self.assertEqual( - getattr(server, "stale_job_abandon_counts", {}).get( - "append_epoch_stale", 0 - ), - 0, - ) - outbox_row = ledger._block_candidate_outbox[submission.block_hash_hex] - self.assertEqual(outbox_row["state"], "submitted") - self.assertIsNone(outbox_row["last_error"]) - self.assertEqual(server.accepted_block_count, 1) - - def test_landing_builds_audit_bundle_outside_balance_serializer(self) -> None: - # The builder/verifier subprocess time must stay off the - # payout-balance mutation lock: job delivery and reconciliation - # queue behind that lock, and holding it across the bundle build was - # the dominant term of the finalization delivery stall. - server, state, ledger = submit_coordinator() - server.stop_after_block = False - server.max_blocks = 10 - lock_held_during_build: list[bool] = [] - build_calls: list[int] = [] - - def probe_lock_from_other_thread() -> bool: - # An RLock re-acquires freely on the owning thread, so the probe - # must come from a thread that cannot be the owner. - acquired: list[bool] = [] - - def attempt() -> None: - got = server._payout_balance_mutation_lock.acquire( - blocking=False - ) - if got: - server._payout_balance_mutation_lock.release() - acquired.append(not got) - - prober = threading.Thread(target=attempt) - prober.start() - prober.join(5) - return bool(acquired and acquired[0]) - - def fake_build_audit_bundle(**_kwargs: object) -> dict[str, object]: - build_calls.append(1) - lock_held_during_build.append(probe_lock_from_other_thread()) - return verified_block_bundle() - - with tempfile.TemporaryDirectory() as tempdir: - server.audit_dir = Path(tempdir) - server.evidence_path = Path(tempdir) / "evidence.json" - server.ledger_writer_public_key_hex = "aa" * 32 - block_hash = "cc" * 32 - server.rpc = SubmitRpc( - tip="00" * 32, - block_hash=block_hash, - ledger=ledger, - ) - server.build_audit_bundle = fake_build_audit_bundle # type: ignore[method-assign] - server.verify_bundle = ( # type: ignore[method-assign] - lambda *_args, **_kwargs: verified_audit_report() - ) - submission = SimpleNamespace( - coinbase_tx_hex="c0ffee", - block_hash_hex=block_hash, - block_hex="00", - ) - pending = SimpleNamespace(share_id="miner-a:" + block_hash) - accepted = server.submit_block_candidate( - block_candidate(server, state, submission, pending_share=pending) - ) - - self.assertTrue(accepted) - self.assertEqual(build_calls, [1]) - self.assertEqual(lock_held_during_build, [False]) - self.assertEqual(len(ledger.persisted), 1) - self.assertEqual(len(ledger.confirmed), 1) - - def test_block_submitted_before_audit_bundle_build(self) -> None: - # The audit build runs with the balance serializer released, but - # never before submitblock: announcement of a freshly solved block - # must not wait on audit construction (a lost race is a lost round). - server, state, ledger = submit_coordinator() - server.stop_after_block = False - server.max_blocks = 10 - order: list[str] = [] - block_hash = "cc" * 32 - - class OrderRecordingRpc(SubmitRpc): - def call( - self, - method: str, - params: list[object] | None = None, - ) -> object: - if method == "submitblock": - order.append("submitblock") - return super().call(method, params) - - def ordered_build(**_kwargs: object) -> dict[str, object]: - order.append("build_audit_bundle") - return verified_block_bundle() - - with tempfile.TemporaryDirectory() as tempdir: - server.audit_dir = Path(tempdir) - server.evidence_path = Path(tempdir) / "evidence.json" - server.ledger_writer_public_key_hex = "aa" * 32 - server.rpc = OrderRecordingRpc( - tip="00" * 32, - block_hash=block_hash, - ledger=ledger, - ) - server.build_audit_bundle = ordered_build # type: ignore[method-assign] - server.verify_bundle = ( # type: ignore[method-assign] - lambda *_args, **_kwargs: verified_audit_report() - ) - submission = SimpleNamespace( - coinbase_tx_hex="c0ffee", - block_hash_hex=block_hash, - block_hex="00", - ) - pending = SimpleNamespace(share_id="miner-a:" + block_hash) - accepted = server.submit_block_candidate( - block_candidate(server, state, submission, pending_share=pending) - ) - - self.assertTrue(accepted) - self.assertEqual(order, ["submitblock", "build_audit_bundle"]) - self.assertEqual(len(ledger.persisted), 1) - self.assertEqual(len(ledger.confirmed), 1) - - def test_successful_fast_lane_reconciles_observed_post_submit_tip(self) -> None: - parent_hash = "00" * 32 - block_hash = "cd" * 32 - server, state, ledger = submit_coordinator(tip=parent_hash) - server.stop_after_block = False - server.max_blocks = 10 - server.rpc = SubmitAcceptingTemplateRpc( - old_tip=parent_hash, - block_hash=block_hash, - ledger=ledger, - ) - reconciled: list[tuple[str, bool]] = [] - landing_inputs: list[tuple[str, bool]] = [] - - def reconcile( - tip_hash: str, - *, - _coalesce_same_tip: bool = True, - ) -> bool: - reconciled.append((tip_hash, _coalesce_same_tip)) - return True - - original_land = server._land_and_confirm_block_candidate - - def observe_landing( - candidate: PrismBlockCandidate, - **kwargs: object, - ) -> object: - landing_inputs.append( - (str(kwargs["current_tip"]), bool(kwargs["already_active"])) - ) - return original_land(candidate, **kwargs) # type: ignore[arg-type] - - server.ensure_reorg_reconciled_for_tip = reconcile # type: ignore[method-assign] - server._land_and_confirm_block_candidate = observe_landing # type: ignore[method-assign] - server.build_audit_bundle = ( # type: ignore[method-assign] - lambda **_kwargs: verified_block_bundle() - ) - server.verify_bundle = ( # type: ignore[method-assign] - lambda *_args, **_kwargs: verified_audit_report() - ) - submission = SimpleNamespace( - coinbase_tx_hex="c0ffee", - block_hash_hex=block_hash, - block_hex="00", - ) - - with tempfile.TemporaryDirectory() as tempdir: - server.audit_dir = Path(tempdir) - server.evidence_path = Path(tempdir) / "evidence.json" - server.ledger_writer_public_key_hex = "aa" * 32 - accepted = server.submit_block_candidate( - block_candidate(server, state, submission) - ) - - self.assertTrue(accepted) - self.assertEqual(landing_inputs, [(block_hash, False)]) - self.assertEqual(reconciled, [(block_hash, False)]) - - def test_submitter_offers_block_before_writer_admission_with_rpc_deadline(self) -> None: - server, state, _ledger = submit_coordinator() - server.block_submit_rpc_timeout_seconds = 0.75 - order: list[object] = [] - - class TimeoutRecordingRpc(TipRpc): - def call( - self, - method: str, - params: list[object] | None = None, - *, - timeout: float | None = None, - ) -> object: - if method == "submitblock": - order.append(("submitblock", timeout)) - return None - return super().call(method, params) - - class WriterAdmission: - def __enter__(self) -> None: - order.append("writer-admission") - - def __exit__(self, *_args: object) -> None: - return None - - candidate = block_candidate( - server, - state, - SimpleNamespace( - coinbase_tx_hex="00", - block_hash_hex="ce" * 32, - block_hex="00", - share_pass=True, - block_pass=True, - ), - ) - server.rpc = TimeoutRecordingRpc("00" * 32) - server._writer_operation = lambda _component: WriterAdmission() # type: ignore[method-assign] - - def account( - _candidate: PrismBlockCandidate, - *, - node_submission: object, - ) -> bool: - order.append("accounting") - self.assertIsNone(getattr(node_submission, "result")) - return True - - server._submit_next_block_candidate_writer = account # type: ignore[method-assign] - server.enqueue_block_candidate(candidate) - - self.assertTrue(server.submit_next_block_candidate()) - self.assertEqual( - order, - [("submitblock", 0.75), "writer-admission", "accounting"], - ) - - def test_accepted_candidate_with_moved_tip_finalizes_as_submitted(self) -> None: - old_tip = "00" * 32 - new_tip = "11" * 32 - block_hash = "cc" * 32 - server, state, _recording = submit_coordinator(tip=old_tip) - ledger = SingleWriterShareLedger() - server.ledger = ledger - pending = self._pending_append("accepted-tip-race").pending_share - candidate = block_candidate( - server, - state, - SimpleNamespace( - coinbase_tx_hex="00", - block_hash_hex=block_hash, - block_hex="00", - share_pass=True, - block_pass=True, - ), - pending_share=pending, - ) - ledger.append_batch( - [(pending, server.block_candidate_intent(candidate))] - ) - submitted: list[str] = [] - abandoned: list[str] = [] - original_submitted = ledger.mark_block_candidate_submitted - original_abandoned = ledger.mark_block_candidate_abandoned - - def mark_submitted(*, block_hash: str) -> bool: - submitted.append(block_hash) - return original_submitted(block_hash=block_hash) - - def mark_abandoned(*, block_hash: str, error: str) -> bool: - abandoned.append(block_hash) - return original_abandoned(block_hash=block_hash, error=error) - - ledger.mark_block_candidate_submitted = mark_submitted # type: ignore[method-assign] - ledger.mark_block_candidate_abandoned = mark_abandoned # type: ignore[method-assign] - server._begin_accepted_block_payout_preview( - block_hash, - block_height=10, - ) - server._mark_accepted_block_payout_landed( - block_hash, - block_height=10, - ) - with server.lock: - # This is the process-local record written immediately before the - # "qbit accepted direct PRISM block" log. - server._accounted_accepted_block_hashes.add(block_hash) - server.rpc = TipRpc(new_tip) - server.build_audit_bundle = ( # type: ignore[method-assign] - lambda **_kwargs: (_ for _ in ()).throw( - AssertionError("an accounted candidate must not be rebuilt") - ) - ) - server.enqueue_block_candidate(candidate) - - self.assertTrue(server.submit_next_block_candidate()) - - self.assertEqual(submitted, [block_hash]) - self.assertEqual(abandoned, []) - self.assertEqual(ledger.pending_block_candidates(), []) - self.assertNotIn( - PRISM_REJECTION_STALE_JOB, - server.block_candidate_abandoned_counts, - ) - self.assertNotIn(block_hash, server._accepted_block_payout_previews) - self.assertNotIn( - block_hash, - server._invalidated_accepted_block_payout_previews, - ) - - def test_terminal_abandon_immediately_clears_payout_preview(self) -> None: - server, _state, _ledger = submit_coordinator() - block_hash = "a4" * 32 - server._begin_accepted_block_payout_preview( - block_hash, - block_height=10, - ) - server._mark_accepted_block_payout_landed( - block_hash, - block_height=10, - ) - - accepted_race_won = server._abandon_block_candidate( - PRISM_REJECTION_STALE_JOB, - "tip moved", - block_hash=block_hash, - worker="miner-a", - ) - - self.assertFalse(accepted_race_won) - self.assertNotIn(block_hash, server._accepted_block_payout_previews) - self.assertIn( - block_hash, - server._invalidated_accepted_block_payout_previews, - ) - # The tombstone protects pending durable replay, but unlike the leaked - # landed transition it does not block payout reconciliation. - with server._payout_balance_mutation(): - pass - - def test_tip_moved_abandon_yields_to_acceptance_during_cleanup(self) -> None: - old_tip = "00" * 32 - new_tip = "11" * 32 - block_hash = "a5" * 32 - server, state, _recording = submit_coordinator(tip=old_tip) - configure_temporary_audit_root(self, server) - ledger = SingleWriterShareLedger() - server.ledger = ledger - pending = self._pending_append("accepted-during-clear").pending_share - candidate = block_candidate( - server, - state, - SimpleNamespace( - coinbase_tx_hex="00", - block_hash_hex=block_hash, - block_hex="00", - share_pass=True, - block_pass=True, - ), - pending_share=pending, - ) - ledger.append_batch( - [(pending, server.block_candidate_intent(candidate))] - ) - submitted: list[str] = [] - abandoned: list[str] = [] - original_submitted = ledger.mark_block_candidate_submitted - original_abandoned = ledger.mark_block_candidate_abandoned - original_clear = server._clear_accepted_block_payout_preview - - def mark_submitted(*, block_hash: str) -> bool: - submitted.append(block_hash) - return original_submitted(block_hash=block_hash) - - def mark_abandoned(*, block_hash: str, error: str) -> bool: - abandoned.append(block_hash) - return original_abandoned(block_hash=block_hash, error=error) - - def clear_while_accepting( - candidate_hash: str, - *, - invalidate_published: bool = False, - ) -> None: - original_clear( - candidate_hash, - invalidate_published=invalidate_published, - ) - if invalidate_published: - with server.lock: - server._accounted_accepted_block_hashes.add( - candidate_hash.lower() - ) - - ledger.mark_block_candidate_submitted = mark_submitted # type: ignore[method-assign] - ledger.mark_block_candidate_abandoned = mark_abandoned # type: ignore[method-assign] - server._clear_accepted_block_payout_preview = clear_while_accepting # type: ignore[method-assign] - server._begin_accepted_block_payout_preview( - block_hash, - block_height=10, - ) - server._mark_accepted_block_payout_landed( - block_hash, - block_height=10, - ) - server.rpc = RejectingSubmitTipRpc(new_tip) - server.enqueue_block_candidate(candidate) - - self.assertTrue(server.submit_next_block_candidate()) - - self.assertEqual(submitted, [block_hash]) - self.assertEqual(abandoned, []) - self.assertNotIn( - PRISM_REJECTION_STALE_JOB, - server.block_candidate_abandoned_counts, - ) - metrics = server.metrics_payload() - self.assertIn( - 'qbit_prism_stale_job_abandons_total{class="tip_moved"} 0', - metrics, - ) - self.assertIn( - 'qbit_prism_stale_job_abandons_total{class="balance_stale"} 0', - metrics, - ) - self.assertEqual(ledger.pending_block_candidates(), []) - outbox_row = ledger._block_candidate_outbox[block_hash] - self.assertEqual(outbox_row["state"], "submitted") - self.assertIsNone(outbox_row["last_error"]) - self.assertNotIn(block_hash, server._accepted_block_payout_previews) - self.assertNotIn( - block_hash, - server._invalidated_accepted_block_payout_previews, - ) - - def test_crash_after_submit_before_attempt_mark_replays_duplicate_once(self) -> None: - parent_hash = "00" * 32 - block_hash = "cf" * 32 - ledger = SingleWriterShareLedger() - first, state, _recording = submit_coordinator(tip=parent_hash) - first.ledger = ledger - first.stop_after_block = False - first.max_blocks = 10 - pending = self._pending_append("submit-before-mark-crash").pending_share - candidate = block_candidate( - first, - state, - SimpleNamespace( - coinbase_tx_hex="c0ffee", - block_hash_hex=block_hash, - block_hex="00", - share_pass=True, - block_pass=True, - ), - pending_share=pending, - ) - intent = first.block_candidate_intent(candidate) - ledger.append_batch([(pending, intent)]) - - class RestartAwareRpc(FakeRpc): - def __init__(self) -> None: - self.submit_results: list[object] = [] - - def call( - self, - method: str, - params: list[object] | None = None, - *, - timeout: float | None = None, - ) -> object: - if method == "submitblock": - result = None if not self.submit_results else "duplicate" - self.submit_results.append(result) - return result - if method == "getbestblockhash": - return parent_hash - if method == "getblockhash": - return block_hash - if method == "getblockcount": - return 9 - return super().call(method, params) - - rpc = RestartAwareRpc() - first.rpc = rpc - original_mark = ledger.mark_block_candidate_attempted - - def crash_before_mark(*, block_hash: str) -> bool: - raise SystemExit(f"simulated crash before marking {block_hash}") - - ledger.mark_block_candidate_attempted = crash_before_mark # type: ignore[method-assign] - first.enqueue_block_candidate(candidate) - with self.assertRaisesRegex(SystemExit, "simulated crash"): - first.submit_next_block_candidate() - - self.assertEqual(rpc.submit_results, [None]) - self.assertEqual(ledger.pending_block_candidates(), [intent]) - self.assertEqual( - ledger._block_candidate_outbox[block_hash]["attempt_count"], - 0, - ) - - ledger.mark_block_candidate_attempted = original_mark # type: ignore[method-assign] - persisted_calls = 0 - confirmed_calls = 0 - original_persist = ledger.persist_accepted_block - original_confirm = ledger.confirm_accepted_block - - def persist_once(**kwargs: object) -> dict[str, object]: - nonlocal persisted_calls - persisted_calls += 1 - return original_persist(**kwargs) - - def confirm_once(**kwargs: object) -> dict[str, object]: - nonlocal confirmed_calls - confirmed_calls += 1 - return original_confirm(**kwargs) - - ledger.persist_accepted_block = persist_once # type: ignore[method-assign] - ledger.confirm_accepted_block = confirm_once # type: ignore[method-assign] - - restarted, _restart_state, _recording = submit_coordinator(tip=parent_hash) - restarted.ledger = ledger - restarted.rpc = rpc - restarted.stop_after_block = False - restarted.max_blocks = 10 - with tempfile.TemporaryDirectory() as tempdir: - restarted.audit_dir = Path(tempdir) - restarted.evidence_path = Path(tempdir) / "evidence.json" - restarted.ledger_writer_public_key_hex = "aa" * 32 - restarted.build_audit_bundle = ( # type: ignore[method-assign] - lambda **_kwargs: verified_block_bundle() - ) - restarted.verify_bundle = ( # type: ignore[method-assign] - lambda *_args, **_kwargs: verified_audit_report() - ) - - self.assertEqual(restarted.replay_pending_block_candidates(), 1) - self.assertTrue(restarted.submit_next_block_candidate()) - self.assertEqual(restarted.replay_pending_block_candidates(), 0) - - self.assertEqual(rpc.submit_results, [None, "duplicate"]) - self.assertEqual(persisted_calls, 1) - self.assertEqual(confirmed_calls, 1) - self.assertEqual(restarted.accepted_block_count, 1) - self.assertEqual(ledger.pending_block_candidates(), []) - self.assertEqual( - ledger._block_candidate_outbox[block_hash]["state"], - "submitted", - ) - - def test_block_submitter_honors_stop_after_block_above_one_block_capacity( - self, - ) -> None: - server, state, ledger = submit_coordinator() - server.accepted_block_count = 1 - server.max_blocks = 2 - server.stop_after_block = True - submission = SimpleNamespace( - coinbase_tx_hex="c0ffee", - block_hash_hex="de" * 32, - block_hex="00", - ) - - accepted = server.submit_block_candidate( - block_candidate(server, state, submission) - ) - - self.assertFalse(accepted) - self.assertEqual( - server.block_candidate_abandoned_counts[PRISM_REJECTION_POOL_CLOSED], - 1, - ) - self.assertEqual(ledger.persisted, []) - - @staticmethod - def _reconcile_coordinator_with_artifact_counter( - ledger: ReorgLedger, - rpc: ReorgRpc, - ) -> tuple[PrismCoordinator, list[tuple[int, int, bool, bool]]]: - """Coordinator armed so candidate preparation would build an artifact.""" - server = coordinator() - server.reorg_reconciler_enabled = True - server.ledger = ledger - server.rpc = rpc - server._ensure_job_cache_state() - # Keep the speculative background preparer out of the way so the - # counter observes only builds requested by reconcile passes. - server._payout_artifact_executor_shutdown = True - server._pool_ready_latched = True - server._template_artifacts = SimpleNamespace(network_difficulty=1) - build_calls: list[tuple[int, int, bool, bool]] = [] - - def fake_build( - expected_payout_state_generation: int, - artifact_payout_state_generation: int, - network_difficulty: int, - force_full_rescan: bool = False, - bypass_build_interval: bool = False, - during_publication: bool = False, - *, - force_prior_balances_read: bool = False, - ) -> None: - del network_difficulty, bypass_build_interval, during_publication - build_calls.append( - ( - expected_payout_state_generation, - artifact_payout_state_generation, - force_full_rescan, - force_prior_balances_read, - ) - ) - return None - - server._build_payout_ledger_artifact = fake_build # type: ignore[method-assign] - return server, build_calls - - -class PrismStampedJobFloorTests(unittest.TestCase): - """The listener floor must hold on the wire, not just in vardiff policy. - - Stamped jobs are the single choke point for every mining.set_difficulty - the coordinator sends, and marketplace verification judges the first one. - The regression here is a young chain: qbit network difficulty below the - high-diff floor used to drag the advertised difficulty down with it. - """ - - def stamp_coordinator(self) -> PrismCoordinator: - server = coordinator() - server.job_counter = 0 - server.share_weights_by_username = {} - server.default_share_weight = 1 - return server - - def cached_bundle(self) -> CachedJobBundle: - # bits 207fffff: regtest-grade network difficulty (~4.7e-10), far - # below the 500k marketplace floor. - qbit_target = target_from_compact("207fffff") - base_job = direct_stratum.DirectQbitStratumJob( - job_id="prism-template-base", - previousblockhash_display="00" * 32, - prevhash="00" * 32, - coinb1="", - coinb2="", - full_coinbase_prefix="", - full_coinbase_suffix="", - merkle_branch=(), - transaction_hexes=(), - version="20000000", - nbits="207fffff", - ntime="6553f100", - qbit_target=qbit_target, - share_target=qbit_target, - share_difficulty=Decimal("1"), - extranonce1_hex="ffffffff", - extranonce2_size=8, - clean_jobs=True, - ) - return CachedJobBundle( - key=("test",), - template=gbt_template("00" * 32), - template_fingerprint="fp", - coinbase_manifest={}, - shares_json=[], - prior_balances=[], - found_block={"network_difficulty": 1}, - collection_only=False, - issued_at_ms=12345, - base_job=base_job, - built_monotonic=time.monotonic(), - ) - - def highdiff_client(self) -> ClientState: - state = client() - state.worker = worker_identity() - state.listener_vardiff_config = highdiff_vardiff_config() - state.minimum_advertised_difficulty = Decimal("500000") - state.share_difficulty = Decimal("500000") - return state - - def test_block_worthy_submission_below_share_target_submits_synchronously(self) -> None: - # With the floor above network difficulty a hash can solve a block - # while missing the advertised share target. It is a valid share only - # if the block lands, so it submits synchronously (not via the async - # queue) and the share credit lands with it. - server, state, ledger = submit_coordinator() - submission = SimpleNamespace( - header_hex="aa" * 80, - block_hash_hex="bb" * 32, - share_pass=False, - block_pass=True, - ) - submitted: list[object] = [] - - def fake_submit(candidate: object) -> bool: - submitted.append(candidate) - server.append_accepted_share( - candidate.client, candidate.context, candidate.submission, candidate.pending_share - ) - return True - - server.submit_block_candidate = fake_submit # type: ignore[method-assign] - with patch( - "lab.prism.prism_coordinator.direct_stratum.assemble_submission", - return_value=submission, - ): - should_close = server.handle_submit( - state, - ["miner-a", "job-1", "00" * 8, "00000001", "00000002"], - ) - - self.assertFalse(should_close) - self.assertEqual(server.rejection_counts_by_reason[PRISM_REJECTION_LOW_DIFFICULTY], 0) - self.assertEqual(len(submitted), 1) - self.assertTrue(submitted[0].credit_share_on_accept) - # Nothing was queued to the async submitter; it landed inline. - self.assertEqual(server.block_candidate_queue.qsize(), 0) - self.assertEqual(len(ledger.pending), 1) - - def test_below_target_block_intent_is_durable_before_synchronous_submit(self) -> None: - server, state, _recording = submit_coordinator() - ledger = SingleWriterShareLedger() - server.ledger = ledger - submission = SimpleNamespace( - header_hex="aa" * 80, - coinbase_tx_hex="00", - block_hex="00", - block_hash_hex="bc" * 32, - share_pass=False, - block_pass=True, - ) - - def fake_submit(candidate: PrismBlockCandidate) -> bool: - pending = ledger.pending_block_candidates() - self.assertEqual(len(pending), 1) - self.assertTrue(pending[0]["credit_share_on_accept"]) - server.append_accepted_share( - candidate.client, - candidate.context, - candidate.submission, - candidate.pending_share, - candidate_intent=server.block_candidate_intent(candidate), - ) - return True - - server.submit_block_candidate = fake_submit # type: ignore[method-assign] - with patch( - "lab.prism.prism_coordinator.direct_stratum.assemble_submission", - return_value=submission, - ): - self.assertFalse( - server.handle_submit( - state, - ["miner-a", "job-1", "00" * 8, "00000001", "00000002"], - ) - ) - - self.assertEqual(len(ledger), 1) - self.assertEqual(ledger.pending_block_candidates(), []) - - def test_below_target_intent_failure_does_not_create_unsafe_retry_slot(self) -> None: - server, state, _recording = submit_coordinator() - ledger = SingleWriterShareLedger() - server.ledger = ledger - submission = SimpleNamespace( - header_hex="aa" * 80, - coinbase_tx_hex="00", - block_hex="00", - block_hash_hex="be" * 32, - share_pass=False, - block_pass=True, - ) - submit_calls = 0 - - def fail_intent(_intent: dict[str, object]) -> bool: - raise RuntimeError("outbox unavailable") - - def unsafe_submit(_candidate: PrismBlockCandidate) -> bool: - nonlocal submit_calls - submit_calls += 1 - return True - - ledger.persist_block_candidate_intent = fail_intent # type: ignore[method-assign] - server.submit_block_candidate = unsafe_submit # type: ignore[method-assign] - with patch( - "lab.prism.prism_coordinator.direct_stratum.assemble_submission", - return_value=submission, - ): - with self.assertRaisesRegex(RuntimeError, "outbox unavailable"): - server.handle_submit( - state, - ["miner-a", "job-1", "00" * 8, "00000001", "00000002"], - ) - - self.assertEqual(submit_calls, 0) - self.assertIsNone(getattr(server, "_retry_block_candidate", None)) - self.assertEqual(ledger.pending_block_candidates(), []) - - def test_block_worthy_below_target_rejects_low_difficulty_when_block_fails(self) -> None: - # If the block does not land, the below-share-target hash earns nothing - # and the miner is rejected as low-difficulty -- never acked accepted - # with no ledger row. - server, state, ledger = submit_coordinator() - submission = SimpleNamespace( - header_hex="aa" * 80, - block_hash_hex="bb" * 32, - share_pass=False, - block_pass=True, - ) - def reject_candidate(_candidate: PrismBlockCandidate) -> bool: - server._abandon_block_candidate( - PRISM_REJECTION_SUBMITBLOCK_REJECTED, - "rejected", - block_hash=_candidate.submission.block_hash_hex, - worker="miner-a", - ) - return False - - server.submit_block_candidate = reject_candidate # type: ignore[method-assign] - with patch( - "lab.prism.prism_coordinator.direct_stratum.assemble_submission", - return_value=submission, - ): - with self.assertRaises(StratumError) as raised: - server.handle_submit( - state, - ["miner-a", "job-1", "00" * 8, "00000001", "00000002"], - ) - - self.assertEqual(raised.exception.reason, PRISM_REJECTION_LOW_DIFFICULTY) - self.assertEqual(len(ledger.pending), 0) - self.assertEqual(server.block_candidate_queue.qsize(), 0) - # The reject is counted (globally and for the worker), not just the - # block-abandonment reason -- this synchronous path used to skip it. - self.assertEqual(server.rejection_counts_by_reason[PRISM_REJECTION_LOW_DIFFICULTY], 1) - self.assertEqual(server.low_difficulty_share_count, 1) - - def test_below_target_transient_outcome_closes_without_definitive_reject(self) -> None: - server, state, _recording = submit_coordinator() - ledger = SingleWriterShareLedger() - server.ledger = ledger - submission = SimpleNamespace( - header_hex="aa" * 80, - coinbase_tx_hex="00", - block_hex="00", - block_hash_hex="bd" * 32, - share_pass=False, - block_pass=True, - ) - server.submit_block_candidate = lambda _candidate: False # type: ignore[method-assign] - - submit_params = ["miner-a", "job-1", "00" * 8, "00000001", "00000002"] - # Each retry rebuilds its candidate intent with a fresh acknowledgment - # stamp. Force every call onto a new millisecond so the durable-outbox - # idempotency is exercised across acknowledgment-stamp drift instead of - # depending on both attempts landing within the same millisecond. - clock_ms = iter(range(1_700_000_000_000, 1_700_000_070_000, 7)) - with patch( - "lab.prism.prism_coordinator.direct_stratum.assemble_submission", - return_value=submission, - ), patch( - "lab.prism.prism_coordinator.now_ms", - side_effect=clock_ms.__next__, - ): - for _attempt in range(2): - with self.assertRaisesRegex(RuntimeError, "pending durable retry"): - server.handle_submit(state, submit_params) - self.assertEqual(server.recent_share_keys, set()) - - self.assertEqual(server.rejection_counts_by_reason[PRISM_REJECTION_LOW_DIFFICULTY], 0) - self.assertEqual(server.duplicate_share_count, 0) - self.assertEqual(len(ledger), 0) - self.assertEqual(len(ledger.pending_block_candidates()), 1) - self.assertIsNotNone(server._retry_block_candidate) - assert server._retry_block_candidate is not None - self.assertEqual( - server._retry_block_candidate.submission.block_hash_hex, - submission.block_hash_hex, - ) - - def test_low_difficulty_submission_without_block_solve_is_rejected(self) -> None: - server, state, ledger = submit_coordinator() - submission = SimpleNamespace( - header_hex="aa" * 80, - block_hash_hex="bb" * 32, - share_pass=False, - block_pass=False, - ) - with patch( - "lab.prism.prism_coordinator.direct_stratum.assemble_submission", - return_value=submission, - ): - with self.assertRaises(StratumError) as raised: - server.handle_submit( - state, - ["miner-a", "job-1", "00" * 8, "00000001", "00000002"], - ) - - self.assertEqual(raised.exception.code, 23) - self.assertEqual(raised.exception.reason, PRISM_REJECTION_LOW_DIFFICULTY) - self.assertEqual(server.rejection_counts_by_reason[PRISM_REJECTION_LOW_DIFFICULTY], 1) - self.assertEqual(len(ledger.pending), 0) - - def test_collection_only_below_target_block_solve_submits_solver_pays_all(self) -> None: - # A collection job's signed bootstrap manifest already commits the - # whole coinbase to the submitting worker, so a solved block on a - # collection job is submitted (synchronously here, since the share - # missed its target) instead of being withheld -- the first block on a - # fresh ledger must never be silently ledgered away. - server, state, ledger = submit_coordinator() - server.jobs["job-1"].collection_only = True - submission = SimpleNamespace( - header_hex="aa" * 80, - block_hash_hex="bb" * 32, - share_pass=False, - block_pass=True, - ) - submitted: list[object] = [] - - def fake_submit(candidate: object) -> bool: - submitted.append(candidate) - server.append_accepted_share( - candidate.client, candidate.context, candidate.submission, candidate.pending_share - ) - return True - - server.submit_block_candidate = fake_submit # type: ignore[method-assign] - with patch( - "lab.prism.prism_coordinator.direct_stratum.assemble_submission", - return_value=submission, - ): - should_close = server.handle_submit( - state, - ["miner-a", "job-1", "00" * 8, "00000001", "00000002"], - ) - - self.assertFalse(should_close) - self.assertEqual(server.rejection_counts_by_reason[PRISM_REJECTION_LOW_DIFFICULTY], 0) - self.assertEqual(len(submitted), 1) - self.assertTrue(submitted[0].credit_share_on_accept) - self.assertTrue(submitted[0].context.collection_only) - self.assertEqual(len(ledger.pending), 1) - self.assertEqual(server.collection_block_submission_count, 1) - - def test_collection_job_block_solve_is_credited_and_enqueued_not_withheld(self) -> None: - # A solved block that also met its share target on a collection job is - # credited immediately and queued for the submitter thread, exactly - # like a ready-window candidate. - server, state, ledger = submit_coordinator() - server.jobs["job-1"].collection_only = True - submission = SimpleNamespace( - header_hex="aa" * 80, - block_hash_hex="cc" * 32, - share_pass=True, - block_pass=True, - ) - with patch( - "lab.prism.prism_coordinator.direct_stratum.assemble_submission", - return_value=submission, - ): - should_close = server.handle_submit( - state, - ["miner-a", "job-1", "00" * 8, "00000001", "00000002"], - ) - - self.assertFalse(should_close) - self.assertEqual(len(ledger.pending), 1) - self.assertEqual(server.block_candidate_queue.qsize(), 1) - queued = server.block_candidate_queue.get_nowait() - self.assertTrue(queued.context.collection_only) - self.assertFalse(queued.credit_share_on_accept) - self.assertEqual(server.collection_block_submission_count, 1) - - def test_block_candidate_intent_round_trips_compact_payout_state(self) -> None: - server, state, _ledger = submit_coordinator() - server.jobs["job-1"].collection_only = True - server.jobs["job-1"].prospective_prior_balances = ( - ("miner-a", "miner-a", "11" * 32, 25), - ) - pending = PendingShare( - share_id="miner-a:" + "dd" * 32, - miner_id="miner-a", - order_key="miner-a", - p2mr_program_hex="11" * 32, - share_difficulty=1, - network_difficulty=1, - template_height=9, - job_id="job-1", - job_issued_at_ms=1, - accepted_at_ms=2, - ntime=1, - ) - submission = SimpleNamespace( - coinbase_tx_hex="00", - block_hash_hex="dd" * 32, - block_hex="00", - share_pass=True, - block_pass=True, - ) - candidate = block_candidate(server, state, submission, pending_share=pending) - - intent = server.block_candidate_intent(candidate) - self.assertIs(intent["collection_only"], True) - replayed = server.block_candidate_from_intent(intent) - self.assertTrue(replayed.context.collection_only) - self.assertEqual( - replayed.context.prospective_prior_balances, - (("miner-a", "miner-a", "11" * 32, 25),), - ) - - # Intents persisted before the flag existed replay as ready-window - # candidates, which is all the outbox could ever have contained then. - intent.pop("collection_only") - intent.pop("prospective_prior_balances") - replayed = server.block_candidate_from_intent(intent) - self.assertFalse(replayed.context.collection_only) - self.assertIsNone(replayed.context.prospective_prior_balances) - - def test_below_target_accepted_tail_serializes_moved_tip_replay(self) -> None: - old_tip = "00" * 32 - new_tip = "11" * 32 - block_hash = "b7" * 32 - server, state, _recording = submit_coordinator(tip=old_tip) - server.max_blocks = 10 - server.stop_after_block = False - ledger = SingleWriterShareLedger() - server.ledger = ledger - server.rpc = SubmitRpc( - tip=old_tip, - block_hash=block_hash, - ledger=ledger, - ) - server.build_audit_bundle = ( # type: ignore[method-assign] - lambda **_kwargs: verified_block_bundle() - ) - server.verify_bundle = ( # type: ignore[method-assign] - lambda *_args, **_kwargs: verified_audit_report() - ) - server.ledger_writer_public_key_hex = "aa" * 32 - server.refresh_jobs_after_pending_accepted_block = ( # type: ignore[method-assign] - lambda *_args, **_kwargs: 0 - ) - submission = SimpleNamespace( - header_hex="b7" * 80, - coinbase_tx_hex="c0ffee", - block_hex="00", - block_hash_hex=block_hash, - share_pass=False, - block_pass=True, - ) - - durable_confirmation = threading.Event() - accepted_tail_paused = threading.Event() - release_accepted_tail = threading.Event() - replay_guard_blocked = threading.Event() - confirmation_calls: list[str] = [] - submitted: list[str] = [] - abandoned: list[str] = [] - synchronous_results: list[bool] = [] - replay_results: list[bool] = [] - errors: list[BaseException] = [] - replay_thread: threading.Thread | None = None - original_confirm = ledger.confirm_accepted_block - original_stats = server.accepted_share_stats - original_submitted = ledger.mark_block_candidate_submitted - original_abandoned = ledger.mark_block_candidate_abandoned - - class ObservedDispositionLock: - def __init__(self) -> None: - self.lock = threading.Lock() - - def acquire( - self, - blocking: bool = True, - timeout: float = -1, - ) -> bool: - if threading.current_thread() is replay_thread: - if self.lock.acquire(blocking=False): - return True - # A failed non-blocking acquisition proves the accepted - # attempt still owns this exact same-hash guard. - replay_guard_blocked.set() - if not blocking: - return self.lock.acquire(blocking=False) - if timeout < 0: - return self.lock.acquire() - return self.lock.acquire(timeout=timeout) - - def release(self) -> None: - self.lock.release() - - server._ensure_block_candidate_disposition_state() - with server._block_candidate_disposition_registry_lock: - server._block_candidate_disposition_flights[block_hash] = ( # type: ignore[assignment] - SimpleNamespace(lock=ObservedDispositionLock(), users=0) - ) - - def confirm_accepted_block( - *, - block_hash: str, - active_tip_height: int, - ) -> dict[str, int | str]: - result = original_confirm( - block_hash=block_hash, - active_tip_height=active_tip_height, - ) - confirmation_calls.append(block_hash) - durable_confirmation.set() - return result - - def pause_in_accepted_tail() -> tuple[int, int]: - accepted_tail_paused.set() - if not release_accepted_tail.wait(10): - raise AssertionError("timed out waiting to release accepted success tail") - return original_stats() - - def mark_submitted(*, block_hash: str) -> bool: - submitted.append(block_hash) - return original_submitted(block_hash=block_hash) - - def mark_abandoned(*, block_hash: str, error: str) -> bool: - abandoned.append(block_hash) - return original_abandoned(block_hash=block_hash, error=error) - - ledger.confirm_accepted_block = confirm_accepted_block # type: ignore[method-assign] - ledger.mark_block_candidate_submitted = mark_submitted # type: ignore[method-assign] - ledger.mark_block_candidate_abandoned = mark_abandoned # type: ignore[method-assign] - server.accepted_share_stats = pause_in_accepted_tail # type: ignore[method-assign] - - def submit_synchronously() -> None: - try: - synchronous_results.append( - server.handle_submit( - state, - [ - "miner-a", - "job-1", - "00" * 8, - "00000001", - "00000002", - ], - ) - ) - except BaseException as exc: # noqa: BLE001 - asserted below - errors.append(exc) - - def replay_pending_candidate() -> None: - try: - replay_results.append(server.submit_next_block_candidate()) - except BaseException as exc: # noqa: BLE001 - asserted below - errors.append(exc) - - with tempfile.TemporaryDirectory() as tempdir, patch( - "lab.prism.prism_coordinator.direct_stratum.assemble_submission", - return_value=submission, - ): - server.audit_dir = Path(tempdir) - server.evidence_path = Path(tempdir) / "evidence.json" - synchronous_thread = threading.Thread(target=submit_synchronously) - synchronous_thread.start() - try: - self.assertTrue( - accepted_tail_paused.wait(5), - msg=f"synchronous submit exited early: {errors!r}", - ) - self.assertTrue(durable_confirmation.is_set()) - self.assertEqual(len(ledger), 1) - self.assertEqual(len(ledger.pending_block_candidates()), 1) - with server.lock: - self.assertNotIn( - block_hash, - server._accounted_accepted_block_hashes, - ) - - self.assertEqual(server.replay_pending_block_candidates(), 1) - server.rpc = TipRpc(new_tip) - replay_thread = threading.Thread(target=replay_pending_candidate) - replay_thread.start() - self.assertTrue(replay_guard_blocked.wait(5)) - self.assertTrue(replay_thread.is_alive()) - self.assertEqual(submitted, []) - self.assertEqual(abandoned, []) - finally: - release_accepted_tail.set() - synchronous_thread.join(10) - if replay_thread is not None: - replay_thread.join(10) - - self.assertFalse(synchronous_thread.is_alive()) - self.assertIsNotNone(replay_thread) - assert replay_thread is not None - self.assertFalse(replay_thread.is_alive()) - if errors: - raise errors[0] - self.assertEqual(synchronous_results, [False]) - self.assertEqual(replay_results, [True]) - self.assertEqual(confirmation_calls, [block_hash]) - self.assertGreaterEqual(len(submitted), 1) - self.assertEqual(abandoned, []) - self.assertEqual(len(ledger), 1) - self.assertEqual(ledger.pending_block_candidates(), []) - self.assertEqual(server.accepted_block_count, 1) - self.assertIn(block_hash, server._accounted_accepted_block_hashes) - self.assertNotIn( - PRISM_REJECTION_STALE_JOB, - server.block_candidate_abandoned_counts, - ) - self.assertNotIn(block_hash, server._accepted_block_payout_previews) - self.assertNotIn( - block_hash, - server._invalidated_accepted_block_payout_previews, - ) - self.assertEqual(server._block_candidate_disposition_flights, {}) - - def _drive_interleaved_distinct_hash_landings( - self, - *, - lease_error: BaseException | None = None, - lease_error_only_inside_publication_guard: bool = False, - ) -> SimpleNamespace: - """Interleave two distinct-hash landings across the production tails. - - Block A lands on the synchronous miner-connection tail and pauses in - its confirm->publish gap (the accepted-share-stats aggregate) with the - payout balance lock and the publication order guard released. Block B - lands through the production submitter->accounting-actor handoff - inside that gap and publishes first, so A reaches publication already - superseded on the publication ordinal. ``lease_error`` is raised from - the coordinator's fresh-lease fence when provided, modelling a writer - whose lease authority cannot be proven inside the gap. With - ``lease_error_only_inside_publication_guard`` the restore-authority - proof succeeds anywhere outside the store's publication order guard - and fails inside it, modelling a writer deposed while queued on the - publication locks. - """ - old_tip = "00" * 32 - hash_a = "a1" * 32 - hash_b = "b2" * 32 - server, state, _recording = submit_coordinator(tip=old_tip) - server.max_blocks = 10 - server.stop_after_block = False - ledger = SingleWriterShareLedger() - server.ledger = ledger - server.build_audit_bundle = ( # type: ignore[method-assign] - lambda **_kwargs: verified_block_bundle() - ) - server.verify_bundle = ( # type: ignore[method-assign] - lambda *_args, **kwargs: verified_audit_report( - block_height=int(kwargs["expected_block_height"]) - ) - ) - server.ledger_writer_public_key_hex = "aa" * 32 - server.refresh_jobs_after_pending_accepted_block = ( # type: ignore[method-assign] - lambda *_args, **_kwargs: 0 - ) - publication_guard_holders: set[int] = set() - if lease_error is not None: - - def failing_lease_fence(component: str) -> None: - if lease_error_only_inside_publication_guard: - if component != "superseded_audit_envelope_restore": - return - if threading.get_ident() not in publication_guard_holders: - return - raise lease_error - - server._require_fresh_ledger_lease_for_external_side_effect = ( # type: ignore[method-assign] - failing_lease_fence - ) - submission_a = SimpleNamespace( - header_hex="a1" * 80, - coinbase_tx_hex="c0ffee", - block_hex="00", - block_hash_hex=hash_a, - share_pass=False, - block_pass=True, - ) - context_b = SimpleNamespace( - job=SimpleNamespace( - job_id="job-2", - share_target=target_from_compact("207fffff"), - share_difficulty=Decimal("1"), - transaction_hexes=(), - ), - template={ - "previousblockhash": hash_a, - "height": 11, - "coinbasevalue": 50_00000000, - }, - found_block={"network_difficulty": 1}, - issued_at_ms=12345, - collection_only=False, - worker=state.worker, - shares_json=[], - prior_balances=[], - ) - server.jobs["job-2"] = context_b - submission_b = SimpleNamespace( - header_hex="b2" * 80, - coinbase_tx_hex="c0ffee", - block_hex="01", - block_hash_hex=hash_b, - share_pass=True, - block_pass=True, - ) - - class TwoBlockRpc: - def __init__(self) -> None: - self.tip = old_tip - self.height = 9 - self.hashes = {9: old_tip} - self.submitted: list[str] = [] - - def call(self, method: str, params: object = None) -> object: - if method == "getbestblockhash": - return self.tip - if method == "getblockcount": - return self.height - if method == "submitblock": - block_hex = str((params or [""])[0]) # type: ignore[index] - block_hash = hash_a if block_hex == "00" else hash_b - self.height += 1 - self.tip = block_hash - self.hashes[self.height] = block_hash - self.submitted.append(block_hash) - return None - if method == "getblockhash": - return self.hashes[int((params or [0])[0])] # type: ignore[index] - raise RuntimeError(method) - - server.rpc = TwoBlockRpc() - - sync_gap_open = threading.Event() - release_sync_gap = threading.Event() - confirmed_hashes: list[str] = [] - synchronous_results: list[bool] = [] - sync_errors: list[BaseException] = [] - actor_errors: list[BaseException] = [] - synchronous_thread: threading.Thread | None = None - original_confirm = ledger.confirm_accepted_block - original_stats = server.accepted_share_stats - - def confirm_accepted_block( - *, - block_hash: str, - active_tip_height: int, - ) -> dict[str, int | str]: - result = original_confirm( - block_hash=block_hash, - active_tip_height=active_tip_height, - ) - confirmed_hashes.append(block_hash) - return result - - def pause_synchronous_gap() -> tuple[int, int]: - # Only A's synchronous tail pauses; B's accounting-actor landing - # runs the same aggregate inside the gap unimpeded. - if threading.current_thread() is synchronous_thread: - sync_gap_open.set() - if not release_sync_gap.wait(10): - raise AssertionError( - "timed out waiting to release the synchronous gap" - ) - return original_stats() - - ledger.confirm_accepted_block = confirm_accepted_block # type: ignore[method-assign] - server.accepted_share_stats = pause_synchronous_gap # type: ignore[method-assign] - - def submit_synchronously() -> None: - try: - synchronous_results.append( - server.handle_submit( - state, - [ - "miner-a", - "job-1", - "00" * 8, - "00000001", - "00000002", - ], - ) - ) - except BaseException as exc: # noqa: BLE001 - asserted below - sync_errors.append(exc) - - with tempfile.TemporaryDirectory() as tempdir, patch( - "lab.prism.prism_coordinator.direct_stratum.assemble_submission", - return_value=submission_a, - ): - server.audit_dir = Path(tempdir) - server.evidence_path = Path(tempdir) / "evidence.json" - audit_store = server._ensure_audit_artifact_store() - if lease_error_only_inside_publication_guard: - real_publication_order_guard = audit_store.publication_order_guard - - @contextlib.contextmanager - def tracking_publication_order_guard(): - with real_publication_order_guard(): - publication_guard_holders.add(threading.get_ident()) - try: - yield - finally: - publication_guard_holders.discard( - threading.get_ident() - ) - - audit_store.publication_order_guard = ( # type: ignore[method-assign] - tracking_publication_order_guard - ) - envelope_a = audit_store.live_envelope_path( - block_height=10, - block_hash=hash_a, - ) - envelope_b = audit_store.live_envelope_path( - block_height=11, - block_hash=hash_b, - ) - synchronous_thread = threading.Thread(target=submit_synchronously) - synchronous_thread.start() - actor_thread: threading.Thread | None = None - try: - self.assertTrue( - sync_gap_open.wait(5), - msg=f"synchronous submit exited early: {sync_errors!r}", - ) - # A is durably confirmed with ordinal 1 and both the payout - # balance lock and the publication order guard are released, - # but its live envelope is not yet written. - self.assertEqual(confirmed_hashes, [hash_a]) - self.assertEqual(ledger.audit_publication_sequence_floor(), 1) - self.assertFalse(envelope_a.exists()) - - # B rides the production handoff: the submitter claims the - # disposition, offers the block to the node, and transfers the - # lease to the accounting actor, which lands and publishes. - server.enqueue_block_candidate( - block_candidate(server, state, submission_b, job_id="job-2") - ) - handoff: list[object] = [] - server._enqueue_block_accounting_task = ( # type: ignore[method-assign] - lambda task: (handoff.append(task), True)[1] - ) - self.assertTrue( - server.submit_next_block_candidate(defer_accounting=True) - ) - self.assertEqual(len(handoff), 1) - - def run_accounting_actor() -> None: - try: - server._run_block_accounting_task(handoff[0]) - except BaseException as exc: # noqa: BLE001 - asserted below - actor_errors.append(exc) - - actor_thread = threading.Thread(target=run_accounting_actor) - actor_thread.start() - actor_thread.join(10) - self.assertFalse(actor_thread.is_alive()) - self.assertEqual(actor_errors, []) - self.assertEqual(confirmed_hashes, [hash_a, hash_b]) - self.assertEqual(ledger.audit_publication_sequence_floor(), 2) - self.assertTrue(envelope_b.exists()) - # The interleave is real: B published while A had not. - self.assertFalse(envelope_a.exists()) - finally: - release_sync_gap.set() - synchronous_thread.join(10) - if actor_thread is not None: - actor_thread.join(10) - - self.assertFalse(synchronous_thread.is_alive()) - if sync_errors: - raise sync_errors[0] - # Whatever happened to A's envelope, the publication itself must - # have completed on both tails. - self.assertEqual(synchronous_results, [False]) - self.assertEqual(server.accepted_block_count, 2) - - # The current evidence reference still names the highest - # published sequence; a superseded publication never moves it. - self.assertTrue(envelope_b.exists()) - latest = audit_store.latest_evidence() - assert latest is not None - self.assertEqual(latest["block_hash"], hash_b) - self.assertEqual( - latest["audit_publication_identity"]["sequence"], - 2, - ) - return SimpleNamespace( - hash_a=hash_a, - envelope_a_exists=envelope_a.exists(), - landed_a=( - json.loads(envelope_a.read_text(encoding="utf-8")) - if envelope_a.exists() - else None - ), - ) - - def test_interleaved_distinct_hash_landings_write_both_live_envelopes(self) -> None: - """Both live envelopes survive an interleaved distinct-hash landing. - - A's own height+hash live envelope is its block's only published - evidence pointer and must be written even though A publishes already - superseded; skipping it was the silent permanent loss of the original - defect. This pins the production opt-in wiring end to end: with - ``restore_superseded_envelope=False`` at the publish call site, this - test fails. - """ - result = self._drive_interleaved_distinct_hash_landings() - self.assertTrue(result.envelope_a_exists) - assert result.landed_a is not None - self.assertEqual(result.landed_a["block_hash"], result.hash_a) - self.assertEqual(result.landed_a["block_height"], 10) - - def test_withheld_lease_authority_skips_restore_without_failing_publication( - self, - ) -> None: - """A failed lease proof degrades the restore; it never fails the landing. - - The fresh-lease fence gates only the superseded-envelope restore - authority. When it raises, the publication must still complete - exactly as before the restore existed -- withholding degrades to the - historical behaviour of skipping the superseded envelope write, - because failing the publication on a lease hiccup would be a live - regression on the found-block path. - """ - from lab.prism.share_ledger import WriterLeaseRenewalDeferred - - result = self._drive_interleaved_distinct_hash_landings( - lease_error=WriterLeaseRenewalDeferred( - "writer lease renewal is deferred behind an in-flight write" - ), - ) - self.assertFalse(result.envelope_a_exists) - - def test_restore_authority_is_proven_after_the_publication_locks( - self, - ) -> None: - """Deposal during the publication lock wait withholds the restore. - - The fresh-lease proof once ran before the payout balance lock and - the publication order guard were acquired -- a check-then-act across - an unbounded wait, so a writer deposed while queued on those locks - still restored. This fence models that deposal: the proof succeeds - anywhere outside the publication order guard and fails inside it, - so the restore is withheld only if authority is evaluated at the - moment of action, with the locks held. - """ - from lab.prism.share_ledger import WriterLeaseRenewalDeferred - - result = self._drive_interleaved_distinct_hash_landings( - lease_error=WriterLeaseRenewalDeferred( - "writer lease was deposed during the publication lock wait" - ), - lease_error_only_inside_publication_guard=True, - ) - self.assertFalse(result.envelope_a_exists) - - -class PrismCoordinatorAcceptedBlockGapTests(unittest.TestCase): - """Blockwait-first acceptance must survive every abandon-capable path. - - Regression coverage for the accepted-block blind spot left open by #89: - the accepted registry was only written in the direct-accept submit tail, - so a candidate whose submitblock ack was lost -- acceptance arriving via - a blockwait tip observation instead -- could be terminally abandoned as - stale-job by a retry that read one racing chain snapshot. The abandon - withdrew the landed payout preview, fenced payout publication, and left - template refreshes coordination-blocked until the watchdog restart. - """ - - def _accepted_tail_scaffolding(self, server: PrismCoordinator, tempdir: str) -> tuple[list[str], list[str]]: - server._ensure_job_cache_state() - server.reorg_reconciler_enabled = False - server.block_candidate_retry_initial_seconds = 0.0 - server.audit_dir = Path(tempdir) - server.evidence_path = Path(tempdir) / "evidence.json" - server.ledger_writer_public_key_hex = "aa" * 32 - server.build_audit_bundle = ( # type: ignore[method-assign] - lambda **_kwargs: verified_block_bundle() - ) - server.verify_bundle = ( # type: ignore[method-assign] - lambda *_args, **kwargs: verified_audit_report( - block_height=int(kwargs["expected_block_height"]) - ) - ) - submitted: list[str] = [] - abandoned: list[str] = [] - server.ledger.mark_block_candidate_submitted = ( # type: ignore[attr-defined] - lambda **kwargs: submitted.append(str(kwargs["block_hash"])) or True - ) - server.ledger.mark_block_candidate_abandoned = ( # type: ignore[attr-defined] - lambda **kwargs: abandoned.append(str(kwargs["block_hash"])) or True - ) - return submitted, abandoned - - def _age_retained_acceptance_past_window( - self, server: PrismCoordinator - ) -> None: - """Backdate any stashed definitive ack past the acceptance window. - - These interleavings pin the terminal machinery that runs once - first-party acceptance evidence has gone stale; a fresh stash - correctly vetoes the terminal decision and defers instead (see - test_definitive_ack_with_unprovable_chain_view_defers_not_abandons). - """ - real_stash = server._stash_retained_block_candidate_node_submission - - def stash_then_age(block_hash: str, node_submission: object) -> None: - real_stash(block_hash, node_submission) - with server.lock: - stamped = getattr( - server, - "_block_candidate_retained_submission_monotonic", - None, - ) - if stamped and block_hash.lower() in stamped: - stamped[block_hash.lower()] -= 400.0 - - server._stash_retained_block_candidate_node_submission = stash_then_age # type: ignore[method-assign] - - def _retained_candidate(self, server: PrismCoordinator) -> PrismBlockCandidate: - with server.lock: - candidate = server._retry_block_candidate - server._retry_block_candidate = None - self.assertIsNotNone(candidate) - return candidate - - def _chained_context( - self, - server: PrismCoordinator, - *, - job_id: str, - parent_hash: str, - height: int, - ) -> None: - base = server.jobs["job-1"] - server.jobs[job_id] = SimpleNamespace( - job=SimpleNamespace( - job_id=job_id, - share_target=base.job.share_target, - share_difficulty=base.job.share_difficulty, - transaction_hexes=(), - ), - template={ - "previousblockhash": parent_hash, - "height": height, - "coinbasevalue": 50_00000000, - }, - found_block={"network_difficulty": 1}, - issued_at_ms=12345, - collection_only=False, - worker=base.worker, - shares_json=[], - prior_balances=[], - ) - - def test_blockwait_observed_acceptance_survives_stale_chain_probe(self) -> None: - # The 2026-07-25 mainnet interleaving, end to end: lost submitblock - # ack, blockwait reports the pool's own hash as the new tip, and the - # retry disposition reads a racing snapshot in which the tip moved on - # and the header probe cannot prove the candidate active. The - # candidate must defer (never terminally abandon), keep its landed - # payout preview, and finalize as submitted once the view settles -- - # with no "qbit accepted direct" tail having run before the defer. - parent = "00" * 32 - block_hash = "b1" * 32 - racing_tip = "77" * 32 - server, state, ledger = submit_coordinator(tip=parent) - server.max_blocks = 2 - server.stop_after_block = False - with tempfile.TemporaryDirectory() as tempdir: - submitted, abandoned = self._accepted_tail_scaffolding(server, tempdir) - rpc = LostAckSubmitRpc( - start_tip=parent, - hash_by_hex={"00": block_hash}, - ) - server.rpc = rpc - candidate = block_candidate( - server, - state, - SimpleNamespace( - coinbase_tx_hex="c0ffee", - block_hash_hex=block_hash, - block_hex="00", - share_pass=True, - block_pass=True, - ), - ) - - # Attempt 1: submitblock lands the block but the ack is lost. - self.assertTrue(server._submit_next_block_candidate_writer(candidate)) - self.assertEqual(rpc.submitblock_calls, 1) - self.assertIn(block_hash, server._accepted_block_payout_previews) - self.assertTrue( - server._accepted_block_payout_previews[block_hash].landed - ) - candidate = self._retained_candidate(server) - - # Blockwait sees the pool's own hash as the new tip. This is the - # only acceptance signal: no direct-submit ack ever arrived and - # the accepted success tail has not run. - self.assertTrue(server.observe_tip_for_refresh(block_hash)) - self.assertIn(block_hash, server._tip_observed_accepted_block_hashes) - self.assertNotIn(block_hash, server._accounted_accepted_block_hashes) - - # Attempt 2: the disposition reads a racing snapshot -- tip moved - # past the candidate and the header probe reports not-found. #89 - # abandoned here; the observation evidence must defer instead. - rpc.racing_tip = racing_tip - self.assertTrue(server._submit_next_block_candidate_writer(candidate)) - self.assertNotIn( - PRISM_REJECTION_STALE_JOB, - getattr(server, "block_candidate_abandoned_counts", {}), - ) - self.assertEqual(abandoned, []) - self.assertEqual(server.block_candidate_accept_pending_defer_count, 1) - # The landed preview is preserved -- not withdrawn, not - # tombstoned -- so payout publication stays unfenced and the - # reconcile/template path stays unblocked. - self.assertIn(block_hash, server._accepted_block_payout_previews) - self.assertTrue( - server._accepted_block_payout_previews[block_hash].landed - ) - self.assertNotIn( - block_hash, - server._invalidated_accepted_block_payout_previews, - ) - self.assertFalse(server._payout_state_publication_blocked) - candidate = self._retained_candidate(server) - - # Attempt 3: the chain view settles with the pool's own block as - # the tip; the resume path runs the full accepted success tail - # and finalizes the durable outbox as submitted. - rpc.racing_tip = None - self.assertTrue(server._submit_next_block_candidate_writer(candidate)) - - self.assertEqual(submitted, [block_hash]) - self.assertEqual(abandoned, []) - self.assertEqual(rpc.submitblock_calls, 3) - self.assertIn(block_hash, server._accounted_accepted_block_hashes) - self.assertEqual(server.accepted_block_count, 1) - self.assertEqual(len(ledger.persisted), 1) - self.assertEqual(len(ledger.confirmed), 1) - self.assertNotIn(block_hash, server._accepted_block_payout_previews) - self.assertNotIn( - block_hash, - server._invalidated_accepted_block_payout_previews, - ) - self.assertFalse(server._payout_state_publication_blocked) - self.assertIsNone( - server._accepted_block_payout_transition_for_parent( - block_hash, - parent_height=10, - ) - ) - self.assertNotIn( - block_hash, server._outstanding_block_candidate_hashes - ) - self.assertNotIn( - block_hash, server._tip_observed_accepted_block_hashes - ) - self.assertIn( - "qbit_prism_block_candidate_accept_pending_defers_total 1", - server.metrics_payload(), - ) - - def test_three_blocks_quick_succession_lost_acks_finalize_all(self) -> None: - # Three pool blocks land back to back (the 20:58 / 21:06:58 / - # 21:07:20 pattern), every submitblock ack is lost, and one retry - # even reads a transient foreign-tip snapshot. All three candidates - # must finalize as submitted with payout publication unfenced. - parent = "00" * 32 - hashes = ["b1" * 32, "b2" * 32, "b3" * 32] - block_hexes = ["0a", "0b", "0c"] - server, state, ledger = submit_coordinator(tip=parent) - server.max_blocks = 5 - server.stop_after_block = False - with tempfile.TemporaryDirectory() as tempdir: - submitted, abandoned = self._accepted_tail_scaffolding(server, tempdir) - rpc = LostAckSubmitRpc( - start_tip=parent, - hash_by_hex=dict(zip(block_hexes, hashes)), - ) - server.rpc = rpc - parents = [parent, hashes[0], hashes[1]] - for index, (block_hash, block_hex) in enumerate( - zip(hashes, block_hexes) - ): - job_id = f"job-chain-{index}" - self._chained_context( - server, - job_id=job_id, - parent_hash=parents[index], - height=10 + index, - ) - candidate = block_candidate( - server, - state, - SimpleNamespace( - coinbase_tx_hex="c0ffee", - block_hash_hex=block_hash, - block_hex=block_hex, - share_pass=True, - block_pass=True, - ), - job_id=job_id, - ) - - # Lost-ack landing. - self.assertTrue( - server._submit_next_block_candidate_writer(candidate) - ) - candidate = self._retained_candidate(server) - # Blockwait-first acceptance. - self.assertTrue(server.observe_tip_for_refresh(block_hash)) - - if index == 1: - # One retry reads a transient foreign-tip snapshot in - # which nothing can be proven active; it must defer. - rpc.racing_tip = "77" * 32 - self.assertTrue( - server._submit_next_block_candidate_writer(candidate) - ) - self.assertEqual(abandoned, []) - rpc.racing_tip = None - candidate = self._retained_candidate(server) - - # Settled view: the resume path finalizes as submitted. - self.assertTrue( - server._submit_next_block_candidate_writer(candidate) - ) - self.assertIn( - block_hash, server._accounted_accepted_block_hashes - ) - - self.assertEqual(submitted, hashes) - self.assertEqual(abandoned, []) - self.assertEqual(server.accepted_block_count, 3) - self.assertEqual( - getattr(server, "block_candidate_abandoned_counts", {}), - {}, - ) - self.assertEqual(len(ledger.persisted), 3) - self.assertEqual(len(ledger.confirmed), 3) - self.assertFalse(server._payout_state_publication_blocked) - self.assertEqual(server._accepted_block_payout_previews, {}) - self.assertEqual( - server._invalidated_accepted_block_payout_previews, {} - ) - self.assertEqual(server._outstanding_block_candidate_hashes, set()) - self.assertEqual(server._tip_observed_accepted_block_hashes, {}) - - def test_observed_window_expiry_keeps_genuinely_stale_abandon_terminal(self) -> None: - # A candidate that was once observed as the tip but has stayed off - # the active chain past the observation window is genuinely stale - # (permanently reorged out): the terminal abandon and its payout - # preview withdrawal must still happen. - parent = "00" * 32 - block_hash = "d4" * 32 - server, state, _ledger = submit_coordinator(tip=parent) - server.observed_tip_accept_window_seconds = 60.0 - with tempfile.TemporaryDirectory() as tempdir: - submitted, abandoned = self._accepted_tail_scaffolding(server, tempdir) - rpc = LostAckSubmitRpc( - start_tip=parent, - hash_by_hex={"00": block_hash}, - ) - server.rpc = rpc - candidate = block_candidate( - server, - state, - SimpleNamespace( - coinbase_tx_hex="c0ffee", - block_hash_hex=block_hash, - block_hex="00", - share_pass=True, - block_pass=True, - ), - ) - self.assertTrue(server._submit_next_block_candidate_writer(candidate)) - candidate = self._retained_candidate(server) - self.assertTrue(server.observe_tip_for_refresh(block_hash)) - - # The chain moved on without the candidate and the observation - # aged out: the block was reorged away for good. - rpc.racing_tip = "77" * 32 - rpc.active.pop(block_hash, None) - with server.lock: - server._tip_observed_accepted_block_hashes[block_hash] = ( - time.monotonic() - 61.0 - ) - - self.assertTrue(server._submit_next_block_candidate_writer(candidate)) - - self.assertEqual(abandoned, [block_hash]) - self.assertEqual(submitted, []) - self.assertEqual( - server.block_candidate_abandoned_counts[PRISM_REJECTION_STALE_JOB], - 1, - ) - self.assertNotIn(block_hash, server._accepted_block_payout_previews) - self.assertNotIn(block_hash, server._accounted_accepted_block_hashes) - - def test_block_candidate_acceptance_pending_probe_semantics(self) -> None: - block_hash = "e5" * 32 - server, _state, _ledger = submit_coordinator() - server._ensure_job_cache_state() - server.observed_tip_accept_window_seconds = 60.0 - server._register_outstanding_block_candidate(block_hash) - - def observed(age_seconds: float | None) -> None: - with server.lock: - server._tip_observed_accepted_block_hashes.pop(block_hash, None) - if age_seconds is not None: - server._tip_observed_accepted_block_hashes[block_hash] = ( - time.monotonic() - age_seconds - ) - - # Own hash is the fresh best tip: accepted, and the probe itself - # registers the observation for later checks. - observed(None) - server.rpc = AcceptanceProbeRpc(tip=block_hash) - self.assertTrue(server._block_candidate_acceptance_pending(block_hash)) - self.assertIn(block_hash, server._tip_observed_accepted_block_hashes) - - # Active header at the expected height: accepted. - observed(None) - server.rpc = AcceptanceProbeRpc( - tip="11" * 32, - header={"height": 10, "confirmations": 2}, - ) - self.assertTrue( - server._block_candidate_acceptance_pending( - block_hash, expected_height=10 - ) - ) - - # Provably active at the wrong height: abandonable even when the - # hash was recently observed. - observed(1.0) - server.rpc = AcceptanceProbeRpc( - tip="11" * 32, - header={"height": 9, "confirmations": 2}, - ) - self.assertFalse( - server._block_candidate_acceptance_pending( - block_hash, expected_height=10 - ) - ) - - # Unprovable probe + fresh observation: accepted (defer). - observed(1.0) - server.rpc = AcceptanceProbeRpc(tip="11" * 32, header=None) - self.assertTrue( - server._block_candidate_acceptance_pending( - block_hash, expected_height=10 - ) - ) - - # Unprovable probe + expired observation: genuinely stale. - observed(61.0) - self.assertFalse( - server._block_candidate_acceptance_pending( - block_hash, expected_height=10 - ) - ) - - # Unprovable probe + no observation: genuinely stale. - observed(None) - self.assertFalse( - server._block_candidate_acceptance_pending( - block_hash, expected_height=10 - ) - ) - - # Probe failure + fresh observation: acceptance evidence wins. - observed(1.0) - server.rpc = AcceptanceProbeRpc(tip="11" * 32, fail=True) - self.assertTrue( - server._block_candidate_acceptance_pending( - block_hash, expected_height=10 - ) - ) - - # Probe failure + no observation: unchanged legacy behavior. - observed(None) - self.assertFalse( - server._block_candidate_acceptance_pending( - block_hash, expected_height=10 - ) - ) - - # A best-tip lookup failure must not suppress the active-header - # probe: the header alone proves acceptance, with no observation. - observed(None) - server.rpc = AcceptanceProbeRpc( - tip="11" * 32, - header={"height": 10, "confirmations": 2}, - fail_tip=True, - ) - self.assertTrue( - server._block_candidate_acceptance_pending( - block_hash, expected_height=10 - ) - ) - - # ... and the header's wrong-height verdict also stands alone, - # overriding even a fresh observation. - observed(1.0) - server.rpc = AcceptanceProbeRpc( - tip="11" * 32, - header={"height": 9, "confirmations": 2}, - fail_tip=True, - ) - self.assertFalse( - server._block_candidate_acceptance_pending( - block_hash, expected_height=10 - ) - ) - - def test_post_persist_stale_view_defers_before_rejecting_prepared_rows(self) -> None: - # Codex P1 / Bugbot: the post-persistence active-hash check could - # reject the prepared payout rows and only then reach the abandon, - # which now defers on acceptance evidence -- leaving rows - # rejected/reversed that a later confirmation can never promote. - # The acceptance check must run BEFORE reject_prepared_block. - parent = "00" * 32 - block_hash = "c7" * 32 - racing_winner = "77" * 32 - server, state, ledger = submit_coordinator(tip=parent) - server.max_blocks = 2 - server.stop_after_block = False - with tempfile.TemporaryDirectory() as tempdir: - submitted, abandoned = self._accepted_tail_scaffolding(server, tempdir) - rpc = LostAckSubmitRpc( - start_tip=parent, - hash_by_hex={"00": block_hash}, - ) - server.rpc = rpc - candidate = block_candidate( - server, - state, - SimpleNamespace( - coinbase_tx_hex="c0ffee", - block_hash_hex=block_hash, - block_hex="00", - share_pass=True, - block_pass=True, - ), - ) - self.assertTrue(server._submit_next_block_candidate_writer(candidate)) - candidate = self._retained_candidate(server) - self.assertTrue(server.observe_tip_for_refresh(block_hash)) - - # Retry resumes on the own-hash tip, but the post-persist - # getblockhash read races to a foreign winner for the height. - rpc.getblockhash_override = racing_winner - self.assertTrue(server._submit_next_block_candidate_writer(candidate)) - - # Deferred without touching the prepared payout rows. - self.assertEqual(ledger.rejected, []) - self.assertEqual(abandoned, []) - self.assertNotIn( - "block-stale", - getattr(server, "block_candidate_abandoned_counts", {}), - ) - candidate = self._retained_candidate(server) - - # The view settles with the pool's block active again: the - # replayed tail confirms the still-prepared rows. - rpc.getblockhash_override = None - self.assertTrue(server._submit_next_block_candidate_writer(candidate)) - self.assertEqual(submitted, [block_hash]) - self.assertEqual(ledger.rejected, []) - self.assertEqual(len(ledger.confirmed), 1) - self.assertIn(block_hash, server._accounted_accepted_block_hashes) - - def test_post_persist_stale_view_still_rejects_without_acceptance_evidence(self) -> None: - # The terminal half of the same site: submitblock succeeds but a - # sibling steals the height right after persistence and the pool's - # block was never observed as the tip. With no FRESH acceptance - # evidence (the retained ack has aged past the acceptance window), - # the terminal decision seals first and only then are the prepared - # rows rejected, exactly once. - parent = "00" * 32 - block_hash = "c8" * 32 - racing_winner = "77" * 32 - server, state, ledger = submit_coordinator(tip=parent) - server.max_blocks = 2 - server.stop_after_block = False - with tempfile.TemporaryDirectory() as tempdir: - submitted, abandoned = self._accepted_tail_scaffolding(server, tempdir) - rpc = LostAckSubmitRpc( - start_tip=parent, - hash_by_hex={"00": block_hash}, - ) - rpc.lose_acks = False - server.rpc = rpc - # The definitive ack retained at the offer would veto the - # terminal decision while fresh; these tests pin the sealed - # terminal machinery, so age it past the acceptance window. - self._age_retained_acceptance_past_window(server) - candidate = block_candidate( - server, - state, - SimpleNamespace( - coinbase_tx_hex="c0ffee", - block_hash_hex=block_hash, - block_hex="00", - share_pass=True, - block_pass=True, - ), - ) - real_persist = ledger.persist_accepted_block - - def persist_then_lose_race(**kwargs: object) -> dict[str, object]: - result = real_persist(**kwargs) - # A sibling wins the height after persistence: the tip and - # the height's active hash both belong to the foreign winner - # and the pool's block never became the chain tip. - rpc.tip = racing_winner - rpc.active.pop(block_hash, None) - rpc.getblockhash_override = racing_winner - return result - - ledger.persist_accepted_block = persist_then_lose_race # type: ignore[method-assign] - - self.assertTrue(server._submit_next_block_candidate_writer(candidate)) - - self.assertEqual(len(ledger.rejected), 1) - self.assertEqual(abandoned, [block_hash]) - self.assertEqual(submitted, []) - self.assertEqual( - server.block_candidate_abandoned_counts["block-stale"], - 1, - ) - self.assertNotIn(block_hash, server._accounted_accepted_block_hashes) - - def test_acceptance_evidence_arriving_during_withdrawal_defers(self) -> None: - # Bugbot: the payout-preview withdrawal inside the abandon can block - # long enough for a blockwait observation to arrive; the terminal - # commitment must consult that late evidence, not only the - # completed-tail registry. - block_hash = "a9" * 32 - server, _state, _ledger = submit_coordinator() - server._ensure_job_cache_state() - server._register_outstanding_block_candidate(block_hash) - server._begin_accepted_block_payout_preview(block_hash, block_height=10) - server._mark_accepted_block_payout_landed(block_hash, block_height=10) - server.rpc = AcceptanceProbeRpc(tip="11" * 32, header=None) - real_clear = server._clear_accepted_block_payout_preview - - def observing_clear( - hash_arg: str, - *, - invalidate_published: bool = False, - ) -> None: - if invalidate_published: - # A blockwait observation lands while the withdrawal blocks. - with server.lock: - server._tip_observed_accepted_block_hashes[block_hash] = ( - time.monotonic() - ) - return real_clear( - hash_arg, - invalidate_published=invalidate_published, - ) - - server._clear_accepted_block_payout_preview = observing_clear # type: ignore[method-assign] - - accepted_race_won = server._abandon_block_candidate( - PRISM_REJECTION_STALE_JOB, - "tip moved before submit: test", - block_hash=block_hash, - worker=None, - preserve_if_accepted=True, - expected_height=10, - ) - - self.assertFalse(accepted_race_won) - outcome = getattr(server, "_block_candidate_outcome", None) - self.assertEqual( - getattr(outcome, "reason", None), - PRISM_REJECTION_BLOCK_ACCEPT_PENDING, - ) - self.assertNotIn( - PRISM_REJECTION_STALE_JOB, - getattr(server, "block_candidate_abandoned_counts", {}), - ) - self.assertEqual(server.block_candidate_accept_pending_defer_count, 1) - # The landed barrier is restored (and its fail-closed tombstone - # popped) so descendant builders wait on the preview instead of - # failing closed while the deferred retry heals it. - self.assertIn(block_hash, server._accepted_block_payout_previews) - self.assertTrue( - server._accepted_block_payout_previews[block_hash].landed - ) - self.assertNotIn( - block_hash, - server._invalidated_accepted_block_payout_previews, - ) - - def test_replay_restores_acceptance_evidence_from_durable_block_state(self) -> None: - # Codex round 5: the observation registry dies with the process. A - # durable prepared/confirmed pool-block row proves a prior process's - # submitblock succeeded, so startup replay must restore acceptance - # evidence before a replay disposition can race a transient fork - # view and terminally abandon the accepted block. - server, state, _recording = submit_coordinator() - ledger = SingleWriterShareLedger() - server.ledger = ledger - accepted_hash = "e7" * 32 - unproven_hash = "e8" * 32 - unreadable_hash = "e9" * 32 - for index, block_hash in enumerate( - (accepted_hash, unproven_hash, unreadable_hash), start=1 - ): - pending = PendingShare( - share_id=f"miner-a:{block_hash}", - miner_id="miner-a", - order_key="miner-a", - p2mr_program_hex="11" * 32, - share_difficulty=1, - network_difficulty=1, - template_height=9, - job_id="job-1", - job_issued_at_ms=1, - accepted_at_ms=index, - ntime=1, - ) - candidate = block_candidate( - server, - state, - SimpleNamespace( - coinbase_tx_hex="00", - block_hash_hex=block_hash, - block_hex="00", - share_pass=True, - block_pass=True, - ), - pending_share=pending, - ) - ledger.append_batch( - [(pending, server.block_candidate_intent(candidate))] - ) - def durable_block_state(*, block_hash: str) -> dict[str, object] | None: - if block_hash == accepted_hash: - return {"chain_state": "prepared", "maturity_state": "immature"} - if block_hash == unreadable_hash: - raise RuntimeError("postgres unavailable") - return None - - ledger.pool_block_state = durable_block_state # type: ignore[attr-defined] - - self.assertEqual(server.replay_pending_block_candidates(), 3) - replayed = [ - server._block_replay_candidate_queue.get_nowait() - for _ in range(3) - ] - for candidate in replayed: - server._restore_replayed_candidate_acceptance_evidence(candidate) - - self.assertIn(accepted_hash, server._tip_observed_accepted_block_hashes) - self.assertIn(accepted_hash, server._outstanding_block_candidate_hashes) - # No durable acceptance proof: replays without synthetic evidence. - self.assertNotIn( - unproven_hash, server._tip_observed_accepted_block_hashes - ) - # An unreadable durable state fails safe: the candidate replays - # protected, bounded by the observation window, instead of exposed - # to a transient fork view racing its first disposition. - self.assertIn( - unreadable_hash, server._tip_observed_accepted_block_hashes - ) - - def test_late_defer_republishes_withdrawn_preview_and_unfences(self) -> None: - # Bugbot round 5: when the withdrawal's own publication loses its - # race, delivery is fenced before the late-acceptance defer runs. - # The defer must republish the withdrawn preview so admission - # reopens immediately instead of staying coordination-blocked - # across the deferral cycles. - block_hash = "ba" * 32 - server, _state, _ledger = submit_coordinator() - server._ensure_job_cache_state() - server._register_outstanding_block_candidate(block_hash) - server._begin_accepted_block_payout_preview(block_hash, block_height=10) - server._mark_accepted_block_payout_landed(block_hash, block_height=10) - server._publish_accepted_block_payout_preview(block_hash, []) - self.assertFalse(server._payout_state_publication_blocked) - server.rpc = AcceptanceProbeRpc(tip="11" * 32, header=None) - - real_publish = server._publish_payout_state_candidate - - def withdrawal_publication_lost(candidate: object) -> object: - if getattr(candidate, "accepted_block_withdrawal", False): - return None - return real_publish(candidate) - - server._publish_payout_state_candidate = withdrawal_publication_lost # type: ignore[method-assign] - real_clear = server._clear_accepted_block_payout_preview - - def observing_clear( - hash_arg: str, - *, - invalidate_published: bool = False, - ) -> None: - if invalidate_published: - with server.lock: - server._tip_observed_accepted_block_hashes[block_hash] = ( - time.monotonic() - ) - return real_clear( - hash_arg, - invalidate_published=invalidate_published, - ) - - server._clear_accepted_block_payout_preview = observing_clear # type: ignore[method-assign] - - accepted_race_won = server._abandon_block_candidate( - PRISM_REJECTION_STALE_JOB, - "tip moved before submit: test", - block_hash=block_hash, - worker=None, - preserve_if_accepted=True, - expected_height=10, - ) - - self.assertFalse(accepted_race_won) - outcome = getattr(server, "_block_candidate_outcome", None) - self.assertEqual( - getattr(outcome, "reason", None), - PRISM_REJECTION_BLOCK_ACCEPT_PENDING, - ) - # The barrier is restored AND publication admission reopened: the - # withdrawn preview was republished despite the withdrawal's own - # publication loss having fenced delivery. - self.assertIn(block_hash, server._accepted_block_payout_previews) - self.assertTrue( - server._accepted_block_payout_previews[block_hash].landed - ) - self.assertIsNotNone( - server._accepted_block_payout_previews[block_hash].preview - ) - self.assertNotIn( - block_hash, - server._invalidated_accepted_block_payout_previews, - ) - self.assertFalse(server._payout_state_publication_blocked) - - def test_late_gate_reprobes_after_withdrawal_heals_buried_block(self) -> None: - # Bugbot round 7: an unknown pre-withdrawal probe verdict must - # re-probe after the (blocking) withdrawal. A buried accepted block - # is not always re-observed as the tip -- blockwait reports only the - # newest of rapid connects -- so the recovered probe alone, with no - # observation evidence at all, must defer the terminal abandon. - block_hash = "bc" * 32 - server, _state, _ledger = submit_coordinator() - server._ensure_job_cache_state() - server._register_outstanding_block_candidate(block_hash) - server._begin_accepted_block_payout_preview(block_hash, block_height=10) - server._mark_accepted_block_payout_landed(block_hash, block_height=10) - rpc = AcceptanceProbeRpc(tip="11" * 32, header=None) - server.rpc = rpc - real_clear = server._clear_accepted_block_payout_preview - - def healing_clear( - hash_arg: str, - *, - invalidate_published: bool = False, - ) -> None: - if invalidate_published: - # The chain view heals while the withdrawal blocks: the - # candidate is provably active again, two blocks deep. - rpc.header = {"height": 10, "confirmations": 2} - return real_clear( - hash_arg, - invalidate_published=invalidate_published, - ) - - server._clear_accepted_block_payout_preview = healing_clear # type: ignore[method-assign] - - accepted_race_won = server._abandon_block_candidate( - PRISM_REJECTION_STALE_JOB, - "tip moved before submit: test", - block_hash=block_hash, - worker=None, - preserve_if_accepted=True, - expected_height=10, - ) - - self.assertFalse(accepted_race_won) - outcome = getattr(server, "_block_candidate_outcome", None) - self.assertEqual( - getattr(outcome, "reason", None), - PRISM_REJECTION_BLOCK_ACCEPT_PENDING, - ) - self.assertNotIn( - PRISM_REJECTION_STALE_JOB, - getattr(server, "block_candidate_abandoned_counts", {}), - ) - self.assertIn(block_hash, server._accepted_block_payout_previews) - self.assertTrue( - server._accepted_block_payout_previews[block_hash].landed - ) - self.assertNotIn( - block_hash, - server._invalidated_accepted_block_payout_previews, - ) - - def test_wrong_height_probe_verdict_overrules_late_observation(self) -> None: - # Bugbot round 6: the probe must win both directions in the late - # check too. An observation arriving during the withdrawal cannot - # revive a candidate the pre-check probe proved active at the wrong - # height -- header heights are immutable, so that verdict stands and - # the abandon commits terminally instead of deferring to window - # expiry. - block_hash = "bb" * 32 - server, _state, _ledger = submit_coordinator() - server._ensure_job_cache_state() - server._register_outstanding_block_candidate(block_hash) - server._begin_accepted_block_payout_preview(block_hash, block_height=10) - server._mark_accepted_block_payout_landed(block_hash, block_height=10) - server.rpc = AcceptanceProbeRpc( - tip="11" * 32, - header={"height": 9, "confirmations": 2}, - ) - real_clear = server._clear_accepted_block_payout_preview - - def observing_clear( - hash_arg: str, - *, - invalidate_published: bool = False, - ) -> None: - if invalidate_published: - with server.lock: - server._tip_observed_accepted_block_hashes[block_hash] = ( - time.monotonic() - ) - return real_clear( - hash_arg, - invalidate_published=invalidate_published, - ) - - server._clear_accepted_block_payout_preview = observing_clear # type: ignore[method-assign] - - accepted_race_won = server._abandon_block_candidate( - "block-stale", - "candidate active at unexpected height 9", - block_hash=block_hash, - worker=None, - expected_height=10, - ) - - self.assertFalse(accepted_race_won) - outcome = getattr(server, "_block_candidate_outcome", None) - self.assertEqual(getattr(outcome, "reason", None), "block-stale") - self.assertNotIn( - "block-stale", - server.block_candidate_abandoned_counts, - ) - self.assertEqual( - getattr(server, "block_candidate_accept_pending_defer_count", 0), - 0, - ) - - def test_terminal_seal_excludes_observations_after_the_commit(self) -> None: - # Codex round 3: acceptance evidence arriving after the terminal - # commit but before the follow-up durable rejection must be excluded - # deterministically. The terminal commit seals the hash (drops it - # from outstanding) in the same critical section, so a later - # observation is a no-op instead of contradicting the sealed - # disposition mid-rejection. - block_hash = "b8" * 32 - server, _state, _ledger = submit_coordinator() - server._ensure_job_cache_state() - server._register_outstanding_block_candidate(block_hash) - server.rpc = AcceptanceProbeRpc(tip="11" * 32, header=None) - - accepted_race_won = server._abandon_block_candidate( - PRISM_REJECTION_STALE_JOB, - "tip moved before submit: test", - block_hash=block_hash, - worker=None, - preserve_if_accepted=True, - expected_height=10, - ) - - self.assertFalse(accepted_race_won) - self.assertNotIn( - PRISM_REJECTION_STALE_JOB, - server.block_candidate_abandoned_counts, - ) - self.assertNotIn(block_hash, server._outstanding_block_candidate_hashes) - - # A blockwait observation lands right after the sealed commit: it - # must not register acceptance evidence for the sealed hash. - self.assertTrue(server.observe_tip_for_refresh(block_hash)) - self.assertNotIn(block_hash, server._tip_observed_accepted_block_hashes) - outcome = getattr(server, "_block_candidate_outcome", None) - self.assertIsNotNone(outcome) - server._record_committed_block_candidate_abandonment(block_hash, outcome) - self.assertEqual( - server.block_candidate_abandoned_counts[PRISM_REJECTION_STALE_JOB], - 1, - ) - - def test_observation_during_prepared_row_rejection_cannot_split_state(self) -> None: - # The exact round-3 interleaving at the post-persist site: a - # blockwait observation fires inside the gap between the sealed - # terminal decision and reject_prepared_block (during the - # getblockcount call). The seal excludes it, so the rows are - # rejected exactly once with no contradictory evidence registered - # and no defer emitted after the seal. - parent = "00" * 32 - block_hash = "c9" * 32 - racing_winner = "77" * 32 - server, state, ledger = submit_coordinator(tip=parent) - server.max_blocks = 2 - server.stop_after_block = False - with tempfile.TemporaryDirectory() as tempdir: - submitted, abandoned = self._accepted_tail_scaffolding(server, tempdir) - rpc = LostAckSubmitRpc( - start_tip=parent, - hash_by_hex={"00": block_hash}, - ) - rpc.lose_acks = False - server.rpc = rpc - # The definitive ack retained at the offer would veto the - # terminal decision while fresh; these tests pin the sealed - # terminal machinery, so age it past the acceptance window. - self._age_retained_acceptance_past_window(server) - candidate = block_candidate( - server, - state, - SimpleNamespace( - coinbase_tx_hex="c0ffee", - block_hash_hex=block_hash, - block_hex="00", - share_pass=True, - block_pass=True, - ), - ) - race_state: dict[str, bool] = {"post_persist": False} - real_persist = ledger.persist_accepted_block - - def persist_then_lose_race(**kwargs: object) -> dict[str, object]: - result = real_persist(**kwargs) - rpc.tip = racing_winner - rpc.active.pop(block_hash, None) - rpc.getblockhash_override = racing_winner - race_state["post_persist"] = True - return result - - ledger.persist_accepted_block = persist_then_lose_race # type: ignore[method-assign] - real_call = rpc.call - - def observing_call( - method: str, - params: list[object] | None = None, - ) -> object: - if method == "getblockcount" and race_state["post_persist"]: - # Blockwait reports the candidate as tip inside the - # seal -> reject gap. - server.observe_tip_for_refresh(block_hash) - return real_call(method, params) - - rpc.call = observing_call # type: ignore[method-assign] - - self.assertTrue(server._submit_next_block_candidate_writer(candidate)) - - self.assertEqual(len(ledger.rejected), 1) - self.assertEqual(abandoned, [block_hash]) - self.assertEqual(submitted, []) - self.assertEqual( - server.block_candidate_abandoned_counts["block-stale"], - 1, - ) - self.assertNotIn( - block_hash, server._tip_observed_accepted_block_hashes - ) - self.assertNotIn( - block_hash, server._outstanding_block_candidate_hashes - ) - self.assertEqual( - getattr(server, "block_candidate_accept_pending_defer_count", 0), - 0, - ) - - def test_retained_candidate_reregisters_for_tip_observations(self) -> None: - # Codex round 8: the terminal seal stops observation matching, but - # when the terminal cleanup itself fails (reject_prepared_block - # raising) the candidate is retained for retry -- and evidence - # arriving during that backoff gap must register again, not vanish. - parent = "00" * 32 - block_hash = "cd" * 32 - racing_winner = "77" * 32 - server, state, ledger = submit_coordinator(tip=parent) - server.max_blocks = 2 - server.stop_after_block = False - with tempfile.TemporaryDirectory() as tempdir: - submitted, abandoned = self._accepted_tail_scaffolding(server, tempdir) - rpc = LostAckSubmitRpc( - start_tip=parent, - hash_by_hex={"00": block_hash}, - ) - rpc.lose_acks = False - server.rpc = rpc - # The definitive ack retained at the offer would veto the - # terminal decision while fresh; these tests pin the sealed - # terminal machinery, so age it past the acceptance window. - self._age_retained_acceptance_past_window(server) - candidate = block_candidate( - server, - state, - SimpleNamespace( - coinbase_tx_hex="c0ffee", - block_hash_hex=block_hash, - block_hex="00", - share_pass=True, - block_pass=True, - ), - ) - real_persist = ledger.persist_accepted_block - - def persist_then_lose_race(**kwargs: object) -> dict[str, object]: - result = real_persist(**kwargs) - rpc.tip = racing_winner - rpc.active.pop(block_hash, None) - rpc.getblockhash_override = racing_winner - return result - - ledger.persist_accepted_block = persist_then_lose_race # type: ignore[method-assign] - reject_attempts: list[dict[str, object]] = [] - - def failing_reject(**kwargs: object) -> dict[str, object]: - reject_attempts.append(kwargs) - raise RuntimeError("psql briefly unavailable") - - ledger.reject_prepared_block = failing_reject # type: ignore[method-assign] - ledger.pool_block_state = ( # type: ignore[attr-defined] - lambda *, block_hash: { - "chain_state": "prepared", - "maturity_state": "immature", - } - ) - - # The sealed terminal pass aborts inside the rejection (site - # attempt, then the writer's terminal-cleanup attempt): the - # candidate is retained and must match observations again. - self.assertTrue(server._submit_next_block_candidate_writer(candidate)) - self.assertEqual(len(reject_attempts), 2) - self.assertIsNotNone(getattr(server, "_retry_block_candidate", None)) - self.assertIn( - block_hash, server._outstanding_block_candidate_hashes - ) - self.assertNotIn( - "block-stale", - server.block_candidate_abandoned_counts, - ) - - # Blockwait reports the pool's own hash during the backoff gap: - # the evidence registers instead of vanishing behind the seal. - self.assertTrue(server.observe_tip_for_refresh(block_hash)) - self.assertIn( - block_hash, server._tip_observed_accepted_block_hashes - ) - self.assertEqual(abandoned, []) - - # The retry can subsequently prove the candidate active and - # complete as submitted. The earlier reversible seals must never - # leave a contradictory abandonment count behind. - ledger.persist_accepted_block = real_persist # type: ignore[method-assign] - rpc.tip = block_hash - rpc.height = 10 - rpc.active[block_hash] = 10 - rpc.getblockhash_override = None - recovered_candidate = self._retained_candidate(server) - self.assertTrue( - server._submit_next_block_candidate_writer(recovered_candidate) - ) - self.assertEqual(submitted, [block_hash]) - self.assertEqual(abandoned, []) - self.assertEqual(server.accepted_block_count, 1) - self.assertNotIn( - "block-stale", - server.block_candidate_abandoned_counts, - ) - - def test_definitive_ack_with_unprovable_chain_view_defers_not_abandons( - self, - ) -> None: - # Release-review finding: a definitive submitblock success followed - # by a chain view that cannot prove the block landed (foreign tip, - # header unknown, height taken by a sibling in that instant's view) - # must defer as accept-pending on the strength of the retained ack - # -- never terminally abandon accounting for a block the node - # accepted. The stash ages on the observation window, so a real - # orphan still terminalizes once the ack is stale (the sealed - # rejection tests above). - parent = "00" * 32 - block_hash = "ce" * 32 - racing_winner = "77" * 32 - server, state, ledger = submit_coordinator(tip=parent) - server.max_blocks = 2 - server.stop_after_block = False - with tempfile.TemporaryDirectory() as tempdir: - submitted, abandoned = self._accepted_tail_scaffolding(server, tempdir) - rpc = LostAckSubmitRpc( - start_tip=parent, - hash_by_hex={"00": block_hash}, - ) - rpc.lose_acks = False - server.rpc = rpc - candidate = block_candidate( - server, - state, - SimpleNamespace( - coinbase_tx_hex="c0ffee", - block_hash_hex=block_hash, - block_hex="00", - share_pass=True, - block_pass=True, - ), - ) - real_persist = ledger.persist_accepted_block - - def persist_then_lose_race(**kwargs: object) -> dict[str, object]: - result = real_persist(**kwargs) - rpc.tip = racing_winner - rpc.active.pop(block_hash, None) - rpc.getblockhash_override = racing_winner - return result - - ledger.persist_accepted_block = persist_then_lose_race # type: ignore[method-assign] - - self.assertTrue(server._submit_next_block_candidate_writer(candidate)) - - # Deferred, not terminal: prepared rows untouched, candidate - # retained for retry, observation matching still open. - self.assertEqual(ledger.rejected, []) - self.assertEqual(abandoned, []) - self.assertEqual(submitted, []) - self.assertGreaterEqual( - getattr(server, "block_candidate_accept_pending_defer_count", 0), - 1, - ) - self.assertIsNotNone(getattr(server, "_retry_block_candidate", None)) - self.assertNotIn( - "block-stale", - server.block_candidate_abandoned_counts, - ) - self.assertIn( - block_hash, server._outstanding_block_candidate_hashes - ) - - def test_retained_acceptance_evidence_ages_on_observation_window( - self, - ) -> None: - # The stash is acceptance evidence only while fresh: past the - # observation window it stands down so a genuinely orphaned block - # (whose probes never prove anything either way) regains - # abandonability instead of deferring forever. - server, _state, _ledger = submit_coordinator() - block_hash = "cf" * 32 - server._stash_retained_block_candidate_node_submission( - block_hash, - SimpleNamespace(attempted=True, error=None, result=None), - ) - self.assertTrue(server._block_candidate_acceptance_retained(block_hash)) - with server.lock: - server._block_candidate_retained_submission_monotonic[ - block_hash - ] -= 400.0 - self.assertFalse(server._block_candidate_acceptance_retained(block_hash)) - # A non-positive window means evidence never expires, mirroring the - # tip-observation semantics. - server.observed_tip_accept_window_seconds = 0.0 - self.assertTrue(server._block_candidate_acceptance_retained(block_hash)) - - def test_pool_closed_gate_requires_probe_proven_acceptance(self) -> None: - # Bugbot: observation evidence alone must not open the pool-closed - # gate -- an off-chain candidate would fall through toward - # submitblock after the pool stopped accepting blocks. Unprovable - # views defer via the abandon path; expired evidence goes terminal. - parent = "00" * 32 - block_hash = "d9" * 32 - server, state, _ledger = submit_coordinator(tip=parent) - server.max_blocks = 1 - server.stop_after_block = False - server.accepted_block_count = 1 - server.observed_tip_accept_window_seconds = 60.0 - with tempfile.TemporaryDirectory() as tempdir: - submitted, abandoned = self._accepted_tail_scaffolding(server, tempdir) - rpc = LostAckSubmitRpc( - start_tip=parent, - hash_by_hex={"00": block_hash}, - ) - server.rpc = rpc - server._register_outstanding_block_candidate(block_hash) - with server.lock: - server._tip_observed_accepted_block_hashes[block_hash] = ( - time.monotonic() - ) - candidate = block_candidate( - server, - state, - SimpleNamespace( - coinbase_tx_hex="c0ffee", - block_hash_hex=block_hash, - block_hex="00", - share_pass=True, - block_pass=True, - ), - ) - - # Fresh observation, unprovable chain view: defer, and above all - # never reach submitblock through the closed pool. - self.assertTrue(server._submit_next_block_candidate_writer(candidate)) - self.assertEqual(rpc.submitblock_calls, 0) - self.assertEqual(abandoned, []) - candidate = self._retained_candidate(server) - - # Evidence expires with the candidate still absent: terminal. - with server.lock: - server._tip_observed_accepted_block_hashes[block_hash] = ( - time.monotonic() - 61.0 - ) - self.assertTrue(server._submit_next_block_candidate_writer(candidate)) - self.assertEqual(rpc.submitblock_calls, 0) - self.assertEqual(abandoned, [block_hash]) - self.assertEqual(submitted, []) - self.assertEqual( - server.block_candidate_abandoned_counts[ - PRISM_REJECTION_POOL_CLOSED - ], - 1, - ) - - def test_pool_closed_gate_yields_to_on_chain_candidate(self) -> None: - # A candidate already on the active chain must complete its payout - # accounting even after the pool stops accepting new blocks; the - # pool-closed gate previously abandoned it terminally. - parent = "00" * 32 - block_hash = "f6" * 32 - server, state, ledger = submit_coordinator(tip=parent) - server.max_blocks = 1 - server.stop_after_block = False - server.accepted_block_count = 1 - with tempfile.TemporaryDirectory() as tempdir: - submitted, abandoned = self._accepted_tail_scaffolding(server, tempdir) - rpc = LostAckSubmitRpc( - start_tip=parent, - hash_by_hex={"00": block_hash}, - ) - # The block is already active with the pool's own hash as tip - # (for example after a watchdog restart replayed the outbox). - rpc.tip = block_hash - rpc.height = 10 - rpc.active[block_hash] = 10 - server.rpc = rpc - server.ensure_reorg_reconciled_for_tip = ( # type: ignore[method-assign] - lambda _tip, *, _coalesce_same_tip: not _coalesce_same_tip - ) - candidate = block_candidate( - server, - state, - SimpleNamespace( - coinbase_tx_hex="c0ffee", - block_hash_hex=block_hash, - block_hex="00", - share_pass=True, - block_pass=True, - ), - ) - - self.assertTrue(server._submit_next_block_candidate_writer(candidate)) - - self.assertEqual(submitted, [block_hash]) - self.assertEqual(abandoned, []) - self.assertNotIn( - PRISM_REJECTION_POOL_CLOSED, - getattr(server, "block_candidate_abandoned_counts", {}), - ) - self.assertIn(block_hash, server._accounted_accepted_block_hashes) - self.assertEqual(server.accepted_block_count, 2) - - -class PrismCoordinatorReliabilityTests(unittest.TestCase): - def _bare_coordinator(self) -> PrismCoordinator: - server = PrismCoordinator.__new__(PrismCoordinator) - server.lock = threading.RLock() - server.stop_event = threading.Event() - server._heartbeats = {} - server._watchdog_pauses = {} - server._heartbeats_lock = threading.Lock() - server.watchdog_timeout_seconds = 120.0 - server.watchdog_interval_seconds = 15.0 - return server - - def test_block_candidate_progress_stamps_only_on_submitter_thread(self) -> None: - server = self._bare_coordinator() - clock = {"now": 1000.0} - with patch( - "lab.prism.prism_coordinator.time.monotonic", - side_effect=lambda: clock["now"], - ): - server._record_heartbeat("block_submitter") - clock["now"] += server.watchdog_timeout_seconds + 1.0 - self.assertEqual( - server._overdue_heartbeats(clock["now"]), ["block_submitter"] - ) - # Without a registered owner the helper never stamps. - server._record_block_candidate_progress() - self.assertEqual( - server._overdue_heartbeats(clock["now"]), ["block_submitter"] - ) - # The owner thread's stamps clear the overdue state. - server._block_submitter_thread_ident = threading.get_ident() - server._record_block_candidate_progress() - self.assertEqual(server._overdue_heartbeats(clock["now"]), []) - - def test_client_thread_disposition_cannot_mask_frozen_submitter(self) -> None: - server = self._bare_coordinator() - clock = {"now": 1000.0} - owner_ready = threading.Event() - release_owner = threading.Event() - - def frozen_owner() -> None: - server._block_submitter_thread_ident = threading.get_ident() - server._record_heartbeat("block_submitter") - owner_ready.set() - release_owner.wait(5) - - with patch( - "lab.prism.prism_coordinator.time.monotonic", - side_effect=lambda: clock["now"], - ): - owner = threading.Thread(target=frozen_owner) - owner.start() - try: - self.assertTrue(owner_ready.wait(5)) - clock["now"] += server.watchdog_timeout_seconds + 1.0 - self.assertEqual( - server._overdue_heartbeats(clock["now"]), ["block_submitter"] - ) - # A client connection thread reaching a disposition boundary - # (synchronous below-target solve) must not refresh the - # frozen dedicated thread's liveness budget. - foreign = threading.Thread( - target=server._record_block_candidate_progress - ) - foreign.start() - foreign.join(5) - self.assertEqual( - server._overdue_heartbeats(clock["now"]), ["block_submitter"] - ) - finally: - release_owner.set() - owner.join(5) - - def test_audit_verifier_subprocess_has_explicit_timeout(self) -> None: - server = self._bare_coordinator() - server.bundle_build_timeout_seconds = 0.25 - with tempfile.TemporaryDirectory() as tempdir: - server.audit_dir = Path(tempdir) / "audit" - server.evidence_path = Path(tempdir) / "evidence.json" - store = server._ensure_audit_artifact_store() - try: - # The verifier budget is the coordinator-configured bundle - # build budget; the store must never silently fall back to a - # fixed default while a shorter budget is configured. - self.assertEqual(store.verifier_timeout_seconds, 0.25) - started = time.monotonic() - with patch( - "lab.prism.audit_artifacts.prism_tool_command", - lambda _name: ["python3", "-c", "import time; time.sleep(5)"], - ): - with self.assertRaisesRegex(RuntimeError, "timed out"): - server.verify_bundle( - Path(tempdir) / "candidate.json", - "00", - "11" * 32, - expected_coinbase_value_sats=1, - ) - self.assertLess(time.monotonic() - started, 5.0) - finally: - store.close() - - def test_sixty_second_attempt_mark_stall_does_not_delay_rpc_or_heartbeat(self) -> None: - server, state, recording = submit_coordinator() - entered_mark = threading.Event() - release_mark = threading.Event() - submitted = threading.Event() - - class StallingLedger(RecordingLedger): - def __init__(self) -> None: - super().__init__() - self.mark_calls = 0 - - def mark_block_candidate_attempted(self, *, block_hash: str) -> bool: - self.mark_calls += 1 - entered_mark.set() - release_mark.wait(60.0) - return True - - class DeadlineRpc(SubmitRpc): - def __init__(self, ledger: RecordingLedger) -> None: - super().__init__( - tip="00" * 32, - block_hash="d2" * 32, - ledger=ledger, - ) - self.timeouts: list[float | None] = [] - - def call( - self, - method: str, - params: list[object] | None = None, - *, - timeout: float | None = None, - ) -> object: - if method == "submitblock": - self.timeouts.append(timeout) - submitted.set() - if len(self.timeouts) > 1: - return "duplicate" - return super().call(method, params) - - ledger = StallingLedger() - server.ledger = ledger - rpc = DeadlineRpc(ledger) - server.rpc = rpc - server.block_submit_rpc_timeout_seconds = 0.4 - server.block_submit_db_timeout_seconds = 0.05 - server.block_candidate_retry_initial_seconds = 0.01 - server.block_candidate_retry_max_seconds = 0.01 - server.watchdog_timeout_seconds = 0.2 - server._heartbeats = {} - server._heartbeat_phases = {} - server._watchdog_pauses = {} - server._heartbeats_lock = threading.Lock() - candidate = block_candidate( - server, - state, - SimpleNamespace( - coinbase_tx_hex="00", - block_hash_hex="d2" * 32, - block_hex="00", - share_pass=True, - block_pass=True, - ), - ) - server.enqueue_block_candidate(candidate) - started = time.monotonic() - submitter = threading.Thread(target=server.block_submit_loop) - with patch("builtins.print"): - submitter.start() - try: - self.assertTrue(submitted.wait(2.0)) - self.assertLess(time.monotonic() - started, 2.0) - self.assertTrue(entered_mark.wait(1.0)) - with server._heartbeats_lock: - first_heartbeat = server._heartbeats["block_submitter"] - heartbeat_deadline = time.monotonic() + 1.0 - while time.monotonic() < heartbeat_deadline: - with server._heartbeats_lock: - latest_heartbeat = server._heartbeats["block_submitter"] - if latest_heartbeat > first_heartbeat: - break - time.sleep(0.01) - self.assertGreater(latest_heartbeat, first_heartbeat) - self.assertEqual( - server._overdue_heartbeats(time.monotonic()), - [], - ) - self.assertEqual(ledger.mark_calls, 1) - self.assertEqual(rpc.timeouts[0], 0.4) - finally: - server.stop_event.set() - submitter.join(2.0) - release_mark.set() - self.assertFalse(submitter.is_alive()) - - def test_accounting_retry_backoff_defers_instead_of_sleeping_under_admission(self) -> None: - server, state, _recording = submit_coordinator() - server.block_candidate_retry_initial_seconds = 30.0 - server.block_candidate_retry_max_seconds = 30.0 - - class FailingMarkLedger(RecordingLedger): - def mark_block_candidate_attempted(self, *, block_hash: str) -> bool: - raise RuntimeError("attempt marker unavailable") - - server.ledger = FailingMarkLedger() - candidate = block_candidate( - server, - state, - SimpleNamespace( - coinbase_tx_hex="00", - block_hash_hex="ab" * 32, - block_hex="ab", - share_pass=True, - block_pass=True, - ), - ) - block_hash = "ab" * 32 - server._block_accounting_thread_ident = threading.get_ident() - server._block_accounting_holds_disposition = True - started = time.monotonic() - with patch("builtins.print"): - handled = server._submit_next_block_candidate_writer( - candidate, - node_submission=SimpleNamespace( - attempted=False, - result=None, - error=None, - ), - disposition_held=True, - ) - elapsed = time.monotonic() - started - self.assertTrue(handled) - # The 30s backoff must be recorded as a deadline, never slept while - # the accounting thread holds admission and the disposition lease. - self.assertLess(elapsed, 10.0) - self.assertIs( - getattr(server, "_block_accounting_deferred_retry_candidate", None), - candidate, - ) - with server.lock: - deadline = server._block_candidate_retry_not_before.get(block_hash) - self.assertIsNotNone(deadline) - self.assertGreater(deadline, time.monotonic()) - - # After the lease releases, the deferred candidate parks in the retry - # slot and the dequeue honors the backoff deadline. - server._block_accounting_holds_disposition = False - with server.lock: - server._block_accounting_deferred_retry_candidate = None - server._merge_block_candidate_retry_locked( - "_retry_block_candidate", - candidate, - ) - with patch("builtins.print"): - self.assertFalse( - server.submit_next_block_candidate(defer_accounting=True) - ) - self.assertIs(getattr(server, "_retry_block_candidate", None), candidate) - - # Once the deadline passes the same candidate dequeues again. - with server.lock: - server._block_candidate_retry_not_before[block_hash] = ( - time.monotonic() - 1.0 - ) - with patch("builtins.print"): - self.assertTrue( - server.submit_next_block_candidate(defer_accounting=True) - ) - - def test_live_retry_reuses_definitive_node_acceptance_without_reoffer(self) -> None: - parent_hash = "00" * 32 - block_hash = "ce" * 32 - moved_tip = "11" * 32 - ledger = SingleWriterShareLedger() - server, state, _recording = submit_coordinator(tip=parent_hash) - server.ledger = ledger - server.stop_after_block = False - server.max_blocks = 10 - server.block_candidate_retry_initial_seconds = 0.0 - server.block_candidate_retry_max_seconds = 0.0 - from lab.prism.share_ledger import PendingShare - - pending = PendingShare( - share_id="miner-a:live-retry-duplicate", - miner_id="miner-a", - order_key="miner-a", - p2mr_program_hex="11" * 32, - share_difficulty=1, - network_difficulty=1, - template_height=10, - job_id="job-1", - job_issued_at_ms=1, - accepted_at_ms=2, - ntime=1_700_000_000, - ) - candidate = block_candidate( - server, - state, - SimpleNamespace( - coinbase_tx_hex="c0ffee", - block_hash_hex=block_hash, - block_hex="00", - share_pass=True, - block_pass=True, - ), - pending_share=pending, - ) - intent = server.block_candidate_intent(candidate) - ledger.append_batch([(pending, intent)]) - - expected_height = int(candidate.context.template["height"]) - header_state = {"confirmations": -1} - - class MovedTipRpc(FakeRpc): - def __init__(self) -> None: - self.submit_results: list[object] = [] - - def call( - self, - method: str, - params: list[object] | None = None, - *, - timeout: float | None = None, - ) -> object: - if method == "submitblock": - result = None if not self.submit_results else "duplicate" - self.submit_results.append(result) - return result - if method == "getbestblockhash": - # The chain advances as soon as the first offer lands, so - # the in-process retry observes a moved tip. - return parent_hash if not self.submit_results else moved_tip - if method == "getblockhash": - return block_hash - if method == "getblockcount": - return 9 - if method == "getblockheader": - # Unprovable during the tip race (found, not yet active), - # then settled and active for the final pass. - return { - "height": expected_height, - "confirmations": header_state["confirmations"], - } - return super().call(method, params) - - rpc = MovedTipRpc() - server.rpc = rpc - original_mark = ledger.mark_block_candidate_attempted - mark_state = {"failures": 0} - - def flaky_mark(*, block_hash: str) -> bool: - # Fail twice, starting at the first attempt marker after the - # fresh node offer, mirroring statement timeouts under sustained - # saturation. The retained acceptance must survive the failed - # retry too, not just the first failure. - if len(rpc.submit_results) == 1 and mark_state["failures"] < 2: - mark_state["failures"] += 1 - raise RuntimeError("attempt marker timed out") - return original_mark(block_hash=block_hash) - - ledger.mark_block_candidate_attempted = flaky_mark # type: ignore[method-assign] - server.enqueue_block_candidate(candidate) - with tempfile.TemporaryDirectory() as tempdir: - server.audit_dir = Path(tempdir) - server.evidence_path = Path(tempdir) / "evidence.json" - server.ledger_writer_public_key_hex = "aa" * 32 - server.build_audit_bundle = ( # type: ignore[method-assign] - lambda **_kwargs: verified_block_bundle() - ) - server.verify_bundle = ( # type: ignore[method-assign] - lambda *_args, **_kwargs: verified_audit_report() - ) - with patch("builtins.print"): - # First pass: fresh accept from the node, then the attempt - # marker fails and the candidate is retained for an - # in-process retry carrying the definitive node result. - self.assertTrue(server.submit_next_block_candidate()) - self.assertEqual(rpc.submit_results, [None]) - self.assertEqual( - ledger._block_candidate_outbox[block_hash]["state"], - "pending", - ) - # Retry 1 fails the marker again: the retained acceptance - # must survive a consumed-and-failed retry, not just the - # first failure. - self.assertTrue(server.submit_next_block_candidate()) - self.assertEqual(rpc.submit_results, [None]) - self.assertEqual( - ledger._block_candidate_outbox[block_hash]["state"], - "pending", - ) - # Retry 2: the stashed acceptance is still reused, so the - # node is never asked again (no "duplicate" classification, - # no chain probe reliance) and the landing tail finalizes - # exactly as if the first pass had continued. - self.assertTrue(server.submit_next_block_candidate()) - self.assertEqual(rpc.submit_results, [None]) - self.assertEqual( - ledger._block_candidate_outbox[block_hash]["state"], - "submitted", - ) - self.assertNotIn( - PRISM_REJECTION_STALE_JOB, - getattr(server, "block_candidate_abandoned_counts", {}), - ) - self.assertEqual(server.accepted_block_count, 1) - - def test_block_accounting_primary_queue_is_bounded_by_default(self) -> None: - server, _state, _recording = submit_coordinator() - server._ensure_block_accounting_state() - # A bounded primary is what makes the documented result-preserving - # spillover ordering reachable; the overflow queue stays unbounded. - self.assertEqual(server._block_accounting_queue.maxsize, 8) - self.assertEqual(server._block_accounting_overflow_queue.maxsize, 0) - - def test_accounting_saturation_does_not_convoy_node_offers(self) -> None: - server, state, _recording = submit_coordinator() - server.max_blocks = 10 - server.stop_after_block = False - server.block_candidate_retry_initial_seconds = 0.01 - server._block_accounting_queue = queue.PriorityQueue(maxsize=1) - entered_accounting = threading.Event() - release_accounting = threading.Event() - submitted: list[str] = [] - - class RecordingRpc(TipRpc): - def call( - self, - method: str, - params: list[object] | None = None, - *, - timeout: float | None = None, - ) -> object: - if method == "submitblock": - submitted.append(str((params or [""])[0])) - return None - return super().call(method, params) - - server.rpc = RecordingRpc("00" * 32) - - def blocked_accounting( - _candidate: PrismBlockCandidate, - **_kwargs: object, - ) -> bool: - entered_accounting.set() - release_accounting.wait(5) - return True - - server._call_block_candidate_writer = blocked_accounting # type: ignore[method-assign] - for tag in ("a1", "b2", "c3", "d4"): - candidate = block_candidate( - server, - state, - SimpleNamespace( - coinbase_tx_hex="00", - block_hash_hex=tag * 32, - block_hex=tag, - share_pass=True, - block_pass=True, - ), - ) - server.enqueue_block_candidate(candidate) - - submitter = threading.Thread(target=server.block_submit_loop) - with patch("builtins.print"): - submitter.start() - try: - self.assertTrue(entered_accounting.wait(1)) - deadline = time.monotonic() + 2 - while len(submitted) < 4 and time.monotonic() < deadline: - time.sleep(0.01) - self.assertEqual(submitted, ["a1", "b2", "c3", "d4"]) - finally: - server.stop_event.set() - release_accounting.set() - submitter.join(2) - accounting = getattr(server, "_block_accounting_thread", None) - if accounting is not None: - accounting.join(2) - self.assertFalse(submitter.is_alive()) - - def test_replay_batch_reaches_node_while_oldest_accounting_stalls(self) -> None: - server, state, _recording = submit_coordinator() - ledger = SingleWriterShareLedger() - server.ledger = ledger - server.max_blocks = 10 - server.stop_after_block = False - submitted: list[str] = [] - release_accounting = threading.Event() - entered_accounting = threading.Event() - - for index, tag in enumerate(("a5", "b6"), start=1): - candidate = block_candidate( - server, - state, - SimpleNamespace( - coinbase_tx_hex="00", - block_hash_hex=tag * 32, - block_hex=tag, - share_pass=True, - block_pass=True, - ), - ) - pending = PendingShare( - share_id=f"miner-a:{tag * 32}", - miner_id="miner-a", - order_key="miner-a", - p2mr_program_hex="11" * 32, - share_difficulty=1, - network_difficulty=1, - template_height=9, - job_id="job-1", - job_issued_at_ms=1, - accepted_at_ms=index, - ntime=1, - ) - candidate = dataclass_replace(candidate, pending_share=pending) - ledger.append_batch( - [(pending, server.block_candidate_intent(candidate))] - ) - - self.assertEqual(server.replay_pending_block_candidates(), 2) - - class RecordingRpc(TipRpc): - def call( - self, - method: str, - params: list[object] | None = None, - *, - timeout: float | None = None, - ) -> object: - if method == "submitblock": - submitted.append(str((params or [""])[0])) - return None - return super().call(method, params) - - server.rpc = RecordingRpc("00" * 32) - - def blocked_accounting( - _candidate: PrismBlockCandidate, - **_kwargs: object, - ) -> bool: - entered_accounting.set() - release_accounting.wait(5) - return True - - server._call_block_candidate_writer = blocked_accounting # type: ignore[method-assign] - submitter = threading.Thread(target=server.block_submit_loop) - with patch("builtins.print"): - submitter.start() - try: - self.assertTrue(entered_accounting.wait(1)) - deadline = time.monotonic() + 2 - while len(submitted) < 2 and time.monotonic() < deadline: - time.sleep(0.01) - self.assertEqual(submitted, ["a5", "b6"]) - finally: - server.stop_event.set() - release_accounting.set() - submitter.join(2) - accounting = getattr(server, "_block_accounting_thread", None) - if accounting is not None: - accounting.join(2) - - def test_startup_block_replay_timeout_starts_submitter_and_converges(self) -> None: - server, state, _recording = submit_coordinator() - server.max_blocks = 10 - server.stop_after_block = False - server.block_submit_db_timeout_seconds = 0.01 - # The startup enumeration is landing-class (issue #188 fix 4). - server.block_landing_db_timeout_seconds = 0.01 - server.block_landing_db_timeout_max_seconds = 0.01 - server.block_candidate_retry_initial_seconds = 0.01 - ledger = SingleWriterShareLedger() - server.ledger = ledger - block_hash = "e5" * 32 - pending = PendingShare( - share_id=f"miner-a:{block_hash}", - miner_id="miner-a", - order_key="miner-a", - p2mr_program_hex="11" * 32, - share_difficulty=1, - network_difficulty=1, - template_height=9, - job_id="job-1", - job_issued_at_ms=1, - accepted_at_ms=1, - ntime=1, - ) - candidate = block_candidate( - server, - state, - SimpleNamespace( - coinbase_tx_hex="00", - block_hash_hex=block_hash, - block_hex="e5", - share_pass=True, - block_pass=True, - ), - pending_share=pending, - ) - ledger.append_batch( - [(pending, server.block_candidate_intent(candidate))] - ) - - query_started = threading.Event() - release_query = threading.Event() - query_calls = 0 - startup_call: list[object] = [] - original_pending_rows = ledger.pending_block_candidate_rows - - def slow_pending_rows(*, limit: int = 32) -> list[dict[str, object]]: - nonlocal query_calls - query_calls += 1 - if query_calls == 1: - with server._block_submitter_ledger_calls_lock: - startup_call.append( - server._block_submitter_ledger_calls[ - ("replay-outbox-query", 32) - ] - ) - query_started.set() - if not release_query.wait(5): - raise AssertionError("timed out waiting to release outbox query") - return original_pending_rows(limit=limit) - - ledger.pending_block_candidate_rows = slow_pending_rows # type: ignore[method-assign] - original_record_wait = server._record_block_submitter_wait - loop_reuse_waiting = threading.Event() - - def observed_record_wait(phase: str) -> None: - original_record_wait(phase) - if ( - phase == "replay-outbox-query" - and getattr(server, "_block_submitter_thread_ident", None) - == threading.get_ident() - ): - loop_reuse_waiting.set() - - server._record_block_submitter_wait = observed_record_wait # type: ignore[method-assign] - - submitted: list[str] = [] - - class RecordingRpc(TipRpc): - def call( - self, - method: str, - params: list[object] | None = None, - *, - timeout: float | None = None, - ) -> object: - if method == "getblockcount": - return 9 - if method == "submitblock": - submitted.append(str((params or [""])[0])) - return None - return super().call(method, params) - - server.rpc = RecordingRpc("00" * 32) - accounted: list[str] = [] - - def account_candidate( - replayed: PrismBlockCandidate, - **_kwargs: object, - ) -> bool: - replayed_hash = replayed.submission.block_hash_hex - accounted.append(replayed_hash) - ledger.mark_block_candidate_submitted(block_hash=replayed_hash) - server.stop_event.set() - return True - - server._call_block_candidate_writer = account_candidate # type: ignore[method-assign] - - profile = SimpleNamespace(heartbeat_name="stratum_accept_default") - listener_closed = threading.Event() - listener = SimpleNamespace(close=listener_closed.set) - server.listener_profiles = [profile] - server.bind = "127.0.0.1" - server.port = 0 - server.min_ready_miners = 1 - server.audit_bind = None - server.audit_port = 0 - server.hot_path_log_enabled = False - server.blockwait_enabled = False - server.vardiff_idle_sweep_seconds = 0.0 - server.stratum_initial_job_timeout_seconds = 0.0 - server.watchdog_enabled = False - server.watchdog_interval_seconds = 0.1 - server.template_refresh_failure_exit_seconds = 120.0 - server.coordination_blocked_exit_seconds = 120.0 - server.open_stratum_listeners = ( # type: ignore[method-assign] - lambda _stack: [(listener, profile)] - ) - server.validate_live_chain_identity = lambda: None # type: ignore[method-assign] - server.validate_live_template_and_fee_policy = lambda: None # type: ignore[method-assign] - server.prism_payout_policy = lambda: {} # type: ignore[method-assign] - server.prewarm_startup_jobs = lambda: None # type: ignore[method-assign] - server.watchdog_loop = lambda: None # type: ignore[method-assign] - server.blockpoll_loop = lambda: None # type: ignore[method-assign] - server.replay_recovered_shares = lambda: 0 # type: ignore[method-assign] - server.share_append_loop = lambda: None # type: ignore[method-assign] - server.shutdown = lambda *, reason="graceful": True # type: ignore[method-assign] - - def drain_threads(threads: list[tuple[threading.Thread, float]]) -> None: - for thread, timeout in threads: - thread.join(timeout) - - server.drain_non_writer_components = drain_threads # type: ignore[method-assign] - - def accept_after_startup(_listener: object, _profile: object) -> None: - self.assertTrue(loop_reuse_waiting.wait(2)) - self.assertIsNotNone( - getattr(server, "_block_submitter_thread_ident", None) - ) - self.assertTrue(query_started.wait(1)) - # The loop is waiting on the exact startup call while that worker - # remains blocked; no replacement query was spawned. - with server._block_submitter_ledger_calls_lock: - self.assertIs( - server._block_submitter_ledger_calls.get( - ("replay-outbox-query", 32) - ), - startup_call[0], - ) - self.assertEqual(query_calls, 1) - release_query.set() - deadline = time.monotonic() + 2 - while not accounted and time.monotonic() < deadline: - time.sleep(0.01) - self.assertTrue(accounted) - - server.accept_loop = accept_after_startup # type: ignore[method-assign] - - try: - with patch("builtins.print") as printed: - server._serve_with_listener_stack(SimpleNamespace()) # type: ignore[arg-type] - finally: - server.stop_event.set() - release_query.set() - - ledger.pending_block_candidate_rows = original_pending_rows # type: ignore[method-assign] - startup_logs = [ - " ".join(str(value) for value in call.args) - for call in printed.call_args_list - ] - self.assertTrue( - any( - "prism coordinator: startup block candidate replay timed out " - "phase=replay-outbox-query timeout=0.01s" in message - for message in startup_logs - ) - ) - self.assertTrue(query_started.is_set()) - self.assertTrue(listener_closed.is_set()) - self.assertEqual(submitted, ["e5"]) - self.assertEqual(accounted, [block_hash]) - self.assertEqual(ledger.pending_block_candidates(), []) - - def _pending_outbox_backlog( - self, - server: PrismCoordinator, - state: ClientState, - ledger: SingleWriterShareLedger, - count: int, - ) -> list[str]: - """Persist ``count`` pending outbox rows and return their hashes.""" - hashes: list[str] = [] - for index in range(1, count + 1): - block_hash = f"{index:064x}" - hashes.append(block_hash) - pending = PendingShare( - share_id=f"miner-a:{block_hash}", - miner_id="miner-a", - order_key="miner-a", - p2mr_program_hex="11" * 32, - share_difficulty=1, - network_difficulty=1, - template_height=9, - job_id="job-1", - job_issued_at_ms=1, - accepted_at_ms=index, - ntime=1, - ) - candidate = block_candidate( - server, - state, - SimpleNamespace( - coinbase_tx_hex="00", - block_hash_hex=block_hash, - block_hex="00", - share_pass=True, - block_pass=True, - ), - pending_share=pending, - ) - ledger.append_batch( - [(pending, server.block_candidate_intent(candidate))] - ) - return hashes - - def test_configured_single_candidate_pages_preserve_complete_startup_replay(self) -> None: - server, state, _recording = submit_coordinator() - ledger = SingleWriterShareLedger() - server.ledger = ledger - server.config = SimpleNamespace(block=SimpleNamespace(replay_page_size=1)) - hashes = self._pending_outbox_backlog(server, state, ledger, 2) - original_rows = ledger.pending_block_candidate_rows - queries = [] - - def recording_rows(*, limit=32, after_cursor=None): - # Even after adopting the first parent, later accepted work must - # remain discoverable before the job-build gate can open. - self.assertTrue(server._block_replay_enumeration_owed()) - rows = original_rows(limit=limit, after_cursor=after_cursor) - queries.append((limit, after_cursor, len(rows))) - return rows - - ledger.pending_block_candidate_rows = recording_rows - server._note_block_replay_enumeration_owed() - with patch("builtins.print"): - self.assertEqual(server.replay_pending_block_candidates(), 2) - - self.assertEqual([query[0] for query in queries], [1, 1, 1]) - self.assertEqual([query[2] for query in queries], [1, 1, 0]) - self.assertIsNone(queries[0][1]) - self.assertNotEqual(queries[1][1], queries[2][1]) - self.assertEqual( - [ - server._block_replay_candidate_queue.get_nowait().submission.block_hash_hex - for _ in hashes - ], - hashes, - ) - self.assertEqual(set(server._accepted_block_payout_previews), set(hashes)) - self.assertFalse(server._block_replay_enumeration_owed()) - # Enumeration never marks accepted-but-unaccounted work terminal. - self.assertEqual(len(original_rows()), 2) - - def test_single_candidate_page_failure_keeps_gate_closed_and_retry_deduplicates(self) -> None: - server, state, _recording = submit_coordinator() - ledger = SingleWriterShareLedger() - server.ledger = ledger - server.config = SimpleNamespace(block=SimpleNamespace(replay_page_size=1)) - hashes = self._pending_outbox_backlog(server, state, ledger, 2) - original_rows = ledger.pending_block_candidate_rows - - def failing_rows(*, limit=32, after_cursor=None): - if after_cursor is not None: - raise TimeoutError("second candidate unavailable") - return original_rows(limit=limit, after_cursor=after_cursor) - - ledger.pending_block_candidate_rows = failing_rows - server._note_block_replay_enumeration_owed() - with patch("builtins.print"): - with self.assertRaisesRegex(TimeoutError, "second candidate unavailable"): - server.replay_pending_block_candidates() - self.assertTrue(server._block_replay_enumeration_owed()) - self.assertEqual(server._block_replay_candidate_queue.qsize(), 1) - self.assertEqual(len(original_rows()), 2) - ledger.pending_block_candidate_rows = original_rows - self.assertEqual(server.replay_pending_block_candidates(), 1) - - self.assertFalse(server._block_replay_enumeration_owed()) - self.assertEqual(server._block_replay_candidate_queue.qsize(), 2) - self.assertEqual(set(server._accepted_block_payout_previews), set(hashes)) - - def test_configured_single_candidate_pages_apply_to_ancestor_redrive(self) -> None: - server, state, _recording = submit_coordinator() - ledger = SingleWriterShareLedger() - server.ledger = ledger - server.config = SimpleNamespace(block=SimpleNamespace(replay_page_size=1)) - hashes = self._pending_outbox_backlog(server, state, ledger, 2) - service = server._ensure_block_candidate_service() - self.assertFalse(server._block_replay_enumeration_owed()) - - with patch.object( - service, "_consume_ancestor_redrive_requests", return_value=(hashes[1],) - ), patch.object( - server, - "_run_block_submitter_ledger_call", - wraps=server._run_block_submitter_ledger_call, - ) as calls, patch("builtins.print"): - self.assertEqual(server.replay_pending_block_candidates(), 2) - - queries = [call for call in calls.call_args_list if call.args[1] == "replay-outbox-query"] - keys = [call.args[0] for call in queries] - self.assertEqual([key[1] for key in keys], [1, 1, 1]) - self.assertEqual([key[2] for key in keys], [1, 2, 3]) - self.assertEqual(len(set(keys)), 3) - self.assertTrue(all(call.kwargs["call_class"] == "landing" for call in queries)) - self.assertEqual(server._block_replay_candidate_queue.qsize(), 2) - self.assertEqual(set(server._accepted_block_payout_previews), set(hashes)) - - def test_startup_enumeration_pages_a_backlog_larger_than_one_window( - self, - ) -> None: - """A backlog past the enumeration cap must still enumerate completely. - - The doubling window this replaces fails closed once one query would - have to hold the entire backlog, so ~5,000 pending rows kept the - job-build gate shut until the outbox drained under the cap on its - own (testnet4, 2026-08-18: ~25 minutes of refused job builds). A - keyset walk adopts every row in bounded pages and ends only on a - page proven short. - """ - server, state, _recording = submit_coordinator() - ledger = SingleWriterShareLedger() - server.ledger = ledger - hashes = self._pending_outbox_backlog(server, state, ledger, 5) - - queries: list[tuple[int, object]] = [] - original_rows = ledger.pending_block_candidate_rows - - def recording_rows( - *, - limit: int = 32, - after_cursor: object | None = None, - ) -> list[dict[str, object]]: - queries.append((limit, after_cursor)) - return original_rows(limit=limit, after_cursor=after_cursor) - - ledger.pending_block_candidate_rows = recording_rows # type: ignore[method-assign] - server._note_block_replay_enumeration_owed() - - # Both constants shrink together so five pending rows reproduce the - # incident's shape: under the doubling window this backlog starts at - # the cap, every pass is "truncated", and the gate never opens. - with patch( - "lab.prism.block_candidates.MAX_BLOCK_REPLAY_ENUMERATION_ROWS", - 2, - ), patch( - "lab.prism.block_candidates.MAX_PENDING_BLOCK_CANDIDATES", - 2, - ), patch("builtins.print"): - queued = server.replay_pending_block_candidates() - - # Every durable row was adopted exactly once. - self.assertEqual(queued, len(hashes)) - self.assertEqual(server._block_replay_candidate_queue.qsize(), len(hashes)) - replayed = [ - server._block_replay_candidate_queue.get_nowait() - for _ in range(len(hashes)) - ] - self.assertEqual( - sorted( - str(candidate.submission.block_hash_hex) for candidate in replayed - ), - sorted(hashes), - ) - # Three pages of two, two, one: each page is the same bounded query, - # and the short one is what proves the enumeration complete. - self.assertEqual([limit for limit, _cursor in queries], [2, 2, 2]) - self.assertIsNone(queries[0][1]) - self.assertIsNotNone(queries[1][1]) - self.assertNotEqual(queries[1][1], queries[2][1]) - # Every restored row armed its payout barrier before the gate opened. - self.assertEqual(len(server._accepted_block_payout_previews), len(hashes)) - self.assertFalse(server._block_replay_enumeration_owed()) - - def test_startup_enumeration_pages_are_distinct_ledger_calls(self) -> None: - """Distinct pages must not collapse onto one in-flight ledger call. - - A timed-out outbox call stays registered under its dedupe key so the - next paced retry can reuse it. If every page shared one key, page - N+1 would be answered by page N's call and a whole page of pending - candidates would vanish from an enumeration that then declared - itself complete. - """ - server, state, _recording = submit_coordinator() - ledger = SingleWriterShareLedger() - server.ledger = ledger - self._pending_outbox_backlog(server, state, ledger, 3) - - keys: list[tuple[object, ...]] = [] - original_call = server._run_block_submitter_ledger_call - - def recording_call(key, phase, operation, **kwargs): # type: ignore[no-untyped-def] - if phase == "replay-outbox-query": - keys.append(key) - return original_call(key, phase, operation, **kwargs) - - server._run_block_submitter_ledger_call = recording_call # type: ignore[method-assign] - server._note_block_replay_enumeration_owed() - - with patch( - "lab.prism.block_candidates.MAX_BLOCK_REPLAY_ENUMERATION_ROWS", - 1, - ), patch("builtins.print"): - self.assertEqual(server.replay_pending_block_candidates(), 3) - - self.assertEqual(len(keys), 4) - self.assertEqual(len(set(keys)), 4) - # Every key is hashable and carries its page ordinal. - self.assertEqual([key[2] for key in keys], [1, 2, 3, 4]) - - def test_steady_state_poll_never_paginates(self) -> None: - """Without an owed enumeration the poll stays one small query.""" - server, state, _recording = submit_coordinator() - ledger = SingleWriterShareLedger() - server.ledger = ledger - self._pending_outbox_backlog(server, state, ledger, 3) - - queries: list[tuple[int, object]] = [] - original_rows = ledger.pending_block_candidate_rows - - def recording_rows( - *, - limit: int = 32, - after_cursor: object | None = None, - ) -> list[dict[str, object]]: - queries.append((limit, after_cursor)) - return original_rows(limit=limit, after_cursor=after_cursor) - - ledger.pending_block_candidate_rows = recording_rows # type: ignore[method-assign] - - with patch( - "lab.prism.block_candidates.MAX_BLOCK_REPLAY_ENUMERATION_ROWS", - 1, - ), patch("builtins.print"): - self.assertEqual(server.replay_pending_block_candidates(), 3) - - self.assertEqual(queries, [(MAX_PENDING_BLOCK_CANDIDATES, None)]) - - def test_startup_enumeration_falls_back_when_cursors_are_unsupported( - self, - ) -> None: - """A ledger without cursor support keeps today's exact semantics. - - The wrapper here takes ``**kwargs`` -- the shape an instrumenting or - forwarding double really has -- so signature introspection cannot - rule the cursor out and the keyword probe is what discovers the - legacy ledger underneath. The pass must then walk the old doubling - window and, past the cap, still fail closed with enumeration owed. - """ - server, _state, _recording = submit_coordinator() - requested: list[int] = [] - - def legacy_rows(*, limit: int = 32) -> list[dict[str, object]]: - requested.append(limit) - return [durable_candidate_row(index) for index in range(limit)] - - def forwarding_rows(**kwargs: object) -> list[dict[str, object]]: - return legacy_rows(**kwargs) # type: ignore[arg-type] - - server.ledger = SimpleNamespace( - pending_block_candidate_rows=forwarding_rows, - ) - server._note_block_replay_enumeration_owed() - - with patch("builtins.print"): - queued = server.replay_pending_block_candidates() - - # The probe raised TypeError and adopted nothing, so the windowed - # walk starts from the first batch exactly as it always did. - self.assertEqual(requested, [32, 64, 128, 256, 512, 1024]) - self.assertEqual(queued, MAX_BLOCK_REPLAY_ENUMERATION_ROWS) - # Fail closed: an unproven enumeration must not open the job-build - # gate. - self.assertTrue(server._block_replay_enumeration_owed()) - - def test_startup_enumeration_fails_closed_on_a_page_without_a_cursor( - self, - ) -> None: - """A full page that cannot advance the walk must not claim success.""" - server, _state, _recording = submit_coordinator() - pages: list[object] = [] - - def uncursored_rows( - *, - limit: int = 32, - after_cursor: object | None = None, - ) -> list[dict[str, object]]: - pages.append(after_cursor) - return [durable_candidate_row(index) for index in range(limit)] - - server.ledger = SimpleNamespace( - pending_block_candidate_rows=uncursored_rows, - ) - server._note_block_replay_enumeration_owed() - - with patch( - "lab.prism.block_candidates.MAX_BLOCK_REPLAY_ENUMERATION_ROWS", - 2, - ), patch("builtins.print"): - self.assertEqual(server.replay_pending_block_candidates(), 2) - - # One page, then a stop: the rows carry no cursor, so there is no - # way to prove what follows them. - self.assertEqual(pages, [None]) - self.assertTrue(server._block_replay_enumeration_owed()) - - def test_startup_block_replay_catches_psql_server_timeout(self) -> None: - server, _state, _recording = submit_coordinator() - ledger = PsqlShareLedger.__new__(PsqlShareLedger) - ledger._command = ["psql"] - ledger._native = None - ledger._operation_timeout_local = threading.local() - - def timed_out_rows(*, limit: int = 32) -> list[dict[str, object]]: - ledger._run_sql(f"SELECT {limit};") - return [] - - ledger.pending_block_candidate_rows = timed_out_rows # type: ignore[method-assign] - server.ledger = ledger - completed = subprocess.CompletedProcess( - args=["psql"], - returncode=3, - stdout="", - stderr=( - "ERROR: 57014: canceling statement due to statement timeout" - ), - ) - with patch( - "lab.prism.share_ledger.subprocess.run", - return_value=completed, - ), patch("builtins.print") as printed: - self.assertTrue(server._run_startup_block_candidate_replay()) - - startup_logs = [ - " ".join(str(value) for value in call.args) - for call in printed.call_args_list - ] - self.assertTrue( - any( - "startup block candidate replay timed out " - "phase=replay-outbox-query" in message - for message in startup_logs - ) - ) - - def test_startup_block_replay_keeps_hard_database_errors_fatal(self) -> None: - server, _state, _recording = submit_coordinator() - - def failed_rows(*, limit: int = 32) -> list[dict[str, object]]: - raise RuntimeError(f"postgres failed limit={limit}") - - server.ledger = SimpleNamespace( - pending_block_candidate_rows=failed_rows, - ) - with self.assertRaisesRegex(RuntimeError, "postgres failed"): - server._run_startup_block_candidate_replay() - - def test_startup_block_replay_preserves_shutdown_stop(self) -> None: - server = self._bare_coordinator() - - def shutdown_replay() -> int: - raise ShutdownInProgress("shutdown won") - - server.replay_pending_block_candidates = shutdown_replay # type: ignore[method-assign] - - self.assertFalse(server._run_startup_block_candidate_replay()) - - def test_stuck_rpc_worker_pool_requests_nonzero_restart(self) -> None: - server, _state, _recording = submit_coordinator() - server.block_submit_stuck_call_exit_seconds = 0.04 - release = threading.Event() - entered: list[str] = [] - - class IgnoringRpc: - def call( - self, - method: str, - params: list[object] | None = None, - *, - timeout: float | None = None, - ) -> object: - entered.append(str((params or [""])[0])) - release.wait(5) - return None - - server.rpc = IgnoringRpc() - try: - for tag in ("11", "22"): - with self.assertRaises(TimeoutError): - server._run_submitblock_rpc_with_hard_deadline( - block_hash=tag * 32, - block_hex=tag, - timeout_seconds=0.01, - ) - time.sleep(0.05) - with self.assertRaises(TimeoutError): - server._run_submitblock_rpc_with_hard_deadline( - block_hash="33" * 32, - block_hex="33", - timeout_seconds=0.01, - ) - self.assertEqual(entered, ["11", "22"]) - self.assertTrue(server.stop_event.is_set()) - self.assertTrue(server._fatal_exit_requested) - finally: - release.set() - - def test_block_landing_db_timeout_escalates_and_clears(self) -> None: - server, _state, _recording = submit_coordinator() - # The doubling sequence below reaches the reviewed 120s cap, which - # only a tolerance of at least 240s can grant: the landing budget is - # clamped to half the configured watchdog. Pin the production - # tolerance explicitly so this test keeps asserting the escalation - # ladder itself rather than the ceiling. - server.watchdog_timeout_seconds = 300.0 - server.block_landing_db_timeout_seconds = 30.0 - server.block_landing_db_timeout_max_seconds = 120.0 - block_hash = "ab" * 32 - self.assertEqual(server._block_landing_db_timeout(block_hash), 30.0) - server._note_block_landing_timeout(block_hash) - self.assertEqual(server._block_landing_db_timeout(block_hash), 60.0) - server._note_block_landing_timeout(block_hash) - self.assertEqual(server._block_landing_db_timeout(block_hash), 120.0) - server._note_block_landing_timeout(block_hash) - self.assertEqual(server._block_landing_db_timeout(block_hash), 120.0) - self.assertEqual(server._block_landing_db_timeout("cd" * 32), 30.0) - server._clear_block_candidate_retry_state(block_hash) - self.assertEqual(server._block_landing_db_timeout(block_hash), 30.0) - - def test_escalated_landing_budget_stays_inside_watchdog_tolerance(self) -> None: - """Escalation may never grant a budget the watchdog will not wait for. - - A landing step is spent on the watchdog-monitored block-work thread, - so a budget above the tolerance does not buy a longer attempt: the - watchdog hard-exits mid-landing, the in-memory escalation counts die - with the process, and the restart replays the same doomed attempt - from the base budget forever (issue #125). The ceiling is derived - from the configured tolerance, so raising the watchdog restores the - reviewed cap without touching this clamp. - """ - server, _state, _recording = submit_coordinator() - server.watchdog_timeout_seconds = 120.0 - server.block_landing_db_timeout_seconds = 30.0 - server.block_landing_db_timeout_max_seconds = 120.0 - block_hash = "ab" * 32 - for _ in range(8): - server._note_block_landing_timeout(block_hash) - escalated = server._block_landing_db_timeout(block_hash) - self.assertLessEqual(escalated, server.watchdog_timeout_seconds / 2.0) - self.assertEqual(escalated, 60.0) - # The production override is pinned too: a 300s tolerance leaves a - # 150s ceiling, so the reviewed 120s cap is still granted in full and - # deployments running that override see no behavior change. - server.watchdog_timeout_seconds = 300.0 - self.assertEqual(server._block_landing_db_timeout(block_hash), 120.0) - - def test_watchdog_ceiling_clamps_the_configured_base_not_only_the_cap(self) -> None: - """A tolerance below the base budget must lower the base too. - - The first landing attempt already spends the base budget on the - watchdog-monitored thread, so a base above the ceiling trips the - watchdog before any escalation has happened at all -- the clamp - cannot be a cap-only concern. - """ - server, _state, _recording = submit_coordinator() - server.watchdog_timeout_seconds = 40.0 - server.block_landing_db_timeout_seconds = 30.0 - server.block_landing_db_timeout_max_seconds = 120.0 - block_hash = "ab" * 32 - service = server._ensure_block_candidate_service() - self.assertEqual(service._block_landing_watchdog_ceiling(), 20.0) - with contextlib.redirect_stdout(io.StringIO()): - self.assertEqual(server._block_landing_db_timeout(block_hash), 20.0) - for _ in range(8): - server._note_block_landing_timeout(block_hash) - self.assertEqual(server._block_landing_db_timeout(block_hash), 20.0) - - def test_disabled_watchdog_grants_the_configured_landing_budget(self) -> None: - """No hard-exit hazard means no reason to spend the operator's cap. - - The clamp exists solely because a landing budget above the watchdog - tolerance buys a hard exit instead of a longer attempt. A deployment - that turned the watchdog off has no such exit to avoid, so halving - its cap would be pure loss. The attribute is read defensively and - defaults to enabled, since clamping is the safe answer. - """ - server, _state, _recording = submit_coordinator() - server.watchdog_timeout_seconds = 120.0 - server.block_landing_db_timeout_seconds = 30.0 - server.block_landing_db_timeout_max_seconds = 120.0 - block_hash = "ab" * 32 - for _ in range(8): - server._note_block_landing_timeout(block_hash) - - server.watchdog_enabled = False - service = server._ensure_block_candidate_service() - self.assertEqual(service._block_landing_watchdog_ceiling(), float("inf")) - with contextlib.redirect_stdout(io.StringIO()) as quiet: - self.assertEqual(server._block_landing_db_timeout(block_hash), 120.0) - self.assertEqual(server._block_landing_db_timeout(None), 30.0) - # An infinite ceiling never compares below a configured value, so the - # clamp notice must stay silent as well. - self.assertEqual(quiet.getvalue(), "") - - server.watchdog_enabled = True - with contextlib.redirect_stdout(io.StringIO()): - self.assertEqual(server._block_landing_db_timeout(block_hash), 60.0) - - def test_lifecycle_config_still_carries_the_watchdog_enabled_switch(self) -> None: - """The clamp's defensive read must not outlive the attribute itself. - - ``getattr(..., True)`` would keep clamping silently if the setting - were ever renamed, so pin the name the coordinator assigns from. - """ - self.assertIn( - "watchdog_enabled", - {field.name for field in dataclasses.fields(LifecycleConfig)}, - ) - - def test_landing_budget_clamp_is_announced_exactly_once(self) -> None: - """A silently reduced budget is a config mismatch nobody can see. - - Without the notice the operator's configured cap simply does not - happen -- at the 120s default tolerance a 120s cap becomes 60s -- and - "escalation exhausted at my cap" reads identically to "escalation - never reached my cap". It is emitted once because the budget is - recomputed on every landing attempt, and the flag flips under the - coordinator lock so concurrent first landings cannot both print. - """ - server, _state, _recording = submit_coordinator() - server.watchdog_timeout_seconds = 120.0 - server.block_landing_db_timeout_seconds = 30.0 - server.block_landing_db_timeout_max_seconds = 120.0 - block_hash = "ab" * 32 - captured = io.StringIO() - with contextlib.redirect_stdout(captured): - barrier = threading.Barrier(4, timeout=10) - - def landing_budget() -> None: - barrier.wait() - for _ in range(5): - server._block_landing_db_timeout(block_hash) - - threads = [threading.Thread(target=landing_budget) for _ in range(4)] - for thread in threads: - thread.start() - for thread in threads: - thread.join(10) - server._block_landing_db_timeout(block_hash) - lines = [ - line - for line in captured.getvalue().splitlines() - if "landing db budget clamped by watchdog" in line - ] - self.assertEqual(len(lines), 1) - self.assertIn("configured_max=120s", lines[0]) - self.assertIn("granted_max=60s", lines[0]) - self.assertIn("ceiling=60s", lines[0]) - self.assertIn("watchdog_timeout=120s", lines[0]) - - def test_landing_scope_stamps_progress_during_ledger_admission(self) -> None: - """Waiting for the ledger's writer lock must not be heartbeat-silent. - - Admission happens before any statement is sent, so no server-side - deadline bounds it and nothing reports until the gate opens. The - landing scope runs on the block-work owner thread the watchdog - monitors, so it installs the ledger's progress hook and the wait - stamps its phase in watchdog-sized slices instead. - """ - server, _state, _recording = submit_coordinator() - server.watchdog_timeout_seconds = 120.0 - server.block_landing_db_timeout_seconds = 30.0 - server.block_submit_db_timeout_seconds = 1.0 - installed: list[float] = [] - stamped: list[str] = [] - - @contextlib.contextmanager - def operation_progress(on_progress, *, slice_seconds: float): - installed.append(slice_seconds) - on_progress() - yield - - @contextlib.contextmanager - def statement_timeout(seconds: float): - yield - - server.ledger = SimpleNamespace( - statement_timeout=statement_timeout, - operation_progress=operation_progress, - ) - server._record_block_submitter_wait = lambda phase: stamped.append(phase) - with server._block_landing_ledger_statement_timeout_scope("ab" * 32): - pass - with server._block_submitter_ledger_statement_timeout_scope(): - pass - self.assertEqual(len(installed), 2) - for slice_seconds in installed: - self.assertGreater(slice_seconds, 0.0) - self.assertLessEqual(slice_seconds, server.watchdog_timeout_seconds) - self.assertEqual( - stamped, - [ - "wait-ledger-admission:landing", - "wait-ledger-admission:submit", - ], - ) - - def test_landing_scope_accepts_ledgers_without_a_progress_hook(self) -> None: - """Duck-typed ledgers predating the hook keep working unchanged.""" - server, _state, _recording = submit_coordinator() - server.watchdog_timeout_seconds = 120.0 - server.block_landing_db_timeout_seconds = 30.0 - scopes: list[float] = [] - - @contextlib.contextmanager - def statement_timeout(seconds: float): - scopes.append(seconds) - yield - - server.ledger = SimpleNamespace(statement_timeout=statement_timeout) - with server._block_landing_ledger_statement_timeout_scope("ab" * 32): - pass - self.assertEqual(scopes, [30.0]) - - def _landing_phase_recorder(self, server: PrismCoordinator) -> list[str]: - """Capture the block-work phases one landing pass stamps, in order.""" - phases: list[str] = [] - server._record_block_submitter_phase = phases.append # type: ignore[method-assign] - return phases - - def test_landing_brackets_the_reorg_walk_and_prior_balance_check(self) -> None: - """Neither stretch may sit inside the landing scope unnamed. - - Between the current-tip RPC and the audit build the landing thread - spends a full reorg reconciliation -- ledger statements and chain - RPCs, one pair per watched block -- and then a prior-balances read. - The budget granted to a landing step is derived from the watchdog on - the assumption that a heartbeat-silent stretch is about one - statement long, so an unstamped multi-statement stretch here is - exactly the hazard the derivation was supposed to remove (issue - #125). - """ - parent_hash = "00" * 32 - block_hash = "cd" * 32 - server, state, ledger = submit_coordinator(tip=parent_hash) - ledger.durable_payout_state = True - phases = self._landing_phase_recorder(server) - server.ensure_reorg_reconciled_for_tip = ( # type: ignore[method-assign] - lambda _tip, **_kwargs: True - ) - # A stale payout base abandons the candidate right after the check, - # so the landing stops with both bracketed stretches behind it. - server.prior_balances_match_current = ( # type: ignore[method-assign] - lambda _balances: False - ) - submission = SimpleNamespace( - coinbase_tx_hex="c0ffee", - block_hash_hex=block_hash, - block_hex="00", - ) - landed = server._land_and_confirm_block_candidate( - block_candidate(server, state, submission), - current_tip=block_hash, - already_active=False, - worker="miner-a", - node_submission=_BlockCandidateNodeSubmission(attempted=False), - ) - self.assertIsNone(landed) - self.assertEqual( - phases, - [ - "reorg-reconcile", - "reorg-reconcile:complete", - "prior-balances-check", - "prior-balances-check:complete", - ], - ) - - def test_landing_brackets_the_tip_height_rpc(self) -> None: - """A node round trip between two stamped stretches gets its own name.""" - - class StaleHeightTipRpc(TipRpc): - def call( - self, - method: str, - params: list[object] | None = None, - ) -> object: - if method == "getblockcount": - return 42 - return super().call(method, params) - - parent_hash = "00" * 32 - block_hash = "cd" * 32 - server, state, ledger = submit_coordinator(tip=parent_hash) - ledger.durable_payout_state = True - server.rpc = StaleHeightTipRpc(parent_hash) - phases = self._landing_phase_recorder(server) - server.ensure_reorg_reconciled_for_tip = ( # type: ignore[method-assign] - lambda _tip, **_kwargs: True - ) - server.prior_balances_match_current = ( # type: ignore[method-assign] - lambda _balances: True - ) - submission = SimpleNamespace( - coinbase_tx_hex="c0ffee", - block_hash_hex=block_hash, - block_hex="00", - ) - landed = server._land_and_confirm_block_candidate( - block_candidate(server, state, submission), - current_tip=block_hash, - already_active=False, - worker="miner-a", - node_submission=_BlockCandidateNodeSubmission(attempted=False), - ) - # The reported tip is far ahead of the template, so the candidate - # abandons as stale immediately after the bracketed RPC. - self.assertIsNone(landed) - self.assertEqual( - phases[-2:], - ["tip-height-rpc", "tip-height-rpc:complete"], - ) - - def test_block_submitter_phase_never_stamps_from_a_foreign_thread(self) -> None: - """A stamp off the owner thread must not refresh a wedged owner. - - The landing brackets added around the reorg walk, the prior-balance - check and the tip-height RPC all run through this stamper, and the - same code paths also execute on client connection threads. If those - stamps counted, a frozen dedicated thread would look alive for as - long as clients kept solving. - """ - server, _state, _ledger = submit_coordinator() - server.watchdog_timeout_seconds = 120.0 - clock = {"now": 1000.0} - owner_ready = threading.Event() - release_owner = threading.Event() - - def frozen_owner() -> None: - server._block_submitter_thread_ident = threading.get_ident() - server._record_heartbeat("block_submitter") - owner_ready.set() - release_owner.wait(5) - - with patch( - "lab.prism.prism_coordinator.time.monotonic", - side_effect=lambda: clock["now"], - ): - owner = threading.Thread(target=frozen_owner) - owner.start() - try: - self.assertTrue(owner_ready.wait(5)) - clock["now"] += server.watchdog_timeout_seconds + 1.0 - self.assertEqual( - server._overdue_heartbeats(clock["now"]), - ["block_submitter"], - ) - foreign = threading.Thread( - target=lambda: server._record_block_submitter_phase( - "prior-balances-check" - ) - ) - foreign.start() - foreign.join(5) - self.assertEqual( - server._overdue_heartbeats(clock["now"]), - ["block_submitter"], - ) - self.assertNotEqual( - getattr( - server._ensure_block_candidate_service(), - "_block_submitter_phase", - None, - ), - "prior-balances-check", - ) - finally: - release_owner.set() - owner.join(5) - - def test_landing_scope_uses_landing_budget_from_first_attempt(self) -> None: - server, _state, _recording = submit_coordinator() - server.block_submit_db_timeout_seconds = 0.01 - server.block_landing_db_timeout_seconds = 45.0 - server.block_landing_db_timeout_max_seconds = 120.0 - scopes: list[float] = [] - - @contextlib.contextmanager - def statement_timeout(seconds: float): - scopes.append(seconds) - yield - - server.ledger = SimpleNamespace(statement_timeout=statement_timeout) - with server._block_landing_ledger_statement_timeout_scope("ab" * 32): - pass - self.assertEqual(scopes, [45.0]) - metrics = server.block_ledger_call_class_metrics() - self.assertEqual(metrics["landing"]["calls_total"], 1) - self.assertEqual(metrics["landing"]["timeouts_total"], 0) - self.assertEqual(metrics["landing"]["last_budget_seconds"], 45.0) - - def test_landing_scope_timeout_escalates_next_attempt_budget(self) -> None: - server, _state, _recording = submit_coordinator() - # The escalated 90s budget asserted below needs a tolerance of at - # least 180s; the landing budget is otherwise clamped to half the - # configured watchdog. - server.watchdog_timeout_seconds = 300.0 - server.block_landing_db_timeout_seconds = 45.0 - server.block_landing_db_timeout_max_seconds = 120.0 - scopes: list[float] = [] - - @contextlib.contextmanager - def statement_timeout(seconds: float): - scopes.append(seconds) - yield - - server.ledger = SimpleNamespace(statement_timeout=statement_timeout) - block_hash = "ab" * 32 - with self.assertRaises(TimeoutError): - with server._block_landing_ledger_statement_timeout_scope(block_hash): - raise LedgerOperationTimeout("postgres operation exceeded 45s") - with server._block_landing_ledger_statement_timeout_scope(block_hash): - pass - self.assertEqual(scopes, [45.0, 90.0]) - metrics = server.block_ledger_call_class_metrics() - self.assertEqual(metrics["landing"]["calls_total"], 2) - self.assertEqual(metrics["landing"]["timeouts_total"], 1) - self.assertEqual(metrics["landing"]["last_budget_seconds"], 90.0) - - def test_landing_scope_ignores_non_ledger_timeouts(self) -> None: - """The scope guards a body that also runs node RPCs; an RPC timeout - is not a database cancellation and must not escalate the next - PostgreSQL budget or fire the landing-timeout alert.""" - server, _state, _recording = submit_coordinator() - server.block_landing_db_timeout_seconds = 45.0 - server.block_landing_db_timeout_max_seconds = 120.0 - scopes: list[float] = [] - - @contextlib.contextmanager - def statement_timeout(seconds: float): - scopes.append(seconds) - yield - - server.ledger = SimpleNamespace(statement_timeout=statement_timeout) - block_hash = "ab" * 32 - with self.assertRaises(TimeoutError): - with server._block_landing_ledger_statement_timeout_scope(block_hash): - raise TimeoutError("getblockhash exceeded 1s") - with server._block_landing_ledger_statement_timeout_scope(block_hash): - pass - self.assertEqual(scopes, [45.0, 45.0]) - metrics = server.block_ledger_call_class_metrics() - self.assertEqual(metrics["landing"]["calls_total"], 2) - self.assertEqual(metrics["landing"]["timeouts_total"], 0) - - def test_fast_class_ledger_call_metrics_record_timeouts(self) -> None: - server, _state, _recording = submit_coordinator() - server.block_submit_db_timeout_seconds = 0.01 - release = threading.Event() - - def blocking_operation() -> None: - release.wait(5) - - try: - with self.assertRaises(TimeoutError): - server._run_block_submitter_ledger_call( - ("metrics-key",), - "metrics-test", - blocking_operation, - ) - finally: - release.set() - metrics = server.block_ledger_call_class_metrics() - self.assertEqual(metrics["fast"]["timeouts_total"], 1) - self.assertEqual(metrics["fast"]["last_budget_seconds"], 0.01) - - def test_stuck_ledger_worker_pool_requests_nonzero_restart(self) -> None: - server, _state, _recording = submit_coordinator() - server.block_submit_db_timeout_seconds = 0.01 - server.block_submit_stuck_call_exit_seconds = 0.04 - release = threading.Event() - entered: list[str] = [] - - def blocking_operation(tag: str) -> str: - entered.append(tag) - release.wait(5) - return tag - - try: - for tag in ("one", "two"): - with self.assertRaises(TimeoutError): - server._run_block_submitter_ledger_call( - (tag,), - f"test-{tag}", - lambda tag=tag: blocking_operation(tag), - ) - time.sleep(0.05) - with self.assertRaises(TimeoutError): - server._run_block_submitter_ledger_call( - ("three",), - "test-three", - lambda: blocking_operation("three"), - ) - self.assertEqual(entered, ["one", "two"]) - self.assertTrue(server.stop_event.is_set()) - self.assertTrue(server._fatal_exit_requested) - finally: - release.set() - - def test_one_stuck_ledger_call_does_not_restart_with_spare_capacity(self) -> None: - server, _state, _recording = submit_coordinator() - server.block_submit_db_timeout_seconds = 0.01 - server.block_submit_stuck_call_exit_seconds = 0.04 - release = threading.Event() - entered = threading.Event() - - def blocking_operation() -> None: - entered.set() - release.wait(5) - - try: - with self.assertRaises(TimeoutError): - server._run_block_submitter_ledger_call( - ("same-key",), - "same-key", - blocking_operation, - ) - self.assertTrue(entered.wait(1)) - time.sleep(0.05) - with self.assertRaises(TimeoutError): - server._run_block_submitter_ledger_call( - ("same-key",), - "same-key", - blocking_operation, - ) - self.assertFalse(server.stop_event.is_set()) - self.assertFalse(getattr(server, "_fatal_exit_requested", False)) - finally: - release.set() - - def test_two_stuck_ledger_calls_restart_on_existing_key_retry(self) -> None: - server, _state, _recording = submit_coordinator() - server.block_submit_db_timeout_seconds = 0.01 - server.block_submit_stuck_call_exit_seconds = 0.04 - release = threading.Event() - entered: list[str] = [] - - def blocking_operation(tag: str) -> None: - entered.append(tag) - release.wait(5) - - try: - for tag in ("one", "two"): - with self.assertRaises(TimeoutError): - server._run_block_submitter_ledger_call( - (tag,), - tag, - lambda tag=tag: blocking_operation(tag), - ) - time.sleep(0.05) - # No third key is required: a retry reusing either poisoned call - # still observes that every bounded worker slot is exhausted. - with self.assertRaises(TimeoutError): - server._run_block_submitter_ledger_call( - ("one",), - "one", - lambda: blocking_operation("one"), - ) - self.assertEqual(entered, ["one", "two"]) - self.assertTrue(server.stop_event.is_set()) - self.assertTrue(server._fatal_exit_requested) - finally: - release.set() - - def _ledger_worker_thread(self, phase: str) -> threading.Thread: - """The bounded worker still running one stalled ledger call.""" - name = f"prism-block-ledger-{phase}" - for thread in threading.enumerate(): - if thread.name == name: - return thread - raise AssertionError(f"no bounded ledger worker named {name}") - - def _assert_ledger_registry_drained(self, server: PrismCoordinator) -> None: - """Neither an entry nor a worker slot outlives a finished call.""" - with server._block_submitter_ledger_calls_lock: - self.assertEqual(dict(server._block_submitter_ledger_calls), {}) - acquired = [ - server._block_submitter_ledger_worker_slots.acquire(blocking=False) - for _ in range(MAX_BLOCK_SUBMITTER_STUCK_CALL_WORKERS) - ] - for _ in [slot for slot in acquired if slot]: - server._block_submitter_ledger_worker_slots.release() - self.assertEqual(acquired, [True] * MAX_BLOCK_SUBMITTER_STUCK_CALL_WORKERS) - - def _drive_late_ledger_call( - self, - server: PrismCoordinator, - key: tuple[object, ...], - phase: str, - result: object = None, - error: BaseException | None = None, - ) -> _BlockSubmitterLedgerCall: - """Expire the coordinator-side wait, then let the worker land late. - - Returns the call the registry held, once its worker has fully - exited, so a caller can assert on the registry without racing the - removal the worker performs on its way out. - """ - release = threading.Event() - self.addCleanup(release.set) - - def stall() -> object: - if not release.wait(5): - raise AssertionError("the stalled ledger call was never released") - if error is not None: - raise error - return result - - with self.assertRaises(TimeoutError): - server._run_block_submitter_ledger_call(key, phase, stall) - with server._block_submitter_ledger_calls_lock: - call = server._block_submitter_ledger_calls[key] - worker = self._ledger_worker_thread(phase) - release.set() - worker.join(5) - self.assertFalse(worker.is_alive()) - return call - - def _overlapping_ledger_callers( - self, - server: PrismCoordinator, - key: tuple[object, ...], - phase: str, - result: object = None, - error: BaseException | None = None, - count: int = 3, - ) -> tuple[list[object], list[BaseException], list[str]]: - """Park ``count`` callers on one in-flight call, then let it land.""" - started = threading.Event() - release = threading.Event() - self.addCleanup(release.set) - runs: list[str] = [] - - def operation() -> object: - runs.append("run") - started.set() - if not release.wait(5): - raise AssertionError("the shared ledger call was never released") - if error is not None: - raise error - return result - - waiting: set[int] = set() - waiting_lock = threading.Lock() - original_record_wait = server._record_block_submitter_wait - - def observed_record_wait(observed: str) -> None: - original_record_wait(observed) - if observed == phase: - with waiting_lock: - waiting.add(threading.get_ident()) - - server._record_block_submitter_wait = observed_record_wait # type: ignore[method-assign] - - results: list[object] = [] - failures: list[BaseException] = [] - - def caller() -> None: - try: - results.append( - server._run_block_submitter_ledger_call(key, phase, operation) - ) - except BaseException as exc: # pragma: no cover - failure detail - failures.append(exc) - - callers = [threading.Thread(target=caller) for _ in range(count)] - callers[0].start() - self.assertTrue(started.wait(5)) - for thread in callers[1:]: - thread.start() - # Every late arrival has to be parked on the one in-flight call - # before it lands, or nothing here is a statement about overlap. - deadline = time.monotonic() + 5 - while time.monotonic() < deadline: - with waiting_lock: - if len(waiting) == count: - break - time.sleep(0.005) - with waiting_lock: - self.assertEqual(len(waiting), count) - release.set() - for thread in callers: - thread.join(5) - self.assertFalse(thread.is_alive()) - return results, failures, runs - - def test_late_ledger_completions_return_the_registry_to_empty(self) -> None: - """Unique keys that time out and then land late must not pile up. - - The shipped collapse write is keyed by the exact page of block - hashes it abandons, so a write that finally lands after the caller's - deadline is the very thing that empties that page out of the outbox: - no later invocation of that key exists to observe the completion and - drop the entry. The registry has to come back to empty by itself, or - every timed-out page leaks a completed call object and its whole - hash tuple for the life of the process. - """ - server, _state, _recording = submit_coordinator() - server.block_submit_db_timeout_seconds = 0.01 - rounds = 6 - sizes: list[int] = [] - for index in range(rounds): - call = self._drive_late_ledger_call( - server, - ("late", index), - f"late-{index}", - result=index, - ) - # The worker really did finish its operation before retiring it. - self.assertEqual(call.result, index) - with server._block_submitter_ledger_calls_lock: - sizes.append(len(server._block_submitter_ledger_calls)) - - self.assertEqual(sizes, [0] * rounds) - # Nothing was retired while it still owed a bounded worker slot. - self._assert_ledger_registry_drained(server) - - def test_a_late_ledger_success_is_retired_and_then_replayed(self) -> None: - """A late success answered nobody, so it leaves nothing behind.""" - server, _state, _recording = submit_coordinator() - server.block_submit_db_timeout_seconds = 0.01 - runs: list[str] = [] - - def fresh() -> str: - runs.append("fresh") - return "fresh-result" - - call = self._drive_late_ledger_call( - server, - ("late-success",), - "late-success", - result="late-result", - ) - self.assertEqual(call.result, "late-result") - self._assert_ledger_registry_drained(server) - # The key is free again, so the next caller replays the idempotent - # operation instead of being answered by a call it never made. - server.block_submit_db_timeout_seconds = 5.0 - self.assertEqual( - server._run_block_submitter_ledger_call( - ("late-success",), - "late-success", - fresh, - ), - "fresh-result", - ) - self.assertEqual(runs, ["fresh"]) - self._assert_ledger_registry_drained(server) - - def test_a_late_ledger_error_is_retired_and_then_replayed(self) -> None: - """A late failure is not owed to a caller that never waited on it.""" - server, _state, _recording = submit_coordinator() - server.block_submit_db_timeout_seconds = 0.01 - failure = RuntimeError("late ledger failure") - runs: list[str] = [] - - def fresh() -> str: - runs.append("fresh") - return "fresh-result" - - call = self._drive_late_ledger_call( - server, - ("late-error",), - "late-error", - error=failure, - ) - self.assertIs(call.error, failure) - self._assert_ledger_registry_drained(server) - server.block_submit_db_timeout_seconds = 5.0 - self.assertEqual( - server._run_block_submitter_ledger_call( - ("late-error",), - "late-error", - fresh, - ), - "fresh-result", - ) - self.assertEqual(runs, ["fresh"]) - self._assert_ledger_registry_drained(server) - - def test_retirement_never_drops_an_unfinished_or_replaced_call(self) -> None: - """The two guards single flight and slot accounting rest on.""" - server, _state, _recording = submit_coordinator() - service = server._ensure_block_candidate_service() - service._ensure_block_submitter_ledger_call_state() - running = _BlockSubmitterLedgerCall() - successor = _BlockSubmitterLedgerCall() - stale = _BlockSubmitterLedgerCall() - stale.done.set() - finished = _BlockSubmitterLedgerCall() - finished.done.set() - with server._block_submitter_ledger_calls_lock: - server._block_submitter_ledger_calls[("running",)] = running - server._block_submitter_ledger_calls[("replaced",)] = successor - server._block_submitter_ledger_calls[("finished",)] = finished - - # A worker that is still out is what same-key single flight and the - # bounded pool are counted from; it is never dropped. - service._retire_finished_block_submitter_ledger_call(("running",), running) - # A finished call cannot drop the fresh call that has taken its key. - service._retire_finished_block_submitter_ledger_call(("replaced",), stale) - # Its own finished entry does go. - service._retire_finished_block_submitter_ledger_call(("finished",), finished) - - with server._block_submitter_ledger_calls_lock: - self.assertEqual( - dict(server._block_submitter_ledger_calls), - {("running",): running, ("replaced",): successor}, - ) - - def test_overlapping_same_key_waiters_share_one_bounded_worker(self) -> None: - """Single flight is unchanged: one operation, one slot, one answer.""" - server, _state, _recording = submit_coordinator() - server.block_submit_db_timeout_seconds = 5.0 - results, failures, runs = self._overlapping_ledger_callers( - server, - ("shared",), - "shared", - result="shared-result", - ) - self.assertEqual(failures, []) - self.assertEqual(runs, ["run"]) - self.assertEqual(results, ["shared-result"] * 3) - self._assert_ledger_registry_drained(server) - - def test_overlapping_same_key_waiters_all_receive_the_failure(self) -> None: - """Retiring the entry cannot cost a parked waiter what it is owed.""" - server, _state, _recording = submit_coordinator() - server.block_submit_db_timeout_seconds = 5.0 - failure = RuntimeError("shared ledger failure") - results, failures, runs = self._overlapping_ledger_callers( - server, - ("shared-error",), - "shared-error", - error=failure, - ) - self.assertEqual(results, []) - self.assertEqual(runs, ["run"]) - self.assertEqual(failures, [failure] * 3) - self._assert_ledger_registry_drained(server) - - def test_parent_retry_displacement_preserves_durable_descendant(self) -> None: - server, state, _recording = submit_coordinator() - - def candidate_at(tag: str, height: int) -> PrismBlockCandidate: - candidate = block_candidate( - server, - state, - SimpleNamespace( - coinbase_tx_hex="00", - block_hash_hex=tag * 32, - block_hex=tag, - share_pass=True, - block_pass=True, - ), - ) - context = SimpleNamespace(**vars(candidate.context)) - context.template = {**candidate.context.template, "height": height} - return dataclass_replace( - candidate, - context=context, - durable_replay=True, - ) - - descendant = candidate_at("b8", 11) - parent = candidate_at("a8", 10) - descendant_hash = descendant.submission.block_hash_hex - server._ensure_block_candidate_disposition_state() - server._ensure_block_replay_state() - server._block_replay_inflight_hashes.add(descendant_hash) - with server.lock: - server._retry_block_candidate = descendant - server._merge_block_candidate_retry_locked( - "_retry_block_candidate", - parent, - ) - - self.assertIs(server._retry_block_candidate, parent) - self.assertIs( - server._block_disposition_waiting_retries[descendant_hash], - descendant, - ) - - # Once the parent reaches a terminal state, the descendant's preserved - # wakeup is selected even though durable replay still deduplicates it. - with server.lock: - server._retry_block_candidate = None - captured: list[object] = [] - server.max_blocks = 10 - server.stop_after_block = False - server._node_submission_for_candidate = ( # type: ignore[method-assign] - lambda _candidate: SimpleNamespace( - attempted=True, - result="duplicate", - error=None, - ) - ) - server._enqueue_block_accounting_task = ( # type: ignore[method-assign] - lambda task: (captured.append(task), True)[1] - ) - - self.assertTrue(server.submit_next_block_candidate(defer_accounting=True)) - self.assertEqual(len(captured), 1) - task = captured[0] - self.assertIs(task.candidate, descendant) - server._release_block_candidate_disposition(task.disposition_lease) - - def test_synchronous_waiter_joins_finalize_only_registry(self) -> None: - server, state, _recording = submit_coordinator() - candidate = block_candidate( - server, - state, - SimpleNamespace( - coinbase_tx_hex="00", - block_hash_hex="c8" * 32, - block_hex="c8", - share_pass=False, - block_pass=True, - ), - credit_share_on_accept=True, - ) - block_hash = candidate.submission.block_hash_hex - server._block_candidate_finalize_retries = {} - server._block_candidate_finalize_retries[block_hash] = (True, "") - server._node_submission_for_candidate = ( # type: ignore[method-assign] - lambda _candidate: (_ for _ in ()).throw( - AssertionError("finalize-only retry must not call qbitd") - ) - ) - finalized: list[dict[str, object]] = [] - - def finalize( - _candidate: PrismBlockCandidate, - **kwargs: object, - ) -> bool: - finalized.append(kwargs) - return True - - server._finalize_block_candidate = finalize # type: ignore[method-assign] - - self.assertTrue(server._submit_synchronous_block_candidate(candidate)) - self.assertEqual(len(finalized), 1) - self.assertTrue(finalized[0]["accepted"]) - self.assertEqual(finalized[0]["block_hash"], block_hash) - - def test_synchronous_waiter_preserves_finalize_only_abandonment(self) -> None: - server, state, _recording = submit_coordinator() - candidate = block_candidate( - server, - state, - SimpleNamespace( - coinbase_tx_hex="00", - block_hash_hex="c9" * 32, - block_hex="c9", - share_pass=False, - block_pass=True, - ), - credit_share_on_accept=True, - ) - block_hash = candidate.submission.block_hash_hex - server._block_candidate_finalize_retries = { - block_hash: (False, "terminal rejection") - } - server._node_submission_for_candidate = ( # type: ignore[method-assign] - lambda _candidate: (_ for _ in ()).throw( - AssertionError("finalize-only retry must not call qbitd") - ) - ) - finalized: list[dict[str, object]] = [] - server._finalize_block_candidate = ( # type: ignore[method-assign] - lambda _candidate, **kwargs: (finalized.append(kwargs), True)[1] - ) - - self.assertFalse(server._submit_synchronous_block_candidate(candidate)) - self.assertEqual(len(finalized), 1) - self.assertFalse(finalized[0]["accepted"]) - self.assertEqual(finalized[0]["error"], "terminal rejection") - - def test_synchronous_abandon_rejects_prepared_state_before_outbox(self) -> None: - server, state, ledger = submit_coordinator() - block_hash = "ca" * 32 - candidate = block_candidate( - server, - state, - SimpleNamespace( - coinbase_tx_hex="00", - block_hash_hex=block_hash, - block_hex="ca", - share_pass=False, - block_pass=True, - ), - credit_share_on_accept=True, - ) - events: list[str] = [] - prepared = {"value": True} - - def pool_block_state(*, block_hash: str) -> dict[str, str]: - self.assertEqual(block_hash, candidate.submission.block_hash_hex) - events.append("state") - return { - "chain_state": "prepared" if prepared["value"] else "rejected", - "maturity_state": "immature", - } - - def reject_prepared_block(**_kwargs: object) -> dict[str, object]: - events.append("reject") - prepared["value"] = False - return {"backend": "fake", "rejected_count": 1} - - def mark_abandoned(**_kwargs: object) -> bool: - events.append("abandon") - return True - - ledger.pool_block_state = pool_block_state # type: ignore[attr-defined] - ledger.reject_prepared_block = reject_prepared_block # type: ignore[method-assign] - ledger.mark_block_candidate_abandoned = mark_abandoned # type: ignore[attr-defined] - - class CleanupRpc(TipRpc): - def call( - self, - method: str, - params: list[object] | None = None, - ) -> object: - if method == "getblockcount": - return 9 - return super().call(method, params) - - server.rpc = CleanupRpc("00" * 32) - server._node_submission_for_candidate = ( # type: ignore[method-assign] - lambda _candidate: SimpleNamespace( - attempted=False, - result=None, - error=None, - ) - ) - server._mark_block_candidate_attempted = ( # type: ignore[method-assign] - lambda _block_hash: True - ) - - def stage_terminal_rejection( - _candidate: PrismBlockCandidate, - *, - node_submission: object, - ) -> bool: - self.assertIsNotNone(node_submission) - outcome = server._block_candidate_outcome - outcome.reason = PRISM_REJECTION_POOL_CLOSED - outcome.error = "pool is no longer accepting blocks" - outcome.stale_job_class = None - return False - - server._submit_block_candidate_serialized = ( # type: ignore[method-assign] - stage_terminal_rejection - ) - - self.assertFalse(server._submit_synchronous_block_candidate(candidate)) - self.assertEqual(events, ["state", "reject", "abandon"]) - self.assertEqual( - server.block_candidate_abandoned_counts, - {PRISM_REJECTION_POOL_CLOSED: 1}, - ) - self.assertFalse(server._block_candidate_terminal_outcome(block_hash)) - - def test_synchronous_cleanup_failure_keeps_candidate_pending(self) -> None: - server, state, ledger = submit_coordinator() - block_hash = "cb" * 32 - candidate = block_candidate( - server, - state, - SimpleNamespace( - coinbase_tx_hex="00", - block_hash_hex=block_hash, - block_hex="cb", - share_pass=False, - block_pass=True, - ), - credit_share_on_accept=True, - ) - events: list[str] = [] - ledger.pool_block_state = ( # type: ignore[attr-defined] - lambda *, block_hash: { - "chain_state": "prepared", - "maturity_state": "immature", - } - ) - - def fail_reject(**_kwargs: object) -> dict[str, object]: - events.append("reject") - raise RuntimeError("postgres unavailable") - - def unexpected_abandon(**_kwargs: object) -> bool: - events.append("abandon") - return True - - ledger.reject_prepared_block = fail_reject # type: ignore[method-assign] - ledger.mark_block_candidate_abandoned = unexpected_abandon # type: ignore[attr-defined] - - class CleanupRpc(TipRpc): - def call( - self, - method: str, - params: list[object] | None = None, - ) -> object: - if method == "getblockcount": - return 9 - return super().call(method, params) - - server.rpc = CleanupRpc("00" * 32) - server._node_submission_for_candidate = ( # type: ignore[method-assign] - lambda _candidate: SimpleNamespace( - attempted=False, - result=None, - error=None, - ) - ) - server._mark_block_candidate_attempted = ( # type: ignore[method-assign] - lambda _block_hash: True - ) - - def stage_terminal_rejection( - _candidate: PrismBlockCandidate, - *, - node_submission: object, - ) -> bool: - self.assertIsNotNone(node_submission) - outcome = server._block_candidate_outcome - outcome.reason = PRISM_REJECTION_POOL_CLOSED - outcome.error = "pool is no longer accepting blocks" - outcome.stale_job_class = None - return False - - server._submit_block_candidate_serialized = ( # type: ignore[method-assign] - stage_terminal_rejection - ) - - with self.assertRaisesRegex( - RuntimeError, - "could not reject prepared state for terminal candidate", - ): - server._submit_synchronous_block_candidate(candidate) - - self.assertEqual(events, ["reject"]) - self.assertIs(server._retry_block_candidate, candidate) - self.assertEqual(server.block_candidate_abandoned_counts, {}) - self.assertIsNone(server._block_candidate_terminal_outcome(block_hash)) - self.assertEqual( - server._block_candidate_outcome.reason, - PRISM_REJECTION_BACKEND_RPC_UNAVAILABLE, - ) - - def test_restart_resubmit_honors_terminal_abandoned_outbox(self) -> None: - server, state, _recording = submit_coordinator() - ledger = SingleWriterShareLedger() - server.ledger = ledger - block_hash = "e8" * 32 - submission = SimpleNamespace( - header_hex="aa" * 80, - coinbase_tx_hex="00", - block_hash_hex=block_hash, - block_hex="e8", - share_pass=False, - block_pass=True, - ) - first_pending = server.pending_share_from_submission( - context=server.jobs["job-1"], - submission=submission, - ntime_hex="00000001", - ) - first_candidate = block_candidate( - server, - state, - submission, - pending_share=first_pending, - credit_share_on_accept=True, - ) - self.assertTrue( - ledger.persist_block_candidate_intent( - server.block_candidate_intent(first_candidate) - ) - ) - self.assertTrue( - ledger.mark_block_candidate_abandoned( - block_hash=block_hash, - error="terminal before restart", - ) - ) - server._finish_pending_share_commit(first_pending) - submitblock_calls: list[str] = [] - - class NoResubmitRpc(TipRpc): - def call( - self, - method: str, - params: list[object] | None = None, - *, - timeout: float | None = None, - ) -> object: - if method == "submitblock": - submitblock_calls.append(str((params or [""])[0])) - return None - return super().call(method, params) - - server.rpc = NoResubmitRpc("00" * 32) - with patch( - "lab.prism.prism_coordinator.direct_stratum.assemble_submission", - return_value=submission, - ): - with self.assertRaises(StratumError) as raised: - server.handle_submit( - state, - ["miner-a", "job-1", "00" * 8, "00000001", "00000002"], - ) - - self.assertEqual(raised.exception.code, 23) - self.assertEqual(submitblock_calls, []) - self.assertEqual(ledger._block_candidate_outbox[block_hash]["state"], "abandoned") - self.assertEqual(len(ledger), 0) - - def test_restart_resubmit_coalesces_terminal_submitted_outbox(self) -> None: - first_server, first_state, _recording = submit_coordinator() - ledger = SingleWriterShareLedger() - first_server.ledger = ledger - block_hash = "f8" * 32 - submission = SimpleNamespace( - header_hex="ab" * 80, - coinbase_tx_hex="00", - block_hash_hex=block_hash, - block_hex="f8", - share_pass=True, - block_pass=True, - ) - first_pending = PendingShare( - share_id=f"miner-a:{block_hash}", - miner_id="miner-a", - order_key="miner-a", - p2mr_program_hex="11" * 32, - share_difficulty=7, - network_difficulty=1, - template_height=9, - job_id="job-1", - job_issued_at_ms=12345, - accepted_at_ms=1, - ntime=1, - ) - first_candidate = block_candidate( - first_server, - first_state, - submission, - pending_share=first_pending, - ) - first_record = ledger.append_batch( - [ - ( - first_pending, - first_server.block_candidate_intent(first_candidate), - ) - ] - )[0] - self.assertTrue(first_record.newly_inserted) - self.assertTrue( - ledger.mark_block_candidate_submitted(block_hash=block_hash) - ) - - # Model a clean restart: volatile duplicate/disposition state is gone, - # while the ledger and terminal candidate outbox remain authoritative. - server, state, _recording = submit_coordinator() - server.ledger = ledger - submitblock_calls: list[str] = [] - - class NoResubmitRpc(TipRpc): - def call( - self, - method: str, - params: list[object] | None = None, - *, - timeout: float | None = None, - ) -> object: - if method == "submitblock": - submitblock_calls.append(str((params or [""])[0])) - return None - return super().call(method, params) - - server.rpc = NoResubmitRpc("00" * 32) - with patch( - "lab.prism.prism_coordinator.direct_stratum.assemble_submission", - return_value=submission, - ): - self.assertFalse( - server.handle_submit( - state, - ["miner-a", "job-1", "00" * 8, "00000001", "00000002"], - ) - ) - - self.assertEqual(submitblock_calls, []) - self.assertEqual( - ledger._block_candidate_outbox[block_hash]["state"], "submitted" - ) - self.assertEqual(len(ledger), 1) - self.assertEqual(ledger._shares[0].accepted_at_ms, 1) - self.assertTrue(server.block_candidate_queue.empty()) - self.assertEqual( - server.worker_share_counts["miner-a"], - {"submitted": 1, "accepted": 0, "grace": 0}, - ) - - def test_exact_share_replay_does_not_repeat_process_credit(self) -> None: - server, state, _recording = submit_coordinator() - ledger = SingleWriterShareLedger() - server.ledger = ledger - block_hash = "d8" * 32 - submission = SimpleNamespace( - coinbase_tx_hex="00", - block_hash_hex=block_hash, - block_hex="d8", - share_pass=True, - block_pass=True, - ) - pending = PendingShare( - share_id=f"miner-a:{block_hash}", - miner_id="miner-a", - order_key="miner-a", - p2mr_program_hex="11" * 32, - share_difficulty=1, - network_difficulty=1, - template_height=9, - job_id="job-1", - job_issued_at_ms=12345, - accepted_at_ms=12346, - ntime=1, - ) - candidate = block_candidate( - server, - state, - submission, - pending_share=pending, - ) - intent = server.block_candidate_intent(candidate) - worker_credits: list[str] = [] - vardiff_credits: list[str] = [] - server.note_worker_accepted_share = ( # type: ignore[method-assign] - lambda worker, _policy: worker_credits.append(worker) - ) - server.note_vardiff_accepted_share = ( # type: ignore[method-assign] - lambda _client, job: vardiff_credits.append(job.job_id) - ) - - self.assertEqual( - server.append_accepted_share( - state, - candidate.context, - submission, - pending, - candidate_intent=intent, - ), - "pending", - ) - self.assertEqual( - server.append_accepted_share( - state, - candidate.context, - submission, - pending, - candidate_intent=intent, - ), - "pending", - ) - - self.assertEqual(len(ledger), 1) - self.assertEqual(worker_credits, ["miner-a"]) - self.assertEqual(vardiff_credits, ["job-1"]) - - def test_same_hash_busy_lease_preserves_retry_behind_live_work(self) -> None: - server, state, _recording = submit_coordinator() - server.block_candidate_retry_initial_seconds = 0.0 - candidate = block_candidate( - server, - state, - SimpleNamespace( - coinbase_tx_hex="00", - block_hash_hex="a7" * 32, - block_hex="a7", - share_pass=True, - block_pass=True, - ), - ) - block_hash = candidate.submission.block_hash_hex - lease = server._claim_block_candidate_disposition( - block_hash, - blocking=True, - ) - assert lease is not None - server._retry_block_candidate = candidate - server._ensure_block_replay_state() - server._block_replay_inflight_hashes.add(block_hash) - try: - self.assertTrue( - server.submit_next_block_candidate(defer_accounting=True) - ) - self.assertIsNone(server._retry_block_candidate) - self.assertIs( - server._block_disposition_waiting_retries[block_hash], - candidate, - ) - finally: - server._release_block_candidate_disposition(lease) - - captured: list[object] = [] - server._node_submission_for_candidate = ( # type: ignore[method-assign] - lambda _candidate: SimpleNamespace( - attempted=True, - result="duplicate", - error=None, - ) - ) - server._enqueue_block_accounting_task = ( # type: ignore[method-assign] - lambda task: (captured.append(task), True)[1] - ) - self.assertTrue(server.submit_next_block_candidate(defer_accounting=True)) - self.assertEqual(len(captured), 1) - task = captured[0] - server._release_block_candidate_disposition(task.disposition_lease) - - def test_permanently_closed_pool_hands_outbox_to_accounting(self) -> None: - server, state, _recording = submit_coordinator() - server.max_blocks = 1 - server.accepted_block_count = 1 - candidate = block_candidate( - server, - state, - SimpleNamespace( - coinbase_tx_hex="00", - block_hash_hex="f2" * 32, - block_hex="f2", - share_pass=True, - block_pass=True, - ), - ) - captured: list[object] = [] - server._enqueue_block_accounting_task = ( # type: ignore[method-assign] - lambda task: (captured.append(task), True)[1] - ) - server.rpc = SimpleNamespace( - call=lambda *_args, **_kwargs: (_ for _ in ()).throw( - AssertionError("closed pool must not call submitblock") - ) - ) - server.enqueue_block_candidate(candidate) - - self.assertTrue(server.submit_next_block_candidate(defer_accounting=True)) - self.assertEqual(len(captured), 1) - task = captured[0] - self.assertFalse(task.node_submission.attempted) - self.assertIsNone(getattr(server, "_retry_block_candidate", None)) - server._release_block_candidate_disposition(task.disposition_lease) - - def test_accounted_block_replaces_its_capacity_reservation(self) -> None: - server, state, _recording = submit_coordinator() - server.max_blocks = 2 - server.stop_after_block = False - block_hash = "e2" * 32 - candidate = block_candidate( - server, - state, - SimpleNamespace( - coinbase_tx_hex="c0ffee", - block_hash_hex=block_hash, - block_hex="e2", - share_pass=True, - block_pass=True, - ), - ) - server._ensure_block_candidate_disposition_state() - server._block_fast_lane_reservations.add(block_hash) - report = verified_audit_report() - verification_identity = AuditArtifactStore.build_verification_identity( - trust_source="embedded_test_only", - trusted_writer_public_key_hex="aa" * 32, - literal_sha256="cd" * 32, - literal_byte_len=1, - report=report, - ) - _recording._audit_publication_sequences[block_hash] = 1 - server._land_and_confirm_block_candidate = ( # type: ignore[method-assign] - lambda *_args, **_kwargs: ( - verified_block_bundle(), - report, - {"persisted": True}, - {"confirmed_count": 1}, - AuditPublicationIdentity(1, 10, block_hash), - verification_identity, - ) - ) - server.accepted_share_stats = lambda: (0, 0) # type: ignore[method-assign] - with tempfile.TemporaryDirectory() as tempdir: - server.audit_dir = Path(tempdir) - server.evidence_path = Path(tempdir) / "evidence.json" - self.assertTrue( - server._submit_block_candidate_serialized( - candidate, - node_submission=SimpleNamespace( - attempted=True, - result=None, - error=None, - ), - ) - ) - - self.assertEqual(server.accepted_block_count, 1) - self.assertNotIn(block_hash, server._block_fast_lane_reservations) - self.assertTrue(server._reserve_block_fast_lane_slot("e3" * 32)) - - def test_synchronous_candidate_waits_for_fast_lane_capacity(self) -> None: - server, state, _recording = submit_coordinator() - server.max_blocks = 1 - server.stop_after_block = True - reserved_hash = "e4" * 32 - candidate = block_candidate( - server, - state, - SimpleNamespace( - coinbase_tx_hex="c0ffee", - block_hash_hex="e5" * 32, - block_hex="e5", - share_pass=False, - block_pass=True, - ), - credit_share_on_accept=True, - ) - server._ensure_block_candidate_disposition_state() - self.assertTrue(server._reserve_block_fast_lane_slot(reserved_hash)) - server._node_submission_for_candidate = ( # type: ignore[method-assign] - lambda _candidate: (_ for _ in ()).throw( - AssertionError("capacity-blocked candidate must not reach qbitd") - ) - ) - - with self.assertRaisesRegex(RuntimeError, "fast-lane capacity"): - server._submit_synchronous_block_candidate(candidate) - - self.assertIs(server._retry_block_candidate, candidate) - self.assertEqual(server._block_fast_lane_reservations, {reserved_hash}) - - def test_node_offer_exception_retains_reserved_candidate(self) -> None: - server, state, _recording = submit_coordinator() - server.max_blocks = 1 - server.stop_after_block = True - candidate = block_candidate( - server, - state, - SimpleNamespace( - coinbase_tx_hex="c0ffee", - block_hash_hex="e6" * 32, - block_hex="e6", - share_pass=True, - block_pass=True, - ), - ) - block_hash = candidate.submission.block_hash_hex - server._node_submission_for_candidate = ( # type: ignore[method-assign] - lambda _candidate: (_ for _ in ()).throw( - RuntimeError("node offer bookkeeping failed") - ) - ) - server.enqueue_block_candidate(candidate) - - with self.assertRaisesRegex(RuntimeError, "node offer bookkeeping failed"): - server.submit_next_block_candidate(defer_accounting=True) - - self.assertIs(server._retry_block_candidate, candidate) - self.assertIn(block_hash, server._block_fast_lane_reservations) - self.assertFalse(server._reserve_block_fast_lane_slot("e7" * 32)) - lease = server._claim_block_candidate_disposition( - block_hash, - blocking=False, - ) - self.assertIsNotNone(lease) - assert lease is not None - server._release_block_candidate_disposition(lease) - - def test_accounting_enqueue_exception_retains_reserved_candidate(self) -> None: - server, state, _recording = submit_coordinator() - server.max_blocks = 1 - server.stop_after_block = True - candidate = block_candidate( - server, - state, - SimpleNamespace( - coinbase_tx_hex="c0ffee", - block_hash_hex="e8" * 32, - block_hex="e8", - share_pass=True, - block_pass=True, - ), - ) - block_hash = candidate.submission.block_hash_hex - server._node_submission_for_candidate = ( # type: ignore[method-assign] - lambda _candidate: SimpleNamespace( - attempted=True, - result=None, - error=None, - ) - ) - server._enqueue_block_accounting_task = ( # type: ignore[method-assign] - lambda _task: (_ for _ in ()).throw( - RuntimeError("accounting enqueue failed") - ) - ) - server.enqueue_block_candidate(candidate) - - with self.assertRaisesRegex(RuntimeError, "accounting enqueue failed"): - server.submit_next_block_candidate(defer_accounting=True) - - self.assertIs(server._retry_block_candidate, candidate) - self.assertIn(block_hash, server._block_fast_lane_reservations) - self.assertFalse(server._reserve_block_fast_lane_slot("e9" * 32)) - lease = server._claim_block_candidate_disposition( - block_hash, - blocking=False, - ) - self.assertIsNotNone(lease) - assert lease is not None - server._release_block_candidate_disposition(lease) - - -class PendingShareFloorSeamTests(unittest.TestCase): - """S3 floor-holder seams used by the block-candidate credit paths. - - At this layer the pending-commit floor keeps the current id-keyed single - holder per stamped share; the attempt/candidate release seams are the - staged split points a later layer widens into independent holders. These - cases pin the seam wiring so a lost release path cannot silently unhook - the snapshot-anchor floor from candidate handling. - """ - - def _stamped_pending_share(self, server, share_id: str): - share_writer = server._ensure_share_writer_service() - pending = PendingShare( - share_id=share_id, - miner_id="miner-a", - order_key="miner-a", - p2mr_program_hex="11" * 32, - share_difficulty=1, - network_difficulty=1, - template_height=9, - job_id="job-1", - job_issued_at_ms=1, - accepted_at_ms=100, - ntime=1_700_000_000, - ) - with share_writer._pending_share_commit_lock: - share_writer._pending_share_commit_floor[id(pending)] = [ - pending, - time.monotonic(), - False, - ] - return pending - - def test_attempt_and_candidate_seams_release_the_owner_floor(self) -> None: - server = coordinator() - share_writer = server._ensure_share_writer_service() - - attempt = self._stamped_pending_share(server, "miner-a:attempt") - self.assertEqual(server._job_snapshot_anchor_ms(1_000), 99) - server._finish_pending_share_attempt(attempt) - self.assertEqual(share_writer._pending_share_commit_floor, {}) - self.assertEqual(server._job_snapshot_anchor_ms(1_000), 999) - - candidate = self._stamped_pending_share(server, "miner-a:candidate") - server._finish_pending_share_candidate(candidate) - self.assertEqual(share_writer._pending_share_commit_floor, {}) - - def test_distinct_same_hash_stamps_hold_independent_floor_entries(self) -> None: - server = coordinator() - share_writer = server._ensure_share_writer_service() - - first = self._stamped_pending_share(server, "miner-a:same-hash") - second = self._stamped_pending_share(server, "miner-a:same-hash") - - # Releasing one stamped object's holder cannot drop the other live - # holder for the same durable hash: the anchor floor stays clamped. - server._finish_pending_share_candidate(second) - self.assertEqual(len(share_writer._pending_share_commit_floor), 1) - self.assertIs( - share_writer._pending_share_commit_floor[id(first)][0], - first, - ) - self.assertEqual(server._job_snapshot_anchor_ms(1_000), 99) - - server._finish_pending_share_attempt(first) - self.assertEqual(share_writer._pending_share_commit_floor, {}) - - -class DuplicateDropSnapshotFloorTests(unittest.TestCase): - """Duplicate-dropped credit candidates release their adopted floor holders. - - A replayed credit-bearing candidate adopts its reconstructed PendingShare - onto the snapshot-anchor floor at decode, keyed by object identity, so a - same-hash duplicate dropped on any path carries a holder no other object's - disposition can release. Without a release at the drop, the job/payout - snapshot anchor stays clamped below the replayed stamp until restart - (#76 review finding). - """ - - def _durable_credit_candidate( - self, - server: PrismCoordinator, - state: ClientState, - ledger: SingleWriterShareLedger, - tag: str, - stamp: int, - ) -> PrismBlockCandidate: - pending = PendingShare( - share_id=f"miner-a:{tag * 32}", - miner_id="miner-a", - order_key="miner-a", - p2mr_program_hex="11" * 32, - share_difficulty=1, - network_difficulty=1, - template_height=9, - job_id="job-1", - job_issued_at_ms=1, - accepted_at_ms=stamp, - ntime=1, - ) - value = dataclass_replace( - block_candidate( - server, - state, - SimpleNamespace( - coinbase_tx_hex="00", - block_hash_hex=tag * 32, - block_hex="00", - share_pass=False, - block_pass=True, - ), - pending_share=pending, - ), - credit_share_on_accept=True, - ) - ledger.append_batch([(pending, server.block_candidate_intent(value))]) - return value - - def test_replay_dedupe_drop_releases_adopted_floor_holder(self) -> None: - server, state, _recording = submit_coordinator() - ledger = SingleWriterShareLedger() - server.ledger = ledger - self._durable_credit_candidate(server, state, ledger, "a7", 100) - share_writer = server._ensure_share_writer_service() - - with patch("builtins.print"): - self.assertEqual(server.replay_pending_block_candidates(), 1) - first = server._block_replay_candidate_queue.get_nowait() - self.assertEqual(len(share_writer._pending_share_commit_floor), 1) - - # The submitter holds the dequeued candidate mid-disposition, so the - # steady-state poll re-reads the still-pending outbox row: the decode - # adopts a fresh holder before the in-flight dedupe drops the object. - with patch("builtins.print"): - self.assertEqual(server.replay_pending_block_candidates(), 0) - self.assertEqual(len(share_writer._pending_share_commit_floor), 1) - - # Once the surviving object's disposition releases its own holder, no - # leaked duplicate keeps clamping the anchor below the replayed stamp. - server._finish_pending_share_candidate(first.pending_share) - self.assertEqual(server._job_snapshot_anchor_ms(1_000), 999) - - def test_terminal_short_circuit_releases_replayed_duplicate_holder(self) -> None: - server, state, _recording = submit_coordinator() - ledger = SingleWriterShareLedger() - server.ledger = ledger - candidate = self._durable_credit_candidate(server, state, ledger, "b7", 100) - with patch("builtins.print"): - self.assertEqual(server.replay_pending_block_candidates(), 1) - - # A same-hash disposition (e.g. the miner's synchronous resubmit of - # the below-target share) lands terminally while the replayed - # duplicate is still queued behind live solves. - server._record_block_candidate_terminal_outcome( - candidate.submission.block_hash_hex, - accepted=True, - ) - self.assertEqual(server._job_snapshot_anchor_ms(1_000), 99) - - self.assertTrue(server.submit_next_block_candidate()) - - self.assertEqual(server._job_snapshot_anchor_ms(1_000), 999) - - def test_terminal_outcome_releases_parked_same_hash_duplicate_holder(self) -> None: - server, state, _recording = submit_coordinator() - ledger = SingleWriterShareLedger() - server.ledger = ledger - candidate = self._durable_credit_candidate(server, state, ledger, "c7", 100) - with patch("builtins.print"): - self.assertEqual(server.replay_pending_block_candidates(), 1) - parked = server._block_replay_candidate_queue.get_nowait() - - # submit_next parks a candidate that lost the same-hash disposition - # claim; the winning flight then records the terminal outcome and - # discards the parked wakeup. - server._ensure_block_candidate_disposition_state() - with server.lock: - server._block_disposition_waiting_retries[ - candidate.submission.block_hash_hex - ] = parked - server._record_block_candidate_terminal_outcome( - candidate.submission.block_hash_hex, - accepted=True, - ) - - self.assertEqual(server._job_snapshot_anchor_ms(1_000), 999) - - def test_same_hash_retry_merge_releases_displaced_duplicate_holder(self) -> None: - server, state, _recording = submit_coordinator() - ledger = SingleWriterShareLedger() - server.ledger = ledger - self._durable_credit_candidate(server, state, ledger, "d7", 100) - with patch("builtins.print"): - self.assertEqual(server.replay_pending_block_candidates(), 1) - displaced = server._block_replay_candidate_queue.get_nowait() - server._retain_block_candidate_for_retry(displaced) - - # The miner resubmits the same below-target block: a distinct live - # stamped object takes the retry slot and the replayed object dies. - live = dataclass_replace( - block_candidate( - server, - state, - SimpleNamespace( - coinbase_tx_hex="00", - block_hash_hex="d7" * 32, - block_hex="00", - share_pass=False, - block_pass=True, - ), - pending_share=PendingShare( - share_id="miner-a:" + "d7" * 32, - miner_id="miner-a", - order_key="miner-a", - p2mr_program_hex="11" * 32, - share_difficulty=1, - network_difficulty=1, - template_height=9, - job_id="job-1", - job_issued_at_ms=1, - accepted_at_ms=100, - ntime=1, - ), - ), - credit_share_on_accept=True, - ) - server._retain_block_candidate_for_retry(live) - - self.assertIs(server._retry_block_candidate, live) - self.assertEqual(server._job_snapshot_anchor_ms(1_000), 999) - - -# -- issue #198: bounded collapse cleanup-retry backlog ------------------------ - -from contextlib import redirect_stderr, redirect_stdout # noqa: E402 -from io import StringIO # noqa: E402 -from typing import Any # noqa: E402 - -from lab.prism.block_candidates import ( # noqa: E402 - BLOCK_CANDIDATE_COLLAPSE_CLEANUP_STEPS, - MAX_BLOCK_CANDIDATE_TERMINAL_OUTCOMES, - PRISM_BLOCK_CANDIDATE_CLEANUP_BACKPRESSURE_CALLERS, - PRISM_BLOCK_CANDIDATE_COLLAPSE_OUTCOMES, - PRISM_BLOCK_CANDIDATE_COLLAPSE_REASON, -) -from lab.prism.coordinator_config import ( # noqa: E402 - DEFAULT_BLOCK_CANDIDATE_CLEANUP_RETRY_BACKLOG_MAX, - MAX_BLOCK_CANDIDATE_CLEANUP_RETRY_BACKLOG_MAX, - load_coordinator_config, -) -from tests.prism_candidate_storm import OBSERVED_TESTNET_CANDIDATE_STORM # noqa: E402 -from tests.test_prism_block_candidate_collapse import ( # noqa: E402 - CollapseFixture as _CollapseFixture, - _break_cleanup_step as _break_collapse_cleanup_step, - _hash as _collapse_hash, -) - -_BACKLOG_KNOB = "PRISM_BLOCK_CANDIDATE_CLEANUP_RETRY_BACKLOG_MAX" - - -def _minimal_config_environment(root: Path) -> dict[str, str]: - return { - "QBIT_RPC_HOST": "qbit.example", - "QBIT_RPC_USER": "rpc-user", - "QBIT_RPC_PASSWORD": "rpc-password", - "PRISM_ALLOW_MEMORY_LEDGER": "1", - "PRISM_ALLOW_TEST_SIGNING_SEEDS": "1", - "PRISM_ALLOW_BUNDLE_EMBEDDED_LEDGER_KEY": "1", - "PRISM_AUDIT_DIR": str(root), - "PRISM_EVIDENCE_PATH": str(root / "evidence.json"), - } - - -class BlockCandidateCleanupRetryBacklogBoundTests(unittest.TestCase): - """Issue #198: the cleanup-retry backlog is bounded at admission. - - A won row whose post-write cleanup failed keeps one in-memory retry - record, the only replay source it has left. Under a systemic cleanup - fault the collapse would otherwise mint one record per row it wins -- - unbounded at storm scale -- so once the registry holds the configured - number of records the collapse stops handing rows to its fenced bulk - terminalization. Everything already terminal keeps its record, its - exact pending-share holders, and its fence; everything declined stays - durable and pending for the per-row path; and once the backlog drains - admission resumes and the preserved rows collapse on the next walk. - - Every test here is deterministic: the retry pass is driven directly, - the walk is the shipped enumeration, and no thread or sleep is used. - """ - - BOUND = DEFAULT_BLOCK_CANDIDATE_CLEANUP_RETRY_BACKLOG_MAX - - @staticmethod - def _quiet() -> Any: - stack = contextlib.ExitStack() - stack.enter_context(redirect_stdout(StringIO())) - stack.enter_context(redirect_stderr(StringIO())) - return stack - - @staticmethod - def _snapshot(fixture: _CollapseFixture) -> dict[str, Any]: - return fixture.service.collapsed_candidate_cleanup_backlog_snapshot() - - def _drain(self, fixture: _CollapseFixture) -> int: - passes = 0 - with self._quiet(): - while fixture.service._run_one_collapsed_block_candidate_cleanup_retry(): - passes += 1 - return passes - - def _walk(self, fixture: _CollapseFixture) -> tuple[int, str]: - """The shipped enumeration walk, both streams captured.""" - errors = StringIO() - with redirect_stderr(errors): - queued, log = fixture.replay() - return queued, log - - # -- the knob ---------------------------------------------------------- - - def test_the_default_bound_is_derived_from_the_pinned_contracts(self) -> None: - fixture = _CollapseFixture() - self.assertEqual( - fixture.service._collapse_cleanup_retry_backlog_max(), - self.BOUND, - ) - # The restated limit is the fence registry's bound, and the default - # pins at most half of it. - self.assertEqual( - MAX_BLOCK_CANDIDATE_CLEANUP_RETRY_BACKLOG_MAX, - MAX_BLOCK_CANDIDATE_TERMINAL_OUTCOMES, - ) - self.assertLessEqual(self.BOUND, MAX_BLOCK_CANDIDATE_TERMINAL_OUTCOMES // 2) - # It absorbs the observed storm's siblings and one full replay page - # without engaging. - self.assertGreaterEqual(self.BOUND, OBSERVED_TESTNET_CANDIDATE_STORM - 1) - self.assertGreaterEqual(self.BOUND, MAX_BLOCK_REPLAY_ENUMERATION_ROWS) - self.assertIn("backlog_deferred", PRISM_BLOCK_CANDIDATE_COLLAPSE_OUTCOMES) - self.assertEqual( - PRISM_BLOCK_CANDIDATE_CLEANUP_BACKPRESSURE_CALLERS, - ("replay-page", "dequeue"), - ) - - def test_the_bound_reads_the_coordinator_then_the_loaded_config(self) -> None: - fixture = _CollapseFixture() - server = fixture.server - server.config = SimpleNamespace( - block=SimpleNamespace(candidate_cleanup_retry_backlog_max=77) - ) - self.assertEqual(fixture.service._collapse_cleanup_retry_backlog_max(), 77) - server.block_candidate_cleanup_retry_backlog_max = 5 - self.assertEqual(fixture.service._collapse_cleanup_retry_backlog_max(), 5) - # A misconfigured runtime value degrades to admitting one row at a - # time from an empty backlog, never to a collapse that admits nothing. - server.block_candidate_cleanup_retry_backlog_max = 0 - self.assertEqual(fixture.service._collapse_cleanup_retry_backlog_max(), 1) - self.assertEqual( - fixture.service._collapse_cleanup_admission_headroom(), - (1, 0, 1), - ) - - # Direct embedder/test overrides bypass startup validation, so they - # are clamped to the terminal-fence registry's hard capacity. - server.block_candidate_cleanup_retry_backlog_max = ( - MAX_BLOCK_CANDIDATE_CLEANUP_RETRY_BACKLOG_MAX + 1 - ) - self.assertEqual( - fixture.service._collapse_cleanup_retry_backlog_max(), - MAX_BLOCK_CANDIDATE_CLEANUP_RETRY_BACKLOG_MAX, - ) - self.assertEqual( - fixture.service._collapse_cleanup_admission_headroom(), - ( - MAX_BLOCK_CANDIDATE_CLEANUP_RETRY_BACKLOG_MAX, - 0, - MAX_BLOCK_CANDIDATE_CLEANUP_RETRY_BACKLOG_MAX, - ), - ) - - def test_startup_validation_refuses_an_unusable_bound(self) -> None: - with tempfile.TemporaryDirectory() as tmp: - base = _minimal_config_environment(Path(tmp)) - self.assertEqual( - load_coordinator_config(base).block.candidate_cleanup_retry_backlog_max, - self.BOUND, - ) - for raw, message in ( - ("0", f"{_BACKLOG_KNOB} must be positive"), - ("-1", f"{_BACKLOG_KNOB} must be positive"), - ("abc", f"{_BACKLOG_KNOB} must be an integer"), - ( - str(MAX_BLOCK_CANDIDATE_CLEANUP_RETRY_BACKLOG_MAX + 1), - f"{_BACKLOG_KNOB} cannot exceed " - f"{MAX_BLOCK_CANDIDATE_CLEANUP_RETRY_BACKLOG_MAX}", - ), - ): - with self.subTest(raw=raw), self.assertRaisesRegex(SystemExit, message): - load_coordinator_config({**base, _BACKLOG_KNOB: raw}) - for raw in ("1", "2048", str(MAX_BLOCK_CANDIDATE_CLEANUP_RETRY_BACKLOG_MAX)): - with self.subTest(raw=raw): - config = load_coordinator_config({**base, _BACKLOG_KNOB: raw}) - self.assertEqual( - config.block.candidate_cleanup_retry_backlog_max, - int(raw), - ) - - # -- the gauges -------------------------------------------------------- - - def test_the_snapshot_reports_depth_age_holders_and_pins(self) -> None: - fixture = _CollapseFixture() - empty = self._snapshot(fixture) - self.assertEqual( - empty, - { - "depth": 0, - "backlog_max": self.BOUND, - "oldest_age_seconds": -1.0, - "pending_share_holders": 0, - "terminal_outcome_pins": 0, - "backpressure_active": False, - "backpressure_engagements": 0, - }, - ) - target = _collapse_hash(1) - candidate = fixture.seed([target], credit_share_on_accept=True)[0] - fixture.server._ensure_share_writer_service().adopt_pending_share( - candidate.pending_share - ) - fixture.service.candidate_queue.put_nowait(candidate) - broken = _break_collapse_cleanup_step( - fixture.server, - "_finish_pending_share_commit", - ) - with self._quiet(): - fixture.collapse() - snapshot = self._snapshot(fixture) - self.assertEqual(snapshot["depth"], 1) - self.assertEqual(snapshot["pending_share_holders"], 1) - self.assertEqual(snapshot["terminal_outcome_pins"], 1) - self.assertFalse(snapshot["backpressure_active"]) - self.assertGreaterEqual(snapshot["oldest_age_seconds"], 0.0) - # The retained holder is the exact object the queue held. - record = fixture.service._block_candidate_collapse_cleanup_retries[target] - self.assertEqual(len(record.shares), 1) - self.assertIs(record.shares[0], candidate.pending_share) - broken["on"] = False - self.assertEqual(self._drain(fixture), 1) - self.assertEqual(self._snapshot(fixture)["depth"], 0) - self.assertEqual(self._snapshot(fixture)["oldest_age_seconds"], -1.0) - - def test_pins_count_only_records_whose_fence_is_published(self) -> None: - """A record whose fence publication degraded still owes it.""" - fixture = _CollapseFixture() - target = _collapse_hash(1) - fixture.seed([target]) - # The degraded branch the publication itself contains: nothing is - # stamped, and the terminal-outcome step is what still owes it. - fixture.service._publish_collapsed_candidate_terminal_fence = ( - lambda abandoned: None - ) - broken = _break_collapse_cleanup_step( - fixture.server, - "_record_block_candidate_terminal_outcome", - ) - with self._quiet(): - fixture.collapse() - snapshot = self._snapshot(fixture) - self.assertEqual(snapshot["depth"], 1) - self.assertEqual(snapshot["terminal_outcome_pins"], 0) - self.assertIn("terminal-outcome", fixture.cleanup_backlog()[target]) - broken["on"] = False - self.assertEqual(self._drain(fixture), 1) - with fixture.server.lock: - self.assertIs( - fixture.service._block_candidate_terminal_outcomes[target], - False, - ) - self.assertEqual(self._snapshot(fixture)["depth"], 0) - - def test_the_oldest_age_is_measured_from_the_first_deferral(self) -> None: - fixture = _CollapseFixture() - target = _collapse_hash(1) - fixture.seed([target]) - broken = _break_collapse_cleanup_step( - fixture.server, - "_clear_block_candidate_retry_state", - ) - with self._quiet(): - fixture.collapse() - registry = fixture.service._block_candidate_collapse_cleanup_retries - with fixture.server.lock: - registry[target].deferred_monotonic = time.monotonic() - 60.0 - # A failed retry takes the record out and re-registers it; the - # stamp must survive that round trip. - with self._quiet(): - self.assertTrue(fixture.retry_cleanup(force_due=True)[0]) - with fixture.server.lock: - self.assertEqual(registry[target].attempts, 1) - self.assertGreaterEqual(self._snapshot(fixture)["oldest_age_seconds"], 60.0) - broken["on"] = False - self.assertEqual(self._drain(fixture), 1) - - def test_an_inflight_retry_remains_counted_and_pins_its_fence(self) -> None: - """The accounting attempt cannot open an admission or fence gap.""" - fixture = _CollapseFixture() - service = fixture.service - server = fixture.server - server.block_candidate_cleanup_retry_backlog_max = 1 - target = _collapse_hash(1) - service._defer_collapsed_candidate_cleanup(target, ("retry-state",)) - service._ensure_block_candidate_disposition_state() - with server.lock: - service._stamp_block_candidate_terminal_outcome( - service._block_candidate_terminal_outcomes, - target, - False, - ) - - observed: dict[str, Any] = {} - - def observe_inflight(*_args: Any, **_kwargs: Any) -> tuple[dict, dict]: - observed["registry_empty"] = not ( - service._block_candidate_collapse_cleanup_retries - ) - observed["inflight"] = target in ( - service._block_candidate_collapse_cleanup_inflight - ) - observed["headroom"] = service._collapse_cleanup_admission_headroom() - observed["snapshot"] = self._snapshot(fixture) - # Make this fence the oldest entry in an overflowing outcome - # registry. The in-flight authority must still pin it while an - # unrelated unpinned outcome is evicted. - with server.lock: - outcomes = service._block_candidate_terminal_outcomes - for index in range(MAX_BLOCK_CANDIDATE_TERMINAL_OUTCOMES + 1): - other = _collapse_hash((1 << 40) + index) - service._stamp_block_candidate_terminal_outcome( - outcomes, - other, - False, - ) - service._bound_block_candidate_terminal_outcomes() - observed["fence_pinned"] = target in outcomes - return {}, {target: ()} - - service._run_collapsed_candidate_cleanup_steps = observe_inflight - with self._quiet(): - self.assertTrue( - service._run_one_collapsed_block_candidate_cleanup_retry() - ) - - self.assertTrue(observed["registry_empty"]) - self.assertTrue(observed["inflight"]) - self.assertEqual(observed["headroom"], (0, 1, 1)) - snapshot = observed["snapshot"] - self.assertEqual(snapshot["depth"], 1) - self.assertEqual(snapshot["terminal_outcome_pins"], 1) - self.assertTrue(snapshot["backpressure_active"]) - self.assertTrue(observed["fence_pinned"]) - self.assertEqual(self._snapshot(fixture)["depth"], 0) - - # -- backpressure at the shipped bound: the replay-page walk ------------ - - def test_bulk_terminalization_stops_admitting_rows_at_the_bound(self) -> None: - """Sustained failure at the shipped bound, then recovery, page path.""" - overflow = 128 - hashes = [_collapse_hash(index) for index in range(1, self.BOUND + overflow + 1)] - fixture = _CollapseFixture() - fixture.seed(hashes) - broken = _break_collapse_cleanup_step( - fixture.server, - "_clear_block_candidate_retry_state", - ) - # Four full pages are admitted and mint BOUND records; the fifth - # page finds no headroom and is preserved before any chain read. - queued, log = self._walk(fixture) - counts = fixture.counts() - snapshot = self._snapshot(fixture) - self.assertEqual(snapshot["depth"], self.BOUND) - self.assertTrue(snapshot["backpressure_active"]) - self.assertEqual(snapshot["backpressure_engagements"], 1) - self.assertEqual(snapshot["terminal_outcome_pins"], self.BOUND) - self.assertEqual(counts["abandoned"], self.BOUND) - self.assertEqual(counts["cleanup_failed"], self.BOUND) - self.assertEqual(counts["backlog_deferred"], overflow) - self.assertEqual(counts["considered"], self.BOUND) - # The preserved rows stayed durable and were adopted for the - # per-row path exactly as an unprobed height leaves them. - self.assertEqual(len(fixture.pending()), overflow) - self.assertEqual(queued, overflow) - self.assertEqual(len(fixture.ledger.abandon_calls), 4) - self.assertIn( - "cleanup backpressure engaged caller=replay-page " - f"rows_preserved={overflow} admitted=0 backlog={self.BOUND} " - f"backlog_max={self.BOUND}", - log, - ) - warning = next(line for line in log.splitlines() if "backpressure engaged" in line) - for block_hash in hashes: - self.assertNotIn(block_hash, warning) - # Sustained failure: retries keep failing, nothing is shed. - for attempt in range(1, 17): - with self._quiet(): - self.assertTrue(fixture.retry_cleanup(force_due=True)[0]) - self.assertEqual(self._snapshot(fixture)["depth"], self.BOUND) - self.assertEqual(fixture.counts()["cleanup_retry_failed"], 16) - self.assertEqual(self._snapshot(fixture)["terminal_outcome_pins"], self.BOUND) - # A second walk while the fault persists admits nothing more and - # re-adopts nothing; the warning is rate limited, the counter is not. - queued, log = self._walk(fixture) - self.assertEqual(queued, 0) - self.assertNotIn("backpressure engaged", log) - counts = fixture.counts() - self.assertEqual(counts["abandoned"], self.BOUND) - self.assertEqual(counts["backlog_deferred"], 2 * overflow) - self.assertEqual(self._snapshot(fixture)["backpressure_engagements"], 2) - self.assertEqual(len(fixture.ledger.abandon_calls), 4) - # Recovery: the backlog drains one hash per pass, admission resumes, - # and the preserved remainder collapses on the next walk. - broken["on"] = False - self.assertEqual(self._drain(fixture), self.BOUND) - snapshot = self._snapshot(fixture) - self.assertEqual(snapshot["depth"], 0) - self.assertFalse(snapshot["backpressure_active"]) - self.assertEqual(fixture.counts()["cleanup_recovered"], self.BOUND) - self.assertEqual(fixture.cleanup_backlog(), {}) - queued, log = self._walk(fixture) - self.assertEqual(queued, 0) - self.assertEqual(fixture.pending(), set()) - counts = fixture.counts() - self.assertEqual(counts["abandoned"], self.BOUND + overflow) - self.assertEqual(counts["backlog_deferred"], 2 * overflow) - self.assertEqual(len(fixture.ledger.abandon_calls), 5) - # Every terminal row is fenced and none was ever offered. - self.assertNotIn("submitblock", fixture.rpc.counts()) - with fixture.server.lock: - outcomes = fixture.service._block_candidate_terminal_outcomes - self.assertTrue(all(outcomes.get(block_hash) is False for block_hash in hashes)) - self.assertEqual( - fixture.server.block_candidate_abandoned_counts, - {PRISM_BLOCK_CANDIDATE_COLLAPSE_REASON: self.BOUND + overflow}, - ) - - def test_forced_walk_pauses_after_one_full_backpressured_page(self) -> None: - """A deep fault cannot move the durable storm into the replay queue.""" - page_rows = MAX_BLOCK_REPLAY_ENUMERATION_ROWS - total = 2 * page_rows + 7 - fixture = _CollapseFixture() - fixture.server.block_candidate_cleanup_retry_backlog_max = page_rows - fixture.seed([_collapse_hash(index) for index in range(1, total + 1)]) - _break_collapse_cleanup_step( - fixture.server, - "_clear_block_candidate_retry_state", - ) - - queued, log = self._walk(fixture) - - self.assertEqual(self._snapshot(fixture)["depth"], page_rows) - self.assertEqual(queued, page_rows) - self.assertEqual( - fixture.service._block_replay_candidate_queue.qsize(), - page_rows, - ) - self.assertEqual(len(fixture.pending()), page_rows + 7) - self.assertIn("enumeration page=1", log) - self.assertIn("enumeration page=2", log) - self.assertNotIn("enumeration page=3", log) - self.assertIn("paused after cleanup backpressure", log) - self.assertTrue(fixture.server._block_replay_enumeration_owed()) - - # Even a direct forced re-entry waits for the adopted page to drain; - # it cannot append the seven rows still sitting on a later page. - engagements = self._snapshot(fixture)["backpressure_engagements"] - queued_again, log_again = self._walk(fixture) - self.assertEqual(queued_again, 0) - self.assertEqual(log_again, "") - self.assertEqual( - fixture.service._block_replay_candidate_queue.qsize(), - page_rows, - ) - self.assertEqual( - self._snapshot(fixture)["backpressure_engagements"], - engagements, - ) - self.assertTrue(fixture.server._block_replay_enumeration_owed()) - - # -- backpressure at the shipped bound: the dequeue-time skip ----------- - - def test_the_dequeue_time_skip_honours_the_same_bound(self) -> None: - fixture = _CollapseFixture() - service = fixture.service - server = fixture.server - # Fill the backlog to the bound through the shipped deferral. - with self._quiet(): - for index in range(self.BOUND): - service._defer_collapsed_candidate_cleanup( - _collapse_hash((1 << 20) + index), - ("retry-state",), - ) - self.assertEqual(self._snapshot(fixture)["depth"], self.BOUND) - fixture.rpc.results["submitblock"] = None - server.block_submit_rpc_timeout_seconds = 0.5 - - def offers() -> int: - return sum(1 for method, _ in fixture.rpc.calls if method == "submitblock") - - first = _collapse_hash(1) - candidate = fixture.seed([first])[0] - service.candidate_queue.put_nowait(candidate) - with self._quiet(): - self.assertTrue(server.submit_next_block_candidate(defer_accounting=True)) - counts = fixture.counts() - # Not considered, not skipped: preserved for the ordinary offer path, - # which offered it. - self.assertEqual(counts["backlog_deferred"], 1) - self.assertEqual(counts["dequeue_considered"], 0) - self.assertEqual(counts["dequeue_skipped"], 0) - self.assertEqual(offers(), 1) - self.assertIn(first, fixture.pending()) - self.assertEqual(self._snapshot(fixture)["depth"], self.BOUND) - self.assertEqual(self._snapshot(fixture)["backpressure_engagements"], 1) - # Once the backlog drains the skip is back. - self.assertEqual(self._drain(fixture), self.BOUND) - second = _collapse_hash(2) - candidate = fixture.seed([second])[0] - service.candidate_queue.put_nowait(candidate) - with self._quiet(): - self.assertTrue(server.submit_next_block_candidate(defer_accounting=True)) - counts = fixture.counts() - self.assertEqual(counts["dequeue_considered"], 1) - self.assertEqual(counts["dequeue_skipped"], 1) - self.assertEqual(counts["backlog_deferred"], 1) - self.assertEqual(offers(), 1) - self.assertNotIn(second, fixture.pending()) - - # -- what backpressure preserves, and what it never touches ------------- - - def test_holders_and_fences_survive_backpressure_and_release_once(self) -> None: - fixture = _CollapseFixture() - server = fixture.server - server.block_candidate_cleanup_retry_backlog_max = 2 - hashes = [_collapse_hash(index) for index in range(1, 5)] - candidates = fixture.seed(hashes, credit_share_on_accept=True) - writer = server._ensure_share_writer_service() - for candidate in candidates: - writer.adopt_pending_share(candidate.pending_share) - fixture.service.candidate_queue.put_nowait(candidate) - by_hash = dict(zip(hashes, candidates)) - releases: dict[int, int] = {} - original = server._finish_pending_share_commit - broken = {"on": True} - - def finish(pending_share: Any) -> None: - if broken["on"]: - raise RuntimeError("floor release boom") - releases[id(pending_share)] = releases.get(id(pending_share), 0) + 1 - original(pending_share) - - server._finish_pending_share_commit = finish - with self._quiet(): - fixture.collapse() - admitted = set(fixture.cleanup_backlog()) - preserved = set(hashes) - admitted - self.assertEqual(len(admitted), 2) - self.assertEqual(fixture.pending(), preserved) - self.assertEqual(fixture.counts()["backlog_deferred"], 2) - registry = fixture.service._block_candidate_collapse_cleanup_retries - for block_hash in admitted: - self.assertEqual(registry[block_hash].shares, (by_hash[block_hash].pending_share,)) - with server.lock: - outcomes = fixture.service._block_candidate_terminal_outcomes - for block_hash in admitted: - self.assertIs(outcomes[block_hash], False) - for block_hash in preserved: - self.assertNotIn(block_hash, outcomes) - # Sustained failure keeps the exact holder objects and the fences. - for _ in range(2): - with self._quiet(): - self.assertTrue(fixture.retry_cleanup(force_due=True)[0]) - for block_hash in admitted: - self.assertEqual(registry[block_hash].shares, (by_hash[block_hash].pending_share,)) - snapshot = self._snapshot(fixture) - self.assertEqual(snapshot["pending_share_holders"], 2) - self.assertEqual(snapshot["terminal_outcome_pins"], 2) - self.assertEqual({id(candidate.pending_share) for candidate in candidates}, set(fixture.floor())) - # Recovery releases exactly the admitted holders, once each; the - # preserved candidates keep theirs until their own collapse. - broken["on"] = False - self.assertEqual(self._drain(fixture), 2) - self.assertEqual( - set(fixture.floor()), - {id(by_hash[block_hash].pending_share) for block_hash in preserved}, - ) - with self._quiet(): - fixture.collapse() - self.assertEqual(fixture.pending(), set()) - self.assertEqual(fixture.floor(), {}) - self.assertEqual(set(releases.values()), {1}) - self.assertEqual(len(releases), 4) - self.assertEqual(fixture.cleanup_backlog(), {}) - self.assertNotIn("submitblock", fixture.rpc.counts()) - - def test_admission_resumes_as_the_backlog_drains(self) -> None: - fixture = _CollapseFixture() - fixture.server.block_candidate_cleanup_retry_backlog_max = 4 - hashes = [_collapse_hash(index) for index in range(1, 9)] - fixture.seed(hashes) - broken = _break_collapse_cleanup_step( - fixture.server, - "_clear_block_candidate_retry_state", - ) - with self._quiet(): - fixture.collapse() - self.assertEqual(self._snapshot(fixture)["depth"], 4) - self.assertEqual(len(fixture.pending()), 4) - self.assertEqual(fixture.counts()["backlog_deferred"], 4) - broken["on"] = False - for _ in range(2): - with self._quiet(): - self.assertTrue(fixture.retry_cleanup()[0]) - # Two records drained, so two rows of headroom: exactly two more - # rows are admitted and the other two preserved again. - with self._quiet(): - fixture.collapse() - counts = fixture.counts() - self.assertEqual(counts["abandoned"], 6) - self.assertEqual(counts["backlog_deferred"], 6) - self.assertEqual(len(fixture.pending()), 2) - self.assertEqual(self._snapshot(fixture)["depth"], 2) - self.assertEqual(self._drain(fixture), 2) - with self._quiet(): - fixture.collapse() - self.assertEqual(fixture.pending(), set()) - self.assertEqual(fixture.counts()["abandoned"], 8) - self.assertEqual(self._snapshot(fixture)["backpressure_engagements"], 2) - - def test_the_backpressure_warning_is_bounded_and_hash_free(self) -> None: - fixture = _CollapseFixture() - fixture.server.block_candidate_cleanup_retry_backlog_max = 2 - hashes = [_collapse_hash(index) for index in range(1, 7)] - fixture.seed(hashes) - broken = _break_collapse_cleanup_step( - fixture.server, - "_clear_block_candidate_retry_state", - ) - with redirect_stderr(StringIO()): - _retained, first_log = fixture.collapse() - _retained, second_log = fixture.collapse() - broken["on"] = False - warnings = [ - line - for log in (first_log, second_log) - for line in log.splitlines() - if "cleanup backpressure engaged" in line - ] - self.assertEqual(len(warnings), 1) - # Two admitted from an empty backlog, four preserved: the bound is - # what the pass would have exceeded, not what it had reached. - self.assertIn( - "caller=replay-page rows_preserved=4 admitted=2 backlog=0 backlog_max=2", - warnings[0], - ) - self.assertIn("oldest_seconds=", warnings[0]) - for block_hash in hashes: - self.assertNotIn(block_hash, warnings[0]) - # The counter and the engagement count moved for both occasions. - self.assertEqual(fixture.counts()["backlog_deferred"], 8) - self.assertEqual(self._snapshot(fixture)["backpressure_engagements"], 2) - self.assertEqual(fixture.counts()["selected"], 6) - self.assertEqual(fixture.counts()["considered"], 6) - with self.assertRaises(ValueError): - fixture.service._note_block_candidate_cleanup_backpressure( - caller="not-a-caller", - rows=1, - admitted=0, - depth=2, - maximum=2, - ) - - def test_the_fault_vocabulary_the_rig_injects_matches_the_cleanup_steps(self) -> None: - from tests.prism_candidate_storm import CLEANUP_FAULTS - - self.assertEqual( - set(CLEANUP_FAULTS) - {"floor-index"}, - set(BLOCK_CANDIDATE_COLLAPSE_CLEANUP_STEPS), - ) diff --git a/tests/test_prism_block_finalization.py b/tests/test_prism_block_finalization.py deleted file mode 100644 index 1ac94f367..000000000 --- a/tests/test_prism_block_finalization.py +++ /dev/null @@ -1,271 +0,0 @@ -#!/usr/bin/env python3 -"""Direct tests for the PRISM accepted-block finalization owner.""" - -from __future__ import annotations - -import ast -from pathlib import Path -from types import SimpleNamespace -import unittest -from unittest import mock - -import lab.prism.block_finalization as block_finalization_module -from lab.prism.block_finalization import ( - FINALIZATION_PHASES, - BlockFinalizationService, -) - - -class RecordingFinalizationService(BlockFinalizationService): - def __init__(self, *, admission: bool = True, landing: bool = True) -> None: - super().__init__(SimpleNamespace()) - self.events: list[str] = [] - self.admission_result: object = ( - SimpleNamespace(block_hash="ab" * 32) if admission else None - ) - self.landing_result = object() if landing else None - self.already_accounted = False - - def _record_block_candidate_progress( - self, - phase: str = "accounting-progress", - ) -> None: - return None - - def _admit_candidate( - self, - candidate: object, - *, - node_submission: object, - ) -> object | None: - self.events.append("admission") - return self.admission_result - - def _land_candidate(self, admission: object) -> object | None: - self.events.append("land_confirm") - return self.landing_result - - def _candidate_already_accounted(self, block_hash: str) -> bool: - self.events.append("accounted_check") - return self.already_accounted - - def _persist_ctv_and_credit( - self, - admission: object, - landed: object, - ) -> dict[str, object]: - self.events.append("ctv_credit") - return {"stored": True} - - def _build_finalization_evidence( - self, - admission: object, - landed: object, - ctv_persistence: dict[str, object], - ) -> object: - self.events.append("evidence") - return object() - - def _publish_finalization_evidence( - self, - landed: object, - prepared: object, - ) -> dict[str, object]: - self.events.append("audit_publish") - return {"published": True} - - def _account_finalized_candidate( - self, - admission: object, - landed: object, - published_evidence: dict[str, object], - ) -> bool: - self.events.append("accounting") - return True - - -class BlockFinalizationServiceTests(unittest.TestCase): - def test_success_runs_named_phases_once_in_durable_order(self) -> None: - service = RecordingFinalizationService() - monotonic_values = iter(range(13)) - - with mock.patch( - "lab.prism.block_finalization.time.monotonic", - side_effect=lambda: float(next(monotonic_values)), - ): - self.assertTrue( - service._submit_block_candidate_serialized( - object(), # type: ignore[arg-type] - node_submission=object(), # type: ignore[arg-type] - ) - ) - - self.assertEqual( - service.events, - [ - "admission", - "land_confirm", - "accounted_check", - "ctv_credit", - "evidence", - "audit_publish", - "accounting", - ], - ) - snapshot = service.metrics_snapshot() - for phase in FINALIZATION_PHASES: - self.assertEqual(snapshot["phases"][phase]["count"], 1) - self.assertEqual(snapshot["phases"][phase]["sum"], 1.0) - self.assertEqual(snapshot["phases"][phase]["max"], 1.0) - - def test_terminal_boundaries_do_not_run_later_phases(self) -> None: - rejected = RecordingFinalizationService(admission=False) - self.assertFalse( - rejected._submit_block_candidate_serialized( - object(), # type: ignore[arg-type] - node_submission=object(), # type: ignore[arg-type] - ) - ) - self.assertEqual(rejected.events, ["admission"]) - - # A tri-state admission verdict (already-recorded acceptance or an - # accepted race win) is terminal for the whole run: the ordered - # phases after admission never execute in either direction. - for terminal_result in (True, False): - short_circuit = RecordingFinalizationService() - short_circuit.admission_result = terminal_result - self.assertIs( - short_circuit._submit_block_candidate_serialized( - object(), # type: ignore[arg-type] - node_submission=object(), # type: ignore[arg-type] - ), - terminal_result, - ) - self.assertEqual(short_circuit.events, ["admission"]) - - not_landed = RecordingFinalizationService(landing=False) - self.assertFalse( - not_landed._submit_block_candidate_serialized( - object(), # type: ignore[arg-type] - node_submission=object(), # type: ignore[arg-type] - ) - ) - self.assertEqual(not_landed.events, ["admission", "land_confirm"]) - - replay = RecordingFinalizationService() - replay.already_accounted = True - self.assertTrue( - replay._submit_block_candidate_serialized( - object(), # type: ignore[arg-type] - node_submission=object(), # type: ignore[arg-type] - ) - ) - self.assertEqual( - replay.events, - ["admission", "land_confirm", "accounted_check"], - ) - - def test_candidate_interarrival_metric_is_bounded_aggregate_evidence(self) -> None: - service = BlockFinalizationService(SimpleNamespace()) - with mock.patch( - "lab.prism.block_finalization.time.monotonic", - side_effect=[3.0, 7.5, 13.0], - ): - service._note_candidate_started() - service._note_candidate_started() - service._note_candidate_started() - - intervals = service.metrics_snapshot()["candidate_intervals"] - self.assertEqual(intervals["count"], 2) - self.assertEqual(intervals["sum"], 10.0) - self.assertEqual(intervals["min"], 4.5) - lines = service.metrics_lines() - self.assertIn( - "qbit_prism_block_candidate_interarrival_seconds_count 2", - lines, - ) - self.assertIn( - "qbit_prism_block_candidate_interarrival_seconds_min 4.500000", - lines, - ) - - -class BlockFinalizationPortSurfaceTests(unittest.TestCase): - """Mechanically compare runtime accesses against the declared port.""" - - # The verify_bundle override is read through runtime.__dict__ on purpose - # (a per-instance escape hatch kept out of the typed port), so the AST - # sees a "__dict__" attribute access that no Protocol should declare. - PORT_ACCESS_ALLOWLIST = frozenset({"__dict__"}) - - @staticmethod - def _module_tree() -> ast.Module: - source_path = Path(block_finalization_module.__file__) - return ast.parse(source_path.read_text(encoding="utf-8")) - - @classmethod - def _class_node(cls, tree: ast.Module, name: str) -> ast.ClassDef: - for node in ast.walk(tree): - if isinstance(node, ast.ClassDef) and node.name == name: - return node - raise AssertionError(f"class {name} not found in module source") - - def test_every_runtime_access_is_declared_in_the_port(self) -> None: - tree = self._module_tree() - service = self._class_node(tree, "BlockFinalizationService") - port = self._class_node(tree, "BlockFinalizationPort") - - port_members: set[str] = set() - for statement in port.body: - if isinstance( - statement, (ast.FunctionDef, ast.AsyncFunctionDef) - ): - port_members.add(statement.name) - elif isinstance(statement, ast.AnnAssign) and isinstance( - statement.target, ast.Name - ): - port_members.add(statement.target.id) - self.assertTrue(port_members, "port declared no members") - - runtime_accesses: set[str] = set() - for node in ast.walk(service): - if ( - isinstance(node, ast.Attribute) - and isinstance(node.value, ast.Attribute) - and isinstance(node.value.value, ast.Name) - and node.value.value.id == "self" - and node.value.attr == "runtime" - ): - runtime_accesses.add(node.attr) - if ( - isinstance(node, ast.Call) - and isinstance(node.func, ast.Name) - and node.func.id == "getattr" - and len(node.args) >= 2 - and isinstance(node.args[0], ast.Attribute) - and isinstance(node.args[0].value, ast.Name) - and node.args[0].value.id == "self" - and node.args[0].attr == "runtime" - and isinstance(node.args[1], ast.Constant) - and isinstance(node.args[1].value, str) - ): - runtime_accesses.add(node.args[1].value) - self.assertTrue(runtime_accesses, "service accessed nothing via runtime") - - undeclared = ( - runtime_accesses - port_members - self.PORT_ACCESS_ALLOWLIST - ) - self.assertEqual( - undeclared, - set(), - "runtime accesses missing from BlockFinalizationPort: " - f"{sorted(undeclared)}", - ) - - def test_service_no_longer_forwards_attribute_reads(self) -> None: - self.assertNotIn("__getattr__", vars(BlockFinalizationService)) - self.assertNotIn("__setattr__", vars(BlockFinalizationService)) - - -if __name__ == "__main__": - unittest.main() diff --git a/tests/test_prism_bounded_canonical_finalization.py b/tests/test_prism_bounded_canonical_finalization.py deleted file mode 100644 index 06e3d328e..000000000 --- a/tests/test_prism_bounded_canonical_finalization.py +++ /dev/null @@ -1,1304 +0,0 @@ -#!/usr/bin/env python3 -"""Bounded canonical candidate finalization and audit body publication (#255). - -End to end through the real Rust builder and verifier when the tools are -available: the compiler streams any share sequence into the builder and -hands back a bounded view of the canonical artifact; the audit store -publishes the canonical bundle, the compact body and its share segments -from that artifact through bounded chunks while preserving exact canonical -bytes, digests, signature and coinbase validation, inode identity binding -and idempotent retries; and accepted-block persistence consumes the view. -The fake-builder cases cover the compiler seam without the tools. -""" - -from __future__ import annotations - -from collections.abc import Sequence -import gc -import gzip -import hashlib -import json -import os -from pathlib import Path -import re -import subprocess -import tempfile -import unittest -from unittest import mock -import weakref -from typing import Any - -from lab.prism.audit_artifacts import ( - AuditArtifactConfig, - AuditArtifactStore, - gzip_canonical_bundle_bytes, -) -from lab.prism.audit_bundle_view import ( - SCAN_CHUNK_BYTES, - CanonicalAuditBundleView, - LazyRecordSequence, -) -from lab.prism.bundle_compiler import _iter_build_input_chunks -from lab.prism.prism_tools import prism_tool_command -from lab.prism.share_json_stream import iter_json_object_text_chunks -from lab.prism.share_ledger import PsqlShareLedger -from tests.prism_coordinator_test_support import coordinator as make_coordinator -from tests.prism_vardiff_test_support import fake_audit_bundle_popen - - -BLOCK_HASH = "a1" * 32 -PARENT_HASH = "b2" * 32 -ANCHOR_MS = 1_700_000_000_000 -COINBASE_VALUE_SATS = 50_00000000 -SIGNING_SEED = "42" * 32 -LEDGER_SEED = "43" * 32 -_TOOLS_STATE: dict[str, bool] = {} - - -def _repo_root() -> Path: - return Path(__file__).resolve().parents[1] - - -def real_tools_available() -> bool: - """True when the qbit-prism builder and verifier can run here.""" - if "available" in _TOOLS_STATE: - return _TOOLS_STATE["available"] - if not os.environ.get("PRISM_TOOL_BIN_DIR"): - local = _repo_root() / "target" / "release" - if (local / "qbit-prism-build-audit-bundle").exists(): - os.environ["PRISM_TOOL_BIN_DIR"] = str(local) - available = True - for name in ("qbit-prism-build-audit-bundle", "qbit-prism-audit-verify"): - try: - completed = subprocess.run( - prism_tool_command(name) + ["--help"], - capture_output=True, - timeout=600, - check=False, - ) - except (OSError, subprocess.TimeoutExpired): - available = False - break - if completed.returncode != 0: - available = False - break - _TOOLS_STATE["available"] = available - return available - - -def share_record(index: int, *, oversized_bytes: int = 0) -> dict[str, object]: - record: dict[str, object] = { - "share_seq": index + 1, - "share_id": "Z" * oversized_bytes if oversized_bytes else f"share-{index + 1}", - "miner_id": f"miner-{index % 3}", - "order_key": f"o{index % 3}", - "p2mr_program_hex": ("%02x" % (index % 3)) * 32, - "share_difficulty": 1, - "network_difficulty": 1000, - "template_height": 10, - "job_id": "job-1", - "job_issued_at_ms": ANCHOR_MS - 1000, - "accepted_at_ms": ANCHOR_MS - 500, - "ntime": 1_700_000_000, - "credit_policy": None if index % 2 else "standard", - } - return record - - -def found_block(height: int = 10) -> dict[str, object]: - return { - "block_height": height, - "coinbase_value_sats": COINBASE_VALUE_SATS, - "network_difficulty": 1000, - "anchor_job_issued_at_ms": ANCHOR_MS, - } - - -class GuardedShareSequence(Sequence): - """A replayable share window that refuses to be copied into a list. - - ``list(seq)`` consults ``__len__`` for its length hint; the streaming - encoder only iterates. Indexing is allowed for the segment slices. - """ - - def __init__(self, records: list[dict[str, object]]) -> None: - self._records = records - self.iterations = 0 - - def __iter__(self): # type: ignore[no-untyped-def] - self.iterations += 1 - yield from self._records - - def __len__(self) -> int: - raise AssertionError("share window must not be materialized") - - def __getitem__(self, index: int | slice) -> Any: - return self._records[index] - - -def coordinator_server() -> Any: - result = make_coordinator() - server = result[0] if isinstance(result, tuple) else result - server.signing_seed_hex = SIGNING_SEED - server.ledger_attestation_signing_seed_hex = LEDGER_SEED - return server - - -class FakeLeasePsqlShareLedger(PsqlShareLedger): - """Scripted psql ledger: every statement is recorded, results are queued.""" - - def __init__(self, lease_results: list[dict[str, object]], **kwargs: Any) -> None: - self.lease_results = list(lease_results) - self.lease_queries: list[str] = [] - kwargs.setdefault("native_client_mode", "0") - super().__init__( - psql_command="psql postgresql://example.invalid/qbit", - lease_retry_sleep=lambda _seconds: None, - **kwargs, - ) - - def _make_writer_lease_guard(self, _database_url: str | None) -> Any: - class FakeGuard: - held = True - - def try_acquire(self) -> bool: - return True - - def close(self) -> None: - self.held = False - - return FakeGuard() - - def _run_json(self, sql: str) -> Any: - self.lease_queries.append(sql) - if not self.lease_results: - raise AssertionError("unexpected extra ledger statement") - return self.lease_results.pop(0) - - -def acquired_lease() -> dict[str, object]: - return { - "acquired": True, - "writer_id": "writer-a", - "writer_epoch": 1, - "writer_session_token": "session-a", - } - - -def persist_result() -> dict[str, object]: - return { - "backend": "postgres-psql", - "share_count": 0, - "block_count": 1, - "bundle_count": 1, - "payout_entry_count": 3, - "carry_forward_count": 3, - "onchain_output_count": 0, - } - - -def payload_from_sql(sql: str) -> dict[str, Any]: - match = re.search(r"\$(qbit_prism_json(?:_x)*)\$(.*)\$\1\$::jsonb", sql, re.S) - if match is None: - raise AssertionError("persist statement carries no payload literal") - return json.loads(match.group(2)) - - -class _RealBuildMixin: - def build_candidate( - self, - store: AuditArtifactStore, - server: Any, - shares: Sequence[dict[str, object]], - *, - block_hash: str = BLOCK_HASH, - height: int = 10, - cancellation: Any = None, - ) -> tuple[Any, CanonicalAuditBundleView]: - candidate = store.issue_candidate(block_hash=block_hash) - adopted: list[Path] = [] - - def adopt(path: Path, value: os.stat_result) -> None: - store.adopt_compiler_candidate(candidate, path=path, value=value) - adopted.append(path) - - parent_fd = store.duplicate_root_directory_fd() - try: - view = server.build_audit_bundle( - shares=shares, - found_block=found_block(height), - prior_balances=[], - coinbase_script_sig_suffix_hex="00", - witness_merkle_leaves_hex=["ab" * 32], - canonical_output_path=candidate.path, - canonical_output_parent_fd=parent_fd, - canonical_output_adopter=adopt, - cancellation=cancellation, - ) - except BaseException: - store.discard_candidate(candidate) - raise - finally: - os.close(parent_fd) - assert adopted == [candidate.path] - return candidate, view - - def verify_candidate( - self, - store: AuditArtifactStore, - candidate: Any, - view: CanonicalAuditBundleView, - *, - height: int = 10, - ) -> dict[str, Any]: - verified = store.verify_candidate( - candidate, - coinbase_tx_hex=view["signed_coinbase_manifest"]["manifest"]["coinbase_tx_hex"], - expected_coinbase_value_sats=COINBASE_VALUE_SATS, - expected_block_height=height, - trusted_writer_public_key_hex=AuditArtifactStore.trusted_writer_key( - None, - view, - allow_embedded_test_key=True, - ), - trust_source="embedded_test_only", - ) - store.require_current_verified_candidate(verified, candidate) - assert verified.canonical_copy_eligible - return dict(verified.report) - - -@unittest.skipUnless(real_tools_available(), "qbit-prism builder and verifier are unavailable") -class RealBuilderBoundedFinalizationTests(_RealBuildMixin, unittest.TestCase): - def make_store(self, root: Path, *, share_segment_size: int) -> tuple[AuditArtifactStore, mock.Mock]: - canonicalizer = mock.Mock( - side_effect=AssertionError("bounded publication must not canonicalize a bundle"), - ) - store = AuditArtifactStore( - AuditArtifactConfig( - root=root, - evidence_path=root / "evidence.json", - share_segment_size=share_segment_size, - ), - canonicalizer=canonicalizer, - ) - self.addCleanup(store.close) - return store, canonicalizer - - def test_canonical_build_returns_bounded_view_from_a_streamed_sequence(self) -> None: - with tempfile.TemporaryDirectory() as tmp: - root = Path(tmp) - store, _canonicalizer = self.make_store(root, share_segment_size=100) - server = coordinator_server() - records = [share_record(index) for index in range(1300)] - window = GuardedShareSequence(records) - candidate, view = self.build_candidate(store, server, window) - try: - self.assertIsInstance(view, CanonicalAuditBundleView) - self.assertGreaterEqual(window.iterations, 1) - raw = candidate.path.read_bytes() - self.assertEqual(view.sha256_hex, hashlib.sha256(raw).hexdigest()) - self.assertEqual(view.byte_length, len(raw)) - loaded = json.loads(raw) - self.assertEqual(view, loaded) - self.assertEqual(loaded, view) - self.assertIsInstance(view["shares"], LazyRecordSequence) - self.assertEqual(len(view["shares"]), 1300) - self.assertIsInstance(view["reward_manifest"]["shares"], LazyRecordSequence) - self.assertEqual(view["shares"][0]["share_seq"], 1) - self.assertEqual(view["shares"][-1]["share_seq"], 1300) - # The typed Rust order is preserved; the sorted v1 identity - # encoding is a different contract and is not used here. - self.assertEqual(list(view)[:3], ["schema", "shares", "found_block"]) - report = self.verify_candidate(store, candidate, view) - self.assertEqual(report["audit_bundle_sha256_hex"], view.sha256_hex) - self.assertEqual( - AuditArtifactStore.trusted_writer_key(None, view, allow_embedded_test_key=True), - AuditArtifactStore.trusted_writer_key(None, loaded, allow_embedded_test_key=True), - ) - self.assertEqual( - view["payout_policy_manifest"]["accounts"], - loaded["payout_policy_manifest"]["accounts"], - ) - finally: - view.close() - store.discard_candidate(candidate) - - def test_bounded_publication_preserves_bytes_digests_and_compact_roundtrip(self) -> None: - with tempfile.TemporaryDirectory() as tmp: - root = Path(tmp) - store, canonicalizer = self.make_store(root, share_segment_size=250) - server = coordinator_server() - records = [share_record(index) for index in range(3000)] - candidate, view = self.build_candidate(store, server, GuardedShareSequence(records)) - try: - raw = candidate.path.read_bytes() - self.assertGreater(len(raw), 2 * SCAN_CHUNK_BYTES) - report = self.verify_candidate(store, candidate, view) - expected = report["audit_bundle_sha256_hex"] - body_uri = str(store.body_path(BLOCK_HASH, expected)) - payload = {"block_hash": BLOCK_HASH, "audit_bundle_sha256": expected} - real_loads = json.loads - real_dumps = json.dumps - - def bounded_loads(text: Any, *args: Any, **kwargs: Any) -> Any: - if len(text) > SCAN_CHUNK_BYTES: - raise AssertionError("publication decoded a window-sized document") - return real_loads(text, *args, **kwargs) - - def bounded_dumps(value: Any, *args: Any, **kwargs: Any) -> str: - encoded = real_dumps(value, *args, **kwargs) - if len(encoded) > SCAN_CHUNK_BYTES: - raise AssertionError("publication encoded a window-sized document") - return encoded - - with mock.patch("lab.prism.audit_artifacts.json.loads", side_effect=bounded_loads), mock.patch( - "lab.prism.audit_artifacts.json.dumps", - side_effect=bounded_dumps, - ): - published = store.prepare_external_audit_body( - payload, - view, - body_uri=body_uri, - canonical_bundle_path=candidate.path, - ) - self.assertEqual(published, body_uri) - canonicalizer.assert_not_called() - body_path = Path(body_uri) - body = json.loads(body_path.read_bytes()) - self.assertEqual(body["schema"], "qbit.prism.audit-bundle.v2") - self.assertEqual(body["share_count"], 3000) - parts = body["share_window_proof"]["share_parts"] - self.assertEqual(len(parts), 12) - self.assertTrue(all(part["kind"] == "segment_range" for part in parts)) - # Exact canonical bytes and their reproducible compression. - self.assertEqual(store.read_canonical_audit_bundle(BLOCK_HASH, expected), raw) - self.assertEqual(store.stored_canonical_bundle_sha256(BLOCK_HASH, expected), expected) - gz_path = store.canonical_bundle_path(BLOCK_HASH, expected) - self.assertEqual(gz_path.read_bytes(), gzip_canonical_bundle_bytes(raw)) - self.assertEqual(gzip.decompress(gz_path.read_bytes()), raw) - # The compact body reconstructs to the exact logical bundle and - # the reconstruction's Rust canonical digest is the advertised - # one (the reader canonicalizes; the publication never did). - canonicalizer.side_effect = None - canonicalizer.return_value = None - from lab.prism.bundle_compiler import canonical_bundle_bytes - - canonicalizer.side_effect = canonical_bundle_bytes - reconstructed = store.read_external_body(body_uri, expected_sha256=expected) - self.assertEqual(reconstructed, json.loads(raw)) - self.assertEqual(reconstructed, view) - canonicalizer.side_effect = AssertionError("retry must not canonicalize") - self.assertEqual( - store.audit_body_byte_len(body_uri, view, candidate.path), - body_path.stat().st_size, - ) - metrics = store.metrics_snapshot() - self.assertEqual(metrics["canonical_bundle"]["files"], 1) - self.assertEqual(metrics["body"]["files"], 1) - self.assertEqual(metrics["share_segment"]["files"], 12) - # An exact retry after the body landed is a durable no-op that - # neither rewrites nor reconstructs. - before = { - path: (path.stat().st_ino, path.stat().st_mtime_ns) - for path in root.iterdir() - if path.is_file() - } - with mock.patch.object( - store, - "resolve_audit_bundle_v2", - side_effect=AssertionError("retry must not reconstruct the window"), - ), mock.patch.object( - store, - "read_audit_share_segment", - side_effect=AssertionError("retry must not load a segment whole"), - ): - retried = store.prepare_external_audit_body( - payload, - view, - body_uri=body_uri, - canonical_bundle_path=candidate.path, - ) - self.assertEqual(retried, body_uri) - after = { - path: (path.stat().st_ino, path.stat().st_mtime_ns) - for path in root.iterdir() - if path.is_file() - } - self.assertEqual(before, after) - # The candidate artifact itself is a dot-prefixed .tmp name; - # no writer temporaries may remain beside it. - self.assertEqual( - [path for path in root.glob(".*.tmp") if "candidate" not in path.name], - [], - ) - finally: - view.close() - store.discard_candidate(candidate) - - def test_view_without_a_canonical_path_publishes_from_its_own_source(self) -> None: - with tempfile.TemporaryDirectory() as tmp: - root = Path(tmp) - store, canonicalizer = self.make_store(root, share_segment_size=0) - server = coordinator_server() - candidate, view = self.build_candidate(store, server, [share_record(i) for i in range(7)]) - try: - raw = candidate.path.read_bytes() - expected = view.sha256_hex - body_uri = str(store.body_path(BLOCK_HASH, expected)) - published = store.prepare_external_audit_body( - {"block_hash": BLOCK_HASH, "audit_bundle_sha256": expected}, - view, - body_uri=body_uri, - ) - self.assertEqual(published, body_uri) - # No segmenting configured: the body is the literal canonical - # bundle, copied from the artifact source. - self.assertEqual(Path(body_uri).read_bytes(), raw) - self.assertEqual(store.read_canonical_audit_bundle(BLOCK_HASH, expected), raw) - canonicalizer.assert_not_called() - self.assertEqual(store.audit_body_byte_len(None, view), len(raw)) - finally: - view.close() - store.discard_candidate(candidate) - - def test_tampered_or_foreign_artifacts_are_rejected(self) -> None: - with tempfile.TemporaryDirectory() as tmp: - root = Path(tmp) - store, _canonicalizer = self.make_store(root, share_segment_size=4) - server = coordinator_server() - candidate, view = self.build_candidate(store, server, [share_record(i) for i in range(9)]) - other, other_view = self.build_candidate( - store, - server, - [share_record(i) for i in range(10)], - block_hash=PARENT_HASH, - ) - try: - raw = candidate.path.read_bytes() - expected = view.sha256_hex - payload = {"block_hash": BLOCK_HASH, "audit_bundle_sha256": expected} - body_uri = str(store.body_path(BLOCK_HASH, expected)) - # A view over a different artifact than the canonical path. - with self.assertRaisesRegex(RuntimeError, "does not match logical bundle"): - store.prepare_external_audit_body( - payload, - other_view, - body_uri=body_uri, - canonical_bundle_path=candidate.path, - ) - # A dictionary that differs in one record. - tampered = json.loads(raw) - tampered["shares"][4]["share_difficulty"] = 2 - with self.assertRaisesRegex(RuntimeError, "does not match logical bundle"): - store.validate_canonical_source(candidate.path, expected, tampered) - store.validate_canonical_source(candidate.path, expected, json.loads(raw)) - store.validate_canonical_source(candidate.path, expected, view) - # The artifact rewritten under the view after the scan. - candidate.path.write_bytes(raw + b" ") - with self.assertRaisesRegex(RuntimeError, "sha256 mismatch"): - store.prepare_external_audit_body( - payload, - view, - body_uri=body_uri, - canonical_bundle_path=candidate.path, - ) - with self.assertRaises(Exception): - store.prepare_external_audit_body(payload, view, body_uri=body_uri) - self.assertEqual(list(root.glob("prism-audit-bundle-body-*")), []) - finally: - view.close() - other_view.close() - store.discard_candidate(candidate) - store.discard_candidate(other) - - def test_oversized_single_field_publishes_through_bounded_writers(self) -> None: - oversized = 2 * SCAN_CHUNK_BYTES + 3 - with tempfile.TemporaryDirectory() as tmp: - root = Path(tmp) - store, canonicalizer = self.make_store(root, share_segment_size=3) - server = coordinator_server() - records = [ - share_record(index, oversized_bytes=oversized if index == 4 else 0) - for index in range(11) - ] - candidate, view = self.build_candidate(store, server, records) - try: - raw = candidate.path.read_bytes() - self.assertGreater(len(raw), 2 * oversized) - report = self.verify_candidate(store, candidate, view) - expected = report["audit_bundle_sha256_hex"] - body_uri = str(store.body_path(BLOCK_HASH, expected)) - published = store.prepare_external_audit_body( - {"block_hash": BLOCK_HASH, "audit_bundle_sha256": expected}, - view, - body_uri=body_uri, - canonical_bundle_path=candidate.path, - ) - self.assertEqual(published, body_uri) - canonicalizer.assert_not_called() - self.assertEqual(store.read_canonical_audit_bundle(BLOCK_HASH, expected), raw) - from lab.prism.bundle_compiler import canonical_bundle_bytes - - canonicalizer.side_effect = canonical_bundle_bytes - self.assertEqual( - store.read_external_body(body_uri, expected_sha256=expected), - json.loads(raw), - ) - segments = sorted(root.glob("prism-audit-share-segment-slot-*.json")) - self.assertEqual(len(segments), 4) - self.assertTrue(any(path.stat().st_size > oversized for path in segments)) - finally: - view.close() - store.discard_candidate(candidate) - - def test_multiple_generations_publish_and_release_independently(self) -> None: - gc.disable() - try: - with tempfile.TemporaryDirectory() as tmp: - root = Path(tmp) - store, _canonicalizer = self.make_store(root, share_segment_size=20) - server = coordinator_server() - digests: list[str] = [] - refs: list[weakref.ref] = [] - for generation in range(3): - block_hash = ("%02x" % (0xC0 + generation)) * 32 - records = [share_record(index) for index in range(40 + generation)] - candidate, view = self.build_candidate( - store, - server, - GuardedShareSequence(records), - block_hash=block_hash, - height=10 + generation, - ) - try: - report = self.verify_candidate(store, candidate, view, height=10 + generation) - expected = report["audit_bundle_sha256_hex"] - body_uri = str(store.body_path(block_hash, expected)) - self.assertEqual( - store.prepare_external_audit_body( - {"block_hash": block_hash, "audit_bundle_sha256": expected}, - view, - body_uri=body_uri, - canonical_bundle_path=candidate.path, - ), - body_uri, - ) - digests.append(expected) - refs.append(weakref.ref(view)) - source_fd = view.source.fileno() - finally: - view.close() - store.discard_candidate(candidate) - with self.assertRaises(OSError): - os.fstat(source_fd) - del view - self.assertIsNone(refs[-1]()) - self.assertEqual(len(set(digests)), 3) - metrics = store.metrics_snapshot() - self.assertEqual(metrics["body"]["files"], 3) - self.assertEqual(metrics["canonical_bundle"]["files"], 3) - self.assertEqual(metrics["candidate"]["files"], 0) - finally: - gc.enable() - - def test_cancellation_during_output_scan_cleans_the_candidate(self) -> None: - with tempfile.TemporaryDirectory() as tmp: - root = Path(tmp) - store, _canonicalizer = self.make_store(root, share_segment_size=0) - server = coordinator_server() - - class Cancelled(RuntimeError): - pass - - class Cancellation: - def __init__(self) -> None: - self.scan_checks = 0 - - def is_set(self) -> bool: - return False - - def raise_if_cancelled(self, phase: str) -> None: - if phase == "builder output scan": - self.scan_checks += 1 - if self.scan_checks == 2: - raise Cancelled(phase) - - cancellation = Cancellation() - records = [share_record(index) for index in range(1500)] - with self.assertRaises(Cancelled): - self.build_candidate(store, server, records, cancellation=cancellation) - self.assertEqual(cancellation.scan_checks, 2) - self.assertEqual(list(root.glob(".prism-live-audit-bundle-candidate-*")), []) - self.assertEqual(store.metrics_snapshot()["candidate"]["files"], 0) - - def test_accepted_block_persistence_consumes_the_view(self) -> None: - with tempfile.TemporaryDirectory() as tmp: - root = Path(tmp) / "audit" - root.mkdir() - canonicalizer = mock.Mock(side_effect=AssertionError("persistence must not canonicalize")) - ledger = FakeLeasePsqlShareLedger( - [ - acquired_lease(), - {"existing_block": False, "existing_body_uri": None}, - persist_result(), - ], - audit_body_dir=root, - audit_bundle_canonicalizer=canonicalizer, - audit_share_segment_size=50, - ) - store = ledger._audit_artifact_store - assert store is not None - server = coordinator_server() - candidate, view = self.build_candidate(store, server, GuardedShareSequence( - [share_record(index) for index in range(120)] - )) - try: - raw = candidate.path.read_bytes() - report = self.verify_candidate(store, candidate, view) - persistence = ledger.persist_accepted_block( - block_hash=BLOCK_HASH, - block_height=10, - parent_hash=PARENT_HASH, - final_bundle=view, - audit_report=report, - canonical_bundle_path=candidate.path, - ) - canonicalizer.assert_not_called() - body_uri = str(store.body_path(BLOCK_HASH, report["audit_bundle_sha256_hex"])) - self.assertEqual(persistence["body_uri"], body_uri) - self.assertEqual(persistence["audit_body_byte_len"], Path(body_uri).stat().st_size) - payload = payload_from_sql(ledger.lease_queries[-1]) - loaded = json.loads(raw) - self.assertIsNone(payload["audit_bundle"]) - self.assertEqual(payload["body_uri"], body_uri) - self.assertEqual(payload["accounts"], loaded["payout_policy_manifest"]["accounts"]) - self.assertEqual(payload["witness_merkle_leaves_hex"], loaded["witness_merkle_leaves_hex"]) - self.assertEqual( - payload["audit_commitment_leaves_hex"], - loaded["audit_commitment_leaves_hex"], - ) - self.assertEqual(payload["found_block_network_difficulty"], 1000) - self.assertEqual(payload["schema_version"], loaded["schema"]) - self.assertEqual(store.read_canonical_audit_bundle(BLOCK_HASH, view.sha256_hex), raw) - finally: - view.close() - store.discard_candidate(candidate) - - def test_inline_persistence_lane_splices_canonical_bytes(self) -> None: - with tempfile.TemporaryDirectory() as tmp: - root = Path(tmp) - store, _canonicalizer = self.make_store(root, share_segment_size=0) - server = coordinator_server() - candidate, view = self.build_candidate(store, server, [share_record(i) for i in range(5)]) - try: - raw = candidate.path.read_bytes() - report = self.verify_candidate(store, candidate, view) - ledger = FakeLeasePsqlShareLedger([acquired_lease(), persist_result()]) - persistence = ledger.persist_accepted_block( - block_hash=BLOCK_HASH, - block_height=10, - parent_hash=PARENT_HASH, - final_bundle=view, - audit_report=report, - ) - self.assertEqual(persistence["body_uri"], "") - self.assertEqual(persistence["audit_body_byte_len"], len(raw)) - payload = payload_from_sql(ledger.lease_queries[-1]) - self.assertEqual(payload["audit_bundle"], json.loads(raw)) - self.assertIsNone(payload["body_uri"]) - self.assertIn( - json.dumps(json.loads(raw), separators=(",", ":"))[:64], - ledger.lease_queries[-1], - ) - finally: - view.close() - store.discard_candidate(candidate) - - -def ctv_manifest_set() -> dict[str, Any]: - """A two-chunk CTV fanout manifest set shaped for the recovery payload.""" - - def manifest(index: int, output_sum: int, covenant: int) -> dict[str, Any]: - return { - "schema": "qbit.prism.ctv-fanout-manifest.v1", - "precommitment": { - "chunk_index": index, - "chunk_count": 2, - "fanout_fee_sats": 0, - "anchor_vout": 1, - "ctv_hash_hex": ("e%x" % index) * 32, - "fanout_tx_template_hex": ("0a%02x" % index) * 40, - "fanout_output_sum_sats": output_sum, - "block_height": 10, - }, - "precommitment_sha256_hex": ("f%x" % index) * 32, - "commitment_witness_leaf_hex": ("a%x" % index) * 32, - "parent_coinbase_txid": "cc" * 32, - "parent_coinbase_tx_hex": "0102", - "parent_coinbase_vout": 1, - "covenant_output_value_sats": covenant, - "fanout_tx_hex": ("0b%02x" % index) * 60, - "fanout_txid": ("d%x" % index) * 32, - } - - return { - "schema": "qbit.prism.ctv-fanout-manifest-set.v1", - "block_height": 10, - "settlement_mode": "ctv_fanout", - "parent_coinbase_txid": "cc" * 32, - "fanout_count": 2, - "fanout_output_sum_sats": 30, - "covenant_output_value_sats": 40, - "manifests": [manifest(0, 10, 15), manifest(1, 20, 25)], - } - - -def synthetic_bundle(share_count: int, *, recipients: int = 3) -> dict[str, Any]: - """A canonical-shaped bundle with recipient-scaled members for the store tests.""" - shares = [share_record(index) for index in range(share_count)] - accounts = [ - { - "recipient_id": f"miner-{index}", - "order_key": f"o{index}", - "p2mr_program_hex": ("%02x" % (index % 251)) * 32, - "gross_amount_sats": 1000 + index, - "prior_balance_sats": 0, - "candidate_balance_sats": 1000 + index, - "onchain_amount_sats": 0, - "carry_forward_balance_sats": 1000 + index, - "action": "accrued", - } - for index in range(recipients) - ] - return { - "schema": "qbit.prism.audit-bundle.v1", - "shares": shares, - "found_block": found_block(), - "prior_balances": [ - {"recipient_id": f"miner-{index}", "order_key": f"o{index}", "p2mr_program_hex": ("%02x" % (index % 251)) * 32, "balance_sats": index} - for index in range(recipients) - ], - "payout_policy": {"p2mr_spend_input_bytes": 1}, - "witness_merkle_leaves_hex": ["ab" * 32], - "audit_commitment_leaves_hex": ["cd" * 32], - "ledger_window_attestation": {"signature": {"public_key_hex": "44" * 32}}, - "reward_manifest": { - "schema": "qbit.prism.reward-manifest.v1", - "included_share_count": share_count, - "shares": [{"share_seq": index + 1, "counted_difficulty": 1} for index in range(share_count)], - "entitlements": [{"recipient_id": f"miner-{index}", "weight": 1} for index in range(recipients)], - }, - "payout_policy_manifest": {"accounts": accounts, "onchain_entitlements": []}, - "ctv_fanout_manifest_set": ctv_manifest_set(), - "signed_coinbase_manifest": {"manifest": {"coinbase_tx_hex": "00", "payout_count": 0}}, - } - - -def bounded_json_guards(module_path: str, limit: int, *, guard_loads: bool = True) -> Any: - """Refuse json.dumps/json.loads calls above ``limit`` while active. - - The patch lands on the shared ``json`` module, so it covers every - caller in the process; tests whose fakes legitimately decode a whole - captured payload pass ``guard_loads=False``. - """ - real_dumps = json.dumps - real_loads = json.loads - - def guarded_dumps(value: Any, *args: Any, **kwargs: Any) -> str: - encoded = real_dumps(value, *args, **kwargs) - if len(encoded) > limit: - raise AssertionError(f"{module_path} encoded {len(encoded)} bytes in one call") - return encoded - - def guarded_loads(text: Any, *args: Any, **kwargs: Any) -> Any: - if len(text) > limit: - raise AssertionError(f"{module_path} decoded {len(text)} bytes in one call") - return real_loads(text, *args, **kwargs) - - class Guards: - def __enter__(self) -> "Guards": - self._patches = [ - mock.patch(f"{module_path}.json.dumps", side_effect=guarded_dumps), - ] - if guard_loads: - self._patches.append( - mock.patch(f"{module_path}.json.loads", side_effect=guarded_loads) - ) - for patch in self._patches: - patch.start() - return self - - def __exit__(self, *exc: object) -> None: - for patch in reversed(self._patches): - patch.stop() - - return Guards() - - -class RemainingBoundaryTests(unittest.TestCase): - """The follow-up closures: parts index, pressure, persistence and CTV.""" - - def make_store(self, root: Path, *, share_segment_size: int) -> AuditArtifactStore: - store = AuditArtifactStore( - AuditArtifactConfig( - root=root, - evidence_path=root / "evidence.json", - share_segment_size=share_segment_size, - ), - canonicalizer=lambda bundle: json.dumps(bundle, separators=(",", ":")).encode(), - ) - self.addCleanup(store.close) - return store - - def write_view(self, root: Path, bundle: dict[str, Any]) -> tuple[Path, bytes, CanonicalAuditBundleView]: - raw = json.dumps(bundle, separators=(",", ":")).encode() - path = root / f".prism-live-audit-bundle-candidate-{BLOCK_HASH}-{'0' * 32}.json.tmp" - path.write_bytes(raw) - view = CanonicalAuditBundleView.scan_path(path) - self.addCleanup(view.close) - return path, raw, view - - def test_small_segment_size_keeps_the_part_index_on_disk(self) -> None: - from lab.prism.audit_bundle_view import LazyRecordSequence as Lazy - - # segment_size=1 is the stress (one part per share); 250 the control. - for segment_size, expected_parts in ((1, 1500), (7, 215), (250, 6)): - with self.subTest(segment_size=segment_size), tempfile.TemporaryDirectory() as tmp: - root = Path(tmp) - store = self.make_store(root, share_segment_size=segment_size) - bundle = synthetic_bundle(1500) - path, raw, view = self.write_view(root, bundle) - expected = hashlib.sha256(raw).hexdigest() - body_uri = str(store.body_path(BLOCK_HASH, expected)) - with bounded_json_guards("lab.prism.audit_artifacts", 256 * 1024): - scratch: list[Any] = [] - parts = store.audit_share_range_parts(view["shares"], scratch=scratch) - assert parts is not None - self.assertIsInstance(parts, Lazy) - self.assertEqual(len(parts), expected_parts) - self.assertEqual(parts[0]["first_share_seq"], 1) - self.assertEqual(parts[-1]["last_share_seq"], 1500) - for scratch_view in scratch: - scratch_view.close() - published = store.prepare_external_audit_body( - {"block_hash": BLOCK_HASH, "audit_bundle_sha256": expected}, - view, - body_uri=body_uri, - canonical_bundle_path=path, - ) - self.assertEqual(published, body_uri) - body = json.loads(Path(body_uri).read_bytes()) - proof = body["share_window_proof"] - self.assertEqual(len(proof["share_parts"]), expected_parts) - # The incremental parts digest equals the reader's whole - # encoding, and the compact body reconstructs exactly. - self.assertEqual( - proof["share_parts_digest_hex"], - hashlib.sha256(store.storage_json_bytes({"share_parts": proof["share_parts"]})).hexdigest(), - ) - self.assertEqual(store.read_external_body(body_uri, expected_sha256=expected), bundle) - # A retry verifies through the lazily parsed part index. - with bounded_json_guards("lab.prism.audit_artifacts", 256 * 1024): - self.assertEqual( - store.prepare_external_audit_body( - {"block_hash": BLOCK_HASH, "audit_bundle_sha256": expected}, - view, - body_uri=body_uri, - canonical_bundle_path=path, - ), - body_uri, - ) - - def test_owned_open_failures_retire_descriptors_before_the_exception_returns(self) -> None: - with tempfile.TemporaryDirectory() as tmp: - root = Path(tmp) - store = self.make_store(root, share_segment_size=4) - bundle = synthetic_bundle(6) - path, _raw, _view = self.write_view(root, bundle) - opened: list[int] = [] - real_open = os.open - - def recording_open(*args: Any, **kwargs: Any) -> int: - fd = real_open(*args, **kwargs) - if kwargs.get("dir_fd") == store._root_fd and args[0] == path.name: - opened.append(fd) - return fd - - gc_was_enabled = gc.isenabled() - gc.disable() - try: - for opener in ( - lambda: store._open_owned_artifact_source(path), - lambda: store._scan_owned_artifact(path, lazy_paths=(("shares",),)), - ): - opened.clear() - checks = {"count": 0} - real_validate = store._validate_owned_parent - - def failing_second_check(target: Path) -> None: - checks["count"] += 1 - if checks["count"] == 2: - raise RuntimeError("audit artifact root identity changed") - real_validate(target) - - retained: BaseException | None = None - with mock.patch("lab.prism.audit_artifacts.os.open", side_effect=recording_open), mock.patch.object( - store, - "_validate_owned_parent", - side_effect=failing_second_check, - ): - try: - opener() - except RuntimeError as exc: - retained = exc - # The exception and its traceback frames (which reference - # the view or source locals) are still retained here; the - # descriptors must already be closed regardless. - assert retained is not None - self.assertIn("identity changed", str(retained)) - self.assertIsNotNone(retained.__traceback__) - self.assertEqual(len(opened), 1) - for fd in opened: - with self.assertRaises(OSError): - os.fstat(fd) - del retained - finally: - if gc_was_enabled: - gc.enable() - - def test_scratch_pressure_is_reported_not_classified_as_mismatch(self) -> None: - from lab.prism.audit_bundle_view import ArtifactResourcePressure - - with tempfile.TemporaryDirectory() as tmp: - root = Path(tmp) - store = self.make_store(root, share_segment_size=100) - bundle = synthetic_bundle(500) - path, raw, view = self.write_view(root, bundle) - expected = hashlib.sha256(raw).hexdigest() - body_uri = str(store.body_path(BLOCK_HASH, expected)) - real_temporary_file = tempfile.TemporaryFile - calls = {"count": 0} - - def failing_after_first(*args: Any, **kwargs: Any) -> Any: - calls["count"] += 1 - if calls["count"] > 1: - raise OSError("no scratch space") - return real_temporary_file(*args, **kwargs) - - with mock.patch("lab.prism.audit_bundle_view.tempfile.TemporaryFile", side_effect=failing_after_first): - with self.assertRaises(ArtifactResourcePressure): - store.prepare_external_audit_body( - {"block_hash": BLOCK_HASH, "audit_bundle_sha256": expected}, - view, - body_uri=body_uri, - canonical_bundle_path=path, - ) - self.assertFalse(Path(body_uri).exists()) - # Once the pressure passes the same call publishes normally. - self.assertEqual( - store.prepare_external_audit_body( - {"block_hash": BLOCK_HASH, "audit_bundle_sha256": expected}, - view, - body_uri=body_uri, - canonical_bundle_path=path, - ), - body_uri, - ) - - def test_persistence_streams_lazy_accounts_and_the_inline_artifact(self) -> None: - with tempfile.TemporaryDirectory() as tmp: - root = Path(tmp) - bundle = synthetic_bundle(1200, recipients=900) - path, raw, view = self.write_view(root, bundle) - expected = hashlib.sha256(raw).hexdigest() - report = { - "coinbase_txid": "11" * 32, - "coinbase_manifest_sha256_hex": "22" * 32, - "audit_bundle_sha256_hex": expected, - "coinbase_tx_hex": "00", - } - # Inline lane: no body store; the artifact bytes are spliced. - inline = FakeLeasePsqlShareLedger([acquired_lease(), persist_result()]) - with bounded_json_guards("lab.prism.share_ledger", 256 * 1024): - persistence = inline.persist_accepted_block( - block_hash=BLOCK_HASH, - block_height=10, - parent_hash=PARENT_HASH, - final_bundle=view, - audit_report=report, - ) - self.assertEqual(persistence["audit_body_byte_len"], len(raw)) - sql = inline.lease_queries[-1] - self.assertEqual(sql.count(raw.decode("utf-8")), 1) - payload = payload_from_sql(sql) - self.assertEqual(payload["audit_bundle"], bundle) - self.assertEqual(payload["accounts"], bundle["payout_policy_manifest"]["accounts"]) - self.assertEqual(payload["witness_merkle_leaves_hex"], bundle["witness_merkle_leaves_hex"]) - # External lane: the body pointer replaces the inline body and the - # recipient-scaled accounts stream from the view. - external = FakeLeasePsqlShareLedger( - [acquired_lease(), {"existing_block": False, "existing_body_uri": None}, persist_result()], - audit_body_dir=root, - audit_bundle_canonicalizer=mock.Mock(side_effect=AssertionError("must not canonicalize")), - audit_share_segment_size=100, - ) - with bounded_json_guards("lab.prism.share_ledger", 256 * 1024): - persistence = external.persist_accepted_block( - block_hash=BLOCK_HASH, - block_height=10, - parent_hash=PARENT_HASH, - final_bundle=view, - audit_report=report, - canonical_bundle_path=path, - ) - payload = payload_from_sql(external.lease_queries[-1]) - self.assertIsNone(payload["audit_bundle"]) - self.assertEqual(payload["body_uri"], persistence["body_uri"]) - self.assertEqual(payload["accounts"], bundle["payout_policy_manifest"]["accounts"]) - - def test_ctv_manifest_set_persists_and_digests_through_the_view(self) -> None: - from lab.prism.audit_bundle_view import LazyRecordSequence as Lazy, streamed_sha256_json_hex - from lab.prism.share_ledger import ctv_fanout_recovery_payload, sha256_json_hex - - with tempfile.TemporaryDirectory() as tmp: - root = Path(tmp) - bundle = synthetic_bundle(20) - _path, _raw, view = self.write_view(root, bundle) - manifest_set = view["ctv_fanout_manifest_set"] - self.assertIsInstance(manifest_set["manifests"], Lazy) - digest = streamed_sha256_json_hex(manifest_set) - self.assertEqual(digest, sha256_json_hex(bundle["ctv_fanout_manifest_set"])) - reference = ctv_fanout_recovery_payload( - block_hash=BLOCK_HASH, - manifest_set=bundle["ctv_fanout_manifest_set"], - manifest_set_sha256=digest, - ) - ledger = FakeLeasePsqlShareLedger( - [acquired_lease(), {"backend": "postgres-psql", "fanout_set_count": 1, "fanout_artifact_count": 2}], - ) - with bounded_json_guards("lab.prism.share_ledger", 256 * 1024): - ledger.persist_ctv_fanout_manifest_set( - block_hash=BLOCK_HASH, - manifest_set=manifest_set, - manifest_set_sha256=digest, - ) - payload = payload_from_sql(ledger.lease_queries[-1]) - for key in ("writer_id", "writer_epoch", "writer_session_token"): - payload.pop(key) - self.assertEqual(payload, reference) - # The in-memory ledger keeps plain state from the same payload. - from lab.prism.share_ledger import SingleWriterShareLedger - - memory = SingleWriterShareLedger() - memory.persist_ctv_fanout_manifest_set( - block_hash=BLOCK_HASH, - manifest_set=manifest_set, - manifest_set_sha256=digest, - ) - self.assertEqual(memory._ctv_fanout_sets[BLOCK_HASH], reference) - - def test_finalization_passes_lazy_members_to_its_consumers(self) -> None: - from types import SimpleNamespace - - from lab.prism.audit_bundle_view import streamed_sha256_json_hex - from lab.prism.block_finalization import BlockFinalizationService, FinalizationAdmission, LandedCandidate - from lab.prism.share_ledger import sha256_json_hex - - with tempfile.TemporaryDirectory() as tmp: - root = Path(tmp) - bundle = synthetic_bundle(20) - _path, _raw, view = self.write_view(root, bundle) - recorded: dict[str, Any] = {} - - def persist(**kwargs: Any) -> dict[str, Any]: - recorded.update(kwargs) - return {"backend": "memory"} - - runtime = SimpleNamespace(ledger=SimpleNamespace(persist_ctv_fanout_manifest_set=persist)) - service = BlockFinalizationService(runtime) # type: ignore[arg-type] - service._record_block_candidate_progress = lambda *args, **kwargs: None # type: ignore[method-assign] - admission = SimpleNamespace( - candidate=SimpleNamespace(credit_share_on_accept=False), - block_hash=BLOCK_HASH, - context=None, - submission=None, - ) - landed = LandedCandidate( - final_bundle=view, - report={}, - persistence={}, - confirmation={}, - audit_publication_identity=None, # type: ignore[arg-type] - audit_verification_identity={}, - ) - result = service._persist_ctv_and_credit(admission, landed) # type: ignore[arg-type] - self.assertEqual(result, {"backend": "memory"}) - self.assertIs(recorded["manifest_set"], view["ctv_fanout_manifest_set"]) - self.assertEqual(recorded["manifest_set_sha256"], sha256_json_hex(bundle["ctv_fanout_manifest_set"])) - self.assertEqual(recorded["manifest_set_sha256"], streamed_sha256_json_hex(view["ctv_fanout_manifest_set"])) - del FinalizationAdmission - - def test_payout_preview_walks_lazy_accounts(self) -> None: - with tempfile.TemporaryDirectory() as tmp: - root = Path(tmp) - bundle = synthetic_bundle(20, recipients=40) - _path, _raw, view = self.write_view(root, bundle) - server = coordinator_server() - self.assertEqual( - server._accepted_block_payout_preview_from_bundle(view, prior_balances=[]), - server._accepted_block_payout_preview_from_bundle(bundle, prior_balances=[]), - ) - self.assertEqual( - len(server._accepted_block_payout_preview_from_bundle(view, prior_balances=[])), - 40, - ) - - -class CompilerInputStreamingTests(unittest.TestCase): - def test_build_input_chunks_match_json_dumps_for_lists_and_sequences(self) -> None: - records = [share_record(index) for index in range(700)] - payload = { - "found_block": found_block(), - "prior_balances": [], - "payout_policy": {"p2mr_spend_input_bytes": 1}, - "coinbase_script_sig_suffix_hex": "00", - "witness_merkle_leaves_hex": ["ab" * 32], - "shares": records, - "ctv_settlement": None, - } - expected = json.dumps(payload, separators=(",", ":")) - self.assertEqual( - "".join(iter_json_object_text_chunks(payload, array_keys=("shares",))), - expected, - ) - for shares in (records, tuple(records), GuardedShareSequence(records)): - with self.subTest(kind=type(shares).__name__): - chunks = list( - _iter_build_input_chunks( - {**payload, "shares": shares}, - array_keys=("shares", "compact_share_identities", "compact_shares"), - batch_records=64, - chunk_chars=4096, - ) - ) - self.assertEqual("".join(chunks), expected) - self.assertTrue(all(chunk.isascii() for chunk in chunks)) - self.assertGreater(len(chunks), 1) - - def test_compiler_streams_a_sequence_without_copying_it(self) -> None: - server = coordinator_server() - captured: dict[str, object] = {} - records = [share_record(index) for index in range(300)] - window = GuardedShareSequence(records) - canonical = json.dumps({"ok": True, "shares": records[:2]}, separators=(",", ":")) - with tempfile.TemporaryDirectory() as tmp, mock.patch( - "lab.prism.bundle_compiler.subprocess.Popen", - fake_audit_bundle_popen(captured, output_text=canonical), - ): - output_path = Path(tmp) / "candidate.audit.json" - bundle = server.build_audit_bundle( - shares=window, - found_block=found_block(), - prior_balances=[], - coinbase_script_sig_suffix_hex="00", - canonical_output_path=output_path, - ) - try: - self.assertIsInstance(bundle, CanonicalAuditBundleView) - self.assertEqual(captured["payload"]["shares"], records) - self.assertEqual(window.iterations, 1) - self.assertEqual(bundle, json.loads(canonical)) - self.assertIsInstance(bundle["shares"], LazyRecordSequence) - self.assertEqual(output_path.read_bytes(), canonical.encode()) - finally: - bundle.close() - - def test_compiler_splices_pre_encoded_windows_without_parsing(self) -> None: - from lab.prism.bundle_compiler import _iter_share_window_items - from lab.prism.share_json_stream import canonical_share_items_bytes - - records = [share_record(index) for index in range(1500)] - items = canonical_share_items_bytes(records) - - class MirrorLike(Sequence): - def __init__(self) -> None: - self.canonical_items = items - self.record_count = len(records) - - def __len__(self) -> int: - return self.record_count - - def __iter__(self): # type: ignore[no-untyped-def] - raise AssertionError("a pre-encoded window must not be parsed") - - def __getitem__(self, index: int | slice) -> Any: - raise AssertionError("a pre-encoded window must not be parsed") - - class PageLike: - def __init__(self, records_slice: list[dict[str, object]]) -> None: - self.canonical_json_items = canonical_share_items_bytes(records_slice) - - class PagedLike(Sequence): - def __init__(self) -> None: - self.pages = (PageLike(records[:700]), PageLike([]), PageLike(records[700:])) - - def __len__(self) -> int: - return len(records) - - def __iter__(self): # type: ignore[no-untyped-def] - raise AssertionError("a page-backed window must not be parsed") - - def __getitem__(self, index: int | slice) -> Any: - raise AssertionError("a page-backed window must not be parsed") - - for window in (MirrorLike(), PagedLike()): - with self.subTest(kind=type(window).__name__): - spliced = b"".join( - piece if isinstance(piece, bytes) else piece.encode() - for piece in _iter_share_window_items(window, batch_records=64, chunk_chars=4096) - ) - self.assertEqual(spliced, items) - self.assertEqual(json.loads(b"[" + spliced + b"]"), json.loads(b"[" + items + b"]")) - # Through the compiler: the builder receives those exact records and - # the recipient-scaled balances stream in batches. - server = coordinator_server() - captured: dict[str, object] = {} - balances = [ - {"recipient_id": f"m{index}", "order_key": f"o{index}", "p2mr_program_hex": "aa" * 32, "balance_sats": index} - for index in range(5000) - ] - with tempfile.TemporaryDirectory() as tmp, mock.patch( - "lab.prism.bundle_compiler.subprocess.Popen", - fake_audit_bundle_popen(captured, output_text='{"ok":true}'), - ), bounded_json_guards("lab.prism.bundle_compiler", 256 * 1024, guard_loads=False): - bundle = server.build_audit_bundle( - shares=MirrorLike(), - found_block=found_block(), - prior_balances=balances, - coinbase_script_sig_suffix_hex="00", - witness_merkle_leaves_hex=["ab" * 32] * 3000, - canonical_output_path=Path(tmp) / "candidate.audit.json", - ) - try: - self.assertEqual(captured["payload"]["shares"], json.loads(b"[" + items + b"]")) - self.assertEqual(captured["payload"]["prior_balances"], balances) - self.assertEqual(len(captured["payload"]["witness_merkle_leaves_hex"]), 3000) - finally: - bundle.close() - - def test_compiler_rejects_non_sequence_windows_before_writing(self) -> None: - server = coordinator_server() - with tempfile.TemporaryDirectory() as tmp: - with self.assertRaises(TypeError): - server.build_audit_bundle( - shares="not-a-window", # type: ignore[arg-type] - found_block=found_block(), - prior_balances=[], - coinbase_script_sig_suffix_hex="00", - canonical_output_path=Path(tmp) / "candidate.audit.json", - ) - - -if __name__ == "__main__": - unittest.main() diff --git a/tests/test_prism_bounded_executor.py b/tests/test_prism_bounded_executor.py deleted file mode 100644 index 6dc5c9ccf..000000000 --- a/tests/test_prism_bounded_executor.py +++ /dev/null @@ -1,181 +0,0 @@ -#!/usr/bin/env python3 -"""Focused tests for PRISM's bounded priority executor.""" - -from __future__ import annotations - -import gc -import queue -import threading -import unittest -import weakref - -from lab.prism import prism_coordinator -from lab.prism.bounded_executor import ( - _BoundedPriorityExecutor, - _DeliveryQueueFull, -) - - -class _Payload: - pass - - -class _Task: - def __init__(self, result: _Payload, *, fail: bool) -> None: - self.result = result - self.fail = fail - - def __call__(self, argument: _Payload, *, keyword: _Payload) -> _Payload: - if self.fail: - raise RuntimeError("injected task failure") - return self.result - - -class BoundedExecutorRetentionTests(unittest.TestCase): - def check_idle_worker_releases_task(self, outcome: str) -> None: - # Queue the task before starting a worker, so cancellation and the - # return to an idle get() are deterministic and require no sleeps. - waiting = threading.Semaphore(0) - - class ObservedQueue(queue.PriorityQueue): - def get(self, *args, **kwargs): - waiting.release() - return super().get(*args, **kwargs) - - gc.collect() - was_enabled = gc.isenabled() - gc.disable() - executor = _BoundedPriorityExecutor(max_workers=0, max_queue_size=1) - executor._queue = ObservedQueue(maxsize=1) - try: - argument, keyword, result = _Payload(), _Payload(), _Payload() - task = _Task(result, fail=outcome == "failed") - future = executor.submit(task, argument, keyword=keyword) - references = { - "argument": weakref.ref(argument), - "keyword": weakref.ref(keyword), - "result": weakref.ref(result), - "task": weakref.ref(task), - "future": weakref.ref(future), - } - if outcome == "cancelled": - self.assertTrue(future.cancel()) - del argument, keyword, result, task, future - thread = threading.Thread(target=executor._worker, daemon=True) - executor._threads.append(thread) - thread.start() - self.assertTrue(waiting.acquire(timeout=5)) - self.assertTrue(waiting.acquire(timeout=5)) - self.assertEqual(executor.stats(), (0, 0)) - self.assertEqual(executor._queue.unfinished_tasks, 0) - self.assertTrue(thread.is_alive()) - self.assertEqual( - {name: ref() is not None for name, ref in references.items()}, - dict.fromkeys(references, False), - ) - finally: - executor.shutdown(wait=True, cancel_futures=True) - if was_enabled: - gc.enable() - gc.collect() - - def test_idle_worker_releases_successful_task(self) -> None: - self.check_idle_worker_releases_task("succeeded") - - def test_idle_worker_releases_failed_task(self) -> None: - self.check_idle_worker_releases_task("failed") - - def test_idle_worker_releases_cancelled_task(self) -> None: - self.check_idle_worker_releases_task("cancelled") - - -class BoundedPriorityExecutorTests(unittest.TestCase): - def test_compatibility_reexports_reference_executor_owner(self) -> None: - self.assertIs( - prism_coordinator._BoundedPriorityExecutor, - _BoundedPriorityExecutor, - ) - self.assertIs(prism_coordinator._DeliveryQueueFull, _DeliveryQueueFull) - - def test_queue_bound_excludes_active_worker_and_reports_active_count(self) -> None: - executor = _BoundedPriorityExecutor(max_workers=1, max_queue_size=1) - blocker_started = threading.Event() - release = threading.Event() - - def blocker() -> None: - blocker_started.set() - release.wait(5) - - try: - executor.submit(blocker) - self.assertTrue(blocker_started.wait(5)) - queued = executor.submit(lambda: None) - self.assertEqual(executor.stats(), (1, 1)) - with self.assertRaisesRegex(_DeliveryQueueFull, "queue is full"): - executor.submit(lambda: None) - release.set() - queued.result(5) - finally: - release.set() - executor.shutdown(wait=True, cancel_futures=True) - - def test_shutdown_cancels_queued_future_and_joins_named_workers(self) -> None: - executor = _BoundedPriorityExecutor(max_workers=2, max_queue_size=2) - blocker_started = [threading.Event(), threading.Event()] - release = threading.Event() - - def blocker(index: int) -> None: - blocker_started[index].set() - release.wait(5) - - executor.submit(blocker, 0) - executor.submit(blocker, 1) - self.assertTrue(all(started.wait(5) for started in blocker_started)) - queued = executor.submit(lambda: None) - - executor.shutdown(wait=False, cancel_futures=True) - self.assertTrue(queued.cancelled()) - self.assertEqual( - [thread.name for thread in executor._threads], - ["prism-job-delivery-1", "prism-job-delivery-2"], - ) - release.set() - executor.shutdown(wait=True) - - self.assertTrue(all(not thread.is_alive() for thread in executor._threads)) - - def test_cancel_discards_queued_entry_and_frees_bounded_admission(self) -> None: - executor = _BoundedPriorityExecutor(max_workers=1, max_queue_size=1) - blocker_started = threading.Event() - release = threading.Event() - - def blocker() -> None: - blocker_started.set() - release.wait(5) - - try: - executor.submit(blocker) - self.assertTrue(blocker_started.wait(5)) - queued = executor.submit(lambda: "victim") - with self.assertRaises(_DeliveryQueueFull): - executor.submit(lambda: "rejected") - - self.assertTrue(executor.cancel(queued)) - self.assertTrue(queued.cancelled()) - # The cancelled entry is removed from the priority queue itself, - # so bounded admission is immediately available again. - replacement = executor.submit(lambda: "replacement") - self.assertEqual(executor.stats()[0], 1) - - release.set() - self.assertEqual(replacement.result(5), "replacement") - # A worker that already dequeued an entry owns the task_done - # path; cancelling a running/finished future removes nothing. - self.assertFalse(executor.cancel(replacement)) - finally: - release.set() - executor.shutdown(wait=True, cancel_futures=True) - - -if __name__ == "__main__": - unittest.main() diff --git a/tests/test_prism_builder_pipe_transport.py b/tests/test_prism_builder_pipe_transport.py deleted file mode 100644 index b2c535c2d..000000000 --- a/tests/test_prism_builder_pipe_transport.py +++ /dev/null @@ -1,1771 +0,0 @@ -"""Builder pipe transport: readiness waits in place of fixed polling (#236). - -The --serve daemon transport and the one-shot writer drive non-blocking -pipes so cancellation, supersession and the absolute deadline are checked -between syscalls. They used to sleep a fixed 20 ms after every EAGAIN; a -readiness wait bounded by the deadline and the same check cadence now takes -its place. These tests pin the transport contract that must survive that -swap -- exact bytes and framing, exactly-once failure accounting, the -timeout/cancellation/supersession classifications, spool poisoning without -prefix replay, and selector cleanup on every path -- first with scripted -syscalls (deterministic EAGAIN/EINTR/spurious-readiness/short-write/EOF -sequences) and then over real pipes with a peer thread or subprocess. - -No timing threshold here is tight enough to be a performance claim; the -throughput measurement lives in ``tests/perf/window_builder_pipe_latency.py``. -""" - -from __future__ import annotations - -import hashlib -import json -import os -import selectors -import sys -import tempfile -import threading -import time -import unittest -from pathlib import Path -from types import SimpleNamespace -from typing import Any, Callable -from unittest.mock import patch - -from lab.prism import bundle_compiler as compiler_module -from lab.prism.bundle_compiler import ( - PRISM_BUILDER_PIPE_READ_CHUNK_BYTES, - PRISM_BUILDER_PIPE_WAIT_SLICE_SECONDS, - PRISM_TIP_REFRESH_ADMISSION_POLL_SECONDS, - BundleCompiler, - _PipeReadinessWaiter, - _ServeBuilderClient, - _ServeBuilderUnavailable, -) -from tests.prism_coordinator_test_support import ( - FAKE_SERVE_BUILDER_COMMAND, - PayoutLedgerArtifact, - coordinator, - spool_share, -) - - -FAR_DEADLINE_SECONDS = 30.0 - - -class _Superseded(RuntimeError): - """Stands in for JobBuildSuperseded.""" - - -class _Cancelled(RuntimeError): - """Stands in for the coordinator's cancellation error.""" - - -class _Control: - """Minimal BundleBuildControlPort.""" - - def __init__(self) -> None: - self.cancel_event = threading.Event() - self.process = None - - -class _Cancellation: - """Minimal CancellationPort backed by an event.""" - - def __init__(self) -> None: - self.event = threading.Event() - self.phases: list[str] = [] - - def is_set(self) -> bool: - return self.event.is_set() - - def raise_if_cancelled(self, phase: str) -> None: - self.phases.append(phase) - if self.event.is_set(): - raise _Cancelled(phase) - - -class _Runtime: - """Only the scheduler counters the transport helpers touch.""" - - def __init__(self) -> None: - self._job_build_scheduler_lock = threading.Lock() - self.job_build_worker_counts = { - "starts": 0, - "restarts": 0, - "crashes": 0, - "terminations": 0, - } - self._job_build_worker_restart_pending = False - - -class _Serialization: - """Only the spool-poisoning surface the splice helper touches.""" - - def __init__(self) -> None: - self.spool_failures = 0 - - def mark_spool_failed(self) -> None: - self.spool_failures += 1 - - -class _PipeEnd: - def __init__(self, file_descriptor: int) -> None: - self._file_descriptor = file_descriptor - - def fileno(self) -> int: - return self._file_descriptor - - -class _FakeClock: - """A monotonic clock the scripted tests advance by hand.""" - - def __init__(self, start: float = 1_000.0) -> None: - self.now = start - - def monotonic(self) -> float: - return self.now - - def advance(self, seconds: float) -> None: - self.now += seconds - - def sleep(self, seconds: float) -> None: - self.advance(seconds) - - -class _ScriptedWaiter: - """Drop-in for _PipeReadinessWaiter that never blocks. - - Records every wait with the deadline slack it was given (against the - scripted fake clock) and answers readiness from a class-level script - (True when the script is empty). Each wait may advance the fake clock, - so a deadline is exhausted by construction rather than by real time - passing: the transport loops can be walked through EAGAIN, spurious - readiness and timeouts deterministically. - """ - - instances: list[_ScriptedWaiter] = [] - readiness: list[bool] = [] - clock: _FakeClock = _FakeClock() - advance_per_wait: float = 0.0 - - def __init__(self, file_descriptor: int, events: int) -> None: - self.file_descriptor = file_descriptor - self.events = events - self.waits = 0 - self.slack: list[float] = [] - self.closed = 0 - type(self).instances.append(self) - - def __enter__(self) -> _ScriptedWaiter: - return self - - def __exit__(self, *exc_info: object) -> None: - self.close() - - def wait(self, deadline: float) -> bool: - self.waits += 1 - cls = type(self) - self.slack.append(deadline - cls.clock.monotonic()) - cls.clock.advance(cls.advance_per_wait) - script = cls.readiness - if script: - return script.pop(0) - return True - - def close(self) -> None: - self.closed += 1 - - -class _RecordingWaiter(_PipeReadinessWaiter): - """The real waiter, with its instances and waits observable.""" - - instances: list[_RecordingWaiter] = [] - - def __init__(self, file_descriptor: int, events: int) -> None: - super().__init__(file_descriptor, events) - self.closed = 0 - type(self).instances.append(self) - - def close(self) -> None: - self.closed += 1 - super().close() - - -class _GatedWaiter(_RecordingWaiter): - """The real waiter, releasing a held-back peer when the transport blocks. - - Tests that need the transport to block by construction hold their peer - back until the transport has actually entered a wait: at that moment - the pipe is empty (reads) or full (writes) whatever the scheduler did, - so a wait-count assertion is not a timing assumption. The release is an - in-process event for thread peers and a marker file for child processes. - """ - - instances: list[_GatedWaiter] = [] - released = threading.Event() - marker: str | None = None - - @classmethod - def arm(cls, marker: str | None = None) -> None: - cls.instances = [] - cls.released = threading.Event() - cls.marker = marker - - def wait(self, deadline: float) -> bool: - gate = type(self) - if not gate.released.is_set(): - if gate.marker is not None: - Path(gate.marker).touch() - gate.released.set() - return super().wait(deadline) - - -class _PollFailingSelector: - """A DefaultSelector stand-in that registers, then cannot poll. - - Models SelectSelector over a Windows pipe handle (registration succeeds, - ``select`` raises), with a hook to spend fake time inside the failed - poll. Lifecycle counts are class-level so a test can assert the - selector was built and closed exactly once. - """ - - built = 0 - closes = 0 - selects: list[float | None] = [] - error: BaseException = OSError(10038, "not a socket") - on_select: Callable[[], None] | None = None - - @classmethod - def reset( - cls, - *, - error: BaseException | None = None, - on_select: Callable[[], None] | None = None, - ) -> None: - cls.built = 0 - cls.closes = 0 - cls.selects = [] - cls.error = OSError(10038, "not a socket") if error is None else error - cls.on_select = on_select - - def __init__(self) -> None: - type(self).built += 1 - self.registered: list[tuple[int, int]] = [] - - def register(self, file_descriptor: int, events: int, data: Any = None) -> None: - self.registered.append((file_descriptor, events)) - - def select(self, timeout: float | None = None) -> list[Any]: - cls = type(self) - cls.selects.append(timeout) - if cls.on_select is not None: - cls.on_select() - raise cls.error - - def close(self) -> None: - type(self).closes += 1 - - -class _Do: - """A scripted syscall step: run a side effect, then answer or raise.""" - - def __init__(self, effect: Callable[[], None], answer: Any) -> None: - self.effect = effect - self.answer = answer - - -class _SyscallScript: - """Scripted os.read / os.write / os.splice answers for one descriptor. - - Answers are consumed in order: bytes (read) or an int (write/splice) - are returned, an exception instance is raised, and a _Do runs its - effect first. Every other descriptor reaches the real syscall. - """ - - def __init__(self, file_descriptor: int) -> None: - self.file_descriptor = file_descriptor - self.reads: list[Any] = [] - self.writes: list[Any] = [] - self.splices: list[Any] = [] - self.read_sizes: list[int] = [] - self.write_log: list[bytes] = [] - self.splice_log: list[tuple[int, int]] = [] - - def _answer(self, script: list[Any], name: str) -> Any: - if not script: - raise AssertionError(f"unscripted os.{name} call") - action = script.pop(0) - if isinstance(action, _Do): - action.effect() - action = action.answer - if isinstance(action, BaseException): - raise action - return action - - def install(self, case: unittest.TestCase) -> None: - real_read = os.read - real_write = os.write - real_splice = getattr(os, "splice", None) - - def read(file_descriptor: int, size: int) -> bytes: - if file_descriptor != self.file_descriptor: - return real_read(file_descriptor, size) - self.read_sizes.append(size) - chunk = self._answer(self.reads, "read") - if len(chunk) > size: - raise AssertionError("scripted read exceeds the requested size") - return chunk - - def write(file_descriptor: int, data: Any) -> int: - if file_descriptor != self.file_descriptor: - return real_write(file_descriptor, data) - written = self._answer(self.writes, "write") - self.write_log.append(bytes(data[:written])) - return written - - def splice(src: int, dst: int, count: int, **kwargs: Any) -> int: - if dst != self.file_descriptor: - assert real_splice is not None - return real_splice(src, dst, count, **kwargs) - self.splice_log.append((int(kwargs.get("offset_src", -1)), count)) - return self._answer(self.splices, "splice") - - for name, replacement in (("read", read), ("write", write)): - patcher = patch.object(os, name, replacement) - patcher.start() - case.addCleanup(patcher.stop) - splice_patcher = patch.object(os, "splice", splice, create=True) - splice_patcher.start() - case.addCleanup(splice_patcher.stop) - - -class _TransportCase(unittest.TestCase): - """Shared fixtures: a compiler over a counter-only runtime and pipes.""" - - def setUp(self) -> None: - self.runtime = _Runtime() - self.compiler = BundleCompiler( - self.runtime, # type: ignore[arg-type] - superseded_error=_Superseded, - cancellation_error_types=(_Cancelled,), - build_control_type=_Control, - ) - self._open_fds: list[int] = [] - - def tearDown(self) -> None: - for file_descriptor in self._open_fds: - try: - os.close(file_descriptor) - except OSError: - pass - - def _close_fd(self, file_descriptor: int) -> None: - try: - os.close(file_descriptor) - except OSError: - pass - if file_descriptor in self._open_fds: - self._open_fds.remove(file_descriptor) - - def _client(self) -> tuple[_ServeBuilderClient, int, int]: - """A client over two real pipes. - - Returns the client plus the peer ends: the descriptor a fake daemon - writes its stdout into, and the one it reads its stdin from. - """ - stdout_read, stdout_write = os.pipe() - stdin_read, stdin_write = os.pipe() - self._open_fds += [stdout_read, stdout_write, stdin_read, stdin_write] - os.set_blocking(stdout_read, False) - os.set_blocking(stdin_write, False) - client = _ServeBuilderClient( - process=SimpleNamespace( # type: ignore[arg-type] - stdin=_PipeEnd(stdin_write), - stdout=_PipeEnd(stdout_read), - poll=lambda: None, - ) - ) - return client, stdout_write, stdin_read - - def _scripted(self) -> None: - """Scripted waiter plus a hand-advanced clock for the module's deadlines.""" - self.clock = _FakeClock() - _ScriptedWaiter.instances = [] - _ScriptedWaiter.readiness = [] - _ScriptedWaiter.clock = self.clock - _ScriptedWaiter.advance_per_wait = 0.0 - for target, replacement in ( - ("_PipeReadinessWaiter", _ScriptedWaiter), - ( - "time", - SimpleNamespace( - monotonic=self.clock.monotonic, - sleep=self.clock.sleep, - ), - ), - ): - patcher = patch.object(compiler_module, target, replacement) - patcher.start() - self.addCleanup(patcher.stop) - - def _gated(self, marker: str | None = None) -> None: - _GatedWaiter.arm(marker) - patcher = patch.object( - compiler_module, - "_PipeReadinessWaiter", - _GatedWaiter, - ) - patcher.start() - self.addCleanup(patcher.stop) - - def _recording(self) -> None: - _RecordingWaiter.instances = [] - patcher = patch.object( - compiler_module, - "_PipeReadinessWaiter", - _RecordingWaiter, - ) - patcher.start() - self.addCleanup(patcher.stop) - - def assertNoFailureAccounted(self) -> None: - self.assertEqual( - self.runtime.job_build_worker_counts, - {"starts": 0, "restarts": 0, "crashes": 0, "terminations": 0}, - ) - self.assertFalse(self.runtime._job_build_worker_restart_pending) - - def assertOneCrashAccounted(self) -> None: - self.assertEqual(self.runtime.job_build_worker_counts["crashes"], 1) - self.assertEqual(self.runtime.job_build_worker_counts["terminations"], 0) - self.assertTrue(self.runtime._job_build_worker_restart_pending) - - -class ReadinessWaiterTests(_TransportCase): - """The helper itself: laziness, bounds, degradation and cleanup.""" - - def test_wait_slice_matches_the_retired_polling_cadence(self) -> None: - self.assertEqual( - PRISM_BUILDER_PIPE_WAIT_SLICE_SECONDS, - min(0.02, PRISM_TIP_REFRESH_ADMISSION_POLL_SECONDS), - ) - - def test_registration_is_lazy_and_close_is_idempotent(self) -> None: - read_end, write_end = os.pipe() - self._open_fds += [read_end, write_end] - with patch.object( - selectors, - "DefaultSelector", - side_effect=AssertionError("selector built without a wait"), - ): - waiter = _PipeReadinessWaiter(read_end, selectors.EVENT_READ) - self.assertEqual(waiter.waits, 0) - waiter.close() - waiter.close() - with waiter: - pass - - def test_ready_descriptor_returns_true_without_sleeping(self) -> None: - read_end, write_end = os.pipe() - self._open_fds += [read_end, write_end] - os.write(write_end, b"x") - with _PipeReadinessWaiter(read_end, selectors.EVENT_READ) as waiter: - started = time.monotonic() - ready = waiter.wait(time.monotonic() + FAR_DEADLINE_SECONDS) - elapsed = time.monotonic() - started - self.assertTrue(ready) - self.assertEqual(waiter.waits, 1) - self.assertLess(elapsed, 1.0) - - def test_idle_descriptor_wakes_on_the_cadence_not_the_deadline(self) -> None: - read_end, write_end = os.pipe() - self._open_fds += [read_end, write_end] - with _PipeReadinessWaiter(read_end, selectors.EVENT_READ) as waiter: - started = time.monotonic() - ready = waiter.wait(time.monotonic() + FAR_DEADLINE_SECONDS) - elapsed = time.monotonic() - started - self.assertFalse(ready) - # Bounded by the cancellation-check slice (20 ms), not by the far - # deadline; the upper bound is generous for a loaded runner. - self.assertGreaterEqual(elapsed, PRISM_BUILDER_PIPE_WAIT_SLICE_SECONDS / 2) - self.assertLess(elapsed, 1.0) - - def test_wait_is_bounded_by_the_remaining_deadline(self) -> None: - read_end, write_end = os.pipe() - self._open_fds += [read_end, write_end] - with _PipeReadinessWaiter(read_end, selectors.EVENT_READ) as waiter: - started = time.monotonic() - ready = waiter.wait(time.monotonic() + 0.005) - elapsed = time.monotonic() - started - self.assertFalse(ready) - self.assertLess(elapsed, 1.0) - - def test_exhausted_deadline_returns_false_without_a_selector(self) -> None: - read_end, write_end = os.pipe() - self._open_fds += [read_end, write_end] - with patch.object( - selectors, - "DefaultSelector", - side_effect=AssertionError("selector built for a dead deadline"), - ), patch.object(compiler_module.time, "sleep") as sleep: - with _PipeReadinessWaiter(read_end, selectors.EVENT_READ) as waiter: - self.assertFalse(waiter.wait(time.monotonic() - 1.0)) - self.assertEqual(waiter.waits, 1) - sleep.assert_not_called() - - def test_unwatchable_descriptor_degrades_to_one_bounded_sleep(self) -> None: - class _Refusing: - built = 0 - - def __init__(self) -> None: - type(self).built += 1 - - def register(self, *args: object) -> None: - raise PermissionError("regular files cannot be polled") - - def close(self) -> None: - pass - - read_end, write_end = os.pipe() - self._open_fds += [read_end, write_end] - with patch.object(selectors, "DefaultSelector", _Refusing), patch.object( - compiler_module.time, - "sleep", - ) as sleep: - with _PipeReadinessWaiter(read_end, selectors.EVENT_READ) as waiter: - self.assertFalse(waiter.wait(time.monotonic() + FAR_DEADLINE_SECONDS)) - self.assertFalse(waiter.wait(time.monotonic() + FAR_DEADLINE_SECONDS)) - self.assertEqual(_Refusing.built, 1) - self.assertEqual(sleep.call_count, 2) - for call in sleep.call_args_list: - self.assertLessEqual(call.args[0], PRISM_BUILDER_PIPE_WAIT_SLICE_SECONDS) - self.assertGreater(call.args[0], 0.0) - - def test_close_releases_the_registration(self) -> None: - closed: list[object] = [] - real_selector = selectors.DefaultSelector - - class _Observed(real_selector): # type: ignore[misc,valid-type] - def close(self) -> None: - closed.append(self) - super().close() - - read_end, write_end = os.pipe() - self._open_fds += [read_end, write_end] - with patch.object(selectors, "DefaultSelector", _Observed): - waiter = _PipeReadinessWaiter(read_end, selectors.EVENT_READ) - waiter.wait(time.monotonic() + 0.001) - self.assertEqual(closed, []) - waiter.close() - self.assertEqual(len(closed), 1) - self.assertIsNone(waiter._selector) - - def _fake_clock_module(self) -> tuple[_FakeClock, list[float]]: - """Drive the waiter's clock and record its sleeps.""" - clock = _FakeClock() - sleeps: list[float] = [] - - def sleep(seconds: float) -> None: - sleeps.append(seconds) - clock.advance(seconds) - - patcher = patch.object( - compiler_module, - "time", - SimpleNamespace(monotonic=clock.monotonic, sleep=sleep), - ) - patcher.start() - self.addCleanup(patcher.stop) - return clock, sleeps - - def test_poll_failure_detaches_once_and_falls_back_to_bounded_sleep(self) -> None: - read_end, write_end = os.pipe() - self._open_fds += [read_end, write_end] - clock, sleeps = self._fake_clock_module() - _PollFailingSelector.reset() - with patch.object(selectors, "DefaultSelector", _PollFailingSelector): - waiter = _PipeReadinessWaiter(read_end, selectors.EVENT_READ) - first = waiter.wait(clock.now + FAR_DEADLINE_SECONDS) - second = waiter.wait(clock.now + FAR_DEADLINE_SECONDS) - waiter.close() - waiter.close() - - self.assertEqual((first, second), (False, False)) - self.assertEqual(waiter.waits, 2) - # Built and polled once; released exactly once at the failed poll, - # never rebuilt for the next wait, and not closed again on exit. - self.assertEqual(_PollFailingSelector.built, 1) - self.assertEqual(len(_PollFailingSelector.selects), 1) - self.assertEqual(_PollFailingSelector.closes, 1) - self.assertIsNone(waiter._selector) - self.assertTrue(waiter._unwatchable) - # The failed poll spent no clock, so the first wait sleeps its whole - # slice; the second, unwatchable, sleeps one full slice too. - self.assertEqual( - [round(seconds, 6) for seconds in sleeps], - [PRISM_BUILDER_PIPE_WAIT_SLICE_SECONDS] * 2, - ) - - def test_poll_failure_sleeps_only_the_remainder_of_the_slice(self) -> None: - read_end, write_end = os.pipe() - self._open_fds += [read_end, write_end] - clock, sleeps = self._fake_clock_module() - spent = 0.005 - _PollFailingSelector.reset(on_select=lambda: clock.advance(spent)) - with patch.object(selectors, "DefaultSelector", _PollFailingSelector): - with _PipeReadinessWaiter(read_end, selectors.EVENT_READ) as waiter: - ready = waiter.wait(clock.now + FAR_DEADLINE_SECONDS) - after_first = clock.now - again = waiter.wait(clock.now + FAR_DEADLINE_SECONDS) - - self.assertEqual((ready, again), (False, False)) - # The failed poll consumed 5 ms of the 20 ms slice: the fallback - # sleeps the 15 ms left, so the wait as a whole still spans exactly - # one slice; the next (unwatchable) wait sleeps one full slice. - self.assertEqual( - [round(seconds, 6) for seconds in sleeps], - [ - round(PRISM_BUILDER_PIPE_WAIT_SLICE_SECONDS - spent, 6), - PRISM_BUILDER_PIPE_WAIT_SLICE_SECONDS, - ], - ) - self.assertEqual(round(after_first - 1_000.0, 6), PRISM_BUILDER_PIPE_WAIT_SLICE_SECONDS) - self.assertEqual(_PollFailingSelector.built, 1) - self.assertEqual(_PollFailingSelector.closes, 1) - - def test_poll_failure_after_spending_the_slice_sleeps_no_further(self) -> None: - read_end, write_end = os.pipe() - self._open_fds += [read_end, write_end] - clock, sleeps = self._fake_clock_module() - _PollFailingSelector.reset( - on_select=lambda: clock.advance(PRISM_BUILDER_PIPE_WAIT_SLICE_SECONDS * 3) - ) - with patch.object(selectors, "DefaultSelector", _PollFailingSelector): - with _PipeReadinessWaiter(read_end, selectors.EVENT_READ) as waiter: - ready = waiter.wait(clock.now + FAR_DEADLINE_SECONDS) - - self.assertFalse(ready) - self.assertEqual(sleeps, []) - self.assertEqual(_PollFailingSelector.closes, 1) - self.assertTrue(waiter._unwatchable) - - def test_poll_failure_never_overruns_the_absolute_deadline(self) -> None: - read_end, write_end = os.pipe() - self._open_fds += [read_end, write_end] - clock, sleeps = self._fake_clock_module() - _PollFailingSelector.reset(on_select=lambda: clock.advance(0.004)) - with patch.object(selectors, "DefaultSelector", _PollFailingSelector): - with _PipeReadinessWaiter(read_end, selectors.EVENT_READ) as waiter: - deadline = clock.now + 0.010 - ready = waiter.wait(deadline) - overshoot = clock.now - deadline - - self.assertFalse(ready) - # 10 ms of budget, 4 ms spent by the failed poll: the fallback sleeps - # the 6 ms to the deadline, not the 16 ms left of the slice. - self.assertEqual([round(seconds, 6) for seconds in sleeps], [0.006]) - self.assertLessEqual(round(overshoot, 9), 0.0) - - def test_poll_failure_with_a_spent_deadline_returns_without_sleeping(self) -> None: - read_end, write_end = os.pipe() - self._open_fds += [read_end, write_end] - clock, sleeps = self._fake_clock_module() - _PollFailingSelector.reset(on_select=lambda: clock.advance(0.050)) - with patch.object(selectors, "DefaultSelector", _PollFailingSelector): - with _PipeReadinessWaiter(read_end, selectors.EVENT_READ) as waiter: - ready = waiter.wait(clock.now + 0.010) - # Subsequent waits against a dead deadline stay free too. - later = waiter.wait(clock.now - 1.0) - - self.assertEqual((ready, later), (False, False)) - self.assertEqual(sleeps, []) - self.assertEqual(_PollFailingSelector.built, 1) - self.assertEqual(_PollFailingSelector.closes, 1) - - def test_interrupted_poll_is_a_plain_wakeup_and_keeps_the_selector(self) -> None: - read_end, write_end = os.pipe() - self._open_fds += [read_end, write_end] - _PollFailingSelector.reset(error=InterruptedError()) - with patch.object(selectors, "DefaultSelector", _PollFailingSelector), patch.object( - compiler_module.time, - "sleep", - ) as sleep: - waiter = _PipeReadinessWaiter(read_end, selectors.EVENT_READ) - first = waiter.wait(time.monotonic() + FAR_DEADLINE_SECONDS) - second = waiter.wait(time.monotonic() + FAR_DEADLINE_SECONDS) - self.assertEqual(_PollFailingSelector.closes, 0) - waiter.close() - - self.assertEqual((first, second), (False, False)) - # EINTR is not a fault: no sleep, the same selector polled again, - # released only by close(). - sleep.assert_not_called() - self.assertEqual(_PollFailingSelector.built, 1) - self.assertEqual(len(_PollFailingSelector.selects), 2) - self.assertEqual(_PollFailingSelector.closes, 1) - self.assertFalse(waiter._unwatchable) - - -class UnpollableDescriptorTransportTests(_TransportCase): - """A transport helper over a descriptor the selector cannot poll.""" - - def setUp(self) -> None: - super().setUp() - self._recording() - self.client, _stdout_write, _stdin_read = self._client() - self.script = _SyscallScript(self.client.process.stdout.fileno()) - self.script.install(self) - self.clock = _FakeClock() - self.sleeps: list[float] = [] - - def sleep(seconds: float) -> None: - self.sleeps.append(seconds) - self.clock.advance(seconds) - - for target, replacement in ( - ("time", SimpleNamespace(monotonic=self.clock.monotonic, sleep=sleep)), - ): - patcher = patch.object(compiler_module, target, replacement) - patcher.start() - self.addCleanup(patcher.stop) - selector_patch = patch.object(selectors, "DefaultSelector", _PollFailingSelector) - selector_patch.start() - self.addCleanup(selector_patch.stop) - - def test_read_line_completes_through_the_bounded_sleep_fallback(self) -> None: - _PollFailingSelector.reset() - self.script.reads = [BlockingIOError(), BlockingIOError(), b"ok\n"] - - line = self.compiler._serve_builder_read_line( - self.client, - self.clock.now + FAR_DEADLINE_SECONDS, - None, - None, - ) - - self.assertEqual(line, b"ok") - self.assertEqual(self.script.reads, []) - # First EAGAIN: the poll fails and the wait finishes on the sleep; - # second EAGAIN: unwatchable, straight to the sleep. One selector, - # released once at the failed poll, nothing left for close(). - self.assertEqual( - [round(seconds, 6) for seconds in self.sleeps], - [PRISM_BUILDER_PIPE_WAIT_SLICE_SECONDS] * 2, - ) - self.assertEqual(_PollFailingSelector.built, 1) - self.assertEqual(len(_PollFailingSelector.selects), 1) - self.assertEqual(_PollFailingSelector.closes, 1) - (waiter,) = _RecordingWaiter.instances - self.assertEqual(waiter.waits, 2) - self.assertEqual(waiter.closed, 1) - self.assertIsNone(waiter._selector) - self.assertNoFailureAccounted() - - def test_failed_poll_that_spends_the_deadline_times_out_without_sleeping(self) -> None: - _PollFailingSelector.reset(on_select=lambda: self.clock.advance(0.050)) - self.script.reads = [BlockingIOError(), b"never\n"] - - with self.assertRaisesRegex(_ServeBuilderUnavailable, "timed out"): - self.compiler._serve_builder_read_line( - self.client, - self.clock.now + 0.010, - None, - None, - ) - - self.assertEqual(self.sleeps, []) - self.assertEqual(self.script.reads, [b"never\n"]) - self.assertEqual(_PollFailingSelector.closes, 1) - (waiter,) = _RecordingWaiter.instances - self.assertEqual(waiter.waits, 1) - self.assertEqual(waiter.closed, 1) - self.assertNoFailureAccounted() - - -class ScriptedReadLineTests(_TransportCase): - """Envelope line reads walked through scripted syscalls.""" - - def setUp(self) -> None: - super().setUp() - self._scripted() - self.client, _stdout_write, _stdin_read = self._client() - self.script = _SyscallScript(self.client.process.stdout.fileno()) - self.script.install(self) - self.deadline = self.clock.now + FAR_DEADLINE_SECONDS - - def _read_line(self, cancellation=None, control=None) -> bytes: - return self.compiler._serve_builder_read_line( - self.client, - self.deadline, - cancellation, - control, - ) - - def test_eagain_waits_once_then_returns_line_and_keeps_tail(self) -> None: - self.script.reads = [BlockingIOError(), b'{"ok":true}\nTAIL'] - - line = self._read_line() - - self.assertEqual(line, b'{"ok":true}') - self.assertEqual(bytes(self.client.stdout_buffer), b"TAIL") - self.assertEqual(self.script.reads, []) - (waiter,) = _ScriptedWaiter.instances - self.assertEqual(waiter.events, selectors.EVENT_READ) - self.assertEqual(waiter.file_descriptor, self.script.file_descriptor) - self.assertEqual(waiter.waits, 1) - self.assertEqual(waiter.closed, 1) - self.assertEqual(waiter.slack, [FAR_DEADLINE_SECONDS]) - self.assertEqual(self.script.read_sizes, [PRISM_BUILDER_PIPE_READ_CHUNK_BYTES] * 2) - self.assertNoFailureAccounted() - - def test_line_already_buffered_takes_no_syscall_and_no_wait(self) -> None: - self.client.stdout_buffer += b"first\nsecond" - - self.assertEqual(self._read_line(), b"first") - - self.assertEqual(bytes(self.client.stdout_buffer), b"second") - self.assertEqual(self.script.read_sizes, []) - (waiter,) = _ScriptedWaiter.instances - self.assertEqual(waiter.waits, 0) - self.assertEqual(waiter.closed, 1) - - def test_line_assembled_across_chunks(self) -> None: - self.script.reads = [b"ab", BlockingIOError(), b"cd", b"ef\n"] - - self.assertEqual(self._read_line(), b"abcdef") - - self.assertEqual(bytes(self.client.stdout_buffer), b"") - self.assertEqual(_ScriptedWaiter.instances[0].waits, 1) - - def test_eintr_retries_immediately_without_waiting(self) -> None: - self.script.reads = [InterruptedError(), InterruptedError(), b"line\n"] - - self.assertEqual(self._read_line(), b"line") - - (waiter,) = _ScriptedWaiter.instances - self.assertEqual(waiter.waits, 0) - self.assertEqual(waiter.closed, 1) - - def test_spurious_readiness_retries_and_waits_again(self) -> None: - self.script.reads = [BlockingIOError(), BlockingIOError(), b"x\n"] - _ScriptedWaiter.readiness = [True, True] - - self.assertEqual(self._read_line(), b"x") - - self.assertEqual(_ScriptedWaiter.instances[0].waits, 2) - - def test_cadence_wakeup_without_readiness_retries(self) -> None: - self.script.reads = [BlockingIOError(), BlockingIOError(), b"x\n"] - _ScriptedWaiter.readiness = [False, False] - - self.assertEqual(self._read_line(), b"x") - - self.assertEqual(_ScriptedWaiter.instances[0].waits, 2) - - def test_eof_mid_frame_is_one_crash(self) -> None: - self.script.reads = [b"partial", b""] - - with self.assertRaisesRegex(_ServeBuilderUnavailable, "exited mid-request"): - self._read_line() - - self.assertOneCrashAccounted() - self.assertEqual(_ScriptedWaiter.instances[0].closed, 1) - - def test_eof_after_supersession_is_superseded_not_a_crash(self) -> None: - control = _Control() - self.script.reads = [ - b"partial", - _Do(control.cancel_event.set, b""), - ] - - with self.assertRaisesRegex(_Superseded, "terminated after supersession"): - self._read_line(control=control) - - self.assertNoFailureAccounted() - self.assertEqual(_ScriptedWaiter.instances[0].closed, 1) - - def test_deadline_while_blocked_times_out_without_accounting(self) -> None: - self.deadline = self.clock.now + 1.0 - self.script.reads = [BlockingIOError()] * 3 - _ScriptedWaiter.readiness = [False, False] - _ScriptedWaiter.advance_per_wait = 0.6 - - with self.assertRaisesRegex(_ServeBuilderUnavailable, "timed out"): - self._read_line() - - self.assertNoFailureAccounted() - (waiter,) = _ScriptedWaiter.instances - # Two waits, each handed the true remaining slack: the second - # exhausts the deadline and the loop's own check raises before any - # further syscall. - self.assertEqual(waiter.waits, 2) - self.assertEqual([round(slack, 6) for slack in waiter.slack], [1.0, 0.4]) - self.assertEqual(len(self.script.reads), 1) - self.assertEqual(waiter.closed, 1) - - def test_cancellation_while_blocked_raises_and_cleans_up(self) -> None: - cancellation = _Cancellation() - self.script.reads = [ - _Do(cancellation.event.set, BlockingIOError()), - b"never\n", - ] - - with self.assertRaises(_Cancelled): - self._read_line(cancellation=cancellation) - - self.assertEqual(self.script.reads, [b"never\n"]) - self.assertNoFailureAccounted() - self.assertEqual(_ScriptedWaiter.instances[0].closed, 1) - - def test_supersession_while_blocked_raises_and_cleans_up(self) -> None: - control = _Control() - self.script.reads = [ - _Do(control.cancel_event.set, BlockingIOError()), - b"never\n", - ] - - with self.assertRaisesRegex(_Superseded, "canceled after supersession"): - self._read_line(control=control) - - self.assertEqual(self.script.reads, [b"never\n"]) - self.assertNoFailureAccounted() - self.assertEqual(_ScriptedWaiter.instances[0].closed, 1) - - def test_read_error_is_a_daemon_anomaly(self) -> None: - self.script.reads = [OSError(5, "io error")] - - with self.assertRaisesRegex(_ServeBuilderUnavailable, "read failed"): - self._read_line() - - self.assertNoFailureAccounted() - self.assertEqual(_ScriptedWaiter.instances[0].closed, 1) - - -class ScriptedReadExactTests(_TransportCase): - """Raw-section reads walked through scripted syscalls.""" - - def setUp(self) -> None: - super().setUp() - self._scripted() - self.client, _stdout_write, _stdin_read = self._client() - self.script = _SyscallScript(self.client.process.stdout.fileno()) - self.script.install(self) - self.deadline = self.clock.now + FAR_DEADLINE_SECONDS - - def _read_exact(self, byte_count: int, cancellation=None, control=None) -> bytes: - return self.compiler._serve_builder_read_exact( - self.client, - byte_count, - self.deadline, - cancellation, - control, - ) - - def test_drains_buffer_then_reads_exactly_across_short_reads(self) -> None: - self.client.stdout_buffer += b"ABC" - self.script.reads = [b"DE", BlockingIOError(), b"FGH", b"UNREAD"] - - self.assertEqual(self._read_exact(8), b"ABCDEFGH") - - self.assertEqual(bytes(self.client.stdout_buffer), b"") - self.assertEqual(self.script.reads, [b"UNREAD"]) - # Never asks the pipe for more than the frame still owes. - self.assertEqual(self.script.read_sizes, [5, 3, 3]) - (waiter,) = _ScriptedWaiter.instances - self.assertEqual(waiter.waits, 1) - self.assertEqual(waiter.closed, 1) - self.assertNoFailureAccounted() - - def test_satisfied_from_buffer_leaves_the_rest_for_the_next_frame(self) -> None: - self.client.stdout_buffer += b"RAW\nnext-line\n" - - self.assertEqual(self._read_exact(3), b"RAW") - self.assertEqual(self._read_exact(1), b"\n") - - self.assertEqual(bytes(self.client.stdout_buffer), b"next-line\n") - self.assertEqual(self.script.read_sizes, []) - self.assertTrue(all(w.waits == 0 for w in _ScriptedWaiter.instances)) - self.assertTrue(all(w.closed == 1 for w in _ScriptedWaiter.instances)) - - def test_large_frame_reads_in_pipe_sized_chunks(self) -> None: - payload = bytes(range(256)) * 1024 # 256 KiB - chunk = PRISM_BUILDER_PIPE_READ_CHUNK_BYTES - self.script.reads = [ - payload[:chunk], - BlockingIOError(), - payload[chunk : 2 * chunk], - payload[2 * chunk : 2 * chunk + 10], - payload[2 * chunk + 10 : 3 * chunk], - payload[3 * chunk :], - ] - - self.assertEqual(self._read_exact(len(payload)), payload) - - # Every read asks for one pipe capacity while more than that is - # owed, whatever the pipe actually delivered last time. - self.assertEqual(self.script.read_sizes, [chunk] * 6) - self.assertEqual(_ScriptedWaiter.instances[0].waits, 1) - - def test_zero_length_frame_returns_empty_without_io(self) -> None: - self.assertEqual(self._read_exact(0), b"") - self.assertEqual(self.script.read_sizes, []) - - def test_eof_mid_frame_is_one_crash(self) -> None: - self.script.reads = [b"12", b""] - - with self.assertRaisesRegex(_ServeBuilderUnavailable, "exited mid-request"): - self._read_exact(4) - - self.assertOneCrashAccounted() - self.assertEqual(_ScriptedWaiter.instances[0].closed, 1) - - def test_eintr_and_spurious_readiness(self) -> None: - self.script.reads = [ - InterruptedError(), - BlockingIOError(), - BlockingIOError(), - b"ok", - ] - _ScriptedWaiter.readiness = [True, False] - - self.assertEqual(self._read_exact(2), b"ok") - - self.assertEqual(_ScriptedWaiter.instances[0].waits, 2) - - def test_deadline_times_out_without_accounting(self) -> None: - self.deadline = self.clock.now + 1.0 - self.script.reads = [BlockingIOError()] * 3 - _ScriptedWaiter.readiness = [False, False] - _ScriptedWaiter.advance_per_wait = 0.6 - - with self.assertRaisesRegex(_ServeBuilderUnavailable, "timed out"): - self._read_exact(1) - - self.assertNoFailureAccounted() - (waiter,) = _ScriptedWaiter.instances - self.assertEqual(waiter.waits, 2) - self.assertEqual([round(slack, 6) for slack in waiter.slack], [1.0, 0.4]) - self.assertEqual(len(self.script.reads), 1) - self.assertEqual(waiter.closed, 1) - - def test_cancellation_and_supersession_while_blocked(self) -> None: - cancellation = _Cancellation() - self.script.reads = [_Do(cancellation.event.set, BlockingIOError())] - with self.assertRaises(_Cancelled): - self._read_exact(1, cancellation=cancellation) - - control = _Control() - self.script.reads = [_Do(control.cancel_event.set, BlockingIOError())] - with self.assertRaisesRegex(_Superseded, "canceled after supersession"): - self._read_exact(1, control=control) - - self.assertNoFailureAccounted() - self.assertTrue(all(w.closed == 1 for w in _ScriptedWaiter.instances)) - - -class ScriptedWriteTests(_TransportCase): - """Request writes walked through scripted syscalls.""" - - def setUp(self) -> None: - super().setUp() - self._scripted() - self.client, _stdout_write, _stdin_read = self._client() - self.script = _SyscallScript(self.client.process.stdin.fileno()) - self.script.install(self) - self.deadline = self.clock.now + FAR_DEADLINE_SECONDS - - def _write(self, data: bytes, cancellation=None, control=None) -> int: - return self.compiler._serve_builder_write( - self.client, - data, - self.deadline, - cancellation, - control, - ) - - def test_short_writes_and_eagain_deliver_every_byte_once_in_order(self) -> None: - self.script.writes = [3, BlockingIOError(), InterruptedError(), 4, 3] - - self.assertEqual(self._write(b"0123456789"), 10) - - self.assertEqual(b"".join(self.script.write_log), b"0123456789") - self.assertEqual(self.script.write_log, [b"012", b"3456", b"789"]) - (waiter,) = _ScriptedWaiter.instances - self.assertEqual(waiter.events, selectors.EVENT_WRITE) - self.assertEqual(waiter.file_descriptor, self.script.file_descriptor) - self.assertEqual(waiter.waits, 1) - self.assertEqual(waiter.closed, 1) - self.assertNoFailureAccounted() - - def test_empty_write_takes_no_syscall(self) -> None: - self.assertEqual(self._write(b""), 0) - self.assertEqual(self.script.write_log, []) - - def test_broken_pipe_is_closed_input(self) -> None: - self.script.writes = [2, BrokenPipeError()] - - with self.assertRaisesRegex(_ServeBuilderUnavailable, "input pipe closed"): - self._write(b"abcd") - - self.assertEqual(self.script.write_log, [b"ab"]) - self.assertNoFailureAccounted() - self.assertEqual(_ScriptedWaiter.instances[0].closed, 1) - - def test_broken_pipe_after_supersession_is_superseded(self) -> None: - control = _Control() - self.script.writes = [_Do(control.cancel_event.set, BrokenPipeError())] - - with self.assertRaisesRegex(_Superseded, "terminated after supersession"): - self._write(b"abcd", control=control) - - self.assertNoFailureAccounted() - - def test_zero_progress_is_closed_input(self) -> None: - self.script.writes = [0] - - with self.assertRaisesRegex(_ServeBuilderUnavailable, "input pipe closed"): - self._write(b"abcd") - - self.assertNoFailureAccounted() - - def test_write_error_is_a_daemon_anomaly(self) -> None: - self.script.writes = [OSError(5, "io error")] - - with self.assertRaisesRegex(_ServeBuilderUnavailable, "write failed"): - self._write(b"abcd") - - self.assertNoFailureAccounted() - - def test_deadline_times_out_without_accounting(self) -> None: - self.deadline = self.clock.now + 1.0 - self.script.writes = [BlockingIOError()] * 3 - _ScriptedWaiter.readiness = [False, False] - _ScriptedWaiter.advance_per_wait = 0.6 - - with self.assertRaisesRegex(_ServeBuilderUnavailable, "timed out"): - self._write(b"abcd") - - self.assertNoFailureAccounted() - (waiter,) = _ScriptedWaiter.instances - self.assertEqual(waiter.waits, 2) - self.assertEqual([round(slack, 6) for slack in waiter.slack], [1.0, 0.4]) - self.assertEqual(len(self.script.writes), 1) - self.assertEqual(self.script.write_log, []) - self.assertEqual(waiter.closed, 1) - - def test_cancellation_and_supersession_while_blocked(self) -> None: - cancellation = _Cancellation() - self.script.writes = [_Do(cancellation.event.set, BlockingIOError())] - with self.assertRaises(_Cancelled): - self._write(b"abcd", cancellation=cancellation) - - control = _Control() - self.script.writes = [_Do(control.cancel_event.set, BlockingIOError())] - with self.assertRaisesRegex(_Superseded, "canceled after supersession"): - self._write(b"abcd", control=control) - - self.assertNoFailureAccounted() - self.assertTrue(all(w.closed == 1 for w in _ScriptedWaiter.instances)) - - -class ScriptedSpliceTests(_TransportCase): - """Spool splices walked through scripted syscalls.""" - - def setUp(self) -> None: - super().setUp() - self._scripted() - self.client, _stdout_write, _stdin_read = self._client() - self.script = _SyscallScript(self.client.process.stdin.fileno()) - self.script.install(self) - self.deadline = self.clock.now + FAR_DEADLINE_SECONDS - self.serialization = _Serialization() - self.spool = tempfile.TemporaryFile() - self.addCleanup(self.spool.close) - - def _splice(self, spool_size: int, cancellation=None, control=None) -> int: - return self.compiler._serve_builder_splice_spool( - self.client, - self.serialization, # type: ignore[arg-type] - self.spool, - spool_size, - self.deadline, - cancellation, - control, - ) - - def test_eagain_waits_for_the_pipe_and_resumes_from_the_exact_offset(self) -> None: - self.script.splices = [4, BlockingIOError(), InterruptedError(), 6] - - self.assertEqual(self._splice(10), 10) - - self.assertEqual( - self.script.splice_log, - [(0, 10), (4, 6), (4, 6), (4, 6)], - ) - (waiter,) = _ScriptedWaiter.instances - self.assertEqual(waiter.events, selectors.EVENT_WRITE) - self.assertEqual(waiter.file_descriptor, self.script.file_descriptor) - self.assertEqual(waiter.waits, 1) - self.assertEqual(waiter.closed, 1) - self.assertEqual(self.serialization.spool_failures, 0) - self.assertNoFailureAccounted() - - def test_failure_after_partial_progress_poisons_spool_and_never_replays(self) -> None: - self.script.splices = [4, OSError(22, "unsupported")] - - with self.assertRaisesRegex(_ServeBuilderUnavailable, "spool transfer failed"): - self._splice(10) - - self.assertEqual(self.script.splice_log, [(0, 10), (4, 6)]) - self.assertEqual(self.script.write_log, []) - self.assertEqual(self.serialization.spool_failures, 1) - self.assertNoFailureAccounted() - self.assertEqual(_ScriptedWaiter.instances[0].closed, 1) - - def test_broken_pipe_is_closed_input_and_keeps_the_spool(self) -> None: - self.script.splices = [BrokenPipeError()] - - with self.assertRaisesRegex(_ServeBuilderUnavailable, "input pipe closed"): - self._splice(10) - - self.assertEqual(self.serialization.spool_failures, 0) - self.assertNoFailureAccounted() - - def test_errors_after_supersession_are_superseded(self) -> None: - control = _Control() - self.script.splices = [_Do(control.cancel_event.set, BrokenPipeError())] - with self.assertRaisesRegex(_Superseded, "terminated after supersession"): - self._splice(10, control=control) - - control = _Control() - self.script.splices = [_Do(control.cancel_event.set, OSError(5, "io"))] - with self.assertRaisesRegex(_Superseded, "terminated after supersession"): - self._splice(10, control=control) - - self.assertEqual(self.serialization.spool_failures, 0) - self.assertNoFailureAccounted() - - def test_zero_progress_is_closed_input(self) -> None: - self.script.splices = [0] - - with self.assertRaisesRegex(_ServeBuilderUnavailable, "input pipe closed"): - self._splice(10) - - def test_deadline_times_out_without_accounting(self) -> None: - self.deadline = self.clock.now + 1.0 - self.script.splices = [BlockingIOError()] * 3 - _ScriptedWaiter.readiness = [False, False] - _ScriptedWaiter.advance_per_wait = 0.6 - - with self.assertRaisesRegex(_ServeBuilderUnavailable, "timed out"): - self._splice(10) - - self.assertNoFailureAccounted() - self.assertEqual(self.serialization.spool_failures, 0) - (waiter,) = _ScriptedWaiter.instances - self.assertEqual(waiter.waits, 2) - self.assertEqual([round(slack, 6) for slack in waiter.slack], [1.0, 0.4]) - self.assertEqual(len(self.script.splices), 1) - self.assertEqual(waiter.closed, 1) - - def test_cancellation_while_blocked(self) -> None: - cancellation = _Cancellation() - self.script.splices = [_Do(cancellation.event.set, BlockingIOError())] - - with self.assertRaises(_Cancelled): - self._splice(10, cancellation=cancellation) - - self.assertNoFailureAccounted() - - -def _slow_reader(file_descriptor: int, sink: bytearray, *, chunk: int, pause: float) -> None: - """Drain a blocking pipe end slowly so the writer side fills the pipe.""" - while True: - try: - data = os.read(file_descriptor, chunk) - except OSError: - break - if not data: - break - sink += data - time.sleep(pause) - - -def _bursty_writer(file_descriptor: int, payload: bytes, *, burst: int, pause: float) -> None: - """Write a payload in paced bursts, then close the pipe end.""" - view = memoryview(payload) - try: - while view: - written = os.write(file_descriptor, view[:burst]) - view = view[written:] - time.sleep(pause) - finally: - os.close(file_descriptor) - - -class RealPipeTransportTests(_TransportCase): - """The transport over real pipes with a peer thread.""" - - def setUp(self) -> None: - super().setUp() - self._recording() - self.client, self.stdout_write, self.stdin_read = self._client() - self.deadline = time.monotonic() + FAR_DEADLINE_SECONDS - self.threads: list[threading.Thread] = [] - - def tearDown(self) -> None: - for thread in self.threads: - thread.join(10.0) - super().tearDown() - - def _start(self, target: Callable[[], None]) -> None: - thread = threading.Thread(target=target, daemon=True) - thread.start() - self.threads.append(thread) - - def test_read_exact_streams_a_multi_megabyte_frame_exactly(self) -> None: - self._gated() - payload = os.urandom(4 << 20) - stdout_write = self.stdout_write - self._open_fds.remove(stdout_write) - - def writer() -> None: - # Nothing is written until the reader has blocked on the empty - # pipe, so the first read meets EAGAIN by construction. - _GatedWaiter.released.wait(10.0) - _bursty_writer(stdout_write, payload, burst=256 << 10, pause=0.0) - - self._start(writer) - - received = self.compiler._serve_builder_read_exact( - self.client, - len(payload), - self.deadline, - None, - None, - ) - - self.assertEqual(len(received), len(payload)) - self.assertEqual( - hashlib.sha256(received).hexdigest(), - hashlib.sha256(payload).hexdigest(), - ) - self.assertEqual(bytes(self.client.stdout_buffer), b"") - (waiter,) = _GatedWaiter.instances - self.assertTrue(_GatedWaiter.released.is_set()) - self.assertGreaterEqual(waiter.waits, 1) - self.assertEqual(waiter.closed, 1) - self.assertNoFailureAccounted() - - def test_line_then_raw_section_then_line_keeps_framing(self) -> None: - raw = os.urandom(200_000) - envelope = json.dumps({"ok": True, "window_items_len": len(raw)}).encode() - stream = envelope + b"\n" + raw + b"\n" + b'{"ok":true,"next":1}\n' - stdout_write = self.stdout_write - self._open_fds.remove(stdout_write) - self._start(lambda: _bursty_writer(stdout_write, stream, burst=7_000, pause=0.0)) - - line = self.compiler._serve_builder_read_line( - self.client, self.deadline, None, None - ) - self.assertEqual(json.loads(line)["window_items_len"], len(raw)) - section = self.compiler._serve_builder_read_exact( - self.client, len(raw), self.deadline, None, None - ) - self.assertEqual(section, raw) - self.assertEqual( - self.compiler._serve_builder_read_exact( - self.client, 1, self.deadline, None, None - ), - b"\n", - ) - self.assertEqual( - json.loads( - self.compiler._serve_builder_read_line( - self.client, self.deadline, None, None - ) - ), - {"ok": True, "next": 1}, - ) - self.assertEqual(bytes(self.client.stdout_buffer), b"") - self.assertTrue(all(w.closed == 1 for w in _RecordingWaiter.instances)) - - def test_write_streams_to_a_slow_reader_exactly(self) -> None: - self._gated() - payload = os.urandom(2 << 20) - received = bytearray() - stdin_read = self.stdin_read - - def reader() -> None: - # Nothing is drained until the writer has blocked on the full - # pipe, so the transfer meets EAGAIN by construction. - _GatedWaiter.released.wait(10.0) - _slow_reader(stdin_read, received, chunk=32 << 10, pause=0.0) - - self._start(reader) - - written = self.compiler._serve_builder_write( - self.client, payload, self.deadline, None, None - ) - self._close_fd(self.client.process.stdin.fileno()) - self.threads[-1].join(10.0) - - self.assertEqual(written, len(payload)) - self.assertEqual(bytes(received), payload) - (waiter,) = _GatedWaiter.instances - self.assertTrue(_GatedWaiter.released.is_set()) - self.assertGreaterEqual(waiter.waits, 1) - self.assertEqual(waiter.closed, 1) - self.assertNoFailureAccounted() - - @unittest.skipUnless(hasattr(os, "splice"), "os.splice is Linux-only") - def test_splice_streams_the_spool_to_a_slow_reader_exactly(self) -> None: - payload = os.urandom(2 << 20) - spool = tempfile.TemporaryFile() - self.addCleanup(spool.close) - spool.write(payload) - spool.flush() - serialization = _Serialization() - received = bytearray() - stdin_read = self.stdin_read - self._gated() - - def reader() -> None: - _GatedWaiter.released.wait(10.0) - _slow_reader(stdin_read, received, chunk=32 << 10, pause=0.0) - - self._start(reader) - - moved = self.compiler._serve_builder_splice_spool( - self.client, - serialization, # type: ignore[arg-type] - spool, - len(payload), - self.deadline, - None, - None, - ) - self._close_fd(self.client.process.stdin.fileno()) - self.threads[-1].join(10.0) - - self.assertEqual(moved, len(payload)) - self.assertEqual(bytes(received), payload) - self.assertEqual(serialization.spool_failures, 0) - (waiter,) = _GatedWaiter.instances - self.assertTrue(_GatedWaiter.released.is_set()) - self.assertGreaterEqual(waiter.waits, 1) - self.assertEqual(waiter.closed, 1) - self.assertNoFailureAccounted() - - def test_idle_daemon_times_out_at_the_deadline_without_accounting(self) -> None: - deadline = time.monotonic() + 0.1 - started = time.monotonic() - - with self.assertRaisesRegex(_ServeBuilderUnavailable, "timed out"): - self.compiler._serve_builder_read_line(self.client, deadline, None, None) - - elapsed = time.monotonic() - started - self.assertGreaterEqual(elapsed, 0.05) - self.assertLess(elapsed, 5.0) - # The wait path itself is pinned by the scripted timeout tests; a - # real deadline cannot guarantee the loop reached a wait first. - (waiter,) = _RecordingWaiter.instances - self.assertEqual(waiter.closed, 1) - self.assertNoFailureAccounted() - - def test_supersession_interrupts_a_blocked_read_on_the_cadence(self) -> None: - control = _Control() - threading.Timer(0.05, control.cancel_event.set).start() - started = time.monotonic() - - with self.assertRaisesRegex(_Superseded, "canceled after supersession"): - self.compiler._serve_builder_read_line( - self.client, self.deadline, None, control - ) - - self.assertLess(time.monotonic() - started, 5.0) - self.assertNoFailureAccounted() - self.assertEqual(_RecordingWaiter.instances[0].closed, 1) - - def test_cancellation_interrupts_a_blocked_write_on_the_cadence(self) -> None: - cancellation = _Cancellation() - stdin_write = self.client.process.stdin.fileno() - # Fill the pipe so the transport must wait for the peer to drain. - filler = b"\0" * (64 << 10) - while True: - try: - os.write(stdin_write, filler) - except BlockingIOError: - break - threading.Timer(0.05, cancellation.event.set).start() - started = time.monotonic() - - with self.assertRaises(_Cancelled): - self.compiler._serve_builder_write( - self.client, b"more", self.deadline, cancellation, None - ) - - self.assertLess(time.monotonic() - started, 5.0) - self.assertNoFailureAccounted() - self.assertEqual(_RecordingWaiter.instances[0].closed, 1) - - def test_daemon_exit_mid_frame_is_one_crash(self) -> None: - os.write(self.stdout_write, b"only-part") - self._close_fd(self.stdout_write) - - with self.assertRaisesRegex(_ServeBuilderUnavailable, "exited mid-request"): - self.compiler._serve_builder_read_exact( - self.client, 200, self.deadline, None, None - ) - - self.assertOneCrashAccounted() - self.assertEqual(_RecordingWaiter.instances[0].closed, 1) - - def test_daemon_reader_gone_is_closed_input(self) -> None: - self._close_fd(self.stdin_read) - - with self.assertRaisesRegex(_ServeBuilderUnavailable, "input pipe closed"): - self.compiler._serve_builder_write( - self.client, b"request\n", self.deadline, None, None - ) - - self.assertNoFailureAccounted() - self.assertEqual(_RecordingWaiter.instances[0].closed, 1) - - def test_daemon_reader_gone_while_blocked_wakes_the_writer(self) -> None: - stdin_write = self.client.process.stdin.fileno() - filler = b"\0" * (64 << 10) - while True: - try: - os.write(stdin_write, filler) - except BlockingIOError: - break - stdin_read = self.stdin_read - threading.Timer(0.05, lambda: self._close_fd(stdin_read)).start() - started = time.monotonic() - - with self.assertRaisesRegex(_ServeBuilderUnavailable, "input pipe closed"): - self.compiler._serve_builder_write( - self.client, b"more", self.deadline, None, None - ) - - self.assertLess(time.monotonic() - started, 5.0) - self.assertNoFailureAccounted() - - -# Echoes its stdin payload like the echo builder, but reads nothing until -# the marker file named by PRISM_TEST_READER_GATE exists: the coordinator's -# writer therefore fills the pipe and blocks by construction, and the gated -# waiter creates the marker on that first wait. -GATED_READER_BUILDER_COMMAND = [ - sys.executable, - "-c", - ( - "import json, os, sys, time\n" - "marker = os.environ.get('PRISM_TEST_READER_GATE')\n" - "while marker and not os.path.exists(marker):\n" - " time.sleep(0.001)\n" - "data = bytearray()\n" - "while True:\n" - " chunk = sys.stdin.buffer.read(65536)\n" - " if not chunk:\n" - " break\n" - " data += chunk\n" - "json.dump({'received': json.loads(bytes(data)), 'transport': 'one-shot'}," - " sys.stdout)\n" - ), -] - - -class OneShotWriterTests(unittest.TestCase): - """The one-shot subprocess writer waits for readiness like the daemon.""" - - def _coordinator(self): - server, _rpc = coordinator() - server.signing_seed_hex = "42" * 32 - server.ledger_attestation_signing_seed_hex = "43" * 32 - self.addCleanup(server.shutdown_serve_builder) - self.addCleanup(server.retire_share_window_spool) - return server - - def _serialization(self, server, shares): - return server._share_window_serialization_for_artifact( - PayoutLedgerArtifact( - generation=1, - payout_state_generation=0, - network_difficulty=1, - accepted_share_count=len(shares), - shares_json=tuple(shares), - prior_balances=(), - prepared_monotonic=time.monotonic(), - snapshot_anchor_ms=None, - ), - shares, - ) - - def _gate(self) -> str: - directory = tempfile.mkdtemp(prefix="prism-reader-gate-") - self.addCleanup(lambda: __import__("shutil").rmtree(directory, ignore_errors=True)) - return os.path.join(directory, "go") - - def _build(self, server, shares, *, serialization=None, command=None, gate=None): - environment = {"PRISM_BUILDER_SERVE": "0"} - if gate is not None: - environment["PRISM_TEST_READER_GATE"] = gate - _GatedWaiter.arm(gate) - waiter_class: type = _GatedWaiter - else: - _RecordingWaiter.instances = [] - waiter_class = _RecordingWaiter - with patch.dict(os.environ, environment), patch( - "lab.prism.prism_coordinator.prism_tool_command", - return_value=list(command or GATED_READER_BUILDER_COMMAND), - ), patch.object(compiler_module, "_PipeReadinessWaiter", waiter_class): - return server.build_audit_bundle( - shares=shares, - found_block={ - "block_height": 10, - "coinbase_value_sats": 50_00000000, - "network_difficulty": 1, - "anchor_job_issued_at_ms": 1_700_000_000_000, - }, - prior_balances=[], - coinbase_script_sig_suffix_hex="00", - summary_only=True, - payout_policy={"policy": "day-one"}, - share_serialization=serialization, - ) - - def test_buffered_writer_streams_to_a_gated_child_exactly(self) -> None: - server = self._coordinator() - server.bundle_build_timeout_seconds = 30.0 - # Well over one pipe capacity, so the fragment writer must block - # before the child is released to read anything. - shares = [spool_share(seq) for seq in range(1, 6_001)] - gate = self._gate() - - result = self._build(server, shares, gate=gate) - - self.assertEqual(result["transport"], "one-shot") - self.assertEqual(len(result["received"]["compact_shares"]), 6_000) - self.assertEqual(result["received"]["found_block"]["block_height"], 10) - self.assertTrue(os.path.exists(gate)) - (waiter,) = _GatedWaiter.instances - self.assertEqual(waiter._events, selectors.EVENT_WRITE) - self.assertGreaterEqual(waiter.waits, 1) - self.assertEqual(waiter.closed, 1) - with server._job_build_scheduler_lock: - counts = dict(server.job_build_worker_counts) - self.assertEqual(counts["crashes"], 0) - self.assertEqual(counts["terminations"], 0) - - @unittest.skipUnless(hasattr(os, "splice"), "os.splice is Linux-only") - def test_spool_splice_streams_to_a_gated_child_exactly(self) -> None: - server = self._coordinator() - server.bundle_build_timeout_seconds = 30.0 - shares = [spool_share(seq) for seq in range(1, 6_001)] - serialization = self._serialization(server, shares) - gate = self._gate() - - result = self._build(server, shares, serialization=serialization, gate=gate) - - self.assertEqual(result["transport"], "one-shot") - self.assertEqual(len(result["received"]["compact_shares"]), 6_000) - self.assertFalse(serialization._spool_failed) - self.assertTrue(os.path.exists(gate)) - (waiter,) = _GatedWaiter.instances - self.assertGreaterEqual(waiter.waits, 1) - self.assertEqual(waiter.closed, 1) - - def test_child_that_stops_reading_fails_the_build_without_hanging(self) -> None: - server = self._coordinator() - server.bundle_build_timeout_seconds = 0.5 - shares = [spool_share(seq) for seq in range(1, 6_001)] - stalled = [ - sys.executable, - "-c", - "import sys, time\nsys.stdin.buffer.read(1024)\ntime.sleep(30)\n", - ] - started = time.monotonic() - - with self.assertRaisesRegex(RuntimeError, "timed out"): - self._build(server, shares, command=stalled) - - self.assertLess(time.monotonic() - started, 10.0) - (waiter,) = _RecordingWaiter.instances - self.assertGreaterEqual(waiter.waits, 1) - self.assertEqual(waiter.closed, 1) - self.assertEqual(server.tip_refresh_worker_failures, 1) - - -class DaemonRoundTripTests(unittest.TestCase): - """A large window through the real daemon protocol over real pipes.""" - - def test_large_window_upload_and_hit_over_subprocess_pipes(self) -> None: - server, _rpc = coordinator() - server.signing_seed_hex = "42" * 32 - server.ledger_attestation_signing_seed_hex = "43" * 32 - self.addCleanup(server.shutdown_serve_builder) - self.addCleanup(server.retire_share_window_spool) - shares = [spool_share(seq) for seq in range(1, 20_001)] - serialization = server._share_window_serialization_for_artifact( - PayoutLedgerArtifact( - generation=1, - payout_state_generation=0, - network_difficulty=1, - accepted_share_count=len(shares), - shares_json=tuple(shares), - prior_balances=(), - prepared_monotonic=time.monotonic(), - snapshot_anchor_ms=None, - ), - shares, - ) - _RecordingWaiter.instances = [] - - def build(height: int): - with patch.dict(os.environ, {"FAKE_SERVE_BUILDER_MODE": "ok"}), patch( - "lab.prism.prism_coordinator.prism_tool_command", - return_value=list(FAKE_SERVE_BUILDER_COMMAND), - ), patch.object( - compiler_module, - "_PipeReadinessWaiter", - _RecordingWaiter, - ): - return server.build_audit_bundle( - shares=shares, - found_block={ - "block_height": height, - "coinbase_value_sats": 50_00000000, - "network_difficulty": 1, - "anchor_job_issued_at_ms": 1_700_000_000_000, - }, - prior_balances=[], - coinbase_script_sig_suffix_hex="00", - summary_only=True, - payout_policy={"policy": "day-one"}, - share_serialization=serialization, - ) - - first = build(10) - second = build(11) - - self.assertEqual(first["transport"], "serve") - self.assertTrue(first["request_had_window"]) - self.assertEqual(len(first["window"]["compact_shares"]), 20_000) - self.assertEqual(second["transport"], "serve") - self.assertFalse(second["request_had_window"]) - self.assertEqual(len(second["window"]["compact_shares"]), 20_000) - with server._serve_builder_metrics_lock: - counts = dict(server.serve_builder_counts) - self.assertEqual(counts["requests"], 2) - self.assertEqual(counts["fallbacks"], 0) - # Every transport operation registered and released its waiter; how - # often the fake daemon's parse outran the coordinator is scheduling - # and is not asserted. - self.assertTrue(_RecordingWaiter.instances) - self.assertTrue(all(w.closed == 1 for w in _RecordingWaiter.instances)) - - -if __name__ == "__main__": - unittest.main() diff --git a/tests/test_prism_candidate_codec.py b/tests/test_prism_candidate_codec.py deleted file mode 100644 index 246a2b8c3..000000000 --- a/tests/test_prism_candidate_codec.py +++ /dev/null @@ -1,457 +0,0 @@ -#!/usr/bin/env python3 -"""Bounded candidate codec regressions (issue #255). - -Two contracts are pinned here. First, byte identity with the historical -v1 identity oracle: for every valid fixture the chunked body concatenates -to exactly ``json.dumps(block_candidate_identity(intent), sort_keys=True, -separators=(",", ":"))`` and ``candidate_sha256`` is the oracle's digest. -Second, boundedness by mechanism, not by timing: every ``json.dumps`` call -the encoder makes is instrumented and asserted to cover at most the batch -record cap, and every chunk it emits is asserted to be the fixed chunk -size except the last. Nothing here measures elapsed time. -""" - -from __future__ import annotations - -import hashlib -import json -import os -import sys -import tempfile -import unittest -from pathlib import Path -from typing import Any -from unittest import mock - -sys.path.insert(0, str(Path(__file__).resolve().parents[1])) - -from lab.prism import candidate_codec as codec # noqa: E402 -from lab.prism.share_ledger import ( # noqa: E402 - AcceptedShareRecord, - DaemonShareJsonSequence, - IncrementalShareJsonSequence, - _IncrementalShareWindowPage, - block_candidate_identity, - block_candidate_identity_sha256, - sha256_json_hex, -) - -HASH_A = "aa" * 32 -PARENT = "bb" * 32 -RUN_SLOW = os.environ.get("PRISM_CANDIDATE_CODEC_SLOW") == "1" - - -def oracle_bytes(intent: dict[str, Any]) -> bytes: - """The historical identity JSON, computed the historical way.""" - identity = block_candidate_identity({**intent, "block_hash_hex": intent["block_hash_hex"].lower()}) - return json.dumps(identity, sort_keys=True, separators=(",", ":")).encode("utf-8") - - -def share(index: int) -> dict[str, Any]: - return { - "share_seq": index, - "share_id": f"miner-{index % 7}:é\U0001F600{'x' * (index % 3)}", - "miner_id": f"m{index % 7}", - "order_key": "k", - "p2mr_program_hex": "ab" * 32, - "share_difficulty": 10**30 + index, - "network_difficulty": 1.5e16 if index % 5 else 3, - "template_height": 5, - "job_id": 'j"q\\', - "job_issued_at_ms": 1_700_000_000_000 + index, - "accepted_at_ms": 1_700_000_000_001 + index, - "ntime": 1_700_000_000, - "credit_policy": None if index % 2 else "stale-grace", - } - - -def intent_for(share_count: int, **overrides: Any) -> dict[str, Any]: - fields: dict[str, Any] = { - "schema": codec.CANDIDATE_INTENT_SCHEMA, - "block_hash_hex": HASH_A.upper(), - "block_hex": "00" * 100, - "coinbase_tx_hex": "01", - "parent_hash": PARENT, - "expected_height": 10, - "template": {"previousblockhash": PARENT, "height": 10, "coinbasevalue": 5}, - "shares_json": [share(index) for index in range(share_count)], - "prior_balances": [{"recipient_id": "a", "balance_sats": 1}], - "found_block": {"network_difficulty": 1.0, "witness": [1, 2, {"z": None}]}, - "prospective_prior_balances": None, - "witness_merkle_leaves_hex": ["cc" * 32] * 3, - "extranonce1_hex": "00", - "extranonce2_hex": "01", - "username": "u", - "pending_share": {"share_id": "s", "accepted_at_ms": 123}, - "credit_share_on_accept": False, - "collection_only": False, - } - fields.update(overrides) - return fields - - -class _Instrumented: - """Instrument ``json.dumps`` inside the codec and collect chunk facts.""" - - def __init__(self) -> None: - self.dumps_inputs: list[int] = [] - self.dumps_outputs: list[int] = [] - self.chunks: list[codec.BodyChunk] = [] - self._real = json.dumps - - def dumps(self, value: Any, *args: Any, **kwargs: Any) -> str: - text = self._real(value, *args, **kwargs) - self.dumps_inputs.append(len(value) if isinstance(value, (list, tuple)) else 1) - self.dumps_outputs.append(len(text)) - return text - - def collect(self, chunk: codec.BodyChunk) -> None: - self.chunks.append(chunk) - - -class CodecByteIdentityTests(unittest.TestCase): - def assert_body_matches_oracle(self, fields: dict[str, Any], *, chunk_bytes: int = 4096) -> codec.PreparedCandidateIntent: - prepared = codec.prepare_candidate_intent(fields, chunk_bytes=chunk_bytes) - got = bytearray() - chunks: list[codec.BodyChunk] = [] - - def collect(chunk: codec.BodyChunk) -> None: - chunks.append(chunk) - got.extend(chunk.data) - - prepared.body.write_chunks(collect) - expected = oracle_bytes(fields) - self.assertEqual(bytes(got), expected) - self.assertEqual(prepared.candidate_sha256, hashlib.sha256(expected).hexdigest()) - self.assertEqual(prepared.candidate_sha256, sha256_json_hex(block_candidate_identity({**fields, "block_hash_hex": HASH_A}))) - self.assertEqual(block_candidate_identity_sha256(prepared), prepared.candidate_sha256) - # Fixed chunk boundaries: every chunk but the last is exactly the size. - self.assertEqual([chunk.ordinal for chunk in chunks], list(range(len(chunks)))) - self.assertTrue(all(len(chunk.data) == chunk_bytes for chunk in chunks[:-1])) - self.assertEqual(sum(len(chunk.data) for chunk in chunks), prepared.manifest.byte_count) - self.assertEqual(len(chunks), prepared.manifest.chunk_count) - for chunk in chunks: - self.assertEqual(hashlib.sha256(chunk.data).hexdigest(), chunk.sha256) - return prepared - - def test_share_counts_around_batch_boundaries(self) -> None: - for count in (0, 1, 255, 256, 257, 511, 512, 513, 1300): - with self.subTest(count=count): - prepared = self.assert_body_matches_oracle(intent_for(count)) - self.assertEqual(prepared.manifest.share_count, count) - self.assertIs(prepared["shares_json"], prepared.shares) - self.assertEqual(prepared["pending_share"]["accepted_at_ms"], 123) - self.assertTrue(prepared.accepted_at_present) - - @unittest.skipUnless(RUN_SLOW, "set PRISM_CANDIDATE_CODEC_SLOW=1 for the 375,000-share oracle") - def test_stress_window_matches_oracle(self) -> None: - prepared = self.assert_body_matches_oracle(intent_for(375_000), chunk_bytes=codec.CANDIDATE_BODY_CHUNK_BYTES) - self.assertEqual(prepared.manifest.share_count, 375_000) - - def test_giant_string_field_spans_chunk_and_slice_boundaries(self) -> None: - big = ("é\"\\" + "x" * 1000 + "\U0001F600") * 300 - prepared = self.assert_body_matches_oracle(intent_for(3, block_hex=big), chunk_bytes=1000) - self.assertGreater(prepared.manifest.span_count, 1) - - def test_giant_field_inside_one_record(self) -> None: - fields = intent_for(600) - fields["shares_json"][300]["share_id"] = "y" * (5 * 1024 * 1024) - self.assert_body_matches_oracle(fields, chunk_bytes=codec.CANDIDATE_BODY_CHUNK_BYTES) - - def test_first_oversized_record_is_bounded_before_encoding(self) -> None: - fields = intent_for(260) - fields["shares_json"][0]["share_id"] = "😀" * 100_000 - fields["shares_json"][257]["share_id"] = "x" * (5 * 1024 * 1024) - expected = oracle_bytes(fields) - observer = _Instrumented() - with mock.patch.object(codec.json, "dumps", observer.dumps): - prepared = codec.prepare_candidate_intent(fields) - prepared.body.write_chunks(observer.collect) - self.assertEqual(b"".join(chunk.data for chunk in observer.chunks), expected) - self.assertLessEqual(max(observer.dumps_outputs), codec.CODEC_BATCH_TARGET_BYTES) - - def test_large_integers_floats_nulls_and_field_order(self) -> None: - fields = intent_for( - 2, - found_block={"network_difficulty": 2**200, "float": 1e300, "tiny": 5e-324, "neg": -0.0, "none": None}, - extension_zzz={"b": [1, {"a": None}], "a": ""}, - extension_aaa=[[], {}, "", 0, False], - ) - prepared = self.assert_body_matches_oracle(fields) - self.assertEqual(list(prepared), list(fields)) - - def test_numeric_keys_and_oversized_nested_key_keep_bounded_encoding(self) -> None: - fields = intent_for(1, extension={1: "x" * 300_000, 2.5: ["y" * 300_000]}, - found_block={"z" * 300_000: True}) - expected = oracle_bytes(fields) - observer = _Instrumented() - with mock.patch.object(codec.json, "dumps", observer.dumps): - prepared = codec.prepare_candidate_intent(fields) - prepared.body.write_chunks(observer.collect) - self.assertEqual(b"".join(chunk.data for chunk in observer.chunks), expected) - self.assertLessEqual(max(observer.dumps_outputs), codec.CODEC_BATCH_TARGET_BYTES) - - def test_absent_and_null_stamp_are_distinct(self) -> None: - absent = intent_for(2) - del absent["pending_share"]["accepted_at_ms"] - prepared_absent = self.assert_body_matches_oracle(absent) - self.assertFalse(prepared_absent.accepted_at_present) - null = intent_for(2, pending_share={"share_id": "s", "accepted_at_ms": None}) - prepared_null = self.assert_body_matches_oracle(null) - self.assertTrue(prepared_null.accepted_at_present) - self.assertNotEqual(prepared_absent.candidate_sha256, prepared_null.candidate_sha256) - - def test_stamp_drift_keeps_identity(self) -> None: - first = codec.prepare_candidate_intent(intent_for(3)) - second = codec.prepare_candidate_intent(intent_for(3, pending_share={"share_id": "s", "accepted_at_ms": 999})) - self.assertEqual(first.candidate_sha256, second.candidate_sha256) - self.assertEqual(second.accepted_at_ms, 999) - - def test_candidate_only_intent_without_shares(self) -> None: - fields = {"schema": codec.CANDIDATE_INTENT_SCHEMA, "block_hash_hex": HASH_A, "block_hex": "00"} - prepared = self.assert_body_matches_oracle(fields) - self.assertFalse(prepared.has_shares) - self.assertNotIn("shares_json", prepared) - - def test_empty_and_large_balances_and_witness_arrays(self) -> None: - empty = intent_for(1, prior_balances=[], witness_merkle_leaves_hex=[], prospective_prior_balances=[]) - self.assert_body_matches_oracle(empty) - large = intent_for( - 1, - prior_balances=[{"recipient_id": f"r{index}", "balance_sats": index} for index in range(5000)], - prospective_prior_balances=[["a", "b", "c", index] for index in range(5000)], - witness_merkle_leaves_hex=["cc" * 32] * 20_000, - ) - prepared = self.assert_body_matches_oracle(large) - self.assertGreaterEqual(prepared.manifest.span_count, 4) - - -class CodecRejectionTests(unittest.TestCase): - def test_nan_and_infinity_are_rejected_before_credit(self) -> None: - for value in (float("nan"), float("inf"), -float("inf")): - with self.subTest(value=value), self.assertRaises(codec.CandidateCodecError): - codec.prepare_candidate_intent(intent_for(1, found_block={"network_difficulty": value})) - with self.subTest(value=value, where="share"), self.assertRaises(codec.CandidateCodecError): - codec.prepare_candidate_intent(intent_for(0, shares_json=[{"a": value}])) - - def test_unsupported_types_raise_type_error_like_json_dumps(self) -> None: - for value in (b"bytes", {1, 2}, object()): - with self.subTest(value=type(value).__name__), self.assertRaises(TypeError): - codec.prepare_candidate_intent(intent_for(1, found_block={"blob": value})) - - def test_jsonb_incompatible_text_is_rejected_everywhere(self) -> None: - nul = chr(0) - high, low = "\ud83d", "\ude00" - cases = { - "nul in field": intent_for(1, username="a" + nul + "b"), - "nul in key": intent_for(1, **{"ext" + nul: 1}), - "nul in share": intent_for(0, shares_json=[{"a": "x" + nul}]), - "nul in giant": intent_for(1, block_hex="y" * 70000 + nul), - "nul in nested share list": intent_for(0, shares_json=[{"a": [nul]}]), - "lone high": intent_for(1, username=high), - "lone low in share": intent_for(0, shares_json=[{"a": low}]), - "lone high in giant": intent_for(1, block_hex="y" * 70000 + high), - "nul in non-string-key dict": intent_for(1, found_block={1: "x" + nul}), - "lone in non-string-key dict": intent_for(1, found_block={1: high}), - "high then high": intent_for(0, shares_json=[{"a": high + high + low}]), - } - for label, fields in cases.items(): - with self.subTest(label=label), self.assertRaises(codec.CandidateCodecError): - codec.prepare_candidate_intent(fields) - - def test_jsonb_compatible_escapes_are_accepted(self) -> None: - high, low = "\ud83d", "\ude00" - for label, fields in { - "valid pair": intent_for(1, username="\U0001F600", shares_json=[{"a": "\U0001F600"}]), - "adjacent lone units forming a pair": intent_for(1, username=high + low), - "escaped backslash-u text": intent_for(1, username="\\u0000", shares_json=[{"a": "\\u0000", "b": "\\\\ud83d"}]), - "pair at slice edge": intent_for(1, block_hex="y" * 65535 + "\U0001F600" + "z"), - "literal surrogate pair at slice edge": intent_for(1, block_hex="y" * (codec.CODEC_STRING_SLICE_CHARS - 1) + high + low + "z"), - }.items(): - with self.subTest(label=label): - prepared = codec.prepare_candidate_intent(fields) - got = bytearray() - prepared.body.write_chunks(lambda chunk: got.extend(chunk.data)) - self.assertEqual(bytes(got), oracle_bytes(fields)) - - def test_missing_hash_and_bad_share_container(self) -> None: - with self.assertRaises(codec.CandidateCodecError): - codec.prepare_candidate_intent({"schema": "x", "shares_json": []}) - with self.assertRaises(codec.CandidateCodecError): - codec.prepare_candidate_intent(intent_for(0, shares_json="not a sequence")) - - def test_changed_source_cannot_publish(self) -> None: - fields = intent_for(5) - prepared = codec.prepare_candidate_intent(fields) - fields["shares_json"].append(share(99)) - with self.assertRaises(codec.CandidateBodyIntegrityError): - prepared.body.write_chunks(lambda chunk: None) - - def test_mutated_prepared_intent_is_re_prepared(self) -> None: - prepared = codec.prepare_candidate_intent(intent_for(2)) - original = prepared.candidate_sha256 - prepared.pop("collection_only") - self.assertTrue(prepared.dirty) - refreshed = codec.prepare_candidate_intent(prepared) - self.assertNotEqual(refreshed.candidate_sha256, original) - self.assertEqual(refreshed.candidate_sha256, hashlib.sha256(oracle_bytes(dict(prepared))).hexdigest()) - - -class CodecBoundednessTests(unittest.TestCase): - def test_every_dumps_call_and_chunk_is_bounded(self) -> None: - fields = intent_for(3000, prior_balances=[{"recipient_id": f"r{i}", "balance_sats": i} for i in range(3000)]) - probe = _Instrumented() - with mock.patch.object(codec.json, "dumps", probe.dumps): - prepared = codec.prepare_candidate_intent(fields, chunk_bytes=codec.CANDIDATE_BODY_CHUNK_BYTES) - prepared.body.write_chunks(probe.collect) - self.assertTrue(probe.dumps_inputs) - self.assertLessEqual(max(probe.dumps_inputs), codec.CODEC_BATCH_RECORDS) - self.assertLessEqual(max(probe.dumps_outputs), codec.CODEC_BATCH_OVERSIZED_BYTES) - self.assertTrue(all(len(chunk.data) <= codec.CANDIDATE_BODY_CHUNK_BYTES for chunk in probe.chunks)) - - def test_daemon_sequence_is_copied_verbatim_without_parsing(self) -> None: - records = [share(index) for index in range(2000)] - records[700]["share_id"] = 'trap},{"accepted_at_ms":1' - items = b",".join(json.dumps(record, sort_keys=True, separators=(",", ":"), default=str).encode() for record in records) - daemon = DaemonShareJsonSequence(items, len(records)) - fields = intent_for(0, shares_json=daemon) - prepared = codec.prepare_candidate_intent(fields, chunk_bytes=4096) - got = bytearray() - prepared.body.write_chunks(lambda chunk: got.extend(chunk.data)) - self.assertIsNone(daemon._parsed, "the daemon sequence must not be parsed to stage it") - self.assertEqual(bytes(got), oracle_bytes(intent_for(0, shares_json=records))) - self.assertEqual(prepared.manifest.share_count, 2000) - - def test_page_backed_window_pages_are_copied_verbatim(self) -> None: - records = tuple( - AcceptedShareRecord(share_seq=i, share_id=f"s{i}", miner_id="m", order_key="k", p2mr_program_hex="ab" * 32, share_difficulty=1, network_difficulty=2, template_height=3, job_id="j", job_issued_at_ms=4, accepted_at_ms=5, ntime=6) - for i in range(1100) - ) - pages = tuple(_IncrementalShareWindowPage.from_records(records[i : i + 512]) for i in range(0, 1100, 512)) - window = IncrementalShareJsonSequence(pages=pages, record_count=1100) - prepared = codec.prepare_candidate_intent(intent_for(0, shares_json=window), chunk_bytes=4096) - got = bytearray() - prepared.body.write_chunks(lambda chunk: got.extend(chunk.data)) - self.assertEqual(bytes(got), oracle_bytes(intent_for(0, shares_json=list(window)))) - self.assertGreaterEqual(prepared.manifest.page_count, 5) - - def test_replay_header_is_bounded_and_defers_oversized_facts(self) -> None: - bounded = codec.replay_header_from_fields(intent_for(1)) - self.assertFalse(bounded["oversized"]) - self.assertEqual(bounded["pending_share"]["accepted_at_ms"], 123) - oversized = codec.replay_header_from_fields(intent_for(1, username="u" * 1000, pending_share={"share_id": "s" * 1000, "accepted_at_ms": 123})) - self.assertTrue(oversized["oversized"]) - self.assertIsNone(oversized["username"]) - self.assertIsNone(oversized["pending_share"]["share_id"]) - self.assertEqual(oversized["pending_share"]["accepted_at_ms"], 123) - self.assertLess(len(json.dumps(oversized)), 4096) - - def test_replay_header_rejects_giant_text_before_encoding_it(self) -> None: - observer = _Instrumented() - fields = intent_for(1, username="u" * (5 * 1024 * 1024)) - with mock.patch.object(codec.json, "dumps", observer.dumps): - header = codec.replay_header_from_fields(fields) - self.assertIsNone(header["username"]) - self.assertLessEqual(max(observer.dumps_outputs), 4096) - - -class SpoolRoundTripTests(unittest.TestCase): - def _spool(self, prepared: codec.PreparedCandidateIntent) -> codec.SpoolCandidateBody: - directory = tempfile.mkdtemp() - self.addCleanup(lambda: [os.unlink(os.path.join(directory, name)) for name in os.listdir(directory)] and os.rmdir(directory) if os.path.isdir(directory) else None) - return codec.write_spool_from_body(os.path.join(directory, "b.body"), os.path.join(directory, "b.idx"), prepared.body) - - def test_pages_decode_and_random_access_use_the_disk_index(self) -> None: - for count in (0, 1, 256, 1300): - with self.subTest(count=count): - fields = intent_for(count, prior_balances=[{"recipient_id": f"r{i}", "balance_sats": i} for i in range(4000 if count == 1300 else 1)]) - prepared = codec.prepare_candidate_intent(fields, chunk_bytes=4096) - spool = self._spool(prepared) - hydrated = codec.prepared_intent_from_spool(spool, accepted_at_present=True, accepted_at_ms=123) - shares = hydrated["shares_json"] - self.assertEqual(len(shares), count) - self.assertEqual(list(shares), fields["shares_json"]) - if count: - self.assertEqual(shares[0], fields["shares_json"][0]) - self.assertEqual(shares[-1], fields["shares_json"][-1]) - self.assertEqual(shares[count // 2], fields["shares_json"][count // 2]) - self.assertEqual(shares.canonical_json_sha256(), hashlib.sha256(json.dumps(fields["shares_json"], sort_keys=True, separators=(",", ":"), default=str).encode()).hexdigest()) - self.assertEqual(list(hydrated["prior_balances"]), fields["prior_balances"]) - if count == 1300: - self.assertIsInstance(hydrated["prior_balances"], codec.SpoolJsonArraySequence) - self.assertEqual(hydrated["pending_share"]["accepted_at_ms"], 123) - regot = bytearray() - hydrated.body.write_chunks(lambda chunk: regot.extend(chunk.data)) - self.assertEqual(bytes(regot), oracle_bytes(fields)) - self.assertEqual(codec.prepare_candidate_intent(dict(hydrated), chunk_bytes=4096).candidate_sha256, prepared.candidate_sha256) - paths = (spool.path, spool.index.path) - spool.close() - self.assertFalse(any(os.path.exists(path) for path in paths)) - - def test_daemon_pages_ignore_quoted_and_nested_boundary_traps(self) -> None: - records = [share(index) for index in range(2000)] - records[700]["share_id"] = 'trap},{"accepted_at_ms":1' - records[511]["nested"] = [{"a": 1}, {"b": "\\\"},{"}] - items = b",".join(json.dumps(record, sort_keys=True, separators=(",", ":"), default=str).encode() for record in records) - prepared = codec.prepare_candidate_intent(intent_for(0, shares_json=DaemonShareJsonSequence(items, 2000)), chunk_bytes=4096) - spool = self._spool(prepared) - span = spool.index.span("shares_json") - self.assertTrue(span.pages_exact) - sequence = codec.SpoolJsonArraySequence(spool, span) - self.assertEqual(sequence[1999], records[1999]) - self.assertEqual(list(sequence), records) - self.assertTrue(spool.index.span("shares_json").pages_exact) - self.assertEqual(sequence[700], records[700]) - spool.close() - - def test_giant_string_and_oversized_record_decode_by_slices(self) -> None: - big = ("é\"\\" + "x" * 1000 + "\U0001F600") * 300 - fields = intent_for(600, block_hex=big) - fields["shares_json"][300]["share_id"] = "y" * (5 * 1024 * 1024) - prepared = codec.prepare_candidate_intent(fields, chunk_bytes=codec.CANDIDATE_BODY_CHUNK_BYTES) - spool = self._spool(prepared) - hydrated = codec.prepared_intent_from_spool(spool, accepted_at_present=True, accepted_at_ms=123) - self.assertEqual(hydrated["block_hex"], big) - self.assertEqual(hydrated["shares_json"][300], fields["shares_json"][300]) - spool.close() - - def test_truncated_and_corrupt_spools_fail_closed(self) -> None: - prepared = codec.prepare_candidate_intent(intent_for(300), chunk_bytes=4096) - spool = self._spool(prepared) - with open(spool.path, "r+b") as handle: - handle.seek(10) - handle.write(b"Z") - with self.assertRaises(codec.CandidateBodyIntegrityError): - list(spool.iter_chunks()) - with open(spool.path, "r+b") as handle: - handle.truncate(100) - with self.assertRaises(codec.CandidateBodyIntegrityError): - list(spool.iter_chunks()) - spool.close() - - def test_manifest_and_span_validation(self) -> None: - manifest = codec.prepare_candidate_intent(intent_for(3)).manifest - self.assertEqual(codec.CandidateBodyManifest.from_json(manifest.to_json()), manifest) - for broken in ( - {**manifest.to_json(), "chunk_count": manifest.chunk_count + 1}, - {**manifest.to_json(), "storage_version": 3}, - {**manifest.to_json(), "shares_end": manifest.byte_count + 1}, - {**manifest.to_json(), "candidate_sha256": "zz"}, - ): - with self.subTest(broken=broken), self.assertRaises(codec.CandidateBodyIntegrityError): - codec.CandidateBodyManifest.from_json(broken) - with self.assertRaises(codec.CandidateBodyIntegrityError): - codec.FieldSpan.from_json({"field": "x", "kind": "blob", "start": 0, "end": 1}) - - def test_jsonb_equivalence_matches_postgres_rules(self) -> None: - from decimal import Decimal - - self.assertTrue(codec.jsonb_equivalent({"a": Decimal("1.0"), "b": [1, 2]}, {"b": [Decimal(1), 2], "a": 1})) - self.assertTrue(codec.jsonb_equivalent(Decimal("1E+16"), 10**16)) - self.assertFalse(codec.jsonb_equivalent(True, 1)) - self.assertFalse(codec.jsonb_equivalent({"a": 1}, {"a": 1, "b": None})) - self.assertFalse(codec.jsonb_equivalent([1, 2], [2, 1])) - - -if __name__ == "__main__": - unittest.main() diff --git a/tests/test_prism_candidate_helper.py b/tests/test_prism_candidate_helper.py deleted file mode 100644 index 3c8f58212..000000000 --- a/tests/test_prism_candidate_helper.py +++ /dev/null @@ -1,75 +0,0 @@ -"""Exercise helper backpressure and shutdown with real child processes.""" - -from __future__ import annotations - -import subprocess -import sys -import threading -import time -import unittest -from unittest import mock - -from lab.prism import candidate_store as store -from lab.prism.candidate_codec import prepare_candidate_intent - - -class CandidateHelperLifecycleTests(unittest.TestCase): - def helper(self, **kwargs): - return store.LegacyCandidateHelper( - store.LegacyTransport(database_url="unused", psql_command=()), **kwargs - ) - - def test_stalled_input_obeys_deadline_and_reaps_child(self): - body = prepare_candidate_intent({ - "block_hash_hex": "aa" * 32, "shares_json": [], "pad": "x" * (2 * 1024 * 1024), - }).body - real_popen = subprocess.Popen - children = [] - - def sleeping_child(_args, **kwargs): - process = real_popen([sys.executable, "-c", "import time; time.sleep(60)"], **kwargs) - children.append(process) - return process - - started = time.monotonic() - with mock.patch.object(store.subprocess, "Popen", sleeping_child): - with self.assertRaisesRegex(store.CandidateStorageError, "deadline"): - self.helper(timeout_seconds=0.2).compare("aa" * 32, body) - self.assertLess(time.monotonic() - started, 3.0) - self.assertEqual(len(children), 1) - self.assertIsNotNone(children[0].poll()) - self.assertTrue(all(pipe.closed for pipe in (children[0].stdin, children[0].stdout, children[0].stderr))) - self.assertFalse(any(t.name.startswith("prism-legacy-helper-") for t in threading.enumerate())) - - def test_cancellation_wrapper_waits_for_same_admission_slot(self): - cancelled = threading.Event() - helper = self.helper(timeout_seconds=1.0) - wrapper = helper.with_cancellation(cancelled.is_set) - helper._lock.acquire() - cancelled.set() - try: - with mock.patch.object(store.subprocess, "Popen") as spawn: - with self.assertRaisesRegex(store.CandidateStorageError, "cancelled"): - wrapper.convert("aa" * 32, "unused.body", "unused.idx") - spawn.assert_not_called() - finally: - helper._lock.release() - - def test_child_diagnostic_backpressure_does_not_deadlock(self): - real_popen = subprocess.Popen - - def noisy_child(_args, **kwargs): - return real_popen([ - sys.executable, "-c", - "import sys; sys.stdin.readline(); " - "sys.stderr.write('x' * 1000000); sys.stderr.flush(); " - "sys.stdout.write('{\"equal\":true}\\n'); sys.stdout.flush(); sys.stdin.read()", - ], **kwargs) - - body = prepare_candidate_intent({"block_hash_hex": "aa" * 32, "shares_json": []}).body - with mock.patch.object(store.subprocess, "Popen", noisy_child): - self.assertTrue(self.helper(timeout_seconds=3.0).compare("aa" * 32, body)) - - -if __name__ == "__main__": - unittest.main() diff --git a/tests/test_prism_candidate_replay_bounds.py b/tests/test_prism_candidate_replay_bounds.py deleted file mode 100644 index 221ba629f..000000000 --- a/tests/test_prism_candidate_replay_bounds.py +++ /dev/null @@ -1,119 +0,0 @@ -"""Descriptor backpressure bounds obligations and preserves complete replay.""" - -from types import SimpleNamespace -import unittest -from unittest import mock - -from lab.prism.candidate_codec import replay_header_from_fields -from lab.prism.candidate_store import CandidateHeaderPage -from tests.prism_postgres_candidate_gate import intent -from tests.prism_vardiff_test_support import submit_coordinator - - -class ReplayDescriptorBoundsTests(unittest.TestCase): - def test_overflow_pauses_before_adoption_and_replays_every_row_after_drain(self): - for page_size in (2, 1024): - with self.subTest(page_size=page_size): - server, _state, _recording = submit_coordinator() - service = server._ensure_block_candidate_service() - rows = [] - for index in range(1, 12): - fields = intent(f"{index:064x}", 0, credit=True) - rows.append({"block_hash": fields["block_hash_hex"], "storage_version": 1, - "candidate_sha256": "ab" * 32, "cursor": index, - "header": replay_header_from_fields(fields)}) - pending = list(rows) - queries = [] - - def headers(*, limit, after_cursor, max_bytes): - remaining = [row for row in pending if row["cursor"] > (after_cursor or 0)] - page = remaining[:limit] - queries.append((after_cursor, len(page))) - return CandidateHeaderPage(tuple(page), page[-1]["cursor"] if page else None, - len(remaining) <= limit, len(page), False) - - server.ledger = SimpleNamespace(candidate_hydration_deferred=True, - pending_block_candidate_headers=headers) - server.config = SimpleNamespace(block=SimpleNamespace(replay_page_size=page_size)) - server._run_block_submitter_ledger_call = lambda _key, _phase, call, **_kwargs: call() - service._collapse_superseded_block_candidates = lambda rows, **_kwargs: rows - previews = set() - server._begin_accepted_block_payout_preview = lambda key, **_kwargs: previews.add(key) - server._clear_accepted_block_payout_preview = previews.discard - server._finish_pending_share_candidate = mock.Mock() - server._note_block_replay_enumeration_owed() - seen = [] - writer = service.ports.share_writer() - with mock.patch.object(writer, "adopt_pending_share"), \ - mock.patch("lab.prism.block_candidates.MAX_BLOCK_REPLAY_DESCRIPTORS_IN_MEMORY", 3), \ - mock.patch("builtins.print"): - while pending: - queued = server.replay_pending_block_candidates() - self.assertEqual(queued, min(3, len(pending))) - self.assertLessEqual(len(service._block_replay_floor_holders), 3) - self.assertEqual(len(previews), queued) - if len(pending) > 3: - self.assertTrue(server._block_replay_enumeration_owed()) - before = len(queries) - self.assertEqual(server.replay_pending_block_candidates(), 0) - self.assertEqual(len(queries), before) - else: - self.assertFalse(server._block_replay_enumeration_owed()) - while not service._block_replay_candidate_queue.empty(): - descriptor = service._block_replay_candidate_queue.get_nowait() - seen.append(descriptor.block_hash) - pending[:] = [row for row in pending if row["block_hash"] != descriptor.block_hash] - service._drop_replay_descriptor(descriptor) - self.assertEqual(service._block_replay_floor_holders, {}) - self.assertEqual(previews, set()) - self.assertEqual(seen, [row["block_hash"] for row in rows]) - self.assertEqual(server._finish_pending_share_candidate.call_count, len(rows)) - - def test_oversized_header_is_registered_and_hydrates_the_body_pending_share(self): - # share_id is "<username>:<block hash>": at 200 bytes only share_id - # exceeds the header bound, at 300 bytes username does too. - for username_bytes in (200, 300): - with self.subTest(username_bytes=username_bytes): - server, _state, _recording = submit_coordinator() - service = server._ensure_block_candidate_service() - fields = intent("ab" * 32, 0, credit=True) - fields["username"] = "q" * username_bytes - fields["pending_share"]["share_id"] = f"{fields['username']}:{fields['block_hash_hex']}" - header = replay_header_from_fields(fields) - self.assertTrue(header["oversized"]) - self.assertIsNone(header["pending_share"]["share_id"]) - row = {"block_hash": fields["block_hash_hex"], "storage_version": 1, - "candidate_sha256": "ab" * 32, "cursor": 1, "header": header} - - def headers(*, limit, after_cursor, max_bytes): - page = (row,) if after_cursor is None else () - return CandidateHeaderPage(page, 1, True, len(page), False) - - server.ledger = SimpleNamespace( - candidate_hydration_deferred=True, - pending_block_candidate_headers=headers, - hydrate_block_candidate_intent=lambda _row, **_kwargs: fields, - ) - server.config = SimpleNamespace(block=SimpleNamespace(replay_page_size=8)) - server._run_block_submitter_ledger_call = lambda _key, _phase, call, **_kwargs: call() - server._block_landing_db_timeout = lambda: 5.0 - service._collapse_superseded_block_candidates = lambda rows, **_kwargs: rows - server._begin_accepted_block_payout_preview = mock.Mock() - server._queue_invalid_block_candidate_for_quarantine = mock.Mock() - server._finish_pending_share_candidate = mock.Mock() - server._note_block_replay_enumeration_owed() - writer = service.ports.share_writer() - with mock.patch.object(writer, "adopt_pending_share") as adopt, \ - mock.patch("builtins.print"): - self.assertEqual(server.replay_pending_block_candidates(), 1) - descriptor = service._block_replay_candidate_queue.get_nowait() - self.assertTrue(descriptor.header_oversized) - stand_in = service._block_replay_floor_holders[descriptor.block_hash] - candidate = service._hydrate_replay_descriptor(descriptor) - server._queue_invalid_block_candidate_for_quarantine.assert_not_called() - self.assertEqual(candidate.pending_share.share_id, fields["pending_share"]["share_id"]) - # The body's share takes the floor before the stand-in leaves it. - self.assertEqual([call.args[0] for call in adopt.call_args_list], - [stand_in, candidate.pending_share]) - server._finish_pending_share_candidate.assert_called_once_with(stand_in) - self.assertEqual(service._block_replay_floor_holders, {}) diff --git a/tests/test_prism_candidate_schema_refusal.py b/tests/test_prism_candidate_schema_refusal.py deleted file mode 100644 index d9e179260..000000000 --- a/tests/test_prism_candidate_schema_refusal.py +++ /dev/null @@ -1,82 +0,0 @@ -"""Every storage version refuses a database without the 002 migration (#255).""" - -from types import SimpleNamespace -import unittest -from unittest.mock import Mock - -from lab.prism import recover_pending_blocks as recovery -from lab.prism.candidate_store import IncompatibleCandidateSchema -from lab.prism.share_ledger import PsqlShareLedger - -# (has_body_table, declared capability): nothing migrated, and a partial -# migration whose capability row (declared last) never landed. -UNMIGRATED = ((False, None), (True, None)) - - -def ledger(storage_version, has_body_table, declared): - value = PsqlShareLedger.__new__(PsqlShareLedger) - value._candidate_storage_version = storage_version - - def run_json(sql): - if "to_regclass" in sql: - return {"has_body_table": has_body_table, "has_capabilities": declared is not None} - return {"declared": declared} - - value._run_json = run_json - return value - - -def reader(has_body_table, declared): - def execute(sql, _params=None): - if "to_regclass" in sql: - row = {"has_body_table": has_body_table, "has_capabilities": declared is not None} - else: - row = {"capability_value": declared} - return SimpleNamespace(fetchone=lambda: row) - - value = recovery.RecoveryReader.__new__(recovery.RecoveryReader) - value.connection = SimpleNamespace(execute=execute) - return value - - -class LedgerSchemaRefusalTests(unittest.TestCase): - def test_unmigrated_schema_is_refused_for_every_storage_version(self): - for storage_version in (1, 2): - for has_body_table, declared in UNMIGRATED: - with self.subTest(storage_version=storage_version, has_body_table=has_body_table), \ - self.assertRaisesRegex(IncompatibleCandidateSchema, "002_candidate_bodies.sql"): - ledger(storage_version, has_body_table, declared).verify_candidate_schema() - - def test_migrated_schema_is_accepted_and_a_newer_one_refused(self): - for storage_version in (1, 2): - with self.subTest(storage_version=storage_version): - self.assertEqual( - ledger(storage_version, True, 2).verify_candidate_schema(), - {"declared": 2, "has_body_table": True}, - ) - with self.assertRaisesRegex(IncompatibleCandidateSchema, "rollback floor"): - ledger(storage_version, True, 3).verify_candidate_schema() - - -class RecoverySchemaRefusalTests(unittest.TestCase): - def test_reader_applies_the_ledger_rule(self): - for has_body_table, declared in UNMIGRATED: - with self.subTest(has_body_table=has_body_table), \ - self.assertRaisesRegex(recovery.RecoveryError, "002_candidate_bodies.sql"): - reader(has_body_table, declared).require_candidate_schema() - with self.assertRaisesRegex(recovery.RecoveryError, "rollback floor"): - reader(True, 3).require_candidate_schema() - reader(True, 2).require_candidate_schema() - - def test_plan_refuses_an_unmigrated_database_before_reading_candidates(self): - legacy = reader(False, None) - legacy.metadata = Mock() - rpc = SimpleNamespace(call=Mock()) - with self.assertRaisesRegex(recovery.RecoveryError, "002_candidate_bodies.sql"): - recovery.plan_recovery(legacy, rpc, ["ab" * 32]) - legacy.metadata.assert_not_called() - rpc.call.assert_not_called() - - -if __name__ == "__main__": - unittest.main() diff --git a/tests/test_prism_candidate_spool_integration.py b/tests/test_prism_candidate_spool_integration.py deleted file mode 100644 index 30cbed82a..000000000 --- a/tests/test_prism_candidate_spool_integration.py +++ /dev/null @@ -1,80 +0,0 @@ -"""Lazy candidate fields remain usable by the replay and membership callers.""" - -import gc -import json -from pathlib import Path -import tempfile -import hashlib -import unittest -from unittest import mock -import weakref - -from lab.prism.block_candidates import block_candidate_from_intent -from lab.prism.candidate_codec import prepare_candidate_intent, prepared_intent_from_spool, write_spool_from_body -from lab.prism.candidate_window import copy_text_chunks, recorded_share_ids -from tests.prism_postgres_candidate_gate import intent - - -class CandidateSpoolIntegrationTests(unittest.TestCase): - def test_metadata_aggregate_and_giant_key_keep_native_decodes_bounded(self): - fields = intent("cd" * 32, 1, credit=False) - fields.update({f"extra_{index}": "x" * 24000 for index in range(60)}) - fields["found_block"]["k" * (2 * 1024 * 1024)] = True - prepared = prepare_candidate_intent(fields) - loads = json.loads - sizes = [] - - def bounded_loads(data, *args, **kwargs): - sizes.append(len(data)) - return loads(data, *args, **kwargs) - - with tempfile.TemporaryDirectory() as directory: - body = write_spool_from_body(str(Path(directory) / "body"), str(Path(directory) / "index"), prepared.body) - try: - with mock.patch("json.loads", side_effect=bounded_loads): - hydrated = prepared_intent_from_spool(body, accepted_at_present=True, accepted_at_ms=123) - self.assertEqual(hydrated["found_block"]["network_difficulty"], 1000) - self.assertEqual(hydrated["extra_59"], fields["extra_59"]) - self.assertLessEqual(max(sizes), 1024 * 1024) - finally: - body.close() - - def test_large_block_and_share_identifier_survive_replay_without_coercion(self): - fields = intent("ab" * 32, 3, credit=False) - fields["block_hex"] = "ab" * (1024 * 1024) - fields["shares_json"][0]["share_id"] = "id" * (1024 * 1024) - prepared = prepare_candidate_intent(fields) - enabled = gc.isenabled() - with tempfile.TemporaryDirectory() as directory: - gc.disable() - try: - body = write_spool_from_body(str(Path(directory) / "body"), str(Path(directory) / "index"), prepared.body) - observed = weakref.ref(body) - hydrated = prepared_intent_from_spool( - body, accepted_at_present=prepared.accepted_at_present, - accepted_at_ms=prepared.accepted_at_ms, - ) - candidate = block_candidate_from_intent(hydrated) - self.assertTrue(callable(getattr(candidate.submission.block_hex, "iter_byte_chunks", None))) - self.assertEqual(candidate.submission.block_hex.byte_length, len(fields["block_hex"]) + 2) - # A changed hydrated intent must re-encode its lazy fields, - # rather than accidentally reusing the original digest/body. - hydrated["username"] = "changed" - updated = prepare_candidate_intent(hydrated) - from tests.test_prism_candidate_codec import oracle_bytes - fields["username"] = "changed" - self.assertEqual(updated.candidate_sha256, hashlib.sha256(oracle_bytes(fields)).hexdigest()) - del updated - copied = b"".join(copy_text_chunks(recorded_share_ids(candidate.context.shares_json))) - expected = b"".join(copy_text_chunks(recorded_share_ids(fields["shares_json"]))) - self.assertEqual(copied, expected) - del candidate, hydrated, body - self.assertIsNone(observed(), "replay's caller graph retained the body without cyclic GC") - self.assertEqual(list(Path(directory).iterdir()), []) - finally: - if enabled: - gc.enable() - - -if __name__ == "__main__": - unittest.main() diff --git a/tests/test_prism_candidate_spool_view.py b/tests/test_prism_candidate_spool_view.py deleted file mode 100644 index 72bcc1032..000000000 --- a/tests/test_prism_candidate_spool_view.py +++ /dev/null @@ -1,845 +0,0 @@ -#!/usr/bin/env python3 -"""Bounded candidate spool hydration regressions (issue #255). - -Every C call the reader makes is instrumented and asserted against the -view module's hard threshold, for a body carrying an oversized share -record, an oversized nested array and oversized top-level strings. The -remaining tests pin the adapter contract root builds on: streamed strings -that never split an escape or a surrogate pair, mapping/sequence views -that compare with plain values, verbatim byte access that preserves every -digest, random access through the disk index, legacy hinted pages, failure -classification (corrupt versus closed versus resource pressure), ownership -under retained exceptions with the cyclic collector disabled, and the -supervised isolated-record helper. -""" - -from __future__ import annotations - -import contextlib -import gc -import hashlib -import json -import os -import shutil -import subprocess -import sys -import tempfile -import threading -import time -import unittest -from dataclasses import replace -from pathlib import Path -from typing import Any -from unittest import mock - -sys.path.insert(0, str(Path(__file__).resolve().parents[1])) - -from lab.prism import audit_bundle_view as audit # noqa: E402 -from lab.prism import candidate_codec as codec # noqa: E402 -from lab.prism import candidate_spool_view as views # noqa: E402 -from lab.prism.share_ledger import block_candidate_identity # noqa: E402 - -HASH_A = "aa" * 32 -PARENT = "bb" * 32 -COMPACT = (",", ":") - - -def canonical(value: Any) -> bytes: - return json.dumps(value, sort_keys=True, separators=COMPACT).encode("utf-8") - - -def oracle_bytes(intent: dict[str, Any]) -> bytes: - identity = block_candidate_identity({**intent, "block_hash_hex": intent["block_hash_hex"].lower()}) - return canonical(identity) - - -def share(index: int) -> dict[str, Any]: - return { - "share_seq": index, - "share_id": f"miner-{index % 7}:é\U0001F600{'x' * (index % 3)}", - "miner_id": f"m{index % 7}", - "order_key": "k", - "p2mr_program_hex": "ab" * 32, - "share_difficulty": 10**30 + index, - "network_difficulty": 1.5e16 if index % 5 else 3, - "template_height": 5, - "job_id": 'j"q\\', - "job_issued_at_ms": 1_700_000_000_000 + index, - "accepted_at_ms": 1_700_000_000_001 + index, - "ntime": 1_700_000_000, - "credit_policy": None if index % 2 else "stale-grace", - } - - -def intent_for(share_count: int, **overrides: Any) -> dict[str, Any]: - fields: dict[str, Any] = { - "schema": codec.CANDIDATE_INTENT_SCHEMA, - "block_hash_hex": HASH_A.upper(), - "block_hex": "00" * 100, - "coinbase_tx_hex": "01", - "parent_hash": PARENT, - "expected_height": 10, - "template": {"previousblockhash": PARENT, "height": 10, "coinbasevalue": 5}, - "shares_json": [share(index) for index in range(share_count)], - "prior_balances": [{"recipient_id": "a", "balance_sats": 1}], - "found_block": {"network_difficulty": 1.0, "witness": [1, 2, {"z": None}]}, - "prospective_prior_balances": None, - "witness_merkle_leaves_hex": ["cc" * 32] * 3, - "extranonce1_hex": "00", - "extranonce2_hex": "01", - "username": "u", - "pending_share": {"share_id": "s", "accepted_at_ms": 123}, - "credit_share_on_accept": False, - "collection_only": False, - } - fields.update(overrides) - return fields - - -class _LoadsProbe: - """Record the input and output size of every ``json.loads`` call.""" - - def __init__(self) -> None: - self.inputs: list[int] = [] - self.outputs: list[int] = [] - self._real = json.loads - - def loads(self, data: Any, *args: Any, **kwargs: Any) -> Any: - value = self._real(data, *args, **kwargs) - self.inputs.append(len(data)) - if isinstance(value, (str, list, tuple, dict)): - self.outputs.append(len(value)) - else: - self.outputs.append(1) - return value - - -class SpoolFixture(unittest.TestCase): - def bounded(self, threshold: int) -> Any: - """Lower the view threshold for a test, keeping the invariants. - - The encoder spans a top-level field only above its fast-path - ceiling and the reader slices strings below the threshold, so both - knobs follow the threshold down; in production the ordering is the - same (64 KiB fast path, 64 KiB slices, 1 MiB threshold). - """ - stack = contextlib.ExitStack() - stack.enter_context(mock.patch.object(views, "SPOOL_VIEW_DECODE_BYTES", threshold)) - stack.enter_context(mock.patch.object(codec, "CODEC_FAST_PATH_BYTES", min(codec.CODEC_FAST_PATH_BYTES, threshold))) - stack.enter_context(mock.patch.object(views, "SPOOL_VIEW_STRING_SLICE_BYTES", min(views.SPOOL_VIEW_STRING_SLICE_BYTES, threshold))) - return stack - - def spool(self, fields: dict[str, Any], *, chunk_bytes: int = 4096) -> tuple[codec.PreparedCandidateIntent, codec.SpoolCandidateBody]: - prepared = codec.prepare_candidate_intent(fields, chunk_bytes=chunk_bytes) - directory = tempfile.mkdtemp() - self.addCleanup(shutil.rmtree, directory, True) - body = codec.write_spool_from_body( - os.path.join(directory, "b.body"), os.path.join(directory, "b.idx"), prepared.body - ) - self.addCleanup(body.close) - return prepared, body - - def hydrate(self, fields: dict[str, Any], **kwargs: Any) -> tuple[codec.PreparedCandidateIntent, codec.SpoolCandidateBody]: - _prepared, body = self.spool(fields, **kwargs) - hydrated = codec.prepared_intent_from_spool(body, accepted_at_present=True, accepted_at_ms=123) - return hydrated, body - - -class BoundednessTests(SpoolFixture): - def test_every_reader_c_call_is_bounded_by_the_threshold(self) -> None: - threshold = 32 * 1024 - big = ("é\"\\" + "y" * 500 + "\U0001F600") * 400 # ~200 KiB encoded - fields = intent_for(700, username=big, block_hex="ab" * 40_000, coinbase_tx_hex="cd" * 40_000) - fields["shares_json"][300]["share_id"] = "s" * (6 * threshold) - fields["shares_json"][300]["nested"] = [{"k": i, "v": "w" * 100} for i in range(2000)] - fields["found_block"]["witness"] = [[i, i * 1.5, f"leaf-{i}"] for i in range(20_000)] - fields["pending_share"] = {"share_id": "p" * (3 * threshold), "accepted_at_ms": 123, "miner_id": "m"} - probe = _LoadsProbe() - reads: list[int] = [] - real_read = codec.SpoolCandidateBody.read_span - - def read_span(self_: Any, start: int, end: int) -> bytes: - reads.append(end - start) - return real_read(self_, start, end) - - with self.bounded(threshold), \ - mock.patch.object(codec.json, "loads", probe.loads), \ - mock.patch.object(views.json, "loads", probe.loads), \ - mock.patch.object(codec.SpoolCandidateBody, "read_span", read_span): - hydrated, _body = self.hydrate(fields) - # Touch everything: every share, every member of the oversized - # record, every top-level fact, every streamed string. - self.assertEqual(len(list(hydrated["shares_json"])), 700) - record = hydrated["shares_json"][300] - self.assertIsInstance(record, views.SpoolObjectView) - self.assertEqual(dict(record).keys(), fields["shares_json"][300].keys()) - self.assertEqual(record["share_id"], fields["shares_json"][300]["share_id"]) - self.assertEqual(list(record["nested"]), fields["shares_json"][300]["nested"]) - self.assertIsInstance(hydrated["username"], views.SpoolStringView) - self.assertEqual("".join(hydrated["username"].iter_text_chunks()), big) - self.assertIsInstance(hydrated["block_hex"], views.SpoolStringView) - self.assertEqual(hydrated["block_hex"], "ab" * 40_000) - self.assertEqual(hydrated["coinbase_tx_hex"], "cd" * 40_000) - self.assertIsInstance(hydrated["coinbase_tx_hex"], str) - witness = hydrated["found_block"]["witness"] - self.assertIsInstance(witness, views.SpoolArrayView) - self.assertEqual(len(witness), 20_000) - self.assertEqual(list(witness), fields["found_block"]["witness"]) - self.assertEqual(witness[19_999], fields["found_block"]["witness"][19_999]) - self.assertEqual(hydrated["pending_share"]["accepted_at_ms"], 123) - self.assertEqual(hydrated["pending_share"]["share_id"], "p" * (3 * threshold)) - header = hydrated.replay_header() - self.assertTrue(header["oversized"]) - self.assertIsNone(header["username"]) - self.assertEqual(header["pending_share"]["accepted_at_ms"], 123) - self.assertTrue(probe.inputs) - # The skeleton is the one call bounded by structure rather than the - # threshold: every non-spanned top-level field is under the encoder's - # fast-path ceiling, and this intent has twenty of them. - skeleton_bound = 20 * threshold - self.assertLessEqual(max(probe.inputs), max(threshold + 2, skeleton_bound)) - self.assertLessEqual(sorted(probe.inputs)[-2], threshold + 2) - self.assertLessEqual(max(reads), max(threshold, views.SPOOL_VIEW_READ_BYTES)) - string_outputs = [size for size, inp in zip(probe.outputs, probe.inputs) if inp <= views.SPOOL_VIEW_STRING_SLICE_BYTES + 16] - self.assertTrue(string_outputs) - self.assertLessEqual(max(probe.outputs), max(threshold, codec.CODEC_BATCH_RECORDS * 4)) - - def test_share_id_above_the_page_decode_ceiling_is_a_streamed_string(self) -> None: - # The exact defect: one record over CODEC_PAGE_DECODE_BYTES made the - # page "not valid JSON". It is now an object view whose share_id - # streams; every other record on the page stays a plain dict. - giant = "g" * (codec.CODEC_PAGE_DECODE_BYTES + 1024 * 1024) - fields = intent_for(600) - fields["shares_json"][300]["share_id"] = giant - probe = _LoadsProbe() - with mock.patch.object(codec.json, "loads", probe.loads), mock.patch.object(views.json, "loads", probe.loads): - hydrated, body = self.hydrate(fields, chunk_bytes=codec.CANDIDATE_BODY_CHUNK_BYTES) - shares = hydrated["shares_json"] - record = shares[300] - self.assertIsInstance(record, views.SpoolObjectView) - self.assertIsInstance(record["share_id"], views.SpoolStringView) - self.assertEqual(record["share_id"].byte_length, len(giant) + 2) - self.assertEqual(record["share_id"].text_length(), len(giant)) - self.assertTrue(record["share_id"] == giant) - self.assertEqual(record["share_seq"], 300) - self.assertEqual(record, fields["shares_json"][300]) - self.assertIsInstance(shares[299], dict) - self.assertIsInstance(shares[301], dict) - self.assertEqual(shares[299], fields["shares_json"][299]) - self.assertEqual(shares[599], fields["shares_json"][599]) - self.assertLessEqual(max(probe.inputs), views.SPOOL_VIEW_DECODE_BYTES + 2) - digest = hashlib.sha256() - body.write_chunks(lambda chunk: digest.update(chunk.data)) - self.assertEqual(digest.hexdigest(), hashlib.sha256(oracle_bytes(fields)).hexdigest()) - - -class RoundTripTests(SpoolFixture): - def test_small_and_unicode_strings_round_trip_at_every_slice_cut(self) -> None: - unit = "\U0001F600" * 5 + "\\" * 7 + 'q"' + "é" + "\U0001F600" + "\\\\u0041" + "\n\t" - text = unit * 40 - fields = intent_for(1, username=text, block_hex="0" * 10) - with self.bounded(256): - hydrated, _body = self.hydrate(fields) - view = hydrated["username"] - self.assertIsInstance(view, views.SpoolStringView) - encoded = json.dumps(text) - for slice_bytes in range(32, 97): - with self.subTest(slice_bytes=slice_bytes): - chunks = list(view.iter_text_chunks(slice_bytes=slice_bytes)) - self.assertEqual("".join(chunks), text) - self.assertTrue(all(chunks)) - self.assertEqual(view, text) - self.assertNotEqual(view, text + "x") - self.assertTrue(text == view) - self.assertEqual("".join(view.iter_encoded_chunks()), encoded) - self.assertEqual(b"".join(view.iter_byte_chunks(chunk_bytes=7)), encoded.encode("ascii")) - self.assertEqual(view.canonical_json_sha256(), hashlib.sha256(encoded.encode()).hexdigest()) - self.assertEqual(views.materialize_spool_views(view), text) - self.assertEqual("".join(views.iter_string_text_chunks(view)), text) - self.assertEqual(list(views.iter_string_text_chunks("plain")), ["plain"]) - with self.assertRaises(TypeError): - str(view) - self.assertIn("SpoolStringView", repr(view)) - self.assertTrue(view) - # Small strings are plain values with the historical shape. - self.assertEqual(hydrated["block_hex"], "0" * 10) - self.assertIsInstance(hydrated["block_hex"], str) - self.assertEqual(hydrated["shares_json"][0]["share_id"], share(0)["share_id"]) - - def test_raw_utf8_and_split_numbers_survive_tiny_reads(self) -> None: - # A body written with raw UTF-8 (a legacy encoder that did not - # escape) and read five bytes at a time: no code point, escape or - # number token may be split by a read boundary. - witness = [ - 1, -2, 3.5e-7, 12345678901234567890, -0.0, 1e300, True, None, - 'trap},{"k":1', "\\\\\"", "☃ snow \U0001F600", {"n": [1.25, "x", {"deep": [[], {}]}]}, [], {}, - ] * 40 - fields = intent_for(1, found_block={"network_difficulty": 1.0, "witness": witness}) - _prepared, body = self.spool(fields) - raw_path = body.path + ".raw" - text = json.loads(body.read_span(0, body.manifest.byte_count).decode("ascii")) - raw = json.dumps(text, sort_keys=True, separators=COMPACT, ensure_ascii=False).encode("utf-8") - with open(raw_path, "wb") as handle: - handle.write(raw) - self.addCleanup(lambda: os.path.exists(raw_path) and os.unlink(raw_path)) - start = raw.index(b'"found_block":{') + len(b'"found_block":') - end = raw.index(b',"parent_hash"') - raw_body = mock.Mock(spec=codec.SpoolCandidateBody) - raw_body.alive = True - raw_body.read_span = lambda s, e: raw[s:e] - with mock.patch.object(views, "SPOOL_VIEW_READ_BYTES", 5), \ - mock.patch.object(views, "SPOOL_VIEW_STRING_SLICE_BYTES", 33), \ - self.bounded(40): - found = views.decode_body_span(raw_body, start, end) - self.assertIsInstance(found, views.SpoolObjectView) - items = found["witness"] - self.assertIsInstance(items, views.SpoolArrayView) - self.assertEqual(len(items), len(witness)) - self.assertEqual(list(items), witness) - self.assertEqual(items[10], witness[10]) - self.assertEqual(items[-1], witness[-1]) - self.assertEqual(items[3:6], tuple(witness[3:6])) - self.assertEqual(items, witness) - self.assertEqual(found, fields["found_block"]) - self.assertEqual("".join(items.iter_encoded_chunks()), json.dumps(witness, sort_keys=True, separators=COMPACT, ensure_ascii=False)) - - def test_random_access_and_slicing_through_the_disk_index(self) -> None: - records = [share(index) for index in range(1300)] - records[700]["share_id"] = 'trap},{"accepted_at_ms":1' - records[511]["nested"] = [{"a": 1}, {"b": "\\\"},{"}] - records[900]["share_id"] = "big" * 30_000 - fields = intent_for(0, shares_json=records) - with self.bounded(16 * 1024): - hydrated, _body = self.hydrate(fields) - shares = hydrated["shares_json"] - self.assertIsInstance(shares, codec.SpoolJsonArraySequence) - self.assertEqual(len(shares), 1300) - for index in (0, 1, 255, 256, 511, 700, 899, 900, 901, 1299, -1, -300): - with self.subTest(index=index): - self.assertEqual(shares[index], records[index]) - self.assertEqual(shares[898:903], tuple(records[898:903])) - self.assertEqual(shares[::400], tuple(records[::400])) - with self.assertRaises(IndexError): - shares[1300] - with self.assertRaises(TypeError): - shares["x"] - self.assertIsInstance(shares[900], views.SpoolObjectView) - self.assertIsInstance(shares[900]["share_id"], views.SpoolStringView) - self.assertEqual(list(shares), records) - self.assertTrue(shares == records) - self.assertFalse(shares == records[:-1]) - expected_array = canonical(records) - self.assertEqual(shares.canonical_json_sha256(), hashlib.sha256(expected_array).hexdigest()) - self.assertEqual(b"".join(shares.iter_byte_chunks()), expected_array) - staged = bytearray(b"[") - for data, _count in shares.canonical_share_pages(): - if staged[-1:] != b"[" and staged[-1:] != b",": - staged += b"," - staged += data - staged += b"]" - self.assertEqual(json.loads(bytes(staged)), records) - - -class RestagingTests(SpoolFixture): - def test_hydrated_body_and_every_field_keep_their_bytes(self) -> None: - fields = intent_for(600, username="u" * 200_000, block_hex="ef" * 100_000) - fields["shares_json"][300]["share_id"] = "y" * 300_000 - fields["found_block"]["witness"] = [{"leaf": i} for i in range(30_000)] - fields["pending_share"] = {"share_id": "p" * 200_000, "accepted_at_ms": 123} - expected = oracle_bytes(fields) - with self.bounded(64 * 1024): - hydrated, body = self.hydrate(fields) - got = bytearray() - hydrated.body.write_chunks(lambda chunk: got.extend(chunk.data)) - self.assertEqual(bytes(got), expected) - self.assertIs(codec.prepare_candidate_intent(hydrated), hydrated) - for name in ("username", "block_hex"): - view = hydrated[name] - self.assertIsInstance(view, views.SpoolStringView) - self.assertEqual(b"".join(view.iter_byte_chunks()), canonical(fields[name])) - found = hydrated["found_block"] - self.assertIsInstance(found, views.SpoolObjectView) - self.assertEqual(b"".join(found.iter_byte_chunks()), canonical(fields["found_block"])) - self.assertEqual(found.canonical_json_sha256(), hashlib.sha256(canonical(fields["found_block"])).hexdigest()) - self.assertEqual(b"".join(found["witness"].iter_byte_chunks()), canonical(fields["found_block"]["witness"])) - record = hydrated["shares_json"][300] - self.assertEqual(b"".join(record.iter_byte_chunks()), canonical(fields["shares_json"][300])) - self.assertEqual(b"".join(record["share_id"].iter_byte_chunks()), canonical(fields["shares_json"][300]["share_id"])) - # Re-staging the hydrated body to a second spool reproduces it - # byte for byte, oversized page and all, and hydrates equal. - directory = tempfile.mkdtemp() - self.addCleanup(shutil.rmtree, directory, True) - second = codec.write_spool_from_body(os.path.join(directory, "c.body"), os.path.join(directory, "c.idx"), hydrated.body) - self.addCleanup(second.close) - self.assertEqual(second.manifest, body.manifest) - again = codec.prepared_intent_from_spool(second, accepted_at_present=True, accepted_at_ms=123) - self.assertEqual(again["shares_json"][300], fields["shares_json"][300]) - self.assertEqual(again["found_block"], fields["found_block"]) - # The shares sequence re-encodes through the verbatim page - # protocol, streaming the oversized page as raw pieces. - restaged = codec.prepare_candidate_intent(intent_for(0, shares_json=hydrated["shares_json"]), chunk_bytes=4096) - regot = bytearray() - restaged.body.write_chunks(lambda chunk: regot.extend(chunk.data)) - self.assertEqual(bytes(regot), oracle_bytes(intent_for(0, shares_json=fields["shares_json"]))) - - def test_streaming_encoder_and_materializer_accept_spool_views(self) -> None: - fields = intent_for(2, username="w" * 5000, found_block={"network_difficulty": 1.0, "witness": list(range(3000))}) - with self.bounded(1024): - hydrated, _body = self.hydrate(fields) - username = hydrated["username"] - found = hydrated["found_block"] - self.assertIsInstance(username, views.SpoolStringView) - self.assertIsInstance(found, views.SpoolObjectView) - payload = {"u": username, "f": found, "w": found["witness"], "n": 1} - plain = {"u": fields["username"], "f": fields["found_block"], "w": fields["found_block"]["witness"], "n": 1} - self.assertEqual("".join(audit.iter_json_chunks(payload, sort_keys=True)), json.dumps(plain, sort_keys=True, separators=COMPACT)) - self.assertEqual("".join(audit.iter_json_chunks(payload)), json.dumps(plain, separators=COMPACT)) - self.assertEqual(audit.materialize_json(payload), plain) - self.assertEqual(views.materialize_spool_views(payload), plain) - self.assertEqual(audit.streamed_sha256_json_hex(payload), hashlib.sha256(canonical(plain)).hexdigest()) - self.assertTrue(views.is_spool_view(username)) - self.assertFalse(views.is_spool_view("x")) - - -class MappingViewTests(SpoolFixture): - def test_object_view_mapping_protocol_and_pending_stamp(self) -> None: - members: dict[str, Any] = {f"k{i:04d}": {"v": i, "s": "x" * 50} for i in range(600)} - members["share_id"] = "S" * 5000 - members["accepted_at_ms"] = 123 - # The body stores keys sorted and the header projection keeps at - # most 32 members (the historical shape), so the pending share stays - # small in members while its share_id alone crosses the threshold. - pending = {"share_id": "S" * 5000, "accepted_at_ms": 123, "miner_id": "m"} - fields = intent_for(1, pending_share=pending, found_block=dict(members)) - with self.bounded(2048), \ - mock.patch.object(views, "SPOOL_VIEW_INDEX_MEMORY_ENTRIES", 8): - hydrated, _body = self.hydrate(fields) - pending = hydrated["pending_share"] - self.assertIsInstance(pending, dict) - self.assertEqual(pending["accepted_at_ms"], 123) - self.assertIsInstance(pending["share_id"], views.SpoolStringView) - self.assertEqual(pending["share_id"], "S" * 5000) - self.assertEqual(pending["miner_id"], "m") - self.assertEqual(set(pending), {"share_id", "accepted_at_ms", "miner_id"}) - header = hydrated.replay_header() - self.assertTrue(header["oversized"]) - self.assertEqual(header["pending_share"]["accepted_at_ms"], 123) - self.assertIsNone(header["pending_share"]["share_id"]) - self.assertTrue(header["accepted_at_present"]) - found = hydrated["found_block"] - self.assertIsInstance(found, views.SpoolObjectView) - # The body stores keys sorted, as the historical json.loads did. - ordered = sorted(members) - self.assertEqual(len(found), len(members)) - self.assertEqual(list(found), ordered) - self.assertEqual(list(found.keys()), ordered) - self.assertEqual([key for key, _ in found.items()], ordered) - self.assertEqual(list(found.values())[1], members[ordered[1]]) - self.assertEqual(found.get("absent", "d"), "d") - self.assertIn("k0599", found) - self.assertNotIn("nope", found) - self.assertNotIn(5, found) - with self.assertRaises(KeyError): - found["nope"] - self.assertEqual(found, members) - self.assertTrue(members == found) - self.assertNotEqual(found, {**members, "extra": 1}) - self.assertNotEqual(found, {**members, "k0000": 0}) - self.assertEqual(dict(found), members) - self.assertEqual(views.materialize_spool_views(found), members) - self.assertTrue(found._ensure_indexed().spilled) - self.assertIn("members=602", repr(found)) - found.close() - with self.assertRaises(codec.CandidateBodyIntegrityError): - found["k0001"] - # Without the key cache (too many members) lookups scan keys. - with mock.patch.object(views, "SPOOL_VIEW_KEY_CACHE_ENTRIES", 4): - again = views.SpoolObjectView(found.body, *found.byte_range) - self.assertEqual(again["k0450"], members["k0450"]) - self.assertIsNone(again._keys) - self.assertEqual(again["share_id"], members["share_id"]) - with self.assertRaises(KeyError): - again["missing"] - - def test_decoded_metadata_boundary_and_lazy_block(self) -> None: - fields = intent_for(1, coinbase_tx_hex="ab" * 3000, block_hex="cd" * 3000, extranonce1_hex="ef" * 3000, username="u" * 6000) - with self.bounded(1024): - hydrated, _body = self.hydrate(fields) - for name in ("coinbase_tx_hex", "extranonce1_hex"): - self.assertIsInstance(hydrated[name], str) - self.assertEqual(hydrated[name], fields[name]) - self.assertIsInstance(hydrated["block_hex"], views.SpoolStringView) - self.assertEqual(hydrated["block_hex"].byte_length, len(fields["block_hex"]) + 2) - self.assertEqual(b"".join(hydrated["block_hex"].iter_byte_chunks()), canonical(fields["block_hex"])) - self.assertIsInstance(hydrated["username"], views.SpoolStringView) - with self.assertRaises(TypeError) as caught: - str(hydrated["username"]) - self.assertNotIsInstance(caught.exception, codec.CandidateBodyIntegrityError) - self.assertEqual(codec.decode_json_string_span(hydrated.body, hydrated.body.index.span("username")), fields["username"]) - self.assertIn("block_hex", hydrated) - self.assertEqual(hydrated.block_hash, HASH_A) - - -class FailureTests(SpoolFixture): - def test_corrupt_truncated_missing_and_closed_bodies_are_classified(self) -> None: - fields = intent_for(300, username="u" * 20_000) - fields["found_block"]["witness"] = list(range(5000)) - with self.bounded(4096): - hydrated, body = self.hydrate(fields) - username = hydrated["username"] - witness = hydrated["found_block"]["witness"] - self.assertEqual(username, fields["username"]) - self.assertEqual(len(witness), 5000) - span = body.index.span("username") - # An invalid escape inside the streamed string: corruption. - with open(body.path, "r+b") as handle: - handle.seek(span.start + 10) - handle.write(b"\\x") - with self.assertRaises(codec.CandidateBodyIntegrityError): - list(username.iter_text_chunks()) - with self.assertRaises(codec.CandidateBodyIntegrityError): - list(body.iter_chunks()) - # Unbalanced delimiters in a lazy array: corruption. - witness_span = body.index.span("found_block") - with open(body.path, "r+b") as handle: - handle.seek(witness_span.start + 20) - handle.write(b"]") - fresh = views.SpoolObjectView(body, witness_span.start, witness_span.end) - with self.assertRaises(codec.CandidateBodyIntegrityError): - fresh["witness"] - # A page index that lies about its span: corruption, not a crash. - with self.assertRaises(codec.CandidateBodyIntegrityError): - codec.SpoolJsonArraySequence(body, replace(body.index.span("shares_json"), start=10**9))[0] - # Truncation inside the share array: the read is shorter than - # the manifest (username follows the shares in key order). - with open(body.path, "r+b") as handle: - handle.truncate(body.index.span("shares_json").start + 100) - with self.assertRaises(codec.CandidateBodyIntegrityError): - hydrated["shares_json"][299] - with self.assertRaises(codec.CandidateBodyIntegrityError): - b"".join(username.iter_byte_chunks()) - # A vanished index file: corruption-class, never a bare OSError. - os.unlink(body.index.path) - with self.assertRaises(codec.CandidateBodyIntegrityError): - hydrated["shares_json"][0] - # A released body: every view fails closed. - hydrated.release() - self.assertFalse(body.alive) - with self.assertRaises(codec.CandidateBodyIntegrityError): - username.iter_text_chunks().__next__() - with self.assertRaises(codec.CandidateBodyIntegrityError): - views.SpoolArrayView(body, witness_span.start, witness_span.end)[0] - with self.assertRaises(codec.CandidateBodyIntegrityError): - hydrated["shares_json"][1] - - def test_mismatched_delimiters_and_empty_values_are_corruption(self) -> None: - raw = b'{"a":[1,2}' - fake = mock.Mock(spec=codec.SpoolCandidateBody) - fake.alive = True - fake.read_span = lambda s, e: raw[s:e] - with self.bounded(2): - with self.assertRaises(codec.CandidateBodyIntegrityError): - views.decode_body_span(fake, 5, 10) - with self.assertRaises(codec.CandidateBodyIntegrityError): - views.decode_body_span(fake, 5, 5) - with self.assertRaises(codec.CandidateBodyIntegrityError): - list(views.iter_member_spans(fake, 1, 9)) - with self.assertRaises(codec.CandidateBodyIntegrityError): - list(views.iter_json_string_text_chunks(fake, 0, 4)) - - def test_legacy_hinted_pages_stay_readable_ignoring_bad_hints(self) -> None: - records = [share(index) for index in range(900)] - records[400]["share_id"] = 'trap},{"share_seq":9' * 20 - records[650]["nested"] = [{"b": "\\\"},{"}] * 3 - fields = intent_for(0, shares_json=records) - _prepared, body = self.spool(fields) - span = body.index.span("shares_json") - data = body.read_span(span.start, span.end) - # A legacy index whose hints came from a substring scan: one lands - # inside a quoted trap, one inside a nested object. - directory = tempfile.mkdtemp() - self.addCleanup(shutil.rmtree, directory, True) - legacy_path = os.path.join(directory, "legacy.body") - shutil.copyfile(body.path, legacy_path) - index = codec.SpoolFieldIndex.create(os.path.join(directory, "legacy.idx")) - hints = [span.start + 1] - hints.append(span.start + data.index(b"trap},{") + 5) - hints.append(span.start + data.index(b'\\"},{"') + 4) - hinted = replace(span, page_count=len(hints), pages_exact=False) - index.begin_field(hinted) - index.append_pages("shares_json", [(offset, 0) for offset in hints]) - legacy = codec.SpoolCandidateBody(legacy_path, body.manifest, index) - self.addCleanup(legacy.close) - sequence = codec.SpoolJsonArraySequence(legacy, hinted) - with self.bounded(8192): - self.assertEqual(sequence[400], records[400]) - self.assertEqual(sequence[899], records[899]) - self.assertEqual(sequence[0], records[0]) - self.assertEqual(list(sequence), records) - self.assertFalse(legacy.index.span("shares_json").pages_exact) - self.assertIsNotNone(sequence._walk_index) - staged = codec.prepare_candidate_intent(intent_for(0, shares_json=sequence), chunk_bytes=4096) - self.assertEqual(staged.candidate_sha256, hashlib.sha256(oracle_bytes(fields)).hexdigest()) - sequence.close() - - -class OwnershipTests(SpoolFixture): - def test_scratch_indices_close_on_error_and_by_refcount_with_gc_disabled(self) -> None: - fields = intent_for(1, found_block={"network_difficulty": 1.0, "witness": list(range(4000))}) - created: list[Any] = [] - real_temporary_file = tempfile.TemporaryFile - - def recording_temporary_file(*args: Any, **kwargs: Any) -> Any: - handle = real_temporary_file(*args, **kwargs) - created.append(handle) - return handle - - gc_was_enabled = gc.isenabled() - gc.disable() - self.addCleanup(lambda: gc.enable() if gc_was_enabled else None) - with self.bounded(512), \ - mock.patch.object(views, "SPOOL_VIEW_INDEX_MEMORY_ENTRIES", 4), \ - mock.patch.object(views.tempfile, "TemporaryFile", recording_temporary_file): - hydrated, body = self.hydrate(fields) - found = hydrated["found_block"] - # Hydration may spill its aggregate metadata index first. - self.assertTrue(all(handle.closed for handle in created)) - created.clear() - # 1. An error while the index is being built closes its scratch - # file before the exception leaves, even though the retained - # exception keeps the frame (and the index) alive. - real_spans = views.iter_item_spans - - def failing_spans(*args: Any, **kwargs: Any) -> Any: - for count, item in enumerate(real_spans(*args, **kwargs)): - if count == 50: - raise OSError("disk went away") - yield item - - retained: list[BaseException] = [] - with mock.patch.object(views, "iter_item_spans", failing_spans): - try: - len(found["witness"]) - except OSError as exc: - retained.append(exc) - self.assertEqual(len(retained), 1) - self.assertEqual(len(created), 1) - self.assertTrue(created[0].closed) - # 2. A successful index spills to a scratch file that closes when - # the last view referencing it is dropped -- by reference - # count alone, since no view takes part in a cycle. - witness = found["witness"] - self.assertEqual(len(witness), 4000) - self.assertEqual(len(created), 2) - self.assertFalse(created[1].closed) - fd = created[1].fileno() - os.fstat(fd) - del witness - self.assertTrue(created[1].closed) - with self.assertRaises(OSError): - os.fstat(fd) - # 3. Resource pressure while spilling is retryable, and leaves - # no descriptor behind. - with mock.patch.object(views.tempfile, "TemporaryFile", side_effect=OSError("no temp")): - with self.assertRaises(audit.ArtifactResourcePressure) as pressure: - len(views.SpoolArrayView(body, *found["witness"].byte_range)) - self.assertNotIsInstance(pressure.exception, codec.CandidateBodyIntegrityError) - self.assertEqual(len(created), 2) - # 4. Explicit release: the spool pair is unlinked while a view - # (kept alive by a retained traceback) still exists, and the - # view fails closed instead of reading a stale file. - witness = found["witness"] - paths = (body.path, body.index.path) - try: - raise RuntimeError("retained while views are alive") - except RuntimeError as exc: - retained.append(exc) - hydrated.release() - self.assertFalse(any(os.path.exists(path) for path in paths)) - with self.assertRaises(codec.CandidateBodyIntegrityError): - witness[0] - views.close_spool_views(hydrated.facts) - self.assertTrue(all(handle.closed for handle in created)) - del retained - - def test_hydration_failure_leaves_no_open_scratch(self) -> None: - # A corrupt member value inside a lazy pending_share: hydration's - # shallow copy fails after the (spilled) member index was built, the - # exception is retained, and the scratch file is closed anyway. - pending = {f"k{i:03d}": {"s": "x" * 20, "v": i} for i in range(40)} - pending["accepted_at_ms"] = 123 - fields = intent_for(2, pending_share=pending) - with self.bounded(512): - _prepared, body = self.spool(fields) - span = body.index.span("pending_share") - data = body.read_span(span.start, span.end) - with open(body.path, "r+b") as handle: - handle.seek(span.start + data.index(b'"v":7}') + 4) - handle.write(b"x") - created: list[Any] = [] - real_temporary_file = tempfile.TemporaryFile - - def recording(*args: Any, **kwargs: Any) -> Any: - handle = real_temporary_file(*args, **kwargs) - created.append(handle) - return handle - - with self.bounded(512), \ - mock.patch.object(views, "SPOOL_VIEW_INDEX_MEMORY_ENTRIES", 2), \ - mock.patch.object(views.tempfile, "TemporaryFile", recording): - retained: list[BaseException] = [] - try: - codec.prepared_intent_from_spool(body, accepted_at_present=True, accepted_at_ms=123) - except codec.CandidateBodyIntegrityError as exc: - retained.append(exc) - self.assertEqual(len(retained), 1) - self.assertTrue(created) - self.assertTrue(all(handle.closed for handle in created)) - del retained - - -class HelperSupervisionTests(unittest.TestCase): - def source(self, raw: bytes) -> audit.ArtifactSource: - directory = tempfile.mkdtemp() - self.addCleanup(shutil.rmtree, directory, True) - path = os.path.join(directory, "record.json") - with open(path, "wb") as handle: - handle.write(raw) - source = audit.ArtifactSource(os.open(path, os.O_RDONLY), path=Path(path)) - self.addCleanup(source.close) - return source - - def helper_threads(self) -> list[str]: - return [thread.name for thread in threading.enumerate() if thread.name.startswith("prism-audit-record-helper")] - - def test_hanging_helper_obeys_the_deadline_and_is_reaped(self) -> None: - raw = b'{"a":"' + b"x" * 5000 + b'"}' - source = self.source(raw) - real_popen = subprocess.Popen - children: list[subprocess.Popen[bytes]] = [] - - def sleeping_child(_args: Any, **kwargs: Any) -> Any: - process = real_popen([sys.executable, "-c", "import time; time.sleep(60)"], **kwargs) - children.append(process) - return process - - started = time.monotonic() - with mock.patch.object(audit.subprocess, "Popen", sleeping_child): - with self.assertRaises(audit.ArtifactResourcePressure): - audit.normalize_record_isolated(source, 0, len(raw), timeout_seconds=0.3) - self.assertLess(time.monotonic() - started, 5.0) - self.assertEqual(len(children), 1) - self.assertIsNotNone(children[0].poll()) - self.assertTrue(all(pipe.closed for pipe in (children[0].stdin, children[0].stdout, children[0].stderr))) - self.assertEqual(self.helper_threads(), []) - - def test_cancellation_abandons_the_helper(self) -> None: - raw = b'{"a":"' + b"x" * 5000 + b'"}' - source = self.source(raw) - real_popen = subprocess.Popen - children: list[subprocess.Popen[bytes]] = [] - calls = {"count": 0} - - class Cancelled(Exception): - pass - - def cancellation() -> None: - calls["count"] += 1 - if calls["count"] > 2: - raise Cancelled() - - def sleeping_child(_args: Any, **kwargs: Any) -> Any: - process = real_popen([sys.executable, "-c", "import time; time.sleep(60)"], **kwargs) - children.append(process) - return process - - with mock.patch.object(audit.subprocess, "Popen", sleeping_child): - with self.assertRaises(Cancelled): - audit.normalize_record_isolated(source, 0, len(raw), timeout_seconds=30.0, cancellation=cancellation) - self.assertIsNotNone(children[0].poll()) - self.assertEqual(self.helper_threads(), []) - - def test_admission_slots_are_shared_and_deadline_bounded(self) -> None: - raw = b'{"a":1}' - source = self.source(raw) - admission = audit.HelperAdmission(1) - admission.acquire(lambda: None) - try: - with mock.patch.object(audit.subprocess, "Popen") as spawn: - with self.assertRaises(audit.ArtifactResourcePressure): - audit.normalize_record_isolated(source, 0, len(raw), timeout_seconds=0.2, admission=admission) - spawn.assert_not_called() - finally: - admission.release() - # The slot is released after use, so a second run proceeds. - record = audit.normalize_record_isolated(source, 0, len(raw), admission=admission) - try: - self.assertEqual(record, {"a": 1}) - finally: - record.close() - admission.acquire(lambda: None) - admission.release() - - def test_helper_exit_statuses_and_noisy_diagnostics_are_classified(self) -> None: - raw = b'{"a":"' + b"x" * 100 + b'"}' - source = self.source(raw) - real_popen = subprocess.Popen - - def child(script: str) -> Any: - def spawn(_args: Any, **kwargs: Any) -> Any: - return real_popen([sys.executable, "-c", script], **kwargs) - return spawn - - noisy = "import sys; sys.stdin.read(); sys.stderr.write('e' * 1000000); sys.stderr.flush(); sys.exit(2)" - with mock.patch.object(audit.subprocess, "Popen", child(noisy)): - with self.assertRaises(json.JSONDecodeError) as malformed: - audit.normalize_record_isolated(source, 0, len(raw), timeout_seconds=30.0) - self.assertIsInstance(malformed.exception, audit.CanonicalArtifactSyntaxError) - self.assertLessEqual(len(str(malformed.exception)), audit.RAW_RECORD_HELPER_DIAGNOSTIC_BYTES + 100) - with mock.patch.object(audit.subprocess, "Popen", child("import sys; sys.stdin.read(); sys.exit(3)")): - with self.assertRaises(audit.ArtifactResourcePressure): - audit.normalize_record_isolated(source, 0, len(raw), timeout_seconds=30.0) - with mock.patch.object(audit.subprocess, "Popen", child("import sys; sys.stdin.read(); sys.stdout.write('not json\\n'); sys.stdout.flush()")): - with self.assertRaises(audit.ArtifactResourcePressure): - audit.normalize_record_isolated(source, 0, len(raw), timeout_seconds=30.0) - with mock.patch.object(audit.tempfile, "TemporaryFile", side_effect=OSError("disk full")): - with self.assertRaises(audit.ArtifactResourcePressure): - audit.normalize_record_isolated(source, 0, len(raw), timeout_seconds=30.0) - self.assertEqual(self.helper_threads(), []) - record = audit.normalize_record_isolated(source, 0, len(raw), member_limit=16) - try: - self.assertIn("a", record.omitted_members) - self.assertEqual(record, {"a": "x" * 100}) - finally: - record.close() - - def test_cancellation_reaches_lazy_audit_reads(self) -> None: - bundle = {"shares": [{"share_seq": i, "share_id": "s" * 3000} for i in range(40)]} - source_path = Path(tempfile.mkdtemp()) - self.addCleanup(shutil.rmtree, source_path, True) - path = source_path / "bundle.json" - path.write_bytes(json.dumps(bundle, separators=COMPACT).encode()) - state = {"cancel": False} - - class Cancelled(Exception): - pass - - def cancellation() -> None: - if state["cancel"]: - raise Cancelled() - - view = audit.CanonicalAuditBundleView.scan_path(path, cancellation=cancellation, chunk_bytes=1024) - try: - self.assertEqual(view["shares"][39]["share_seq"], 39) - state["cancel"] = True - with self.assertRaises(Cancelled): - view["shares"][39] - with mock.patch.object(audit, "RECORD_DECODE_SOFT_LIMIT_BYTES", 512): - with self.assertRaises(Cancelled): - view["shares"][3] - finally: - view.close() - - -if __name__ == "__main__": - unittest.main() diff --git a/tests/test_prism_candidate_storage_races.py b/tests/test_prism_candidate_storage_races.py deleted file mode 100644 index 85bb026aa..000000000 --- a/tests/test_prism_candidate_storage_races.py +++ /dev/null @@ -1,252 +0,0 @@ -"""Two-session PostgreSQL regressions for immutable candidate publication.""" - -from __future__ import annotations - -import os -import threading -import time -import unittest -from unittest import mock -import uuid - -from lab.prism.candidate_codec import prepare_candidate_intent -from lab.prism.candidate_store import retire_orphan_bodies_sql -from lab.prism.share_ledger import PsqlShareLedger -from tests.prism_postgres_candidate_gate import intent, pending_share_for - - -@unittest.skipUnless(os.environ.get("PRISM_TEST_DATABASE_URL"), "requires disposable PostgreSQL") -class CandidateStorageRaceTests(unittest.TestCase): - def setUp(self): - import psycopg - from psycopg.conninfo import make_conninfo - - self.admin = psycopg.connect(os.environ["PRISM_TEST_DATABASE_URL"], autocommit=True) - self.addCleanup(self.admin.close) - schema = "candidate_race_" + uuid.uuid4().hex - self.admin.execute(f'CREATE SCHEMA "{schema}"') - self.addCleanup(lambda: self.admin.execute(f'DROP SCHEMA "{schema}" CASCADE')) - url = make_conninfo(os.environ["PRISM_TEST_DATABASE_URL"], options=f"-csearch_path={schema}") - self.ledger = PsqlShareLedger( - database_url=url, psql_command="psql", native_client_mode="native", - writer_id="candidate-race", initialize_schema=True, - ) - self.addCleanup(self.ledger.close) - self.addCleanup(self.ledger.release_writer_lease) - self.first = psycopg.connect(url) - self.second = psycopg.connect(url) - self.addCleanup(self.first.close) - self.addCleanup(self.second.close) - self.prepared = prepare_candidate_intent(intent(uuid.uuid4().hex * 2, 4, credit=False)) - - def stage_without_sealing(self): - execute = self.ledger._run_json - captured = [] - - def capture(sql): - if "observed_chunk_count" in sql: - captured.append(sql) - return {"sealed": 1} - return execute(sql) - - with mock.patch.object(self.ledger, "_run_json", side_effect=capture): - body_id = self.ledger.stage_candidate_body(self.prepared) - self.assertEqual(len(captured), 1) - return body_id, captured[0] - - def publication(self): - body_id = self.ledger.stage_candidate_body(self.prepared) - with mock.patch.object(self.ledger, "stage_candidate_body", return_value=body_id), \ - mock.patch.object(self.ledger, "_run_fenced_json", return_value={"inserted": 1}) as execute: - self.ledger.persist_block_candidate_intent(self.prepared) - return body_id, execute.call_args.args[0] - - def retire_sql(self): - return retire_orphan_bodies_sql( - {**self.ledger._writer_identity_payload(), "stale_staging_seconds": 0}, - jsonb=self.ledger._jsonb_literal, - ) - - def race(self, first_sql, second_sql): - """Commit the first statement only after the second demonstrably waits.""" - initial = self.first.execute(first_sql).fetchone() - result = [] - - def run(): - try: - cursor = self.second.execute(second_sql) - result.append(cursor.fetchone() if cursor.description else None) - self.second.commit() - except Exception as exc: - result.append(exc) - self.second.rollback() - - thread = threading.Thread(target=run) - thread.start() - deadline = time.monotonic() + 10 - blocked = False - try: - while thread.is_alive() and time.monotonic() < deadline: - row = self.admin.execute( - "SELECT wait_event_type FROM pg_stat_activity WHERE pid = %s", - (self.second.info.backend_pid,), - ).fetchone() - if row and row[0] == "Lock": - blocked = True - break - time.sleep(0.01) - finally: - self.first.commit() - thread.join(10) - if thread.is_alive(): - self.second.cancel() - thread.join(5) - self.assertTrue(blocked, "second statement never reached the row-lock barrier") - self.assertFalse(thread.is_alive()) - self.assertEqual(len(result), 1) - return initial, result[0] - - def extra_part_sql(self, body_id, kind): - if kind == "chunk": - return ( - "INSERT INTO qbit_block_candidate_body_chunk VALUES " - f"('{body_id}', 999999, '\\x00', '{'0' * 64}') RETURNING ordinal" - ) - return ( - "INSERT INTO qbit_block_candidate_body_page VALUES " - f"('{body_id}', 'shares_json', 999999, 0, 0) RETURNING page_ordinal" - ) - - def test_seal_first_rejects_late_chunk(self): - body_id, seal = self.stage_without_sealing() - first, second = self.race(seal, self.extra_part_sql(body_id, "chunk")) - self.assertEqual(first[0]["sealed"], 1) - self.assertIsInstance(second, Exception) - self.assertIn("accepts no parts", str(second)) - - def test_seal_first_rejects_late_page(self): - body_id, seal = self.stage_without_sealing() - _, second = self.race(seal, self.extra_part_sql(body_id, "page")) - self.assertIsInstance(second, Exception) - self.assertIn("accepts no parts", str(second)) - - def assert_part_first_prevents_seal(self, kind): - body_id, seal = self.stage_without_sealing() - _, second = self.race(self.extra_part_sql(body_id, kind), seal) - if isinstance(second, Exception): - self.assertTrue("incomplete" in str(second) or "could not serialize" in str(second), str(second)) - else: - self.assertEqual(second[0]["sealed"], 0, "seal missed the part committed during its lock wait") - state = self.first.execute("SELECT state FROM qbit_block_candidate_body WHERE body_id = %s", (body_id,)).fetchone()[0] - self.assertEqual(state, "staging") - - def test_chunk_first_is_included_in_seal_validation(self): - self.assert_part_first_prevents_seal("chunk") - - def test_page_first_is_included_in_seal_validation(self): - self.assert_part_first_prevents_seal("page") - - def test_repeatable_read_sealer_cannot_miss_concurrent_upload(self): - self.second.execute("SET TRANSACTION ISOLATION LEVEL REPEATABLE READ") - self.assert_part_first_prevents_seal("page") - - def test_publication_first_prevents_orphan_retirement(self): - body_id, publish = self.publication() - first, second = self.race(publish, self.retire_sql()) - self.assertEqual(first[0]["inserted"], 1) - if isinstance(second, Exception): - self.assertIn("referenced by a pending outbox row", str(second)) - else: - self.assertEqual(second[0]["retired"], []) - self.assertEqual(self.first.execute("SELECT state FROM qbit_block_candidate_body WHERE body_id = %s", (body_id,)).fetchone()[0], "sealed") - - def test_retirement_first_prevents_publication(self): - body_id, publish = self.publication() - first, second = self.race(self.retire_sql(), publish) - self.assertEqual(first[0]["retired"], [body_id]) - self.assertEqual(second[0]["error"], "block candidate body is not sealed") - self.assertEqual(self.first.execute("SELECT count(*) FROM qbit_block_candidate_outbox WHERE body_id = %s", (body_id,)).fetchone()[0], 0) - - def test_repeatable_read_retirement_cannot_miss_publication(self): - body_id, publish = self.publication() - self.second.execute("SET TRANSACTION ISOLATION LEVEL REPEATABLE READ") - _, second = self.race(publish, self.retire_sql()) - self.assertIsInstance(second, Exception) - self.assertIn("could not serialize", str(second)) - self.assertEqual(self.first.execute("SELECT state FROM qbit_block_candidate_body WHERE body_id = %s", (body_id,)).fetchone()[0], "sealed") - - def test_database_publication_guard_invalidates_older_retirement_snapshot(self): - body_id = self.ledger.stage_candidate_body(self.prepared) - publish = ( - "INSERT INTO qbit_block_candidate_outbox " - "(block_hash, candidate_sha256, storage_version, body_id) VALUES " - f"('{self.prepared.block_hash}', '{self.prepared.candidate_sha256}', 2, '{body_id}') " - "RETURNING body_id" - ) - self.second.execute("SET TRANSACTION ISOLATION LEVEL REPEATABLE READ") - _, second = self.race(publish, self.retire_sql()) - self.assertIsInstance(second, Exception) - self.assertIn("could not serialize", str(second)) - self.assertEqual(self.first.execute("SELECT state FROM qbit_block_candidate_body WHERE body_id = %s", (body_id,)).fetchone()[0], "sealed") - - def test_content_comparator_rejects_forced_digest_collision(self): - body_id = self.ledger.stage_candidate_body(self.prepared) - fields = dict(self.prepared) - fields["username"] = "distinct" - different = prepare_candidate_intent(fields) - # The append caller has already accepted a matching digest. Exercise - # its independent content check with deliberately different bytes. - self.assertFalse(self.ledger.compare_candidate_body(body_id, different)) - - def test_content_comparator_preserves_jsonb_numeric_equality(self): - body_id = self.ledger.stage_candidate_body(self.prepared) - fields = dict(self.prepared) - fields["found_block"] = {"network_difficulty": 1000.0} - equivalent = prepare_candidate_intent(fields) - self.assertNotEqual(equivalent.candidate_sha256, self.prepared.candidate_sha256) - self.assertTrue(self.ledger.compare_candidate_body(body_id, equivalent)) - - def test_takeover_after_comparison_cannot_credit_share(self): - self.ledger.persist_block_candidate_intent(self.prepared) - compare = self.ledger.compare_candidate_body - - def compare_then_takeover(*args): - result = compare(*args) - self.first.execute( - "UPDATE qbit_ledger_writer_lease SET writer_session_token = %s WHERE singleton", - (uuid.uuid4().hex,), - ) - self.first.commit() - return result - - with mock.patch.object(self.ledger, "compare_candidate_body", side_effect=compare_then_takeover): - with self.assertRaisesRegex(RuntimeError, "writer lease is not active"): - self.ledger.append_batch([(pending_share_for(self.prepared), self.prepared)]) - self.assertEqual(self.first.execute("SELECT count(*) FROM qbit_share_ledger").fetchone()[0], 0) - self.assertEqual(self.first.execute("SELECT state, share_id FROM qbit_block_candidate_outbox").fetchone(), ("pending", None)) - - def test_body_retired_before_publication_is_restaged_without_duplicate_credit(self): - publish = self.ledger._publish_candidate_batch - retired = [] - - def retire_once(payloads, count): - if not retired: - body_id = payloads[0]["candidate"]["body_id"] - self.first.execute( - "UPDATE qbit_block_candidate_body SET state = 'retired', retired_at = clock_timestamp() WHERE body_id = %s", - (body_id,), - ) - self.first.commit() - retired.append(body_id) - return publish(payloads, count) - - with mock.patch.object(self.ledger, "_publish_candidate_batch", side_effect=retire_once): - self.ledger.append_batch([(pending_share_for(self.prepared), self.prepared)]) - self.assertEqual(self.first.execute("SELECT count(*) FROM qbit_share_ledger").fetchone()[0], 1) - row = self.first.execute("SELECT state, body_id FROM qbit_block_candidate_outbox").fetchone() - self.assertEqual(row[0], "pending") - self.assertNotEqual(row[1], retired[0]) - - -if __name__ == "__main__": - unittest.main() diff --git a/tests/test_prism_candidate_storm.py b/tests/test_prism_candidate_storm.py deleted file mode 100644 index c5028fa08..000000000 --- a/tests/test_prism_candidate_storm.py +++ /dev/null @@ -1,1150 +0,0 @@ -#!/usr/bin/env python3 -"""Regression coverage for the reusable PRISM candidate-storm rig.""" - -from __future__ import annotations - -import unittest - -from lab.prism.block_candidates import ( - BLOCK_CANDIDATE_COLLAPSE_CLEANUP_STEPS, - MAX_BLOCK_REPLAY_ENUMERATION_ROWS, - MAX_PENDING_BLOCK_CANDIDATES, -) -from lab.prism.coordinator_config import ( - DEFAULT_BLOCK_CANDIDATE_CLEANUP_RETRY_BACKLOG_MAX, -) -from tests.prism_candidate_storm import ( - CLEANUP_FAULT_NONE, - CLEANUP_FAULTS, - OBSERVED_TESTNET_CANDIDATE_STORM, - PERTURBATIONS, - STORM_PARENT_HASH, - CandidateStormRig, - CleanupBacklogReport, - CleanupFaultInjector, - cleanup_backlog_marginal_bytes_per_record, -) - -# A storm large enough to overflow the live queue by 2x and to exercise the -# replay enumeration's legacy widening window more than once, small enough -# to drain through the per-row path in well under a second. -DECIDED_STORM_CANDIDATES = 100 - - -class CandidateStormInstrumentTests(unittest.TestCase): - def test_observed_storm_exposes_live_and_restart_marker_cardinality( - self, - ) -> None: - """Pin the ownership semantics that falsified the first #183 design. - - Outstanding covers all live admissions, not just the bounded queue, - and replay-inflight covers every adopted restart row. Neither marker - is evidence that the candidate has already reached qbitd. - """ - - self.assertGreater( - OBSERVED_TESTNET_CANDIDATE_STORM, - MAX_BLOCK_REPLAY_ENUMERATION_ROWS, - ) - self.assertGreater( - OBSERVED_TESTNET_CANDIDATE_STORM, - MAX_PENDING_BLOCK_CANDIDATES, - ) - rig = CandidateStormRig() - - live = rig.seed_live() - self.assertEqual(live.durable_pending, OBSERVED_TESTNET_CANDIDATE_STORM) - self.assertEqual(live.live_queue_capacity, MAX_PENDING_BLOCK_CANDIDATES) - self.assertEqual(live.live_queued, MAX_PENDING_BLOCK_CANDIDATES) - self.assertEqual( - live.live_coalesced, - OBSERVED_TESTNET_CANDIDATE_STORM - MAX_PENDING_BLOCK_CANDIDATES, - ) - self.assertEqual( - live.outstanding_marked, - OBSERVED_TESTNET_CANDIDATE_STORM, - ) - self.assertTrue(live.outstanding_covers_all_durable) - self.assertEqual(live.replay_queued, 0) - self.assertEqual(live.replay_inflight_marked, 0) - - restarted = rig.restart_and_enumerate() - self.assertEqual( - restarted.durable_pending, - OBSERVED_TESTNET_CANDIDATE_STORM, - ) - self.assertEqual(restarted.live_queued, 0) - self.assertEqual(restarted.live_coalesced, 0) - self.assertEqual(restarted.outstanding_marked, 0) - self.assertEqual( - restarted.replay_queued, - OBSERVED_TESTNET_CANDIDATE_STORM, - ) - self.assertEqual( - restarted.replay_inflight_marked, - OBSERVED_TESTNET_CANDIDATE_STORM, - ) - self.assertTrue(restarted.replay_inflight_covers_all_durable) - self.assertEqual( - restarted.accepted_parent_previews, - OBSERVED_TESTNET_CANDIDATE_STORM, - ) - self.assertFalse(restarted.replay_enumeration_owed) - - -class DecidedCandidateStormTests(unittest.TestCase): - """Pin what the rig reports once the storm height is decided. - - These are component facts about the shipped tree, not a selector: they - say which in-memory and durable facts exist for never-offered siblings - versus candidates with a real or ambiguous node offer, and what the - per-row disposition path costs to drain the sibling set. - """ - - def test_decided_height_leaves_siblings_without_offer_evidence(self) -> None: - rig = CandidateStormRig(candidates=DECIDED_STORM_CANDIDATES) - rig.seed_live() - winner = rig.decide_height() - self.assertEqual(winner, rig.block_hashes[-1]) - self.assertEqual(rig.decided_rpc.call("getbestblockhash"), winner) - self.assertEqual( - rig.decided_rpc.call("getblockhash", [rig.storm_height]), - winner, - ) - - live = rig.snapshot(rig.live_server) - self.assertEqual(live.outstanding_marked, DECIDED_STORM_CANDIDATES) - self.assertEqual(live.selector_evidence_marked, 0) - ownership = rig.ownership(rig.live_server) - self.assertEqual(len(ownership), DECIDED_STORM_CANDIDATES) - decided_header = rig.decided_rpc.call("getblockheader", [winner]) - for record in ownership: - self.assertEqual(record.outbox_state, "pending") - self.assertEqual(record.attempt_count, 0) - self.assertEqual(record.parent_hash, STORM_PARENT_HASH) - self.assertEqual(record.expected_height, rig.storm_height) - # Equal work: the decided block and every sibling were built at - # the same difficulty, so height occupancy is decisive here. - self.assertEqual(record.network_difficulty, decided_header["difficulty"]) - self.assertTrue(record.outstanding) - self.assertFalse(record.selector_evidence) - self.assertEqual( - sum(1 for record in ownership if record.live_queued), - MAX_PENDING_BLOCK_CANDIDATES, - ) - # The winner is the last seeded candidate: a coalesced wakeup in the - # live view, so the queue holds only siblings. - self.assertFalse(ownership[-1].live_queued) - - restarted = rig.restart_and_enumerate() - self.assertEqual(restarted.replay_inflight_marked, DECIDED_STORM_CANDIDATES) - self.assertEqual(restarted.accepted_parent_previews, DECIDED_STORM_CANDIDATES) - self.assertEqual(restarted.selector_evidence_marked, 0) - self.assertIs(rig.restarted_server.rpc, rig.decided_rpc) - self.assertTrue( - all(record.replay_queued for record in rig.ownership(rig.restarted_server)) - ) - - def test_selector_evidence_gauge_counts_each_evidence_kind_once(self) -> None: - rig = CandidateStormRig(candidates=DECIDED_STORM_CANDIDATES) - rig.seed_live() - rig.decide_height() - server = rig.live_server - hashes = rig.block_hashes - undo = [ - rig.perturb(server, kind, hashes[index]) - for index, kind in enumerate(PERTURBATIONS) - ] - try: - by_hash = {record.block_hash: record for record in rig.ownership(server)} - self.assertTrue(by_hash[hashes[0]].disposition_held) - self.assertTrue(by_hash[hashes[1]].node_acceptance_retained) - self.assertTrue(by_hash[hashes[2]].tip_observed) - self.assertTrue(by_hash[hashes[3]].retry_held) - self.assertEqual(by_hash[hashes[4]].pool_block_chain_state, "prepared") - self.assertIs(by_hash[hashes[5]].terminal_outcome, False) - for block_hash in hashes[:len(PERTURBATIONS)]: - self.assertTrue(by_hash[block_hash].selector_evidence) - # Process ownership is unchanged by evidence: still outstanding. - self.assertTrue(by_hash[block_hash].outstanding) - self.assertEqual( - sum(1 for record in by_hash.values() if record.selector_evidence), - len(PERTURBATIONS), - ) - # The union is conservative, but the rig still says which rows - # actually attest a node offer. Three do not: the retry holder, - # the disposition lease and the terminal registry are each - # reachable without any submitblock. - widenings = { - "retry_slot": "retry_held_without_offer", - "lease_held": "lease_held_without_offer", - "terminal": "terminal_without_offer", - } - for kind, attested in widenings.items(): - with self.subTest(perturbation=kind): - record = by_hash[hashes[PERTURBATIONS.index(kind)]] - self.assertFalse(record.node_offer_evidence) - for name in widenings.values(): - self.assertEqual(getattr(record, name), name == attested) - for index, kind in enumerate(PERTURBATIONS): - if kind in widenings: - continue - with self.subTest(perturbation=kind): - self.assertTrue(by_hash[hashes[index]].node_offer_evidence) - for name in widenings.values(): - self.assertFalse(getattr(by_hash[hashes[index]], name)) - snapshot = rig.snapshot(server) - self.assertEqual( - snapshot.selector_evidence_marked, - len(PERTURBATIONS), - ) - self.assertEqual( - snapshot.node_offer_evidence_marked, - len(PERTURBATIONS) - len(widenings), - ) - self.assertEqual(snapshot.unoffered_retry_marked, 1) - self.assertEqual(snapshot.unoffered_lease_marked, 1) - self.assertEqual(snapshot.unoffered_terminal_marked, 1) - # The published split accounts for the whole union: every row the - # conservative contract covers is either an attested offer or one - # of the three named widenings, and here the widenings are - # disjoint. - self.assertEqual( - snapshot.selector_evidence_marked, - snapshot.node_offer_evidence_marked - + snapshot.unoffered_retry_marked - + snapshot.unoffered_lease_marked - + snapshot.unoffered_terminal_marked, - ) - finally: - for release in undo: - if release is not None: - release() - - def test_retry_retention_alone_is_not_node_offer_evidence(self) -> None: - """Parking a wakeup in the retry holder must not read as an offer. - - ``_retain_block_candidate_for_retry`` runs on shipped paths that - never reached qbitd -- most plainly when - ``_reserve_block_fast_lane_slot`` declines and ``submit_next`` parks - the candidate *before* any submitblock -- so a rig that folded the - holder into its offer evidence would credit a node call that never - happened. The #183 must-not-abandon union still has to cover the - row, because the holder cannot distinguish that path from a genuine - offer retrying its tail. - """ - - rig = CandidateStormRig(candidates=DECIDED_STORM_CANDIDATES) - rig.seed_live() - rig.decide_height() - server = rig.live_server - parked = rig.block_hashes[0] - - before = {record.block_hash: record for record in rig.ownership(server)} - self.assertTrue(before[parked].live_queued) - self.assertFalse(before[parked].retry_held) - self.assertFalse(before[parked].selector_evidence) - calls_before = dict(rig.decided_rpc.calls) - - # Move a queued, never-offered wakeup into the retry slot, exactly as - # a declined fast-lane reservation does. - rig.perturb(server, "retry_slot", parked) - record = {r.block_hash: r for r in rig.ownership(server)}[parked] - self.assertTrue(record.retry_held) - - # Nothing was offered: no RPC happened, and no fact downstream of an - # offer exists for the row. - self.assertEqual(rig.decided_rpc.calls, calls_before) - self.assertFalse(record.node_offer_evidence) - self.assertTrue(record.retry_held_without_offer) - self.assertFalse(record.disposition_held) - self.assertFalse(record.node_acceptance_retained) - self.assertFalse(record.tip_observed) - self.assertFalse(record.accounted_accepted) - self.assertIsNone(record.pool_block_chain_state) - # Nor is the row reported terminal: retry retention is not a - # disposition, so the durable row is untouched and unattempted. - self.assertIsNone(record.terminal_outcome) - self.assertEqual(record.outbox_state, "pending") - self.assertEqual(record.attempt_count, 0) - # The conservative contract still covers it. - self.assertTrue(record.selector_evidence) - - snapshot = rig.snapshot(server) - self.assertEqual(snapshot.selector_evidence_marked, 1) - self.assertEqual(snapshot.node_offer_evidence_marked, 0) - self.assertEqual(snapshot.unoffered_retry_marked, 1) - - def test_lease_retention_alone_is_not_node_offer_evidence(self) -> None: - """Holding the disposition lease must not read as an offer. - - ``submit_next`` claims the lease on the dequeued hash *before* it - consults the terminal outcome, the accounted/capacity close and - ``_reserve_block_fast_lane_slot``; each of those paths releases it - again without ever reaching submitblock. The flight registry this is - read from is weaker still -- ``_claim_block_candidate_disposition`` - registers a user before taking the flight's lock, so a blocked waiter - is present too. A rig that folded the lease into its offer evidence - would credit a node call that never happened, and #183's safety - comparison is published against exactly that split. The - must-not-abandon union still has to cover the row, because a held - lease may equally span an offer already in flight. - """ - - rig = CandidateStormRig(candidates=DECIDED_STORM_CANDIDATES) - rig.seed_live() - rig.decide_height() - server = rig.live_server - leased = rig.block_hashes[0] - - before = {record.block_hash: record for record in rig.ownership(server)} - self.assertFalse(before[leased].disposition_held) - self.assertFalse(before[leased].selector_evidence) - calls_before = dict(rig.decided_rpc.calls) - - # Claim the lease through the shipped entry point, exactly as - # submit_next does before it has decided anything about the row. - release = rig.perturb(server, "lease_held", leased) - try: - record = {r.block_hash: r for r in rig.ownership(server)}[leased] - self.assertTrue(record.disposition_held) - - # Nothing was offered: no RPC happened, and no fact downstream of - # an offer exists for the row. - self.assertEqual(rig.decided_rpc.calls, calls_before) - self.assertFalse(record.node_offer_evidence) - self.assertTrue(record.lease_held_without_offer) - self.assertFalse(record.retry_held) - self.assertFalse(record.retry_held_without_offer) - self.assertFalse(record.node_acceptance_retained) - self.assertFalse(record.tip_observed) - self.assertFalse(record.accounted_accepted) - self.assertIsNone(record.pool_block_chain_state) - # Nor is the row disposed: the lease is a claim on the decision, - # not the decision, so the durable row is untouched and - # unattempted. - self.assertIsNone(record.terminal_outcome) - self.assertEqual(record.outbox_state, "pending") - self.assertEqual(record.attempt_count, 0) - # The conservative contract still covers it. - self.assertTrue(record.selector_evidence) - - snapshot = rig.snapshot(server) - self.assertEqual(snapshot.selector_evidence_marked, 1) - self.assertEqual(snapshot.node_offer_evidence_marked, 0) - self.assertEqual(snapshot.unoffered_lease_marked, 1) - self.assertEqual(snapshot.unoffered_retry_marked, 0) - finally: - release() - - # Releasing drops the row out of the union again: the widening reads - # the live flight registry rather than a sticky marker, so it cannot - # accumulate the storm the way outstanding/replay-inflight do. - after = {r.block_hash: r for r in rig.ownership(server)}[leased] - self.assertFalse(after.disposition_held) - self.assertFalse(after.selector_evidence) - self.assertEqual(rig.snapshot(server).selector_evidence_marked, 0) - self.assertEqual(rig.decided_rpc.calls, calls_before) - - def test_capacity_closed_terminalization_is_not_node_offer_evidence( - self, - ) -> None: - """A terminal outcome reached with zero submitblock is not an offer. - - Whenever pool capacity is already closed, ``submit_next`` builds a - ``_BlockCandidateNodeSubmission(attempted=False)`` and hands it - straight to the accounting queue, which marks the durable attempt and - terminalizes the outbox row without calling qbitd once. A rig that - folded the terminal registry into its offer evidence would publish - that row as an attested node offer, and #183's safety comparison is - published against exactly that split. The conservative - must-not-abandon union still has to cover the row, because a terminal - outcome may equally be the seal on an offer that returned. - """ - - rig = CandidateStormRig(candidates=DECIDED_STORM_CANDIDATES) - rig.seed_live() - winner = rig.decide_height() - server = rig.live_server - closed = rig.block_hashes[0] - - before = {r.block_hash: r for r in rig.ownership(server)}[closed] - self.assertIsNone(before.terminal_outcome) - self.assertFalse(before.selector_evidence) - self.assertNotIn("submitblock", rig.decided_rpc.calls) - - # Close pool capacity through the shipped coordinator tunable the - # production check reads, then drive one row of the shipped path. - # ``max_rounds`` cannot bound this call: it budgets node offers and - # this path makes none, so the single row is stepped by withholding - # every other one instead. - server.max_blocks = 0 - drain = rig.drain_per_row( - server, - stop_before_hash=winner, - preserve_hashes=[ - block_hash for block_hash in rig.block_hashes if block_hash != closed - ], - ) - - # The rig's own RPC call log is the proof: the row was terminalized - # and not one submitblock was made, on this call or ever -- so the - # call performed no rounds at all, and the report's offer gauge - # agrees with its RPC delta instead of contradicting it. - self.assertEqual(drain.rounds, 0) - self.assertEqual(drain.rpc_calls.get("submitblock", 0), 0) - self.assertNotIn("submitblock", rig.decided_rpc.calls) - self.assertEqual(drain.abandoned_hashes, frozenset({closed})) - self.assertEqual(drain.submitted_hashes, frozenset()) - self.assertEqual( - drain.withheld_hashes, - frozenset(rig.block_hashes) - {closed}, - ) - # The durable row still records an attempt, so attempt_count is no - # substitute for the missing RPC: submit_writer marks admission to a - # processing phase before the offer branch it never took. - self.assertEqual(drain.ledger_attempt_marks, 1) - - record = {r.block_hash: r for r in rig.ownership(server)}[closed] - self.assertEqual(record.outbox_state, "abandoned") - self.assertIs(record.terminal_outcome, False) - self.assertEqual(record.attempt_count, 1) - self.assertFalse(record.node_offer_evidence) - self.assertTrue(record.terminal_without_offer) - # Nothing else attests an offer for the row either. - self.assertFalse(record.node_acceptance_retained) - self.assertFalse(record.tip_observed) - self.assertFalse(record.accounted_accepted) - self.assertIsNone(record.pool_block_chain_state) - self.assertFalse(record.retry_held) - self.assertFalse(record.retry_held_without_offer) - self.assertFalse(record.disposition_held) - self.assertFalse(record.lease_held_without_offer) - # The conservative contract still covers it. - self.assertTrue(record.selector_evidence) - - snapshot = rig.snapshot(server) - self.assertEqual(snapshot.node_offer_evidence_marked, 0) - self.assertEqual(snapshot.unoffered_terminal_marked, 1) - self.assertEqual(snapshot.unoffered_retry_marked, 0) - self.assertEqual(snapshot.unoffered_lease_marked, 0) - self.assertEqual(snapshot.selector_evidence_marked, 1) - - # The oracle's exact-set properties are unchanged by the narrowed - # gauge: reopening capacity drains every remaining sibling through - # the ordinary offer path, and the two calls partition the sibling - # set exactly, each reporting only its own transitions. - server.max_blocks = DECIDED_STORM_CANDIDATES + 1 - rest = rig.drain_per_row(server, stop_before_hash=winner) - self.assertEqual( - rest.rpc_calls["submitblock"], - DECIDED_STORM_CANDIDATES - 2, - ) - self.assertTrue(rest.abandoned_hashes.isdisjoint(drain.abandoned_hashes)) - self.assertEqual( - drain.abandoned_hashes | rest.abandoned_hashes, - frozenset(rig.block_hashes) - {winner}, - ) - self.assertEqual(rest.pending_hashes, frozenset({winner})) - self.assertEqual(rest.submitted_hashes, frozenset()) - self.assertEqual(rest.withheld_hashes, frozenset()) - # Every abandoned row is now conservatively covered and none of them - # is published as an attested offer. - after = rig.ownership(server) - self.assertEqual( - sum(1 for r in after if r.terminal_without_offer), - DECIDED_STORM_CANDIDATES - 1, - ) - self.assertEqual( - rig.snapshot(server).node_offer_evidence_marked, - 0, - ) - - def test_replay_adoption_alone_drops_a_tip_observation(self) -> None: - """Why ``perturb('tip_observed')`` also registers the hash outstanding. - - Restart adoption never marks a hash outstanding -- only the node - offer does -- so a blockwait observation of a replayed candidate is - dropped until it is dequeued and offered. A selector that reads the - observation set alone therefore cannot see a replayed candidate's - acceptance, which is why the #181 dequeue-time skip has to read the - chain for replayed rows. - """ - - rig = CandidateStormRig(candidates=DECIDED_STORM_CANDIDATES) - rig.seed_live() - rig.restart_and_enumerate() - rig.decide_height() - server = rig.restarted_server - replayed = rig.block_hashes[0] - self.assertEqual(rig.snapshot(server).outstanding_marked, 0) - - server._note_tip_observation_for_candidates(replayed) - by_hash = {record.block_hash: record for record in rig.ownership(server)} - self.assertFalse(by_hash[replayed].tip_observed) - self.assertFalse(by_hash[replayed].selector_evidence) - self.assertEqual(rig.snapshot(server).selector_evidence_marked, 0) - - rig.perturb(server, "tip_observed", replayed) - by_hash = {record.block_hash: record for record in rig.ownership(server)} - self.assertTrue(by_hash[replayed].tip_observed) - self.assertEqual(rig.snapshot(server).selector_evidence_marked, 1) - - def test_per_row_drain_cost_is_linear_in_siblings_live_and_after_restart( - self, - ) -> None: - siblings = DECIDED_STORM_CANDIDATES - 1 - expected_rpc = { - # One fast-lane offer per sibling, then the admission tip read plus - # the abandon seal's pre- and post-withdrawal chain probes: three - # getbestblockhash/getblockheader pairs per sibling. - "submitblock": siblings, - "getbestblockhash": 3 * siblings, - "getblockheader": 3 * siblings, - } - - live_rig = CandidateStormRig(candidates=DECIDED_STORM_CANDIDATES) - live_rig.seed_live() - winner = live_rig.decide_height() - drain = live_rig.drain_per_row(live_rig.live_server, stop_before_hash=winner) - self.assertEqual(drain.rounds, siblings) - # The exact set, not just its size: this is what a selector is - # compared against. - self.assertEqual( - drain.abandoned_hashes, - frozenset(live_rig.block_hashes) - {winner}, - ) - self.assertEqual(drain.pending_hashes, frozenset({winner})) - self.assertEqual(drain.submitted_hashes, frozenset()) - self.assertEqual(drain.withheld_hashes, frozenset()) - # Every offer was handed to the real accounting queue and run there. - self.assertEqual(drain.accounting_tasks, siblings) - self.assertEqual(drain.ledger_attempt_marks, siblings) - self.assertEqual(drain.rpc_calls, expected_rpc) - # 32 queued wakeups drain first; the coalesced remainder is recovered - # through the outbox query in 32-row windows: 32 + 32 + 4 rows. - self.assertEqual(drain.replay_enumerations, 3) - after = {record.block_hash: record for record in live_rig.ownership(live_rig.live_server)} - self.assertEqual(after[winner].outbox_state, "pending") - self.assertFalse(after[winner].selector_evidence) - self.assertTrue( - all( - record.outbox_state == "abandoned" and record.terminal_outcome is False - for block_hash, record in after.items() - if block_hash != winner - ) - ) - # Every sibling's offer-time barrier was withdrawn by its abandonment; - # the one left is the winner's, armed when the live process's own - # recovery enumeration adopted it from the outbox (adoption arms a - # barrier per row exactly as restart adoption does). - self.assertEqual(live_rig.snapshot(live_rig.live_server).accepted_parent_previews, 1) - - restart_rig = CandidateStormRig(candidates=DECIDED_STORM_CANDIDATES) - restart_rig.seed_live() - restart_rig.restart_and_enumerate() - winner = restart_rig.decide_height() - drain = restart_rig.drain_per_row( - restart_rig.restarted_server, - stop_before_hash=winner, - ) - self.assertEqual(drain.rounds, siblings) - self.assertEqual( - drain.abandoned_hashes, - frozenset(restart_rig.block_hashes) - {winner}, - ) - self.assertEqual(drain.pending_hashes, frozenset({winner})) - self.assertEqual(drain.accounting_tasks, siblings) - self.assertEqual(drain.ledger_attempt_marks, siblings) - self.assertEqual(drain.rpc_calls, expected_rpc) - # Restart adoption already queued every row; no enumeration is needed. - self.assertEqual(drain.replay_enumerations, 0) - # Only the undrained winner's adoption-time barrier remains. - self.assertEqual( - restart_rig.snapshot(restart_rig.restarted_server).accepted_parent_previews, - 1, - ) - - def test_per_row_drain_runs_the_production_submitter_split(self) -> None: - """The drain must defer accounting, as ``BlockCandidateService.run`` does. - - Two shipped behaviours exist only on that split, and both decide - whether a row is abandoned: ``submit_next`` claims the disposition - lease *blocking* when accounting is inline, so one candidate whose - lease is held elsewhere would stop the drain forever; and - ``_restore_replayed_candidate_acceptance_evidence`` runs only inside - the accounting task, so an inline drain abandons a replayed candidate - with a prepared pool-block row that production preserves. - """ - - rig = CandidateStormRig(candidates=DECIDED_STORM_CANDIDATES) - rig.seed_live() - winner = rig.decide_height() - server = rig.live_server - held = rig.block_hashes[0] - release = rig.perturb(server, "lease_held", held) - try: - drain = rig.drain_per_row(server, stop_before_hash=winner) - finally: - release() - # Bounded: the held lease parked one wakeup instead of blocking, and - # every other sibling still drained. - self.assertEqual(drain.lease_blocked_hashes, frozenset({held})) - self.assertNotIn(held, drain.abandoned_hashes) - self.assertIn(held, drain.pending_hashes) - self.assertEqual( - drain.abandoned_hashes, - frozenset(rig.block_hashes) - {winner, held}, - ) - # The lease-held row was never offered, so it cost no submitblock. - self.assertEqual( - drain.rpc_calls["submitblock"], - DECIDED_STORM_CANDIDATES - 2, - ) - - def test_max_rounds_budgets_offers_not_wakeups_when_a_lease_is_held( - self, - ) -> None: - """A parked no-offer row must not spend the bounded drain's budget. - - ``submit_next_block_candidate(defer_accounting=True)`` returns True - for the head whose disposition lease it could not claim: it only - parks the wakeup in ``_block_disposition_waiting_retries`` and - returns, having offered qbitd nothing. A ``max_rounds`` budget spent - on that boolean would end a bounded drain with zero submitblock - calls, zero accounting tasks and zero terminalizations, while the - report still claimed a round -- so the gauge would contradict this - report's own RPC, accounting and terminal-set fields, and a caller - stepping the oracle one offered row at a time would silently get no - row at all. - """ - - rig = CandidateStormRig(candidates=DECIDED_STORM_CANDIDATES) - rig.seed_live() - winner = rig.decide_height() - server = rig.live_server - service = server._ensure_block_candidate_service() - # The head of the live lane, so the parked row is the first one the - # bounded drain reaches. - held = str( - list(service.candidate_queue.queue)[0].submission.block_hash_hex - ).lower() - - release = rig.perturb(server, "lease_held", held) - try: - drain = rig.drain_per_row( - server, - stop_before_hash=winner, - max_rounds=1, - ) - # Read while the lease still stands: that is the state the drain - # saw and the reason it parked the row. - record = {r.block_hash: r for r in rig.ownership(server)}[held] - finally: - release() - - # The parked row cost no offer, so it cost no round; the drain moved - # past it and spent its single round on a row it really offered. - self.assertEqual(drain.rounds, 1) - self.assertEqual(drain.rpc_calls.get("submitblock", 0), 1) - self.assertEqual(drain.accounting_tasks, 1) - self.assertEqual(drain.ledger_attempt_marks, 1) - # Every cost field agrees with the terminal sets: exactly one row was - # terminalized, and it is not the parked one. - self.assertEqual(len(drain.abandoned_hashes), 1) - self.assertNotIn(held, drain.abandoned_hashes) - self.assertEqual(drain.submitted_hashes, frozenset()) - self.assertEqual(drain.lease_blocked_hashes, frozenset({held})) - self.assertEqual(drain.withheld_hashes, frozenset({held})) - self.assertIn(held, drain.pending_hashes) - self.assertEqual( - drain.pending_hashes, - frozenset(rig.block_hashes) - drain.abandoned_hashes, - ) - - # The parked row itself is untouched: no offer, no durable attempt, - # no terminal outcome -- only the lease this test holds. - self.assertEqual(record.outbox_state, "pending") - self.assertEqual(record.attempt_count, 0) - self.assertIsNone(record.terminal_outcome) - self.assertFalse(record.node_offer_evidence) - self.assertTrue(record.disposition_held) - self.assertTrue(record.lease_held_without_offer) - self.assertTrue(record.selector_evidence) - - def test_per_row_drain_preserves_every_offer_evidence_perturbation( - self, - ) -> None: - """What the shipped path does with each evidence shape, in both views. - - Only two shapes reach a terminal abandonment: a wakeup sitting in the - retry slot (dequeued and re-offered like any other), and, in the live - view only, a prepared pool-block row (rejected first, then - abandoned). The same prepared row after restart is preserved, because - the accounting task restores its acceptance evidence. ``lease_held`` - carries no node-offer evidence either, but it is spared for an - unrelated reason: the drain cannot claim its lease, so the shipped - path parks that wakeup instead of deciding it. - """ - - for view in ("live", "restart"): - with self.subTest(view=view): - rig = CandidateStormRig(candidates=DECIDED_STORM_CANDIDATES) - rig.seed_live() - if view == "restart": - rig.restart_and_enumerate() - winner = rig.decide_height() - server = rig.restarted_server if view == "restart" else rig.live_server - hashes = dict(zip(PERTURBATIONS, rig.block_hashes)) - undo = [ - rig.perturb(server, kind, block_hash) - for kind, block_hash in hashes.items() - ] - self.assertEqual( - rig.snapshot(server).selector_evidence_marked, - len(PERTURBATIONS), - ) - try: - drain = rig.drain_per_row( - server, - stop_before_hash=winner, - # A retained definitive success re-runs the landing - # tail rather than the abandon path, and that tail is - # outside this instrument. - preserve_hashes=[hashes["retained_success"]], - ) - finally: - for release in undo: - if release is not None: - release() - - abandoned = {"retry_slot"} - if view == "live": - abandoned.add("prepared_pool_block") - for kind, block_hash in hashes.items(): - with self.subTest(perturbation=kind): - if kind in abandoned: - self.assertIn(block_hash, drain.abandoned_hashes) - else: - self.assertNotIn(block_hash, drain.abandoned_hashes) - self.assertIn(block_hash, drain.pending_hashes) - self.assertEqual( - drain.lease_blocked_hashes, - frozenset({hashes["lease_held"]}), - ) - expected_deferred = {hashes["tip_observed"]} - if view == "restart": - expected_deferred.add(hashes["prepared_pool_block"]) - self.assertEqual(drain.deferred_hashes, frozenset(expected_deferred)) - self.assertEqual( - drain.abandoned_rows, - DECIDED_STORM_CANDIDATES - 1 - len(hashes) + len(abandoned), - ) - self.assertEqual(drain.submitted_hashes, frozenset()) - self.assertIn(winner, drain.pending_hashes) - - def test_preserved_wakeups_survive_the_drain_that_withheld_them(self) -> None: - """``preserve_hashes`` is invocation-scoped, so it must not eat wakeups. - - A withheld row is lifted out of the dequeue order without being - offered or disposed, which leaves its ownership markers standing. - After restart adoption that includes ``_block_replay_inflight_hashes``, - and ``_enqueue_replayed_block_candidate`` drops any re-adoption of an - in-flight hash as a duplicate -- so a drain that discarded the lifted - wakeup would leave the row pending with nothing left to dispose of it - and no enumeration able to rebuild it. Partial and repeated drains - re-apply the withholding every call, so the wakeup has to survive each - one, and a later unprotected drain has to be able to process the row. - """ - - for view in ("live", "restart"): - with self.subTest(view=view): - rig = CandidateStormRig(candidates=DECIDED_STORM_CANDIDATES) - rig.seed_live() - if view == "restart": - rig.restart_and_enumerate() - winner = rig.decide_height() - server = rig.restarted_server if view == "restart" else rig.live_server - service = server._ensure_block_candidate_service() - protected = rig.block_hashes[0] - - def queued_wakeups() -> set[str]: - return { - str(item.submission.block_hash_hex).lower() - for lane in ( - service.candidate_queue, - service._block_replay_candidate_queue, - ) - for item in list(lane.queue) - } - - self.assertIn(protected, queued_wakeups()) - - # Repeated partial drains: each one withholds the row again. - steps = [ - rig.drain_per_row( - server, - stop_before_hash=winner, - preserve_hashes=[protected], - max_rounds=2, - ) - for _ in range(2) - ] - for index, step in enumerate(steps): - with self.subTest(step=index): - self.assertEqual(step.rounds, 2) - self.assertEqual(step.withheld_hashes, frozenset({protected})) - self.assertNotIn(protected, step.abandoned_hashes) - self.assertIn(protected, step.pending_hashes) - self.assertEqual(len(step.abandoned_hashes), 2) - self.assertTrue( - steps[0].abandoned_hashes.isdisjoint(steps[1].abandoned_hashes) - ) - self.assertIn(protected, queued_wakeups()) - - # ...then an unbounded preserving drain over the remainder. - first = rig.drain_per_row( - server, - stop_before_hash=winner, - preserve_hashes=[protected], - ) - self.assertEqual(first.withheld_hashes, frozenset({protected})) - self.assertNotIn(protected, first.abandoned_hashes) - self.assertEqual( - first.pending_hashes, - frozenset({protected, winner}), - ) - - # The protected row still owns exactly one in-memory wakeup, - # in a lane a later drain dequeues from... - self.assertIn(protected, queued_wakeups()) - self.assertEqual( - sum( - 1 - for lane in ( - service.candidate_queue, - service._block_replay_candidate_queue, - ) - for item in list(lane.queue) - if str(item.submission.block_hash_hex).lower() == protected - ), - 1, - ) - record = {r.block_hash: r for r in rig.ownership(server)}[protected] - self.assertTrue(record.live_queued or record.replay_queued) - # ...and its durable row and evidence are untouched, exactly - # as the caller arranged them. - self.assertEqual(record.outbox_state, "pending") - self.assertEqual(record.attempt_count, 0) - self.assertIsNone(record.terminal_outcome) - self.assertFalse(record.selector_evidence) - self.assertFalse(record.node_offer_evidence) - - # A later unprotected drain reaches it and disposes of it for - # real -- one offer, one accounting task, one durable attempt. - second = rig.drain_per_row(server, stop_before_hash=winner) - self.assertEqual(second.rounds, 1) - self.assertEqual(second.accounting_tasks, 1) - self.assertEqual(second.ledger_attempt_marks, 1) - self.assertEqual(second.rpc_calls["submitblock"], 1) - self.assertEqual(second.abandoned_hashes, frozenset({protected})) - self.assertEqual(second.withheld_hashes, frozenset()) - self.assertEqual(second.pending_hashes, frozenset({winner})) - # Every call partitions the sibling set; nothing was stranded. - self.assertEqual( - steps[0].abandoned_hashes - | steps[1].abandoned_hashes - | first.abandoned_hashes - | second.abandoned_hashes, - frozenset(rig.block_hashes) - {winner}, - ) - - def test_bounded_drains_report_only_their_own_transitions(self) -> None: - """``max_rounds`` reports this call's work, not the rig's history. - - A bounded drain is how a caller steps the oracle one row at a time. - If the report named every terminal row standing in the outbox instead - of the ones this call moved, the second step would claim the first - step's abandonment, and no per-step comparison against a selector - would mean anything. - """ - - steps = 3 - rig = CandidateStormRig(candidates=DECIDED_STORM_CANDIDATES) - rig.seed_live() - winner = rig.decide_height() - server = rig.live_server - - seen: frozenset[str] = frozenset() - for step in range(steps): - with self.subTest(step=step): - report = rig.drain_per_row( - server, - stop_before_hash=winner, - max_rounds=1, - ) - # One row offered, one accounted, one durable attempt marked. - self.assertEqual(report.rounds, 1) - self.assertEqual(report.accounting_tasks, 1) - self.assertEqual(report.ledger_attempt_marks, 1) - self.assertEqual(report.rpc_calls["submitblock"], 1) - # The first 32 wakeups are already queued, so no step needs - # the recovery enumeration. - self.assertEqual(report.replay_enumerations, 0) - # ...and exactly one transition, never an earlier step's. - self.assertEqual(len(report.abandoned_hashes), 1) - self.assertEqual(report.submitted_hashes, frozenset()) - self.assertEqual(report.withheld_hashes, frozenset()) - self.assertTrue(report.abandoned_hashes.isdisjoint(seen)) - seen |= report.abandoned_hashes - # pending_hashes is the complementary residue of the same - # outbox, so it shrinks by exactly the rows drained so far. - self.assertEqual( - report.pending_hashes, - frozenset(rig.block_hashes) - seen, - ) - self.assertEqual(len(seen), steps) - - # The bounded steps and the unbounded remainder partition the sibling - # set exactly: no row is reported twice and none is dropped. - final = rig.drain_per_row(server, stop_before_hash=winner) - self.assertEqual(final.rounds, DECIDED_STORM_CANDIDATES - 1 - steps) - self.assertTrue(final.abandoned_hashes.isdisjoint(seen)) - self.assertEqual( - seen | final.abandoned_hashes, - frozenset(rig.block_hashes) - {winner}, - ) - self.assertEqual(final.submitted_hashes, frozenset()) - self.assertEqual(final.withheld_hashes, frozenset()) - self.assertEqual(final.pending_hashes, frozenset({winner})) - - def test_per_row_drain_at_the_observed_cardinality(self) -> None: - """The same facts at the incident's 3,120 rows, live and after restart. - - 3,120 is above both the 32-entry live queue and the 1,024-row replay - page, so the live view exercises the recovery enumeration 97 times - and the restart view exercises multi-page adoption. - """ - - siblings = OBSERVED_TESTNET_CANDIDATE_STORM - 1 - expected_rpc = { - "submitblock": siblings, - "getbestblockhash": 3 * siblings, - "getblockheader": 3 * siblings, - } - for view, enumerations in (("live", 97), ("restart", 0)): - with self.subTest(view=view): - rig = CandidateStormRig() - rig.seed_live() - if view == "restart": - rig.restart_and_enumerate() - winner = rig.decide_height() - server = rig.restarted_server if view == "restart" else rig.live_server - drain = rig.drain_per_row(server, stop_before_hash=winner) - self.assertEqual( - drain.abandoned_hashes, - frozenset(rig.block_hashes) - {winner}, - ) - self.assertEqual(drain.pending_hashes, frozenset({winner})) - self.assertEqual(drain.submitted_hashes, frozenset()) - self.assertEqual(drain.withheld_hashes, frozenset()) - self.assertEqual(drain.rounds, siblings) - self.assertEqual(drain.accounting_tasks, siblings) - self.assertEqual(drain.ledger_attempt_marks, siblings) - self.assertEqual(drain.rpc_calls, expected_rpc) - self.assertEqual(sum(drain.rpc_calls.values()), 21_833) - self.assertEqual(drain.replay_enumerations, enumerations) - # Only the undrained winner's barrier is left standing. - self.assertEqual( - rig.snapshot(server).accepted_parent_previews, - 1, - ) - - def test_parent_transition_lookup_probes_once_per_distinct_height(self) -> None: - rig = CandidateStormRig(candidates=DECIDED_STORM_CANDIDATES) - rig.seed_live() - rig.restart_and_enumerate() - rig.decide_height() - server = rig.restarted_server - self.assertEqual( - rig.snapshot(server).accepted_parent_previews, - DECIDED_STORM_CANDIDATES, - ) - before = dict(rig.decided_rpc.calls) - # A parent with no exact transition: the lookup falls through to the - # ancestor scan, which #193 deduplicates by candidate height. - selected = server._accepted_block_payout_transition_for_parent( - "22" * 32, - parent_height=rig.storm_height, - ) - self.assertEqual( - rig.decided_rpc.calls.get("getblockhash", 0) - before.get("getblockhash", 0), - 1, - ) - self.assertEqual(selected, (rig.block_hashes[-1], False)) - - -class CleanupBacklogStormTests(unittest.TestCase): - """Issue #198: wall-clock corroboration at the observed cardinality. - - The deterministic contract lives in ``tests/test_prism_block_candidates``; - these runs drive the shipped collapse walk over 3,120 durable rows under - each injected cleanup dependency fault, hold the fault for a bounded run - of failing retries, heal it, and require the rig's recovery proof to - hold: the backlog drains through the shipped retry pass, every collapsed - hash is cleaned up exactly once, no terminal row is replay-adopted or - re-offered, and the decided winner keeps its pending-share floor - authority. Wall-clock bounds are deliberately loose; they corroborate - the measured throughput rather than pin it. - """ - - SIBLINGS = OBSERVED_TESTNET_CANDIDATE_STORM - 1 - - def _assert_recovered(self, report: CleanupBacklogReport) -> None: - self.assertEqual(report.depth_after_recovery, 0, report) - self.assertEqual(report.recovered_records, report.deferred_records, report) - self.assertEqual(report.recovery_passes, report.deferred_records, report) - self.assertEqual(set(report.excess_cleanup_calls.values()), {0}, report) - self.assertEqual(set(report.missing_cleanup_calls.values()), {0}, report) - self.assertEqual(report.post_recovery_cleanup_calls, 0, report) - self.assertEqual(report.floor_double_releases, 0, report) - self.assertEqual(report.terminal_rows_adopted, 0, report) - self.assertEqual(report.node_offers, 0, report) - self.assertEqual(report.drain_rounds, 0, report) - self.assertEqual(report.pending_rows_after_drain, 1, report) - self.assertTrue(report.recovered_cleanly, report) - - def test_the_fault_vocabulary_covers_every_cleanup_dependency(self) -> None: - self.assertEqual( - set(CLEANUP_FAULTS), - set(BLOCK_CANDIDATE_COLLAPSE_CLEANUP_STEPS) | {"floor-index"}, - ) - rig = CandidateStormRig(candidates=DECIDED_STORM_CANDIDATES) - rig.seed_live() - with self.assertRaises(ValueError): - CleanupFaultInjector(rig.live_server, "not-a-cleanup-dependency") - with self.assertRaises(ValueError): - rig.measure_cleanup_backlog("not-a-cleanup-dependency") - - def test_every_cleanup_dependency_fault_at_the_observed_storm_recovers( - self, - ) -> None: - for fault in CLEANUP_FAULTS: - with self.subTest(fault=fault): - rig = CandidateStormRig(credit_shares=True) - report = rig.measure_cleanup_backlog( - fault, - view="restart", - sustained_passes=16, - ) - # The shipped bound absorbs the whole storm: every sibling - # is collapsed and every one of them owes cleanup. - self.assertEqual(report.backlog_max, DEFAULT_BLOCK_CANDIDATE_CLEANUP_RETRY_BACKLOG_MAX) - self.assertEqual(report.collapsed_rows, self.SIBLINGS) - self.assertEqual(report.deferred_records, self.SIBLINGS) - self.assertEqual(report.preserved_rows, 0) - self.assertEqual(report.backpressure_engagements, 0) - self.assertFalse(report.backpressure_active) - self.assertEqual(report.terminal_outcome_pins, self.SIBLINGS) - # The restart view holds every row in the replay queue, so - # every record retains its holder -- except under the - # index abort, which never got to read them. - expected_holders = 0 if fault == "floor-index" else self.SIBLINGS - self.assertEqual(report.pending_share_holders, expected_holders) - self.assertGreater(report.registry_bytes_per_record, 0.0) - self.assertGreaterEqual(report.oldest_age_seconds, 0.0) - # Sustained failure sheds nothing. - self.assertEqual(report.sustained_passes, 16) - self.assertEqual(report.sustained_failed_passes, 16) - self.assertEqual(report.depth_after_sustained, self.SIBLINGS) - self.assertEqual(report.holders_after_sustained, expected_holders) - self.assertEqual(report.pins_after_sustained, self.SIBLINGS) - # Recovery, and the floor authority it leaves behind. - self._assert_recovered(report) - self.assertEqual(report.collapsed_rows_after_recovery, 0) - self.assertEqual(report.floor_holders_before, OBSERVED_TESTNET_CANDIDATE_STORM) - self.assertEqual(report.floor_holders_after, 1) - self.assertTrue(report.winner_floor_holder_retained) - # Wall-clock corroboration of the measured drain rate. - self.assertLess(report.recovery_seconds, 60.0) - self.assertGreater(report.retry_records_per_second, 50.0) - - def test_the_bound_engages_at_the_observed_storm_and_admission_resumes( - self, - ) -> None: - bound = MAX_BLOCK_REPLAY_ENUMERATION_ROWS - rig = CandidateStormRig(credit_shares=True, backlog_max=bound) - report = rig.measure_cleanup_backlog( - "retry-state", - view="restart", - sustained_passes=16, - ) - self.assertEqual(report.backlog_max, bound) - # The first page fills the backlog; exactly one following page is - # preserved and adopted before pagination pauses for it to drain. - self.assertEqual(report.collapsed_rows, bound) - self.assertEqual(report.deferred_records, bound) - self.assertEqual( - report.preserved_rows, - MAX_BLOCK_REPLAY_ENUMERATION_ROWS, - ) - self.assertEqual(report.backpressure_engagements, 1) - self.assertTrue(report.backpressure_active) - self.assertEqual(report.pending_share_holders, bound) - self.assertEqual(report.terminal_outcome_pins, bound) - self.assertEqual(report.depth_after_sustained, bound) - self.assertEqual(report.holders_after_sustained, bound) - # Once drained, the next walk collapses everything the bound - # preserved, and the rig's recovery proof still holds over the - # union. - self._assert_recovered(report) - self.assertEqual(report.collapsed_rows_after_recovery, self.SIBLINGS - bound) - self.assertEqual(report.floor_holders_after, 1) - self.assertTrue(report.winner_floor_holder_retained) - - def test_the_live_view_retains_only_the_queued_holders(self) -> None: - """Only a queued wakeup owns a holder the cleanup can retain.""" - rig = CandidateStormRig() - report = rig.measure_cleanup_backlog( - "pending-share-floor", - view="live", - sustained_passes=8, - ) - self.assertEqual(report.collapsed_rows, self.SIBLINGS) - # The coalesced siblings had no in-memory object, so their cleanup - # had no floor step to fail: only the bounded queue's wakeups owe. - self.assertEqual(report.deferred_records, MAX_PENDING_BLOCK_CANDIDATES) - self.assertEqual(report.pending_share_holders, MAX_PENDING_BLOCK_CANDIDATES) - self.assertEqual(report.terminal_outcome_pins, MAX_PENDING_BLOCK_CANDIDATES) - self._assert_recovered(report) - - def test_a_clean_walk_leaves_no_backlog_and_the_marginal_cost_is_measurable( - self, - ) -> None: - clean = CandidateStormRig(candidates=DECIDED_STORM_CANDIDATES).measure_cleanup_backlog( - CLEANUP_FAULT_NONE, - view="restart", - ) - self.assertEqual(clean.deferred_records, 0) - self.assertEqual(clean.collapsed_rows, DECIDED_STORM_CANDIDATES - 1) - self.assertEqual(clean.registry_bytes_per_record, 0.0) - self.assertEqual(clean.pending_share_holders, 0) - self.assertTrue(clean.recovered_cleanly) - faulted = CandidateStormRig(candidates=DECIDED_STORM_CANDIDATES).measure_cleanup_backlog( - "retry-state", - view="restart", - ) - self.assertEqual(faulted.deferred_records, DECIDED_STORM_CANDIDATES - 1) - # The process-wide live-byte delta also includes allocator and cache - # churn from the clean walk, so it is diagnostic rather than a - # monotonic assertion. The owner-filtered delta and the independent - # deep registry walk isolate the retained retry records. - self.assertGreater( - cleanup_backlog_marginal_bytes_per_record(faulted, clean, owner_only=True), - 0.0, - ) - self.assertGreater(faulted.registry_bytes_per_record, 0.0) - - -if __name__ == "__main__": - unittest.main() diff --git a/tests/test_prism_candidate_window.py b/tests/test_prism_candidate_window.py deleted file mode 100644 index 74aa9b4c3..000000000 --- a/tests/test_prism_candidate_window.py +++ /dev/null @@ -1,411 +0,0 @@ -from __future__ import annotations - -import gc -import itertools -import os -from pathlib import Path -import shlex -import shutil -import threading -import time -from types import SimpleNamespace -import unittest -from unittest import mock -import weakref - -from lab.prism.candidate_window import ( - COPY_TEXT_MAX_BYTES, - COPY_TEXT_MAX_RECORDS, - copy_text_chunks, - disk_window_covers, - recorded_share_ids, -) - - -class CandidateWindowTests(unittest.TestCase): - def test_membership_checks_omissions_not_equality_or_order(self) -> None: - self.assertTrue(disk_window_covers(iter(["b", "a", "a", "extra"]), [ - {"share_id": "a"}, {"share_id": "b"}, - ])) - self.assertFalse(disk_window_covers(iter(["a", "extra"]), [ - {"share_id": "a"}, {"share_id": "late"}, - ])) - self.assertTrue(disk_window_covers(iter([]), [])) - self.assertFalse(disk_window_covers(iter([]), [{"share_id": "a"}])) - - def test_legacy_string_and_non_dictionary_semantics(self) -> None: - recorded = [None, {"share_id": 42}, {}, {"share_id": "é"}] - self.assertTrue(disk_window_covers(recorded_share_ids(recorded), [ - "ignored", {"share_id": "42"}, {"share_id": None}, {"share_id": "é"}, - ])) - self.assertFalse(disk_window_covers(recorded_share_ids(recorded), [ - {"share_id": "e\u0301"}, - ])) - - def test_copy_escapes_commands_nulls_and_delimiters(self) -> None: - chunks = list(copy_text_chunks(["\\.\nDROP TABLE x;", "\\N", "a\tb\rc", ""])) - self.assertEqual( - b"".join(chunks), b"\\\\.\\nDROP TABLE x;\n\\\\N\na\\tb\\rc\n\n", - ) - - def test_oversized_field_obeys_byte_bound_and_checks_deadline(self) -> None: - value = "🌐\n\\\té" * 30000 - checks = 0 - - def check() -> None: - nonlocal checks - checks += 1 - - count = 0 - for chunk in copy_text_chunks([value], check=check): - self.assertLessEqual(len(chunk), COPY_TEXT_MAX_BYTES) - count += 1 - self.assertEqual(checks, count) - self.assertGreater(checks, 30) - - def test_lazy_identifier_is_streamed_and_compared_exactly_under_collision(self) -> None: - class Text: - def __init__(self, suffix): - self.suffix = suffix - - def iter_text_chunks(self): - yield "🌐\n\\\té" * 30000 - yield self.suffix - - def __str__(self): - raise AssertionError("lazy identifier was materialized") - - value = Text("a") - chunks = list(copy_text_chunks(recorded_share_ids([{"share_id": value}]))) - self.assertTrue(all(len(chunk) <= COPY_TEXT_MAX_BYTES for chunk in chunks)) - collision = SimpleNamespace(update=lambda data: None, digest=lambda: b"collision") - with mock.patch("lab.prism.candidate_window.hashlib.sha256", return_value=collision): - self.assertTrue(disk_window_covers([value], [{"share_id": Text("a")}])) - self.assertFalse(disk_window_covers([value], [{"share_id": Text("b")}])) - - def test_cancel_during_long_field_stops_consumption(self) -> None: - calls = 0 - - def cancel() -> None: - nonlocal calls - calls += 1 - if calls == 3: - raise InterruptedError("cancelled") - - with self.assertRaises(InterruptedError): - for _chunk in copy_text_chunks(["x" * 50000], check=cancel): - pass - self.assertEqual(calls, 3) - - def test_copy_rejects_nul_after_a_bounded_prefix(self) -> None: - emitted = 0 - with self.assertRaisesRegex(ValueError, "share ID contains a NUL"): - for chunk in copy_text_chunks(["x" * (3 * COPY_TEXT_MAX_BYTES) + "\x00"]): - self.assertLessEqual(len(chunk), COPY_TEXT_MAX_BYTES) - emitted += 1 - self.assertGreater(emitted, 0) - - def test_tiny_records_still_have_a_work_bound(self) -> None: - consumed = 0 - at_check = 0 - - def values(): - nonlocal consumed - for _ in range(1300): - consumed += 1 - yield "" - - def check(): - nonlocal at_check - self.assertLessEqual(consumed - at_check, COPY_TEXT_MAX_RECORDS) - at_check = consumed - - for chunk in copy_text_chunks(values(), check=check): - self.assertLessEqual(len(chunk), COPY_TEXT_MAX_BYTES) - self.assertEqual(at_check, 1300) - - def test_record_graphs_release_without_cyclic_gc(self) -> None: - class Row(dict): - pass - - references = [] - - def rows(): - for index in range(2000): - row = Row(share_id=f"share-{index}", payload=bytearray(10000)) - references.append(weakref.ref(row)) - yield row - - enabled = gc.isenabled() - gc.disable() - try: - self.assertTrue(disk_window_covers(recorded_share_ids(rows()), [ - {"share_id": "share-1999"}, - ])) - self.assertTrue(all(reference() is None for reference in references)) - finally: - if enabled: - gc.enable() - - -class PayoutMembershipPortTests(unittest.TestCase): - def setUp(self) -> None: - from lab.prism.payout_state import PayoutStateService - - self.covers = mock.Mock(return_value=True) - self.aggregate = mock.Mock(side_effect=AssertionError("aggregate window read")) - self.runtime = SimpleNamespace( - ledger=SimpleNamespace( - candidate_window_covers=self.covers, audit_share_window=self.aggregate, - ), - _job_cache_lock=threading.RLock(), - _record_payout_artifact_event=mock.Mock(), - _incremental_payout_artifact_window=object(), - ) - self.service = PayoutStateService( - self.runtime, shutdown_error=RuntimeError, now_ms=lambda: 0, - ) - # This object cannot be materialized or iterated by the payout caller. - self.shares = object() - self.context = SimpleNamespace( - found_block={"anchor_job_issued_at_ms": "2000", "network_difficulty": "1000000"}, - shares_json=self.shares, - ) - - def test_bounded_port_receives_original_view_and_returns_omission_verdict(self) -> None: - for verdict in (True, False): - with self.subTest(verdict=verdict): - self.covers.return_value = verdict - self.assertIs(self.service._replayed_payout_window_reproducible(self.context), verdict) - self.covers.assert_called_with( - self.shares, anchor_job_issued_at_ms=2000, network_difficulty=1000000, - ) - self.aggregate.assert_not_called() - - def test_missing_anchor_fails_closed_before_any_read(self) -> None: - self.context.found_block.pop("anchor_job_issued_at_ms") - self.assertFalse(self.service._replayed_payout_window_reproducible(self.context)) - self.covers.assert_not_called() - self.aggregate.assert_not_called() - - def test_bounded_port_divergence_invalidates_mirror_and_fails_closed(self) -> None: - from lab.prism.share_ledger import DaemonWindowMirrorDivergence - - self.covers.side_effect = DaemonWindowMirrorDivergence("refuted mirror") - self.assertFalse(self.service._replayed_payout_window_reproducible(self.context)) - self.runtime._record_payout_artifact_event.assert_called_once_with("window_mirror_divergence") - self.assertIsNone(self.runtime._incremental_payout_artifact_window) - self.aggregate.assert_not_called() - - def test_bounded_port_timeout_propagates_for_retry_without_aggregate_fallback(self) -> None: - from lab.prism.share_ledger import LedgerOperationTimeout - - error = LedgerOperationTimeout("expired") - self.covers.side_effect = error - with self.assertRaises(LedgerOperationTimeout) as raised: - self.service._replayed_payout_window_reproducible(self.context) - self.assertIs(raised.exception, error) - self.aggregate.assert_not_called() - - -@unittest.skipUnless(os.environ.get("PRISM_TEST_DATABASE_URL"), "requires disposable PostgreSQL") -class CandidateWindowPostgresTests(unittest.TestCase): - """Run with an initialized disposable database, never a production DSN.""" - - def setUp(self) -> None: - import psycopg - from lab.prism.share_ledger import PsqlShareLedger - - self.connection = psycopg.connect(os.environ["PRISM_TEST_DATABASE_URL"], autocommit=True) - self.addCleanup(self.connection.close) - self.connection.execute("TRUNCATE qbit_share_ledger CASCADE") - self.ledger = PsqlShareLedger( - database_url=os.environ["PRISM_TEST_DATABASE_URL"], psql_command="psql", - native_client_mode="native", writer_id="candidate-window-tests", - ) - self.addCleanup(self.ledger.close) - self.addCleanup(self.ledger.release_writer_lease) - - def insert(self, share_id: str, *, accepted_at_ms: int = 1000) -> None: - self.connection.execute(""" - INSERT INTO qbit_share_ledger ( - share_id, miner_id, payout_order_key, p2mr_program, - share_difficulty, network_difficulty, template_height, job_id, - job_issued_at, ntime, accepted_at, credit_policy, accepted, - writer_id, writer_epoch - ) VALUES (%s, 'miner', 'miner', decode(repeat('aa', 32), 'hex'), - 1, 1000000, 1, 'job', to_timestamp(0), 0, - to_timestamp(%s::double precision / 1000.0), NULL, true, 'test', 1) - """, (share_id, accepted_at_ms)) - - def covers(self, rows) -> bool: - return self.ledger.candidate_window_covers( - rows, anchor_job_issued_at_ms=2000, network_difficulty=1000000, - ) - - def test_native_exact_subset_and_late_visible_append(self) -> None: - self.insert("a") - self.assertTrue(self.covers([{"share_id": "a"}, {"share_id": "extra"}])) - self.insert("late") - self.assertFalse(self.covers([{"share_id": "a"}])) - self.assertTrue(self.covers([{"share_id": "late"}, {"share_id": "a"}])) - - def test_native_copy_special_values_and_no_temporary_table_leak(self) -> None: - for share_id in ("\\.\nSELECT 1;", "\\N", "🌐\té\r\n", "extra"): - self.insert(share_id) - self.assertTrue(self.covers([ - {"share_id": value} - for value in ("extra", "\\.\nSELECT 1;", "🌐\té\r\n", "\\N") - ])) - with self.ledger._native.connection() as connection: - self.assertIsNone(connection.execute( - "SELECT to_regclass('pg_temp.qbit_candidate_recorded_ids')" - ).fetchone()[0]) - - def test_native_failure_rolls_back_private_copy_and_can_retry(self) -> None: - self.insert("a") - - def failing_rows(): - yield {"share_id": "a"} - raise InterruptedError("stop") - - with self.assertRaises(InterruptedError): - self.covers(failing_rows()) - self.assertTrue(self.covers([{"share_id": "a"}])) - - def test_native_expired_deadline_does_not_publish_partial_answer(self) -> None: - from lab.prism.share_ledger import LedgerOperationTimeout - - self.insert("a") - with self.ledger.operation_timeout(1): - self.ledger._operation_timeout_local.deadline = self.ledger._monotonic() - 1 - with self.assertRaises(LedgerOperationTimeout): - self.covers([{"share_id": "a"}]) - self.assertTrue(self.covers([{"share_id": "a"}])) - - def test_native_nul_failure_rolls_back_and_can_retry(self) -> None: - self.insert("a") - with self.assertRaisesRegex(ValueError, "share ID contains a NUL"): - self.covers([ - {"share_id": "a"}, - {"share_id": "x" * (3 * COPY_TEXT_MAX_BYTES) + "\x00"}, - ]) - self.assertTrue(self.covers([{"share_id": "a"}])) - - def assert_streaming_timeout_and_retry(self, share_id: str) -> None: - from lab.prism.share_ledger import LedgerOperationTimeout - - with self.ledger.operation_timeout(0.1): - with self.assertRaises(LedgerOperationTimeout): - self.covers(itertools.repeat({"share_id": share_id})) - self.assertEqual(self.connection.execute(""" - SELECT count(*) FROM pg_class c JOIN pg_namespace n ON c.relnamespace = n.oid - WHERE n.nspname LIKE 'pg_temp_%' AND c.relname = 'qbit_candidate_recorded_ids' - """).fetchone()[0], 0) - self.assertEqual(self.connection.execute(""" - SELECT count(*) FROM pg_stat_activity - WHERE datname = current_database() AND state = 'idle in transaction' - """).fetchone()[0], 0) - self.assertTrue(self.covers([{"share_id": share_id}])) - - def test_native_deadline_expires_while_streaming(self) -> None: - self.insert("a") - self.assert_streaming_timeout_and_retry("a") - - def test_native_slow_consumer_applies_backpressure_and_rolls_back(self) -> None: - from lab.prism import candidate_window - from lab.prism.share_ledger import LedgerOperationTimeout - - self.insert("a") - create = candidate_window._CREATE + """ -CREATE FUNCTION pg_temp.slow_candidate_copy() RETURNS trigger LANGUAGE plpgsql AS $$ -BEGIN PERFORM pg_sleep(0.02); RETURN NEW; END $$; -CREATE TRIGGER slow_candidate_copy BEFORE INSERT ON pg_temp.qbit_candidate_recorded_ids -FOR EACH ROW EXECUTE FUNCTION pg_temp.slow_candidate_copy(); -""" - factory = candidate_window._flushing_copy_writer - longest_write = 0.0 - maximum_rss = 0 - - def rss() -> int: - status = Path("/proc/self/status") - if not status.exists(): - return 0 - for line in status.read_text().splitlines(): - if line.startswith("VmRSS:"): - return int(line.split()[1]) * 1024 - return 0 - - initial_rss = rss() - - def instrument(cursor): - writer = factory(cursor) - write = writer.write - - def measured(data): - nonlocal longest_write, maximum_rss - started = time.monotonic() - try: - return write(data) - finally: - longest_write = max(longest_write, time.monotonic() - started) - maximum_rss = max(maximum_rss, rss()) - - writer.write = measured - return writer - - # Reuse one record so the only potential window-sized allocation is - # inside the transport. The stock Linux writer buffers this 64 MiB - # stream without blocking when the trigger stalls the backend. - rows = itertools.repeat({"share_id": "x" * 4095}, 16384) - with mock.patch.object(candidate_window, "_CREATE", create), mock.patch.object( - candidate_window, "_flushing_copy_writer", instrument, - ), self.ledger.operation_timeout(1.5): - with self.assertRaises(LedgerOperationTimeout): - self.covers(rows) - self.assertGreater(longest_write, 0.1) - if initial_rss: - self.assertLess(maximum_rss - initial_rss, 32 * 1024 * 1024) - self.assertEqual(self.connection.execute(""" - SELECT count(*) FROM pg_class c JOIN pg_namespace n ON c.relnamespace = n.oid - WHERE n.nspname LIKE 'pg_temp_%' AND c.relname = 'qbit_candidate_recorded_ids' - """).fetchone()[0], 0) - self.assertEqual(self.connection.execute(""" - SELECT count(*) FROM pg_stat_activity - WHERE datname = current_database() AND state = 'idle in transaction' - """).fetchone()[0], 0) - self.assertTrue(self.covers([{"share_id": "a"}])) - - @unittest.skipUnless( - shutil.which("psql") or os.environ.get("PRISM_TEST_PSQL_COMMAND_WITH_ENV"), - "requires PostgreSQL client", - ) - def test_psql_stream_roundtrip_and_failure(self) -> None: - # Use the existing instance's guards/gates and an independent psql - # connection. The writer lease is not read or updated by this call. - native = self.ledger._native - old_command = self.ledger._command - self.ledger._native = None - self.ledger._command = shlex.split(os.environ.get( - "PRISM_TEST_PSQL_COMMAND_WITH_ENV", - "psql " + shlex.quote(os.environ["PRISM_TEST_DATABASE_URL"]), - )) - try: - value = "\\.\nSELECT 1;🌐\t\\N" - self.insert(value) - self.assertTrue(self.covers([{"share_id": value}, {"share_id": "extra"}])) - self.assertFalse(self.covers([{"share_id": "extra"}])) - with self.assertRaisesRegex(ValueError, "share ID contains a NUL"): - self.covers([{"share_id": value + "\x00"}]) - self.assertTrue(self.covers([{"share_id": value}])) - self.assert_streaming_timeout_and_retry(value) - self.assertEqual(self.connection.execute( - "SELECT count(*) FROM qbit_share_ledger" - ).fetchone()[0], 1) - finally: - self.ledger._native = native - self.ledger._command = old_command - - -if __name__ == "__main__": - unittest.main() diff --git a/tests/test_prism_capacity_evidence.py b/tests/test_prism_capacity_evidence.py deleted file mode 100644 index 5a9f16728..000000000 --- a/tests/test_prism_capacity_evidence.py +++ /dev/null @@ -1,335 +0,0 @@ -#!/usr/bin/env python3 - -from __future__ import annotations - -import json -import subprocess -import sys -import tempfile -import unittest -from datetime import datetime, timezone -from decimal import Decimal -from pathlib import Path - -from lab.prism import capacity_evidence - - -ROOT = Path(__file__).resolve().parents[1] -FIXTURE = ROOT / "tests" / "fixtures" / "prism-capacity-evidence.json" -NOW = datetime(2026, 7, 13, 12, 30, tzinfo=timezone.utc) - - -class PrismCapacityEvidenceTests(unittest.TestCase): - def payload(self, *, qualification: bool = True) -> dict[str, object]: - payload = json.loads(FIXTURE.read_text(encoding="utf-8")) - if qualification: - payload["artifact_kind"] = "qualification" - payload["run_id"] = "91d514da-2c6f-4a8e-8964-b5f64c46ba18" - return payload - - def expected_configuration(self) -> dict[str, str]: - return { - "PRISM_STRATUM_SHARE_DIFF": "1024.0", - "PRISM_STRATUM_VARDIFF": "1", - "PRISM_STRATUM_VARDIFF_TARGET_SECONDS": "15", - "PRISM_STRATUM_VARDIFF_MIN_DIFF": "1024", - "PRISM_STRATUM_VARDIFF_START_DIFF": "4096", - "PRISM_STRATUM_VARDIFF_MAX_DIFF": "65536", - "PRISM_STRATUM_VARDIFF_RETARGET_SECONDS": "90", - "PRISM_STRATUM_VARDIFF_MAX_STEP_UP": "4", - "PRISM_STRATUM_VARDIFF_MAX_STEP_DOWN": "4", - "PRISM_STRATUM_VARDIFF_EWMA_ALPHA": "0.4", - "PRISM_STRATUM_VARDIFF_RETARGET_TOLERANCE": "0.25", - "PRISM_STRATUM_VARDIFF_IDLE_SWEEP_SECONDS": "15", - "PRISM_SHARE_COMMIT_BATCH_SIZE": "64", - "PRISM_SHARE_COMMIT_LINGER_MILLISECONDS": "5", - "PRISM_SHARE_COMMIT_TIMEOUT_SECONDS": "15", - "PRISM_STRATUM_SEND_TIMEOUT_SECONDS": "20", - } - - def expected_subject(self) -> dict[str, str]: - return { - "coordinator_revision": "a" * 40, - "coordinator_image_digest": "sha256:" + "b" * 64, - "postgres_server_version": "16.4", - "database_profile_sha256": "c" * 64, - } - - def validate( - self, - payload: dict[str, object] | None = None, - **overrides: object, - ) -> capacity_evidence.CapacityEvidenceSummary: - options: dict[str, object] = { - "expected_configuration": self.expected_configuration(), - "expected_subject": self.expected_subject(), - "expected_forecast_peak_shares_per_second": "100", - "expected_ack_p99_limit_milliseconds": "50", - "current_time": NOW, - } - options.update(overrides) - return capacity_evidence.validate_capacity_evidence( - payload if payload is not None else self.payload(), - **options, - ) - - def test_qualification_proves_two_times_capacity_and_identifier_reconciliation(self) -> None: - summary = self.validate() - - self.assertEqual(summary.measured_shares_per_second, Decimal("200")) - self.assertEqual(summary.capacity_multiple, Decimal("2")) - self.assertEqual(summary.acknowledged_shares, 120000) - self.assertEqual(summary.ack_p50_milliseconds, Decimal("6")) - self.assertEqual(summary.ack_p99_milliseconds, Decimal("40")) - - def test_committed_fixture_is_example_only(self) -> None: - payload = self.payload(qualification=False) - - with self.assertRaisesRegex(capacity_evidence.EvidenceError, "example capacity evidence"): - self.validate(payload) - - summary = self.validate(payload, allow_example=True) - self.assertEqual(summary.capacity_multiple, Decimal("2")) - - def test_qualification_requires_external_policy_and_subject(self) -> None: - missing_options = ( - ("expected_configuration", "expected deployment configuration"), - ("expected_subject", "expected deployment subject"), - ("expected_forecast_peak_shares_per_second", "externally configured forecast"), - ("expected_ack_p99_limit_milliseconds", "externally configured ACK p99"), - ) - for option, message in missing_options: - with self.subTest(option=option): - with self.assertRaisesRegex(capacity_evidence.EvidenceError, message): - self.validate(**{option: None}) - - def test_rejects_stale_and_future_qualification_artifacts(self) -> None: - stale = self.payload() - stale["generated_at"] = "2026-07-10T12:00:00Z" - with self.assertRaisesRegex(capacity_evidence.EvidenceError, "older than"): - self.validate(stale) - - future = self.payload() - future["generated_at"] = "2026-07-14T12:00:00Z" - with self.assertRaisesRegex(capacity_evidence.EvidenceError, "too far in the future"): - self.validate(future) - - def test_runtime_restart_can_revalidate_binding_without_time_bomb(self) -> None: - stale = self.payload() - stale["generated_at"] = "2026-01-01T00:00:00Z" - - summary = self.validate(stale, enforce_freshness=False) - - self.assertEqual(summary.capacity_multiple, Decimal("2")) - - def test_rejects_subject_for_different_runtime_or_database(self) -> None: - for key in self.expected_subject(): - with self.subTest(key=key): - expected = self.expected_subject() - expected[key] = "different" - with self.assertRaisesRegex(capacity_evidence.EvidenceError, f"subject.{key}"): - self.validate(expected_subject=expected) - - def test_rejects_weakened_postgres_durability(self) -> None: - for setting in ("fsync", "full_page_writes", "synchronous_commit"): - with self.subTest(setting=setting): - payload = self.payload() - durability = payload["durability"] - assert isinstance(durability, dict) - durability[setting] = "off" - with self.assertRaisesRegex(capacity_evidence.EvidenceError, f"durability.{setting}"): - self.validate(payload) - - def test_binds_every_load_affecting_configuration_value(self) -> None: - alternatives = { - "PRISM_STRATUM_SHARE_DIFF": "2048", - "PRISM_STRATUM_VARDIFF": "0", - "PRISM_STRATUM_VARDIFF_TARGET_SECONDS": "20", - "PRISM_STRATUM_VARDIFF_MIN_DIFF": "2048", - "PRISM_STRATUM_VARDIFF_START_DIFF": "8192", - "PRISM_STRATUM_VARDIFF_MAX_DIFF": "131072", - "PRISM_STRATUM_VARDIFF_RETARGET_SECONDS": "120", - "PRISM_STRATUM_VARDIFF_MAX_STEP_UP": "2", - "PRISM_STRATUM_VARDIFF_MAX_STEP_DOWN": "2", - "PRISM_STRATUM_VARDIFF_EWMA_ALPHA": "0.5", - "PRISM_STRATUM_VARDIFF_RETARGET_TOLERANCE": "0.3", - "PRISM_STRATUM_VARDIFF_IDLE_SWEEP_SECONDS": "20", - "PRISM_SHARE_COMMIT_BATCH_SIZE": "32", - "PRISM_SHARE_COMMIT_LINGER_MILLISECONDS": "7", - "PRISM_SHARE_COMMIT_TIMEOUT_SECONDS": "20", - "PRISM_STRATUM_SEND_TIMEOUT_SECONDS": "25", - } - self.assertEqual(set(alternatives), set(capacity_evidence.CONFIGURATION_KEYS)) - for key, alternative in alternatives.items(): - with self.subTest(key=key): - expected = self.expected_configuration() - expected[key] = alternative - with self.assertRaisesRegex(capacity_evidence.EvidenceError, f"configuration.{key}"): - self.validate(expected_configuration=expected) - - def test_rejects_vardiff_start_outside_qualified_range(self) -> None: - payload = self.payload() - configuration = payload["configuration"] - assert isinstance(configuration, dict) - configuration["PRISM_STRATUM_VARDIFF_MAX_DIFF"] = "2048" - - with self.assertRaisesRegex(capacity_evidence.EvidenceError, "minimum <= start <= maximum"): - self.validate(payload) - - def test_rejects_invalid_vardiff_math_parameters(self) -> None: - for key, value, message in ( - ("PRISM_STRATUM_VARDIFF_MAX_STEP_UP", "0.5", "must be at least 1"), - ("PRISM_STRATUM_VARDIFF_MAX_STEP_DOWN", "0.5", "must be at least 1"), - ("PRISM_STRATUM_VARDIFF_EWMA_ALPHA", "1.1", "must not exceed 1"), - ): - with self.subTest(key=key): - payload = self.payload() - configuration = payload["configuration"] - assert isinstance(configuration, dict) - configuration[key] = value - with self.assertRaisesRegex(capacity_evidence.EvidenceError, message): - self.validate(payload) - - def test_rejects_lab_difficulty_profile(self) -> None: - payload = self.payload() - configuration = payload["configuration"] - assert isinstance(configuration, dict) - configuration["PRISM_STRATUM_SHARE_DIFF"] = "1e-9" - - with self.assertRaisesRegex(capacity_evidence.EvidenceError, "lab-only 1e-9"): - self.validate(payload) - - def test_rejects_self_declared_forecast_or_latency_limit(self) -> None: - with self.assertRaisesRegex(capacity_evidence.EvidenceError, "forecast_peak.*deployment"): - self.validate(expected_forecast_peak_shares_per_second="101") - with self.assertRaisesRegex(capacity_evidence.EvidenceError, "ack_p99_limit.*deployment"): - self.validate(expected_ack_p99_limit_milliseconds="51") - - def test_rejects_ack_limit_beyond_commit_timeout(self) -> None: - payload = self.payload() - payload["ack_p99_limit_milliseconds"] = 16000 - - with self.assertRaisesRegex(capacity_evidence.EvidenceError, "cannot exceed.*COMMIT_TIMEOUT"): - self.validate(payload, expected_ack_p99_limit_milliseconds="16000") - - def test_rejects_phase_duration_mismatch_and_trivial_phases(self) -> None: - mismatch = self.payload() - phases = mismatch["phases"] - assert isinstance(phases, dict) - steady = phases["steady_state"] - assert isinstance(steady, dict) - steady["duration_seconds"] = 399 - with self.assertRaisesRegex(capacity_evidence.EvidenceError, "phase durations"): - self.validate(mismatch) - - trivial = self.payload() - phases = trivial["phases"] - assert isinstance(phases, dict) - reconnect = phases["reconnect"] - assert isinstance(reconnect, dict) - reconnect["duration_seconds"] = 59 - with self.assertRaisesRegex(capacity_evidence.EvidenceError, "at least 60"): - self.validate(trivial) - - def test_requires_two_times_capacity_in_every_fault_phase(self) -> None: - payload = self.payload() - phases = payload["phases"] - assert isinstance(phases, dict) - steady = phases["steady_state"] - reconnect = phases["reconnect"] - assert isinstance(steady, dict) and isinstance(reconnect, dict) - for field in ( - "offered_valid_shares", - "acknowledged_shares", - "postgres_unique_committed_shares", - ): - steady[field] = 90000 - reconnect[field] = 10000 - - with self.assertRaisesRegex(capacity_evidence.EvidenceError, "phases.reconnect sustained rate"): - self.validate(payload) - - def test_rejects_token_fault_phases(self) -> None: - reconnect_payload = self.payload() - phases = reconnect_payload["phases"] - assert isinstance(phases, dict) - reconnect = phases["reconnect"] - assert isinstance(reconnect, dict) - reconnect["reconnect_events"] = 1 - with self.assertRaisesRegex(capacity_evidence.EvidenceError, "reconnect_events must be at least 10"): - self.validate(reconnect_payload) - - database_payload = self.payload() - phases = database_payload["phases"] - assert isinstance(phases, dict) - slow_database = phases["slow_database"] - assert isinstance(slow_database, dict) - slow_database["database_delay_milliseconds"] = 1 - with self.assertRaisesRegex( - capacity_evidence.EvidenceError, - "database_delay_milliseconds must be at least 10", - ): - self.validate(database_payload) - - def test_rejects_valid_share_loss_and_identifier_mismatch(self) -> None: - missing = self.payload() - missing["missing_acknowledged_share_ids"] = 1 - with self.assertRaisesRegex(capacity_evidence.EvidenceError, "ACK-to-Postgres reconciliation"): - self.validate(missing) - - digest_mismatch = self.payload() - digest_mismatch["postgres_share_ids_sha256"] = "f" * 64 - with self.assertRaisesRegex(capacity_evidence.EvidenceError, "identifier digests differ"): - self.validate(digest_mismatch) - - rejected = self.payload() - rejected["rejected_valid_shares"] = 1 - with self.assertRaisesRegex(capacity_evidence.EvidenceError, "every offered valid share"): - self.validate(rejected) - - def test_rejects_phase_latency_over_external_limit(self) -> None: - payload = self.payload() - phases = payload["phases"] - assert isinstance(phases, dict) - slow_database = phases["slow_database"] - assert isinstance(slow_database, dict) - latency = slow_database["ack_latency_milliseconds"] - assert isinstance(latency, dict) - latency["p99"] = 51 - - with self.assertRaisesRegex(capacity_evidence.EvidenceError, "required 50ms"): - self.validate(payload) - - def test_rejects_unknown_schema_fields(self) -> None: - payload = self.payload() - payload["unchecked_claim"] = True - - with self.assertRaisesRegex(capacity_evidence.EvidenceError, "unknown fields"): - self.validate(payload) - - def test_loader_rejects_duplicate_json_keys(self) -> None: - with tempfile.TemporaryDirectory() as directory: - path = Path(directory) / "duplicate.json" - path.write_text('{"schema":"first","schema":"second"}', encoding="utf-8") - - with self.assertRaisesRegex(capacity_evidence.EvidenceError, "duplicate JSON key"): - capacity_evidence.load_capacity_evidence(path) - - def test_cli_requires_explicit_test_override_for_example_fixture(self) -> None: - command = [sys.executable, str(ROOT / "scripts" / "prism_capacity_evidence.py"), str(FIXTURE)] - rejected = subprocess.run(command, cwd=ROOT, text=True, capture_output=True, check=False) - accepted = subprocess.run( - [*command, "--allow-example-evidence-for-tests"], - cwd=ROOT, - text=True, - capture_output=True, - check=False, - ) - - self.assertNotEqual(rejected.returncode, 0) - self.assertIn("example capacity evidence", rejected.stderr) - self.assertEqual(accepted.returncode, 0, accepted.stderr) - - -if __name__ == "__main__": - unittest.main() diff --git a/tests/test_prism_completion_callbacks.py b/tests/test_prism_completion_callbacks.py deleted file mode 100644 index b5fdf9948..000000000 --- a/tests/test_prism_completion_callbacks.py +++ /dev/null @@ -1,238 +0,0 @@ -#!/usr/bin/env python3 -"""A completed future must not keep obsolete callback owners alive.""" - -from __future__ import annotations - -from concurrent.futures import Future -from contextlib import redirect_stderr, redirect_stdout -import gc -import io -import threading -from types import SimpleNamespace -import unittest -import weakref - -from lab.prism.future_callbacks import add_releasing_done_callback -from lab.prism.job_delivery import JobDeliveryService, PendingInitialJob -from lab.prism.reorg_reconciler import ReorgReconcilerService - - -class _Callback: - def __init__(self, calls: list[str], name: str) -> None: - self.calls = calls - self.name = name - - def __call__(self, future: Future) -> None: - assert future.done() - self.calls.append(self.name) - - -class CompletionCallbackTests(unittest.TestCase): - def test_pending_callbacks_stay_alive_and_complete_in_registration_order(self) -> None: - future = Future() - calls: list[str] = [] - references = [] - for name in ("first", "second"): - callback = _Callback(calls, name) - references.append(weakref.ref(callback)) - add_releasing_done_callback(future, callback) - del callback - self.assertTrue(all(ref() is not None for ref in references)) - future.set_result(17) - self.assertEqual(calls, ["first", "second"]) - self.assertTrue(all(ref() is None for ref in references)) - # The caller can keep the result future without keeping its callbacks. - self.assertEqual(future.result(), 17) - - def test_finished_future_invokes_and_releases_callback_inline(self) -> None: - future = Future() - future.set_result(17) - calls: list[str] = [] - callback = _Callback(calls, "inline") - reference = weakref.ref(callback) - add_releasing_done_callback(future, callback) - del callback - self.assertEqual(calls, ["inline"]) - self.assertIsNone(reference()) - - -class InitialDeliveryCallbackRetentionTests(unittest.TestCase): - def test_failure_handler_preserves_error_without_capturing_request(self) -> None: - gc.collect() - was_enabled = gc.isenabled() - gc.disable() - try: - references = self.finish_failed_request() - self.assertTrue(all(ref() is None for ref in references)) - finally: - if was_enabled: - gc.enable() - gc.collect() - - def finish_failed_request(self): - class Client: - connection_id = 17 - - client = Client() - request = PendingInitialJob( - client=client, - authorization_generation=1, - worker=None, - requested_monotonic=0, - deadline_monotonic=None, - ) - future = Future() - error = RuntimeError("initial build failed") - future.set_exception(error) - request.future = future - pending = {client: request} - disconnected = [] - runtime = SimpleNamespace( - lock=threading.Lock(), - job_build_failure_count=0, - disconnect_client=lambda peer: disconnected.append(peer.connection_id), - ) - owner = SimpleNamespace( - _runtime=runtime, - _initial_job_admission_lock=threading.Lock(), - _pending_initial_jobs=pending, - initial_job_failed_count=0, - ) - output, errors = io.StringIO(), io.StringIO() - with redirect_stdout(output), redirect_stderr(errors): - JobDeliveryService._initial_job_future_finished(owner, request, future) - self.assertEqual(runtime.job_build_failure_count, 1) - self.assertEqual(owner.initial_job_failed_count, 1) - self.assertEqual(disconnected, [17]) - self.assertFalse(pending) - self.assertIn("connection=17", output.getvalue()) - self.assertIn("RuntimeError: initial build failed", errors.getvalue()) - self.assertIs(future.exception(), error) - self.assertIsNone(error.__traceback__) - return [weakref.ref(obj) for obj in (client, request, future)] - - def check_releases_request(self, *, cancel: bool, already_done: bool) -> None: - gc.collect() - was_enabled = gc.isenabled() - gc.disable() - try: - references = self.complete_request(cancel=cancel, already_done=already_done) - self.assertEqual( - {name: ref() is not None for name, ref in references.items()}, - dict.fromkeys(references, False), - ) - finally: - if was_enabled: - gc.enable() - gc.collect() - - def complete_request(self, *, cancel: bool, already_done: bool): - class Client: - pass - - client = Client() - request = PendingInitialJob( - client=client, - authorization_generation=1, - worker=None, - requested_monotonic=0, - deadline_monotonic=None, - ) - future = Future() - pending = {client: request} - calls = [] - - def finish(completed_request, completed_future): - self.assertIs(completed_request, pending.pop(completed_request.client)) - self.assertIs(completed_request.future, completed_future) - calls.append(True) - - runtime = SimpleNamespace( - _submit_delivery_task=lambda *args, **kwargs: future, - initial_job_executor=lambda: None, - _run_initial_job=lambda *args: None, - _initial_job_future_finished=finish, - ) - owner = SimpleNamespace( - _runtime=runtime, - _initial_job_admission_lock=threading.Lock(), - _pending_initial_jobs=pending, - ) - references = { - "client": weakref.ref(client), - "request": weakref.ref(request), - "future": weakref.ref(future), - } - if already_done: - future.set_result(True) - self.assertTrue(JobDeliveryService._submit_initial_job_request(owner, request)) - if cancel: - self.assertTrue(future.cancel()) - elif not already_done: - future.set_result(True) - self.assertEqual(calls, [True]) - self.assertFalse(pending) - return references - - def test_completed_request_releases_disconnected_client(self) -> None: - self.check_releases_request(cancel=False, already_done=False) - - def test_cancelled_request_releases_disconnected_client(self) -> None: - self.check_releases_request(cancel=True, already_done=False) - - def test_already_completed_request_releases_disconnected_client(self) -> None: - self.check_releases_request(cancel=False, already_done=True) - - -class StalePrefetchCallbackRetentionTests(unittest.TestCase): - def check_releases_prefetch(self, *, already_done: bool, outcome: str) -> None: - gc.collect() - was_enabled = gc.isenabled() - gc.disable() - try: - reference = self.discard_prefetch(already_done=already_done, outcome=outcome) - self.assertIsNone(reference()) - finally: - if was_enabled: - gc.enable() - gc.collect() - - def discard_prefetch(self, *, already_done: bool, outcome: str): - future = Future() - error = KeyboardInterrupt("prefetch failed") - - def complete(): - if outcome == "failed": - future.set_exception(error) - elif outcome == "cancelled": - future.cancel() - else: - future.set_result(True) - - if already_done: - complete() - ReorgReconcilerService.discard_stale_prefetch(future) - if not already_done: - complete() - if outcome == "failed": - self.assertIs(future.exception(), error) - # Observing an obsolete error must not add a new traceback whose - # callback frame points back to the discarded future. - self.assertIsNone(error.__traceback__) - return weakref.ref(future) - - def test_failed_pending_prefetch_does_not_capture_consumer_frame(self) -> None: - self.check_releases_prefetch(already_done=False, outcome="failed") - - def test_failed_finished_prefetch_does_not_capture_consumer_frame(self) -> None: - self.check_releases_prefetch(already_done=True, outcome="failed") - - def test_cancelled_prefetch_is_discarded(self) -> None: - self.check_releases_prefetch(already_done=False, outcome="cancelled") - - def test_successful_prefetch_is_discarded(self) -> None: - self.check_releases_prefetch(already_done=False, outcome="succeeded") - - -if __name__ == "__main__": - unittest.main() diff --git a/tests/test_prism_compose_profile.py b/tests/test_prism_compose_profile.py deleted file mode 100644 index 494301c46..000000000 --- a/tests/test_prism_compose_profile.py +++ /dev/null @@ -1,564 +0,0 @@ -#!/usr/bin/env python3 - -from __future__ import annotations - -import json -import os -import shutil -import subprocess -import unittest -from pathlib import Path -from unittest.mock import patch - -from lab.prism import public_read_service - - -ROOT = Path(__file__).resolve().parents[1] - - -class PrismComposeProfileTests(unittest.TestCase): - @classmethod - def setUpClass(cls) -> None: - docker = shutil.which("docker") - if docker is None: - raise unittest.SkipTest("docker CLI is not installed") - - version = subprocess.run( - [docker, "compose", "version"], - cwd=ROOT, - text=True, - stdout=subprocess.PIPE, - stderr=subprocess.PIPE, - check=False, - ) - if version.returncode != 0: - raise unittest.SkipTest(f"docker compose is unavailable: {version.stderr.strip()}") - - env = os.environ.copy() - env.update( - { - "QBIT_SRC_DIR": str(ROOT), - "QBIT_NODE_EXTRA_ARG": "-listen=1", - "PRISM_STRATUM_PORT": "43340", - "PRISM_STRATUM_PORT_HOST": "127.0.0.1:43340", - "PRISM_STRATUM_HIGHDIFF_PORT": "44334", - "PRISM_PUBLIC_STRATUM_URL": "stratum+tcp://public-pool.example:3335", - "PRISM_PUBLIC_STRATUM_HIGHDIFF_URL": "stratum+tcp://public-pool.example:4334", - "PRISM_PUBLIC_POOL_FEE_BPS": "200", - "PRISM_CTV_SETTLEMENT_ENABLED": "1", - "PRISM_DIRECT_COINBASE_PAYOUT_FLOOR_BITS": "20971520", - "PRISM_MAX_COINBASE_SETTLEMENT_OUTPUTS": "15", - "PRISM_MAX_DIRECT_COINBASE_OUTPUTS": "7", - "PRISM_MAX_CTV_FANOUT_RECIPIENTS_PER_TRANSACTION": "999", - "PRISM_RESERVED_COINBASE_OUTPUTS": "1", - "PRISM_CTV_FANOUT_FEE_MARKET_RATE_BITS_PER_1000_WEIGHT": "25", - "PRISM_CTV_FANOUT_FEE_PREMIUM_BPS": "13000", - "PRISM_COINBASE_OUTPUT_POLICY": "pool-fee-first", - "PRISM_CTV_BROADCASTER_ENABLED": "1", - "PRISM_CTV_BROADCASTER_WALLET": "fanout-broadcaster", - "PRISM_CTV_BROADCASTER_FEE_BITS": "0", - "PRISM_CTV_BROADCASTER_LIMIT": "7", - "PRISM_CTV_BROADCASTER_CHUNK_SIZE": "3", - "PRISM_CTV_BROADCASTER_INTERVAL_SECONDS": "11", - "PRISM_CTV_BROADCAST_ATTEMPT_DETAIL_LIMIT": "9", - "PRISM_CTV_BROADCAST_RETRY_BACKOFF_SECONDS": "17", - "PRISM_BLOCKWAIT_ENABLED": "0", - "PRISM_BLOCKWAIT_TIMEOUT_SECONDS": "13", - "PRISM_BLOCK_REPLAY_PAGE_SIZE": "1", - "PRISM_COORDINATION_BLOCKED_EXIT_SECONDS": "37", - "PRISM_HEALTH_PENDING_REFRESH_MAX_AGE_SECONDS": "23", - "PRISM_HEALTH_TIP_POLL_MAX_AGE_SECONDS": "29", - "PRISM_METRICS_REFRESH_SECONDS": "17", - "PRISM_STRATUM_STALE_GRACE_SECONDS": "4", - "PRISM_STRATUM_SAME_TIP_JOB_RETENTION_SECONDS": "31", - "PRISM_STRATUM_SAME_TIP_JOB_RETENTION_PER_CONNECTION": "65", - "PRISM_TIP_REFRESH_EPOCH_FANOUT": "1", - "PRISM_TIP_REFRESH_MAX_WORKERS": "7", - "PRISM_STRATUM_VARDIFF_IDLE_SWEEP_SECONDS": "19", - "PRISM_WORKER_METRICS_LIMIT": "8", - "PRISM_STRATUM_MAX_CONNECTIONS": "1900", - "PRISM_STRATUM_MAX_CONNECTIONS_PER_USERNAME": "400", - "PRISM_STRATUM_MAX_PENDING_INITIAL_JOBS": "120", - "PRISM_STRATUM_INITIAL_JOB_TIMEOUT_SECONDS": "27", - "PRISM_MINING_HEALTH_STARTUP_GRACE_SECONDS": "29", - "PRISM_STRATUM_ACCEPT_RESOURCE_EXHAUSTION_BACKOFF_SECONDS": "2", - "PRISM_WRITER_QUIESCENCE_TIMEOUT_SECONDS": "9", - "PRISM_PAYOUT_ADDRESS_CACHE_MAX_ENTRIES": "2048", - "PRISM_PAYOUT_ADDRESS_CACHE_TTL_SECONDS": "1800", - "PRISM_COORDINATOR_NOFILE_SOFT": "60000", - "PRISM_COORDINATOR_NOFILE_HARD": "65000", - } - ) - completed = subprocess.run( - [ - docker, - "compose", - "--env-file", - str(ROOT / "config/upstream.env.example"), - "--env-file", - str(ROOT / ".env.example"), - "-f", - str(ROOT / "compose.yaml"), - "--project-name", - "qbit-prism-compose-test", - "--profile", - "prism", - "config", - "--format", - "json", - ], - cwd=ROOT, - env=env, - text=True, - stdout=subprocess.PIPE, - stderr=subprocess.PIPE, - check=False, - ) - if completed.returncode != 0: - raise AssertionError( - "docker compose --profile prism config failed\n" - f"stdout:\n{completed.stdout}\n" - f"stderr:\n{completed.stderr}" - ) - cls.config = json.loads(completed.stdout) - - def test_prism_profile_services_render(self) -> None: - services = self.config["services"] - - self.assertIn("qbitd", services) - self.assertIn("prism-postgres", services) - self.assertIn("prism-coordinator", services) - - def test_qbit_prelaunch_tip_age_is_absent_by_default(self) -> None: - env = self.config["services"]["qbitd"]["environment"] - - self.assertEqual(env["QBIT_PRODUCTION"], "0") - self.assertEqual(env["QBIT_TOOLS_PRODUCTION"], "0") - self.assertEqual(env["QBIT_CHAIN"], "regtest") - self.assertEqual(env["CKPOOL_NON_TEST_READINESS_GATE"], "1") - self.assertEqual(env["QBIT_MAINNET_LAUNCH_READINESS_CHECKS_ENABLED"], "") - self.assertIsNone(env["QBIT_MAINNET_PRELAUNCH_MAX_TIP_AGE_SECONDS"]) - - def test_qbit_extra_argument_remains_one_independent_argv_entry(self) -> None: - command = self.config["services"]["qbitd"]["command"] - - self.assertIn("-listen=1", command) - self.assertEqual(command.count("-listen=1"), 1) - - def test_prism_coordinator_gets_required_environment(self) -> None: - env = self._service_environment("prism-coordinator") - - self.assertEqual(env["QBIT_RPC_HOST"], "qbitd") - self.assertEqual(env["QBIT_PRODUCTION"], "0") - self.assertEqual(env["QBIT_TOOLS_PRODUCTION"], "0") - self.assertEqual(env["PRISM_DATABASE_URL"], "postgresql://qbit:change-this@prism-postgres:5432/qbit") - self.assertEqual(env["PRISM_POSTGRES_INIT_SCHEMA"], "1") - self.assertEqual(env["PRISM_POSTGRES_READ_CONCURRENCY"], "4") - self.assertEqual(env["PRISM_POSTGRES_IDLE_IN_TRANSACTION_TIMEOUT_SECONDS"], "15") - self.assertEqual(env["PRISM_POSTGRES_TCP_KEEPALIVES_IDLE_SECONDS"], "30") - self.assertEqual(env["PRISM_POSTGRES_TCP_KEEPALIVES_INTERVAL_SECONDS"], "10") - self.assertEqual(env["PRISM_POSTGRES_TCP_KEEPALIVES_COUNT"], "3") - self.assertEqual(env["PRISM_LEDGER_LEASE_TTL_SECONDS"], "60") - self.assertEqual(env["PRISM_LEDGER_LEASE_ACQUIRE_LOCK_TIMEOUT_SECONDS"], "5") - self.assertEqual(env["PRISM_LEDGER_LEASE_ACQUIRE_ATTEMPTS"], "5") - self.assertEqual(env["PRISM_WRITER_QUIESCENCE_TIMEOUT_SECONDS"], "9") - self.assertEqual(env["PRISM_WATCHDOG_ENABLED"], "1") - self.assertEqual(env["PRISM_WATCHDOG_TIMEOUT_SECONDS"], "120") - self.assertEqual(env["PRISM_WATCHDOG_INTERVAL_SECONDS"], "15") - self.assertEqual(env["PRISM_BLOCKWAIT_ENABLED"], "0") - self.assertEqual(env["PRISM_BLOCKWAIT_TIMEOUT_SECONDS"], "13") - self.assertEqual(env["PRISM_BLOCK_REPLAY_PAGE_SIZE"], "1") - self.assertEqual(env["PRISM_COORDINATION_BLOCKED_EXIT_SECONDS"], "37") - self.assertEqual(env["PRISM_HEALTH_PENDING_REFRESH_MAX_AGE_SECONDS"], "23") - self.assertEqual(env["PRISM_HEALTH_TIP_POLL_MAX_AGE_SECONDS"], "29") - self.assertEqual(env["PRISM_METRICS_REFRESH_SECONDS"], "17") - self.assertEqual(env["PRISM_STRATUM_STALE_GRACE_SECONDS"], "4") - self.assertEqual(env["PRISM_STRATUM_SAME_TIP_JOB_RETENTION_SECONDS"], "31") - self.assertEqual( - env["PRISM_STRATUM_SAME_TIP_JOB_RETENTION_PER_CONNECTION"], - "65", - ) - self.assertEqual(env["PRISM_TIP_REFRESH_EPOCH_FANOUT"], "1") - self.assertEqual(env["PRISM_TIP_REFRESH_MAX_WORKERS"], "7") - self.assertEqual(env["PRISM_STRATUM_VARDIFF_IDLE_SWEEP_SECONDS"], "19") - self.assertEqual(env["PRISM_WORKER_METRICS_LIMIT"], "8") - self.assertEqual(env["PRISM_STRATUM_MAX_CONNECTIONS"], "1900") - self.assertEqual(env["PRISM_STRATUM_MAX_CONNECTIONS_PER_USERNAME"], "400") - self.assertEqual(env["PRISM_STRATUM_MAX_PENDING_INITIAL_JOBS"], "120") - self.assertEqual(env["PRISM_STRATUM_INITIAL_JOB_TIMEOUT_SECONDS"], "27") - self.assertEqual(env["PRISM_MINING_HEALTH_STARTUP_GRACE_SECONDS"], "29") - self.assertEqual( - env["PRISM_STRATUM_ACCEPT_RESOURCE_EXHAUSTION_BACKOFF_SECONDS"], - "2", - ) - self.assertEqual(env["PRISM_PAYOUT_ADDRESS_CACHE_MAX_ENTRIES"], "2048") - self.assertEqual(env["PRISM_PAYOUT_ADDRESS_CACHE_TTL_SECONDS"], "1800") - self.assertEqual(env["PRISM_STRATUM_BIND"], "0.0.0.0") - self.assertEqual(env["PRISM_STRATUM_PORT"], "43340") - self.assertEqual(env["PRISM_PUBLIC_REWARD_WINDOW_CACHE_SECONDS"], "30") - self.assertEqual(env["PRISM_AUDIT_BIND"], "127.0.0.1") - self.assertEqual(env["PRISM_AUDIT_PORT"], "3341") - self.assertEqual(env["PRISM_AUDIT_SHARE_SEGMENT_SIZE"], "10000") - self.assertEqual(env["PRISM_AUDIT_LIVE_BUNDLE_RETENTION"], "5") - self.assertEqual(env["PRISM_AUDIT_CANDIDATE_RETENTION_SECONDS"], "86400") - self.assertEqual(env["PRISM_STOP_AFTER_BLOCK"], "0") - self.assertEqual(env["PRISM_CTV_SETTLEMENT_ENABLED"], "1") - self.assertEqual(env["PRISM_DIRECT_COINBASE_PAYOUT_FLOOR_BITS"], "20971520") - self.assertEqual(env["PRISM_MAX_COINBASE_SETTLEMENT_OUTPUTS"], "15") - self.assertEqual(env["PRISM_MAX_DIRECT_COINBASE_OUTPUTS"], "7") - self.assertEqual(env["PRISM_MAX_CTV_FANOUT_RECIPIENTS_PER_TRANSACTION"], "999") - self.assertEqual(env["PRISM_RESERVED_COINBASE_OUTPUTS"], "1") - self.assertEqual(env["PRISM_CTV_FANOUT_FEE_MARKET_RATE_BITS_PER_1000_WEIGHT"], "25") - self.assertEqual(env["PRISM_CTV_FANOUT_FEE_PREMIUM_BPS"], "13000") - self.assertEqual(env["PRISM_COINBASE_OUTPUT_POLICY"], "pool-fee-first") - self.assertEqual(env["PRISM_CTV_BROADCASTER_ENABLED"], "1") - self.assertEqual(env["PRISM_CTV_BROADCASTER_WALLET"], "fanout-broadcaster") - self.assertEqual(env["PRISM_CTV_BROADCASTER_FEE_BITS"], "0") - self.assertEqual(env["PRISM_CTV_BROADCASTER_LIMIT"], "7") - self.assertEqual(env["PRISM_CTV_BROADCASTER_CHUNK_SIZE"], "3") - self.assertEqual(env["PRISM_CTV_BROADCASTER_INTERVAL_SECONDS"], "11") - self.assertEqual(env["PRISM_CTV_BROADCAST_ATTEMPT_DETAIL_LIMIT"], "9") - self.assertEqual(env["PRISM_CTV_BROADCAST_RETRY_BACKOFF_SECONDS"], "17") - self.assertEqual(env["PRISM_USERNAME_FALLBACK_ADDRESS"], "") - self.assertEqual(env["PRISM_ALLOW_MEMORY_LEDGER"], "0") - self.assertEqual(env["PRISM_ALLOW_TEST_SIGNING_SEEDS"], "0") - self.assertEqual(env["PRISM_ALLOW_BUNDLE_EMBEDDED_LEDGER_KEY"], "0") - - def test_prism_stratum_port_publish_is_configurable(self) -> None: - ports = self.config["services"]["prism-coordinator"].get("ports", []) - - self.assertTrue( - any( - str(port.get("target")) == "43340" - and str(port.get("published")) == "43340" - and port.get("host_ip") == "127.0.0.1" - for port in ports - if isinstance(port, dict) - ), - f"did not find configurable PRISM Stratum port in {ports!r}", - ) - - def test_prism_audit_directory_is_persistent(self) -> None: - volumes = self.config["services"]["prism-coordinator"].get("volumes", []) - - self.assertTrue( - any( - volume.get("type") == "volume" - and volume.get("source") == "prism-audit-data" - and volume.get("target") == "/var/lib/qbit-prism/audit" - for volume in volumes - if isinstance(volume, dict) - ), - f"did not find persistent PRISM audit volume in {volumes!r}", - ) - - def test_prism_services_restart_for_auto_recovery(self) -> None: - # The coordinator restarts on crashes and watchdog exits, while clean - # bounded-run exits (PRISM_STOP_AFTER_BLOCK / PRISM_MAX_BLOCKS) stay - # stopped. Postgres should still come back after daemon/host restarts. - self.assertEqual(self.config["services"]["prism-coordinator"].get("restart"), "on-failure") - self.assertEqual(self.config["services"]["prism-postgres"].get("restart"), "unless-stopped") - - def test_prism_healthcheck_propagates_http_503_failure(self) -> None: - healthcheck = self.config["services"]["prism-coordinator"]["healthcheck"] - command = " ".join(healthcheck["test"]) - - # urllib raises HTTPError for a 503, so the container command exits - # non-zero instead of treating any HTTP response as healthy. - self.assertIn("/healthz", command) - self.assertIn("urllib.request.urlopen", command) - self.assertEqual(healthcheck["start_period"], "15s") - self.assertEqual(healthcheck["retries"], 3) - - def test_prism_coordinator_descriptor_limit_is_configurable(self) -> None: - nofile = self.config["services"]["prism-coordinator"]["ulimits"]["nofile"] - - self.assertEqual(nofile, {"soft": 60000, "hard": 65000}) - - # -- the extracted public read tier (issue #145) ------------------------ - - def test_public_api_runs_the_public_read_service_from_the_prism_image(self) -> None: - service = self.config["services"]["prism-public-api"] - - self.assertEqual(service["command"], ["python3", "-m", "lab.prism.public_read_service"]) - self.assertEqual( - service["build"]["dockerfile"], - "lab/prism/Dockerfile", - ) - self.assertEqual( - service["image"], - self.config["services"]["prism-coordinator"]["image"], - ) - self.assertEqual(service.get("restart"), "on-failure") - - def test_public_api_survives_coordinator_restarts(self) -> None: - # The public tier must not depend on the coordinator: independence from - # coordinator restarts is a large part of why the surface was extracted. - # It waits on the standby it reads, which is also not the coordinator's - # primary -- so a primary restart does not take the dashboard with it. - depends_on = self.config["services"]["prism-public-api"].get("depends_on", {}) - - self.assertIn("prism-postgres-replica", depends_on) - self.assertEqual( - depends_on["prism-postgres-replica"]["condition"], - "service_healthy", - ) - self.assertNotIn("prism-coordinator", depends_on) - - def test_public_api_mounts_the_audit_volume_read_only(self) -> None: - volumes = self.config["services"]["prism-public-api"].get("volumes", []) - audit = [ - volume - for volume in volumes - if isinstance(volume, dict) and volume.get("source") == "prism-audit-data" - ] - - self.assertEqual(1, len(audit), f"expected one audit mount in {volumes!r}") - self.assertTrue( - audit[0].get("read_only"), - f"the audit artifact volume must be mounted read-only: {audit[0]!r}", - ) - - def test_public_api_receives_the_moved_public_knobs(self) -> None: - env = self._service_environment("prism-public-api") - - self.assertEqual(env["PRISM_PUBLIC_STRATUM_URL"], "stratum+tcp://public-pool.example:3335") - self.assertEqual(env["PRISM_PUBLIC_STRATUM_HIGHDIFF_URL"], "stratum+tcp://public-pool.example:4334") - self.assertEqual(env["PRISM_PUBLIC_POOL_FEE_BPS"], "200") - self.assertEqual(env["PRISM_PUBLIC_CACHE_ENABLED"], "1") - self.assertEqual(env["PRISM_PUBLIC_CACHE_TTL_SECONDS"], "5") - self.assertEqual(env["PRISM_PUBLIC_AGGREGATE_CACHE_TTL_SECONDS"], "30") - self.assertEqual(env["PRISM_PUBLIC_CONFIG_CACHE_TTL_SECONDS"], "300") - self.assertEqual(env["PRISM_PUBLIC_ARTIFACT_CACHE_TTL_SECONDS"], "86400") - self.assertEqual(env["PRISM_PUBLIC_CACHE_MAX_ENTRIES"], "512") - self.assertEqual(env["PRISM_PUBLIC_CACHE_DEBUG_HEADERS"], "0") - self.assertEqual( - self._public_database_url(), - "postgresql://qbit:change-this@prism-postgres-replica:5432/qbit", - ) - self.assertEqual(env["PRISM_AUDIT_DIR"], "/var/lib/qbit-prism/audit") - self.assertEqual(env["QBIT_RPC_HOST"], "qbitd") - - def test_public_api_receives_the_advertised_stratum_ports(self) -> None: - # The service runs no Stratum listener, but mining-configuration still - # renders from these: PRISM_STRATUM_PORT is the primary endpoint's - # fallback port and the High-diff endpoint appears only when - # PRISM_STRATUM_HIGHDIFF_PORT is set. Without the pass-through the - # rendered body silently drops the high-diff listener the coordinator - # is running. - env = self._service_environment("prism-public-api") - - self.assertEqual(env["PRISM_STRATUM_PORT"], "43340") - self.assertEqual(env["PRISM_STRATUM_HIGHDIFF_PORT"], "44334") - - def test_the_coordinator_no_longer_carries_the_moved_public_knobs(self) -> None: - # The coordinator serves no route that reads these any more; leaving - # them behind would imply it still did. - env = self._service_environment("prism-coordinator") - - for name in ( - "PRISM_PUBLIC_STRATUM_URL", - "PRISM_PUBLIC_POOL_NAME", - "PRISM_PUBLIC_CACHE_ENABLED", - "PRISM_PUBLIC_CACHE_TTL_SECONDS", - "PRISM_PUBLIC_ARTIFACT_CACHE_TTL_SECONDS", - "PRISM_PUBLIC_EXPLORER_TX_URL_PREFIX", - ): - with self.subTest(name=name): - self.assertNotIn(name, env) - - def test_the_reward_window_cache_knob_stays_on_both(self) -> None: - # Read by the ledger itself, and both processes construct one. - for service in ("prism-coordinator", "prism-public-api"): - with self.subTest(service=service): - env = self._service_environment(service) - self.assertEqual(env["PRISM_PUBLIC_REWARD_WINDOW_CACHE_SECONDS"], "30") - - def test_public_api_never_receives_writer_or_signing_configuration(self) -> None: - # A read tier that could acquire a writer lease or sign a manifest - # would not be a read tier. - env = self._service_environment("prism-public-api") - - self.assertEqual(env.get("PRISM_ALLOW_MEMORY_LEDGER"), "0") - for name in env: - self.assertFalse( - name.startswith("PRISM_LEDGER_WRITER_"), - f"{name} must not reach the public read tier", - ) - for name in ( - "PRISM_MANIFEST_SIGNING_SEED_HEX", - "PRISM_LEDGER_ATTESTATION_SIGNING_SEED_HEX", - "PRISM_POSTGRES_INIT_SCHEMA", - ): - with self.subTest(name=name): - self.assertNotIn(name, env) - - def test_public_api_publishes_its_port_and_healthcheck(self) -> None: - service = self.config["services"]["prism-public-api"] - published = { - str(port.get("published")) - for port in service.get("ports", []) - if isinstance(port, dict) - } - self.assertIn("3342", published) - - command = " ".join(service["healthcheck"]["test"]) - self.assertIn("/healthz", command) - self.assertIn("urllib.request.urlopen", command) - - def test_primary_pins_the_hba_file_carrying_the_replication_rules(self) -> None: - postgres = self.config["services"]["prism-postgres"] - - self.assertEqual( - postgres["command"], - ["postgres", "-c", "hba_file=/etc/postgresql/pg_hba.conf"], - ) - mounts = [ - volume - for volume in postgres.get("volumes", []) - if isinstance(volume, dict) - and volume.get("target") == "/etc/postgresql/pg_hba.conf" - ] - self.assertEqual( - len(mounts), 1, f"missing pg_hba mount in {postgres.get('volumes')!r}" - ) - self.assertTrue(mounts[0].get("read_only", False)) - self.assertEqual( - mounts[0].get("source"), - str(ROOT / "config/prism-postgres/pg_hba.conf"), - ) - - def test_primary_hba_file_authorizes_replication_and_keeps_local_access( - self, - ) -> None: - rules = [ - line.split() - for line in (ROOT / "config/prism-postgres/pg_hba.conf") - .read_text(encoding="utf-8") - .splitlines() - if line.strip() and not line.lstrip().startswith("#") - ] - - # The image's initdb bootstrap, pg_isready and `docker exec psql` all - # come in over the unix socket, and this file replaces the generated - # one, so losing the local rule would break the existing healthcheck. - self.assertEqual(rules[0][:4], ["local", "all", "all", "trust"]) - self.assertIn(["host", "all", "all", "127.0.0.1/32", "scram-sha-256"], rules) - self.assertIn(["host", "all", "all", "::1/128", "scram-sha-256"], rules) - self.assertIn(["host", "all", "all", "0.0.0.0/0", "scram-sha-256"], rules) - self.assertIn(["host", "all", "all", "::/0", "scram-sha-256"], rules) - # Compose cannot interpolate this file, so the replication rules name - # the default superuser literally. - self.assertIn( - ["host", "replication", "qbit", "0.0.0.0/0", "scram-sha-256"], rules - ) - self.assertIn(["host", "replication", "qbit", "::/0", "scram-sha-256"], rules) - - def test_replica_bootstraps_from_the_primary(self) -> None: - replica = self.config["services"]["prism-postgres-replica"] - env = self._service_environment("prism-postgres-replica") - - self.assertEqual(replica["image"], "postgres:16-alpine") - self.assertEqual( - replica["depends_on"]["prism-postgres"]["condition"], "service_healthy" - ) - self.assertEqual( - replica["entrypoint"], ["/usr/local/bin/prism-replica-entrypoint.sh"] - ) - self.assertEqual( - env["PRISM_POSTGRES_REPLICATION_SLOT"], "prism_public_replica" - ) - # The data directory is cloned by pg_basebackup, so these credentials - # exist for the replication connection and the healthcheck's psql. - self.assertEqual(env["POSTGRES_USER"], "qbit") - self.assertEqual(env["POSTGRES_DB"], "qbit") - self.assertEqual(env["PGPASSWORD"], env["POSTGRES_PASSWORD"]) - - targets = { - volume.get("target"): volume - for volume in replica.get("volumes", []) - if isinstance(volume, dict) - } - self.assertEqual( - targets["/var/lib/postgresql/data"].get("source"), - "prism-postgres-replica-data", - ) - self.assertEqual( - targets["/usr/local/bin/prism-replica-entrypoint.sh"].get("source"), - str(ROOT / "config/prism-postgres/replica-entrypoint.sh"), - ) - - def test_replica_bootstrap_script_retries_and_creates_the_slot(self) -> None: - script = (ROOT / "config/prism-postgres/replica-entrypoint.sh").read_text( - encoding="utf-8" - ) - - self.assertIn("pg_basebackup", script) - self.assertIn("--wal-method=stream", script) - self.assertIn("--write-recovery-conf", script) - self.assertIn('--slot="$REPLICATION_SLOT"', script) - self.assertIn("--create-slot", script) - # A cold `docker compose up` must survive the primary not being - # reachable yet, on a bounded budget rather than forever. - self.assertIn("PRISM_POSTGRES_REPLICA_BASEBACKUP_ATTEMPTS:-60", script) - self.assertIn("PRISM_POSTGRES_REPLICA_BASEBACKUP_RETRY_SECONDS:-5", script) - # Hand off to the stock entrypoint, which skips init on a populated - # data directory and drops privileges to the postgres account. - self.assertIn("exec docker-entrypoint.sh postgres", script) - # A bind mount aimed by mistake at a real cluster must stop the script - # rather than be cleared by the incomplete-bootstrap path below it. - self.assertIn("pg_controldata", script) - - def test_replica_healthcheck_requires_a_live_standby(self) -> None: - command = " ".join( - self.config["services"]["prism-postgres-replica"]["healthcheck"]["test"] - ) - - # An out-of-band promotion must report unhealthy rather than quietly - # become a second writable cluster behind the public read tier. - self.assertIn("pg_is_in_recovery()", command) - self.assertIn("pg_isready", command) - - def test_public_read_service_reads_the_replica_not_the_primary(self) -> None: - env = self._service_environment("prism-public-api") - - self.assertEqual(env["PRISM_DATABASE_URL"], "") - self.assertEqual(env["PRISM_PUBLIC_DATABASE_HOST"], "prism-postgres-replica") - database_url = self._public_database_url() - self.assertIn("prism-postgres-replica", database_url) - self.assertNotIn("@prism-postgres:", database_url) - self.assertEqual(env["PRISM_PUBLIC_REPLICA_MODE"], "require") - depends_on = self.config["services"]["prism-public-api"]["depends_on"] - self.assertEqual({"prism-postgres-replica"}, set(depends_on)) - self.assertEqual( - "service_healthy", depends_on["prism-postgres-replica"]["condition"] - ) - - # The coordinator keeps the primary; the split is the point. - coordinator = self._service_environment("prism-coordinator") - self.assertIn("@prism-postgres:", coordinator["PRISM_DATABASE_URL"]) - - def _public_database_url(self) -> str: - # Inspect the DSN consumed by both ledger backends after runtime encoding. - env = self._service_environment("prism-public-api") - with patch.dict(os.environ, env, clear=True), patch.object( - public_read_service, "PsqlShareLedger" - ) as ledger, patch.object(public_read_service, "build_audit_artifact_store", return_value=None): - public_read_service.build_ledger_from_env(env) - return ledger.call_args.kwargs["database_url"] - - def _service_environment(self, name: str) -> dict[str, str]: - raw_env = self.config["services"][name]["environment"] - if isinstance(raw_env, dict): - return {str(key): str(value) for key, value in raw_env.items()} - if isinstance(raw_env, list): - result = {} - for item in raw_env: - key, _, value = str(item).partition("=") - result[key] = value - return result - raise TypeError(f"unexpected environment shape for {name}: {type(raw_env).__name__}") - - -if __name__ == "__main__": - unittest.main() diff --git a/tests/test_prism_concurrency_harness.py b/tests/test_prism_concurrency_harness.py deleted file mode 100644 index 3fab3c18e..000000000 --- a/tests/test_prism_concurrency_harness.py +++ /dev/null @@ -1,992 +0,0 @@ -#!/usr/bin/env python3 -"""Self-tests for the deterministic concurrency harness. - -A harness that lies is worse than no harness: every scenario built on it -inherits the lie, and a green suite then argues *against* fixing a real -defect. These tests pin the three properties the lease scenarios depend on. - -1. **Classification is anchored to production SQL.** ``FakePostgres`` answers - statements by recognising them, so a statement it recognises wrongly is a - silent divergence. The classifier is driven from real ``PsqlShareLedger`` - calls, and anything unrecognised raises rather than returning a plausible - answer. - -2. **The ports match production.** The fakes stand in for - ``_NativePostgresClient`` and ``_NativePostgresLeaseGuard``; if those - signatures move, the fakes must break loudly rather than silently stop - modelling something. - -3. **Order is an input, not an accident.** The scheduler must produce an - identical trace every run, must refuse to step an actor that cannot - progress, and must fail fast when an actor blocks on something the - harness does not model instead of hanging the suite. -""" - -from __future__ import annotations - -import inspect -import threading -import unittest -from typing import Any - -from lab.prism import share_ledger as share_ledger_module -from lab.prism.share_ledger import ( - DEFAULT_LEASE_ACQUIRE_LOCK_TIMEOUT_SECONDS, - PendingShare, -) -from tests.prism_concurrency_harness import ( - HarnessError, - acquire_result_arms, - acquire_terminal_arm_keys, - LandingOp, - LeaseHarness, - LeaseOp, - NotRunnable, - SchedulerStall, - UnsupportedStatement, - VirtualClock, - advisory_lock_pair, - assert_deterministic, - classify, -) - - -#: The explicit not-acquired/retry terminal arm, captured verbatim from the -#: signed ledger change that introduces it (PR #168, commit 869bb7e). That -#: reference is documentation of where these bytes came from and nothing more: -#: this suite never reads that commit, and must not. A test that shells out to -#: git is a test that fails for reasons which have nothing to do with the tree -#: under test -- a shallow clone, a pruned object, an offline runner -- and one -#: that can fetch on demand makes the suite's result depend on the network. -#: The fixture is checked in so the model is pinned against a fixed input. -#: -#: Re-capture by hand if the ledger change re-words the arm; the constants it -#: interpolates are WRITER_LEASE_ACQUIRE_RETRY_KEY and -#: WRITER_LEASE_ACQUIRE_RETRY_SUBJECT. -LEDGER_RETRY_TERMINAL_ARM = """, - json_build_object( - 'acquired', false, - 'lease_snapshot_retry', true, - 'lease', 'qbit ledger writer lease', - 'retry_reason', - 'the qbit ledger writer lease row was committed by a concurrent first acquisition after this statement snapshot' - ) -);""" - -#: What the model must report for that arm. Written out independently of the -#: fixture above rather than derived from parsing it: deriving both sides from -#: one parse would make the assertion agree with itself no matter what the -#: parse did. -LEDGER_RETRY_TERMINAL_OBJECT = { - "acquired": False, - "lease_snapshot_retry": True, - "lease": "qbit ledger writer lease", - "retry_reason": ( - "the qbit ledger writer lease row was committed by a concurrent " - "first acquisition after this statement snapshot" - ), -} - - -def _without_ledger_retry_terminal_arm(sql: str) -> str: - """Return the historical two-arm acquire shape from current shipped SQL.""" - stripped = sql.rstrip() - if not stripped.endswith(LEDGER_RETRY_TERMINAL_ARM): - raise AssertionError("shipped acquire SQL does not carry the pinned terminal arm") - return stripped[: -len(LEDGER_RETRY_TERMINAL_ARM)] + "\n);" - - -def _source_of(member: Any) -> str: - """Return a member's source, or an empty string when it has none.""" - try: - return inspect.getsource(member) - except (OSError, TypeError): - return "" - - -class StatementClassificationTests(unittest.TestCase): - """The classifier is pinned to the SQL PsqlShareLedger actually emits.""" - - def test_every_lease_statement_is_recognised_from_real_calls(self) -> None: - """Drive each lease operation through production code and classify it. - - This is the pin that matters: the fragments in ``_SIGNATURES`` are - lifted from ``share_ledger.py``, and this test proves they still - match what that module generates rather than what it generated when - the harness was written. - """ - with LeaseHarness() as harness: - alpha = harness.coordinator("alpha") - alpha.start() - harness.run_until(alpha, "done:startup") - - alpha.call(alpha.ledger.renew_writer_lease, label="renew") - alpha.call( - alpha.ledger.verify_writer_lease_guard_session, - label="verify", - ) - alpha.call(alpha.ledger.release_writer_lease, label="release") - - # A second coordinator, to pin the acquire path a successor takes. - # This one finds the lease released and takes it through the - # ordinary expiry CAS rather than adopting; the adoption CAS is - # pinned separately by test_adoption_statement_classifies, which - # is the only place that statement is emitted here. - beta = harness.coordinator( - "beta", - session_token="heartbeat-v1:beta-successor", - ) - beta.start() - harness.run_until(beta, "done:startup") - - # Every statement above reached FakePostgres and was classified - # as the operation it performs; an unrecognised one would have - # raised UnsupportedStatement inside the ledger call. - self.assertEqual( - harness.statement_kinds(), - ["acquire", "renew", "verify", "release", "acquire"], - ) - - def test_adoption_statement_classifies(self) -> None: - """The adoption CAS is the one statement startup alone never emits.""" - ledger_sql = share_ledger_module.PsqlShareLedger.__dict__[ - "_try_adopt_writer_lease" - ] - self.assertIn("observed_writer_session_token", inspect.getsource(ledger_sql)) - - with LeaseHarness() as harness: - alpha = harness.coordinator("alpha") - alpha.start() - harness.run_until(alpha, "done:startup") - # Build the statement through the real method, capturing it at the - # port rather than reconstructing it here. - observed = { - "writer_session_token": "heartbeat-v1:predecessor", - "lease_updated_at": "2026-06-26 19:49:22.233718+00", - } - alpha.call( - lambda: alpha.ledger._try_adopt_writer_lease(observed), - label="adopt", - ) - self.assertEqual(harness.server.statements[-1].kind, LeaseOp.ADOPT) - self.assertEqual( - harness.server.statements[-1].payload[ - "observed_writer_session_token" - ], - "heartbeat-v1:predecessor", - ) - - def test_unknown_statement_fails_loudly(self) -> None: - with self.assertRaises(UnsupportedStatement) as caught: - classify("SELECT 1") - self.assertIn("does not model this statement", str(caught.exception)) - - def test_deferred_state_machines_name_their_follow_up(self) -> None: - for sql, expected in ( - ("INSERT INTO qbit_share_ledger (share_id)", "share submission"), - ("INSERT INTO qbit_ctv_fanouts (block_hash)", "CTV fanout"), - ): - with self.subTest(sql=sql): - with self.assertRaises(UnsupportedStatement) as caught: - classify(sql) - self.assertIn(expected, str(caught.exception)) - self.assertIn("#128", str(caught.exception)) - - def test_a_landing_statement_shape_that_moved_fails_loudly(self) -> None: - """A landing table alone is not a classification. - - The deferred marker for the outbox is gone now that landing is - modelled, so an outbox statement whose shape the classifier does not - recognise has to reach the unmodelled-statement error rather than - being answered by whichever landing evaluator matched loosest. - """ - with self.assertRaises(UnsupportedStatement) as caught: - classify("UPDATE qbit_block_candidate_outbox SET quarantined = true") - self.assertIn("does not model this statement", str(caught.exception)) - self.assertIn("_LANDING_SIGNATURES", str(caught.exception)) - - def test_shipped_share_append_is_not_read_as_the_outbox_state_machine( - self, - ) -> None: - """A share append must not be misnamed as the outbox state machine. - - ``_append_batch_with_replay_outcomes`` opens the very same - ``INSERT INTO qbit_block_candidate_outbox (`` and raises the very same - ``'block candidate payload mismatch'`` as - ``persist_block_candidate_intent``. Those two fragments alone therefore - match a share submission, and the classifier answered one with the - outbox evaluator: the statement was reported against the wrong state - machine, so the failure argued for an outbox bug that did not exist and - hid the deferral that did. This drives the shipped ``append_batch`` and - requires the captured statement to reach ``_DEFERRED_SIGNATURES``. - """ - share = PendingShare( - share_id="miner-a:1", - miner_id="miner-a", - order_key="miner-a", - p2mr_program_hex="22" * 32, - share_difficulty=1, - network_difficulty=1, - template_height=9, - job_id="job-1", - job_issued_at_ms=1, - accepted_at_ms=2, - ntime=1_700_000_000, - ) - candidate = {"block_hash_hex": "ab" * 32} - - captured: list[str] = [] - with LeaseHarness() as harness: - alpha = harness.coordinator("alpha") - alpha.start() - harness.run_until(alpha, "done:startup") - - # Capture at the port. The classifier rejects this statement, so it - # never reaches server.statements; the raw SQL has to be taken on - # the way in rather than read back out of the recognised trace. - executed = harness.server.execute - - def capturing_execute(backend: Any, sql: str, **kwargs: Any) -> Any: - captured.append(sql) - return executed(backend, sql, **kwargs) - - harness.server.execute = capturing_execute # type: ignore[method-assign] - - # Pin the historical ambiguous SQL discriminator. The v2 append - # has a distinct version-CAS shape and its own storage gate. - alpha.ledger._candidate_storage_version = 1 - with self.assertRaises(UnsupportedStatement) as caught: - alpha.call( - lambda: alpha.ledger.append_batch([(share, candidate)]), - label="append", - ) - - # The statement really is ambiguous under the two original fragments; - # if production stops emitting both, this test is no longer guarding - # anything and should fail rather than quietly pass. - appends = [sql for sql in captured if "INSERT INTO qbit_share_ledger" in sql] - self.assertEqual(len(appends), 1) - sql = appends[0] - self.assertIn("INSERT INTO qbit_block_candidate_outbox (", sql) - self.assertIn("'block candidate payload mismatch'", sql) - self.assertIn("INSERT INTO qbit_share_ledger", sql) - - # The discriminator: a share append claims a colliding outbox row, - # it does not leave it alone. - self.assertIn("ON CONFLICT (block_hash) DO UPDATE", sql) - self.assertNotIn("ON CONFLICT (block_hash) DO NOTHING", sql) - - # Both the in-flight rejection and a direct classification of the - # captured statement have to name the same state machine. - with self.assertRaises(UnsupportedStatement) as direct: - classify(sql) - for message in (str(caught.exception), str(direct.exception)): - self.assertIn("share submission and dedupe", message) - self.assertIn("#128", message) - self.assertNotIn(LandingOp.OUTBOX_RECORD.value, message) - - def test_block_candidate_intent_still_classifies_as_the_outbox_record( - self, - ) -> None: - """The discriminator must not cost the outbox its own classification. - - Narrowing ``OUTBOX_RECORD`` is only correct if the statement it exists - for still matches, so this pins the other direction of the same change. - """ - with LeaseHarness() as harness: - alpha = harness.coordinator("alpha") - alpha.start() - harness.run_until(alpha, "done:startup") - alpha.call( - lambda: alpha.ledger.persist_block_candidate_intent( - {"block_hash_hex": "ab" * 32} - ), - label="intent", - ) - statement = harness.server.statements[-1] - self.assertEqual(statement.kind, LandingOp.OUTBOX_RECORD) - self.assertEqual(statement.payload["block_hash"], "ab" * 32) - self.assertIn("ON CONFLICT (block_hash) DO NOTHING", statement.sql) - - def test_the_outbox_discriminator_is_unique_to_the_intent_writer( - self, - ) -> None: - """Pin the fragment's uniqueness to production, not to this test. - - ``OUTBOX_RECORD`` is now identified by a conflict action. That is only - a valid identity for the two versioned intent writers; if another - writer starts inserting the candidate with ``DO NOTHING`` the - classifier is ambiguous again and this fails at that moment rather - than at the next confusing scenario failure. - """ - discriminator = "ON CONFLICT (block_hash) DO NOTHING" - emitters = sorted( - name - for name, member in vars(share_ledger_module.PsqlShareLedger).items() - if discriminator in _source_of(member) - ) - self.assertEqual(emitters, ["_persist_block_candidate_intent_v1", "persist_block_candidate_intent"]) - - def test_classifier_extracts_the_configured_ttl(self) -> None: - with LeaseHarness(lease_ttl_seconds=17.5) as harness: - alpha = harness.coordinator("alpha") - alpha.start() - harness.run_until(alpha, "done:startup") - self.assertEqual( - harness.server.statements[0].lease_ttl_seconds, - 17.5, - ) - row = harness.lease_row() - assert row is not None - self.assertEqual( - (row.lease_expires_at - row.updated_at).total_seconds(), - 17.5, - ) - - def test_advisory_lock_pair_matches_production_splitting(self) -> None: - key = share_ledger_module._writer_lease_advisory_lock_key( - "prism-coordinator", - 1, - ) - classid, objid = advisory_lock_pair(key) - self.assertEqual(classid, (key >> 32) & 0xFFFFFFFF) - self.assertEqual(objid, key & 0xFFFFFFFF) - - -class PortContractTests(unittest.TestCase): - """The fakes must keep matching the production classes they replace. - - The repository runs no mypy — CI is `compileall` plus `unittest discover` - — so structural typing is never checked statically. Without these the two - protocols would be documentation: nothing would notice a fake that - stopped implementing one, or a production class that outgrew it. - """ - - def _guard_stub(self) -> Any: - guard = share_ledger_module._NativePostgresLeaseGuard.__new__( - share_ledger_module._NativePostgresLeaseGuard - ) - guard._connection = None - guard._advisory_lock_key = 0 - guard._query_lock = threading.Lock() - guard._closed = False - guard._held = True - return guard - - def test_production_classes_satisfy_their_own_ports(self) -> None: - client = share_ledger_module._NativePostgresClient.__new__( - share_ledger_module._NativePostgresClient - ) - self.assertIsInstance(client, share_ledger_module.LedgerSqlPort) - self.assertIsInstance( - self._guard_stub(), - share_ledger_module.LeaseGuardPort, - ) - - def test_fakes_satisfy_the_ports_they_stand_in_for(self) -> None: - from tests.prism_concurrency_harness import FakeLeaseGuard, FakeSqlBackend - - with LeaseHarness() as harness: - alpha = harness.coordinator("alpha") - alpha.start() - harness.run_until(alpha, "done:startup") - - self.assertIsInstance(alpha.sql_backend, FakeSqlBackend) - self.assertIsInstance(alpha.guard, FakeLeaseGuard) - self.assertIsInstance( - alpha.sql_backend, - share_ledger_module.LedgerSqlPort, - ) - self.assertIsInstance( - alpha.guard, - share_ledger_module.LeaseGuardPort, - ) - - def test_every_port_method_matches_production_signature(self) -> None: - """Full signatures, not just parameter names. - - Comparing `list(signature.parameters)` alone would stay green if a - keyword-only argument became positional or lost its default, which is - exactly the kind of drift that makes a fake silently stop modelling - the thing it replaces. - """ - from tests.prism_concurrency_harness import FakeLeaseGuard, FakeSqlBackend - - for production, fake, methods in ( - ( - share_ledger_module._NativePostgresClient, - FakeSqlBackend, - ("run_json", "run_script", "close"), - ), - ( - share_ledger_module._NativePostgresLeaseGuard, - FakeLeaseGuard, - ("run_json", "try_acquire", "close"), - ), - ): - for method in methods: - with self.subTest(production=production.__name__, method=method): - self.assertEqual( - inspect.signature(getattr(production, method)), - inspect.signature(getattr(fake, method)), - ) - - def test_sql_backend_matches_native_client_signature(self) -> None: - from tests.prism_concurrency_harness import FakeSqlBackend - - production = inspect.signature( - share_ledger_module._NativePostgresClient.run_json - ) - fake = inspect.signature(FakeSqlBackend.run_json) - self.assertEqual(list(production.parameters), list(fake.parameters)) - - def test_lease_guard_matches_native_guard_signature(self) -> None: - from tests.prism_concurrency_harness import FakeLeaseGuard - - production = inspect.signature( - share_ledger_module._NativePostgresLeaseGuard.run_json - ) - fake = inspect.signature(FakeLeaseGuard.run_json) - self.assertEqual(list(production.parameters), list(fake.parameters)) - - def test_ledger_accepts_the_fakes_through_its_constructor(self) -> None: - """No bound method is reassigned to install the harness.""" - parameters = inspect.signature( - share_ledger_module.PsqlShareLedger.__init__ - ).parameters - for name in ("sql_backend_factory", "lease_guard_factory", "monotonic"): - self.assertIn(name, parameters) - - -class VirtualClockTests(unittest.TestCase): - def test_time_only_moves_when_advanced(self) -> None: - clock = VirtualClock() - first = clock.monotonic() - for _ in range(1000): - clock.now() - self.assertEqual(clock.monotonic(), first) - clock.advance(2.5) - self.assertEqual(clock.monotonic(), first + 2.5) - - def test_time_cannot_move_backwards(self) -> None: - clock = VirtualClock() - with self.assertRaises(HarnessError): - clock.advance(-1) - - def test_timestamp_text_is_postgres_shaped(self) -> None: - clock = VirtualClock() - self.assertEqual( - VirtualClock.timestamp_text(clock.now()), - "2026-06-26 19:49:22.233718+00", - ) - - -class SchedulerTests(unittest.TestCase): - def test_stepping_a_blocked_actor_is_refused(self) -> None: - with LeaseHarness() as harness: - alpha = harness.coordinator("alpha") - alpha.start() - harness.run_until(alpha, "done:startup") - - def sleep_forever() -> None: - harness.sleep(30.0) - - alpha.submit(sleep_forever, label="sleep") - harness.run_until(alpha, "sleep:30") - - with self.assertRaises(NotRunnable): - harness.step(alpha) - - # Time is the only thing that can release it, and the harness can - # say exactly when. - self.assertEqual( - harness.scheduler.next_deadline(), - harness.clock.monotonic() + 30.0, - ) - harness.advance_to_next_deadline() - harness.run_until(alpha, "done:sleep") - - def test_blocking_outside_the_model_is_reported_not_hung(self) -> None: - """An actor waiting on something unmodelled must fail, not hang.""" - with LeaseHarness(baton_timeout_seconds=0.4) as harness: - alpha = harness.coordinator("alpha") - unmodelled = threading.Lock() - unmodelled.acquire() - alpha.submit(unmodelled.acquire, label="wedge") - - with self.assertRaises(SchedulerStall) as caught: - harness.run_until(alpha, "done:wedge") - self.assertIn("does not model", str(caught.exception)) - unmodelled.release() - - def test_repeated_runs_produce_an_identical_schedule(self) -> None: - def scenario(harness: LeaseHarness) -> dict[str, object]: - # Distinct epochs: each computes a different advisory-guard key, - # so both reach the lease upsert and genuinely race for the row. - alpha = harness.coordinator("alpha", writer_epoch=1) - beta = harness.coordinator("beta", writer_epoch=2) - alpha.start() - beta.start() - # Both statements are in flight before either executes. - harness.run_until(alpha, "alpha.begin:acquire") - harness.run_until(beta, "beta.begin:acquire") - harness.drain([alpha, beta]) - beta_error = beta.actor.calls[0].error - return { - "holder": harness.lease_holder_session(), - "kinds": harness.statement_kinds(), - "alpha_failed": alpha.actor.calls[0].error is not None, - "beta_error": None if beta_error is None else str(beta_error), - "acquire_arms": acquire_result_arms(harness.server.statements[0].sql), - } - - run = assert_deterministic(self, scenario, repeats=25) - # The schedule decides the winner, and it decides it the same way - # every run. That reproducibility is the property under test; the - # identity of the winner is just what this schedule happens to pick. - self.assertEqual(run.outcome["holder"], "heartbeat-v1:alpha-1") - # The race loser receives the explicit retry sentinel and performs one - # fresh-snapshot acquire, so the current three-arm statement executes - # three acquire operations in this schedule. - self.assertEqual(run.outcome["kinds"], ["acquire", "acquire", "acquire"]) - self.assertFalse(run.outcome["alpha_failed"]) - # What the loser gets is a property of the shipped statement, so it is - # asserted against the statement rather than pinned to a constant. On - # the two-arm shape its DO UPDATE matches nothing and the COALESCE - # fallback reads a snapshot taken before the winner's row existed, so - # the whole statement evaluates to SQL NULL. Issue #140's terminal arm - # retires that outcome. See - # test_first_acquire_race_loser_matches_the_shipped_acquire_shape. - if run.outcome["acquire_arms"] == 2: - self.assertEqual( - run.outcome["beta_error"], - "postgres query returned no JSON", - ) - else: - self.assertNotIn( - "returned no JSON", - str(run.outcome["beta_error"]), - ) - - def test_first_acquire_race_loser_matches_the_shipped_acquire_shape(self) -> None: - """Two coordinators racing the first-ever acquire. - - What the loser gets is decided by the shipped statement, so this test - reads the statement and asserts what that shape can produce, instead - of pinning whichever outcome was current when it was written. - - **Two-arm shape (this baseline).** The losing - ``INSERT ... ON CONFLICT DO UPDATE`` affects zero rows, and the - ``COALESCE`` arm that would report the holder reads this statement's - own snapshot -- taken before the winner's row was committed -- so it - finds nothing and the statement returns SQL NULL. - ``parse_single_json_value`` turns that into ``postgres query returned - no JSON``, which ``_ensure_writer_lease`` does not catch, so - ``PsqlShareLedger.__init__`` dies with an error naming nothing about - leases or about the coordinator that holds it. That is a real - production outcome that reads like a driver fault, which is why it was - pinned rather than left implicit. - - **Terminal-arm shape (issue #140).** The arm exists precisely so that - race reports a lease outcome, so the generic no-JSON diagnostic must - no longer be reachable. This asserts its retirement rather than a - message this branch has never seen: the operator-facing text is the - ledger PR's to choose, but "not the driver fault" is the whole point - of the change and is safe to require here. - """ - with LeaseHarness() as harness: - alpha = harness.coordinator("alpha", writer_epoch=1) - beta = harness.coordinator("beta", writer_epoch=2) - alpha.start() - beta.start() - harness.run_until(alpha, "alpha.begin:acquire") - harness.run_until(beta, "beta.begin:acquire") - harness.drain([alpha, beta]) - - alpha.actor.calls[0].value() - arms = acquire_result_arms(harness.server.statements[0].sql) - if arms == 2: - with self.assertRaisesRegex( - RuntimeError, - "postgres query returned no JSON", - ): - beta.actor.calls[0].value() - else: - error = beta.actor.calls[0].error - self.assertNotIn("returned no JSON", str(error)) - - # Contrast, and unchanged by either shape: once the winner's row - # is visible in the loser's snapshot, the same race produces the - # ordinary refusal. - gamma = harness.coordinator("gamma", writer_epoch=3) - gamma.start() - harness.run_until(gamma, "done:startup") - with self.assertRaisesRegex( - RuntimeError, - "qbit ledger writer lease is held by", - ): - gamma.actor.calls[0].value() - - def test_acquire_arm_count_is_read_from_the_shipped_statement(self) -> None: - """The compatibility switch is computed, never declared. - - If this read a constant or a version flag, the whole arrangement would - be a guess about production that drifts silently. It reads the - statement ``PsqlShareLedger`` actually emitted, and pins what that - statement is at this baseline so a shape change cannot pass unnoticed. - """ - with LeaseHarness() as harness: - alpha = harness.coordinator("alpha") - alpha.start() - harness.run_until(alpha, "done:startup") - statement = harness.server.statements[0] - self.assertEqual(statement.kind, LeaseOp.ACQUIRE) - sql = statement.sql - - # The shipped shape has the acquired, holder, and explicit retry arms. - # Asserting the terminal keys as well as the count stops a renamed arm - # from being read as unchanged. - self.assertEqual(acquire_result_arms(sql), 3) - self.assertEqual( - acquire_terminal_arm_keys(sql), - ( - "acquired", - "lease_snapshot_retry", - "lease", - "retry_reason", - ), - ) - - # A shape neither this model nor #140 describes stays loud. - with self.assertRaises(UnsupportedStatement) as caught: - acquire_result_arms("SELECT 1;") - self.assertIn("SELECT COALESCE", str(caught.exception)) - - def test_the_explicit_not_acquired_arm_is_modelled_when_present(self) -> None: - """Exercise the integrated #140 arm and the historical two-arm shape. - - The current statement must carry the pinned terminal arm and the model - must answer it. Removing only that captured arm recreates the old - two-arm statement and proves the compatibility switch still switches. - - The arm is the captured fixture and the expected object is written out - separately, so this holds the model against a fixed input without - reading anything outside this tree. - """ - with LeaseHarness() as harness: - alpha = harness.coordinator("alpha") - alpha.start() - harness.run_until(alpha, "done:startup") - server = harness.server - shipped = server.statements[0].sql - - terminal = shipped.rstrip() - baseline = _without_ledger_retry_terminal_arm(shipped) - self.assertEqual(acquire_result_arms(terminal), 3) - self.assertEqual( - acquire_terminal_arm_keys(terminal), - ("acquired", "lease_snapshot_retry", "lease", "retry_reason"), - ) - - statement = classify(terminal) - answered = server._unobserved_not_acquired(statement) - - # The whole point: this shape cannot evaluate to SQL NULL, so the - # ports can no longer raise "postgres query returned no JSON". The - # values are the statement's own literals, including the sentence -- - # a boolean here would mean the model was answering the key's name. - self.assertEqual(answered, LEDGER_RETRY_TERMINAL_OBJECT) - # Spelled out separately: answering the key's *name* is the exact - # defect the first policy had, and it would put a boolean here. Dict - # equality alone reads as a typo check and would not say that. - assert answered is not None - self.assertIsInstance(answered["retry_reason"], str) - self.assertIs(answered["lease_snapshot_retry"], True) - self.assertIs(answered["acquired"], False) - - # And the historical shape still must, or the switch is not switching. - self.assertIsNone(server._unobserved_not_acquired(classify(baseline))) - - def test_a_terminal_literal_survives_commas_and_quotes(self) -> None: - """Punctuation inside a reason string must not become an argument break. - - The sentence this arm ships today happens to contain no comma, so - nothing else in this suite exercises the splitter's quoting. That is - a thin place to leave under a model whose whole job is to report the - wording production chose: the first comma added to that sentence - would otherwise silently re-pair every key and value after it, and - the failure would look like a renamed key rather than a parser bug. - """ - reason = ( - "the lease row was committed by a concurrent acquisition, " - "after this statement''s snapshot" - ) - arm = ( - ",\n json_build_object(\n" - " 'acquired', false,\n" - " 'lease_snapshot_retry', true,\n" - " 'lease', 'qbit ledger writer lease',\n" - " 'retry_reason',\n" - f" '{reason}'\n" - " )\n);" - ) - with LeaseHarness() as harness: - alpha = harness.coordinator("alpha") - alpha.start() - harness.run_until(alpha, "done:startup") - server = harness.server - baseline = _without_ledger_retry_terminal_arm(server.statements[0].sql) - terminal = baseline.rstrip()[:-2] + arm - - self.assertEqual(acquire_result_arms(terminal), 3) - self.assertEqual( - acquire_terminal_arm_keys(terminal), - ("acquired", "lease_snapshot_retry", "lease", "retry_reason"), - ) - answered = server._unobserved_not_acquired(classify(terminal)) - - assert answered is not None - self.assertEqual( - answered["retry_reason"], - "the lease row was committed by a concurrent acquisition, " - "after this statement's snapshot", - ) - self.assertEqual(answered["lease"], "qbit ledger writer lease") - - def test_an_unteachable_terminal_arm_key_stays_loud(self) -> None: - """A key this model cannot answer must fail, not be invented. - - The value policy for the terminal arm is deliberately narrow: - ``acquired`` is false, the holder and lease columns are null because - nothing is visible, and a retry flag is true. Anything else is a shape - nobody has taught this model, and guessing at it would be exactly the - harness lie the suite's docstring warns about. - """ - with LeaseHarness() as harness: - alpha = harness.coordinator("alpha") - alpha.start() - harness.run_until(alpha, "done:startup") - server = harness.server - shipped = server.statements[0].sql - terminal = shipped.rstrip()[:-2] + ( - ",\n (\n SELECT json_build_object(\n" - " 'acquired', false,\n" - " 'backoff_hint_seconds', 3\n" - " )\n )\n);" - ) - statement = classify(terminal) - with self.assertRaises(UnsupportedStatement) as caught: - server._unobserved_not_acquired(statement) - self.assertIn("backoff_hint_seconds", str(caught.exception)) - - def test_trace_names_the_actor_and_the_stop(self) -> None: - with LeaseHarness() as harness: - alpha = harness.coordinator("alpha") - alpha.start() - harness.run_until(alpha, "done:startup") - self.assertEqual( - harness.trace, - [ - "alpha@alpha.begin:acquire", - "alpha@alpha.done:acquire", - # The acquire runs under #123's acquisition deadline, so - # it commits explicitly and offers a precommit stop. - "alpha@alpha.precommit", - "alpha@done:startup", - ], - ) - - -class PostgresModelTests(unittest.TestCase): - """The PostgreSQL behaviours the lease lifecycle actually relies on.""" - - def test_advisory_lock_is_exclusive_and_session_scoped(self) -> None: - with LeaseHarness() as harness: - alpha = harness.coordinator("alpha", writer_epoch=1) - alpha.start() - harness.run_until(alpha, "done:startup") - assert alpha.guard is not None - self.assertTrue(alpha.guard.held) - - # A same-identity twin computes the same key and cannot take it. - key = alpha.guard.advisory_lock_key - other = harness.server.connect(application_name="twin") - self.assertFalse(harness.server.try_advisory_lock(other, key)) - - # Closing the holder's session releases it server-side. - alpha.guard.close() - self.assertTrue(harness.server.try_advisory_lock(other, key)) - - def test_tuple_lock_is_held_for_the_life_of_a_transaction(self) -> None: - with LeaseHarness() as harness: - alpha = harness.coordinator("alpha") - alpha.start() - harness.run_until(alpha, "done:startup") - - def deadline_scoped_write() -> object: - with alpha.ledger.operation_timeout(30.0): - return alpha.ledger.renew_writer_lease() - - alpha.submit(deadline_scoped_write, label="write") - harness.run_until(alpha, "alpha.done:renew") - # Statement executed, COMMIT not yet sent: the lock is held and - # the committed row still carries the pre-write value. - self.assertIsNotNone(harness.server.lease_lock_holder) - row = harness.lease_row() - assert row is not None - self.assertIsNotNone(row.xmax) - - harness.run_until(alpha, "done:write") - self.assertIsNone(harness.server.lease_lock_holder) - row = harness.lease_row() - assert row is not None - self.assertIsNone(row.xmax) - - def test_vanishing_during_an_autocommit_statement_leaves_no_orphan(self) -> None: - """The model must not offer #123's shape on a path that cannot produce it. - - A client that vanishes mid-statement leaves an orphan only when the - COMMIT is a separate message it never sends. Autocommit has no such - message: PostgreSQL finishes the statement and commits it. A harness - that let a scenario build the orphan shape on such a path would argue - for a fix to a defect that path cannot reach. - - This used to drive the startup lease acquire, which was autocommit - because no caller armed a deadline on it. #123's fix arms one, so the - acquire is no longer an example of this property — see - ``test_vanishing_mid_acquire_orphans_now_that_it_is_deadline_scoped`` - for what that path does instead. An unscoped renewal is still a plain - autocommit statement and demonstrates the property unchanged; the - property under test never was about the acquire in particular. - """ - with LeaseHarness() as harness: - alpha = harness.coordinator("alpha") - alpha.start() - harness.run_until(alpha, "done:startup") - - # No operation_timeout: run_json gets timeout_seconds=None, so - # the model runs it the way _NativePostgresClient does, without - # an enclosing conn.transaction(). - alpha.submit(lambda: alpha.ledger.renew_writer_lease(), label="renew") - harness.run_until(alpha, "alpha.done:renew") - self.assertIsNotNone( - harness.server.lease_lock_holder, - "the statement is mid-flight and holds the tuple lock", - ) - - alpha.vanish() - - self.assertIsNone( - harness.server.lease_lock_holder, - "the server finished and committed the autocommit statement", - ) - row = harness.lease_row() - assert row is not None - self.assertEqual(row.writer_session_token, "heartbeat-v1:alpha-1") - self.assertIsNone( - row.xmax, - "no transaction is left holding the row", - ) - - def test_vanishing_mid_acquire_orphans_now_that_it_is_deadline_scoped( - self, - ) -> None: - """#123's fix widens the orphan shape onto the startup acquire itself. - - ``_run_lease_acquisition_json`` wraps the acquire in - ``statement_timeout(...)`` so that a successor cannot queue forever - on the lease tuple. That deadline is also what makes - ``_NativePostgresClient`` run the statement inside an explicit - transaction with a separate COMMIT — so a coordinator that vanishes - mid-acquire now leaves exactly the orphan #123 is about, on a path - that previously could not produce one. - - This is an accepted trade rather than a regression, and it is pinned - here so it stays deliberate. The orphan #123 describes was unbounded; - this one is bounded by ``idle_in_transaction_session_timeout`` - (``PRISM_POSTGRES_IDLE_IN_TRANSACTION_TIMEOUT_SECONDS``, 15s by - default), which the same fix sets on every coordinator session. That - is why the session guard is not optional, and - docs/prism-ledger-ops.md records the trade. - - The reap itself is not asserted here: this harness models no - idle-in-transaction reaper, and faking one would pin the model's - opinion rather than PostgreSQL's. That the option is actually set on - every connection path is pinned directly against the shipped client - by ``PostgresSessionGuardTests`` in tests/test_prism_share_ledger.py. - What this test pins is what the model does support: the orphan is - real, it holds the lease lock, and a successor that meets it is - bounded rather than stuck. - """ - with LeaseHarness() as harness: - alpha = harness.coordinator("alpha", writer_epoch=1) - alpha.start() - # A checkpoint that did not exist on this path before the fix. - harness.run_until(alpha, "alpha.precommit") - alpha.vanish() - - orphan = harness.server.lease_lock_holder - self.assertIsNotNone(orphan) - assert orphan is not None - self.assertTrue(orphan.orphaned) - self.assertTrue(orphan.explicit, "the deadline is what opened it") - self.assertIsNone( - harness.lease_row(), - "the acquire never committed, so no lease row exists at all", - ) - - # The successor still gets out, which is the half that matters: - # this orphan costs a bounded startup delay, not an outage. - beta = harness.coordinator("beta", writer_epoch=2) - beta.start() - harness.run_until_blocked(beta, "beta.lockwait:acquire") - self.assertEqual( - beta.actor.wake_at, - harness.clock.monotonic() - + DEFAULT_LEASE_ACQUIRE_LOCK_TIMEOUT_SECONDS, - ) - - def test_vanishing_inside_a_deadline_scoped_statement_does_orphan(self) -> None: - """The contrast: a separate COMMIT the client never sends.""" - with LeaseHarness() as harness: - alpha = harness.coordinator("alpha") - alpha.start() - harness.run_until(alpha, "done:startup") - - def deadline_scoped_write() -> object: - with alpha.ledger.operation_timeout(30.0): - return alpha.ledger.renew_writer_lease() - - alpha.submit(deadline_scoped_write, label="write") - harness.run_until(alpha, "alpha.precommit") - alpha.vanish() - - orphan = harness.server.lease_lock_holder - self.assertIsNotNone(orphan) - assert orphan is not None - self.assertTrue(orphan.orphaned) - - def test_autocommit_statements_never_leave_a_transaction_open(self) -> None: - """No caller deadline, so no explicit transaction to be caught inside. - - Also drove the startup acquire until #123's fix armed a deadline on - it. Asserting the absence of a precommit stop only means something on - a statement that genuinely has no caller deadline, so this drives an - unscoped renewal; on the acquire the assertion would now be checking - the wrong path and passing for the wrong reason. - """ - with LeaseHarness() as harness: - alpha = harness.coordinator("alpha") - alpha.start() - harness.run_until(alpha, "done:startup") - - mark = len(harness.trace) - alpha.submit(lambda: alpha.ledger.renew_writer_lease(), label="renew") - harness.run_until(alpha, "done:renew") - - self.assertNotIn("alpha@alpha.precommit", harness.trace[mark:]) - self.assertIsNone(harness.server.lease_lock_holder) - - -if __name__ == "__main__": - unittest.main() diff --git a/tests/test_prism_container_hardening.py b/tests/test_prism_container_hardening.py new file mode 100644 index 000000000..c36911fdd --- /dev/null +++ b/tests/test_prism_container_hardening.py @@ -0,0 +1,145 @@ +#!/usr/bin/env python3 +"""Hold the PRISM image and lab Compose services to the #288 hardening contract.""" + +from __future__ import annotations + +import json +import os +import re +import shutil +import subprocess +import unittest +from pathlib import Path + + +ROOT = Path(__file__).resolve().parents[1] +DOCKERFILE = ROOT / "lab" / "prism" / "Dockerfile" +NATIVE_SETTINGS = ROOT / "crates" / "qbit-prism-server" / "src" / "config" / "native-settings.txt" +PRISM_SERVICES = ("prism-coordinator", "prism-public-api") + + +def runtime_stage() -> list[str]: + """Return the logical instructions of the final stage, continuations joined.""" + text = DOCKERFILE.read_text(encoding="utf-8").replace("\\\n", " ") + lines = [line.strip() for line in text.splitlines()] + instructions = [line for line in lines if line and not line.startswith("#")] + last_from = max(i for i, line in enumerate(instructions) if line.startswith("FROM ")) + return instructions[last_from:] + + +class PrismDockerfileTests(unittest.TestCase): + def test_runtime_base_is_digest_pinned(self) -> None: + self.assertRegex(runtime_stage()[0], r"^FROM debian:bookworm-slim@sha256:[0-9a-f]{64}$") + + def test_runtime_runs_as_numeric_non_root_user_after_setup(self) -> None: + stage = runtime_stage() + users = [i for i, line in enumerate(stage) if line.startswith("USER ")] + self.assertEqual(len(users), 1) + match = re.fullmatch(r"USER (\d+):(\d+)", stage[users[0]]) + self.assertIsNotNone(match, stage[users[0]]) + self.assertNotEqual(match.group(1), "0") + self.assertNotEqual(match.group(2), "0") + setup = [i for i, line in enumerate(stage) if line.startswith(("RUN ", "COPY "))] + self.assertGreater(users[0], max(setup)) + self.assertTrue(any(f"--uid {match.group(1)}" in line for line in stage)) + + def test_entrypoint_disables_core_dumps_and_execs_the_command(self) -> None: + self.assertIn( + 'ENTRYPOINT ["/bin/sh", "-c", "ulimit -c 0 && exec \\"$@\\"", "qbit-prism"]', + runtime_stage(), + ) + + def test_healthcheck_probes_the_operator_listener(self) -> None: + healthchecks = [line for line in runtime_stage() if line.startswith("HEALTHCHECK ")] + self.assertEqual(len(healthchecks), 1) + self.assertTrue( + re.search(r' CMD \["qbit-prism-server", "healthcheck"\]$', healthchecks[0]), + healthchecks[0], + ) + self.assertEqual(runtime_stage()[-1], 'CMD ["qbit-prism-server"]') + + +class PrismImageSmokeWiringTests(unittest.TestCase): + def test_ci_runs_the_image_smoke_and_settings_guard(self) -> None: + workflow = (ROOT / ".github" / "workflows" / "ci.yml").read_text(encoding="utf-8") + self.assertIn("run: bash .github/scripts/test-prism-image-runtime.sh", workflow) + self.assertIn("PRISM_IMAGE: qbit-lab-prism:ci", workflow) + self.assertIn("run: python3 scripts/check_prism_settings.py", workflow) + + +class PrismLabComposeTests(unittest.TestCase): + @classmethod + def setUpClass(cls) -> None: + docker = shutil.which("docker") + if docker is None: + raise unittest.SkipTest("docker CLI is not installed") + inherited = ("PATH", "HOME", "DOCKER_HOST", "DOCKER_CONTEXT", "XDG_CONFIG_HOME") + env = {key: os.environ[key] for key in inherited if key in os.environ} + env["QBIT_SRC_DIR"] = str(ROOT) + completed = subprocess.run( + [docker, "compose", + "--env-file", str(ROOT / "config/upstream.env.example"), + "--env-file", str(ROOT / ".env.example"), + "-f", str(ROOT / "compose.yaml"), + "--project-name", "qbit-prism-hardening-contract", + "--profile", "prism", "config", "--format", "json"], + cwd=ROOT, env=env, text=True, capture_output=True, check=False, + ) + if completed.returncode != 0 and "compose" in completed.stderr and "not a docker command" in completed.stderr: + raise unittest.SkipTest("docker compose is unavailable") + if completed.returncode != 0: + raise AssertionError(f"lab compose render failed: {completed.stderr}") + cls.services = json.loads(completed.stdout)["services"] + + def test_prism_containers_disable_core_dumps(self) -> None: + for name in PRISM_SERVICES: + with self.subTest(service=name): + core = self.services[name]["ulimits"]["core"] + # The rendered form omits zero-valued soft and hard limits. + limits = ( + (core.get("soft", 0), core.get("hard", 0)) if isinstance(core, dict) else (core, core) + ) + self.assertEqual(limits, (0, 0)) + + def test_role_healthchecks_match_their_listeners(self) -> None: + self.assertEqual( + self.services["prism-coordinator"]["healthcheck"]["test"], + ["CMD", "qbit-prism-server", "healthcheck"], + ) + self.assertEqual( + self.services["prism-public-api"]["healthcheck"]["test"], + ["CMD", "qbit-prism-server", "healthcheck", "--public-api"], + ) + + def test_frontend_receives_mounted_secret_and_operator_token_settings(self) -> None: + env = self.services["prism-coordinator"]["environment"] + for name in ( + "PRISM_MANIFEST_SIGNING_SEED_HEX_FILE", + "PRISM_LEDGER_ATTESTATION_SIGNING_SEED_HEX_FILE", + "PRISM_OPERATOR_BEARER_TOKEN", + "PRISM_OPERATOR_BEARER_TOKEN_FILE", + ): + with self.subTest(name=name): + self.assertEqual(env[name], "") + + def test_public_api_is_seedless(self) -> None: + env = self.services["prism-public-api"]["environment"] + self.assertFalse( + [name for name in env if "SIGNING_SEED" in name or "OPERATOR_BEARER_TOKEN" in name] + ) + + def test_native_processes_receive_only_native_prism_settings(self) -> None: + # Production check-config rejects an unread PRISM_* name, so Compose-only + # values such as PRISM_POSTGRES_PASSWORD must stay out of these services. + native = { + line for line in NATIVE_SETTINGS.read_text(encoding="utf-8").splitlines() + if line.startswith("PRISM_") + } + for name in PRISM_SERVICES: + with self.subTest(service=name): + passed = {key for key in self.services[name]["environment"] if key.startswith("PRISM_")} + self.assertEqual(passed - native, set()) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_prism_coordinator_config.py b/tests/test_prism_coordinator_config.py deleted file mode 100644 index d31c83bd4..000000000 --- a/tests/test_prism_coordinator_config.py +++ /dev/null @@ -1,2616 +0,0 @@ -#!/usr/bin/env python3 -"""Focused PRISM coordinator config tests.""" -# ruff: noqa: F403, F405 - -from __future__ import annotations - -import sys -import tempfile -import unittest -from pathlib import Path - -from lab.prism.coordinator_config import ( - DEFAULT_PRISM_VARDIFF_INITIAL_MAX_STEP_UP, - MAX_PRISM_PYTHON_SWITCH_INTERVAL_SECONDS, - MIN_PRISM_PYTHON_SWITCH_INTERVAL_SECONDS, - apply_python_switch_interval, - env_decimal, - load_coordinator_config, - resolve_python_switch_interval_seconds, -) -from tests.prism_vardiff_test_support import * - - -class PrismPythonSwitchIntervalTests(unittest.TestCase): - """#143: the interpreter switch interval is a tunable, validated at startup.""" - - def setUp(self) -> None: - # setswitchinterval is process-global, so a test that applies one must - # put it back or every later test in this process inherits it. - original = sys.getswitchinterval() - self.addCleanup(sys.setswitchinterval, original) - - def test_unset_leaves_the_interpreter_default_untouched(self) -> None: - before = sys.getswitchinterval() - - self.assertIsNone(resolve_python_switch_interval_seconds(environ={})) - self.assertIsNone(apply_python_switch_interval(environ={})) - - # Absent means "do not call setswitchinterval", not "call it with 5ms". - self.assertEqual(sys.getswitchinterval(), before) - - def test_empty_value_behaves_as_unset(self) -> None: - # compose.yaml passes ${PRISM_PYTHON_SWITCH_INTERVAL_SECONDS:-}, which - # reaches the container as an empty string rather than as an absent - # variable. If that ever stopped reading as unset, every default - # deployment would refuse to start. - before = sys.getswitchinterval() - - self.assertIsNone( - apply_python_switch_interval( - environ={"PRISM_PYTHON_SWITCH_INTERVAL_SECONDS": ""} - ) - ) - - self.assertEqual(sys.getswitchinterval(), before) - - def test_configured_value_is_applied_and_returned(self) -> None: - applied = apply_python_switch_interval( - environ={"PRISM_PYTHON_SWITCH_INTERVAL_SECONDS": "0.001"} - ) - - self.assertEqual(applied, 0.001) - self.assertAlmostEqual(sys.getswitchinterval(), 0.001) - - def test_non_numeric_value_refuses_startup(self) -> None: - with self.assertRaisesRegex( - SystemExit, "PRISM_PYTHON_SWITCH_INTERVAL_SECONDS must be a number" - ): - resolve_python_switch_interval_seconds( - environ={"PRISM_PYTHON_SWITCH_INTERVAL_SECONDS": "fast"} - ) - - def test_non_positive_and_non_finite_values_refuse_startup(self) -> None: - for raw, message in ( - ("0", "must be positive"), - ("-0.005", "must be positive"), - ("nan", "must be finite"), - ("inf", "must be finite"), - ): - with self.subTest(raw=raw), self.assertRaisesRegex(SystemExit, message): - resolve_python_switch_interval_seconds( - environ={"PRISM_PYTHON_SWITCH_INTERVAL_SECONDS": raw} - ) - - def test_sub_microsecond_value_refuses_startup(self) -> None: - # CPython stores the interval in whole microseconds, so anything below - # one truncates to zero -- a switch check at every opportunity, which - # is the opposite of what lowering the value is reaching for. - for raw in ("0.0000009", "1e-07", "1e-09"): - with self.subTest(raw=raw), self.assertRaisesRegex( - SystemExit, "cannot be below .* truncate it to zero" - ): - resolve_python_switch_interval_seconds( - environ={"PRISM_PYTHON_SWITCH_INTERVAL_SECONDS": raw} - ) - - def test_the_floor_is_the_interpreters_actual_resolution(self) -> None: - # Pins the bound to the interpreter rather than to a number we chose: - # the floor must apply intact, and anything below it must truncate to - # zero. If a future CPython changes its resolution, this fails rather - # than letting the bound silently drift out of agreement with it. - floor = MIN_PRISM_PYTHON_SWITCH_INTERVAL_SECONDS - - applied = apply_python_switch_interval( - environ={"PRISM_PYTHON_SWITCH_INTERVAL_SECONDS": repr(floor)} - ) - self.assertEqual(applied, floor) - self.assertEqual(sys.getswitchinterval(), floor) - - sys.setswitchinterval(floor / 2) - self.assertEqual( - sys.getswitchinterval(), - 0.0, - "sub-floor intervals no longer truncate to zero; revisit the bound", - ) - - def test_value_above_the_ceiling_refuses_startup(self) -> None: - # A misplaced decimal point ("5" meaning 5ms) would otherwise let a - # thread hold the GIL for whole seconds, which reads as unexplained - # ack-latency rather than as a configuration error. - with self.assertRaisesRegex( - SystemExit, "PRISM_PYTHON_SWITCH_INTERVAL_SECONDS cannot exceed" - ): - resolve_python_switch_interval_seconds( - environ={"PRISM_PYTHON_SWITCH_INTERVAL_SECONDS": "5"} - ) - - def test_the_ceiling_itself_is_accepted(self) -> None: - ceiling = MAX_PRISM_PYTHON_SWITCH_INTERVAL_SECONDS - - self.assertEqual( - resolve_python_switch_interval_seconds( - environ={"PRISM_PYTHON_SWITCH_INTERVAL_SECONDS": str(ceiling)} - ), - ceiling, - ) - - def test_rejected_values_never_reach_the_interpreter(self) -> None: - before = sys.getswitchinterval() - - with self.assertRaises(SystemExit): - apply_python_switch_interval( - environ={"PRISM_PYTHON_SWITCH_INTERVAL_SECONDS": "5"} - ) - - self.assertEqual(sys.getswitchinterval(), before) - - -class PrismCoordinatorVardiffTests(unittest.TestCase): - def test_decimal_env_rejects_invalid_and_non_finite_values(self) -> None: - with self.assertRaisesRegex(SystemExit, "TEST_DECIMAL must be a decimal number"): - env_decimal("TEST_DECIMAL", "1", environ={"TEST_DECIMAL": "not-decimal"}) - for raw in ("NaN", "sNaN", "Infinity", "-Infinity"): - with self.subTest(raw=raw), self.assertRaisesRegex( - SystemExit, - "TEST_DECIMAL must be finite", - ): - env_decimal("TEST_DECIMAL", "1", environ={"TEST_DECIMAL": raw}) - - def test_send_error_includes_canonical_reason_id_data(self) -> None: - server = coordinator() - sent: list[dict[str, object]] = [] - client = SimpleNamespace(send=lambda payload: sent.append(payload)) - - server.send_error(client, "submit-1", 21, "stale job", reason=PRISM_REJECTION_STALE_JOB) # type: ignore[arg-type] - - self.assertEqual( - sent, - [ - { - "id": "submit-1", - "result": None, - "error": [21, "stale job", {"reason_id": PRISM_REJECTION_STALE_JOB}], - } - ], - ) - - def test_scaled_target_difficulty_uses_pow_limit_units(self) -> None: - pow_limit = target_from_compact("207fffff") - - self.assertEqual(scaled_target_difficulty(pow_limit), 1_000_000) - self.assertEqual(scaled_target_difficulty(pow_limit // 4), 4_000_000) - - def test_qbit_gbt_rules_include_signet_rule_only_for_signet(self) -> None: - self.assertEqual(qbit_gbt_rules("regtest"), ["segwit"]) - self.assertEqual(qbit_gbt_rules("testnet4"), ["segwit"]) - self.assertEqual(qbit_gbt_rules("signet"), ["segwit", "signet"]) - - def test_qbit_template_fingerprint_ignores_clock_only_fields(self) -> None: - base = gbt_template("00" * 32, curtime=1) - base["longpollid"] = "10:0" - base["mintime"] = 1 - clock_only = gbt_template("00" * 32, curtime=2) - clock_only["longpollid"] = "10:1" - clock_only["mintime"] = 2 - changed_value = dict(clock_only) - changed_value["coinbasevalue"] = 49_99999999 - - self.assertEqual(qbit_template_fingerprint(base), qbit_template_fingerprint(clock_only)) - self.assertNotEqual(qbit_template_fingerprint(base), qbit_template_fingerprint(changed_value)) - - def test_resolve_version_mask_uses_gbt_versionrollingmask(self) -> None: - server = PrismCoordinator.__new__(PrismCoordinator) - rpc = TemplateRpc({"versionrollingmask": "1fffe000"}) - server.rpc = rpc - server.qbit_chain = "signet" - - selection = server.resolve_version_rolling_mask(0x000000FF) - - self.assertEqual(selection.selected_mask, 0x1FFFE000) - self.assertEqual(selection.source, "qbit_getblocktemplate") - self.assertEqual(rpc.calls, [("getblocktemplate", [{"rules": ["segwit", "signet"]}])]) - - def test_resolve_version_mask_falls_back_only_when_gbt_missing_or_unavailable(self) -> None: - missing = PrismCoordinator.__new__(PrismCoordinator) - missing.rpc = TemplateRpc({}) - missing.qbit_chain = "regtest" - - missing_selection = missing.resolve_version_rolling_mask(direct_stratum.QBIT_VERSION_ROLLING_MASK) - - self.assertEqual(missing_selection.selected_mask, direct_stratum.QBIT_VERSION_ROLLING_MASK) - self.assertEqual(missing_selection.source, "fallback") - self.assertEqual(missing_selection.detail, "missing_versionrollingmask") - - unavailable = PrismCoordinator.__new__(PrismCoordinator) - unavailable.rpc = FakeRpc() - unavailable.qbit_chain = "regtest" - - unavailable_selection = unavailable.resolve_version_rolling_mask(direct_stratum.QBIT_VERSION_ROLLING_MASK) - - self.assertEqual(unavailable_selection.selected_mask, direct_stratum.QBIT_VERSION_ROLLING_MASK) - self.assertEqual(unavailable_selection.source, "fallback") - self.assertTrue(unavailable_selection.detail.startswith("probe_error:")) - - def test_resolve_version_mask_disables_only_on_gbt_zero_mask(self) -> None: - server = PrismCoordinator.__new__(PrismCoordinator) - server.rpc = TemplateRpc({"versionrollingmask": "00000000"}) - server.qbit_chain = "regtest" - - selection = server.resolve_version_rolling_mask(direct_stratum.QBIT_VERSION_ROLLING_MASK) - - self.assertEqual(selection.selected_mask, 0) - self.assertEqual(selection.source, "qbit_getblocktemplate") - self.assertEqual(selection.detail, "disabled_by_zero_mask") - - def test_resolve_version_mask_rejects_invalid_gbt_mask(self) -> None: - server = PrismCoordinator.__new__(PrismCoordinator) - server.rpc = TemplateRpc({"versionrollingmask": "not-hex"}) - server.qbit_chain = "regtest" - - with self.assertRaisesRegex(SystemExit, "invalid getblocktemplate.versionrollingmask"): - server.resolve_version_rolling_mask(direct_stratum.QBIT_VERSION_ROLLING_MASK) - - def test_configure_negotiates_requested_mask_with_gbt_server_mask(self) -> None: - server = coordinator() - server.version_mask = 0x1FFFE000 - state = client() - captured: dict[str, object] = {} - server.send_result = lambda _client, request_id, result: captured.update( # type: ignore[method-assign] - {"request_id": request_id, "result": result} - ) - - server.handle_configure( - state, - "configure-1", - [ - ["version-rolling"], - {"version-rolling.mask": "0000f000"}, - ], - ) - - self.assertEqual(captured["request_id"], "configure-1") - self.assertEqual( - captured["result"], - { - "version-rolling": True, - "version-rolling.mask": "0000e000", - }, - ) - self.assertEqual(state.version_mask, 0x0000E000) - - def test_configure_disables_version_rolling_when_gbt_mask_is_zero(self) -> None: - server = coordinator() - server.version_mask = 0 - state = client() - captured: dict[str, object] = {} - server.send_result = lambda _client, request_id, result: captured.update( # type: ignore[method-assign] - {"request_id": request_id, "result": result} - ) - - server.handle_configure( - state, - "configure-1", - [ - ["version-rolling"], - {"version-rolling.mask": "ffffffff"}, - ], - ) - - self.assertEqual( - captured["result"], - { - "version-rolling": False, - "version-rolling.mask": "00000000", - }, - ) - self.assertEqual(state.version_mask, 0) - - def test_accepted_share_difficulty_uses_actual_target_unless_overridden(self) -> None: - server = coordinator() - worker = WorkerIdentity( - username="miner-a", - payout_address="miner-a", - worker_name=None, - script_pubkey_hex="5220" + "11" * 32, - p2mr_program_hex="11" * 32, - ) - context = SimpleNamespace( - worker=worker, - job=SimpleNamespace(share_target=target_from_compact("207fffff") // 2), - ) - server.share_weights_by_username = {} - - self.assertEqual(server.accepted_share_difficulty(context), 2_000_000) - - server.share_weights_by_username = {"miner-a": 7} - self.assertEqual(server.accepted_share_difficulty(context), 7) - - def test_resolve_worker_accepts_bare_payout_address(self) -> None: - server = PrismCoordinator.__new__(PrismCoordinator) - rpc = AddressValidationRpc(script_byte="22") - server.rpc = rpc - - worker = server.resolve_worker(PAYOUT_ADDRESS) - - self.assertEqual(rpc.validated, [PAYOUT_ADDRESS]) - self.assertEqual(worker.username, PAYOUT_ADDRESS) - self.assertEqual(worker.payout_address, PAYOUT_ADDRESS) - self.assertIsNone(worker.worker_name) - self.assertEqual(worker.p2mr_program_hex, "22" * 32) - - def test_resolve_worker_accepts_address_worker_username(self) -> None: - server = PrismCoordinator.__new__(PrismCoordinator) - rpc = AddressValidationRpc(script_byte="33") - server.rpc = rpc - - worker = server.resolve_worker(f"{PAYOUT_ADDRESS}.rig-a") - - self.assertEqual(rpc.validated, [PAYOUT_ADDRESS]) - self.assertEqual(worker.username, f"{PAYOUT_ADDRESS}.rig-a") - self.assertEqual(worker.payout_address, PAYOUT_ADDRESS) - self.assertEqual(worker.worker_name, "rig-a") - self.assertEqual(worker.p2mr_program_hex, "33" * 32) - - def test_resolve_worker_caches_successful_address_validation(self) -> None: - server = PrismCoordinator.__new__(PrismCoordinator) - rpc = AddressValidationRpc(script_byte="33") - server.rpc = rpc - - first = server.resolve_worker(f"{PAYOUT_ADDRESS}.rig-a") - second = server.resolve_worker(f"{PAYOUT_ADDRESS}.rig-b") - - self.assertEqual(rpc.validated, [PAYOUT_ADDRESS]) - self.assertEqual(first.p2mr_program_hex, second.p2mr_program_hex) - - def test_payout_address_cache_evicts_least_recently_used_entry(self) -> None: - server = PrismCoordinator.__new__(PrismCoordinator) - server.payout_address_cache_max_entries = 2 - server.payout_address_cache_ttl_seconds = 60 - - class AnyAddressRpc: - def __init__(self) -> None: - self.validated: list[str] = [] - - def call(self, method: str, params: list[object] | None = None) -> object: - address = str((params or [""])[0]) - self.validated.append(address) - return {"isvalid": True, "scriptPubKey": "5220" + "33" * 32} - - rpc = AnyAddressRpc() - server.rpc = rpc - - server.validate_p2mr_address("address-a", label="test") - server.validate_p2mr_address("address-b", label="test") - server.validate_p2mr_address("address-a", label="test") - server.validate_p2mr_address("address-c", label="test") - - self.assertEqual(rpc.validated, ["address-a", "address-b", "address-c"]) - self.assertEqual(list(server._p2mr_address_cache), ["address-a", "address-c"]) - - server.validate_p2mr_address("address-b", label="test") - self.assertEqual(rpc.validated[-1], "address-b") - self.assertEqual(len(server._p2mr_address_cache), 2) - - def test_payout_address_cache_revalidates_expired_entry(self) -> None: - server = PrismCoordinator.__new__(PrismCoordinator) - server.payout_address_cache_max_entries = 2 - server.payout_address_cache_ttl_seconds = 5 - rpc = AddressValidationRpc(script_byte="33") - server.rpc = rpc - now = 100.0 - - with patch( - "lab.prism.prism_coordinator.time.monotonic", - side_effect=lambda: now, - ): - server.validate_p2mr_address(PAYOUT_ADDRESS, label="test") - now = 104.0 - server.validate_p2mr_address(PAYOUT_ADDRESS, label="test") - now = 106.0 - server.validate_p2mr_address(PAYOUT_ADDRESS, label="test") - - self.assertEqual(rpc.validated, [PAYOUT_ADDRESS, PAYOUT_ADDRESS]) - - def test_concurrent_worker_resolution_singleflights_address_validation(self) -> None: - server = PrismCoordinator.__new__(PrismCoordinator) - entered = threading.Event() - release = threading.Event() - - class BlockingAddressRpc(AddressValidationRpc): - def call(self, method: str, params: list[object] | None = None) -> object: - if method == "validateaddress": - entered.set() - if not release.wait(timeout=5): - raise TimeoutError("test did not release validateaddress") - return super().call(method, params) - - rpc = BlockingAddressRpc(script_byte="33") - server.rpc = rpc - server._ensure_p2mr_address_cache_state() - workers: list[WorkerIdentity] = [] - errors: list[BaseException] = [] - - def resolve(index: int) -> None: - try: - workers.append(server.resolve_worker(f"{PAYOUT_ADDRESS}.rig-{index}")) - except BaseException as exc: # pragma: no cover - surfaced below - errors.append(exc) - - threads = [threading.Thread(target=resolve, args=(index,)) for index in range(8)] - for thread in threads: - thread.start() - self.assertTrue(entered.wait(timeout=5)) - release.set() - for thread in threads: - thread.join(timeout=5) - - self.assertFalse(errors) - self.assertEqual(len(workers), 8) - self.assertEqual(rpc.validated, [PAYOUT_ADDRESS]) - - def test_concurrent_failed_worker_resolution_shares_singleflight_error(self) -> None: - server = PrismCoordinator.__new__(PrismCoordinator) - entered = threading.Event() - release = threading.Event() - - class FailingAddressRpc: - def __init__(self) -> None: - self.calls = 0 - - def call(self, method: str, params: list[object] | None = None) -> object: - self.calls += 1 - entered.set() - if not release.wait(timeout=5): - raise TimeoutError("test did not release validateaddress") - raise RuntimeError("qbitd unavailable") - - rpc = FailingAddressRpc() - server.rpc = rpc - server._ensure_p2mr_address_cache_state() - errors: list[BaseException] = [] - - def resolve() -> None: - try: - server.resolve_worker(PAYOUT_ADDRESS) - except BaseException as exc: - errors.append(exc) - - threads = [threading.Thread(target=resolve) for _ in range(8)] - for thread in threads: - thread.start() - self.assertTrue(entered.wait(timeout=5)) - deadline = time.monotonic() + 5 - while time.monotonic() < deadline: - with server._p2mr_address_cache_lock: - pending = server._p2mr_address_validation_inflight[PAYOUT_ADDRESS] - if pending.waiters == 7: - break - time.sleep(0.001) - self.assertEqual(pending.waiters, 7) - release.set() - for thread in threads: - thread.join(timeout=5) - - self.assertEqual(rpc.calls, 1) - self.assertEqual(len(errors), 8) - self.assertTrue(all("qbitd unavailable" in str(exc) for exc in errors)) - - def test_resolve_worker_rejects_invalid_base_address_with_worker_suffix(self) -> None: - server = PrismCoordinator.__new__(PrismCoordinator) - rpc = AddressValidationRpc() - server.rpc = rpc - server.username_fallback_address = None - - with self.assertRaises(StratumError) as raised: - server.resolve_worker("not-a-qbit-address.rig-a") - - self.assertEqual(raised.exception.code, 20) - self.assertEqual(rpc.validated, ["not-a-qbit-address"]) - - def test_resolve_worker_uses_configured_fallback_for_invalid_username(self) -> None: - fallback_address = "tq1fallback" - server = PrismCoordinator.__new__(PrismCoordinator) - rpc = AddressValidationRpc(valid_address=fallback_address, script_byte="44") - server.rpc = rpc - server.username_fallback_address = fallback_address - - worker = server.resolve_worker("not-a-qbit-address.rig-a") - - self.assertEqual(rpc.validated, ["not-a-qbit-address", fallback_address]) - self.assertEqual(worker.username, "not-a-qbit-address.rig-a") - self.assertEqual(worker.payout_address, fallback_address) - self.assertEqual(worker.worker_name, "rig-a") - self.assertEqual(worker.p2mr_program_hex, "44" * 32) - - def test_resolve_worker_uses_testnet_default_fallback_for_invalid_username(self) -> None: - server = PrismCoordinator.__new__(PrismCoordinator) - rpc = AddressValidationRpc(valid_address=DEFAULT_TESTNET_USERNAME_FALLBACK_ADDRESS, script_byte="55") - server.rpc = rpc - - with patch.dict(os.environ, {"QBIT_CHAIN": "testnet4"}, clear=True): - worker = server.resolve_worker("not-a-qbit-address") - - self.assertEqual(rpc.validated, ["not-a-qbit-address", DEFAULT_TESTNET_USERNAME_FALLBACK_ADDRESS]) - self.assertEqual(worker.username, "not-a-qbit-address") - self.assertEqual(worker.payout_address, DEFAULT_TESTNET_USERNAME_FALLBACK_ADDRESS) - self.assertIsNone(worker.worker_name) - self.assertEqual(worker.p2mr_program_hex, "55" * 32) - - def test_default_username_fallback_is_testnet_only_unless_configured(self) -> None: - with patch.dict(os.environ, {"QBIT_CHAIN": "testnet4"}, clear=True): - self.assertEqual(default_prism_username_fallback_address(), DEFAULT_TESTNET_USERNAME_FALLBACK_ADDRESS) - with patch.dict(os.environ, {"QBIT_CHAIN": "regtest"}, clear=True): - self.assertIsNone(default_prism_username_fallback_address()) - with patch.dict( - os.environ, - {"QBIT_CHAIN": "regtest", "PRISM_USERNAME_FALLBACK_ADDRESS": "qbrt1fallback"}, - clear=True, - ): - self.assertEqual(default_prism_username_fallback_address(), "qbrt1fallback") - - def test_prism_payout_policy_defaults_to_no_pool_fee(self) -> None: - server = coordinator() - - with patch.dict(os.environ, {}, clear=True): - policy = server.prism_payout_policy() - - self.assertEqual( - policy, - { - "p2mr_spend_input_bytes": 3_680, - "target_feerate_sats_per_byte": 1, - "safety_multiplier": 4, - }, - ) - - def test_prism_payout_policy_allows_fixed_min_output_bits_override(self) -> None: - server = coordinator() - - with patch.dict(os.environ, {"PRISM_PAYOUT_MIN_OUTPUT_BITS": "10000"}, clear=True): - policy = server.prism_payout_policy() - - self.assertEqual( - policy, - { - "p2mr_spend_input_bytes": 3_680, - "target_feerate_sats_per_byte": 1, - "safety_multiplier": 4, - "min_output_sats": 10_000, - }, - ) - - def test_prism_payout_policy_falls_back_to_legacy_min_output_sats_override(self) -> None: - server = coordinator() - - with patch.dict(os.environ, {"PRISM_PAYOUT_MIN_OUTPUT_SATS": "10000"}, clear=True): - policy = server.prism_payout_policy() - - self.assertEqual(policy["min_output_sats"], 10_000) - - def test_prism_payout_policy_min_output_bits_overrides_legacy_sats_override(self) -> None: - server = coordinator() - - with patch.dict( - os.environ, - { - "PRISM_PAYOUT_MIN_OUTPUT_BITS": "11000", - "PRISM_PAYOUT_MIN_OUTPUT_SATS": "10000", - }, - clear=True, - ): - policy = server.prism_payout_policy() - - self.assertEqual(policy["min_output_sats"], 11_000) - - def test_prism_coinbase_tag_defaults_to_prism(self) -> None: - with patch.dict(os.environ, {}, clear=True): - self.assertEqual(default_prism_coinbase_tag_hex(), "/PRISM/".encode("ascii").hex()) - - def test_prism_coinbase_tag_is_configurable_and_can_be_disabled(self) -> None: - with patch.dict(os.environ, {"PRISM_COINBASE_TAG": "/CUSTOM/"}, clear=True): - self.assertEqual(default_prism_coinbase_tag_hex(), "/CUSTOM/".encode("ascii").hex()) - with patch.dict(os.environ, {"PRISM_COINBASE_TAG": ""}, clear=True): - self.assertEqual(default_prism_coinbase_tag_hex(), "") - - def test_prism_coinbase_tag_rejects_non_printable_or_long_values(self) -> None: - for tag, message in ( - ("PRISM\n", "printable ASCII"), - ("P" * 41, "at most 40 bytes"), - ("PRISM-π", "ASCII"), - ): - with self.subTest(tag=tag), patch.dict( - os.environ, {"PRISM_COINBASE_TAG": tag}, clear=True - ): - with self.assertRaisesRegex(SystemExit, message): - default_prism_coinbase_tag_hex() - - def test_coinbase_script_sig_suffix_places_pool_tag_before_extranonce(self) -> None: - server = coordinator() - server.coinbase_tag_hex = "/PRISM/".encode("ascii").hex() - - suffix = server.coinbase_script_sig_suffix_hex("aabbccdd", "00" * 8) - - self.assertEqual(suffix, "/PRISM/".encode("ascii").hex() + "aabbccdd" + "00" * 8) - self.assertTrue(suffix.endswith("aabbccdd" + "00" * 8)) - - def test_prism_payout_policy_allows_formula_overrides(self) -> None: - server = coordinator() - - with patch.dict( - os.environ, - { - "PRISM_PAYOUT_P2MR_SPEND_INPUT_BYTES": "2500", - "PRISM_PAYOUT_TARGET_FEERATE_BITS_PER_BYTE": "2", - "PRISM_PAYOUT_SAFETY_MULTIPLIER": "3", - }, - clear=True, - ): - policy = server.prism_payout_policy() - - self.assertEqual( - policy, - { - "p2mr_spend_input_bytes": 2_500, - "target_feerate_sats_per_byte": 2, - "safety_multiplier": 3, - }, - ) - - def test_prism_payout_policy_formula_uses_legacy_feerate_alias(self) -> None: - server = coordinator() - - with patch.dict(os.environ, {"PRISM_PAYOUT_TARGET_FEERATE_SATS_PER_BYTE": "2"}, clear=True): - policy = server.prism_payout_policy() - - self.assertEqual(policy["target_feerate_sats_per_byte"], 2) - - def test_prism_payout_policy_formula_bits_feerate_overrides_legacy_alias(self) -> None: - server = coordinator() - - with patch.dict( - os.environ, - { - "PRISM_PAYOUT_TARGET_FEERATE_BITS_PER_BYTE": "3", - "PRISM_PAYOUT_TARGET_FEERATE_SATS_PER_BYTE": "2", - }, - clear=True, - ): - policy = server.prism_payout_policy() - - self.assertEqual(policy["target_feerate_sats_per_byte"], 3) - - def test_prism_payout_policy_rejects_invalid_floor_settings(self) -> None: - cases = [ - ({"PRISM_PAYOUT_MIN_OUTPUT_BITS": "0"}, "PRISM_PAYOUT_MIN_OUTPUT_BITS must be positive"), - ( - {"PRISM_PAYOUT_MIN_OUTPUT_BITS": "not-int"}, - "PRISM_PAYOUT_MIN_OUTPUT_BITS must be an integer", - ), - ({"PRISM_PAYOUT_MIN_OUTPUT_SATS": "0"}, "PRISM_PAYOUT_MIN_OUTPUT_SATS must be positive"), - ( - {"PRISM_PAYOUT_MIN_OUTPUT_SATS": "not-int"}, - "PRISM_PAYOUT_MIN_OUTPUT_SATS must be an integer", - ), - ( - {"PRISM_PAYOUT_SAFETY_MULTIPLIER": "0"}, - "PRISM_PAYOUT_SAFETY_MULTIPLIER must be positive", - ), - ( - {"PRISM_PAYOUT_TARGET_FEERATE_BITS_PER_BYTE": "0"}, - "PRISM_PAYOUT_TARGET_FEERATE_BITS_PER_BYTE must be positive", - ), - ] - for env_vars, expected in cases: - with self.subTest(env_vars=env_vars), patch.dict(os.environ, env_vars, clear=True): - server = coordinator() - with self.assertRaisesRegex(SystemExit, expected): - server.prism_payout_policy() - - def test_prism_pool_fee_address_config_validates_p2mr_address(self) -> None: - server = coordinator() - server.rpc = AddressRpc(valid_address="tq1fee", script_byte="88") - - with patch.dict( - os.environ, - { - "PRISM_POOL_FEE_ENABLED": "1", - "PRISM_POOL_FEE_BPS": "125", - "PRISM_POOL_FEE_ADDRESS": "tq1fee", - }, - clear=True, - ): - policy = server.prism_payout_policy() - - self.assertEqual(server.rpc.validated, ["tq1fee"]) - self.assertEqual( - policy["pool_fee_policy"], - { - "fee_bps": 125, - "recipient_id": "tq1fee", - "order_key": "tq1fee", - "p2mr_program_hex": "88" * 32, - }, - ) - - def test_prism_pool_fee_enabled_allows_zero_bps_policy(self) -> None: - server = coordinator() - server.rpc = AddressRpc(valid_address="tq1fee", script_byte="66") - - with patch.dict( - os.environ, - { - "PRISM_POOL_FEE_ENABLED": "1", - "PRISM_POOL_FEE_BPS": "0", - "PRISM_POOL_FEE_ADDRESS": "tq1fee", - }, - clear=True, - ): - policy = server.prism_payout_policy() - - self.assertEqual(policy["pool_fee_policy"]["fee_bps"], 0) - self.assertEqual(policy["pool_fee_policy"]["p2mr_program_hex"], "66" * 32) - - def test_prism_pool_fee_program_config_requires_recipient_identity(self) -> None: - server = coordinator() - - with patch.dict( - os.environ, - { - "PRISM_POOL_FEE_ENABLED": "1", - "PRISM_POOL_FEE_BPS": "125", - "PRISM_POOL_FEE_P2MR_PROGRAM_HEX": "55" * 32, - }, - clear=True, - ): - with self.assertRaisesRegex(SystemExit, "PRISM_POOL_FEE_RECIPIENT_ID"): - server.prism_payout_policy() - - def test_prism_pool_fee_program_config_uses_explicit_order_key(self) -> None: - server = coordinator() - - with patch.dict( - os.environ, - { - "PRISM_POOL_FEE_ENABLED": "1", - "PRISM_POOL_FEE_BPS": "125", - "PRISM_POOL_FEE_P2MR_PROGRAM_HEX": "55" * 32, - "PRISM_POOL_FEE_RECIPIENT_ID": "pool-fee", - "PRISM_POOL_FEE_ORDER_KEY": "000-pool-fee", - }, - clear=True, - ): - policy = server.prism_payout_policy() - - self.assertEqual( - policy["pool_fee_policy"], - { - "fee_bps": 125, - "recipient_id": "pool-fee", - "order_key": "000-pool-fee", - "p2mr_program_hex": "55" * 32, - }, - ) - - def test_prism_pool_fee_config_rejects_ambiguous_or_invalid_settings(self) -> None: - cases = [ - ( - {"PRISM_POOL_FEE_ENABLED": "1", "PRISM_POOL_FEE_ADDRESS": "tq1fee"}, - "PRISM_POOL_FEE_BPS", - ), - ( - { - "PRISM_POOL_FEE_ENABLED": "1", - "PRISM_POOL_FEE_BPS": "10001", - "PRISM_POOL_FEE_ADDRESS": "tq1fee", - }, - "between 0 and 10000", - ), - ( - { - "PRISM_POOL_FEE_ENABLED": "1", - "PRISM_POOL_FEE_BPS": "125", - "PRISM_POOL_FEE_ADDRESS": "tq1fee", - "PRISM_POOL_FEE_P2MR_PROGRAM_HEX": "55" * 32, - }, - "exactly one", - ), - ] - for env_vars, expected in cases: - with self.subTest(env_vars=env_vars), patch.dict(os.environ, env_vars, clear=True): - server = coordinator() - server.rpc = AddressRpc(valid_address="tq1fee") - with self.assertRaisesRegex(SystemExit, expected): - server.prism_payout_policy() - - def test_prism_pool_fee_config_rejects_disabled_fee_settings(self) -> None: - cases = [ - {"PRISM_POOL_FEE_BPS": "125"}, - {"PRISM_POOL_FEE_ADDRESS": "tq1fee"}, - {"PRISM_POOL_FEE_P2MR_PROGRAM_HEX": "55" * 32}, - {"PRISM_POOL_FEE_RECIPIENT_ID": "pool-fee"}, - { - "PRISM_POOL_FEE_ENABLED": "0", - "PRISM_POOL_FEE_BPS": "125", - "PRISM_POOL_FEE_ADDRESS": "tq1fee", - }, - ] - for env_vars in cases: - with self.subTest(env_vars=env_vars), patch.dict(os.environ, env_vars, clear=True): - server = coordinator() - server.rpc = AddressRpc(valid_address="tq1fee") - with self.assertRaisesRegex(SystemExit, "PRISM_POOL_FEE_ENABLED=1"): - server.prism_payout_policy() - - def test_prism_pool_fee_config_rejects_non_p2mr_fee_address(self) -> None: - server = coordinator() - server.rpc = AddressRpc(valid_address="tq1fee", p2mr=False) - - with patch.dict( - os.environ, - { - "PRISM_POOL_FEE_ENABLED": "1", - "PRISM_POOL_FEE_BPS": "125", - "PRISM_POOL_FEE_ADDRESS": "tq1fee", - }, - clear=True, - ): - with self.assertRaisesRegex(SystemExit, "P2MR"): - server.prism_payout_policy() - - def test_prism_ctv_settlement_config_uses_legacy_unit_aliases(self) -> None: - server = coordinator() - - with patch.dict( - os.environ, - { - "PRISM_CTV_SETTLEMENT_ENABLED": "1", - "PRISM_DIRECT_COINBASE_PAYOUT_FLOOR_SATS": "10485760", - "PRISM_CTV_FANOUT_FEE_MARKET_RATE_SATS_PER_1000_WEIGHT": "25", - }, - clear=True, - ): - config = server.prism_ctv_settlement_config() - - self.assertIsNotNone(config) - assert config is not None - self.assertEqual(config["direct_floor_sats"], 10_485_760) - self.assertEqual( - config["fanout_fee_rate_policy"], - {"market_fee_rate_sats_per_1000_weight": 25, "premium_bps": 12_000}, - ) - - def test_prism_ctv_settlement_config_uses_node_fee_estimate_by_default(self) -> None: - server = coordinator() - rpc = FeeEstimateRpc({"feerate": "0.00001001"}) - server.rpc = rpc - - with patch.dict(os.environ, {"PRISM_CTV_SETTLEMENT_ENABLED": "1"}, clear=True): - config = server.prism_ctv_settlement_config() - - self.assertIsNotNone(config) - assert config is not None - self.assertEqual( - config["fanout_fee_rate_policy"], - {"market_fee_rate_sats_per_1000_weight": 1001, "premium_bps": 12_000}, - ) - self.assertIn(("estimatesmartfee", [2]), rpc.calls) - - def test_prism_ctv_settlement_config_caches_node_fee_estimate_per_block_height(self) -> None: - server = coordinator() - rpc = FeeEstimateRpc({"feerate": "0.00001001"}) - server.rpc = rpc - - with patch.dict(os.environ, {"PRISM_CTV_SETTLEMENT_ENABLED": "1"}, clear=True): - first = server.prism_ctv_settlement_config(block_height=10) - rpc.estimate = {"feerate": "0.00002000"} - second = server.prism_ctv_settlement_config(block_height=10) - next_height = server.prism_ctv_settlement_config(block_height=11) - - assert first is not None - assert second is not None - assert next_height is not None - self.assertEqual( - first["fanout_fee_rate_policy"], - {"market_fee_rate_sats_per_1000_weight": 1001, "premium_bps": 12_000}, - ) - self.assertEqual(second["fanout_fee_rate_policy"], first["fanout_fee_rate_policy"]) - self.assertEqual( - next_height["fanout_fee_rate_policy"], - {"market_fee_rate_sats_per_1000_weight": 2000, "premium_bps": 12_000}, - ) - self.assertEqual( - [call for call in rpc.calls if call[0] == "estimatesmartfee"], - [("estimatesmartfee", [2]), ("estimatesmartfee", [2])], - ) - - def test_prism_ctv_settlement_config_separates_fee_cache_by_parent_hash(self) -> None: - server = coordinator() - rpc = FeeEstimateRpc({"feerate": "0.00001001"}) - server.rpc = rpc - - with patch.dict(os.environ, {"PRISM_CTV_SETTLEMENT_ENABLED": "1"}, clear=True): - first = server.prism_ctv_settlement_config(block_height=10, parent_hash="aa" * 32) - rpc.estimate = {"feerate": "0.00002000"} - same_parent = server.prism_ctv_settlement_config(block_height=10, parent_hash="aa" * 32) - reorg_parent = server.prism_ctv_settlement_config(block_height=10, parent_hash="bb" * 32) - - assert first is not None - assert same_parent is not None - assert reorg_parent is not None - self.assertEqual(same_parent["fanout_fee_rate_policy"], first["fanout_fee_rate_policy"]) - self.assertEqual( - reorg_parent["fanout_fee_rate_policy"], - {"market_fee_rate_sats_per_1000_weight": 2000, "premium_bps": 12_000}, - ) - self.assertEqual( - [call for call in rpc.calls if call[0] == "estimatesmartfee"], - [("estimatesmartfee", [2]), ("estimatesmartfee", [2])], - ) - - def test_prism_ctv_settlement_config_fails_closed_when_fee_estimate_unavailable(self) -> None: - server = coordinator() - server.rpc = FeeEstimateRpc({"errors": ["insufficient data"]}) - - with patch.dict(os.environ, {"PRISM_CTV_SETTLEMENT_ENABLED": "1"}, clear=True): - with self.assertRaisesRegex(RuntimeError, "unable to compute PRISM CTV fanout fee rate"): - server.prism_ctv_settlement_config() - - def test_prism_ctv_settlement_config_retries_after_fee_estimate_failure(self) -> None: - server = coordinator() - rpc = FeeEstimateRpc({"errors": ["insufficient data"]}) - server.rpc = rpc - - with patch.dict(os.environ, {"PRISM_CTV_SETTLEMENT_ENABLED": "1"}, clear=True): - with self.assertRaises(RuntimeError): - server.prism_ctv_settlement_config(block_height=10) - rpc.estimate = {"feerate": "0.00002000"} - recovered = server.prism_ctv_settlement_config(block_height=10) - - assert recovered is not None - self.assertEqual( - recovered["fanout_fee_rate_policy"], - {"market_fee_rate_sats_per_1000_weight": 2000, "premium_bps": 12_000}, - ) - self.assertEqual( - [call for call in rpc.calls if call[0] == "estimatesmartfee"], - [("estimatesmartfee", [2]), ("estimatesmartfee", [2])], - ) - - def test_ctv_broadcaster_daemon_uses_coordinator_ledger_and_config(self) -> None: - server = coordinator() - server.ctv_broadcaster_wallet = None - server.ctv_broadcaster_fee_sats = 0 - server.ctv_broadcaster_limit = 7 - captured: dict[str, object] = {} - - class FakeDaemon: - def __init__(self, ledger: object, broadcaster: object, *, fee_sats: int) -> None: - captured["ledger"] = ledger - captured["broadcaster"] = broadcaster - captured["fee_sats"] = fee_sats - - def run_once( - self, - *, - limit: int, - progress_callback: object, - chunk_size: int, - tip_refresh_pending: object, - chunk_callback: object, - ) -> object: - captured["limit"] = limit - captured["chunk_size"] = chunk_size - captured["tip_refresh_pending"] = tip_refresh_pending - captured["chunk_callback"] = chunk_callback - return SimpleNamespace( - scanned_count=1, - submitted_count=0, - updated_count=1, - failed_count=0, - yielded_to_tip_refresh=False, - ) - - with patch("lab.prism.prism_coordinator.CtvFanoutBroadcastDaemon", FakeDaemon): - result = server.run_ctv_fanout_broadcaster_once() - - self.assertIs(captured["ledger"], server.ledger) - self.assertEqual(captured["fee_sats"], 0) - self.assertEqual(captured["limit"], 7) - self.assertEqual(captured["chunk_size"], 5) - self.assertTrue(callable(captured["tip_refresh_pending"])) - self.assertTrue(callable(captured["chunk_callback"])) - self.assertEqual(result.updated_count, 1) - self.assertIsNotNone(captured["broadcaster"]) - - def test_ctv_broadcaster_daemon_requires_wallet_for_cpfp_fee(self) -> None: - server = coordinator() - server.ctv_broadcaster_wallet = None - server.ctv_broadcaster_fee_sats = 1 - - with self.assertRaisesRegex(ValueError, "ctv_broadcaster_wallet is required"): - server.make_ctv_fanout_broadcast_daemon() - - def test_make_ledger_requires_explicit_memory_opt_in(self) -> None: - server = PrismCoordinator.__new__(PrismCoordinator) - - with patch.dict(os.environ, {}, clear=True): - with self.assertRaises(SystemExit): - server.make_ledger() - - with patch.dict(os.environ, {"PRISM_ALLOW_MEMORY_LEDGER": "1"}, clear=True): - ledger = server.make_ledger() - - self.assertEqual(ledger.backend_name, "memory") - - def test_trusted_ledger_key_must_be_configured_or_explicitly_test_mode(self) -> None: - server = PrismCoordinator.__new__(PrismCoordinator) - - with patch.dict(os.environ, {}, clear=True): - with self.assertRaises(SystemExit): - server.load_trusted_ledger_writer_public_key() - - with patch.dict(os.environ, {"PRISM_LEDGER_WRITER_PUBLIC_KEY_HEX": "aa" * 32}, clear=True): - self.assertEqual(server.load_trusted_ledger_writer_public_key(), "aa" * 32) - - with patch.dict(os.environ, {"PRISM_ALLOW_BUNDLE_EMBEDDED_LEDGER_KEY": "1"}, clear=True): - self.assertIsNone(server.load_trusted_ledger_writer_public_key()) - - def test_fixed_ledger_session_token_requires_explicit_opt_in(self) -> None: - server = PrismCoordinator.__new__(PrismCoordinator) - env = { - "PRISM_POSTGRES_PSQL_COMMAND": "psql postgresql://example.invalid/qbit", - "PRISM_LEDGER_WRITER_SESSION_TOKEN": "fixed-session", - "PRISM_PUBLIC_REWARD_WINDOW_CACHE_SECONDS": "47", - } - - with patch.dict(os.environ, env, clear=True): - with self.assertRaises(SystemExit): - server.make_ledger() - - with patch.dict(os.environ, {**env, "PRISM_ALLOW_FIXED_LEDGER_SESSION_TOKEN": "1"}, clear=True): - with patch("lab.prism.prism_coordinator.PsqlShareLedger") as fake_ledger: - fake_ledger.return_value = SimpleNamespace(backend_name="postgres-psql") - ledger = server.make_ledger() - - self.assertEqual(ledger.backend_name, "postgres-psql") - self.assertEqual(fake_ledger.call_args.kwargs["writer_session_token"], "fixed-session") - self.assertEqual( - fake_ledger.call_args.kwargs["reward_window_cache_seconds"], - 47.0, - ) - - def test_same_tip_retention_requires_connection_derived_production_bound(self) -> None: - with self.assertRaisesRegex( - SystemExit, - "PRISM_STRATUM_SAME_TIP_JOB_RETENTION_PER_CONNECTION", - ): - validate_same_tip_job_retention_limits( - retention_seconds=30, - per_connection=0, - max_connections=0, - production=False, - ) - with self.assertRaisesRegex(SystemExit, "PRISM_STRATUM_MAX_CONNECTIONS"): - validate_same_tip_job_retention_limits( - retention_seconds=30, - per_connection=64, - max_connections=0, - production=True, - ) - - validate_same_tip_job_retention_limits( - retention_seconds=30, - per_connection=64, - max_connections=1_900, - production=True, - ) - validate_same_tip_job_retention_limits( - retention_seconds=30, - per_connection=64, - max_connections=0, - production=False, - ) - validate_same_tip_job_retention_limits( - retention_seconds=0, - per_connection=0, - max_connections=0, - production=True, - ) - - def test_production_gate_rejects_prism_test_bypasses_without_capacity_evidence(self) -> None: - base = { - "QBIT_PRODUCTION": "1", - "QBIT_RPC_USER": "qbitrpc", - "QBIT_RPC_PASSWORD": "not-default", - "PRISM_POSTGRES_PSQL_COMMAND": "psql postgresql://example.invalid/qbit", - "PRISM_POSTGRES_PASSWORD": "not-default", - "PRISM_MANIFEST_SIGNING_SEED_HEX": "42" * 32, - "PRISM_LEDGER_ATTESTATION_SIGNING_SEED_HEX": "43" * 32, - "PRISM_LEDGER_WRITER_PUBLIC_KEY_HEX": "44" * 32, - "PRISM_LEDGER_WRITER_ID": "managed-writer", - "PRISM_LEDGER_WRITER_EPOCH": "7", - "PRISM_AUDIT_DIR": "/var/lib/qbit/prism/audit", - "PRISM_EVIDENCE_PATH": "/var/lib/qbit/prism/evidence.json", - "PRISM_STRATUM_STALE_GRACE_SECONDS": "3", - "PRISM_STRATUM_SHARE_DIFF": "1024", - "PRISM_STRATUM_VARDIFF_MIN_DIFF": "1024", - "PRISM_STRATUM_VARDIFF_START_DIFF": "4096", - "PRISM_STRATUM_VARDIFF_MAX_DIFF": "65536", - "PRISM_STRATUM_MAX_CONNECTIONS": "1900", - } - for name in ( - "PRISM_ALLOW_MEMORY_LEDGER", - "PRISM_ALLOW_TEST_SIGNING_SEEDS", - "PRISM_ALLOW_BUNDLE_EMBEDDED_LEDGER_KEY", - "PRISM_ALLOW_FIXED_LEDGER_SESSION_TOKEN", - ): - with self.subTest(name=name), patch.dict(os.environ, {**base, name: "1"}, clear=True): - with self.assertRaisesRegex(SystemExit, name): - validate_prism_production_gate() - - with patch.dict(os.environ, base, clear=True): - validate_prism_production_gate() - - with patch.dict( - os.environ, - {**base, "PRISM_STRATUM_MAX_CONNECTIONS": "0"}, - clear=True, - ): - with self.assertRaisesRegex(SystemExit, "PRISM_STRATUM_MAX_CONNECTIONS"): - validate_prism_production_gate() - - with patch.dict( - os.environ, - {**base, "PRISM_STRATUM_INITIAL_JOB_TIMEOUT_SECONDS": "0"}, - clear=True, - ): - with self.assertRaisesRegex( - SystemExit, - "PRISM_STRATUM_INITIAL_JOB_TIMEOUT_SECONDS", - ): - validate_prism_production_gate() - - with patch.dict( - os.environ, - {**base, "PRISM_STRATUM_MAX_PENDING_INITIAL_JOBS": "0"}, - clear=True, - ): - with self.assertRaisesRegex( - SystemExit, - "PRISM_STRATUM_MAX_PENDING_INITIAL_JOBS", - ): - validate_prism_production_gate() - - # Mainnet follows the same bounded-grace rule as every other chain: - # zero grace rejects every in-flight prior-tip share at each block, - # which miners read as pool failure. - with patch.dict( - os.environ, - {**base, "QBIT_CHAIN": "mainnet", "PRISM_STRATUM_STALE_GRACE_SECONDS": "3"}, - clear=True, - ): - validate_prism_production_gate() - - # Off mainnet, production mode accepts a bounded grace window too. - with patch.dict( - os.environ, - {**base, "PRISM_STRATUM_STALE_GRACE_SECONDS": "2"}, - clear=True, - ): - validate_prism_production_gate() - - with patch.dict(os.environ, {**base, "PRISM_POSTGRES_PASSWORD": "change-this"}, clear=True): - with self.assertRaisesRegex(SystemExit, "PRISM_POSTGRES_PASSWORD"): - validate_prism_production_gate() - - with patch.dict( - os.environ, - { - **base, - "PRISM_POSTGRES_PASSWORD": "not-default", - "PRISM_POSTGRES_PSQL_COMMAND": "", - "PRISM_DATABASE_URL": "postgresql://qbit:change-this@prism-postgres:5432/qbit", - }, - clear=True, - ): - with self.assertRaisesRegex(SystemExit, "PRISM_DATABASE_URL"): - validate_prism_production_gate() - - def test_production_gate_rejects_unsafe_difficulty_without_capacity_gate(self) -> None: - base = { - "QBIT_PRODUCTION": "1", - "QBIT_RPC_USER": "qbitrpc", - "QBIT_RPC_PASSWORD": "not-default", - "PRISM_POSTGRES_PSQL_COMMAND": "psql postgresql://example.invalid/qbit", - "PRISM_POSTGRES_PASSWORD": "not-default", - "PRISM_MANIFEST_SIGNING_SEED_HEX": "42" * 32, - "PRISM_LEDGER_ATTESTATION_SIGNING_SEED_HEX": "43" * 32, - "PRISM_LEDGER_WRITER_PUBLIC_KEY_HEX": "44" * 32, - "PRISM_LEDGER_WRITER_ID": "managed-writer", - "PRISM_LEDGER_WRITER_EPOCH": "7", - "PRISM_AUDIT_DIR": "/var/lib/qbit/prism/audit", - "PRISM_EVIDENCE_PATH": "/var/lib/qbit/prism/evidence.json", - "PRISM_STRATUM_STALE_GRACE_SECONDS": "3", - "PRISM_STRATUM_SHARE_DIFF": "1024", - "PRISM_STRATUM_VARDIFF_MIN_DIFF": "1024", - "PRISM_STRATUM_VARDIFF_START_DIFF": "4096", - "PRISM_STRATUM_VARDIFF_MAX_DIFF": "65536", - } - cases = ( - ({"PRISM_STRATUM_SHARE_DIFF": ""}, "requires PRISM_STRATUM_SHARE_DIFF"), - ({"PRISM_STRATUM_SHARE_DIFF": "not-a-decimal"}, "must be a decimal number"), - ({"PRISM_STRATUM_SHARE_DIFF": "NaN"}, "PRISM_STRATUM_SHARE_DIFF must be positive"), - ({"PRISM_STRATUM_SHARE_DIFF": "0"}, "PRISM_STRATUM_SHARE_DIFF must be positive"), - ({"PRISM_STRATUM_SHARE_DIFF": "1e-9"}, "lab-only 1e-9 difficulty"), - ({"PRISM_STRATUM_VARDIFF_MIN_DIFF": "8192"}, "minimum exceeds its start"), - ({"PRISM_STRATUM_VARDIFF_START_DIFF": "131072"}, "start exceeds its maximum"), - ) - for overrides, message in cases: - with self.subTest(overrides=overrides), patch.dict( - os.environ, - {**base, **overrides}, - clear=True, - ): - with self.assertRaisesRegex(SystemExit, message): - validate_prism_production_gate() - - def test_mainnet_implies_production_gate(self) -> None: - with patch.dict( - os.environ, - { - "QBIT_CHAIN": "mainnet", - "PRISM_STRATUM_STALE_GRACE_SECONDS": "3", - }, - clear=True, - ): - with self.assertRaisesRegex(SystemExit, "requires PRISM_STRATUM_SHARE_DIFF"): - validate_prism_production_gate() - - def test_compatibility_production_flag_implies_production_gate(self) -> None: - with patch.dict( - os.environ, - { - "QBIT_CHAIN": "testnet4", - "QBIT_TOOLS_PRODUCTION": "1", - "PRISM_STRATUM_STALE_GRACE_SECONDS": "3", - }, - clear=True, - ): - with self.assertRaisesRegex(SystemExit, "requires PRISM_STRATUM_SHARE_DIFF"): - validate_prism_production_gate() - - def test_mainnet_ctv_requires_static_fee_rate_before_runtime_startup(self) -> None: - env = { - "QBIT_CHAIN": "mainnet", - "QBIT_RPC_USER": "qbitrpc", - "QBIT_RPC_PASSWORD": "not-default", - "PRISM_POSTGRES_PSQL_COMMAND": "psql postgresql://example.invalid/qbit", - "PRISM_POSTGRES_PASSWORD": "not-default", - "PRISM_MANIFEST_SIGNING_SEED_HEX": "42" * 32, - "PRISM_LEDGER_ATTESTATION_SIGNING_SEED_HEX": "43" * 32, - "PRISM_LEDGER_WRITER_PUBLIC_KEY_HEX": "44" * 32, - "PRISM_LEDGER_WRITER_ID": "managed-writer", - "PRISM_LEDGER_WRITER_EPOCH": "7", - "PRISM_AUDIT_DIR": "/var/lib/qbit/prism/audit", - "PRISM_EVIDENCE_PATH": "/var/lib/qbit/prism/evidence.json", - "PRISM_STRATUM_STALE_GRACE_SECONDS": "3", - "PRISM_STRATUM_SHARE_DIFF": "1024", - "PRISM_STRATUM_VARDIFF_MIN_DIFF": "1024", - "PRISM_STRATUM_VARDIFF_START_DIFF": "4096", - "PRISM_STRATUM_VARDIFF_MAX_DIFF": "65536", - "PRISM_CTV_SETTLEMENT_ENABLED": "1", - } - with ( - patch.dict(os.environ, env, clear=True), - self.assertRaisesRegex( - SystemExit, - "PRISM_CTV_FANOUT_FEE_MARKET_RATE_BITS_PER_1000_WEIGHT", - ), - ): - validate_prism_production_gate() - - def test_live_chain_identity_accepts_main_alias_and_pinned_genesis(self) -> None: - genesis = "12" * 32 - server = PrismCoordinator.__new__(PrismCoordinator) - server.qbit_chain = "mainnet" - - class Rpc: - def call(self, method: str, params: object = None) -> object: - if method == "getblockchaininfo": - return { - "chain": "main", - "initialblockdownload": False, - "blocks": 100, - "headers": 100, - } - if method == "getnetworkinfo": - return {"connections": 2} - if method == "getblockhash" and params == [0]: - return genesis - raise RuntimeError(method) - - server.rpc = Rpc() - with patch.dict(os.environ, {"QBIT_EXPECTED_GENESIS_HASH": genesis}, clear=True): - server.validate_live_chain_identity() - - def test_live_chain_identity_rejects_incomplete_public_readiness(self) -> None: - server = PrismCoordinator.__new__(PrismCoordinator) - server.qbit_chain = "mainnet" - genesis = "12" * 32 - - cases = ( - ({"chain": "main", "blocks": 10, "headers": 10}, {"connections": 1}, "initial block"), - ( - {"chain": "main", "initialblockdownload": False, "blocks": 9, "headers": 10}, - {"connections": 1}, - "not caught up", - ), - ( - {"chain": "main", "initialblockdownload": False, "blocks": 10, "headers": 10}, - {"connections": 0}, - "requires at least 1", - ), - ) - for blockchain_info, network_info, message in cases: - with self.subTest(message=message): - server.rpc = SimpleNamespace( - call=lambda method, params=None: ( - blockchain_info - if method == "getblockchaininfo" - else network_info - if method == "getnetworkinfo" - else genesis - ) - ) - with ( - patch.dict(os.environ, {"QBIT_EXPECTED_GENESIS_HASH": genesis}, clear=True), - self.assertRaisesRegex(RuntimeError, message), - ): - server.validate_live_chain_identity() - - def test_live_template_preflight_enforces_freshness_and_relay_fee_floor(self) -> None: - server = PrismCoordinator.__new__(PrismCoordinator) - previous_hash = "34" * 32 - template = {"height": 1, "curtime": int(time.time()), "previousblockhash": previous_hash} - server.current_template_artifacts = lambda: SimpleNamespace( - template=template, - previousblockhash=previous_hash, - ) - server.rpc = SimpleNamespace( - call=lambda method, params=None: { - "minrelaytxfee": "0.00001000", - "mempoolminfee": "0.00001000", - } - ) - server._ctv_fanout_market_fee_rate_cache = {} - - enabled = { - "PRISM_CTV_SETTLEMENT_ENABLED": "1", - "PRISM_CTV_FANOUT_FEE_MARKET_RATE_BITS_PER_1000_WEIGHT": "1000", - "PRISM_TEMPLATE_MAX_AGE_SECONDS": "120", - } - with patch.dict(os.environ, enabled, clear=True): - server.validate_live_template_and_fee_policy() - - server._ctv_fanout_market_fee_rate_cache = {} - with ( - patch.dict( - os.environ, - {**enabled, "PRISM_CTV_FANOUT_FEE_MARKET_RATE_BITS_PER_1000_WEIGHT": "1"}, - clear=True, - ), - self.assertRaisesRegex(RuntimeError, "below the connected node relay floor"), - ): - server.validate_live_template_and_fee_policy() - - template["curtime"] = int(time.time()) - 121 - with ( - patch.dict(os.environ, enabled, clear=True), - self.assertRaisesRegex(RuntimeError, "block template is stale"), - ): - server.validate_live_template_and_fee_policy() - - def test_live_chain_identity_rejects_wrong_chain_or_genesis(self) -> None: - server = PrismCoordinator.__new__(PrismCoordinator) - server.qbit_chain = "mainnet" - server.rpc = SimpleNamespace( - call=lambda method, params=None: ( - {"chain": "regtest"} if method == "getblockchaininfo" else "34" * 32 - ) - ) - with patch.dict(os.environ, {"QBIT_EXPECTED_GENESIS_HASH": "12" * 32}, clear=True): - with self.assertRaisesRegex(RuntimeError, "does not match RPC chain"): - server.validate_live_chain_identity() - - server.rpc = SimpleNamespace( - call=lambda method, params=None: ( - {"chain": "main"} if method == "getblockchaininfo" else "34" * 32 - ) - ) - with patch.dict(os.environ, {"QBIT_EXPECTED_GENESIS_HASH": "12" * 32}, clear=True): - with self.assertRaisesRegex(RuntimeError, "does not match the connected"): - server.validate_live_chain_identity() - - def _pending_append(self, tag: str, accepted_at_ms: int = 2) -> PendingShareAppend: - from lab.prism.share_ledger import PendingShare - - return PendingShareAppend( - pending_share=PendingShare( - share_id=f"miner-a:{tag}", - miner_id="miner-a", - order_key="miner-a", - p2mr_program_hex="11" * 32, - share_difficulty=1, - network_difficulty=1, - template_height=10, - job_id="job-1", - job_issued_at_ms=1, - accepted_at_ms=accepted_at_ms, - ntime=1_700_000_000, - ), - username="miner-a", - job_id="job-1", - block_hash_hex=tag * 32, - collection_only=False, - credit_policy=None, - ) - - def test_prism_coinbase_output_policy_defaults_to_canonical_and_is_omitted(self) -> None: - server = coordinator() - server.rpc = AddressRpc(valid_address="tq1fee", script_byte="88") - - with patch.dict( - os.environ, - { - "PRISM_POOL_FEE_ENABLED": "1", - "PRISM_POOL_FEE_BPS": "125", - "PRISM_POOL_FEE_ADDRESS": "tq1fee", - }, - clear=True, - ): - policy = server.prism_payout_policy() - - self.assertNotIn("coinbase_output_policy", policy) - - def test_prism_coinbase_output_policy_explicit_canonical_is_omitted(self) -> None: - server = coordinator() - server.rpc = AddressRpc(valid_address="tq1fee", script_byte="88") - - with patch.dict( - os.environ, - { - "PRISM_COINBASE_OUTPUT_POLICY": "canonical", - "PRISM_POOL_FEE_ENABLED": "1", - "PRISM_POOL_FEE_BPS": "125", - "PRISM_POOL_FEE_ADDRESS": "tq1fee", - }, - clear=True, - ): - policy = server.prism_payout_policy() - - self.assertNotIn("coinbase_output_policy", policy) - - def test_prism_coinbase_output_policy_pool_fee_first_is_included(self) -> None: - server = coordinator() - server.rpc = AddressRpc(valid_address="tq1fee", script_byte="88") - - with patch.dict( - os.environ, - { - "PRISM_COINBASE_OUTPUT_POLICY": "pool-fee-first", - "PRISM_POOL_FEE_ENABLED": "1", - "PRISM_POOL_FEE_BPS": "125", - "PRISM_POOL_FEE_ADDRESS": "tq1fee", - }, - clear=True, - ): - policy = server.prism_payout_policy() - - self.assertEqual(policy["coinbase_output_policy"], "pool-fee-first") - self.assertEqual(policy["pool_fee_policy"]["fee_bps"], 125) - - def test_prism_coinbase_output_policy_rejects_unknown_values(self) -> None: - for invalid in ("fee-first", "POOL-FEE-FIRST", "canonical "): - with self.subTest(invalid=invalid), patch.dict( - os.environ, - { - "PRISM_COINBASE_OUTPUT_POLICY": invalid, - "PRISM_POOL_FEE_ENABLED": "1", - "PRISM_POOL_FEE_BPS": "125", - "PRISM_POOL_FEE_ADDRESS": "tq1fee", - }, - clear=True, - ): - server = coordinator() - server.rpc = AddressRpc(valid_address="tq1fee") - with self.assertRaisesRegex( - SystemExit, - "PRISM_COINBASE_OUTPUT_POLICY must be one of: canonical, pool-fee-first", - ): - server.prism_payout_policy() - - def test_prism_coinbase_output_policy_pool_fee_first_requires_pool_fee(self) -> None: - cases = [ - {"PRISM_COINBASE_OUTPUT_POLICY": "pool-fee-first"}, - { - "PRISM_COINBASE_OUTPUT_POLICY": "pool-fee-first", - "PRISM_POOL_FEE_ENABLED": "0", - }, - ] - for env_vars in cases: - with self.subTest(env_vars=env_vars), patch.dict(os.environ, env_vars, clear=True): - server = coordinator() - with self.assertRaisesRegex(SystemExit, "requires PRISM_POOL_FEE_ENABLED=1"): - server.prism_payout_policy() - - def test_generated_coordinator_session_advertises_heartbeat_capability(self) -> None: - server = PrismCoordinator.__new__(PrismCoordinator) - env = { - "PRISM_POSTGRES_PSQL_COMMAND": "psql postgresql://example.invalid/qbit", - } - - with patch.dict(os.environ, env, clear=True), patch( - "lab.prism.prism_coordinator.PsqlShareLedger" - ) as fake_ledger: - fake_ledger.return_value = SimpleNamespace(backend_name="postgres-psql") - server.make_ledger() - - self.assertTrue( - fake_ledger.call_args.kwargs["writer_session_token"].startswith( - WRITER_LEASE_HEARTBEAT_SESSION_PREFIX - ) - ) - - -class PrismVardiffInitialConvergenceKillSwitchTests(unittest.TestCase): - """PRISM_STRATUM_VARDIFF_INITIAL_CONVERGENCE=0 must fully disengage. - - The switch is documented as restoring the pre-#132 behavior exactly, so - turning it off has to stop the initial-convergence settings from having - any say in whether the coordinator starts. VardiffConfig enforces its - invariants on every instance, so the loader is what has to keep an inert - setting from reaching them as a startup failure. - """ - - START = Decimal("0.000000001") - - # Each entry is one initial-convergence setting that is invalid only - # relative to the relaxation, plus the message the enabled loader rejects - # it with. Raising MAX_STEP_UP past the initial bound is the reproducer - # that motivated this: it is an ordinary tuning change that becomes - # unrelated to anything once the relaxation is off. - DISABLED_SHOULD_TOLERATE = ( - ( - {"PRISM_STRATUM_VARDIFF_MAX_STEP_UP": "128"}, - "PRISM_STRATUM_VARDIFF_INITIAL_MAX_STEP_UP is below", - ), - ( - {"PRISM_STRATUM_VARDIFF_INITIAL_MAX_STEP_UP": "2"}, - "PRISM_STRATUM_VARDIFF_INITIAL_MAX_STEP_UP is below", - ), - ( - {"PRISM_STRATUM_VARDIFF_INITIAL_MIN_SHARES": "0"}, - "PRISM_STRATUM_VARDIFF_INITIAL_MIN_SHARES must be at least 1", - ), - ( - {"PRISM_STRATUM_VARDIFF_INITIAL_MIN_STEP_UP": "0.5"}, - "PRISM_STRATUM_VARDIFF_INITIAL_MIN_STEP_UP must be at least 1", - ), - ( - {"PRISM_STRATUM_VARDIFF_INITIAL_MIN_SECONDS": "0"}, - "PRISM_STRATUM_VARDIFF_INITIAL_MIN_SECONDS must be positive", - ), - ) - - def test_disabled_initial_settings_never_fail_startup(self) -> None: - for overrides, _message in self.DISABLED_SHOULD_TOLERATE: - with self.subTest(overrides=overrides): - config = load_prism_vardiff_config( - self.START, - environ={ - "PRISM_STRATUM_VARDIFF_INITIAL_CONVERGENCE": "0", - **overrides, - }, - ) - - self.assertFalse(config.initial_convergence_enabled) - # Whatever the loader had to do to the inert values, the - # config it returns must satisfy the invariants rather than - # having escaped them. - self.assertGreaterEqual( - config.initial_max_step_factor, config.max_step_factor - ) - self.assertGreaterEqual(config.initial_min_step_factor, 1) - self.assertGreaterEqual(config.initial_min_accepted_shares, 1) - self.assertGreater(config.initial_min_elapsed_seconds, 0) - - def test_disabled_initial_settings_are_not_parsed(self) -> None: - config = load_prism_vardiff_config( - self.START, - environ={ - "PRISM_STRATUM_VARDIFF_INITIAL_CONVERGENCE": "0", - "PRISM_STRATUM_VARDIFF_INITIAL_MAX_STEP_UP": "not-a-decimal", - "PRISM_STRATUM_VARDIFF_INITIAL_MIN_SHARES": "not-an-integer", - "PRISM_STRATUM_VARDIFF_INITIAL_MIN_STEP_UP": "NaN", - "PRISM_STRATUM_VARDIFF_INITIAL_MIN_SECONDS": "-1", - }, - ) - - self.assertFalse(config.initial_convergence_enabled) - self.assertEqual( - config.initial_max_step_factor, - DEFAULT_PRISM_VARDIFF_INITIAL_MAX_STEP_UP, - ) - - def test_enabled_validation_is_unchanged(self) -> None: - for overrides, message in self.DISABLED_SHOULD_TOLERATE: - with self.subTest(overrides=overrides): - with self.assertRaises(SystemExit) as raised: - load_prism_vardiff_config(self.START, environ=dict(overrides)) - - self.assertIn(message, str(raised.exception)) - - def test_enabled_still_accepts_a_valid_custom_relaxation(self) -> None: - config = load_prism_vardiff_config( - self.START, - environ={ - "PRISM_STRATUM_VARDIFF_MAX_STEP_UP": "8", - "PRISM_STRATUM_VARDIFF_INITIAL_MAX_STEP_UP": "128", - "PRISM_STRATUM_VARDIFF_INITIAL_MIN_SHARES": "4", - "PRISM_STRATUM_VARDIFF_INITIAL_MIN_STEP_UP": "2", - }, - ) - - self.assertTrue(config.initial_convergence_enabled) - # Operator values reach the config untouched while the switch is on. - self.assertEqual(config.max_step_factor, Decimal("8")) - self.assertEqual(config.initial_max_step_factor, Decimal("128")) - self.assertEqual(config.initial_min_accepted_shares, 4) - self.assertEqual(config.initial_min_step_factor, Decimal("2")) - - def test_validation_unrelated_to_the_relaxation_still_applies_when_off( - self, - ) -> None: - # The switch relaxes the initial-convergence settings only. The - # observation-only arrival threshold and the ordinary step bound are - # read regardless of it, so they keep failing closed. - for overrides, message in ( - ( - {"PRISM_STRATUM_VARDIFF_HIGH_DIFF_ARRIVAL_DIFF": "0"}, - "PRISM_STRATUM_VARDIFF_HIGH_DIFF_ARRIVAL_DIFF must be positive", - ), - ( - {"PRISM_STRATUM_VARDIFF_MAX_STEP_UP": "not-a-number"}, - "PRISM_STRATUM_VARDIFF_MAX_STEP_UP must be a decimal number", - ), - ): - with self.subTest(overrides=overrides): - with self.assertRaises(SystemExit) as raised: - load_prism_vardiff_config( - self.START, - environ={ - "PRISM_STRATUM_VARDIFF_INITIAL_CONVERGENCE": "0", - **overrides, - }, - ) - - self.assertIn(message, str(raised.exception)) - - def test_tolerated_settings_stay_inert(self) -> None: - # The whole justification for accepting these configurations is that - # the relaxation is off, so pin that: neither the early trigger nor - # the wider step bound may act on a config the loader had to adjust. - config = load_prism_vardiff_config( - self.START, - environ={ - "PRISM_STRATUM_VARDIFF_INITIAL_CONVERGENCE": "0", - "PRISM_STRATUM_VARDIFF_MAX_STEP_UP": "128", - "PRISM_STRATUM_VARDIFF_MAX_DIFF": "1000000000", - }, - ) - current = Decimal("100") - - self.assertFalse( - vardiff.initial_convergence_ready( - current_difficulty=current, - accepted_shares=1000, - accepted_difficulty=Decimal("1000000"), - elapsed_seconds=Decimal("60"), - config=config, - ) - ) - # Even asked for the relaxation directly, the step stays on the - # ordinary bound: 128x, not the coerced initial factor. - self.assertEqual( - vardiff.calculate_next_difficulty( - current_difficulty=current, - accepted_shares=1000, - elapsed_seconds=Decimal("60"), - config=config, - accepted_difficulty=Decimal("100000000"), - initial_convergence=True, - ), - current * config.max_step_factor, - ) - - -class PrismCoordinatorReliabilityTests(unittest.TestCase): - def _bare_coordinator(self) -> PrismCoordinator: - server = PrismCoordinator.__new__(PrismCoordinator) - server.lock = threading.RLock() - server.stop_event = threading.Event() - server._heartbeats = {} - server._watchdog_pauses = {} - server._heartbeats_lock = threading.Lock() - server.watchdog_timeout_seconds = 120.0 - server.watchdog_interval_seconds = 15.0 - return server - - def test_positive_float_env_rejects_non_finite_values(self) -> None: - for raw in ("nan", "inf", "-inf"): - with self.subTest(raw=raw), patch.dict( - os.environ, {"PRISM_WATCHDOG_TIMEOUT_SECONDS": raw}, clear=True - ): - with self.assertRaisesRegex(SystemExit, "PRISM_WATCHDOG_TIMEOUT_SECONDS must be finite"): - env_positive_float("PRISM_WATCHDOG_TIMEOUT_SECONDS", 120.0) - - -class PrismListenerProfileTests(unittest.TestCase): - def test_highdiff_listener_disabled_without_port(self) -> None: - with patch.dict(os.environ, {}, clear=False): - clear_stratum_diff_env() - base = load_prism_vardiff_config(Decimal("0.000000001")) - self.assertIsNone(load_prism_highdiff_listener("0.0.0.0", base)) - - def test_highdiff_listener_defaults_to_nicehash_floor(self) -> None: - with patch.dict(os.environ, {}, clear=False): - clear_stratum_diff_env() - os.environ["PRISM_STRATUM_HIGHDIFF_PORT"] = "4334" - base = load_prism_vardiff_config(Decimal("0.000000001")) - profile = load_prism_highdiff_listener("10.0.0.1", base) - - assert profile is not None - self.assertEqual(profile.name, "highdiff") - self.assertEqual(profile.bind, "10.0.0.1") - self.assertEqual(profile.port, 4334) - self.assertEqual(profile.heartbeat_name, "stratum_accept_highdiff") - self.assertEqual(profile.share_difficulty, Decimal("500000")) - self.assertEqual(profile.minimum_advertised_difficulty, Decimal("500000")) - self.assertEqual(profile.vardiff_config.min_difficulty, Decimal("500000")) - self.assertEqual(profile.vardiff_config.startup_difficulty, Decimal("500000")) - self.assertEqual(profile.vardiff_config.max_difficulty, Decimal("4294967296")) - # Everything but the difficulty bounds is inherited from the base config. - self.assertEqual(profile.vardiff_config.enabled, base.enabled) - self.assertEqual( - profile.vardiff_config.target_share_interval_seconds, - base.target_share_interval_seconds, - ) - self.assertEqual( - profile.vardiff_config.retarget_interval_seconds, - base.retarget_interval_seconds, - ) - self.assertEqual(profile.vardiff_config.max_step_factor, base.max_step_factor) - - def test_highdiff_listener_env_overrides(self) -> None: - with patch.dict(os.environ, {}, clear=False): - clear_stratum_diff_env() - os.environ["PRISM_STRATUM_HIGHDIFF_PORT"] = "4335" - os.environ["PRISM_STRATUM_HIGHDIFF_BIND"] = "127.0.0.2" - os.environ["PRISM_STRATUM_HIGHDIFF_MIN_DIFF"] = "600000" - os.environ["PRISM_STRATUM_HIGHDIFF_START_DIFF"] = "1000000" - os.environ["PRISM_STRATUM_HIGHDIFF_MAX_DIFF"] = "8000000" - os.environ["PRISM_STRATUM_HIGHDIFF_SHARE_DIFF"] = "700000" - base = load_prism_vardiff_config(Decimal("0.000000001")) - profile = load_prism_highdiff_listener("0.0.0.0", base) - - assert profile is not None - self.assertEqual(profile.bind, "127.0.0.2") - self.assertEqual(profile.port, 4335) - self.assertEqual(profile.share_difficulty, Decimal("700000")) - self.assertEqual(profile.minimum_advertised_difficulty, Decimal("600000")) - self.assertEqual(profile.vardiff_config.min_difficulty, Decimal("600000")) - self.assertEqual(profile.vardiff_config.startup_difficulty, Decimal("1000000")) - self.assertEqual(profile.vardiff_config.max_difficulty, Decimal("8000000")) - - def test_highdiff_listener_rejects_inconsistent_bounds(self) -> None: - base = load_prism_vardiff_config(Decimal("0.000000001")) - with patch.dict(os.environ, {}, clear=False): - clear_stratum_diff_env() - os.environ["PRISM_STRATUM_HIGHDIFF_PORT"] = "4334" - os.environ["PRISM_STRATUM_HIGHDIFF_MIN_DIFF"] = "1000000" - with self.assertRaises(SystemExit): - load_prism_highdiff_listener("0.0.0.0", base) - with patch.dict(os.environ, {}, clear=False): - clear_stratum_diff_env() - os.environ["PRISM_STRATUM_HIGHDIFF_PORT"] = "4334" - os.environ["PRISM_STRATUM_HIGHDIFF_MAX_DIFF"] = "400000" - with self.assertRaises(SystemExit): - load_prism_highdiff_listener("0.0.0.0", base) - for bad_port in ("not-a-port", "0", "70000"): - with patch.dict(os.environ, {}, clear=False): - clear_stratum_diff_env() - os.environ["PRISM_STRATUM_HIGHDIFF_PORT"] = bad_port - with self.assertRaises(SystemExit): - load_prism_highdiff_listener("0.0.0.0", base) - - def test_client_startup_difficulty_uses_listener_profile(self) -> None: - server = coordinator() - profile = StratumListenerProfile( - name="highdiff", - bind="0.0.0.0", - port=4334, - share_difficulty=Decimal("700000"), - vardiff_config=highdiff_vardiff_config(), - heartbeat_name="stratum_accept_highdiff", - ) - self.assertEqual(server.client_startup_difficulty(profile), Decimal("500000")) - # Without a profile the default listener behavior is unchanged. - self.assertEqual(server.client_startup_difficulty(), Decimal("0.000000001")) - # With vardiff disabled the listener's fixed share difficulty applies. - fixed_profile = StratumListenerProfile( - name="highdiff", - bind="0.0.0.0", - port=4334, - share_difficulty=Decimal("700000"), - vardiff_config=highdiff_vardiff_config(enabled=False), - heartbeat_name="stratum_accept_highdiff", - ) - self.assertEqual(server.client_startup_difficulty(fixed_profile), Decimal("700000")) - - def test_stratum_accept_heartbeat_names(self) -> None: - server = coordinator() - # Coordinators built without listener profiles (tests, legacy) keep the - # historical single heartbeat name. - self.assertEqual(server.stratum_accept_heartbeat_names(), ("stratum_accept",)) - server.listener_profiles = [ - StratumListenerProfile( - name="default", - bind="0.0.0.0", - port=3340, - share_difficulty=Decimal("1"), - vardiff_config=server.vardiff_config, - heartbeat_name="stratum_accept", - ), - StratumListenerProfile( - name="highdiff", - bind="0.0.0.0", - port=4334, - share_difficulty=Decimal("500000"), - vardiff_config=highdiff_vardiff_config(), - heartbeat_name="stratum_accept_highdiff", - ), - ] - self.assertEqual( - server.stratum_accept_heartbeat_names(), - ("stratum_accept", "stratum_accept_highdiff"), - ) - - def test_parse_stratum_password_options(self) -> None: - self.assertEqual(parse_stratum_password_options(""), (None, None)) - self.assertEqual(parse_stratum_password_options("x"), (None, None)) - self.assertEqual(parse_stratum_password_options("d=8192"), (Decimal("8192"), None)) - self.assertEqual( - parse_stratum_password_options("md=4096,d=8192"), - (Decimal("8192"), Decimal("4096")), - ) - self.assertEqual( - parse_stratum_password_options("D=500000, MD=500000"), - (Decimal("500000"), Decimal("500000")), - ) - self.assertEqual(parse_stratum_password_options("d=abc"), (None, None)) - self.assertEqual(parse_stratum_password_options("d=-5,md=0"), (None, None)) - self.assertEqual(parse_stratum_password_options("foo=1,md=2048"), (None, Decimal("2048"))) - - def test_password_d_below_highdiff_floor_is_clamped(self) -> None: - server = coordinator() - state = client() - state.listener_vardiff_config = highdiff_vardiff_config() - state.share_difficulty = Decimal("500000") - state.requested_difficulty = Decimal("1000") - - target = server.apply_client_difficulty_requests(state) - - self.assertEqual(target, Decimal("500000")) - assert state.vardiff_config is not None - self.assertEqual(state.vardiff_config.min_difficulty, Decimal("500000")) - self.assertEqual(state.vardiff_config.startup_difficulty, Decimal("500000")) - - def test_password_md_raises_personal_floor_and_retarget_respects_it(self) -> None: - server = coordinator() - state = client() - state.requested_min_difficulty = Decimal("256") - state.share_difficulty = Decimal("1") - - target = server.apply_client_difficulty_requests(state) - - self.assertEqual(target, Decimal("256")) - assert state.vardiff_config is not None - self.assertEqual(state.vardiff_config.min_difficulty, Decimal("256")) - self.assertEqual(state.vardiff_config.max_difficulty, Decimal("1024")) - - # A zero-share retarget window wants to step down 4x; the personal - # floor must hold it at 256. - state.share_difficulty = Decimal("256") - server.retarget_client( - state, - current_difficulty=Decimal("256"), - accepted_shares=0, - submitted_shares=0, - accepted_difficulty=Decimal("0"), - elapsed_seconds=Decimal("2"), - ) - self.assertIsNone(state.pending_share_difficulty) - self.assertEqual(state.share_difficulty, Decimal("256")) - - def test_apply_requests_is_stable_across_reapplication(self) -> None: - server = coordinator() - state = client() - state.listener_vardiff_config = highdiff_vardiff_config() - state.requested_min_difficulty = Decimal("600000") - state.requested_difficulty = Decimal("700000") - - first = server.apply_client_difficulty_requests(state) - second = server.apply_client_difficulty_requests(state) - - self.assertEqual(first, Decimal("700000")) - self.assertEqual(second, Decimal("700000")) - assert state.vardiff_config is not None - # Recomputed from the pristine listener config: the floor is the md= - # value, not a compounded one. - self.assertEqual(state.vardiff_config.min_difficulty, Decimal("600000")) - - def test_suggest_difficulty_before_subscribe_applies_directly(self) -> None: - server = coordinator() - state = ClientState(sock=object(), address=("127.0.0.1", 1), connection_id=2, extranonce1_hex="00000002") - sent: list[object] = [] - state.send = lambda payload: sent.append(payload) # type: ignore[method-assign] - - server.handle_suggest_difficulty(state, 7, [512]) - - self.assertEqual(state.suggested_difficulty, Decimal("512")) - self.assertEqual(state.share_difficulty, Decimal("512")) - self.assertIsNone(state.pending_share_difficulty) - self.assertEqual(sent, [{"id": 7, "result": True, "error": None}]) - - def test_suggest_difficulty_post_authorize_advertises_with_job(self) -> None: - server = coordinator() - state = client() - state.share_difficulty = Decimal("1") - sent: list[object] = [] - state.send = lambda payload: sent.append(payload) # type: ignore[method-assign] - jobs: dict[str, object] = {"count": 0} - - def fake_send_job(client: object, clean_jobs: bool) -> bool: - jobs.update({"count": jobs["count"] + 1, "clean": clean_jobs}) - return True - - server.maybe_send_job = fake_send_job # type: ignore[method-assign] - - server.handle_suggest_difficulty(state, 8, [512]) - - self.assertEqual(state.pending_share_difficulty, Decimal("512")) - self.assertEqual(jobs["count"], 1) - self.assertTrue(jobs["clean"]) - self.assertEqual(sent, [{"id": 8, "result": True, "error": None}]) - - def test_suggest_difficulty_rolls_back_pending_on_build_failure(self) -> None: - server = coordinator() - state = client() - state.share_difficulty = Decimal("1") - state.difficulty_generation = 7 - state.send = lambda payload: None # type: ignore[method-assign] - server.maybe_send_job = lambda client, *, clean_jobs: False # type: ignore[method-assign] - - server.handle_suggest_difficulty(state, 9, [512]) - - self.assertIsNone(state.pending_share_difficulty) - self.assertEqual(state.share_difficulty, Decimal("1")) - self.assertEqual(state.difficulty_generation, 7) - - def test_suggest_difficulty_yields_to_password_d_option(self) -> None: - server = coordinator() - state = client() - state.requested_difficulty = Decimal("512") - state.share_difficulty = Decimal("512") - state.send = lambda payload: None # type: ignore[method-assign] - server.maybe_send_job = lambda client, *, clean_jobs: True # type: ignore[method-assign] - - server.handle_suggest_difficulty(state, 10, [128]) - - # d= wins: the suggestion is recorded but the resolved target stays at - # the explicit password difficulty, so nothing is re-advertised. - self.assertEqual(state.suggested_difficulty, Decimal("128")) - self.assertEqual(state.share_difficulty, Decimal("512")) - self.assertIsNone(state.pending_share_difficulty) - - def test_suggest_difficulty_ignores_junk_values(self) -> None: - server = coordinator() - state = client() - state.share_difficulty = Decimal("1") - sent: list[object] = [] - state.send = lambda payload: sent.append(payload) # type: ignore[method-assign] - - for junk in ([], ["nan"], ["-4"], ["0"], [None]): - server.handle_suggest_difficulty(state, 11, junk) # type: ignore[arg-type] - - self.assertIsNone(state.vardiff_config) - self.assertEqual(state.share_difficulty, Decimal("1")) - self.assertEqual(len(sent), 5) - - def test_highdiff_share_diff_tracks_start_and_validates_bounds(self) -> None: - base = load_prism_vardiff_config(Decimal("0.000000001")) - with patch.dict(os.environ, {}, clear=False): - clear_stratum_diff_env() - os.environ["PRISM_STRATUM_HIGHDIFF_PORT"] = "4334" - os.environ["PRISM_STRATUM_HIGHDIFF_MIN_DIFF"] = "1000000" - os.environ["PRISM_STRATUM_HIGHDIFF_START_DIFF"] = "1000000" - profile = load_prism_highdiff_listener("0.0.0.0", base) - assert profile is not None - # Unset fixed difficulty tracks the start difficulty instead of a - # constant that could fall below a raised floor. - self.assertEqual(profile.share_difficulty, Decimal("1000000")) - - # An explicit fixed difficulty outside the listener bounds must fail - # startup: advertising below the floor is exactly what the high-diff - # listener exists to prevent. - with patch.dict(os.environ, {}, clear=False): - clear_stratum_diff_env() - os.environ["PRISM_STRATUM_HIGHDIFF_PORT"] = "4334" - os.environ["PRISM_STRATUM_HIGHDIFF_SHARE_DIFF"] = "1000" - with self.assertRaises(SystemExit): - load_prism_highdiff_listener("0.0.0.0", base) - with patch.dict(os.environ, {}, clear=False): - clear_stratum_diff_env() - os.environ["PRISM_STRATUM_HIGHDIFF_PORT"] = "4334" - os.environ["PRISM_STRATUM_HIGHDIFF_SHARE_DIFF"] = "8589934592" - with self.assertRaises(SystemExit): - load_prism_highdiff_listener("0.0.0.0", base) - - def authorize_server_and_client(self) -> tuple[PrismCoordinator, ClientState, list[object]]: - server = coordinator() - server.rpc = AddressValidationRpc() - server.username_fallback_address = None - server.maybe_send_job = lambda client, *, clean_jobs: True # type: ignore[method-assign] - state = ClientState(sock=object(), address=("127.0.0.1", 1), connection_id=3, extranonce1_hex="00000003") - state.subscribed = True - sent: list[object] = [] - state.send = lambda payload: sent.append(payload) # type: ignore[method-assign] - return server, state, sent - - def test_authorize_password_applies_before_first_job(self) -> None: - server, state, sent = self.authorize_server_and_client() - - server.handle_request( - state, - {"id": 5, "method": "mining.authorize", "params": [PAYOUT_ADDRESS, "d=0.5,md=0.25"]}, - ) - - self.assertTrue(state.authorized) - self.assertEqual(state.requested_difficulty, Decimal("0.5")) - self.assertEqual(state.requested_min_difficulty, Decimal("0.25")) - # Applied directly (no job exists yet), so the first - # set_difficulty/notify pair advertises the requested value. - self.assertEqual(state.share_difficulty, Decimal("0.5")) - self.assertIsNone(state.pending_share_difficulty) - assert state.vardiff_config is not None - self.assertEqual(state.vardiff_config.min_difficulty, Decimal("0.25")) - self.assertEqual(sent, [{"id": 5, "result": True, "error": None}]) - - def test_reauthorize_with_plain_password_clears_stale_overrides(self) -> None: - server, state, _ = self.authorize_server_and_client() - server.handle_request( - state, - {"id": 5, "method": "mining.authorize", "params": [PAYOUT_ADDRESS, "d=0.5,md=0.25"]}, - ) - assert state.vardiff_config is not None - - server.handle_request( - state, - {"id": 6, "method": "mining.authorize", "params": [PAYOUT_ADDRESS, "x"]}, - ) - - # The new password carries no options: prior overrides are dropped and - # the client falls back to the pristine listener policy (its current - # difficulty is left alone; vardiff drifts it under listener bounds). - self.assertIsNone(state.requested_difficulty) - self.assertIsNone(state.requested_min_difficulty) - self.assertIsNone(state.vardiff_config) - self.assertEqual(state.share_difficulty, Decimal("0.5")) - - def test_reauthorize_with_new_difficulty_sends_single_job_pair(self) -> None: - server, state, _ = self.authorize_server_and_client() - send_job_calls: list[bool] = [] - - def counting_send_job(current: ClientState, *, clean_jobs: bool) -> bool: - send_job_calls.append(clean_jobs) - current.active_job = SimpleNamespace( - template={"previousblockhash": "aa" * 32}, - payout_state_generation=0, - ) - server.current_tip_first_seen = ("aa" * 32, None) - return True - - server.maybe_send_job = counting_send_job # type: ignore[method-assign] - - server.handle_request( - state, - {"id": 5, "method": "mining.authorize", "params": [PAYOUT_ADDRESS, "x"]}, - ) - self.assertEqual(len(send_job_calls), 1) - - # A re-authorize whose new d= advertises a fresh difficulty/job pair - # must not be followed by a second back-to-back pair. - server.handle_request( - state, - {"id": 6, "method": "mining.authorize", "params": [PAYOUT_ADDRESS, "d=0.5"]}, - ) - self.assertEqual(len(send_job_calls), 2) - self.assertEqual(state.pending_share_difficulty, Decimal("0.5")) - - def test_authorize_rejects_and_disconnects_above_username_connection_limit(self) -> None: - server, first, _ = self.authorize_server_and_client() - server.stratum_max_connections_per_username = 1 - second = ClientState( - sock=object(), - address=("127.0.0.1", 2), - connection_id=4, - extranonce1_hex="00000004", - ) - second.send = lambda payload: None # type: ignore[method-assign] - server.clients.update({first, second}) - - server.handle_request( - first, - {"id": 5, "method": "mining.authorize", "params": [PAYOUT_ADDRESS, "x"]}, - ) - with self.assertRaises(StratumError) as raised: - server.handle_request( - second, - {"id": 6, "method": "mining.authorize", "params": [PAYOUT_ADDRESS, "x"]}, - ) - - self.assertTrue(raised.exception.disconnect) - self.assertEqual(raised.exception.message, "too many connections for username") - self.assertEqual(server.connection_limit_rejection_counts["username"], 1) - self.assertFalse(second.authorized) - - def test_reauthorize_limit_error_preserves_live_session(self) -> None: - server, live, _ = self.authorize_server_and_client() - server.stratum_max_connections_per_username = 1 - occupant = ClientState( - sock=object(), - address=("127.0.0.1", 2), - connection_id=4, - extranonce1_hex="00000004", - ) - occupant.send = lambda payload: None # type: ignore[method-assign] - server.clients.update({live, occupant}) - - server.handle_request( - live, - { - "id": 5, - "method": "mining.authorize", - "params": [f"{PAYOUT_ADDRESS}.original", "x"], - }, - ) - server.handle_request( - occupant, - { - "id": 6, - "method": "mining.authorize", - "params": [f"{PAYOUT_ADDRESS}.full", "x"], - }, - ) - original_worker = live.worker - - with self.assertRaises(StratumError) as raised: - server.handle_request( - live, - { - "id": 7, - "method": "mining.authorize", - "params": [f"{PAYOUT_ADDRESS}.full", "x"], - }, - ) - - self.assertFalse(raised.exception.disconnect) - self.assertTrue(live.authorized) - self.assertIs(live.worker, original_worker) - self.assertEqual(live.username, f"{PAYOUT_ADDRESS}.original") - - def test_username_connection_limit_is_disabled_by_default(self) -> None: - server = coordinator() - first = client() - second = client() - server.clients.update({first, second}) - worker = WorkerIdentity( - username=PAYOUT_ADDRESS, - payout_address=PAYOUT_ADDRESS, - worker_name=None, - script_pubkey_hex="5220" + "11" * 32, - p2mr_program_hex="11" * 32, - ) - - self.assertTrue(server.reserve_client_username(first, worker)) - self.assertTrue(server.reserve_client_username(second, worker)) - self.assertEqual(server.connection_limit_rejection_counts["username"], 0) - - def test_username_limit_does_not_count_idle_clients_for_empty_username(self) -> None: - server = coordinator() - server.stratum_max_connections_per_username = 1 - idle = client() - first = client() - second = client() - server.clients.update({idle, first, second}) - worker = WorkerIdentity( - username="", - payout_address=PAYOUT_ADDRESS, - worker_name=None, - script_pubkey_hex="5220" + "11" * 32, - p2mr_program_hex="11" * 32, - ) - - self.assertTrue(server.reserve_client_username(first, worker)) - self.assertFalse(server.reserve_client_username(second, worker)) - self.assertEqual(server.connection_limit_rejection_counts["username"], 1) - - def test_accept_loop_rejects_above_global_connection_limit(self) -> None: - server = coordinator() - server.stratum_max_connections = 1 - server.clients.add(client()) - - class AcceptedSocket: - def __init__(self) -> None: - self.closed = False - - def close(self) -> None: - self.closed = True - - accepted = AcceptedSocket() - - class OneConnectionListener: - def __init__(self) -> None: - self.calls = 0 - - def accept(self) -> tuple[object, tuple[str, int]]: - self.calls += 1 - if self.calls == 1: - return accepted, ("127.0.0.1", 1000) - server.stop_event.set() - raise socket.timeout() - - profile = StratumListenerProfile( - name="default", - bind="127.0.0.1", - port=3340, - share_difficulty=server.share_difficulty, - vardiff_config=server.vardiff_config, - heartbeat_name="stratum_accept", - ) - - server.accept_loop(OneConnectionListener(), profile) # type: ignore[arg-type] - - self.assertTrue(accepted.closed) - self.assertEqual(server.connection_limit_rejection_counts["global"], 1) - - def test_accept_loop_recovers_from_descriptor_exhaustion(self) -> None: - server = coordinator() - server.stratum_accept_resource_exhaustion_backoff_seconds = 0 - - class ExhaustedListener: - def __init__(self) -> None: - self.calls = 0 - - def accept(self) -> tuple[object, tuple[str, int]]: - self.calls += 1 - if self.calls == 1: - raise OSError(errno.EMFILE, "too many open files") - server.stop_event.set() - raise socket.timeout() - - listener = ExhaustedListener() - profile = StratumListenerProfile( - name="default", - bind="127.0.0.1", - port=3340, - share_difficulty=server.share_difficulty, - vardiff_config=server.vardiff_config, - heartbeat_name="stratum_accept", - ) - - server.accept_loop(listener, profile) # type: ignore[arg-type] - - self.assertEqual(listener.calls, 2) - self.assertEqual(server.accept_resource_exhaustion_count, 1) - - def test_resource_backoff_keeps_accept_watchdog_heartbeat_fresh(self) -> None: - server = coordinator() - server.stratum_accept_resource_exhaustion_backoff_seconds = 0.03 - server.watchdog_timeout_seconds = 0.01 - - server._wait_after_stratum_resource_failure("stratum_accept") - - self.assertFalse(server._overdue_heartbeats(time.monotonic())) - - def test_accept_loop_recovers_when_handler_thread_cannot_start(self) -> None: - server = coordinator() - server.connection_counter = 0 - server.stratum_send_timeout_seconds = 0 - server.stratum_accept_resource_exhaustion_backoff_seconds = 0 - - class AcceptedSocket: - def __init__(self) -> None: - self.closed = False - - def settimeout(self, timeout: object) -> None: - pass - - def shutdown(self, how: int) -> None: - pass - - def close(self) -> None: - self.closed = True - - accepted = AcceptedSocket() - - class OneConnectionListener: - def __init__(self) -> None: - self.calls = 0 - - def accept(self) -> tuple[object, tuple[str, int]]: - self.calls += 1 - if self.calls == 1: - return accepted, ("127.0.0.1", 1000) - server.stop_event.set() - raise socket.timeout() - - listener = OneConnectionListener() - profile = StratumListenerProfile( - name="default", - bind="127.0.0.1", - port=3340, - share_difficulty=server.share_difficulty, - vardiff_config=server.vardiff_config, - heartbeat_name="stratum_accept", - ) - - with patch.object(threading.Thread, "start", side_effect=RuntimeError("can't start new thread")): - server.accept_loop(listener, profile) # type: ignore[arg-type] - - self.assertEqual(listener.calls, 2) - self.assertTrue(accepted.closed) - self.assertFalse(server.clients) - self.assertEqual(server.connection_setup_failure_count, 1) - - def test_handle_client_cleans_up_when_makefile_hits_descriptor_limit(self) -> None: - server = coordinator() - - class MakefileFailureSocket: - def __init__(self) -> None: - self.closed = False - - def makefile(self, *args: object, **kwargs: object) -> object: - raise OSError(errno.EMFILE, "too many open files") - - def shutdown(self, how: int) -> None: - pass - - def close(self) -> None: - self.closed = True - - sock = MakefileFailureSocket() - state = ClientState( - sock=sock, # type: ignore[arg-type] - address=("127.0.0.1", 1000), - connection_id=1, - extranonce1_hex="00000001", - ) - server.clients.add(state) - - server.handle_client(state) - - self.assertTrue(sock.closed) - self.assertNotIn(state, server.clients) - self.assertEqual(server.accept_resource_exhaustion_count, 1) - - def test_accept_loop_assigns_listener_profiles_and_unique_extranonce(self) -> None: - server = coordinator() - server.connection_counter = 0 - server.stratum_send_timeout_seconds = 0.0 - - def listening_socket() -> socket.socket: - listener = socket.socket(socket.AF_INET, socket.SOCK_STREAM) - listener.bind(("127.0.0.1", 0)) - listener.listen() - listener.settimeout(0.1) - return listener - - default_listener = listening_socket() - highdiff_listener = listening_socket() - default_profile = StratumListenerProfile( - name="default", - bind="127.0.0.1", - port=default_listener.getsockname()[1], - share_difficulty=server.share_difficulty, - vardiff_config=server.vardiff_config, - heartbeat_name="stratum_accept", - ) - highdiff_profile = StratumListenerProfile( - name="highdiff", - bind="127.0.0.1", - port=highdiff_listener.getsockname()[1], - share_difficulty=Decimal("500000"), - vardiff_config=highdiff_vardiff_config(), - heartbeat_name="stratum_accept_highdiff", - minimum_advertised_difficulty=Decimal("500000"), - ) - threads = [ - threading.Thread(target=server.accept_loop, args=(default_listener, default_profile), daemon=True), - threading.Thread(target=server.accept_loop, args=(highdiff_listener, highdiff_profile), daemon=True), - ] - for thread in threads: - thread.start() - connections = [] - try: - connections.append(socket.create_connection(("127.0.0.1", default_profile.port), timeout=5)) - connections.append(socket.create_connection(("127.0.0.1", highdiff_profile.port), timeout=5)) - deadline = time.monotonic() + 5 - while time.monotonic() < deadline: - with server.lock: - if len(server.clients) == 2: - break - time.sleep(0.01) - with server.lock: - clients_by_listener = {c.listener_name: c for c in server.clients} - self.assertEqual(set(clients_by_listener), {"default", "highdiff"}) - self.assertEqual( - clients_by_listener["default"].share_difficulty, - Decimal("0.000000001"), - ) - self.assertEqual( - clients_by_listener["highdiff"].share_difficulty, - Decimal("500000"), - ) - self.assertIs( - clients_by_listener["highdiff"].listener_vardiff_config, - highdiff_profile.vardiff_config, - ) - self.assertEqual( - clients_by_listener["highdiff"].minimum_advertised_difficulty, - Decimal("500000"), - ) - self.assertEqual( - clients_by_listener["default"].minimum_advertised_difficulty, - Decimal("0"), - ) - extranonces = {c.extranonce1_hex for c in clients_by_listener.values()} - self.assertEqual(extranonces, {"00000001", "00000002"}) - self.assertIn("stratum_accept", server._heartbeats) - self.assertIn("stratum_accept_highdiff", server._heartbeats) - finally: - server.stop_event.set() - for connection in connections: - connection.close() - default_listener.close() - highdiff_listener.close() - for thread in threads: - thread.join(timeout=5) - - -class LedgerSessionGuardConfigTests(unittest.TestCase): - # (env var, LedgerConfig field, default) for the postgres session guards - # and the bounded startup lease acquisition, all six added for issue #123. - SESSION_GUARD_SETTINGS = ( - ( - "PRISM_POSTGRES_IDLE_IN_TRANSACTION_TIMEOUT_SECONDS", - "postgres_idle_in_transaction_timeout_seconds", - 15.0, - ), - ( - "PRISM_POSTGRES_TCP_KEEPALIVES_IDLE_SECONDS", - "postgres_tcp_keepalives_idle_seconds", - 30, - ), - ( - "PRISM_POSTGRES_TCP_KEEPALIVES_INTERVAL_SECONDS", - "postgres_tcp_keepalives_interval_seconds", - 10, - ), - ( - "PRISM_POSTGRES_TCP_KEEPALIVES_COUNT", - "postgres_tcp_keepalives_count", - 3, - ), - ( - "PRISM_LEDGER_LEASE_ACQUIRE_LOCK_TIMEOUT_SECONDS", - "lease_acquire_lock_timeout_seconds", - 5.0, - ), - ( - "PRISM_LEDGER_LEASE_ACQUIRE_ATTEMPTS", - "lease_acquire_attempts", - 5, - ), - ) - - @staticmethod - def minimal_environment(root: Path) -> dict[str, str]: - return { - "QBIT_RPC_HOST": "qbit.example", - "QBIT_RPC_USER": "rpc-user", - "QBIT_RPC_PASSWORD": "rpc-password", - "PRISM_ALLOW_MEMORY_LEDGER": "1", - "PRISM_ALLOW_TEST_SIGNING_SEEDS": "1", - "PRISM_ALLOW_BUNDLE_EMBEDDED_LEDGER_KEY": "1", - "PRISM_AUDIT_DIR": str(root), - "PRISM_EVIDENCE_PATH": str(root / "evidence.json"), - } - - def test_session_guard_defaults_land_on_ledger_config(self) -> None: - with tempfile.TemporaryDirectory() as temp_dir: - config = load_coordinator_config( - self.minimal_environment(Path(temp_dir)) - ) - - for _env_name, field_name, default in self.SESSION_GUARD_SETTINGS: - with self.subTest(field=field_name): - self.assertEqual(getattr(config.ledger, field_name), default) - - def test_replay_page_size_defaults_and_overrides(self) -> None: - with tempfile.TemporaryDirectory() as temp_dir: - source = self.minimal_environment(Path(temp_dir)) - self.assertEqual(load_coordinator_config(source).block.replay_page_size, 1024) - for size in (1, 2, 32, 1024): - with self.subTest(size=size): - config = load_coordinator_config( - {**source, "PRISM_BLOCK_REPLAY_PAGE_SIZE": str(size)} - ) - self.assertEqual(config.block.replay_page_size, size) - - def test_invalid_replay_page_size_refuses_startup(self) -> None: - with tempfile.TemporaryDirectory() as temp_dir: - source = self.minimal_environment(Path(temp_dir)) - for raw in ("", "0", "-1", "1025", "1.5", "invalid"): - with self.subTest(raw=raw), self.assertRaisesRegex( - SystemExit, "PRISM_BLOCK_REPLAY_PAGE_SIZE" - ): - load_coordinator_config({**source, "PRISM_BLOCK_REPLAY_PAGE_SIZE": raw}) - - def test_session_guard_overrides_land_on_ledger_config(self) -> None: - overrides = { - "PRISM_POSTGRES_IDLE_IN_TRANSACTION_TIMEOUT_SECONDS": "7.5", - "PRISM_POSTGRES_TCP_KEEPALIVES_IDLE_SECONDS": "17", - "PRISM_POSTGRES_TCP_KEEPALIVES_INTERVAL_SECONDS": "6", - "PRISM_POSTGRES_TCP_KEEPALIVES_COUNT": "2", - "PRISM_LEDGER_LEASE_ACQUIRE_LOCK_TIMEOUT_SECONDS": "3.5", - "PRISM_LEDGER_LEASE_ACQUIRE_ATTEMPTS": "9", - } - with tempfile.TemporaryDirectory() as temp_dir: - config = load_coordinator_config( - {**self.minimal_environment(Path(temp_dir)), **overrides} - ) - - ledger = config.ledger - self.assertEqual(ledger.postgres_idle_in_transaction_timeout_seconds, 7.5) - self.assertEqual(ledger.postgres_tcp_keepalives_idle_seconds, 17) - self.assertEqual(ledger.postgres_tcp_keepalives_interval_seconds, 6) - self.assertEqual(ledger.postgres_tcp_keepalives_count, 2) - self.assertEqual(ledger.lease_acquire_lock_timeout_seconds, 3.5) - self.assertEqual(ledger.lease_acquire_attempts, 9) - - def test_session_guard_settings_fail_closed_on_invalid_values(self) -> None: - # Empty, non-numeric, zero, and negative values must all abort the - # load: a disarmed or nonsensical guard silently recreates the - # unbounded startup blocking these settings exist to prevent. - for env_name, _field_name, _default in self.SESSION_GUARD_SETTINGS: - for raw in ("", "not-a-number", "0", "-1"): - with self.subTest(name=env_name, raw=raw): - with tempfile.TemporaryDirectory() as temp_dir: - source = { - **self.minimal_environment(Path(temp_dir)), - env_name: raw, - } - with self.assertRaises(SystemExit): - load_coordinator_config(source) diff --git a/tests/test_prism_coordinator_config_loading.py b/tests/test_prism_coordinator_config_loading.py deleted file mode 100644 index f6acb97b8..000000000 --- a/tests/test_prism_coordinator_config_loading.py +++ /dev/null @@ -1,352 +0,0 @@ -#!/usr/bin/env python3 -"""Direct tests for the extracted coordinator configuration and RPC seams.""" - -from __future__ import annotations - -import contextlib -import dataclasses -import io -import subprocess -import sys -import tempfile -import time -import unittest -from dataclasses import replace -from pathlib import Path -from types import SimpleNamespace -from unittest.mock import patch - -from lab.prism.coordinator_config import ( - CoordinatorConfig, - LifecycleConfig, - load_coordinator_config, -) -from lab.prism.prism_coordinator import JsonRpc as CompatibilityJsonRpc -from lab.prism.prism_coordinator import PrismCoordinator -from lab.prism.rpc import JsonRpc -from lab.prism.run_ctv_broadcaster_daemon import JsonRpc as DaemonJsonRpc - - -def minimal_environment(root: Path) -> dict[str, str]: - return { - "QBIT_RPC_HOST": "qbit.example", - "QBIT_RPC_USER": "rpc-user", - "QBIT_RPC_PASSWORD": "rpc-password", - "PRISM_ALLOW_MEMORY_LEDGER": "1", - "PRISM_ALLOW_TEST_SIGNING_SEEDS": "1", - "PRISM_ALLOW_BUNDLE_EMBEDDED_LEDGER_KEY": "1", - "PRISM_AUDIT_DIR": str(root), - "PRISM_EVIDENCE_PATH": str(root / "evidence.json"), - } - - -class CoordinatorConfigLoadingTests(unittest.TestCase): - def construct(self, source: dict[str, str]) -> PrismCoordinator: - config = load_coordinator_config(source) - with patch.object(JsonRpc, "call", side_effect=RuntimeError("offline")): - return PrismCoordinator(config) - - def test_loader_accepts_mapping_without_reading_process_environment(self) -> None: - with tempfile.TemporaryDirectory() as temp_dir: - source = minimal_environment(Path(temp_dir)) - with patch.dict("os.environ", {}, clear=True): - config = load_coordinator_config(source) - - self.assertIsInstance(config, CoordinatorConfig) - self.assertEqual(config.rpc.host, "qbit.example") - self.assertEqual(config.rpc.port, 18452) - self.assertEqual(config.stratum.bind, "127.0.0.1") - self.assertEqual(config.lifecycle.pending_refresh_health_deadline_seconds, 15.0) - self.assertEqual(config.lifecycle.coherent_tip_poll_health_deadline_seconds, 15.0) - - def test_loader_captures_pr54_health_deadlines_in_frozen_lifecycle_config(self) -> None: - with tempfile.TemporaryDirectory() as temp_dir: - source = { - **minimal_environment(Path(temp_dir)), - "PRISM_HEALTH_REFRESH_SECONDS": "7", - "PRISM_METRICS_REFRESH_SECONDS": "11", - "PRISM_HEALTH_PENDING_REFRESH_MAX_AGE_SECONDS": "23", - "PRISM_HEALTH_TIP_POLL_MAX_AGE_SECONDS": "29", - "PRISM_MINING_HEALTH_STARTUP_GRACE_SECONDS": "31", - } - config = load_coordinator_config(source) - - self.assertEqual( - config.lifecycle, - replace( - config.lifecycle, - health_refresh_seconds=7.0, - metrics_refresh_seconds=11.0, - pending_refresh_health_deadline_seconds=23.0, - coherent_tip_poll_health_deadline_seconds=29.0, - mining_health_startup_grace_seconds=31.0, - ), - ) - self.assertIsInstance(config.lifecycle, LifecycleConfig) - with self.assertRaises(dataclasses.FrozenInstanceError): - config.lifecycle.health_refresh_seconds = 1.0 # type: ignore[misc] - - def test_loader_owns_public_reward_window_cache_interval(self) -> None: - with tempfile.TemporaryDirectory() as temp_dir: - source = { - **minimal_environment(Path(temp_dir)), - "PRISM_PUBLIC_REWARD_WINDOW_CACHE_SECONDS": "47", - } - config = load_coordinator_config(source) - - self.assertEqual(config.ledger.reward_window_cache_seconds, 47.0) - - def test_coordinator_uses_supplied_snapshot_without_reloading_environment(self) -> None: - with tempfile.TemporaryDirectory() as temp_dir: - config = load_coordinator_config(minimal_environment(Path(temp_dir))) - with ( - patch( - "lab.prism.prism_coordinator.load_coordinator_config", - side_effect=AssertionError("unexpected environment reload"), - ), - patch.object(JsonRpc, "call", side_effect=RuntimeError("offline")), - ): - coordinator = PrismCoordinator(config) - - self.assertIs(coordinator.config, config) - self.assertEqual(coordinator.bind, config.stratum.bind) - self.assertEqual( - coordinator.health_pending_refresh_max_age_seconds, - config.lifecycle.pending_refresh_health_deadline_seconds, - ) - self.assertEqual( - coordinator.metrics_refresh_seconds, - config.lifecycle.metrics_refresh_seconds, - ) - - def test_metrics_refresh_interval_must_be_positive(self) -> None: - with tempfile.TemporaryDirectory() as temp_dir: - source = { - **minimal_environment(Path(temp_dir)), - "PRISM_METRICS_REFRESH_SECONDS": "0", - } - with self.assertRaisesRegex( - SystemExit, - "PRISM_METRICS_REFRESH_SECONDS must be positive", - ): - load_coordinator_config(source) - - def test_watchdog_timeout_floor_rejects_sub_second_tolerances(self) -> None: - with tempfile.TemporaryDirectory() as temp_dir: - base = minimal_environment(Path(temp_dir)) - for raw in ("0.0005", "0.5", "0.999"): - with self.subTest(raw=raw): - with self.assertRaisesRegex( - SystemExit, - "PRISM_WATCHDOG_TIMEOUT_SECONDS must be at least 1", - ): - load_coordinator_config( - {**base, "PRISM_WATCHDOG_TIMEOUT_SECONDS": raw} - ) - for raw, expected in (("1.0", 1.0), ("120", 120.0)): - with self.subTest(raw=raw): - config = load_coordinator_config( - {**base, "PRISM_WATCHDOG_TIMEOUT_SECONDS": raw} - ) - self.assertEqual( - config.lifecycle.watchdog_timeout_seconds, expected - ) - - def test_idle_in_transaction_reap_outside_lease_budget_warns(self) -> None: - with tempfile.TemporaryDirectory() as temp_dir: - base = minimal_environment(Path(temp_dir)) - inverted = { - **base, - "PRISM_LEDGER_LEASE_ACQUIRE_LOCK_TIMEOUT_SECONDS": "2", - "PRISM_LEDGER_LEASE_ACQUIRE_ATTEMPTS": "3", - "PRISM_POSTGRES_IDLE_IN_TRANSACTION_TIMEOUT_SECONDS": "10", - } - stderr = io.StringIO() - with contextlib.redirect_stderr(stderr): - load_coordinator_config(inverted) - warning = stderr.getvalue() - self.assertIn("prism coordinator: ", warning) - self.assertIn( - "PRISM_POSTGRES_IDLE_IN_TRANSACTION_TIMEOUT_SECONDS (10s)", - warning, - ) - self.assertIn( - "PRISM_LEDGER_LEASE_ACQUIRE_LOCK_TIMEOUT_SECONDS (2s)", - warning, - ) - self.assertIn("PRISM_LEDGER_LEASE_ACQUIRE_ATTEMPTS (3)", warning) - self.assertIn("lower the idle-in-transaction timeout", warning) - - stderr = io.StringIO() - with contextlib.redirect_stderr(stderr): - load_coordinator_config(base) - self.assertEqual(stderr.getvalue(), "") - - def test_zero_argument_coordinator_still_loads_environment(self) -> None: - with tempfile.TemporaryDirectory() as temp_dir: - source = minimal_environment(Path(temp_dir)) - with ( - patch.dict("os.environ", source, clear=True), - patch.object(JsonRpc, "call", side_effect=RuntimeError("offline")), - ): - coordinator = PrismCoordinator() - - self.assertEqual(coordinator.rpc.host, "qbit.example") - self.assertEqual(coordinator.config.rpc.user, "rpc-user") - - def test_dormant_live_chain_validation_is_deferred_until_use(self) -> None: - with tempfile.TemporaryDirectory() as temp_dir: - source = { - **minimal_environment(Path(temp_dir)), - "PRISM_MIN_PEERS": "0", - "PRISM_TEMPLATE_MAX_AGE_SECONDS": "not-an-int", - } - coordinator = self.construct(source) - - class PublicChainRpc: - def call(self, method: str, params: list[object] | None = None) -> object: - if method == "getblockchaininfo": - return { - "chain": "testnet4", - "initialblockdownload": False, - "blocks": 10, - "headers": 10, - } - if method == "getnetworkinfo": - return {"connections": 1} - raise RuntimeError(method) - - coordinator.qbit_chain = "testnet4" - coordinator.rpc = PublicChainRpc() # type: ignore[assignment] - # The extracted immutable snapshot defers live-boundary validation; - # the consuming methods read the live process environment at call - # time in the current tree, so the invalid values are staged there. - deferred = { - "PRISM_MIN_PEERS": "0", - "PRISM_TEMPLATE_MAX_AGE_SECONDS": "not-an-int", - } - with patch.dict("os.environ", deferred): - with self.assertRaisesRegex(SystemExit, "PRISM_MIN_PEERS must be positive"): - coordinator.validate_live_chain_identity() - - coordinator.current_template_artifacts = lambda: SimpleNamespace( # type: ignore[method-assign] - template={"curtime": int(time.time())}, - previousblockhash="11" * 32, - ) - with self.assertRaisesRegex( - SystemExit, "PRISM_TEMPLATE_MAX_AGE_SECONDS must be an integer" - ): - coordinator.validate_live_template_and_fee_policy() - - def test_disabled_ctv_settlement_ignores_invalid_settlement_only_values(self) -> None: - with tempfile.TemporaryDirectory() as temp_dir: - source = { - **minimal_environment(Path(temp_dir)), - "PRISM_CTV_SETTLEMENT_ENABLED": "0", - "PRISM_DIRECT_COINBASE_PAYOUT_FLOOR_BITS": "not-an-int", - "PRISM_RESERVED_COINBASE_OUTPUTS": "also-not-an-int", - } - coordinator = self.construct(source) - - with patch.dict( - "os.environ", - { - "PRISM_CTV_SETTLEMENT_ENABLED": "0", - "PRISM_DIRECT_COINBASE_PAYOUT_FLOOR_BITS": "not-an-int", - "PRISM_RESERVED_COINBASE_OUTPUTS": "also-not-an-int", - }, - ): - self.assertIsNone(coordinator.prism_ctv_settlement_config()) - - def test_enabled_ctv_settlement_validates_at_consuming_boundary(self) -> None: - with tempfile.TemporaryDirectory() as temp_dir: - source = { - **minimal_environment(Path(temp_dir)), - "PRISM_CTV_SETTLEMENT_ENABLED": "1", - "PRISM_DIRECT_COINBASE_PAYOUT_FLOOR_BITS": "not-an-int", - } - coordinator = self.construct(source) - - with patch.dict( - "os.environ", - { - "PRISM_CTV_SETTLEMENT_ENABLED": "1", - "PRISM_DIRECT_COINBASE_PAYOUT_FLOOR_BITS": "not-an-int", - }, - ): - with self.assertRaisesRegex( - SystemExit, "PRISM_DIRECT_COINBASE_PAYOUT_FLOOR_BITS must be an integer" - ): - coordinator.prism_ctv_settlement_config() - - def test_payout_validation_is_deferred_until_policy_is_consumed(self) -> None: - with tempfile.TemporaryDirectory() as temp_dir: - source = { - **minimal_environment(Path(temp_dir)), - "PRISM_PAYOUT_P2MR_SPEND_INPUT_BYTES": "not-an-int", - "PRISM_POOL_FEE_ENABLED": "", - } - coordinator = self.construct(source) - - with patch.dict( - "os.environ", - { - "PRISM_PAYOUT_P2MR_SPEND_INPUT_BYTES": "not-an-int", - "PRISM_POOL_FEE_ENABLED": "", - }, - ): - with self.assertRaisesRegex( - SystemExit, "PRISM_PAYOUT_P2MR_SPEND_INPUT_BYTES must be an integer" - ): - coordinator.prism_payout_policy() - - def test_explicit_disabled_broadcaster_defers_empty_settlement_flag(self) -> None: - with tempfile.TemporaryDirectory() as temp_dir: - source = { - **minimal_environment(Path(temp_dir)), - "PRISM_CTV_BROADCASTER_ENABLED": "0", - "PRISM_CTV_SETTLEMENT_ENABLED": "", - } - coordinator = self.construct(source) - - self.assertFalse(coordinator.ctv_broadcaster_enabled) - with patch.dict("os.environ", {"PRISM_CTV_SETTLEMENT_ENABLED": ""}): - with self.assertRaisesRegex( - SystemExit, "PRISM_CTV_SETTLEMENT_ENABLED is required" - ): - coordinator.prism_ctv_settlement_config() - - def test_broadcaster_default_still_consumes_settlement_flag_during_construction(self) -> None: - with tempfile.TemporaryDirectory() as temp_dir: - source = { - **minimal_environment(Path(temp_dir)), - "PRISM_CTV_SETTLEMENT_ENABLED": "", - } - with self.assertRaisesRegex( - SystemExit, "PRISM_CTV_SETTLEMENT_ENABLED is required" - ): - load_coordinator_config(source) - - def test_rpc_compatibility_reexport_points_to_leaf_class(self) -> None: - self.assertIs(CompatibilityJsonRpc, JsonRpc) - self.assertIs(DaemonJsonRpc, JsonRpc) - - def test_importing_rpc_leaf_does_not_import_coordinator(self) -> None: - result = subprocess.run( - [ - sys.executable, - "-c", - ( - "import sys; import lab.prism.rpc; " - "assert 'lab.prism.prism_coordinator' not in sys.modules" - ), - ], - check=False, - capture_output=True, - text=True, - ) - self.assertEqual(result.returncode, 0, result.stderr) - - -if __name__ == "__main__": - unittest.main() diff --git a/tests/test_prism_coordinator_job_cache.py b/tests/test_prism_coordinator_job_cache.py deleted file mode 100644 index 56304ddf0..000000000 --- a/tests/test_prism_coordinator_job_cache.py +++ /dev/null @@ -1,384 +0,0 @@ -#!/usr/bin/env python3 -"""Direct ownership tests for the extracted PRISM template and job services. - -These cases target the J1 owner boundary itself: symbol identity, descriptor -routing, template-artifact generation ordering, and bundle cache-admission -fencing through the real ``JobBundleService`` / -``TemplateArtifactRepository`` APIs. The publication-priority and scheduler -behavior itself is asserted by the upstream suites in -``tests/test_prism_job_builder.py`` and ``tests/test_prism_job_build_promises.py``. -""" - -from __future__ import annotations - -from collections import OrderedDict -from dataclasses import replace as dataclass_replace -import time -from types import SimpleNamespace -import unittest - -from lab.prism import job_bundle, prism_coordinator, template_artifacts -from lab.prism.job_bundle import CachedJobBundle, JobBuildKey -from lab.prism.prism_coordinator import PayoutStateArtifact -from lab.prism.template_artifacts import CachedTemplateArtifacts -from tests.prism_coordinator_test_support import base_template, coordinator - - -def cached_bundle( - artifacts: CachedTemplateArtifacts, - *, - payout_generation: int, - payout_artifact_sha256: str, - key: tuple[object, ...] = ("candidate",), - collection_only: bool = True, - payout_append_invalidation_epoch: int = 0, -) -> CachedJobBundle: - build_key = JobBuildKey( - best_tip_hash=artifacts.previousblockhash, - previous_block_hash=artifacts.previousblockhash, - template_fingerprint=artifacts.fingerprint, - template_generation=artifacts.generation, - payout_state_generation=payout_generation, - payout_artifact_sha256=payout_artifact_sha256, - mode="collection" if collection_only else "ready", - collection_identity=("miner", "22" * 32) if collection_only else None, - block_height=int(artifacts.template["height"]), - coinbase_value_sats=int(artifacts.template["coinbasevalue"]), - network_difficulty=artifacts.network_difficulty, - issued_at_ms=1, - payout_policy_sha256="payout", - ctv_settlement_sha256=None, - witness_merkle_sha256="witness", - transaction_set_sha256="transactions", - coinbase_suffix_hex="00", - signing_key_sha256="signing", - ledger_signing_key_sha256="ledger", - numeric_context_sha256="numeric", - payout_append_invalidation_epoch=payout_append_invalidation_epoch, - ) - return CachedJobBundle( - key=key, - template=artifacts.template, - template_fingerprint=artifacts.fingerprint, - coinbase_manifest={"coinbase_tx_hex": "00"}, - shares_json=[], - prior_balances=[], - found_block={"network_difficulty": artifacts.network_difficulty}, - collection_only=collection_only, - issued_at_ms=1, - base_job=SimpleNamespace(), # type: ignore[arg-type] - built_monotonic=time.monotonic(), - template_generation=artifacts.generation, - payout_state_generation=payout_generation, - collection_identity=build_key.collection_identity, - build_key=build_key, - ) - - -def seed_published_payout_artifact(server: object, sha256: str) -> int: - """Publish a payout artifact through the P1 owner and return the generation.""" - server._ensure_payout_state_service() # type: ignore[attr-defined] - generation = int(server._payout_state_generation) # type: ignore[attr-defined] - server._published_payout_state = dataclass_replace( # type: ignore[attr-defined] - server._published_payout_state, # type: ignore[attr-defined] - artifact=PayoutStateArtifact( - generation=generation, - source_generation=0, - prior_balances_json="[]", - prior_balances_sha256=sha256, - prepared_monotonic=time.monotonic(), - ), - ) - return generation - - -class MovedSymbolIdentityTests(unittest.TestCase): - def test_moved_symbol_reexports_preserve_identity(self) -> None: - self.assertIs(prism_coordinator.CachedJobBundle, job_bundle.CachedJobBundle) - # PR 80 removed the JobBuildKey compatibility re-export. - self.assertFalse(hasattr(prism_coordinator, "JobBuildKey")) - self.assertIs( - prism_coordinator.JobBuildSuperseded, job_bundle.JobBuildSuperseded - ) - self.assertIs(prism_coordinator._JobBuildRequest, job_bundle.JobBuildRequest) - self.assertIs(prism_coordinator._JobBuildFlight, job_bundle.JobBuildFlight) - self.assertIs( - prism_coordinator._JobBuildCancellation, job_bundle.JobBuildCancellation - ) - self.assertIs( - prism_coordinator._JobBundleBuildControl, - job_bundle.JobBundleBuildControl, - ) - self.assertIs( - prism_coordinator.CachedTemplateArtifacts, - template_artifacts.CachedTemplateArtifacts, - ) - self.assertIs( - prism_coordinator.QbitTipTemplateSnapshot, - template_artifacts.QbitTipTemplateSnapshot, - ) - # qbit_template_fingerprint intentionally remains a local coordinator - # compatibility definition at this layer; the owner carries its own - # copy with identical semantics (no reverse import). - template = base_template() - self.assertEqual( - prism_coordinator.qbit_template_fingerprint(template), - template_artifacts.qbit_template_fingerprint(template), - ) - - def test_service_owns_job_cache_state_and_descriptors_route(self) -> None: - server, _rpc = coordinator() - service = server._ensure_job_bundle_service() - - self.assertIs(server._job_cache_lock, service._job_cache_lock) - self.assertIs(server._job_bundle_cache, service._job_bundle_cache) - self.assertIs( - server._job_build_scheduler_lock, service._job_build_scheduler_lock - ) - self.assertIs(server.job_cache_hit_counts, service.job_cache_hit_counts) - # The repository deliberately shares the historical combined lock. - self.assertIs( - service.template_repository._job_cache_lock, - service._job_cache_lock, - ) - - replacement: OrderedDict[tuple[object, ...], CachedJobBundle] = OrderedDict() - server._job_bundle_cache = replacement - self.assertIs(service._job_bundle_cache, replacement) - - -class TemplateArtifactRepositoryTests(unittest.TestCase): - def test_repository_derive_reuses_expensive_fields_and_orders_stores(self) -> None: - server, _rpc = coordinator() - repository = server._ensure_job_bundle_service().template_repository - - first = repository.derive( - base_template(), generation=repository.reserve_generation() - ) - self.assertTrue(repository.store_artifacts(first)) - - second = repository.derive( - first.template, generation=repository.reserve_generation() - ) - - self.assertEqual(second.template, first.template) - # A same-fingerprint derivation reuses the expensive derived fields - # of the stored observation instead of recomputing them. - self.assertIs(second.transaction_hexes, first.transaction_hexes) - self.assertIs( - second.witness_merkle_leaves_hex, first.witness_merkle_leaves_hex - ) - self.assertTrue(repository.store_artifacts(second)) - self.assertFalse(repository.store_artifacts(first)) - self.assertIs(repository.current_artifacts(), second) - - def test_store_artifacts_prunes_stale_fingerprints_and_cancels_builds( - self, - ) -> None: - server, rpc = coordinator() - service = server._ensure_job_bundle_service() - repository = service.template_repository - old = server.store_template_artifacts(dict(rpc.template)) - assert old is not None - cancellations: list[tuple[str, bool]] = [] - server._cancel_obsolete_job_builds = ( # type: ignore[method-assign] - lambda reason, *, keep_published_snapshot=False: cancellations.append( - (reason, keep_published_snapshot) - ) - ) - with service._job_cache_lock: - service._job_bundle_cache = OrderedDict( - ( - (("old", 0), SimpleNamespace(template_fingerprint=old.fingerprint)), - (("other", 0), SimpleNamespace(template_fingerprint="ff" * 32)), - ) - ) - - replacement = repository.derive( - base_template(height=11, prevhash="22" * 32), - generation=repository.reserve_generation(), - ) - self.assertNotEqual(replacement.fingerprint, old.fingerprint) - self.assertTrue(repository.store_artifacts(replacement)) - - self.assertEqual( - cancellations, - [("template fingerprint superseded", True)], - ) - with service._job_cache_lock: - self.assertEqual(dict(service._job_bundle_cache), {}) - self.assertIs(repository.current_artifacts(), replacement) - - def test_repository_adoption_seeds_generation_like_first_touch(self) -> None: - server, _rpc = coordinator() - repository = server._ensure_job_bundle_service().template_repository - seeded = repository.derive(base_template(), generation=5) - - repository.adopt_template_artifacts(seeded) - - self.assertIs(repository.current_artifacts(), seeded) - self.assertEqual(repository.reserve_generation(), 6) - - repository.adopt_template_artifact_generation(10) - # An explicitly seeded counter is authoritative: later artifact - # adoption may replace the artifacts but not rewind the counter. - repository.adopt_template_artifacts( - repository.derive(base_template(), generation=3) - ) - self.assertEqual(repository.reserve_generation(), 11) - - -class JobBundleCacheAdmissionTests(unittest.TestCase): - def admission_fixture(self): # type: ignore[no-untyped-def] - server, rpc = coordinator() - service = server._ensure_job_bundle_service() - artifacts = server.store_template_artifacts(dict(rpc.template)) - assert artifacts is not None - generation = seed_published_payout_artifact(server, "payout-sha") - server._artifacts_buildable_locked = lambda _artifacts: True # type: ignore[method-assign] - server._published_snapshot_artifacts_locked = ( # type: ignore[method-assign] - lambda _artifacts: False - ) - return server, service, artifacts, generation - - def test_cache_admission_requires_current_payout_state(self) -> None: - server, service, artifacts, generation = self.admission_fixture() - - exact = cached_bundle( - artifacts, - payout_generation=generation, - payout_artifact_sha256="payout-sha", - key=("exact",), - ) - self.assertTrue(service._cache_job_bundle_if_current(exact, artifacts)) - with service._job_cache_lock: - self.assertIn(("exact",), service._job_bundle_cache) - - stale_generation = cached_bundle( - artifacts, - payout_generation=generation + 1, - payout_artifact_sha256="payout-sha", - key=("stale-generation",), - ) - self.assertFalse( - service._cache_job_bundle_if_current(stale_generation, artifacts) - ) - - stale_artifact = cached_bundle( - artifacts, - payout_generation=generation, - payout_artifact_sha256="different-sha", - key=("stale-artifact",), - ) - self.assertFalse( - service._cache_job_bundle_if_current(stale_artifact, artifacts) - ) - - stale_epoch = cached_bundle( - artifacts, - payout_generation=generation, - payout_artifact_sha256="payout-sha", - key=("stale-epoch",), - collection_only=False, - payout_append_invalidation_epoch=7, - ) - self.assertFalse( - service._cache_job_bundle_if_current(stale_epoch, artifacts) - ) - - with service._job_cache_lock: - self.assertEqual(list(service._job_bundle_cache), [("exact",)]) - - def test_cache_admission_requires_current_template_or_pinned_snapshot( - self, - ) -> None: - server, service, old, generation = self.admission_fixture() - repository = service.template_repository - stale_bundle = cached_bundle( - old, - payout_generation=generation, - payout_artifact_sha256="payout-sha", - key=("old-after-publication",), - ) - server._cancel_obsolete_job_builds = ( # type: ignore[method-assign] - lambda _reason, **_kwargs: None - ) - replacement = repository.derive( - base_template(height=11, prevhash="22" * 32), - generation=repository.reserve_generation(), - ) - self.assertTrue(repository.store_artifacts(replacement)) - - # An old completion cannot cache after a newer fingerprint published. - self.assertFalse( - service._cache_job_bundle_if_current(stale_bundle, old) - ) - with service._job_cache_lock: - self.assertNotIn(("old-after-publication",), service._job_bundle_cache) - - # The sole exception is a pinned rebuild of exactly the published - # snapshot, which direct issuance would otherwise rebuild all window. - server._published_snapshot_artifacts_locked = ( # type: ignore[method-assign] - lambda candidate: candidate.fingerprint == old.fingerprint - ) - self.assertTrue(service._cache_job_bundle_if_current(stale_bundle, old)) - with service._job_cache_lock: - self.assertIn(("old-after-publication",), service._job_bundle_cache) - - def test_collection_admission_requires_exact_generation_unlike_ready( - self, - ) -> None: - server, service, artifacts, generation = self.admission_fixture() - repository = service.template_repository - # A same-fingerprint, newer-generation observation of the same - # template supersedes the stored one without a fingerprint change. - newer = repository.derive( - artifacts.template, generation=repository.reserve_generation() - ) - self.assertTrue(repository.store_artifacts(newer)) - - old_collection = cached_bundle( - artifacts, - payout_generation=generation, - payout_artifact_sha256="payout-sha", - key=("old-collection",), - collection_only=True, - ) - old_ready = cached_bundle( - artifacts, - payout_generation=generation, - payout_artifact_sha256="payout-sha", - key=("old-ready",), - collection_only=False, - ) - - self.assertFalse( - service._cache_job_bundle_if_current(old_collection, artifacts) - ) - self.assertTrue(service._cache_job_bundle_if_current(old_ready, artifacts)) - with service._job_cache_lock: - self.assertNotIn(("old-collection",), service._job_bundle_cache) - self.assertIn(("old-ready",), service._job_bundle_cache) - - -class JobBuildExecutorOwnershipTests(unittest.TestCase): - def test_executor_is_single_flight_named_and_shutdown_is_terminal(self) -> None: - server, _rpc = coordinator() - service = server._ensure_job_bundle_service() - - executor = service._job_build_executor_locked() - self.assertIs(service._job_build_executor_locked(), executor) - self.assertEqual(executor._thread_name_prefix, "prism-job-build") - - service.shutdown_job_build_executor() - - self.assertTrue(service._job_build_executor_shutdown) - with self.assertRaisesRegex(RuntimeError, "shut down"): - service._job_build_executor_locked() - # The coordinator facade delegates to the same owner state. - with self.assertRaisesRegex(RuntimeError, "shut down"): - server._job_build_executor_locked() - - -if __name__ == "__main__": - unittest.main() diff --git a/tests/test_prism_coordinator_metrics.py b/tests/test_prism_coordinator_metrics.py deleted file mode 100644 index 7cf402e0a..000000000 --- a/tests/test_prism_coordinator_metrics.py +++ /dev/null @@ -1,1198 +0,0 @@ -#!/usr/bin/env python3 -"""Focused PRISM coordinator metrics tests.""" -# ruff: noqa: F821 - -from __future__ import annotations - -from pathlib import Path -import tempfile -import unittest -from unittest import mock -from lab.prism import audit_http as audit_http_module -from tests import prism_coordinator_test_support as _job_support -from tests import prism_vardiff_test_support as _vardiff_support - - -def install_isolated_coordinator( - test_case: unittest.TestCase, - factory: object, -) -> None: - assert callable(factory) - - def isolated(*args: object, **kwargs: object) -> object: - result = factory(*args, **kwargs) - server = result[0] if isinstance(result, tuple) else result - temporary = tempfile.TemporaryDirectory() - server.audit_dir = Path(temporary.name) / "audit" - server.evidence_path = Path(temporary.name) / "state" / "evidence.json" - - def cleanup() -> None: - store = server.__dict__.get("_audit_artifact_store") - if store is not None: - store.close() - temporary.cleanup() - - test_case.addCleanup(cleanup) - return result - - globals()["coordinator"] = isolated - - -class _VardiffSupportTestCase(unittest.TestCase): - def setUp(self) -> None: - globals().update( - {name: getattr(_vardiff_support, name) for name in _vardiff_support.__all__} - ) - install_isolated_coordinator(self, _vardiff_support.coordinator) - - -class _JobSupportTestCase(unittest.TestCase): - def setUp(self) -> None: - globals().update( - {name: getattr(_job_support, name) for name in _job_support.__all__} - ) - install_isolated_coordinator(self, _job_support.coordinator) - - -class PrismCoordinatorVardiffTests(_VardiffSupportTestCase): - def test_metrics_include_issue_scope_operational_gauges(self) -> None: - server = coordinator() - server.submitted_share_count = 10 - server.stale_share_count = 2 - server.duplicate_share_count = 1 - server.low_difficulty_share_count = 3 - server.grace_credited_share_count = 6 - server.idle_retarget_count = 7 - server.rejection_counts_by_reason[PRISM_REJECTION_STALE_JOB] = 2 - server.rejection_counts_by_reason["duplicate-share"] = 1 - server.rejection_counts_by_reason["low-difficulty"] = 3 - server.tip_refresh_job_count = 4 - server.post_accept_refresh_failure_count = 5 - server.connection_limit_rejection_counts = {"global": 2, "username": 3} - server.accept_resource_exhaustion_count = 4 - server.connection_setup_failure_count = 5 - server._ensure_vardiff_idle_state() - with server._vardiff_idle_lock: - server.vardiff_idle_clients_inspected = 8 - server.vardiff_idle_skip_counts["busy"] = 2 - server.vardiff_idle_queue_depth = 1 - server.vardiff_idle_inflight = 2 - server.vardiff_idle_task_failures = 1 - server._observe_vardiff_idle_seconds("sweep", 0.005) - server._observe_vardiff_idle_seconds("task", 0.01) - - metrics = server.metrics_payload() - - self.assertIn("qbit_prism_submitted_shares_total 10", metrics) - self.assertIn("qbit_prism_stale_shares_total 2", metrics) - self.assertIn("qbit_prism_duplicate_shares_total 1", metrics) - self.assertIn("qbit_prism_low_difficulty_shares_total 3", metrics) - self.assertIn("qbit_prism_grace_credited_shares_total 6", metrics) - self.assertIn("qbit_prism_stratum_active_connections 0", metrics) - self.assertIn("qbit_prism_stratum_connection_limit 384", metrics) - self.assertIn("qbit_prism_stratum_peak_active_connections 0", metrics) - self.assertIn("qbit_prism_stratum_pending_initial_jobs 0", metrics) - self.assertIn("qbit_prism_stratum_pending_initial_job_limit 128", metrics) - self.assertIn( - "qbit_prism_stratum_oldest_genuinely_pending_initial_job_seconds 0.0", - metrics, - ) - self.assertIn( - "qbit_prism_stratum_current_tip_coverage_gap_seconds 0.0", - metrics, - ) - self.assertIn("qbit_prism_stratum_current_tip_job_coverage 1.0", metrics) - self.assertIn("qbit_prism_stratum_handler_threads 0", metrics) - self.assertIn("qbit_prism_job_delivery_queue_depth 0", metrics) - self.assertIn("qbit_prism_job_delivery_active_workers 0", metrics) - self.assertIn( - 'qbit_prism_stratum_connection_limit_rejections_total{scope="global"} 2', - metrics, - ) - self.assertIn( - 'qbit_prism_stratum_connection_limit_rejections_total{scope="username"} 3', - metrics, - ) - self.assertIn("qbit_prism_stratum_accept_resource_exhaustions_total 4", metrics) - self.assertIn("qbit_prism_stratum_connection_setup_failures_total 5", metrics) - self.assertIn('qbit_prism_rejections_total{reason_id="stale-job"} 2', metrics) - self.assertIn('qbit_prism_rejections_total{reason_id="duplicate-share"} 1', metrics) - self.assertIn('qbit_prism_rejections_total{reason_id="low-difficulty"} 3', metrics) - self.assertIn("qbit_prism_tip_refresh_jobs_total 4", metrics) - self.assertIn("qbit_prism_post_accept_refresh_failures_total 5", metrics) - self.assertIn("qbit_prism_vardiff_idle_retargets_total 7", metrics) - self.assertIn("qbit_prism_vardiff_idle_clients_inspected_total 8", metrics) - self.assertIn('qbit_prism_vardiff_idle_skips_total{reason="busy"} 2', metrics) - self.assertIn("qbit_prism_vardiff_idle_queue_depth 1", metrics) - self.assertIn("qbit_prism_vardiff_idle_inflight 2", metrics) - self.assertIn("qbit_prism_vardiff_idle_task_failures_total 1", metrics) - self.assertIn("qbit_prism_vardiff_idle_sweep_seconds_count 1", metrics) - self.assertIn("qbit_prism_vardiff_idle_retarget_task_seconds_count 1", metrics) - self.assertIn("qbit_prism_stale_share_percent 20", metrics) - self.assertIn("qbit_prism_coinbase_weight_headroom_bytes 1999750", metrics) - self.assertIn("qbit_prism_vardiff_enabled 1", metrics) - self.assertIn("qbit_prism_qbitd_initial_block_download 0", metrics) - self.assertIn("qbit_prism_qbitd_peers 4", metrics) - - def test_hot_lock_and_pending_candidate_metrics_are_bounded(self) -> None: - server = coordinator() - server.lock = _ObservedRLock() - lock_wait_started = threading.Event() - - def contend() -> None: - lock_wait_started.set() - with server.lock: - pass - - with server.lock: - waiter = threading.Thread(target=contend) - waiter.start() - self.assertTrue(lock_wait_started.wait(1)) - time.sleep(0.02) - waiter.join(1) - self.assertFalse(waiter.is_alive()) - - ledger = SingleWriterShareLedger() - block_hash = "bc" * 32 - ledger.persist_block_candidate_intent( - { - "schema": "qbit.prism.block-candidate-intent.v1", - "block_hash_hex": block_hash, - "block_hex": "00", - } - ) - server.ledger = ledger - before_attempt = "\n".join( - server.coordinator_lock_metrics_lines() - + server.block_submitter_metrics_lines() - ) - self.assertIn("qbit_prism_coordinator_lock_contentions_total 1", before_attempt) - self.assertIn("qbit_prism_block_candidates_pending 1", before_attempt) - self.assertRegex( - before_attempt, - r"qbit_prism_block_candidate_oldest_unattempted_seconds 0\.\d+", - ) - - self.assertTrue(ledger.mark_block_candidate_attempted(block_hash=block_hash)) - after_attempt = "\n".join(server.block_submitter_metrics_lines()) - self.assertIn( - "qbit_prism_block_candidate_oldest_unattempted_seconds 0.000000", - after_attempt, - ) - - def test_coordinator_lock_counts_every_granted_acquisition(self) -> None: - server = coordinator() - server.lock = _ObservedRLock() - - with server.lock: - # Re-entrant re-acquisition is an acquire() call and counts as one: - # the contended counter is per call, so the denominator must be too. - with server.lock: - pass - self.assertTrue(server.lock.acquire(blocking=False)) - server.lock.release() - - acquisitions, contentions, _sum, _max = ( - server.coordinator_lock_contention_snapshot() - ) - self.assertEqual((acquisitions, contentions), (3, 0)) - - lock_wait_started = threading.Event() - - def contend() -> None: - lock_wait_started.set() - with server.lock: - pass - - with server.lock: - waiter = threading.Thread(target=contend) - waiter.start() - self.assertTrue(lock_wait_started.wait(1)) - time.sleep(0.02) - waiter.join(1) - self.assertFalse(waiter.is_alive()) - - acquisitions, contentions, _sum, _max = ( - server.coordinator_lock_contention_snapshot() - ) - # The waiter's grant is one acquisition and one contention; the holder's - # own acquisition was uncontended. - self.assertEqual((acquisitions, contentions), (5, 1)) - - metrics = "\n".join(server.coordinator_lock_metrics_lines()) - self.assertIn("qbit_prism_coordinator_lock_acquisitions_total 5", metrics) - self.assertIn("qbit_prism_coordinator_lock_contentions_total 1", metrics) - - def test_coordinator_lock_timeout_is_contention_without_acquisition(self) -> None: - server = coordinator() - server.lock = _ObservedRLock() - timed_out = threading.Event() - - def give_up() -> None: - if not server.lock.acquire(timeout=0.05): - timed_out.set() - - with server.lock: - waiter = threading.Thread(target=give_up) - waiter.start() - waiter.join(1) - self.assertFalse(waiter.is_alive()) - self.assertTrue(timed_out.is_set()) - - acquisitions, contentions, _sum, _max = ( - server.coordinator_lock_contention_snapshot() - ) - # Only the holder acquired. The abandoned wait is real contention, which - # is why the derived contended fraction is an upper bound. - self.assertEqual((acquisitions, contentions), (1, 1)) - - def test_accepted_share_rate_is_left_to_the_counter(self) -> None: - server = coordinator() - - metrics = server.metrics_payload() - - # The gauge divided the ledger's lifetime accepted count by this - # process's uptime. Removed rather than redefined: a series that - # disappears is loud, a series that silently changes meaning is not. - self.assertNotIn("qbit_prism_shares_per_second", metrics) - self.assertRegex(metrics, r"\nqbit_prism_accepted_shares_total \d+\n") - self.assertIn("qbit_prism_vardiff_lane_accepted_shares_total", metrics) - - def test_metrics_include_bounded_worker_share_and_rejection_counters(self) -> None: - server = coordinator() - server.worker_metrics_limit = 1 - - server.note_worker_submitted_share("miner-a") - server.note_worker_accepted_share("miner-a", PRISM_CREDIT_POLICY_STALE_GRACE) - server.note_worker_submitted_share("miner-b") - server.record_rejection(PRISM_REJECTION_LOW_DIFFICULTY, worker="miner-b") - - metrics = server.metrics_payload() - - self.assertIn('qbit_prism_worker_submitted_shares_total{worker="miner-a"} 1', metrics) - self.assertIn('qbit_prism_worker_accepted_shares_total{worker="miner-a"} 1', metrics) - self.assertIn('qbit_prism_worker_grace_credited_shares_total{worker="miner-a"} 1', metrics) - self.assertIn('qbit_prism_worker_submitted_shares_total{worker="_other"} 1', metrics) - self.assertIn( - 'qbit_prism_worker_rejections_total{worker="_other",reason_id="low-difficulty"} 1', - metrics, - ) - - def test_metrics_include_ctv_broadcaster_progress_and_pass_duration(self) -> None: - server = coordinator() - server._record_ctv_fanout_broadcaster_progress() - server._record_ctv_fanout_broadcaster_progress() - server.observe_ctv_fanout_broadcaster_pass(102.0) - server.observe_ctv_fanout_broadcaster_chunk( - SimpleNamespace(processed_count=1, elapsed_seconds=0.25) - ) - server._record_ctv_fanout_broadcaster_yield() - - metrics = server.metrics_payload() - - self.assertIn("qbit_prism_ctv_fanout_broadcaster_processed_rows_total 2", metrics) - self.assertIn( - 'qbit_prism_ctv_fanout_broadcaster_pass_seconds_bucket{le="60"} 0', - metrics, - ) - self.assertIn( - 'qbit_prism_ctv_fanout_broadcaster_pass_seconds_bucket{le="120"} 1', - metrics, - ) - self.assertIn("qbit_prism_ctv_fanout_broadcaster_pass_seconds_sum 102.000000", metrics) - self.assertIn("qbit_prism_ctv_fanout_broadcaster_pass_seconds_count 1", metrics) - self.assertIn( - "qbit_prism_ctv_fanout_broadcaster_tip_refresh_yields_total 1", - metrics, - ) - self.assertIn( - 'qbit_prism_ctv_fanout_broadcaster_chunk_seconds_bucket{le="0.25"} 1', - metrics, - ) - self.assertIn( - 'qbit_prism_ctv_fanout_broadcaster_chunk_rows_bucket{le="1"} 1', - metrics, - ) - - def test_zero_worker_metric_limit_uses_overflow_bucket(self) -> None: - server = coordinator() - server.worker_metrics_limit = 0 - - server.note_worker_submitted_share("miner-a") - - self.assertEqual(set(server.worker_share_counts), {"_other"}) - self.assertEqual(server.worker_share_counts["_other"]["submitted"], 1) - - def test_unauthorized_submit_does_not_admit_payload_worker_metric_label(self) -> None: - server, state, _ledger = submit_coordinator() - server.worker_metrics_limit = 1 - - with self.assertRaises(StratumError) as raised: - server.handle_submit( - state, - ["spoofed-miner", "job-1", "00" * 8, "00000001", "00000002"], - ) - - self.assertEqual(raised.exception.reason, PRISM_REJECTION_UNAUTHORIZED_WORKER) - self.assertNotIn("spoofed-miner", server.worker_share_counts) - self.assertEqual(server.worker_share_counts["miner-a"]["submitted"], 0) - self.assertEqual( - server.worker_rejection_counts[("miner-a", PRISM_REJECTION_UNAUTHORIZED_WORKER)], - 1, - ) - - server.note_worker_submitted_share("miner-a") - - self.assertNotIn(PRISM_WORKER_METRICS_OVERFLOW_LABEL, server.worker_share_counts) - self.assertEqual(server.worker_share_counts["miner-a"]["submitted"], 1) - - def test_metrics_include_audit_artifact_storage_gauges(self) -> None: - server = coordinator() - with tempfile.TemporaryDirectory() as tempdir: - server.audit_dir = Path(tempdir) - files = { - f"prism-audit-bundle-body-{'aa' * 32}-{'bb' * 32}.json": b"abc", - f"prism-audit-share-segment-1-1-{'cc' * 32}.json": b"defg", - f"prism-live-audit-bundle-1-{'dd' * 32}.json": b"hi", - f"prism-live-audit-bundle-candidate-{'ee' * 32}.json": b"j", - f".prism-live-audit-bundle-candidate-{'ff' * 32}.json.tmp": b"klmno", - "operator-note.txt": b"pqrstu", - } - for name, body in files.items(): - (Path(tempdir) / name).write_bytes(body) - - metrics = server.metrics_payload() - - self.assertIn('qbit_prism_audit_artifact_bytes{kind="body"} 3', metrics) - self.assertIn('qbit_prism_audit_artifact_files{kind="body"} 1', metrics) - self.assertIn('qbit_prism_audit_artifact_bytes{kind="share_segment"} 4', metrics) - self.assertIn('qbit_prism_audit_artifact_files{kind="share_segment"} 1', metrics) - self.assertIn('qbit_prism_audit_artifact_bytes{kind="live_bundle"} 2', metrics) - self.assertIn('qbit_prism_audit_artifact_files{kind="live_bundle"} 1', metrics) - self.assertIn('qbit_prism_audit_artifact_bytes{kind="candidate"} 6', metrics) - self.assertIn('qbit_prism_audit_artifact_files{kind="candidate"} 2', metrics) - self.assertIn('qbit_prism_audit_artifact_bytes{kind="other"} 6', metrics) - self.assertIn('qbit_prism_audit_artifact_files{kind="other"} 1', metrics) - self.assertIn("qbit_prism_audit_artifact_scan_error 0", metrics) - - def test_metrics_payload_reads_one_a1_snapshot(self) -> None: - server = coordinator() - snapshot = { - "body": {"files": 1, "bytes": 2}, - "share_segment": {"files": 3, "bytes": 4}, - "live_bundle": {"files": 5, "bytes": 6}, - "candidate": {"files": 7, "bytes": 8}, - "other": {"files": 9, "bytes": 10}, - "scan_error": 0, - } - reader = mock.Mock(return_value=snapshot) - server.audit_artifact_metrics = reader # type: ignore[method-assign] - - metrics = server.metrics_payload() - - reader.assert_called_once_with() - self.assertIn('qbit_prism_audit_artifact_files{kind="body"} 1', metrics) - self.assertIn('qbit_prism_audit_artifact_bytes{kind="other"} 10', metrics) - self.assertIn("qbit_prism_audit_artifact_scan_error 0", metrics) - - def _pending_append(self, tag: str, accepted_at_ms: int = 2) -> PendingShareAppend: - from lab.prism.share_ledger import PendingShare - - return PendingShareAppend( - pending_share=PendingShare( - share_id=f"miner-a:{tag}", - miner_id="miner-a", - order_key="miner-a", - p2mr_program_hex="11" * 32, - share_difficulty=1, - network_difficulty=1, - template_height=10, - job_id="job-1", - job_issued_at_ms=1, - accepted_at_ms=accepted_at_ms, - ntime=1_700_000_000, - ), - username="miner-a", - job_id="job-1", - block_hash_hex=tag * 32, - collection_only=False, - credit_policy=None, - ) - - def test_share_ack_latency_histogram_observes_accept_and_reject(self) -> None: - # The read-to-ack instrument for share-ingest saturation, driven - # through the real handler loop: both outcomes are observed after - # their response write, so accepted-vs-rejected separates commit - # pressure from transport/thread saturation. - tip = "00" * 32 - server, state, _ledger = submit_coordinator(tip=tip) - sent: list[dict[str, object]] = [] - state.send = lambda payload: sent.append(payload) # type: ignore[method-assign] - submission = SimpleNamespace( - header_hex="af" * 80, - block_hash_hex="cf" * 32, - share_pass=True, - block_pass=False, - ) - coordinator_sock, peer = socket.socketpair() - state.sock = coordinator_sock - accepted_line = json.dumps( - { - "id": 7, - "method": "mining.submit", - "params": ["miner-a", "job-1", "00" * 8, "00000001", "00000002"], - } - ) - rejected_line = json.dumps( - { - "id": 8, - "method": "mining.submit", - "params": [ - "miner-a", - "missing-job", - "00" * 8, - "00000001", - "00000002", - ], - } - ) - - with patch( - "lab.prism.prism_coordinator.direct_stratum.assemble_submission", - return_value=submission, - ): - handler = threading.Thread(target=server.handle_client, args=(state,)) - handler.start() - peer.sendall((accepted_line + "\n" + rejected_line + "\n").encode()) - peer.close() - handler.join(5) - - self.assertFalse(handler.is_alive()) - self.assertEqual(sent[0], {"id": 7, "result": True, "error": None}) - self.assertEqual(sent[1]["id"], 8) - self.assertIsNotNone(sent[1]["error"]) - metrics = "\n".join(server.share_ack_metrics_lines()) - self.assertIn( - 'qbit_prism_share_ack_seconds_count{result="accepted"} 1', - metrics, - ) - self.assertIn( - 'qbit_prism_share_ack_seconds_count{result="rejected"} 1', - metrics, - ) - - def test_block_submit_histogram_measures_landed_to_rpc_interval(self) -> None: - # The race-critical span (candidate landed -> submitblock returned) - # must be observed exactly once per attempted submit, independent of - # the post-submit audit/persistence bookkeeping the outbox - # created_at/completed_at interval also includes. - old_tip = "00" * 32 - block_hash = "ab" * 32 - server, state, ledger = submit_coordinator(tip=old_tip) - server.stop_after_block = False - server.max_blocks = 10 - server.clients = {state} - sent: list[dict[str, object]] = [] - state.send = lambda payload: sent.append(payload) # type: ignore[method-assign] - - def build_fresh_job(client: ClientState, *, clean_jobs: bool) -> object: - return prism_context( - "fresh-job", - block_hash, - worker=state.worker, - difficulty=Decimal("1"), - clean_jobs=clean_jobs, - ) - - server.build_job_for_client = build_fresh_job # type: ignore[method-assign] - submission = SimpleNamespace( - header_hex="aa" * 80, - share_pass=True, - block_pass=True, - coinbase_tx_hex="c0ffee", - block_hash_hex=block_hash, - block_hex="00", - ) - - with tempfile.TemporaryDirectory() as tempdir: - server.audit_dir = Path(tempdir) - server.evidence_path = Path(tempdir) / "evidence.json" - server.ledger_writer_public_key_hex = "aa" * 32 - server.rpc = SubmitAcceptingTemplateRpc( - old_tip=old_tip, - block_hash=block_hash, - ledger=ledger, - ) - server.build_audit_bundle = lambda **_kwargs: verified_block_bundle() # type: ignore[method-assign] - server.verify_bundle = lambda *_args, **_kwargs: verified_audit_report() # type: ignore[method-assign] - - with patch( - "lab.prism.prism_coordinator.direct_stratum.assemble_submission", - return_value=submission, - ): - server.handle_submit( - state, - ["miner-a", "job-1", "00" * 8, "00000001", "00000002"], - ) - self.assertTrue(server.submit_next_block_candidate()) - - self.assertEqual(server.accepted_block_count, 1) - metrics = "\n".join(server.block_submitter_metrics_lines()) - self.assertIn("qbit_prism_block_submit_seconds_count 1", metrics) - self.assertIn( - 'qbit_prism_block_submit_seconds_bucket{le="+Inf"} 1', - metrics, - ) - # The candidate landed moments ago in this process, so the interval - # must fall inside the finite buckets, not only the +Inf overflow. - self.assertIn( - 'qbit_prism_block_submit_seconds_bucket{le="30"} 1', - metrics, - ) - - -class PrismCoordinatorReliabilityTests(_VardiffSupportTestCase): - def _bare_coordinator(self) -> PrismCoordinator: - server = PrismCoordinator.__new__(PrismCoordinator) - server.lock = threading.RLock() - server.stop_event = threading.Event() - server._heartbeats = {} - server._watchdog_pauses = {} - server._heartbeats_lock = threading.Lock() - server.watchdog_timeout_seconds = 120.0 - server.watchdog_interval_seconds = 15.0 - return server - - def test_overdue_heartbeats_flags_only_stale_subsystems(self) -> None: - server = self._bare_coordinator() - server._record_heartbeat("stratum_accept") - server._record_heartbeat("qbit_blockpoll") - now = time.monotonic() - - self.assertEqual(server._overdue_heartbeats(now), []) - - with server._heartbeats_lock: - server._heartbeats["qbit_blockpoll"] = now - 1_000.0 - - self.assertEqual(server._overdue_heartbeats(now), ["qbit_blockpoll"]) - - def test_block_submitter_retry_wait_heartbeats_in_bounded_slices(self) -> None: - server = self._bare_coordinator() - server.watchdog_timeout_seconds = 0.3 - clock = {"now": 0.0} - waits: list[float] = [] - overdue_samples: list[list[str]] = [] - - class AdvancingStopEvent: - def is_set(self) -> bool: - return False - - def wait(self, timeout: float) -> bool: - waits.append(timeout) - clock["now"] += timeout - overdue_samples.append(server._overdue_heartbeats(clock["now"])) - return False - - server.stop_event = AdvancingStopEvent() # type: ignore[assignment] - with patch( - "lab.prism.block_candidates.time.monotonic", - side_effect=lambda: clock["now"], - ): - self.assertFalse(server._wait_for_block_candidate_retry(1.0)) - - self.assertEqual(waits, [0.25, 0.25, 0.25, 0.25]) - self.assertEqual(overdue_samples, [[], [], [], []]) - self.assertEqual( - server._heartbeats["block_submitter"], - 1.0, - ) - - def test_blocked_candidate_phase_remains_watchdog_eligible(self) -> None: - server, state, _recording = submit_coordinator() - server._heartbeats = {} - server._watchdog_pauses = {} - server._heartbeats_lock = threading.Lock() - server.watchdog_timeout_seconds = 0.05 - entered_submission = threading.Event() - release_submission = threading.Event() - candidate = block_candidate( - server, - state, - SimpleNamespace( - coinbase_tx_hex="00", - block_hash_hex="fa" * 32, - block_hex="00", - share_pass=True, - block_pass=True, - ), - ) - - def blocked_submission(_candidate: PrismBlockCandidate) -> bool: - entered_submission.set() - release_submission.wait(2) - return True - - server.submit_block_candidate = blocked_submission # type: ignore[method-assign] - server.enqueue_block_candidate(candidate) - server._record_heartbeat("block_submitter") - submitter = threading.Thread(target=server.submit_next_block_candidate) - submitter.start() - try: - self.assertTrue(entered_submission.wait(1)) - time.sleep(0.08) - self.assertEqual( - server._overdue_heartbeats(time.monotonic()), - ["block_submitter"], - ) - finally: - release_submission.set() - submitter.join(2) - self.assertFalse(submitter.is_alive()) - - def test_progressing_ctv_pass_longer_than_watchdog_timeout_stays_healthy(self) -> None: - server = self._bare_coordinator() - server.ctv_broadcaster_limit = 200 - server.ctv_broadcaster_interval_seconds = 30.0 - clock = {"now": 0.0} - overdue_samples: list[list[str]] = [] - seen_limits: list[int] = [] - - class StopAfterOnePass: - def is_set(self) -> bool: - return False - - def wait(self, timeout: float) -> bool: - return True - - class ProgressingDaemon: - def run_once(self, *, limit: int, progress_callback: object, **_kwargs: object) -> object: - seen_limits.append(limit) - assert callable(progress_callback) - for _ in range(3): - clock["now"] += 80.0 - overdue_samples.append(server._overdue_heartbeats(clock["now"])) - progress_callback() - return SimpleNamespace( - scanned_count=3, - submitted_count=0, - updated_count=3, - failed_count=0, - yielded_to_tip_refresh=False, - ) - - server.stop_event = StopAfterOnePass() # type: ignore[assignment] - server.ctv_fanout_broadcast_daemon = ProgressingDaemon() - - with patch("lab.prism.prism_coordinator.time.monotonic", side_effect=lambda: clock["now"]), patch( - "builtins.print" - ): - server.ctv_fanout_broadcaster_loop() - - self.assertGreater(clock["now"], server.watchdog_timeout_seconds) - self.assertEqual(seen_limits, [200]) - self.assertEqual(overdue_samples, [[], [], []]) - self.assertEqual(server.ctv_broadcaster_processed_rows_total, 3) - self.assertEqual(server.ctv_broadcaster_pass_count, 1) - - def test_ctv_pass_completion_heartbeat_precedes_interval_wait(self) -> None: - server = self._bare_coordinator() - server.ctv_broadcaster_limit = 200 - server.ctv_broadcaster_interval_seconds = 30.0 - clock = {"now": 0.0} - wait_observation: dict[str, object] = {} - - class StopAfterIntervalWait: - def is_set(self) -> bool: - return False - - def wait(self, timeout: float) -> bool: - wait_observation["timeout"] = timeout - wait_observation["heartbeat"] = server._heartbeats["ctv_fanout_broadcaster"] - wait_observation["overdue_after_wait"] = server._overdue_heartbeats( - clock["now"] + timeout - ) - return True - - class IncidentDurationDaemon: - def run_once(self, *, limit: int, progress_callback: object, **_kwargs: object) -> object: - clock["now"] += 102.0 - return SimpleNamespace( - scanned_count=200, - submitted_count=0, - updated_count=200, - failed_count=0, - yielded_to_tip_refresh=False, - ) - - server.stop_event = StopAfterIntervalWait() # type: ignore[assignment] - server.ctv_fanout_broadcast_daemon = IncidentDurationDaemon() - - with patch("lab.prism.prism_coordinator.time.monotonic", side_effect=lambda: clock["now"]), patch( - "builtins.print" - ): - server.ctv_fanout_broadcaster_loop() - - self.assertEqual(wait_observation["timeout"], 30.0) - self.assertEqual(wait_observation["heartbeat"], 102.0) - self.assertEqual(wait_observation["overdue_after_wait"], []) - - def test_ctv_pass_without_progress_remains_watchdog_eligible(self) -> None: - server = self._bare_coordinator() - server.ctv_broadcaster_limit = 200 - server.ctv_broadcaster_interval_seconds = 30.0 - clock = {"now": 0.0} - entered_row = threading.Event() - release_row = threading.Event() - - class BlockingDaemon: - def run_once(self, *, limit: int, progress_callback: object, **_kwargs: object) -> object: - entered_row.set() - release_row.wait() - return SimpleNamespace( - scanned_count=1, - submitted_count=0, - updated_count=1, - failed_count=0, - yielded_to_tip_refresh=False, - ) - - server.ctv_fanout_broadcast_daemon = BlockingDaemon() - broadcaster_thread = threading.Thread(target=server.ctv_fanout_broadcaster_loop) - with patch("lab.prism.prism_coordinator.time.monotonic", side_effect=lambda: clock["now"]), patch( - "builtins.print" - ): - broadcaster_thread.start() - self.assertTrue(entered_row.wait(timeout=1.0)) - clock["now"] = server.watchdog_timeout_seconds + 1.0 - self.assertEqual( - server._overdue_heartbeats(clock["now"]), - ["ctv_fanout_broadcaster"], - ) - server.stop_event.set() - release_row.set() - broadcaster_thread.join(timeout=1.0) - - self.assertFalse(broadcaster_thread.is_alive()) - - def test_watchdog_pause_suppresses_known_long_critical_section(self) -> None: - server = self._bare_coordinator() - server._record_heartbeat("stratum_accept") - server._record_heartbeat("qbit_blockpoll") - now = time.monotonic() - with server._heartbeats_lock: - server._heartbeats["stratum_accept"] = now - 1_000.0 - server._heartbeats["qbit_blockpoll"] = now - 1_000.0 - - self.assertEqual(server._overdue_heartbeats(now), ["qbit_blockpoll", "stratum_accept"]) - - with server._watchdog_paused("qbit_blockpoll", "stratum_accept"): - self.assertEqual(server._overdue_heartbeats(now + 1_000.0), []) - - self.assertEqual(server._overdue_heartbeats(time.monotonic()), []) - - def test_block_submit_pause_names_cover_registered_refresh_and_idle_threads(self) -> None: - server = self._bare_coordinator() - for name in ("stratum_accept", "qbit_blockpoll", "qbit_blockwait", "vardiff_idle_sweep"): - server._record_heartbeat(name) - now = time.monotonic() - with server._heartbeats_lock: - for name in server._heartbeats: - server._heartbeats[name] = now - 1_000.0 - - pause_names = server._registered_watchdog_heartbeat_names( - "qbit_blockpoll", - "qbit_blockwait", - "vardiff_idle_sweep", - "stratum_accept", - ) - - with server._watchdog_paused(*pause_names): - self.assertEqual(server._overdue_heartbeats(now + 1_000.0), []) - - def test_pause_names_skip_removed_blockwait_without_resurrecting_heartbeat(self) -> None: - server = self._bare_coordinator() - server._record_heartbeat("qbit_blockpoll") - server._record_heartbeat("qbit_blockwait") - server._remove_watchdog_heartbeat("qbit_blockwait") - - pause_names = server._registered_watchdog_heartbeat_names("qbit_blockpoll", "qbit_blockwait") - - self.assertEqual(pause_names, ("qbit_blockpoll",)) - with server._watchdog_paused(*pause_names): - pass - self.assertNotIn("qbit_blockwait", server._heartbeats) - - def test_wedged_evidence_read_stays_watchdog_eligible(self) -> None: - server = self._bare_coordinator() - clock = {"now": 1000.0} - entered = threading.Event() - release = threading.Event() - - def wedged_stats() -> tuple[int, int]: - entered.set() - if not release.wait(5): - raise AssertionError("evidence read was not released") - return (7, 3) - - server.accepted_share_stats = wedged_stats # type: ignore[method-assign] - with patch( - "lab.prism.prism_coordinator.time.monotonic", - side_effect=lambda: clock["now"], - ): - server._record_heartbeat("block_submitter") - reader = threading.Thread(target=server.accepted_share_stats) - reader.start() - try: - self.assertTrue(entered.wait(5)) - clock["now"] += server.watchdog_timeout_seconds + 1.0 - # A blocked evidence read is deliberately NOT excused from - # the liveness budget: exit-and-replay is the recovery path - # for a genuinely wedged ledger read, and no disposition - # code may pause the submitter's heartbeat to hide it. - self.assertEqual( - server._overdue_heartbeats(clock["now"]), ["block_submitter"] - ) - self.assertNotIn("block_submitter", server._watchdog_pauses) - finally: - release.set() - reader.join(5) - - def test_metrics_include_accepted_stats_reconcile_status(self) -> None: - server = self._bare_coordinator() - server.ledger = SimpleNamespace( - accepted_stats_reconcile_status=lambda: { - "age_seconds": 12.5, - "failures": 3, - } - ) - lines = server._accepted_stats_reconcile_metric_lines() - self.assertIn( - "qbit_prism_accepted_stats_reconcile_failures_total 3", lines - ) - self.assertIn( - "qbit_prism_accepted_stats_reconcile_age_seconds 12.500000", lines - ) - # Ledgers without the accessor (file-backed, duck-typed) emit no - # reconcile lines rather than failing the scrape. - server.ledger = SimpleNamespace() - self.assertEqual(server._accepted_stats_reconcile_metric_lines(), []) - - def test_overdue_submitter_heartbeat_names_the_stuck_phase(self) -> None: - server = self._bare_coordinator() - clock = {"now": 1_000.0} - server._block_submitter_thread_ident = threading.get_ident() - with patch( - "lab.prism.prism_coordinator.time.monotonic", - side_effect=lambda: clock["now"], - ): - server._record_block_submitter_phase("replay-outbox-query") - clock["now"] += server.watchdog_timeout_seconds + 1.0 - self.assertEqual( - server._overdue_heartbeats(clock["now"]), - ["block_submitter:replay-outbox-query"], - ) - - def test_watchdog_starts_before_synchronous_startup_work(self) -> None: - source = inspect.getsource(PrismCoordinator._serve_with_listener_stack) - watchdog_start = source.index("target=self.watchdog_loop") - - self.assertEqual(source.count("target=self.watchdog_loop"), 1) - self.assertLess(watchdog_start, source.index("self.prewarm_startup_jobs")) - self.assertLess(watchdog_start, source.index("self.replay_recovered_shares")) - - def test_watchdog_hard_exits_when_fatal_stop_wins_during_startup(self) -> None: - server = self._bare_coordinator() - server._fatal_exit_requested = True - server.stop_event.set() - - with patch("lab.prism.prism_coordinator.os._exit") as hard_exit: - server.watchdog_loop() - - hard_exit.assert_called_once_with(1) - - -class PrismStampedJobFloorTests(_VardiffSupportTestCase): - """The listener floor must hold on the wire, not just in vardiff policy. - - Stamped jobs are the single choke point for every mining.set_difficulty - the coordinator sends, and marketplace verification judges the first one. - The regression here is a young chain: qbit network difficulty below the - high-diff floor used to drag the advertised difficulty down with it. - """ - - def stamp_coordinator(self) -> PrismCoordinator: - server = coordinator() - server.job_counter = 0 - server.share_weights_by_username = {} - server.default_share_weight = 1 - return server - - def cached_bundle(self) -> CachedJobBundle: - # bits 207fffff: regtest-grade network difficulty (~4.7e-10), far - # below the 500k marketplace floor. - qbit_target = target_from_compact("207fffff") - base_job = direct_stratum.DirectQbitStratumJob( - job_id="prism-template-base", - previousblockhash_display="00" * 32, - prevhash="00" * 32, - coinb1="", - coinb2="", - full_coinbase_prefix="", - full_coinbase_suffix="", - merkle_branch=(), - transaction_hexes=(), - version="20000000", - nbits="207fffff", - ntime="6553f100", - qbit_target=qbit_target, - share_target=qbit_target, - share_difficulty=Decimal("1"), - extranonce1_hex="ffffffff", - extranonce2_size=8, - clean_jobs=True, - ) - return CachedJobBundle( - key=("test",), - template=gbt_template("00" * 32), - template_fingerprint="fp", - coinbase_manifest={}, - shares_json=[], - prior_balances=[], - found_block={"network_difficulty": 1}, - collection_only=False, - issued_at_ms=12345, - base_job=base_job, - built_monotonic=time.monotonic(), - ) - - def highdiff_client(self) -> ClientState: - state = client() - state.worker = worker_identity() - state.listener_vardiff_config = highdiff_vardiff_config() - state.minimum_advertised_difficulty = Decimal("500000") - state.share_difficulty = Decimal("500000") - return state - - def test_post_block_notification_stamps_caller_heartbeat(self) -> None: - server, _state, _ledger = submit_coordinator() - seen: list[str] = [] - server._record_heartbeat = seen.append # type: ignore[method-assign] - server.refresh_jobs_after_accepted_block( - block_height=10, block_hash="bb" * 32, heartbeat_name="block_submitter" - ) - self.assertEqual(seen, ["block_submitter"]) - - # The client-thread pending refresh keeps the default poller heartbeat. - seen.clear() - server.refresh_jobs_after_accepted_block(block_height=11, block_hash="cc" * 32) - self.assertEqual(seen, ["qbit_blockpoll"]) - - -class HealthSnapshotTests(_JobSupportTestCase): - def test_health_payload_uses_aggregate_stats_not_all_shares(self) -> None: - ledger = FakeLedger() - server, _ = coordinator(ledger=ledger) - mark_progress_healthy(server) - payload = server.health_payload() - self.assertTrue(payload["ok"]) - self.assertEqual(payload["accepted_share_count"], 3) - self.assertEqual(payload["ready_miner_count"], 3) - self.assertGreaterEqual(ledger.stats_calls, 1) - - def test_cached_health_payload_computes_inline_without_refresher(self) -> None: - server, _ = coordinator() - mark_progress_healthy(server) - status, payload = server.cached_health_payload() - self.assertEqual(status, 200) - self.assertTrue(payload["ok"]) - - def test_cached_health_payload_serves_snapshot_and_flags_staleness(self) -> None: - server, _ = coordinator() - mark_progress_healthy(server) - server.refresh_health_snapshot() - server._ensure_observability_service().set_loop_running_for_test(True) - - status, payload = server.cached_health_payload() - self.assertEqual(status, 200) - self.assertIn("snapshot_age_seconds", payload) - - # Even if the ledger becomes unusable, the snapshot keeps serving. - server.ledger = None # type: ignore[assignment] - status, payload = server.cached_health_payload() - self.assertEqual(status, 200) - - server._ensure_observability_service().set_health_snapshot_monotonic_for_test( - time.monotonic() - 1_000 - ) - status, payload = server.cached_health_payload() - self.assertEqual(status, 503) - self.assertFalse(payload["ok"]) - - def test_accepted_share_stats_falls_back_to_all_shares(self) -> None: - server, _ = coordinator(ledger=SingleWriterShareLedger()) - self.assertEqual(server.accepted_share_stats(), (0, 0)) - - def test_single_writer_ledger_stats(self) -> None: - ledger = SingleWriterShareLedger() - self.assertEqual( - ledger.accepted_share_stats(), - {"accepted_share_count": 0, "distinct_miner_count": 0}, - ) - - def test_audit_server_marks_refresher_before_listener_dispatch(self) -> None: - """A probe retrying connection-refused queues in the accept backlog at - listen() time and can be served the instant the listener thread - starts. The refresher flag must already be set by then, or - cached_health_payload's legacy inline path would run the cold - accepted-share aggregate synchronously on the handler thread instead - of returning the documented starting state.""" - server, _ = coordinator() - server.audit_bind = "127.0.0.1" - server.audit_port = 0 - # The spawned refresher loops exit immediately; the running flag is - # what gates the handler path and must survive. - server.stop_event.set() - flag_at_bind: list[bool] = [] - - class RecordingServer: - def __init__(self, address: object, handler_cls: object) -> None: - flag_at_bind.append( - server._ensure_observability_service() - .state() - .health_refresh_loop_running - ) - self.server_address = address - self.ready = threading.Event() - self.released = threading.Event() - - def serve_unless_startup_cancelled(self, *, poll_interval: float) -> None: - del poll_interval - self.ready.set() - self.released.wait(2.0) - - def cancel_startup(self) -> bool: - return True - - def shutdown(self) -> None: - self.released.set() - - def server_close(self) -> None: - return None - - with patch.object( - audit_http_module, "_BoundedThreadingHttpServer", RecordingServer - ), patch("builtins.print"): - server.start_audit_server() - try: - self.assertEqual(flag_at_bind, [True]) - self.assertTrue( - server._ensure_observability_service() - .state() - .health_refresh_loop_running - ) - finally: - self.assertTrue(server._ensure_audit_http_facade().stop()) - - -class MetricsSnapshotTests(_JobSupportTestCase): - def test_cached_metrics_do_not_reenter_complete_renderer(self) -> None: - server, _ = coordinator() - complete = server.refresh_metrics_snapshot() - server._render_metrics_payload = mock.Mock( # type: ignore[method-assign] - side_effect=AssertionError("cached scrape reached live renderer") - ) - - status, cached = server.cached_metrics_payload() - - self.assertEqual(status, 200) - self.assertTrue(cached.startswith(complete)) - server._render_metrics_payload.assert_not_called() - self.assertIn("qbit_prism_metrics_snapshot_generation 1", cached) - - def test_renderer_emits_semantic_current_work_ratio_gauge(self) -> None: - # Upstream #107: the complete document carries the semantic-currency - # gauge derived from fingerprint, payout generation, and client - # session currency. - server, _ = coordinator() - - metrics = server._render_metrics_payload() - - self.assertIn( - "qbit_prism_stratum_semantic_current_work_ratio 1.0", - metrics, - ) - - -class JobBuildMetricsTests(_JobSupportTestCase): - def test_metrics_include_job_build_histogram_and_cache_counters(self) -> None: - server, _ = coordinator() - install_fake_bundle_builder(server) - server.build_job_for_client(client(1), clean_jobs=True) - server.build_job_for_client(client(2), clean_jobs=True) - server.observe_job_build_elapsed(0.3, {"bundle": 0.2, "stamp": 0.01}) - server._observe_tip_refresh_build_phase("payout_state_derivation", 0.25) - server._record_tip_refresh_ipc_bytes("input", 123) - - metrics = server.metrics_payload() - - self.assertIn('qbit_prism_job_build_seconds_bucket{le="0.5"} 1', metrics) - self.assertIn('qbit_prism_job_build_seconds_bucket{le="+Inf"} 1', metrics) - self.assertIn("qbit_prism_job_build_seconds_count 1", metrics) - self.assertIn('qbit_prism_job_cache_hits_total{cache="bundle"} 1', metrics) - self.assertIn('qbit_prism_job_cache_misses_total{cache="bundle"} 1', metrics) - self.assertIn('qbit_prism_job_build_phase_seconds_total{phase="bundle"} 0.2', metrics) - self.assertIn( - 'qbit_prism_tip_refresh_bundle_phase_seconds_count{phase="payout_state_derivation"} 1', - metrics, - ) - self.assertIn( - 'qbit_prism_tip_refresh_builder_ipc_bytes_total{direction="input"} 123', - metrics, - ) - self.assertIn("qbit_prism_tip_refresh_bundle_queue_depth 0", metrics) - self.assertIn("qbit_prism_tip_refresh_bundle_inflight 0", metrics) - self.assertIn("qbit_prism_connected_clients 0", metrics) - - def test_metrics_split_payout_preparation_publication_and_delivery(self) -> None: - server, _ = coordinator() - install_fake_bundle_builder(server) - state = client(1) - state.send = lambda _payload: None # type: ignore[method-assign] - - self.assertEqual(server._advance_payout_state_generation(), 1) - self.assertTrue(server.maybe_send_job(state, clean_jobs=True)) - - metrics = server.metrics_payload() - - self.assertIn("qbit_prism_payout_preparation_seconds_count 1", metrics) - self.assertIn("qbit_prism_payout_publish_seconds_count 1", metrics) - self.assertIn( - "qbit_prism_payout_invalidation_first_delivery_seconds_count 1", - metrics, - ) - self.assertIn( - 'qbit_prism_payout_gate_wait_seconds_count{generation="current"} 1', - metrics, - ) - self.assertIn("qbit_prism_payout_candidates_discarded_total 0", metrics) - - def test_poll_observes_reorg_reconcile_phase(self) -> None: - # The reconcile stage runs serially before every refresh build; one - # poll must record exactly one reorg_reconcile phase observation. - server, _ = coordinator() - install_fake_bundle_builder(server) - state = client(1) - state.send = lambda _payload: None # type: ignore[method-assign] - server.clients = {state} - - try: - refreshed = server.poll_qbit_tip_template_once() - finally: - server.shutdown_tip_refresh_executor() - - self.assertEqual(refreshed, 1) - metrics = server.metrics_payload() - self.assertIn( - 'qbit_prism_tip_refresh_bundle_phase_seconds_count{phase="reorg_reconcile"} 1', - metrics, - ) diff --git a/tests/test_prism_coordinator_shutdown.py b/tests/test_prism_coordinator_shutdown.py deleted file mode 100644 index 1a3aa5db0..000000000 --- a/tests/test_prism_coordinator_shutdown.py +++ /dev/null @@ -1,2130 +0,0 @@ -#!/usr/bin/env python3 - -from __future__ import annotations - -import queue -import re -import signal -import threading -import time -import unittest -from types import SimpleNamespace -from unittest.mock import patch - -from lab.prism import prism_coordinator -from lab.prism.coordinator_shutdown import ( - CoordinatorShutdownController, - ShutdownInProgress, -) -from lab.prism.prism_coordinator import ( - PendingShareAppend, - PRISM_REJECTION_POOL_CLOSED, - PrismCoordinator, - StratumError, - WriterLeaseRenewalDeferred, -) - - -class RecordingLeaseLedger: - backend_name = "recording" - - def __init__(self) -> None: - self.release_calls = 0 - self.released = threading.Event() - - def release_writer_lease(self) -> bool: - self.release_calls += 1 - self.released.set() - return True - - -class WatchdogLeaseLedger(RecordingLeaseLedger): - def __init__(self) -> None: - super().__init__() - self.release_thread: threading.Thread | None = None - - def release_writer_lease_fresh_connection(self) -> bool: - self.release_thread = threading.current_thread() - return self.release_writer_lease() - - -class HeartbeatLeaseLedger(RecordingLeaseLedger): - writer_lease_fast_adoption_capable = True - writer_lease_guard_required = True - - def __init__(self) -> None: - super().__init__() - self.renew_calls = 0 - self.renew_thread: threading.Thread | None = None - self.renewed = threading.Event() - - def renew_writer_lease_heartbeat( - self, - *, - on_query_start=None, - ) -> dict[str, int | str]: - if on_query_start is not None: - on_query_start() - self.renew_calls += 1 - self.renew_thread = threading.current_thread() - self.renewed.set() - return {"backend": "recording", "renewed_count": 1} - - -class DeferredRenewalLeaseLedger(RecordingLeaseLedger): - """Verification proves liveness with renewal deferred to an own write. - - Models a fenced write that outlasted the lease TTL: the committed row is - expired, the SKIP LOCKED renewal skipped the tuple lock, and PostgreSQL - attributes that lock to this process's own pooled backend. The session - is live only as long as that write eventually commits. - """ - - writer_lease_fast_adoption_capable = True - writer_lease_guard_required = True - - def __init__(self) -> None: - super().__init__() - self.verify_calls = 0 - self.verified = threading.Event() - - def verify_writer_lease_guard_session( - self, - *, - on_query_start=None, - ) -> dict[str, int | str | bool]: - if on_query_start is not None: - on_query_start() - self.verify_calls += 1 - self.verified.set() - return { - "backend": "recording", - "verified_count": 1, - "renewed_count": 0, - "renewal_deferred_to_own_write": True, - } - - -class GuardVerifyLeaseLedger(RecordingLeaseLedger): - """Fake ledger with the non-locking guarded-session verification. - - ``lease_row_lock`` models the qbit_ledger_writer_lease tuple lock a - fenced write holds for its entire transaction. The verification never - needs it; the legacy locking renewal would (and in production died on - the guard's statement timeout — the block-39416 restart loop), so this - fake fails the test outright if the coordinator still calls it. - """ - - writer_lease_fast_adoption_capable = True - writer_lease_guard_required = True - - def __init__(self) -> None: - super().__init__() - self.lease_row_lock = threading.Lock() - self.fenced_write_started = threading.Event() - self.verified = threading.Event() - self.verify_calls = 0 - self.verify_calls_during_fenced_write = 0 - self.renew_calls = 0 - self.guard_lost = False - - def verify_writer_lease_guard_session( - self, - *, - on_query_start=None, - ) -> dict[str, int | str]: - if on_query_start is not None: - on_query_start() - if self.guard_lost: - raise RuntimeError("postgres writer lease guard is not held") - self.verify_calls += 1 - if self.lease_row_lock.locked(): - self.verify_calls_during_fenced_write += 1 - self.verified.set() - return {"backend": "recording", "verified_count": 1} - - def renew_writer_lease_heartbeat( - self, - *, - on_query_start=None, - ) -> dict[str, int | str]: - self.renew_calls += 1 - raise AssertionError( - "lease-row renewal must not run on the guarded session" - ) - - def persist_accepted_block( - self, - *, - duration_seconds: float, - ) -> dict[str, int | str]: - with self.lease_row_lock: - self.fenced_write_started.set() - time.sleep(duration_seconds) - return {"backend": "recording", "block_row_count": 1} - - -def coordinator( - ledger: object | None = None, - *, - timeout: float = 0.5, -) -> PrismCoordinator: - server = PrismCoordinator.__new__(PrismCoordinator) - server.lock = threading.RLock() - server.stop_event = threading.Event() - server.writer_quiescence_timeout_seconds = timeout - server._shutdown_controller = CoordinatorShutdownController(timeout) - server.ledger = ledger or RecordingLeaseLedger() - return server - - -class CoordinatorShutdownControllerTests(unittest.TestCase): - def test_compatibility_reexports_reference_shutdown_owner(self) -> None: - self.assertIs( - prism_coordinator.CoordinatorShutdownController, - CoordinatorShutdownController, - ) - self.assertIs(prism_coordinator.ShutdownInProgress, ShutdownInProgress) - - def test_nested_writer_inherits_admission_after_shutdown_request(self) -> None: - controller = CoordinatorShutdownController(0.5) - outer = controller.enter_writer("outer") - controller.request_shutdown(signal.SIGTERM) - inner = controller.enter_writer("inner") - - controller.exit_writer(inner) - controller.exit_writer(outer) - - self.assertEqual(controller.snapshot()["active_writers"], {}) - with self.assertRaisesRegex(ShutdownInProgress, "coordinator is shutting down"): - controller.enter_writer("late") - - def test_transferable_writer_token_finishes_idempotently_on_another_thread( - self, - ) -> None: - controller = CoordinatorShutdownController(0.5) - token = controller.reserve_writer("share_persistence") - - finisher = threading.Thread(target=lambda: (token.finish(), token.finish())) - finisher.start() - finisher.join(1) - - self.assertFalse(finisher.is_alive()) - self.assertTrue(token.finished) - self.assertEqual(controller.snapshot()["active_writers"], {}) - - -class PrismCoordinatorShutdownTests(unittest.TestCase): - def test_refresh_timeout_still_drains_build_executors(self) -> None: - server = coordinator() - calls: list[str] = [] - server.shutdown_initial_job_executor = ( # type: ignore[method-assign] - lambda: calls.append("initial") - ) - server.shutdown_job_build_executor = ( # type: ignore[method-assign] - lambda: calls.append("job_build") - ) - server.shutdown_payout_artifact_executor = ( # type: ignore[method-assign] - lambda: calls.append("payout_artifact") - ) - server.shutdown_reconcile_prefetch_executor = ( # type: ignore[method-assign] - lambda: calls.append("reconcile_prefetch") - ) - server.retire_share_window_spool = ( # type: ignore[method-assign] - lambda: calls.append("spool") - ) - server.shutdown_serve_builder = ( # type: ignore[method-assign] - lambda: calls.append("serve_builder") - ) - - server.shutdown_tip_refresh_executor() - - self.assertEqual( - calls, - [ - "initial", - "job_build", - "payout_artifact", - "reconcile_prefetch", - "spool", - "serve_builder", - ], - ) - - def test_startup_replay_shutdown_stops_cleanly_and_releases_lease_once( - self, - ) -> None: - ledger = RecordingLeaseLedger() - server = coordinator(ledger) - - def rejected_replay() -> int: - server.request_shutdown(signal.SIGTERM) - raise ShutdownInProgress("PRISM coordinator is shutting down") - - started = time.monotonic() - with patch("builtins.print"): - self.assertFalse( - server._run_startup_writer_replay( - rejected_replay, - drain_threads=[], - ) - ) - elapsed = time.monotonic() - started - - # No writer is active, so the replay exit never waits out the - # quiescence budget before releasing the lease exactly once. - self.assertLess(elapsed, 0.45) - self.assertEqual(ledger.release_calls, 1) - snapshot = server._ensure_shutdown_controller().snapshot() - self.assertEqual(snapshot["active_writers"], {}) - self.assertFalse(snapshot["lease_release_withheld"]) - self.assertEqual(snapshot["release_withheld_total"], 0) - self.assertEqual(snapshot["lease_release_outcomes"]["success"], 1) - with patch("builtins.print"): - server.shutdown(reason="main_finally") - self.assertEqual(ledger.release_calls, 1) - - def test_lease_heartbeat_start_without_ledger_is_noop(self) -> None: - server = PrismCoordinator.__new__(PrismCoordinator) - - with patch.object(prism_coordinator.threading, "Thread") as thread: - self.assertIsNone(server._start_ledger_lease_heartbeat()) - - thread.assert_not_called() - - def test_lease_heartbeat_start_fails_closed_when_required_guard_lost( - self, - ) -> None: - class LostGuardAtStartupLedger(HeartbeatLeaseLedger): - writer_lease_fast_adoption_capable = False - - server = coordinator(LostGuardAtStartupLedger()) - - with patch.object( - server, - "_ledger_lease_heartbeat_hard_exit", - ) as hard_exit: - self.assertIsNone(server._start_ledger_lease_heartbeat()) - - hard_exit.assert_called_once() - - def test_lease_heartbeat_start_skips_guardless_ledger_without_exit( - self, - ) -> None: - server = coordinator(RecordingLeaseLedger()) - - with patch.object( - server, - "_ledger_lease_heartbeat_hard_exit", - ) as hard_exit: - self.assertIsNone(server._start_ledger_lease_heartbeat()) - - hard_exit.assert_not_called() - - def test_external_side_effect_gate_fails_closed_after_guard_loss(self) -> None: - class LostGuardLedger(HeartbeatLeaseLedger): - def renew_writer_lease_heartbeat(self) -> dict[str, int | str]: - raise RuntimeError("postgres session was lost during host suspend") - - server = coordinator(LostGuardLedger()) - - with patch.object( - server, - "_ledger_lease_heartbeat_hard_exit", - ) as hard_exit, self.assertRaisesRegex( - ShutdownInProgress, - "writer lease guard verification failed", - ): - server._require_fresh_ledger_lease_for_external_side_effect( - "submitblock" - ) - - hard_exit.assert_called_once() - - def test_external_side_effect_gate_allows_fresh_guarded_session(self) -> None: - ledger = HeartbeatLeaseLedger() - server = coordinator(ledger) - - server._require_fresh_ledger_lease_for_external_side_effect( - "submitblock" - ) - - self.assertEqual(ledger.renew_calls, 1) - self.assertIsNot(ledger.renew_thread, threading.current_thread()) - - def test_external_side_effect_gate_does_not_regress_freshness_stamp( - self, - ) -> None: - ledger = HeartbeatLeaseLedger() - server = coordinator(ledger) - verification_started_monotonic = 100.0 - fresher_heartbeat_monotonic = 101.0 - original_join = threading.Thread.join - - def join_then_publish_fresher_heartbeat( - thread: threading.Thread, - timeout: float | None = None, - ) -> None: - original_join(thread, timeout) - self.assertFalse(thread.is_alive()) - self.assertTrue(ledger.renewed.is_set()) - server._ledger_lease_heartbeat_last_success_monotonic = ( - fresher_heartbeat_monotonic - ) - - with patch.object( - prism_coordinator.time, - "monotonic", - return_value=verification_started_monotonic, - ), patch.object( - prism_coordinator.threading.Thread, - "join", - new=join_then_publish_fresher_heartbeat, - ), patch( - "lab.prism.prism_coordinator.os._exit", - side_effect=AssertionError("unexpected hard exit"), - ) as process_exit, patch.object( - server, - "_watchdog_hard_exit", - ) as watchdog_hard_exit: - server._require_fresh_ledger_lease_for_external_side_effect( - "submitblock" - ) - - self.assertEqual( - server._ledger_lease_heartbeat_last_success_monotonic, - fresher_heartbeat_monotonic, - ) - process_exit.assert_not_called() - watchdog_hard_exit.assert_not_called() - - def test_external_side_effect_gate_defers_renewal_blocked_by_own_write( - self, - ) -> None: - """Own-write lease deferral withholds the RPC without fencing the process. - - The guarded session is live — the heartbeat exemption keeps a slow - fenced write from restart-looping the coordinator — but that - survival argument assumes the write commits. A rollback would hand - the expired row to a queued different-identity claimant while the - external effect is in flight, so the fence must refuse the RPC. It - must refuse without the hard exit: the process is healthy, and - callers retry deferred work on their own cadence (broadcast pass - interval, block-candidate outbox replay), succeeding once the - commit lands a renewal. - """ - ledger = DeferredRenewalLeaseLedger() - server = coordinator(ledger) - - with patch.object( - server, - "_ledger_lease_heartbeat_hard_exit", - ) as hard_exit, self.assertRaisesRegex( - WriterLeaseRenewalDeferred, - "renewal is deferred", - ): - server._require_fresh_ledger_lease_for_external_side_effect( - "submitblock" - ) - - hard_exit.assert_not_called() - self.assertEqual(ledger.verify_calls, 1) - # The verification still proved the session live; the freshness - # stamp must advance so the heartbeat monitor sees progress. - self.assertIsNotNone( - getattr( - server, - "_ledger_lease_heartbeat_last_success_monotonic", - None, - ) - ) - - def test_lease_heartbeat_tolerates_renewal_deferred_to_own_write(self) -> None: - """The heartbeat keeps running while renewal defers to an own write. - - Hard-exiting here would roll back the very write the lease defers - to and restart-loop on every similarly slow block; only the - external-side-effect fence treats the deferral as disqualifying. - """ - ledger = DeferredRenewalLeaseLedger() - server = coordinator(ledger) - server.ledger_lease_heartbeat_seconds = 0.01 - - with patch.object( - server, - "_ledger_lease_heartbeat_hard_exit", - ) as hard_exit: - thread = server._start_ledger_lease_heartbeat() - self.assertIsNotNone(thread) - self.assertTrue(ledger.verified.wait(0.2)) - server._ledger_lease_heartbeat_stop_event.set() - assert thread is not None - thread.join(1.0) - - hard_exit.assert_not_called() - self.assertGreaterEqual(ledger.verify_calls, 1) - - def test_external_side_effect_gate_bounds_blocked_guard_verification(self) -> None: - verification_started = threading.Event() - release_verification = threading.Event() - - class BlockingGuardLedger(HeartbeatLeaseLedger): - def renew_writer_lease_heartbeat(self) -> dict[str, int | str]: - verification_started.set() - release_verification.wait(1) - return {"backend": "recording", "renewed_count": 1} - - ledger = BlockingGuardLedger() - server = coordinator(ledger) - server.ledger_lease_external_fence_timeout_seconds = 0.01 - - try: - started = time.monotonic() - with patch.object( - server, - "_ledger_lease_heartbeat_hard_exit", - ) as hard_exit, self.assertRaisesRegex( - ShutdownInProgress, - "writer lease guard verification timed out", - ): - server._require_fresh_ledger_lease_for_external_side_effect( - "ctv_submitpackage" - ) - elapsed = time.monotonic() - started - finally: - release_verification.set() - - self.assertTrue(verification_started.is_set()) - self.assertLess(elapsed, 0.1) - hard_exit.assert_called_once() - if ledger.renew_thread is not None: - ledger.renew_thread.join(0.2) - - def test_external_fence_survives_guard_queue_contention(self) -> None: - guard_query_lock = threading.Lock() - holder_ready = threading.Event() - release_holder = threading.Event() - - class ContendedGuardLedger(HeartbeatLeaseLedger): - def renew_writer_lease_heartbeat( - self, - *, - on_query_start=None, - ) -> dict[str, int | str]: - with guard_query_lock: - return super().renew_writer_lease_heartbeat( - on_query_start=on_query_start - ) - - def hold_guard_like_inflight_heartbeat() -> None: - with guard_query_lock: - holder_ready.set() - release_holder.wait(1) - - ledger = ContendedGuardLedger() - server = coordinator(ledger) - server.ledger_lease_external_fence_timeout_seconds = 0.05 - server.ledger_lease_heartbeat_failure_seconds = 0.75 - holder = threading.Thread(target=hold_guard_like_inflight_heartbeat) - holder.start() - self.assertTrue(holder_ready.wait(0.2)) - unblock = threading.Timer(0.15, release_holder.set) - unblock.start() - - try: - with patch.object( - server, - "_ledger_lease_heartbeat_hard_exit", - ) as hard_exit: - server._require_fresh_ledger_lease_for_external_side_effect( - "submitblock" - ) - finally: - release_holder.set() - unblock.cancel() - holder.join(1.0) - - hard_exit.assert_not_called() - self.assertEqual(ledger.renew_calls, 1) - - def test_external_fence_bounds_queue_wait_by_failure_budget(self) -> None: - release_queue_slot = threading.Event() - - class QueuedForeverLedger(HeartbeatLeaseLedger): - def renew_writer_lease_heartbeat( - self, - *, - on_query_start=None, - ) -> dict[str, int | str]: - release_queue_slot.wait(1) - return super().renew_writer_lease_heartbeat( - on_query_start=on_query_start - ) - - server = coordinator(QueuedForeverLedger()) - server.ledger_lease_external_fence_timeout_seconds = 0.02 - server.ledger_lease_heartbeat_failure_seconds = 0.05 - - started = time.monotonic() - try: - with patch.object( - server, - "_ledger_lease_heartbeat_hard_exit", - ) as hard_exit, self.assertRaisesRegex( - ShutdownInProgress, - "writer lease guard verification timed out", - ): - server._require_fresh_ledger_lease_for_external_side_effect( - "ctv_submitpackage" - ) - elapsed = time.monotonic() - started - finally: - release_queue_slot.set() - - self.assertLess(elapsed, 0.5) - hard_exit.assert_called_once() - - def test_fast_adoptable_lease_renews_on_dedicated_heartbeat_thread(self) -> None: - ledger = HeartbeatLeaseLedger() - server = coordinator(ledger) - server.ledger_lease_heartbeat_seconds = 0.01 - - thread = server._start_ledger_lease_heartbeat() - self.assertIsNotNone(thread) - self.assertTrue(ledger.renewed.wait(0.2)) - server.stop_event.set() - assert thread is not None - time.sleep(0.02) - self.assertTrue(thread.is_alive()) - self.assertTrue(server._stop_ledger_lease_heartbeat()) - thread.join(0.2) - - self.assertFalse(thread.is_alive()) - self.assertGreaterEqual(ledger.renew_calls, 1) - self.assertIs(ledger.renew_thread, thread) - self.assertIsNot(ledger.renew_thread, threading.current_thread()) - - def test_heartbeat_prefers_non_locking_guard_verification(self) -> None: - ledger = GuardVerifyLeaseLedger() - server = coordinator(ledger) - server.ledger_lease_heartbeat_seconds = 0.01 - - thread = server._start_ledger_lease_heartbeat() - self.assertIsNotNone(thread) - self.assertTrue(ledger.verified.wait(0.2)) - self.assertTrue(server._stop_ledger_lease_heartbeat()) - assert thread is not None - thread.join(0.2) - - self.assertGreaterEqual(ledger.verify_calls, 1) - self.assertEqual(ledger.renew_calls, 0) - - def test_heartbeat_survives_fenced_write_holding_lease_row_past_timeout(self) -> None: - """Regression: block-39416 restart loop. - - persist_accepted_block holds the lease tuple for several multiples of - the guard's statement timeout and of the heartbeat failure budget - (production: 1.7s versus 0.5s/0.75s, scaled here). The heartbeat must - keep passing throughout, no hard exit fires, and the fenced write - completes. - """ - ledger = GuardVerifyLeaseLedger() - server = coordinator(ledger) - server.ledger_lease_heartbeat_seconds = 0.01 - server.ledger_lease_heartbeat_failure_seconds = 0.1 - server.ledger_lease_heartbeat_monitor_seconds = 0.005 - - with patch( - "lab.prism.prism_coordinator.os._exit", - side_effect=AssertionError("unexpected hard exit"), - ) as process_exit, patch.object( - server, - "_watchdog_hard_exit", - ) as hard_exit: - thread = server._start_ledger_lease_heartbeat() - self.assertIsNotNone(thread) - self.assertTrue(ledger.verified.wait(0.2)) - - result = ledger.persist_accepted_block(duration_seconds=0.3) - - assert thread is not None - self.assertTrue(thread.is_alive()) - self.assertTrue(server._stop_ledger_lease_heartbeat()) - thread.join(0.2) - - self.assertEqual(result, {"backend": "recording", "block_row_count": 1}) - self.assertGreaterEqual(ledger.verify_calls_during_fenced_write, 1) - self.assertEqual(ledger.renew_calls, 0) - process_exit.assert_not_called() - hard_exit.assert_not_called() - - def test_heartbeat_hard_exits_promptly_after_guard_connection_loss(self) -> None: - ledger = GuardVerifyLeaseLedger() - server = coordinator(ledger) - server.ledger_lease_heartbeat_seconds = 0.01 - exited = threading.Event() - - with patch("builtins.print"), patch("traceback.print_exc"), patch.object( - server, - "_watchdog_hard_exit", - side_effect=lambda *_args, **_kwargs: exited.set(), - ) as hard_exit: - thread = server._start_ledger_lease_heartbeat() - self.assertIsNotNone(thread) - self.assertTrue(ledger.verified.wait(0.2)) - - ledger.guard_lost = True - self.assertTrue(exited.wait(0.5)) - assert thread is not None - thread.join(0.5) - server._ledger_lease_heartbeat_stop_event.set() - - hard_exit.assert_called_once_with("lease_heartbeat", timeout_seconds=0.1) - - def test_external_fence_is_non_locking_during_accepted_block_persistence(self) -> None: - ledger = GuardVerifyLeaseLedger() - server = coordinator(ledger) - # Tighter than the fenced write's duration: a fence that queued on - # the lease tuple would time out and hard-exit here. - server.ledger_lease_external_fence_timeout_seconds = 0.05 - persist_thread = threading.Thread( - target=ledger.persist_accepted_block, - kwargs={"duration_seconds": 0.3}, - ) - persist_thread.start() - self.assertTrue(ledger.fenced_write_started.wait(0.5)) - - try: - with patch.object( - server, - "_ledger_lease_heartbeat_hard_exit", - ) as hard_exit: - server._require_fresh_ledger_lease_for_external_side_effect( - "submitblock" - ) - finally: - persist_thread.join(1.0) - - self.assertFalse(persist_thread.is_alive()) - hard_exit.assert_not_called() - self.assertGreaterEqual(ledger.verify_calls_during_fenced_write, 1) - self.assertEqual(ledger.renew_calls, 0) - - def test_external_fence_fails_closed_after_guard_loss_with_verification(self) -> None: - ledger = GuardVerifyLeaseLedger() - ledger.guard_lost = True - server = coordinator(ledger) - - with patch.object( - server, - "_ledger_lease_heartbeat_hard_exit", - ) as hard_exit, self.assertRaisesRegex( - ShutdownInProgress, - "writer lease guard verification failed", - ): - server._require_fresh_ledger_lease_for_external_side_effect( - "submitblock" - ) - - hard_exit.assert_called_once() - - def test_lease_heartbeat_failure_uses_bounded_hard_exit_path(self) -> None: - class FailingHeartbeatLedger(HeartbeatLeaseLedger): - def renew_writer_lease_heartbeat(self) -> dict[str, int | str]: - raise RuntimeError("database unavailable") - - server = coordinator(FailingHeartbeatLedger()) - - with patch("builtins.print"), patch("traceback.print_exc"), patch.object( - server, - "_watchdog_hard_exit", - ) as hard_exit: - server.ledger_lease_heartbeat_loop() - - hard_exit.assert_called_once_with("lease_heartbeat", timeout_seconds=0.1) - - def test_heartbeat_hard_exit_does_not_deadlock_lease_release(self) -> None: - released = threading.Event() - exited = threading.Event() - - class FailingReleaseLedger(HeartbeatLeaseLedger): - def renew_writer_lease_heartbeat(self) -> dict[str, int | str]: - raise RuntimeError("database unavailable") - - def release_writer_lease_fresh_connection(self) -> bool: - released.set() - return True - - server = coordinator(FailingReleaseLedger()) - - with patch("builtins.print"), patch("traceback.print_exc"), patch( - "lab.prism.prism_coordinator.os._exit", - side_effect=lambda _code: exited.set(), - ): - thread = threading.Thread(target=server.ledger_lease_heartbeat_loop) - server._ledger_lease_heartbeat_thread = thread - thread.start() - self.assertTrue(exited.wait(1.0)) - thread.join(1.0) - - self.assertFalse(thread.is_alive()) - # The heartbeat thread that armed the exit is parked joining the - # release worker; the release must still reach the database instead - # of burning the exit budget joining that same thread. - self.assertTrue(released.is_set()) - - def test_lease_heartbeat_hard_exit_never_waits_on_blocked_output(self) -> None: - server = coordinator(HeartbeatLeaseLedger()) - release_print = threading.Event() - hard_exit_called = threading.Event() - - def blocking_print(*_args: object, **_kwargs: object) -> None: - release_print.wait(1) - - with patch("builtins.print", side_effect=blocking_print) as print_call, patch.object( - server, - "_watchdog_hard_exit", - side_effect=lambda *_args, **_kwargs: hard_exit_called.set(), - ) as hard_exit: - caller = threading.Thread( - target=server._ledger_lease_heartbeat_hard_exit, - args=("heartbeat failed",), - kwargs={"include_traceback": False}, - ) - caller.start() - reached_hard_exit = hard_exit_called.wait(0.1) - release_print.set() - caller.join(0.2) - - self.assertTrue(reached_hard_exit) - self.assertFalse(caller.is_alive()) - print_call.assert_not_called() - hard_exit.assert_called_once_with("lease_heartbeat", timeout_seconds=0.1) - - def test_heartbeat_start_waits_for_first_exact_session_renewal(self) -> None: - renew_started = threading.Event() - release_renew = threading.Event() - - class ArmingHeartbeatLedger(HeartbeatLeaseLedger): - def renew_writer_lease_heartbeat(self) -> dict[str, int | str]: - renew_started.set() - release_renew.wait(1) - return {"backend": "recording", "renewed_count": 1} - - server = coordinator(ArmingHeartbeatLedger()) - result: list[threading.Thread | None] = [] - starter = threading.Thread( - target=lambda: result.append(server._start_ledger_lease_heartbeat()) - ) - starter.start() - self.assertTrue(renew_started.wait(0.2)) - time.sleep(0.02) - self.assertTrue(starter.is_alive()) - - release_renew.set() - starter.join(0.2) - self.assertFalse(starter.is_alive()) - self.assertEqual(len(result), 1) - self.assertIsNotNone(result[0]) - self.assertTrue(server._stop_ledger_lease_heartbeat()) - - def test_stalled_lease_heartbeat_exits_before_adoption_silence(self) -> None: - renew_started = threading.Event() - release_renew = threading.Event() - - class BlockingHeartbeatLedger(HeartbeatLeaseLedger): - def renew_writer_lease_heartbeat(self) -> dict[str, int | str]: - renew_started.set() - release_renew.wait(1) - return {"backend": "recording", "renewed_count": 1} - - server = coordinator(BlockingHeartbeatLedger()) - # A scaled-down but internally coherent policy: the interval stays - # below the failure budget and the budget plus the exit envelope - # stays inside the adoption silence, so the heartbeat starts and the - # stall — not a refused configuration — is what fires the exit. - server.ledger_lease_heartbeat_seconds = 0.005 - server.ledger_lease_heartbeat_failure_seconds = 0.03 - server.ledger_lease_heartbeat_monitor_seconds = 0.005 - server.ledger_lease_heartbeat_exit_timeout_seconds = 0.01 - thread: threading.Thread | None = None - - try: - with patch("builtins.print"), patch.object( - server, - "_watchdog_hard_exit", - ) as hard_exit: - thread = server._start_ledger_lease_heartbeat() - self.assertTrue(renew_started.wait(0.2)) - deadline = time.monotonic() + 0.2 - while not hard_exit.called and time.monotonic() < deadline: - time.sleep(0.001) - - hard_exit.assert_called_once_with( - "lease_heartbeat", - timeout_seconds=0.01, - ) - finally: - release_renew.set() - heartbeat_stop = getattr( - server, - "_ledger_lease_heartbeat_stop_event", - None, - ) - if heartbeat_stop is not None: - heartbeat_stop.set() - if thread is not None: - thread.join(0.2) - - def test_heartbeat_survives_steady_slow_but_successful_verifications( - self, - ) -> None: - """Regression: healthy-writer hard exit at legal verification latency. - - The success timestamp is the call-start edge, so with steady - verification duration D the freshest edge reaches age 2D plus the - idle interval just before the next success lands. Keyed on success - edges alone (D=0.15, interval=0.05, budget=0.25 here), the monitor - fires ~0.35s into a run in which every verification succeeds. The - attempt-start and completion progress marks keep the observable - silence at D, so a sole writer whose statements stay inside their - legal budgets is never hard-exited. - """ - - class SlowVerifyLedger(RecordingLeaseLedger): - writer_lease_fast_adoption_capable = True - writer_lease_guard_required = True - - def __init__(self) -> None: - super().__init__() - self.verify_calls = 0 - - def verify_writer_lease_guard_session( - self, - ) -> dict[str, int | str]: - time.sleep(0.15) - self.verify_calls += 1 - return {"backend": "recording", "verified_count": 1} - - ledger = SlowVerifyLedger() - server = coordinator(ledger) - server.ledger_lease_heartbeat_seconds = 0.05 - server.ledger_lease_heartbeat_failure_seconds = 0.25 - server.ledger_lease_heartbeat_monitor_seconds = 0.01 - - with patch( - "lab.prism.prism_coordinator.os._exit", - side_effect=AssertionError("unexpected hard exit"), - ) as process_exit, patch.object( - server, - "_watchdog_hard_exit", - ) as hard_exit: - thread = server._start_ledger_lease_heartbeat() - self.assertIsNotNone(thread) - deadline = time.monotonic() + 2.0 - while ledger.verify_calls < 4 and time.monotonic() < deadline: - time.sleep(0.01) - self.assertGreaterEqual(ledger.verify_calls, 4) - assert thread is not None - self.assertTrue(thread.is_alive()) - self.assertTrue(server._stop_ledger_lease_heartbeat()) - thread.join(0.5) - - process_exit.assert_not_called() - hard_exit.assert_not_called() - - def test_slow_but_advancing_first_heartbeat_does_not_startup_exit( - self, - ) -> None: - """The startup fencing deadline keys on activity, not arming age. - - A first verification whose end-to-end duration exceeds the whole - failure budget while every observable segment stays inside it - (queue wait, then a statement round trip) must arm the heartbeat - rather than hard-exit at startup. A genuinely wedged first beat - still ages out because it stamps nothing. - """ - - class AdvancingFirstBeatLedger(RecordingLeaseLedger): - writer_lease_fast_adoption_capable = True - writer_lease_guard_required = True - - def __init__(self) -> None: - super().__init__() - self.saw_query_start = threading.Event() - - def verify_writer_lease_guard_session( - self, - *, - on_query_start=None, - ) -> dict[str, int | str]: - time.sleep(0.10) - if on_query_start is not None: - on_query_start() - self.saw_query_start.set() - time.sleep(0.10) - return {"backend": "recording", "verified_count": 1} - - ledger = AdvancingFirstBeatLedger() - server = coordinator(ledger) - server.ledger_lease_heartbeat_seconds = 0.05 - server.ledger_lease_heartbeat_failure_seconds = 0.16 - server.ledger_lease_heartbeat_monitor_seconds = 0.01 - - with patch( - "lab.prism.prism_coordinator.os._exit", - side_effect=AssertionError("unexpected hard exit"), - ) as process_exit, patch.object( - server, - "_watchdog_hard_exit", - ) as hard_exit: - thread = server._start_ledger_lease_heartbeat() - self.assertIsNotNone(thread) - self.assertTrue(ledger.saw_query_start.is_set()) - assert thread is not None - self.assertTrue(server._stop_ledger_lease_heartbeat()) - thread.join(0.5) - - process_exit.assert_not_called() - hard_exit.assert_not_called() - - def test_server_proven_cap_preserves_adoption_envelope_after_death( - self, - ) -> None: - """Client-side marks must not stretch hard exit past adoption. - - A silent guard-session death leaves one attempt-start and one - query-slot mark up to an idle interval after the death, so the - plain activity budget alone would fire only at - death + interval + budget — past a replacement's CAS eligibility - when interval + budget reaches the adoption silence. The - server-proven cap measures from completed round trips, which - cannot postdate the death, and must fire first (cap 0.26s here - versus interval + budget = 0.30s). - """ - first_done = threading.Event() - release_hang = threading.Event() - calls: list[int] = [] - - class DyingSessionLedger(RecordingLeaseLedger): - writer_lease_fast_adoption_capable = True - writer_lease_guard_required = True - _lease_adoption_silence_seconds = 0.28 - - def verify_writer_lease_guard_session( - self, - *, - on_query_start=None, - ) -> dict[str, int | str]: - calls.append(len(calls)) - if on_query_start is not None: - on_query_start() - if len(calls) == 1: - first_done.set() - return {"backend": "recording", "verified_count": 1} - # The guard session died silently between beats: this - # statement black-holes with no error and no response. - release_hang.wait(5.0) - return {"backend": "recording", "verified_count": 1} - - ledger = DyingSessionLedger() - server = coordinator(ledger) - server.ledger_lease_heartbeat_seconds = 0.05 - server.ledger_lease_heartbeat_failure_seconds = 0.25 - server.ledger_lease_heartbeat_monitor_seconds = 0.005 - server.ledger_lease_heartbeat_exit_timeout_seconds = 0.005 - # The scheduler slack is a term of the safety inequality, so a - # scaled-down policy has to scale it too: left at the production - # value it would not fit inside this 0.28s silence and the - # heartbeat would (correctly) refuse to start. - server.ledger_lease_heartbeat_scheduler_slack_seconds = 0.005 - thread: threading.Thread | None = None - - try: - with patch("builtins.print"), patch.object( - server, - "_watchdog_hard_exit", - ) as hard_exit: - thread = server._start_ledger_lease_heartbeat() - self.assertIsNotNone(thread) - self.assertTrue(first_done.wait(0.5)) - deadline = time.monotonic() + 1.0 - while not hard_exit.called and time.monotonic() < deadline: - time.sleep(0.002) - self.assertTrue(hard_exit.called) - reason = str( - getattr(server, "_ledger_lease_heartbeat_failure_reason", "") - ) - match = re.search( - r"progress for ([0-9.]+)s \(server-proven ([0-9.]+)s\)", - reason, - ) - self.assertIsNotNone(match, reason) - assert match is not None - activity_age = float(match.group(1)) - server_age = float(match.group(2)) - # The cap fired on stale server evidence while the client - # marks were still inside the plain activity budget. - self.assertLess(activity_age, 0.25) - self.assertGreaterEqual(server_age, 0.26) - finally: - release_hang.set() - heartbeat_stop = getattr( - server, - "_ledger_lease_heartbeat_stop_event", - None, - ) - if heartbeat_stop is not None: - heartbeat_stop.set() - if thread is not None: - thread.join(0.5) - - def test_heartbeat_rejection_honors_private_silence_attribute(self) -> None: - """An operator silence override reaches the safety inequality. - - PsqlShareLedger keeps the adoption silence on a private attribute - with no public alias. The policy must read it: a 0.5s silence - cannot contain the default 1.25s failure budget plus the exit - envelope, so this is a real double-writer hazard, not a tuning - preference, and the heartbeat must refuse to start rather than - run with a broken exit-before-adoption argument. - """ - - class ShortSilenceLedger(GuardVerifyLeaseLedger): - _lease_adoption_silence_seconds = 0.5 - - ledger = ShortSilenceLedger() - server = coordinator(ledger) - server.ledger_lease_heartbeat_seconds = 0.01 - - with patch.object( - server, - "_ledger_lease_heartbeat_hard_exit", - ) as hard_exit: - self.assertIsNone(server._start_ledger_lease_heartbeat()) - - hard_exit.assert_called_once() - message = str(hard_exit.call_args.args[0]) - self.assertIn("refusing to start the ledger lease heartbeat", message) - self.assertIn("adoption silence 0.5s", message) - # No thread may exist: refusing means refusing to run, not - # starting and then complaining. - self.assertIsNone(server._ledger_lease_heartbeat_thread) - self.assertEqual(ledger.verify_calls, 0) - - def test_heartbeat_unsafe_timing_is_refused_at_start(self) -> None: - """A failure budget that overruns the adoption silence is refused. - - The budget the monitor enforces has to fit inside the silence - window alongside the exit envelope, or a coordinator that lost its - guarded session can still be running when a replacement becomes - adoption-eligible. Before issue #212 this printed a warning and - started anyway. - """ - ledger = GuardVerifyLeaseLedger() - server = coordinator(ledger) - server.ledger_lease_heartbeat_seconds = 0.01 - server.ledger_lease_heartbeat_failure_seconds = 2.5 - - with patch.object( - server, - "_ledger_lease_heartbeat_hard_exit", - ) as hard_exit: - self.assertIsNone(server._start_ledger_lease_heartbeat()) - - hard_exit.assert_called_once() - message = str(hard_exit.call_args.args[0]) - self.assertIn("refusing to start the ledger lease heartbeat", message) - self.assertIn("failure budget 2.5s", message) - self.assertIsNone(server._ledger_lease_heartbeat_thread) - - def test_heartbeat_interval_at_or_above_budget_is_refused(self) -> None: - """One idle wait must not be able to exhaust the liveness budget.""" - ledger = GuardVerifyLeaseLedger() - server = coordinator(ledger) - server.ledger_lease_heartbeat_seconds = 1.25 - server.ledger_lease_heartbeat_failure_seconds = 1.25 - - with patch.object( - server, - "_ledger_lease_heartbeat_hard_exit", - ) as hard_exit: - self.assertIsNone(server._start_ledger_lease_heartbeat()) - - hard_exit.assert_called_once() - self.assertIn( - "must exceed the heartbeat interval", - str(hard_exit.call_args.args[0]), - ) - - def test_heartbeat_warns_without_tail_latency_headroom(self) -> None: - """Safe but unstable timing warns instead of refusing. - - A silence window that still satisfies the exit-before-adoption - inequality but leaves the server-proven cap below the largest gap a - healthy coordinator can produce permits no double writer — it just - guarantees issue #212's false exits. Labs and tests deliberately - run tiny policies, so this stays a warning. - """ - - class TightSilenceLedger(GuardVerifyLeaseLedger): - _lease_adoption_silence_seconds = 0.5 - - ledger = TightSilenceLedger() - server = coordinator(ledger) - server.ledger_lease_heartbeat_seconds = 0.01 - server.ledger_lease_heartbeat_failure_seconds = 0.2 - server.ledger_lease_heartbeat_monitor_seconds = 0.005 - server.ledger_lease_heartbeat_exit_timeout_seconds = 0.01 - # Scaled with the rest of the policy so this stays the safe-but-tight - # case: the guard's own statement timeout is not scaled, so the - # staleness cap still lands below the healthy-gap bound and the - # advisory fires. - server.ledger_lease_heartbeat_scheduler_slack_seconds = 0.005 - - with patch("builtins.print") as printed, patch.object( - server, - "_ledger_lease_heartbeat_hard_exit", - ) as hard_exit: - thread = server._start_ledger_lease_heartbeat() - self.assertIsNotNone(thread) - self.assertTrue(server._stop_ledger_lease_heartbeat()) - assert thread is not None - thread.join(0.2) - - hard_exit.assert_not_called() - warnings = [ - call.args[0] - for call in printed.call_args_list - if call.args and "no tail-latency headroom" in str(call.args[0]) - ] - self.assertEqual(len(warnings), 1) - - def test_heartbeat_default_timing_does_not_warn(self) -> None: - ledger = GuardVerifyLeaseLedger() - server = coordinator(ledger) - server.ledger_lease_heartbeat_seconds = 0.01 - - with patch("builtins.print") as printed: - thread = server._start_ledger_lease_heartbeat() - self.assertIsNotNone(thread) - self.assertTrue(server._stop_ledger_lease_heartbeat()) - assert thread is not None - thread.join(0.2) - for call in printed.call_args_list: - if call.args: - self.assertNotIn( - "no tail-latency headroom", - str(call.args[0]), - ) - self.assertNotIn( - "heartbeat timing is misconfigured", - str(call.args[0]), - ) - - def test_external_fence_reports_recheck_progress_to_the_monitor( - self, - ) -> None: - """The fence's verification feeds the monitor's progress mark too. - - The verification holds the guard's serialized slot for its whole - statement sequence, so the periodic heartbeat queues behind it and - records nothing meanwhile. Without this caller passing the - progress callback, a fence verification's lawful second statement - ages the heartbeat's last success past the single-statement - failure budget and the monitor hard-exits a healthy coordinator — - the hazard already closed for the heartbeat loop's own recheck. - """ - - class RecheckFenceLedger(RecordingLeaseLedger): - writer_lease_fast_adoption_capable = True - writer_lease_guard_required = True - - def __init__(self) -> None: - super().__init__() - self.saw_progress_callback = threading.Event() - - def verify_writer_lease_guard_session( - self, - *, - on_query_start=None, - on_statement_progress=None, - ) -> dict[str, int | str]: - if on_query_start is not None: - on_query_start() - if on_statement_progress is not None: - # The attribution recheck's first statement completed. - on_statement_progress() - self.saw_progress_callback.set() - return { - "backend": "recording", - "verified_count": 1, - "renewed_count": 1, - } - - ledger = RecheckFenceLedger() - server = coordinator(ledger) - - server._require_fresh_ledger_lease_for_external_side_effect( - "submitblock" - ) - - self.assertTrue(ledger.saw_progress_callback.is_set()) - self.assertIsNotNone( - getattr( - server, - "_ledger_lease_heartbeat_last_progress_monotonic", - None, - ) - ) - - def test_recheck_statement_progress_keeps_monitor_from_hard_exiting( - self, - ) -> None: - """A lawful two-statement verification outlasts the failure budget. - - The monitor's budget must stay under the adoption silence, so it is - sized for one statement; the attribution recheck's second statement - would age the last success past it (two in-deadline statements plus - the heartbeat interval exceed the budget below). The completed - first round trip is the same session-answers evidence a success - proves, so its progress mark must keep the monitor from restarting - a healthy coordinator, while a wedged statement — which reports no - progress — still ages out (previous test). - """ - first_statement_seconds = 0.2 - second_statement_seconds = 0.2 - - class RecheckingLedger(HeartbeatLeaseLedger): - def __init__(self) -> None: - super().__init__() - self.verify_calls = 0 - self.recheck_completed = threading.Event() - - def verify_writer_lease_guard_session( - self, - *, - on_query_start=None, - on_statement_progress=None, - ) -> dict[str, int | str]: - if on_query_start is not None: - on_query_start() - self.verify_calls += 1 - if self.verify_calls == 1: - # Seed the success stamp with a fast verification. - return {"backend": "recording", "renewed_count": 1} - # The ambiguous shape: both statements individually within - # the guard's statement deadline, together past the - # monitor's failure budget. - time.sleep(first_statement_seconds) - if on_statement_progress is not None: - on_statement_progress() - time.sleep(second_statement_seconds) - self.recheck_completed.set() - return {"backend": "recording", "renewed_count": 1} - - ledger = RecheckingLedger() - server = coordinator(ledger) - server.ledger_lease_heartbeat_seconds = 0.01 - server.ledger_lease_heartbeat_failure_seconds = 0.3 - server.ledger_lease_heartbeat_monitor_seconds = 0.005 - thread: threading.Thread | None = None - - try: - with patch( - "lab.prism.prism_coordinator.os._exit", - side_effect=AssertionError("unexpected hard exit"), - ) as process_exit, patch.object( - server, - "_watchdog_hard_exit", - ) as hard_exit: - thread = server._start_ledger_lease_heartbeat() - self.assertIsNotNone(thread) - self.assertTrue(ledger.recheck_completed.wait(2)) - process_exit.assert_not_called() - hard_exit.assert_not_called() - finally: - heartbeat_stop = getattr( - server, - "_ledger_lease_heartbeat_stop_event", - None, - ) - if heartbeat_stop is not None: - heartbeat_stop.set() - if thread is not None: - thread.join(1) - - def test_watchdog_exit_releases_on_fresh_thread_within_deadline(self) -> None: - ledger = WatchdogLeaseLedger() - server = coordinator(ledger) - server.watchdog_lease_release_timeout_seconds = 0.2 - watchdog_thread = threading.current_thread() - - started = time.monotonic() - with patch("builtins.print"), patch( - "lab.prism.prism_coordinator.os._exit", - side_effect=SystemExit(1), - ) as hard_exit, self.assertRaises(SystemExit): - server._watchdog_hard_exit("liveness") - elapsed = time.monotonic() - started - - hard_exit.assert_called_once_with(1) - self.assertLess(elapsed, 0.2) - self.assertEqual(ledger.release_calls, 1) - self.assertIsNotNone(ledger.release_thread) - self.assertIsNot(ledger.release_thread, watchdog_thread) - self.assertEqual(ledger.release_thread.name, "prism-watchdog-lease-release") - - def test_all_watchdog_branches_arm_exit_without_blocking_output(self) -> None: - cases = ( - ("coordination", ("coordination", 2.0, 1.0, 1.0), []), - ("publication", ("publication", 0.0, 1.0, 1.0), []), - ("liveness", (None, 0.0, 1.0, 1.0), ["block_submitter"]), - ) - for expected_reason, publication_state, overdue in cases: - with self.subTest(reason=expected_reason): - server = coordinator() - server.watchdog_interval_seconds = 0.0 - server.watchdog_timeout_seconds = 300.0 - server.watchdog_enabled = True - server._publication_watchdog_state = ( # type: ignore[method-assign] - lambda _now, state=publication_state: state - ) - server._overdue_heartbeats = ( # type: ignore[method-assign] - lambda _now, value=overdue: value - ) - release_print = threading.Event() - hard_exit_called = threading.Event() - - def blocking_print(*_args: object, **_kwargs: object) -> None: - release_print.wait(1) - - def hard_exit(reason: str) -> None: - self.assertEqual(reason, expected_reason) - hard_exit_called.set() - raise SystemExit(1) - - with patch( - "builtins.print", - side_effect=blocking_print, - ) as print_call, patch.object( - server, - "_watchdog_hard_exit", - side_effect=hard_exit, - ): - caller = threading.Thread(target=server.watchdog_loop) - caller.start() - reached_hard_exit = hard_exit_called.wait(0.1) - release_print.set() - caller.join(0.2) - - self.assertTrue(reached_hard_exit) - self.assertFalse(caller.is_alive()) - print_call.assert_not_called() - - def test_watchdog_exit_hard_exits_when_fresh_db_release_hangs(self) -> None: - release_started = threading.Event() - unblock_release = threading.Event() - release_finished = threading.Event() - - class BlockingLedger(RecordingLeaseLedger): - def release_writer_lease_fresh_connection(self) -> bool: - release_started.set() - unblock_release.wait(1) - release_finished.set() - return True - - server = coordinator(BlockingLedger()) - server.watchdog_lease_release_timeout_seconds = 0.02 - - started = time.monotonic() - try: - with patch("builtins.print"), patch( - "lab.prism.prism_coordinator.os._exit", - side_effect=SystemExit(1), - ) as hard_exit, self.assertRaises(SystemExit): - server._watchdog_hard_exit("publication") - finally: - unblock_release.set() - elapsed = time.monotonic() - started - - hard_exit.assert_called_once_with(1) - self.assertTrue(release_started.is_set()) - self.assertLess(elapsed, 0.2) - self.assertTrue(release_finished.wait(0.2)) - - def test_watchdog_release_never_waits_on_shutdown_logging(self) -> None: - ledger = WatchdogLeaseLedger() - server = coordinator(ledger) - server.watchdog_lease_release_timeout_seconds = 0.05 - release_log = threading.Event() - - def blocking_log(*_args: object, **_kwargs: object) -> None: - release_log.wait(1) - - try: - with patch.object( - server, - "_shutdown_log", - side_effect=blocking_log, - ) as shutdown_log, patch("builtins.print"), patch( - "lab.prism.prism_coordinator.os._exit", - side_effect=SystemExit(1), - ), self.assertRaises(SystemExit): - server._watchdog_hard_exit("liveness") - released_before_log_unblocked = ledger.released.is_set() - finally: - release_log.set() - - self.assertTrue(released_before_log_unblocked) - shutdown_log.assert_not_called() - - def test_watchdog_blocked_diagnostic_cannot_extend_exit_deadline(self) -> None: - ledger = WatchdogLeaseLedger() - server = coordinator(ledger) - server.watchdog_lease_release_timeout_seconds = 0.02 - diagnostic_started = threading.Event() - release_diagnostic = threading.Event() - - def blocking_print(*_args: object, **_kwargs: object) -> None: - diagnostic_started.set() - release_diagnostic.wait(1) - - started = time.monotonic() - try: - with patch("builtins.print", side_effect=blocking_print), patch( - "lab.prism.prism_coordinator.os._exit", - side_effect=SystemExit(1), - ), self.assertRaises(SystemExit): - server._watchdog_hard_exit("liveness") - elapsed = time.monotonic() - started - finally: - release_diagnostic.set() - if ledger.release_thread is not None: - ledger.release_thread.join(0.2) - - self.assertTrue(ledger.released.is_set()) - self.assertTrue(diagnostic_started.is_set()) - self.assertLess(elapsed, 0.2) - - def test_watchdog_exit_hard_exits_when_release_thread_cannot_start(self) -> None: - server = coordinator() - - with patch( - "lab.prism.prism_coordinator.threading.Thread", - side_effect=RuntimeError("cannot start new thread"), - ), patch( - "lab.prism.prism_coordinator.os._exit", - side_effect=SystemExit(1), - ) as hard_exit, self.assertRaises(SystemExit): - server._watchdog_hard_exit("liveness") - - hard_exit.assert_called_once_with(1) - - def test_watchdog_exit_withholds_release_while_writer_is_active(self) -> None: - ledger = WatchdogLeaseLedger() - server = coordinator(ledger) - server.watchdog_lease_release_timeout_seconds = 0.02 - controller = server._ensure_shutdown_controller() - active_writer = controller.reserve_writer("block_submitter") - - try: - with patch("builtins.print"), patch( - "lab.prism.prism_coordinator.os._exit", - side_effect=SystemExit(1), - ), self.assertRaises(SystemExit): - server._watchdog_hard_exit("liveness") - deadline = time.monotonic() + 0.2 - while ( - not controller.snapshot()["lease_release_withheld"] - and time.monotonic() < deadline - ): - time.sleep(0.001) - self.assertTrue(controller.snapshot()["lease_release_withheld"]) - finally: - active_writer.finish() - - self.assertEqual(ledger.release_calls, 0) - - def test_withheld_shutdown_keeps_lease_heartbeat_alive(self) -> None: - ledger = HeartbeatLeaseLedger() - server = coordinator(ledger, timeout=0.02) - server.ledger_lease_heartbeat_seconds = 0.005 - heartbeat_thread = server._start_ledger_lease_heartbeat() - self.assertIsNotNone(heartbeat_thread) - controller = server._ensure_shutdown_controller() - active_writer = controller.reserve_writer("accepted_block_handling") - renewals_before_shutdown = ledger.renew_calls - - try: - with patch("builtins.print"): - self.assertFalse(server.shutdown(reason="active_writer")) - deadline = time.monotonic() + 0.1 - while ( - ledger.renew_calls <= renewals_before_shutdown - and time.monotonic() < deadline - ): - time.sleep(0.001) - - self.assertGreater(ledger.renew_calls, renewals_before_shutdown) - assert heartbeat_thread is not None - self.assertTrue(heartbeat_thread.is_alive()) - self.assertEqual(ledger.release_calls, 0) - finally: - active_writer.finish() - self.assertTrue(server._stop_ledger_lease_heartbeat()) - - def test_normal_shutdown_releases_lease_promptly_and_exports_metrics(self) -> None: - ledger = RecordingLeaseLedger() - server = coordinator(ledger) - - started = time.monotonic() - with patch("builtins.print"): - self.assertTrue(server.shutdown(reason="normal_return")) - elapsed = time.monotonic() - started - - self.assertLess(elapsed, 0.2) - self.assertEqual(ledger.release_calls, 1) - metrics = "\n".join(server.shutdown_metrics_lines()) - self.assertIn("qbit_prism_shutdowns_total 1", metrics) - self.assertIn( - 'qbit_prism_shutdown_writer_quiescence_total{outcome="success"} 1', - metrics, - ) - self.assertIn( - 'qbit_prism_shutdown_lease_release_total{outcome="success"} 1', - metrics, - ) - - def test_safe_release_stops_lease_heartbeat_before_database_handoff(self) -> None: - release_saw_heartbeat_alive: list[bool] = [] - - class OrderedHeartbeatLedger(HeartbeatLeaseLedger): - def release_writer_lease(self) -> bool: - heartbeat = getattr(server, "_ledger_lease_heartbeat_thread", None) - release_saw_heartbeat_alive.append( - bool(heartbeat is not None and heartbeat.is_alive()) - ) - return super().release_writer_lease() - - ledger = OrderedHeartbeatLedger() - server = coordinator(ledger) - self.assertIsNotNone(server._start_ledger_lease_heartbeat()) - - with patch("builtins.print"): - self.assertTrue(server.shutdown(reason="ordered_release")) - - self.assertEqual(release_saw_heartbeat_alive, [False]) - self.assertEqual(ledger.release_calls, 1) - - def test_blocked_non_writer_drain_starts_only_after_lease_release(self) -> None: - ledger = RecordingLeaseLedger() - server = coordinator(ledger) - drain_started = threading.Event() - unblock_drain = threading.Event() - - def blocked_executor_drain() -> None: - drain_started.set() - unblock_drain.wait(1) - - server.shutdown_tip_refresh_executor = blocked_executor_drain # type: ignore[method-assign] - with patch("builtins.print"): - self.assertTrue(server.shutdown()) - self.assertTrue(ledger.released.is_set()) - - drain_thread = threading.Thread(target=server.drain_non_writer_components) - with patch("builtins.print"): - drain_thread.start() - self.assertTrue(drain_started.wait(0.2)) - self.assertTrue(ledger.released.is_set()) - self.assertTrue(drain_thread.is_alive()) - unblock_drain.set() - drain_thread.join(1) - self.assertFalse(drain_thread.is_alive()) - - def test_audit_listener_stops_during_non_writer_drain_after_lease_release( - self, - ) -> None: - ledger = RecordingLeaseLedger() - server = coordinator(ledger) - stop_saw_release: list[bool] = [] - server._audit_http_facade = SimpleNamespace( - stop=lambda: stop_saw_release.append(ledger.released.is_set()) or True - ) - server._background_services = SimpleNamespace( - threads_to_drain=lambda: (), - ) - server.shutdown_vardiff_idle_executor = lambda: None # type: ignore[method-assign] - server.shutdown_tip_refresh_executor = lambda: None # type: ignore[method-assign] - - with patch("builtins.print"): - self.assertTrue(server.shutdown()) - server.drain_non_writer_components() - - self.assertEqual(stop_saw_release, [True]) - - def test_audit_listener_stop_timeout_is_logged_during_drain(self) -> None: - # The facade's bounded stop() reports False on a stop_timeout; the - # drain must surface that as an audit_http_stop shutdown log line - # instead of blocking or swallowing it. - ledger = RecordingLeaseLedger() - server = coordinator(ledger) - shutdown_events: list[tuple[str, dict[str, object]]] = [] - server._shutdown_log = ( # type: ignore[method-assign] - lambda event, **fields: shutdown_events.append((event, fields)) - ) - server._audit_http_facade = SimpleNamespace(stop=lambda: False) - server._background_services = SimpleNamespace( - threads_to_drain=lambda: (), - ) - server.shutdown_vardiff_idle_executor = lambda: None # type: ignore[method-assign] - server.shutdown_tip_refresh_executor = lambda: None # type: ignore[method-assign] - - with patch("builtins.print"): - self.assertTrue(server.shutdown()) - server.drain_non_writer_components() - - self.assertIn(("audit_http_stop", {"outcome": "timeout"}), shutdown_events) - - def test_pending_share_batch_flushes_before_release(self) -> None: - append_started = threading.Event() - allow_flush = threading.Event() - release_saw_ack: list[bool] = [] - timeline: list[str] = [] - - class Ledger(RecordingLeaseLedger): - def append_batch(self, entries: object) -> list[object]: - append_started.set() - allow_flush.wait(1) - timeline.append("share_flush") - return [SimpleNamespace(share_seq=1)] - - def release_writer_lease(self) -> bool: - release_saw_ack.append(entry.committed.is_set()) - timeline.append("lease_release") - return super().release_writer_lease() - - ledger = Ledger() - server = coordinator(ledger, timeout=1) - server.share_append_queue = queue.Queue(maxsize=2) - entry = PendingShareAppend( - pending_share=SimpleNamespace( - job_issued_at_ms=0, - accepted_at_ms=0, - ), - username="miner-a", - job_id="job-a", - block_hash_hex="aa" * 32, - collection_only=False, - credit_policy=None, - ) - server.enqueue_share_append(entry) - - def flush_one() -> None: - queued = server.share_append_queue.get_nowait() - server._append_share_batch([queued]) - - writer_thread = threading.Thread(target=flush_one) - writer_thread.start() - self.assertTrue(append_started.wait(0.2)) - - shutdown_result: list[bool] = [] - shutdown_thread = threading.Thread( - target=lambda: shutdown_result.append(server.shutdown()) - ) - with patch("builtins.print"): - shutdown_thread.start() - time.sleep(0.03) - self.assertEqual(ledger.release_calls, 0) - allow_flush.set() - writer_thread.join(1) - shutdown_thread.join(1) - - self.assertEqual(shutdown_result, [True]) - self.assertEqual(timeline, ["share_flush", "lease_release"]) - self.assertEqual(release_saw_ack, [True]) - - def test_share_writer_stays_alive_for_admitted_not_yet_queued_submit(self) -> None: - class Ledger(RecordingLeaseLedger): - def append_batch(self, entries: object) -> list[object]: - return [SimpleNamespace(share_seq=1)] - - ledger = Ledger() - server = coordinator(ledger, timeout=1) - server.share_append_queue = queue.Queue(maxsize=2) - server.share_commit_batch_size = 1 - server.share_commit_linger_seconds = 0 - server.share_commit_timeout_seconds = 1 - server._record_heartbeat = lambda _name: None # type: ignore[method-assign] - producer_admitted = threading.Event() - allow_enqueue = threading.Event() - - entry = PendingShareAppend( - pending_share=SimpleNamespace( - job_issued_at_ms=0, - accepted_at_ms=0, - ), - username="miner-a", - job_id="job-a", - block_hash_hex="aa" * 32, - collection_only=False, - credit_policy=None, - ) - - def producer() -> None: - with server._writer_operation("share_submission"): - producer_admitted.set() - allow_enqueue.wait(1) - server.enqueue_share_append(entry, wait=True) - - producer_thread = threading.Thread(target=producer) - writer_thread = threading.Thread(target=server.share_append_loop) - producer_thread.start() - writer_thread.start() - self.assertTrue(producer_admitted.wait(0.2)) - - shutdown_thread = threading.Thread(target=server.shutdown) - with patch("builtins.print"): - shutdown_thread.start() - time.sleep(0.03) - self.assertTrue(writer_thread.is_alive()) - self.assertEqual(ledger.release_calls, 0) - allow_enqueue.set() - producer_thread.join(1) - writer_thread.join(1) - shutdown_thread.join(1) - - self.assertFalse(producer_thread.is_alive(), "producer thread leaked") - self.assertFalse(writer_thread.is_alive(), "share writer thread leaked") - self.assertFalse(shutdown_thread.is_alive(), "shutdown thread leaked") - self.assertTrue(entry.committed.is_set()) - self.assertEqual(ledger.release_calls, 1) - - def test_event_only_stop_cannot_strand_late_admitted_share(self) -> None: - class Ledger(RecordingLeaseLedger): - def append_batch(self, entries: object) -> list[object]: - return [SimpleNamespace(share_seq=1)] - - server = coordinator(Ledger()) - server.share_append_queue = queue.Queue(maxsize=2) - server.share_commit_batch_size = 1 - server.share_commit_linger_seconds = 0 - server._record_heartbeat = lambda _name: None # type: ignore[method-assign] - entry = PendingShareAppend( - pending_share=SimpleNamespace( - job_issued_at_ms=0, - accepted_at_ms=0, - ), - username="miner-a", - job_id="job-a", - block_hash_hex="aa" * 32, - collection_only=False, - credit_policy=None, - ) - - # Model a request that passed admission immediately before a legacy - # event-only stop. The writer must remain available until admission is - # actually closed, so the share cannot be stranded behind its exit. - server.stop_event.set() - writer_thread = threading.Thread(target=server.share_append_loop) - writer_thread.start() - time.sleep(0.25) - self.assertTrue(writer_thread.is_alive()) - - server.enqueue_share_append(entry) - self.assertTrue(entry.committed.wait(1)) - server.request_shutdown(signal.SIGTERM) - writer_thread.join(1) - - self.assertFalse(writer_thread.is_alive()) - self.assertIsNotNone(entry.record) - - def test_blocked_writer_withholds_release_and_names_component(self) -> None: - ledger = RecordingLeaseLedger() - server = coordinator(ledger, timeout=0.03) - entered = threading.Event() - unblock = threading.Event() - - def blocked_writer() -> None: - with server._writer_operation("accepted_block_handling"): - entered.set() - unblock.wait(1) - - writer = threading.Thread(target=blocked_writer) - writer.start() - self.assertTrue(entered.wait(0.2)) - with patch("builtins.print") as printed: - self.assertFalse(server.shutdown()) - - self.assertEqual(ledger.release_calls, 0) - snapshot = server._ensure_shutdown_controller().snapshot() - self.assertEqual(snapshot["release_withheld_total"], 1) - rendered = " ".join(str(call) for call in printed.call_args_list) - self.assertIn("accepted_block_handling", rendered) - unblock.set() - writer.join(1) - controller = server._ensure_shutdown_controller() - self.assertTrue(controller.claim_non_writer_drain()) - controller.finish_non_writer_drain(0.0) - with patch("builtins.print"): - self.assertFalse(server.shutdown(reason="finally")) - self.assertFalse(server.release_ledger_lease()) - self.assertEqual(ledger.release_calls, 0) - self.assertTrue(controller.snapshot()["lease_release_withheld"]) - - def test_repeated_shutdown_and_finally_release_at_most_once(self) -> None: - ledger = RecordingLeaseLedger() - server = coordinator(ledger) - with patch("builtins.print"): - self.assertTrue(server.shutdown(reason="serve_exit")) - self.assertTrue(server.shutdown(reason="main_finally")) - self.assertTrue(server.release_ledger_lease()) - self.assertEqual(ledger.release_calls, 1) - - def test_sigterm_closes_writer_admission_and_records_release_latency(self) -> None: - ledger = RecordingLeaseLedger() - server = coordinator(ledger) - server.request_shutdown(signal.SIGTERM) - - with self.assertRaises(ShutdownInProgress): - with server._writer_operation("share_submission"): - pass - with patch("builtins.print"): - self.assertTrue(server.shutdown(reason="signal")) - - snapshot = server._ensure_shutdown_controller().snapshot() - self.assertTrue(snapshot["sigterm_release_observed"]) - self.assertGreaterEqual(snapshot["sigterm_to_lease_release_seconds"], 0) - - def test_submit_admission_race_returns_pool_closed_stratum_error(self) -> None: - server = coordinator() - - def rejected_submit(_client: object, _params: object) -> bool: - server.request_shutdown(signal.SIGTERM) - raise ShutdownInProgress("PRISM coordinator is shutting down") - - server.handle_submit = rejected_submit # type: ignore[method-assign] - with self.assertRaises(StratumError) as raised: - server.handle_request( - SimpleNamespace(), - {"id": 1, "method": "mining.submit", "params": []}, - ) - - self.assertEqual(raised.exception.code, 20) - self.assertEqual(raised.exception.reason, PRISM_REJECTION_POOL_CLOSED) - self.assertTrue(raised.exception.disconnect) - - def test_block_submit_loop_exits_if_shutdown_wins_admission_race(self) -> None: - server = coordinator() - submit_calls: list[bool] = [] - server._record_heartbeat = lambda _name: None # type: ignore[method-assign] - - def rejected_replay() -> int: - server.request_shutdown(signal.SIGTERM) - raise ShutdownInProgress("PRISM coordinator is shutting down") - - server.replay_pending_block_candidates = rejected_replay # type: ignore[method-assign] - server.submit_next_block_candidate = ( # type: ignore[method-assign] - lambda **_kwargs: submit_calls.append(True) or False - ) - - server.block_submit_loop() - - self.assertEqual(submit_calls, []) - - def test_blockpoll_shutdown_race_does_not_take_hard_exit_path(self) -> None: - server = coordinator() - server.blockpoll_seconds = 0 - server._record_heartbeat = lambda _name: None # type: ignore[method-assign] - - def rejected_poll() -> int: - server.request_shutdown(signal.SIGTERM) - raise ShutdownInProgress("PRISM coordinator is shutting down") - - server.poll_qbit_tip_template_once = rejected_poll # type: ignore[method-assign] - with patch("lab.prism.prism_coordinator.os._exit") as hard_exit: - server.blockpoll_loop() - - hard_exit.assert_not_called() - - def test_ctv_loop_exits_cleanly_if_shutdown_wins_admission_race(self) -> None: - server = coordinator() - pass_observations: list[float] = [] - server._record_heartbeat = lambda _name: None # type: ignore[method-assign] - server.observe_ctv_fanout_broadcaster_pass = ( # type: ignore[method-assign] - pass_observations.append - ) - - def rejected_pass(**_kwargs: object) -> object: - server.request_shutdown(signal.SIGTERM) - raise ShutdownInProgress("PRISM coordinator is shutting down") - - server.run_ctv_fanout_broadcaster_once = rejected_pass # type: ignore[method-assign] - with patch("builtins.print") as printed, patch( - "lab.prism.prism_coordinator.traceback.print_exc" - ) as traceback_printed: - server.ctv_fanout_broadcaster_loop() - - self.assertEqual(pass_observations, []) - printed.assert_not_called() - traceback_printed.assert_not_called() - - def test_startup_replays_stop_cleanly_after_shutdown_admission_closes(self) -> None: - server = coordinator() - server.request_shutdown(signal.SIGTERM) - - for replay in ( - server.replay_pending_block_candidates, - server.replay_recovered_shares, - ): - with self.subTest(replay=replay.__name__): - self.assertFalse(server._run_startup_writer_replay(replay)) - - drain_threads = [(threading.current_thread(), 0.0)] - timeline: list[object] = [] - server.shutdown = ( # type: ignore[method-assign] - lambda *, reason: timeline.append(("shutdown", reason)) or True - ) - server.drain_non_writer_components = ( # type: ignore[method-assign] - lambda threads: timeline.append(("drain", threads)) - ) - self.assertFalse( - server._run_startup_writer_replay( - server.replay_recovered_shares, - drain_threads=drain_threads, - ) - ) - self.assertEqual( - timeline, - [ - ("shutdown", "serve_startup_exit"), - ("drain", drain_threads), - ], - ) - - def test_replacement_can_acquire_immediately_after_graceful_release(self) -> None: - lease_lock = threading.Lock() - holder: list[str | None] = [None] - - class LeaseLedger(RecordingLeaseLedger): - def __init__(self, session: str) -> None: - super().__init__() - self.session = session - - def acquire(self) -> bool: - with lease_lock: - if holder[0] is not None: - return False - holder[0] = self.session - return True - - def release_writer_lease(self) -> bool: - with lease_lock: - if holder[0] != self.session: - return False - holder[0] = None - return super().release_writer_lease() - - old = LeaseLedger("old") - replacement = LeaseLedger("replacement") - self.assertTrue(old.acquire()) - server = coordinator(old) - with patch("builtins.print"): - self.assertTrue(server.shutdown()) - self.assertTrue(replacement.acquire()) - - def test_no_ledger_mutation_is_admitted_after_release(self) -> None: - ledger = RecordingLeaseLedger() - server = coordinator(ledger) - with patch("builtins.print"): - self.assertTrue(server.shutdown()) - - with self.assertRaises(ShutdownInProgress): - server.replay_recovered_shares() - self.assertEqual(ledger.release_calls, 1) - - def test_shutdown_race_preserves_single_writer_invariant(self) -> None: - ledger = RecordingLeaseLedger() - server = coordinator(ledger, timeout=1) - admitted = threading.Event() - finish_writer = threading.Event() - mutation_after_release: list[bool] = [] - - def existing_writer() -> None: - with server._writer_operation("payout_reconciliation"): - admitted.set() - finish_writer.wait(1) - mutation_after_release.append(ledger.released.is_set()) - - writer = threading.Thread(target=existing_writer) - writer.start() - self.assertTrue(admitted.wait(0.2)) - server.request_shutdown(signal.SIGTERM) - with self.assertRaises(ShutdownInProgress): - with server._writer_operation("ctv_broadcast_state"): - pass - - shutdown_thread = threading.Thread(target=server.shutdown) - with patch("builtins.print"): - shutdown_thread.start() - time.sleep(0.03) - self.assertFalse(ledger.released.is_set()) - finish_writer.set() - writer.join(1) - shutdown_thread.join(1) - - self.assertEqual(mutation_after_release, [False]) - self.assertTrue(ledger.released.is_set()) - - def test_main_normal_sigterm_and_exception_paths_run_controlled_finally(self) -> None: - handlers: dict[int, object] = {} - - class FakeCoordinator: - def __init__(self, mode: str) -> None: - self.mode = mode - self.events: list[object] = [] - - def request_shutdown(self, signum: int | None = None) -> None: - self.events.append(("request", signum)) - - def serve(self) -> None: - self.events.append("serve") - if self.mode == "sigterm": - handler = handlers[signal.SIGTERM] - assert callable(handler) - handler(signal.SIGTERM, None) - if self.mode == "exception": - raise RuntimeError("serve failed") - - def shutdown(self, *, reason: str) -> bool: - self.events.append(("shutdown", reason)) - return True - - def drain_non_writer_components(self) -> None: - self.events.append("drain") - - for mode in ("normal", "sigterm", "exception"): - with self.subTest(mode=mode): - fake = FakeCoordinator(mode) - handlers.clear() - with patch.object( - prism_coordinator, - "PrismCoordinator", - return_value=fake, - ), patch.object( - prism_coordinator.signal, - "signal", - side_effect=lambda signum, handler: handlers.__setitem__(signum, handler), - ): - if mode == "exception": - with self.assertRaisesRegex(RuntimeError, "serve failed"): - prism_coordinator.main() - else: - self.assertEqual(prism_coordinator.main(), 0) - if mode == "sigterm": - self.assertIn(("request", signal.SIGTERM), fake.events) - self.assertEqual(fake.events[-2:], [("shutdown", "main_finally"), "drain"]) - - -if __name__ == "__main__": - unittest.main() diff --git a/tests/test_prism_ctv_refresh_priority.py b/tests/test_prism_ctv_refresh_priority.py deleted file mode 100644 index 3ad28dc9e..000000000 --- a/tests/test_prism_ctv_refresh_priority.py +++ /dev/null @@ -1,315 +0,0 @@ -#!/usr/bin/env python3 -"""Focused coordinator coverage for CTV yielding to new-tip refreshes.""" - -from __future__ import annotations - -import threading -import unittest -from types import SimpleNamespace -from unittest.mock import patch - -from lab.prism.ctv_broadcaster import CONFIRMED, BroadcastAttempt -from lab.prism.ctv_broadcaster_daemon import ( - CtvFanoutBroadcastDaemon, - CtvFanoutChunkResult, - CtvFanoutDaemonResult, -) -from lab.prism.prism_coordinator import PrismCoordinator - - -OLD_TIP = "11" * 32 -NEW_TIP = "22" * 32 -NEWER_TIP = "33" * 32 - - -def pending_row(index: int) -> dict[str, object]: - return { - "fanout_txid": f"{index + 1:064x}", - "fanout_tx_hex": "0300000001", - "anchor_vout": None, - "parent_coinbase_txid": "aa" * 32, - "parent_coinbase_vout": 1, - "block_hash": "bb" * 32, - "block_height": 100 + index, - "settlement_status": "broadcastable", - "row_index": index, - } - - -class FakeLedger: - def __init__(self, rows: list[dict[str, object]]) -> None: - self.rows = rows - self.lock = threading.Lock() - self.updates: list[tuple[str, str]] = [] - - def pending_ctv_fanout_statuses(self, *, limit: int = 100) -> list[dict[str, object]]: - with self.lock: - eligible = [ - dict(row) - for row in self.rows - if row["settlement_status"] not in {"confirmed", "reorged", "failed"} - ] - return eligible[:limit] - - def update_ctv_fanout_status( - self, - *, - fanout_txid: str, - settlement_status: str, - ) -> dict[str, int | str]: - with self.lock: - self.updates.append((fanout_txid, settlement_status)) - for row in self.rows: - if row["fanout_txid"] == fanout_txid: - row["settlement_status"] = settlement_status - break - return {"backend": "fake", "updated_count": 1} - - def record_ctv_fanout_broadcast_attempt(self, **_kwargs: object) -> dict[str, int | str]: - raise AssertionError("confirmed fake rows must use the status-update path") - - -class BlockingBroadcaster: - def __init__(self) -> None: - self.first_row_entered = threading.Event() - self.release_first_row = threading.Event() - self.calls: list[str] = [] - - def broadcast(self, artifact: object, _fee_sats: int) -> BroadcastAttempt: - fanout_txid = str(getattr(artifact, "fanout_txid")) - self.calls.append(fanout_txid) - if len(self.calls) == 1: - self.first_row_entered.set() - if not self.release_first_row.wait(timeout=2.0): - raise AssertionError("test did not release the first CTV chunk") - return BroadcastAttempt( - fanout_txid=fanout_txid, - status=CONFIRMED, - submitted=False, - ) - - -class ParentRpc: - def call(self, method: str, params: list[object] | None = None) -> object: - if method == "getblock": - assert params is not None - return {"previousblockhash": OLD_TIP} - raise AssertionError(f"unexpected RPC {method}") - - -class StopAfterOnePass: - def is_set(self) -> bool: - return False - - def wait(self, _timeout: float) -> bool: - return True - - -def coordinator() -> PrismCoordinator: - server = PrismCoordinator.__new__(PrismCoordinator) - server.lock = threading.RLock() - server.stop_event = threading.Event() - server.rpc = ParentRpc() - server.current_tip_first_seen = (OLD_TIP, None) - server.current_tip_parent = None - server.current_tip_observation_sequence = 1 - server.tip_observation_sequence = 1 - server.tip_template_snapshot = SimpleNamespace(bestblockhash=OLD_TIP) - server.evicted_job_graveyard = {} - server.prune_evicted_job_graveyard = lambda **_kwargs: None # type: ignore[method-assign] - server.ctv_broadcaster_limit = 3 - server.ctv_broadcaster_chunk_size = 1 - server.ctv_broadcaster_interval_seconds = 30.0 - server.ctv_fanout_broadcast_daemon = None - return server - - -class PrismCtvRefreshPriorityTests(unittest.TestCase): - def test_pending_tip_yields_after_current_chunk_and_later_pass_finishes_rows(self) -> None: - server = coordinator() - ledger = FakeLedger([pending_row(index) for index in range(3)]) - broadcaster = BlockingBroadcaster() - server.ctv_fanout_broadcast_daemon = CtvFanoutBroadcastDaemon( - ledger, - broadcaster, # type: ignore[arg-type] - fee_sats=0, - ) - results: list[CtvFanoutDaemonResult] = [] - errors: list[BaseException] = [] - - def run_pass() -> None: - try: - results.append(server.run_ctv_fanout_broadcaster_once()) - except BaseException as exc: # pragma: no cover - asserted below - errors.append(exc) - - pass_thread = threading.Thread(target=run_pass) - pass_thread.start() - self.assertTrue(broadcaster.first_row_entered.wait(timeout=2.0)) - - pending_token = server._mark_tip_refresh_pending(NEW_TIP) - self.assertTrue(server._tip_refresh_pending()) - broadcaster.release_first_row.set() - pass_thread.join(timeout=2.0) - - self.assertFalse(pass_thread.is_alive()) - self.assertEqual(errors, []) - self.assertEqual(len(results), 1) - self.assertEqual(results[0].scanned_count, 1) - self.assertTrue(results[0].yielded_to_tip_refresh) - self.assertEqual(len(broadcaster.calls), 1) - # The priority check is made only after the row's durable mutation has - # returned, so no ledger critical section survives the yielded pass. - self.assertTrue(ledger.lock.acquire(blocking=False)) - ledger.lock.release() - - server._clear_tip_refresh_pending(pending_token) - self.assertFalse(server._tip_refresh_pending()) - later_result = server.run_ctv_fanout_broadcaster_once() - - self.assertEqual(later_result.scanned_count, 2) - self.assertFalse(later_result.yielded_to_tip_refresh) - self.assertEqual(len(broadcaster.calls), 3) - self.assertEqual(len(set(broadcaster.calls)), 3) - self.assertEqual( - {row["settlement_status"] for row in ledger.rows}, - {"confirmed"}, - ) - - def test_older_refresh_cannot_clear_a_newer_pending_tip(self) -> None: - server = coordinator() - - first_token = server._mark_tip_refresh_pending(NEW_TIP) - newer_token = server._mark_tip_refresh_pending(NEWER_TIP) - - self.assertNotEqual(first_token, newer_token) - server._clear_tip_refresh_pending(first_token) - self.assertTrue(server._tip_refresh_pending()) - server._clear_tip_refresh_pending(newer_token) - self.assertFalse(server._tip_refresh_pending()) - - def test_poll_local_mark_cannot_replace_newer_pending_work(self) -> None: - server = coordinator() - - owned_token = server._claim_tip_refresh_pending() - self.assertIsNone(owned_token) - newer_token = server._mark_tip_refresh_pending(NEWER_TIP) - - retained_token = server._mark_tip_refresh_pending_for_poll( - owned_token, - NEW_TIP, - ) - - self.assertIsNone(retained_token) - self.assertEqual(server._tip_refresh_pending_token, newer_token) - self.assertTrue(server._tip_refresh_pending()) - - def test_routine_same_tip_observation_does_not_starve_ctv(self) -> None: - server = coordinator() - cancellations: list[bool] = [] - server._active_tip_refresh = ( # type: ignore[assignment] - SimpleNamespace(tip_hash=OLD_TIP, observation_sequence=1), - SimpleNamespace(cancel=lambda: cancellations.append(True)), - ) - - for _ in range(5): - self.assertTrue(server.observe_tip_first_seen(OLD_TIP)) - - self.assertEqual(cancellations, []) - self.assertEqual(server.current_tip_observation_sequence, 1) - self.assertFalse(server._tip_refresh_pending()) - - def test_new_tip_detection_marks_priority_without_parent_lookup(self) -> None: - server = coordinator() - server._fetch_tip_parent_hash = ( # type: ignore[method-assign] - lambda _tip: self.fail("tip detection must not fetch parent metadata") - ) - - self.assertTrue(server.observe_tip_for_refresh(NEW_TIP)) - - self.assertEqual(server.latest_detected_tip[0], NEW_TIP) - self.assertEqual(server.current_tip_first_seen[0], OLD_TIP) - self.assertTrue(server._tip_refresh_pending()) - - def test_yield_and_chunk_metrics_are_unlabeled_and_bounded(self) -> None: - server = coordinator() - server.stop_event = StopAfterOnePass() # type: ignore[assignment] - - class MetricDaemon: - def run_once( - self, - *, - limit: int, - chunk_size: int, - progress_callback: object, - tip_refresh_pending: object, - chunk_callback: object, - ) -> CtvFanoutDaemonResult: - self_limit = limit - self_chunk_size = chunk_size - self.assert_wiring(self_limit, self_chunk_size) - assert callable(progress_callback) - assert callable(tip_refresh_pending) - assert callable(chunk_callback) - progress_callback() - progress_callback() - chunk_callback(CtvFanoutChunkResult(processed_count=2, elapsed_seconds=0.25)) - return CtvFanoutDaemonResult( - scanned_count=2, - submitted_count=0, - updated_count=2, - failed_count=0, - yielded_to_tip_refresh=True, - ) - - @staticmethod - def assert_wiring(limit: int, chunk_size: int) -> None: - if limit != 3 or chunk_size != 1: - raise AssertionError((limit, chunk_size)) - - server.ctv_fanout_broadcast_daemon = MetricDaemon() # type: ignore[assignment] - with patch("builtins.print"): - server.ctv_fanout_broadcaster_loop() - - lines = server.ctv_fanout_broadcaster_metrics_lines() - metrics = "\n".join(lines) - self.assertIn( - "qbit_prism_ctv_fanout_broadcaster_tip_refresh_yields_total 1", - metrics, - ) - self.assertIn( - "qbit_prism_ctv_fanout_broadcaster_chunk_rows_sum 2", - metrics, - ) - self.assertIn( - "qbit_prism_ctv_fanout_broadcaster_chunk_rows_count 1", - metrics, - ) - self.assertIn( - "qbit_prism_ctv_fanout_broadcaster_chunk_seconds_sum 0.250000", - metrics, - ) - self.assertIn( - "qbit_prism_ctv_fanout_broadcaster_chunk_seconds_count 1", - metrics, - ) - - new_metric_lines = [ - line - for line in lines - if line.startswith("qbit_prism_ctv_fanout_broadcaster_chunk_") - or line.startswith( - "qbit_prism_ctv_fanout_broadcaster_tip_refresh_yields_total" - ) - ] - for line in new_metric_lines: - labels = line.split("{", 1)[1].split("}", 1)[0] if "{" in line else "" - self.assertTrue(not labels or labels.startswith('le="'), line) - self.assertNotIn("fanout", labels) - self.assertNotIn("txid", labels) - self.assertNotIn("address", labels) - self.assertNotIn("row", labels) - - -if __name__ == "__main__": - unittest.main() diff --git a/tests/test_prism_ctv_runtime.py b/tests/test_prism_ctv_runtime.py deleted file mode 100644 index f298742d1..000000000 --- a/tests/test_prism_ctv_runtime.py +++ /dev/null @@ -1,547 +0,0 @@ -"""Direct tests for the coordinator-facing CTV runtime service.""" - -from __future__ import annotations - -from contextlib import contextmanager -import threading -import unittest -from unittest.mock import patch - -from lab.prism.coordinator_shutdown import ShutdownInProgress -from lab.prism.ctv_broadcaster_daemon import ( - CtvFanoutChunkResult, - CtvFanoutDaemonResult, -) -from lab.prism.ctv_runtime import ( - CTV_BROADCAST_STATE_COMPONENT, - CtvRuntimeConfig, - CtvRuntimeService, -) -from lab.prism.prism_coordinator import PrismCoordinator - - -class StopAfterOnePass: - def is_set(self) -> bool: - return False - - def wait(self, _timeout: float) -> bool: - return True - - -class PrismCtvRuntimeTests(unittest.TestCase): - @staticmethod - def config(**overrides: object) -> CtvRuntimeConfig: - values: dict[str, object] = { - "enabled": True, - "wallet": None, - "fee_sats": 0, - "limit": 7, - "chunk_size": 2, - "interval_seconds": 30.0, - } - values.update(overrides) - return CtvRuntimeConfig(**values) # type: ignore[arg-type] - - def test_run_once_constructs_daemon_and_holds_exact_writer_admission(self) -> None: - ledger = object() - admission: list[str] = [] - captured: dict[str, object] = {} - heartbeats: list[bool] = [] - - def rpc_call(*_args: object, **_kwargs: object) -> None: - return None - - class FakeBroadcaster: - def __init__(self, call: object, *, funding_wallet: str | None) -> None: - captured["rpc_call"] = call - captured["wallet"] = funding_wallet - - class FakeDaemon: - def __init__( - self, - daemon_ledger: object, - broadcaster: object, - *, - fee_sats: int, - ) -> None: - captured["ledger"] = daemon_ledger - captured["broadcaster"] = broadcaster - captured["fee_sats"] = fee_sats - - def run_once(self, **kwargs: object) -> CtvFanoutDaemonResult: - captured.update(kwargs) - progress_callback = kwargs["progress_callback"] - chunk_callback = kwargs["chunk_callback"] - tip_refresh_pending = kwargs["tip_refresh_pending"] - assert callable(progress_callback) - assert callable(chunk_callback) - assert callable(tip_refresh_pending) - progress_callback() - chunk_callback( - CtvFanoutChunkResult(processed_count=1, elapsed_seconds=0.25) - ) - captured["tip_pending"] = tip_refresh_pending() - return CtvFanoutDaemonResult(1, 0, 1, 0) - - @contextmanager - def writer_admission(component: str): - admission.append(f"enter:{component}") - try: - yield - finally: - admission.append(f"exit:{component}") - - service = CtvRuntimeService( - rpc_call=rpc_call, - ledger_source=lambda: ledger, - writer_admission=writer_admission, - tip_refresh_pending=lambda: False, - heartbeat=lambda: heartbeats.append(True), - stop_signal=threading.Event, - config=self.config(wallet="fee-wallet", fee_sats=900), - daemon_type=FakeDaemon, # type: ignore[arg-type] - broadcaster_type=FakeBroadcaster, # type: ignore[arg-type] - ) - - result = service.run_once(progress_callback=service.record_progress) - - self.assertEqual(result.updated_count, 1) - self.assertEqual( - admission, - [ - f"enter:{CTV_BROADCAST_STATE_COMPONENT}", - f"exit:{CTV_BROADCAST_STATE_COMPONENT}", - ], - ) - self.assertIs(captured["ledger"], ledger) - self.assertIs(captured["rpc_call"], rpc_call) - self.assertEqual(captured["wallet"], "fee-wallet") - self.assertEqual(captured["fee_sats"], 900) - self.assertEqual(captured["limit"], 7) - self.assertEqual(captured["chunk_size"], 2) - self.assertFalse(captured["tip_pending"]) - self.assertEqual(len(heartbeats), 2) - metrics = "\n".join(service.metrics_lines()) - self.assertIn( - "qbit_prism_ctv_fanout_broadcaster_processed_rows_total 1", - metrics, - ) - self.assertIn( - "qbit_prism_ctv_fanout_broadcaster_chunk_seconds_sum 0.250000", - metrics, - ) - - def test_closed_writer_admission_does_not_construct_daemon(self) -> None: - constructed: list[bool] = [] - - class RejectingAdmission: - def __enter__(self) -> None: - raise ShutdownInProgress("closed") - - def __exit__(self, *_args: object) -> None: - return None - - class UnexpectedDaemon: - def __init__(self, *_args: object, **_kwargs: object) -> None: - constructed.append(True) - - service = CtvRuntimeService( - rpc_call=lambda *_args, **_kwargs: None, - ledger_source=lambda: object(), - writer_admission=lambda _component: RejectingAdmission(), - tip_refresh_pending=lambda: False, - heartbeat=lambda: None, - stop_signal=threading.Event, - config=self.config(), - daemon_type=UnexpectedDaemon, # type: ignore[arg-type] - ) - - with self.assertRaises(ShutdownInProgress): - service.run_once() - - self.assertEqual(constructed, []) - - def test_loop_records_completion_before_wait_and_preserves_summary(self) -> None: - clock_values = iter((10.0, 112.0)) - heartbeats: list[float] = [] - wait_observation: dict[str, object] = {} - - class ObservedStop(StopAfterOnePass): - def wait(self, timeout: float) -> bool: - wait_observation["timeout"] = timeout - wait_observation["pass_count"] = service.pass_count - return True - - @contextmanager - def writer_admission(_component: str): - yield - - class OnePassDaemon: - def run_once(self, **_kwargs: object) -> CtvFanoutDaemonResult: - return CtvFanoutDaemonResult(0, 0, 0, 0, True) - - observed_stop = ObservedStop() - service = CtvRuntimeService( - rpc_call=lambda *_args, **_kwargs: None, - ledger_source=lambda: object(), - writer_admission=writer_admission, - tip_refresh_pending=lambda: False, - heartbeat=lambda: heartbeats.append(1.0), - stop_signal=lambda: observed_stop, - config=self.config(), - monotonic=lambda: next(clock_values), - ) - service.daemon = OnePassDaemon() # type: ignore[assignment] - - service.loop() - - self.assertEqual(wait_observation, {"timeout": 30.0, "pass_count": 1}) - self.assertEqual(len(heartbeats), 2) - metrics = "\n".join(service.metrics_lines()) - self.assertIn( - "qbit_prism_ctv_fanout_broadcaster_pass_seconds_sum 102.000000", - metrics, - ) - self.assertIn( - "qbit_prism_ctv_fanout_broadcaster_tip_refresh_yields_total 1", - metrics, - ) - self.assertEqual( - service.startup_summary(), - "prism coordinator: CTV fanout broadcaster enabled " - "mode=direct fee_bits=0 wallet=none interval=30s limit=7 chunk_size=2", - ) - - def test_cpfp_wallet_validation_remains_at_daemon_construction(self) -> None: - @contextmanager - def writer_admission(_component: str): - yield - - service = CtvRuntimeService( - rpc_call=lambda *_args, **_kwargs: None, - ledger_source=lambda: object(), - writer_admission=writer_admission, - tip_refresh_pending=lambda: False, - heartbeat=lambda: None, - stop_signal=threading.Event, - config=self.config(wallet=None, fee_sats=1), - ) - - with self.assertRaisesRegex( - ValueError, - "ctv_broadcaster_wallet is required", - ): - service.make_daemon() - - def test_lazy_handoff_consumes_overrides_and_updates_only_live_config(self) -> None: - server = PrismCoordinator.__new__(PrismCoordinator) - server.ctv_broadcaster_limit = 7 - - self.assertEqual( - server.__dict__["_ctv_runtime_compat_config"], - {"limit": 7}, - ) - - runtime = server._ensure_ctv_runtime() - - self.assertEqual(runtime.config.limit, 7) - self.assertNotIn("_ctv_runtime_compat_config", server.__dict__) - - server.ctv_broadcaster_limit = 11 - - self.assertIs(server._ensure_ctv_runtime(), runtime) - self.assertEqual(runtime.config.limit, 11) - self.assertNotIn("_ctv_runtime_compat_config", server.__dict__) - self.assertEqual(server._legacy_ctv_runtime_config().limit, 100) - - def test_wallet_or_fee_config_change_rebuilds_daemon_for_next_pass(self) -> None: - constructions: list[tuple[str | None, int]] = [] - runs: list[tuple[int, int]] = [] - - class FakeBroadcaster: - def __init__( - self, - _rpc_call: object, - *, - funding_wallet: str | None, - ) -> None: - self.wallet = funding_wallet - - class FakeDaemon: - def __init__( - self, - _ledger: object, - broadcaster: FakeBroadcaster, - *, - fee_sats: int, - ) -> None: - constructions.append((broadcaster.wallet, fee_sats)) - - def run_once(self, **kwargs: object) -> CtvFanoutDaemonResult: - runs.append((int(kwargs["limit"]), int(kwargs["chunk_size"]))) - return CtvFanoutDaemonResult(0, 0, 0, 0) - - @contextmanager - def writer_admission(_component: str): - yield - - service = CtvRuntimeService( - rpc_call=lambda *_args, **_kwargs: None, - ledger_source=lambda: object(), - writer_admission=writer_admission, - tip_refresh_pending=lambda: False, - heartbeat=lambda: None, - stop_signal=threading.Event, - config=self.config(), - daemon_type=FakeDaemon, # type: ignore[arg-type] - broadcaster_type=FakeBroadcaster, # type: ignore[arg-type] - ) - - service.run_once() - first = service.daemon - service.replace_config(limit=11, chunk_size=4) - service.run_once() - self.assertIs(service.daemon, first) - - service.replace_config(wallet="fee-wallet", fee_sats=900) - service.run_once() - - self.assertIsNot(service.daemon, first) - self.assertEqual(constructions, [(None, 0), ("fee-wallet", 900)]) - self.assertEqual(runs, [(7, 2), (11, 4), (11, 4)]) - - def test_concurrent_live_config_updates_retain_both_frozen_fields(self) -> None: - server = PrismCoordinator.__new__(PrismCoordinator) - runtime = server._ensure_ctv_runtime() - start = threading.Barrier(3) - errors: list[BaseException] = [] - - def update_limit() -> None: - try: - start.wait(timeout=2.0) - server.ctv_broadcaster_limit = 17 - except BaseException as exc: # pragma: no cover - asserted below - errors.append(exc) - - def update_chunk_size() -> None: - try: - start.wait(timeout=2.0) - server.ctv_broadcaster_chunk_size = 3 - except BaseException as exc: # pragma: no cover - asserted below - errors.append(exc) - - threads = [ - threading.Thread(target=update_limit), - threading.Thread(target=update_chunk_size), - ] - for thread in threads: - thread.start() - start.wait(timeout=2.0) - for thread in threads: - thread.join(timeout=2.0) - - self.assertTrue(all(not thread.is_alive() for thread in threads)) - self.assertEqual(errors, []) - snapshot = runtime.config - self.assertEqual(snapshot.limit, 17) - self.assertEqual(snapshot.chunk_size, 3) - - def test_lazy_handoff_is_singleton_under_deterministic_contention(self) -> None: - server = PrismCoordinator.__new__(PrismCoordinator) - server._record_heartbeat = lambda _name: None # type: ignore[method-assign] - arrivals = threading.Barrier(2) - underlying_lock = threading.Lock() - candidates: list[CtvRuntimeService] = [] - results: list[CtvRuntimeService] = [] - errors: list[BaseException] = [] - - class CoordinatedInitLock: - def __enter__(self) -> None: - arrivals.wait(timeout=2.0) - underlying_lock.acquire() - - def __exit__(self, *_args: object) -> None: - underlying_lock.release() - - server._ctv_runtime_init_lock = CoordinatedInitLock() # type: ignore[assignment] - original_make = server._make_ctv_runtime_service - - def counted_make( - config: CtvRuntimeConfig | None = None, - ) -> CtvRuntimeService: - candidate = original_make(config) - candidates.append(candidate) - return candidate - - server._make_ctv_runtime_service = counted_make # type: ignore[method-assign] - - def ensure_runtime() -> None: - try: - results.append(server._ensure_ctv_runtime()) - except BaseException as exc: # pragma: no cover - asserted below - errors.append(exc) - - threads = [threading.Thread(target=ensure_runtime) for _ in range(2)] - for thread in threads: - thread.start() - for thread in threads: - thread.join(timeout=2.0) - - self.assertTrue(all(not thread.is_alive() for thread in threads)) - self.assertEqual(errors, []) - self.assertEqual(len(candidates), 1) - self.assertEqual(len(results), 2) - self.assertIs(results[0], candidates[0]) - self.assertIs(results[1], candidates[0]) - marker = object() - results[0].daemon = marker # type: ignore[assignment] - self.assertIs(results[1].daemon, marker) - results[0].record_progress() - results[1].record_progress() - self.assertEqual(results[0].processed_rows_total, 2) - - def test_daemon_getter_waits_for_paused_lazy_handoff_publish(self) -> None: - server = PrismCoordinator.__new__(PrismCoordinator) - daemon_marker = object() - server.ctv_fanout_broadcast_daemon = daemon_marker # type: ignore[assignment] - underlying_lock = threading.Lock() - getter_attempted = threading.Event() - factory_popped_compat = threading.Event() - release_factory = threading.Event() - getter_returned = threading.Event() - runtime_results: list[CtvRuntimeService] = [] - daemon_results: list[object | None] = [] - errors: list[BaseException] = [] - - class ObservedInitLock: - def __enter__(self) -> None: - if threading.current_thread().name == "ctv-daemon-getter": - getter_attempted.set() - underlying_lock.acquire() - - def __exit__(self, *_args: object) -> None: - underlying_lock.release() - - server._ctv_runtime_init_lock = ObservedInitLock() # type: ignore[assignment] - original_make = server._make_ctv_runtime_service - - def paused_make( - config: CtvRuntimeConfig | None = None, - ) -> CtvRuntimeService: - runtime = original_make(config) - if "_ctv_runtime_compat_daemon" in server.__dict__: - raise AssertionError("compat daemon was not consumed") - factory_popped_compat.set() - if not release_factory.wait(timeout=2.0): - raise AssertionError("test did not release runtime publication") - return runtime - - server._make_ctv_runtime_service = paused_make # type: ignore[method-assign] - - def initialize() -> None: - try: - runtime_results.append(server._ensure_ctv_runtime()) - except BaseException as exc: # pragma: no cover - asserted below - errors.append(exc) - - def read_daemon() -> None: - try: - daemon_results.append(server.ctv_fanout_broadcast_daemon) - except BaseException as exc: # pragma: no cover - asserted below - errors.append(exc) - finally: - getter_returned.set() - - initializer = threading.Thread(target=initialize, name="ctv-runtime-initializer") - getter = threading.Thread(target=read_daemon, name="ctv-daemon-getter") - initializer.start() - self.assertTrue(factory_popped_compat.wait(timeout=2.0)) - getter.start() - self.assertTrue(getter_attempted.wait(timeout=2.0)) - self.assertFalse(getter_returned.is_set()) - - release_factory.set() - initializer.join(timeout=2.0) - getter.join(timeout=2.0) - - self.assertFalse(initializer.is_alive()) - self.assertFalse(getter.is_alive()) - self.assertEqual(errors, []) - self.assertEqual(len(runtime_results), 1) - self.assertEqual(daemon_results, [daemon_marker]) - self.assertIs(runtime_results[0].daemon, daemon_marker) - self.assertNotIn("_ctv_runtime_compat_daemon", server.__dict__) - - def test_coordinator_facade_routes_every_retained_loop_patch_seam(self) -> None: - server = PrismCoordinator.__new__(PrismCoordinator) - server.stop_event = StopAfterOnePass() # type: ignore[assignment] - server.ctv_broadcaster_limit = 7 - server.ctv_broadcaster_chunk_size = 2 - server.ctv_broadcaster_interval_seconds = 30.0 - server.tip_refresh_is_pending = lambda: False # type: ignore[method-assign] - server._record_heartbeat = lambda _name: None # type: ignore[method-assign] - events: list[object] = [] - - @contextmanager - def writer_admission(component: str): - events.append(("admit", component)) - yield - - server._writer_operation = writer_admission # type: ignore[method-assign] - - class FacadeDaemon: - def run_once(self, **kwargs: object) -> CtvFanoutDaemonResult: - progress_callback = kwargs["progress_callback"] - chunk_callback = kwargs["chunk_callback"] - assert callable(progress_callback) - assert callable(chunk_callback) - progress_callback() - chunk_callback( - CtvFanoutChunkResult(processed_count=2, elapsed_seconds=0.25) - ) - return CtvFanoutDaemonResult(2, 0, 2, 0, True) - - server.ctv_fanout_broadcast_daemon = FacadeDaemon() # type: ignore[assignment] - server._record_ctv_fanout_broadcaster_progress = ( # type: ignore[method-assign] - lambda: events.append("progress") - ) - server.observe_ctv_fanout_broadcaster_chunk = ( # type: ignore[method-assign] - lambda result: events.append(("chunk", result.processed_count)) - ) - server.observe_ctv_fanout_broadcaster_pass = ( # type: ignore[method-assign] - lambda elapsed: events.append(("pass", elapsed)) - ) - server._record_ctv_fanout_broadcaster_yield = ( # type: ignore[method-assign] - lambda: events.append("yield") - ) - original_run_once = server.run_ctv_fanout_broadcaster_once - - def patched_run_once(**kwargs: object) -> CtvFanoutDaemonResult: - events.append("run_once") - return original_run_once(**kwargs) # type: ignore[arg-type] - - server.run_ctv_fanout_broadcaster_once = patched_run_once # type: ignore[method-assign] - - with patch("builtins.print"): - server.ctv_fanout_broadcaster_loop() - - self.assertEqual( - [event if isinstance(event, str) else event[0] for event in events], - ["run_once", "admit", "progress", "chunk", "pass", "yield"], - ) - self.assertEqual(events[1], ("admit", CTV_BROADCAST_STATE_COMPONENT)) - # The registry spec is declared with a call-time-resolved target - # (post-construction loop monkeypatches keep working), so the spec - # contract is asserted on its lifecycle fields rather than a bound - # method identity. - registry = server._make_background_service_registry() - specification = registry.snapshot("ctv_fanout_broadcaster").specification - self.assertEqual(specification.name, "ctv_fanout_broadcaster") - self.assertEqual(specification.thread_name, "prism-ctv-fanout-broadcaster") - self.assertEqual(specification.join_timeout, 1.0) - self.assertTrue(specification.daemon) - self.assertTrue(specification.watchdog_monitored) - - -if __name__ == "__main__": - unittest.main() diff --git a/tests/test_prism_dequeue_stale_skip.py b/tests/test_prism_dequeue_stale_skip.py deleted file mode 100644 index aec43903c..000000000 --- a/tests/test_prism_dequeue_stale_skip.py +++ /dev/null @@ -1,928 +0,0 @@ -#!/usr/bin/env python3 -"""Regression coverage for the dequeue-time stale sibling skip (#181 item 2). - -The 2026-08-21 testnet4 validation showed that #196 removed the -restart/replay candidate population but left the live/fast-path one: the -coordinator log still carried bursts of individual -``block candidate abandoned reason=stale-job: tip moved before submit``. -Each of those rows costs one ``submitblock``, three ``getbestblockhash``, -three ``getblockheader`` and two ledger writes -- plus an accounting task, a -fast-lane reservation and an accepted-block payout-preview barrier armed and -withdrawn -- to learn a fact the chain could have answered before the offer. - -These tests pin what the skip does about that, against two references it may -never disagree with: #194's per-row oracle (``drain_per_row``, the shipped -disposition path) and #196's predicate S (whose clauses, evidence set, -fencing and cleanup the skip reuses rather than restates). - -The invariant every efficacy test asserts is one-directional: -``skip_set - oracle_abandoned`` must be empty. Conservative oracle-only -deltas are acceptable -- the skip is an optimisation over a path that still -runs -- while a single selector-only hash is a block the per-row path would -have kept. -""" - -from __future__ import annotations - -import queue -import sys -import unittest -from contextlib import redirect_stdout -from io import StringIO -from pathlib import Path -from typing import Any - -if not __package__: - sys.path.insert(0, str(Path(__file__).resolve().parents[1])) - -from lab.prism.block_candidates import ( # noqa: E402 - MAX_PENDING_BLOCK_CANDIDATES, - PRISM_BLOCK_CANDIDATE_COLLAPSE_OUTCOMES, - _BlockCandidateNodeSubmission, - _dequeued_candidate_collapse_row, - block_candidate_intent, -) -from tests.prism_candidate_storm import ( # noqa: E402 - PERTURBATIONS, - CandidateStormRig, -) -from tests.test_prism_block_candidate_collapse import ( # noqa: E402 - DECIDED_HEIGHT, - DECIDED_WINNER, - MIN_WORK_BITS, - STORM_PARENT, - CollapseChainRpc, - CollapseFixture, - CollapseLedger, - _hash, - _scaled, -) - - -class _NoBatchLedger(CollapseLedger): - """A compatibility ledger with no fenced batch abandonment at all.""" - - def __getattr__(self, name: str) -> Any: - if name == "mark_block_candidates_abandoned": - raise AttributeError(name) - return super().__getattr__(name) - - mark_block_candidates_abandoned = None # type: ignore[assignment] - - -class _NoPoolBlockLedger(CollapseLedger): - """A compatibility ledger that cannot answer clause 2 at all.""" - - def __getattr__(self, name: str) -> Any: - if name == "pool_block_state": - raise AttributeError(name) - return super().__getattr__(name) - - pool_block_state = None # type: ignore[assignment] - - -def _offerable(fixture: CollapseFixture) -> CollapseFixture: - """Let the fixture's chain answer ``submitblock`` instead of raising. - - ``FakeRpc`` refuses every unknown method, which would turn a preserved - candidate's node offer into an error submission. These tests care - whether the offer *happened*, so the chain answers the way qbitd does - for a valid block that did not become the tip. - """ - fixture.rpc.results["submitblock"] = "inconclusive" - return fixture - - -def _drive(fixture: CollapseFixture, candidate: Any) -> str: - """Admit one candidate and run exactly one shipped ``submit_next`` over it. - - ``defer_accounting=True`` is the split ``BlockCandidateService.run`` - uses. No accounting lane is drained here, so a candidate the skip - preserved is left with its offer made and its durable row still pending - -- which is precisely the observable difference from one it skipped. - """ - buffer = StringIO() - with redirect_stdout(buffer): - fixture.server.enqueue_block_candidate(candidate) - fixture.server.submit_next_block_candidate(defer_accounting=True) - return buffer.getvalue() - - -def _record_skips(server: Any) -> set[str]: - """Capture the hashes the *skip itself* terminalized, and only those. - - ``PerRowDrainReport.abandoned_hashes`` names every durable transition a - drain made, which under ``dequeue_skip=True`` is the union of the skip's - own set and whatever the per-row path went on to abandon for candidates - the skip preserved. The contract's invariant is about the skip's set, so - it has to be read where the decision is made. - """ - service = server._ensure_block_candidate_service() - shipped = service._skip_superseded_block_candidate_at_dequeue - seen: set[str] = set() - - def recording(candidate: Any, **kwargs: Any) -> bool: - skipped = shipped(candidate, **kwargs) - if skipped: - seen.add(str(candidate.submission.block_hash_hex).lower()) - return skipped - - service._skip_superseded_block_candidate_at_dequeue = recording - return seen - - -def _accounting_depth(fixture: CollapseFixture) -> int: - service = fixture.service - service._ensure_block_accounting_state() - return ( - service._block_accounting_queue.qsize() - + service._block_accounting_overflow_queue.qsize() - + service._block_accounting_accepted_queue.qsize() - ) - - -def _skipped(fixture: CollapseFixture, candidate: Any) -> bool: - """Drive one candidate and report whether the skip terminalized it.""" - before = fixture.pending() - _drive(fixture, candidate) - block_hash = str(candidate.submission.block_hash_hex).lower() - gone = block_hash in before and block_hash not in fixture.pending() - skipped = fixture.counts()["dequeue_skipped"] > 0 - assert gone == skipped, "durable state and the skip counter disagree" - return skipped - - -class DequeueSkipRowShapeTests(unittest.TestCase): - """The skip judges an in-memory candidate; it must present the durable row.""" - - def test_the_dequeue_row_matches_the_durable_intent_field_by_field( - self, - ) -> None: - """Predicate S must read the same facts whichever side built the row. - - ``_dequeued_candidate_collapse_row`` restates a subset of - ``block_candidate_intent`` rather than calling it, because the full - intent re-serializes the block and recomputes every witness merkle - leaf. This is what keeps the restatement honest. - """ - fixture = CollapseFixture() - candidate = fixture.seed([_hash(1)], network_difficulty=7)[0] - intent = block_candidate_intent(candidate) - row = _dequeued_candidate_collapse_row(candidate, pool_block_exists=False) - self.assertEqual(row["block_hash"], intent["block_hash_hex"]) - built = row["candidate"] - for field in ("block_hash_hex", "parent_hash", "expected_height"): - self.assertEqual(built[field], intent[field], field) - for field in ("previousblockhash", "height"): - self.assertEqual( - built["template"][field], - intent["template"][field], - field, - ) - self.assertEqual(built["found_block"], intent["found_block"]) - self.assertEqual( - built["pending_share"]["job_id"], - intent["pending_share"]["job_id"], - ) - self.assertIs(row["pool_block_exists"], False) - self.assertIs( - _dequeued_candidate_collapse_row(candidate, pool_block_exists=True)[ - "pool_block_exists" - ], - True, - ) - - def test_the_outcome_label_set_stays_closed(self) -> None: - fixture = CollapseFixture() - self.assertEqual( - tuple(fixture.counts()), - PRISM_BLOCK_CANDIDATE_COLLAPSE_OUTCOMES, - ) - for outcome in ("dequeue_considered", "dequeue_skipped", "dequeue_preserved"): - self.assertIn(outcome, PRISM_BLOCK_CANDIDATE_COLLAPSE_OUTCOMES) - - -class DequeueSkipEfficacyTests(unittest.TestCase): - """The skip against #194's per-row oracle, live and after restart.""" - - def _rig(self, candidates: int, view: str) -> tuple[CandidateStormRig, Any, str]: - rig = CandidateStormRig(candidates=candidates) - rig.seed_live() - if view == "restart": - rig.restart_and_enumerate() - server = rig.restarted_server if view == "restart" else rig.live_server - winner = rig.decide_height() - return rig, server, winner - - def _oracle(self, candidates: int, view: str) -> tuple[set[str], Any]: - rig, server, winner = self._rig(candidates, view) - # drain_per_row suppresses both shipped set-oriented dispositions by - # default, so this stays the per-row path with no patching here. - report = rig.drain_per_row(server, stop_before_hash=winner) - return set(report.abandoned_hashes), report - - def _skip(self, candidates: int, view: str) -> tuple[set[str], Any, Any, str]: - rig, server, winner = self._rig(candidates, view) - skip_set = _record_skips(server) - report = rig.drain_per_row( - server, - stop_before_hash=winner, - dequeue_skip=True, - ) - return skip_set, report, server, winner - - def _compare(self, candidates: int, view: str) -> tuple[set[str], Any, Any]: - oracle, oracle_report = self._oracle(candidates, view) - skipped, skip_report, _server, winner = self._skip(candidates, view) - self.assertEqual( - skipped - oracle, - set(), - f"the skip abandoned {len(skipped - oracle)} hash(es) the shipped " - f"per-row path refused ({candidates} candidates, {view} view)", - ) - # And the drain as a whole -- skip plus whatever per-row disposed of - # afterwards -- terminalized nothing the oracle kept either. - self.assertEqual(set(skip_report.abandoned_hashes) - oracle, set()) - self.assertNotIn(winner, skipped) - return skipped, skip_report, oracle_report - - def test_every_stale_sibling_is_skipped_and_the_winner_is_not(self) -> None: - for view in ("live", "restart"): - with self.subTest(view=view): - skipped, _report, _oracle = self._compare(100, view) - self.assertEqual(len(skipped), 99) - - def test_agreement_holds_at_the_observed_storm_cardinality(self) -> None: - for view in ("live", "restart"): - with self.subTest(view=view): - skipped, _report, _oracle = self._compare(3_120, view) - self.assertEqual(len(skipped), 3_119) - - def test_the_skip_costs_no_submitblock_and_fewer_chain_reads(self) -> None: - """The cost claim, measured against the oracle on the same population.""" - for view in ("live", "restart"): - with self.subTest(view=view): - oracle, oracle_report = self._oracle(100, view) - skipped, skip_report, _server, _winner = self._skip(100, view) - self.assertEqual(len(oracle), 99) - self.assertEqual(len(skipped), 99) - self.assertEqual(set(skip_report.abandoned_hashes), skipped) - # The dominant per-sibling cost is the node offer itself. - self.assertEqual(oracle_report.rpc_calls["submitblock"], 99) - self.assertNotIn("submitblock", skip_report.rpc_calls) - oracle_rpcs = sum(oracle_report.rpc_calls.values()) - skip_rpcs = sum(skip_report.rpc_calls.values()) - self.assertEqual(oracle_rpcs, 693) - self.assertLess(skip_rpcs, oracle_rpcs) - # Four reads per skipped sibling: the clause-3 tip, the - # selector's tip height, and the pre-write revalidation's - # own fresh tip and occupant. The decided height and its - # header are read once for the whole burst, because the - # burst shares one probe budget for as long as the best tip - # holds. - self.assertEqual(skip_rpcs, 4 * 99 + 2) - self.assertEqual(skip_report.rpc_calls["getblockheader"], 1) - # Two durable writes per sibling become one: the attempt - # mark the offer stamps is never made at all. - self.assertEqual(oracle_report.ledger_attempt_marks, 99) - self.assertEqual(skip_report.ledger_attempt_marks, 0) - # And no stale sibling reaches per-row accounting. - self.assertEqual(oracle_report.accounting_tasks, 99) - self.assertEqual(skip_report.accounting_tasks, 0) - - def test_a_replay_adopted_row_is_classified_from_the_chain(self) -> None: - """The restart-specific fact: replay adoption marks nothing outstanding. - - A skip that consulted only the observation set would get both halves - of this wrong -- it would have no evidence that the winner is the - winner, and none that a sibling is superseded. - """ - rig = CandidateStormRig(candidates=8) - rig.seed_live() - rig.restart_and_enumerate() - server = rig.restarted_server - winner = rig.decide_height() - service = server._ensure_block_candidate_service() - with server.lock: - self.assertEqual( - set(service._outstanding_block_candidate_hashes), - set(), - ) - self.assertEqual( - set(service._tip_observed_accepted_block_hashes), - set(), - ) - self.assertEqual( - len(service._block_replay_inflight_hashes), - 8, - ) - report = rig.drain_per_row( - server, - stop_before_hash=winner, - dequeue_skip=True, - ) - self.assertEqual(len(report.abandoned_hashes), 7) - self.assertNotIn(winner, report.abandoned_hashes) - # Now drive the winner's own dequeue. Nothing in memory says it was - # accepted; only the chain does, through clause 4's "the occupant is - # this candidate". - offers_before = int(rig.decided_rpc.calls.get("submitblock", 0)) - with redirect_stdout(StringIO()): - server.submit_next_block_candidate(defer_accounting=True) - self.assertEqual( - int(rig.decided_rpc.calls.get("submitblock", 0)) - offers_before, - 1, - ) - self.assertIn( - winner, - { - str(row["block_hash"]).lower() - for row in rig.ledger.pending_block_candidate_rows(limit=64) - }, - ) - counts = service.block_candidate_collapse_snapshot() - self.assertEqual(counts["dequeue_skipped"], 7) - self.assertEqual(counts["dequeue_preserved"], 1) - - -class DequeueSkipEvidenceTests(unittest.TestCase): - """Every member of #196's evidence set still rejects at dequeue.""" - - def test_the_baseline_sibling_is_skipped(self) -> None: - fixture = _offerable(CollapseFixture()) - candidate = fixture.seed([_hash(1)])[0] - self.assertTrue(_skipped(fixture, candidate)) - self.assertEqual(_accounting_depth(fixture), 0) - self.assertNotIn("submitblock", fixture.rpc.counts()) - - def test_a_candidate_building_on_the_best_tip_is_never_skipped(self) -> None: - """Clause 3, read from the chain: this is the block waiting to land.""" - fixture = _offerable( - CollapseFixture( - rpc=CollapseChainRpc( - tip=STORM_PARENT, - tip_height=DECIDED_HEIGHT - 1, - active={DECIDED_HEIGHT - 1: STORM_PARENT}, - ) - ) - ) - candidate = fixture.seed([_hash(1)])[0] - self.assertFalse(_skipped(fixture, candidate)) - self.assertEqual(fixture.counts()["dequeue_preserved"], 1) - # The acceptance path pays exactly one extra chain read for the - # clause-3 answer, and never a durable one. - self.assertEqual(fixture.rpc.counts()["getbestblockhash"], 1) - self.assertNotIn("getblockcount", fixture.rpc.counts()) - self.assertEqual(fixture.rpc.counts()["submitblock"], 1) - - def test_ownership_markers_alone_do_not_preserve_a_stale_sibling(self) -> None: - """Outstanding/replay-inflight mean owned, not offered (#183's note).""" - fixture = _offerable(CollapseFixture()) - candidate = fixture.seed([_hash(1)])[0] - fixture.server._register_outstanding_block_candidate(_hash(1)) - with fixture.server.lock: - fixture.service._block_replay_inflight_hashes.add(_hash(1)) - self.assertTrue(_skipped(fixture, candidate)) - - def test_each_evidence_source_preserves_its_hash(self) -> None: - target = _hash(1) - - def deferred_accounting_retry(fixture: CollapseFixture) -> None: - duplicate = fixture.seed([target])[0] - with fixture.server.lock: - fixture.service._block_accounting_deferred_retry_candidate = ( - duplicate - ) - - def waiting_retry(fixture: CollapseFixture) -> None: - duplicate = fixture.seed([target])[0] - with fixture.server.lock: - fixture.service._block_disposition_waiting_retries[target] = ( - duplicate - ) - - def retained_submission(fixture: CollapseFixture) -> None: - fixture.service._stash_retained_block_candidate_node_submission( - target, - _BlockCandidateNodeSubmission(attempted=True, result=None), - ) - - def tip_observed(fixture: CollapseFixture) -> None: - fixture.server._register_outstanding_block_candidate(target) - with redirect_stdout(StringIO()): - fixture.server._note_tip_observation_for_candidates(target) - - def accounted_accepted(fixture: CollapseFixture) -> None: - fixture.server._ensure_job_cache_state() - with fixture.server.lock: - fixture.server._accounted_accepted_block_hashes.add(target) - - def prepared_pool_block(fixture: CollapseFixture) -> None: - fixture.ledger.inner.persist_accepted_block( - block_hash=target, - block_height=DECIDED_HEIGHT, - parent_hash=STORM_PARENT, - final_bundle={}, - audit_report={}, - ) - - for name, install in ( - ("deferred_accounting_retry", deferred_accounting_retry), - ("waiting_retry", waiting_retry), - ("retained_submission", retained_submission), - ("tip_observed", tip_observed), - ("accounted_accepted", accounted_accepted), - ("prepared_pool_block", prepared_pool_block), - ): - with self.subTest(evidence=name): - fixture = _offerable(CollapseFixture()) - candidate = fixture.seed([target])[0] - install(fixture) - self.assertFalse(_skipped(fixture, candidate)) - self.assertIn(target, fixture.pending()) - self.assertEqual(fixture.counts()["dequeue_skipped"], 0) - if name != "prepared_pool_block": - # In-memory evidence is probed before any round trip, so - # an excluded candidate leaves the skip having spent - # neither a chain read nor the durable pool-block probe. - # ``prepared_pool_block`` is the one clause that is a - # durable fact, so it is answered after the tip read. - self.assertNotIn( - "getbestblockhash", - fixture.rpc.counts(), - ) - else: - self.assertEqual( - fixture.rpc.counts()["getbestblockhash"], - 1, - ) - - def test_a_recorded_terminal_outcome_never_reaches_the_skip(self) -> None: - """The duplicate-drop fence runs first, and abandons nothing.""" - fixture = _offerable(CollapseFixture()) - candidate = fixture.seed([_hash(1)])[0] - fixture.service._record_block_candidate_terminal_outcome( - _hash(1), - accepted=False, - ) - _drive(fixture, candidate) - self.assertEqual(fixture.counts()["dequeue_considered"], 0) - self.assertIn(_hash(1), fixture.pending()) - - def test_a_finalize_retry_never_reaches_the_skip(self) -> None: - """A finalize-only replay is a different branch of ``submit_next``.""" - fixture = _offerable(CollapseFixture()) - candidate = fixture.seed([_hash(1)])[0] - with fixture.server.lock: - fixture.service.finalize_retries[_hash(1)] = (False, "boom") - _drive(fixture, candidate) - self.assertEqual(fixture.counts()["dequeue_considered"], 0) - self.assertEqual(fixture.counts()["dequeue_skipped"], 0) - - def test_a_lease_held_elsewhere_parks_the_wakeup_before_the_skip(self) -> None: - """``submit_next`` claims the lease first; an unclaimable one defers.""" - fixture = _offerable(CollapseFixture()) - candidate = fixture.seed([_hash(1)])[0] - lease = fixture.server._claim_block_candidate_disposition( - _hash(1), - blocking=False, - ) - self.assertIsNotNone(lease) - self.addCleanup( - fixture.server._release_block_candidate_disposition, - lease, - ) - _drive(fixture, candidate) - self.assertEqual(fixture.counts()["dequeue_considered"], 0) - self.assertIn(_hash(1), fixture.pending()) - self.assertIn( - _hash(1), - fixture.service._block_disposition_waiting_retries, - ) - - def test_the_passs_own_lease_is_the_only_flight_exempted(self) -> None: - """The self-exemption is what makes the skip reachable at all. - - ``submit_next`` holds this hash's disposition lease by the time the - skip runs, so #196's clause 1 would reject the row forever without - it -- and any exemption wider than this one hash would let a - genuinely mid-offer sibling through. - """ - fixture = _offerable(CollapseFixture()) - target, other = _hash(1), _hash(2) - first = fixture.seed([target])[0] - second = fixture.seed([other])[0] - held = fixture.server._claim_block_candidate_disposition( - other, - blocking=False, - ) - self.assertIsNotNone(held) - self.addCleanup( - fixture.server._release_block_candidate_disposition, - held, - ) - # The pass's own flight does not count; the sibling's does. - self.assertTrue(_skipped(fixture, first)) - self.assertNotIn(target, fixture.pending()) - _drive(fixture, second) - self.assertIn(other, fixture.pending()) - - -class DequeueSkipClauseTests(unittest.TestCase): - """The chain clauses, at the dequeue boundary.""" - - def test_a_lighter_occupant_never_destroys_a_heavier_sibling(self) -> None: - """Clause 4b, and the per-row path re-decides the height instead. - - Under testnet4's 20-minute minimum-difficulty rule a light foreign - block can occupy a height our full-difficulty sibling outweighs. - Skipping it pre-offer would destroy a block the per-row path wins. - """ - fixture = _offerable( - CollapseFixture( - rpc=CollapseChainRpc( - tip=DECIDED_WINNER, - tip_height=DECIDED_HEIGHT, - active={DECIDED_HEIGHT: DECIDED_WINNER}, - bits={DECIDED_WINNER: MIN_WORK_BITS}, - ) - ) - ) - occupant = _scaled(MIN_WORK_BITS) - heavier = fixture.seed([_hash(1)], network_difficulty=occupant * 8)[0] - self.assertFalse(_skipped(fixture, heavier)) - self.assertIn(_hash(1), fixture.pending()) - # Still offered, and still handed to per-row accounting to decide. - self.assertEqual(fixture.rpc.counts()["submitblock"], 1) - self.assertEqual(_accounting_depth(fixture), 1) - - def test_an_equal_work_occupant_still_supersedes(self) -> None: - fixture = _offerable( - CollapseFixture( - rpc=CollapseChainRpc( - tip=DECIDED_WINNER, - tip_height=DECIDED_HEIGHT, - active={DECIDED_HEIGHT: DECIDED_WINNER}, - bits={DECIDED_WINNER: MIN_WORK_BITS}, - ) - ) - ) - occupant = _scaled(MIN_WORK_BITS) - sibling = fixture.seed([_hash(2)], network_difficulty=occupant)[0] - self.assertTrue(_skipped(fixture, sibling)) - - def test_an_undecided_height_is_never_skipped(self) -> None: - """Clause 4: a height above the tip is still ours to win.""" - fixture = _offerable( - CollapseFixture( - rpc=CollapseChainRpc( - tip=DECIDED_WINNER, - tip_height=DECIDED_HEIGHT - 1, - active={DECIDED_HEIGHT - 1: DECIDED_WINNER}, - ) - ) - ) - candidate = fixture.seed([_hash(1)])[0] - self.assertFalse(_skipped(fixture, candidate)) - self.assertEqual(fixture.rpc.counts()["submitblock"], 1) - - def test_a_candidate_that_is_itself_the_occupant_is_never_skipped(self) -> None: - fixture = _offerable( - CollapseFixture( - rpc=CollapseChainRpc( - tip=_hash(1), - tip_height=DECIDED_HEIGHT, - active={DECIDED_HEIGHT: _hash(1)}, - ) - ) - ) - candidate = fixture.seed([_hash(1)])[0] - self.assertFalse(_skipped(fixture, candidate)) - - -class DequeueSkipFailClosedTests(unittest.TestCase): - """Every unreadable fact falls through to the offer path.""" - - def _preserved_on(self, method: str) -> None: - fixture = _offerable(CollapseFixture()) - candidate = fixture.seed([_hash(1)])[0] - fixture.rpc.failures.add(method) - self.assertFalse(_skipped(fixture, candidate)) - self.assertIn(_hash(1), fixture.pending()) - self.assertEqual(fixture.counts()["dequeue_preserved"], 1) - self.assertGreaterEqual(fixture.counts()["fail_closed"], 1) - self.assertEqual(fixture.rpc.counts()["submitblock"], 1) - - def test_a_failed_best_tip_read_preserves_the_candidate(self) -> None: - self._preserved_on("getbestblockhash") - - def test_a_failed_active_block_read_preserves_the_candidate(self) -> None: - self._preserved_on("getblockhash") - - def test_a_failed_header_read_preserves_the_candidate(self) -> None: - self._preserved_on("getblockheader") - - def test_a_failed_tip_height_read_preserves_the_candidate(self) -> None: - self._preserved_on("getblockcount") - - def test_a_failed_pool_block_probe_preserves_the_candidate(self) -> None: - fixture = _offerable(CollapseFixture()) - candidate = fixture.seed([_hash(1)])[0] - - def explode(**_kwargs: object) -> object: - raise RuntimeError("pool block state unavailable") - - fixture.ledger.inner.pool_block_state = explode - self.assertFalse(_skipped(fixture, candidate)) - self.assertEqual(fixture.counts()["dequeue_preserved"], 1) - self.assertGreaterEqual(fixture.counts()["fail_closed"], 1) - self.assertEqual(fixture.rpc.counts()["submitblock"], 1) - - def test_a_ledger_without_a_pool_block_probe_never_skips(self) -> None: - """Clause 2 has no durable answer, so nothing may be concluded.""" - fixture = _offerable(CollapseFixture(ledger=_NoPoolBlockLedger())) - candidate = fixture.seed([_hash(1)])[0] - self.assertFalse(_skipped(fixture, candidate)) - self.assertEqual(fixture.counts()["dequeue_considered"], 0) - self.assertEqual(fixture.rpc.counts()["submitblock"], 1) - - def test_a_ledger_without_a_fenced_batch_write_never_skips(self) -> None: - fixture = _offerable(CollapseFixture(ledger=_NoBatchLedger())) - candidate = fixture.seed([_hash(1)])[0] - self.assertFalse(_skipped(fixture, candidate)) - self.assertEqual(fixture.counts()["dequeue_considered"], 0) - # Not even a chain read is spent on a structural absence. - self.assertNotIn("getbestblockhash", fixture.rpc.counts()) - self.assertEqual(fixture.rpc.counts()["submitblock"], 1) - - def test_a_lost_fenced_write_preserves_the_candidate(self) -> None: - """The write is the authority: a row it did not return is not ours.""" - fixture = _offerable(CollapseFixture()) - candidate = fixture.seed([_hash(1)])[0] - fixture.ledger.abandon_hook = lambda _hashes, _error: () - self.assertFalse(_skipped(fixture, candidate)) - self.assertIn(_hash(1), fixture.pending()) - self.assertEqual(fixture.counts()["write_lost"], 1) - self.assertEqual(fixture.counts()["dequeue_preserved"], 1) - self.assertEqual(fixture.rpc.counts()["submitblock"], 1) - - def test_a_failed_fenced_write_preserves_the_candidate(self) -> None: - def explode(_hashes: object, _error: object) -> object: - raise RuntimeError("fenced write unavailable") - - fixture = _offerable(CollapseFixture()) - candidate = fixture.seed([_hash(1)])[0] - fixture.ledger.abandon_hook = explode - self.assertFalse(_skipped(fixture, candidate)) - self.assertIn(_hash(1), fixture.pending()) - self.assertEqual(fixture.rpc.counts()["submitblock"], 1) - - def test_a_tip_that_moved_under_the_lease_drops_the_write(self) -> None: - """The pre-write revalidation runs at dequeue exactly as it does on a page.""" - fixture = _offerable(CollapseFixture()) - candidate = fixture.seed([_hash(1)])[0] - seen: list[str] = [] - - def move_tip(rpc: Any, method: str, _params: object) -> None: - if method != "getbestblockhash": - return - seen.append(method) - if len(seen) == 2: - # Between selection and the fenced write, the candidate's - # own parent became the tip again. - rpc.tip = STORM_PARENT - - fixture.rpc.before_call = move_tip - self.assertFalse(_skipped(fixture, candidate)) - self.assertIn(_hash(1), fixture.pending()) - self.assertEqual(fixture.counts()["revalidation_dropped"], 1) - self.assertEqual(fixture.rpc.counts()["submitblock"], 1) - - -class DequeueSkipGuardTests(unittest.TestCase): - """An escape from the skip must not strand the lease or the object.""" - - def test_a_raising_skip_retains_the_candidate_and_frees_the_lease( - self, - ) -> None: - fixture = _offerable(CollapseFixture()) - candidate = fixture.seed([_hash(1)], credit_share_on_accept=True)[0] - - def explode(_candidate: Any, **_kwargs: Any) -> bool: - raise RuntimeError("skip exploded") - - fixture.service._skip_superseded_block_candidate_at_dequeue = explode - with self.assertRaises(RuntimeError): - with redirect_stdout(StringIO()): - fixture.server.enqueue_block_candidate(candidate) - fixture.server.submit_next_block_candidate(defer_accounting=True) - # The lease is free again, so the next pass for this hash is not - # parked forever... - lease = fixture.server._claim_block_candidate_disposition( - _hash(1), - blocking=False, - ) - self.assertIsNotNone(lease) - fixture.server._release_block_candidate_disposition(lease) - # ...and the object that owns the floor holder is still reachable. - with fixture.server.lock: - self.assertIs(fixture.service.retry_candidate, candidate) - self.assertIn(_hash(1), fixture.pending()) - - -class DequeueSkipCleanupTests(unittest.TestCase): - """A skipped candidate is terminalized with #196's full cleanup.""" - - def test_the_dequeued_objects_floor_holder_is_released(self) -> None: - """Asserted directly: the holder is keyed by the object's identity. - - The apply's own pending-share step indexes holders by scanning the - live and replay queues, and a dequeued object is in neither, so the - skip has to release this one itself. - """ - fixture = CollapseFixture() - candidate = fixture.seed([_hash(1)], credit_share_on_accept=True)[0] - writer = fixture.server._ensure_share_writer_service() - writer.adopt_pending_share(candidate.pending_share) - self.assertIn(id(candidate.pending_share), fixture.floor()) - self.assertTrue(_skipped(fixture, candidate)) - self.assertNotIn(id(candidate.pending_share), fixture.floor()) - - def test_every_skipped_candidate_in_a_burst_releases_its_floor(self) -> None: - fixture = CollapseFixture() - hashes = [_hash(index) for index in range(1, 6)] - candidates = fixture.seed(hashes, credit_share_on_accept=True) - writer = fixture.server._ensure_share_writer_service() - for candidate in candidates: - writer.adopt_pending_share(candidate.pending_share) - self.assertEqual(len(fixture.floor()), len(candidates)) - for candidate in candidates: - _drive(fixture, candidate) - self.assertEqual(fixture.counts()["dequeue_skipped"], len(candidates)) - self.assertEqual(fixture.floor(), {}) - self.assertEqual(fixture.pending(), set()) - - def test_the_preview_barrier_is_withdrawn_and_left_untombstoned(self) -> None: - fixture = CollapseFixture() - candidate = fixture.seed([_hash(1)])[0] - server = fixture.server - server._register_outstanding_block_candidate(_hash(1)) - server._begin_accepted_block_payout_preview( - _hash(1), - block_height=DECIDED_HEIGHT, - ) - server._mark_accepted_block_payout_landed( - _hash(1), - block_height=DECIDED_HEIGHT, - ) - with server._accepted_block_payout_preview_condition: - self.assertIn(_hash(1), server._accepted_block_payout_previews) - self.assertTrue(_skipped(fixture, candidate)) - with server._accepted_block_payout_preview_condition: - # Withdrawn *and* untombstoned: the durable row is terminal, so - # nothing is left to fence, and a retained tombstone is exactly - # the preview wait this work exists to end. - self.assertNotIn(_hash(1), server._accepted_block_payout_previews) - self.assertEqual( - server._invalidated_accepted_block_payout_previews, - {}, - ) - - def test_the_skip_publishes_the_terminal_fence_and_the_counters(self) -> None: - fixture = CollapseFixture() - candidate = fixture.seed([_hash(1)])[0] - self.assertTrue(_skipped(fixture, candidate)) - service = fixture.service - self.assertIs(service._block_candidate_terminal_outcome(_hash(1)), False) - with fixture.server.lock: - self.assertNotIn( - _hash(1), - service._outstanding_block_candidate_hashes, - ) - self.assertNotIn(_hash(1), service._block_fast_lane_reservations) - self.assertEqual( - fixture.server.block_candidate_abandoned_counts, - {"stale-job": 1}, - ) - self.assertEqual(fixture.cleanup_backlog(), {}) - - def test_a_skip_reserves_no_max_block_capacity(self) -> None: - """A stale sibling must not consume capacity on its way to abandonment.""" - fixture = CollapseFixture() - fixture.server.max_blocks = 1 - fixture.server.stop_after_block = False - candidates = fixture.seed( - [_hash(1), _hash(2), _hash(3)], - ) - for candidate in candidates: - _drive(fixture, candidate) - self.assertEqual(fixture.counts()["dequeue_skipped"], 3) - self.assertEqual(fixture.pending(), set()) - with fixture.server.lock: - self.assertEqual(fixture.service._block_fast_lane_reservations, set()) - - -class DequeueSkipPerturbationTests(unittest.TestCase): - """No evidence shape ever produces a skip-only hash, in either view.""" - - def test_no_perturbation_ever_produces_a_skip_only_hash(self) -> None: - candidates = 20 - target = _hash(7) - for view in ("live", "restart"): - for kind in PERTURBATIONS: - with self.subTest(view=view, perturbation=kind): - oracle = self._drain(candidates, view, kind, target, False) - skipped = self._drain(candidates, view, kind, target, True) - self.assertEqual( - skipped - oracle, - set(), - f"{kind} produced a skip-only hash in the {view} view", - ) - - def test_five_of_six_shapes_exclude_the_perturbed_hash_outright(self) -> None: - """And the sixth is the retry slot, which the dequeue itself consumes. - - ``perturb('retry_slot')`` moves a never-offered wakeup into the - global retry holder -- what the shipped path does when the fast-lane - reservation declines. ``submit_next`` clears that holder as it - dequeues, so by the time the skip judges the object, the holder no - longer names it and cannot be evidence about it. The per-row oracle - abandons that row too (it re-offers and finds the tip moved), so the - subset invariant above still holds and no block the per-row path - would have won is destroyed; it is simply a place where the skip is - no more conservative than the path it replaces. - """ - candidates = 20 - target = _hash(7) - for view in ("live", "restart"): - for kind in PERTURBATIONS: - with self.subTest(view=view, perturbation=kind): - skipped = self._drain(candidates, view, kind, target, True) - if kind == "retry_slot": - self.assertIn(target, skipped) - else: - self.assertNotIn(target, skipped) - - def _drain( - self, - candidates: int, - view: str, - kind: str, - target: str, - dequeue_skip: bool, - ) -> set[str]: - rig = CandidateStormRig(candidates=candidates) - rig.seed_live() - if view == "restart": - rig.restart_and_enumerate() - server = rig.restarted_server if view == "restart" else rig.live_server - winner = rig.decide_height() - with redirect_stdout(StringIO()): - undo = rig.perturb(server, kind, target) - skip_set = _record_skips(server) if dequeue_skip else set() - try: - report = rig.drain_per_row( - server, - stop_before_hash=winner, - dequeue_skip=dequeue_skip, - ) - finally: - if undo is not None: - undo() - return skip_set if dequeue_skip else set(report.abandoned_hashes) - - -class DequeueAdmissionUnchangedTests(unittest.TestCase): - """D5: the live queue stays FIFO and stays bounded at 32.""" - - def test_the_live_queue_is_still_a_bounded_fifo_of_thirty_two(self) -> None: - self.assertEqual(MAX_PENDING_BLOCK_CANDIDATES, 32) - fixture = CollapseFixture() - service = fixture.service - service.candidate_queue = None - candidates = fixture.seed([_hash(index) for index in range(1, 40)]) - with redirect_stdout(StringIO()): - admitted = [ - fixture.server.enqueue_block_candidate(candidate) - for candidate in candidates - ] - queue_obj = service.candidate_queue - self.assertIsInstance(queue_obj, queue.Queue) - self.assertNotIsInstance(queue_obj, queue.PriorityQueue) - self.assertEqual(queue_obj.maxsize, 32) - self.assertEqual(queue_obj.qsize(), 32) - # The first 32 wakeups were admitted; the rest coalesced, and their - # durable rows are what keeps them replayable. - self.assertEqual(admitted, [True] * 32 + [False] * 7) - self.assertEqual(int(service.wakeups_coalesced), 7) - self.assertEqual( - [ - str(item.submission.block_hash_hex).lower() - for item in list(queue_obj.queue) - ], - [_hash(index) for index in range(1, 33)], - ) - - -if __name__ == "__main__": - unittest.main() diff --git a/tests/test_prism_direct_stratum.py b/tests/test_prism_direct_stratum.py deleted file mode 100644 index 171b742df..000000000 --- a/tests/test_prism_direct_stratum.py +++ /dev/null @@ -1,727 +0,0 @@ -#!/usr/bin/env python3 - -from __future__ import annotations - -import dataclasses -import unittest -from decimal import Decimal -from unittest import mock - -from lab.auxpow import stratum_codec -from lab.prism import direct_stratum - - -EXTRANONCE1 = "11111111" -EXTRANONCE2_SIZE = 8 -PLACEHOLDER = EXTRANONCE1 + ("00" * EXTRANONCE2_SIZE) - - -def tx_output(value_sats: int, script_hex: str) -> str: - return value_sats.to_bytes(8, "little").hex() + direct_stratum.compact_size(len(bytes.fromhex(script_hex))).hex() + script_hex - - -def synthetic_coinbase_template(witness_commitment: str | None = None) -> str: - script_sig = "5100" + PLACEHOLDER - p2mr_script = "5220" + ("aa" * 32) - witness_commitment = witness_commitment or "6a24aa21a9ed" + ("bb" * 32) - txin = ( - "01" - + ("00" * 32) - + "ffffffff" - + direct_stratum.compact_size(len(bytes.fromhex(script_sig))).hex() - + script_sig - + "feffffff" - ) - txout = "02" + tx_output(5_000_000_000, p2mr_script) + tx_output(0, witness_commitment) - witness = "01" + "20" + ("00" * 32) - return "02000000" + "0001" + txin + txout + witness + "00000000" - - -def coinbase_with( - scriptsig_suffix_hex: str, - *, - payout_outputs: list[tuple[int, str]] | None = None, - witness_commitment_hex: str | None = None, - witness_nonce_hex: str = "00" * 32, -) -> str: - """Build a coinbase whose scriptSig is ``OP_1 OP_0`` (height) plus an arbitrary suffix. - - The suffix is where the extranonce placeholder lives. ``payout_outputs``, - ``witness_commitment_hex`` and ``witness_nonce_hex`` let a test deliberately - embed the same bytes elsewhere in the coinbase to exercise collisions. - """ - script_sig = "5100" + scriptsig_suffix_hex - witness_commitment_hex = witness_commitment_hex or ("6a24aa21a9ed" + ("bb" * 32)) - outputs = list(payout_outputs or [(5_000_000_000, "5220" + ("aa" * 32))]) - outputs.append((0, witness_commitment_hex)) - txin = ( - "01" - + ("00" * 32) - + "ffffffff" - + direct_stratum.compact_size(len(bytes.fromhex(script_sig))).hex() - + script_sig - + "feffffff" - ) - txout = direct_stratum.compact_size(len(outputs)).hex() + "".join( - tx_output(value, script) for value, script in outputs - ) - witness = ( - "01" - + direct_stratum.compact_size(len(bytes.fromhex(witness_nonce_hex))).hex() - + witness_nonce_hex - ) - return "02000000" + "0001" + txin + txout + witness + "00000000" - - -def manifest_for(coinbase_tx_hex: str) -> dict[str, str]: - return { - "coinbase_tx_hex": coinbase_tx_hex, - "coinbase_txid": direct_stratum.transaction_txid_display(coinbase_tx_hex), - } - - -def synthetic_transaction(seed: str) -> str: - script = seed * 3 - return ( - "01000000" - + "01" - + (seed * 32) - + "00000000" - + direct_stratum.compact_size(len(bytes.fromhex(script))).hex() - + script - + "ffffffff" - + "01" - + tx_output(1, "51") - + "00000000" - ) - - -def synthetic_witness_transaction(seed: str) -> str: - script = seed * 3 - witness_item = seed * 5 - return ( - "01000000" - + "0001" - + "01" - + (seed * 32) - + "00000000" - + direct_stratum.compact_size(len(bytes.fromhex(script))).hex() - + script - + "ffffffff" - + "01" - + tx_output(1, "51") - + "01" - + direct_stratum.compact_size(len(bytes.fromhex(witness_item))).hex() - + witness_item - + "00000000" - ) - - -def merkle_root(hashes: list[bytes]) -> bytes: - while len(hashes) > 1: - hashes = [ - stratum_codec.double_sha256( - hashes[index] + (hashes[index + 1] if index + 1 < len(hashes) else hashes[index]) - ) - for index in range(0, len(hashes), 2) - ] - return hashes[0] - - -def witness_commitment_script(transaction_hexes: list[str]) -> str: - root = merkle_root([bytes(32)] + [direct_stratum.transaction_wtxid_internal(tx_hex) for tx_hex in transaction_hexes]) - commitment = stratum_codec.double_sha256(root + bytes(32)) - return "6a24aa21a9ed" + commitment.hex() - - -def transaction_end(data: bytes, offset: int) -> int: - offset += 4 - has_witness = data[offset] == 0 and data[offset + 1] != 0 - if has_witness: - offset += 2 - input_count, offset = direct_stratum.read_compact_size(data, offset, field_name="transaction inputs") - for input_index in range(input_count): - offset = direct_stratum.skip_bytes(data, offset, 36, field_name=f"transaction input[{input_index}] prevout") - script_len, offset = direct_stratum.read_compact_size( - data, - offset, - field_name=f"transaction input[{input_index}] script", - ) - offset = direct_stratum.skip_bytes(data, offset, script_len, field_name=f"transaction input[{input_index}] script") - offset = direct_stratum.skip_bytes(data, offset, 4, field_name=f"transaction input[{input_index}] sequence") - output_count, offset = direct_stratum.read_compact_size(data, offset, field_name="transaction outputs") - for output_index in range(output_count): - offset = direct_stratum.skip_bytes(data, offset, 8, field_name=f"transaction output[{output_index}] value") - script_len, offset = direct_stratum.read_compact_size( - data, - offset, - field_name=f"transaction output[{output_index}] script", - ) - offset = direct_stratum.skip_bytes(data, offset, script_len, field_name=f"transaction output[{output_index}] script") - if has_witness: - for input_index in range(input_count): - item_count, offset = direct_stratum.read_compact_size(data, offset, field_name=f"transaction witness[{input_index}]") - for item_index in range(item_count): - item_len, offset = direct_stratum.read_compact_size( - data, - offset, - field_name=f"transaction witness[{input_index}][{item_index}]", - ) - offset = direct_stratum.skip_bytes( - data, - offset, - item_len, - field_name=f"transaction witness[{input_index}][{item_index}]", - ) - return direct_stratum.skip_bytes(data, offset, 4, field_name="transaction locktime") - - -def block_transaction_hexes(block_hex: str) -> list[str]: - data = bytes.fromhex(block_hex) - offset = 80 - tx_count, offset = direct_stratum.read_compact_size(data, offset, field_name="block transaction count") - result = [] - for _ in range(tx_count): - start = offset - offset = transaction_end(data, offset) - result.append(data[start:offset].hex()) - if offset != len(data): - raise ValueError("block has trailing bytes") - return result - - -def template(transactions: list[str] | None = None) -> dict[str, object]: - return { - "previousblockhash": "00" * 32, - "version": 0x20000000, - "bits": "207fffff", - "curtime": 1_700_000_000, - "transactions": [{"data": tx_hex} for tx_hex in transactions or []], - } - - -class PrismDirectStratumTests(unittest.TestCase): - def test_builder_coinbase_template_splits_without_witness_and_reassembles_submitblock(self) -> None: - coinbase_tx_hex = synthetic_coinbase_template() - manifest = { - "coinbase_tx_hex": coinbase_tx_hex, - "coinbase_txid": direct_stratum.transaction_txid_display(coinbase_tx_hex), - } - job = direct_stratum.make_job_from_builder_manifest( - job_id="job-1", - template=template(), - manifest=manifest, - extranonce1_hex=EXTRANONCE1, - extranonce2_size=EXTRANONCE2_SIZE, - desired_share_difficulty=Decimal("1"), - ) - - submission = direct_stratum.assemble_submission( - job, - extranonce2_hex="cafebabe00000001", - ntime_hex=job.ntime, - nonce_hex="0000002a", - version_bits_hex="00002000", - version_mask=direct_stratum.QBIT_VERSION_ROLLING_MASK, - ) - - self.assertEqual(submission.applied_version_hex, "20002000") - self.assertEqual( - submission.coinbase_tx_hex, - coinbase_tx_hex.replace(PLACEHOLDER, EXTRANONCE1 + "cafebabe00000001", 1), - ) - self.assertEqual( - direct_stratum.strip_witness_transaction(submission.coinbase_tx_hex).hex(), - submission.coinbase_txid_preimage_hex, - ) - self.assertTrue(submission.block_pass) - self.assertEqual(submission.block_hex, submission.header_hex + "01" + submission.coinbase_tx_hex) - self.assertEqual(len(bytes.fromhex(submission.header_hex)), 80) - - def test_merkle_branch_uses_transaction_txids_excluding_witness(self) -> None: - extra_tx = synthetic_transaction("22") - manifest = { - "coinbase_tx_hex": synthetic_coinbase_template(), - "coinbase_txid": direct_stratum.transaction_txid_display(synthetic_coinbase_template()), - } - job = direct_stratum.make_job_from_builder_manifest( - job_id="job-branch", - template=template([extra_tx]), - manifest=manifest, - extranonce1_hex=EXTRANONCE1, - extranonce2_size=EXTRANONCE2_SIZE, - desired_share_difficulty=Decimal("1"), - ) - - self.assertEqual(job.merkle_branch, (direct_stratum.transaction_txid_internal(extra_tx).hex(),)) - - submission = direct_stratum.assemble_submission( - job, - extranonce2_hex="0102030405060708", - ntime_hex=job.ntime, - nonce_hex="00000000", - ) - coinbase_txid = stratum_codec.double_sha256(bytes.fromhex(submission.coinbase_txid_preimage_hex)) - expected_merkle_root = stratum_codec.double_sha256(coinbase_txid + direct_stratum.transaction_txid_internal(extra_tx)) - expected_header = stratum_codec.serialize_header_from_stratum_fields( - version_hex=job.version, - prevhash_hex=job.prevhash, - merkle_root_serialized=expected_merkle_root, - ntime_hex=job.ntime, - nbits_hex=job.nbits, - nonce_hex="00000000", - ) - - self.assertEqual(submission.header_hex, expected_header.hex()) - self.assertTrue(submission.block_pass) - self.assertTrue(submission.block_hex.endswith(submission.coinbase_tx_hex + extra_tx)) - - def test_witness_commitment_uses_template_wtxids_for_serialized_block(self) -> None: - witness_tx = synthetic_witness_transaction("44") - self.assertNotEqual( - direct_stratum.transaction_txid_internal(witness_tx), - direct_stratum.transaction_wtxid_internal(witness_tx), - ) - commitment_script = witness_commitment_script([witness_tx]) - coinbase_tx_hex = synthetic_coinbase_template(commitment_script) - manifest = { - "coinbase_tx_hex": coinbase_tx_hex, - "coinbase_txid": direct_stratum.transaction_txid_display(coinbase_tx_hex), - } - job = direct_stratum.make_job_from_builder_manifest( - job_id="job-witness", - template=template([witness_tx]), - manifest=manifest, - extranonce1_hex=EXTRANONCE1, - extranonce2_size=EXTRANONCE2_SIZE, - desired_share_difficulty=Decimal("1"), - ) - - self.assertEqual(job.merkle_branch, (direct_stratum.transaction_txid_internal(witness_tx).hex(),)) - self.assertEqual( - direct_stratum.witness_merkle_leaves_hex(job.transaction_hexes), - [direct_stratum.transaction_wtxid_internal(witness_tx).hex()], - ) - - # Nonce 1 rather than 0: bits 207fffff put the network target at - # roughly half the hash space, and only a hash that solved the block - # serializes one to inspect. - submission = direct_stratum.assemble_submission( - job, - extranonce2_hex="0102030405060708", - ntime_hex=job.ntime, - nonce_hex="00000001", - ) - self.assertTrue(submission.block_pass) - block_txs = block_transaction_hexes(submission.block_hex) - self.assertEqual(block_txs, [submission.coinbase_tx_hex, witness_tx]) - self.assertIn(witness_commitment_script(block_txs[1:]), block_txs[0]) - - def test_rejects_mismatched_txid_and_placeholder_not_in_scriptsig(self) -> None: - coinbase_tx_hex = synthetic_coinbase_template() - manifest = { - "coinbase_tx_hex": coinbase_tx_hex, - "coinbase_txid": "00" * 32, - } - with self.assertRaisesRegex(ValueError, "coinbase_txid"): - direct_stratum.make_job_from_builder_manifest( - job_id="bad-txid", - template=template(), - manifest=manifest, - extranonce1_hex=EXTRANONCE1, - extranonce2_size=EXTRANONCE2_SIZE, - desired_share_difficulty=Decimal("1"), - ) - - # A coinbase whose scriptSig does not carry the placeholder at its tail is - # rejected outright rather than silently mis-split. - wrong_suffix = coinbase_with("ab" * (len(bytes.fromhex(PLACEHOLDER)))) - with self.assertRaisesRegex(ValueError, "does not end its coinbase scriptSig"): - direct_stratum.split_coinbase_extranonce( - wrong_suffix, PLACEHOLDER, field_name="coinbase txid preimage" - ) - # A scriptSig shorter than the placeholder cannot contain it. - with self.assertRaisesRegex(ValueError, "shorter than the extranonce placeholder"): - direct_stratum.split_coinbase_extranonce( - coinbase_with(""), PLACEHOLDER, field_name="coinbase txid preimage" - ) - # Truncated / non-coinbase hex is reported, never an unhandled crash. - with self.assertRaisesRegex(ValueError, "too short to be a coinbase transaction"): - direct_stratum.split_coinbase_extranonce("aa", PLACEHOLDER, field_name="coinbase txid preimage") - - def test_split_is_robust_when_placeholder_bytes_appear_in_outputs_and_witness(self) -> None: - # The extranonce placeholder bytes are deliberately echoed in a payout - # output script, in the witness commitment, and in the witness nonce. The - # old substring split raised "extranonce placeholder more than once" here - # and tore down the miner's client thread. - placeholder_bytes = bytes.fromhex(PLACEHOLDER) - collision_output_script = ( - "6a" + direct_stratum.compact_size(len(placeholder_bytes)).hex() + PLACEHOLDER - ) - witness_commitment = "6a24aa21a9ed" + PLACEHOLDER + ("bb" * (32 - len(placeholder_bytes))) - witness_nonce = PLACEHOLDER + ("00" * (32 - len(placeholder_bytes))) - extranonce2_hex = "cafebabe00000001" - - coinbase_tx_hex = coinbase_with( - PLACEHOLDER, - payout_outputs=[(5_000_000_000, "5220" + ("aa" * 32)), (7, collision_output_script)], - witness_commitment_hex=witness_commitment, - witness_nonce_hex=witness_nonce, - ) - # Sanity: the placeholder really does appear several times in the coinbase. - self.assertGreaterEqual(coinbase_tx_hex.count(PLACEHOLDER), 4) - - job = direct_stratum.make_job_from_builder_manifest( - job_id="job-collision", - template=template(), - manifest=manifest_for(coinbase_tx_hex), - extranonce1_hex=EXTRANONCE1, - extranonce2_size=EXTRANONCE2_SIZE, - desired_share_difficulty=Decimal("1"), - ) - submission = direct_stratum.assemble_submission( - job, - extranonce2_hex=extranonce2_hex, - ntime_hex=job.ntime, - nonce_hex="00000000", - ) - - # Only the scriptSig occurrence is rewritten; the colliding copies in the - # outputs, witness commitment, and witness nonce are preserved byte-for-byte. - expected = coinbase_with( - EXTRANONCE1 + extranonce2_hex, - payout_outputs=[(5_000_000_000, "5220" + ("aa" * 32)), (7, collision_output_script)], - witness_commitment_hex=witness_commitment, - witness_nonce_hex=witness_nonce, - ) - self.assertEqual(submission.coinbase_tx_hex, expected) - self.assertEqual(submission.coinbase_tx_hex.count(PLACEHOLDER), 3) - self.assertEqual( - direct_stratum.strip_witness_transaction(submission.coinbase_tx_hex).hex(), - submission.coinbase_txid_preimage_hex, - ) - - def test_split_anchors_on_scriptsig_when_placeholder_collides_with_null_outpoint(self) -> None: - # A low connection id yields extranonce1="00000000", so the placeholder is - # all zero bytes -- which is contained in the 32-byte all-zero null outpoint - # that appears *before* the scriptSig. A naive first-occurrence search would - # split inside the outpoint and corrupt the coinbase (and the duplicate check - # would raise). Structural parsing anchors correctly on the scriptSig. - extranonce1 = "00000000" - placeholder = extranonce1 + ("00" * EXTRANONCE2_SIZE) - extranonce2_hex = "0123456789abcdef" - coinbase_tx_hex = coinbase_with(placeholder) - - # The naive substring search would land inside the null outpoint, well - # before the real scriptSig insertion point. - scriptsig_offset = coinbase_tx_hex.index("5100" + placeholder) + len("5100") - self.assertLess(coinbase_tx_hex.find(placeholder) * 1, scriptsig_offset) - self.assertGreater(coinbase_tx_hex.count(placeholder), 1) - - job = direct_stratum.make_job_from_builder_manifest( - job_id="job-zero-extranonce", - template=template(), - manifest=manifest_for(coinbase_tx_hex), - extranonce1_hex=extranonce1, - extranonce2_size=EXTRANONCE2_SIZE, - desired_share_difficulty=Decimal("1"), - ) - submission = direct_stratum.assemble_submission( - job, - extranonce2_hex=extranonce2_hex, - ntime_hex=job.ntime, - nonce_hex="00000000", - ) - - self.assertEqual(submission.coinbase_tx_hex, coinbase_with(extranonce1 + extranonce2_hex)) - # The null outpoint ahead of the scriptSig is untouched. - self.assertTrue( - submission.coinbase_tx_hex.startswith("02000000" + "0001" + "01" + ("00" * 32) + "ffffffff") - ) - self.assertEqual( - direct_stratum.strip_witness_transaction(submission.coinbase_tx_hex).hex(), - submission.coinbase_txid_preimage_hex, - ) - - def test_version_bits_are_limited_to_negotiated_gbt_mask(self) -> None: - coinbase_tx_hex = synthetic_coinbase_template() - manifest = { - "coinbase_tx_hex": coinbase_tx_hex, - "coinbase_txid": direct_stratum.transaction_txid_display(coinbase_tx_hex), - } - job = direct_stratum.make_job_from_builder_manifest( - job_id="job-mask", - template=template(), - manifest=manifest, - extranonce1_hex=EXTRANONCE1, - extranonce2_size=EXTRANONCE2_SIZE, - desired_share_difficulty=Decimal("1"), - ) - - submission = direct_stratum.assemble_submission( - job, - extranonce2_hex="0000000000000001", - ntime_hex=job.ntime, - nonce_hex="00000000", - version_bits_hex="00002000", - version_mask=direct_stratum.QBIT_VERSION_ROLLING_MASK, - ) - - self.assertEqual(submission.applied_version_hex, "20002000") - - with self.assertRaisesRegex(ValueError, "outside the negotiated mask"): - direct_stratum.assemble_submission( - job, - extranonce2_hex="0000000000000001", - ntime_hex=job.ntime, - nonce_hex="00000000", - version_bits_hex="000000ff", - version_mask=direct_stratum.QBIT_VERSION_ROLLING_MASK, - ) - - def test_select_version_rolling_mask_uses_gbt_field_or_fallback(self) -> None: - advertised = direct_stratum.select_version_rolling_mask( - {"versionrollingmask": "1fffe000"}, - 0x000000FF, - ) - missing = direct_stratum.select_version_rolling_mask({}, direct_stratum.QBIT_VERSION_ROLLING_MASK) - disabled = direct_stratum.select_version_rolling_mask( - {"versionrollingmask": "00000000"}, - direct_stratum.QBIT_VERSION_ROLLING_MASK, - ) - - self.assertEqual(advertised.selected_mask, 0x1FFFE000) - self.assertEqual(advertised.source, "qbit_getblocktemplate") - self.assertEqual(missing.selected_mask, direct_stratum.QBIT_VERSION_ROLLING_MASK) - self.assertEqual(missing.source, "fallback") - self.assertEqual(disabled.selected_mask, 0) - self.assertEqual(disabled.detail, "disabled_by_zero_mask") - - with self.assertRaisesRegex(ValueError, "invalid getblocktemplate.versionrollingmask"): - direct_stratum.select_version_rolling_mask( - {"versionrollingmask": "not-hex"}, - direct_stratum.QBIT_VERSION_ROLLING_MASK, - ) - - -class DeferredBlockSerializationTests(unittest.TestCase): - """The block is serialized only for a hash that actually solved it. - - ``block_hex`` costs O(block bytes) of GIL-held decode, concatenate and - re-encode, and only the block-landing path ever reads it, so an ordinary - accepted share must not pay for it (#143). These tests pin both halves of - that bargain: the share path never builds the bytes, and the block path - always has them before a candidate can exist. - """ - - # qbit_target == 1, so no hash solves the block. - UNSOLVABLE_BITS = "03000001" - # Difficulty this low clamps the share target to 2**256-1, so every hash - # passes the share target while missing the (unsolvable) network target. - ALWAYS_SHARE_DIFFICULTY = Decimal("1e-40") - - def job( - self, - *, - bits: str | None = None, - transactions: list[str] | None = None, - desired_share_difficulty: Decimal = Decimal("1"), - ) -> direct_stratum.DirectQbitStratumJob: - coinbase_tx_hex = synthetic_coinbase_template() - built_template = template(transactions) - if bits is not None: - built_template["bits"] = bits - return direct_stratum.make_job_from_builder_manifest( - job_id="job-serialization", - template=built_template, - manifest=manifest_for(coinbase_tx_hex), - extranonce1_hex=EXTRANONCE1, - extranonce2_size=EXTRANONCE2_SIZE, - desired_share_difficulty=desired_share_difficulty, - ) - - def assemble( - self, - job: direct_stratum.DirectQbitStratumJob, - ) -> direct_stratum.DirectQbitSubmission: - return direct_stratum.assemble_submission( - job, - extranonce2_hex="cafebabe00000001", - ntime_hex=job.ntime, - nonce_hex="0000002a", - ) - - def test_ordinary_share_never_serializes_the_block(self) -> None: - job = self.job( - bits=self.UNSOLVABLE_BITS, - transactions=[synthetic_transaction("22")], - desired_share_difficulty=self.ALWAYS_SHARE_DIFFICULTY, - ) - - with mock.patch.object( - direct_stratum, - "serialize_block", - side_effect=AssertionError("serialize_block ran on the share path"), - ) as serialize: - submission = self.assemble(job) - - serialize.assert_not_called() - self.assertTrue(submission.share_pass) - self.assertFalse(submission.block_pass) - self.assertEqual(submission.block_hex, "") - - def test_block_worthy_share_materializes_the_block_in_assembly(self) -> None: - extra_tx = synthetic_transaction("22") - job = self.job(transactions=[extra_tx]) - - submission = self.assemble(job) - - self.assertTrue(submission.block_pass) - # Byte-identical to what the unconditional call produced before the - # share path stopped paying for it. - self.assertEqual( - submission.block_hex, - direct_stratum.serialize_block( - bytes.fromhex(submission.header_hex), - submission.coinbase_tx_hex, - job.transaction_hexes, - ).hex(), - ) - self.assertEqual( - block_transaction_hexes(submission.block_hex), - [submission.coinbase_tx_hex, extra_tx], - ) - - def test_block_hex_attribute_exists_on_every_submission(self) -> None: - """The landing path's duck-typed guard must stay meaningful. - - ``block_candidates`` probes ``hasattr(submission, "block_hex")`` to - tell a real submission from an embedder that retains no block bytes. - Skipping the work must not skip the attribute, or every ordinary share - would start looking like such an embedder. - """ - share_only = self.job( - bits=self.UNSOLVABLE_BITS, - desired_share_difficulty=self.ALWAYS_SHARE_DIFFICULTY, - ) - - submission = self.assemble(share_only) - - self.assertTrue(hasattr(submission, "block_hex")) - self.assertIsInstance(submission.block_hex, str) - - def test_coinbase_validation_still_runs_on_a_share_that_missed_the_block( - self, - ) -> None: - """Deferring the block bytes must not defer coinbase validation. - - ``serialize_block`` used to decode the full coinbase on every share as - a side effect. The explicit txid-preimage check is what actually - guards that value, and it has to keep firing at assembly time -- not - at block landing, which is the worst possible moment to discover it. - """ - job = self.job( - bits=self.UNSOLVABLE_BITS, - desired_share_difficulty=self.ALWAYS_SHARE_DIFFICULTY, - ) - # Move the full coinbase's locktime without moving the Stratum txid - # preimage's: the witness strip no longer reproduces the preimage. - tampered = dataclasses.replace( - job, - full_coinbase_suffix=job.full_coinbase_suffix[:-8] + "01000000", - ) - - with self.assertRaisesRegex(ValueError, "does not match Stratum txid preimage"): - self.assemble(tampered) - - def test_template_transactions_are_validated_once_at_job_build(self) -> None: - """Per-template validation amortizes across every share of that job. - - ``serialize_block`` re-validated the hex of every transaction on every - share. The values are immutable and were already validated here, at - build, so dropping the per-share pass moves no detection anywhere. - """ - malformed = template(["zz"]) - - with self.assertRaisesRegex( - ValueError, - r"template transaction\[0\] data must be hexadecimal", - ): - direct_stratum.make_job_from_builder_manifest( - job_id="job-malformed", - template=malformed, - manifest=manifest_for(synthetic_coinbase_template()), - extranonce1_hex=EXTRANONCE1, - extranonce2_size=EXTRANONCE2_SIZE, - desired_share_difficulty=Decimal("1"), - ) - - # A well-formed template is validated and normalized at build, so the - # tuple every later share reads is already known good. - extra_tx = synthetic_transaction("22") - job = self.job(transactions=[extra_tx.upper()]) - self.assertEqual(job.transaction_hexes, (extra_tx,)) - - def test_explicit_transaction_hexes_are_validated_at_job_build(self) -> None: - """The bundle path passes transactions in directly; it validates too.""" - coinbase_tx_hex = synthetic_coinbase_template() - - with self.assertRaisesRegex( - ValueError, - r"transaction_hexes\[0\] must be hexadecimal", - ): - direct_stratum.make_job_from_builder_manifest( - job_id="job-malformed-explicit", - template=template(), - manifest=manifest_for(coinbase_tx_hex), - extranonce1_hex=EXTRANONCE1, - extranonce2_size=EXTRANONCE2_SIZE, - desired_share_difficulty=Decimal("1"), - transaction_hexes=("zz",), - ) - - -class EffectiveShareTargetTests(unittest.TestCase): - def test_network_cap_governs_without_minimum(self) -> None: - # bits 207fffff: network difficulty ~4.7e-10, so a 500k desired - # difficulty is capped down to the (easier) network target. - qbit_target = direct_stratum.target_from_compact_hex("207fffff") - - share_target = direct_stratum.effective_share_target(Decimal("500000"), qbit_target) - - self.assertEqual(share_target, qbit_target) - self.assertLess(direct_stratum.target_difficulty(share_target), Decimal("1")) - - def test_minimum_advertised_difficulty_overrides_network_cap(self) -> None: - qbit_target = direct_stratum.target_from_compact_hex("207fffff") - - share_target = direct_stratum.effective_share_target( - Decimal("500000"), - qbit_target, - minimum_advertised_difficulty=Decimal("500000"), - ) - - self.assertEqual(share_target, direct_stratum.difficulty_target(Decimal("500000"))) - # Compare at wire precision: set_difficulty sends float(difficulty), - # and Decimal round-tripping can sit within 1e-27 of the floor. - self.assertGreaterEqual( - float(direct_stratum.target_difficulty(share_target)), - 500000.0, - ) - - def test_minimum_is_inert_when_desired_already_meets_it(self) -> None: - # Mature chain: network target harder than the floor target, desired - # difficulty above the floor. The floor changes nothing. - qbit_target = direct_stratum.difficulty_target(Decimal("80000000")) - - share_target = direct_stratum.effective_share_target( - Decimal("600000"), - qbit_target, - minimum_advertised_difficulty=Decimal("500000"), - ) - - self.assertEqual(share_target, direct_stratum.difficulty_target(Decimal("600000"))) - - -if __name__ == "__main__": - unittest.main() diff --git a/tests/test_prism_endpoint_registry.py b/tests/test_prism_endpoint_registry.py deleted file mode 100644 index c0e640171..000000000 --- a/tests/test_prism_endpoint_registry.py +++ /dev/null @@ -1,672 +0,0 @@ -#!/usr/bin/env python3 -"""Pin the endpoint classification registry against real HTTP dispatch. - -The registry in ``lab/prism/endpoint_registry.py`` is the reviewable form of -the issue #145 public/operator split. It is only worth reviewing if it cannot -drift from the code, so these tests assert both directions: every classified -path is actually routed, and every routed path is classified. -""" - -from __future__ import annotations - -import ast -import json -import unittest -import urllib.error -import urllib.request -from pathlib import Path - -from lab.prism import endpoint_registry, public_api -from lab.prism.audit_http import AuditHttpConfig, AuditHttpFacade -from lab.prism.endpoint_registry import ( - Audience, - Disposition, - ENDPOINTS, - LedgerAccess, -) -from lab.prism.observability import METRICS_STATE_FRESH, MetricsSnapshotResponse - - -ROOT = Path(__file__).resolve().parents[1] -AUDIT_HTTP_PATH = ROOT / "lab" / "prism" / "audit_http.py" -PUBLIC_API_PATH = ROOT / "lab" / "prism" / "public_api.py" - -HEX64 = "ab" * 32 -PLACEHOLDERS = { - "{block_hash}": HEX64, - "{fanout_txid}": HEX64, - "{commitment_leaf_hex}": HEX64, - "{sha256}": HEX64, - "{recipient_id}": "miner-1", -} - -UNKNOWN_ENDPOINT_BODY = {"error": "unknown endpoint"} -UNKNOWN_PUBLIC_MESSAGE = "unknown public dashboard endpoint" - - -def concrete_path(path: str) -> str: - for placeholder, value in PLACEHOLDERS.items(): - path = path.replace(placeholder, value) - if "{" in path: - raise AssertionError(f"unsubstituted placeholder in {path!r}") - return path - - -class _Sentinel(Exception): - """Raised by the fake coordinator once dispatch reaches a real handler.""" - - -class _ExplodingLedger: - backend_name = "fixture" - - def __getattr__(self, name: str): - def _call(*args: object, **kwargs: object): - raise _Sentinel(name) - - return _call - - -class _ExplodingRpc: - def call(self, *args: object, **kwargs: object): - raise _Sentinel("rpc") - - -class _FakeCoordinator: - def __init__(self) -> None: - self.ledger = _ExplodingLedger() - self.rpc = _ExplodingRpc() - - -class _RoutingPort: - """Answers every AuditHttpPort method so only routing is under test.""" - - def cached_health_payload(self) -> tuple[int, dict[str, object]]: - return 200, {"ok": True} - - def cached_metrics_payload(self) -> MetricsSnapshotResponse: - return MetricsSnapshotResponse( - status=200, - body="fixture 1\n", - state=METRICS_STATE_FRESH, - age_seconds=0, - ) - - def latest_evidence_payload(self) -> dict[str, object] | None: - return {"schema": "fixture"} - - def owed_balances_payload(self) -> dict[str, object]: - return {"balances": []} - - def carry_forward_integrity_payload(self) -> dict[str, object]: - return {"ok": True} - - def miner_status_payload(self, recipient_id: str) -> dict[str, object]: - return {"recipient_id": recipient_id} - - def ledger_backend(self) -> str: - return "fixture" - - def audit_share_window(self, **kwargs: object) -> list[dict[str, object]]: - return [{"fixture": True}] - - def audit_block_payouts(self, **kwargs: object) -> list[dict[str, object]]: - return [{"fixture": True}] - - def audit_ctv_fanouts(self, **kwargs: object) -> list[dict[str, object]]: - return [{"fixture": True}] - - def audit_ctv_fanout_manifest_set(self, **kwargs: object) -> dict[str, object]: - return {"fixture": True} - - def ctv_fanout_status(self, **kwargs: object) -> dict[str, object]: - return {"fixture": True} - - def pending_ctv_fanout_statuses(self, **kwargs: object) -> list[dict[str, object]]: - return [{"fixture": True}] - - def audit_bundle(self, **kwargs: object) -> dict[str, object]: - return {"fixture": True} - - def audit_bundle_by_commitment(self, **kwargs: object) -> dict[str, object]: - return {"fixture": True} - - -def string_literals(source_path: Path, function_name: str) -> set[str]: - """Absolute path literals mentioned inside one function.""" - - tree = ast.parse(source_path.read_text(encoding="utf-8")) - for node in ast.walk(tree): - if isinstance(node, ast.FunctionDef) and node.name == function_name: - return { - child.value - for child in ast.walk(node) - if isinstance(child, ast.Constant) - and isinstance(child.value, str) - and child.value.startswith("/") - } - raise AssertionError(f"{function_name} not found in {source_path.name}") - - -class EndpointRegistryShapeTests(unittest.TestCase): - def test_paths_are_unique_across_the_registry(self) -> None: - seen: dict[str, str] = {} - for endpoint in ENDPOINTS: - for path in endpoint.paths: - self.assertNotIn( - path, - seen, - f"{path} classified twice ({seen.get(path)} and {endpoint.primary_path})", - ) - seen[path] = endpoint.primary_path - - def test_only_the_public_read_surface_is_extracted(self) -> None: - for endpoint in ENDPOINTS: - expected = ( - Disposition.EXTRACT - if endpoint.audience is Audience.PUBLIC_READ - else Disposition.RETAIN - ) - self.assertIs( - endpoint.disposition, - expected, - f"{endpoint.primary_path}: {endpoint.audience} must be {expected}", - ) - - def test_extracted_surface_is_exactly_public_v1(self) -> None: - # PRISM.md: "operators can expose only /public/v1 through a reverse - # proxy or dashboard frontend". The extraction must not widen that. - for path in endpoint_registry.extracted_paths(): - self.assertTrue( - path.startswith("/public/v1/"), - f"{path} is not part of the documented public API surface", - ) - - def test_audit_routes_are_never_extracted(self) -> None: - # docs/public-dashboard-api/README.md and - # tests/test_public_dashboard_api_contract.py both forbid exposing - # /audit/* as public API. - for path in endpoint_registry.extracted_paths(): - self.assertFalse(path.startswith("/audit/")) - for prefix in ("/audit/", "/healthz", "/metrics", "/owed"): - self.assertTrue( - any(path.startswith(prefix) for path in endpoint_registry.retained_paths()), - f"expected retained paths under {prefix}", - ) - - def test_extracted_and_retained_partition_the_registry(self) -> None: - extracted = set(endpoint_registry.extracted_paths()) - retained = set(endpoint_registry.retained_paths()) - self.assertEqual(set(), extracted & retained) - every_path = {path for endpoint in ENDPOINTS for path in endpoint.paths} - self.assertEqual(every_path, extracted | retained) - - def test_writer_lock_extractions_record_a_blocker(self) -> None: - # The Endpoint constructor enforces this; assert the set is non-empty so - # the rule cannot be satisfied vacuously by dropping the classification. - blockers = dict(endpoint_registry.extraction_blockers()) - for endpoint in ENDPOINTS: - if ( - LedgerAccess.WRITER_LOCK in endpoint.access - and endpoint.disposition is Disposition.EXTRACT - ): - self.assertIn(endpoint.primary_path, blockers) - self.assertTrue(blockers) - - def test_health_and_metrics_are_retained_operator_surfaces(self) -> None: - retained = endpoint_registry.retained_paths() - self.assertIn("/healthz", retained) - self.assertIn("/metrics", retained) - - def test_drift_count_and_owed_dump_stay_on_the_coordinator(self) -> None: - retained = endpoint_registry.retained_paths() - for path in ( - "/audit/carry-forward-integrity", - "/audit/ledger-integrity", - "/owed", - "/owed-balances", - "/miners/{recipient_id}/status", - "/payouts/{recipient_id}/status", - ): - self.assertIn(path, retained) - - def test_payout_affecting_surface_is_named(self) -> None: - payout = { - path - for endpoint in endpoint_registry.endpoints_by_audience( - Audience.PAYOUT_AFFECTING - ) - for path in endpoint.paths - } - self.assertEqual( - { - "/audit/carry-forward-integrity", - "/audit/ledger-integrity", - "/owed", - "/owed-balances", - "/miners/{recipient_id}/status", - "/payouts/{recipient_id}/status", - }, - payout, - ) - - def test_no_extracted_route_takes_the_writer_lock_without_a_blocker(self) -> None: - blockers = dict(endpoint_registry.extraction_blockers()) - extracted = set(endpoint_registry.extracted_paths()) - for path in endpoint_registry.writer_lock_paths(): - if path in extracted: - self.assertIn(path, blockers) - - -class StalenessBudgetTests(unittest.TestCase): - """The staleness contract must be derived and stated, never hand-picked.""" - - def extracted_endpoints(self) -> list[endpoint_registry.Endpoint]: - return [ - endpoint - for endpoint in ENDPOINTS - if endpoint.disposition is Disposition.EXTRACT - ] - - def test_every_extracted_route_states_a_budget_or_declares_immutability(self) -> None: - for endpoint in self.extracted_endpoints(): - with self.subTest(path=endpoint.primary_path): - self.assertTrue( - endpoint.max_staleness_seconds is not None - or endpoint.immutable_content, - f"{endpoint.primary_path} states no staleness tolerance", - ) - - def test_retained_routes_carry_no_public_staleness_budget(self) -> None: - # The contract covers the extracted public surface. A budget on a - # retained route would imply the coordinator enforces one, which it - # does not. - for endpoint in ENDPOINTS: - if endpoint.disposition is Disposition.EXTRACT: - continue - with self.subTest(path=endpoint.primary_path): - self.assertIsNone(endpoint.max_staleness_seconds) - self.assertFalse(endpoint.immutable_content) - - def test_budgets_match_the_documented_derivation(self) -> None: - # Pin each budget to the formula rather than to a literal, so changing - # a cache default without revisiting the budget fails here. - aggregate = endpoint_registry.PUBLIC_AGGREGATE_CACHE_TTL_DEFAULT_SECONDS - plain = endpoint_registry.PUBLIC_CACHE_TTL_DEFAULT_SECONDS - config = endpoint_registry.PUBLIC_CONFIG_CACHE_TTL_DEFAULT_SECONDS - reward_window = endpoint_registry.POOL_REWARD_WINDOW_CACHE_DEFAULT_SECONDS - derive = endpoint_registry.staleness_budget_seconds - expected = { - "/public/v1/pool-summary": derive(cache_ttl_seconds=aggregate), - "/public/v1/blocks": derive(cache_ttl_seconds=plain), - "/public/v1/leaderboard": derive(cache_ttl_seconds=plain), - "/public/v1/hashrate-series": derive(cache_ttl_seconds=aggregate), - # A cheap grouped scan of the small pool-blocks table, so it is a - # plain row read rather than an aggregate despite serving a chart. - "/public/v1/block-markers": derive(cache_ttl_seconds=plain), - "/public/v1/mining-configuration": derive(cache_ttl_seconds=config), - # The one route with a second cache in series beneath it. - "/public/v1/miners/{recipient_id}": derive( - cache_ttl_seconds=plain, - underlying_cache_seconds=reward_window, - ), - "/public/v1/miners/{recipient_id}/earnings": derive(cache_ttl_seconds=plain), - "/public/v1/miners/{recipient_id}/payouts": derive(cache_ttl_seconds=plain), - "/public/v1/miners/{recipient_id}/workers": derive(cache_ttl_seconds=aggregate), - "/public/v1/blocks/{block_hash}/settlement-artifacts": derive( - cache_ttl_seconds=plain - ), - "/public/v1/fanouts/pending": derive(cache_ttl_seconds=plain), - "/public/v1/fanouts/{fanout_txid}": derive(cache_ttl_seconds=plain), - } - actual = { - endpoint.primary_path: endpoint.max_staleness_seconds - for endpoint in self.extracted_endpoints() - if endpoint.max_staleness_seconds is not None - } - self.assertEqual(expected, actual) - - def test_the_aggregate_ttl_really_is_what_the_cache_policy_applies(self) -> None: - # The derivation above assumes public_cache_policy classifies these - # three as aggregates and everything else as a plain row read. Assert - # that against the policy itself so the two cannot drift apart. - for path, expected_ttl in ( - ("/public/v1/pool-summary", endpoint_registry.PUBLIC_AGGREGATE_CACHE_TTL_DEFAULT_SECONDS), - ("/public/v1/hashrate-series", endpoint_registry.PUBLIC_AGGREGATE_CACHE_TTL_DEFAULT_SECONDS), - ("/public/v1/miners/miner-1/workers", endpoint_registry.PUBLIC_AGGREGATE_CACHE_TTL_DEFAULT_SECONDS), - ("/public/v1/blocks", endpoint_registry.PUBLIC_CACHE_TTL_DEFAULT_SECONDS), - ("/public/v1/block-markers", endpoint_registry.PUBLIC_CACHE_TTL_DEFAULT_SECONDS), - ("/public/v1/miners/miner-1/earnings", endpoint_registry.PUBLIC_CACHE_TTL_DEFAULT_SECONDS), - ("/public/v1/mining-configuration", endpoint_registry.PUBLIC_CONFIG_CACHE_TTL_DEFAULT_SECONDS), - ): - with self.subTest(path=path): - self.assertEqual( - expected_ttl, - public_api.public_cache_policy(path).ttl_seconds, - ) - - def test_no_budget_falls_below_the_documented_floor(self) -> None: - for endpoint in self.extracted_endpoints(): - if endpoint.max_staleness_seconds is None: - continue - with self.subTest(path=endpoint.primary_path): - self.assertGreaterEqual( - endpoint.max_staleness_seconds, - endpoint_registry.MINIMUM_PUBLIC_STALE_SECONDS, - ) - - def test_only_the_artifact_route_is_exempt_from_staleness(self) -> None: - # Content-addressed and immutable: correct at any age. Stated in the - # registry rather than left implicit, so the exemption is reviewable. - exempt = [ - endpoint.primary_path - for endpoint in ENDPOINTS - if endpoint.immutable_content - ] - self.assertEqual(["/public/v1/artifacts/{sha256}"], exempt) - - def test_immutable_content_and_a_budget_are_mutually_exclusive(self) -> None: - with self.assertRaises(ValueError): - endpoint_registry.Endpoint( - paths=("/public/v1/example",), - audience=Audience.PUBLIC_READ, - disposition=Disposition.EXTRACT, - access=(LedgerAccess.READ_SLOT,), - rationale="fixture", - immutable_content=True, - max_staleness_seconds=30, - ) - - def test_an_extracted_route_cannot_omit_its_staleness_tolerance(self) -> None: - with self.assertRaises(ValueError): - endpoint_registry.Endpoint( - paths=("/public/v1/example",), - audience=Audience.PUBLIC_READ, - disposition=Disposition.EXTRACT, - access=(LedgerAccess.READ_SLOT,), - rationale="fixture", - ) - - def test_derivation_rejects_negative_cache_seconds(self) -> None: - with self.assertRaises(ValueError): - endpoint_registry.staleness_budget_seconds(cache_ttl_seconds=-1) - with self.assertRaises(ValueError): - endpoint_registry.staleness_budget_seconds( - cache_ttl_seconds=5, - underlying_cache_seconds=-1, - ) - - -class EndpointForRequestPathTests(unittest.TestCase): - """Concrete request paths must resolve to the endpoint that serves them.""" - - def test_literal_paths_win_over_templates(self) -> None: - # /public/v1/fanouts/pending is a literal list route that would also - # match /public/v1/fanouts/{fanout_txid}; dispatch() gives the literal - # precedence, so the registry lookup must too. - endpoint = endpoint_registry.endpoint_for_request_path( - "/public/v1/fanouts/pending" - ) - assert endpoint is not None - self.assertEqual("/public/v1/fanouts/pending", endpoint.primary_path) - - def test_templates_match_concrete_segments(self) -> None: - for path, expected in ( - (f"/public/v1/fanouts/{HEX64}", "/public/v1/fanouts/{fanout_txid}"), - ("/public/v1/miners/miner-1", "/public/v1/miners/{recipient_id}"), - ( - "/public/v1/miners/miner-1/workers", - "/public/v1/miners/{recipient_id}/workers", - ), - ( - f"/public/v1/blocks/{HEX64}/settlement-artifacts", - "/public/v1/blocks/{block_hash}/settlement-artifacts", - ), - ): - with self.subTest(path=path): - endpoint = endpoint_registry.endpoint_for_request_path(path) - assert endpoint is not None - self.assertEqual(expected, endpoint.primary_path) - - def test_unknown_paths_resolve_to_nothing(self) -> None: - for path in ("/public/v1/not-a-route", "/public/v1", "/nope"): - with self.subTest(path=path): - self.assertIsNone(endpoint_registry.endpoint_for_request_path(path)) - - def test_every_extracted_path_resolves(self) -> None: - for path in endpoint_registry.extracted_paths(): - with self.subTest(path=path): - self.assertIsNotNone( - endpoint_registry.endpoint_for_request_path(concrete_path(path)) - ) - - -class EndpointRegistryRoutingTests(unittest.TestCase): - """Every classified path must actually be routed today.""" - - @classmethod - def setUpClass(cls) -> None: - cls.facade = AuditHttpFacade( - _RoutingPort(), # type: ignore[arg-type] - AuditHttpConfig("127.0.0.1", 0, join_timeout_seconds=1.0), - ) - state = cls.facade.start() - assert state.bound_address is not None - cls.host, cls.port = state.bound_address - - @classmethod - def tearDownClass(cls) -> None: - cls.facade.stop() - - @staticmethod - def decode(body: bytes) -> object: - # /metrics answers Prometheus text rather than JSON; the routing - # assertion only needs "not the unknown-endpoint body". - try: - return json.loads(body or b"null") - except json.JSONDecodeError: - return body.decode(errors="replace") - - def get(self, path: str) -> tuple[int, object]: - url = f"http://{self.host}:{self.port}{path}" - try: - with urllib.request.urlopen(url, timeout=5) as response: - return response.status, self.decode(response.read()) - except urllib.error.HTTPError as error: - with error: - return error.code, self.decode(error.read()) - - def test_every_non_public_registry_path_is_routed(self) -> None: - for endpoint in ENDPOINTS: - for path in endpoint.paths: - if path.startswith("/public/v1"): - continue - with self.subTest(path=path): - status, body = self.get(concrete_path(path)) - self.assertNotEqual( - body, - UNKNOWN_ENDPOINT_BODY, - f"{path} is classified but not routed", - ) - self.assertNotEqual(status, 404, f"{path} returned 404") - - def test_metrics_is_served_as_prometheus_text(self) -> None: - url = f"http://{self.host}:{self.port}/metrics" - with urllib.request.urlopen(url, timeout=5) as response: - self.assertEqual(response.status, 200) - self.assertTrue( - response.headers["Content-Type"].startswith("text/plain") - ) - - def test_unclassified_path_still_404s(self) -> None: - status, body = self.get("/definitely-not-a-route") - self.assertEqual(status, 404) - self.assertEqual(body, UNKNOWN_ENDPOINT_BODY) - - -class PublicDispatchRoutingTests(unittest.TestCase): - """Every /public/v1 path in the registry must be routed by dispatch().""" - - def dispatch(self, path: str) -> None: - public_api.dispatch(_FakeCoordinator(), path, {}) - - def test_every_public_registry_path_is_routed(self) -> None: - for path in endpoint_registry.extracted_paths(): - if not path.startswith("/public/v1"): - continue - with self.subTest(path=path): - try: - self.dispatch(concrete_path(path)) - except _Sentinel: - # Reached a real handler, which is what "routed" means. - continue - except public_api.PublicApiError as error: - self.assertNotEqual( - error.message, - UNKNOWN_PUBLIC_MESSAGE, - f"{path} is classified but dispatch() does not route it", - ) - - def test_unknown_public_path_is_rejected(self) -> None: - with self.assertRaises(public_api.PublicApiError) as caught: - self.dispatch("/public/v1/not-a-route") - self.assertEqual(caught.exception.message, UNKNOWN_PUBLIC_MESSAGE) - - -class RegistryCoverageTests(unittest.TestCase): - """The HTTP layer's route literals must match the registry exactly. - - The guard is a frozen snapshot of the string literals in each dispatch - function, compared for exact set equality. A looser substring match was - rejected on review: a new route that merely *extends* an existing literal - (for example a new ``/public/v1/blocks/...`` sub-route) would pass - unclassified, which is precisely the drift this guard exists to catch. - """ - - # Every "/"-prefixed string constant in audit_http.do_GET, including the - # prefix fragments the dispatcher matches against ("/audit/blocks/", - # "/status", ...). Adding a route without classifying it in - # lab/prism/endpoint_registry.py fails the snapshot equality first. - EXPECTED_DO_GET_LITERALS = frozenset( - { - "/", - "/audit/block/", - "/audit/blocks/", - "/audit/carry-forward-integrity", - "/audit/commitments/", - "/audit/fanouts/", - "/audit/fanouts/pending", - "/audit/latest", - "/audit/ledger-integrity", - "/audit/share-window", - "/bundle", - "/ctv-fanout-manifest-set", - "/ctv-fanouts", - "/healthz", - "/metrics", - "/miners/", - "/owed", - "/owed-balances", - "/payouts", - "/payouts/", - "/status", - } - ) - - # Every "/"-prefixed string constant in public_api.dispatch. - EXPECTED_DISPATCH_LITERALS = frozenset( - { - "/", - "/public/v1/artifacts/", - "/public/v1/block-markers", - "/public/v1/blocks", - "/public/v1/blocks/", - "/public/v1/fanouts/", - "/public/v1/fanouts/pending", - "/public/v1/hashrate-series", - "/public/v1/leaderboard", - "/public/v1/miners/", - "/public/v1/mining-configuration", - "/public/v1/pool-summary", - "/settlement-artifacts", - } - ) - - def registry_paths(self) -> set[str]: - return {path for endpoint in ENDPOINTS for path in endpoint.paths} - - def assert_snapshot(self, literals: set[str], expected: frozenset[str]) -> None: - added = sorted(literals - expected) - removed = sorted(expected - literals) - self.assertEqual( - (added, removed), - ([], []), - "route literals in the HTTP layer changed without a matching " - "classification update in lab/prism/endpoint_registry.py " - f"(new: {added}, gone: {removed})", - ) - - def test_audit_http_route_literals_match_snapshot(self) -> None: - literals = string_literals(AUDIT_HTTP_PATH, "do_GET") - self.assertIn("/healthz", literals) - self.assert_snapshot(literals, self.EXPECTED_DO_GET_LITERALS) - - def test_public_dispatch_route_literals_match_snapshot(self) -> None: - literals = string_literals(PUBLIC_API_PATH, "dispatch") - self.assertIn("/public/v1/pool-summary", literals) - self.assert_snapshot(literals, self.EXPECTED_DISPATCH_LITERALS) - - def test_every_registry_path_is_reachable_through_a_route_literal(self) -> None: - # The snapshots pin the literals; this pins the other direction: every - # classified path must actually be matched by a literal from the - # dispatcher that owns it, so the registry cannot list a path nothing - # serves. - public = set(endpoint_registry.extracted_paths()) - for path in self.registry_paths(): - surface = ( - self.EXPECTED_DISPATCH_LITERALS - if path in public - else self.EXPECTED_DO_GET_LITERALS - ) - with self.subTest(path=path): - self.assertTrue( - any( - path == literal or path.startswith(literal) - for literal in surface - if literal != "/" - ), - f"{path} is classified but no route literal in its " - "dispatcher can match it", - ) - - def test_path_counts_are_pinned(self) -> None: - # The extraction set and the total surface are contract numbers: - # PR 2 moved exactly the 13 original /public/v1 routes, - # /public/v1/block-markers joined the extracted surface later, and the - # registry classifies all 32 request paths the HTTP surface serves. - self.assertEqual(14, len(endpoint_registry.extracted_paths())) - self.assertEqual(32, len(self.registry_paths())) - - def test_http_handler_is_get_only(self) -> None: - # The registry -- and the snapshot guard above -- only inspect do_GET - # and dispatch. If a do_POST ever appears, its routes would be - # invisible to the coverage tests, so pin the assumption the registry - # rests on: the HTTP surface answers GET only. - source = AUDIT_HTTP_PATH.read_text(encoding="utf-8") - handler_methods = sorted( - node.name - for node in ast.walk(ast.parse(source)) - if isinstance(node, ast.FunctionDef) - and node.name.startswith("do_") - and node.name.isupper() is False - and node.name != "do_GET" - ) - self.assertEqual( - [], - handler_methods, - "the endpoint registry only covers GET; new HTTP verb methods " - "need registry coverage first", - ) - - -if __name__ == "__main__": - unittest.main() diff --git a/tests/test_prism_first_job_latency.py b/tests/test_prism_first_job_latency.py deleted file mode 100644 index 4d2cdd1b6..000000000 --- a/tests/test_prism_first_job_latency.py +++ /dev/null @@ -1,916 +0,0 @@ -#!/usr/bin/env python3 -"""New-connection first-job latency regressions. - -Covers the coordinator changes that flatten the first-notify tail: initial -requests subscribing to in-flight publication-priority builds, retryable -payout-gate non-admission, content-addressed share-window serialization caching, -worker-count env knobs, and hashrate-ordered tip-refresh fanout admission. -""" - -from __future__ import annotations - -import dataclasses -import json -import threading -import time -import unittest -from concurrent.futures import Future -from decimal import Decimal -from contextlib import contextmanager -from types import SimpleNamespace -from typing import Iterator -from unittest.mock import patch - -import lab.prism.prism_coordinator as prism_coordinator -from lab.prism.prism_coordinator import ( - DEFAULT_PRISM_INITIAL_JOB_MAX_WORKERS, - DEFAULT_PRISM_JOB_BUILD_EXECUTOR_WORKERS, - PRISM_DELIVERY_PRIORITY_INITIAL, - PRISM_DELIVERY_PRIORITY_NEW_TIP, - PRISM_DELIVERY_PRIORITY_SAME_TIP, - PayoutLedgerArtifact, - _compact_share_payload, - _JobBuildCancellation, - canonical_json_sha256, -) -from lab.prism.payout_state import ( - PayoutDeliveryAdmission as _PayoutDeliveryAdmission, -) -from lab.prism.prism_coordinator import ClientState, PrismCoordinator -from tests.prism_coordinator_test_support import ( - base_template, - client, - coordinator, - install_fake_bundle_builder, -) -from tests.prism_vardiff_test_support import fake_audit_bundle_popen -from tests.test_prism_initial_job_delivery import wait_until - - -def payout_ledger_artifact( - shares: list[dict[str, object]], - *, - generation: int = 1, - payout_state_generation: int = 0, - network_difficulty: int = 2, -) -> PayoutLedgerArtifact: - return PayoutLedgerArtifact( - generation=generation, - payout_state_generation=payout_state_generation, - network_difficulty=network_difficulty, - accepted_share_count=len(shares), - shares_json=tuple(shares), - prior_balances=(), - prepared_monotonic=time.monotonic(), - snapshot_anchor_ms=1_700_000_000_000, - ) - - -def window_shares(count: int = 4) -> list[dict[str, object]]: - return [ - { - "share_seq": index + 1, - "share_id": f"share-{index + 1}", - "miner_id": "miner-a" if index % 2 == 0 else "miner-b", - "order_key": "miner-a" if index % 2 == 0 else "miner-b", - "p2mr_program_hex": "22" * 32, - "share_difficulty": 1, - "network_difficulty": 2, - "template_height": 9, - "job_id": f"job-{index + 1}", - "job_issued_at_ms": 1_000 + index, - "accepted_at_ms": 2_000 + index, - "ntime": 1_700_000_000, - } - for index in range(count) - ] - - -class InitialBuildSubscriptionTests(unittest.TestCase): - def test_initial_request_rides_priority_build_result(self) -> None: - """A deferred initial request consumes the priority build's bundle - instead of rebuilding it after the priority window closes.""" - server, rpc = coordinator() - install_fake_bundle_builder(server) - artifacts = server.store_template_artifacts(dict(rpc.template)) - assert artifacts is not None - expected = server.shared_job_bundle(artifacts, mode="ready") - with server._job_cache_lock: - server._job_bundle_cache.clear() - - priority_request = SimpleNamespace( - idle_retarget=False, - mode="ready", - equivalence_key=("priority",), - cache_key=("priority",), - cancellation=_JobBuildCancellation(timeout_seconds=30.0), - promise=Future(), - publication_critical=True, - request_source="tip_refresh", - requested_monotonic=time.monotonic(), - priority_admission_recorded=True, - ) - with server._job_build_scheduler_lock: - server._job_build_active = SimpleNamespace(request=priority_request) - server._job_build_retiring = None - server._job_build_pending = None - - constructed_sources: list[object] = [] - original_new_request = server._new_job_build_request - - def observed_new_request(*args: object, **kwargs: object) -> object: - constructed_sources.append(kwargs.get("request_source")) - return original_new_request(*args, **kwargs) # type: ignore[arg-type] - - server._new_job_build_request = observed_new_request # type: ignore[method-assign] - - results: list[object] = [] - errors: list[BaseException] = [] - - def run_initial() -> None: - try: - results.append( - server.shared_job_bundle( - artifacts, - mode="ready", - request_source="initial", - ) - ) - except BaseException as exc: # noqa: BLE001 - surfaced below - errors.append(exc) - - thread = threading.Thread(target=run_initial) - thread.start() - try: - wait_until( - lambda: ( - server.initial_job_prepared_work_counts["subscribed"] == 1 - ) - ) - self.assertTrue(thread.is_alive()) - # Production ordering: the build result resolves, then the - # scheduler sweeps the finished flight and signals the change. - priority_request.promise.set_result(expected) - with server._job_build_scheduler_lock: - server._job_build_active = None - server._job_build_priority_changed.set() - thread.join(5) - finally: - with server._job_build_scheduler_lock: - server._job_build_active = None - server._job_build_priority_changed.set() - thread.join(5) - - self.assertFalse(thread.is_alive()) - self.assertEqual(errors, []) - self.assertEqual(len(results), 1) - self.assertIs(results[0], expected) - self.assertEqual(server.initial_job_prepared_work_counts["deferred"], 1) - self.assertEqual( - server.initial_job_prepared_work_counts["subscribed"], - 1, - ) - self.assertEqual( - server.initial_job_prepared_work_counts["cache_hit"], - 1, - ) - # The whole point: no second immutable request was ever constructed. - self.assertEqual(constructed_sources, []) - - def test_cached_bundle_served_without_waiting_out_priority_window( - self, - ) -> None: - """An initial request whose bundle is already cached must not queue - behind an unrelated publication-priority reservation.""" - server, rpc = coordinator() - install_fake_bundle_builder(server) - artifacts = server.store_template_artifacts(dict(rpc.template)) - assert artifacts is not None - expected = server.shared_job_bundle(artifacts, mode="ready") - - priority_token, _requested = ( - server._begin_job_build_priority_preparation() - ) - try: - started = time.monotonic() - served = server.shared_job_bundle( - artifacts, - mode="ready", - request_source="initial", - ) - elapsed = time.monotonic() - started - finally: - server._finish_job_build_priority_preparation(priority_token) - - self.assertIs(served, expected) - # The old admission gate would have parked this request for the whole - # priority window; the probe returns without any deferred accounting. - self.assertLess(elapsed, 1.0) - self.assertEqual(server.initial_job_prepared_work_counts["deferred"], 0) - self.assertGreaterEqual( - server.initial_job_prepared_work_counts["cache_hit"], - 1, - ) - - def test_metrics_render_subscribed_prepared_work(self) -> None: - server, _rpc = coordinator() - metrics = "\n".join(server.job_build_metrics_lines()) - self.assertIn( - 'qbit_prism_initial_job_prepared_work_total{result="subscribed"} 0', - metrics, - ) - - -class RetryablePayoutGateTests(unittest.TestCase): - def test_gate_non_admission_retries_and_delivers_without_disconnect( - self, - ) -> None: - """Transient gate rejections retry with the request's own backoff and - deadline instead of retiring the connection.""" - server, _rpc = coordinator() - install_fake_bundle_builder(server) - server.prewarm_current_tip_ready_bundle() - - real_gate = server._payout_state_delivery_gate - - class FlakyGate: - def __init__(self, delegate: object, failures: int) -> None: - self.delegate = delegate - self.failures = failures - self.non_admissions = 0 - - @contextmanager - def delivery_cancelable( - self, - cancelled: object, - **kwargs: object, - ) -> Iterator[object]: - if self.failures > 0: - self.failures -= 1 - self.non_admissions += 1 - generation = int(kwargs.get("generation") or 0) - yield _PayoutDeliveryAdmission( - admitted=False, - wait_seconds=0.0, - generation=generation, - published_generation=generation, - relation="current", - ) - return - with self.delegate.delivery_cancelable( # type: ignore[attr-defined] - cancelled, - **kwargs, - ) as admission: - yield admission - - flaky = FlakyGate(real_gate, failures=2) - server._payout_state_delivery_gate = flaky # type: ignore[assignment] - disconnected: list[ClientState] = [] - server.disconnect_client = disconnected.append # type: ignore[method-assign] - - state = client(1) - state.authorization_generation = 1 - state.authorized_monotonic = time.monotonic() - state.send = lambda _payload: None # type: ignore[method-assign] - server.clients = {state} - server.request_initial_job_delivery(state) - try: - wait_until(lambda: state.active_job is not None) - finally: - server.shutdown_tip_refresh_executor() - - self.assertEqual(flaky.non_admissions, 2) - self.assertEqual(disconnected, []) - self.assertEqual(server.initial_job_failed_count, 0) - self.assertEqual(server.initial_job_cancelled_count, 0) - self.assertEqual(server.initial_job_sent_count, 1) - - def test_cancelled_request_still_terminal_on_non_admission(self) -> None: - """A cancelled request must not spin the retry loop: cancellation - stays terminal even when the gate would also have rejected.""" - server, _rpc = coordinator() - install_fake_bundle_builder(server) - server.prewarm_current_tip_ready_bundle() - artifacts = server.job_issuance_template_artifacts() - bundle = server.shared_job_bundle(artifacts, mode="ready") - - state = client(1) - state.authorization_generation = 1 - server.clients = {state} - request = prism_coordinator.PendingInitialJob( - client=state, - connection_id=state.connection_id, - authorization_generation=1, - worker=state.worker, - requested_monotonic=time.monotonic(), - deadline_monotonic=None, - ) - request.cancelled.set() - - self.assertIs( - server._deliver_initial_bundle(request, artifacts, bundle), - False, - ) - - -class ShareWindowSerializationCacheTests(unittest.TestCase): - def test_generation_retags_reuse_and_content_change_invalidates(self) -> None: - server, _rpc = coordinator() - shares = window_shares() - artifact = payout_ledger_artifact(shares) - - first = server._share_window_serialization_for_artifact( - artifact, - shares, - ) - self.assertEqual( - first.share_snapshot_sha256, - canonical_json_sha256(shares), - ) - self.assertIs( - server._share_window_serialization_for_artifact(artifact, shares), - first, - ) - - tail_chunks = first.compact_tail_chunks(shares) - expected_identities, expected_compact = _compact_share_payload(shares) - # The tail completes a JSON object whose only other member is the - # (empty) head, so it can be parsed back by prefixing "{". - tail = json.loads("{" + "".join(tail_chunks)[1:]) - self.assertEqual( - tail["compact_share_identities"], - [list(identity) for identity in expected_identities], - ) - self.assertEqual( - tail["compact_shares"], - [list(entry) for entry in expected_compact], - ) - # The tail encodes once per canonical window and is reused verbatim. - self.assertIs(first.compact_tail_chunks(shares), tail_chunks) - - artifact_bump = dataclasses.replace(artifact, generation=2) - rebuilt = server._share_window_serialization_for_artifact( - artifact_bump, - shares, - ) - self.assertIs(rebuilt, first) - - payout_bump = dataclasses.replace( - artifact, - generation=3, - payout_state_generation=artifact.payout_state_generation + 1, - ) - self.assertIs( - server._share_window_serialization_for_artifact( - payout_bump, - shares, - ), - rebuilt, - ) - - changed_shares = [*shares, window_shares(5)[-1]] - changed = payout_ledger_artifact( - changed_shares, - generation=4, - payout_state_generation=1, - ) - self.assertIsNot( - server._share_window_serialization_for_artifact( - changed, - changed_shares, - ), - first, - ) - - def test_concurrent_builders_share_one_digest_computation(self) -> None: - server, _rpc = coordinator() - shares = window_shares() - artifact = payout_ledger_artifact(shares) - digest_calls = 0 - original_sha = prism_coordinator.canonical_json_sha256 - - def counting_sha(value: object) -> str: - nonlocal digest_calls - digest_calls += 1 - return original_sha(value) - - barrier = threading.Barrier(2) - results: list[object] = [] - results_lock = threading.Lock() - - def compute() -> None: - barrier.wait() - serialization = server._share_window_serialization_for_artifact( - artifact, - shares, - ) - with results_lock: - results.append(serialization) - - with patch.object( - prism_coordinator, - "canonical_json_sha256", - counting_sha, - ): - threads = [threading.Thread(target=compute) for _ in range(2)] - for thread in threads: - thread.start() - for thread in threads: - thread.join(5) - - self.assertEqual(len(results), 2) - self.assertIs(results[0], results[1]) - self.assertEqual(digest_calls, 1) - - def test_precomposed_builder_payload_matches_inline_encoding(self) -> None: - """The composed pipe write must be byte-for-byte JSON-equivalent to - the historical single json.dump of the same payload.""" - server, _rpc = coordinator() - server.signing_seed_hex = "42" * 32 - server.ledger_attestation_signing_seed_hex = "43" * 32 - shares = window_shares() - artifact = payout_ledger_artifact(shares) - serialization = server._share_window_serialization_for_artifact( - artifact, - shares, - ) - summary = { - "found_block": {"block_height": 10}, - "signed_coinbase_manifest": { - "manifest": {"coinbase_tx_hex": "c0ffee"}, - }, - } - build_kwargs = dict( - shares=shares, - found_block={ - "block_height": 10, - "coinbase_value_sats": 50_00000000, - "network_difficulty": 2, - "anchor_job_issued_at_ms": 1_700_000_000_000, - }, - prior_balances=[{"miner_id": "miner-a", "balance_sats": 1}], - coinbase_script_sig_suffix_hex="00", - witness_merkle_leaves_hex=["aa" * 32], - summary_only=True, - payout_policy={"policy": "test"}, - ctv_settlement={"kind": "test"}, - ) - - inline_captured: dict[str, object] = {} - with patch( - "lab.prism.prism_coordinator.subprocess.Popen", - fake_audit_bundle_popen( - inline_captured, - output_text=json.dumps(summary, separators=(",", ":")), - ), - ): - server.build_audit_bundle(**build_kwargs) - - cached_captured: dict[str, object] = {} - with patch( - "lab.prism.prism_coordinator.subprocess.Popen", - fake_audit_bundle_popen( - cached_captured, - output_text=json.dumps(summary, separators=(",", ":")), - ), - ): - server.build_audit_bundle( - **build_kwargs, - share_serialization=serialization, - ) - - self.assertIn("compact_shares", inline_captured["payload"]) - self.assertEqual( - cached_captured["payload"], - inline_captured["payload"], - ) - - def test_share_count_mismatch_falls_back_to_inline_encoding(self) -> None: - server, _rpc = coordinator() - server.signing_seed_hex = "42" * 32 - server.ledger_attestation_signing_seed_hex = "43" * 32 - shares = window_shares() - stale = prism_coordinator._ShareWindowSerialization( - key=(0, 1, 32), - share_count=len(shares) + 1, - share_snapshot_sha256="00" * 32, - ) - summary = { - "found_block": {"block_height": 10}, - "signed_coinbase_manifest": { - "manifest": {"coinbase_tx_hex": "c0ffee"}, - }, - } - captured: dict[str, object] = {} - with patch( - "lab.prism.prism_coordinator.subprocess.Popen", - fake_audit_bundle_popen( - captured, - output_text=json.dumps(summary, separators=(",", ":")), - ), - ): - server.build_audit_bundle( - shares=shares, - found_block={ - "block_height": 10, - "coinbase_value_sats": 50_00000000, - }, - prior_balances=[], - coinbase_script_sig_suffix_hex="00", - summary_only=True, - payout_policy={"policy": "test"}, - ctv_settlement={"kind": "test"}, - share_serialization=stale, - ) - - payload = captured["payload"] - assert isinstance(payload, dict) - expected_identities, expected_compact = _compact_share_payload(shares) - self.assertEqual( - payload["compact_share_identities"], - [list(identity) for identity in expected_identities], - ) - self.assertEqual( - payload["compact_shares"], - [list(entry) for entry in expected_compact], - ) - - -class WorkerKnobTests(unittest.TestCase): - def test_defaults_unchanged(self) -> None: - self.assertEqual(DEFAULT_PRISM_INITIAL_JOB_MAX_WORKERS, 4) - self.assertEqual(DEFAULT_PRISM_JOB_BUILD_EXECUTOR_WORKERS, 2) - - def test_knobs_are_env_wired(self) -> None: - with open(prism_coordinator.__file__, encoding="utf-8") as module_source: - source = module_source.read() - self.assertRegex( - source, - r"resolve_initial_job_max_workers\(\s*" - r'env_optional_positive_int\("PRISM_INITIAL_JOB_MAX_WORKERS"\)', - ) - self.assertRegex( - source, - r'env_positive_int\(\s*"PRISM_JOB_BUILD_EXECUTOR_WORKERS",\s*' - r"DEFAULT_PRISM_JOB_BUILD_EXECUTOR_WORKERS,", - ) - - def test_metrics_render_configured_worker_knobs(self) -> None: - server, _rpc = coordinator() - server.initial_job_max_workers = 7 - server.job_build_executor_workers = 1 - initial_metrics = "\n".join(server.initial_delivery_metrics_lines()) - build_metrics = "\n".join(server.job_build_metrics_lines()) - self.assertIn( - "qbit_prism_initial_job_delivery_configured_workers 7", - initial_metrics, - ) - self.assertIn( - "qbit_prism_job_build_configured_workers 1", - build_metrics, - ) - - def test_job_build_executor_honors_configured_workers(self) -> None: - server, _rpc = coordinator() - server.job_build_executor_workers = 1 - try: - with server._job_build_scheduler_lock: - executor = server._job_build_executor_locked() - self.assertEqual(executor._max_workers, 1) - finally: - server.shutdown_job_build_executor() - - def test_initial_job_workers_reject_widths_past_pending_bound(self) -> None: - validate = prism_coordinator.validate_initial_job_max_workers - self.assertEqual(validate(4, 128), 4) - self.assertEqual(validate(128, 128), 128) - with self.assertRaisesRegex(SystemExit, "cannot exceed"): - validate(129, 128) - with self.assertRaisesRegex(SystemExit, "positive"): - validate(0, 128) - - def test_implicit_initial_worker_default_caps_to_pending_bound(self) -> None: - resolve = prism_coordinator.resolve_initial_job_max_workers - # Pre-knob deployments with tiny pending bounds stay bootable. - self.assertEqual(resolve(None, 2), 2) - self.assertEqual( - resolve(None, 128), - DEFAULT_PRISM_INITIAL_JOB_MAX_WORKERS, - ) - self.assertEqual(resolve(2, 2), 2) - with self.assertRaisesRegex(SystemExit, "cannot exceed"): - resolve(4, 2) - - def test_job_build_executor_workers_rejects_unusable_widths(self) -> None: - validate = prism_coordinator.validate_job_build_executor_workers - self.assertEqual(validate(1), 1) - self.assertEqual( - validate(DEFAULT_PRISM_JOB_BUILD_EXECUTOR_WORKERS), - DEFAULT_PRISM_JOB_BUILD_EXECUTOR_WORKERS, - ) - with self.assertRaisesRegex(SystemExit, "cannot exceed"): - validate(DEFAULT_PRISM_JOB_BUILD_EXECUTOR_WORKERS + 1) - with self.assertRaisesRegex(SystemExit, "positive"): - validate(0) - - def test_initial_job_executor_honors_configured_workers(self) -> None: - server, _rpc = coordinator() - server._ensure_initial_job_state() - server.initial_job_max_workers = 3 - try: - executor = server.initial_job_executor() - self.assertEqual(executor.max_workers, 3) - finally: - server.shutdown_initial_job_executor() - - -class FanoutOrderingTests(unittest.TestCase): - def test_fanout_wave_duration_histogram_is_observed(self) -> None: - # One completed prepared wave records exactly one wall-clock span - # (first task submission to last successful delivery). - server, _rpc = coordinator() - install_fake_bundle_builder(server) - states = [client(1), client(2)] - for state in states: - state.send = lambda _payload: None # type: ignore[method-assign] - server.clients = set(states) - - snapshot = server.fetch_qbit_tip_template_snapshot() - server.observe_tip_first_seen(snapshot.bestblockhash) - server.pool_readiness_latched() - server.tip_template_snapshot = snapshot - bundle = server.prepare_tip_refresh_bundle(snapshot) - try: - sent, first_delivery, last_delivery, failed = ( - server._fanout_prepared_tip_refresh( - states, - bundle, - snapshot, - heartbeat_name="qbit_blockpoll", - ) - ) - finally: - server.shutdown_tip_refresh_executor() - - self.assertEqual(sent, 2) - self.assertEqual(failed, 0) - self.assertIsNotNone(first_delivery) - self.assertIsNotNone(last_delivery) - metrics = server.metrics_payload() - self.assertIn( - "qbit_prism_tip_refresh_fanout_wave_seconds_count 1", - metrics, - ) - - def test_send_job_update_precomposed_bytes_match_full_serialization( - self, - ) -> None: - # The precomposed notify line must be byte-identical to serializing - # the payload dicts, or protocol bytes would silently diverge from - # what monkeypatched-send tests observe. - server, _rpc = coordinator() - install_fake_bundle_builder(server) - state = client(1) - - class RecorderSock: - def __init__(self) -> None: - self.chunks: list[bytes] = [] - - def sendall(self, data: bytes) -> None: - self.chunks.append(data) - - sock = RecorderSock() - state.sock = sock # type: ignore[assignment] - - snapshot = server.fetch_qbit_tip_template_snapshot() - server.observe_tip_first_seen(snapshot.bestblockhash) - server.pool_readiness_latched() - server.tip_template_snapshot = snapshot - try: - bundle = server.prepare_tip_refresh_bundle(snapshot) - finally: - server.shutdown_tip_refresh_executor() - job = bundle.base_job - self.assertIsNotNone(job.notify_shared_params_json) - - server.send_job_update(state, job) - - expected = ( - json.dumps(server.difficulty_payload(job.share_difficulty)).encode() - + b"\n" - + json.dumps(PrismCoordinator.job_payload(job)).encode() - + b"\n" - ) - self.assertEqual(b"".join(sock.chunks), expected) - - def test_priority_constants_put_first_jobs_ahead_of_the_wave(self) -> None: - self.assertLess( - PRISM_DELIVERY_PRIORITY_INITIAL, - PRISM_DELIVERY_PRIORITY_NEW_TIP, - ) - self.assertLess( - PRISM_DELIVERY_PRIORITY_NEW_TIP, - PRISM_DELIVERY_PRIORITY_SAME_TIP, - ) - - def test_fanout_wave_serves_descending_hashrate_first(self) -> None: - server, rpc = coordinator() - install_fake_bundle_builder(server) - server.tip_refresh_max_workers = 1 - slow_stale = client(1) - newcomer = client(2) - fast_stale = client(3) - states = [slow_stale, newcomer, fast_stale] - notified: list[int] = [] - - def sender(state: ClientState) -> object: - def send(payload: dict[str, object]) -> None: - if payload["method"] == "mining.notify": - notified.append(state.connection_id) - - return send - - for state in states: - state.send = sender(state) # type: ignore[method-assign] - server.clients = set(states) - - snapshot = server.fetch_qbit_tip_template_snapshot() - server.observe_tip_first_seen(snapshot.bestblockhash) - server.pool_readiness_latched() - server.tip_template_snapshot = snapshot - bundle = server.prepare_tip_refresh_bundle(snapshot) - try: - server._fanout_prepared_tip_refresh( - states, - bundle, - snapshot, - heartbeat_name="qbit_blockpoll", - ) - self.assertEqual(sorted(notified), [1, 2, 3]) - - # A job-less newcomer burns zero hashrate while stale clients burn - # their full rate on old-tip work, so the next wave must serve the - # fastest stale client first and the newcomer last -- even when - # the newcomer's configured difficulty is the highest in the wave, - # it keeps only its initial-delivery queue priority. The fast - # client ranks by its vardiff estimate; the slow one exercises the - # share_difficulty fallback used before an estimate exists. - newcomer.active_job = None - newcomer.active_job_ids = set() - newcomer.share_difficulty = Decimal("64") - slow_stale.share_difficulty = Decimal("8") - fast_stale.vardiff_difficulty_estimate = Decimal("32") - notified.clear() - rpc.tip = "22" * 32 - rpc.template = base_template(height=11, prevhash="22" * 32) - replacement = server.fetch_qbit_tip_template_snapshot() - server.observe_tip_first_seen(replacement.bestblockhash) - server.tip_template_snapshot = replacement - replacement_bundle = server.prepare_tip_refresh_bundle(replacement) - - submissions: list[tuple[int, int]] = [] - - def recording_submit( - executor: object, - function: object, - *args: object, - priority: int, - ) -> Future[object]: - target = args[0] - assert isinstance(target, ClientState) - submissions.append((target.connection_id, priority)) - return PrismCoordinator._submit_delivery_task( - executor, - function, # type: ignore[arg-type] - *args, - priority=priority, - ) - - server._submit_delivery_task = recording_submit # type: ignore[method-assign] - server._fanout_prepared_tip_refresh( - [slow_stale, newcomer, fast_stale], - replacement_bundle, - replacement, - heartbeat_name="qbit_blockpoll", - ) - finally: - server.shutdown_tip_refresh_executor() - - self.assertEqual(notified, [3, 1, 2]) - self.assertEqual( - [connection_id for connection_id, _priority in submissions], - [3, 1, 2], - ) - self.assertEqual( - submissions[2], - (2, PRISM_DELIVERY_PRIORITY_INITIAL), - ) - self.assertEqual( - { - priority - for connection_id, priority in submissions - if connection_id != 2 - }, - {PRISM_DELIVERY_PRIORITY_NEW_TIP}, - ) - - -class SimulatedChurnLatencyTests(unittest.TestCase): - def test_p99_first_notify_stays_sub_second_under_tip_churn(self) -> None: - """Clients joining through sustained tip churn get first work fast. - - Scaled-down version of the production complaint: ~45-150 clients - churning at a ~47s tip cadence saw multi-second first notifies - because every publication-priority build deferred, cancelled, and - outcompeted initial work. Cadence and build cost are scaled ~1:100; - the sub-second p99 bound is intentionally generous for slow CI. - """ - server, rpc = coordinator() - install_fake_bundle_builder(server) - original_build = server.build_shared_job_bundle - - def costly_build(*args: object, **kwargs: object) -> object: - time.sleep(0.02) - return original_build(*args, **kwargs) # type: ignore[arg-type] - - server.build_shared_job_bundle = costly_build # type: ignore[method-assign] - disconnected: list[ClientState] = [] - original_disconnect = server.disconnect_client - - def record_disconnect(state: ClientState) -> None: - disconnected.append(state) - original_disconnect(state) - - server.disconnect_client = record_disconnect # type: ignore[method-assign] - server.clients = set() - - stop_churn = threading.Event() - poll_failures: list[BaseException] = [] - - def churn_tips() -> None: - height = 11 - while not stop_churn.is_set(): - if stop_churn.wait(0.3): - return - prevhash = f"{height:02x}" * 32 - rpc.tip = prevhash - rpc.template = base_template(height=height, prevhash=prevhash) - try: - server.poll_qbit_tip_template_once() - except BaseException as exc: # noqa: BLE001 - surfaced below - poll_failures.append(exc) - return - height += 1 - - first_notify: dict[int, float] = {} - notify_lock = threading.Lock() - - def notify_stamper(state: ClientState) -> object: - def send(payload: dict[str, object]) -> None: - if payload.get("method") == "mining.notify": - with notify_lock: - first_notify.setdefault( - state.connection_id, - time.monotonic(), - ) - - return send - - churn_thread = threading.Thread(target=churn_tips) - churn_thread.start() - requested_at: dict[int, float] = {} - try: - server.poll_qbit_tip_template_once() - for index in range(30): - state = client(100 + index) - state.authorization_generation = 1 - state.authorized_monotonic = time.monotonic() - state.send = notify_stamper(state) # type: ignore[method-assign] - with server.lock: - server.clients.add(state) - requested_at[state.connection_id] = time.monotonic() - self.assertTrue(server.schedule_initial_job(state)) - time.sleep(0.1) - wait_until( - lambda: len(first_notify) == len(requested_at), - timeout=30.0, - ) - finally: - stop_churn.set() - churn_thread.join(10) - server.shutdown_tip_refresh_executor() - - self.assertEqual(poll_failures, []) - self.assertEqual(disconnected, []) - latencies = [ - first_notify[connection_id] - requested - for connection_id, requested in requested_at.items() - ] - self.assertEqual(len(latencies), 30) - ranked = sorted(latencies) - p99 = ranked[max(0, int(len(ranked) * 0.99) - 1)] - self.assertLess(p99, 1.0) - - -if __name__ == "__main__": - unittest.main() diff --git a/tests/test_prism_ha_qualification.py b/tests/test_prism_ha_qualification.py new file mode 100644 index 000000000..05fcc9635 --- /dev/null +++ b/tests/test_prism_ha_qualification.py @@ -0,0 +1,272 @@ +import copy +import importlib.util +import json +import os +from pathlib import Path +import signal +import subprocess +import sys +import tempfile +import time +from types import SimpleNamespace +import unittest +from unittest.mock import Mock, patch + + +spec = importlib.util.spec_from_file_location( + "ha_qualification", Path(__file__).resolve().parents[1] / "scripts/prism_ha_qualification.py") +qualification = importlib.util.module_from_spec(spec) +spec.loader.exec_module(qualification) + + +def rendered_fixture(): + env = qualification.fixture_environment() + services = {} + for index, name in enumerate(qualification.FRONTENDS, 1): + services[name] = { + "environment": { + "PRISM_INSTANCE_ID": env[f"PRISM_HA_INSTANCE_ID_{index}"], + "QBIT_RPC_URL": env[f"PRISM_HA_RPC_URL_{index}"], + "PRISM_AUDIT_PORT": env["PRISM_HA_AUDIT_PORT"], + "PRISM_AUDIT_BIND": env["PRISM_HA_AUDIT_BIND"], + "PRISM_DATABASE_URL": env["PRISM_DATABASE_URL"], + "PRISM_STRATUM_PORT": "3340", + "PRISM_STRATUM_HIGHDIFF_PORT": env["PRISM_STRATUM_HIGHDIFF_PORT"], + }, + "ports": [ + {"host_ip": "127.0.0.1", "target": 3340, "published": str(18440 if index == 1 else 18443)}, + {"host_ip": "127.0.0.1", "target": 18441, "published": str(18441 if index == 1 else 18444)}, + {"host_ip": "127.0.0.1", "target": 18446, "published": str(18442 if index == 1 else 18445)}, + ], + } + return {"services": services} + + +class QualificationTests(unittest.TestCase): + def test_inherited_secrets_and_external_docker_target_are_not_interpolated(self): + with patch.dict(os.environ, {"PRISM_DATABASE_URL": "secret", "QBIT_RPC_URL": "secret", + "DOCKER_HOST": "tcp://external.invalid:2375"}): + env = qualification.fixture_environment() + self.assertNotIn("secret", json.dumps(env)) + self.assertNotIn("DOCKER_HOST", env) + self.assertNotIn("QBIT_RPC_URL", env) + + def test_projection_omits_all_non_allowlisted_environment(self): + document = rendered_fixture() + document["services"][qualification.FRONTENDS[0]]["environment"]["SIGNING_SECRET"] = "secret" + result = qualification.check_render(document, qualification.fixture_environment()) + self.assertNotIn("secret", json.dumps(result)) + self.assertNotIn("postgresql", json.dumps(result)) + self.assertEqual(len(result), 2) + + def test_wrong_writer_instance_rpc_health_and_port_fail(self): + document = rendered_fixture() + for key in ("PRISM_INSTANCE_ID", "QBIT_RPC_URL", "PRISM_DATABASE_URL", "PRISM_AUDIT_PORT", "PRISM_AUDIT_BIND", "PRISM_STRATUM_HIGHDIFF_PORT"): + with self.subTest(key=key): + bad = copy.deepcopy(document) + bad["services"][qualification.FRONTENDS[1]]["environment"][key] = "secret" + with self.assertRaisesRegex(ValueError, "differs from fixture") as raised: + qualification.check_render(bad, qualification.fixture_environment()) + self.assertNotIn("secret", str(raised.exception)) + document["services"][qualification.FRONTENDS[1]]["ports"][0]["host_ip"] = "0.0.0.0" + with self.assertRaisesRegex(ValueError, "loopback ports"): + qualification.check_render(document, qualification.fixture_environment()) + + def test_high_difficulty_and_unexpected_exposed_ports_fail(self): + document = rendered_fixture() + document["services"][qualification.FRONTENDS[1]]["ports"][2]["host_ip"] = "0.0.0.0" + with self.assertRaisesRegex(ValueError, "loopback ports"): + qualification.check_render(document, qualification.fixture_environment()) + document = rendered_fixture() + document["services"][qualification.FRONTENDS[0]]["ports"].append( + {"host_ip": "0.0.0.0", "target": 5432, "published": "5432"}) + with self.assertRaisesRegex(ValueError, "loopback ports"): + qualification.check_render(document, qualification.fixture_environment()) + + def test_render_never_prints_failure_stdout_or_stderr(self): + failure = subprocess.CompletedProcess([], 1, "secret config", "secret diagnostics") + with patch.object(qualification.subprocess, "run", return_value=failure) as call: + with self.assertRaisesRegex(RuntimeError, "withheld") as raised: + qualification.render() + self.assertNotIn("secret", str(raised.exception)) + self.assertIn(os.devnull, call.call_args.args[0]) + + def test_cleanup_only_stops_the_exact_owned_data_directory(self): + with tempfile.TemporaryDirectory() as parent: + root = Path(parent) / "owned" + data = root / "prism-load-unique" / "primary" + data.mkdir(parents=True) + (data / "PG_VERSION").write_text("16") + with patch.object(qualification.subprocess, "run", side_effect=[ + subprocess.CompletedProcess([], 0), subprocess.CompletedProcess([], 0), + subprocess.CompletedProcess([], 3), + ]) as call: + result = qualification.cleanup_owned(root, Path("/fixture/bin")) + self.assertTrue(result["owned_parent_removed"]) + self.assertEqual(len(call.call_args_list), 3) + for args in call.call_args_list: + self.assertEqual(args.args[0][:3], ["/fixture/bin/pg_ctl", "-D", str(data)]) + + def test_unknown_exit_keeps_owned_directory(self): + with tempfile.TemporaryDirectory() as parent: + root = Path(parent) / "owned" + data = root / "prism-load-unique" / "primary" + data.mkdir(parents=True) + (data / "PG_VERSION").write_text("16") + with patch.object(qualification.subprocess, "run", return_value=subprocess.CompletedProcess([], 1)): + with self.assertRaisesRegex(RuntimeError, "unverified"): + qualification.cleanup_owned(root, Path("/fixture/bin")) + self.assertTrue(data.exists()) + + def test_symlink_and_unknown_entries_are_never_cleaned(self): + with tempfile.TemporaryDirectory() as parent: + root = Path(parent) / "owned" + root.mkdir() + outside = Path(parent) / "outside" + outside.mkdir() + (root / "prism-load-other").symlink_to(outside) + with patch.object(qualification.subprocess, "run") as call: + with self.assertRaisesRegex(RuntimeError, "unexpected entry"): + qualification.cleanup_owned(root, Path("/fixture/bin")) + call.assert_not_called() + self.assertTrue(outside.exists()) + + def test_partial_initialization_has_no_running_postmaster(self): + with tempfile.TemporaryDirectory() as parent: + root = Path(parent) / "owned" + data = root / "prism-load-unique" / "primary" + data.mkdir(parents=True) + with patch.object(qualification.subprocess, "run") as call: + qualification.cleanup_owned(root, Path("/fixture/bin")) + call.assert_not_called() + self.assertFalse(root.exists()) + + def test_wrapper_records_ownership_before_start_and_sets_explicit_locale(self): + with tempfile.TemporaryDirectory() as parent: + base = Path(parent) + binary = base / "pg_ctl" + binary.write_text("fixture") + binary.chmod(0o700) + owned = base / "owned" + owned.mkdir() + args = SimpleNamespace(example_bin=binary, runtime_tests=binary, ack_tests=binary, + pg_bin_dir=base, out=base / "evidence") + + def launch(command, **kwargs): + self.assertTrue((args.out / "resource-census.json").is_file()) + self.assertEqual(kwargs["env"]["LC_ALL"], "C") + self.assertEqual(kwargs["env"]["TMPDIR"], str(owned)) + self.assertNotIn("PRISM_DATABASE_URL", kwargs["env"]) + self.assertTrue(kwargs["start_new_session"]) + functional = args.out / "functional" + functional.mkdir() + (functional / "ha-functional.json").write_text(json.dumps({ + "result": "passed", "cleanup": {"complete": True}, + })) + return SimpleNamespace(pid=42, returncode=0, communicate=lambda **kw: ("", ""), poll=lambda: 0) + + with patch.object(qualification.tempfile, "mkdtemp", return_value=str(owned)), \ + patch.object(qualification.subprocess, "Popen", side_effect=launch), \ + patch.object(qualification.subprocess, "run", side_effect=[ + subprocess.CompletedProcess([], 0, "a" * 40), subprocess.CompletedProcess([], 0, ""), + ]): + result = qualification.run(args) + self.assertEqual(result["functional"], "passed") + census = json.loads((args.out / "resource-census.json").read_text()) + self.assertEqual(census["process_exit"], 0) + self.assertTrue(census["cleanup"]["owned_parent_removed"]) + self.assertFalse(owned.exists()) + + def test_failed_process_start_still_cleans_and_records_unknown_exit(self): + with tempfile.TemporaryDirectory() as parent: + base = Path(parent) + binary = base / "pg_ctl" + binary.write_text("fixture") + binary.chmod(0o700) + owned = base / "owned" + owned.mkdir() + args = SimpleNamespace(example_bin=binary, runtime_tests=binary, ack_tests=binary, + pg_bin_dir=base, out=base / "evidence") + with patch.object(qualification.tempfile, "mkdtemp", return_value=str(owned)), \ + patch.object(qualification.subprocess, "Popen", side_effect=OSError("start failed")), \ + patch.object(qualification.subprocess, "run", side_effect=[ + subprocess.CompletedProcess([], 0, "a" * 40), subprocess.CompletedProcess([], 0, ""), + ]): + with self.assertRaisesRegex(OSError, "start failed"): + qualification.run(args) + census = json.loads((args.out / "resource-census.json").read_text()) + self.assertIsNone(census["process_exit"]) + self.assertTrue(census["cleanup"]["owned_parent_removed"]) + self.assertFalse(owned.exists()) + + def test_wrapper_sigterm_stops_owned_child_and_records_cleanup(self): + with tempfile.TemporaryDirectory() as parent: + base = Path(parent) + binary = base / "pg_ctl" + binary.write_text( + f"#!{sys.executable}\n" + "import pathlib, sys, time\n" + "out = pathlib.Path(sys.argv[sys.argv.index('--out') + 1])\n" + "out.mkdir()\n" + "(out / 'ready').touch()\n" + "time.sleep(60)\n") + binary.chmod(0o700) + out = base / "evidence" + process = subprocess.Popen([ + sys.executable, str(qualification.ROOT / "scripts/prism_ha_qualification.py"), + "run", "--example-bin", str(binary), "--runtime-tests", str(binary), + "--ack-tests", str(binary), "--pg-bin-dir", str(base), "--out", str(out), + ], stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True) + try: + deadline = time.monotonic() + 10 + while not (out / "functional" / "ready").exists(): + self.assertIsNone(process.poll(), "fixture wrapper exited before readiness") + self.assertLess(time.monotonic(), deadline, "fixture readiness timed out") + time.sleep(0.01) + process.send_signal(signal.SIGTERM) + process.communicate(timeout=10) + self.assertNotEqual(process.returncode, 0) + census = json.loads((out / "resource-census.json").read_text()) + self.assertEqual(census["process_exit"], -signal.SIGTERM) + self.assertIn("interrupted", census["failure"]) + self.assertTrue(census["cleanup"]["owned_parent_removed"]) + self.assertFalse(Path(census["owned_parent"]).exists()) + with self.assertRaises(ProcessLookupError): + os.kill(census["process_pid"], 0) + finally: + if process.poll() is None: + process.terminate() + process.communicate(timeout=10) + + def test_diagnostic_copy_failure_preserves_original_interruption(self): + with tempfile.TemporaryDirectory() as parent: + base = Path(parent) + binary = base / "pg_ctl" + binary.write_text("fixture") + binary.chmod(0o700) + owned = base / "owned" + cluster = owned / "prism-load-fixture" + cluster.mkdir(parents=True) + (cluster / "primary.log").write_text("fixture diagnostics") + args = SimpleNamespace(example_bin=binary, runtime_tests=binary, ack_tests=binary, + pg_bin_dir=base, out=base / "evidence") + process = SimpleNamespace(pid=42, returncode=-signal.SIGTERM, poll=lambda: -signal.SIGTERM, + communicate=Mock(side_effect=[KeyboardInterrupt(), ("", "")])) + with patch.object(qualification.tempfile, "mkdtemp", return_value=str(owned)), \ + patch.object(qualification.subprocess, "Popen", return_value=process), \ + patch.object(qualification.os, "killpg") as kill, \ + patch.object(qualification.shutil, "copyfile", side_effect=OSError("copy failed")), \ + patch.object(qualification.subprocess, "run", side_effect=[ + subprocess.CompletedProcess([], 0, "a" * 40), subprocess.CompletedProcess([], 0, ""), + ]): + with self.assertRaisesRegex(RuntimeError, "functional run failed"): + qualification.run(args) + kill.assert_called_once_with(42, signal.SIGTERM) + census = json.loads((args.out / "resource-census.json").read_text()) + self.assertIn("interrupted", census["failure"]) + self.assertIn("diagnostics", census["failure"]) + self.assertTrue(census["cleanup"]["owned_parent_removed"]) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_prism_ha_readiness_probe.py b/tests/test_prism_ha_readiness_probe.py new file mode 100644 index 000000000..aefda6d60 --- /dev/null +++ b/tests/test_prism_ha_readiness_probe.py @@ -0,0 +1,78 @@ +import unittest +import json +import math +import subprocess +import sys + +from scripts.prism_ha_readiness_probe import ProbeConfig, ReadinessProbe, run, run_timeline + + +OK = {"status": 200, "ok": True, "schema": "qbit.prism.audit-health.v1"} +BAD = {"status": 503, "ok": False} + + +class ReadinessProbeTests(unittest.TestCase): + def test_initial_unknown_and_two_success_rise(self): + self.assertEqual(run([(OK, 0), (OK, 0)]), ["unknown", "up"]) + + def test_contract_rejects_false_timeout_unreachable_and_malformed(self): + p = ReadinessProbe() + for response, elapsed in [(BAD, 0), (OK, 1.01), (None, 0), ({"status": 200}, 0)]: + self.assertEqual(p.observe(response, elapsed), "unknown") + self.assertEqual(p.failures, 4) + for elapsed in (-1.0, math.nan, math.inf): + self.assertNotEqual(p.observe(OK, elapsed), "up") + + def test_six_failures_eject_and_two_successes_recover(self): + p = ReadinessProbe() + for _ in range(2): + p.observe(OK) + self.assertEqual(p.state, "up") + self.assertEqual(run([(BAD, 0)] * 5, ProbeConfig()), ["unknown"] * 5) # initial state is explicit + for _ in range(6): + p.observe(BAD) + self.assertEqual(p.state, "down") + p.observe(OK) + self.assertEqual(p.state, "down") + p.observe(OK) + self.assertEqual(p.state, "up") + + def test_timing_bound_and_monotonic_schedule(self): + events = [(float(i * 2), BAD, 1.0) for i in range(6)] + self.assertEqual(run_timeline(events), ["unknown"] * 5 + ["down"]) + self.assertLessEqual(events[-1][0] + 1.0, 13.0) + with self.assertRaises(ValueError): + run_timeline([(0.0, BAD, 0), (1.0, BAD, 0)]) + with self.assertRaises(ValueError): + run_timeline([(0.0, BAD, 3.0), (2.0, BAD, 0)]) + with self.assertRaises(ValueError): + run_timeline([(0.0, BAD, math.nan)]) + + def test_transient_rebuild_does_not_eject(self): + p = ReadinessProbe() + p.observe(OK); p.observe(OK) + for _ in range(5): + p.observe(BAD) + self.assertEqual(p.state, "up") + + def test_nondefaults_and_invalid_config(self): + config = ProbeConfig(interval_s=0.5, timeout_s=0.25, fall=3, rise=3) + self.assertEqual(run_timeline([(i * 0.5, BAD, 0) for i in range(3)], config), ["unknown", "unknown", "down"]) + self.assertEqual(run([(BAD, 0)] * 3 + [(OK, 0)] * 3, config), ["unknown", "unknown", "down", "down", "down", "up"]) + for kwargs in ({"interval_s": 0}, {"timeout_s": 0}, {"fall": 0}, {"rise": 0}, {"timeout_s": math.nan}, {"interval_s": math.inf}): + with self.assertRaises(ValueError): + ProbeConfig(**kwargs) + + def test_cli_preserves_state_across_multiple_json_lines(self): + rows = "\n".join(json.dumps({"response": OK, "elapsed_s": value}) for value in (0.0, 1.0)) + "\n" + result = subprocess.run([sys.executable, "scripts/prism_ha_readiness_probe.py"], input=rows, text=True, capture_output=True, check=True) + self.assertEqual([json.loads(line)["state"] for line in result.stdout.splitlines()], ["unknown", "up"]) + + def test_cli_malformed_duration_is_a_failed_probe(self): + row = json.dumps({"response": OK, "elapsed_s": None}) + "\n" + result = subprocess.run([sys.executable, "scripts/prism_ha_readiness_probe.py"], input=row, text=True, capture_output=True, check=True) + self.assertEqual(json.loads(result.stdout)["state"], "unknown") + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_prism_heap_census.py b/tests/test_prism_heap_census.py deleted file mode 100644 index 08ae40afa..000000000 --- a/tests/test_prism_heap_census.py +++ /dev/null @@ -1,1155 +0,0 @@ -#!/usr/bin/env python3 -"""Issue #226 part 2: the bounded heap census, malloc_trim, allocator pins. - -Every test here pins a safety property of an instrument that walks a -multi-gigabyte heap or takes glibc's arena locks: that it is off unless an -operator switches it on, that it is bounded when on, that no request path -can reach it, that the trim is paced and reports its effect through the -always-on telemetry, that the image's allocator environment cannot change -silently, and that the soak's resident-set bound is an automated verdict. -""" - -from __future__ import annotations - -import ast -import gc -import json -import os -import re -import signal -import subprocess -import sys -import tempfile -import threading -import time -import tracemalloc -import unittest -from pathlib import Path -from types import SimpleNamespace -from unittest import mock - -from lab.prism import process_telemetry -from lab.prism.coordinator_config import ( - DEFAULT_PRISM_HEAP_CENSUS_CONFIG, - DEFAULT_PRISM_HEAP_CENSUS_MAX_BYTES, - DEFAULT_PRISM_HEAP_CENSUS_MAX_SECONDS, - DEFAULT_PRISM_HEAP_CENSUS_MIN_INTERVAL_SECONDS, - DEFAULT_PRISM_HEAP_CENSUS_TOP_N, - MAX_PRISM_HEAP_CENSUS_MAX_SECONDS, - MAX_PRISM_HEAP_CENSUS_TOP_N, - MIN_PRISM_MALLOC_TRIM_INTERVAL_SECONDS, - env_heap_census_config, - load_coordinator_config, -) -from lab.prism.metrics import MetricsRenderer -from lab.prism.process_telemetry import ( - HEAP_CENSUS_FORMAT, - HEAP_CENSUS_RETAINED_FILES, - HeapCensus, - HeapCensusConfig, - HeapCensusService, - MallocTrimmer, - ProcessHeapSample, - ProcessHeapTelemetry, - evaluate_rss_bound, - heap_census_signal, - install_heap_census, - malloc_trim_signal, - parse_malloc_info_xml, - read_anonymous_map_shape, - read_malloc_arena_count, - read_malloc_info_summary, - read_resident_memory_bytes, -) -from tests import prism_vardiff_test_support as support - -ROOT = Path(__file__).resolve().parents[1] -PRISM_DIR = ROOT / "lab" / "prism" -DOCKERFILE = PRISM_DIR / "Dockerfile" -COMPOSE_FILE = ROOT / "compose.yaml" -ENV_EXAMPLE = ROOT / ".env.example" - -# SIGRTMIN is Linux-only. The fake signal module carries a synthetic number so -# every fake-driven test runs on macOS too; only the test that delivers a real -# signal to this process gates its real-time-signal half on the platform. -_HAS_SIGRTMIN = hasattr(signal, "SIGRTMIN") -_SIGRTMIN = getattr(signal, "SIGRTMIN", 32) -_HAS_PROCFS = Path("/proc/self/statm").exists() - -# The image's allocator environment, as decided in lab/prism/Dockerfile and -# argued in docs/mainnet-deployment.md. A change here must be a decision. -PINNED_MALLOC_ARENA_MAX = "2" - -# Every name through which the walk or the trim can be reached. No module -# that serves a request may mention any of them. -CENSUS_ENTRY_POINTS = ( - "HeapCensus", - "HeapCensusService", - "install_heap_census", - "MallocTrimmer", - "resolve_malloc_trim", - "gc.get_objects", - "tracemalloc.take_snapshot", - "malloc_trim", -) - - -# A malloc_info document in glibc's shape: per heap, one <size> row per -# non-empty size class (fastbin classes first), an <unsorted> row, the fast -# and rest totals (rest = bins + top, never fastbins), then the global totals. -_MALLOC_INFO_FIXTURE = """<malloc version="1"> -<heap nr="0"> -<sizes> - <size from="33" to="33" total="198" count="6"/> - <size from="49" to="49" total="294" count="6"/> - <size from="145" to="145" total="145" count="1"/> - <size from="20001" to="20001" total="40002" count="2"/> - <unsorted from="5000" to="5000" total="5000" count="1"/> -</sizes> -<total type="fast" count="12" size="492"/> -<total type="rest" count="5" size="145147"/> -<system type="current" size="1000000"/> -<system type="max" size="1000000"/> -<aspace type="total" size="1000000"/> -<aspace type="mprotect" size="1000000"/> -</heap> -<heap nr="1"> -<sizes> -</sizes> -<total type="fast" count="0" size="0"/> -<total type="rest" count="1" size="70000"/> -<system type="current" size="200000"/> -<system type="max" size="200000"/> -<aspace type="total" size="200000"/> -<aspace type="mprotect" size="200000"/> -</heap> -<total type="fast" count="12" size="492"/> -<total type="rest" count="6" size="215147"/> -<total type="mmap" count="0" size="0"/> -<system type="current" size="1200000"/> -<system type="max" size="1200000"/> -<aspace type="total" size="1200000"/> -<aspace type="mprotect" size="1200000"/> -</malloc> -""" - - -def _coordinator_service_environment() -> dict[str, str]: - """The prism-coordinator service's environment mapping, from the raw text. - - Parsed without a YAML dependency: the service block runs from its key to - the next two-space-indented key, and its environment entries are the - six-space-indented ``NAME: value`` lines inside it. - """ - lines = COMPOSE_FILE.read_text(encoding="utf-8").splitlines() - start = lines.index(" prism-coordinator:") - block: list[str] = [] - for line in lines[start + 1:]: - if re.match(r"^ [A-Za-z0-9_-]+:\s*$", line): - break - block.append(line) - environment: dict[str, str] = {} - inside = False - for line in block: - if re.match(r"^ environment:\s*$", line): - inside = True - continue - if inside and re.match(r"^ [A-Za-z0-9_-]+:", line): - inside = False - if inside: - match = re.match(r"^ ([A-Z0-9_]+): (.*)$", line) - if match: - environment[match.group(1)] = match.group(2).strip() - return environment - - -def _env_example_values() -> dict[str, str]: - values: dict[str, str] = {} - for line in ENV_EXAMPLE.read_text(encoding="utf-8").splitlines(): - if line and not line.startswith("#") and "=" in line: - key, value = line.split("=", 1) - values[key] = value - return values - - -def _config(**overrides: object) -> HeapCensusConfig: - values: dict[str, object] = { - "enabled": True, - "output_dir": "unused", - "top_n": 5, - "max_bytes": 65_536, - "max_seconds": 10.0, - "min_interval_seconds": 60.0, - "tracemalloc_enabled": False, - "malloc_trim_signal_enabled": False, - "malloc_trim_interval_seconds": 0.0, - } - values.update(overrides) - return HeapCensusConfig(**values) # type: ignore[arg-type] - - -def _sample(**overrides: int) -> ProcessHeapSample: - values: dict[str, object] = { - "allocated_blocks": 1_000, - "gc_trigger_count": (1, 2, 3), - "gc_collections": (4, 5, 6), - "gc_collected": (7, 8, 9), - "gc_uncollectable": (0, 0, 0), - "threads": 64, - "malloc_info_available": True, - "malloc_arena_bytes": 10_000, - "malloc_in_use_bytes": 6_000, - "malloc_free_bytes": 4_000, - "malloc_mmapped_bytes": 500, - } - values.update(overrides) - return ProcessHeapSample(**values) # type: ignore[arg-type] - - -class _FakeClock: - def __init__(self, now: float = 1_000.0) -> None: - self.now = now - - def __call__(self) -> float: - return self.now - - def advance(self, seconds: float) -> None: - self.now += seconds - - -class _FakeSignalModule: - """Records registrations instead of installing real handlers.""" - - SIGUSR1 = signal.SIGUSR1 - SIGRTMIN = _SIGRTMIN - SIG_DFL = signal.SIG_DFL - - def __init__(self) -> None: - self.registered: dict[int, object] = {} - - def signal(self, signum: int, handler: object) -> None: - self.registered[signum] = handler - - def getsignal(self, signum: int) -> object: - return self.registered.get(signum, self.SIG_DFL) - - -class HeapCensusDisabledByDefaultTests(unittest.TestCase): - def test_heap_census_is_disabled_by_default(self) -> None: - """Without the switch nothing is registered and the entry point is inert.""" - config = env_heap_census_config(environ={}) - self.assertFalse(config.enabled) - self.assertFalse(config.tracemalloc_enabled) - self.assertFalse(config.malloc_trim_signal_enabled) - self.assertEqual(config.malloc_trim_interval_seconds, 0.0) - self.assertFalse(config.anything_armed) - # The shipped lifecycle default is what the loader produces from an - # empty environment, switches and bounds alike. - self.assertEqual(config, DEFAULT_PRISM_HEAP_CENSUS_CONFIG) - fake_signal = _FakeSignalModule() - with ( - mock.patch.object( - gc, - "get_objects", - side_effect=AssertionError("gc.get_objects ran with the census off"), - ) as get_objects, - mock.patch.object( - tracemalloc, - "start", - side_effect=AssertionError("tracemalloc started with the census off"), - ) as tracemalloc_start, - ): - self.assertIsNone(install_heap_census(config, signal_module=fake_signal)) - self.assertEqual(fake_signal.registered, {}) - self.assertFalse(tracemalloc.is_tracing()) - with tempfile.TemporaryDirectory() as directory: - census = HeapCensus(_config(enabled=False, output_dir=directory)) - self.assertIsNone(census.take("direct call")) - self.assertIsNone(census.last_outcome) - self.assertEqual(os.listdir(directory), []) - service = HeapCensusService(_config(enabled=False, output_dir=directory)) - try: - self.assertFalse(service.request_census("signal")) - self.assertIsNone(service.dispatch("census", "direct")) - self.assertEqual(os.listdir(directory), []) - finally: - service.close() - get_objects.assert_not_called() - tracemalloc_start.assert_not_called() - # No worker thread exists for a census that never armed. - self.assertFalse( - any( - thread.name == HeapCensusService.THREAD_NAME - for thread in threading.enumerate() - ) - ) - - def test_census_switches_are_strict_and_bounded(self) -> None: - """A typo or an unbounded value refuses startup rather than arming.""" - for bad in ( - {"PRISM_HEAP_CENSUS": "maybe"}, - {"PRISM_HEAP_CENSUS_TRACEMALLOC": "1"}, - {"PRISM_HEAP_CENSUS": "1", "PRISM_HEAP_CENSUS_TRACEMALLOC": "sometimes"}, - {"PRISM_HEAP_CENSUS_TOP_N": "0"}, - {"PRISM_HEAP_CENSUS_TOP_N": str(MAX_PRISM_HEAP_CENSUS_TOP_N + 1)}, - {"PRISM_HEAP_CENSUS_MAX_BYTES": "10"}, - {"PRISM_HEAP_CENSUS_MAX_BYTES": str(64 * 1024 * 1024)}, - {"PRISM_HEAP_CENSUS_MAX_SECONDS": "0"}, - {"PRISM_HEAP_CENSUS_MAX_SECONDS": str(MAX_PRISM_HEAP_CENSUS_MAX_SECONDS + 1)}, - {"PRISM_HEAP_CENSUS_MIN_INTERVAL_SECONDS": "-1"}, - {"PRISM_MALLOC_TRIM_SIGNAL": "yes please"}, - { - "PRISM_MALLOC_TRIM_INTERVAL_SECONDS": str( - MIN_PRISM_MALLOC_TRIM_INTERVAL_SECONDS / 2 - ) - }, - ): - with self.subTest(bad=bad), self.assertRaises(SystemExit): - env_heap_census_config(environ=bad) - armed = env_heap_census_config( - environ={ - "PRISM_HEAP_CENSUS": "on", - "PRISM_HEAP_CENSUS_TRACEMALLOC": "true", - "PRISM_MALLOC_TRIM_SIGNAL": "1", - "PRISM_MALLOC_TRIM_INTERVAL_SECONDS": "600", - "PRISM_AUDIT_DIR": "/var/lib/qbit-prism/audit", - } - ) - self.assertTrue(armed.enabled) - self.assertTrue(armed.tracemalloc_enabled) - self.assertTrue(armed.malloc_trim_signal_enabled) - self.assertEqual(armed.malloc_trim_interval_seconds, 600.0) - self.assertEqual(armed.output_dir, "/var/lib/qbit-prism/audit/heap-census") - self.assertEqual(armed.top_n, DEFAULT_PRISM_HEAP_CENSUS_TOP_N) - self.assertEqual(armed.max_bytes, DEFAULT_PRISM_HEAP_CENSUS_MAX_BYTES) - self.assertEqual(armed.max_seconds, DEFAULT_PRISM_HEAP_CENSUS_MAX_SECONDS) - self.assertEqual( - armed.min_interval_seconds, DEFAULT_PRISM_HEAP_CENSUS_MIN_INTERVAL_SECONDS - ) - # The loader carries the validated snapshot, so a bad bound refuses - # coordinator startup, not just a standalone read. - with tempfile.TemporaryDirectory() as tmp: - root = Path(tmp) - base = { - "QBIT_RPC_HOST": "qbit.example", - "QBIT_RPC_USER": "rpc-user", - "QBIT_RPC_PASSWORD": "rpc-password", - "PRISM_ALLOW_MEMORY_LEDGER": "1", - "PRISM_ALLOW_TEST_SIGNING_SEEDS": "1", - "PRISM_ALLOW_BUNDLE_EMBEDDED_LEDGER_KEY": "1", - "PRISM_AUDIT_DIR": str(root), - "PRISM_EVIDENCE_PATH": str(root / "evidence.json"), - } - shipped = load_coordinator_config(base) - self.assertFalse(shipped.lifecycle.heap_census.enabled) - self.assertFalse(shipped.lifecycle.heap_census.anything_armed) - self.assertEqual( - shipped.lifecycle.heap_census.output_dir, str(root / "heap-census") - ) - with self.assertRaisesRegex(SystemExit, "PRISM_HEAP_CENSUS_TOP_N must be at most"): - load_coordinator_config( - {**base, "PRISM_HEAP_CENSUS": "1", "PRISM_HEAP_CENSUS_TOP_N": "9999"} - ) - with self.assertRaisesRegex(SystemExit, "PRISM_HEAP_CENSUS must be a boolean"): - load_coordinator_config({**base, "PRISM_HEAP_CENSUS": "maybe"}) - loaded = load_coordinator_config({**base, "PRISM_HEAP_CENSUS": "1"}) - self.assertTrue(loaded.lifecycle.heap_census.enabled) - self.assertEqual( - loaded.lifecycle.heap_census.output_dir, - str(loaded.audit.directory / "heap-census"), - ) - - -class HeapCensusBoundedReportTests(unittest.TestCase): - def test_heap_census_writes_a_bounded_report_to_a_file(self) -> None: - """On a heap with more types than N, the file holds exactly N and fits.""" - - class Retained: - pass - - # More distinct tracked types than the census may list, each with - # enough instances to outrank the interpreter's own bookkeeping. - kinds = [type(f"CensusProbe{index}", (Retained,), {}) for index in range(12)] - keep = [kind() for kind in kinds for _ in range(50)] - with tempfile.TemporaryDirectory() as directory: - config = _config(output_dir=os.path.join(directory, "nested"), top_n=5, max_bytes=16_384) - census = HeapCensus(config) - outcome = census.take("test") - self.assertIsNotNone(outcome) - assert outcome is not None - path = Path(outcome.path) - self.assertTrue(path.exists()) - self.assertEqual(path.parent, Path(config.output_dir)) - self.assertLessEqual(path.stat().st_size, config.max_bytes) - self.assertEqual(outcome.bytes_written, path.stat().st_size) - report = json.loads(path.read_text(encoding="utf-8")) - self.assertEqual(report["format"], HEAP_CENSUS_FORMAT) - self.assertEqual(report["reason"], "test") - self.assertEqual(report["bounds"], {"top_n": 5, "max_bytes": 16_384, "max_seconds": 10.0}) - self.assertEqual(len(report["types_by_count"]), 5) - self.assertEqual(len(report["types_by_bytes"]), 5) - self.assertEqual(report["top_n_written"], 5) - self.assertFalse(report["walk"]["truncated"]) - self.assertGreater(report["walk"]["distinct_types"], 5) - self.assertEqual(report["walk"]["objects_walked"], report["walk"]["tracked_objects"]) - self.assertGreaterEqual(report["walk"]["tracked_objects"], len(keep)) - self.assertEqual(report["tracemalloc"], {"tracing": False}) - self.assertEqual(report["pid"], os.getpid()) - self.assertIn("resident_memory_bytes", report["process"]) - self.assertIn("malloc_arena_count", report["process"]) - self.assertIn("anonymous_map", report["process"]) - # Each entry is a name, a count and a shallow byte total; the - # list is sorted by its key and nothing in it is a repr of an - # object (no addresses, no payload). - counts = [entry["count"] for entry in report["types_by_count"]] - self.assertEqual(counts, sorted(counts, reverse=True)) - for entry in report["types_by_count"] + report["types_by_bytes"]: - self.assertEqual(set(entry), {"type", "count", "shallow_bytes"}) - self.assertNotIn("0x", entry["type"]) - # The byte cap is met by shrinking N, never by cutting JSON. - tight = HeapCensus(_config(output_dir=directory, top_n=200, max_bytes=4_096)) - tight_outcome = tight.take("tight") - assert tight_outcome is not None - tight_report = json.loads(Path(tight_outcome.path).read_text(encoding="utf-8")) - self.assertLessEqual(Path(tight_outcome.path).stat().st_size, 4_096) - self.assertLess(tight_report["top_n_written"], 200) - self.assertTrue(tight_report["fit_to_max_bytes"]) - self.assertEqual(len(tight_report["types_by_count"]), tight_report["top_n_written"]) - # The walk cap: with no time at all the pass stops after its - # first stride and says so, and the report is still written. - clock = _FakeClock() - capped = HeapCensus( - _config(output_dir=directory, max_seconds=0.001), - clock=_advancing(clock, 1.0), - ) - capped_outcome = capped.take("capped") - assert capped_outcome is not None - self.assertTrue(capped_outcome.walk_truncated) - self.assertLess(capped_outcome.objects_walked, capped_outcome.tracked_objects) - capped_report = json.loads(Path(capped_outcome.path).read_text(encoding="utf-8")) - self.assertTrue(capped_report["walk"]["truncated"]) - # Files are retained up to a fixed count; the oldest go first, - # including the first report this census wrote. - for index in range(HEAP_CENSUS_RETAINED_FILES + 3): - self.assertIsNotNone(census.take(f"fill-{index}")) - names = sorted( - name - for name in os.listdir(config.output_dir) - if name.startswith("heap-census-") - ) - self.assertEqual(len(names), HEAP_CENSUS_RETAINED_FILES) - self.assertFalse(path.exists()) - self.assertFalse(any(name.endswith(".tmp") for name in os.listdir(config.output_dir))) - del keep - - def test_types_sharing_a_printed_name_merge_rather_than_overwrite(self) -> None: - """Two type objects with one printed name add up; neither hides the other.""" - - def make_probe_class() -> type: - class CensusMergeProbe: - pass - - # The same short printed name for both, as two modules defining - # one class name would have; the nested qualname would only - # truncate into a third collision. - CensusMergeProbe.__qualname__ = "CensusMergeProbe" - return CensusMergeProbe - - first = make_probe_class() - second = make_probe_class() - self.assertIsNot(first, second) - self.assertEqual( - (first.__module__, first.__qualname__), (second.__module__, second.__qualname__) - ) - # Two more whose long names truncate to the same printed name. - long_a = type("L" * 150, (object,), {}) - long_b = type("L" * 149 + "Z", (object,), {}) - keep = [first() for _ in range(30)] + [second() for _ in range(20)] - keep += [long_a() for _ in range(7)] + [long_b() for _ in range(5)] - with tempfile.TemporaryDirectory() as directory: - outcome = HeapCensus(_config(output_dir=directory, top_n=200)).take("merge") - assert outcome is not None - report = json.loads(Path(outcome.path).read_text(encoding="utf-8")) - del keep - by_name = {entry["type"]: entry for entry in report["types_by_count"]} - merged = [name for name in by_name if name.endswith(".CensusMergeProbe")] - self.assertEqual(len(merged), 1, merged) - self.assertEqual(by_name[merged[0]]["count"], 50) - self.assertGreater(by_name[merged[0]]["shallow_bytes"], 0) - truncated = [name for name in by_name if name.endswith("...") and "LLLL" in name] - self.assertEqual(len(truncated), 1, truncated) - self.assertEqual(by_name[truncated[0]]["count"], 12) - # The walk still reports how many type objects it saw, and how many - # printed names they merged into. - self.assertGreater(report["walk"]["distinct_types"], report["walk"]["distinct_type_names"]) - - def test_retention_keeps_the_newest_reports_regardless_of_mtime(self) -> None: - """A tight loop past the limit evicts the oldest even when every mtime ties.""" - with tempfile.TemporaryDirectory() as directory: - census = HeapCensus(_config(output_dir=directory, top_n=3)) - total = HEAP_CENSUS_RETAINED_FILES + 8 - for index in range(total - 1): - self.assertIsNotNone(census.take(f"fill-{index}")) - # Every existing report now carries the same timestamp, so the - # eviction that the final census triggers cannot lean on mtime. - same_stamp = 1_700_000_000 - for name in os.listdir(directory): - os.utime(os.path.join(directory, name), (same_stamp, same_stamp)) - self.assertIsNotNone(census.take(f"fill-{total - 1}")) - names = sorted( - name for name in os.listdir(directory) if name.startswith("heap-census-") - ) - self.assertEqual(len(names), HEAP_CENSUS_RETAINED_FILES) - reasons = [ - json.loads(Path(directory, name).read_text(encoding="utf-8"))["reason"] - for name in names - ] - expected = [f"fill-{index}" for index in range(total - HEAP_CENSUS_RETAINED_FILES, total)] - self.assertEqual(reasons, expected) - # Name order is write order: the sequence is zero-padded, so the - # tenth report does not sort before the ninth. - sequences = [int(name.rsplit("-", 1)[1].split(".")[0]) for name in names] - self.assertEqual(sequences, sorted(sequences)) - self.assertEqual(sequences, list(range(total - HEAP_CENSUS_RETAINED_FILES + 1, total + 1))) - - def test_heap_census_reports_tracemalloc_sites_only_while_tracing(self) -> None: - """The site lists appear only when tracing, and are capped at N too.""" - self.assertFalse(tracemalloc.is_tracing()) - tracemalloc.start(1) - try: - junk = [bytearray(2_048) for _ in range(200)] - with tempfile.TemporaryDirectory() as directory: - outcome = HeapCensus(_config(output_dir=directory, top_n=3)).take("traced") - assert outcome is not None - self.assertTrue(outcome.tracemalloc_traced) - report = json.loads(Path(outcome.path).read_text(encoding="utf-8")) - traced = report["tracemalloc"] - self.assertTrue(traced["tracing"]) - self.assertEqual(traced["frames"], 1) - self.assertLessEqual(len(traced["by_line"]), 3) - self.assertLessEqual(len(traced["by_file"]), 3) - self.assertGreater(traced["traced_bytes"], 0) - for entry in traced["by_line"] + traced["by_file"]: - self.assertEqual(set(entry), {"site", "bytes", "count"}) - del junk - finally: - tracemalloc.stop() - - def test_malloc_info_split_excludes_fastbins_from_bins(self) -> None: - """Fastbin <size> rows are not part of <total type="rest">; top must not absorb them.""" - summary = parse_malloc_info_xml(_MALLOC_INFO_FIXTURE) - self.assertTrue(summary.available) - self.assertEqual(summary.arenas, 2) - self.assertEqual(summary.fast_bytes, 492) - # Heap 0: rows 198+294 (fast) + 145 + 40002 + unsorted 5000; rest is - # bins plus top = 145147, so bins are 45147 and top is 100000. - # Heap 1: nothing in any bin, rest 70000 is all top. - self.assertEqual(summary.bin_bytes, 45_147) - self.assertEqual(summary.top_bytes, 170_000) - self.assertEqual(summary.system_bytes, 1_200_000) - # The identity the split must keep: fast + bins + top is exactly the - # sum of glibc's own fast and rest totals. - self.assertEqual(summary.fast_bytes + summary.bin_bytes + summary.top_bytes, 492 + 215_147) - # A document without heaps folds to zero arenas, not to a crash. - empty = parse_malloc_info_xml('<malloc version="1">\n</malloc>\n') - self.assertEqual((empty.arenas, empty.top_bytes, empty.bin_bytes), (0, 0, 0)) - # And the live reading on this host, where glibc offers it, is - # internally consistent: no negative part, parts under system bytes. - live = read_malloc_info_summary() - if live.available: - self.assertGreaterEqual(live.top_bytes, 0) - self.assertGreaterEqual(live.bin_bytes, 0) - self.assertGreaterEqual(live.fast_bytes, 0) - self.assertLessEqual(live.top_bytes + live.bin_bytes + live.fast_bytes, live.system_bytes) - - def test_process_readings_are_available_on_this_host(self) -> None: - """The readings the census and the storm instrument share are real here.""" - resident = read_resident_memory_bytes() - shape = read_anonymous_map_shape() - arenas = read_malloc_arena_count() - if not Path("/proc/self/maps").exists(): - self.skipTest("no procfs on this platform") - self.assertGreater(resident, 0) - self.assertTrue(shape.available) - self.assertGreaterEqual(shape.regions, 1) - self.assertEqual( - shape.regions, sum(shape.band_regions.values()) - ) - self.assertEqual(shape.total_bytes, sum(shape.band_bytes.values())) - if ProcessHeapTelemetry().sample().malloc_info_available: - self.assertGreaterEqual(arenas, 1) - # A pid that does not exist reads as unavailable, not as zero. - self.assertEqual(read_resident_memory_bytes(2**22 + 12345), -1) - self.assertFalse(read_anonymous_map_shape(2**22 + 12345).available) - - -def _advancing(clock: _FakeClock, step: float): - """A clock that moves ``step`` forward on every read.""" - - def read() -> float: - clock.advance(step) - return clock.now - - return read - - -class HeapCensusRequestPathTests(unittest.TestCase): - def test_heap_census_is_not_reachable_from_any_request_path(self) -> None: - """Structurally: only main() in the coordinator can arm it, and no server module names it.""" - # 1. The coordinator references the installer in exactly one place, - # inside main(), after the shutdown handlers. - source = (PRISM_DIR / "prism_coordinator.py").read_text(encoding="utf-8") - tree = ast.parse(source) - main_node = next( - node for node in tree.body - if isinstance(node, ast.FunctionDef) and node.name == "main" - ) - main_span = range(main_node.lineno, main_node.end_lineno + 1) - references = [ - node.lineno - for node in ast.walk(tree) - if isinstance(node, ast.Name) and node.id == "install_heap_census" - ] - self.assertEqual(len(references), 1, references) - self.assertIn(references[0], main_span) - for name in ("HeapCensus", "HeapCensusService", "MallocTrimmer", "gc.get_objects"): - self.assertNotIn(name, source) - # 2. No module that serves a request names any entry point. The - # census module itself, the config that validates its switches, - # and the coordinator's main() are the only exceptions. - exempt = {"process_telemetry.py", "coordinator_config.py", "prism_coordinator.py"} - offenders: dict[str, list[str]] = {} - for module in sorted(PRISM_DIR.glob("*.py")): - if module.name in exempt: - continue - text = module.read_text(encoding="utf-8") - hits = [name for name in CENSUS_ENTRY_POINTS if name in text] - if hits: - offenders[module.name] = hits - self.assertEqual(offenders, {}) - for served in ("audit_http.py", "public_read_service.py", "metrics.py"): - self.assertTrue((PRISM_DIR / served).exists(), served) - # 3. A full metrics render, the one path every scrape takes, never - # walks the heap or trims, even with the census armed in-process. - server = support.coordinator() - with tempfile.TemporaryDirectory() as directory: - service = HeapCensusService(_config(output_dir=directory)) - try: - with ( - mock.patch.object( - gc, - "get_objects", - side_effect=AssertionError("gc.get_objects walked the heap on a scrape"), - ) as get_objects, - mock.patch.object( - tracemalloc, - "take_snapshot", - side_effect=AssertionError("tracemalloc snapshot on a scrape"), - ) as take_snapshot, - mock.patch.object( - HeapCensus, - "take", - side_effect=AssertionError("census taken on a scrape"), - ) as take, - mock.patch.object( - MallocTrimmer, - "trim_once", - side_effect=AssertionError("malloc_trim on a scrape"), - ) as trim, - ): - document = MetricsRenderer(server).render() - self.assertIn("qbit_prism_process_allocated_blocks ", document) - # The signal handlers, run as the interpreter would run - # them, only wake the worker: no walk in handler context. - fake_signal = _FakeSignalModule() - armed = install_heap_census( - _config(output_dir=directory, malloc_trim_signal_enabled=True), - signal_module=fake_signal, - start_thread=False, - log=lambda _line: None, - ) - assert armed is not None - try: - census_handler = fake_signal.registered[heap_census_signal(fake_signal)] - trim_handler = fake_signal.registered[malloc_trim_signal(fake_signal)] - census_handler(signal.SIGUSR1, None) - trim_handler(_SIGRTMIN + 1, None) - finally: - armed.close() - get_objects.assert_not_called() - take_snapshot.assert_not_called() - take.assert_not_called() - trim.assert_not_called() - finally: - service.close() - self.assertEqual( - [name for name in os.listdir(directory) if name.startswith("heap-census-")], - [], - ) - - def test_signal_handlers_queue_work_for_the_worker_thread(self) -> None: - """A real SIGUSR1 to this process produces a census on the worker, not inline. - - The trim half needs a real-time signal, which macOS does not have; - there only the SIGUSR1 half runs. - """ - with tempfile.TemporaryDirectory() as directory: - previous_census = signal.getsignal(signal.SIGUSR1) - previous_trim = signal.getsignal(_SIGRTMIN + 1) if _HAS_SIGRTMIN else None - trimmer = MallocTrimmer( - SimpleNamespace(sample=lambda: _sample()), # type: ignore[arg-type] - resolver=lambda: (lambda _pad: 1), - rss_reader=lambda: 5, - log=lambda _line: None, - ) - service = HeapCensusService( - _config(output_dir=directory, malloc_trim_signal_enabled=True, min_interval_seconds=0.0), - trimmer=trimmer, - log=lambda _line: None, - ) - try: - fake_signal = _FakeSignalModule() - with mock.patch.object( - process_telemetry, "HeapCensusService", return_value=service - ): - armed = install_heap_census( - service.config, signal_module=fake_signal, log=lambda _line: None - ) - self.assertIs(armed, service) - # Install the recorded handlers for real, deliver the - # signals to ourselves, and wait for the worker. - signal.signal(signal.SIGUSR1, fake_signal.registered[signal.SIGUSR1]) - if _HAS_SIGRTMIN: - signal.signal(_SIGRTMIN + 1, fake_signal.registered[_SIGRTMIN + 1]) - try: - os.kill(os.getpid(), signal.SIGUSR1) - if _HAS_SIGRTMIN: - os.kill(os.getpid(), _SIGRTMIN + 1) - # Bounded wait: the worker has ten seconds to serve both. - deadline = time.monotonic() + 10.0 - while time.monotonic() < deadline and ( - service.census.last_outcome is None - or (_HAS_SIGRTMIN and trimmer.calls == 0) - ): - time.sleep(0.02) - finally: - signal.signal(signal.SIGUSR1, previous_census) - if _HAS_SIGRTMIN: - signal.signal(_SIGRTMIN + 1, previous_trim) - outcome = service.census.last_outcome - self.assertIsNotNone(outcome) - assert outcome is not None - self.assertTrue(Path(outcome.path).exists()) - self.assertIn("signal:", json.loads(Path(outcome.path).read_text())["reason"]) - if _HAS_SIGRTMIN: - self.assertEqual(trimmer.calls, 1) - assert trimmer.last_result is not None - self.assertTrue(trimmer.last_result.reason.startswith("signal:")) - else: - self.assertEqual(trimmer.calls, 0) - finally: - service.close() - - def test_close_disarms_signals_and_silences_wake(self) -> None: - """After close() the previous handlers are back and a late signal writes nothing.""" - fake_signal = _FakeSignalModule() - earlier_census = object() - earlier_trim = object() - fake_signal.registered[signal.SIGUSR1] = earlier_census - fake_signal.registered[_SIGRTMIN + 1] = earlier_trim - with tempfile.TemporaryDirectory() as directory: - service = install_heap_census( - _config(output_dir=directory, malloc_trim_signal_enabled=True), - signal_module=fake_signal, - start_thread=False, - log=lambda _line: None, - ) - assert service is not None - census_handler = fake_signal.registered[signal.SIGUSR1] - trim_handler = fake_signal.registered[_SIGRTMIN + 1] - self.assertIsNot(census_handler, earlier_census) - self.assertIsNot(trim_handler, earlier_trim) - self.assertTrue(service.request_census("before close")) - service.close() - # The handlers this service armed are gone, and what they - # replaced is back. - self.assertIs(fake_signal.registered[signal.SIGUSR1], earlier_census) - self.assertIs(fake_signal.registered[_SIGRTMIN + 1], earlier_trim) - # A handler someone kept a reference to, or a request made after - # close, is a no-op: no write into a recycled descriptor. - with mock.patch.object(os, "write", side_effect=AssertionError("wrote after close")) as write: - self.assertFalse(service.request_census("late")) - self.assertFalse(service.request_trim("late")) - self.assertFalse(service._wake(b"c")) - census_handler(signal.SIGUSR1, None) - trim_handler(_SIGRTMIN + 1, None) - write.assert_not_called() - service.close() # idempotent - self.assertEqual( - [name for name in os.listdir(directory) if name.startswith("heap-census-")], - [], - ) - # With nothing registered beforehand the restore lands on SIG_DFL, - # not on a stale handler. - bare = _FakeSignalModule() - service = install_heap_census( - _config(output_dir="unused-never-written", malloc_trim_signal_enabled=True), - signal_module=bare, - start_thread=False, - log=lambda _line: None, - ) - assert service is not None - service.close() - self.assertIs(bare.registered[signal.SIGUSR1], signal.SIG_DFL) - self.assertIs(bare.registered[_SIGRTMIN + 1], signal.SIG_DFL) - - -class MallocTrimTests(unittest.TestCase): - def test_malloc_trim_action_is_bounded_and_reports_its_effect(self) -> None: - """At most one trim per interval, deltas reported through the WP1a sample.""" - calls: list[int] = [] - samples = iter( - [ - _sample(malloc_arena_bytes=10_000, malloc_in_use_bytes=6_000, malloc_free_bytes=4_000), - _sample(malloc_arena_bytes=7_000, malloc_in_use_bytes=6_000, malloc_free_bytes=1_000), - _sample(malloc_arena_bytes=7_000, malloc_in_use_bytes=6_000, malloc_free_bytes=1_000), - _sample(malloc_arena_bytes=7_000, malloc_in_use_bytes=6_000, malloc_free_bytes=1_000), - ] - ) - rss = iter([9_000, 6_500, 6_500, 6_500]) - clock = _FakeClock() - lines: list[str] = [] - - def fake_trim(pad: int) -> int: - calls.append(pad) - return 1 - - trimmer = MallocTrimmer( - SimpleNamespace(sample=lambda: next(samples)), # type: ignore[arg-type] - resolver=lambda: fake_trim, - clock=clock, - rss_reader=lambda: next(rss), - log=lines.append, - ) - service = HeapCensusService( - _config(enabled=False, min_interval_seconds=60.0, malloc_trim_interval_seconds=0.0), - trimmer=trimmer, - clock=clock, - log=lines.append, - ) - try: - first = service.dispatch("trim", "signal:35") - assert first is not None - self.assertEqual(calls, [0]) - self.assertTrue(first.released) - self.assertEqual(first.arena_delta, -3_000) - self.assertEqual(first.free_delta, -3_000) - self.assertEqual(first.in_use_delta, 0) - self.assertEqual(first.resident_delta, -2_500) - self.assertEqual(first.as_dict()["reason"], "signal:35") - self.assertTrue(any(line.startswith("prism malloc_trim ") for line in lines)) - # Two more requests inside the interval are suppressed, and say so. - clock.advance(10.0) - self.assertIsNone(service.dispatch("trim", "signal:35")) - clock.advance(10.0) - self.assertIsNone(service.dispatch("trim", "signal:35")) - self.assertEqual(calls, [0]) - self.assertEqual(service.suppressed["trim"], 2) - self.assertTrue(any("malloc_trim suppressed" in line for line in lines)) - # Past the interval the next request runs. - clock.advance(60.0) - self.assertIsNotNone(service.dispatch("trim", "periodic")) - self.assertEqual(calls, [0, 0]) - self.assertEqual(trimmer.calls, 2) - finally: - service.close() - # The periodic action is paced by its own interval through the - # same gate, and is not due before it. - clock = _FakeClock() - periodic = HeapCensusService( - _config(enabled=False, min_interval_seconds=0.0, malloc_trim_interval_seconds=120.0), - trimmer=MallocTrimmer( - SimpleNamespace(sample=lambda: _sample()), # type: ignore[arg-type] - resolver=lambda: fake_trim, - clock=clock, - rss_reader=lambda: 1, - log=lambda _line: None, - ), - clock=clock, - log=lambda _line: None, - ) - try: - calls.clear() - self.assertFalse(periodic.periodic_trim_due()) - self.assertTrue(periodic.run_once(0.0)) - self.assertEqual(calls, []) - clock.advance(120.0) - self.assertTrue(periodic.periodic_trim_due()) - self.assertTrue(periodic.run_once(0.0)) - self.assertEqual(calls, [0]) - self.assertFalse(periodic.periodic_trim_due()) - finally: - periodic.close() - # An absent symbol is remembered: one log line, no call, no retry. - absent_lines: list[str] = [] - absent = MallocTrimmer( - SimpleNamespace(sample=lambda: _sample()), # type: ignore[arg-type] - resolver=mock.Mock(side_effect=AttributeError("malloc_trim: symbol not found")), - log=absent_lines.append, - ) - self.assertIsNone(absent.trim_once("a")) - self.assertIsNone(absent.trim_once("b")) - self.assertEqual(absent.calls, 0) - self.assertEqual(len(absent_lines), 1) - - def test_real_malloc_trim_releases_free_bytes_and_keeps_in_use(self) -> None: - """On glibc the real call frees retained free chunks and touches nothing live.""" - telemetry = ProcessHeapTelemetry() - if not telemetry.sample().malloc_info_available: - self.skipTest("mallinfo2 unavailable on this platform") - # Fragment the main arena a little: allocate and free medium chunks - # that stay under the mmap threshold, so glibc keeps them. - junk = [bytes([index % 251]) * (48 * 1024) for index in range(512)] - held = junk[::2] - del junk - result = MallocTrimmer(telemetry, log=lambda _line: None).trim_once("test") - self.assertIsNotNone(result) - assert result is not None - self.assertIsInstance(result.released, bool) - # Reading RSS and sampling mallinfo2 allocate Python/ctypes objects - # between the two samples. Their exact net allocation varies across - # Python builds, even though malloc_trim preserves every live chunk. - # Check the live allocation and its contents instead of requiring the - # sampling machinery itself to have a zero-byte delta. - live_bytes = sum(map(len, held)) - self.assertGreaterEqual(result.before.malloc_in_use_bytes, live_bytes) - self.assertGreaterEqual(result.after.malloc_in_use_bytes, live_bytes) - for index, chunk in enumerate(held): - self.assertEqual(chunk, bytes([(index * 2) % 251]) * (48 * 1024)) - self.assertLessEqual(result.free_delta, 0) - self.assertLessEqual(result.arena_delta, 0) - self.assertGreaterEqual(result.seconds, 0.0) - del held - - -class AllocatorEnvironmentPinTests(unittest.TestCase): - def test_allocator_environment_defaults_are_pinned(self) -> None: - """The image sets exactly MALLOC_ARENA_MAX, at the decided value, overridably.""" - dockerfile = DOCKERFILE.read_text(encoding="utf-8") - args = dict(re.findall(r"^ARG\s+([A-Z0-9_]+)=(\S*)\s*$", dockerfile, re.MULTILINE)) - envs = dict(re.findall(r"^ENV\s+([A-Z0-9_]+)=(\S*)\s*$", dockerfile, re.MULTILINE)) - self.assertEqual(args.get("MALLOC_ARENA_MAX"), PINNED_MALLOC_ARENA_MAX) - self.assertEqual(envs.get("MALLOC_ARENA_MAX"), "${MALLOC_ARENA_MAX}") - allocator_envs = { - name for name in envs - if name.startswith("MALLOC_") or name in ("GLIBC_TUNABLES", "PYTHONMALLOC") - } - self.assertEqual(allocator_envs, {"MALLOC_ARENA_MAX"}) - self.assertNotIn("PYTHONMALLOC", dockerfile.replace("# PYTHONMALLOC is deliberately unset", "")) - # Never an empty allocator value: glibc reads "" as 0. - for name, value in {**args, **envs}.items(): - if name.startswith("MALLOC_"): - self.assertNotEqual(value, "", name) - # The ARG precedes the ENV that expands it, and both precede CMD. - self.assertLess(dockerfile.index("ARG MALLOC_ARENA_MAX="), dockerfile.index("ENV MALLOC_ARENA_MAX=")) - self.assertLess(dockerfile.index("ENV MALLOC_ARENA_MAX="), dockerfile.index("CMD [")) - # The deployment notes record the same decision. - deployment = (ROOT / "docs" / "mainnet-deployment.md").read_text(encoding="utf-8") - self.assertIn(f"MALLOC_ARENA_MAX={PINNED_MALLOC_ARENA_MAX}", deployment) - # A dotenv value reaches the container only through the compose - # passthrough, whose default must equal the image default so the - # two cannot drift, and must never be empty (glibc reads "" as 0). - environment = _coordinator_service_environment() - self.assertEqual( - environment.get("MALLOC_ARENA_MAX"), - f"${{MALLOC_ARENA_MAX:-{PINNED_MALLOC_ARENA_MAX}}}", - ) - self.assertEqual(_env_example_values().get("MALLOC_ARENA_MAX"), PINNED_MALLOC_ARENA_MAX) - # glibc honours the variable: a child started with it capped cannot - # grow past the cap however many threads allocate. This is the - # mechanism the soak relies on, proven on this host. - probe = ( - "import ctypes, threading, sys\n" - "sys.path.insert(0, %r)\n" - "from lab.prism.process_telemetry import read_malloc_arena_count\n" - "barrier = threading.Barrier(16)\n" - "def work():\n" - " keep = [bytes(8192) for _ in range(32)]\n" - " barrier.wait()\n" - " del keep\n" - "threads = [threading.Thread(target=work) for _ in range(16)]\n" - "[t.start() for t in threads]; [t.join() for t in threads]\n" - "print(read_malloc_arena_count())\n" - ) % str(ROOT) - if read_malloc_arena_count() < 0: - self.skipTest("malloc_info unavailable on this platform") - capped = subprocess.run( - [sys.executable, "-c", probe], - env={**os.environ, "MALLOC_ARENA_MAX": PINNED_MALLOC_ARENA_MAX}, - capture_output=True, - text=True, - check=True, - ) - uncapped_env = {k: v for k, v in os.environ.items() if k != "MALLOC_ARENA_MAX"} - uncapped = subprocess.run( - [sys.executable, "-c", probe], - env=uncapped_env, - capture_output=True, - text=True, - check=True, - ) - self.assertLessEqual(int(capped.stdout.strip()), int(PINNED_MALLOC_ARENA_MAX)) - self.assertGreater(int(uncapped.stdout.strip()), int(PINNED_MALLOC_ARENA_MAX)) - - - def test_census_switches_pass_through_compose_with_the_shipped_defaults(self) -> None: - """Every switch reaches the coordinator container, defaulting to what the config ships.""" - environment = _coordinator_service_environment() - example = _env_example_values() - expected_defaults = { - "PRISM_HEAP_CENSUS": "0", - "PRISM_HEAP_CENSUS_TOP_N": str(DEFAULT_PRISM_HEAP_CENSUS_TOP_N), - "PRISM_HEAP_CENSUS_MAX_BYTES": str(DEFAULT_PRISM_HEAP_CENSUS_MAX_BYTES), - "PRISM_HEAP_CENSUS_MAX_SECONDS": f"{DEFAULT_PRISM_HEAP_CENSUS_MAX_SECONDS:g}", - "PRISM_HEAP_CENSUS_MIN_INTERVAL_SECONDS": f"{DEFAULT_PRISM_HEAP_CENSUS_MIN_INTERVAL_SECONDS:g}", - "PRISM_HEAP_CENSUS_TRACEMALLOC": "0", - "PRISM_MALLOC_TRIM_SIGNAL": "0", - "PRISM_MALLOC_TRIM_INTERVAL_SECONDS": "0", - } - for name, default in expected_defaults.items(): - with self.subTest(name=name): - self.assertEqual(environment.get(name), f"${{{name}:-{default}}}") - self.assertEqual(example.get(name), default) - # The directory is optional: an empty passthrough must reach the - # coordinator as unset, and the reader must then use the default. - self.assertEqual(environment.get("PRISM_HEAP_CENSUS_DIR"), "${PRISM_HEAP_CENSUS_DIR:-}") - config = env_heap_census_config( - environ={"PRISM_HEAP_CENSUS_DIR": "", "PRISM_AUDIT_DIR": "/var/lib/qbit-prism/audit"} - ) - self.assertEqual(config.output_dir, "/var/lib/qbit-prism/audit/heap-census") - # The defaults the passthroughs carry are the ones the loader - # produces from an empty environment. - shipped = env_heap_census_config(environ={}) - passthrough_environ = { - name: default for name, default in expected_defaults.items() - } - passthrough_environ["PRISM_HEAP_CENSUS_DIR"] = "" - self.assertEqual(env_heap_census_config(environ=passthrough_environ), shipped) - - -class RssBoundCheckTests(unittest.TestCase): - def test_rss_bound_check_fails_when_the_stated_multiple_is_exceeded(self) -> None: - """Synthetic series: the #226 slope fails, a flat post-warm-up series passes.""" - mib = 1024 * 1024 - hour = 3600.0 - # The mainnet shape: 140 MiB at start, +390 MB every hour, for 24 h. - leaking = [(h * hour, int(140 * mib + h * 390_000_000)) for h in range(25)] - verdict = evaluate_rss_bound(leaking, warmup_seconds=hour, multiple=2.0) - self.assertFalse(verdict.passed) - # The baseline is the warm-up peak (the one-hour sample), so the - # bound is twice that and the third-hour sample is the first over it. - self.assertEqual(verdict.baseline_bytes, int(140 * mib + 390_000_000)) - self.assertEqual(verdict.first_breach_at_seconds, 3 * hour) - self.assertGreater(verdict.peak_ratio, 2.0) - self.assertGreater(verdict.slope_bytes_per_hour or 0, 380_000_000) - self.assertTrue(any("exceeded 2x" in reason for reason in verdict.reasons)) - # Bounded: warm-up to 400 MiB (the window materializes), then a - # storm to 613 MiB that drains, then flat. Under 2x of 400 MiB. - bounded = [(0.0, 140 * mib), (0.5 * hour, 400 * mib), (hour, 390 * mib)] - bounded += [(h * hour, 613 * mib if h == 3 else 400 * mib + (h % 3) * mib) for h in range(2, 25)] - verdict = evaluate_rss_bound(bounded, warmup_seconds=hour, multiple=2.0) - self.assertTrue(verdict.passed, verdict.reasons) - self.assertEqual(verdict.baseline_bytes, 400 * mib) - self.assertEqual(verdict.bound_bytes, 800 * mib) - self.assertEqual(verdict.peak_bytes, 613 * mib) - self.assertIsNone(verdict.first_breach_at_seconds) - # The same series fails a tighter multiple, at the storm. - verdict = evaluate_rss_bound(bounded, warmup_seconds=hour, multiple=1.5) - self.assertFalse(verdict.passed) - self.assertEqual(verdict.first_breach_at_seconds, 3 * hour) - # Absolute epoch timestamps are re-based; -1 samples are ignored. - epoch = 1_788_000_000.0 - shifted = [(epoch + t, r) for t, r in bounded] + [(epoch + 5 * hour, -1)] - self.assertTrue(evaluate_rss_bound(shifted, warmup_seconds=hour, multiple=2.0).passed) - # Too short a series, or one with nothing after warm-up, is not a pass. - short = evaluate_rss_bound(bounded[:3], warmup_seconds=hour, multiple=2.0) - self.assertFalse(short.passed) - self.assertIn("no sample after the warm-up window", short.reasons) - brief = evaluate_rss_bound(bounded, warmup_seconds=hour, multiple=2.0, min_span_seconds=48 * hour) - self.assertFalse(brief.passed) - self.assertTrue(any("spans" in reason for reason in brief.reasons)) - self.assertFalse(evaluate_rss_bound([], warmup_seconds=hour, multiple=2.0).passed) - # The optional slope guard catches a slow leak that stays under 2x - # (15 MiB/h from a 420 MiB warm-up peak ends the day at 760 MiB). - slow = [(h * hour, int(400 * mib + h * 15 * mib)) for h in range(25)] - self.assertTrue(evaluate_rss_bound(slow, warmup_seconds=hour, multiple=2.0).passed) - guarded = evaluate_rss_bound( - slow, warmup_seconds=hour, multiple=2.0, max_slope_bytes_per_hour=10 * mib - ) - self.assertFalse(guarded.passed) - self.assertTrue(any("slope" in reason for reason in guarded.reasons)) - with self.assertRaises(ValueError): - evaluate_rss_bound(bounded, warmup_seconds=hour, multiple=1.0) - - def test_rss_bound_cli_exit_status_is_the_verdict(self) -> None: - """The operator's command exits 0 on pass, 1 on fail, 2 on bad input.""" - mib = 1024 * 1024 - with tempfile.TemporaryDirectory() as directory: - passing = Path(directory) / "pass.csv" - passing.write_text( - "# epoch_seconds,rss_bytes\n" - + "".join(f"{h * 3600},{400 * mib}\n" for h in range(25)), - encoding="utf-8", - ) - failing = Path(directory) / "fail.csv" - failing.write_text( - "".join(f"{h * 3600},{140 * mib + h * 390_000_000}\n" for h in range(25)), - encoding="utf-8", - ) - broken = Path(directory) / "broken.csv" - broken.write_text("not,a,number\n", encoding="utf-8") - base = [sys.executable, "-m", "lab.prism.process_telemetry", "rss-bound"] - ok = subprocess.run( - [*base, "--samples", str(passing)], - cwd=ROOT, - capture_output=True, - text=True, - ) - self.assertEqual(ok.returncode, 0, ok.stderr) - self.assertTrue(json.loads(ok.stdout)["passed"]) - bad = subprocess.run( - [*base, "--samples", str(failing)], - cwd=ROOT, - capture_output=True, - text=True, - ) - self.assertEqual(bad.returncode, 1, bad.stderr) - self.assertFalse(json.loads(bad.stdout)["passed"]) - self.assertEqual(json.loads(bad.stdout)["first_breach_at_seconds"], 10800.0) - usage = subprocess.run( - [*base, "--samples", str(broken)], - cwd=ROOT, - capture_output=True, - text=True, - ) - self.assertEqual(usage.returncode, 2) - sample = subprocess.run( - [sys.executable, "-m", "lab.prism.process_telemetry", "rss-sample", "--pid", str(os.getpid())], - cwd=ROOT, - capture_output=True, - text=True, - ) - epoch, rss = sample.stdout.strip().split(",") - self.assertGreater(int(epoch), 0) - if _HAS_PROCFS: - self.assertEqual(sample.returncode, 0, sample.stderr) - self.assertGreater(int(rss), 0) - else: - # Documented contract off Linux: the -1 sentinel and exit 1, - # so a capture loop cannot mistake "unavailable" for a value. - self.assertEqual(sample.returncode, 1, sample.stderr) - self.assertEqual(int(rss), -1) - - -if __name__ == "__main__": - unittest.main() diff --git a/tests/test_prism_hot_path.py b/tests/test_prism_hot_path.py deleted file mode 100644 index 3e4b369e6..000000000 --- a/tests/test_prism_hot_path.py +++ /dev/null @@ -1,978 +0,0 @@ -#!/usr/bin/env python3 -"""Hot-path regressions behind the 2026-07-16 stale/unknown-job reject outage. - -Covers the coordinator-side commitments: -- mining.submit classifies against the poller/blockwait-observed tip instead - of a blocking per-share getbestblockhash RPC, with a freshness bound that - falls back to the RPC when tip observation stalls (never fail-open on a - frozen snapshot); -- per-share / per-job stdout logging is debug-gated; -- prepared fanout passes perform O(1) live validations regardless of client - count (the base branch's validation-token architecture keeps per-client - deliveries RPC-free; pinned here as an outage regression guard). -""" - -from __future__ import annotations - -import contextlib -from concurrent.futures import ThreadPoolExecutor -import io -import threading -import time -import unittest -from decimal import Decimal -from unittest import mock - -from lab.auxpow import vardiff -from lab.prism import direct_stratum -from lab.prism.prism_coordinator import ( - ClientState, - PRISM_REJECTION_REASON_IDS, - PRISM_REJECTION_STALE_JOB, - PrismCoordinator, - StratumError, - TemplateRefreshBlocked, - WorkerIdentity, -) -from lab.prism.coordinator_config import default_prism_coinbase_tag_hex -from lab.prism.share_ledger import AcceptedShareRecord, PendingShare - -PAYOUT_ADDRESS = "tq1z70ukpvs96kye6jmgvl3nttevtkrq8uu89snkpm6m8gwqukw8u5dsz32kwa" -EXTRANONCE2_SIZE = 8 -OLD_TIP = "11" * 32 -NEW_TIP = "33" * 32 - - -class FakeShare: - def __init__(self, miner_id: str, share_seq: int) -> None: - self.miner_id = miner_id - self.share_seq = share_seq - - def to_prism_json(self) -> dict[str, object]: - return {"share_seq": self.share_seq, "miner_id": self.miner_id} - - -class FakeAppendLedger: - backend_name = "fake" - - def __init__(self) -> None: - self.miners = ["miner-a", "miner-b", "miner-c"] - self.appended: list[PendingShare] = [] - - def accepted_share_stats(self) -> dict[str, int]: - return { - "accepted_share_count": len(self.miners) + len(self.appended), - "distinct_miner_count": len(self.miners), - } - - def snapshot_at_job_issue( - self, - anchor_job_issued_at_ms: int, - *, - window_weight: int | None = None, - ) -> list[FakeShare]: - return [ - FakeShare(miner_id=miner, share_seq=seq + 1) - for seq, miner in enumerate(self.miners) - ] - - def current_prior_balances(self) -> list[dict[str, object]]: - return [] - - def append_batch( - self, - entries: list[tuple[PendingShare, dict[str, object] | None]], - ) -> list[AcceptedShareRecord]: - records = [] - for pending, _candidate in entries: - self.appended.append(pending) - records.append( - AcceptedShareRecord( - share_seq=len(self.appended), - share_id=pending.share_id, - miner_id=pending.miner_id, - order_key=pending.order_key, - p2mr_program_hex=pending.p2mr_program_hex, - share_difficulty=pending.share_difficulty, - network_difficulty=pending.network_difficulty, - template_height=pending.template_height, - job_id=pending.job_id, - job_issued_at_ms=pending.job_issued_at_ms, - accepted_at_ms=pending.accepted_at_ms, - ntime=pending.ntime, - credit_policy=pending.credit_policy, - ) - ) - return records - - def append(self, pending: PendingShare) -> AcceptedShareRecord: - return self.append_batch([(pending, None)])[0] - - def metrics(self) -> dict[str, int]: - return {"blocks": 0, "owed_accounts": 0} - - -class FakeRpc: - def __init__(self, template: dict[str, object], tip: str) -> None: - self.template = template - self.tip = tip - self.blockchain_info: dict[str, object] = { - "initialblockdownload": False, - "blocks": 100, - "headers": 100, - } - self.calls: list[str] = [] - - def call(self, method: str, params: list[object] | None = None, **_kwargs: object) -> object: - self.calls.append(method) - if method == "getblocktemplate": - return dict(self.template) - if method == "getbestblockhash": - return self.tip - if method == "getblockchaininfo": - return dict(self.blockchain_info) - if method == "getblockcount": - return int(self.blockchain_info["blocks"]) - if method == "getblock": - return {"previousblockhash": OLD_TIP} - raise AssertionError(f"unexpected RPC {method}") - - def count(self, method: str) -> int: - return sum(1 for name in self.calls if name == method) - - -class FakeSock: - def __init__(self) -> None: - self.sent: list[bytes] = [] - self._lock = threading.Lock() - - def sendall(self, data: bytes) -> None: - with self._lock: - self.sent.append(data) - - def shutdown(self, how: int) -> None: - return None - - def close(self) -> None: - return None - - -class CountingRLock: - """RLock test double that counts acquisitions by the calling thread.""" - - def __init__(self) -> None: - self._lock = threading.RLock() - self._local = threading.local() - self.first_acquisition = threading.Event() - - def acquire(self, *args: object, **kwargs: object) -> bool: - acquired = self._lock.acquire(*args, **kwargs) - if acquired: - self._local.acquisitions = self.acquisitions_for_current_thread() + 1 - self.first_acquisition.set() - return acquired - - def release(self) -> None: - self._lock.release() - - def __enter__(self) -> CountingRLock: - self.acquire() - return self - - def __exit__(self, *_args: object) -> None: - self.release() - - def acquisitions_for_current_thread(self) -> int: - return int(getattr(self._local, "acquisitions", 0)) - - -class NotifyingRLock: - """RLock test double that exposes when another path attempts admission.""" - - def __init__(self) -> None: - self._lock = threading.RLock() - self.attempted = threading.Event() - - def acquire(self, *args: object, **kwargs: object) -> bool: - self.attempted.set() - return self._lock.acquire(*args, **kwargs) - - def release(self) -> None: - self._lock.release() - - def __enter__(self) -> NotifyingRLock: - self.acquire() - return self - - def __exit__(self, *_args: object) -> None: - self.release() - - -def base_template(height: int = 10, prevhash: str = OLD_TIP) -> dict[str, object]: - return { - "height": height, - "previousblockhash": prevhash, - "bits": "1b00ffff", - "version": 0x20000000, - "curtime": 1_700_000_000, - "coinbasevalue": 50_00000000, - "transactions": [], - } - - -def worker(payout: str = PAYOUT_ADDRESS) -> WorkerIdentity: - return WorkerIdentity( - username=payout, - payout_address=payout, - worker_name=None, - script_pubkey_hex="5220" + "22" * 32, - p2mr_program_hex="22" * 32, - ) - - -def client(connection_id: int) -> ClientState: - state = ClientState.__new__(ClientState) - state.sock = FakeSock() - state.address = ("127.0.0.1", 40_000 + connection_id) - state.connection_id = connection_id - state.extranonce1_hex = f"{connection_id:08x}" - state.subscribed = True - state.authorized = True - identity = worker() - state.username = identity.username - state.worker = identity - state.version_mask = 0 - state.active_job = None - state.share_difficulty = Decimal("0.000000001") - state.pending_share_difficulty = None - state.minimum_advertised_difficulty = Decimal("0") - state.vardiff_config = None - state.listener_vardiff_config = None - state.requested_difficulty = None - state.suggested_difficulty = None - state.requested_min_difficulty = None - state.vardiff_window_started_monotonic = time.monotonic() - state.vardiff_window_accepted = 0 - state.vardiff_window_submitted = 0 - state.vardiff_window_work = Decimal("0") - state.vardiff_difficulty_estimate = None - state.vardiff_lock = threading.RLock() - state.active_job_ids = set() - state.post_accept_refresh_block = None - state.tip_work_delivered = None - state.job_update_lock = threading.RLock() - state.send_lock = threading.Lock() - return state - - -def coordinator( - *, - template: dict[str, object] | None = None, -) -> tuple[PrismCoordinator, FakeRpc]: - server = PrismCoordinator.__new__(PrismCoordinator) - template = template or base_template() - rpc = FakeRpc(template, tip=str(template["previousblockhash"])) - server.rpc = rpc - server.qbit_chain = "regtest" - server.lock = threading.RLock() - server._recent_share_lock = threading.Lock() - server._share_accounting_lock = threading.Lock() - server.stop_event = threading.Event() - server.clients = set() - server.jobs = {} - server.job_counter = 0 - server.connection_counter = 0 - server.accepted_block_count = 0 - server.max_blocks = 1_000 - server.started_monotonic = time.monotonic() - server.submitted_share_count = 0 - server.stale_share_count = 0 - server.duplicate_share_count = 0 - server.low_difficulty_share_count = 0 - server.rejection_counts_by_reason = {reason: 0 for reason in PRISM_REJECTION_REASON_IDS} - server.job_build_failure_count = 0 - server.tip_refresh_job_count = 0 - server.post_accept_refresh_failure_count = 0 - server.reorg_reconciler_enabled = False - server.reorg_inactive_block_count = 0 - server.reorg_reactivated_block_count = 0 - server.reorg_reconcile_skip_count = 0 - server.reorg_reconcile_error_count = 0 - server.matured_payout_count = 0 - server.last_reorg_reconciled_tip_hash = None - server.last_reorg_reconciled_trusted = False - server.last_reorg_reconciled_monotonic = None - server.latest_evidence = None - server.latest_coinbase_size_bytes = None - server.tip_template_snapshot = None - server.recent_share_keys = set() - server.extranonce2_size = EXTRANONCE2_SIZE - server.coinbase_tag_hex = default_prism_coinbase_tag_hex() - server.share_difficulty = Decimal("0.000000001") - server.vardiff_config = vardiff.VardiffConfig( - enabled=True, - target_share_interval_seconds=Decimal("15"), - min_difficulty=Decimal("0.000000001"), - max_difficulty=Decimal("1024"), - retarget_interval_seconds=Decimal("90"), - max_step_factor=Decimal("4"), - startup_difficulty=Decimal("0.000000001"), - max_step_down_factor=Decimal("4"), - ewma_alpha=Decimal("0.4"), - retarget_tolerance=Decimal("0.25"), - ) - server.default_share_weight = 1 - server.share_weights_by_username = {} - server.min_ready_miners = 3 - server.ledger = FakeAppendLedger() - server.blockpoll_seconds = 2.0 - # Failed-refresh spacing is opt-in per test: its holdoff waits on real - # time, which deadlocks tests that freeze time.monotonic around failing - # polls. Pacing behavior is covered by test_prism_refresh_retry_pacing. - server.tip_refresh_failure_holdoff_seconds = 0.0 - server.job_bundle_cache_seconds = 10.0 - server.template_cache_seconds = 2.0 - server.reorg_reconcile_cache_seconds = 5.0 - server.health_refresh_seconds = 5.0 - server.stratum_send_timeout_seconds = 20.0 - server.stale_grace_seconds = 0.0 - server.hot_path_log_enabled = False - server.share_writer_active = False - server._ensure_job_cache_state() - return server, rpc - - -def synthetic_manifest_coinbase_hex(suffix_hex: str) -> str: - height_push = "03aabbcc" - script_sig = height_push + suffix_hex - script_sig_bytes = bytes.fromhex(script_sig) - output = (50_00000000).to_bytes(8, "little").hex() + "0151" - return ( - "01000000" - + "01" - + "00" * 32 - + "ffffffff" - + direct_stratum.compact_size(len(script_sig_bytes)).hex() - + script_sig - + "ffffffff" - + "01" - + output - + "00000000" - ) - - -def install_fake_bundle_builder(server: PrismCoordinator) -> dict[str, object]: - recorded: dict[str, object] = {"calls": 0} - - def fake_build_audit_bundle(**kwargs: object) -> dict[str, object]: - recorded["calls"] = int(recorded["calls"]) + 1 - suffix_hex = str(kwargs["coinbase_script_sig_suffix_hex"]) - return { - "found_block": dict(kwargs["found_block"]), - "signed_coinbase_manifest": { - "manifest": { - "coinbase_tx_hex": synthetic_manifest_coinbase_hex(suffix_hex), - } - }, - } - - server.build_audit_bundle = fake_build_audit_bundle # type: ignore[method-assign] - server._ensure_bundle_compiler().build_audit_bundle = ( # type: ignore[method-assign] - fake_build_audit_bundle - ) - return recorded - - -def register_job(server: PrismCoordinator, state: ClientState) -> object: - context = server.build_job_for_client(state, clean_jobs=True) - state.active_job = context - with server.lock: - server.jobs[context.job.job_id] = context - state.active_job_ids.add(context.job.job_id) - return context - - -def observe_tip(server: PrismCoordinator, tip_hash: str, *, age_seconds: float = 0.0) -> None: - """Install an observed tip the way blockpoll/blockwait would.""" - with server.lock: - server.current_tip_first_seen = (tip_hash, None) - server.current_tip_observed_monotonic = time.monotonic() - age_seconds - - -def submit_params(state: ClientState, context: object) -> list[object]: - """Params for a share that deterministically passes the share target - without also being a block solution.""" - extranonce2_hex = "11" * EXTRANONCE2_SIZE - for nonce in range(200_000): - nonce_hex = f"{nonce:08x}" - submission = direct_stratum.assemble_submission( - context.job, - extranonce2_hex=extranonce2_hex, - ntime_hex=context.job.ntime, - nonce_hex=nonce_hex, - ) - if submission.share_pass and not submission.block_pass: - return [ - state.username, - context.job.job_id, - extranonce2_hex, - context.job.ntime, - nonce_hex, - ] - raise AssertionError("no share-passing nonce found") - - -class SubmitTipCheckTests(unittest.TestCase): - def test_submit_tip_selection_is_atomic_with_observation_update(self) -> None: - server, rpc = coordinator() - observed_at = time.monotonic() - with server.lock: - server.current_tip_first_seen = (OLD_TIP, None) - server.current_tip_observed_monotonic = observed_at - - freshness_check_started = threading.Event() - release_freshness_check = threading.Event() - tip_update_finished = threading.Event() - selected: list[str] = [] - errors: list[BaseException] = [] - real_monotonic = time.monotonic - - def controlled_monotonic() -> float: - if threading.current_thread().name == "submit-tip": - freshness_check_started.set() - if not release_freshness_check.wait(5): - raise AssertionError("freshness check was not released") - return observed_at - return real_monotonic() - - def select_tip() -> None: - try: - selected.append(server.submit_stale_check_tip()) - except BaseException as exc: # noqa: BLE001 - surfaced below - errors.append(exc) - - def update_tip() -> None: - with server.lock: - server.current_tip_first_seen = (NEW_TIP, real_monotonic()) - server.current_tip_observed_monotonic = real_monotonic() - tip_update_finished.set() - - submit_thread = threading.Thread(target=select_tip, name="submit-tip") - update_thread = threading.Thread(target=update_tip, name="tip-observer") - with mock.patch( - "lab.prism.prism_coordinator.time.monotonic", - side_effect=controlled_monotonic, - ): - submit_thread.start() - try: - self.assertTrue(freshness_check_started.wait(5)) - update_thread.start() - # A new observation cannot supersede the selected hash before - # submit_stale_check_tip returns its point-in-time result. - self.assertFalse(tip_update_finished.wait(0.1)) - finally: - release_freshness_check.set() - submit_thread.join(5) - if update_thread.ident is not None: - update_thread.join(5) - - self.assertFalse(submit_thread.is_alive()) - self.assertFalse(update_thread.is_alive()) - self.assertEqual(errors, []) - self.assertEqual(selected, [OLD_TIP]) - self.assertTrue(tip_update_finished.is_set()) - self.assertEqual(rpc.count("getbestblockhash"), 0) - - def test_submit_uses_observed_tip_without_rpc(self) -> None: - server, rpc = coordinator() - install_fake_bundle_builder(server) - state = client(1) - context = register_job(server, state) - observe_tip(server, OLD_TIP) - rpc.calls.clear() - - accepted_and_closed = server.handle_submit(state, submit_params(state, context)) - - self.assertFalse(accepted_and_closed) - self.assertEqual(rpc.count("getbestblockhash"), 0) - self.assertEqual(len(server.ledger.appended), 1) - - def test_submit_rejects_stale_job_from_observed_tip_without_rpc(self) -> None: - server, rpc = coordinator() - install_fake_bundle_builder(server) - state = client(1) - context = register_job(server, state) - params = submit_params(state, context) - observe_tip(server, NEW_TIP) - rpc.calls.clear() - - with self.assertRaises(StratumError) as caught: - server.handle_submit(state, params) - - self.assertEqual(caught.exception.reason, PRISM_REJECTION_STALE_JOB) - self.assertEqual(rpc.count("getbestblockhash"), 0) - self.assertEqual(server.rejection_counts_by_reason[PRISM_REJECTION_STALE_JOB], 1) - self.assertEqual(len(server.ledger.appended), 0) - - def test_submit_falls_back_to_rpc_without_observation(self) -> None: - server, rpc = coordinator() - install_fake_bundle_builder(server) - state = client(1) - context = register_job(server, state) - self.assertIsNone(getattr(server, "current_tip_first_seen", None)) - params = submit_params(state, context) - rpc.calls.clear() - - server.handle_submit(state, params) - - self.assertEqual(rpc.count("getbestblockhash"), 1) - self.assertEqual(len(server.ledger.appended), 1) - - def test_submit_falls_back_to_rpc_when_observation_goes_stale(self) -> None: - # The Codex P1 regression: a poller that stops refreshing after a tip - # change must not let submits classify against the frozen snapshot - # forever. Once the observation exceeds its freshness budget the - # submit path re-reads the live tip. - server, rpc = coordinator() - install_fake_bundle_builder(server) - server.submit_tip_max_age_seconds = 10.0 - state = client(1) - context = register_job(server, state) - params = submit_params(state, context) - # Observation is stale, and the live tip has moved on: the share must - # be classified against the RPC tip and rejected as stale. - observe_tip(server, OLD_TIP, age_seconds=11.0) - rpc.tip = NEW_TIP - rpc.calls.clear() - - with self.assertRaises(StratumError) as caught: - server.handle_submit(state, params) - - self.assertEqual(caught.exception.reason, PRISM_REJECTION_STALE_JOB) - self.assertEqual(rpc.count("getbestblockhash"), 1) - self.assertEqual(len(server.ledger.appended), 0) - - def test_submit_keeps_published_tip_during_bounded_replacement_build(self) -> None: - server, rpc = coordinator() - install_fake_bundle_builder(server) - server.submit_tip_max_age_seconds = 10.0 - server.template_refresh_failure_exit_seconds = 120.0 - state = client(1) - context = register_job(server, state) - params = submit_params(state, context) - # The ordinary 10-second observation freshness has expired, but a new - # tip is only detected: replacement work has not been published yet. - observe_tip(server, OLD_TIP, age_seconds=11.0) - rpc.tip = NEW_TIP - self.assertTrue(server.observe_tip_for_refresh(NEW_TIP)) - rpc.calls.clear() - - server.handle_submit(state, params) - - self.assertEqual(rpc.count("getbestblockhash"), 0) - self.assertEqual(len(server.ledger.appended), 1) - self.assertEqual(server.current_tip_first_seen[0], OLD_TIP) - self.assertEqual(server.latest_detected_tip[0], NEW_TIP) - - def test_unpublished_divergence_lease_does_not_renew_and_expires(self) -> None: - server, rpc = coordinator() - server.submit_tip_max_age_seconds = 10.0 - server.template_refresh_failure_exit_seconds = 120.0 - observe_tip(server, OLD_TIP, age_seconds=11.0) - rpc.tip = NEW_TIP - self.assertTrue(server.observe_tip_for_refresh(NEW_TIP)) - divergence_started = server.tip_refresh_divergence_started_monotonic - self.assertIsNotNone(divergence_started) - - newest_tip = "44" * 32 - rpc.tip = newest_tip - self.assertTrue(server.observe_tip_for_refresh(newest_tip)) - self.assertEqual( - server.tip_refresh_divergence_started_monotonic, - divergence_started, - ) - with server.lock: - server.tip_refresh_divergence_started_monotonic = ( - time.monotonic() - 121.0 - ) - rpc.calls.clear() - - self.assertEqual(server.submit_stale_check_tip(), newest_tip) - self.assertEqual(rpc.count("getbestblockhash"), 1) - - def test_submit_tip_max_age_zero_restores_per_share_rpc(self) -> None: - server, rpc = coordinator() - install_fake_bundle_builder(server) - server.submit_tip_max_age_seconds = 0.0 - state = client(1) - context = register_job(server, state) - params = submit_params(state, context) - observe_tip(server, OLD_TIP) - rpc.calls.clear() - - server.handle_submit(state, params) - - self.assertEqual(rpc.count("getbestblockhash"), 1) - - -class HotPathLoggingTests(unittest.TestCase): - def drive_job_and_share(self, server: PrismCoordinator, rpc: FakeRpc) -> None: - state = client(1) - self.assertTrue(server.maybe_send_job(state, clean_jobs=True)) - context = state.active_job - observe_tip(server, OLD_TIP) - server.handle_submit(state, submit_params(state, context)) - - def test_per_share_and_per_job_prints_off_by_default(self) -> None: - server, rpc = coordinator() - install_fake_bundle_builder(server) - stdout = io.StringIO() - with contextlib.redirect_stdout(stdout): - self.drive_job_and_share(server, rpc) - output = stdout.getvalue() - self.assertNotIn("building job", output) - self.assertNotIn("sent job", output) - self.assertNotIn("accepted share", output) - - def test_per_share_and_per_job_prints_enabled_by_flag(self) -> None: - server, rpc = coordinator() - install_fake_bundle_builder(server) - server.hot_path_log_enabled = True - stdout = io.StringIO() - with contextlib.redirect_stdout(stdout): - self.drive_job_and_share(server, rpc) - output = stdout.getvalue() - self.assertIn("building job connection=1", output) - self.assertIn("sent job connection=1", output) - self.assertIn("accepted share seq=1", output) - - -class HotPathLockIsolationTests(unittest.TestCase): - def test_first_touch_installs_one_hot_path_lock_set(self) -> None: - server = PrismCoordinator.__new__(PrismCoordinator) - server.extranonce2_size = EXTRANONCE2_SIZE - state = ClientState.__new__(ClientState) - start = threading.Barrier(32) - - def initialize(_index: int) -> tuple[object, object, object, bool]: - start.wait(timeout=2) - vardiff_lock = server._client_vardiff_lock(state) - reserved = server._reserve_recent_share_key(("same", "header")) - return ( - vardiff_lock, - server._ensure_share_submission_service().recent_shares, - server._share_accounting_lock, - reserved, - ) - - with ThreadPoolExecutor(max_workers=32) as executor: - results = list(executor.map(initialize, range(32))) - - first = results[0] - self.assertTrue(all(result[0] is first[0] for result in results)) - self.assertTrue(all(result[1] is first[1] for result in results)) - self.assertTrue(all(result[2] is first[2] for result in results)) - self.assertEqual(sum(result[3] for result in results), 1) - - def test_idle_vardiff_wait_cannot_hold_coordinator_lock(self) -> None: - server, _rpc = coordinator() - state = client(1) - vardiff_lock = NotifyingRLock() - state.vardiff_lock = vardiff_lock # type: ignore[assignment] - active_job = object() - state.active_job = active_job - server.clients.add(state) - server.latest_detected_tip = None - request = type("IdleRequest", (), {})() - request.client = state - request.connection_id = state.connection_id - request.worker = state.worker - request.active_job = active_job - request.current_difficulty = state.share_difficulty - request.window_started_monotonic = state.vardiff_window_started_monotonic - attempting = threading.Event() - - def validate_idle_request() -> None: - attempting.set() - server._idle_request_skip_reason(request) - - with vardiff_lock: - vardiff_lock.attempted.clear() - validator = threading.Thread(target=validate_idle_request) - validator.start() - self.assertTrue(attempting.wait(1)) - self.assertTrue(vardiff_lock.attempted.wait(1)) - acquired = server.lock.acquire(timeout=0.25) - self.assertTrue(acquired) - if acquired: - server.lock.release() - validator.join(1) - self.assertFalse(validator.is_alive()) - - def test_job_delivery_vardiff_wait_cannot_hold_coordinator_lock(self) -> None: - server, _rpc = coordinator() - install_fake_bundle_builder(server) - state = client(1) - vardiff_lock = NotifyingRLock() - state.vardiff_lock = vardiff_lock # type: ignore[assignment] - - with vardiff_lock: - vardiff_lock.attempted.clear() - sender = threading.Thread( - target=lambda: server.maybe_send_job(state, clean_jobs=True), - ) - sender.start() - self.assertTrue(vardiff_lock.attempted.wait(2)) - acquired = server.lock.acquire(timeout=0.25) - self.assertTrue(acquired) - if acquired: - server.lock.release() - sender.join(2) - self.assertFalse(sender.is_alive()) - - def test_real_submissions_cannot_starve_tip_publication(self) -> None: - """A 600-client/100-submit burst admits tip publication promptly.""" - server, _rpc = coordinator() - install_fake_bundle_builder(server) - clients = [client(index + 1) for index in range(600)] - contexts = [register_job(server, state) for state in clients] - with server.lock: - server.clients.update(clients) - params = [ - submit_params(state, context) - for state, context in zip(clients[:100], contexts[:100], strict=True) - ] - counting_lock = CountingRLock() - server.lock = counting_lock # type: ignore[assignment] - - # Hold accepted submissions after their single control snapshot so tip - # publication is guaranteed to compete with a live submit burst, not - # merely start after fast in-memory ledger commits have all finished. - ledger_entered = threading.Event() - release_ledger = threading.Event() - append_batch = server.ledger.append_batch - - def blocking_append_batch( - entries: list[tuple[PendingShare, dict[str, object] | None]], - ) -> list[AcceptedShareRecord]: - ledger_entered.set() - if not release_ledger.wait(2): - raise AssertionError("tip publication did not release share commits") - return append_batch(entries) - - server.ledger.append_batch = blocking_append_batch # type: ignore[method-assign] - - start = threading.Event() - ready = threading.Event() - active_lock = threading.Lock() - ready_count = 0 - active_count = 0 - active_during_publication = 0 - publication_wait = float("inf") - - def submit_share(index: int) -> int: - nonlocal ready_count, active_count - with active_lock: - ready_count += 1 - active_count += 1 - if ready_count == 64: - ready.set() - self.assertTrue(start.wait(2)) - before = counting_lock.acquisitions_for_current_thread() - try: - try: - server.handle_submit(clients[index], params[index]) - except StratumError as exc: - # Submits whose admission snapshot follows the publication - # correctly observe the old job as stale. - if exc.reason != PRISM_REJECTION_STALE_JOB: - raise - return counting_lock.acquisitions_for_current_thread() - before - finally: - with active_lock: - active_count -= 1 - - def publish_tip_state() -> None: - nonlocal active_during_publication, publication_wait - try: - self.assertTrue(ledger_entered.wait(2)) - started = time.monotonic() - with counting_lock: - publication_wait = time.monotonic() - started - server.current_tip_first_seen = (NEW_TIP, time.monotonic()) - server.current_tip_observed_monotonic = time.monotonic() - with active_lock: - active_during_publication = active_count - finally: - release_ledger.set() - - with ThreadPoolExecutor( - max_workers=64, - thread_name_prefix="share-submit", - ) as executor: - futures = [executor.submit(submit_share, index) for index in range(100)] - self.assertTrue(ready.wait(2)) - tip_thread = threading.Thread( - target=publish_tip_state, - name="tip-publication", - ) - tip_thread.start() - start.set() - acquisitions_per_submit = [future.result(timeout=5) for future in futures] - tip_thread.join(2) - - self.assertFalse(tip_thread.is_alive()) - self.assertGreater(active_during_publication, 0) - self.assertLess(publication_wait, 0.5) - # Before the consolidated snapshot, every normal handle_submit crossed - # this lock separately for pool state, job lookup, and tip selection. - self.assertEqual(acquisitions_per_submit, [1] * 100) - self.assertEqual(server.submitted_share_count, 100) - - def test_concurrent_vardiff_accounting_is_exact_without_coordinator_lock(self) -> None: - server, _rpc = coordinator() - state = client(1) - state.vardiff_window_started_monotonic = time.monotonic() - fake_job = type("FakeJob", (), {"share_difficulty": Decimal("2")})() - - def account(_index: int) -> None: - server.note_vardiff_submitted_share(state) - server.note_vardiff_accepted_share(state, fake_job) # type: ignore[arg-type] - - with ThreadPoolExecutor(max_workers=32) as executor: - list(executor.map(account, range(1_000))) - - with state.vardiff_lock: - self.assertEqual(state.vardiff_window_submitted, 1_000) - self.assertEqual(state.vardiff_window_accepted, 1_000) - self.assertEqual(state.vardiff_window_work, Decimal("2000")) - self.assertEqual(server.submitted_share_count, 1_000) - - def test_slow_stratum_socket_cannot_hold_coordinator_lock(self) -> None: - server, _rpc = coordinator() - install_fake_bundle_builder(server) - state = client(1) - send_started = threading.Event() - release_send = threading.Event() - - def slow_send(_payload: dict[str, object]) -> None: - send_started.set() - self.assertTrue(release_send.wait(2)) - - state.send = slow_send # type: ignore[method-assign] - sender = threading.Thread( - target=lambda: server.maybe_send_job(state, clean_jobs=True) - ) - sender.start() - try: - self.assertTrue(send_started.wait(2)) - acquired = server.lock.acquire(timeout=0.25) - self.assertTrue(acquired) - if acquired: - server.lock.release() - finally: - release_send.set() - sender.join(5) - self.assertFalse(sender.is_alive()) - - -class FanOutRpcEconomyTests(unittest.TestCase): - """Outage regression guard: a refresh pass's RPC cost must not scale with - the number of connected clients. The base branch's validation-token - architecture performs O(1) live validations per pass and keeps the - per-client delivery path RPC-free; this pins that property against the - 2026-07-16 failure mode (two serialized RPC round trips per client).""" - - def run_pass(self, client_count: int) -> FakeRpc: - server, rpc = coordinator() - install_fake_bundle_builder(server) - states = [client(index + 1) for index in range(client_count)] - server.clients = set(states) - try: - refreshed = server.poll_qbit_tip_template_once() - finally: - server.shutdown_tip_refresh_executor() - self.assertEqual(refreshed, client_count) - for state in states: - self.assertIsNotNone(state.active_job) - return rpc - - def test_refresh_pass_rpc_count_is_independent_of_client_count(self) -> None: - small = self.run_pass(1) - large = self.run_pass(8) - for method in ( - "getbestblockhash", - "getblocktemplate", - "getblockchaininfo", - "getblockcount", - "getblock", - ): - self.assertEqual(small.count(method), large.count(method), method) - - def test_driver_live_trust_check_aborts_queued_delivery(self) -> None: - server, rpc = coordinator() - install_fake_bundle_builder(server) - server.tip_refresh_max_workers = 1 - server.reorg_reconciler_enabled = True - - def reconcile_live_chain_view(tip_hash: str) -> bool: - self.assertEqual(tip_hash, OLD_TIP) - return not server.qbit_chain_view_untrusted() - - server.ensure_reorg_reconciled_for_tip = reconcile_live_chain_view # type: ignore[method-assign] - first = client(1) - second = client(2) - server.clients = [first, second] # type: ignore[assignment] - first_delivery_started = threading.Event() - release_first_delivery = threading.Event() - second_notifications: list[dict[str, object]] = [] - - def block_first_delivery(payload: dict[str, object]) -> None: - if payload["method"] == "mining.notify": - first_delivery_started.set() - self.assertTrue(release_first_delivery.wait(10)) - - first.send = block_first_delivery # type: ignore[method-assign] - second.send = second_notifications.append # type: ignore[method-assign] - refreshed: list[int] = [] - errors: list[BaseException] = [] - - def poll() -> None: - try: - refreshed.append(server.poll_qbit_tip_template_once()) - except BaseException as exc: # noqa: BLE001 - surfaced below - errors.append(exc) - - poll_thread = threading.Thread(target=poll) - poll_thread.start() - try: - self.assertTrue(first_delivery_started.wait(5)) - chain_info_calls_before_mismatch = rpc.count("getblockchaininfo") - rpc.blockchain_info["headers"] = 101 - deadline = time.monotonic() + 5 - while ( - rpc.count("getblockchaininfo") <= chain_info_calls_before_mismatch - and time.monotonic() < deadline - ): - time.sleep(0.02) - self.assertGreater( - rpc.count("getblockchaininfo"), - chain_info_calls_before_mismatch, - "driver live trust check never ran", - ) - finally: - release_first_delivery.set() - poll_thread.join(10) - server.shutdown_tip_refresh_executor() - - self.assertFalse(poll_thread.is_alive()) - self.assertEqual(refreshed, []) - self.assertEqual(len(errors), 1) - self.assertIsInstance(errors[0], TemplateRefreshBlocked) - self.assertIn("chain view became untrusted", str(errors[0])) - self.assertEqual(second_notifications, []) - self.assertIsNone(second.active_job) - - -if __name__ == "__main__": - unittest.main() diff --git a/tests/test_prism_immutable_refresh_artifacts.py b/tests/test_prism_immutable_refresh_artifacts.py deleted file mode 100644 index f6af607b3..000000000 --- a/tests/test_prism_immutable_refresh_artifacts.py +++ /dev/null @@ -1,303 +0,0 @@ -#!/usr/bin/env python3 -"""Expected behavior for immutable PRISM tip-refresh artifacts.""" - -from __future__ import annotations - -import threading -import unittest -from dataclasses import replace as dataclass_replace - -from lab.prism.prism_coordinator import ( - CachedJobBundle, - CachedTemplateArtifacts, - ClientState, - TemplateRefreshBlocked, - WorkerIdentity, - qbit_template_fingerprint, -) -from tests.prism_coordinator_test_support import ( - base_template, - client, - coordinator, - install_fake_bundle_builder, -) - - -class ImmutableRefreshArtifactTests(unittest.TestCase): - def test_fetched_snapshot_owns_exact_derived_artifacts(self) -> None: - server, _ = coordinator() - original_store = server.store_template_artifacts - stored: list[CachedTemplateArtifacts | None] = [] - - def recording_store( - template: dict[str, object], - *, - generation: int | None = None, - ) -> CachedTemplateArtifacts | None: - artifacts = original_store(template, generation=generation) - stored.append(artifacts) - return artifacts - - server.store_template_artifacts = recording_store # type: ignore[method-assign] - - snapshot = server.fetch_qbit_tip_template_snapshot() - - self.assertEqual(len(stored), 1) - self.assertIsNotNone(stored[0]) - self.assertIs(snapshot.template_artifacts, stored[0]) - assert snapshot.template_artifacts is not None - self.assertEqual( - snapshot.template_fingerprint, - snapshot.template_artifacts.fingerprint, - ) - self.assertEqual( - snapshot.template_generation, - snapshot.template_artifacts.generation, - ) - self.assertEqual( - snapshot.previousblockhash, - snapshot.template_artifacts.previousblockhash, - ) - self.assertEqual( - snapshot.bestblockhash, - snapshot.template_artifacts.previousblockhash, - ) - - def test_fetch_fails_closed_when_exact_artifacts_cannot_be_derived(self) -> None: - server, _ = coordinator() - server.store_template_artifacts = ( # type: ignore[method-assign] - lambda _template, *, generation=None: None - ) - - with self.assertRaises(TemplateRefreshBlocked): - server.fetch_qbit_tip_template_snapshot() - - self.assertIsNone(server.tip_template_snapshot) - self.assertIsNone(server._template_artifacts) - - def test_prepare_rejects_mismatched_snapshot_artifact_invariants(self) -> None: - server, _ = coordinator() - install_fake_bundle_builder(server) - state = client(1) - server.clients = {state} - snapshot = server.fetch_qbit_tip_template_snapshot() - artifacts = snapshot.template_artifacts - self.assertIsNotNone(artifacts) - assert artifacts is not None - - with self.assertRaises(TemplateRefreshBlocked): - server.prepare_tip_refresh_bundle( - dataclass_replace(snapshot, template_artifacts=None), - ) - with self.assertRaises(TemplateRefreshBlocked): - server.prepare_tip_refresh_bundle( - dataclass_replace(snapshot, bestblockhash="55" * 32), - ) - - changed_template = dict(artifacts.template) - changed_template["coinbasevalue"] = int(changed_template["coinbasevalue"]) + 1 - wrong_parent_template = dict(artifacts.template) - wrong_parent_template["previousblockhash"] = "44" * 32 - cases = { - "fingerprint": dataclass_replace(artifacts, fingerprint="ff" * 32), - "generation": dataclass_replace( - artifacts, - generation=artifacts.generation + 1, - ), - "previousblockhash": dataclass_replace( - artifacts, - previousblockhash="33" * 32, - ), - "template_fingerprint": dataclass_replace( - artifacts, - template=changed_template, - ), - "template_previousblockhash": dataclass_replace( - artifacts, - template=wrong_parent_template, - ), - } - - for name, mismatched in cases.items(): - with self.subTest(name=name): - mismatched_snapshot = dataclass_replace( - snapshot, - template_artifacts=mismatched, - ) - with self.assertRaises(TemplateRefreshBlocked): - server.prepare_tip_refresh_bundle(mismatched_snapshot) - - self.assertEqual(server.job_counter, 0) - self.assertIsNone(state.active_job) - - def test_concurrent_global_cache_replacement_does_not_abort_snapshot_artifacts( - self, - ) -> None: - server, rpc = coordinator() - recorded = install_fake_bundle_builder(server) - states = [client(1), client(2)] - server.clients = set(states) - snapshot = server.fetch_qbit_tip_template_snapshot() - artifacts_a = snapshot.template_artifacts - self.assertIsNotNone(artifacts_a) - assert artifacts_a is not None - - build_entered = threading.Event() - release_build = threading.Event() - original_shared_job_bundle = server.shared_job_bundle - bundles: list[CachedJobBundle] = [] - errors: list[BaseException] = [] - - def blocked_shared_job_bundle( - artifacts: CachedTemplateArtifacts, - identity: WorkerIdentity | None = None, - **kwargs: object, - ) -> CachedJobBundle: - self.assertIs(artifacts, artifacts_a) - build_entered.set() - if not release_build.wait(5): - raise AssertionError("immutable artifact build was not released") - return original_shared_job_bundle(artifacts, identity, **kwargs) - - def prepare() -> None: - try: - bundles.append(server.prepare_tip_refresh_bundle(snapshot)) - except BaseException as exc: # noqa: BLE001 - surfaced by the test - errors.append(exc) - - server.shared_job_bundle = blocked_shared_job_bundle # type: ignore[method-assign] - thread = threading.Thread(target=prepare) - thread.start() - try: - self.assertTrue(build_entered.wait(5)) - template_b = dict(rpc.template) - template_b["coinbasevalue"] = int(template_b["coinbasevalue"]) + 1 - artifacts_b = server.store_template_artifacts(template_b) - self.assertIsNotNone(artifacts_b) - assert artifacts_b is not None - self.assertNotEqual(artifacts_a.fingerprint, artifacts_b.fingerprint) - self.assertIs(server._template_artifacts, artifacts_b) - finally: - release_build.set() - thread.join(5) - - self.assertFalse(thread.is_alive()) - self.assertEqual(errors, []) - self.assertEqual(len(bundles), 1) - bundle = bundles[0] - self.assertEqual(bundle.template_fingerprint, artifacts_a.fingerprint) - self.assertEqual(bundle.template_generation, artifacts_a.generation) - self.assertEqual( - bundle.template["previousblockhash"], - artifacts_a.previousblockhash, - ) - self.assertEqual( - qbit_template_fingerprint(bundle.template), - artifacts_a.fingerprint, - ) - self.assertEqual( - bundle.found_block["coinbase_value_sats"], - artifacts_a.template["coinbasevalue"], - ) - self.assertEqual(recorded["calls"], 1) - - def test_newer_observation_supersedes_old_build_before_send_and_runs_promptly( - self, - ) -> None: - tip_a = "11" * 32 - tip_b = "22" * 32 - template_a = base_template(height=10, prevhash=tip_a) - template_b = base_template(height=11, prevhash=tip_b) - server, rpc = coordinator(template=template_a) - install_fake_bundle_builder(server) - server._pool_ready_latched = True - states = [client(1), client(2)] - server.clients = set(states) - sent_fingerprints: list[str] = [] - sent_lock = threading.Lock() - - def record_send(state: ClientState, payload: dict[str, object]) -> None: - if payload["method"] != "mining.notify": - return - active_job = state.active_job - self.assertIsNotNone(active_job) - with sent_lock: - sent_fingerprints.append(active_job.template_fingerprint) - - for state in states: - state.send = ( # type: ignore[method-assign] - lambda payload, state=state: record_send(state, payload) - ) - - first_build_entered = threading.Event() - release_first_build = threading.Event() - original_shared_job_bundle = server.shared_job_bundle - build_count = 0 - build_count_lock = threading.Lock() - - def block_first_shared_job_bundle( - artifacts: CachedTemplateArtifacts, - identity: WorkerIdentity | None = None, - **kwargs: object, - ) -> CachedJobBundle: - nonlocal build_count - with build_count_lock: - build_count += 1 - current_build = build_count - if current_build == 1: - first_build_entered.set() - if not release_first_build.wait(5): - raise AssertionError("superseded artifact build was not released") - return original_shared_job_bundle(artifacts, identity, **kwargs) - - server.shared_job_bundle = block_first_shared_job_bundle # type: ignore[method-assign] - old_results: list[int] = [] - old_errors: list[BaseException] = [] - new_results: list[int] = [] - new_errors: list[BaseException] = [] - - def poll(results: list[int], errors: list[BaseException]) -> None: - try: - results.append(server.poll_qbit_tip_template_once()) - except BaseException as exc: # noqa: BLE001 - surfaced by the test - errors.append(exc) - - old_thread = threading.Thread(target=poll, args=(old_results, old_errors)) - new_thread = threading.Thread(target=poll, args=(new_results, new_errors)) - old_thread.start() - try: - self.assertTrue(first_build_entered.wait(5)) - rpc.tip = tip_b - rpc.template = template_b - self.assertTrue(server.observe_tip_for_refresh(tip_b)) - # A competing producer observes/cancels immediately, but cannot - # enter the heavy lane while the obsolete owner unwinds. - new_thread.start() - finally: - release_first_build.set() - old_thread.join(5) - new_thread.join(5) - - self.assertFalse(old_thread.is_alive()) - self.assertFalse(new_thread.is_alive()) - self.assertEqual(old_results, []) - self.assertEqual(len(old_errors), 1) - self.assertIsInstance(old_errors[0], TemplateRefreshBlocked) - self.assertEqual(new_errors, []) - self.assertEqual(new_results, [0]) - try: - self.assertEqual( - server.poll_qbit_tip_template_once(), - len(states), - ) - finally: - server.shutdown_tip_refresh_executor() - self.assertEqual( - sent_fingerprints, - [qbit_template_fingerprint(template_b)] * len(states), - ) - self.assertNotIn(qbit_template_fingerprint(template_a), sent_fingerprints) - - -if __name__ == "__main__": - unittest.main() diff --git a/tests/test_prism_incremental_payout_window.py b/tests/test_prism_incremental_payout_window.py deleted file mode 100644 index e468c9199..000000000 --- a/tests/test_prism_incremental_payout_window.py +++ /dev/null @@ -1,822 +0,0 @@ -#!/usr/bin/env python3 - -from __future__ import annotations - -import hashlib -import json -import math -import random -import unittest -from dataclasses import replace -from typing import Iterable, Sequence -from unittest.mock import patch - -from lab.prism.share_json_stream import ( - ShareArrayJsonSequence, - canonical_share_array_sha256, - canonical_share_items_bytes, - iter_json_array_text_chunks, - iter_json_object_text_chunks, - release_share_list_incrementally, - share_array_json_view, -) -from lab.prism.share_ledger import ( - AcceptedShareRecord, - IncrementalShareWindow, - IncrementalWindowFallback, - _IncrementalShareWindowPage, -) - - -RANDOM_MASTER_SEED = 0x51A7_E2026 -RANDOM_SEQUENCE_COUNT = 10_000 - - -def accepted_share( - share_seq: int, - *, - share_difficulty: int, - job_issued_at_ms: int, - accepted_at_ms: int, - credit_policy: str | None = None, -) -> AcceptedShareRecord: - return AcceptedShareRecord( - share_seq=share_seq, - share_id=f"share-{share_seq}", - miner_id=f"miner-{share_seq % 11}", - order_key=f"{share_seq % 11:02d}:miner-{share_seq % 11}", - p2mr_program_hex=f"{share_seq % 256:02x}" * 32, - share_difficulty=share_difficulty, - network_difficulty=1_000_000 + (share_seq % 17), - template_height=800_000 + (share_seq // 32), - job_id=f"job-{share_seq}", - job_issued_at_ms=job_issued_at_ms, - accepted_at_ms=accepted_at_ms, - ntime=1_700_000_000 + share_seq, - credit_policy=credit_policy, - ) - - -def full_rescan_oracle( - records: Iterable[AcceptedShareRecord], - *, - anchor_job_issued_at_ms: int, - window_weight: int, -) -> tuple[AcceptedShareRecord, ...]: - """Model qbit_share_ledger's bounded snapshot without sharing cache code.""" - eligible = sorted( - ( - record - for record in records - if record.job_issued_at_ms <= anchor_job_issued_at_ms - and record.accepted_at_ms <= anchor_job_issued_at_ms - ), - key=lambda record: record.share_seq, - ) - retained_newest_first: list[AcceptedShareRecord] = [] - cumulative_weight = 0 - for record in reversed(eligible): - if cumulative_weight >= window_weight: - break - retained_newest_first.append(record) - cumulative_weight += record.share_difficulty - retained_newest_first.reverse() - return tuple(retained_newest_first) - - -def canonical_json_text(value: object) -> str: - return json.dumps(value, sort_keys=True, separators=(",", ":"), default=str) - - -def canonical_json_sha256(value: object) -> str: - return hashlib.sha256(canonical_json_text(value).encode()).hexdigest() - - -def carry_forward_near_floor( - rng: random.Random, -) -> tuple[int, tuple[tuple[str, str, str, int], ...]]: - floor_sats = rng.choice((1, 2, 10, 546, 1_000, 10_000)) - boundary_amounts = ( - 0, - max(0, floor_sats - 1), - floor_sats, - floor_sats + 1, - (2 * floor_sats) - 1, - ) - carry = tuple( - ( - f"carry-miner-{index}", - f"{index:02d}:carry-miner-{index}", - f"{(index + 192) % 256:02x}" * 32, - rng.choice(boundary_amounts), - ) - for index in range(rng.randint(0, 4)) - ) - return floor_sats, carry - - -def artifact_equivalence_projection( - records: Sequence[AcceptedShareRecord], - *, - carry_forward: tuple[tuple[str, str, str, int], ...], - payout_floor_sats: int, -) -> tuple[tuple[dict[str, object], ...], str, str]: - """Project the money-path inputs whose ordering and bytes must not change. - - The boundary classification deliberately includes balances immediately - below, at, and above a payout-like floor. Incremental maintenance does not - own payout math, but it must hand exactly the same share and carry tuples to - that math as a full rebuild. - """ - shares_json = tuple(record.to_prism_json() for record in records) - balances_json = tuple( - { - "recipient_id": recipient_id, - "order_key": order_key, - "p2mr_program_hex": p2mr_program_hex, - "balance_sats": balance_sats, - } - for recipient_id, order_key, p2mr_program_hex, balance_sats in carry_forward - ) - settlement_boundary = tuple( - { - "recipient_id": recipient_id, - "paid_sats": balance_sats if balance_sats >= payout_floor_sats else 0, - "carry_forward_sats": balance_sats if balance_sats < payout_floor_sats else 0, - } - for recipient_id, _order_key, _program, balance_sats in carry_forward - ) - artifact_json = canonical_json_text( - { - "shares": shares_json, - "prior_balances": balances_json, - "payout_floor_sats": payout_floor_sats, - "settlement_boundary": settlement_boundary, - } - ) - return shares_json, artifact_json, hashlib.sha256(artifact_json.encode()).hexdigest() - - -class IncrementalShareWindowGoldenTests(unittest.TestCase): - def assert_window_parity( - self, - window: IncrementalShareWindow, - history: Sequence[AcceptedShareRecord], - *, - anchor_job_issued_at_ms: int, - window_weight: int, - rng: random.Random, - ) -> None: - incremental_records = window.records() - oracle_records = full_rescan_oracle( - history, - anchor_job_issued_at_ms=anchor_job_issued_at_ms, - window_weight=window_weight, - ) - self.assertIsInstance(incremental_records, tuple) - self.assertEqual(incremental_records, oracle_records) - self.assertEqual( - tuple(record.share_seq for record in incremental_records), - tuple(sorted(record.share_seq for record in incremental_records)), - ) - - incremental_json = window.json_records() - oracle_json = tuple(record.to_prism_json() for record in oracle_records) - self.assertEqual(tuple(incremental_json), oracle_json) - self.assertEqual( - incremental_json.canonical_json_sha256(), - canonical_json_sha256(oracle_json), - ) - - payout_floor_sats, carry_forward = carry_forward_near_floor(rng) - incremental_artifact = artifact_equivalence_projection( - incremental_records, - carry_forward=carry_forward, - payout_floor_sats=payout_floor_sats, - ) - oracle_artifact = artifact_equivalence_projection( - oracle_records, - carry_forward=carry_forward, - payout_floor_sats=payout_floor_sats, - ) - self.assertEqual(incremental_artifact, oracle_artifact) - - def assert_stats_shape(self, stats: object) -> None: - for name in ("added_rows", "expired_rows", "touched_pages"): - value = getattr(stats, name) - self.assertIsInstance(value, int) - self.assertGreaterEqual(value, 0) - - def test_full_snapshot_keeps_exact_crossing_row_and_ascending_order(self) -> None: - anchor_ms = 10_000 - records = [ - accepted_share( - 1, - share_difficulty=2, - job_issued_at_ms=anchor_ms - 4, - accepted_at_ms=anchor_ms - 3, - ), - accepted_share( - 2, - share_difficulty=3, - job_issued_at_ms=anchor_ms - 2, - accepted_at_ms=anchor_ms - 1, - credit_policy="stale-grace", - ), - accepted_share( - 3, - share_difficulty=8, - job_issued_at_ms=anchor_ms, - accepted_at_ms=anchor_ms, - ), - accepted_share( - 4, - share_difficulty=100, - job_issued_at_ms=anchor_ms + 1, - accepted_at_ms=anchor_ms, - ), - ] - - window = IncrementalShareWindow.from_full_snapshot( - records, - anchor_job_issued_at_ms=anchor_ms, - window_weight=10, - page_size=2, - ) - - self.assertEqual(tuple(record.share_seq for record in window.records()), (2, 3)) - self.assertEqual(window.records()[0].credit_policy, "stale-grace") - expected_json = tuple(record.to_prism_json() for record in records[1:3]) - self.assertEqual( - tuple(record.to_prism_json() for record in window.records()), - expected_json, - ) - self.assertEqual( - canonical_json_sha256(tuple(record.to_prism_json() for record in window.records())), - canonical_json_sha256(expected_json), - ) - - def test_advance_rejects_non_append_deltas_without_mutating_the_window(self) -> None: - anchor_ms = 20_000 - records = [ - accepted_share( - share_seq, - share_difficulty=1, - job_issued_at_ms=anchor_ms, - accepted_at_ms=anchor_ms, - ) - for share_seq in range(1, 7) - ] - window = IncrementalShareWindow.from_full_snapshot( - records, - anchor_job_issued_at_ms=anchor_ms, - window_weight=5, - page_size=2, - ) - before = window.records() - corrected = replace(records[-1], share_difficulty=9) - unordered = ( - accepted_share( - 8, - share_difficulty=1, - job_issued_at_ms=anchor_ms + 1, - accepted_at_ms=anchor_ms, - ), - accepted_share( - 7, - share_difficulty=1, - job_issued_at_ms=anchor_ms + 1, - accepted_at_ms=anchor_ms, - ), - ) - - for delta, next_anchor_ms in ( - ((corrected,), anchor_ms), - ((records[-2],), anchor_ms), - (unordered, anchor_ms + 1), - ((), anchor_ms - 1), - ): - with self.subTest(delta=tuple(record.share_seq for record in delta), anchor=next_anchor_ms): - with self.assertRaises(IncrementalWindowFallback): - window.advance( - delta, - anchor_job_issued_at_ms=next_anchor_ms, - ) - self.assertEqual(window.records(), before) - - def test_10_000_seeded_randomized_sequences_match_full_rescan(self) -> None: - for sequence_number in range(RANDOM_SEQUENCE_COUNT): - seed = RANDOM_MASTER_SEED + sequence_number - rng = random.Random(seed) - with self.subTest(seed=seed): - anchor_ms = 1_000_000 + (sequence_number * 100) - history: list[AcceptedShareRecord] = [] - next_share_seq = 1 - - for _ in range(rng.randint(1, 8)): - history.append( - accepted_share( - next_share_seq, - share_difficulty=rng.choice((1, 2, 3, 7, 19, 101, 10_000)), - job_issued_at_ms=anchor_ms - rng.randint(0, 30), - accepted_at_ms=anchor_ms - rng.randint(0, 30), - credit_policy=( - "stale-grace" if rng.randrange(4) == 0 else None - ), - ) - ) - next_share_seq += 1 - - recent_weight = sum(record.share_difficulty for record in history[-3:]) - window_weight = rng.choice( - ( - 1, - 2, - 10, - max(1, recent_weight - 1), - max(1, recent_weight), - recent_weight + 1, - 100_000, - ) - ) - page_size = rng.randint(2, 8) - window = IncrementalShareWindow.from_full_snapshot( - history, - anchor_job_issued_at_ms=anchor_ms, - window_weight=window_weight, - page_size=page_size, - ) - self.assert_window_parity( - window, - history, - anchor_job_issued_at_ms=anchor_ms, - window_weight=window_weight, - rng=rng, - ) - - immediate_anchor_ms = anchor_ms + 1 - immediate_batch: list[AcceptedShareRecord] = [] - for _ in range(rng.randint(1, 4)): - record = accepted_share( - next_share_seq, - share_difficulty=rng.choice((1, 2, 5, 31, 5_000, 50_000)), - job_issued_at_ms=immediate_anchor_ms, - accepted_at_ms=anchor_ms - rng.randint(0, 3), - credit_policy="stale-grace" if rng.randrange(3) == 0 else None, - ) - immediate_batch.append(record) - history.append(record) - next_share_seq += 1 - previous_count = len(window.records()) - window, stats = window.advance( - immediate_batch, - anchor_job_issued_at_ms=immediate_anchor_ms, - ) - anchor_ms = immediate_anchor_ms - self.assert_stats_shape(stats) - self.assertEqual(stats.added_rows, len(immediate_batch)) - self.assertEqual( - stats.expired_rows, - previous_count + len(immediate_batch) - len(window.records()), - ) - self.assert_window_parity( - window, - history, - anchor_job_issued_at_ms=anchor_ms, - window_weight=window_weight, - rng=rng, - ) - - transition_anchor_ms = anchor_ms + rng.randint(2, 30) - delayed_batch: list[AcceptedShareRecord] = [] - for index in range(rng.randint(1, 3)): - transition_on_job_time = (index + sequence_number) % 2 == 0 - record = accepted_share( - next_share_seq, - share_difficulty=rng.choice((1, 3, 13, 1_000, 100_000)), - job_issued_at_ms=( - transition_anchor_ms - if transition_on_job_time - else anchor_ms - rng.randint(0, 2) - ), - accepted_at_ms=( - anchor_ms - rng.randint(0, 2) - if transition_on_job_time - else transition_anchor_ms - ), - credit_policy="stale-grace" if rng.randrange(2) == 0 else None, - ) - delayed_batch.append(record) - history.append(record) - next_share_seq += 1 - - window, stats = window.advance( - (), - anchor_job_issued_at_ms=transition_anchor_ms - 1, - ) - self.assert_stats_shape(stats) - self.assertEqual((stats.added_rows, stats.expired_rows), (0, 0)) - self.assert_window_parity( - window, - history, - anchor_job_issued_at_ms=transition_anchor_ms - 1, - window_weight=window_weight, - rng=rng, - ) - - previous_count = len(window.records()) - window, stats = window.advance( - delayed_batch, - anchor_job_issued_at_ms=transition_anchor_ms, - ) - anchor_ms = transition_anchor_ms - self.assert_stats_shape(stats) - self.assertEqual(stats.added_rows, len(delayed_batch)) - self.assertEqual( - stats.expired_rows, - previous_count + len(delayed_batch) - len(window.records()), - ) - self.assert_window_parity( - window, - history, - anchor_job_issued_at_ms=anchor_ms, - window_weight=window_weight, - rng=rng, - ) - - correction_index = rng.randrange(len(history)) - original = history[correction_index] - corrected = replace( - original, - share_difficulty=original.share_difficulty + rng.randint(1, 101), - credit_policy=( - None if original.credit_policy == "stale-grace" else "stale-grace" - ), - ) - before_fallback = window.records() - with self.assertRaises(IncrementalWindowFallback): - window.advance( - (corrected,), - anchor_job_issued_at_ms=anchor_ms, - ) - self.assertEqual(window.records(), before_fallback) - history[correction_index] = corrected - window = IncrementalShareWindow.from_full_snapshot( - history, - anchor_job_issued_at_ms=anchor_ms, - window_weight=window_weight, - page_size=page_size, - ) - self.assert_window_parity( - window, - history, - anchor_job_issued_at_ms=anchor_ms, - window_weight=window_weight, - rng=rng, - ) - - post_rebuild_anchor_ms = anchor_ms + 1 - post_rebuild_batch: list[AcceptedShareRecord] = [] - for _ in range(rng.randint(1, 3)): - record = accepted_share( - next_share_seq, - share_difficulty=rng.choice((1, 2, 11, 257, 25_000)), - job_issued_at_ms=post_rebuild_anchor_ms, - accepted_at_ms=anchor_ms, - credit_policy="stale-grace" if rng.randrange(3) == 0 else None, - ) - post_rebuild_batch.append(record) - history.append(record) - next_share_seq += 1 - window, stats = window.advance( - post_rebuild_batch, - anchor_job_issued_at_ms=post_rebuild_anchor_ms, - ) - anchor_ms = post_rebuild_anchor_ms - self.assert_stats_shape(stats) - self.assertEqual(stats.added_rows, len(post_rebuild_batch)) - self.assert_window_parity( - window, - history, - anchor_job_issued_at_ms=anchor_ms, - window_weight=window_weight, - rng=rng, - ) - - def test_90_000_share_window_plus_500_delta_touches_only_edge_pages(self) -> None: - anchor_ms = 2_000_000 - initial_count = 90_000 - delta_count = 500 - page_size = 512 - initial = tuple( - accepted_share( - share_seq, - share_difficulty=1, - job_issued_at_ms=anchor_ms, - accepted_at_ms=anchor_ms, - credit_policy="stale-grace" if share_seq % 101 == 0 else None, - ) - for share_seq in range(1, initial_count + 1) - ) - window = IncrementalShareWindow.from_full_snapshot( - initial, - anchor_job_issued_at_ms=anchor_ms, - window_weight=initial_count, - page_size=page_size, - ) - delta = tuple( - accepted_share( - share_seq, - share_difficulty=1, - job_issued_at_ms=anchor_ms + 1, - accepted_at_ms=anchor_ms + 1, - credit_policy="stale-grace" if share_seq % 101 == 0 else None, - ) - for share_seq in range(initial_count + 1, initial_count + delta_count + 1) - ) - - initial_pages = window.pages - prism_json_calls = 0 - original_to_prism_json = AcceptedShareRecord.to_prism_json - - def counting_to_prism_json( - record: AcceptedShareRecord, - ) -> dict[str, object]: - nonlocal prism_json_calls - prism_json_calls += 1 - return original_to_prism_json(record) - - with patch.object( - AcceptedShareRecord, - "to_prism_json", - autospec=True, - side_effect=counting_to_prism_json, - ): - window, stats = window.advance( - delta, - anchor_job_issued_at_ms=anchor_ms + 1, - ) - incremental_json = window.json_records() - incremental_digest = incremental_json.canonical_json_sha256() - - self.assert_stats_shape(stats) - self.assertEqual(stats.added_rows, delta_count) - self.assertEqual(stats.expired_rows, delta_count) - # Only the partially retained head and append-edge pages may be - # inspected or rewritten. Fully expired and newly allocated pages do - # not count; any larger value exposes retained-interior page work. - self.assertLessEqual(stats.touched_pages, 2) - # Production materialization reuses JSON fragments on every retained - # page. Only the two changed edge pages plus a new delta page invoke - # per-share conversion; the ~89.5k retained interior is untouched. - self.assertLessEqual(prism_json_calls, 2 * page_size) - self.assertGreaterEqual( - sum( - 1 - for page in window.pages - if any(page is initial_page for initial_page in initial_pages) - ), - len(initial_pages) - 2, - ) - expected = full_rescan_oracle( - (*initial, *delta), - anchor_job_issued_at_ms=anchor_ms + 1, - window_weight=initial_count, - ) - self.assertEqual(window.records(), expected) - expected_json = tuple(record.to_prism_json() for record in expected) - self.assertEqual(tuple(incremental_json), expected_json) - self.assertEqual( - incremental_digest, - canonical_json_sha256(expected_json), - ) - self.assertEqual(len(window.records()), initial_count) - self.assertEqual(window.records()[0].share_seq, delta_count + 1) - self.assertEqual(window.records()[-1].share_seq, initial_count + delta_count) - self.assertLess( - stats.touched_pages, - math.ceil(initial_count / page_size), - ) - - -def awkward_share_json(seq: int) -> dict[str, object]: - """A prism-JSON share that exercises every encoder edge at once. - - Non-ASCII and non-BMP text (surrogate-pair escapes), JSON-significant - punctuation inside strings, integers past 64 and 128 bits, and the - optional ``credit_policy`` member both present and absent. - """ - payload: dict[str, object] = { - "share_seq": seq, - "share_id": f"miner-\u00e9.rig{seq}:\U0001F600,}}{{\"\\", - "miner_id": f"miner-{seq % 5}", - "order_key": f"{seq % 5:02d}:miner-{seq % 5}", - "p2mr_program_hex": f"{seq % 256:02x}" * 32, - "share_difficulty": (1 << 130) + seq, - "network_difficulty": (1 << 70) + 7, - "template_height": 800_000 + seq, - "job_id": f"job-\u2603-{seq}", - "job_issued_at_ms": 1_700_000_000_000 + seq, - "accepted_at_ms": 1_700_000_000_100 + seq, - "ntime": 1_700_000_000 + seq, - } - if seq % 3 == 0: - payload["credit_policy"] = "template-\u00fc" - return payload - - -class BoundedShareJsonEncodingTests(unittest.TestCase): - """The batched encoders reproduce ``json.dumps`` byte for byte (#236). - - Every helper in ``share_json_stream`` exists so that no single C call - covers a whole 200k-share window; the contract that makes that safe is - that the concatenated output is identical to the historical one-call - encoding, whatever batch and chunk sizes are in force. - """ - - BATCH_SIZES = (1, 2, 3, 7, 512) - CHUNK_SIZES = (1, 16, 64 * 1024) - - def _shares(self, count: int) -> list[dict[str, object]]: - return [awkward_share_json(seq) for seq in range(1, count + 1)] - - def test_array_chunks_match_json_dumps_for_every_batch_and_chunk_size( - self, - ) -> None: - for count in (0, 1, 2, 13, 1_100): - shares = self._shares(count) - canonical = json.dumps( - shares, sort_keys=True, separators=(",", ":"), default=str - )[1:-1] - compact = json.dumps(shares, separators=(",", ":"))[1:-1] - for batch in self.BATCH_SIZES: - for chunk in self.CHUNK_SIZES: - with self.subTest(count=count, batch=batch, chunk=chunk): - canonical_chunks = list( - iter_json_array_text_chunks( - shares, - sort_keys=True, - default=str, - batch_records=batch, - chunk_chars=chunk, - ) - ) - self.assertEqual("".join(canonical_chunks), canonical) - self.assertEqual( - "".join( - iter_json_array_text_chunks( - shares, - batch_records=batch, - chunk_chars=chunk, - ) - ), - compact, - ) - # No chunk splits a record: every chunk is a run of - # whole records, so it parses once re-framed. - for piece in canonical_chunks: - json.loads("[" + piece.strip(",") + "]") - self.assertTrue( - all(piece.isascii() for piece in canonical_chunks) - ) - - def test_object_chunks_match_json_dumps(self) -> None: - shares = self._shares(700) - fields = { - "request": "prepare_window", - "mode": "full", - "anchor_job_issued_at_ms": 1_700_000_000_000, - "records": shares, - "window_weight": 1 << 100, - "empty": [], - "pair": (1, "two"), - "nested": {"z": 1, "a": [1.5, None, True]}, - } - for batch in (1, 512): - for chunk in self.CHUNK_SIZES: - with self.subTest(batch=batch, chunk=chunk): - self.assertEqual( - "".join( - iter_json_object_text_chunks( - fields, - array_keys=("records", "empty", "pair"), - batch_records=batch, - chunk_chars=chunk, - ) - ), - json.dumps(fields, separators=(",", ":")), - ) - self.assertEqual( - "".join( - iter_json_object_text_chunks( - fields, - array_keys=("records",), - sort_keys=True, - default=str, - batch_records=batch, - chunk_chars=chunk, - ) - ), - json.dumps( - fields, - sort_keys=True, - separators=(",", ":"), - default=str, - ), - ) - # A non-string key defers to json.dumps' own coercion rules. - mixed = {"a": 1, 2: [1, 2]} - self.assertEqual( - "".join(iter_json_object_text_chunks(mixed, array_keys=(2,))), - json.dumps(mixed, separators=(",", ":")), - ) - - def test_page_canonical_items_match_per_record_encoding(self) -> None: - records = tuple( - accepted_share( - seq, - share_difficulty=(1 << 129) + seq, - job_issued_at_ms=1_000 + seq, - accepted_at_ms=1_000 + seq, - credit_policy="unicode-\u00e9-\U0001F600" if seq % 2 else None, - ) - for seq in range(1, 600) - ) - page = _IncrementalShareWindowPage.from_records(records) - expected_items = b",".join( - canonical_json_text(record.to_prism_json()).encode() - for record in records - ) - self.assertEqual(page.canonical_json_items, expected_items) - self.assertEqual( - canonical_share_items_bytes(page.prism_json_records), - expected_items, - ) - window = IncrementalShareWindow.from_full_snapshot( - records, - anchor_job_issued_at_ms=10_000, - window_weight=sum(int(r.share_difficulty) for r in records), - page_size=128, - ) - self.assertEqual( - window.json_records().canonical_json_sha256(), - canonical_json_sha256( - tuple(record.to_prism_json() for record in records) - ), - ) - self.assertEqual( - _IncrementalShareWindowPage.from_records(()).canonical_json_items, - b"", - ) - - def test_incremental_release_empties_lists_in_bounded_slices(self) -> None: - values = list(range(10_001)) - release_share_list_incrementally(values, batch_records=4_000) - self.assertEqual(values, []) - # Non-lists (tuples, the paged view) are left to ordinary GC. - frozen = (1, 2, 3) - release_share_list_incrementally(frozen) - self.assertEqual(frozen, (1, 2, 3)) - # A batch size below one is clamped rather than looping forever. - values = [1, 2] - release_share_list_incrementally(values, batch_records=0) - self.assertEqual(values, []) - - def test_share_array_view_digests_like_the_whole_array(self) -> None: - shares = self._shares(1_500) - expected = canonical_json_sha256(shares) - self.assertEqual(canonical_share_array_sha256(shares), expected) - self.assertEqual(canonical_share_array_sha256(tuple(shares)), expected) - self.assertEqual(canonical_share_array_sha256([]), canonical_json_sha256([])) - view = share_array_json_view(shares) - self.assertIsInstance(view, ShareArrayJsonSequence) - self.assertEqual(view.canonical_json_sha256(), expected) - self.assertEqual(len(view), len(shares)) - self.assertEqual(list(view), shares) - self.assertIs(view[3], shares[3]) - self.assertEqual(view[-2:], shares[-2:]) - # A sequence that already carries its digest is handed back as is, - # and the streaming digest defers to it. - window = IncrementalShareWindow.from_full_snapshot( - [ - accepted_share( - seq, - share_difficulty=1, - job_issued_at_ms=seq, - accepted_at_ms=seq, - ) - for seq in range(1, 4) - ], - anchor_job_issued_at_ms=10, - window_weight=3, - ) - paged = window.json_records() - self.assertIs(share_array_json_view(paged), paged) - self.assertEqual( - canonical_share_array_sha256(paged), - paged.canonical_json_sha256(), - ) - - -if __name__ == "__main__": - unittest.main() diff --git a/tests/test_prism_initial_job_delivery.py b/tests/test_prism_initial_job_delivery.py deleted file mode 100644 index c4f120b6d..000000000 --- a/tests/test_prism_initial_job_delivery.py +++ /dev/null @@ -1,868 +0,0 @@ -#!/usr/bin/env python3 -"""Startup prewarm and reconnect-storm initial job delivery regressions.""" - -from __future__ import annotations - -import threading -import time -import unittest -from unittest.mock import patch - -from lab.prism.payout_state import AcceptedParentPayoutPreviewPending -from lab.prism.prism_coordinator import ( - PendingInitialJob, - PRISM_JOB_EXTRANONCE1_PLACEHOLDER_HEX, - TemplateRefreshBlocked, -) -from tests.prism_coordinator_test_support import ( - EXTRANONCE2_SIZE, - ObservedRLock, - base_template, - client, - coordinator, - install_fake_bundle_builder, - worker, -) - - -def wait_until(predicate: object, *, timeout: float = 10.0) -> None: - deadline = time.monotonic() + timeout - while time.monotonic() < deadline: - if predicate(): # type: ignore[operator] - return - time.sleep(0.01) - raise AssertionError("condition did not become true before the deadline") - - -class PrismInitialJobDeliveryTests(unittest.TestCase): - def test_blocked_startup_prewarm_defers_to_background_refresh(self) -> None: - server, _rpc = coordinator() - retry_scheduled = threading.Event() - - def blocked() -> object: - raise TemplateRefreshBlocked("template raced startup") - - server.prewarm_current_tip_ready_bundle = blocked # type: ignore[method-assign] - server._schedule_tip_refresh_retry = retry_scheduled.set # type: ignore[method-assign] - - self.assertIsNone(server.prewarm_startup_jobs()) - self.assertTrue(retry_scheduled.is_set()) - - def test_startup_prewarm_builds_worker_independent_ready_bundle(self) -> None: - server, rpc = coordinator() - recorded = install_fake_bundle_builder(server) - - bundle = server.prewarm_current_tip_ready_bundle() - - self.assertIsNotNone(bundle) - assert bundle is not None - self.assertFalse(bundle.collection_only) - self.assertEqual( - bundle.key, - ( - bundle.template_fingerprint, - bundle.template["previousblockhash"], - "ready", - 0, - 0, - ), - ) - self.assertEqual(recorded["calls"], 1) - self.assertEqual(server.ledger.snapshot_calls, 1) - self.assertEqual(server.tip_template_snapshot.bestblockhash, rpc.tip) - self.assertIs(server._prepared_ready_bundle, bundle) - self.assertEqual( - recorded["suffixes"], - [ - server.coinbase_tag_hex - + PRISM_JOB_EXTRANONCE1_PLACEHOLDER_HEX - + "00" * EXTRANONCE2_SIZE - ], - ) - - def test_initial_delivery_vardiff_wait_does_not_hold_coordinator_lock(self) -> None: - server, _rpc = coordinator() - install_fake_bundle_builder(server) - bundle = server.prewarm_current_tip_ready_bundle() - assert bundle is not None - artifacts = server.current_template_artifacts() - state = client(1) - state.authorization_generation = 1 - state.difficulty_generation = 0 - state.authorized_monotonic = time.monotonic() - state.send = lambda _payload: None # type: ignore[method-assign] - vardiff_lock = ObservedRLock() - state.vardiff_lock = vardiff_lock # type: ignore[assignment] - server.clients = {state} - server._ensure_initial_job_state() - request = PendingInitialJob( - client=state, - authorization_generation=1, - worker=state.worker, - requested_monotonic=time.monotonic(), - deadline_monotonic=None, - connection_id=state.connection_id, - difficulty_generation=0, - ) - server.pending_initial_jobs = {state: request} - results: list[bool | None] = [] - - with vardiff_lock: - vardiff_lock.acquire_attempted.clear() - vardiff_lock.observe_acquires = True - delivery = threading.Thread( - target=lambda: results.append( - server._deliver_initial_bundle(request, artifacts, bundle) - ) - ) - delivery.start() - self.assertTrue(vardiff_lock.acquire_attempted.wait(2)) - acquired = server.lock.acquire(timeout=0.25) - self.assertTrue(acquired) - if acquired: - server.lock.release() - - delivery.join(2) - self.assertFalse(delivery.is_alive()) - self.assertEqual(results, [True]) - - def test_initial_delivery_rechecks_append_epoch_after_vardiff_wait( - self, - ) -> None: - # The payout check before the Vardiff wait is only a fast path: a - # late-visible append can bump the invalidation epoch while a share - # holds this client's Vardiff lock, and the commit boundary must - # re-read the live epoch or a pre-append ready bundle becomes the - # miner's active job. - server, _rpc = coordinator() - install_fake_bundle_builder(server) - bundle = server.prewarm_current_tip_ready_bundle() - assert bundle is not None - artifacts = server.current_template_artifacts() - state = client(1) - state.authorization_generation = 1 - state.difficulty_generation = 0 - state.authorized_monotonic = time.monotonic() - state.send = lambda _payload: None # type: ignore[method-assign] - vardiff_lock = ObservedRLock() - state.vardiff_lock = vardiff_lock # type: ignore[assignment] - server.clients = {state} - server._ensure_initial_job_state() - request = PendingInitialJob( - client=state, - authorization_generation=1, - worker=state.worker, - requested_monotonic=time.monotonic(), - deadline_monotonic=None, - connection_id=state.connection_id, - difficulty_generation=0, - ) - server.pending_initial_jobs = {state: request} - results: list[bool | None] = [] - - with vardiff_lock: - vardiff_lock.acquire_attempted.clear() - vardiff_lock.observe_acquires = True - delivery = threading.Thread( - target=lambda: results.append( - server._deliver_initial_bundle(request, artifacts, bundle) - ) - ) - delivery.start() - self.assertTrue(vardiff_lock.acquire_attempted.wait(2)) - # The delivery passed its pre-wait payout check and is blocked on - # the Vardiff lock; a late-visible append now invalidates the - # bundle's window. - with server._job_cache_lock: - server._payout_ledger_append_invalidation_epoch += 1 - - delivery.join(2) - self.assertFalse(delivery.is_alive()) - self.assertEqual(results, [None]) - self.assertIsNone(state.active_job) - self.assertEqual(state.active_job_ids, set()) - - def test_initial_delivery_rechecks_published_digest_after_vardiff_wait( - self, - ) -> None: - # A periodic self-check repair swaps the published balance snapshot - # in place: neither the payout generation nor the append epoch moves, - # so only the digest identifies work keyed to the refuted balances. - # The commit boundary must re-check it or a bundle probed before the - # repair becomes the miner's active job with the refuted balances. - server, _rpc = coordinator() - install_fake_bundle_builder(server) - bundle = server.prewarm_current_tip_ready_bundle() - assert bundle is not None - artifacts = server.current_template_artifacts() - state = client(1) - state.authorization_generation = 1 - state.difficulty_generation = 0 - state.authorized_monotonic = time.monotonic() - state.send = lambda _payload: None # type: ignore[method-assign] - vardiff_lock = ObservedRLock() - state.vardiff_lock = vardiff_lock # type: ignore[assignment] - server.clients = {state} - server._ensure_initial_job_state() - request = PendingInitialJob( - client=state, - authorization_generation=1, - worker=state.worker, - requested_monotonic=time.monotonic(), - deadline_monotonic=None, - connection_id=state.connection_id, - difficulty_generation=0, - ) - server.pending_initial_jobs = {state: request} - results: list[bool | None] = [] - - with vardiff_lock: - vardiff_lock.acquire_attempted.clear() - vardiff_lock.observe_acquires = True - delivery = threading.Thread( - target=lambda: results.append( - server._deliver_initial_bundle(request, artifacts, bundle) - ) - ) - delivery.start() - self.assertTrue(vardiff_lock.acquire_attempted.wait(2)) - # The delivery passed its pre-wait payout check and is blocked on - # the Vardiff lock; the periodic self-check now refutes and - # repairs the published balances the bundle was keyed to. - with server._job_cache_lock: - published = server._published_payout_state.artifact - assert published is not None - self.assertTrue( - server._publish_self_check_repaired_balances( - 0, - stale_prior_balances_sha256=( - published.prior_balances_sha256 - ), - balances=[ - { - "recipient_id": "carry", - "order_key": "01:carry", - "p2mr_program_hex": "66" * 32, - "balance_sats": 999, - } - ], - ) - ) - - delivery.join(2) - self.assertFalse(delivery.is_alive()) - self.assertEqual(results, [None]) - self.assertIsNone(state.active_job) - self.assertEqual(state.active_job_ids, set()) - - def test_initial_delivery_retries_transient_reorg_and_build_failures(self) -> None: - server, _rpc = coordinator() - install_fake_bundle_builder(server) - server.prewarm_current_tip_ready_bundle() - state = client(1) - state.authorization_generation = 1 - state.authorized_monotonic = time.monotonic() - state.send = lambda _payload: None # type: ignore[method-assign] - server.clients = {state} - - reorg_attempts = 0 - - def transient_reorg() -> bool: - nonlocal reorg_attempts - reorg_attempts += 1 - return reorg_attempts > 1 - - artifact_attempts = 0 - original_artifacts = server.current_template_artifacts - - def transient_artifacts() -> object: - nonlocal artifact_attempts - artifact_attempts += 1 - if artifact_attempts == 1: - raise RuntimeError("temporary template failure") - return original_artifacts() - - server.ensure_reorg_reconciled_for_current_tip = transient_reorg # type: ignore[method-assign] - server.current_template_artifacts = transient_artifacts # type: ignore[method-assign] - - server.request_initial_job_delivery(state) - try: - wait_until(lambda: state.active_job is not None) - finally: - server.shutdown_tip_refresh_executor() - - self.assertEqual(reorg_attempts, 3) - self.assertEqual(artifact_attempts, 2) - self.assertEqual(server.job_build_failure_count, 1) - self.assertIn(state, server.clients) - self.assertEqual( - server.progress_health_snapshot()["eligible_clients_requiring_refresh"], - 0, - ) - - def test_initial_delivery_retries_preview_pending_without_counting_it( - self, - ) -> None: - # Boundary 1 (#182): the initial-job loop coalesces accepted-parent - # preview backpressure onto its ordinary bounded retry. The client - # stays connected, the job still lands, and the benign wait never - # reaches the generic job_build_failure_count accounting reserved for - # real build failures. - server, _rpc = coordinator() - install_fake_bundle_builder(server) - server.prewarm_current_tip_ready_bundle() - state = client(1) - state.authorization_generation = 1 - state.authorized_monotonic = time.monotonic() - state.send = lambda _payload: None # type: ignore[method-assign] - server.clients = {state} - - bundle_attempts = 0 - original_shared_job_bundle = server.shared_job_bundle - - def blocked_then_ready(*args: object, **kwargs: object) -> object: - nonlocal bundle_attempts - bundle_attempts += 1 - if bundle_attempts == 1: - raise AcceptedParentPayoutPreviewPending( - "accepted parent payout preview is not ready yet", - parent_hash="cc" * 32, - waited_seconds=0.25, - timeout_count=1, - ) - return original_shared_job_bundle(*args, **kwargs) # type: ignore[operator] - - server.shared_job_bundle = blocked_then_ready # type: ignore[method-assign] - - with patch("builtins.print") as logged: - try: - server.request_initial_job_delivery(state) - wait_until(lambda: state.active_job is not None) - finally: - server.shutdown_tip_refresh_executor() - log_lines = [ - str(call.args[0]) for call in logged.call_args_list if call.args - ] - - self.assertEqual(bundle_attempts, 2) - # Retried, not counted: the benign wait stays out of the generic - # build-failure budget. - self.assertEqual(server.job_build_failure_count, 0) - self.assertIn(state, server.clients) - self.assertIsNotNone(state.active_job) - # The retry carries an explicit reason instead of a failure traceback. - self.assertTrue( - any("accepted_parent_preview_pending" in line for line in log_lines), - log_lines, - ) - - def test_initial_delivery_backs_off_after_superseded_work(self) -> None: - server, rpc = coordinator() - install_fake_bundle_builder(server) - bundle = server.prewarm_current_tip_ready_bundle() - state = client(1) - state.authorization_generation = 1 - state.difficulty_generation = 0 - server.clients = {state} - server._ensure_initial_job_state() - # Lapse the published authority entirely: age the freshness stamp and - # close the divergence lease so the live RPC read owns classification - # again, which is the only state where prepared published-tip work - # must be dropped instead of delivered. - server.current_tip_observed_monotonic = ( - time.monotonic() - - float(getattr(server, "submit_tip_max_age_seconds", 10.0)) - - 1.0 - ) - server.template_refresh_failure_exit_seconds = 0.0 - request = PendingInitialJob( - client=state, - authorization_generation=1, - worker=state.worker, - requested_monotonic=0.0, - deadline_monotonic=None, - connection_id=state.connection_id, - difficulty_generation=0, - ) - server.pending_initial_jobs = {state: request} - artifacts = server.current_template_artifacts() - waits: list[float] = [] - request.cancelled.wait = ( # type: ignore[method-assign] - lambda timeout: waits.append(timeout) or False - ) - original_call = rpc.call - best_tip_calls = 0 - - def tip_churn_once( - method: str, - params: list[object] | None = None, - ) -> object: - nonlocal best_tip_calls - if method == "getbestblockhash": - best_tip_calls += 1 - if best_tip_calls == 1: - return "ff" * 32 - return artifacts.previousblockhash - return original_call(method, params) - - deliveries = iter((None, True)) - rpc.call = tip_churn_once # type: ignore[method-assign] - server.current_template_artifacts = lambda: artifacts # type: ignore[method-assign] - server.shared_job_bundle = lambda *_args, **_kwargs: bundle # type: ignore[method-assign] - server._deliver_initial_bundle = ( # type: ignore[method-assign] - lambda *_args: next(deliveries) - ) - - self.assertTrue(server._run_initial_job(request)) - self.assertEqual(best_tip_calls, 3) - self.assertEqual(waits, [0.05, 0.1]) - - def test_initial_delivery_keeps_authoritative_published_work_on_tip_churn(self) -> None: - server, rpc = coordinator() - install_fake_bundle_builder(server) - bundle = server.prewarm_current_tip_ready_bundle() - state = client(1) - state.authorization_generation = 1 - state.difficulty_generation = 0 - server.clients = {state} - server._ensure_initial_job_state() - request = PendingInitialJob( - client=state, - authorization_generation=1, - worker=state.worker, - requested_monotonic=0.0, - deadline_monotonic=None, - connection_id=state.connection_id, - difficulty_generation=0, - ) - server.pending_initial_jobs = {state: request} - artifacts = server.current_template_artifacts() - waits: list[float] = [] - request.cancelled.wait = ( # type: ignore[method-assign] - lambda timeout: waits.append(timeout) or False - ) - original_call = rpc.call - detected_tip = "ee" * 32 - best_tip_calls = 0 - - def churned_tip( - method: str, - params: list[object] | None = None, - ) -> object: - nonlocal best_tip_calls - if method == "getbestblockhash": - best_tip_calls += 1 - return detected_tip - return original_call(method, params) - - rpc.call = churned_tip # type: ignore[method-assign] - server.current_template_artifacts = lambda: artifacts # type: ignore[method-assign] - server.shared_job_bundle = lambda *_args, **_kwargs: bundle # type: ignore[method-assign] - server._deliver_initial_bundle = ( # type: ignore[method-assign] - lambda *_args: True - ) - - # The published tip is fresh, so a live read racing ahead of the - # refresh must not drop deliverable published-tip work; it is only - # recorded as a detection for the refresh path to act on. - self.assertTrue(server._run_initial_job(request)) - self.assertEqual(best_tip_calls, 1) - self.assertEqual(waits, []) - self.assertEqual(server.latest_detected_tip[0], detected_tip) - published = server.current_tip_first_seen - assert published is not None - self.assertEqual(published[0], artifacts.previousblockhash) - - def test_250_client_reconnect_storm_uses_one_build_without_client_locks(self) -> None: - server, _rpc = coordinator() - recorded = install_fake_bundle_builder(server) - server.tip_refresh_max_workers = 8 - server.stratum_max_pending_initial_jobs = 250 - # The storm asserts single-flight coalescing, i.e. no REDUNDANT - # builds. The harness's short template (2s) and bundle (10s) TTLs are - # load-dependent confounds: an expiry mid-storm forces a legitimate - # fresh build (a template refetch sweeps the deliberately blocked - # flight; bundle eviction rebuilds current work), failing the - # one-build invariant without any coalescing defect. Pin both TTLs - # past the test's lifetime. - server.template_cache_seconds = 300.0 - server.job_bundle_cache_seconds = 300.0 - server.prewarm_current_tip_ready_bundle() - with server._job_cache_lock: - server._job_bundle_cache.clear() - server._prepared_ready_bundle = None - server._prepared_ready_snapshot = None - - build_entered = threading.Event() - release_build = threading.Event() - original_build = server.build_shared_job_bundle - build_calls = 0 - build_calls_lock = threading.Lock() - - def blocked_build(*args: object, **kwargs: object) -> object: - nonlocal build_calls - with build_calls_lock: - build_calls += 1 - build_entered.set() - # Generous budget: a timeout here raises inside the build - # executor, fails the flight, and later workers then correctly - # rebuild -- surfacing as a flaky build count, not a test error. - self.assertTrue(release_build.wait(120)) - return original_build(*args, **kwargs) # type: ignore[arg-type] - - server.build_shared_job_bundle = blocked_build # type: ignore[method-assign] - clients = [client(index + 1) for index in range(250)] - now = time.monotonic() - for state in clients: - state.authorization_generation = 1 - state.authorized_monotonic = now - state.send = lambda _payload: None # type: ignore[method-assign] - server.clients = set(clients) - - barrier = threading.Barrier(len(clients) + 1) - request_threads = [ - threading.Thread( - target=lambda state=state: ( - barrier.wait(), - server.request_initial_job_delivery(state), - ) - ) - for state in clients - ] - for thread in request_threads: - thread.start() - barrier.wait() - try: - self.assertTrue(build_entered.wait(10)) - # Hold the release until the other three delivery workers have - # coalesced onto the in-flight build. Releasing on first entry - # alone lets the build complete while a straggling worker is - # still constructing its request; that worker then finds an - # empty single-flight lane a moment before the winner publishes - # the cache entry and legitimately starts a second build -- - # a razor-thin startup race, not a coalescing defect. - wait_until( - lambda: ( - server.initial_job_prepared_work_counts["singleflight"] - >= 3 - ) - ) - for thread in request_threads: - thread.join(5) - self.assertTrue(all(not thread.is_alive() for thread in request_threads)) - executor = server.initial_job_executor() - _queued, active_workers = executor.stats() - self.assertLessEqual(active_workers, 4) - self.assertLessEqual( - len(server.pending_initial_jobs), - 250, - ) - for state in clients: - acquired = state.job_update_lock.acquire(timeout=0.2) - self.assertTrue(acquired) - if acquired: - state.job_update_lock.release() - finally: - release_build.set() - - try: - wait_until(lambda: all(state.active_job is not None for state in clients)) - wait_until(lambda: not server.pending_initial_jobs) - finally: - server.shutdown_tip_refresh_executor() - - self.assertEqual(build_calls, 1) - self.assertEqual(recorded["calls"], 2) - # The prewarm build published its fenced ledger window as the payout - # ledger artifact, so the storm's single coalesced rebuild reuses it - # instead of taking a second snapshot. - self.assertEqual(server.ledger.snapshot_calls, 1) - self.assertIsNotNone(server.last_initial_job_delivery_monotonic) - expected_tip = server.tip_template_snapshot.bestblockhash - self.assertTrue( - all( - state.active_job.template["previousblockhash"] == expected_tip - for state in clients - ) - ) - - def test_reauthorization_supersedes_pending_identity(self) -> None: - server, _rpc = coordinator() - install_fake_bundle_builder(server) - server.prewarm_current_tip_ready_bundle() - with server._job_cache_lock: - server._job_bundle_cache.clear() - - build_entered = threading.Event() - release_build = threading.Event() - original_build = server.build_shared_job_bundle - - def blocked_build(*args: object, **kwargs: object) -> object: - build_entered.set() - self.assertTrue(release_build.wait(5)) - return original_build(*args, **kwargs) # type: ignore[arg-type] - - server.build_shared_job_bundle = blocked_build # type: ignore[method-assign] - state = client(1, worker(username="old-worker")) - state.authorization_generation = 1 - state.authorized_monotonic = time.monotonic() - state.send = lambda _payload: None # type: ignore[method-assign] - server.clients = {state} - server.request_initial_job_delivery(state) - self.assertTrue(build_entered.wait(5)) - - replacement = worker(username="new-worker") - with state.job_update_lock: - state.worker = replacement - state.username = replacement.username - state.authorization_generation += 1 - state.authorized_monotonic = time.monotonic() - server.request_initial_job_delivery(state) - queued_replacement = server.pending_initial_jobs[state] - self.assertIsNone(queued_replacement.future) - self.assertIsNotNone(queued_replacement.predecessor) - release_build.set() - try: - wait_until(lambda: state.active_job is not None) - finally: - server.shutdown_tip_refresh_executor() - - self.assertEqual(state.active_job.worker, replacement) - self.assertEqual(state.active_job.authorization_generation, 2) - - def test_payout_generation_supersedes_pending_initial_bundle(self) -> None: - server, _rpc = coordinator() - install_fake_bundle_builder(server) - server.prewarm_current_tip_ready_bundle() - with server._job_cache_lock: - server._job_bundle_cache.clear() - - build_entered = threading.Event() - release_build = threading.Event() - original_build = server.build_shared_job_bundle - build_calls = 0 - - def blocked_build(*args: object, **kwargs: object) -> object: - nonlocal build_calls - build_calls += 1 - if build_calls == 1: - build_entered.set() - self.assertTrue(release_build.wait(5)) - return original_build(*args, **kwargs) # type: ignore[arg-type] - - server.build_shared_job_bundle = blocked_build # type: ignore[method-assign] - state = client(1) - state.authorization_generation = 1 - state.authorized_monotonic = time.monotonic() - state.send = lambda _payload: None # type: ignore[method-assign] - server.clients = {state} - server.request_initial_job_delivery(state) - self.assertTrue(build_entered.wait(5)) - - self.assertEqual(server._advance_payout_state_generation(), 1) - release_build.set() - try: - wait_until( - lambda: state.active_job is not None - and state.active_job.payout_state_generation == 1 - ) - finally: - server.shutdown_tip_refresh_executor() - - self.assertEqual(state.active_job.payout_state_generation, 1) - self.assertNotEqual(state.active_job.payout_state_generation, 0) - - def test_collection_mode_initial_bundles_remain_identity_specific(self) -> None: - from tests.prism_coordinator_test_support import FakeLedger - - server, _rpc = coordinator(ledger=FakeLedger(miners=["solo"])) - install_fake_bundle_builder(server) - server.prewarm_current_tip_ready_bundle() - first = client(1, worker(payout="tq1worker-a", username="worker-a")) - second = client(2, worker(payout="tq1worker-b", username="worker-b")) - for state in (first, second): - state.authorization_generation = 1 - state.authorized_monotonic = time.monotonic() - state.send = lambda _payload: None # type: ignore[method-assign] - server.clients = {first, second} - - server.request_initial_job_delivery(first) - server.request_initial_job_delivery(second) - try: - wait_until( - lambda: first.active_job is not None - and second.active_job is not None - ) - finally: - server.shutdown_tip_refresh_executor() - - self.assertTrue(first.active_job.collection_only) - self.assertTrue(second.active_job.collection_only) - self.assertNotEqual(first.active_job.worker, second.active_job.worker) - collection_keys = { - key - for key in server._job_bundle_cache - if len(key) >= 8 and key[2] == "collection" - } - self.assertEqual( - {key[6] for key in collection_keys}, - {"tq1worker-a", "tq1worker-b"}, - ) - - def test_new_tip_supersedes_blocked_build_without_duplicate_client_task(self) -> None: - tip_a = "11" * 32 - tip_b = "22" * 32 - server, rpc = coordinator(template=base_template(prevhash=tip_a)) - install_fake_bundle_builder(server) - server.prewarm_current_tip_ready_bundle() - with server._job_cache_lock: - server._job_bundle_cache.clear() - - first_build_entered = threading.Event() - release_first_build = threading.Event() - original_build = server.build_shared_job_bundle - build_count = 0 - build_lock = threading.Lock() - - def block_first_build(*args: object, **kwargs: object) -> object: - nonlocal build_count - with build_lock: - build_count += 1 - current = build_count - if current == 1: - first_build_entered.set() - self.assertTrue(release_first_build.wait(10)) - return original_build(*args, **kwargs) # type: ignore[arg-type] - - server.build_shared_job_bundle = block_first_build # type: ignore[method-assign] - state = client(1) - state.authorization_generation = 1 - state.authorized_monotonic = time.monotonic() - state.send = lambda _payload: None # type: ignore[method-assign] - server.clients = {state} - server.request_initial_job_delivery(state) - self.assertTrue(first_build_entered.wait(5)) - - rpc.tip = tip_b - rpc.template = base_template(height=11, prevhash=tip_b) - server.observe_tip_first_seen(tip_b) - time.sleep(0.05) - self.assertEqual(build_count, 1) - self.assertIsNone(state.active_job) - release_first_build.set() - try: - wait_until( - lambda: state.active_job is not None - and state.active_job.template["previousblockhash"] == tip_b, - timeout=5, - ) - finally: - server.shutdown_tip_refresh_executor() - - self.assertGreaterEqual(build_count, 2) - self.assertEqual(state.active_job.template["previousblockhash"], tip_b) - self.assertGreaterEqual(server.shared_bundle_build_counts["superseded"], 1) - - def test_health_turns_non_green_after_stalled_delivery_deadline(self) -> None: - server, _rpc = coordinator() - install_fake_bundle_builder(server) - server.prewarm_current_tip_ready_bundle() - state = client(1) - state.authorization_generation = 1 - state.authorized_monotonic = time.monotonic() - 60 - server.clients = {state} - server.started_monotonic = time.monotonic() - 60 - server.mining_health_startup_grace_seconds = 5 - server.stratum_initial_job_timeout_seconds = 5 - - status, payload = server.cached_health_payload() - - self.assertEqual(status, 503) - self.assertFalse(payload["ok"]) - self.assertFalse(payload["mining_ready"]) - self.assertEqual(payload["authorized_clients"], 1) - self.assertEqual(payload["clients_with_current_tip_job"], 0) - - server.clients.clear() - server._ensure_observability_service().clear_health_snapshot_for_test() - status, payload = server.cached_health_payload() - self.assertEqual(status, 200) - self.assertTrue(payload["ok"]) - - def test_zero_initial_timeout_disables_delivery_stall_health_deadline(self) -> None: - server, _rpc = coordinator() - install_fake_bundle_builder(server) - server.prewarm_current_tip_ready_bundle() - state = client(1) - state.authorization_generation = 1 - state.authorized_monotonic = time.monotonic() - 60 - server.clients = {state} - server.started_monotonic = time.monotonic() - 60 - server.mining_health_startup_grace_seconds = 5 - server.stratum_initial_job_timeout_seconds = 0 - - status, payload = server.cached_health_payload() - - self.assertEqual(status, 200) - self.assertTrue(payload["ok"]) - self.assertTrue(payload["mining_ready"]) - self.assertFalse(payload["initial_delivery_stalled"]) - self.assertNotIn("initial-delivery-stalled", payload["unhealthy_reasons"]) - - def test_health_flags_sustained_eighty_seven_percent_coverage_loss(self) -> None: - server, _rpc = coordinator() - install_fake_bundle_builder(server) - bundle = server.prewarm_current_tip_ready_bundle() - assert bundle is not None - clients = [client(index + 1) for index in range(8)] - old = time.monotonic() - 60 - for state in clients: - state.authorization_generation = 1 - state.authorized_monotonic = old - state.tip_work_delivered = ("ff" * 32, old) - clients[0].active_job = server.stamp_job_for_client( - clients[0], bundle, clean_jobs=True - ) - server.clients = set(clients) - server.started_monotonic = old - server.mining_health_startup_grace_seconds = 5 - server.stratum_initial_job_timeout_seconds = 5 - server._ensure_observability_service().set_delivery_failure_started_monotonic_for_test( - time.monotonic() - 5 - ) - - status, payload = server.cached_health_payload() - - self.assertEqual(status, 503) - self.assertEqual(payload["clients_with_current_tip_job"], 1) - self.assertEqual(payload["current_tip_job_coverage_ratio"], 0.125) - self.assertIn("initial-delivery-stalled", payload["unhealthy_reasons"]) - - def test_same_tip_obsolete_template_does_not_count_as_current(self) -> None: - server, rpc = coordinator() - install_fake_bundle_builder(server) - bundle = server.prewarm_current_tip_ready_bundle() - assert bundle is not None - state = client(1) - state.authorization_generation = 1 - state.active_job = server.stamp_job_for_client( - state, bundle, clean_jobs=True - ) - server.clients = {state} - self.assertEqual( - server.mining_delivery_snapshot()["clients_with_current_tip_jobs"], - 1, - ) - - rpc.template = base_template(height=11, prevhash=rpc.tip) - rpc.template["coinbasevalue"] = int(rpc.template["coinbasevalue"]) + 1 - server.tip_template_snapshot = server.fetch_qbit_tip_template_snapshot() - - self.assertEqual( - server.mining_delivery_snapshot()["clients_with_current_tip_jobs"], - 0, - ) - - -if __name__ == "__main__": - unittest.main() diff --git a/tests/test_prism_issue_224_cadence.py b/tests/test_prism_issue_224_cadence.py deleted file mode 100644 index e6b30182c..000000000 --- a/tests/test_prism_issue_224_cadence.py +++ /dev/null @@ -1,596 +0,0 @@ -"""Issue #224 Wave 2: a dense reconcile cadence over one exact share window. - -Wave 1 taught a confirmed payout mutation to ask candidate preparation for -a live prior-balances reread instead of the O(window) oracle rescan, on the -invariant that a reconcile pass never writes ``qbit_share_ledger`` -(recorded in ``lab.prism.accepted_preview_telemetry``). Its tests pin one -mutation at a time. The alert hour was a dense accepted-tip cadence, so -this module drives several consecutive trusted passes over one fixed -accepted-share window through the real coordinator adapters while payout -state moves through the reconciliation-owned mutations the ledger actually -performs -- a maturation, an inactive marking, a reactivation -- interleaved -with passes that change nothing and with the post-confirm and landing entry -points. - -Per pass it records the intent handed to candidate preparation, the ledger -artifact that came back, the balances that were published and the ledger -reads that were paid for, and pins them against each other and against the -fixed-cardinality telemetry: a mutating pass rereads balances over the -armed window and never rescans it, a non-mutating pass invents neither, and -a lost mutator response mid-cadence still fails closed to the full rescan. -Everything runs on a frozen anchor clock; nothing here sleeps or compares -elapsed time. -""" - -from __future__ import annotations - -import re -import unittest -from collections import Counter -from typing import Callable -from unittest.mock import patch - -from lab.prism.accepted_preview_telemetry import ( - PRISM_ACCEPTED_LANDING_PHASES, - PRISM_LEDGER_READ_OPERATIONS, - PRISM_PAYOUT_WINDOW_FULL_RESCAN_PATHS, - PRISM_PAYOUT_WINDOW_FULL_RESCAN_REASONS, - PRISM_REORG_RECONCILE_CALLERS, - PRISM_REORG_RECONCILE_STEPS, - ensure_accepted_preview_telemetry, - fold_ledger_read_stats, -) -from lab.prism.metrics import MetricsRenderer -from lab.prism.share_ledger import PsqlShareLedger -from tests.test_prism_issue_224_core import ( - BALANCE_REREAD_INTENT, - FULL_RESCAN_INTENT, - PLAIN_INTENT, - CarryLedger, - QuietCoordinatorTestCase, - configured_coordinator, - full_rescan_total, - pass_counts, - step_count, -) - - -# One confirmed pool block below the fake node's tip (height 100). The -# cadence flips the active chain at its height between its own hash and a -# competitor to drive the inactive and reactivated transitions. -POOL_BLOCK_HASH = "ab" * 32 -POOL_BLOCK_HEIGHT = 90 -COMPETING_BLOCK_HASH = "cd" * 32 -FIXED_NOW_MS = 1_000_000 - -# The carry aggregate after each mutation. The values are arbitrary but -# pairwise distinct, so a publication that reused an earlier generation's -# balances cannot equal the live read by coincidence. ``CarryLedger`` owns -# the initial 546 and the post-maturation 777. -INITIAL_BALANCE = 546 -MATURED_BALANCE = 777 -INACTIVE_BALANCE = 101 -REACTIVATED_BALANCE = 909 -LOST_RESPONSE_BALANCE = 4321 - - -class CadenceLedger(CarryLedger): - """A carry ledger whose pool-block state feeds the watch loop. - - ``SingleWriterShareLedger`` already models pool-block chain state (the - prepared/confirmed/inactive transitions with their audit ordinals); the - memory backend just never exposes those rows to ``reorg_watch_blocks``. - This fixture exposes them, and moves the carry aggregate on every - mutation that reports a row changed, the way the PostgreSQL functions - move ``qbit_payout_carry_forward`` -- and, exactly as in PostgreSQL, - never touches an accepted share. - - The prior-balances read is attributed with the recorder borrowed from - ``PsqlShareLedger``, so the operation name, the cell shape and the - rendering are the shipped ones; the fixture supplies only the trigger. - """ - - _ensure_ledger_read_timings = PsqlShareLedger._ensure_ledger_read_timings - _note_ledger_read_timing = PsqlShareLedger._note_ledger_read_timing - ledger_read_gate_stats = PsqlShareLedger.ledger_read_gate_stats - - def __init__(self) -> None: - super().__init__() - self.lose_next_maturation_response = False - - def current_prior_balances(self) -> list[dict[str, object]]: - balances = super().current_prior_balances() - self._note_ledger_read_timing( - "current_prior_balances", - gate_wait_seconds=0.0, - execute_seconds=0.0, - timed_out=False, - ) - return balances - - def reorg_watch_blocks(self, *, active_tip_height: int) -> list[dict[str, object]]: - with self._lock: - return [ - { - "block_hash": block_hash, - "block_height": block[0], - "chain_state": block[1], - } - for block_hash, block in self._memory_pool_blocks.items() - if block[1] in {"confirmed", "inactive"} - ] - - def mark_pool_block_inactive(self, **kwargs: object) -> dict[str, int | str]: - result = super().mark_pool_block_inactive(**kwargs) # type: ignore[arg-type] - if result["inactive_count"]: - self.balance_sats = INACTIVE_BALANCE - return result - - def reactivate_pool_block(self, **kwargs: object) -> dict[str, int | str]: - result = super().reactivate_pool_block(**kwargs) # type: ignore[arg-type] - if result["reactivated_count"]: - self.balance_sats = REACTIVATED_BALANCE - return result - - def mark_mature_pool_payouts(self, *, active_tip_height: int) -> dict[str, int | str]: - if self.lose_next_maturation_response: - # The statement committed server-side; only its response was - # lost, so the aggregate moved and the caller cannot know it. - self.lose_next_maturation_response = False - self.matured_pending = 0 - self.balance_sats = LOST_RESPONSE_BALANCE - raise ConnectionError("mutation response lost") - return super().mark_mature_pool_payouts(active_tip_height=active_tip_height) - - -class CandidateRecorder: - """Every candidate preparation: the intent asked for, the artifact built. - - The reconciler reaches ``_prepared_payout_state_candidate`` through a - port resolved at call time, so a per-instance wrapper sees exactly the - kwargs it forwarded and the real candidate the coordinator prepared. - """ - - def __init__(self, server) -> None: - self.records: list[tuple[dict[str, bool], object]] = [] - original = server._prepared_payout_state_candidate - - def prepared(captured, **kwargs): - candidate = original(captured, **kwargs) - self.records.append((dict(kwargs), candidate.ledger_artifact)) - return candidate - - server._prepared_payout_state_candidate = prepared - - -def cadence_coordinator(): - """A reconciling coordinator over one confirmed pool block. - - The debounce decision compares ``time.monotonic()`` against the minimum - build interval; it is held far above any test's wall-clock so the - cadence rides the armed window by construction rather than by racing an - elapsed-time threshold. - """ - ledger = CadenceLedger() - server, rpc, _ledger, _artifacts = configured_coordinator(ledger) - server.reorg_reconciler_enabled = True - server.reorg_reconcile_cache_seconds = 0.0 - server.payout_artifact_min_build_interval_seconds = 3_600.0 - active_chain = {POOL_BLOCK_HEIGHT: POOL_BLOCK_HASH} - original_call = rpc.call - - def call(method: str, params: list[object] | None = None) -> object: - if method == "getblockhash": - rpc.calls.append(method) - return active_chain[int(params[0])] # type: ignore[index] - return original_call(method, params) - - rpc.call = call # type: ignore[method-assign] - ledger.persist_accepted_block( - block_hash=POOL_BLOCK_HASH, - block_height=POOL_BLOCK_HEIGHT, - parent_hash="11" * 32, - final_bundle={}, - audit_report={}, - ) - ledger.confirm_accepted_block( - block_hash=POOL_BLOCK_HASH, - active_tip_height=POOL_BLOCK_HEIGHT, - ) - return server, rpc, ledger, active_chain - - -def label_values(lines: list[str]) -> dict[str, set[str]]: - values: dict[str, set[str]] = {} - for line in lines: - for name, value in re.findall(r'(\w+)="([^"]*)"', line): - values.setdefault(name, set()).add(value) - return values - - -def sample(lines: list[str], series: str) -> int: - """The integer value of one exact series line, which must exist once.""" - matches = [ - line for line in lines if line.startswith(series + " ") - ] - if len(matches) != 1: - raise AssertionError(f"{series}: {len(matches)} lines") - return int(float(matches[0].split(" ", 1)[1])) - - -class DenseCadenceTests(QuietCoordinatorTestCase): - """Consecutive trusted passes over one fixed window, through the real adapters.""" - - def setUp(self) -> None: - super().setUp() - self.server, self.rpc, self.ledger, self.active_chain = cadence_coordinator() - self.telemetry = ensure_accepted_preview_telemetry(self.server) - self.recorder = CandidateRecorder(self.server) - self.tip = str(self.rpc.tip) - # What the telemetry must report once the cadence is over, kept as - # the cadence runs so the story is told by the passes themselves. - self.expected_passes: Counter[str] = Counter() - self.expected_steps: Counter[tuple[str, str]] = Counter() - self.window_identity: tuple[object, ...] | None = None - self.armed_window: object = None - clock = patch("lab.prism.prism_coordinator.now_ms", return_value=FIXED_NOW_MS) - clock.start() - self.addCleanup(clock.stop) - - # -- helpers ----------------------------------------------------------- - - def published(self): - with self.server._job_cache_lock: - published = self.server._published_payout_state.artifact - return published - - def published_generation(self) -> int: - published = self.published() - return 0 if published is None else int(published.generation) - - def run_pass( - self, - label: str, - *, - intent: dict[str, bool] | None, - reads: int, - mutations: int, - window_mode: str | None = None, - caller: str = "other", - run: Callable[[], object] | None = None, - ) -> object: - """Run one pass and pin its per-pass contract. - - ``intent`` is the preparation kwargs the pass must forward, or - ``None`` when it must prepare nothing; ``reads`` is how many live - prior-balances reads it may pay for; ``mutations`` how many ledger - mutators it runs (the maturation sweep always runs once). - """ - reads_before = self.ledger.prior_balance_reads - records_before = len(self.recorder.records) - generation_before = self.published_generation() - - result = ( - run() - if run is not None - else self.server.reconcile_prism_pool_blocks_once(tip_hash=self.tip) - ) - - prepared = intent is not None - self.expected_passes[caller] += 1 - self.expected_steps[(caller, "admission_wait")] += 1 - # getblockcount plus one getblockhash for the watched row. - self.expected_steps[(caller, "chain_probe")] += 2 - # The watch rows plus the stranded-prepared sweep. - self.expected_steps[(caller, "watch_query")] += 2 - self.expected_steps[(caller, "mutations")] += mutations - self.expected_steps[(caller, "candidate_prepare")] += int(prepared) - self.expected_steps[(caller, "publish")] += int(prepared) - - records = self.recorder.records[records_before:] - self.assertEqual( - self.ledger.prior_balance_reads, reads_before + reads, label - ) - if not prepared: - self.assertEqual(records, [], label) - self.assertEqual(self.published_generation(), generation_before, label) - return result - - self.assertEqual(len(records), 1, label) - kwargs, artifact = records[0] - self.assertEqual(kwargs, intent, label) - self.assertIsNotNone(artifact, label) - self.assertEqual(artifact.window_build_mode, window_mode, label) - # The publication carries exactly the live carry aggregate. - published = self.published() - self.assertIsNotNone(published, label) - self.assertEqual(published.generation, generation_before + 1, label) - self.assertEqual( - [row["balance_sats"] for row in published.prior_balances()], - [self.ledger.balance_sats], - label, - ) - identity = ( - artifact.snapshot_anchor_ms, - artifact.share_snapshot_sha256, - tuple(artifact.shares_json), - ) - if self.window_identity is None: - self.window_identity = identity - self.armed_window = self.server._incremental_payout_artifact_window - self.assertIsNotNone(self.armed_window, label) - return result - # The same window: same anchor, same digest, same shares, and the - # very object the cold start armed -- never replaced, never - # rescanned. - self.assertEqual(identity, self.window_identity, label) - self.assertIs( - self.server._incremental_payout_artifact_window, - self.armed_window, - label, - ) - self.assertIsNone(artifact.window_full_rescan_reason, label) - return result - - def assert_no_window_rescan_since_cold_start(self) -> None: - self.assertEqual(self.ledger.full_snapshot_calls, 1) - self.assertEqual(self.ledger.delta_snapshot_calls, 0) - self.assertEqual(self.server.payout_artifact_event_counts["full_rescan"], 1) - snapshot = self.telemetry.snapshot() - self.assertEqual(full_rescan_total(snapshot), 1) - self.assertEqual(snapshot["full_rescans"][("cold_start", "in_process")]["count"], 1) - - # -- the cadence -------------------------------------------------------- - - def test_confirmed_mutations_reread_balances_over_one_exact_window(self) -> None: - ledger, chain = self.ledger, self.active_chain - self.assertEqual(ledger.balance_sats, INITIAL_BALANCE) - - # 1. Cold start: the tip is reserved, so the pass publishes; the - # window comes from the oracle once and the balances from the - # ledger once, with the plain pre-#224 intent. - first = self.run_pass( - "cold start", - intent=PLAIN_INTENT, - reads=1, - mutations=1, - window_mode="full_rescan", - ) - self.assertEqual(first["published_generation"], 1) - self.assertEqual(first["watched_blocks"], 1) - - # 2. A maturation moves the aggregate: reread, same window. - ledger.matured_pending = 1 - second = self.run_pass( - "maturation", - intent=BALANCE_REREAD_INTENT, - reads=1, - mutations=1, - window_mode="debounced", - ) - self.assertEqual(second["matured_payouts"], 1) - self.assertEqual(ledger.balance_sats, MATURED_BALANCE) - - # 3. Nothing moved and nothing requires publication: no - # preparation, no read, no generation. - third = self.run_pass("quiet", intent=None, reads=0, mutations=1) - self.assertIsNone(third["published_generation"]) - - # 4. The block leaves the active chain: inactive, reread. - chain[POOL_BLOCK_HEIGHT] = COMPETING_BLOCK_HASH - fourth = self.run_pass( - "inactive", - intent=BALANCE_REREAD_INTENT, - reads=1, - mutations=2, - window_mode="debounced", - ) - self.assertEqual(fourth["inactive_blocks"], 1) - self.assertEqual(ledger.balance_sats, INACTIVE_BALANCE) - - # 5. Still off the active chain: the inactive row is left alone. - fifth = self.run_pass("quiet while inactive", intent=None, reads=0, mutations=1) - self.assertEqual(fifth["inactive_blocks"], 0) - self.assertEqual(fifth["reactivated_blocks"], 0) - - # 6. The chain flips back: reactivated, reread. - chain[POOL_BLOCK_HEIGHT] = POOL_BLOCK_HASH - sixth = self.run_pass( - "reactivated", - intent=BALANCE_REREAD_INTENT, - reads=1, - mutations=2, - window_mode="debounced", - ) - self.assertEqual(sixth["reactivated_blocks"], 1) - self.assertEqual(ledger.balance_sats, REACTIVATED_BALANCE) - - # 7. A post-confirm forced publication with nothing to mutate keeps - # the plain intent and reuses the published balances verbatim. - seventh = self.run_pass( - "post-confirm republish", - caller="post_confirm", - run=lambda: self.server.reconcile_prism_pool_blocks_once( - tip_hash=self.tip, - _force_publish=True, - _source_reserved=True, - ), - intent=PLAIN_INTENT, - reads=0, - mutations=1, - window_mode="debounced", - ) - self.assertIsNotNone(seventh["published_generation"]) - - # 8. The landing's in-lock pass with a second maturation. - ledger.matured_pending = 1 - self.run_pass( - "landing maturation", - caller="landing", - run=lambda: self.assertTrue( - self.server.ensure_reorg_reconciled_for_tip( - self.tip, _coalesce_same_tip=False - ) - ), - intent=BALANCE_REREAD_INTENT, - reads=1, - mutations=1, - window_mode="debounced", - ) - self.assertEqual(ledger.balance_sats, MATURED_BALANCE) - - # The window was read from the oracle exactly once, for the cold - # start, and never rescanned; every later read was a balance read - # owned by a payout-changing pass. - self.assert_no_window_rescan_since_cold_start() - self.assertEqual(ledger.prior_balance_reads, 5) - self.assertEqual(self.published_generation(), 6) - self.assert_telemetry_tells_the_same_story() - - def assert_telemetry_tells_the_same_story(self) -> None: - snapshot = self.telemetry.snapshot() - self.assertEqual( - pass_counts(snapshot), - {caller: self.expected_passes[caller] for caller in PRISM_REORG_RECONCILE_CALLERS}, - ) - for caller in PRISM_REORG_RECONCILE_CALLERS: - for step in PRISM_REORG_RECONCILE_STEPS: - self.assertEqual( - step_count(snapshot, caller, step), - self.expected_steps[(caller, step)], - (caller, step), - ) - # Closed products, and nothing the cadence did grew them. - self.assertEqual( - set(snapshot["reconcile_steps"]), - { - (caller, step) - for caller in PRISM_REORG_RECONCILE_CALLERS - for step in PRISM_REORG_RECONCILE_STEPS - }, - ) - self.assertEqual( - set(snapshot["full_rescans"]), - { - (reason, path) - for reason in PRISM_PAYOUT_WINDOW_FULL_RESCAN_REASONS - for path in PRISM_PAYOUT_WINDOW_FULL_RESCAN_PATHS - }, - ) - # The reads the passes paid for are the reads the ledger attributed, - # under the one contract operation, and nothing else. - folded = fold_ledger_read_stats(self.ledger.ledger_read_gate_stats()) - self.assertEqual(list(folded), ["current_prior_balances"]) - self.assertIn("current_prior_balances", PRISM_LEDGER_READ_OPERATIONS) - self.assertEqual( - folded["current_prior_balances"]["calls_total"], - self.ledger.prior_balance_reads, - ) - # And the operator sees the same story on /metrics with only - # closed-vocabulary labels. - renderer = MetricsRenderer(self.server) - lines = ( - renderer.accepted_preview_attribution_metrics_lines() - + renderer._ledger_read_gate_metric_lines() - ) - labels = label_values(lines) - self.assertEqual(labels["phase"], set(PRISM_ACCEPTED_LANDING_PHASES)) - self.assertEqual(labels["caller"], set(PRISM_REORG_RECONCILE_CALLERS)) - self.assertEqual(labels["step"], set(PRISM_REORG_RECONCILE_STEPS)) - self.assertEqual(labels["reason"], set(PRISM_PAYOUT_WINDOW_FULL_RESCAN_REASONS)) - self.assertEqual(labels["path"], set(PRISM_PAYOUT_WINDOW_FULL_RESCAN_PATHS)) - self.assertEqual(labels["operation"], {"current_prior_balances"}) - self.assertEqual( - sample(lines, 'qbit_prism_ledger_read_calls_total{operation="current_prior_balances"}'), - self.ledger.prior_balance_reads, - ) - for caller in PRISM_REORG_RECONCILE_CALLERS: - self.assertEqual( - sample(lines, f'qbit_prism_reorg_reconcile_pass_seconds_count{{caller="{caller}"}}'), - self.expected_passes[caller], - ) - self.assertEqual( - sample( - lines, - 'qbit_prism_payout_window_full_rescan_seconds_count' - '{reason="reconcile_invalidation",path="in_process"}', - ), - 0, - ) - self.assertEqual( - sample( - lines, - 'qbit_prism_payout_window_full_rescan_seconds_count' - '{reason="cold_start",path="in_process"}', - ), - 1, - ) - - def test_a_lost_mutator_response_mid_cadence_keeps_the_full_rescan(self) -> None: - """The fail-closed path is unchanged under the same cadence. - - The maturation may have committed before its response was lost, so - the row counts are unknowable. The pass republishes against the - forced full rescan, which necessarily rereads the current balances, - and records the rescan under its own reason. - """ - ledger = self.ledger - self.run_pass( - "cold start", - intent=PLAIN_INTENT, - reads=1, - mutations=1, - window_mode="full_rescan", - ) - ledger.matured_pending = 1 - self.run_pass( - "maturation", - intent=BALANCE_REREAD_INTENT, - reads=1, - mutations=1, - window_mode="debounced", - ) - self.assert_no_window_rescan_since_cold_start() - generation_before = self.published_generation() - armed_before = self.server._incremental_payout_artifact_window - - ledger.lose_next_maturation_response = True - with self.assertRaisesRegex(ConnectionError, "mutation response lost"): - self.server.reconcile_prism_pool_blocks_once(tip_hash=self.tip) - - kwargs, artifact = self.recorder.records[-1] - self.assertEqual(kwargs, FULL_RESCAN_INTENT) - self.assertEqual(artifact.window_build_mode, "full_rescan") - self.assertEqual(artifact.window_full_rescan_reason, "reconcile_invalidation") - # The oracle ran again and re-armed a fresh window whose content is - # the same (no share landed) but which is no longer the cold-start - # object. - self.assertEqual(ledger.full_snapshot_calls, 2) - assert self.window_identity is not None - self.assertEqual(artifact.share_snapshot_sha256, self.window_identity[1]) - self.assertIsNot(self.server._incremental_payout_artifact_window, armed_before) - # The republication carries the balances the lost mutation left. - self.assertEqual(ledger.prior_balance_reads, 3) - published = self.published() - assert published is not None - self.assertEqual(published.generation, generation_before + 1) - self.assertEqual( - [row["balance_sats"] for row in published.prior_balances()], - [LOST_RESPONSE_BALANCE], - ) - service = self.server._ensure_reorg_reconciler_service() - self.assertEqual(service.snapshot().reconcile_error_count, 1) - snapshot = self.telemetry.snapshot() - self.assertEqual(pass_counts(snapshot)["other"], 3) - self.assertEqual(step_count(snapshot, "other", "mutations"), 3) - self.assertEqual(step_count(snapshot, "other", "candidate_prepare"), 3) - self.assertEqual(step_count(snapshot, "other", "publish"), 3) - rescans = snapshot["full_rescans"] - self.assertEqual(rescans[("reconcile_invalidation", "in_process")]["count"], 1) - self.assertEqual(rescans[("cold_start", "in_process")]["count"], 1) - self.assertEqual(full_rescan_total(snapshot), 2) - self.assertEqual(self.server.payout_artifact_event_counts["full_rescan"], 2) - - -if __name__ == "__main__": - unittest.main() diff --git a/tests/test_prism_issue_224_core.py b/tests/test_prism_issue_224_core.py deleted file mode 100644 index 0fc3064e5..000000000 --- a/tests/test_prism_issue_224_core.py +++ /dev/null @@ -1,1040 +0,0 @@ -"""Issue #224 Wave 1 core: reconciliation rereads balances, never the window. - -A reconcile pass mutates pool-block, payout-entry and carry state but never -``qbit_share_ledger`` (the invariant recorded in -``lab.prism.accepted_preview_telemetry``). At a fixed anchor the accepted -share window therefore stays exact after a maturation, an inactive marking -or a reactivation, and only the prior balances need a live reread. Before -this wave every confirmed mutation answered with ``force_full_window_rescan`` -and paid an O(window) oracle read under the writer lock on the accepted -preview's critical path. - -These tests pin the split between the two intents, the fail-closed paths -that keep the full rescan, and the fixed-cardinality attribution the -telemetry contract defines for reconcile passes, steps, callers and actual -full rescans. -""" - -from __future__ import annotations - -import threading -import time -import unittest -from contextlib import contextmanager, nullcontext -from types import SimpleNamespace -from unittest.mock import patch - -from lab.prism.accepted_preview_telemetry import ( - AcceptedPreviewTelemetry, - PRISM_PAYOUT_WINDOW_FULL_RESCAN_PATHS, - PRISM_PAYOUT_WINDOW_FULL_RESCAN_REASONS, - PRISM_REORG_RECONCILE_CALLERS, - PRISM_REORG_RECONCILE_STEPS, - ensure_accepted_preview_telemetry, -) -from lab.prism.payout_state import ( - PayoutStatePublicationBlocked, - _PayoutWindowMaterialization, -) -from lab.prism.reorg_reconciler import ReorgPorts, ReorgReconcilerService -from lab.prism.share_ledger import IncrementalWindowAdvanceStats -from tests.prism_coordinator_test_support import ( - IncrementalRecordingLedger, - append_incremental_share, - coordinator, -) - - -BALANCE_REREAD_INTENT = { - "force_full_window_rescan": False, - "force_prior_balances_read": True, -} -PLAIN_INTENT = {"force_full_window_rescan": False} -FULL_RESCAN_INTENT = {"force_full_window_rescan": True} - - -def full_rescan_total(snapshot: dict[str, object]) -> int: - rescans = snapshot["full_rescans"] - assert isinstance(rescans, dict) - return sum(int(stats["count"]) for stats in rescans.values()) - - -def pass_counts(snapshot: dict[str, object]) -> dict[str, int]: - passes = snapshot["reconcile_passes"] - assert isinstance(passes, dict) - return {caller: int(stats["count"]) for caller, stats in passes.items()} - - -def step_count(snapshot: dict[str, object], caller: str, step: str) -> int: - steps = snapshot["reconcile_steps"] - assert isinstance(steps, dict) - return int(steps[(caller, step)]["count"]) - - -# --- reconciler-level fixtures ------------------------------------------- - - -class MutatingLedger: - """One confirmed row that leaves the active chain plus a maturation.""" - - def __init__( - self, - *, - rows: list[dict[str, object]] | None = None, - matured: int = 1, - lose_mutation_response: bool = False, - ) -> None: - self.rows = ( - [{"block_height": 8, "block_hash": "aa", "chain_state": "confirmed"}] - if rows is None - else rows - ) - self.matured = matured - self.lose_mutation_response = lose_mutation_response - self.inactivated: list[str] = [] - - def reorg_watch_blocks(self, *, active_tip_height: int) -> list[dict[str, object]]: - return [dict(row) for row in self.rows] - - def mark_pool_block_inactive(self, **kwargs: object) -> dict[str, int]: - self.inactivated.append(str(kwargs["block_hash"])) - if self.lose_mutation_response: - raise ConnectionError("mutation response lost") - return {"inactive_count": 1} - - def reactivate_pool_block(self, **kwargs: object) -> dict[str, int]: - return {"reactivated_count": 0} - - def mark_mature_pool_payouts(self, **_kwargs: object) -> dict[str, int]: - return {"matured_count": self.matured} - - -def make_ports( - *, - ledger: object, - telemetry: AcceptedPreviewTelemetry | None = None, - chain_untrusted: bool = False, - publication_required: bool = False, - publish: object = 7, - serialized_refusals: list[BaseException] | None = None, -) -> tuple[ReorgPorts, list[tuple[str, object]], dict[str, ReorgReconcilerService]]: - """Ports whose coordinator seams route back into the service under test. - - ``reconcile_with_admission`` and ``reconcile_serialized`` stand in for - the coordinator's public entry and serialized adapter, so the caller - stamp and the admission stamp cross the same seams they cross in the - coordinator; ``serialized_refusals`` raises in the adapter before the - core pass, the way a landed preview refuses a pass. - """ - events: list[tuple[str, object]] = [] - holder: dict[str, ReorgReconcilerService] = {} - state_lock = threading.RLock() - active_chain = {8: "not-aa"} - refusals = list(serialized_refusals or []) - - def rpc_call(method: str, params: object = None) -> object: - if method == "getbestblockhash": - return "tip" - if method == "getblockcount": - return 10 - if method == "getblockhash": - height = int(params[0]) # type: ignore[index] - events.append(("getblockhash", height)) - return active_chain[height] - raise AssertionError(method) - - @contextmanager - def publication_guard(): - yield - - def prepared_candidate(captured, **kwargs): - events.append(("prepare", kwargs)) - return SimpleNamespace(captured=captured) - - def reconcile_serialized(**kwargs: object) -> dict[str, object]: - events.append(("serialized", dict(kwargs))) - if refusals: - raise refusals.pop(0) - return holder["service"].reconcile( - tip_hash=kwargs.get("tip_hash"), # type: ignore[arg-type] - force_publish=bool(kwargs.get("_force_publish", False)), - source_reserved=bool(kwargs.get("_source_reserved", False)), - ) - - ports = ReorgPorts( - rpc_call=rpc_call, - ledger=lambda: ledger, - ensure_job_cache_state=lambda: None, - state_lock=lambda: state_lock, - source_tip=lambda: "tip", - reserve_external_tip=lambda tip: events.append(("reserve", tip)), - max_supersession_retries=lambda: 1, - prepare_lock=nullcontext, - capture_source=lambda: (0, 0, "tip", "test", 0.0), - prepared_candidate=prepared_candidate, - captured_publication_required=lambda _captured: publication_required, - block_publication=lambda **kwargs: events.append(("block", kwargs)), - publication_guard=publication_guard, - publish_candidate=lambda _candidate: publish, # type: ignore[return-value] - observe_preparation=lambda elapsed: None, - chain_view_untrusted=lambda: chain_untrusted, - reorg_proof_snapshot=lambda: ("tip", 0), - flight_wait_seconds=lambda: 30.0, - prefetch_join_timeout_seconds=lambda: 20.0, - reconcile_with_admission=lambda **kwargs: ( - holder["service"].reconcile_with_flights(**kwargs) - ), - reconcile_serialized=reconcile_serialized, - ensure_tip=lambda tip: holder["service"].ensure_tip(tip), - accepted_preview_telemetry=( - (lambda: telemetry) if telemetry is not None else None - ), - ) - return ports, events, holder - - -def make_service(**kwargs: object) -> tuple[ - ReorgReconcilerService, - list[tuple[str, object]], - AcceptedPreviewTelemetry, -]: - telemetry = kwargs.pop("telemetry", None) or AcceptedPreviewTelemetry() - ports, events, holder = make_ports(telemetry=telemetry, **kwargs) # type: ignore[arg-type] - service = ReorgReconcilerService(ports, cache_seconds=0.0) - holder["service"] = service - return service, events, telemetry - - -def prepare_intents(events: list[tuple[str, object]]) -> list[object]: - return [payload for name, payload in events if name == "prepare"] - - -class ReconcileIntentTests(unittest.TestCase): - """The confirmed-mutation path asks for balances, not the window.""" - - def test_confirmed_mutations_request_a_balance_reread_over_the_window( - self, - ) -> None: - ledger = MutatingLedger() - service, events, _telemetry = make_service(ledger=ledger) - - summary = service.reconcile(tip_hash="tip") - - self.assertEqual(ledger.inactivated, ["aa"]) - self.assertEqual(summary["inactive_blocks"], 1) - self.assertEqual(summary["matured_payouts"], 1) - self.assertEqual(summary["published_generation"], 7) - # Exactly one preparation, and it carries the narrow #224 intent: - # the exact window is reused, only the carry aggregate is reread. - self.assertEqual(prepare_intents(events), [BALANCE_REREAD_INTENT]) - # Publication still fences delivery until the reread balances land. - self.assertIn(("block", {"force": True}), events) - - def test_maturation_alone_requests_the_reread(self) -> None: - ledger = MutatingLedger(rows=[], matured=3) - service, events, _telemetry = make_service(ledger=ledger) - - summary = service.reconcile(tip_hash="tip") - - self.assertEqual(summary["matured_payouts"], 3) - self.assertEqual(prepare_intents(events), [BALANCE_REREAD_INTENT]) - - def test_a_pass_without_mutations_keeps_the_plain_intent(self) -> None: - ledger = MutatingLedger(rows=[], matured=0) - service, events, _telemetry = make_service( - ledger=ledger, - publication_required=True, - ) - - service.reconcile(tip_hash="tip") - - # No reread is forced (the published balances are still exact) and - # the pre-#224 keyword shape reaches preparation seams unchanged. - self.assertEqual(prepare_intents(events), [PLAIN_INTENT]) - - def test_no_mutation_and_no_required_publication_prepares_nothing( - self, - ) -> None: - ledger = MutatingLedger(rows=[], matured=0) - service, events, _telemetry = make_service(ledger=ledger) - - summary = service.reconcile(tip_hash="tip") - - self.assertIsNone(summary["published_generation"]) - self.assertEqual(prepare_intents(events), []) - - def test_lost_mutator_response_keeps_the_forced_full_rescan(self) -> None: - """Error-after-possible-commit stays fail-closed and unchanged. - - The mutator may have committed before its response was lost, so the - row counts are unknowable. The pass republishes against the forced - full rescan exactly as before: that path necessarily rereads the - current prior balances, so stale balances can never be reused, and - the narrow reread intent is not what it asks for. - """ - ledger = MutatingLedger(lose_mutation_response=True) - service, events, telemetry = make_service(ledger=ledger) - - with self.assertRaisesRegex(ConnectionError, "mutation response lost"): - service.reconcile(tip_hash="tip") - - self.assertEqual(prepare_intents(events), [FULL_RESCAN_INTENT]) - self.assertIn(("block", {"force": True}), events) - self.assertEqual(service.snapshot().reconcile_error_count, 1) - # The error exit still records its pass and the steps it reached, - # including the error candidate's preparation and publication. - snapshot = telemetry.snapshot() - self.assertEqual(pass_counts(snapshot)["other"], 1) - self.assertEqual(step_count(snapshot, "other", "mutations"), 1) - self.assertEqual(step_count(snapshot, "other", "candidate_prepare"), 1) - self.assertEqual(step_count(snapshot, "other", "publish"), 1) - - def test_a_read_phase_failure_forces_nothing(self) -> None: - class ReadFailingLedger(MutatingLedger): - def reorg_watch_blocks(self, *, active_tip_height: int): - raise TimeoutError("reconcile prefetch join exceeded 20s") - - service, events, telemetry = make_service(ledger=ReadFailingLedger()) - - with self.assertRaises(TimeoutError): - service.reconcile(tip_hash="tip") - - self.assertEqual(prepare_intents(events), []) - snapshot = telemetry.snapshot() - self.assertEqual(pass_counts(snapshot)["other"], 1) - self.assertEqual(step_count(snapshot, "other", "watch_query"), 1) - self.assertEqual(step_count(snapshot, "other", "candidate_prepare"), 0) - - def test_untrusted_forced_publication_prepares_without_any_intent( - self, - ) -> None: - service, events, _telemetry = make_service( - ledger=MutatingLedger(), - chain_untrusted=True, - ) - - summary = service.reconcile(tip_hash="tip", force_publish=True) - - self.assertTrue(summary["untrusted"]) - self.assertEqual(prepare_intents(events), [{}]) - - -class ReconcileAttributionTests(unittest.TestCase): - """Pass/step/caller observations through the real entry points.""" - - def test_direct_pass_records_pass_and_every_step_it_ran(self) -> None: - service, _events, telemetry = make_service(ledger=MutatingLedger()) - - service.reconcile(tip_hash="tip") - - snapshot = telemetry.snapshot() - counts = pass_counts(snapshot) - self.assertEqual(counts["other"], 1) - for caller in PRISM_REORG_RECONCILE_CALLERS: - if caller != "other": - self.assertEqual(counts[caller], 0, caller) - # getblockcount plus one getblockhash; the watch query; the inactive - # marking plus the maturation; the preparation; the publication. - self.assertEqual(step_count(snapshot, "other", "chain_probe"), 2) - self.assertEqual(step_count(snapshot, "other", "watch_query"), 1) - self.assertEqual(step_count(snapshot, "other", "mutations"), 2) - self.assertEqual(step_count(snapshot, "other", "candidate_prepare"), 1) - self.assertEqual(step_count(snapshot, "other", "publish"), 1) - # A direct core call never crossed the serialized adapter. - self.assertEqual(step_count(snapshot, "other", "admission_wait"), 0) - # Nothing here is a full rescan. - self.assertEqual(full_rescan_total(snapshot), 0) - - def test_untrusted_pass_records_its_pass_and_no_steps(self) -> None: - service, _events, telemetry = make_service( - ledger=MutatingLedger(), - chain_untrusted=True, - ) - - service.reconcile(tip_hash="tip") - - snapshot = telemetry.snapshot() - self.assertEqual(pass_counts(snapshot)["other"], 1) - for step in PRISM_REORG_RECONCILE_STEPS: - self.assertEqual(step_count(snapshot, "other", step), 0, step) - - def test_forced_publication_without_context_is_post_confirm(self) -> None: - service, events, telemetry = make_service(ledger=MutatingLedger()) - - service.reconcile_with_flights( - tip_hash="tip", - _force_publish=True, - _source_reserved=True, - ) - - snapshot = telemetry.snapshot() - self.assertEqual(pass_counts(snapshot)["post_confirm"], 1) - self.assertEqual(pass_counts(snapshot)["other"], 0) - # Crossing the adapter records the admission wait for that caller. - self.assertEqual( - step_count(snapshot, "post_confirm", "admission_wait"), 1 - ) - # The adapter seam still receives exactly its historical kwargs. - self.assertIn( - ( - "serialized", - { - "tip_hash": "tip", - "_force_publish": True, - "_source_reserved": True, - }, - ), - events, - ) - - def test_direct_forced_core_call_is_post_confirm_too(self) -> None: - service, _events, telemetry = make_service(ledger=MutatingLedger()) - service.reconcile(tip_hash="tip", force_publish=True) - self.assertEqual(pass_counts(telemetry.snapshot())["post_confirm"], 1) - - def test_flight_bypass_attributes_the_landing(self) -> None: - service, events, telemetry = make_service(ledger=MutatingLedger()) - - self.assertTrue(service.ensure_tip("tip", _coalesce_same_tip=False)) - - snapshot = telemetry.snapshot() - self.assertEqual(pass_counts(snapshot)["landing"], 1) - self.assertEqual(step_count(snapshot, "landing", "admission_wait"), 1) - self.assertEqual(step_count(snapshot, "landing", "candidate_prepare"), 1) - self.assertIn(("serialized", {"tip_hash": "tip"}), events) - - def test_coalescing_ensure_tip_without_context_is_other(self) -> None: - service, _events, telemetry = make_service(ledger=MutatingLedger()) - self.assertTrue(service.ensure_tip("tip")) - self.assertEqual(pass_counts(telemetry.snapshot())["other"], 1) - - def test_job_build_entry_attributes_job_build(self) -> None: - service, _events, telemetry = make_service(ledger=MutatingLedger()) - - self.assertTrue(service.ensure_current()) - - snapshot = telemetry.snapshot() - self.assertEqual(pass_counts(snapshot)["job_build"], 1) - self.assertEqual(step_count(snapshot, "job_build", "admission_wait"), 1) - # The existing lookup family and the new pass family join on the - # same spelling. - self.assertEqual( - service.reorg_reconcile_lookup_snapshot()[("job_build", "serial")], - 1, - ) - - def test_prefetch_lane_attributes_tip_refresh(self) -> None: - service, _events, telemetry = make_service(ledger=MutatingLedger()) - - self.assertTrue(service.prefetch_pass("tip")) - self.assertTrue(service.prefetch_pass("tip", prove=True)) - # The bounded re-prove routes through the same lane. - service.shutdown_prefetch_executor() - self.assertTrue(service.snapshot_tip_bounded("tip")) - - self.assertEqual(pass_counts(telemetry.snapshot())["tip_refresh"], 3) - - def test_explicit_caller_wins_and_unknown_callers_are_refused(self) -> None: - service, _events, telemetry = make_service(ledger=MutatingLedger()) - - service.ensure_tip("tip", _caller="tip_refresh") - service.reconcile_with_flights(tip_hash="tip", _caller="job_build") - service.reconcile(tip_hash="tip", caller="landing") - - counts = pass_counts(telemetry.snapshot()) - self.assertEqual(counts["tip_refresh"], 1) - self.assertEqual(counts["job_build"], 1) - self.assertEqual(counts["landing"], 1) - with self.assertRaisesRegex(ValueError, "unknown reorg reconcile caller"): - service.reconcile(tip_hash="tip", caller="benchmark-run-42") - with self.assertRaisesRegex(ValueError, "unknown reorg reconcile caller"): - service.ensure_tip("tip", _caller="benchmark-run-42") - - def test_outer_entry_point_keeps_its_label_through_nested_seams(self) -> None: - """A prefetched pass reaching ensure_tip via the coordinator stays tip_refresh.""" - service, _events, telemetry = make_service(ledger=MutatingLedger()) - - with service._attributed("tip_refresh"): - service.reconcile_with_flights(tip_hash="tip", _force_publish=True) - - counts = pass_counts(telemetry.snapshot()) - self.assertEqual(counts["tip_refresh"], 1) - self.assertEqual(counts["post_confirm"], 0) - # The stamp is restored once the entry point returns. - self.assertIsNone(service._attributed_caller()) - - def test_adapter_refusal_clears_the_admission_stamp(self) -> None: - service, _events, telemetry = make_service( - ledger=MutatingLedger(), - serialized_refusals=[ - PayoutStatePublicationBlocked("accepted block payout pending") - ], - ) - - with self.assertRaises(PayoutStatePublicationBlocked): - service.reconcile_with_flights(tip_hash="tip") - # A later direct call on this thread must not inherit the stamp. - service.reconcile(tip_hash="tip") - - snapshot = telemetry.snapshot() - self.assertEqual(pass_counts(snapshot)["other"], 1) - self.assertEqual(step_count(snapshot, "other", "admission_wait"), 0) - - def test_disabled_service_records_nothing(self) -> None: - ports, _events, holder = make_ports(ledger=MutatingLedger()) - service = ReorgReconcilerService(ports, enabled=False) - holder["service"] = service - service.reconcile(tip_hash="tip") - self.assertTrue(service.ensure_tip("tip", _coalesce_same_tip=False)) - self.assertEqual( - sum(pass_counts(service.accepted_preview_telemetry.snapshot()).values()), - 0, - ) - - def test_ports_without_an_owner_get_one_private_stable_owner(self) -> None: - ports, _events, holder = make_ports(ledger=MutatingLedger()) - service = ReorgReconcilerService(ports) - holder["service"] = service - owner = service.accepted_preview_telemetry - self.assertIsInstance(owner, AcceptedPreviewTelemetry) - self.assertIs(service.accepted_preview_telemetry, owner) - - service.reconcile(tip_hash="tip") - self.assertEqual(pass_counts(owner.snapshot())["other"], 1) - - def test_port_supplied_owner_is_used_as_is(self) -> None: - shared = AcceptedPreviewTelemetry() - service, _events, telemetry = make_service( - ledger=MutatingLedger(), - telemetry=shared, - ) - self.assertIs(telemetry, shared) - self.assertIs(service.accepted_preview_telemetry, shared) - - -# --- payout-state level --------------------------------------------------- - - -class CarryLedger(IncrementalRecordingLedger): - """Incremental ledger whose carry aggregate moves under the tests' control.""" - - def __init__(self) -> None: - super().__init__() - self.balance_sats = 546 - self.prior_balance_reads = 0 - self.matured_pending = 0 - self.fail_full_snapshot = False - - def current_prior_balances(self) -> list[dict[str, object]]: - self.prior_balance_reads += 1 - return [ - { - "recipient_id": "carry", - "order_key": "01:carry", - "p2mr_program_hex": "66" * 32, - "balance_sats": self.balance_sats, - } - ] - - def snapshot_at_job_issue(self, anchor_job_issued_at_ms, *, window_weight=None): - if self.fail_full_snapshot: - self.full_snapshot_calls += 1 - raise TimeoutError("postgres statement deadline expired") - return super().snapshot_at_job_issue( - anchor_job_issued_at_ms, - window_weight=window_weight, - ) - - def mark_mature_pool_payouts(self, *, active_tip_height: int) -> dict[str, int | str]: - matured, self.matured_pending = self.matured_pending, 0 - if matured: - # Maturation moves carry balances; the share ledger is untouched. - self.balance_sats = 777 - return {"backend": "memory", "matured_count": matured} - - -def configured_coordinator(ledger: CarryLedger | None = None): - """A ready coordinator over a three-share window on a carry ledger. - - ``ledger`` lets the Wave 2 cadence module supply its subclass without - re-stating this setup. - """ - ledger = CarryLedger() if ledger is None else ledger - append_incremental_share(ledger, share_seq=1, accepted_at_ms=999_900) - append_incremental_share(ledger, share_seq=2, accepted_at_ms=999_910) - append_incremental_share(ledger, share_seq=3, accepted_at_ms=999_920) - server, rpc = coordinator(ledger=ledger) - artifacts = server.store_template_artifacts(dict(rpc.template)) - assert artifacts is not None - server._pool_ready_latched = True - server.payout_artifact_min_build_interval_seconds = 60.0 - server.payout_artifact_full_rescan_seconds = 3_600.0 - return server, rpc, ledger, artifacts - - -class QuietCoordinatorTestCase(unittest.TestCase): - """Keep the artifact lifecycle log lines out of the test output.""" - - def setUp(self) -> None: - super().setUp() - printer = patch("builtins.print") - printer.start() - self.addCleanup(printer.stop) - - -class PayoutWindowIntentTests(QuietCoordinatorTestCase): - def test_balance_reread_reuses_the_exact_window_and_reads_live_carry( - self, - ) -> None: - server, _rpc, ledger, artifacts = configured_coordinator() - telemetry = ensure_accepted_preview_telemetry(server) - with patch("lab.prism.prism_coordinator.now_ms", return_value=1_000_000): - published = server._current_payout_state_artifact() - initial = server._build_payout_ledger_artifact( - 0, 0, artifacts.network_difficulty - ) - assert initial is not None - self.assertEqual(initial.window_build_mode, "full_rescan") - self.assertEqual(initial.window_full_rescan_reason, "cold_start") - self.assertEqual(ledger.full_snapshot_calls, 1) - self.assertTrue(server._install_payout_ledger_artifact(initial)) - - ledger.balance_sats = 777 - reads_before = ledger.prior_balance_reads - reused = server._build_payout_ledger_artifact( - 0, 0, artifacts.network_difficulty - ) - assert reused is not None - # Without the intent the published bytes are reused verbatim. - self.assertEqual(reused.prior_balances[0]["balance_sats"], 546) - self.assertEqual( - reused.prior_balances_sha256, published.prior_balances_sha256 - ) - self.assertEqual(ledger.prior_balance_reads, reads_before) - - reread = server._build_payout_ledger_artifact( - 0, - 0, - artifacts.network_difficulty, - force_prior_balances_read=True, - ) - - assert reread is not None - # The window is the armed one: no oracle read, same digest, same - # anchor, and no rescan reason. - self.assertEqual(reread.window_build_mode, "debounced") - self.assertIsNone(reread.window_full_rescan_reason) - self.assertEqual(reread.share_snapshot_sha256, initial.share_snapshot_sha256) - self.assertEqual(reread.snapshot_anchor_ms, initial.snapshot_anchor_ms) - self.assertEqual(ledger.full_snapshot_calls, 1) - self.assertEqual(ledger.delta_snapshot_calls, 0) - # The balances are live. - self.assertEqual(reread.prior_balances[0]["balance_sats"], 777) - self.assertEqual(ledger.prior_balance_reads, reads_before + 1) - self.assertNotEqual( - reread.prior_balances_sha256, published.prior_balances_sha256 - ) - # A balance reread is never an observation of the rescan family. - snapshot = telemetry.snapshot() - self.assertEqual(full_rescan_total(snapshot), 1) - self.assertEqual(snapshot["full_rescans"][("cold_start", "in_process")]["count"], 1) - - def test_balance_reread_rides_the_delta_path_when_shares_landed(self) -> None: - server, _rpc, ledger, artifacts = configured_coordinator() - server.payout_artifact_min_build_interval_seconds = 0.0 - clock_ms = [1_000_000] - with patch( - "lab.prism.prism_coordinator.now_ms", - side_effect=lambda: clock_ms[0], - ): - server._current_payout_state_artifact() - initial = server._build_payout_ledger_artifact( - 0, 0, artifacts.network_difficulty - ) - assert initial is not None - self.assertTrue(server._install_payout_ledger_artifact(initial)) - append_incremental_share(ledger, share_seq=4, accepted_at_ms=1_000_010) - clock_ms[0] = 1_000_020 - ledger.balance_sats = 777 - reread = server._build_payout_ledger_artifact( - 0, - 0, - artifacts.network_difficulty, - force_prior_balances_read=True, - ) - - assert reread is not None - self.assertEqual(reread.window_build_mode, "incremental") - self.assertEqual(reread.window_delta_rows, 1) - self.assertEqual(len(reread.shares_json), 4) - self.assertEqual(ledger.full_snapshot_calls, 1) - self.assertEqual(ledger.delta_snapshot_calls, 1) - self.assertEqual(reread.prior_balances[0]["balance_sats"], 777) - self.assertEqual( - full_rescan_total(ensure_accepted_preview_telemetry(server).snapshot()), - 1, - ) - - def test_force_full_window_rescan_keeps_its_whole_meaning(self) -> None: - server, _rpc, ledger, artifacts = configured_coordinator() - telemetry = ensure_accepted_preview_telemetry(server) - with patch("lab.prism.prism_coordinator.now_ms", return_value=1_000_000): - server._current_payout_state_artifact() - initial = server._build_payout_ledger_artifact( - 0, 0, artifacts.network_difficulty - ) - assert initial is not None - self.assertTrue(server._install_payout_ledger_artifact(initial)) - ledger.balance_sats = 777 - reads_before = ledger.prior_balance_reads - forced = server._build_payout_ledger_artifact( - 0, - 0, - artifacts.network_difficulty, - True, - ) - - assert forced is not None - self.assertEqual(forced.window_build_mode, "full_rescan") - self.assertEqual(forced.window_full_rescan_reason, "reconcile_invalidation") - self.assertEqual(ledger.full_snapshot_calls, 2) - # A full rescan necessarily obtains the current prior balances. - self.assertEqual(forced.prior_balances[0]["balance_sats"], 777) - self.assertEqual(ledger.prior_balance_reads, reads_before + 1) - rescans = telemetry.snapshot()["full_rescans"] - self.assertEqual(rescans[("reconcile_invalidation", "in_process")]["count"], 1) - self.assertEqual(rescans[("cold_start", "in_process")]["count"], 1) - - def test_prepared_candidate_carries_both_intents_distinctly(self) -> None: - server, rpc, ledger, artifacts = configured_coordinator() - telemetry = ensure_accepted_preview_telemetry(server) - captured = (0, 0, str(rpc.tip), "external_tip", time.monotonic()) - with patch("lab.prism.prism_coordinator.now_ms", return_value=1_000_000): - server._current_payout_state_artifact() - initial = server._build_payout_ledger_artifact( - 0, 0, artifacts.network_difficulty - ) - assert initial is not None - self.assertTrue(server._install_payout_ledger_artifact(initial)) - ledger.balance_sats = 777 - - # Compatibility: the pre-#224 signature still works. - plain = server._prepared_payout_state_candidate(captured) - reread = server._prepared_payout_state_candidate( - captured, - force_prior_balances_read=True, - ) - forced = server._prepared_payout_state_candidate( - captured, - force_full_window_rescan=True, - ) - - assert plain.ledger_artifact is not None - assert reread.ledger_artifact is not None - assert forced.ledger_artifact is not None - self.assertEqual(plain.ledger_artifact.window_build_mode, "debounced") - self.assertEqual(plain.ledger_artifact.prior_balances[0]["balance_sats"], 546) - self.assertEqual(reread.ledger_artifact.window_build_mode, "debounced") - self.assertEqual(reread.ledger_artifact.prior_balances[0]["balance_sats"], 777) - self.assertEqual(forced.ledger_artifact.window_build_mode, "full_rescan") - self.assertEqual( - forced.ledger_artifact.window_full_rescan_reason, - "reconcile_invalidation", - ) - self.assertEqual(forced.ledger_artifact.prior_balances[0]["balance_sats"], 777) - self.assertEqual(ledger.full_snapshot_calls, 2) - self.assertEqual( - telemetry.snapshot()["full_rescans"][ - ("reconcile_invalidation", "in_process") - ]["count"], - 1, - ) - - def test_narrow_build_seams_are_not_handed_the_new_keyword(self) -> None: - """Preparation forwards the reread intent only when it applies.""" - server, rpc, _ledger, _artifacts = configured_coordinator() - seen: list[tuple[object, ...]] = [] - - def narrow_build( - expected_payout_state_generation: int, - artifact_payout_state_generation: int, - network_difficulty: int, - force_full_rescan: bool = False, - bypass_build_interval: bool = False, - during_publication: bool = False, - ) -> None: - seen.append( - ( - expected_payout_state_generation, - artifact_payout_state_generation, - force_full_rescan, - ) - ) - return None - - server._build_payout_ledger_artifact = narrow_build # type: ignore[method-assign] - captured = (0, 0, str(rpc.tip), "external_tip", time.monotonic()) - server._prepared_payout_state_candidate(captured) - server._prepared_payout_state_candidate(captured, force_full_window_rescan=True) - self.assertEqual(seen, [(0, 1, False), (0, 1, True)]) - with self.assertRaises(TypeError): - server._prepared_payout_state_candidate( - captured, - force_prior_balances_read=True, - ) - - -class FullRescanTelemetryTests(QuietCoordinatorTestCase): - def test_every_recorded_reason_and_path_is_in_the_contract(self) -> None: - server, _rpc, _ledger, _artifacts = configured_coordinator() - snapshot = ensure_accepted_preview_telemetry(server).snapshot() - for reason, path in snapshot["full_rescans"]: - self.assertIn(reason, PRISM_PAYOUT_WINDOW_FULL_RESCAN_REASONS) - self.assertIn(path, PRISM_PAYOUT_WINDOW_FULL_RESCAN_PATHS) - - def test_cache_lifecycle_rescans_record_their_fixed_reason(self) -> None: - server, _rpc, ledger, artifacts = configured_coordinator() - telemetry = ensure_accepted_preview_telemetry(server) - with patch("lab.prism.prism_coordinator.now_ms", return_value=1_000_000): - initial = server._build_payout_ledger_artifact( - 0, 0, artifacts.network_difficulty - ) - assert initial is not None - # A late-visible append bumps the epoch without waiting for the - # preparation lock; the next build must fail closed to the oracle. - with server._job_cache_lock: - server._payout_ledger_append_invalidation_epoch += 1 - appended = server._build_payout_ledger_artifact( - 0, 0, artifacts.network_difficulty - ) - assert appended is not None - self.assertEqual(appended.window_full_rescan_reason, "late_visible_append") - # An explicit append-side invalidation leaves its one-shot reason. - with server._payout_state_prepare_lock: - server._incremental_payout_artifact_window = None - server._incremental_payout_artifact_window_invalidation_reason = ( - "cache_invalidated" - ) - invalidated = server._build_payout_ledger_artifact( - 0, 0, artifacts.network_difficulty - ) - assert invalidated is not None - self.assertEqual(invalidated.window_full_rescan_reason, "cache_invalidated") - - self.assertEqual(ledger.full_snapshot_calls, 3) - rescans = telemetry.snapshot()["full_rescans"] - self.assertEqual(rescans[("cold_start", "in_process")]["count"], 1) - self.assertEqual(rescans[("late_visible_append", "in_process")]["count"], 1) - self.assertEqual(rescans[("cache_invalidated", "in_process")]["count"], 1) - self.assertEqual(full_rescan_total(telemetry.snapshot()), 3) - for stats in rescans.values(): - self.assertGreaterEqual(stats["sum"], 0.0) - self.assertGreaterEqual(stats["max"], 0.0) - - def test_daemon_fold_records_the_daemon_path(self) -> None: - server, _rpc, _ledger, artifacts = configured_coordinator() - telemetry = ensure_accepted_preview_telemetry(server) - service = server._ensure_payout_state_service() - - def daemon_fold(*, records, snapshot_anchor_ms, reason, **_kwargs): - shares_json = tuple(record.to_prism_json() for record in records) - return _PayoutWindowMaterialization( - shares_json=shares_json, - share_snapshot_sha256=service._canonical_json_sha256(shares_json), - snapshot_anchor_ms=int(snapshot_anchor_ms), - mode="full_rescan", - record_count=len(shares_json), - stats=IncrementalWindowAdvanceStats(0, 0, 0), - full_rescan_reason=reason, - ) - - with ( - patch.object(service, "_window_pipeline_rust_enabled", return_value=True), - patch.object(service, "_daemon_full_window_materialization", daemon_fold), - patch("lab.prism.prism_coordinator.now_ms", return_value=1_000_000), - ): - built = server._build_payout_ledger_artifact( - 0, 0, artifacts.network_difficulty - ) - - assert built is not None - self.assertEqual(built.window_build_mode, "full_rescan") - rescans = telemetry.snapshot()["full_rescans"] - self.assertEqual(rescans[("cold_start", "daemon")]["count"], 1) - self.assertEqual(rescans[("cold_start", "in_process")]["count"], 0) - - def test_periodic_self_check_records_the_periodic_reason_in_process( - self, - ) -> None: - server, _rpc, ledger, artifacts = configured_coordinator() - telemetry = ensure_accepted_preview_telemetry(server) - server.payout_artifact_min_build_interval_seconds = 0.0 - server.payout_artifact_full_rescan_seconds = 0.0 - with patch("lab.prism.prism_coordinator.now_ms", return_value=1_000_000): - initial = server._build_payout_ledger_artifact( - 0, 0, artifacts.network_difficulty - ) - assert initial is not None - checked = server._build_payout_ledger_artifact( - 0, 0, artifacts.network_difficulty - ) - - assert checked is not None - self.assertEqual(checked.window_build_mode, "self_check_match") - self.assertEqual(ledger.full_snapshot_calls, 2) - rescans = telemetry.snapshot()["full_rescans"] - self.assertEqual(rescans[("cold_start", "in_process")]["count"], 1) - self.assertEqual(rescans[("periodic_self_check", "in_process")]["count"], 1) - - def test_failed_self_check_records_its_failure_reason(self) -> None: - server, _rpc, ledger, artifacts = configured_coordinator() - telemetry = ensure_accepted_preview_telemetry(server) - server.payout_artifact_min_build_interval_seconds = 0.0 - server.payout_artifact_full_rescan_seconds = 0.0 - with patch("lab.prism.prism_coordinator.now_ms", return_value=1_000_000): - initial = server._build_payout_ledger_artifact( - 0, 0, artifacts.network_difficulty - ) - assert initial is not None - ledger.fail_full_snapshot = True - checked = server._build_payout_ledger_artifact( - 0, 0, artifacts.network_difficulty - ) - - assert checked is not None - self.assertEqual(checked.window_build_mode, "incremental_self_check_failed") - rescans = telemetry.snapshot()["full_rescans"] - self.assertEqual( - rescans[("periodic_self_check_failed", "in_process")]["count"], 1 - ) - - def test_a_rescan_that_dies_with_the_ledger_still_owns_its_time(self) -> None: - server, _rpc, ledger, artifacts = configured_coordinator() - telemetry = ensure_accepted_preview_telemetry(server) - ledger.fail_full_snapshot = True - with patch("lab.prism.prism_coordinator.now_ms", return_value=1_000_000): - self.assertIsNone( - server._build_payout_ledger_artifact( - 0, 0, artifacts.network_difficulty - ) - ) - rescans = telemetry.snapshot()["full_rescans"] - self.assertEqual(rescans[("cold_start", "in_process")]["count"], 1) - - -# --- coordinator integration ------------------------------------------------ - - -class CoordinatorReconcileIntegrationTests(QuietCoordinatorTestCase): - """The real adapters: writer admission, mutation lock, service core.""" - - def reconciling_coordinator(self): - server, rpc, ledger, artifacts = configured_coordinator() - server.reorg_reconciler_enabled = True - # Every entry point must run a real pass, not a memo hit. - server.reorg_reconcile_cache_seconds = 0.0 - return server, rpc, ledger, artifacts - - def test_maturation_reuses_the_window_and_publishes_reread_balances( - self, - ) -> None: - server, rpc, ledger, _artifacts = self.reconciling_coordinator() - telemetry = ensure_accepted_preview_telemetry(server) - tip = str(rpc.tip) - with patch("lab.prism.prism_coordinator.now_ms", return_value=1_000_000): - first = server.reconcile_prism_pool_blocks_once(tip_hash=tip) - self.assertEqual(first["published_generation"], 1) - self.assertEqual(ledger.full_snapshot_calls, 1) - with server._job_cache_lock: - first_published = server._published_payout_state.artifact - assert first_published is not None - self.assertEqual(first_published.prior_balances()[0]["balance_sats"], 546) - - ledger.matured_pending = 1 - reads_before = ledger.prior_balance_reads - second = server.reconcile_prism_pool_blocks_once(tip_hash=tip) - - self.assertEqual(second["matured_payouts"], 1) - self.assertEqual(second["published_generation"], 2) - # The exact window was reused: no second oracle read anywhere. - self.assertEqual(ledger.full_snapshot_calls, 1) - self.assertEqual(server.payout_artifact_event_counts["full_rescan"], 1) - # The balances the maturation moved were reread and published. - self.assertEqual(ledger.prior_balance_reads, reads_before + 1) - with server._job_cache_lock: - published = server._published_payout_state.artifact - assert published is not None - self.assertEqual(published.generation, 2) - self.assertEqual(published.prior_balances()[0]["balance_sats"], 777) - self.assertNotEqual( - published.prior_balances_sha256, first_published.prior_balances_sha256 - ) - # Attribution: two ``other`` passes that crossed the real adapter, - # each preparing and publishing; one cold-start rescan, and no - # reconcile_invalidation rescan at all. - snapshot = telemetry.snapshot() - self.assertEqual(pass_counts(snapshot)["other"], 2) - self.assertEqual(step_count(snapshot, "other", "admission_wait"), 2) - # Per pass: the watch query plus the stranded-prepared sweep read. - self.assertEqual(step_count(snapshot, "other", "watch_query"), 4) - self.assertEqual(step_count(snapshot, "other", "chain_probe"), 2) - self.assertEqual(step_count(snapshot, "other", "mutations"), 2) - self.assertEqual(step_count(snapshot, "other", "candidate_prepare"), 2) - self.assertEqual(step_count(snapshot, "other", "publish"), 2) - rescans = snapshot["full_rescans"] - self.assertEqual(rescans[("cold_start", "in_process")]["count"], 1) - self.assertEqual(rescans[("reconcile_invalidation", "in_process")]["count"], 0) - self.assertEqual(rescans[("reconcile_invalidation", "daemon")]["count"], 0) - self.assertEqual(full_rescan_total(snapshot), 1) - - def test_entry_points_attribute_through_the_real_adapters(self) -> None: - server, rpc, _ledger, _artifacts = self.reconciling_coordinator() - telemetry = ensure_accepted_preview_telemetry(server) - tip = str(rpc.tip) - with patch("lab.prism.prism_coordinator.now_ms", return_value=1_000_000): - self.assertTrue( - server.ensure_reorg_reconciled_for_tip(tip, _coalesce_same_tip=False) - ) - confirmed = server.reconcile_prism_pool_blocks_once( - tip_hash=tip, - _force_publish=True, - _source_reserved=True, - ) - self.assertIsNotNone(confirmed["published_generation"]) - self.assertTrue(server.ensure_reorg_reconciled_for_current_tip()) - prefetch = server._submit_reconcile_prefetch(tip) - assert prefetch is not None - try: - self.assertTrue(prefetch.result(timeout=10.0)) - finally: - server.shutdown_reconcile_prefetch_executor() - self.assertTrue(server.ensure_reorg_reconciled_for_tip(tip)) - - counts = pass_counts(telemetry.snapshot()) - self.assertEqual(counts["landing"], 1) - self.assertEqual(counts["post_confirm"], 1) - self.assertEqual(counts["job_build"], 1) - self.assertEqual(counts["tip_refresh"], 1) - self.assertEqual(counts["other"], 1) - snapshot = telemetry.snapshot() - for caller in PRISM_REORG_RECONCILE_CALLERS: - self.assertEqual(step_count(snapshot, caller, "admission_wait"), 1, caller) - # The service records into the same owner the renderer reads. - self.assertIs( - server._ensure_reorg_reconciler_service().accepted_preview_telemetry, - telemetry, - ) - - def test_landed_preview_refusal_records_no_pass(self) -> None: - server, rpc, _ledger, _artifacts = self.reconciling_coordinator() - telemetry = ensure_accepted_preview_telemetry(server) - tip = str(rpc.tip) - server._begin_accepted_block_payout_preview("aa" * 32, block_height=11) - server._mark_accepted_block_payout_landed("aa" * 32, block_height=11) - with self.assertRaises(PayoutStatePublicationBlocked): - server.reconcile_prism_pool_blocks_once(tip_hash=tip) - self.assertEqual(sum(pass_counts(telemetry.snapshot()).values()), 0) - - -if __name__ == "__main__": - unittest.main() diff --git a/tests/test_prism_issue_224_landing.py b/tests/test_prism_issue_224_landing.py deleted file mode 100644 index a26da2f86..000000000 --- a/tests/test_prism_issue_224_landing.py +++ /dev/null @@ -1,1094 +0,0 @@ -#!/usr/bin/env python3 -"""Accepted-landing attribution for issue #224. - -``PrismAcceptedPreviewPublicationLatencyHigh`` proves a tail exists between -definitive node acceptance and the visible payout preview, but until this lane -nothing said *which stretch of the landing* owned a sample. These tests pin -the properties that make the answer trustworthy rather than merely present: - -* the phase label vocabulary is closed -- the seven names of - ``PRISM_ACCEPTED_LANDING_PHASES`` and nothing a call site can compute; -* the interval keeps its shipped definition -- it starts at definitive node - acceptance, not at the pre-RPC landed fence that is armed before qbitd has - seen the block, and it closes once at the first publication; -* one publication yields exactly one retained diagnostic and one structured - line, and a retry or republication yields none; -* a block hash and height reach the diagnostic and the log line only, never a - metric label; -* the in-flight per-hash timing state cannot leak -- terminal abandonment - drops it and the map is bounded exactly as the acceptance stamps are; -* the landing itself is unchanged apart from the telemetry: the balance - serializer is still taken once and released once around the same body, and - a landing still lands, confirms and publishes. -""" - -from __future__ import annotations - -import ast -import json -import threading -import unittest -from contextlib import contextmanager -from io import StringIO -from pathlib import Path -from types import SimpleNamespace -from typing import Any, Iterator -from unittest import mock - -import lab.prism.block_candidates as block_candidates_module -import lab.prism.block_finalization as block_finalization_module -from lab.prism.accepted_preview_telemetry import ( - AcceptedPreviewPublicationDiagnostic, - LANDING_PHASE_BALANCE_LOCK_WAIT, - LANDING_PHASE_CHAIN_PROBE, - LANDING_PHASE_LANE_WAIT, - LANDING_PHASE_PREVIEW_PREPARE, - LANDING_PHASE_PREVIEW_PUBLISH, - LANDING_PHASE_PRIOR_BALANCES_CHECK, - LANDING_PHASE_RECONCILE, - PRISM_ACCEPTED_LANDING_PHASES, - RECONCILE_CALLER_LANDING, - ensure_accepted_preview_telemetry, -) -from lab.prism.block_candidates import ( - MAX_ACCEPTED_BLOCK_PREVIEW_PUBLICATION_STAMPS, - PRISM_ACCEPTED_LANDING_LOG_EVENT, -) -from tests.prism_vardiff_test_support import ( - SubmitRpc, - block_candidate, - submit_coordinator, - tempfile, - verified_audit_report, - verified_block_bundle, -) - - -OWNED_MODULES = (block_candidates_module, block_finalization_module) -# Every call whose argument list carries a landing phase name. A phase that is -# a computed string anywhere in this list is a new metric series. -PHASE_ARGUMENT_CALLS = frozenset( - { - "observe_landing_phase", - "landing_phase", - "_landing_phase", - "_accepted_landing_phase", - "_commit_accepted_landing_phase", - } -) - - -def _module_tree(module: Any) -> ast.Module: - return ast.parse(Path(module.__file__).read_text(encoding="utf-8")) - - -def _builds_a_string(node: ast.AST) -> bool: - """Whether this expression produces a string the source did not fix.""" - if isinstance(node, ast.Constant): - return isinstance(node.value, str) - if isinstance(node, ast.JoinedStr): - return True - if isinstance(node, ast.BinOp): - return any( - _builds_a_string(operand) for operand in (node.left, node.right) - ) - if isinstance(node, ast.Call) and isinstance(node.func, ast.Attribute): - return node.func.attr in {"format", "join", "lower", "upper"} - return False - - -def _call_name(node: ast.Call) -> str | None: - if isinstance(node.func, ast.Attribute): - return node.func.attr - if isinstance(node.func, ast.Name): - return node.func.id - return None - - -class LandingPhaseVocabularyTests(unittest.TestCase): - """The seven fixed labels, and no way for a call site to invent an eighth.""" - - def test_owned_modules_reference_every_contract_phase_constant(self) -> None: - referenced: set[str] = set() - for module in OWNED_MODULES: - for node in ast.walk(_module_tree(module)): - if isinstance(node, ast.Name) and node.id.startswith( - "LANDING_PHASE_" - ): - referenced.add(node.id) - self.assertEqual( - referenced, - { - "LANDING_PHASE_LANE_WAIT", - "LANDING_PHASE_BALANCE_LOCK_WAIT", - "LANDING_PHASE_RECONCILE", - "LANDING_PHASE_PRIOR_BALANCES_CHECK", - "LANDING_PHASE_CHAIN_PROBE", - "LANDING_PHASE_PREVIEW_PREPARE", - "LANDING_PHASE_PREVIEW_PUBLISH", - }, - ) - self.assertEqual( - tuple(sorted(PRISM_ACCEPTED_LANDING_PHASES)), - tuple( - sorted( - ( - LANDING_PHASE_LANE_WAIT, - LANDING_PHASE_BALANCE_LOCK_WAIT, - LANDING_PHASE_RECONCILE, - LANDING_PHASE_PRIOR_BALANCES_CHECK, - LANDING_PHASE_CHAIN_PROBE, - LANDING_PHASE_PREVIEW_PREPARE, - LANDING_PHASE_PREVIEW_PUBLISH, - ) - ) - ), - ) - - def test_no_phase_argument_is_a_computed_string(self) -> None: - # A literal, an f-string, or a concatenation reaching one of these - # calls is how a dynamic label gets born. Names are the only shape - # allowed, and the recorder validates the name against the contract. - offenders: list[str] = [] - for module in OWNED_MODULES: - for node in ast.walk(_module_tree(module)): - if not isinstance(node, ast.Call): - continue - if _call_name(node) not in PHASE_ARGUMENT_CALLS: - continue - arguments = list(node.args) + [ - keyword.value for keyword in node.keywords - ] - for argument in arguments: - if _builds_a_string(argument): - offenders.append( - f"{Path(module.__file__).name}:{node.lineno}" - ) - self.assertEqual(offenders, []) - - def test_unknown_phase_names_are_refused_before_the_body_runs(self) -> None: - server, _state, _ledger = submit_coordinator() - service = server._ensure_block_candidate_service() - entered = False - - with self.assertRaises(ValueError): - with service._accepted_landing_phase("ab" * 32, "not_a_phase"): - entered = True # pragma: no cover - the guard runs first - self.assertFalse(entered) - - telemetry = ensure_accepted_preview_telemetry(server) - with self.assertRaises(ValueError): - telemetry.observe_landing_phase("not_a_phase", 1.0) - - def test_phase_metric_cells_carry_only_the_seven_fixed_labels(self) -> None: - server, _state, _ledger = submit_coordinator() - service = server._ensure_block_candidate_service() - block_hash = "ab" * 32 - service._note_accepted_block_preview_acceptance(block_hash) - for phase in PRISM_ACCEPTED_LANDING_PHASES: - with service._accepted_landing_phase(block_hash, phase): - pass - - snapshot = ensure_accepted_preview_telemetry(server).snapshot() - self.assertEqual( - set(snapshot["landing_phases"]), - set(PRISM_ACCEPTED_LANDING_PHASES), - ) - for phase in PRISM_ACCEPTED_LANDING_PHASES: - self.assertEqual(snapshot["landing_phases"][phase]["count"], 1) - # Nothing anywhere in the metric cells names a block. - self.assertNotIn(block_hash, repr(snapshot)) - - -class LandingLaneWaitTests(unittest.TestCase): - """``lane_wait`` measures from acceptance, not from the landed fence.""" - - def _service(self) -> tuple[Any, Any]: - server, _state, _ledger = submit_coordinator() - return server, server._ensure_block_candidate_service() - - def test_lane_wait_starts_at_definitive_acceptance(self) -> None: - server, service = self._service() - block_hash = "ab" * 32 - with mock.patch( - "lab.prism.block_candidates.time.monotonic", - side_effect=[100.0, 104.5], - ): - service._note_accepted_block_preview_acceptance(block_hash) - service._close_accepted_landing_lane_wait(block_hash, block_height=10) - - snapshot = ensure_accepted_preview_telemetry(server).snapshot() - lane_wait = snapshot["landing_phases"][LANDING_PHASE_LANE_WAIT] - self.assertEqual(lane_wait["count"], 1) - self.assertAlmostEqual(float(lane_wait["sum"]), 4.5) - self.assertEqual( - service.accepted_landing_attribution_snapshot()[block_hash][ - LANDING_PHASE_LANE_WAIT - ], - 4.5, - ) - - def test_pre_rpc_landed_fencing_does_not_start_the_lane_wait(self) -> None: - # _mark_accepted_block_payout_landed arms the landed barrier *before* - # the submitblock RPC, so a lane wait measured from it would include - # the offer itself and would restart on every retry. Only the - # definitive-acceptance stamp opens an interval. - server, service = self._service() - block_hash = "cd" * 32 - server._begin_accepted_block_payout_preview(block_hash, block_height=10) - server._mark_accepted_block_payout_landed(block_hash, block_height=10) - - service._close_accepted_landing_lane_wait(block_hash, block_height=10) - - snapshot = ensure_accepted_preview_telemetry(server).snapshot() - self.assertEqual( - snapshot["landing_phases"][LANDING_PHASE_LANE_WAIT]["count"], - 0, - ) - self.assertEqual(service.accepted_landing_attribution_snapshot(), {}) - - def test_lane_wait_is_recorded_once_per_open_interval(self) -> None: - # A retried landing re-enters the lane. Measuring again from the same - # acceptance stamp would charge the retry's queue time to a stretch - # the first attempt already owns, and would keep growing it. - server, service = self._service() - block_hash = "ef" * 32 - with mock.patch( - "lab.prism.block_candidates.time.monotonic", - side_effect=[10.0, 11.0, 30.0, 90.0], - ): - service._note_accepted_block_preview_acceptance(block_hash) - service._close_accepted_landing_lane_wait(block_hash) - service._close_accepted_landing_lane_wait(block_hash) - service._close_accepted_landing_lane_wait(block_hash) - - lane_wait = ensure_accepted_preview_telemetry(server).snapshot()[ - "landing_phases" - ][LANDING_PHASE_LANE_WAIT] - self.assertEqual(lane_wait["count"], 1) - self.assertAlmostEqual(float(lane_wait["sum"]), 1.0) - - def test_a_candidate_shape_without_a_hash_starts_nothing(self) -> None: - _server, service = self._service() - service._note_accepted_landing_lane_start(SimpleNamespace()) - self.assertEqual(service.accepted_landing_attribution_snapshot(), {}) - - def _candidate(self, block_hash: str, height: int = 10) -> Any: - return SimpleNamespace( - submission=SimpleNamespace(block_hash_hex=block_hash), - context=SimpleNamespace(template={"height": height}), - ) - - def test_a_lane_closed_before_the_stamp_is_closed_again_after_it( - self, - ) -> None: - # A fresh candidate carries no acceptance stamp until the node offer - # the landing itself makes returns definitively, so the close that - # runs before that offer finds no interval to measure. The second - # close is what turns a first-pass landing into a recorded sample - # instead of a silent gap: a zero sample and a missing sample read - # identically in the diagnostic, and only the family count separates - # them. - server, service = self._service() - block_hash = "5c" * 32 - candidate = self._candidate(block_hash) - - service._note_accepted_landing_lane_start(candidate) - service._note_accepted_block_preview_acceptance(block_hash) - service._note_accepted_landing_lane_start(candidate) - - lane_wait = ensure_accepted_preview_telemetry(server).snapshot()[ - "landing_phases" - ][LANDING_PHASE_LANE_WAIT] - self.assertEqual(lane_wait["count"], 1) - self.assertEqual( - service.accepted_landing_attribution_snapshot()[block_hash][ - LANDING_PHASE_LANE_WAIT - ], - float(lane_wait["sum"]), - ) - - def test_a_retained_acceptance_keeps_the_earlier_tighter_sample( - self, - ) -> None: - # The reverse case: the stamp already existed, so the close before - # the retained offer is looked up owns the sample and the one after - # it must not extend the stretch with the landing's own work. - server, service = self._service() - block_hash = "6d" * 32 - candidate = self._candidate(block_hash) - with mock.patch( - "lab.prism.block_candidates.time.monotonic", - side_effect=[10.0, 12.0, 45.0], - ): - service._note_accepted_block_preview_acceptance(block_hash) - service._note_accepted_landing_lane_start(candidate) - service._note_accepted_landing_lane_start(candidate) - - lane_wait = ensure_accepted_preview_telemetry(server).snapshot()[ - "landing_phases" - ][LANDING_PHASE_LANE_WAIT] - self.assertEqual(lane_wait["count"], 1) - self.assertAlmostEqual(float(lane_wait["sum"]), 2.0) - - def test_the_lane_closes_before_the_attempt_marker_is_written(self) -> None: - # ``_mark_block_candidate_attempted`` is the landing's own PostgreSQL - # write, not queue time. Closing the lane after it charges the write - # to ``lane_wait`` -- and a marker that times out keeps charging the - # retry backoff there too, so the one stretch that exonerates the - # landing grows precisely when the writer is degraded. The ordering is - # the whole guarantee, so it is pinned where it is written. - tree = _module_tree(block_candidates_module) - ordered = 0 - for function in ast.walk(tree): - if not isinstance(function, (ast.FunctionDef, ast.AsyncFunctionDef)): - continue - lane_starts = [ - node.lineno - for node in ast.walk(function) - if isinstance(node, ast.Call) - and _call_name(node) == "_note_accepted_landing_lane_start" - ] - markers = [ - node.lineno - for node in ast.walk(function) - if isinstance(node, ast.Call) - and _call_name(node) == "_mark_block_candidate_attempted" - ] - if not lane_starts or not markers: - continue - offers = [ - node.lineno - for node in ast.walk(function) - if isinstance(node, ast.Call) - and _call_name(node) - == "_node_submission_for_candidate_or_retained" - ] - ordered += 1 - self.assertLess( - max(lane_starts), - min(markers), - f"{function.name} closes the lane after its attempt marker", - ) - # A fresh candidate is stamped by the offer itself, so a path that - # only closed the lane before making it would record no sample at - # all for every first-pass landing. - self.assertTrue( - offers and max(lane_starts) > max(offers), - f"{function.name} never closes the lane after its node offer", - ) - # Both queued landing paths write the marker; a rename that loses one - # would otherwise make this test vacuously true. - self.assertEqual(ordered, 2) - - -class LandingPublicationDiagnosticTests(unittest.TestCase): - """One closed interval, one retained diagnostic, one structured line.""" - - def _service(self) -> tuple[Any, Any]: - server, _state, _ledger = submit_coordinator() - return server, server._ensure_block_candidate_service() - - def _publish(self, server: Any, block_hash: str, result: str) -> list[dict]: - stream = StringIO() - with mock.patch("sys.stdout", stream): - server._observe_accepted_block_preview_publication( - block_hash, - result=result, - ) - lines = [] - for line in stream.getvalue().splitlines(): - prefix, _, payload = line.partition("prism coordinator: ") - if prefix or not payload.startswith("{"): - continue - record = json.loads(payload) - if record.get("event") == PRISM_ACCEPTED_LANDING_LOG_EVENT: - lines.append(record) - return lines - - def test_first_published_publication_closes_one_interval(self) -> None: - server, service = self._service() - block_hash = "ab" * 32 - service._note_accepted_block_preview_acceptance(block_hash) - with service._accepted_landing_phase( - block_hash, - LANDING_PHASE_RECONCILE, - block_height=10, - ): - pass - - logged = self._publish(server, block_hash, "published") - - telemetry = ensure_accepted_preview_telemetry(server) - diagnostics = telemetry.diagnostics_snapshot() - self.assertEqual(len(diagnostics), 1) - self.assertEqual(telemetry.snapshot()["diagnostics_recorded"], 1) - diagnostic = diagnostics[0] - self.assertEqual(diagnostic.block_hash, block_hash) - self.assertEqual(diagnostic.block_height, 10) - self.assertEqual(diagnostic.result, "published") - self.assertEqual(diagnostic.reconcile_caller, RECONCILE_CALLER_LANDING) - self.assertGreaterEqual(diagnostic.acceptance_to_publication_seconds, 0.0) - self.assertIsNotNone(diagnostic.recorded_monotonic) - self.assertEqual( - set(diagnostic.phase_seconds), - set(PRISM_ACCEPTED_LANDING_PHASES), - ) - self.assertEqual(len(logged), 1) - self.assertEqual(logged[0]["block_hash"], block_hash) - self.assertEqual(logged[0]["block_height"], 10) - self.assertEqual(logged[0]["result"], "published") - # The shipped histogram still closed exactly one interval. - histograms = service.accepted_block_preview_publication_snapshot() - self.assertEqual(histograms["published"]["count"], 1) - self.assertEqual(histograms["degraded"]["count"], 0) - - def test_first_degraded_publication_closes_one_interval(self) -> None: - # The fenced local-retention branch also makes the preview visible to - # waiting children, so it closes the interval and earns a diagnostic - # of its own -- labelled apart, never merged into the healthy one. - server, service = self._service() - block_hash = "cd" * 32 - service._note_accepted_block_preview_acceptance(block_hash) - - logged = self._publish(server, block_hash, "degraded") - - telemetry = ensure_accepted_preview_telemetry(server) - diagnostics = telemetry.diagnostics_snapshot() - self.assertEqual(len(diagnostics), 1) - self.assertEqual(diagnostics[0].result, "degraded") - self.assertEqual(len(logged), 1) - self.assertEqual(logged[0]["result"], "degraded") - histograms = service.accepted_block_preview_publication_snapshot() - self.assertEqual(histograms["degraded"]["count"], 1) - self.assertEqual(histograms["published"]["count"], 0) - - def test_retry_and_republication_do_not_double_count(self) -> None: - server, service = self._service() - block_hash = "ef" * 32 - service._note_accepted_block_preview_acceptance(block_hash) - first = self._publish(server, block_hash, "published") - - # A matching republication, a degraded relabel, and a second - # definitive offer of the same hash: none of them is a new interval. - again = self._publish(server, block_hash, "published") - relabelled = self._publish(server, block_hash, "degraded") - service._note_accepted_block_preview_acceptance(block_hash) - reoffered = self._publish(server, block_hash, "published") - - telemetry = ensure_accepted_preview_telemetry(server) - self.assertEqual(len(first), 1) - self.assertEqual(again, []) - self.assertEqual(relabelled, []) - self.assertEqual(reoffered, []) - self.assertEqual(len(telemetry.diagnostics_snapshot()), 1) - self.assertEqual(telemetry.snapshot()["diagnostics_recorded"], 1) - histograms = service.accepted_block_preview_publication_snapshot() - self.assertEqual(histograms["published"]["count"], 1) - self.assertEqual(histograms["degraded"]["count"], 0) - - def test_phases_after_the_interval_closed_reach_no_diagnostic(self) -> None: - # A retry's stretches still belong in the process-wide family -- they - # are landing work that ran -- but they can never be claimed by a - # landing whose interval is already measured and recorded. - server, service = self._service() - block_hash = "12" * 32 - service._note_accepted_block_preview_acceptance(block_hash) - self._publish(server, block_hash, "published") - - with service._accepted_landing_phase( - block_hash, - LANDING_PHASE_RECONCILE, - block_height=10, - ): - pass - - telemetry = ensure_accepted_preview_telemetry(server) - self.assertEqual(len(telemetry.diagnostics_snapshot()), 1) - self.assertEqual( - telemetry.diagnostics_snapshot()[0].phase_seconds[ - LANDING_PHASE_RECONCILE - ], - 0.0, - ) - self.assertEqual( - telemetry.snapshot()["landing_phases"][LANDING_PHASE_RECONCILE][ - "count" - ], - 1, - ) - self.assertEqual(service.accepted_landing_attribution_snapshot(), {}) - - def test_publication_without_an_acceptance_stamp_records_nothing(self) -> None: - server, _service = self._service() - logged = self._publish(server, "ab" * 32, "published") - telemetry = ensure_accepted_preview_telemetry(server) - self.assertEqual(logged, []) - self.assertEqual(telemetry.diagnostics_snapshot(), ()) - - def test_publish_span_is_closed_by_the_visibility_instant(self) -> None: - # The publication closes the interval from inside the publish call, so - # a plain timer would record ``preview_publish`` after the diagnostic - # was already built from an empty cell. - server, service = self._service() - block_hash = "34" * 32 - service._note_accepted_block_preview_acceptance(block_hash) - with service._accepted_landing_publish_span(block_hash, block_height=11): - server._observe_accepted_block_preview_publication( - block_hash, - result="published", - ) - - telemetry = ensure_accepted_preview_telemetry(server) - diagnostics = telemetry.diagnostics_snapshot() - self.assertEqual(len(diagnostics), 1) - self.assertEqual(diagnostics[0].block_height, 11) - self.assertGreater( - diagnostics[0].phase_seconds[LANDING_PHASE_PREVIEW_PUBLISH], - 0.0, - ) - # Closed exactly once: the observer took the span, and the context - # manager's exit found nothing left to record. - self.assertEqual( - telemetry.snapshot()["landing_phases"][ - LANDING_PHASE_PREVIEW_PUBLISH - ]["count"], - 1, - ) - - def test_a_publish_that_publishes_nothing_still_records_its_phase(self) -> None: - server, service = self._service() - block_hash = "56" * 32 - service._note_accepted_block_preview_acceptance(block_hash) - with service._accepted_landing_publish_span(block_hash): - pass - - telemetry = ensure_accepted_preview_telemetry(server) - self.assertEqual( - telemetry.snapshot()["landing_phases"][ - LANDING_PHASE_PREVIEW_PUBLISH - ]["count"], - 1, - ) - self.assertEqual(telemetry.diagnostics_snapshot(), ()) - - def test_hash_and_height_reach_the_diagnostic_and_log_only(self) -> None: - server, service = self._service() - block_hash = "78" * 32 - service._note_accepted_block_preview_acceptance(block_hash) - with service._accepted_landing_phase( - block_hash, - LANDING_PHASE_CHAIN_PROBE, - block_height=4242, - ): - pass - logged = self._publish(server, block_hash, "published") - - telemetry = ensure_accepted_preview_telemetry(server) - self.assertEqual(len(logged), 1) - self.assertEqual(logged[0]["block_hash"], block_hash) - self.assertEqual(logged[0]["block_height"], 4242) - for phase in PRISM_ACCEPTED_LANDING_PHASES: - self.assertIn(f"phase_{phase}_seconds", logged[0]) - - rendered = repr(telemetry.snapshot()) - self.assertNotIn(block_hash, rendered) - self.assertNotIn("4242", rendered) - self.assertNotIn( - block_hash, - repr(service.accepted_block_preview_publication_snapshot()), - ) - - def test_the_shipped_metrics_render_carries_no_hash_or_height(self) -> None: - # End to end through the renderer the alert is read from: the phase - # family is there, its only label is the fixed phase name, and the - # block this landing measured is nowhere in the exposition. - server, service = self._service() - block_hash = "bc" * 32 - service._note_accepted_block_preview_acceptance(block_hash) - with service._accepted_landing_phase( - block_hash, - LANDING_PHASE_RECONCILE, - block_height=4242, - ): - pass - self._publish(server, block_hash, "published") - - rendered = server._ensure_metrics_renderer().render() - - self.assertIn( - 'qbit_prism_accepted_block_landing_phase_seconds_count{phase="reconcile"} 1', - rendered, - ) - for phase in PRISM_ACCEPTED_LANDING_PHASES: - self.assertIn( - "qbit_prism_accepted_block_landing_phase_seconds_sum" - f'{{phase="{phase}"}}', - rendered, - ) - self.assertNotIn(block_hash, rendered) - self.assertNotIn("4242", rendered) - self.assertNotIn(PRISM_ACCEPTED_LANDING_LOG_EVENT, rendered) - - def test_rescan_and_ledger_totals_stay_at_the_neutral_values(self) -> None: - # This lane can obtain no per-landing causal evidence for either - # without reading a process-wide delta, and such a read would claim - # whatever else the process was doing as this block's work. The - # separately attributed families answer those questions instead. - server, service = self._service() - block_hash = "9a" * 32 - service._note_accepted_block_preview_acceptance(block_hash) - logged = self._publish(server, block_hash, "published") - - diagnostic = ensure_accepted_preview_telemetry(server).diagnostics_snapshot()[0] - neutral = AcceptedPreviewPublicationDiagnostic( - block_hash=block_hash, - block_height=None, - result="published", - acceptance_to_publication_seconds=0.0, - ) - self.assertEqual(diagnostic.full_rescan_reason, neutral.full_rescan_reason) - self.assertEqual(diagnostic.full_rescan_path, neutral.full_rescan_path) - self.assertEqual(diagnostic.full_rescan_seconds, neutral.full_rescan_seconds) - self.assertEqual( - diagnostic.ledger_gate_wait_seconds, - neutral.ledger_gate_wait_seconds, - ) - self.assertEqual( - diagnostic.ledger_execute_seconds, - neutral.ledger_execute_seconds, - ) - self.assertIsNone(logged[0]["full_rescan_reason"]) - self.assertEqual(logged[0]["ledger_execute_seconds"], 0.0) - - -class LandingAttributionCleanupTests(unittest.TestCase): - """In-flight per-hash timing state is bounded and cannot leak.""" - - def _service(self) -> tuple[Any, Any]: - server, _state, _ledger = submit_coordinator() - return server, server._ensure_block_candidate_service() - - def test_terminal_abandonment_discards_the_in_flight_state(self) -> None: - server, service = self._service() - block_hash = "ab" * 32 - service._note_accepted_block_preview_acceptance(block_hash) - with service._accepted_landing_phase( - block_hash, - LANDING_PHASE_RECONCILE, - ): - pass - self.assertIn(block_hash, service.accepted_landing_attribution_snapshot()) - - service._record_committed_block_candidate_abandonment( - block_hash, - SimpleNamespace(reason="submitblock-rejected", stale_job_class=None), - ) - - self.assertEqual(service.accepted_landing_attribution_snapshot(), {}) - # The metric family keeps the stretch that actually ran. - self.assertEqual( - ensure_accepted_preview_telemetry(server).snapshot()["landing_phases"][ - LANDING_PHASE_RECONCILE - ]["count"], - 1, - ) - - def test_a_retryable_abandonment_keeps_the_open_interval(self) -> None: - # A retryable disposition is not terminal: the same interval is still - # running and its next attempt will close it. - _server, service = self._service() - block_hash = "cd" * 32 - service._note_accepted_block_preview_acceptance(block_hash) - with service._accepted_landing_phase( - block_hash, - LANDING_PHASE_RECONCILE, - ): - pass - retryable = next(iter(service.retryable_reasons)) - - service._record_committed_block_candidate_abandonment( - block_hash, - SimpleNamespace(reason=retryable, stale_job_class=None), - ) - - self.assertIn(block_hash, service.accepted_landing_attribution_snapshot()) - - def test_in_flight_state_is_bounded_like_the_acceptance_stamps(self) -> None: - _server, service = self._service() - for index in range(MAX_ACCEPTED_BLOCK_PREVIEW_PUBLICATION_STAMPS + 64): - block_hash = f"{index:064x}" - service._note_accepted_block_preview_acceptance(block_hash) - with service._accepted_landing_phase( - block_hash, - LANDING_PHASE_RECONCILE, - ): - pass - - retained = service.accepted_landing_attribution_snapshot() - self.assertLessEqual( - len(retained), - MAX_ACCEPTED_BLOCK_PREVIEW_PUBLICATION_STAMPS, - ) - self.assertLessEqual( - len(service._accepted_block_preview_acceptance_monotonic), - MAX_ACCEPTED_BLOCK_PREVIEW_PUBLICATION_STAMPS, - ) - # Eviction is oldest-first, so the newest landing is the survivor. - newest = f"{MAX_ACCEPTED_BLOCK_PREVIEW_PUBLICATION_STAMPS + 63:064x}" - self.assertIn(newest, retained) - - def test_an_unstamped_publication_leaves_nothing_in_flight(self) -> None: - # A startup replay or a chain-proven candidate publishes without an - # in-process acceptance stamp, so the observer returns before it can - # pop anything. A record opened by the publish span would then report - # a finished publication as in flight for as long as the bound kept - # it. - server, service = self._service() - block_hash = "7b" * 32 - with service._accepted_landing_publish_span(block_hash, block_height=12): - server._observe_accepted_block_preview_publication( - block_hash, - result="published", - ) - - self.assertEqual(service.accepted_landing_attribution_snapshot(), {}) - telemetry = ensure_accepted_preview_telemetry(server) - self.assertEqual(telemetry.diagnostics_snapshot(), ()) - # The stretch still ran, so the phase family still owns its sample. - self.assertEqual( - telemetry.snapshot()["landing_phases"][ - LANDING_PHASE_PREVIEW_PUBLISH - ]["count"], - 1, - ) - - def test_unstamped_publications_never_evict_a_live_landing(self) -> None: - # The failure this guards is the replay storm: enough unattributable - # publications to fill the shared cap would push live landings out of - # it before their diagnostics were ever emitted. - server, service = self._service() - live = "01" * 32 - service._note_accepted_block_preview_acceptance(live) - with service._accepted_landing_phase(live, LANDING_PHASE_RECONCILE): - pass - for index in range(MAX_ACCEPTED_BLOCK_PREVIEW_PUBLICATION_STAMPS + 8): - with service._accepted_landing_publish_span(f"{index:064x}"): - pass - - self.assertEqual( - list(service.accepted_landing_attribution_snapshot()), - [live], - ) - server._observe_accepted_block_preview_publication( - live, - result="published", - ) - diagnostics = ensure_accepted_preview_telemetry(server).diagnostics_snapshot() - self.assertEqual([entry.block_hash for entry in diagnostics], [live]) - - def test_publication_pops_the_in_flight_state(self) -> None: - server, service = self._service() - block_hash = "ef" * 32 - service._note_accepted_block_preview_acceptance(block_hash) - with service._accepted_landing_phase( - block_hash, - LANDING_PHASE_RECONCILE, - ): - pass - server._observe_accepted_block_preview_publication( - block_hash, - result="published", - ) - self.assertEqual(service.accepted_landing_attribution_snapshot(), {}) - - def test_a_raising_stretch_still_records_and_is_not_masked(self) -> None: - server, service = self._service() - block_hash = "12" * 32 - service._note_accepted_block_preview_acceptance(block_hash) - - with self.assertRaisesRegex(RuntimeError, "reconcile exploded"): - with service._accepted_landing_phase( - block_hash, - LANDING_PHASE_RECONCILE, - ): - raise RuntimeError("reconcile exploded") - - telemetry = ensure_accepted_preview_telemetry(server) - self.assertEqual( - telemetry.snapshot()["landing_phases"][LANDING_PHASE_RECONCILE][ - "count" - ], - 1, - ) - self.assertIn( - LANDING_PHASE_RECONCILE, - service.accepted_landing_attribution_snapshot()[block_hash], - ) - - def test_a_raising_publish_span_still_records_and_is_not_masked(self) -> None: - server, service = self._service() - block_hash = "34" * 32 - service._note_accepted_block_preview_acceptance(block_hash) - - with self.assertRaisesRegex(RuntimeError, "publication exploded"): - with service._accepted_landing_publish_span(block_hash): - raise RuntimeError("publication exploded") - - self.assertEqual( - ensure_accepted_preview_telemetry(server).snapshot()["landing_phases"][ - LANDING_PHASE_PREVIEW_PUBLISH - ]["count"], - 1, - ) - - def test_a_refused_diagnostic_never_fails_a_publication(self) -> None: - # Attribution is observability: the preview is already visible to - # waiting children by the time the record is built. - server, service = self._service() - block_hash = "56" * 32 - service._note_accepted_block_preview_acceptance(block_hash) - with mock.patch.object( - block_candidates_module, - "AcceptedPreviewPublicationDiagnostic", - side_effect=ValueError("refused"), - ): - server._observe_accepted_block_preview_publication( - block_hash, - result="published", - ) - - telemetry = ensure_accepted_preview_telemetry(server) - self.assertEqual(telemetry.diagnostics_snapshot(), ()) - # The shipped interval still closed exactly once. - self.assertEqual( - service.accepted_block_preview_publication_snapshot()["published"][ - "count" - ], - 1, - ) - - -class LandingBehaviourCompatibilityTests(unittest.TestCase): - """The landing itself is unchanged apart from the telemetry it emits.""" - - def test_balance_serializer_is_taken_once_around_the_same_body(self) -> None: - # The acquisition is timed through an ExitStack rather than a plain - # ``with``. The lock must still be entered once before the body and - # released once after it, on the early-return path too. - server, state, _ledger = submit_coordinator() - service = server._ensure_block_finalization_service() - events: list[str] = [] - real_lock = server._block_submitter_lock - - @contextmanager - def recording(lock: Any, name: str) -> Iterator[None]: - events.append(f"enter:{name}") - try: - with real_lock(lock, name): - yield - finally: - events.append(f"exit:{name}") - - server._block_submitter_lock = recording # type: ignore[method-assign] - server._defer_for_pending_parent_payout_transition = ( # type: ignore[method-assign] - lambda **_kwargs: (events.append("body") or True) - ) - submission = SimpleNamespace( - coinbase_tx_hex="c0ffee", - block_hash_hex="94" * 32, - block_hex="00", - ) - candidate = block_candidate(server, state, submission) - - landed = service._land_and_confirm_block_candidate( - candidate, - current_tip="00" * 32, - already_active=False, - worker="miner-a", - node_submission=block_candidates_module._BlockCandidateNodeSubmission( - attempted=False - ), - ) - - self.assertIsNone(landed) - self.assertEqual( - events, - [ - "enter:payout-balance-mutation", - "body", - "exit:payout-balance-mutation", - ], - ) - # The wait was attributed even though the landing deferred. - self.assertEqual( - ensure_accepted_preview_telemetry(server).snapshot()["landing_phases"][ - LANDING_PHASE_BALANCE_LOCK_WAIT - ]["count"], - 1, - ) - # And the serializer is genuinely free again. - self.assertTrue(server._payout_balance_mutation_lock.acquire(blocking=False)) - server._payout_balance_mutation_lock.release() - - def test_prior_balances_reread_keeps_its_stamps_and_verdict(self) -> None: - server, _state, _ledger = submit_coordinator() - service = server._ensure_block_finalization_service() - stamps: list[str] = [] - server._record_block_submitter_phase = ( # type: ignore[method-assign] - lambda phase: stamps.append(phase) - ) - server.prior_balances_match_current = ( # type: ignore[method-assign] - lambda _balances: False - ) - - without_context = service._stamped_prior_balances_match_current([]) - with_context = service._stamped_prior_balances_match_current( - [], - block_hash="ab" * 32, - block_height=10, - ) - - self.assertFalse(without_context) - self.assertFalse(with_context) - self.assertEqual( - stamps, - [ - "prior-balances-check", - "prior-balances-check:complete", - "prior-balances-check", - "prior-balances-check:complete", - ], - ) - # Only the landing-scoped call is a landing phase; the same helper - # called without a hash still times nothing. - self.assertEqual( - ensure_accepted_preview_telemetry(server).snapshot()["landing_phases"][ - LANDING_PHASE_PRIOR_BALANCES_CHECK - ]["count"], - 1, - ) - - def test_a_landing_still_lands_confirms_and_publishes(self) -> None: - server, state, ledger = submit_coordinator() - block_hash = "94" * 32 - with tempfile.TemporaryDirectory() as tmp: - server.audit_dir = Path(tmp) - server.evidence_path = Path(tmp) / "evidence.json" - server.ledger_writer_public_key_hex = "aa" * 32 - server._ensure_audit_artifact_store() - server.rpc = SubmitRpc( - tip="00" * 32, - block_hash=block_hash, - ledger=ledger, - ) - server.build_audit_bundle = ( # type: ignore[method-assign] - lambda **_kwargs: verified_block_bundle() - ) - server.verify_bundle = ( # type: ignore[method-assign] - lambda *_args, **_kwargs: verified_audit_report() - ) - submission = SimpleNamespace( - coinbase_tx_hex="c0ffee", - block_hash_hex=block_hash, - block_hex="00", - ) - candidate = block_candidate(server, state, submission) - - self.assertTrue(server.submit_block_candidate(candidate)) - - self.assertEqual(server.accepted_block_count, 1) - service = server._ensure_block_candidate_service() - histograms = service.accepted_block_preview_publication_snapshot() - published = int(histograms["published"]["count"]) - degraded = int(histograms["degraded"]["count"]) - self.assertEqual(published + degraded, 1) - - telemetry = ensure_accepted_preview_telemetry(server) - diagnostics = telemetry.diagnostics_snapshot() - self.assertEqual(len(diagnostics), 1) - diagnostic = diagnostics[0] - self.assertEqual(diagnostic.block_hash, block_hash) - self.assertEqual(diagnostic.block_height, 10) - self.assertEqual(diagnostic.reconcile_caller, RECONCILE_CALLER_LANDING) - # The measured stretches are stretches *of* the interval they close, - # so they can never sum past it. - self.assertLessEqual( - sum(diagnostic.phase_seconds.values()), - diagnostic.acceptance_to_publication_seconds + 1e-6, - ) - # Nothing is left in flight once the landing completed. - self.assertEqual(service.accepted_landing_attribution_snapshot(), {}) - - def test_the_service_records_nothing_without_the_b1_seam(self) -> None: - # The focused fakes that drive finalization directly carry no - # block-candidate service; the timers must degrade to no-ops rather - # than reaching through a runtime that has no seam. - service = block_finalization_module.BlockFinalizationService( - SimpleNamespace() - ) - self.assertIsNone(service._accepted_landing_owner()) - with service._landing_phase( - LANDING_PHASE_RECONCILE, - block_hash="ab" * 32, - ): - pass - with service._landing_publish_span(block_hash="ab" * 32): - pass - self.assertNotIn( - "_accepted_preview_telemetry", - vars(service.runtime), - ) - - -class LandingAttributionThreadingTests(unittest.TestCase): - """The recorders never nest the leaf lock under anything of their own.""" - - def test_concurrent_landings_keep_their_own_attribution(self) -> None: - server, _state, _ledger = submit_coordinator() - service = server._ensure_block_candidate_service() - hashes = [f"{index:064x}" for index in range(8)] - for block_hash in hashes: - service._note_accepted_block_preview_acceptance(block_hash) - - barrier = threading.Barrier(len(hashes)) - - def land(block_hash: str) -> None: - barrier.wait() - with service._accepted_landing_phase( - block_hash, - LANDING_PHASE_RECONCILE, - block_height=10, - ): - pass - server._observe_accepted_block_preview_publication( - block_hash, - result="published", - ) - - threads = [ - threading.Thread(target=land, args=(block_hash,)) - for block_hash in hashes - ] - for thread in threads: - thread.start() - for thread in threads: - thread.join() - - telemetry = ensure_accepted_preview_telemetry(server) - diagnostics = telemetry.diagnostics_snapshot() - self.assertEqual( - sorted(diagnostic.block_hash for diagnostic in diagnostics), - sorted(hashes), - ) - self.assertEqual( - telemetry.snapshot()["landing_phases"][LANDING_PHASE_RECONCILE][ - "count" - ], - len(hashes), - ) - self.assertEqual(service.accepted_landing_attribution_snapshot(), {}) - - -if __name__ == "__main__": - unittest.main() diff --git a/tests/test_prism_issue_224_ledger.py b/tests/test_prism_issue_224_ledger.py deleted file mode 100644 index 58ecc2d50..000000000 --- a/tests/test_prism_issue_224_ledger.py +++ /dev/null @@ -1,759 +0,0 @@ -#!/usr/bin/env python3 -"""Issue #224 Wave 1: attribution for the ledger reads a preview waits on. - -``PrismAcceptedPreviewPublicationLatencyHigh`` fired while the coordinator -stayed healthy, and nothing on ``/metrics`` said which stretch of the landing -owned the sample. Four ledger reads sit in that stretch -- the payout-window -snapshot and its incremental delta at job issue, and the two prior-balances -reads -- and each of them can be slow for either of two unrelated reasons: -coordinator-local admission (a convoy behind an accounting write) or -PostgreSQL execution. One duration covering both cannot tell an operator -which, which is exactly how #211 was misdiagnosed on the enumeration path -before it was split. - -This wave routes those four through the same attributed read helper the -enumeration already uses, under the closed operation vocabulary -``lab.prism.accepted_preview_telemetry`` froze in Wave 0. The scenarios here -pin the two things that could go wrong while doing it: - -* the *measurement* is right -- the fixed operation name, gate wait apart - from execution, an admission expiry counted as a gate timeout with no - execution sample, a statement expiry counted as an execution timeout; -* the *behavior* is unchanged -- each read keeps the admission primitive it - already had (the writer lock stays the writer lock, the read slot stays the - read slot), sends byte-identical SQL, returns identical records, gives the - primitive back on every failure path, and preserves the original exception. - -The gates below are real ``Lock``/``BoundedSemaphore`` objects behind a -recording wrapper, so "the read released what it took" is a fact about an -actual primitive rather than about a counter. The clock is scripted, so the -recorded seconds are exact. -""" - -from __future__ import annotations - -import copy -import re -import sys -import threading -import unittest -from pathlib import Path -from typing import Any - -sys.path.insert(0, str(Path(__file__).resolve().parents[1])) - -from lab.prism.accepted_preview_telemetry import ( # noqa: E402 - PRISM_LEDGER_READ_OPERATIONS, - fold_ledger_read_stats, -) -from lab.prism.share_ledger import ( # noqa: E402 - AcceptedShareRecord, - LedgerOperationTimeout, - PsqlShareLedger, - ReadOnlyLedgerError, - _RefusingWriterGate, -) - -REPO_ROOT = Path(__file__).resolve().parents[1] - -BLOCK_HASH = "aa" * 32 - -# The operation each method must record under, and the admission primitive it -# must keep. Both halves are the contract: an attribution that renamed the -# operation would break the join with the Wave 0 vocabulary, and one that -# moved a read between primitives would change which writes it serializes -# against -- a concurrency change wearing a metrics change's clothes. -READ_SLOT = "read slot" -WRITER_LOCK = "writer lock" - -# The longest a scenario will wait for admission before calling the gate -# stranded. Every legitimate acquisition below is uncontended and returns -# immediately, so this is never reached by a passing run -- it exists so a -# helper that stopped releasing its gate fails a scenario instead of parking -# the suite on a primitive nobody will ever give back. -STRANDED_GATE_SECONDS = 2.0 - - -class _Clock: - """A monotonic clock these scenarios advance explicitly. - - Every recorded duration below comes from this clock, so a gate wait of - 0.25s is 0.25s in the assertion whether or not the primitive was really - contended and whatever the host was doing at the time. - """ - - def __init__(self, start: float = 1_000.0) -> None: - self.now = float(start) - - def __call__(self) -> float: - return self.now - - def advance(self, seconds: float) -> None: - self.now += float(seconds) - - -class _RecordingGate: - """A real admission primitive that also reports who took it, and when. - - Deliberately a wrapper rather than a stub. The failure scenarios assert - that a read which raises still hands its gate back, and a counter-only - fake cannot tell a released primitive from a leaked one -- the next - acquisition would succeed either way. Wrapping a real ``Lock`` / - ``BoundedSemaphore`` makes a leak show up where it would in production, - as the next caller failing to be admitted (bounded here by - ``STRANDED_GATE_SECONDS`` so the failure is a report, not a hang). - """ - - def __init__( - self, - name: str, - primitive: Any, - journal: list[str], - clock: _Clock, - *, - wait_seconds: float = 0.0, - ) -> None: - self.name = name - self.primitive = primitive - self.journal = journal - self.clock = clock - self.wait_seconds = wait_seconds - self.admissions = 0 - self.releases = 0 - - def acquire(self, timeout: float | None = None) -> bool: - self.journal.append(f"{self.name}:acquire") - acquired = self.primitive.acquire( - timeout=STRANDED_GATE_SECONDS if timeout is None else timeout - ) - # Admission wall-clock is scripted, and is charged whether admission - # succeeded or expired: a caller that waited out its budget waited. - self.clock.advance(self.wait_seconds) - if acquired: - self.admissions += 1 - self.journal.append(f"{self.name}:{'admitted' if acquired else 'expired'}") - return acquired - - def release(self) -> None: - self.releases += 1 - self.journal.append(f"{self.name}:release") - self.primitive.release() - - @property - def outstanding(self) -> int: - return self.admissions - self.releases - - def is_free(self) -> bool: - """Whether the wrapped primitive can still be taken.""" - if not self.primitive.acquire(timeout=0): - return False - self.primitive.release() - return True - - -class _AttributedReadLedger(PsqlShareLedger): - """A ledger carrying only the read plumbing these scenarios touch. - - Built through ``__new__`` in the style the rest of the ledger suite uses - for single-statement scenarios: the class under test is the shipped - ``PsqlShareLedger``, and only the seams a read actually reaches are - supplied. - """ - - def __init__( - self, - results: list[Any], - *, - gate_wait_seconds: float = 0.0, - execute_seconds: float = 0.0, - read_concurrency: int = 1, - ) -> None: - self.clock = _Clock() - self.journal: list[str] = [] - self.queries: list[str] = [] - self.results = list(results) - self.execute_seconds = execute_seconds - self._monotonic = self.clock - self._native = None - self.writer_gate = _RecordingGate( - WRITER_LOCK, - threading.Lock(), - self.journal, - self.clock, - wait_seconds=gate_wait_seconds, - ) - self.read_gate = _RecordingGate( - READ_SLOT, - threading.BoundedSemaphore(read_concurrency), - self.journal, - self.clock, - wait_seconds=gate_wait_seconds, - ) - self._lock = self.writer_gate - self._read_semaphore = self.read_gate - self._prior_balances_read_stats_lock = threading.Lock() - self._prior_balances_reads_total = 0 - self._prior_balances_read_last_seconds = 0.0 - self._prior_balances_read_max_seconds = 0.0 - self._ledger_read_timings: dict[str, dict[str, float | int]] = {} - self._ledger_read_timings_lock = threading.Lock() - - def arm_deadline(self, remaining_seconds: float) -> None: - """Give the calling thread a bounded operation budget.""" - deadline_local = threading.local() - deadline_local.deadline = self.clock.now + remaining_seconds - self._operation_timeout_local = deadline_local - - def _run_json(self, sql: str) -> Any: - self.queries.append(sql) - self.journal.append("statement") - self.clock.advance(self.execute_seconds) - if not self.results: - raise AssertionError(f"unexpected extra query: {sql[:120]}") - result = self.results.pop(0) - if isinstance(result, BaseException): - raise result - return result - - -class _UnattributedReadLedger(_AttributedReadLedger): - """The same ledger with the pre-#224 admission body restored. - - ``_run_attributed_read_json`` is replaced by exactly what each call site - used to do: take the gate through ``_operation_gate`` and run the same - retry-safe statement, recording nothing. It is the differential control - for "the SQL and the results did not change" -- both ledgers run the - shipped method bodies, and only the read helper differs. - """ - - def _run_attributed_read_json( - self, - sql: str, - *, - operation: str, - gate: Any, - gate_name: str, - ) -> Any: - with self._operation_gate(gate, gate_name): - return self._run_retry_safe_read_json(sql) - - def _run_attributed_read_json_rows( - self, - sql: str, - *, - operation: str, - gate: Any, - gate_name: str, - row_converter: Any = None, - ) -> list[Any]: - # The row-result reads (#236: the payout-window snapshot and delta) - # get the same restoration: the caller's gate through - # ``_operation_gate`` and the same retry-safe row statement, - # recording nothing. - with self._operation_gate(gate, gate_name): - return self._run_retry_safe_read_json_rows( - sql, - row_converter=row_converter, - ) - - -def share_row(share_seq: int, *, miner: str = "miner-a") -> dict[str, Any]: - """One accepted-share payload in the shape the snapshot SQL returns.""" - return { - "share_seq": share_seq, - "share_id": f"share-{share_seq}", - "miner_id": miner, - "order_key": miner, - "p2mr_program_hex": "11" * 32, - "share_difficulty": "4096", - "network_difficulty": "1048576", - "template_height": 800_000 + share_seq, - "job_id": f"job-{share_seq}", - "job_issued_at_ms": 1_000 + share_seq, - "accepted_at_ms": 1_001 + share_seq, - "ntime": 1_700_000_000 + share_seq, - "credit_policy": None, - } - - -def balance_row(miner: str = "miner-a") -> dict[str, Any]: - """One carry-forward balance in the shape both balance reads return.""" - return { - "recipient_id": miner, - "order_key": miner, - "p2mr_program_hex": "22" * 32, - "balance_sats": "2500", - } - - -# Each attributed read, as (operation, gate name, canned result, call). The -# scenarios iterate this so a fifth read added to the contract without a -# scenario is a missing row here rather than a silently untested path. -def read_cases() -> tuple[tuple[str, str, Any, Any], ...]: - return ( - ( - "payout_window_snapshot", - READ_SLOT, - [share_row(1), share_row(2)], - lambda ledger: ledger.snapshot_at_job_issue(1_005, window_weight=64), - ), - ( - "payout_window_delta", - READ_SLOT, - [share_row(3)], - lambda ledger: ledger.snapshot_between_job_issues(1_005, 1_006), - ), - ( - "prior_balances_after_pool_block", - READ_SLOT, - [balance_row()], - lambda ledger: ledger.prior_balances_after_pool_block( - block_hash=BLOCK_HASH - ), - ), - ( - "current_prior_balances", - WRITER_LOCK, - [balance_row()], - lambda ledger: ledger.current_prior_balances(), - ), - ) - - -class LedgerReadAttributionTests(unittest.TestCase): - def test_each_read_records_under_its_contract_operation(self) -> None: - """The four names are the Wave 0 ones, spelled exactly.""" - for operation, _gate_name, result, call in read_cases(): - with self.subTest(operation=operation): - ledger = _AttributedReadLedger([result]) - - call(ledger) - - self.assertIn(operation, PRISM_LEDGER_READ_OPERATIONS) - stats = ledger.ledger_read_gate_stats() - # Exactly one operation, exactly once: a read that recorded - # twice would double-count the landing it is meant to explain. - self.assertEqual(list(stats), [operation]) - self.assertEqual(stats[operation]["calls_total"], 1) - - def test_the_shipped_enumeration_attribution_is_untouched(self) -> None: - """``pending_block_candidate_rows`` keeps its name and its read slot.""" - ledger = _AttributedReadLedger( - [[{"block_hash": BLOCK_HASH, "candidate": {}, "pool_block_exists": False}]] - ) - - rows = ledger.pending_block_candidate_rows(limit=8) - - self.assertEqual([row["block_hash"] for row in rows], [BLOCK_HASH]) - self.assertEqual(rows[0]["pool_block_exists"], False) - self.assertEqual( - list(ledger.ledger_read_gate_stats()), - ["pending_block_candidate_rows"], - ) - self.assertEqual(ledger.read_gate.admissions, 1) - self.assertEqual(ledger.writer_gate.admissions, 0) - - def test_each_read_keeps_the_admission_primitive_it_had(self) -> None: - """The writer-lock read stays a writer-lock read, and vice versa. - - This is the invariant most at risk while adding attribution: the - attributed helper used to be hard-wired to the read semaphore, so - routing the prior-balances reread through it is exactly where that - read could have been silently moved off the writer lock it needs. - """ - for operation, gate_name, result, call in read_cases(): - with self.subTest(operation=operation): - ledger = _AttributedReadLedger([result]) - - call(ledger) - - writer_lock = gate_name == WRITER_LOCK - taken = ledger.writer_gate if writer_lock else ledger.read_gate - untaken = ledger.read_gate if writer_lock else ledger.writer_gate - self.assertEqual(taken.admissions, 1) - self.assertEqual(taken.releases, 1) - # Not merely "the right gate was taken": the other one was - # never touched, so no read acquired both or swapped classes. - self.assertEqual(untaken.admissions, 0) - self.assertEqual(untaken.releases, 0) - # And admission precedes the statement it admits. - self.assertEqual( - ledger.journal, - [ - f"{gate_name}:acquire", - f"{gate_name}:admitted", - "statement", - f"{gate_name}:release", - ], - ) - - def test_read_slot_reads_never_reach_the_writer_lock(self) -> None: - """A refusing writer gate is invisible to the three read-slot reads. - - The read-only ledger substitutes ``_RefusingWriterGate`` for the - writer lock, so any path that quietly took it fails loudly. The three - read-slot reads must be unaffected; the prior-balances reread must - still be refused, which is the same statement made from the other - side: it really is on the writer lock. - """ - for operation, gate_name, result, call in read_cases(): - with self.subTest(operation=operation): - ledger = _AttributedReadLedger([result]) - ledger._lock = _RefusingWriterGate() - - if gate_name == WRITER_LOCK: - with self.assertRaises(ReadOnlyLedgerError): - call(ledger) - # Refused before any statement was sent. - self.assertEqual(ledger.queries, []) - self.assertEqual(ledger.read_gate.admissions, 0) - else: - self.assertEqual(len(call(ledger)), len(result)) - self.assertEqual(len(ledger.queries), 1) - self.assertEqual(ledger.read_gate.admissions, 1) - - def test_gate_wait_and_execution_are_separate_samples(self) -> None: - """The whole point: which half of a slow read was slow.""" - for operation, _gate_name, result, call in read_cases(): - with self.subTest(operation=operation): - ledger = _AttributedReadLedger( - [result], - gate_wait_seconds=0.25, - execute_seconds=1.5, - ) - - call(ledger) - - stats = ledger.ledger_read_gate_stats()[operation] - self.assertAlmostEqual(stats["gate_wait_seconds_total"], 0.25) - self.assertAlmostEqual(stats["gate_wait_seconds_max"], 0.25) - self.assertAlmostEqual(stats["execute_seconds_total"], 1.5) - self.assertAlmostEqual(stats["execute_seconds_max"], 1.5) - # Neither timeout counter fires on a read that completed. - self.assertEqual(stats["gate_timeouts_total"], 0) - self.assertEqual(stats["execute_timeouts_total"], 0) - - def test_admission_timeout_is_a_gate_timeout_with_no_execution_sample( - self, - ) -> None: - """A budget spent queueing is never charged to PostgreSQL. - - The gate is genuinely held by another holder, so the expiry is a real - admission expiry rather than a scripted one, and it names the - primitive the caller was waiting for. - """ - for operation, gate_name, result, call in read_cases(): - with self.subTest(operation=operation): - ledger = _AttributedReadLedger([result], gate_wait_seconds=4.0) - held = ( - ledger.writer_gate if gate_name == WRITER_LOCK else ledger.read_gate - ) - self.assertTrue(held.primitive.acquire(timeout=0)) - self.addCleanup(held.primitive.release) - ledger.arm_deadline(0.01) - - with self.assertRaises(LedgerOperationTimeout) as raised: - call(ledger) - - self.assertIn(gate_name, str(raised.exception)) - # No statement was ever sent, so there is nothing to charge to - # the server -- not even a zero-length execution sample. - self.assertEqual(ledger.queries, []) - stats = ledger.ledger_read_gate_stats()[operation] - self.assertEqual(stats["calls_total"], 1) - self.assertEqual(stats["gate_timeouts_total"], 1) - self.assertEqual(stats["execute_timeouts_total"], 0) - self.assertEqual(stats["execute_seconds_total"], 0.0) - self.assertEqual(stats["execute_seconds_max"], 0.0) - self.assertAlmostEqual(stats["gate_wait_seconds_max"], 4.0) - # A failed admission releases nothing, because it took nothing. - self.assertEqual(held.admissions, 0) - self.assertEqual(held.releases, 0) - - def test_execution_timeout_is_an_execution_timeout(self) -> None: - """A deadline that expired inside the statement belongs to the server. - - Including the tail a server-cancelled statement spends returning: the - elapsed execution is recorded, not discarded, so the operator sees how - long PostgreSQL actually held the budget. - """ - for operation, _gate_name, _result, call in read_cases(): - with self.subTest(operation=operation): - ledger = _AttributedReadLedger( - [LedgerOperationTimeout("postgres operation exceeded 5s")], - gate_wait_seconds=0.1, - execute_seconds=5.0, - ) - - with self.assertRaisesRegex(LedgerOperationTimeout, "exceeded 5s"): - call(ledger) - - stats = ledger.ledger_read_gate_stats()[operation] - self.assertEqual(stats["calls_total"], 1) - self.assertEqual(stats["gate_timeouts_total"], 0) - self.assertEqual(stats["execute_timeouts_total"], 1) - self.assertAlmostEqual(stats["execute_seconds_max"], 5.0) - self.assertAlmostEqual(stats["gate_wait_seconds_max"], 0.1) - - def test_failing_reads_release_the_primitive_and_keep_the_error(self) -> None: - """A raising statement leaks nothing and masks nothing. - - Attribution runs in a ``finally``; a version of it that recorded - before releasing, or that swallowed the statement's exception into a - bookkeeping one, would strand the coordinator's writer lock or hide - the failure that mattered. - """ - for operation, gate_name, _result, call in read_cases(): - with self.subTest(operation=operation): - failure = RuntimeError("postgres query failed: connection reset") - ledger = _AttributedReadLedger([failure]) - gate = ( - ledger.writer_gate if gate_name == WRITER_LOCK else ledger.read_gate - ) - - with self.assertRaises(RuntimeError) as raised: - call(ledger) - - # The original exception object, not a wrapper and not a - # timeout misclassification. - self.assertIs(raised.exception, failure) - self.assertEqual(gate.admissions, 1) - self.assertEqual(gate.releases, 1) - self.assertEqual(gate.outstanding, 0) - self.assertTrue(gate.is_free()) - # A failure is still a call, and still not a timeout. - stats = ledger.ledger_read_gate_stats()[operation] - self.assertEqual(stats["calls_total"], 1) - self.assertEqual(stats["gate_timeouts_total"], 0) - self.assertEqual(stats["execute_timeouts_total"], 0) - - def test_a_released_gate_admits_the_next_read(self) -> None: - """The proof that release was real: the same gate serves again. - - Run against a read slot of one and a single writer lock, so a read - that failed to give its primitive back cannot be admitted a second - time (see ``STRANDED_GATE_SECONDS``). - """ - for operation, _gate_name, result, call in read_cases(): - with self.subTest(operation=operation): - ledger = _AttributedReadLedger( - [RuntimeError("first attempt failed"), result], - read_concurrency=1, - ) - - with self.assertRaises(RuntimeError): - call(ledger) - second = call(ledger) - - self.assertEqual(len(second), len(result)) - self.assertEqual( - ledger.ledger_read_gate_stats()[operation]["calls_total"], 2 - ) - - -class UnchangedBehaviorTests(unittest.TestCase): - """Everything except the bookkeeping must be byte-for-byte what it was.""" - - def test_sql_and_results_match_the_unattributed_path(self) -> None: - """Differential against the pre-#224 admission body. - - Both ledgers run the shipped method; only the read helper differs -- - one attributes, one reproduces the old ``_operation_gate`` + - ``_run_retry_safe_read_json`` pair. Identical SQL text and identical - return values is the statement that this wave added a measurement and - nothing else. - """ - for operation, _gate_name, result, call in read_cases(): - with self.subTest(operation=operation): - # Deep copies: both balance reads convert ``balance_sats`` - # in place, and a shared row would let one ledger's parsing - # feed the other's. - attributed = _AttributedReadLedger([copy.deepcopy(result)]) - legacy = _UnattributedReadLedger([copy.deepcopy(result)]) - - new_value = call(attributed) - old_value = call(legacy) - - self.assertEqual(len(attributed.queries), 1) - self.assertEqual(attributed.queries, legacy.queries) - self.assertEqual(new_value, old_value) - # The legacy path records nothing, so equal results cannot be - # an artifact of both sides sharing the new helper. - self.assertEqual(legacy.ledger_read_gate_stats(), {}) - - def test_snapshot_records_parse_exactly_as_before(self) -> None: - """Result parsing is untouched: same fields, same types, same order.""" - ledger = _AttributedReadLedger([[share_row(1), share_row(2, miner="miner-b")]]) - - records = ledger.snapshot_at_job_issue(1_005, window_weight=64) - - self.assertEqual([type(record) for record in records], [AcceptedShareRecord] * 2) - self.assertEqual([record.share_seq for record in records], [1, 2]) - self.assertEqual([record.miner_id for record in records], ["miner-a", "miner-b"]) - self.assertEqual([record.share_difficulty for record in records], [4096, 4096]) - self.assertIsNone(records[0].credit_policy) - - def test_balances_are_still_returned_as_integers(self) -> None: - """Both balance reads keep converting ``balance_sats`` out of text.""" - for call in ( - lambda ledger: ledger.prior_balances_after_pool_block(block_hash=BLOCK_HASH), - lambda ledger: ledger.current_prior_balances(), - ): - with self.subTest(call=call): - ledger = _AttributedReadLedger([[balance_row()]]) - - balances = call(ledger) - - self.assertEqual( - balances, - [{**balance_row(), "balance_sats": 2500}], - ) - - def test_snapshot_sql_still_carries_its_window_bound(self) -> None: - """The statements themselves are the ones the audit path depends on.""" - ledger = _AttributedReadLedger([[share_row(1)], [share_row(2)]]) - - ledger.snapshot_at_job_issue(1_005, window_weight=64) - ledger.snapshot_between_job_issues(1_005, 1_006) - - snapshot_sql, delta_sql = ledger.queries - self.assertIn("WITH RECURSIVE pages AS", snapshot_sql) - self.assertIn("cumulative_difficulty - share_difficulty < 64::numeric", snapshot_sql) - self.assertIn("ORDER BY share_seq ASC", snapshot_sql) - # The delta's two disjoint branches are what keep a share from being - # credited twice across consecutive anchors. - self.assertIn("UNION ALL", delta_sql) - self.assertIn("ledger.accepted_at > to_timestamp", delta_sql) - self.assertIn("ledger.job_issued_at > to_timestamp", delta_sql) - - def test_empty_delta_still_short_circuits_without_a_gate_or_a_statement( - self, - ) -> None: - """An unmoved anchor reads nothing, so it records nothing. - - A sample here would be a lie about work that never happened, and would - inflate the delta operation's call count on every idle rebuild. - """ - ledger = _AttributedReadLedger([]) - - self.assertEqual(ledger.snapshot_between_job_issues(1_005, 1_005), []) - - self.assertEqual(ledger.queries, []) - self.assertEqual(ledger.journal, []) - self.assertEqual(ledger.ledger_read_gate_stats(), {}) - - def test_backwards_anchor_is_still_rejected_before_admission(self) -> None: - ledger = _AttributedReadLedger([]) - - with self.assertRaisesRegex(ValueError, "snapshot anchor moved backwards"): - ledger.snapshot_between_job_issues(1_006, 1_005) - - self.assertEqual(ledger.journal, []) - - def test_legacy_prior_balances_aggregate_is_preserved(self) -> None: - """``qbit_prism_prior_balances_*`` keeps its old meaning. - - It still counts only successful reads and still covers admission and - execution together. The new per-operation sample duplicates part of it - on purpose: an operator comparing the existing series across this - upgrade must not find it silently redefined. - """ - ledger = _AttributedReadLedger( - [[balance_row()], RuntimeError("postgres query failed")], - gate_wait_seconds=0.5, - execute_seconds=2.0, - ) - - ledger.current_prior_balances() - with self.assertRaises(RuntimeError): - ledger.current_prior_balances() - - legacy = ledger.prior_balances_read_stats() - # One success, one failure: the failure is absent from the legacy - # aggregate, exactly as before, and present in the new one. - self.assertEqual(legacy["reads_total"], 1) - self.assertGreaterEqual(legacy["last_seconds"], 0.0) - self.assertGreaterEqual(legacy["max_seconds"], legacy["last_seconds"]) - self.assertEqual( - ledger.ledger_read_gate_stats()["current_prior_balances"]["calls_total"], - 2, - ) - - -class LabelBoundednessTests(unittest.TestCase): - def test_every_attributed_call_site_names_a_contract_operation(self) -> None: - """The literals in the module are the closed vocabulary, and all of it. - - ``tests.test_prism_accepted_preview_telemetry`` already fails on a - literal outside the contract. This is the other direction: the four - names this wave was asked for are actually present, so a lane that - wired up only some of them cannot pass. - """ - source = (REPO_ROOT / "lab" / "prism" / "share_ledger.py").read_text() - literals = set(re.findall(r'operation="([^"]+)"', source)) - - self.assertEqual( - literals, - { - "pending_block_candidate_rows", - "observe_block_candidate_outbox_row", - "pending_block_candidate_headers", - "read_block_candidate_body_manifest", - "read_block_candidate_body_page", - "read_block_candidate_body_pages", - "read_block_candidate_body_spans", - "payout_window_snapshot", - "payout_window_delta", - "current_prior_balances", - "prior_balances_after_pool_block", - }, - ) - self.assertEqual(sorted(literals - set(PRISM_LEDGER_READ_OPERATIONS)), []) - - def test_an_out_of_contract_name_folds_to_other_before_rendering(self) -> None: - """A drifted call site costs an attribution, never a new series. - - The recorder itself keeps whatever name it is handed -- it is a plain - dict, and rejecting there would turn a metrics mistake into a failed - ledger read. Boundedness is the renderer's job, and this is where the - two meet. - """ - ledger = _AttributedReadLedger([[balance_row()], [balance_row()]]) - - ledger.current_prior_balances() - ledger._run_attributed_read_json( - "SELECT json_build_object('ok', true);", - operation="block_hash=" + BLOCK_HASH, - gate=ledger._read_semaphore, - gate_name=READ_SLOT, - ) - - raw = ledger.ledger_read_gate_stats() - self.assertEqual(len(raw), 2) - folded = fold_ledger_read_stats(raw) - self.assertEqual( - sorted(folded), - ["current_prior_balances", "other"], - ) - for operation in folded: - self.assertIn(operation, PRISM_LEDGER_READ_OPERATIONS) - - def test_many_unknown_names_still_render_as_one_series(self) -> None: - """Unbounded cardinality at the call site stays one label at the edge.""" - ledger = _AttributedReadLedger([[] for _ in range(8)]) - - for index in range(8): - ledger._run_attributed_read_json( - "SELECT json_build_object('ok', true);", - operation=f"snapshot_at_job_issue_{index}", - gate=ledger._read_semaphore, - gate_name=READ_SLOT, - ) - - folded = fold_ledger_read_stats(ledger.ledger_read_gate_stats()) - self.assertEqual(list(folded), ["other"]) - self.assertEqual(folded["other"]["calls_total"], 8) - - -if __name__ == "__main__": - unittest.main() diff --git a/tests/test_prism_job_build_exception_retention.py b/tests/test_prism_job_build_exception_retention.py deleted file mode 100644 index 6acc3badc..000000000 --- a/tests/test_prism_job_build_exception_retention.py +++ /dev/null @@ -1,604 +0,0 @@ -#!/usr/bin/env python3 -"""Cancelled job builds must release producer and waiter references. - -A build that observes supersession or timeout raises from inside the executor -task. The stored error's traceback reached the request through the finished -producer frames (and, on Python 3.14, through the executor context frame that -still owned the task's argument tuple), and every waiter that re-raised the -stored instance appended its own frame owning the request and promise. Both -halves formed request -> promise -> error -> traceback -> request cycles that -only cyclic GC reclaimed. - -These regressions drive the real coordinator facade, scheduler, executor -callback, builder checkpoint, and waiter loop with automatic collection -disabled, and assert release by reference counting alone. Threads are -synchronized with events; nothing inspects frame locals or referrers. -""" - -from __future__ import annotations - -import gc -import threading -import time -import unittest -import weakref -from concurrent.futures import Future -from typing import Callable - -from lab.prism.job_bundle import ( - JobBuildCancelled, - JobBuildSuperseded, - _await_job_build_promise, -) -from tests.prism_coordinator_test_support import ( - FakeLedger, - coordinator, - install_fake_bundle_builder, -) - -# The builder checkpoints ledger conversion every 256 rows. Cancelling while -# row CANCEL_ROW converts lets the next checkpoint raise from the real builder -# frame while the ledger snapshot and the converted rows are still live. -WINDOW_ROWS = 1024 -CANCEL_ROW = 300 -WAIT_SECONDS = 20.0 -WAITER_FRAME = "_shared_job_bundle_after_priority_admission" -PRODUCER_FRAMES = ( - "_execute_job_build_request", - "build_shared_job_bundle", - "raise_if_cancelled", -) - - -def wait_until(predicate: Callable[[], bool], *, timeout: float = WAIT_SECONDS) -> None: - deadline = time.monotonic() + timeout - while time.monotonic() < deadline: - if predicate(): - return - time.sleep(0.005) - raise AssertionError("condition did not become true before the deadline") - - -def traceback_names(error: BaseException | None) -> list[str]: - """Function names along a traceback, retaining no frame.""" - - names: list[str] = [] - entry = error.__traceback__ if error is not None else None - while entry is not None: - names.append(entry.tb_frame.f_code.co_name) - entry = entry.tb_next - return names - - -def describe(error: BaseException | None) -> tuple[type, tuple, list[str]] | None: - if error is None: - return None - return type(error), error.args, traceback_names(error) - - -class ShareRow(dict): - """dict subclass so a converted share row is weakref-able.""" - - -class Snapshot(list): - """list subclass so a ledger snapshot is weakref-able.""" - - -class LedgerRecord: - def __init__(self, ledger: "WindowLedger", snapshot_index: int, index: int) -> None: - self.ledger = ledger - self.snapshot_index = snapshot_index - self.index = index - self.payload = { - "miner_id": f"miner-{index % 3}", - "share_seq": index + 1, - "blob": "x" * 64, - } - - def to_prism_json(self) -> ShareRow: - self.ledger.on_convert(self.snapshot_index, self.index) - row = ShareRow(self.payload) - if self.index == 0: - self.ledger.first_rows.append(weakref.ref(row)) - return row - - -class WindowLedger(FakeLedger): - """A ledger whose snapshot rows call back into the test as they convert.""" - - def __init__(self) -> None: - super().__init__(miners=["miner-0", "miner-1", "miner-2"]) - self.on_convert: Callable[[int, int], None] = lambda snapshot_index, index: None - self.snapshots: list[weakref.ref] = [] - self.first_rows: list[weakref.ref] = [] - - def accepted_share_stats(self) -> dict[str, int]: - self.stats_calls += 1 - return {"accepted_share_count": WINDOW_ROWS, "distinct_miner_count": 3} - - def snapshot_at_job_issue( - self, - anchor_job_issued_at_ms: int, - *, - window_weight: int | None = None, - ) -> Snapshot: - self.snapshot_calls += 1 - snapshot_index = self.snapshot_calls - snapshot = Snapshot( - LedgerRecord(self, snapshot_index, index) for index in range(WINDOW_ROWS) - ) - self.snapshots.append(weakref.ref(snapshot)) - return snapshot - - -class _Payload: - pass - - -class JobBuildExceptionRetentionTests(unittest.TestCase): - def setUp(self) -> None: - gc.collect() - self.gc_enabled = gc.isenabled() - gc.disable() - self.ledger = WindowLedger() - self.server, rpc = coordinator(ledger=self.ledger) - install_fake_bundle_builder(self.server) - self.service = self.server._ensure_job_bundle_service() - self.artifacts = self.server.store_template_artifacts(dict(rpc.template)) - self.assertIsNotNone(self.artifacts) - self.requests: list[weakref.ref] = [] - self.promises: list[weakref.ref] = [] - self.flight_promise: weakref.ref | None = None - original_new_request = self.server._new_job_build_request - - def observe_new_request(*args: object, **kwargs: object) -> object: - request = original_new_request(*args, **kwargs) # type: ignore[arg-type] - self.requests.append(weakref.ref(request)) - self.promises.append(weakref.ref(request.promise)) - return request - - self.server._new_job_build_request = observe_new_request # type: ignore[method-assign] - - def tearDown(self) -> None: - self.server.shutdown_job_build_executor() - if self.gc_enabled: - gc.enable() - gc.collect() - - # -- harness ----------------------------------------------------------- - - def hold_first_conversion( - self, - after: Callable[[int], None] | None = None, - ) -> tuple[threading.Event, threading.Event]: - """Pause the first snapshot's conversion at CANCEL_ROW until released.""" - - reached = threading.Event() - proceed = threading.Event() - - def on_convert(snapshot_index: int, index: int) -> None: - if snapshot_index == 1 and index == CANCEL_ROW: - reached.set() - if not proceed.wait(WAIT_SECONDS): - raise TimeoutError("test never released the paused build") - if after is not None: - after(index) - - self.ledger.on_convert = on_convert - return reached, proceed - - def run_waiters( - self, - count: int, - *, - retry_superseded: bool = False, - ) -> tuple[list[threading.Thread], list[dict[str, object]]]: - """Run real bundle waiters, recording only plain data about outcomes.""" - - outcomes: list[dict[str, object]] = [{} for _ in range(count)] - - def stored_error() -> BaseException | None: - reference = self.flight_promise or self.promises[0] - promise = reference() - return None if promise is None else promise.exception(timeout=WAIT_SECONDS) - - def wait(slot: int) -> None: - outcome = outcomes[slot] - try: - self.server.shared_job_bundle( - self.artifacts, - mode="ready", - retry_superseded=retry_superseded, - ) - except JobBuildCancelled as error: - stored = stored_error() - outcome.update( - kind="cancelled", - type=type(error), - args=error.args, - names=traceback_names(error), - notes=list(getattr(error, "__notes__", [])), - attributes={ - key: value - for key, value in vars(error).items() - if key != "__notes__" - }, - cause=describe(error.__cause__), - context=describe(error.__context__), - suppress_context=error.__suppress_context__, - stored_is_error=stored is error, - stored_names=traceback_names(stored), - ) - except BaseException as error: # noqa: BLE001 - asserted below - stored = stored_error() - outcome.update( - kind="unexpected", - type=type(error), - args=error.args, - stored_is_error=stored is error, - ) - else: - outcome.update(kind="bundle") - - threads = [ - threading.Thread(target=wait, args=(slot,), name=f"bundle-waiter-{slot}") - for slot in range(count) - ] - for thread in threads: - thread.start() - return threads, outcomes - - def await_joined_flight(self, waiters: int) -> None: - """Every waiter must have joined the paused flight before it is cancelled.""" - - wait_until( - lambda: self.service.job_build_scheduler_counts["requests"] == waiters - ) - with self.service._job_build_scheduler_lock: - flight = self.service._job_build_active - self.assertIsNotNone(flight) - self.flight_promise = weakref.ref(flight.request.promise) - self.assertEqual(self.ledger.snapshot_calls, 1) - - def cancel_active_flight(self, reason: str) -> None: - with self.service._job_build_scheduler_lock: - flight = self.service._job_build_active - self.assertIsNotNone(flight) - self.assertTrue(self.service._cancel_job_build_flight_locked(flight, reason)) - self.assertEqual(flight.request.cancellation.reason, reason) - - def join_all(self, threads: list[threading.Thread]) -> None: - for thread in threads: - thread.join(WAIT_SECONDS) - self.assertFalse(thread.is_alive(), f"{thread.name} did not finish") - # Joining the executor thread drops the finished work item and the - # task argument tuple it still holds between completion and exit. - self.server.shutdown_job_build_executor() - - def tracked_references(self) -> dict[str, weakref.ref]: - references: dict[str, weakref.ref] = {} - for index, reference in enumerate(self.requests): - references[f"request[{index}]"] = reference - for index, reference in enumerate(self.promises): - references[f"promise[{index}]"] = reference - for index, reference in enumerate(self.ledger.snapshots): - references[f"snapshot[{index}]"] = reference - for index, reference in enumerate(self.ledger.first_rows): - references[f"first_row[{index}]"] = reference - return references - - def assert_released(self, references: dict[str, weakref.ref]) -> None: - self.assertFalse(gc.isenabled()) - self.assertEqual( - {name: reference() is not None for name, reference in references.items()}, - dict.fromkeys(references, False), - ) - - def assert_cancelled_outcome( - self, - outcome: dict[str, object], - error_type: type[JobBuildCancelled], - message_prefix: str, - ) -> None: - self.assertEqual(outcome["kind"], "cancelled", outcome) - self.assertIs(outcome["type"], error_type) - args = outcome["args"] - assert isinstance(args, tuple) - self.assertEqual(len(args), 1) - self.assertTrue(str(args[0]).startswith(message_prefix), args) - names = outcome["names"] - assert isinstance(names, list) - self.assertIn(WAITER_FRAME, names) - self.assertFalse(outcome["stored_is_error"]) - stored_names = outcome["stored_names"] - assert isinstance(stored_names, list) - self.assertNotIn(WAITER_FRAME, stored_names) - self.assertIn("_execute_job_build_request", stored_names) - - # -- supersession and timeout through the real checkpoint -------------- - - def check_cancelled_flight_releases_for_every_waiter( - self, - reason: str, - error_type: type[JobBuildCancelled], - message_prefix: str, - ) -> None: - reached, proceed = self.hold_first_conversion() - threads, outcomes = self.run_waiters(3) - self.assertTrue(reached.wait(WAIT_SECONDS)) - self.await_joined_flight(3) - self.cancel_active_flight(reason) - proceed.set() - self.join_all(threads) - for outcome in outcomes: - self.assert_cancelled_outcome(outcome, error_type, message_prefix) - names = outcome["names"] - assert isinstance(names, list) - for producer_frame in PRODUCER_FRAMES: - self.assertIn(producer_frame, names) - self.assertEqual(outcome["cause"], None) - self.assertEqual(outcome["context"], None) - self.assertEqual(len(self.requests), 3) - self.assertEqual(self.ledger.snapshot_calls, 1) - # Scheduler accounting is unchanged: supersession is an obsolete - # result, a timeout is a failed build. - superseded = reason != "timeout" - self.assertEqual( - self.service.job_build_scheduler_counts["obsolete_results"], - 1 if superseded else 0, - ) - self.assertEqual( - self.service.shared_bundle_build_counts["superseded"], - 1 if superseded else 0, - ) - self.assertEqual( - self.service.shared_bundle_build_counts["failed"], - 0 if superseded else 1, - ) - self.assert_released(self.tracked_references()) - - def test_superseded_flight_releases_window_for_every_waiter(self) -> None: - self.check_cancelled_flight_releases_for_every_waiter( - "superseded", - JobBuildSuperseded, - "job build superseded at ledger_snapshot_conversion", - ) - - def test_timed_out_flight_releases_window_for_every_waiter(self) -> None: - self.check_cancelled_flight_releases_for_every_waiter( - "timeout", - JobBuildCancelled, - "job build timeout at ledger_snapshot_conversion", - ) - - def test_in_frame_retry_releases_the_superseded_build(self) -> None: - reached, proceed = self.hold_first_conversion() - threads, outcomes = self.run_waiters(1, retry_superseded=True) - self.assertTrue(reached.wait(WAIT_SECONDS)) - self.await_joined_flight(1) - self.cancel_active_flight("superseded") - proceed.set() - self.join_all(threads) - self.assertEqual(outcomes[0], {"kind": "bundle"}) - self.assertEqual(self.ledger.snapshot_calls, 2) - self.assertEqual(len(self.requests), 2) - # The retry's cached bundle legitimately keeps its converted rows; - # everything the superseded build owned must be gone. - self.assert_released( - { - "superseded_request": self.requests[0], - "superseded_promise": self.promises[0], - "superseded_snapshot": self.ledger.snapshots[0], - "superseded_first_row": self.ledger.first_rows[0], - "retry_request": self.requests[1], - "retry_promise": self.promises[1], - "retry_snapshot": self.ledger.snapshots[1], - } - ) - - # -- orphan sweep resolves under the scheduler lock -------------------- - - def test_orphan_sweep_resolves_wedged_flight_and_releases_window(self) -> None: - self.server.job_build_orphan_sweep_grace_seconds = 0.0 # type: ignore[attr-defined] - flights: list[weakref.ref] = [] - - def never_arm(flight: object) -> None: - # A done callback that died before resolving leaves the flight - # wedged in its slot; the admission sweep must settle it. - flights.append(weakref.ref(flight)) - - self.server._arm_job_build_locked = never_arm # type: ignore[method-assign] - reached, proceed = self.hold_first_conversion() - threads, outcomes = self.run_waiters(2) - self.assertTrue(reached.wait(WAIT_SECONDS)) - self.await_joined_flight(2) - self.cancel_active_flight("superseded") - proceed.set() - wait_until(lambda: flights[0]().future.done()) - with self.service._job_build_scheduler_lock: - self.assertEqual(self.service._evict_orphaned_job_build_flights_locked(), []) - logs = self.service._evict_orphaned_job_build_flights_locked() - self.assertEqual(len(logs), 1) - self.assertIn("slot=active", logs[0]) - self.assertIn("cancelled=True", logs[0]) - self.assertEqual(self.service.job_build_scheduler_counts["orphan_evicted"], 1) - self.join_all(threads) - for outcome in outcomes: - self.assert_cancelled_outcome( - outcome, - JobBuildSuperseded, - "job build superseded at ledger_snapshot_conversion", - ) - references = self.tracked_references() - references["flight"] = flights[0] - self.assert_released(references) - - # -- chained producer failures reported as supersession ----------------- - - def check_chained_cancellation_releases_nested_frames(self, chaining: str) -> None: - def reject_row(index: int) -> None: - raise ValueError(f"ledger row {index} rejected") - - reached, proceed = self.hold_first_conversion(after=reject_row) - coordinator_entry = self.server._execute_job_build_request - - def _execute_job_build_request(request: object) -> object: - # Stand-in for an executor entry that reports a nested producer - # failure as supersession, as the bundle compiler does with a - # subprocess failure. Like the real entry, its frame owns the - # request while the nested frames own the payout window. - try: - return coordinator_entry(request) # type: ignore[arg-type] - except ValueError as failure: - error = JobBuildSuperseded( - "job build superseded at ledger_snapshot_conversion; row rejected" - ) - error.add_note("rejected row converts on the next build") - error.phase = "ledger_snapshot_conversion" # type: ignore[attr-defined] - if chaining == "cause": - raise error from failure - if chaining == "suppressed": - raise error from None - raise error - - self.server._execute_job_build_request = _execute_job_build_request # type: ignore[method-assign] - threads, outcomes = self.run_waiters(2) - self.assertTrue(reached.wait(WAIT_SECONDS)) - self.await_joined_flight(2) - proceed.set() - self.join_all(threads) - failure = (ValueError, (f"ledger row {CANCEL_ROW} rejected",)) - for outcome in outcomes: - self.assert_cancelled_outcome( - outcome, - JobBuildSuperseded, - "job build superseded at ledger_snapshot_conversion; row rejected", - ) - self.assertEqual(outcome["notes"], ["rejected row converts on the next build"]) - self.assertEqual(outcome["attributes"], {"phase": "ledger_snapshot_conversion"}) - cause = outcome["cause"] - context = outcome["context"] - if chaining == "cause": - self.assertTrue(outcome["suppress_context"]) - self.assertIsNotNone(cause) - chained = cause - else: - self.assertIsNone(cause) - self.assertEqual(outcome["suppress_context"], chaining == "suppressed") - chained = context - assert chained is not None - self.assertEqual(chained[:2], failure) - for producer_frame in ("_execute_job_build_request", "build_shared_job_bundle", "to_prism_json"): - self.assertIn(producer_frame, chained[2]) - self.assertEqual(len(self.requests), 2) - self.assertEqual(self.ledger.snapshot_calls, 1) - self.assert_released(self.tracked_references()) - - def test_cancellation_with_cause_releases_nested_producer_frames(self) -> None: - self.check_chained_cancellation_releases_nested_frames("cause") - - def test_cancellation_with_context_releases_nested_producer_frames(self) -> None: - self.check_chained_cancellation_releases_nested_frames("context") - - def test_cancellation_with_suppressed_context_releases_nested_frames(self) -> None: - self.check_chained_cancellation_releases_nested_frames("suppressed") - - # -- unexpected failures keep their behavior ---------------------------- - - def test_unexpected_build_failure_still_raises_the_stored_instance(self) -> None: - def reject_row(index: int) -> None: - raise ValueError(f"ledger row {index} rejected") - - reached, proceed = self.hold_first_conversion(after=reject_row) - threads, outcomes = self.run_waiters(1) - self.assertTrue(reached.wait(WAIT_SECONDS)) - self.await_joined_flight(1) - proceed.set() - self.join_all(threads) - self.assertEqual(outcomes[0]["kind"], "unexpected") - self.assertIs(outcomes[0]["type"], ValueError) - self.assertEqual(outcomes[0]["args"], (f"ledger row {CANCEL_ROW} rejected",)) - self.assertTrue(outcomes[0]["stored_is_error"]) - self.assertEqual(self.service.shared_bundle_build_counts["failed"], 1) - # An unexpected failure keeps its full traceback for post-mortem - # inspection, so only collection reclaims that build. - gc.collect() - self.assert_released(self.tracked_references()) - - -class WaiterCopyTests(unittest.TestCase): - def setUp(self) -> None: - gc.collect() - self.gc_enabled = gc.isenabled() - gc.disable() - - def tearDown(self) -> None: - if self.gc_enabled: - gc.enable() - gc.collect() - - def test_waiter_copy_preserves_metadata_and_leaves_stored_traceback_alone(self) -> None: - payload = _Payload() - promise: Future[object] = Future() - try: - raise ValueError("compiler exited 137") - except ValueError as failure: - stored = JobBuildSuperseded("job build superseded at bundle_assembly") - stored.add_note("compiler exit 137") - stored.phase = "bundle_assembly" # type: ignore[attr-defined] - stored.__cause__ = failure - stored.__suppress_context__ = True - promise.set_exception(stored) - references = {"payload": weakref.ref(payload), "promise": weakref.ref(promise)} - - def waiter(owned_payload: _Payload, owned_promise: Future[object]) -> None: - _await_job_build_promise(owned_promise, 1.0) - - try: - waiter(payload, promise) - except JobBuildSuperseded as error: - self.assertIsNot(error, stored) - self.assertIs(type(error), JobBuildSuperseded) - self.assertEqual(error.args, stored.args) - self.assertEqual(error.__notes__, ["compiler exit 137"]) # type: ignore[attr-defined] - self.assertEqual(error.phase, "bundle_assembly") # type: ignore[attr-defined] - self.assertIs(error.__cause__, stored.__cause__) - self.assertIs(error.__context__, stored.__context__) - self.assertTrue(error.__suppress_context__) - self.assertEqual( - traceback_names(error)[-2:], - ["waiter", "_await_job_build_promise"], - ) - else: - self.fail("waiter did not observe the cancellation") - # The stored instance was never raised: no waiter frame reached it. - self.assertIsNone(stored.__traceback__) - self.assertIs(promise.exception(), stored) - del stored, payload, promise - self.assertFalse(gc.isenabled()) - self.assertEqual( - {name: reference() is not None for name, reference in references.items()}, - dict.fromkeys(references, False), - ) - - def test_waiter_keeps_future_result_semantics_for_other_outcomes(self) -> None: - pending: Future[object] = Future() - with self.assertRaises(TimeoutError): - _await_job_build_promise(pending, 0.001) - completed: Future[object] = Future() - result = _Payload() - completed.set_result(result) - self.assertIs(_await_job_build_promise(completed, 1.0), result) - failed: Future[object] = Future() - failure = RuntimeError("build failed") - failed.set_exception(failure) - try: - _await_job_build_promise(failed, 1.0) - except RuntimeError as error: - self.assertIs(error, failure) - else: - self.fail("unexpected failure was not re-raised") - - -if __name__ == "__main__": - unittest.main() diff --git a/tests/test_prism_job_build_promises.py b/tests/test_prism_job_build_promises.py deleted file mode 100644 index cf21a5698..000000000 --- a/tests/test_prism_job_build_promises.py +++ /dev/null @@ -1,388 +0,0 @@ -#!/usr/bin/env python3 -"""Single-flight job-build promise lifecycle regressions. - -A cancelled or superseded flight must never leave joiners parked on an -unresolved promise: cancellation resolves the shared promise exceptionally at -the moment of termination, and dedup admission refuses (and eventually -evicts) flights whose completion never released their scheduler slot, so the -next requester becomes the new owner instead of orbiting an abandoned -flight. -""" - -from __future__ import annotations - -import threading -import unittest -from concurrent.futures import Future - -from lab.prism.prism_coordinator import ( - JobBuildCancelled, - JobBuildSuperseded, - PrismCoordinator, - _JobBuildFlight, - _JobBuildRequest, -) -from tests.prism_coordinator_test_support import ( - coordinator, - install_fake_bundle_builder, - synthetic_manifest_coinbase_hex, - worker, -) - - -def install_gated_bundle_builder( - server: PrismCoordinator, -) -> tuple[threading.Event, threading.Event, dict[str, object]]: - """A counting fake audit-bundle builder that parks until released. - - Returns (entered, release, recorded): ``entered`` is set when a build has - reached the builder, ``release`` lets it finish. Waiter-wakeup assertions - run while the build is provably still executing. - """ - - entered = threading.Event() - release = threading.Event() - recorded: dict[str, object] = {"calls": 0} - - def gated_build_audit_bundle(**kwargs: object) -> dict[str, object]: - recorded["calls"] = int(recorded["calls"]) + 1 - entered.set() - if not release.wait(timeout=30.0): - raise AssertionError("bundle builder gate never released") - suffix_hex = str(kwargs["coinbase_script_sig_suffix_hex"]) - return { - "found_block": dict(kwargs["found_block"]), # type: ignore[call-overload] - "payout_policy_manifest": {"accounts": []}, - "signed_coinbase_manifest": { - "manifest": { - "coinbase_tx_hex": synthetic_manifest_coinbase_hex( - suffix_hex - ), - } - }, - } - - server.build_audit_bundle = gated_build_audit_bundle # type: ignore[method-assign] - return entered, release, recorded - - -class _PromiseHarness(unittest.TestCase): - def setUp(self) -> None: - self.server, self.rpc = coordinator() - self.identity = worker() - self.artifacts = self.server.store_template_artifacts( - dict(self.rpc.template) - ) - assert self.artifacts is not None - self.cache_key = self.server._job_bundle_key( - self.artifacts, - mode="ready", - payout_state_generation=0, - worker=self.identity, - ) - - def tearDown(self) -> None: - self.server.shutdown_job_build_executor() - - def new_request(self) -> _JobBuildRequest: - return self.server._new_job_build_request( - self.artifacts, - self.identity, - mode="ready", - payout_state_generation=0, - cache_key=self.cache_key, - ) - - -class CancelledFlightWakesJoinersTests(_PromiseHarness): - def test_cancelling_running_build_delivers_at_drain(self) -> None: - # Owner starts a build that parks inside the builder; a joiner shares - # the same flight. Cancelling a flight whose build is still executing - # keeps today's ordering: the promise resolves when the drain - # completes, never earlier -- and, with the completion path hardened, - # never later. - entered, release, recorded = install_gated_bundle_builder(self.server) - owner = self.new_request() - owner_promise = self.server._request_job_build(owner) - self.assertTrue(entered.wait(timeout=5.0)) - - joiner = self.new_request() - joiner_promise = self.server._request_job_build(joiner) - self.assertIs(joiner_promise, owner_promise) - - with self.server._job_build_scheduler_lock: - flight = self.server._job_build_active - assert flight is not None - self.assertTrue( - self.server._cancel_job_build_flight_locked( - flight, - "superseded", - ) - ) - self.assertFalse(owner_promise.done()) - - release.set() - error = owner_promise.exception(timeout=5.0) - self.assertIsInstance(error, JobBuildSuperseded) - - # The retry after the drain becomes a fresh owner and builds. - rebuild = self.new_request() - rebuild_promise = self.server._request_job_build(rebuild) - self.assertIsNot(rebuild_promise, owner_promise) - rebuild_promise.result(timeout=10.0) - self.assertEqual(recorded["calls"], 2) - - def test_cancelling_never_started_flight_wakes_waiters_immediately( - self, - ) -> None: - # Without a live executor future nothing else can ever resolve the - # promise: cancellation itself must wake the waiters instead of - # leaving them to burn their full wait deadline. - owner = self.new_request() - flight = _JobBuildFlight(request=owner) - self.server._job_build_active = flight - - with self.server._job_build_scheduler_lock: - self.assertTrue( - self.server._cancel_job_build_flight_locked( - flight, - "superseded", - ) - ) - - error = owner.promise.exception(timeout=1.0) - self.assertIsInstance(error, JobBuildSuperseded) - self.server._job_build_active = None - - def test_timeout_cancellation_maps_to_job_build_cancelled(self) -> None: - owner = self.new_request() - flight = _JobBuildFlight(request=owner) - self.server._job_build_active = flight - - with self.server._job_build_scheduler_lock: - self.server._cancel_job_build_flight_locked(flight, "timeout") - - error = owner.promise.exception(timeout=1.0) - self.assertIsInstance(error, JobBuildCancelled) - self.assertNotIsInstance(error, JobBuildSuperseded) - self.server._job_build_active = None - - def test_cancel_obsolete_job_builds_resolves_orphaned_promises(self) -> None: - # The detection-sweep termination path shares the same guarantee: a - # flight it cancels without a live future resolves immediately, and a - # draining build keeps today's resolve-at-drain ordering. - entered, release, _recorded = install_gated_bundle_builder(self.server) - owner = self.new_request() - owner_promise = self.server._request_job_build(owner) - self.assertTrue(entered.wait(timeout=5.0)) - - self.server._cancel_obsolete_job_builds( - "superseded by a newer template observation" - ) - self.assertFalse(owner_promise.done()) - - release.set() - error = owner_promise.exception(timeout=5.0) - self.assertIsInstance(error, JobBuildSuperseded) - - -class OrphanedFlightEvictionTests(_PromiseHarness): - def test_orphaned_flight_is_evicted_and_not_reserved(self) -> None: - # Manufacture the wedge a died done-callback leaves behind: a flight - # whose executor future finished but whose promise was never resolved - # and whose slot was never released. Repeated requests must not orbit - # it: the sweep resolves the promise from the finished future, frees - # the slot, and the next requester becomes the new owner. The gated - # builder keeps the new owner's build parked so its slot ownership - # stays observable. - entered, release, _recorded = install_gated_bundle_builder(self.server) - self.server.job_build_orphan_sweep_grace_seconds = 0.0 - - corpse_request = self.new_request() - corpse_future: Future = Future() - corpse_future.set_exception(RuntimeError("builder thread died")) - corpse = _JobBuildFlight( - request=corpse_request, - future=corpse_future, - ) - self.server._job_build_active = corpse - - # First admission cannot yet distinguish the corpse from a completion - # whose callback is still in flight; it only marks the observation. - first = self.new_request() - first_promise = self.server._request_job_build(first) - self.assertIs(first_promise, corpse_request.promise) - self.assertFalse(corpse_request.promise.done()) - - # Past the grace, the next admission evicts: the abandoned promise - # resolves (waking the first requester) and the corpse never owns the - # slot again. - second = self.new_request() - second_promise = self.server._request_job_build(second) - self.assertIsNot(second_promise, corpse_request.promise) - self.assertIsInstance( - corpse_request.promise.exception(timeout=1.0), - RuntimeError, - ) - self.assertTrue(entered.wait(timeout=5.0)) - with self.server._job_build_scheduler_lock: - active = self.server._job_build_active - assert active is not None - self.assertIs(active.request, second) - self.assertEqual( - self.server.job_build_scheduler_counts["orphan_evicted"], - 1, - ) - release.set() - second_promise.result(timeout=10.0) - - def test_running_flight_is_never_swept(self) -> None: - # The sweep must not touch a flight whose future is still executing: - # evicting it would let a replacement start and break the bounded - # two-build concurrency invariant. - entered, release, _recorded = install_gated_bundle_builder(self.server) - self.server.job_build_orphan_sweep_grace_seconds = 0.0 - owner = self.new_request() - owner_promise = self.server._request_job_build(owner) - self.assertTrue(entered.wait(timeout=5.0)) - - with self.server._job_build_scheduler_lock: - self.server._evict_orphaned_job_build_flights_locked() - self.server._evict_orphaned_job_build_flights_locked() - flight = self.server._job_build_active - assert flight is not None - self.assertIs(flight.request, owner) - self.assertEqual( - self.server.job_build_scheduler_counts["orphan_evicted"], - 0, - ) - release.set() - owner_promise.result(timeout=10.0) - - -class StructuralResolutionFaultInjectionTests(_PromiseHarness): - def test_poisoned_completion_bookkeeping_still_resolves_promise(self) -> None: - # The completion callback's promise resolution must survive a raise - # anywhere in its bookkeeping (done callbacks swallow exceptions, so - # without the try/finally the promise would strand until the sweep). - install_fake_bundle_builder(self.server) - - def poisoned_promote() -> None: - raise RuntimeError("bookkeeping poisoned by test") - - self.server._promote_pending_job_build_locked = ( # type: ignore[method-assign] - poisoned_promote - ) - try: - owner = self.new_request() - promise = self.server._request_job_build(owner) - self.assertIsNotNone(promise.result(timeout=10.0)) - # The sweep played no part: the callback itself resolved it. - self.assertEqual( - self.server.job_build_scheduler_counts["orphan_evicted"], - 0, - ) - finally: - del self.server._promote_pending_job_build_locked - - def test_promote_start_failure_resolves_consumed_pending_promise( - self, - ) -> None: - # A pending request consumed by promotion whose executor start fails - # has no flight and therefore no completion callback left; promotion - # itself must settle the promise for its waiters. - pending = self.new_request() - self.server._job_build_pending = pending - - def failing_start(request: _JobBuildRequest) -> _JobBuildFlight: - raise RuntimeError("executor unavailable") - - self.server._start_job_build_locked = ( # type: ignore[method-assign] - failing_start - ) - try: - with self.server._job_build_scheduler_lock: - with self.assertRaises(RuntimeError): - self.server._promote_pending_job_build_locked() - finally: - del self.server._start_job_build_locked - self.assertIsNone(self.server._job_build_pending) - self.assertIsInstance( - pending.promise.exception(timeout=1.0), - JobBuildSuperseded, - ) - - def test_resolved_priority_corpse_does_not_defer_routine_requesters( - self, - ) -> None: - # A resolved-promise publication-critical corpse is not live priority - # work: deferring on its already-done promise would wake instantly - # and hot-spin until the sweep's grace elapses. The routine requester - # must take ownership through the normal supersession path instead. - entered, release, _recorded = install_gated_bundle_builder(self.server) - corpse_request = self.server._new_job_build_request( - self.artifacts, - self.identity, - mode="ready", - payout_state_generation=0, - cache_key=self.cache_key, - publication_critical=True, - ) - corpse_request.promise.set_exception( - JobBuildSuperseded("resolved before the slot was released") - ) - corpse_future: Future = Future() - corpse_future.set_exception(RuntimeError("builder thread died")) - self.server._job_build_active = _JobBuildFlight( - request=corpse_request, - future=corpse_future, - ) - - routine = self.new_request() - promise = self.server._request_job_build(routine) - self.assertIs(promise, routine.promise) - self.assertTrue(entered.wait(timeout=5.0)) - release.set() - promise.result(timeout=10.0) - - def test_shutdown_settles_terminal_corpse_waiters(self) -> None: - # Executor shutdown re-fires callbacks for queued and running - # futures, but a terminal future's callback never fires again: - # shutdown is the last chance to settle its waiters. - corpse = self.new_request() - corpse_future: Future = Future() - corpse_future.set_exception(RuntimeError("builder thread died")) - self.server._job_build_active = _JobBuildFlight( - request=corpse, - future=corpse_future, - ) - self.server.shutdown_job_build_executor() - self.assertIsInstance( - corpse.promise.exception(timeout=1.0), - JobBuildSuperseded, - ) - - -class HealthySingleFlightTests(_PromiseHarness): - def test_equivalent_requests_share_one_build(self) -> None: - entered, release, recorded = install_gated_bundle_builder(self.server) - owner = self.new_request() - owner_promise = self.server._request_job_build(owner) - self.assertTrue(entered.wait(timeout=5.0)) - - joiner = self.new_request() - joiner_promise = self.server._request_job_build(joiner) - self.assertIs(joiner_promise, owner_promise) - - release.set() - owner_bundle = owner_promise.result(timeout=10.0) - self.assertIs(joiner_promise.result(timeout=10.0), owner_bundle) - self.assertEqual(recorded["calls"], 1) - self.assertEqual( - self.server.job_build_scheduler_counts["orphan_evicted"], - 0, - ) - - -if __name__ == "__main__": - unittest.main() diff --git a/tests/test_prism_job_build_retention.py b/tests/test_prism_job_build_retention.py deleted file mode 100644 index 145f902e0..000000000 --- a/tests/test_prism_job_build_retention.py +++ /dev/null @@ -1,144 +0,0 @@ -#!/usr/bin/env python3 -"""Completed job flights must release their windows without cyclic GC.""" - -from __future__ import annotations - -import gc -import unittest -import weakref -from concurrent.futures import CancelledError, Future -from types import SimpleNamespace -from unittest.mock import patch - -from lab.prism.job_bundle import JobBuildFlight, JobBundleService - - -class _Payload: - pass - - -class JobBuildCallbackRetentionTests(unittest.TestCase): - def setUp(self) -> None: - gc.collect() - self.gc_enabled = gc.isenabled() - gc.disable() - self.completions = 0 - self.fail_completion = False - self.owner = SimpleNamespace(_runtime=self) - - def tearDown(self) -> None: - if self.gc_enabled: - gc.enable() - gc.collect() - - def _job_build_done(self, flight: JobBuildFlight, future: Future) -> None: - self.assertIs(flight.future, future) - self.assertTrue(future.done()) - self.completions += 1 - if self.fail_completion: - raise RuntimeError("injected completion bookkeeping failure") - - def arm(self, *, already_finished: bool = False): - request = _Payload() - result = _Payload() - future = Future() - flight = JobBuildFlight(request=request, future=future) - references = { - "request": weakref.ref(request), - "result": weakref.ref(result), - "flight": weakref.ref(flight), - "future": weakref.ref(future), - } - if already_finished: - future.set_result(result) - JobBundleService._arm_job_build_locked(self.owner, flight) - return future, result, references - - def assert_released(self, references) -> None: - self.assertFalse(gc.isenabled()) - self.assertEqual( - {name: ref() is not None for name, ref in references.items()}, - dict.fromkeys(references, False), - ) - - def test_completed_flights_and_results_release_without_collection(self) -> None: - for _ in range(10): - future, result, references = self.arm() - future.set_result(result) - del future, result - self.assert_released(references) - self.assertEqual(self.completions, 10) - - def test_callback_owns_pending_flight_until_completion(self) -> None: - future, result, references = self.arm() - self.assertIsNotNone(references["flight"]()) - self.assertIsNotNone(references["request"]()) - self.assertEqual(self.completions, 0) - future.set_result(result) - self.assertIsNone(references["flight"]()) - self.assertIsNone(references["request"]()) - self.assertIs(future.result(), result) - del future, result - self.assert_released(references) - self.assertEqual(self.completions, 1) - - def test_already_finished_future_completes_inline(self) -> None: - future, result, references = self.arm(already_finished=True) - self.assertEqual(self.completions, 1) - self.assertIs(future.result(), result) - del future, result - self.assert_released(references) - - def test_cancelled_future_releases_flight(self) -> None: - future, result, references = self.arm() - self.assertTrue(future.cancel()) - del future, result - self.assert_released(references) - self.assertEqual(self.completions, 1) - - def test_exceptional_future_releases_callback_ownership(self) -> None: - future, result, references = self.arm() - error = RuntimeError("build failed") - future.set_exception(error) - self.assertIs(future.exception(), error) - del future, result - self.assert_released(references) - self.assertEqual(self.completions, 1) - - def test_bookkeeping_failure_releases_callback_ownership(self) -> None: - future, result, references = self.arm() - self.fail_completion = True - # Capturing logging records here would itself retain the callback's - # traceback and flight, masking the ownership under test. - with patch("concurrent.futures._base.LOGGER.exception", lambda *a, **kw: None): - future.set_result(result) - self.assertEqual(self.completions, 1) - self.assertIs(future.result(), result) - del future, result - self.assert_released(references) - - def check_failed_outcome_releases_request(self, *, cancelled: bool) -> None: - request = _Payload() - request.cancellation = SimpleNamespace(is_set=lambda: False) - future = Future() - references = {"request": weakref.ref(request), "future": weakref.ref(future)} - if cancelled: - future.cancel() - else: - future.set_exception(RuntimeError("build failed")) - result, error = JobBundleService._job_build_flight_outcome(request, future) - self.assertIsNone(result) - self.assertIsInstance(error, CancelledError if cancelled else RuntimeError) - self.assertIsNone(error.__traceback__) - del request, future, result, error - self.assert_released(references) - - def test_failed_outcome_does_not_capture_request_in_traceback(self) -> None: - self.check_failed_outcome_releases_request(cancelled=False) - - def test_cancelled_outcome_does_not_capture_request_in_traceback(self) -> None: - self.check_failed_outcome_releases_request(cancelled=True) - - -if __name__ == "__main__": - unittest.main() diff --git a/tests/test_prism_job_builder.py b/tests/test_prism_job_builder.py deleted file mode 100644 index f806bef3f..000000000 --- a/tests/test_prism_job_builder.py +++ /dev/null @@ -1,4810 +0,0 @@ -#!/usr/bin/env python3 -"""Focused PRISM coordinator job builder tests.""" -# ruff: noqa: F403, F405 - -from __future__ import annotations - -import json -import unittest -from tests.prism_coordinator_test_support import * - -from lab.prism.bundle_compiler import ( - _compact_share_tail_chunks, - _iter_prepare_window_request_chunks, -) -from lab.prism.share_ledger import DaemonShareWindowMirror - - -class SnapshotAnchorFloorTests(unittest.TestCase): - def _hold_floor(self, server: PrismCoordinator, share: PendingShare) -> None: - server._ensure_pending_share_commit_state() - with server._pending_share_commit_lock: - server._pending_share_commit_floor[id(share)] = [ - share, - time.monotonic(), - False, - ] - - def test_job_bundle_anchor_clamps_below_pending_share_commit(self) -> None: - # The issued snapshot must be reproducible from the durable ledger: - # while a stamped share's commit is pending, the job anchor (which the - # bundle declares as anchor_job_issued_at_ms) has to predate it, or - # qbit_audit_share_window at the declared anchor would include a share - # the published window omitted. - ledger = AnchorRecordingLedger() - server, _rpc = coordinator(ledger=ledger) - install_fake_bundle_builder(server) - stamped_ms = now_ms() - 5 - share = stamped_pending_share(stamped_ms) - self._hold_floor(server, share) - - bundle = server.build_shared_job_bundle( - server.current_template_artifacts(), - worker(), - ) - self.assertEqual(ledger.anchors[-1], stamped_ms - 1) - self.assertEqual( - bundle.found_block["anchor_job_issued_at_ms"], stamped_ms - 1 - ) - self.assertEqual(bundle.issued_at_ms, stamped_ms - 1) - - server._finish_pending_share_commit(share) - # The issued time is frozen per template generation; drop the frozen - # entry so the rebuild stamps a fresh anchor now that no commit is - # pending. - with server._job_cache_lock: - server._job_build_issued_at_ms.clear() - rebuilt = server.build_shared_job_bundle( - server.current_template_artifacts(), - worker(), - ) - self.assertGreaterEqual(ledger.anchors[-1], stamped_ms) - self.assertGreaterEqual( - int(rebuilt.found_block["anchor_job_issued_at_ms"]), stamped_ms - ) - - def test_payout_artifact_declares_its_own_snapshot_anchor(self) -> None: - # An artifact snapshot is taken at its own earlier anchor. A bundle - # reusing the artifact must declare that anchor rather than the - # fresher job-issue time: a share stamped between the two anchors is - # excluded from the artifact by construction, yet a re-derivation at - # the job-issue anchor would include it. - ledger = AnchorRecordingLedger() - server, _rpc = coordinator(ledger=ledger) - install_fake_bundle_builder(server) - artifacts = server.current_template_artifacts() - clamp_now_ms = now_ms() - 6 - # Anchor selection sits strictly below the clamp instant so a share - # stamped in the same millisecond can never tie the anchor. - artifact_anchor_ms = clamp_now_ms - 1 - with patch( - "lab.prism.prism_coordinator.now_ms", - return_value=clamp_now_ms, - ): - artifact = server._build_payout_ledger_artifact( - 0, 0, artifacts.network_difficulty - ) - assert artifact is not None - self.assertEqual(artifact.snapshot_anchor_ms, artifact_anchor_ms) - self.assertEqual(ledger.anchors[-1], artifact_anchor_ms) - - # Construction re-validates that the passed artifact is the installed - # current one. - with server._job_cache_lock: - server._payout_ledger_artifact = artifact - bundle = server.build_shared_job_bundle( - artifacts, - worker(), - payout_artifact=artifact, - ) - self.assertEqual( - bundle.found_block["anchor_job_issued_at_ms"], - artifact.snapshot_anchor_ms, - ) - self.assertGreater(bundle.issued_at_ms, int(artifact.snapshot_anchor_ms)) - - def test_artifact_paths_proceed_at_the_clamped_anchor_floor( - self, - ) -> None: - # The pending-commit clamp is anchor selection, not a fence: every - # share stamped at or below the clamped anchor is already durable, - # so the window read at that anchor is exact and reproducible, and - # both artifact producers publish it. Shares stamped above the - # anchor deterministically belong to the next window. - ledger = AnchorRecordingLedger() - server, _rpc = coordinator(ledger=ledger) - install_fake_bundle_builder(server) - artifacts = server.current_template_artifacts() - stamped_ms = now_ms() - 5 - share = stamped_pending_share(stamped_ms) - self._hold_floor(server, share) - try: - artifact = server._build_payout_ledger_artifact( - 0, 0, artifacts.network_difficulty - ) - self.assertIsNotNone(artifact) - assert artifact is not None - self.assertEqual(artifact.snapshot_anchor_ms, stamped_ms - 1) - self.assertEqual(ledger.anchors[-1], stamped_ms - 1) - - # The synchronous build proceeds at the clamped anchor and seeds - # its window for reuse under that same anchor; cache publication - # installs the seed. - bundle = server.build_shared_job_bundle(artifacts, worker()) - self.assertEqual(bundle.issued_at_ms, stamped_ms - 1) - seeded = bundle.prepared_ledger_artifact - self.assertIsNotNone(seeded) - assert seeded is not None - self.assertEqual(seeded.snapshot_anchor_ms, stamped_ms - 1) - server._install_payout_ledger_artifact(seeded) - with server._job_cache_lock: - published = server._payout_ledger_artifact - self.assertIsNotNone(published) - assert published is not None - self.assertEqual(published.snapshot_anchor_ms, stamped_ms - 1) - finally: - server._finish_pending_share_commit(share) - - def test_background_build_refuses_a_pathologically_old_floor(self) -> None: - # A floor held further below now than the audit anchor ceiling (a - # wedged writer or leaked release) would arm an artifact that is - # born expired; refuse before paying the window walk so the - # re-arm backoff paces retries. - ledger = AnchorRecordingLedger() - server, _rpc = coordinator(ledger=ledger) - install_fake_bundle_builder(server) - artifacts = server.current_template_artifacts() - share = stamped_pending_share(now_ms() - 400_000) - self._hold_floor(server, share) - try: - self.assertIsNone( - server._build_payout_ledger_artifact( - 0, 0, artifacts.network_difficulty - ) - ) - self.assertEqual(ledger.snapshot_calls, 0) - finally: - server._finish_pending_share_commit(share) - - def test_anchor_selection_excludes_same_millisecond_stamps(self) -> None: - # Millisecond granularity can hand a share stamped right after - # anchor selection the same accepted_at_ms as the clamp instant, - # and such a share is never protected by the pending floor. The - # window predicate is anchor-inclusive, so the anchor must sit - # strictly below the clamp-time millisecond or that share would - # join audit replays of a window that never contained it. - server, _rpc = coordinator(ledger=AnchorRecordingLedger()) - clamp_now = now_ms() - self.assertEqual( - server._job_snapshot_anchor_ms(clamp_now), - clamp_now - 1, - ) - - share = stamped_pending_share(clamp_now - 7) - self._hold_floor(server, share) - try: - self.assertEqual( - server._job_snapshot_anchor_ms(clamp_now), - clamp_now - 8, - ) - finally: - server._finish_pending_share_commit(share) - - -class JobBundleCacheTests(unittest.TestCase): - def test_tip_template_snapshot_stays_coherent_across_tip_transition(self) -> None: - old_tip = "11" * 32 - new_tip = "22" * 32 - server, rpc = coordinator(template=base_template(prevhash=old_tip)) - new_template = base_template(height=11, prevhash=new_tip) - original_call = rpc.call - - def transition_during_template_fetch( - method: str, - params: list[object] | None = None, - ) -> object: - if method == "getblocktemplate": - rpc.tip = new_tip - rpc.template = new_template - return original_call(method, params) - - rpc.call = transition_during_template_fetch # type: ignore[method-assign] - - snapshot = server.fetch_qbit_tip_template_snapshot() - - self.assertEqual(snapshot.bestblockhash, new_tip) - self.assertEqual(snapshot.previousblockhash, new_tip) - self.assertEqual( - snapshot.template_fingerprint, - qbit_template_fingerprint(new_template), - ) - self.assertEqual(rpc.calls[:2], ["getblocktemplate", "getbestblockhash"]) - - def test_one_heavy_build_shared_across_clients_with_per_client_stamping(self) -> None: - server, rpc = coordinator() - recorded = install_fake_bundle_builder(server) - clients = [client(1), client(2), client(3)] - - contexts = [server.build_job_for_client(c, clean_jobs=True) for c in clients] - - self.assertEqual(recorded["calls"], 1) - self.assertEqual(rpc.count("getblocktemplate"), 1) - self.assertEqual(server.ledger.snapshot_calls, 1) - # The heavy build uses the placeholder extranonce1, never a client's. - self.assertEqual( - recorded["suffixes"], - [ - server.coinbase_tag_hex - + PRISM_JOB_EXTRANONCE1_PLACEHOLDER_HEX - + "00" * EXTRANONCE2_SIZE - ], - ) - job_ids = {context.job.job_id for context in contexts} - self.assertEqual(len(job_ids), 3) - self.assertEqual( - [context.job.extranonce1_hex for context in contexts], - [c.extranonce1_hex for c in clients], - ) - # coinb1/coinb2 exclude the extranonce window entirely, so the shared - # split is byte-identical for every client. - self.assertEqual(len({context.job.coinb1 for context in contexts}), 1) - self.assertEqual(len({context.job.coinb2 for context in contexts}), 1) - self.assertTrue(all(not hasattr(context, "bundle") for context in contexts)) - self.assertTrue( - all(context.prospective_prior_balances == () for context in contexts) - ) - cached = next(iter(server._job_bundle_cache.values())) - self.assertFalse(hasattr(cached, "bundle")) - self.assertEqual(cached.prospective_prior_balances, ()) - self.assertEqual( - cached.coinbase_manifest["coinbase_tx_hex"], - synthetic_manifest_coinbase_hex(recorded["suffixes"][0]), - ) - self.assertIs(contexts[0].shares_json, contexts[1].shares_json) - - def test_latest_wins_scheduler_preserves_synchronous_builder_output(self) -> None: - server, rpc = coordinator() - recorded = install_fake_bundle_builder(server) - artifacts = server.store_template_artifacts(dict(rpc.template)) - assert artifacts is not None - identity = worker() - cache_key = server._job_bundle_key( - artifacts, - mode="ready", - payout_state_generation=0, - worker=identity, - ) - - with patch("lab.prism.prism_coordinator.now_ms", return_value=1_700_000_001_000): - direct_request = server._new_job_build_request( - artifacts, - identity, - mode="ready", - payout_state_generation=0, - cache_key=cache_key, - ) - direct = server.build_shared_job_bundle( - artifacts, - identity, - mode="ready", - payout_state_generation=0, - key=cache_key, - build_request=direct_request, - ) - scheduled_request = server._new_job_build_request( - artifacts, - identity, - mode="ready", - payout_state_generation=0, - cache_key=cache_key, - ) - scheduled = server._request_job_build(scheduled_request).result(5) - - server.shutdown_job_build_executor() - self.assertEqual(recorded["calls"], 2) - self.assertEqual(scheduled.key, direct.key) - self.assertEqual(scheduled.template, direct.template) - self.assertEqual(scheduled.template_fingerprint, direct.template_fingerprint) - self.assertEqual(scheduled.coinbase_manifest, direct.coinbase_manifest) - self.assertEqual(scheduled.shares_json, direct.shares_json) - self.assertEqual(scheduled.prior_balances, direct.prior_balances) - self.assertEqual(scheduled.found_block, direct.found_block) - self.assertEqual(scheduled.collection_only, direct.collection_only) - self.assertEqual(scheduled.issued_at_ms, direct.issued_at_ms) - self.assertEqual(scheduled.base_job, direct.base_job) - self.assertEqual(scheduled.build_key, direct.build_key) - - def test_stamped_job_reassembles_coinbase_with_client_extranonce(self) -> None: - server, _ = coordinator() - install_fake_bundle_builder(server) - state = client(0x2A) - - context = server.build_job_for_client(state, clean_jobs=True) - extranonce2_hex = "11" * EXTRANONCE2_SIZE - submission = direct_stratum.assemble_submission( - context.job, - extranonce2_hex=extranonce2_hex, - ntime_hex="65000000", - nonce_hex="00000001", - ) - - expected_suffix = server.coinbase_tag_hex + state.extranonce1_hex + extranonce2_hex - coinbase = bytes.fromhex(submission.coinbase_tx_hex) - script_start, script_len = direct_stratum.coinbase_scriptsig_span( - coinbase, field_name="stamped coinbase" - ) - script_sig_hex = coinbase[script_start : script_start + script_len].hex() - self.assertTrue(script_sig_hex.endswith(expected_suffix)) - self.assertNotIn(PRISM_JOB_EXTRANONCE1_PLACEHOLDER_HEX, script_sig_hex[len("03aabbcc") :]) - - def test_template_fingerprint_change_invalidates_bundle_cache(self) -> None: - server, rpc = coordinator() - recorded = install_fake_bundle_builder(server) - server.build_job_for_client(client(1), clean_jobs=True) - - new_template = base_template(height=11, prevhash="22" * 32) - rpc.template = new_template - rpc.tip = str(new_template["previousblockhash"]) - server.store_template_artifacts(dict(new_template)) - - context = server.build_job_for_client(client(2), clean_jobs=True) - - self.assertEqual(recorded["calls"], 2) - self.assertEqual(context.template_fingerprint, qbit_template_fingerprint(new_template)) - # Bundles for the old fingerprint are evicted. - self.assertEqual( - {entry.template_fingerprint for entry in server._job_bundle_cache.values()}, - {qbit_template_fingerprint(new_template)}, - ) - - def test_bundle_cache_ttl_expiry_rebuilds(self) -> None: - server, _ = coordinator() - recorded = install_fake_bundle_builder(server) - server.job_bundle_cache_seconds = 0.05 - - server.build_job_for_client(client(1), clean_jobs=True) - time.sleep(0.06) - server.build_job_for_client(client(2), clean_jobs=True) - - self.assertEqual(recorded["calls"], 2) - - def test_bundle_cache_lookup_prunes_every_expired_snapshot(self) -> None: - server, _ = coordinator() - install_fake_bundle_builder(server) - artifacts = server.store_template_artifacts(base_template()) - assert artifacts is not None - current = server.shared_job_bundle(artifacts, worker()) - expired_key = ("expired-template", "ready") - expired = dataclass_replace( - current, - key=expired_key, - template_fingerprint="expired-template", - built_monotonic=time.monotonic() - 60, - ) - server._job_bundle_cache[expired_key] = expired - - looked_up = server._lookup_job_bundle(current.key) - - self.assertIs(looked_up, current) - self.assertNotIn(expired_key, server._job_bundle_cache) - self.assertEqual(list(server._job_bundle_cache.values()), [current]) - - def test_zero_ttl_disables_bundle_cache(self) -> None: - server, _ = coordinator() - recorded = install_fake_bundle_builder(server) - server.job_bundle_cache_seconds = 0.0 - - server.build_job_for_client(client(1), clean_jobs=True) - server.build_job_for_client(client(2), clean_jobs=True) - - self.assertEqual(recorded["calls"], 2) - - def test_payout_state_change_during_build_retries_before_cache_or_return(self) -> None: - server, rpc = coordinator() - recorded = install_fake_bundle_builder(server) - artifacts = server.store_template_artifacts(dict(rpc.template)) - self.assertIsNotNone(artifacts) - assert artifacts is not None - identity = worker() - original_build = server.build_shared_job_bundle - built_generations: list[int] = [] - - def mutate_after_first_build( - build_artifacts: object, - build_worker: WorkerIdentity | None, - **kwargs: object, - ) -> object: - bundle = original_build( # type: ignore[arg-type] - build_artifacts, - build_worker, - **kwargs, - ) - built_generations.append(bundle.payout_state_generation) - if len(built_generations) == 1: - server._advance_payout_state_generation() - return bundle - - server.build_shared_job_bundle = mutate_after_first_build # type: ignore[method-assign] - - bundle = server.shared_job_bundle(artifacts, identity) - cached = server.shared_job_bundle(artifacts, identity) - - self.assertEqual(built_generations, [0, 1]) - self.assertEqual(recorded["calls"], 2) - self.assertEqual(bundle.payout_state_generation, 1) - self.assertIs(cached, bundle) - - def test_escaped_stale_bundle_is_rejected_before_direct_delivery(self) -> None: - server, _rpc = coordinator() - install_fake_bundle_builder(server) - server._ensure_tip_refresh_state() - state = client(1) - sent: list[dict[str, object]] = [] - state.send = sent.append # type: ignore[method-assign] - original_shared_job_bundle = server.shared_job_bundle - advanced = False - - def advance_after_bundle(*args: object, **kwargs: object) -> object: - nonlocal advanced - bundle = original_shared_job_bundle(*args, **kwargs) # type: ignore[arg-type] - if not advanced: - advanced = True - server._advance_payout_state_generation() - return bundle - - server.shared_job_bundle = advance_after_bundle # type: ignore[method-assign] - - self.assertFalse(server.maybe_send_job(state, clean_jobs=True)) - self.assertEqual(sent, []) - self.assertIsNone(state.active_job) - self.assertEqual(server._payout_state_generation, 1) - self.assertTrue(server._tip_refresh_retry.is_set()) - - self.assertTrue(server.maybe_send_job(state, clean_jobs=True)) - self.assertIsNotNone(state.active_job) - self.assertEqual(state.active_job.payout_state_generation, 1) - self.assertEqual( - [payload["method"] for payload in sent], - ["mining.set_difficulty", "mining.notify"], - ) - - def test_priority_decision_uses_one_publication_snapshot(self) -> None: - server, _rpc = coordinator() - state = client(1) - context = SimpleNamespace( - payout_state_generation=0, - template={"previousblockhash": "11" * 32}, - ) - server.ensure_reorg_reconciled_for_current_tip = ( # type: ignore[method-assign] - lambda **_kwargs: True - ) - server.build_job_for_client = ( # type: ignore[method-assign] - lambda *_args, **_kwargs: context - ) - original_lock = server._job_cache_lock - - class PublishAfterPrioritySnapshot: - advanced = False - - def __enter__(self) -> object: - original_lock.acquire() - return self - - def __exit__( - self, - _exc_type: object, - _exc: object, - _traceback: object, - ) -> None: - original_lock.release() - if not self.advanced: - self.advanced = True - server._payout_state_generation = 1 - - priorities: list[bool] = [] - - class RecordingGate: - @contextmanager - def delivery_cancelable( - self, - _cancelled: object, - *, - priority: bool, - **_kwargs: object, - ) -> object: - priorities.append(priority) - yield False - - server._job_cache_lock = PublishAfterPrioritySnapshot() # type: ignore[assignment] - server._payout_state_delivery_gate = RecordingGate() # type: ignore[assignment] - - self.assertFalse(server.maybe_send_job(state, clean_jobs=True)) - self.assertEqual(priorities, [True]) - self.assertEqual(server._payout_state_generation, 1) - - def test_zero_template_ttl_fetches_template_per_build(self) -> None: - server, rpc = coordinator() - install_fake_bundle_builder(server) - server.template_cache_seconds = 0.0 - - server.build_job_for_client(client(1), clean_jobs=True) - server.build_job_for_client(client(2), clean_jobs=True) - - self.assertEqual(rpc.count("getblocktemplate"), 2) - - def test_late_stale_template_fetch_cannot_replace_newer_artifacts(self) -> None: - server, rpc = coordinator() - server.template_cache_seconds = 0.0 - stale_template = dict(rpc.template) - current_template = base_template(height=11, prevhash="22" * 32) - fetch_started = threading.Event() - release_fetch = threading.Event() - results: list[object] = [] - errors: list[BaseException] = [] - original_call = rpc.call - thread: threading.Thread - - def blocking_call( - method: str, - params: list[object] | None = None, - ) -> object: - if method == "getblocktemplate" and threading.current_thread() is thread: - fetch_started.set() - if not release_fetch.wait(5): - raise AssertionError("stale template fetch was not released") - return dict(stale_template) - return original_call(method, params) - - def fetch_stale_artifacts() -> None: - try: - results.append(server.current_template_artifacts()) - except BaseException as exc: # noqa: BLE001 - surface to the test - errors.append(exc) - - rpc.call = blocking_call # type: ignore[method-assign] - thread = threading.Thread(target=fetch_stale_artifacts) - thread.start() - try: - self.assertTrue(fetch_started.wait(5)) - current_artifacts = server.store_template_artifacts(current_template) - self.assertIsNotNone(current_artifacts) - assert current_artifacts is not None - self.assertGreater(current_artifacts.generation, 1) - finally: - release_fetch.set() - thread.join(5) - - self.assertFalse(thread.is_alive()) - self.assertEqual(errors, []) - self.assertEqual(results, [current_artifacts]) - self.assertIs(server._template_artifacts, current_artifacts) - self.assertEqual( - current_artifacts.fingerprint, - qbit_template_fingerprint(current_template), - ) - - def test_collection_mode_bundles_are_keyed_per_worker(self) -> None: - server, _ = coordinator(ledger=FakeLedger(miners=["solo"])) - recorded = install_fake_bundle_builder(server) - server.min_ready_miners = 3 - - worker_a = worker(payout="tq1worker-a") - worker_b = worker(payout="tq1worker-b") - context_a1 = server.build_job_for_client(client(1, worker_a), clean_jobs=True) - context_a2 = server.build_job_for_client(client(2, worker_a), clean_jobs=True) - context_b = server.build_job_for_client(client(3, worker_b), clean_jobs=True) - - self.assertTrue(context_a1.collection_only) - self.assertTrue(context_b.collection_only) - self.assertEqual(recorded["calls"], 2) - self.assertTrue( - all(not hasattr(context, "bundle") for context in (context_a1, context_a2, context_b)) - ) - self.assertIs(context_a1.shares_json, context_a2.shares_json) - self.assertIsNot(context_a1.shares_json, context_b.shares_json) - - def test_collection_bundle_cache_rebuilds_when_pool_becomes_ready(self) -> None: - ledger = FakeLedger(miners=["solo"]) - server, _ = coordinator(ledger=ledger) - recorded = install_fake_bundle_builder(server) - state = client(1) - - collection_context = server.build_job_for_client(state, clean_jobs=True) - ledger.miners = ["miner-a", "miner-b", "miner-c"] - ready_context = server.build_job_for_client(state, clean_jobs=True) - - self.assertTrue(collection_context.collection_only) - self.assertFalse(ready_context.collection_only) - self.assertEqual(recorded["calls"], 2) - self.assertEqual(ledger.snapshot_calls, 1) - - def test_ready_empty_snapshot_does_not_fall_back_to_worker_collection(self) -> None: - ledger = ReadyLedgerWithEmptyFirstSnapshot() - server, _ = coordinator(ledger=ledger) - recorded = install_fake_bundle_builder(server) - state = client(1) - - with self.assertRaisesRegex( - RuntimeError, - "ready-pool ledger snapshot contained no payout shares", - ): - server.build_job_for_client(state, clean_jobs=True) - ready_context = server.build_job_for_client(state, clean_jobs=True) - - self.assertFalse(ready_context.collection_only) - self.assertEqual(recorded["calls"], 1) - self.assertEqual(ledger.snapshot_calls, 2) - - def test_vardiff_difficulty_is_stamped_per_client(self) -> None: - server, _ = coordinator() - install_fake_bundle_builder(server) - easy = client(1) - hard = client(2) - hard.pending_share_difficulty = Decimal("512") - - easy_context = server.build_job_for_client(easy, clean_jobs=True) - hard_context = server.build_job_for_client(hard, clean_jobs=True) - - self.assertEqual(easy_context.job.coinb1, hard_context.job.coinb1) - self.assertGreater(easy_context.job.share_target, hard_context.job.share_target) - self.assertEqual(hard_context.job.share_difficulty, Decimal("512")) - - def test_template_artifacts_reuse_derivations_when_fingerprint_unchanged(self) -> None: - server, _ = coordinator() - template = base_template() - first = server.store_template_artifacts(dict(template)) - refreshed_template = dict(template) - refreshed_template["curtime"] = int(template["curtime"]) + 30 - second = server.store_template_artifacts(refreshed_template) - - self.assertIsNotNone(first) - self.assertIsNotNone(second) - assert first is not None and second is not None - self.assertEqual(first.fingerprint, second.fingerprint) - self.assertIs(first.transaction_hexes, second.transaction_hexes) - self.assertIs(first.witness_merkle_leaves_hex, second.witness_merkle_leaves_hex) - - def test_poll_seeds_template_cache_for_client_builds(self) -> None: - server, rpc = coordinator() - install_fake_bundle_builder(server) - - refreshed = server.poll_qbit_tip_template_once() - self.assertEqual(refreshed, 0) - self.assertEqual(rpc.count("getblocktemplate"), 1) - - server.build_job_for_client(client(1), clean_jobs=True) - self.assertEqual(rpc.count("getblocktemplate"), 1) - - def test_reorg_reconciliation_cached_per_tip(self) -> None: - server, rpc = coordinator() - server.reorg_reconciler_enabled = True - reconcile_calls: list[str | None] = [] - - def fake_reconcile(*, tip_hash: str | None = None) -> dict[str, object]: - reconcile_calls.append(tip_hash) - server._note_reorg_reconcile_outcome(tip_hash, trusted=True) - return {"untrusted": False} - - server.reconcile_prism_pool_blocks_once = fake_reconcile # type: ignore[method-assign] - - self.assertTrue(server.ensure_reorg_reconciled_for_current_tip()) - self.assertTrue(server.ensure_reorg_reconciled_for_current_tip()) - self.assertEqual(len(reconcile_calls), 1) - - rpc.tip = "33" * 32 - self.assertTrue(server.ensure_reorg_reconciled_for_current_tip()) - self.assertEqual(len(reconcile_calls), 2) - self.assertEqual(reconcile_calls[-1], "33" * 32) - - def test_reorg_cache_rechecks_chain_view_before_reuse(self) -> None: - server, rpc = coordinator() - server.reorg_reconciler_enabled = True - rpc.blockchain_info["headers"] = 101 - server._note_reorg_reconcile_outcome(rpc.tip, trusted=True) - reconcile_calls: list[str | None] = [] - - def fake_reconcile(*, tip_hash: str | None = None) -> dict[str, object]: - reconcile_calls.append(tip_hash) - server._note_reorg_reconcile_outcome(tip_hash, trusted=False) - return {"untrusted": True} - - server.reconcile_prism_pool_blocks_once = fake_reconcile # type: ignore[method-assign] - - self.assertFalse(server.ensure_reorg_reconciled_for_current_tip()) - self.assertEqual(rpc.count("getblockchaininfo"), 1) - self.assertEqual(reconcile_calls, [rpc.tip]) - self.assertFalse(server.last_reorg_reconciled_trusted) - - def test_single_flight_builds_once_under_concurrency(self) -> None: - server, _ = coordinator() - recorded = install_fake_bundle_builder(server) - original_builder = server.build_audit_bundle - build_started = threading.Event() - - def slow_builder(**kwargs: object) -> dict[str, object]: - build_started.set() - time.sleep(0.05) - return original_builder(**kwargs) - - server.build_audit_bundle = slow_builder # type: ignore[method-assign] - errors: list[BaseException] = [] - - def build(connection_id: int) -> None: - try: - server.build_job_for_client(client(connection_id), clean_jobs=True) - except BaseException as exc: # noqa: BLE001 - surface to the test - errors.append(exc) - - threads = [threading.Thread(target=build, args=(index + 1,)) for index in range(6)] - for thread in threads: - thread.start() - for thread in threads: - thread.join() - - self.assertEqual(errors, []) - self.assertEqual(recorded["calls"], 1) - - def test_observed_tip_change_rejects_stale_cached_bundle_delivery(self) -> None: - old_tip = "11" * 32 - new_tip = "22" * 32 - server, rpc = coordinator(template=base_template(prevhash=old_tip)) - install_fake_bundle_builder(server) - state = client(1) - sent: list[dict[str, object]] = [] - state.send = sent.append # type: ignore[method-assign] - server.clients = {state} - server.observe_tip_first_seen(old_tip, observation_sequence=1) - - self.assertTrue(server.maybe_send_job(state, clean_jobs=True)) - self.assertEqual(len(sent), 2) - sent.clear() - - rpc.tip = new_tip - server.observe_tip_first_seen(new_tip, observation_sequence=2) - - self.assertFalse(server.maybe_send_job(state, clean_jobs=True)) - self.assertEqual(sent, []) - - def test_ready_ledger_snapshot_holds_payout_mutation_lock(self) -> None: - server, rpc = coordinator() - install_fake_bundle_builder(server) - artifacts = server.store_template_artifacts(dict(rpc.template)) - assert artifacts is not None - entered_snapshot = threading.Event() - release_snapshot = threading.Event() - original_snapshot = server.ledger.snapshot_at_job_issue - - def blocked_snapshot(*args: object, **kwargs: object) -> object: - entered_snapshot.set() - self.assertTrue(release_snapshot.wait(2)) - return original_snapshot(*args, **kwargs) - - server.ledger.snapshot_at_job_issue = blocked_snapshot # type: ignore[method-assign] - errors: list[BaseException] = [] - build_thread = threading.Thread( - target=lambda: self._capture_error( - errors, - lambda: server.shared_job_bundle(artifacts, mode="ready"), - ) - ) - build_thread.start() - try: - self.assertTrue(entered_snapshot.wait(2)) - mutation_acquired = server._payout_state_prepare_lock.acquire( - blocking=False - ) - if mutation_acquired: - server._payout_state_prepare_lock.release() - self.assertFalse(mutation_acquired) - finally: - release_snapshot.set() - build_thread.join(2) - - self.assertFalse(build_thread.is_alive()) - self.assertEqual(errors, []) - - def test_ready_build_identity_separates_clock_only_generations(self) -> None: - server, rpc = coordinator() - first = server.store_template_artifacts(dict(rpc.template)) - second_template = dict(rpc.template) - second_template["curtime"] = int(second_template["curtime"]) + 1 - second = server.store_template_artifacts(second_template) - assert first is not None and second is not None - self.assertEqual(first.fingerprint, second.fingerprint) - self.assertNotEqual(first.generation, second.generation) - payout_generation = server._payout_state_generation - - with patch( - "lab.prism.prism_coordinator.now_ms", - side_effect=[1_700_000_000_000, 1_700_000_001_000], - ): - first_request = server._new_job_build_request( - first, - None, - mode="ready", - payout_state_generation=payout_generation, - cache_key=server._job_bundle_key( - first, - mode="ready", - payout_state_generation=payout_generation, - worker=None, - ), - ) - second_request = server._new_job_build_request( - second, - None, - mode="ready", - payout_state_generation=payout_generation, - cache_key=server._job_bundle_key( - second, - mode="ready", - payout_state_generation=payout_generation, - worker=None, - ), - ) - - self.assertNotEqual( - first_request.equivalence_key, - second_request.equivalence_key, - ) - self.assertNotEqual( - first_request.key.issued_at_ms, - second_request.key.issued_at_ms, - ) - - def test_precomputed_payout_artifact_matches_inline_output_exactly(self) -> None: - server, rpc = coordinator() - install_fake_bundle_builder(server) - artifacts = server.store_template_artifacts(dict(rpc.template)) - assert artifacts is not None - - with patch("lab.prism.prism_coordinator.now_ms", return_value=1_700_000_123_000): - inline = server.shared_job_bundle(artifacts, mode="ready") - with server._job_cache_lock: - server._job_bundle_cache.clear() - server._prepare_payout_ledger_artifact( - server._payout_state_generation, - artifacts.network_difficulty, - ) - prepared = server.shared_job_bundle(artifacts, mode="ready") - - self.assertEqual(prepared.base_job, inline.base_job) - self.assertEqual(prepared.coinbase_manifest, inline.coinbase_manifest) - self.assertEqual(prepared.shares_json, inline.shares_json) - self.assertEqual(prepared.prior_balances, inline.prior_balances) - self.assertEqual(prepared.found_block, inline.found_block) - self.assertGreater(prepared.payout_artifact_generation, 0) - - def test_accepted_preview_patches_artifact_across_normal_clear(self) -> None: - class CountingBalanceLedger(FakeLedger): - def __init__(self) -> None: - super().__init__() - self.prior_balance_reads = 0 - self.database_balances = [ - { - "recipient_id": "stale-miner", - "order_key": "stale-miner", - "p2mr_program_hex": "22" * 32, - "balance_sats": 1, - } - ] - - def current_prior_balances(self) -> list[dict[str, object]]: - self.prior_balance_reads += 1 - return [dict(balance) for balance in self.database_balances] - - ledger = CountingBalanceLedger() - server, rpc = coordinator(ledger=ledger) - recorded = install_fake_bundle_builder(server) - artifacts = server.store_template_artifacts(dict(rpc.template)) - assert artifacts is not None - server._pool_ready_latched = True - parent_hash = str(rpc.template["previousblockhash"]) - parent_height = int(rpc.template["height"]) - 1 - preview = [ - { - "recipient_id": "miner-a", - "order_key": "miner-a", - "p2mr_program_hex": "11" * 32, - "balance_sats": 25, - } - ] - - server._begin_accepted_block_payout_preview( - parent_hash, - block_height=parent_height, - ) - server._publish_accepted_block_payout_preview(parent_hash, preview) - - artifact = server._payout_ledger_artifact - self.assertIsNotNone(artifact) - assert artifact is not None - self.assertEqual(artifact.prior_balances, tuple(preview)) - self.assertEqual( - artifact.payout_state_generation, - server._payout_state_generation, - ) - self.assertGreater(artifact.generation, 0) - - ledger.prior_balance_reads = 0 - preview_bundle = server.shared_job_bundle(artifacts, mode="ready") - - self.assertEqual(preview_bundle.prior_balances, preview) - self.assertEqual(recorded["last_kwargs"]["prior_balances"], preview) # type: ignore[index] - self.assertEqual( - preview_bundle.payout_artifact_generation, - artifact.generation, - ) - self.assertEqual(ledger.prior_balance_reads, 0) - - payout_generation = server._payout_state_generation - ledger.database_balances = [dict(balance) for balance in preview] - server._clear_accepted_block_payout_preview(parent_hash) - self.assertEqual(server._payout_state_generation, payout_generation) - self.assertIs(server._payout_ledger_artifact, artifact) - self.assertNotIn(parent_hash, server._accepted_block_payout_previews) - self.assertNotIn( - parent_hash, - server._invalidated_accepted_block_payout_previews, - ) - with server._job_cache_lock: - server._job_bundle_cache.clear() - - post_clear_bundle = server.shared_job_bundle(artifacts, mode="ready") - - self.assertEqual(post_clear_bundle.prior_balances, preview) - self.assertEqual(recorded["last_kwargs"]["prior_balances"], preview) # type: ignore[index] - self.assertEqual( - post_clear_bundle.payout_artifact_generation, - artifact.generation, - ) - self.assertEqual(ledger.prior_balance_reads, 0) - - def test_valid_precomputed_artifact_skips_tip_path_ledger_snapshot(self) -> None: - server, rpc = coordinator() - install_fake_bundle_builder(server) - snapshot = server.fetch_qbit_tip_template_snapshot() - assert snapshot.template_artifacts is not None - server._prepare_payout_ledger_artifact( - server._payout_state_generation, - snapshot.template_artifacts.network_difficulty, - ) - server.ledger.snapshot_calls = 0 - - bundle = server.prepare_tip_refresh_bundle(snapshot) - - self.assertFalse(bundle.collection_only) - self.assertGreater(bundle.payout_artifact_generation, 0) - self.assertEqual(server.ledger.snapshot_calls, 0) - - def test_mismatched_precomputed_artifact_falls_back_to_inline_snapshot(self) -> None: - server, rpc = coordinator() - install_fake_bundle_builder(server) - artifacts = server.store_template_artifacts(dict(rpc.template)) - assert artifacts is not None - server._current_payout_state_artifact() - server._prepare_payout_ledger_artifact( - server._payout_state_generation, - artifacts.network_difficulty, - ) - with server._job_cache_lock: - published = server._published_payout_state - assert published.artifact is not None - changed_balances = [{"miner_id": "miner-z", "balance_sats": 1}] - server._published_payout_state = dataclass_replace( - published, - artifact=dataclass_replace( - published.artifact, - prior_balances_json=canonical_json_text(changed_balances), - prior_balances_sha256=canonical_json_sha256(changed_balances), - ), - ) - - self.assertIsNone( - server._usable_payout_ledger_artifact( - server._payout_state_generation, - artifacts.network_difficulty, - ) - ) - self.assertIsNone(server._payout_ledger_artifact) - server.ledger.snapshot_calls = 0 - - bundle = server.shared_job_bundle(artifacts, mode="ready") - - self.assertEqual(bundle.payout_artifact_generation, 0) - self.assertEqual(server.ledger.snapshot_calls, 1) - - def test_new_tip_cancels_blocked_old_bundle_without_publication(self) -> None: - old_tip = "11" * 32 - new_tip = "22" * 32 - server, rpc = coordinator(template=base_template(prevhash=old_tip)) - recorded = install_fake_bundle_builder(server) - original_builder = server.build_audit_bundle - build_started = threading.Event() - - def cancelable_builder(**kwargs: object) -> dict[str, object]: - control = server._job_build_phase_local.bundle_build_control - build_started.set() - self.assertTrue(control.cancel_event.wait(2)) - return original_builder(**kwargs) - - server.build_audit_bundle = cancelable_builder # type: ignore[method-assign] - artifacts = server.store_template_artifacts(dict(rpc.template)) - assert artifacts is not None - server.observe_tip_first_seen(old_tip, observation_sequence=1) - errors: list[BaseException] = [] - thread = threading.Thread( - target=lambda: self._capture_error( - errors, - lambda: server.shared_job_bundle(artifacts, mode="ready"), - ) - ) - thread.start() - self.assertTrue(build_started.wait(2)) - - server.observe_tip_first_seen(new_tip, observation_sequence=2) - thread.join(2) - - self.assertFalse(thread.is_alive()) - self.assertEqual(len(errors), 1) - self.assertIsInstance(errors[0], TemplateRefreshBlocked) - self.assertEqual(recorded["calls"], 1) - self.assertEqual(server._active_job_bundle_builds, {}) - self.assertEqual(server.tip_refresh_build_inflight, 0) - self.assertFalse(any( - entry.template_fingerprint == artifacts.fingerprint - for entry in server._job_bundle_cache.values() - )) - self.assertEqual(server.tip_refresh_superseded_results, 1) - - def test_builder_crash_and_timeout_fail_closed_then_recover(self) -> None: - server, _rpc = coordinator() - server.prism_ctv_settlement_config = lambda **_kwargs: None # type: ignore[method-assign] - server.signing_seed_hex = "42" * 32 - server.ledger_attestation_signing_seed_hex = "43" * 32 - build_kwargs = { - "shares": [], - "found_block": { - "block_height": 10, - "coinbase_value_sats": 50_00000000, - "network_difficulty": 1, - "anchor_job_issued_at_ms": 1_700_000_000_000, - }, - "prior_balances": [], - "coinbase_script_sig_suffix_hex": "00", - } - - with patch( - "lab.prism.prism_coordinator.prism_tool_command", - return_value=[sys.executable, "-c", "raise SystemExit(7)"], - ): - with self.assertRaisesRegex(RuntimeError, "failed"): - server.build_audit_bundle(**build_kwargs) - - server.bundle_build_timeout_seconds = 0.01 - with patch( - "lab.prism.prism_coordinator.prism_tool_command", - return_value=[ - sys.executable, - "-c", - "import time; time.sleep(5)", - ], - ): - with self.assertRaisesRegex(RuntimeError, "timed out"): - server.build_audit_bundle(**build_kwargs) - - server.bundle_build_timeout_seconds = 1.0 - recovery_script = ( - "import json,sys; json.load(sys.stdin); " - "json.dump({'recovered': True}, sys.stdout)" - ) - with patch( - "lab.prism.prism_coordinator.prism_tool_command", - return_value=[sys.executable, "-c", recovery_script], - ): - recovered = server.build_audit_bundle(**build_kwargs) - self.assertEqual(recovered, {"recovered": True}) - - def test_audit_builder_child_does_not_inherit_open_socket(self) -> None: - server, _rpc = coordinator() - server.prism_ctv_settlement_config = lambda **_kwargs: None # type: ignore[method-assign] - server.signing_seed_hex = "42" * 32 - server.ledger_attestation_signing_seed_hex = "43" * 32 - probe_script = ( - "import json,os,sys; json.load(sys.stdin); fd=int(sys.argv[1]); " - "inherited=True; " - "\ntry: os.fstat(fd)" - "\nexcept OSError: inherited=False" - "\njson.dump({'inherited_socket': inherited}, sys.stdout)" - ) - with socket.socket() as parent_socket: - parent_socket.set_inheritable(True) - with patch( - "lab.prism.prism_coordinator.prism_tool_command", - return_value=[ - sys.executable, - "-c", - probe_script, - str(parent_socket.fileno()), - ], - ): - result = server.build_audit_bundle( - shares=[], - found_block={ - "block_height": 10, - "coinbase_value_sats": 50_00000000, - "network_difficulty": 1, - "anchor_job_issued_at_ms": 1_700_000_000_000, - }, - prior_balances=[], - coinbase_script_sig_suffix_hex="00", - ) - self.assertEqual(result, {"inherited_socket": False}) - - def test_repeated_superseded_builds_leave_state_bounded(self) -> None: - server, rpc = coordinator() - install_fake_bundle_builder(server) - original_builder = server.build_audit_bundle - starts: queue.Queue[None] = queue.Queue() - - def cancelable_builder(**kwargs: object) -> dict[str, object]: - control = server._job_build_phase_local.bundle_build_control - starts.put(None) - self.assertTrue(control.cancel_event.wait(2)) - return original_builder(**kwargs) - - server.build_audit_bundle = cancelable_builder # type: ignore[method-assign] - current_tip = str(rpc.tip) - server.observe_tip_first_seen(current_tip, observation_sequence=1) - errors: list[BaseException] = [] - for index in range(8): - rpc.template = base_template(height=10 + index, prevhash=current_tip) - artifacts = server.store_template_artifacts(dict(rpc.template)) - assert artifacts is not None - thread = threading.Thread( - target=lambda current=artifacts: self._capture_error( - errors, - lambda: server.shared_job_bundle(current, mode="ready"), - ) - ) - thread.start() - starts.get(timeout=2) - current_tip = f"{index + 2:064x}" - rpc.tip = current_tip - server.observe_tip_first_seen( - current_tip, - observation_sequence=index + 2, - ) - thread.join(2) - self.assertFalse(thread.is_alive()) - - self.assertEqual(len(errors), 8) - self.assertTrue(all(isinstance(exc, TemplateRefreshBlocked) for exc in errors)) - self.assertEqual(server._active_job_bundle_builds, {}) - self.assertEqual(server.tip_refresh_build_inflight, 0) - self.assertEqual(server.tip_refresh_build_queue_depth, 0) - self.assertLessEqual( - len(server._job_bundle_cache), - MAX_PRISM_JOB_BUNDLE_CACHE_ENTRIES, - ) - self.assertEqual(server.tip_refresh_superseded_results, 8) - - @staticmethod - def _capture_error( - errors: list[BaseException], - operation: object, - ) -> None: - try: - operation() # type: ignore[operator] - except BaseException as exc: # noqa: BLE001 - test thread handoff - errors.append(exc) - - def test_same_fingerprint_bundle_rebinds_exact_template_observation(self) -> None: - server, _ = coordinator() - install_fake_bundle_builder(server) - identity = worker() - first = server.store_template_artifacts(base_template()) - assert first is not None - original = server.shared_job_bundle(first, identity) - updated_template = dict(first.template) - updated_template["curtime"] = int(updated_template["curtime"]) + 30 - second = server.store_template_artifacts(updated_template) - assert second is not None - - rebound = server.shared_job_bundle(second, identity) - - self.assertEqual(first.fingerprint, second.fingerprint) - self.assertIs(rebound.template, second.template) - self.assertIsNot(rebound.template, original.template) - self.assertEqual(rebound.template_generation, second.generation) - self.assertEqual(rebound.base_job.ntime, f'{updated_template["curtime"]:08x}') - - def test_clock_only_refresh_does_not_discard_inflight_ready_build(self) -> None: - server, rpc = coordinator() - recorded = install_fake_bundle_builder(server) - first = server.store_template_artifacts(dict(rpc.template)) - assert first is not None - build_entered = threading.Event() - release_build = threading.Event() - original_build = server.build_shared_job_bundle - build_calls = 0 - build_calls_lock = threading.Lock() - - def blocking_build(*args: object, **kwargs: object) -> object: - nonlocal build_calls - with build_calls_lock: - build_calls += 1 - build_entered.set() - self.assertTrue(release_build.wait(5)) - return original_build(*args, **kwargs) # type: ignore[arg-type] - - server.build_shared_job_bundle = blocking_build # type: ignore[method-assign] - results: list[list[object]] = [[], []] - errors: list[list[BaseException]] = [[], []] - - def build(index: int, build_artifacts: object) -> None: - try: - results[index].append( - server.shared_job_bundle( # type: ignore[arg-type] - build_artifacts, - mode="ready", - ) - ) - except BaseException as exc: # noqa: BLE001 - surfaced below - errors[index].append(exc) - - first_thread = threading.Thread(target=build, args=(0, first)) - second_thread: threading.Thread | None = None - first_thread.start() - try: - self.assertTrue(build_entered.wait(5)) - updated_template = dict(first.template) - updated_template["curtime"] = int(updated_template["curtime"]) + 30 - second = server.store_template_artifacts(updated_template) - assert second is not None - second_thread = threading.Thread(target=build, args=(1, second)) - second_thread.start() - time.sleep(0.05) - with build_calls_lock: - self.assertEqual(build_calls, 1) - finally: - release_build.set() - first_thread.join(5) - if second_thread is not None: - second_thread.join(5) - - self.assertFalse(first_thread.is_alive()) - assert second_thread is not None - self.assertFalse(second_thread.is_alive()) - self.assertEqual(errors, [[], []]) - self.assertEqual([len(items) for items in results], [1, 1]) - built = results[0][0] - self.assertIs(built.template, first.template) # type: ignore[union-attr] - rebound = results[1][0] - self.assertIs(rebound.template, second.template) - self.assertEqual(rebound.template_generation, second.generation) - self.assertEqual(recorded["calls"], 1) - - def test_same_fingerprint_collection_bundle_rebuilds_exact_observation(self) -> None: - server, _ = coordinator(ledger=FakeLedger(miners=["miner-a"])) - recorded = install_fake_bundle_builder(server) - identity = worker() - first = server.store_template_artifacts(base_template()) - assert first is not None - original = server.shared_job_bundle(first, identity) - updated_template = dict(first.template) - updated_template["curtime"] = int(updated_template["curtime"]) + 30 - second = server.store_template_artifacts(updated_template) - assert second is not None - - rebuilt = server.shared_job_bundle(second, identity) - - self.assertTrue(original.collection_only) - self.assertTrue(rebuilt.collection_only) - self.assertEqual(first.fingerprint, second.fingerprint) - self.assertEqual(recorded["calls"], 2) - self.assertIsNot(rebuilt.coinbase_manifest, original.coinbase_manifest) - self.assertIs(rebuilt.template, second.template) - self.assertEqual(rebuilt.template_generation, second.generation) - - def test_job_bundle_cache_is_bounded(self) -> None: - server, _ = coordinator() - install_fake_bundle_builder(server) - artifacts = server.store_template_artifacts(base_template()) - assert artifacts is not None - bundle = server.shared_job_bundle(artifacts, worker()) - - for index in range(MAX_PRISM_JOB_BUNDLE_CACHE_ENTRIES + 5): - candidate = dataclass_replace( - bundle, - key=(artifacts.fingerprint, "test", index), - ) - server._cache_job_bundle_if_current(candidate, artifacts) - - self.assertEqual( - len(server._job_bundle_cache), - MAX_PRISM_JOB_BUNDLE_CACHE_ENTRIES, - ) - self.assertNotIn( - (artifacts.fingerprint, "test", 0), - server._job_bundle_cache, - ) - - def test_job_bundle_cache_preserves_coordinator_lock_order(self) -> None: - server, _ = coordinator() - install_fake_bundle_builder(server) - artifacts = server.store_template_artifacts(base_template()) - assert artifacts is not None - bundle = server.shared_job_bundle(artifacts, mode="ready") - observed_cache_lock = ObservedRLock() - server._job_cache_lock = observed_cache_lock # type: ignore[assignment] - errors: list[BaseException] = [] - - def cache_bundle() -> None: - try: - server._cache_job_bundle_if_current(bundle, artifacts) - except BaseException as exc: # noqa: BLE001 - surfaced below - errors.append(exc) - - observed_cache_lock.acquire() - observed_cache_lock.observe_acquires = True - cache_thread = threading.Thread(target=cache_bundle) - coordinator_lock_acquired = False - try: - cache_thread.start() - self.assertTrue(observed_cache_lock.acquire_attempted.wait(5)) - coordinator_lock_acquired = server.lock.acquire(timeout=0.25) - if coordinator_lock_acquired: - server.lock.release() - finally: - observed_cache_lock.release() - cache_thread.join(5) - - self.assertTrue(coordinator_lock_acquired) - self.assertFalse(cache_thread.is_alive()) - self.assertEqual(errors, []) - - def test_active_gap_replaces_older_pending_job_build(self) -> None: - server, rpc = coordinator() - first = server.store_template_artifacts(dict(rpc.template)) - second_template = dict(rpc.template) - second_template["coinbasevalue"] = int(second_template["coinbasevalue"]) + 1 - second = server.store_template_artifacts(second_template) - assert first is not None and second is not None - payout_generation = server._payout_state_generation - - def request_for(artifacts: object) -> object: - return server._new_job_build_request( - artifacts, # type: ignore[arg-type] - None, - mode="ready", - payout_state_generation=payout_generation, - cache_key=server._job_bundle_key( - artifacts, # type: ignore[arg-type] - mode="ready", - payout_state_generation=payout_generation, - worker=None, - ), - ) - - pending = request_for(first) - newest = request_for(second) - server._job_build_active = None - server._job_build_retiring = SimpleNamespace(request=pending) - server._job_build_pending = pending - server._start_job_build_locked = ( # type: ignore[method-assign] - lambda request: SimpleNamespace(request=request, future=None) - ) - server._arm_job_build_locked = lambda _flight: None # type: ignore[method-assign] - - promise = server._request_job_build(newest) # type: ignore[arg-type] - - self.assertIs(promise, newest.promise) # type: ignore[union-attr] - self.assertIsNone(server._job_build_pending) - assert server._job_build_active is not None - self.assertIs(server._job_build_active.request, newest) - self.assertTrue(pending.promise.done()) # type: ignore[union-attr] - self.assertIsInstance( - pending.promise.exception(), # type: ignore[union-attr] - JobBuildSuperseded, - ) - - def test_publication_critical_build_cannot_be_displaced_by_initial_work( - self, - ) -> None: - server, rpc = coordinator() - old_artifacts = server.store_template_artifacts(dict(rpc.template)) - new_artifacts = server.store_template_artifacts( - base_template(height=11, prevhash="22" * 32) - ) - assert old_artifacts is not None and new_artifacts is not None - payout_generation = server._payout_state_generation - - def request_for( - artifacts: object, - *, - publication_critical: bool, - request_source: str, - ) -> object: - return server._new_job_build_request( - artifacts, # type: ignore[arg-type] - None, - mode="ready", - payout_state_generation=payout_generation, - cache_key=server._job_bundle_key( - artifacts, # type: ignore[arg-type] - mode="ready", - payout_state_generation=payout_generation, - worker=None, - ), - publication_critical=publication_critical, - request_source=request_source, - ) - - latest = request_for( - new_artifacts, - publication_critical=True, - request_source="tip_refresh", - ) - reconnect = request_for( - old_artifacts, - publication_critical=False, - request_source="initial", - ) - same_tip_reconnect = request_for( - new_artifacts, - publication_critical=False, - request_source="initial", - ) - latest_flight = SimpleNamespace(request=latest) - server._job_build_active = latest_flight - server._job_build_retiring = None - server._job_build_pending = None - - deferred = server._request_job_build(reconnect) # type: ignore[arg-type] - coalesced = server._request_job_build( # type: ignore[arg-type] - same_tip_reconnect - ) - - self.assertIs(server._job_build_active, latest_flight) - self.assertIs(coalesced, latest.promise) # type: ignore[union-attr] - self.assertFalse(latest.cancellation.is_set()) # type: ignore[union-attr] - self.assertFalse(deferred.done()) - self.assertEqual( - server.job_build_priority_counts["routine_deferred"], - 1, - ) - self.assertEqual( - server.initial_job_prepared_work_counts["deferred"], - 1, - ) - self.assertEqual( - server.initial_job_prepared_work_counts["singleflight"], - 1, - ) - - latest.promise.set_result(object()) # type: ignore[union-attr] - with self.assertRaises(JobBuildSuperseded): - deferred.result() - - metrics = "\n".join(server.job_build_metrics_lines()) - self.assertIn( - 'qbit_prism_job_build_priority_events_total{result="routine_deferred"} 1', - metrics, - ) - self.assertIn( - 'qbit_prism_initial_job_prepared_work_total{result="deferred"} 1', - metrics, - ) - self.assertIn( - 'qbit_prism_initial_job_prepared_work_total{result="singleflight"} 1', - metrics, - ) - - def test_publication_critical_build_preempts_routine_builder_capacity( - self, - ) -> None: - server, rpc = coordinator() - old_artifacts = server.store_template_artifacts(dict(rpc.template)) - new_artifacts = server.store_template_artifacts( - base_template(height=11, prevhash="22" * 32) - ) - assert old_artifacts is not None and new_artifacts is not None - payout_generation = server._payout_state_generation - - def request_for( - artifacts: object, - *, - publication_critical: bool, - request_source: str, - ) -> object: - return server._new_job_build_request( - artifacts, # type: ignore[arg-type] - None, - mode="ready", - payout_state_generation=payout_generation, - cache_key=server._job_bundle_key( - artifacts, # type: ignore[arg-type] - mode="ready", - payout_state_generation=payout_generation, - worker=None, - ), - publication_critical=publication_critical, - request_source=request_source, - ) - - routine = request_for( - old_artifacts, - publication_critical=False, - request_source="initial", - ) - latest = request_for( - new_artifacts, - publication_critical=True, - request_source="tip_refresh", - ) - routine_flight = SimpleNamespace(request=routine) - server._job_build_active = routine_flight - server._job_build_retiring = None - server._job_build_pending = None - server._start_job_build_locked = ( # type: ignore[method-assign] - lambda request: SimpleNamespace(request=request, future=None) - ) - server._arm_job_build_locked = lambda _flight: None # type: ignore[method-assign] - - promise = server._request_job_build(latest) # type: ignore[arg-type] - - self.assertIs(promise, latest.promise) # type: ignore[union-attr] - self.assertTrue(routine.cancellation.is_set()) # type: ignore[union-attr] - self.assertIs(server._job_build_retiring, routine_flight) - assert server._job_build_active is not None - self.assertIs(server._job_build_active.request, latest) - self.assertEqual( - server.job_build_priority_counts["routine_preempted"], - 1, - ) - - def test_publication_critical_build_restarts_unhealthy_exact_flight( - self, - ) -> None: - for unhealthy in ("almost_expired", "stalled"): - with self.subTest(unhealthy=unhealthy): - server, rpc = coordinator() - artifacts = server.store_template_artifacts(dict(rpc.template)) - assert artifacts is not None - payout_generation = server._payout_state_generation - - def request_for( - *, - publication_critical: bool, - priority_requested_monotonic: float | None = None, - ) -> object: - return server._new_job_build_request( - artifacts, - None, - mode="ready", - payout_state_generation=payout_generation, - cache_key=server._job_bundle_key( - artifacts, - mode="ready", - payout_state_generation=payout_generation, - worker=None, - ), - publication_critical=publication_critical, - request_source=( - "tip_refresh" - if publication_critical - else "initial" - ), - priority_requested_monotonic=( - priority_requested_monotonic - ), - ) - - routine = request_for(publication_critical=False) - now = time.monotonic() - if unhealthy == "almost_expired": - routine.cancellation.started_monotonic = now - 59.99 - routine.cancellation.deadline_monotonic = now + 0.01 - routine.cancellation.last_checkpoint_monotonic = now - else: - routine.cancellation.started_monotonic = now - 1.0 - routine.cancellation.deadline_monotonic = now + 59.0 - routine.cancellation.last_checkpoint_monotonic = ( - now - server.job_build_cancel_grace_seconds - 0.01 - ) - priority_requested = now - 59.0 - latest = request_for( - publication_critical=True, - priority_requested_monotonic=priority_requested, - ) - routine_flight = SimpleNamespace(request=routine) - server._job_build_active = routine_flight - server._job_build_retiring = None - server._job_build_pending = None - server._start_job_build_locked = ( # type: ignore[method-assign] - lambda request: SimpleNamespace(request=request, future=None) - ) - server._arm_job_build_locked = lambda _flight: None # type: ignore[method-assign] - - promise = server._request_job_build(latest) # type: ignore[arg-type] - - self.assertIs(promise, latest.promise) - self.assertIsNot(promise, routine.promise) - self.assertTrue(routine.cancellation.is_set()) - self.assertIs(server._job_build_retiring, routine_flight) - assert server._job_build_active is not None - self.assertIs(server._job_build_active.request, latest) - self.assertEqual( - latest.requested_monotonic, - priority_requested, - ) - self.assertGreater( - latest.cancellation.deadline_monotonic - - time.monotonic(), - server.job_build_timeout_seconds - 1.0, - ) - self.assertEqual( - server.job_build_priority_counts["routine_preempted"], - 1, - ) - - def test_publication_priority_precedes_immutable_request_preparation( - self, - ) -> None: - server, rpc = coordinator() - install_fake_bundle_builder(server) - artifacts = server.store_template_artifacts(dict(rpc.template)) - assert artifacts is not None - critical_preparation_entered = threading.Event() - release_critical_preparation = threading.Event() - initial_preparation_entered = threading.Event() - original_new_request = server._new_job_build_request - - def observed_new_request(*args: object, **kwargs: object) -> object: - request_source = str(kwargs.get("request_source", "routine")) - if request_source == "tip_refresh": - critical_preparation_entered.set() - if not release_critical_preparation.wait(5): - raise AssertionError("test did not release priority preparation") - elif request_source == "initial": - initial_preparation_entered.set() - return original_new_request(*args, **kwargs) # type: ignore[arg-type] - - server._new_job_build_request = observed_new_request # type: ignore[method-assign] - results: dict[str, list[object]] = {"critical": [], "initial": []} - errors: list[BaseException] = [] - - def build(label: str, *, publication_critical: bool) -> None: - try: - results[label].append( - server.shared_job_bundle( - artifacts, - mode="ready", - publication_critical=publication_critical, - request_source=( - "tip_refresh" if publication_critical else "initial" - ), - ) - ) - except BaseException as exc: # noqa: BLE001 - surfaced below - errors.append(exc) - - critical_thread = threading.Thread( - target=build, - args=("critical",), - kwargs={"publication_critical": True}, - ) - initial_thread = threading.Thread( - target=build, - args=("initial",), - kwargs={"publication_critical": False}, - ) - critical_thread.start() - try: - self.assertTrue(critical_preparation_entered.wait(5)) - initial_thread.start() - self.assertFalse(initial_preparation_entered.wait(0.1)) - metrics = "\n".join(server.job_build_metrics_lines()) - self.assertIn("qbit_prism_job_build_priority_active 1", metrics) - finally: - release_critical_preparation.set() - critical_thread.join(5) - initial_thread.join(5) - - self.assertFalse(critical_thread.is_alive()) - self.assertFalse(initial_thread.is_alive()) - self.assertEqual(errors, []) - self.assertEqual([len(results[label]) for label in results], [1, 1]) - self.assertIs(results["critical"][0], results["initial"][0]) - self.assertFalse(initial_preparation_entered.is_set()) - self.assertEqual( - server.initial_job_prepared_work_counts["deferred"], - 1, - ) - self.assertEqual( - server.initial_job_prepared_work_counts["cache_hit"], - 1, - ) - self.assertEqual( - server.job_build_priority_admission_seconds["count"], - 1, - ) - self.assertGreaterEqual( - server.job_build_priority_admission_seconds["sum"], - 0.1, - ) - - def test_priority_reservation_cancels_admitted_routine_preparation( - self, - ) -> None: - server, rpc = coordinator() - install_fake_bundle_builder(server) - artifacts = server.store_template_artifacts(dict(rpc.template)) - assert artifacts is not None - routine_in_payout_lookup = threading.Event() - release_routine_lookup = threading.Event() - routine_request_constructed = threading.Event() - original_usable_artifact = server._usable_payout_ledger_artifact - original_new_request = server._new_job_build_request - routine_thread: threading.Thread - - def blocked_usable_artifact(*args: object, **kwargs: object) -> object: - if threading.current_thread() is routine_thread: - # The pre-admission cache probe also consults the payout - # artifact; only the lookup made under an admitted routine - # preparation exercises the cancellation contract here. - with server._job_build_scheduler_lock: - admitted = bool(server._job_build_routine_preparations) - if admitted: - routine_in_payout_lookup.set() - if not release_routine_lookup.wait(5): - raise AssertionError("test did not release payout lookup") - return original_usable_artifact(*args, **kwargs) # type: ignore[arg-type] - - def observed_new_request(*args: object, **kwargs: object) -> object: - if kwargs.get("request_source") == "initial": - routine_request_constructed.set() - return original_new_request(*args, **kwargs) # type: ignore[arg-type] - - server._usable_payout_ledger_artifact = blocked_usable_artifact # type: ignore[method-assign] - server._new_job_build_request = observed_new_request # type: ignore[method-assign] - errors: list[BaseException] = [] - - def build_routine() -> None: - try: - server.shared_job_bundle( - artifacts, - mode="ready", - retry_superseded=False, - request_source="initial", - ) - except BaseException as exc: # noqa: BLE001 - surfaced below - errors.append(exc) - - routine_thread = threading.Thread(target=build_routine) - routine_thread.start() - priority_token: int | None = None - try: - self.assertTrue(routine_in_payout_lookup.wait(5)) - with server._job_build_scheduler_lock: - routine_cancellations = tuple( - cancellation_ref() - for cancellation_ref in ( - server._job_build_routine_preparations.values() - ) - ) - self.assertEqual(len(routine_cancellations), 1) - self.assertIsNotNone(routine_cancellations[0]) - priority_token, _requested = ( - server._begin_job_build_priority_preparation() - ) - assert routine_cancellations[0] is not None - self.assertTrue(routine_cancellations[0].is_set()) - finally: - release_routine_lookup.set() - routine_thread.join(5) - if priority_token is not None: - server._finish_job_build_priority_preparation(priority_token) - - self.assertFalse(routine_thread.is_alive()) - self.assertEqual(len(errors), 1) - self.assertIsInstance(errors[0], JobBuildSuperseded) - self.assertFalse(routine_request_constructed.is_set()) - with server._job_build_scheduler_lock: - self.assertEqual(len(server._job_build_routine_preparations), 0) - - def test_publication_critical_collection_promotes_past_ready_work(self) -> None: - server, rpc = coordinator(ledger=FakeLedger(miners=["miner-a"])) - old_artifacts = server.store_template_artifacts(dict(rpc.template)) - new_artifacts = server.store_template_artifacts( - base_template(height=11, prevhash="22" * 32) - ) - assert old_artifacts is not None and new_artifacts is not None - payout_generation = server._payout_state_generation - ready = server._new_job_build_request( - old_artifacts, - None, - mode="ready", - payout_state_generation=payout_generation, - cache_key=server._job_bundle_key( - old_artifacts, - mode="ready", - payout_state_generation=payout_generation, - worker=None, - ), - request_source="initial", - ) - latest_identity = worker("tq1latest", "tq1latest.rig") - latest = server._new_job_build_request( - new_artifacts, - latest_identity, - mode="collection", - payout_state_generation=payout_generation, - cache_key=server._job_bundle_key( - new_artifacts, - mode="collection", - payout_state_generation=payout_generation, - worker=latest_identity, - ), - publication_critical=True, - request_source="tip_refresh", - ) - ready_flight = SimpleNamespace(request=ready) - server._job_build_active = ready_flight - server._job_build_retiring = None - server._job_build_pending = latest - armed: list[object] = [] - server._start_job_build_locked = ( # type: ignore[method-assign] - lambda request: SimpleNamespace(request=request, future=None) - ) - server._arm_job_build_locked = armed.append # type: ignore[method-assign] - - server._promote_pending_job_build_locked() - - self.assertTrue(ready.cancellation.is_set()) - self.assertIs(server._job_build_retiring, ready_flight) - self.assertIsNone(server._job_build_pending) - assert server._job_build_active is not None - self.assertIs(server._job_build_active.request, latest) - self.assertEqual(armed, [server._job_build_active]) - self.assertEqual( - server.job_build_priority_counts["routine_preempted"], - 1, - ) - - def test_routine_pending_never_promotes_over_publication_critical_flight( - self, - ) -> None: - for placement in ("active", "retiring"): - with self.subTest(placement=placement): - server, rpc = coordinator() - critical_artifacts = server.store_template_artifacts( - dict(rpc.template) - ) - routine_artifacts = server.store_template_artifacts( - base_template(height=11, prevhash="22" * 32) - ) - assert critical_artifacts is not None - assert routine_artifacts is not None - payout_generation = server._payout_state_generation - - def request_for( - artifacts: object, - *, - publication_critical: bool, - ) -> object: - return server._new_job_build_request( - artifacts, # type: ignore[arg-type] - None, - mode="ready", - payout_state_generation=payout_generation, - cache_key=server._job_bundle_key( - artifacts, # type: ignore[arg-type] - mode="ready", - payout_state_generation=payout_generation, - worker=None, - ), - publication_critical=publication_critical, - request_source=( - "tip_refresh" if publication_critical else "initial" - ), - ) - - critical = request_for( - critical_artifacts, - publication_critical=True, - ) - routine = request_for( - routine_artifacts, - publication_critical=False, - ) - critical_flight = SimpleNamespace(request=critical) - server._job_build_active = ( - critical_flight if placement == "active" else None - ) - server._job_build_retiring = ( - critical_flight if placement == "retiring" else None - ) - server._job_build_pending = routine - server._start_job_build_locked = ( # type: ignore[method-assign] - lambda _request: self.fail( - "routine pending work displaced publication-critical work" - ) - ) - - server._promote_pending_job_build_locked() - - self.assertIs(server._job_build_pending, routine) - self.assertFalse(critical.cancellation.is_set()) - self.assertIs( - ( - server._job_build_active - if placement == "active" - else server._job_build_retiring - ), - critical_flight, - ) - - def test_cancelled_ready_does_not_block_collection_promotion(self) -> None: - for placement in ("active", "retiring"): - with self.subTest(placement=placement): - server, rpc = coordinator(ledger=FakeLedger(miners=["miner-a"])) - artifacts = server.store_template_artifacts(dict(rpc.template)) - assert artifacts is not None - payout_generation = server._payout_state_generation - - def request_for( - mode: str, - identity: WorkerIdentity | None, - ) -> object: - return server._new_job_build_request( - artifacts, - identity, - mode=mode, - payout_state_generation=payout_generation, - cache_key=server._job_bundle_key( - artifacts, - mode=mode, - payout_state_generation=payout_generation, - worker=identity, - ), - ) - - ready = request_for("ready", None) - collection = request_for( - "collection", - worker("tq1collection", "tq1collection.rig"), - ) - self.assertTrue( # type: ignore[union-attr] - ready.cancellation.cancel("superseded") - ) - ready_flight = SimpleNamespace(request=ready) - if placement == "active": - server._job_build_active = ready_flight - server._job_build_retiring = None - else: - server._job_build_active = None - server._job_build_retiring = ready_flight - server._job_build_pending = collection - armed: list[object] = [] - server._start_job_build_locked = ( # type: ignore[method-assign] - lambda request: SimpleNamespace(request=request, future=None) - ) - server._arm_job_build_locked = armed.append # type: ignore[method-assign] - - server._promote_pending_job_build_locked() - - self.assertIsNone(server._job_build_pending) - assert server._job_build_active is not None - self.assertIs(server._job_build_active.request, collection) - self.assertIs(server._job_build_retiring, ready_flight) - self.assertEqual(armed, [server._job_build_active]) - - def test_immediate_collection_completion_does_not_reoccupy_slot(self) -> None: - server, rpc = coordinator(ledger=FakeLedger(miners=["miner-a"])) - artifacts = server.store_template_artifacts(dict(rpc.template)) - assert artifacts is not None - payout_generation = server._payout_state_generation - identities = [ - worker(f"tq1worker-{index}", f"tq1worker-{index}.rig") - for index in range(2) - ] - - def request_for(identity: WorkerIdentity) -> object: - return server._new_job_build_request( - artifacts, - identity, - mode="collection", - payout_state_generation=payout_generation, - cache_key=server._job_bundle_key( - artifacts, - mode="collection", - payout_state_generation=payout_generation, - worker=identity, - ), - ) - - pending = request_for(identities[0]) - incoming = request_for(identities[1]) - results: dict[int, object] = {} - - def completed_flight(request: object) -> object: - result = SimpleNamespace(request=request) - results[id(request)] = result - future: Future[object] = Future() - future.set_result(result) - return SimpleNamespace(request=request, future=future) - - server._job_build_pending = pending - server._start_job_build_locked = completed_flight # type: ignore[method-assign] - - promise = server._request_job_build(incoming) # type: ignore[arg-type] - - self.assertIs(promise.result(), results[id(incoming)]) - self.assertIs( # type: ignore[union-attr] - pending.promise.result(), - results[id(pending)], - ) - self.assertIsNone(server._job_build_active) - self.assertIsNone(server._job_build_retiring) - self.assertIsNone(server._job_build_pending) - - def test_independent_collection_workers_do_not_supersede_each_other(self) -> None: - server, rpc = coordinator(ledger=FakeLedger(miners=["miner-a"])) - recorded = install_fake_bundle_builder(server) - artifacts = server.store_template_artifacts(dict(rpc.template)) - assert artifacts is not None - identities = [ - worker(f"tq1worker-{index}", f"tq1worker-{index}.rig") - for index in range(4) - ] - entered = [threading.Event() for _identity in identities] - releases = [threading.Event() for _identity in identities] - original_build = server.build_shared_job_bundle - active_builds = 0 - max_active_builds = 0 - active_lock = threading.Lock() - - def blocking_build( - build_artifacts: object, - identity: WorkerIdentity, - **kwargs: object, - ) -> object: - nonlocal active_builds, max_active_builds - index = identities.index(identity) - with active_lock: - active_builds += 1 - max_active_builds = max(max_active_builds, active_builds) - entered[index].set() - try: - self.assertTrue(releases[index].wait(5)) - return original_build( - build_artifacts, # type: ignore[arg-type] - identity, - **kwargs, - ) - finally: - with active_lock: - active_builds -= 1 - - server.build_shared_job_bundle = blocking_build # type: ignore[method-assign] - results: list[list[object]] = [[] for _identity in identities] - errors: list[list[BaseException]] = [[] for _identity in identities] - - def build(index: int) -> None: - try: - results[index].append( - server.shared_job_bundle( - artifacts, - identities[index], - mode="collection", - ) - ) - except BaseException as exc: # noqa: BLE001 - surfaced below - errors[index].append(exc) - - threads = [ - threading.Thread(target=build, args=(index,)) - for index in range(len(identities)) - ] - threads[0].start() - try: - self.assertTrue(entered[0].wait(5)) - threads[1].start() - self.assertTrue(entered[1].wait(5)) - threads[2].start() - pending_deadline = time.monotonic() + 5 - while time.monotonic() < pending_deadline: - with server._job_build_scheduler_lock: - pending = server._job_build_pending - if pending is not None and pending.worker == identities[2]: - break - time.sleep(0.01) - else: - self.fail("third collection build was not queued") - threads[3].start() - self.assertEqual(server.job_build_scheduler_counts["starts"], 2) - releases[0].set() - self.assertTrue(entered[2].wait(5)) - releases[1].set() - self.assertTrue(entered[3].wait(5)) - self.assertEqual(server.job_build_scheduler_counts["starts"], 4) - finally: - for release in releases: - release.set() - for thread in threads: - thread.join(5) - - self.assertTrue(all(not thread.is_alive() for thread in threads)) - self.assertEqual(errors, [[], [], [], []]) - self.assertEqual([len(items) for items in results], [1, 1, 1, 1]) - self.assertEqual(recorded["calls"], 4) - self.assertLessEqual(max_active_builds, 2) - self.assertEqual(server.job_build_scheduler_counts["supersessions"], 0) - - def test_collection_independence_requires_one_immutable_cohort(self) -> None: - server, rpc = coordinator(ledger=FakeLedger(miners=["miner-a"])) - first = server.store_template_artifacts(dict(rpc.template)) - assert first is not None - second_template = dict(rpc.template) - second_template["curtime"] = int(second_template["curtime"]) + 1 - second = server.store_template_artifacts(second_template) - assert second is not None - payout_generation = server._payout_state_generation - first_worker = worker("tq1worker-1", "tq1worker-1.rig") - second_worker = worker("tq1worker-2", "tq1worker-2.rig") - - def request_for( - build_artifacts: object, - identity: WorkerIdentity, - ) -> object: - return server._new_job_build_request( - build_artifacts, # type: ignore[arg-type] - identity, - mode="collection", - payout_state_generation=payout_generation, - cache_key=server._job_bundle_key( - build_artifacts, # type: ignore[arg-type] - mode="collection", - payout_state_generation=payout_generation, - worker=identity, - ), - ) - - first_request = request_for(first, first_worker) - peer_request = request_for(first, second_worker) - newer_request = request_for(second, first_worker) - - self.assertTrue( - server._collection_job_builds_are_independent( - first_request, # type: ignore[arg-type] - peer_request, # type: ignore[arg-type] - ) - ) - self.assertFalse( - server._collection_job_builds_are_independent( - first_request, # type: ignore[arg-type] - newer_request, # type: ignore[arg-type] - ) - ) - - def test_ready_build_cancels_both_live_collection_flights(self) -> None: - server, rpc = coordinator(ledger=FakeLedger(miners=["miner-a"])) - server._ensure_tip_refresh_state() - install_fake_bundle_builder(server) - artifacts = server.store_template_artifacts(dict(rpc.template)) - assert artifacts is not None - identities = [ - worker(f"tq1worker-{index}", f"tq1worker-{index}.rig") - for index in range(2) - ] - entered = [threading.Event(), threading.Event()] - cancellation_observed = [threading.Event(), threading.Event()] - release_cancelled = threading.Event() - original_build = server.build_shared_job_bundle - - def blocking_build( - build_artifacts: object, - identity: WorkerIdentity | None, - **kwargs: object, - ) -> object: - request = kwargs["build_request"] - if identity in identities: - index = identities.index(identity) - entered[index].set() - while not request.cancellation.is_set(): # type: ignore[union-attr] - if release_cancelled.wait(0.01): - raise AssertionError("collection build was not cancelled") - cancellation_observed[index].set() - release_cancelled.wait(5) - request.cancellation.raise_if_cancelled( # type: ignore[union-attr] - "test collection hold" - ) - return original_build( - build_artifacts, # type: ignore[arg-type] - identity, - **kwargs, - ) - - server.build_shared_job_bundle = blocking_build # type: ignore[method-assign] - payout_generation = server._payout_state_generation - - def request_for( - mode: str, - identity: WorkerIdentity | None, - ) -> object: - return server._new_job_build_request( - artifacts, - identity, - mode=mode, - payout_state_generation=payout_generation, - cache_key=server._job_bundle_key( - artifacts, - mode=mode, - payout_state_generation=payout_generation, - worker=identity, - ), - ) - - collection_requests = [ - request_for("collection", identity) for identity in identities - ] - collection_promises = [] - ready_promise = None - try: - collection_promises.append( - server._request_job_build(collection_requests[0]) # type: ignore[arg-type] - ) - self.assertTrue(entered[0].wait(5)) - collection_promises.append( - server._request_job_build(collection_requests[1]) # type: ignore[arg-type] - ) - self.assertTrue(entered[1].wait(5)) - - ready_request = request_for("ready", None) - ready_promise = server._request_job_build( # type: ignore[arg-type] - ready_request - ) - self.assertTrue(cancellation_observed[0].wait(5)) - self.assertTrue(cancellation_observed[1].wait(5)) - self.assertFalse(collection_promises[0].done()) - self.assertFalse(collection_promises[1].done()) - self.assertFalse(ready_promise.done()) - self.assertEqual(server.job_build_scheduler_counts["starts"], 2) - finally: - release_cancelled.set() - - assert ready_promise is not None - ready_bundle = ready_promise.result(timeout=5) - self.assertFalse(ready_bundle.collection_only) - for request, promise in zip(collection_requests, collection_promises): - self.assertTrue(request.cancellation.is_set()) # type: ignore[union-attr] - self.assertIsInstance( - promise.exception(timeout=5), - JobBuildSuperseded, - ) - self.assertEqual(server.job_build_scheduler_counts["starts"], 3) - - def test_ready_build_cancels_retiring_only_collection_flight(self) -> None: - server, rpc = coordinator(ledger=FakeLedger(miners=["miner-a"])) - server._ensure_tip_refresh_state() - install_fake_bundle_builder(server) - artifacts = server.store_template_artifacts(dict(rpc.template)) - assert artifacts is not None - identities = [ - worker(f"tq1worker-{index}", f"tq1worker-{index}.rig") - for index in range(2) - ] - entered = [threading.Event(), threading.Event()] - release_active = threading.Event() - retiring_cancelled = threading.Event() - release_retiring = threading.Event() - original_build = server.build_shared_job_bundle - - def blocking_build( - build_artifacts: object, - identity: WorkerIdentity | None, - **kwargs: object, - ) -> object: - request = kwargs["build_request"] - if identity == identities[0]: - entered[0].set() - while not request.cancellation.is_set(): # type: ignore[union-attr] - if release_retiring.wait(0.01): - raise AssertionError("retiring build was not cancelled") - retiring_cancelled.set() - release_retiring.wait(5) - request.cancellation.raise_if_cancelled( # type: ignore[union-attr] - "test retiring-only hold" - ) - elif identity == identities[1]: - entered[1].set() - self.assertTrue(release_active.wait(5)) - return original_build( - build_artifacts, # type: ignore[arg-type] - identity, - **kwargs, - ) - - server.build_shared_job_bundle = blocking_build # type: ignore[method-assign] - payout_generation = server._payout_state_generation - - def request_for( - mode: str, - identity: WorkerIdentity | None, - ) -> object: - return server._new_job_build_request( - artifacts, - identity, - mode=mode, - payout_state_generation=payout_generation, - cache_key=server._job_bundle_key( - artifacts, - mode=mode, - payout_state_generation=payout_generation, - worker=identity, - ), - ) - - first_request = request_for("collection", identities[0]) - second_request = request_for("collection", identities[1]) - first_promise = server._request_job_build( # type: ignore[arg-type] - first_request - ) - self.assertTrue(entered[0].wait(5)) - second_promise = server._request_job_build( # type: ignore[arg-type] - second_request - ) - self.assertTrue(entered[1].wait(5)) - try: - release_active.set() - second_bundle = second_promise.result(timeout=5) - self.assertTrue(second_bundle.collection_only) - with server._job_build_scheduler_lock: - self.assertIsNone(server._job_build_active) - assert server._job_build_retiring is not None - self.assertIs( - server._job_build_retiring.request, - first_request, - ) - - ready_request = request_for("ready", None) - ready_promise = server._request_job_build( # type: ignore[arg-type] - ready_request - ) - self.assertTrue(retiring_cancelled.wait(5)) - ready_bundle = ready_promise.result(timeout=5) - self.assertFalse(ready_bundle.collection_only) - self.assertTrue(first_request.cancellation.is_set()) # type: ignore[union-attr] - finally: - release_active.set() - release_retiring.set() - - self.assertIsInstance( - first_promise.exception(timeout=5), - JobBuildSuperseded, - ) - - def test_collection_retries_do_not_supersede_ready_build(self) -> None: - server, rpc = coordinator(ledger=FakeLedger(miners=["miner-a"])) - server._ensure_tip_refresh_state() - install_fake_bundle_builder(server) - artifacts = server.store_template_artifacts(dict(rpc.template)) - assert artifacts is not None - identities = [ - worker(f"tq1worker-{index}", f"tq1worker-{index}.rig") - for index in range(2) - ] - collection_entered = [threading.Event(), threading.Event()] - collection_cancelled = [threading.Event(), threading.Event()] - release_cancelled = threading.Event() - ready_entered = threading.Event() - release_ready = threading.Event() - stop_collections = threading.Event() - ready_requests: list[object] = [] - collection_requests: list[object | None] = [None, None] - original_build = server.build_shared_job_bundle - - def blocking_build( - build_artifacts: object, - identity: WorkerIdentity | None, - **kwargs: object, - ) -> object: - request = kwargs["build_request"] - if request.mode == "ready": # type: ignore[union-attr] - ready_requests.append(request) - ready_entered.set() - self.assertTrue(release_ready.wait(5)) - elif identity in identities: - index = identities.index(identity) - collection_requests[index] = request - collection_entered[index].set() - while not request.cancellation.is_set(): # type: ignore[union-attr] - if release_cancelled.wait(0.01): - raise AssertionError("collection build was not cancelled") - collection_cancelled[index].set() - release_cancelled.wait(5) - request.cancellation.raise_if_cancelled( # type: ignore[union-attr] - "test collection retry hold" - ) - return original_build( - build_artifacts, # type: ignore[arg-type] - identity, - **kwargs, - ) - - server.build_shared_job_bundle = blocking_build # type: ignore[method-assign] - collection_results: list[list[object]] = [[], []] - collection_errors: list[list[BaseException]] = [[], []] - ready_results: list[object] = [] - ready_errors: list[BaseException] = [] - - def build_collection(index: int) -> None: - try: - collection_results[index].append( - server.shared_job_bundle( - artifacts, - identities[index], - mode="collection", - cancelled=stop_collections.is_set, - ) - ) - except BaseException as exc: # noqa: BLE001 - surfaced below - collection_errors[index].append(exc) - - def build_ready() -> None: - try: - ready_results.append( - server.shared_job_bundle( - artifacts, - mode="ready", - retry_superseded=False, - ) - ) - except BaseException as exc: # noqa: BLE001 - surfaced below - ready_errors.append(exc) - - collection_threads = [ - threading.Thread(target=build_collection, args=(index,)) - for index in range(2) - ] - ready_thread = threading.Thread(target=build_ready) - try: - collection_threads[0].start() - self.assertTrue(collection_entered[0].wait(5)) - collection_threads[1].start() - self.assertTrue(collection_entered[1].wait(5)) - ready_thread.start() - self.assertTrue(collection_cancelled[0].wait(5)) - self.assertTrue(collection_cancelled[1].wait(5)) - self.assertEqual(server.job_build_scheduler_counts["starts"], 2) - - release_cancelled.set() - self.assertTrue(ready_entered.wait(5)) - retry_deadline = time.monotonic() + 5 - while ( - server.job_build_scheduler_counts["requests"] < 5 - and time.monotonic() < retry_deadline - ): - time.sleep(0.01) - self.assertGreaterEqual( - server.job_build_scheduler_counts["requests"], - 5, - ) - self.assertEqual(server.job_build_scheduler_counts["starts"], 3) - self.assertEqual(len(ready_requests), 1) - self.assertFalse( - ready_requests[0].cancellation.is_set() # type: ignore[union-attr] - ) - - stop_collections.set() - for thread in collection_threads: - thread.join(2) - release_ready.set() - ready_thread.join(5) - finally: - stop_collections.set() - release_cancelled.set() - release_ready.set() - for thread in collection_threads: - if thread.ident is not None: - thread.join(5) - if ready_thread.ident is not None: - ready_thread.join(5) - - self.assertTrue(all(not thread.is_alive() for thread in collection_threads)) - self.assertFalse(ready_thread.is_alive()) - self.assertEqual(collection_results, [[], []]) - self.assertEqual([len(errors) for errors in collection_errors], [1, 1]) - self.assertEqual(ready_errors, []) - self.assertEqual(len(ready_results), 1) - self.assertFalse(ready_results[0].collection_only) # type: ignore[union-attr] - for request in collection_requests: - assert request is not None - self.assertTrue(request.cancellation.is_set()) # type: ignore[union-attr] - - def test_shutdown_cancels_builder_with_full_helper_input_pipe(self) -> None: - server, rpc = coordinator() - server.signing_seed_hex = "42" * 32 - server.ledger_attestation_signing_seed_hex = "43" * 32 - artifacts = server.store_template_artifacts(dict(rpc.template)) - assert artifacts is not None - payout_generation = server._payout_state_generation - request = server._new_job_build_request( - artifacts, - None, - mode="ready", - payout_state_generation=payout_generation, - cache_key=server._job_bundle_key( - artifacts, - mode="ready", - payout_state_generation=payout_generation, - worker=None, - ), - ) - helper_started = threading.Event() - helper_processes: list[subprocess.Popen[str]] = [] - real_popen = subprocess.Popen - - def capture_popen(*args: object, **kwargs: object) -> subprocess.Popen[str]: - process = real_popen(*args, **kwargs) # type: ignore[arg-type] - helper_processes.append(process) - helper_started.set() - return process - - def fill_helper_pipe(*_args: object, **kwargs: object) -> object: - build_request = kwargs["build_request"] - return server.build_audit_bundle( - shares=[], - found_block={ - "block_height": 10, - "coinbase_value_sats": 50_00000000, - "network_difficulty": 1, - "anchor_job_issued_at_ms": 1_700_000_000_000, - }, - prior_balances=[ - { - "miner_id": "pipe-filler", - "balance_sats": 1, - "padding": "x" * (4 * 1024 * 1024), - } - ], - coinbase_script_sig_suffix_hex="00", - cancellation=build_request.cancellation, # type: ignore[union-attr] - ) - - server.build_shared_job_bundle = fill_helper_pipe # type: ignore[method-assign] - shutdown_finished = threading.Event() - shutdown_errors: list[BaseException] = [] - - def shutdown() -> None: - try: - server.shutdown_job_build_executor() - except BaseException as exc: # noqa: BLE001 - surfaced below - shutdown_errors.append(exc) - finally: - shutdown_finished.set() - - with patch( - "lab.prism.prism_coordinator.prism_tool_command", - return_value=[sys.executable, "-c", "import time; time.sleep(30)"], - ), patch( - "lab.prism.prism_coordinator.subprocess.Popen", - side_effect=capture_popen, - ): - promise = server._request_job_build(request) - self.assertTrue( - helper_started.wait(5), - repr(promise.exception(timeout=1)) if promise.done() else None, - ) - time.sleep(0.1) - self.assertFalse(promise.done()) - shutdown_thread = threading.Thread(target=shutdown) - shutdown_thread.start() - shutdown_returned = shutdown_finished.wait(2) - if not shutdown_returned: - for process in helper_processes: - if process.poll() is None: - process.kill() - shutdown_thread.join(5) - - self.assertTrue(shutdown_returned) - self.assertFalse(shutdown_thread.is_alive()) - self.assertEqual(shutdown_errors, []) - self.assertIsInstance(promise.exception(timeout=1), JobBuildSuperseded) - - def test_control_cancel_during_serialization_is_supersession(self) -> None: - server, rpc = coordinator() - server.signing_seed_hex = "42" * 32 - server.ledger_attestation_signing_seed_hex = "43" * 32 - artifacts = server.store_template_artifacts(dict(rpc.template)) - assert artifacts is not None - helper_started = threading.Event() - helper_processes: list[subprocess.Popen[str]] = [] - real_popen = subprocess.Popen - - def capture_popen(*args: object, **kwargs: object) -> subprocess.Popen[str]: - process = real_popen(*args, **kwargs) # type: ignore[arg-type] - helper_processes.append(process) - helper_started.set() - return process - - def fill_helper_pipe(*_args: object, **kwargs: object) -> object: - build_request = kwargs["build_request"] - return server.build_audit_bundle( - shares=[], - found_block={ - "block_height": 10, - "coinbase_value_sats": 50_00000000, - "network_difficulty": 1, - "anchor_job_issued_at_ms": 1_700_000_000_000, - }, - prior_balances=[ - { - "miner_id": "pipe-filler", - "balance_sats": 1, - "padding": "x" * (4 * 1024 * 1024), - } - ], - coinbase_script_sig_suffix_hex="00", - cancellation=build_request.cancellation, # type: ignore[union-attr] - ) - - server.build_shared_job_bundle = fill_helper_pipe # type: ignore[method-assign] - errors: list[BaseException] = [] - - def build() -> None: - try: - server.shared_job_bundle( - artifacts, - mode="ready", - retry_superseded=False, - ) - except BaseException as exc: # noqa: BLE001 - surfaced below - errors.append(exc) - - with patch( - "lab.prism.prism_coordinator.prism_tool_command", - return_value=[sys.executable, "-c", "import time; time.sleep(30)"], - ), patch( - "lab.prism.prism_coordinator.subprocess.Popen", - side_effect=capture_popen, - ): - build_thread = threading.Thread(target=build) - build_thread.start() - self.assertTrue(helper_started.wait(5)) - with server._job_cache_lock: - controls = list(server._active_job_bundle_builds.values()) - self.assertEqual(len(controls), 1) - controls[0].cancel_event.set() - build_thread.join(2) - if build_thread.is_alive(): - for process in helper_processes: - if process.poll() is None: - process.kill() - build_thread.join(5) - - self.assertFalse(build_thread.is_alive()) - self.assertEqual(len(errors), 1) - self.assertIsInstance(errors[0], JobBuildSuperseded) - self.assertEqual(server.shared_bundle_build_counts["superseded"], 1) - self.assertEqual(server.shared_bundle_build_counts["failed"], 0) - self.assertEqual(server.tip_refresh_superseded_results, 1) - - def test_full_helper_input_pipe_obeys_builder_timeout(self) -> None: - server, _rpc = coordinator() - server.signing_seed_hex = "42" * 32 - server.ledger_attestation_signing_seed_hex = "43" * 32 - server.bundle_build_timeout_seconds = 0.05 - helper_started = threading.Event() - helper_processes: list[subprocess.Popen[str]] = [] - real_popen = subprocess.Popen - - def capture_popen(*args: object, **kwargs: object) -> subprocess.Popen[str]: - process = real_popen(*args, **kwargs) # type: ignore[arg-type] - helper_processes.append(process) - helper_started.set() - return process - - errors: list[BaseException] = [] - - def build() -> None: - try: - server.build_audit_bundle( - shares=[], - found_block={ - "block_height": 10, - "coinbase_value_sats": 50_00000000, - "network_difficulty": 1, - "anchor_job_issued_at_ms": 1_700_000_000_000, - }, - prior_balances=[ - { - "miner_id": "pipe-filler", - "balance_sats": 1, - "padding": "x" * (4 * 1024 * 1024), - } - ], - coinbase_script_sig_suffix_hex="00", - ) - except BaseException as exc: # noqa: BLE001 - surfaced below - errors.append(exc) - - with patch( - "lab.prism.prism_coordinator.prism_tool_command", - return_value=[sys.executable, "-c", "import time; time.sleep(30)"], - ), patch( - "lab.prism.prism_coordinator.subprocess.Popen", - side_effect=capture_popen, - ): - build_thread = threading.Thread(target=build) - build_thread.start() - self.assertTrue(helper_started.wait(5)) - build_thread.join(2) - if build_thread.is_alive(): - for process in helper_processes: - if process.poll() is None: - process.kill() - build_thread.join(5) - - self.assertFalse(build_thread.is_alive()) - self.assertEqual(len(errors), 1) - self.assertIsInstance(errors[0], RuntimeError) - self.assertIn("timed out", str(errors[0])) - - def test_untrusted_outcome_for_other_tip_keeps_memo_armed(self) -> None: - server, rpc = coordinator() - server.reorg_reconciler_enabled = True - server._note_reorg_reconcile_outcome(rpc.tip, trusted=True) - reconcile_calls: list[str | None] = [] - - def fake_reconcile(*, tip_hash: str | None = None) -> dict[str, object]: - reconcile_calls.append(tip_hash) - server._note_reorg_reconcile_outcome(tip_hash, trusted=True) - return {"untrusted": False} - - server.reconcile_prism_pool_blocks_once = fake_reconcile # type: ignore[method-assign] - - # A superseded/untrusted pass for a different tip must not unarm the - # cached trusted outcome for the current tip. - server._note_reorg_reconcile_outcome("44" * 32, trusted=False) - self.assertTrue(server.ensure_reorg_reconciled_for_current_tip()) - self.assertEqual(reconcile_calls, []) - - # An untrusted outcome for the current tip itself unarms only it. - server._note_reorg_reconcile_outcome(rpc.tip, trusted=False) - self.assertTrue(server.ensure_reorg_reconciled_for_current_tip()) - self.assertEqual(reconcile_calls, [rpc.tip]) - - def test_reconcile_error_outcome_clears_every_memo_entry(self) -> None: - server, rpc = coordinator() - server.reorg_reconciler_enabled = True - server._note_reorg_reconcile_outcome(rpc.tip, trusted=True) - server._note_reorg_reconcile_outcome("55" * 32, trusted=True) - reconcile_calls: list[str | None] = [] - - def fake_reconcile(*, tip_hash: str | None = None) -> dict[str, object]: - reconcile_calls.append(tip_hash) - server._note_reorg_reconcile_outcome(tip_hash, trusted=True) - return {"untrusted": False} - - server.reconcile_prism_pool_blocks_once = fake_reconcile # type: ignore[method-assign] - - server._note_reorg_reconcile_outcome( - "55" * 32, - trusted=False, - clear_memo=True, - ) - - self.assertTrue(server.ensure_reorg_reconciled_for_current_tip()) - self.assertEqual(reconcile_calls, [rpc.tip]) - - def test_tip_flip_back_within_ttl_requires_fresh_reconcile(self) -> None: - server, rpc = coordinator() - server.reorg_reconciler_enabled = True - tip_a = rpc.tip - self.addCleanup(server.shutdown_tip_refresh_executor) - server.observe_tip_for_refresh(tip_a) - server._note_reorg_reconcile_outcome(tip_a, trusted=True) - reconcile_calls: list[str | None] = [] - - def fake_reconcile(*, tip_hash: str | None = None) -> dict[str, object]: - reconcile_calls.append(tip_hash) - return {"untrusted": False} - - server.reconcile_prism_pool_blocks_once = fake_reconcile # type: ignore[method-assign] - - self.assertTrue(server.ensure_reorg_reconciled_for_current_tip()) - self.assertEqual(reconcile_calls, []) - - # Detecting an intervening tip ends tip A's memo epoch: when the - # chain flips back to A within the cache TTL, pool-block chain state - # must be re-proven by a fresh pass, not assumed from the pre-flip - # outcome. - server.observe_tip_for_refresh("bb" * 32) - - self.assertTrue(server.ensure_reorg_reconciled_for_current_tip()) - self.assertEqual(reconcile_calls, [tip_a]) - - def test_stale_pass_cannot_rearm_memo_after_newer_tip_detected(self) -> None: - server, rpc = coordinator() - server.reorg_reconciler_enabled = True - tip_a = rpc.tip - self.addCleanup(server.shutdown_tip_refresh_executor) - server.observe_tip_for_refresh(tip_a) - server.observe_tip_for_refresh("bb" * 32) - - # A pass for tip A finishing after tip B was detected must not arm - # the memo for A: its epoch already ended at the newer observation. - server._note_reorg_reconcile_outcome(tip_a, trusted=True) - reconcile_calls: list[str | None] = [] - - def fake_reconcile(*, tip_hash: str | None = None) -> dict[str, object]: - reconcile_calls.append(tip_hash) - return {"untrusted": False} - - server.reconcile_prism_pool_blocks_once = fake_reconcile # type: ignore[method-assign] - - self.assertTrue(server.ensure_reorg_reconciled_for_current_tip()) - self.assertEqual(reconcile_calls, [tip_a]) - - def test_rearmed_artifact_is_not_shadowed_by_stale_cached_bundle( - self, - ) -> None: - server, rpc = coordinator() - recorded = install_fake_bundle_builder(server) - artifacts = server.store_template_artifacts(dict(rpc.template)) - assert artifacts is not None - - stale = server.shared_job_bundle(artifacts, mode="ready") - self.assertEqual(recorded["calls"], 1) - self.assertEqual(len(stale.shares_json), 3) - - # A share lands and the debounced re-arm rebuilds a fresher window. - server.ledger.miners = [*server.ledger.miners, "late-share"] - server._prepare_payout_ledger_artifact( - server._payout_state_generation, - artifacts.network_difficulty, - ) - with server._job_cache_lock: - fresh_artifact = server._payout_ledger_artifact - assert fresh_artifact is not None - self.assertEqual(fresh_artifact.accepted_share_count, 4) - - rebuilt = server.shared_job_bundle(artifacts, mode="ready") - - # The pre-re-arm bundle is still inside its cache TTL under the - # no-artifact key; its older window must not shadow the fresher - # artifact. - self.assertEqual( - rebuilt.payout_artifact_generation, - fresh_artifact.generation, - ) - self.assertEqual(len(rebuilt.shares_json), 4) - self.assertEqual(recorded["calls"], 2) - - def test_in_flight_build_keeps_aged_artifact_it_already_selected( - self, - ) -> None: - server, rpc = coordinator() - install_fake_bundle_builder(server) - artifacts = server.store_template_artifacts(dict(rpc.template)) - assert artifacts is not None - server._prepare_payout_ledger_artifact( - server._payout_state_generation, - artifacts.network_difficulty, - ) - server.payout_artifact_max_anchor_age_seconds = 5.0 - with server._job_cache_lock: - armed = server._payout_ledger_artifact - assert armed is not None - assert armed.snapshot_anchor_ms is not None - aged = dataclass_replace( - armed, - snapshot_anchor_ms=int(armed.snapshot_anchor_ms) - 6_000, - ) - server._payout_ledger_artifact = aged - - # New reuse decisions reject a window past the audit ceiling... - self.assertIsNone( - server._usable_payout_ledger_artifact( - server._payout_state_generation, - artifacts.network_difficulty, - rearm_on_fence_failure=False, - ) - ) - # ...but a build that selected it while fresh completes with it: the - # in-build re-validation checks supersession and the balances fence - # only, so queue delay past the bound cannot scrap the reuse into - # the full snapshot it was armed to avoid. - server.ledger.snapshot_calls = 0 - bundle = server.build_shared_job_bundle( - artifacts, - worker(), - payout_artifact=aged, - ) - self.assertEqual(server.ledger.snapshot_calls, 0) - self.assertEqual( - bundle.found_block["anchor_job_issued_at_ms"], - aged.snapshot_anchor_ms, - ) - - def test_in_flight_build_survives_same_window_anchor_refresh(self) -> None: - server, rpc = coordinator() - install_fake_bundle_builder(server) - artifacts = server.store_template_artifacts(dict(rpc.template)) - assert artifacts is not None - server._prepare_payout_ledger_artifact( - server._payout_state_generation, - artifacts.network_difficulty, - ) - with server._job_cache_lock: - armed = server._payout_ledger_artifact - assert armed is not None - assert armed.snapshot_anchor_ms is not None - # An in-flight build selected the armed artifact while it - # carried an older anchor. - selected = dataclass_replace( - armed, - snapshot_anchor_ms=int(armed.snapshot_anchor_ms) - 5, - ) - server._payout_ledger_artifact = selected - - # A same-window rebuild refreshes the anchor in place: the stored - # instance swaps while the generation -- the re-key authority -- - # and the window bytes stay identical. - server._prepare_payout_ledger_artifact( - server._payout_state_generation, - artifacts.network_difficulty, - ) - with server._job_cache_lock: - refreshed = server._payout_ledger_artifact - assert refreshed is not None - self.assertIsNot(refreshed, selected) - self.assertEqual(refreshed.generation, selected.generation) - self.assertEqual( - refreshed.share_snapshot_sha256, - selected.share_snapshot_sha256, - ) - - # The build holding the pre-refresh instance completes its reuse - # instead of being scrapped into a full snapshot by the swap. - server.ledger.snapshot_calls = 0 - bundle = server.build_shared_job_bundle( - artifacts, - worker(), - payout_artifact=selected, - ) - self.assertEqual(server.ledger.snapshot_calls, 0) - self.assertEqual( - bundle.found_block["anchor_job_issued_at_ms"], - selected.snapshot_anchor_ms, - ) - - def test_generation_current_cached_bundle_serves_past_reanchor_floor( - self, - ) -> None: - server, rpc = coordinator() - recorded = install_fake_bundle_builder(server) - first = server.store_template_artifacts(dict(rpc.template)) - assert first is not None - with server._payout_artifact_executor_lock: - server._payout_artifact_executor_shutdown = True - # Re-anchor floor below the bundle-cache TTL so the two gates are - # distinguishable inside the TTL window. - server.payout_artifact_reanchor_seconds = 5.0 - - server.shared_job_bundle(first, mode="ready") - self.assertEqual(recorded["calls"], 1) - with server._job_cache_lock: - armed = server._payout_ledger_artifact - assert armed is not None - - # A bundle keyed to the currently armed artifact generation carries - # exactly the armed window: it keeps serving past the re-anchor - # floor (its window is as fresh as reuse itself), bounded upstream - # by the cache TTL. - second = server.store_template_artifacts( - base_template(height=11, prevhash="22" * 32) - ) - assert second is not None - reused = server.shared_job_bundle(second, mode="ready") - self.assertEqual(reused.payout_artifact_generation, armed.generation) - self.assertEqual(recorded["calls"], 2) - with server._job_cache_lock: - for key, entry in list(server._job_bundle_cache.items()): - if entry.payout_artifact_generation == armed.generation: - server._job_bundle_cache[key] = dataclass_replace( - entry, - built_monotonic=float(entry.built_monotonic) - 7.0, - ) - server.shared_job_bundle(second, mode="ready") - self.assertEqual(recorded["calls"], 2) - - def test_cached_no_artifact_bundle_gates_on_build_age_not_anchor( - self, - ) -> None: - # 2026-07-29 regression companion: the gate must follow build age - # (freshness of the work) inside the audit ceiling, never the - # wall-clock anchor, which is frozen per template generation. - server, rpc = coordinator() - recorded = install_fake_bundle_builder(server) - artifacts = server.store_template_artifacts(dict(rpc.template)) - assert artifacts is not None - with server._payout_artifact_executor_lock: - server._payout_artifact_executor_shutdown = True - server.payout_artifact_reanchor_seconds = 5.0 - - seeded = server.shared_job_bundle(artifacts, mode="ready") - self.assertEqual(recorded["calls"], 1) - - # Fresh build age serves from cache once the armed artifact is - # gone. - with server._job_cache_lock: - server._payout_ledger_artifact = None - server.shared_job_bundle(artifacts, mode="ready") - self.assertEqual(recorded["calls"], 1) - - # Build age past the re-anchor floor rebuilds even though the - # bundle-cache TTL has not lapsed. - with server._job_cache_lock: - server._payout_ledger_artifact = None - for key, entry in list(server._job_bundle_cache.items()): - server._job_bundle_cache[key] = dataclass_replace( - entry, - built_monotonic=float(entry.built_monotonic) - 7.0, - ) - server.shared_job_bundle(artifacts, mode="ready") - self.assertEqual(recorded["calls"], 2) - - # The audit ceiling on the declared anchor stays as the backstop - # even for a freshly built entry. - aged_anchor_ms = ( - int(seeded.found_block["anchor_job_issued_at_ms"]) - 400_000 - ) - with server._job_cache_lock: - server._payout_ledger_artifact = None - for key, entry in list(server._job_bundle_cache.items()): - aged_found_block = dict(entry.found_block) - aged_found_block["anchor_job_issued_at_ms"] = aged_anchor_ms - server._job_bundle_cache[key] = dataclass_replace( - entry, - found_block=aged_found_block, - ) - server.shared_job_bundle(artifacts, mode="ready") - self.assertEqual(recorded["calls"], 3) - - def test_generation_bump_does_not_scrap_in_flight_build(self) -> None: - # 2026-07-29 regression: every differing window bumps the armed - # artifact generation, and the in-build re-validation scrapped any - # build whose selected artifact lost the slot -- a rebuild storm at - # production share rates. A build must finish on the copy it - # selected: its window stays audit-reproducible at its declared - # anchor while the payout generation and published balances hold. - server, rpc = coordinator() - install_fake_bundle_builder(server) - artifacts = server.store_template_artifacts(dict(rpc.template)) - assert artifacts is not None - server._prepare_payout_ledger_artifact( - server._payout_state_generation, - artifacts.network_difficulty, - ) - with server._job_cache_lock: - selected = server._payout_ledger_artifact - assert selected is not None - assert selected.snapshot_anchor_ms is not None - - fresher_window = [ - {"share_seq": seq, "miner_id": "miner-a"} for seq in (1, 2, 3, 4) - ] - fresher = dataclass_replace( - selected, - generation=0, - accepted_share_count=4, - shares_json=tuple(fresher_window), - snapshot_anchor_ms=int(selected.snapshot_anchor_ms) + 5, - share_snapshot_sha256=canonical_json_sha256(fresher_window), - ) - self.assertTrue(server._install_payout_ledger_artifact(fresher)) - with server._job_cache_lock: - bumped = server._payout_ledger_artifact - assert bumped is not None - self.assertGreater(bumped.generation, selected.generation) - - server.ledger.snapshot_calls = 0 - bundle = server.build_shared_job_bundle( - artifacts, - worker(), - payout_artifact=selected, - ) - self.assertEqual(server.ledger.snapshot_calls, 0) - self.assertEqual( - bundle.found_block["anchor_job_issued_at_ms"], - selected.snapshot_anchor_ms, - ) - self.assertEqual(bundle.shares_json, list(selected.shares_json)) - - def test_rebuilt_bundle_serves_under_aged_template_generation( - self, - ) -> None: - # 2026-07-29 regression: issued_at_ms is frozen per template - # generation and predates the walk, so a wall-clock anchor gate on - # cached bundles declared every rebuilt bundle dead on arrival once - # its template generation outlived the bound. Below the audit - # ceiling, cache service must follow build age, not anchor age. - server, rpc = coordinator() - recorded = install_fake_bundle_builder(server) - artifacts = server.store_template_artifacts(dict(rpc.template)) - assert artifacts is not None - with server._payout_artifact_executor_lock: - server._payout_artifact_executor_shutdown = True - with server._job_cache_lock: - server._job_build_issued_at_ms[artifacts.generation] = ( - now_ms() - 20_000 - ) - - built = server.shared_job_bundle(artifacts, mode="ready") - self.assertEqual(recorded["calls"], 1) - self.assertEqual( - built.found_block["anchor_job_issued_at_ms"], - built.issued_at_ms, - ) - - served = server.shared_job_bundle(artifacts, mode="ready") - self.assertEqual(recorded["calls"], 1) - self.assertEqual(served.issued_at_ms, built.issued_at_ms) - - def test_admission_deadline_fails_fast_and_visibly(self) -> None: - server, _rpc = coordinator() - server.routine_admission_deadline_seconds = 0.05 - server._publication_priority_scheduled_locked = ( # type: ignore[method-assign] - lambda: True - ) - - with self.assertRaises(JobBuildAdmissionDeadlineExceeded) as raised: - server._begin_routine_job_build_preparation( - request_source="initial", - cancelled=None, - ) - - # The deadline is coordination churn, not a failure budget event: - # callers retry with their normal pacing. - self.assertIsInstance(raised.exception, TemplateRefreshBlocked) - with server._job_build_scheduler_lock: - self.assertEqual( - server.initial_job_prepared_work_counts["admission_deadline"], - 1, - ) - - -class ShareWindowSpoolTests(unittest.TestCase): - """Spool-file handoff of the serialized share window to the builder.""" - - def _coordinator(self) -> PrismCoordinator: - server, _rpc = coordinator() - server.signing_seed_hex = "42" * 32 - server.ledger_attestation_signing_seed_hex = "43" * 32 - self.addCleanup(server.retire_share_window_spool) - return server - - def _ledger_artifact( - self, - shares: list[dict[str, object]], - *, - generation: int, - ) -> PayoutLedgerArtifact: - return PayoutLedgerArtifact( - generation=generation, - payout_state_generation=0, - network_difficulty=1, - accepted_share_count=len(shares), - shares_json=tuple(shares), - prior_balances=(), - prepared_monotonic=time.monotonic(), - snapshot_anchor_ms=None, - ) - - def _build_with_echo_builder( - self, - server: PrismCoordinator, - shares: list[dict[str, object]], - serialization: object, - *, - height: int, - ) -> dict[str, object]: - # These tests cover the one-shot spool handoff; pin the persistent - # builder off so the echo helper is never spawned as a daemon. - with patch.dict(os.environ, {"PRISM_BUILDER_SERVE": "0"}), patch( - "lab.prism.prism_coordinator.prism_tool_command", - return_value=list(ECHO_BUILDER_COMMAND), - ): - return server.build_audit_bundle( - shares=shares, - found_block={ - "block_height": height, - "coinbase_value_sats": 50_00000000, - "network_difficulty": 1, - "anchor_job_issued_at_ms": 1_700_000_000_000, - }, - prior_balances=[], - coinbase_script_sig_suffix_hex="00", - summary_only=True, - payout_policy={"policy": "day-one"}, - share_serialization=serialization, # type: ignore[arg-type] - ) - - def _expected_payload( - self, - shares: list[dict[str, object]], - *, - height: int, - ) -> dict[str, object]: - identities, compact_shares = _compact_share_payload(shares) - return { - "found_block": { - "block_height": height, - "coinbase_value_sats": 50_00000000, - "network_difficulty": 1, - "anchor_job_issued_at_ms": 1_700_000_000_000, - }, - "prior_balances": [], - "payout_policy": {"policy": "day-one"}, - "coinbase_script_sig_suffix_hex": "00", - "witness_merkle_leaves_hex": [], - "compact_share_identities": [ - list(identity) for identity in identities - ], - "compact_shares": [list(share) for share in compact_shares], - } - - def _awkward_shares(self, count: int) -> list[dict[str, object]]: - shares = [] - for seq in range(1, count + 1): - share = spool_share(seq) - share["share_id"] = f"miner-\u00e9.rig{seq}:\U0001F600,}}{{\"\\" - share["miner_id"] = f"miner-{seq % 4}-\u2603" - share["order_key"] = f"{seq % 4:02d}:miner-{seq % 4}" - share["share_difficulty"] = (1 << 130) + seq - share["credit_policy"] = "policy-\u00fc" if seq % 3 else None - shares.append(share) - return shares - - def _historical_tail(self, shares: list[dict[str, object]]) -> str: - identities, compact_shares = _compact_share_payload(shares) - return ( - ',"compact_share_identities":' - + json.dumps(identities, separators=(",", ":")) - + ',"compact_shares":' - + json.dumps(compact_shares, separators=(",", ":")) - + "}" - ) - - def test_compact_tail_chunks_match_the_historical_fragment_bytes(self) -> None: - for count in (0, 1, 2, 1_300): - shares = self._awkward_shares(count) - expected = self._historical_tail(shares) - for batch, chunk in ((512, 64 * 1024), (1, 1), (7, 100), (5_000, 10)): - with self.subTest(count=count, batch=batch, chunk=chunk): - chunks = _compact_share_tail_chunks( - shares, - batch_records=batch, - chunk_chars=chunk, - ) - self.assertEqual("".join(chunks), expected) - self.assertTrue(all(piece.isascii() for piece in chunks)) - - def test_spool_and_in_memory_tail_hold_identical_bounded_bytes(self) -> None: - server = self._coordinator() - shares = self._awkward_shares(1_300) - serialization = server._share_window_serialization_for_artifact( - self._ledger_artifact(shares, generation=1), - shares, - ) - expected = self._historical_tail(shares) - chunks = serialization.compact_tail_chunks(shares) - self.assertGreater(len(chunks), 4) - self.assertEqual("".join(chunks), expected) - self.assertIs(serialization.compact_tail_chunks(shares), chunks) - self.assertEqual(serialization.compact_json_bytes, len(expected)) - lease = serialization.acquire_spooled_tail(shares) - assert lease is not None - spool_file, spool_size = lease - try: - spool_file.seek(0) - self.assertEqual(spool_file.read(), expected.encode("utf-8")) - self.assertEqual(spool_size, len(expected)) - finally: - serialization.release_spooled_tail() - # Both transports deliver the same parsed window. - first = self._build_with_echo_builder(server, shares, serialization, height=10) - with patch( - "lab.prism.prism_coordinator._share_window_spool_file", - side_effect=OSError("temp filesystem unavailable"), - ): - fresh = server._share_window_serialization_for_artifact( - self._ledger_artifact(shares, generation=2), - shares, - ) - fresh.retire_spool() - second = self._build_with_echo_builder(server, shares, fresh, height=10) - self.assertEqual(first["received"], self._expected_payload(shares, height=10)) - self.assertEqual(second["received"], first["received"]) - - def test_spool_feeds_builder_and_is_written_once_per_generation( - self, - ) -> None: - server = self._coordinator() - shares = [spool_share(seq) for seq in range(1, 4)] - serialization = server._share_window_serialization_for_artifact( - self._ledger_artifact(shares, generation=1), - shares, - ) - spool_creations: list[int] = [] - real_spool_file = prism_coordinator_module._share_window_spool_file - - def counting_spool_file() -> object: - spool_creations.append(1) - return real_spool_file() - - with patch( - "lab.prism.prism_coordinator._share_window_spool_file", - side_effect=counting_spool_file, - ): - first = self._build_with_echo_builder( - server, - shares, - serialization, - height=10, - ) - second = self._build_with_echo_builder( - server, - shares, - serialization, - height=11, - ) - - self.assertEqual( - first["received"], - self._expected_payload(shares, height=10), - ) - self.assertEqual( - second["received"], - self._expected_payload(shares, height=11), - ) - self.assertEqual(len(spool_creations), 1) - - def test_spool_creation_failure_falls_back_to_pipe_writes(self) -> None: - server = self._coordinator() - shares = [spool_share(seq) for seq in range(1, 4)] - serialization = server._share_window_serialization_for_artifact( - self._ledger_artifact(shares, generation=1), - shares, - ) - - with patch( - "lab.prism.prism_coordinator._share_window_spool_file", - side_effect=OSError("temp filesystem unavailable"), - ) as spool_factory: - first = self._build_with_echo_builder( - server, - shares, - serialization, - height=10, - ) - second = self._build_with_echo_builder( - server, - shares, - serialization, - height=11, - ) - - self.assertEqual( - first["received"], - self._expected_payload(shares, height=10), - ) - self.assertEqual( - second["received"], - self._expected_payload(shares, height=11), - ) - # The failure is sticky: a broken temp filesystem is not retried on - # every subsequent build of the same generation. - self.assertEqual(spool_factory.call_count, 1) - - def test_splice_failure_poisons_spool_and_falls_back_to_pipe(self) -> None: - server = self._coordinator() - shares = [spool_share(seq) for seq in range(1, 4)] - serialization = server._share_window_serialization_for_artifact( - self._ledger_artifact(shares, generation=1), - shares, - ) - - with patch( - "lab.prism.prism_coordinator.os.splice", - side_effect=OSError("splice unsupported"), - ): - first = self._build_with_echo_builder( - server, - shares, - serialization, - height=10, - ) - second = self._build_with_echo_builder( - server, - shares, - serialization, - height=11, - ) - - self.assertEqual( - first["received"], - self._expected_payload(shares, height=10), - ) - self.assertEqual( - second["received"], - self._expected_payload(shares, height=11), - ) - self.assertTrue(serialization._spool_failed) - self.assertIsNone(serialization._spool_file) - - def test_mid_stream_splice_failure_poisons_spool_for_later_builds( - self, - ) -> None: - server = self._coordinator() - shares = [spool_share(seq) for seq in range(1, 4)] - serialization = server._share_window_serialization_for_artifact( - self._ledger_artifact(shares, generation=1), - shares, - ) - - # The first chunk moves and the second fails: this build is - # unrepairable (partial tail already in the pipe), but the spool must - # be poisoned so later builds stream the cached fragments instead of - # repeating the failure. - with patch( - "lab.prism.prism_coordinator.os.splice", - side_effect=[16, OSError("splice io error")], - ): - with self.assertRaises(OSError): - self._build_with_echo_builder( - server, - shares, - serialization, - height=10, - ) - self.assertTrue(serialization._spool_failed) - - recovered = self._build_with_echo_builder( - server, - shares, - serialization, - height=11, - ) - self.assertEqual( - recovered["received"], - self._expected_payload(shares, height=11), - ) - - def test_content_rotation_retires_spool_after_last_lease(self) -> None: - server = self._coordinator() - shares = [spool_share(seq) for seq in range(1, 4)] - first = server._share_window_serialization_for_artifact( - self._ledger_artifact(shares, generation=1), - shares, - ) - lease = first.acquire_spooled_tail(shares) - self.assertIsNotNone(lease) - assert lease is not None - spool_file, spool_size = lease - self.assertGreater(spool_size, 0) - - changed_shares = [*shares, spool_share(4)] - second = server._share_window_serialization_for_artifact( - self._ledger_artifact(changed_shares, generation=2), - changed_shares, - ) - self.assertIsNot(second, first) - # Retired: no new leases, but the in-flight transfer keeps the - # descriptor alive until it releases. - self.assertIsNone(first.acquire_spooled_tail(shares)) - self.assertFalse(spool_file.closed) - first.release_spooled_tail() - self.assertTrue(spool_file.closed) - - def test_generation_retag_reuses_content_keyed_serialization(self) -> None: - server = self._coordinator() - shares = [spool_share(seq) for seq in range(1, 4)] - - first = server._share_window_serialization_for_artifact( - self._ledger_artifact(shares, generation=1), - shares, - ) - second = server._share_window_serialization_for_artifact( - self._ledger_artifact(shares, generation=2), - shares, - ) - - self.assertIs(second, first) - - def test_shutdown_retires_current_spool(self) -> None: - server = self._coordinator() - shares = [spool_share(seq) for seq in range(1, 4)] - serialization = server._share_window_serialization_for_artifact( - self._ledger_artifact(shares, generation=1), - shares, - ) - lease = serialization.acquire_spooled_tail(shares) - assert lease is not None - serialization.release_spooled_tail() - self.assertFalse(lease[0].closed) - - server.retire_share_window_spool() - - self.assertTrue(lease[0].closed) - self.assertIsNone(serialization.acquire_spooled_tail(shares)) - - -class ServeBuilderTests(unittest.TestCase): - """Persistent --serve builder client, its fallback, and supersession.""" - - def _coordinator(self) -> PrismCoordinator: - server, _rpc = coordinator() - server.signing_seed_hex = "42" * 32 - server.ledger_attestation_signing_seed_hex = "43" * 32 - self.addCleanup(server.shutdown_serve_builder) - self.addCleanup(server.retire_share_window_spool) - return server - - def _serialization( - self, - server: PrismCoordinator, - shares: list[dict[str, object]], - *, - generation: int = 1, - ) -> object: - return server._share_window_serialization_for_artifact( - PayoutLedgerArtifact( - generation=generation, - payout_state_generation=0, - network_difficulty=1, - accepted_share_count=len(shares), - shares_json=tuple(shares), - prior_balances=(), - prepared_monotonic=time.monotonic(), - snapshot_anchor_ms=None, - ), - shares, - ) - - def _build( - self, - server: PrismCoordinator, - shares: list[dict[str, object]], - serialization: object, - *, - height: int, - mode: str = "ok", - cancellation: object = None, - ) -> dict[str, object]: - with patch.dict( - os.environ, - {"FAKE_SERVE_BUILDER_MODE": mode}, - ), patch( - "lab.prism.prism_coordinator.prism_tool_command", - return_value=list(FAKE_SERVE_BUILDER_COMMAND), - ): - return server.build_audit_bundle( - shares=shares, - found_block={ - "block_height": height, - "coinbase_value_sats": 50_00000000, - "network_difficulty": 1, - "anchor_job_issued_at_ms": 1_700_000_000_000, - }, - prior_balances=[], - coinbase_script_sig_suffix_hex="00", - summary_only=True, - payout_policy={"policy": "day-one"}, - share_serialization=serialization, # type: ignore[arg-type] - cancellation=cancellation, # type: ignore[arg-type] - ) - - @staticmethod - def _prepare_record(seq: int, **overrides: object) -> dict[str, object]: - record: dict[str, object] = { - "share_seq": seq, - "share_id": f"share-\u00e9-{seq}:\U0001F600", - "miner_id": f"miner-{seq % 3}", - "order_key": f"{seq % 3:02d}:miner-{seq % 3}", - "p2mr_program_hex": "22" * 32, - "share_difficulty": 1, - "network_difficulty": 1, - "template_height": 9, - "job_id": f"job-{seq}", - "job_issued_at_ms": 1_700_000_000_000 + seq, - "accepted_at_ms": 1_700_000_000_100 + seq, - "ntime": 1_700_000_000 + seq, - } - record.update(overrides) - return record - - def _prepare( - self, - server: PrismCoordinator, - records: list[dict[str, object]], - *, - mode: str = "ok", - ) -> object: - with patch.dict( - os.environ, - {"FAKE_SERVE_BUILDER_MODE": mode}, - ), patch( - "lab.prism.prism_coordinator.prism_tool_command", - return_value=list(FAKE_SERVE_BUILDER_COMMAND), - ): - # An anchor past every stamp and a weight equal to the record - # count: the fold retains the whole snapshot. - return server.prepare_payout_window( - mode="full", - records_json=records, - anchor_job_issued_at_ms=1_700_000_100_000, - append_invalidation_epoch=0, - window_weight=len(records) or 1, - page_size=512, - ) - - def test_prepare_request_streams_the_same_bytes_as_one_json_dumps(self) -> None: - records = [ - self._prepare_record(seq, share_difficulty=(1 << 130) + seq) - for seq in range(1, 1_301) - ] - for fields in ( - { - "request": "prepare_window", - "mode": "full", - "append_invalidation_epoch": 3, - "anchor_job_issued_at_ms": 1_700_000_001_000, - "records": records, - "window_weight": 1 << 100, - "page_size": 512, - }, - { - "request": "prepare_window", - "mode": "advance", - "append_invalidation_epoch": 3, - "anchor_job_issued_at_ms": 1_700_000_001_000, - "records": [], - "base_digest": "ab" * 32, - }, - ): - with self.subTest(mode=fields["mode"]): - chunks = list(_iter_prepare_window_request_chunks(fields)) - self.assertEqual( - b"".join(chunks), - json.dumps(fields, separators=(",", ":")).encode("utf-8") - + b"\n", - ) - self.assertEqual(chunks[-1], b"\n") - if fields["records"]: - self.assertGreater(len(chunks), 3) - - def test_prepare_request_reaches_the_daemon_in_bounded_chunks(self) -> None: - server = self._coordinator() - # Several record batches and past the pipe capacity, so the request - # is written as many chunks and the daemon folds it byte-faithfully. - records = [self._prepare_record(seq) for seq in range(1, 2_001)] - try: - outcome = self._prepare(server, records) - self.assertEqual(outcome.status, "prepared") - self.assertEqual(outcome.record_count, len(records)) - mirror = DaemonShareWindowMirror.from_full_items( - anchor_job_issued_at_ms=1_700_000_100_000, - window_weight=len(records), - page_size=512, - record_count=outcome.record_count, - canonical_items=outcome.window_items, - share_snapshot_sha256=outcome.share_snapshot_sha256, - ) - self.assertEqual( - [share["share_id"] for share in mirror.json_records()], - [record["share_id"] for record in records], - ) - client = server._serve_builder - assert client is not None - self.assertFalse(client.request_incomplete) - with server._serve_builder_metrics_lock: - counts = dict(server.serve_builder_counts) - self.assertEqual(counts["window_prepares"], 1) - self.assertEqual(counts["fallbacks"], 0) - finally: - server.shutdown_serve_builder() - - def test_prepare_request_that_fails_to_encode_mid_stream_retires_daemon( - self, - ) -> None: - server = self._coordinator() - records = [self._prepare_record(seq) for seq in range(1, 1_301)] - # The failure lands in the third batch: two batches are already on - # the daemon's stdin as an unterminated line when it surfaces. - records[1_200] = self._prepare_record(1_201, share_id=object()) - try: - outcome = self._prepare(server, records) - self.assertIsNone(outcome) - self.assertIsNone(server._serve_builder) - with server._serve_builder_metrics_lock: - counts = dict(server.serve_builder_counts) - self.assertEqual(counts["fallbacks"], 1) - self.assertEqual(counts["window_prepares"], 0) - # The replacement daemon starts clean and serves the next - # preparation; nothing of the partial line survived. - recovered = self._prepare(server, records[:1_200]) - self.assertEqual(recovered.status, "prepared") - self.assertEqual(recovered.record_count, 1_200) - with server._serve_builder_metrics_lock: - counts = dict(server.serve_builder_counts) - self.assertEqual(counts["spawns"], 2) - finally: - server.shutdown_serve_builder() - - def test_prepare_request_daemon_death_mid_request_degrades_in_process( - self, - ) -> None: - server = self._coordinator() - records = [self._prepare_record(seq) for seq in range(1, 3_001)] - try: - outcome = self._prepare( - server, - records, - mode="crash-during-prepare", - ) - self.assertIsNone(outcome) - self.assertIsNone(server._serve_builder) - with server._serve_builder_metrics_lock: - counts = dict(server.serve_builder_counts) - self.assertEqual(counts["fallbacks"], 1) - finally: - server.shutdown_serve_builder() - - def test_cache_miss_uploads_window_then_hits_on_next_request(self) -> None: - server = self._coordinator() - shares = [spool_share(seq) for seq in range(1, 4)] - serialization = self._serialization(server, shares) - identities, compact_shares = _compact_share_payload(shares) - expected_window = { - "compact_share_identities": [ - list(identity) for identity in identities - ], - "compact_shares": [list(share) for share in compact_shares], - } - try: - first = self._build(server, shares, serialization, height=10) - second = self._build(server, shares, serialization, height=11) - - self.assertEqual(first["transport"], "serve") - self.assertTrue(first["request_had_window"]) - self.assertEqual(first["window"], expected_window) - self.assertEqual(first["found_block"]["block_height"], 10) - - self.assertEqual(second["transport"], "serve") - self.assertFalse(second["request_had_window"]) - self.assertEqual(second["window"], expected_window) - self.assertEqual(second["found_block"]["block_height"], 11) - - with server._serve_builder_metrics_lock: - counts = dict(server.serve_builder_counts) - window_counts = dict(server.serve_builder_window_cache_counts) - self.assertEqual(counts["requests"], 2) - self.assertEqual(counts["spawns"], 1) - self.assertEqual(counts["fallbacks"], 0) - self.assertEqual(counts["window_uploads"], 1) - self.assertEqual(window_counts, {"hits": 1, "misses": 1}) - finally: - server.shutdown_serve_builder() - - def test_needs_window_bounce_reuploads_and_succeeds(self) -> None: - server = self._coordinator() - shares = [spool_share(seq) for seq in range(1, 4)] - serialization = self._serialization(server, shares) - try: - first = self._build(server, shares, serialization, height=10) - self.assertTrue(first["request_had_window"]) - client = server._serve_builder - assert client is not None - second_shares = [spool_share(seq) for seq in range(1, 5)] - second_serialization = self._serialization( - server, - second_shares, - generation=2, - ) - # Pretend this window was already uploaded: the daemon's - # needs_window bounce must repair the divergence transparently. - client.note_uploaded_window( - second_serialization.share_snapshot_sha256 # type: ignore[attr-defined] - ) - - server._ensure_tip_refresh_state() - server._job_build_phase_local.tip_refresh_metrics = True - try: - second = self._build( - server, - second_shares, - second_serialization, - height=11, - ) - finally: - server._job_build_phase_local.tip_refresh_metrics = False - - self.assertEqual(second["transport"], "serve") - self.assertTrue(second["request_had_window"]) - self.assertEqual(second["found_block"]["block_height"], 11) - with server._serve_builder_metrics_lock: - counts = dict(server.serve_builder_counts) - self.assertEqual(counts["requests"], 2) - self.assertEqual(counts["fallbacks"], 0) - # One build observes serialization_copy exactly three times -- - # fragment precompose, the accumulated transport writes, and the - # daemon's reported timings -- matching the one-shot transport - # even when a needs_window bounce re-sent the request. - histogram = server.tip_refresh_build_phase_histograms[ - "serialization_copy" - ] - self.assertEqual(histogram["count"], 3) - finally: - server.shutdown_serve_builder() - - def test_hit_promotion_keeps_mirror_and_daemon_lru_aligned(self) -> None: - server = self._coordinator() - shares_a = [spool_share(seq) for seq in range(1, 4)] - shares_b = [spool_share(seq) for seq in range(1, 5)] - shares_c = [spool_share(seq) for seq in range(1, 6)] - ser_a = self._serialization(server, shares_a, generation=1) - ser_b = self._serialization(server, shares_b, generation=2) - ser_c = self._serialization(server, shares_c, generation=3) - try: - self.assertTrue( - self._build(server, shares_a, ser_a, height=10)[ - "request_had_window" - ] - ) - self.assertTrue( - self._build(server, shares_b, ser_b, height=11)[ - "request_had_window" - ] - ) - # The hit promotes window A on both sides; uploading a third - # window must therefore evict B everywhere. - self.assertFalse( - self._build(server, shares_a, ser_a, height=12)[ - "request_had_window" - ] - ) - self.assertTrue( - self._build(server, shares_c, ser_c, height=13)[ - "request_had_window" - ] - ) - - aligned = self._build(server, shares_a, ser_a, height=14) - - self.assertEqual(aligned["transport"], "serve") - self.assertFalse(aligned["request_had_window"]) - with server._serve_builder_metrics_lock: - counts = dict(server.serve_builder_counts) - self.assertEqual(counts["requests"], 5) - self.assertEqual(counts["window_uploads"], 3) - self.assertEqual(counts["fallbacks"], 0) - finally: - server.shutdown_serve_builder() - - def test_daemon_crash_falls_back_to_one_shot(self) -> None: - server = self._coordinator() - shares = [spool_share(seq) for seq in range(1, 4)] - serialization = self._serialization(server, shares) - try: - result = self._build( - server, - shares, - serialization, - height=10, - mode="crash-before-response", - ) - - self.assertEqual(result["transport"], "one-shot") - self.assertEqual( - result["received"]["found_block"]["block_height"], - 10, - ) - self.assertIn("compact_shares", result["received"]) - with server._serve_builder_metrics_lock: - counts = dict(server.serve_builder_counts) - self.assertIsNone(server._serve_builder) - self.assertEqual(counts["fallbacks"], 1) - self.assertEqual(counts["requests"], 0) - finally: - server.shutdown_serve_builder() - - def test_daemon_splice_failure_poisons_spool_and_one_shot_recovers( - self, - ) -> None: - server = self._coordinator() - shares = [spool_share(seq) for seq in range(1, 4)] - serialization = self._serialization(server, shares) - try: - # The daemon upload dies mid-splice; the spool is poisoned before - # falling back, so the fresh one-shot subprocess streams the - # cached fragments instead of retrying the same failing transfer. - with patch( - "lab.prism.prism_coordinator.os.splice", - side_effect=[16, OSError("splice io error")], - ): - result = self._build(server, shares, serialization, height=10) - - self.assertEqual(result["transport"], "one-shot") - self.assertIn("compact_shares", result["received"]) - self.assertTrue(serialization._spool_failed) # type: ignore[attr-defined] - with server._serve_builder_metrics_lock: - counts = dict(server.serve_builder_counts) - self.assertEqual(counts["fallbacks"], 1) - finally: - server.shutdown_serve_builder() - - def test_protocol_mismatch_falls_back_to_one_shot(self) -> None: - server = self._coordinator() - shares = [spool_share(seq) for seq in range(1, 4)] - serialization = self._serialization(server, shares) - try: - result = self._build( - server, - shares, - serialization, - height=10, - mode="protocol-mismatch", - ) - - self.assertEqual(result["transport"], "one-shot") - with server._serve_builder_metrics_lock: - counts = dict(server.serve_builder_counts) - self.assertIsNone(server._serve_builder) - self.assertEqual(counts["fallbacks"], 1) - # The mismatched daemon was a real worker launch and must appear - # in the lifecycle totals alongside the one-shot fallback. - with server._job_build_scheduler_lock: - worker_counts = dict(server.job_build_worker_counts) - self.assertEqual(worker_counts["starts"], 2) - finally: - server.shutdown_serve_builder() - - def test_disabled_serve_builder_uses_one_shot_directly(self) -> None: - server = self._coordinator() - shares = [spool_share(seq) for seq in range(1, 4)] - serialization = self._serialization(server, shares) - with patch.dict(os.environ, {"PRISM_BUILDER_SERVE": "0"}): - result = self._build(server, shares, serialization, height=10) - - self.assertEqual(result["transport"], "one-shot") - with server._serve_builder_metrics_lock: - counts = dict(server.serve_builder_counts) - self.assertEqual(counts["spawns"], 0) - - def test_daemon_timeout_fallback_shares_the_build_deadline(self) -> None: - server = self._coordinator() - # An already-exhausted budget: the daemon attempt must not grant the - # one-shot fallback a fresh full deadline, and the timeout is counted - # as one worker failure, not two. - server.bundle_build_timeout_seconds = 0.0 - shares = [spool_share(seq) for seq in range(1, 4)] - serialization = self._serialization(server, shares) - - with self.assertRaisesRegex( - RuntimeError, - "qbit-prism-build-audit-bundle timed out", - ): - self._build( - server, - shares, - serialization, - height=10, - mode="hang-after-request", - ) - - with server._serve_builder_metrics_lock: - counts = dict(server.serve_builder_counts) - self.assertEqual(counts["fallbacks"], 1) - self.assertEqual(counts["spawns"], 0) - self.assertEqual(server.tip_refresh_worker_failures, 1) - # The live daemon killed at the handshake deadline is a recorded - # termination, and the one-shot replacement consumes the pending - # restart: two launches, one termination, one restart. - with server._job_build_scheduler_lock: - worker_counts = dict(server.job_build_worker_counts) - self.assertFalse(server._job_build_worker_restart_pending) - self.assertEqual(worker_counts["starts"], 2) - self.assertEqual(worker_counts["terminations"], 1) - self.assertEqual(worker_counts["restarts"], 1) - - def test_idle_daemon_death_counts_crash_and_restart(self) -> None: - server = self._coordinator() - shares = [spool_share(seq) for seq in range(1, 4)] - serialization = self._serialization(server, shares) - try: - first = self._build(server, shares, serialization, height=10) - self.assertEqual(first["transport"], "serve") - client = server._serve_builder - assert client is not None - client.process.kill() - client.process.wait(timeout=5) - - second = self._build(server, shares, serialization, height=11) - - self.assertEqual(second["transport"], "serve") - with server._job_build_scheduler_lock: - counts = dict(server.job_build_worker_counts) - self.assertFalse(server._job_build_worker_restart_pending) - self.assertEqual(counts["crashes"], 1) - self.assertEqual(counts["restarts"], 1) - self.assertEqual(server.tip_refresh_worker_restarts, 1) - with server._serve_builder_metrics_lock: - serve_counts = dict(server.serve_builder_counts) - self.assertEqual(serve_counts["spawns"], 2) - self.assertEqual(serve_counts["fallbacks"], 0) - finally: - server.shutdown_serve_builder() - - def test_daemon_respawn_counts_as_worker_restart(self) -> None: - server = self._coordinator() - shares = [spool_share(seq) for seq in range(1, 4)] - serialization = self._serialization(server, shares) - try: - # A prior worker ended abnormally (crash or supersession kill) - # with no immediate fallback consuming the flag; the replacement - # daemon spawn must claim the restart instead of leaking it to - # an unrelated later one-shot build. - server._ensure_tip_refresh_state() - with server._job_build_scheduler_lock: - server._job_build_worker_restart_pending = True - - result = self._build(server, shares, serialization, height=10) - - self.assertEqual(result["transport"], "serve") - with server._job_build_scheduler_lock: - counts = dict(server.job_build_worker_counts) - self.assertFalse(server._job_build_worker_restart_pending) - self.assertEqual(counts["restarts"], 1) - self.assertGreaterEqual(counts["starts"], 1) - self.assertEqual(server.tip_refresh_worker_restarts, 1) - finally: - server.shutdown_serve_builder() - - def test_daemon_detaches_from_build_control_after_request(self) -> None: - server = self._coordinator() - shares = [spool_share(seq) for seq in range(1, 4)] - serialization = self._serialization(server, shares) - control = prism_coordinator_module._JobBundleBuildControl( - key=("serve-detach",), - previousblockhash="11" * 32, - payout_state_generation=0, - payout_artifact_generation=1, - ) - with server._job_cache_lock: - server._active_job_bundle_builds[control.key] = control - server._job_build_phase_local.bundle_build_control = control - try: - result = self._build(server, shares, serialization, height=10) - - self.assertEqual(result["transport"], "serve") - # The completed request detached the daemon: a late supersession - # of this build has no process reference left to terminate. - self.assertIsNone(control.process) - control.cancel_event.set() - client = server._serve_builder - assert client is not None - self.assertIsNone(client.process.poll()) - finally: - server._job_build_phase_local.bundle_build_control = None - control.cancel_event.set() - with server._job_cache_lock: - server._active_job_bundle_builds.pop(control.key, None) - server.shutdown_serve_builder() - - def test_supersession_cancels_in_flight_daemon_request(self) -> None: - server = self._coordinator() - shares = [spool_share(seq) for seq in range(1, 4)] - serialization = self._serialization(server, shares) - control = prism_coordinator_module._JobBundleBuildControl( - key=("serve-test",), - previousblockhash="11" * 32, - payout_state_generation=0, - payout_artifact_generation=1, - ) - with server._job_cache_lock: - server._active_job_bundle_builds[control.key] = control - errors: list[BaseException] = [] - results: list[object] = [] - - def build() -> None: - # The bundle-build control travels on the builder thread's local - # state, exactly as _execute_job_build installs it. - server._job_build_phase_local.bundle_build_control = control - try: - results.append( - self._build( - server, - shares, - serialization, - height=10, - mode="hang-after-request", - ) - ) - except BaseException as exc: # noqa: BLE001 - surfaced below - errors.append(exc) - finally: - server._job_build_phase_local.bundle_build_control = None - - thread = threading.Thread(target=build) - thread.start() - try: - deadline = time.monotonic() + 5.0 - while time.monotonic() < deadline: - if control.process is not None: - break - time.sleep(0.005) - self.assertIsNotNone(control.process) - - control.cancel_event.set() - thread.join(5.0) - finally: - control.cancel_event.set() - thread.join(1.0) - server.shutdown_serve_builder() - - self.assertFalse(thread.is_alive()) - self.assertEqual(results, []) - self.assertEqual(len(errors), 1) - self.assertIsInstance(errors[0], JobBuildSuperseded) - self.assertIsNone(server._serve_builder) - - -class ReorgReconcileRefreshPathTests(unittest.TestCase): - """The tip-refresh reconcile stage: memo reuse and fetch overlap.""" - - def test_poll_memo_hit_skips_full_reconcile_pass(self) -> None: - server, rpc = coordinator() - install_fake_bundle_builder(server) - server.reorg_reconciler_enabled = True - state = client(1) - state.send = lambda _payload: None # type: ignore[method-assign] - server.clients = {state} - with server.lock: - server._ensure_reorg_reconciler_service()._reorg_reconcile_trusted_memo[rpc.tip] = time.monotonic() - - def unexpected_pass(**_kwargs: object) -> dict[str, object]: - raise AssertionError( - "memo-armed poll must not queue a serialized reconcile pass" - ) - - server.reconcile_prism_pool_blocks_once = ( # type: ignore[method-assign] - unexpected_pass - ) - - try: - refreshed = server.poll_qbit_tip_template_once() - finally: - server.shutdown_tip_refresh_executor() - - self.assertEqual(refreshed, 1) - # A memo hit never needs the prefetch worker. - self.assertIsNone(server._ensure_reorg_reconciler_service()._reconcile_prefetch_executor) - metrics = server.metrics_payload() - self.assertIn( - 'qbit_prism_reorg_reconcile_lookups_total{path="tip_refresh",source="memo_hit"} 1', - metrics, - ) - self.assertIn( - 'qbit_prism_tip_refresh_bundle_phase_seconds_count{phase="reorg_reconcile"} 1', - metrics, - ) - - def test_poll_overlaps_reconcile_with_template_fetch(self) -> None: - # Deadlock-free only when the reconcile pass and the template fetch - # genuinely run concurrently: the fetch refuses to return before the - # reconcile starts, and the reconcile refuses to return before the - # fetch starts. - fetch_started = threading.Event() - reconcile_running = threading.Event() - - class OverlapProbeRpc(FakeRpc): - def call( - self, - method: str, - params: list[object] | None = None, - ) -> object: - if method == "getblocktemplate": - fetch_started.set() - assert reconcile_running.wait(5.0), ( - "reconcile prefetch did not start while the template " - "fetch was in flight" - ) - return super().call(method, params) - - template = base_template() - rpc = OverlapProbeRpc(template, tip=str(template["previousblockhash"])) - server, _ = coordinator(template=template) - server.rpc = rpc - install_fake_bundle_builder(server) - server.reorg_reconciler_enabled = True - state = client(1) - state.send = lambda _payload: None # type: ignore[method-assign] - server.clients = {state} - reconciled_tips: list[str] = [] - - def fake_ensure(tip_hash: str) -> bool: - reconciled_tips.append(tip_hash) - reconcile_running.set() - assert fetch_started.wait(5.0), ( - "template fetch did not start while the reconcile pass was " - "in flight" - ) - # A real trusted pass arms the per-tip memo; the join validates - # the overlapped result against that armed entry. - with server.lock: - server._ensure_reorg_reconciler_service()._reorg_reconcile_trusted_memo[tip_hash] = time.monotonic() - return True - - server.ensure_reorg_reconciled_for_tip = ( # type: ignore[method-assign] - fake_ensure - ) - - try: - refreshed = server.poll_qbit_tip_template_once() - finally: - server.shutdown_tip_refresh_executor() - - self.assertEqual(refreshed, 1) - self.assertEqual(reconciled_tips, [rpc.tip]) - metrics = server.metrics_payload() - self.assertIn( - 'qbit_prism_reorg_reconcile_lookups_total{path="tip_refresh",source="overlap"} 1', - metrics, - ) - - def test_poll_reconciles_snapshot_tip_serially_when_tip_moves_before_fetch( - self, - ) -> None: - # The cheap probe observed a tip that was replaced before the - # template fetch. The prefetched pass proved the superseded hash, so - # the poll must reconcile the snapshot's newer tip on its own thread. - template = base_template() - new_tip = str(template["previousblockhash"]) - old_tip = "ab" * 32 - - class MovingTipRpc(FakeRpc): - def __init__(self) -> None: - super().__init__(template, tip=new_tip) - self.best_tip_calls = 0 - - def call( - self, - method: str, - params: list[object] | None = None, - ) -> object: - if method == "getbestblockhash": - self.best_tip_calls += 1 - if self.best_tip_calls == 1: - self.calls.append(method) - return old_tip - return super().call(method, params) - - rpc = MovingTipRpc() - server, _ = coordinator(template=template) - server.rpc = rpc - install_fake_bundle_builder(server) - server.reorg_reconciler_enabled = True - state = client(1) - state.send = lambda _payload: None # type: ignore[method-assign] - server.clients = {state} - reconcile_calls: list[str] = [] - prefetch_drained = threading.Event() - - def fake_ensure(tip_hash: str) -> bool: - reconcile_calls.append(tip_hash) - if tip_hash == old_tip: - prefetch_drained.set() - return True - - server.ensure_reorg_reconciled_for_tip = ( # type: ignore[method-assign] - fake_ensure - ) - - try: - refreshed = server.poll_qbit_tip_template_once() - finally: - server.shutdown_tip_refresh_executor() - - self.assertEqual(refreshed, 1) - self.assertTrue(prefetch_drained.wait(5.0)) - self.assertIn(new_tip, reconcile_calls) - self.assertEqual( - sorted(reconcile_calls), - sorted([old_tip, new_tip]), - ) - metrics = server.metrics_payload() - self.assertIn( - 'qbit_prism_reorg_reconcile_lookups_total{path="tip_refresh",source="serial"} 1', - metrics, - ) - - def test_poll_reproves_when_memo_evicted_during_template_fetch(self) -> None: - # A tip that flips away and back during the template fetch evicts - # the memo entry so post-flip pool-block state is re-proven. The - # join must judge the memo at join time, never the pre-fetch probe. - template = base_template() - server, _ = coordinator(template=template) - server.reorg_reconciler_enabled = True - tip = str(template["previousblockhash"]) - - class EvictingRpc(FakeRpc): - def call( - self, - method: str, - params: list[object] | None = None, - ) -> object: - if method == "getblocktemplate": - # Models a detection during the fetch unarming the memo. - with server.lock: - server._ensure_reorg_reconciler_service()._reorg_reconcile_trusted_memo.clear() - return super().call(method, params) - - rpc = EvictingRpc(template, tip=tip) - server.rpc = rpc - install_fake_bundle_builder(server) - state = client(1) - state.send = lambda _payload: None # type: ignore[method-assign] - server.clients = {state} - with server.lock: - server._ensure_reorg_reconciler_service()._reorg_reconcile_trusted_memo[tip] = time.monotonic() - reconcile_calls: list[str] = [] - - def fake_ensure(tip_hash: str) -> bool: - reconcile_calls.append(tip_hash) - return True - - server.ensure_reorg_reconciled_for_tip = ( # type: ignore[method-assign] - fake_ensure - ) - - try: - refreshed = server.poll_qbit_tip_template_once() - finally: - server.shutdown_tip_refresh_executor() - - self.assertEqual(refreshed, 1) - self.assertEqual(reconcile_calls, [tip]) - metrics = server.metrics_payload() - self.assertIn( - 'qbit_prism_reorg_reconcile_lookups_total{path="tip_refresh",source="serial"} 1', - metrics, - ) - self.assertIn( - 'qbit_prism_reorg_reconcile_lookups_total{path="tip_refresh",source="memo_hit"} 0', - metrics, - ) - - def test_poll_reproves_when_memo_rearmed_across_detection_epoch(self) -> None: - # A pass whose execution straddles a flip away and back can re-arm - # the memo after the flip-back (the latest detected hash matches - # again), but its proof belongs to the closed epoch. The join must - # refuse memo entries when the detection epoch moved mid-fetch. - template = base_template() - server, _ = coordinator(template=template) - server.reorg_reconciler_enabled = True - tip = str(template["previousblockhash"]) - - class FlipBackRpc(FakeRpc): - def call( - self, - method: str, - params: list[object] | None = None, - ) -> object: - if method == "getblocktemplate": - # Models a flip away and back detected during the fetch - # (two epoch bumps) with a straddling pass re-arming the - # memo for the same hash afterwards. - with server.lock: - server.tip_detection_epoch = ( - int(getattr(server, "tip_detection_epoch", 0)) + 2 - ) - server._ensure_reorg_reconciler_service()._reorg_reconcile_trusted_memo[tip] = ( - time.monotonic() - ) - return super().call(method, params) - - rpc = FlipBackRpc(template, tip=tip) - server.rpc = rpc - install_fake_bundle_builder(server) - state = client(1) - state.send = lambda _payload: None # type: ignore[method-assign] - server.clients = {state} - with server.lock: - server._ensure_reorg_reconciler_service()._reorg_reconcile_trusted_memo[tip] = time.monotonic() - reconcile_calls: list[str] = [] - - def fake_ensure(tip_hash: str) -> bool: - reconcile_calls.append(tip_hash) - return True - - server.ensure_reorg_reconciled_for_tip = ( # type: ignore[method-assign] - fake_ensure - ) - - try: - refreshed = server.poll_qbit_tip_template_once() - finally: - server.shutdown_tip_refresh_executor() - - self.assertEqual(refreshed, 1) - self.assertEqual(reconcile_calls, [tip]) - metrics = server.metrics_payload() - self.assertIn( - 'qbit_prism_reorg_reconcile_lookups_total{path="tip_refresh",source="serial"} 1', - metrics, - ) - - def test_overlap_join_blocks_when_trust_flips_after_prefetch(self) -> None: - # The prefetched pass proves trust when it runs, up to a fetch - # window earlier. Headers running ahead without a best-hash change - # (an arriving reorg, no detection) must still block the refresh at - # the join, matching the memo branch's live trust check. - template = base_template() - server, _ = coordinator(template=template) - server.reorg_reconciler_enabled = True - tip = str(template["previousblockhash"]) - - class TrustFlipRpc(FakeRpc): - def call( - self, - method: str, - params: list[object] | None = None, - ) -> object: - if method == "getblocktemplate": - # Headers run ahead after the fetch: untrusted view with - # no tip detection (epoch unchanged). - self.blockchain_info["headers"] = 101 - return super().call(method, params) - - rpc = TrustFlipRpc(template, tip=tip) - server.rpc = rpc - install_fake_bundle_builder(server) - state = client(1) - state.send = lambda _payload: None # type: ignore[method-assign] - server.clients = {state} - server.ensure_reorg_reconciled_for_tip = ( # type: ignore[method-assign] - lambda _tip: True - ) - - try: - with self.assertRaises(TemplateRefreshBlocked) as raised: - server.poll_qbit_tip_template_once() - finally: - server.shutdown_tip_refresh_executor() - - self.assertIn("untrusted", str(raised.exception)) - self.assertIsNone(state.active_job) - - def test_tip_refresh_join_bounds_a_stalled_reconcile_prefetch(self) -> None: - # The overlap join must never park the poll loop on a crawling - # prefetched pass: it times out into the normal blocked-retry path - # within the CONFIGURED budget (not merely eventually), retries - # re-join the identical still-running future without starting - # another pass, and the pass completing unblocks the next poll. - server, rpc = coordinator() - server.reorg_reconciler_enabled = True - server.reconcile_prefetch_join_timeout_seconds = 0.05 - started = threading.Event() - release = threading.Event() - pass_calls = [0] - - def slow_ensure(tip_hash: str) -> bool: - pass_calls[0] += 1 - started.set() - assert release.wait(10.0) - return True - - server.ensure_reorg_reconciled_for_tip = ( # type: ignore[method-assign] - slow_ensure - ) - try: - join_started = time.monotonic() - with self.assertRaises(TemplateRefreshBlocked): - server.poll_qbit_tip_template_once() - # Generous scheduler tolerance, but far below any hardcoded - # multi-second budget: the configured 0.05s must be what fired. - self.assertLess(time.monotonic() - join_started, 1.0) - self.assertTrue(started.is_set()) - with server._ensure_reorg_reconciler_service()._reconcile_prefetch_executor_lock: - pending = server._ensure_reorg_reconciler_service()._reconcile_prefetch_pending - self.assertIsNotNone(pending) - assert pending is not None - first_future = pending[1] - self.assertFalse(first_future.done()) - - # A retry re-joins the same running future: same slot identity, - # no additional pass started. - with self.assertRaises(TemplateRefreshBlocked): - server.poll_qbit_tip_template_once() - with server._ensure_reorg_reconciler_service()._reconcile_prefetch_executor_lock: - pending_again = server._ensure_reorg_reconciler_service()._reconcile_prefetch_pending - assert pending_again is not None - self.assertIs(pending_again[1], first_future) - self.assertEqual(pass_calls[0], 1) - - release.set() - self.assertTrue(first_future.result(5.0)) - server.poll_qbit_tip_template_once() - finally: - release.set() - server.shutdown_reconcile_prefetch_executor() - - def test_serial_reconcile_branch_uses_the_bounded_join(self) -> None: - # With the memo disabled no overlap prefetch exists, so the pass - # lands in the serial re-prove branch -- which must route through - # the same prefetch slot and bounded join instead of parking the - # poll loop synchronously inside a crawling pass. - server, rpc = coordinator() - server.reorg_reconciler_enabled = True - server.reorg_reconcile_cache_seconds = 0.0 - server.reconcile_prefetch_join_timeout_seconds = 0.05 - started = threading.Event() - release = threading.Event() - - def slow_ensure(tip_hash: str) -> bool: - started.set() - assert release.wait(10.0) - return True - - server.ensure_reorg_reconciled_for_tip = ( # type: ignore[method-assign] - slow_ensure - ) - try: - join_started = time.monotonic() - with self.assertRaises(TemplateRefreshBlocked): - server.poll_qbit_tip_template_once() - self.assertLess(time.monotonic() - join_started, 1.0) - self.assertTrue(started.is_set()) - with server._ensure_reorg_reconciler_service()._reconcile_prefetch_executor_lock: - pending = server._ensure_reorg_reconciler_service()._reconcile_prefetch_pending - self.assertIsNotNone(pending) - assert pending is not None - self.assertFalse(pending[1].done()) - - release.set() - self.assertTrue(pending[1].result(5.0)) - server.poll_qbit_tip_template_once() - finally: - release.set() - server.shutdown_reconcile_prefetch_executor() - - def test_pass_raised_timeout_is_not_logged_as_join_expiry(self) -> None: - # socket.timeout IS builtins.TimeoutError: a pass that raises it - # completes the future exceptionally and must surface as an ordinary - # blocked pass -- not as a join expiry, which would misdirect - # operators toward the join while nothing is running. - server, rpc = coordinator() - server.reorg_reconciler_enabled = True - server.reorg_reconcile_cache_seconds = 0.0 - server.reconcile_prefetch_join_timeout_seconds = 5.0 - - def timing_out_ensure(tip_hash: str) -> bool: - raise TimeoutError("pass-owned timeout") - - server.ensure_reorg_reconciled_for_tip = ( # type: ignore[method-assign] - timing_out_ensure - ) - try: - captured = io.StringIO() - join_started = time.monotonic() - with contextlib.redirect_stdout(captured): - with self.assertRaises(TemplateRefreshBlocked): - server.poll_qbit_tip_template_once() - self.assertLess(time.monotonic() - join_started, 2.0) - self.assertNotIn("join exceeded", captured.getvalue()) - finally: - server.shutdown_reconcile_prefetch_executor() - - def test_reconcile_prefetch_slot_is_reused_across_failed_attempts(self) -> None: - # A refresh attempt that dies before its join (template-RPC outage) - # must not queue another serialized pass per retry: the slot holds - # at most one outstanding prefetch, reused for the same tip and - # replaced (with the queued task cancelled) on a tip change. - server, _ = coordinator() - server.reorg_reconciler_enabled = True - started = threading.Event() - release = threading.Event() - - def slow_ensure(tip_hash: str) -> bool: - started.set() - assert release.wait(5.0) - return True - - server.ensure_reorg_reconciled_for_tip = ( # type: ignore[method-assign] - slow_ensure - ) - try: - first = server._submit_reconcile_prefetch("aa" * 32) - assert first is not None - self.assertTrue(started.wait(5.0)) - second = server._submit_reconcile_prefetch("aa" * 32) - self.assertIs(second, first) - third = server._submit_reconcile_prefetch("bb" * 32) - assert third is not None - self.assertIsNot(third, first) - fourth = server._submit_reconcile_prefetch("cc" * 32) - assert fourth is not None - self.assertIsNot(fourth, third) - self.assertTrue(third.cancelled()) - release.set() - self.assertTrue(first.result(5.0)) - self.assertTrue(fourth.result(5.0)) - finally: - release.set() - server.shutdown_reconcile_prefetch_executor() - - def test_job_build_memo_lookups_are_counted(self) -> None: - server, rpc = coordinator() - server.reorg_reconciler_enabled = True - with server.lock: - server._ensure_reorg_reconciler_service()._reorg_reconcile_trusted_memo[rpc.tip] = time.monotonic() - - self.assertTrue(server.ensure_reorg_reconciled_for_current_tip()) - - with server.lock: - server._ensure_reorg_reconciler_service()._reorg_reconcile_trusted_memo[rpc.tip] = ( - time.monotonic() - 100.0 - ) - serial_tips: list[str] = [] - - def fake_ensure(tip_hash: str) -> bool: - serial_tips.append(tip_hash) - return True - - server.ensure_reorg_reconciled_for_tip = ( # type: ignore[method-assign] - fake_ensure - ) - - self.assertTrue(server.ensure_reorg_reconciled_for_current_tip()) - - self.assertEqual(serial_tips, [rpc.tip]) - metrics = server.metrics_payload() - self.assertIn( - 'qbit_prism_reorg_reconcile_lookups_total{path="job_build",source="memo_hit"} 1', - metrics, - ) - self.assertIn( - 'qbit_prism_reorg_reconcile_lookups_total{path="job_build",source="serial"} 1', - metrics, - ) - - def test_poll_falls_back_to_serial_reconcile_after_prefetch_shutdown( - self, - ) -> None: - server, rpc = coordinator() - install_fake_bundle_builder(server) - server.reorg_reconciler_enabled = True - state = client(1) - state.send = lambda _payload: None # type: ignore[method-assign] - server.clients = {state} - server.shutdown_reconcile_prefetch_executor() - reconcile_calls: list[str] = [] - - def fake_ensure(tip_hash: str) -> bool: - reconcile_calls.append(tip_hash) - return True - - server.ensure_reorg_reconciled_for_tip = ( # type: ignore[method-assign] - fake_ensure - ) - - try: - refreshed = server.poll_qbit_tip_template_once() - finally: - server.shutdown_tip_refresh_executor() - - self.assertEqual(refreshed, 1) - self.assertEqual(reconcile_calls, [rpc.tip]) - self.assertIsNone(server._ensure_reorg_reconciler_service()._reconcile_prefetch_executor) - metrics = server.metrics_payload() - self.assertIn( - 'qbit_prism_reorg_reconcile_lookups_total{path="tip_refresh",source="serial"} 1', - metrics, - ) - - -class JobContextStampTests(unittest.TestCase): - def test_job_context_stamps_client_version_mask(self) -> None: - # Cross-connection resumes validate version bits against the mask - # negotiated on the delivering connection, so the context must - # record it at stamp time. - server, _ = coordinator() - install_fake_bundle_builder(server) - state = client(1) - state.version_mask = 0x1FFFE000 - - context = server.build_job_for_client(state, clean_jobs=True) - - self.assertEqual(context.version_mask, 0x1FFFE000) - - -class ExternallyTerminatedBuilderTests(unittest.TestCase): - """A build superseded while its helper runs terminates without a crash.""" - - def test_superseded_running_builder_terminates_without_crash(self) -> None: - server, rpc = coordinator() - server.signing_seed_hex = "42" * 32 - server.ledger_attestation_signing_seed_hex = "43" * 32 - artifacts = server.store_template_artifacts(dict(rpc.template)) - assert artifacts is not None - real_popen = subprocess.Popen - - def build_with_control(*_args: object, **kwargs: object) -> object: - build_request = kwargs["build_request"] - return server.build_audit_bundle( - shares=[], - found_block={ - "block_height": 10, - "coinbase_value_sats": 50_00000000, - "network_difficulty": 1, - "anchor_job_issued_at_ms": 1_700_000_000_000, - }, - prior_balances=[], - coinbase_script_sig_suffix_hex="00", - cancellation=build_request.cancellation, # type: ignore[union-attr] - ) - - server.build_shared_job_bundle = build_with_control # type: ignore[method-assign] - - def cancel_then_exit( - *args: object, - **kwargs: object, - ) -> subprocess.Popen[str]: - process = real_popen(*args, **kwargs) # type: ignore[arg-type] - original_poll = process.poll - original_wait = process.wait - first_poll = True - - def poll() -> int | None: - nonlocal first_poll - if not first_poll: - return original_poll() - first_poll = False - with server._job_cache_lock: - controls = list(server._active_job_bundle_builds.values()) - self.assertEqual(len(controls), 1) - # Supersession lands while the helper is still running; the - # compiler owns the terminate/kill sequence itself, so this - # never counts as a worker crash. - controls[0].cancel_event.set() - return None - - process.poll = poll # type: ignore[method-assign] - del original_wait - return process - - with patch( - "lab.prism.prism_coordinator.prism_tool_command", - return_value=[sys.executable, "-c", "import time; time.sleep(30)"], - ), patch( - "lab.prism.prism_coordinator.subprocess.Popen", - side_effect=cancel_then_exit, - ): - with self.assertRaises(JobBuildSuperseded): - server.shared_job_bundle( - artifacts, - mode="ready", - retry_superseded=False, - ) - - service = server._ensure_job_bundle_service() - self.assertEqual(service.shared_bundle_build_counts["superseded"], 1) - self.assertEqual(service.shared_bundle_build_counts["failed"], 0) - self.assertEqual(server.job_build_worker_counts["terminations"], 1) - self.assertEqual(server.job_build_worker_counts["crashes"], 0) - self.assertEqual(server.job_build_failure_count, 0) diff --git a/tests/test_prism_job_delivery.py b/tests/test_prism_job_delivery.py deleted file mode 100644 index 1c41f360b..000000000 --- a/tests/test_prism_job_delivery.py +++ /dev/null @@ -1,207 +0,0 @@ -#!/usr/bin/env python3 -"""Direct contract tests for the coordinator-free S2 delivery boundary. - -The current owner keeps one call-time ``JobDeliveryRuntime`` typed port and -descriptor-routed compatibility state (the old stack's per-capability port -dataclasses and authority objects were deliberately not built). These cases -pin the boundary itself: symbol identity, descriptor routing and adoption, -the narrow ``next_job_id`` / ``complete_delivery`` adapters, and legacy -retained-state conversion. Delivery behavior (stamping, retained jobs, -initial jobs, difficulty adjacency) is asserted by the upstream suites in -``tests/test_prism_retained_jobs.py``, ``tests/test_prism_initial_job_delivery.py``, -``tests/test_prism_tip_refresh_delivery.py`` and -``tests/test_prism_job_builder.py`` (``JobContextStampTests``). -""" - -from __future__ import annotations - -from collections import OrderedDict -import time -from types import SimpleNamespace -import unittest - -import lab.prism.job_delivery as job_delivery_module -from lab.prism import prism_coordinator -from lab.prism.job_delivery import ( - EvictedJobEntry, - JobBuildFailed, - PendingInitialJob, - PrismJobContext, -) -from lab.prism.prism_coordinator import ( - EvictedJobEntry as FacadeEvictedJobEntry, - PendingInitialJob as FacadePendingInitialJob, - PrismJobContext as FacadePrismJobContext, -) -from tests.prism_vardiff_test_support import client, coordinator - -TIP_A = "11" * 32 - - -class S2FacadeIdentityTests(unittest.TestCase): - def test_facade_reexports_exact_s2_identities(self) -> None: - self.assertIs(FacadePrismJobContext, PrismJobContext) - self.assertIs(FacadeEvictedJobEntry, EvictedJobEntry) - self.assertIs(FacadePendingInitialJob, PendingInitialJob) - self.assertIs(prism_coordinator._JobBuildFailed, JobBuildFailed) - # PR 80 removed the admission-poll compatibility re-export. - self.assertFalse( - hasattr(prism_coordinator, "PRISM_TIP_REFRESH_ADMISSION_POLL_SECONDS") - ) - # The owner is a leaf: it never imports the coordinator module. - self.assertFalse(hasattr(job_delivery_module, "PrismCoordinator")) - - -class S2DescriptorRoutingTests(unittest.TestCase): - def test_coordinator_descriptors_route_s2_state_to_owner(self) -> None: - server = coordinator() - service = server._ensure_job_delivery_service() - - self.assertIs(server.jobs, service.jobs) - self.assertIs(server.pending_initial_jobs, service.pending_initial_jobs) - self.assertIs(server.evicted_job_graveyard, service.evicted_job_graveyard) - - server.job_counter = 5 - self.assertEqual(service.job_counter, 5) - service.initial_job_sent_count = 3 - self.assertEqual(server.initial_job_sent_count, 3) - - def test_coordinator_adopts_replaced_jobs_and_pending_initial_mappings( - self, - ) -> None: - server = coordinator() - service = server._ensure_job_delivery_service() - - jobs_replacement: dict[str, PrismJobContext] = {} - server.jobs = jobs_replacement - self.assertIs(service.jobs, jobs_replacement) - - # The admission map is private to the owner and every access to it is - # taken under the admission lock, so assigning through the legacy name - # adopts the mapping's contents rather than retaining (and exposing) - # the caller's mutable object (#159). - state = client() - request = PendingInitialJob( - client=state, - authorization_generation=1, - worker=state.worker, - requested_monotonic=time.monotonic(), - deadline_monotonic=None, - connection_id=state.connection_id, - difficulty_generation=0, - ) - pending_replacement: dict[object, PendingInitialJob] = {state: request} - server.pending_initial_jobs = pending_replacement - - self.assertIsNot(service.pending_initial_jobs, pending_replacement) - self.assertIs(service.pending_initial_jobs[state], request) - self.assertEqual(len(server.pending_initial_jobs), 1) - # Adoption is a copy: later admission changes are not written back - # into the caller's dict, and the caller cannot mutate admission state - # behind the lock by holding on to it. - server.cancel_initial_job_delivery(state) - self.assertEqual(len(server.pending_initial_jobs), 0) - self.assertEqual(pending_replacement, {state: request}) - - -class S2NarrowAdapterTests(unittest.TestCase): - def test_next_job_id_allocates_through_the_owner_counter(self) -> None: - server = coordinator() - service = server._ensure_job_delivery_service() - server.job_counter = 41 - - self.assertEqual(service.next_job_id(), "prism-42") - # The coordinator adapter is a pure delegate over the same counter. - self.assertEqual(server._next_job_delivery_id(), "prism-43") - self.assertEqual(server.job_counter, 43) - - def test_complete_delivery_notes_tip_work_before_progress_proof(self) -> None: - server = coordinator() - service = server._ensure_job_delivery_service() - state = client() - events: list[tuple[str, object]] = [] - server.note_tip_work_delivered = ( # type: ignore[method-assign] - lambda delivered_client, parent: events.append( - ("tip", (delivered_client, parent)) - ) - ) - server._record_progress_delivery = ( # type: ignore[method-assign] - lambda delivered_client, context, monotonic: events.append( - ("progress", (delivered_client, context, monotonic)) - ) - ) - context = SimpleNamespace(template={"previousblockhash": TIP_A}) - - service.complete_delivery(state, context, 3.5) - - self.assertEqual( - [name for name, _payload in events], ["tip", "progress"] - ) - self.assertEqual(events[0][1], (state, TIP_A)) - self.assertEqual(events[1][1], (state, context, 3.5)) - - # The coordinator adapter routes through the same owner method. - events.clear() - server._complete_job_delivery(state, context, 4.5) - self.assertEqual( - [name for name, _payload in events], ["tip", "progress"] - ) - - -class RetainedStateAdoptionTests(unittest.TestCase): - def test_ensure_evicted_job_state_converts_legacy_tuple_entries(self) -> None: - server = coordinator() - service = server._ensure_job_delivery_service() - live_client = SimpleNamespace(connection_id=7) - server.clients = [live_client] - context = SimpleNamespace(template={"previousblockhash": TIP_A}) - evicted_at = time.monotonic() - # A legacy plain-dict graveyard with tuple entries, seeded through the - # compatibility field exactly as pre-extraction tests did. - server.evicted_job_graveyard = { - "prism-legacy": (context, 7, evicted_at), - } - - server._ensure_evicted_job_state() - - graveyard = service.evicted_job_graveyard - self.assertIsInstance(graveyard, OrderedDict) - entry = graveyard["prism-legacy"] - self.assertIsInstance(entry, EvictedJobEntry) - self.assertIs(entry.context, context) - self.assertEqual(entry.connection_id, 7) - self.assertEqual(entry.evicted_monotonic, evicted_at) - self.assertEqual(entry.previousblockhash, TIP_A) - self.assertIs(entry.client, live_client) - # Conversion rebuilds the per-connection retained indexes. - self.assertIn(7, service.evicted_jobs_by_connection) - self.assertIn("prism-legacy", service.evicted_jobs_by_connection[7]) - - def test_ensure_evicted_job_state_rebuilds_indexes_on_tip_change(self) -> None: - server = coordinator() - service = server._ensure_job_delivery_service() - server.clients = set() - context = SimpleNamespace(template={"previousblockhash": TIP_A}) - entry = EvictedJobEntry( - context=context, - connection_id=3, - evicted_monotonic=time.monotonic(), - previousblockhash=TIP_A, - client=None, - ) - service.evicted_job_graveyard = OrderedDict({"prism-kept": entry}) - service.evicted_jobs_by_connection = {} - service.evicted_job_index_tip_hash = "stale-tip" - - server._ensure_evicted_job_state() - - self.assertIn(3, service.evicted_jobs_by_connection) - self.assertIn("prism-kept", service.evicted_jobs_by_connection[3]) - self.assertEqual( - service.evicted_job_index_tip_hash, - server._current_published_tip_hash_locked(), - ) - - -if __name__ == "__main__": - unittest.main() diff --git a/tests/test_prism_l3_promote.py b/tests/test_prism_l3_promote.py new file mode 100644 index 000000000..67f741400 --- /dev/null +++ b/tests/test_prism_l3_promote.py @@ -0,0 +1,182 @@ +"""Tests for scripts/prism_l3_promote.py (#550): the tag's promote-or-rerun.""" + +from __future__ import annotations + +import contextlib +import io +import json +from pathlib import Path +import subprocess +import sys +import unittest + +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT / "scripts")) + +import prism_l3_promote as promote # noqa: E402 + +REPO = "Qbit-Org/qbit-mining-bootstrap" +TREE = "a" * 40 +OTHER_TREE = "b" * 40 +THIS_RUN = 900 + + +def verdict(**overrides) -> dict: + value = { + "schema": promote.VERDICT_SCHEMA, + "suite": "l3-full", + "tree": TREE, + "complete": True, + "passed": True, + } + value.update(overrides) + return value + + +def run(run_id: int, **overrides) -> dict: + value = { + "id": run_id, + "event": "pull_request", + "path": promote.WORKFLOW_PATH, + "status": "completed", + "conclusion": "success", + "head_repository": {"full_name": REPO}, + "html_url": f"https://github.com/{REPO}/actions/runs/{run_id}", + "pull_requests": [{"number": 600}], + } + value.update(overrides) + return value + + +class FakeGh: + """Answers the three calls the script makes, from canned runs.""" + + def __init__(self, runs: list[tuple[dict, dict | None, str]], fail_listing: bool = False): + # (run, verdict or None for an unreadable one, artifact created_at) + self.runs = {r["id"]: (r, v) for r, v, _ in runs} + self.created = {r["id"]: created for r, _, created in runs} + self.fail_listing = fail_listing + self.downloads: list[int] = [] + + def __call__(self, args: list[str]) -> str: + if args[0] == "api" and "actions/artifacts" in args[1]: + if self.fail_listing: + raise subprocess.CalledProcessError(1, ["gh", *args], stderr="HTTP 502") + self.listing_query = args[1] + return json.dumps({"artifacts": [ + {"workflow_run": {"id": run_id}, "expired": False, "created_at": created} + for run_id, created in self.created.items() + ]}) + if args[0] == "api" and "actions/runs/" in args[1]: + return json.dumps(self.runs[int(args[1].rsplit("/", 1)[1])][0]) + if args[:2] == ["run", "download"]: + run_id = int(args[2]) + self.downloads.append(run_id) + found = self.runs[run_id][1] + if found is None: + raise subprocess.CalledProcessError(1, ["gh", *args], stderr="no artifact") + directory = Path(args[args.index("-D") + 1]) + (directory / "verdict.json").write_text(json.dumps(found), encoding="utf-8") + return "" + raise AssertionError(f"unexpected gh call {args}") + + +class Decide(unittest.TestCase): + def test_a_matching_complete_pull_request_run_is_promoted(self) -> None: + gh = FakeGh([(run(10), verdict(), "2026-09-30T10:00:00Z")]) + decision = promote.decide(gh, REPO, TREE, THIS_RUN) + self.assertTrue(decision["promote"], decision) + self.assertEqual(decision["source_run"], 10) + self.assertEqual(decision["source_pull_requests"], [600]) + # Only the artifact named for this tree is listed. + self.assertIn(f"name=prism-l3-verdict-{TREE}", gh.listing_query) + + def test_no_run_on_this_tree_reruns(self) -> None: + decision = promote.decide(FakeGh([]), REPO, TREE, THIS_RUN) + self.assertFalse(decision["promote"]) + self.assertTrue(decision["reason"].startswith("rerun:")) + + def test_a_verdict_for_another_tree_is_not_promoted(self) -> None: + gh = FakeGh([(run(10), verdict(tree=OTHER_TREE), "2026-09-30T10:00:00Z")]) + decision = promote.decide(gh, REPO, TREE, THIS_RUN) + self.assertFalse(decision["promote"]) + self.assertIn("tree", decision["considered"][0]["rejected"]) + + def test_the_newest_eligible_run_wins_and_ineligible_ones_are_recorded(self) -> None: + gh = FakeGh([ + (run(10), verdict(passed=False), "2026-09-29T10:00:00Z"), + (run(11), verdict(), "2026-09-30T10:00:00Z"), + (run(12, head_repository={"full_name": "someone/fork"}), verdict(), "2026-09-30T11:00:00Z"), + (run(13, event="workflow_dispatch"), verdict(), "2026-09-30T12:00:00Z"), + (run(14, conclusion="cancelled"), verdict(), "2026-09-30T13:00:00Z"), + (run(15), verdict(complete=False), "2026-09-30T14:00:00Z"), + (run(16), verdict(suite="l3-reduced"), "2026-09-30T15:00:00Z"), + (run(17), None, "2026-09-30T16:00:00Z"), + (run(18, path=".github/workflows/prism-load-nightly.yml"), verdict(), "2026-09-30T17:00:00Z"), + ]) + decision = promote.decide(gh, REPO, TREE, THIS_RUN) + self.assertTrue(decision["promote"], decision) + self.assertEqual(decision["source_run"], 11) + rejected = {entry["run"]: entry["rejected"] for entry in decision["considered"]} + self.assertEqual(sorted(rejected), [12, 13, 14, 15, 16, 17, 18]) + self.assertIn("fork", rejected[12]) + self.assertIn("workflow_dispatch", rejected[13]) + self.assertIn("cancelled", rejected[14]) + self.assertIn("every planned run", rejected[15]) + self.assertIn("l3-reduced", rejected[16]) + self.assertIn("could not be read", rejected[17]) + self.assertIn("prism-load-nightly", rejected[18]) + # Runs that fail on their metadata are never downloaded. + self.assertNotIn(12, gh.downloads) + self.assertNotIn(10, gh.downloads) + + def test_a_complete_run_that_failed_a_gate_is_promoted_with_its_verdict(self) -> None: + gh = FakeGh([(run(10, conclusion="failure"), verdict(passed=False), "2026-09-30T10:00:00Z")]) + decision = promote.decide(gh, REPO, TREE, THIS_RUN) + self.assertTrue(decision["promote"]) + self.assertFalse(decision["verdict"]["passed"]) + self.assertIn("not every run passed", decision["reason"]) + + def test_this_run_is_never_its_own_source(self) -> None: + gh = FakeGh([(run(THIS_RUN), verdict(), "2026-09-30T10:00:00Z")]) + self.assertFalse(promote.decide(gh, REPO, TREE, THIS_RUN)["promote"]) + + def test_a_failed_lookup_reruns_rather_than_promotes(self) -> None: + decision = promote.decide(FakeGh([], fail_listing=True), REPO, TREE, THIS_RUN) + self.assertFalse(decision["promote"]) + self.assertIn("lookup failed", decision["reason"]) + + def test_an_artifact_naming_no_run_is_skipped_not_a_crash(self) -> None: + gh = FakeGh([(run(10), verdict(), "2026-09-30T10:00:00Z")]) + listing = gh.__call__ + + def with_orphan(args: list[str]) -> str: + out = listing(args) + if args[0] == "api" and "actions/artifacts" in args[1]: + body = json.loads(out) + body["artifacts"].insert(0, {"workflow_run": None, "expired": False, "created_at": "2026-10-01T00:00:00Z"}) + body["artifacts"].insert(0, {"expired": False, "created_at": "2026-10-01T00:00:00Z"}) + out = json.dumps(body) + return out + + decision = promote.decide(with_orphan, REPO, TREE, THIS_RUN) + self.assertTrue(decision["promote"], decision) + self.assertEqual(decision["source_run"], 10) + + def test_a_verdict_without_a_pass_flag_is_not_promoted(self) -> None: + broken = verdict() + del broken["passed"] + gh = FakeGh([(run(10), broken, "2026-09-30T10:00:00Z")]) + self.assertFalse(promote.decide(gh, REPO, TREE, THIS_RUN)["promote"]) + + +class Arguments(unittest.TestCase): + def test_a_tree_that_is_not_a_full_id_is_refused(self) -> None: + with contextlib.redirect_stderr(io.StringIO()): + with self.assertRaises(SystemExit): + promote.main(["--repo", REPO, "--run-id", "1"]) + self.assertEqual(promote.main(["--repo", REPO, "--tree", "HEAD", "--run-id", "1"]), 2) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_prism_landing_append_epoch.py b/tests/test_prism_landing_append_epoch.py deleted file mode 100644 index 8534b9d27..000000000 --- a/tests/test_prism_landing_append_epoch.py +++ /dev/null @@ -1,515 +0,0 @@ -#!/usr/bin/env python3 -"""The append-invalidation epoch, asked per anchor instead of per counter. - -Issue #126: the epoch is one global counter, so a landing that compares its -own baseline against the live value learns only that *somebody's* payout -window was invalidated. A share that becomes durable stamped later than this -candidate's declared anchor -- but early enough to predate some newer exposed -window -- bumps that counter without touching this candidate's window. On the -fallback submit lane, where nothing has been offered yet, the pre-offer fence -read the inequality and terminally abandoned the block as -``append_epoch_stale``: it never reached qbitd at all, and a valid block's -reward was forfeited. - -What the fix may not do is forgive a window that genuinely was invalidated, -so these scenarios are written in pairs. The same interleaving, at the same -fence, differs only in the stamp of the appended row: one stamp that cannot -predate the candidate's declared anchor must land, and one that can must -still abandon with nothing submitted. The boundary itself (a stamp exactly -equal to the anchor) is on the terminal side, because -``_pending_share_predates_anchor`` treats an equal stamp as predating. - -Both landing fences are covered. The pre-offer one is advisory -- it reads -the epoch under the job-cache lock and releases it -- so its pair drains the -appender before the landing starts. The authoritative one holds -``_payout_append_landing_fence_lock`` across the read and ``submitblock``, so -its pair parks the accounting tail at ``phase:tip-height-rpc`` (past the -advisory read, before the fence arms) and lets the appender commit its bump -into exactly that gap. - -The two fail-closed cases that have nothing to do with stamps are here too: -a baseline epoch older than the retained stamp history cannot be answered, -and neither can a candidate that declares no anchor. Both abandon, which is -what the bare counter did for every candidate. - -The scoped predicate reads recorded bumps, so it is only as good as the -guarantee that every predating append bumps. The disarm-gap scenario pins -that guarantee at its one weak point: a seeded window's anchor is exposed -only by the armed artifact, and the bump that consults it also disarms it, -so without the watermark fold a second append could predate the candidate's -window with the anchor set empty -- no bump, no stamp, and a fence that -reads only harmless history submits an invalidated window. - -Finally, the recorded stamp itself is pinned. A row predates an anchor iff -BOTH its stamps do -- iff their max does -- and every equal-stamp scenario -above records the same number under max(), min(), or either stamp alone. -The straddling pair pulls the stamps apart so only max() survives it. -""" - -from __future__ import annotations - -import sys -import unittest -from pathlib import Path -from types import SimpleNamespace -from typing import Any - -sys.path.insert(0, str(Path(__file__).resolve().parents[1])) - -from lab.prism.payout_state import ( # noqa: E402 - PRISM_PAYOUT_APPEND_INVALIDATION_STAMP_HISTORY, -) -from tests.prism_concurrency_harness import assert_deterministic # noqa: E402 -from tests.prism_landing_harness import LandingHarness # noqa: E402 - -BLOCK = "cc" * 32 - -# The candidate's own window. Every share stamp below is placed relative to -# this: at or before it invalidates the candidate, after it does not. -ANCHOR_MS = 12_000 -# A newer published window, live while this candidate lands. It is what makes -# an append that misses the candidate's window still bump the global epoch -- -# without it there would be no defect to scope, because nothing would predate -# a live anchor at all. -NEWER_WINDOW_ANCHOR_MS = 20_000 -# Stamped after the candidate's anchor and before the newer window's: it -# invalidates that newer window and nothing of this candidate's. -LATER_THAN_ANCHOR_MS = 15_000 -# Stamped inside the candidate's own window, and on its exact boundary. -INSIDE_ANCHOR_MS = 9_000 -AT_ANCHOR_MS = ANCHOR_MS -# A newer SEEDED window. Unlike NEWER_WINDOW_ANCHOR_MS it is never handed -# to the published-window watermark: its anchor is exposed only by the -# armed payout-ledger artifact, which the first bump disarms -- the -# precondition for the disarm gap. -NEWER_SEEDED_ANCHOR_MS = 20_000 -# The acceptance stamp of a row whose two stamps differ while both sit -# inside the candidate's window. -ALSO_INSIDE_ANCHOR_MS = 11_000 - -# The last phase the landing names before the fallback lane's authoritative -# fence. Stopping here puts a tail past the advisory pre-offer read, with no -# fence lock held and ``submitblock`` not yet reached -- the one gap in which -# an append-side bump can still overtake a landing that already checked. -BEFORE_AUTHORITATIVE_FENCE = "phase:tip-height-rpc" - - -def _landing_outcome(harness: LandingHarness, found: Any) -> dict[str, object]: - """The durable verdict on one candidate, as a comparable digest.""" - row = harness.pool_block(found.block_hash) - return { - "submitted": list(harness.rpc.submitted), - "abandoned": dict(harness.pool.block_candidate_abandoned_counts), - "stale_classes": dict( - getattr(harness.pool, "stale_job_abandon_counts", None) or {} - ), - "outbox": harness.outbox_state(found), - "chain_state": None if row is None else row.chain_state, - "envelope_written": harness.envelope_written(found), - "live_epoch": harness.live_append_epoch(), - "summary": harness.landing_summary(), - } - - -def _append_then_land( - harness: LandingHarness, - *, - stamp_ms: int, -) -> dict[str, object]: - """The reproduction from #126, parameterised by the appended stamp. - - The append is fully drained before the landing starts, so the epoch has - already moved when the pre-offer fence takes its advisory read. Which - verdict that fence reaches is then a question about ``stamp_ms`` alone. - """ - harness.boot() - found = harness.found_block( - BLOCK, - height=10, - anchor_job_issued_at_ms=ANCHOR_MS, - ) - harness.publish_window_anchor(NEWER_WINDOW_ANCHOR_MS) - appended = harness.append_late_visible_share(stamp_ms=stamp_ms) - harness.drain([harness.appender]) - - harness.land_on_accounting_tail(found) - harness.drain([harness.accounting, harness.appender]) - - outcome = _landing_outcome(harness, found) - # Evidence that the premise held: the row really did advance the global - # epoch. A scenario where it did not would prove nothing about scoping. - outcome["bumped_to"] = appended.value() - return outcome - - -def _land_across_the_authoritative_fence( - harness: LandingHarness, - *, - stamp_ms: int, -) -> dict[str, object]: - """Commit the bump between the advisory read and the fence-held one. - - The accounting tail parks past the pre-offer fence, which therefore saw - an unmoved epoch; the appender then commits its bump into that gap and - the tail resumes into the authoritative fence. ``epoch_at_advisory_read`` - is recorded so a scenario cannot silently degenerate into the drained - shape above. - """ - harness.boot() - harness.break_at(BEFORE_AUTHORITATIVE_FENCE) - found = harness.found_block( - BLOCK, - height=10, - anchor_job_issued_at_ms=ANCHOR_MS, - ) - harness.publish_window_anchor(NEWER_WINDOW_ANCHOR_MS) - - harness.land_on_accounting_tail(found) - harness.run_until(harness.accounting, BEFORE_AUTHORITATIVE_FENCE) - epoch_at_advisory_read = harness.live_append_epoch() - - appended = harness.append_late_visible_share(stamp_ms=stamp_ms) - harness.drain([harness.appender]) - harness.drain([harness.accounting, harness.appender]) - - outcome = _landing_outcome(harness, found) - outcome["epoch_at_advisory_read"] = epoch_at_advisory_read - outcome["bumped_to"] = appended.value() - return outcome - - -def _silent_append_after_disarm(harness: LandingHarness) -> dict[str, object]: - """The disarm gap: a predating append that leaves no bump at all. - - A newer SEEDED window's anchor lives only on the armed artifact -- - deliberately NOT ``publish_window_anchor()``, whose seedless watermark - is immortal and would hide the gap. The first append predates only that - newer window, so it bumps; the bump also disarms the artifact, and with - it the window's only anchor exposure. The second append genuinely - predates the candidate's declared anchor, but if the disarm emptied the - anchor set it records nothing: no bump, no stamp, and the landing's - anchor-scoped fences read only the first bump's harmless stamp and - submit a window a durable share invalidated. The fold of the armed - artifact's anchor into the published-window watermark is what closes - this: the second append still finds an anchor to predate and leaves the - stamp the fences then refuse. - """ - harness.boot() - found = harness.found_block( - BLOCK, - height=10, - anchor_job_issued_at_ms=ANCHOR_MS, - ) - # A staging liberty: production arms a full PayoutLedgerArtifact, but - # the anchor-set arithmetic under test reads only snapshot_anchor_ms. - harness.pool._payout_ledger_artifact = SimpleNamespace( - snapshot_anchor_ms=NEWER_SEEDED_ANCHOR_MS - ) - bump = harness.append_late_visible_share(stamp_ms=LATER_THAN_ANCHOR_MS) - harness.drain([harness.appender]) - silent = harness.append_late_visible_share( - stamp_ms=INSIDE_ANCHOR_MS, - share_id="appender:silent", - ) - harness.drain([harness.appender]) - - harness.land_on_accounting_tail(found) - harness.drain([harness.accounting, harness.appender]) - - outcome = _landing_outcome(harness, found) - outcome["bumped_to"] = bump.value() - outcome["silent_bump"] = silent.value() - outcome["artifact_disarmed"] = harness.pool._payout_ledger_artifact is None - return outcome - - -def _append_with_distinct_stamps_then_land( - harness: LandingHarness, - *, - job_issued_at_ms: int, - accepted_at_ms: int, -) -> dict[str, object]: - """The drained shape above, with the row's two stamps pulled apart. - - The recorded bump stamp is ``max(job_issued_at_ms, accepted_at_ms)``: a - row predates an anchor iff BOTH stamps do, i.e. iff that max does. The - equal-stamp scenarios in this file cannot tell that choice from - ``min()`` or either stamp alone. A straddling row -- issued at or - before the candidate's anchor, accepted after -- is where they part - ways: it does not predate the candidate's window, so recording anything - but the max would abandon a block whose window was never invalidated, - which is #126's over-abandon back again. - """ - harness.boot() - found = harness.found_block( - BLOCK, - height=10, - anchor_job_issued_at_ms=ANCHOR_MS, - ) - harness.publish_window_anchor(NEWER_WINDOW_ANCHOR_MS) - appended = harness.append_late_visible_share( - stamp_ms=job_issued_at_ms, - accepted_at_ms=accepted_at_ms, - ) - harness.drain([harness.appender]) - - harness.land_on_accounting_tail(found) - harness.drain([harness.accounting, harness.appender]) - - outcome = _landing_outcome(harness, found) - outcome["bumped_to"] = appended.value() - return outcome - - -def _land_after_pruned_history(harness: LandingHarness) -> dict[str, object]: - """Push the candidate's baseline off the end of the retained stamps. - - Every append here misses the candidate's window, so an unbounded history - would land the block. The point is that a bounded one must not: once the - baseline epoch is older than the oldest retained bump, the landing cannot - prove anything about the bumps in between and has to abandon. - """ - harness.boot() - found = harness.found_block( - BLOCK, - height=10, - anchor_job_issued_at_ms=ANCHOR_MS, - ) - harness.publish_window_anchor(NEWER_WINDOW_ANCHOR_MS) - bumps = PRISM_PAYOUT_APPEND_INVALIDATION_STAMP_HISTORY + 2 - for index in range(bumps): - harness.append_late_visible_share( - stamp_ms=LATER_THAN_ANCHOR_MS, - share_id=f"appender:pruned-{index}", - ) - # One drain for the whole queue: the appends are independent of each - # other and of the landing, which has not started. - harness.drain([harness.appender], limit=4 * bumps) - - harness.land_on_accounting_tail(found) - harness.drain([harness.accounting, harness.appender]) - - outcome = _landing_outcome(harness, found) - outcome["bumps"] = bumps - return outcome - - -def _land_without_a_declared_anchor(harness: LandingHarness) -> dict[str, object]: - """Land a candidate whose job declared no payout window anchor. - - Nothing scopes the epoch question for such a candidate -- the landing - cannot even expose an anchor for the append side to bump against -- so a - moved epoch stays as terminal as it was before #126. - """ - harness.boot() - found = harness.found_block( - BLOCK, - height=10, - anchor_job_issued_at_ms=ANCHOR_MS, - ) - found.candidate.context.found_block.pop("anchor_job_issued_at_ms") - harness.publish_window_anchor(NEWER_WINDOW_ANCHOR_MS) - appended = harness.append_late_visible_share(stamp_ms=LATER_THAN_ANCHOR_MS) - harness.drain([harness.appender]) - - harness.land_on_accounting_tail(found) - harness.drain([harness.accounting, harness.appender]) - - outcome = _landing_outcome(harness, found) - outcome["bumped_to"] = appended.value() - return outcome - - -class AppendEpochScopedToTheDeclaredAnchorTests(unittest.TestCase): - """#126: only an append that predates *this* anchor may abandon.""" - - def _deterministic(self, scenario: Any) -> dict[str, object]: - outcome = assert_deterministic( - self, - scenario, - harness_factory=LandingHarness, - ).outcome - assert isinstance(outcome, dict) - return outcome - - def _assert_landed(self, outcome: dict[str, object]) -> None: - """The block reached qbitd, confirmed durably, and was not abandoned.""" - self.assertEqual(outcome["submitted"], [f"block:{BLOCK}"]) - self.assertEqual(outcome["chain_state"], "confirmed") - self.assertEqual(outcome["outbox"], "submitted") - self.assertTrue(outcome["envelope_written"]) - self.assertEqual(outcome["abandoned"], {}) - self.assertEqual(outcome["stale_classes"], {}) - - def _assert_abandoned_as_append_epoch_stale( - self, - outcome: dict[str, object], - ) -> None: - """The block was withheld from qbitd and terminally abandoned.""" - self.assertEqual(outcome["submitted"], []) - self.assertIsNone(outcome["chain_state"]) - self.assertEqual(outcome["outbox"], "abandoned") - self.assertFalse(outcome["envelope_written"]) - self.assertEqual(outcome["abandoned"], {"stale-job": 1}) - self.assertEqual( - outcome["stale_classes"], - {"tip_moved": 0, "balance_stale": 0, "append_epoch_stale": 1}, - ) - - # -- the over-abandon --------------------------------------------------- - - def test_an_append_that_misses_this_window_lands_the_block(self) -> None: - """#126's reproduction: the block must reach qbitd and confirm. - - Before the fix this abandoned with ``submitted == []`` and - ``block_candidate_abandoned_counts == {"stale-job": 1}`` -- the - forfeited reward the issue is about. - """ - outcome = self._deterministic( - lambda harness: _append_then_land( - harness, - stamp_ms=LATER_THAN_ANCHOR_MS, - ) - ) - # The premise: the append really did move the global epoch, so the - # bare comparison this fence used to make would have fired. - self.assertEqual(outcome["bumped_to"], 1) - self.assertEqual(outcome["live_epoch"], 1) - self._assert_landed(outcome) - - # -- fail-closed, preserved -------------------------------------------- - - def test_an_append_inside_this_window_still_abandons(self) -> None: - outcome = self._deterministic( - lambda harness: _append_then_land(harness, stamp_ms=INSIDE_ANCHOR_MS) - ) - self.assertEqual(outcome["bumped_to"], 1) - self._assert_abandoned_as_append_epoch_stale(outcome) - - def test_an_append_exactly_at_the_declared_anchor_still_abandons(self) -> None: - """The boundary belongs to the terminal side. - - A row predates an anchor when *both* its stamps are at or before it, - so a share stamped exactly at the anchor is inside the candidate's - window and its coinbase would omit a durable share. - """ - outcome = self._deterministic( - lambda harness: _append_then_land(harness, stamp_ms=AT_ANCHOR_MS) - ) - self.assertEqual(outcome["bumped_to"], 1) - self._assert_abandoned_as_append_epoch_stale(outcome) - - # -- the disarm gap ------------------------------------------------------ - - def test_a_predating_append_after_the_seeded_anchor_disarms_still_abandons( - self, - ) -> None: - """A bump that disarms the artifact must not blind the next append. - - Before the watermark fold this submitted: the second append - returned ``None`` (no live anchor covered it), the fences saw only - the first bump's harmless stamp, and ``submitted`` held the block - -- a coinbase whose window omitted a durable share. - """ - outcome = self._deterministic(_silent_append_after_disarm) - # The premise: the first append moved the epoch by predating the - # seeded window, and doing so disarmed the artifact that was that - # window's only anchor exposure. - self.assertEqual(outcome["bumped_to"], 1) - self.assertTrue(outcome["artifact_disarmed"]) - # The fix: the second append still finds an anchor to predate -- - # the disarm folded the seeded anchor into the watermark -- so its - # bump and stamp exist for the fences to refuse. - self.assertEqual(outcome["silent_bump"], 2) - self.assertEqual(outcome["live_epoch"], 2) - self._assert_abandoned_as_append_epoch_stale(outcome) - - # -- the recorded stamp is the max of the row's two ---------------------- - - def test_a_straddling_append_lands_the_block(self) -> None: - """Issued at or before the anchor, accepted after: not predating. - - The row invalidates the newer published window (both stamps are at - or before its anchor), so the epoch moves -- but the candidate's - own window is intact, and only a recorded stamp of - ``max(job_issued_at_ms, accepted_at_ms)`` says so. ``min()`` or the - issue stamp alone records 9_000 here and abandons a valid block. - """ - outcome = self._deterministic( - lambda harness: _append_with_distinct_stamps_then_land( - harness, - job_issued_at_ms=INSIDE_ANCHOR_MS, - accepted_at_ms=LATER_THAN_ANCHOR_MS, - ) - ) - self.assertEqual(outcome["bumped_to"], 1) - self._assert_landed(outcome) - - def test_an_append_with_distinct_stamps_inside_the_window_still_abandons( - self, - ) -> None: - """And the same distinct-stamp row is terminal once both predate.""" - outcome = self._deterministic( - lambda harness: _append_with_distinct_stamps_then_land( - harness, - job_issued_at_ms=INSIDE_ANCHOR_MS, - accepted_at_ms=ALSO_INSIDE_ANCHOR_MS, - ) - ) - self.assertEqual(outcome["bumped_to"], 1) - self._assert_abandoned_as_append_epoch_stale(outcome) - - # -- the authoritative fence ------------------------------------------- - - def test_a_bump_after_the_advisory_read_is_terminal_when_it_predates( - self, - ) -> None: - """The fence-held check still catches what the advisory one missed.""" - outcome = self._deterministic( - lambda harness: _land_across_the_authoritative_fence( - harness, - stamp_ms=INSIDE_ANCHOR_MS, - ) - ) - # The interleaving is the whole point: the pre-offer fence read an - # unmoved epoch, so this verdict can only come from the fence-held - # comparison that runs after it. - self.assertEqual(outcome["epoch_at_advisory_read"], 0) - self.assertEqual(outcome["bumped_to"], 1) - self._assert_abandoned_as_append_epoch_stale(outcome) - - def test_a_bump_after_the_advisory_read_is_harmless_when_it_does_not( - self, - ) -> None: - """And forgives the same interleaving when the stamp cannot predate.""" - outcome = self._deterministic( - lambda harness: _land_across_the_authoritative_fence( - harness, - stamp_ms=LATER_THAN_ANCHOR_MS, - ) - ) - self.assertEqual(outcome["epoch_at_advisory_read"], 0) - self.assertEqual(outcome["bumped_to"], 1) - self._assert_landed(outcome) - - # -- what cannot be answered ------------------------------------------- - - def test_a_baseline_older_than_the_retained_history_abandons(self) -> None: - """Pruning narrows acceptance; it never widens it.""" - outcome = self._deterministic(_land_after_pruned_history) - self.assertEqual( - outcome["live_epoch"], - PRISM_PAYOUT_APPEND_INVALIDATION_STAMP_HISTORY + 2, - ) - # Every one of those appends missed the candidate's window, so this - # abandon is the retention bound talking and nothing else. - self._assert_abandoned_as_append_epoch_stale(outcome) - - def test_a_candidate_without_a_declared_anchor_abandons(self) -> None: - outcome = self._deterministic(_land_without_a_declared_anchor) - self.assertEqual(outcome["bumped_to"], 1) - self._assert_abandoned_as_append_epoch_stale(outcome) - - -if __name__ == "__main__": - unittest.main() diff --git a/tests/test_prism_landing_budget_watchdog.py b/tests/test_prism_landing_budget_watchdog.py deleted file mode 100644 index caf5f9d27..000000000 --- a/tests/test_prism_landing_budget_watchdog.py +++ /dev/null @@ -1,710 +0,0 @@ -#!/usr/bin/env python3 -"""One landing step's budget against the watchdog that is timing it. - -Issue #125: a landing-class ledger step is two waits, not one. The caller -first queues for the ledger's writer lock -- no statement has been sent, so -neither ``statement_timeout`` nor any server-side cancellation bounds it -- -and only then runs the statement it was queuing for, under its own full -budget. Both are spent on the block-work thread the watchdog monitors, and -before ``49f04c3`` both were heartbeat-silent. At the reviewed 120s -escalation cap that is ~240s of silence against a 120s tolerance: the -watchdog hard-exits mid-landing, the in-memory escalation counter dies with -the process, and the restart comes back at the 30s base to ride the same -cycle again. No accounting is lost -- the outbox row stays pending and -replays -- but found-block settlement is deferred behind a restart loop. - -``49f04c3`` answers it in two halves, and these scenarios are shaped around -them: - -* the admission wait is served in slices whenever a progress hook is - installed, so a queued landing stamps its heartbeat instead of going - quiet -- while the caller's own deadline stays the sole authority on when - the wait ends; -* every configured landing budget, base and escalated cap alike, is clamped - to a fraction of the watchdog tolerance, so admission plus statement - cannot outrun the monitor timing them. - -These pass on ``2.x.x``. Their evidence is not that the fix is correct -- -its own PR argued that against unit-level doubles -- but that the -interleaving #125 describes can now be *driven*: a real landing tail queuing -on the real ledger writer lock, escalating its real budget across real -timed-out attempts, on a clock that moves only when this file says so. -""" - -from __future__ import annotations - -import sys -import threading -import unittest -from collections.abc import Callable -from pathlib import Path -from typing import Any - -sys.path.insert(0, str(Path(__file__).resolve().parents[1])) - -from tests.prism_concurrency_harness import Actor, assert_deterministic # noqa: E402 -from tests.prism_landing_harness import ( # noqa: E402 - PARENT_HASH, - LandingHarness, -) - -BLOCK_A = "aa" * 32 -BLOCK_B = "bb" * 32 - -# The landing names its own boundaries; these are the two it names that this -# file turns on. ``phase:accounting`` is the last stamp before the accounting -# tail opens its landing-class budget scope, so an actor parked there is -# parked exactly one step short of the wait #125 is about. -# ``wait-ledger-admission:landing`` is the stamp the fix added *inside* that -# wait -- arming it turns every heartbeat into a checkpoint, which makes the -# stamps part of the order-stable trace rather than something counted on the -# side. -LANDING_SCOPE = "phase:accounting" -ADMISSION_STAMP = "phase:wait-ledger-admission:landing" - -WRITER_LOCK_HELD = "writer-lock-held" -CLAMP_LINE = "prism coordinator: landing db budget clamped by watchdog" -GATE_TIMEOUT = "timed out waiting for postgres writer lock" - - -# -------------------------------------------------------------------------- -# Driving -# -------------------------------------------------------------------------- - - -def _own_block_work(harness: LandingHarness) -> None: - """Let the accounting actor stand in for the block_accounting thread. - - Block-work stamps are gated to the two threads that own a heartbeat slot - (``_block_work_heartbeat_owner``), which is the whole reason a client - connection thread cannot refresh the accounting thread's liveness budget - on its behalf. ``block_accounting_loop`` claims that slot by stamping its - own ident; the harness dequeues nothing and drives - ``_run_block_accounting_task`` directly, so the claim has to be made - here. Without it the admission stamps still fire -- the fix does not - depend on ownership -- but they land on the unowned fallback slot with no - phase attached, which is not the shape production runs and not the shape - #125 describes. - """ - service = harness.pool._ensure_block_candidate_service() - - def claim() -> None: - service._block_accounting_thread_ident = threading.get_ident() - - harness.accounting.submit(claim, label="own-block-work") - harness.run_until(harness.accounting, "done:own-block-work") - - -def _pump( - harness: LandingHarness, - actor: Actor, - label: str, - *, - on_stop: Callable[[str, float], None] | None = None, - limit: int = 20_000, -) -> None: - """Step ``actor`` to ``label``, moving virtual time when it waits on it. - - ``run_until`` cannot drive a sliced wait: an actor parked on a slice - deadline is not runnable until the clock reaches it, and only the - controller can move a virtual clock. Advancing to the *next* pending - deadline rather than by a fixed step is what keeps the schedule a - function of the scenario -- every second that passes here is a second - some deadline in the code under test asked for, and an actor parked with - no deadline at all is reported rather than waited on, because nothing - would ever release it. - """ - for _ in range(limit): - if not actor.runnable(): - if harness.advance_to_next_deadline() is None: - raise AssertionError( - f"actor {actor.name!r} is parked at {actor.stop!r} with no " - "deadline; nothing bounds this wait" - ) - continue - stop = harness.step(actor) - if on_stop is not None: - on_stop(stop, harness.clock.monotonic()) - if stop == label: - return - raise AssertionError( - f"actor {actor.name!r} did not reach {label!r} within {limit} steps; " - f"last stop {actor.stop!r}" - ) - - -def _hold_writer_lock(harness: LandingHarness) -> None: - """Park the client actor holding the ledger's writer lock. - - A landing queues on admission only if something else is already inside - the gate. The client tail is the honest holder: in production it is the - other thread that lands blocks, and a share append on it takes the same - lock. Parking it at a checkpoint rather than blocking it means the - controller decides exactly when the queue clears. - """ - lock = harness.pool.ledger._lock - - def hold() -> None: - lock.acquire() - harness.scheduler.checkpoint(WRITER_LOCK_HELD) - lock.release() - - harness.client.submit(hold, label="hold-writer-lock") - harness.run_until(harness.client, WRITER_LOCK_HELD) - - -def _release_writer_lock(harness: LandingHarness) -> None: - harness.run_until(harness.client, "done:hold-writer-lock") - - -def _granted_budget(harness: LandingHarness, block_hash: str | None) -> float: - """What production would actually grant this hash's next landing step. - - Read through the coordinator rather than recomputed here, because the - clamp arithmetic is the thing under test; a second copy of it in the - test would agree with itself and prove nothing. It runs on an actor - because it takes the coordinator lock, which is a harness lock. - """ - call = harness.accounting.submit( - lambda: harness.pool._block_landing_db_timeout(block_hash), - label="landing-budget", - ) - harness.run_until(harness.accounting, "done:landing-budget") - return float(call.value()) - - -def _wait_slice(harness: LandingHarness) -> float: - service = harness.pool._ensure_block_candidate_service() - return float(service._block_work_wait_slice()) - - -def _timeout_count(harness: LandingHarness, block_hash: str) -> int: - service = harness.pool._ensure_block_candidate_service() - counts = getattr(service, "_block_landing_timeout_counts", None) or {} - return int(counts.get(block_hash, 0)) - - -def _land_while_queued( - harness: LandingHarness, - found: Any, - *, - release_after_stamps: int | None = None, -) -> dict[str, Any]: - """Run one accounting-tail landing attempt against a held writer lock. - - The tail is stopped at the last boundary before its landing-class scope - opens, the lock is taken there, and the tail is then let go: everything - from that point on -- the admission wait, its stamps, the deadline that - ends it -- is the real landing running under its real budget. - - ``release_after_stamps`` hands the lock back mid-wait, which is the only - way to get a landing that queues *and then still runs*; leaving it held - can only ever produce the timing-out half of the story. - """ - harness.break_at(LANDING_SCOPE) - call = harness.land_on_accounting_tail(found) - harness.run_until(harness.accounting, LANDING_SCOPE) - started = harness.clock.monotonic() - _hold_writer_lock(harness) - - stamps: list[float] = [] - after_admission: list[float] = [] - queued = True - - def observe(stop: str, when: float) -> None: - nonlocal queued - if stop != ADMISSION_STAMP: - return - # The same phase covers both reports the fix added: the slices of an - # admission wait, and the round trip a gate-holding step completes - # between two statements it cannot release the gate between. Which - # one a stamp is depends only on whether this landing is still - # outside the gate, so they are split on that rather than the label. - if not queued: - after_admission.append(when) - return - stamps.append(when) - if release_after_stamps is not None and len(stamps) == release_after_stamps: - _release_writer_lock(harness) - queued = False - - _pump( - harness, - harness.accounting, - f"done:accounting-land:{found.block_hash[:4]}", - on_stop=observe, - ) - finished = harness.clock.monotonic() - # Admission ends when the lock is handed over -- the last stamp taken - # while it was still shut -- or, when it never is, at the deadline one - # slice past the last stamp. Nothing else in a landing consumes virtual - # time, so the tail's own completion pins the timing-out case exactly. - admitted_at = stamps[-1] if release_after_stamps is not None else finished - return { - "started": started, - "stamps": stamps, - "after_admission": after_admission, - "admitted_at": admitted_at, - "finished": finished, - "admission_seconds": admitted_at - started, - "call": call, - } - - -def _silent_gaps(started: float, stamps: list[float], ended: float) -> list[float]: - """Every interval the block-work heartbeat went unstamped, in order.""" - marks = [started, *stamps, ended] - return [round(later - earlier, 6) for earlier, later in zip(marks, marks[1:])] - - -def _prepared_harness( - harness: LandingHarness, - *, - configured_max_seconds: float, -) -> None: - """Boot, claim the block-work slot, and make every admission stamp visible. - - ``block_landing_db_timeout_max_seconds`` is the escalation cap an - operator configures; ``LandingHarness`` exposes the base budget and the - watchdog tolerance as constructor arguments but not the cap, so it is - set here before anything can read it. - """ - harness.pool.block_landing_db_timeout_max_seconds = float(configured_max_seconds) - harness.boot() - _own_block_work(harness) - harness.break_at(ADMISSION_STAMP) - - -# -------------------------------------------------------------------------- -# 1. The admission wait is no longer silent -# -------------------------------------------------------------------------- - - -def _stamped_admission_wait(harness: LandingHarness) -> dict[str, Any]: - """A landing that queues on the writer lock until its budget runs out.""" - _prepared_harness(harness, configured_max_seconds=120.0) - found = harness.found_block(BLOCK_A, height=10, parent_hash=PARENT_HASH) - budget = _granted_budget(harness, BLOCK_A) - - attempt = _land_while_queued(harness, found) - _release_writer_lock(harness) - - return { - "budget": budget, - "slice": _wait_slice(harness), - "stamps": len(attempt["stamps"]), - "waited": round(attempt["admission_seconds"], 6), - "silent_gaps": sorted( - set( - _silent_gaps( - attempt["started"], - attempt["stamps"], - attempt["admitted_at"], - ) - ) - ), - # The tail swallows the timeout and leaves the row for replay, which - # is the behaviour #125 says keeps accounting correct. The gate error - # itself reaches stderr with the traceback. - "tail_error": ( - type(attempt["call"].error).__name__ if attempt["call"].error else None - ), - "gate_timed_out": GATE_TIMEOUT in harness.errors, - "escalation_armed": _timeout_count(harness, BLOCK_A), - "outbox": harness.outbox_state(found), - "summary": harness.landing_summary(), - } - - -def _sliced_and_unsliced_wait(harness: LandingHarness) -> dict[str, Any]: - """The same gate, the same budget, once with the hook and once without. - - Slicing is only safe if it is invisible to the caller's deadline, so the - claim needs a control rather than an argument: one landing-class step - under the production scope that installs the hook, and one bare - ``statement_timeout`` at the same budget, both queued behind the same - held lock on the same actor. Anything the slicing changed about *when* - the wait ends would show up as a difference between the two. - """ - _prepared_harness(harness, configured_max_seconds=120.0) - ledger = harness.pool.ledger - budget = _granted_budget(harness, BLOCK_B) - _hold_writer_lock(harness) - - def queue(scope: Callable[[], Any], label: str) -> dict[str, Any]: - def run() -> None: - with scope(): - # A landing-class read that takes the writer lock, so the - # wait under test is the ledger's own admission gate rather - # than something the harness invented. - ledger.current_prior_balances() - - call = harness.accounting.submit(run, label=label) - started = harness.clock.monotonic() - stamps: list[float] = [] - _pump( - harness, - harness.accounting, - f"done:{label}", - on_stop=lambda stop, when: ( - stamps.append(when) if stop == ADMISSION_STAMP else None - ), - ) - return { - "stamps": len(stamps), - "waited": round(harness.clock.monotonic() - started, 6), - "error": type(call.error).__name__ if call.error else None, - "message": str(call.error) if call.error else None, - } - - hooked = queue( - lambda: harness.pool._block_landing_ledger_statement_timeout_scope(BLOCK_B), - "hooked-admission", - ) - unhooked = queue(lambda: ledger.statement_timeout(budget), "unhooked-admission") - _release_writer_lock(harness) - - return {"budget": budget, "hooked": hooked, "unhooked": unhooked} - - -class AdmissionWaitStampTests(unittest.TestCase): - """#125, first half: a queued landing must not go quiet.""" - - def test_a_queued_landing_stamps_its_heartbeat_while_it_waits(self) -> None: - outcome = assert_deterministic( - self, - _stamped_admission_wait, - harness_factory=LandingHarness, - ).outcome - assert isinstance(outcome, dict) - - # The default deployment: a 120s watchdog grants 30s to a first - # landing attempt, and the wait is served in watchdog-sized slices. - self.assertEqual(outcome["budget"], 30.0) - self.assertEqual(outcome["slice"], 0.25) - - # The assertion the fix exists for. Before it, this wait was one - # unbroken silence; now every slice boundary reports liveness, and - # the longest gap the watchdog could observe is one slice. - self.assertEqual(outcome["stamps"], 119) - self.assertEqual(outcome["silent_gaps"], [0.25]) - - # ...and the slicing bought that without lengthening the wait: the - # caller's own deadline still ends it, to the microsecond. - self.assertEqual(outcome["waited"], 30.0) - - # The landing failed the way #125 says it does: the gate deadline - # fires, the tail keeps the durable row for replay, and the next - # attempt for this hash is armed to escalate. - self.assertTrue(outcome["gate_timed_out"]) - self.assertIsNone(outcome["tail_error"]) - self.assertEqual(outcome["outbox"], "pending") - self.assertEqual(outcome["escalation_armed"], 1) - - def test_slicing_the_wait_does_not_lengthen_it(self) -> None: - outcome = assert_deterministic( - self, - _sliced_and_unsliced_wait, - harness_factory=LandingHarness, - ).outcome - assert isinstance(outcome, dict) - - hooked = outcome["hooked"] - unhooked = outcome["unhooked"] - - # One wait reports 119 times, the other not once... - self.assertEqual(hooked["stamps"], 119) - self.assertEqual(unhooked["stamps"], 0) - - # ...and they end at the same instant, with the same error. The - # deadline is the authority; the hook only decides how loudly the - # caller waits for it. - self.assertEqual(hooked["waited"], outcome["budget"]) - self.assertEqual(unhooked["waited"], outcome["budget"]) - self.assertEqual(hooked["error"], "LedgerOperationTimeout") - self.assertEqual(unhooked["error"], "LedgerOperationTimeout") - self.assertEqual(hooked["message"], unhooked["message"]) - self.assertIn(GATE_TIMEOUT, hooked["message"]) - - -# -------------------------------------------------------------------------- -# 2. The escalation ladder cannot outrun the watchdog -# -------------------------------------------------------------------------- - - -def _escalation_ladder( - harness: LandingHarness, - *, - configured_max_seconds: float, - attempts: int, -) -> dict[str, Any]: - """Time out ``attempts`` landings for one hash and read the ladder off. - - The escalation counter is keyed by block hash and moved only by an - observed landing-class timeout, so the ladder is driven the way - production climbs it -- one genuinely timed-out attempt per rung -- - rather than by writing the counter. - """ - _prepared_harness(harness, configured_max_seconds=configured_max_seconds) - found = harness.found_block(BLOCK_A, height=10, parent_hash=PARENT_HASH) - watchdog = float(harness.pool.watchdog_timeout_seconds) - - rungs: list[dict[str, Any]] = [] - for _ in range(attempts): - granted = _granted_budget(harness, BLOCK_A) - attempt = _land_while_queued(harness, found) - _release_writer_lock(harness) - rungs.append( - { - "granted": granted, - "waited": round(attempt["admission_seconds"], 6), - "stamps": len(attempt["stamps"]), - # #125's arithmetic: admission and statement each receive a - # full budget, so this is what one landing step can cost the - # heartbeat before the watchdog is entitled to give up. - "admission_plus_statement": 2.0 * granted, - "fits_watchdog": 2.0 * granted <= watchdog, - } - ) - - return { - "watchdog": watchdog, - "configured_base": float(harness.pool.block_landing_db_timeout_seconds), - "configured_max": float(harness.pool.block_landing_db_timeout_max_seconds), - "rungs": rungs, - "clamped": harness.output.count(CLAMP_LINE), - "escalations": _timeout_count(harness, BLOCK_A), - "outbox": harness.outbox_state(found), - } - - -# ``assert_deterministic`` takes a one-argument scenario, so each -# configuration under test is a named binding of the ladder above rather -# than a parameter the test method passes in. - - -def _clamped_cap_ladder(harness: LandingHarness) -> dict[str, Any]: - return _escalation_ladder(harness, configured_max_seconds=120.0, attempts=3) - - -def _clamped_base_ladder(harness: LandingHarness) -> dict[str, Any]: - return _escalation_ladder(harness, configured_max_seconds=120.0, attempts=2) - - -def _unclamped_ladder(harness: LandingHarness) -> dict[str, Any]: - return _escalation_ladder(harness, configured_max_seconds=60.0, attempts=3) - - -class EscalatedBudgetCeilingTests(unittest.TestCase): - """#125, second half: escalation stops below the watchdog, and says so.""" - - def test_the_reviewed_cap_is_clamped_to_the_default_watchdog(self) -> None: - """The exact deployment #125 was reported against. - - A 120s tolerance with the reviewed 120s escalation cap is the - configuration that produces the ~240s heartbeat gap: two full - budgets, one monitor, and no slack at all between them. - """ - outcome = assert_deterministic( - self, - _clamped_cap_ladder, - harness_factory=LandingHarness, - ).outcome - assert isinstance(outcome, dict) - - self.assertEqual(outcome["watchdog"], 120.0) - self.assertEqual(outcome["configured_base"], 30.0) - self.assertEqual(outcome["configured_max"], 120.0) - - # What #125 measured: the configured cap alone is the whole watchdog - # tolerance, so admission plus statement is twice it. - self.assertGreater(2.0 * outcome["configured_max"], outcome["watchdog"]) - - granted = [rung["granted"] for rung in outcome["rungs"]] - self.assertEqual(granted, [30.0, 60.0, 60.0]) - - # Every rung, including the exhausted one, leaves the watchdog able - # to tell a slow landing from a wedged one. - for rung in outcome["rungs"]: - self.assertTrue(rung["fits_watchdog"], rung) - self.assertLessEqual(rung["admission_plus_statement"], outcome["watchdog"]) - - # The ladder was climbed by real timed-out landings, not asserted: - # each rung spent its whole granted budget queuing, and stamped - # every slice of it. - self.assertEqual([rung["waited"] for rung in outcome["rungs"]], granted) - self.assertEqual([rung["stamps"] for rung in outcome["rungs"]], [119, 239, 239]) - self.assertEqual(outcome["escalations"], 3) - self.assertEqual(outcome["outbox"], "pending") - - # A reduced budget is never silent -- and it is said once, not on - # every budget read, of which this scenario made several per rung. - self.assertEqual(outcome["clamped"], 1) - - def test_a_tight_watchdog_clamps_the_base_budget_too(self) -> None: - """The ceiling is not a cap-only rule. - - An operator who tightens the watchdog without touching the landing - budgets moves the ceiling below the *base*, and the first attempt -- - the one that never escalated -- is already the one that would outrun - the monitor. Escalation then has nowhere to climb, which is the - correct answer: a budget that cannot be spent safely is not a budget. - """ - outcome = assert_deterministic( - self, - _clamped_base_ladder, - harness_factory=LandingHarness, - watchdog_timeout_seconds=40.0, - ).outcome - assert isinstance(outcome, dict) - - self.assertEqual(outcome["watchdog"], 40.0) - self.assertEqual(outcome["configured_base"], 30.0) - self.assertGreater(2.0 * outcome["configured_base"], outcome["watchdog"]) - - granted = [rung["granted"] for rung in outcome["rungs"]] - self.assertEqual(granted, [20.0, 20.0]) - for rung in outcome["rungs"]: - self.assertTrue(rung["fits_watchdog"], rung) - - self.assertEqual([rung["waited"] for rung in outcome["rungs"]], granted) - self.assertEqual([rung["stamps"] for rung in outcome["rungs"]], [79, 79]) - self.assertEqual(outcome["clamped"], 1) - - def test_a_budget_the_watchdog_can_afford_is_left_alone(self) -> None: - """The clamp is a ceiling, not a policy. - - A 300s tolerance with a 60s cap has room for both halves of a - landing step, so nothing is reduced and nothing is logged. This is - the case that keeps the ceiling honest: if it lowered budgets it did - not need to, the reported clamp would stop meaning anything. - """ - outcome = assert_deterministic( - self, - _unclamped_ladder, - harness_factory=LandingHarness, - watchdog_timeout_seconds=300.0, - ).outcome - assert isinstance(outcome, dict) - - self.assertEqual(outcome["watchdog"], 300.0) - self.assertEqual(outcome["configured_max"], 60.0) - - granted = [rung["granted"] for rung in outcome["rungs"]] - self.assertEqual(granted, [30.0, 60.0, 60.0]) - for rung in outcome["rungs"]: - self.assertTrue(rung["fits_watchdog"], rung) - - self.assertEqual([rung["waited"] for rung in outcome["rungs"]], granted) - self.assertEqual(outcome["clamped"], 0) - - -# -------------------------------------------------------------------------- -# 3. Both halves at once -# -------------------------------------------------------------------------- - - -def _queue_then_land_at_the_escalated_budget( - harness: LandingHarness, -) -> dict[str, Any]: - """Burn a whole escalated admission budget, then land on what is left. - - #125's worst case is not a landing that times out -- that one merely - retries. It is a landing that queues for almost everything admission - will give it and *then* starts a statement with a fresh full budget, - because that is the pair the watchdog sees back to back. So the first - attempt is spent to arm the escalation, and the second is handed the - lock one slice before its own deadline: the longest admission wait that - still ends in a landing. - """ - _prepared_harness(harness, configured_max_seconds=120.0) - found = harness.found_block(BLOCK_A, height=10, parent_hash=PARENT_HASH) - watchdog = float(harness.pool.watchdog_timeout_seconds) - slice_seconds = _wait_slice(harness) - - # Rung one: a full base budget spent queuing, which is what escalates. - base = _granted_budget(harness, BLOCK_A) - _land_while_queued(harness, found) - _release_writer_lock(harness) - - # Rung two: the escalated budget, released with one slice to spare. - escalated = _granted_budget(harness, BLOCK_A) - attempt = _land_while_queued( - harness, - found, - release_after_stamps=int(escalated / slice_seconds) - 1, - ) - - admission = attempt["admission_seconds"] - return { - "watchdog": watchdog, - "base": base, - "escalated": escalated, - "admission": round(admission, 6), - "stamps": len(attempt["stamps"]), - "stamps_after_admission": len(attempt["after_admission"]), - "longest_silence": max( - _silent_gaps( - attempt["started"], - attempt["stamps"], - attempt["admitted_at"], - ) - ), - # The quantity #125 is about: what this one landing step can still - # cost the block-work heartbeat, counting the admission it actually - # spent plus the untouched statement budget waiting on the far side - # of the gate. - "worst_case": round(admission + escalated, 6), - "landed": attempt["call"].error is None, - "outbox": harness.outbox_state(found), - "sequence": harness.publication_sequence(found), - "envelope": harness.envelope_written(found), - "summary": harness.landing_summary(), - } - - -class CombinedWorstCaseTests(unittest.TestCase): - """#125 end to end: queue on admission, then spend the escalated budget.""" - - def test_a_queued_escalated_landing_stays_inside_the_watchdog(self) -> None: - outcome = assert_deterministic( - self, - _queue_then_land_at_the_escalated_budget, - harness_factory=LandingHarness, - ).outcome - assert isinstance(outcome, dict) - - self.assertEqual(outcome["base"], 30.0) - self.assertEqual(outcome["escalated"], 60.0) - - # The landing queued for all but one slice of its escalated budget. - self.assertEqual(outcome["admission"], 59.75) - self.assertEqual(outcome["stamps"], 239) - - # Admission plus a fresh statement budget -- the exact pair that read - # as ~240s of silence at the unclamped cap -- now fits the tolerance. - self.assertEqual(outcome["worst_case"], 119.75) - self.assertLessEqual(outcome["worst_case"], outcome["watchdog"]) - - # And the watchdog never had to infer any of that: the longest stretch - # it saw without a stamp was one slice. - self.assertEqual(outcome["longest_silence"], 0.25) - - # Past the gate the same contract holds one level down. A landing - # step that must issue two statements without releasing the writer - # lock reports the first one's completed round trip, so the pair - # costs the monitor one budget of silence rather than two -- the - # same doubling #125 found on the other side of the gate. - self.assertEqual(outcome["stamps_after_admission"], 1) - - # The point of staying alive: the block lands. Nearly the whole - # escalated budget went to queuing and the landing still completed, - # published its ordinal and wrote its envelope. - self.assertTrue(outcome["landed"]) - self.assertEqual(outcome["outbox"], "submitted") - self.assertEqual(outcome["sequence"], 1) - self.assertTrue(outcome["envelope"]) - - -if __name__ == "__main__": - unittest.main() diff --git a/tests/test_prism_landing_harness_model.py b/tests/test_prism_landing_harness_model.py deleted file mode 100644 index ea765cb0f..000000000 --- a/tests/test_prism_landing_harness_model.py +++ /dev/null @@ -1,1388 +0,0 @@ -#!/usr/bin/env python3 -"""Self-tests for the landing half of the deterministic concurrency harness. - -``tests/test_prism_concurrency_harness.py`` defends the lease half. This file -defends the half that models block candidate landing, for the same reason: a -fake that answers an unrecognised statement plausibly drifts away from -production without anyone noticing, and every scenario built on it then -inherits the drift. When a landing scenario fails, these tests are what makes -"the code under test is wrong" the first explanation rather than "the model -is wrong". - -Four claims are pinned here, each one something a landing scenario relies on -without restating: - -1. **The publication-ordinal allocator follows ``qbit_confirm_pool_block``.** - The ordinal comes from a sequence rather than from ``MAX + 1``, so a fresh - confirmation burns a value, an exact replay burns nothing, a terminally - disposed row reports the distinct superseded disposition, and the durable - floor — a ``MAX`` over the table — tolerates the gap an aborted - confirmation leaves. #133 turns on precisely the difference between the - sequence and the floor, so the model owes the SQL an exact account of it. - -2. **Transaction scope is real.** A deadline-scoped landing statement sends - its COMMIT as a separate message, and the window in between is where the - interesting interleavings live. A model that published a landing write at - statement time would close that window and hide the defects inside it. - -3. **Classification is anchored to production SQL.** Every landing statement - is driven through the shipped ``PsqlShareLedger`` method that emits it, so - a change to that SQL breaks classification loudly instead of quietly - changing what the fake models. - -4. **The synchronisation primitives behave like the ones they replace.** The - landing topology is the first one where two threads of a single process - contend a process-local lock, so the harness locks are load-bearing: an - ordering they can express but a real lock cannot would be an ordering a - scenario could "prove" and production could never reach. -""" - -from __future__ import annotations - -import contextlib -import sys -import unittest -from collections.abc import Callable, Iterator -from dataclasses import replace -from pathlib import Path -from typing import Any - -sys.path.insert(0, str(Path(__file__).resolve().parents[1])) - -from tests.prism_concurrency_harness import ( # noqa: E402 - Actor, - Call, - HarnessError, - LandingOp, - PoolBlockRow, - UnsupportedStatement, - assert_deterministic, - classify, -) -from tests.prism_landing_harness import ( # noqa: E402 - PARENT_HASH, - LandingHarness, -) - -BLOCK_A = "aa" * 32 -BLOCK_B = "bb" * 32 -BLOCK_C = "cc" * 32 -BLOCK_D = "dd" * 32 - -# The top of the publish step, named by the landing itself. A tail stopped -# here has confirmed durably and released every lock it took to do so. -PUBLISH_BOUNDARY = "progress:evidence-write" - -# A successor's session token. Nothing but the token has to differ: the -# landing statements fence on the whole identity triple. -SUCCESSOR_TOKEN = "heartbeat-v1:successor" - - -def _run( - harness: LandingHarness, - actor: Actor, - work: Callable[[], Any], - label: str, -) -> Call: - """Run ``work`` to completion on ``actor`` and hand back its call record. - - The call record rather than its value, because half the assertions here - are about the error a statement raised. - """ - call = actor.submit(work, label=label) - harness.run_until(actor, f"done:{label}") - return call - - -def _prepared( - harness: LandingHarness, - block_hash: str, - *, - height: int, - parent_hash: str = PARENT_HASH, - chain_state: str = "prepared", - maturity_state: str = "immature", -) -> PoolBlockRow: - """Seed one committed ``qbit_pool_blocks`` row. - - Landing a whole block to reach a given row state would put the landing - state machine between the test and the allocator, which is the wrong - thing under test here: these cases are about what the allocator does with - a row, not about how the row got there. - """ - row = PoolBlockRow( - block_hash=block_hash, - block_height=height, - parent_hash=parent_hash, - chain_state=chain_state, - maturity_state=maturity_state, - ) - harness.server.pool_blocks[block_hash] = row - return row - - -@contextlib.contextmanager -def _deposed_writer(harness: LandingHarness) -> Iterator[None]: - """Hand the writer lease to a successor for the duration of the block. - - A landing statement carries its writer identity inline, so moving the - committed lease row is the whole of a takeover as far as one statement is - concerned. Building a second coordinator to produce the successor would - add a lease lifecycle to a test that is not about lease lifecycles. - """ - live = harness.server.lease - assert live is not None - harness.server.lease = replace(live, writer_session_token=SUCCESSOR_TOKEN) - try: - yield - finally: - harness.server.lease = live - - -def _captured_sql( - harness: LandingHarness, - actor: Actor, - work: Callable[[], Any], - label: str, -) -> str: - """Production's SQL for one landing statement, without its effects. - - Every landing statement fences on the writer lease before it writes - anything, so running one under a deposed writer records the statement — - classified, with its payload recovered — while leaving every table - untouched. That is what lets a test replay genuine production SQL into a - transaction it intends to abort, instead of hand-writing the statement - whose fidelity is the thing being tested. - """ - with _deposed_writer(harness): - call = _run(harness, actor, work, label) - assert call.error is not None, "a deposed writer's landing statement must fail" - return harness.server.statements[-1].sql - - -def _read_back_sql(harness: LandingHarness, ledger: Any) -> str: - """The pool-block state read, as ``pool_block_state`` emits it. - - A read takes no lease CTE, so it needs none of ``_captured_sql``'s - deposition: running it is already free of effects. - """ - _run( - harness, - harness.accounting, - lambda: ledger.pool_block_state(block_hash=BLOCK_A), - "capture-state", - ).value() - return harness.server.statements[-1].sql - - -class PublicationOrdinalAllocatorTests(unittest.TestCase): - """``qbit_confirm_pool_block``, as the model claims to implement it.""" - - def test_a_fresh_confirmation_assigns_the_next_sequence_value(self) -> None: - with LandingHarness() as harness: - ledger = harness.boot() - _prepared(harness, BLOCK_A, height=10) - _prepared(harness, BLOCK_B, height=11, parent_hash=BLOCK_A) - - first = _run( - harness, - harness.client, - lambda: ledger.confirm_accepted_block( - block_hash=BLOCK_A, - active_tip_height=10, - ), - "confirm-a", - ).value() - second = _run( - harness, - harness.client, - lambda: ledger.confirm_accepted_block( - block_hash=BLOCK_B, - active_tip_height=11, - ), - "confirm-b", - ).value() - - self.assertEqual(first["confirmed_count"], 1) - self.assertEqual(first["audit_publication_sequence"], 1) - self.assertEqual(second["audit_publication_sequence"], 2) - - row = harness.pool_block(BLOCK_A) - assert row is not None - self.assertEqual(row.chain_state, "confirmed") - # Confirmation moves the chain state only. Maturity is the reorg - # window's business and 100 blocks away. - self.assertEqual(row.maturity_state, "immature") - - def test_an_exact_replay_returns_one_without_burning_a_sequence_value( - self, - ) -> None: - """The half of the allocator's contract a counter model would get wrong. - - A replay that burned a value would leave the row's ordinal behind the - sequence, so the next block would jump — and the durable floor would - rise for a confirmation that changed nothing. The proof is the *next* - block's ordinal, not the replay's own return value. - """ - with LandingHarness() as harness: - ledger = harness.boot() - _prepared(harness, BLOCK_A, height=10) - _prepared(harness, BLOCK_B, height=11, parent_hash=BLOCK_A) - - _run( - harness, - harness.client, - lambda: ledger.confirm_accepted_block( - block_hash=BLOCK_A, - active_tip_height=10, - ), - "confirm-a", - ).value() - replay = _run( - harness, - harness.client, - lambda: ledger.confirm_accepted_block( - block_hash=BLOCK_A, - active_tip_height=10, - ), - "replay-a", - ).value() - - self.assertEqual(replay["confirmed_count"], 1) - self.assertEqual(replay["audit_publication_sequence"], 1) - - after_replay = _run( - harness, - harness.client, - lambda: ledger.confirm_accepted_block( - block_hash=BLOCK_B, - active_tip_height=11, - ), - "confirm-b", - ).value() - self.assertEqual( - after_replay["audit_publication_sequence"], - 2, - "the replay burned a sequence value it must not have burned", - ) - - def test_a_terminally_disposed_row_reports_the_superseded_disposition( - self, - ) -> None: - """-1 is a distinct answer, not a variant of 0. - - A caller reading -1 abandons the candidate knowing the ledger - explained itself; a caller reading 0 has to treat the state as - unexplained. Collapsing the two would let a routine reorg race look - like corruption, so every terminal shape the SQL names is checked - here rather than one representative. - """ - terminal = { - "reorg-quarantined": ("11" * 32, {"chain_state": "inactive"}), - "rejected": ("22" * 32, {"chain_state": "rejected"}), - "reversed": ("33" * 32, {"chain_state": "reversed"}), - # Confirmed but reversed: the maturity arm of the same test, and - # the one a chain_state-only model would answer as a replay. - "confirmed-then-reversed": ( - "44" * 32, - {"chain_state": "confirmed", "maturity_state": "reversed"}, - ), - } - with LandingHarness() as harness: - ledger = harness.boot() - for name, (block_hash, state) in terminal.items(): - with self.subTest(disposition=name): - _prepared(harness, block_hash, height=10, **state) - result = _run( - harness, - harness.client, - lambda h=block_hash: ledger.confirm_accepted_block( - block_hash=h, - active_tip_height=10, - ), - f"confirm-{name}", - ).value() - self.assertEqual(result["confirmed_count"], -1) - self.assertNotIn("audit_publication_sequence", result) - - # None of them burned an ordinal, so nothing was allocated at all. - self.assertEqual( - _run( - harness, - harness.client, - ledger.audit_publication_sequence_floor, - "floor", - ).value(), - 0, - ) - - def test_a_height_mismatch_and_a_missing_row_both_report_no_match(self) -> None: - """0 keeps its own meaning: no row, or a live row this does not match.""" - with LandingHarness() as harness: - ledger = harness.boot() - _prepared(harness, BLOCK_A, height=10) - - mismatched = _run( - harness, - harness.client, - lambda: ledger.confirm_accepted_block( - block_hash=BLOCK_A, - active_tip_height=11, - ), - "wrong-height", - ).value() - missing = _run( - harness, - harness.client, - lambda: ledger.confirm_accepted_block( - block_hash=BLOCK_C, - active_tip_height=10, - ), - "no-row", - ).value() - - self.assertEqual(mismatched["confirmed_count"], 0) - self.assertEqual(missing["confirmed_count"], 0) - row = harness.pool_block(BLOCK_A) - assert row is not None - self.assertEqual(row.chain_state, "prepared") - self.assertIsNone(row.audit_publication_sequence) - - def test_the_durable_floor_is_a_max_and_tolerates_a_burned_gap(self) -> None: - """``nextval`` is non-transactional, so an abort leaves a hole. - - The floor read is a ``MAX`` over the table for exactly this reason: it - follows the ordinals that became durable, not the ones the sequence - handed out. A model that counted allocations instead would put the - floor ahead of every published block after the first aborted - confirmation, and #133's whole argument is about how a block's ordinal - compares to that floor. - """ - with LandingHarness() as harness: - ledger = harness.boot() - _prepared(harness, BLOCK_A, height=10) - _prepared(harness, BLOCK_C, height=12, parent_hash=BLOCK_A) - _prepared(harness, BLOCK_D, height=13, parent_hash=BLOCK_C) - - _run( - harness, - harness.client, - lambda: ledger.confirm_accepted_block( - block_hash=BLOCK_A, - active_tip_height=10, - ), - "confirm-a", - ).value() - self.assertEqual(harness.publication_floor(), 1) - - confirm_c = _captured_sql( - harness, - harness.client, - lambda: ledger.confirm_accepted_block( - block_hash=BLOCK_C, - active_tip_height=12, - ), - "capture-confirm-c", - ) - - def aborted_confirmation() -> int | None: - backend = harness.server.connect(application_name="aborted") - transaction = harness.server._begin(backend, explicit=True) - harness.server.execute( - backend, - confirm_c, - transaction=transaction, - timeout_seconds=30.0, - tag="aborted", - ) - burned = transaction.staged_pool_blocks[ - BLOCK_C - ].audit_publication_sequence - harness.server._rollback(transaction) - return burned - - burned = _run( - harness, - harness.client, - aborted_confirmation, - "abort-confirm-c", - ).value() - self.assertEqual(burned, 2) - - floor_after_abort = _run( - harness, - harness.client, - ledger.audit_publication_sequence_floor, - "floor-after-abort", - ).value() - self.assertEqual( - floor_after_abort, - 1, - "an aborted confirmation must not raise the durable floor", - ) - - after = _run( - harness, - harness.client, - lambda: ledger.confirm_accepted_block( - block_hash=BLOCK_D, - active_tip_height=13, - ), - "confirm-d", - ).value() - self.assertEqual( - after["audit_publication_sequence"], - 3, - "the burned value must never be handed out again", - ) - self.assertEqual(harness.publication_floor(), 3) - - -class LandingTransactionVisibilityTests(unittest.TestCase): - """The confirm→COMMIT window, which is where landing defects live.""" - - def test_a_staged_landing_write_is_invisible_to_another_session(self) -> None: - with LandingHarness() as harness: - ledger = harness.boot() - _prepared(harness, BLOCK_A, height=10) - # Captured up front, so the reader replays one identical statement - # on both sides of the COMMIT. - state_sql = _read_back_sql(harness, ledger) - reader = harness.server.connect(application_name="reader") - - def read_pool_block() -> Any: - return harness.server.execute( - reader, - state_sql, - transaction=None, - timeout_seconds=None, - tag="reader", - ) - - def confirm_under_deadline() -> Any: - # A caller deadline is what makes _NativePostgresClient wrap - # the statement in BEGIN / statement / COMMIT, so it is also - # what opens the window this test is about. - with ledger.operation_timeout(30.0): - return ledger.confirm_accepted_block( - block_hash=BLOCK_A, - active_tip_height=10, - ) - - landing = harness.client.submit(confirm_under_deadline, label="confirm") - harness.run_until(harness.client, "ledger.precommit") - - assert harness.sql_backend is not None - transaction = harness.sql_backend._in_use[0].transaction - assert transaction is not None - staged = transaction.staged_pool_blocks[BLOCK_A] - self.assertEqual(staged.chain_state, "confirmed") - self.assertEqual(staged.audit_publication_sequence, 1) - - before = _run( - harness, - harness.accounting, - read_pool_block, - "read-before-commit", - ).value() - self.assertEqual(before["state"]["chain_state"], "prepared") - self.assertIsNone(before["state"]["audit_publication_sequence"]) - - harness.drain([harness.client]) - landing.value() - - after = _run( - harness, - harness.accounting, - read_pool_block, - "read-after-commit", - ).value() - self.assertEqual(after["state"]["chain_state"], "confirmed") - self.assertEqual(after["state"]["audit_publication_sequence"], 1) - - def test_a_transaction_sees_its_own_landing_write_before_commit(self) -> None: - """READ COMMITTED: committed rows, plus this transaction's own writes. - - The other half of the window. A landing reads its own confirmation - back — that is how it learns its publication ordinal — so a model that - staged writes where even their own session could not see them would - break every landing rather than only the interleaved ones. - """ - with LandingHarness() as harness: - ledger = harness.boot() - _prepared(harness, BLOCK_A, height=10) - state_sql = _read_back_sql(harness, ledger) - confirm_sql = _captured_sql( - harness, - harness.client, - lambda: ledger.confirm_accepted_block( - block_hash=BLOCK_A, - active_tip_height=10, - ), - "capture-confirm", - ) - - def confirm_then_read() -> Any: - backend = harness.server.connect(application_name="own-session") - transaction = harness.server._begin(backend, explicit=True) - harness.server.execute( - backend, - confirm_sql, - transaction=transaction, - timeout_seconds=30.0, - tag="own", - ) - own = harness.server.execute( - backend, - state_sql, - transaction=transaction, - timeout_seconds=30.0, - tag="own", - ) - harness.server._commit(transaction) - return own - - own = _run( - harness, - harness.client, - confirm_then_read, - "own-session-read", - ).value() - self.assertEqual(own["state"]["chain_state"], "confirmed") - self.assertEqual(own["state"]["audit_publication_sequence"], 1) - # And it really was the same transaction's write that it saw. - row = harness.pool_block(BLOCK_A) - assert row is not None - self.assertEqual(row.chain_state, "confirmed") - - def test_a_rolled_back_landing_leaves_no_row_but_keeps_the_burned_ordinal( - self, - ) -> None: - """Rollback undoes rows. It does not undo ``nextval``. - - Both halves matter to a landing scenario: a durable-looking row that - an abort left behind would make a scenario assert a landing that never - happened, and an ordinal handed back on abort would make the sequence - look like the counter it deliberately is not. - """ - with LandingHarness() as harness: - ledger = harness.boot() - _prepared(harness, BLOCK_A, height=10) - _prepared(harness, BLOCK_C, height=12, parent_hash=BLOCK_A) - _prepared(harness, BLOCK_D, height=13, parent_hash=BLOCK_C) - - _run( - harness, - harness.client, - lambda: ledger.confirm_accepted_block( - block_hash=BLOCK_A, - active_tip_height=10, - ), - "confirm-a", - ).value() - - found = harness.found_block(BLOCK_C, height=12, parent_hash=BLOCK_A) - intent = harness.pool.block_candidate_intent(found.candidate) - outbox_sql = _captured_sql( - harness, - harness.client, - lambda: ledger.persist_block_candidate_intent(intent), - "capture-outbox", - ) - confirm_sql = _captured_sql( - harness, - harness.client, - lambda: ledger.confirm_accepted_block( - block_hash=BLOCK_C, - active_tip_height=12, - ), - "capture-confirm", - ) - - def aborted_landing() -> None: - backend = harness.server.connect(application_name="aborted") - transaction = harness.server._begin(backend, explicit=True) - for sql in (outbox_sql, confirm_sql): - harness.server.execute( - backend, - sql, - transaction=transaction, - timeout_seconds=30.0, - tag="aborted", - ) - self.assertIn(BLOCK_C, transaction.staged_outbox) - self.assertNotIn(BLOCK_C, harness.server.outbox) - harness.server._rollback(transaction) - - _run(harness, harness.client, aborted_landing, "abort").value() - - self.assertNotIn(BLOCK_C, harness.server.outbox) - row = harness.pool_block(BLOCK_C) - assert row is not None - self.assertEqual(row.chain_state, "prepared") - self.assertIsNone(row.audit_publication_sequence) - - next_ordinal = _run( - harness, - harness.client, - lambda: ledger.confirm_accepted_block( - block_hash=BLOCK_D, - active_tip_height=13, - ), - "confirm-d", - ).value() - self.assertEqual(next_ordinal["audit_publication_sequence"], 3) - - -class LandingStatementClassificationTests(unittest.TestCase): - """The classifier is pinned to the SQL PsqlShareLedger actually emits.""" - - def test_every_modelled_landing_statement_is_recognised_from_real_calls( - self, - ) -> None: - """Drive every ``LandingOp`` through production code and classify it. - - The fragments in ``_LANDING_SIGNATURES`` were lifted from - ``share_ledger.py``; this is what proves they still match what that - module generates rather than what it generated when the landing model - was written. A full landing covers most of them on its own — that it - does is itself part of the claim, because a scenario that never - reaches a statement never checks it either. - """ - with LandingHarness() as harness: - ledger = harness.boot() - found = harness.found_block(BLOCK_A, height=10, parent_hash=PARENT_HASH) - landed = harness.land_on_client_tail(found) - harness.drain([harness.client]) - self.assertIsNone(landed.error) - - # The four a successful landing never emits: nothing was - # rejected, neither the share history nor the pool-block state - # is read on the happy path, and the height-bounded as-of balance - # read belongs to confirmed-ancestor replay only (issue #209). - _prepared(harness, BLOCK_B, height=11, parent_hash=BLOCK_A) - _run( - harness, - harness.client, - lambda: ledger.reject_prepared_block( - block_hash=BLOCK_B, - active_tip_height=11, - ), - "reject", - ).value() - _run(harness, harness.client, ledger.all_shares, "shares").value() - _run( - harness, - harness.client, - lambda: ledger.pool_block_state(block_hash=BLOCK_A), - "state", - ).value() - _run( - harness, - harness.client, - lambda: ledger.prior_balances_after_pool_block( - block_hash=BLOCK_A, - ), - "as-of", - ).value() - - # The replay path's two statements (issues #196 and #211). Neither - # is on a landing's own critical path -- the enumeration recovers - # missed wakeups and the batch abandon collapses superseded - # siblings -- so both are driven explicitly here. - _prepared(harness, BLOCK_C, height=12, parent_hash=BLOCK_B) - _run( - harness, - harness.client, - lambda: ledger.persist_block_candidate_intent( - { - "schema": "qbit.prism.block-candidate-intent.v1", - "block_hash_hex": BLOCK_C, - "block_hex": "00", - } - ), - "intent-c", - ).value() - page = _run( - harness, - harness.client, - lambda: ledger._legacy_pending_block_candidate_rows(limit=32), - "pending-page", - ).value() - self.assertEqual([row["block_hash"] for row in page], [BLOCK_C]) - # BLOCK_C has a prepared pool-block row, so the fenced batch write - # re-checks it and reports nothing: the page fact and the write's - # own veto are exercised together. - self.assertTrue(page[0]["pool_block_exists"]) - self.assertEqual( - _run( - harness, - harness.client, - lambda: ledger.mark_block_candidates_abandoned( - block_hashes=[BLOCK_C], - error="superseded by a decided height", - ), - "batch-abandon", - ).value(), - (), - ) - - # Issue #255: the chunked-body statements. A candidate with a - # share array wider than the fast path gets a span and page - # index; the metadata-first page, a hydration (manifest, spans, - # pages, chunk pages), a retired staging body and one janitor - # step drive the rest. - block_d = "dd" * 32 - self.assertEqual( - _run( - harness, - harness.client, - ledger.verify_candidate_schema, - "schema-check", - ).value(), - {"declared": 2, "has_body_table": True}, - ) - _run( - harness, - harness.client, - lambda: ledger.persist_block_candidate_intent( - { - "schema": "qbit.prism.block-candidate-intent.v1", - "block_hash_hex": block_d, - "block_hex": "00", - "parent_hash": BLOCK_C, - "expected_height": 13, - "template": {"previousblockhash": BLOCK_C, "height": 13}, - "found_block": {"network_difficulty": 1}, - "pending_share": {"job_id": "job-d"}, - "credit_share_on_accept": False, - "username": "miner-d", - "shares_json": [{"share_seq": index, "share_id": f"s{index}"} for index in range(4000)], - } - ), - "intent-d", - ).value() - headers = _run( - harness, - harness.client, - lambda: ledger.pending_block_candidate_headers(limit=32), - "header-page", - ).value() - # BLOCK_C stayed pending (its pool-block row vetoed the abandon). - self.assertEqual([row["block_hash"] for row in headers.rows], [BLOCK_C, block_d]) - self.assertTrue(headers.exhausted) - hydrated = _run( - harness, - harness.client, - lambda: ledger.hydrate_block_candidate_intent(headers.rows[1]), - "hydrate-d", - ).value() - self.assertEqual(len(hydrated["shares_json"]), 4000) - self.assertEqual(hydrated["shares_json"][3999]["share_id"], "s3999") - hydrated.body.close() - _run( - harness, - harness.client, - lambda: ledger._retire_staging_body("ab" * 16), - "retire-staging", - ).value() - _run( - harness, - harness.client, - ledger.reap_retired_candidate_bodies, - "janitor", - ).value() - - observed = { - kind - for kind in harness.statement_kinds() - if kind in {op.value for op in LandingOp} - } - self.assertEqual(observed, {op.value for op in LandingOp}) - - def test_a_landing_table_alone_is_not_a_classification(self) -> None: - """A signature match, not a table mention, is what answers a statement. - - The deferred markers for the landing tables are gone now that landing - is modelled, so a statement against one of them that matches no - signature has to reach the unmodelled-statement error rather than - being answered by whichever evaluator matched loosest. - """ - for sql in ( - "SELECT count(*) FROM qbit_pool_blocks;", - "DELETE FROM qbit_block_candidate_outbox WHERE block_hash = 'ab';", - "UPDATE qbit_pool_blocks SET maturity_state = 'mature';", - ): - with self.subTest(sql=sql): - with self.assertRaises(UnsupportedStatement) as caught: - classify(sql) - self.assertIn("does not model this statement", str(caught.exception)) - self.assertIn("_LANDING_SIGNATURES", str(caught.exception)) - - def test_deferred_state_machines_still_name_their_follow_up(self) -> None: - """Landing borders two state machines #128 defers, and says so. - - A landing scenario that wanders into share submission or CTV fanout - gets an error naming the follow-up rather than a bare model gap, and - that remains true now that landing statements are recognised ahead of - the deferred markers. - """ - for sql, expected in ( - ("INSERT INTO qbit_share_ledger (share_id)", "share submission"), - ( - "UPDATE qbit_ctv_fanouts SET state = 'broadcast'", - "CTV fanout", - ), - ): - with self.subTest(sql=sql): - with self.assertRaises(UnsupportedStatement) as caught: - classify(sql) - self.assertIn(expected, str(caught.exception)) - self.assertIn("#128", str(caught.exception)) - - def test_the_writer_identity_is_recovered_from_the_statement(self) -> None: - """The model is never told who a session belongs to; it reads the SQL. - - The two shapes recover it differently — the pool-block functions take - it as inlined arguments, the outbox statements inline it into their - lease CTE — so both are checked. If either recovery broke, the fence - below would fail open rather than closed. - """ - with LandingHarness() as harness: - ledger = harness.boot() - _prepared(harness, BLOCK_A, height=10) - found = harness.found_block(BLOCK_A, height=10, parent_hash=PARENT_HASH) - - _run( - harness, - harness.client, - lambda: ledger.confirm_accepted_block( - block_hash=BLOCK_A, - active_tip_height=10, - ), - "confirm", - ).value() - confirm = next( - statement - for statement in reversed(harness.server.statements) - if statement.kind is LandingOp.CONFIRM_BLOCK - ) - self.assertEqual(confirm.payload["block_hash"], BLOCK_A) - self.assertEqual(confirm.payload["active_tip_height"], 10) - - _run( - harness, - harness.client, - lambda: ledger.persist_block_candidate_intent( - harness.pool.block_candidate_intent(found.candidate) - ), - "outbox", - ).value() - outbox = harness.server.statements[-1] - self.assertEqual(outbox.kind, LandingOp.OUTBOX_RECORD) - self.assertEqual(outbox.payload["block_hash"], BLOCK_A) - - for statement in (confirm, outbox): - with self.subTest(kind=statement.kind.value): - self.assertEqual( - statement.payload["writer_id"], - "prism-coordinator", - ) - self.assertEqual(statement.payload["writer_epoch"], 1) - self.assertEqual( - statement.payload["writer_session_token"], - harness.session_token, - ) - - def test_a_deposed_writer_fails_the_lease_cte_in_the_shape_sql_dictates( - self, - ) -> None: - """Same fence, two observable shapes, and the difference is real. - - ``qbit_confirm_pool_block`` and ``qbit_reject_prepared_pool_block`` - ``RAISE`` when their lease UPDATE affects no rows, so the client sees - an error. The outbox statements are plain CTEs whose ``CASE`` reports - the same condition as a value, so the client sees a result and its - caller is what turns it into an error. A model that flattened the two - would let a scenario put a fenced-out landing on the wrong side of a - ``try``. - """ - with LandingHarness() as harness: - ledger = harness.boot() - _prepared(harness, BLOCK_A, height=10) - found = harness.found_block(BLOCK_A, height=10, parent_hash=PARENT_HASH) - - confirm_sql = _captured_sql( - harness, - harness.client, - lambda: ledger.confirm_accepted_block( - block_hash=BLOCK_A, - active_tip_height=10, - ), - "capture-confirm", - ) - reject_sql = _captured_sql( - harness, - harness.client, - lambda: ledger.reject_prepared_block( - block_hash=BLOCK_A, - active_tip_height=10, - ), - "capture-reject", - ) - outbox_sql = _captured_sql( - harness, - harness.client, - lambda: ledger.persist_block_candidate_intent( - harness.pool.block_candidate_intent(found.candidate) - ), - "capture-outbox", - ) - - assert harness.sql_backend is not None - backend = harness.sql_backend - with _deposed_writer(harness): - for name, sql in ( - ("confirm", confirm_sql), - ("reject_prepared", reject_sql), - ): - with self.subTest(statement=name): - call = _run( - harness, - harness.client, - lambda s=sql: backend.run_json(s), - f"fenced-{name}", - ) - self.assertIsInstance(call.error, RuntimeError) - self.assertEqual( - str(call.error), - "writer lease is not active", - ) - - outbox = _run( - harness, - harness.client, - lambda: backend.run_json(outbox_sql), - "fenced-outbox", - ).value() - self.assertEqual( - outbox, - {"error": "writer lease is not active"}, - ) - - # Fenced out means fenced out: nothing reached a table either way. - row = harness.pool_block(BLOCK_A) - assert row is not None - self.assertEqual(row.chain_state, "prepared") - self.assertEqual(harness.server.outbox, {}) - - -class HarnessLockTests(unittest.TestCase): - """A contended process-local lock is an interleaving point, not a stall.""" - - def test_two_actors_are_mutually_excluded_and_granted_in_arrival_order( - self, - ) -> None: - """FIFO, because the schedule is an input here. - - Without a queue the model could grant in an order neither a real - futex-backed lock nor the scheduler would produce, and a scenario - could then "prove" an interleaving production cannot reach. - """ - with LandingHarness() as harness: - lock = harness.lock("payout-balance-mutation") - granted: list[str] = [] - - def hold(name: str) -> Callable[[], None]: - def run() -> None: - lock.acquire() - granted.append(name) - harness.scheduler.checkpoint(f"{name}:holding") - lock.release() - - return run - - harness.client.submit(hold("client"), label="client-hold") - harness.accounting.submit(hold("accounting"), label="accounting-hold") - harness.appender.submit(hold("appender"), label="appender-hold") - - harness.run_until(harness.client, "client:holding") - # Arrival order is chosen by the controller, not by thread timing. - parked = "lock:payout-balance-mutation" - harness.run_until_blocked(harness.accounting, parked) - harness.run_until_blocked(harness.appender, parked) - self.assertEqual(granted, ["client"]) - self.assertIs(lock.owner, harness.client) - - harness.step(harness.client) - self.assertTrue(harness.accounting.runnable()) - self.assertFalse( - harness.appender.runnable(), - "a later arrival must not overtake a queued waiter", - ) - - harness.run_until(harness.accounting, "accounting:holding") - self.assertEqual(granted, ["client", "accounting"]) - self.assertFalse(harness.appender.runnable()) - - harness.step(harness.accounting) - harness.run_until(harness.appender, "appender:holding") - self.assertEqual(granted, ["client", "accounting", "appender"]) - harness.drain([harness.client, harness.accounting, harness.appender]) - self.assertIsNone(lock.owner) - - def test_a_non_blocking_acquire_refuses_rather_than_parking(self) -> None: - with LandingHarness() as harness: - lock = harness.lock("tip-refresh") - - def hold() -> None: - lock.acquire() - harness.scheduler.checkpoint("holding") - lock.release() - - harness.client.submit(hold, label="hold") - harness.run_until(harness.client, "holding") - - probe = _run( - harness, - harness.accounting, - lambda: lock.acquire(blocking=False), - "try-acquire", - ) - self.assertFalse(probe.value()) - # It refused instead of queueing, so nothing is waiting on the - # holder and the holder owes it nothing. - self.assertFalse(harness.accounting.blocked) - harness.drain([harness.client, harness.accounting]) - - def test_a_timed_acquire_expires_against_the_virtual_clock(self) -> None: - with LandingHarness() as harness: - lock = harness.lock("job-cache") - - def hold() -> None: - lock.acquire() - harness.scheduler.checkpoint("holding") - lock.release() - - harness.client.submit(hold, label="hold") - harness.run_until(harness.client, "holding") - - started = harness.clock.monotonic() - timed = harness.accounting.submit( - lambda: lock.acquire(timeout=5.0), - label="timed-acquire", - ) - harness.run_until_blocked(harness.accounting, "lock:job-cache") - self.assertEqual(harness.accounting.wake_at, started + 5.0) - - harness.advance_to_next_deadline() - harness.run_until(harness.accounting, "done:timed-acquire") - self.assertFalse(timed.value()) - self.assertEqual(harness.clock.monotonic(), started + 5.0) - # Nothing slept: the wait was measured, not endured. - self.assertEqual(harness.sleeps, []) - harness.drain([harness.client, harness.accounting]) - - def test_re_acquiring_a_non_reentrant_lock_is_reported_not_hidden(self) -> None: - """A real mutex deadlocks there, and a harness that hid it would lie. - - Silently granting the second acquire would turn a production deadlock - into a green scenario, which is the single most expensive thing this - harness could get wrong. - """ - with LandingHarness() as harness: - lock = harness.lock("payout-append-landing-fence") - - def double_acquire() -> None: - lock.acquire() - lock.acquire() - - call = _run(harness, harness.client, double_acquire, "double") - with self.assertRaises(HarnessError) as caught: - call.value() - self.assertIn("production would deadlock here", str(caught.exception)) - - -class HarnessRLockTests(unittest.TestCase): - def test_the_owner_re_enters_and_a_waiter_waits_for_depth_zero(self) -> None: - with LandingHarness() as harness: - lock = harness.rlock("coordinator") - - def hold() -> None: - lock.acquire() - lock.acquire() - harness.scheduler.checkpoint("depth-two") - lock.release() - harness.scheduler.checkpoint("depth-one") - lock.release() - - harness.client.submit(hold, label="hold") - waiter = harness.accounting.submit(lock.acquire, label="waiter") - harness.run_until(harness.client, "depth-two") - harness.run_until_blocked(harness.accounting, "lock:coordinator") - - harness.run_until(harness.client, "depth-one") - self.assertFalse( - harness.accounting.runnable(), - "an outer release still leaves the lock held", - ) - - harness.step(harness.client) - self.assertTrue(harness.accounting.runnable()) - harness.run_until(harness.accounting, "done:waiter") - self.assertTrue(waiter.value()) - self.assertIs(lock.owner, harness.accounting) - - -class HarnessConditionTests(unittest.TestCase): - """``wait`` must drop the lock, or the notifier could never take it.""" - - def test_a_wait_releases_the_lock_and_a_notify_from_elsewhere_wakes_it( - self, - ) -> None: - with LandingHarness() as harness: - condition = harness.condition("accepted-block-payout-preview") - notifier_held_the_lock: list[bool] = [] - reacquired: list[bool] = [] - - def wait_for_notify() -> bool: - with condition: - notified = condition.wait() - # Held again before wait returns, because a predicate loop - # written against a real condition re-reads shared state - # on this line. - reacquired.append(condition._lock.held_by_current()) - return notified - - def notify() -> None: - with condition: - notifier_held_the_lock.append(condition._lock.locked()) - condition.notify_all() - - waiter = harness.accounting.submit(wait_for_notify, label="wait") - harness.run_until_blocked( - harness.accounting, - "cond:accepted-block-payout-preview", - ) - self.assertIsNone( - condition._lock.owner, - "a parked waiter must not still hold the condition's lock", - ) - - _run(harness, harness.client, notify, "notify").value() - self.assertEqual(notifier_held_the_lock, [True]) - self.assertTrue(harness.accounting.runnable()) - - harness.run_until(harness.accounting, "done:wait") - self.assertTrue(waiter.value()) - self.assertEqual(reacquired, [True]) - self.assertIsNone(condition._lock.owner) - - def test_a_timed_wait_expires_against_the_virtual_clock(self) -> None: - with LandingHarness() as harness: - condition = harness.condition("unfenced-append-drained") - started = harness.clock.monotonic() - - def wait_briefly() -> bool: - with condition: - return condition.wait(7.5) - - timed = harness.appender.submit(wait_briefly, label="timed-wait") - harness.run_until_blocked(harness.appender, "cond:unfenced-append-drained") - self.assertEqual(harness.appender.wake_at, started + 7.5) - - harness.advance_to_next_deadline() - harness.run_until(harness.appender, "done:timed-wait") - self.assertFalse(timed.value(), "an expiry is not a notification") - self.assertEqual(harness.clock.monotonic(), started + 7.5) - self.assertEqual(harness.sleeps, []) - - def test_re_entrant_depth_survives_a_wait(self) -> None: - """The coordinator's condition sits on a reentrant lock. - - A wait that re-acquired to depth one would leave the caller's outer - ``with`` releasing a lock it no longer holds — a failure a long way - from its cause. Proved behaviourally: after one release the lock is - still held, which can only be true if the depth came back as two. - """ - with LandingHarness() as harness: - condition = harness.condition("accepted-block-payout-preview") - depth_after_wait: list[bool] = [] - - def nested_wait() -> None: - condition.acquire() - condition.acquire() - condition.wait() - condition.release() - depth_after_wait.append(condition._lock.locked()) - condition.release() - - harness.accounting.submit(nested_wait, label="nested") - harness.run_until_blocked( - harness.accounting, - "cond:accepted-block-payout-preview", - ) - _run(harness, harness.client, condition.notify_all, "notify").value() - harness.run_until(harness.accounting, "done:nested") - - self.assertEqual(depth_after_wait, [True]) - self.assertIsNone(condition._lock.owner) - - -class HarnessSemaphoreTests(unittest.TestCase): - """The ledger's read slots, which a landing scenario can exhaust on purpose.""" - - def test_exhaustion_parks_and_a_release_admits_one_waiter(self) -> None: - with LandingHarness() as harness: - semaphore = harness.semaphore("ledger-read-slot", 1) - - def hold() -> None: - semaphore.acquire() - harness.scheduler.checkpoint("holding-slot") - semaphore.release() - - harness.client.submit(hold, label="hold") - harness.run_until(harness.client, "holding-slot") - self.assertEqual(semaphore.value, 0) - - waiter = harness.accounting.submit(semaphore.acquire, label="wait-slot") - harness.run_until_blocked(harness.accounting, "semaphore:ledger-read-slot") - - harness.step(harness.client) - self.assertTrue(harness.accounting.runnable()) - harness.run_until(harness.accounting, "done:wait-slot") - self.assertTrue(waiter.value()) - self.assertEqual(semaphore.value, 0) - - def test_releasing_above_the_initial_value_is_refused(self) -> None: - """``BoundedSemaphore``'s check, kept: an over-release is a real bug. - - A slot handed back twice inflates read concurrency for the rest of the - run, and the scenario that then passes is measuring a pool production - never had. - """ - with LandingHarness() as harness: - semaphore = harness.semaphore("ledger-read-slot", 2) - call = _run(harness, harness.client, semaphore.release, "over-release") - with self.assertRaises(HarnessError) as caught: - call.value() - self.assertIn("released too many times", str(caught.exception)) - - -class HarnessEventTests(unittest.TestCase): - def test_a_wait_parks_until_the_event_is_set(self) -> None: - with LandingHarness() as harness: - event = harness.event("block-landed") - waiter = harness.appender.submit(event.wait, label="await-event") - harness.run_until_blocked(harness.appender, "event:block-landed") - self.assertFalse(event.is_set()) - - _run(harness, harness.client, event.set, "set-event").value() - self.assertTrue(harness.appender.runnable()) - harness.run_until(harness.appender, "done:await-event") - self.assertTrue(waiter.value()) - - def test_a_timed_wait_expires_against_the_virtual_clock(self) -> None: - with LandingHarness() as harness: - event = harness.event("block-landed") - started = harness.clock.monotonic() - timed = harness.appender.submit( - lambda: event.wait(3.0), - label="timed-event", - ) - harness.run_until_blocked(harness.appender, "event:block-landed") - self.assertEqual(harness.appender.wake_at, started + 3.0) - - harness.advance_to_next_deadline() - harness.run_until(harness.appender, "done:timed-event") - self.assertFalse(timed.value()) - self.assertEqual(harness.clock.monotonic(), started + 3.0) - self.assertEqual(harness.sleeps, []) - - -def _landing_with_a_replay_in_the_publish_window( - harness: LandingHarness, -) -> dict[str, object]: - """One landing, with the allocator driven inside its confirm→publish gap. - - The gap is the landing model's one genuinely concurrent window — the - landing has confirmed durably and released every lock it took to do so — - so it is the right place to prove the allocator's answers do not depend on - which actor the controller steps next. - """ - ledger = harness.boot() - harness.break_at(PUBLISH_BOUNDARY) - found = harness.found_block(BLOCK_A, height=10, parent_hash=PARENT_HASH) - landed = harness.land_on_client_tail(found) - harness.run_until(harness.client, PUBLISH_BOUNDARY) - - _prepared(harness, BLOCK_B, height=11, parent_hash=BLOCK_A) - _prepared(harness, BLOCK_C, height=12, parent_hash=BLOCK_B) - replay = harness.accounting.submit( - lambda: ledger.confirm_accepted_block( - block_hash=BLOCK_A, - active_tip_height=10, - ), - label="replay-a", - ) - rejected = harness.accounting.submit( - lambda: ledger.reject_prepared_block( - block_hash=BLOCK_B, - active_tip_height=11, - ), - label="reject-b", - ) - harness.run_until(harness.accounting, "done:reject-b") - harness.drain([harness.client, harness.accounting]) - - # After the window rather than inside it, and deliberately so. A bare - # confirmation allocates an ordinal that no publication ever follows, and - # a landing still holding a lower one then fails its own publish against a - # floor nothing published raised. That is the landing refusing to publish - # behind the ledger, not a model artefact — but it belongs to a scenario - # about the landing, and this one is about the allocator. - confirmed = _run( - harness, - harness.accounting, - lambda: ledger.confirm_accepted_block( - block_hash=BLOCK_C, - active_tip_height=12, - ), - "confirm-c", - ) - - return { - "landed": landed.error is None and landed.result is True, - "a_sequence": harness.publication_sequence(found), - "replay": replay.value(), - "rejected": rejected.value(), - "confirmed": confirmed.value(), - "floor": harness.publication_floor(), - "summary": harness.landing_summary(), - } - - -class LandingModelDeterminismTests(unittest.TestCase): - """A scenario that is merely usually right is worse than no scenario.""" - - def test_the_landing_model_is_order_stable_across_repeated_runs(self) -> None: - outcome = assert_deterministic( - self, - _landing_with_a_replay_in_the_publish_window, - harness_factory=LandingHarness, - ).outcome - assert isinstance(outcome, dict) - - self.assertTrue(outcome["landed"]) - self.assertEqual(outcome["a_sequence"], 1) - # The replay ran in the gap and reported the ordinal already assigned. - self.assertEqual(outcome["replay"]["confirmed_count"], 1) - self.assertEqual(outcome["replay"]["audit_publication_sequence"], 1) - self.assertEqual(outcome["rejected"]["rejected_count"], 1) - # And it burned nothing: the next fresh confirmation takes 2. - self.assertEqual(outcome["confirmed"]["audit_publication_sequence"], 2) - self.assertEqual(outcome["floor"], 2) - - -if __name__ == "__main__": - unittest.main() diff --git a/tests/test_prism_landing_interleavings.py b/tests/test_prism_landing_interleavings.py deleted file mode 100644 index 2eb542aad..000000000 --- a/tests/test_prism_landing_interleavings.py +++ /dev/null @@ -1,206 +0,0 @@ -#!/usr/bin/env python3 -"""Two distinct found blocks interleaved across PRISM's two landing tails. - -Issue #133: when two *distinct* found blocks land inside one confirm→publish -window and one of them lands on the synchronous client-thread route, the -earlier block's live audit envelope was silently and permanently never -written. Nothing was lost but the published evidence pointer — the block was -mined, paid and audited correctly — which is exactly why no error appeared -and no test caught it. The reachable case was untested because the per-hash -disposition lease provides no mutual exclusion across hashes, and no test -interleaved two hashes. - -That defect is fixed on ``2.x.x`` by ``03e733e``, so these scenarios pass. -The evidence they carry is not that the fix works — its own PR asserted that -against a hand-built store — but that the *interleaving* can now be driven -end to end, through the real landing tails, the real publication-ordinal -allocator and a real audit root. Nothing could do that before. -""" - -from __future__ import annotations - -import sys -import unittest -from pathlib import Path - -sys.path.insert(0, str(Path(__file__).resolve().parents[1])) - -from tests.prism_concurrency_harness import assert_deterministic # noqa: E402 -from tests.prism_landing_harness import ( # noqa: E402 - PARENT_HASH, - LandingHarness, -) - -BLOCK_A = "aa" * 32 -BLOCK_B = "bb" * 32 - -# The top of the publish step, named by the landing itself. Every tail stops -# here having confirmed durably and having released the payout-balance -# serializer, the publication order guard and the ledger's stats -# single-flight — which is precisely the window #133 describes as open to a -# descendant. -PUBLISH_BOUNDARY = "progress:evidence-write" - - -def _two_tail_landing(harness: LandingHarness) -> dict[str, object]: - """Land A on the client tail and B on the accounting tail, interleaved. - - Block B is A's child: #133's trigger is explicit that the window is open - to descendants because A clears its accepted-block payout preview — the - child-deferral gate — before it publishes. - """ - harness.boot() - harness.break_at(PUBLISH_BOUNDARY) - block_a = harness.found_block(BLOCK_A, height=10, parent_hash=PARENT_HASH) - block_b = harness.found_block(BLOCK_B, height=11, parent_hash=BLOCK_A) - - landed_a = harness.land_on_client_tail(block_a) - landed_b = harness.land_on_accounting_tail(block_b) - - # A confirms and stops at the top of its publish step. - harness.run_until(harness.client, PUBLISH_BOUNDARY) - a_sequence = harness.publication_sequence(block_a) - floor_before_b = harness.publication_floor() - - # B lands entirely inside that window: confirm, ordinal, publish. - harness.run_until(harness.accounting, PUBLISH_BOUNDARY) - harness.drain([harness.accounting]) - b_sequence = harness.publication_sequence(block_b) - - # A resumes and publishes against a floor B has already raised. - harness.drain([harness.client, harness.accounting]) - - return { - "a_sequence": a_sequence, - "b_sequence": b_sequence, - "floor_before_b": floor_before_b, - "floor_after": harness.publication_floor(), - "a_landed": landed_a.error is None and landed_a.result is True, - "b_landed": landed_b.error is None, - "a_envelope_written": harness.envelope_written(block_a), - "b_envelope_written": harness.envelope_written(block_b), - "a_outbox": harness.outbox_state(block_a), - "b_outbox": harness.outbox_state(block_b), - "summary": harness.landing_summary(), - } - - -class TwoDistinctHashLandingTests(unittest.TestCase): - """#133: the earlier ordinal must keep its own published evidence.""" - - def test_harness_drives_the_two_distinct_hash_interleaving(self) -> None: - outcome = assert_deterministic( - self, - _two_tail_landing, - harness_factory=LandingHarness, - ).outcome - assert isinstance(outcome, dict) - - # The interleaving actually happened: A took the lower ordinal, B - # took the next one and raised the durable floor past A while A was - # still between its confirmation and its publication. - self.assertEqual(outcome["a_sequence"], 1) - self.assertEqual(outcome["b_sequence"], 2) - self.assertEqual(outcome["floor_before_b"], 1) - self.assertEqual(outcome["floor_after"], 2) - - # Both blocks land, and neither is abandoned by the race. - self.assertTrue(outcome["a_landed"]) - self.assertTrue(outcome["b_landed"]) - self.assertEqual(outcome["a_outbox"], "submitted") - self.assertEqual(outcome["b_outbox"], "submitted") - - # The assertion #133 exists for. A published at sequence 1 against a - # floor of 2, so it is superseded on the evidence pointer and never - # becomes the current reference — but its own height+hash envelope is - # its only published evidence, it competes with nothing, and it must - # be on disk. Before 03e733e this file was silently absent. - self.assertTrue(outcome["a_envelope_written"]) - self.assertTrue(outcome["b_envelope_written"]) - - def test_the_superseded_publication_is_reported_not_silent(self) -> None: - """A superseded envelope write must leave a trace an operator can find. - - The original defect's whole character was silence: no error, no log, - a permanent gap in the public audit trail. A fix that restored the - file but stayed quiet would still leave nobody able to tell the - interleaving had happened. - """ - with LandingHarness() as harness: - _two_tail_landing(harness) - # Audit-path diagnostics go to stderr: worker processes reserve - # stdout for a strict JSON-lines protocol. - self.assertIn( - "wrote live envelope for superseded publication", - harness.errors, - ) - self.assertIn("sequence=1 floor_sequence=2", harness.errors) - - def test_restore_authority_is_gated_on_a_fresh_lease_proof(self) -> None: - """Withheld authority degrades to the old behaviour, not to a failure. - - Only the live writer for a landing may mutate the shared audit root. - A process whose writer lease was taken over inside the confirm→publish - gap reaches the same call site with stale reads behind it, and - publishing a stale report as a block's public evidence is worse than - leaving the envelope missing. So the restore is gated — and when the - gate refuses, the publication must still succeed, because failing a - found-block landing over a lease hiccup would be a live regression. - """ - - def scenario(harness: LandingHarness) -> dict[str, object]: - harness.withheld_lease_components = { - "superseded_audit_envelope_restore" - } - outcome = _two_tail_landing(harness) - return { - "a_envelope_written": outcome["a_envelope_written"], - "b_envelope_written": outcome["b_envelope_written"], - "a_landed": outcome["a_landed"], - "b_landed": outcome["b_landed"], - } - - outcome = assert_deterministic( - self, - scenario, - harness_factory=LandingHarness, - ).outcome - assert isinstance(outcome, dict) - # The landing still succeeds on both tails. - self.assertTrue(outcome["a_landed"]) - self.assertTrue(outcome["b_landed"]) - # B is unaffected: it is not superseded, so it never asks. - self.assertTrue(outcome["b_envelope_written"]) - # A degrades to the pre-fix behaviour rather than failing. - self.assertFalse(outcome["a_envelope_written"]) - - def test_the_disposition_lease_does_not_serialise_distinct_hashes(self) -> None: - """The topology the defect needs, asserted rather than assumed. - - If the per-hash disposition lease did serialise across hashes, this - whole scenario would be unreachable and the tests above would be - proving nothing. The claim is load-bearing, so it is checked: with A - holding its own disposition and parked mid-landing, B's tail claims - its disposition and completes a full landing. - """ - with LandingHarness() as harness: - harness.boot() - harness.break_at(PUBLISH_BOUNDARY) - block_a = harness.found_block(BLOCK_A, height=10, parent_hash=PARENT_HASH) - block_b = harness.found_block(BLOCK_B, height=11, parent_hash=BLOCK_A) - harness.land_on_client_tail(block_a) - landed_b = harness.land_on_accounting_tail(block_b) - - harness.run_until(harness.client, PUBLISH_BOUNDARY) - self.assertIsNone(harness.publication_sequence(block_b)) - - # A is mid-landing and holds its hash's disposition. B proceeds - # anyway, all the way through its own durable confirmation. - harness.run_until(harness.accounting, PUBLISH_BOUNDARY) - harness.drain([harness.accounting]) - self.assertIsNone(landed_b.error) - self.assertEqual(harness.publication_sequence(block_b), 2) - - -if __name__ == "__main__": - unittest.main() diff --git a/tests/test_prism_lease_heartbeat_tail_latency.py b/tests/test_prism_lease_heartbeat_tail_latency.py deleted file mode 100644 index 871a5dad5..000000000 --- a/tests/test_prism_lease_heartbeat_tail_latency.py +++ /dev/null @@ -1,978 +0,0 @@ -#!/usr/bin/env python3 -"""False lease-heartbeat exits under normal tail latency (issue #212). - -What could not be tested before -------------------------------- -``union-mainnet`` restarted 69 times during one rapid-block burst while -PostgreSQL, qbitd, the RAID array, CPU, memory and disk were all healthy and -the coordinator still held its PostgreSQL advisory guard. The exits were -real protective exits: the heartbeat's server-proven envelope had genuinely -elapsed. It had elapsed because the envelope was sized for an idle interval -and not for the tail of a lawful verification. - -Reproducing that needs three things at once — a guard statement running near -its server-side timeout, the coordinator's own threads being descheduled -behind the burst, and a fenced write holding the lease tuple — and needs them -in a fixed order. Running the coordinator and hoping produces the schedule -by accident at best. This module puts the real -:class:`LedgerLeaseHeartbeatService` loops on the #128 deterministic harness: -its heartbeat and monitor become two harness actors, every interval they -measure reads the harness's virtual clock, and modelled server time is -charged per statement. Nothing sleeps. - -What is proved --------------- -``RapidBlockBurstTests`` - A production-shaped burst — near-bound guard statements, repeated - scheduler stalls, an in-flight fenced write on the lease tuple, and - external-effect fence callers competing for the guard's query slot — - produces no hard exit, while the peak staleness it reaches is above - what the pre-#212 policy tolerated. The regression is anchored to the - old bounds, so a future retune that reintroduces them fails here. - -``ProofRenewalSplitTests`` - The frequent statement is the cheap ownership proof, renewal escalates - on the same beat the committed row enters the authority margin, and - neither ever queues on the lease tuple's row lock. - -``FailClosedTests`` - The safety half. A lost guard session, a wedged proof that never - answers, and a guarded statement cancelled by its own statement timeout - each terminate this coordinator's authority strictly before a successor - becomes eligible to adopt — measured against the successor's real - guard-acquisition edge, not against a constant. -""" - -from __future__ import annotations - -from typing import Callable -import unittest - -from lab.prism.background_services import ( - LedgerLeaseHeartbeatPorts, - LedgerLeaseHeartbeatService, -) -from lab.prism.writer_lease_timing import ( - DEFAULT_WRITER_LEASE_HEARTBEAT_POLICY, - LEASE_HEARTBEAT_MODE_FENCE, - WriterLeaseHeartbeatPolicy, - WriterLeaseVerificationAttempt, -) -from tests.prism_concurrency_harness import ( - GUARD_STATEMENT_TIMEOUT_SECONDS, - LeaseHarness, -) -from tests.test_prism_writer_lease_policy import PRE_212_POLICY - -# The shipped policy, used verbatim: these scenarios exist to judge the -# defaults operators actually run, not a scaled-down stand-in. -POLICY = DEFAULT_WRITER_LEASE_HEARTBEAT_POLICY - -# Guard statements run just inside their server-side ceiling — the tail the -# burst produced, not a pathological value. Anything above the ceiling is -# cancelled by PostgreSQL and is a different scenario (see FailClosedTests). -TAIL_STATEMENT_SECONDS = 0.45 - -# One descheduling of the coordinator's threads behind the burst. Well inside -# the policy's scheduler-slack allowance and far outside what the pre-#212 -# envelope could absorb. -SCHEDULER_STALL_SECONDS = 0.35 - -# Virtual seconds of burst. Long enough to cross the own-write authority -# margin (half the 60s lease TTL) and prove the renewal escalation fires on -# its own, rather than only on the first beat. -BURST_SECONDS = 40.0 - -LEASE_TTL_SECONDS = 60.0 - -# A fenced write's caller deadline. Arming one is what makes the ledger wrap -# the statement in an explicit transaction whose COMMIT is a separate client -# message, which is how the lease tuple stays row-locked across the burst. -FENCED_WRITE_BUDGET_SECONDS = 600.0 - - -class HeartbeatUnderTest: - """The real heartbeat service, wired onto the deterministic harness. - - Substitution goes through ports, exactly as the harness does for the - ledger: the loops, the fail-closed bookkeeping, the phase attribution - and the monitor's arithmetic are the shipped implementations. Only the - clock, the synchronisation primitives and the process-exit seam are - supplied by the harness, so what a scenario observes is what production - would do on the same schedule. - """ - - def __init__( - self, - harness: LeaseHarness, - coordinator, - *, - policy: WriterLeaseHeartbeatPolicy = POLICY, - name: str = "hb", - ) -> None: - self.harness = harness - self.policy = policy - self.exits: list[tuple[float, str, str | None]] = [] - service: LedgerLeaseHeartbeatService - - def lease_hard_exit(message: str, *, include_traceback: bool) -> None: - # The coordinator's delegate, spelled the same way: the real - # fail-closed bookkeeping (single-arming, exit-thread exemption, - # failure reason) has to run for the assertions to mean anything. - service.hard_exit(message, include_traceback=include_traceback) - - def watchdog_hard_exit(reason: str, *, timeout_seconds: float) -> None: - # Stands in for os._exit: record when the process would have - # gone, and stop both loops, because a dead process runs nothing. - self.exits.append( - (harness.clock.monotonic(), reason, service.failure_reason) - ) - service.stop_event.set() - - ports = LedgerLeaseHeartbeatPorts( - ledger=lambda: coordinator.ledger, - heartbeat_seconds=lambda: policy.heartbeat_interval_seconds, - failure_seconds=lambda: policy.failure_budget_seconds, - monitor_seconds=lambda: policy.monitor_interval_seconds, - exit_timeout_seconds=lambda: policy.exit_margin_seconds, - scheduler_slack_seconds=lambda: policy.scheduler_slack_seconds, - external_fence_timeout_seconds=lambda: 0.5, - lease_hard_exit=lease_hard_exit, - watchdog_hard_exit=watchdog_hard_exit, - heartbeat_loop=lambda: service.heartbeat_loop(), - monitor_loop=lambda: service.monitor_loop(), - monotonic=harness.clock.monotonic, - ) - service = LedgerLeaseHeartbeatService(ports) - # The freshness and failure locks stay real: the service never - # yields while holding either, and under the baton scheduler no two - # actors run at once, so neither can be contended. The stop/ready/ - # failed events do have to be scheduler-visible, because the loops - # wait on them with a timeout and that wait is what makes the - # heartbeat interval and the monitor poll observable. - service.stop_event = harness.event(f"{name}.stop") - service.ready = harness.event(f"{name}.ready") - service.failed = harness.event(f"{name}.failed") - armed = harness.clock.monotonic() - service.last_success_monotonic = armed - service.last_server_proven_monotonic = armed - self.service = service - self.armed_monotonic = armed - self.heartbeat = harness.scheduler.actor(f"{name}-heartbeat") - self.monitor = harness.scheduler.actor(f"{name}-monitor") - self.heartbeat.submit(service.heartbeat_loop, label="heartbeat-loop") - self.monitor.submit(service.monitor_loop, label="monitor-loop") - self.peak_server_proven_age = 0.0 - self.peak_activity_age = 0.0 - - # -- observation ------------------------------------------------------- - - @property - def exited(self) -> bool: - return bool(self.exits) - - @property - def exit_monotonic(self) -> float: - if not self.exits: - raise AssertionError("the heartbeat did not hard-exit") - return self.exits[0][0] - - @property - def exit_reason(self) -> str: - if not self.exits: - raise AssertionError("the heartbeat did not hard-exit") - return self.exits[0][2] or "" - - def observe(self) -> None: - """Sample staleness the way the monitor sees it, for peak reporting.""" - self.peak_server_proven_age = max( - self.peak_server_proven_age, - self.service.server_proven_age_seconds(), - ) - self.peak_activity_age = max( - self.peak_activity_age, - self.service.activity_age_seconds(), - ) - - # -- driving ----------------------------------------------------------- - - def fence_call(self, ledger) -> Callable[[], object]: - """The external-effect fence's guard call, minus its worker thread. - - ``require_fresh_lease_for_external_side_effect`` runs the - verification on a daemon thread the baton scheduler cannot drive, - so a scenario calls the same shipped verification with the same two - callbacks wired the same way. What is under test through this seam - is the guard's single query slot and the marks a fence leaves - behind, both of which are identical either way — and leaving the - marks out would model a guard caller that does not exist. - """ - attempt = WriterLeaseVerificationAttempt( - LEASE_HEARTBEAT_MODE_FENCE, - monotonic=self.harness.clock.monotonic, - ) - - def run() -> object: - return ledger.verify_writer_lease_guard_session( - on_query_start=attempt.slot_acquired, - on_statement_end=lambda: self.service.record_server_proven( - attempt.statement_completed() - ), - ) - - return run - - def advance_until_beat(self, others: list) -> None: - """Jump forward until the heartbeat's next beat is due. - - Other actors keep their own turns as their timers come up, so the - heartbeat is not privileged by the fast-forward. - """ - while not self.heartbeat.runnable(): - if self.harness.advance_to_next_deadline() is None: - raise AssertionError("nothing bounds the heartbeat's idle wait") - for actor in others: - while actor.runnable(): - self.harness.step(actor) - - def run_monitor_maximally_late( - self, - *, - rounds: int, - first_poll_monotonic: float | None = None, - ) -> None: - """Drive only the monitor, each poll as late as the policy allows. - - The worst schedule the safety inequality claims to survive: the - heartbeat thread gets no turn at all, and the monitor — the thread - that has to notice and exit — wakes ``monitor_interval + - scheduler_slack`` after its previous poll every single time. - - ``first_poll_monotonic`` pins the phase of that grid. Lateness - alone is not the worst case: a poll that happens to land just after - a bound elapses notices immediately, and the scenario then proves - nothing about the term being tested. Placing one poll immediately - *before* the bound forces the next one to arrive a full late - interval after it, which is the case the envelope has to cover. - """ - late_by = ( - self.policy.monitor_interval_seconds - + self.policy.scheduler_slack_seconds - ) - if first_poll_monotonic is not None: - self.harness.clock.advance_to( - max(first_poll_monotonic, self.harness.clock.monotonic()) - ) - while self.monitor.runnable(): - self.harness.step(self.monitor) - self.observe() - for _ in range(rounds): - if self.exited: - return - self.harness.clock.advance(late_by) - while self.monitor.runnable(): - self.harness.step(self.monitor) - self.observe() - - def stall_heartbeat(self, seconds: float) -> None: - """Deschedule the coordinator's heartbeat thread for ``seconds``. - - Only the monitor runs, in its own poll steps, which is what a GIL - stall behind a rapid-block burst looks like from the monitor's side: - time passes, no new activity mark appears, and the decision to - hard-exit or not is taken on the policy alone. - """ - target = self.harness.clock.monotonic() + seconds - while self.harness.clock.monotonic() < target and not self.exited: - self.harness.clock.advance_to( - min( - target, - self.harness.clock.monotonic() - + self.policy.monitor_interval_seconds, - ) - ) - while self.monitor.runnable(): - self.harness.step(self.monitor) - self.observe() - - def run( - self, - actors: list, - *, - seconds: float, - stall_every: int = 0, - stall_seconds: float = SCHEDULER_STALL_SECONDS, - limit: int = 8000, - ) -> None: - """Drain every runnable actor, then jump to the next pending timer.""" - started = self.harness.clock.monotonic() - rounds = 0 - while ( - self.harness.clock.monotonic() - started < seconds - and rounds < limit - and not self.exited - ): - self.harness.drain(actors) - self.observe() - if stall_every and rounds % stall_every == stall_every - 1: - self.stall_heartbeat(stall_seconds) - if self.harness.advance_to_next_deadline() is None: - break - rounds += 1 - - def quiesce(self, actors: list) -> None: - self.service.stop_event.set() - self.harness.drain(actors) - - -def guard_statement_latency(seconds: float): - """Charge ``seconds`` of modelled server time to each guarded statement.""" - - def latency(statement) -> float: - return seconds if statement.kind.value in {"prove", "verify"} else 0.0 - - return latency - - -def statement_counts(harness: LeaseHarness) -> dict[str, int]: - kinds = harness.statement_kinds() - return {kind: kinds.count(kind) for kind in set(kinds)} - - -class RapidBlockBurstTests(unittest.TestCase): - def test_burst_with_tail_latency_and_stalls_never_hard_exits(self) -> None: - """The regression, on the schedule that produced it. - - Guard statements run just under their server-side ceiling, the - coordinator's threads are descheduled repeatedly, a fenced write - holds the lease tuple for the whole burst, and external-effect - fences compete for the guard's single query slot. Nothing here is - a fault; every restart the burst produced was a healthy - coordinator being told its own session was gone. - """ - with LeaseHarness( - lease_ttl_seconds=LEASE_TTL_SECONDS, - capture_output=True, - ) as harness: - alpha = harness.coordinator("alpha") - alpha.start() - harness.run_until(alpha, "done:startup") - heartbeat = HeartbeatUnderTest(harness, alpha) - harness.server.statement_latency_seconds = guard_statement_latency( - TAIL_STATEMENT_SECONDS - ) - - # A long fenced write parked mid-statement: PostgreSQL holds the - # lease tuple's row lock and the COMMIT has not been sent. This - # is the shape persist_accepted_block leaves behind. - def fenced_write() -> object: - with alpha.ledger.operation_timeout(FENCED_WRITE_BUDGET_SECONDS): - return alpha.ledger.renew_writer_lease() - - alpha.submit(fenced_write, label="fenced-write") - harness.run_until(alpha, "alpha.done:renew") - - fence = harness.scheduler.actor("external-fence") - actors = [heartbeat.heartbeat, heartbeat.monitor, fence] - heartbeat.run( - actors, - seconds=BURST_SECONDS, - stall_every=7, - ) - heartbeat.quiesce(actors) - - self.assertFalse( - heartbeat.exited, - f"healthy coordinator hard-exited: {heartbeat.exits}", - ) - # The burst really did produce the staleness the issue reported: - # both pre-#212 bounds are crossed by a coordinator that never - # stopped holding its guard. - self.assertGreaterEqual( - heartbeat.peak_server_proven_age, - PRE_212_POLICY.server_proven_cap_seconds, - ) - self.assertLess( - heartbeat.peak_server_proven_age, - POLICY.server_proven_cap_seconds, - ) - self.assertLess( - heartbeat.peak_activity_age, - POLICY.failure_budget_seconds, - ) - # The lease tuple was locked throughout, and no guarded - # statement ever queued for it. - self.assertIsNotNone(harness.server.lease_lock_holder) - self.assertEqual( - [stop for stop in harness.trace if "guard" in stop and "lockwait" in stop], - [], - ) - - def test_monitor_lateness_is_attributed_and_survived(self) -> None: - """A stalled monitor thread spends envelope like a slow statement. - - The monitor decides the exit, so its own lateness has to be - measurable separately from database latency — otherwise an operator - reading an exit cannot tell "PostgreSQL was slow" from "this - process could not get scheduled". - """ - with LeaseHarness(lease_ttl_seconds=LEASE_TTL_SECONDS) as harness: - alpha = harness.coordinator("alpha") - alpha.start() - harness.run_until(alpha, "done:startup") - heartbeat = HeartbeatUnderTest(harness, alpha) - actors = [heartbeat.heartbeat, heartbeat.monitor] - heartbeat.run(actors, seconds=1.0) - - # Deschedule the monitor alone: the heartbeat keeps proving. - late_by = 0.4 - harness.advance(heartbeat.policy.monitor_interval_seconds + late_by) - while heartbeat.monitor.runnable(): - harness.step(heartbeat.monitor) - heartbeat.run(actors, seconds=1.0) - heartbeat.quiesce(actors) - - self.assertFalse(heartbeat.exited) - self.assertGreaterEqual( - heartbeat.service.monitor_wake_delay_seconds, - late_by, - ) - snapshot = heartbeat.service.snapshot() - self.assertGreaterEqual( - float(snapshot["monitor_wake_delay_seconds"]), - late_by, - ) - - -class ProofRenewalSplitTests(unittest.TestCase): - def test_beats_prove_ownership_and_escalate_only_when_renewal_is_due( - self, - ) -> None: - """Frequent ownership proof, infrequent TTL renewal — same guarantees. - - The first beat renews, because startup readiness must prove the - renewing path works. After that every beat asks the cheap ownership - question until the committed row falls inside the own-write - authority margin (half the 60s TTL), at which point renewal - escalates on that same beat. - """ - with LeaseHarness(lease_ttl_seconds=LEASE_TTL_SECONDS) as harness: - alpha = harness.coordinator("alpha") - alpha.start() - harness.run_until(alpha, "done:startup") - heartbeat = HeartbeatUnderTest(harness, alpha) - harness.server.statement_latency_seconds = guard_statement_latency( - TAIL_STATEMENT_SECONDS - ) - actors = [heartbeat.heartbeat, heartbeat.monitor] - heartbeat.run(actors, seconds=BURST_SECONDS) - heartbeat.quiesce(actors) - - counts = statement_counts(harness) - self.assertFalse(heartbeat.exited) - # Proof dominates by a wide margin: renewal runs on the first - # beat and once more when half the TTL has eroded. - self.assertGreater(counts["prove"], 20) - self.assertEqual(counts["verify"], 2) - self.assertGreater(counts["prove"], 10 * counts["verify"]) - - attempts = heartbeat.service.attempt_counts - outcomes = heartbeat.service.outcome_counts - self.assertEqual(attempts["renew"], counts["verify"]) - self.assertEqual(attempts["proof"], counts["prove"]) - # Exactly one proof reported the row renewal-due; that beat - # escalated immediately rather than waiting for the next one. - self.assertEqual(outcomes["renewal_due"], 1) - self.assertEqual(outcomes["renewed"], counts["verify"]) - self.assertEqual(outcomes["failed"], 0) - - def test_proof_never_queues_behind_an_in_flight_fenced_write(self) -> None: - """The block-39416 property, preserved across the split. - - The proof takes no row lock at all — it asks two ``EXISTS`` - questions — so a fenced write holding the lease tuple for the whole - of its transaction cannot make a beat queue and die on the guard's - statement timeout. - """ - with LeaseHarness(lease_ttl_seconds=LEASE_TTL_SECONDS) as harness: - alpha = harness.coordinator("alpha") - alpha.start() - harness.run_until(alpha, "done:startup") - - def fenced_write() -> object: - with alpha.ledger.operation_timeout(FENCED_WRITE_BUDGET_SECONDS): - return alpha.ledger.renew_writer_lease() - - alpha.submit(fenced_write, label="fenced-write") - harness.run_until(alpha, "alpha.done:renew") - self.assertIsNotNone(harness.server.lease_lock_holder) - - heartbeat = HeartbeatUnderTest(harness, alpha) - actors = [heartbeat.heartbeat, heartbeat.monitor] - heartbeat.run(actors, seconds=2.0) - heartbeat.quiesce(actors) - - self.assertFalse(heartbeat.exited) - self.assertGreater(statement_counts(harness)["prove"], 1) - self.assertEqual( - [stop for stop in harness.trace if "lockwait" in stop], - [], - ) - - def test_guard_slot_contention_is_attributed_to_queue_wait(self) -> None: - """A beat that queued behind a fence must not look like slow SQL. - - The guard session is one connection with one query slot, so an - external-effect fence and the heartbeat serialize. Charging that - wait to the database would send an operator to PostgreSQL for a - problem that is in this process. - """ - with LeaseHarness(lease_ttl_seconds=LEASE_TTL_SECONDS) as harness: - alpha = harness.coordinator("alpha") - alpha.start() - harness.run_until(alpha, "done:startup") - heartbeat = HeartbeatUnderTest(harness, alpha) - harness.server.statement_latency_seconds = guard_statement_latency( - TAIL_STATEMENT_SECONDS - ) - actors = [heartbeat.heartbeat, heartbeat.monitor] - heartbeat.run(actors, seconds=1.0) - - # Park a fence caller inside the guard's slot, then let the - # heartbeat's next beat arrive behind it. - fence = harness.scheduler.actor("external-fence") - fence.submit( - heartbeat.fence_call(alpha.ledger), - label="submitblock-fence", - ) - harness.run_until(fence, "alpha.guard.begin:verify") - heartbeat.advance_until_beat([heartbeat.monitor]) - harness.run_until_blocked( - heartbeat.heartbeat, - "alpha.guard.slot:wait", - ) - # Only now does the fence's statement spend its server time, so - # every second of it is queue wait for the heartbeat. - harness.drain([fence] + actors) - heartbeat.quiesce(actors + [fence]) - - self.assertFalse(heartbeat.exited) - queued = [ - phases - for phases in (heartbeat.service.worst_phases,) - if phases.slot_wait_seconds > 0.0 - ] - self.assertTrue( - queued, - "a beat queued behind the fence but recorded no slot wait", - ) - self.assertGreaterEqual( - heartbeat.service.worst_phases.slot_wait_seconds, - TAIL_STATEMENT_SECONDS, - ) - - -class FailClosedTests(unittest.TestCase): - """Losing authority must still be fatal, and fatal early enough.""" - - def takeover_deadline(self, harness: LeaseHarness, successor) -> float: - """Earliest virtual instant the successor may compare-and-swap. - - Measured from the instant the successor actually took the writer - advisory guard, because that is the edge - ``_writer_lease_adoption_wait_seconds`` counts its silence from — - deliberately, so a predecessor that just lost the guard always gets - its whole self-fencing budget however stale its lease row looks. - """ - successor.start() - return self.step_until_guard_held(harness, successor) - - def step_until_guard_held(self, harness: LeaseHarness, successor) -> float: - for _ in range(50): - if successor.guard is not None and successor.guard.held: - return harness.clock.monotonic() + POLICY.adoption_silence_seconds - if not successor.actor.runnable(): - break - harness.step(successor.actor) - raise AssertionError("the successor never acquired the writer guard") - - def assert_guard_withheld(self, harness: LeaseHarness, successor) -> None: - """A successor cannot even start its silence while we hold the guard. - - This is the first fence, ahead of any timing argument: the advisory - lock is a session lock, so a predecessor that is wedged — rather - than dead — keeps it, and the successor's adoption clock has not - begun. - """ - successor.start() - for _ in range(20): - if not successor.actor.runnable(): - break - harness.step(successor.actor) - self.assertFalse( - successor.guard is not None and successor.guard.held, - "a successor took the writer guard while the predecessor held it", - ) - - def test_lost_guard_session_exits_before_takeover_eligibility(self) -> None: - """An injected guard-session loss fences the old coordinator first.""" - with LeaseHarness(lease_ttl_seconds=LEASE_TTL_SECONDS) as harness: - alpha = harness.coordinator("alpha") - alpha.start() - harness.run_until(alpha, "done:startup") - heartbeat = HeartbeatUnderTest(harness, alpha) - harness.server.statement_latency_seconds = guard_statement_latency( - TAIL_STATEMENT_SECONDS - ) - actors = [heartbeat.heartbeat, heartbeat.monitor] - heartbeat.run(actors, seconds=1.0) - self.assertFalse(heartbeat.exited) - - # The guard connection drops: the advisory lock is released - # server-side and this coordinator no longer owns anything. - assert alpha.guard is not None - alpha.guard.close() - lost_at = harness.clock.monotonic() - - beta = harness.coordinator("beta") - deadline = self.takeover_deadline(harness, beta) - heartbeat.run( - actors + [beta.actor], - seconds=POLICY.adoption_silence_seconds, - ) - - self.assertTrue(heartbeat.exited, "a lost guard did not fence") - self.assertLess(heartbeat.exit_monotonic, deadline) - self.assertLess( - heartbeat.exit_monotonic - lost_at, - POLICY.adoption_silence_seconds, - ) - self.assertIn("ledger lease heartbeat failed", heartbeat.exit_reason) - # The exit names the phase breakdown of the attempt that failed. - self.assertIn("mode=proof", heartbeat.exit_reason) - self.assertIn("outcome=failed", heartbeat.exit_reason) - self.assertEqual(heartbeat.service.outcome_counts["failed"], 1) - - def test_wedged_proof_exits_before_takeover_eligibility(self) -> None: - """A statement that never answers still terminates authority. - - A wedged beat stamps nothing at all after its attempt start, so the - liveness budget is the bound that fires first and the server-proven - edge — which no client-side mark can move — is stale behind it. The - exit has to land inside the adoption silence, and the reason has to - say which of the two bounds decided it. - """ - with LeaseHarness(lease_ttl_seconds=LEASE_TTL_SECONDS) as harness: - alpha = harness.coordinator("alpha") - alpha.start() - harness.run_until(alpha, "done:startup") - heartbeat = HeartbeatUnderTest(harness, alpha) - actors = [heartbeat.heartbeat, heartbeat.monitor] - heartbeat.run(actors, seconds=1.0) - wedged_at = harness.clock.monotonic() - - def wedge(statement) -> float: - if statement.kind.value in {"prove", "verify"}: - # No answer, no error: the beat is abandoned mid - # statement exactly as a black-holed session leaves it. - harness.scheduler.park_forever("alpha.guard.wedged") - return 0.0 - - harness.server.statement_latency_seconds = wedge - - beta = harness.coordinator("beta") - # A wedged coordinator still holds the session advisory lock, so - # the successor cannot even begin its silence window yet. - self.assert_guard_withheld(harness, beta) - heartbeat.run( - actors + [beta.actor], - seconds=POLICY.adoption_silence_seconds, - ) - - self.assertTrue(heartbeat.exited, "a wedged proof did not fence") - self.assertLess( - heartbeat.exit_monotonic - wedged_at, - POLICY.adoption_silence_seconds, - ) - # The process is gone, which is what finally drops the guard. - # Only then does the successor's full silence window start, so - # the old coordinator's authority ended strictly first. - assert alpha.guard is not None - alpha.guard.close() - deadline = self.step_until_guard_held(harness, beta) - self.assertLess(heartbeat.exit_monotonic, deadline) - reason = heartbeat.exit_reason - self.assertIn("stopped making progress", reason) - # The server-proven edge is the bound that decides a wedge: it - # is a whole idle interval staler than the client-side activity - # mark the abandoned beat left behind, so it reaches its cap - # first. That ordering is the point of measuring the adoption - # envelope from completed round trips. - self.assertIn("Tripped: server-proven", reason) - self.assertIn("adoption envelope cap", reason) - self.assertGreater( - heartbeat.service.server_proven_age_seconds(), - heartbeat.service.activity_age_seconds(), - ) - # Every term of the policy that produced the decision is in the - # exit, so an operator never has to guess which bound tripped. - self.assertIn("Last attempt:", reason) - self.assertIn("Monitor wake delay:", reason) - self.assertIn("server_proven_cap=", reason) - - def test_statement_cancelled_by_its_own_timeout_fails_closed(self) -> None: - """A guarded statement PostgreSQL cancels is loss, not latency. - - The guard session's statement_timeout is the ceiling the whole - envelope is sized against. A statement that reaches it produces an - error, and an error on the guarded session is loss of the guarded - session — never something to retry into the next beat. - """ - with LeaseHarness(lease_ttl_seconds=LEASE_TTL_SECONDS) as harness: - alpha = harness.coordinator("alpha") - alpha.start() - harness.run_until(alpha, "done:startup") - heartbeat = HeartbeatUnderTest(harness, alpha) - actors = [heartbeat.heartbeat, heartbeat.monitor] - heartbeat.run(actors, seconds=1.0) - self.assertFalse(heartbeat.exited) - - harness.server.statement_latency_seconds = guard_statement_latency( - GUARD_STATEMENT_TIMEOUT_SECONDS * 2 - ) - beta = harness.coordinator("beta") - self.assert_guard_withheld(harness, beta) - heartbeat.run( - actors + [beta.actor], - seconds=POLICY.adoption_silence_seconds, - ) - - self.assertTrue(heartbeat.exited) - self.assertIn("ledger lease heartbeat failed", heartbeat.exit_reason) - assert alpha.guard is not None - alpha.guard.close() - self.assertLess( - heartbeat.exit_monotonic, - self.step_until_guard_held(harness, beta), - ) - - def assert_exit_precedes_eligibility( - self, - harness: LeaseHarness, - heartbeat: HeartbeatUnderTest, - alpha, - successor, - *, - authority_lost_monotonic: float, - ) -> None: - """The whole safety claim, measured rather than assumed. - - Three things have to hold together: the old coordinator exits, it - exits within one adoption silence of losing authority, and the - successor's own eligibility edge — its real guard acquisition plus - the silence — is strictly later than that exit. - """ - self.assertTrue(heartbeat.exited, "authority was lost without an exit") - # The recorded instant is when the monitor *decided*; the process is - # gone at most one exit margin later, and that is what has to beat - # the adoption edge. - self.assertLess( - heartbeat.exit_monotonic - + POLICY.exit_margin_seconds - - authority_lost_monotonic, - POLICY.adoption_silence_seconds, - ) - if alpha.guard is not None and alpha.guard.held: - # The process is gone; that is what finally drops the session - # advisory lock and lets any successor start its silence. - alpha.guard.close() - eligible_at = self.step_until_guard_held(harness, successor) - self.assertLess( - heartbeat.exit_monotonic + POLICY.exit_margin_seconds, - eligible_at, - ) - - def test_wedged_proof_and_maximally_late_monitor_still_exit_in_time( - self, - ) -> None: - """The combined worst case the exit envelope is sized for. - - Authority is gone (the guard session answers nothing), the thread - that would notice by raising is descheduled entirely, and the - monitor — the only thread left that can act — wakes as late as the - policy's scheduler slack permits on every poll. This is the - schedule that made the first cut of the envelope unsafe: reserving - only the poll intervals, a maximally late monitor observed the - stale edge after the successor's adoption edge. - """ - with LeaseHarness(lease_ttl_seconds=LEASE_TTL_SECONDS) as harness: - alpha = harness.coordinator("alpha") - alpha.start() - harness.run_until(alpha, "done:startup") - heartbeat = HeartbeatUnderTest(harness, alpha) - # Beats run their statements at the guard's server-side ceiling. - # That maximises the gap between the newest completed round trip - # and the client-side marks the next beat leaves before it - # wedges, which is exactly when the envelope cap — not the - # liveness budget — is the bound that has to carry the - # guarantee. - harness.server.statement_latency_seconds = guard_statement_latency( - GUARD_STATEMENT_TIMEOUT_SECONDS - ) - actors = [heartbeat.heartbeat, heartbeat.monitor] - heartbeat.run(actors, seconds=2.0) - self.assertFalse(heartbeat.exited) - - # The last instant this coordinator's authority was provable. - # The guarded session cannot have died before it, so a - # successor's guard acquisition — and therefore its silence - # window — cannot start earlier either. Measuring from here is - # conservative in the direction that matters. - proven_at = float(heartbeat.service.last_server_proven_monotonic) - - def wedge(statement) -> float: - if statement.kind.value in {"prove", "verify"}: - harness.scheduler.park_forever("alpha.guard.wedged") - return 0.0 - - harness.server.statement_latency_seconds = wedge - # Let one beat reach the wedge — leaving its attempt-start and - # slot-acquisition marks behind, which is what makes the - # client-side liveness clock look healthier than the session - # is — then abandon the heartbeat thread: from here only the - # monitor can save the deployment. - heartbeat.advance_until_beat([heartbeat.monitor]) - while heartbeat.heartbeat.runnable(): - harness.step(heartbeat.heartbeat) - - beta = harness.coordinator("beta") - self.assert_guard_withheld(harness, beta) - # Worst phase: one poll lands a hair before the cap elapses, so - # the poll that can actually notice is a full late interval - # after it. - heartbeat.run_monitor_maximally_late( - rounds=200, - first_poll_monotonic=( - proven_at + POLICY.server_proven_cap_seconds - 0.001 - ), - ) - - # A precondition, not a bound: the point is that the monitor - # really was at least as late as the policy budgets for, so a - # lower bound is what this checks. - self.assertGreaterEqual( - heartbeat.service.monitor_wake_delay_seconds, - POLICY.scheduler_slack_seconds - 1e-6, - "the monitor was not actually driven at maximum lateness", - ) - self.assertTrue(heartbeat.exited, "a wedged proof did not fence") - # The whole safety claim, at the worst schedule the policy - # tolerates: gone before the silence window — which starts no - # earlier than the last provable instant — can elapse. Asserted - # before anything about wording, so a regression in the envelope - # reports the breach rather than a changed message. - self.assertLess( - heartbeat.exit_monotonic - + POLICY.exit_margin_seconds - - proven_at, - POLICY.adoption_silence_seconds, - "the old coordinator was still exiting after a successor " - "became eligible to adopt", - ) - # The envelope cap is the bound that fired, not the liveness - # budget the fresh client marks were still inside. - self.assertIn("Tripped: server-proven", heartbeat.exit_reason) - self.assert_exit_precedes_eligibility( - harness, - heartbeat, - alpha, - beta, - authority_lost_monotonic=proven_at, - ) - - def test_lost_guard_with_stalled_heartbeat_and_late_monitor_exits_in_time( - self, - ) -> None: - """The same worst case, entered through guard-session loss. - - The advisory lock is genuinely gone, so this coordinator has no - authority at all — and its heartbeat thread, which would normally - raise on the next beat, never gets scheduled again. The monitor - has to carry the exit alone, maximally late, and still finish - before the successor that can now take the guard becomes eligible. - """ - with LeaseHarness(lease_ttl_seconds=LEASE_TTL_SECONDS) as harness: - alpha = harness.coordinator("alpha") - alpha.start() - harness.run_until(alpha, "done:startup") - heartbeat = HeartbeatUnderTest(harness, alpha) - actors = [heartbeat.heartbeat, heartbeat.monitor] - heartbeat.run(actors, seconds=1.0) - self.assertFalse(heartbeat.exited) - - assert alpha.guard is not None - alpha.guard.close() - authority_lost = harness.clock.monotonic() - - # The successor may take the guard immediately — the silence - # window it must then wait out is the only thing standing - # between it and the lease row. - beta = harness.coordinator("beta") - eligible_at = self.takeover_deadline(harness, beta) - - # From here the heartbeat thread never runs again. - heartbeat.run_monitor_maximally_late(rounds=200) - - self.assertTrue(heartbeat.exited, "a lost guard did not fence") - self.assertGreaterEqual( - heartbeat.service.monitor_wake_delay_seconds, - POLICY.scheduler_slack_seconds - 1e-6, - ) - self.assertLess( - heartbeat.exit_monotonic - + POLICY.exit_margin_seconds - - authority_lost, - POLICY.adoption_silence_seconds, - ) - self.assertLess( - heartbeat.exit_monotonic + POLICY.exit_margin_seconds, - eligible_at, - ) - - def test_unsafe_policy_refuses_to_arm_the_heartbeat(self) -> None: - """Startup rejection, on the real service. - - A policy whose failure budget cannot fit inside the adoption - silence alongside the exit envelope is a double-writer hazard. The - service must refuse to start rather than run with a broken - exit-before-adoption argument. - """ - with LeaseHarness(lease_ttl_seconds=LEASE_TTL_SECONDS) as harness: - alpha = harness.coordinator("alpha") - alpha.start() - harness.run_until(alpha, "done:startup") - unsafe = WriterLeaseHeartbeatPolicy( - adoption_silence_seconds=POLICY.adoption_silence_seconds, - heartbeat_interval_seconds=0.25, - failure_budget_seconds=POLICY.adoption_silence_seconds, - monitor_interval_seconds=0.05, - exit_margin_seconds=0.1, - ) - heartbeat = HeartbeatUnderTest(harness, alpha, policy=unsafe) - starter = harness.scheduler.actor("serve") - started = starter.submit(heartbeat.service.start, label="start") - harness.run_until(starter, "done:start") - - self.assertIsNone(started.value()) - self.assertTrue(heartbeat.exited) - self.assertIn( - "refusing to start the ledger lease heartbeat", - heartbeat.exit_reason, - ) - self.assertIn("adoption silence", heartbeat.exit_reason) - # Nothing was armed: no guarded statement ran. - self.assertNotIn("prove", statement_counts(harness)) - - -if __name__ == "__main__": - unittest.main() diff --git a/tests/test_prism_lease_heartbeat_under_fenced_write.py b/tests/test_prism_lease_heartbeat_under_fenced_write.py deleted file mode 100644 index 5f94c0357..000000000 --- a/tests/test_prism_lease_heartbeat_under_fenced_write.py +++ /dev/null @@ -1,631 +0,0 @@ -#!/usr/bin/env python3 -"""The lease heartbeat self-fencing shape: a heartbeat queued behind our own write. - -Every fenced write this coordinator performs — a share append, -``persist_accepted_block`` — row-locks the singleton ``qbit_ledger_writer_lease`` -tuple for the whole of its transaction. The pre-fix heartbeat renewed the lease -with a plain ``UPDATE``, which queues for that tuple. On the guard connection, -which carries a server-side ``statement_timeout``, queueing is fatal: the -heartbeat's own statement is cancelled, the coordinator reads that as loss of -its guarded session, and a perfectly healthy writer hard-exits because it was -briefly busy writing. The coordinator fenced itself out with its own success. - -The fix made the heartbeat's renewal non-blocking — -``FOR NO KEY UPDATE SKIP LOCKED`` — and proved liveness by reads instead: the -session answers, the guard advisory lock is still held by this backend, and -the last committed lease row still names this exact session. A skipped renewal -is harmless while that committed row is unexpired, because the fenced write -that skipped it refreshes the TTL when it commits. - -Nothing could interleave a real heartbeat with a real in-flight fenced write -before the deterministic harness existed: the fenced write has to be parked -*inside* its statement, after PostgreSQL has taken the tuple lock and before -the client's COMMIT, and no amount of running the coordinator reliably lands -there. This module parks it there on purpose and drives the shipped -``verify_writer_lease_guard_session`` across the gap. - -Four things are proved, and one of them is the reason the others mean anything: - -``HeartbeatUnderFencedWriteTests`` - The heartbeat survives an in-flight fenced write, skipping renewal - without ever queueing, and renewal resumes the moment that write commits. - ``test_the_queueing_renewal_spelling_dies_where_the_heartbeat_survives`` - is the control: it runs the *pre-fix* spelling into the same lock and - watches it die on the guard's statement timeout. Without that control, - ``renewed_count == 0`` would be equally consistent with a heartbeat that - never tried. - -``OwnWriteRenewalDeferralTests`` - The one lock-blocked-and-expired case that must not fail closed: this - process's own fenced write outlasting the TTL. The locker is attributable - through ``pg_stat_activity`` to this process's pool ``application_name``, - so the verification defers rather than self-fencing — and the same - expired row locked by anyone else still fails closed. - -``GuardQuerySlotTests`` - The guard connection is one session, so its callers serialize on one - query slot rather than pipelining onto the same backend. -""" - -from __future__ import annotations - -import threading -import unittest - -from lab.prism import share_ledger as share_ledger_module -from tests.prism_concurrency_harness import ( - LeaseHarness, - LockTimeout, - assert_deterministic, -) - -# Lease TTL for every scenario here. The own-write authority margin defaults -# to half the TTL (``_resolve_lease_authority_margin_seconds``), so a freshly -# acquired row at 60s is 30s clear of the margin: a renewal skipped over it -# defers nothing, which is what makes scenario 1's `renewal_deferred_to_own_write` -# assertion distinguishable from scenario 4's. -LEASE_TTL_SECONDS = 60.0 - -# The caller deadline the fenced write runs under. A slow -# ``persist_accepted_block`` is the real shape; the value only has to be a -# deadline, because arming one is what makes _NativePostgresClient wrap the -# statement in an explicit transaction whose COMMIT is a separate client -# message — the gap this module parks in. Chosen well beyond every clock -# advance below so the write's own budget never explains what we observe. -FENCED_WRITE_BUDGET_SECONDS = 600.0 - -# Past LEASE_TTL_SECONDS, so the committed row the heartbeat reads has lapsed -# while the fenced write that would have refreshed it is still in flight. -TTL_LAPSE_SECONDS = 70.0 - -# Three passes, because one is not a heartbeat. The point is that the guarded -# session keeps answering for the whole span of the fenced write, not that it -# answered once. -HEARTBEAT_PASSES = 3 - - -def drive_heartbeat_under_fenced_write(harness: LeaseHarness) -> dict[str, object]: - """Park a fenced write mid-statement and heartbeat across it. - - 1. ``alpha`` starts and takes the writer lease. - 2. ``alpha`` begins a deadline-scoped fenced statement. Stopping at - ``alpha.done:renew`` leaves PostgreSQL exactly where a long fenced - write leaves it: the statement has executed and holds the lease tuple - lock, and the COMMIT has not been sent. - 3. A second actor runs the shipped heartbeat verification on ``alpha``'s - guard connection, repeatedly. It has to be a second actor: ``alpha`` - is parked inside its own statement and also holds the ledger's writer - lock, so a heartbeat submitted to ``alpha`` could never run — which is - precisely the production shape, where the heartbeat is a separate - thread from whatever is writing. - """ - alpha = harness.coordinator("alpha") - alpha.start() - harness.run_until(alpha, "done:startup") - - def fenced_write() -> object: - # Renewal is the smallest fenced write that touches the lease tuple, - # and it is shipped code rather than a stand-in. Any fenced write has - # this shape: deadline armed, tuple locked, COMMIT still to come. - with alpha.ledger.operation_timeout(FENCED_WRITE_BUDGET_SECONDS): - return alpha.ledger.renew_writer_lease() - - alpha.submit(fenced_write, label="fenced-write") - harness.run_until(alpha, "alpha.done:renew") - - heartbeat = harness.scheduler.actor("heartbeat") - verifications = [] - for pass_number in range(1, HEARTBEAT_PASSES + 1): - label = f"heartbeat-{pass_number}" - call = heartbeat.submit( - alpha.ledger.verify_writer_lease_guard_session, - label=label, - ) - harness.run_until(heartbeat, f"done:{label}") - verifications.append(call.value()) - - return { - "verifications": verifications, - "fenced_write_still_holds_lease_lock": ( - harness.server.lease_lock_holder is not None - ), - "fenced_write_finished": alpha.actor.calls[-1].done, - "lockwait_stops": [stop for stop in harness.trace if "lockwait" in stop], - "statements": harness.statement_kinds(), - "lease_holder_session": harness.lease_holder_session(), - } - - -def drive_own_write_outlasting_the_ttl(harness: LeaseHarness) -> dict[str, object]: - """Let the TTL lapse under this coordinator's own in-flight fenced write. - - Same park as above, then the virtual clock runs past the committed row's - ``lease_expires_at``. The renewal that would have refreshed it is the very - write holding the lock, so the heartbeat now reads a row that is expired, - locked, and unrenewable — the shape that must fail closed when the locker - is a competing claimant, and must not when it is us. - """ - alpha = harness.coordinator("alpha") - alpha.start() - harness.run_until(alpha, "done:startup") - - def fenced_write() -> object: - with alpha.ledger.operation_timeout(FENCED_WRITE_BUDGET_SECONDS): - return alpha.ledger.renew_writer_lease() - - alpha.submit(fenced_write, label="fenced-write") - harness.run_until(alpha, "alpha.done:renew") - - harness.advance(TTL_LAPSE_SECONDS) - - heartbeat = harness.scheduler.actor("heartbeat") - call = heartbeat.submit( - alpha.ledger.verify_writer_lease_guard_session, - label="heartbeat", - ) - harness.run_until(heartbeat, "done:heartbeat") - - row = harness.lease_row() - return { - "verification": call.value(), - "committed_row_expired": ( - row is not None and row.lease_expires_at <= harness.clock.now() - ), - "committed_row_locked": row is not None and row.xmax is not None, - "lockwait_stops": [stop for stop in harness.trace if "lockwait" in stop], - "statements": harness.statement_kinds(), - } - - -def drive_foreign_locker_over_the_expired_row( - harness: LeaseHarness, -) -> dict[str, object]: - """The same expired-and-lock-blocked read, with the lock held by someone else. - - ``beta`` is a different-epoch claimant, so it has a different advisory - guard key and is not held off by ``alpha``'s guard. Parking it at - ``beta.done:acquire`` stops it in the gap its expiry CAS opens: it holds - the tuple lock and has staged a takeover that has not committed, so the - committed row ``alpha``'s heartbeat reads still names ``alpha`` and is - still expired. That stale read is exactly what the queueing renewal used - to catch by waiting, and what the non-blocking spelling has to catch by - failing closed. - """ - alpha = harness.coordinator("alpha", writer_epoch=1) - alpha.start() - harness.run_until(alpha, "done:startup") - - harness.advance(TTL_LAPSE_SECONDS) - - beta = harness.coordinator("beta", writer_epoch=2) - beta.start() - harness.run_until(beta, "beta.done:acquire") - - heartbeat = harness.scheduler.actor("heartbeat") - call = heartbeat.submit( - alpha.ledger.verify_writer_lease_guard_session, - label="heartbeat", - ) - harness.run_until(heartbeat, "done:heartbeat") - - row = harness.lease_row() - return { - "error": None if call.error is None else str(call.error), - "error_type": None if call.error is None else type(call.error).__name__, - "committed_row_session": None if row is None else row.writer_session_token, - "committed_row_expired": ( - row is not None and row.lease_expires_at <= harness.clock.now() - ), - "lockwait_stops": [stop for stop in harness.trace if "lockwait" in stop], - "statements": harness.statement_kinds(), - } - - -class HeartbeatUnderFencedWriteTests(unittest.TestCase): - """A heartbeat crossing an in-flight fenced write, and the control that dates it.""" - - def test_heartbeat_survives_the_whole_fenced_write_without_queueing(self) -> None: - """Skipped, never queued, never raised — for every pass of the write. - - ``renewed_count == 0`` on its own is only half the claim. The other - half is ``lockwait_stops``: the harness gives a statement that queues - for the lease tuple a named checkpoint, so an empty list is positive - evidence that the verification declined the lock rather than waiting - on it and getting lucky. ``verified_count == 1`` alongside it says the - session was proven live by reads while the renewal was skipped, which - is the whole substance of the fix. - """ - with LeaseHarness(lease_ttl_seconds=LEASE_TTL_SECONDS) as harness: - observed = drive_heartbeat_under_fenced_write(harness) - - self.assertTrue( - observed["fenced_write_still_holds_lease_lock"], - "the fenced write must still be holding the lease tuple lock, " - "or the heartbeat had nothing to survive", - ) - self.assertFalse(observed["fenced_write_finished"]) - self.assertEqual(len(observed["verifications"]), HEARTBEAT_PASSES) - for pass_number, result in enumerate(observed["verifications"], start=1): - with self.subTest(heartbeat_pass=pass_number): - self.assertEqual(result["verified_count"], 1) - self.assertEqual(result["renewed_count"], 0) - # The committed row is a full TTL fresh and the margin is - # half a TTL, so this skip is over a row with plenty of - # runway: liveness *and* authority, no deferral. - self.assertFalse(result["renewal_deferred_to_own_write"]) - self.assertEqual( - observed["lockwait_stops"], - [], - "a verification that stopped at a lockwait checkpoint queued " - "for the lease tuple, which is the self-fencing bug itself", - ) - self.assertEqual( - observed["statements"], - ["acquire", "renew", "verify", "verify", "verify"], - "each heartbeat must be one non-blocking statement; a second " - "statement would mean the attribution recheck fired, and this " - "row is not ambiguous", - ) - - def test_renewal_resumes_once_the_fenced_write_commits(self) -> None: - """The skip is deferral to the write, not a heartbeat that stopped renewing. - - The fenced write refreshes the TTL itself when it commits, so the - renewal the heartbeat skipped was never lost. Proving that the very - next verification renews is what separates "declined the lock" from - "silently gave up on renewing". - """ - with LeaseHarness(lease_ttl_seconds=LEASE_TTL_SECONDS) as harness: - drive_heartbeat_under_fenced_write(harness) - alpha = harness.coordinators["alpha"] - heartbeat = harness.scheduler.actors["heartbeat"] - - harness.run_until(alpha, "alpha.precommit") - harness.run_until(alpha, "done:fenced-write") - self.assertIsNone( - harness.server.lease_lock_holder, - "the committed fenced write must have released the tuple lock", - ) - - call = heartbeat.submit( - alpha.ledger.verify_writer_lease_guard_session, - label="heartbeat-after-commit", - ) - harness.run_until(heartbeat, "done:heartbeat-after-commit") - - self.assertEqual( - call.value(), - { - "backend": "postgres-psql", - "verified_count": 1, - "renewed_count": 1, - "renewal_deferred_to_own_write": False, - }, - ) - - def test_the_queueing_renewal_spelling_dies_where_the_heartbeat_survives( - self, - ) -> None: - """Control: the pre-fix renewal, run into the same lock, is cancelled. - - ``_renew_writer_lease_with`` is the plain locking ``UPDATE`` the - heartbeat used to run. Handed the guard's ``run_json`` it becomes the - pre-fix heartbeat exactly, and the guard session's server-side - ``statement_timeout`` is what turns its queue into a hard failure — - which the coordinator reads as loss of its guarded session. - - This test is what makes ``renewed_count == 0`` above mean "declined a - lock that was really there" rather than "nothing was attempted". - """ - with LeaseHarness(lease_ttl_seconds=LEASE_TTL_SECONDS) as harness: - drive_heartbeat_under_fenced_write(harness) - alpha = harness.coordinators["alpha"] - heartbeat = harness.scheduler.actors["heartbeat"] - guard = alpha.guard - assert guard is not None - - call = heartbeat.submit( - lambda: alpha.ledger._renew_writer_lease_with(guard.run_json), - label="queueing-renewal", - ) - # The pre-fix spelling does what the fixed one refuses to: it - # queues on the tuple the fenced write holds. - harness.run_until_blocked(heartbeat, "alpha.guard.lockwait:renew") - - self.assertIsNotNone( - heartbeat.wake_at, - "the guard session's statement timeout must bound this wait; " - "an unbounded one would be a different defect", - ) - self.assertIsNotNone( - harness.advance_to_next_deadline(), - "the guard statement timeout is the only pending deadline", - ) - harness.run_until(heartbeat, "done:queueing-renewal") - - # Specifically the guard connection's server-side - # statement_timeout, not a caller-armed lock_timeout: the - # heartbeat runs on a session whose bound it cannot widen, which - # is why queueing there is fatal rather than merely slow. - with self.assertRaisesRegex( - LockTimeout, - "canceling statement due to statement timeout", - ): - call.value() - self.assertTrue( - harness.server.lease_lock_holder is not None, - "the fenced write still holds the lock it queued for; the " - "renewal died, the write did not", - ) - - def test_interleaving_is_order_stable_across_repeated_runs(self) -> None: - """A flaky concurrency test is worse than none; prove this one is not.""" - run = assert_deterministic( - self, - drive_heartbeat_under_fenced_write, - repeats=25, - lease_ttl_seconds=LEASE_TTL_SECONDS, - ) - - self.assertEqual( - list(run.trace), - [ - "alpha@alpha.begin:acquire", - "alpha@alpha.done:acquire", - # Since #123 the acquire/adopt statements carry an acquisition - # deadline, so each commits explicitly and offers a precommit stop. - "alpha@alpha.precommit", - "alpha@done:startup", - "alpha@alpha.begin:renew", - "alpha@alpha.done:renew", - "heartbeat@alpha.guard.begin:verify", - "heartbeat@alpha.guard.snapshot:verify", - "heartbeat@alpha.guard.done:verify", - "heartbeat@done:heartbeat-1", - "heartbeat@alpha.guard.begin:verify", - "heartbeat@alpha.guard.snapshot:verify", - "heartbeat@alpha.guard.done:verify", - "heartbeat@done:heartbeat-2", - "heartbeat@alpha.guard.begin:verify", - "heartbeat@alpha.guard.snapshot:verify", - "heartbeat@alpha.guard.done:verify", - "heartbeat@done:heartbeat-3", - ], - ) - - -class OwnWriteRenewalDeferralTests(unittest.TestCase): - """Expired, lock-blocked, and ours: defer. Expired and someone else's: fail closed.""" - - def test_own_write_outlasting_the_ttl_defers_instead_of_self_fencing(self) -> None: - """Raising here would roll back a valid write and restart-loop on the next one. - - The locker is this process's own fenced write, so no competing expiry - claim can be in flight behind it — the tuple's exclusive lock is the - proof — and its commit refreshes ``lease_expires_at`` before any - queued claimant re-evaluates its CAS. The session provably survives, - so the verification must not raise. It survives as a *process* only: - the argument assumes the write commits, and a rollback would hand the - expired row straight to a queued claimant, so the same result carries - ``renewal_deferred_to_own_write`` to tell external-side-effect fences - to withhold their RPC. - """ - with LeaseHarness(lease_ttl_seconds=LEASE_TTL_SECONDS) as harness: - observed = drive_own_write_outlasting_the_ttl(harness) - - self.assertTrue( - observed["committed_row_expired"], - "the committed row must have lapsed for this to be the " - "fail-closed shape at all", - ) - self.assertTrue(observed["committed_row_locked"]) - self.assertEqual( - observed["verification"], - { - "backend": "postgres-psql", - "verified_count": 1, - "renewed_count": 0, - "renewal_deferred_to_own_write": True, - }, - ) - self.assertEqual(observed["lockwait_stops"], []) - self.assertEqual( - observed["statements"], - ["acquire", "renew", "verify"], - "the locker was attributable on the first statement, so no " - "attribution recheck was warranted", - ) - - def test_an_expired_row_locked_by_a_foreign_application_fails_closed(self) -> None: - """The contrast that gives the deferral its meaning. - - Identical read — expired committed row, renewal skipped over a held - lock — and the only difference is who holds the lock. A locker that - cannot be attributed to this process's pool ``application_name`` may - be a different-identity expiry claim whose commit lands right after - this snapshot, and the stale token read would name us until it does. - A skipped renewal is not evidence of liveness there, so it raises. - """ - with LeaseHarness(lease_ttl_seconds=LEASE_TTL_SECONDS) as harness: - observed = drive_foreign_locker_over_the_expired_row(harness) - - self.assertEqual( - observed["committed_row_session"], - "heartbeat-v1:alpha-1", - "the claimant's takeover must still be uncommitted, or the " - "verification would fail on identity instead", - ) - self.assertTrue(observed["committed_row_expired"]) - self.assertEqual(observed["error_type"], "RuntimeError") - self.assertIsNotNone(observed["error"]) - self.assertIn( - "expired and its renewal was lock-blocked", - str(observed["error"]), - ) - self.assertEqual( - observed["lockwait_stops"], - [], - "failing closed must still be non-blocking; the point of the " - "fix is that the guard never queues, not that it never raises", - ) - self.assertEqual( - observed["statements"], - ["acquire", "acquire", "verify", "verify"], - "an unattributable locker over an expired row is the one " - "ambiguous shape, so it earns exactly one recheck before the " - "fail-closed raise", - ) - - def test_own_write_deferral_is_order_stable_across_repeated_runs(self) -> None: - """The TTL-lapse interleaving pins a schedule too, not just an outcome.""" - run = assert_deterministic( - self, - drive_own_write_outlasting_the_ttl, - repeats=25, - lease_ttl_seconds=LEASE_TTL_SECONDS, - ) - - self.assertEqual( - list(run.trace), - [ - "alpha@alpha.begin:acquire", - "alpha@alpha.done:acquire", - "alpha@alpha.precommit", - "alpha@done:startup", - "alpha@alpha.begin:renew", - "alpha@alpha.done:renew", - "heartbeat@alpha.guard.begin:verify", - "heartbeat@alpha.guard.snapshot:verify", - "heartbeat@alpha.guard.done:verify", - "heartbeat@done:heartbeat", - ], - ) - - -class GuardQuerySlotTests(unittest.TestCase): - """One guard connection, one statement at a time.""" - - def test_production_guard_serializes_callers_on_one_query_slot(self) -> None: - """The shipped guard, not the harness's model of it. - - The scheduler test below drives ``FakeLeaseGuard``, whose slot is a - reimplementation: deleting ``with self._query_lock:`` from - ``_NativePostgresLeaseGuard.run_json`` would leave every assertion - there green. This one holds the production lock and shows the - production method waiting for it, which is the claim the module - actually wants to make. It needs no server — the serialization is a - plain lock around the statement, above the connection. - """ - guard = share_ledger_module._NativePostgresLeaseGuard.__new__( - share_ledger_module._NativePostgresLeaseGuard - ) - guard._advisory_lock_key = 0 - guard._query_lock = threading.Lock() - guard._closed = False - guard._held = True - guard._connection = None - - entered = threading.Event() - arrived = threading.Event() - released = threading.Event() - - def second_caller() -> None: - entered.set() - # Reaching the statement requires the slot; `held` is True and - # `_connection` is None, so arrival past the slot is observable as - # the AttributeError the statement itself then raises. Catching - # that exact type is what makes this evidence: any other outcome - # means run_json did not get where this test claims it got. - try: - guard.run_json("SELECT 1") - except AttributeError: - arrived.set() - released.set() - - guard._query_lock.acquire() - caller = threading.Thread(target=second_caller, daemon=True) - caller.start() - self.assertTrue(entered.wait(2.0)) - self.assertFalse( - released.wait(0.2), - "run_json entered the statement while another caller held the " - "query slot; the guard session is not serialized", - ) - guard._query_lock.release() - self.assertTrue( - released.wait(2.0), - "releasing the slot must let the queued caller proceed", - ) - self.assertTrue( - arrived.is_set(), - "the queued caller must have reached the statement itself, not " - "merely returned", - ) - caller.join(timeout=2.0) - - def test_a_second_caller_parks_on_the_guard_query_slot(self) -> None: - """The harness models that serialization as an interleaving point. - - Both callers run the non-blocking verification, so neither can be - queued on the lease tuple; the only thing that can hold one up is the - other's occupancy of the single guard connection. This pins the - harness's rendering of it — that the second caller stops at a named - checkpoint and is released by the holder rather than by a timeout — - so scenarios can schedule around it. The production serialization - itself is pinned by the test above. - """ - with LeaseHarness(lease_ttl_seconds=LEASE_TTL_SECONDS) as harness: - alpha = harness.coordinator("alpha") - alpha.start() - harness.run_until(alpha, "done:startup") - - first = harness.scheduler.actor("hb-first") - second = harness.scheduler.actor("hb-second") - first_call = first.submit( - alpha.ledger.verify_writer_lease_guard_session, - label="first", - ) - second_call = second.submit( - alpha.ledger.verify_writer_lease_guard_session, - label="second", - ) - - harness.run_until(first, "alpha.guard.begin:verify") - harness.run_until_blocked(second, "alpha.guard.slot:wait") - self.assertIsNone( - second.wake_at, - "the slot wait is released by the holder, not by a timeout", - ) - - harness.run_until(first, "done:first") - self.assertTrue( - second.runnable(), - "releasing the slot must be what makes the second caller " - "runnable again", - ) - harness.run_until(second, "done:second") - - self.assertEqual(first_call.value()["verified_count"], 1) - self.assertEqual(second_call.value()["verified_count"], 1) - self.assertEqual( - harness.trace, - [ - "alpha@alpha.begin:acquire", - "alpha@alpha.done:acquire", - "alpha@alpha.precommit", - "alpha@done:startup", - "hb-first@alpha.guard.begin:verify", - "hb-second@alpha.guard.slot:wait", - "hb-first@alpha.guard.snapshot:verify", - "hb-first@alpha.guard.done:verify", - "hb-first@done:first", - "hb-second@alpha.guard.begin:verify", - "hb-second@alpha.guard.snapshot:verify", - "hb-second@alpha.guard.done:verify", - "hb-second@done:second", - ], - ) - - -if __name__ == "__main__": - unittest.main() diff --git a/tests/test_prism_lease_monitor_stalls.py b/tests/test_prism_lease_monitor_stalls.py deleted file mode 100644 index 10015180b..000000000 --- a/tests/test_prism_lease_monitor_stalls.py +++ /dev/null @@ -1,1019 +0,0 @@ -#!/usr/bin/env python3 -"""In-process lease-monitor stalls: attribution and response (issue #227). - -What production showed ----------------------- -Three days after PR #214 shipped the writer-lease timing policy, -``union-mainnet`` recorded a heartbeat-monitor wake delay of 0.648s while -the host was 93% idle. The stall was in-process — a 5.1s payout-window -rescan holding the interpreter — and two things about the telemetry made -it undiagnosable: - -* ``guard_sql`` was measured client-side, so the worst attempt showed - 0.714s of "database time" against a server-enforced 0.50s statement - timeout while ``scheduler_delay`` never exceeded 22 microseconds in 645k - attempts. The attribution literally could not see the stall. -* The monitor's lateness was exported only as a lifetime high-water mark, - so every smaller stall was invisible and the alert could fire once per - process lifetime. - -What is proved here -------------------- -``InRoundTripStallAttributionTests`` - The required scenario: on the #128 deterministic harness (virtual - clock, baton scheduler, nothing sleeps) a 0.65s scheduler stall is - injected *inside* a guard round trip — PostgreSQL has answered, the - heartbeat thread does not get the interpreter back — and the new - attribution reports it as scheduler time, not ``guard_sql``. Plus the - accumulator-level arithmetic of the split. - -``MonitorWakeTelemetryTests`` - The point of #227: the rolling-window maximum falls back down once a - stall ages out and the record's age advances, unlike the lifetime - gauge. - -``GcPauseTelemetryTests`` - Pause durations land in the right generation's series. - -``StallProbeTests`` - Repeated over-half-slack wakes produce at most the configured number - of stack samples per window — a count, never a duration. - -``StallProbeFrameOwnershipTests`` - Completed samples, sampling caps, and formatting errors release sampled - workers' locals without waiting for cyclic GC (issue #247). - -``LatenessBeyondSlackResponseTests`` - The decided response (option (b) in the ``writer_lease_timing`` - docstring): the monitor does not hard-exit a healthy coordinator, the - breach is counted with its overrun past the exit-guarantee bound, and - one structured warning names it. -""" - -from __future__ import annotations - -from dataclasses import replace -import gc -import re -import threading -import unittest -from unittest import mock -import weakref - -from lab.prism.background_services import ( - GcPauseTelemetry, - LedgerLeaseHeartbeatPorts, - LedgerLeaseHeartbeatService, - MonitorWakeTelemetry, - StallProbe, - verifier_callbacks, -) -from lab.prism.process_telemetry import PRISM_GC_GENERATIONS -from lab.prism.writer_lease_timing import ( - DEFAULT_WRITER_LEASE_HEARTBEAT_POLICY, - LEASE_HEARTBEAT_MODE_PROOF, - LEASE_HEARTBEAT_OUTCOME_PROVEN, - LEASE_MONITOR_LATE_WAKE_SLACK_FRACTIONS, - LEASE_MONITOR_STALL_PROBE_MAX_SAMPLES_PER_WINDOW, - LEASE_MONITOR_STALL_PROBE_TRIGGER_SLACK_FRACTION, - LEASE_MONITOR_STALL_PROBE_WINDOW_SECONDS, - LEASE_MONITOR_WAKE_DELAY_BUCKETS, - LEASE_MONITOR_WAKE_DELAY_WINDOW_SECONDS, - OBSERVED_MONITOR_WAKE_DELAY_SECONDS_227, - WriterLeaseHeartbeatPolicy, - WriterLeaseVerificationAttempt, -) -from tests.prism_concurrency_harness import LeaseHarness -from tests.test_prism_lease_heartbeat_tail_latency import ( - HeartbeatUnderTest, - guard_statement_latency, -) - -POLICY = DEFAULT_WRITER_LEASE_HEARTBEAT_POLICY - -# The stall issue #227's acceptance criterion names: a scheduler stall of -# this length inside one guard round trip. Above the 0.50s slack and above -# the 0.55s bound at which exit-before-adoption stops being guaranteed. -IN_ROUND_TRIP_STALL_SECONDS = 0.65 - -# Modelled server time per guard statement in the harness scenario: a -# perfectly ordinary proof, so the whole stall is unambiguously not SQL. -SERVER_EXECUTION_SECONDS = 0.05 - -LEASE_TTL_SECONDS = 60.0 - - -class Clock: - """A settable virtual clock for the accumulator- and telemetry-level tests.""" - - def __init__(self, now: float = 1_000.0) -> None: - self.now = now - - def __call__(self) -> float: - return self.now - - def advance(self, seconds: float) -> float: - self.now += seconds - return self.now - - -class InRoundTripStallAttributionTests(unittest.TestCase): - def test_scheduler_stall_inside_a_guard_round_trip_is_not_charged_to_guard_sql( - self, - ) -> None: - """The required test (issue #227, acceptance criterion 1). - - The heartbeat is driven to the checkpoint at which the fake - PostgreSQL has finished executing the ownership proof — the - answer has left the server — and the virtual clock then advances - 0.65s before the heartbeat actor is stepped again, which is what a - GIL stall between the answer arriving and the heartbeat thread - resuming looks like. The statement reported 0.05s of server - execution, so the attribution must put the 0.65s in scheduler time - (and in its in-round-trip sub-phase, client resume), not in - ``guard_sql``. On the merge base this scenario reports - ``guard_sql`` of 0.70s and a scheduler delay of zero. - """ - with LeaseHarness(lease_ttl_seconds=LEASE_TTL_SECONDS) as harness: - alpha = harness.coordinator("alpha") - alpha.start() - harness.run_until(alpha, "done:startup") - heartbeat = HeartbeatUnderTest(harness, alpha) - warnings: list[str] = [] - heartbeat.service._ports = replace( - heartbeat.service._ports, log=warnings.append - ) - harness.server.statement_latency_seconds = guard_statement_latency( - SERVER_EXECUTION_SECONDS - ) - # The shipped ledger accepts the server-timing callback, so - # the wiring under test is production's, not a stub's. - self.assertIn( - "on_statement_server_seconds", - verifier_callbacks(alpha.ledger.prove_writer_lease_guard_session), - ) - actors = [heartbeat.heartbeat, heartbeat.monitor] - # The first beat renews; settle into the proof cadence. - heartbeat.run(actors, seconds=1.0) - self.assertFalse(heartbeat.exited) - proofs_before = heartbeat.service.attempt_counts[LEASE_HEARTBEAT_MODE_PROOF] - - # Fast-forward to the next beat, then run the heartbeat actor - # exactly up to the point where the fake server has finished - # the proof statement (its modelled 0.05s already charged). - heartbeat.advance_until_beat([heartbeat.monitor]) - harness.run_until(heartbeat.heartbeat, "alpha.guard.done:prove") - answered_at = harness.clock.monotonic() - - # PostgreSQL has answered. This process does not get back to - # the heartbeat thread for 0.65s. - harness.advance(IN_ROUND_TRIP_STALL_SECONDS) - harness.drain(actors) - heartbeat.quiesce(actors) - - self.assertFalse( - heartbeat.exited, - f"healthy coordinator hard-exited: {heartbeat.exits}", - ) - self.assertEqual( - heartbeat.service.attempt_counts[LEASE_HEARTBEAT_MODE_PROOF], - proofs_before + 1, - ) - phases = heartbeat.service.last_phases - self.assertEqual(phases.mode, LEASE_HEARTBEAT_MODE_PROOF) - self.assertEqual(phases.outcome, LEASE_HEARTBEAT_OUTCOME_PROVEN) - self.assertTrue(phases.server_reported) - # The database is charged only what it reported. - self.assertAlmostEqual( - phases.guard_sql_seconds, SERVER_EXECUTION_SECONDS, places=6 - ) - self.assertLess(phases.guard_sql_seconds, IN_ROUND_TRIP_STALL_SECONDS) - # The stall is scheduler time, and specifically the part of it - # that fell inside the round trip. - self.assertGreaterEqual( - phases.scheduler_delay_seconds, IN_ROUND_TRIP_STALL_SECONDS - 1e-6 - ) - self.assertAlmostEqual( - phases.client_resume_seconds, IN_ROUND_TRIP_STALL_SECONDS, places=6 - ) - self.assertAlmostEqual( - phases.slot_wait_seconds - + phases.guard_sql_seconds - + phases.scheduler_delay_seconds, - phases.total_seconds, - places=9, - ) - # The metrics view carries the same split under fixed keys. - snapshot = heartbeat.service.snapshot() - last = snapshot["last_phase_seconds"] - assert isinstance(last, dict) - self.assertAlmostEqual(last["guard_sql"], SERVER_EXECUTION_SECONDS, places=6) - self.assertAlmostEqual( - last["guard_client_resume"], IN_ROUND_TRIP_STALL_SECONDS, places=6 - ) - # The server-proven edge is still the conservative send edge: - # the stall neither made the session look fresher nor older - # than the instant the statement left. - self.assertAlmostEqual( - heartbeat.service.last_server_proven_monotonic, - answered_at - SERVER_EXECUTION_SECONDS, - places=6, - ) - self.assertIn("client_resume=0.650s", phases.summary()) - - def test_guard_sql_splits_into_server_execution_and_client_resume(self) -> None: - """Known server time plus a known client gap land where they should. - - ``slot_wait + guard_sql + scheduler_delay == total`` still holds, - and ``client_resume`` is the in-round-trip share of the residual. - """ - clock = Clock() - attempt = WriterLeaseVerificationAttempt( - LEASE_HEARTBEAT_MODE_PROOF, monotonic=clock - ) - clock.advance(0.02) # queue wait for the guard's slot - attempt.slot_acquired() - clock.advance(0.10) # the server executes - clock.advance(0.65) # ... and this process does not resume - attempt.statement_completed() - attempt.statement_server_seconds(0.10) - clock.advance(0.03) # result handling after the round trip - phases = attempt.finish(LEASE_HEARTBEAT_OUTCOME_PROVEN) - - self.assertTrue(phases.server_reported) - self.assertAlmostEqual(phases.slot_wait_seconds, 0.02) - self.assertAlmostEqual(phases.guard_sql_seconds, 0.10) - self.assertAlmostEqual(phases.client_resume_seconds, 0.65) - self.assertAlmostEqual(phases.scheduler_delay_seconds, 0.68) - self.assertAlmostEqual(phases.total_seconds, 0.80) - self.assertAlmostEqual( - phases.slot_wait_seconds - + phases.guard_sql_seconds - + phases.scheduler_delay_seconds, - phases.total_seconds, - ) - self.assertLessEqual(phases.client_resume_seconds, phases.scheduler_delay_seconds) - self.assertIn("server-timed", phases.summary()) - - def test_server_time_accumulates_across_a_two_statement_verification(self) -> None: - """The recheck's server time adds to the first statement's.""" - clock = Clock() - attempt = WriterLeaseVerificationAttempt("renew", monotonic=clock) - attempt.slot_acquired() - clock.advance(0.20) - attempt.statement_completed() - attempt.statement_server_seconds(0.15) - clock.advance(0.30) - attempt.statement_completed() - attempt.statement_server_seconds(0.25) - phases = attempt.finish("renewed") - self.assertEqual(phases.statement_count, 2) - self.assertAlmostEqual(phases.guard_sql_seconds, 0.40) - self.assertAlmostEqual(phases.client_resume_seconds, 0.10) - self.assertAlmostEqual(phases.scheduler_delay_seconds, 0.10) - - def test_unreported_or_malformed_server_time_keeps_the_conservative_split( - self, - ) -> None: - for reported in (None, "not a number", float("nan"), -1.0): - with self.subTest(reported=reported): - clock = Clock() - attempt = WriterLeaseVerificationAttempt( - LEASE_HEARTBEAT_MODE_PROOF, monotonic=clock - ) - attempt.slot_acquired() - clock.advance(0.70) - attempt.statement_completed() - attempt.statement_server_seconds(reported) # type: ignore[arg-type] - phases = attempt.finish(LEASE_HEARTBEAT_OUTCOME_PROVEN) - self.assertFalse(phases.server_reported) - self.assertAlmostEqual(phases.guard_sql_seconds, 0.70) - self.assertAlmostEqual(phases.client_resume_seconds, 0.0) - self.assertAlmostEqual(phases.scheduler_delay_seconds, 0.0) - - def test_server_time_cannot_exceed_the_span_it_fits_inside(self) -> None: - """A server figure larger than the client span is clamped, not trusted.""" - clock = Clock() - attempt = WriterLeaseVerificationAttempt( - LEASE_HEARTBEAT_MODE_PROOF, monotonic=clock - ) - attempt.slot_acquired() - clock.advance(0.30) - attempt.statement_completed() - attempt.statement_server_seconds(0.45) - phases = attempt.finish(LEASE_HEARTBEAT_OUTCOME_PROVEN) - self.assertAlmostEqual(phases.guard_sql_seconds, 0.30) - self.assertAlmostEqual(phases.client_resume_seconds, 0.0) - - -class MonitorWakeTelemetryTests(unittest.TestCase): - def test_rolling_window_maximum_decays_and_record_age_advances(self) -> None: - """Unlike the lifetime gauge, the window maximum comes back down. - - The lifetime record stays at 0.648s forever (that is the dashboard - continuity the existing gauge keeps), while the rolling maximum - reports the stall only for one window and the record age says how - long ago the record was set. - """ - telemetry = MonitorWakeTelemetry() - slack = POLICY.scheduler_slack_seconds - now = 10_000.0 - telemetry.observe(0.001, slack_seconds=slack, now=now) - stall_at = now + 20.0 - telemetry.observe( - OBSERVED_MONITOR_WAKE_DELAY_SECONDS_227, - slack_seconds=slack, - now=stall_at, - ) - self.assertAlmostEqual( - telemetry.lifetime_max_seconds, OBSERVED_MONITOR_WAKE_DELAY_SECONDS_227 - ) - self.assertAlmostEqual( - telemetry.window_max_seconds(stall_at), - OBSERVED_MONITOR_WAKE_DELAY_SECONDS_227, - ) - self.assertAlmostEqual(telemetry.record_age_seconds(stall_at), 0.0) - - # Quiet wakes for the rest of the window: the stall still dominates. - later = stall_at + LEASE_MONITOR_WAKE_DELAY_WINDOW_SECONDS / 2.0 - telemetry.observe(0.002, slack_seconds=slack, now=later) - self.assertAlmostEqual( - telemetry.window_max_seconds(later), - OBSERVED_MONITOR_WAKE_DELAY_SECONDS_227, - ) - self.assertAlmostEqual( - telemetry.record_age_seconds(later), - LEASE_MONITOR_WAKE_DELAY_WINDOW_SECONDS / 2.0, - ) - - # Once the stall ages out, the window maximum is the recent quiet - # wake, the lifetime record is unchanged, and its age keeps growing. - aged_out = stall_at + LEASE_MONITOR_WAKE_DELAY_WINDOW_SECONDS + 10.0 - telemetry.observe(0.003, slack_seconds=slack, now=aged_out) - self.assertAlmostEqual(telemetry.window_max_seconds(aged_out), 0.003) - self.assertAlmostEqual( - telemetry.lifetime_max_seconds, OBSERVED_MONITOR_WAKE_DELAY_SECONDS_227 - ) - self.assertAlmostEqual( - telemetry.record_age_seconds(aged_out), - LEASE_MONITOR_WAKE_DELAY_WINDOW_SECONDS + 10.0, - ) - # And with no wakes at all inside the window, it is zero. - self.assertAlmostEqual( - telemetry.window_max_seconds( - aged_out + 2.0 * LEASE_MONITOR_WAKE_DELAY_WINDOW_SECONDS - ), - 0.0, - ) - - snapshot = telemetry.snapshot(aged_out) - self.assertEqual(snapshot["count"], 4) - self.assertEqual( - set(snapshot["late_wakes"]), set(LEASE_MONITOR_LATE_WAKE_SLACK_FRACTIONS) - ) - self.assertEqual(tuple(snapshot["buckets"]), LEASE_MONITOR_WAKE_DELAY_BUCKETS) - - def test_histogram_is_cumulative_and_threshold_counters_are_exact(self) -> None: - telemetry = MonitorWakeTelemetry() - slack = 0.5 - for delay in (0.004, 0.26, 0.41, 0.5, 0.648): - telemetry.observe(delay, slack_seconds=slack, now=1.0) - snapshot = telemetry.snapshot(1.0) - buckets = snapshot["buckets"] - assert isinstance(buckets, dict) - self.assertEqual(buckets[0.005], 1) - self.assertEqual(buckets[0.25], 1) - self.assertEqual(buckets[0.4], 2) - self.assertEqual(buckets[0.5], 4) - self.assertEqual(buckets[0.75], 5) - self.assertEqual(buckets[2.0], 5) - self.assertEqual(snapshot["count"], 5) - self.assertAlmostEqual(snapshot["sum"], 0.004 + 0.26 + 0.41 + 0.5 + 0.648) - late = snapshot["late_wakes"] - assert isinstance(late, dict) - # >= 0.25, >= 0.4, >= 0.5 respectively. - self.assertEqual(late, {"0.5": 4, "0.8": 3, "1.0": 2}) - - -class GcPauseTelemetryTests(unittest.TestCase): - def test_gc_pause_histogram_records_per_generation(self) -> None: - """A pause lands in the series of the generation that paused.""" - clock = Clock() - telemetry = GcPauseTelemetry(clock=clock) - telemetry._callback("start", {"generation": 2}) - clock.advance(0.30) - telemetry._callback("stop", {"generation": 2, "collected": 5, "uncollectable": 0}) - telemetry._callback("start", {"generation": 0}) - clock.advance(0.002) - telemetry._callback("stop", {"generation": 0, "collected": 1, "uncollectable": 0}) - - snapshot = telemetry.snapshot() - self.assertEqual(set(snapshot), set(PRISM_GC_GENERATIONS)) - self.assertEqual(snapshot["2"]["count"], 1) - self.assertAlmostEqual(snapshot["2"]["last_seconds"], 0.30) - self.assertAlmostEqual(snapshot["2"]["max_seconds"], 0.30) - self.assertEqual(snapshot["0"]["count"], 1) - self.assertAlmostEqual(snapshot["0"]["last_seconds"], 0.002) - self.assertEqual(snapshot["1"]["count"], 0) - buckets_2 = snapshot["2"]["buckets"] - assert isinstance(buckets_2, dict) - self.assertEqual(buckets_2[0.25], 0) - self.assertEqual(buckets_2[0.5], 1) - worst_generation, worst_seconds = telemetry.worst() - self.assertEqual(worst_generation, "2") - self.assertAlmostEqual(worst_seconds, 0.30) - # An unknown generation never grows the label set. - telemetry.record("3", 5.0) - self.assertEqual(set(telemetry.snapshot()), set(PRISM_GC_GENERATIONS)) - - def test_installed_collector_callbacks_record_a_real_collection(self) -> None: - telemetry = GcPauseTelemetry() - telemetry.install() - telemetry.install() # idempotent - self.addCleanup(telemetry.uninstall) - self.assertEqual(gc.callbacks.count(telemetry._callback), 1) - gc.collect() # a full collection reports generation 2 - self.assertGreaterEqual(telemetry.snapshot()["2"]["count"], 1) - telemetry.uninstall() - self.assertNotIn(telemetry._callback, gc.callbacks) - self.assertFalse(telemetry.installed) - - -class StallProbeTests(unittest.TestCase): - def test_stall_probe_is_rate_limited(self) -> None: - """Repeated over-half-slack wakes yield at most N samples per window.""" - probe = StallProbe() - self.assertEqual( - probe.max_samples_per_window, LEASE_MONITOR_STALL_PROBE_MAX_SAMPLES_PER_WINDOW - ) - self.assertEqual(probe.window_seconds, LEASE_MONITOR_STALL_PROBE_WINDOW_SECONDS) - triggers = 10 * LEASE_MONITOR_STALL_PROBE_MAX_SAMPLES_PER_WINDOW - sampled = [ - probe.sample(now=100.0 + index, wake_delay_seconds=0.3) - for index in range(triggers) - ] - self.assertEqual(probe.samples_total, LEASE_MONITOR_STALL_PROBE_MAX_SAMPLES_PER_WINDOW) - self.assertEqual( - probe.suppressed_total, - triggers - LEASE_MONITOR_STALL_PROBE_MAX_SAMPLES_PER_WINDOW, - ) - self.assertEqual( - sum(1 for text in sampled if text is not None), - LEASE_MONITOR_STALL_PROBE_MAX_SAMPLES_PER_WINDOW, - ) - # The next window re-arms the budget. - self.assertIsNotNone( - probe.sample( - now=100.0 + LEASE_MONITOR_STALL_PROBE_WINDOW_SECONDS, - wake_delay_seconds=0.3, - ) - ) - self.assertEqual( - probe.samples_total, LEASE_MONITOR_STALL_PROBE_MAX_SAMPLES_PER_WINDOW + 1 - ) - - def test_probe_samples_other_threads_not_itself(self) -> None: - parked = threading.Event() - release = threading.Event() - - def park() -> None: - parked.set() - release.wait(5.0) - - worker = threading.Thread(target=park, name="stall-probe-target", daemon=True) - worker.start() - self.addCleanup(release.set) - self.assertTrue(parked.wait(5.0)) - stacks = StallProbe().capture_stacks() - self.assertIn("stall-probe-target", stacks) - self.assertIn("park@", stacks) - self.assertNotIn(threading.current_thread().name + " (", stacks) - # Bounded output: every frame names a bare file, never a path. - frame_files = re.findall(r"@([^:\s]+):\d+", stacks) - self.assertTrue(frame_files) - for frame_file in frame_files: - self.assertNotIn("/", frame_file) - - def test_service_level_probe_holds_the_limit_under_a_stall_storm(self) -> None: - """Through the monitor's own entry point, the count still holds.""" - clock = Clock() - service, logged, exits = _service_with_fake_ports(clock) - trigger = LEASE_MONITOR_STALL_PROBE_TRIGGER_SLACK_FRACTION * POLICY.scheduler_slack_seconds - for _ in range(50): - clock.advance(0.05) - _wake(service, trigger) - self.assertEqual( - service.stall_probe.samples_total, - LEASE_MONITOR_STALL_PROBE_MAX_SAMPLES_PER_WINDOW, - ) - self.assertEqual( - service.stall_probe.suppressed_total, - 50 - LEASE_MONITOR_STALL_PROBE_MAX_SAMPLES_PER_WINDOW, - ) - # One probe line per sample, none for suppressed triggers, no exit. - self.assertEqual(len(logged), LEASE_MONITOR_STALL_PROBE_MAX_SAMPLES_PER_WINDOW) - self.assertEqual(exits, []) - self.assertEqual(service.exit_guarantee_breaches, 0) - - -class StallProbeFrameOwnershipTests(unittest.TestCase): - """A diagnostic must not keep finished workers alive until cyclic GC (#247).""" - - def _assert_worker_released( - self, probe: StallProbe, *, fail_formatting: bool = False - ) -> None: - class Payload: - pass - - parked = threading.Event() - release = threading.Event() - payload_refs: list[weakref.ReferenceType[Payload]] = [] - - def park() -> None: - payload = Payload() - payload_refs.append(weakref.ref(payload)) - parked.set() - release.wait(10.0) - - worker = threading.Thread(target=park, name="stall-probe-owner", daemon=True) - gc_was_enabled = gc.isenabled() - gc.disable() - failure = None - try: - worker.start() - try: - self.assertTrue(parked.wait(5.0)) - if fail_formatting: - expected = RuntimeError("stack formatting failed") - with mock.patch( - "lab.prism.background_services.os", - ) as probe_os: - probe_os.path.basename.side_effect = expected - try: - probe.capture(now=100.0, wake_delay_seconds=0.3) - except RuntimeError as error: - # Keep the traceback alive too: its capture_stacks - # frame must no longer own the sampled frames. - failure = error - self.assertIs(failure, expected) - else: - stacks = probe.capture(now=100.0, wake_delay_seconds=0.3) - self.assertEqual(probe.last_sample["stacks"], stacks) - finally: - release.set() - worker.join(5.0) - self.assertFalse(worker.is_alive()) - self.assertEqual(len(payload_refs), 1) - # No gc.collect() before this assertion. Keeping the probe, its - # last_sample, and any formatting traceback must retain only text. - self.assertIsNone(payload_refs[0]()) - finally: - failure = None - gc.collect() - if gc_was_enabled: - gc.enable() - - def test_completed_sample_releases_worker_without_cyclic_gc(self) -> None: - self._assert_worker_released(StallProbe()) - - def test_sampling_limits_release_worker_without_cyclic_gc(self) -> None: - for limits in ( - {"max_threads": 0}, - {"max_threads": 1}, - {"max_frames_per_thread": 0}, - {"max_frames_per_thread": 1}, - ): - with self.subTest(**limits): - self._assert_worker_released(StallProbe(**limits)) - - def test_formatting_error_releases_worker_with_traceback_alive(self) -> None: - self._assert_worker_released(StallProbe(), fail_formatting=True) - - def test_empty_snapshot(self) -> None: - with mock.patch("lab.prism.background_services.sys._current_frames", return_value={}): - self.assertEqual(StallProbe().capture_stacks(), "") - - -def _service_with_fake_ports( - clock: Clock, - *, - policy: WriterLeaseHeartbeatPolicy = POLICY, -) -> tuple[LedgerLeaseHeartbeatService, list[str], list[str]]: - """A heartbeat service on a virtual clock with a recording log and exit.""" - logged: list[str] = [] - exits: list[str] = [] - - class Ledger: - _lease_adoption_silence_seconds = policy.adoption_silence_seconds - - ports = LedgerLeaseHeartbeatPorts( - ledger=lambda: Ledger(), - heartbeat_seconds=lambda: policy.heartbeat_interval_seconds, - failure_seconds=lambda: policy.failure_budget_seconds, - monitor_seconds=lambda: policy.monitor_interval_seconds, - exit_timeout_seconds=lambda: policy.exit_margin_seconds, - external_fence_timeout_seconds=lambda: 0.5, - lease_hard_exit=lambda message, *, include_traceback: exits.append(message), - watchdog_hard_exit=lambda reason, *, timeout_seconds: exits.append(reason), - heartbeat_loop=lambda: None, - monitor_loop=lambda: None, - monotonic=clock, - scheduler_slack_seconds=lambda: policy.scheduler_slack_seconds, - log=logged.append, - ) - service = LedgerLeaseHeartbeatService(ports) - service.last_success_monotonic = clock() - service.last_server_proven_monotonic = clock() - return service, logged, exits - - -def _wake( - service: LedgerLeaseHeartbeatService, - wake_delay: float, - policy: WriterLeaseHeartbeatPolicy = POLICY, -): - """One monitor wake through the shipped accounting, diagnostics run inline. - - The monitor loop hands the returned job to a diagnostics thread after - the exit decision; the tier tests here run it synchronously so their - assertions are deterministic. The threading itself is pinned by - ``InstrumentCostAndOrderingTests``. - """ - job = service._observe_monitor_wake(wake_delay, policy) - if job is not None: - job() - return job - - -class LatenessBeyondSlackResponseTests(unittest.TestCase): - def test_lateness_beyond_slack_triggers_the_documented_response(self) -> None: - """Option (b), pinned: no exit, a counted breach, one structured warning. - - Driven at the production number. A 0.648s wake is beyond the - 0.50s slack *and* beyond the 0.55s bound of inequality (4), so it - is an exit-guarantee breach with a 0.098s overrun. The monitor - accepts it — the fences carry the residual — and says so. - """ - clock = Clock() - service, logged, exits = _service_with_fake_ports(clock) - clock.advance(1.0) - _wake(service, OBSERVED_MONITOR_WAKE_DELAY_SECONDS_227) - - self.assertEqual(exits, []) - self.assertEqual(service.exit_guarantee_breaches, 1) - self.assertAlmostEqual( - service.worst_exit_guarantee_overrun_seconds, - OBSERVED_MONITOR_WAKE_DELAY_SECONDS_227 - - POLICY.max_guaranteed_monitor_lateness_seconds, - ) - self.assertEqual(len(logged), 1) - warning = logged[0] - self.assertIn("WARNING", warning) - self.assertIn("monitor wake delay 0.648s exceeded the 0.500s scheduler slack", warning) - self.assertIn("issue #227", warning) - self.assertIn("exit-before-adoption was NOT guaranteed for this beat", warning) - self.assertIn("0.098s after the adoption edge", warning) - self.assertIn("session-token fenced", warning) - self.assertIn("Last attempt:", warning) - self.assertIn("Policy:", warning) - # A stack sample rode along with the first warning. - self.assertIn("Threads:", warning) - self.assertEqual(service.stall_probe.samples_total, 1) - breach = service.last_slack_breach - assert breach is not None - self.assertAlmostEqual(breach["wake_delay_seconds"], 0.648) - self.assertAlmostEqual(breach["overrun_seconds"], 0.098) - - # The metric surface reports it under fixed keys. - snapshot = service.snapshot() - self.assertEqual(snapshot["exit_guarantee_breaches"], 1) - self.assertAlmostEqual(snapshot["worst_exit_guarantee_overrun_seconds"], 0.098) - wakes = snapshot["monitor_wakes"] - assert isinstance(wakes, dict) - self.assertEqual(wakes["late_wakes"], {"0.5": 1, "0.8": 1, "1.0": 1}) - self.assertAlmostEqual(wakes["lifetime_max_seconds"], 0.648) - self.assertAlmostEqual(wakes["window_max_seconds"], 0.648) - policy_seconds = snapshot["policy_seconds"] - assert isinstance(policy_seconds, dict) - self.assertAlmostEqual(policy_seconds["max_guaranteed_monitor_lateness"], 0.55) - - def test_lateness_inside_the_reserve_warns_without_a_guarantee_breach(self) -> None: - clock = Clock() - service, logged, exits = _service_with_fake_ports(clock) - clock.advance(1.0) - _wake(service, 0.52) - self.assertEqual(exits, []) - self.assertEqual(service.exit_guarantee_breaches, 0) - self.assertEqual(service.worst_exit_guarantee_overrun_seconds, 0.0) - self.assertEqual(len(logged), 1) - self.assertIn("exit-before-adoption still held for this beat", logged[0]) - self.assertIn("strictness reserve consumed", logged[0]) - - def test_wakes_within_slack_are_counted_but_never_warned(self) -> None: - clock = Clock() - service, logged, exits = _service_with_fake_ports(clock) - for delay in (0.0, 0.01, 0.2, 0.24): - clock.advance(0.05) - self.assertIsNone(_wake(service, delay)) - self.assertEqual(logged, []) - self.assertEqual(exits, []) - self.assertEqual(service.exit_guarantee_breaches, 0) - self.assertEqual(service.stall_probe.samples_total, 0) - self.assertEqual(service.snapshot()["monitor_wakes"]["count"], 4) - - def test_breach_warnings_are_rate_limited_like_the_probe(self) -> None: - """A stall storm produces bounded warnings, every breach still counted.""" - clock = Clock() - service, logged, exits = _service_with_fake_ports(clock) - storm = 4 * LEASE_MONITOR_STALL_PROBE_MAX_SAMPLES_PER_WINDOW - for _ in range(storm): - clock.advance(0.7) - _wake(service, 0.7) - self.assertEqual(service.exit_guarantee_breaches, storm) - self.assertEqual(len(logged), LEASE_MONITOR_STALL_PROBE_MAX_SAMPLES_PER_WINDOW) - self.assertEqual( - service.suppressed_slack_breach_warnings, - storm - LEASE_MONITOR_STALL_PROBE_MAX_SAMPLES_PER_WINDOW, - ) - # Suppressed warnings are exported, so a quiet period and a - # rate-limited flood do not look the same on /metrics. - self.assertEqual( - service.snapshot()["slack_breach_warnings_suppressed"], - storm - LEASE_MONITOR_STALL_PROBE_MAX_SAMPLES_PER_WINDOW, - ) - self.assertEqual(exits, []) - - def test_late_monitor_beyond_slack_survives_on_the_harness(self) -> None: - """The real loops: a 0.648s-late monitor over a healthy heartbeat. - - The tail-latency suite proves survival at 0.4s (inside the slack). - This is the #227 case, beyond the slack: the monitor must not - hard-exit — the heartbeat kept proving, so nothing tripped a bound - — and must have recorded the exit-guarantee breach. - """ - with LeaseHarness(lease_ttl_seconds=LEASE_TTL_SECONDS) as harness: - alpha = harness.coordinator("alpha") - alpha.start() - harness.run_until(alpha, "done:startup") - heartbeat = HeartbeatUnderTest(harness, alpha) - warnings: list[str] = [] - heartbeat.service._ports = replace( - heartbeat.service._ports, log=warnings.append - ) - actors = [heartbeat.heartbeat, heartbeat.monitor] - heartbeat.run(actors, seconds=1.0) - - harness.advance( - heartbeat.policy.monitor_interval_seconds - + OBSERVED_MONITOR_WAKE_DELAY_SECONDS_227 - ) - while heartbeat.monitor.runnable(): - harness.step(heartbeat.monitor) - heartbeat.run(actors, seconds=1.0) - heartbeat.quiesce(actors) - - self.assertFalse(heartbeat.exited, heartbeat.exits) - service = heartbeat.service - self.assertGreaterEqual(service.monitor_wake_delay_seconds, 0.648) - self.assertEqual(service.exit_guarantee_breaches, 1) - self.assertGreaterEqual(service.worst_exit_guarantee_overrun_seconds, 0.098) - # The warning is written by the diagnostics thread, after the - # monitor's decision; wait for it before reading the sink. - diagnostics = service.diagnostics_thread - self.assertIsNotNone(diagnostics) - assert diagnostics is not None - diagnostics.join(5.0) - self.assertFalse(diagnostics.is_alive()) - self.assertEqual(len(warnings), 1) - self.assertIn("NOT guaranteed", warnings[0]) - snapshot = service.snapshot() - self.assertEqual(snapshot["monitor_wakes"]["late_wakes"]["1.0"], 1) - - def test_hard_exit_reason_quotes_window_max_record_age_and_gc(self) -> None: - """The exit message no longer quotes only a possibly hours-old record.""" - with LeaseHarness(lease_ttl_seconds=LEASE_TTL_SECONDS) as harness: - alpha = harness.coordinator("alpha") - alpha.start() - harness.run_until(alpha, "done:startup") - heartbeat = HeartbeatUnderTest(harness, alpha) - heartbeat.service._ports = replace( - heartbeat.service._ports, log=lambda line: None - ) - heartbeat.service.gc_pauses.record("2", 0.123) - actors = [heartbeat.heartbeat, heartbeat.monitor] - heartbeat.run(actors, seconds=1.0) - # Deschedule the heartbeat for good: the monitor must age it - # out and exit, quoting the new figures. - heartbeat.stall_heartbeat(heartbeat.policy.server_proven_cap_seconds + 1.0) - self.assertTrue(heartbeat.exited) - reason = heartbeat.exit_reason - self.assertIn("Monitor wake delay:", reason) - self.assertIn("lifetime record", reason) - self.assertIn(f"in the last {LEASE_MONITOR_WAKE_DELAY_WINDOW_SECONDS:g}s", reason) - self.assertIn("exit-guarantee breach(es)", reason) - self.assertIn("Worst GC pause: 0.123s (generation 2)", reason) - - -class _SnapshotOnWrite(list): - """A bucket list that takes a snapshot the instant any bucket is bumped. - - Stands in for a scrape interleaving with a writer between two of the - writer's stores; the invariant under test is that no interleaving can - render a bucket above the histogram's count. - """ - - def __init__(self, values, on_write) -> None: - super().__init__(values) - self._on_write = on_write - - def __setitem__(self, index, value) -> None: - super().__setitem__(index, value) - self._on_write() - - -DIAGNOSTICS_THREAD_NAME = "prism-ledger-lease-monitor-diagnostics" - - -class InstrumentCostAndOrderingTests(unittest.TestCase): - """The instrument must not extend the path it measures. - - The monitor thread decides the hard exit. Everything this package adds - to that thread has to be O(1) and has to come *after* the decision, or - it consumes the very envelope it reports on — and work done between - polls is not measured as lateness at all. These tests pin the - mechanism (which thread, in which order, which calls are never made), - not a duration. - """ - - def test_observing_a_wake_does_no_capture_or_logging_itself(self) -> None: - """Accounting is O(1); the frame walk and the log line are deferred.""" - clock = Clock() - service, logged, exits = _service_with_fake_ports(clock) - captures: list[str] = [] - - def capture_stacks() -> str: - captures.append(threading.current_thread().name) - return "stacks" - - service.stall_probe.capture_stacks = capture_stacks # type: ignore[method-assign] - clock.advance(1.0) - job = service._observe_monitor_wake(OBSERVED_MONITOR_WAKE_DELAY_SECONDS_227, POLICY) - # Counted and admitted, but nothing walked and nothing written yet. - self.assertEqual(service.exit_guarantee_breaches, 1) - self.assertEqual(captures, []) - self.assertEqual(logged, []) - self.assertEqual(service.stall_probe.samples_total, 0) - self.assertIsNotNone(job) - assert job is not None - # The shipped runner hands the job to a named daemon thread. - service._run_diagnostics(job) - thread = service.diagnostics_thread - assert thread is not None - thread.join(5.0) - self.assertFalse(thread.is_alive()) - self.assertEqual(captures, [DIAGNOSTICS_THREAD_NAME]) - self.assertEqual(len(logged), 1) - self.assertIn("Threads:\nstacks", logged[0]) - self.assertEqual(service.stall_probe.samples_total, 1) - self.assertEqual(exits, []) - - def test_probe_and_warning_run_after_the_exit_decision_and_off_the_monitor_thread( - self, - ) -> None: - """On the real loops: decide first, diagnose after, never on the monitor. - - First a healthy heartbeat with a 0.648s-late monitor: the sample and - the warning both happen, on the diagnostics thread, after the - monitor has decided not to exit. Then a dead heartbeat with a wake - far beyond the slack: the monitor exits, on its own thread, and no - capture or log line runs ahead of that exit (nor after it — the - exit reason carries the figures). - """ - with LeaseHarness(lease_ttl_seconds=LEASE_TTL_SECONDS) as harness: - alpha = harness.coordinator("alpha") - alpha.start() - harness.run_until(alpha, "done:startup") - heartbeat = HeartbeatUnderTest(harness, alpha) - service = heartbeat.service - monitor_thread_name = heartbeat.monitor._thread.name - events: list[tuple[str, str]] = [] - - def capture_stacks() -> str: - events.append(("capture", threading.current_thread().name)) - return "stacks" - - service.stall_probe.capture_stacks = capture_stacks # type: ignore[method-assign] - original_exit = service._ports.lease_hard_exit - - def lease_hard_exit(message: str, *, include_traceback: bool) -> None: - events.append(("exit", threading.current_thread().name)) - original_exit(message, include_traceback=include_traceback) - - service._ports = replace( - service._ports, - log=lambda line: events.append(("log", threading.current_thread().name)), - lease_hard_exit=lease_hard_exit, - ) - actors = [heartbeat.heartbeat, heartbeat.monitor] - heartbeat.run(actors, seconds=1.0) - - # Healthy heartbeat, late monitor. - harness.advance( - heartbeat.policy.monitor_interval_seconds - + OBSERVED_MONITOR_WAKE_DELAY_SECONDS_227 - ) - while heartbeat.monitor.runnable(): - harness.step(heartbeat.monitor) - self.assertFalse(heartbeat.exited) - diagnostics = service.diagnostics_thread - self.assertIsNotNone(diagnostics) - assert diagnostics is not None - diagnostics.join(5.0) - self.assertFalse(diagnostics.is_alive()) - self.assertEqual([kind for kind, _ in events], ["capture", "log"]) - self.assertEqual( - {name for _, name in events}, {DIAGNOSTICS_THREAD_NAME} - ) - self.assertNotEqual(DIAGNOSTICS_THREAD_NAME, monitor_thread_name) - self.assertEqual(service.exit_guarantee_breaches, 1) - events.clear() - - # Dead heartbeat: only the monitor is stepped, after sleeping - # through the whole cap. Its wake is far beyond the slack, so - # both tiers are due — and neither may run ahead of the exit. - harness.advance(heartbeat.policy.server_proven_cap_seconds + 1.0) - while heartbeat.monitor.runnable() and not heartbeat.exited: - harness.step(heartbeat.monitor) - self.assertTrue(heartbeat.exited) - self.assertEqual(events, [("exit", monitor_thread_name)]) - # The wake was still accounted (O(1), before the decision). - self.assertEqual(service.exit_guarantee_breaches, 2) - self.assertIn("exit-guarantee breach(es)", heartbeat.exit_reason) - heartbeat.quiesce(actors) - - def test_stack_capture_never_touches_linecache(self) -> None: - """No stat, no source read: the walk uses frame objects only. - - ``traceback.extract_stack`` stats every file and reads every line; - even ``StackSummary.extract(lookup_lines=False)`` still calls - ``linecache.checkcache`` per file. The sample prints function - names, basenames and line numbers, none of which need the source. - """ - parked = threading.Event() - release = threading.Event() - - def park() -> None: - parked.set() - release.wait(5.0) - - worker = threading.Thread(target=park, name="linecache-probe-target", daemon=True) - worker.start() - self.addCleanup(release.set) - self.assertTrue(parked.wait(5.0)) - with mock.patch("linecache.checkcache") as checkcache, mock.patch( - "linecache.lazycache" - ) as lazycache, mock.patch("linecache.getline") as getline, mock.patch( - "linecache.getlines" - ) as getlines, mock.patch("linecache.updatecache") as updatecache: - stacks = StallProbe().capture_stacks() - for patched in (checkcache, lazycache, getline, getlines, updatecache): - patched.assert_not_called() - self.assertIn("linecache-probe-target", stacks) - self.assertIn("park@", stacks) - - def test_gc_pause_buckets_never_exceed_count_mid_record(self) -> None: - """A scrape between two of record()'s stores sees bucket <= count.""" - telemetry = GcPauseTelemetry(clock=Clock()) - seen: list[dict[str, object]] = [] - telemetry._bucket_counts["2"] = _SnapshotOnWrite( - telemetry._bucket_counts["2"], - lambda: seen.append(telemetry.snapshot()["2"]), - ) - telemetry.record("2", 0.3) - self.assertTrue(seen) - for mid in seen: - buckets = mid["buckets"] - assert isinstance(buckets, dict) - self.assertTrue( - all(value <= mid["count"] for value in buckets.values()), mid - ) - - def test_wake_buckets_never_exceed_count_mid_observe(self) -> None: - telemetry = MonitorWakeTelemetry() - seen: list[dict[str, object]] = [] - telemetry._bucket_counts = _SnapshotOnWrite( - telemetry._bucket_counts, - lambda: seen.append(telemetry.snapshot(1.0)), - ) - telemetry.observe(0.3, slack_seconds=0.5, now=1.0) - self.assertTrue(seen) - for mid in seen: - buckets = mid["buckets"] - assert isinstance(buckets, dict) - self.assertTrue( - all(value <= mid["count"] for value in buckets.values()), mid - ) - - -if __name__ == "__main__": - unittest.main() diff --git a/tests/test_prism_lease_orphaned_lock.py b/tests/test_prism_lease_orphaned_lock.py deleted file mode 100644 index 9a793cf6c..000000000 --- a/tests/test_prism_lease_orphaned_lock.py +++ /dev/null @@ -1,384 +0,0 @@ -#!/usr/bin/env python3 -"""Issue #123: an orphaned lease-tuple lock blocking writer failover. - -#123 describes a coordinator that vanishes mid-statement — a network -partition, a SIGSTOP, a VM pause — after PostgreSQL has finished executing a -deadline-scoped landing statement but before the client's COMMIT arrives. The -server keeps the transaction open and keeps the ``qbit_ledger_writer_lease`` -tuple lock. Before the fix nothing bounded how long it held them: -``idle_in_transaction_session_timeout`` was set nowhere, server-side TCP -keepalives were configured nowhere, and the startup lease upsert ran with no -``lock_timeout``, so a successor coordinator waited inside -``PsqlShareLedger.__init__`` until TCP keepalive teardown — hours at OS -defaults. - -The issue was written as a hypothesis with a hand-derived trigger sequence -because no test could express it: reproducing the interleaving meant a live -PostgreSQL, a partition, and luck. This module drives it deterministically -against the harness instead, with the successor running the shipped -acquisition path — the real retry loop, the real CAS SQL, the real -``_run_json`` transaction shape. - -Two tests, doing two different jobs: - -``OrphanedLeaseLockInterleavingTests`` - Proves the harness still expresses the interleaving: the schedule is - stable across repeated runs, the orphan really does hold the tuple lock, - and the successor really does park on it. That evidence is what #128 - asked for, and the fix does not remove any of it — the successor still - queues behind an orphan it cannot clear. What changed is that the wait is - now bounded, so the tests pin the bound instead of its absence. - -``OrphanedLeaseLockFailoverBoundTests`` - Asserts the behaviour #123's fix must produce: a successor's startup - finishes, one way or the other, within a bounded time. - - That assertion was authored against the fixed behaviour while 2.x.x was - still unfixed, and carried an ``expectedFailure`` marker so the suite - would go red the day it started passing. It now passes, so the marker is - gone — removed by the commit that fixed #123, exactly as this file - demanded. The assertion itself is unchanged and is the strongest - evidence the fix works: it was written by someone who had not seen the - fix, against a model of PostgreSQL that knows nothing about it. -""" - -from __future__ import annotations - -import unittest - -from lab.prism.share_ledger import ( - DEFAULT_LEASE_ACQUIRE_ATTEMPTS, - DEFAULT_LEASE_ACQUIRE_LOCK_TIMEOUT_SECONDS, - LedgerOperationTimeout, -) -from tests.prism_concurrency_harness import ( - LeaseHarness, - assert_deterministic, -) - -# How long the vanished client's landing statement was budgeted for. #120's -# escalated landing budget tops out at 120s; the exact value does not matter -# to the interleaving, only that a deadline is armed, because that is what -# makes _NativePostgresClient wrap the statement in an explicit transaction -# whose COMMIT is a separate client message. -LANDING_BUDGET_SECONDS = 120.0 - -# Well past any plausible watchdog, restart or retry interval, and still far -# short of the "on the order of hours" TCP keepalive teardown #123 describes. -FAILOVER_HORIZON_SECONDS = 6 * 60 * 60 - -# PsqlShareLedger's floor between acquisition attempts (_lease_retry_sleep is -# called with _lease_retry_min_sleep_seconds, which is min(0.25, max sleep)). -# Not a tunable of this fix; named so the elapsed-time assertion below can be -# exact rather than approximate. -RETRY_SLEEP_SECONDS = 0.25 - - -def drive_orphaned_lock_interleaving(harness: LeaseHarness) -> dict[str, object]: - """Run #123's trigger sequence and report what the successor sees. - - 1. ``alpha`` starts and takes the writer lease. - 2. ``alpha`` runs a deadline-scoped fenced statement, which PostgreSQL - executes inside an explicit transaction holding the lease tuple lock. - 3. ``alpha`` vanishes in the gap between the statement completing and its - COMMIT — no RST reaches the server, so the transaction stays open. - 4. ``beta``, a failover twin at the next writer epoch, starts up. Its - advisory-guard key differs from ``alpha``'s, so it is not held off - there and reaches the lease upsert, which is where #123 bites. - """ - alpha = harness.coordinator("alpha", writer_epoch=1) - alpha.start() - harness.run_until(alpha, "done:startup") - - def fenced_landing_statement() -> object: - # Any deadline-scoped fenced write has this shape; renewal is the - # smallest one that touches the lease tuple, and it is real shipped - # code rather than a stand-in. - with alpha.ledger.operation_timeout(LANDING_BUDGET_SECONDS): - return alpha.ledger.renew_writer_lease() - - alpha.submit(fenced_landing_statement, label="landing") - # Stop in the gap PostgreSQL is left in: statement executed, COMMIT not - # sent. This is the exact instant #123 names. - harness.run_until(alpha, "alpha.precommit") - alpha.vanish() - - orphan = harness.server.lease_lock_holder - - beta = harness.coordinator("beta", writer_epoch=2) - beta.start() - harness.run_until_blocked(beta, "beta.lockwait:acquire") - - return { - "orphan_open": orphan is not None and orphan.orphaned, - "orphan_holds_lease_lock": harness.server.lease_lock_holder is orphan, - "beta_blocked_at": beta.actor.block_reason, - "beta_parked_at": harness.clock.monotonic(), - "beta_wake_at": beta.actor.wake_at, - "scheduler_next_deadline": harness.scheduler.next_deadline(), - "lease_holder_session": harness.lease_holder_session(), - "statements": harness.statement_kinds(), - } - - -def drive_successor_to_quiescence( - harness: LeaseHarness, - successor: object, - *, - horizon_seconds: float = FAILOVER_HORIZON_SECONDS, -) -> bool: - """Run the successor until it finishes or nothing can move it again. - - Draining once is not enough to decide "did the wait end". A fix built - from ``lock_timeout`` plus a bounded retry — which is what #123 proposes - — parks the successor on a succession of short deadlines, and a single - ``advance`` followed by a single ``drain`` would leave it stopped at the - first of them and report a wait that never ended. Advancing to each - pending deadline in turn asks the question the assertion actually means: - is there any sequence of timeouts that gets this coordinator out? - - Returns True when startup finished, by acquiring or by raising. Returns - False when the successor is stuck with no deadline left to reach, which - is the state ``2.x.x`` produces. - """ - startup = successor.actor.calls[0] # type: ignore[attr-defined] - deadline = harness.clock.monotonic() + horizon_seconds - while not startup.done and harness.clock.monotonic() < deadline: - harness.drain([successor]) - if startup.done: - break - if harness.advance_to_next_deadline() is None: - # Nothing in the system is waiting on time, so no amount of - # further waiting changes anything. - break - return startup.done - - -class OrphanedLeaseLockInterleavingTests(unittest.TestCase): - """The harness can express and reproduce #123's interleaving.""" - - def test_successor_parks_on_the_orphaned_lease_tuple_lock(self) -> None: - with LeaseHarness() as harness: - observed = drive_orphaned_lock_interleaving(harness) - - self.assertTrue( - observed["orphan_open"], - "the vanished client's transaction must still be open server-side", - ) - self.assertTrue( - observed["orphan_holds_lease_lock"], - "the orphaned transaction must still hold the lease tuple lock", - ) - self.assertEqual(observed["beta_blocked_at"], "beta.lockwait:acquire") - # The interleaving #123 names is intact: the orphan holds the - # tuple lock and the successor is parked on it. What the fix - # changes is the only thing that was ever wrong with it — the - # wait now ends. _run_lease_acquisition_json arms - # lock_timeout/statement_timeout on every attempt, so the - # successor parks with a wake time rather than forever. - self.assertEqual( - observed["beta_wake_at"], - observed["beta_parked_at"] - + DEFAULT_LEASE_ACQUIRE_LOCK_TIMEOUT_SECONDS, - "the successor must park under the configured acquisition " - "lock deadline, not indefinitely", - ) - self.assertEqual( - observed["scheduler_next_deadline"], - observed["beta_wake_at"], - "and that deadline must be the thing the whole system is " - "waiting on: nothing else is pending, so it is what releases " - "the successor", - ) - # The predecessor's renewal never committed, so the committed row - # still names its session: the successor is queued behind a write - # whose effect will never land. - self.assertEqual( - observed["lease_holder_session"], - "heartbeat-v1:alpha-1", - ) - self.assertEqual( - observed["statements"], - ["acquire", "renew", "acquire"], - ) - - def test_wait_ends_in_bounded_attempts_and_a_visible_failure(self) -> None: - """How the wait ends, now that it ends. - - This test used to assert ``finished is False``: six virtual hours - passed and the successor was still parked, because nothing bounded - the wait. The fix inverts that premise outright, and simply flipping - the assertion would duplicate - ``OrphanedLeaseLockFailoverBoundTests``, which already pins *that* - the wait ends. - - So it pins *how* it ends instead, which nothing else covers: the - successor spends exactly its configured retry budget on the lock, - each attempt bounded by the configured deadline, and then fails - visibly with a ``RuntimeError`` that chains the underlying - ``LedgerOperationTimeout``. Retrying matters because an orphan is - often reaped partway through the budget; giving up visibly matters - because a coordinator that cannot take the lease must exit for its - supervisor rather than hang in ``__init__``. The evidence the old - assertion carried — that nothing releases the orphan's lock, so the - successor never actually starts — is kept below. - """ - with LeaseHarness() as harness: - drive_orphaned_lock_interleaving(harness) - beta = harness.coordinators["beta"] - before = harness.clock.monotonic() - - finished = drive_successor_to_quiescence(harness, beta) - - self.assertTrue(finished, "the bounded retry must terminate") - self.assertFalse( - beta.started, - "and it must not have started: the orphan still holds the " - "lease lock, so there was never a lease to take", - ) - # The orphan is untouched — this is still #123's interleaving, - # not a scenario that quietly resolved itself. - orphan = harness.server.lease_lock_holder - self.assertIsNotNone(orphan) - assert orphan is not None - self.assertTrue(orphan.orphaned) - - # Exactly the configured budget of attempts, no more and no - # fewer: one attempt would abandon orphans that clear on their - # own, and an unbounded count is the outage this fix removes. - self.assertEqual( - harness.statement_kinds().count("acquire"), - 1 + DEFAULT_LEASE_ACQUIRE_ATTEMPTS, - "alpha's successful acquire plus beta's full retry budget", - ) - # Every attempt ran the deadline out, so the wall time is the - # budget itself rather than an accident of scheduling. - self.assertEqual( - harness.clock.monotonic() - before, - DEFAULT_LEASE_ACQUIRE_ATTEMPTS - * DEFAULT_LEASE_ACQUIRE_LOCK_TIMEOUT_SECONDS - + (DEFAULT_LEASE_ACQUIRE_ATTEMPTS - 1) * RETRY_SLEEP_SECONDS, - ) - - with self.assertRaisesRegex( - RuntimeError, - "did not complete within its .*deadline on any of " - f"{DEFAULT_LEASE_ACQUIRE_ATTEMPTS} attempts", - ) as raised: - beta.actor.calls[0].value() - # Chained, not swallowed: the operator diagnoses from the cause - # rather than from this layer's guess at one. - self.assertIsInstance( - raised.exception.__cause__, - LedgerOperationTimeout, - ) - - def test_interleaving_is_order_stable_across_repeated_runs(self) -> None: - """A flaky concurrency test is worse than none; prove this one is not.""" - run = assert_deterministic( - self, - drive_orphaned_lock_interleaving, - repeats=25, - ) - - self.assertEqual( - list(run.trace), - [ - "alpha@alpha.begin:acquire", - "alpha@alpha.done:acquire", - # New since the fix: the startup acquire is deadline-scoped, - # so it runs in an explicit transaction with its own COMMIT. - # docs/prism-ledger-ops.md records what that costs. - "alpha@alpha.precommit", - "alpha@done:startup", - "alpha@alpha.begin:renew", - "alpha@alpha.done:renew", - "alpha@alpha.precommit", - "alpha@vanished", - "beta@beta.begin:acquire", - "beta@beta.lockwait:acquire", - ], - ) - - def test_committing_the_predecessor_releases_the_successor(self) -> None: - """Control: the successor is blocked by the lock, not by the harness. - - If the vanished client's transaction commits after all, the successor - wakes on the next step and its upsert proceeds. That is what makes the - parked state above attributable to the orphaned lock rather than to a - scenario that simply never scheduled the successor. - """ - with LeaseHarness() as harness: - drive_orphaned_lock_interleaving(harness) - beta = harness.coordinators["beta"] - orphan = harness.server.lease_lock_holder - assert orphan is not None - - harness.server._commit(orphan) - - self.assertTrue(beta.actor.runnable()) - harness.run_until(beta, "done:startup") - - # Once the lock is gone the twin gets to read the row and reaches - # its decision immediately. A different epoch cannot wait on - # another identity's unexpired lease, so it refuses loudly. That - # is the shape #123's fix should produce within a bounded time: - # the twin has no useful work to do behind that lock, and every - # second it spends queued for it is pure outage. - with self.assertRaisesRegex( - RuntimeError, - "qbit ledger writer lease is held by prism-coordinator epoch=1", - ): - beta.actor.calls[0].value() - self.assertEqual( - harness.lease_holder_session(), - "heartbeat-v1:alpha-1", - ) - - -class OrphanedLeaseLockFailoverBoundTests(unittest.TestCase): - """The bound #123's fix must establish. Does not hold until #123 is fixed.""" - - def test_successor_startup_must_not_wait_unboundedly(self) -> None: - """A successor must fail fast and visibly, not queue for hours. - - #123's proposed fix runs the startup lease upsert with - ``SET LOCAL lock_timeout`` and a bounded retry. Under that fix the - successor's acquisition ends within a bounded time — by acquiring, or - by raising something an operator can see — rather than waiting on a - lock nobody will ever release. - - The successor is driven through every deadline it reaches rather than - drained once, so a fix whose bounded wait spans several short - timeouts satisfies this as readily as one that fails on the first. - Asserting anything narrower would leave the person who lands #123 - looking at a red test their correct fix did not turn green, and - reaching for the same weakening this test exists to refuse. - - This assertion was deliberately written against the fixed behaviour - rather than against the behaviour of the day. It carried an - ``expectedFailure`` marker until #123 was fixed; the marker is gone - because the bound now holds, which is the removal this docstring - asked the fixing commit to make. The assertion is otherwise - untouched. - """ - with LeaseHarness() as harness: - drive_orphaned_lock_interleaving(harness) - beta = harness.coordinators["beta"] - - finished = drive_successor_to_quiescence(harness, beta) - - self.assertTrue( - finished, - "the successor's PsqlShareLedger.__init__ is still waiting on " - f"the orphaned lease-tuple lock after " - f"{FAILOVER_HORIZON_SECONDS / 3600:g} virtual hours, with no " - "deadline left anywhere that could release it. #123 has " - "regressed: the startup lease upsert is reaching the tuple " - "lock without a lock_timeout again, so nothing bounds the " - "wait.", - ) - - -if __name__ == "__main__": - unittest.main() diff --git a/tests/test_prism_lease_renewal_vs_expiry_claim.py b/tests/test_prism_lease_renewal_vs_expiry_claim.py deleted file mode 100644 index 1a4534f32..000000000 --- a/tests/test_prism_lease_renewal_vs_expiry_claim.py +++ /dev/null @@ -1,564 +0,0 @@ -#!/usr/bin/env python3 -"""Writer-lease heartbeat renewal racing a competing expiry claim. - -``PsqlShareLedger.verify_writer_lease_guard_session`` renews the lease TTL -with ``FOR NO KEY UPDATE SKIP LOCKED`` so a heartbeat never queues behind a -fenced write holding the ``qbit_ledger_writer_lease`` tuple lock. Declining -to queue buys liveness but loses the re-evaluation the old locking renewal -performed after a contending transaction committed: once the committed row's -TTL has lapsed, the lock the heartbeat skipped may be a different-identity -``_try_acquire_writer_lease`` taking the row through its expiry CAS, and the -stale committed snapshot would keep naming this session right up until that -claim commits. The defect shape this module covers is that gap: a heartbeat -that reads its own stale token off an expired, foreign-locked row and calls -itself alive would authorize a fenced-out writer. - -The production docstring states four invariants, and each gets a scenario: - -1. Uncontended, the heartbeat renews: ``renewed_count == 1`` and the - committed row's ``lease_expires_at`` moves a full TTL ahead of the - virtual clock. -2. Expired row plus a lock held by a different-identity claimant fails - closed ("expired and its renewal was lock-blocked"), and it is the - *combination* that raises: the same foreign lock over an unexpired row - verifies fine with ``renewed_count == 0``. -3. The one ambiguous result shape — expired, renewal skipped, no - attributable locker — earns exactly one recheck on a fresh snapshot, - because this coordinator's own fenced write committing between the first - statement's MVCC snapshot and its live ``pg_stat_activity`` probe - presents identically to a competing claim. The recheck resolves it and - the verification returns instead of restart-looping a healthy writer. -4. A claim that commits wins: the next verification raises "writer lease is - not active" instead of reporting a renewal. - -The competing claim is a real second coordinator at the next writer epoch -running the shipped startup acquisition. Its expiry CAS executes and takes -the tuple lock, and the harness's ``done:acquire`` checkpoint sits between -statement execution and the transaction's commit, so parking the claimant -there holds the claim in flight — executed, locking, uncommitted — for as -long as the scenario needs. -""" - -from __future__ import annotations - -import unittest -from datetime import timedelta - -from tests.prism_concurrency_harness import ( - LeaseHarness, - assert_deterministic, -) - -LEASE_TTL_SECONDS = 60.0 - -# Push the virtual clock just past lease_expires_at. The exact overshoot is -# irrelevant; the fail-closed branch triggers on `lease_expires_at <= -# clock_timestamp()`, not on how stale the row is. -EXPIRY_LAPSE_SECONDS = LEASE_TTL_SECONDS + 1.0 - -# For the unexpired control: far enough in that time visibly passed, far -# enough out that the row is neither expired nor inside the authority margin -# (TTL/2 = 30s), so the success path exercised is plain "skip and trust the -# committed row" with no deferral semantics in play. -UNEXPIRED_ADVANCE_SECONDS = 20.0 - -# Budget for the coordinator's own fenced write in the recheck scenario. Any -# armed deadline works; a deadline is what makes the client wrap the -# statement in an explicit transaction whose COMMIT is a separate message, -# giving the scenario a precommit stop inside the snapshot-to-probe window. -LANDING_BUDGET_SECONDS = 30.0 - - -def drive_expired_lock_blocked_renewal(harness: LeaseHarness) -> dict[str, object]: - """Scenario 2's raise: expired row, tuple lock held by a foreign claim. - - 1. ``alpha`` starts and holds the lease; the clock then lapses the TTL. - 2. ``claimant``, at the next writer epoch, starts up. Its advisory-guard - key differs from ``alpha``'s, so it reaches the lease upsert, whose - expiry CAS executes and takes the tuple lock. It is parked at - ``done:acquire``: the claim is in flight and uncommitted. - 3. ``alpha``'s heartbeat verification runs. Its ``SKIP LOCKED`` renewal - skips, the committed snapshot still names ``alpha``, and the row is - expired — the shape whose meaning one statement cannot decide, so the - verification spends its one recheck on a fresh snapshot, sees the - same contended expiry, and fails closed. - """ - alpha = harness.coordinator("alpha", writer_epoch=1) - alpha.start() - harness.run_until(alpha, "done:startup") - row_at_startup = harness.lease_row() - assert row_at_startup is not None - - harness.advance(EXPIRY_LAPSE_SECONDS) - - claimant = harness.coordinator("claimant", writer_epoch=2) - claimant.start() - harness.run_until(claimant, "claimant.begin:acquire") - harness.run_until(claimant, "claimant.done:acquire") - claim_tx = harness.server.lease_lock_holder - - verify = alpha.submit( - alpha.ledger.verify_writer_lease_guard_session, - label="verify", - ) - harness.run_until(alpha, "done:verify") - - row = harness.lease_row() - assert row is not None - return { - "claim_took_lock": claim_tx is not None, - "claim_still_holds_lock": harness.server.lease_lock_holder is claim_tx, - "claim_uncommitted": not claimant.started, - "verify_raised": verify.error is not None, - "verify_error": str(verify.error), - "committed_row_session": row.writer_session_token, - "committed_row_expired": row.lease_expires_at <= harness.clock.now(), - "committed_row_unrenewed": ( - row.lease_expires_at == row_at_startup.lease_expires_at - ), - "statement_kinds": harness.statement_kinds(), - } - - -def drive_unexpired_lock_blocked_renewal(harness: LeaseHarness) -> dict[str, object]: - """Scenario 2's control: the identical foreign lock over a live row. - - Same claimant, same parked uncommitted acquire holding the tuple lock; - the only difference is that the committed row has not expired (the - claimant's upsert therefore observed the holder instead of running the - expiry CAS — either way its statement locks the row until commit). The - skipped renewal must be trusted on the committed row's own validity. - """ - alpha = harness.coordinator("alpha", writer_epoch=1) - alpha.start() - harness.run_until(alpha, "done:startup") - row_at_startup = harness.lease_row() - assert row_at_startup is not None - - harness.advance(UNEXPIRED_ADVANCE_SECONDS) - - claimant = harness.coordinator("claimant", writer_epoch=2) - claimant.start() - harness.run_until(claimant, "claimant.begin:acquire") - harness.run_until(claimant, "claimant.done:acquire") - claim_tx = harness.server.lease_lock_holder - - verify = alpha.submit( - alpha.ledger.verify_writer_lease_guard_session, - label="verify", - ) - harness.run_until(alpha, "done:verify") - - row = harness.lease_row() - assert row is not None - return { - "claim_still_holds_lock": ( - claim_tx is not None - and harness.server.lease_lock_holder is claim_tx - ), - "verify_raised": verify.error is not None, - "result": verify.result, - "committed_row_session": row.writer_session_token, - "committed_row_unrenewed": ( - row.lease_expires_at == row_at_startup.lease_expires_at - ), - "statement_kinds": harness.statement_kinds(), - } - - -def drive_own_write_attribution_recheck(harness: LeaseHarness) -> dict[str, object]: - """Scenario 3: the coordinator's own commit lands inside the probe gap. - - ``alpha``'s deadline-scoped fenced write has executed its statement — - holding the tuple lock, its renewal staged — but has not yet sent - COMMIT. The heartbeat verification's first statement takes its MVCC - snapshot in that window: expired row, renewal skipped. The scenario then - commits the fenced write at the ``guard.snapshot:verify`` checkpoint, - exactly between that snapshot and the statement's live - ``pg_stat_activity`` probe, so the probe finds the locker's - ``backend_xid`` already cleared: an own write that just landed is now - indistinguishable from a competing claim. The verification must resolve - this with one recheck, not a raise. - """ - alpha = harness.coordinator("alpha", writer_epoch=1) - alpha.start() - harness.run_until(alpha, "done:startup") - - harness.advance(EXPIRY_LAPSE_SECONDS) - - def fenced_landing() -> object: - # Renewal under an armed deadline: the smallest shipped statement - # with the fenced-write transaction shape (BEGIN / statement / - # separate COMMIT) that locks and rewrites the lease tuple. - with alpha.ledger.operation_timeout(LANDING_BUDGET_SECONDS): - return alpha.ledger.renew_writer_lease() - - alpha.submit(fenced_landing, label="landing") - harness.run_until(alpha, "alpha.precommit") - landing_tx = harness.server.lease_lock_holder - row_in_snapshot = harness.lease_row() - assert row_in_snapshot is not None - statements_before = len(harness.server.statements) - - # The heartbeat runs on its own thread inside the coordinator process; - # alpha's actor is parked inside the fenced write, so the verification - # needs its own actor to interleave with it. - progress: list[str] = [] - heartbeat = harness.scheduler.actor("heartbeat") - verify = heartbeat.submit( - lambda: alpha.ledger.verify_writer_lease_guard_session( - on_statement_progress=lambda: progress.append("round-trip"), - ), - label="verify", - ) - harness.run_until(heartbeat, "alpha.guard.snapshot:verify") - - # What the first statement's snapshot saw: the documented ambiguous - # inputs, minus the locker attribution the commit is about to spoil. - first_statement_shape = { - "row_expired": row_in_snapshot.lease_expires_at <= harness.clock.now(), - "row_names_this_session": ( - row_in_snapshot.writer_session_token == alpha.session_token - ), - "renewal_lock_blocked": ( - landing_tx is not None - and harness.server.lease_lock_holder is landing_tx - ), - "row_xmax_is_own_write": ( - landing_tx is not None and row_in_snapshot.xmax == landing_tx.xid - ), - } - - # The misfire: the own fenced write commits between the snapshot and the - # pg_stat_activity probe, clearing its backend_xid while the snapshot - # still shows the expired row that write had locked. - harness.run_until(alpha, "done:landing") - harness.run_until(heartbeat, "done:verify") - - row = harness.lease_row() - assert row is not None - return { - "first_statement_shape": first_statement_shape, - "verify_raised": verify.error is not None, - "result": verify.result, - "progress_calls": len(progress), - "verify_statements_run": ( - len(harness.server.statements) - statements_before - ), - "statement_kinds": harness.statement_kinds(), - "lease_holder_session": harness.lease_holder_session(), - "lease_unexpired_after": ( - row.lease_expires_at > harness.clock.now() - ), - } - - -def drive_committed_expiry_claim(harness: LeaseHarness) -> dict[str, object]: - """Scenario 4: the competing claim commits and this session is fenced.""" - alpha = harness.coordinator("alpha", writer_epoch=1) - alpha.start() - harness.run_until(alpha, "done:startup") - - harness.advance(EXPIRY_LAPSE_SECONDS) - - claimant = harness.coordinator("claimant", writer_epoch=2) - claimant.start() - # Run the claimant's startup to completion: its expiry CAS commits and - # the singleton row now names the claimant's session. - harness.run_until(claimant, "done:startup") - - verify = alpha.submit( - alpha.ledger.verify_writer_lease_guard_session, - label="verify", - ) - harness.run_until(alpha, "done:verify") - - return { - "claimant_started": claimant.started, - "verify_raised": verify.error is not None, - "verify_error": str(verify.error), - "lease_holder_session": harness.lease_holder_session(), - "statement_kinds": harness.statement_kinds(), - } - - -class UncontendedRenewalTests(unittest.TestCase): - """Scenario 1: a healthy heartbeat renews without contention.""" - - def test_heartbeat_renews_a_full_ttl_ahead_of_the_clock(self) -> None: - """The committed row, not just the return value, must move. - - The return value alone could report a renewal whose UPDATE matched - zero rows; asserting ``lease_expires_at`` and ``updated_at`` against - the virtual clock proves the renewal landed on the singleton row and - pushed the TTL a full interval ahead of *now*, not ahead of the old - expiry. - """ - with LeaseHarness(lease_ttl_seconds=LEASE_TTL_SECONDS) as harness: - alpha = harness.coordinator("alpha", writer_epoch=1) - alpha.start() - harness.run_until(alpha, "done:startup") - - # Half a TTL of virtual time passes so the renewed expiry is - # visibly different from the acquisition's. - harness.advance(LEASE_TTL_SECONDS / 2) - - result = alpha.call( - alpha.ledger.verify_writer_lease_guard_session, - label="verify", - ) - self.assertEqual( - result, - { - "backend": "postgres-psql", - "verified_count": 1, - "renewed_count": 1, - "renewal_deferred_to_own_write": False, - }, - ) - - row = harness.lease_row() - assert row is not None - self.assertEqual(row.writer_session_token, "heartbeat-v1:alpha-1") - self.assertEqual( - row.lease_expires_at, - harness.clock.now() + timedelta(seconds=LEASE_TTL_SECONDS), - ) - self.assertEqual(row.updated_at, harness.clock.now()) - self.assertIsNone(row.xmax, "the renewal's transaction committed") - - -class ExpiryClaimFailClosedTests(unittest.TestCase): - """Scenario 2: expired + foreign-locked raises; either alone does not.""" - - def test_expired_row_with_foreign_lock_fails_closed(self) -> None: - with LeaseHarness(lease_ttl_seconds=LEASE_TTL_SECONDS) as harness: - observed = drive_expired_lock_blocked_renewal(harness) - - self.assertTrue(observed["claim_took_lock"]) - self.assertTrue( - observed["claim_still_holds_lock"], - "the competing claim must still be in flight when the " - "heartbeat verifies; a committed claim is scenario 4", - ) - self.assertTrue(observed["claim_uncommitted"]) - self.assertTrue(observed["verify_raised"]) - self.assertRegex( - str(observed["verify_error"]), - "expired and its renewal was lock-blocked", - ) - # The stale committed snapshot is exactly what makes the raise - # necessary: it still names this session, so a token read alone - # would have called the fenced-out writer alive. - self.assertEqual( - observed["committed_row_session"], - "heartbeat-v1:alpha-1", - ) - self.assertTrue(observed["committed_row_expired"]) - self.assertTrue(observed["committed_row_unrenewed"]) - # Two verify statements: the ambiguous shape (the claimant's - # backend is not attributable to alpha's pool) earns the one - # recheck, whose fresh snapshot sees the same contended expiry - # and still fails closed — the docstring's residual outcome. - self.assertEqual( - observed["statement_kinds"], - ["acquire", "acquire", "verify", "verify"], - ) - - def test_same_lock_over_an_unexpired_row_verifies_clean(self) -> None: - """The control that pins the raise on the *combination*. - - If the foreign lock alone raised, every heartbeat racing an ordinary - fenced write would kill a healthy coordinator; the non-blocking - renewal is only allowed to distrust a skip once the committed row's - own validity has lapsed. Identical claimant, identical held lock, - unexpired row: the verification must succeed, renewing nothing. - """ - with LeaseHarness(lease_ttl_seconds=LEASE_TTL_SECONDS) as harness: - observed = drive_unexpired_lock_blocked_renewal(harness) - - self.assertTrue(observed["claim_still_holds_lock"]) - self.assertFalse(observed["verify_raised"]) - self.assertEqual( - observed["result"], - { - "backend": "postgres-psql", - "verified_count": 1, - "renewed_count": 0, - "renewal_deferred_to_own_write": False, - }, - ) - self.assertEqual( - observed["committed_row_session"], - "heartbeat-v1:alpha-1", - ) - self.assertTrue( - observed["committed_row_unrenewed"], - "a skipped renewal must leave the committed expiry alone", - ) - # Unexpired means unambiguous: no recheck, one verify statement. - self.assertEqual( - observed["statement_kinds"], - ["acquire", "acquire", "verify"], - ) - - def test_fail_closed_interleaving_is_order_stable(self) -> None: - """A fail-closed path that only usually triggers is a split brain. - - The exact schedule is the assertion: the claim executes and parks - uncommitted before the verification starts, and the verification's - two statements (first read, one recheck) both run against the still - contended row. Any drift in this trace means the scenario stopped - testing the interleaving it claims to test. - """ - run = assert_deterministic( - self, - drive_expired_lock_blocked_renewal, - repeats=25, - lease_ttl_seconds=LEASE_TTL_SECONDS, - ) - - self.assertEqual( - list(run.trace), - [ - "alpha@alpha.begin:acquire", - "alpha@alpha.done:acquire", - # Since #123 the acquire/adopt statements carry an acquisition - # deadline, so each commits explicitly and offers a precommit stop. - "alpha@alpha.precommit", - "alpha@done:startup", - "claimant@claimant.begin:acquire", - "claimant@claimant.done:acquire", - "alpha@alpha.guard.begin:verify", - "alpha@alpha.guard.snapshot:verify", - "alpha@alpha.guard.done:verify", - "alpha@alpha.guard.begin:verify", - "alpha@alpha.guard.snapshot:verify", - "alpha@alpha.guard.done:verify", - "alpha@done:verify", - ], - ) - - -class AttributionRecheckTests(unittest.TestCase): - """Scenario 3: the ambiguous shape earns exactly one fresh-snapshot look.""" - - def test_own_commit_in_probe_window_resolves_by_recheck(self) -> None: - with LeaseHarness(lease_ttl_seconds=LEASE_TTL_SECONDS) as harness: - observed = drive_own_write_attribution_recheck(harness) - - # The first statement really produced the ambiguous shape: the - # snapshot row was expired, still named this session, and its - # locker was this coordinator's own in-flight write — which the - # live probe could no longer attribute after the commit. - self.assertEqual( - observed["first_statement_shape"], - { - "row_expired": True, - "row_names_this_session": True, - "renewal_lock_blocked": True, - "row_xmax_is_own_write": True, - }, - ) - # The verification returned; raising here is the misfire the - # recheck exists to prevent — it would roll back nothing (the - # write already committed) and restart-loop a healthy writer on - # every fenced write slow enough to outlast the TTL. - self.assertFalse(observed["verify_raised"]) - self.assertEqual( - observed["result"], - { - "backend": "postgres-psql", - "verified_count": 1, - "renewed_count": 1, - "renewal_deferred_to_own_write": False, - }, - ) - # WRITER_LEASE_VERIFICATION_MAX_STATEMENTS is 2: exactly one - # recheck ran, and it announced the first statement's completed - # round trip to the liveness monitor exactly once. - self.assertEqual(observed["progress_calls"], 1) - self.assertEqual(observed["verify_statements_run"], 2) - self.assertEqual( - observed["statement_kinds"], - ["acquire", "renew", "verify", "verify"], - ) - # The recheck's fresh snapshot saw the committed refresh and the - # second statement's renewal landed on the uncontended row. - self.assertEqual( - observed["lease_holder_session"], - "heartbeat-v1:alpha-1", - ) - self.assertTrue(observed["lease_unexpired_after"]) - - def test_recheck_interleaving_is_order_stable(self) -> None: - """The misfire window is two checkpoints wide; pin the schedule. - - The own write must commit after ``guard.snapshot:verify`` (the first - statement's MVCC snapshot is taken) and before that statement's live - probe runs. If the trace drifts — the commit landing before the - snapshot, or the recheck not running — the test would silently stop - covering the one-direction misfire the production docstring names. - """ - run = assert_deterministic( - self, - drive_own_write_attribution_recheck, - repeats=25, - lease_ttl_seconds=LEASE_TTL_SECONDS, - ) - - self.assertEqual( - list(run.trace), - [ - "alpha@alpha.begin:acquire", - "alpha@alpha.done:acquire", - "alpha@alpha.precommit", - "alpha@done:startup", - "alpha@alpha.begin:renew", - "alpha@alpha.done:renew", - "alpha@alpha.precommit", - "heartbeat@alpha.guard.begin:verify", - "heartbeat@alpha.guard.snapshot:verify", - "alpha@done:landing", - "heartbeat@alpha.guard.done:verify", - "heartbeat@alpha.guard.begin:verify", - "heartbeat@alpha.guard.snapshot:verify", - "heartbeat@alpha.guard.done:verify", - "heartbeat@done:verify", - ], - ) - - -class CommittedExpiryClaimTests(unittest.TestCase): - """Scenario 4: a claim that wins fences this session outright.""" - - def test_committed_claim_fences_the_session(self) -> None: - """Losing the row must read as fencing, not as a lock-blocked retry. - - Once the claimant's CAS commits, the committed row no longer names - this session, which is conclusive — not the ambiguous shape — so - the verification raises the identity failure in one statement - rather than spending its recheck or, worse, reporting a renewal - against a row it no longer owns. - """ - with LeaseHarness(lease_ttl_seconds=LEASE_TTL_SECONDS) as harness: - observed = drive_committed_expiry_claim(harness) - - self.assertTrue(observed["claimant_started"]) - self.assertEqual( - observed["lease_holder_session"], - "heartbeat-v1:claimant-2", - ) - self.assertTrue(observed["verify_raised"]) - self.assertRegex( - str(observed["verify_error"]), - "writer lease is not active", - ) - self.assertEqual( - observed["statement_kinds"], - ["acquire", "acquire", "verify"], - ) - - -if __name__ == "__main__": - unittest.main() diff --git a/tests/test_prism_lease_takeover.py b/tests/test_prism_lease_takeover.py deleted file mode 100644 index 2f7171c9f..000000000 --- a/tests/test_prism_lease_takeover.py +++ /dev/null @@ -1,746 +0,0 @@ -#!/usr/bin/env python3 -"""Writer-lease takeover: the proof fast adoption owes before it fences a peer. - -Fast adoption is the only path by which one PRISM coordinator takes the -writer lease from a same-identity predecessor *before* the TTL has run out, -and therefore the only path that can hand two live processes overlapping -authority if its proof is wrong. The proof has three independent parts, and -each one fails in a way that is invisible to a single-process test: - -``_initialize_writer_lease_guard`` - The successor must hold the predecessor's PostgreSQL session advisory - lock. Two coordinators sharing ``(writer_id, writer_epoch)`` hash to the - same key, so a live predecessor keeps its successor out of the lease - upsert entirely: the successor never observes the row, never runs a CAS, - and spins. A guard that failed open would not change any *outcome* a - functional test can see -- adoption would still work -- so the only - executable statement of it is that no statement reaches the server. - -``_writer_lease_adoption_wait_seconds`` - Silence is measured from two edges and the later one wins: the lease - row's ``updated_at`` age, and the time since *this* process acquired the - guard. The guard edge is the one that is easy to drop and impossible to - notice, because dropping it is only wrong when the row is *already* - stale: a long fenced transaction such as ``persist_accepted_block`` - withholds ``updated_at`` refreshes, so the row can look minutes silent at - the instant the predecessor dies. A row-only implementation adopts - instantly in exactly that case and gives the dying predecessor none of - its heartbeat failure budget to self-fence. The second scenario here - constructs precisely that state -- a row far staler than the silence - interval -- and requires a full interval of waiting anyway. - -``_try_adopt_writer_lease`` - The CAS is the final database fence. Winning it rewrites the session - token, which fences every later predecessor mutation. Losing it must not - be terminal: the renewal that beat it may have been the predecessor's - final act before dying, so refusing that token permanently would recreate - the whole-TTL outage adoption exists to avoid. The successor must - re-observe and wait a *fresh* full silence interval instead. - -The last scenario covers the other half of the identity rule. Fast adoption -is deliberately unavailable across writer epochs -- ``_can_wait_for_writer_lease`` -requires an exact ``(writer_id, writer_epoch)`` match, and a different epoch -hashes to a different advisory key, so nothing in this process can prove -anything about the holder. A different epoch must therefore refuse loudly -while the lease is live, and take it through the ordinary expiry CAS once it -has lapsed. - -Every scenario is driven through the shipped code: the real retry loop, the -real adoption arithmetic, the real CAS SQL. Nothing is patched; the harness -substitutes PostgreSQL, the clock and the retry sleep at the constructor -seams ``PsqlShareLedger`` already exposes. -""" - -from __future__ import annotations - -import unittest - -from tests.prism_concurrency_harness import ( - Coordinator, - LeaseHarness, - assert_deterministic, -) - -# A long TTL keeps every scenario inside the *unexpired* half of the lease -# lifecycle, which is the only half where adoption is reachable at all: an -# expired row is taken by the ordinary expiry CAS in `_try_acquire_writer_lease` -# and never consults the silence arithmetic. -LEASE_TTL_SECONDS = 600.0 - -# Small and explicit so the waiting arithmetic below is readable as arithmetic -# rather than taken on trust. Production defaults to 1.0s -# (`DEFAULT_WRITER_LEASE_ADOPTION_SILENCE_SECONDS`); the mechanism does not -# depend on the value, only on which edge it is measured from. -ADOPTION_SILENCE_SECONDS = 5.0 - -# How stale the lease row is made before the successor ever starts. Chosen to -# be far larger than the silence interval, so a row-edge-only implementation -# would compute a wait of zero and adopt on its first observation. -PREDECESSOR_SILENT_SECONDS = 120.0 - -# `_lease_retry_min_sleep_seconds` is `min(0.25, lease_retry_max_sleep_seconds)`, -# and the guard spin sleeps exactly that between attempts. -GUARD_SPIN_SLEEP_SECONDS = 0.25 - -# How many times the gated successor is made to go round the guard spin. Any -# number above one works; more than one is the point, because the warning is -# asserted to be printed once for the whole wait rather than once per lap. -GUARD_SPIN_LAPS = 4 - -GUARD_HELD_WARNING = ( - "prism ledger writer guard held by a live same-identity coordinator; " - "waiting before lease acquisition" -) - -HARNESS_KWARGS = { - "lease_ttl_seconds": LEASE_TTL_SECONDS, - "lease_adoption_silence_seconds": ADOPTION_SILENCE_SECONDS, -} - - -def start_predecessor(harness: LeaseHarness) -> Coordinator: - """Bring up the lease holder every scenario takes over from.""" - alpha = harness.coordinator("alpha", writer_epoch=1) - alpha.start() - harness.run_until(alpha, "done:startup") - return alpha - - -def drive_guard_gated_successor(harness: LeaseHarness) -> dict[str, object]: - """Start a same-identity successor while the predecessor is alive. - - ``beta`` shares ``alpha``'s ``(writer_id, writer_epoch)``, so - ``_writer_lease_advisory_lock_key`` hands both the same key and ``beta`` - cannot get past ``_initialize_writer_lease_guard``. It is stepped round - the spin several times to show the wait is a loop rather than a single - stall, and that the loop reaches the server not at all. - """ - alpha = start_predecessor(harness) - statements_before = harness.statement_kinds() - - beta = harness.coordinator("beta", writer_epoch=1) - beta.start() - harness.run_until_blocked(beta, f"sleep:{GUARD_SPIN_SLEEP_SECONDS:g}") - for _ in range(GUARD_SPIN_LAPS - 1): - harness.advance_to_next_deadline() - harness.run_until_blocked(beta, f"sleep:{GUARD_SPIN_SLEEP_SECONDS:g}") - - return { - "statements_before": statements_before, - "statements_after": harness.statement_kinds(), - "successor_stops": [ - entry.split("@", 1)[1] - for entry in harness.trace - if entry.startswith(f"{beta.name}@") - ], - "successor_started": beta.started, - "sleeps": list(harness.sleeps), - "same_advisory_key": ( - beta.guard.advisory_lock_key == alpha.guard.advisory_lock_key - ), - "predecessor_guard_held": alpha.guard.held, - "successor_guard_held": beta.guard.held, - "warning_count": harness.output.count(GUARD_HELD_WARNING), - } - - -def drive_fast_adoption(harness: LeaseHarness) -> dict[str, object]: - """Adopt from a predecessor whose lease row is already long stale. - - The clock is advanced ``PREDECESSOR_SILENT_SECONDS`` *before* the - predecessor's guard is released, which is the shape a long fenced - transaction produces: the row has not been refreshed for minutes, yet the - predecessor only stopped being able to act at the instant the guard went. - Everything the successor may conclude about silence therefore has to be - measured from its own guard acquisition. - """ - alpha = start_predecessor(harness) - harness.advance(PREDECESSOR_SILENT_SECONDS) - alpha.guard.close() - - beta = harness.coordinator("beta", writer_epoch=1) - beta.start() - # The successor takes the guard, observes the row, and finds it cannot - # adopt yet. This is the first moment it could have adopted if silence - # were read off the row alone. - harness.run_until_blocked(beta, f"sleep:{ADOPTION_SILENCE_SECONDS:g}") - observed_row = harness.lease_row() - row_age_at_first_observation = ( - harness.clock.now() - observed_row.updated_at - ).total_seconds() - statements_at_first_observation = harness.statement_kinds() - monotonic_at_first_observation = harness.clock.monotonic() - - harness.advance_to_next_deadline() - harness.run_until(beta, "done:startup") - - adopted_row = harness.lease_row() - return { - "row_age_at_first_observation": row_age_at_first_observation, - "statements_at_first_observation": statements_at_first_observation, - "waited_seconds": ( - harness.clock.monotonic() - monotonic_at_first_observation - ), - "sleeps": list(harness.sleeps), - "statements": harness.statement_kinds(), - "session_token": adopted_row.writer_session_token, - "predecessor_token": alpha.session_token, - "successor_token": beta.session_token, - "lease_expires_at": str(adopted_row.lease_expires_at), - "ttl_ahead_seconds": ( - adopted_row.lease_expires_at - harness.clock.now() - ).total_seconds(), - "adoption_announced": ( - "adopted from same-identity predecessor" in harness.output - ), - } - - -def drive_lost_adoption_cas(harness: LeaseHarness) -> dict[str, object]: - """Land a predecessor renewal between the successor's observation and CAS. - - The interleaving is forced by PostgreSQL's own ordering rather than by - scheduling luck. At ``beta.done:acquire`` the successor's transaction - still holds the lease tuple lock, so the predecessor's renewal queues - behind it; letting the successor run on to its COMMIT releases the lock, - at which point the renewal lands and changes ``updated_at`` -- the exact - column the adoption CAS compares. The successor is then at - ``beta.begin:adopt`` with a stale observation and no way to know it. - - Since #123's fix the observation is deadline-scoped, so that COMMIT is an - explicit message and reaching it takes a ``beta.precommit`` stop and then - a step, where an autocommit statement committed itself in one. The - interleaving is the same one and lands at the same place; only the number - of scheduler steps needed to reach it changed. - """ - alpha = start_predecessor(harness) - harness.advance(PREDECESSOR_SILENT_SECONDS) - alpha.guard.close() - - beta = harness.coordinator("beta", writer_epoch=1) - beta.start() - harness.run_until_blocked(beta, f"sleep:{ADOPTION_SILENCE_SECONDS:g}") - harness.advance_to_next_deadline() - - # Re-observation: adoption is now permitted, and the successor holds the - # tuple lock until this statement's autocommit COMMIT. - harness.run_until(beta, "beta.done:acquire") - renewal = alpha.submit(alpha.ledger.renew_writer_lease, label="renew") - harness.run_until_blocked(alpha, "alpha.lockwait:renew") - - # Commit the observation and carry the successor to the CAS statement, - # which has begun a transaction but not yet taken the lock. - harness.run_until(beta, "beta.precommit") - harness.step(beta) - stop_before_cas = beta.actor.stop - harness.run_until(alpha, "done:renew") - - # The CAS now runs against a row the renewal moved out from under it. - harness.run_until_blocked(beta, f"sleep:{ADOPTION_SILENCE_SECONDS:g}") - statements_after_lost_cas = harness.statement_kinds() - sleeps_after_lost_cas = list(harness.sleeps) - monotonic_after_lost_cas = harness.clock.monotonic() - - harness.advance_to_next_deadline() - harness.run_until(beta, "done:startup") - - adopted_row = harness.lease_row() - return { - "stop_before_cas": stop_before_cas, - "renewal": renewal.value(), - "statements_after_lost_cas": statements_after_lost_cas, - "sleeps_after_lost_cas": sleeps_after_lost_cas, - "waited_after_lost_cas": ( - harness.clock.monotonic() - monotonic_after_lost_cas - ), - "sleeps": list(harness.sleeps), - "statements": harness.statement_kinds(), - "session_token": adopted_row.writer_session_token, - "successor_token": beta.session_token, - } - - -class WriterGuardGatesTakeoverTests(unittest.TestCase): - """A live same-identity predecessor keeps its successor out entirely.""" - - def test_successor_never_reaches_the_lease_upsert(self) -> None: - """Not "does not adopt" -- does not *observe*. - - The advisory guard is the only part of the proof that speaks to what - the predecessor can still do outside the database, so it has to hold - before any lease statement runs, not merely before the CAS. Asserting - that no statement reached the server is the difference between a - guard that gates takeover and one that only delays it. - """ - with LeaseHarness(**HARNESS_KWARGS) as harness: - observed = drive_guard_gated_successor(harness) - - self.assertTrue( - observed["same_advisory_key"], - "two coordinators at the same (writer_id, writer_epoch) must " - "contend for one advisory key, or the guard proves nothing", - ) - self.assertTrue(observed["predecessor_guard_held"]) - self.assertFalse(observed["successor_guard_held"]) - - self.assertEqual( - observed["statements_after"], - observed["statements_before"], - "the gated successor must not have executed any lease " - "statement; it is held off before the upsert, not at it", - ) - self.assertEqual(observed["statements_after"], ["acquire"]) - self.assertNotIn( - "beta.begin:acquire", - observed["successor_stops"], - "reaching begin:acquire would mean the successor observed the " - "lease row while its predecessor was still able to act", - ) - self.assertEqual( - observed["successor_stops"], - [f"sleep:{GUARD_SPIN_SLEEP_SECONDS:g}"] * GUARD_SPIN_LAPS, - ) - self.assertEqual( - observed["sleeps"], - [("beta", GUARD_SPIN_SLEEP_SECONDS)] * GUARD_SPIN_LAPS, - ) - self.assertFalse( - observed["successor_started"], - "PsqlShareLedger.__init__ has not returned", - ) - - def test_guard_contention_is_reported_once_not_once_per_retry(self) -> None: - """The spin is unbounded; the log line must not be. - - ``_initialize_writer_lease_guard`` retries every 0.25s for as long as - the predecessor lives, so a per-attempt warning would emit four lines - a second for the whole of a healthy coordinator's life and bury the - one line an operator needs. The ``warned`` latch is the behaviour, and - a spin of several laps producing one line is the only way to see it. - """ - with LeaseHarness(**HARNESS_KWARGS) as harness: - observed = drive_guard_gated_successor(harness) - - self.assertEqual(len(observed["sleeps"]), GUARD_SPIN_LAPS) - self.assertEqual(observed["warning_count"], 1) - - def test_releasing_the_guard_lets_the_gated_successor_proceed(self) -> None: - """Control: the successor is held by the guard, not by the schedule. - - Without this, the parked state above is equally consistent with a - scenario that simply never gave the successor a chance to run. Closing - the predecessor's guard and doing nothing else must be enough to carry - the same successor all the way through adoption. - """ - with LeaseHarness(**HARNESS_KWARGS) as harness: - drive_guard_gated_successor(harness) - alpha = harness.coordinators["alpha"] - beta = harness.coordinators["beta"] - - alpha.guard.close() - - harness.advance_to_next_deadline() - harness.run_until_blocked( - beta, f"sleep:{ADOPTION_SILENCE_SECONDS:g}" - ) - self.assertTrue( - beta.guard.held, - "the successor took the guard on its next lap", - ) - harness.advance_to_next_deadline() - harness.run_until(beta, "done:startup") - - self.assertTrue(beta.started) - self.assertEqual( - harness.lease_holder_session(), - beta.session_token, - ) - - -class AdoptionSilenceIsMeasuredFromGuardAcquisitionTests(unittest.TestCase): - """The guard edge, not the row edge, is what bounds the wait.""" - - def test_successor_waits_a_full_interval_after_taking_the_guard(self) -> None: - """A row that is already minutes stale buys the successor nothing. - - This is the case the two-edge rule exists for. The predecessor's - ``updated_at`` is ``PREDECESSOR_SILENT_SECONDS`` old -- far past the - silence interval -- when the successor takes the guard, so the row - edge contributes a wait of zero. If the row edge were the only one, - the successor would adopt on its first observation and the dying - predecessor would get none of its heartbeat failure budget to - self-fence. The wait must be a full interval measured from this - process's own guard acquisition instead. - """ - with LeaseHarness(**HARNESS_KWARGS) as harness: - observed = drive_fast_adoption(harness) - - self.assertGreater( - observed["row_age_at_first_observation"], - ADOPTION_SILENCE_SECONDS, - "the row must already be staler than the silence interval, or " - "this scenario is not testing the guard edge at all", - ) - self.assertEqual( - observed["row_age_at_first_observation"], - PREDECESSOR_SILENT_SECONDS, - ) - self.assertEqual( - observed["sleeps"], - [("beta", ADOPTION_SILENCE_SECONDS)], - "exactly one wait, of exactly one full silence interval", - ) - self.assertEqual( - observed["waited_seconds"], - ADOPTION_SILENCE_SECONDS, - ) - - def test_observation_precedes_the_cas_by_a_whole_interval(self) -> None: - """The statement order is the executable form of the arithmetic. - - A wait is only a fence if nothing was written during it. The successor - must observe, wait, re-observe, and only then run the CAS -- so every - statement before the first ``adopt`` is a read-only ``acquire`` - attempt, and there are two of them: one either side of the interval. - """ - with LeaseHarness(**HARNESS_KWARGS) as harness: - observed = drive_fast_adoption(harness) - - self.assertEqual( - observed["statements_at_first_observation"], - ["acquire", "acquire"], - "the predecessor's startup acquire, then the successor's " - "first observation -- and no adopt", - ) - self.assertEqual( - observed["statements"], - ["acquire", "acquire", "acquire", "adopt"], - ) - statements = observed["statements"] - self.assertEqual( - set(statements[: statements.index("adopt")]), - {"acquire"}, - ) - - def test_adoption_schedule_is_order_stable_across_repeated_runs(self) -> None: - """The wait is a property of the schedule, so pin the schedule. - - Everything this class asserts is a claim about *when* statements ran - relative to one another. That claim is only worth as much as the - reproducibility of the interleaving that produced it. - """ - run = assert_deterministic( - self, - drive_fast_adoption, - repeats=25, - **HARNESS_KWARGS, - ) - - self.assertEqual( - list(run.trace), - [ - "alpha@alpha.begin:acquire", - "alpha@alpha.done:acquire", - # Since #123 the acquire/adopt statements carry an acquisition - # deadline, so each commits explicitly and offers a precommit stop. - "alpha@alpha.precommit", - "alpha@done:startup", - "beta@beta.begin:acquire", - "beta@beta.done:acquire", - "beta@beta.precommit", - "beta@sleep:5", - "beta@beta.begin:acquire", - "beta@beta.done:acquire", - "beta@beta.precommit", - "beta@beta.begin:adopt", - "beta@beta.done:adopt", - "beta@beta.precommit", - "beta@done:startup", - ], - ) - - -class AdoptionRewritesTheSessionTokenTests(unittest.TestCase): - """What a won CAS leaves behind, and what it takes away.""" - - def test_adoption_installs_the_successor_for_a_full_ttl(self) -> None: - with LeaseHarness(**HARNESS_KWARGS) as harness: - observed = drive_fast_adoption(harness) - - self.assertEqual( - observed["session_token"], - observed["successor_token"], - ) - self.assertNotEqual( - observed["session_token"], - observed["predecessor_token"], - ) - self.assertEqual( - observed["ttl_ahead_seconds"], - LEASE_TTL_SECONDS, - "adoption installs a full fresh TTL, not the remainder of the " - "predecessor's", - ) - self.assertTrue(observed["adoption_announced"]) - self.assertIn( - "prism ledger writer lease adopted from same-identity " - "predecessor session=heartbeat-v1:alpha-1", - harness.output, - ) - - def test_predecessor_is_fenced_by_the_rewritten_token(self) -> None: - """The CAS is the fence; nothing tells the predecessor it lost. - - A replaced predecessor is still a running process holding a ledger - object that believes it owns the lease. Its renewal must fail the same - way a fenced write does, so it fails fast rather than continuing to - act, and its release must decline to touch a row it no longer owns -- - expiring a successor's live lease would open the window adoption just - closed. - """ - with LeaseHarness(**HARNESS_KWARGS) as harness: - drive_fast_adoption(harness) - alpha = harness.coordinators["alpha"] - beta = harness.coordinators["beta"] - - with self.assertRaisesRegex( - RuntimeError, - "writer lease is not active", - ): - alpha.call(alpha.ledger.renew_writer_lease, label="renew") - - self.assertFalse( - alpha.call(alpha.ledger.release_writer_lease, label="release"), - "a lease already reassigned must not be expired by its " - "predecessor", - ) - self.assertEqual( - harness.lease_holder_session(), - beta.session_token, - ) - row = harness.lease_row() - self.assertGreater(row.lease_expires_at, harness.clock.now()) - self.assertEqual( - harness.statement_kinds(), - ["acquire", "acquire", "acquire", "adopt", "renew", "release"], - "both fenced calls really did reach the server and were " - "refused by the row, not short-circuited in the client", - ) - - -class LostAdoptionCasTests(unittest.TestCase): - """A CAS loss re-observes; it never refuses the token permanently.""" - - def test_lost_cas_re_observes_after_a_fresh_silence_interval(self) -> None: - """Neither give up nor retry immediately -- both are unsafe. - - The renewal that beat the CAS proves the predecessor was alive - moments ago, so retrying straight away would adopt from a process - that had just demonstrated it can still act. Giving up instead would - recreate the whole-TTL outage fast adoption exists to avoid, because - that renewal may equally have been the predecessor's final act. The - comment in ``_ensure_writer_lease`` picks the third option, and this - is it: re-observe, and require a fresh full silence interval -- which - the renewal's own ``updated_at`` now supplies through the row edge. - """ - with LeaseHarness(**HARNESS_KWARGS) as harness: - observed = drive_lost_adoption_cas(harness) - - self.assertEqual(observed["stop_before_cas"], "beta.begin:adopt") - self.assertEqual(observed["renewal"]["renewed_count"], 1) - - self.assertEqual( - observed["statements_after_lost_cas"], - [ - "acquire", # predecessor startup - "acquire", # successor's first observation - "acquire", # successor re-observes after the guard wait - "renew", # predecessor's renewal wins the tuple lock - "adopt", # the CAS, now against a moved updated_at - ], - "the renewal must land between the observation and the CAS, " - "or the CAS was never actually contested", - ) - self.assertEqual( - observed["sleeps_after_lost_cas"], - [ - ("beta", ADOPTION_SILENCE_SECONDS), - ("beta", ADOPTION_SILENCE_SECONDS), - ], - "a second full interval, not a shortened one and not none", - ) - self.assertEqual( - observed["waited_after_lost_cas"], - ADOPTION_SILENCE_SECONDS, - ) - - self.assertEqual( - observed["statements"], - [ - "acquire", - "acquire", - "acquire", - "renew", - "adopt", - "acquire", - "adopt", - ], - "re-observe before the second CAS: adopting on the first " - "observation's token would be adopting on stale evidence", - ) - self.assertEqual( - observed["session_token"], - observed["successor_token"], - "the successor still ends up holding the lease", - ) - - def test_interleaving_is_order_stable_across_repeated_runs(self) -> None: - """A flaky concurrency test is worse than none; prove this one is not.""" - run = assert_deterministic( - self, - drive_lost_adoption_cas, - repeats=25, - **HARNESS_KWARGS, - ) - - self.assertEqual( - list(run.trace), - [ - "alpha@alpha.begin:acquire", - "alpha@alpha.done:acquire", - "alpha@alpha.precommit", - "alpha@done:startup", - "beta@beta.begin:acquire", - "beta@beta.done:acquire", - "beta@beta.precommit", - "beta@sleep:5", - "beta@beta.begin:acquire", - "beta@beta.done:acquire", - "alpha@alpha.begin:renew", - "alpha@alpha.lockwait:renew", - "beta@beta.precommit", - "beta@beta.begin:adopt", - "alpha@alpha.done:renew", - "alpha@done:renew", - "beta@beta.done:adopt", - "beta@beta.precommit", - "beta@sleep:5", - "beta@beta.begin:acquire", - "beta@beta.done:acquire", - "beta@beta.precommit", - "beta@beta.begin:adopt", - "beta@beta.done:adopt", - "beta@beta.precommit", - "beta@done:startup", - ], - ) - - -class WriterEpochFencingTests(unittest.TestCase): - """A different epoch gets the TTL and nothing else.""" - - def test_different_epoch_refuses_an_unexpired_lease(self) -> None: - """Refusing loudly is the only safe answer, and it must be immediate. - - A different epoch hashes to a different advisory key, so it is not - held off by the holder's guard and does reach the lease upsert -- and - having reached it, it can prove nothing at all about whether the - holder is alive. ``_can_wait_for_writer_lease`` therefore requires an - exact identity match, which makes every different-epoch outcome - against a live lease a raise rather than a wait or a CAS. - """ - with LeaseHarness(**HARNESS_KWARGS) as harness: - alpha = start_predecessor(harness) - - gamma = harness.coordinator("gamma", writer_epoch=2) - gamma.start() - harness.run_until(gamma, "done:startup") - - self.assertNotEqual( - gamma.guard.advisory_lock_key, - alpha.guard.advisory_lock_key, - "different epochs must not share an advisory key, or this " - "coordinator would have been gated instead of refused", - ) - self.assertIn( - "gamma@gamma.begin:acquire", - harness.trace, - "the different-epoch guard was acquired and the upsert was " - "reached, so the refusal comes from the identity check and " - "not from guard contention", - ) - # `PsqlShareLedger.__init__` closes what it opened when it raises. - # A refused claimant that kept its advisory lock would gate the - # legitimate holder's own successor for the life of the process. - self.assertFalse(gamma.guard.held) - self.assertFalse( - gamma.started, - "PsqlShareLedger.__init__ must not return", - ) - with self.assertRaisesRegex( - RuntimeError, - "qbit ledger writer lease is held by prism-coordinator epoch=1", - ): - gamma.actor.calls[0].value() - - self.assertEqual( - harness.statement_kinds(), - ["acquire", "acquire"], - "one observation and no adopt: fast adoption is unreachable " - "across epochs", - ) - self.assertEqual(harness.sleeps, []) - self.assertEqual( - harness.lease_holder_session(), - alpha.session_token, - ) - - def test_different_epoch_takes_an_expired_lease_immediately(self) -> None: - """The expiry CAS is the different-epoch path, and it does not wait.""" - with LeaseHarness(**HARNESS_KWARGS) as harness: - alpha = start_predecessor(harness) - - gamma = harness.coordinator("gamma", writer_epoch=2) - gamma.start() - harness.run_until(gamma, "done:startup") - self.assertIsNotNone(gamma.actor.calls[0].error) - - row = harness.lease_row() - remaining = ( - row.lease_expires_at - harness.clock.now() - ).total_seconds() - self.assertEqual(remaining, LEASE_TTL_SECONDS) - harness.advance(remaining + 1.0) - - delta = harness.coordinator("delta", writer_epoch=2) - delta.start() - harness.run_until(delta, "done:startup") - - self.assertTrue(delta.started) - adopted = harness.lease_row() - self.assertEqual(adopted.writer_epoch, 2) - self.assertEqual(adopted.writer_id, alpha.writer_id) - self.assertEqual( - adopted.writer_session_token, - delta.session_token, - ) - self.assertEqual( - harness.statement_kinds(), - ["acquire", "acquire", "acquire"], - "the expiry CAS lands on the first attempt; no adopt " - "statement is involved", - ) - self.assertEqual( - harness.sleeps, - [], - "an expired lease is taken without waiting at all", - ) - self.assertEqual( - adopted.lease_expires_at - harness.clock.now(), - row.lease_expires_at - row.updated_at, - "the claimant installs a full fresh TTL", - ) - - -if __name__ == "__main__": - unittest.main() diff --git a/tests/test_prism_legacy_import_admission_gate.py b/tests/test_prism_legacy_import_admission_gate.py new file mode 100644 index 000000000..2c3cbab23 --- /dev/null +++ b/tests/test_prism_legacy_import_admission_gate.py @@ -0,0 +1,677 @@ +"""The go-live gate for frontends admitting traffic while import-audits runs (#734). + +The hermetic tests put a fake `qbit-prism-server` and a fake `psql` on PATH; +the fake psql answers each of the gate's queries from the environment and logs +its command line. The real-schema test migrates a fresh database with a real +`qbit-prism-server` and runs the gate's SQL against it with the real psql; it +runs only when PRISM_TEST_DATABASE_URL and PRISM_ADMISSION_GATE_SERVER_BIN are +set. +""" +import json +import os +from pathlib import Path +import shutil +import socket +import stat +import subprocess +import tempfile +import textwrap +import threading +import time +import unittest +import urllib.parse +import uuid + + +REPO = Path(__file__).resolve().parents[1] +GATE = REPO / "scripts/prism_legacy_import_admission_gate.sh" +PASSWORD = "s3cret-p@ss" +DSN = "postgres://gate-user:s3cret-p%40ss@db.example.invalid/prism?sslmode=verify-full" +TOKEN = "operator-token-s3cret" +STANDBY_PASSWORD = "st4ndby-pw" +STANDBY_DSN = "postgres://reader:st4ndby-pw@standby.example.invalid/prism?sslmode=verify-full" +STANDBY_URL = "postgres://reader@standby.example.invalid/prism?sslmode=verify-full" +# The writer's and the standby's system identifier and database name, as both probes print them. +ID = "7000000000000000001|prism" +TESTNET_FALLBACK = "tq1zlsq9dpxz8mennhdpr9nf9s0f2tjtq6gxs9m84k6xglhkfp92q2zszzu4m3" +TEST_DATABASE_URL = os.environ.get("PRISM_TEST_DATABASE_URL", "") +SERVER_BIN = os.environ.get("PRISM_ADMISSION_GATE_SERVER_BIN", "") + +FAKE_SERVER = textwrap.dedent("""\ + #!/usr/bin/env python3 + import os, sys + command = sys.argv[1:2] + if command == ["check-config"]: + print("WARNING: block submission is disabled (PRISM_BLOCK_SUBMIT_ENABLED=0)") + if os.environ.get("FAKE_CHECK_CONFIG_FAIL"): + print("Error: PRISM_FEE_PERCENT leaves dust unsettled") + sys.exit(1) + elif command == ["submission-hold"]: + print('{"held": true, "set_by": "drill"}') + elif command == ["healthcheck"]: + if os.environ.get("FAKE_HEALTH_FAIL"): + print("Error: PRISM health is not ready", file=sys.stderr) + sys.exit(1) + else: + sys.exit(f"fake qbit-prism-server: unexpected {sys.argv[1:]}") +""") + +# Answers each query from the environment, so a test changes one fact at a +# time, and logs its command line and PGPASSWORD. pg_stat_activity comes +# first: the import probe's patterns name qbit_pool_audit_bundles. +FAKE_PSQL = textwrap.dedent("""\ + #!/usr/bin/env python3 + import json, os, sys + ID = "7000000000000000001|prism" + args = sys.argv[1:] + query = sys.stdin.read() + with open(os.environ["FAKE_PSQL_LOG"], "a") as log: + log.write(json.dumps({"argv": args, "pgpassword": os.environ.get("PGPASSWORD"), + "query": query}) + "\\n") + failing = os.environ.get("FAKE_PSQL_FAIL") + if failing and failing in query: + sys.exit(os.environ.get("FAKE_PSQL_STDERR") + or f"psql: error: connection to {args[0]} failed") + answers = [ + ("FROM pg_stat_activity", "FAKE_IMPORT", "1|3|0"), + ("FROM qbit_prism_instances", "FAKE_CENSUS", + '2|0|0|0|[{"id": "frontend-a", "age_s": 1.0, "status": "{}"}]'), + ("FROM qbit_pool_audit_bundles", "FAKE_COMPLETENESS", "0|0"), + ("qbit_carry_forward_integrity_report", "FAKE_INTEGRITY", "0|0"), + ("FROM pg_settings", "FAKE_DURABILITY", + "fsync=on full_page_writes=on synchronous_commit=on"), + ("pg_stat_replication", "FAKE_STANDBY", "true|1|1"), + ("FROM qbit_share_ledger", "FAKE_RECENT_MINER", "qb1zrecentminer"), + ("pg_is_in_recovery", "FAKE_RECOVERY", + "true|0/3000148|paused|3|off|30s|30s|" + ID + "|00000001"), + ("synchronous_standby_names", "FAKE_WRITER", "on||0/3000200|" + ID + "|00000001"), + ] + for needle, variable, default in answers: + if needle in query: + print(os.environ.get(variable, default)) + sys.exit(0) + sys.exit(f"fake psql: unexpected query {query!r}") +""") + + +def write_executable(path, text): + path.write_text(text) + path.chmod(path.stat().st_mode | stat.S_IXUSR | stat.S_IXGRP | stat.S_IXOTH) + + +class FakeStratum: + """A highdiff listener that answers subscribe and authorize, then sends the + given raw lines, or keeps sending notifications without a difficulty, and + records the username the gate authorized with.""" + + def __init__(self, difficulty=None, lines=None, chatty=False): + self.lines = lines if lines is not None else [ + json.dumps({"id": None, "method": "mining.set_difficulty", + "params": [difficulty]})] + self.chatty = chatty + self.usernames = [] + self.server = socket.socket() + self.server.bind(("127.0.0.1", 0)) + self.server.listen(1) + self.port = self.server.getsockname()[1] + self.stop = threading.Event() + self.thread = threading.Thread(target=self.serve, daemon=True) + self.thread.start() + + def serve(self): + connection, _ = self.server.accept() + with connection, connection.makefile("rwb") as stream: + try: + for _ in range(2): + request = json.loads(stream.readline()) + if request["method"] == "mining.authorize": + self.usernames.append(request["params"][0]) + stream.write((json.dumps({"id": request["id"], "result": True, + "error": None}) + "\n").encode()) + stream.flush() + for line in self.lines: + stream.write((line + "\n").encode()) + stream.flush() + while self.chatty and not self.stop.wait(1): + stream.write(b'{"id": null, "method": "mining.notify", "params": []}\n') + stream.flush() + except OSError: + pass + + def close(self): + self.stop.set() + self.thread.join(timeout=5) + self.server.close() + + +class AdmissionGateTests(unittest.TestCase): + def setUp(self): + self.tmp = tempfile.TemporaryDirectory() + self.addCleanup(self.tmp.cleanup) + self.bin = Path(self.tmp.name) / "bin" + self.bin.mkdir() + self.log = Path(self.tmp.name) / "psql.log" + write_executable(self.bin / "qbit-prism-server", FAKE_SERVER) + write_executable(self.bin / "psql", FAKE_PSQL) + + def run_gate(self, **env): + base = {key: value for key, value in os.environ.items() + if not key.startswith(("PRISM_", "FAKE_", "PG", "QBIT_"))} + base.update(PATH=f"{self.bin}{os.pathsep}{os.environ['PATH']}", + PRISM_DATABASE_URL=DSN, PRISM_OPERATOR_BEARER_TOKEN=TOKEN, + FAKE_PSQL_LOG=str(self.log)) + base.update({key: value for key, value in env.items() if value is not None}) + for key, value in env.items(): + if value is None: + base.pop(key, None) + run = subprocess.run(["bash", str(GATE)], env=base, capture_output=True, text=True, + timeout=60) + for secret in (PASSWORD, "s3cret-p%40ss", TOKEN, STANDBY_PASSWORD): + self.assertNotIn(secret, run.stdout + run.stderr) + return run + + def psql_calls(self): + if not self.log.exists(): + return [] + return [json.loads(line) for line in self.log.read_text().splitlines()] + + def assert_fails_with(self, line, **env): + run = self.run_gate(**env) + self.assertEqual(run.returncode, 1, run.stdout) + self.assertIn(line, run.stdout) + self.assertIn("ADMISSION GATE: FAIL", run.stdout) + + def test_every_fact_passing_admits(self): + run = self.run_gate() + self.assertEqual(run.returncode, 0, run.stdout) + for line in ("PASS check-config", + "INFO check-config WARNING: block submission is disabled", + "PASS heartbeat census read: [{\"id\": \"frontend-a\"", + "INFO HA: 2 live, 0 stale, 0 inactive, 0 unknown (freshness 15s)", + "INFO submission hold: {\"held\": true", + "INFO audit completeness: 0 legacy rows pending", + "PASS native audit rows complete", + "PASS carry-forward integrity (mismatch 0, drift 0)", + "PASS durability: fsync=on full_page_writes=on synchronous_commit=on", + "INFO offer standby not configured", + "PASS healthcheck (frontend ready on the observed tip)", + "INFO highdiff not configured on this host", + "ADMISSION GATE: PASS"): + self.assertIn(line, run.stdout) + self.assertFalse([line for line in run.stdout.splitlines() if line.startswith("WARN")]) + + def test_the_password_never_reaches_a_command_line(self): + for url, expected in ( + (DSN, "postgres://gate-user@db.example.invalid/prism?sslmode=verify-full"), + ("postgresql://gate-user@db.example.invalid/prism?password=s3cret-p%40ss" + "&sslmode=require", + "postgresql://gate-user@db.example.invalid/prism?sslmode=require")): + with self.subTest(url=url): + self.log.unlink(missing_ok=True) + run = self.run_gate(PRISM_DATABASE_URL=url) + self.assertEqual(run.returncode, 0, run.stdout) + calls = self.psql_calls() + self.assertTrue(calls) + for call in calls: + self.assertEqual(call["argv"][0], expected) + self.assertNotIn(PASSWORD, json.dumps(call["argv"])) + self.assertEqual(call["pgpassword"], PASSWORD) + + def test_a_url_without_a_password_leaves_pgpassword_alone(self): + run = self.run_gate(PRISM_DATABASE_URL="postgres://gate-user@db.example.invalid/prism") + self.assertEqual(run.returncode, 0, run.stdout) + self.assertTrue(all(call["pgpassword"] is None for call in self.psql_calls())) + + def test_a_url_that_is_not_postgres_is_refused(self): + run = self.run_gate(PRISM_DATABASE_URL="mysql://gate-user:pw@db.example.invalid/prism") + self.assertEqual(run.returncode, 2) + self.assertIn("must be a postgres:// or postgresql:// URL", run.stderr) + + def test_pending_legacy_rows_are_reported_not_failed(self): + run = self.run_gate(FAKE_COMPLETENESS="9531|0") + self.assertEqual(run.returncode, 0, run.stdout) + self.assertIn("INFO audit completeness: 9531 legacy rows pending", run.stdout) + self.assertIn("INFO import-audits: 1 session(s), last statement 3s ago", run.stdout) + + def test_a_missing_import_session_is_a_warning(self): + run = self.run_gate(FAKE_COMPLETENESS="9531|0", FAKE_IMPORT="0|-|0") + self.assertEqual(run.returncode, 0, run.stdout) + self.assertIn("WARN no import-audits session found", run.stdout) + run = self.run_gate(FAKE_COMPLETENESS="9531|0", FAKE_IMPORT="0|-|4") + self.assertEqual(run.returncode, 0, run.stdout) + self.assertIn("INFO import-audits progress not visible to this role", run.stdout) + run = self.run_gate(FAKE_COMPLETENESS="9531|0", FAKE_PSQL_FAIL="FROM pg_stat_activity") + self.assertEqual(run.returncode, 0, run.stdout) + self.assertIn("WARN could not read pg_stat_activity", run.stdout) + + def test_the_census_warns_as_self_check_does(self): + run = self.run_gate(FAKE_CENSUS='1|1|0|0|[]') + self.assertEqual(run.returncode, 0, run.stdout) + self.assertIn("INFO HA: 1 live, 1 stale, 0 inactive, 0 unknown", run.stdout) + self.assertIn("WARN Fewer than two live frontends observed", run.stdout) + run = self.run_gate(FAKE_CENSUS='2|0|0|1|[]') + self.assertEqual(run.returncode, 0, run.stdout) + self.assertIn("WARN Unrecognized or future-dated heartbeats; HA is unknown", run.stdout) + run = self.run_gate(PRISM_HEALTH_REFRESH_SECONDS="09") + self.assertEqual(run.returncode, 0, run.stdout + run.stderr) + self.assertIn("(freshness 27s)", run.stdout) + run = self.run_gate(PRISM_HEALTH_REFRESH_SECONDS="10") + self.assertIn("(freshness 30s)", run.stdout) + self.assertTrue(any(["-v", "fresh=30"] == call["argv"][i:i + 2] + for call in self.psql_calls() for i in range(len(call["argv"])))) + + def test_incomplete_native_rows_fail(self): + self.assert_fails_with("FAIL 2 native audit rows are incomplete", + FAKE_COMPLETENESS="5|2") + + def test_a_carry_forward_mismatch_or_drift_fails(self): + self.assert_fails_with("FAIL carry-forward integrity: mismatch_count=1 " + "current_drift_count=0", FAKE_INTEGRITY="1|0") + self.assert_fails_with("current_drift_count=3", FAKE_INTEGRITY="0|3") + + def test_durability_off_fails(self): + self.assert_fails_with( + "FAIL durability: fsync=on full_page_writes=on synchronous_commit=off", + FAKE_DURABILITY="fsync=on full_page_writes=on synchronous_commit=off") + + def test_the_offer_standby_must_be_exactly_one_flushing_readable_standby(self): + run = self.run_gate(PRISM_OFFER_STANDBY_APPLICATION_NAME=" pair-b ") + self.assertEqual(run.returncode, 0, run.stdout) + self.assertIn("PASS offer standby usable", run.stdout) + self.assertTrue(any("app=pair-b" in call["argv"] for call in self.psql_calls())) + for answer in ("true|0|0", "true|2|2", "true|1|0", "false|1|1"): + with self.subTest(answer=answer): + self.assert_fails_with( + f"FAIL offer standby not usable (readable|standbys|flushing = {answer})", + PRISM_OFFER_STANDBY_APPLICATION_NAME="pair-b", FAKE_STANDBY=answer) + + def test_a_zero_flush_wait_turns_the_offer_standby_off(self): + for wait in ("0", " 0 ", "00"): + with self.subTest(wait=wait): + run = self.run_gate(PRISM_OFFER_STANDBY_APPLICATION_NAME="pair-b", + PRISM_OFFER_STANDBY_FLUSH_WAIT_MS=wait, + FAKE_STANDBY="true|0|0") + self.assertEqual(run.returncode, 0, run.stdout) + self.assertIn("INFO offer standby not configured", run.stdout) + self.assert_fails_with("FAIL offer standby not usable", + PRISM_OFFER_STANDBY_APPLICATION_NAME="pair-b", + PRISM_OFFER_STANDBY_FLUSH_WAIT_MS="250", FAKE_STANDBY="true|0|0") + + def test_an_unready_frontend_fails(self): + self.assert_fails_with("FAIL healthcheck: Error: PRISM health is not ready", + FAKE_HEALTH_FAIL="1") + + def test_a_refused_configuration_fails(self): + self.assert_fails_with("FAIL check-config: Error: PRISM_FEE_PERCENT leaves dust", + FAKE_CHECK_CONFIG_FAIL="1") + + def test_a_failed_query_fails_without_showing_the_dsn(self): + for needle, line in ( + ("FROM qbit_prism_instances", "FAIL heartbeat census read failed"), + ("FROM qbit_pool_audit_bundles a", "FAIL audit completeness read failed"), + ("qbit_carry_forward_integrity_report", + "FAIL carry-forward integrity report failed"), + ("FROM pg_settings", "FAIL durability read failed")): + with self.subTest(query=needle): + self.assert_fails_with(line, FAKE_PSQL_FAIL=needle) + self.assert_fails_with("readable|standbys|flushing = error", + PRISM_OFFER_STANDBY_APPLICATION_NAME="pair-b", + FAKE_PSQL_FAIL="pg_stat_replication") + + def test_every_query_runs_under_timeouts(self): + # The integrity report gets 120 s; every other read 15 s; every one a 5 s lock wait. + run = self.run_gate() + self.assertEqual(run.returncode, 0, run.stdout) + calls = self.psql_calls() + self.assertTrue(calls) + for call in calls: + query = call["query"] + self.assertIn("SET default_transaction_read_only = on;", query) + self.assertIn("SET lock_timeout = '5s';", query) + limit = "120s" if "qbit_carry_forward_integrity_report" in query else "15s" + self.assertIn(f"SET statement_timeout = '{limit}';", query) + + def report_calls(self): + return [call for call in self.psql_calls() + if "qbit_carry_forward_integrity_report" in call["query"]] + + def standby_gate(self, **env): + return self.run_gate(PRISM_INTEGRITY_REPORT_DATABASE_URL=STANDBY_DSN, **env) + + def test_the_integrity_report_can_run_on_a_standby(self): + run = self.standby_gate() + self.assertEqual(run.returncode, 0, run.stdout) + self.assertIn("INFO integrity report on a standby: replayed to 0/3000148 on timeline " + "00000001 (writer at 0/3000200), replay state 'paused', last replayed " + "transaction 3s ago", run.stdout) + self.assertIn("PASS carry-forward integrity on the standby (mismatch 0, drift 0)", + run.stdout) + self.assertFalse([line for line in run.stdout.splitlines() if line.startswith("WARN")]) + report = self.report_calls() + self.assertEqual([call["argv"][0] for call in report], [STANDBY_URL]) + self.assertEqual(report[0]["pgpassword"], STANDBY_PASSWORD) + self.assertIn("SET statement_timeout = '600s';", report[0]["query"]) + writer = [call for call in self.psql_calls() if call["argv"][0] != STANDBY_URL] + self.assertTrue(any("synchronous_standby_names" in call["query"] for call in writer)) + self.assertTrue(all(call["pgpassword"] == PASSWORD for call in writer)) + self.assertFalse(any(STANDBY_PASSWORD in argument + for call in self.psql_calls() for argument in call["argv"])) + + def test_a_standby_url_without_a_password_does_not_borrow_the_writers(self): + run = self.run_gate( + PRISM_INTEGRITY_REPORT_DATABASE_URL="postgres://reader@standby.example.invalid/prism") + self.assertEqual(run.returncode, 0, run.stdout) + standby = [call for call in self.psql_calls() + if call["argv"][0] == "postgres://reader@standby.example.invalid/prism"] + self.assertEqual(len(standby), 2) + self.assertTrue(all(call["pgpassword"] is None for call in standby)) + + def test_the_integrity_standby_must_be_fit_for_the_report(self): + for recovery, line in ( + ("false|-|-|-|off|30s|30s|" + ID + "|00000001", + "FAIL PRISM_INTEGRITY_REPORT_DATABASE_URL is not a hot standby " + "(in recovery: false)"), + ("true|0/3000148|paused|3|off|30s|30s|7000000000000000002|prism|00000001", + "FAIL the integrity standby is not a replica of this writer's database"), + ("true|0/3000148|paused|3|off|30s|30s|7000000000000000001|staging|00000001", + "FAIL the integrity standby is not a replica of this writer's database"), + ("true|0/3000148|paused|3|on|30s|30s|" + ID + "|00000001", + "FAIL the integrity standby has hot_standby_feedback on"), + ("true|-|paused|-|off|30s|30s|" + ID + "|00000001", + "FAIL the integrity standby has replayed no transaction yet"), + ("true|0/3000148|paused|900|off|30s|30s|" + ID + "|00000001", + "FAIL the integrity standby's last replayed transaction is 900s old " + "(limit 300s)")): + with self.subTest(recovery=recovery): + self.log.unlink(missing_ok=True) + self.assert_fails_with(line, PRISM_INTEGRITY_REPORT_DATABASE_URL=STANDBY_DSN, + FAKE_RECOVERY=recovery) + self.assertFalse(self.report_calls()) + + def test_a_standby_on_another_timeline_is_refused(self): + # WAL positions on different timelines aren't comparable: after a failover, a standby + # left on the old timeline can even look ahead of the new writer. + self.assert_fails_with("FAIL the integrity standby is on timeline 00000001 but the writer " + "is on 00000002", + PRISM_INTEGRITY_REPORT_DATABASE_URL=STANDBY_DSN, + FAKE_WRITER="on||0/1000000|" + ID + "|00000002", + FAKE_RECOVERY="true|0/9000000|paused|3|off|30s|30s|" + ID + + "|00000001") + self.assertFalse(self.report_calls()) + + def test_the_age_limit_can_be_raised(self): + run = self.standby_gate( + FAKE_RECOVERY="true|0/3000148|paused|900|off|30s|30s|" + ID + "|00000001", + PRISM_INTEGRITY_REPORT_MAX_AGE_SECONDS="1200") + self.assertEqual(run.returncode, 0, run.stdout) + + def test_an_unusable_age_limit_is_refused_whatever_the_route(self): + for value in ("5m", "0", "99999999999999999999"): + with self.subTest(value=value): + run = self.run_gate(PRISM_INTEGRITY_REPORT_MAX_AGE_SECONDS=value) + self.assertEqual(run.returncode, 2, run.stdout) + self.assertIn("PRISM_INTEGRITY_REPORT_MAX_AGE_SECONDS must be a whole number of " + "seconds", run.stderr) + + def test_a_blank_standby_url_counts_as_unset(self): + run = self.run_gate(PRISM_INTEGRITY_REPORT_DATABASE_URL=" ") + self.assertEqual(run.returncode, 0, run.stdout) + self.assertIn("PASS carry-forward integrity (mismatch 0, drift 0)", run.stdout) + + def test_a_standby_that_is_not_yet_paused_is_a_warning(self): + for state in ("not paused", "pause requested"): + with self.subTest(state=state): + run = self.standby_gate( + FAKE_RECOVERY=f"true|0/3000148|{state}|1|off|30s|30s|" + ID + "|00000001") + self.assertEqual(run.returncode, 0, run.stdout) + self.assertIn(f"WARN the integrity standby's replay state is '{state}', " + "not 'paused'", run.stdout) + self.assertIn("PASS carry-forward integrity on the standby", run.stdout) + # Infinite conflict delays are the documented alternative to a pause. + run = self.standby_gate( + FAKE_RECOVERY="true|0/3000148|not paused|1|off|-1|-1|" + ID + "|00000001") + self.assertEqual(run.returncode, 0, run.stdout) + self.assertFalse([line for line in run.stdout.splitlines() if line.startswith("WARN")]) + + def test_a_writer_waiting_for_apply_is_refused(self): + self.assert_fails_with("FAIL the writer waits for standby apply " + "(synchronous_commit=remote_apply)", + PRISM_INTEGRITY_REPORT_DATABASE_URL=STANDBY_DSN, + FAKE_WRITER="remote_apply|FIRST 1 (prism_standby_1)|0/3000200|" + + ID + "|00000001") + self.assertFalse(self.report_calls()) + # Without synchronous standbys, remote_apply waits for nothing. + run = self.standby_gate(FAKE_WRITER="remote_apply||0/3000200|" + ID + "|00000001") + self.assertEqual(run.returncode, 0, run.stdout) + + def test_a_cancelled_report_is_named_without_the_server_details(self): + for stderr, reason in ( + ("ERROR: canceling statement due to conflict with recovery\n" + "DETAIL: User query might have needed to see row versions that must be " + "removed. (standby.example.invalid)", + "(cancelled by a recovery conflict: pause the standby's replay or set " + "max_standby_streaming_delay = -1 there)"), + ("ERROR: canceling statement due to statement timeout " + "(standby.example.invalid)", "(statement timeout)")): + with self.subTest(reason=reason): + run = self.standby_gate(FAKE_PSQL_FAIL="qbit_carry_forward_integrity_report", + FAKE_PSQL_STDERR=stderr) + self.assertEqual(run.returncode, 1, run.stdout) + self.assertIn("FAIL carry-forward integrity report failed on the standby " + + reason, run.stdout) + self.assertNotIn("standby.example.invalid", run.stdout + run.stderr) + timeout = "ERROR: canceling statement due to statement timeout" + self.assert_fails_with("FAIL carry-forward integrity report failed (statement timeout)", + FAKE_PSQL_FAIL="qbit_carry_forward_integrity_report", + FAKE_PSQL_STDERR=timeout) + self.assert_fails_with("FAIL heartbeat census read failed (statement timeout)", + FAKE_PSQL_FAIL="FROM qbit_prism_instances", FAKE_PSQL_STDERR=timeout) + + def test_the_integrity_check_can_be_skipped_and_says_so(self): + run = self.run_gate(PRISM_GATE_INTEGRITY="skip") + self.assertEqual(run.returncode, 0, run.stdout) + self.assertIn("WARN carry-forward integrity skipped (PRISM_GATE_INTEGRITY=skip)", + run.stdout) + self.assertIn("ADMISSION GATE: PASS (legacy audit import pending, reported above) " + "(carry-forward integrity skipped)", run.stdout) + self.assertFalse(self.report_calls()) + run = self.standby_gate(PRISM_GATE_INTEGRITY="skip") + self.assertEqual(run.returncode, 0, run.stdout) + self.assertIn("WARN PRISM_INTEGRITY_REPORT_DATABASE_URL is set but not used", run.stdout) + self.assertFalse(self.report_calls()) + + def test_an_unknown_integrity_mode_fails(self): + self.assert_fails_with("FAIL PRISM_GATE_INTEGRITY must be empty or skip", + PRISM_GATE_INTEGRITY="off") + self.assertFalse(self.report_calls()) + + def test_an_integrity_url_that_is_not_postgres_is_refused(self): + run = self.run_gate(PRISM_INTEGRITY_REPORT_DATABASE_URL="mysql://reader@standby/prism") + self.assertEqual(run.returncode, 2) + self.assertIn("PRISM_INTEGRITY_REPORT_DATABASE_URL must be a postgres:// or " + "postgresql:// URL", run.stderr) + + def test_the_frontend_environment_is_required(self): + run = self.run_gate(PRISM_DATABASE_URL=None) + self.assertNotEqual(run.returncode, 0) + self.assertIn("export the frontend environment first", run.stderr) + self.assertNotIn("ADMISSION GATE", run.stdout) + + def probe(self, listener, **env): + self.addCleanup(listener.close) + run = self.run_gate(PRISM_STRATUM_HIGHDIFF_BIND="127.0.0.1", + PRISM_STRATUM_HIGHDIFF_PORT=str(listener.port), + PRISM_STRATUM_HIGHDIFF_MIN_DIFF="500000", **env) + listener.close() + return run + + def test_the_highdiff_floor_is_probed_as_self_check_probes_it(self): + for difficulty, passes in ((600000, True), (500000, True), (1000, False)): + with self.subTest(difficulty=difficulty): + listener = FakeStratum(difficulty) + run = self.probe(listener, PRISM_SELF_CHECK_ADDRESS="qb1zselfcheck") + self.assertEqual(listener.usernames, ["qb1zselfcheck"]) + if passes: + self.assertEqual(run.returncode, 0, run.stdout) + self.assertIn(f"PASS highdiff first difficulty {float(difficulty)} >= " + "500000.0", run.stdout) + else: + self.assertEqual(run.returncode, 1, run.stdout) + self.assertIn("FAIL highdiff probe: 1000.0 < 500000.0", run.stdout) + + def test_the_highdiff_probe_authorizes_as_self_check_does(self): + cases = ( + ({"PRISM_USERNAME_FALLBACK_ADDRESS": "qb1zfallback", + "PRISM_POOL_FEE_ADDRESS": "qb1zfee"}, "qb1zfallback"), + ({"PRISM_POOL_FEE_ADDRESS": "qb1zfee"}, "qb1zfee"), + ({"QBIT_CHAIN": "testnet4", "PRISM_POOL_FEE_ADDRESS": "qb1zfee"}, TESTNET_FALLBACK), + ({}, "qb1zrecentminer"), + ({"PRISM_SELF_CHECK_ADDRESS": " qb1zspaced ", "PRISM_POOL_FEE_ADDRESS": "qb1zfee"}, + " qb1zspaced "), + ({"PRISM_SELF_CHECK_ADDRESS": " ", "PRISM_USERNAME_FALLBACK_ADDRESS": " "}, + "qb1zrecentminer"), + ) + for env, username in cases: + with self.subTest(env=env): + listener = FakeStratum(600000) + run = self.probe(listener, **env) + self.assertEqual(run.returncode, 0, run.stdout) + self.assertEqual(listener.usernames, [username]) + + def test_an_empty_pool_without_an_address_fails_as_self_check_does(self): + listener = FakeStratum(600000) + run = self.probe(listener, FAKE_RECENT_MINER="") + self.assertEqual(run.returncode, 1, run.stdout) + self.assertIn("FAIL highdiff probe: set PRISM_SELF_CHECK_ADDRESS", run.stdout) + + def test_highdiff_is_on_only_with_a_port(self): + run = self.run_gate(PRISM_STRATUM_HIGHDIFF_BIND="0.0.0.0", + PRISM_STRATUM_HIGHDIFF_PORT=" ") + self.assertEqual(run.returncode, 0, run.stdout) + self.assertIn("INFO highdiff not configured on this host", run.stdout) + + def test_the_probe_refuses_what_self_check_refuses(self): + for name, line in ( + ("a string difficulty", + '{"id": null, "method": "mining.set_difficulty", "params": ["600000"]}'), + ("an infinite difficulty", + '{"id": null, "method": "mining.set_difficulty", "params": [Infinity]}'), + ("a zero difficulty", + '{"id": null, "method": "mining.set_difficulty", "params": [0]}')): + with self.subTest(name): + run = self.probe(FakeStratum(lines=[line]), PRISM_SELF_CHECK_ADDRESS="qb1z") + self.assertEqual(run.returncode, 1, run.stdout) + self.assertIn("FAIL highdiff probe: invalid advertised difficulty", run.stdout) + run = self.probe(FakeStratum(lines=['{"id": 3, "result": null, "error": false}']), + PRISM_SELF_CHECK_ADDRESS="qb1z") + self.assertEqual(run.returncode, 1, run.stdout) + self.assertIn("FAIL highdiff probe: Stratum probe rejected: False", run.stdout) + + def test_the_probe_gives_up_after_15_seconds_in_total(self): + started = time.monotonic() + run = self.probe(FakeStratum(lines=[], chatty=True), PRISM_SELF_CHECK_ADDRESS="qb1z") + self.assertLess(time.monotonic() - started, 30) + self.assertEqual(run.returncode, 1, run.stdout) + self.assertIn("FAIL highdiff probe: Stratum difficulty probe timed out", run.stdout) + + +class RealSchemaTests(unittest.TestCase): + """The gate's SQL against a database `qbit-prism-server migrate` created.""" + + @classmethod + def setUpClass(cls): + if TEST_DATABASE_URL and SERVER_BIN: + return + # Like the gated Rust tests: where integration is required, a missing input fails. + if os.environ.get("PRISM_TEST_REQUIRE_INTEGRATION") == "1": + raise AssertionError("PRISM_TEST_REQUIRE_INTEGRATION=1 but PRISM_TEST_DATABASE_URL " + "or PRISM_ADMISSION_GATE_SERVER_BIN is unset") + raise unittest.SkipTest("set PRISM_TEST_DATABASE_URL and PRISM_ADMISSION_GATE_SERVER_BIN") + + def setUp(self): + self.tmp = tempfile.TemporaryDirectory() + self.addCleanup(self.tmp.cleanup) + self.bin = Path(self.tmp.name) / "bin" + self.bin.mkdir() + write_executable(self.bin / "qbit-prism-server", FAKE_SERVER) + pg_bin = os.environ.get("PRISM_TEST_PG_BIN_DIR") + self.psql = str(Path(pg_bin) / "psql") if pg_bin else shutil.which("psql") + self.assertIsNotNone(self.psql, "psql is required") + name = f"prism_admission_gate_{uuid.uuid4().hex[:12]}" + self.sql(TEST_DATABASE_URL, f"CREATE DATABASE {name}") + self.addCleanup(self.sql, TEST_DATABASE_URL, f"DROP DATABASE IF EXISTS {name}") + parts = urllib.parse.urlsplit(TEST_DATABASE_URL) + self.url = urllib.parse.urlunsplit(parts._replace(path=f"/{name}")) + migrate = subprocess.run([SERVER_BIN, "migrate"], capture_output=True, text=True, + env={"PATH": os.environ["PATH"], + "HOME": os.environ.get("HOME", "/"), + "PRISM_DATABASE_URL": self.url}, timeout=600) + self.assertEqual(migrate.returncode, 0, migrate.stderr) + + def sql(self, url, statement): + run = subprocess.run([self.psql, url, "-XAtq", "-v", "ON_ERROR_STOP=1", + "-c", "SET session_replication_role = replica", + "-c", statement], capture_output=True, text=True, timeout=120) + self.assertEqual(run.returncode, 0, run.stderr) + + def run_gate(self, **extra): + env = {key: value for key, value in os.environ.items() if not key.startswith("PRISM_")} + env.update(PATH=f"{self.bin}{os.pathsep}{Path(self.psql).parent}{os.pathsep}" + f"{os.environ['PATH']}", + PRISM_DATABASE_URL=self.url, **extra) + return subprocess.run(["bash", str(GATE)], env=env, capture_output=True, text=True, + timeout=120) + + def test_the_gate_reads_a_migrated_schema(self): + # The body-present CHECK needs audit_bundle or body_uri; replica role + # skips the block foreign key these bare rows would need. + self.sql(self.url, "INSERT INTO qbit_pool_audit_bundles" + "(block_hash, audit_bundle_sha256, coinbase_tx_hex, body_uri) VALUES " + "(repeat('a', 64), repeat('1', 64), '00', '/audit/a.json'), " + "(repeat('b', 64), repeat('2', 64), '00', '/audit/b.json')") + self.sql(self.url, "INSERT INTO qbit_prism_instances(instance_id, status) VALUES " + "('frontend-a', '{\"schema\": \"qbit.prism.audit-health.v1\", " + "\"ready\": true}'), ('frontend-b', '{\"state\": \"starting\", " + "\"candidate_offer_lifecycle\": 1}')") + run = self.run_gate() + self.assertEqual(run.returncode, 0, run.stdout + run.stderr) + for line in ("PASS heartbeat census read: [{\"id\": \"frontend-a\"", + "INFO HA: 1 live, 0 stale, 1 inactive, 0 unknown (freshness 15s)", + "WARN Fewer than two live frontends observed", + "INFO audit completeness: 2 legacy rows pending", + "WARN no import-audits session found", + "PASS native audit rows complete", + "PASS carry-forward integrity (mismatch 0, drift 0)", + "PASS durability: fsync=on full_page_writes=on", + "ADMISSION GATE: PASS"): + self.assertIn(line, run.stdout) + + # A native row whose snapshot is missing, and one with a snapshot but + # no object body: self-check counts both as incomplete. + self.sql(self.url, "INSERT INTO qbit_prism_audit_snapshots" + "(snapshot_sha256, first_share_seq, last_share_seq, anchor_ms, " + "share_count) VALUES (repeat('e', 64), 1, 1, 0, 1)") + self.sql(self.url, "INSERT INTO qbit_pool_audit_bundles" + "(block_hash, audit_bundle_sha256, coinbase_tx_hex, audit_bundle, " + "body_uri, share_snapshot_sha256) VALUES " + "(repeat('c', 64), repeat('3', 64), '00', '{}', NULL, repeat('f', 64)), " + "(repeat('d', 64), repeat('4', 64), '00', NULL, '/audit/d.json', " + "repeat('e', 64))") + run = self.run_gate() + self.assertEqual(run.returncode, 1, run.stdout + run.stderr) + self.assertIn("FAIL 2 native audit rows are incomplete", run.stdout) + self.assertIn("INFO audit completeness: 2 legacy rows pending", run.stdout) + + def test_a_primary_is_refused_as_the_integrity_standby(self): + # The recovery probe must run on a primary too: pg_get_wal_replay_pause_state() raises + # there, so it sits behind pg_is_in_recovery(), and the writer-position comparison and + # the identity read must parse. + run = self.run_gate(PRISM_INTEGRITY_REPORT_DATABASE_URL=self.url) + self.assertEqual(run.returncode, 1, run.stdout + run.stderr) + self.assertIn("FAIL PRISM_INTEGRITY_REPORT_DATABASE_URL is not a hot standby " + "(in recovery: false)", run.stdout) + self.assertNotIn("carry-forward integrity (mismatch", run.stdout) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_prism_legacy_parts_digest_check.py b/tests/test_prism_legacy_parts_digest_check.py new file mode 100644 index 000000000..79a033f95 --- /dev/null +++ b/tests/test_prism_legacy_parts_digest_check.py @@ -0,0 +1,572 @@ +"""List the legacy audit rows import-audits refuses for 2.x's window-proof parts +digest, and write their sidecars in shards (#731).""" +import contextlib +import gzip +import hashlib +import importlib.util +import io +import json +import os +from pathlib import Path +import shutil +import stat +import subprocess +import sys +import tempfile +import textwrap +import unittest + + +REPO = Path(__file__).resolve().parents[1] +SCRIPT = REPO / "scripts/prism_legacy_parts_digest_check.py" +SIDECAR_TOOL = REPO / "scripts/prism_legacy_range_sidecars.py" +SHARDED = REPO / "scripts/prism_legacy_range_sidecars_sharded.sh" +FIXTURE = REPO / "tests/fixtures/prism-legacy-v2-audit-body" +spec = importlib.util.spec_from_file_location("legacy_parts_digest_check", SCRIPT) +module = importlib.util.module_from_spec(spec) +spec.loader.exec_module(module) +tool_spec = importlib.util.spec_from_file_location("legacy_range_sidecars", SIDECAR_TOOL) +tool = importlib.util.module_from_spec(tool_spec) +tool_spec.loader.exec_module(tool) + +PREFIX = "/var/lib/qbit-mining-pool/prism/audit" +SEGMENT = "qbit.prism.audit-share-segment.v1" +BODY_REF = "qbit.prism.audit-body-ref.v1" +V2 = "qbit.prism.audit-bundle.v2" +PROOF = "qbit.prism.window-completeness-proof.v1" +HEADER = "block_hash,body_uri,audit_bundle_sha256" +SLOT = "prism-audit-share-segment-slot-1-8.json" +CANONICALIZE_BIN = os.environ.get("PRISM_AUDIT_CANONICALIZE_BIN", "") +# The sharded runner splits with GNU split's round robin and resolves paths +# with GNU realpath -m; BSD split has no --version. +GNU_COREUTILS = shutil.which("split") is not None and subprocess.run( + ["split", "--version"], capture_output=True).returncode == 0 + +# The 3.x canonicalizer's contract, for the shapes these tests write: refuse a +# v2 body whose parts digest is not the sorted-key form 3.x recomputes, resolve +# the share parts, check each range digest as serde_json writes it, require +# sha256(canonical bundle) == audit_bundle_sha256, and print the canonical +# bytes. Its canonical form is the bundle as compact JSON. +STUB = textwrap.dedent("""\ + import hashlib, json, sys + from pathlib import Path + + def serde(value, sort_keys=False): + return json.dumps(value, separators=(",", ":"), ensure_ascii=False, + sort_keys=sort_keys).encode() + + path = Path(sys.argv[sys.argv.index("--input") + 1]) + body = json.loads(path.read_bytes()) + if body["schema"] == "qbit.prism.audit-body-ref.v1": + parts = body["share_parts"] + else: + proof = body["share_window_proof"] + parts = proof["share_parts"] + expected = proof.get("share_parts_digest_hex") + if expected and hashlib.sha256(serde({"share_parts": parts}, True)).hexdigest() != expected: + sys.exit("qbit-prism-audit-canonicalize: invalid audit body ref: " + "window proof share_parts_digest_hex mismatch") + shares = [] + for part in parts: + if part["kind"] == "inline": + shares += part["shares"] + continue + slot = Path(part["body_uri"]) + slot = slot if slot.is_absolute() else path.parent / slot + segment = json.loads(slot.read_bytes()) + first, last = part["first_share_seq"], part["last_share_seq"] + selected = [s for s in segment["shares"] if first <= s["share_seq"] <= last] + payload = {"schema": "qbit.prism.audit-share-segment.v1", "first_share_seq": first, + "last_share_seq": last, "share_count": len(selected), "shares": selected} + if hashlib.sha256(serde(payload)).hexdigest() != part["range_sha256"]: + sys.exit("qbit-prism-audit-canonicalize: audit body ref hash mismatch") + shares += selected + bundle = dict(body["bundle_without_shares"]) + bundle["shares"] = shares + canonical = serde(bundle) + if hashlib.sha256(canonical).hexdigest() != body["audit_bundle_sha256"]: + sys.exit("qbit-prism-audit-canonicalize: audit body ref hash mismatch") + sys.stdout.buffer.write(canonical) +""") + + +def compact(value, ascii_only=True): + return json.dumps(value, separators=(",", ":"), ensure_ascii=ascii_only).encode() + + +def digest_as_2x_wrote_it(parts): + """2.x's share_parts_digest_hex: insertion order, ASCII-escaped.""" + return hashlib.sha256(compact({"share_parts": parts})).hexdigest() + + +def digest_as_3x_recomputes_it(parts): + return hashlib.sha256(json.dumps({"share_parts": parts}, separators=(",", ":"), + sort_keys=True, ensure_ascii=False).encode()).hexdigest() + + +def share(seq): + return { + "share_seq": seq, "share_id": f"{seq:064x}", + "miner_id": f"qb1zexample.rig{seq:02d}", "order_key": "qb1zexample", + "p2mr_program_hex": "ab" * 32, "share_difficulty": 2 ** 40 + seq, + "network_difficulty": 2 ** 42, "template_height": 53313, "job_id": f"job-{seq}", + "job_issued_at_ms": 1755815340000 + seq, "accepted_at_ms": 1755815341000 + seq, + "ntime": 1755815340, + } + + +def write_stub(directory): + canonicalizer = Path(directory) / "qbit-prism-audit-canonicalize" + canonicalizer.write_text(f"#!{sys.executable}\n" + STUB) + canonicalizer.chmod(canonicalizer.stat().st_mode | stat.S_IXUSR) + return canonicalizer + + +class Tree: + """A restored audit tree of 2.x v2 bodies without sidecars, and an empty + import root. Each block's body has the same six shares in one slot.""" + + def __init__(self, root, *, blocks=1): + self.root = Path(root) + self.audit = self.root / "audit" + self.audit.mkdir(parents=True) + self.import_root = self.root / "import-root" + self.import_root.mkdir() + self.shares = [share(seq) for seq in range(1, 7)] + self.bundle_without_shares = {"found_block": {"block_height": 53313}, "schema": "v1.1"} + bundle = dict(self.bundle_without_shares) + bundle["shares"] = self.shares + self.canonical = compact(bundle, False) + self.digest = hashlib.sha256(self.canonical).hexdigest() + slot = {"schema": SEGMENT, "first_share_seq": 1, "last_share_seq": 8, + "share_count": len(self.shares), "shares": self.shares} + (self.audit / SLOT).write_bytes(compact(slot)) + self.parts = [self.range_part(1, 4), self.range_part(5, 6)] + self.blocks = [f"{index + 1:02x}" * 32 for index in range(blocks)] + for block in self.blocks: + self.write_body(block, digest_as_2x_wrote_it(self.parts)) + self.rows = self.root / "rows.csv" + self.write_rows() + + def range_part(self, first, last): + """A segment_range part with 2.x's keys, in 2.x's order.""" + selected = self.shares[first - 1:last] + payload = {"schema": SEGMENT, "first_share_seq": first, "last_share_seq": last, + "share_count": len(selected), "shares": selected} + return {"kind": "segment_range", "segment_first_share_seq": 1, + "segment_last_share_seq": 8, "first_share_seq": first, "last_share_seq": last, + "share_count": len(selected), "range_sha256": hashlib.sha256(compact(payload)).hexdigest(), + "body_uri": f"{PREFIX}/{SLOT}"} + + def body_name(self, block): + return f"prism-audit-bundle-body-{block}-{self.digest}.json" + + def body_uri(self, block): + return f"{PREFIX}/{self.body_name(block)}" + + def write_body(self, block, parts_digest, *, parts=None, schema=V2, digest=None, where=None): + proof = {"schema": PROOF, "share_segment_size": 8, "first_share_seq": 1, + "last_share_seq": len(self.shares), "share_count": len(self.shares)} + if parts_digest is not None: + proof["share_parts_digest_hex"] = parts_digest + proof["share_parts"] = self.parts if parts is None else parts + body = {"schema": schema, "audit_bundle_sha256": digest or self.digest, + "share_count": len(self.shares), "bundle_without_shares": self.bundle_without_shares, + "share_window_proof": proof} + path = (where or self.audit) / self.body_name(block) + path.write_bytes(compact(body)) + return path + + def write_rows(self, rows=None): + rows = rows if rows is not None else [ + (block, self.body_uri(block), self.digest) for block in self.blocks] + self.rows.write_text(HEADER + "\n" + "".join(f"{b},{u},{d}\n" for b, u, d in rows)) + + def sidecar(self, block): + return self.import_root / f"prism-audit-bundle-canonical-{block}-{self.digest}.json.gz" + + +def run_checker(argv): + stdout, stderr = io.StringIO(), io.StringIO() + with contextlib.redirect_stdout(stdout), contextlib.redirect_stderr(stderr): + code = module.main([str(arg) for arg in argv]) + rows = [json.loads(line) for line in stdout.getvalue().splitlines()] + return code, rows, json.loads(stderr.getvalue()) + + +def check(tree, *extra): + return run_checker(["--rows", tree.rows, "--audit-root", tree.audit, + "--sidecar-dir", tree.import_root, *extra]) + + +class PartsDigestCheckTests(unittest.TestCase): + def setUp(self): + self.tmp = tempfile.TemporaryDirectory() + self.addCleanup(self.tmp.cleanup) + self.dir = Path(self.tmp.name) + + def test_a_2x_body_without_a_sidecar_is_refused_and_listed(self): + tree = Tree(self.dir, blocks=2) + affected = self.dir / "affected.csv" + code, rows, summary = check(tree, "--affected-csv", affected) + self.assertEqual(code, 3) + self.assertEqual([row["status"] for row in rows], ["2x-order-only"] * 2) + self.assertEqual([(row["parts"], row["has_inline"]) for row in rows], [(2, False)] * 2) + self.assertEqual(summary, {"import_refuses": 2, "totals": {"2x-order-only": 2}}) + # The refused rows, in the sidecar tool's --rows format. + self.assertEqual(affected.read_text(), tree.rows.read_text()) + + def test_a_sidecar_in_the_import_root_means_import_never_reads_the_body(self): + tree = Tree(self.dir, blocks=2) + (tree.audit / tree.body_name(tree.blocks[0])).unlink() + tree.sidecar(tree.blocks[0]).write_bytes(b"") + code, rows, summary = check(tree) + self.assertEqual(code, 3) + self.assertEqual([row["status"] for row in rows], ["has-sidecar", "2x-order-only"]) + self.assertEqual(rows[0]["sidecar"], str(tree.sidecar(tree.blocks[0]))) + self.assertEqual(summary["import_refuses"], 1) + tree.sidecar(tree.blocks[1]).write_bytes(b"") + code, rows, summary = check(tree) + self.assertEqual((code, summary["import_refuses"]), (0, 0)) + + def test_a_digest_in_the_form_3x_recomputes_passes(self): + tree = Tree(self.dir) + tree.write_body(tree.blocks[0], digest_as_3x_recomputes_it(tree.parts)) + code, [row], summary = check(tree) + self.assertEqual((code, row["status"]), (0, "ok-in-3x")) + self.assertEqual(summary, {"import_refuses": 0, "totals": {"ok-in-3x": 1}}) + + def test_a_digest_in_neither_form_is_refused_and_listed(self): + tree = Tree(self.dir) + tree.write_body(tree.blocks[0], "00" * 32) + affected = self.dir / "affected.csv" + code, [row], summary = check(tree, "--affected-csv", affected) + self.assertEqual((code, row["status"]), (3, "matches-neither")) + self.assertEqual(summary["import_refuses"], 1) + self.assertEqual(affected.read_text(), tree.rows.read_text()) + + def test_unknown_keys_and_absent_segment_bounds_are_dropped_as_3x_drops_them(self): + tree = Tree(self.dir) + parts = [dict(tree.parts[0], note="2.x kept it, 3.x has no such field"), + dict(tree.parts[1], segment_first_share_seq=None, segment_last_share_seq=None)] + seen_by_3x = [{key: value for key, value in part.items() + if key != "note" and value is not None} for part in parts] + tree.write_body(tree.blocks[0], digest_as_3x_recomputes_it(seen_by_3x), parts=parts) + code, [row], _ = check(tree) + self.assertEqual((code, row["status"]), (0, "ok-in-3x")) + tree.write_body(tree.blocks[0], digest_as_2x_wrote_it(parts), parts=parts) + code, [row], _ = check(tree) + self.assertEqual((code, row["status"]), (3, "2x-order-only")) + + def test_an_empty_parts_digest_is_checked_as_3x_checks_it(self): + # 3.x reads "" as Some(""), compares it and refuses the body. + tree = Tree(self.dir) + tree.write_body(tree.blocks[0], "") + code, [row], summary = check(tree) + self.assertEqual((code, row["status"], summary["import_refuses"]), (3, "matches-neither", 1)) + + def test_a_body_ref_for_another_digest_is_a_body_problem(self): + tree = Tree(self.dir) + tree.write_body(tree.blocks[0], "00" * 32, schema=BODY_REF, digest="11" * 32) + code, [row], summary = check(tree) + self.assertEqual((code, row["status"], summary["import_refuses"]), (3, "body-digest-differs", 0)) + + def test_one_malformed_body_does_not_end_the_run(self): + tree = Tree(self.dir, blocks=2) + parts = [dict(tree.parts[0], kind=["segment_range"]), tree.parts[1]] + tree.write_body(tree.blocks[0], digest_as_2x_wrote_it(parts), parts=parts) + affected = self.dir / "affected.csv" + code, rows, summary = check(tree, "--affected-csv", affected) + self.assertEqual(code, 3) + self.assertEqual([row["status"] for row in rows], ["body-unreadable", "2x-order-only"]) + self.assertIn("TypeError", rows[0]["error"]) + self.assertEqual(summary["import_refuses"], 1) + self.assertEqual(len(affected.read_text().splitlines()), 2) + + def test_a_v2_body_without_the_digest_is_not_checked(self): + tree = Tree(self.dir) + tree.write_body(tree.blocks[0], None) + code, [row], _ = check(tree) + self.assertEqual((code, row["status"]), (0, "no-parts-digest")) + + def test_a_body_ref_or_a_plain_bundle_is_not_checked(self): + tree = Tree(self.dir, blocks=2) + tree.write_body(tree.blocks[0], "00" * 32, schema=BODY_REF) + bundle = dict(tree.bundle_without_shares, shares=tree.shares) + (tree.audit / tree.body_name(tree.blocks[1])).write_bytes(compact(bundle)) + code, rows, summary = check(tree) + self.assertEqual(code, 0) + self.assertEqual([(row["status"], row["schema"]) for row in rows], + [("no-window-proof", BODY_REF), ("no-window-proof", "v1.1")]) + self.assertEqual(summary["import_refuses"], 0) + + def test_body_problems_fail_the_run_but_are_not_listed_for_the_sidecar_tool(self): + tree = Tree(self.dir, blocks=4) + missing, garbled, malformed, other_digest = tree.blocks + (tree.audit / tree.body_name(missing)).unlink() + (tree.audit / tree.body_name(garbled)).write_bytes(b"{not json") + tree.write_body(malformed, "00" * 32, parts="not a list") + tree.write_body(other_digest, digest_as_2x_wrote_it(tree.parts), digest="11" * 32) + tree.write_rows([(block, tree.body_uri(block), tree.digest) for block in tree.blocks] + + [("ff" * 32, "", tree.digest)]) + affected = self.dir / "affected.csv" + code, rows, summary = check(tree, "--affected-csv", affected) + self.assertEqual(code, 3) + self.assertEqual([row["status"] for row in rows], + ["body-missing", "body-unreadable", "body-unreadable", + "body-digest-differs", "body-missing"]) + self.assertIn("JSONDecodeError", rows[1]["error"]) + self.assertEqual(rows[2]["error"], "share_window_proof.share_parts is not a list of objects") + self.assertEqual(summary["import_refuses"], 0) + self.assertEqual(affected.read_text(), HEADER + "\n") + + def test_the_uri_prefix_maps_onto_the_audit_root_and_relative_uris_onto_the_import_root(self): + tree = Tree(self.dir, blocks=4) + prefixed, file_uri, relative, absolute = tree.blocks + # Import resolves a relative URI against its --root, the sidecar dir. + (tree.audit / tree.body_name(relative)).rename(tree.import_root / tree.body_name(relative)) + elsewhere = self.dir / "elsewhere" + elsewhere.mkdir() + (tree.audit / tree.body_name(absolute)).rename(elsewhere / tree.body_name(absolute)) + tree.write_rows([ + (prefixed, tree.body_uri(prefixed), tree.digest), + (file_uri, "file://" + tree.body_uri(file_uri), tree.digest), + (relative, tree.body_name(relative), tree.digest), + (absolute, str(elsewhere / tree.body_name(absolute)), tree.digest), + ]) + code, rows, _ = check(tree) + self.assertEqual([row["status"] for row in rows], ["2x-order-only"] * 4) + self.assertEqual([row["body"] for row in rows], [ + str(tree.audit / tree.body_name(prefixed)), str(tree.audit / tree.body_name(file_uri)), + str(tree.import_root / tree.body_name(relative)), + str(elsewhere / tree.body_name(absolute))]) + # Without --sidecar-dir a relative URI resolves against the audit root. + code, rows, _ = run_checker(["--rows", tree.rows, "--audit-root", tree.audit]) + self.assertEqual(rows[2]["status"], "body-missing") + self.assertEqual(rows[2]["body"], str(tree.audit / tree.body_name(relative))) + + def test_the_csv_alias_and_body_file_arguments(self): + tree = Tree(self.dir) + by_rows = check(tree) + by_csv = run_checker(["--csv", tree.rows, "--audit-root", tree.audit, + "--sidecar-dir", tree.import_root]) + self.assertEqual(by_rows, by_csv) + body = tree.audit / tree.body_name(tree.blocks[0]) + code, [row], _ = run_checker([body]) + self.assertEqual((code, row["status"], row["body"]), (3, "2x-order-only", str(body))) + self.assertNotIn("block_hash", row) + + def test_bad_arguments_are_refused(self): + tree = Tree(self.dir) + tree.rows.write_text("block_hash,body_uri\n") + for argv in (["--rows", tree.rows], [], ["--jobs", "0", tree.rows]): + with self.subTest(argv=argv), contextlib.redirect_stderr(io.StringIO()): + with self.assertRaises(SystemExit) as raised: + module.main([str(arg) for arg in argv]) + self.assertEqual(raised.exception.code, 2) + + def test_parallel_jobs_report_exactly_what_one_process_does(self): + # Run as a script: a pool's workers import the checker by its path. + tree = Tree(self.dir, blocks=6) + tree.write_body(tree.blocks[1], digest_as_3x_recomputes_it(tree.parts)) + tree.write_body(tree.blocks[2], "00" * 32) + tree.sidecar(tree.blocks[3]).write_bytes(b"") + (tree.audit / tree.body_name(tree.blocks[4])).unlink() + runs = [] + for jobs in ("1", "3"): + affected = self.dir / f"affected-{jobs}.csv" + run = subprocess.run( + [sys.executable, str(SCRIPT), "--rows", str(tree.rows), "--audit-root", + str(tree.audit), "--sidecar-dir", str(tree.import_root), "--jobs", jobs, + "--affected-csv", str(affected)], capture_output=True, text=True) + runs.append((run.returncode, run.stdout, run.stderr, affected.read_text())) + self.assertEqual(runs[0], runs[1]) + self.assertEqual(runs[0][0], 3) + self.assertEqual([json.loads(line)["status"] for line in runs[0][1].splitlines()], + ["2x-order-only", "ok-in-3x", "matches-neither", "has-sidecar", + "body-missing", "2x-order-only"]) + + def test_the_affected_rows_feed_the_sidecar_tool_and_clear_the_refusals(self): + tree = Tree(self.dir, blocks=2) + affected = self.dir / "affected.csv" + check(tree, "--affected-csv", affected) + report = self.dir / "report.jsonl" + with contextlib.redirect_stdout(io.StringIO()): + code = tool.main(["--rows", str(affected), "--audit-root", str(tree.audit), + "--canonicalizer", str(write_stub(self.dir)), + "--sidecar-dir", str(tree.import_root), + "--work-dir", str(self.dir / "work"), "--report", str(report), + "--write"]) + rows = [json.loads(line) for line in report.read_text().splitlines()] + self.assertEqual((code, [row["status"] for row in rows]), (0, ["sidecar-written"] * 2)) + self.assertIn("share_parts_digest_hex mismatch", rows[0]["original_error"]) + for block in tree.blocks: + self.assertEqual(gzip.decompress(tree.sidecar(block).read_bytes()), tree.canonical) + code, rows, summary = check(tree) + self.assertEqual((code, summary), (0, {"import_refuses": 0, "totals": {"has-sidecar": 2}})) + + +@unittest.skipUnless(GNU_COREUTILS, "the sharded runner needs GNU coreutils") +class ShardedRunnerTests(unittest.TestCase): + """prism_legacy_range_sidecars_sharded.sh over the sidecar tool, with a + canonicalizer stub that refuses 2.x's parts digest as 3.x does.""" + + def setUp(self): + self.tmp = tempfile.TemporaryDirectory() + self.addCleanup(self.tmp.cleanup) + self.dir = Path(self.tmp.name) + self.canonicalizer = write_stub(self.dir) + + def run_shards(self, tree, out, *, shards="2", rows=None): + return subprocess.run( + ["bash", str(SHARDED), str(rows or tree.rows), str(tree.audit), str(tree.import_root), + str(self.canonicalizer), str(SIDECAR_TOOL), str(out), shards], + capture_output=True, text=True) + + def statuses(self, out): + return sorted(json.loads(line)["status"] for report in sorted(Path(out).glob("report-*.jsonl")) + for line in report.read_text().splitlines()) + + def test_shards_write_every_sidecar_once_and_a_rerun_verifies_them(self): + tree = Tree(self.dir, blocks=5) + run = self.run_shards(tree, self.dir / "out1") + self.assertEqual(run.returncode, 0, run.stdout + run.stderr) + shards = sorted((self.dir / "out1").glob("shard-*.csv")) + self.assertEqual([shard.name for shard in shards], ["shard-00.csv", "shard-01.csv"]) + sharded = [line for shard in shards for line in shard.read_text().splitlines()[1:]] + self.assertTrue(all(shard.read_text().startswith(HEADER + "\n") for shard in shards)) + self.assertEqual(sorted(sharded), sorted(tree.rows.read_text().splitlines()[1:])) + self.assertEqual(self.statuses(self.dir / "out1"), ["sidecar-written"] * 5) + for block in tree.blocks: + self.assertEqual(gzip.decompress(tree.sidecar(block).read_bytes()), tree.canonical) + self.assertIn("temporary files left in IMPORT_ROOT: 0", run.stdout) + rerun = self.run_shards(tree, self.dir / "out2", shards="3") + self.assertEqual(rerun.returncode, 0, rerun.stdout + rerun.stderr) + self.assertEqual(self.statuses(self.dir / "out2"), ["sidecar-ok"] * 5) + + def test_a_shard_that_fails_fails_the_run(self): + tree = Tree(self.dir, blocks=4) + (tree.audit / tree.body_name(tree.blocks[0])).unlink() + run = self.run_shards(tree, self.dir / "out") + self.assertNotEqual(run.returncode, 0) + self.assertEqual(self.statuses(self.dir / "out"), ["body-missing"] + ["sidecar-written"] * 3) + self.assertEqual(sorted(path.read_text().strip() for path in (self.dir / "out").glob("exit-*")), + ["0", "3"]) + + def test_the_out_dir_guards_and_an_empty_rows_file(self): + tree = Tree(self.dir) + inside = tree.import_root / "out" + run = self.run_shards(tree, inside) + self.assertEqual((run.returncode, inside.exists()), (2, False)) + self.assertIn("OUT_DIR must not be inside IMPORT_ROOT", run.stderr) + busy = self.dir / "busy" + busy.mkdir() + (busy / "left-over").write_text("") + run = self.run_shards(tree, busy) + self.assertEqual(run.returncode, 2) + self.assertIn("is not empty", run.stderr) + header_only = self.dir / "header-only.csv" + header_only.write_text(HEADER + "\n") + run = self.run_shards(tree, self.dir / "out", rows=header_only) + self.assertEqual(run.returncode, 0, run.stderr) + self.assertIn("nothing to do", run.stdout) + self.assertEqual(list((self.dir / "out").iterdir()), []) + for shards in ("0", "100", "x"): + run = self.run_shards(tree, self.dir / f"out-{shards}", shards=shards) + self.assertEqual(run.returncode, 2, shards) + self.assertIn("N must be 1 to 99", run.stderr) + usage = subprocess.run(["bash", str(SHARDED), "rows.csv"], capture_output=True, text=True) + self.assertEqual(usage.returncode, 2) + self.assertIn("usage: prism_legacy_range_sidecars_sharded.sh", usage.stderr) + + +@unittest.skipUnless(CANONICALIZE_BIN, "set PRISM_AUDIT_CANONICALIZE_BIN, as " + "`make test-prism-legacy-range-sidecars` does") +class RealCanonicalizerTests(unittest.TestCase): + """The real 3.x canonicalizer against a body 2.x's own writer produced + (tests/fixtures/prism-legacy-v2-audit-body).""" + + def setUp(self): + self.tmp = tempfile.TemporaryDirectory() + self.addCleanup(self.tmp.cleanup) + self.dir = Path(self.tmp.name) + [self.fixture_body] = FIXTURE.glob("prism-audit-bundle-body-*.json") + self.body = json.loads(self.fixture_body.read_bytes()) + self.digest = self.body["audit_bundle_sha256"] + + def canonicalize(self, path): + return subprocess.run([CANONICALIZE_BIN, "--input", str(path)], capture_output=True) + + def test_the_canonicalizer_refuses_exactly_what_the_checker_lists(self): + proof = self.body["share_window_proof"] + # The checker's 2.x form reproduces the digest 2.x's writer stored. + self.assertEqual(module.digest_2x(proof["share_parts"]), proof["share_parts_digest_hex"]) + code, [row], _ = run_checker([self.fixture_body]) + self.assertEqual((code, row["status"]), (3, "2x-order-only")) + refused = self.canonicalize(self.fixture_body) + self.assertNotEqual(refused.returncode, 0) + self.assertIn(b"window proof share_parts_digest_hex mismatch", refused.stderr) + + # With the slots beside the body, the digest's form alone decides. + for path in FIXTURE.glob("prism-audit-share-segment-*.json"): + shutil.copy(path, self.dir / path.name) + for part in proof["share_parts"]: + part["body_uri"] = Path(part["body_uri"]).name + body = self.dir / "body.json" + for digest, status, accepted in ((module.digest_2x, "2x-order-only", False), + (module.digest_3x, "ok-in-3x", True)): + with self.subTest(status=status): + proof["share_parts_digest_hex"] = digest(proof["share_parts"]) + body.write_bytes(compact(self.body)) + code, [row], _ = run_checker([body]) + self.assertEqual(row["status"], status) + run = self.canonicalize(body) + self.assertEqual(run.returncode == 0, accepted, run.stderr) + if accepted: + self.assertEqual(hashlib.sha256(run.stdout).hexdigest(), self.digest) + else: + self.assertIn(b"share_parts_digest_hex mismatch", run.stderr) + + @unittest.skipUnless(GNU_COREUTILS, "the sharded runner needs GNU coreutils") + def test_the_sharded_runner_proves_2x_bodies_with_the_real_canonicalizer(self): + audit = self.dir / "audit" + audit.mkdir() + import_root = self.dir / "import-root" + import_root.mkdir() + for path in FIXTURE.glob("prism-audit-share-segment-*.json"): + shutil.copy(path, audit / path.name) + blocks = [f"{index:02x}" * 32 for index in (0xa1, 0xb2, 0xc3)] + rows = self.dir / "rows.csv" + lines = [HEADER] + for block in blocks: + name = f"prism-audit-bundle-body-{block}-{self.digest}.json" + shutil.copy(self.fixture_body, audit / name) + lines.append(f"{block},{PREFIX}/{name},{self.digest}") + rows.write_text("\n".join(lines) + "\n") + affected = self.dir / "affected.csv" + code, _, summary = run_checker(["--rows", rows, "--audit-root", audit, + "--sidecar-dir", import_root, "--affected-csv", affected]) + self.assertEqual((code, summary["import_refuses"]), (3, 3)) + + def shards(out): + run = subprocess.run( + ["bash", str(SHARDED), str(affected), str(audit), str(import_root), + CANONICALIZE_BIN, str(SIDECAR_TOOL), str(out), "2"], + capture_output=True, text=True) + self.assertEqual(run.returncode, 0, run.stdout + run.stderr) + return sorted(json.loads(line)["status"] for report in Path(out).glob("report-*.jsonl") + for line in report.read_text().splitlines()) + + self.assertEqual(shards(self.dir / "out1"), ["sidecar-written"] * 3) + for block in blocks: + sidecar = import_root / f"prism-audit-bundle-canonical-{block}-{self.digest}.json.gz" + self.assertEqual(hashlib.sha256(gzip.decompress(sidecar.read_bytes())).hexdigest(), + self.digest) + code, _, summary = run_checker(["--rows", rows, "--audit-root", audit, + "--sidecar-dir", import_root]) + self.assertEqual((code, summary), (0, {"import_refuses": 0, "totals": {"has-sidecar": 3}})) + self.assertEqual(shards(self.dir / "out2"), ["sidecar-ok"] * 3) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_prism_legacy_range_sidecars.py b/tests/test_prism_legacy_range_sidecars.py new file mode 100644 index 000000000..7e34032bc --- /dev/null +++ b/tests/test_prism_legacy_range_sidecars.py @@ -0,0 +1,430 @@ +"""Prove legacy bodies whose range digests use Python's escaped JSON (#709).""" +import contextlib +import errno +import gzip +import hashlib +import importlib.util +import io +import json +import os +from pathlib import Path +import stat +import subprocess +import sys +import tempfile +import textwrap +import unittest +from unittest import mock + + +REPO = Path(__file__).resolve().parents[1] +SCRIPT = REPO / "scripts/prism_legacy_range_sidecars.py" +spec = importlib.util.spec_from_file_location("legacy_range_sidecars", SCRIPT) +module = importlib.util.module_from_spec(spec) +spec.loader.exec_module(module) + +PREFIX = "/var/lib/qbit-mining-pool/prism/audit" +SEGMENT = "qbit.prism.audit-share-segment.v1" +BODY_REF = "qbit.prism.audit-body-ref.v1" +V2 = "qbit.prism.audit-bundle.v2" +CANONICALIZE_BIN = os.environ.get("PRISM_AUDIT_CANONICALIZE_BIN", "") + +# The 3.x canonicalizer's contract, for the shapes these tests write: resolve +# the share parts, check each range digest as serde_json writes it (raw +# UTF-8), require sha256(canonical bundle) == audit_bundle_sha256, and print +# the canonical bytes. Its canonical form is the bundle as compact JSON. +STUB = textwrap.dedent("""\ + import hashlib, json, sys + from pathlib import Path + + def serde(value): + return json.dumps(value, separators=(",", ":"), ensure_ascii=False).encode() + + path = Path(sys.argv[sys.argv.index("--input") + 1]) + body = json.loads(path.read_bytes()) + if body["schema"] == "qbit.prism.audit-body-ref.v1": + parts = body["share_parts"] + else: + parts = body["share_window_proof"]["share_parts"] + shares = [] + for part in parts: + if part["kind"] == "inline": + shares += part["shares"] + continue + slot = Path(part["body_uri"]) + slot = slot if slot.is_absolute() else path.parent / slot + segment = json.loads(slot.read_bytes()) + first, last = part["first_share_seq"], part["last_share_seq"] + selected = [s for s in segment["shares"] if first <= s["share_seq"] <= last] + payload = {"schema": "qbit.prism.audit-share-segment.v1", "first_share_seq": first, + "last_share_seq": last, "share_count": len(selected), "shares": selected} + key = "range_sha256" if part["kind"] == "segment_range" else "prefix_sha256" + if hashlib.sha256(serde(payload)).hexdigest() != part[key]: + sys.exit("qbit-prism-audit-canonicalize: audit body ref hash mismatch") + shares += selected + bundle = dict(body["bundle_without_shares"]) + bundle["shares"] = shares + canonical = serde(bundle) + if hashlib.sha256(canonical).hexdigest() != body["audit_bundle_sha256"]: + sys.exit("qbit-prism-audit-canonicalize: audit body ref hash mismatch") + sys.stdout.buffer.write(canonical) +""") + + +def compact(value, ascii_only): + return json.dumps(value, separators=(",", ":"), ensure_ascii=ascii_only).encode() + + +def share(seq, worker): + return { + "share_seq": seq, "share_id": f"{seq:064x}", + "miner_id": f"qb1zexample.{worker}", "order_key": "qb1zexample", + "p2mr_program_hex": "ab" * 32, "share_difficulty": 2 ** 70 + seq, + "network_difficulty": 2 ** 72, "template_height": 53313, "job_id": f"job-{seq}", + "job_issued_at_ms": 1755815340000 + seq, "accepted_at_ms": 1755815341000 + seq, + "ntime": 1755815340, + } + + +def range_digest(first, last, shares, ascii_only): + payload = {"schema": SEGMENT, "first_share_seq": first, "last_share_seq": last, + "share_count": len(shares), "shares": shares} + return hashlib.sha256(compact(payload, ascii_only)).hexdigest() + + +def escaped_part(kind, first, last, shares, body_uri): + """A range or prefix part as 1.x and 2.x wrote it: the digest is Python's + escaped JSON.""" + key = "range_sha256" if kind == "segment_range" else "prefix_sha256" + return {"kind": kind, "first_share_seq": first, "last_share_seq": last, + "share_count": len(shares), key: range_digest(first, last, shares, True), + "body_uri": body_uri} + + +def parted_body(schema, digest, bundle_without_shares, share_count, parts): + common = {"schema": schema, "audit_bundle_sha256": digest, "share_count": share_count, + "bundle_without_shares": bundle_without_shares} + if schema == BODY_REF: + return dict(common, share_parts=parts) + return dict(common, share_window_proof={ + "schema": "qbit.prism.window-completeness-proof.v1", "first_share_seq": 1, + "last_share_seq": share_count, "share_count": share_count, "share_parts": parts}) + + +class Fixture: + """A restored audit tree holding one 1.x-style parted body.""" + + def __init__(self, root, *, schema=BODY_REF, workers=None, absolute_slot_uri=True, + first_kind="segment_range"): + self.root = Path(root) + workers = workers or ["rig01", "rig02", "rig03", "Bjørn-rig", "rig05", "Bjørn-rig"] + self.shares = [share(seq, worker) for seq, worker in enumerate(workers, start=1)] + self.bundle_without_shares = {"found_block": {"block_height": 53313}, "schema": "v1.1"} + bundle = dict(self.bundle_without_shares) + bundle["shares"] = self.shares + self.canonical = compact(bundle, False) + self.digest = hashlib.sha256(self.canonical).hexdigest() + self.block = "00" * 16 + "11e43" + "0" * 27 + (self.root / "segments").mkdir(parents=True) + (self.root / "bodies").mkdir() + slot = {"schema": SEGMENT, "first_share_seq": 1, "last_share_seq": len(self.shares), + "share_count": len(self.shares), "shares": self.shares} + self.slot = self.root / "segments/slot-1-10.json" + self.slot.write_bytes(compact(slot, True)) + parts = [ + escaped_part(first_kind, 1, 3, self.shares[:3], "../segments/slot-1-10.json"), + escaped_part("segment_range", 4, 6, self.shares[3:], + (f"{PREFIX}/segments/slot-1-10.json" if absolute_slot_uri + else "../segments/slot-1-10.json")), + ] + body = parted_body(schema, self.digest, self.bundle_without_shares, len(self.shares), + parts) + self.body = self.root / f"bodies/{self.block}.json" + self.body.write_bytes(compact(body, True)) + self.rows = self.root / "rows.csv" + self.write_rows(f"{PREFIX}/bodies/{self.block}.json") + self.sidecar_dir = self.root / "sidecars" + self.sidecar_dir.mkdir() + self.sidecar = (self.sidecar_dir / + f"prism-audit-bundle-canonical-{self.block}-{self.digest}.json.gz") + + def write_rows(self, body_uri, block=None): + self.rows.write_text("block_hash,body_uri,audit_bundle_sha256\n" + f"{block or self.block},{body_uri},{self.digest}\n") + + +class LegacyRangeSidecarTests(unittest.TestCase): + def setUp(self): + self.tmp = tempfile.TemporaryDirectory() + self.addCleanup(self.tmp.cleanup) + self.canonicalizer = Path(self.tmp.name) / "qbit-prism-audit-canonicalize" + self.canonicalizer.write_text(f"#!{sys.executable}\n" + STUB) + self.canonicalizer.chmod(self.canonicalizer.stat().st_mode | stat.S_IXUSR) + + def fixture(self, **kwargs): + return Fixture(Path(self.tmp.name) / "audit", **kwargs) + + def run_tool(self, fx, *, write, paths=None): + report = Path(self.tmp.name) / "report.jsonl" + paths = paths or {"--audit-root": fx.root, "--sidecar-dir": fx.sidecar_dir, + "--work-dir": Path(self.tmp.name) / "work"} + argv = ["--rows", str(fx.rows), "--uri-prefix", PREFIX, + "--canonicalizer", str(self.canonicalizer), "--report", str(report)] + for flag, path in paths.items(): + argv += [flag, str(path)] + if write: + argv.append("--write") + stdout, stderr = io.StringIO(), io.StringIO() + with contextlib.redirect_stdout(stdout), contextlib.redirect_stderr(stderr): + code = module.main(argv) + rows = [json.loads(line) for line in report.read_text().splitlines()] + return code, rows, stderr.getvalue() + + def test_escaped_range_digest_is_reproduced_and_the_body_proven(self): + fx = self.fixture() + code, [row], _ = self.run_tool(fx, write=True) + self.assertEqual(code, 0) + self.assertEqual(row["status"], "sidecar-written") + self.assertEqual([part["status"] for part in row["parts"]], + ["ok-in-3x", "python-escaped"]) + self.assertIn({"share_seq": 4, "field": "miner_id", "text": "qb1zexample.Bjørn-rig"}, + row["parts"][1]["non_ascii"]) + self.assertEqual(gzip.decompress(fx.sidecar.read_bytes()), fx.canonical) + + def test_v2_body_is_proven_through_a_body_ref_rewrite(self): + fx = self.fixture(schema=V2) + code, [row], _ = self.run_tool(fx, write=True) + self.assertEqual((code, row["status"]), (0, "sidecar-written")) + self.assertEqual(gzip.decompress(fx.sidecar.read_bytes()), fx.canonical) + + def test_an_escaped_prefix_digest_is_proven_too(self): + fx = self.fixture(workers=["Bjørn-rig", "rig02", "rig03", "rig04", "rig05", "rig06"], + first_kind="segment_prefix") + code, [row], _ = self.run_tool(fx, write=True) + self.assertEqual((code, row["status"]), (0, "sidecar-written")) + self.assertEqual([(part["kind"], part["status"]) for part in row["parts"]], + [("segment_prefix", "python-escaped"), ("segment_range", "ok-in-3x")]) + self.assertEqual(gzip.decompress(fx.sidecar.read_bytes()), fx.canonical) + + def test_dry_run_classifies_without_writing(self): + fx = self.fixture() + code, [row], _ = self.run_tool(fx, write=False) + self.assertEqual((code, row["status"]), (0, "proven")) + self.assertFalse(fx.sidecar.exists()) + + def test_an_ascii_body_already_verifies_and_gets_no_sidecar(self): + fx = self.fixture(workers=["rig01", "rig02", "rig03", "rig04", "rig05", "rig06"], + absolute_slot_uri=False) + code, [row], _ = self.run_tool(fx, write=True) + self.assertEqual((code, row["status"]), (0, "ok-in-3x")) + self.assertFalse(fx.sidecar.exists()) + + def test_a_tree_mounted_away_from_its_uris_still_gets_a_proven_sidecar(self): + # The body names its slot by the production path, which this mount + # does not have; the mapped rewrite proves it and the sidecar spares + # import the slot lookup. + fx = self.fixture(workers=["rig01", "rig02", "rig03", "rig04", "rig05", "rig06"]) + code, [row], _ = self.run_tool(fx, write=True) + self.assertEqual((code, row["status"]), (0, "sidecar-written")) + self.assertEqual([part["status"] for part in row["parts"]], ["ok-in-3x", "ok-in-3x"]) + self.assertEqual(gzip.decompress(fx.sidecar.read_bytes()), fx.canonical) + + def test_relative_paths_do_not_depend_on_the_work_directory(self): + # The second range keeps its slot, named by the production path; the + # rewritten body-ref, read from the work directory, must still find it. + fx = self.fixture(workers=["Bjørn-rig", "rig02", "rig03", "rig04", "rig05", "rig06"]) + self.addCleanup(os.chdir, os.getcwd()) + os.chdir(self.tmp.name) + code, [row], _ = self.run_tool(fx, write=True, paths={ + "--audit-root": "audit", "--sidecar-dir": "audit/sidecars", "--work-dir": "work"}) + self.assertEqual((code, row["status"]), (0, "sidecar-written")) + self.assertEqual([part["status"] for part in row["parts"]], + ["python-escaped", "ok-in-3x"]) + self.assertEqual(gzip.decompress(fx.sidecar.read_bytes()), fx.canonical) + + def test_a_relative_body_uri_resolves_against_the_import_root(self): + # Import resolves it against its --root, which is --sidecar-dir, even + # when the tree is mounted for the tool somewhere else. + fx = self.fixture(absolute_slot_uri=False) + fx.write_rows(f"bodies/{fx.block}.json") + elsewhere = Path(self.tmp.name) / "elsewhere" + elsewhere.mkdir() + code, [row], _ = self.run_tool(fx, write=True, paths={ + "--audit-root": elsewhere, "--sidecar-dir": fx.root, + "--work-dir": Path(self.tmp.name) / "work"}) + self.assertEqual((code, row["status"]), (0, "sidecar-written")) + self.assertEqual(row["body"], str(fx.body.resolve())) + sidecar = fx.root / fx.sidecar.name + self.assertEqual(gzip.decompress(sidecar.read_bytes()), fx.canonical) + + def test_a_valid_existing_sidecar_is_kept_and_counts_as_done(self): + fx = self.fixture() + original = gzip.compress(fx.canonical) + fx.sidecar.write_bytes(original) + code, [row], _ = self.run_tool(fx, write=True) + self.assertEqual((code, row["status"]), (0, "sidecar-ok")) + self.assertEqual(fx.sidecar.read_bytes(), original) + + def test_write_refuses_to_overwrite_an_existing_sidecar(self): + fx = self.fixture() + fx.sidecar.write_bytes(b"not the canonical bundle") + code, [row], stderr = self.run_tool(fx, write=True) + self.assertEqual((code, row["status"]), (3, "sidecar-bad")) + self.assertIn(str(fx.sidecar), stderr) + self.assertEqual(fx.sidecar.read_bytes(), b"not the canonical bundle") + + def test_a_sidecar_with_bytes_after_its_gzip_member_is_bad(self): + # Import reads only the first member, so anything after it is refused + # rather than trusted. + fx = self.fixture() + fx.sidecar.write_bytes(gzip.compress(fx.canonical) + gzip.compress(b"\n")) + code, [row], stderr = self.run_tool(fx, write=True) + self.assertEqual((code, row["status"]), (3, "sidecar-bad")) + self.assertIn(str(fx.sidecar), stderr) + + def test_a_sidecar_linked_from_outside_the_import_root_is_bad(self): + # Import refuses a sidecar that resolves outside its --root, however + # good the bytes. + fx = self.fixture() + outside = Path(self.tmp.name) / "elsewhere.json.gz" + outside.write_bytes(gzip.compress(fx.canonical)) + fx.sidecar.symlink_to(outside) + code, [row], stderr = self.run_tool(fx, write=True) + self.assertEqual((code, row["status"]), (3, "sidecar-bad")) + self.assertIn(str(fx.sidecar), stderr) + + def test_write_refuses_a_sidecar_that_appears_while_proving(self): + fx = self.fixture() + target = fx.sidecar + self.assertEqual(module.write_sidecar(target, b"first"), "sidecar-written") + self.assertEqual(module.write_sidecar(target, b"second"), "sidecar-conflict") + self.assertEqual(gzip.decompress(target.read_bytes()), b"first") + self.assertEqual([path.name for path in fx.sidecar_dir.iterdir()], [target.name]) + + def test_a_sidecar_that_cannot_be_published_whole_is_not_published(self): + # Import stops on a partial sidecar, so publication is the atomic + # hard link or nothing, and the row is reported. + failures = { + "no hard links": mock.patch.object( + module.os, "link", side_effect=OSError(errno.EPERM, "no links")), + "disk full": mock.patch.object( + module.os, "fsync", side_effect=OSError(errno.ENOSPC, "disk full")), + } + for name, failure in failures.items(): + with self.subTest(name), tempfile.TemporaryDirectory() as tmp: + fx = Fixture(Path(tmp) / "audit") + with failure: + code, [row], stderr = self.run_tool(fx, write=True) + self.assertEqual((code, row["status"]), (3, "sidecar-write-failed")) + self.assertIn(str(fx.sidecar), stderr) + self.assertEqual(list(fx.sidecar_dir.iterdir()), []) + + def test_a_corrupt_body_is_reported_and_gets_no_sidecar(self): + fx = self.fixture() + slot = json.loads(fx.slot.read_bytes()) + slot["shares"][4]["share_difficulty"] += 1 # damage inside the failing range + fx.slot.write_bytes(compact(slot, True)) + code, [row], _ = self.run_tool(fx, write=True) + self.assertEqual(code, 3) + self.assertEqual(row["status"], "canonicalizer-refused") + self.assertEqual(row["parts"][1]["status"], "unexplained") + self.assertFalse(fx.sidecar.exists()) + + def test_an_unreadable_body_is_reported_and_gets_no_sidecar(self): + fx = self.fixture() + fx.body.write_bytes(b'{"schema": "qbit.prism.audit-body-ref.v1", "share_') + code, [row], _ = self.run_tool(fx, write=True) + self.assertEqual((code, row["status"]), (3, "body-unreadable")) + self.assertFalse(fx.sidecar.exists()) + + def test_a_missing_slot_is_reported(self): + fx = self.fixture() + fx.slot.unlink() + code, [row], _ = self.run_tool(fx, write=True) + self.assertEqual((code, row["status"]), (3, "slot-incomplete")) + self.assertFalse(fx.sidecar.exists()) + + def test_a_row_that_is_not_a_hash_identity_is_refused(self): + fx = self.fixture() + fx.write_rows(f"{PREFIX}/bodies/{fx.block}.json", block="../../escape") + code, [row], _ = self.run_tool(fx, write=True) + self.assertEqual((code, row["status"]), (3, "bad-row")) + self.assertEqual(list(fx.sidecar_dir.iterdir()), []) + + def test_write_needs_a_sidecar_dir(self): + fx = self.fixture() + with contextlib.redirect_stderr(io.StringIO()), self.assertRaises(SystemExit): + module.main(["--rows", str(fx.rows), "--audit-root", str(fx.root), + "--canonicalizer", str(self.canonicalizer), "--write"]) + + +@unittest.skipUnless(CANONICALIZE_BIN, "set PRISM_AUDIT_CANONICALIZE_BIN, as " + "`make test-prism-legacy-range-sidecars` does") +class RealCanonicalizerTests(unittest.TestCase): + """The real 3.x canonicalizer refuses a 1.x-style body whose range digest + is escaped JSON, and proves the tool's rewrite. The bundle comes from the + qbit-prism-build-audit-bundle beside it, with non-ASCII worker names.""" + + def real_bundle(self, tmp): + fixture = json.loads( + (REPO / "crates/qbit-prism/fixtures/power-law-accrual.prism-fixture.json").read_bytes()) + for accepted in fixture["shares"][1::3]: + accepted["miner_id"] += ".Bjørn-rig" + # The fixture's carry-forward, as crates/qbit-prism/tests/audit_cli.rs passes it. + prior = [{"recipient_id": "miner-whale", "order_key": "01", + "p2mr_program_hex": "11" * 32, "balance_sats": 4_800}] + builder_input = Path(tmp) / "input.json" + builder_input.write_bytes(compact({"found_block": fixture["found_block"], + "shares": fixture["shares"], + "prior_balances": prior}, False)) + builder = Path(CANONICALIZE_BIN).with_name("qbit-prism-build-audit-bundle") + return subprocess.run( + [str(builder), "--input", str(builder_input), "--signing-key-seed-hex", "42" * 32, + "--ledger-signing-key-seed-hex", "43" * 32, "--canonical-output"], + capture_output=True, check=True).stdout + + def test_real_canonicalizer_refuses_the_body_and_proves_the_rewrite(self): + for schema in (BODY_REF, V2): + with self.subTest(schema=schema), tempfile.TemporaryDirectory() as tmp: + canonical = self.real_bundle(tmp) + digest = hashlib.sha256(canonical).hexdigest() + bundle = json.loads(canonical) + shares = bundle.pop("shares") + root = Path(tmp) / "audit" + (root / "segments").mkdir(parents=True) + slot = {"schema": SEGMENT, "first_share_seq": 1, "last_share_seq": len(shares), + "share_count": len(shares), "shares": shares} + (root / "segments/slot.json").write_bytes(compact(slot, True)) + # The ASCII prefix is kept and checked by the real binary; the + # range holds the non-ASCII workers and is inlined. + parts = [escaped_part("segment_prefix", 1, 1, shares[:1], "segments/slot.json"), + escaped_part("segment_range", 2, len(shares), shares[1:], + "segments/slot.json")] + body = parted_body(schema, digest, bundle, len(shares), parts) + (root / "body.json").write_bytes(compact(body, True)) + refused, error = module.canonicalize(CANONICALIZE_BIN, root / "body.json") + self.assertIsNone(refused) + self.assertIn("hash mismatch", error) + + rows = Path(tmp) / "rows.csv" + rows.write_text("block_hash,body_uri,audit_bundle_sha256\n" + f"{'cd' * 32},{PREFIX}/body.json,{digest}\n") + sidecars = Path(tmp) / "sidecars" + sidecars.mkdir() + report = Path(tmp) / "report.jsonl" + with contextlib.redirect_stdout(io.StringIO()): + code = module.main(["--rows", str(rows), "--audit-root", str(root), + "--canonicalizer", CANONICALIZE_BIN, + "--sidecar-dir", str(sidecars), + "--work-dir", str(Path(tmp) / "work"), + "--report", str(report), "--write"]) + [row] = [json.loads(line) for line in report.read_text().splitlines()] + self.assertEqual((code, row["status"]), (0, "sidecar-written")) + self.assertEqual([part["status"] for part in row["parts"]], + ["ok-in-3x", "python-escaped"]) + [sidecar] = sidecars.iterdir() + self.assertEqual(gzip.decompress(sidecar.read_bytes()), canonical) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_prism_load_ab.py b/tests/test_prism_load_ab.py new file mode 100644 index 000000000..67dcfcc6a --- /dev/null +++ b/tests/test_prism_load_ab.py @@ -0,0 +1,781 @@ +"""Tests for scripts/prism_load_ab.py, the A/B release benchmark driver (#511).""" + +from __future__ import annotations + +import contextlib +import io +import json +import os +import signal +import time +import subprocess +from pathlib import Path +import sys +import tempfile +import unittest +from unittest import mock + +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT / "scripts")) + +import prism_load_ab as ab # noqa: E402 + +# The real preset check, before any test stubs it out. +VALIDATE_PRESET = ab.validate_preset + +# `qbit-prism-load --help`'s long flags at 5d0042f6, #271's base and the +# oldest build #479 STEP 1 compared. +FLAGS_5D0042F6 = frozenset( + """--ack-p99-limit-ms --allow-debug-server --allow-dirty-tree + --allow-unverified-server-revision --blockpoll-seconds --burst-rate --burst-seconds + --cadence --cadence-gaps --cadence-rate --cadence-seconds --database-url + --db-max-connections --example-artifact --external-tips + --forecast-peak-shares-per-second --frontends --keep-artifacts + --lock-sample-interval-ms --max-outstanding-per-session --mid-flight-kill + --min-mem-available-mib --out --pg-bin-dir --plan --process-sample-interval-ms --rate + --reconnect-seconds --reconnect-target --replication --runtime-workers + --scheduled-blocks --seed-share-bytes --server-bin --sessions + --share-commit-timeout-seconds --slow-database-seconds --slow-db-delay-ms + --steady-state-rate --steady-state-seconds --warmup-seconds --window-shares + --work-timeout --help --version""".split() +) + +# 5d0042f6's history has none of the commits that introduced a legacy flag. +OLDER = lambda introduced: False # noqa: E731 + +# #479 STEP 1's argument list: #271's 20k list, unchanged. +ARGS_479 = ( + "--frontends 1 --sessions 2000 --window-shares 20000 --replication async --plan d1 " + "--forecast-peak-shares-per-second 2000 --ack-p99-limit-ms 1000 --min-mem-available-mib 6144" +).split() + + +def running(pid: int) -> bool: + """Whether `pid` is a live process; a zombie awaiting its reaper is not.""" + try: + state = Path(f"/proc/{pid}/stat").read_text().rsplit(")", 1)[1].split()[0] + except (FileNotFoundError, ProcessLookupError): + return False + return state != "Z" + + +def number_or_text(word: str | bool) -> float | str | bool: + try: + return float(word) + except (TypeError, ValueError): + return word + + +def pairs(words: list[str]) -> dict[str, str | bool]: + out: dict[str, str | bool] = {} + index = 0 + while index < len(words): + flag = words[index] + if index + 1 < len(words) and not words[index + 1].startswith("--"): + out[flag] = words[index + 1] + index += 2 + else: + out[flag] = True + index += 1 + return out + + +class Resolution(unittest.TestCase): + def setUp(self) -> None: + self.legacy = ab.load_legacy() + self.operational = ab.operational_flags() + + def preset(self, name: str) -> dict: + return ab.load_preset(ab.PRESETS / f"{name}.json") + + def current_flags(self, preset: dict) -> frozenset[str]: + return frozenset(preset["args"]) | self.operational | {"--help", "--version"} + + def test_the_operational_flags_are_read_from_preset_rs(self) -> None: + self.assertIn("--server-bin", self.operational) + self.assertIn("--preset", self.operational) + self.assertNotIn("--sessions", self.operational) + + def test_d1_20k_runs_on_5d0042f6_with_479s_argument_list(self) -> None: + preset = self.preset("throughput-20k-window-1fe") + words, dropped = ab.resolve_argv(preset["args"], FLAGS_5D0042F6, self.legacy, self.operational, "base", OLDER) + resolved = pairs(words) + expected = pairs(ARGS_479) + for flag, value in expected.items(): + self.assertTrue(ab.same(number_or_text(resolved[flag]), number_or_text(value)), flag) + self.assertTrue(set(resolved) <= FLAGS_5D0042F6) + self.assertIn("--stratum-max-pending-initial-jobs", dropped) + self.assertIn("--background-shares-per-second", dropped) + self.assertEqual(set(dropped), set(preset["args"]) - FLAGS_5D0042F6) + + def test_a_current_build_runs_every_pinned_flag_and_leaves_nothing_off(self) -> None: + preset = self.preset("throughput-20k-window-1fe") + words, dropped = ab.resolve_argv(preset["args"], self.current_flags(preset), self.legacy, self.operational, "candidate", OLDER) + self.assertEqual(dropped, []) + self.assertIn("--stratum-max-pending-initial-jobs", words) + self.assertEqual(words[words.index("--stratum-max-pending-initial-jobs") + 1], "2016") + self.assertNotIn("--database-url", words) + self.assertNotIn("--mid-flight-kill", words) + + def test_a_workload_an_old_build_cannot_run_is_refused_with_every_reason(self) -> None: + preset = self.preset("tip-delivery-2000-miners-400k-2fe-retarget") + with self.assertRaises(ab.DriverError) as caught: + ab.resolve_argv(preset["args"], FLAGS_5D0042F6, self.legacy, self.operational, "base", OLDER) + message = str(caught.exception) + self.assertIn("--retarget-bits", message) + self.assertIn("--background-shares-per-second", message) + self.assertIn("--stratum-max-pending-initial-jobs", message) + self.assertIn("needs 1016", message) + + def test_a_missing_flag_with_no_legacy_rule_is_refused(self) -> None: + preset = self.preset("throughput-20k-window-1fe") + flags = self.current_flags(preset) - {"--plan"} + with self.assertRaisesRegex(ab.DriverError, "no --plan and legacy-flags.json has no rule"): + ab.resolve_argv(preset["args"], flags, self.legacy, self.operational, "base", OLDER) + + def test_a_newer_harness_flag_the_preset_does_not_pin_is_refused(self) -> None: + preset = self.preset("throughput-20k-window-1fe") + flags = self.current_flags(preset) | {"--some-new-knob"} + with self.assertRaisesRegex(ab.DriverError, "result flag --some-new-knob"): + ab.resolve_argv(preset["args"], flags, self.legacy, self.operational, "candidate", OLDER) + + def test_inert_flags_are_dropped_only_while_their_phase_is_off(self) -> None: + rule = self.legacy["--churn-rate"] + self.assertTrue(ab.legacy_holds("--churn-rate", rule, {"--churn-rate": 50, "--churn-seconds": 0})[0]) + self.assertFalse(ab.legacy_holds("--churn-rate", rule, {"--churn-rate": 50, "--churn-seconds": 300})[0]) + self.assertFalse(ab.legacy_holds("--churn-rate", rule, {"--churn-rate": 50})[0]) + + def test_numbers_compare_as_numbers_and_never_as_booleans(self) -> None: + self.assertTrue(ab.same(0, 0.0)) + self.assertFalse(ab.same(False, 0)) + self.assertFalse(ab.same(None, 0)) + self.assertTrue(ab.same(None, None)) + + def test_the_admission_formula_matches_the_old_harness(self) -> None: + rule = self.legacy["--stratum-max-pending-initial-jobs"] + flag = "--stratum-max-pending-initial-jobs" + cases = [(2000, 1, 2016), (2000, 2, 1016), (2000, 4, 516), (200, 2, 128), (2001, 2, 1017)] + for sessions, frontends, implied in cases: + args = {"--sessions": sessions, "--frontends": frontends, flag: implied} + self.assertTrue(ab.legacy_holds(flag, rule, args)[0], (sessions, frontends)) + args[flag] = implied + 1 + self.assertFalse(ab.legacy_holds(flag, rule, args)[0]) + + def test_every_checked_in_d1_preset_runs_on_5d0042f6(self) -> None: + for path in sorted(ab.PRESETS.glob("throughput-*.json")): + if path.stem.endswith("-addresses"): + continue + with self.subTest(preset=path.stem): + preset = ab.load_preset(path) + ab.resolve_argv(preset["args"], FLAGS_5D0042F6, self.legacy, self.operational, "base", OLDER) + + def test_help_flags_are_the_option_lines_never_a_flag_a_description_mentions(self) -> None: + text = """Options: + --pg-bin-dir <PG_BIN_DIR> + Directory holding `initdb`. Defaults to `pg_config --bindir` + --mid-flight-kill + -h, --help + Print help (see a summary with '-h') + -V, --version +""" + self.assertEqual(ab.parse_help_flags(text), {"--pg-bin-dir", "--mid-flight-kill", "--help", "--version"}) + + def test_a_harness_that_lost_a_legacy_flag_is_refused_not_treated_as_older(self) -> None: + preset = ab.load_preset(ab.preset_path("throughput-20k-window-1fe")) + flags = self.current_flags(preset) - {"--seed"} + seed = self.legacy["--seed"]["introduced"] + with self.assertRaisesRegex(ab.DriverError, "the flag was removed"): + ab.resolve_argv(preset["args"], flags, self.legacy, self.operational, "candidate", + lambda introduced: introduced == seed) + # A build older than the flag runs without it, as the rule says. + _, dropped = ab.resolve_argv(preset["args"], flags, self.legacy, self.operational, "base", OLDER) + self.assertEqual(dropped, ["--seed"]) + + def test_the_legacy_table_is_well_formed(self) -> None: + for flag, rule in self.legacy.items(): + self.assertTrue(flag.startswith("--")) + self.assertTrue(rule.get("why"), flag) + self.assertRegex(rule["introduced"], r"^[0-9a-f]{40}$", flag) + with tempfile.TemporaryDirectory() as directory: + broken = Path(directory) / "legacy.json" + broken.write_text(json.dumps({"schema": ab.LEGACY_SCHEMA, "flags": {"--x": {"equals": 1, "formula": "y"}}})) + with self.assertRaises(ab.DriverError): + ab.load_legacy(broken) + + +class Series(unittest.TestCase): + def setUp(self) -> None: + # The comparator's preset check needs a Rust build; these tests + # exercise the driver around it. + patcher = mock.patch.object(ab, "validate_preset", lambda comparator, preset: None) + patcher.start() + self.addCleanup(patcher.stop) + compare = mock.patch.object(ab, "compare_bin", lambda explicit: Path("/nonexistent/compare")) + compare.start() + self.addCleanup(compare.stop) + + def test_a_preset_the_comparator_rejects_stops_the_series_before_any_build(self) -> None: + with tempfile.TemporaryDirectory() as directory: + fake = Path(directory) / "compare" + fake.write_text("#!/bin/sh\necho 'preset x: max_shortfall is -1' >&2\nexit 2\n") + fake.chmod(0o755) + with self.assertRaisesRegex(ab.DriverError, "is not valid: preset x: max_shortfall is -1"): + VALIDATE_PRESET(fake, "x.json") + + def test_the_preset_is_validated_before_anything_is_built(self) -> None: + events: list[str] = [] + + def build(label, ref, out, skip_build): + events.append(f"build {label}") + raise ab.DriverError("stop") + + with tempfile.TemporaryDirectory() as directory, \ + mock.patch.object(ab, "validate_preset", lambda comparator, preset: events.append("validate")), \ + mock.patch.object(ab, "prepare_build", build), self.assertRaises(ab.DriverError): + ab.main(["--base", "a", "--candidate", "b", "--preset", "throughput-20k-window-1fe", + "--out", directory, "--dry-run"]) + self.assertEqual(events, ["validate", "build base"]) + + def test_repeats_alternate_which_build_goes_first(self) -> None: + self.assertEqual( + ab.interleaved(3), + [(1, "base"), (1, "candidate"), (2, "candidate"), (2, "base"), (3, "base"), (3, "candidate")], + ) + + def test_the_load_gate_waits_for_a_quiet_host_and_gives_up_at_its_deadline(self) -> None: + clock = [0.0] + loads = iter([5.0, 4.0, 2.5]) + seen = ab.wait_for_quiet(3.0, 100, 10, lambda: next(loads), lambda s: clock.__setitem__(0, clock[0] + s), lambda: clock[0]) + self.assertEqual(seen, 2.5) + clock[0] = 0.0 + with self.assertRaises(ab.IncompleteSeries): + ab.wait_for_quiet(3.0, 25, 10, lambda: 9.0, lambda s: clock.__setitem__(0, clock[0] + s), lambda: clock[0]) + + def test_pg_test_fsync_is_read_from_the_one_write_section(self) -> None: + text = """5 seconds per test +O_DIRECT supported on this platform for open_datasync and open_sync. + +Compare file sync methods using one 8kB write: +(in "wal_sync_method" preference order, except fdatasync is Linux's default) + open_datasync 3937.123 ops/sec 254 usecs/op + fdatasync 3467.456 ops/sec 288 usecs/op + fsync 2123.000 ops/sec 471 usecs/op + +Compare file sync methods using two 8kB writes: +(in "wal_sync_method" preference order, except fdatasync is Linux's default) + open_datasync 1900.000 ops/sec 526 usecs/op + fdatasync 1700.000 ops/sec 588 usecs/op +""" + self.assertEqual( + ab.parse_pg_test_fsync(text), + {"fdatasync_ops_per_second": 3467.456, "fdatasync_usecs_per_op": 288.0}, + ) + self.assertIsNone(ab.parse_pg_test_fsync("could not open output file")) + + def test_nonfinite_and_non_positive_settings_are_refused(self) -> None: + base = ["--base", "a", "--candidate", "b", "--preset", "throughput-20k-window-1fe", "--out", "x"] + for bad in (["--max-load", "nan"], ["--max-load", "0"], ["--repeats", "0"], ["--cooldown-seconds", "inf"], + ["--run-ceiling-seconds", "-1"]): + with self.subTest(bad=bad), self.assertRaises(SystemExit), contextlib.redirect_stderr(io.StringIO()): + ab.parse_args(base + bad) + self.assertEqual(ab.parse_args(base + ["--cooldown-seconds", "0"]).cooldown_seconds, 0.0) + + def test_a_run_records_its_exit_code_and_host_samples(self) -> None: + with tempfile.TemporaryDirectory() as directory: + runs = Path(directory) + record = ab.execute_run("r1", [sys.executable, "-c", "print('hi'); raise SystemExit(3)"], + runs, dict(os.environ), runs, 60, 0.05) + self.assertEqual(record["exit_code"], 3) + self.assertFalse(record["ceiling_hit"]) + self.assertIsNotNone(record["load_max"]) + self.assertIn("hi", (runs / "r1.log").read_text()) + self.assertTrue((runs / "r1.host.jsonl").read_text().strip()) + self.assertTrue((runs / "r1.ps-before.txt").exists() and (runs / "r1.ps-after.txt").exists()) + + def test_a_run_past_its_ceiling_is_killed_with_its_process_group(self) -> None: + with tempfile.TemporaryDirectory() as directory: + runs = Path(directory) + record = ab.execute_run("r2", [sys.executable, "-c", "import time; time.sleep(60)"], + runs, dict(os.environ), runs, 0.5, 0.1) + self.assertIsNone(record["exit_code"]) + self.assertTrue(record["ceiling_hit"]) + + def test_an_interrupted_run_takes_its_process_group_down_before_propagating(self) -> None: + started: list[subprocess.Popen] = [] + + class Interrupted(subprocess.Popen): + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + started.append(self) + + def wait(self, timeout=None): + if timeout == 30: + raise KeyboardInterrupt + return super().wait(timeout) + + with tempfile.TemporaryDirectory() as directory, mock.patch.object(ab.subprocess, "Popen", Interrupted): + runs = Path(directory) + with self.assertRaises(KeyboardInterrupt): + ab.execute_run("r3", [sys.executable, "-c", "import time; time.sleep(60)"], + runs, dict(os.environ), runs, 30, 0.1) + self.assertIsNotNone(started[-1].poll()) + + def test_a_resume_that_moves_the_clusters_or_the_host_is_refused(self) -> None: + base = ["--base", "a", "--candidate", "b", "--preset", "throughput-20k-window-1fe", "--out", "x", "--resume"] + options = ab.parse_args(base + ["--tmpdir", "/tmp/pload-ab"]) + previous = {"host": ab.host_facts(), "settings": ab.settings_of(options)} + ab.check_resumable(previous, options) + ab.check_resumable(previous, ab.parse_args(base + ["--tmpdir", "/tmp/pload-ab", "--skip-build"])) + for moved in (["--max-load", "100"], ["--cooldown-seconds", "0"], ["--lock-file", "/tmp/other.lock"]): + with self.subTest(moved=moved), self.assertRaisesRegex(ab.DriverError, "cannot resume"): + ab.check_resumable(previous, ab.parse_args(base + ["--tmpdir", "/tmp/pload-ab"] + moved)) + with self.assertRaisesRegex(ab.DriverError, "--tmpdir was /tmp/pload-ab"): + ab.check_resumable(previous, ab.parse_args(base + ["--tmpdir", "/mnt/other"])) + with self.assertRaisesRegex(ab.DriverError, "--repeats was 5, not 3"): + ab.check_resumable({**previous, "settings": {**previous["settings"], "repeats": 5}}, options) + for key, value in (("hostname", "elsewhere"), ("nproc", 999), ("mem_total_mib", 1)): + moved = {**previous, "host": {**previous["host"], key: value}} + with self.subTest(key=key), self.assertRaisesRegex(ab.DriverError, f"the host's {key} was {value}"): + ab.check_resumable(moved, options) + + def test_the_builds_run_under_the_benchmark_lock(self) -> None: + events: list[str] = [] + + @contextlib.contextmanager + def lock(path): + events.append("lock") + try: + yield + finally: + events.append("unlock") + + def build(label, ref, out, skip_build): + events.append(f"build {label}") + raise ab.DriverError("stop") + + with tempfile.TemporaryDirectory() as directory, \ + mock.patch.object(ab, "benchmark_lock", lock), mock.patch.object(ab, "prepare_build", build): + with self.assertRaises(ab.DriverError): + ab.main(["--base", "a", "--candidate", "b", "--preset", "throughput-20k-window-1fe", + "--out", directory, "--dry-run"]) + self.assertEqual(events, ["lock", "build base", "unlock"]) + + def test_a_missing_pg_test_fsync_is_bad_input_not_a_fail(self) -> None: + with tempfile.TemporaryDirectory() as directory: + with self.assertRaisesRegex(ab.DriverError, "pg_test_fsync, .*: not executable"): + ab.main(["--base", "a", "--candidate", "b", "--preset", "throughput-20k-window-1fe", + "--out", directory, "--pg-bin-dir", directory]) + with self.assertRaisesRegex(ab.DriverError, "running"): + ab.record_fsync(Path(directory), Path(directory), Path(directory) / "f.txt", 1) + + def test_a_failed_flush_probe_stops_the_series_as_bad_input(self) -> None: + with tempfile.TemporaryDirectory() as directory: + folder = Path(directory) + probe = folder / "pg_test_fsync" + for script, why in [ + ("echo 'could not open output file' >&2; exit 1", "exited 1"), + ("echo '5 seconds per test'; exit 0", "exited 0 without"), + ]: + probe.write_text(f"#!/bin/sh\n{script}\n") + probe.chmod(0o755) + with self.subTest(why=why), self.assertRaisesRegex(ab.DriverError, why): + ab.record_fsync(folder, folder, folder / "fsync.txt", 1) + self.assertTrue((folder / "fsync.txt").exists(), "the probe's output is kept") + + def test_a_series_written_while_waiting_for_the_lock_needs_resume(self) -> None: + with tempfile.TemporaryDirectory() as directory: + @contextlib.contextmanager + def lock(path): + # The first invocation finishes while this one waits. + (Path(directory) / "manifest.json").write_text("{}") + yield + + with mock.patch.object(ab, "benchmark_lock", lock), \ + self.assertRaisesRegex(ab.DriverError, "pass --resume"): + ab.main(["--base", "a", "--candidate", "b", "--preset", "throughput-20k-window-1fe", + "--out", directory, "--dry-run"]) + + def test_relative_paths_are_made_absolute_before_anything_uses_them(self) -> None: + seen = {} + + def series(options, *rest): + seen["tmpdir"] = options.tmpdir + return 0 + + with tempfile.TemporaryDirectory() as directory, mock.patch.object(ab, "run_series", series), \ + mock.patch.object(ab, "benchmark_lock", lambda path: contextlib.nullcontext()): + ab.main(["--base", "a", "--candidate", "b", "--preset", "throughput-20k-window-1fe", + "--out", directory, "--dry-run", "--tmpdir", "relative/tmp"]) + self.assertTrue(seen["tmpdir"].is_absolute()) + + def test_an_unexpected_failure_exits_2_never_1(self) -> None: + cases = [ + (RuntimeError("boom"), ab.EXIT_INPUT), + (FileNotFoundError("no comparator"), ab.EXIT_INPUT), + (ab.DriverError("bad"), ab.EXIT_INPUT), + (ab.IncompleteSeries("loaded"), ab.EXIT_INCOMPLETE), + ] + for error, status in cases: + def fail(argv, error=error): + raise error + + with self.subTest(error=error), mock.patch.object(ab, "main", fail), \ + contextlib.redirect_stderr(io.StringIO()): + self.assertEqual(ab.run_cli([]), status) + + def test_a_relative_cargo_target_dir_is_anchored_where_cargo_ran(self) -> None: + self.assertEqual(ab.comparator_path(None), ab.ROOT / "target" / "release" / "qbit-prism-load-compare") + self.assertEqual(ab.comparator_path("out/t"), ab.ROOT / "out" / "t" / "release" / "qbit-prism-load-compare") + self.assertEqual(ab.comparator_path("/abs/t"), Path("/abs/t/release/qbit-prism-load-compare")) + + def test_sigterm_to_the_driver_stops_the_detached_harness(self) -> None: + with tempfile.TemporaryDirectory() as directory: + pid_file = Path(directory) / "harness.pid" + harness = f"import os, time; open({str(pid_file)!r}, 'w').write(str(os.getpid())); time.sleep(120)" + driver = ( + f"import sys; sys.path.insert(0, {str(ab.ROOT / 'scripts')!r}); import prism_load_ab as ab\n" + "from pathlib import Path\n" + "import os\n" + "ab.main = lambda argv: ab.execute_run('r', [sys.executable, '-c', " + f"{harness!r}], Path({directory!r}), dict(os.environ), Path({directory!r}), 120, 0.1) and 0\n" + "sys.exit(ab.run_cli([]))\n" + ) + proc = subprocess.Popen([sys.executable, "-c", driver], stderr=subprocess.PIPE, text=True) + deadline = time.monotonic() + 10 + while not pid_file.exists() or not pid_file.read_text(): + self.assertLess(time.monotonic(), deadline) + time.sleep(0.05) + harness_pid = int(pid_file.read_text()) + proc.send_signal(signal.SIGTERM) + _, stderr = proc.communicate(timeout=30) + self.assertEqual(proc.returncode, 128 + signal.SIGTERM, stderr) + with self.assertRaises(ProcessLookupError): + os.kill(harness_pid, 0) + + def test_a_signal_during_the_spawn_is_raised_after_it_and_stops_the_child(self) -> None: + started: list[subprocess.Popen] = [] + + class SignalledDuringSpawn(subprocess.Popen): + def __init__(self, args, *rest, **kwargs): + super().__init__(args, *rest, **kwargs) + if args[0] == sys.executable: # the harness, not the ps snapshots + started.append(self) + # The signal lands before the constructor returns. + ab.on_termination(signal.SIGTERM, None) + + with tempfile.TemporaryDirectory() as directory, \ + mock.patch.object(ab.subprocess, "Popen", SignalledDuringSpawn): + runs = Path(directory) + with self.assertRaises(ab.Terminated): + ab.execute_run("r4", [sys.executable, "-c", "import time; time.sleep(60)"], + runs, dict(os.environ), runs, 60, 0.1) + self.assertIsNotNone(started[-1].poll()) + with self.assertRaises(ab.Terminated): + ab.on_termination(signal.SIGTERM, None) + + def test_a_descendant_that_outlives_the_harness_is_stopped_before_the_run_returns(self) -> None: + with tempfile.TemporaryDirectory() as directory: + runs = Path(directory) + pid_file = runs / "descendant.pid" + # The harness starts a descendant that ignores SIGTERM, then exits. + descendant = "import signal, time; signal.signal(signal.SIGTERM, signal.SIG_IGN); time.sleep(120)" + harness = ( + "import subprocess, sys; " + f"p = subprocess.Popen([sys.executable, '-c', {descendant!r}]); " + f"open({str(pid_file)!r}, 'w').write(str(p.pid))" + ) + original = ab.stop_group + with mock.patch.object(ab, "stop_group", lambda child: original(child, 1.0, 0.05)): + record = ab.execute_run("r5", [sys.executable, "-c", harness], runs, dict(os.environ), runs, 60, 0.1) + self.assertEqual(record["exit_code"], 0) + self.assertFalse(running(int(pid_file.read_text()))) + + def test_a_second_invocation_on_the_same_out_is_refused_whatever_its_lock_file(self) -> None: + with tempfile.TemporaryDirectory() as directory: + with ab.series_lock(Path(directory) / ".series.lock"): + with self.assertRaisesRegex(ab.DriverError, "another invocation is using"): + ab.main(["--base", "a", "--candidate", "b", "--preset", "throughput-20k-window-1fe", + "--out", directory, "--dry-run", "--lock-file", str(Path(directory) / "other.lock")]) + + def test_a_signal_just_after_the_deferred_spawn_still_stops_the_child(self) -> None: + started: list[subprocess.Popen] = [] + real_deferred = ab.deferred_termination + + @contextlib.contextmanager + def deferral_then_signal(): + with real_deferred() as pending: + yield pending + # The signal lands the instant the deferral ends. + ab.on_termination(signal.SIGTERM, None) + + class Recording(subprocess.Popen): + def __init__(self, args, *rest, **kwargs): + super().__init__(args, *rest, **kwargs) + if args[0] == sys.executable: + started.append(self) + + with tempfile.TemporaryDirectory() as directory, \ + mock.patch.object(ab.subprocess, "Popen", Recording), \ + mock.patch.object(ab, "deferred_termination", deferral_then_signal): + runs = Path(directory) + with self.assertRaises(ab.Terminated): + ab.execute_run("r6", [sys.executable, "-c", "import time; time.sleep(60)"], + runs, dict(os.environ), runs, 60, 0.1) + self.assertIsNotNone(started[-1].poll()) + + def test_a_signal_during_the_after_exit_sweep_still_stops_the_descendant(self) -> None: + with tempfile.TemporaryDirectory() as directory: + runs = Path(directory) + pid_file = runs / "descendant.pid" + descendant = "import signal, time; signal.signal(signal.SIGTERM, signal.SIG_IGN); time.sleep(120)" + harness = ( + "import subprocess, sys; " + f"p = subprocess.Popen([sys.executable, '-c', {descendant!r}]); " + f"open({str(pid_file)!r}, 'w').write(str(p.pid))" + ) + original = ab.stop_group + + def signalled_stop(child): + # The signal lands as the sweep begins; it must be held. + ab.on_termination(signal.SIGTERM, None) + original(child, 1.0, 0.05) + + with mock.patch.object(ab, "stop_group", signalled_stop), self.assertRaises(ab.Terminated): + ab.execute_run("r7", [sys.executable, "-c", harness], runs, dict(os.environ), runs, 60, 0.1) + self.assertFalse(running(int(pid_file.read_text()))) + + def test_a_setsid_server_left_by_a_killed_harness_is_found_and_stopped(self) -> None: + # As qbit-prism-server and pg_ctl's postmaster do, the descendant + # leaves the harness's process group with setsid() (and ignores + # SIGTERM); the harness then dies without stopping it. + self.assertTrue(ab.become_subreaper()) + with tempfile.TemporaryDirectory() as directory: + runs = Path(directory) + pid_file = runs / "server.pid" + server = "import os, signal, time; os.setsid(); signal.signal(signal.SIGTERM, signal.SIG_IGN); time.sleep(120)" + harness = ( + "import os, signal, subprocess, sys, time; " + f"p = subprocess.Popen([sys.executable, '-c', {server!r}]); " + f"open({str(pid_file)!r}, 'w').write(str(p.pid)); " + "time.sleep(0.3); os.kill(os.getpid(), signal.SIGKILL)" + ) + sweep = ab.sweep_descendants + with mock.patch.object(ab, "sweep_descendants", lambda: sweep(1.0, 0.05)): + record = ab.execute_run("r8", [sys.executable, "-c", harness], runs, dict(os.environ), runs, 60, 0.1) + self.assertEqual(record["exit_code"], -signal.SIGKILL) + server_pid = int(pid_file.read_text()) + self.assertFalse(running(server_pid)) + + def test_a_series_is_refused_when_the_driver_cannot_become_a_subreaper(self) -> None: + argv = ["--base", "a", "--candidate", "b", "--preset", "throughput-20k-window-1fe"] + with tempfile.TemporaryDirectory() as directory, mock.patch.object(ab, "become_subreaper", lambda: False): + for name in ("pg_test_fsync", *ab.required_pg_binaries()): + binary = Path(directory) / name + binary.write_text("#!/bin/sh\n") + binary.chmod(0o755) + with self.assertRaisesRegex(ab.DriverError, "child subreaper"): + ab.main(argv + ["--out", directory, "--pg-bin-dir", directory]) + # A dry run still compiles both builds, so it needs one too. + with self.assertRaisesRegex(ab.DriverError, "child subreaper"): + ab.main(argv + ["--out", directory, "--dry-run"]) + # So does one over existing builds: it compiles the comparator. + with self.assertRaisesRegex(ab.DriverError, "child subreaper"): + ab.main(argv + ["--out", directory, "--dry-run", "--skip-build"]) + + def test_a_build_interrupted_by_a_signal_leaves_nothing_running_under_the_lock(self) -> None: + self.assertTrue(ab.become_subreaper()) + events: list[str] = [] + with tempfile.TemporaryDirectory() as directory: + pid_file = Path(directory) / "rustc.pid" + # "cargo" starts a "rustc" that ignores SIGTERM, then the driver + # is signalled mid-build. + rustc = "import signal, time; signal.signal(signal.SIGTERM, signal.SIG_IGN); time.sleep(120)" + cargo = ( + "import subprocess, sys, time; " + f"p = subprocess.Popen([sys.executable, '-c', {rustc!r}]); " + f"open({str(pid_file)!r}, 'w').write(str(p.pid)); time.sleep(120)" + ) + + def build(label, ref, out, skip_build): + subprocess.Popen([sys.executable, "-c", cargo]) + while not pid_file.exists() or not pid_file.read_text(): + time.sleep(0.05) + raise ab.Terminated(signal.SIGTERM) + + @contextlib.contextmanager + def lock(path): + try: + yield + finally: + events.append("unlock") + events.append("rustc alive" if running(int(pid_file.read_text())) else "rustc stopped") + + sweep = ab.sweep_descendants + with mock.patch.object(ab, "prepare_build", build), mock.patch.object(ab, "benchmark_lock", lock), \ + mock.patch.object(ab, "sweep_descendants", lambda: sweep(1.0, 0.05)), \ + self.assertRaises(ab.Terminated): + ab.main(["--base", "a", "--candidate", "b", "--preset", "throughput-20k-window-1fe", + "--out", directory, "--dry-run"]) + self.assertEqual(events, ["unlock", "rustc stopped"]) + + def test_every_server_binary_the_harness_needs_is_checked_before_the_series(self) -> None: + self.assertEqual(ab.required_pg_binaries(), ("initdb", "pg_ctl", "pg_basebackup")) + with tempfile.TemporaryDirectory() as directory: + fsync = Path(directory) / "pg_test_fsync" + fsync.write_text("#!/bin/sh\n") + fsync.chmod(0o755) + with self.assertRaisesRegex(ab.DriverError, "initdb, .*pg_ctl, .*pg_basebackup: not executable"): + ab.main(["--base", "a", "--candidate", "b", "--preset", "throughput-20k-window-1fe", + "--out", directory, "--pg-bin-dir", directory]) + + def test_a_skipped_build_reuses_only_binaries_the_driver_built_for_that_commit(self) -> None: + commits = {"now": "a" * 40} + + def git(*args: str, cwd: Path = ab.ROOT) -> str: + return commits["now"] if args[0] == "rev-parse" else "" + + with tempfile.TemporaryDirectory() as directory: + out = Path(directory) + release = out / "builds" / "base" / "target" / "release" + release.mkdir(parents=True) + for binary in ab.BINARIES: + (release / binary).write_bytes(b"left here by " + binary.encode()) + built = subprocess.CompletedProcess([], 0) + with mock.patch.object(ab, "git", git), \ + mock.patch.object(ab.subprocess, "run", return_value=built) as cargo, \ + contextlib.redirect_stderr(io.StringIO()): + # Binaries the driver did not build are not reused. + with self.assertRaisesRegex(ab.DriverError, "holds no build this driver made"): + ab.prepare_build("base", "v1", out, skip_build=True) + cargo.assert_not_called() + # The driver's own build records them, and then they are. + ab.prepare_build("base", "v1", out, skip_build=False) + self.assertEqual(cargo.call_count, 1) + ab.prepare_build("base", "v1", out, skip_build=True) + self.assertEqual(cargo.call_count, 1) + # A binary replaced since, or a worktree moved to another + # commit, is not. + (release / "qbit-prism-load").write_bytes(b"copied in") + with self.assertRaisesRegex(ab.DriverError, "holds no build this driver made"): + ab.prepare_build("base", "v1", out, skip_build=True) + ab.prepare_build("base", "v1", out, skip_build=False) + commits["now"] = "b" * 40 + with self.assertRaisesRegex(ab.DriverError, "holds no build this driver made"): + ab.prepare_build("base", "v2", out, skip_build=True) + # A failed build leaves no record to reuse. + cargo.return_value = subprocess.CompletedProcess([], 101) + with self.assertRaisesRegex(ab.DriverError, "failed with exit 101"): + ab.prepare_build("base", "v2", out, skip_build=False) + self.assertFalse((release / ab.BUILD_RECORD).exists()) + + def test_url_secrets_are_redacted_as_the_harness_redacts_them(self) -> None: + password = "hunter2-Sup3r_Secret" + # The cases of the harness's own redaction tests (tests/harness.rs). + for url, expected in [ + (f"postgresql://alex:{password}@db.example:5432/qbit?sslmode=disable&application_name=load-fe-0", + "postgresql://alex:<redacted>@db.example:5432/qbit?sslmode=disable&application_name=load-fe-0"), + (f"https://svc:{password}@api.example/v1#frag", "https://svc:<redacted>@api.example/v1#frag"), + (f"postgresql://db.example/qbit?password={password}&sslmode=require", + "postgresql://db.example/qbit?password=<redacted>&sslmode=require"), + (f"postgresql://db.example/qbit?pass%77ord={password}&sslmode=require", + "postgresql://db.example/qbit?pass%77ord=<redacted>&sslmode=require"), + (f"postgresql://db.example/qbit?%70%61%73%73%77%6F%72%64={password}", + "postgresql://db.example/qbit?%70%61%73%73%77%6F%72%64=<redacted>"), + (f"postgresql://alex:{password}@db.example/qbit?PASS%57ORD={password}#f", + "postgresql://alex:<redacted>@db.example/qbit?PASS%57ORD=<redacted>#f"), + ]: + self.assertEqual(ab.redact_url_secrets(url), expected) + for untouched in ["postgresql://u@127.0.0.1:5432/postgres", "info", "0.0000000122070312", + "postgresql://h/d?pass+word=x&pass%zzword=y", "--database-url"]: + self.assertEqual(ab.redact_url_secrets(untouched), untouched) + + def test_a_database_url_password_reaches_the_harness_but_not_the_manifest_or_the_log(self) -> None: + url = "postgresql://alex:hunter2@db.example/qbit" + launched: list[list[str]] = [] + + def build(label, ref, out, skip_build): + return {"label": label, "ref": ref, "commit": label[0] * 40, "worktree": f"builds/{label}"} + + def run(run_id, command, *rest): + launched.append(command) + return {"exit_code": 0, "ceiling_hit": False, "load_max": 0.1, "mem_available_min_mib": 1} + + with tempfile.TemporaryDirectory() as directory: + out = Path(directory) + options = ab.parse_args(["--base", "a", "--candidate", "b", "--preset", "throughput-20k-window-1fe", + "--out", directory, "--repeats", "1", "--cooldown-seconds", "0"]) + preset = {"name": "throughput-20k-window-1fe", "_path": "p.json", "_sha256": "ab", + "args": {"--database-url": url}} + comparison = subprocess.CompletedProcess([], 0, stdout="", stderr="") + with mock.patch.object(ab, "prepare_build", build), \ + mock.patch.object(ab, "descends_from", lambda commit, ancestor: False), \ + mock.patch.object(ab, "help_flags", lambda harness: frozenset(["--database-url"])), \ + mock.patch.object(ab, "record_fsync", lambda *args: {"fdatasync_usecs_per_op": 288.0}), \ + mock.patch.object(ab, "wait_for_quiet", lambda *args: 0.1), \ + mock.patch.object(ab, "execute_run", run), \ + mock.patch.object(ab.subprocess, "run", return_value=comparison), \ + contextlib.redirect_stderr(io.StringIO()) as log: + ab.run_series(options, out, preset, {}, frozenset(), out / "manifest.json", None) + manifest = (out / "manifest.json").read_text() + self.assertEqual(len(launched), 2) + self.assertTrue(all(url in command for command in launched)) + self.assertNotIn("hunter2", manifest) + self.assertNotIn("hunter2", log.getvalue()) + self.assertIn("postgresql://alex:<redacted>@db.example/qbit", manifest) + # Builds whose history predates a report field are recorded as such. + self.assertEqual( + [build["predates_report_fields"] for build in json.loads(manifest)["builds"]], + [["mem_available_unread_checks"]] * 2, + ) + + def test_a_resume_whose_rebuilt_binaries_differ_is_refused(self) -> None: + def build(label, ref, out, skip_build): + return {"label": label, "ref": ref, "commit": label[0] * 40, "worktree": f"builds/{label}", + "sha256": {"qbit-prism-load": f"{label}-rebuilt", "qbit-prism-server": label}} + + with tempfile.TemporaryDirectory() as directory: + out = Path(directory) + options = ab.parse_args(["--base", "a", "--candidate", "b", "--preset", "throughput-20k-window-1fe", + "--out", directory, "--resume"]) + preset = {"name": "throughput-20k-window-1fe", "_path": "p.json", "_sha256": "ab", "args": {}} + previous = {"builds": [ + {"label": label, "commit": label[0] * 40, + "sha256": {"qbit-prism-load": label, "qbit-prism-server": label}} + for label in ab.LABELS + ], "runs": [], "pg_test_fsync": {}} + with mock.patch.object(ab, "prepare_build", build), \ + mock.patch.object(ab, "descends_from", lambda commit, ancestor: True), \ + mock.patch.object(ab, "help_flags", lambda harness: frozenset()), \ + self.assertRaisesRegex(ab.DriverError, "not the ones its recorded runs used"): + ab.run_series(options, out, preset, {}, frozenset(), out / "manifest.json", previous) + + def test_the_sweep_waits_out_a_descendant_that_outlives_sigkill(self) -> None: + # A process in uninterruptible I/O survives SIGKILL for a while: here + # 30 of the fake clock's seconds, three times the old give-up window. + clock = {"now": 0.0} + alive_until = 30.0 + + def monotonic() -> float: + clock["now"] += 1.0 + return clock["now"] + + survivor = 999_999 + with mock.patch.object(ab, "descendants", lambda root: [survivor] if clock["now"] < alive_until else []), \ + mock.patch.object(ab, "_alive", lambda pid: True), \ + mock.patch.object(ab.os, "kill", lambda pid, sig: None), \ + mock.patch.object(ab.time, "monotonic", monotonic), \ + mock.patch.object(ab.time, "sleep", lambda seconds: None), \ + contextlib.redirect_stderr(io.StringIO()) as log: + ab.sweep_descendants(grace_seconds=1.0) + self.assertGreaterEqual(clock["now"], alive_until) + self.assertIn(f"waiting for {survivor} to exit after SIGKILL", log.getvalue()) + + def test_a_named_preset_resolves_to_the_checked_in_file(self) -> None: + self.assertEqual(ab.preset_path("throughput-20k-window-1fe"), ab.PRESETS / "throughput-20k-window-1fe.json") + with contextlib.redirect_stderr(io.StringIO()) as warning: + self.assertEqual(ab.preset_path("d1-20k"), ab.PRESETS / "throughput-20k-window-1fe.json") + self.assertIn("deprecated", warning.getvalue()) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_prism_load_bridge.py b/tests/test_prism_load_bridge.py new file mode 100644 index 000000000..cdb693bef --- /dev/null +++ b/tests/test_prism_load_bridge.py @@ -0,0 +1,281 @@ +"""Tests for scripts/prism_load_bridge.py, the bridging lane's row (#552).""" + +from __future__ import annotations + +import contextlib +import copy +import io +import json +from pathlib import Path +import re +import sys +import tempfile +import unittest + +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT / "scripts")) + +import prism_load_bridge as bridge # noqa: E402 +import prism_load_matrix as matrix # noqa: E402 + + +def latency(p50: float | None, p99: float | None) -> dict: + summary = {"unit": "milliseconds", "clock": "client monotonic", "samples": 10, "p50": p50, "p99": p99} + if p50 is None: + summary.update(samples=0, unavailable_reason="no samples were recorded") + return summary + + +def report(side: str, ack: tuple[float | None, float | None] = (2.0, 9.0), tips=(40.0, 60.0, 80.0), + rss=((100_000, 120_000), (110_000, 130_000))) -> dict: + phases = [] + for index, name in enumerate(("warm_up", "steady_state", "reconnect")): + phases.append({ + "name": name, + "client_ack_latency": latency(*ack), + "processes": [{"instance_id": f"fe{fe}", "peak_rss_kib": rss[fe][min(index, 1)]} for fe in range(2)], + }) + node: dict = {"url": "http://127.0.0.1:1", "submissions": []} + if side == "real": + node = { + "mode": "qbitd", + "qbitd": {"peak_rss_kib": {"pool": 300_000, "peer": 280_000}}, + "relay": {"rpc_latency_milliseconds": { + "submitblock": {"count": 2, "p50": 12.0, "p99": 30.0, "max": 30.0}, + "getblocktemplate": {"count": 50, "p50": 4.0, "p99": 11.0, "max": 15.0}, + }}, + } + return { + "schema": bridge.REPORT_SCHEMA, + "preset": {"name": bridge.PRESETS[side]}, + "topology": {"frontends": 2}, + "phases": phases, + "time_to_usable_work": {"tips": [ + {"all_sessions_milliseconds": value, + "all_sessions_unavailable_reason": None if value is not None else "3 of 100 sessions got no usable work"} + for value in tips + ]}, + "node": node, + } + + +class Runs: + """Two run directories shaped as prism-load-run.sh leaves them.""" + + def __init__(self, root: Path) -> None: + self.root = root + self.fake = root / "fake" + self.real = root / "real" + self.write(self.fake, report("fake")) + self.write(self.real, report("real", ack=(5.0, 20.0), tips=(90.0, 100.0, 150.0), + rss=((150_000, 160_000), (150_000, 170_000)))) + + def write(self, directory: Path, body: dict | None, harness: str = "0", gate: str = "0", + runner: str = "blacksmith-8vcpu-ubuntu-2404") -> None: + directory.mkdir(parents=True, exist_ok=True) + path = directory / "load-harness-report.json" + if body is None: + path.unlink(missing_ok=True) + else: + path.write_text(json.dumps(body), encoding="utf-8") + (directory / "harness-exit-code").write_text(harness + "\n", encoding="utf-8") + (directory / "gate-exit-code").write_text(gate + "\n", encoding="utf-8") + (directory / "host.json").write_text(json.dumps({"runner_label": runner}), encoding="utf-8") + + def row(self, order: str = "fake,real") -> tuple[int, dict, str]: + out = self.root / "row.json" + summary = self.root / "summary.md" + summary.unlink(missing_ok=True) + with contextlib.redirect_stdout(io.StringIO()): + code = bridge.main([ + "--fake", str(self.fake), "--real", str(self.real), "--order", order, + "--out", str(out), "--summary", str(summary), "--commit", "abc123", + ]) + return code, json.loads(out.read_text(encoding="utf-8")), summary.read_text(encoding="utf-8") + + +def by_name(row: dict) -> dict[str, dict]: + return {entry["metric"]: entry for entry in row["metrics"]} + + +class ComparisonRow(unittest.TestCase): + def setUp(self) -> None: + self.directory = tempfile.TemporaryDirectory() + self.runs = Runs(Path(self.directory.name)) + + def tearDown(self) -> None: + self.directory.cleanup() + + def test_each_metric_is_real_minus_fake_with_its_unit(self) -> None: + code, row, summary = self.runs.row() + self.assertEqual(code, 0) + self.assertEqual(row["schema"], bridge.SCHEMA) + self.assertTrue(row["trusted"], row["untrusted_reasons"]) + self.assertIn("Not a D1 verdict", row["statement"]) + self.assertEqual(row["order"], ["fake", "real"]) + self.assertEqual(row["runner"], "blacksmith-8vcpu-ubuntu-2404") + metrics = by_name(row) + ack = metrics["client_ack_latency p99 (steady_state)"] + self.assertEqual((ack["fake"], ack["real"], ack["real_minus_fake"], ack["unit"]), (9.0, 20.0, 11.0, "milliseconds")) + self.assertIn("client_ack_latency p50 (reconnect)", metrics) + tip_p50 = metrics["time to usable work, slowest session, p50 over external tips"] + self.assertEqual((tip_p50["fake"], tip_p50["real"], tip_p50["real_minus_fake"]), (60.0, 100.0, 40.0)) + self.assertEqual(metrics["time to usable work, slowest session, max over external tips"]["real_minus_fake"], 70.0) + # VmHWM's run peak is the largest any phase saw, per frontend. + fe1 = metrics["frontend 1 peak RSS"] + self.assertEqual((fe1["fake"], fe1["real"], fe1["unit"]), (130_000, 170_000, "KiB")) + self.assertEqual(metrics["frontends peak RSS, summed"]["real_minus_fake"], (160_000 + 170_000) - (120_000 + 130_000)) + self.assertIn("| client_ack_latency p99 (steady_state) | 9.0 ms | 20.0 ms | +11.0 ms |", summary) + self.assertIn("Not a D1 verdict", summary) + + def test_what_only_the_real_node_measures_is_unknown_on_the_fake_side_never_zero(self) -> None: + _, row, summary = self.runs.row() + metrics = by_name(row) + for name in ("node submitblock latency p50", "node submitblock latency p99", + "node getblocktemplate latency p99", "pool node peak RSS"): + entry = metrics[name] + self.assertIsNone(entry["fake"], name) + self.assertIsNotNone(entry["real"], name) + self.assertIsNone(entry["real_minus_fake"], name) + self.assertTrue(entry["unavailable_reason"].startswith("fake: "), entry) + self.assertEqual(metrics["node submitblock latency p99"]["real"], 30.0) + self.assertEqual(metrics["pool node peak RSS"]["real"], 300_000) + self.assertRegex(summary, r"\| node submitblock latency p50 \| unknown \| 12\.0 ms \| unknown \[\d+\] \|") + self.assertIn(bridge.FAKE_HAS_NO_RELAY, summary) + + def test_a_real_zero_difference_stays_zero(self) -> None: + self.runs.write(self.runs.real, report("real")) + _, row, summary = self.runs.row() + ack = by_name(row)["client_ack_latency p50 (warm_up)"] + self.assertEqual(ack["real_minus_fake"], 0.0) + self.assertIsNone(ack["unavailable_reason"]) + self.assertIn("| client_ack_latency p50 (warm_up) | 2.0 ms | 2.0 ms | 0.0 ms |", summary) + + def test_a_tip_some_session_missed_leaves_the_tip_statistics_unknown(self) -> None: + self.runs.write(self.runs.real, report("real", tips=(90.0, None, 150.0))) + _, row, _ = self.runs.row() + for statistic in ("p50", "max"): + entry = by_name(row)[f"time to usable work, slowest session, {statistic} over external tips"] + self.assertIsNone(entry["real"]) + self.assertIsNone(entry["real_minus_fake"]) + self.assertIn("tip 1: 3 of 100 sessions got no usable work", entry["unavailable_reason"]) + + def test_an_unmeasured_ack_latency_is_unknown_with_the_harness_reason(self) -> None: + self.runs.write(self.runs.fake, report("fake", ack=(None, None))) + _, row, _ = self.runs.row() + entry = by_name(row)["client_ack_latency p99 (steady_state)"] + self.assertIsNone(entry["fake"]) + self.assertIsNone(entry["real_minus_fake"]) + self.assertIn("no samples were recorded", entry["unavailable_reason"]) + + def test_a_missing_side_still_writes_the_row_and_marks_it_untrusted(self) -> None: + self.runs.write(self.runs.real, None, harness="6", gate="1") + code, row, summary = self.runs.row(order="real,fake") + self.assertEqual(code, 0) + self.assertFalse(row["trusted"]) + self.assertIn("real: load-harness-report.json is missing", row["untrusted_reasons"]) + self.assertIn("real: harness_exit_code is 6", row["untrusted_reasons"]) + self.assertIn("real: gate_exit_code is 1", row["untrusted_reasons"]) + for entry in row["metrics"]: + self.assertIsNone(entry["real"], entry["metric"]) + self.assertIsNone(entry["real_minus_fake"], entry["metric"]) + self.assertIn("**Not trusted:**", summary) + self.assertIn("order real then fake", summary) + + def test_a_gate_that_left_no_exit_code_is_missing_not_a_pass(self) -> None: + (self.runs.fake / "gate-exit-code").unlink() + _, row, _ = self.runs.row() + self.assertFalse(row["trusted"]) + self.assertIn("fake: gate_exit_code is missing", row["untrusted_reasons"]) + + def test_another_schema_or_preset_is_refused_by_name(self) -> None: + body = report("fake") + body["schema"] = "qbit.prism.load-harness.v0" + self.runs.write(self.runs.fake, body) + _, row, _ = self.runs.row() + self.assertIn(f"fake: load-harness-report.json has schema 'qbit.prism.load-harness.v0', not {bridge.REPORT_SCHEMA}", + row["untrusted_reasons"]) + swapped = copy.deepcopy(report("real")) + self.runs.write(self.runs.fake, swapped) + _, row, _ = self.runs.row() + self.assertIn("fake: load-harness-report.json is from preset 'short-plan-real-node', not short-plan-fake-node", + row["untrusted_reasons"]) + + def test_sides_on_different_runners_are_untrusted(self) -> None: + self.runs.write(self.runs.real, report("real"), runner="blacksmith-16vcpu-ubuntu-2404") + _, row, _ = self.runs.row() + self.assertFalse(row["trusted"]) + self.assertIsNone(row["runner"]) + + def test_an_order_other_than_the_two_sides_is_refused(self) -> None: + with contextlib.redirect_stderr(io.StringIO()): + self.assertEqual(bridge.main([ + "--fake", "f", "--real", "r", "--order", "fake,fake", "--out", str(self.runs.root / "x.json"), + ]), 2) + + +class CheckedInPair(unittest.TestCase): + """The row is only a node cost while the two presets differ only in --node.""" + + def test_the_presets_differ_only_in_node(self) -> None: + presets = matrix.load(matrix.PRESETS) + fake, real = presets[bridge.PRESETS["fake"]], presets[bridge.PRESETS["real"]] + self.assertEqual((fake["args"]["--node"], real["args"]["--node"]), ("fake", "qbitd")) + differing = sorted(key for key in fake["args"] if fake["args"][key] != real["args"].get(key)) + self.assertEqual(differing, ["--node"]) + self.assertEqual(set(fake["args"]), set(real["args"])) + for key in ("runner", "timeout_minutes", "gates"): + self.assertEqual(fake[key], real[key], key) + self.assertEqual(fake["args"]["--plan"], "short") + + +class Workflow(unittest.TestCase): + """The bridging job in prism-load-nightly.yml runs both halves on one runner.""" + + def setUp(self) -> None: + text = (ROOT / ".github/workflows/prism-load-nightly.yml").read_text(encoding="utf-8") + match = re.search(r"^ bridging:\n(.*?)(?=^ [a-z-]+:\n)", text, re.S | re.M) + self.assertIsNotNone(match, "prism-load-nightly.yml has no bridging job") + self.job = match.group(1) + self.text = text + + def test_it_runs_both_presets_and_the_row_in_one_job(self) -> None: + self.assertEqual(self.job.count(".github/scripts/prism-load-run.sh"), 1) + for preset in bridge.PRESETS.values(): + self.assertIn(preset, self.job) + self.assertIn("scripts/prism_load_bridge.py", self.job) + # The real half needs the pinned qbitd, as the matrix's real-node job does. + self.assertIn(".github/scripts/install-prism-qbit.sh", self.job) + + def test_each_half_runs_under_its_presets_own_timeout(self) -> None: + self.assertIn('minutes="$(jq -er .timeout_minutes "crates/qbit-prism-load/presets/${preset[${side}]}.json")"', self.job) + self.assertRegex(self.job, r'timeout --kill-after=\d+ "\$\{minutes\}m" \\\n\s+bash \.github/scripts/prism-load-run\.sh') + + def test_its_timeout_holds_both_presets_timeouts(self) -> None: + presets = matrix.load(matrix.PRESETS) + timeout = int(re.search(r"^ timeout-minutes: (\d+)$", self.job, re.M).group(1)) + both = sum(presets[name]["timeout_minutes"] for name in bridge.PRESETS.values()) + self.assertGreaterEqual(timeout, both + 10) + self.assertLessEqual(timeout, 360) + + def test_the_runner_is_the_presets_runner(self) -> None: + presets = matrix.load(matrix.PRESETS) + runner = re.search(r"^ runs-on: (\S+)$", self.job, re.M).group(1) + self.assertEqual(runner, presets[bridge.PRESETS["real"]]["runner"]) + # host.json records the label the job gives, so it must be this one. + self.assertEqual(re.search(r"^ RUNNER_LABEL: (\S+)$", self.job, re.M).group(1), runner) + + def test_a_bridge_that_reached_no_verdict_keeps_the_commit_untested(self) -> None: + self.assertIn("name: prism-load-gated-bridging", self.job) + self.assertIn('touch "${RUNNER_TEMP}/gated/bridging"', self.job) + report = re.search(r"^ report:\n(.*?)(?=^ [a-z-]+:\n)", self.text, re.S | re.M).group(1) + self.assertIn("echo bridging; } | while read -r preset", report) + self.assertIn("pattern: prism-load-gated-*", report) + + def test_a_failed_bridging_run_reaches_the_failure_report(self) -> None: + needs = re.search(r"^ report:\n(?:.*\n)*? needs: \[(.*)\]", self.text, re.M).group(1) + self.assertIn("bridging", [name.strip() for name in needs.split(",")]) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_prism_load_matrix.py b/tests/test_prism_load_matrix.py new file mode 100644 index 000000000..5e961f0c6 --- /dev/null +++ b/tests/test_prism_load_matrix.py @@ -0,0 +1,217 @@ +"""Tests for scripts/prism_load_matrix.py (#521).""" + +from __future__ import annotations + +import contextlib +import io +import json +from pathlib import Path +import sys +import tempfile +import unittest + +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT / "scripts")) + +import prism_load_matrix as matrix # noqa: E402 + + +class CheckedInPresets(unittest.TestCase): + def setUp(self) -> None: + self.presets = matrix.load(matrix.PRESETS) + + def test_the_schedule_runs_the_nightly_presets_and_not_the_manual_or_smoke_ones(self) -> None: + names = [entry["preset"] for entry in matrix.select(self.presets, "nightly")] + for name in ("throughput-20k-window-1fe", "tip-delivery-2000-miners-400k-2fe-retarget", "mainnet-shape-130-addresses", "mainnet-shape-650-addresses"): + self.assertIn(name, names) + self.assertNotIn("mainnet-shape-2600-addresses", names) + self.assertNotIn("pr-smoke", names) + + def test_the_weekly_schedule_runs_the_long_soak_and_the_long_fault_set(self) -> None: + entries = matrix.select(self.presets, "weekly") + self.assertEqual( + [entry["preset"] for entry in entries], ["faults-long-real-node", "soak-weekly"] + ) + # Each under the six-hour job limit, with the soak's own minutes inside it. + for entry in entries: + self.assertLess(entry["timeout_minutes"], 360) + soak = next(entry for entry in entries if entry["preset"] == "soak-weekly") + self.assertLess(self.presets["soak-weekly"]["soak"]["minutes"], soak["timeout_minutes"]) + nightly = [entry["preset"] for entry in matrix.select(self.presets, "nightly")] + everything = [entry["preset"] for entry in matrix.select(self.presets, "all")] + for name in ("soak-weekly", "faults-long-real-node"): + self.assertNotIn(name, nightly) + self.assertNotIn(name, everything) + + def test_all_adds_the_manual_presets(self) -> None: + names = [entry["preset"] for entry in matrix.select(self.presets, "all")] + self.assertIn("mainnet-shape-2600-addresses", names) + self.assertNotIn("pr-smoke", names) + + def test_a_named_preset_runs_on_its_own_runner_for_its_own_timeout(self) -> None: + [entry] = matrix.select(self.presets, "mainnet-shape-2600-addresses") + self.assertEqual(entry["runner"], self.presets["mainnet-shape-2600-addresses"]["runner"]) + self.assertEqual(entry["timeout_minutes"], self.presets["mainnet-shape-2600-addresses"]["timeout_minutes"]) + self.assertTrue(entry["runner"].startswith("blacksmith-")) + + def test_unknown_and_empty_selections_are_refused(self) -> None: + with self.assertRaises(matrix.SelectionError): + matrix.select(self.presets, "no-such-preset") + with self.assertRaises(matrix.SelectionError): + matrix.select(self.presets, " , ") + + def test_every_runner_label_is_known_to_actionlint(self) -> None: + config = (ROOT / ".github" / "actionlint.yaml").read_text(encoding="utf-8") + for preset in self.presets.values(): + self.assertIn(f"- {preset['runner']}", config, preset["name"]) + + +class Aliases(unittest.TestCase): + def test_every_deprecated_name_selects_the_preset_it_was_renamed_to(self) -> None: + presets = matrix.load(matrix.PRESETS) + aliases = matrix.load_aliases(matrix.PRESETS) + self.assertEqual(aliases["d1-20k"], "throughput-20k-window-1fe") + self.assertEqual(aliases["smoke"], "pr-smoke") + for old, new in aliases.items(): + self.assertIn(new, presets, old) + self.assertNotIn(old, presets, old) + [entry] = matrix.select(presets, old, aliases) + self.assertEqual(entry["preset"], new) + + def test_without_the_alias_file_an_old_name_is_unknown(self) -> None: + with self.assertRaises(matrix.SelectionError): + matrix.select(matrix.load(matrix.PRESETS), "tip-275", {}) + + +class Malformed(unittest.TestCase): + def test_a_preset_whose_name_is_not_its_file_is_refused(self) -> None: + with tempfile.TemporaryDirectory() as directory: + Path(directory, "a.json").write_text( + json.dumps({"schema": matrix.SCHEMA, "name": "b"}), encoding="utf-8" + ) + with self.assertRaises(matrix.SelectionError): + matrix.load(Path(directory)) + + +class Suites(unittest.TestCase): + """The checked-in suites (#550) and what a malformed one is refused for.""" + + def setUp(self) -> None: + self.presets = matrix.load(matrix.PRESETS) + self.suites = matrix.load_suites(matrix.PRESETS, self.presets) + + def test_the_full_l3_runs_every_473_cell_three_times_on_one_runner_class(self) -> None: + entries = matrix.select(self.presets, "suite:l3-full", suites=self.suites) + names = sorted({entry["preset"] for entry in entries}) + self.assertEqual(len(names), 11) + for name in ( + "throughput-200k-window-1fe-async", + "throughput-400k-window-1fe-async", + "throughput-400k-window-2fe-sync", + "throughput-400k-window-2fe-async-3-blocks", + "dense-cadence-400k-window-1fe-async", + "dense-cadence-400k-window-2fe-async", + "throughput-500k-window-4fe-async", + ): + self.assertIn(name, names) + self.assertEqual(len(entries), 33) + self.assertEqual({entry["runner"] for entry in entries}, {"blacksmith-32vcpu-ubuntu-2404"}) + self.assertEqual( + sorted(entry["id"] for entry in entries if entry["preset"] == names[0]), + [f"{names[0]}-r1", f"{names[0]}-r2", f"{names[0]}-r3"], + ) + # One job per repeat: every id is distinct. + self.assertEqual(len({entry["id"] for entry in entries}), 33) + for entry in entries: + self.assertEqual(entry["timeout_minutes"], self.presets[entry["preset"]]["timeout_minutes"]) + + def test_the_reduced_l3_is_400k_at_1_2_and_4_frontends_async_once(self) -> None: + entries = matrix.select(self.presets, "suite:l3-reduced", suites=self.suites) + names = [entry["preset"] for entry in entries] + for fe in (1, 2, 4): + self.assertIn(f"throughput-400k-window-{fe}fe-async", names) + self.assertEqual({entry["repeat"] for entry in entries}, {1}) + + def test_the_reduced_l3_carries_555s_session_points_and_the_mainnet_shapes(self) -> None: + names = [e["preset"] for e in matrix.select(self.presets, "suite:l3-reduced", suites=self.suites)] + for name in ( + "mainnet-shape-130-addresses", + "mainnet-shape-650-addresses", + "mainnet-shape-2600-addresses", + "rental-churn-bursts-and-storms", + ): + self.assertIn(name, names) + base = self.presets["throughput-400k-window-2fe-async"] + for sessions in (5000, 10000): + name = f"session-point-{sessions}-sessions-400k-2fe-async" + self.assertIn(name, names) + point = self.presets[name] + # The 2,000-session cell's workload but for the session count and + # its admission, sessions per frontend + 16. + differs = {flag for flag, value in point["args"].items() if base["args"][flag] != value} + self.assertEqual(differs, {"--sessions", "--stratum-max-pending-initial-jobs"}, name) + self.assertEqual(point["args"]["--sessions"], sessions) + self.assertEqual(point["args"]["--stratum-max-pending-initial-jobs"], sessions // 2 + 16) + self.assertEqual(point["args"]["--steady-state-rate"], 500) + # The D1 plan's burst is the 2,000/s-for-60-s burst variant. + self.assertEqual(point["args"]["--plan"], "d1") + self.assertEqual((point["args"]["--burst-rate"], point["args"]["--burst-seconds"]), (2000, 60)) + # A capacity-envelope point, not a D1 verdict. + self.assertFalse(point["gates"]["d1_verdict_table"], name) + + def test_a_suite_preset_can_keep_its_own_schedule(self) -> None: + # throughput-400k-window-1fe-async is nightly and in both L3 suites. + self.assertEqual(self.presets["throughput-400k-window-1fe-async"]["schedule"], "nightly") + nightly = [entry["preset"] for entry in matrix.select(self.presets, "nightly")] + self.assertIn("throughput-400k-window-1fe-async", nightly) + self.assertNotIn("repeat", matrix.select(self.presets, "nightly")[0]) + + def test_every_suite_runner_is_known_to_actionlint(self) -> None: + config = (ROOT / ".github" / "actionlint.yaml").read_text(encoding="utf-8") + for name, suite in self.suites.items(): + if "runner" in suite: + self.assertIn(f"- {suite['runner']}", config, name) + + def test_an_unknown_suite_is_refused(self) -> None: + with self.assertRaises(matrix.SelectionError): + matrix.select(self.presets, "suite:l3-nope", suites=self.suites) + + def test_a_malformed_suite_is_refused_and_only_a_suite_selection_reads_the_file(self) -> None: + good = ( + 'schema = "qbit.prism.load-suites.v1"\n[suites.s]\ndescription = "d"\nlane = "L3"\n' + 'issues = ["#550"]\nrepeats = 2\npresets = ["pr-smoke"]\n' + ) + cases = { + "schema": good.replace("load-suites.v1", "load-suites.v0"), + "unknown key": good + "extra = 1\n", + "missing key": good.replace('lane = "L3"\n', ""), + "unknown preset": good.replace('["pr-smoke"]', '["no-such-preset"]'), + "alias": good.replace('["pr-smoke"]', '["smoke"]'), + "repeated preset": good.replace('["pr-smoke"]', '["pr-smoke", "pr-smoke"]'), + "zero repeats": good.replace("repeats = 2", "repeats = 0"), + "too many repeats": good.replace("repeats = 2", "repeats = 6"), + "boolean repeats": good.replace("repeats = 2", "repeats = true"), + "runner": good + 'runner = "ubuntu-latest"\n', + "issue": good.replace('["#550"]', '["550"]'), + } + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + for preset in matrix.PRESETS.glob("*.json"): + (root / preset.name).write_bytes(preset.read_bytes()) + (root / "suites.toml").write_text(good, encoding="utf-8") + presets = matrix.load(root) + self.assertEqual(len(matrix.select(presets, "suite:s", suites=matrix.load_suites(root, presets))), 2) + for name, text in cases.items(): + with self.subTest(case=name): + (root / "suites.toml").write_text(text, encoding="utf-8") + with self.assertRaises(matrix.SelectionError): + matrix.load_suites(root, presets) + # A broken suites file never stops the nightly's plan. + with contextlib.redirect_stdout(io.StringIO()): + self.assertEqual(matrix.main(["nightly", "--presets", str(root)]), 0) + with contextlib.redirect_stderr(io.StringIO()): + self.assertEqual(matrix.main(["suite:s", "--presets", str(root)]), 2) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_prism_load_probe.py b/tests/test_prism_load_probe.py new file mode 100644 index 000000000..5217e4ebe --- /dev/null +++ b/tests/test_prism_load_probe.py @@ -0,0 +1,739 @@ +"""Tests for scripts/prism_load_probe.py (#541, #542).""" + +from __future__ import annotations + +import contextlib +import io +import json +from pathlib import Path +import sys +import tempfile +import unittest + +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT / "scripts")) + +import prism_load_matrix # noqa: E402 +import prism_load_probe as probe # noqa: E402 + +PG_TEST_FSYNC = """\ +5 seconds per test +O_DIRECT supported on this platform for open_datasync and open_sync. + +Compare file sync methods using one 8kB write: +(in wal_sync_method preference order, except fdatasync is Linux's default) + open_datasync 2051.170 ops/sec 488 usecs/op + fdatasync 2034.433 ops/sec 492 usecs/op + fsync 1011.234 ops/sec 989 usecs/op + fsync_writethrough n/a + open_sync 1003.030 ops/sec 997 usecs/op + +Compare file sync methods using two 8kB writes: +(in wal_sync_method preference order, except fdatasync is Linux's default) + open_datasync 1025.100 ops/sec 976 usecs/op + fdatasync 1900.000 ops/sec 526 usecs/op +""" + + +class Matrix(unittest.TestCase): + def test_every_class_runs_each_backend_and_the_fsync_vms(self) -> None: + result = probe.matrices("8, 16,32", "both", "3") + self.assertEqual(len(result["probe"]["include"]), 6) + self.assertEqual( + {(e["class"], e["target_cache"]) for e in result["probe"]["include"]}, + {(c, t) for c in (8, 16, 32) for t in probe.TARGET_CACHES}, + ) + self.assertEqual(result["probe"]["include"][0]["runner"], "blacksmith-8vcpu-ubuntu-2404") + self.assertEqual(len(result["fsync"]["include"]), 9) + self.assertTrue(result["fsync_enabled"]) + + def test_no_fsync_vms_leaves_a_valid_matrix_and_a_false_flag(self) -> None: + result = probe.matrices("8", "sticky-disk", "0") + self.assertEqual(len(result["fsync"]["include"]), 1) + self.assertFalse(result["fsync_enabled"]) + self.assertEqual([e["target_cache"] for e in result["probe"]["include"]], ["sticky-disk"]) + + def test_inputs_outside_what_the_workflow_knows_are_refused(self) -> None: + for classes, cache, vms in ( + ("12", "both", "1"), + ("8,x", "both", "1"), + (" , ", "both", "1"), + ("8", "s3", "1"), + ("8", "both", "11"), + ("8", "both", "-1"), + ): + with self.subTest(classes=classes, cache=cache, vms=vms): + with self.assertRaises(probe.ProbeError): + probe.matrices(classes, cache, vms) + + def test_the_probe_presets_are_checked_in(self) -> None: + presets = prism_load_matrix.load(prism_load_matrix.PRESETS) + for name in probe.PRESETS: + self.assertIn(name, presets) + self.assertEqual(presets["short-plan-20k-window-1fe"]["args"]["--plan"], "short") + self.assertEqual(presets["throughput-20k-window-1fe"]["args"]["--plan"], "d1") + for name in probe.PRESETS: + self.assertEqual(presets[name]["args"]["--window-shares"], 20000, name) + + def test_short_20k_is_not_in_the_nightly_set(self) -> None: + presets = prism_load_matrix.load(prism_load_matrix.PRESETS) + names = [e["preset"] for e in prism_load_matrix.select(presets, "nightly")] + self.assertNotIn("short-plan-20k-window-1fe", names) + + +class PgTestFsync(unittest.TestCase): + def test_the_one_write_fdatasync_line_is_the_flush_cost(self) -> None: + self.assertEqual( + probe.parse_pg_test_fsync(PG_TEST_FSYNC), + {"method": "fdatasync", "ops_per_second": 2034.433, "usecs_per_op": 492.0}, + ) + + def test_a_run_marked_failed_after_the_one_write_section_is_unknown(self) -> None: + for marker in ("pg_test_fsync failed\n", "pg_test_fsync failed; see above\n"): + parsed = probe.parse_pg_test_fsync(PG_TEST_FSYNC + marker) + self.assertIsNone(parsed["ops_per_second"], marker) + self.assertIsNone(parsed["usecs_per_op"], marker) + + def test_a_failed_run_is_unknown_not_zero(self) -> None: + parsed = probe.parse_pg_test_fsync("pg_test_fsync failed; see above\n") + self.assertIsNone(parsed["ops_per_second"]) + self.assertIsNone(parsed["usecs_per_op"]) + + +class Measure(unittest.TestCase): + def test_exit_code_wall_time_and_memory_are_recorded(self) -> None: + result = probe.measure([sys.executable, "-c", "import sys; sys.exit(3)"], interval=0.05) + self.assertEqual(result["exit_code"], 3) + self.assertGreaterEqual(result["wall_seconds"], 0) + if Path("/proc/meminfo").exists(): + self.assertGreater(result["peak_used_mib"], 0) + self.assertGreaterEqual(result["peak_used_mib"], result["baseline_used_mib"]) + + def test_meminfo_parsing(self) -> None: + info = probe.meminfo_kib("MemTotal: 1000 kB\nMemAvailable: 250 kB\nHugePages_Total: 0\n") + self.assertEqual(probe.used_kib(info), 750) + self.assertIsNone(probe.used_kib({"MemTotal": 1})) + + +def report(**overrides) -> dict: + document = { + "dirty": False, + "artifact_kind": "qualification", + "versions": { + "server_build_profile": "release", + "server_revision_evidence": {"status": "established"}, + }, + "window": {"seed": {"seconds": 12.5, "rows": 20000}}, + "phases": [ + {"name": "warm_up", "duration_seconds": 30.0, "shortfall": 0, "completed": True}, + {"name": "steady_state", "duration_seconds": 60.0, "shortfall": 2, "completed": True}, + ], + "time_to_usable_work": {"tips": [ + {"all_sessions_milliseconds": 100}, + {"all_sessions_milliseconds": 300}, + {"all_sessions_milliseconds": None}, + ]}, + } + document.update(overrides) + return document + + +class Host(unittest.TestCase): + def test_the_runner_label_leads_the_fingerprint_when_given(self) -> None: + # #549: each nightly preset's artifact names the runner it asked for + # beside the disk facts. + with tempfile.TemporaryDirectory() as tmp: + out = Path(tmp) / "host.json" + self.assertEqual(probe.main(["host", "--out", str(out), "--dir", tmp, + "--runner", "blacksmith-8vcpu-ubuntu-2404"]), 0) + facts = json.loads(out.read_text()) + self.assertEqual(next(iter(facts)), "runner_label") + self.assertEqual(facts["runner_label"], "blacksmith-8vcpu-ubuntu-2404") + self.assertIn("block_device", facts) + self.assertIn("options", facts["filesystem"]) + + self.assertEqual(probe.main(["host", "--out", str(out), "--dir", tmp]), 0) + self.assertNotIn("runner_label", json.loads(out.read_text())) + + +class Rows(unittest.TestCase): + def test_provenance_names_each_bypassed_check(self) -> None: + self.assertEqual(probe.provenance(report()), "pass") + self.assertTrue(probe.provenance(None).startswith("no report")) + verdict = probe.provenance(report( + dirty=True, + artifact_kind="example", + versions={"server_build_profile": "debug", + "server_revision_evidence": {"status": "unestablished"}}, + )) + for part in ("dirty tree", "revision unestablished", "server not release", + "artifact_kind example"): + self.assertIn(part, verdict) + + def test_a_run_directory_becomes_its_result(self) -> None: + with tempfile.TemporaryDirectory() as directory: + run = Path(directory) + (run / "load-harness-report.json").write_text(json.dumps(report())) + (run / "harness-exit-code").write_text("0\n") + (run / "probe-measure.json").write_text(json.dumps( + {"exit_code": 1, "wall_seconds": 400.0, "peak_used_mib": 9000.0, + "baseline_used_mib": 1000.0})) + (run / "pg_test_fsync.txt").write_text(PG_TEST_FSYNC) + result = probe.run_result(run) + self.assertEqual(result["harness_exit_code"], 0) + self.assertEqual(result["gate_exit_code"], 1) + self.assertEqual(result["seed_seconds"], 12.5) + self.assertEqual(result["phases"]["steady_state"]["shortfall"], 2) + # A tip some session never got work on leaves the p99 unmeasured, + # as the gate has it, while the missing count is kept. + self.assertIsNone(result["tip_last_notify_p99_ms"]) + self.assertEqual(result["tips_missing_a_session"], 1) + self.assertEqual(result["fsync"]["usecs_per_op"], 492.0) + self.assertEqual(result["provenance"], "pass") + + def test_a_run_that_never_started_is_unknown_throughout(self) -> None: + with tempfile.TemporaryDirectory() as directory: + result = probe.run_result(Path(directory, "missing")) + self.assertIsNone(result["harness_exit_code"]) + self.assertIsNone(result["gate_exit_code"]) + self.assertIsNone(result["seed_seconds"]) + self.assertIsNone(result["tip_last_notify_p99_ms"]) + + def test_the_tip_p99_is_measured_when_every_tip_was_served(self) -> None: + served = {"tips": [{"all_sessions_milliseconds": 100}, + {"all_sessions_milliseconds": 300}]} + with tempfile.TemporaryDirectory() as directory: + Path(directory, "load-harness-report.json").write_text( + json.dumps(report(time_to_usable_work=served))) + result = probe.run_result(Path(directory)) + self.assertEqual(result["tip_last_notify_p99_ms"], 300) + self.assertEqual(result["tips_missing_a_session"], 0) + + def test_nearest_rank_matches_the_gate(self) -> None: + self.assertEqual(probe.nearest_rank([5, 1, 3], 0.99), 5) + self.assertEqual(probe.nearest_rank(list(range(1, 101)), 0.99), 99) + self.assertIsNone(probe.nearest_rank([], 0.99)) + + +class Table(unittest.TestCase): + def test_rows_on_disk_become_one_table(self) -> None: + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + run = root / "short-plan-20k-window-1fe" + run.mkdir() + (run / "load-harness-report.json").write_text(json.dumps(report())) + (run / "pg_test_fsync.txt").write_text(PG_TEST_FSYNC) + rows = root / "rows" + rows.mkdir() + for path in (root / "build.json", root / "host.json"): + path.write_text("{}") + status = probe.main([ + "row", "--out", str(rows / "8-sticky-disk.json"), "--class", "8", + "--runner", "blacksmith-8vcpu-ubuntu-2404", "--target-cache", "sticky-disk", + "--target-restored", "false", "--target-bytes-before", "0", + "--target-bytes-after", "", "--restore-seconds", "1.5", "--save-seconds", "", + "--build", str(root / "build.json"), "--host", str(root / "host.json"), + "--run", f"short-plan-20k-window-1fe={run}", + ]) + self.assertEqual(status, 0) + for vm, ops in ((1, "2034.433"), (2, "1017.2")): + fsync = root / f"fsync-{vm}.txt" + fsync.write_text(PG_TEST_FSYNC.replace("2034.433", ops)) + self.assertEqual(probe.main([ + "fsync-row", "--out", str(rows / f"fsync-8-{vm}.json"), "--class", "8", + "--runner", "blacksmith-8vcpu-ubuntu-2404", "--vm", str(vm), + "--host", str(root / "host.json"), "--fsync", str(fsync), + ]), 0) + (rows / "unrelated.json").write_text(json.dumps({"schema": "other"})) + text = probe.table(probe.load_rows(rows)) + self.assertIn("### 8 vCPU · sticky-disk · repeat 1 (cold target)", text) + self.assertIn("2,034 ops/s", text) + # Without the plan, the columns are the presets the rows ran. + self.assertIn("| Measure | short-plan-20k-window-1fe |\n", text) + self.assertNotIn("throughput-20k-window-1fe", text) + self.assertIn("steady_state 2", text) + self.assertIn("over 2 VMs, max/min 2.00", text) + + def test_a_vm_whose_pg_test_fsync_failed_is_counted_as_unknown(self) -> None: + self.assertEqual( + probe.spread([2000.0, None, 1000.0]), + "1,000–2,000 over 2 VMs, max/min 2.00; 1 of 3 VMs unknown (pg_test_fsync failed or " + "the job wrote no row)", + ) + self.assertEqual(probe.spread([None, None]), + "unknown; 2 of 2 VMs unknown (pg_test_fsync failed or the job wrote no row)") + + def test_a_planned_job_that_wrote_no_row_is_named_and_counted(self) -> None: + expected = probe.matrices("8,16", "sticky-disk", "3") + rows = [ + {"schema": probe.ROW_SCHEMA, "kind": "fsync", "class": 8, "vm": vm, + "runner": "r", "host": {}, "fsync": {"ops_per_second": ops}} + for vm, ops in ((1, 2000.0), (2, 1000.0)) + ] + text = probe.table(rows, expected) + for size in (8, 16): + self.assertIn(f"### {size} vCPU · sticky-disk · repeat 1: no row", text) + self.assertIn("| 8 vCPU | 1,000–2,000 over 2 VMs, max/min 2.00; 1 of 3 VMs unknown", text) + self.assertIn("| 16 vCPU | unknown; 3 of 3 VMs unknown", text) + # With the plan, a planned run a row lacks is shown as not run. + row = probe_row(1, [run()]) + row["class"] = 8 + planned = probe.matrices("8", "sticky-disk", "0", f"{PRESET},short-plan-20k-window-1fe") + text = probe.table([row], planned) + self.assertIn(f"| Measure | {PRESET} | short-plan-20k-window-1fe |", text) + self.assertIn("| not run |", text) + # With no fsync jobs planned, none is expected. + self.assertNotIn("fdatasync across VMs", probe.table([], probe.matrices("8", "both", "0"))) + + def test_the_run_argument_names_a_probe_preset(self) -> None: + with self.assertRaises(probe.ProbeError): + probe.parse_run("growth-20x=/tmp/x") + with self.assertRaises(probe.ProbeError): + probe.parse_run("short-plan-20k-window-1fe") + + +class RepeatPlan(unittest.TestCase): + """#542: repeats of one commit per class, as separate jobs and in-job.""" + + def test_repeats_are_separate_jobs_and_the_job_timeout_covers_every_run(self) -> None: + plan = probe.matrices("8,32", "sticky-disk", "0", "throughput-20k-window-1fe", + "10", "2") + self.assertEqual(len(plan["probe"]["include"]), 20) + self.assertEqual({e["repeat"] for e in plan["probe"]["include"]}, set(range(1, 11))) + self.assertEqual(plan["presets"], "throughput-20k-window-1fe") + self.assertEqual(plan["in_job_repeats"], 2) + # Two 60-minute runs plus slack and each run's cleanup, inside setup, + # build and reports. + self.assertEqual(plan["runs_minutes"], 2 * 60 + probe.RUN_STEP_SLACK_MINUTES + + 2 * probe.CLEANUP_MINUTES_PER_RUN) + self.assertEqual(plan["job_minutes"], probe.SETUP_MINUTES + probe.BUILD_MINUTES + + plan["runs_minutes"] + probe.REPORT_MINUTES) + ceiling = plan["ceiling"] + self.assertEqual(ceiling["probe_jobs"], 20) + self.assertEqual(ceiling["two_vcpu_minutes"], + 10 * plan["job_minutes"] * (4 + 16) + probe.OVERHEAD_JOB_MINUTES) + self.assertAlmostEqual(ceiling["usd"], round(ceiling["two_vcpu_minutes"] * 0.004, 2)) + self.assertIn("Spend ceiling", probe.plan_summary(plan)) + + def test_the_default_plan_is_541s(self) -> None: + plan = probe.matrices("8,16,32", "both", "3") + self.assertEqual(plan["presets"], ",".join(probe.PRESETS)) + self.assertEqual(plan["in_job_repeats"], 1) + self.assertEqual({e["repeat"] for e in plan["probe"]["include"]}, {1}) + + def test_a_deprecated_name_runs_its_preset_and_duplicates_collapse(self) -> None: + with contextlib.redirect_stderr(io.StringIO()) as warned: + plan = probe.matrices("16", "sticky-disk", "0", + "d1-473-400k-fe2-async, throughput-400k-window-2fe-async," + "throughput-400k-window-4fe-async") + self.assertIn("d1-473-400k-fe2-async is deprecated", warned.getvalue()) + self.assertEqual(plan["presets"], + "throughput-400k-window-2fe-async,throughput-400k-window-4fe-async") + + def test_plans_the_workflow_cannot_run_are_refused_before_any_runner(self) -> None: + for presets, repeats, in_job, needle in ( + ("throughput-20k-window-1fe", "0", "1", "repeats"), + ("throughput-20k-window-1fe", "21", "1", "repeats"), + ("throughput-20k-window-1fe", "1.5", "1", "repeats"), + ("throughput-20k-window-1fe", "+2", "1", "repeats"), + ("throughput-20k-window-1fe", "", "1", "repeats"), + ("throughput-20k-window-1fe", "1", "6", "in-job repeats"), + ("throughput-20k-window-1fe", "1", "0", "in-job repeats"), + ("no-such-preset", "1", "1", "unknown preset"), + (" , ", "1", "1", "names no preset"), + ("short-plan-real-node", "1", "1", "real node"), + ("soak-weekly", "1", "1", "ceiling"), + ("throughput-20k-window-1fe,throughput-400k-window-1fe-async," + "throughput-400k-window-2fe-async", "1", "2", "ceiling"), + ("throughput-20k-window-1fe", "11", "1", "over 64"), + ): + with self.subTest(presets=presets, repeats=repeats, in_job=in_job): + with self.assertRaises(probe.ProbeError) as raised: + probe.matrices("8,16,32", "both", "0", presets, repeats, in_job) + self.assertIn(needle, str(raised.exception)) + + def test_the_cli_prints_every_output_the_workflow_reads(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + summary = Path(tmp) / "summary.md" + out = io.StringIO() + with contextlib.redirect_stdout(out): + status = probe.main(["matrix", "--classes", "8", "--target-cache", "sticky-disk", + "--fsync-vms", "0", "--repeats", "3", + "--summary", str(summary)]) + self.assertEqual(status, 0) + outputs = dict(line.split("=", 1) for line in out.getvalue().splitlines()) + keys = set(outputs) + # The workflow splits the preset list on commas, so it is bare text. + self.assertEqual(outputs["presets"], ",".join(probe.PRESETS)) + self.assertEqual(json.loads(outputs["job_minutes"]), 202) + self.assertEqual(json.loads(outputs["probe"])["include"][2]["repeat"], 3) + self.assertEqual(keys, {"probe", "fsync", "fsync_enabled", "presets", + "in_job_repeats", "runs_minutes", "job_minutes", "ceiling"}) + self.assertIn("3 probe job(s)", summary.read_text()) + with contextlib.redirect_stderr(io.StringIO()): + self.assertEqual(probe.main(["matrix", "--classes", "8", "--target-cache", + "both", "--fsync-vms", "0", "--repeats", "x"]), 2) + + +class MeasureTimeout(unittest.TestCase): + def test_a_timeout_ends_the_whole_process_group(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + pid_file = Path(tmp) / "grandchild" + script = f"sleep 120 & echo $! > {pid_file}; wait" + result = probe.measure(["bash", "-c", script], interval=0.05, timeout_seconds=1) + self.assertTrue(result["timed_out"]) + self.assertNotEqual(result["exit_code"], 0) + grandchild = int(pid_file.read_text()) + # Gone, or a zombie a slow PID 1 has not reaped yet: not running. + state = probe.process_state(grandchild) + self.assertTrue(state is None or state[0] in ("Z", "X"), state) + self.assertFalse(probe.group_alive(grandchild)) + + def test_the_cli_exits_124_on_a_timeout(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + out = Path(tmp) / "m.json" + status = probe.main(["measure", "--out", str(out), "--timeout-seconds", "0.5", + "--", "sleep", "30"]) + self.assertEqual(status, 124) + self.assertTrue(json.loads(out.read_text())["timed_out"]) + status = probe.main(["measure", "--out", str(out), "--", "true"]) + self.assertEqual(status, 0) + self.assertFalse(json.loads(out.read_text())["timed_out"]) + + +def write_run(directory: Path, harness: int = 0, **report_overrides) -> Path: + directory.mkdir(parents=True, exist_ok=True) + (directory / "load-harness-report.json").write_text(json.dumps(report(**report_overrides))) + (directory / "harness-exit-code").write_text(f"{harness}\n") + (directory / "probe-measure.json").write_text(json.dumps( + {"exit_code": 0, "timed_out": False, "wall_seconds": 400.0, "peak_used_mib": 9000.0, + "baseline_used_mib": 1000.0, "mem_total_mib": 32000.0})) + (directory / "pg_test_fsync.txt").write_text(PG_TEST_FSYNC) + return directory + + +class RowsV2(unittest.TestCase): + def test_a_preset_given_again_is_its_next_in_job_repeat(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + first = write_run(root / "a" / "1") + second = write_run(root / "a" / "2", harness=6) + other = write_run(root / "b" / "1") + for path in (root / "build.json", root / "host.json"): + path.write_text("{}") + out = root / "row.json" + self.assertEqual(probe.main([ + "row", "--out", str(out), "--class", "32", "--runner", "r", + "--target-cache", "sticky-disk", "--target-restored", "true", + "--repeat", "4", "--commit", "abc", + "--build", str(root / "build.json"), "--host", str(root / "host.json"), + "--run", f"throughput-20k-window-1fe={first}", + "--run", f"throughput-20k-window-1fe={second}", + "--run", f"throughput-400k-window-2fe-async={other}", + ]), 0) + row = json.loads(out.read_text()) + self.assertEqual(row["schema"], probe.ROW_SCHEMA) + self.assertEqual((row["repeat"], row["commit"]), (4, "abc")) + runs = row["runs"]["throughput-20k-window-1fe"] + self.assertEqual(len(runs), 2) + self.assertIsNone(runs[0]["unmeasured_reason"]) + self.assertEqual(runs[1]["unmeasured_reason"], "harness exit 6") + self.assertEqual(runs[0]["headroom_mib"], 23000.0) + self.assertEqual(len(row["runs"]["throughput-400k-window-2fe-async"]), 1) + + def test_a_run_argument_must_name_a_checked_in_preset(self) -> None: + known = {"throughput-400k-window-2fe-async"} + self.assertEqual(probe.parse_run("throughput-400k-window-2fe-async=/x", known)[0], + "throughput-400k-window-2fe-async") + for text in ("no-such=/x", "throughput-400k-window-2fe-async=", "x"): + with self.subTest(text=text), self.assertRaises(probe.ProbeError): + probe.parse_run(text, known) + + def test_phase_latency_and_rate_are_read_only_in_their_units(self) -> None: + phases = [{"name": "steady_state", "shortfall": 0, "duration_seconds": 300.0, + "completed": True, "achieved_rate_shares_per_second": 499.5, + "client_ack_latency": {"unit": "milliseconds", "p50": 3.0, "p99": 12.5}}, + {"name": "burst", "shortfall": 10, "achieved_rate_shares_per_second": None, + "client_ack_latency": {"unit": "seconds", "p50": 0.1, "p99": 0.5}}] + with tempfile.TemporaryDirectory() as tmp: + result = probe.run_result(write_run(Path(tmp), phases=phases)) + steady, burst = result["phases"]["steady_state"], result["phases"]["burst"] + self.assertEqual((steady["ack_p50_ms"], steady["ack_p99_ms"]), (3.0, 12.5)) + self.assertEqual(steady["achieved_rate_shares_per_second"], 499.5) + self.assertIsNone(burst["ack_p99_ms"]) + self.assertIsNone(burst["achieved_rate_shares_per_second"]) + + def test_what_makes_a_run_a_measurement(self) -> None: + base = {"harness_exit_code": 0, "provenance": "pass", "timed_out": False, + "gate_exit_code": 0} + self.assertIsNone(probe.unmeasured_reason(base)) + # A failed gate still measured the run. + self.assertIsNone(probe.unmeasured_reason(dict(base, gate_exit_code=1))) + self.assertEqual(probe.unmeasured_reason(dict(base, timed_out=True)), "timed out") + self.assertEqual(probe.unmeasured_reason(dict(base, harness_exit_code=None)), + "no harness exit code") + self.assertEqual(probe.unmeasured_reason(dict(base, harness_exit_code=3)), + "harness exit 3") + self.assertTrue(probe.unmeasured_reason(dict(base, provenance="fail: dirty tree")) + .startswith("provenance")) + # Only a gate verdict (0 or 1) makes the run a sample. + for gate in (2, 101, None, -15): + self.assertEqual(probe.unmeasured_reason(dict(base, gate_exit_code=gate)), + f"gate exit {gate}") + + def test_v1_rows_are_read_and_unknown_schemas_refused(self) -> None: + v1 = {"schema": probe.ROW_SCHEMA_V1, "kind": "probe", "class": 8, "runner": "r", + "target_cache": "sticky-disk", "runs": {"throughput-20k-window-1fe": {"x": 1}}} + with tempfile.TemporaryDirectory() as tmp: + Path(tmp, "v1.json").write_text(json.dumps(v1)) + (row,) = probe.load_rows(Path(tmp)) + self.assertEqual(row["schema"], probe.ROW_SCHEMA) + self.assertEqual(row["repeat"], 1) + self.assertIsNone(row["commit"]) + # A v1 run with no harness exit code is unknown, not a sample. + self.assertEqual(row["runs"]["throughput-20k-window-1fe"], + [{"x": 1, "unmeasured_reason": "no harness exit code"}]) + Path(tmp, "v9.json").write_text(json.dumps( + dict(v1, schema="qbit.prism.runner-probe-row.v9"))) + with self.assertRaises(probe.ProbeError): + probe.load_rows(Path(tmp)) + + +PRESET = "throughput-20k-window-1fe" + + +def run(rate: float | None = 500.0, ack99: float = 10.0, peak: float = 9000.0, + usecs: float | None = 492.0, harness: int | None = 0, steady_shortfall: int = 0, + burst_shortfall: int = 50, total: float = 32000.0) -> dict: + """A run as run_result writes it.""" + result = { + "harness_exit_code": harness, "gate_exit_code": 0, "timed_out": False, + "wall_seconds": 600.0, "peak_used_mib": peak, "baseline_used_mib": 1000.0, + "mem_total_mib": total, "headroom_mib": total - peak, + "fsync": {"method": "fdatasync", "ops_per_second": None, "usecs_per_op": usecs}, + "provenance": "pass", "seed_seconds": 10.0, "seed_rows": 20000, + "phases": { + "steady_state": {"shortfall": steady_shortfall, "ack_p50_ms": 2.0, + "ack_p99_ms": ack99, "achieved_rate_shares_per_second": rate}, + # D1 presets gate steady_state only; the burst's shortfall is + # reported, not gated (#473). + "burst": {"shortfall": burst_shortfall, "ack_p50_ms": 5.0, "ack_p99_ms": 900.0, + "achieved_rate_shares_per_second": 1500.0}, + }, + "tip_last_notify_p99_ms": 800, "tips_missing_a_session": 0, + } + result["unmeasured_reason"] = probe.unmeasured_reason(result) + return result + + +def probe_row(repeat: int, runs: list[dict], size: int = 8, commit: str | None = "c0ffee", + write_cache: str = "write back") -> dict: + return {"schema": probe.ROW_SCHEMA, "kind": "probe", "class": size, + "runner": probe.runner_label(size), "target_cache": "sticky-disk", + "repeat": repeat, "commit": commit, "target": {}, "build": {}, + "host": {"block_device": {"write_cache": write_cache, "fua": "0", "model": "m"}}, + "runs": {PRESET: runs}} + + +def group(document: dict, size: int, band: str, preset: str = PRESET) -> dict: + (found,) = [g for g in document["groups"] + if (g["class"], g["preset"], g["fsync_band"]) == (size, preset, band)] + return found + + +class Variance(unittest.TestCase): + def setUp(self) -> None: + self.presets, _ = probe.load_presets() + + def expected(self, repeats: int, in_job: int = 2, classes: str = "8") -> dict: + return probe.matrices(classes, "sticky-disk", "0", PRESET, str(repeats), str(in_job)) + + def test_bands(self) -> None: + self.assertEqual(probe.fsync_band(None), "unknown") + self.assertEqual(probe.fsync_band(100), "0-125us") + self.assertEqual(probe.fsync_band(125), "125-250us") + self.assertEqual(probe.fsync_band(492), "250-500us") + self.assertEqual(probe.fsync_band(4000), "ge4000us") + + def test_statistics(self) -> None: + result = probe.stats([10.0, 12.0, 14.0, 100.0], planned=6) + self.assertEqual(result["n"], 4) + self.assertEqual(result["unknown"], 2) + self.assertEqual(result["median"], 13.0) + # |x - 13| = 3, 1, 1, 87: median 2. + self.assertEqual(result["mad"], 2.0) + # Sample standard deviation 44.03 over a mean of 34. + self.assertAlmostEqual(result["cv"], 1.295, places=3) + self.assertEqual((result["min"], result["max"]), (10.0, 100.0)) + self.assertIsNone(probe.stats([5.0])["cv"]) + self.assertIsNone(probe.stats([0.0, 0.0])["cv"]) + self.assertEqual(probe.stats([0.0, 0.0])["mad"], 0.0) + empty = probe.stats([], planned=3) + self.assertEqual((empty["n"], empty["unknown"], empty["median"]), (0, 3, None)) + + def test_missing_and_failed_runs_are_unknown_never_zero(self) -> None: + rows = [ + probe_row(1, [run(rate=500.0), run(rate=490.0)]), + probe_row(2, [run(rate=480.0), run(rate=None, harness=6)]), + probe_row(3, [run(rate=510.0, usecs=1500.0), run(rate=505.0, usecs=1500.0)]), + ] + # Repeat 4 was planned and wrote no row. + document = probe.variance(rows, self.expected(4), self.presets) + self.assertEqual(document["schema"], probe.VARIANCE_SCHEMA) + self.assertEqual(document["commit"], "c0ffee") + self.assertEqual(document["jobs"]["planned"], 4) + self.assertEqual(document["jobs"]["missing"], + [{"class": 8, "target_cache": "sticky-disk", "repeat": 4}]) + everything = group(document, 8, "all") + self.assertEqual(everything["runs"], { + "planned": 8, "measured": 5, "unknown": 3, + "unknown_reasons": {"job wrote no row": 2, "harness exit 6": 1}}) + rate = everything["metrics"]["steady_state.achieved_rate_shares_per_second"] + self.assertEqual(rate["unit"], probe.METRIC_UNITS["achieved_rate_shares_per_second"]) + self.assertEqual(rate["all"]["n"], 5) + self.assertEqual(rate["all"]["unknown"], 3) + self.assertEqual(rate["all"]["median"], 500.0) + self.assertEqual(rate["all"]["min"], 480.0) + # VM-to-VM: one median per VM that measured (495, 480, 507.5), the + # missing VM unknown. + self.assertEqual(rate["vm_to_vm"]["n"], 3) + self.assertEqual(rate["vm_to_vm"]["unknown"], 1) + self.assertEqual(rate["vm_to_vm"]["median"], 495.0) + # Run-to-run: only VMs with two measured runs. + self.assertEqual(rate["run_to_run"]["jobs"], 2) + self.assertEqual(rate["run_to_run"]["median_mad"], 3.75) + # Bands: two VMs at ~0.5 ms, one at 1.5 ms, the missing one unknown. + self.assertEqual(group(document, 8, "250-500us")["runs"]["planned"], 4) + self.assertEqual(group(document, 8, "1000-2000us")["runs"]["measured"], 2) + self.assertEqual(group(document, 8, "unknown")["runs"]["unknown_reasons"], + {"job wrote no row": 2}) + self.assertIsNone(group(document, 8, "unknown")["size_evidence"]) + evidence = everything["size_evidence"] + self.assertIsNone(evidence["fits"]) + self.assertIn("3 of 8 run(s) unknown", evidence["why"]) + self.assertEqual(everything["disks"], [{"write_cache": "write back", "fua": "0", + "model": "m", "vms": 3}]) + self.assertEqual({vm["fsync_band"] for vm in document["jobs"]["by_vm"]}, + {"250-500us", "1000-2000us", "unknown"}) + + def test_a_class_fits_only_on_zero_gated_shortfall_and_headroom(self) -> None: + rows = [probe_row(r, [run(), run()]) for r in (1, 2)] + evidence = group(probe.variance(rows, self.expected(2), self.presets), 8, "all")[ + "size_evidence"] + # The burst's shortfall is not gated on the D1 preset. + self.assertEqual(evidence["gated_phases"], ["steady_state"]) + self.assertEqual(evidence["gated_shortfall_max"], 0) + self.assertEqual(evidence["headroom_mib_required"], 8000.0) + self.assertTrue(evidence["fits"], evidence["why"]) + + rows[1]["runs"][PRESET][0] = run(steady_shortfall=3) + evidence = group(probe.variance(rows, self.expected(2), self.presets), 8, "all")[ + "size_evidence"] + self.assertFalse(evidence["fits"]) + self.assertEqual(evidence["runs_with_gated_shortfall"], 1) + + rows[1]["runs"][PRESET][0] = run(peak=26000.0) + evidence = group(probe.variance(rows, self.expected(2), self.presets), 8, "all")[ + "size_evidence"] + self.assertFalse(evidence["fits"]) + self.assertIn("under the headroom", evidence["why"]) + + # Each run against its own MemTotal: a 64 GiB VM with 10 GiB left + # fails even when a 32 GiB VM of the class needs only 8 GiB. + rows[1]["runs"][PRESET][0] = run(total=65536.0, peak=55536.0) + evidence = group(probe.variance(rows, self.expected(2), self.presets), 8, "all")[ + "size_evidence"] + self.assertFalse(evidence["fits"]) + self.assertEqual(evidence["headroom_mib_required"], 16384.0) + + # The preset's own memory floor (6 GiB) wins over a small fraction. + evidence = group(probe.variance([probe_row(1, [run(peak=27000.0), run()])], + self.expected(1), self.presets, headroom_fraction=0.1), + 8, "all")["size_evidence"] + self.assertEqual(evidence["headroom_mib_required"], 6144.0) + self.assertFalse(evidence["fits"]) + + def test_a_group_with_every_run_unknown_still_lists_its_metrics(self) -> None: + rows = [probe_row(1, [run(harness=6), run(harness=6)])] + everything = group(probe.variance(rows, self.expected(2), self.presets), 8, "all") + rate = everything["metrics"]["steady_state.achieved_rate_shares_per_second"] + self.assertEqual((rate["all"]["n"], rate["all"]["unknown"]), (0, 4)) + self.assertEqual((rate["vm_to_vm"]["n"], rate["vm_to_vm"]["unknown"]), (0, 2)) + self.assertIsNone(rate["run_to_run"]) + self.assertEqual(everything["metrics"]["peak_used_mib"]["all"]["unknown"], 4) + + def test_an_all_phase_preset_lists_the_phases_its_unknown_runs_reported(self) -> None: + # pr-smoke's gates name no phases, so every driven phase is gated. + self.assertIsNone(probe.gated_phases(self.presets["pr-smoke"])) + row = probe_row(1, [run(harness=6)]) + row["runs"] = {"pr-smoke": row["runs"][PRESET]} + planned = probe.matrices("8", "sticky-disk", "0", "pr-smoke", "1", "1") + everything = group(probe.variance([row], planned, self.presets), 8, "all", "pr-smoke") + for phase in ("steady_state", "burst"): + metric = everything["metrics"][f"{phase}.shortfall"] + self.assertEqual((metric["all"]["n"], metric["all"]["unknown"]), (0, 1)) + + def test_one_commit_one_row_per_job(self) -> None: + with self.assertRaises(probe.ProbeError): + probe.variance([probe_row(1, [run()]), probe_row(2, [run()], commit="other")]) + with self.assertRaises(probe.ProbeError): + probe.variance([probe_row(1, [run()]), probe_row(1, [run()])]) + with self.assertRaises(probe.ProbeError): + probe.variance([], headroom_fraction=float("nan")) + + def test_without_the_plan_missing_jobs_cannot_be_counted_and_say_so(self) -> None: + document = probe.variance([probe_row(1, [run()])], None, self.presets) + self.assertIsNone(document["jobs"]["planned"]) + self.assertEqual(group(document, 8, "all")["runs"]["planned"], 1) + + def test_the_cli_writes_the_document_and_a_summary(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + rows = Path(tmp) / "rows" + rows.mkdir() + for r in (1, 2): + (rows / f"8-sticky-disk-{r}.json").write_text( + json.dumps(probe_row(r, [run(), run(rate=495.0)]))) + (rows / "fsync-8-1.json").write_text(json.dumps( + {"schema": probe.ROW_SCHEMA, "kind": "fsync", "class": 8, "vm": 1, + "runner": "r", "host": {}, "fsync": {"ops_per_second": 1.0}})) + expected = Path(tmp) / "expected.json" + expected.write_text(json.dumps(self.expected(3))) + out = Path(tmp) / "variance.json" + text = io.StringIO() + with contextlib.redirect_stdout(text): + self.assertEqual(probe.main(["variance", str(rows), "--expected", str(expected), + "--out", str(out), "--markdown"]), 0) + document = json.loads(out.read_text()) + self.assertEqual(document["schema"], probe.VARIANCE_SCHEMA) + self.assertEqual(document["jobs"]["missing"][0]["repeat"], 3) + self.assertIn("| 8 vCPU | throughput-20k-window-1fe | 4 of 6 |", text.getvalue()) + self.assertIn("unknown: 2 of 6 run(s) unknown", text.getvalue()) + with contextlib.redirect_stderr(io.StringIO()): + self.assertEqual(probe.main(["variance", str(rows), "--markdown"]), 2) + with self.assertRaises(SystemExit), contextlib.redirect_stderr(io.StringIO()): + probe.main(["variance", str(rows), "--headroom-fraction", "1.5"]) + + +class Verdict(unittest.TestCase): + def test_every_planned_run_must_exit_0(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + (root / "build.json").write_text(json.dumps({"exit_code": 0})) + for i in (1, 2): + path = root / PRESET / str(i) / "probe-measure.json" + path.parent.mkdir(parents=True) + path.write_text(json.dumps({"exit_code": 0, "timed_out": False})) + self.assertEqual(probe.verdict(root, [PRESET], 2), []) + # A run that never wrote its measurement is a failure. + self.assertEqual(probe.verdict(root, [PRESET], 3), + [f"{PRESET}/3/probe-measure: exit None"]) + (root / PRESET / "2" / "probe-measure.json").write_text( + json.dumps({"exit_code": -15, "timed_out": True})) + self.assertEqual(probe.verdict(root, [PRESET], 2), + [f"{PRESET}/2/probe-measure: exit -15 (timed out)"]) + self.assertEqual(probe.main(["verdict", "--probe", str(root), "--presets", PRESET, + "--in-job-repeats", "1"]), 0) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_prism_load_regress.py b/tests/test_prism_load_regress.py new file mode 100644 index 000000000..95ebfb935 --- /dev/null +++ b/tests/test_prism_load_regress.py @@ -0,0 +1,458 @@ +"""Tests for scripts/prism_load_regress.py (#551).""" + +from __future__ import annotations + +import copy +import dataclasses +import hashlib +import io +import json +import subprocess +from contextlib import redirect_stderr, redirect_stdout +from pathlib import Path +import sys +import tempfile +import unittest + +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT / "scripts")) + +import prism_load_regress as regress # noqa: E402 + +FIXTURE = ROOT / "tests" / "fixtures" / "prism-load-trend" / "regression" +CONFIG = regress.load_config(regress.CONFIG) + + +def commit(n: int) -> str: + return hashlib.sha1(f"synthetic commit {n:02d}".encode()).hexdigest() + + +def fixture_history() -> list[dict]: + rows, notes = regress.read_history(FIXTURE / "history") + assert not notes, notes + return rows + + +def fixture_current() -> dict: + return regress.read_row_files(FIXTURE / "current")[0] + + +def rerun(template: dict, n: int, **phase) -> dict: + """The fixture's row as run 1000+n on synthetic commit n.""" + row = copy.deepcopy(template) + row.update(run_id=1000 + n, commit=commit(n), recorded_at=f"2026-09-{n:02d}T02:40:00Z", + run_url=f"https://github.com/Qbit-Org/qbit-mining-bootstrap/actions/runs/{1000 + n}") + row["headline"]["phases"]["steady_state"].update(phase) + return row + + +def by_metric(verdict: dict) -> dict[str, dict]: + return {r["metric"]: r for r in verdict["regressions"]} + + +class SyntheticRegression(unittest.TestCase): + """#551's acceptance shape: a sleep inside the share append.""" + + def test_the_fixture_flags_the_share_append_numbers_with_the_commit_range(self) -> None: + verdict = regress.evaluate([fixture_current()], fixture_history(), CONFIG, None, None) + flagged = by_metric(verdict) + self.assertEqual(set(flagged), {"server_ack_mean_ms", "client_ack_p99_ms", "client_ack_p50_ms"}) + for entry in flagged.values(): + self.assertEqual(entry["commit_range"], {"from": commit(12), "to": commit(13)}) + self.assertEqual(entry["last_good"]["run_id"], 1012) + self.assertEqual(entry["first_bad"]["run_id"], 1013) + self.assertEqual(entry["runs_in_streak"], 1) + self.assertEqual(entry["phase"], "steady_state") + self.assertEqual(entry["fsync_band"], "fast") + self.assertEqual(flagged["server_ack_mean_ms"]["current"]["value"], 12.1) + self.assertAlmostEqual(flagged["server_ack_mean_ms"]["baseline"]["median"], 2.0, places=2) + self.assertTrue(verdict["provisional"]) + self.assertTrue(verdict["report_only"]) + + def test_the_last_good_run_says_whether_it_was_itself_judged(self) -> None: + # Seven earlier runs: runs 6 and 7 were judged within their own + # baseline of five; with exactly five, run 13's range starts at an + # unjudged run, which is within the limit that flagged run 13. + judged = by_metric(regress.evaluate([fixture_current()], fixture_history()[-7:], CONFIG, + None, None))["server_ack_mean_ms"] + self.assertEqual(judged["last_good"]["basis"], "within its own baseline") + first = regress.evaluate([fixture_current()], fixture_history()[-5:], CONFIG, None, None) + entry = by_metric(first)["server_ack_mean_ms"] + self.assertEqual(entry["commit_range"]["from"], commit(12)) + self.assertTrue(entry["last_good"]["basis"].startswith("within this run's limit (not itself judged")) + self.assertIn("not itself judged", regress.markdown(first, None, "https://github.com")) + + def test_the_next_night_keeps_the_range_where_the_regression_began(self) -> None: + history = fixture_history() + [fixture_current()] + tonight = rerun(fixture_current(), 14, server_ack_mean_ms=11.8) + entry = by_metric(regress.evaluate([tonight], history, CONFIG, None, None))["server_ack_mean_ms"] + self.assertEqual(entry["commit_range"], {"from": commit(12), "to": commit(13)}) + self.assertEqual(entry["runs_in_streak"], 2) + # The baseline is retaken from before the streak, not from itself. + self.assertAlmostEqual(entry["baseline"]["median"], 2.0, places=2) + + def test_an_unmeasured_run_widens_the_range_and_never_splits_a_streak(self) -> None: + history = fixture_history() + # Run 12 completed but did not measure the number: it is no baseline + # and no verdict. Before the regression, the range spans it (the + # change may be its commit); inside one, first_bad stays at the + # earlier measured bad run. + history[-1]["headline"]["phases"]["steady_state"]["server_ack_mean_ms"] = None + entry = by_metric(regress.evaluate([fixture_current()], history, CONFIG, None, None))[ + "server_ack_mean_ms"] + self.assertEqual(entry["commit_range"], {"from": commit(11), "to": commit(13)}) + history[-2]["headline"]["phases"]["steady_state"]["server_ack_mean_ms"] = 12.0 + entry = by_metric(regress.evaluate([fixture_current()], history, CONFIG, None, None))[ + "server_ack_mean_ms"] + self.assertEqual(entry["commit_range"], {"from": commit(10), "to": commit(11)}) + + def test_a_lasting_regression_stays_flagged_until_it_has_lasted_a_window(self) -> None: + history = fixture_history() + [fixture_current()] + for n in range(14, 26): + history.append(rerun(fixture_current(), n, server_ack_mean_ms=12.0)) + # Night 26: thirteen flagged runs so far, all out of the baseline. + entry = by_metric(regress.evaluate([rerun(fixture_current(), 26, server_ack_mean_ms=12.0)], + history, CONFIG, None, None))["server_ack_mean_ms"] + self.assertEqual(entry["commit_range"], {"from": commit(12), "to": commit(13)}) + self.assertEqual(entry["runs_in_streak"], 14) + self.assertAlmostEqual(entry["baseline"]["median"], 2.0, places=2) + # Night 27: fourteen flagged runs in a row are the accepted level. + history.append(rerun(fixture_current(), 26, server_ack_mean_ms=12.0)) + verdict = regress.evaluate([rerun(fixture_current(), 27, server_ack_mean_ms=12.0)], + history, CONFIG, None, None) + self.assertNotIn("server_ack_mean_ms", by_metric(verdict)) + + def test_a_run_within_the_noise_is_not_flagged(self) -> None: + current = rerun(fixture_current(), 13, server_ack_mean_ms=2.05, client_ack_p99_ms=41.0, + client_ack_p50_ms=10.2) + verdict = regress.evaluate([current], fixture_history(), CONFIG, None, None) + self.assertEqual(verdict["regressions"], []) + self.assertEqual(verdict["runs"][0]["numbers"]["regressed"], 0) + + def test_a_rate_drop_is_a_regression_where_lower_is_worse(self) -> None: + current = rerun(fixture_current(), 13, server_ack_mean_ms=2.0, client_ack_p99_ms=40.0, + client_ack_p50_ms=10.0, achieved_rate_shares_per_second=420.0) + flagged = by_metric(regress.evaluate([current], fixture_history(), CONFIG, None, None)) + self.assertEqual(set(flagged), {"achieved_rate_shares_per_second"}) + self.assertEqual(flagged["achieved_rate_shares_per_second"]["worse"], "lower") + + def test_the_cli_reports_and_still_exits_zero(self) -> None: + with tempfile.TemporaryDirectory() as scratch: + out, md, gh = (Path(scratch) / name for name in ("v.json", "v.md", "gh")) + with redirect_stdout(io.StringIO()): + code = regress.main(["--history", str(FIXTURE / "history"), "--rows", str(FIXTURE / "current"), + "--json", str(out), "--markdown", str(md), "--github-output", str(gh), + "--repository", "Qbit-Org/qbit-mining-bootstrap"]) + self.assertEqual(code, 0) + self.assertEqual(gh.read_text(), "regressions=3\n") + self.assertEqual(json.loads(out.read_text())["schema"], regress.VERDICT_SCHEMA) + text = md.read_text() + self.assertIn(f"/compare/{commit(12)}...{commit(13)}", text) + self.assertIn("provisional", text) + + +class NoVerdictIsNotAPass(unittest.TestCase): + def test_a_short_series_is_unknown(self) -> None: + verdict = regress.evaluate([fixture_current()], fixture_history()[:3], CONFIG, None, None) + self.assertEqual(verdict["regressions"], []) + numbers = verdict["runs"][0]["numbers"] + self.assertEqual(numbers["within"], 0) + self.assertGreater(numbers["unknown"], 0) + + def test_runs_in_another_fsync_band_are_not_a_baseline(self) -> None: + history = fixture_history() + for row in history: + row["fsync"]["usecs_per_op"] = 5000.0 + verdict = regress.evaluate([fixture_current()], history, CONFIG, None, None) + self.assertEqual(verdict["regressions"], []) + self.assertEqual(verdict["runs"][0]["fsync_band"], "fast") + self.assertEqual(verdict["runs"][0]["numbers"]["within"], 0) + + def test_an_unmeasured_fsync_cost_gives_no_verdict(self) -> None: + current = fixture_current() + current["fsync"]["usecs_per_op"] = None + entry = regress.evaluate([current], fixture_history(), CONFIG, None, None)["runs"][0] + self.assertEqual(entry["status"], "unknown") + self.assertIn("fsync", entry["reason"]) + + def test_a_changed_preset_starts_a_new_baseline(self) -> None: + current = fixture_current() + current["preset_sha256"] = "0" * 64 + verdict = regress.evaluate([current], fixture_history(), CONFIG, None, None) + self.assertEqual(verdict["regressions"], []) + + def test_an_incomplete_run_is_not_evaluated_nor_a_baseline(self) -> None: + current = fixture_current() + current["outcome"]["harness_exit_code"] = 4 + entry = regress.evaluate([current], fixture_history(), CONFIG, None, None)["runs"][0] + self.assertEqual(entry["status"], "not evaluated") + history = fixture_history() + for row in history[:10]: + row["outcome"]["harness_exit_code"] = 3 + verdict = regress.evaluate([fixture_current()], history, CONFIG, None, None) + self.assertEqual(verdict["regressions"], []) + + def test_a_number_the_run_did_not_measure_is_unknown(self) -> None: + current = rerun(fixture_current(), 13, server_ack_mean_ms=None) + verdict = regress.evaluate([current], fixture_history(), CONFIG, None, None) + self.assertNotIn("server_ack_mean_ms", by_metric(verdict)) + self.assertIn("steady_state.server_ack_mean_ms: not measured in this run", + verdict["runs"][0]["unknown"]) + + def test_a_preset_that_left_no_row_is_shown(self) -> None: + missing = {"schema": regress.ROW_SCHEMA, "kind": "missing", "lane": "L2", "run_id": 1013, + "run_attempt": 1, "repository": "Qbit-Org/qbit-mining-bootstrap", + "preset": "mainnet-shape-130-addresses"} + current = rerun(fixture_current(), 13, server_ack_mean_ms=2.0, client_ack_p99_ms=40.0, + client_ack_p50_ms=10.0) + verdict = regress.evaluate([current, missing], fixture_history(), CONFIG, None, None) + entry = [e for e in verdict["runs"] if e["preset"] == "mainnet-shape-130-addresses"] + self.assertEqual(len(entry), 1) + self.assertEqual(entry[0]["status"], "not evaluated") + text = regress.markdown(verdict, None, "https://github.com") + self.assertIn("1 planned preset(s) left no measurement", text) + + def test_a_later_missing_attempt_hides_the_earlier_measurement(self) -> None: + current = fixture_current() + missing = {"schema": regress.ROW_SCHEMA, "kind": "missing", "lane": "L2", + "run_id": current["run_id"], "run_attempt": 2, "repository": current["repository"], + "preset": current["preset"]} + verdict = regress.evaluate([current, missing], fixture_history(), CONFIG, None, None) + self.assertEqual(verdict["regressions"], []) + self.assertEqual([e["status"] for e in verdict["runs"]], ["not evaluated"]) + self.assertEqual(verdict["runs"][0]["run_attempt"], 2) + + def test_a_later_missing_attempt_in_another_lane_still_supersedes(self) -> None: + current = fixture_current() + missing = {"schema": regress.ROW_SCHEMA, "kind": "missing", "lane": "weekly", + "run_id": current["run_id"], "run_attempt": 2, "repository": current["repository"], + "preset": current["preset"]} + verdict = regress.evaluate([current, missing], fixture_history(), CONFIG, None, None) + self.assertEqual(verdict["regressions"], []) + self.assertEqual([(e["status"], e["run_attempt"]) for e in verdict["runs"]], [("not evaluated", 2)]) + + def test_a_lane_the_rule_does_not_read_is_named(self) -> None: + current = fixture_current() + current["lane"] = "L5" + entry = regress.evaluate([current], fixture_history(), CONFIG, None, None)["runs"][0] + self.assertEqual(entry["status"], "not evaluated") + + +class History(unittest.TestCase): + def test_the_latest_attempt_of_a_run_stands(self) -> None: + history = fixture_history() + # Run 1012's first attempt was slow; its second is the one that counts. + slow = copy.deepcopy(history[-1]) + slow["run_attempt"] = 0 + slow["headline"]["phases"]["steady_state"]["server_ack_mean_ms"] = 50.0 + kept = regress.latest_attempts(history + [slow]) + self.assertEqual(len(kept), len(history)) + self.assertNotIn(50.0, [r["headline"]["phases"]["steady_state"]["server_ack_mean_ms"] for r in kept]) + + def test_the_current_run_is_never_its_own_baseline(self) -> None: + history = fixture_history() + [fixture_current()] + verdict = regress.evaluate([fixture_current()], history, CONFIG, None, None) + self.assertEqual(by_metric(verdict)["server_ack_mean_ms"]["commit_range"]["from"], commit(12)) + + def test_foreign_and_broken_lines_are_skipped_and_counted(self) -> None: + with tempfile.TemporaryDirectory() as scratch: + lane = Path(scratch) / "trend" / "L2" + lane.mkdir(parents=True) + good = (FIXTURE / "history" / "trend" / "L2" / "2026-09.jsonl").read_text() + (lane / "2026-09.jsonl").write_text( + good + '{"schema": "qbit.prism.load-trend-row.v9"}\nnot json\n\n') + rows, notes = regress.read_history(Path(scratch)) + self.assertEqual(len(rows), 12) + self.assertEqual(len(notes), 2) + + +VARIANCE = ROOT / "tests" / "fixtures" / "prism-load-trend" / "variance" + + +def real_variance() -> dict: + """#542's document as scripts/prism_load_probe.py `variance` writes it.""" + data, note = regress.load_variance(VARIANCE / "runner-probe-variance.json") + assert note is None, note + return data + + +def spread_group(data: dict, band: str) -> dict: + (found,) = [g for g in data["groups"] if g["fsync_band"] == band] + return found + + +class Variance(unittest.TestCase): + def write(self, scratch: str, document: dict) -> Path: + path = Path(scratch) / "variance.json" + path.write_text(json.dumps(document)) + return path + + def test_the_fixture_is_what_the_probe_writes(self) -> None: + # Regenerate with generate.py when #542's document changes shape. + done = subprocess.run([sys.executable, str(VARIANCE / "generate.py"), "--check"]) + self.assertEqual(done.returncode, 0, "the variance fixture is stale; run generate.py") + + def test_other_schemas_and_versions_are_refused_by_name(self) -> None: + for schema in ("something.v7", "qbit.prism.runner-probe-variance.v2", None): + with self.subTest(schema=schema), tempfile.TemporaryDirectory() as scratch: + data, note = regress.load_variance(self.write(scratch, dict(real_variance(), schema=schema))) + self.assertIsNone(data) + self.assertIn(repr(schema), note) + self.assertIn("provisional", note) + with tempfile.TemporaryDirectory() as scratch: + data, note = regress.load_variance(self.write(scratch, {"schema": regress.VARIANCE_SCHEMA})) + self.assertIsNone(data) + self.assertIn("no groups", note) + + def test_the_spread_is_all_runs_cv_from_the_runs_own_band(self) -> None: + data = real_variance() + verdict = regress.evaluate([fixture_current()], fixture_history(), CONFIG, data, None) + entry = by_metric(verdict)["client_ack_p99_ms"] + cv = spread_group(data, "500-1000us")["metrics"]["steady_state.ack_p99_ms"]["all"]["cv"] + self.assertEqual(entry["baseline"]["spread_source"], "variance (#542, 500-1000us band, n 8)") + self.assertAlmostEqual(entry["baseline"]["spread"], cv * entry["baseline"]["median"]) + # #542 does not measure the server-side ACK: provisional, and named. + self.assertTrue(by_metric(verdict)["server_ack_mean_ms"]["baseline"]["spread_source"] + .startswith("provisional")) + self.assertTrue(verdict["provisional"]) + self.assertTrue(any("steady_state.server_ack_mean_ms" in n for n in verdict["notes"])) + # VM 4 is slower: VM-to-VM over 2x run-to-run points at #511's A/B. + self.assertTrue(any("#511" in n and "steady_state.client_ack_p99_ms" in n for n in verdict["notes"])) + + def test_a_band_with_no_spread_falls_back_to_all_bands_then_provisional(self) -> None: + data = real_variance() + band = spread_group(data, "500-1000us") + band["metrics"]["steady_state.ack_p99_ms"]["all"].update(n=1, cv=None) + verdict = regress.evaluate([fixture_current()], fixture_history(), CONFIG, data, None) + self.assertEqual(by_metric(verdict)["client_ack_p99_ms"]["baseline"]["spread_source"], + "variance (#542, all bands, n 8)") + for group in data["groups"]: + group["metrics"]["steady_state.ack_p99_ms"]["all"].update(cv=None) + verdict = regress.evaluate([fixture_current()], fixture_history(), CONFIG, data, None) + self.assertTrue(by_metric(verdict)["client_ack_p99_ms"]["baseline"]["spread_source"] + .startswith("provisional")) + + def test_each_replayed_run_is_judged_with_its_own_band(self) -> None: + # Earlier runs at 499 µs/op are the config's `fast` band like the + # current run, but #542's 250-500us band: there the ack p99 spread is + # made huge, so run 12's inflated p99 is within its own limit, and the + # range of run 13's regression cannot start before it. + data = real_variance() + wide = copy.deepcopy(spread_group(data, "500-1000us")) + wide["fsync_band"] = "250-500us" + wide["metrics"]["steady_state.ack_p99_ms"]["all"]["cv"] = 5.0 + data["groups"].append(wide) + history = fixture_history() + history[-1]["fsync"]["usecs_per_op"] = 499.0 + history[-1]["headline"]["phases"]["steady_state"]["client_ack_p99_ms"] = 90.0 + verdict = regress.evaluate([fixture_current()], history, CONFIG, data, None) + entry = by_metric(verdict)["client_ack_p99_ms"] + self.assertEqual(entry["commit_range"], {"from": commit(12), "to": commit(13)}) + # In the current run's band, run 12 is flagged too: the regression + # then began at run 12, after run 11. + history[-1]["fsync"]["usecs_per_op"] = 500.0 + verdict = regress.evaluate([fixture_current()], history, CONFIG, data, None) + self.assertEqual(by_metric(verdict)["client_ack_p99_ms"]["commit_range"], + {"from": commit(11), "to": commit(12)}) + + def test_another_runner_class_has_no_spread(self) -> None: + current = fixture_current() + history = fixture_history() + for row in [current, *history]: + row["runner_class"] = "blacksmith-16vcpu-ubuntu-2404" + verdict = regress.evaluate([current], history, CONFIG, real_variance(), None) + self.assertTrue(by_metric(verdict)["client_ack_p99_ms"]["baseline"]["spread_source"] + .startswith("provisional")) + + def test_full_coverage_clears_provisional_and_a_measured_zero_counts(self) -> None: + covered = tuple(m for m in CONFIG.metrics if m.key in regress.VARIANCE_METRICS + and m.key != "tip_last_notify_p99_ms") + calibrated = dataclasses.replace(CONFIG, provisional=False, metrics=covered) + data = real_variance() + verdict = regress.evaluate([fixture_current()], fixture_history(), calibrated, data, None) + self.assertFalse(verdict["provisional"], verdict["notes"]) + spread_group(data, "500-1000us")["metrics"]["steady_state.ack_p99_ms"]["all"]["cv"] = 0.0 + verdict = regress.evaluate([fixture_current()], fixture_history(), calibrated, data, None) + self.assertFalse(verdict["provisional"]) + # A replayed (and judged) run whose #542 band, and the all-band + # group, have no spread keeps it provisional, though this run's own + # band has one. + history = fixture_history() + history[8]["fsync"]["usecs_per_op"] = 499.0 + partial = copy.deepcopy(data) + del spread_group(partial, "all")["metrics"]["steady_state.ack_p99_ms"] + verdict = regress.evaluate([fixture_current()], history, calibrated, partial, None) + self.assertTrue(verdict["provisional"]) + self.assertTrue(any("steady_state.client_ack_p99_ms" in n for n in verdict["notes"])) + del spread_group(data, "500-1000us")["metrics"]["steady_state.ack_p50_ms"] + del spread_group(data, "all")["metrics"]["steady_state.ack_p50_ms"] + verdict = regress.evaluate([fixture_current()], fixture_history(), calibrated, data, None) + self.assertTrue(verdict["provisional"]) + self.assertTrue(any("steady_state.client_ack_p50_ms" in n for n in verdict["notes"])) + + +class Config(unittest.TestCase): + def load(self, text: str) -> regress.Config: + with tempfile.TemporaryDirectory() as scratch: + path = Path(scratch) / "c.toml" + path.write_text(text) + return regress.load_config(path) + + def base(self) -> str: + return regress.CONFIG.read_text() + + def test_the_checked_in_config_loads_and_is_provisional(self) -> None: + self.assertTrue(CONFIG.provisional) + self.assertEqual(CONFIG.lanes, ("L2",)) + self.assertEqual(CONFIG.band_names, ("fast", "medium", "slow")) + + def test_invalid_values_are_refused(self) -> None: + for old, new in [ + ("k = 3.0", "k = 0.0"), + ("k = 3.0", 'k = "3"'), + ("min_baseline = 5", "min_baseline = 20"), + ("window = 14", "window = 14\nwidow = 3"), + ('names = ["fast", "medium", "slow"]', 'names = ["fast", "slow"]'), + ("edges_usecs = [1000.0, 4000.0]", "edges_usecs = [4000.0, 1000.0]"), + ('worse = "lower"', 'worse = "down"'), + ("min_absolute_change = 1.0\n", "min_absolute_change = nan\n"), + ('schema = "qbit.prism.load-regression-config.v1"', 'schema = "v0"'), + ]: + with self.subTest(new=new), self.assertRaises(regress.RegressError): + self.load(self.base().replace(old, new, 1)) + + def test_bands(self) -> None: + for value, band in [(None, "unknown"), (0, "unknown"), (float("nan"), "unknown"), (True, "unknown"), + (500, "fast"), (1000.0, "medium"), (3999.9, "medium"), (4000, "slow")]: + with self.subTest(value=value): + self.assertEqual(regress.fsync_band(value, CONFIG), band) + + def test_no_rows_is_no_verdict(self) -> None: + with tempfile.TemporaryDirectory() as scratch: + with redirect_stderr(io.StringIO()) as err: + code = regress.main(["--history", scratch, "--rows", str(Path(scratch) / "absent")]) + self.assertEqual(code, 2) + self.assertIn("no trend rows", err.getvalue()) + + def test_rows_with_no_preset_run_say_nothing_was_compared(self) -> None: + job = {"schema": regress.ROW_SCHEMA, "kind": "job", "lane": "L4", "run_id": 1} + verdict = regress.evaluate([job], fixture_history(), CONFIG, None, None) + self.assertIn("No verdict", regress.markdown(verdict, None, "https://github.com")) + + def test_a_run_with_every_number_unknown_is_no_verdict(self) -> None: + verdict = regress.evaluate([fixture_current()], fixture_history()[:3], CONFIG, None, None) + text = regress.markdown(verdict, None, "https://github.com") + self.assertIn("No verdict", text) + self.assertNotIn("No number regressed", text) + + def test_a_bad_config_exits_two(self) -> None: + with tempfile.TemporaryDirectory() as scratch: + path = Path(scratch) / "c.toml" + path.write_text('schema = "nope"\n') + with redirect_stderr(io.StringIO()): + code = regress.main(["--history", scratch, "--rows", scratch, "--config", str(path)]) + self.assertEqual(code, 2) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_prism_load_trend.py b/tests/test_prism_load_trend.py new file mode 100644 index 000000000..910a2f8d3 --- /dev/null +++ b/tests/test_prism_load_trend.py @@ -0,0 +1,561 @@ +"""Tests for scripts/prism_load_trend.py (#551).""" + +from __future__ import annotations + +import argparse +import json +import os +from pathlib import Path +import subprocess +import sys +import tempfile +import unittest +from unittest import mock + +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT / "scripts")) + +import prism_load_trend as trend # noqa: E402 + +PG_TEST_FSYNC = """\ +Compare file sync methods using one 8kB write: +(in wal_sync_method preference order, except fdatasync is Linux's default) + open_datasync 2051.170 ops/sec 488 usecs/op + fdatasync 2034.433 ops/sec 492 usecs/op + fsync 1011.234 ops/sec 989 usecs/op +""" + + +def ack_delta(count: float, seconds: float, buckets: dict[str, float], unavailable=None) -> dict: + return {"instance_id": "fe", "counts": {"accepted": count}, "sums": {"accepted": seconds}, + "bucket_deltas": {"accepted": buckets}, "unavailable_reason": unavailable} + + +REPORT = { + "schema": "qbit.prism.load-harness.v1", + "run_id": "harness-1", + "started_at": "2026-09-30T02:20:00Z", + "finished_at": "2026-09-30T02:30:00Z", + "dirty": False, + "artifact_kind": "qualification", + "versions": {"server_build_profile": "release", + "server_revision_evidence": {"status": "established"}}, + "durability_findings": [], + "time_to_usable_work": {"tips": [{"all_sessions_milliseconds": 120.0}, + {"all_sessions_milliseconds": 300.0}]}, + "phases": [ + { + "name": "steady_state", "in_artifact": True, "completed": True, + "duration_seconds": 120.0, "target_rate_shares_per_second": 500, + "achieved_rate_shares_per_second": 499.5, "shortfall": 0, + "client_ack_latency": {"unit": "milliseconds", "p50": 9.0, "p99": 38.0}, + "server_share_ack_seconds": [ + ack_delta(100, 0.2, {"0.001": 10, "0.005": 99, "0.01": 100, "+Inf": 100}), + ack_delta(100, 0.4, {"0.001": 0, "0.005": 98, "0.01": 100, "+Inf": 100}), + ], + "order_lock": {"max_waiters": 0, "mean_waiters": 0.0}, + "processes": [{"peak_rss_kib": 204800}, {"peak_rss_kib": 102400}], + "rejected_valid_shares": 0, + "reconciliation": {"missing": 0}, + }, + { + "name": "warmup", "in_artifact": False, "completed": True, + "client_ack_latency": {"unit": "seconds", "p99": 0.04}, + "server_share_ack_seconds": [ack_delta(10, 0.1, {}, unavailable="scrape failed")], + "order_lock": {"max_waiters": None, "mean_waiters": None}, + "processes": [{"peak_rss_kib": None}], + }, + ], +} + + +def run_args(**overrides) -> argparse.Namespace: + values = dict(run_id=77, run_attempt=1, event="schedule", repository="Qbit-Org/qbit-mining-bootstrap", + server_url="https://github.com", ref="3.x.x", commit="a" * 40, + recorded_at="2026-09-30T02:40:00Z", presets=None) + values.update(overrides) + return argparse.Namespace(**values) + + +class Row(unittest.TestCase): + def setUp(self) -> None: + self.scratch = tempfile.TemporaryDirectory() + self.addCleanup(self.scratch.cleanup) + base = Path(self.scratch.name) + self.out = base / "out" + self.out.mkdir() + self.presets = base / "presets" + self.presets.mkdir() + (self.presets / "nightly-one.json").write_text(json.dumps({"name": "nightly-one", "schedule": "nightly"})) + (self.presets / "soak-weekly.json").write_text(json.dumps({"name": "soak-weekly", "schedule": "weekly"})) + + def build(self, preset: str = "nightly-one") -> dict: + args = run_args(presets=self.presets, dir=self.out, preset=preset, lane="auto", + runner_class="blacksmith-8vcpu-ubuntu-2404", artifact_name=f"prism-load-{preset}", + artifact_url="https://example.invalid/a/artifacts/1", artifact_id="", + config=trend.regress.CONFIG) + return trend.build_row(args) + + def write_full_run(self) -> None: + (self.out / "load-harness-report.json").write_text(json.dumps(REPORT)) + (self.out / "pg_test_fsync.txt").write_text(PG_TEST_FSYNC) + (self.out / "harness-exit-code").write_text("0\n") + (self.out / "gate-exit-code").write_text("0\n") + (self.out / "host.json").write_text(json.dumps({ + "runner_label": "blacksmith-8vcpu-ubuntu-2404", "nproc": 8, "cpu_model": "EPYC", + "mem_total_mib": 32000.0, "kernel": "6.8", "filesystem": {"type": "ext4", "options": "rw"}, + "block_device": {"write_cache": "write back", "fua": "0"}})) + + def test_a_full_run_has_its_headline_numbers(self) -> None: + self.write_full_run() + row = self.build() + trend.validate(row) + self.assertEqual(row["lane"], "L2") + self.assertEqual(row["fsync"]["usecs_per_op"], 492.0) + self.assertEqual(row["fsync"]["band"], "fast") + self.assertEqual(row["outcome"], {"harness_exit_code": 0, "gate_exit_code": 0, "result": "pass", + "provenance": "pass"}) + self.assertEqual(row["host"]["write_cache"], "write back") + self.assertEqual(len(row["preset_sha256"]), 64) + steady = row["headline"]["phases"]["steady_state"] + self.assertAlmostEqual(steady["server_ack_mean_ms"], 3.0) + self.assertEqual(steady["server_ack_p99_upper_ms"], 10.0) + self.assertEqual(steady["client_ack_p99_ms"], 38.0) + self.assertEqual(steady["peak_rss_mib"], 200.0) + self.assertEqual(steady["order_lock_max_waiters"], 0) + self.assertEqual(steady["order_lock_mean_waiters"], 0.0) + self.assertEqual(row["headline"]["run"]["tip_last_notify_p99_ms"], 300.0) + self.assertEqual(row["headline"]["run"]["durability_findings"], 0) + self.assertEqual(row["artifact"]["name"], "prism-load-nightly-one") + self.assertEqual(row["artifact"]["id"], 1) + self.assertIsNone(row["asset_url"]) + + def test_what_was_not_measured_is_null_not_zero(self) -> None: + self.write_full_run() + warmup = self.build()["headline"]["phases"]["warmup"] + self.assertFalse(warmup["in_artifact"]) + # A latency in another unit, an unavailable server delta, a sampler + # that saw nothing and a missing peak RSS are all unknown. + for key in ("client_ack_p99_ms", "server_ack_mean_ms", "server_ack_p99_upper_ms", + "order_lock_max_waiters", "order_lock_mean_waiters", "peak_rss_mib", + "achieved_rate_shares_per_second", "shortfall"): + with self.subTest(key=key): + self.assertIsNone(warmup[key]) + + def test_a_run_that_left_nothing_has_a_row_of_unknowns(self) -> None: + (self.out / "pg_test_fsync.txt").write_text("pg_test_fsync failed; see above\n") + row = self.build() + self.assertEqual(row["outcome"]["result"], "no verdict") + self.assertIsNone(row["outcome"]["harness_exit_code"]) + self.assertEqual(row["fsync"]["band"], "unknown") + self.assertIsNone(row["fsync"]["usecs_per_op"]) + self.assertIsNone(row["host"]) + self.assertIsNone(row["report"]) + self.assertEqual(row["headline"]["phases"], {}) + self.assertIsNone(row["headline"]["run"]["tip_last_notify_p99_ms"]) + self.assertIsNone(row["headline"]["run"]["durability_findings"]) + + def test_the_weekly_soak_is_l5(self) -> None: + self.write_full_run() + self.assertEqual(self.build("soak-weekly")["lane"], "weekly") + + def test_a_later_attempt_links_its_own_attempt(self) -> None: + row = trend.base_row(run_args(run_attempt=2), "preset", "L2") + self.assertTrue(row["run_url"].endswith("/runs/77/attempts/2")) + + +class RerunRows(unittest.TestCase): + def test_a_rerun_preset_job_that_left_no_row_is_missing_at_its_attempt(self) -> None: + with tempfile.TemporaryDirectory() as scratch: + base = Path(scratch) + rows_dir, presets = base / "rows", base / "presets" + rows_dir.mkdir() + presets.mkdir() + for name in ("a", "b"): + (presets / f"{name}.json").write_text(json.dumps({"schedule": "nightly"})) + stale = trend.base_row(run_args(run_attempt=1), "preset", "L2") + stale["preset"] = name + (rows_dir / f"{name}.json").write_text(json.dumps(stale)) + jobs = base / "jobs.jsonl" + # `a` was re-run in attempt 2 and died before building its row. + jobs.write_text('{"name": "a", "run_attempt": 2}\n{"name": "b", "run_attempt": 1}\n' + '{"name": "evidence", "run_attempt": 2}\n') + args = run_args(run_attempt=2, presets=presets, needs="{}", rows=rows_dir, jobs=jobs, + matrix=json.dumps({"include": [{"preset": "a"}, {"preset": "b"}]})) + rows = trend.job_rows(args) + self.assertEqual([(r["kind"], r["preset"], r["run_attempt"]) for r in rows], [("missing", "a", 2)]) + self.assertTrue(rows[0]["run_url"].endswith("/runs/77/attempts/2")) + + def test_a_missing_row_links_the_preset_jobs_attempt_not_this_jobs(self) -> None: + with tempfile.TemporaryDirectory() as scratch: + base = Path(scratch) + (base / "rows").mkdir() + (base / "presets").mkdir() + (base / "presets" / "a.json").write_text(json.dumps({"schedule": "nightly"})) + (base / "jobs.jsonl").write_text('{"name": "a", "run_attempt": 1}\n') + args = run_args(run_attempt=2, presets=base / "presets", needs="{}", rows=base / "rows", + jobs=base / "jobs.jsonl", matrix=json.dumps({"include": [{"preset": "a"}]})) + (row,) = trend.job_rows(args) + self.assertEqual(row["run_attempt"], 1) + self.assertTrue(row["run_url"].endswith("/runs/77")) + + +class PlanFilter(unittest.TestCase): + def test_rows_outside_the_plan_and_job_attempts(self) -> None: + with tempfile.TemporaryDirectory() as scratch: + base = Path(scratch) + (base / "rows").mkdir() + (base / "presets").mkdir() + for name in ("kept", "dropped"): + (base / "presets" / f"{name}.json").write_text(json.dumps({"schedule": "nightly"})) + row = trend.base_row(run_args(run_attempt=1), "preset", "L2") + row["preset"] = name + (base / "rows" / f"{name}.json").write_text(json.dumps(row)) + needs = {"bridging": {"result": "success", "outputs": {"commit": "d" * 40, "attempt": "1"}}} + args = run_args(run_attempt=3, presets=base / "presets", needs=json.dumps(needs), + rows=base / "rows", jobs=None, + matrix=json.dumps({"include": [{"preset": "kept"}]})) + with mock.patch("sys.stderr"): + rows = trend.job_rows(args) + left = sorted(p.name for p in (base / "rows").glob("*.json")) + self.assertEqual(left, ["kept.json"]) + (bridging,) = rows + # Re-running only the evidence job (attempt 3) does not move the + # bridging job's row off the attempt it ran in. + self.assertEqual((bridging["job"], bridging["run_attempt"]), ("bridging", 1)) + self.assertTrue(bridging["run_url"].endswith("/runs/77")) + + +class Validate(unittest.TestCase): + def row(self, **overrides) -> dict: + row = trend.base_row(run_args(), "preset", "L2") + row.update(preset="p") + row.update(overrides) + return row + + def test_pull_request_and_dispatch_rows_never_write(self) -> None: + for event in ("pull_request", "workflow_dispatch", "pull_request_target", None): + with self.subTest(event=event), self.assertRaisesRegex(trend.TrendError, "artifacts only"): + trend.validate(self.row(event=event)) + + def test_only_a_v_tag_push_writes(self) -> None: + trend.validate(self.row(event="push", ref="refs/tags/v3.0.0")) + for ref in ("refs/heads/3.x.x", "3.x.x", None, "refs/tags/ci-evidence/2026-09"): + with self.subTest(ref=ref), self.assertRaisesRegex(trend.TrendError, "tag push"): + trend.validate(self.row(event="push", ref=ref)) + + def test_a_job_rerun_on_a_moved_branch_is_a_new_row(self) -> None: + job = self.row(kind="job", lane="L4", job="live-nightly", outcome={"result": "success"}) + moved = dict(job, commit="b" * 40) + self.assertNotEqual(trend.identity(job), trend.identity(moved)) + # A genuine re-run of the job, even with the same outcome, is a new row. + self.assertNotEqual(trend.identity(job), trend.identity(dict(job, run_attempt=2))) + self.assertEqual(trend.identity(job), trend.identity(dict(job, recorded_at="2026-10-01T00:00:00Z"))) + + def test_malformed_rows_are_refused(self) -> None: + for overrides in ({"schema": "v0"}, {"lane": "L9"}, {"kind": "other"}, {"run_id": 0}, + {"run_attempt": True}, {"recorded_at": "2026-09-30"}, {"repository": "x"}, + {"preset": None}, {"kind": "job", "job": "live-nightly"}): + with self.subTest(overrides=overrides), self.assertRaises(trend.TrendError): + trend.validate(self.row(**overrides)) + + def test_a_promotion_row_is_written_by_cite_or_a_writer_job(self) -> None: + trend.validate(self.row(kind="promotion", event="cite")) + trend.validate(self.row(kind="promotion", event="schedule")) + with self.assertRaises(trend.TrendError): + trend.validate(self.row(kind="promotion", event="workflow_dispatch")) + + def test_months(self) -> None: + self.assertEqual(trend.month_of(self.row()), "2026-09") + + +class JobRows(unittest.TestCase): + def test_jobs_that_ran_and_presets_that_left_no_row(self) -> None: + with tempfile.TemporaryDirectory() as scratch: + rows_dir = Path(scratch) / "rows" + rows_dir.mkdir() + (rows_dir / "a.json").write_text(json.dumps({"kind": "preset", "preset": "a", "run_attempt": 1})) + presets = Path(scratch) / "presets" + presets.mkdir() + (presets / "b.json").write_text(json.dumps({"schedule": "nightly"})) + needs = { + "live-nightly": {"result": "failure", "outputs": {"commit": "c" * 40, "attempt": "1"}}, + "live-weekly": {"result": "skipped", "outputs": {}}, + "stratum-fuzz": {"result": "success", "outputs": {}}, + "run": {"result": "failure", "outputs": {}}, + } + args = run_args(presets=presets, needs=json.dumps(needs), rows=rows_dir, jobs=None, + matrix=json.dumps({"include": [{"preset": "a"}, {"preset": "b"}]})) + rows = trend.job_rows(args) + by = {(r["kind"], r.get("job") or r.get("preset")): r for r in rows} + self.assertEqual(set(by), {("job", "live-nightly"), ("job", "stratum-fuzz"), ("missing", "b")}) + self.assertEqual(by[("job", "live-nightly")]["lane"], "L4") + self.assertEqual(by[("job", "live-nightly")]["commit"], "c" * 40) + self.assertEqual(by[("job", "live-nightly")]["outcome"], {"result": "failure"}) + self.assertEqual(by[("job", "live-nightly")]["run_attempt"], 1) + # The fuzz job did not say which commit it ran: unknown, not the plan's. + self.assertIsNone(by[("job", "stratum-fuzz")]["commit"]) + missing = by[("missing", "b")] + self.assertTrue(missing["outcome"]["result"].startswith("missing")) + self.assertNotIn("headline", missing) + # It can never take the identity of the preset's real row. + real = dict(missing, kind="preset") + self.assertNotEqual(trend.identity(missing), trend.identity(real)) + for row in rows: + trend.validate(row) + + +def git(*args: str, cwd: Path) -> str: + return subprocess.run(["git", *args], cwd=cwd, check=True, capture_output=True, text=True).stdout + + +class Remote(unittest.TestCase): + """`append` and `fetch` against a local bare repository.""" + + def setUp(self) -> None: + self.scratch = tempfile.TemporaryDirectory() + self.addCleanup(self.scratch.cleanup) + env = {"GIT_CONFIG_GLOBAL": os.devnull, "GIT_CONFIG_NOSYSTEM": "1", + "GIT_AUTHOR_NAME": "t", "GIT_AUTHOR_EMAIL": "t@example.invalid", + "GIT_COMMITTER_NAME": "t", "GIT_COMMITTER_EMAIL": "t@example.invalid"} + patcher = mock.patch.dict(os.environ, env) + patcher.start() + self.addCleanup(patcher.stop) + base = Path(self.scratch.name) + self.bare = base / "remote.git" + git("init", "--bare", "-q", str(self.bare), cwd=base) + self.url = self.bare.as_uri() + self.writers = [] + for name in ("a", "b"): + clone = base / name + clone.mkdir() + git("init", "-q", cwd=clone) + git("remote", "add", "origin", self.url, cwd=clone) + self.writers.append(clone) + + def branch(self, index: int = 0, cls=trend.Branch) -> trend.Branch: + return cls(self.writers[index], "origin", trend.Deadline(60)) + + def row(self, run_id: int, preset: str = "p", attempt: int = 1, lane: str = "L2", + recorded_at: str = "2026-09-30T02:40:00Z") -> dict: + row = trend.base_row(run_args(run_id=run_id, run_attempt=attempt, recorded_at=recorded_at), "preset", lane) + row.update(preset=preset) + return row + + def recorded(self, path: str = "trend/L2/2026-09.jsonl") -> list[dict]: + text = git("--git-dir", str(self.bare), "show", f"{trend.BRANCH}:{path}", cwd=self.bare) + return [json.loads(line) for line in text.splitlines()] + + def quiet(self, *args, **kwargs): + return trend.append_rows(*args, backoff=0, log=lambda *_: None, **kwargs) + + def test_the_first_write_creates_the_orphan_branch(self) -> None: + result = self.quiet(self.branch(), [self.row(1), self.row(1, preset="q")], attempts=3) + self.assertEqual(result["added"], 2) + self.assertEqual([r["preset"] for r in self.recorded()], ["p", "q"]) + readme = git("--git-dir", str(self.bare), "show", f"{trend.BRANCH}:README.md", cwd=self.bare) + self.assertIn("append-only", readme) + parents = git("--git-dir", str(self.bare), "rev-list", "--parents", "-n1", trend.BRANCH, cwd=self.bare) + self.assertEqual(len(parents.split()), 1, "the branch's first commit has no parent") + + def test_a_retried_write_adds_nothing(self) -> None: + self.quiet(self.branch(), [self.row(1)], attempts=3) + result = self.quiet(self.branch(1), [self.row(1), self.row(1)], attempts=3) + self.assertEqual(result["added"], 0) + self.assertEqual(len(self.recorded()), 1) + + def test_a_row_rebuilt_months_later_is_still_recognised(self) -> None: + self.quiet(self.branch(), [self.row(1, recorded_at="2026-01-31T23:00:00Z")], attempts=1) + result = self.quiet(self.branch(), [self.row(1, recorded_at="2026-03-01T01:00:00Z")], attempts=1) + self.assertEqual(result["added"], 0) + listed = git("--git-dir", str(self.bare), "ls-tree", "-r", "--name-only", trend.BRANCH, cwd=self.bare) + self.assertNotIn("2026-03", listed) + + def test_a_later_attempt_is_a_new_row(self) -> None: + self.quiet(self.branch(), [self.row(1)], attempts=3) + self.quiet(self.branch(), [self.row(1, attempt=2)], attempts=3) + self.assertEqual([r["run_attempt"] for r in self.recorded()], [1, 2]) + + def test_rows_go_to_their_lane_and_month(self) -> None: + self.quiet(self.branch(), [self.row(1), self.row(2, lane="L5"), + self.row(3, recorded_at="2026-10-01T00:05:00Z")], attempts=3) + self.assertEqual(len(self.recorded("trend/L2/2026-09.jsonl")), 1) + self.assertEqual(len(self.recorded("trend/L5/2026-09.jsonl")), 1) + self.assertEqual(len(self.recorded("trend/L2/2026-10.jsonl")), 1) + + def test_concurrent_writers_lose_no_row(self) -> None: + test = self + + class Racing(trend.Branch): + raced = False + + def push(self, commit: str): + # Another writer lands between this writer's fetch and push. + if not Racing.raced: + Racing.raced = True + test.quiet(test.branch(1), [test.row(2)], attempts=1) + return super().push(commit) + + self.quiet(self.branch(), [self.row(3)], attempts=1) + result = self.quiet(self.branch(0, Racing), [self.row(1)], attempts=3) + self.assertEqual(result["attempts"], 2) + self.assertEqual(sorted(r["run_id"] for r in self.recorded()), [1, 2, 3]) + + def test_concurrent_creation_of_the_branch_loses_no_row(self) -> None: + test = self + + class Racing(trend.Branch): + raced = False + + def push(self, commit: str): + if not Racing.raced: + Racing.raced = True + test.quiet(test.branch(1), [test.row(2)], attempts=1) + return super().push(commit) + + self.quiet(self.branch(0, Racing), [self.row(1)], attempts=3) + self.assertEqual(sorted(r["run_id"] for r in self.recorded()), [1, 2]) + + def test_an_unconfirmed_push_that_landed_is_not_written_twice(self) -> None: + class Unconfirmed(trend.Branch): + calls = 0 + + def push(self, commit: str): + Unconfirmed.calls += 1 + pushed, error = super().push(commit) + return (False, "timed out") if Unconfirmed.calls == 1 else (pushed, error) + + result = self.quiet(self.branch(0, Unconfirmed), [self.row(1)], attempts=3) + self.assertEqual(result, {"added": 0, "skipped": 1, "attempts": 2}) + self.assertEqual(len(self.recorded()), 1) + + def test_a_promotion_recovered_by_a_rerun_is_its_own_row(self) -> None: + failed = self.row(1) + failed.update(promotion={"status": "failed", "release": "ci-evidence/2026-09", "error": "x"}, + asset_url=None) + self.quiet(self.branch(), [failed], attempts=1) + recovered = self.row(1) + recovered.update(promotion={"status": "promoted", "release": "ci-evidence/2026-09", "asset": "a"}, + asset_url="https://example.invalid/a") + self.quiet(self.branch(), [recovered], attempts=1) + self.quiet(self.branch(), [recovered], attempts=1) + rows = self.recorded() + self.assertEqual([r["kind"] for r in rows], ["preset", "promotion"]) + self.assertEqual(rows[0]["promotion"]["status"], "failed") + self.assertEqual(rows[1]["asset_url"], "https://example.invalid/a") + self.assertEqual((rows[1]["run_id"], rows[1]["run_attempt"], rows[1]["preset"]), (1, 1, "p")) + + def test_bounded_attempts_end_in_an_error(self) -> None: + class Refused(trend.Branch): + def push(self, commit: str): + return False, "rejected" + + with self.assertRaisesRegex(trend.TrendError, "not recorded after 2 attempts"): + self.quiet(self.branch(0, Refused), [self.row(1)], attempts=2) + + def test_a_failed_listing_is_not_a_missing_branch(self) -> None: + broken = trend.Branch(self.writers[0], "nowhere", trend.Deadline(60)) + with self.assertRaises(trend.TrendError): + self.quiet(broken, [self.row(1)], attempts=1) + + def test_a_refused_row_writes_nothing(self) -> None: + bad = self.row(1) + bad["event"] = "pull_request" + with self.assertRaises(trend.TrendError): + self.quiet(self.branch(), [self.row(2), bad], attempts=1) + listed = git("ls-remote", "--heads", self.url, cwd=self.writers[0]) + self.assertEqual(listed, "") + + def test_fetch_exports_the_trend_or_nothing(self) -> None: + out = Path(self.scratch.name) / "history" + self.assertIsNone(trend.export(self.branch(), out)) + self.assertEqual(list(out.iterdir()), []) + self.quiet(self.branch(), [self.row(1)], attempts=1) + self.assertIsNotNone(trend.export(self.branch(1), out)) + rows, notes = trend.regress.read_history(out) + self.assertEqual((len(rows), notes), (1, [])) + + +class Promotion(unittest.TestCase): + def rows(self, scratch: str) -> Path: + directory = Path(scratch) + specs = [("l2-pass", "L2", "pass"), ("l2-fail", "L2", "fail"), ("l2-regressed", "L2", "pass"), + ("soak", "weekly", "pass"), ("l4-job", "L4", None)] + for name, lane, result in specs: + row = trend.base_row(run_args(), "job" if result is None else "preset", lane) + row.update(preset=name, outcome={"result": result}, promotion=None, asset_url=None, + artifact={"name": f"prism-load-{name}", "url": None}) + (directory / f"{name}.json").write_text(json.dumps(row)) + return directory + + def test_the_policy_promotes_soaks_and_flagged_l2_runs(self) -> None: + verdict = {"regressions": [{"run_id": 77, "run_attempt": 1, "preset": "l2-regressed"}]} + calls = [] + + def uploader(row, tag, repository, create, deadline): + calls.append((row["preset"], tag, create)) + if row["preset"] == "l2-fail": + raise trend.TrendError("upload refused") + if row["preset"] == "l2-regressed": + raise OSError(28, "No space left on device") + return f"https://example.invalid/{trend.asset_name(row)}" + + with tempfile.TemporaryDirectory() as scratch: + directory = self.rows(scratch) + failed = trend.promote(directory, "policy", verdict, "monthly", "o/r", trend.Deadline(60), + uploader=uploader, log=lambda *_: None) + rows = {p.stem: json.loads(p.read_text()) for p in directory.glob("*.json")} + self.assertEqual(sorted(c[0] for c in calls), ["l2-fail", "l2-regressed", "soak"]) + self.assertTrue(all(tag == "ci-evidence/2026-09" and create for _, tag, create in calls)) + self.assertEqual(failed, 2) + self.assertEqual(rows["l2-regressed"]["promotion"]["status"], "failed") + self.assertIn("No space left", rows["l2-regressed"]["promotion"]["error"]) + self.assertEqual(rows["soak"]["promotion"]["status"], "promoted") + self.assertEqual(rows["soak"]["asset_url"], + "https://example.invalid/weekly-soak-run77-attempt1.tar.gz") + self.assertEqual(rows["l2-fail"]["promotion"]["status"], "failed") + self.assertIsNone(rows["l2-fail"]["asset_url"]) + self.assertIsNone(rows["l2-pass"]["promotion"]) + + def test_a_tag_release_takes_every_row_and_is_not_created_here(self) -> None: + calls = [] + with tempfile.TemporaryDirectory() as scratch: + trend.promote(self.rows(scratch), "all", None, "v3.0.0", "o/r", trend.Deadline(60), + uploader=lambda row, tag, repo, create, d: calls.append((tag, create)) or "u", + log=lambda *_: None) + self.assertEqual(calls, [("v3.0.0", False)] * 4) + + def test_the_artifact_id_comes_from_the_upload(self) -> None: + self.assertEqual(trend.artifact_id_of("42", None), 42) + self.assertEqual(trend.artifact_id_of("", "https://github.com/o/r/actions/runs/7/artifacts/99"), 99) + self.assertIsNone(trend.artifact_id_of("", "https://example.invalid/x")) + + def test_a_bundle_is_fetched_only_by_its_exact_artifact(self) -> None: + row = {"kind": "preset", "run_id": 7, "run_attempt": 2, "preset": "p", + "artifact": {"name": "prism-load-p", "id": 99, "url": None}} + with tempfile.TemporaryDirectory() as scratch, \ + mock.patch.object(trend, "gh_json", return_value={ + "name": "prism-load-p", "workflow_run": {"id": 8}, "expired": False}), \ + self.assertRaisesRegex(trend.TrendError, "not the row's"): + trend.download_artifact(row, "o/r", Path(scratch) / "b", trend.Deadline(60)) + row["artifact"]["id"] = None + with self.assertRaisesRegex(trend.TrendError, "cannot be selected exactly"): + trend.download_artifact(row, "o/r", Path("/nonexistent"), trend.Deadline(60)) + + def test_a_cited_artifact_is_attributed_to_the_attempt_that_uploaded_it(self) -> None: + answers = { + "repos/o/r/actions/runs/7": {"run_attempt": 3}, + "repos/o/r/actions/runs/7/attempts/3": {"run_started_at": "2026-09-30T05:00:00Z"}, + "repos/o/r/actions/runs/7/attempts/2": {"run_started_at": "2026-09-30T03:00:00Z"}, + } + meta = {"id": 99, "workflow_run": {"id": 7}, "created_at": "2026-09-30T03:30:00Z"} + with mock.patch.object(trend, "gh_json", side_effect=lambda path, deadline: answers[path]): + self.assertEqual(trend.attempt_of_artifact(meta, "o/r", trend.Deadline(60)), 2) + + def test_asset_names_are_safe_and_unique_per_attempt(self) -> None: + row = {"lane": "L2", "preset": "a b/c", "run_id": 5, "run_attempt": 2} + self.assertEqual(trend.asset_name(row), "L2-a_b_c-run5-attempt2.tar.gz") + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_prism_metrics.py b/tests/test_prism_metrics.py deleted file mode 100644 index b85ff3b97..000000000 --- a/tests/test_prism_metrics.py +++ /dev/null @@ -1,3480 +0,0 @@ -#!/usr/bin/env python3 -"""PRISM metrics renderer extraction tests. - -The frozen reference implementations below are byte-copied from the -coordinator formatter bodies as they stood immediately before the PR 80 -metrics extraction (``self`` renamed to ``server``). The full-render parity -test proves the extracted ``MetricsRenderer`` reproduces the pre-extraction -Prometheus document byte-for-byte for identical inputs. -""" - -from __future__ import annotations - -from collections import OrderedDict -from contextlib import redirect_stdout -from decimal import Decimal -import gc -import hashlib -from io import StringIO -from pathlib import Path -import re -import tempfile -import time -import tracemalloc -import unittest -from types import SimpleNamespace -from unittest import mock - -from lab.prism.coordinator_config import ( - DEFAULT_ACCEPTED_PARENT_UNRESOLVED_DEPTH_MAX, - DEFAULT_BLOCK_CANDIDATE_CLEANUP_RETRY_BACKLOG_MAX, - DEFAULT_PRISM_INITIAL_JOB_MAX_WORKERS, - DEFAULT_PRISM_JOB_BUILD_EXECUTOR_WORKERS, -) -from lab.prism.accepted_preview_telemetry import ( - PRISM_ACCEPTED_LANDING_PHASES, - PRISM_PAYOUT_WINDOW_FULL_RESCAN_PATHS, - PRISM_PAYOUT_WINDOW_FULL_RESCAN_REASONS, - PRISM_REORG_RECONCILE_CALLERS, - PRISM_REORG_RECONCILE_STEPS, - AcceptedPreviewTelemetry, - ensure_accepted_preview_telemetry, - fold_ledger_read_stats, -) -from lab.prism.block_candidates import ( - PRISM_ACCEPTED_BLOCK_PREVIEW_PUBLICATION_RESULTS, - PRISM_ACCEPTED_BLOCK_PREVIEW_PUBLICATION_SECONDS_BUCKETS, - PRISM_BLOCK_CANDIDATE_COLLAPSE_OUTCOMES, - PRISM_STALE_JOB_ABANDON_CLASSES, -) -from lab.prism.bundle_compiler import _ServeBuilderClient, _ShareWindowSerialization -from lab.prism.job_bundle import ( - PRISM_JOB_BUILD_PHASES, - PRISM_JOB_BUILD_SECONDS_BUCKETS, - PRISM_JOB_CACHE_KINDS, -) -from lab.prism.job_delivery import ( - PRISM_EVICTED_JOB_CAPACITY_SCOPES, - PRISM_EVICTED_JOB_CLASSES, - PRISM_EVICTED_JOB_SUBMIT_OUTCOMES, -) -from lab.prism.metrics import ( - PRISM_COMPONENT_BYTE_KINDS, - PRISM_COMPONENT_ENTRY_KINDS, - MetricsRenderer, -) -from lab.prism.payout_state import ( - AcceptedBlockPayoutTransition, - PRISM_PAYOUT_WINDOW_BUILD_OUTCOMES, - PRISM_PAYOUT_WINDOW_BUILD_PHASES, - PayoutLedgerArtifact, - _IncrementalPayoutArtifactWindow, -) -from lab.prism.prism_coordinator import PRISM_REJECTION_REASON_IDS -from lab.prism.process_telemetry import ( - PRISM_GC_GENERATIONS, - MallInfo2, - ProcessHeapSample, - ProcessHeapTelemetry, -) -from lab.prism.reorg_reconciler import ( - PRISM_REORG_RECONCILE_LOOKUP_PATHS, - PRISM_REORG_RECONCILE_LOOKUP_SOURCES, - _ReconcileFlight, -) -from lab.prism.share_ledger import ( - DaemonShareWindowMirror, - IncrementalShareJsonSequence, - IncrementalShareWindow, - _IncrementalShareWindowPage, -) -from lab.prism.share_submission import ( - PRISM_SHARE_ACK_RESULTS, - empty_share_ack_histograms, -) -from lab.prism.vardiff_service import PRISM_VARDIFF_RESUME_OUTCOMES -from lab.prism.writer_lease_timing import ( - LEASE_HEARTBEAT_MODES, - LEASE_HEARTBEAT_OUTCOMES, - LEASE_HEARTBEAT_PHASES, - LEASE_HEARTBEAT_POLICY_TERMS, - LEASE_MONITOR_LATE_WAKE_SLACK_FRACTIONS, - LEASE_MONITOR_WAKE_DELAY_BUCKETS, -) -from lab.prism.background_services import PRISM_GC_PAUSE_SECONDS_BUCKETS -from tests import prism_vardiff_test_support as support - - -def reference_share_ack_metrics_lines(server) -> list[str]: - service = server.__dict__.get("_share_submission_service") - if service is not None: - histograms = service.share_ack_snapshot() - else: - legacy = server.__dict__.get("share_ack_histograms") - histograms = ( - { - result: { - "buckets": dict(histogram["buckets"]), - "sum": float(histogram["sum"]), - "count": int(histogram["count"]), - } - for result, histogram in legacy.items() - } - if legacy is not None - else empty_share_ack_histograms() - ) - lines = [ - "# HELP qbit_prism_share_ack_seconds mining.submit line arrival to Stratum response, by outcome.", - "# TYPE qbit_prism_share_ack_seconds histogram", - ] - for result in PRISM_SHARE_ACK_RESULTS: - histogram = histograms[result] - buckets = histogram["buckets"] - lines.extend( - f'qbit_prism_share_ack_seconds_bucket{{result="{result}",le="{bucket:g}"}} {int(buckets.get(bucket, 0))}' - for bucket in PRISM_JOB_BUILD_SECONDS_BUCKETS - ) - lines.append( - f'qbit_prism_share_ack_seconds_bucket{{result="{result}",le="+Inf"}} {histogram["count"]}' - ) - lines.append( - f'qbit_prism_share_ack_seconds_sum{{result="{result}"}} {histogram["sum"]:.6f}' - ) - lines.append( - f'qbit_prism_share_ack_seconds_count{{result="{result}"}} {histogram["count"]}' - ) - return lines - - -def reference_coordinator_lock_metrics_lines(server) -> list[str]: - snapshot = getattr(server.lock, "contention_snapshot", None) - if callable(snapshot): - acquisition_count, contention_count, wait_sum, wait_max = snapshot() - else: - acquisition_count, contention_count, wait_sum, wait_max = 0, 0, 0.0, 0.0 - return [ - "# HELP qbit_prism_coordinator_lock_acquisitions_total Coordinator control-plane lock acquisitions granted, contended or not; divide qbit_prism_coordinator_lock_contentions_total by this for the contended fraction. A timed-out wait is counted as contention without an acquisition, so that fraction is an upper bound where a caller passes a timeout.", - "# TYPE qbit_prism_coordinator_lock_acquisitions_total counter", - f"qbit_prism_coordinator_lock_acquisitions_total {int(acquisition_count)}", - "# HELP qbit_prism_coordinator_lock_contentions_total Coordinator control-plane lock acquisitions that had to wait.", - "# TYPE qbit_prism_coordinator_lock_contentions_total counter", - f"qbit_prism_coordinator_lock_contentions_total {int(contention_count)}", - "# HELP qbit_prism_coordinator_lock_wait_seconds Coordinator control-plane lock wait duration for contended acquisitions.", - "# TYPE qbit_prism_coordinator_lock_wait_seconds summary", - f"qbit_prism_coordinator_lock_wait_seconds_sum {float(wait_sum):.6f}", - f"qbit_prism_coordinator_lock_wait_seconds_count {int(contention_count)}", - "# HELP qbit_prism_coordinator_lock_wait_seconds_max Longest observed coordinator control-plane lock wait.", - "# TYPE qbit_prism_coordinator_lock_wait_seconds_max gauge", - f"qbit_prism_coordinator_lock_wait_seconds_max {float(wait_max):.6f}", - ] - - -def reference_block_submitter_metrics_lines(server) -> list[str]: - pending_metrics = { - "pending_count": -1, - "oldest_pending_age_seconds": -1.0, - "oldest_unattempted_age_seconds": -1.0, - } - pending_snapshot = getattr(server.ledger, "block_candidate_pending_metrics", None) - if callable(pending_snapshot): - try: - pending_metrics.update(pending_snapshot()) - except Exception: - pass - service = server._ensure_block_candidate_service() - backoff_active, backoff_remaining, backoff_delay = ( - service.backoff_snapshot() - ) - submit_buckets, submit_sum, submit_count = ( - service.block_submit_seconds_snapshot() - ) - collapse_counts = service.block_candidate_collapse_snapshot() - preview_publication = service.accepted_block_preview_publication_snapshot() - preview_publication_lines: list[str] = [] - for result in PRISM_ACCEPTED_BLOCK_PREVIEW_PUBLICATION_RESULTS: - histogram = preview_publication[result] - buckets = histogram["buckets"] - count = int(histogram["count"]) - preview_publication_lines.extend( - f"qbit_prism_accepted_block_preview_publication_seconds_bucket" - f'{{result="{result}",le="{bucket:g}"}} {int(buckets.get(bucket, 0))}' - for bucket in PRISM_ACCEPTED_BLOCK_PREVIEW_PUBLICATION_SECONDS_BUCKETS - ) - preview_publication_lines.append( - f"qbit_prism_accepted_block_preview_publication_seconds_bucket" - f'{{result="{result}",le="+Inf"}} {count}' - ) - preview_publication_lines.append( - f"qbit_prism_accepted_block_preview_publication_seconds_sum" - f'{{result="{result}"}} {float(histogram["sum"]):.6f}' - ) - preview_publication_lines.append( - f"qbit_prism_accepted_block_preview_publication_seconds_count" - f'{{result="{result}"}} {count}' - ) - return [ - "# HELP qbit_prism_block_submit_seconds Seconds from a block candidate landing in this process to its submitblock RPC returning.", - "# TYPE qbit_prism_block_submit_seconds histogram", - *[ - f'qbit_prism_block_submit_seconds_bucket{{le="{bucket:g}"}} {int(submit_buckets.get(bucket, 0))}' - for bucket in PRISM_JOB_BUILD_SECONDS_BUCKETS - ], - f'qbit_prism_block_submit_seconds_bucket{{le="+Inf"}} {submit_count}', - f"qbit_prism_block_submit_seconds_sum {submit_sum:.6f}", - f"qbit_prism_block_submit_seconds_count {submit_count}", - "# HELP qbit_prism_accepted_block_preview_publication_seconds Seconds from definitive qbitd acceptance of a block candidate to its payout preview becoming visible to waiting child work, by publication result.", - "# TYPE qbit_prism_accepted_block_preview_publication_seconds histogram", - *preview_publication_lines, - "# HELP qbit_prism_block_candidates_pending Durable block candidates awaiting a terminal outcome, or -1 if unavailable.", - "# TYPE qbit_prism_block_candidates_pending gauge", - f"qbit_prism_block_candidates_pending {int(pending_metrics['pending_count'])}", - "# HELP qbit_prism_block_candidate_oldest_pending_seconds Age of the oldest durable pending block candidate, or -1 if unavailable.", - "# TYPE qbit_prism_block_candidate_oldest_pending_seconds gauge", - f"qbit_prism_block_candidate_oldest_pending_seconds {float(pending_metrics['oldest_pending_age_seconds']):.6f}", - "# HELP qbit_prism_block_candidate_oldest_unattempted_seconds Age of the oldest durable candidate that has never entered processing, or -1 if unavailable.", - "# TYPE qbit_prism_block_candidate_oldest_unattempted_seconds gauge", - f"qbit_prism_block_candidate_oldest_unattempted_seconds {float(pending_metrics['oldest_unattempted_age_seconds']):.6f}", - "# HELP qbit_prism_block_submitter_retry_backoff_active Whether the submitter is in an intentional interruptible retry wait.", - "# TYPE qbit_prism_block_submitter_retry_backoff_active gauge", - f"qbit_prism_block_submitter_retry_backoff_active {1 if backoff_active else 0}", - "# HELP qbit_prism_block_submitter_retry_backoff_remaining_seconds Remaining intentional submitter retry wait.", - "# TYPE qbit_prism_block_submitter_retry_backoff_remaining_seconds gauge", - f"qbit_prism_block_submitter_retry_backoff_remaining_seconds {backoff_remaining:.6f}", - "# HELP qbit_prism_block_submitter_retry_backoff_seconds Current intentional submitter retry delay.", - "# TYPE qbit_prism_block_submitter_retry_backoff_seconds gauge", - f"qbit_prism_block_submitter_retry_backoff_seconds {backoff_delay:.6f}", - "# HELP qbit_prism_block_candidate_collapse_total Durable pending block candidates handled by the decided-height collapse, by bounded outcome. The outcome label set is closed: no block hash, parent hash, or job ID is ever a label.", - "# TYPE qbit_prism_block_candidate_collapse_total counter", - *[ - f'qbit_prism_block_candidate_collapse_total{{outcome="{outcome}"}} {int(collapse_counts.get(outcome, 0))}' - for outcome in PRISM_BLOCK_CANDIDATE_COLLAPSE_OUTCOMES - ], - ] - - -def reference_block_candidate_cleanup_backlog_metrics_lines(server) -> list[str]: - # Issue #198. Pinned deliberately: seven unlabelled single-series - # families over the B1 owner's fixed-key backlog snapshot. Adding a - # label, a key, or a family here is a metrics-contract change and must - # be mirrored in docs/prism-overload-alerts.md. - snapshot = ( - server._ensure_block_candidate_service() - .collapsed_candidate_cleanup_backlog_snapshot() - ) - return [ - "# HELP qbit_prism_block_candidate_cleanup_retry_backlog Durably terminal collapsed block candidates whose in-memory cleanup is still owed and retried by the accounting lane.", - "# TYPE qbit_prism_block_candidate_cleanup_retry_backlog gauge", - f"qbit_prism_block_candidate_cleanup_retry_backlog {int(snapshot['depth'])}", - "# HELP qbit_prism_block_candidate_cleanup_retry_backlog_max Configured cleanup-retry backlog depth at which the decided-height collapse stops admitting rows to bulk terminalization.", - "# TYPE qbit_prism_block_candidate_cleanup_retry_backlog_max gauge", - f"qbit_prism_block_candidate_cleanup_retry_backlog_max {int(snapshot['backlog_max'])}", - "# HELP qbit_prism_block_candidate_cleanup_retry_oldest_seconds Age of the oldest owed collapse cleanup since it was first deferred, or -1 when none is owed.", - "# TYPE qbit_prism_block_candidate_cleanup_retry_oldest_seconds gauge", - f"qbit_prism_block_candidate_cleanup_retry_oldest_seconds {float(snapshot['oldest_age_seconds']):.6f}", - "# HELP qbit_prism_block_candidate_cleanup_retry_pending_share_holders Pending-share floor holders retained by owed collapse cleanups, by exact object identity.", - "# TYPE qbit_prism_block_candidate_cleanup_retry_pending_share_holders gauge", - f"qbit_prism_block_candidate_cleanup_retry_pending_share_holders {int(snapshot['pending_share_holders'])}", - "# HELP qbit_prism_block_candidate_cleanup_retry_terminal_outcome_pins Terminal-outcome fences the cleanup-retry backlog pins against eviction.", - "# TYPE qbit_prism_block_candidate_cleanup_retry_terminal_outcome_pins gauge", - f"qbit_prism_block_candidate_cleanup_retry_terminal_outcome_pins {int(snapshot['terminal_outcome_pins'])}", - "# HELP qbit_prism_block_candidate_cleanup_backpressure_active Whether the cleanup-retry backlog is at its bound and bulk terminalization is refusing new rows.", - "# TYPE qbit_prism_block_candidate_cleanup_backpressure_active gauge", - f"qbit_prism_block_candidate_cleanup_backpressure_active {1 if snapshot['backpressure_active'] else 0}", - "# HELP qbit_prism_block_candidate_cleanup_backpressure_total Occasions on which the cleanup-retry backlog bound preserved at least one row from bulk terminalization.", - "# TYPE qbit_prism_block_candidate_cleanup_backpressure_total counter", - f"qbit_prism_block_candidate_cleanup_backpressure_total {int(snapshot['backpressure_engagements'])}", - ] - - -# The closed family set issue #198 exports; a test pins the renderer to it. -BLOCK_CANDIDATE_CLEANUP_BACKLOG_METRIC_FAMILIES = ( - ("qbit_prism_block_candidate_cleanup_retry_backlog", "gauge"), - ("qbit_prism_block_candidate_cleanup_retry_backlog_max", "gauge"), - ("qbit_prism_block_candidate_cleanup_retry_oldest_seconds", "gauge"), - ("qbit_prism_block_candidate_cleanup_retry_pending_share_holders", "gauge"), - ("qbit_prism_block_candidate_cleanup_retry_terminal_outcome_pins", "gauge"), - ("qbit_prism_block_candidate_cleanup_backpressure_active", "gauge"), - ("qbit_prism_block_candidate_cleanup_backpressure_total", "counter"), -) - - -# Issue #226 part 1: the closed family sets of the always-on heap telemetry -# and the component-cardinality gauges; tests pin the renderer to both. -PROCESS_HEAP_METRIC_FAMILIES = ( - ("qbit_prism_process_allocated_blocks", "gauge"), - ("qbit_prism_process_gc_trigger_count", "gauge"), - ("qbit_prism_process_gc_collections_total", "counter"), - ("qbit_prism_process_gc_collected_objects_total", "counter"), - ("qbit_prism_process_gc_uncollectable_objects_total", "counter"), - ("qbit_prism_process_threads", "gauge"), - ("qbit_prism_process_malloc_info_available", "gauge"), - ("qbit_prism_process_malloc_arena_bytes", "gauge"), - ("qbit_prism_process_malloc_in_use_bytes", "gauge"), - ("qbit_prism_process_malloc_free_bytes", "gauge"), - ("qbit_prism_process_malloc_mmapped_bytes", "gauge"), -) -# The only labelled heap families; their sole label is the closed -# three-value generation set. -PROCESS_HEAP_GENERATION_FAMILIES = frozenset( - { - "qbit_prism_process_gc_trigger_count", - "qbit_prism_process_gc_collections_total", - "qbit_prism_process_gc_collected_objects_total", - "qbit_prism_process_gc_uncollectable_objects_total", - } -) -COMPONENT_CARDINALITY_METRIC_FAMILIES = ( - ("qbit_prism_component_entries", "gauge"), - ("qbit_prism_component_bytes", "gauge"), -) -# The parity fixture pins the collector to this sample: allocation and GC -# counters move between the reference and the actual render. -PINNED_HEAP_SAMPLE = ProcessHeapSample( - allocated_blocks=123_456, - gc_trigger_count=(5, 6, 7), - gc_collections=(100, 10, 1), - gc_collected=(1_000, 100, 10), - gc_uncollectable=(0, 0, 2), - threads=64, - malloc_info_available=True, - malloc_arena_bytes=9_000, - malloc_in_use_bytes=6_000, - malloc_free_bytes=3_000, - malloc_mmapped_bytes=500, -) - - -def reference_process_heap_metrics_lines(sample: ProcessHeapSample) -> list[str]: - lines = [ - "# HELP qbit_prism_process_allocated_blocks Memory blocks currently allocated by the CPython allocator (sys.getallocatedblocks); a live-object proxy that rises with retention and stays flat under pure allocator fragmentation.", - "# TYPE qbit_prism_process_allocated_blocks gauge", - f"qbit_prism_process_allocated_blocks {int(sample.allocated_blocks)}", - "# HELP qbit_prism_process_gc_trigger_count gc.get_count(): the cyclic collector's per-generation trigger counters, compared against gc.get_threshold() to decide when to collect. Generation 0 is allocations minus deallocations, NOT a count of retained objects. CPython 3.13 made the collector incremental, so the third entry is unused and reads 0; the series is kept for a fixed family set.", - "# TYPE qbit_prism_process_gc_trigger_count gauge", - ] - lines.extend( - f'qbit_prism_process_gc_trigger_count{{generation="{generation}"}} {int(count)}' - for generation, count in zip(PRISM_GC_GENERATIONS, sample.gc_trigger_count) - ) - lines.extend( - [ - "# HELP qbit_prism_process_gc_collections_total Cyclic collector passes completed per generation since process start (gc.get_stats).", - "# TYPE qbit_prism_process_gc_collections_total counter", - ] - ) - lines.extend( - f'qbit_prism_process_gc_collections_total{{generation="{generation}"}} {int(count)}' - for generation, count in zip(PRISM_GC_GENERATIONS, sample.gc_collections) - ) - lines.extend( - [ - "# HELP qbit_prism_process_gc_collected_objects_total Objects the cyclic collector freed per generation since process start.", - "# TYPE qbit_prism_process_gc_collected_objects_total counter", - ] - ) - lines.extend( - f'qbit_prism_process_gc_collected_objects_total{{generation="{generation}"}} {int(count)}' - for generation, count in zip(PRISM_GC_GENERATIONS, sample.gc_collected) - ) - lines.extend( - [ - "# HELP qbit_prism_process_gc_uncollectable_objects_total Objects the cyclic collector found unreachable but could not free per generation since process start; growth is a leak the collector already knows about.", - "# TYPE qbit_prism_process_gc_uncollectable_objects_total counter", - ] - ) - lines.extend( - f'qbit_prism_process_gc_uncollectable_objects_total{{generation="{generation}"}} {int(count)}' - for generation, count in zip(PRISM_GC_GENERATIONS, sample.gc_uncollectable) - ) - lines.extend( - [ - "# HELP qbit_prism_process_threads Live Python threads (threading.active_count); each one is a candidate glibc malloc arena.", - "# TYPE qbit_prism_process_threads gauge", - f"qbit_prism_process_threads {int(sample.threads)}", - "# HELP qbit_prism_process_malloc_info_available Whether glibc mallinfo2 is bound and PRISM_MALLOC_TELEMETRY is on (1), or the allocator byte gauges are unavailable (0): the symbol is glibc 2.33+ only and absent on musl and macOS. While 0 the byte gauges render -1, never 0.", - "# TYPE qbit_prism_process_malloc_info_available gauge", - f"qbit_prism_process_malloc_info_available {1 if sample.malloc_info_available else 0}", - "# HELP qbit_prism_process_malloc_arena_bytes Heap space glibc malloc holds for its arenas across every thread arena, excluding mmapped chunks (mallinfo2.arena), or -1 when unavailable.", - "# TYPE qbit_prism_process_malloc_arena_bytes gauge", - f"qbit_prism_process_malloc_arena_bytes {int(sample.malloc_arena_bytes)}", - "# HELP qbit_prism_process_malloc_in_use_bytes Bytes in allocated chunks across every glibc malloc arena (mallinfo2.uordblks), or -1 when unavailable; arena minus this is the fragmentation glibc has not returned to the kernel.", - "# TYPE qbit_prism_process_malloc_in_use_bytes gauge", - f"qbit_prism_process_malloc_in_use_bytes {int(sample.malloc_in_use_bytes)}", - "# HELP qbit_prism_process_malloc_free_bytes Bytes in free chunks glibc malloc retains across every arena (mallinfo2.fordblks), or -1 when unavailable.", - "# TYPE qbit_prism_process_malloc_free_bytes gauge", - f"qbit_prism_process_malloc_free_bytes {int(sample.malloc_free_bytes)}", - "# HELP qbit_prism_process_malloc_mmapped_bytes Bytes in chunks glibc malloc served directly through mmap (mallinfo2.hblkhd), or -1 when unavailable.", - "# TYPE qbit_prism_process_malloc_mmapped_bytes gauge", - f"qbit_prism_process_malloc_mmapped_bytes {int(sample.malloc_mmapped_bytes)}", - ] - ) - return lines - - -def reference_component_cardinality_metrics_lines(server) -> list[str]: - server._ensure_job_cache_state() - bundles = server._ensure_job_bundle_service() - payout = server._ensure_payout_state_service() - writer = server._ensure_share_writer_service() - compiler = server._ensure_bundle_compiler() - candidates = server._ensure_block_candidate_service() - candidates._ensure_block_replay_state() - candidates._ensure_block_candidate_disposition_state() - reconciler = server.__dict__.get("_reorg_reconciler_service") - serialization = bundles._share_window_serialization - cached_window = payout._incremental_payout_artifact_window - artifact = payout._payout_ledger_artifact - client = compiler._serve_builder - window = cached_window.window if cached_window is not None else None - pages = records = canonical = mirror_records = mirror_bytes = 0 - if isinstance(window, DaemonShareWindowMirror): - # The mirror is the cached window on the Rust path, so it is accounted - # once, under daemon_window_mirror_*; payout_window_* stays zero. - mirror_records = int(window.record_count) - mirror_bytes = len(window.canonical_items) - elif window is not None: - pages = len(window.pages) - records = sum(len(page.records) for page in window.pages) - canonical = sum(len(page.canonical_json_items) for page in window.pages) - with server.lock: - server._ensure_evicted_job_state() - locked = { - "evicted_job_graveyard": len(server.evicted_job_graveyard), - "evicted_same_tip_job_ids": len(server.evicted_same_tip_job_ids), - "outstanding_block_candidate_hashes": len( - candidates._outstanding_block_candidate_hashes - ), - "tip_observed_accepted_block_hashes": len( - candidates._tip_observed_accepted_block_hashes - ), - "counted_block_candidate_abandonments": len( - candidates._counted_block_candidate_abandonments - ), - "ancestor_redrive_records": len(candidates._ancestor_redrive_records), - "block_candidate_terminal_outcomes": len( - candidates._block_candidate_terminal_outcomes - ), - "block_disposition_waiting_retries": len( - candidates._block_disposition_waiting_retries - ), - "block_candidate_dequeued_hashes": len( - candidates._block_candidate_dequeued_hashes - ), - "accounted_accepted_block_hashes": len( - server._accounted_accepted_block_hashes - ), - "reconcile_trusted_memo": ( - len(reconciler._reorg_reconcile_trusted_memo) - if reconciler is not None - else 0 - ), - } - entries = { - "payout_window_pages": pages, - "payout_window_records": records, - "payout_ledger_artifact_shares": ( - len(artifact.shares_json) if artifact is not None else 0 - ), - "accepted_block_payout_previews": len(payout._accepted_block_payout_previews), - "invalidated_accepted_block_payout_previews": len( - payout._invalidated_accepted_block_payout_previews - ), - "payout_append_invalidation_stamps": len( - payout._payout_append_invalidation_stamps - ), - "payout_window_inflight_scan_anchors": len( - payout._payout_window_inflight_scan_anchors - ), - "payout_unfenced_append_inflight_stamps": len( - payout._payout_unfenced_append_inflight_stamps - ), - "share_window_serialization_shares": ( - int(serialization.share_count) if serialization is not None else 0 - ), - "job_contexts": len(getattr(server, "jobs", {})), - "job_bundle_cache": len(bundles._job_bundle_cache), - "job_build_issued_stamps": len(bundles._job_build_issued_at_ms), - "bundle_preparation_flights": len(bundles._bundle_preparation_flights), - "active_job_bundle_builds": len(bundles._active_job_bundle_builds), - "block_candidate_queue": candidates.candidate_queue.qsize(), - "block_replay_queue": candidates._block_replay_candidate_queue.qsize(), - "block_replay_inflight_hashes": len(candidates._block_replay_inflight_hashes), - "block_quarantine_queue": candidates._block_quarantine_queue.qsize(), - "block_quarantine_hashes": len(candidates._block_quarantine_hashes), - "accepted_block_preview_stamps": len( - candidates._accepted_block_preview_acceptance_monotonic - ), - "block_candidate_disposition_flights": len( - candidates._block_candidate_disposition_flights - ), - "reconcile_flights": ( - len(reconciler._reconcile_flights) if reconciler is not None else 0 - ), - "pending_share_commit_floor": len(writer._pending_share_commit_floor), - "daemon_window_mirror_records": mirror_records, - "daemon_uploaded_windows": ( - len(client.uploaded_windows) if client is not None else 0 - ), - **locked, - } - byte_sizes = { - "payout_window_canonical_json": canonical, - "daemon_window_mirror_canonical_items": mirror_bytes, - "share_window_serialization_spool": ( - int(serialization._spool_size) if serialization is not None else 0 - ), - "share_window_serialization_compact_json": ( - int(serialization.compact_json_bytes) - if serialization is not None - else 0 - ), - } - return [ - "# HELP qbit_prism_component_entries Entries retained by each in-process coordinator component; the component label set is closed and carries no job id, tip hash, generation, or worker name.", - "# TYPE qbit_prism_component_entries gauge", - *[ - f'qbit_prism_component_entries{{component="{kind}"}} {int(entries[kind])}' - for kind in PRISM_COMPONENT_ENTRY_KINDS - ], - "# HELP qbit_prism_component_bytes Bytes retained by each byte-sized coordinator payload (the cached payout window's canonical JSON, the daemon window mirror, and the share-window serialization compact strings); the component label set is closed. All are resident bytes except share_window_serialization_spool, which is an on-disk temporary file and is not part of RSS.", - "# TYPE qbit_prism_component_bytes gauge", - *[ - f'qbit_prism_component_bytes{{component="{kind}"}} {int(byte_sizes[kind])}' - for kind in PRISM_COMPONENT_BYTE_KINDS - ], - ] - - -def reference_landing_observability_metrics_lines(server) -> list[str]: - lines: list[str] = [ - "# HELP qbit_prism_block_ledger_calls_total Submitter ledger calls by deadline class.", - "# TYPE qbit_prism_block_ledger_calls_total counter", - "# HELP qbit_prism_block_ledger_call_timeouts_total Submitter ledger deadline expiries by deadline class.", - "# TYPE qbit_prism_block_ledger_call_timeouts_total counter", - "# HELP qbit_prism_block_ledger_call_budget_seconds Most recent statement budget applied per deadline class (escalates for retried landings).", - "# TYPE qbit_prism_block_ledger_call_budget_seconds gauge", - "# HELP qbit_prism_block_ledger_call_last_duration_seconds Duration of the most recent call per deadline class.", - "# TYPE qbit_prism_block_ledger_call_last_duration_seconds gauge", - "# HELP qbit_prism_block_ledger_call_max_duration_seconds Longest observed call per deadline class since process start.", - "# TYPE qbit_prism_block_ledger_call_max_duration_seconds gauge", - ] - for call_class, stats in sorted( - server.block_ledger_call_class_metrics().items() - ): - label = server.prometheus_label_value(call_class) - lines.extend( - [ - f'qbit_prism_block_ledger_calls_total{{call_class="{label}"}} {int(stats["calls_total"])}', - f'qbit_prism_block_ledger_call_timeouts_total{{call_class="{label}"}} {int(stats["timeouts_total"])}', - f'qbit_prism_block_ledger_call_budget_seconds{{call_class="{label}"}} {float(stats["last_budget_seconds"]):.6f}', - f'qbit_prism_block_ledger_call_last_duration_seconds{{call_class="{label}"}} {float(stats["last_duration_seconds"]):.6f}', - f'qbit_prism_block_ledger_call_max_duration_seconds{{call_class="{label}"}} {float(stats["max_duration_seconds"]):.6f}', - ] - ) - unresolved_depth = server._accepted_parent_unresolved_depth() - unresolved_depth_cap = server._accepted_parent_unresolved_depth_cap() - unresolved_ages = server.accepted_parent_unresolved_ages_seconds() - oldest_unresolved = max(unresolved_ages) if unresolved_ages else -1.0 - with server.lock: - preview_wait_timeouts = int( - getattr(server, "_accepted_parent_preview_wait_timeouts", 0) - ) - lines.extend( - [ - "# HELP qbit_prism_accepted_parent_unresolved_transitions Landed accepted-block transitions whose durable bookkeeping is unresolved.", - "# TYPE qbit_prism_accepted_parent_unresolved_transitions gauge", - f"qbit_prism_accepted_parent_unresolved_transitions {unresolved_depth}", - "# HELP qbit_prism_accepted_parent_unresolved_depth_max Configured unresolved accepted-parent depth at which job admission fails closed.", - "# TYPE qbit_prism_accepted_parent_unresolved_depth_max gauge", - f"qbit_prism_accepted_parent_unresolved_depth_max {unresolved_depth_cap}", - "# HELP qbit_prism_accepted_parent_unresolved_oldest_seconds Age of the oldest unresolved accepted-parent transition, or -1 when none.", - "# TYPE qbit_prism_accepted_parent_unresolved_oldest_seconds gauge", - f"qbit_prism_accepted_parent_unresolved_oldest_seconds {oldest_unresolved:.6f}", - "# HELP qbit_prism_accepted_parent_preview_wait_timeouts_total Child job builds that timed out waiting for an accepted-parent payout preview.", - "# TYPE qbit_prism_accepted_parent_preview_wait_timeouts_total counter", - f"qbit_prism_accepted_parent_preview_wait_timeouts_total {preview_wait_timeouts}", - "# HELP qbit_prism_accepted_parent_redrive_attempts_total In-process ancestor re-drives armed after repeated finalization deferrals on one pending accepted-parent transition.", - "# TYPE qbit_prism_accepted_parent_redrive_attempts_total counter", - "qbit_prism_accepted_parent_redrive_attempts_total " - f"{int(getattr(server, 'accepted_parent_redrive_attempt_count', 0))}", - "# HELP qbit_prism_accepted_parent_redrive_resolved_total Pending accepted-parent transitions that resolved after at least one in-process re-drive attempt.", - "# TYPE qbit_prism_accepted_parent_redrive_resolved_total counter", - "qbit_prism_accepted_parent_redrive_resolved_total " - f"{int(getattr(server, 'accepted_parent_redrive_resolved_count', 0))}", - "# HELP qbit_prism_accepted_parent_redrive_exhausted_total Ancestors whose re-drive attempt cap was exhausted with the transition still pending; the publication-progress watchdog remains the backstop.", - "# TYPE qbit_prism_accepted_parent_redrive_exhausted_total counter", - "qbit_prism_accepted_parent_redrive_exhausted_total " - f"{int(getattr(server, 'accepted_parent_redrive_exhausted_count', 0))}", - ] - ) - prior_stats_fn = getattr(server.ledger, "prior_balances_read_stats", None) - if callable(prior_stats_fn): - prior_stats = prior_stats_fn() - lines.extend( - [ - "# HELP qbit_prism_prior_balances_reads_total Prior-balances reads served by the ledger.", - "# TYPE qbit_prism_prior_balances_reads_total counter", - f"qbit_prism_prior_balances_reads_total {int(prior_stats['reads_total'])}", - "# HELP qbit_prism_prior_balances_read_last_seconds Duration of the most recent prior-balances read.", - "# TYPE qbit_prism_prior_balances_read_last_seconds gauge", - f"qbit_prism_prior_balances_read_last_seconds {float(prior_stats['last_seconds']):.6f}", - "# HELP qbit_prism_prior_balances_read_max_seconds Longest prior-balances read since process start.", - "# TYPE qbit_prism_prior_balances_read_max_seconds gauge", - f"qbit_prism_prior_balances_read_max_seconds {float(prior_stats['max_seconds']):.6f}", - ] - ) - read_gate_fn = getattr(server.ledger, "ledger_read_gate_stats", None) - read_gate_stats = ( - fold_ledger_read_stats(read_gate_fn() or {}) - if callable(read_gate_fn) - else {} - ) - if read_gate_stats: - families = ( - ( - "qbit_prism_ledger_read_calls_total", - "counter", - "Ledger read-slot operations completed or failed, by operation.", - "calls_total", - True, - ), - ( - "qbit_prism_ledger_read_gate_wait_seconds_total", - "counter", - "Cumulative coordinator-local read-slot admission wait, by operation.", - "gate_wait_seconds_total", - False, - ), - ( - "qbit_prism_ledger_read_gate_wait_seconds_max", - "gauge", - "Longest coordinator-local read-slot admission wait since process start, by operation.", - "gate_wait_seconds_max", - False, - ), - ( - "qbit_prism_ledger_read_gate_timeouts_total", - "counter", - "Read-slot operations whose deadline expired before admission, so no statement was ever sent.", - "gate_timeouts_total", - True, - ), - ( - "qbit_prism_ledger_read_execute_seconds_total", - "counter", - "Cumulative PostgreSQL execution time for admitted read-slot statements, by operation.", - "execute_seconds_total", - False, - ), - ( - "qbit_prism_ledger_read_execute_seconds_max", - "gauge", - "Longest PostgreSQL execution time for an admitted read-slot statement since process start, by operation.", - "execute_seconds_max", - False, - ), - ( - "qbit_prism_ledger_read_execute_timeouts_total", - "counter", - "Admitted read-slot statements whose deadline expired inside PostgreSQL, cancel lag included.", - "execute_timeouts_total", - True, - ), - ) - operations = [ - (server.prometheus_label_value(str(operation)), stats) - for operation, stats in sorted(read_gate_stats.items()) - ] - for name, metric_type, help_text, field, integral in families: - lines.append(f"# HELP {name} {help_text}") - lines.append(f"# TYPE {name} {metric_type}") - for label, stats in operations: - value = stats.get(field, 0) - rendered = f"{int(value)}" if integral else f"{float(value):.6f}" - lines.append(f'{name}{{operation="{label}"}} {rendered}') - startup_phases = server.startup_phase_seconds() - if startup_phases: - lines.extend( - [ - "# HELP qbit_prism_startup_phase_seconds Seconds from serve() start to each startup phase, recorded once.", - "# TYPE qbit_prism_startup_phase_seconds gauge", - ] - ) - for phase, seconds in sorted(startup_phases.items()): - label = server.prometheus_label_value(phase) - lines.append( - f'qbit_prism_startup_phase_seconds{{phase="{label}"}} {float(seconds):.6f}' - ) - return lines - - -def reference_accepted_stats_reconcile_metric_lines(server) -> list[str]: - status_fn = getattr(server.ledger, "accepted_stats_reconcile_status", None) - if not callable(status_fn): - return [] - status = status_fn() - lines = [ - "# HELP qbit_prism_accepted_stats_reconcile_failures_total Failed background accepted-stats reconcile passes.", - "# TYPE qbit_prism_accepted_stats_reconcile_failures_total counter", - f"qbit_prism_accepted_stats_reconcile_failures_total {int(status.get('failures') or 0)}", - ] - age = status.get("age_seconds") - if age is not None: - lines.extend( - [ - "# HELP qbit_prism_accepted_stats_reconcile_age_seconds Seconds since the accepted-share counters were last reconciled against the ledger.", - "# TYPE qbit_prism_accepted_stats_reconcile_age_seconds gauge", - f"qbit_prism_accepted_stats_reconcile_age_seconds {float(age):.6f}", - ] - ) - return lines - - -def reference_lease_heartbeat_metrics_lines(server) -> list[str]: - snapshot = server._ensure_lease_heartbeat_service().snapshot() - attempts = snapshot["attempts"] - outcomes = snapshot["outcomes"] - last_phases = snapshot["last_phase_seconds"] - worst_phases = snapshot["worst_phase_seconds"] - policy_seconds = snapshot["policy_seconds"] - wakes = snapshot["monitor_wakes"] - probe = snapshot["stall_probe"] - assert isinstance(attempts, dict) - assert isinstance(outcomes, dict) - assert isinstance(last_phases, dict) - assert isinstance(worst_phases, dict) - assert isinstance(policy_seconds, dict) - assert isinstance(wakes, dict) - assert isinstance(probe, dict) - wake_buckets = wakes["buckets"] - late_wakes = wakes["late_wakes"] - assert isinstance(wake_buckets, dict) - assert isinstance(late_wakes, dict) - return [ - "# HELP qbit_prism_lease_heartbeat_running Whether the writer-lease heartbeat thread is alive.", - "# TYPE qbit_prism_lease_heartbeat_running gauge", - f"qbit_prism_lease_heartbeat_running {int(bool(snapshot['running']))}", - "# HELP qbit_prism_lease_heartbeat_attempts_total Guard verifications by mode.", - "# TYPE qbit_prism_lease_heartbeat_attempts_total counter", - *[ - f'qbit_prism_lease_heartbeat_attempts_total{{mode="{mode}"}} {int(attempts.get(mode, 0))}' - for mode in LEASE_HEARTBEAT_MODES - ], - "# HELP qbit_prism_lease_heartbeat_outcomes_total Guard verification outcomes.", - "# TYPE qbit_prism_lease_heartbeat_outcomes_total counter", - *[ - f'qbit_prism_lease_heartbeat_outcomes_total{{outcome="{outcome}"}} {int(outcomes.get(outcome, 0))}' - for outcome in LEASE_HEARTBEAT_OUTCOMES - ], - "# HELP qbit_prism_lease_heartbeat_phase_seconds Phase breakdown of the latest guard verification.", - "# TYPE qbit_prism_lease_heartbeat_phase_seconds gauge", - *[ - f'qbit_prism_lease_heartbeat_phase_seconds{{phase="{phase}"}} {float(last_phases.get(phase, 0.0)):.6f}' - for phase in LEASE_HEARTBEAT_PHASES - ], - "# HELP qbit_prism_lease_heartbeat_worst_phase_seconds Phase breakdown of the slowest guard verification observed.", - "# TYPE qbit_prism_lease_heartbeat_worst_phase_seconds gauge", - *[ - f'qbit_prism_lease_heartbeat_worst_phase_seconds{{phase="{phase}"}} {float(worst_phases.get(phase, 0.0)):.6f}' - for phase in LEASE_HEARTBEAT_PHASES - ], - "# HELP qbit_prism_lease_heartbeat_activity_age_seconds Age of the newest monitor-visible heartbeat activity mark.", - "# TYPE qbit_prism_lease_heartbeat_activity_age_seconds gauge", - f"qbit_prism_lease_heartbeat_activity_age_seconds {float(snapshot['activity_age_seconds']):.6f}", - "# HELP qbit_prism_lease_heartbeat_server_proven_age_seconds Age of the newest completed guard round trip.", - "# TYPE qbit_prism_lease_heartbeat_server_proven_age_seconds gauge", - f"qbit_prism_lease_heartbeat_server_proven_age_seconds {float(snapshot['server_proven_age_seconds']):.6f}", - "# HELP qbit_prism_lease_heartbeat_monitor_wake_delay_seconds Worst observed lateness of the heartbeat monitor's own poll.", - "# TYPE qbit_prism_lease_heartbeat_monitor_wake_delay_seconds gauge", - f"qbit_prism_lease_heartbeat_monitor_wake_delay_seconds {float(snapshot['monitor_wake_delay_seconds']):.6f}", - "# HELP qbit_prism_lease_heartbeat_policy_seconds Terms of the validated writer-lease heartbeat timing policy.", - "# TYPE qbit_prism_lease_heartbeat_policy_seconds gauge", - *[ - f'qbit_prism_lease_heartbeat_policy_seconds{{term="{term}"}} {float(policy_seconds.get(term, 0.0)):.6f}' - for term in LEASE_HEARTBEAT_POLICY_TERMS - ], - "# HELP qbit_prism_lease_heartbeat_monitor_wake_lateness_seconds Lateness of every heartbeat-monitor poll wake (issue #227); express the wake-delay alert on this or on the late-wake counters, not on the lifetime gauge.", - "# TYPE qbit_prism_lease_heartbeat_monitor_wake_lateness_seconds histogram", - *[ - f'qbit_prism_lease_heartbeat_monitor_wake_lateness_seconds_bucket{{le="{bucket:g}"}} {int(wake_buckets.get(bucket, 0))}' - for bucket in LEASE_MONITOR_WAKE_DELAY_BUCKETS - ], - f'qbit_prism_lease_heartbeat_monitor_wake_lateness_seconds_bucket{{le="+Inf"}} {int(wakes["count"])}', - f'qbit_prism_lease_heartbeat_monitor_wake_lateness_seconds_sum {float(wakes["sum"]):.6f}', - f'qbit_prism_lease_heartbeat_monitor_wake_lateness_seconds_count {int(wakes["count"])}', - "# HELP qbit_prism_lease_heartbeat_monitor_late_wakes_total Monitor wakes at least the labelled fraction of the policy's scheduler slack late.", - "# TYPE qbit_prism_lease_heartbeat_monitor_late_wakes_total counter", - *[ - f'qbit_prism_lease_heartbeat_monitor_late_wakes_total{{slack_fraction="{fraction}"}} {int(late_wakes.get(fraction, 0))}' - for fraction in LEASE_MONITOR_LATE_WAKE_SLACK_FRACTIONS - ], - "# HELP qbit_prism_lease_heartbeat_monitor_wake_delay_window_max_seconds Worst monitor wake lateness inside the trailing 300s window; falls back once a stall ages out, unlike the lifetime gauge.", - "# TYPE qbit_prism_lease_heartbeat_monitor_wake_delay_window_max_seconds gauge", - f"qbit_prism_lease_heartbeat_monitor_wake_delay_window_max_seconds {float(wakes['window_max_seconds']):.6f}", - "# HELP qbit_prism_lease_heartbeat_monitor_wake_delay_record_age_seconds Seconds since the lifetime monitor wake-delay record was last raised.", - "# TYPE qbit_prism_lease_heartbeat_monitor_wake_delay_record_age_seconds gauge", - f"qbit_prism_lease_heartbeat_monitor_wake_delay_record_age_seconds {float(wakes['record_age_seconds']):.6f}", - "# HELP qbit_prism_lease_heartbeat_monitor_exit_guarantee_breaches_total Monitor wakes later than the policy's max_guaranteed_monitor_lateness: beats on which exit-before-adoption was not guaranteed (issue #227).", - "# TYPE qbit_prism_lease_heartbeat_monitor_exit_guarantee_breaches_total counter", - f"qbit_prism_lease_heartbeat_monitor_exit_guarantee_breaches_total {int(snapshot['exit_guarantee_breaches'])}", - "# HELP qbit_prism_lease_heartbeat_monitor_worst_exit_guarantee_overrun_seconds Worst amount by which a late monitor wake could have placed the hard exit after the successor's adoption edge.", - "# TYPE qbit_prism_lease_heartbeat_monitor_worst_exit_guarantee_overrun_seconds gauge", - f"qbit_prism_lease_heartbeat_monitor_worst_exit_guarantee_overrun_seconds {float(snapshot['worst_exit_guarantee_overrun_seconds']):.6f}", - "# HELP qbit_prism_lease_heartbeat_stall_probe_samples_total Stack samples the stall probe took on monitor wakes at least half a scheduler slack late.", - "# TYPE qbit_prism_lease_heartbeat_stall_probe_samples_total counter", - f"qbit_prism_lease_heartbeat_stall_probe_samples_total {int(probe['samples'])}", - "# HELP qbit_prism_lease_heartbeat_stall_probe_suppressed_total Stall-probe triggers refused by its rate limit (at most 3 samples per 60s).", - "# TYPE qbit_prism_lease_heartbeat_stall_probe_suppressed_total counter", - f"qbit_prism_lease_heartbeat_stall_probe_suppressed_total {int(probe['suppressed'])}", - "# HELP qbit_prism_lease_heartbeat_monitor_breach_warnings_suppressed_total Lateness-beyond-slack warnings the monitor refused under its rate limit (at most 3 per 60s); the breaches themselves are still counted.", - "# TYPE qbit_prism_lease_heartbeat_monitor_breach_warnings_suppressed_total counter", - f"qbit_prism_lease_heartbeat_monitor_breach_warnings_suppressed_total {int(snapshot['slack_breach_warnings_suppressed'])}", - ] - - -def reference_gc_pause_metrics_lines(server) -> list[str]: - pauses = server._ensure_lease_heartbeat_service().snapshot()["gc_pauses"] - assert isinstance(pauses, dict) - lines = [ - "# HELP qbit_prism_process_gc_pause_seconds Cyclic collector pause duration per generation, from gc.callbacks; the qbit_prism_process_gc_* count families say how many passes ran, this says how long each held the interpreter.", - "# TYPE qbit_prism_process_gc_pause_seconds histogram", - ] - for generation in PRISM_GC_GENERATIONS: - entry = pauses[generation] - buckets = entry["buckets"] - lines.extend( - f'qbit_prism_process_gc_pause_seconds_bucket{{generation="{generation}",le="{bucket:g}"}} {int(buckets.get(bucket, 0))}' - for bucket in PRISM_GC_PAUSE_SECONDS_BUCKETS - ) - lines.append( - f'qbit_prism_process_gc_pause_seconds_bucket{{generation="{generation}",le="+Inf"}} {int(entry["count"])}' - ) - lines.append( - f'qbit_prism_process_gc_pause_seconds_sum{{generation="{generation}"}} {float(entry["sum"]):.6f}' - ) - lines.append( - f'qbit_prism_process_gc_pause_seconds_count{{generation="{generation}"}} {int(entry["count"])}' - ) - lines.extend( - [ - "# HELP qbit_prism_process_gc_last_pause_seconds Duration of the most recent cyclic collector pass per generation.", - "# TYPE qbit_prism_process_gc_last_pause_seconds gauge", - *[ - f'qbit_prism_process_gc_last_pause_seconds{{generation="{generation}"}} {float(pauses[generation]["last_seconds"]):.6f}' - for generation in PRISM_GC_GENERATIONS - ], - "# HELP qbit_prism_process_gc_max_pause_seconds Longest cyclic collector pass observed per generation since the lease heartbeat armed.", - "# TYPE qbit_prism_process_gc_max_pause_seconds gauge", - *[ - f'qbit_prism_process_gc_max_pause_seconds{{generation="{generation}"}} {float(pauses[generation]["max_seconds"]):.6f}' - for generation in PRISM_GC_GENERATIONS - ], - ] - ) - return lines - - -def reference_shutdown_metrics_lines(server) -> list[str]: - snapshot = server._ensure_shutdown_controller().snapshot() - quiescence = snapshot["writer_quiescence_outcomes"] - release = snapshot["lease_release_outcomes"] - active = snapshot["active_writers"] - assert isinstance(quiescence, dict) - assert isinstance(release, dict) - assert isinstance(active, dict) - return [ - "# HELP qbit_prism_shutdowns_total Controlled coordinator shutdown sequences started.", - "# TYPE qbit_prism_shutdowns_total counter", - f"qbit_prism_shutdowns_total {int(snapshot['shutdowns_total'])}", - "# HELP qbit_prism_shutdown_writer_operations Active admitted ledger-mutating operations by component.", - "# TYPE qbit_prism_shutdown_writer_operations gauge", - *[ - f'qbit_prism_shutdown_writer_operations{{component="{server.prometheus_label_value(str(component))}"}} {int(count)}' - for component, count in sorted(active.items()) - ], - "# HELP qbit_prism_shutdown_writer_quiescence_total Writer-quiescence outcomes.", - "# TYPE qbit_prism_shutdown_writer_quiescence_total counter", - *[ - f'qbit_prism_shutdown_writer_quiescence_total{{outcome="{outcome}"}} {int(quiescence.get(outcome, 0))}' - for outcome in ("success", "timeout") - ], - "# HELP qbit_prism_shutdown_writer_quiescence_seconds Duration of the latest writer-quiescence barrier.", - "# TYPE qbit_prism_shutdown_writer_quiescence_seconds gauge", - f"qbit_prism_shutdown_writer_quiescence_seconds {float(snapshot['writer_quiescence_seconds']):.6f}", - "# HELP qbit_prism_shutdown_lease_release_attempts_total Writer-lease release attempts.", - "# TYPE qbit_prism_shutdown_lease_release_attempts_total counter", - f"qbit_prism_shutdown_lease_release_attempts_total {int(snapshot['lease_release_attempts_total'])}", - "# HELP qbit_prism_shutdown_lease_release_total Writer-lease release outcomes.", - "# TYPE qbit_prism_shutdown_lease_release_total counter", - *[ - f'qbit_prism_shutdown_lease_release_total{{outcome="{outcome}"}} {int(release.get(outcome, 0))}' - for outcome in ("success", "not_held", "unsupported", "failure") - ], - "# HELP qbit_prism_shutdown_lease_release_seconds Duration of the latest writer-lease release attempt.", - "# TYPE qbit_prism_shutdown_lease_release_seconds gauge", - f"qbit_prism_shutdown_lease_release_seconds {float(snapshot['lease_release_seconds']):.6f}", - "# HELP qbit_prism_shutdown_sigterm_to_lease_release_seconds Time from SIGTERM admission close to safe lease release, or -1 if unobserved.", - "# TYPE qbit_prism_shutdown_sigterm_to_lease_release_seconds gauge", - "qbit_prism_shutdown_sigterm_to_lease_release_seconds " - + ( - f"{float(snapshot['sigterm_to_lease_release_seconds']):.6f}" - if snapshot["sigterm_release_observed"] - else "-1" - ), - "# HELP qbit_prism_shutdown_release_withheld_total Shutdowns that withheld lease release because a writer did not quiesce.", - "# TYPE qbit_prism_shutdown_release_withheld_total counter", - f"qbit_prism_shutdown_release_withheld_total {int(snapshot['release_withheld_total'])}", - "# HELP qbit_prism_shutdown_non_writer_drain_seconds Duration of cleanup after writer lease handling.", - "# TYPE qbit_prism_shutdown_non_writer_drain_seconds gauge", - f"qbit_prism_shutdown_non_writer_drain_seconds {float(snapshot['non_writer_drain_seconds']):.6f}", - ] - - -def reference_initial_delivery_metrics_lines(server) -> list[str]: - server._ensure_initial_job_state() - mining = server.mining_delivery_snapshot() - with server.lock: - counts = { - "sent": server.initial_job_sent_count, - "cancelled": server.initial_job_cancelled_count, - "coalesced": server.initial_job_coalesced_count, - "failed": server.initial_job_failed_count, - "superseded": server.initial_job_superseded_count, - } - latency_sum = server.initial_job_delivery_latency_seconds_sum - latency_count = server.initial_job_delivery_latency_count - queue_capacity_reclaimed = ( - server.initial_job_queue_capacity_reclaimed_count - ) - executor = getattr(server, "_initial_job_executor", None) - queued, slots = executor.stats() if executor is not None else (0, 0) - configured_workers = int( - getattr( - server, - "initial_job_max_workers", - DEFAULT_PRISM_INITIAL_JOB_MAX_WORKERS, - ) - ) - with server._bundle_preparation_lock: - build_counts = dict(server.shared_bundle_build_counts) - preparation_sum = server.shared_bundle_preparation_seconds_sum - preparation_count = server.shared_bundle_preparation_count - waiters = server.shared_bundle_preparation_waiters - return [ - "# HELP qbit_prism_stratum_subscribed_clients Subscribed Stratum clients.", - "# TYPE qbit_prism_stratum_subscribed_clients gauge", - f'qbit_prism_stratum_subscribed_clients {mining["subscribed_clients"]}', - "# HELP qbit_prism_stratum_authorized_clients Subscribed and authorized Stratum clients.", - "# TYPE qbit_prism_stratum_authorized_clients gauge", - f'qbit_prism_stratum_authorized_clients {mining["authorized_clients"]}', - "# HELP qbit_prism_clients_without_current_tip_job Authorized clients without usable current-tip work.", - "# TYPE qbit_prism_clients_without_current_tip_job gauge", - f'qbit_prism_clients_without_current_tip_job {mining["clients_without_current_tip_job"]}', - "# HELP qbit_prism_clients_with_no_active_job Authorized clients with no active job at all.", - "# TYPE qbit_prism_clients_with_no_active_job gauge", - f'qbit_prism_clients_with_no_active_job {mining["clients_with_no_active_job"]}', - "# HELP qbit_prism_clients_with_current_tip_job Authorized clients with usable current-tip work.", - "# TYPE qbit_prism_clients_with_current_tip_job gauge", - f'qbit_prism_clients_with_current_tip_job {mining["clients_with_current_tip_job"]}', - "# HELP qbit_prism_current_tip_job_coverage_ratio Fraction of authorized clients with current-tip work.", - "# TYPE qbit_prism_current_tip_job_coverage_ratio gauge", - f'qbit_prism_current_tip_job_coverage_ratio {float(mining["current_tip_job_coverage_ratio"]):.12g}', - "# HELP qbit_prism_initial_job_deliveries_pending Coalesced initial deliveries queued or running.", - "# TYPE qbit_prism_initial_job_deliveries_pending gauge", - f'qbit_prism_initial_job_deliveries_pending {mining["clients_pending_initial_job"]}', - "# HELP qbit_prism_initial_job_delivery_tasks_inflight Bounded shared delivery slots currently occupied.", - "# TYPE qbit_prism_initial_job_delivery_tasks_inflight gauge", - f"qbit_prism_initial_job_delivery_tasks_inflight {slots}", - "# HELP qbit_prism_initial_job_delivery_queue_depth Initial-job tasks waiting for a dedicated worker.", - "# TYPE qbit_prism_initial_job_delivery_queue_depth gauge", - f"qbit_prism_initial_job_delivery_queue_depth {queued}", - "# HELP qbit_prism_initial_job_delivery_active_workers Dedicated initial-job workers currently running tasks.", - "# TYPE qbit_prism_initial_job_delivery_active_workers gauge", - f"qbit_prism_initial_job_delivery_active_workers {slots}", - "# HELP qbit_prism_initial_job_delivery_configured_workers Configured dedicated initial-job worker count.", - "# TYPE qbit_prism_initial_job_delivery_configured_workers gauge", - f"qbit_prism_initial_job_delivery_configured_workers {configured_workers}", - "# HELP qbit_prism_initial_job_delivery_seconds Authorization-to-current-job latency.", - "# TYPE qbit_prism_initial_job_delivery_seconds summary", - f"qbit_prism_initial_job_delivery_seconds_sum {latency_sum:.6f}", - f"qbit_prism_initial_job_delivery_seconds_count {latency_count}", - "# HELP qbit_prism_initial_job_requests_total Initial delivery outcomes.", - "# TYPE qbit_prism_initial_job_requests_total counter", - *[ - f'qbit_prism_initial_job_requests_total{{result="{result}"}} {count}' - for result, count in sorted(counts.items()) - ], - "# HELP qbit_prism_initial_job_queue_capacity_reclaimed_total Queued initial-job slots reclaimed immediately by cancellation.", - "# TYPE qbit_prism_initial_job_queue_capacity_reclaimed_total counter", - f"qbit_prism_initial_job_queue_capacity_reclaimed_total {queue_capacity_reclaimed}", - "# HELP qbit_prism_shared_bundle_preparation_seconds Heavy shared bundle preparation wall time.", - "# TYPE qbit_prism_shared_bundle_preparation_seconds summary", - f"qbit_prism_shared_bundle_preparation_seconds_sum {preparation_sum:.6f}", - f"qbit_prism_shared_bundle_preparation_seconds_count {preparation_count}", - "# HELP qbit_prism_shared_bundle_preparation_waiters Callers waiting on the keyed shared preparation flight.", - "# TYPE qbit_prism_shared_bundle_preparation_waiters gauge", - f"qbit_prism_shared_bundle_preparation_waiters {waiters}", - "# HELP qbit_prism_shared_bundle_builds_total Shared bundle builds by terminal outcome.", - "# TYPE qbit_prism_shared_bundle_builds_total counter", - *[ - f'qbit_prism_shared_bundle_builds_total{{result="{result}"}} {count}' - for result, count in sorted(build_counts.items()) - ], - ] - - -def reference_job_build_metrics_lines(server) -> list[str]: - server._ensure_job_cache_state() - with server._job_cache_lock: - bucket_counts = dict(server.job_build_seconds_bucket_counts) - build_sum = server.job_build_seconds_sum - build_count = server.job_build_count - phase_seconds = dict(server.job_build_phase_seconds) - hit_counts = dict(server.job_cache_hit_counts) - miss_counts = dict(server.job_cache_miss_counts) - health_refresh_failures = ( - server._ensure_observability_service() - .state() - .health_snapshot_refresh_failure_count - ) - with server._job_build_scheduler_lock: - scheduler_counts = dict(server.job_build_scheduler_counts) - priority_counts = dict(server.job_build_priority_counts) - priority_admission_seconds = dict( - server.job_build_priority_admission_seconds - ) - initial_prepared_counts = dict( - server.initial_job_prepared_work_counts - ) - cancellation_seconds = dict(server.job_build_cancellation_seconds) - replacement_seconds = dict(server.job_build_replacement_start_seconds) - worker_counts = dict(server.job_build_worker_counts) - active_builds = int(server._job_build_active is not None) - pending_builds = int(server._job_build_pending is not None) - priority_requests = tuple( - request - for request in ( - ( - server._job_build_active.request - if server._job_build_active is not None - else None - ), - ( - server._job_build_retiring.request - if server._job_build_retiring is not None - else None - ), - server._job_build_pending, - ) - if request is not None - and not request.cancellation.is_set() - and server._job_build_is_publication_critical(request) - ) - priority_preparations = tuple( - server._job_build_priority_preparations.values() - ) - priority_active = int( - bool(priority_requests or priority_preparations) - ) - priority_age_seconds = max( - ( - time.monotonic() - request.requested_monotonic - for request in priority_requests - ), - default=0.0, - ) - priority_age_seconds = max( - priority_age_seconds, - max( - ( - time.monotonic() - started - for started in priority_preparations - ), - default=0.0, - ), - ) - lock = getattr(server, "lock", None) - if lock is not None: - with lock: - connected_clients = len(getattr(server, "clients", ())) - else: - connected_clients = len(getattr(server, "clients", ())) - lines = [ - "# HELP qbit_prism_job_build_seconds Wall time from client job build or prepared submission to completion, including skipped prepared tasks.", - "# TYPE qbit_prism_job_build_seconds histogram", - ] - for bucket in PRISM_JOB_BUILD_SECONDS_BUCKETS: - lines.append( - f'qbit_prism_job_build_seconds_bucket{{le="{bucket:g}"}} {bucket_counts.get(bucket, 0)}' - ) - lines.extend( - [ - f'qbit_prism_job_build_seconds_bucket{{le="+Inf"}} {build_count}', - f"qbit_prism_job_build_seconds_sum {build_sum:.6f}", - f"qbit_prism_job_build_seconds_count {build_count}", - "# HELP qbit_prism_job_build_phase_seconds_total Cumulative job build wall time by phase.", - "# TYPE qbit_prism_job_build_phase_seconds_total counter", - *[ - f'qbit_prism_job_build_phase_seconds_total{{phase="{phase}"}} {phase_seconds.get(phase, 0.0):.6f}' - for phase in PRISM_JOB_BUILD_PHASES - ], - "# HELP qbit_prism_job_cache_hits_total Job build cache hits by cache kind.", - "# TYPE qbit_prism_job_cache_hits_total counter", - *[ - f'qbit_prism_job_cache_hits_total{{cache="{kind}"}} {int(hit_counts.get(kind, 0))}' - for kind in PRISM_JOB_CACHE_KINDS - ], - "# HELP qbit_prism_job_cache_misses_total Job build cache misses by cache kind.", - "# TYPE qbit_prism_job_cache_misses_total counter", - *[ - f'qbit_prism_job_cache_misses_total{{cache="{kind}"}} {int(miss_counts.get(kind, 0))}' - for kind in PRISM_JOB_CACHE_KINDS - ], - "# HELP qbit_prism_health_snapshot_refresh_failures_total Background health snapshot refreshes that raised.", - "# TYPE qbit_prism_health_snapshot_refresh_failures_total counter", - f"qbit_prism_health_snapshot_refresh_failures_total {health_refresh_failures}", - "# HELP qbit_prism_connected_clients Currently connected Stratum clients.", - "# TYPE qbit_prism_connected_clients gauge", - f"qbit_prism_connected_clients {connected_clients}", - "# HELP qbit_prism_job_build_requests_total Immutable job build requests admitted to the latest-wins scheduler.", - "# TYPE qbit_prism_job_build_requests_total counter", - f'qbit_prism_job_build_requests_total {int(scheduler_counts.get("requests", 0))}', - "# HELP qbit_prism_job_build_starts_total Immutable job builds started by the bounded executor.", - "# TYPE qbit_prism_job_build_starts_total counter", - f'qbit_prism_job_build_starts_total {int(scheduler_counts.get("starts", 0))}', - "# HELP qbit_prism_job_build_completions_total Immutable job build executions completed.", - "# TYPE qbit_prism_job_build_completions_total counter", - f'qbit_prism_job_build_completions_total {int(scheduler_counts.get("completions", 0))}', - "# HELP qbit_prism_job_build_supersessions_total Active or pending builds replaced by a newer immutable key.", - "# TYPE qbit_prism_job_build_supersessions_total counter", - f'qbit_prism_job_build_supersessions_total {int(scheduler_counts.get("supersessions", 0))}', - "# HELP qbit_prism_job_build_obsolete_results_total Obsolete build results discarded before cache or delivery.", - "# TYPE qbit_prism_job_build_obsolete_results_total counter", - f'qbit_prism_job_build_obsolete_results_total {int(scheduler_counts.get("obsolete_results", 0))}', - "# HELP qbit_prism_job_build_orphan_evicted_total Finished flights evicted from scheduler slots after their completion never released them.", - "# TYPE qbit_prism_job_build_orphan_evicted_total counter", - f'qbit_prism_job_build_orphan_evicted_total {int(scheduler_counts.get("orphan_evicted", 0))}', - "# HELP qbit_prism_job_build_active Current latest-generation build executions.", - "# TYPE qbit_prism_job_build_active gauge", - f"qbit_prism_job_build_active {active_builds}", - "# HELP qbit_prism_job_build_pending Newest build request waiting for a bounded executor slot.", - "# TYPE qbit_prism_job_build_pending gauge", - f"qbit_prism_job_build_pending {pending_builds}", - "# HELP qbit_prism_job_build_cancellation_seconds Cancellation signal to obsolete execution completion.", - "# TYPE qbit_prism_job_build_cancellation_seconds summary", - f'qbit_prism_job_build_cancellation_seconds_sum {float(cancellation_seconds.get("sum", 0.0)):.6f}', - f'qbit_prism_job_build_cancellation_seconds_count {int(cancellation_seconds.get("count", 0))}', - "# HELP qbit_prism_job_build_replacement_start_seconds Supersession signal to replacement build start.", - "# TYPE qbit_prism_job_build_replacement_start_seconds summary", - f'qbit_prism_job_build_replacement_start_seconds_sum {float(replacement_seconds.get("sum", 0.0)):.6f}', - f'qbit_prism_job_build_replacement_start_seconds_count {int(replacement_seconds.get("count", 0))}', - "# HELP qbit_prism_job_build_priority_events_total Publication-critical scheduler admissions and routine-work displacement.", - "# TYPE qbit_prism_job_build_priority_events_total counter", - *[ - f'qbit_prism_job_build_priority_events_total{{result="{result}"}} {int(priority_counts.get(result, 0))}' - for result in ( - "started", - "coalesced", - "queued", - "routine_deferred", - "routine_preempted", - ) - ], - "# HELP qbit_prism_job_build_priority_admission_seconds Publication-priority reservation to builder start or exact-flight coalescing.", - "# TYPE qbit_prism_job_build_priority_admission_seconds summary", - f'qbit_prism_job_build_priority_admission_seconds_sum {float(priority_admission_seconds.get("sum", 0.0)):.6f}', - f'qbit_prism_job_build_priority_admission_seconds_count {int(priority_admission_seconds.get("count", 0))}', - "# HELP qbit_prism_job_build_priority_active Whether publication-critical build work is preparing, running, retiring, or pending.", - "# TYPE qbit_prism_job_build_priority_active gauge", - f"qbit_prism_job_build_priority_active {priority_active}", - "# HELP qbit_prism_job_build_priority_age_seconds Age of the oldest admitted publication-critical build request.", - "# TYPE qbit_prism_job_build_priority_age_seconds gauge", - f"qbit_prism_job_build_priority_age_seconds {priority_age_seconds:.6f}", - "# HELP qbit_prism_initial_job_prepared_work_total Initial jobs that reused, coalesced behind, or deferred to prepared shared work.", - "# TYPE qbit_prism_initial_job_prepared_work_total counter", - *[ - f'qbit_prism_initial_job_prepared_work_total{{result="{result}"}} {int(initial_prepared_counts.get(result, 0))}' - for result in ( - "cache_hit", - "singleflight", - "deferred", - "subscribed", - "admission_deadline", - ) - ], - "# HELP qbit_prism_job_build_worker_events_total Pure builder subprocess lifecycle events.", - "# TYPE qbit_prism_job_build_worker_events_total counter", - *[ - f'qbit_prism_job_build_worker_events_total{{event="{event}"}} {int(worker_counts.get(event, 0))}' - for event in ("starts", "terminations", "crashes", "restarts") - ], - "# HELP qbit_prism_job_build_configured_workers Configured bounded job-build executor worker count.", - "# TYPE qbit_prism_job_build_configured_workers gauge", - ( - "qbit_prism_job_build_configured_workers " - f"{int(getattr(server, 'job_build_executor_workers', DEFAULT_PRISM_JOB_BUILD_EXECUTOR_WORKERS))}" - ), - ] - ) - return lines - - -def reference_vardiff_convergence_metrics_lines(server) -> list[str]: - snapshot = server.vardiff_convergence_snapshot() - lane_accepted = snapshot["lane_accepted_shares"] - assert isinstance(lane_accepted, dict) - resume_outcomes = snapshot["resume_outcomes"] - assert isinstance(resume_outcomes, dict) - lanes = [ - str(profile.name) - for profile in getattr(server, "listener_profiles", ()) or () - ] - lanes.extend(sorted(lane for lane in lane_accepted if lane not in lanes)) - elapsed = max(0.001, time.monotonic() - server.started_monotonic) - return [ - "# HELP qbit_prism_vardiff_sessions_at_max_difficulty Sessions whose current difficulty sits at their effective vardiff ceiling.", - "# TYPE qbit_prism_vardiff_sessions_at_max_difficulty gauge", - f"qbit_prism_vardiff_sessions_at_max_difficulty {int(snapshot['sessions_at_max_difficulty'])}", - "# HELP qbit_prism_vardiff_lane_accepted_shares_total Accepted shares by Stratum lane.", - "# TYPE qbit_prism_vardiff_lane_accepted_shares_total counter", - *[ - f'qbit_prism_vardiff_lane_accepted_shares_total{{lane="{server.prometheus_label_value(lane)}"}} {int(lane_accepted.get(lane, 0))}' - for lane in lanes - ], - "# HELP qbit_prism_vardiff_lane_accepted_shares_per_second Accepted shares per second by Stratum lane, averaged since coordinator start; a long-run average cannot show a transient reconnect storm -- use rate(qbit_prism_vardiff_lane_accepted_shares_total[5m]) for that. The numerator counts shares this process newly committed on the live submission path, not the ledger lifetime total published by qbit_prism_accepted_shares_total: an exact replay of an already-durable share and a startup recovery-journal replay are both acked accepted without being lane-counted, so the lanes sum to this coordinator's own commit rate and need not reconcile against an accepted-ack count.", - "# TYPE qbit_prism_vardiff_lane_accepted_shares_per_second gauge", - *[ - f'qbit_prism_vardiff_lane_accepted_shares_per_second{{lane="{server.prometheus_label_value(lane)}"}} {int(lane_accepted.get(lane, 0)) / elapsed:.12g}' - for lane in lanes - ], - "# HELP qbit_prism_vardiff_resume_total Reconnect difficulty resume attempts by outcome; clamped means the retained value was pulled into the lane's plausibility bounds, overridden means an adopted value was superseded by an explicit difficulty request in the same authorize, so resumed + clamped - overridden is the number that stuck.", - "# TYPE qbit_prism_vardiff_resume_total counter", - *[ - f'qbit_prism_vardiff_resume_total{{outcome="{outcome}"}} {int(resume_outcomes.get(outcome, 0))}' - for outcome in PRISM_VARDIFF_RESUME_OUTCOMES - ], - "# HELP qbit_prism_vardiff_resume_retained_sessions Worker sessions holding a retained difficulty eligible for resume.", - "# TYPE qbit_prism_vardiff_resume_retained_sessions gauge", - f"qbit_prism_vardiff_resume_retained_sessions {int(snapshot['retained_sessions'])}", - ] - - -def reference_accepted_preview_attribution_metrics_lines(server) -> list[str]: - snapshot = ensure_accepted_preview_telemetry(server).snapshot() - landing = snapshot["landing_phases"] - passes = snapshot["reconcile_passes"] - steps = snapshot["reconcile_steps"] - rescans = snapshot["full_rescans"] - - def summary(name: str, labels: str, stats: dict[str, object]) -> list[str]: - return [ - f'{name}_sum{{{labels}}} {float(stats["sum"]):.6f}', - f'{name}_count{{{labels}}} {int(stats["count"])}', - f'{name}_max{{{labels}}} {float(stats["max"]):.6f}', - ] - - lines = [ - "# HELP qbit_prism_accepted_block_landing_phase_seconds Accepted-block landing wall time between definitive node acceptance and payout preview publication, by bounded sub-phase; prior_balances_check is a prior-balances reread, never a payout-window rescan.", - "# TYPE qbit_prism_accepted_block_landing_phase_seconds summary", - ] - for phase in PRISM_ACCEPTED_LANDING_PHASES: - lines.extend( - summary( - "qbit_prism_accepted_block_landing_phase_seconds", - f'phase="{phase}"', - landing[phase], - ) - ) - lines.extend( - [ - "# HELP qbit_prism_reorg_reconcile_pass_seconds Reorg reconciliation passes by bounded caller; _count is the pass count, and a pass minus the sum of its steps is the unattributed remainder.", - "# TYPE qbit_prism_reorg_reconcile_pass_seconds summary", - ] - ) - for caller in PRISM_REORG_RECONCILE_CALLERS: - lines.extend( - summary( - "qbit_prism_reorg_reconcile_pass_seconds", - f'caller="{caller}"', - passes[caller], - ) - ) - lines.extend( - [ - "# HELP qbit_prism_reorg_reconcile_step_seconds Reorg reconciliation wall time by bounded caller and step; admission_wait is writer admission plus the payout-balance mutation lock, and candidate_prepare is where a reconcile_invalidation full rescan executes.", - "# TYPE qbit_prism_reorg_reconcile_step_seconds summary", - ] - ) - for caller in PRISM_REORG_RECONCILE_CALLERS: - for step in PRISM_REORG_RECONCILE_STEPS: - lines.extend( - summary( - "qbit_prism_reorg_reconcile_step_seconds", - f'caller="{caller}",step="{step}"', - steps[(caller, step)], - ) - ) - lines.extend( - [ - "# HELP qbit_prism_payout_window_full_rescan_seconds Full payout-window oracle rescans by bounded reason and window pipeline path; a prior-balances reread is a ledger read operation, not an observation here.", - "# TYPE qbit_prism_payout_window_full_rescan_seconds summary", - ] - ) - for reason in PRISM_PAYOUT_WINDOW_FULL_RESCAN_REASONS: - for path in PRISM_PAYOUT_WINDOW_FULL_RESCAN_PATHS: - lines.extend( - summary( - "qbit_prism_payout_window_full_rescan_seconds", - f'reason="{reason}",path="{path}"', - rescans[(reason, path)], - ) - ) - return lines - - -def reference_render_metrics_payload(server) -> str: - ledger_metrics = server.ledger.metrics() - audit_metrics = server.audit_artifact_metrics() - mining_metrics = server.mining_delivery_snapshot() - process_rss_bytes, process_open_fds = server.process_resource_metrics() - accepted_share_count = server.accepted_share_stats()[0] - server._ensure_share_hot_path_state() - with server._share_accounting_lock: - submitted_share_count = int(getattr(server, "submitted_share_count", 0)) - stale_share_count = int(getattr(server, "stale_share_count", 0)) - duplicate_share_count = int(getattr(server, "duplicate_share_count", 0)) - low_difficulty_share_count = int( - getattr(server, "low_difficulty_share_count", 0) - ) - collection_block_submission_count = int( - getattr(server, "collection_block_submission_count", 0) - ) - rejection_counts = dict( - getattr(server, "rejection_counts_by_reason", {}) - ) - grace_credited_share_count = int( - getattr(server, "grace_credited_share_count", 0) - ) - block_solves_dropped_counts = dict( - getattr( - server, - "block_solves_dropped_counts", - {"stale_grace": 0}, - ) - ) - stale_percent = 0.0 - if submitted_share_count > 0: - stale_percent = (stale_share_count / submitted_share_count) * 100.0 - idle_retarget_count = int(getattr(server, "idle_retarget_count", 0)) - with server.lock: - server._ensure_connection_capacity_state() - active_connection_count = len(server.clients) - connection_limit_rejection_counts = dict( - server.connection_limit_rejection_counts - ) - accept_resource_exhaustion_count = int( - getattr(server, "accept_resource_exhaustion_count", 0) - ) - connection_setup_failure_count = int( - getattr(server, "connection_setup_failure_count", 0) - ) - server._ensure_evicted_job_state() - server.prune_evicted_job_graveyard(force=False) - same_tip_context_count = len(server.evicted_same_tip_job_ids) - evicted_job_context_counts = { - "same_tip": same_tip_context_count, - "stale_grace": len(server.evicted_job_graveyard) - same_tip_context_count, - } - evicted_job_submit_counts = dict(server.evicted_job_submit_counts) - evicted_job_expiration_counts = dict(server.evicted_job_expiration_counts) - evicted_job_capacity_eviction_counts = dict( - server.evicted_job_capacity_eviction_counts - ) - stale_job_abandon_counts = dict( - getattr( - server, - "stale_job_abandon_counts", - { - abandon_class: 0 - for abandon_class in PRISM_STALE_JOB_ABANDON_CLASSES - }, - ) - ) - server._ensure_worker_metrics_state() - with server.worker_metrics_lock: - worker_share_counts = { - label: dict(counts) - for label, counts in server.worker_share_counts.items() - } - worker_rejection_counts = dict(server.worker_rejection_counts) - coinbase_weight_headroom = 2_000_000 - latest_coinbase_size_bytes = getattr(server, "latest_coinbase_size_bytes", None) - if latest_coinbase_size_bytes is not None: - coinbase_weight_headroom = 2_000_000 - int(latest_coinbase_size_bytes) - ctv_pending = 0 - ctv_broadcastable = 0 - ctv_failed = 0 - pending_ctv_fanouts = getattr(server.ledger, "pending_ctv_fanout_statuses", None) - if callable(pending_ctv_fanouts): - try: - for fanout in pending_ctv_fanouts(limit=1_000): - ctv_pending += 1 - status = str(fanout.get("settlement_status", "")) - if status == "broadcastable": - ctv_broadcastable += 1 - elif status == "failed": - ctv_failed += 1 - except Exception: - ctv_pending = -1 - ctv_broadcastable = -1 - ctv_failed = -1 - if ctv_failed >= 0: - ctv_failed = int(ledger_metrics.get("ctv_fanouts_failed", ctv_failed)) - ibd = 0 - peers = 0 - try: - blockchain_info = server.rpc.call("getblockchaininfo") - if isinstance(blockchain_info, dict) and blockchain_info.get("initialblockdownload"): - ibd = 1 - except Exception: - ibd = -1 - try: - network_info = server.rpc.call("getnetworkinfo") - if isinstance(network_info, dict): - peers = int(network_info.get("connections", 0)) - except Exception: - peers = -1 - lines = [ - "# HELP qbit_prism_accepted_shares_total Accepted shares recorded by the canonical PRISM ledger.", - "# TYPE qbit_prism_accepted_shares_total counter", - f"qbit_prism_accepted_shares_total {accepted_share_count}", - *reference_accepted_stats_reconcile_metric_lines(server), - *reference_landing_observability_metrics_lines(server), - "# HELP qbit_prism_submitted_shares_total Stratum share submissions seen by the PRISM coordinator.", - "# TYPE qbit_prism_submitted_shares_total counter", - f"qbit_prism_submitted_shares_total {submitted_share_count}", - "# HELP qbit_prism_stratum_active_connections Active admitted Stratum connections across all listeners.", - "# TYPE qbit_prism_stratum_active_connections gauge", - f"qbit_prism_stratum_active_connections {active_connection_count}", - "# HELP qbit_prism_stratum_connection_limit Configured global Stratum connection ceiling; zero means unlimited.", - "# TYPE qbit_prism_stratum_connection_limit gauge", - f"qbit_prism_stratum_connection_limit {mining_metrics['connection_capacity']}", - "# HELP qbit_prism_stratum_peak_active_connections Peak admitted Stratum connections since process start.", - "# TYPE qbit_prism_stratum_peak_active_connections gauge", - f"qbit_prism_stratum_peak_active_connections {mining_metrics['peak_active_connections']}", - "# HELP qbit_prism_stratum_subscribed_connections Active subscribed Stratum connections.", - "# TYPE qbit_prism_stratum_subscribed_connections gauge", - f"qbit_prism_stratum_subscribed_connections {mining_metrics['subscribed_connections']}", - "# HELP qbit_prism_stratum_authorized_connections Active subscribed and authorized Stratum connections.", - "# TYPE qbit_prism_stratum_authorized_connections gauge", - f"qbit_prism_stratum_authorized_connections {mining_metrics['authorized_connections']}", - "# HELP qbit_prism_stratum_pending_initial_jobs Authorized clients awaiting their first usable current-tip job.", - "# TYPE qbit_prism_stratum_pending_initial_jobs gauge", - f"qbit_prism_stratum_pending_initial_jobs {mining_metrics['pending_initial_jobs']}", - "# HELP qbit_prism_stratum_pending_initial_job_limit Configured bound for clients awaiting their first usable job.", - "# TYPE qbit_prism_stratum_pending_initial_job_limit gauge", - f"qbit_prism_stratum_pending_initial_job_limit {mining_metrics['pending_initial_job_capacity']}", - "# HELP qbit_prism_stratum_oldest_pending_initial_job_seconds Age of the oldest pending first-job request.", - "# TYPE qbit_prism_stratum_oldest_pending_initial_job_seconds gauge", - f"qbit_prism_stratum_oldest_pending_initial_job_seconds {mining_metrics['oldest_pending_initial_job_age_seconds']}", - "# HELP qbit_prism_stratum_oldest_genuinely_pending_initial_job_seconds Age of the oldest authorized client that has never received usable work.", - "# TYPE qbit_prism_stratum_oldest_genuinely_pending_initial_job_seconds gauge", - f"qbit_prism_stratum_oldest_genuinely_pending_initial_job_seconds {mining_metrics['oldest_genuinely_pending_initial_job_age_seconds']}", - "# HELP qbit_prism_stratum_current_tip_coverage_gap_seconds Continuous age of current-tip job coverage below 95 percent.", - "# TYPE qbit_prism_stratum_current_tip_coverage_gap_seconds gauge", - f"qbit_prism_stratum_current_tip_coverage_gap_seconds {mining_metrics['current_tip_coverage_gap_age_seconds']}", - "# HELP qbit_prism_stratum_initial_job_queue_rejections_total Sessions closed because bounded first-job delivery was full.", - "# TYPE qbit_prism_stratum_initial_job_queue_rejections_total counter", - f"qbit_prism_stratum_initial_job_queue_rejections_total {mining_metrics['initial_job_queue_rejections']}", - "# HELP qbit_prism_stratum_initial_job_timeouts_total Sessions disconnected after first-job delivery timed out.", - "# TYPE qbit_prism_stratum_initial_job_timeouts_total counter", - f"qbit_prism_stratum_initial_job_timeouts_total {mining_metrics['initial_job_timeout_disconnects']}", - "# HELP qbit_prism_stratum_initial_job_tasks_total First-job tasks canceled or coalesced before duplicate work.", - "# TYPE qbit_prism_stratum_initial_job_tasks_total counter", - f'qbit_prism_stratum_initial_job_tasks_total{{result="cancelled"}} {mining_metrics["initial_job_cancelled_tasks"]}', - f'qbit_prism_stratum_initial_job_tasks_total{{result="coalesced"}} {mining_metrics["initial_job_coalesced_tasks"]}', - "# HELP qbit_prism_stratum_initial_job_queue_capacity_reclaimed_total Queued first-job admission slots reclaimed immediately by cancellation.", - "# TYPE qbit_prism_stratum_initial_job_queue_capacity_reclaimed_total counter", - f'qbit_prism_stratum_initial_job_queue_capacity_reclaimed_total {mining_metrics["initial_job_queue_capacity_reclaimed"]}', - "# HELP qbit_prism_stratum_clients_with_current_tip_jobs Authorized clients holding usable current-tip work.", - "# TYPE qbit_prism_stratum_clients_with_current_tip_jobs gauge", - f"qbit_prism_stratum_clients_with_current_tip_jobs {mining_metrics['clients_with_current_tip_jobs']}", - "# HELP qbit_prism_stratum_current_tip_job_coverage Ratio of authorized clients holding current-tip work.", - "# TYPE qbit_prism_stratum_current_tip_job_coverage gauge", - f"qbit_prism_stratum_current_tip_job_coverage {mining_metrics['current_tip_job_coverage']}", - "# HELP qbit_prism_stratum_semantic_current_work_ratio Ratio of authorized clients whose work matches the current template fingerprint, payout generation, and client session.", - "# TYPE qbit_prism_stratum_semantic_current_work_ratio gauge", - f"qbit_prism_stratum_semantic_current_work_ratio {mining_metrics['semantic_current_work_ratio']}", - "# HELP qbit_prism_stratum_handler_threads Active per-connection Stratum handler threads.", - "# TYPE qbit_prism_stratum_handler_threads gauge", - f"qbit_prism_stratum_handler_threads {mining_metrics['handler_threads']}", - "# HELP qbit_prism_job_delivery_queue_depth Current bounded delivery executor queue depth.", - "# TYPE qbit_prism_job_delivery_queue_depth gauge", - f"qbit_prism_job_delivery_queue_depth {mining_metrics['delivery_executor_queue_depth']}", - "# HELP qbit_prism_job_delivery_active_workers Delivery executor workers currently running tasks.", - "# TYPE qbit_prism_job_delivery_active_workers gauge", - f"qbit_prism_job_delivery_active_workers {mining_metrics['delivery_executor_active_workers']}", - "# HELP qbit_prism_process_resident_memory_bytes Current process RSS bytes, or -1 when unavailable.", - "# TYPE qbit_prism_process_resident_memory_bytes gauge", - f"qbit_prism_process_resident_memory_bytes {process_rss_bytes}", - "# HELP qbit_prism_process_open_file_descriptors Current process open descriptor count, or -1 when unavailable.", - "# TYPE qbit_prism_process_open_file_descriptors gauge", - f"qbit_prism_process_open_file_descriptors {process_open_fds}", - "# HELP qbit_prism_stratum_connection_limit_rejections_total Stratum connections rejected by an explicitly configured admission limit.", - "# TYPE qbit_prism_stratum_connection_limit_rejections_total counter", - *[ - f'qbit_prism_stratum_connection_limit_rejections_total{{scope="{scope}"}} {int(connection_limit_rejection_counts.get(scope, 0))}' - for scope in ("global", "username") - ], - "# HELP qbit_prism_stratum_accept_resource_exhaustions_total Recoverable Stratum accept or client-setup failures caused by process or system descriptor exhaustion.", - "# TYPE qbit_prism_stratum_accept_resource_exhaustions_total counter", - f"qbit_prism_stratum_accept_resource_exhaustions_total {accept_resource_exhaustion_count}", - "# HELP qbit_prism_stratum_connection_setup_failures_total Admitted Stratum connections cleaned up after socket or handler-thread setup failure.", - "# TYPE qbit_prism_stratum_connection_setup_failures_total counter", - f"qbit_prism_stratum_connection_setup_failures_total {connection_setup_failure_count}", - "# HELP qbit_prism_stale_shares_total Stratum shares rejected or ignored as stale.", - "# TYPE qbit_prism_stale_shares_total counter", - f"qbit_prism_stale_shares_total {stale_share_count}", - "# HELP qbit_prism_duplicate_shares_total Duplicate Stratum shares rejected.", - "# TYPE qbit_prism_duplicate_shares_total counter", - f"qbit_prism_duplicate_shares_total {duplicate_share_count}", - "# HELP qbit_prism_low_difficulty_shares_total Low-difficulty Stratum shares rejected.", - "# TYPE qbit_prism_low_difficulty_shares_total counter", - f"qbit_prism_low_difficulty_shares_total {low_difficulty_share_count}", - "# HELP qbit_prism_collection_block_submissions_total Solver-pays-all block candidates submitted from collection-mode jobs.", - "# TYPE qbit_prism_collection_block_submissions_total counter", - f"qbit_prism_collection_block_submissions_total {collection_block_submission_count}", - "# HELP qbit_prism_grace_credited_shares_total Accepted shares credited by the stale-grace policy.", - "# TYPE qbit_prism_grace_credited_shares_total counter", - f"qbit_prism_grace_credited_shares_total {grace_credited_share_count}", - "# HELP qbit_prism_block_solves_dropped_total Block-passing submissions intentionally excluded from block submission by a bounded policy reason.", - "# TYPE qbit_prism_block_solves_dropped_total counter", - *[ - f'qbit_prism_block_solves_dropped_total{{reason="{reason}"}} {int(block_solves_dropped_counts.get(reason, 0))}' - for reason in ("stale_grace",) - ], - "# HELP qbit_prism_rejections_total PRISM share or block rejections by canonical reason ID.", - "# TYPE qbit_prism_rejections_total counter", - *[ - f'qbit_prism_rejections_total{{reason_id="{reason}"}} {int(rejection_counts.get(reason, 0))}' - for reason in PRISM_REJECTION_REASON_IDS - ], - "# HELP qbit_prism_worker_submitted_shares_total Stratum share submissions by bounded worker label.", - "# TYPE qbit_prism_worker_submitted_shares_total counter", - *[ - f'qbit_prism_worker_submitted_shares_total{{worker="{server.prometheus_label_value(label)}"}} {int(counts.get("submitted", 0))}' - for label, counts in sorted(worker_share_counts.items()) - ], - "# HELP qbit_prism_worker_accepted_shares_total Accepted shares by bounded worker label.", - "# TYPE qbit_prism_worker_accepted_shares_total counter", - *[ - f'qbit_prism_worker_accepted_shares_total{{worker="{server.prometheus_label_value(label)}"}} {int(counts.get("accepted", 0))}' - for label, counts in sorted(worker_share_counts.items()) - ], - "# HELP qbit_prism_worker_grace_credited_shares_total Stale-grace credited shares by bounded worker label.", - "# TYPE qbit_prism_worker_grace_credited_shares_total counter", - *[ - f'qbit_prism_worker_grace_credited_shares_total{{worker="{server.prometheus_label_value(label)}"}} {int(counts.get("grace", 0))}' - for label, counts in sorted(worker_share_counts.items()) - ], - "# HELP qbit_prism_worker_rejections_total PRISM share or block rejections by bounded worker label and reason ID.", - "# TYPE qbit_prism_worker_rejections_total counter", - *[ - f'qbit_prism_worker_rejections_total{{worker="{server.prometheus_label_value(label)}",reason_id="{reason}"}} {int(count)}' - for (label, reason), count in sorted(worker_rejection_counts.items()) - ], - "# HELP qbit_prism_job_build_failures_total Job builds skipped after a template/coinbase error without dropping the client.", - "# TYPE qbit_prism_job_build_failures_total counter", - f"qbit_prism_job_build_failures_total {server.job_build_failure_count}", - "# HELP qbit_prism_block_candidates_dropped_total Legacy counter; durable candidate outbox rows are never dropped on queue overflow.", - "# TYPE qbit_prism_block_candidates_dropped_total counter", - f"qbit_prism_block_candidates_dropped_total {int(getattr(server, 'block_candidates_dropped', 0))}", - "# HELP qbit_prism_block_candidate_wakeups_coalesced_total Candidate queue wakeups coalesced while the durable outbox retained the work.", - "# TYPE qbit_prism_block_candidate_wakeups_coalesced_total counter", - f"qbit_prism_block_candidate_wakeups_coalesced_total {int(getattr(server, 'block_candidate_wakeups_coalesced', 0))}", - "# HELP qbit_prism_block_candidate_retries_total Transient candidate outcomes retained for durable retry.", - "# TYPE qbit_prism_block_candidate_retries_total counter", - f"qbit_prism_block_candidate_retries_total {int(getattr(server, 'block_candidate_retry_count', 0))}", - "# HELP qbit_prism_block_candidate_accept_pending_defers_total Terminal abandonments refused because the candidate is (or was recently observed as) an active chain block; the candidate retries until its accepted success tail finalizes it as submitted.", - "# TYPE qbit_prism_block_candidate_accept_pending_defers_total counter", - f"qbit_prism_block_candidate_accept_pending_defers_total {int(getattr(server, 'block_candidate_accept_pending_defer_count', 0))}", - "# HELP qbit_prism_block_candidate_poisoned_total Invalid durable candidate intents quarantined from replay.", - "# TYPE qbit_prism_block_candidate_poisoned_total counter", - f"qbit_prism_block_candidate_poisoned_total {int(getattr(server, 'block_candidate_poisoned_count', 0))}", - "# HELP qbit_prism_block_candidates_abandoned_total Block candidates that did not land (lost tip race or failed submit), by reason. Not share rejections: the underlying share was accepted.", - "# TYPE qbit_prism_block_candidates_abandoned_total counter", - *[ - f'qbit_prism_block_candidates_abandoned_total{{reason_id="{reason}"}} {int(count)}' - for reason, count in sorted(getattr(server, "block_candidate_abandoned_counts", {}).items()) - ], - "# HELP qbit_prism_stale_job_abandons_total Terminal stale-job block candidate abandonments by bounded cause.", - "# TYPE qbit_prism_stale_job_abandons_total counter", - *[ - f'qbit_prism_stale_job_abandons_total{{class="{abandon_class}"}} {int(stale_job_abandon_counts.get(abandon_class, 0))}' - for abandon_class in PRISM_STALE_JOB_ABANDON_CLASSES - ], - "# HELP qbit_prism_share_append_queue_depth Accepted shares waiting on the ledger writer thread.", - "# TYPE qbit_prism_share_append_queue_depth gauge", - f"qbit_prism_share_append_queue_depth {server.share_append_queue.qsize() if getattr(server, 'share_append_queue', None) is not None else 0}", - "# HELP qbit_prism_share_append_failures_total Shares in group commits that failed before acknowledgement.", - "# TYPE qbit_prism_share_append_failures_total counter", - f"qbit_prism_share_append_failures_total {int(getattr(server, 'share_append_failure_count', 0))}", - "# HELP qbit_prism_shares_recovered_to_disk_total Legacy pre-commit-ACK shares written to the upgrade recovery file.", - "# TYPE qbit_prism_shares_recovered_to_disk_total counter", - f"qbit_prism_shares_recovered_to_disk_total {int(getattr(server, 'shares_recovered_to_disk', 0))}", - "# HELP qbit_prism_shares_replayed_total Recovery-file shares replayed into the ledger at startup.", - "# TYPE qbit_prism_shares_replayed_total counter", - f"qbit_prism_shares_replayed_total {int(getattr(server, 'shares_replayed', 0))}", - "# HELP qbit_prism_share_replay_conflicts_total Recovery-file rows quarantined because the durable row disagrees with the journal payload.", - "# TYPE qbit_prism_share_replay_conflicts_total counter", - f"qbit_prism_share_replay_conflicts_total {int(getattr(server, 'share_replay_conflicts', 0))}", - "# HELP qbit_prism_tip_refresh_jobs_total Client jobs refreshed after qbit tip/template changes.", - "# TYPE qbit_prism_tip_refresh_jobs_total counter", - f"qbit_prism_tip_refresh_jobs_total {server.tip_refresh_job_count}", - "# HELP qbit_prism_active_job_contexts Current retained PRISM job contexts.", - "# TYPE qbit_prism_active_job_contexts gauge", - f"qbit_prism_active_job_contexts {len(getattr(server, 'jobs', {}))}", - "# HELP qbit_prism_evicted_job_contexts Evicted job contexts retained by safety class.", - "# TYPE qbit_prism_evicted_job_contexts gauge", - *[ - f'qbit_prism_evicted_job_contexts{{class="{job_class}"}} {evicted_job_context_counts[job_class]}' - for job_class in PRISM_EVICTED_JOB_CLASSES - ], - "# HELP qbit_prism_evicted_job_submits_total Accepted submits validated against an evicted job context.", - "# TYPE qbit_prism_evicted_job_submits_total counter", - *[ - f'qbit_prism_evicted_job_submits_total{{outcome="{outcome}"}} {int(evicted_job_submit_counts.get(outcome, 0))}' - for outcome in PRISM_EVICTED_JOB_SUBMIT_OUTCOMES - ], - "# HELP qbit_prism_evicted_job_expirations_total Retained job contexts removed after their class TTL.", - "# TYPE qbit_prism_evicted_job_expirations_total counter", - *[ - f'qbit_prism_evicted_job_expirations_total{{class="{job_class}"}} {int(evicted_job_expiration_counts.get(job_class, 0))}' - for job_class in PRISM_EVICTED_JOB_CLASSES - ], - "# HELP qbit_prism_evicted_job_capacity_evictions_total Same-tip retained contexts removed by a configured count limit.", - "# TYPE qbit_prism_evicted_job_capacity_evictions_total counter", - *[ - f'qbit_prism_evicted_job_capacity_evictions_total{{scope="{scope}"}} {int(evicted_job_capacity_eviction_counts.get(scope, 0))}' - for scope in PRISM_EVICTED_JOB_CAPACITY_SCOPES - ], - "# HELP qbit_prism_post_accept_refresh_failures_total Immediate clean-job refreshes that failed after direct block acceptance.", - "# TYPE qbit_prism_post_accept_refresh_failures_total counter", - f"qbit_prism_post_accept_refresh_failures_total {server.post_accept_refresh_failure_count}", - "# HELP qbit_prism_reorg_inactive_blocks_total PRISM pool blocks quarantined after leaving the active chain.", - "# TYPE qbit_prism_reorg_inactive_blocks_total counter", - f"qbit_prism_reorg_inactive_blocks_total {server.reorg_inactive_block_count}", - "# HELP qbit_prism_reorg_reactivated_blocks_total Quarantined PRISM pool blocks restored after returning to the active chain.", - "# TYPE qbit_prism_reorg_reactivated_blocks_total counter", - f"qbit_prism_reorg_reactivated_blocks_total {server.reorg_reactivated_block_count}", - "# HELP qbit_prism_reorg_reconcile_skips_total Reorg reconciliation passes skipped because qbitd chain view was not trusted.", - "# TYPE qbit_prism_reorg_reconcile_skips_total counter", - f"qbit_prism_reorg_reconcile_skips_total {server.reorg_reconcile_skip_count}", - "# HELP qbit_prism_reorg_reconcile_errors_total Reorg reconciliation errors that prevented ordered job issuance.", - "# TYPE qbit_prism_reorg_reconcile_errors_total counter", - f"qbit_prism_reorg_reconcile_errors_total {server.reorg_reconcile_error_count}", - "# HELP qbit_prism_reorg_reconcile_lookups_total Reconcile demand by caller path and the source that satisfied it.", - "# TYPE qbit_prism_reorg_reconcile_lookups_total counter", - *[ - f'qbit_prism_reorg_reconcile_lookups_total{{path="{path}",source="{source}"}} ' - f"{int(getattr(server, 'reorg_reconcile_lookup_counts', {}).get((path, source), 0))}" - for path in PRISM_REORG_RECONCILE_LOOKUP_PATHS - for source in PRISM_REORG_RECONCILE_LOOKUP_SOURCES - ], - "# HELP qbit_prism_matured_payouts_total Payout entries marked mature by the coordinator tip reconciliation path.", - "# TYPE qbit_prism_matured_payouts_total counter", - f"qbit_prism_matured_payouts_total {server.matured_payout_count}", - "# HELP qbit_prism_vardiff_idle_retargets_total Vardiff retargets triggered by the idle zero-accepted-share sweep.", - "# TYPE qbit_prism_vardiff_idle_retargets_total counter", - f"qbit_prism_vardiff_idle_retargets_total {idle_retarget_count}", - "# HELP qbit_prism_stale_share_percent Percent of submitted shares classified stale.", - "# TYPE qbit_prism_stale_share_percent gauge", - f"qbit_prism_stale_share_percent {stale_percent:.12g}", - "# HELP qbit_prism_blocks_accepted_total Blocks accepted through the PRISM coordinator.", - "# TYPE qbit_prism_blocks_accepted_total counter", - f"qbit_prism_blocks_accepted_total {server.accepted_block_count}", - "# HELP qbit_prism_persisted_blocks Persisted PRISM pool block rows.", - "# TYPE qbit_prism_persisted_blocks gauge", - f"qbit_prism_persisted_blocks {ledger_metrics['blocks']}", - "# HELP qbit_prism_inactive_pool_blocks PRISM pool block rows currently quarantined as inactive.", - "# TYPE qbit_prism_inactive_pool_blocks gauge", - f"qbit_prism_inactive_pool_blocks {ledger_metrics.get('inactive_blocks', 0)}", - "# HELP qbit_prism_reversed_pool_blocks PRISM pool block rows terminally reversed.", - "# TYPE qbit_prism_reversed_pool_blocks gauge", - f"qbit_prism_reversed_pool_blocks {ledger_metrics.get('reversed_blocks', 0)}", - "# HELP qbit_prism_rejected_pool_blocks PRISM pool block rows rejected before confirmation.", - "# TYPE qbit_prism_rejected_pool_blocks gauge", - f"qbit_prism_rejected_pool_blocks {ledger_metrics.get('rejected_blocks', 0)}", - "# HELP qbit_prism_owed_accounts Current accounts with positive carried owed balances.", - "# TYPE qbit_prism_owed_accounts gauge", - f"qbit_prism_owed_accounts {ledger_metrics['owed_accounts']}", - "# HELP qbit_prism_coinbase_weight_headroom_bytes Remaining qbit block weight bytes after the latest pool coinbase.", - "# TYPE qbit_prism_coinbase_weight_headroom_bytes gauge", - f"qbit_prism_coinbase_weight_headroom_bytes {coinbase_weight_headroom}", - "# HELP qbit_prism_ctv_fanouts_pending Pending non-terminal CTV fanouts known to the ledger, or -1 if unavailable.", - "# TYPE qbit_prism_ctv_fanouts_pending gauge", - f"qbit_prism_ctv_fanouts_pending {ctv_pending}", - "# HELP qbit_prism_ctv_fanouts_broadcastable CTV fanouts that are mature enough to broadcast, or -1 if unavailable.", - "# TYPE qbit_prism_ctv_fanouts_broadcastable gauge", - f"qbit_prism_ctv_fanouts_broadcastable {ctv_broadcastable}", - "# HELP qbit_prism_ctv_fanouts_failed CTV fanouts with failed or rejected broadcast state, or -1 if unavailable.", - "# TYPE qbit_prism_ctv_fanouts_failed gauge", - f"qbit_prism_ctv_fanouts_failed {ctv_failed}", - "# HELP qbit_prism_vardiff_enabled Whether PRISM Stratum vardiff is enabled.", - "# TYPE qbit_prism_vardiff_enabled gauge", - f"qbit_prism_vardiff_enabled {1 if server.vardiff_config.enabled else 0}", - "# HELP qbit_prism_qbitd_initial_block_download qbitd initialblockdownload status, or -1 if unavailable.", - "# TYPE qbit_prism_qbitd_initial_block_download gauge", - f"qbit_prism_qbitd_initial_block_download {ibd}", - "# HELP qbit_prism_qbitd_peers qbitd peer count, or -1 if unavailable.", - "# TYPE qbit_prism_qbitd_peers gauge", - f"qbit_prism_qbitd_peers {peers}", - "# HELP qbit_prism_audit_artifact_bytes Bytes used by PRISM audit artifacts in PRISM_AUDIT_DIR by artifact kind.", - "# TYPE qbit_prism_audit_artifact_bytes gauge", - *[ - f'qbit_prism_audit_artifact_bytes{{kind="{kind}"}} {audit_metrics[kind]["bytes"]}' - for kind in ("body", "share_segment", "live_bundle", "candidate", "other") - ], - "# HELP qbit_prism_audit_artifact_files PRISM audit artifact file count in PRISM_AUDIT_DIR by artifact kind.", - "# TYPE qbit_prism_audit_artifact_files gauge", - *[ - f'qbit_prism_audit_artifact_files{{kind="{kind}"}} {audit_metrics[kind]["files"]}' - for kind in ("body", "share_segment", "live_bundle", "candidate", "other") - ], - "# HELP qbit_prism_audit_artifact_scan_error Whether the latest PRISM_AUDIT_DIR metric scan failed.", - "# TYPE qbit_prism_audit_artifact_scan_error gauge", - f"qbit_prism_audit_artifact_scan_error {audit_metrics['scan_error']}", - ] - lines.extend(reference_lease_heartbeat_metrics_lines(server)) - lines.extend(reference_shutdown_metrics_lines(server)) - lines.extend(reference_coordinator_lock_metrics_lines(server)) - lines.extend(reference_block_submitter_metrics_lines(server)) - lines.extend(reference_block_candidate_cleanup_backlog_metrics_lines(server)) - lines.extend(reference_share_ack_metrics_lines(server)) - lines.extend(server.ctv_fanout_broadcaster_metrics_lines()) - lines.extend(server.vardiff_idle_metrics_lines()) - lines.extend(reference_vardiff_convergence_metrics_lines(server)) - lines.extend(server.block_finalization_metrics_lines()) - lines.extend(reference_job_build_metrics_lines(server)) - lines.extend(server.tip_refresh_metrics_lines()) - lines.extend(server.payout_state_metrics_lines()) - lines.extend(reference_initial_delivery_metrics_lines(server)) - lines.extend(server.progress_health_metrics_lines()) - lines.extend(reference_accepted_preview_attribution_metrics_lines(server)) - lines.extend(reference_gc_pause_metrics_lines(server)) - lines.extend(reference_process_heap_metrics_lines(PINNED_HEAP_SAMPLE)) - lines.extend(reference_component_cardinality_metrics_lines(server)) - return "\n".join(lines) + "\n" - - -class MetricsRenderParityTests(unittest.TestCase): - """Byte-for-byte parity between the renderer and the frozen reference.""" - - def _seeded_coordinator(self): - server = support.coordinator() - temporary = tempfile.TemporaryDirectory() - server.audit_dir = Path(temporary.name) / "audit" - server.evidence_path = Path(temporary.name) / "state" / "evidence.json" - - def cleanup() -> None: - store = server.__dict__.get("_audit_artifact_store") - if store is not None: - store.close() - temporary.cleanup() - - self.addCleanup(cleanup) - # Pin the /proc-derived gauges: RSS moves between the two renders. - server.process_resource_metrics = lambda: (123_456_789, 42) - # Pin issue #226's heap collector the same way: allocation and GC - # counters move between the reference and the actual render. The - # pinned renderer is the one metrics_payload() resolves. - server.__dict__["_metrics_renderer"] = MetricsRenderer( - server, - process_telemetry=SimpleNamespace(sample=lambda: PINNED_HEAP_SAMPLE), - ) - # Issue #226's component gauges, seeded through the shipped owners - # so the parity document carries non-zero values: a bundle-cache - # entry, a cached two-page payout window, a serialization slot with - # a spool, a reconcile flight and memo entry, a pending-share floor - # holder, a terminal outcome, and one daemon-uploaded window. - bundles = server._ensure_job_bundle_service() - bundles._job_bundle_cache[("parity-bundle",)] = object() - bundles._share_window_serialization = _ShareWindowSerialization( - key=("parity-digest", 3, 0), - share_count=3, - share_snapshot_sha256="parity-digest", - ) - bundles._share_window_serialization._spool_size = 4_096 - parity_pages = ( - _IncrementalShareWindowPage( - records=(object(), object()), - total_difficulty=0, - prism_json_records=({}, {}), - canonical_json_items=b'{"a":1},{"b":2}', - ), - _IncrementalShareWindowPage( - records=(object(),), - total_difficulty=0, - prism_json_records=({},), - canonical_json_items=b'{"c":3}', - ), - ) - payout = server._ensure_payout_state_service() - payout._incremental_payout_artifact_window = _IncrementalPayoutArtifactWindow( - window=IncrementalShareWindow( - anchor_job_issued_at_ms=1, - window_weight=1, - page_size=512, - pages=parity_pages, - total_difficulty=0, - ), - shares_json=IncrementalShareJsonSequence(pages=parity_pages, record_count=3), - share_snapshot_sha256="parity-digest", - refreshed_monotonic=0.0, - full_rescan_monotonic=0.0, - full_rescan_attempt_monotonic=0.0, - ) - reconciler = server._ensure_reorg_reconciler_service() - reconciler._reconcile_flights["parity-tip"] = _ReconcileFlight() - reconciler._reorg_reconcile_trusted_memo["parity-tip"] = 1.0 - server._ensure_share_writer_service().adopt_pending_share( - SimpleNamespace(share_id="parity-share") - ) - candidate_service = server._ensure_block_candidate_service() - candidate_service._ensure_block_candidate_disposition_state() - candidate_service._block_candidate_terminal_outcomes["ef" * 32] = True - daemon_client = _ServeBuilderClient(process=SimpleNamespace()) - daemon_client.note_uploaded_window("parity-digest") - server._ensure_bundle_compiler()._serve_builder = daemon_client - # Share accounting and rejection counters. - server.submitted_share_count = 10 - server.stale_share_count = 2 - server.duplicate_share_count = 1 - server.low_difficulty_share_count = 3 - server.collection_block_submission_count = 4 - server.grace_credited_share_count = 6 - server.idle_retarget_count = 7 - server.rejection_counts_by_reason["stale-job"] = 2 - server.rejection_counts_by_reason["duplicate-share"] = 1 - # Worker-labeled counters. - server.worker_share_counts = { - "alice": {"submitted": 5, "accepted": 4, "grace": 1}, - "bob": {"submitted": 2, "accepted": 1}, - } - server.worker_rejection_counts = {("alice", "stale-job"): 2} - # Connection admission counters. - server.connection_limit_rejection_counts = {"global": 2, "username": 3} - server.accept_resource_exhaustion_count = 4 - server.connection_setup_failure_count = 5 - # Share-ACK histograms via the submission owner. - server._observe_share_ack_seconds("accepted", 0.05) - server._observe_share_ack_seconds("rejected", 0.2) - # Block-candidate producer state (routed to the B1 owner). - server.block_candidate_retry_count = 2 - server.block_candidate_accept_pending_defer_count = 1 - server.block_candidate_poisoned_count = 1 - server.block_candidate_wakeups_coalesced = 3 - server.block_candidate_abandoned_counts = {"stale-job": 2} - server.stale_job_abandon_counts = { - abandon_class: index - for index, abandon_class in enumerate(PRISM_STALE_JOB_ABANDON_CLASSES) - } - server.block_solves_dropped_counts = {"stale_grace": 2} - server._observe_block_submit_seconds(0.5) - # Acceptance-to-preview-publication needs both halves of the interval: - # the submitter-thread acceptance stamp and the publication that - # closes it. Seeding through the shipped B1 entry points keeps the - # parity document exercising the real observer, not a hand-built - # histogram. - service = server._ensure_block_candidate_service() - service._note_accepted_block_preview_acceptance("ab" * 32) - server._observe_accepted_block_preview_publication( - "ab" * 32, - result="published", - ) - # Issue #198's cleanup-retry backlog, seeded through the shipped - # deferral and backpressure entry points: one owed hash carrying one - # retained holder and a published fence, and one engagement that - # preserved two rows, so every new family renders a non-zero value. - service._record_block_candidate_terminal_outcome("cd" * 32, accepted=False) - service._defer_collapsed_candidate_cleanup( - "cd" * 32, - ("retry-state",), - shares=(SimpleNamespace(share_id="miner-a:" + "cd" * 32),), - ) - with redirect_stdout(StringIO()): - service._note_block_candidate_cleanup_backpressure( - caller="replay-page", - rows=2, - admitted=0, - depth=1, - maximum=1, - ) - server._record_block_ledger_call( - call_class="landing", - budget_seconds=30.0, - duration_seconds=0.25, - timed_out=False, - ) - server._record_block_ledger_call( - call_class="fast", - budget_seconds=1.0, - duration_seconds=1.2, - timed_out=True, - ) - # Reorg reconciler outcome and lookup counters. - server.reorg_inactive_block_count = 2 - server.reorg_reactivated_block_count = 1 - server.reorg_reconcile_skip_count = 3 - server.reorg_reconcile_error_count = 1 - server.matured_payout_count = 4 - server._record_reorg_reconcile_lookup("job_build", "memo_hit") - server._record_reorg_reconcile_lookup("job_build", "serial") - server._record_reorg_reconcile_lookup("tip_refresh", "overlap") - # Landing observability inputs. - server._accepted_parent_preview_wait_timeouts = 2 - server._startup_phase_origin_monotonic = time.monotonic() - 1.0 - server._record_startup_phase_once("audit_listener_bound") - server._record_startup_phase_once("node_rpc_ready") - # Ledger-side optional snapshots. - server.ledger.accepted_stats_reconcile_status = lambda: { - "failures": 1, - "age_seconds": 2.5, - } - server.ledger.prior_balances_read_stats = lambda: { - "reads_total": 3, - "last_seconds": 0.01, - "max_seconds": 0.02, - } - # Issue #211's attribution: local admission and server execution are - # separate series so a budget exhaustion names the half it went to. - # Issue #224 Wave 0: a contract operation renders under its own - # name, an operation outside PRISM_LEDGER_READ_OPERATIONS folds - # into ``other`` instead of opening a series. - server.ledger.ledger_read_gate_stats = lambda: { - "pending_block_candidate_rows": { - "calls_total": 7, - "gate_wait_seconds_total": 0.125, - "gate_wait_seconds_max": 0.1, - "gate_timeouts_total": 1, - "execute_seconds_total": 4.5, - "execute_seconds_max": 2.25, - "execute_timeouts_total": 2, - }, - "current_prior_balances": { - "calls_total": 3, - "gate_wait_seconds_total": 0.5, - "gate_wait_seconds_max": 0.4, - "gate_timeouts_total": 0, - "execute_seconds_total": 0.75, - "execute_seconds_max": 0.5, - "execute_timeouts_total": 0, - }, - "legacy_probe": { - "calls_total": 1, - "gate_wait_seconds_total": 0.01, - "gate_wait_seconds_max": 0.01, - "gate_timeouts_total": 0, - "execute_seconds_total": 0.02, - "execute_seconds_max": 0.02, - "execute_timeouts_total": 0, - }, - } - # Issue #224 Wave 0's attribution families, seeded through the - # shared owner every lane will record into, including one - # out-of-vocabulary rescan reason that must fold into ``other``. - telemetry = ensure_accepted_preview_telemetry(server) - telemetry.observe_landing_phase("reconcile", 0.25) - telemetry.observe_landing_phase("reconcile", 0.75) - telemetry.observe_landing_phase("preview_publish", 0.05) - telemetry.observe_reconcile_pass("landing", 0.5) - telemetry.observe_reconcile_step("landing", "watch_query", 0.125) - telemetry.observe_reconcile_step("post_confirm", "candidate_prepare", 3.5) - telemetry.observe_payout_window_full_rescan( - "reconcile_invalidation", "daemon", 3.5 - ) - telemetry.observe_payout_window_full_rescan( - "not-a-contract-reason", "in_process", 0.5 - ) - server.ledger.block_candidate_pending_metrics = lambda: { - "pending_count": 2, - "oldest_pending_age_seconds": 1.5, - "oldest_unattempted_age_seconds": 0.5, - } - server.ledger.pending_ctv_fanout_statuses = lambda limit: [ - {"settlement_status": "broadcastable"}, - {"settlement_status": "failed"}, - {"settlement_status": "pending"}, - ] - # Job-build histogram and phase state via the J1 owner. - server._ensure_job_cache_state() - server.observe_job_build_elapsed(0.05, {"assemble": 0.02, "payout": 0.01}) - server._record_job_cache_event("template", hit=True) - server._record_job_cache_event("bundle", hit=False) - # Vardiff convergence/resume state via the vardiff owner. - server.listener_profiles = [ - SimpleNamespace(name="default"), - SimpleNamespace(name="highdiff"), - ] - vardiff_service = server._ensure_vardiff_service() - with vardiff_service._vardiff_convergence_lock: - vardiff_service.vardiff_lane_accepted_counts["default"] = 9 - vardiff_service.vardiff_lane_accepted_counts["highdiff"] = 2 - vardiff_service.vardiff_resume_outcome_counts["resumed"] = 3 - vardiff_service.vardiff_resume_outcome_counts["clamped"] = 1 - vardiff_service.vardiff_resume_outcome_counts["overridden"] = 2 - vardiff_service.session_difficulty_store.record( - ("default", "alice"), - Decimal("32768"), - now=time.monotonic(), - share_backed=True, - ) - return server - - def test_full_render_parity_with_frozen_reference(self) -> None: - server = self._seeded_coordinator() - frozen_now = time.monotonic() - with mock.patch.object(time, "monotonic", return_value=frozen_now): - expected = reference_render_metrics_payload(server) - actual = server.metrics_payload() - self.assertEqual(expected, actual) - # The parity document must exercise the families the acceptance - # checklist names; an accidentally empty fixture would prove nothing. - for needle in ( - "qbit_prism_block_candidate_accept_pending_defers_total 1", - 'qbit_prism_share_ack_seconds_count{result="accepted"} 1', - "qbit_prism_block_submit_seconds_count 1", - 'qbit_prism_accepted_block_preview_publication_seconds_count{result="published"} 1', - 'qbit_prism_reorg_reconcile_lookups_total{path="job_build",source="memo_hit"} 1', - 'qbit_prism_block_solves_dropped_total{reason="stale_grace"} 2', - 'qbit_prism_stale_job_abandons_total{class="', - "qbit_prism_stratum_semantic_current_work_ratio", - "qbit_prism_job_build_configured_workers", - 'qbit_prism_initial_job_prepared_work_total{result="admission_deadline"}', - "qbit_prism_job_build_orphan_evicted_total", - "qbit_prism_accepted_stats_reconcile_failures_total 1", - "qbit_prism_accepted_stats_reconcile_age_seconds 2.500000", - 'qbit_prism_block_ledger_calls_total{call_class="fast"} 1', - 'qbit_prism_block_ledger_call_timeouts_total{call_class="fast"} 1', - "qbit_prism_prior_balances_reads_total 3", - 'qbit_prism_ledger_read_calls_total{operation="pending_block_candidate_rows"} 7', - 'qbit_prism_ledger_read_gate_wait_seconds_max{operation="pending_block_candidate_rows"} 0.100000', - 'qbit_prism_ledger_read_gate_timeouts_total{operation="pending_block_candidate_rows"} 1', - 'qbit_prism_ledger_read_execute_seconds_max{operation="pending_block_candidate_rows"} 2.250000', - 'qbit_prism_ledger_read_execute_timeouts_total{operation="pending_block_candidate_rows"} 2', - 'qbit_prism_startup_phase_seconds{phase="audit_listener_bound"}', - "qbit_prism_accepted_parent_preview_wait_timeouts_total 2", - "qbit_prism_tip_refresh_wave_outcomes_total", - "qbit_prism_payout_artifact_events_total", - 'qbit_prism_payout_window_build_phase_seconds_count{phase="ledger_read",outcome="completed"}', - 'qbit_prism_payout_window_build_phase_seconds_count{phase="lock_wait",outcome="failed"}', - "qbit_prism_vardiff_sessions_at_max_difficulty", - 'qbit_prism_vardiff_resume_total{outcome="clamped"}', - 'qbit_prism_vardiff_resume_total{outcome="overridden"} 2', - 'qbit_prism_vardiff_lane_accepted_shares_per_second{lane="default"}', - 'qbit_prism_lease_heartbeat_attempts_total{mode="proof"}', - 'qbit_prism_lease_heartbeat_phase_seconds{phase="guard_slot_wait"}', - 'qbit_prism_lease_heartbeat_phase_seconds{phase="guard_client_resume"}', - 'qbit_prism_lease_heartbeat_policy_seconds{term="scheduler_slack"}', - 'qbit_prism_lease_heartbeat_policy_seconds{term="exit_envelope"}', - 'qbit_prism_lease_heartbeat_policy_seconds{term="server_proven_cap"}', - 'qbit_prism_lease_heartbeat_policy_seconds{term="max_guaranteed_monitor_lateness"} 0.550000', - 'qbit_prism_lease_heartbeat_monitor_wake_lateness_seconds_bucket{le="+Inf"}', - 'qbit_prism_lease_heartbeat_monitor_late_wakes_total{slack_fraction="1.0"}', - "qbit_prism_lease_heartbeat_monitor_wake_delay_window_max_seconds", - "qbit_prism_lease_heartbeat_monitor_wake_delay_record_age_seconds", - "qbit_prism_lease_heartbeat_monitor_exit_guarantee_breaches_total", - "qbit_prism_lease_heartbeat_stall_probe_samples_total", - "qbit_prism_lease_heartbeat_monitor_breach_warnings_suppressed_total", - 'qbit_prism_process_gc_pause_seconds_bucket{generation="2",le="+Inf"}', - 'qbit_prism_process_gc_max_pause_seconds{generation="0"}', - "qbit_prism_block_candidate_cleanup_retry_backlog 1", - f"qbit_prism_block_candidate_cleanup_retry_backlog_max {DEFAULT_BLOCK_CANDIDATE_CLEANUP_RETRY_BACKLOG_MAX}", - "qbit_prism_block_candidate_cleanup_retry_pending_share_holders 1", - "qbit_prism_block_candidate_cleanup_retry_terminal_outcome_pins 1", - "qbit_prism_block_candidate_cleanup_backpressure_active 0", - "qbit_prism_block_candidate_cleanup_backpressure_total 1", - 'qbit_prism_block_candidate_collapse_total{outcome="backlog_deferred"} 2', - 'qbit_prism_ledger_read_calls_total{operation="current_prior_balances"} 3', - 'qbit_prism_ledger_read_calls_total{operation="other"} 1', - 'qbit_prism_accepted_block_landing_phase_seconds_count{phase="reconcile"} 2', - 'qbit_prism_accepted_block_landing_phase_seconds_max{phase="reconcile"} 0.750000', - 'qbit_prism_accepted_block_landing_phase_seconds_sum{phase="lane_wait"} 0.000000', - 'qbit_prism_reorg_reconcile_pass_seconds_count{caller="landing"} 1', - 'qbit_prism_reorg_reconcile_step_seconds_sum{caller="landing",step="watch_query"} 0.125000', - 'qbit_prism_reorg_reconcile_step_seconds_max{caller="post_confirm",step="candidate_prepare"} 3.500000', - 'qbit_prism_payout_window_full_rescan_seconds_count{reason="reconcile_invalidation",path="daemon"} 1', - 'qbit_prism_payout_window_full_rescan_seconds_count{reason="other",path="in_process"} 1', - 'qbit_prism_payout_window_full_rescan_seconds_count{reason="window_daemon_state_lost",path="daemon"} 0', - "qbit_prism_process_allocated_blocks 123456", - 'qbit_prism_process_gc_trigger_count{generation="2"} 7', - 'qbit_prism_process_gc_uncollectable_objects_total{generation="2"} 2', - "qbit_prism_process_threads 64", - "qbit_prism_process_malloc_info_available 1", - "qbit_prism_process_malloc_arena_bytes 9000", - "qbit_prism_process_malloc_mmapped_bytes 500", - 'qbit_prism_component_entries{component="payout_window_pages"} 2', - 'qbit_prism_component_entries{component="payout_window_records"} 3', - 'qbit_prism_component_entries{component="share_window_serialization_shares"} 3', - 'qbit_prism_component_entries{component="job_bundle_cache"} 1', - # The #198 seed above publishes one terminal outcome of its own. - 'qbit_prism_component_entries{component="block_candidate_terminal_outcomes"} 2', - 'qbit_prism_component_entries{component="reconcile_flights"} 1', - 'qbit_prism_component_entries{component="reconcile_trusted_memo"} 1', - 'qbit_prism_component_entries{component="pending_share_commit_floor"} 1', - 'qbit_prism_component_entries{component="daemon_uploaded_windows"} 1', - 'qbit_prism_component_bytes{component="payout_window_canonical_json"} 22', - 'qbit_prism_component_bytes{component="share_window_serialization_spool"} 4096', - ): - self.assertIn(needle, actual) - self.assertNotIn('operation="legacy_probe"', actual) - self.assertNotIn("not-a-contract-reason", actual) - # The oldest-age gauge is a real elapsed interval, not the -1 - # sentinel, once a record is owed. - oldest = [ - line - for line in actual.splitlines() - if line.startswith("qbit_prism_block_candidate_cleanup_retry_oldest_seconds ") - ] - self.assertEqual(len(oldest), 1) - self.assertGreaterEqual(float(oldest[0].split()[1]), 0.0) - - def test_payout_window_build_phase_family_is_fixed_cardinality(self) -> None: - """Issue #228: one histogram family over closed phase and outcome sets.""" - server = self._seeded_coordinator() - payload = server.metrics_payload() - family = "qbit_prism_payout_window_build_phase_seconds" - self.assertEqual( - PRISM_PAYOUT_WINDOW_BUILD_PHASES, - ("ledger_read", "record_conversion", "daemon_prepare", "lock_wait"), - ) - self.assertEqual( - PRISM_PAYOUT_WINDOW_BUILD_OUTCOMES, - ("completed", "debounced", "aborted", "failed"), - ) - self.assertEqual(payload.count(f"# TYPE {family} histogram\n"), 1) - series = [ - line - for line in payload.splitlines() - if line.startswith(f"{family}_") - ] - self.assertTrue(series) - cells_seen: set[tuple[str, str]] = set() - for line in series: - labels = line.split("{", 1)[1].split("}", 1)[0] - keys = {pair.split("=", 1)[0] for pair in labels.split(",")} - # Only the closed phase and outcome labels and the histogram's - # own ``le``: no per-tip, per-generation or per-reason label - # may ever ride this family. - self.assertTrue(keys <= {"phase", "outcome", "le"}, line) - phase = labels.split('phase="', 1)[1].split('"', 1)[0] - outcome = labels.split('outcome="', 1)[1].split('"', 1)[0] - cells_seen.add((phase, outcome)) - self.assertEqual( - cells_seen, - { - (phase, outcome) - for phase in PRISM_PAYOUT_WINDOW_BUILD_PHASES - for outcome in PRISM_PAYOUT_WINDOW_BUILD_OUTCOMES - }, - ) - for phase, outcome in cells_seen: - labels = f'phase="{phase}",outcome="{outcome}"' - self.assertEqual( - sum(1 for line in series if line.startswith(f"{family}_count{{{labels}}}")), - 1, - ) - self.assertEqual( - sum(1 for line in series if line.startswith(f"{family}_sum{{{labels}}}")), - 1, - ) - - def test_renderer_bypasses_cached_metrics_path(self) -> None: - server = self._seeded_coordinator() - server.refresh_metrics_snapshot = mock.Mock( # type: ignore[method-assign] - side_effect=AssertionError("renderer must not re-enter the cache") - ) - server.cached_metrics_payload = mock.Mock( # type: ignore[method-assign] - side_effect=AssertionError("renderer must not re-enter the cache") - ) - payload = MetricsRenderer(server).render() - self.assertIn("qbit_prism_accepted_shares_total", payload) - - -class RemovedCompatibilitySurfaceTests(unittest.TestCase): - """PR 80's deleted re-exports and health properties must stay absent.""" - - REMOVED_MODULE_EXPORTS = ( - # coordinator_config re-exports - "DEFAULT_HIGHDIFF_DIFFICULTY", - "DEFAULT_HIGHDIFF_MAX_DIFFICULTY", - "DEFAULT_MIN_OUTPUT_FEERATE_SATS_PER_BYTE", - "DEFAULT_MIN_OUTPUT_SAFETY_MULTIPLIER", - "DEFAULT_P2MR_SPEND_INPUT_BYTES", - "DEFAULT_PRISM_COINBASE_TAG", - "DEFAULT_PRISM_VARDIFF_IDLE_SWEEP_SECONDS", - "DEFAULT_TESTNET_USERNAME_FALLBACK_ADDRESS", - "MAX_PRISM_COINBASE_TAG_BYTES", - "default_prism_coinbase_tag_hex", - "env_decimal", - "env_nonnegative_int_with_legacy", - "env_optional_bool", - "env_seed_hex", - "load_prism_highdiff_listener", - "load_prism_vardiff_config", - "production_mode", - "require_production_env", - "validate_prism_production_gate", - "validate_same_tip_job_retention_limits", - # stratum_session re-exports - "parse_stratum_password_options", - "parse_worker_username", - "split_worker_username", - # ctv re-exports - "MAX_CTV_FANOUT_BROADCASTER_CHUNK_SIZE", - "PRISM_CTV_BROADCASTER_CHUNK_ROWS_BUCKETS", - "PRISM_CTV_BROADCASTER_CHUNK_SECONDS_BUCKETS", - "PRISM_CTV_BROADCASTER_SECONDS_BUCKETS", - # job_bundle re-exports - "CollectionIdentityUnavailable", - "JobBuildKey", - "_JobBuildCancelled", - # job_delivery re-exports - "DEFAULT_PRISM_EVICTED_JOB_PRUNE_INTERVAL_SECONDS", - "InitialJobSnapshot", - "InitialJobTracker", - "MAX_ACTIVE_PRISM_JOBS_PER_CLIENT", - "PRISM_TIP_REFRESH_ADMISSION_POLL_SECONDS", - # tip_refresh re-exports - "PRISM_TIP_REFRESH_CANCELLATION_STAGES", - "PRISM_TIP_REFRESH_RESULTS", - "PublishedTipSnapshot", - # progress_health re-exports - "BundleBuildToken", - "RefreshActivityToken", - # payout_state aliases and re-export - "_AcceptedBlockPayoutTransition", - "_PayoutDeliveryAdmission", - "_PayoutStateDeliveryGate", - "_PayoutStatePublicationBlocked", - "PublishedPayoutState", - # coordinator_shutdown re-export - "_WriterOperationToken", - # duplicate job-build registries (canonical in job_bundle) - "PRISM_JOB_BUILD_PHASES", - "PRISM_JOB_CACHE_KINDS", - # reorg machinery moved to lab.prism.reorg_reconciler - "_ReconcileFlight", - "PRISM_REORG_RECONCILE_MEMO_MAX_TIPS", - "PRISM_REORG_RECONCILE_LOOKUP_PATHS", - "PRISM_REORG_RECONCILE_LOOKUP_SOURCES", - "PRISM_RECONCILE_PREFETCH_JOIN_TIMEOUT_CEILING_SECONDS", - ) - - KEPT_MODULE_EXPORTS = ( - "BackgroundServiceRegistry", - "BackgroundServiceSpec", - "CoordinatorShutdownController", - "ShutdownInProgress", - "ledger_writer_operation", - "DEFAULT_SHARE_COMMIT_BATCH_SIZE", - "DEFAULT_SHARE_COMMIT_LINGER_MILLISECONDS", - "DEFAULT_SHARE_COMMIT_TIMEOUT_SECONDS", - "_BoundedPriorityExecutor", - "_DeliveryQueueFull", - ) - - REMOVED_HEALTH_PROPERTIES = ( - "_health_snapshot", - "_health_snapshot_monotonic", - "_health_refresh_loop_running", - "_health_snapshot_lock", - "health_snapshot_refresh_failure_count", - ) - - def test_removed_reexports_are_absent_from_the_coordinator(self) -> None: - import lab.prism.prism_coordinator as prism_coordinator - - for name in self.REMOVED_MODULE_EXPORTS: - self.assertFalse(hasattr(prism_coordinator, name), name) - for name in self.KEPT_MODULE_EXPORTS: - self.assertTrue(hasattr(prism_coordinator, name), name) - # The buckets survive only as an import of the canonical owner. - self.assertIs( - prism_coordinator.PRISM_JOB_BUILD_SECONDS_BUCKETS, - PRISM_JOB_BUILD_SECONDS_BUCKETS, - ) - - def test_health_compatibility_properties_are_gone(self) -> None: - from lab.prism.observability import ObservabilityService - from lab.prism.prism_coordinator import PrismCoordinator - - for name in self.REMOVED_HEALTH_PROPERTIES: - self.assertFalse(hasattr(PrismCoordinator, name), name) - for name in ( - "lock", - "set_health_snapshot_for_compatibility", - "set_health_snapshot_monotonic_for_compatibility", - "set_loop_running_for_compatibility", - "set_refresh_failure_count_for_compatibility", - ): - self.assertFalse(hasattr(ObservabilityService, name), name) - for name in ( - "replace_lock_for_test", - "clear_health_snapshot_for_test", - "set_health_snapshot_monotonic_for_test", - "set_loop_running_for_test", - ): - self.assertTrue(hasattr(ObservabilityService, name), name) - - -class MetricsRendererTests(unittest.TestCase): - def test_shutdown_formatter_consumes_one_owner_snapshot(self) -> None: - snapshot_calls = 0 - - def snapshot() -> dict[str, object]: - nonlocal snapshot_calls - snapshot_calls += 1 - return { - "shutdowns_total": 1, - "writer_quiescence_outcomes": {"success": 1, "timeout": 0}, - "lease_release_outcomes": { - "success": 1, - "not_held": 0, - "unsupported": 0, - "failure": 0, - }, - "active_writers": {"candidate": 2}, - "writer_quiescence_seconds": 0.25, - "lease_release_attempts_total": 1, - "lease_release_seconds": 0.125, - "sigterm_release_observed": True, - "sigterm_to_lease_release_seconds": 0.5, - "release_withheld_total": 0, - "non_writer_drain_seconds": 0.75, - } - - port = SimpleNamespace( - _ensure_shutdown_controller=lambda: SimpleNamespace( - snapshot=snapshot - ), - prometheus_label_value=lambda value: value, - ) - lines = MetricsRenderer(port).shutdown_metrics_lines() # type: ignore[arg-type] - - self.assertEqual(snapshot_calls, 1) - self.assertIn("qbit_prism_shutdowns_total 1", lines) - self.assertIn( - 'qbit_prism_shutdown_writer_operations{component="candidate"} 2', - lines, - ) - self.assertIn( - "qbit_prism_shutdown_sigterm_to_lease_release_seconds 0.500000", - lines, - ) - - def test_block_submitter_formatter_consumes_one_merged_snapshot(self) -> None: - snapshot_calls = 0 - - def snapshot() -> dict[str, object]: - nonlocal snapshot_calls - snapshot_calls += 1 - return { - "pending_count": 2, - "oldest_pending_age_seconds": 1.5, - "oldest_unattempted_age_seconds": 0.5, - "backoff_active": True, - "backoff_remaining_seconds": 0.25, - "backoff_delay_seconds": 1.0, - "submit_seconds_buckets": {0.05: 1}, - "submit_seconds_sum": 0.04, - "submit_seconds_count": 1, - "accepted_preview_publication": { - "published": { - "buckets": { - bucket: (1 if bucket >= 0.5 else 0) - for bucket in ( - PRISM_ACCEPTED_BLOCK_PREVIEW_PUBLICATION_SECONDS_BUCKETS - ) - }, - "sum": 0.4, - "count": 1, - }, - "degraded": { - "buckets": { - bucket: 0 - for bucket in ( - PRISM_ACCEPTED_BLOCK_PREVIEW_PUBLICATION_SECONDS_BUCKETS - ) - }, - "sum": 0.0, - "count": 0, - }, - }, - } - - port = SimpleNamespace( - block_submitter_snapshot=snapshot, - block_candidate_collapse_snapshot=lambda: { - outcome: 0 for outcome in PRISM_BLOCK_CANDIDATE_COLLAPSE_OUTCOMES - }, - ) - lines = MetricsRenderer(port).block_submitter_metrics_lines() # type: ignore[arg-type] - - self.assertEqual(snapshot_calls, 1) - self.assertIn("qbit_prism_block_candidates_pending 2", lines) - self.assertIn("qbit_prism_block_submitter_retry_backoff_active 1", lines) - self.assertIn("qbit_prism_block_submit_seconds_count 1", lines) - self.assertIn( - 'qbit_prism_block_submit_seconds_bucket{le="0.05"} 1', - lines, - ) - self.assertIn( - "qbit_prism_accepted_block_preview_publication_seconds_bucket" - '{result="published",le="0.5"} 1', - lines, - ) - # The 5 s child wait budget must be an exact bucket boundary: the - # acceptance criterion is a p95 below it, which a Prometheus - # histogram can only answer at a boundary it carries. - self.assertIn( - "qbit_prism_accepted_block_preview_publication_seconds_bucket" - '{result="published",le="5"} 1', - lines, - ) - self.assertIn( - "qbit_prism_accepted_block_preview_publication_seconds_count" - '{result="published"} 1', - lines, - ) - self.assertIn( - "qbit_prism_accepted_block_preview_publication_seconds_count" - '{result="degraded"} 0', - lines, - ) - - def test_cleanup_backlog_formatter_consumes_one_owner_snapshot(self) -> None: - """Issue #198: one snapshot read, seven unlabelled families.""" - snapshot_calls = 0 - - def snapshot() -> dict[str, object]: - nonlocal snapshot_calls - snapshot_calls += 1 - return { - "depth": 3, - "backlog_max": 4096, - "oldest_age_seconds": 12.5, - "pending_share_holders": 2, - "terminal_outcome_pins": 3, - "backpressure_active": False, - "backpressure_engagements": 4, - } - - port = SimpleNamespace( - _ensure_block_candidate_service=lambda: SimpleNamespace( - collapsed_candidate_cleanup_backlog_snapshot=snapshot, - ), - ) - lines = MetricsRenderer(port).block_candidate_cleanup_backlog_metrics_lines() # type: ignore[arg-type] - - self.assertEqual(snapshot_calls, 1) - self.assertIn("qbit_prism_block_candidate_cleanup_retry_backlog 3", lines) - self.assertIn("qbit_prism_block_candidate_cleanup_retry_backlog_max 4096", lines) - self.assertIn( - "qbit_prism_block_candidate_cleanup_retry_oldest_seconds 12.500000", - lines, - ) - self.assertIn( - "qbit_prism_block_candidate_cleanup_retry_pending_share_holders 2", - lines, - ) - self.assertIn( - "qbit_prism_block_candidate_cleanup_retry_terminal_outcome_pins 3", - lines, - ) - self.assertIn("qbit_prism_block_candidate_cleanup_backpressure_active 0", lines) - self.assertIn("qbit_prism_block_candidate_cleanup_backpressure_total 4", lines) - # Fixed cardinality: no series in the family carries a label. - for line in lines: - if not line.startswith("#"): - self.assertNotIn("{", line) - self.assertEqual( - [ - (line.split()[2], line.split()[3]) - for line in lines - if line.startswith("# TYPE ") - ], - list(BLOCK_CANDIDATE_CLEANUP_BACKLOG_METRIC_FAMILIES), - ) - - def test_cleanup_backlog_families_render_from_the_shipped_owner(self) -> None: - """An empty backlog renders the -1 age sentinel and the default bound.""" - server = support.coordinator() - lines = MetricsRenderer(server).block_candidate_cleanup_backlog_metrics_lines() - self.assertIn("qbit_prism_block_candidate_cleanup_retry_backlog 0", lines) - self.assertIn( - "qbit_prism_block_candidate_cleanup_retry_backlog_max " - f"{DEFAULT_BLOCK_CANDIDATE_CLEANUP_RETRY_BACKLOG_MAX}", - lines, - ) - self.assertIn( - "qbit_prism_block_candidate_cleanup_retry_oldest_seconds -1.000000", - lines, - ) - self.assertIn("qbit_prism_block_candidate_cleanup_backpressure_active 0", lines) - # A pinned coordinator bound is the one exported, so an alert can be - # written as a ratio against the running configuration. - server.block_candidate_cleanup_retry_backlog_max = 12 - lines = MetricsRenderer(server).block_candidate_cleanup_backlog_metrics_lines() - self.assertIn("qbit_prism_block_candidate_cleanup_retry_backlog_max 12", lines) - self.assertEqual(lines, reference_block_candidate_cleanup_backlog_metrics_lines(server)) - - def test_collapse_counter_carries_the_backlog_deferred_outcome(self) -> None: - self.assertIn("backlog_deferred", PRISM_BLOCK_CANDIDATE_COLLAPSE_OUTCOMES) - server = support.coordinator() - lines = MetricsRenderer(server).block_submitter_metrics_lines() - self.assertIn( - 'qbit_prism_block_candidate_collapse_total{outcome="backlog_deferred"} 0', - lines, - ) - - def test_accepted_preview_attribution_formatter_consumes_one_owner_snapshot( - self, - ) -> None: - """Issue #224 Wave 0: one snapshot read, four closed-product families.""" - snapshot_calls = 0 - telemetry = AcceptedPreviewTelemetry() - telemetry.observe_landing_phase("balance_lock_wait", 1.5) - telemetry.observe_reconcile_pass("tip_refresh", 0.25) - telemetry.observe_reconcile_step("tip_refresh", "admission_wait", 0.2) - telemetry.observe_payout_window_full_rescan( - "window_daemon_state_lost", "in_process", 2.0 - ) - real_snapshot = telemetry.snapshot - - def snapshot() -> dict[str, object]: - nonlocal snapshot_calls - snapshot_calls += 1 - return real_snapshot() - - telemetry.snapshot = snapshot # type: ignore[method-assign] - port = SimpleNamespace(_accepted_preview_telemetry=telemetry) - lines = MetricsRenderer(port).accepted_preview_attribution_metrics_lines() # type: ignore[arg-type] - - self.assertEqual(snapshot_calls, 1) - self.assertIn( - 'qbit_prism_accepted_block_landing_phase_seconds_max{phase="balance_lock_wait"} 1.500000', - lines, - ) - self.assertIn( - 'qbit_prism_reorg_reconcile_pass_seconds_count{caller="tip_refresh"} 1', - lines, - ) - self.assertIn( - 'qbit_prism_reorg_reconcile_step_seconds_sum{caller="tip_refresh",step="admission_wait"} 0.200000', - lines, - ) - self.assertIn( - 'qbit_prism_payout_window_full_rescan_seconds_count{reason="window_daemon_state_lost",path="in_process"} 1', - lines, - ) - # Every cell of every closed product renders, at zero when unobserved. - expected_samples = 3 * ( - len(PRISM_ACCEPTED_LANDING_PHASES) - + len(PRISM_REORG_RECONCILE_CALLERS) - + len(PRISM_REORG_RECONCILE_CALLERS) * len(PRISM_REORG_RECONCILE_STEPS) - + len(PRISM_PAYOUT_WINDOW_FULL_RESCAN_REASONS) - * len(PRISM_PAYOUT_WINDOW_FULL_RESCAN_PATHS) - ) - self.assertEqual( - len([line for line in lines if not line.startswith("#")]), - expected_samples, - ) - - def test_ledger_read_operations_outside_the_contract_fold_into_other( - self, - ) -> None: - """Issue #224 Wave 0: the operation label cannot grow past the vocabulary.""" - stats = { - "calls_total": 2, - "gate_wait_seconds_total": 0.25, - "gate_wait_seconds_max": 0.2, - "gate_timeouts_total": 1, - "execute_seconds_total": 1.0, - "execute_seconds_max": 0.75, - "execute_timeouts_total": 0, - } - port = SimpleNamespace( - ledger=SimpleNamespace( - ledger_read_gate_stats=lambda: { - "zeta_probe": dict(stats), - "pending_block_candidate_rows": dict(stats), - "alpha_probe": dict(stats, gate_wait_seconds_max=0.9), - } - ), - prometheus_label_value=lambda value: value, - ) - lines = MetricsRenderer(port)._ledger_read_gate_metric_lines() # type: ignore[arg-type] - - operations = [ - line.split('operation="', 1)[1].split('"', 1)[0] - for line in lines - if line.startswith("qbit_prism_ledger_read_calls_total{") - ] - self.assertEqual(operations, ["other", "pending_block_candidate_rows"]) - self.assertIn('qbit_prism_ledger_read_calls_total{operation="other"} 4', lines) - self.assertIn( - 'qbit_prism_ledger_read_gate_wait_seconds_max{operation="other"} 0.900000', - lines, - ) - self.assertIn( - 'qbit_prism_ledger_read_gate_timeouts_total{operation="other"} 2', - lines, - ) - self.assertIn( - 'qbit_prism_ledger_read_calls_total{operation="pending_block_candidate_rows"} 2', - lines, - ) - - def test_share_ack_formatter_renders_owner_snapshot(self) -> None: - histograms = empty_share_ack_histograms() - histograms["accepted"]["buckets"][PRISM_JOB_BUILD_SECONDS_BUCKETS[0]] = 2 - histograms["accepted"]["sum"] = 0.01 - histograms["accepted"]["count"] = 2 - port = SimpleNamespace(share_ack_snapshot=lambda: histograms) - lines = MetricsRenderer(port).share_ack_metrics_lines() # type: ignore[arg-type] - self.assertIn( - f'qbit_prism_share_ack_seconds_bucket{{result="accepted",le="{PRISM_JOB_BUILD_SECONDS_BUCKETS[0]:g}"}} 2', - lines, - ) - self.assertIn( - 'qbit_prism_share_ack_seconds_count{result="accepted"} 2', - lines, - ) - for result in PRISM_SHARE_ACK_RESULTS: - self.assertIn( - f'qbit_prism_share_ack_seconds_count{{result="{result}"}} ' - f'{histograms[result]["count"]}', - lines, - ) - - -class AcceptedParentUnresolvedDepthGaugeTests(unittest.TestCase): - """The depth gauge must read exactly what the admission fence counts.""" - - def _coordinator(self): - server = support.coordinator() - server._ensure_job_cache_state() - return server - - def _landing_lines(self, server) -> list[str]: - return MetricsRenderer(server).landing_observability_metrics_lines() - - def _arm_landed_without_timestamp(self, server, block_hash: str) -> None: - """Reproduce the landed transitions that carry no monotonic stamp. - - Both the degraded preview-publication fallback and the durable - candidate replay reconstruct a transition from the dataclass default, - so ``landed`` is armed while ``landed_monotonic`` stays ``None``. - """ - with server._accepted_block_payout_preview_condition: - server._accepted_block_payout_previews[block_hash] = ( - AcceptedBlockPayoutTransition(block_height=11, landed=True) - ) - - def test_depth_gauge_counts_a_landed_transition_with_no_timestamp(self) -> None: - server = self._coordinator() - self._arm_landed_without_timestamp(server, "cd" * 32) - # The age list cannot see this transition, so a gauge derived from it - # would report zero while the fence already counts depth one. - self.assertEqual(server.accepted_parent_unresolved_ages_seconds(), []) - self.assertEqual(server._accepted_parent_unresolved_depth(), 1) - lines = self._landing_lines(server) - self.assertIn("qbit_prism_accepted_parent_unresolved_transitions 1", lines) - # No timestamp means no age to invent: the oldest gauge reports -1 - # rather than fabricating an age for the transition depth just counted. - self.assertIn( - "qbit_prism_accepted_parent_unresolved_oldest_seconds -1.000000", - lines, - ) - - def test_depth_gauge_matches_the_fence_across_mixed_transitions(self) -> None: - server = self._coordinator() - timestamped = "ce" * 32 - server._begin_accepted_block_payout_preview(timestamped, block_height=10) - server._mark_accepted_block_payout_landed(timestamped, block_height=10) - self._arm_landed_without_timestamp(server, "cf" * 32) - # A pending (not yet landed) transition stays out of both signals. - server._begin_accepted_block_payout_preview("d0" * 32, block_height=12) - self.assertEqual(len(server.accepted_parent_unresolved_ages_seconds()), 1) - self.assertEqual(server._accepted_parent_unresolved_depth(), 2) - lines = self._landing_lines(server) - self.assertIn("qbit_prism_accepted_parent_unresolved_transitions 2", lines) - self.assertNotIn( - "qbit_prism_accepted_parent_unresolved_oldest_seconds -1.000000", - lines, - ) - - def test_depth_gauge_is_zero_without_landed_transitions(self) -> None: - server = self._coordinator() - lines = self._landing_lines(server) - self.assertIn("qbit_prism_accepted_parent_unresolved_transitions 0", lines) - self.assertIn( - "qbit_prism_accepted_parent_unresolved_oldest_seconds -1.000000", - lines, - ) - - def test_exported_cap_is_the_runtime_configured_cap(self) -> None: - server = self._coordinator() - # The unconfigured runtime falls back to the shipped default. - self.assertEqual( - server._accepted_parent_unresolved_depth_cap(), - DEFAULT_ACCEPTED_PARENT_UNRESOLVED_DEPTH_MAX, - ) - self.assertIn( - "qbit_prism_accepted_parent_unresolved_depth_max " - f"{DEFAULT_ACCEPTED_PARENT_UNRESOLVED_DEPTH_MAX}", - self._landing_lines(server), - ) - # A non-default configured cap must reach the exposition too, so an - # alert can compare the depth against the limit actually enforced. - non_default = DEFAULT_ACCEPTED_PARENT_UNRESOLVED_DEPTH_MAX + 3 - server.accepted_parent_unresolved_depth_max = non_default - self.assertEqual(server._accepted_parent_unresolved_depth_cap(), non_default) - self.assertIn( - f"qbit_prism_accepted_parent_unresolved_depth_max {non_default}", - self._landing_lines(server), - ) - - -_SERIES_LINE = re.compile(r"^(?P<family>[a-z_]+)(?:\{(?P<labels>[^}]*)\})? (?P<value>\S+)$") - - -def _parse_series(line: str) -> tuple[str, dict[str, str], str]: - """Split one exposition sample into (family, labels, value).""" - match = _SERIES_LINE.match(line) - if match is None: - raise AssertionError(f"not a sample line: {line!r}") - labels: dict[str, str] = {} - if match.group("labels"): - for pair in match.group("labels").split(","): - key, _, quoted = pair.partition("=") - labels[key] = quoted.strip('"') - return match.group("family"), labels, match.group("value") - - -def _type_families(lines: list[str]) -> list[tuple[str, str]]: - return [ - (line.split()[2], line.split()[3]) - for line in lines - if line.startswith("# TYPE ") - ] - - -def _series_value(lines: list[str], series: str) -> int: - matches = [line for line in lines if line.startswith(series + " ")] - if len(matches) != 1: - raise AssertionError(f"{series!r} rendered {len(matches)} times") - return int(matches[0].split()[-1]) - - -class HeapAndComponentCardinalityTelemetryTests(unittest.TestCase): - """Issue #226 part 1: always-on heap telemetry and component gauges. - - The five behaviours the work package pins: the exact family set, the - mallinfo2-absent path rendering as unavailable rather than zero, every - component gauge reading its shipped owner, no unbounded label anywhere - in the component families, and a scrape that never walks the heap. - """ - - TIP = "aa" * 32 - - def _seeded_owners(self) -> tuple[object, dict[str, int], dict[str, int]]: - """Seed every component through its shipped owner; return the expectations. - - Each seed goes through the real owning structure -- the service - field, the queue, the LRU, the floor -- never a stub on the port, - so a renamed owner field breaks the read instead of reporting 0. - """ - server = support.coordinator() - server._ensure_job_cache_state() - server.jobs = {"job-1": object(), "job-2": object()} - bundles = server._ensure_job_bundle_service() - bundles._job_bundle_cache[("bundle-a",)] = object() - bundles._job_bundle_cache[("bundle-b",)] = object() - bundles._job_build_issued_at_ms[7] = 7_000 - bundles._bundle_preparation_flights[("flight",)] = object() - bundles._active_job_bundle_builds[("build",)] = object() - serialization = _ShareWindowSerialization( - key=("digest", 5, 0), - share_count=5, - share_snapshot_sha256="digest", - ) - serialization._spool_size = 4_096 - serialization._compact_tail_chunks = ("[" + "x" * 98 + "]", "y" * 20) - serialization._compact_tail_chars = 120 - bundles._share_window_serialization = serialization - pages = ( - _IncrementalShareWindowPage( - records=(object(), object()), - total_difficulty=0, - prism_json_records=({}, {}), - canonical_json_items=b'{"a":1},{"b":2}', - ), - _IncrementalShareWindowPage( - records=(object(),), - total_difficulty=0, - prism_json_records=({},), - canonical_json_items=b'{"c":3}', - ), - ) - payout = server._ensure_payout_state_service() - payout._incremental_payout_artifact_window = _IncrementalPayoutArtifactWindow( - window=IncrementalShareWindow( - anchor_job_issued_at_ms=1, - window_weight=1, - page_size=512, - pages=pages, - total_difficulty=0, - ), - shares_json=IncrementalShareJsonSequence(pages=pages, record_count=3), - share_snapshot_sha256="digest", - refreshed_monotonic=0.0, - full_rescan_monotonic=0.0, - full_rescan_attempt_monotonic=0.0, - ) - payout._payout_ledger_artifact = PayoutLedgerArtifact( - generation=1, - payout_state_generation=1, - network_difficulty=1, - accepted_share_count=3, - shares_json=IncrementalShareJsonSequence(pages=pages, record_count=3), - prior_balances=(), - prepared_monotonic=0.0, - ) - payout._accepted_block_payout_previews["b1" * 32] = AcceptedBlockPayoutTransition( - block_height=11, - landed=True, - ) - payout._invalidated_accepted_block_payout_previews["b2" * 32] = 5 - payout._payout_append_invalidation_stamps[1] = 100 - payout._payout_append_invalidation_stamps[2] = 200 - payout._payout_window_inflight_scan_anchors[1] = 100 - payout._payout_unfenced_append_inflight_stamps[1] = 100 - server.evicted_job_graveyard = { - "job-old": ( - SimpleNamespace( - job_id="job-old", - template={"previousblockhash": "bb" * 32}, - ), - 1, - time.monotonic(), - ), - } - candidates = server._ensure_block_candidate_service() - candidates._ensure_block_replay_state() - candidates._ensure_block_candidate_disposition_state() - candidates.candidate_queue.put_nowait(object()) - candidates._block_replay_candidate_queue.put_nowait(object()) - candidates._block_replay_candidate_queue.put_nowait(object()) - candidates._block_replay_inflight_hashes.add("c1" * 32) - candidates._block_quarantine_queue.put_nowait(object()) - candidates._block_quarantine_hashes.add("c2" * 32) - candidates._outstanding_block_candidate_hashes.update({"c3" * 32, "c4" * 32}) - candidates._tip_observed_accepted_block_hashes["c5" * 32] = 1.0 - candidates._counted_block_candidate_abandonments.add("c6" * 32) - candidates._accepted_block_preview_acceptance_monotonic["c7" * 32] = None - candidates._ancestor_redrive_records["c8" * 32] = object() - candidates._block_candidate_terminal_outcomes["c9" * 32] = True - candidates._block_candidate_terminal_outcomes["ca" * 32] = False - candidates._block_candidate_disposition_flights["cb" * 32] = object() - candidates._block_disposition_waiting_retries["cc" * 32] = object() - candidates._block_candidate_dequeued_hashes["cd" * 32] = 1 - server._accounted_accepted_block_hashes.add("ce" * 32) - reconciler = server._ensure_reorg_reconciler_service() - reconciler._reconcile_flights[self.TIP] = _ReconcileFlight() - reconciler._reorg_reconcile_trusted_memo[self.TIP] = 1.0 - reconciler._reorg_reconcile_trusted_memo["dd" * 32] = 2.0 - writer = server._ensure_share_writer_service() - writer.adopt_pending_share(SimpleNamespace(share_id="miner-a:s1")) - writer.adopt_pending_share(SimpleNamespace(share_id="miner-a:s2")) - daemon_client = _ServeBuilderClient(process=SimpleNamespace()) - daemon_client.note_uploaded_window("digest") - daemon_client.note_uploaded_window("digest-2") - server._ensure_bundle_compiler()._serve_builder = daemon_client - expected_entries = { - "payout_window_pages": 2, - "payout_window_records": 3, - "payout_ledger_artifact_shares": 3, - "accepted_block_payout_previews": 1, - "invalidated_accepted_block_payout_previews": 1, - "payout_append_invalidation_stamps": 2, - "payout_window_inflight_scan_anchors": 1, - "payout_unfenced_append_inflight_stamps": 1, - "share_window_serialization_shares": 5, - "job_contexts": 2, - "job_bundle_cache": 2, - "job_build_issued_stamps": 1, - "bundle_preparation_flights": 1, - "active_job_bundle_builds": 1, - "evicted_job_graveyard": 1, - # With no published tip the delivery owner classes every retained - # context as same-tip, so its same-tip index holds the entry too. - "evicted_same_tip_job_ids": 1, - "block_candidate_queue": 1, - "block_replay_queue": 2, - "block_replay_inflight_hashes": 1, - "block_quarantine_queue": 1, - "block_quarantine_hashes": 1, - "outstanding_block_candidate_hashes": 2, - "tip_observed_accepted_block_hashes": 1, - "counted_block_candidate_abandonments": 1, - "accepted_block_preview_stamps": 1, - "ancestor_redrive_records": 1, - "block_candidate_terminal_outcomes": 2, - "block_candidate_disposition_flights": 1, - "block_disposition_waiting_retries": 1, - "block_candidate_dequeued_hashes": 1, - "accounted_accepted_block_hashes": 1, - "reconcile_flights": 1, - "reconcile_trusted_memo": 2, - "pending_share_commit_floor": 2, - "daemon_window_mirror_records": 0, - "daemon_uploaded_windows": 2, - } - expected_bytes = { - "payout_window_canonical_json": 22, - "daemon_window_mirror_canonical_items": 0, - "share_window_serialization_spool": 4_096, - "share_window_serialization_compact_json": 120, - } - return server, expected_entries, expected_bytes - - def test_process_heap_telemetry_families_are_pinned(self) -> None: - """The exact (family, type) sequence renders, in order, fixed cardinality.""" - server = support.coordinator() - renderer = MetricsRenderer(server) - heap = renderer.process_heap_metrics_lines() - self.assertEqual(_type_families(heap), list(PROCESS_HEAP_METRIC_FAMILIES)) - rendered_generations: dict[str, list[str]] = {} - for line in heap: - if line.startswith("#"): - continue - family, labels, _value = _parse_series(line) - if family in PROCESS_HEAP_GENERATION_FAMILIES: - # The one permitted label, drawn from the closed set. - self.assertEqual(set(labels), {"generation"}) - self.assertIn(labels["generation"], PRISM_GC_GENERATIONS) - rendered_generations.setdefault(family, []).append(labels["generation"]) - else: - self.assertNotIn("{", line) - for family in PROCESS_HEAP_GENERATION_FAMILIES: - self.assertEqual(rendered_generations[family], list(PRISM_GC_GENERATIONS)) - components = renderer.component_cardinality_metrics_lines() - self.assertEqual( - _type_families(components), - list(COMPONENT_CARDINALITY_METRIC_FAMILIES), - ) - entry_kinds = [ - _parse_series(line)[1]["component"] - for line in components - if line.startswith("qbit_prism_component_entries{") - ] - byte_kinds = [ - _parse_series(line)[1]["component"] - for line in components - if line.startswith("qbit_prism_component_bytes{") - ] - self.assertEqual(entry_kinds, list(PRISM_COMPONENT_ENTRY_KINDS)) - self.assertEqual(byte_kinds, list(PRISM_COMPONENT_BYTE_KINDS)) - self.assertEqual(len(set(PRISM_COMPONENT_ENTRY_KINDS)), len(PRISM_COMPONENT_ENTRY_KINDS)) - # Every component the issue names has a gauge. - for named in ( - "payout_window_pages", - "share_window_serialization_shares", - "job_contexts", - "job_bundle_cache", - "evicted_job_graveyard", - "block_candidate_queue", - "block_replay_queue", - "reconcile_flights", - "pending_share_commit_floor", - "daemon_window_mirror_records", - ): - self.assertIn(named, PRISM_COMPONENT_ENTRY_KINDS) - # The full document carries both blocks, in this order, at its tail. - type_lines = [ - line for line in server.metrics_payload().splitlines() - if line.startswith("# TYPE ") - ] - expected_tail = [ - f"# TYPE {family} {kind}" - for family, kind in ( - PROCESS_HEAP_METRIC_FAMILIES + COMPONENT_CARDINALITY_METRIC_FAMILIES - ) - ] - self.assertEqual(type_lines[-len(expected_tail):], expected_tail) - - def test_mallinfo_absence_renders_unavailable_not_zero(self) -> None: - """A failed ctypes lookup renders availability 0 and -1 bytes, never 0.""" - server = support.coordinator() - resolver = mock.Mock(side_effect=AttributeError("mallinfo2: symbol not found")) - telemetry = ProcessHeapTelemetry(mallinfo_resolver=resolver) - renderer = MetricsRenderer(server, process_telemetry=telemetry) - first = renderer.process_heap_metrics_lines() - second = renderer.process_heap_metrics_lines() - self.assertEqual(_series_value(first, "qbit_prism_process_malloc_info_available"), 0) - for gauge in ("arena", "in_use", "free", "mmapped"): - self.assertEqual( - _series_value(first, f"qbit_prism_process_malloc_{gauge}_bytes"), - -1, - ) - # The interpreter readings are real even when glibc's are not. - self.assertGreater(_series_value(first, "qbit_prism_process_allocated_blocks"), 0) - self.assertGreaterEqual(_series_value(first, "qbit_prism_process_threads"), 1) - # Resolved once and remembered: no per-scrape lookup, no per-scrape log. - self.assertEqual(resolver.call_count, 1) - self.assertEqual(_type_families(second), _type_families(first)) - self.assertEqual(_series_value(second, "qbit_prism_process_malloc_info_available"), 0) - # The family set is identical to the available case, so a dashboard - # built on either platform reads the same series. - self.assertEqual(_type_families(first), list(PROCESS_HEAP_METRIC_FAMILIES)) - # A symbol that binds but fails at call time degrades the same way - # and is not retried on the next scrape. - broken = mock.Mock(side_effect=OSError("mallinfo2 call failed")) - renderer = MetricsRenderer( - server, - process_telemetry=ProcessHeapTelemetry(mallinfo_resolver=lambda: broken), - ) - lines = renderer.process_heap_metrics_lines() - renderer.process_heap_metrics_lines() - self.assertEqual(_series_value(lines, "qbit_prism_process_malloc_info_available"), 0) - self.assertEqual(_series_value(lines, "qbit_prism_process_malloc_arena_bytes"), -1) - self.assertEqual(broken.call_count, 1) - # Where the symbol binds, the struct's fields render; the fake makes - # this branch platform-independent too. - def fake_mallinfo2() -> MallInfo2: - info = MallInfo2() - info.arena = 9_000 - info.uordblks = 6_000 - info.fordblks = 3_000 - info.hblkhd = 500 - return info - - lines = MetricsRenderer( - server, - process_telemetry=ProcessHeapTelemetry(mallinfo_resolver=lambda: fake_mallinfo2), - ).process_heap_metrics_lines() - self.assertEqual(_series_value(lines, "qbit_prism_process_malloc_info_available"), 1) - self.assertEqual(_series_value(lines, "qbit_prism_process_malloc_arena_bytes"), 9_000) - self.assertEqual(_series_value(lines, "qbit_prism_process_malloc_in_use_bytes"), 6_000) - self.assertEqual(_series_value(lines, "qbit_prism_process_malloc_free_bytes"), 3_000) - self.assertEqual(_series_value(lines, "qbit_prism_process_malloc_mmapped_bytes"), 500) - # The operator switch renders unavailable without probing at all. - resolver = mock.Mock() - lines = MetricsRenderer( - server, - process_telemetry=ProcessHeapTelemetry( - malloc_enabled=False, - mallinfo_resolver=resolver, - ), - ).process_heap_metrics_lines() - self.assertEqual(_series_value(lines, "qbit_prism_process_malloc_info_available"), 0) - self.assertEqual(_series_value(lines, "qbit_prism_process_malloc_in_use_bytes"), -1) - resolver.assert_not_called() - - def test_component_cardinality_gauges_render_from_the_shipped_owner(self) -> None: - """Each gauge reads the real owning structure, not a stub.""" - server, expected_entries, expected_bytes = self._seeded_owners() - self.assertEqual(set(expected_entries), set(PRISM_COMPONENT_ENTRY_KINDS)) - self.assertEqual(set(expected_bytes), set(PRISM_COMPONENT_BYTE_KINDS)) - lines = MetricsRenderer(server).component_cardinality_metrics_lines() - for kind, value in expected_entries.items(): - self.assertIn( - f'qbit_prism_component_entries{{component="{kind}"}} {value}', - lines, - ) - for kind, value in expected_bytes.items(): - self.assertIn( - f'qbit_prism_component_bytes{{component="{kind}"}} {value}', - lines, - ) - self.assertEqual(lines, reference_component_cardinality_metrics_lines(server)) - # A mirror-backed window reports the daemon's bytes and records and - # no in-process pages: the same slot, the other shipped window type. - items = b'{"a":1},{"b":2}' - digest = hashlib.sha256(b"[" + items + b"]").hexdigest() - mirror = DaemonShareWindowMirror.from_full_items( - anchor_job_issued_at_ms=1, - window_weight=1, - page_size=512, - record_count=2, - canonical_items=items, - share_snapshot_sha256=digest, - ) - payout = server._ensure_payout_state_service() - payout._incremental_payout_artifact_window = _IncrementalPayoutArtifactWindow( - window=mirror, - shares_json=mirror.json_records(), - share_snapshot_sha256=digest, - refreshed_monotonic=0.0, - full_rescan_monotonic=0.0, - full_rescan_attempt_monotonic=0.0, - ) - lines = MetricsRenderer(server).component_cardinality_metrics_lines() - # Accounted exactly once. The mirror *is* the cached window here, so - # its size appears only under daemon_window_mirror_*; reporting it - # under payout_window_* as well would make an operator summing - # qbit_prism_component_bytes for RSS attribution count it twice. - self.assertIn('qbit_prism_component_entries{component="payout_window_pages"} 0', lines) - self.assertIn('qbit_prism_component_entries{component="payout_window_records"} 0', lines) - self.assertIn( - 'qbit_prism_component_entries{component="daemon_window_mirror_records"} 2', - lines, - ) - self.assertIn( - 'qbit_prism_component_bytes{component="payout_window_canonical_json"} 0', - lines, - ) - self.assertIn( - f'qbit_prism_component_bytes{{component="daemon_window_mirror_canonical_items"}} {len(items)}', - lines, - ) - # The same property stated as a sum, so a future backing that reports - # under both slots fails here rather than in an operator's dashboard. - self.assertEqual( - sum( - int(line.rsplit(" ", 1)[1]) - for line in lines - if line.startswith("qbit_prism_component_bytes{") - and ( - 'component="payout_window_canonical_json"' in line - or 'component="daemon_window_mirror_canonical_items"' in line - ) - ), - len(items), - ) - # An empty window slot and an absent daemon client are true zeros. - payout._incremental_payout_artifact_window = None - server._ensure_bundle_compiler()._serve_builder = None - lines = MetricsRenderer(server).component_cardinality_metrics_lines() - self.assertIn('qbit_prism_component_entries{component="payout_window_records"} 0', lines) - self.assertIn('qbit_prism_component_entries{component="daemon_uploaded_windows"} 0', lines) - # The reads are direct: renaming an owner field breaks the scrape - # loudly instead of reporting a silent zero. - reconciler = server._ensure_reorg_reconciler_service() - flights = reconciler._reconcile_flights - del reconciler._reconcile_flights - try: - with self.assertRaises(AttributeError): - MetricsRenderer(server).component_cardinality_metrics_lines() - finally: - reconciler._reconcile_flights = flights - candidates = server._ensure_block_candidate_service() - outstanding = candidates._outstanding_block_candidate_hashes - del candidates._outstanding_block_candidate_hashes - try: - with self.assertRaises(AttributeError): - MetricsRenderer(server).component_cardinality_metrics_lines() - finally: - candidates._outstanding_block_candidate_hashes = outstanding - - def test_component_cardinality_gauges_carry_no_unbounded_label(self) -> None: - """No series carries a label that varies per job, tip, hash, or generation.""" - server, _entries, _bytes = self._seeded_owners() - - def series_set(lines: list[str]) -> list[tuple[str, str]]: - observed: list[tuple[str, str]] = [] - for line in lines: - if line.startswith("#"): - continue - family, labels, _value = _parse_series(line) - self.assertEqual(set(labels), {"component"}, line) - closed = ( - PRISM_COMPONENT_ENTRY_KINDS - if family == "qbit_prism_component_entries" - else PRISM_COMPONENT_BYTE_KINDS - ) - self.assertIn(labels["component"], closed, line) - observed.append((family, labels["component"])) - return observed - - before = MetricsRenderer(server).component_cardinality_metrics_lines() - baseline = series_set(before) - self.assertEqual(len(baseline), len(set(baseline))) - self.assertEqual( - len(baseline), - len(PRISM_COMPONENT_ENTRY_KINDS) + len(PRISM_COMPONENT_BYTE_KINDS), - ) - # Grow every unbounded population the issue names -- more jobs, more - # tips, more candidate hashes, more generations, more workers -- and - # the series set must not move by one. - seeded_identifiers = [] - for index in range(5): - job_id = f"job-growth-{index}" - tip_hash = f"{index:02x}" * 32 - block_hash = f"e{index}" * 32 - seeded_identifiers.extend((job_id, tip_hash, block_hash, f"worker-{index}")) - server.jobs[job_id] = object() - reconciler = server._ensure_reorg_reconciler_service() - reconciler._reorg_reconcile_trusted_memo[tip_hash] = float(index) - reconciler._reconcile_flights[tip_hash] = _ReconcileFlight() - candidates = server._ensure_block_candidate_service() - candidates._outstanding_block_candidate_hashes.add(block_hash) - candidates._block_candidate_terminal_outcomes[block_hash] = True - server._ensure_payout_state_service()._payout_append_invalidation_stamps[ - 10 + index - ] = index - server._ensure_share_writer_service().adopt_pending_share( - SimpleNamespace(share_id=f"worker-{index}:{block_hash}") - ) - after = MetricsRenderer(server).component_cardinality_metrics_lines() - self.assertEqual(series_set(after), baseline) - self.assertEqual(_type_families(after), _type_families(before)) - rendered = "\n".join(after) - for identifier in seeded_identifiers: - self.assertNotIn(identifier, rendered) - # The values moved; only the label set stayed put. - self.assertGreater( - _series_value(after, 'qbit_prism_component_entries{component="job_contexts"}'), - _series_value(before, 'qbit_prism_component_entries{component="job_contexts"}'), - ) - - def test_heap_telemetry_scrape_does_not_walk_the_heap(self) -> None: - """The collector never calls gc.get_objects or starts tracemalloc.""" - server, _entries, _bytes = self._seeded_owners() - self.assertFalse(tracemalloc.is_tracing()) - with ( - mock.patch.object( - gc, - "get_objects", - side_effect=AssertionError("gc.get_objects walked the heap on a scrape"), - ) as get_objects, - mock.patch.object( - gc, - "collect", - side_effect=AssertionError("gc.collect ran on a scrape"), - ) as collect, - mock.patch.object( - tracemalloc, - "start", - side_effect=AssertionError("tracemalloc started on a scrape"), - ) as start, - mock.patch.object( - tracemalloc, - "take_snapshot", - side_effect=AssertionError("tracemalloc snapshot on a scrape"), - ) as take_snapshot, - ): - renderer = MetricsRenderer(server) - heap = renderer.process_heap_metrics_lines() - components = renderer.component_cardinality_metrics_lines() - document = renderer.render() - self.assertEqual(_type_families(heap), list(PROCESS_HEAP_METRIC_FAMILIES)) - self.assertEqual( - _type_families(components), - list(COMPONENT_CARDINALITY_METRIC_FAMILIES), - ) - self.assertIn("qbit_prism_process_allocated_blocks ", document) - get_objects.assert_not_called() - collect.assert_not_called() - start.assert_not_called() - take_snapshot.assert_not_called() - self.assertFalse(tracemalloc.is_tracing()) - - -# Issue #227: the closed family sets of the monitor-lateness signals and the -# GC pause durations; tests pin the renderer to both. -LEASE_MONITOR_STALL_METRIC_FAMILIES = ( - ("qbit_prism_lease_heartbeat_monitor_wake_lateness_seconds", "histogram"), - ("qbit_prism_lease_heartbeat_monitor_late_wakes_total", "counter"), - ("qbit_prism_lease_heartbeat_monitor_wake_delay_window_max_seconds", "gauge"), - ("qbit_prism_lease_heartbeat_monitor_wake_delay_record_age_seconds", "gauge"), - ("qbit_prism_lease_heartbeat_monitor_exit_guarantee_breaches_total", "counter"), - ("qbit_prism_lease_heartbeat_monitor_worst_exit_guarantee_overrun_seconds", "gauge"), - ("qbit_prism_lease_heartbeat_stall_probe_samples_total", "counter"), - ("qbit_prism_lease_heartbeat_stall_probe_suppressed_total", "counter"), - ("qbit_prism_lease_heartbeat_monitor_breach_warnings_suppressed_total", "counter"), -) -GC_PAUSE_METRIC_FAMILIES = ( - ("qbit_prism_process_gc_pause_seconds", "histogram"), - ("qbit_prism_process_gc_last_pause_seconds", "gauge"), - ("qbit_prism_process_gc_max_pause_seconds", "gauge"), -) - - -class LeaseMonitorStallTelemetryTests(unittest.TestCase): - """Issue #227: the re-armable monitor-lateness signals and GC pauses.""" - - @staticmethod - def _series_keys(lines: list[str]) -> list[tuple[str, tuple[tuple[str, str], ...]]]: - return [ - (family, tuple(sorted(labels.items()))) - for family, labels, _value in ( - _parse_series(line) for line in lines if not line.startswith("#") - ) - ] - - def test_monitor_wake_delay_histogram_and_threshold_counters_are_pinned( - self, - ) -> None: - """Family set, bucket boundaries and the slack-fraction counters. - - All fixed cardinality: the series set after two thousand more - observations is identical to the series set before them. The - lifetime ``..._monitor_wake_delay_seconds`` gauge still renders. - """ - server = support.coordinator() - service = server._ensure_lease_heartbeat_service() - slack = service.policy().scheduler_slack_seconds - # The renderer's snapshot reads the service clock (time.monotonic - # here), so the seeded wakes are stamped on that clock too; the - # rolling window and the record age are then read at "now". - now = time.monotonic() - 10.0 - for delay in [0.0] * 100 + [0.3] * 40 + [0.45] * 30 + [0.6] * 20: - now += 0.05 - service.monitor_wakes.observe(delay, slack_seconds=slack, now=now) - service.monitor_wakes.observe(0.648, slack_seconds=slack, now=now + 0.05) - # The lifetime gauge is the monitor loop's own high-water mark. - service.monitor_wake_delay_seconds = 0.648 - renderer = MetricsRenderer(server) - lines = renderer.lease_heartbeat_metrics_lines() - - families = _type_families(lines) - self.assertIn( - ("qbit_prism_lease_heartbeat_monitor_wake_delay_seconds", "gauge"), families - ) - self.assertIn("qbit_prism_lease_heartbeat_monitor_wake_delay_seconds 0.648000", lines) - self.assertEqual( - families[-len(LEASE_MONITOR_STALL_METRIC_FAMILIES):], - list(LEASE_MONITOR_STALL_METRIC_FAMILIES), - ) - # Bucket boundaries are the closed set plus +Inf, cumulative. - histogram = "qbit_prism_lease_heartbeat_monitor_wake_lateness_seconds" - bucket_lines = [line for line in lines if line.startswith(histogram + "_bucket{")] - self.assertEqual( - [_parse_series(line)[1] for line in bucket_lines], - [{"le": f"{bucket:g}"} for bucket in LEASE_MONITOR_WAKE_DELAY_BUCKETS] - + [{"le": "+Inf"}], - ) - counts = [int(_parse_series(line)[2]) for line in bucket_lines] - self.assertEqual(counts, sorted(counts)) - self.assertEqual(counts[-1], 191) - self.assertEqual(_series_value(lines, histogram + "_count"), 191) - by_bound = dict(zip(LEASE_MONITOR_WAKE_DELAY_BUCKETS, counts)) - self.assertEqual(by_bound[0.001], 100) - self.assertEqual(by_bound[0.4], 140) - self.assertEqual(by_bound[0.5], 170) - self.assertEqual(by_bound[0.75], 191) - # The 0.5 / 0.8 / 1.0 x slack counters, exactly that label set. - counter = "qbit_prism_lease_heartbeat_monitor_late_wakes_total" - fraction_lines = [line for line in lines if line.startswith(counter + "{")] - self.assertEqual( - [_parse_series(line)[1]["slack_fraction"] for line in fraction_lines], - list(LEASE_MONITOR_LATE_WAKE_SLACK_FRACTIONS), - ) - self.assertEqual(_series_value(lines, counter + '{slack_fraction="0.5"}'), 91) - self.assertEqual(_series_value(lines, counter + '{slack_fraction="0.8"}'), 51) - self.assertEqual(_series_value(lines, counter + '{slack_fraction="1.0"}'), 21) - # Rolling maximum, record age, the breach counters and the probe. - self.assertIn( - "qbit_prism_lease_heartbeat_monitor_wake_delay_window_max_seconds 0.648000", - lines, - ) - self.assertTrue( - any( - line.startswith( - "qbit_prism_lease_heartbeat_monitor_wake_delay_record_age_seconds " - ) - for line in lines - ) - ) - self.assertEqual( - _series_value(lines, "qbit_prism_lease_heartbeat_monitor_exit_guarantee_breaches_total"), - 0, - ) - self.assertEqual( - _series_value(lines, "qbit_prism_lease_heartbeat_stall_probe_samples_total"), 0 - ) - # Suppressed breach warnings are exported alongside the probe's. - service.suppressed_slack_breach_warnings = 4 - self.assertEqual( - _series_value( - renderer.lease_heartbeat_metrics_lines(), - "qbit_prism_lease_heartbeat_monitor_breach_warnings_suppressed_total", - ), - 4, - ) - # The policy term alerts compare against, and the new phase. - self.assertIn( - 'qbit_prism_lease_heartbeat_policy_seconds{term="max_guaranteed_monitor_lateness"} 0.550000', - lines, - ) - self.assertEqual( - [ - _parse_series(line)[1]["phase"] - for line in lines - if line.startswith("qbit_prism_lease_heartbeat_phase_seconds{") - ], - list(LEASE_HEARTBEAT_PHASES), - ) - self.assertIn("guard_client_resume", LEASE_HEARTBEAT_PHASES) - # Fixed cardinality: traffic moves values, never the series set. - before = self._series_keys(lines) - for _ in range(2_000): - now += 0.05 - service.monitor_wakes.observe(1.7, slack_seconds=slack, now=now) - self.assertEqual(self._series_keys(renderer.lease_heartbeat_metrics_lines()), before) - - def test_gc_pause_families_are_pinned_to_the_closed_generation_set(self) -> None: - server = support.coordinator() - service = server._ensure_lease_heartbeat_service() - service.gc_pauses.record("2", 0.3) - service.gc_pauses.record("0", 0.0004) - service.gc_pauses.record("7", 9.0) # not a CPython generation: dropped - renderer = MetricsRenderer(server) - lines = renderer.gc_pause_metrics_lines() - self.assertEqual(_type_families(lines), list(GC_PAUSE_METRIC_FAMILIES)) - rendered: dict[str, list[str]] = {} - for line in lines: - if line.startswith("#"): - continue - family, labels, _value = _parse_series(line) - self.assertEqual(set(labels) - {"le"}, {"generation"}) - self.assertIn(labels["generation"], PRISM_GC_GENERATIONS) - if family == "qbit_prism_process_gc_pause_seconds_bucket": - rendered.setdefault(labels["generation"], []).append(labels["le"]) - for generation in PRISM_GC_GENERATIONS: - self.assertEqual( - rendered[generation], - [f"{bucket:g}" for bucket in PRISM_GC_PAUSE_SECONDS_BUCKETS] + ["+Inf"], - ) - self.assertEqual( - _series_value(lines, 'qbit_prism_process_gc_pause_seconds_count{generation="2"}'), - 1, - ) - self.assertEqual( - _series_value(lines, 'qbit_prism_process_gc_pause_seconds_count{generation="1"}'), - 0, - ) - self.assertIn('qbit_prism_process_gc_last_pause_seconds{generation="2"} 0.300000', lines) - self.assertIn('qbit_prism_process_gc_max_pause_seconds{generation="0"} 0.000400', lines) - # In the full document the block sits immediately before the issue - # #226 heap block it complements. - type_lines = [ - line for line in server.metrics_payload().splitlines() - if line.startswith("# TYPE ") - ] - index = type_lines.index("# TYPE qbit_prism_process_gc_pause_seconds histogram") - self.assertEqual( - type_lines[index : index + len(GC_PAUSE_METRIC_FAMILIES)], - [f"# TYPE {family} {kind}" for family, kind in GC_PAUSE_METRIC_FAMILIES], - ) - self.assertEqual( - type_lines[index + len(GC_PAUSE_METRIC_FAMILIES)], - "# TYPE qbit_prism_process_allocated_blocks gauge", - ) - - -if __name__ == "__main__": - unittest.main() diff --git a/tests/test_prism_observability.py b/tests/test_prism_observability.py deleted file mode 100644 index 8de991ce6..000000000 --- a/tests/test_prism_observability.py +++ /dev/null @@ -1,1119 +0,0 @@ -#!/usr/bin/env python3 -"""Direct ownership and safety tests for PRISM health observability.""" - -from __future__ import annotations - -from dataclasses import replace -import threading -import time -import unittest -from types import SimpleNamespace - -from lab.prism.observability import ( - HEALTH_SCHEMA, - METRICS_STALE_WARNING, - METRICS_STATES, - METRICS_STATE_FRESH, - METRICS_STATE_HEADER, - METRICS_STATE_STALE, - METRICS_STATE_UNAVAILABLE, - MiningDeliveryInputs, - ObservabilityService, -) -from lab.prism.prism_coordinator import ( - ClientState, - PrismCoordinator, - WorkerIdentity, -) - - -def healthy_inputs() -> MiningDeliveryInputs: - return MiningDeliveryInputs( - active_connections=0, - connection_capacity=64, - peak_active_connections=0, - subscribed_connections=0, - authorized_connections=0, - pending_initial_jobs=0, - pending_initial_job_capacity=32, - oldest_pending_initial_job_age_seconds=0.0, - oldest_genuinely_pending_initial_job_age_seconds=0.0, - clients_with_current_tip_jobs=0, - clients_with_semantically_current_work=0, - clients_with_no_active_job=0, - last_initial_job_delivery_monotonic=None, - initial_job_timeout_seconds=5.0, - initial_job_queue_rejections=0, - initial_job_timeout_disconnects=0, - initial_job_cancelled_tasks=0, - initial_job_coalesced_tasks=0, - initial_job_queue_capacity_reclaimed=0, - handler_threads=0, - delivery_executor_queue_depth=0, - delivery_executor_active_workers=0, - started_monotonic=0.0, - startup_grace_seconds=0.0, - stale_unknown_rejections=0, - submitted_shares=0, - job_preparation_pending=False, - current_observed_tip=None, - prepared_bundle_current=False, - prepared_bundle_tip=None, - prepared_bundle_template_generation=None, - prepared_bundle_payout_generation=None, - ) - - -def incident_inputs() -> MiningDeliveryInputs: - """The 2026-08-31 fleet: 4 of 33 miners on the exact observed tip while - all 33 held semantically current work and nothing was genuinely pending.""" - - return replace( - healthy_inputs(), - active_connections=33, - subscribed_connections=33, - authorized_connections=33, - clients_with_current_tip_jobs=4, - clients_with_semantically_current_work=33, - last_initial_job_delivery_monotonic=99.5, - ) - - -class FakeObservabilityPort: - def __init__(self) -> None: - self.now = 100.0 - self.inputs = healthy_inputs() - self.progress: dict[str, object] = { - "ok": True, - "reason": None, - "reasons": [], - } - self.raise_on_stats = False - self.stats_calls = 0 - self.metrics_payload = "qbit_prism_fixture 1\n" - self.metrics_error: Exception | None = None - self.metrics_render_count = 0 - self.startup_phases: list[str] = [] - self.log_messages: list[str] = [] - self.exception_count = 0 - - def monotonic(self) -> float: - return self.now - - def mining_delivery_inputs(self, now: float) -> MiningDeliveryInputs: - self.assert_current_time(now) - return self.inputs - - def assert_current_time(self, now: float) -> None: - if now != self.now: - raise AssertionError(f"unexpected monotonic time: {now}") - - def accepted_share_stats(self) -> tuple[int, int]: - self.stats_calls += 1 - if self.raise_on_stats: - raise RuntimeError("stats unavailable") - return 3, 2 - - def ledger_backend(self) -> str: - return "memory" - - def block_counts(self) -> tuple[int, int]: - return 1, 2 - - def progress_health(self) -> dict[str, object]: - return dict(self.progress) - - def health_refresh_seconds(self) -> float: - return 1.0 - - def render_metrics_payload(self) -> str: - self.metrics_render_count += 1 - if self.metrics_error is not None: - raise self.metrics_error - return self.metrics_payload - - def metrics_refresh_seconds(self) -> float: - return 1.0 - - def record_startup_phase(self, phase: str) -> None: - self.startup_phases.append(phase) - - def stop_requested(self) -> bool: - return False - - def wait_for_stop(self, timeout: float) -> bool: - if timeout != 1.0: - raise AssertionError(f"unexpected wait: {timeout}") - return True - - def log(self, message: str) -> None: - self.log_messages.append(message) - - def log_exception(self) -> None: - self.exception_count += 1 - - -class ObservabilityServiceTests(unittest.TestCase): - def test_cached_base_health_uses_fresh_progress_overlay(self) -> None: - port = FakeObservabilityPort() - service = ObservabilityService(port) - service.refresh_health_snapshot() - self.assertNotIn("reason", service.state().health_snapshot or {}) - - port.progress = { - "ok": False, - "reason": "tip_poll_stale", - "reasons": ["tip_poll_stale"], - } - status, payload = service.cached_health_payload() - - self.assertEqual(status, 503) - self.assertFalse(payload["ok"]) - self.assertEqual(payload["reason"], "tip_poll_stale") - - def test_refresh_stamps_health_snapshot_warm_startup_phase(self) -> None: - port = FakeObservabilityPort() - service = ObservabilityService(port) - - service.refresh_health_snapshot() - - self.assertEqual(port.startup_phases, ["health_snapshot_warm"]) - - def test_fresh_progress_cannot_mask_cached_base_failure(self) -> None: - port = FakeObservabilityPort() - port.inputs = replace( - port.inputs, - subscribed_connections=1, - authorized_connections=1, - oldest_genuinely_pending_initial_job_age_seconds=6.0, - ) - service = ObservabilityService(port) - service.refresh_health_snapshot() - - port.inputs = healthy_inputs() - status, payload = service.cached_health_payload() - - self.assertEqual(status, 503) - self.assertFalse(payload["ok"]) - self.assertIn("initial-delivery-stalled", payload["unhealthy_reasons"]) - - def test_running_loop_without_snapshot_reports_starting_state(self) -> None: - # Upstream #120: while the background warm-up runs, the endpoint - # must fail closed with an explicit starting state instead of - # collecting ledger aggregates inline on the handler thread. - port = FakeObservabilityPort() - service = ObservabilityService(port) - self.assertTrue(service.begin_refresh_loop()) - - status, payload = service.cached_health_payload() - - self.assertEqual(status, 503) - self.assertFalse(payload["ok"]) - self.assertEqual(payload["state"], "starting") - self.assertEqual(payload["schema"], HEALTH_SCHEMA) - self.assertEqual(port.stats_calls, 0) - - def test_stale_snapshot_fails_closed_with_current_progress(self) -> None: - port = FakeObservabilityPort() - service = ObservabilityService(port) - service.refresh_health_snapshot() - port.now += 16.0 - port.progress = { - "ok": False, - "reason": "refresh_pending_too_long", - "reasons": ["refresh_pending_too_long"], - } - - status, payload = service.cached_health_payload() - - self.assertEqual(status, 503) - self.assertFalse(payload["ok"]) - self.assertEqual(payload["schema"], HEALTH_SCHEMA) - self.assertEqual(payload["error"], "health snapshot is stale") - self.assertEqual(payload["reason"], "refresh_pending_too_long") - - def test_delivery_timers_recover_after_sustained_overload(self) -> None: - port = FakeObservabilityPort() - port.inputs = replace( - port.inputs, - active_connections=1, - subscribed_connections=1, - authorized_connections=1, - pending_initial_jobs=32, - ) - service = ObservabilityService(port) - - self.assertTrue(service.mining_delivery_snapshot()["mining_ready"]) - port.now += 6.0 - stalled = service.mining_delivery_snapshot() - self.assertFalse(stalled["mining_ready"]) - self.assertEqual( - stalled["unhealthy_reasons"], - [ - "initial-delivery-stalled", - "pending-initial-jobs-saturated", - ], - ) - - port.inputs = replace( - port.inputs, - pending_initial_jobs=0, - clients_with_current_tip_jobs=1, - clients_with_semantically_current_work=1, - ) - recovered = service.mining_delivery_snapshot() - self.assertTrue(recovered["mining_ready"]) - state = service.state() - self.assertIsNone(state.mining_overload_started_monotonic) - self.assertIsNone(state.mining_delivery_failure_started_monotonic) - self.assertIsNone(state.mining_semantic_work_gap_started_monotonic) - - def test_semantic_gauges_report_count_and_six_decimal_ratio(self) -> None: - port = FakeObservabilityPort() - port.inputs = replace( - port.inputs, - active_connections=3, - subscribed_connections=3, - authorized_connections=3, - clients_with_current_tip_jobs=3, - clients_with_semantically_current_work=1, - ) - service = ObservabilityService(port) - - snapshot = service.mining_delivery_snapshot() - - self.assertEqual(snapshot["clients_with_semantically_current_work"], 1) - self.assertEqual(snapshot["semantic_current_work_ratio"], round(1 / 3, 6)) - - def test_semantic_ratio_is_one_with_zero_authorized_clients(self) -> None: - port = FakeObservabilityPort() - service = ObservabilityService(port) - - snapshot = service.mining_delivery_snapshot() - - self.assertEqual(snapshot["clients_with_semantically_current_work"], 0) - self.assertEqual(snapshot["semantic_current_work_ratio"], 1.0) - self.assertTrue(snapshot["mining_ready"]) - - def test_delivery_timer_observations_are_applied_in_capture_order(self) -> None: - first_captured = threading.Event() - release_first = threading.Event() - second_captured = threading.Event() - - class OrderedPort(FakeObservabilityPort): - def mining_delivery_inputs(self, now: float) -> MiningDeliveryInputs: - if now == 100.0: - first_captured.set() - if not release_first.wait(1.0): - raise AssertionError("first health capture was not released") - return replace( - self.inputs, - authorized_connections=1, - clients_with_current_tip_jobs=0, - ) - second_captured.set() - return replace( - self.inputs, - authorized_connections=1, - clients_with_current_tip_jobs=1, - ) - - port = OrderedPort() - service = ObservabilityService(port) - snapshots: list[dict[str, object]] = [] - first = threading.Thread( - target=lambda: snapshots.append( - service.mining_delivery_snapshot(now=100.0) - ) - ) - second = threading.Thread( - target=lambda: snapshots.append( - service.mining_delivery_snapshot(now=101.0) - ) - ) - first.start() - self.assertTrue(first_captured.wait(1.0)) - second.start() - self.assertFalse(second_captured.wait(0.05)) - release_first.set() - for thread in (first, second): - thread.join(1.0) - self.assertFalse(thread.is_alive()) - - self.assertEqual(len(snapshots), 2) - self.assertTrue(second_captured.is_set()) - state = service.state() - self.assertIsNone(state.mining_overload_started_monotonic) - self.assertIsNone(state.mining_delivery_failure_started_monotonic) - - def test_refresh_loop_counts_failure_and_keeps_one_shot_latch(self) -> None: - port = FakeObservabilityPort() - port.raise_on_stats = True - service = ObservabilityService(port) - self.assertTrue(service.begin_refresh_loop()) - self.assertFalse(service.begin_refresh_loop()) - - service.health_snapshot_loop() - - state = service.state() - # Upstream #120: the running flag is a one-shot latch. Loop exit must - # not re-open the legacy inline collection path for late requests. - self.assertTrue(state.health_refresh_loop_running) - self.assertEqual(state.health_snapshot_refresh_failure_count, 1) - self.assertEqual( - port.log_messages, - ["prism coordinator: health snapshot refresh failed"], - ) - self.assertEqual(port.exception_count, 1) - - def test_metrics_snapshot_preserves_renderer_bytes_and_adds_diagnostics(self) -> None: - port = FakeObservabilityPort() - service = ObservabilityService(port) - - rendered = service.refresh_metrics_snapshot() - status, cached = service.cached_metrics_payload() - - self.assertEqual(rendered, port.metrics_payload) - self.assertEqual(status, 200) - self.assertTrue(cached.startswith(port.metrics_payload)) - self.assertEqual(port.metrics_render_count, 1) - self.assertIn("qbit_prism_metrics_snapshot_available 1\n", cached) - self.assertIn("qbit_prism_metrics_snapshot_stale 0\n", cached) - self.assertIn("qbit_prism_metrics_snapshot_generation 1\n", cached) - - def test_metrics_failure_preserves_prior_complete_generation(self) -> None: - port = FakeObservabilityPort() - service = ObservabilityService(port) - service.refresh_metrics_snapshot() - port.metrics_payload = "partial" - - with self.assertRaises(ValueError): - service.refresh_metrics_snapshot() - - state = service.metrics_state() - self.assertEqual(state.metrics_snapshot, "qbit_prism_fixture 1\n") - self.assertEqual(state.metrics_collection_generation, 1) - self.assertEqual(state.metrics_collection_failure_count, 1) - self.assertEqual(state.metrics_failure_invalid_payload_count, 1) - status, cached = service.cached_metrics_payload() - self.assertEqual(status, 200) - self.assertTrue(cached.startswith("qbit_prism_fixture 1\n")) - - port.now += 16.0 - response = service.cached_metrics_payload() - - # The failed refresh left the last complete generation in place and - # #184 keeps serving it. Refusing here discarded that payload along - # with the failure counters that explain it, because Prometheus drops - # the body of a non-200 response. - self.assertEqual(response.status, 200) - self.assertEqual(response.state, METRICS_STATE_STALE) - self.assertTrue(response.body.startswith("qbit_prism_fixture 1\n")) - self.assertIn("qbit_prism_metrics_snapshot_stale 1\n", response.body) - self.assertIn( - 'qbit_prism_metrics_collection_failures_total{class="invalid_payload"} 1', - response.body, - ) - - def test_metrics_without_snapshot_fails_closed_without_collecting(self) -> None: - port = FakeObservabilityPort() - service = ObservabilityService(port) - - response = service.cached_metrics_payload() - - # Warm-up is the one case that still refuses: there is no complete - # payload to serve, so the body is diagnostics only and a scraper must - # not store it as this process's metrics. - self.assertEqual(response.status, 503) - self.assertEqual(response.state, METRICS_STATE_UNAVAILABLE) - self.assertIsNone(response.age_seconds) - self.assertEqual(port.metrics_render_count, 0) - self.assertIn("qbit_prism_metrics_snapshot_available 0\n", response.body) - self.assertIn( - "qbit_prism_metrics_snapshot_age_seconds -1.000\n", - response.body, - ) - self.assertNotIn("qbit_prism_fixture", response.body) - - def test_stale_complete_snapshot_is_served_as_200_with_truthful_age( - self, - ) -> None: - port = FakeObservabilityPort() - service = ObservabilityService(port) - service.refresh_metrics_snapshot() - - port.now += 47.5 - - response = service.cached_metrics_payload() - - self.assertEqual(response.status, 200) - self.assertEqual(response.state, METRICS_STATE_STALE) - # Byte-for-byte the generation that was published, with the current - # diagnostics appended -- nothing re-rendered on this thread. - self.assertTrue(response.body.startswith(port.metrics_payload)) - self.assertEqual(port.metrics_render_count, 1) - self.assertIn("qbit_prism_metrics_snapshot_available 1\n", response.body) - self.assertIn("qbit_prism_metrics_snapshot_stale 1\n", response.body) - self.assertIn( - "qbit_prism_metrics_snapshot_age_seconds 47.500\n", - response.body, - ) - # Floored, not rounded: Age must never overstate freshness. - self.assertEqual(response.age_seconds, 47) - - def test_blocked_refresh_past_budget_serves_prior_document_promptly( - self, - ) -> None: - port = FakeObservabilityPort() - service = ObservabilityService(port) - service.refresh_metrics_snapshot() - entered = threading.Event() - release = threading.Event() - render_calls: list[str] = [] - - def blocked_render() -> str: - render_calls.append("render") - entered.set() - release.wait(5.0) - return "qbit_prism_fixture 2\n" - - port.render_metrics_payload = blocked_render # type: ignore[method-assign] - collector = threading.Thread(target=service.refresh_metrics_snapshot) - collector.start() - try: - self.assertTrue(entered.wait(2.0)) - # Far past the staleness budget, with the collector still wedged - # inside the renderer holding the collection lock. - port.now += 600.0 - - started = time.monotonic() - response = service.cached_metrics_payload() - elapsed = time.monotonic() - started - finally: - release.set() - collector.join(5.0) - - # Never waited on the stuck collector, and never re-entered the - # renderer itself: the only render is the collector's own. - self.assertLess(elapsed, 1.0) - self.assertEqual(render_calls, ["render"]) - self.assertEqual(response.status, 200) - self.assertEqual(response.state, METRICS_STATE_STALE) - self.assertTrue(response.body.startswith("qbit_prism_fixture 1\n")) - self.assertEqual(response.age_seconds, 600) - self.assertFalse(collector.is_alive()) - - def test_metrics_response_metadata_is_exact_and_bounded(self) -> None: - port = FakeObservabilityPort() - service = ObservabilityService(port) - - unavailable = service.cached_metrics_payload() - service.refresh_metrics_snapshot() - port.now += 2.0 - fresh = service.cached_metrics_payload() - port.now += 20.0 - stale = service.cached_metrics_payload() - - self.assertEqual( - unavailable.response_headers(), - { - "Cache-Control": "no-store", - METRICS_STATE_HEADER: "unavailable", - }, - ) - self.assertEqual( - fresh.response_headers(), - { - "Cache-Control": "no-store", - METRICS_STATE_HEADER: "fresh", - "Age": "2", - }, - ) - self.assertEqual( - stale.response_headers(), - { - "Cache-Control": "no-store", - METRICS_STATE_HEADER: "stale", - "Age": "22", - "Warning": METRICS_STALE_WARNING, - }, - ) - # Bounded vocabulary, and the warn-code line is the registered 110. - self.assertEqual(METRICS_STATES, ("fresh", "stale", "unavailable")) - for response in (unavailable, fresh, stale): - self.assertIn(response.state, METRICS_STATES) - self.assertEqual( - METRICS_STALE_WARNING, - '110 qbit-prism "metrics snapshot is stale; ' - 'serving last complete payload"', - ) - # Still destructures as the pair every pre-#184 caller unpacks. - status, body = fresh - self.assertEqual((status, body), (fresh.status, fresh.body)) - self.assertEqual(fresh.state, METRICS_STATE_FRESH) - - def test_metrics_exception_is_bounded_and_recovery_replaces_snapshot( - self, - ) -> None: - port = FakeObservabilityPort() - service = ObservabilityService(port) - service.refresh_metrics_snapshot() - port.metrics_error = RuntimeError("dynamic backend detail") - - with self.assertRaises(RuntimeError): - service.refresh_metrics_snapshot() - - state = service.metrics_state() - self.assertEqual(state.metrics_failure_exception_count, 1) - self.assertEqual(state.metrics_last_failure_class, "exception") - _, failed = service.cached_metrics_payload() - self.assertNotIn("dynamic backend detail", failed) - - port.metrics_error = None - port.metrics_payload = "qbit_prism_fixture 2\n" - service.refresh_metrics_snapshot() - - recovered = service.metrics_state() - self.assertEqual(recovered.metrics_collection_generation, 2) - self.assertEqual(recovered.metrics_collection_success_count, 2) - self.assertEqual(recovered.metrics_collection_failure_count, 1) - self.assertIsNone(recovered.metrics_last_failure_class) - status, cached = service.cached_metrics_payload() - self.assertEqual(status, 200) - self.assertTrue(cached.startswith("qbit_prism_fixture 2\n")) - self.assertIn( - 'qbit_prism_metrics_collection_failures_total{class="exception"} 1', - cached, - ) - - def test_slow_metrics_collection_does_not_block_or_partially_replace_cache( - self, - ) -> None: - port = FakeObservabilityPort() - service = ObservabilityService(port) - service.refresh_metrics_snapshot() - entered = threading.Event() - release = threading.Event() - - def slow_render() -> str: - entered.set() - release.wait(2.0) - return "qbit_prism_fixture 2\n" - - port.render_metrics_payload = slow_render # type: ignore[method-assign] - collector = threading.Thread(target=service.refresh_metrics_snapshot) - collector.start() - self.assertTrue(entered.wait(1.0)) - - status, during = service.cached_metrics_payload() - - self.assertEqual(status, 200) - self.assertTrue(during.startswith("qbit_prism_fixture 1\n")) - release.set() - collector.join(2.0) - self.assertFalse(collector.is_alive()) - _, after = service.cached_metrics_payload() - self.assertTrue(after.startswith("qbit_prism_fixture 2\n")) - - def test_metrics_and_health_loop_state_are_independent(self) -> None: - port = FakeObservabilityPort() - service = ObservabilityService(port) - - self.assertTrue(service.begin_refresh_loop()) - self.assertTrue(service.begin_metrics_refresh_loop()) - self.assertTrue(service.state().health_refresh_loop_running) - self.assertTrue(service.metrics_state().metrics_refresh_loop_running) - - service.metrics_snapshot_loop() - - self.assertTrue(service.state().health_refresh_loop_running) - self.assertFalse(service.metrics_state().metrics_refresh_loop_running) - - -class SemanticCoverageDeliveryHealthTests(unittest.TestCase): - """Issue #216: the delivery stall is timed on semantic coverage.""" - - def test_rapid_tip_churn_with_complete_semantic_coverage_stays_ready( - self, - ) -> None: - port = FakeObservabilityPort() - port.inputs = incident_inputs() - service = ObservabilityService(port) - - # Exact coverage churns below 95% for four times the 5s deadline - # while every miner keeps semantically current work, nothing is - # genuinely pending, and deliveries keep landing. The refresher's - # cached /healthz answer must hold 200 on every sample, not flap. - for step, exact in enumerate((4, 12, 1, 20, 7, 30)): - port.now = 100.0 + 4.0 * step - port.inputs = replace( - port.inputs, - clients_with_current_tip_jobs=exact, - last_initial_job_delivery_monotonic=port.now - 0.5, - ) - service.refresh_health_snapshot() - status, payload = service.cached_health_payload() - - self.assertEqual(status, 200, payload) - self.assertTrue(payload["ok"]) - self.assertTrue(payload["mining_ready"]) - self.assertFalse(payload["initial_delivery_stalled"]) - self.assertEqual(payload["unhealthy_reasons"], []) - self.assertEqual(payload["pending_initial_jobs"], 0) - self.assertEqual( - payload["oldest_genuinely_pending_initial_job_age_seconds"], - 0.0, - ) - self.assertLess(payload["current_tip_job_coverage"], 0.95) - self.assertEqual(payload["semantic_current_work_ratio"], 1.0) - # The exact gap keeps reporting the churn as telemetry. - self.assertEqual( - payload["current_tip_coverage_gap_age_seconds"], - 4.0 * step, - ) - self.assertEqual(payload["semantic_current_work_gap_age_seconds"], 0.0) - - state = service.state() - self.assertEqual(state.mining_delivery_failure_started_monotonic, 100.0) - self.assertIsNone(state.mining_semantic_work_gap_started_monotonic) - - def test_genuine_first_job_starvation_fails_closed_after_deadline( - self, - ) -> None: - port = FakeObservabilityPort() - port.inputs = replace( - healthy_inputs(), - active_connections=3, - subscribed_connections=3, - authorized_connections=3, - pending_initial_jobs=3, - oldest_pending_initial_job_age_seconds=4.999, - oldest_genuinely_pending_initial_job_age_seconds=4.999, - clients_with_no_active_job=3, - ) - service = ObservabilityService(port) - - self.assertTrue(service.mining_delivery_snapshot()["mining_ready"]) - - # Only the genuine pending age crosses the deadline here: both - # coverage gaps are a millisecond old, so starvation fires on its own. - port.now += 0.001 - port.inputs = replace( - port.inputs, - oldest_pending_initial_job_age_seconds=5.0, - oldest_genuinely_pending_initial_job_age_seconds=5.0, - ) - starved = service.mining_delivery_snapshot() - - self.assertFalse(starved["mining_ready"]) - self.assertTrue(starved["initial_delivery_stalled"]) - self.assertIn("initial-delivery-stalled", starved["unhealthy_reasons"]) - self.assertEqual(starved["semantic_current_work_gap_age_seconds"], 0.001) - - def test_sustained_semantic_work_loss_fails_closed_after_deadline( - self, - ) -> None: - # Every miner holds work, so nothing is genuinely pending, but none - # of it matches the current template content and none is on the - # observed tip: fanout has stopped. This must still fail closed. - port = FakeObservabilityPort() - port.inputs = replace( - healthy_inputs(), - active_connections=4, - subscribed_connections=4, - authorized_connections=4, - clients_with_current_tip_jobs=0, - clients_with_semantically_current_work=0, - ) - service = ObservabilityService(port) - - self.assertTrue(service.mining_delivery_snapshot()["mining_ready"]) - port.now = 104.999 - grace = service.mining_delivery_snapshot() - self.assertTrue(grace["mining_ready"]) - self.assertEqual(grace["semantic_current_work_gap_age_seconds"], 4.999) - self.assertEqual(grace["current_tip_coverage_gap_age_seconds"], 4.999) - - port.now = 105.0 - stalled = service.mining_delivery_snapshot() - self.assertFalse(stalled["mining_ready"]) - self.assertTrue(stalled["initial_delivery_stalled"]) - self.assertEqual(stalled["unhealthy_reasons"], ["initial-delivery-stalled"]) - self.assertEqual( - stalled["oldest_genuinely_pending_initial_job_age_seconds"], - 0.0, - ) - - # Semantic recovery alone ends the stall and closes the semantic gap - # while the strict gauge is still lagging the observed tip. - port.now = 106.0 - port.inputs = replace(port.inputs, clients_with_semantically_current_work=4) - recovered = service.mining_delivery_snapshot() - self.assertTrue(recovered["mining_ready"]) - self.assertFalse(recovered["initial_delivery_stalled"]) - self.assertEqual(recovered["semantic_current_work_gap_age_seconds"], 0.0) - self.assertEqual(recovered["current_tip_coverage_gap_age_seconds"], 6.0) - state = service.state() - self.assertIsNone(state.mining_semantic_work_gap_started_monotonic) - self.assertEqual(state.mining_delivery_failure_started_monotonic, 100.0) - - def test_semantic_gap_must_be_sustained_not_sampled(self) -> None: - # A long-running exact gap plus one poor semantic sample is not a - # stall: the semantic gap has its own timer and must itself last a - # full deadline before health fails. - port = FakeObservabilityPort() - port.inputs = incident_inputs() - service = ObservabilityService(port) - service.mining_delivery_snapshot() - - port.now = 120.0 - port.inputs = replace(port.inputs, clients_with_semantically_current_work=20) - dip = service.mining_delivery_snapshot() - self.assertTrue(dip["mining_ready"]) - self.assertEqual(dip["current_tip_coverage_gap_age_seconds"], 20.0) - self.assertEqual(dip["semantic_current_work_gap_age_seconds"], 0.0) - - port.now = 124.0 - port.inputs = replace(port.inputs, clients_with_semantically_current_work=33) - back = service.mining_delivery_snapshot() - self.assertTrue(back["mining_ready"]) - self.assertIsNone(service.state().mining_semantic_work_gap_started_monotonic) - - port.now = 128.0 - port.inputs = replace(port.inputs, clients_with_semantically_current_work=20) - self.assertTrue(service.mining_delivery_snapshot()["mining_ready"]) - port.now = 132.999 - self.assertTrue(service.mining_delivery_snapshot()["mining_ready"]) - port.now = 133.0 - stalled = service.mining_delivery_snapshot() - self.assertFalse(stalled["mining_ready"]) - self.assertEqual(stalled["unhealthy_reasons"], ["initial-delivery-stalled"]) - self.assertEqual(stalled["semantic_current_work_gap_age_seconds"], 5.0) - - def test_exact_tip_work_without_semantic_evidence_keeps_health_ready( - self, - ) -> None: - # An embedder that publishes only the observed tip (no template - # snapshot) reads semantic coverage 0 for every client while the - # strict gauge is complete. In production strict currency is checked - # against the same fingerprint and payout generation, so this state - # means "no semantic evidence", never "semantically stale", and the - # strict gauge must still count as delivered work. - port = FakeObservabilityPort() - port.inputs = replace( - healthy_inputs(), - active_connections=4, - subscribed_connections=4, - authorized_connections=4, - clients_with_current_tip_jobs=4, - clients_with_semantically_current_work=0, - ) - service = ObservabilityService(port) - - service.mining_delivery_snapshot() - port.now += 10.0 - payload = service.mining_delivery_snapshot() - - self.assertTrue(payload["mining_ready"]) - self.assertFalse(payload["initial_delivery_stalled"]) - self.assertEqual(payload["semantic_current_work_gap_age_seconds"], 10.0) - self.assertEqual(payload["current_tip_coverage_gap_age_seconds"], 0.0) - - def test_incident_payload_retains_exact_and_semantic_telemetry(self) -> None: - port = FakeObservabilityPort() - port.inputs = incident_inputs() - service = ObservabilityService(port) - service.mining_delivery_snapshot() - - port.now += 6.0 - payload = service.mining_delivery_snapshot() - - self.assertEqual(payload["authorized_connections"], 33) - self.assertEqual(payload["clients_with_current_tip_jobs"], 4) - self.assertEqual(payload["current_tip_job_coverage"], 0.121212) - self.assertEqual(payload["clients_with_semantically_current_work"], 33) - self.assertEqual(payload["semantic_current_work_ratio"], 1.0) - self.assertEqual(payload["pending_initial_jobs"], 0) - self.assertEqual( - payload["oldest_genuinely_pending_initial_job_age_seconds"], - 0.0, - ) - self.assertEqual(payload["clients_with_no_active_job"], 0) - self.assertEqual(payload["current_tip_coverage_gap_age_seconds"], 6.0) - self.assertEqual(payload["semantic_current_work_gap_age_seconds"], 0.0) - # Compatibility aliases keep their exact-tip meaning. - self.assertEqual(payload["clients_with_current_tip_job"], 4) - self.assertEqual(payload["clients_without_current_tip_job"], 29) - self.assertEqual(payload["current_tip_job_coverage_ratio"], 4 / 33) - self.assertEqual(payload["oldest_initial_job_pending_seconds"], 0.0) - self.assertTrue(payload["mining_ready"]) - self.assertTrue(payload["mining_delivery_healthy"]) - self.assertFalse(payload["initial_delivery_stalled"]) - self.assertFalse(payload["overload"]) - self.assertEqual(payload["unhealthy_reasons"], []) - - def test_overload_now_keeps_using_exact_coverage(self) -> None: - port = FakeObservabilityPort() - port.inputs = replace( - incident_inputs(), - active_connections=64, - subscribed_connections=64, - authorized_connections=64, - clients_with_current_tip_jobs=8, - clients_with_semantically_current_work=64, - ) - service = ObservabilityService(port) - - strict_pressure = service.mining_delivery_snapshot() - - self.assertTrue(strict_pressure["connection_capacity_saturated"]) - self.assertTrue(strict_pressure["overload"]) - self.assertTrue(strict_pressure["mining_ready"]) - self.assertEqual( - service.state().mining_overload_started_monotonic, - 100.0, - ) - - # Semantic loss with complete exact coverage is not overload. - port.now += 1.0 - port.inputs = replace( - port.inputs, - clients_with_current_tip_jobs=64, - clients_with_semantically_current_work=0, - ) - semantic_only = service.mining_delivery_snapshot() - - self.assertTrue(semantic_only["connection_capacity_saturated"]) - self.assertFalse(semantic_only["overload"]) - self.assertEqual(semantic_only["overload_age_seconds"], 0.0) - self.assertIsNone(service.state().mining_overload_started_monotonic) - - def test_reject_storm_keeps_using_exact_coverage(self) -> None: - port = FakeObservabilityPort() - port.inputs = replace( - incident_inputs(), - submitted_shares=100, - stale_unknown_rejections=95, - ) - service = ObservabilityService(port) - - storm = service.mining_delivery_snapshot() - - self.assertTrue(storm["overload"]) - self.assertFalse(storm["mining_ready"]) - self.assertEqual( - storm["unhealthy_reasons"], - ["stale-unknown-rejection-storm"], - ) - - # The same rejection ratio with complete exact coverage and no - # semantic coverage is not a storm: the semantic gauge does not - # corroborate it. - port.now += 1.0 - port.inputs = replace( - port.inputs, - clients_with_current_tip_jobs=33, - clients_with_semantically_current_work=0, - ) - quiet = service.mining_delivery_snapshot() - - self.assertFalse(quiet["overload"]) - self.assertTrue(quiet["mining_ready"]) - self.assertEqual(quiet["unhealthy_reasons"], []) - - def test_cap_saturated_reason_keeps_using_exact_coverage(self) -> None: - port = FakeObservabilityPort() - port.inputs = replace( - incident_inputs(), - active_connections=64, - subscribed_connections=64, - authorized_connections=64, - clients_with_current_tip_jobs=8, - clients_with_semantically_current_work=64, - ) - service = ObservabilityService(port) - service.mining_delivery_snapshot() - - port.now += 5.0 - persistent = service.mining_delivery_snapshot() - - # Strict-tip pressure under a full cap is surfaced on its own; the - # delivery stall does not ride along on the exact gap. - self.assertFalse(persistent["mining_ready"]) - self.assertEqual( - persistent["unhealthy_reasons"], - ["connection-capacity-saturated"], - ) - self.assertFalse(persistent["initial_delivery_stalled"]) - self.assertEqual(persistent["current_tip_coverage_gap_age_seconds"], 5.0) - - # Exact coverage restored under the same full cap clears the reason - # even though semantic coverage is now gone. - port.now += 5.0 - port.inputs = replace( - port.inputs, - clients_with_current_tip_jobs=64, - clients_with_semantically_current_work=0, - ) - cleared = service.mining_delivery_snapshot() - - self.assertTrue(cleared["connection_capacity_saturated"]) - self.assertTrue(cleared["mining_ready"]) - self.assertEqual(cleared["unhealthy_reasons"], []) - - -def bare_semantic_coordinator() -> PrismCoordinator: - server = PrismCoordinator.__new__(PrismCoordinator) - server.lock = threading.RLock() - server.stop_event = threading.Event() - server.clients = set() - server.jobs = {} - server.connection_counter = 0 - server.job_counter = 0 - server.stratum_max_connections = 8 - server.stratum_max_connections_per_username = 0 - server.stratum_max_pending_initial_jobs = 4 - server.stratum_initial_job_timeout_seconds = 30.0 - server.mining_health_startup_grace_seconds = 30.0 - server.tip_refresh_max_workers = 1 - server.tip_template_snapshot = None - server.started_monotonic = time.monotonic() - server.submitted_share_count = 0 - server.rejection_counts_by_reason = {} - server._ensure_job_cache_state() - server._ensure_tip_refresh_state() - server._ensure_initial_job_state() - return server - - -def semantic_client(server: PrismCoordinator, connection_id: int) -> ClientState: - state = ClientState( - sock=SimpleNamespace(), - address=("127.0.0.1", 40_000 + connection_id), - connection_id=connection_id, - extranonce1_hex=f"{connection_id:08x}", - ) - state.subscribed = True - state.authorized = True - state.authorization_generation = 1 - state.worker = WorkerIdentity( - username=f"miner-{connection_id}", - payout_address=f"miner-{connection_id}", - worker_name=None, - script_pubkey_hex="5220" + "11" * 32, - p2mr_program_hex="11" * 32, - ) - state.username = state.worker.username - server.clients.add(state) - return state - - -class SemanticCurrencyAdapterTests(unittest.TestCase): - def test_template_generation_and_identity_do_not_affect_the_count( - self, - ) -> None: - # Upstream #107: template generation and template object identity do - # not participate in semantic currency. Client-session generations do, - # and the strict fail-closed gauge stays untouched by semantic matches. - server = bare_semantic_coordinator() - fingerprint = "ff" * 32 - server.tip_template_snapshot = SimpleNamespace( - bestblockhash="aa" * 32, - template_fingerprint=fingerprint, - template_generation=7, - template_artifacts=None, - ) - matching = semantic_client(server, 1) - matching.active_job = SimpleNamespace( - template={"previousblockhash": "aa" * 32}, - template_fingerprint=fingerprint, - # Deliberately different template generation and a distinct - # template object: neither may disqualify semantic currency. - template_generation=3, - payout_state_generation=0, - connection_id=matching.connection_id, - authorization_generation=matching.authorization_generation, - difficulty_generation=matching.difficulty_generation, - ) - mismatched = semantic_client(server, 2) - mismatched.active_job = SimpleNamespace( - template={"previousblockhash": "aa" * 32}, - template_fingerprint="ee" * 32, - template_generation=7, - payout_state_generation=0, - connection_id=mismatched.connection_id, - authorization_generation=mismatched.authorization_generation, - difficulty_generation=mismatched.difficulty_generation, - ) - - snapshot = server.mining_delivery_snapshot() - - self.assertEqual(snapshot["clients_with_semantically_current_work"], 1) - self.assertEqual(snapshot["semantic_current_work_ratio"], 0.5) - self.assertEqual(snapshot["clients_with_current_tip_jobs"], 0) - self.assertEqual(snapshot["current_tip_job_coverage"], 0.0) - self.assertTrue(snapshot["mining_ready"]) - - def test_stale_payout_generation_disqualifies_semantic_currency(self) -> None: - server = bare_semantic_coordinator() - fingerprint = "ff" * 32 - server.tip_template_snapshot = SimpleNamespace( - bestblockhash="aa" * 32, - template_fingerprint=fingerprint, - template_generation=7, - template_artifacts=None, - ) - stale = semantic_client(server, 1) - stale.active_job = SimpleNamespace( - template={"previousblockhash": "aa" * 32}, - template_fingerprint=fingerprint, - template_generation=7, - payout_state_generation=99, - connection_id=stale.connection_id, - authorization_generation=stale.authorization_generation, - difficulty_generation=stale.difficulty_generation, - ) - - snapshot = server.mining_delivery_snapshot() - - self.assertEqual(snapshot["clients_with_semantically_current_work"], 0) - self.assertEqual(snapshot["semantic_current_work_ratio"], 0.0) - - def test_session_generation_changes_disqualify_semantic_currency(self) -> None: - server = bare_semantic_coordinator() - fingerprint = "ff" * 32 - server.tip_template_snapshot = SimpleNamespace( - bestblockhash="aa" * 32, - template_fingerprint=fingerprint, - template_generation=7, - template_artifacts=None, - ) - reauthorized = semantic_client(server, 1) - reauthorized.active_job = SimpleNamespace( - template={"previousblockhash": "aa" * 32}, - template_fingerprint=fingerprint, - payout_state_generation=0, - connection_id=reauthorized.connection_id, - authorization_generation=reauthorized.authorization_generation, - difficulty_generation=reauthorized.difficulty_generation, - ) - retargeted = semantic_client(server, 2) - retargeted.active_job = SimpleNamespace( - template={"previousblockhash": "aa" * 32}, - template_fingerprint=fingerprint, - payout_state_generation=0, - connection_id=retargeted.connection_id, - authorization_generation=retargeted.authorization_generation, - difficulty_generation=retargeted.difficulty_generation, - ) - - reauthorized.authorization_generation += 1 - retargeted.difficulty_generation += 1 - snapshot = server.mining_delivery_snapshot() - - self.assertEqual(snapshot["clients_with_semantically_current_work"], 0) - self.assertEqual(snapshot["semantic_current_work_ratio"], 0.0) - - -if __name__ == "__main__": - unittest.main() diff --git a/tests/test_prism_payout_state.py b/tests/test_prism_payout_state.py deleted file mode 100644 index 93ba78546..000000000 --- a/tests/test_prism_payout_state.py +++ /dev/null @@ -1,6043 +0,0 @@ -#!/usr/bin/env python3 -"""Focused PRISM coordinator payout state tests.""" -# ruff: noqa: F403, F405 - -from __future__ import annotations - -import json -import unittest - -from lab.prism.payout_state import AcceptedParentPayoutPreviewPending -from lab.prism.payout_state import _snapshot_window_weight_within_band -from lab.prism.payout_state import PRISM_PAYOUT_WINDOW_BUILD_OUTCOMES -from lab.prism.payout_state import PRISM_PAYOUT_WINDOW_BUILD_PHASES -from lab.prism.payout_state import PRISM_REWARD_WINDOW_MULTIPLIER -from lab.prism.payout_state import PRISM_SNAPSHOT_WINDOW_MARGIN -from lab.prism.payout_state import PRISM_TIP_REFRESH_WINDOW_STALENESS_SECONDS -from tests.prism_coordinator_test_support import * - - -class JobBundleCacheTests(unittest.TestCase): - def test_child_bundle_waits_for_pending_parent_preview_without_confirmed_read( - self, - ) -> None: - class PreviewLedger(FakeLedger): - def __init__(self) -> None: - super().__init__() - self.current_balance_reads = 0 - - def current_prior_balances(self) -> list[dict[str, object]]: - self.current_balance_reads += 1 - return [] - - class ObservedCondition(threading.Condition): - def __init__(self) -> None: - super().__init__() - self.wait_entered = threading.Event() - - def wait(self, timeout: float | None = None) -> bool: - self.wait_entered.set() - return super().wait(timeout) - - ledger = PreviewLedger() - server, rpc = coordinator(ledger=ledger) - recorded = install_fake_bundle_builder(server) - artifacts = server.store_template_artifacts(dict(rpc.template)) - self.assertIsNotNone(artifacts) - assert artifacts is not None - parent_hash = str(rpc.template["previousblockhash"]) - preview_condition = ObservedCondition() - server._accepted_block_payout_preview_condition = preview_condition - server.accepted_block_payout_preview_wait_seconds = 10 - preview = [ - { - "recipient_id": "miner-a", - "order_key": "miner-a", - "p2mr_program_hex": "11" * 32, - "balance_sats": 25, - } - ] - bundles: list[object] = [] - errors: list[BaseException] = [] - - def build_child_bundle() -> None: - try: - bundles.append(server.shared_job_bundle(artifacts, worker())) - except BaseException as exc: # pragma: no cover - asserted below - errors.append(exc) - - server._begin_accepted_block_payout_preview(parent_hash) - thread = threading.Thread(target=build_child_bundle, daemon=True) - thread.start() - try: - preview_wait_reached = preview_condition.wait_entered.wait(2) - waiting_for_preview = thread.is_alive() - confirmed_reads_while_pending = ledger.current_balance_reads - builds_while_pending = int(recorded["calls"]) - finally: - server._publish_accepted_block_payout_preview(parent_hash, preview) - thread.join(5) - server._clear_accepted_block_payout_preview(parent_hash) - - self.assertTrue(preview_wait_reached) - self.assertTrue(waiting_for_preview) - self.assertEqual(confirmed_reads_while_pending, 0) - self.assertEqual(builds_while_pending, 0) - self.assertFalse(thread.is_alive()) - self.assertEqual(errors, []) - self.assertEqual(len(bundles), 1) - bundle = bundles[0] - self.assertEqual(bundle.prior_balances, preview) # type: ignore[union-attr] - self.assertEqual(recorded["last_kwargs"]["prior_balances"], preview) # type: ignore[index] - # Preview publication prepares an artifact from one confirmed snapshot; - # the zero count captured above proves the waiting child did not read it. - self.assertEqual(ledger.current_balance_reads, 1) - self.assertEqual( # type: ignore[union-attr] - bundle.payout_state_generation, - server._payout_state_generation, - ) - - self.assertEqual(server._prior_balances_for_job_parent(parent_hash), []) - self.assertEqual(ledger.current_balance_reads, 2) - - def test_parent_preview_publication_is_idempotent_and_withdrawal_invalidates( - self, - ) -> None: - server, _rpc = coordinator() - parent_hash = "ab" * 32 - preview = [ - { - "recipient_id": "miner-a", - "order_key": "miner-a", - "p2mr_program_hex": "11" * 32, - "balance_sats": 25, - } - ] - - server._begin_accepted_block_payout_preview(parent_hash) - self.assertEqual( - server._publish_accepted_block_payout_preview(parent_hash, preview), - preview, - ) - self.assertEqual(server._payout_state_generation, 1) - - self.assertEqual( - server._publish_accepted_block_payout_preview(parent_hash, preview), - preview, - ) - self.assertEqual(server._payout_state_generation, 1) - with self.assertRaisesRegex(RuntimeError, "changed during retry"): - server._publish_accepted_block_payout_preview( - parent_hash, - [{**preview[0], "balance_sats": 26}], - ) - - server._clear_accepted_block_payout_preview( - parent_hash, - invalidate_published=True, - ) - self.assertEqual(server._payout_state_generation, 2) - self.assertEqual(server._accepted_block_payout_previews, {}) - self.assertEqual( - server._invalidated_accepted_block_payout_previews, - {parent_hash: None}, - ) - with self.assertRaisesRegex(TemplateRefreshBlocked, "was withdrawn"): - server._prior_balances_for_job_parent(parent_hash) - - server._begin_accepted_block_payout_preview(parent_hash) - self.assertEqual(server._invalidated_accepted_block_payout_previews, {}) - server._clear_accepted_block_payout_preview(parent_hash) - - def test_unpublished_parent_preview_retries_and_reopens_delivery(self) -> None: - server, _rpc = coordinator() - server.payout_reconcile_supersession_retries = 2 - parent_hash = "ac" * 32 - preview = [ - { - "recipient_id": "miner-a", - "order_key": "miner-a", - "p2mr_program_hex": "11" * 32, - "balance_sats": 25, - } - ] - - server._begin_accepted_block_payout_preview(parent_hash) - with patch.object( - server, - "_publish_payout_state_candidate", - return_value=None, - ) as publish_candidate: - self.assertEqual( - server._publish_accepted_block_payout_preview(parent_hash, preview), - preview, - ) - - transition = server._accepted_block_payout_previews[parent_hash] - self.assertEqual(publish_candidate.call_count, 3) - self.assertIsNotNone(transition.preview) - self.assertIsNone(transition.published_generation) - self.assertEqual(server._payout_state_generation, 0) - self.assertTrue(server._payout_state_publication_fenced()) - self.assertTrue(server._payout_state_delivery_gate._delivery_blocked) - - self.assertEqual( - server._publish_accepted_block_payout_preview(parent_hash, preview), - preview, - ) - - transition = server._accepted_block_payout_previews[parent_hash] - self.assertEqual(transition.published_generation, 1) - self.assertEqual(server._payout_state_generation, 1) - self.assertFalse(server._payout_state_publication_fenced()) - self.assertFalse(server._payout_state_delivery_gate._delivery_blocked) - - def test_ancestor_scan_probes_each_height_once(self) -> None: - # Every offered or replay-adopted candidate arms a preview barrier, and - # siblings of one solved block share a height -- so without dedupe this - # scan issues one getblockhash per barrier for a single answer. The - # 2026-08-20 load test reached 3,120 barriers at one height. - server, rpc = coordinator() - original_rpc_call = rpc.call - heights_probed: list[int] = [] - - def counting_call( - method: str, - params: list[object] | None = None, - ) -> object: - if method == "getblockhash": - assert params is not None - heights_probed.append(int(params[0])) - return "ff" * 32 - return original_rpc_call(method, params) - - rpc.call = counting_call # type: ignore[method-assign] - for index in range(50): - server._begin_accepted_block_payout_preview( - f"{index:064x}", - block_height=10, - ) - for index in range(50, 60): - server._begin_accepted_block_payout_preview( - f"{index:064x}", - block_height=11, - ) - - server._accepted_block_payout_transition_for_parent( - "ab" * 32, - parent_height=12, - ) - - # Two distinct heights, sixty barriers: two probes, not sixty. - self.assertEqual(sorted(heights_probed), [10, 11]) - - def test_ancestor_scan_dedupe_is_scoped_to_one_scan(self) -> None: - # The active block at a height changes under reorgs, so the memo must - # not outlive the scan that built it. - server, rpc = coordinator() - original_rpc_call = rpc.call - probe_count = 0 - - def counting_call( - method: str, - params: list[object] | None = None, - ) -> object: - nonlocal probe_count - if method == "getblockhash": - probe_count += 1 - return "ff" * 32 - return original_rpc_call(method, params) - - rpc.call = counting_call # type: ignore[method-assign] - server._begin_accepted_block_payout_preview("cd" * 32, block_height=10) - - for _ in range(3): - server._accepted_block_payout_transition_for_parent( - "ab" * 32, - parent_height=12, - ) - - self.assertEqual(probe_count, 3) - - def test_withdrawn_landed_transition_blocks_active_descendant_fallback( - self, - ) -> None: - class CountingLedger(FakeLedger): - def __init__(self) -> None: - super().__init__() - self.current_balance_reads = 0 - - def current_prior_balances(self) -> list[dict[str, object]]: - self.current_balance_reads += 1 - return [] - - ledger = CountingLedger() - server, rpc = coordinator(ledger=ledger) - accepted_hash = "bc" * 32 - descendant_hash = "bd" * 32 - original_rpc_call = rpc.call - - def active_chain_call( - method: str, - params: list[object] | None = None, - ) -> object: - if method == "getblockhash": - self.assertEqual(params, [10]) - return accepted_hash - return original_rpc_call(method, params) - - rpc.call = active_chain_call # type: ignore[method-assign] - server._begin_accepted_block_payout_preview( - accepted_hash, - block_height=10, - ) - server._mark_accepted_block_payout_landed( - accepted_hash, - block_height=10, - ) - server._clear_accepted_block_payout_preview( - accepted_hash, - invalidate_published=True, - ) - - with self.assertRaisesRegex(TemplateRefreshBlocked, "was withdrawn"): - server._prior_balances_for_job_parent( - descendant_hash, - parent_height=11, - ) - self.assertEqual(ledger.current_balance_reads, 0) - self.assertTrue(server._tip_refresh_retry.is_set()) - - def test_inactive_landed_ancestor_rejects_preview_patched_artifact( - self, - ) -> None: - accepted_hash = "c0" * 32 - alternate_tip = "c1" * 32 - server, rpc = coordinator( - template=base_template(height=12, prevhash=alternate_tip) - ) - recorded = install_fake_bundle_builder(server) - artifacts = server.store_template_artifacts(dict(rpc.template)) - assert artifacts is not None - self.assertTrue(server.pool_readiness_latched()) - preview = [ - { - "recipient_id": "miner-a", - "order_key": "miner-a", - "p2mr_program_hex": "11" * 32, - "balance_sats": 25, - } - ] - server._begin_accepted_block_payout_preview( - accepted_hash, - block_height=10, - ) - server._mark_accepted_block_payout_landed( - accepted_hash, - block_height=10, - ) - server._publish_accepted_block_payout_preview(accepted_hash, preview) - with server._job_cache_lock: - artifact = server._payout_ledger_artifact - self.assertIsNotNone(artifact) - assert artifact is not None - self.assertEqual(list(artifact.prior_balances), preview) - - original_rpc_call = rpc.call - - def alternate_chain_call( - method: str, - params: list[object] | None = None, - ) -> object: - if method == "getblockhash": - self.assertEqual(params, [10]) - return "c2" * 32 - return original_rpc_call(method, params) - - rpc.call = alternate_chain_call # type: ignore[method-assign] - - with self.assertRaisesRegex( - TemplateRefreshBlocked, - "no longer active", - ): - server.shared_job_bundle(artifacts, mode="ready") - - self.assertEqual(recorded["calls"], 0) - self.assertTrue(server._tip_refresh_retry.is_set()) - - def test_waiting_child_does_not_fall_back_after_transition_withdrawal( - self, - ) -> None: - class CountingLedger(FakeLedger): - def __init__(self) -> None: - super().__init__() - self.current_balance_reads = 0 - - def current_prior_balances(self) -> list[dict[str, object]]: - self.current_balance_reads += 1 - return [] - - class ObservedCondition(threading.Condition): - def __init__(self) -> None: - super().__init__() - self.wait_entered = threading.Event() - - def wait(self, timeout: float | None = None) -> bool: - self.wait_entered.set() - return super().wait(timeout) - - ledger = CountingLedger() - server, _rpc = coordinator(ledger=ledger) - parent_hash = "be" * 32 - preview_condition = ObservedCondition() - server._accepted_block_payout_preview_condition = preview_condition - server.accepted_block_payout_preview_wait_seconds = 10 - server._begin_accepted_block_payout_preview( - parent_hash, - block_height=10, - ) - server._mark_accepted_block_payout_landed( - parent_hash, - block_height=10, - ) - errors: list[BaseException] = [] - - def read_parent_balances() -> None: - try: - server._prior_balances_for_job_parent( - parent_hash, - parent_height=10, - ) - except BaseException as exc: # pragma: no cover - asserted below - errors.append(exc) - - thread = threading.Thread(target=read_parent_balances, daemon=True) - thread.start() - try: - self.assertTrue(preview_condition.wait_entered.wait(2)) - self.assertTrue(thread.is_alive()) - finally: - server._clear_accepted_block_payout_preview( - parent_hash, - invalidate_published=True, - ) - thread.join(5) - - self.assertFalse(thread.is_alive()) - self.assertEqual(len(errors), 1) - self.assertIsInstance(errors[0], TemplateRefreshBlocked) - self.assertIn("was withdrawn", str(errors[0])) - self.assertEqual(ledger.current_balance_reads, 0) - - def test_pending_parent_preview_wait_is_bounded_and_retryable(self) -> None: - server, _rpc = coordinator() - parent_hash = "ac" * 32 - server.accepted_block_payout_preview_wait_seconds = 0.01 - server._begin_accepted_block_payout_preview(parent_hash) - - with self.assertRaisesRegex(TemplateRefreshBlocked, "not ready"): - server._prior_balances_for_job_parent(parent_hash) - - self.assertTrue(server._tip_refresh_retry.is_set()) - server._clear_accepted_block_payout_preview(parent_hash) - - def test_replayed_active_ancestor_blocks_descendant_until_preview(self) -> None: - class PreviewLedger(FakeLedger): - def __init__(self) -> None: - super().__init__() - self.current_balance_reads = 0 - - def current_prior_balances(self) -> list[dict[str, object]]: - self.current_balance_reads += 1 - return [] - - class ObservedCondition(threading.Condition): - def __init__(self) -> None: - super().__init__() - self.wait_entered = threading.Event() - - def wait(self, timeout: float | None = None) -> bool: - self.wait_entered.set() - return super().wait(timeout) - - ledger = PreviewLedger() - server, rpc = coordinator(ledger=ledger) - accepted_hash = "ad" * 32 - descendant_hash = "ae" * 32 - preview = [ - { - "recipient_id": "miner-a", - "order_key": "miner-a", - "p2mr_program_hex": "11" * 32, - "balance_sats": 25, - } - ] - original_rpc_call = rpc.call - - def active_chain_call( - method: str, - params: list[object] | None = None, - ) -> object: - if method == "getblockhash": - self.assertEqual(params, [10]) - return accepted_hash - return original_rpc_call(method, params) - - rpc.call = active_chain_call # type: ignore[method-assign] - preview_condition = ObservedCondition() - server._accepted_block_payout_preview_condition = preview_condition - server.accepted_block_payout_preview_wait_seconds = 10 - server._begin_accepted_block_payout_preview(accepted_hash, block_height=10) - balances: list[list[dict[str, object]]] = [] - errors: list[BaseException] = [] - - def read_descendant_balances() -> None: - try: - balances.append( - server._prior_balances_for_job_parent( - descendant_hash, - parent_height=11, - ) - ) - except BaseException as exc: # pragma: no cover - asserted below - errors.append(exc) - - thread = threading.Thread(target=read_descendant_balances, daemon=True) - thread.start() - try: - self.assertTrue(preview_condition.wait_entered.wait(2)) - self.assertTrue(thread.is_alive()) - self.assertEqual(ledger.current_balance_reads, 0) - finally: - server._publish_accepted_block_payout_preview(accepted_hash, preview) - thread.join(5) - server._clear_accepted_block_payout_preview(accepted_hash) - - self.assertFalse(thread.is_alive()) - self.assertEqual(errors, []) - self.assertEqual(balances, [preview]) - self.assertEqual(ledger.current_balance_reads, 0) - - def test_landed_transition_bars_reconciliation_before_preview(self) -> None: - server, _rpc = coordinator() - block_hash = "af" * 32 - server._begin_accepted_block_payout_preview(block_hash, block_height=10) - server._mark_accepted_block_payout_landed(block_hash, block_height=10) - - with self.assertRaisesRegex( - _PayoutStatePublicationBlocked, - "confirmation is still pending", - ): - with server._payout_balance_mutation(): - self.fail("landed transition must bar payout reconciliation") - with self.assertRaisesRegex( - _PayoutStatePublicationBlocked, - "confirmation is still pending", - ): - server.reconcile_prism_pool_blocks_once() - - server._clear_accepted_block_payout_preview(block_hash) - with server._payout_balance_mutation(): - pass - - def test_readiness_latch_during_preparation_admission_reselects_ready_mode(self) -> None: - ledger = FakeLedger(miners=["solo"]) - server, rpc = coordinator(ledger=ledger) - recorded = install_fake_bundle_builder(server) - artifacts = server.store_template_artifacts(dict(rpc.template)) - self.assertIsNotNone(artifacts) - assert artifacts is not None - first_lookup_entered = threading.Event() - release_first_lookup = threading.Event() - original_lookup = server._lookup_job_bundle - lookup_calls = 0 - - def block_first_lookup(key: tuple[object, ...]) -> object: - nonlocal lookup_calls - lookup_calls += 1 - if lookup_calls == 1: - first_lookup_entered.set() - self.assertTrue(release_first_lookup.wait(2.0)) - return original_lookup(key) - - server._lookup_job_bundle = block_first_lookup # type: ignore[method-assign] - bundles: list[object] = [] - errors: list[BaseException] = [] - - def build_bundle() -> None: - try: - bundles.append(server.shared_job_bundle(artifacts, worker())) - except BaseException as exc: # pragma: no cover - assertion reports it - errors.append(exc) - - build_thread = threading.Thread(target=build_bundle) - build_thread.start() - self.assertTrue(first_lookup_entered.wait(2.0)) - ledger.miners = ["miner-a", "miner-b", "miner-c"] - release_first_lookup.set() - - build_thread.join(2.0) - - self.assertFalse(build_thread.is_alive()) - self.assertEqual(errors, []) - self.assertEqual(len(bundles), 1) - bundle = bundles[0] - self.assertFalse(bundle.collection_only) # type: ignore[union-attr] - self.assertEqual(recorded["calls"], 1) - self.assertEqual(ledger.snapshot_calls, 1) - - def test_payout_generation_advance_does_not_cancel_new_generation_fanout(self) -> None: - server, _rpc = coordinator() - server._ensure_tip_refresh_state() - cancellations: list[str] = [] - - class CurrentToken: - payout_state_generation = 1 - - class Cancellation: - def cancel(self) -> None: - cancellations.append("cancelled") - - class InjectCurrentFanoutLock: - def __enter__(self) -> object: - server._active_tip_refresh = (CurrentToken(), Cancellation()) - return self - - def __exit__(self, *_args: object) -> None: - return None - - # Model a new-generation refresh registering after the cache-state - # increment but before the invalidator reaches the coordinator lock. - server.lock = InjectCurrentFanoutLock() # type: ignore[assignment] - - self.assertEqual(server._advance_payout_state_generation(), 1) - self.assertEqual(cancellations, []) - self.assertFalse(server.tip_refresh_is_pending()) - self.assertFalse(server._tip_refresh_retry.is_set()) - - def test_payout_generation_retry_marks_tip_refresh_pending(self) -> None: - server, _rpc = coordinator() - server._ensure_tip_refresh_state() - - self.assertFalse(server.tip_refresh_is_pending()) - self.assertEqual(server._advance_payout_state_generation(), 1) - - self.assertTrue(server.tip_refresh_is_pending()) - self.assertTrue(server._tip_refresh_retry.is_set()) - - self.assertEqual(server.poll_qbit_tip_template_once(), 0) - self.assertFalse(server.tip_refresh_is_pending()) - - def test_payout_only_advance_bounds_publish_supersession(self) -> None: - server, _rpc = coordinator() - server.payout_reconcile_supersession_retries = 2 - real_publish = server._publish_payout_state_candidate - publish_attempts = 0 - - def supersede_before_publish(candidate: object) -> int | None: - nonlocal publish_attempts - publish_attempts += 1 - server._reserve_payout_state_source( - "external_tip", - tip_hash=f"{publish_attempts + 30:064x}", - ) - return real_publish(candidate) # type: ignore[arg-type] - - server._publish_payout_state_candidate = supersede_before_publish # type: ignore[method-assign] - - with self.assertRaisesRegex( - TemplateRefreshBlocked, - "payout-only invalidation was superseded", - ): - server._advance_payout_state_generation() - - self.assertEqual(publish_attempts, 3) - self.assertEqual(server._payout_state_generation, 0) - self.assertTrue(server._payout_state_publication_blocked) - self.assertTrue(server._payout_state_delivery_gate._delivery_blocked) - self.assertTrue(server.tip_refresh_is_pending()) - - def test_payout_publication_fence_is_not_a_job_build_failure(self) -> None: - server, _rpc = coordinator() - install_fake_bundle_builder(server) - state = client(1) - state.send = lambda _payload: None # type: ignore[method-assign] - server.clients = {state} - server._pool_ready_latched = True - server._reserve_payout_state_source("payout_only") - server._block_payout_state_publication() - - self.assertFalse(server.maybe_send_job(state, clean_jobs=True)) - with self.assertRaisesRegex(TemplateRefreshBlocked, "pending publication"): - server.poll_qbit_tip_template_once() - - self.assertEqual(server.job_build_failure_count, 0) - self.assertEqual(server.tip_refresh_client_counts["failed"], 0) - self.assertEqual(server.tip_refresh_client_counts["skipped"], 1) - - def test_successful_poll_clears_payout_pending_created_during_reconcile(self) -> None: - server, _rpc = coordinator() - server._ensure_tip_refresh_state() - reconcile_entered = threading.Event() - allow_reconcile = threading.Event() - results: list[int] = [] - errors: list[BaseException] = [] - - def reconcile(_tip_hash: str) -> bool: - reconcile_entered.set() - if not allow_reconcile.wait(5): - raise AssertionError("test did not release reconciliation") - return True - - def poll() -> None: - try: - results.append(server.poll_qbit_tip_template_once()) - except BaseException as exc: # pragma: no cover - asserted below - errors.append(exc) - - server.ensure_reorg_reconciled_for_tip = reconcile # type: ignore[method-assign] - server._mark_tip_refresh_pending("seed") - poll_thread = threading.Thread(target=poll) - poll_thread.start() - try: - self.assertTrue(reconcile_entered.wait(5)) - self.assertEqual(server._advance_payout_state_generation(), 1) - finally: - allow_reconcile.set() - poll_thread.join(5) - - self.assertFalse(poll_thread.is_alive()) - self.assertEqual(errors, []) - self.assertEqual(results, [0]) - self.assertEqual(server._payout_state_generation, 1) - self.assertIsNotNone(server.last_successful_template_refresh_monotonic) - self.assertFalse(server.tip_refresh_is_pending()) - - def test_failed_poll_preserves_pending_signal_until_successful_retry(self) -> None: - server, _rpc = coordinator() - server._ensure_tip_refresh_state() - pending_token = server._mark_tip_refresh_pending("seed") - - def fail_reconciliation(_tip_hash: str) -> bool: - server._schedule_tip_refresh_retry() - raise RuntimeError("ledger unavailable") - - server.ensure_reorg_reconciled_for_tip = fail_reconciliation # type: ignore[method-assign] - - with self.assertRaisesRegex( - TemplateRefreshBlocked, - "qbit reorg reconciliation failed", - ): - server.poll_qbit_tip_template_once() - - self.assertTrue(server._tip_refresh_retry.is_set()) - self.assertTrue(server.tip_refresh_is_pending()) - self.assertEqual(server._tip_refresh_pending_token, pending_token) - - # Model blockpoll claiming the immediate wake and completing the retry. - server._tip_refresh_retry.clear() - server.ensure_reorg_reconciled_for_tip = lambda _tip: True # type: ignore[method-assign] - - self.assertEqual(server.poll_qbit_tip_template_once(), 0) - self.assertFalse(server.tip_refresh_is_pending()) - - def test_shutdown_during_reconciliation_stops_poll_without_refresh_failure(self) -> None: - server, _rpc = coordinator() - - def rejected_reconciliation(_tip_hash: str) -> bool: - server.stop_event.set() - raise ShutdownInProgress("PRISM coordinator is shutting down") - - server.ensure_reorg_reconciled_for_tip = ( # type: ignore[method-assign] - rejected_reconciliation - ) - - self.assertEqual(server.poll_qbit_tip_template_once(), 0) - self.assertIsNone( - getattr(server, "last_successful_template_refresh_monotonic", None) - ) - - def test_completed_refresh_cannot_clear_newer_payout_pending(self) -> None: - server, _rpc = coordinator() - snapshot = server.fetch_qbit_tip_template_snapshot() - sequence = server._reserve_tip_observation_sequence() - self.assertTrue( - server.observe_tip_first_seen( - snapshot.bestblockhash, - observation_sequence=sequence, - publish_refresh_observation=True, - ) - ) - with server.lock: - server.tip_template_snapshot = snapshot - completed_generation = server._payout_state_generation - self.assertEqual(server._advance_payout_state_generation(), 1) - newer_token = server._tip_refresh_pending_token - - self.assertFalse( - server._clear_tip_refresh_pending_for_completed_refresh( - snapshot, - sequence, - completed_generation, - ) - ) - self.assertEqual(server._tip_refresh_pending_token, newer_token) - self.assertTrue(server.tip_refresh_is_pending()) - - @staticmethod - def _capture_error( - errors: list[BaseException], - operation: object, - ) -> None: - try: - operation() # type: ignore[operator] - except BaseException as exc: # noqa: BLE001 - test thread handoff - errors.append(exc) - - def test_preview_publication_degrades_when_artifact_preparation_times_out( - self, - ) -> None: - server, _rpc = coordinator() - parent_hash = "ab" * 32 - preview = [ - { - "recipient_id": "miner-a", - "order_key": "miner-a", - "p2mr_program_hex": "11" * 32, - "balance_sats": 25, - } - ] - - def timing_out_preparation(*_args: object, **_kwargs: object) -> object: - raise TimeoutError("postgres operation exceeded 30s") - - server._prepared_payout_state_candidate = ( # type: ignore[method-assign] - timing_out_preparation - ) - server._begin_accepted_block_payout_preview(parent_hash) - with patch("builtins.print"): - self.assertEqual( - server._publish_accepted_block_payout_preview(parent_hash, preview), - preview, - ) - - # Children of the pending parent receive the immutable compact - # preview even though no payout-state generation was published. - transition = server._accepted_block_payout_previews[parent_hash] - self.assertIsNone(transition.published_generation) - self.assertTrue(transition.landed) - self.assertEqual( - server._prior_balances_for_job_parent(parent_hash), - preview, - ) - # Payout-state publication itself stays gated until the landing - # reaches a terminal durable state. - with self.assertRaisesRegex( - _PayoutStatePublicationBlocked, "still pending" - ): - with server._payout_balance_mutation(): - pass - server._clear_accepted_block_payout_preview(parent_hash) - - def test_unresolved_accepted_parent_depth_cap_fails_closed(self) -> None: - server, _rpc = coordinator() - server.accepted_parent_unresolved_depth_max = 2 - server.accepted_block_payout_preview_wait_seconds = 0.01 - hashes = ["a1" * 32, "a2" * 32, "a3" * 32] - for height, block_hash in enumerate(hashes, start=10): - server._begin_accepted_block_payout_preview( - block_hash, block_height=height - ) - server._mark_accepted_block_payout_landed( - block_hash, block_height=height - ) - self.assertEqual(len(server.accepted_parent_unresolved_ages_seconds()), 3) - with self.assertRaisesRegex(TemplateRefreshBlocked, "exceeds cap"): - server._await_pending_parent_payout_preview(hashes[0]) - # At the cap issuance still blocks: a child issued now could land - # and create a (cap + 1)th unresolved transition, so the configured - # maximum would no longer bound the prospective balance chain. - server._clear_accepted_block_payout_preview(hashes[2]) - with self.assertRaisesRegex(TemplateRefreshBlocked, "exceeds cap"): - server._await_pending_parent_payout_preview(hashes[0]) - # Below the cap the ordinary bounded wait applies instead. - server._clear_accepted_block_payout_preview(hashes[1]) - with self.assertRaisesRegex(TemplateRefreshBlocked, "not ready yet"): - server._await_pending_parent_payout_preview(hashes[0]) - server._clear_accepted_block_payout_preview(hashes[0]) - self.assertEqual(server.accepted_parent_unresolved_ages_seconds(), []) - - def test_landing_observability_metrics_exposition(self) -> None: - import contextlib as _contextlib - import time as _time - - server, _rpc = coordinator() - server.block_landing_db_timeout_seconds = 45.0 - server.block_landing_db_timeout_max_seconds = 90.0 - - @_contextlib.contextmanager - def statement_timeout(seconds: float): - yield - - original_ledger = server.ledger - server.ledger = SimpleNamespace( - statement_timeout=statement_timeout, - prior_balances_read_stats=lambda: { - "reads_total": 3, - "last_seconds": 0.004, - "max_seconds": 0.021, - }, - ) - block_hash = "ab" * 32 - with self.assertRaises(TimeoutError): - with server._block_landing_ledger_statement_timeout_scope(block_hash): - raise LedgerOperationTimeout("postgres operation exceeded 45s") - server.ledger = original_ledger - - server._begin_accepted_block_payout_preview(block_hash, block_height=10) - server._mark_accepted_block_payout_landed(block_hash, block_height=10) - server.accepted_block_payout_preview_wait_seconds = 0.0 - with self.assertRaisesRegex(TemplateRefreshBlocked, "not ready yet"): - server._await_pending_parent_payout_preview(block_hash) - server._startup_phase_origin_monotonic = _time.monotonic() - server._record_startup_phase_once("audit_listener_bound") - - prior_stats = { - "reads_total": 3, - "last_seconds": 0.004, - "max_seconds": 0.021, - } - server.ledger.prior_balances_read_stats = ( # type: ignore[attr-defined] - lambda: prior_stats - ) - lines = "\n".join(server.landing_observability_metrics_lines()) - self.assertIn( - 'qbit_prism_block_ledger_call_timeouts_total{call_class="landing"} 1', - lines, - ) - self.assertIn( - 'qbit_prism_block_ledger_call_budget_seconds{call_class="landing"} 45.000000', - lines, - ) - self.assertIn( - "qbit_prism_accepted_parent_unresolved_transitions 1", - lines, - ) - self.assertIn( - "qbit_prism_accepted_parent_preview_wait_timeouts_total 1", - lines, - ) - self.assertNotIn( - "qbit_prism_accepted_parent_unresolved_oldest_seconds -1", - lines, - ) - self.assertIn("qbit_prism_prior_balances_reads_total 3", lines) - self.assertIn( - "qbit_prism_prior_balances_read_max_seconds 0.021000", - lines, - ) - self.assertIn( - 'qbit_prism_startup_phase_seconds{phase="audit_listener_bound"}', - lines, - ) - server._clear_accepted_block_payout_preview(block_hash) - - def test_startup_replay_gate_blocks_job_builds_until_enumeration(self) -> None: - server, _rpc = coordinator() - enumerated: list[int] = [] - - class EnumLedger(FakeLedger): - def pending_block_candidate_rows( - self, *, limit: int = 32 - ) -> list[dict[str, object]]: - enumerated.append(limit) - return [] - - server.ledger = EnumLedger() - server._note_block_replay_enumeration_owed() - with self.assertRaisesRegex(TemplateRefreshBlocked, "has not enumerated"): - server._await_pending_parent_payout_preview("ab" * 32) - self.assertEqual(server.replay_pending_block_candidates(), 0) - self.assertEqual(enumerated, [32]) - self.assertFalse(server._block_replay_enumeration_owed()) - self.assertIsNone(server._await_pending_parent_payout_preview("ab" * 32)) - - def test_owed_enumeration_bypasses_live_candidate_short_circuits(self) -> None: - server, _rpc = coordinator() - enumerated: list[int] = [] - - class EnumLedger(FakeLedger): - def pending_block_candidate_rows( - self, *, limit: int = 32 - ) -> list[dict[str, object]]: - enumerated.append(limit) - return [] - - server.ledger = EnumLedger() - server.block_landing_db_timeout_seconds = 7.5 - server._retry_block_candidate = object() - # Steady state: a retained retry short-circuits the outbox poll. - self.assertEqual(server.replay_pending_block_candidates(), 0) - self.assertEqual(enumerated, []) - # Owed enumeration is correctness-critical and bypasses the - # short-circuits, running with the landing-class budget and - # recording under the landing metrics class. - server._note_block_replay_enumeration_owed() - self.assertEqual(server.replay_pending_block_candidates(), 0) - self.assertEqual(enumerated, [32]) - self.assertFalse(server._block_replay_enumeration_owed()) - metrics = server.block_ledger_call_class_metrics() - self.assertEqual(metrics["landing"]["last_budget_seconds"], 7.5) - self.assertNotIn("fast", metrics) - - def test_worker_reported_deadline_counts_as_landing_timeout(self) -> None: - """A server-side cancellation completes the ledger worker with a - timeout error before the coordinator-side wait expires; the - completion path must still count it for the landing-timeout alert.""" - server, _rpc = coordinator() - - class CancelledLedger(FakeLedger): - def pending_block_candidate_rows( - self, *, limit: int = 32 - ) -> list[dict[str, object]]: - raise LedgerOperationTimeout("postgres statement deadline expired") - - server.ledger = CancelledLedger() - server._note_block_replay_enumeration_owed() - with patch("builtins.print"): - self.assertTrue(server._run_startup_block_candidate_replay()) - self.assertTrue(server._block_replay_enumeration_owed()) - metrics = server.block_ledger_call_class_metrics() - self.assertEqual(metrics["landing"]["calls_total"], 1) - self.assertEqual(metrics["landing"]["timeouts_total"], 1) - self.assertNotIn("fast", metrics) - - def test_startup_enumeration_escalates_past_truncated_batches(self) -> None: - """A full first batch must not end enumeration: rows beyond the batch - window (potentially the active parent) still need their payout - barriers armed before job builds unblock.""" - server, _rpc = coordinator() - requested: list[int] = [] - total_pending = 33 - - class TruncatingLedger(FakeLedger): - def pending_block_candidate_rows( - self, *, limit: int = 32 - ) -> list[dict[str, object]]: - requested.append(limit) - return [ - durable_candidate_row(index) - for index in range(min(limit, total_pending)) - ] - - server.ledger = TruncatingLedger() - server._note_block_replay_enumeration_owed() - self.assertEqual(server.replay_pending_block_candidates(), total_pending) - self.assertEqual(requested, [32, 64]) - self.assertFalse(server._block_replay_enumeration_owed()) - self.assertEqual( - server._block_replay_candidate_queue.qsize(), total_pending - ) - # Every restored row armed its payout barrier before the gate opened. - self.assertEqual( - len(server._accepted_block_payout_previews), total_pending - ) - - def test_startup_enumeration_stays_owed_at_truncation_cap(self) -> None: - """An outbox larger than the enumeration cap keeps job builds blocked - instead of silently declaring enumeration complete.""" - server, _rpc = coordinator() - requested: list[int] = [] - - class EndlessLedger(FakeLedger): - def pending_block_candidate_rows( - self, *, limit: int = 32 - ) -> list[dict[str, object]]: - requested.append(limit) - return [durable_candidate_row(index) for index in range(limit)] - - server.ledger = EndlessLedger() - server._note_block_replay_enumeration_owed() - with patch("builtins.print"): - queued = server.replay_pending_block_candidates() - self.assertEqual(queued, 1024) - self.assertEqual(requested, [32, 64, 128, 256, 512, 1024]) - self.assertTrue(server._block_replay_enumeration_owed()) - with self.assertRaisesRegex(TemplateRefreshBlocked, "has not enumerated"): - server._await_pending_parent_payout_preview("ab" * 32) - - def test_steady_state_poll_restores_a_single_batch(self) -> None: - """Without an owed enumeration, a full batch stays a single query; - the next poll picks up later rows after the queue drains.""" - server, _rpc = coordinator() - requested: list[int] = [] - - class FullBatchLedger(FakeLedger): - def pending_block_candidate_rows( - self, *, limit: int = 32 - ) -> list[dict[str, object]]: - requested.append(limit) - return [durable_candidate_row(index) for index in range(limit)] - - server.ledger = FullBatchLedger() - self.assertEqual(server.replay_pending_block_candidates(), 32) - self.assertEqual(requested, [32]) - self.assertFalse(server._block_replay_enumeration_owed()) - - def test_startup_replay_timeout_keeps_job_builds_blocked(self) -> None: - server, _rpc = coordinator() - release = threading.Event() - - class StallLedger(FakeLedger): - def pending_block_candidate_rows( - self, *, limit: int = 32 - ) -> list[dict[str, object]]: - release.wait(5) - return [] - - server.ledger = StallLedger() - server.block_landing_db_timeout_seconds = 0.01 - server.block_landing_db_timeout_max_seconds = 0.01 - try: - with patch("builtins.print"): - self.assertTrue(server._run_startup_block_candidate_replay()) - self.assertTrue(server._block_replay_enumeration_owed()) - with self.assertRaisesRegex( - TemplateRefreshBlocked, "has not enumerated" - ): - server._await_pending_parent_payout_preview("ab" * 32) - # The startup enumeration ran with the landing budget, so its - # timeout fires the documented landing-timeout alert instead of - # hiding inside the fast-call series. - metrics = server.block_ledger_call_class_metrics() - self.assertEqual(metrics["landing"]["timeouts_total"], 1) - self.assertNotIn("fast", metrics) - finally: - release.set() - - def test_startup_phase_seconds_records_first_occurrence_only(self) -> None: - import time as _time - - server, _rpc = coordinator() - self.assertEqual(server.startup_phase_seconds(), {}) - server._record_startup_phase_once("audit_listener_bound") - self.assertEqual(server.startup_phase_seconds(), {}) - server._startup_phase_origin_monotonic = _time.monotonic() - 1.0 - server._record_startup_phase_once("audit_listener_bound") - first = server.startup_phase_seconds()["audit_listener_bound"] - self.assertGreaterEqual(first, 1.0) - server._record_startup_phase_once("audit_listener_bound") - self.assertEqual( - server.startup_phase_seconds()["audit_listener_bound"], first - ) - - def test_job_issuance_continues_during_padded_landing(self) -> None: - """Acceptance shape for issue #188: a landing padded past its normal - duration must not stop child job issuance, and payout-state - publication stays gated until the landing reaches a terminal state. - The pad is logical (the barrier simply stays unresolved) so the test - is instant while pinning the same ordering a 10s landing produces.""" - server, _rpc = coordinator() - parent_hash = "ab" * 32 - preview = [ - { - "recipient_id": "miner-a", - "order_key": "miner-a", - "p2mr_program_hex": "11" * 32, - "balance_sats": 25, - } - ] - server._begin_accepted_block_payout_preview(parent_hash, block_height=10) - server._mark_accepted_block_payout_landed(parent_hash, block_height=10) - server._publish_accepted_block_payout_preview(parent_hash, preview) - # The landing bookkeeping is still unresolved; every child build keeps - # receiving the immutable preview instead of blocking or reading - # confirmed balances. - for _ in range(5): - self.assertEqual( - server._prior_balances_for_job_parent(parent_hash), - preview, - ) - ages = server.accepted_parent_unresolved_ages_seconds() - self.assertEqual(len(ages), 1) - with self.assertRaisesRegex( - _PayoutStatePublicationBlocked, "still pending" - ): - with server._payout_balance_mutation(): - pass - # Landing completes: the transition clears and confirmed reads resume. - server._clear_accepted_block_payout_preview(parent_hash) - self.assertEqual( - server._prior_balances_for_job_parent( - parent_hash, fallback_balances=[] - ), - [], - ) - self.assertEqual(server.accepted_parent_unresolved_ages_seconds(), []) - - def test_unmark_landed_clears_unresolved_age(self) -> None: - server, _rpc = coordinator() - block_hash = "cd" * 32 - server._begin_accepted_block_payout_preview(block_hash, block_height=10) - self.assertEqual(server.accepted_parent_unresolved_ages_seconds(), []) - server._mark_accepted_block_payout_landed(block_hash, block_height=10) - ages = server.accepted_parent_unresolved_ages_seconds() - self.assertEqual(len(ages), 1) - self.assertGreaterEqual(ages[0], 0.0) - server._unmark_accepted_block_payout_landed(block_hash) - self.assertEqual(server.accepted_parent_unresolved_ages_seconds(), []) - server._clear_accepted_block_payout_preview(block_hash) - - def test_preview_publication_survives_window_build_timeout(self) -> None: - """Issue #188 acceptance: when the payout window build dies with the - slow ledger during found-block publication, the preview must still - cross the atomic publication boundary. Admission then reopens and - child jobs build against the verified compact preview instead of - stalling behind the global publication fence until the ledger - recovers.""" - accepted_hash = "c4" * 32 - server, rpc = coordinator( - template=base_template(height=12, prevhash=accepted_hash) - ) - recorded = install_fake_bundle_builder(server) - artifacts = server.store_template_artifacts(dict(rpc.template)) - assert artifacts is not None - self.assertTrue(server.pool_readiness_latched()) - preview = [ - { - "recipient_id": "miner-a", - "order_key": "miner-a", - "p2mr_program_hex": "11" * 32, - "balance_sats": 25, - } - ] - - def timing_out_window_build( - *_args: object, **_kwargs: object - ) -> PayoutLedgerArtifact: - raise LedgerOperationTimeout("postgres statement deadline expired") - - server._build_payout_ledger_artifact = ( # type: ignore[method-assign] - timing_out_window_build - ) - server._begin_accepted_block_payout_preview(accepted_hash, block_height=11) - server._mark_accepted_block_payout_landed(accepted_hash, block_height=11) - generation_before = server._payout_state_generation - with patch("builtins.print"): - published = server._publish_accepted_block_payout_preview( - accepted_hash, preview - ) - self.assertEqual(published, preview) - # The publication crossed the atomic boundary: a generation was - # installed and delivery admission reopened despite the dead build. - self.assertFalse(server._payout_state_publication_fenced()) - self.assertEqual(server._payout_state_generation, generation_before + 1) - transition = server._accepted_block_payout_previews[accepted_hash] - self.assertEqual(transition.published_generation, generation_before + 1) - # Child job construction is admitted and consumes the preview. - bundle = server.shared_job_bundle(artifacts, mode="ready") - self.assertIsNotNone(bundle) - self.assertEqual(recorded["calls"], 1) - last_kwargs = recorded["last_kwargs"] - assert isinstance(last_kwargs, dict) - self.assertEqual(list(last_kwargs["prior_balances"]), preview) - server._clear_accepted_block_payout_preview(accepted_hash) - - def test_landed_transition_poll_uses_coordination_blocked_budget(self) -> None: - server, _rpc = coordinator() - server.reorg_reconciler_enabled = True - block_hash = "af" * 32 - server._begin_accepted_block_payout_preview(block_hash, block_height=10) - server._mark_accepted_block_payout_landed(block_hash, block_height=10) - - with ( - patch( - "lab.prism.prism_coordinator.time.monotonic", - return_value=100.0, - ), - self.assertRaisesRegex( - _PayoutStatePublicationBlocked, - "confirmation is still pending", - ), - ): - server.poll_qbit_tip_template_once() - - self.assertIsNone( - getattr(server, "template_refresh_failure_started_monotonic", None) - ) - self.assertEqual( - server.coordination_blocked_streak_age_seconds(105.0), - 5.0, - ) - - def test_sync_ledger_snapshot_publishes_reusable_artifact(self) -> None: - server, rpc = coordinator() - install_fake_bundle_builder(server) - first = server.store_template_artifacts(dict(rpc.template)) - assert first is not None - # Pin distinct per-generation anchors near the live clock so the - # armed window stays anchored inside the audit ceiling. - first_anchor_ms = now_ms() - 20 - with server._job_cache_lock: - server._job_build_issued_at_ms[first.generation] = first_anchor_ms - - inline = server.shared_job_bundle(first, mode="ready") - - self.assertEqual(inline.payout_artifact_generation, 0) - self.assertEqual(server.ledger.snapshot_calls, 1) - with server._job_cache_lock: - artifact = server._payout_ledger_artifact - self.assertIsNotNone(artifact) - assert artifact is not None - self.assertGreater(artifact.generation, 0) - self.assertEqual( - artifact.payout_state_generation, - server._payout_state_generation, - ) - self.assertEqual( - artifact.accepted_share_count, - len(server.ledger.miners), - ) - self.assertEqual(list(artifact.shares_json), inline.shares_json) - self.assertEqual(artifact.snapshot_anchor_ms, inline.issued_at_ms) - - second = server.store_template_artifacts( - base_template(height=11, prevhash="22" * 32) - ) - assert second is not None - with server._job_cache_lock: - server._job_build_issued_at_ms[second.generation] = ( - first_anchor_ms + 10 - ) - - reused = server.shared_job_bundle(second, mode="ready") - - self.assertEqual(reused.payout_artifact_generation, artifact.generation) - self.assertEqual(server.ledger.snapshot_calls, 1) - self.assertEqual(reused.shares_json, inline.shares_json) - self.assertNotEqual(reused.issued_at_ms, inline.issued_at_ms) - self.assertEqual( - reused.found_block["anchor_job_issued_at_ms"], - artifact.snapshot_anchor_ms, - ) - - def test_sync_artifact_publication_survives_share_commit_mid_read( - self, - ) -> None: - class MidReadCommitLedger(FakeLedger): - def snapshot_at_job_issue( - self, - anchor_job_issued_at_ms: int, - *, - window_weight: int | None = None, - ) -> list[FakeShare]: - result = super().snapshot_at_job_issue( - anchor_job_issued_at_ms, - window_weight=window_weight, - ) - # A share becomes durable while the window read is in - # flight. The read is scoped by the frozen anchor -- the - # commit lands above it and belongs to the next window -- so - # anchor-scoped publication proceeds. - self.miners = [*self.miners, f"mid-read-{self.snapshot_calls}"] - return result - - server, rpc = coordinator(ledger=MidReadCommitLedger()) - install_fake_bundle_builder(server) - artifacts = server.store_template_artifacts(dict(rpc.template)) - assert artifacts is not None - - bundle = server.shared_job_bundle(artifacts, mode="ready") - - self.assertFalse(bundle.collection_only) - self.assertEqual(bundle.payout_artifact_generation, 0) - with server._job_cache_lock: - artifact = server._payout_ledger_artifact - self.assertIsNotNone(artifact) - assert artifact is not None - self.assertEqual(artifact.snapshot_anchor_ms, bundle.issued_at_ms) - self.assertEqual(list(artifact.shares_json), bundle.shares_json) - - # The armed window keeps serving under continuous commits: the live - # durable count has moved past the artifact's, and reuse must not - # care. - second = server.store_template_artifacts( - base_template(height=11, prevhash="22" * 32) - ) - assert second is not None - reused = server.shared_job_bundle(second, mode="ready") - self.assertEqual(reused.payout_artifact_generation, artifact.generation) - self.assertEqual(server.ledger.snapshot_calls, 1) - self.assertEqual( - reused.found_block["anchor_job_issued_at_ms"], - artifact.snapshot_anchor_ms, - ) - - def test_sync_publication_allowed_when_anchor_predates_durable_share( - self, - ) -> None: - server, rpc = coordinator() - install_fake_bundle_builder(server) - artifacts = server.store_template_artifacts(dict(rpc.template)) - assert artifacts is not None - - first = server.shared_job_bundle(artifacts, mode="ready") - self.assertFalse(first.collection_only) - with server._job_cache_lock: - self.assertIsNotNone(server._payout_ledger_artifact) - # A share becomes durable after this generation's anchor was - # frozen. Force the synchronous path to run again for the same - # frozen anchor. - server._payout_ledger_artifact = None - server._job_bundle_cache.clear() - server.ledger.miners = [*server.ledger.miners, "post-anchor-share"] - - second = server.shared_job_bundle(artifacts, mode="ready") - - self.assertEqual(second.payout_artifact_generation, 0) - # The window read at the frozen anchor deterministically excludes - # the later durable share -- it belongs to the next window -- so the - # anchor-scoped publication proceeds even though the live count has - # moved past the frozen anchor. - with server._job_cache_lock: - republished = server._payout_ledger_artifact - self.assertIsNotNone(republished) - assert republished is not None - self.assertEqual(republished.snapshot_anchor_ms, second.issued_at_ms) - self.assertEqual(second.issued_at_ms, first.issued_at_ms) - - def test_racing_commit_does_not_disable_sync_artifact_seeding(self) -> None: - class RacingStatsLedger(FakeLedger): - def __init__(self) -> None: - super().__init__( - miners=["miner-a", "miner-b", "miner-c", "miner-d"] - ) - self._pending_counts = iter([3, 4]) - - def accepted_share_stats(self) -> dict[str, int]: - self.stats_calls += 1 - try: - count = next(self._pending_counts) - except StopIteration: - count = 4 - return { - "accepted_share_count": count, - "distinct_miner_count": 4, - } - - server, rpc = coordinator(ledger=RacingStatsLedger()) - install_fake_bundle_builder(server) - artifacts = server.store_template_artifacts(dict(rpc.template)) - assert artifacts is not None - - # A commit races the informational count read. The window is scoped - # by the frozen anchor regardless of which side of the read the - # commit landed on, so seeding proceeds; the count is diagnostics, - # not a fence. - bundle = server.shared_job_bundle(artifacts, mode="ready") - - self.assertFalse(bundle.collection_only) - with server._job_cache_lock: - artifact = server._payout_ledger_artifact - self.assertIsNotNone(artifact) - assert artifact is not None - self.assertEqual(artifact.snapshot_anchor_ms, bundle.issued_at_ms) - self.assertEqual(list(artifact.shares_json), bundle.shares_json) - - def test_same_window_background_rebuild_keeps_artifact_generation( - self, - ) -> None: - server, rpc = coordinator() - install_fake_bundle_builder(server) - artifacts = server.store_template_artifacts(dict(rpc.template)) - assert artifacts is not None - - server.shared_job_bundle(artifacts, mode="ready") - with server._job_cache_lock: - installed = server._payout_ledger_artifact - assert installed is not None - - # No share committed since: the speculative rebuild reads the same - # window under a fresh anchor and must not spin the generation, - # which would re-key bundle lookups for nothing -- but it is still a - # successful preparation, so an accumulated re-arm backoff releases - # and the fresher anchor advances the staleness clock in place. - with server._payout_artifact_executor_lock: - server._payout_artifact_rearm_backoff = 4 - server._prepare_payout_ledger_artifact( - server._payout_state_generation, - artifacts.network_difficulty, - ) - with server._job_cache_lock: - refreshed = server._payout_ledger_artifact - assert refreshed is not None - self.assertEqual(refreshed.generation, installed.generation) - self.assertEqual( - refreshed.share_snapshot_sha256, - installed.share_snapshot_sha256, - ) - assert installed.snapshot_anchor_ms is not None - assert refreshed.snapshot_anchor_ms is not None - self.assertGreaterEqual( - int(refreshed.snapshot_anchor_ms), - int(installed.snapshot_anchor_ms), - ) - with server._payout_artifact_executor_lock: - self.assertEqual(server._payout_artifact_rearm_backoff, 1) - - # A new durable share makes the rebuild a genuine replacement. - server.ledger.miners = [*server.ledger.miners, "late-share"] - server._prepare_payout_ledger_artifact( - server._payout_state_generation, - artifacts.network_difficulty, - ) - with server._job_cache_lock: - replaced = server._payout_ledger_artifact - assert replaced is not None - self.assertGreater(replaced.generation, installed.generation) - self.assertEqual(replaced.accepted_share_count, 4) - - def test_delayed_older_snapshot_does_not_regress_artifact(self) -> None: - server, rpc = coordinator() - install_fake_bundle_builder(server) - artifacts = server.store_template_artifacts(dict(rpc.template)) - assert artifacts is not None - server.shared_job_bundle(artifacts, mode="ready") - with server._job_cache_lock: - current = server._payout_ledger_artifact - assert current is not None - assert current.snapshot_anchor_ms is not None - - # A snapshot taken at an earlier anchor finishes its window - # conversion late, so its preparation timestamp is newer than the - # installed artifact's; the anchor still proves it is the older - # window. - older_window = [ - {"share_seq": seq, "miner_id": "miner-a"} for seq in (1, 2) - ] - delayed = PayoutLedgerArtifact( - generation=0, - payout_state_generation=server._payout_state_generation, - network_difficulty=int(current.network_difficulty), - accepted_share_count=2, - shares_json=tuple(older_window), - prior_balances=(), - prepared_monotonic=time.monotonic(), - snapshot_anchor_ms=int(current.snapshot_anchor_ms) - 5, - share_snapshot_sha256=canonical_json_sha256(older_window), - ) - server._install_payout_ledger_artifact(delayed) - with server._job_cache_lock: - self.assertIs(server._payout_ledger_artifact, current) - - # A window snapshotted at a fresher anchor replaces regardless of - # its preparation timestamp ordering. - newer_window = [ - {"share_seq": seq, "miner_id": "miner-a"} for seq in (1, 2, 3, 4) - ] - fresher = dataclass_replace( - delayed, - accepted_share_count=4, - shares_json=tuple(newer_window), - prepared_monotonic=current.prepared_monotonic - 1.0, - snapshot_anchor_ms=int(current.snapshot_anchor_ms) + 5, - share_snapshot_sha256=canonical_json_sha256(newer_window), - ) - server._install_payout_ledger_artifact(fresher) - with server._job_cache_lock: - replaced = server._payout_ledger_artifact - assert replaced is not None - self.assertGreater(replaced.generation, current.generation) - self.assertEqual(replaced.accepted_share_count, 4) - - def test_difficulty_change_replaces_same_window_artifact(self) -> None: - server, rpc = coordinator() - install_fake_bundle_builder(server) - artifacts = server.store_template_artifacts(dict(rpc.template)) - assert artifacts is not None - server.shared_job_bundle(artifacts, mode="ready") - with server._job_cache_lock: - installed = server._payout_ledger_artifact - assert installed is not None - - # A retarget moves the live template to a new difficulty while the - # share window stays identical. The rebuild at the live difficulty - # must replace the artifact: keeping the old-difficulty artifact - # would fail the reuse fence on every build while the same-window - # skip kept discarding its replacement. - retarget_difficulty = int(installed.network_difficulty) + 1 - with server._job_cache_lock: - live = server._template_artifacts - assert live is not None - server._template_artifacts = dataclass_replace( - live, - network_difficulty=retarget_difficulty, - ) - server._prepare_payout_ledger_artifact( - server._payout_state_generation, - retarget_difficulty, - ) - with server._job_cache_lock: - replaced = server._payout_ledger_artifact - assert replaced is not None - self.assertGreater(replaced.generation, installed.generation) - self.assertEqual(replaced.network_difficulty, retarget_difficulty) - - # A pre-retarget build delayed after its snapshot carries the same - # window at the old difficulty; whether its anchor trails or ties, - # the live-difficulty artifact must stay. - delayed = dataclass_replace( - installed, - generation=0, - prepared_monotonic=time.monotonic(), - ) - server._install_payout_ledger_artifact(delayed) - with server._job_cache_lock: - self.assertIs(server._payout_ledger_artifact, replaced) - - # Even a delayed pre-retarget build that clamped a fresher anchor - # must not displace it: a wrong-difficulty install would fail every - # reuse probe on the difficulty check, which never re-arms. - assert replaced.snapshot_anchor_ms is not None - leading = dataclass_replace( - installed, - generation=0, - prepared_monotonic=time.monotonic(), - snapshot_anchor_ms=int(replaced.snapshot_anchor_ms) + 5, - ) - server._install_payout_ledger_artifact(leading) - with server._job_cache_lock: - self.assertIs(server._payout_ledger_artifact, replaced) - - def test_fence_failure_rearms_artifact_preparation_with_debounce(self) -> None: - server, rpc = coordinator() - install_fake_bundle_builder(server) - artifacts = server.store_template_artifacts(dict(rpc.template)) - assert artifacts is not None - server.payout_artifact_rearm_min_seconds = 5.0 - server._prepare_payout_ledger_artifact( - server._payout_state_generation, - artifacts.network_difficulty, - ) - - class RecordingExecutor: - def __init__(self) -> None: - self.submissions = 0 - - def submit(self, _fn: object) -> Future[None]: - # Recorded but never run: the request slot stays observable. - self.submissions += 1 - return Future() - - executor = RecordingExecutor() - with server._payout_artifact_executor_lock: - server._payout_artifact_executor = executor # type: ignore[assignment] - - usable = server._usable_payout_ledger_artifact( - server._payout_state_generation, - artifacts.network_difficulty, - ) - self.assertIsNotNone(usable) - with server._payout_artifact_executor_lock: - self.assertIsNone(server._payout_artifact_requested) - - # Age the armed window past the re-anchor floor: the probe keeps - # serving the artifact while the debounced background re-anchor is - # scheduled. - server.payout_artifact_reanchor_seconds = 10.0 - with server._job_cache_lock: - armed = server._payout_ledger_artifact - assert armed is not None - assert armed.snapshot_anchor_ms is not None - server._payout_ledger_artifact = dataclass_replace( - armed, - snapshot_anchor_ms=int(armed.snapshot_anchor_ms) - 11_000, - ) - - self.assertIsNotNone( - server._usable_payout_ledger_artifact( - server._payout_state_generation, - artifacts.network_difficulty, - ) - ) - with server._payout_artifact_executor_lock: - self.assertEqual( - server._payout_artifact_requested, - ( - server._payout_state_generation, - int(artifacts.network_difficulty), - ), - ) - self.assertEqual(executor.submissions, 1) - # Every past-floor probe counts, even when the rearm itself is - # debounced -- the canary needs "serving past the floor" to be - # visible independently of rearm_scheduled. - self.assertEqual( - server.payout_artifact_event_counts["probe_past_floor"], - 1, - ) - # The one-slot worker dequeues the request; a served probe inside - # the interval must not slip a duplicate into the emptied slot. - server._payout_artifact_requested = None - - self.assertIsNotNone( - server._usable_payout_ledger_artifact( - server._payout_state_generation, - artifacts.network_difficulty, - ) - ) - with server._payout_artifact_executor_lock: - self.assertIsNone(server._payout_artifact_requested) - - # Once the interval has elapsed the next served probe re-arms. - with server._payout_artifact_executor_lock: - server._payout_artifact_last_schedule_monotonic = ( - time.monotonic() - 6.0 - ) - self.assertIsNotNone( - server._usable_payout_ledger_artifact( - server._payout_state_generation, - artifacts.network_difficulty, - ) - ) - with server._payout_artifact_executor_lock: - self.assertEqual( - server._payout_artifact_requested, - ( - server._payout_state_generation, - int(artifacts.network_difficulty), - ), - ) - - def test_aborted_speculative_rebuilds_back_off_and_reset(self) -> None: - ledger = AnchorRecordingLedger() - server, rpc = coordinator(ledger=ledger) - install_fake_bundle_builder(server) - artifacts = server.store_template_artifacts(dict(rpc.template)) - assert artifacts is not None - server.payout_artifact_rearm_min_seconds = 5.0 - - # A pending-commit floor older than the audit anchor ceiling aborts - # the rebuild before the window walk (the artifact would arm born - # expired); each abort doubles the re-arm interval instead of - # retrying the reward-window walk at the floor forever. - share = stamped_pending_share(now_ms() - 400_000) - server._ensure_pending_share_commit_state() - with server._pending_share_commit_lock: - server._pending_share_commit_floor[id(share)] = [ - share, - time.monotonic(), - False, - ] - server._prepare_payout_ledger_artifact(0, artifacts.network_difficulty) - with server._payout_artifact_executor_lock: - self.assertEqual(server._payout_artifact_rearm_backoff, 2) - server._prepare_payout_ledger_artifact(0, artifacts.network_difficulty) - with server._payout_artifact_executor_lock: - self.assertEqual(server._payout_artifact_rearm_backoff, 4) - self.assertEqual(ledger.snapshot_calls, 0) - - # Elapsed time beyond the floor but inside the scaled interval must - # not re-arm. - with server._payout_artifact_executor_lock: - server._payout_artifact_last_schedule_monotonic = ( - time.monotonic() - 10.0 - ) - server._rearm_payout_ledger_artifact_after_fence_failure( - 0, - artifacts.network_difficulty, - ) - with server._payout_artifact_executor_lock: - self.assertIsNone(server._payout_artifact_requested) - - # A rebuild that finally arms resets the backoff to the floor. - server._finish_pending_share_commit(share) - server._prepare_payout_ledger_artifact(0, artifacts.network_difficulty) - with server._job_cache_lock: - self.assertIsNotNone(server._payout_ledger_artifact) - with server._payout_artifact_executor_lock: - self.assertEqual(server._payout_artifact_rearm_backoff, 1) - - def test_payout_publication_resets_rearm_backoff(self) -> None: - server, rpc = coordinator() - install_fake_bundle_builder(server) - artifacts = server.store_template_artifacts(dict(rpc.template)) - assert artifacts is not None - server._pool_ready_latched = True - with server._payout_artifact_executor_lock: - server._payout_artifact_rearm_backoff = 8 - - # The accepted-block preview publishes a new payout generation whose - # candidate artifact installs through the atomic publication pointer - # swap; the accumulated backoff must release with it. - parent_hash = str(rpc.template["previousblockhash"]) - server._begin_accepted_block_payout_preview( - parent_hash, - block_height=int(rpc.template["height"]) - 1, - ) - server._publish_accepted_block_payout_preview( - parent_hash, - [ - { - "recipient_id": "miner-a", - "order_key": "miner-a", - "p2mr_program_hex": "11" * 32, - "balance_sats": 25, - } - ], - ) - - self.assertGreater(server._payout_state_generation, 0) - with server._job_cache_lock: - self.assertIsNotNone(server._payout_ledger_artifact) - with server._payout_artifact_executor_lock: - self.assertEqual(server._payout_artifact_rearm_backoff, 1) - - def test_landed_preview_suppresses_fence_failure_rearm(self) -> None: - server, rpc = coordinator() - install_fake_bundle_builder(server) - artifacts = server.store_template_artifacts(dict(rpc.template)) - assert artifacts is not None - server._prepare_payout_ledger_artifact( - server._payout_state_generation, - artifacts.network_difficulty, - ) - scheduled: list[tuple[int, int]] = [] - server._schedule_payout_ledger_artifact_preparation = ( # type: ignore[method-assign] - lambda generation, difficulty, **_kwargs: scheduled.append( - (generation, difficulty) - ) - ) - accepted_hash = "d0" * 32 - server._begin_accepted_block_payout_preview( - accepted_hash, - block_height=int(rpc.template["height"]) - 1, - ) - server._mark_accepted_block_payout_landed( - accepted_hash, - block_height=int(rpc.template["height"]) - 1, - ) - - # Age the armed window past the re-anchor floor: the probe keeps - # serving while attempting the background re-anchor. - server.payout_artifact_reanchor_seconds = 10.0 - with server._job_cache_lock: - armed = server._payout_ledger_artifact - assert armed is not None - assert armed.snapshot_anchor_ms is not None - server._payout_ledger_artifact = dataclass_replace( - armed, - snapshot_anchor_ms=int(armed.snapshot_anchor_ms) - 11_000, - ) - - self.assertIsNotNone( - server._usable_payout_ledger_artifact( - server._payout_state_generation, - artifacts.network_difficulty, - ) - ) - # A speculative rebuild would read database balances the landed - # prospective state supersedes; preparation resumes only through the - # durable-confirmation call site. - self.assertEqual(scheduled, []) - - def test_artifact_reuse_serves_past_install_age_to_anchor_ceiling( - self, - ) -> None: - server, rpc = coordinator() - install_fake_bundle_builder(server) - artifacts = server.store_template_artifacts(dict(rpc.template)) - assert artifacts is not None - server._prepare_payout_ledger_artifact( - server._payout_state_generation, - artifacts.network_difficulty, - ) - with server._job_cache_lock: - armed = server._payout_ledger_artifact - assert armed is not None - assert armed.snapshot_anchor_ms is not None - # The balances digest is memoized at construction so the serving - # probe and the reused-build fences never re-canonicalize the - # tuple on the request path. - self.assertEqual( - armed.prior_balances_sha256, - canonical_json_sha256(armed.prior_balances), - ) - - # A window whose anchor is past the re-anchor floor still serves -- - # post-anchor shares belong to the next window by construction, and - # validity is carried by the generation/balances fences, never by - # wall-clock age below the ceiling. (The re-anchor and - # anchor-ceiling env knobs wire these same attributes in __init__.) - server.payout_artifact_reanchor_seconds = 10.0 - server.ledger.miners = [*server.ledger.miners, "late-1", "late-2"] - with server._job_cache_lock: - server._payout_ledger_artifact = dataclass_replace( - armed, - snapshot_anchor_ms=int(armed.snapshot_anchor_ms) - 15_000, - ) - self.assertIsNotNone( - server._usable_payout_ledger_artifact( - server._payout_state_generation, - artifacts.network_difficulty, - rearm_on_fence_failure=False, - ) - ) - - # 2026-07-30 brownout regression: install age must never retire the - # window. A 10s install budget at the production build cadence (a - # shared build every 10-14s, a 4-8s walk) expired between - # consecutive builds, sending every build back through the - # synchronous reward-window walk. Only payout events and the audit - # ceiling end reuse. - with server._job_cache_lock: - server._payout_ledger_artifact = dataclass_replace( - armed, - prepared_monotonic=float(armed.prepared_monotonic) - 3600.0, - ) - self.assertIsNotNone( - server._usable_payout_ledger_artifact( - server._payout_state_generation, - artifacts.network_difficulty, - rearm_on_fence_failure=False, - ) - ) - - # The wall-clock anchor ceiling stays as the audit-facing backstop. - server.payout_artifact_max_anchor_age_seconds = 5.0 - with server._job_cache_lock: - server._payout_ledger_artifact = dataclass_replace( - armed, - snapshot_anchor_ms=int(armed.snapshot_anchor_ms) - 6_000, - ) - self.assertIsNone( - server._usable_payout_ledger_artifact( - server._payout_state_generation, - artifacts.network_difficulty, - rearm_on_fence_failure=False, - ) - ) - - def test_reuse_kill_switch_disables_probe_and_background_walks( - self, - ) -> None: - server, rpc = coordinator() - install_fake_bundle_builder(server) - artifacts = server.store_template_artifacts(dict(rpc.template)) - assert artifacts is not None - server._prepare_payout_ledger_artifact( - server._payout_state_generation, - artifacts.network_difficulty, - ) - with server._job_cache_lock: - assert server._payout_ledger_artifact is not None - - # Off: the probe refuses a perfectly valid armed artifact and the - # scheduler runs no background reward-window walks -- the disabled - # state restores pre-reuse delivery economics (the synchronous - # path keeps the re-landed anchor-selection semantics), so a - # delivery regression is an env flip instead of a rollback. - server.payout_artifact_reuse_enabled = False - self.assertIsNone( - server._usable_payout_ledger_artifact( - server._payout_state_generation, - artifacts.network_difficulty, - ) - ) - server._schedule_payout_ledger_artifact_preparation( - server._payout_state_generation, - artifacts.network_difficulty, - ) - with server._payout_artifact_executor_lock: - self.assertIsNone(server._payout_artifact_requested) - self.assertIsNone(server._payout_artifact_future) - - # Back on: the armed artifact serves again without a rebuild. - server.payout_artifact_reuse_enabled = True - self.assertIsNotNone( - server._usable_payout_ledger_artifact( - server._payout_state_generation, - artifacts.network_difficulty, - ) - ) - - # Off also gates arming itself: a prepared window is discarded at - # install, so nothing churns install events or re-keys the idle - # bundle fast path to an unusable generation while disabled. - server.payout_artifact_reuse_enabled = False - with server._job_cache_lock: - server._payout_ledger_artifact = None - server._prepare_payout_ledger_artifact( - server._payout_state_generation, - artifacts.network_difficulty, - ) - with server._job_cache_lock: - self.assertIsNone(server._payout_ledger_artifact) - - def test_anchor_ceiling_must_clear_reanchor_floor(self) -> None: - # One env var used to be able to re-create the 2026-07-30 incident: - # a ceiling the background re-anchor cannot beat (floor + debounce + - # the window walk) rejects every probe while the rebuild runs - # debounced in parallel -- walk-per-build. __init__ refuses the - # configuration at startup through this validation. - with self.assertRaises(SystemExit): - prism_coordinator_module.validate_payout_artifact_age_bounds( - 60.0, - 60.0, - ) - with self.assertRaises(SystemExit): - prism_coordinator_module.validate_payout_artifact_age_bounds( - 120.0, - 60.0, - ) - # The same margin applies to a custom build cadence; validating only - # the re-anchor floor allowed 299s builds under a 300s ceiling. - with self.assertRaises(SystemExit): - prism_coordinator_module.validate_payout_artifact_age_bounds( - 300.0, - 299.0, - ) - # The default split (300s ceiling, 60s floor) passes. - prism_coordinator_module.validate_payout_artifact_age_bounds( - 300.0, - 60.0, - ) - - def test_slow_window_walk_still_arms_reusable_artifact(self) -> None: - # 2026-07-29 regression: the reward-window walk takes multiple - # seconds at production volume, and freshness measured from the - # wall-clock anchor rejected every slow build on arrival. The walk - # here advances the wall clock past the freshness budget; the - # artifact must still arm, serve reuse, and reset the re-arm - # backoff. - clock = [now_ms()] - - class SlowWalkLedger(FakeLedger): - def snapshot_at_job_issue( - self, - anchor_job_issued_at_ms: int, - *, - window_weight: int | None = None, - ) -> list[FakeShare]: - result = super().snapshot_at_job_issue( - anchor_job_issued_at_ms, - window_weight=window_weight, - ) - clock[0] += 15_000 - return result - - server, rpc = coordinator(ledger=SlowWalkLedger()) - install_fake_bundle_builder(server) - artifacts = server.store_template_artifacts(dict(rpc.template)) - assert artifacts is not None - with server._payout_artifact_executor_lock: - server._payout_artifact_rearm_backoff = 4 - with patch( - "lab.prism.prism_coordinator.now_ms", - side_effect=lambda: clock[0], - ): - server._prepare_payout_ledger_artifact( - server._payout_state_generation, - artifacts.network_difficulty, - ) - with server._job_cache_lock: - armed = server._payout_ledger_artifact - self.assertIsNotNone(armed) - assert armed is not None - assert armed.snapshot_anchor_ms is not None - self.assertGreater( - clock[0] - int(armed.snapshot_anchor_ms), - 10_000, - ) - self.assertIs( - server._usable_payout_ledger_artifact( - server._payout_state_generation, - artifacts.network_difficulty, - ), - armed, - ) - with server._payout_artifact_executor_lock: - self.assertEqual(server._payout_artifact_rearm_backoff, 1) - metrics = server.metrics_payload() - self.assertIn( - 'qbit_prism_payout_artifact_events_total{event="installed"} 1', - metrics, - ) - self.assertIn( - 'qbit_prism_payout_artifact_events_total{event="built"} 1', - metrics, - ) - self.assertIn( - 'qbit_prism_payout_artifact_events_total{event="incremental"}', - metrics, - ) - self.assertIn( - 'qbit_prism_payout_artifact_events_total{event="self_check_failed"}', - metrics, - ) - self.assertIn( - 'qbit_prism_payout_artifact_events_total{event="found_block_cached"}', - metrics, - ) - - def test_born_expired_install_paces_backoff_instead_of_resetting( - self, - ) -> None: - # A walk that outlives the audit ceiling arms nothing. Crediting it - # as a success reset the re-arm backoff to the floor and re-walked - # the reward window continuously -- the 2026-07-29 livelock. It - # must pace exactly like a failed preparation. - clock = [now_ms()] - - class GlacialWalkLedger(FakeLedger): - def snapshot_at_job_issue( - self, - anchor_job_issued_at_ms: int, - *, - window_weight: int | None = None, - ) -> list[FakeShare]: - result = super().snapshot_at_job_issue( - anchor_job_issued_at_ms, - window_weight=window_weight, - ) - clock[0] += 400_000 - return result - - server, rpc = coordinator(ledger=GlacialWalkLedger()) - install_fake_bundle_builder(server) - artifacts = server.store_template_artifacts(dict(rpc.template)) - assert artifacts is not None - with patch( - "lab.prism.prism_coordinator.now_ms", - side_effect=lambda: clock[0], - ): - server._prepare_payout_ledger_artifact( - server._payout_state_generation, - artifacts.network_difficulty, - ) - with server._job_cache_lock: - self.assertIsNone(server._payout_ledger_artifact) - with server._payout_artifact_executor_lock: - self.assertEqual(server._payout_artifact_rearm_backoff, 2) - server._prepare_payout_ledger_artifact( - server._payout_state_generation, - artifacts.network_difficulty, - ) - with server._payout_artifact_executor_lock: - self.assertEqual(server._payout_artifact_rearm_backoff, 4) - with server._payout_artifact_executor_lock: - self.assertEqual( - server.payout_artifact_event_counts["born_expired"], - 2, - ) - - def test_equal_anchor_reprove_credits_freshness(self) -> None: - # A pending-commit floor pins the snapshot anchor, so a fence-failure - # rebuild re-proves the same window at the SAME anchor. That re-prove - # must advance the freshness clock: no fresher window is - # constructible while the floor holds, and rejecting the credit - # would re-walk the reward window (with backoff reset each round) - # until the audit ceiling -- the livelock shape this re-land - # removes. - server, rpc = coordinator() - install_fake_bundle_builder(server) - artifacts = server.store_template_artifacts(dict(rpc.template)) - assert artifacts is not None - share = stamped_pending_share(now_ms() - 5) - server._ensure_pending_share_commit_state() - with server._pending_share_commit_lock: - server._pending_share_commit_floor[id(share)] = [ - share, - time.monotonic(), - False, - ] - try: - server._prepare_payout_ledger_artifact( - server._payout_state_generation, - artifacts.network_difficulty, - ) - with server._job_cache_lock: - armed = server._payout_ledger_artifact - assert armed is not None - server._payout_ledger_artifact = dataclass_replace( - armed, - prepared_monotonic=float(armed.prepared_monotonic) - 11.0, - ) - # Install age no longer gates the probe (2026-07-30 brownout - # regression); the armed window keeps serving while pinned. - self.assertIsNotNone( - server._usable_payout_ledger_artifact( - server._payout_state_generation, - artifacts.network_difficulty, - rearm_on_fence_failure=False, - ) - ) - - server._prepare_payout_ledger_artifact( - server._payout_state_generation, - artifacts.network_difficulty, - ) - with server._job_cache_lock: - reproved = server._payout_ledger_artifact - assert reproved is not None - self.assertEqual(reproved.generation, armed.generation) - self.assertEqual( - reproved.snapshot_anchor_ms, - armed.snapshot_anchor_ms, - ) - self.assertIsNotNone( - server._usable_payout_ledger_artifact( - server._payout_state_generation, - artifacts.network_difficulty, - ) - ) - finally: - server._finish_pending_share_commit(share) - - def test_publication_restamps_candidate_artifact_freshness(self) -> None: - # A payout-state candidate builds its artifact before the atomic - # publication, and the delivery-gate drain between the two can - # outlive the reuse budget. The install must restamp freshness so a - # freshly published generation never arms an already-stale artifact. - server, rpc = coordinator() - install_fake_bundle_builder(server) - artifacts = server.store_template_artifacts(dict(rpc.template)) - assert artifacts is not None - server._pool_ready_latched = True - real_build = server._build_payout_ledger_artifact - - def slow_publication_build(*args: object) -> object: - built = real_build(*args) - if built is None: - return None - return dataclass_replace( - built, - prepared_monotonic=time.monotonic() - 11.0, - ) - - server._build_payout_ledger_artifact = slow_publication_build # type: ignore[method-assign] - parent_hash = str(rpc.template["previousblockhash"]) - server._begin_accepted_block_payout_preview( - parent_hash, - block_height=int(rpc.template["height"]) - 1, - ) - server._publish_accepted_block_payout_preview( - parent_hash, - [ - { - "recipient_id": "miner-a", - "order_key": "miner-a", - "p2mr_program_hex": "11" * 32, - "balance_sats": 25, - } - ], - ) - - self.assertGreater(server._payout_state_generation, 0) - usable = server._usable_payout_ledger_artifact( - server._payout_state_generation, - artifacts.network_difficulty, - ) - self.assertIsNotNone(usable) - - def test_usable_probe_survives_equal_window_freshness_restamp( - self, - ) -> None: - # An equal-window re-prove restamps the armed artifact by replacing - # the object while a reuse probe is hashing balances outside the - # cache lock. The probe must treat the restamped copy as the same - # armed window instead of failing closed into a synchronous - # reward-window walk: under a pinned pending-commit floor the - # restamp IS the intentional recovery path, so the race is routine. - server, rpc = coordinator() - install_fake_bundle_builder(server) - artifacts = server.store_template_artifacts(dict(rpc.template)) - assert artifacts is not None - server._prepare_payout_ledger_artifact( - server._payout_state_generation, - artifacts.network_difficulty, - ) - with server._job_cache_lock: - armed = server._payout_ledger_artifact - assert armed is not None - # Strip the memoized balances digest: the race window this test - # drives opens while the probe hashes balances outside the cache - # lock, which only happens on the fallback path for artifacts - # without a memo (legacy or test-constructed). The restamp - # admission below must keep protecting that path. - armed = dataclass_replace(armed, prior_balances_sha256=None) - server._payout_ledger_artifact = armed - - real_sha256 = prism_coordinator_module.canonical_json_sha256 - restamped_once = [False] - - def restamp_during_hash(value: object) -> str: - if not restamped_once[0]: - restamped_once[0] = True - self.assertTrue( - server._install_payout_ledger_artifact( - dataclass_replace( - armed, - generation=0, - prepared_monotonic=time.monotonic(), - ) - ) - ) - return real_sha256(value) - - with patch.object( - prism_coordinator_module, - "canonical_json_sha256", - restamp_during_hash, - ): - usable = server._usable_payout_ledger_artifact( - server._payout_state_generation, - artifacts.network_difficulty, - ) - self.assertTrue(restamped_once[0]) - with server._job_cache_lock: - restamped = server._payout_ledger_artifact - assert restamped is not None - self.assertIsNot(restamped, armed) - self.assertIsNotNone(usable) - self.assertIs(usable, restamped) - self.assertEqual(restamped.generation, armed.generation) - - def test_publication_discards_born_expired_candidate_artifact( - self, - ) -> None: - # Candidate construction plus the delivery-gate drain can push the - # candidate artifact's declared anchor past the audit ceiling. The - # atomic publication must apply the same born-expired admission rule - # as _install_payout_ledger_artifact: arming the artifact would fail - # every reuse probe on anchor age -- with the re-arm suppressed - # while the accepted preview awaits durability -- instead of letting - # the post-publication probe schedule recovery. - server, rpc = coordinator() - install_fake_bundle_builder(server) - artifacts = server.store_template_artifacts(dict(rpc.template)) - assert artifacts is not None - server._pool_ready_latched = True - real_build = server._build_payout_ledger_artifact - ceiling_ms = server._payout_artifact_max_anchor_age_ms() - - def expired_anchor_build(*args: object) -> object: - built = real_build(*args) - if built is None: - return None - assert built.snapshot_anchor_ms is not None - return dataclass_replace( - built, - snapshot_anchor_ms=int(built.snapshot_anchor_ms) - - int(ceiling_ms) - - 1_000, - ) - - server._build_payout_ledger_artifact = expired_anchor_build # type: ignore[method-assign] - parent_hash = str(rpc.template["previousblockhash"]) - server._begin_accepted_block_payout_preview( - parent_hash, - block_height=int(rpc.template["height"]) - 1, - ) - generation_before = server._payout_state_generation - server._publish_accepted_block_payout_preview( - parent_hash, - [ - { - "recipient_id": "miner-a", - "order_key": "miner-a", - "p2mr_program_hex": "11" * 32, - "balance_sats": 25, - } - ], - ) - self.assertGreater(server._payout_state_generation, generation_before) - with server._job_cache_lock: - self.assertIsNone(server._payout_ledger_artifact) - with server._payout_artifact_executor_lock: - born_expired = int( - server.payout_artifact_event_counts.get("born_expired", 0) - ) - self.assertGreaterEqual(born_expired, 1) - self.assertIsNone( - server._usable_payout_ledger_artifact( - server._payout_state_generation, - artifacts.network_difficulty, - ) - ) - - def test_publication_records_installed_event(self) -> None: - # The atomic publication arms the candidate artifact through its own - # pointer swap, not _install_payout_ledger_artifact. It must still - # count in the installed lifecycle event family, or every - # publication-path install is invisible to the observability the - # event counter exists for. - server, rpc = coordinator() - install_fake_bundle_builder(server) - artifacts = server.store_template_artifacts(dict(rpc.template)) - assert artifacts is not None - server._pool_ready_latched = True - with server._payout_artifact_executor_lock: - installed_before = int( - server.payout_artifact_event_counts.get("installed", 0) - ) - parent_hash = str(rpc.template["previousblockhash"]) - server._begin_accepted_block_payout_preview( - parent_hash, - block_height=int(rpc.template["height"]) - 1, - ) - server._publish_accepted_block_payout_preview( - parent_hash, - [ - { - "recipient_id": "miner-a", - "order_key": "miner-a", - "p2mr_program_hex": "11" * 32, - "balance_sats": 25, - } - ], - ) - with server._job_cache_lock: - self.assertIsNotNone(server._payout_ledger_artifact) - with server._payout_artifact_executor_lock: - installed_after = int( - server.payout_artifact_event_counts.get("installed", 0) - ) - self.assertEqual(installed_after, installed_before + 1) - - def test_reused_anchor_bundle_preview_matches_fresh_build_through_guard( - self, - ) -> None: - # The #50 payout-preview guard compares the preview computed from - # the issued job against one recomputed at landing. A reused-anchor - # bundle must therefore reproduce, byte for byte, what a fresh - # ledger read at the same anchor produces -- the audit - # reproducibility contract the artifact's snapshot_anchor_ms - # declaration documents. - programs = { - "miner-a": "aa" * 32, - "miner-b": "bb" * 32, - "miner-c": "cc" * 32, - "miner-d": "dd" * 32, - } - - class AnchorScopedLedger(FakeLedger): - """Window reads respect the anchor like the real ledger.""" - - def __init__(self) -> None: - super().__init__(miners=[]) - self.stamped: list[tuple[int, FakeShare]] = [] - - def add_share(self, miner_id: str, accepted_at_ms: int) -> None: - self.stamped.append( - ( - int(accepted_at_ms), - FakeShare( - miner_id=miner_id, - share_seq=len(self.stamped) + 1, - ), - ) - ) - - def accepted_share_stats(self) -> dict[str, int]: - self.stats_calls += 1 - return { - "accepted_share_count": len(self.stamped), - "distinct_miner_count": len( - {share.miner_id for _, share in self.stamped} - ), - } - - def snapshot_at_job_issue( - self, - anchor_job_issued_at_ms: int, - *, - window_weight: int | None = None, - ) -> list[FakeShare]: - self.snapshot_calls += 1 - return [ - share - for stamp, share in self.stamped - if stamp <= int(anchor_job_issued_at_ms) - ] - - anchor_ms = now_ms() - 50 - ledger = AnchorScopedLedger() - ledger.add_share("miner-a", anchor_ms - 30) - ledger.add_share("miner-b", anchor_ms - 20) - ledger.add_share("miner-c", anchor_ms - 10) - server, rpc = coordinator(ledger=ledger) - - def preview_bundle_builder(**kwargs: object) -> dict[str, object]: - suffix_hex = str(kwargs["coinbase_script_sig_suffix_hex"]) - weights: dict[str, int] = {} - for share in kwargs["shares"]: # type: ignore[union-attr] - miner = str(share["miner_id"]) # type: ignore[index] - weights[miner] = weights.get(miner, 0) + 1 - return { - "found_block": dict(kwargs["found_block"]), # type: ignore[call-overload] - "payout_policy_manifest": { - "accounts": [ - { - "account_type": "miner", - "recipient_id": miner, - "order_key": miner, - "p2mr_program_hex": programs[miner], - "carry_forward_balance_sats": 1_000 * weight, - } - for miner, weight in sorted(weights.items()) - ] - }, - "signed_coinbase_manifest": { - "manifest": { - "coinbase_tx_hex": synthetic_manifest_coinbase_hex( - suffix_hex - ), - } - }, - } - - server.build_audit_bundle = preview_bundle_builder # type: ignore[method-assign] - - first = server.store_template_artifacts(dict(rpc.template)) - assert first is not None - with server._job_cache_lock: - server._job_build_issued_at_ms[first.generation] = anchor_ms - - seeded_bundle = server.shared_job_bundle(first, mode="ready") - self.assertEqual(seeded_bundle.payout_artifact_generation, 0) - with server._job_cache_lock: - artifact = server._payout_ledger_artifact - assert artifact is not None - self.assertEqual(artifact.snapshot_anchor_ms, anchor_ms) - - # Shares keep landing after the anchor; the durable count moves past - # the artifact's, which must not matter for reuse. - ledger.add_share("miner-a", anchor_ms + 20) - ledger.add_share("miner-d", anchor_ms + 25) - - second = server.store_template_artifacts( - base_template(height=11, prevhash="22" * 32) - ) - assert second is not None - reused = server.shared_job_bundle(second, mode="ready") - self.assertEqual(reused.payout_artifact_generation, artifact.generation) - self.assertEqual( - reused.found_block["anchor_job_issued_at_ms"], - anchor_ms, - ) - - # Control: a fresh synchronous build pinned to the same anchor pays - # its own window read and must reproduce the reused output exactly, - # excluding the post-anchor shares deterministically. - with server._job_cache_lock: - server._payout_ledger_artifact = None - server._job_bundle_cache.clear() - third = server.store_template_artifacts( - base_template(height=12, prevhash="33" * 32) - ) - assert third is not None - with server._job_cache_lock: - server._job_build_issued_at_ms[third.generation] = anchor_ms - snapshot_calls_before = ledger.snapshot_calls - fresh = server.shared_job_bundle(third, mode="ready") - self.assertEqual(ledger.snapshot_calls, snapshot_calls_before + 1) - self.assertEqual(fresh.payout_artifact_generation, 0) - - self.assertEqual(reused.shares_json, fresh.shares_json) - self.assertEqual( - fresh.found_block["anchor_job_issued_at_ms"], - anchor_ms, - ) - assert reused.prospective_prior_balances is not None - assert fresh.prospective_prior_balances is not None - self.assertEqual( - canonical_json_text(list(reused.prospective_prior_balances)), - canonical_json_text(list(fresh.prospective_prior_balances)), - ) - - # A window at a fresher anchor covers the post-anchor shares and - # produces a different preview; built now to prove the guard - # equality below is load-bearing. - with server._job_cache_lock: - server._payout_ledger_artifact = None - server._job_bundle_cache.clear() - fourth = server.store_template_artifacts( - base_template(height=13, prevhash="44" * 32) - ) - assert fourth is not None - with server._job_cache_lock: - server._job_build_issued_at_ms[fourth.generation] = anchor_ms + 30 - divergent = server.shared_job_bundle(fourth, mode="ready") - assert divergent.prospective_prior_balances is not None - self.assertNotEqual( - canonical_json_text(list(divergent.prospective_prior_balances)), - canonical_json_text(list(reused.prospective_prior_balances)), - ) - - # The #50 guard sequence at landing: the issued preview (from the - # reused-anchor job) publishes first; the verified preview - # recomputed at the same anchor must then publish idempotently. - block_hash = "d1" * 32 - server._begin_accepted_block_payout_preview( - block_hash, - block_height=int(rpc.template["height"]), - ) - issued = server._materialize_prior_balance_preview( - reused.prospective_prior_balances - ) - server._publish_accepted_block_payout_preview(block_hash, issued) - generation_after_issued = server._payout_state_generation - verified = server._materialize_prior_balance_preview( - fresh.prospective_prior_balances - ) - server._publish_accepted_block_payout_preview(block_hash, verified) - self.assertEqual( - server._payout_state_generation, - generation_after_issued, - ) - with server._accepted_block_payout_preview_condition: - transition = server._accepted_block_payout_previews[block_hash] - self.assertEqual( - transition.preview, - server._serialize_prior_balance_preview(issued), - ) - - # A preview from any other anchor's window trips the guard exactly - # as submit_block_candidate's landing rebuild would. - with self.assertRaisesRegex(RuntimeError, "changed during retry"): - server._publish_accepted_block_payout_preview( - block_hash, - server._materialize_prior_balance_preview( - divergent.prospective_prior_balances - ), - ) - - -class IncrementalPayoutArtifactTests(unittest.TestCase): - def configured_server( - self, - ) -> tuple[PrismCoordinator, IncrementalRecordingLedger, object]: - ledger = IncrementalRecordingLedger() - append_incremental_share(ledger, share_seq=1, accepted_at_ms=999_900) - append_incremental_share(ledger, share_seq=2, accepted_at_ms=999_910) - append_incremental_share(ledger, share_seq=3, accepted_at_ms=999_920) - server, rpc = coordinator(ledger=ledger) - artifacts = server.store_template_artifacts(dict(rpc.template)) - assert artifacts is not None - server._pool_ready_latched = True - server.payout_artifact_min_build_interval_seconds = 60.0 - server.payout_artifact_full_rescan_seconds = 3_600.0 - return server, ledger, artifacts - - @staticmethod - def replay_shaped_append_entry( - *, - accepted_at_ms: int, - block_hash_hex: str, - ) -> PendingShareAppend: - persisted = stamped_pending_share(accepted_at_ms) - pending = PendingShare(**dict(vars(persisted))) - return PendingShareAppend( - pending_share=pending, - username="miner-a", - job_id=pending.job_id, - block_hash_hex=block_hash_hex, - collection_only=False, - credit_policy=None, - candidate_intent={ - "block_hash_hex": block_hash_hex, - "pending_share": dict(vars(pending)), - "credit_share_on_accept": True, - }, - ) - - def test_difficulty_drift_within_band_keeps_incremental_window( - self, - ) -> None: - # ASERT retargets every block, so the snapshot weight drifts a - # fraction of a percent per tip change. Within the retained margin - # that drift must ride the incremental window: the pre-band behavior - # ran the full oracle (a multi-second writer-lock hold) on every - # block, which starved landing-window ledger reads (mainnet alert - # #236). - server, ledger, artifacts = self.configured_server() - clock_ms = [1_000_000] - with patch( - "lab.prism.prism_coordinator.now_ms", - side_effect=lambda: clock_ms[0], - ): - initial = server._build_payout_ledger_artifact( - 0, 0, artifacts.network_difficulty - ) - assert initial is not None - self.assertEqual(initial.window_build_mode, "full_rescan") - self.assertEqual(ledger.full_snapshot_calls, 1) - - server.payout_artifact_min_build_interval_seconds = 0.0 - clock_ms[0] = 1_000_020 - drifted_difficulty = ( - int(artifacts.network_difficulty) * 101 + 99 - ) // 100 - drifted = server._build_payout_ledger_artifact( - 0, 0, drifted_difficulty - ) - assert drifted is not None - self.assertEqual(drifted.window_build_mode, "incremental") - self.assertEqual(ledger.full_snapshot_calls, 1) - # The artifact still stamps the live difficulty: the exact - # 8 x difficulty reward cutoff is re-derived downstream, so the - # cached superset only has to cover it. - self.assertEqual( - drifted.network_difficulty, drifted_difficulty - ) - - def test_difficulty_drift_beyond_band_floor_forces_full_rescan( - self, - ) -> None: - server, ledger, artifacts = self.configured_server() - clock_ms = [1_000_000] - with patch( - "lab.prism.prism_coordinator.now_ms", - side_effect=lambda: clock_ms[0], - ): - initial = server._build_payout_ledger_artifact( - 0, 0, artifacts.network_difficulty - ) - assert initial is not None - server.payout_artifact_min_build_interval_seconds = 0.0 - clock_ms[0] = 1_000_020 - # Cached weight below 5/8 of the live snapshot weight: the - # retained superset no longer covers the live reward window - # with headroom, so the oracle must re-center. - surged = server._build_payout_ledger_artifact( - 0, 0, int(artifacts.network_difficulty) * 2 - ) - assert surged is not None - self.assertEqual(surged.window_build_mode, "full_rescan") - self.assertEqual( - surged.window_full_rescan_reason, - "snapshot_window_weight_out_of_band", - ) - self.assertEqual(ledger.full_snapshot_calls, 2) - - def test_difficulty_drop_beyond_band_ceiling_forces_full_rescan( - self, - ) -> None: - server, ledger, artifacts = self.configured_server() - clock_ms = [1_000_000] - with patch( - "lab.prism.prism_coordinator.now_ms", - side_effect=lambda: clock_ms[0], - ): - initial = server._build_payout_ledger_artifact( - 0, 0, artifacts.network_difficulty - ) - assert initial is not None - server.payout_artifact_min_build_interval_seconds = 0.0 - clock_ms[0] = 1_000_020 - # Cached weight more than twice the live snapshot weight: the - # stale superset is bounded above, so a difficulty collapse - # re-centers rather than serving an oversized window forever. - collapsed = server._build_payout_ledger_artifact( - 0, 0, max(1, int(artifacts.network_difficulty) // 3) - ) - assert collapsed is not None - self.assertEqual(collapsed.window_build_mode, "full_rescan") - self.assertEqual( - collapsed.window_full_rescan_reason, - "snapshot_window_weight_out_of_band", - ) - self.assertEqual(ledger.full_snapshot_calls, 2) - - def test_self_check_under_small_drift_matches_and_keeps_weight( - self, - ) -> None: - # The periodic runtime-check compares like-for-like at the cached - # weight (in-band ASERT drift is not a pipeline mismatch) and, on - # ordinary drift, adopts the oracle window still at the cached - # weight: adopting a digest the daemon never prepared would answer - # needs_full on the next advance and re-pay the full oracle read - # at every runtime-check. - server, ledger, artifacts = self.configured_server() - clock_ms = [1_000_000] - with patch( - "lab.prism.prism_coordinator.now_ms", - side_effect=lambda: clock_ms[0], - ): - initial = server._build_payout_ledger_artifact( - 0, 0, artifacts.network_difficulty - ) - assert initial is not None - server.payout_artifact_min_build_interval_seconds = 0.0 - server.payout_artifact_full_rescan_seconds = 0.0 - clock_ms[0] = 1_000_020 - drifted_difficulty = ( - int(artifacts.network_difficulty) * 101 + 99 - ) // 100 - checked = server._build_payout_ledger_artifact( - 0, 0, drifted_difficulty - ) - assert checked is not None - self.assertEqual(checked.window_build_mode, "self_check_match") - window = server._incremental_payout_artifact_window - assert window is not None - self.assertEqual( - int(window.window.window_weight), - PRISM_REWARD_WINDOW_MULTIPLIER - * PRISM_SNAPSHOT_WINDOW_MARGIN - * int(artifacts.network_difficulty), - ) - self.assertEqual(ledger.full_snapshot_calls, 2) - - def test_self_check_recenters_only_a_well_oversized_weight( - self, - ) -> None: - # When difficulty falls far enough that the cached weight sits more - # than 25% above the live snapshot weight, the runtime-check adopts - # the oracle window at the live weight so a falling market cannot - # pin an oversized oracle scan; the match verdict still compares at - # the cached weight. - server, ledger, artifacts = self.configured_server() - clock_ms = [1_000_000] - with patch( - "lab.prism.prism_coordinator.now_ms", - side_effect=lambda: clock_ms[0], - ): - initial = server._build_payout_ledger_artifact( - 0, 0, artifacts.network_difficulty - ) - assert initial is not None - server.payout_artifact_min_build_interval_seconds = 0.0 - server.payout_artifact_full_rescan_seconds = 0.0 - clock_ms[0] = 1_000_020 - # cached/live = 4/3 > 5/4, still inside the 2x band ceiling. - collapsed_difficulty = max( - 1, int(artifacts.network_difficulty) * 3 // 4 - ) - checked = server._build_payout_ledger_artifact( - 0, 0, collapsed_difficulty - ) - assert checked is not None - self.assertEqual(checked.window_build_mode, "self_check_match") - window = server._incremental_payout_artifact_window - assert window is not None - self.assertEqual( - int(window.window.window_weight), - PRISM_REWARD_WINDOW_MULTIPLIER - * PRISM_SNAPSHOT_WINDOW_MARGIN - * collapsed_difficulty, - ) - self.assertEqual(ledger.full_snapshot_calls, 2) - - def test_late_visible_replay_append_forces_next_build_to_full_oracle( - self, - ) -> None: - server, ledger, artifacts = self.configured_server() - clock_ms = [1_000_000] - with patch( - "lab.prism.prism_coordinator.now_ms", - side_effect=lambda: clock_ms[0], - ): - initial = server._build_payout_ledger_artifact( - 0, 0, artifacts.network_difficulty - ) - assert initial is not None - self.assertEqual(initial.window_build_mode, "full_rescan") - - server.payout_artifact_min_build_interval_seconds = 0.0 - clock_ms[0] = 1_000_020 - advanced = server._build_payout_ledger_artifact( - 0, 0, artifacts.network_difficulty - ) - assert advanced is not None - self.assertEqual(advanced.window_build_mode, "incremental") - self.assertTrue(server._install_payout_ledger_artifact(advanced)) - anchor_ms = int(advanced.snapshot_anchor_ms) - - entry = self.replay_shaped_append_entry( - accepted_at_ms=anchor_ms - 10, - block_hash_hex="44" * 32, - ) - self.assertTrue(server._append_share_batch([entry])) - self.assertIsNone(server._incremental_payout_artifact_window) - with server._job_cache_lock: - self.assertIsNone(server._payout_ledger_artifact) - self.assertFalse(server._install_payout_ledger_artifact(advanced)) - self.assertIsNone( - server._cached_found_block_payout_artifact( - base_generation=0, - artifact_payout_state_generation=1, - network_difficulty=artifacts.network_difficulty, - fallback_reason="prepare_lock_busy", - ) - ) - - clock_ms[0] = 1_000_040 - rebuilt = server._build_payout_ledger_artifact( - 0, 0, artifacts.network_difficulty - ) - - assert rebuilt is not None - self.assertEqual(rebuilt.window_build_mode, "full_rescan") - self.assertEqual( - rebuilt.window_full_rescan_reason, - "late_visible_append", - ) - self.assertEqual(ledger.full_snapshot_calls, 2) - self.assertIn( - entry.pending_share.share_id, - {str(share["share_id"]) for share in rebuilt.shares_json}, - ) - - def test_replay_shaped_batch_commits_under_the_landing_fence(self) -> None: - # The durable append and its epoch bump must share the landing fence - # boundary: a bump that only starts after append_batch() returned - # leaves a gap where a landing verifies the pre-bump epoch and enters - # submitblock with the row already durable but unpaid. A batch holding - # a replay-shaped row therefore holds the fence across the ledger - # commit itself; an ordinary batch stays off the lock entirely. - server, ledger, artifacts = self.configured_server() - clock_ms = [1_000_000] - with patch( - "lab.prism.prism_coordinator.now_ms", - side_effect=lambda: clock_ms[0], - ): - initial = server._build_payout_ledger_artifact( - 0, 0, artifacts.network_difficulty - ) - assert initial is not None - server.payout_artifact_min_build_interval_seconds = 0.0 - clock_ms[0] = 1_000_020 - advanced = server._build_payout_ledger_artifact( - 0, 0, artifacts.network_difficulty - ) - assert advanced is not None - self.assertTrue(server._install_payout_ledger_artifact(advanced)) - anchor_ms = int(advanced.snapshot_anchor_ms) - - fence_held_during_commit: list[bool] = [] - original_append_batch = ledger.append_batch - - def recording_append_batch(entries: list[object]) -> list[object]: - fence_held_during_commit.append( - server._payout_append_landing_fence_lock.locked() - ) - return original_append_batch(entries) - - ledger.append_batch = recording_append_batch # type: ignore[method-assign] - - fresh = self.replay_shaped_append_entry( - accepted_at_ms=anchor_ms + 50, - block_hash_hex="88" * 32, - ) - self.assertTrue(server._append_share_batch([fresh])) - self.assertEqual(fence_held_during_commit, [False]) - with server._job_cache_lock: - self.assertEqual( - server._payout_ledger_append_invalidation_epoch, 0 - ) - - predating = self.replay_shaped_append_entry( - accepted_at_ms=anchor_ms - 10, - block_hash_hex="99" * 32, - ) - self.assertTrue(server._append_share_batch([predating])) - self.assertEqual(fence_held_during_commit, [False, True]) - with server._job_cache_lock: - self.assertEqual( - server._payout_ledger_append_invalidation_epoch, 1 - ) - self.assertIsNone(server._payout_ledger_artifact) - self.assertIsNone(server._incremental_payout_artifact_window) - - def test_replay_append_during_cold_scan_invalidates_inflight_walk( - self, - ) -> None: - # A cold-start walk exposes no cached window or armed artifact while - # its ledger read runs. A replayed row committing mid-walk with - # timestamps below the walk's anchor must still record an - # invalidation, or the walk installs a window that can never - # rediscover the row through delta reads. - server, ledger, artifacts = self.configured_server() - clock_ms = [1_000_000] - entry = self.replay_shaped_append_entry( - accepted_at_ms=999_950, - block_hash_hex="55" * 32, - ) - original_snapshot = ledger.snapshot_at_job_issue - raced = threading.Event() - append_threads: list[threading.Thread] = [] - - def racing_snapshot( - anchor_job_issued_at_ms: int, - *, - window_weight: int | None = None, - ) -> list[object]: - # Capture the read result first: the replayed row committing - # afterwards models a database snapshot taken before the commit. - records = original_snapshot( - anchor_job_issued_at_ms, - window_weight=window_weight, - ) - if not raced.is_set(): - raced.set() - thread = threading.Thread( - target=server._append_share_batch, - args=([entry],), - ) - thread.start() - append_threads.append(thread) - deadline = time.monotonic() + 2.0 - while time.monotonic() < deadline: - with server._job_cache_lock: - if server._payout_ledger_append_invalidation_epoch: - break - time.sleep(0.001) - return records - - ledger.snapshot_at_job_issue = racing_snapshot # type: ignore[method-assign] - with patch( - "lab.prism.prism_coordinator.now_ms", - side_effect=lambda: clock_ms[0], - ): - initial = server._build_payout_ledger_artifact( - 0, 0, artifacts.network_difficulty - ) - assert initial is not None - self.assertEqual(initial.window_build_mode, "full_rescan") - self.assertNotIn( - entry.pending_share.share_id, - {str(share["share_id"]) for share in initial.shares_json}, - ) - self.assertTrue(append_threads) - append_threads[0].join(timeout=2.0) - self.assertFalse(append_threads[0].is_alive()) - with server._job_cache_lock: - self.assertEqual( - server._payout_ledger_append_invalidation_epoch, 1 - ) - self.assertFalse(server._install_payout_ledger_artifact(initial)) - with server._job_cache_lock: - self.assertIsNone(server._payout_ledger_artifact) - - server.payout_artifact_min_build_interval_seconds = 0.0 - clock_ms[0] = 1_000_040 - rebuilt = server._build_payout_ledger_artifact( - 0, 0, artifacts.network_difficulty - ) - - assert rebuilt is not None - self.assertEqual(rebuilt.window_build_mode, "full_rescan") - self.assertEqual( - rebuilt.window_full_rescan_reason, - "late_visible_append", - ) - self.assertIn( - entry.pending_share.share_id, - {str(share["share_id"]) for share in rebuilt.shares_json}, - ) - - def test_inflight_scan_anchor_records_invalidation_without_windows( - self, - ) -> None: - # Directly exercise the published in-flight anchor: with no cached - # window and no armed artifact, only a predating append may advance - # the epoch, and fresh shares above the anchor must leave the walk - # undisturbed. - server, _ledger, _artifacts = self.configured_server() - server._ensure_job_cache_state() - predating = stamped_pending_share(999_950) - fresh = stamped_pending_share(1_000_050) - - server._invalidate_incremental_payout_window_for_append(predating) - with server._job_cache_lock: - self.assertEqual(server._payout_ledger_append_invalidation_epoch, 0) - - with patch( - "lab.prism.prism_coordinator.now_ms", - side_effect=lambda: 1_000_000, - ): - token = server._expose_inflight_scan_anchor(999_999) - try: - server._invalidate_incremental_payout_window_for_append(fresh) - with server._job_cache_lock: - self.assertEqual( - server._payout_ledger_append_invalidation_epoch, 0 - ) - server._invalidate_incremental_payout_window_for_append(predating) - finally: - server._retire_inflight_scan_anchor(token) - with server._job_cache_lock: - self.assertEqual(server._payout_ledger_append_invalidation_epoch, 1) - self.assertEqual(server._payout_window_inflight_scan_anchors, {}) - self.assertEqual( - server._incremental_payout_artifact_window_invalidation_reason, - "late_visible_append", - ) - - def test_replay_append_during_bundle_construction_invalidates_build( - self, - ) -> None: - # A cold synchronous ready build exposes no cached window or armed - # artifact once its ledger read returns, while manifest construction - # and cache publication still lie ahead. A replayed row committing in - # that gap with timestamps below the read's anchor must still advance - # the epoch -- the exposure outlives the read -- so the publication - # fence supersedes the bundle instead of seeding a window that delta - # reads can never rediscover the row from. - server, _ledger, _artifacts = self.configured_server() - install_fake_bundle_builder(server) - clock_ms = [1_000_000] - entry = self.replay_shaped_append_entry( - accepted_at_ms=999_950, - block_hash_hex="66" * 32, - ) - raced = threading.Event() - original_build = server.build_audit_bundle - - def racing_build(**kwargs: object) -> dict[str, object]: - # The ledger read has already returned; the replayed row commits - # while the coinbase manifest is being constructed. - if not raced.is_set(): - raced.set() - server._append_share_batch([entry]) - return original_build(**kwargs) - - server.build_audit_bundle = racing_build # type: ignore[method-assign] - with patch( - "lab.prism.prism_coordinator.now_ms", - side_effect=lambda: clock_ms[0], - ): - with self.assertRaises(JobBuildSuperseded): - server.build_shared_job_bundle( - server.current_template_artifacts(), - worker(), - ) - with server._job_cache_lock: - self.assertEqual( - server._payout_ledger_append_invalidation_epoch, 1 - ) - self.assertIsNone(server._payout_ledger_artifact) - - # A fresh build keyed to the advanced epoch re-reads the ledger - # and rediscovers the replayed row. - with server._job_cache_lock: - server._job_build_issued_at_ms.clear() - clock_ms[0] = 1_000_040 - rebuilt = server.build_shared_job_bundle( - server.current_template_artifacts(), - worker(), - ) - self.assertIn( - entry.pending_share.share_id, - {str(share["share_id"]) for share in rebuilt.shares_json}, - ) - - def test_seeded_artifact_exposure_survives_until_install_fence( - self, - ) -> None: - # The synchronous build's exposed scan anchor rides its seeded - # artifact: a replayed row committing after the bundle returned but - # before cache publication installs the seed must advance the epoch, - # and the doomed seed's install fence must discard it (retiring the - # exposure) instead of arming the pre-append window. - server, _ledger, _artifacts = self.configured_server() - install_fake_bundle_builder(server) - with patch( - "lab.prism.prism_coordinator.now_ms", - side_effect=lambda: 1_000_000, - ): - bundle = server.build_shared_job_bundle( - server.current_template_artifacts(), - worker(), - ) - seed = bundle.prepared_ledger_artifact - assert seed is not None - self.assertIsNotNone(seed.inflight_scan_anchor_token) - - entry = self.replay_shaped_append_entry( - accepted_at_ms=999_950, - block_hash_hex="77" * 32, - ) - self.assertTrue(server._append_share_batch([entry])) - with server._job_cache_lock: - self.assertEqual( - server._payout_ledger_append_invalidation_epoch, 1 - ) - - self.assertFalse(server._install_payout_ledger_artifact(seed)) - with server._job_cache_lock: - self.assertIsNone(server._payout_ledger_artifact) - self.assertEqual(server._payout_window_inflight_scan_anchors, {}) - - def test_late_append_disarms_before_prepare_lock_is_available(self) -> None: - server, _ledger, artifacts = self.configured_server() - clock_ms = [1_000_000] - with patch( - "lab.prism.prism_coordinator.now_ms", - side_effect=lambda: clock_ms[0], - ): - artifact = server._build_payout_ledger_artifact( - 0, 0, artifacts.network_difficulty - ) - assert artifact is not None - self.assertTrue(server._install_payout_ledger_artifact(artifact)) - anchor_ms = int(artifact.snapshot_anchor_ms) - pending = stamped_pending_share(anchor_ms - 10) - - server._payout_state_prepare_lock.acquire() - # Exercise the artifact anchor independently: an already-cold - # incremental cache must not hide an unsafe armed artifact. - server._incremental_payout_artifact_window = None - invalidation_finished = threading.Event() - - def invalidate() -> None: - server._invalidate_incremental_payout_window_for_append( - pending - ) - invalidation_finished.set() - - thread = threading.Thread(target=invalidate) - thread.start() - try: - deadline = time.monotonic() + 1.0 - while time.monotonic() < deadline: - with server._job_cache_lock: - if server._payout_ledger_artifact is None: - break - time.sleep(0.001) - with server._job_cache_lock: - self.assertIsNone(server._payout_ledger_artifact) - self.assertGreater( - server._payout_ledger_append_invalidation_epoch, - artifact.append_invalidation_epoch, - ) - self.assertFalse(invalidation_finished.is_set()) - self.assertFalse( - server._install_payout_ledger_artifact(artifact) - ) - finally: - server._payout_state_prepare_lock.release() - thread.join(timeout=1.0) - self.assertFalse(thread.is_alive()) - self.assertTrue(invalidation_finished.is_set()) - - def test_atomic_publication_rejects_pre_append_artifact(self) -> None: - server, _ledger, artifacts = self.configured_server() - clock_ms = [1_000_000] - with patch( - "lab.prism.prism_coordinator.now_ms", - side_effect=lambda: clock_ms[0], - ): - candidate = server._current_payout_state_candidate() - stale = candidate.ledger_artifact - assert stale is not None - anchor_ms = int(stale.snapshot_anchor_ms) - server._invalidate_incremental_payout_window_for_append( - stamped_pending_share(anchor_ms - 10) - ) - with server._payout_artifact_executor_lock: - server._payout_artifact_executor_shutdown = True - published = server._publish_payout_state_candidate(candidate) - - self.assertIsNotNone(published) - with server._job_cache_lock: - self.assertIsNone(server._payout_ledger_artifact) - self.assertGreater( - server._payout_ledger_append_invalidation_epoch, - stale.append_invalidation_epoch, - ) - - def test_late_append_supersedes_in_flight_sync_seed_bundle(self) -> None: - server, _ledger, artifacts = self.configured_server() - clock_ms = [1_000_000] - with patch( - "lab.prism.prism_coordinator.now_ms", - side_effect=lambda: clock_ms[0], - ): - artifact = server._build_payout_ledger_artifact( - 0, 0, artifacts.network_difficulty - ) - assert artifact is not None - self.assertTrue(server._install_payout_ledger_artifact(artifact)) - pending = stamped_pending_share( - int(artifact.snapshot_anchor_ms) - 10 - ) - - def invalidating_builder(**kwargs: object) -> dict[str, object]: - server._invalidate_incremental_payout_window_for_append( - pending - ) - suffix_hex = str(kwargs["coinbase_script_sig_suffix_hex"]) - return { - "found_block": dict(kwargs["found_block"]), - "payout_policy_manifest": {"accounts": []}, - "signed_coinbase_manifest": { - "manifest": { - "coinbase_tx_hex": synthetic_manifest_coinbase_hex( - suffix_hex - ), - } - }, - } - - server.build_audit_bundle = invalidating_builder # type: ignore[method-assign] - with self.assertRaises(JobBuildSuperseded): - server.build_shared_job_bundle( - artifacts, - worker(), - payout_artifact=None, - ) - - with server._job_cache_lock: - self.assertIsNone(server._payout_ledger_artifact) - - def test_normal_append_preserves_incremental_window(self) -> None: - server, ledger, artifacts = self.configured_server() - clock_ms = [1_000_000] - with patch( - "lab.prism.prism_coordinator.now_ms", - side_effect=lambda: clock_ms[0], - ): - initial = server._build_payout_ledger_artifact( - 0, 0, artifacts.network_difficulty - ) - assert initial is not None - cached = server._incremental_payout_artifact_window - self.assertIsNotNone(cached) - self.assertTrue(server._install_payout_ledger_artifact(initial)) - with server._job_cache_lock: - armed = server._payout_ledger_artifact - append_epoch = ( - server._payout_ledger_append_invalidation_epoch - ) - self.assertIsNotNone(armed) - - entry = self.replay_shaped_append_entry( - accepted_at_ms=1_000_010, - block_hash_hex="55" * 32, - ) - self.assertTrue(server._append_share_entry(entry)) - self.assertIs(server._incremental_payout_artifact_window, cached) - with server._job_cache_lock: - self.assertIs(server._payout_ledger_artifact, armed) - self.assertEqual( - server._payout_ledger_append_invalidation_epoch, - append_epoch, - ) - self.assertIs( - server._usable_payout_ledger_artifact( - 0, - artifacts.network_difficulty, - ), - armed, - ) - - server.payout_artifact_min_build_interval_seconds = 0.0 - clock_ms[0] = 1_000_020 - advanced = server._build_payout_ledger_artifact( - 0, 0, artifacts.network_difficulty - ) - - assert advanced is not None - self.assertEqual(advanced.window_build_mode, "incremental") - self.assertIsNone(advanced.window_full_rescan_reason) - self.assertEqual(advanced.window_touched_pages, 1) - self.assertEqual(ledger.full_snapshot_calls, 1) - self.assertEqual(ledger.delta_snapshot_calls, 1) - self.assertIn( - entry.pending_share.share_id, - {str(share["share_id"]) for share in advanced.shares_json}, - ) - - def test_debounce_then_delta_and_forced_full_rescan(self) -> None: - server, ledger, artifacts = self.configured_server() - clock_ms = [1_000_000] - with patch( - "lab.prism.prism_coordinator.now_ms", - side_effect=lambda: clock_ms[0], - ): - initial = server._build_payout_ledger_artifact( - 0, 0, artifacts.network_difficulty - ) - assert initial is not None - self.assertEqual(initial.window_build_mode, "full_rescan") - self.assertEqual(ledger.full_snapshot_calls, 1) - - append_incremental_share( - ledger, - share_seq=4, - accepted_at_ms=1_000_010, - ) - clock_ms[0] = 1_000_020 - debounced = server._build_payout_ledger_artifact( - 0, 0, artifacts.network_difficulty - ) - assert debounced is not None - self.assertEqual(debounced.window_build_mode, "debounced") - self.assertEqual(ledger.full_snapshot_calls, 1) - self.assertEqual(ledger.delta_snapshot_calls, 0) - self.assertEqual(len(debounced.shares_json), 3) - - server.payout_artifact_min_build_interval_seconds = 0.0 - advanced = server._build_payout_ledger_artifact( - 0, 0, artifacts.network_difficulty - ) - assert advanced is not None - self.assertEqual(advanced.window_build_mode, "incremental") - self.assertEqual(advanced.window_delta_rows, 1) - self.assertEqual(ledger.delta_snapshot_calls, 1) - self.assertEqual(len(advanced.shares_json), 4) - - # A payout mutation (the reconciler's #224 intent) re-reads the - # balances and advances the append-only window from its delta; - # it is never a full rescan. - mutated = server._build_payout_ledger_artifact( - 0, - 0, - artifacts.network_difficulty, - force_prior_balances_read=True, - ) - assert mutated is not None - self.assertEqual(mutated.window_build_mode, "incremental") - self.assertIsNone(mutated.window_full_rescan_reason) - self.assertEqual(ledger.full_snapshot_calls, 1) - self.assertEqual(ledger.delta_snapshot_calls, 2) - # The positional flag keeps its full fail-closed meaning. - forced = server._build_payout_ledger_artifact( - 0, - 0, - artifacts.network_difficulty, - True, - ) - assert forced is not None - self.assertEqual(forced.window_build_mode, "full_rescan") - self.assertEqual( - forced.window_full_rescan_reason, - "reconcile_invalidation", - ) - self.assertEqual(ledger.full_snapshot_calls, 2) - self.assertEqual( - forced.share_snapshot_sha256, mutated.share_snapshot_sha256 - ) - - def test_periodic_self_check_compares_delta_to_full_oracle(self) -> None: - server, ledger, artifacts = self.configured_server() - clock_ms = [1_000_000] - with patch( - "lab.prism.prism_coordinator.now_ms", - side_effect=lambda: clock_ms[0], - ): - self.assertIsNotNone( - server._build_payout_ledger_artifact( - 0, 0, artifacts.network_difficulty - ) - ) - append_incremental_share( - ledger, - share_seq=4, - accepted_at_ms=1_000_010, - ) - clock_ms[0] = 1_000_020 - server.payout_artifact_min_build_interval_seconds = 0.0 - server.payout_artifact_full_rescan_seconds = 0.0 - - checked = server._build_payout_ledger_artifact( - 0, 0, artifacts.network_difficulty - ) - - assert checked is not None - self.assertEqual(checked.window_build_mode, "self_check_match") - self.assertEqual(ledger.delta_snapshot_calls, 1) - self.assertEqual(ledger.full_snapshot_calls, 2) - - def test_routine_build_runs_overdue_self_check_despite_preview_refresh( - self, - ) -> None: - # Accepted-block previews bypass the build interval and refresh - # refreshed_monotonic on every block, yet never run the periodic - # oracle themselves. Under a sustained sub-interval block cadence - # every routine build used to return from the debounce branch - # before reaching the self-check condition, so the configured - # full-rescan window and balance oracle could be postponed - # indefinitely. An overdue self-check now disarms the debounce - # for routine builds while the urgent preview path stays - # debounce-free and oracle-free. - server, ledger, artifacts = self.configured_server() - clock_ms = [1_000_000] - with patch( - "lab.prism.prism_coordinator.now_ms", - side_effect=lambda: clock_ms[0], - ): - self.assertIsNotNone( - server._build_payout_ledger_artifact( - 0, 0, artifacts.network_difficulty - ) - ) - append_incremental_share( - ledger, - share_seq=4, - accepted_at_ms=1_000_010, - ) - clock_ms[0] = 1_000_020 - preview = server._build_payout_ledger_artifact( - 0, 0, artifacts.network_difficulty, False, True - ) - assert preview is not None - self.assertEqual(preview.window_build_mode, "incremental") - # The next preview arrives with the self-check long overdue; - # the urgent path must still not pay the oracle walk. - server.payout_artifact_full_rescan_seconds = 0.0 - append_incremental_share( - ledger, - share_seq=5, - accepted_at_ms=1_000_030, - ) - clock_ms[0] = 1_000_040 - urgent = server._build_payout_ledger_artifact( - 0, 0, artifacts.network_difficulty, False, True - ) - assert urgent is not None - self.assertEqual(urgent.window_build_mode, "incremental") - self.assertEqual(ledger.full_snapshot_calls, 1) - # A routine build inside the min build interval used to - # debounce here -- forever, while blocks kept refreshing the - # window above -- and the self-check never ran. - checked = server._build_payout_ledger_artifact( - 0, 0, artifacts.network_difficulty - ) - - assert checked is not None - self.assertEqual(checked.window_build_mode, "self_check_match") - self.assertEqual(ledger.full_snapshot_calls, 2) - - def test_periodic_self_check_replaces_divergent_delta_with_full_bytes( - self, - ) -> None: - class OmittingDeltaLedger(IncrementalRecordingLedger): - def snapshot_between_job_issues( - self, - previous_anchor_job_issued_at_ms: int, - anchor_job_issued_at_ms: int, - ) -> list[object]: - del previous_anchor_job_issued_at_ms, anchor_job_issued_at_ms - self.delta_snapshot_calls += 1 - return [] - - ledger = OmittingDeltaLedger() - for share_seq in range(1, 4): - append_incremental_share( - ledger, - share_seq=share_seq, - accepted_at_ms=999_900 + share_seq, - ) - server, rpc = coordinator(ledger=ledger) - artifacts = server.store_template_artifacts(dict(rpc.template)) - assert artifacts is not None - server._pool_ready_latched = True - clock_ms = [1_000_000] - with patch( - "lab.prism.prism_coordinator.now_ms", - side_effect=lambda: clock_ms[0], - ): - self.assertIsNotNone( - server._build_payout_ledger_artifact( - 0, 0, artifacts.network_difficulty - ) - ) - append_incremental_share( - ledger, - share_seq=4, - accepted_at_ms=1_000_010, - ) - clock_ms[0] = 1_000_020 - server.payout_artifact_min_build_interval_seconds = 0.0 - server.payout_artifact_full_rescan_seconds = 0.0 - repaired = server._build_payout_ledger_artifact( - 0, 0, artifacts.network_difficulty - ) - forced = server._build_payout_ledger_artifact( - 0, - 0, - artifacts.network_difficulty, - True, - ) - - assert repaired is not None and forced is not None - self.assertEqual(forced.window_build_mode, "full_rescan") - self.assertEqual(repaired.window_build_mode, "self_check_mismatch") - self.assertEqual(list(repaired.shares_json), list(forced.shares_json)) - self.assertEqual( - repaired.share_snapshot_sha256, - forced.share_snapshot_sha256, - ) - - def test_periodic_self_check_invalidates_drifted_published_balances( - self, - ) -> None: - class DriftingCarryLedger(IncrementalRecordingLedger): - def __init__(self) -> None: - super().__init__() - self.balance_sats = 546 - self.prior_balance_reads = 0 - - def current_prior_balances(self) -> list[dict[str, object]]: - self.prior_balance_reads += 1 - return [ - { - "recipient_id": "carry", - "order_key": "01:carry", - "p2mr_program_hex": "66" * 32, - "balance_sats": self.balance_sats, - } - ] - - ledger = DriftingCarryLedger() - for share_seq in range(1, 4): - append_incremental_share( - ledger, - share_seq=share_seq, - accepted_at_ms=999_900 + share_seq, - ) - server, rpc = coordinator(ledger=ledger) - artifacts = server.store_template_artifacts(dict(rpc.template)) - assert artifacts is not None - server._pool_ready_latched = True - server.payout_artifact_min_build_interval_seconds = 0.0 - server.payout_artifact_full_rescan_seconds = 3_600.0 - clock_ms = [1_000_000] - monotonic_seconds = [100.0] - build_logs: list[dict[str, object]] = [] - server._payout_artifact_log = ( # type: ignore[method-assign] - lambda event, **fields: build_logs.append( - {"event": event, **fields} - ) - ) - - with ( - patch( - "lab.prism.prism_coordinator.now_ms", - side_effect=lambda: clock_ms[0], - ), - patch( - "lab.prism.prism_coordinator.time.monotonic", - side_effect=lambda: monotonic_seconds[0], - ), - ): - published = server._current_payout_state_artifact() - initial = server._build_payout_ledger_artifact( - 0, - 0, - artifacts.network_difficulty, - ) - assert initial is not None - self.assertTrue(server._install_payout_ledger_artifact(initial)) - - ledger.balance_sats = 777 - append_incremental_share( - ledger, - share_seq=4, - accepted_at_ms=1_000_010, - ) - clock_ms[0] = 1_000_020 - monotonic_seconds[0] = 3_701.0 - checked = server._build_payout_ledger_artifact( - 0, - 0, - artifacts.network_difficulty, - ) - - with server._job_cache_lock: - self.assertIsNone(server._payout_ledger_artifact) - - # The mismatch build publishes the repaired balances for the same - # generation, so the next build resumes byte reuse from the - # repaired snapshot instead of paying another carry read against - # a still-stale publication. - clock_ms[0] = 1_000_030 - monotonic_seconds[0] = 3_702.0 - following = server._build_payout_ledger_artifact( - 0, - 0, - artifacts.network_difficulty, - ) - - assert checked is not None and following is not None - self.assertEqual(checked.window_build_mode, "self_check_match") - self.assertEqual(checked.prior_balances[0]["balance_sats"], 777) - self.assertEqual(following.prior_balances[0]["balance_sats"], 777) - self.assertEqual(ledger.prior_balance_reads, 2) - self.assertEqual(ledger.full_snapshot_calls, 2) - repaired = server._published_payout_state.artifact - assert repaired is not None - self.assertNotEqual( - repaired.prior_balances_sha256, - published.prior_balances_sha256, - ) - self.assertEqual( - repaired.prior_balances_sha256, - checked.prior_balances_sha256, - ) - self.assertIsNone( - server._payout_prior_balances_reuse_invalidated_sha256 - ) - self.assertEqual( - server.payout_artifact_event_counts["balance_check_mismatch"], - 1, - ) - self.assertEqual( - server.payout_artifact_event_counts["balance_repair_published"], - 1, - ) - self.assertEqual( - server.payout_artifact_event_counts["self_check_match"], - 1, - ) - built_sources = [ - entry["prior_balances_source"] - for entry in build_logs - if entry["event"] == "payout_artifact_built" - ] - self.assertEqual(built_sources, ["published", "ledger", "published"]) - mismatch_logs = [ - entry - for entry in build_logs - if entry["event"] == "payout_artifact_balance_check_mismatch" - ] - self.assertEqual(len(mismatch_logs), 1) - repair_logs = [ - entry - for entry in build_logs - if entry["event"] == "payout_artifact_balance_repair_published" - ] - self.assertEqual(len(repair_logs), 1) - - def test_balance_backstop_failure_retains_oracle_repaired_window( - self, - ) -> None: - # The share-window oracle and the carry-balance backstop are - # independent reads. A backstop failure after a successful oracle - # must not resurrect the delta window the oracle just refuted. - class FailingCarryOmittingDeltaLedger(IncrementalRecordingLedger): - def __init__(self) -> None: - super().__init__() - self.fail_balance_read = False - self.balance_read_attempts = 0 - - def snapshot_between_job_issues( - self, - previous_anchor_job_issued_at_ms: int, - anchor_job_issued_at_ms: int, - ) -> list[object]: - del previous_anchor_job_issued_at_ms, anchor_job_issued_at_ms - self.delta_snapshot_calls += 1 - return [] - - def current_prior_balances(self) -> list[dict[str, object]]: - self.balance_read_attempts += 1 - if self.fail_balance_read: - raise RuntimeError("carry aggregate unavailable") - return [ - { - "recipient_id": "carry", - "order_key": "01:carry", - "p2mr_program_hex": "66" * 32, - "balance_sats": 546, - } - ] - - ledger = FailingCarryOmittingDeltaLedger() - for share_seq in range(1, 4): - append_incremental_share( - ledger, - share_seq=share_seq, - accepted_at_ms=999_900 + share_seq, - ) - server, rpc = coordinator(ledger=ledger) - artifacts = server.store_template_artifacts(dict(rpc.template)) - assert artifacts is not None - server._pool_ready_latched = True - server.payout_artifact_min_build_interval_seconds = 0.0 - server.payout_artifact_full_rescan_seconds = 3_600.0 - clock_ms = [1_000_000] - monotonic_seconds = [100.0] - with ( - patch( - "lab.prism.prism_coordinator.now_ms", - side_effect=lambda: clock_ms[0], - ), - patch( - "lab.prism.prism_coordinator.time.monotonic", - side_effect=lambda: monotonic_seconds[0], - ), - ): - server._current_payout_state_artifact() - initial = server._build_payout_ledger_artifact( - 0, - 0, - artifacts.network_difficulty, - ) - assert initial is not None - self.assertTrue(server._install_payout_ledger_artifact(initial)) - - ledger.fail_balance_read = True - append_incremental_share( - ledger, - share_seq=4, - accepted_at_ms=1_000_010, - ) - clock_ms[0] = 1_000_020 - monotonic_seconds[0] = 3_701.0 - checked = server._build_payout_ledger_artifact( - 0, - 0, - artifacts.network_difficulty, - ) - - append_incremental_share( - ledger, - share_seq=5, - accepted_at_ms=1_000_021, - ) - clock_ms[0] = 1_000_030 - monotonic_seconds[0] = 3_702.0 - following = server._build_payout_ledger_artifact( - 0, - 0, - artifacts.network_difficulty, - ) - - assert checked is not None and following is not None - self.assertEqual(checked.window_build_mode, "self_check_mismatch") - self.assertEqual( - checked.window_full_rescan_reason, - "periodic_self_check_balance_check_failed", - ) - # The omitting delta produced three retained shares; the oracle's - # four-share window must be the one served and cached. - self.assertEqual(len(checked.shares_json), 4) - # One read published the priming state; the only other attempt is - # the backstop that failed. - self.assertEqual(ledger.balance_read_attempts, 2) - # The unverified balances keep reusing the published snapshot. - self.assertEqual( - checked.prior_balances_sha256, - initial.prior_balances_sha256, - ) - self.assertIsNone( - server._payout_prior_balances_reuse_invalidated_sha256 - ) - self.assertEqual( - server.payout_artifact_event_counts["self_check_mismatch"], - 1, - ) - self.assertEqual( - server.payout_artifact_event_counts["self_check_failed"], - 0, - ) - self.assertEqual( - server.payout_artifact_event_counts["balance_check_mismatch"], - 0, - ) - # The failed backstop still stamps the attempt: the next build stays - # on the delta path instead of retrying the oracle immediately. - self.assertEqual(following.window_build_mode, "incremental") - self.assertEqual(ledger.full_snapshot_calls, 2) - - def test_periodic_self_check_repair_arms_and_serves(self) -> None: - # The repaired publication is what lets the mismatch build's fresh - # balances actually reach miners: its artifact must arm against the - # repaired digest and the reuse probe must serve it, instead of the - # armed-artifact fence clearing the repair against a stale published - # snapshot until an unrelated generation change. - class DriftingCarryLedger(IncrementalRecordingLedger): - def __init__(self) -> None: - super().__init__() - self.balance_sats = 546 - - def current_prior_balances(self) -> list[dict[str, object]]: - return [ - { - "recipient_id": "carry", - "order_key": "01:carry", - "p2mr_program_hex": "66" * 32, - "balance_sats": self.balance_sats, - } - ] - - ledger = DriftingCarryLedger() - for share_seq in range(1, 4): - append_incremental_share( - ledger, - share_seq=share_seq, - accepted_at_ms=999_900 + share_seq, - ) - server, rpc = coordinator(ledger=ledger) - artifacts = server.store_template_artifacts(dict(rpc.template)) - assert artifacts is not None - server._pool_ready_latched = True - server.payout_artifact_min_build_interval_seconds = 0.0 - server.payout_artifact_full_rescan_seconds = 3_600.0 - clock_ms = [1_000_000] - monotonic_seconds = [100.0] - - with ( - patch( - "lab.prism.prism_coordinator.now_ms", - side_effect=lambda: clock_ms[0], - ), - patch( - "lab.prism.prism_coordinator.time.monotonic", - side_effect=lambda: monotonic_seconds[0], - ), - ): - server._current_payout_state_artifact() - initial = server._build_payout_ledger_artifact( - 0, 0, artifacts.network_difficulty - ) - assert initial is not None - self.assertTrue(server._install_payout_ledger_artifact(initial)) - - ledger.balance_sats = 777 - clock_ms[0] = 1_000_020 - monotonic_seconds[0] = 3_701.0 - checked = server._build_payout_ledger_artifact( - 0, 0, artifacts.network_difficulty - ) - assert checked is not None - self.assertEqual(checked.prior_balances[0]["balance_sats"], 777) - - self.assertTrue(server._install_payout_ledger_artifact(checked)) - served = server._usable_payout_ledger_artifact( - 0, - artifacts.network_difficulty, - ) - - assert served is not None - self.assertEqual(served.prior_balances[0]["balance_sats"], 777) - self.assertEqual( - served.prior_balances_sha256, - checked.prior_balances_sha256, - ) - with server._job_cache_lock: - self.assertIsNotNone(server._payout_ledger_artifact) - - def test_self_check_repair_supersedes_active_jobs(self) -> None: - # The in-place repair changes neither the payout generation nor the - # append epoch, so an already-stamped job passes every admission - # fence while carrying the refuted balances. The repair must - # schedule a refresh wave, and the wave's reselection must identify - # such jobs by their payout digest, or they stay mineable (and a - # block solve commits the stale allocation) until an unrelated - # generation change. - server, _ledger, _artifacts = self.configured_server() - install_fake_bundle_builder(server) - server._ensure_tip_refresh_state() - state = client(1) - with patch( - "lab.prism.prism_coordinator.now_ms", - side_effect=lambda: 1_000_000, - ): - bundle = server.build_shared_job_bundle( - server.current_template_artifacts(), - worker(), - ) - context = server.stamp_job_for_client( - state, - bundle, - clean_jobs=True, - ) - state.active_job = context - snapshot = SimpleNamespace( - bestblockhash=str(bundle.template["previousblockhash"]), - previousblockhash=str(bundle.template["previousblockhash"]), - template_fingerprint=bundle.template_fingerprint, - ) - with server._job_cache_lock: - published = server._published_payout_state.artifact - assert published is not None - self.assertEqual( - context.payout_artifact_sha256, - published.prior_balances_sha256, - ) - self.assertFalse( - server.client_needs_tip_template_refresh(state, snapshot) - ) - - self.assertTrue( - server._publish_self_check_repaired_balances( - 0, - stale_prior_balances_sha256=published.prior_balances_sha256, - balances=[ - { - "recipient_id": "carry", - "order_key": "01:carry", - "p2mr_program_hex": "66" * 32, - "balance_sats": 999, - } - ], - ) - ) - - self.assertTrue( - server.client_needs_tip_template_refresh(state, snapshot) - ) - self.assertTrue(server._tip_refresh_pending_event.is_set()) - self.assertTrue(server._tip_refresh_retry.is_set()) - - def test_self_check_repair_retires_refuted_jobs_clean(self) -> None: - # Reselection alone is not enough: the wave stamps the replacement - # with clean_jobs from client_tip_changed_for_snapshot, and a - # clean_jobs=False delivery leaves the refuted job ID admissible in - # active_job_ids. The digest mismatch must force clean-job - # retirement, or a late block submission on the old job still - # commits the refuted allocation after corrected work is delivered. - server, _ledger, _artifacts = self.configured_server() - install_fake_bundle_builder(server) - server._ensure_tip_refresh_state() - state = client(1) - with patch( - "lab.prism.prism_coordinator.now_ms", - side_effect=lambda: 1_000_000, - ): - bundle = server.build_shared_job_bundle( - server.current_template_artifacts(), - worker(), - ) - context = server.stamp_job_for_client( - state, - bundle, - clean_jobs=True, - ) - state.active_job = context - snapshot = SimpleNamespace( - bestblockhash=str(bundle.template["previousblockhash"]), - previousblockhash=str(bundle.template["previousblockhash"]), - template_fingerprint=bundle.template_fingerprint, - ) - with server._job_cache_lock: - published = server._published_payout_state.artifact - assert published is not None - self.assertFalse( - server.client_tip_changed_for_snapshot(state, snapshot) - ) - - self.assertTrue( - server._publish_self_check_repaired_balances( - 0, - stale_prior_balances_sha256=published.prior_balances_sha256, - balances=[ - { - "recipient_id": "carry", - "order_key": "01:carry", - "p2mr_program_hex": "66" * 32, - "balance_sats": 999, - } - ], - ) - ) - - self.assertTrue( - server.client_tip_changed_for_snapshot(state, snapshot) - ) - - def test_late_append_invalidation_supersedes_active_jobs(self) -> None: - # A late-visible append advances only the append epoch: the payout - # generation and published digest both survive, so an active job - # mining the pre-append window passes every other fence. The - # invalidation must schedule a refresh wave, the reselection must - # identify the job by its stamped epoch, and the replacement must - # retire it clean -- otherwise miners keep working (and can solve a - # block from) a window that omitted the late share until some - # unrelated tip event. - server, _ledger, _artifacts = self.configured_server() - install_fake_bundle_builder(server) - server._ensure_tip_refresh_state() - state = client(1) - with patch( - "lab.prism.prism_coordinator.now_ms", - side_effect=lambda: 1_000_000, - ): - bundle = server.build_shared_job_bundle( - server.current_template_artifacts(), - worker(), - ) - context = server.stamp_job_for_client( - state, - bundle, - clean_jobs=True, - ) - state.active_job = context - snapshot = SimpleNamespace( - bestblockhash=str(bundle.template["previousblockhash"]), - previousblockhash=str(bundle.template["previousblockhash"]), - template_fingerprint=bundle.template_fingerprint, - ) - self.assertFalse( - server.client_needs_tip_template_refresh(state, snapshot) - ) - self.assertFalse( - server.client_tip_changed_for_snapshot(state, snapshot) - ) - - with patch( - "lab.prism.prism_coordinator.now_ms", - side_effect=lambda: 1_000_000, - ): - token = server._expose_inflight_scan_anchor(999_999) - try: - server._invalidate_incremental_payout_window_for_append( - stamped_pending_share(999_950) - ) - finally: - server._retire_inflight_scan_anchor(token) - with server._job_cache_lock: - self.assertEqual( - server._payout_ledger_append_invalidation_epoch, 1 - ) - - self.assertTrue( - server.client_needs_tip_template_refresh(state, snapshot) - ) - self.assertTrue( - server.client_tip_changed_for_snapshot(state, snapshot) - ) - self.assertTrue(server._tip_refresh_pending_event.is_set()) - self.assertTrue(server._tip_refresh_retry.is_set()) - - def test_direct_delivery_rechecks_published_digest_at_commit(self) -> None: - # A periodic self-check repair swaps the published balance snapshot - # in place: neither the payout generation nor the append epoch moves, - # so only the digest identifies a direct-delivery context keyed to - # the refuted balances. When the repair lands after the build but - # before the commit boundary, the delivery must re-check the digest - # or the miner's active job carries the refuted allocation. - server, _ledger, _artifacts = self.configured_server() - install_fake_bundle_builder(server) - server._ensure_tip_refresh_state() - state = client(1) - sent: list[dict[str, object]] = [] - state.send = sent.append # type: ignore[method-assign] - original_build_job_for_client = server.build_job_for_client - repaired = False - - def repair_after_build(*args: object, **kwargs: object) -> object: - nonlocal repaired - context = original_build_job_for_client(*args, **kwargs) # type: ignore[arg-type] - if not repaired: - repaired = True - with server._job_cache_lock: - published = server._published_payout_state.artifact - assert published is not None - self.assertEqual( - context.payout_artifact_sha256, - published.prior_balances_sha256, - ) - self.assertTrue( - server._publish_self_check_repaired_balances( - 0, - stale_prior_balances_sha256=( - published.prior_balances_sha256 - ), - balances=[ - { - "recipient_id": "carry", - "order_key": "01:carry", - "p2mr_program_hex": "66" * 32, - "balance_sats": 999, - } - ], - ) - ) - return context - - server.build_job_for_client = repair_after_build # type: ignore[method-assign] - - with patch( - "lab.prism.prism_coordinator.now_ms", - side_effect=lambda: 1_000_000, - ): - self.assertFalse(server.maybe_send_job(state, clean_jobs=True)) - self.assertEqual(sent, []) - self.assertIsNone(state.active_job) - self.assertEqual(state.active_job_ids, set()) - self.assertEqual(server.jobs, {}) - - def test_incremental_and_forced_full_artifacts_match_with_carry_boundaries( - self, - ) -> None: - class CarryLedger(IncrementalRecordingLedger): - def current_prior_balances(self) -> list[dict[str, object]]: - return [ - { - "recipient_id": "below", - "order_key": "01:below", - "p2mr_program_hex": "11" * 32, - "balance_sats": 545, - }, - { - "recipient_id": "at", - "order_key": "02:at", - "p2mr_program_hex": "22" * 32, - "balance_sats": 546, - }, - { - "recipient_id": "above", - "order_key": "03:above", - "p2mr_program_hex": "33" * 32, - "balance_sats": 547, - }, - ] - - ledger = CarryLedger() - for share_seq in range(1, 4): - append_incremental_share( - ledger, - share_seq=share_seq, - accepted_at_ms=999_900 + share_seq, - ) - server, rpc = coordinator(ledger=ledger) - artifacts = server.store_template_artifacts(dict(rpc.template)) - assert artifacts is not None - server._pool_ready_latched = True - server.payout_artifact_min_build_interval_seconds = 0.0 - clock_ms = [1_000_000] - with patch( - "lab.prism.prism_coordinator.now_ms", - side_effect=lambda: clock_ms[0], - ): - self.assertIsNotNone( - server._build_payout_ledger_artifact( - 0, 0, artifacts.network_difficulty - ) - ) - append_incremental_share( - ledger, - share_seq=4, - accepted_at_ms=1_000_010, - ) - clock_ms[0] = 1_000_020 - incremental = server._build_payout_ledger_artifact( - 0, 0, artifacts.network_difficulty - ) - forced = server._build_payout_ledger_artifact( - 0, - 0, - artifacts.network_difficulty, - True, - ) - - assert incremental is not None and forced is not None - self.assertEqual(incremental.window_build_mode, "incremental") - self.assertEqual(forced.window_build_mode, "full_rescan") - incremental_bytes = canonical_json_text( - { - "shares": list(incremental.shares_json), - "prior_balances": list(incremental.prior_balances), - } - ).encode() - forced_bytes = canonical_json_text( - { - "shares": list(forced.shares_json), - "prior_balances": list(forced.prior_balances), - } - ).encode() - self.assertEqual(incremental_bytes, forced_bytes) - self.assertEqual( - incremental.share_snapshot_sha256, - forced.share_snapshot_sha256, - ) - self.assertEqual( - incremental.prior_balances_sha256, - forced.prior_balances_sha256, - ) - - def test_normal_refresh_reuses_generation_owned_prior_balances(self) -> None: - class CountingCarryLedger(IncrementalRecordingLedger): - prior_balance_reads = 0 - - def current_prior_balances(self) -> list[dict[str, object]]: - self.prior_balance_reads += 1 - return [ - { - "recipient_id": "carry", - "order_key": "01:carry", - "p2mr_program_hex": "66" * 32, - "balance_sats": 545, - } - ] - - ledger = CountingCarryLedger() - for share_seq in range(1, 4): - append_incremental_share( - ledger, - share_seq=share_seq, - accepted_at_ms=999_900 + share_seq, - ) - server, rpc = coordinator(ledger=ledger) - artifacts = server.store_template_artifacts(dict(rpc.template)) - assert artifacts is not None - server._pool_ready_latched = True - server.payout_artifact_min_build_interval_seconds = 0.0 - clock_ms = [1_000_000] - with patch( - "lab.prism.prism_coordinator.now_ms", - side_effect=lambda: clock_ms[0], - ): - published = server._current_payout_state_artifact() - initial = server._build_payout_ledger_artifact( - 0, 0, artifacts.network_difficulty - ) - append_incremental_share( - ledger, - share_seq=4, - accepted_at_ms=1_000_010, - ) - clock_ms[0] = 1_000_020 - incremental = server._build_payout_ledger_artifact( - 0, 0, artifacts.network_difficulty - ) - forced = server._build_payout_ledger_artifact( - 0, - 0, - artifacts.network_difficulty, - True, - ) - - assert initial is not None and incremental is not None and forced is not None - self.assertEqual(ledger.prior_balance_reads, 2) - self.assertEqual( - initial.prior_balances_sha256, - published.prior_balances_sha256, - ) - self.assertEqual( - incremental.prior_balances_sha256, - published.prior_balances_sha256, - ) - - def test_fifteen_second_generation_bumps_refresh_once_per_minute(self) -> None: - server, ledger, artifacts = self.configured_server() - clock_ms = [1_000_000] - monotonic_seconds = [100.0] - modes: list[str | None] = [] - with ( - patch( - "lab.prism.prism_coordinator.now_ms", - side_effect=lambda: clock_ms[0], - ), - patch( - "lab.prism.prism_coordinator.time.monotonic", - side_effect=lambda: monotonic_seconds[0], - ), - ): - initial = server._build_payout_ledger_artifact( - 0, 0, artifacts.network_difficulty - ) - assert initial is not None - modes.append(initial.window_build_mode) - for bump in range(1, 5): - monotonic_seconds[0] = 100.0 + (15.0 * bump) - clock_ms[0] = 1_000_000 + (15_000 * bump) - append_incremental_share( - ledger, - share_seq=3 + bump, - accepted_at_ms=clock_ms[0] - 2, - ) - built = server._build_payout_ledger_artifact( - 0, 0, artifacts.network_difficulty - ) - assert built is not None - modes.append(built.window_build_mode) - - self.assertEqual( - modes, - ["full_rescan", "debounced", "debounced", "debounced", "incremental"], - ) - self.assertEqual(ledger.full_snapshot_calls, 1) - self.assertEqual(ledger.delta_snapshot_calls, 1) - - def test_found_block_candidate_bypasses_normal_interval(self) -> None: - server, ledger, artifacts = self.configured_server() - clock_ms = [1_000_000] - with patch( - "lab.prism.prism_coordinator.now_ms", - side_effect=lambda: clock_ms[0], - ): - self.assertIsNotNone( - server._build_payout_ledger_artifact( - 0, 0, artifacts.network_difficulty - ) - ) - append_incremental_share( - ledger, - share_seq=4, - accepted_at_ms=1_000_010, - ) - # Even an overdue routine oracle must be deferred from the - # immediate found-block publication path. - server.payout_artifact_full_rescan_seconds = 0.0 - clock_ms[0] = 1_000_020 - candidate = server._prepared_payout_state_candidate( - ( - 0, - 1, - str(server.rpc.tip), - "accepted_block_preview", - time.monotonic(), - ) - ) - candidate = server._accepted_block_preview_candidate( - candidate, - block_hash="44" * 32, - preview=(("winner", "01:winner", "55" * 32, 546),), - ) - - artifact = candidate.ledger_artifact - assert artifact is not None - self.assertEqual(artifact.window_build_mode, "incremental") - self.assertEqual(artifact.window_delta_rows, 1) - self.assertEqual(ledger.delta_snapshot_calls, 1) - self.assertEqual(ledger.full_snapshot_calls, 1) - self.assertEqual(len(artifact.shares_json), 4) - - def test_failed_periodic_oracle_is_spaced_from_next_delta_build(self) -> None: - class FailingOracleLedger(IncrementalRecordingLedger): - fail_full_snapshot = False - - def snapshot_at_job_issue( - self, - anchor_job_issued_at_ms: int, - *, - window_weight: int | None = None, - ) -> list[object]: - if self.fail_full_snapshot: - self.full_snapshot_calls += 1 - raise RuntimeError("oracle unavailable") - return super().snapshot_at_job_issue( - anchor_job_issued_at_ms, - window_weight=window_weight, - ) - - ledger = FailingOracleLedger() - for share_seq in range(1, 4): - append_incremental_share( - ledger, - share_seq=share_seq, - accepted_at_ms=999_900 + share_seq, - ) - server, rpc = coordinator(ledger=ledger) - artifacts = server.store_template_artifacts(dict(rpc.template)) - assert artifacts is not None - server._pool_ready_latched = True - server.payout_artifact_min_build_interval_seconds = 0.0 - server.payout_artifact_full_rescan_seconds = 10.0 - clock_ms = [1_000_000] - monotonic_seconds = [100.0] - with ( - patch( - "lab.prism.prism_coordinator.now_ms", - side_effect=lambda: clock_ms[0], - ), - patch( - "lab.prism.prism_coordinator.time.monotonic", - side_effect=lambda: monotonic_seconds[0], - ), - ): - self.assertIsNotNone( - server._build_payout_ledger_artifact( - 0, 0, artifacts.network_difficulty - ) - ) - ledger.fail_full_snapshot = True - append_incremental_share( - ledger, - share_seq=4, - accepted_at_ms=1_000_010, - ) - clock_ms[0] = 1_000_020 - monotonic_seconds[0] = 110.0 - failed_check = server._build_payout_ledger_artifact( - 0, 0, artifacts.network_difficulty - ) - assert failed_check is not None - self.assertEqual( - failed_check.window_build_mode, - "incremental_self_check_failed", - ) - - append_incremental_share( - ledger, - share_seq=5, - accepted_at_ms=1_000_021, - ) - clock_ms[0] = 1_000_030 - monotonic_seconds[0] = 111.0 - next_delta = server._build_payout_ledger_artifact( - 0, 0, artifacts.network_difficulty - ) - - assert next_delta is not None - self.assertEqual(next_delta.window_build_mode, "incremental") - self.assertEqual(ledger.full_snapshot_calls, 2) - - def test_found_block_reuses_armed_window_when_prepare_lock_is_busy( - self, - ) -> None: - class BusyPrepareLock: - def acquire(self, blocking: bool = True) -> bool: - self.assert_nonblocking(blocking) - return False - - @staticmethod - def assert_nonblocking(blocking: bool) -> None: - if blocking: - raise AssertionError("found-block path attempted to wait") - - def release(self) -> None: - raise AssertionError("unacquired lock was released") - - server, ledger, artifacts = self.configured_server() - clock_ms = [1_000_000] - with patch( - "lab.prism.prism_coordinator.now_ms", - side_effect=lambda: clock_ms[0], - ): - initial = server._build_payout_ledger_artifact( - 0, 0, artifacts.network_difficulty - ) - assert initial is not None - self.assertTrue(server._install_payout_ledger_artifact(initial)) - server._payout_state_prepare_lock = BusyPrepareLock() # type: ignore[assignment] - candidate = server._prepared_payout_state_candidate( - ( - 0, - 1, - str(server.rpc.tip), - "accepted_block_preview", - time.monotonic(), - ) - ) - candidate = server._accepted_block_preview_candidate( - candidate, - block_hash="44" * 32, - preview=(("winner", "01:winner", "55" * 32, 546),), - ) - - artifact = candidate.ledger_artifact - assert artifact is not None - self.assertEqual(artifact.window_build_mode, "found_block_cached") - self.assertEqual(artifact.payout_state_generation, 1) - self.assertEqual( - list(artifact.prior_balances), - [ - { - "recipient_id": "winner", - "order_key": "01:winner", - "p2mr_program_hex": "55" * 32, - "balance_sats": 546, - } - ], - ) - self.assertEqual(ledger.full_snapshot_calls, 1) - self.assertEqual(ledger.delta_snapshot_calls, 0) - self.assertEqual( - server.payout_artifact_event_counts["found_block_cached"], - 1, - ) - - def test_found_block_build_failure_is_visible_and_reuses_exact_window( - self, - ) -> None: - class FailingDeltaLedger(IncrementalRecordingLedger): - fail_delta = False - - def snapshot_between_job_issues( - self, - previous_anchor_job_issued_at_ms: int, - anchor_job_issued_at_ms: int, - ) -> list[object]: - if self.fail_delta: - self.delta_snapshot_calls += 1 - raise RuntimeError("delta unavailable") - return super().snapshot_between_job_issues( - previous_anchor_job_issued_at_ms, - anchor_job_issued_at_ms, - ) - - ledger = FailingDeltaLedger() - for share_seq in range(1, 4): - append_incremental_share( - ledger, - share_seq=share_seq, - accepted_at_ms=999_900 + share_seq, - ) - server, rpc = coordinator(ledger=ledger) - artifacts = server.store_template_artifacts(dict(rpc.template)) - assert artifacts is not None - server._pool_ready_latched = True - clock_ms = [1_000_000] - with patch( - "lab.prism.prism_coordinator.now_ms", - side_effect=lambda: clock_ms[0], - ): - initial = server._build_payout_ledger_artifact( - 0, 0, artifacts.network_difficulty - ) - assert initial is not None - self.assertTrue(server._install_payout_ledger_artifact(initial)) - ledger.fail_delta = True - clock_ms[0] = 1_000_020 - candidate = server._prepared_payout_state_candidate( - ( - 0, - 1, - str(server.rpc.tip), - "accepted_block_preview", - time.monotonic(), - ) - ) - - artifact = candidate.ledger_artifact - assert artifact is not None - self.assertEqual(artifact.window_build_mode, "found_block_cached") - self.assertEqual( - server.payout_artifact_event_counts["build_aborted"], - 1, - ) - self.assertEqual( - server.payout_artifact_event_counts["found_block_cached"], - 1, - ) - - def test_old_anchor_does_not_rearm_before_window_cadence(self) -> None: - class RecordingExecutor: - def __init__(self) -> None: - self.submissions = 0 - - def submit(self, _fn: object) -> Future[None]: - self.submissions += 1 - return Future() - - server, _ledger, artifacts = self.configured_server() - clock_ms = [1_000_000] - monotonic_seconds = [100.0] - with ( - patch( - "lab.prism.prism_coordinator.now_ms", - side_effect=lambda: clock_ms[0], - ), - patch( - "lab.prism.prism_coordinator.time.monotonic", - side_effect=lambda: monotonic_seconds[0], - ), - ): - initial = server._build_payout_ledger_artifact( - 0, 0, artifacts.network_difficulty - ) - assert initial is not None - self.assertTrue(server._install_payout_ledger_artifact(initial)) - executor = RecordingExecutor() - with server._payout_artifact_executor_lock: - server._payout_artifact_executor = executor # type: ignore[assignment] - - # A pending-share floor can leave the declared anchor old even - # though its exact pages were just re-proved. Past-floor probes - # must not enqueue five-second debounced balance-only builds. - clock_ms[0] = 1_061_000 - monotonic_seconds[0] = 105.0 - self.assertIsNotNone( - server._usable_payout_ledger_artifact( - 0, - artifacts.network_difficulty, - ) - ) - with server._payout_artifact_executor_lock: - self.assertIsNone(server._payout_artifact_requested) - - monotonic_seconds[0] = 160.0 - self.assertIsNotNone( - server._usable_payout_ledger_artifact( - 0, - artifacts.network_difficulty, - ) - ) - - with server._payout_artifact_executor_lock: - self.assertEqual( - server._payout_artifact_requested, - (0, int(artifacts.network_difficulty)), - ) - self.assertEqual(executor.submissions, 1) - - -@contextmanager -def _captured_payout_artifact_logs(): - """Collect the JSON payout-artifact lifecycle lines a build prints.""" - lines: list[dict[str, object]] = [] - prefix = "prism coordinator: " - - def fake_print(*args: object, **_kwargs: object) -> None: - text = " ".join(str(arg) for arg in args) - if text.startswith(prefix + "{"): - try: - lines.append(json.loads(text[len(prefix):])) - except ValueError: - pass - - with patch("builtins.print", side_effect=fake_print): - yield lines - - -class PayoutWindowRescanBoundTests(unittest.TestCase): - """Issue #228: forced full rescans are rare, attributable, and bounded.""" - - # Wall-clock ceiling a routine (tip-refresh path) candidate preparation - # may take while another build holds the prepare lock and a window inside - # PRISM_TIP_REFRESH_WINDOW_STALENESS_SECONDS is armed. The preparation - # must not queue behind the lock at all, so this is generous for a - # non-blocking path and far below any build the lock holder could be - # running; a join that times out means the path blocked. - NON_BLOCKING_BUDGET_SECONDS = 2.0 - - def configured_server( - self, - ) -> tuple[PrismCoordinator, IncrementalRecordingLedger, object]: - ledger = IncrementalRecordingLedger() - append_incremental_share(ledger, share_seq=1, accepted_at_ms=999_900) - append_incremental_share(ledger, share_seq=2, accepted_at_ms=999_910) - append_incremental_share(ledger, share_seq=3, accepted_at_ms=999_920) - server, rpc = coordinator(ledger=ledger) - artifacts = server.store_template_artifacts(dict(rpc.template)) - assert artifacts is not None - server._pool_ready_latched = True - server.payout_artifact_min_build_interval_seconds = 60.0 - server.payout_artifact_full_rescan_seconds = 3_600.0 - return server, ledger, artifacts - - @staticmethod - def _built_lines( - lines: list[dict[str, object]], - ) -> list[dict[str, object]]: - return [line for line in lines if line.get("event") == "payout_artifact_built"] - - def test_sticky_invalidation_cause_does_not_force_later_full_rescans( - self, - ) -> None: - # The defect: _capture_payout_state_source re-reads the sticky - # source tuple without consuming it, so every later capture at the - # same source generation inherited a withdrawal's cause and forced - # the full oracle again. The withdrawal's own preparation is the - # only one entitled to act on the cause it reserved. - server, ledger, artifacts = self.configured_server() - clock_ms = [1_000_000] - with patch( - "lab.prism.prism_coordinator.now_ms", - side_effect=lambda: clock_ms[0], - ), _captured_payout_artifact_logs() as lines: - initial = server._build_payout_ledger_artifact( - 0, 0, artifacts.network_difficulty - ) - assert initial is not None - self.assertEqual(ledger.full_snapshot_calls, 1) - - captured = server._capture_payout_state_source() - reserved = server._reserve_payout_state_source_if_current( - captured[1], - "accepted_block_preview_withdrawn", - tip_hash=captured[2], - ) - assert reserved is not None - # The withdrawal path's own preparation: explicit flag plus the - # reserved cause. Balances must come from the ledger; the - # append-only share window is advanced, not rescanned. - own = server._prepared_payout_state_candidate( - reserved, - force_full_window_rescan=True, - bypass_build_interval=True, - ) - assert own.ledger_artifact is not None - full_after_own = ledger.full_snapshot_calls - own_line = self._built_lines(lines)[-1] - self.assertEqual( - own_line["invalidation_cause"], - "accepted_block_preview_withdrawn", - ) - self.assertEqual( - own_line["force_origin"], "caller_flag+reserved_cause" - ) - self.assertFalse(own_line["cause_inherited"]) - self.assertTrue(own_line["payout_mutated"]) - self.assertEqual(own_line["prior_balances_source"], "ledger") - - # A routine build through the capture path at the same, still - # current, source generation must not rescan. - server.payout_artifact_min_build_interval_seconds = 0.0 - append_incremental_share( - ledger, share_seq=4, accepted_at_ms=1_000_010 - ) - clock_ms[0] = 1_000_020 - routine = server._current_payout_state_candidate() - assert routine.ledger_artifact is not None - self.assertNotEqual( - routine.ledger_artifact.window_build_mode, "full_rescan" - ) - self.assertEqual(ledger.full_snapshot_calls, full_after_own) - self.assertEqual(routine.ledger_artifact.window_delta_rows, 1) - routine_line = self._built_lines(lines)[-1] - self.assertEqual( - routine_line["invalidation_cause"], - "accepted_block_preview_withdrawn", - ) - self.assertTrue(routine_line["cause_inherited"]) - self.assertEqual(routine_line["force_origin"], "none") - self.assertFalse(routine_line["payout_mutated"]) - # The tuple itself is untouched: supersession readers still see it. - self.assertEqual( - server._payout_state_source[2], - "accepted_block_preview_withdrawn", - ) - - def test_payout_state_source_supersession_still_detected_after_the_fix( - self, - ) -> None: - server, ledger, artifacts = self.configured_server() - clock_ms = [1_000_000] - with patch( - "lab.prism.prism_coordinator.now_ms", - side_effect=lambda: clock_ms[0], - ), _captured_payout_artifact_logs(): - captured = server._capture_payout_state_source() - stale = server._prepared_payout_state_candidate(captured) - # A newer source is reserved before publication: the candidate - # prepared against generation N is discarded, not published. - server._reserve_payout_state_source( - "external_tip", tip_hash="ab" * 32 - ) - server._block_payout_state_publication(force=True) - self.assertIsNone(server._publish_payout_state_candidate(stale)) - self.assertEqual(server.payout_state_candidates_discarded, 1) - self.assertEqual(server._payout_state_generation, 0) - # Reserve-if-current refuses a superseded expectation exactly as - # before, so a withdrawal cannot attach its cause to a source - # generation that already moved. - self.assertIsNone( - server._reserve_payout_state_source_if_current( - captured[1], - "accepted_block_preview_withdrawn", - tip_hash=captured[2], - ) - ) - # The current source still publishes. - fresh = server._current_payout_state_candidate() - self.assertEqual(fresh.source_generation, server._payout_state_source[0]) - self.assertEqual(server._publish_payout_state_candidate(fresh), 1) - self.assertEqual(server._payout_state_generation, 1) - self.assertEqual( - server._published_payout_state.source_generation, - fresh.source_generation, - ) - - def test_window_build_phase_attribution_pins_the_split(self) -> None: - class SlowOracleLedger(IncrementalRecordingLedger): - oracle_delay_seconds = 0.3 - - def snapshot_at_job_issue( - self, - anchor_job_issued_at_ms: int, - *, - window_weight: int | None = None, - ) -> list[object]: - time.sleep(self.oracle_delay_seconds) - return super().snapshot_at_job_issue( - anchor_job_issued_at_ms, - window_weight=window_weight, - ) - - ledger = SlowOracleLedger() - for share_seq in range(1, 4): - append_incremental_share( - ledger, - share_seq=share_seq, - accepted_at_ms=999_900 + share_seq, - ) - server, rpc = coordinator(ledger=ledger) - artifacts = server.store_template_artifacts(dict(rpc.template)) - assert artifacts is not None - server._pool_ready_latched = True - with patch( - "lab.prism.prism_coordinator.now_ms", - return_value=1_000_000, - ), _captured_payout_artifact_logs() as lines: - built = server._build_payout_ledger_artifact( - 0, 0, artifacts.network_difficulty - ) - assert built is not None - self.assertEqual(built.window_build_mode, "full_rescan") - line = self._built_lines(lines)[-1] - phases = line["phase_seconds"] - assert isinstance(phases, dict) - self.assertEqual( - set(phases), set(PRISM_PAYOUT_WINDOW_BUILD_PHASES) - ) - self.assertEqual( - PRISM_PAYOUT_WINDOW_BUILD_PHASES, - ("ledger_read", "record_conversion", "daemon_prepare", "lock_wait"), - ) - # The known-slow ledger read lands in the ledger-read phase and not - # in conversion, daemon, or lock time. - self.assertGreaterEqual( - float(phases["ledger_read"]), ledger.oracle_delay_seconds - ) - other = ( - float(phases["record_conversion"]) - + float(phases["daemon_prepare"]) - + float(phases["lock_wait"]) - ) - self.assertLess(other, 0.1) - self.assertEqual(float(phases["daemon_prepare"]), 0.0) - # The phases account for the reported build within 0.1 s (the - # unattributed remainder is artifact construction and the cheap - # accepted-share count, both microseconds on this ledger). - tolerance_seconds = 0.1 - self.assertLessEqual( - abs(sum(float(value) for value in phases.values()) - float(line["duration_seconds"])), - tolerance_seconds, - ) - # Exported as one fixed-cardinality histogram family over the - # closed phase and outcome sets. - metrics = server.payout_state_metrics_lines() - family = "qbit_prism_payout_window_build_phase_seconds" - count_lines = [ - entry for entry in metrics if entry.startswith(f"{family}_count{{") - ] - exported_phases = { - entry.split('phase="', 1)[1].split('"', 1)[0] for entry in count_lines - } - exported_outcomes = { - entry.split('outcome="', 1)[1].split('"', 1)[0] for entry in count_lines - } - self.assertEqual(exported_phases, set(PRISM_PAYOUT_WINDOW_BUILD_PHASES)) - self.assertEqual(exported_outcomes, set(PRISM_PAYOUT_WINDOW_BUILD_OUTCOMES)) - completed = 'phase="ledger_read",outcome="completed"' - self.assertIn(f"{family}_count{{{completed}}} 1", metrics) - self.assertIn(f"# TYPE {family} histogram", metrics) - ledger_sum = [ - entry for entry in metrics if entry.startswith(f"{family}_sum{{{completed}}}") - ] - self.assertEqual(len(ledger_sum), 1) - self.assertGreaterEqual( - float(ledger_sum[0].split()[-1]), ledger.oracle_delay_seconds - ) - - def test_tip_refresh_wave_does_not_wait_for_a_full_rescan_when_a_recent_window_exists( - self, - ) -> None: - server, ledger, artifacts = self.configured_server() - recorded = install_fake_bundle_builder(server) - clock_ms = [1_000_000] - with patch( - "lab.prism.prism_coordinator.now_ms", - side_effect=lambda: clock_ms[0], - ), _captured_payout_artifact_logs() as lines: - initial = server._build_payout_ledger_artifact( - 0, 0, artifacts.network_difficulty - ) - assert initial is not None - self.assertTrue(server._install_payout_ledger_artifact(initial)) - # Publish the generation's balance artifact up front, as every - # publication does in production, so the job build below never - # needs the prepare lock for a balance read. - server._current_payout_state_artifact() - - # An invalidating build (a forced reorg or withdrawal rescan) - # owns the prepare lock for longer than the whole budget. - lock_held = threading.Event() - release_lock = threading.Event() - - def hold_prepare_lock() -> None: - with server._payout_state_prepare_lock: - lock_held.set() - release_lock.wait(30.0) - - holder = threading.Thread(target=hold_prepare_lock, daemon=True) - holder.start() - self.assertTrue(lock_held.wait(5.0)) - try: - # The routine candidate (what the tip-refresh path prepares) - # comes back inside the budget with the armed window. - candidates: list[PayoutStateCandidate] = [] - routine = threading.Thread( - target=lambda: candidates.append( - server._current_payout_state_candidate() - ), - daemon=True, - ) - routine.start() - routine.join(self.NON_BLOCKING_BUDGET_SECONDS) - self.assertFalse( - routine.is_alive(), - "routine candidate preparation queued behind the rescan", - ) - candidate = candidates[0] - artifact = candidate.ledger_artifact - assert artifact is not None - self.assertEqual(artifact.window_build_mode, "tip_refresh_cached") - self.assertEqual( - artifact.window_full_rescan_reason, "prepare_lock_busy" - ) - self.assertEqual( - artifact.share_snapshot_sha256, - initial.share_snapshot_sha256, - ) - self.assertEqual(artifact.payout_state_generation, 1) - self.assertEqual(ledger.full_snapshot_calls, 1) - self.assertEqual(ledger.delta_snapshot_calls, 0) - cached_line = [ - line - for line in lines - if line.get("event") == "payout_artifact_tip_refresh_cached" - ][-1] - self.assertLessEqual( - float(cached_line["anchor_age_ms"]), - PRISM_TIP_REFRESH_WINDOW_STALENESS_SECONDS * 1000.0, - ) - self.assertEqual( - server.payout_artifact_event_counts["tip_refresh_cached"], - 1, - ) - - # Fanout itself: a ready job build during the same window - # completes on the armed window without touching the lock. - bundles: list[object] = [] - build = threading.Thread( - target=lambda: bundles.append( - server.shared_job_bundle(artifacts, mode="ready") - ), - daemon=True, - ) - build.start() - build.join(self.NON_BLOCKING_BUDGET_SECONDS) - self.assertFalse(build.is_alive(), "job build queued behind the rescan") - self.assertEqual(len(bundles), 1) - self.assertEqual(recorded["calls"], 1) - self.assertEqual(ledger.full_snapshot_calls, 1) - - # Past the accepted staleness bound the routine path must - # queue for a fresh build instead of propagating a window - # the re-anchor has already flagged stale. - clock_ms[0] = 1_000_000 + int( - PRISM_TIP_REFRESH_WINDOW_STALENESS_SECONDS * 1000.0 - ) + 1_000 - stale_candidates: list[PayoutStateCandidate] = [] - stale_routine = threading.Thread( - target=lambda: stale_candidates.append( - server._current_payout_state_candidate() - ), - daemon=True, - ) - stale_routine.start() - stale_routine.join(0.5) - self.assertTrue( - stale_routine.is_alive(), - "a window past the staleness bound was reused", - ) - finally: - release_lock.set() - holder.join(5.0) - stale_routine.join(10.0) - self.assertFalse(stale_routine.is_alive()) - stale_artifact = stale_candidates[0].ledger_artifact - assert stale_artifact is not None - self.assertNotEqual( - stale_artifact.window_build_mode, "tip_refresh_cached" - ) - self.assertEqual( - server.payout_artifact_event_counts["tip_refresh_cached"], 1 - ) - - def test_production_shaped_cadence_keeps_full_rescans_rare(self) -> None: - # Two simulated hours at the mainnet cadence (75 s blocks), with - # accepted-block confirmations, one reorg withdrawal, and maturity - # mutations at the observed 48 h ratio (55 confirmations and 8 - # reorgs per 48 h, rounded up per two hours). The assertion is a - # count, a property of the invalidation logic, never a duration. - server, ledger, artifacts = self.configured_server() - server.payout_artifact_min_build_interval_seconds = 0.0 - difficulty = int(artifacts.network_difficulty) - block_seconds = 75 - simulated_hours = 2 - blocks = simulated_hours * 3600 // block_seconds - confirmations_at = {20, 60} - withdrawal_at = {40} - maturity_at = {30, 70} - preview = [ - { - "recipient_id": "miner-a", - "order_key": "miner-a", - "p2mr_program_hex": "11" * 32, - "balance_sats": 25, - } - ] - clock_ms = [1_000_000] - share_seq = 3 - - def drain_background() -> None: - future = server._payout_artifact_future - if future is not None: - future.result(timeout=30.0) - - def reconciler_pass(*, payout_changed: bool) -> None: - # The reorg reconciler's candidate path, minus the RPC and - # ledger mutations: prepare under the lock with the #224 - # intent (a confirmed mutation rereads balances, never the - # window), fence, publish. - with server._payout_state_prepare_lock: - captured = server._capture_payout_state_source() - if not ( - payout_changed - or server._captured_payout_source_requires_publication(captured) - ): - return - candidate = server._prepared_payout_state_candidate( - captured, - force_full_window_rescan=False, - force_prior_balances_read=payout_changed, - ) - server._block_payout_state_publication(force=True) - self.assertIsNotNone(server._publish_payout_state_candidate(candidate)) - drain_background() - - with patch( - "lab.prism.prism_coordinator.now_ms", - side_effect=lambda: clock_ms[0], - ), _captured_payout_artifact_logs() as lines: - self.assertIsNotNone( - server._build_payout_ledger_artifact(0, 0, difficulty) - ) - pending_own_block: str | None = None - next_reanchor_ms = clock_ms[0] + 60_000 - for block in range(1, blocks + 1): - # Shares land between blocks. - for _ in range(5): - share_seq += 1 - clock_ms[0] += block_seconds * 1000 // 6 - append_incremental_share( - ledger, - share_seq=share_seq, - accepted_at_ms=clock_ms[0] - 5, - ) - clock_ms[0] += block_seconds * 1000 // 6 - if pending_own_block is not None: - # Our accepted block becomes the tip: the source tip - # already names it, so no external_tip is reserved. - tip_hash = pending_own_block - pending_own_block = None - else: - tip_hash = f"{block:064x}" - if server._payout_state_source[1] != tip_hash: - server._reserve_payout_state_source( - "external_tip", tip_hash=tip_hash - ) - reconciler_pass(payout_changed=block in maturity_at) - if block in confirmations_at or block in withdrawal_at: - own_hash = f"{0xA000 + block:064x}" - server._begin_accepted_block_payout_preview( - own_hash, block_height=100 + block - ) - server._mark_accepted_block_payout_landed( - own_hash, block_height=100 + block - ) - server._publish_accepted_block_payout_preview(own_hash, preview) - drain_background() - if block in withdrawal_at: - server._clear_accepted_block_payout_preview( - own_hash, invalidate_published=True - ) - drain_background() - else: - server._clear_accepted_block_payout_preview(own_hash) - pending_own_block = own_hash - if clock_ms[0] >= next_reanchor_ms: - # The background re-anchor cadence. - server._prepare_payout_ledger_artifact( - server._payout_state_generation, difficulty - ) - next_reanchor_ms = clock_ms[0] + 60_000 - - built = self._built_lines(lines) - full_rescans = [ - line for line in built if line["window_build_mode"] == "full_rescan" - ] - reasons = sorted(str(line.get("full_rescan_reason")) for line in full_rescans) - self.assertLessEqual( - len(full_rescans), - 1 * simulated_hours, - f"full rescans per simulated hour exceeded 1: {reasons}", - ) - # The withdrawal path keeps its fail-closed full rescan (#224); the - # maturity passes never produce one. - self.assertLessEqual( - reasons.count("reconcile_invalidation"), len(withdrawal_at) - ) - self.assertEqual( - server.payout_artifact_event_counts["full_rescan"], len(full_rescans) - ) - # Every payout mutation re-read its balances from the ledger, and - # every one that was not an explicit fail-closed rescan advanced the - # append-only window instead of rescanning it. - mutated = [line for line in built if line.get("payout_mutated")] - self.assertGreaterEqual(len(mutated), len(maturity_at) + len(withdrawal_at)) - for line in mutated: - self.assertEqual(line["prior_balances_source"], "ledger") - if not line["window_oracle_forced"]: - self.assertNotEqual(line["window_build_mode"], "full_rescan") - self.assertGreaterEqual( - sum(1 for line in mutated if line["prior_balances_read_forced"]), - len(maturity_at), - ) - # Each build is attributable from its own log line. - for line in built: - self.assertIn("invalidation_cause", line) - self.assertIn("force_origin", line) - self.assertIn("cause_inherited", line) - self.assertGreater(len(built), blocks) - - def test_preview_reservation_ownership_survives_interleaved_foreign_reservation( - self, - ) -> None: - # Review finding: ownership was a single newest-wins slot, so a - # foreign reservation at a later generation displaced the preview's - # record and its own preparation fell through to the routine path, - # losing the found-block immediacy and degrade of issue #188. - class BusyPrepareLock: - def acquire(self, blocking: bool = True) -> bool: - if blocking: - raise AssertionError("found-block path attempted to wait") - return False - - def release(self) -> None: - raise AssertionError("unacquired lock was released") - - class FailingDeltaLedger(IncrementalRecordingLedger): - fail_delta = False - - def snapshot_between_job_issues( - self, - previous_anchor_job_issued_at_ms: int, - anchor_job_issued_at_ms: int, - ) -> list[object]: - if self.fail_delta: - raise RuntimeError("delta unavailable") - return super().snapshot_between_job_issues( - previous_anchor_job_issued_at_ms, - anchor_job_issued_at_ms, - ) - - ledger = FailingDeltaLedger() - for share_seq in range(1, 4): - append_incremental_share( - ledger, - share_seq=share_seq, - accepted_at_ms=999_900 + share_seq, - ) - server, rpc = coordinator(ledger=ledger) - artifacts = server.store_template_artifacts(dict(rpc.template)) - assert artifacts is not None - server._pool_ready_latched = True - server.payout_artifact_min_build_interval_seconds = 0.0 - server.payout_artifact_full_rescan_seconds = 3_600.0 - real_lock = server._payout_state_prepare_lock - clock_ms = [1_000_000] - - def interleave_foreign_work(tip_hash: str) -> None: - # Another thread captures and prepares at the preview's - # generation, then reserves the next generation, before the - # preview path reaches its own preparation. - failures: list[BaseException] = [] - - def foreign() -> None: - try: - server._prepared_payout_state_candidate( - server._capture_payout_state_source() - ) - server._reserve_payout_state_source( - "external_tip", tip_hash=tip_hash - ) - except BaseException as exc: # pragma: no cover - asserted - failures.append(exc) - - thread = threading.Thread(target=foreign, daemon=True) - thread.start() - thread.join(10.0) - self.assertFalse(thread.is_alive()) - self.assertEqual(failures, []) - - with patch( - "lab.prism.prism_coordinator.now_ms", - side_effect=lambda: clock_ms[0], - ), patch("builtins.print"): - initial = server._build_payout_ledger_artifact( - 0, 0, artifacts.network_difficulty - ) - assert initial is not None - self.assertTrue(server._install_payout_ledger_artifact(initial)) - server._current_payout_state_artifact() - - # Cycle one: a busy prepare lock. The preview's preparation must - # not wait and must degrade to the armed window as a found-block - # candidate, not as a routine one. - captured = server._capture_payout_state_source() - reserved = server._reserve_payout_state_source_if_current( - captured[1], "accepted_block_preview", tip_hash="44" * 32 - ) - assert reserved is not None - interleave_foreign_work("55" * 32) - server._payout_state_prepare_lock = BusyPrepareLock() # type: ignore[assignment] - try: - candidate = server._prepared_payout_state_candidate(reserved) - finally: - server._payout_state_prepare_lock = real_lock - artifact = candidate.ledger_artifact - assert artifact is not None - self.assertEqual(artifact.window_build_mode, "found_block_cached") - self.assertEqual(artifact.window_full_rescan_reason, "prepare_lock_busy") - self.assertEqual(candidate.cause, "accepted_block_preview") - self.assertEqual(server.payout_artifact_event_counts["tip_refresh_cached"], 0) - - # Cycle two: a build that dies with the ledger. The preview's - # preparation must degrade instead of raising. - captured = server._capture_payout_state_source() - reserved = server._reserve_payout_state_source_if_current( - captured[1], "accepted_block_preview", tip_hash="66" * 32 - ) - assert reserved is not None - interleave_foreign_work("77" * 32) - ledger.fail_delta = True - clock_ms[0] = 1_000_020 - try: - candidate = server._prepared_payout_state_candidate(reserved) - finally: - ledger.fail_delta = False - artifact = candidate.ledger_artifact - assert artifact is not None - self.assertEqual(artifact.window_build_mode, "found_block_cached") - self.assertEqual(artifact.window_full_rescan_reason, "build_failed") - self.assertEqual(server.payout_artifact_event_counts["found_block_cached"], 2) - - def test_stale_preparation_cannot_rewind_cause_consumption(self) -> None: - # Review finding: a preparation carrying an older source generation - # could reset the consumed marker, after which the newer - # generation's reserved cause was honoured a second time and forced - # a duplicate payout-mutated build. - server, ledger, artifacts = self.configured_server() - clock_ms = [1_000_000] - with patch( - "lab.prism.prism_coordinator.now_ms", - side_effect=lambda: clock_ms[0], - ), _captured_payout_artifact_logs() as lines: - # Publish the generation's balance artifact so a routine build - # can reuse the published bytes: a build that honours the - # withdrawal cause a second time would show up as a ledger read. - server._current_payout_state_artifact() - initial = server._build_payout_ledger_artifact( - 0, 0, artifacts.network_difficulty - ) - assert initial is not None - self.assertEqual( - self._built_lines(lines)[-1]["prior_balances_source"], - "published", - ) - stale_captured = server._capture_payout_state_source() - reserved = server._reserve_payout_state_source_if_current( - stale_captured[1], - "accepted_block_preview_withdrawn", - tip_hash=stale_captured[2], - ) - assert reserved is not None - own = server._prepared_payout_state_candidate( - reserved, - force_full_window_rescan=True, - bypass_build_interval=True, - ) - assert own.ledger_artifact is not None - own_line = self._built_lines(lines)[-1] - self.assertEqual(own_line["force_origin"], "caller_flag+reserved_cause") - self.assertEqual(own_line["prior_balances_source"], "ledger") - # The stale capture, taken before the reservation, is prepared - # afterwards (a slow reconciler pass would do this). It acts on - # nothing, and its candidate is discarded at publication. - server.payout_artifact_min_build_interval_seconds = 0.0 - stale = server._prepared_payout_state_candidate(stale_captured) - self.assertEqual(stale.source_generation, stale_captured[1]) - stale_line = self._built_lines(lines)[-1] - self.assertEqual(stale_line["force_origin"], "none") - self.assertFalse(stale_line["payout_mutated"]) - server._block_payout_state_publication(force=True) - self.assertIsNone(server._publish_payout_state_candidate(stale)) - # The withdrawal generation's cause is not honoured again. - append_incremental_share( - ledger, share_seq=4, accepted_at_ms=1_000_010 - ) - clock_ms[0] = 1_000_020 - routine = server._current_payout_state_candidate() - assert routine.ledger_artifact is not None - routine_line = self._built_lines(lines)[-1] - self.assertEqual( - routine_line["invalidation_cause"], - "accepted_block_preview_withdrawn", - ) - self.assertTrue(routine_line["cause_inherited"]) - self.assertEqual(routine_line["force_origin"], "none") - self.assertFalse(routine_line["payout_mutated"]) - self.assertEqual(routine_line["prior_balances_source"], "published") - - def test_owner_entitlement_survives_a_foreign_preparation_at_a_later_generation( - self, - ) -> None: - # Review finding: pruning below the consumption high-water mark - # deleted the reserver's own record once a foreign preparation at - # a later generation moved the mark, so the preview's preparation - # fell to the routine path. The entitlement now travels with the - # reserved tuple, where no bookkeeping rule can reach it: after a - # burst of foreign reservations and a foreign preparation at the - # newest of them, the owner's preparation still takes the - # immediate path, non-blocking on a busy lock and degrading when - # its build dies with the ledger. - class BusyPrepareLock: - def acquire(self, blocking: bool = True) -> bool: - if blocking: - raise AssertionError("found-block path attempted to wait") - return False - - def release(self) -> None: - raise AssertionError("unacquired lock was released") - - class FailingDeltaLedger(IncrementalRecordingLedger): - fail_delta = False - - def snapshot_between_job_issues( - self, - previous_anchor_job_issued_at_ms: int, - anchor_job_issued_at_ms: int, - ) -> list[object]: - if self.fail_delta: - raise RuntimeError("delta unavailable") - return super().snapshot_between_job_issues( - previous_anchor_job_issued_at_ms, - anchor_job_issued_at_ms, - ) - - ledger = FailingDeltaLedger() - for share_seq in range(1, 4): - append_incremental_share( - ledger, - share_seq=share_seq, - accepted_at_ms=999_900 + share_seq, - ) - server, rpc = coordinator(ledger=ledger) - artifacts = server.store_template_artifacts(dict(rpc.template)) - assert artifacts is not None - server._pool_ready_latched = True - server.payout_artifact_min_build_interval_seconds = 0.0 - server.payout_artifact_full_rescan_seconds = 3_600.0 - service = server._ensure_payout_state_service() - real_lock = server._payout_state_prepare_lock - clock_ms = [1_000_000] - - def foreign_burst_then_preparation(burst: int, tip_base: int) -> None: - failures: list[BaseException] = [] - - def foreign() -> None: - try: - for index in range(burst): - server._reserve_payout_state_source( - "external_tip", tip_hash=f"{tip_base + index:064x}" - ) - # A foreign preparation at the newest generation moves - # the consumption mark past the owner's generation. - server._prepared_payout_state_candidate( - server._capture_payout_state_source() - ) - except BaseException as exc: # pragma: no cover - asserted - failures.append(exc) - - thread = threading.Thread(target=foreign, daemon=True) - thread.start() - thread.join(10.0) - self.assertFalse(thread.is_alive()) - self.assertEqual(failures, []) - - with patch( - "lab.prism.prism_coordinator.now_ms", - side_effect=lambda: clock_ms[0], - ), patch("builtins.print"): - initial = server._build_payout_ledger_artifact( - 0, 0, artifacts.network_difficulty - ) - assert initial is not None - self.assertTrue(server._install_payout_ledger_artifact(initial)) - server._current_payout_state_artifact() - - # Cycle one: busy prepare lock after a burst and a foreign - # preparation at a later generation. - captured = server._capture_payout_state_source() - reserved = server._reserve_payout_state_source_if_current( - captured[1], "accepted_block_preview", tip_hash="44" * 32 - ) - assert reserved is not None - owner_generation = reserved[1] - foreign_burst_then_preparation(200, 0x1000) - self.assertGreater( - service._payout_source_cause_high_water or 0, owner_generation - ) - server._payout_state_prepare_lock = BusyPrepareLock() # type: ignore[assignment] - try: - candidate = server._prepared_payout_state_candidate(reserved) - finally: - server._payout_state_prepare_lock = real_lock - artifact = candidate.ledger_artifact - assert artifact is not None - self.assertEqual(artifact.window_build_mode, "found_block_cached") - self.assertEqual(artifact.window_full_rescan_reason, "prepare_lock_busy") - self.assertEqual(server.payout_artifact_event_counts["tip_refresh_cached"], 0) - # The entitlement is one-shot: presenting the reserved tuple - # again inherits, exactly like a capture would. - self.assertFalse(service._consume_payout_source_cause(reserved)) - # And the mark was not rewound by the owner's older generation. - self.assertGreater( - service._payout_source_cause_high_water or 0, owner_generation - ) - - # Cycle two: a build that dies with the ledger, same - # interleaving. The preview's preparation degrades instead of - # raising. - captured = server._capture_payout_state_source() - reserved = server._reserve_payout_state_source_if_current( - captured[1], "accepted_block_preview", tip_hash="66" * 32 - ) - assert reserved is not None - foreign_burst_then_preparation(3, 0x2000) - ledger.fail_delta = True - clock_ms[0] = 1_000_020 - try: - candidate = server._prepared_payout_state_candidate(reserved) - finally: - ledger.fail_delta = False - artifact = candidate.ledger_artifact - assert artifact is not None - self.assertEqual(artifact.window_build_mode, "found_block_cached") - self.assertEqual(artifact.window_full_rescan_reason, "build_failed") - self.assertEqual(server.payout_artifact_event_counts["found_block_cached"], 2) - - def test_aborted_and_failed_builds_still_contribute_their_phases(self) -> None: - # Review finding: phases were observed only when a build finished, - # which biased the family toward fast builds. Every exit observes - # now, under a closed outcome, so a slow read that ends in an abort - # or a failure keeps its measured time. - class SlowThenEmptyLedger(IncrementalRecordingLedger): - oracle_delay_seconds = 0.2 - raise_after_read = False - - def snapshot_at_job_issue( - self, - anchor_job_issued_at_ms: int, - *, - window_weight: int | None = None, - ) -> list[object]: - # A slow oracle that comes back empty (or dies): the build - # pays for the read and then aborts on the empty window. - self.full_snapshot_calls += 1 - time.sleep(self.oracle_delay_seconds) - if self.raise_after_read: - raise RuntimeError("postgres statement deadline expired") - return [] - - # Shares exist (so the anchor clamp is ordinary and the build gets - # as far as the read), but the oracle answers with an empty window. - ledger = SlowThenEmptyLedger() - for share_seq in range(1, 4): - append_incremental_share( - ledger, - share_seq=share_seq, - accepted_at_ms=999_900 + share_seq, - ) - server, rpc = coordinator(ledger=ledger) - artifacts = server.store_template_artifacts(dict(rpc.template)) - assert artifacts is not None - server._pool_ready_latched = True - family = "qbit_prism_payout_window_build_phase_seconds" - - def cell(metrics: list[str], phase: str, outcome: str, product: str) -> float: - prefix = f'{family}_{product}{{phase="{phase}",outcome="{outcome}"}} ' - values = [entry for entry in metrics if entry.startswith(prefix)] - self.assertEqual(len(values), 1, prefix) - return float(values[0].split()[-1]) - - with patch( - "lab.prism.prism_coordinator.now_ms", - return_value=1_000_000, - ), patch("builtins.print"): - self.assertIsNone( - server._build_payout_ledger_artifact( - 0, 0, artifacts.network_difficulty - ) - ) - metrics = server.payout_state_metrics_lines() - self.assertEqual(cell(metrics, "ledger_read", "aborted", "count"), 1) - self.assertGreaterEqual( - cell(metrics, "ledger_read", "aborted", "sum"), - ledger.oracle_delay_seconds, - ) - self.assertEqual(cell(metrics, "ledger_read", "completed", "count"), 0) - - # The read itself dies: a failed build keeps its time too. - ledger.raise_after_read = True - self.assertIsNone( - server._build_payout_ledger_artifact( - 0, 0, artifacts.network_difficulty - ) - ) - metrics = server.payout_state_metrics_lines() - self.assertEqual(cell(metrics, "ledger_read", "failed", "count"), 1) - self.assertGreaterEqual( - cell(metrics, "ledger_read", "failed", "sum"), - ledger.oracle_delay_seconds, - ) - # A lost generation race aborts before any read and is counted - # as such, with no ledger time. - ledger.raise_after_read = False - self.assertIsNone( - server._build_payout_ledger_artifact( - 7, 7, artifacts.network_difficulty - ) - ) - metrics = server.payout_state_metrics_lines() - self.assertEqual(cell(metrics, "ledger_read", "aborted", "count"), 2) - self.assertLess( - cell(metrics, "ledger_read", "aborted", "sum"), - 2 * ledger.oracle_delay_seconds, - ) - for outcome in PRISM_PAYOUT_WINDOW_BUILD_OUTCOMES: - for phase in PRISM_PAYOUT_WINDOW_BUILD_PHASES: - cell(metrics, phase, outcome, "count") - - def test_deferred_self_check_is_bounded_under_frequent_builds(self) -> None: - # Review finding: the deferral measured freshness against the last - # build, so candidates arriving faster than the staleness bound - # deferred the hourly self-check every time it came due. It is now - # bounded against the last self-check attempt, and a deferral hands - # the check to the background walker at the next publication. - server, ledger, artifacts = self.configured_server() - # Part one: no background walker at all (reuse off), builds every - # 20 s against a 100 s check interval and a 60 s bound, across two - # check intervals. The check must run on the critical path once the - # grace expires, and never later than that. - server.payout_artifact_reuse_enabled = False - server.payout_artifact_min_build_interval_seconds = 0.0 - server.payout_artifact_full_rescan_seconds = 100.0 - self.assertEqual(PRISM_TIP_REFRESH_WINDOW_STALENESS_SECONDS, 60.0) - clock_ms = [1_000_000] - monotonic_seconds = [1_000.0] - share_seq = 3 - outcomes: list[tuple[int, str, bool]] = [] - with patch( - "lab.prism.prism_coordinator.now_ms", - side_effect=lambda: clock_ms[0], - ), patch( - "lab.prism.prism_coordinator.time.monotonic", - side_effect=lambda: monotonic_seconds[0], - ), _captured_payout_artifact_logs() as lines: - initial = server._build_payout_ledger_artifact( - 0, 0, artifacts.network_difficulty - ) - assert initial is not None - for step in range(1, 17): - elapsed = 20 * step - monotonic_seconds[0] = 1_000.0 + elapsed - clock_ms[0] = 1_000_000 + elapsed * 1_000 - share_seq += 1 - append_incremental_share( - ledger, share_seq=share_seq, accepted_at_ms=clock_ms[0] - 5 - ) - candidate = server._current_payout_state_candidate() - assert candidate.ledger_artifact is not None - line = self._built_lines(lines)[-1] - outcomes.append( - ( - elapsed, - str(line["window_build_mode"]), - bool(line["self_check_deferred"]), - ) - ) - self.assertEqual( - [elapsed for elapsed, mode, _ in outcomes if mode == "self_check_match"], - [160, 320], - ) - self.assertEqual( - [elapsed for elapsed, _, deferred in outcomes if deferred], - [100, 120, 140, 260, 280, 300], - ) - self.assertEqual(ledger.full_snapshot_calls, 3) - self.assertIsNone(server._payout_artifact_future) - - # Part two: with the walker available, a deferral is handed to it at - # the next publication and the check runs there, off the critical - # path, before the grace expires. - server, ledger, artifacts = self.configured_server() - server.payout_artifact_min_build_interval_seconds = 0.0 - server.payout_artifact_full_rescan_seconds = 100.0 - clock_ms = [1_000_000] - monotonic_seconds = [1_000.0] - share_seq = 3 - with patch( - "lab.prism.prism_coordinator.now_ms", - side_effect=lambda: clock_ms[0], - ), patch( - "lab.prism.prism_coordinator.time.monotonic", - side_effect=lambda: monotonic_seconds[0], - ), _captured_payout_artifact_logs() as lines: - initial = server._build_payout_ledger_artifact( - 0, 0, artifacts.network_difficulty - ) - assert initial is not None - self.assertTrue(server._install_payout_ledger_artifact(initial)) - server._current_payout_state_artifact() - candidate: PayoutStateCandidate | None = None - for step in range(1, 6): - elapsed = 20 * step - monotonic_seconds[0] = 1_000.0 + elapsed - clock_ms[0] = 1_000_000 + elapsed * 1_000 - share_seq += 1 - append_incremental_share( - ledger, share_seq=share_seq, accepted_at_ms=clock_ms[0] - 5 - ) - candidate = server._current_payout_state_candidate() - assert candidate is not None and candidate.ledger_artifact is not None - deferred_line = self._built_lines(lines)[-1] - self.assertTrue(deferred_line["self_check_deferred"]) - self.assertEqual(deferred_line["window_build_mode"], "incremental") - self.assertEqual(ledger.full_snapshot_calls, 1) - server._block_payout_state_publication(force=True) - self.assertEqual(server._publish_payout_state_candidate(candidate), 1) - future = server._payout_artifact_future - self.assertIsNotNone(future, "publication did not hand the owed check to the walker") - assert future is not None - future.result(timeout=30.0) - background = [ - line - for line in self._built_lines(lines) - if line["invalidation_cause"] is None - and line["window_build_mode"] == "self_check_match" - ] - self.assertEqual(len(background), 1) - self.assertEqual(background[0]["payout_state_generation"], 1) - self.assertEqual(ledger.full_snapshot_calls, 2) - # The next routine build is not overdue and defers nothing. - monotonic_seconds[0] = 1_120.0 - clock_ms[0] = 1_120_000 - share_seq += 1 - append_incremental_share( - ledger, share_seq=share_seq, accepted_at_ms=clock_ms[0] - 5 - ) - later = server._current_payout_state_candidate() - assert later.ledger_artifact is not None - later_line = self._built_lines(lines)[-1] - self.assertFalse(later_line["self_check_deferred"]) - self.assertEqual(later_line["window_build_mode"], "incremental") - self.assertEqual(ledger.full_snapshot_calls, 2) - - -class UnfencedAppendDrainTests(unittest.TestCase): - def test_drain_waits_in_stamped_slices_without_returning_early(self) -> None: - """The drain stays unbounded; only its silence is broken up. - - A landing calls this inline on the watchdog-monitored block-work - thread right after exposing its declared anchor. Bounding the wait is - not an option -- it enforces the fencing contract, and returning - early would let the landing arm its epoch fences ahead of an append - that must be drained first -- so the fix is to wait in - watchdog-sized slices and stamp each one. The ``while`` re-check - makes the timed wait semantically identical to the untimed one. - """ - server, _rpc = coordinator() - server._ensure_job_cache_state() - stamps: list[tuple[str, object]] = [] - server._record_heartbeat = ( # type: ignore[method-assign] - lambda name, phase=None: stamps.append((name, phase)) - ) - server._block_work_wait_slice = lambda: 0.01 # type: ignore[method-assign] - anchor = 1000 - with server._payout_unfenced_append_drained: - server._payout_unfenced_append_inflight_stamps[1] = anchor - 1 - returned = threading.Event() - - def drain() -> None: - server._block_submitter_thread_ident = threading.get_ident() - server._await_unfenced_appends_predating_anchor(anchor) - returned.set() - - worker_thread = threading.Thread(target=drain) - worker_thread.start() - try: - # The predating append is still in flight: no number of expired - # slices may let the drain return. - self.assertFalse(returned.wait(0.2)) - drain_stamps = [ - stamp - for stamp in stamps - if stamp == ("block_submitter", "wait-unfenced-append-drain") - ] - self.assertGreaterEqual(len(drain_stamps), 2) - with server._payout_unfenced_append_drained: - server._payout_unfenced_append_inflight_stamps.pop(1, None) - server._payout_unfenced_append_drained.notify_all() - self.assertTrue(returned.wait(5)) - finally: - with server._payout_unfenced_append_drained: - server._payout_unfenced_append_inflight_stamps.pop(1, None) - server._payout_unfenced_append_drained.notify_all() - worker_thread.join(5) - - def test_drain_with_nothing_in_flight_stamps_nothing(self) -> None: - """The common path never enters the loop, so it never stamps.""" - server, _rpc = coordinator() - server._ensure_job_cache_state() - stamps: list[object] = [] - server._record_heartbeat = ( # type: ignore[method-assign] - lambda name, phase=None: stamps.append((name, phase)) - ) - server._block_submitter_thread_ident = threading.get_ident() - server._await_unfenced_appends_predating_anchor(1000) - self.assertEqual(stamps, []) - - -class AcceptedParentPreviewBackpressureTests(unittest.TestCase): - """#182: the bounded accepted-parent preview wait is typed backpressure.""" - - def test_bounded_wait_timeout_raises_typed_preview_pending(self) -> None: - server, _rpc = coordinator() - server.accepted_block_payout_preview_wait_seconds = 0.0 - block_hash = "b1" * 32 - server._begin_accepted_block_payout_preview(block_hash, block_height=10) - server._mark_accepted_block_payout_landed(block_hash, block_height=10) - before = int(getattr(server, "_accepted_parent_preview_wait_timeouts", 0)) - - with self.assertRaises(AcceptedParentPayoutPreviewPending) as caught: - server._await_pending_parent_payout_preview(block_hash) - - exc = caught.exception - # Compatibility: every caller that catches either parent still catches - # this, so no existing handler changes behavior. - self.assertIsInstance(exc, _PayoutStatePublicationBlocked) - self.assertIsInstance(exc, TemplateRefreshBlocked) - # An explicit, machine-readable retry/log reason. - self.assertEqual(exc.retry_reason, "accepted_parent_preview_pending") - self.assertEqual(exc.parent_hash, block_hash) - self.assertEqual(exc.waited_seconds, 0.0) - self.assertIn("not ready yet", str(exc)) - # The existing timeout counter stays meaningful and is carried on the - # exception so a retry log can report it without a second read. - self.assertEqual(exc.timeout_count, before + 1) - self.assertEqual( - server._accepted_parent_preview_wait_timeouts, before + 1 - ) - server._clear_accepted_block_payout_preview(block_hash) - - def test_published_preview_and_terminal_transition_do_not_raise(self) -> None: - server, _rpc = coordinator() - server.accepted_block_payout_preview_wait_seconds = 0.0 - block_hash = "b2" * 32 - preview = [ - { - "recipient_id": "miner-a", - "order_key": "miner-a", - "p2mr_program_hex": "11" * 32, - "balance_sats": 25, - } - ] - server._begin_accepted_block_payout_preview(block_hash, block_height=10) - server._mark_accepted_block_payout_landed(block_hash, block_height=10) - with patch("builtins.print"): - server._publish_accepted_block_payout_preview(block_hash, preview) - - # A published preview is the success path: no backpressure, no timeout. - self.assertEqual( - server._await_pending_parent_payout_preview(block_hash), preview - ) - self.assertEqual( - int(getattr(server, "_accepted_parent_preview_wait_timeouts", 0)), 0 - ) - # A parent with no governing transition returns None untouched. - server._clear_accepted_block_payout_preview(block_hash) - self.assertIsNone(server._await_pending_parent_payout_preview(block_hash)) - self.assertEqual( - int(getattr(server, "_accepted_parent_preview_wait_timeouts", 0)), 0 - ) - - def test_withdrawn_preview_keeps_its_existing_type(self) -> None: - server, _rpc = coordinator() - server.accepted_block_payout_preview_wait_seconds = 0.0 - block_hash = "b3" * 32 - server._begin_accepted_block_payout_preview(block_hash, block_height=10) - server._mark_accepted_block_payout_landed(block_hash, block_height=10) - server._clear_accepted_block_payout_preview( - block_hash, - invalidate_published=True, - ) - - with self.assertRaisesRegex( - TemplateRefreshBlocked, "was withdrawn" - ) as caught: - server._await_pending_parent_payout_preview(block_hash) - - self.assertNotIsInstance( - caught.exception, AcceptedParentPayoutPreviewPending - ) - # A withdrawal is not a bounded-wait expiry, so it never touches the - # preview-wait timeout counter. - self.assertEqual( - int(getattr(server, "_accepted_parent_preview_wait_timeouts", 0)), 0 - ) - - def test_unresolved_depth_cap_keeps_its_existing_type(self) -> None: - server, _rpc = coordinator() - server.accepted_parent_unresolved_depth_max = 1 - server.accepted_block_payout_preview_wait_seconds = 0.0 - hashes = ["b4" * 32, "b5" * 32] - for height, block_hash in enumerate(hashes, start=10): - server._begin_accepted_block_payout_preview( - block_hash, block_height=height - ) - server._mark_accepted_block_payout_landed( - block_hash, block_height=height - ) - - with self.assertRaisesRegex( - TemplateRefreshBlocked, "exceeds cap" - ) as caught: - server._await_pending_parent_payout_preview(hashes[0]) - - self.assertNotIsInstance( - caught.exception, AcceptedParentPayoutPreviewPending - ) - self.assertEqual( - int(getattr(server, "_accepted_parent_preview_wait_timeouts", 0)), 0 - ) - for block_hash in hashes: - server._clear_accepted_block_payout_preview(block_hash) - - def test_owed_replay_enumeration_keeps_its_existing_type(self) -> None: - server, _rpc = coordinator() - - class EnumLedger(FakeLedger): - def pending_block_candidate_rows( - self, *, limit: int = 32 - ) -> list[dict[str, object]]: - return [] - - server.ledger = EnumLedger() - server._note_block_replay_enumeration_owed() - - with self.assertRaisesRegex( - TemplateRefreshBlocked, "has not enumerated" - ) as caught: - server._await_pending_parent_payout_preview("b6" * 32) - - self.assertNotIsInstance( - caught.exception, AcceptedParentPayoutPreviewPending - ) - - def test_preview_pending_uses_the_coordination_blocked_budget(self) -> None: - """The typed raise reaches the refresh poller as coordination churn. - - Sustained pendency accrues the separate coordination-blocked streak - (which the publication-progress watchdog enforces) instead of arming - the ordinary template-refresh failure budget, so it neither retries - silently forever nor exits on the wrong clock. - """ - server, _rpc = coordinator() - server.reorg_reconciler_enabled = True - server.accepted_block_payout_preview_wait_seconds = 0.0 - block_hash = "b7" * 32 - server._begin_accepted_block_payout_preview(block_hash, block_height=10) - server._mark_accepted_block_payout_landed(block_hash, block_height=10) - server.fetch_qbit_tip_template_snapshot = ( # type: ignore[method-assign] - lambda: (_ for _ in ()).throw( - AcceptedParentPayoutPreviewPending( - "accepted parent payout preview is not ready yet" - ) - ) - ) - - with ( - patch( - "lab.prism.prism_coordinator.time.monotonic", - return_value=100.0, - ), - self.assertRaises(AcceptedParentPayoutPreviewPending), - ): - server.poll_qbit_tip_template_once() - - self.assertIsNone( - getattr(server, "template_refresh_failure_started_monotonic", None) - ) - self.assertFalse(server.template_refresh_failure_expired(10_000.0)) - # The long-lived coordination escalation still runs on its own clock. - self.assertEqual( - server.coordination_blocked_streak_age_seconds(105.0), 5.0 - ) - server._clear_accepted_block_payout_preview(block_hash) - - def test_plain_template_refresh_blocked_still_uses_generic_budget(self) -> None: - """Negative control: only the typed subclass is exempt.""" - server, _rpc = coordinator() - server.reorg_reconciler_enabled = True - server.template_refresh_failure_exit_seconds = 10.0 - server.fetch_qbit_tip_template_snapshot = ( # type: ignore[method-assign] - lambda: (_ for _ in ()).throw( - TemplateRefreshBlocked( - "unable to derive exact artifacts for observed qbit template" - ) - ) - ) - - with ( - patch( - "lab.prism.prism_coordinator.time.monotonic", - return_value=100.0, - ), - self.assertRaises(TemplateRefreshBlocked), - ): - server.poll_qbit_tip_template_once() - - self.assertEqual( - server.template_refresh_failure_started_monotonic, 100.0 - ) - self.assertTrue(server.template_refresh_failure_expired(110.0)) - self.assertEqual( - server.coordination_blocked_streak_age_seconds(110.0), 0.0 - ) - - -class SnapshotWindowWeightBandTests(unittest.TestCase): - def test_equal_weights_are_within_band(self) -> None: - self.assertTrue(_snapshot_window_weight_within_band(1_000, 1_000)) - - def test_floor_boundary_is_exact(self) -> None: - # cached * 8 >= live * 5 holds at exactly 5/8 and fails one below. - self.assertTrue(_snapshot_window_weight_within_band(625, 1_000)) - self.assertFalse(_snapshot_window_weight_within_band(624, 1_000)) - - def test_ceiling_boundary_is_exact(self) -> None: - self.assertTrue(_snapshot_window_weight_within_band(2_000, 1_000)) - self.assertFalse(_snapshot_window_weight_within_band(2_001, 1_000)) - - def test_non_positive_weights_fail_closed(self) -> None: - self.assertFalse(_snapshot_window_weight_within_band(0, 1_000)) - self.assertFalse(_snapshot_window_weight_within_band(1_000, 0)) - self.assertFalse(_snapshot_window_weight_within_band(-16, -16)) - - -if __name__ == "__main__": - unittest.main() diff --git a/tests/test_prism_payout_window_daemon_recenter.py b/tests/test_prism_payout_window_daemon_recenter.py deleted file mode 100644 index f52de2bf6..000000000 --- a/tests/test_prism_payout_window_daemon_recenter.py +++ /dev/null @@ -1,501 +0,0 @@ -#!/usr/bin/env python3 -"""Issue #207: the periodic self-check must leave the Rust daemon prepared. - -The daemon here is an in-process fake that serves ``prepare_window`` with -the real Python fold, exactly like the subprocess fixture the window-pipeline -suite uses, but addressable from the test: it counts full preparations and -advances, answers ``needs_full`` for a digest it does not hold, and can be -evicted on demand. That makes three coordinator contracts checkable without -a Rust build: - -- ordinary in-band drift keeps the cached weight and digest warm, so the - next build advances through the daemon without a second oracle read; -- an oversized re-center adopts the live-weight digest AND prepares the - daemon for it before the next advance, so the deliberate re-center never - pays a second full ``snapshot_at_job_issue`` under the writer lock; -- a genuine eviction still reports ``window_daemon_state_lost`` and takes - the existing recovery path, and the strings stay distinguishable from a - deliberate re-center that could not prepare the daemon. -""" -# ruff: noqa: F403, F405 - -from __future__ import annotations - -import os -import unittest - -from lab.prism.bundle_compiler import PreparedWindowOutcome -from lab.prism.payout_state import PRISM_REWARD_WINDOW_MULTIPLIER -from lab.prism.payout_state import PRISM_SNAPSHOT_WINDOW_MARGIN -from lab.prism.share_ledger import ( - AcceptedShareRecord, - DaemonShareWindowMirror, - IncrementalShareWindow, - IncrementalWindowFallback, - PendingShare, -) -from tests.prism_coordinator_test_support import * - - -def _canonical_items(window: IncrementalShareWindow) -> bytes: - return b",".join( - page.canonical_json_items - for page in window.pages - if page.canonical_json_items - ) - - -class InProcessFakeDaemon: - """``prepare_payout_window`` served by the shipped Python fold. - - Held windows are addressed by digest alone, like the real daemon; an - advance against a digest that is not held answers ``needs_full``. - """ - - def __init__(self) -> None: - self.held: dict[str, IncrementalShareWindow] = {} - self.full_prepares = 0 - self.advances = 0 - self.needs_full_answers = 0 - self.full_prepare_statuses: list[str] = [] - # Statuses to answer the next full preparations with instead of - # folding (fault injection for the unprepared-adoption test). - self.inject_full_statuses: list[str] = [] - - def evict(self) -> None: - self.held.clear() - - def __call__( - self, - *, - mode: str, - records_json: list[dict[str, object]], - anchor_job_issued_at_ms: int, - append_invalidation_epoch: int, - base_digest: str | None = None, - window_weight: int | None = None, - page_size: int | None = None, - wait_for_daemon: bool = True, - ) -> PreparedWindowOutcome | None: - records = [ - AcceptedShareRecord( - **{ - key: value - for key, value in record_json.items() - if value is not None or key == "credit_policy" - } - ) - for record_json in records_json - ] - anchor = int(anchor_job_issued_at_ms) - if mode == "full": - self.full_prepares += 1 - if self.inject_full_statuses: - status = self.inject_full_statuses.pop(0) - self.full_prepare_statuses.append(status) - return PreparedWindowOutcome(status=status) - assert window_weight is not None - window = IncrementalShareWindow.from_full_snapshot( - records, - anchor_job_issued_at_ms=anchor, - window_weight=int(window_weight), - page_size=int(page_size or 512), - ) - digest = window.json_records().canonical_json_sha256() - self.held = {digest: window} - self.full_prepare_statuses.append("prepared") - return PreparedWindowOutcome( - status="prepared", - share_snapshot_sha256=digest, - record_count=window.record_count, - window_items=_canonical_items(window), - ) - assert mode == "advance", mode - base = self.held.get(str(base_digest)) - if base is None: - self.needs_full_answers += 1 - return PreparedWindowOutcome( - status="needs_full", - error=f"prepared window {base_digest} is not held", - ) - old_items = _canonical_items(base) - try: - advanced, stats = base.advance( - records, - anchor_job_issued_at_ms=anchor, - ) - except IncrementalWindowFallback as error: - return PreparedWindowOutcome(status="fallback", error=str(error)) - digest = advanced.json_records().canonical_json_sha256() - new_items = _canonical_items(advanced) - drop = 0 - while drop <= len(old_items): - retained = old_items[drop:] - if new_items[: len(retained)] == retained: - break - drop += 1 - appended = new_items[len(old_items) - drop :] - self.held = {digest: advanced} - self.advances += 1 - return PreparedWindowOutcome( - status="prepared", - share_snapshot_sha256=digest, - record_count=advanced.record_count, - added_rows=stats.added_rows, - expired_rows=stats.expired_rows, - touched_pages=stats.touched_pages, - retained_drop_bytes=drop, - appended_items=appended, - ) - - -def _append_heavy_share( - ledger: SingleWriterShareLedger, - *, - share_seq: int, - accepted_at_ms: int, - share_difficulty: int, -) -> None: - """A share heavy enough that a handful of them fill the reward window.""" - ledger.append( - PendingShare( - share_id=f"heavy-{share_seq}", - miner_id=f"miner-{share_seq % 3}", - order_key=f"miner-{share_seq % 3}", - p2mr_program_hex=f"{share_seq % 256:02x}" * 32, - share_difficulty=share_difficulty, - network_difficulty=share_difficulty, - template_height=9, - job_id=f"job-{share_seq}", - job_issued_at_ms=accepted_at_ms - 1, - accepted_at_ms=accepted_at_ms, - ntime=1_700_000_000 + share_seq, - ) - ) - - -class DaemonRecenterTests(unittest.TestCase): - """Issue #207 regression tests against the in-process fake daemon.""" - - def setUp(self) -> None: - patcher = patch.dict(os.environ, {"PRISM_WINDOW_PIPELINE_RUST": "1"}) - patcher.start() - self.addCleanup(patcher.stop) - - def _server( - self, - ) -> tuple[PrismCoordinator, IncrementalRecordingLedger, object, InProcessFakeDaemon]: - ledger = IncrementalRecordingLedger() - server, rpc = coordinator(ledger=ledger) - artifacts = server.store_template_artifacts(dict(rpc.template)) - assert artifacts is not None - difficulty = int(artifacts.network_difficulty) - # Twenty shares each worth one unit of network difficulty: the - # window at the live weight (16 * difficulty) retains sixteen, so a - # re-center to a smaller weight changes the retained set and hence - # the digest the daemon must hold. - for share_seq in range(1, 21): - _append_heavy_share( - ledger, - share_seq=share_seq, - accepted_at_ms=999_000 + share_seq * 10, - share_difficulty=difficulty, - ) - server._pool_ready_latched = True - server.payout_artifact_min_build_interval_seconds = 60.0 - server.payout_artifact_full_rescan_seconds = 3_600.0 - daemon = InProcessFakeDaemon() - server.prepare_payout_window = daemon # type: ignore[method-assign] - return server, ledger, artifacts, daemon - - @staticmethod - def _oracle_digest( - ledger: IncrementalRecordingLedger, - *, - anchor_ms: int, - window_weight: int, - ) -> str: - records = list( - SingleWriterShareLedger.snapshot_at_job_issue( - ledger, - anchor_ms, - window_weight=None, - ) - ) - window = IncrementalShareWindow.from_full_snapshot( - records, - anchor_job_issued_at_ms=anchor_ms, - window_weight=window_weight, - ) - return window.json_records().canonical_json_sha256() - - def test_oversized_recenter_leaves_the_daemon_prepared(self) -> None: - server, ledger, artifacts, daemon = self._server() - difficulty = int(artifacts.network_difficulty) - clock_ms = [1_000_000] - with patch( - "lab.prism.prism_coordinator.now_ms", - side_effect=lambda: clock_ms[0], - ): - initial = server._build_payout_ledger_artifact(0, 0, difficulty) - assert initial is not None - self.assertEqual(initial.window_build_mode, "full_rescan") - self.assertEqual(daemon.full_prepares, 1) - self.assertEqual(ledger.full_snapshot_calls, 1) - - # Difficulty collapses far enough (cached/live = 4/3 > 5/4) that - # the periodic self-check re-centers the window at the live - # weight; the adopted digest differs from the daemon's. - server.payout_artifact_min_build_interval_seconds = 0.0 - server.payout_artifact_full_rescan_seconds = 0.0 - clock_ms[0] = 1_000_020 - collapsed_difficulty = difficulty * 3 // 4 - live_weight = ( - PRISM_REWARD_WINDOW_MULTIPLIER - * PRISM_SNAPSHOT_WINDOW_MARGIN - * collapsed_difficulty - ) - checked = server._build_payout_ledger_artifact( - 0, 0, collapsed_difficulty - ) - assert checked is not None - self.assertEqual(checked.window_build_mode, "self_check_match") - self.assertEqual( - checked.window_full_rescan_reason, "periodic_self_check" - ) - # The deliberate re-center is flagged on the build itself. - self.assertTrue(checked.window_self_check_recentered) - self.assertFalse(initial.window_self_check_recentered) - self.assertEqual(ledger.full_snapshot_calls, 2) - cached = server._incremental_payout_artifact_window - assert cached is not None - self.assertIsInstance(cached.window, DaemonShareWindowMirror) - self.assertEqual(int(cached.window.window_weight), live_weight) - self.assertIsNone(cached.daemon_unprepared_reason) - # Byte-exact parity with the in-process oracle at the live - # weight, and the daemon holds exactly that digest. - self.assertNotEqual( - checked.share_snapshot_sha256, initial.share_snapshot_sha256 - ) - self.assertEqual( - checked.share_snapshot_sha256, - self._oracle_digest( - ledger, - anchor_ms=int(checked.snapshot_anchor_ms or 0), - window_weight=live_weight, - ), - ) - self.assertEqual(daemon.full_prepares, 2) - self.assertIn(checked.share_snapshot_sha256, daemon.held) - - # The next build advances incrementally: no second full ledger - # snapshot, no needs_full, no window_daemon_state_lost. - server.payout_artifact_full_rescan_seconds = 3_600.0 - _append_heavy_share( - ledger, - share_seq=21, - accepted_at_ms=1_000_030, - share_difficulty=difficulty, - ) - clock_ms[0] = 1_000_040 - advanced = server._build_payout_ledger_artifact( - 0, 0, collapsed_difficulty - ) - assert advanced is not None - self.assertEqual(advanced.window_build_mode, "incremental") - self.assertIsNone(advanced.window_full_rescan_reason) - self.assertEqual(advanced.window_delta_rows, 1) - self.assertEqual(ledger.full_snapshot_calls, 2) - self.assertEqual(daemon.needs_full_answers, 0) - self.assertEqual(daemon.full_prepares, 2) - self.assertEqual( - advanced.share_snapshot_sha256, - self._oracle_digest( - ledger, - anchor_ms=int(advanced.snapshot_anchor_ms or 0), - window_weight=live_weight, - ), - ) - - def test_in_band_drift_keeps_the_cached_weight_and_digest_warm(self) -> None: - server, ledger, artifacts, daemon = self._server() - difficulty = int(artifacts.network_difficulty) - cached_weight = ( - PRISM_REWARD_WINDOW_MULTIPLIER - * PRISM_SNAPSHOT_WINDOW_MARGIN - * difficulty - ) - clock_ms = [1_000_000] - with patch( - "lab.prism.prism_coordinator.now_ms", - side_effect=lambda: clock_ms[0], - ): - initial = server._build_payout_ledger_artifact(0, 0, difficulty) - assert initial is not None - self.assertEqual(daemon.full_prepares, 1) - - # ASERT-sized drift: inside the band and far from the 1.25x - # re-center threshold. The self-check compares and adopts at - # the cached weight, whose digest the daemon already holds. - server.payout_artifact_min_build_interval_seconds = 0.0 - server.payout_artifact_full_rescan_seconds = 0.0 - clock_ms[0] = 1_000_020 - drifted_difficulty = (difficulty * 101 + 99) // 100 - checked = server._build_payout_ledger_artifact( - 0, 0, drifted_difficulty - ) - assert checked is not None - self.assertEqual(checked.window_build_mode, "self_check_match") - self.assertEqual( - checked.window_full_rescan_reason, "periodic_self_check" - ) - self.assertFalse(checked.window_self_check_recentered) - self.assertEqual( - checked.share_snapshot_sha256, initial.share_snapshot_sha256 - ) - cached = server._incremental_payout_artifact_window - assert cached is not None - self.assertIsInstance(cached.window, DaemonShareWindowMirror) - self.assertEqual(int(cached.window.window_weight), cached_weight) - self.assertIsNone(cached.daemon_unprepared_reason) - # Warm: no second full preparation was needed. - self.assertEqual(daemon.full_prepares, 1) - self.assertIn(checked.share_snapshot_sha256, daemon.held) - - server.payout_artifact_full_rescan_seconds = 3_600.0 - _append_heavy_share( - ledger, - share_seq=21, - accepted_at_ms=1_000_030, - share_difficulty=difficulty, - ) - clock_ms[0] = 1_000_040 - advanced = server._build_payout_ledger_artifact( - 0, 0, drifted_difficulty - ) - assert advanced is not None - self.assertEqual(advanced.window_build_mode, "incremental") - self.assertEqual(advanced.window_delta_rows, 1) - self.assertEqual(ledger.full_snapshot_calls, 2) - self.assertEqual(daemon.needs_full_answers, 0) - self.assertEqual(daemon.full_prepares, 1) - - def test_genuine_daemon_eviction_still_reports_window_daemon_state_lost( - self, - ) -> None: - server, ledger, artifacts, daemon = self._server() - difficulty = int(artifacts.network_difficulty) - clock_ms = [1_000_000] - with patch( - "lab.prism.prism_coordinator.now_ms", - side_effect=lambda: clock_ms[0], - ): - initial = server._build_payout_ledger_artifact(0, 0, difficulty) - assert initial is not None - # A respawn or LRU eviction: the daemon forgets its prepared - # state while the coordinator mirror stays digest-verified. - daemon.evict() - _append_heavy_share( - ledger, - share_seq=21, - accepted_at_ms=1_000_010, - share_difficulty=difficulty, - ) - server.payout_artifact_min_build_interval_seconds = 0.0 - clock_ms[0] = 1_000_020 - rebuilt = server._build_payout_ledger_artifact(0, 0, difficulty) - assert rebuilt is not None - self.assertEqual(rebuilt.window_build_mode, "full_rescan") - self.assertEqual( - rebuilt.window_full_rescan_reason, - "window_daemon_state_lost", - ) - # The existing recovery path: one oracle read re-seeds the - # daemon through the full preparation and the cache stays a - # daemon mirror the next build can advance. - self.assertEqual(daemon.needs_full_answers, 1) - self.assertEqual(daemon.full_prepares, 2) - self.assertEqual(ledger.full_snapshot_calls, 2) - cached = server._incremental_payout_artifact_window - assert cached is not None - self.assertIsInstance(cached.window, DaemonShareWindowMirror) - self.assertIn(rebuilt.share_snapshot_sha256, daemon.held) - self.assertEqual( - rebuilt.share_snapshot_sha256, - self._oracle_digest( - ledger, - anchor_ms=int(rebuilt.snapshot_anchor_ms or 0), - window_weight=PRISM_REWARD_WINDOW_MULTIPLIER - * PRISM_SNAPSHOT_WINDOW_MARGIN - * difficulty, - ), - ) - # A genuine eviction is never flagged as a re-center, so - # telemetry can tell the two apart. - self.assertFalse(rebuilt.window_self_check_recentered) - - def test_unprepared_recenter_is_distinguishable_from_daemon_state_loss( - self, - ) -> None: - # When the daemon cannot be prepared at re-center time (busy under - # another build), the adopted mirror is tagged with the daemon's own - # outcome and the next build's rebuild names that outcome rather - # than blaming a daemon eviction; recovery is the existing - # full-oracle path. The reason stays inside the closed vocabulary - # the #224 telemetry owner exports. - server, ledger, artifacts, daemon = self._server() - difficulty = int(artifacts.network_difficulty) - clock_ms = [1_000_000] - with patch( - "lab.prism.prism_coordinator.now_ms", - side_effect=lambda: clock_ms[0], - ): - initial = server._build_payout_ledger_artifact(0, 0, difficulty) - assert initial is not None - server.payout_artifact_min_build_interval_seconds = 0.0 - server.payout_artifact_full_rescan_seconds = 0.0 - clock_ms[0] = 1_000_020 - collapsed_difficulty = difficulty * 3 // 4 - daemon.inject_full_statuses = ["busy"] - checked = server._build_payout_ledger_artifact( - 0, 0, collapsed_difficulty - ) - assert checked is not None - self.assertEqual(checked.window_build_mode, "self_check_match") - self.assertEqual( - checked.window_full_rescan_reason, "periodic_self_check" - ) - self.assertTrue(checked.window_self_check_recentered) - cached = server._incremental_payout_artifact_window - assert cached is not None - self.assertEqual(cached.daemon_unprepared_reason, "window_daemon_busy") - self.assertEqual(daemon.full_prepare_statuses, ["prepared", "busy"]) - - server.payout_artifact_full_rescan_seconds = 3_600.0 - _append_heavy_share( - ledger, - share_seq=21, - accepted_at_ms=1_000_030, - share_difficulty=difficulty, - ) - clock_ms[0] = 1_000_040 - rebuilt = server._build_payout_ledger_artifact( - 0, 0, collapsed_difficulty - ) - assert rebuilt is not None - self.assertEqual(rebuilt.window_build_mode, "full_rescan") - self.assertEqual(rebuilt.window_full_rescan_reason, "window_daemon_busy") - self.assertNotEqual( - rebuilt.window_full_rescan_reason, "window_daemon_state_lost" - ) - self.assertFalse(rebuilt.window_self_check_recentered) - self.assertEqual(daemon.needs_full_answers, 1) - self.assertEqual(daemon.full_prepares, 3) - recovered = server._incremental_payout_artifact_window - assert recovered is not None - self.assertIsNone(recovered.daemon_unprepared_reason) - self.assertIn(rebuilt.share_snapshot_sha256, daemon.held) - - -if __name__ == "__main__": - unittest.main() diff --git a/tests/test_prism_pending_block_recovery.py b/tests/test_prism_pending_block_recovery.py deleted file mode 100644 index 93c0e425f..000000000 --- a/tests/test_prism_pending_block_recovery.py +++ /dev/null @@ -1,520 +0,0 @@ -"""Focused recovery safety tests; native cases run in the PostgreSQL gate.""" - -import hashlib -import json -import os -import tempfile -import threading -import unittest -import uuid -from contextlib import nullcontext -from pathlib import Path -from types import SimpleNamespace -from unittest.mock import Mock, patch - -from lab.auxpow.stratum_codec import header_hash_hex -from lab.prism import direct_stratum -from lab.prism import recover_pending_blocks as recovery -from lab.prism.block_candidates import block_candidate_from_intent -from lab.prism.coordinator_config import load_coordinator_config -from lab.prism.recovery_json import cooperative_json -from lab.prism.share_ledger import PsqlShareLedger, block_candidate_identity_sha256 -from tests.prism_coordinator_test_support import durable_candidate_row - - -def fixture(index=0, parent="11" * 32, height=10): - row = durable_candidate_row(index) - intent = row["candidate"] - header = bytes([index + 1]) * 80 - block_hash = header_hash_hex(header) - intent.update( - block_hex=header.hex(), - block_hash_hex=block_hash, - parent_hash=parent, - expected_height=height, - ) - intent["template"].update(previousblockhash=parent, height=height) - row.update(block_hash=block_hash, candidate_sha256=block_candidate_identity_sha256(intent)) - block = recovery.RecoveryBlock(block_hash, height, parent, "pending") - return block, row - - -def metadata(block, state="pending"): - return dict( - block_hash=block.block_hash, - state=state, - block_height=block.height if state == "submitted" else None, - parent_hash=block.parent_hash if state == "submitted" else None, - chain_state="confirmed" if state == "submitted" else None, - has_audit=state == "submitted", - ) - - -def fake_rpc(blocks): - def call(method, params): - if method == "getblockheader": - block = next(block for block in blocks if block.block_hash == params[0]) - return dict(height=block.height, previousblockhash=block.parent_hash, confirmations=1) - if method == "getblockhash": - return next(block.block_hash for block in blocks if block.height == params[0]) - raise AssertionError(f"unexpected RPC {method}") - - return SimpleNamespace(call=Mock(side_effect=call)) - - -def test_config(root, database_url="postgresql://qbit:qbit@localhost/qbit"): - return load_coordinator_config( - { - "QBIT_RPC_HOST": "localhost", - "QBIT_RPC_USER": "test", - "QBIT_RPC_PASSWORD": "test", - "PRISM_DATABASE_URL": database_url, - "PRISM_ALLOW_TEST_SIGNING_SEEDS": "1", - "PRISM_ALLOW_BUNDLE_EMBEDDED_LEDGER_KEY": "1", - "PRISM_AUDIT_DIR": str(root), - "PRISM_EVIDENCE_PATH": str(root / "evidence.json"), - } - ) - - -class RecoveryTests(unittest.TestCase): - def test_plan_orders_parent_before_child_without_loading_payloads(self): - parent, _ = fixture() - child, _ = fixture(1, parent.block_hash, 11) - rows = {block.block_hash: metadata(block) for block in [parent, child]} - reader = Mock(metadata=lambda value: rows.get(value)) - planned = recovery.plan_recovery( - reader, fake_rpc([parent, child]), [child.block_hash, parent.block_hash] - ) - self.assertEqual(planned, [parent, child]) - reader.candidate.assert_not_called() - - def test_missing_pending_parent_rejects_plan(self): - parent, _ = fixture() - child, _ = fixture(1, parent.block_hash, 11) - rows = {block.block_hash: metadata(block) for block in [parent, child]} - with self.assertRaisesRegex(recovery.RecoveryError, "pending parent"): - recovery.plan_recovery( - Mock(metadata=lambda value: rows.get(value)), - fake_rpc([parent, child]), - [child.block_hash], - ) - - def test_duplicate_missing_abandoned_and_inactive_fail_closed(self): - block, _ = fixture() - for hashes, row in [ - ([], None), - ([block.block_hash] * 2, metadata(block)), - ([block.block_hash], None), - ([block.block_hash], metadata(block, "abandoned")), - ]: - with self.subTest(hashes=hashes, row=row), self.assertRaises(recovery.RecoveryError): - recovery.plan_recovery(Mock(metadata=lambda _: row), fake_rpc([block]), hashes) - rpc = fake_rpc([block]) - rpc.call = Mock( - side_effect=[dict(height=10, previousblockhash=block.parent_hash), "22" * 32] - ) - with self.assertRaisesRegex(recovery.RecoveryError, "active chain"): - recovery.require_active_block(rpc, block.block_hash) - - def test_submitted_without_complete_accounting_is_not_skipped(self): - block, _ = fixture() - row = metadata(block, "submitted") - row["has_audit"] = False - with self.assertRaisesRegex(recovery.RecoveryError, "completion"): - recovery.plan_recovery( - Mock(metadata=lambda _: row), fake_rpc([block]), [block.block_hash] - ) - - def test_payload_identity_height_and_header_are_verified(self): - block, row = fixture() - coordinator = SimpleNamespace(block_candidate_from_intent=block_candidate_from_intent) - candidate = recovery.decode_candidate(coordinator, block, row) - self.assertTrue(candidate.durable_replay) - for field, value in [ - ("shares_json", [{"changed": True}]), - ("expected_height", 12), - ("block_hex", "00" * 80), - ]: - modified = json.loads(json.dumps(row)) - modified["candidate"][field] = value - if field != "shares_json": - modified["candidate_sha256"] = block_candidate_identity_sha256( - modified["candidate"] - ) - with self.subTest(field=field), self.assertRaises(recovery.RecoveryError): - recovery.decode_candidate(coordinator, block, modified) - - def test_chunked_candidate_row_is_hydrated_not_misclassified(self): - """Issue #255: a version-2 row has no jsonb; it decodes via the ledger.""" - from lab.prism.candidate_codec import prepare_candidate_intent - - block, row = fixture() - prepared = prepare_candidate_intent(row["candidate"]) - v2_row = { - "candidate": None, - "candidate_sha256": row["candidate_sha256"], - "storage_version": 2, - "body_id": "ab" * 16, - "replay_header": prepared.replay_header(), - "byte_count": prepared.manifest.byte_count, - "chunk_count": prepared.manifest.chunk_count, - "chunk_bytes": prepared.manifest.chunk_bytes, - "share_count": prepared.manifest.share_count, - "body_state": "sealed", - } - hydrations = [] - - def hydrate(header_row, *, cancelled): - hydrations.append(header_row) - self.assertEqual(header_row["body"]["body_id"], "ab" * 16) - self.assertFalse(cancelled()) - return prepared - - coordinator = SimpleNamespace( - block_candidate_from_intent=block_candidate_from_intent, - ledger=SimpleNamespace(hydrate_block_candidate_intent=hydrate), - stop_event=threading.Event(), - ) - candidate = recovery.decode_candidate(coordinator, block, v2_row) - self.assertTrue(candidate.durable_replay) - self.assertEqual(candidate.submission.block_hash_hex, block.block_hash) - self.assertEqual(len(hydrations), 1) - # A digest that disagrees with the hydrated body is still refused. - with self.assertRaisesRegex(recovery.RecoveryError, "digest mismatch"): - recovery.decode_candidate(coordinator, block, {**v2_row, "candidate_sha256": "0" * 64}) - - def test_unsupported_storage_version_and_unsealed_body_fail_closed(self): - block, row = fixture() - coordinator = SimpleNamespace( - block_candidate_from_intent=block_candidate_from_intent, - ledger=SimpleNamespace(hydrate_block_candidate_intent=Mock()), - stop_event=threading.Event(), - ) - base = {"candidate": None, "candidate_sha256": row["candidate_sha256"]} - for label, v2_row in { - "future version": {**base, "storage_version": 3, "body_id": "ab" * 16, "body_state": "sealed"}, - "unsealed body": {**base, "storage_version": 2, "body_id": "ab" * 16, "body_state": "staging"}, - "missing body": {**base, "storage_version": 2, "body_id": None, "body_state": None}, - }.items(): - with self.subTest(label=label), self.assertRaises(recovery.RecoveryError): - recovery.decode_candidate(coordinator, block, v2_row) - coordinator.ledger.hydrate_block_candidate_intent.assert_not_called() - - def test_apply_finalizes_only_after_normal_accounting_and_checks_durable_result(self): - block, payload = fixture() - events = [] - row = metadata(block) - reader = Mock(metadata=lambda _: row, candidate=lambda _: payload) - - def submit(_candidate, *, node_submission): - self.assertFalse(node_submission.attempted) - events.append("accounting") - return True - - def finalize(*_args, **kwargs): - self.assertTrue(kwargs["accepted"]) - events.append("outbox") - row.update(metadata(block, "submitted")) - - coordinator = Mock( - rpc=fake_rpc([block]), - stop_event=threading.Event(), - block_candidate_from_intent=block_candidate_from_intent, - _writer_operation=lambda _: nullcontext(), - _block_landing_ledger_statement_timeout_scope=lambda _: nullcontext(), - submit_block_candidate=submit, - _finalize_block_candidate=finalize, - ) - recovery.recover_block(coordinator, reader, block) - self.assertEqual(events, ["accounting", "outbox"]) - recovery.recover_block(coordinator, reader, block) - self.assertEqual(events, ["accounting", "outbox"]) - coordinator.serve.assert_not_called() - - def test_accounting_failure_and_lost_lease_leave_outbox_pending(self): - block, payload = fixture() - for failure in ("accounting", "lease", "terminal_update"): - reader = Mock(metadata=lambda _: metadata(block), candidate=lambda _: payload) - coordinator = Mock( - rpc=fake_rpc([block]), - stop_event=threading.Event(), - block_candidate_from_intent=block_candidate_from_intent, - _writer_operation=lambda _: nullcontext(), - _block_landing_ledger_statement_timeout_scope=lambda _: nullcontext(), - ) - coordinator.submit_block_candidate.return_value = failure != "accounting" - if failure == "lease": - coordinator._require_fresh_ledger_lease_for_external_side_effect.side_effect = ( - recovery.RecoveryError("lease lost") - ) - with self.subTest(failure=failure), self.assertRaises(recovery.RecoveryError): - recovery.recover_block(coordinator, reader, block) - if failure != "terminal_update": - coordinator._finalize_block_candidate.assert_not_called() - - def test_recovery_config_preserves_lease_policy_and_signing_identity(self): - with tempfile.TemporaryDirectory() as directory: - before = test_config(Path(directory)) - after = recovery.recovery_config(before) - self.assertEqual(after.lifecycle, before.lifecycle) - self.assertEqual(after.ledger.writer_id, before.ledger.writer_id) - self.assertEqual(after.ledger.writer_epoch, before.ledger.writer_epoch) - self.assertEqual(after.ledger.signing_seed_hex, before.ledger.signing_seed_hex) - self.assertFalse(after.ledger.initialize_schema) - self.assertEqual(after.ledger.native_client_mode, "on") - self.assertIsNone(after.ledger.writer_session_token) - - def test_runner_never_offers_new_blocks(self): - with self.assertRaisesRegex(recovery.RecoveryError, "refuses to submit"): - recovery.RecoveryCoordinator.__new__( - recovery.RecoveryCoordinator - )._submit_block_candidate_to_node(None) - - -class CooperativeJsonTests(unittest.TestCase): - def test_canonical_bytes_identity_and_parser_behavior_match_stdlib(self): - values = [None, True, -(2**90), 1.25, '\u03bb\U0001f680\n\\"', {"z": [1, None], "a": "x"}] - expected = [json.dumps(value, sort_keys=True, separators=(",", ":")) for value in values] - _, payload = fixture() - digest = block_candidate_identity_sha256(payload["candidate"]) - original = json._default_decoder - with cooperative_json(): - for value, text in zip(values, expected): - self.assertEqual(json.dumps(value, sort_keys=True, separators=(",", ":")), text) - self.assertEqual(json.loads(text), value) - self.assertEqual(json.loads(text.encode()), value) - self.assertEqual(block_candidate_identity_sha256(payload["candidate"]), digest) - for malformed in ('{"x":', "[1,]", '"\\q"', "{} garbage"): - with self.assertRaises(json.JSONDecodeError): - json.loads(malformed) - self.assertEqual(json.loads('{"x":1,"x":2}'), {"x": 2}) - self.assertIs(json._default_decoder, original) - - def test_codec_restores_after_error(self): - original = json.scanner.make_scanner - with self.assertRaises(RuntimeError), cooperative_json(): - raise RuntimeError("test") - self.assertIs(json.scanner.make_scanner, original) - - -@unittest.skipUnless( - os.environ.get("PRISM_RECOVERY_TEST_DATABASE_URL"), "requires the native PostgreSQL test gate" -) -class NativeRecoveryTests(unittest.TestCase): - @classmethod - def setUpClass(cls): - import psycopg - from psycopg.conninfo import make_conninfo - - cls.schema = "recovery_" + uuid.uuid4().hex - cls.admin = psycopg.connect(os.environ["PRISM_RECOVERY_TEST_DATABASE_URL"], autocommit=True) - cls.admin.execute(f'CREATE SCHEMA "{cls.schema}"') - # Private schema only: with public on the path, 001's DROP FUNCTION - # statements would remove the shared database's public functions - # before creating the private replacements. Builtins stay implicitly - # available. - cls.url = make_conninfo( - os.environ["PRISM_RECOVERY_TEST_DATABASE_URL"], - options=f"-csearch_path={cls.schema}", - ) - with psycopg.connect(cls.url, autocommit=True) as connection: - connection.execute(Path("crates/qbit-prism/sql/001_share_ledger.sql").read_text()) - # Issue #255: the chunked-body migration the ledger applies - # after 001 on every initialized start. - connection.execute(Path("crates/qbit-prism/sql/002_candidate_bodies.sql").read_text()) - - @classmethod - def tearDownClass(cls): - cls.admin.execute(f'DROP SCHEMA "{cls.schema}" CASCADE') - cls.admin.close() - - def ledger(self, **kwargs): - ledger = PsqlShareLedger( - psql_command="psql", database_url=self.url, native_client_mode="on", **kwargs - ) - self.addCleanup(ledger.close) - return ledger - - def test_exclusive_guard_refuses_live_writer_and_reacquires_after_release(self): - owner = self.ledger(writer_session_token="heartbeat-v1:" + uuid.uuid4().hex) - token = owner._writer_session_token - with self.assertRaisesRegex(recovery.RecoveryError, "exclusive writer lease"): - self.ledger( - writer_session_token="heartbeat-v1:" + uuid.uuid4().hex, - lease_retry_sleep=recovery.refuse_lease_wait, - ) - self.assertEqual(owner._writer_session_token, token) - self.assertTrue(owner.release_writer_lease()) - owner.close() - successor = self.ledger( - writer_session_token="heartbeat-v1:" + uuid.uuid4().hex, - lease_retry_sleep=recovery.refuse_lease_wait, - ) - self.assertTrue(successor.release_writer_lease()) - - def test_reader_is_read_only_and_loads_only_selected_candidate_with_cooperative_codec(self): - import psycopg - from psycopg.types.json import Jsonb - - block, payload = fixture() - other, other_payload = fixture(1) - with psycopg.connect(self.url, autocommit=True) as connection: - for target, row in [(block, payload), (other, other_payload)]: - connection.execute( - "INSERT INTO qbit_block_candidate_outbox (block_hash, candidate, candidate_sha256) VALUES (%s,%s,%s)", - (target.block_hash, Jsonb(row["candidate"]), row["candidate_sha256"]), - ) - with cooperative_json(): - reader = recovery.RecoveryReader(self.url) - try: - self.assertEqual(reader.metadata(block.block_hash)["state"], "pending") - self.assertNotIn("candidate", reader.metadata(block.block_hash)) - self.assertEqual( - reader.candidate(block.block_hash)["candidate"], payload["candidate"] - ) - with self.assertRaises(psycopg.errors.ReadOnlySqlTransaction): - reader.connection.execute( - "UPDATE qbit_block_candidate_outbox SET attempt_count=1" - ) - finally: - reader.close() - - def test_legacy_only_schema_is_refused_by_the_reader_and_the_ledger(self): - # A 001-only database: the staged-upgrade case with - # PRISM_POSTGRES_INIT_SCHEMA=0, which storage version 1 cannot rescue. - import psycopg - from psycopg.conninfo import make_conninfo - - from lab.prism.candidate_store import IncompatibleCandidateSchema - - schema = "recovery_legacy_" + uuid.uuid4().hex - self.admin.execute(f'CREATE SCHEMA "{schema}"') - self.addCleanup(self.admin.execute, f'DROP SCHEMA "{schema}" CASCADE') - url = make_conninfo( - os.environ["PRISM_RECOVERY_TEST_DATABASE_URL"], options=f"-csearch_path={schema}" - ) - with psycopg.connect(url, autocommit=True) as connection: - connection.execute(Path("crates/qbit-prism/sql/001_share_ledger.sql").read_text()) - reader = recovery.RecoveryReader(url) - self.addCleanup(reader.close) - with self.assertRaisesRegex(recovery.RecoveryError, "002_candidate_bodies.sql"): - reader.require_candidate_schema() - ledger = PsqlShareLedger( - psql_command="psql", - database_url=url, - native_client_mode="on", - candidate_storage_version=1, - ) - self.addCleanup(ledger.close) - with self.assertRaisesRegex(IncompatibleCandidateSchema, "002_candidate_bodies.sql"): - ledger.verify_candidate_schema() - - def test_real_finalizer_persists_confirms_publishes_and_completes_outbox(self): - # Real coordinator, PostgreSQL, lease heartbeat, finalizer and audit - # publication. Only the node and cryptographic builder/verifier use - # deterministic fixtures; their own suites cover consensus proofs. - import psycopg - from psycopg.types.json import Jsonb - - from tests.prism_vardiff_test_support import ( - verified_audit_report, - verified_block_bundle, - ) - - parent, parent_payload = fixture(3, height=20) - child, child_payload = fixture(4, parent.block_hash, 21) - with psycopg.connect(self.url, autocommit=True) as connection: - for block, payload in [(parent, parent_payload), (child, child_payload)]: - payload["candidate"]["prospective_prior_balances"] = [] - payload["candidate_sha256"] = block_candidate_identity_sha256(payload["candidate"]) - connection.execute( - "INSERT INTO qbit_block_candidate_outbox (block_hash,candidate,candidate_sha256) VALUES (%s,%s,%s)", - (block.block_hash, Jsonb(payload["candidate"]), payload["candidate_sha256"]), - ) - rpc = fake_rpc([parent, child]) - chain_call = rpc.call.side_effect - - def call(method, params=None, **_kwargs): - if method == "getbestblockhash": - return child.block_hash - if method == "getblockcount": - return child.height - if method == "getblockchaininfo": - return dict( - chain="regtest", - blocks=child.height, - headers=child.height, - initialblockdownload=False, - ) - if method == "getblockhash" and params == [parent.height - 1]: - return parent.parent_hash - return chain_call(method, params) - - rpc.call.side_effect = call - bundle = verified_block_bundle("00") - - def canonical(value): - return json.dumps(value, sort_keys=True, separators=(",", ":")).encode() - - audit_report = verified_audit_report("00") - audit_report.update(audit_bundle_sha256_hex=hashlib.sha256(canonical(bundle)).hexdigest()) - - def verify(*_args, **kwargs): - return {**audit_report, "block_height": kwargs["expected_block_height"]} - - selection = direct_stratum.VersionRollingMaskSelection( - direct_stratum.QBIT_VERSION_ROLLING_MASK, "fallback", "test" - ) - with ( - tempfile.TemporaryDirectory() as directory, - cooperative_json(), - patch("lab.prism.prism_coordinator.JsonRpc", return_value=rpc), - patch.object( - recovery.RecoveryCoordinator, "resolve_version_rolling_mask", return_value=selection - ), - patch.object(recovery.RecoveryCoordinator, "validate_live_chain_identity"), - patch.object(recovery.RecoveryCoordinator, "validate_live_template_and_fee_policy"), - patch.object(recovery.RecoveryCoordinator, "build_audit_bundle", return_value=bundle), - patch("lab.prism.audit_artifacts.AuditArtifactStore.verify_bundle", side_effect=verify), - patch("lab.prism.prism_coordinator.canonical_bundle_bytes", canonical), - patch("lab.prism.share_ledger._default_bundle_canonicalizer", return_value=canonical), - ): - reader = recovery.RecoveryReader(self.url) - self.addCleanup(reader.close) - config = test_config(Path(directory), self.url) - events = [] - blocks = recovery.plan_recovery(reader, rpc, [child.block_hash, parent.block_hash]) - with patch( - "lab.prism.audit_artifacts.AuditArtifactStore.publish_success", - side_effect=RuntimeError("injected publication failure"), - ): - with self.assertRaisesRegex(RuntimeError, "injected publication failure"): - recovery.apply_recovery( - config, reader, blocks, timeout_seconds=30, report=events.append - ) - self.assertEqual(reader.metadata(parent.block_hash)["chain_state"], "confirmed") - self.assertEqual(reader.metadata(parent.block_hash)["state"], "pending") - self.assertIsNone(reader.metadata(child.block_hash)["chain_state"]) - events.clear() - recovery.apply_recovery( - config, reader, blocks, timeout_seconds=30, report=events.append - ) - for block in blocks: - recovery.require_completed(reader.metadata(block.block_hash), block) - self.assertEqual( - [event["block_hash"] for event in events if event["event"] == "completed"], - [parent.block_hash, child.block_hash], - ) - # Re-entry is a durable no-op; no payload or accounting is rebuilt. - with patch.object( - reader, "candidate", side_effect=AssertionError("terminal payload read") - ): - recovery.apply_recovery( - config, reader, blocks, timeout_seconds=30, report=events.append - ) - self.assertFalse(any(call.args[0] == "submitblock" for call in rpc.call.call_args_list)) - - -if __name__ == "__main__": - unittest.main() diff --git a/tests/test_prism_process_telemetry.py b/tests/test_prism_process_telemetry.py deleted file mode 100644 index 041e78b0b..000000000 --- a/tests/test_prism_process_telemetry.py +++ /dev/null @@ -1,309 +0,0 @@ -"""Issue #226 part 1: the process heap collector and its configuration switch. - -The renderer-level contracts (family set, unavailable rendering, owner -reads, closed labels, no heap walk) live in tests/test_prism_metrics.py. -This module pins the collector itself: the glibc struct layout, the -resolve-once behaviour, the closed generation set, and the strict -PRISM_MALLOC_TELEMETRY switch through the coordinator config loader. -""" - -from __future__ import annotations - -import ctypes -import gc -import os -from pathlib import Path -import sys -import tempfile -import unittest -from types import SimpleNamespace -from unittest import mock - -from lab.prism.coordinator_config import ( - DEFAULT_PRISM_MALLOC_TELEMETRY, - env_malloc_telemetry, - load_coordinator_config, -) -from lab.prism.metrics import MetricsRenderer -from lab.prism.process_telemetry import ( - PRISM_GC_GENERATIONS, - MallInfo2, - ProcessHeapTelemetry, - resolve_mallinfo2, -) - - -def _host_has_mallinfo2() -> bool: - try: - resolve_mallinfo2() - except Exception: # noqa: BLE001 - any failure means "not here" - return False - return True - - -class MallInfo2LayoutTests(unittest.TestCase): - def test_struct_matches_glibc_field_order(self) -> None: - """Ten size_t fields in glibc's declared order; a wrong layout would - read the wrong field as the arena on every scrape.""" - names = [name for name, _ctype in MallInfo2._fields_] - self.assertEqual( - names, - [ - "arena", - "ordblks", - "smblks", - "hblks", - "hblkhd", - "usmblks", - "fsmblks", - "uordblks", - "fordblks", - "keepcost", - ], - ) - for _name, ctype in MallInfo2._fields_: - self.assertIs(ctype, ctypes.c_size_t) - self.assertEqual(ctypes.sizeof(MallInfo2), 10 * ctypes.sizeof(ctypes.c_size_t)) - - @unittest.skipUnless( - sys.platform.startswith("linux") and _host_has_mallinfo2(), - "glibc mallinfo2 is not available on this host", - ) - def test_real_binding_reads_a_consistent_struct(self) -> None: - """On a glibc 2.33+ host the bound symbol returns arena = in-use + free.""" - info = resolve_mallinfo2()() - self.assertIsInstance(info, MallInfo2) - self.assertGreater(int(info.arena), 0) - self.assertGreaterEqual(int(info.uordblks), 0) - self.assertGreaterEqual(int(info.fordblks), 0) - self.assertEqual(int(info.arena), int(info.uordblks) + int(info.fordblks)) - - -class ProcessHeapTelemetryTests(unittest.TestCase): - def test_sample_carries_every_interpreter_reading(self) -> None: - sample = ProcessHeapTelemetry(malloc_enabled=False).sample() - self.assertGreater(sample.allocated_blocks, 0) - self.assertGreaterEqual(sample.threads, 1) - for reading in ( - sample.gc_trigger_count, - sample.gc_collections, - sample.gc_collected, - sample.gc_uncollectable, - ): - self.assertEqual(len(reading), len(PRISM_GC_GENERATIONS)) - for value in reading: - self.assertGreaterEqual(value, 0) - self.assertFalse(sample.malloc_info_available) - self.assertEqual(sample.malloc_arena_bytes, -1) - self.assertEqual(sample.malloc_in_use_bytes, -1) - self.assertEqual(sample.malloc_free_bytes, -1) - self.assertEqual(sample.malloc_mmapped_bytes, -1) - - def test_generation_set_stays_closed_when_the_interpreter_reports_fewer(self) -> None: - """A missing generation renders -1 rather than shrinking the series set.""" - with ( - mock.patch.object(gc, "get_count", return_value=(4, 2)), - mock.patch.object( - gc, - "get_stats", - return_value=[ - {"collections": 9, "collected": 8, "uncollectable": 1}, - {"collections": 3, "collected": 2, "uncollectable": 0}, - ], - ), - ): - sample = ProcessHeapTelemetry(malloc_enabled=False).sample() - self.assertEqual(sample.gc_trigger_count, (4, 2, -1)) - self.assertEqual(sample.gc_collections, (9, 3, -1)) - self.assertEqual(sample.gc_collected, (8, 2, -1)) - self.assertEqual(sample.gc_uncollectable, (1, 0, -1)) - - def test_resolver_runs_once_and_its_failure_is_remembered(self) -> None: - resolver = mock.Mock(side_effect=AttributeError("no mallinfo2")) - telemetry = ProcessHeapTelemetry(mallinfo_resolver=resolver) - for _ in range(3): - sample = telemetry.sample() - self.assertFalse(sample.malloc_info_available) - self.assertEqual(sample.malloc_arena_bytes, -1) - self.assertEqual(resolver.call_count, 1) - - def test_bound_symbol_that_fails_at_call_time_is_not_retried(self) -> None: - bound = mock.Mock(side_effect=OSError("call failed")) - telemetry = ProcessHeapTelemetry(mallinfo_resolver=lambda: bound) - first = telemetry.sample() - second = telemetry.sample() - self.assertFalse(first.malloc_info_available) - self.assertFalse(second.malloc_info_available) - self.assertEqual(bound.call_count, 1) - - def test_bound_symbol_fields_map_to_the_sample(self) -> None: - def fake_mallinfo2() -> MallInfo2: - info = MallInfo2() - info.arena = 20_000 - info.uordblks = 12_000 - info.fordblks = 8_000 - info.hblkhd = 1_024 - return info - - sample = ProcessHeapTelemetry(mallinfo_resolver=lambda: fake_mallinfo2).sample() - self.assertTrue(sample.malloc_info_available) - self.assertEqual(sample.malloc_arena_bytes, 20_000) - self.assertEqual(sample.malloc_in_use_bytes, 12_000) - self.assertEqual(sample.malloc_free_bytes, 8_000) - self.assertEqual(sample.malloc_mmapped_bytes, 1_024) - - @unittest.skipUnless( - sys.platform.startswith("linux") and _host_has_mallinfo2(), - "glibc mallinfo2 is not available on this host", - ) - def test_default_collector_binds_glibc_on_this_host(self) -> None: - sample = ProcessHeapTelemetry().sample() - self.assertTrue(sample.malloc_info_available) - self.assertGreater(sample.malloc_arena_bytes, 0) - self.assertGreaterEqual(sample.malloc_in_use_bytes, 0) - self.assertGreaterEqual(sample.malloc_free_bytes, 0) - self.assertGreaterEqual(sample.malloc_mmapped_bytes, 0) - - -class MallocTelemetrySwitchTests(unittest.TestCase): - """PRISM_MALLOC_TELEMETRY: default on, strict values, renderer resolution.""" - - def test_switch_defaults_to_on(self) -> None: - self.assertTrue(DEFAULT_PRISM_MALLOC_TELEMETRY) - self.assertTrue(env_malloc_telemetry(environ={})) - - def test_switch_accepts_boolean_tokens(self) -> None: - for raw in ("1", "true", "YES", "On"): - with self.subTest(raw=raw): - self.assertTrue( - env_malloc_telemetry(environ={"PRISM_MALLOC_TELEMETRY": raw}) - ) - for raw in ("0", "false", "NO", "Off"): - with self.subTest(raw=raw): - self.assertFalse( - env_malloc_telemetry(environ={"PRISM_MALLOC_TELEMETRY": raw}) - ) - - def test_switch_fails_closed_on_non_boolean_values(self) -> None: - for raw in ("2", "banana", "tru"): - with self.subTest(raw=raw): - with self.assertRaisesRegex( - SystemExit, "PRISM_MALLOC_TELEMETRY must be a boolean" - ): - env_malloc_telemetry(environ={"PRISM_MALLOC_TELEMETRY": raw}) - - def test_coordinator_config_load_validates_the_switch_at_startup(self) -> None: - with tempfile.TemporaryDirectory() as tmp: - root = Path(tmp) - base = { - "QBIT_RPC_HOST": "qbit.example", - "QBIT_RPC_USER": "rpc-user", - "QBIT_RPC_PASSWORD": "rpc-password", - "PRISM_ALLOW_MEMORY_LEDGER": "1", - "PRISM_ALLOW_TEST_SIGNING_SEEDS": "1", - "PRISM_ALLOW_BUNDLE_EMBEDDED_LEDGER_KEY": "1", - "PRISM_AUDIT_DIR": str(root), - "PRISM_EVIDENCE_PATH": str(root / "evidence.json"), - } - config = load_coordinator_config(base) - self.assertTrue(config.lifecycle.malloc_telemetry_enabled) - disabled = load_coordinator_config( - {**base, "PRISM_MALLOC_TELEMETRY": "0"} - ) - self.assertFalse(disabled.lifecycle.malloc_telemetry_enabled) - with self.assertRaisesRegex( - SystemExit, "PRISM_MALLOC_TELEMETRY must be a boolean" - ): - load_coordinator_config( - {**base, "PRISM_MALLOC_TELEMETRY": "maybe"} - ) - - def test_renderer_resolves_the_switch_once_at_construction(self) -> None: - # A port that pins the switch wins over the environment. - pinned = MetricsRenderer(SimpleNamespace(malloc_telemetry_enabled=False)) - self.assertFalse(pinned._process_telemetry.malloc_enabled) - # Otherwise the strict environment switch is read, once, at - # construction; a later environment change does not reach a built - # renderer. - with mock.patch.dict(os.environ, {"PRISM_MALLOC_TELEMETRY": "0"}): - from_environment = MetricsRenderer(SimpleNamespace()) - self.assertFalse(from_environment._process_telemetry.malloc_enabled) - with mock.patch.dict(os.environ, {"PRISM_MALLOC_TELEMETRY": "1"}): - self.assertFalse(from_environment._process_telemetry.malloc_enabled) - self.assertTrue( - MetricsRenderer(SimpleNamespace())._process_telemetry.malloc_enabled - ) - with mock.patch.dict(os.environ, {"PRISM_MALLOC_TELEMETRY": ""}, clear=False): - self.assertEqual( - MetricsRenderer(SimpleNamespace())._process_telemetry.malloc_enabled, - DEFAULT_PRISM_MALLOC_TELEMETRY, - ) - - def test_validated_config_snapshot_beats_the_ambient_environment(self) -> None: - """A coordinator built with an explicit config is not overridden by env. - - The lifecycle snapshot is the value the config loader validated at - startup. A caller that constructed the coordinator with telemetry - disabled must not have mallinfo2 run anyway because the ambient - environment happens to say otherwise, and a valid environment change - after startup must not silently override the snapshot either. - """ - disabled = SimpleNamespace( - config=SimpleNamespace( - lifecycle=SimpleNamespace(malloc_telemetry_enabled=False) - ) - ) - with mock.patch.dict(os.environ, {"PRISM_MALLOC_TELEMETRY": "1"}): - self.assertFalse( - MetricsRenderer(disabled)._process_telemetry.malloc_enabled - ) - enabled = SimpleNamespace( - config=SimpleNamespace( - lifecycle=SimpleNamespace(malloc_telemetry_enabled=True) - ) - ) - with mock.patch.dict(os.environ, {"PRISM_MALLOC_TELEMETRY": "0"}): - self.assertTrue( - MetricsRenderer(enabled)._process_telemetry.malloc_enabled - ) - # An explicit port pin still outranks the snapshot. - pinned = SimpleNamespace( - malloc_telemetry_enabled=False, - config=SimpleNamespace( - lifecycle=SimpleNamespace(malloc_telemetry_enabled=True) - ), - ) - self.assertFalse(MetricsRenderer(pinned)._process_telemetry.malloc_enabled) - # With no snapshot at all the environment is still the fallback. - with mock.patch.dict(os.environ, {"PRISM_MALLOC_TELEMETRY": "0"}): - self.assertFalse( - MetricsRenderer( - SimpleNamespace(config=SimpleNamespace()) - )._process_telemetry.malloc_enabled - ) - - def test_gc_trigger_count_is_not_described_as_retained_objects(self) -> None: - """gc.get_count() is a trigger counter, and the family must say so. - - Naming it after retained objects sent the runbook's leak guidance at - internal collector state; an operator reading it as a backlog during an - incident would draw the wrong conclusion. - """ - lines = MetricsRenderer( - SimpleNamespace(), process_telemetry=ProcessHeapTelemetry() - ).process_heap_metrics_lines() - help_line = next( - line - for line in lines - if line.startswith("# HELP qbit_prism_process_gc_trigger_count") - ) - self.assertIn("trigger counters", help_line) - self.assertIn("NOT a count of retained objects", help_line) - self.assertFalse( - [line for line in lines if "gc_pending_objects" in line], - "the misleading family name must not come back", - ) - - -if __name__ == "__main__": - unittest.main() diff --git a/tests/test_prism_progress_health.py b/tests/test_prism_progress_health.py deleted file mode 100644 index 0038ce134..000000000 --- a/tests/test_prism_progress_health.py +++ /dev/null @@ -1,954 +0,0 @@ -#!/usr/bin/env python3 -"""Deterministic monotonic progress-health coverage for PRISM.""" - -from __future__ import annotations - -import threading -import unittest -from types import SimpleNamespace -from unittest.mock import patch - -from concurrent.futures import CancelledError - -from lab.prism.prism_coordinator import QbitTipTemplateSnapshot -from lab.prism.progress_health import ( - ProgressHealthConfig, - ProgressHealthService, - WorkGeneration, -) -from tests.prism_coordinator_test_support import client, coordinator - - -class FakeMonotonicClock: - def __init__(self, now: float = 100.0) -> None: - self.now = now - - def __call__(self) -> float: - return self.now - - def advance(self, seconds: float) -> None: - self.now += seconds - - -def snapshot( - *, generation: int, fingerprint: str, tip: str = "11" * 32 -) -> QbitTipTemplateSnapshot: - return QbitTipTemplateSnapshot( - bestblockhash=tip, - previousblockhash=tip, - template_fingerprint=fingerprint, - template_generation=generation, - ) - - -def context_for( - work: QbitTipTemplateSnapshot, - payout_generation: int, - *, - collection_only: bool = False, -) -> SimpleNamespace: - return SimpleNamespace( - template={"previousblockhash": work.previousblockhash}, - template_fingerprint=work.template_fingerprint, - template_generation=work.template_generation, - payout_state_generation=payout_generation, - collection_only=collection_only, - ) - - -def progress_coordinator() -> tuple[object, FakeMonotonicClock]: - server, _ = coordinator() - clock = FakeMonotonicClock() - server._progress_monotonic = clock - server.started_monotonic = clock.now - server.health_pending_refresh_max_age_seconds = 15.0 - server.health_tip_poll_max_age_seconds = 15.0 - with server._progress_health_lock: - server._progress_current_template_generation = 0 - server._progress_current_template_fingerprint = None - server._progress_current_payout_generation = 0 - server._progress_published_template_generation = 0 - server._progress_published_template_fingerprint = None - server._progress_published_payout_generation = 0 - server._progress_has_published_work = False - server._progress_last_tip_poll_monotonic = None - server._progress_last_delivery_template_generation = 0 - server._progress_last_delivery_template_fingerprint = None - server._progress_last_delivery_payout_generation = 0 - server._progress_last_delivery_monotonic = None - server._progress_pending_since_monotonic = clock.now - server._progress_publication_divergence_since_monotonic = clock.now - server._progress_refresh_signal_pending = False - server._progress_active_refresh_count = 0 - server._progress_last_refresh_activity_monotonic = None - server._progress_bundle_build_counter = 0 - server._progress_bundle_builds.clear() - return server, clock - - -def publish( - server: object, - work: QbitTipTemplateSnapshot, - payout_generation: int = 0, -) -> None: - server._record_progress_tip_poll(work) - server._record_progress_publication(work, payout_generation) - - -class ProgressHealthTests(unittest.TestCase): - def test_health_warmup_is_async_and_reports_starting_state(self) -> None: - import time as _time - - server, _clock = progress_coordinator() - entered = threading.Event() - release = threading.Event() - - def blocking_stats() -> tuple[int, int]: - entered.set() - release.wait(5) - return (0, 0) - - server.accepted_share_stats = blocking_stats # type: ignore[method-assign] - try: - started = _time.monotonic() - server.start_health_snapshot_refresher() - # The bind path never blocks on the cold accepted-share - # aggregate; the warm-up runs in the background loop. - self.assertLess(_time.monotonic() - started, 1.0) - self.assertTrue(entered.wait(2)) - status, payload = server.cached_health_payload() - self.assertEqual(status, 503) - self.assertEqual(payload["state"], "starting") - self.assertFalse(payload["ok"]) - finally: - release.set() - server.stop_event.set() - - def test_publication_progress_uses_template_failure_budget(self) -> None: - server, clock = progress_coordinator() - server.template_refresh_failure_exit_seconds = 10.0 - current = snapshot(generation=1, fingerprint="aa" * 32) - publish(server, current) - replacement = snapshot( - generation=2, - fingerprint="bb" * 32, - tip="22" * 32, - ) - server._record_progress_tip_poll(replacement) - - clock.advance(9.999) - self.assertFalse(server.publication_progress_failure_expired(clock.now)) - clock.advance(0.001) - self.assertTrue(server.publication_progress_failure_expired(clock.now)) - - server._record_progress_publication(replacement, 0) - self.assertFalse(server.publication_progress_failure_expired(clock.now)) - - def test_publication_watchdog_does_not_inherit_client_delivery_age(self) -> None: - server, clock = progress_coordinator() - server.template_refresh_failure_exit_seconds = 10.0 - current = snapshot(generation=1, fingerprint="aa" * 32) - publish(server, current) - miner = client(1) - server.clients.add(miner) - server._progress_reconcile_pending(now=clock.now) - - clock.advance(20.0) - self.assertEqual(server._progress_pending_since_monotonic, 100.0) - self.assertIsNone( - server._progress_publication_divergence_since_monotonic - ) - self.assertFalse(server.publication_progress_failure_expired(clock.now)) - - replacement = snapshot( - generation=2, - fingerprint="bb" * 32, - tip="22" * 32, - ) - with server.lock: - server.latest_detected_tip = (replacement.bestblockhash, 1) - server._progress_note_refresh_pending(clock.now) - - # A delayed delivery of the still-published tip can clear the broader - # client health condition, but it must not clear or age the newer - # publication-divergence deadline. - server._record_progress_delivery( - miner, - context_for(current, 0), - clock.now, - ) - self.assertIsNone(server._progress_pending_since_monotonic) - self.assertEqual( - server._progress_publication_divergence_since_monotonic, - clock.now, - ) - - server._record_progress_tip_poll(replacement, clock.now) - self.assertFalse(server.publication_progress_failure_expired(clock.now)) - clock.advance(9.999) - self.assertFalse(server.publication_progress_failure_expired(clock.now)) - clock.advance(0.001) - self.assertTrue(server.publication_progress_failure_expired(clock.now)) - - def test_publication_divergence_survives_old_tip_delivery(self) -> None: - server, clock = progress_coordinator() - server.template_refresh_failure_exit_seconds = 10.0 - current = snapshot(generation=1, fingerprint="aa" * 32) - publish(server, current) - miner = client(1) - server.clients.add(miner) - replacement = snapshot( - generation=2, - fingerprint="bb" * 32, - tip="22" * 32, - ) - with server.lock: - server.latest_detected_tip = (replacement.bestblockhash, 1) - server._progress_note_refresh_pending(clock.now) - - clock.advance(6.0) - server._record_progress_delivery( - miner, - context_for(current, 0), - clock.now, - ) - - self.assertEqual( - server._progress_publication_divergence_since_monotonic, - 100.0, - ) - clock.advance(3.999) - self.assertFalse(server.publication_progress_failure_expired(clock.now)) - clock.advance(0.001) - self.assertTrue(server.publication_progress_failure_expired(clock.now)) - - def test_publication_divergence_churn_does_not_renew_deadline(self) -> None: - server, clock = progress_coordinator() - server.template_refresh_failure_exit_seconds = 10.0 - publish(server, snapshot(generation=1, fingerprint="aa" * 32)) - latest = None - first_replacement = None - - for generation, marker in ((2, "bb"), (3, "cc"), (4, "dd")): - latest = snapshot( - generation=generation, - fingerprint=marker * 32, - tip=marker * 32, - ) - if first_replacement is None: - first_replacement = latest - with server.lock: - server.latest_detected_tip = (latest.bestblockhash, generation) - server._progress_note_refresh_pending(clock.now) - server._record_progress_tip_poll(latest, clock.now) - self.assertEqual( - server._progress_publication_divergence_since_monotonic, - 100.0, - ) - clock.advance(3.0) - - self.assertIsNotNone(latest) - self.assertFalse(server.publication_progress_failure_expired(clock.now)) - clock.advance(1.0) - self.assertTrue(server.publication_progress_failure_expired(clock.now)) - - assert first_replacement is not None - server._record_progress_publication(first_replacement, 0) - self.assertEqual( - server._progress_publication_divergence_since_monotonic, - 100.0, - ) - - assert latest is not None - server._record_progress_publication(latest, 0) - self.assertIsNone( - server._progress_publication_divergence_since_monotonic - ) - self.assertFalse(server.publication_progress_failure_expired(clock.now)) - - def test_publication_watchdog_fires_with_heartbeat_watchdog_disabled(self) -> None: - server, clock = progress_coordinator() - server.template_refresh_failure_exit_seconds = 10.0 - server.watchdog_enabled = False - server.watchdog_interval_seconds = 0.001 - publish(server, snapshot(generation=1, fingerprint="aa" * 32)) - server._record_progress_tip_poll( - snapshot( - generation=2, - fingerprint="bb" * 32, - tip="22" * 32, - ) - ) - clock.advance(10.0) - - with ( - patch( - "lab.prism.prism_coordinator.time.monotonic", - return_value=clock.now, - ), - patch( - "lab.prism.prism_coordinator.os._exit", - side_effect=SystemExit(1), - ) as exit_process, - patch("builtins.print"), - self.assertRaises(SystemExit), - ): - server.watchdog_loop() - - exit_process.assert_called_once_with(1) - - def test_brief_coordination_block_owns_publication_deadline(self) -> None: - server, clock = progress_coordinator() - server.template_refresh_failure_exit_seconds = 10.0 - server.coordination_blocked_exit_seconds = 30.0 - server.watchdog_enabled = False - server.watchdog_interval_seconds = 0.001 - publish(server, snapshot(generation=1, fingerprint="aa" * 32)) - server._record_progress_tip_poll( - snapshot( - generation=2, - fingerprint="bb" * 32, - tip="22" * 32, - ) - ) - server._record_coordination_blocked_refresh(clock.now) - clock.advance(10.0) - - self.assertTrue(server.publication_progress_failure_expired(clock.now)) - self.assertFalse(server.coordination_blocked_streak_expired(clock.now)) - wait_results = iter((False, True)) - server.stop_event = SimpleNamespace( - wait=lambda _seconds: next(wait_results) - ) - with ( - patch( - "lab.prism.prism_coordinator.time.monotonic", - return_value=clock.now, - ), - patch("lab.prism.prism_coordinator.os._exit") as exit_process, - patch("builtins.print"), - ): - server.watchdog_loop() - - exit_process.assert_not_called() - - def test_coordination_start_during_publication_check_wins_arbitration( - self, - ) -> None: - server, clock = progress_coordinator() - server.template_refresh_failure_exit_seconds = 10.0 - server.coordination_blocked_exit_seconds = 30.0 - server.watchdog_enabled = False - server.watchdog_interval_seconds = 0.001 - publish(server, snapshot(generation=1, fingerprint="aa" * 32)) - server._record_progress_tip_poll( - snapshot( - generation=2, - fingerprint="bb" * 32, - tip="22" * 32, - ) - ) - clock.advance(10.0) - - publication_check_started = threading.Event() - resume_publication_check = threading.Event() - server.stop_event = threading.Event() - original_publication_check = server.publication_progress_failure_expired - thread_errors: list[BaseException] = [] - - def delayed_publication_check(now: float) -> bool: - publication_check_started.set() - if not resume_publication_check.wait(5.0): - raise AssertionError("publication watchdog interleave timed out") - server.stop_event.set() - return original_publication_check(now) - - def run_watchdog() -> None: - try: - server.watchdog_loop() - except BaseException as exc: - thread_errors.append(exc) - - with ( - patch.object( - server, - "publication_progress_failure_expired", - side_effect=delayed_publication_check, - ), - patch( - "lab.prism.prism_coordinator.time.monotonic", - return_value=clock.now, - ), - patch("lab.prism.prism_coordinator.os._exit") as exit_process, - patch("builtins.print"), - ): - watchdog = threading.Thread(target=run_watchdog) - watchdog.start() - self.assertTrue(publication_check_started.wait(5.0)) - server._record_coordination_blocked_refresh(clock.now) - resume_publication_check.set() - watchdog.join(5.0) - - self.assertFalse(watchdog.is_alive()) - self.assertEqual(thread_errors, []) - exit_process.assert_not_called() - - def test_unchanged_tip_for_hours_with_valid_work_stays_healthy(self) -> None: - server, clock = progress_coordinator() - original = snapshot(generation=1, fingerprint="aa" * 32) - publish(server, original) - miner = client(1) - delivered = context_for(original, 0) - miner.active_job = delivered - server.clients.add(miner) - server._record_progress_delivery(miner, delivered, clock.now) - - clock.advance(6 * 60 * 60) - same_work = snapshot(generation=2, fingerprint=original.template_fingerprint) - server._record_progress_tip_poll(same_work) - health = server.progress_health_snapshot() - - self.assertTrue(health["ok"]) - self.assertEqual(health["published_template_generation"], 2) - self.assertGreater(health["last_valid_delivery_age_seconds"], 21_000) - - def test_repeated_successful_same_tip_polls_stay_healthy(self) -> None: - server, clock = progress_coordinator() - fingerprint = "aa" * 32 - publish(server, snapshot(generation=1, fingerprint=fingerprint)) - - for generation in range(2, 20): - clock.advance(10) - server._record_progress_tip_poll( - snapshot(generation=generation, fingerprint=fingerprint) - ) - self.assertTrue(server.progress_health_snapshot()["ok"]) - - def test_new_tip_without_publication_exceeds_deadline_and_returns_503(self) -> None: - server, clock = progress_coordinator() - publish(server, snapshot(generation=1, fingerprint="aa" * 32)) - changed = snapshot(generation=2, fingerprint="bb" * 32, tip="22" * 32) - server._record_progress_tip_poll(changed) - clock.advance(16) - server._record_progress_tip_poll( - snapshot( - generation=3, - fingerprint=changed.template_fingerprint, - tip=changed.bestblockhash, - ) - ) - - status, health = server.cached_health_payload() - - self.assertEqual(status, 503) - self.assertIn("refresh_pending_too_long", health["reasons"]) - self.assertIn("current_generation_not_published", health["reasons"]) - - def test_payout_change_without_replacement_delivery_returns_503(self) -> None: - server, clock = progress_coordinator() - work = snapshot(generation=1, fingerprint="aa" * 32) - publish(server, work) - miner = client(1) - old_delivery = context_for(work, 0) - miner.active_job = old_delivery - server.clients.add(miner) - server._record_progress_delivery(miner, old_delivery, clock.now) - - server._record_progress_payout_generation(1, clock.now) - server._record_progress_publication(work, 1) - clock.advance(16) - same_work = snapshot(generation=2, fingerprint=work.template_fingerprint) - server._record_progress_tip_poll(same_work) - - status, health = server.cached_health_payload() - - self.assertEqual(status, 503) - self.assertIn("current_generation_not_delivered", health["reasons"]) - self.assertEqual(health["current_payout_generation"], 1) - self.assertEqual(health["published_payout_generation"], 1) - - def test_current_generation_delivery_clears_failure_immediately(self) -> None: - server, clock = progress_coordinator() - work = snapshot(generation=1, fingerprint="aa" * 32) - publish(server, work) - miner = client(1) - miner.active_job = context_for(work, 0) - server.clients.add(miner) - server._record_progress_payout_generation(1, clock.now) - server._record_progress_publication(work, 1) - clock.advance(16) - server._record_progress_tip_poll( - snapshot(generation=2, fingerprint=work.template_fingerprint) - ) - self.assertFalse(server.progress_health_snapshot()["ok"]) - - current_delivery = context_for(work, 1) - miner.active_job = current_delivery - server._record_progress_delivery(miner, current_delivery, clock.now) - - status, health = server.cached_health_payload() - self.assertEqual(status, 200) - self.assertTrue(health["ok"]) - self.assertFalse(health["pending_refresh"]) - - def test_blocked_bundle_build_becomes_unhealthy(self) -> None: - server, clock = progress_coordinator() - work = snapshot(generation=1, fingerprint="aa" * 32) - publish(server, work) - server.bundle_build_timeout_seconds = 60.0 - token = server._progress_bundle_build_started() - clock.advance(16) - server._record_progress_tip_poll( - snapshot(generation=2, fingerprint=work.template_fingerprint) - ) - - within_build_timeout = server.progress_health_snapshot() - - self.assertTrue(within_build_timeout["ok"]) - self.assertNotIn("bundle_build_stuck", within_build_timeout["reasons"]) - - clock.advance(45) - server._record_progress_tip_poll( - snapshot(generation=3, fingerprint=work.template_fingerprint) - ) - health = server.progress_health_snapshot() - - self.assertFalse(health["ok"]) - self.assertIn("bundle_build_stuck", health["reasons"]) - self.assertEqual(health["bundle_build_oldest_age_seconds"], 61.0) - server._progress_bundle_build_finished(token) - - def test_no_eligible_miners_need_no_socket_delivery_after_publication(self) -> None: - server, clock = progress_coordinator() - work = snapshot(generation=1, fingerprint="aa" * 32) - server._record_progress_tip_poll(work) - server._record_progress_payout_generation(1, clock.now) - server._record_progress_publication(work, 1) - - health = server.progress_health_snapshot() - - self.assertTrue(health["ok"]) - self.assertEqual(health["eligible_client_count"], 0) - self.assertIsNone(health["last_valid_delivery_age_seconds"]) - - def test_eligible_miners_require_current_generation_delivery(self) -> None: - server, clock = progress_coordinator() - work = snapshot(generation=1, fingerprint="aa" * 32) - publish(server, work) - miner = client(1) - server.clients.add(miner) - server.progress_health_snapshot() - clock.advance(16) - server._record_progress_tip_poll( - snapshot(generation=2, fingerprint=work.template_fingerprint) - ) - - health = server.progress_health_snapshot() - - self.assertFalse(health["ok"]) - self.assertIn("current_generation_not_delivered", health["reasons"]) - self.assertEqual(health["eligible_client_count"], 1) - self.assertEqual(health["eligible_clients_requiring_refresh"], 1) - - def test_partial_fanout_stays_pending_until_every_client_is_current(self) -> None: - server, clock = progress_coordinator() - work = snapshot(generation=1, fingerprint="aa" * 32) - publish(server, work) - delivered = client(1) - missing = client(2) - current_context = context_for(work, 0) - delivered.active_job = current_context - server.clients.update((delivered, missing)) - server._record_progress_delivery(delivered, current_context, clock.now) - clock.advance(16) - server._record_progress_tip_poll( - snapshot(generation=2, fingerprint=work.template_fingerprint) - ) - - health = server.progress_health_snapshot() - - self.assertFalse(health["ok"]) - self.assertTrue(health["pending_refresh"]) - self.assertEqual(health["eligible_clients_requiring_refresh"], 1) - self.assertIn("current_generation_not_delivered", health["reasons"]) - - missing.active_job = current_context - server._record_progress_delivery(missing, current_context, clock.now) - self.assertTrue(server.progress_health_snapshot()["ok"]) - - def test_registered_job_is_not_delivery_proof_before_socket_send(self) -> None: - server, clock = progress_coordinator() - work = snapshot(generation=1, fingerprint="aa" * 32) - publish(server, work) - miner = client(1) - current_context = context_for(work, 0) - miner.active_job = current_context - server.clients.add(miner) - server.progress_health_snapshot() - clock.advance(16) - server._record_progress_tip_poll( - snapshot(generation=2, fingerprint=work.template_fingerprint) - ) - - before_send = server.progress_health_snapshot() - - self.assertEqual(before_send["eligible_clients_requiring_refresh"], 1) - self.assertIn("current_generation_not_delivered", before_send["reasons"]) - - server._record_progress_delivery(miner, current_context, clock.now) - after_send = server.progress_health_snapshot() - self.assertTrue(after_send["ok"]) - self.assertEqual(after_send["eligible_clients_requiring_refresh"], 0) - - def test_readiness_promotion_requires_successful_ready_delivery(self) -> None: - server, clock = progress_coordinator() - work = snapshot(generation=1, fingerprint="aa" * 32) - publish(server, work) - miner = client(1) - collection_context = context_for(work, 0, collection_only=True) - miner.active_job = collection_context - server.clients.add(miner) - server._record_progress_delivery(miner, collection_context, clock.now) - self.assertTrue(server.progress_health_snapshot()["ok"]) - - self.assertTrue(server.pool_readiness_latched()) - promoted = server.progress_health_snapshot() - self.assertTrue(promoted["pending_refresh"]) - self.assertEqual(promoted["eligible_clients_requiring_refresh"], 1) - self.assertTrue(server.client_needs_tip_template_refresh(miner, work)) - - # Fanout registers the ready context before its socket write. A write - # that stalls here must not replace the delivered collection proof. - ready_context = context_for(work, 0, collection_only=False) - miner.active_job = ready_context - server._record_progress_publication(work, 0) - clock.advance(16) - server._record_progress_tip_poll( - snapshot(generation=2, fingerprint=work.template_fingerprint) - ) - - stalled = server.progress_health_snapshot() - self.assertFalse(stalled["ok"]) - self.assertIn("current_generation_not_delivered", stalled["reasons"]) - self.assertEqual(stalled["eligible_clients_requiring_refresh"], 1) - - server._record_progress_delivery(miner, ready_context, clock.now) - delivered = server.progress_health_snapshot() - self.assertTrue(delivered["ok"]) - self.assertFalse(delivered["pending_refresh"]) - self.assertEqual(delivered["eligible_clients_requiring_refresh"], 0) - - def test_startup_is_unready_until_initial_work_is_published(self) -> None: - server, clock = progress_coordinator() - - status, startup = server.cached_health_payload() - self.assertEqual(status, 503) - self.assertIn("current_generation_not_published", startup["reasons"]) - - clock.advance(16) - self.assertIn("tip_poll_stale", server.progress_health_snapshot()["reasons"]) - - publish(server, snapshot(generation=1, fingerprint="aa" * 32)) - status, ready = server.cached_health_payload() - self.assertEqual(status, 200) - self.assertTrue(ready["ok"]) - - def test_stale_cached_ok_cannot_mask_a_progress_failure(self) -> None: - server, clock = progress_coordinator() - original = snapshot(generation=1, fingerprint="aa" * 32) - publish(server, original) - server.health_refresh_seconds = 60.0 - server.refresh_health_snapshot() - server._ensure_observability_service().set_loop_running_for_test(True) - - changed = snapshot(generation=2, fingerprint="bb" * 32, tip="22" * 32) - server._record_progress_tip_poll(changed) - clock.advance(16) - server._record_progress_tip_poll( - snapshot( - generation=3, - fingerprint=changed.template_fingerprint, - tip=changed.bestblockhash, - ) - ) - - status, health = server.cached_health_payload() - - self.assertEqual(status, 503) - self.assertFalse(health["ok"]) - self.assertLess(health["snapshot_age_seconds"], server.health_refresh_seconds) - - def test_wall_clock_changes_do_not_affect_health_decisions(self) -> None: - server, clock = progress_coordinator() - publish(server, snapshot(generation=1, fingerprint="aa" * 32)) - - with patch("lab.prism.prism_coordinator.time.time", return_value=-10**12): - self.assertTrue(server.progress_health_snapshot()["ok"]) - with patch("lab.prism.prism_coordinator.time.time", return_value=10**12): - self.assertTrue(server.progress_health_snapshot()["ok"]) - self.assertEqual(clock.now, 100.0) - - def test_tip_poll_freshness_has_an_independent_deadline(self) -> None: - server, clock = progress_coordinator() - publish(server, snapshot(generation=1, fingerprint="aa" * 32)) - clock.advance(16) - - health = server.progress_health_snapshot() - - self.assertFalse(health["ok"]) - self.assertEqual(health["reasons"], ["tip_poll_stale"]) - - def test_older_poll_cannot_renew_current_generation_freshness(self) -> None: - server, clock = progress_coordinator() - current = snapshot(generation=2, fingerprint="bb" * 32, tip="22" * 32) - publish(server, current) - clock.advance(16) - - server._record_progress_tip_poll( - snapshot(generation=1, fingerprint="aa" * 32) - ) - health = server.progress_health_snapshot() - - self.assertFalse(health["ok"]) - self.assertEqual(health["current_template_generation"], 2) - self.assertEqual(health["tip_poll_age_seconds"], 16.0) - self.assertIn("tip_poll_stale", health["reasons"]) - - def test_progressing_refresh_does_not_report_tip_poll_stale(self) -> None: - server, clock = progress_coordinator() - work = snapshot(generation=1, fingerprint="aa" * 32) - publish(server, work) - server._progress_refresh_started() - clock.advance(10) - server._record_progress_publication(work, 0) - clock.advance(10) - - health = server.progress_health_snapshot() - - self.assertTrue(health["ok"]) - self.assertTrue(health["tip_refresh_in_progress"]) - self.assertEqual(health["tip_poll_age_seconds"], 20.0) - self.assertEqual(health["tip_refresh_progress_age_seconds"], 10.0) - self.assertNotIn("tip_poll_stale", health["reasons"]) - server._progress_refresh_finished() - - def test_stalled_active_refresh_still_reports_tip_poll_stale(self) -> None: - server, clock = progress_coordinator() - publish(server, snapshot(generation=1, fingerprint="aa" * 32)) - server._progress_refresh_started() - clock.advance(16) - - health = server.progress_health_snapshot() - - self.assertFalse(health["ok"]) - self.assertIn("tip_poll_stale", health["reasons"]) - server._progress_refresh_finished() - - def test_progress_health_cannot_mask_base_mining_failure(self) -> None: - server, _ = progress_coordinator() - progress = { - "ok": True, - "reason": None, - "reasons": [], - } - - payload = server._apply_progress_health({"ok": False}, progress) - - self.assertFalse(payload["ok"]) - - def test_healthy_response_fields_remain_backward_compatible(self) -> None: - server, _ = progress_coordinator() - publish(server, snapshot(generation=1, fingerprint="aa" * 32)) - - payload = server.health_payload() - - for field in ( - "ok", - "schema", - "ledger_backend", - "accepted_share_count", - "ready_miner_count", - "accepted_block", - "accepted_block_count", - "max_blocks", - ): - self.assertIn(field, payload) - - def test_progress_metrics_have_bounded_state_and_age_gauges(self) -> None: - server, _ = progress_coordinator() - publish(server, snapshot(generation=1, fingerprint="aa" * 32)) - server._record_coordination_blocked_refresh(100.0) - - with patch( - "lab.prism.prism_coordinator.time.monotonic", - return_value=105.0, - ): - metrics = server.metrics_payload() - - for metric in ( - "qbit_prism_refresh_pending", - "qbit_prism_refresh_pending_age_seconds", - "qbit_prism_tip_poll_age_seconds", - "qbit_prism_current_generation_delivery_age_seconds", - "qbit_prism_bundle_build_oldest_age_seconds", - "# TYPE qbit_prism_template_refresh_coordination_blocked_age_seconds gauge", - "qbit_prism_template_refresh_coordination_blocked_age_seconds 5.000000", - 'qbit_prism_health_state{reason="healthy"} 1', - ): - self.assertIn(metric, metrics) - - server._clear_coordination_blocked_streak() - with patch( - "lab.prism.prism_coordinator.time.monotonic", - return_value=106.0, - ): - cleared_metrics = server.metrics_payload() - self.assertIn( - "qbit_prism_template_refresh_coordination_blocked_age_seconds 0.000000", - cleared_metrics, - ) - - -def owner_progress_service( - *, bundle_build_deadline_seconds: float = 60.0 -) -> tuple[ProgressHealthService, FakeMonotonicClock, ProgressHealthConfig, float]: - clock = FakeMonotonicClock() - service = ProgressHealthService( - started_monotonic=clock.now, - monotonic=clock, - ) - config = ProgressHealthConfig( - pending_refresh_deadline_seconds=15.0, - tip_poll_deadline_seconds=15.0, - bundle_build_deadline_seconds=bundle_build_deadline_seconds, - ) - return service, clock, config, clock.now - - -def work_generation( - *, generation: int, fingerprint: str, payout_generation: int = 0 -) -> WorkGeneration: - return WorkGeneration( - template_generation=generation, - template_fingerprint=fingerprint, - payout_generation=payout_generation, - ) - - -def owner_service_health( - service: ProgressHealthService, - config: ProgressHealthConfig, - started_monotonic: float, - *, - payout_generation: int = 0, -) -> dict[str, object]: - return service.snapshot( - lambda _fingerprint, _payout_generation: (0, 0), - payout_generation, - config, - started_monotonic, - ).as_mapping() - - -class ProgressTokenOwnershipTests(unittest.TestCase): - """Direct R1/J1 token contracts on the extracted G1 owner service.""" - - def test_refresh_token_finishes_on_exception(self) -> None: - service, _clock, config, started = owner_progress_service() - - with self.assertRaisesRegex(RuntimeError, "boom"): - with service.start_refresh(): - raise RuntimeError("boom") - - health = owner_service_health(service, config, started) - self.assertFalse(health["tip_refresh_in_progress"]) - - def test_bundle_token_finishes_on_cancellation(self) -> None: - service, clock, config, started = owner_progress_service( - bundle_build_deadline_seconds=1.0 - ) - - with self.assertRaises(CancelledError): - with service.start_bundle_build(): - raise CancelledError() - clock.advance(2) - - health = owner_service_health(service, config, started) - self.assertEqual(health["bundle_build_oldest_age_seconds"], 0.0) - - def test_token_finish_is_idempotent(self) -> None: - service, _clock, config, started = owner_progress_service() - refresh = service.start_refresh() - build = service.start_bundle_build() - - refresh.finish() - refresh.finish() - build.finish() - build.finish() - - health = owner_service_health(service, config, started) - self.assertFalse(health["tip_refresh_in_progress"]) - self.assertEqual(health["bundle_build_oldest_age_seconds"], 0.0) - - def test_overlapping_refresh_tokens_finish_independently(self) -> None: - service, clock, config, started = owner_progress_service() - first = service.start_refresh() - clock.advance(5) - second = service.start_refresh() - - second.finish() - health = owner_service_health(service, config, started) - self.assertTrue(health["tip_refresh_in_progress"]) - first.note_activity() - first.finish() - - health = owner_service_health(service, config, started) - self.assertFalse(health["tip_refresh_in_progress"]) - - def test_oldest_overlapping_bundle_controls_health(self) -> None: - service, clock, config, started = owner_progress_service( - bundle_build_deadline_seconds=60.0 - ) - current = work_generation(generation=1, fingerprint="aa" * 32) - service.observe_tip(current) - self.assertTrue(service.publish_work(current)) - first = service.start_bundle_build() - clock.advance(10) - second = service.start_bundle_build() - clock.advance(51) - service.observe_tip( - work_generation(generation=2, fingerprint="aa" * 32) - ) - - health = owner_service_health(service, config, started) - self.assertIn("bundle_build_stuck", health["reasons"]) - self.assertEqual(health["bundle_build_oldest_age_seconds"], 61.0) - - first.finish() - health = owner_service_health(service, config, started) - self.assertNotIn("bundle_build_stuck", health["reasons"]) - self.assertEqual(health["bundle_build_oldest_age_seconds"], 51.0) - second.finish() - - def test_multiple_failures_keep_the_fixed_reason_order(self) -> None: - service, clock, config, started = owner_progress_service( - bundle_build_deadline_seconds=60.0 - ) - build = service.start_bundle_build() - clock.advance(61) - - health = owner_service_health(service, config, started) - - self.assertEqual( - health["reasons"], - [ - "tip_poll_stale", - "bundle_build_stuck", - "current_generation_not_published", - ], - ) - build.finish() - - -if __name__ == "__main__": - unittest.main() diff --git a/tests/test_prism_public_credentials.py b/tests/test_prism_public_credentials.py new file mode 100644 index 000000000..63b459f41 --- /dev/null +++ b/tests/test_prism_public_credentials.py @@ -0,0 +1,561 @@ +#!/usr/bin/env python3 +"""Public-reader credential boundary through real Compose merges and startup. + +Only synthetic inputs are rendered; neither the caller's .env nor its database +settings are read. Image integration is enabled by PRISM_CREDENTIAL_TEST_IMAGE +in the existing image smoke gate (or explicitly by a local caller). +""" + +from __future__ import annotations + +import itertools +import json +import os +import re +import shutil +import subprocess +import tempfile +import time +import unittest +import uuid +from pathlib import Path +from urllib.parse import quote + + +ROOT = Path(__file__).resolve().parents[1] +BOOTSTRAP_PASSWORD = "fixture-bootstrap-only-383" +READER_PASSWORD = "fixture-reader-only-383" +WRITER_URL = f"postgresql://prism_bootstrap:{BOOTSTRAP_PASSWORD}@prism-postgres:5432/prism_fixture" +READER_URL = f"postgresql://prism_reader:{READER_PASSWORD}@prism-postgres-replica:5432/prism_fixture" +PASSWORDLESS_URL = "postgresql://prism_reader@prism-postgres-replica:5432/prism_fixture" +LITERAL_PASSWORD = "reader$MISSING #:@/383\\tail" +LITERAL_PASSWORDLESS_URL = PASSWORDLESS_URL.replace("prism_reader@", "prism_literal_reader@") +LITERAL_URL = LITERAL_PASSWORDLESS_URL.replace("prism_literal_reader@", f"prism_literal_reader:{quote(LITERAL_PASSWORD, safe='')}@") +STACKS = tuple( + tuple(name for enabled, name in zip(flags, ( + "compose.production.yaml", "compose.prism-external-db.yaml", "compose.prism-ha.yaml", + )) if enabled) + for flags in itertools.product((False, True), repeat=3) +) + + +def fixture_env(*, production: bool = False, **overrides: str) -> dict[str, str]: + inherited = ("PATH", "HOME", "DOCKER_HOST", "DOCKER_CONTEXT", "XDG_CONFIG_HOME") + return { + **{key: os.environ[key] for key in inherited if key in os.environ}, + "QBIT_SRC_DIR": str(ROOT), + "QBIT_PRODUCTION": "1" if production else "0", + "QBIT_CHAIN": "mainnet" if production else "regtest", + "QBIT_RPC_PASSWORD": "fixture-rpc-only-383", + "PRISM_POSTGRES_USER": "prism_bootstrap", + "PRISM_POSTGRES_PASSWORD": BOOTSTRAP_PASSWORD, + "PRISM_POSTGRES_DB": "prism_fixture", + "PRISM_DATABASE_URL": WRITER_URL, + "PRISM_PUBLIC_STRATUM_URL": "stratum+tcp://pool.example.invalid:3340", + # Ambient libpq credentials must not become container credentials. + "PGPASSWORD": "fixture-ambient-password-383", + **overrides, + } + + +def compose_args(stack: tuple[str, ...], extra_file: str | None = None, + project: str = "prism-reader-credentials", env_file: Path | None = None) -> list[str]: + args = ["docker", "compose", "--env-file", str(ROOT / "config/upstream.env.example")] + if env_file: + args.extend(("--env-file", str(env_file))) + for name in ("compose.yaml", *stack): + args.extend(("-f", str(ROOT / name))) + if extra_file: + args.extend(("-f", extra_file)) + return [*args, "--project-name", project, "--profile", "prism"] + + +def render_public(stack: tuple[str, ...], *, env_file: Path | None = None, **overrides: str) -> dict: + result = subprocess.run( + [*compose_args(stack, env_file=env_file), "config", "--format", "json"], + cwd=ROOT, env=fixture_env(production="compose.production.yaml" in stack, **overrides), + text=True, capture_output=True, check=False, + ) + if result.returncode: + raise AssertionError(f"synthetic Compose render failed: {result.stderr}") + # Never print the complete rendered stack: assertions concern this service. + return json.loads(result.stdout)["services"]["prism-public-api"] + + +class PublicCredentialComposeTests(unittest.TestCase): + @classmethod + def setUpClass(cls) -> None: + if shutil.which("docker") is None: + raise unittest.SkipTest("docker CLI is not installed") + result = subprocess.run(["docker", "compose", "version"], text=True, capture_output=True) + if result.returncode and "not a docker command" in result.stderr: + raise unittest.SkipTest("docker compose is unavailable") + if result.returncode: + raise AssertionError(f"docker compose version failed: {result.stderr}") + + def assert_reader_boundary(self, public: dict, url: str, password: str = "") -> None: + environment = public["environment"] + self.assertEqual(environment["PRISM_DATABASE_URL"], url) + # Compose escapes dollars when serializing its reusable config; actual + # container values are checked verbatim by the runtime test below. + self.assertEqual(environment["PGPASSWORD"], password.replace("$", "$$")) + self.assertNotIn("PRISM_POSTGRES_PASSWORD", environment) + self.assertNotIn("PRISM_PUBLIC_POSTGRES_PASSWORD", environment) + self.assertNotIn(BOOTSTRAP_PASSWORD, json.dumps(public)) + self.assertNotIn("fixture-ambient-password-383", json.dumps(public)) + self.assertEqual(public["command"], ["qbit-prism-server", "public-api"]) + self.assertEqual( + public["healthcheck"]["test"], + ["CMD", "qbit-prism-server", "healthcheck", "--public-api"], + ) + + def test_dedicated_dsn_excludes_bootstrap_password_in_every_stack(self) -> None: + for stack in STACKS: + with self.subTest(stack=stack): + self.assert_reader_boundary( + render_public(stack, PRISM_PUBLIC_DATABASE_URL=READER_URL), READER_URL, + ) + + def test_separate_carrier_has_only_public_reader_provenance(self) -> None: + for stack, password in itertools.product(STACKS, (None, "", "fixture-bad-reader", READER_PASSWORD)): + with self.subTest(stack=stack, password=password): + overrides = {"PRISM_PUBLIC_DATABASE_URL": PASSWORDLESS_URL} + if password is not None: + overrides["PRISM_PUBLIC_POSTGRES_PASSWORD"] = password + self.assert_reader_boundary(render_public(stack, **overrides), PASSWORDLESS_URL, password or "") + + def test_tools_only_production_selector_reaches_every_public_stack(self) -> None: + for stack in STACKS: + with self.subTest(stack=stack): + public = render_public(stack, QBIT_PRODUCTION="0", QBIT_TOOLS_PRODUCTION="1", + QBIT_CHAIN="regtest", PRISM_PUBLIC_DATABASE_URL=READER_URL) + self.assertEqual(public["environment"]["QBIT_PRODUCTION"], "0") + self.assertEqual(public["environment"]["QBIT_TOOLS_PRODUCTION"], "1") + + def test_missing_and_empty_public_dsn_preserve_supported_defaults(self) -> None: + for stack, url, password in itertools.product(STACKS, (None, ""), ("", READER_PASSWORD)): + with self.subTest(stack=stack, url=url, carrier_set=bool(password)): + overrides = {} if url is None else {"PRISM_PUBLIC_DATABASE_URL": url} + overrides["PRISM_PUBLIC_POSTGRES_PASSWORD"] = password + public = render_public(stack, **overrides) + external = "compose.prism-external-db.yaml" in stack + expected = WRITER_URL if external else WRITER_URL.replace("@prism-postgres:", "@prism-postgres-replica:") + self.assertEqual(public["environment"]["PRISM_DATABASE_URL"], expected) + self.assertEqual(public["environment"]["PGPASSWORD"], password) + self.assertEqual(public["environment"]["PRISM_PUBLIC_REPLICA_MODE"], "off" if external else "require") + + def test_literal_env_file_password_and_encoded_dsn_in_every_stack(self) -> None: + with tempfile.TemporaryDirectory(prefix="prism-reader-env-") as directory: + env_file = Path(directory) / "reader.env" + # Match the documented form for both Compose and the shell-based + # preflight loader. The caller's environment has no carrier. + env_file.write_text(f"PRISM_PUBLIC_DATABASE_URL={LITERAL_PASSWORDLESS_URL}\n" + f"PRISM_PUBLIC_POSTGRES_PASSWORD='{LITERAL_PASSWORD}'\n", encoding="utf-8") + sourced = subprocess.run( + ["bash", "-c", 'source "$1"; printf "%s" "$PRISM_PUBLIC_POSTGRES_PASSWORD"', "fixture", str(env_file)], + env=fixture_env(), text=True, capture_output=True, check=True, + ) + self.assertEqual(sourced.stdout, LITERAL_PASSWORD) + for stack in STACKS: + with self.subTest(stack=stack): + self.assert_reader_boundary(render_public(stack, env_file=env_file), + LITERAL_PASSWORDLESS_URL, LITERAL_PASSWORD) + self.assert_reader_boundary(render_public(stack, PRISM_PUBLIC_DATABASE_URL=LITERAL_URL), + LITERAL_URL) + + +@unittest.skipUnless(os.environ.get("PRISM_CREDENTIAL_TEST_IMAGE"), "set PRISM_CREDENTIAL_TEST_IMAGE for disposable image integration") +class PublicCredentialRuntimeTests(unittest.TestCase): + """Run the actual merged public service, image entrypoint and HTTP probe.""" + + @classmethod + def command(cls, *args: str, check: bool = True, input: str | None = None, + env: dict[str, str] | None = None) -> subprocess.CompletedProcess: + return subprocess.run(args, cwd=ROOT, env=fixture_env(**(env or {})), text=True, + input=input, capture_output=True, check=check, timeout=180) + + @classmethod + def sql(cls, query: str, *, reader: bool = False, check: bool = True) -> subprocess.CompletedProcess: + return cls.command( + "docker", "exec", "-i", "--env", "PGPASSWORD", + cls.primary, "psql", "-h", "127.0.0.1", "-U", "prism_reader" if reader else "prism_bootstrap", + "-d", "prism_fixture", "-v", "ON_ERROR_STOP=1", "-Atf", "-", input=query, check=check, + env={"PGPASSWORD": READER_PASSWORD if reader else BOOTSTRAP_PASSWORD}, + ) + + @classmethod + def setUpClass(cls) -> None: + cls.image = os.environ["PRISM_CREDENTIAL_TEST_IMAGE"] + prefix = f"prism-reader-383-{uuid.uuid4().hex[:10]}" + cls.project = prefix + cls.network = f"{prefix}-net" + cls.primary, cls.replica, cls.public = (f"{prefix}-{role}" for role in ("primary", "replica", "public")) + cls.addClassCleanup(cls.command, "docker", "network", "rm", cls.network, check=False) + for container in (cls.primary, cls.replica, cls.public): + cls.addClassCleanup(cls.command, "docker", "rm", "--force", "--volumes", container, check=False) + cls.command("docker", "network", "create", cls.network) + cls.command( + "docker", "run", "-d", "--name", cls.primary, "--network", cls.network, + "--network-alias", "prism-postgres", "--env", "POSTGRES_USER=prism_bootstrap", + "--env", "POSTGRES_PASSWORD", "--env", "POSTGRES_DB=prism_fixture", + "--env", "POSTGRES_HOST_AUTH_METHOD=scram-sha-256", "postgres:16", + env={"POSTGRES_PASSWORD": BOOTSTRAP_PASSWORD}, + ) + for _ in range(60): + if cls.sql("SELECT 1", check=False).returncode == 0: + break + time.sleep(1) + else: + raise AssertionError("disposable PostgreSQL did not become ready") + cls.command("docker", "run", "--rm", "--network", cls.network, "--no-healthcheck", + "--env", "PRISM_DATABASE_URL", cls.image, "qbit-prism-server", "migrate", + env={"PRISM_DATABASE_URL": WRITER_URL}) + cls.sql(f""" + CREATE ROLE prism_reader LOGIN PASSWORD '{READER_PASSWORD}' + NOSUPERUSER NOCREATEDB NOCREATEROLE NOREPLICATION; + GRANT CONNECT ON DATABASE prism_fixture TO prism_reader; + GRANT USAGE ON SCHEMA public TO prism_reader; + GRANT SELECT ON ALL TABLES IN SCHEMA public TO prism_reader; + GRANT pg_read_all_stats TO prism_reader; + CREATE ROLE prism_literal_reader LOGIN PASSWORD '{LITERAL_PASSWORD.replace("'", "''")}' + NOSUPERUSER NOCREATEDB NOCREATEROLE NOREPLICATION; + GRANT prism_reader TO prism_literal_reader; + CREATE ROLE qbit LOGIN PASSWORD 'change-this' + NOSUPERUSER NOCREATEDB NOCREATEROLE NOREPLICATION; + GRANT prism_reader TO qbit; + """) + # Use the shipped standby entrypoint against a disposable primary only. + cls.command("docker", "exec", cls.primary, "sh", "-c", + 'printf "host replication prism_bootstrap all scram-sha-256\\n" >> "$PGDATA/pg_hba.conf"') + cls.sql("SELECT pg_reload_conf()") + cls.command( + "docker", "run", "-d", "--name", cls.replica, "--network", cls.network, + "--network-alias", "prism-postgres-replica", "--env", "PRISM_POSTGRES_USER=prism_bootstrap", + "--env", "PRISM_POSTGRES_PASSWORD", "--env", "PRISM_POSTGRES_DB=prism_fixture", + "--volume", f"{ROOT / 'config/prism-postgres/replica-entrypoint.sh'}:/replica-entrypoint.sh:ro", + "--entrypoint", "bash", "postgres:16", "/replica-entrypoint.sh", + ) + for _ in range(90): + result = cls.command("docker", "exec", cls.replica, "psql", "-U", "prism_bootstrap", + "-d", "prism_fixture", "-Atc", "SELECT pg_is_in_recovery()", check=False) + if result.returncode == 0 and result.stdout.strip() == "t": + break + time.sleep(1) + else: + raise AssertionError("disposable standby did not become ready") + + def start_public(self, stack: tuple[str, ...], *, lab_default: bool = False, + env_file: Path | None = None, **overrides: str) -> dict: + self.command("docker", "rm", "--force", self.public, check=False) + # Only adapt placement: retain the merged environment, command, + # healthcheck and the image's real entrypoint. No production services, + # host data mounts or published host ports are started by this test. + with tempfile.TemporaryDirectory(prefix="prism-reader-compose-") as directory: + overlay = Path(directory) / "runtime.yaml" + overlay.write_text( + "services:\n prism-public-api:\n" + f" container_name: {self.public}\n" + " restart: 'no'\n pull_policy: never\n" + " ports: !override []\n networks: !override [fixture]\n" + "networks:\n fixture:\n external: true\n" + f" name: {self.network}\n", encoding="utf-8", + ) + subprocess.run( + [*compose_args(stack, str(overlay), self.project, env_file), "up", "--detach", "--no-deps", "--no-build", + "--pull", "never", "prism-public-api"], + cwd=ROOT, env=fixture_env(production="compose.production.yaml" in stack, + PRISM_COORDINATOR_IMAGE=self.image, **overrides), + text=True, capture_output=True, check=True, timeout=90, + ) + config = self.command("docker", "inspect", "--format", "{{json .Config}}", self.public).stdout + if not lab_default: + self.assertNotIn(BOOTSTRAP_PASSWORD, config) + self.assertNotIn("fixture-ambient-password-383", config) + if lab_default: + self.assertIn("PGPASSWORD=", json.loads(config)["Env"]) + self.assertTrue(any(item.startswith("PGPASSWORD=") for item in json.loads(config)["Env"])) + return json.loads(config) + + def preflight(self, stack: tuple[str, ...], *, env_file: Path | None = None, + make_deployment: bool = False, extra_files: tuple[str, ...] = (), + **overrides: str) -> subprocess.CompletedProcess: + # The actual shell preflight runs the public image with the same merged + # service environment. Place its one-shot container on our disposable + # network; no production services, volumes or ports are started. + with tempfile.TemporaryDirectory(prefix="prism-reader-preflight-") as directory: + fixture_root = Path(directory) + (fixture_root / "scripts").mkdir() + (fixture_root / "config").mkdir() + for source, destination in ( + (ROOT / "scripts/check-env.sh", fixture_root / "scripts/check-env.sh"), + (ROOT / ".env.example", fixture_root / ".env.example"), + (ROOT / "config/upstream.env.example", fixture_root / "config/upstream.env.example"), + (ROOT / "compose.yaml", fixture_root / "compose.yaml"), + (ROOT / "compose.production.yaml", fixture_root / "compose.production.yaml"), + ): + shutil.copyfile(source, destination) + overlay = fixture_root / "placement.yaml" + overlay.write_text( + "services:\n prism-public-api:\n networks: !override [fixture]\n" + "networks:\n fixture:\n external: true\n" + f" name: {self.network}\n", encoding="utf-8") + args = ["bash", str(fixture_root / "scripts/check-env.sh"), "--public-reader-only"] + if make_deployment: + args.append("--make-deployment") + for name in (*stack, *extra_files, str(overlay)): + args.extend(("--compose-file", str(ROOT / name))) + if env_file: + overrides["DEPLOY_ENV_FILE"] = str(env_file) + result = subprocess.run(args, cwd=ROOT, text=True, capture_output=True, timeout=60, + env=fixture_env(production="compose.production.yaml" in stack, + **{"PRISM_COORDINATOR_IMAGE": self.image, + "COMPOSE_PROJECT_NAME": self.project, **overrides})) + diagnostics = result.stdout + result.stderr + for credential in (BOOTSTRAP_PASSWORD, READER_PASSWORD, LITERAL_PASSWORD, "fixture-bad-reader", + "change-this", "ch%61nge-this"): + self.assertFalse(credential in diagnostics, "preflight exposed a credential") + return result + + def assert_config_exit(self, expected: str) -> None: + deadline = time.monotonic() + 15 + while time.monotonic() < deadline: + state = json.loads(self.command("docker", "inspect", "--format", "{{json .State}}", self.public).stdout) + if not state["Running"]: + self.assertNotEqual(state["ExitCode"], 0) + logs = self.command("docker", "logs", self.public) + diagnostics = logs.stdout + logs.stderr + for credential in (BOOTSTRAP_PASSWORD, READER_PASSWORD, LITERAL_PASSWORD, "change-this", + "ch%61nge-this", "fixture-secret"): + self.assertFalse(credential in diagnostics, "startup exposed a credential") + self.assertTrue(expected in diagnostics, "missing configuration diagnostic") + return + time.sleep(0.1) + self.fail("invalid configuration reached service startup instead of exiting") + + def response(self, path: str) -> tuple[int, dict]: + result = self.command("docker", "exec", self.public, "curl", "--silent", "--show-error", + "--max-time", "2", "--write-out", "\n%{http_code}", + f"http://127.0.0.1:3342{path}") + body, code = result.stdout.rsplit("\n", 1) + return int(code), json.loads(body) + + def assert_health(self, healthy: bool, *, error: str = "database authentication failed", + category: str = "authentication") -> None: + deadline = time.monotonic() + 30 + while True: + try: + status, body = self.response("/healthz") + if body.get("state") != "starting": + break + except subprocess.CalledProcessError: + pass + if time.monotonic() >= deadline: + self.fail("synthetic public listener did not report readiness within 30 seconds") + time.sleep(0.25) + self.assertEqual(status, 200 if healthy else 503, body) + self.assertIs(body["ok"], healthy, body) + self.assertIs(body["database_ready"], healthy, body) + if not healthy: + # A failed initial authentication leaves require-mode replicas + # unprobed. Preserve that public warm-up message while checking the + # causal, sanitized operator diagnostic in every deployment mode. + expected_error = ("public read service is warming up" + if "replica" in body and not body["replica"]["probed"] + else error) + self.assertEqual(body.get("error"), expected_error, body) + logs = self.command("docker", "logs", self.public) + # The runtime formatter may color fields even in captured logs. + diagnostic = re.sub(r"\x1b\[[0-9;]*m", "", logs.stdout + logs.stderr) + self.assertIn("public readiness probe failed", diagnostic) + self.assertIn(f'category="{category}"', diagnostic) + self.assertIn('action="check ', diagnostic) + for private in (BOOTSTRAP_PASSWORD, READER_PASSWORD, LITERAL_PASSWORD, + "prism_bootstrap", "prism_reader", "prism-postgres", + "prism_fixture", "fixture_private_schema_405", "fixture_missing_database_405"): + self.assertNotIn(private, json.dumps(body)) + self.assertNotIn(private, diagnostic) + self.assertNotIn(BOOTSTRAP_PASSWORD, json.dumps(body)) + self.assertNotIn(READER_PASSWORD, json.dumps(body)) + self.assertNotIn(LITERAL_PASSWORD, json.dumps(body)) + probe = self.command("docker", "exec", self.public, "qbit-prism-server", "healthcheck", "--public-api", check=False) + self.assertEqual(probe.returncode == 0, healthy, probe.stderr) + # Blocks exercises application-table reads without requiring a node RPC. + status, body = self.response("/public/v1/blocks") + self.assertEqual(status, 200 if healthy else 503, body) + + def test_reader_dsn_and_http_health_in_every_stack(self) -> None: + for stack in STACKS: + with self.subTest(stack=stack): + self.assertEqual(self.preflight(stack, PRISM_PUBLIC_DATABASE_URL=READER_URL).returncode, 0) + self.start_public(stack, PRISM_PUBLIC_DATABASE_URL=READER_URL) + self.assert_health(True) + + def test_supported_default_dsns_still_connect(self) -> None: + for stack, dsn in itertools.product(STACKS, (None, "")): + with self.subTest(stack=stack, dsn_absent=dsn is None): + overrides = {} if dsn is None else {"PRISM_PUBLIC_DATABASE_URL": dsn} + self.assertEqual(self.preflight(stack, **overrides).returncode, 0) + self.start_public(stack, lab_default=True, **overrides) + self.assert_health(True) + + def test_shipped_lab_password_remains_supported(self) -> None: + overrides = {"PRISM_PUBLIC_DATABASE_URL": "", "PRISM_POSTGRES_USER": "qbit", + "PRISM_POSTGRES_PASSWORD": "change-this"} + self.assertEqual(self.preflight((), **overrides).returncode, 0) + self.start_public((), lab_default=True, **overrides) + self.assert_health(True) + + def test_literal_env_file_password_and_encoded_dsn_authenticate(self) -> None: + stack = ("compose.production.yaml", "compose.prism-external-db.yaml", "compose.prism-ha.yaml") + with tempfile.TemporaryDirectory(prefix="prism-reader-env-") as directory: + env_file = Path(directory) / "reader.env" + env_file.write_text(f"PRISM_PUBLIC_DATABASE_URL={LITERAL_PASSWORDLESS_URL}\n" + f"PRISM_PUBLIC_POSTGRES_PASSWORD='{LITERAL_PASSWORD}'\n", encoding="utf-8") + self.assertEqual(self.preflight(stack, env_file=env_file).returncode, 0) + config = self.start_public(stack, env_file=env_file) + self.assertIn(f"PGPASSWORD={LITERAL_PASSWORD}", config["Env"]) + self.assert_health(True) + self.assertEqual(self.preflight(stack, PRISM_PUBLIC_DATABASE_URL=LITERAL_URL).returncode, 0) + config = self.start_public(stack, PRISM_PUBLIC_DATABASE_URL=LITERAL_URL) + self.assertIn("PGPASSWORD=", config["Env"]) + self.assert_health(True) + + def test_interpolated_carrier_preserves_original_caller_precedence(self) -> None: + stack = ("compose.production.yaml", "compose.prism-external-db.yaml", "compose.prism-ha.yaml") + with tempfile.TemporaryDirectory(prefix="prism-reader-interpolation-") as directory: + env_file = Path(directory) / "reader.env" + env_file.write_text(f"PRISM_PUBLIC_DATABASE_URL={PASSWORDLESS_URL}\n" + "READER_SECRET=change-this\n" + "PRISM_PUBLIC_POSTGRES_PASSWORD=${READER_SECRET}\n", encoding="utf-8") + self.assertEqual(self.preflight(stack, env_file=env_file, READER_SECRET=READER_PASSWORD).returncode, 0) + self.start_public(stack, env_file=env_file, READER_SECRET=READER_PASSWORD) + self.assert_health(True) + + def test_passwordless_dsn_authentication_and_dsn_precedence(self) -> None: + for stack, password in itertools.product(STACKS, (None, "", "fixture-bad-reader", READER_PASSWORD)): + with self.subTest(stack=stack, absent=password is None, empty=password == "", valid=password == READER_PASSWORD): + overrides = {"PRISM_PUBLIC_DATABASE_URL": PASSWORDLESS_URL} + if password is not None: + overrides["PRISM_PUBLIC_POSTGRES_PASSWORD"] = password + self.assertEqual(self.preflight(stack, **overrides).returncode, 0) + self.start_public(stack, **overrides) + self.assert_health(password == READER_PASSWORD) + # An explicitly incorrect DSN password must not be repaired by even a + # valid reader carrier; the DSN is authoritative. An empty query + # password likewise wins over the carrier in SQLx. + stack = ("compose.production.yaml", "compose.prism-external-db.yaml", "compose.prism-ha.yaml") + for password in ("fixture-bad-reader", ""): + with self.subTest(dsn_password=password): + self.assertEqual(self.preflight(stack, PRISM_PUBLIC_DATABASE_URL=f"{PASSWORDLESS_URL}?password={password}", + PRISM_PUBLIC_POSTGRES_PASSWORD=READER_PASSWORD).returncode, 0) + self.start_public(stack, PRISM_PUBLIC_DATABASE_URL=f"{PASSWORDLESS_URL}?password={password}", + PRISM_PUBLIC_POSTGRES_PASSWORD=READER_PASSWORD) + self.assert_health(False) + + def test_malformed_and_production_default_credentials_fail_preflight_and_startup(self) -> None: + cases = ( + ("malformed", "fixture-secret", "", "invalid public PRISM_DATABASE_URL"), + ("invalid-option", PASSWORDLESS_URL + "?sslmode=fixture-secret", "", "invalid public PRISM_DATABASE_URL connection options"), + ("uri-default", PASSWORDLESS_URL.replace("prism_reader@", "prism_reader:change-this@"), "", "non-default database credentials"), + ("encoded-default", PASSWORDLESS_URL.replace("prism_reader@", "prism_reader:ch%61nge-this@"), "", "non-default database credentials"), + ("carrier-default", PASSWORDLESS_URL, "change-this", "non-default database credentials"), + ("query-default", PASSWORDLESS_URL + "?password=ch%61nge-this", READER_PASSWORD, "non-default database credentials"), + ) + for stack in STACKS: + for label, dsn, carrier, message in cases: + if "default" in label and "compose.production.yaml" not in stack: + continue + with self.subTest(stack=stack, case=label): + overrides = {"PRISM_PUBLIC_DATABASE_URL": dsn, "PRISM_PUBLIC_POSTGRES_PASSWORD": carrier} + self.assertNotEqual(self.preflight(stack, **overrides).returncode, 0) + self.start_public(stack, **overrides) + self.assert_config_exit(message) + + def test_tools_only_production_policy_through_preflight_and_startup(self) -> None: + for stack, carrier in itertools.product(STACKS, (False, True)): + with self.subTest(stack=stack, carrier=carrier): + overrides = {"QBIT_PRODUCTION": "0", "QBIT_TOOLS_PRODUCTION": "1", "QBIT_CHAIN": "regtest", + "PRISM_PUBLIC_DATABASE_URL": PASSWORDLESS_URL if carrier else PASSWORDLESS_URL.replace( + "prism_reader@", "prism_reader:ch%61nge-this@"), + "PRISM_PUBLIC_POSTGRES_PASSWORD": "change-this" if carrier else ""} + result = self.preflight(stack, make_deployment=True, **overrides) + self.assertNotEqual(result.returncode, 0) + self.assertIn("production requires non-default database credentials", result.stderr) + self.start_public(stack, **overrides) + self.assert_config_exit("non-default database credentials") + for stack in STACKS: + with self.subTest(stack=stack, valid=True): + overrides = {"QBIT_PRODUCTION": "0", "QBIT_TOOLS_PRODUCTION": "1", "QBIT_CHAIN": "regtest", + "PRISM_PUBLIC_DATABASE_URL": PASSWORDLESS_URL, + "PRISM_PUBLIC_POSTGRES_PASSWORD": READER_PASSWORD} + self.assertEqual(self.preflight(stack, make_deployment=True, **overrides).returncode, 0) + self.start_public(stack, **overrides) + self.assert_health(True) + + def test_make_preflight_builds_fresh_lab_image_and_uses_effective_external_overlay(self) -> None: + image = f"prism-credential-fresh-{uuid.uuid4().hex}:local" + self.assertNotEqual(self.command("docker", "image", "inspect", image, check=False).returncode, 0) + try: + with tempfile.TemporaryDirectory(prefix="prism-reader-build-") as directory: + root = Path(directory) + (root / "Dockerfile").write_text(f"FROM {self.image}\n", encoding="utf-8") + overlay = root / "build.yaml" + overlay.write_text("services:\n prism-public-api:\n build: !override\n" + f" context: {json.dumps(str(root))}\n dockerfile: Dockerfile\n", + encoding="utf-8") + stack = ("compose.prism-external-db.yaml", "compose.prism-ha.yaml") + overrides = {"PRISM_COORDINATOR_IMAGE": image, "PRISM_PUBLIC_DATABASE_URL": "", + # The fixture's FROM image lives in the local + # engine, not in CI's separate remote BuildKit. + "BUILDX_BUILDER": "default", + # Only the external overlay selects this reader DSN. + "PRISM_DATABASE_URL": READER_URL, "PRISM_POSTGRES_PASSWORD": "change-this"} + result = self.preflight(stack, make_deployment=True, extra_files=(str(overlay),), **overrides) + self.assertEqual(result.returncode, 0, result.stderr) + self.assertEqual(self.command("docker", "image", "inspect", image, check=False).returncode, 0) + # The same selected overlay DSN authenticates at real startup. + del overrides["PRISM_COORDINATOR_IMAGE"] + self.start_public(stack, **overrides) + self.assert_health(True) + finally: + self.command("docker", "image", "rm", "--force", image, check=False) + + def test_make_production_preflight_requires_prepared_image_and_effective_overlay(self) -> None: + stack = ("compose.prism-external-db.yaml", "compose.prism-ha.yaml") + overrides = {"QBIT_PRODUCTION": "1", "QBIT_CHAIN": "mainnet", "PRISM_PUBLIC_DATABASE_URL": "", + "PRISM_DATABASE_URL": READER_URL, "PRISM_POSTGRES_PASSWORD": "change-this"} + result = self.preflight(stack, make_deployment=True, **overrides) + self.assertEqual(result.returncode, 0, result.stderr) + self.start_public(("compose.production.yaml", *stack), **overrides) + self.assert_health(True) + absent_image = f"prism-credential-missing-{uuid.uuid4().hex}:local" + result = self.preflight(stack, make_deployment=True, PRISM_COORDINATOR_IMAGE=absent_image, **overrides) + self.assertNotEqual(result.returncode, 0) + self.assertNotEqual(self.command("docker", "image", "inspect", absent_image, check=False).returncode, 0) + + def test_missing_image_is_a_failed_preflight_without_building(self) -> None: + absent_image = f"prism-credential-missing-{uuid.uuid4().hex}:local" + self.assertNotEqual(self.preflight((), PRISM_COORDINATOR_IMAGE=absent_image).returncode, 0) + result = self.command("docker", "image", "inspect", absent_image, check=False) + self.assertNotEqual(result.returncode, 0, "preflight implicitly built an image") + + def test_reader_cannot_write_even_without_session_read_only_guard(self) -> None: + self.assertEqual(self.sql("SELECT count(*) FROM qbit_pool_blocks", reader=True).returncode, 0) + denied = self.sql("SET default_transaction_read_only=off; UPDATE qbit_pool_blocks SET inactive_since=inactive_since WHERE false", + reader=True, check=False) + self.assertNotEqual(denied.returncode, 0) + self.assertIn("permission denied", denied.stderr) + self.assertEqual(self.sql("SELECT rolsuper OR rolcreatedb OR rolcreaterole OR rolreplication FROM pg_roles WHERE rolname=current_user", reader=True).stdout.strip(), "f") + + def test_connection_and_schema_readiness_failures_are_sanitized(self) -> None: + stack = ("compose.production.yaml", "compose.prism-external-db.yaml", "compose.prism-ha.yaml") + self.start_public(stack, PRISM_PUBLIC_DATABASE_URL=READER_URL.replace( + "/prism_fixture", "/fixture_missing_database_405")) + self.assert_health(False, error="database connection failed", category="connection") + self.start_public(stack, PRISM_PUBLIC_DATABASE_URL=( + READER_URL + "?options=-csearch_path%3Dfixture_private_schema_405")) + self.assert_health(False, error="native public read schema is incomplete", category="schema") + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_prism_public_dashboard_api.py b/tests/test_prism_public_dashboard_api.py deleted file mode 100644 index f3094c4bb..000000000 --- a/tests/test_prism_public_dashboard_api.py +++ /dev/null @@ -1,3554 +0,0 @@ -#!/usr/bin/env python3 - -from __future__ import annotations - -import hashlib -import json -import os -import tempfile -import threading -import time -import unittest -import urllib.error -import urllib.request -from contextlib import contextmanager -from datetime import datetime, timezone -from decimal import Decimal -from http.server import ThreadingHTTPServer -from pathlib import Path -from types import SimpleNamespace -from typing import Iterator -from unittest.mock import patch - -from lab.prism import direct_stratum, public_api, public_read_service -from lab.prism.share_ledger import ( - LedgerOperationTimeout, - PendingShare, - PsqlShareLedger, - SingleWriterShareLedger, - canonical_json_text, - sha256_json_hex, -) - - -def make_public_handler(coordinator: object) -> type: - """Serve /public/v1 through the process that owns it. - - These routes left the coordinator's audit listener in issue #145, so the - handler under test here is the extracted public read service's. It wraps - the same public_api.dispatch behind the same PublicResponseCache, which is - what keeps these assertions meaningful across the move; - tests/test_prism_public_read_service.py pins that equivalence directly. - """ - - service = public_read_service.PublicReadService(coordinator) # type: ignore[arg-type] - return public_read_service.make_handler(service) - - -class FanoutPublicRowTests(unittest.TestCase): - def test_cpfp_anchor_spendable_requires_zero_fee_anchor(self) -> None: - base = { - "fanout_txid": "aa" * 32, - "block_hash": "bb" * 32, - "block_height": 123, - "settlement_status": "broadcastable", - "manifest_set_sha256": "cc" * 32, - "manifest_sha256": "dd" * 32, - "parent_coinbase_txid": "ee" * 32, - "parent_coinbase_vout": 0, - "covenant_output_value_sats": 10_000, - "fanout_output_sum_sats": 10_000, - "fanout_tx_hex": "00", - } - - anchored = public_api.fanout_public_row({**base, "anchor_vout": 1}) - fee_bearing = public_api.fanout_public_row( - { - **base, - "anchor_vout": None, - "covenant_output_value_sats": 10_100, - } - ) - - self.assertTrue(anchored["cpfp_anchor_spendable"]) - self.assertFalse(fee_bearing["cpfp_anchor_spendable"]) - self.assertIsNone(fee_bearing["anchor_vout"]) - - -class FakeRpc: - def __init__( - self, - *, - template_difficulty: object | None = None, - template_bits: str | None = "207fffff", - blockchain_info: dict[str, object] | None = None, - network_info: dict[str, object] | None = None, - network_hashps: object = 2_500_000_000_000_000_000, - ) -> None: - self.template_difficulty = template_difficulty - self.template_bits = template_bits - self.blockchain_info = blockchain_info or {} - self.network_info = network_info or {} - # H/s, as qbitd's getnetworkhashps reports. Pass an exception instance - # to simulate a node whose hashrate estimate is unavailable. - self.network_hashps = network_hashps - self.network_hashps_calls: list[list[object] | None] = [] - - def call(self, method: str, params: list[object] | None = None) -> object: - if method == "getblockchaininfo": - return { - "chain": "qbit-signet", - "blocks": 123456, - "bestblockhash": "0" * 63 + "1", - "initialblockdownload": False, - **self.blockchain_info, - } - if method == "getblocktemplate": - template: dict[str, object] = {} - if self.template_bits is not None: - template["bits"] = self.template_bits - if self.template_difficulty is not None: - template["network_difficulty"] = self.template_difficulty - return template - if method == "getnetworkinfo": - return {"connections": 8, **self.network_info} - if method == "getnetworkhashps": - self.network_hashps_calls.append(params) - if isinstance(self.network_hashps, BaseException): - raise self.network_hashps - return self.network_hashps - raise AssertionError(f"unexpected RPC method {method}") - - -def _bucket_timestamp(epoch: int) -> str: - return public_api.iso_datetime(datetime.fromtimestamp(epoch, timezone.utc)) - - -def _epoch_from_timestamp(timestamp: str) -> int: - parsed = datetime.strptime(timestamp, "%Y-%m-%dT%H:%M:%SZ") - return int(parsed.replace(tzinfo=timezone.utc).timestamp()) - - -FIXTURE_DIR = Path(__file__).resolve().parents[1] / "docs" / "public-dashboard-api" / "fixtures" - - -def _load_fixture(name: str) -> dict[str, object]: - with (FIXTURE_DIR / name).open(encoding="utf-8") as fixture_file: - return json.load(fixture_file) - - -class FakePublicLedger: - backend_name = "fake-ledger" - block_hash = "a" * 64 - fanout_txid = "b" * 64 - audit_bundle_sha256 = "c" * 64 - manifest_set_sha256 = "d" * 64 - manifest_sha256 = "e" * 64 - - def __init__(self) -> None: - self.now_ms = int(time.time() * 1000) - self.current_network_difficulty: object | None = None - self.reward_window_network_difficulty: object | None = None - self.reward_leaderboard_network_difficulty: object | None = None - self.leaderboard_calls = 0 - self.reward_leaderboard_calls: list[dict[str, object]] = [] - self.pool_snapshot_calls = 0 - self.hashrate_series_calls = 0 - self.blocks_calls: list[dict[str, object]] = [] - self.block_marker_calls = 0 - self.last_block_markers_anchor: int | None = None - # Found blocks across two recent 1h buckets plus one ancient one, as - # (height, hash, found_at_epoch, chain_state). The newer bucket holds - # four accepted blocks (so the top-3 cap truncates), two of them tied - # on found_at (so the height tiebreak is observable), plus a reversed - # block that must never appear. The ancient block sits far outside - # every bounded range and is only reachable through range=all. - base_hour = int(time.time()) // 3600 * 3600 - self.block_marker_records: list[tuple[int, str, int, str]] = [ - (67440, "1a" * 32, base_hour - 7200 + 300, "confirmed"), - (67441, "1b" * 32, base_hour - 3600 + 60, "confirmed"), - (67442, "1c" * 32, base_hour - 3600 + 120, "confirmed"), - (67443, "1d" * 32, base_hour - 3600 + 240, "confirmed"), - (67444, "1e" * 32, base_hour - 3600 + 240, "confirmed"), - (67439, "1f" * 32, base_hour - 3600 + 240, "reversed"), - (60000, "2a" * 32, base_hour - 400 * 86400, "confirmed"), - ] - - def dashboard_pool_snapshot(self, *, current_network_difficulty: object, generated_at: str) -> dict[str, object]: - self.pool_snapshot_calls += 1 - self.current_network_difficulty = current_network_difficulty - requested_window_weight = public_api.decimal_string(Decimal(str(current_network_difficulty)) * Decimal(8)) - return { - "hashrate_ths": {"h1": "1.5", "h3": "2.5", "h24": "3.5"}, - "participants_3h": 2, - "blocks_found_total": 3, - "prism_blocks_total": 3, - "blocks_reversed_total": 1, - "blocks_inactive_total": 2, - "total_mined_bits": 600, - "latest_block": { - "height": 123450, - "hash": "a" * 64, - "found_at": "2026-06-26T19:55:00Z", - "age_seconds": 300, - "solver_recipient_id": "miner-a", - "solver_worker_name": None, - }, - "reward_window": { - "window_multiplier": 8, - "requested_window_weight": requested_window_weight, - "oldest_share_accepted_at": "2026-06-26T11:20:00Z", - "newest_share_accepted_at": "2026-06-26T20:44:53Z", - "included_share_count": 4, - }, - } - - def dashboard_blocks(self, *, page: int, limit: int, chain_state: str | None = None) -> dict[str, object]: - # chain_state=None records that the caller omitted the argument, which - # is what the default public /blocks path must do. - self.blocks_calls.append({"page": page, "limit": limit, "chain_state": chain_state}) - active_row = { - "height": 123450, - "hash": "a" * 64, - "found_at": "2026-06-26T19:55:00Z", - "network_difficulty": "1000", - "bits": "207fffff", - "solver_recipient_id": "miner-a", - "solver_worker_name": None, - "solver_share_difficulty": "99", - "reward_window_weight": "792", - "coinbase_value_bits": 600, - "audit_bundle_sha256": "b" * 64, - "payout_manifest_sha256": "c" * 64, - "explorer_url": None, - } - if chain_state in (None, "active"): - rows = [active_row] - else: - reversed_row = { - **active_row, - "height": 123449, - "hash": "f" * 64, - "found_at": "2026-06-26T18:55:00Z", - "solver_recipient_id": "miner-b", - "solver_share_difficulty": None, - "audit_bundle_sha256": None, - "payout_manifest_sha256": None, - "chain_state": "reversed", - "disconnected_at": "2026-06-26T19:05:00Z", - } - if chain_state == "all": - rows = [ - {**active_row, "chain_state": "confirmed", "disconnected_at": None}, - reversed_row, - ] - elif chain_state == "reversed": - rows = [reversed_row] - else: - raise ValueError("chain_state must be one of active, all, reversed") - return { - "pagination": {"page": page, "limit": limit, "total_count": len(rows), "total_pages": 1}, - "rows": rows, - } - - def dashboard_leaderboard(self, *, page: int, limit: int, search: str | None = None) -> dict[str, object]: - self.leaderboard_calls += 1 - rows = [ - { - "rank": 1, - "recipient_id": "miner-a", - "display_name": None, - "hashrate_ths_3h": "2.5", - "share_percent": "60", - "hash_percent": "60", - "blocks_found": 1, - "last_share_at": "2026-06-26T20:44:53Z", - } - ] - return { - "started_at": "2026-06-26T17:45:00Z", - "ended_at": "2026-06-26T20:45:00Z", - "totals": { - "pool_hashrate_ths": "2.5", - "pool_accepted_share_difficulty": "100", - "participant_count": 1, - }, - "pagination": {"page": page, "limit": limit, "total_count": 1, "total_pages": 1}, - "rows": rows, - } - - def dashboard_reward_leaderboard( - self, - *, - page: int, - limit: int, - current_network_difficulty: object, - search: str | None = None, - recipient_id: str | None = None, - ) -> dict[str, object]: - self.reward_leaderboard_network_difficulty = current_network_difficulty - self.reward_leaderboard_calls.append( - { - "page": page, - "limit": limit, - "search": search, - "recipient_id": recipient_id, - } - ) - requested_window_weight = Decimal(str(current_network_difficulty)) * Decimal(8) - rows = [ - { - "rank": 1, - "recipient_id": "miner-a", - "display_name": None, - "hashrate_ths": "1.5", - "included_share_count": 3, - "counted_share_difficulty": public_api.decimal_string(requested_window_weight * Decimal("0.6")), - "share_percent": "60", - "blocks_found_total": 1, - "last_share_at": "2026-06-26T20:44:53Z", - }, - { - "rank": 2, - "recipient_id": "miner-b", - "display_name": None, - "hashrate_ths": "1", - "included_share_count": 2, - "counted_share_difficulty": public_api.decimal_string(requested_window_weight * Decimal("0.4")), - "share_percent": "40", - "blocks_found_total": 0, - "last_share_at": "2026-06-26T20:44:00Z", - }, - ] - if recipient_id is not None: - rows = [row for row in rows if row["recipient_id"] == recipient_id] - elif search: - rows = [row for row in rows if search.lower() in str(row["recipient_id"]).lower()] - offset = (page - 1) * limit - return { - "window": { - "id": "reward", - "window_multiplier": 8, - "network_difficulty": public_api.decimal_string(current_network_difficulty), - "requested_window_weight": public_api.decimal_string(requested_window_weight), - "counted_window_weight": public_api.decimal_string(requested_window_weight), - "is_complete": True, - "started_at": "2026-06-26T20:35:00Z", - "ended_at": "2026-06-26T20:45:00Z", - "observed_span_seconds": 600, - "included_share_count": 5, - }, - "totals": { - "pool_hashrate_ths": "2.5", - "pool_counted_share_difficulty": public_api.decimal_string(requested_window_weight), - "participant_count": 2, - "expected_time_to_block_seconds": 75, - }, - "pagination": public_api.pagination(page, limit, len(rows)), - "rows": rows[offset : offset + limit], - } - - def dashboard_miner_reward_window(self, *, recipient_id: str, current_network_difficulty: object) -> dict[str, object]: - self.reward_window_network_difficulty = current_network_difficulty - if recipient_id != "miner-a": - return {"accepted_difficulty": "0", "pool_accepted_difficulty": "0", "share_percent": None} - return {"accepted_difficulty": "50", "pool_accepted_difficulty": "100", "share_percent": "50"} - - def audit_ctv_fanout_manifest_set(self, *, block_hash: str) -> dict[str, object] | None: - if block_hash != self.block_hash: - return None - return { - "schema": "qbit.prism.ctv-fanout-recovery.v1", - "block_hash": self.block_hash, - "block_height": None, - "settlement_mode": "hybrid_coinbase_ctv_fanout", - "audit_bundle_sha256": self.audit_bundle_sha256, - "payout_manifest_sha256": "f" * 64, - "manifest_set_sha256": self.manifest_set_sha256, - "manifest_set_json": "{\"schema\":\"qbit.prism.ctv-fanout-manifest-set.v1\"}", - "manifest_set": {"schema": "qbit.prism.ctv-fanout-manifest-set.v1"}, - "artifacts": [self._fanout_artifact()], - } - - def pending_ctv_fanout_statuses(self, *, limit: int = 100) -> list[dict[str, object]]: - return [self._fanout_status()][:limit] - - def dashboard_pending_fanout_rows(self, *, page: int, limit: int) -> dict[str, object]: - rows = self._pending_fanout_statuses() - offset = (page - 1) * limit - total_count = len(rows) - return { - "pagination": { - "page": page, - "limit": limit, - "total_count": total_count, - "total_pages": (total_count + limit - 1) // limit, - }, - "rows": rows[offset : offset + limit], - } - - def ctv_fanout_status(self, *, fanout_txid: str) -> dict[str, object] | None: - if fanout_txid != self.fanout_txid: - return None - return {**self._fanout_status(), "broadcast_attempts": [{"attempted_at": "2026-06-26 20:45:00+00", "error": None}]} - - def dashboard_public_artifact(self, *, sha256: str) -> dict[str, object] | None: - if sha256 == self.audit_bundle_sha256: - return {"schema": "qbit.prism.audit-bundle.v1"} - if sha256 == self.manifest_set_sha256: - return {"schema": "qbit.prism.ctv-fanout-manifest-set.v1"} - if sha256 == self.manifest_sha256: - return {"schema": "qbit.prism.ctv-fanout-manifest.v1"} - return None - - def _fanout_artifact(self) -> dict[str, object]: - row = self._fanout_status() - row["settlement_status"] = "awaiting_maturity" - row["broadcast_attempts"] = [] - return row - - def _fanout_status(self) -> dict[str, object]: - return { - "schema": "qbit.prism.ctv-fanout-status.v1", - "fanout_txid": self.fanout_txid, - "block_hash": self.block_hash, - "block_height": 123450, - "settlement_status": "broadcastable", - "manifest_set_sha256": self.manifest_set_sha256, - "manifest_sha256": self.manifest_sha256, - "audit_bundle_sha256": self.audit_bundle_sha256, - "parent_coinbase_txid": "1" * 64, - "parent_coinbase_vout": 2, - "anchor_vout": None, - "covenant_output_value_sats": 4200, - "fanout_output_sum_sats": 4100, - "fanout_tx_hex": "02000000000100", - "broadcast_attempts": [], - } - - def _pending_fanout_statuses(self) -> list[dict[str, object]]: - rows = [] - for index in range(1_001): - row = self._fanout_status() - row["fanout_txid"] = self.fanout_txid if index == 0 else f"{index:064x}" - row["chunk_index"] = index - row["chunk_count"] = 1_001 - rows.append(row) - return rows - - def dashboard_hashrate_series( - self, - *, - subject_type: str, - subject_id: str | None, - range_id: str, - bucket: str, - lookback_seconds: int = 0, - range_anchor_epoch: int | None = None, - ) -> list[dict[str, object]]: - self.hashrate_series_calls += 1 - self.last_hashrate_series_lookback = lookback_seconds - self.last_hashrate_series_anchor = range_anchor_epoch - if bucket == "5m": - # Recent bucket epochs (with a gap between them) so the points sit - # inside every bounded range after lookback trimming. - aligned_epoch = int(time.time()) // 300 * 300 - return [ - { - "timestamp": _bucket_timestamp(aligned_epoch - 600), - "hashrate_ths": public_api.hashrate_ths_from_difficulty(600, 300), - "accepted_share_count": 2, - "accepted_share_difficulty": "600", - }, - { - "timestamp": _bucket_timestamp(aligned_epoch), - "hashrate_ths": public_api.hashrate_ths_from_difficulty(1200, 300), - "accepted_share_count": 1, - "accepted_share_difficulty": "1200", - }, - ] - return [ - { - "timestamp": "2026-06-26T20:00:00Z", - "hashrate_ths": "2.5", - "accepted_share_count": 3, - "accepted_share_difficulty": "100", - } - ] - - def dashboard_block_markers( - self, - *, - range_id: str, - bucket: str, - range_anchor_epoch: int | None = None, - ) -> dict[str, object]: - # Mirrors the Postgres read model over the fixture records: reversed - # blocks excluded, floor bucketing on the hashrate grid, the - # hashrate_series_min_epoch lower bound for bounded ranges, per-bucket - # top-3 by found_at DESC then height DESC. - self.block_marker_calls += 1 - self.last_block_markers_anchor = range_anchor_epoch - bucket_seconds = public_api.HASHRATE_SERIES_BUCKET_SECONDS[bucket] - range_seconds = public_api.HASHRATE_SERIES_RANGE_SECONDS[range_id] - min_epoch = None - if range_seconds is not None: - anchor = range_anchor_epoch if range_anchor_epoch is not None else int(time.time()) - min_epoch = public_api.hashrate_series_min_epoch(anchor, range_seconds, bucket_seconds) - buckets: dict[int, list[tuple[int, str, int]]] = {} - total = 0 - for height, block_hash, found_epoch, chain_state in self.block_marker_records: - if chain_state == "reversed": - continue - if min_epoch is not None and found_epoch < min_epoch: - continue - bucket_epoch = found_epoch // bucket_seconds * bucket_seconds - buckets.setdefault(bucket_epoch, []).append((height, block_hash, found_epoch)) - total += 1 - points: list[dict[str, object]] = [] - for bucket_epoch in sorted(buckets): - records = sorted(buckets[bucket_epoch], key=lambda record: (-record[2], -record[0])) - points.append( - { - "timestamp": _bucket_timestamp(bucket_epoch), - "block_count": len(records), - "blocks": [ - { - "height": height, - "hash": block_hash, - "found_at": _bucket_timestamp(found_epoch), - } - for height, block_hash, found_epoch in records[ - : public_api.BLOCK_MARKERS_MAX_BLOCKS_PER_BUCKET - ] - ], - } - ) - return {"total_blocks": total, "points": points} - - def all_shares(self) -> list[object]: - return [ - SimpleNamespace( - miner_id="miner-a", - share_id="miner-a.rig-a:" + "1" * 64, - share_difficulty=100, - accepted_at_ms=self.now_ms, - ), - SimpleNamespace( - miner_id="miner-a", - share_id="miner-a.rig-b:" + "2" * 64, - share_difficulty=80, - accepted_at_ms=self.now_ms, - ), - SimpleNamespace( - miner_id="miner-b", - share_id="miner-b.rig-z:" + "3" * 64, - share_difficulty=50, - accepted_at_ms=self.now_ms, - ), - ] - - def current_owed_balances(self) -> list[dict[str, object]]: - return [{"recipient_id": "miner-a", "balance_sats": 42}] - - def recipient_payout_history(self, *, recipient_id: str, limit: int = 50) -> list[dict[str, object]]: - if recipient_id != "miner-a": - return [] - return self._payout_history()[:limit] - - def dashboard_miner_lifetime_earnings_bits(self, *, recipient_id: str) -> int: - if recipient_id != "miner-a": - return 0 - total = 0 - for row in self._payout_history(): - fallback_gross = int(row.get("onchain_amount_sats", 0)) + int(row.get("carry_forward_balance_sats", 0)) - total += int(row.get("gross_amount_sats", fallback_gross)) - return total - - def dashboard_miner_pending_maturity_bits(self, *, recipient_id: str) -> int: - if recipient_id != "miner-a": - return 0 - return sum( - max(0, int(row.get("onchain_amount_sats", 0)) - int(row.get("settlement_fee_sats", 0))) - for row in self._payout_history() - if row["action"] == "onchain" and row["maturity_state"] == "immature" - ) - - def dashboard_miner_payout_rows(self, *, recipient_id: str, page: int, limit: int) -> dict[str, object]: - history = self.recipient_payout_history(recipient_id=recipient_id, limit=1_000) - rows = [public_api.miner_payout_row(row) for row in history] - offset = (page - 1) * limit - return {"pagination": public_api.pagination(page, limit, len(rows)), "rows": rows[offset : offset + limit]} - - def dashboard_miner_earning_rows(self, *, recipient_id: str, page: int, limit: int) -> dict[str, object]: - history = self.recipient_payout_history(recipient_id=recipient_id, limit=1_000) - rows = [public_api.miner_earning_row(row) for row in history] - offset = (page - 1) * limit - return {"pagination": public_api.pagination(page, limit, len(rows)), "rows": rows[offset : offset + limit]} - - def _payout_history(self) -> list[dict[str, object]]: - return [ - { - "block_hash": self.block_hash, - "block_height": 123450, - "coinbase_txid": "1" * 64, - "recipient_id": "miner-a", - "onchain_amount_sats": 40, - "settlement_fee_sats": 3, - "carry_forward_balance_sats": 2, - "block_gross_amount_sats": 84, - "action": "onchain", - "maturity_state": "immature", - "created_at": "2026-06-26 20:45:00+00", - "found_at": "2026-06-26 20:40:00+00", - }, - { - "block_hash": "2" * 64, - "block_height": 123449, - "coinbase_txid": "3" * 64, - "recipient_id": "miner-a", - "gross_amount_sats": 7, - "onchain_amount_sats": 0, - "settlement_fee_sats": 1, - "carry_forward_balance_sats": 7, - "block_gross_amount_sats": 28, - "action": "accrued", - "maturity_state": "immature", - "created_at": "2026-06-26 19:45:00+00", - "found_at": "2026-06-26 19:40:00+00", - }, - ] - - -class RangeBoundaryPublicLedger(FakePublicLedger): - """Serves one bucket just before the 1w range boundary and one inside it.""" - - def dashboard_hashrate_series( - self, - *, - subject_type: str, - subject_id: str | None, - range_id: str, - bucket: str, - lookback_seconds: int = 0, - range_anchor_epoch: int | None = None, - ) -> list[dict[str, object]]: - self.last_hashrate_series_lookback = lookback_seconds - self.last_hashrate_series_anchor = range_anchor_epoch - # Derive the boundary from the caller-provided anchor, exactly as the - # endpoint's min_epoch trim does, so this stub is deterministic. - anchor_epoch = range_anchor_epoch if range_anchor_epoch is not None else int(time.time()) - bucket_seconds = public_api.HASHRATE_SERIES_BUCKET_SECONDS[bucket] - cutoff_epoch = public_api.hashrate_series_min_epoch( - anchor_epoch, - 7 * 86400, - bucket_seconds, - ) - return [ - { - "timestamp": _bucket_timestamp(cutoff_epoch - bucket_seconds), - "hashrate_ths": public_api.hashrate_ths_from_difficulty( - 600, - bucket_seconds, - ), - "accepted_share_count": 1, - "accepted_share_difficulty": "600", - }, - { - "timestamp": _bucket_timestamp(cutoff_epoch + 4 * bucket_seconds), - "hashrate_ths": public_api.hashrate_ths_from_difficulty( - 1200, - bucket_seconds, - ), - "accepted_share_count": 1, - "accepted_share_difficulty": "1200", - }, - ] - - -class RangeBoundaryBlockMarkerLedger(FakePublicLedger): - """Serves one block just before the 1w marker cutoff and one just after. - - The records are derived from the caller-provided anchor with the same - hashrate_series_min_epoch arithmetic the read models apply, so the test - can pin the trim boundary deterministically: the pre-cutoff block sits in - the straddling bucket that must be dropped whole, and the post-cutoff - block sits in the first fully covered bucket. - """ - - def dashboard_block_markers( - self, - *, - range_id: str, - bucket: str, - range_anchor_epoch: int | None = None, - ) -> dict[str, object]: - anchor_epoch = range_anchor_epoch if range_anchor_epoch is not None else int(time.time()) - bucket_seconds = public_api.HASHRATE_SERIES_BUCKET_SECONDS[bucket] - cutoff_epoch = public_api.hashrate_series_min_epoch( - anchor_epoch, - 7 * 86400, - bucket_seconds, - ) - self.block_marker_records = [ - (100, "aa" * 32, cutoff_epoch - 1, "confirmed"), - (101, "bb" * 32, cutoff_epoch + 60, "confirmed"), - ] - return super().dashboard_block_markers( - range_id=range_id, - bucket=bucket, - range_anchor_epoch=range_anchor_epoch, - ) - - -class BrokenPublicLedger(FakePublicLedger): - def dashboard_leaderboard(self, *, page: int, limit: int, search: str | None = None) -> dict[str, object]: - self.leaderboard_calls += 1 - raise RuntimeError("database password=secret failed") - - -class MissingPendingReadModelLedger(FakePublicLedger): - dashboard_pending_fanout_rows = None - - -class MissingAuditBundleBodyLedger(FakePublicLedger): - def dashboard_public_artifact(self, *, sha256: str) -> dict[str, object] | None: - if sha256 == self.audit_bundle_sha256: - return None - return super().dashboard_public_artifact(sha256=sha256) - - -def sample_ctv_manifest_set() -> dict[str, object]: - # Mirrors the tests/test_prism_share_ledger.py fixture: the minimal - # manifest set that passes ctv_fanout_recovery_payload validation. - parent_coinbase_txid = "11" * 32 - precommitment = { - "chunk_index": 0, - "chunk_count": 1, - "block_height": 123450, - "settlement_mode": "ctv_fanout", - "fanout_tx_template_hex": "0300000001", - "fanout_output_sum_sats": 25_000, - "anchor_vout": 1, - "ctv_hash_hex": "33" * 32, - } - manifest = { - "schema": "qbit.prism.ctv-fanout-manifest.v1", - "precommitment": precommitment, - "precommitment_sha256_hex": "44" * 32, - "commitment_witness_leaf_hex": "55" * 32, - "parent_coinbase_txid": parent_coinbase_txid, - "parent_coinbase_tx_hex": "0200000001", - "parent_coinbase_vout": 2, - "covenant_output_value_sats": 25_000, - "fanout_tx_hex": "0300000002", - "fanout_txid": "22" * 32, - } - return { - "schema": "qbit.prism.ctv-fanout-manifest-set.v1", - "block_height": 123450, - "settlement_mode": "ctv_fanout", - "parent_coinbase_txid": parent_coinbase_txid, - "fanout_count": 1, - "fanout_output_sum_sats": 25_000, - "covenant_output_value_sats": 25_000, - "manifests": [manifest], - } - - -class CanonicalArtifactPublicLedger(FakePublicLedger): - """Artifacts whose canonical text and advertised sha256 are genuine.""" - - def __init__(self) -> None: - super().__init__() - self.manifest_set = sample_ctv_manifest_set() - self.manifest = self.manifest_set["manifests"][0] # type: ignore[index] - self.manifest_sha256 = sha256_json_hex(self.manifest) - self.manifest_set_sha256 = sha256_json_hex(self.manifest_set) - self.audit_bundle = { - "schema": "qbit.prism.audit-bundle.v1", - "found_block": {"block_height": 123450}, - "accepted_shares": [{"share_seq": 1}], - } - # Audit bundles are hashed over these exact canonical bytes. Production - # persists the same sequence beside the compact external-body pointer. - self.audit_bundle_sha256 = hashlib.sha256( - json.dumps(self.audit_bundle, separators=(",", ":")).encode() - ).hexdigest() - - def dashboard_public_artifact(self, *, sha256: str) -> dict[str, object] | None: - if sha256 == self.audit_bundle_sha256: - return self.audit_bundle - if sha256 == self.manifest_set_sha256: - return self.manifest_set - if sha256 == self.manifest_sha256: - return self.manifest - return None - - def dashboard_public_artifact_document(self, *, sha256: str) -> dict[str, object] | None: - payload = self.dashboard_public_artifact(sha256=sha256) - if payload is None: - return None - canonical_json = None - if sha256 == self.audit_bundle_sha256: - canonical_json = json.dumps(self.audit_bundle, separators=(",", ":")) - if sha256 == self.manifest_set_sha256: - canonical_json = canonical_json_text(self.manifest_set) - if sha256 == self.manifest_sha256: - canonical_json = canonical_json_text(self.manifest) - return {"payload": payload, "canonical_json": canonical_json} - - -class TamperedCanonicalArtifactPublicLedger(CanonicalArtifactPublicLedger): - def dashboard_public_artifact_document(self, *, sha256: str) -> dict[str, object] | None: - document = super().dashboard_public_artifact_document(sha256=sha256) - if document is None or document.get("canonical_json") is None: - return document - # Same document, different bytes: must never be served raw. - return {**document, "canonical_json": str(document["canonical_json"]) + " "} - - -class MissingCanonicalArtifactPublicLedger(CanonicalArtifactPublicLedger): - def __init__(self) -> None: - super().__init__() - self.audit_document_calls = 0 - - def dashboard_public_artifact_document(self, *, sha256: str) -> dict[str, object] | None: - if sha256 != self.audit_bundle_sha256: - return super().dashboard_public_artifact_document(sha256=sha256) - self.audit_document_calls += 1 - return { - "payload": self.audit_bundle, - "canonical_json": None, - "canonical_fallback_reason": "missing", - } - - -class DirectCoinbasePublicLedger(FakePublicLedger): - block_hash = "00000000000026e9383a5aeae5fe5c3297f24884f29c4cf1585f71829491f0d9" - block_height = 23342 - audit_bundle_sha256 = "92adad1828dfe2b68deceddf1bfcd153e9a4fd3e9ec8516fd2c13c295129b49f" - payout_manifest_sha256 = "8c37b6b595e76f69c3f7c68a04c5acf34ca4ba10e508be6d56f8398d4a70ba9a" - - def audit_ctv_fanout_manifest_set(self, *, block_hash: str) -> dict[str, object] | None: - return None - - def audit_bundle(self, *, block_hash: str) -> dict[str, object] | None: - if block_hash != self.block_hash: - return None - return { - "block_hash": self.block_hash, - "block_height": self.block_height, - "payout_manifest_sha256": self.payout_manifest_sha256, - "audit_bundle_sha256": self.audit_bundle_sha256, - "audit_bundle": { - "schema": "qbit.prism.audit-bundle.v1", - "found_block": {"block_height": self.block_height}, - "settlement_mode_decision": {"mode": "direct_coinbase"}, - }, - } - - -class ArtifactExistsDirectCoinbasePublicLedger(DirectCoinbasePublicLedger): - def __init__(self) -> None: - self.exists_calls = 0 - self.artifact_calls = 0 - - def dashboard_public_artifact_exists(self, *, sha256: str) -> bool: - self.exists_calls += 1 - return sha256 == self.audit_bundle_sha256 - - def dashboard_public_artifact(self, *, sha256: str) -> dict[str, object] | None: - self.artifact_calls += 1 - raise AssertionError("settlement link availability should use metadata-only exists hook") - - -class ExternalizedAuditBodyResolver(PsqlShareLedger): - def __init__(self, audit_body_dir: str | Path) -> None: - self._audit_body_dir = Path(audit_body_dir) - self.body_read_count = 0 - - def _read_external_body(self, body_uri: object, *, expected_sha256: object) -> dict[str, object] | None: - self.body_read_count += 1 - return super()._read_external_body(body_uri, expected_sha256=expected_sha256) - - -class ExternalizedDirectCoinbasePublicLedger(DirectCoinbasePublicLedger): - def __init__(self, audit_body_dir: str | Path) -> None: - bundle = { - "schema": "qbit.prism.audit-bundle.v1", - "found_block": {"block_height": self.block_height}, - "settlement_mode_decision": {"mode": "direct_coinbase"}, - } - body_bytes = json.dumps(bundle, separators=(",", ":")).encode() - self.audit_bundle_sha256 = hashlib.sha256(body_bytes).hexdigest() - self.body_uri = ( - Path(audit_body_dir) - / f"prism-audit-bundle-body-{self.block_hash}-{self.audit_bundle_sha256}.json" - ) - self.body_uri.write_bytes(body_bytes) - self.resolver = ExternalizedAuditBodyResolver(audit_body_dir) - - def audit_bundle(self, *, block_hash: str) -> dict[str, object] | None: - if block_hash != self.block_hash: - return None - return self.resolver._resolve_audit_bundle_row( - { - "block_hash": self.block_hash, - "block_height": self.block_height, - "payout_manifest_sha256": self.payout_manifest_sha256, - "audit_bundle_sha256": self.audit_bundle_sha256, - "audit_bundle": None, - "body_uri": str(self.body_uri), - } - ) - - -class ValueErrorPublicLedger(FakePublicLedger): - def dashboard_leaderboard(self, *, page: int, limit: int, search: str | None = None) -> dict[str, object]: - self.leaderboard_calls += 1 - raise ValueError("json parse password=secret failed") - - -class SlowPublicLedger(FakePublicLedger): - def __init__(self) -> None: - super().__init__() - self._leaderboard_lock = threading.Lock() - - def dashboard_leaderboard(self, *, page: int, limit: int, search: str | None = None) -> dict[str, object]: - with self._leaderboard_lock: - self.leaderboard_calls += 1 - time.sleep(0.2) - rows = [ - { - "rank": 1, - "recipient_id": "miner-a", - "display_name": None, - "hashrate_ths_3h": "2.5", - "share_percent": "60", - "hash_percent": "60", - "blocks_found": 1, - "last_share_at": "2026-06-26T20:44:53Z", - } - ] - return { - "started_at": "2026-06-26T17:45:00Z", - "ended_at": "2026-06-26T20:45:00Z", - "totals": { - "pool_hashrate_ths": "2.5", - "pool_accepted_share_difficulty": "100", - "participant_count": 1, - }, - "pagination": {"page": page, "limit": limit, "total_count": 1, "total_pages": 1}, - "rows": rows, - } - - -class StatementTimeoutRecordingLedger(FakePublicLedger): - """Duck-typed operation_timeout scope, recording what the handler arms. - - Mirrors PsqlShareLedger's contextmanager signature so the read service's - getattr + callable probe takes the same branch it takes in production. - The operation scope, not the statement scope: one budget must cover the - whole dispatch, however many ledger reads it performs. - """ - - def __init__(self) -> None: - super().__init__() - self.statement_timeout_budgets: list[float] = [] - - @contextmanager - def operation_timeout(self, timeout_seconds: float) -> Iterator[None]: - self.statement_timeout_budgets.append(timeout_seconds) - yield - - -class TimingOutPublicLedger(StatementTimeoutRecordingLedger): - """Every hashrate query dies the way a server-cancelled statement does.""" - - def dashboard_hashrate_series(self, **kwargs: object) -> list[dict[str, object]]: - self.hashrate_series_calls += 1 - raise LedgerOperationTimeout("postgres statement deadline expired") - - -class FakeCoordinator: - def __init__(self, ledger: FakePublicLedger | None = None, rpc: FakeRpc | None = None) -> None: - self.ledger = ledger or FakePublicLedger() - self.rpc = rpc or FakeRpc() - self.bind = "127.0.0.1" - self.port = 3340 - - -class MemoryCoordinator: - def __init__(self) -> None: - self.ledger = SingleWriterShareLedger() - self.rpc = FakeRpc() - - -class PrismPublicDashboardApiTests(unittest.TestCase): - def setUp(self) -> None: - handler = make_public_handler(FakeCoordinator()) # type: ignore[arg-type] - self.server = ThreadingHTTPServer(("127.0.0.1", 0), handler) - self.thread = threading.Thread(target=self.server.serve_forever, daemon=True) - self.thread.start() - self.base_url = f"http://127.0.0.1:{self.server.server_port}" - - def tearDown(self) -> None: - self.server.shutdown() - self.server.server_close() - self.thread.join(timeout=5) - - def get_json(self, path: str) -> dict[str, object]: - with urllib.request.urlopen(self.base_url + path, timeout=5) as response: - return json.loads(response.read()) - - def test_pool_summary_public_shape(self) -> None: - payload = self.get_json("/public/v1/pool-summary") - - self.assertEqual(payload["schema"], "prism.dashboard.pool-summary.v1") - self.assertEqual(payload["network"]["name"], "qbit-signet") - self.assertEqual(payload["network"]["bits"], "207fffff") - self.assertEqual(payload["pool"]["total_mined_bits"], 600) - self.assertEqual(payload["pool"]["latest_block"]["solver_recipient_id"], "miner-a") - - def test_pool_summary_derives_scaled_network_difficulty_from_bits(self) -> None: - # network_difficulty must be PRISM's scaled difficulty (derived from the - # compact bits), NOT the raw getblocktemplate/getblockchaininfo difficulty - # float, so the reward-window weight and ETA line up with scaled per-share - # difficulties. The raw "1.2" below must be ignored. This covers dashboard difficulty and reward-estimate regressions. - ledger = FakePublicLedger() - payload = public_api.pool_summary( - FakeCoordinator(ledger=ledger, rpc=FakeRpc(template_difficulty="1.2", template_bits="207fffff")) - ) - - scaled = str(public_api.scaled_network_difficulty("207fffff")) - self.assertEqual(scaled, "1000000") - self.assertEqual(payload["network"]["network_difficulty"], scaled) - self.assertEqual(ledger.current_network_difficulty, scaled) - self.assertEqual(payload["pool"]["reward_window"]["requested_window_weight"], "8000000") - - def test_pool_summary_expected_time_to_block_is_nonzero_for_real_difficulty(self) -> None: - # Regression: expected_time_to_block_seconds returned 0 because - # network_summary fed the raw (unscaled, ~1e6x too small) RPC difficulty into - # the scaled-units formula. Deriving the scaled difficulty from bits produces - # the real ETA (~tens of seconds for the fake 2.5 TH/s pool at this difficulty). - ledger = FakePublicLedger() - payload = public_api.pool_summary(FakeCoordinator(ledger=ledger, rpc=FakeRpc(template_bits="1b0404cb"))) - - scaled = public_api.scaled_network_difficulty("1b0404cb") - expected_eta = public_api.expected_time_to_block_seconds( - hashrate_ths="2.5", network_difficulty=str(scaled) - ) - self.assertEqual(payload["network"]["network_difficulty"], str(scaled)) - self.assertGreater(payload["pool"]["expected_time_to_block_seconds"], 0) - self.assertEqual(payload["pool"]["expected_time_to_block_seconds"], expected_eta) - - def test_network_summary_returns_503_without_authoritative_compact_bits(self) -> None: - with self.assertRaises(public_api.PublicApiError) as raised: - public_api.network_summary(FakeCoordinator(rpc=FakeRpc(template_bits=None))) - - self.assertEqual(raised.exception.status, 503) - self.assertEqual(raised.exception.code, "qbit_rpc_unavailable") - self.assertEqual(raised.exception.message, "qbit RPC did not provide valid compact bits") - - def test_network_summary_uses_valid_blockchain_bits_when_template_bits_are_missing_or_invalid(self) -> None: - for template_bits in (None, "not-bits"): - with self.subTest(template_bits=template_bits): - payload = public_api.network_summary( - FakeCoordinator( - rpc=FakeRpc( - template_bits=template_bits, - blockchain_info={"bits": "1b0404cb"}, - ) - ) - ) - - self.assertEqual(payload["bits"], "1b0404cb") - self.assertEqual( - payload["network_difficulty"], - str(public_api.scaled_network_difficulty("1b0404cb")), - ) - - def test_missing_compact_bits_fail_endpoints_before_reward_ledger_reads(self) -> None: - ledger = FakePublicLedger() - handler = make_public_handler( - FakeCoordinator(ledger=ledger, rpc=FakeRpc(template_bits=None)) - ) # type: ignore[arg-type] - server = ThreadingHTTPServer(("127.0.0.1", 0), handler) - thread = threading.Thread(target=server.serve_forever, daemon=True) - thread.start() - try: - for path in ( - "/public/v1/pool-summary", - "/public/v1/leaderboard?window=reward", - "/public/v1/miners/miner-a", - ): - with self.subTest(path=path): - with self.assertRaises(urllib.error.HTTPError) as raised: - urllib.request.urlopen( - f"http://127.0.0.1:{server.server_port}{path}", - timeout=5, - ) - self.assertEqual(raised.exception.code, 503) - payload = json.loads(raised.exception.read()) - raised.exception.close() - self.assertEqual(payload["error"]["code"], "upstream_unavailable") - finally: - server.shutdown() - server.server_close() - thread.join(timeout=5) - - self.assertEqual(ledger.pool_snapshot_calls, 0) - self.assertEqual(ledger.reward_leaderboard_calls, []) - self.assertIsNone(ledger.reward_window_network_difficulty) - - def test_network_summary_tolerates_null_or_invalid_rpc_counters(self) -> None: - payload = public_api.network_summary( - FakeCoordinator( - rpc=FakeRpc( - blockchain_info={"blocks": None, "headers": "not-a-height"}, - network_info={"connections": None}, - ) - ) - ) - - self.assertEqual(payload["height"], 0) - self.assertEqual(payload["peers"], 0) - - def test_blocks_leaderboard_and_hashrate_series(self) -> None: - blocks = self.get_json("/public/v1/blocks?page=1&limit=15") - leaderboard = self.get_json("/public/v1/leaderboard?window=3h&page=1&limit=15") - series = self.get_json("/public/v1/hashrate-series?subject=miner:miner-a&range=1w&bucket=auto") - - self.assertEqual(blocks["schema"], "prism.dashboard.blocks.v1") - self.assertEqual(blocks["rows"][0]["bits"], "207fffff") - self.assertIsNotNone(blocks["rows"][0]["network_difficulty"]) - self.assertEqual(leaderboard["schema"], "prism.dashboard.leaderboard.v1") - self.assertEqual(leaderboard["rows"][0]["recipient_id"], "miner-a") - self.assertEqual(series["schema"], "prism.dashboard.hashrate-series.v1") - self.assertEqual(series["subject"], {"type": "miner", "id": "miner-a"}) - self.assertEqual(series["bucket"], "1h") - # 1h buckets already exceed the default smoothing window, so ledger - # rates pass through untouched. - self.assertEqual(series["points"][0]["hashrate_ths"], "2.5") - - BLOCK_ROW_V1_KEYS = { - "height", - "hash", - "found_at", - "network_difficulty", - "bits", - "solver_recipient_id", - "solver_worker_name", - "solver_share_difficulty", - "reward_window_weight", - "coinbase_value_bits", - "audit_bundle_sha256", - "payout_manifest_sha256", - "explorer_url", - } - - def test_blocks_default_response_keeps_v1_shape_and_omits_state_fields(self) -> None: - ledger = FakePublicLedger() - coordinator = FakeCoordinator(ledger=ledger) - - for query in ({}, {"chain_state": ["active"]}): - with self.subTest(query=query): - status, payload = public_api.dispatch(coordinator, "/public/v1/blocks", query) - - self.assertEqual(status, 200) - self.assertEqual(payload["schema"], "prism.dashboard.blocks.v1") - for row in payload["rows"]: - self.assertEqual(set(row), self.BLOCK_ROW_V1_KEYS) - # Both the implicit default and an explicit chain_state=active reach - # the read model without any chain_state argument, so a ledger that - # predates the filter keeps serving the default contract. - self.assertEqual( - ledger.blocks_calls, - [ - {"page": 1, "limit": 15, "chain_state": None}, - {"page": 1, "limit": 15, "chain_state": None}, - ], - ) - - def test_blocks_chain_state_filter_returns_v2_rows_and_filtered_pagination(self) -> None: - ledger = FakePublicLedger() - coordinator = FakeCoordinator(ledger=ledger) - - status, all_payload = public_api.dispatch( - coordinator, "/public/v1/blocks", {"chain_state": ["all"]} - ) - self.assertEqual(status, 200) - self.assertEqual(all_payload["schema"], "prism.dashboard.blocks.v2") - self.assertEqual(all_payload["pagination"]["total_count"], 2) - for row in all_payload["rows"]: - self.assertEqual(set(row), self.BLOCK_ROW_V1_KEYS | {"chain_state", "disconnected_at"}) - self.assertEqual(all_payload["rows"][0]["chain_state"], "confirmed") - self.assertIsNone(all_payload["rows"][0]["disconnected_at"]) - self.assertEqual(all_payload["rows"][1]["chain_state"], "reversed") - self.assertEqual(all_payload["rows"][1]["disconnected_at"], "2026-06-26T19:05:00Z") - - status, reversed_payload = public_api.dispatch( - coordinator, "/public/v1/blocks", {"chain_state": ["reversed"]} - ) - self.assertEqual(status, 200) - self.assertEqual(reversed_payload["schema"], "prism.dashboard.blocks.v2") - self.assertEqual(reversed_payload["pagination"]["total_count"], 1) - self.assertEqual( - [row["chain_state"] for row in reversed_payload["rows"]], - ["reversed"], - ) - self.assertEqual( - ledger.blocks_calls, - [ - {"page": 1, "limit": 15, "chain_state": "all"}, - {"page": 1, "limit": 15, "chain_state": "reversed"}, - ], - ) - - def test_blocks_rejects_unknown_chain_state(self) -> None: - with self.assertRaises(urllib.error.HTTPError) as raised: - self.get_json("/public/v1/blocks?chain_state=orphaned") - - self.assertEqual(raised.exception.code, 400) - payload = json.loads(raised.exception.read()) - raised.exception.close() - self.assertEqual(payload["error"]["code"], "bad_request") - - def test_origin_cache_separates_blocks_chain_states(self) -> None: - ledger = FakePublicLedger() - handler = make_public_handler(FakeCoordinator(ledger=ledger)) # type: ignore[arg-type] - server = ThreadingHTTPServer(("127.0.0.1", 0), handler) - thread = threading.Thread(target=server.serve_forever, daemon=True) - thread.start() - try: - with patch.dict(os.environ, {"PRISM_PUBLIC_CACHE_TTL_SECONDS": "30"}, clear=True): - base = f"http://127.0.0.1:{server.server_port}/public/v1/blocks" - for url in (base, base, f"{base}?chain_state=all"): - with urllib.request.urlopen(url, timeout=5) as response: - payload = json.loads(response.read()) - with urllib.request.urlopen(base, timeout=5) as response: - default_payload = json.loads(response.read()) - finally: - server.shutdown() - server.server_close() - thread.join(timeout=5) - - # chain_state partitions the origin cache automatically because the - # cache key carries the sorted query string: the repeated default - # requests share one entry while chain_state=all computes its own. - self.assertEqual(payload["schema"], "prism.dashboard.blocks.v2") - self.assertEqual(default_payload["schema"], "prism.dashboard.blocks.v1") - self.assertEqual( - [call["chain_state"] for call in ledger.blocks_calls], - [None, "all"], - ) - - def test_pool_summary_reports_reorg_counts_and_network_hashrate(self) -> None: - rpc = FakeRpc(network_hashps=2_500_000_000_000_000_000) - payload = public_api.pool_summary(FakeCoordinator(rpc=rpc)) - - self.assertEqual(payload["pool"]["blocks_reversed_total"], 1) - self.assertEqual(payload["pool"]["blocks_inactive_total"], 2) - # 2.5e18 H/s reported by the node is 2,500,000 TH/s, and the lookup - # asks for the permissionless lane the pool's shares are credited - # against, over the RPC's own default 120-block window at the tip. - self.assertEqual(payload["network"]["hashrate_ths"], "2500000") - self.assertEqual(rpc.network_hashps_calls, [[120, -1, "permissionless"]]) - - def test_pool_summary_network_hashrate_is_null_when_rpc_fails(self) -> None: - rpc = FakeRpc(network_hashps=RuntimeError("getnetworkhashps unavailable")) - payload = public_api.pool_summary(FakeCoordinator(rpc=rpc)) - - # The field is nullable and non-fatal: pool-summary still answers 200 - # with everything else populated, and only compact-bits loss may 503. - self.assertIsNone(payload["network"]["hashrate_ths"]) - self.assertEqual(payload["schema"], "prism.dashboard.pool-summary.v1") - self.assertEqual(payload["pool"]["blocks_found_total"], 3) - - def test_network_hashrate_ths_rejects_non_numeric_estimates(self) -> None: - for network_hashps in (float("nan"), float("inf"), -1, True, "not-a-rate", None, {}): - with self.subTest(network_hashps=network_hashps): - coordinator = FakeCoordinator(rpc=FakeRpc(network_hashps=network_hashps)) - self.assertIsNone(public_api.network_hashrate_ths(coordinator)) - fractional = FakeCoordinator(rpc=FakeRpc(network_hashps=1_500_000_000_000.5)) - self.assertEqual(public_api.network_hashrate_ths(fractional), "1.5000000000005") - - def test_block_markers_shape_alignment_and_truncation(self) -> None: - markers = self.get_json("/public/v1/block-markers?range=1m&bucket=auto") - - self.assertEqual(markers["schema"], "prism.dashboard.block-markers.v1") - self.assertEqual(markers["range"], "1m") - # auto resolves exactly as it does for hashrate-series: 1h for 1m. - self.assertEqual(markers["bucket"], "1h") - self.assertEqual(markers["bucket_seconds"], 3600) - # The reversed block and the out-of-range ancient block are excluded - # from the total; empty buckets are omitted rather than zero-filled. - self.assertEqual(markers["total_blocks"], 5) - points = markers["points"] - self.assertEqual(len(points), 2) - epochs = [_epoch_from_timestamp(str(point["timestamp"])) for point in points] - self.assertEqual(epochs, sorted(epochs)) - for point, epoch in zip(points, epochs): - # Marker buckets sit on the hashrate chart's grid: the same floor - # arithmetic hashrate-series buckets use, so every timestamp is - # bucket-aligned and every block floors into its point's bucket. - self.assertEqual(epoch % 3600, 0) - for block in point["blocks"]: - found_epoch = _epoch_from_timestamp(str(block["found_at"])) - self.assertEqual(found_epoch // 3600 * 3600, epoch) - self.assertEqual(points[0]["block_count"], 1) - self.assertFalse(points[0]["truncated"]) - self.assertEqual(points[0]["blocks"][0]["height"], 67440) - # Four accepted blocks in the newer bucket: only the three most recent - # survive, ordered found_at DESC with height DESC breaking the tie. - self.assertEqual(points[1]["block_count"], 4) - self.assertTrue(points[1]["truncated"]) - self.assertEqual( - [block["height"] for block in points[1]["blocks"]], - [67444, 67443, 67442], - ) - found_ats = [ - _epoch_from_timestamp(str(block["found_at"])) - for block in points[1]["blocks"] - ] - self.assertEqual(found_ats, sorted(found_ats, reverse=True)) - self.assertNotIn("1f" * 32, json.dumps(markers)) - - def test_block_markers_all_range_reaches_unbounded_history(self) -> None: - markers = self.get_json("/public/v1/block-markers?range=all&bucket=1d") - - self.assertEqual(markers["bucket"], "1d") - self.assertEqual(markers["bucket_seconds"], 86400) - # `all` has no lower bound, so the ancient block joins the total. - self.assertEqual(markers["total_blocks"], 6) - points = markers["points"] - self.assertEqual( - sum(int(point["block_count"]) for point in points), - markers["total_blocks"], - ) - self.assertEqual(points[0]["blocks"][0]["hash"], "2a" * 32) - for point in points: - self.assertEqual(_epoch_from_timestamp(str(point["timestamp"])) % 86400, 0) - # An unbounded range carries no anchor: there is no lower bound for - # the ledger to derive from it. - ledger = FakePublicLedger() - ledger.last_block_markers_anchor = -1 - public_api.block_markers(FakeCoordinator(ledger=ledger), range_id="all", bucket="1d") - self.assertIsNone(ledger.last_block_markers_anchor) - - def test_block_markers_pass_range_anchor_and_trim_straddling_bucket(self) -> None: - ledger = RangeBoundaryBlockMarkerLedger() - handler = make_public_handler(FakeCoordinator(ledger=ledger)) # type: ignore[arg-type] - server = ThreadingHTTPServer(("127.0.0.1", 0), handler) - thread = threading.Thread(target=server.serve_forever, daemon=True) - thread.start() - try: - url = f"http://127.0.0.1:{server.server_port}/public/v1/block-markers?range=1w&bucket=5m" - with urllib.request.urlopen(url, timeout=5) as response: - markers = json.loads(response.read()) - finally: - server.shutdown() - server.server_close() - thread.join(timeout=5) - - # The endpoint anchors the range on its own clock and the ledger keeps - # only fully covered buckets, so the block in the straddling bucket is - # dropped whole while the first in-range bucket survives -- the same - # boundary the hashrate series trims against. - self.assertIsNotNone(ledger.last_block_markers_anchor) - self.assertEqual(markers["total_blocks"], 1) - self.assertEqual(len(markers["points"]), 1) - self.assertEqual(markers["points"][0]["blocks"][0]["hash"], "bb" * 32) - self.assertNotIn("aa" * 32, json.dumps(markers)) - cutoff_epoch = public_api.hashrate_series_min_epoch( - int(ledger.last_block_markers_anchor), - 7 * 86400, - 300, - ) - self.assertEqual( - _epoch_from_timestamp(str(markers["points"][0]["timestamp"])), - cutoff_epoch, - ) - - def test_block_markers_reject_invalid_range_bucket_and_clamp_combos(self) -> None: - for path in ( - "/public/v1/block-markers?range=2y", - "/public/v1/block-markers?range=1w&bucket=7m", - # Every clamped combination: finer buckets than the range's chart - # renders are rejected rather than served unboundedly long. - "/public/v1/block-markers?range=1m&bucket=5m", - "/public/v1/block-markers?range=6m&bucket=5m", - "/public/v1/block-markers?range=6m&bucket=1h", - "/public/v1/block-markers?range=all&bucket=5m", - "/public/v1/block-markers?range=all&bucket=1h", - ): - with self.subTest(path=path): - with self.assertRaises(urllib.error.HTTPError) as raised: - self.get_json(path) - self.assertEqual(raised.exception.code, 400) - payload = json.loads(raised.exception.read()) - raised.exception.close() - self.assertEqual(payload["schema"], "prism.dashboard.error.v1") - self.assertEqual(payload["error"]["code"], "bad_request") - - def assert_hashrate_ths(self, actual: object, difficulty: int, seconds: int) -> None: - # Hashrate strings are Decimal-context dependent (direct_stratum pins - # prec=40 in the importing thread while HTTP handler threads use the - # default 28), so compare values with a tight relative tolerance - # instead of exact strings. - expected = ( - Decimal(difficulty) - * public_api.HASHES_PER_QBIT_SCALED_DIFFICULTY - / Decimal(seconds) - / public_api.TERAHASH - ) - self.assertLess(abs(Decimal(str(actual)) - expected), expected * Decimal("1e-25")) - - def test_hashrate_series_smooths_5m_buckets_over_trailing_window(self) -> None: - series = self.get_json("/public/v1/hashrate-series?subject=pool&range=1w&bucket=5m") - - points = series["points"] - self.assertEqual(len(points), 2) - # Raw per-bucket fields are preserved; only the rate is windowed. - self.assertEqual([point["accepted_share_difficulty"] for point in points], ["600", "1200"]) - self.assertEqual([point["accepted_share_count"] for point in points], [2, 1]) - self.assert_hashrate_ths(points[0]["hashrate_ths"], 600, 1800) - # The second point's 30m window spans 20:00 and 20:10 with the empty - # 20:05 bucket counting as zero difficulty. - self.assert_hashrate_ths(points[1]["hashrate_ths"], 1800, 1800) - - def test_hashrate_series_smoothing_disabled_by_env(self) -> None: - with patch.dict(os.environ, {"PRISM_PUBLIC_HASHRATE_SMOOTHING_SECONDS": "0"}): - series = self.get_json("/public/v1/hashrate-series?subject=pool&range=1w&bucket=5m") - - points = series["points"] - self.assert_hashrate_ths(points[0]["hashrate_ths"], 600, 300) - self.assert_hashrate_ths(points[1]["hashrate_ths"], 1200, 300) - - def test_hashrate_series_fetches_lookback_and_trims_pre_range_points(self) -> None: - ledger = RangeBoundaryPublicLedger() - handler = make_public_handler(FakeCoordinator(ledger=ledger)) # type: ignore[arg-type] - server = ThreadingHTTPServer(("127.0.0.1", 0), handler) - thread = threading.Thread(target=server.serve_forever, daemon=True) - thread.start() - try: - url = f"http://127.0.0.1:{server.server_port}/public/v1/hashrate-series?subject=pool&range=1w&bucket=5m" - with urllib.request.urlopen(url, timeout=5) as response: - series = json.loads(response.read()) - finally: - server.shutdown() - server.server_close() - thread.join(timeout=5) - - # The ledger is asked for one extra smoothing window of history so the - # first in-range windows average over real pre-range buckets, anchored - # on the same epoch the endpoint trims against so database clock skew - # cannot shift the boundary. - self.assertEqual(ledger.last_hashrate_series_lookback, 1800) - self.assertIsNotNone(ledger.last_hashrate_series_anchor) - points = series["points"] - # The pre-range context bucket feeds the trailing window but is dropped - # from the response instead of charting an artificial ramp-up. - self.assertEqual(len(points), 1) - self.assertEqual(points[0]["accepted_share_difficulty"], "1200") - self.assert_hashrate_ths(points[0]["hashrate_ths"], 1800, 1800) - - def test_hashrate_series_min_epoch_trims_straddling_bucket(self) -> None: - # Range start aligned to the bucket grid: that bucket is fully in range - # and is the first one kept. - self.assertEqual(public_api.hashrate_series_min_epoch(1000, 700, 300), 300) - # Unaligned range start (350) falls inside bucket [300, 600): that - # straddling bucket would mix pre-range shares into its raw fields, so - # the first kept bucket is the next fully covered one. - self.assertEqual(public_api.hashrate_series_min_epoch(1050, 700, 300), 600) - - def test_smooth_hashrate_series_rolling_total_matches_naive_window_sums(self) -> None: - # The rolling int total must equal an independently computed sum over - # each point's trailing window, across gaps and window evictions. - base_epoch = 1_750_000_000 // 300 * 300 - series = [ - (base_epoch, 100), - (base_epoch + 300, 250), - (base_epoch + 900, 75), - (base_epoch + 1500, 500), - (base_epoch + 1800, 125), - (base_epoch + 3600, 900), - ] - points = [ - { - "timestamp": _bucket_timestamp(epoch), - "hashrate_ths": "0", - "accepted_share_count": 1, - "accepted_share_difficulty": str(difficulty), - } - for epoch, difficulty in series - ] - smoothed = public_api.smooth_hashrate_series_points( - points, bucket_seconds=300, window_seconds=1800 - ) - self.assertEqual(len(smoothed), len(series)) - for (epoch, _), point in zip(series, smoothed): - expected_total = sum(d for ep, d in series if epoch - 1800 < ep <= epoch) - self.assertEqual( - point["hashrate_ths"], - public_api.hashrate_ths_from_difficulty(expected_total, 1800), - ) - - def test_smooth_hashrate_series_points_drops_malformed_points(self) -> None: - # An unparseable point can be neither windowed nor range-checked, so it - # is dropped rather than passed through raw (which would leak pre-range - # lookback buckets past the min_epoch trim). - malformed = { - "timestamp": "not-a-timestamp", - "hashrate_ths": "2.5", - "accepted_share_count": 1, - "accepted_share_difficulty": "100", - } - valid = { - "timestamp": "2026-06-26T20:00:00Z", - "hashrate_ths": "2.5", - "accepted_share_count": 1, - "accepted_share_difficulty": "600", - } - smoothed = public_api.smooth_hashrate_series_points( - [malformed, valid], bucket_seconds=300, window_seconds=1800 - ) - self.assertEqual(len(smoothed), 1) - self.assertEqual(smoothed[0]["accepted_share_difficulty"], "600") - self.assert_hashrate_ths(smoothed[0]["hashrate_ths"], 600, 1800) - # With trimming active, a malformed pre-range context point stays out - # of the response. - cutoff_epoch = 1_750_000_000 // 300 * 300 - in_range = dict(valid, timestamp=_bucket_timestamp(cutoff_epoch)) - smoothed = public_api.smooth_hashrate_series_points( - [malformed, in_range], - bucket_seconds=300, - window_seconds=1800, - min_epoch=cutoff_epoch, - ) - self.assertEqual(len(smoothed), 1) - self.assertEqual(smoothed[0]["timestamp"], _bucket_timestamp(cutoff_epoch)) - - V1_SERIES_KEYS = {"schema", "generated_at", "subject", "range", "bucket", "unit", "points"} - V1_POINT_KEYS = {"timestamp", "hashrate_ths", "accepted_share_count", "accepted_share_difficulty"} - V2_POINT_KEYS = { - "timestamp", - "raw_hashrate_ths", - "smoothed_hashrate_ths", - "accepted_share_count", - "accepted_share_difficulty", - "complete", - } - - def test_hashrate_series_default_view_retains_exact_v1_shape(self) -> None: - fixture = _load_fixture("hashrate-series.json") - for suffix in ("",): - with urllib.request.urlopen( - f"{self.base_url}/public/v1/hashrate-series?subject=pool&range=1w&bucket=5m{suffix}", - timeout=5, - ) as response: - body = response.read() - series = json.loads(body) - - self.assertEqual(series["schema"], "prism.dashboard.hashrate-series.v1") - self.assertEqual(set(series), self.V1_SERIES_KEYS) - self.assertEqual(set(series), set(fixture)) - self.assertEqual(len(series["points"]), 2) - for point in series["points"]: - self.assertEqual(set(point), self.V1_POINT_KEYS) - self.assertEqual(set(point), set(fixture["points"][0])) - # The legacy body carries none of the dual-rate vocabulary. - for token in (b"raw_hashrate_ths", b"smoothed_hashrate_ths", b"bucket_seconds", b"rate_basis", b"smoothing", b"complete"): - self.assertNotIn(token, body) - # And still serves the trailing-window rate as hashrate_ths. - self.assert_hashrate_ths(series["points"][0]["hashrate_ths"], 600, 1800) - self.assert_hashrate_ths(series["points"][1]["hashrate_ths"], 1800, 1800) - - def test_hashrate_series_both_view_returns_raw_and_smoothed_rates(self) -> None: - fixture = _load_fixture("hashrate-series-dual-rate.json") - series = self.get_json("/public/v1/hashrate-series?subject=pool&range=1w&bucket=5m&view=both") - - self.assertEqual(series["schema"], "prism.dashboard.hashrate-series.v2") - self.assertEqual(set(series), set(fixture)) - self.assertEqual(series["subject"], {"type": "pool", "id": None}) - self.assertEqual(series["range"], "1w") - self.assertEqual(series["bucket"], "5m") - self.assertEqual(series["bucket_seconds"], 300) - self.assertEqual(series["unit"], "ths") - self.assertEqual(series["rate_basis"], "accepted_share_difficulty") - self.assertEqual(series["smoothing"], {"method": "trailing", "window_seconds": 1800}) - - points = series["points"] - self.assertEqual(len(points), 2) - for point in points: - self.assertEqual(set(point), self.V2_POINT_KEYS) - self.assertEqual(set(point), set(fixture["points"][0])) - self.assertEqual([point["accepted_share_difficulty"] for point in points], ["600", "1200"]) - self.assertEqual([point["accepted_share_count"] for point in points], [2, 1]) - # Raw divides the bucket's credit by the full 5m bucket; smoothed is the - # 30m trailing estimate v1 serves, so the two differ for the same point. - self.assert_hashrate_ths(points[0]["raw_hashrate_ths"], 600, 300) - self.assert_hashrate_ths(points[0]["smoothed_hashrate_ths"], 600, 1800) - self.assert_hashrate_ths(points[1]["raw_hashrate_ths"], 1200, 300) - self.assert_hashrate_ths(points[1]["smoothed_hashrate_ths"], 1800, 1800) - for point in points: - self.assertNotEqual(Decimal(point["raw_hashrate_ths"]), Decimal(point["smoothed_hashrate_ths"])) - # The older bucket ended before generated_at; the current one is still - # accumulating credit. - self.assertEqual([point["complete"] for point in points], [True, False]) - generated_at_epoch = int( - datetime.strptime(series["generated_at"], "%Y-%m-%dT%H:%M:%SZ").replace(tzinfo=timezone.utc).timestamp() - ) - for point in points: - point_epoch = int( - datetime.strptime(point["timestamp"], "%Y-%m-%dT%H:%M:%SZ").replace(tzinfo=timezone.utc).timestamp() - ) - self.assertEqual(point["complete"], generated_at_epoch >= point_epoch + 300) - - def test_hashrate_series_both_view_reports_no_smoothing_when_disabled(self) -> None: - with patch.dict(os.environ, {"PRISM_PUBLIC_HASHRATE_SMOOTHING_SECONDS": "0"}): - series = self.get_json("/public/v1/hashrate-series?subject=pool&range=1w&bucket=5m&view=both") - - self.assertEqual(series["schema"], "prism.dashboard.hashrate-series.v2") - self.assertEqual(series["smoothing"], {"method": "none", "window_seconds": 300}) - points = series["points"] - self.assertEqual(len(points), 2) - self.assert_hashrate_ths(points[0]["raw_hashrate_ths"], 600, 300) - self.assert_hashrate_ths(points[1]["raw_hashrate_ths"], 1200, 300) - for point in points: - self.assertEqual(point["smoothed_hashrate_ths"], point["raw_hashrate_ths"]) - - def test_hashrate_series_both_view_reports_no_smoothing_for_coarse_buckets(self) -> None: - # The default 30m window is shorter than two 1h buckets, so the v1 - # series passes ledger rates through; v2 says so explicitly. - ledger = RangeBoundaryPublicLedger() - series = public_api.hashrate_series( - FakeCoordinator(ledger=ledger), # type: ignore[arg-type] - subject="miner:miner-a", - range_id="1w", - bucket="auto", - view="both", - ) - - self.assertEqual(series["schema"], "prism.dashboard.hashrate-series.v2") - self.assertEqual(series["subject"], {"type": "miner", "id": "miner-a"}) - self.assertEqual(series["bucket"], "1h") - self.assertEqual(series["bucket_seconds"], 3600) - self.assertEqual(series["smoothing"], {"method": "none", "window_seconds": 3600}) - self.assertEqual(len(series["points"]), 1) - point = series["points"][0] - self.assertEqual(point["accepted_share_count"], 1) - self.assertEqual(point["accepted_share_difficulty"], "1200") - self.assert_hashrate_ths(point["raw_hashrate_ths"], 1200, 3600) - self.assertEqual(point["smoothed_hashrate_ths"], point["raw_hashrate_ths"]) - self.assertTrue(point["complete"]) - - def test_hashrate_series_both_view_trims_partial_leading_coarse_bucket(self) -> None: - ledger = RangeBoundaryPublicLedger() - series = public_api.hashrate_series( - FakeCoordinator(ledger=ledger), # type: ignore[arg-type] - subject="pool", - range_id="1w", - bucket="auto", - view="both", - ) - - self.assertEqual(series["smoothing"], {"method": "none", "window_seconds": 3600}) - self.assertEqual(ledger.last_hashrate_series_lookback, 0) - self.assertIsNotNone(ledger.last_hashrate_series_anchor) - self.assertEqual(len(series["points"]), 1) - self.assertEqual(series["points"][0]["accepted_share_difficulty"], "1200") - self.assertTrue(series["points"][0]["complete"]) - - def test_hashrate_series_both_view_trims_lookback_but_keeps_its_smoothing(self) -> None: - ledger = RangeBoundaryPublicLedger() - handler = make_public_handler(FakeCoordinator(ledger=ledger)) # type: ignore[arg-type] - server = ThreadingHTTPServer(("127.0.0.1", 0), handler) - thread = threading.Thread(target=server.serve_forever, daemon=True) - thread.start() - try: - url = ( - f"http://127.0.0.1:{server.server_port}/public/v1/hashrate-series" - "?subject=pool&range=1w&bucket=5m&view=both" - ) - with urllib.request.urlopen(url, timeout=5) as response: - series = json.loads(response.read()) - finally: - server.shutdown() - server.server_close() - thread.join(timeout=5) - - self.assertEqual(ledger.last_hashrate_series_lookback, 1800) - self.assertIsNotNone(ledger.last_hashrate_series_anchor) - points = series["points"] - # The pre-range bucket is trimmed from the response, but its credit - # still feeds the kept point's trailing window while the raw rate - # describes only the kept bucket. - self.assertEqual(len(points), 1) - self.assertEqual(points[0]["accepted_share_difficulty"], "1200") - self.assert_hashrate_ths(points[0]["raw_hashrate_ths"], 1200, 300) - self.assert_hashrate_ths(points[0]["smoothed_hashrate_ths"], 1800, 1800) - self.assertTrue(points[0]["complete"]) - - def test_hashrate_series_rejects_unknown_view(self) -> None: - for view in ("raw", "smoothed", "BOTH", "both,raw"): - with self.assertRaises(urllib.error.HTTPError) as raised: - self.get_json(f"/public/v1/hashrate-series?subject=pool&range=1w&bucket=5m&view={view}") - - self.assertEqual(raised.exception.code, 400, view) - payload = json.loads(raised.exception.read()) - raised.exception.close() - self.assertEqual(payload["schema"], "prism.dashboard.error.v1") - self.assertEqual( - payload["error"], - {"code": "bad_request", "message": "view must be both or omitted", "request_id": None}, - ) - - def test_origin_cache_separates_hashrate_series_views(self) -> None: - path = "/public/v1/hashrate-series" - base_query = {"subject": ["pool"], "range": ["1w"], "bucket": ["5m"]} - self.assertNotEqual( - public_api.public_cache_key(path, base_query), - public_api.public_cache_key(path, {**base_query, "view": ["both"]}), - ) - - ledger = FakePublicLedger() - handler = make_public_handler(FakeCoordinator(ledger=ledger)) # type: ignore[arg-type] - server = ThreadingHTTPServer(("127.0.0.1", 0), handler) - thread = threading.Thread(target=server.serve_forever, daemon=True) - thread.start() - try: - with patch.dict(os.environ, {"PRISM_PUBLIC_AGGREGATE_CACHE_TTL_SECONDS": "30"}, clear=True): - schemas = [] - for suffix in ("", "&view=both", ""): - with urllib.request.urlopen( - f"http://127.0.0.1:{server.server_port}{path}?subject=pool&range=1w&bucket=5m{suffix}", - timeout=5, - ) as response: - schemas.append(json.loads(response.read())["schema"]) - finally: - server.shutdown() - server.server_close() - thread.join(timeout=5) - - self.assertEqual( - schemas, - [ - "prism.dashboard.hashrate-series.v1", - "prism.dashboard.hashrate-series.v2", - "prism.dashboard.hashrate-series.v1", - ], - ) - # One origin computation per view; the repeated v1 request is a hit. - self.assertEqual(ledger.hashrate_series_calls, 2) - - def test_hashrate_series_smoothing_descriptor_matches_smoother_threshold(self) -> None: - self.assertEqual( - public_api.hashrate_series_smoothing(bucket_seconds=300, window_seconds=1800), - {"method": "trailing", "window_seconds": 1800}, - ) - # Partial buckets are dropped from the effective window. - self.assertEqual( - public_api.hashrate_series_smoothing(bucket_seconds=300, window_seconds=899), - {"method": "trailing", "window_seconds": 600}, - ) - for bucket_seconds, window_seconds in ((300, 0), (300, 599), (3600, 1800), (86400, 86400)): - self.assertEqual( - public_api.hashrate_series_smoothing(bucket_seconds=bucket_seconds, window_seconds=window_seconds), - {"method": "none", "window_seconds": bucket_seconds}, - (bucket_seconds, window_seconds), - ) - - def test_dual_rate_points_complete_flag_follows_bucket_end(self) -> None: - epoch = 1_750_000_000 // 300 * 300 - point = { - "timestamp": _bucket_timestamp(epoch), - "hashrate_ths": "0", - "accepted_share_count": 4, - "accepted_share_difficulty": "900", - } - for generated_at_epoch, complete in ((epoch, False), (epoch + 299, False), (epoch + 300, True), (epoch + 301, True)): - points = public_api.dual_rate_hashrate_series_points( - [point], - bucket_seconds=300, - window_seconds=1800, - generated_at_epoch=generated_at_epoch, - ) - self.assertEqual(len(points), 1) - self.assertEqual(points[0]["complete"], complete, generated_at_epoch - epoch) - self.assertEqual(points[0]["timestamp"], _bucket_timestamp(epoch)) - self.assertEqual(points[0]["accepted_share_count"], 4) - self.assertEqual(points[0]["accepted_share_difficulty"], "900") - self.assertEqual(points[0]["raw_hashrate_ths"], public_api.hashrate_ths_from_difficulty(900, 300)) - - def test_dual_rate_points_match_v1_smoothing_and_trim_lookback(self) -> None: - base_epoch = 1_750_000_000 // 300 * 300 - series = [ - (base_epoch, 100), - (base_epoch + 300, 250), - (base_epoch + 900, 75), - (base_epoch + 1500, 500), - (base_epoch + 1800, 125), - (base_epoch + 3600, 900), - ] - malformed = { - "timestamp": "not-a-timestamp", - "hashrate_ths": "2.5", - "accepted_share_count": 1, - "accepted_share_difficulty": "100", - } - points = [ - { - "timestamp": _bucket_timestamp(epoch), - "hashrate_ths": "0", - "accepted_share_count": 1, - "accepted_share_difficulty": str(difficulty), - } - for epoch, difficulty in series - ] - generated_at_epoch = base_epoch + 3600 + 150 - smoothed = public_api.smooth_hashrate_series_points( - list(points), bucket_seconds=300, window_seconds=1800 - ) - dual_rate = public_api.dual_rate_hashrate_series_points( - [malformed, *reversed(points)], - bucket_seconds=300, - window_seconds=1800, - generated_at_epoch=generated_at_epoch, - ) - self.assertEqual(len(dual_rate), len(series)) - for (epoch, difficulty), v1_point, v2_point in zip(series, smoothed, dual_rate): - self.assertEqual(v2_point["timestamp"], _bucket_timestamp(epoch)) - self.assertEqual(v2_point["smoothed_hashrate_ths"], v1_point["hashrate_ths"]) - self.assertEqual(v2_point["raw_hashrate_ths"], public_api.hashrate_ths_from_difficulty(difficulty, 300)) - self.assertEqual(v2_point["complete"], epoch != base_epoch + 3600) - - # Trimming keeps the lookback credit inside the first kept window. - trimmed = public_api.dual_rate_hashrate_series_points( - points, - bucket_seconds=300, - window_seconds=1800, - min_epoch=base_epoch + 900, - generated_at_epoch=generated_at_epoch, - ) - self.assertEqual([point["timestamp"] for point in trimmed], [_bucket_timestamp(epoch) for epoch, _ in series[2:]]) - self.assertEqual(trimmed[0]["raw_hashrate_ths"], public_api.hashrate_ths_from_difficulty(75, 300)) - self.assertEqual(trimmed[0]["smoothed_hashrate_ths"], public_api.hashrate_ths_from_difficulty(425, 1800)) - - # Below the two-bucket threshold the smoothed rate is the raw rate. - unsmoothed = public_api.dual_rate_hashrate_series_points( - points, - bucket_seconds=300, - window_seconds=300, - generated_at_epoch=generated_at_epoch, - ) - for point in unsmoothed: - self.assertEqual(point["smoothed_hashrate_ths"], point["raw_hashrate_ths"]) - - count_as_decimal = public_api.dual_rate_hashrate_series_points( - [{**points[0], "accepted_share_count": "3.0"}], - bucket_seconds=300, - window_seconds=1800, - generated_at_epoch=generated_at_epoch, - ) - self.assertEqual(count_as_decimal[0]["accepted_share_count"], 3) - - def test_omitted_leaderboard_window_retains_exact_legacy_v1_keys(self) -> None: - payload = self.get_json("/public/v1/leaderboard") - - self.assertEqual(payload["schema"], "prism.dashboard.leaderboard.v1") - self.assertEqual(set(payload), {"schema", "generated_at", "window", "totals", "pagination", "rows"}) - self.assertEqual(set(payload["window"]), {"id", "started_at", "ended_at"}) - self.assertEqual(payload["window"]["id"], "3h") - self.assertEqual( - set(payload["totals"]), - {"pool_hashrate_ths", "pool_accepted_share_difficulty", "participant_count"}, - ) - self.assertEqual( - set(payload["rows"][0]), - { - "rank", - "recipient_id", - "display_name", - "hashrate_ths_3h", - "share_percent", - "hash_percent", - "blocks_found", - "last_share_at", - }, - ) - - def test_reward_leaderboard_uses_v2_work_window_contract(self) -> None: - payload = self.get_json("/public/v1/leaderboard?window=reward&page=1&limit=15") - - self.assertEqual(payload["schema"], "prism.dashboard.leaderboard.v2") - self.assertEqual(payload["generated_at"], payload["window"]["ended_at"]) - self.assertEqual(payload["window"]["id"], "reward") - self.assertEqual(payload["window"]["window_multiplier"], 8) - self.assertTrue(payload["window"]["is_complete"]) - self.assertNotIn("basis", payload["window"]) - self.assertNotIn("newest_share_accepted_at", payload["window"]) - self.assertEqual( - Decimal(payload["window"]["counted_window_weight"]), - Decimal(payload["totals"]["pool_counted_share_difficulty"]), - ) - self.assertEqual([row["rank"] for row in payload["rows"]], [1, 2]) - - def test_reward_leaderboard_exact_lookup_preserves_global_rank_and_totals(self) -> None: - ledger = FakePublicLedger() - coordinator = FakeCoordinator(ledger=ledger) - - status, payload = public_api.dispatch( - coordinator, - "/public/v1/leaderboard", - {"window": ["reward"], "recipient_id": ["miner-b"]}, - ) - - self.assertEqual(status, 200) - self.assertEqual(payload["rows"][0]["rank"], 2) - self.assertEqual(payload["rows"][0]["recipient_id"], "miner-b") - self.assertEqual(payload["totals"]["participant_count"], 2) - self.assertEqual(payload["pagination"]["total_count"], 1) - self.assertEqual(ledger.reward_leaderboard_calls[-1]["recipient_id"], "miner-b") - self.assertEqual(ledger.reward_leaderboard_network_difficulty, "1000000") - - def test_reward_leaderboard_rejects_search_with_exact_recipient(self) -> None: - with self.assertRaises(urllib.error.HTTPError) as raised: - self.get_json("/public/v1/leaderboard?window=reward&search=miner&recipient_id=miner-a") - - self.assertEqual(raised.exception.code, 400) - payload = json.loads(raised.exception.read()) - raised.exception.close() - self.assertEqual(payload["error"]["message"], "search and recipient_id are mutually exclusive") - - def test_legacy_leaderboard_rejects_reward_recipient_filter(self) -> None: - for path in ( - "/public/v1/leaderboard?recipient_id=miner-a", - "/public/v1/leaderboard?window=3h&recipient_id=miner-a", - ): - with self.subTest(path=path): - with self.assertRaises(urllib.error.HTTPError) as raised: - self.get_json(path) - - self.assertEqual(raised.exception.code, 400) - payload = json.loads(raised.exception.read()) - raised.exception.close() - self.assertEqual(payload["error"]["message"], "recipient_id requires window=reward") - - def test_origin_cache_separates_legacy_and_reward_leaderboards(self) -> None: - ledger = FakePublicLedger() - handler = make_public_handler(FakeCoordinator(ledger=ledger)) # type: ignore[arg-type] - server = ThreadingHTTPServer(("127.0.0.1", 0), handler) - thread = threading.Thread(target=server.serve_forever, daemon=True) - thread.start() - try: - with patch.dict(os.environ, {"PRISM_PUBLIC_CACHE_TTL_SECONDS": "30"}, clear=True): - with urllib.request.urlopen( - f"http://127.0.0.1:{server.server_port}/public/v1/leaderboard?window=3h", - timeout=5, - ) as legacy_response: - legacy = json.loads(legacy_response.read()) - with urllib.request.urlopen( - f"http://127.0.0.1:{server.server_port}/public/v1/leaderboard?window=reward", - timeout=5, - ) as reward_response: - reward = json.loads(reward_response.read()) - finally: - server.shutdown() - server.server_close() - thread.join(timeout=5) - - self.assertEqual(legacy["schema"], "prism.dashboard.leaderboard.v1") - self.assertEqual(reward["schema"], "prism.dashboard.leaderboard.v2") - self.assertEqual(ledger.leaderboard_calls, 1) - self.assertEqual(len(ledger.reward_leaderboard_calls), 1) - - def test_public_cache_policy_gives_aggregate_endpoints_longer_default_ttl(self) -> None: - with patch.dict(os.environ, {}, clear=True): - for path in ( - "/public/v1/pool-summary", - "/public/v1/hashrate-series", - "/public/v1/miners/miner-a/workers", - ): - with self.subTest(path=path): - policy = public_api.public_cache_policy(path) - self.assertEqual(policy.ttl_seconds, 30) - self.assertEqual(policy.stale_while_revalidate_seconds, 30) - for path in ( - "/public/v1/miners/miner-a", - "/public/v1/miners/workers", - "/public/v1/miners/miner-a/earnings", - "/public/v1/leaderboard", - "/public/v1/blocks", - ): - with self.subTest(path=path): - self.assertEqual(public_api.public_cache_policy(path).ttl_seconds, 5) - - def test_public_cache_policy_aggregate_ttl_honors_env_and_kill_switch(self) -> None: - with patch.dict( - os.environ, - { - "PRISM_PUBLIC_AGGREGATE_CACHE_TTL_SECONDS": "45", - "PRISM_PUBLIC_AGGREGATE_CACHE_STALE_WHILE_REVALIDATE_SECONDS": "90", - "PRISM_PUBLIC_CACHE_TTL_SECONDS": "7", - }, - clear=True, - ): - policy = public_api.public_cache_policy("/public/v1/pool-summary") - self.assertEqual(policy.ttl_seconds, 45) - self.assertEqual(policy.stale_while_revalidate_seconds, 90) - self.assertEqual(public_api.public_cache_policy("/public/v1/blocks").ttl_seconds, 7) - with patch.dict(os.environ, {"PRISM_PUBLIC_CACHE_ENABLED": "0"}, clear=True): - disabled = public_api.public_cache_policy("/public/v1/pool-summary") - self.assertEqual(disabled.ttl_seconds, 0) - self.assertEqual(disabled.stale_while_revalidate_seconds, 0) - - def test_public_api_successes_emit_cache_headers_and_hit_origin_cache(self) -> None: - ledger = FakePublicLedger() - handler = make_public_handler(FakeCoordinator(ledger=ledger)) # type: ignore[arg-type] - server = ThreadingHTTPServer(("127.0.0.1", 0), handler) - thread = threading.Thread(target=server.serve_forever, daemon=True) - thread.start() - try: - with patch.dict( - os.environ, - { - "PRISM_PUBLIC_CACHE_TTL_SECONDS": "30", - "PRISM_PUBLIC_CACHE_DEBUG_HEADERS": "1", - }, - clear=True, - ): - first_url = f"http://127.0.0.1:{server.server_port}/public/v1/leaderboard?window=3h&page=1&limit=15" - second_url = f"http://127.0.0.1:{server.server_port}/public/v1/leaderboard?limit=15&page=1&window=3h" - with urllib.request.urlopen(first_url, timeout=5) as first: - first_payload = json.loads(first.read()) - first_cache = first.headers.get("X-Prism-Public-Cache") - first_age = first.headers.get("Age") - first_cdn_cache = first.headers.get("CDN-Cache-Control") - first_browser_cache = first.headers.get("Cache-Control") - with urllib.request.urlopen(second_url, timeout=5) as second: - second_payload = json.loads(second.read()) - second_cache = second.headers.get("X-Prism-Public-Cache") - second_age = second.headers.get("Age") - second_cdn_cache = second.headers.get("CDN-Cache-Control") - finally: - server.shutdown() - server.server_close() - thread.join(timeout=5) - - self.assertEqual(first_payload, second_payload) - self.assertEqual(first_cache, "miss") - self.assertEqual(second_cache, "hit") - self.assertEqual(first_age, "0") - self.assertEqual(second_age, "0") - self.assertEqual(first_browser_cache, "public, max-age=0, must-revalidate") - self.assertEqual(first_cdn_cache, "public, max-age=15") - self.assertEqual(second_cdn_cache, first_cdn_cache) - self.assertEqual(ledger.leaderboard_calls, 1) - - def test_public_api_errors_are_no_store_and_not_cached(self) -> None: - ledger = BrokenPublicLedger() - handler = make_public_handler(FakeCoordinator(ledger)) # type: ignore[arg-type] - server = ThreadingHTTPServer(("127.0.0.1", 0), handler) - thread = threading.Thread(target=server.serve_forever, daemon=True) - thread.start() - try: - url = f"http://127.0.0.1:{server.server_port}/public/v1/leaderboard" - for _ in range(2): - with self.assertRaises(urllib.error.HTTPError) as raised: - urllib.request.urlopen(url, timeout=5) - self.assertEqual(raised.exception.code, 500) - self.assertEqual(raised.exception.headers.get("Cache-Control"), "no-store") - self.assertIsNone(raised.exception.headers.get("CDN-Cache-Control")) - raised.exception.close() - finally: - server.shutdown() - server.server_close() - thread.join(timeout=5) - - self.assertEqual(ledger.leaderboard_calls, 2) - - def test_public_api_coalesces_concurrent_origin_cache_misses(self) -> None: - ledger = SlowPublicLedger() - handler = make_public_handler(FakeCoordinator(ledger)) # type: ignore[arg-type] - server = ThreadingHTTPServer(("127.0.0.1", 0), handler) - thread = threading.Thread(target=server.serve_forever, daemon=True) - thread.start() - errors: list[BaseException] = [] - payloads: list[dict[str, object]] = [] - start_barrier = threading.Barrier(6) - - def fetch() -> None: - try: - start_barrier.wait(timeout=5) - with urllib.request.urlopen( - f"http://127.0.0.1:{server.server_port}/public/v1/leaderboard", - timeout=5, - ) as response: - payloads.append(json.loads(response.read())) - except BaseException as exc: # pragma: no cover - re-raised below for clear failure - errors.append(exc) - - workers = [threading.Thread(target=fetch) for _ in range(5)] - try: - with patch.dict(os.environ, {"PRISM_PUBLIC_CACHE_TTL_SECONDS": "30"}, clear=True): - for worker in workers: - worker.start() - start_barrier.wait(timeout=5) - for worker in workers: - worker.join(timeout=10) - finally: - server.shutdown() - server.server_close() - thread.join(timeout=5) - - self.assertEqual(errors, []) - self.assertEqual(len(payloads), 5) - self.assertEqual({payload["schema"] for payload in payloads}, {"prism.dashboard.leaderboard.v1"}) - self.assertEqual(ledger.leaderboard_calls, 1) - - def test_public_api_coalesces_concurrent_misses_when_response_not_cacheable(self) -> None: - # An oversize (non-cacheable) response must still coalesce concurrent - # waiters onto the owner's single origin call instead of re-running it. - ledger = SlowPublicLedger() - handler = make_public_handler(FakeCoordinator(ledger)) # type: ignore[arg-type] - server = ThreadingHTTPServer(("127.0.0.1", 0), handler) - thread = threading.Thread(target=server.serve_forever, daemon=True) - thread.start() - errors: list[BaseException] = [] - payloads: list[dict[str, object]] = [] - start_barrier = threading.Barrier(6) - - def fetch() -> None: - try: - start_barrier.wait(timeout=5) - with urllib.request.urlopen( - f"http://127.0.0.1:{server.server_port}/public/v1/leaderboard", - timeout=5, - ) as response: - payloads.append(json.loads(response.read())) - except BaseException as exc: # pragma: no cover - re-raised below for clear failure - errors.append(exc) - - workers = [threading.Thread(target=fetch) for _ in range(5)] - try: - with patch.dict( - os.environ, - { - "PRISM_PUBLIC_CACHE_TTL_SECONDS": "30", - "PRISM_PUBLIC_CACHE_MAX_RESPONSE_BYTES": "1", - }, - clear=True, - ): - for worker in workers: - worker.start() - start_barrier.wait(timeout=5) - for worker in workers: - worker.join(timeout=10) - finally: - server.shutdown() - server.server_close() - thread.join(timeout=5) - - self.assertEqual(errors, []) - self.assertEqual(len(payloads), 5) - self.assertEqual({payload["schema"] for payload in payloads}, {"prism.dashboard.leaderboard.v1"}) - self.assertEqual(ledger.leaderboard_calls, 1) - - def test_public_cache_key_canonicalizes_hash_casing(self) -> None: - hex_hash = "ab" * 32 - upper_hash = hex_hash.upper() - - artifact_lower = public_api.public_cache_key(f"/public/v1/artifacts/{hex_hash}", {}) - artifact_upper = public_api.public_cache_key(f"/public/v1/artifacts/{upper_hash}", {}) - self.assertEqual(artifact_lower, artifact_upper) - - block_lower = public_api.public_cache_key( - f"/public/v1/blocks/{hex_hash}/settlement-artifacts", {} - ) - block_upper = public_api.public_cache_key( - f"/public/v1/blocks/{upper_hash}/settlement-artifacts", {} - ) - self.assertEqual(block_lower, block_upper) - - fanout_lower = public_api.public_cache_key(f"/public/v1/fanouts/{hex_hash}", {}) - fanout_upper = public_api.public_cache_key(f"/public/v1/fanouts/{upper_hash}", {}) - self.assertEqual(fanout_lower, fanout_upper) - - # Case-sensitive identity routes and the literal pending route are untouched. - self.assertNotEqual( - public_api.public_cache_key("/public/v1/miners/AbC", {}), - public_api.public_cache_key("/public/v1/miners/abc", {}), - ) - self.assertEqual( - public_api.public_cache_key("/public/v1/fanouts/pending", {})[0], - "/public/v1/fanouts/pending", - ) - # A non-hash fanout segment (a case-variant of the literal pending route, - # or any id dispatch would reject) must NOT fold onto the pending list's - # cache key, or a cached pending response could be served for it. - self.assertNotEqual( - public_api.public_cache_key("/public/v1/fanouts/PENDING", {}), - public_api.public_cache_key("/public/v1/fanouts/pending", {}), - ) - self.assertEqual( - public_api.public_cache_key("/public/v1/fanouts/not-a-hash", {})[0], - "/public/v1/fanouts/not-a-hash", - ) - - def test_public_cache_reaps_expired_before_evicting_fresh(self) -> None: - with patch.dict(os.environ, {"PRISM_PUBLIC_CACHE_MAX_ENTRIES": "2"}, clear=True): - cache = public_api.PublicResponseCache() - cache.get_or_compute(key=("/fresh", ()), ttl_seconds=60, compute=lambda: (200, {"v": "fresh"})) - cache.get_or_compute(key=("/expiring", ()), ttl_seconds=1, compute=lambda: (200, {"v": "expiring"})) - time.sleep(1.2) # /expiring is now expired; /fresh is still valid - # Storing a third fresh entry over the 2-entry bound must reap the - # expired /expiring, not evict the still-fresh (but oldest) /fresh. - cache.get_or_compute(key=("/new", ()), ttl_seconds=60, compute=lambda: (200, {"v": "new"})) - - recompute_calls: list[int] = [] - - def recompute() -> tuple[int, object]: - recompute_calls.append(1) - return 200, {"v": "recomputed"} - - status, payload, cache_state, _age = cache.get_or_compute( - key=("/fresh", ()), ttl_seconds=60, compute=recompute - ) - self.assertEqual(cache_state, "HIT") - self.assertEqual(payload, {"v": "fresh"}) - self.assertEqual(recompute_calls, []) - - def test_public_api_cache_headers_use_route_specific_ttls(self) -> None: - ledger = FakePublicLedger() - handler = make_public_handler(FakeCoordinator(ledger=ledger)) # type: ignore[arg-type] - server = ThreadingHTTPServer(("127.0.0.1", 0), handler) - thread = threading.Thread(target=server.serve_forever, daemon=True) - thread.start() - try: - with patch.dict( - os.environ, - { - "PRISM_PUBLIC_CONFIG_CACHE_TTL_SECONDS": "600", - "PRISM_PUBLIC_ARTIFACT_CACHE_TTL_SECONDS": "7200", - }, - clear=True, - ): - with urllib.request.urlopen( - f"http://127.0.0.1:{server.server_port}/public/v1/mining-configuration", - timeout=5, - ) as config_response: - json.loads(config_response.read()) - config_cache = config_response.headers.get("CDN-Cache-Control") - with urllib.request.urlopen( - f"http://127.0.0.1:{server.server_port}/public/v1/artifacts/{ledger.audit_bundle_sha256}", - timeout=5, - ) as artifact_response: - json.loads(artifact_response.read()) - artifact_cache = artifact_response.headers.get("CDN-Cache-Control") - finally: - server.shutdown() - server.server_close() - thread.join(timeout=5) - - self.assertEqual(config_cache, "public, max-age=600, stale-while-revalidate=300") - self.assertEqual(artifact_cache, "public, max-age=7200, stale-while-revalidate=86400, immutable") - - def test_public_api_errors_use_public_error_schema(self) -> None: - with self.assertRaises(urllib.error.HTTPError) as raised: - self.get_json("/public/v1/leaderboard?window=24h") - - self.assertEqual(raised.exception.code, 400) - payload = json.loads(raised.exception.read()) - raised.exception.close() - self.assertEqual(set(payload.keys()), {"schema", "error"}) - self.assertEqual(payload["schema"], "prism.dashboard.error.v1") - self.assertEqual( - payload["error"], - {"code": "bad_request", "message": "window must be one of 3h, reward", "request_id": None}, - ) - - with self.assertRaises(urllib.error.HTTPError) as raised: - self.get_json(f"/public/v1/leaderboard?search={'x' * 129}") - - self.assertEqual(raised.exception.code, 400) - payload = json.loads(raised.exception.read()) - raised.exception.close() - self.assertEqual(payload["error"], {"code": "bad_request", "message": "search must be 128 characters or fewer", "request_id": None}) - - with self.assertRaises(urllib.error.HTTPError) as raised: - self.get_json("/public/v1/hashrate-series?range=window") - - self.assertEqual(raised.exception.code, 400) - payload = json.loads(raised.exception.read()) - raised.exception.close() - self.assertEqual(payload["error"], {"code": "bad_request", "message": "range must be one of 1w, 1m, 6m, all", "request_id": None}) - - with self.assertRaises(urllib.error.HTTPError) as raised: - self.get_json("/public/v1") - - self.assertEqual(raised.exception.code, 404) - payload = json.loads(raised.exception.read()) - raised.exception.close() - self.assertEqual(payload["schema"], "prism.dashboard.error.v1") - self.assertEqual(payload["error"]["code"], "not_found") - - def test_hashrate_from_difficulty_uses_qbit_scaled_pow_limit_units(self) -> None: - live_bucket_difficulty = Decimal("8231979598912131605603") - - hashrate_ths = Decimal(public_api.hashrate_ths_from_difficulty(live_bucket_difficulty, 60 * 60)) - - self.assertGreater(hashrate_ths, Decimal("4.57")) - self.assertLess(hashrate_ths, Decimal("4.58")) - - def test_expected_time_to_block_uses_qbit_scaled_network_difficulty(self) -> None: - self.assertEqual( - public_api.expected_time_to_block_seconds( - hashrate_ths="0.000000000001", - network_difficulty="1000000", - ), - 2, - ) - - def test_expected_time_to_block_matches_hashrate_window_when_difficulty_matches(self) -> None: - network_difficulty = Decimal("4000000") - window_seconds = 3 * 60 * 60 - hashrate_ths = public_api.hashrate_ths_from_difficulty(network_difficulty, window_seconds) - - self.assertEqual( - public_api.expected_time_to_block_seconds( - hashrate_ths=hashrate_ths, - network_difficulty=str(network_difficulty), - ), - window_seconds, - ) - - def test_scaled_network_difficulty_uses_qbit_pow_limit_units(self) -> None: - self.assertEqual(public_api.scaled_network_difficulty("207fffff"), 1_000_000) - - def test_scaled_network_difficulty_rejects_non_positive_targets(self) -> None: - with self.assertRaises(ValueError): - public_api.scaled_network_difficulty("00000000") - - def test_unexpected_public_api_errors_do_not_leak_internal_details(self) -> None: - handler = make_public_handler(FakeCoordinator(BrokenPublicLedger())) # type: ignore[arg-type] - server = ThreadingHTTPServer(("127.0.0.1", 0), handler) - thread = threading.Thread(target=server.serve_forever, daemon=True) - thread.start() - try: - with self.assertRaises(urllib.error.HTTPError) as raised: - urllib.request.urlopen( - f"http://127.0.0.1:{server.server_port}/public/v1/leaderboard", - timeout=5, - ) - - self.assertEqual(raised.exception.code, 500) - raw_payload = raised.exception.read().decode() - raised.exception.close() - finally: - server.shutdown() - server.server_close() - thread.join(timeout=5) - - self.assertNotIn("password=secret", raw_payload) - payload = json.loads(raw_payload) - self.assertEqual(payload["schema"], "prism.dashboard.error.v1") - self.assertEqual(payload["error"], {"code": "internal_error", "message": "internal server error", "request_id": None}) - - def test_public_value_errors_do_not_leak_internal_details(self) -> None: - handler = make_public_handler(FakeCoordinator(ValueErrorPublicLedger())) # type: ignore[arg-type] - server = ThreadingHTTPServer(("127.0.0.1", 0), handler) - thread = threading.Thread(target=server.serve_forever, daemon=True) - thread.start() - try: - with self.assertRaises(urllib.error.HTTPError) as raised: - urllib.request.urlopen( - f"http://127.0.0.1:{server.server_port}/public/v1/leaderboard", - timeout=5, - ) - - self.assertEqual(raised.exception.code, 500) - raw_payload = raised.exception.read().decode() - raised.exception.close() - finally: - server.shutdown() - server.server_close() - thread.join(timeout=5) - - self.assertNotIn("password=secret", raw_payload) - payload = json.loads(raw_payload) - self.assertEqual(payload["schema"], "prism.dashboard.error.v1") - self.assertEqual(payload["error"], {"code": "internal_error", "message": "internal server error", "request_id": None}) - - def test_settlement_fanout_and_artifact_endpoints(self) -> None: - ledger = FakePublicLedger() - settlement = self.get_json(f"/public/v1/blocks/{ledger.block_hash}/settlement-artifacts") - pending = self.get_json("/public/v1/fanouts/pending?page=1&limit=15") - fanout = self.get_json(f"/public/v1/fanouts/{ledger.fanout_txid}") - artifact = self.get_json(f"/public/v1/artifacts/{ledger.manifest_set_sha256}") - audit_bundle = self.get_json(f"/public/v1/artifacts/{ledger.audit_bundle_sha256}") - - self.assertEqual(settlement["schema"], "prism.dashboard.settlement-artifacts.v1") - self.assertEqual(settlement["block_height"], 123450) - self.assertEqual(settlement["artifact_links"][0]["url"], f"/public/v1/artifacts/{ledger.audit_bundle_sha256}") - self.assertEqual(settlement["fanouts"][0]["status"], "broadcastable") - self.assertEqual(settlement["fanouts"][0]["broadcastable_at_height"], 124450) - self.assertEqual(settlement["fanouts"][0]["covenant_output_value_bits"], 4200) - self.assertEqual(settlement["fanouts"][0]["fanout_fee_bits"], 100) - self.assertEqual(settlement["fanouts"][0]["fanout_tx_sha256"], ledger.fanout_txid) - self.assertEqual(pending["schema"], "prism.dashboard.pending-fanouts.v1") - self.assertEqual(pending["rows"][0]["fanout_txid"], ledger.fanout_txid) - self.assertEqual(pending["rows"][0]["broadcastable_at_height"], 124450) - self.assertEqual(pending["rows"][0]["fanout_fee_bits"], 100) - self.assertEqual(fanout["schema"], "prism.dashboard.fanout.v1") - self.assertEqual(fanout["fanout"]["broadcastable_at_height"], 124450) - self.assertEqual(fanout["fanout"]["fanout_fee_bits"], 100) - self.assertEqual(fanout["fanout"]["last_broadcast_attempt_at"], "2026-06-26T20:45:00Z") - self.assertEqual(artifact["schema"], "qbit.prism.ctv-fanout-manifest-set.v1") - self.assertEqual(audit_bundle["schema"], "qbit.prism.audit-bundle.v1") - - def test_settlement_artifacts_returns_direct_coinbase_bundle_without_fanouts(self) -> None: - ledger = DirectCoinbasePublicLedger() - handler = make_public_handler(FakeCoordinator(ledger=ledger)) # type: ignore[arg-type] - server = ThreadingHTTPServer(("127.0.0.1", 0), handler) - thread = threading.Thread(target=server.serve_forever, daemon=True) - thread.start() - try: - with urllib.request.urlopen( - f"http://127.0.0.1:{server.server_port}/public/v1/blocks/{ledger.block_hash}/settlement-artifacts", - timeout=5, - ) as response: - self.assertEqual(response.status, 200) - settlement = json.loads(response.read()) - finally: - server.shutdown() - server.server_close() - thread.join(timeout=5) - - self.assertEqual(settlement["schema"], "prism.dashboard.settlement-artifacts.v1") - self.assertEqual(settlement["block_hash"], ledger.block_hash) - self.assertEqual(settlement["block_height"], ledger.block_height) - self.assertEqual(settlement["settlement_mode"], "direct_coinbase") - self.assertEqual(settlement["audit_bundle_sha256"], ledger.audit_bundle_sha256) - self.assertEqual(settlement["payout_manifest_sha256"], ledger.payout_manifest_sha256) - self.assertEqual(settlement["artifact_links"][0]["kind"], "audit_bundle") - self.assertEqual(settlement["artifact_links"][0]["url"], f"/public/v1/artifacts/{ledger.audit_bundle_sha256}") - self.assertEqual(settlement["fanouts"], []) - - def test_settlement_artifacts_uses_metadata_only_artifact_availability_when_present(self) -> None: - ledger = ArtifactExistsDirectCoinbasePublicLedger() - - settlement = public_api.settlement_artifacts(FakeCoordinator(ledger=ledger), block_hash=ledger.block_hash) - - self.assertEqual(ledger.exists_calls, 1) - self.assertEqual(ledger.artifact_calls, 0) - self.assertEqual(settlement["artifact_links"][0]["kind"], "audit_bundle") - - def test_settlement_artifacts_supports_externalized_direct_coinbase_audit_bundle(self) -> None: - with tempfile.TemporaryDirectory() as tmp: - ledger = ExternalizedDirectCoinbasePublicLedger(tmp) - handler = make_public_handler(FakeCoordinator(ledger=ledger)) # type: ignore[arg-type] - server = ThreadingHTTPServer(("127.0.0.1", 0), handler) - thread = threading.Thread(target=server.serve_forever, daemon=True) - thread.start() - try: - with urllib.request.urlopen( - f"http://127.0.0.1:{server.server_port}/public/v1/blocks/{ledger.block_hash}/settlement-artifacts", - timeout=5, - ) as response: - self.assertEqual(response.status, 200) - settlement = json.loads(response.read()) - finally: - server.shutdown() - server.server_close() - thread.join(timeout=5) - - self.assertEqual(ledger.resolver.body_read_count, 1) - self.assertEqual(settlement["settlement_mode"], "direct_coinbase") - self.assertEqual(settlement["block_height"], ledger.block_height) - self.assertEqual(settlement["audit_bundle_sha256"], ledger.audit_bundle_sha256) - self.assertEqual(settlement["payout_manifest_sha256"], ledger.payout_manifest_sha256) - self.assertEqual(settlement["fanouts"], []) - - def test_settlement_artifacts_returns_404_when_externalized_audit_body_is_unreadable(self) -> None: - with tempfile.TemporaryDirectory() as tmp: - ledger = ExternalizedDirectCoinbasePublicLedger(tmp) - ledger.body_uri.unlink() - handler = make_public_handler(FakeCoordinator(ledger=ledger)) # type: ignore[arg-type] - server = ThreadingHTTPServer(("127.0.0.1", 0), handler) - thread = threading.Thread(target=server.serve_forever, daemon=True) - thread.start() - try: - with self.assertRaises(urllib.error.HTTPError) as raised: - urllib.request.urlopen( - f"http://127.0.0.1:{server.server_port}/public/v1/blocks/{ledger.block_hash}/settlement-artifacts", - timeout=5, - ) - self.assertEqual(raised.exception.code, 404) - raised.exception.close() - finally: - server.shutdown() - server.server_close() - thread.join(timeout=5) - - self.assertEqual(ledger.resolver.body_read_count, 1) - - def test_settlement_artifacts_omit_unavailable_audit_bundle_link(self) -> None: - ledger = MissingAuditBundleBodyLedger() - - settlement = public_api.settlement_artifacts(FakeCoordinator(ledger=ledger), block_hash=ledger.block_hash) - - artifact_kinds = {link["kind"] for link in settlement["artifact_links"]} # type: ignore[index] - self.assertEqual(settlement["audit_bundle_sha256"], ledger.audit_bundle_sha256) - self.assertNotIn("audit_bundle", artifact_kinds) - self.assertIn("ctv_manifest_set", artifact_kinds) - - def test_pending_fanout_pagination_counts_beyond_first_thousand_rows(self) -> None: - pending = self.get_json("/public/v1/fanouts/pending?page=11&limit=100") - - self.assertEqual(pending["pagination"], {"page": 11, "limit": 100, "total_count": 1001, "total_pages": 11}) - self.assertEqual(len(pending["rows"]), 1) - self.assertEqual(pending["rows"][0]["fanout_txid"], f"{1000:064x}") - - def test_pending_fanouts_requires_uncapped_read_model(self) -> None: - with self.assertRaises(public_api.PublicApiError): - public_api.pending_fanouts( - FakeCoordinator(MissingPendingReadModelLedger()), # type: ignore[arg-type] - page=1, - limit=15, - ) - - def test_fanout_transaction_hash_uses_non_witness_txid_fallback(self) -> None: - witness_tx_hex = ( - "02000000" - "0001" - "01" - + "00" * 32 - + "ffffffff" - "00" - "ffffffff" - "00" - "00" - "00000000" - ) - - self.assertEqual(public_api.txid_from_tx_hex(witness_tx_hex), direct_stratum.transaction_txid_display(witness_tx_hex)) - - def test_settlement_bad_hash_and_missing_rows(self) -> None: - with self.assertRaises(urllib.error.HTTPError) as raised: - self.get_json("/public/v1/fanouts/not-a-hash") - - self.assertEqual(raised.exception.code, 400) - raised.exception.close() - - with self.assertRaises(urllib.error.HTTPError) as raised: - self.get_json(f"/public/v1/fanouts/{'9' * 64}") - - self.assertEqual(raised.exception.code, 404) - raised.exception.close() - - with self.assertRaises(urllib.error.HTTPError) as raised: - self.get_json(f"/public/v1/blocks/{'9' * 64}/settlement-artifacts") - - self.assertEqual(raised.exception.code, 404) - raised.exception.close() - - def test_miner_route_validation_uses_public_errors(self) -> None: - with self.assertRaises(urllib.error.HTTPError) as raised: - self.get_json("/public/v1/miners/miner-a/unknown") - - self.assertEqual(raised.exception.code, 404) - raised.exception.close() - - with self.assertRaises(urllib.error.HTTPError) as raised: - self.get_json(f"/public/v1/miners/{'x' * 257}") - - self.assertEqual(raised.exception.code, 400) - payload = json.loads(raised.exception.read()) - raised.exception.close() - self.assertEqual(payload["error"]["code"], "bad_request") - self.assertEqual(payload["error"]["message"], "recipient_id must be 256 characters or fewer") - - def test_mining_configuration_and_miner_detail_endpoints(self) -> None: - with patch.dict(os.environ, {}, clear=True): - config = self.get_json("/public/v1/mining-configuration") - miner = self.get_json("/public/v1/miners/miner-a") - earnings = self.get_json("/public/v1/miners/miner-a/earnings?page=1&limit=15") - payouts = self.get_json("/public/v1/miners/miner-a/payouts?page=1&limit=15") - workers = self.get_json("/public/v1/miners/miner-a/workers?page=1&limit=15") - - endpoint = config["configurations"][0]["stratum_endpoints"][0] - self.assertEqual(config["schema"], "prism.dashboard.mining-configuration.v1") - self.assertEqual(config["configurations"][0]["pool_fee_bps"], 0) - self.assertEqual(len(config["configurations"][0]["stratum_endpoints"]), 1) - self.assertEqual(endpoint["url"], "stratum+tcp://127.0.0.1:3340") - self.assertEqual(endpoint["default_port"], 3340) - self.assertEqual(miner["schema"], "prism.dashboard.miner.v1") - self.assertEqual(miner["owed_balance_bits"], 42) - self.assertEqual(miner["lifetime_earnings_bits"], 49) - self.assertEqual(miner["pending_maturity_bits"], 37) - self.assertEqual(miner["minimum_payout_bits"], 0) - self.assertEqual(miner["estimated_time_to_minimum_payout_seconds"], 0) - self.assertEqual(miner["reward_window_percent"], "50") - self.assertEqual(miner["estimated_next_block"]["share_percent"], "50") - # estimated_reward_bits is now computed (share_percent x distributable coinbase), - # no longer hardcoded null. - self.assertIsInstance(miner["estimated_next_block"]["estimated_reward_bits"], int) - self.assertEqual(miner["workers_currently_hashing"], 2) - self.assertEqual(miner["workers"][0]["worker_name"], "rig-a") - self.assertEqual(earnings["schema"], "prism.dashboard.miner-earnings.v1") - self.assertEqual(earnings["rows"][0]["found_at"], "2026-06-26T20:40:00Z") - self.assertEqual(earnings["rows"][0]["reward_share_percent"], "50") - self.assertEqual(earnings["rows"][0]["gross_earning_bits"], 42) - self.assertEqual(payouts["schema"], "prism.dashboard.miner-payouts.v1") - self.assertEqual(payouts["rows"][0]["created_at"], "2026-06-26T20:45:00Z") - self.assertEqual(payouts["rows"][0]["onchain_amount_bits"], 40) - self.assertEqual(workers["schema"], "prism.dashboard.miner-workers.v1") - self.assertEqual(workers["pagination"]["total_count"], 2) - self.assertEqual(workers["rows"][0]["worker_name"], "rig-a") - self.assertEqual(workers["rows"][0]["status"], "active") - - def test_miner_payout_row_labels_ctv_fanout_outputs(self) -> None: - fanout_txid = "c5" * 32 - base_row = { - "block_hash": "a1" * 32, - "block_height": 19, - "coinbase_txid": "b2" * 32, - "recipient_id": "miner-a", - "onchain_amount_sats": 7_767_471, - "carry_forward_balance_sats": 0, - "action": "onchain", - "maturity_state": "immature", - "created_at": "2026-07-15 21:50:03+00", - } - - with patch.dict(os.environ, {"PRISM_PUBLIC_EXPLORER_TX_URL_PREFIX": "https://explorer.example/tx"}, clear=False): - ctv_row = public_api.miner_payout_row( - { - **base_row, - "fanout_txid": fanout_txid, - "fanout_vout": 1, - "fanout_amount_sats": 7_767_319, - "fanout_fee_sats": 152, - "fanout_gross_amount_sats": 7_767_471, - "fanout_status": "awaiting_maturity", - } - ) - self.assertEqual(ctv_row["transaction_kind"], "ctv_fanout") - self.assertEqual(ctv_row["transaction_id"], fanout_txid) - self.assertEqual(ctv_row["onchain_amount_bits"], 7_767_319) - self.assertEqual(ctv_row["explorer_url"], f"https://explorer.example/tx/{fanout_txid}") - - coinbase_row = public_api.miner_payout_row(dict(base_row)) - self.assertEqual(coinbase_row["transaction_kind"], "coinbase") - self.assertEqual(coinbase_row["transaction_id"], "b2" * 32) - self.assertEqual(coinbase_row["onchain_amount_bits"], 7_767_471) - - # Accrued rows stay carry_forward even if a fanout/coinbase column - # leaks in from a wider read-model row. - accrued_row = public_api.miner_payout_row( - { - **base_row, - "action": "accrued", - "onchain_amount_sats": 0, - "carry_forward_balance_sats": 7, - "fanout_txid": fanout_txid, - } - ) - self.assertEqual(accrued_row["transaction_kind"], "carry_forward") - self.assertIsNone(accrued_row["transaction_id"]) - self.assertEqual(accrued_row["onchain_amount_bits"], 0) - - def test_estimated_next_block_reward_bits_computes_and_handles_edges(self) -> None: - # 50% of a 600-bit coinbase, no pool fee. - self.assertEqual( - public_api.estimated_next_block_reward_bits( - share_percent="50", expected_coinbase_bits=600, pool_fee_bps=0 - ), - 300, - ) - # A 10% pool fee reduces the distributable coinbase before applying the share. - self.assertEqual( - public_api.estimated_next_block_reward_bits( - share_percent="50", expected_coinbase_bits=600, pool_fee_bps=1000 - ), - 270, - ) - # A 0% share is a real answer (0 bits), not "unknown". - self.assertEqual( - public_api.estimated_next_block_reward_bits( - share_percent="0", expected_coinbase_bits=600, pool_fee_bps=0 - ), - 0, - ) - # No share, no coinbase, or a non-positive coinbase -> None so the dashboard - # can fall back to showing only the share percent. - for kwargs in ( - {"share_percent": None, "expected_coinbase_bits": 600, "pool_fee_bps": 0}, - {"share_percent": "50", "expected_coinbase_bits": None, "pool_fee_bps": 0}, - {"share_percent": "50", "expected_coinbase_bits": 0, "pool_fee_bps": 0}, - {"share_percent": "not-a-number", "expected_coinbase_bits": 600, "pool_fee_bps": 0}, - ): - self.assertIsNone(public_api.estimated_next_block_reward_bits(**kwargs)) - - def test_miner_estimated_reward_bits_end_to_end(self) -> None: - coordinator = FakeCoordinator() # latest block coinbase_value_bits=600, share_percent=50 - with patch.dict(os.environ, {"PRISM_PUBLIC_POOL_FEE_BPS": "0"}, clear=False): - miner = public_api.miner(coordinator, recipient_id="miner-a") - self.assertEqual(miner["estimated_next_block"]["estimated_reward_bits"], 300) - - with patch.dict(os.environ, {"PRISM_PUBLIC_POOL_FEE_BPS": "1000"}, clear=False): - miner_with_fee = public_api.miner(coordinator, recipient_id="miner-a") - self.assertEqual(miner_with_fee["estimated_next_block"]["estimated_reward_bits"], 270) - - def test_miner_estimated_reward_bits_none_when_no_reward_window_share(self) -> None: - coordinator = FakeCoordinator() - miner = public_api.miner(coordinator, recipient_id="miner-b") # no window share - self.assertIsNone(miner["estimated_next_block"]["share_percent"]) - self.assertIsNone(miner["estimated_next_block"]["estimated_reward_bits"]) - - def test_public_pool_fee_bps_clamps_and_tolerates_invalid_env(self) -> None: - for value, expected in (("", 0), ("250", 250), ("-5", 0), ("20000", 10_000), ("nan", 0)): - with patch.dict(os.environ, {"PRISM_PUBLIC_POOL_FEE_BPS": value}, clear=False): - self.assertEqual(public_api.public_pool_fee_bps(), expected) - with patch.dict(os.environ, {}, clear=True): - self.assertEqual(public_api.public_pool_fee_bps(), 0) - - def test_mining_configuration_pool_fee_uses_clamped_helper(self) -> None: - # The displayed fee and the reward estimate must read the fee the same way. - coordinator = FakeCoordinator() - with patch.dict(os.environ, {"PRISM_PUBLIC_POOL_FEE_BPS": "20000"}, clear=False): - config = public_api.mining_configuration(coordinator) - self.assertEqual(config["configurations"][0]["pool_fee_bps"], 10_000) - - def test_miner_detail_tolerates_blank_minimum_payout_env(self) -> None: - with patch.dict( - os.environ, - { - "PRISM_PUBLIC_MINIMUM_PAYOUT_BITS": "", - "PRISM_PAYOUT_MIN_OUTPUT_BITS": "", - "PRISM_PAYOUT_MIN_OUTPUT_SATS": "", - }, - clear=True, - ): - miner = self.get_json("/public/v1/miners/miner-a") - - self.assertEqual(miner["minimum_payout_bits"], 0) - self.assertEqual(miner["estimated_time_to_minimum_payout_seconds"], 0) - - def test_miner_detail_uses_payout_minimum_bits_env_as_fallback(self) -> None: - with patch.dict(os.environ, {"PRISM_PAYOUT_MIN_OUTPUT_BITS": "43"}, clear=True): - miner = self.get_json("/public/v1/miners/miner-a") - - self.assertEqual(miner["minimum_payout_bits"], 43) - self.assertIsNone(miner["estimated_time_to_minimum_payout_seconds"]) - - def test_miner_detail_falls_back_to_legacy_sats_env(self) -> None: - with patch.dict(os.environ, {"PRISM_PAYOUT_MIN_OUTPUT_SATS": "43"}, clear=True): - miner = self.get_json("/public/v1/miners/miner-a") - - self.assertEqual(miner["minimum_payout_bits"], 43) - - def test_miner_detail_public_minimum_payout_env_overrides_internal_env(self) -> None: - with patch.dict( - os.environ, - { - "PRISM_PUBLIC_MINIMUM_PAYOUT_BITS": "41", - "PRISM_PAYOUT_MIN_OUTPUT_BITS": "43", - }, - clear=True, - ): - miner = self.get_json("/public/v1/miners/miner-a") - - self.assertEqual(miner["minimum_payout_bits"], 41) - self.assertEqual(miner["estimated_time_to_minimum_payout_seconds"], 0) - - def test_miner_detail_skips_invalid_minimum_payout_env_values(self) -> None: - with patch.dict( - os.environ, - { - "PRISM_PUBLIC_MINIMUM_PAYOUT_BITS": "not-an-int", - "PRISM_PAYOUT_MIN_OUTPUT_BITS": "43", - }, - clear=True, - ): - miner = self.get_json("/public/v1/miners/miner-a") - - self.assertEqual(miner["minimum_payout_bits"], 43) - - def test_mining_configuration_uses_public_stratum_url_and_fee_env(self) -> None: - with patch.dict( - os.environ, - { - "PRISM_PUBLIC_STRATUM_URL": "stratum+tcp://public-pool.example:3335", - "PRISM_PUBLIC_POOL_FEE_BPS": "200", - }, - clear=True, - ): - config = self.get_json("/public/v1/mining-configuration") - - endpoint = config["configurations"][0]["stratum_endpoints"][0] - self.assertEqual(config["configurations"][0]["pool_fee_bps"], 200) - self.assertEqual(endpoint["url"], "stratum+tcp://public-pool.example:3335") - self.assertEqual(endpoint["default_port"], 3335) - - def test_mining_configuration_includes_highdiff_endpoint_when_enabled(self) -> None: - with patch.dict( - os.environ, - { - "PRISM_PUBLIC_STRATUM_HOST": "mine.prism.example", - "PRISM_STRATUM_HIGHDIFF_PORT": "4334", - }, - clear=True, - ): - config = public_api.mining_configuration(FakeCoordinator()) - - endpoints = config["configurations"][0]["stratum_endpoints"] - self.assertEqual( - endpoints, - [ - { - "label": "Primary", - "url": "stratum+tcp://mine.prism.example:3340", - "protocol": "stratum_v1", - "default_port": 3340, - }, - { - "label": "High-diff", - "url": "stratum+tcp://mine.prism.example:4334", - "protocol": "stratum_v1", - "default_port": 4334, - }, - ], - ) - - def test_mining_configuration_derives_highdiff_endpoint_from_public_stratum_url(self) -> None: - with patch.dict( - os.environ, - { - "PRISM_PUBLIC_STRATUM_URL": "stratum+tcp://public-pool.example:3335", - "PRISM_STRATUM_HIGHDIFF_PORT": "4334", - }, - clear=True, - ): - config = public_api.mining_configuration(FakeCoordinator()) - - endpoints = config["configurations"][0]["stratum_endpoints"] - self.assertEqual(endpoints[0]["url"], "stratum+tcp://public-pool.example:3335") - self.assertEqual(endpoints[1]["label"], "High-diff") - self.assertEqual(endpoints[1]["url"], "stratum+tcp://public-pool.example:4334") - self.assertEqual(endpoints[1]["default_port"], 4334) - - def test_mining_configuration_uses_public_highdiff_stratum_url_override(self) -> None: - with patch.dict( - os.environ, - { - "PRISM_PUBLIC_STRATUM_URL": "stratum+tcp://mine.prism.example:3333", - "PRISM_STRATUM_HIGHDIFF_PORT": "4334", - "PRISM_PUBLIC_STRATUM_HIGHDIFF_URL": "stratum+tcp://rentals.prism.example:14334", - }, - clear=True, - ): - config = public_api.mining_configuration(FakeCoordinator()) - - endpoints = config["configurations"][0]["stratum_endpoints"] - self.assertEqual(endpoints[0]["url"], "stratum+tcp://mine.prism.example:3333") - self.assertEqual(endpoints[0]["default_port"], 3333) - self.assertEqual(endpoints[1]["label"], "High-diff") - self.assertEqual(endpoints[1]["url"], "stratum+tcp://rentals.prism.example:14334") - self.assertEqual(endpoints[1]["default_port"], 14334) - - def test_miner_worker_search_and_pagination_use_real_worker_names(self) -> None: - page_two = self.get_json("/public/v1/miners/miner-a/workers?hide_inactive=false&page=2&limit=1") - search = self.get_json("/public/v1/miners/miner-a/workers?hide_inactive=false&search=rig-b&page=1&limit=15") - - self.assertEqual(page_two["pagination"], {"page": 2, "limit": 1, "total_count": 2, "total_pages": 2}) - self.assertEqual(page_two["rows"][0]["worker_name"], "rig-b") - self.assertEqual(search["pagination"]["total_count"], 1) - self.assertEqual(search["rows"][0]["worker_name"], "rig-b") - - with self.assertRaises(urllib.error.HTTPError) as raised: - self.get_json(f"/public/v1/miners/miner-a/workers?search={'x' * 129}") - - self.assertEqual(raised.exception.code, 400) - raised.exception.close() - - def test_miner_payout_and_earning_pagination_is_not_capped_to_first_row(self) -> None: - earnings = self.get_json("/public/v1/miners/miner-a/earnings?page=2&limit=1") - payouts = self.get_json("/public/v1/miners/miner-a/payouts?page=2&limit=1") - - self.assertEqual(earnings["pagination"]["total_count"], 2) - self.assertEqual(earnings["rows"][0]["gross_earning_bits"], 7) - self.assertEqual(earnings["rows"][0]["settlement_fee_bits"], 1) - self.assertEqual(earnings["rows"][0]["net_earning_bits"], 6) - self.assertEqual(earnings["rows"][0]["reward_share_percent"], "25") - self.assertEqual(payouts["pagination"]["total_count"], 2) - self.assertEqual(payouts["rows"][0]["action"], "accrued") - - def test_miner_row_builders_do_not_invent_missing_internal_values(self) -> None: - with self.assertRaises(public_api.PublicApiError): - public_api.miner_payout_row({"block_hash": "a" * 64, "block_height": 1}) - - with self.assertRaises(public_api.PublicApiError): - public_api.miner_earning_row( - { - "block_hash": "a" * 64, - "block_height": 1, - "gross_amount_sats": 1, - "block_gross_amount_sats": 10, - } - ) - - with self.assertRaises(public_api.PublicApiError): - public_api.miner_earning_row( - { - "block_hash": "a" * 64, - "block_height": 1, - "gross_amount_sats": 1, - "created_at": "2026-06-26 20:45:00+00", - } - ) - - def test_pending_maturity_fallback_filters_and_nets_immature_onchain_rows(self) -> None: - class HistoryOnlyLedger: - def recipient_payout_history(self, *, recipient_id: str, limit: int = 50) -> list[dict[str, object]]: - self.request = (recipient_id, limit) - return [ - { - "action": "onchain", - "maturity_state": "immature", - "onchain_amount_sats": 20, - "settlement_fee_sats": 3, - }, - { - "action": "onchain", - "maturity_state": "mature", - "onchain_amount_sats": 50, - "settlement_fee_sats": 2, - }, - { - "action": "accrued", - "maturity_state": "immature", - "onchain_amount_sats": 70, - }, - { - "action": "onchain", - "maturity_state": "immature", - "onchain_amount_sats": 1, - "settlement_fee_sats": 2, - }, - ] - - ledger = HistoryOnlyLedger() - - self.assertEqual(public_api.pending_maturity_bits_for_recipient(ledger, "miner-a"), 17) - self.assertEqual(ledger.request, ("miner-a", 1_000)) - - -class PublicHandlerTestCase(unittest.TestCase): - """One extracted-handler server per ledger, torn down with the test.""" - - def serve(self, ledger: FakePublicLedger) -> str: - handler = make_public_handler(FakeCoordinator(ledger=ledger)) # type: ignore[arg-type] - server = ThreadingHTTPServer(("127.0.0.1", 0), handler) - thread = threading.Thread(target=server.serve_forever, daemon=True) - thread.start() - - def stop() -> None: - server.shutdown() - server.server_close() - thread.join(timeout=5) - - self.addCleanup(stop) - return f"http://127.0.0.1:{server.server_port}" - - def get_error(self, url: str) -> tuple[int, dict[str, object], dict[str, str]]: - with self.assertRaises(urllib.error.HTTPError) as raised: - urllib.request.urlopen(url, timeout=5) - with raised.exception as error: - return error.code, json.loads(error.read()), dict(error.headers.items()) - - -# Every range/bucket pair outside HASHRATE_SERIES_ALLOWED_BUCKETS, and every -# concrete pair inside it. Derived by hand rather than from the constant so a -# vocabulary regression cannot rewrite the test that guards it. -FORBIDDEN_BUCKET_COMBOS = ( - ("1m", "5m"), - ("6m", "5m"), - ("6m", "1h"), - ("all", "5m"), - ("all", "1h"), -) -ALLOWED_BUCKET_COMBOS = ( - ("1w", "5m"), - ("1w", "1h"), - ("1w", "1d"), - ("1m", "1h"), - ("1m", "1d"), - ("6m", "1d"), - ("all", "1d"), -) - - -class HashrateBucketClampTests(PublicHandlerTestCase): - """The static bucket-per-range vocabulary closes the unbounded-cost combos.""" - - def test_forbidden_combinations_are_rejected_before_any_ledger_call(self) -> None: - ledger = FakePublicLedger() - base = self.serve(ledger) - for range_id, bucket in FORBIDDEN_BUCKET_COMBOS: - with self.subTest(range=range_id, bucket=bucket): - status, payload, headers = self.get_error( - f"{base}/public/v1/hashrate-series?range={range_id}&bucket={bucket}" - ) - self.assertEqual(400, status) - self.assertEqual("prism.dashboard.error.v1", payload["schema"]) - self.assertEqual("bad_request", payload["error"]["code"]) - self.assertIn( - f"bucket {bucket} is not allowed for range {range_id}", - payload["error"]["message"], - ) - self.assertEqual("no-store", headers.get("Cache-Control")) - self.assertEqual(0, ledger.hashrate_series_calls) - - def test_the_refusal_names_the_allowed_buckets(self) -> None: - base = self.serve(FakePublicLedger()) - _status, payload, _headers = self.get_error( - f"{base}/public/v1/hashrate-series?range=1m&bucket=5m" - ) - self.assertIn("allowed: 1h, 1d", payload["error"]["message"]) - - def test_allowed_combinations_still_serve(self) -> None: - ledger = FakePublicLedger() - base = self.serve(ledger) - combos = ALLOWED_BUCKET_COMBOS + tuple( - (range_id, "auto") for range_id in public_api.HASHRATE_SERIES_RANGE_SECONDS - ) - for range_id, bucket in combos: - with self.subTest(range=range_id, bucket=bucket): - url = f"{base}/public/v1/hashrate-series?range={range_id}&bucket={bucket}" - with urllib.request.urlopen(url, timeout=5) as response: - payload = json.loads(response.read()) - self.assertEqual("prism.dashboard.hashrate-series.v1", payload["schema"]) - self.assertEqual(len(combos), ledger.hashrate_series_calls) - - def test_every_auto_resolution_is_inside_the_allowed_vocabulary(self) -> None: - for range_id in public_api.HASHRATE_SERIES_RANGE_SECONDS: - resolved = public_api.auto_bucket(range_id) - self.assertIn(resolved, public_api.HASHRATE_SERIES_ALLOWED_BUCKETS[range_id]) - # And the helper itself agrees: a resolved auto bucket never raises. - public_api.allowed_hashrate_bucket(range_id, resolved) - - -class PublicReadStatementTimeoutTests(PublicHandlerTestCase): - """The per-request database deadline and its read_timeout refusal.""" - - def test_a_ledger_deadline_becomes_a_503_read_timeout_and_is_never_cached(self) -> None: - ledger = TimingOutPublicLedger() - base = self.serve(ledger) - # Patch only the deadline knob: clearing the whole environment also - # drops NO_PROXY, and on a proxy-configured machine urllib then routes - # this localhost request through the proxy instead of the test server. - with patch.dict(os.environ): - os.environ.pop("PRISM_PUBLIC_READ_STATEMENT_TIMEOUT_SECONDS", None) - for expected_origin_calls in (1, 2): - status, payload, headers = self.get_error( - f"{base}/public/v1/hashrate-series" - ) - self.assertEqual(503, status) - self.assertEqual("prism.dashboard.error.v1", payload["schema"]) - self.assertEqual("read_timeout", payload["error"]["code"]) - self.assertEqual("no-store", headers.get("Cache-Control")) - self.assertIsNone(headers.get("CDN-Cache-Control")) - # The second request re-ran the origin: a timeout stored no - # cache entry, so one expensive miss cannot poison the cache - # with a refusal. - self.assertEqual(expected_origin_calls, ledger.hashrate_series_calls) - self.assertEqual([20.0, 20.0], ledger.statement_timeout_budgets) - - def test_the_scope_is_entered_with_the_configured_budget(self) -> None: - ledger = StatementTimeoutRecordingLedger() - base = self.serve(ledger) - with patch.dict( - os.environ, - {"PRISM_PUBLIC_READ_STATEMENT_TIMEOUT_SECONDS": "7"}, - ): - with urllib.request.urlopen( - f"{base}/public/v1/hashrate-series", timeout=5 - ) as response: - json.loads(response.read()) - self.assertEqual([7.0], ledger.statement_timeout_budgets) - self.assertEqual(1, ledger.hashrate_series_calls) - - def test_env_zero_disables_the_scope_entirely(self) -> None: - ledger = StatementTimeoutRecordingLedger() - base = self.serve(ledger) - with patch.dict( - os.environ, - {"PRISM_PUBLIC_READ_STATEMENT_TIMEOUT_SECONDS": "0"}, - ): - with urllib.request.urlopen( - f"{base}/public/v1/hashrate-series", timeout=5 - ) as response: - json.loads(response.read()) - self.assertEqual([], ledger.statement_timeout_budgets) - self.assertEqual(1, ledger.hashrate_series_calls) - - -class PublicResponseCacheStaleWhileRevalidateTests(unittest.TestCase): - """In-process stale-while-revalidate on PublicResponseCache. - - Entry expiry is keyed on time.monotonic(), which is not injectable, so - these tests age entries by editing the stored bounds directly -- the - precedent tests/test_prism_public_read_service.py sets for the outage - contract -- rather than sleeping out real TTLs. - """ - - KEY = ("/swr", ()) - TTL = 60 - - def prime(self, cache: public_api.PublicResponseCache, payload: object) -> None: - status, served, state, _age = cache.get_or_compute( - key=self.KEY, ttl_seconds=self.TTL, compute=lambda: (200, payload) - ) - self.assertEqual((200, payload, "MISS"), (status, served, state)) - - def age_entry( - self, - cache: public_api.PublicResponseCache, - *, - age_seconds: float, - key: tuple[str, tuple[tuple[str, tuple[str, ...]], ...]] | None = None, - stale_while_revalidate_seconds: int = 30, - ) -> None: - entry = cache._entries[key if key is not None else self.KEY] - entry.stored_at = time.monotonic() - age_seconds - entry.expires_at = entry.stored_at + self.TTL - entry.stale_until = entry.expires_at + stale_while_revalidate_seconds - - def lookup( - self, - cache: public_api.PublicResponseCache, - compute, - ) -> tuple[int, object, str, int]: - return cache.get_or_compute( - key=self.KEY, - ttl_seconds=self.TTL, - stale_while_revalidate_seconds=30, - compute=compute, - ) - - def wait_for_refresh_slot_to_clear(self, cache: public_api.PublicResponseCache) -> None: - # Bounded: the refresh thread frees the inflight slot in its finally - # clause, so a few milliseconds suffice and 5s is a hard failure. - deadline = time.monotonic() + 5 - while self.KEY in cache._inflight and time.monotonic() < deadline: - time.sleep(0.01) - self.assertNotIn(self.KEY, cache._inflight) - - def test_capacity_reaping_spares_stale_servable_entries(self) -> None: - """Reaping removes entries past their whole window, not merely expired. - - A full cache receiving a new key must shed dead weight first: an entry - past TTL *and* its recorded revalidate window is gone, but one still - inside the window stays exactly as servable as the stale-serve path - promises -- reaping it would turn its next visitor's instant STALE - answer into a blocking recompute. - """ - cache = public_api.PublicResponseCache() - keep_key = ("/swr-keep", ()) - dead_key = ("/swr-dead", ()) - for key in (keep_key, dead_key): - cache.get_or_compute( - key=key, - ttl_seconds=self.TTL, - stale_while_revalidate_seconds=30, - compute=lambda: (200, {"k": key[0]}), - ) - # keep: expired 5s ago, inside its 30s window. dead: past the window. - self.age_entry(cache, age_seconds=65, key=keep_key) - self.age_entry(cache, age_seconds=100, key=dead_key) - with patch.dict(os.environ, {"PRISM_PUBLIC_CACHE_MAX_ENTRIES": "2"}): - cache.get_or_compute( - key=("/swr-new", ()), - ttl_seconds=self.TTL, - stale_while_revalidate_seconds=30, - compute=lambda: (200, {"k": "new"}), - ) - self.assertNotIn(dead_key, cache._entries) - self.assertIn(keep_key, cache._entries) - status, payload, state, age = cache.get_or_compute( - key=keep_key, - ttl_seconds=self.TTL, - stale_while_revalidate_seconds=30, - compute=lambda: (200, {"k": "refreshed"}), - ) - self.assertEqual((200, {"k": "/swr-keep"}, "STALE", 65), (status, payload, state, age)) - deadline = time.monotonic() + 5 - while keep_key in cache._inflight and time.monotonic() < deadline: - time.sleep(0.01) - self.assertNotIn(keep_key, cache._inflight) - - def test_a_failed_refresh_thread_start_frees_the_slot(self) -> None: - """Thread creation failure must not strand the registered slot. - - The refresh slot is registered before Thread.start() runs, and under - resource pressure start() can raise. Without cleanup no thread ever - frees the slot: later stale hits skip their refresh forever and, - past the window, blocking requests coalesce onto an event nobody can - set. The stale answer is still served; the slot is freed and the - event set so the next request proceeds normally. - """ - cache = public_api.PublicResponseCache() - self.prime(cache, {"v": "stale"}) - self.age_entry(cache, age_seconds=65) - - with patch.object( - threading.Thread, "start", side_effect=RuntimeError("can't start new thread") - ): - status, payload, state, age = self.lookup( - cache, lambda: (200, {"v": "unused"}) - ) - - self.assertEqual((200, {"v": "stale"}, "STALE", 65), (status, payload, state, age)) - self.assertNotIn(self.KEY, cache._inflight) - # The next stale hit retries the refresh with a working thread... - refreshed = threading.Event() - - def refresh() -> tuple[int, object]: - refreshed.set() - return 200, {"v": "fresh"} - - status, payload, state, _age = self.lookup(cache, refresh) - self.assertEqual((200, {"v": "stale"}, "STALE"), (status, payload, state)) - self.assertTrue(refreshed.wait(timeout=5)) - self.wait_for_refresh_slot_to_clear(cache) - status, payload, state, _age = self.lookup(cache, lambda: (200, {"v": "wrong"})) - self.assertEqual((200, {"v": "fresh"}, "HIT"), (status, payload, state)) - - def test_a_fresh_entry_is_a_hit_with_the_window_armed(self) -> None: - cache = public_api.PublicResponseCache() - self.prime(cache, {"v": "fresh"}) - - status, payload, state, age = self.lookup(cache, lambda: (200, {"v": "wrong"})) - - self.assertEqual((200, {"v": "fresh"}, "HIT", 0), (status, payload, state, age)) - - def test_a_stale_entry_is_served_while_one_background_refresh_recomputes(self) -> None: - cache = public_api.PublicResponseCache() - self.prime(cache, {"v": "stale"}) - self.age_entry(cache, age_seconds=65) # expired 5s ago, inside the 30s window - refreshed = threading.Event() - - def refresh() -> tuple[int, object]: - refreshed.set() - return 200, {"v": "fresh"} - - status, payload, state, age = self.lookup(cache, refresh) - - # Served immediately, from the expired entry, with its honest age. - self.assertEqual((200, {"v": "stale"}, "STALE", 65), (status, payload, state, age)) - self.assertTrue(refreshed.wait(timeout=5)) - self.wait_for_refresh_slot_to_clear(cache) - status, payload, state, age = self.lookup(cache, lambda: (200, {"v": "wrong"})) - self.assertEqual((200, {"v": "fresh"}, "HIT", 0), (status, payload, state, age)) - - def test_a_failed_refresh_keeps_the_stale_entry_and_frees_the_slot(self) -> None: - cache = public_api.PublicResponseCache() - self.prime(cache, {"v": "stale"}) - self.age_entry(cache, age_seconds=61) - failed = threading.Event() - - def failing_refresh() -> tuple[int, object]: - failed.set() - raise RuntimeError("origin down") - - status, payload, state, _age = self.lookup(cache, failing_refresh) - - self.assertEqual((200, {"v": "stale"}, "STALE"), (status, payload, state)) - self.assertTrue(failed.wait(timeout=5)) - self.wait_for_refresh_slot_to_clear(cache) - # Still servable -- the failure deleted nothing -- and the freed slot - # lets this later stale hit start a second refresh, which succeeds. - recovered = threading.Event() - - def recovering_refresh() -> tuple[int, object]: - recovered.set() - return 200, {"v": "fresh"} - - status, payload, state, _age = self.lookup(cache, recovering_refresh) - self.assertEqual((200, {"v": "stale"}, "STALE"), (status, payload, state)) - self.assertTrue(recovered.wait(timeout=5)) - self.wait_for_refresh_slot_to_clear(cache) - _status, payload, state, _age = self.lookup(cache, lambda: (200, {"v": "wrong"})) - self.assertEqual(({"v": "fresh"}, "HIT"), (payload, state)) - - def test_beyond_the_window_the_next_request_blocks_and_recomputes(self) -> None: - cache = public_api.PublicResponseCache() - self.prime(cache, {"v": "stale"}) - self.age_entry(cache, age_seconds=95) # expired 35s ago, past the 30s window - - status, payload, state, age = self.lookup(cache, lambda: (200, {"v": "fresh"})) - - self.assertEqual((200, {"v": "fresh"}, "MISS", 0), (status, payload, state, age)) - - def test_concurrent_stale_hits_share_exactly_one_refresh(self) -> None: - cache = public_api.PublicResponseCache() - self.prime(cache, {"v": "stale"}) - self.age_entry(cache, age_seconds=61) - release = threading.Event() - compute_calls: list[int] = [] - compute_lock = threading.Lock() - - def slow_refresh() -> tuple[int, object]: - with compute_lock: - compute_calls.append(1) - self.assertTrue(release.wait(timeout=5)) - return 200, {"v": "fresh"} - - results: list[tuple[int, object, str, int]] = [] - results_lock = threading.Lock() - - def stale_hit() -> None: - result = self.lookup(cache, slow_refresh) - with results_lock: - results.append(result) - - workers = [threading.Thread(target=stale_hit) for _ in range(5)] - for worker in workers: - worker.start() - for worker in workers: - worker.join(timeout=5) - - # Every requester was answered immediately from the stale entry; none - # blocked behind the (still running) refresh, and none started one of - # its own. - self.assertEqual(5, len(results)) - for status, payload, state, _age in results: - self.assertEqual((200, {"v": "stale"}, "STALE"), (status, payload, state)) - release.set() - self.wait_for_refresh_slot_to_clear(cache) - self.assertEqual([1], compute_calls) - _status, payload, state, _age = self.lookup(cache, lambda: (200, {"v": "wrong"})) - self.assertEqual(({"v": "fresh"}, "HIT"), (payload, state)) - - -class PublicArtifactByteExactnessTests(unittest.TestCase): - """GET /public/v1/artifacts/{sha256} must return the exact bytes the - advertised sha256 was computed over, or no external verifier can confirm - any artifact. This was the missing regression coverage: the hash side - serializes with canonical_json_text (compact separators) while the - generic write_json path re-serializes with default separators plus a - trailing newline.""" - - def serve(self, ledger: FakePublicLedger) -> str: - handler = make_public_handler(FakeCoordinator(ledger=ledger)) # type: ignore[arg-type] - server = ThreadingHTTPServer(("127.0.0.1", 0), handler) - thread = threading.Thread(target=server.serve_forever, daemon=True) - thread.start() - - def stop() -> None: - server.shutdown() - server.server_close() - thread.join(timeout=5) - - self.addCleanup(stop) - return f"http://127.0.0.1:{server.server_port}" - - def fetch_artifact(self, base_url: str, sha256: str) -> tuple[bytes, dict[str, str]]: - with urllib.request.urlopen(f"{base_url}/public/v1/artifacts/{sha256}", timeout=5) as response: - return response.read(), dict(response.headers) - - def test_manifest_artifact_bytes_hash_to_advertised_sha256(self) -> None: - ledger = CanonicalArtifactPublicLedger() - base_url = self.serve(ledger) - for sha256, document in ( - (ledger.manifest_sha256, ledger.manifest), - (ledger.manifest_set_sha256, ledger.manifest_set), - ): - with self.subTest(sha256=sha256): - body, headers = self.fetch_artifact(base_url, sha256) - - self.assertEqual(hashlib.sha256(body).hexdigest(), sha256) - self.assertEqual(json.loads(body), document) - self.assertEqual(headers.get("Content-Type"), "application/json") - self.assertEqual(int(headers["Content-Length"]), len(body)) - - def test_artifact_bytes_stay_exact_across_cache_hits(self) -> None: - ledger = CanonicalArtifactPublicLedger() - base_url = self.serve(ledger) - - first, first_headers = self.fetch_artifact(base_url, ledger.manifest_set_sha256) - second, second_headers = self.fetch_artifact(base_url, ledger.manifest_set_sha256) - - self.assertEqual(first, second) - self.assertEqual(hashlib.sha256(second).hexdigest(), ledger.manifest_set_sha256) - for headers in (first_headers, second_headers): - self.assertIn("immutable", headers.get("CDN-Cache-Control", "")) - - def test_uppercase_artifact_path_serves_the_same_exact_bytes(self) -> None: - ledger = CanonicalArtifactPublicLedger() - base_url = self.serve(ledger) - - body, _headers = self.fetch_artifact(base_url, ledger.manifest_sha256.upper()) - - self.assertEqual(hashlib.sha256(body).hexdigest(), ledger.manifest_sha256) - - def test_tampered_canonical_text_falls_back_to_reserialized_document(self) -> None: - # Stored canonical text that no longer hashes to its content address - # must never be served as the exact bytes; the endpoint keeps the - # legacy re-serialized response instead. - ledger = TamperedCanonicalArtifactPublicLedger() - base_url = self.serve(ledger) - - body, _headers = self.fetch_artifact(base_url, ledger.manifest_sha256) - - self.assertEqual(body, json.dumps(ledger.manifest, sort_keys=True).encode() + b"\n") - self.assertEqual(json.loads(body), ledger.manifest) - - def test_audit_bundle_artifact_bytes_hash_to_advertised_sha256(self) -> None: - ledger = CanonicalArtifactPublicLedger() - base_url = self.serve(ledger) - - body, _headers = self.fetch_artifact(base_url, ledger.audit_bundle_sha256) - - self.assertEqual(hashlib.sha256(body).hexdigest(), ledger.audit_bundle_sha256) - - def test_missing_canonical_audit_bundle_fallback_is_never_cached(self) -> None: - ledger = MissingCanonicalArtifactPublicLedger() - base_url = self.serve(ledger) - - first, first_headers = self.fetch_artifact(base_url, ledger.audit_bundle_sha256) - second, second_headers = self.fetch_artifact(base_url, ledger.audit_bundle_sha256) - - self.assertEqual(json.loads(first), ledger.audit_bundle) - self.assertEqual(second, first) - self.assertEqual(ledger.audit_document_calls, 2) - for headers in (first_headers, second_headers): - self.assertEqual(headers.get("Cache-Control"), "no-store") - self.assertEqual( - headers.get("X-Prism-Artifact-Canonical-State"), - "missing", - ) - self.assertNotIn("CDN-Cache-Control", headers) - self.assertNotIn("Vercel-CDN-Cache-Control", headers) - self.assertNotIn("Age", headers) - - -class PrismPublicDashboardMemoryLedgerTests(unittest.TestCase): - def test_memory_ledger_public_read_models_are_empty_safe(self) -> None: - coordinator = MemoryCoordinator() - now_ms = int(time.time() * 1000) - coordinator.ledger.append( - PendingShare( - share_id="miner-a:share-1", - miner_id="miner-a", - order_key="miner-a", - p2mr_program_hex="11" * 32, - share_difficulty=100, - network_difficulty=1000, - template_height=123, - job_id="job-1", - job_issued_at_ms=now_ms, - accepted_at_ms=now_ms, - ntime=1, - ) - ) - handler = make_public_handler(coordinator) # type: ignore[arg-type] - server = ThreadingHTTPServer(("127.0.0.1", 0), handler) - thread = threading.Thread(target=server.serve_forever, daemon=True) - thread.start() - try: - with urllib.request.urlopen( - f"http://127.0.0.1:{server.server_port}/public/v1/leaderboard", - timeout=5, - ) as response: - payload = json.loads(response.read()) - finally: - server.shutdown() - server.server_close() - thread.join(timeout=5) - - self.assertEqual(payload["schema"], "prism.dashboard.leaderboard.v1") - self.assertEqual(payload["pagination"]["total_count"], 1) - self.assertEqual(payload["rows"][0]["recipient_id"], "miner-a") - - def test_memory_ledger_supports_blocks_chain_state_filter(self) -> None: - ledger = SingleWriterShareLedger() - - for chain_state in ("active", "all", "reversed"): - with self.subTest(chain_state=chain_state): - payload = ledger.dashboard_blocks(page=1, limit=15, chain_state=chain_state) - self.assertEqual(payload["rows"], []) - self.assertEqual(payload["pagination"]["total_count"], 0) - with self.assertRaises(ValueError): - ledger.dashboard_blocks(page=1, limit=15, chain_state="orphaned") - - def test_memory_ledger_pool_snapshot_reports_zero_reorg_counts(self) -> None: - snapshot = SingleWriterShareLedger().dashboard_pool_snapshot( - current_network_difficulty="1", - generated_at=public_api.utc_now_iso(), - ) - - self.assertEqual(snapshot["blocks_reversed_total"], 0) - self.assertEqual(snapshot["blocks_inactive_total"], 0) - - def test_memory_ledger_block_markers_are_empty_safe(self) -> None: - # The in-memory backend records no found_at for its pool blocks, so - # the marker series degrades to an empty response rather than 500ing, - # the way its other dashboard read models degrade. - coordinator = MemoryCoordinator() - handler = make_public_handler(coordinator) # type: ignore[arg-type] - server = ThreadingHTTPServer(("127.0.0.1", 0), handler) - thread = threading.Thread(target=server.serve_forever, daemon=True) - thread.start() - try: - with urllib.request.urlopen( - f"http://127.0.0.1:{server.server_port}/public/v1/block-markers?range=1m&bucket=1h", - timeout=5, - ) as response: - payload = json.loads(response.read()) - finally: - server.shutdown() - server.server_close() - thread.join(timeout=5) - - self.assertEqual(payload["schema"], "prism.dashboard.block-markers.v1") - self.assertEqual(payload["total_blocks"], 0) - self.assertEqual(payload["points"], []) - - def test_memory_ledger_artifact_responses_verify_end_to_end(self) -> None: - # Full path: the real record path persists the canonical text, and the - # HTTP response returns those exact bytes for every manifest kind. - coordinator = MemoryCoordinator() - manifest_set = sample_ctv_manifest_set() - manifest = manifest_set["manifests"][0] # type: ignore[index] - manifest_set_sha256 = sha256_json_hex(manifest_set) - manifest_sha256 = sha256_json_hex(manifest) - coordinator.ledger.persist_ctv_fanout_manifest_set( - block_hash="aa" * 32, - manifest_set=manifest_set, - manifest_set_sha256=manifest_set_sha256, - ) - handler = make_public_handler(coordinator) # type: ignore[arg-type] - server = ThreadingHTTPServer(("127.0.0.1", 0), handler) - thread = threading.Thread(target=server.serve_forever, daemon=True) - thread.start() - try: - for sha256 in (manifest_set_sha256, manifest_sha256): - with self.subTest(sha256=sha256): - with urllib.request.urlopen( - f"http://127.0.0.1:{server.server_port}/public/v1/artifacts/{sha256}", - timeout=5, - ) as response: - body = response.read() - self.assertEqual(hashlib.sha256(body).hexdigest(), sha256) - finally: - server.shutdown() - server.server_close() - thread.join(timeout=5) - - -if __name__ == "__main__": - unittest.main() diff --git a/tests/test_prism_public_http_limits.py b/tests/test_prism_public_http_limits.py deleted file mode 100644 index f3eb02b64..000000000 --- a/tests/test_prism_public_http_limits.py +++ /dev/null @@ -1,151 +0,0 @@ -"""Socket-level admission and request-header deadlines for the public API.""" - -import gc -import http.client -import socket -import threading -import time -import unittest -import weakref -from unittest.mock import patch - -from lab.prism import public_read_service as public - - -class PublicHttpLimitsTests(unittest.TestCase): - def setUp(self): - self.admitted = threading.Event() - handler = public.make_handler(public.PublicReadService(object())) - admitted = self.admitted - self.handler_refs = [] - handler_refs = self.handler_refs - - class ObservedHandler(handler): - def setup(self): - super().setup() - handler_refs.append(weakref.ref(self)) - admitted.set() - - self.server = public.BoundedPublicHTTPServer( - ("127.0.0.1", 0), ObservedHandler, max_connections=1, - timeout_seconds=0.3, - ) - self.thread = threading.Thread( - target=lambda: self.server.serve_forever(poll_interval=0.01), daemon=True, - ) - self.thread.start() - self.addCleanup(self.stop_server) - - def stop_server(self): - self.server.shutdown() - self.server.server_close() - self.thread.join(2) - - def connect(self): - client = socket.create_connection(self.server.server_address, timeout=2) - self.addCleanup(client.close) - return client - - def assert_closed(self, client): - try: - self.assertEqual(client.recv(4096), b"") - except ConnectionResetError: - pass - - def wait_for_capacity(self): - # Wait on the actual admission boundary; no arbitrary recovery sleep. - self.assertTrue(self.server._connection_slots.acquire(timeout=2)) - self.server._connection_slots.release() - - def test_excess_connections_close_and_idle_worker_retires(self): - idle = self.connect() - self.assertTrue(self.admitted.wait(2)) - self.assert_closed(self.connect()) - self.assert_closed(idle) - self.wait_for_capacity() - client = http.client.HTTPConnection(*self.server.server_address, timeout=2) - self.addCleanup(client.close) - client.request("GET", "/missing") - response = client.getresponse() - self.assertEqual(response.status, 404) - self.assertEqual(response.getheader("Connection"), "close") - response.read() - - def test_slow_headers_cannot_extend_total_deadline(self): - client = self.connect() - self.assertTrue(self.admitted.wait(2)) - dispatched = threading.Event() - handler = self.server.RequestHandlerClass - original = handler.do_GET - - def observe_dispatch(request): - dispatched.set() - original(request) - - patcher = patch.object(handler, "do_GET", observe_dispatch) - patcher.start() - self.addCleanup(patcher.stop) - client.sendall(b"GET /missing HTTP/1.1\r\nX-Slow: ") - stopped = threading.Event() - - def drip(): - while not stopped.wait(0.03): - try: - client.sendall(b"x") - except OSError: - return - - writer = threading.Thread(target=drip, daemon=True) - writer.start() - started = time.monotonic() - try: - self.assert_closed(client) - self.assertLess(time.monotonic() - started, 1.5) - finally: - stopped.set() - writer.join(2) - self.wait_for_capacity() - self.assertFalse(dispatched.is_set(), "expired headers reached application dispatch") - - def test_keep_alive_request_cannot_retain_worker(self): - client = self.connect() - client.sendall(b"GET /missing HTTP/1.1\r\nHost: localhost\r\nConnection: keep-alive\r\n\r\n") - body = b"" - while chunk := client.recv(4096): - body += chunk - self.assertIn(b"404", body) - self.assertIn(b"Connection: close", body) - self.wait_for_capacity() - - def test_completed_handler_retires_without_cyclic_gc(self): - was_enabled = gc.isenabled() - gc.disable() - try: - client = self.connect() - client.sendall(b"GET /missing HTTP/1.1\r\nHost: localhost\r\n\r\n") - while client.recv(4096): - pass - self.wait_for_capacity() - self.assertEqual(len(self.handler_refs), 1) - self.assertIsNone(self.handler_refs[0]()) - finally: - if was_enabled: - gc.enable() - - def test_worker_start_failure_returns_admission_slot(self): - request, peer = socket.socketpair() - self.addCleanup(request.close) - self.addCleanup(peer.close) - with patch.object(public.ThreadingHTTPServer, "process_request", side_effect=RuntimeError): - with self.assertRaises(RuntimeError): - self.server.process_request(request, ("127.0.0.1", 1)) - self.wait_for_capacity() - - def test_invalid_limits_refuse_before_binding(self): - for kwargs in ( - {"max_connections": 0}, {"max_connections": 1025}, - {"timeout_seconds": 0}, {"timeout_seconds": float("nan")}, - {"timeout_seconds": float("inf")}, - ): - with self.subTest(kwargs=kwargs), self.assertRaises(public.PublicReadConfigurationError): - public.BoundedPublicHTTPServer(self.server.server_address, object, **kwargs) diff --git a/tests/test_prism_public_read_service.py b/tests/test_prism_public_read_service.py deleted file mode 100644 index 501432b96..000000000 --- a/tests/test_prism_public_read_service.py +++ /dev/null @@ -1,2571 +0,0 @@ -#!/usr/bin/env python3 -"""Pin the extracted PRISM public read tier against the surface it replaced. - -The extraction step of issue #145 moved exactly the 13 original ``/public/v1`` -routes out of the coordinator's audit/ops listener into -``lab/prism/public_read_service.py``; ``/public/v1/block-markers`` joined the -extracted surface later, bringing it to 14 routes. The issue's read-replica back-end is a -separate follow-up -- this service still reads the primary Postgres, through -bounded read slots rather than the writer path. An extraction is -only worth doing if it changes nothing a client can observe and everything the -coordinator can feel, so these tests assert both halves: - -* **Contract equality** -- every extracted route's status, body and cache - headers still come from ``public_api`` unchanged. ``ContractEqualityTests`` - drives the new handler and ``public_api.dispatch`` against the *same* - coordinator and compares. The concrete pre-extraction values are pinned - separately and unchanged by ``tests/test_prism_public_dashboard_api.py``, - whose 70 assertions were written against the coordinator's listener and now - run against this handler. -* **The point of the move** -- ``NoWriterLockReadTests`` drives all 14 routes - against a ledger that raises if any writer-lock read is reached. Moving the - GIL contention while leaving the lock contention would have been motion - without progress. -* **The staleness contract** -- budgets published on every response, 503 rather - than a silently stale body, and no staleness refusal for immutable - content-addressed artifacts. -* **Fail-closed startup** and **coordinator subtraction**. -""" - -from __future__ import annotations - -import json -import os -import threading -import time -from contextlib import contextmanager -from threading import BoundedSemaphore, Lock -import unittest -import urllib.error -import urllib.parse -import urllib.request -from pathlib import Path -from typing import Iterator -from unittest.mock import patch - -from lab.prism import endpoint_registry, public_api, public_read_service -from lab.prism.audit_http import AuditHttpConfig, AuditHttpFacade -from lab.prism.share_ledger import PsqlShareLedger, ReadOnlyLedgerError - -# The public fixtures these routes are exercised against already exist, written -# against the coordinator's listener before the extraction. Reusing them rather -# than restating them is deliberate: a second copy could drift, and then the -# contract test would be comparing the new service to a fixture that no longer -# describes the old behaviour. -from tests.test_prism_public_dashboard_api import ( - DirectCoinbasePublicLedger, - FakeCoordinator, - FakePublicLedger, -) - - -ROOT = Path(__file__).resolve().parents[1] -AUDIT_HTTP_PATH = ROOT / "lab" / "prism" / "audit_http.py" - -FROZEN_NOW = "2026-01-01T00:00:00Z" - -RECIPIENT_ID = "miner-a" -BLOCK_HASH = FakePublicLedger.block_hash -FANOUT_TXID = FakePublicLedger.fanout_txid -ARTIFACT_SHA256 = FakePublicLedger.audit_bundle_sha256 - -# One concrete request per extracted route. EXTRACTED_ROUTE_COUNT is asserted -# against the registry so a new public route cannot be added without landing -# here too. -EXTRACTED_ROUTES: tuple[tuple[str, str], ...] = ( - ("/public/v1/pool-summary", ""), - ("/public/v1/blocks", ""), - ("/public/v1/leaderboard", ""), - ("/public/v1/hashrate-series", ""), - ("/public/v1/block-markers", ""), - ("/public/v1/mining-configuration", ""), - (f"/public/v1/miners/{RECIPIENT_ID}", ""), - (f"/public/v1/miners/{RECIPIENT_ID}/earnings", ""), - (f"/public/v1/miners/{RECIPIENT_ID}/payouts", ""), - (f"/public/v1/miners/{RECIPIENT_ID}/workers", ""), - (f"/public/v1/blocks/{BLOCK_HASH}/settlement-artifacts", ""), - ("/public/v1/fanouts/pending", ""), - (f"/public/v1/fanouts/{FANOUT_TXID}", ""), - (f"/public/v1/artifacts/{ARTIFACT_SHA256}", ""), -) - -EXTRACTED_ROUTE_COUNT = 14 - -CACHE_HEADERS = ("Cache-Control", "CDN-Cache-Control", "Vercel-CDN-Cache-Control", "Age") - -WRITER_LOCK_LEDGER_METHODS = ( - "current_owed_balances", - "recipient_payout_history", - "audit_bundle", - "audit_bundle_by_commitment", - "audit_block_payouts", - "audit_share_window", - "carry_forward_integrity_report", - # public_api.worker_rows falls back to this full-table read when - # dashboard_miner_worker_rows is absent; on Postgres it is a writer-lock - # read like the rest, so the public tier must never reach it either. - "all_shares", -) - - -class WriterLockReached(AssertionError): - """Raised when a public route reaches a read that fences on the writer.""" - - -class ServedResponse: - """One HTTP response, reduced to what the contract actually covers.""" - - def __init__(self, status: int, body: bytes, headers: dict[str, str]) -> None: - self.status = status - self.body = body - self.headers = headers - - @property - def payload(self) -> object: - return json.loads(self.body) - - def cache_headers(self) -> dict[str, str | None]: - return {name: self.headers.get(name) for name in CACHE_HEADERS} - - -class ServiceHarness: - """A bound public read service over one coordinator, for the test's lifetime.""" - - def __init__( - self, - coordinator: object, - *, - response_cache: object | None = None, - readiness: object | None = None, - metrics: object | None = None, - replica: object | None = None, - ) -> None: - self.service = public_read_service.PublicReadService( - coordinator, - response_cache=response_cache, # type: ignore[arg-type] - metrics=metrics, # type: ignore[arg-type] - readiness=readiness, # type: ignore[arg-type] - replica=replica, # type: ignore[arg-type] - ) - handler = public_read_service.make_handler(self.service) - self.server = public_read_service.BoundedPublicHTTPServer(("127.0.0.1", 0), handler) - self.thread = threading.Thread(target=self.server.serve_forever, daemon=True) - self.thread.start() - self.base_url = f"http://127.0.0.1:{self.server.server_port}" - # Loopback fixtures must never inherit a cached process-wide proxy. - self.opener = urllib.request.build_opener(urllib.request.ProxyHandler({})) - - def get(self, path_and_query: str) -> ServedResponse: - url = self.base_url + path_and_query - try: - with self.opener.open(url, timeout=5) as response: - return ServedResponse( - response.status, - response.read(), - dict(response.headers.items()), - ) - except urllib.error.HTTPError as error: - with error: - return ServedResponse( - error.code, - error.read(), - dict(error.headers.items()), - ) - - def close(self) -> None: - self.server.shutdown() - self.server.server_close() - self.thread.join(timeout=5) - - -def request_path(route: tuple[str, str]) -> str: - path, query = route - return f"{path}?{query}" if query else path - - -class RegistryAgreementTests(unittest.TestCase): - """The routes exercised here must be exactly the routes the registry extracts.""" - - def test_every_extracted_registry_path_is_covered(self) -> None: - covered = set() - for path, _query in EXTRACTED_ROUTES: - endpoint = endpoint_registry.endpoint_for_request_path(path) - self.assertIsNotNone(endpoint, f"{path} resolves to no registry endpoint") - assert endpoint is not None - covered.add(endpoint.primary_path) - self.assertEqual(set(endpoint_registry.extracted_paths()), covered) - - def test_the_extracted_surface_is_still_fourteen_routes(self) -> None: - self.assertEqual( - EXTRACTED_ROUTE_COUNT, - len(endpoint_registry.extracted_paths()), - ) - self.assertEqual(EXTRACTED_ROUTE_COUNT, len(EXTRACTED_ROUTES)) - - -class ContractEqualityTests(unittest.TestCase): - """Same status, same body, same cache headers as dispatch() produces. - - Time is frozen for both sides so ``generated_at`` cannot make two - structurally identical payloads compare unequal. - """ - - def setUp(self) -> None: - self.coordinator = FakeCoordinator(ledger=FullReadModelLedger()) - self.harness = ServiceHarness(self.coordinator) - self.addCleanup(self.harness.close) - patcher = patch.object(public_api, "utc_now_iso", return_value=FROZEN_NOW) - patcher.start() - self.addCleanup(patcher.stop) - - def expected_success(self, path: str, query: str) -> tuple[int, object, dict[str, str]]: - parsed_query = urllib.parse.parse_qs(query) - status, payload = public_api.dispatch(self.coordinator, path, parsed_query) - policy = public_api.public_cache_policy(path) - budget = public_read_service.staleness_budget_for_path(path) - if budget is not None: - ttl = min(policy.ttl_seconds, int(budget)) - policy = public_api.PublicCachePolicy( - ttl, max(0, min(policy.stale_while_revalidate_seconds, int(budget) - ttl)), - policy.immutable, - ) - # A fresh service cache serves the first request as a MISS at age 0, - # which is exactly what the coordinator's handle_public did. - headers = public_api.public_cache_headers( - policy, - cache_state="MISS", - age_seconds=0, - ) - return status, payload, headers - - def test_every_extracted_route_matches_dispatch(self) -> None: - for route in EXTRACTED_ROUTES: - path, query = route - with self.subTest(path=path): - served = self.harness.get(request_path(route)) - status, payload, headers = self.expected_success(path, query) - - self.assertEqual(status, served.status) - self.assertEqual(payload, served.payload) - # Byte-for-byte, not just structurally: the framing (sorted - # keys, trailing newline) is part of what clients receive. - self.assertEqual( - json.dumps(payload, sort_keys=True).encode() + b"\n", - served.body, - ) - for name in CACHE_HEADERS: - self.assertEqual( - headers.get(name), - served.headers.get(name), - f"{path}: {name} changed across the extraction", - ) - - def test_mining_configuration_highdiff_endpoint_survives_the_extraction(self) -> None: - # This process runs no Stratum listener, so the High-diff endpoint in - # the rendered body comes only from PRISM_STRATUM_HIGHDIFF_PORT in this - # process's own environment (compose passes it through; see - # tests/test_prism_compose_profile.py). With the env set the served - # body must still match dispatch and must carry the High-diff endpoint. - route = ("/public/v1/mining-configuration", "") - with patch.dict( - os.environ, - { - "PRISM_PUBLIC_STRATUM_URL": "stratum+tcp://public-pool.example:3335", - "PRISM_STRATUM_HIGHDIFF_PORT": "4334", - }, - clear=True, - ): - served = self.harness.get(request_path(route)) - status, payload, headers = self.expected_success(*route) - - self.assertEqual(status, served.status) - self.assertEqual(payload, served.payload) - for name in CACHE_HEADERS: - self.assertEqual(headers.get(name), served.headers.get(name)) - - endpoints = served.payload["configurations"][0]["stratum_endpoints"] - self.assertEqual( - ["Primary", "High-diff"], - [endpoint["label"] for endpoint in endpoints], - ) - self.assertEqual( - "stratum+tcp://public-pool.example:4334", - endpoints[1]["url"], - ) - self.assertEqual(4334, endpoints[1]["default_port"]) - - def test_harness_bypasses_a_cached_proxy_with_no_proxy_environment_cleared(self) -> None: - proxy_opener = urllib.request.build_opener( - urllib.request.ProxyHandler({"http": "http://127.0.0.1:9"}) - ) - with patch.object(urllib.request, "_opener", proxy_opener), patch.dict(os.environ, {}, clear=True): - self.assertEqual(self.harness.get("/public/v1/blocks").status, 200) - - def test_content_type_is_unchanged(self) -> None: - for route in EXTRACTED_ROUTES: - with self.subTest(path=route[0]): - served = self.harness.get(request_path(route)) - self.assertEqual("application/json", served.headers.get("Content-Type")) - - def test_not_found_error_matches_dispatch(self) -> None: - # A block with no settlement rows: dispatch raises PublicApiError(404). - unknown_block = "f" * 64 - path = f"/public/v1/blocks/{unknown_block}/settlement-artifacts" - with self.assertRaises(public_api.PublicApiError) as raised: - public_api.dispatch(self.coordinator, path, {}) - expected = raised.exception - - served = self.harness.get(path) - - self.assertEqual(404, expected.status) - self.assertEqual(expected.status, served.status) - self.assertEqual( - public_api.error_payload(expected.code, expected.message), - served.payload, - ) - self.assertEqual("no-store", served.headers.get("Cache-Control")) - # Errors are not cached, so they carry no shared-cache or Age headers. - self.assertIsNone(served.headers.get("CDN-Cache-Control")) - self.assertIsNone(served.headers.get("Age")) - - def test_public_api_error_matches_dispatch(self) -> None: - # An explicit PublicApiError raised by dispatch's own validation. - path, query = "/public/v1/leaderboard", "window=bogus" - with self.assertRaises(public_api.PublicApiError) as raised: - public_api.dispatch(self.coordinator, path, urllib.parse.parse_qs(query)) - expected = raised.exception - - served = self.harness.get(f"{path}?{query}") - - self.assertEqual(400, expected.status) - self.assertEqual("invalid_window", expected.code) - self.assertEqual(expected.status, served.status) - self.assertEqual( - public_api.error_payload(expected.code, expected.message), - served.payload, - ) - self.assertEqual("prism.dashboard.error.v1", served.payload["schema"]) - self.assertEqual("no-store", served.headers.get("Cache-Control")) - - def test_reward_filter_conflict_matches_dispatch(self) -> None: - path, query = "/public/v1/leaderboard", "window=reward&search=a&recipient_id=miner-a" - with self.assertRaises(public_api.PublicApiError) as raised: - public_api.dispatch(self.coordinator, path, urllib.parse.parse_qs(query)) - - served = self.harness.get(f"{path}?{query}") - - self.assertEqual(raised.exception.status, served.status) - self.assertEqual( - public_api.error_payload(raised.exception.code, raised.exception.message), - served.payload, - ) - - def test_unknown_public_route_is_a_public_schema_404(self) -> None: - served = self.harness.get("/public/v1/not-a-route") - - self.assertEqual(404, served.status) - self.assertEqual("prism.dashboard.error.v1", served.payload["schema"]) - self.assertEqual("not_found", served.payload["error"]["code"]) - - def test_a_second_request_is_served_from_cache_with_a_real_age(self) -> None: - # The response cache moved with the routes; a HIT must still report Age. - route = ("/public/v1/blocks", "") - first = self.harness.get(request_path(route)) - second = self.harness.get(request_path(route)) - - self.assertEqual(200, first.status) - self.assertEqual(first.body, second.body) - self.assertEqual("0", second.headers.get("Age")) - - -class FullReadModelLedger(FakePublicLedger): - """A ledger implementing every ``dashboard_*`` read model, as Postgres does. - - ``FakePublicLedger`` deliberately leaves some read models out so the - pre-extraction tests could exercise ``public_api``'s duck-typed fallbacks. - The extracted service never runs those fallbacks in production -- - ``PsqlShareLedger`` implements every read model -- and two of the fallbacks - derive hashrate from ``all_shares()`` relative to wall-clock now, which - makes them unusable for an equality comparison. This subclass fills the - gaps with fixed rows so a route's answer depends only on its inputs. - """ - - def dashboard_miner_share_summary(self, *, recipient_id: str) -> dict[str, object]: - return { - "hashrate_ths": {"h1": "1.5", "h3": "2.5", "h24": "3.5"}, - "accepted_3h": 3, - "accepted_difficulty_3h": "600000", - "last_share_at": "2026-06-26T20:44:53Z", - "share_percent": "60", - } - - def dashboard_miner_worker_rows( - self, - *, - recipient_id: str, - page: int, - limit: int, - search: str | None, - hide_inactive: bool, - ) -> dict[str, object]: - rows = [ - { - "worker_name": "rig-1", - "status": "active", - "last_share_at": "2026-06-26T20:44:53Z", - "hashrate_ths_60s": "1.5", - "hashrate_ths_3h": "2.5", - } - ] - if search: - rows = [row for row in rows if search.lower() in str(row["worker_name"]).lower()] - offset = (page - 1) * limit - return { - "pagination": public_api.pagination(page, limit, len(rows)), - "rows": rows[offset : offset + limit], - "active_count": len(rows), - } - - def dashboard_miner_owed_balance_bits(self, *, recipient_id: str) -> int: - return sum( - int(balance["balance_sats"]) - for balance in self.current_owed_balances() - if balance["recipient_id"] == recipient_id - ) - - def dashboard_direct_coinbase_settlement( - self, - *, - block_hash: str, - ) -> dict[str, object] | None: - return None - - # The Postgres implementations of these two are independent SQL. The base - # fixture builds them from recipient_payout_history(), which is itself a - # writer-lock read, so they are re-based here on the raw fixture rows. - def dashboard_miner_payout_rows( - self, - *, - recipient_id: str, - page: int, - limit: int, - ) -> dict[str, object]: - rows = [public_api.miner_payout_row(row) for row in self._payout_history()] - offset = (page - 1) * limit - return { - "pagination": public_api.pagination(page, limit, len(rows)), - "rows": rows[offset : offset + limit], - } - - def dashboard_miner_earning_rows( - self, - *, - recipient_id: str, - page: int, - limit: int, - ) -> dict[str, object]: - rows = [public_api.miner_earning_row(row) for row in self._payout_history()] - offset = (page - 1) * limit - return { - "pagination": public_api.pagination(page, limit, len(rows)), - "rows": rows[offset : offset + limit], - } - - -class NoWriterLockLedger(FullReadModelLedger): - """A public ledger whose writer-lock reads are all landmines. - - Every method in WRITER_LOCK_LEDGER_METHODS is one ``public_api`` would - reach only by falling back off a ``dashboard_*`` read model. On the real - Postgres ledger each of them runs under - ``_operation_gate(self._lock, "writer lock")``, so reaching one from a - public route serializes a dashboard poll against the block-landing writer. - The extraction's whole purpose is that none of them are reachable. - """ - - def __init__(self) -> None: - super().__init__() - self.settlement_calls = 0 - self.owed_balance_calls = 0 - - def dashboard_miner_owed_balance_bits(self, *, recipient_id: str) -> int: - self.owed_balance_calls += 1 - return 1234 - - def dashboard_direct_coinbase_settlement( - self, - *, - block_hash: str, - ) -> dict[str, object] | None: - self.settlement_calls += 1 - return None - - def dashboard_miner_share_summary(self, *, recipient_id: str) -> dict[str, object]: - # FakePublicLedger has no share-summary read model, so public_api would - # fall back to all_shares(). PsqlShareLedger does have one, and it is a - # read-slot query; supply it so the fallback is genuinely unreachable. - return { - "hashrate_ths": {"m1": "0", "m5": "0", "m10": "0", "h3": "2.5", "h24": "2.5"}, - "accepted_3h": 2, - "accepted_difficulty_3h": "180", - "last_share_at": "2026-06-26T20:44:53Z", - "share_percent": "60", - } - - def dashboard_miner_payout_rows( - self, - *, - recipient_id: str, - page: int, - limit: int, - ) -> dict[str, object]: - # The inherited version routes through recipient_payout_history, which - # is a landmine here. On PsqlShareLedger this read model is its own - # read-slot query and touches no fenced path. - rows = [public_api.miner_payout_row(row) for row in self._payout_history()] - offset = (page - 1) * limit - return { - "pagination": public_api.pagination(page, limit, len(rows)), - "rows": rows[offset : offset + limit], - } - - def dashboard_miner_earning_rows( - self, - *, - recipient_id: str, - page: int, - limit: int, - ) -> dict[str, object]: - rows = [public_api.miner_earning_row(row) for row in self._payout_history()] - offset = (page - 1) * limit - return { - "pagination": public_api.pagination(page, limit, len(rows)), - "rows": rows[offset : offset + limit], - } - - -def _install_writer_lock_landmines(ledger: object) -> None: - for name in WRITER_LOCK_LEDGER_METHODS: - def explode(*_args: object, _name: str = name, **_kwargs: object) -> object: - raise WriterLockReached( - f"public route reached the writer-lock read {_name}()" - ) - - setattr(ledger, name, explode) - - -class NoWriterLockReadTests(unittest.TestCase): - """No extracted route may reach a read that fences on the writer lock. - - This is the test that proves the extraction achieved its purpose. If it - fails, the public tier still serializes against the lease-holding writer - and the move bought only GIL separation. - """ - - def setUp(self) -> None: - self.ledger = NoWriterLockLedger() - _install_writer_lock_landmines(self.ledger) - self.coordinator = FakeCoordinator(ledger=self.ledger) - self.harness = ServiceHarness(self.coordinator) - self.addCleanup(self.harness.close) - - def test_no_extracted_route_touches_a_writer_lock_read(self) -> None: - for route in EXTRACTED_ROUTES: - with self.subTest(path=route[0]): - served = self.harness.get(request_path(route)) - # A landmine surfaces as a 500 in the public error schema, - # because the handler converts every unexpected exception. Any - # 5xx here means a writer-lock read was reached. - self.assertLess( - served.status, - 500, - f"{route[0]} reached a writer-lock read (status {served.status})", - ) - - def test_the_landmines_are_actually_armed(self) -> None: - # Guard against the previous test passing vacuously because the stub - # stopped raising. - for name in WRITER_LOCK_LEDGER_METHODS: - with self.subTest(method=name): - with self.assertRaises(WriterLockReached): - getattr(self.ledger, name)() - - def test_the_miner_page_uses_the_owed_balance_read_model(self) -> None: - # /public/v1/miners/{recipient_id} is the most-polled route on the - # surface; it must reach the per-recipient read model, not the - # every-recipient writer-lock dump. - served = self.harness.get(f"/public/v1/miners/{RECIPIENT_ID}") - - self.assertEqual(200, served.status) - self.assertEqual(1, self.ledger.owed_balance_calls) - - def test_direct_coinbase_settlement_uses_the_read_model(self) -> None: - # A block the CTV manifest-set read does not know: this is the branch - # that used to fall through to audit_bundle() under the writer lock. - served = self.harness.get( - f"/public/v1/blocks/{'0e' * 32}/settlement-artifacts" - ) - - self.assertEqual(404, served.status) - self.assertEqual(1, self.ledger.settlement_calls) - - -class StubCache: - """A response cache that answers with a chosen age, to drive staleness.""" - - def __init__(self, age_seconds: int, cache_state: str = "HIT") -> None: - self.age_seconds = age_seconds - self.cache_state = cache_state - self.stale_windows: list[int] = [] - - def get_or_compute( - self, - *, - key: object, - ttl_seconds: int, - compute, - stale_while_revalidate_seconds: int = 0, - ) -> tuple: - del key, ttl_seconds - self.stale_windows.append(stale_while_revalidate_seconds) - status, payload = compute() - return status, payload, self.cache_state, self.age_seconds - - -class StalenessContractTests(unittest.TestCase): - """Publish the budget, refuse past it, and never refuse immutable content.""" - - def harness_at_age(self, age_seconds: int) -> ServiceHarness: - harness = ServiceHarness( - FakeCoordinator(ledger=FullReadModelLedger()), - response_cache=StubCache(age_seconds), - ) - self.addCleanup(harness.close) - return harness - - def test_every_extracted_response_publishes_its_budget(self) -> None: - harness = ServiceHarness(FakeCoordinator(ledger=FullReadModelLedger())) - self.addCleanup(harness.close) - for route in EXTRACTED_ROUTES: - path = route[0] - with self.subTest(path=path): - served = harness.get(request_path(route)) - budget = served.headers.get( - public_read_service.STALENESS_BUDGET_HEADER - ) - self.assertIsNotNone( - budget, - f"{path} served no staleness budget", - ) - endpoint = endpoint_registry.endpoint_for_request_path(path) - assert endpoint is not None - if endpoint.immutable_content: - self.assertEqual( - public_read_service.UNBOUNDED_STALENESS_BUDGET, - budget, - ) - else: - self.assertEqual( - str(int(endpoint.max_staleness_seconds or 0)), - budget, - ) - - def test_a_response_past_its_budget_is_refused(self) -> None: - # One second past the largest budget on the surface, so every - # non-immutable route is over its own. - budgets = [ - endpoint.max_staleness_seconds or 0 - for endpoint in endpoint_registry.ENDPOINTS - if endpoint.disposition is endpoint_registry.Disposition.EXTRACT - ] - harness = self.harness_at_age(int(max(budgets)) + 1) - - for route in EXTRACTED_ROUTES: - path = route[0] - endpoint = endpoint_registry.endpoint_for_request_path(path) - assert endpoint is not None - if endpoint.immutable_content: - continue - with self.subTest(path=path): - served = harness.get(request_path(route)) - - self.assertEqual(503, served.status) - self.assertEqual("no-store", served.headers.get("Cache-Control")) - self.assertEqual( - "prism.dashboard.error.v1", - served.payload["schema"], - ) - message = served.payload["error"]["message"] - budget = endpoint.max_staleness_seconds or 0 - self.assertIn(f"{int(budget)}", message) - self.assertIn(f"{int(max(budgets)) + 1}", message) - - def test_a_refusal_still_reports_the_budget_and_the_observed_age(self) -> None: - harness = self.harness_at_age(10_000) - served = harness.get("/public/v1/pool-summary") - - self.assertEqual(503, served.status) - self.assertEqual( - "90", - served.headers.get(public_read_service.STALENESS_BUDGET_HEADER), - ) - self.assertEqual("10000", served.headers.get("Age")) - - def test_a_response_inside_its_budget_is_served(self) -> None: - # 60s is inside pool-summary's 90s budget and outside the 15s floor - # routes, so this also pins that the check is per-route. - harness = self.harness_at_age(60) - - self.assertEqual(200, harness.get("/public/v1/pool-summary").status) - self.assertEqual(503, harness.get("/public/v1/blocks").status) - - def test_immutable_artifacts_never_refuse_for_staleness(self) -> None: - # Content-addressed: a body that hashes to the requested sha256 is - # correct however old it is, so age must not turn it into a 503. - harness = self.harness_at_age(10_000_000) - served = harness.get(f"/public/v1/artifacts/{ARTIFACT_SHA256}") - - self.assertEqual(200, served.status) - self.assertEqual( - public_read_service.UNBOUNDED_STALENESS_BUDGET, - served.headers.get(public_read_service.STALENESS_BUDGET_HEADER), - ) - - def test_refusals_are_counted_in_metrics(self) -> None: - harness = self.harness_at_age(10_000) - harness.get("/public/v1/pool-summary") - - metrics = harness.get("/metrics") - self.assertEqual(200, metrics.status) - self.assertIn( - "qbit_prism_public_staleness_refusals_total 1", - metrics.body.decode(), - ) - - -class StatementTimeoutScopeLedger(FullReadModelLedger): - """FullReadModelLedger plus PsqlShareLedger's operation_timeout scope shape.""" - - def __init__(self) -> None: - super().__init__() - self.statement_timeout_budgets: list[float] = [] - - @contextmanager - def operation_timeout(self, timeout_seconds: float) -> Iterator[None]: - self.statement_timeout_budgets.append(timeout_seconds) - yield - - -class StatementTimeoutWiringTests(unittest.TestCase): - """The per-request database deadline arms exactly where a read slot is held. - - The deadline's own semantics (budget value, env kill switch, the - read_timeout 503, cache non-poisoning) are pinned in - tests/test_prism_public_dashboard_api.py; what belongs to this service is - the classification -- path_occupies_read_slot(), which is deliberately - wider than the staleness gates' path_reads_database(): a cold immutable - artifact lookup holds a Postgres read slot exactly like any other cold - read even though its body, once found, is correct at any age. - """ - - def test_the_scope_wraps_every_read_slot_route_and_nothing_else(self) -> None: - ledger = StatementTimeoutScopeLedger() - harness = ServiceHarness(FakeCoordinator(ledger=ledger)) - self.addCleanup(harness.close) - - for route in EXTRACTED_ROUTES: - path = route[0] - with self.subTest(path=path): - armed_before = len(ledger.statement_timeout_budgets) - served = harness.get(request_path(route)) - self.assertEqual(200, served.status) - armed = len(ledger.statement_timeout_budgets) > armed_before - # mining-configuration touches no read slot; every other - # extracted route -- the immutable artifact route included -- - # occupies one on a cold request and owes the deadline. - self.assertEqual( - public_read_service.path_occupies_read_slot(path), armed - ) - - def test_artifacts_owe_the_deadline_but_not_the_staleness_gates(self) -> None: - path = f"/public/v1/artifacts/{ARTIFACT_SHA256}" - self.assertTrue(public_read_service.path_occupies_read_slot(path)) - # Deliberately split from path_reads_database: immutable content is - # exempt from replica/outage refusals, never from the deadline. - self.assertFalse(public_read_service.path_reads_database(path)) - - def test_mining_configuration_arms_nothing(self) -> None: - path = "/public/v1/mining-configuration" - self.assertFalse(public_read_service.path_occupies_read_slot(path)) - self.assertFalse(public_read_service.path_reads_database(path)) - - def test_one_budget_covers_a_multi_read_dispatch(self) -> None: - """The scope is entered once around the whole dispatch, not per read. - - The miner detail route performs several sequential ledger reads; a - per-statement scope would hand each a fresh budget and let the - request run many multiples of the configured limit. Exactly one - operation_timeout entry proves the deadline spans them all. - """ - ledger = StatementTimeoutScopeLedger() - harness = ServiceHarness(FakeCoordinator(ledger=ledger)) - self.addCleanup(harness.close) - - served = harness.get(f"/public/v1/miners/{RECIPIENT_ID}") - - self.assertEqual(200, served.status) - self.assertEqual(1, len(ledger.statement_timeout_budgets)) - - -class StaleServeTests(unittest.TestCase): - """Read-service wiring for the in-process stale-while-revalidate window.""" - - def setUp(self) -> None: - self.cache = public_api.PublicResponseCache() - self.metrics = public_read_service.ServiceMetrics() - self.harness = ServiceHarness( - FakeCoordinator(ledger=FullReadModelLedger()), - response_cache=self.cache, - metrics=self.metrics, - ) - self.addCleanup(self.harness.close) - - def age_blocks_entry(self, age_seconds: float) -> None: - """Age the warm /public/v1/blocks entry to a chosen served age. - - The route's default TTL is 5s, so any age beyond that is expired; - the bounds are edited directly for the reason cached_entry() states. - """ - entry = cached_entry(self.cache, "/public/v1/blocks") - assert entry is not None - entry.stored_at = time.monotonic() - age_seconds # type: ignore[union-attr] - entry.expires_at = entry.stored_at + 5.0 # type: ignore[union-attr] - - def test_a_stale_entry_inside_the_budget_is_served_with_its_age(self) -> None: - warm = self.harness.get("/public/v1/blocks") - self.age_blocks_entry(8.0) # 3s past the 5s TTL, inside the 15s budget - - served = self.harness.get("/public/v1/blocks") - - self.assertEqual(200, served.status) - self.assertEqual(warm.body, served.body) - self.assertEqual("8", served.headers.get("Age")) - - def test_cdn_policy_cannot_exceed_the_route_budget(self) -> None: - for ttl, swr, expected in ( - (5, 30, "public, max-age=5, stale-while-revalidate=10"), - (60, 30, "public, max-age=15"), - (5, 0, "public, max-age=5"), - ): - with self.subTest(ttl=ttl, swr=swr), patch.object( - public_api, "public_cache_policy", - return_value=public_api.PublicCachePolicy(ttl, swr), - ): - served = self.harness.get(f"/public/v1/blocks?limit=1&case={ttl}-{swr}") - self.assertEqual(served.status, 200) - self.assertEqual(served.headers["X-Prism-Staleness-Budget-Seconds"], "15") - self.assertEqual(served.headers["CDN-Cache-Control"], expected) - self.assertEqual(served.headers["Vercel-CDN-Cache-Control"], expected) - - def test_stale_serves_are_counted_in_cache_metrics(self) -> None: - self.harness.get("/public/v1/blocks") - self.age_blocks_entry(8.0) - self.harness.get("/public/v1/blocks") - - body = self.harness.get("/metrics").body.decode() - - self.assertIn('qbit_prism_public_cache_total{state="stale"} 1', body) - - def test_the_window_is_clamped_to_the_route_staleness_budget(self) -> None: - """A stale serve must never turn an age the blocking path handled into a 503. - - Age 20 is inside the raw 30s stale-while-revalidate window but past - the 15s budget /public/v1/blocks publishes. Unclamped, the cache - would serve it STALE and the unconditional budget check would refuse - it; clamped to budget minus TTL, the request falls through to the - blocking recompute exactly as it did before the window existed. - """ - self.harness.get("/public/v1/blocks") - self.age_blocks_entry(20.0) - - served = self.harness.get("/public/v1/blocks") - - self.assertEqual(200, served.status) - self.assertEqual("0", served.headers.get("Age")) - - def test_an_outage_withdraws_the_window(self) -> None: - """While degraded, only unexpired entries are served (the #164 contract). - - The stale-while-revalidate window must not widen what an outage may - answer with, and its background refresh must not become a second - origin path the outage gate does not see. - """ - probe = ScriptedReadiness() - readiness = public_read_service.ReadinessProbe(probe, interval_seconds=3600) - readiness.check_once() - cache = public_api.PublicResponseCache() - harness = ServiceHarness( - FakeCoordinator(ledger=FullReadModelLedger()), - response_cache=cache, - readiness=readiness, - ) - self.addCleanup(harness.close) - harness.get("/public/v1/blocks") - entry = cached_entry(cache, "/public/v1/blocks") - assert entry is not None - entry.stored_at = time.monotonic() - 8.0 # type: ignore[union-attr] - entry.expires_at = entry.stored_at + 5.0 # type: ignore[union-attr] - probe.healthy = False - readiness.check_once() - - served = harness.get("/public/v1/blocks") - - self.assertEqual(503, served.status) - self.assertEqual("upstream_unavailable", served.payload["error"]["code"]) - - -class FakeClock: - """Monotonic time the test moves by hand.""" - - def __init__(self) -> None: - self.now = 1_000.0 - - def __call__(self) -> float: - return self.now - - def advance(self, seconds: float) -> None: - self.now += seconds - - -class ScriptedReplica: - """A read_replica_status() whose answers -- and failures -- the test picks.""" - - def __init__(self, *answers: object) -> None: - self.answers = list(answers) - self.calls = 0 - - def __call__(self) -> dict[str, object]: - self.calls += 1 - answer = self.answers[min(self.calls, len(self.answers)) - 1] - if isinstance(answer, Exception): - raise answer - assert isinstance(answer, dict) - return answer - - -def standby(heartbeat: float | None = 1.0, replay_lag: float | None = 2.0) -> dict: - return { - "in_recovery": True, - "receiver_heartbeat_age_seconds": heartbeat, - "replay_lag_seconds": replay_lag, - "apply_backlog_bytes": 4096, - } - - -PRIMARY_STATUS = { - "in_recovery": False, - "receiver_heartbeat_age_seconds": None, - "replay_lag_seconds": None, - "apply_backlog_bytes": None, -} - - -class ReplicaContractTests(unittest.TestCase): - """Serve from a live standby, or refuse and say which part failed. - - The bound is on the walreceiver heartbeat rather than replay lag; see - ReplicaProbe. These tests drive the probe directly with a fake clock, so - they pin the freshness rules without a Postgres. - """ - - def probe( - self, - *answers: object, - max_lag_seconds: float = 60.0, - ) -> tuple[public_read_service.ReplicaProbe, FakeClock]: - clock = FakeClock() - probe = public_read_service.ReplicaProbe( - ScriptedReplica(*answers), - max_lag_seconds=max_lag_seconds, - monotonic=clock, - ) - return probe, clock - - def harness(self, probe: object, **kwargs: object) -> ServiceHarness: - harness = ServiceHarness( - FakeCoordinator(ledger=FullReadModelLedger()), - replica=probe, - **kwargs, # type: ignore[arg-type] - ) - self.addCleanup(harness.close) - return harness - - def test_before_the_first_probe_replica_routes_are_warming_up(self) -> None: - probe, _ = self.probe(standby()) - served = self.harness(probe).get("/public/v1/blocks") - - self.assertEqual(503, served.status) - self.assertIn("warming up", served.payload["error"]["message"]) - - def test_a_live_standby_is_served(self) -> None: - probe, _ = self.probe(standby()) - probe.check() - served = self.harness(probe).get("/public/v1/blocks") - - self.assertEqual(200, served.status) - - def test_a_writable_primary_is_refused(self) -> None: - probe, _ = self.probe(PRIMARY_STATUS) - self.assertFalse(probe.check()) - served = self.harness(probe).get("/public/v1/blocks") - - self.assertEqual(503, served.status) - self.assertIn("not in recovery", served.payload["error"]["message"]) - - def test_a_refusal_uses_the_documented_public_error_enum(self) -> None: - # PUBLIC_ERROR_CODES silently rewrites an unknown code to - # internal_error, so an undocumented one would ship a 503 that says - # "internal error" and never fail a test that only reads the message. - probe, _ = self.probe(PRIMARY_STATUS) - probe.check() - served = self.harness(probe).get("/public/v1/blocks") - - self.assertEqual("prism.dashboard.error.v1", served.payload["schema"]) - self.assertEqual("upstream_unavailable", served.payload["error"]["code"]) - - def test_a_disconnected_walreceiver_is_refused(self) -> None: - probe, _ = self.probe(standby(heartbeat=None)) - probe.check() - served = self.harness(probe).get("/public/v1/blocks") - - self.assertEqual(503, served.status) - self.assertIn("not connected", served.payload["error"]["message"]) - - def test_a_heartbeat_past_the_bound_is_refused(self) -> None: - probe, _ = self.probe(standby(heartbeat=61.0), max_lag_seconds=60.0) - probe.check() - served = self.harness(probe).get("/public/v1/blocks") - - self.assertEqual(503, served.status) - self.assertIn("silent for", served.payload["error"]["message"]) - - def test_a_failing_probe_ages_its_last_good_answer_out(self) -> None: - """The gate fails closed: a dead probe must not pin a fresh verdict. - - This is the whole reason freshness is computed as "heartbeat age when - last probed, plus how long ago that was". Without the second term a - replica that stopped answering entirely keeps serving 200 forever off - its last good snapshot. - """ - probe, clock = self.probe( - standby(heartbeat=1.0), - RuntimeError("connection refused"), - max_lag_seconds=60.0, - ) - probe.check() - harness = self.harness(probe) - - # Probe starts failing; the snapshot is still inside the bound. - for _ in range(100): - with self.assertRaises(RuntimeError): - probe.check() - clock.advance(30.0) - self.assertEqual(200, harness.get("/public/v1/blocks").status) - - # Past the bound, the same unchanged snapshot must now refuse. - clock.advance(30.0) - served = harness.get("/public/v1/blocks") - self.assertEqual(503, served.status) - self.assertIn("silent for", served.payload["error"]["message"]) - self.assertIn("connection refused", served.payload["error"]["message"]) - - def test_a_refusal_is_not_masked_by_a_warm_cache(self) -> None: - """The gate runs before the cache, not after. - - A replica outage does not expire cache entries, so a check placed after - the cache would keep answering 200 from the last good snapshot for as - long as traffic kept it warm. - """ - probe, clock = self.probe(standby(heartbeat=1.0), max_lag_seconds=60.0) - probe.check() - harness = self.harness(probe, response_cache=StubCache(0)) - - self.assertEqual(200, harness.get("/public/v1/blocks").status) - clock.advance(120.0) - self.assertEqual(503, harness.get("/public/v1/blocks").status) - - def test_routes_that_read_no_replica_state_keep_serving(self) -> None: - probe, _ = self.probe(PRIMARY_STATUS) - probe.check() - harness = self.harness(probe) - - self.assertEqual(503, harness.get("/public/v1/blocks").status) - self.assertEqual( - 200, harness.get("/public/v1/mining-configuration").status - ) - - def test_immutable_artifacts_are_not_replica_gated(self) -> None: - # Content-addressed: the body either hashes to the requested sha256 or - # it does not, which replication lag cannot change. The registry gives - # these no staleness budget for the same reason. - probe, _ = self.probe(PRIMARY_STATUS) - probe.check() - harness = self.harness(probe) - - self.assertFalse( - public_read_service.path_reads_database( - f"/public/v1/artifacts/{ARTIFACT_SHA256}" - ) - ) - self.assertEqual( - 200, harness.get(f"/public/v1/artifacts/{ARTIFACT_SHA256}").status - ) - - def test_every_replica_backed_route_is_gated(self) -> None: - """Exhaustive, not a sample: every extracted route that reads a read - slot must refuse, and the classification must come from the registry.""" - - probe, _ = self.probe(PRIMARY_STATUS) - probe.check() - harness = self.harness(probe) - - gated = 0 - for route in EXTRACTED_ROUTES: - path = route[0] - endpoint = endpoint_registry.endpoint_for_request_path(path) - assert endpoint is not None - expected_gated = ( - not endpoint.immutable_content - and endpoint_registry.LedgerAccess.READ_SLOT in endpoint.access - ) - with self.subTest(path=path): - self.assertEqual( - expected_gated, - public_read_service.path_reads_database(path), - ) - served = harness.get(request_path(route)) - if expected_gated: - gated += 1 - self.assertEqual(503, served.status) - self.assertIn( - "not in recovery", served.payload["error"]["message"] - ) - else: - self.assertNotEqual(503, served.status) - self.assertGreater(gated, 0, "no route exercised the replica gate") - - def test_observed_replay_lag_travels_on_the_response(self) -> None: - probe, clock = self.probe(standby(heartbeat=1.0, replay_lag=2.0)) - probe.check() - clock.advance(3.0) - served = self.harness(probe).get("/public/v1/blocks") - - self.assertEqual(200, served.status) - self.assertEqual( - "5.000", - served.headers.get(public_read_service.REPLICA_LAG_HEADER), - ) - - def test_healthz_carries_the_replication_facts(self) -> None: - probe, _ = self.probe(standby(heartbeat=1.0)) - probe.check() - served = self.harness(probe).get("/healthz") - - self.assertEqual(200, served.status) - self.assertEqual(True, served.payload["replica"]["in_recovery"]) - self.assertEqual( - 1.0, served.payload["replica"]["receiver_heartbeat_age_seconds"] - ) - self.assertEqual(60.0, served.payload["replica"]["max_lag_seconds"]) - - def test_healthz_goes_unready_when_the_contract_fails(self) -> None: - probe, _ = self.probe(PRIMARY_STATUS) - probe.check() - served = self.harness(probe).get("/healthz") - - self.assertEqual(503, served.status) - self.assertFalse(served.payload["ok"]) - self.assertIn("not in recovery", served.payload["error"]) - - def test_metrics_publish_the_replica_gauges_and_refusals(self) -> None: - probe, _ = self.probe(standby(heartbeat=1.0, replay_lag=2.0)) - probe.check() - harness = self.harness(probe) - harness.get("/public/v1/blocks") - - body = harness.get("/metrics").body.decode() - self.assertIn("qbit_prism_public_replica_in_recovery 1", body) - self.assertIn( - "qbit_prism_public_replica_heartbeat_age_seconds 1.000", body - ) - self.assertIn("qbit_prism_public_replica_replay_lag_seconds 2.000", body) - self.assertIn("qbit_prism_public_replica_apply_backlog_bytes 4096.000", body) - self.assertIn("qbit_prism_public_replica_max_lag_seconds 60.000", body) - self.assertIn("qbit_prism_public_replica_refusals_total 0", body) - - def test_replica_refusals_are_counted(self) -> None: - probe, _ = self.probe(PRIMARY_STATUS) - probe.check() - harness = self.harness(probe) - harness.get("/public/v1/blocks") - - self.assertIn( - "qbit_prism_public_replica_refusals_total 1", - harness.get("/metrics").body.decode(), - ) - - def test_unknown_numbers_report_minus_one_rather_than_zero(self) -> None: - probe, _ = self.probe(standby(heartbeat=None, replay_lag=None)) - probe.check() - harness = self.harness(probe) - - body = harness.get("/metrics").body.decode() - self.assertIn("qbit_prism_public_replica_heartbeat_age_seconds -1", body) - self.assertIn("qbit_prism_public_replica_replay_lag_seconds -1", body) - - -class ReplicaModeTests(unittest.TestCase): - """PRISM_PUBLIC_REPLICA_MODE decides whether the contract is enforced.""" - - class Ledger: - def read_replica_status(self) -> dict[str, object]: - return standby() - - def test_absent_mode_enforces_nothing(self) -> None: - # The default must not 503 a deployment that merged this before it - # provisioned a standby; the shipped compose sets require explicitly. - self.assertEqual( - public_read_service.REPLICA_MODE_OFF, - public_read_service.resolve_replica_mode({}), - ) - self.assertIsNone(public_read_service.build_replica_probe(self.Ledger(), {})) - - def test_require_builds_the_probe(self) -> None: - probe = public_read_service.build_replica_probe( - self.Ledger(), {"PRISM_PUBLIC_REPLICA_MODE": "require"} - ) - self.assertIsInstance(probe, public_read_service.ReplicaProbe) - - def test_an_unrecognised_mode_refuses_startup(self) -> None: - # Neither default is safe to guess: the two modes disagree about - # whether this process may read the coordinator's primary. - with self.assertRaises( - public_read_service.PublicReadConfigurationError - ) as raised: - public_read_service.resolve_replica_mode( - {"PRISM_PUBLIC_REPLICA_MODE": "yes"} - ) - self.assertIn("PRISM_PUBLIC_REPLICA_MODE", str(raised.exception)) - - -class DatabaseReached(AssertionError): - """Raised when a route consulted the database while readiness said it was down.""" - - -class ScriptedReadiness: - """A readiness probe callable whose answer the test flips. - - Raises rather than returning False, because that is what an unreachable - Postgres actually does to ``dashboard_readiness_probe``, and the two take - different arms inside ReadinessProbe.check_once(). - """ - - def __init__(self, *, healthy: bool = True) -> None: - self.healthy = healthy - self.calls = 0 - - def __call__(self) -> bool: - self.calls += 1 - if not self.healthy: - raise RuntimeError("connection refused") - return True - - -def cached_entry( - cache: public_api.PublicResponseCache, - path: str, -) -> object | None: - """The live cache entry for a path, or None. - - Reaches into PublicResponseCache's storage deliberately. Expiry there is - keyed on time.monotonic(), which is not injectable, so the alternatives for - driving the expired-entry case are sleeping out a real TTL or replacing the - cache with a double -- and a double cannot prove the thing under test, - which is that a degraded request leaves the *real* entry's bounds alone. - """ - return cache._entries.get(public_api.public_cache_key(path, {})) - - -class CoalescingCache: - """A cache that hands back a result some other request computed. - - This is PublicResponseCache's waiter path: a request that arrives while - another is already computing the same key never calls compute() itself and - receives the owner's result. Modelled here rather than raced, because the - property under test is what the handler does with a result it did not - produce, not the cache's own locking. - """ - - def __init__(self, result: tuple) -> None: - self.result = result - self.computes = 0 - - def get_or_compute( - self, - *, - key: object, - ttl_seconds: int, - compute, - stale_while_revalidate_seconds: int = 0, - ) -> tuple: - # a waiter never runs compute() - del key, ttl_seconds, compute, stale_while_revalidate_seconds - return self.result - - -class DatabaseOutageTests(unittest.TestCase): - """Bounded-stale availability while Postgres is unreachable (#164). - - The endpoint is public and the database is not: an outage that turns every - dashboard into an error page is an outage the public sees in full. What the - surface owes instead is its own already-published bound -- serve the warm - entry if there is one inside the route's staleness budget, say plainly that - it is doing so, and refuse otherwise. - - Two properties make that honest rather than a way to hide an outage, and - each has a test below: a degraded request never calls the database (so the - outage is not amplified by the retry traffic it causes), and it never - extends the entry it serves (so ordinary polling cannot walk a stale - dashboard past the bound, one hit at a time, for as long as the outage - lasts). - """ - - def setUp(self) -> None: - self.ledger = FullReadModelLedger() - self.coordinator = FakeCoordinator(ledger=self.ledger) - self.cache = public_api.PublicResponseCache() - self.probe_source = ScriptedReadiness() - self.readiness = public_read_service.ReadinessProbe( - self.probe_source, - interval_seconds=3600, - ) - self.readiness.check_once() - self.metrics = public_read_service.ServiceMetrics() - self.reached: list[str] = [] - self.armed: list[str] = [] - self.harness = ServiceHarness( - self.coordinator, - response_cache=self.cache, - readiness=self.readiness, - metrics=self.metrics, - ) - self.addCleanup(self.harness.close) - - def take_database_down(self, *, arm_landmines: bool = True) -> None: - """Readiness starts failing, and every database read becomes a landmine. - - Everything public_api asks a ledger for is a database read, so arming - all of them turns "the handler went to the origin anyway" into a - recorded method name rather than a subtle difference in a body. - """ - self.probe_source.healthy = False - self.readiness.check_once() - self.assertEqual(503, self.readiness.snapshot()[0]) - if not arm_landmines: - return - names = {name for name in dir(self.ledger) if name.startswith("dashboard_")} - names.update( - name for name in WRITER_LOCK_LEDGER_METHODS if hasattr(self.ledger, name) - ) - for name in sorted(names): - def explode(*_args: object, _name: str = name, **_kwargs: object) -> object: - self.reached.append(_name) - raise DatabaseReached(f"the origin reached {_name}() during an outage") - - setattr(self.ledger, name, explode) - self.armed.append(name) - - def bring_database_up(self) -> None: - for name in self.armed: - delattr(self.ledger, name) - self.armed.clear() - self.probe_source.healthy = True - self.readiness.check_once() - self.assertEqual(200, self.readiness.snapshot()[0]) - - def assert_degraded(self, served: ServedResponse) -> None: - self.assertEqual( - public_read_service.DEGRADED_WARNING, - served.headers.get(public_read_service.DEGRADED_WARNING_HEADER), - ) - self.assertEqual( - public_read_service.DATABASE_STATE_UNAVAILABLE, - served.headers.get(public_read_service.DATABASE_STATE_HEADER), - ) - - def test_a_warm_entry_is_served_and_says_so(self) -> None: - warm = self.harness.get("/public/v1/blocks") - self.assertEqual(200, warm.status) - self.take_database_down() - - served = self.harness.get("/public/v1/blocks") - - self.assertEqual(200, served.status) - self.assertEqual(warm.body, served.body) - self.assertEqual([], self.reached) - # Pinned literally, not through the constant: this string is the - # contract a dashboard frontend parses. - self.assertEqual( - '110 qbit-prism "database unavailable; serving cached response"', - served.headers.get("Warning"), - ) - self.assertEqual( - "unavailable", served.headers.get("X-Prism-Database-State") - ) - # The outage fact and the numerical bound travel together: without the - # second, "stale" has no size. - self.assertEqual("0", served.headers.get("Age")) - self.assertEqual( - "15", - served.headers.get(public_read_service.STALENESS_BUDGET_HEADER), - ) - - def test_a_cold_cache_refuses_without_calling_the_database(self) -> None: - self.take_database_down() - - served = self.harness.get("/public/v1/blocks") - - self.assertEqual(503, served.status) - self.assertEqual([], self.reached) - self.assertEqual("no-store", served.headers.get("Cache-Control")) - self.assertEqual("prism.dashboard.error.v1", served.payload["schema"]) - # PUBLIC_ERROR_CODES rewrites an undocumented code to internal_error, - # so a 503 that only reads right in the message is not enough. - self.assertEqual("upstream_unavailable", served.payload["error"]["code"]) - self.assertIn( - "cannot reach the database", served.payload["error"]["message"] - ) - self.assertEqual( - "unavailable", served.headers.get("X-Prism-Database-State") - ) - # 110 is "Response is Stale". Nothing was served, stale or otherwise. - self.assertIsNone(served.headers.get("Warning")) - self.assertEqual( - "15", - served.headers.get(public_read_service.STALENESS_BUDGET_HEADER), - ) - - def test_a_refusal_stores_nothing(self) -> None: - # Otherwise the first refusal would become a warm entry and the next - # request would serve a 503 body as a cached 503 -- or worse, count as - # the warm entry the outage path is allowed to serve. - self.take_database_down() - self.harness.get("/public/v1/blocks") - - self.assertIsNone(cached_entry(self.cache, "/public/v1/blocks")) - - def test_an_expired_entry_refuses_and_traffic_cannot_refill_it(self) -> None: - self.harness.get("/public/v1/blocks") - entry = cached_entry(self.cache, "/public/v1/blocks") - self.assertIsNotNone(entry) - self.take_database_down() - # Expire it where the cache actually looks, rather than sleeping out a - # real TTL. - entry.expires_at = 0.0 # type: ignore[union-attr] - - for _ in range(3): - self.assertEqual(503, self.harness.get("/public/v1/blocks").status) - - self.assertEqual([], self.reached) - self.assertIsNone(cached_entry(self.cache, "/public/v1/blocks")) - - def test_an_entry_past_the_route_budget_refuses_though_it_is_warm(self) -> None: - """TTL and budget are separate numbers, and the budget is the promise. - - Nothing stops an operator raising PRISM_PUBLIC_*_CACHE_TTL_SECONDS - above the registry budget the same route publishes. When that happens - the entry is unexpired and past the bound at once, and the bound wins. - """ - self.harness.get("/public/v1/blocks") - entry = cached_entry(self.cache, "/public/v1/blocks") - entry.stored_at -= 3600.0 # type: ignore[union-attr] - entry.expires_at += 3600.0 # type: ignore[union-attr] - self.take_database_down() - - for _ in range(3): - served = self.harness.get("/public/v1/blocks") - self.assertEqual(503, served.status) - - self.assertEqual([], self.reached) - self.assertIn( - "exceeds the staleness budget", served.payload["error"]["message"] - ) - self.assertEqual("no-store", served.headers.get("Cache-Control")) - self.assertEqual("3600", served.headers.get("Age")) - self.assertEqual( - "15", - served.headers.get(public_read_service.STALENESS_BUDGET_HEADER), - ) - self.assertEqual( - "unavailable", served.headers.get("X-Prism-Database-State") - ) - self.assertIsNone(served.headers.get("Warning")) - - def test_degraded_traffic_never_extends_the_entry_it_serves(self) -> None: - """The bound has to survive the traffic, or it is not a bound. - - A degraded hit that refreshed stored_at or expires_at would let - ordinary polling walk one snapshot forward indefinitely, and the - staleness budget on every one of those responses would be a number - describing nothing. - """ - self.harness.get("/public/v1/blocks") - entry = cached_entry(self.cache, "/public/v1/blocks") - assert entry is not None - bounds = (entry.stored_at, entry.expires_at) - self.take_database_down() - - for _ in range(5): - self.assert_degraded(self.harness.get("/public/v1/blocks")) - - self.assertEqual(bounds, (entry.stored_at, entry.expires_at)) - # And when those bounds are reached, the outage cannot renew them. - entry.expires_at = 0.0 - self.assertEqual(503, self.harness.get("/public/v1/blocks").status) - - def test_recovery_restores_the_origin_and_drops_the_degraded_headers(self) -> None: - self.harness.get("/public/v1/blocks") - self.take_database_down() - self.assert_degraded(self.harness.get("/public/v1/blocks")) - - self.bring_database_up() - # Expire the warm entry so the answer has to come from the origin. - cached_entry(self.cache, "/public/v1/blocks").expires_at = 0.0 # type: ignore[union-attr] - served = self.harness.get("/public/v1/blocks") - - self.assertEqual(200, served.status) - self.assertIsNone(served.headers.get("Warning")) - self.assertIsNone(served.headers.get("X-Prism-Database-State")) - self.assertEqual("0", served.headers.get("Age")) - # Refilled, which only the origin path can do. - self.assertIsNotNone(cached_entry(self.cache, "/public/v1/blocks")) - - def test_every_database_backed_route_degrades_and_the_rest_do_not(self) -> None: - """Exhaustive and registry-derived, so a new route is classified once.""" - - for route in EXTRACTED_ROUTES: - self.assertEqual( - 200, self.harness.get(request_path(route)).status, route[0] - ) - self.take_database_down() - - degraded = 0 - for route in EXTRACTED_ROUTES: - path = route[0] - endpoint = endpoint_registry.endpoint_for_request_path(path) - assert endpoint is not None - expected = ( - not endpoint.immutable_content - and endpoint_registry.LedgerAccess.READ_SLOT in endpoint.access - ) - with self.subTest(path=path): - self.assertEqual( - expected, public_read_service.path_reads_database(path) - ) - served = self.harness.get(request_path(route)) - self.assertEqual(200, served.status) - if expected: - degraded += 1 - self.assert_degraded(served) - self.assertIsNotNone(served.headers.get("Age")) - self.assertIsNotNone( - served.headers.get( - public_read_service.STALENESS_BUDGET_HEADER - ) - ) - else: - self.assertIsNone(served.headers.get("Warning")) - self.assertIsNone( - served.headers.get("X-Prism-Database-State") - ) - self.assertEqual([], self.reached) - self.assertGreater(degraded, 0, "no route exercised the outage gate") - - def test_the_exempt_routes_stay_available_from_a_cold_cache(self) -> None: - """Mining configuration and immutable artifacts owe answers regardless. - - The first is assembled from the environment and has no database answer - to lose; the second is content-addressed, so its body either hashes to - the requested sha256 or it does not, which an outage cannot change. - Landmines stay disarmed here precisely because these two are expected - to reach their origin. - """ - self.take_database_down(arm_landmines=False) - - refused = 0 - for route in EXTRACTED_ROUTES: - path = route[0] - with self.subTest(path=path): - served = self.harness.get(request_path(route)) - if public_read_service.path_reads_database(path): - refused += 1 - self.assertEqual(503, served.status) - self.assertEqual( - "upstream_unavailable", served.payload["error"]["code"] - ) - self.assertEqual( - "no-store", served.headers.get("Cache-Control") - ) - else: - self.assertEqual(200, served.status) - self.assertIsNone(served.headers.get("Warning")) - self.assertIsNone( - served.headers.get("X-Prism-Database-State") - ) - self.assertEqual(len(EXTRACTED_ROUTES) - 2, refused) - - def test_a_bypassed_cache_refuses_rather_than_calling_the_database(self) -> None: - # With the response cache off there is no warm entry to serve and no - # way to make one, so every request is the cold case. - with patch.dict(os.environ, {"PRISM_PUBLIC_CACHE_ENABLED": "false"}): - self.harness.get("/public/v1/blocks") - self.take_database_down() - served = self.harness.get("/public/v1/blocks") - - self.assertEqual(503, served.status) - self.assertEqual([], self.reached) - self.assertEqual( - "unavailable", served.headers.get("X-Prism-Database-State") - ) - - def test_a_route_that_reads_no_database_is_never_degraded(self) -> None: - self.take_database_down() - - served = self.harness.get("/public/v1/mining-configuration") - - self.assertEqual(200, served.status) - self.assertIsNone(served.headers.get("Warning")) - self.assertIsNone(served.headers.get("X-Prism-Database-State")) - - def test_degraded_responses_and_refusals_are_counted(self) -> None: - self.harness.get("/public/v1/blocks") - self.take_database_down() - self.harness.get("/public/v1/blocks") # warm: degraded 200 - self.harness.get("/public/v1/leaderboard") # cold: refused - - body = self.harness.get("/metrics").body.decode() - - self.assertIn("qbit_prism_public_degraded_responses_total 1", body) - self.assertIn("qbit_prism_public_database_outage_refusals_total 1", body) - - def test_a_healthy_readiness_snapshot_changes_nothing(self) -> None: - # The gate must be invisible while the database is up, including on the - # first, uncached request. - served = self.harness.get("/public/v1/blocks") - - self.assertEqual(200, served.status) - self.assertIsNone(served.headers.get("Warning")) - self.assertIsNone(served.headers.get("X-Prism-Database-State")) - body = self.harness.get("/metrics").body.decode() - self.assertIn("qbit_prism_public_degraded_responses_total 0", body) - self.assertIn("qbit_prism_public_database_outage_refusals_total 0", body) - - def test_a_warming_up_probe_is_treated_as_unavailable(self) -> None: - # Before the first probe completes there is no evidence the database is - # reachable, and this gate fails closed on absent evidence like every - # other freshness check in this service. - readiness = public_read_service.ReadinessProbe( - ScriptedReadiness(), interval_seconds=3600 - ) - harness = ServiceHarness( - FakeCoordinator(ledger=FullReadModelLedger()), readiness=readiness - ) - self.addCleanup(harness.close) - - self.assertEqual(503, harness.get("/public/v1/blocks").status) - self.assertEqual(200, harness.get("/public/v1/mining-configuration").status) - - def test_a_coalesced_refusal_is_never_shared_cacheable(self) -> None: - """A waiter must not turn another request's refusal into a CDN entry. - - get_or_compute() coalesces: a request arriving while another is - computing the same key gets that one's result, whatever its own view of - readiness was. So a request whose probe had already recovered can - receive the outage refusal -- and if the handler decided "is this the - refusal" from the request instead of from the response, it would send - that 503 out under the ordinary shared-cache headers and a CDN would - hold it for the route's TTL. - """ - harness = ServiceHarness( - self.coordinator, - response_cache=CoalescingCache( - (503, public_read_service.DATABASE_UNAVAILABLE_PAYLOAD, "MISS", 0) - ), - readiness=self.readiness, # healthy: this request never degraded - ) - self.addCleanup(harness.close) - - served = harness.get("/public/v1/blocks") - - self.assertEqual(503, served.status) - self.assertEqual("no-store", served.headers.get("Cache-Control")) - self.assertIsNone(served.headers.get("CDN-Cache-Control")) - self.assertEqual( - "unavailable", served.headers.get("X-Prism-Database-State") - ) - - def test_a_coalesced_fresh_body_is_not_labelled_stale(self) -> None: - """The mirror case: the waiter is the degraded one, the body is fresh. - - Warning 110 and X-Prism-Database-State are claims about the body being - served. A response another request just computed from a database that - answered it is not a cached response, whatever this request's probe - said a moment later. - """ - fresh = {"schema": "prism.dashboard.blocks.v1", "rows": []} - self.take_database_down() - harness = ServiceHarness( - self.coordinator, - response_cache=CoalescingCache((200, fresh, "MISS", 0)), - readiness=self.readiness, - ) - self.addCleanup(harness.close) - - served = harness.get("/public/v1/blocks") - - self.assertEqual(200, served.status) - self.assertEqual(fresh, served.payload) - self.assertIsNone(served.headers.get("Warning")) - self.assertIsNone(served.headers.get("X-Prism-Database-State")) - - def test_replica_mode_require_keeps_its_stricter_refusal(self) -> None: - """A warm entry does not soften the standby contract. - - In require mode the replica probe is the readiness probe, so both gates - see the same failure. They do not owe the same answer: reading the - coordinator's primary is refused because of *where* the answer would - come from, which a cached body from that same wrong place does not fix. - """ - clock = FakeClock() - replica = public_read_service.ReplicaProbe( - ScriptedReplica(standby(heartbeat=1.0), PRIMARY_STATUS), - max_lag_seconds=60.0, - monotonic=clock, - ) - readiness = public_read_service.ReadinessProbe( - replica.check, interval_seconds=3600, monotonic=clock - ) - readiness.check_once() - harness = ServiceHarness( - FakeCoordinator(ledger=FullReadModelLedger()), - readiness=readiness, - replica=replica, - ) - self.addCleanup(harness.close) - self.assertEqual(200, harness.get("/public/v1/blocks").status) - - readiness.check_once() - served = harness.get("/public/v1/blocks") - - self.assertEqual(503, served.status) - self.assertIn("not in recovery", served.payload["error"]["message"]) - self.assertIsNone(served.headers.get("Warning")) - - -class FailClosedStartupTests(unittest.TestCase): - """Refuse to start rather than serve wrong facts.""" - - def test_missing_public_stratum_url_refuses_startup(self) -> None: - for environ in ({}, {"PRISM_PUBLIC_STRATUM_URL": ""}, {"PRISM_PUBLIC_STRATUM_URL": " "}): - with self.subTest(environ=environ): - with self.assertRaises( - public_read_service.PublicReadConfigurationError - ) as raised: - public_read_service.require_public_stratum_url(environ) - self.assertIn("PRISM_PUBLIC_STRATUM_URL", str(raised.exception)) - - def test_a_configured_public_stratum_url_is_accepted(self) -> None: - value = public_read_service.require_public_stratum_url( - {"PRISM_PUBLIC_STRATUM_URL": "stratum+tcp://pool.example:3340"} - ) - self.assertEqual("stratum+tcp://pool.example:3340", value) - - def test_invalid_stratum_urls_refuse_before_database_construction(self) -> None: - for url in ( - "pool.example:3340", "https://pool.example:3340", - "stratum+tcp://pool.example", "stratum+tcp://pool.example:not-a-port", - "stratum+tcp://pool.example:0", "stratum+tcp://pool.example:65536", - "stratum+tcp://:3340", "stratum+tcp://[broken:3340", - "stratum+tcp://user:password@pool.example:3340", - "stratum+tcp://pool.example:3340/path", "stratum+tcp://pool.example:3340?q=1", - "stratum+tcp://pool.example:3340#fragment", "stratum+tcp://bad host:3340", - "stratum+tcp://pool.example:33\t40", " stratum+tcp://pool.example:3340 ", - "stratum+tcp://bad%host:3340", "stratum+tcp://bad\\host:3340", - ): - with self.subTest(url=url), patch.object( - public_read_service, "build_ledger_from_env" - ) as ledger: - with self.assertRaises(public_read_service.PublicReadConfigurationError): - public_read_service.build_service({"PRISM_PUBLIC_STRATUM_URL": url}) - ledger.assert_not_called() - - def test_highdiff_url_is_validated_only_when_advertised(self) -> None: - base = {"PRISM_PUBLIC_STRATUM_URL": "stratum+tcp://pool.example:3340"} - for bad in ("https://host/path", "stratum+tcp://host:not-a-port", " "): - with self.subTest(url=bad), patch.object(public_read_service, "build_ledger_from_env") as ledger: - with self.assertRaisesRegex(public_read_service.PublicReadConfigurationError, "PRISM_PUBLIC_STRATUM_HIGHDIFF_URL"): - public_read_service.build_service({**base, "PRISM_STRATUM_HIGHDIFF_PORT": "4334", "PRISM_PUBLIC_STRATUM_HIGHDIFF_URL": bad}) - ledger.assert_not_called() - self.assertEqual(public_read_service.require_public_stratum_url( - {**base, "PRISM_PUBLIC_STRATUM_HIGHDIFF_URL": bad}), base["PRISM_PUBLIC_STRATUM_URL"]) - for explicit in ("", "stratum+ssl://[2001:db8::1]:443"): - self.assertEqual(public_read_service.require_public_stratum_url( - {**base, "PRISM_STRATUM_HIGHDIFF_PORT": "4334", "PRISM_PUBLIC_STRATUM_HIGHDIFF_URL": explicit}), base["PRISM_PUBLIC_STRATUM_URL"]) - - def test_ipv6_and_tls_stratum_urls_are_accepted(self) -> None: - for url in ("stratum+tcp://[2001:db8::1]:3340", "stratum+ssl://pool.example:443"): - self.assertEqual(public_read_service.require_public_stratum_url( - {"PRISM_PUBLIC_STRATUM_URL": url}), url) - - def test_memory_ledger_is_refused(self) -> None: - with self.assertRaises( - public_read_service.PublicReadConfigurationError - ) as raised: - public_read_service.build_ledger_from_env( - { - "PRISM_ALLOW_MEMORY_LEDGER": "1", - "PRISM_DATABASE_URL": "postgresql:///prism", - } - ) - message = str(raised.exception) - self.assertIn("PRISM_ALLOW_MEMORY_LEDGER", message) - # The reason matters as much as the refusal: the memory ledger would - # reintroduce writer-lock reads through public_api's fallbacks. - self.assertIn("dashboard_", message) - - def test_missing_database_configuration_refuses_startup(self) -> None: - with self.assertRaises( - public_read_service.PublicReadConfigurationError - ) as raised: - public_read_service.build_ledger_from_env({}) - self.assertIn("PRISM_DATABASE_URL", str(raised.exception)) - - def test_build_service_refuses_before_touching_the_database(self) -> None: - # The stratum-url check must come first, so a service missing both - # reports the configuration error rather than a connection failure. - with self.assertRaises( - public_read_service.PublicReadConfigurationError - ) as raised: - public_read_service.build_service({}) - self.assertIn("PRISM_PUBLIC_STRATUM_URL", str(raised.exception)) - - def test_the_stand_in_defines_no_stratum_endpoint_attributes(self) -> None: - # public_api.mining_configuration reads getattr(coordinator, "port"/ - # "bind") with an environment fallback. The stand-in must not define - # them, or the fallback -- and with it the required-env check -- would - # be dead code. - coordinator = public_read_service.PublicReadCoordinator( - ledger=object(), - rpc=object(), - ) - self.assertFalse(hasattr(coordinator, "port")) - self.assertFalse(hasattr(coordinator, "bind")) - - -class ReadinessAndMetricsTests(unittest.TestCase): - """The service's own liveness and counters, which are new rather than moved.""" - - def test_healthz_reports_ready_when_the_probe_succeeds(self) -> None: - probe = public_read_service.ReadinessProbe(lambda: True, interval_seconds=60) - probe.check_once() - harness = ServiceHarness(FakeCoordinator(), readiness=probe) - self.addCleanup(harness.close) - - served = harness.get("/healthz") - - self.assertEqual(200, served.status) - self.assertTrue(served.payload["ok"]) - - def test_healthz_reports_unready_when_postgres_is_unreachable(self) -> None: - def failing_probe() -> bool: - raise RuntimeError("connection refused") - - probe = public_read_service.ReadinessProbe(failing_probe, interval_seconds=60) - probe.check_once() - harness = ServiceHarness(FakeCoordinator(), readiness=probe) - self.addCleanup(harness.close) - - served = harness.get("/healthz") - - self.assertEqual(503, served.status) - self.assertFalse(served.payload["ok"]) - self.assertIn("connection refused", str(served.payload["error"])) - - def test_healthz_refuses_a_stale_probe(self) -> None: - clock = [1000.0] - probe = public_read_service.ReadinessProbe( - lambda: True, - interval_seconds=5, - monotonic=lambda: clock[0], - ) - probe.check_once() - clock[0] += probe.stale_after_seconds + 1 - harness = ServiceHarness(FakeCoordinator(), readiness=probe) - self.addCleanup(harness.close) - - served = harness.get("/healthz") - - self.assertEqual(503, served.status) - self.assertIn("stale", str(served.payload["error"])) - - def test_healthz_does_not_query_on_the_request_thread(self) -> None: - # The compose healthcheck polls this forever; a database round trip per - # poll would make liveness checking a source of load. - calls: list[int] = [] - probe = public_read_service.ReadinessProbe( - lambda: (calls.append(1), True)[1], - interval_seconds=3600, - ) - probe.check_once() - harness = ServiceHarness(FakeCoordinator(), readiness=probe) - self.addCleanup(harness.close) - - for _ in range(5): - self.assertEqual(200, harness.get("/healthz").status) - - self.assertEqual(1, len(calls)) - - def test_metrics_are_prometheus_text_for_this_process(self) -> None: - harness = ServiceHarness(FakeCoordinator()) - self.addCleanup(harness.close) - harness.get("/public/v1/blocks") - - served = harness.get("/metrics") - - self.assertEqual(200, served.status) - self.assertTrue( - served.headers.get("Content-Type", "").startswith("text/plain") - ) - body = served.body.decode() - for metric in ( - "qbit_prism_public_requests_total", - "qbit_prism_public_responses_total", - "qbit_prism_public_cache_total", - "qbit_prism_public_staleness_refusals_total", - ): - self.assertIn(metric, body) - - def test_cache_hits_and_misses_are_counted(self) -> None: - harness = ServiceHarness(FakeCoordinator()) - self.addCleanup(harness.close) - harness.get("/public/v1/blocks") - harness.get("/public/v1/blocks") - - body = harness.get("/metrics").body.decode() - - self.assertIn('qbit_prism_public_cache_total{state="miss"} 1', body) - self.assertIn('qbit_prism_public_cache_total{state="hit"} 1', body) - - def test_unknown_paths_outside_the_public_surface_are_plain_404s(self) -> None: - harness = ServiceHarness(FakeCoordinator()) - self.addCleanup(harness.close) - - served = harness.get("/audit/latest") - - self.assertEqual(404, served.status) - self.assertEqual({"error": "unknown endpoint"}, served.payload) - - -class _MinimalAuditPort: - """Answers nothing: an unknown path never reaches a port method.""" - - def ledger_backend(self) -> str: - return "fixture" - - -class CoordinatorSubtractionTests(unittest.TestCase): - """The coordinator's listener no longer serves the public surface.""" - - @classmethod - def setUpClass(cls) -> None: - cls.facade = AuditHttpFacade( - _MinimalAuditPort(), # type: ignore[arg-type] - AuditHttpConfig("127.0.0.1", 0, join_timeout_seconds=1.0), - ) - state = cls.facade.start() - assert state.bound_address is not None - cls.host, cls.port = state.bound_address - - @classmethod - def tearDownClass(cls) -> None: - cls.facade.stop() - - def get(self, path: str) -> tuple[int, object]: - url = f"http://{self.host}:{self.port}{path}" - try: - with urllib.request.urlopen(url, timeout=5) as response: - return response.status, json.loads(response.read()) - except urllib.error.HTTPError as error: - with error: - return error.code, json.loads(error.read()) - - def test_public_routes_are_unknown_endpoints_on_the_coordinator(self) -> None: - for path in ( - "/public/v1/pool-summary", - "/public/v1/blocks", - f"/public/v1/miners/{RECIPIENT_ID}", - "/public/v1", - ): - with self.subTest(path=path): - status, body = self.get(path) - self.assertEqual(404, status) - self.assertEqual({"error": "unknown endpoint"}, body) - - def test_audit_http_no_longer_imports_public_api(self) -> None: - # The public error schema and cache policy left with the routes. An - # import here would mean some public handling stayed behind. - source = AUDIT_HTTP_PATH.read_text(encoding="utf-8") - self.assertNotIn("import public_api", source) - self.assertNotIn("public_api.", source) - - def test_the_audit_http_port_no_longer_exposes_public_payload(self) -> None: - from lab.prism import audit_http - - self.assertFalse(hasattr(audit_http.AuditHttpPort, "public_payload")) - - -class DirectCoinbaseSettlementContractTests(unittest.TestCase): - """The severed writer-lock settlement read must answer identically.""" - - def test_read_model_and_audit_bundle_fallback_agree(self) -> None: - # The read model added in share_ledger returns exactly what - # public_api.direct_coinbase_settlement_payload built from audit_bundle. - fallback_ledger = DirectCoinbasePublicLedger() - expected = public_api.direct_coinbase_settlement_payload( - fallback_ledger, - block_hash=DirectCoinbasePublicLedger.block_hash, - ) - - class ReadModelLedger(DirectCoinbasePublicLedger): - def dashboard_direct_coinbase_settlement( - self, - *, - block_hash: str, - ) -> dict[str, object] | None: - # Stand in for the Postgres read-slot implementation by - # reusing the same source row the fenced path would have read. - return expected - - def audit_bundle(self, *, block_hash: str) -> dict[str, object] | None: - raise WriterLockReached("audit_bundle() is a writer-lock read") - - actual = public_api.direct_coinbase_settlement_payload( - ReadModelLedger(), - block_hash=DirectCoinbasePublicLedger.block_hash, - ) - - self.assertEqual(expected, actual) - self.assertEqual( - { - "block_hash", - "block_height", - "settlement_mode", - "audit_bundle_sha256", - "payout_manifest_sha256", - "artifacts", - }, - set(actual or {}), - ) - - def test_owed_balance_read_model_matches_the_python_sum(self) -> None: - balances = [ - {"recipient_id": "miner-a", "balance_sats": 700}, - {"recipient_id": "miner-b", "balance_sats": 11}, - {"recipient_id": "miner-a", "balance_sats": 300}, - ] - - class SummingLedger: - def current_owed_balances(self) -> list[dict[str, object]]: - return balances - - class ReadModelLedger: - def dashboard_miner_owed_balance_bits(self, *, recipient_id: str) -> int: - return sum( - int(balance["balance_sats"]) - for balance in balances - if balance["recipient_id"] == recipient_id - ) - - def current_owed_balances(self) -> list[dict[str, object]]: - raise WriterLockReached("current_owed_balances() is a writer-lock read") - - self.assertEqual( - public_api.owed_balance_for_recipient(SummingLedger(), "miner-a"), - public_api.owed_balance_for_recipient(ReadModelLedger(), "miner-a"), - ) - self.assertEqual( - 1000, - public_api.owed_balance_for_recipient(ReadModelLedger(), "miner-a"), - ) - - def test_owed_balance_falls_back_for_a_ledger_without_the_read_model(self) -> None: - # The in-memory ledger has no dashboard_* read models; the fallback - # must stay for it. - class MemoryStyleLedger: - def current_owed_balances(self) -> list[dict[str, object]]: - return [{"recipient_id": "miner-a", "balance_sats": 42}] - - self.assertEqual( - 42, - public_api.owed_balance_for_recipient(MemoryStyleLedger(), "miner-a"), - ) - - -class StubbedReadSlotLedger(PsqlShareLedger): - """A PsqlShareLedger with a real writer lock and a canned SQL backend. - - Bypasses __init__ (as the other share-ledger tests do) so the gates under - test are real threading primitives while no database is involved: the - property being asserted is which gate a method takes, and that is decided - entirely by the Python code around the statement. - """ - - def __init__(self, results: dict[str, object]) -> None: - self._lock = threading.Lock() - self._read_semaphore = threading.BoundedSemaphore(4) - self._results = results - self.statements: list[str] = [] - - def _run_json(self, sql: str) -> object: - self.statements.append(sql) - for marker, result in self._results.items(): - if marker in sql: - return result - raise AssertionError(f"unexpected statement: {sql[:120]}") - - -OWED_BALANCE_SQL_RESULT = {"qbit_current_owed_balances": {"owed_balance_bits": "1000"}} -SETTLEMENT_SQL_RESULT = { - "qbit_pool_audit_bundles": { - "block_hash": "ab" * 32, - "block_height": 23342, - "payout_manifest_sha256": "8c" * 32, - "audit_bundle_sha256": "92" * 32, - "audit_bundle": { - "schema": "qbit.prism.audit-bundle.v1", - "found_block": {"block_height": 23342}, - "settlement_mode_decision": {"mode": "direct_coinbase"}, - }, - "body_uri": None, - } -} - - -class ReadSlotNotWriterLockTests(unittest.TestCase): - """The severed reads must take the read slot, structurally. - - Asserted by holding the writer lock from another thread for the whole - call: a method that fences would block until the holder released, so a - call that returns under a held lock cannot have taken it. The two - control cases at the end show the same setup does block the reads these - replaced, so a passing assertion here is not vacuous. - """ - - HOLD_SECONDS = 30.0 - RETURN_TIMEOUT = 5.0 - - def call_with_writer_lock_held(self, ledger: PsqlShareLedger, call) -> object: - """Run call() while another thread holds ledger's writer lock.""" - - acquired = threading.Event() - release = threading.Event() - - def hold() -> None: - with ledger._lock: - acquired.set() - release.wait(self.HOLD_SECONDS) - - holder = threading.Thread(target=hold, name="writer-lock-holder", daemon=True) - holder.start() - self.assertTrue(acquired.wait(5.0), "writer lock holder never started") - try: - result: list[object] = [] - error: list[BaseException] = [] - - def run() -> None: - try: - result.append(call()) - except BaseException as exc: # noqa: BLE001 - reported below - error.append(exc) - - worker = threading.Thread(target=run, name="read-under-lock", daemon=True) - worker.start() - worker.join(timeout=self.RETURN_TIMEOUT) - returned = not worker.is_alive() - finally: - release.set() - holder.join(timeout=5.0) - if error: - raise error[0] - if not returned: - raise AssertionError("call did not return while the writer lock was held") - return result[0] - - def blocks_while_writer_lock_held(self, ledger: PsqlShareLedger, call) -> bool: - try: - self.call_with_writer_lock_held(ledger, call) - except AssertionError as exc: - return "did not return" in str(exc) - return False - - def test_owed_balance_read_model_returns_under_a_held_writer_lock(self) -> None: - ledger = StubbedReadSlotLedger(OWED_BALANCE_SQL_RESULT) - - value = self.call_with_writer_lock_held( - ledger, - lambda: ledger.dashboard_miner_owed_balance_bits(recipient_id="miner-a"), - ) - - self.assertEqual(1000, value) - - def test_settlement_read_model_returns_under_a_held_writer_lock(self) -> None: - ledger = StubbedReadSlotLedger(SETTLEMENT_SQL_RESULT) - - payload = self.call_with_writer_lock_held( - ledger, - lambda: ledger.dashboard_direct_coinbase_settlement(block_hash="ab" * 32), - ) - - assert payload is not None - self.assertEqual("direct_coinbase", payload["settlement_mode"]) - self.assertEqual(23342, payload["block_height"]) - - def test_owed_balance_sql_filters_by_recipient_rather_than_in_python(self) -> None: - ledger = StubbedReadSlotLedger(OWED_BALANCE_SQL_RESULT) - - ledger.dashboard_miner_owed_balance_bits(recipient_id="miner-a") - - statement = ledger.statements[0] - self.assertIn("qbit_current_owed_balances()", statement) - # The same two predicates the Python sum applied, now in SQL. - self.assertIn("owed_balance_sats > 0", statement) - self.assertIn("miner_id = 'miner-a'", statement) - - def test_a_quoted_recipient_id_cannot_escape_its_literal(self) -> None: - ledger = StubbedReadSlotLedger(OWED_BALANCE_SQL_RESULT) - - ledger.dashboard_miner_owed_balance_bits(recipient_id="mi'ner") - - self.assertIn("miner_id = 'mi''ner'", ledger.statements[0]) - - def test_an_empty_recipient_id_is_rejected(self) -> None: - ledger = StubbedReadSlotLedger(OWED_BALANCE_SQL_RESULT) - - with self.assertRaises(ValueError): - ledger.dashboard_miner_owed_balance_bits(recipient_id="") - - def test_settlement_read_model_reports_non_direct_coinbase_as_none(self) -> None: - ledger = StubbedReadSlotLedger( - { - "qbit_pool_audit_bundles": { - **SETTLEMENT_SQL_RESULT["qbit_pool_audit_bundles"], - "audit_bundle": { - "settlement_mode_decision": { - "mode": "hybrid_coinbase_ctv_fanout" - } - }, - } - } - ) - - self.assertIsNone( - ledger.dashboard_direct_coinbase_settlement(block_hash="ab" * 32) - ) - - def test_settlement_read_model_reports_a_missing_bundle_as_none(self) -> None: - ledger = StubbedReadSlotLedger({"qbit_pool_audit_bundles": None}) - - self.assertIsNone( - ledger.dashboard_direct_coinbase_settlement(block_hash="ab" * 32) - ) - - def test_an_unreadable_external_body_is_none_rather_than_an_error(self) -> None: - # public_api.audit_bundle_body_read_failed() tolerated exactly these - # three failures on the fenced path; the read model must keep doing so. - ledger = StubbedReadSlotLedger( - { - "qbit_pool_audit_bundles": { - **SETTLEMENT_SQL_RESULT["qbit_pool_audit_bundles"], - "audit_bundle": None, - "body_uri": "/nonexistent/prism-audit-bundle-body.json", - } - } - ) - - self.assertIsNone( - ledger.dashboard_direct_coinbase_settlement(block_hash="ab" * 32) - ) - - def test_the_replaced_reads_really_do_block_on_the_writer_lock(self) -> None: - # The control: without this, the assertions above could pass because - # nothing in the harness ever takes the writer lock at all. - owed = StubbedReadSlotLedger( - {"qbit_current_owed_balances": [], "count(*)": {"count": 0}} - ) - self.assertTrue( - self.blocks_while_writer_lock_held(owed, owed.current_owed_balances), - "current_owed_balances() no longer fences, so the contrast is void", - ) - - bundle = StubbedReadSlotLedger(SETTLEMENT_SQL_RESULT) - self.assertTrue( - self.blocks_while_writer_lock_held( - bundle, - lambda: bundle.audit_bundle(block_hash="ab" * 32), - ), - "audit_bundle() no longer fences, so the contrast is void", - ) - - def test_a_read_only_ledger_cannot_take_the_writer_lock_at_all(self) -> None: - ledger = PsqlShareLedger( - psql_command="psql postgresql://example.invalid/qbit", - native_client_mode="psql", - read_only=True, - ) - - with self.assertRaises(ReadOnlyLedgerError): - ledger.current_owed_balances() - with self.assertRaises(ReadOnlyLedgerError): - ledger.audit_bundle(block_hash="ab" * 32) - - def test_a_read_only_ledger_refuses_schema_initialization(self) -> None: - with self.assertRaises(ValueError): - PsqlShareLedger( - psql_command="psql postgresql://example.invalid/qbit", - native_client_mode="psql", - read_only=True, - initialize_schema=True, - ) - - -class _GatedLedger(PsqlShareLedger): - """A PsqlShareLedger with real gates and a stubbed SQL layer. - - Built without __init__ (the pattern tests/test_prism_public_dashboard_api.py - already uses) so the writer lock and the read semaphore are the genuine - objects while no database is involved. - """ - - def __init__(self) -> None: - self._lock = Lock() - self._read_semaphore = BoundedSemaphore(4) - self.statements: list[str] = [] - self.row: object = None - - def _text_literal(self, value: object) -> str: - return "'" + str(value).replace("'", "''") + "'" - - def _run_retry_safe_read_json(self, sql: str) -> object: - self.statements.append(sql) - return self.row - - -class WriterLockIsNotTakenTests(unittest.TestCase): - """Structural proof: the new read models never touch the writer gate. - - Each call is driven with the writer lock already held by another thread. A - method that fenced would block there until the test timed out; these must - return, because they take the bounded read slot instead. - """ - - def hold_writer_lock(self, ledger: _GatedLedger) -> None: - acquired = threading.Event() - release = threading.Event() - - def holder() -> None: - ledger._lock.acquire() - acquired.set() - release.wait(10) - ledger._lock.release() - - thread = threading.Thread(target=holder, daemon=True) - thread.start() - self.assertTrue(acquired.wait(5), "helper thread never took the writer lock") - self.addCleanup(thread.join, 5) - self.addCleanup(release.set) - - def call_with_timeout(self, call) -> object: - result: list[object] = [] - error: list[BaseException] = [] - - def run() -> None: - try: - result.append(call()) - except BaseException as exc: # noqa: BLE001 - reported below - error.append(exc) - - thread = threading.Thread(target=run, daemon=True) - thread.start() - thread.join(10) - self.assertFalse( - thread.is_alive(), - "the call blocked on the writer lock instead of taking a read slot", - ) - if error: - raise error[0] - return result[0] - - def test_owed_balance_read_model_does_not_fence(self) -> None: - ledger = _GatedLedger() - ledger.row = {"owed_balance_bits": "1000"} - self.hold_writer_lock(ledger) - - balance = self.call_with_timeout( - lambda: ledger.dashboard_miner_owed_balance_bits(recipient_id=RECIPIENT_ID) - ) - - self.assertEqual(1000, balance) - self.assertEqual(1, len(ledger.statements)) - # Filtered in SQL, not in Python: the point is to stop reading every - # recipient's balance to answer for one. - self.assertIn("miner_id", ledger.statements[0]) - self.assertIn(RECIPIENT_ID, ledger.statements[0]) - self.assertIn("owed_balance_sats > 0", ledger.statements[0]) - - def test_owed_balance_read_model_returns_zero_without_rows(self) -> None: - ledger = _GatedLedger() - ledger.row = {"owed_balance_bits": "0"} - self.hold_writer_lock(ledger) - - self.assertEqual( - 0, - self.call_with_timeout( - lambda: ledger.dashboard_miner_owed_balance_bits( - recipient_id="nobody" - ) - ), - ) - - def test_direct_coinbase_settlement_read_model_does_not_fence(self) -> None: - ledger = _GatedLedger() - ledger.row = { - "block_hash": DirectCoinbasePublicLedger.block_hash, - "block_height": DirectCoinbasePublicLedger.block_height, - "payout_manifest_sha256": DirectCoinbasePublicLedger.payout_manifest_sha256, - "audit_bundle_sha256": DirectCoinbasePublicLedger.audit_bundle_sha256, - "audit_bundle": { - "schema": "qbit.prism.audit-bundle.v1", - "found_block": {"block_height": DirectCoinbasePublicLedger.block_height}, - "settlement_mode_decision": {"mode": "direct_coinbase"}, - }, - "body_uri": None, - } - self.hold_writer_lock(ledger) - - payload = self.call_with_timeout( - lambda: ledger.dashboard_direct_coinbase_settlement( - block_hash=DirectCoinbasePublicLedger.block_hash - ) - ) - - # Identical to what the audit_bundle() path builds for the same row. - expected = public_api.direct_coinbase_settlement_payload( - DirectCoinbasePublicLedger(), - block_hash=DirectCoinbasePublicLedger.block_hash, - ) - self.assertEqual(expected, payload) - - def test_direct_coinbase_settlement_ignores_other_settlement_modes(self) -> None: - ledger = _GatedLedger() - ledger.row = { - "block_hash": DirectCoinbasePublicLedger.block_hash, - "block_height": 1, - "payout_manifest_sha256": None, - "audit_bundle_sha256": None, - "audit_bundle": { - "settlement_mode_decision": {"mode": "ctv_fanout"}, - }, - "body_uri": None, - } - self.hold_writer_lock(ledger) - - self.assertIsNone( - self.call_with_timeout( - lambda: ledger.dashboard_direct_coinbase_settlement( - block_hash=DirectCoinbasePublicLedger.block_hash - ) - ) - ) - - def test_direct_coinbase_settlement_is_none_without_a_bundle(self) -> None: - ledger = _GatedLedger() - ledger.row = None - self.hold_writer_lock(ledger) - - self.assertIsNone( - self.call_with_timeout( - lambda: ledger.dashboard_direct_coinbase_settlement( - block_hash=DirectCoinbasePublicLedger.block_hash - ) - ) - ) - - def test_an_unreadable_external_body_is_reported_as_none(self) -> None: - # public_api.audit_bundle_body_read_failed() tolerates exactly these - # three; the read model must keep that behaviour rather than 500. - for message in ( - "audit bundle body is not retrievable", - "audit bundle body hash mismatch", - "audit bundle body is not valid JSON", - ): - with self.subTest(message=message): - ledger = _GatedLedger() - ledger.row = { - "block_hash": DirectCoinbasePublicLedger.block_hash, - "audit_bundle_sha256": DirectCoinbasePublicLedger.audit_bundle_sha256, - "audit_bundle": None, - "body_uri": "file:///missing", - } - - def explode(_body_uri: object, *, expected_sha256: object = None) -> object: - raise RuntimeError(message) - - ledger._read_external_body = explode # type: ignore[assignment] - - self.assertIsNone( - ledger.dashboard_direct_coinbase_settlement( - block_hash=DirectCoinbasePublicLedger.block_hash - ) - ) - - def test_an_unexpected_body_error_still_raises(self) -> None: - ledger = _GatedLedger() - ledger.row = { - "block_hash": DirectCoinbasePublicLedger.block_hash, - "audit_bundle_sha256": DirectCoinbasePublicLedger.audit_bundle_sha256, - "audit_bundle": None, - "body_uri": "file:///missing", - } - - def explode(_body_uri: object, *, expected_sha256: object = None) -> object: - raise RuntimeError("postgres connection lost") - - ledger._read_external_body = explode # type: ignore[assignment] - - with self.assertRaises(RuntimeError): - ledger.dashboard_direct_coinbase_settlement( - block_hash=DirectCoinbasePublicLedger.block_hash - ) - - def test_the_readiness_probe_does_not_fence(self) -> None: - ledger = _GatedLedger() - ledger.row = {"ok": True} - self.hold_writer_lock(ledger) - - self.assertTrue( - self.call_with_timeout(ledger.dashboard_readiness_probe) - ) - - -if __name__ == "__main__": - unittest.main() diff --git a/tests/test_prism_public_refresh_limits.py b/tests/test_prism_public_refresh_limits.py deleted file mode 100644 index 38e88a9a8..000000000 --- a/tests/test_prism_public_refresh_limits.py +++ /dev/null @@ -1,76 +0,0 @@ -"""Global background refresh admission across cache keys and instances.""" - -import threading -import time -import unittest -from unittest.mock import patch - -from lab.prism import public_api - - -class PublicRefreshLimitsTests(unittest.TestCase): - def test_saturated_refreshes_serve_stale_without_registering_waiters(self): - slots = threading.BoundedSemaphore(1) - entered, release = threading.Event(), threading.Event() - first, second = public_api.PublicResponseCache(), public_api.PublicResponseCache() - key_a, key_b = ("/first", ()), ("/second", ()) - for cache, key in ((first, key_a), (second, key_b)): - cache.get_or_compute(key=key, ttl_seconds=5, compute=lambda: (200, {"old": True})) - cache._entries[key].stored_at = time.monotonic() - 6 - cache._entries[key].expires_at = time.monotonic() - 1 - - def blocked(): - entered.set() - self.assertTrue(release.wait(5)) - return 200, {"new": True} - - def stale(cache, key, compute): - return cache.get_or_compute( - key=key, ttl_seconds=5, stale_while_revalidate_seconds=30, compute=compute, - ) - - with patch.object(public_api, "_PUBLIC_REFRESH_SLOTS", slots): - try: - self.assertEqual(stale(first, key_a, blocked)[2], "STALE") - self.assertTrue(entered.wait(2)) - with patch.object(threading.Thread, "start") as start: - for _ in range(20): - self.assertEqual(stale(second, key_b, lambda: self.fail("unadmitted refresh"))[2], "STALE") - start.assert_not_called() - self.assertNotIn(key_b, second._inflight) - finally: - release.set() - self.assertTrue(slots.acquire(timeout=5)) - slots.release() - # Once capacity returns, the previously deferred key can refresh. - done = threading.Event() - - def recovered(): - done.set() - return 200, {"recovered": True} - - self.assertEqual(stale(second, key_b, recovered)[2], "STALE") - self.assertTrue(done.wait(2)) - self.assertTrue(slots.acquire(timeout=5)) - slots.release() - - def test_start_failure_and_compute_failure_both_return_capacity(self): - slots = threading.BoundedSemaphore(1) - cache = public_api.PublicResponseCache() - key = ("/failure", ()) - cache.get_or_compute(key=key, ttl_seconds=5, compute=lambda: (200, {})) - cache._entries[key].expires_at = time.monotonic() - 1 - - def fail(): - raise RuntimeError("refresh failed") - - with patch.object(public_api, "_PUBLIC_REFRESH_SLOTS", slots): - with patch.object(threading.Thread, "start", side_effect=RuntimeError("start failed")): - cache.get_or_compute(key=key, ttl_seconds=5, stale_while_revalidate_seconds=30, compute=fail) - self.assertTrue(slots.acquire(timeout=2)) - slots.release() - self.assertNotIn(key, cache._inflight) - cache.get_or_compute(key=key, ttl_seconds=5, stale_while_revalidate_seconds=30, compute=fail) - self.assertTrue(slots.acquire(timeout=2)) - slots.release() - self.assertNotIn(key, cache._inflight) diff --git a/tests/test_prism_reconnect_backpressure.py b/tests/test_prism_reconnect_backpressure.py deleted file mode 100644 index 5f6631dc1..000000000 --- a/tests/test_prism_reconnect_backpressure.py +++ /dev/null @@ -1,1132 +0,0 @@ -#!/usr/bin/env python3 -"""Deterministic reconnect-storm admission, backpressure, and health tests.""" - -from __future__ import annotations - -import socket -import threading -import time -import unittest -from concurrent.futures import Future -from decimal import Decimal -from types import SimpleNamespace -from unittest.mock import patch - -from lab.auxpow import vardiff -from lab.prism.job_delivery import ( - PRISM_DELIVERY_PRIORITY_INITIAL, - PRISM_DELIVERY_PRIORITY_NEW_TIP, - PRISM_DELIVERY_PRIORITY_SAME_TIP, -) -from lab.prism.prism_coordinator import ( - ClientState, - PendingInitialJob, - PrismCoordinator, - StratumError, - StratumListenerProfile, - WorkerIdentity, - _BoundedPriorityExecutor, - _JobBuildCancellation, -) - - -class FakeSocket: - def __init__(self) -> None: - self.closed = False - - def settimeout(self, _timeout: object) -> None: - return - - def setsockopt(self, *_args: object) -> None: - return - - def shutdown(self, _how: object) -> None: - return - - def close(self) -> None: - self.closed = True - - -class AcceptSequence: - def __init__(self, server: PrismCoordinator, count: int, *, port: int) -> None: - self.server = server - self.sockets = [FakeSocket() for _ in range(count)] - self.port = port - self.index = 0 - - def accept(self) -> tuple[FakeSocket, tuple[str, int]]: - if self.index == len(self.sockets): - self.server.stop_event.set() - raise socket.timeout - sock = self.sockets[self.index] - address = ("127.0.0.1", self.port + self.index) - self.index += 1 - return sock, address - - -class DormantThread: - created = 0 - - def __init__(self, *args: object, **kwargs: object) -> None: - type(self).created += 1 - - def start(self) -> None: - return - - -def worker(name: str) -> WorkerIdentity: - return WorkerIdentity( - username=name, - payout_address=name, - worker_name=None, - script_pubkey_hex="5220" + "11" * 32, - p2mr_program_hex="11" * 32, - ) - - -def listener(name: str, port: int) -> StratumListenerProfile: - config = vardiff.VardiffConfig( - enabled=False, - target_share_interval_seconds=Decimal("15"), - min_difficulty=Decimal("1"), - max_difficulty=Decimal("1024"), - retarget_interval_seconds=Decimal("90"), - max_step_factor=Decimal("4"), - startup_difficulty=Decimal("1"), - max_step_down_factor=Decimal("4"), - ewma_alpha=Decimal("0.4"), - retarget_tolerance=Decimal("0.25"), - ) - return StratumListenerProfile( - name=name, - bind="127.0.0.1", - port=port, - share_difficulty=Decimal("1"), - vardiff_config=config, - heartbeat_name=f"accept-{name}", - ) - - -def coordinator(*, connection_limit: int = 3, pending_limit: int = 2) -> PrismCoordinator: - server = PrismCoordinator.__new__(PrismCoordinator) - server.lock = threading.RLock() - server.stop_event = threading.Event() - server.clients = set() - server.jobs = {} - server.connection_counter = 0 - server.job_counter = 0 - server.stratum_max_connections = connection_limit - server.stratum_max_connections_per_username = 0 - server.stratum_max_pending_initial_jobs = pending_limit - server.stratum_initial_job_timeout_seconds = 30.0 - server.mining_health_startup_grace_seconds = 30.0 - server.tip_refresh_max_workers = 1 - server.tip_template_snapshot = None - server.started_monotonic = 0.0 - server.submitted_share_count = 0 - server.rejection_counts_by_reason = {} - server.tip_template_snapshot = None - server._record_heartbeat = lambda _name: None # type: ignore[method-assign] - server.apply_stratum_send_timeout = lambda _sock: None # type: ignore[method-assign] - server.client_startup_difficulty = lambda _profile: Decimal("1") # type: ignore[method-assign] - server._ensure_job_cache_state() - server._ensure_tip_refresh_state() - server._ensure_initial_job_state() - return server - - -def client(server: PrismCoordinator, connection_id: int, *, with_job: bool = False) -> ClientState: - state = ClientState( - sock=FakeSocket(), - address=("127.0.0.1", 40_000 + connection_id), - connection_id=connection_id, - extranonce1_hex=f"{connection_id:08x}", - ) - state.subscribed = True - state.authorized = True - state.authorization_generation = 1 - state.worker = worker(f"miner-{connection_id}") - state.username = state.worker.username - if with_job: - state.active_job = SimpleNamespace( - template={"previousblockhash": "aa" * 32}, - payout_state_generation=0, - ) - server.clients.add(state) - return state - - -class PrismReconnectBackpressureTests(unittest.TestCase): - def tearDown(self) -> None: - DormantThread.created = 0 - - def test_global_cap_precedes_client_and_handler_allocation_during_storm(self) -> None: - server = coordinator(connection_limit=3) - accepted = AcceptSequence(server, 1_000, port=10_000) - - with patch("lab.prism.prism_coordinator.threading.Thread", DormantThread): - server.accept_loop(accepted, listener("default", 3340)) # type: ignore[arg-type] - - self.assertEqual(len(server.clients), 3) - self.assertEqual(server.handler_thread_count, 3) - self.assertEqual(DormantThread.created, 3) - self.assertEqual(server.connection_limit_rejection_counts["global"], 997) - self.assertTrue(all(sock.closed for sock in accepted.sockets[3:])) - self.assertTrue(all(not sock.closed for sock in accepted.sockets[:3])) - - def test_cross_listener_accounting_uses_one_global_cap(self) -> None: - server = coordinator(connection_limit=3) - default_accepts = AcceptSequence(server, 2, port=20_000) - with patch("lab.prism.prism_coordinator.threading.Thread", DormantThread): - server.accept_loop(default_accepts, listener("default", 3340)) # type: ignore[arg-type] - server.stop_event.clear() - high_accepts = AcceptSequence(server, 2, port=30_000) - server.accept_loop(high_accepts, listener("highdiff", 3342)) # type: ignore[arg-type] - - self.assertEqual(len(server.clients), 3) - self.assertEqual( - sorted(state.listener_name for state in server.clients), - ["default", "default", "highdiff"], - ) - self.assertTrue(high_accepts.sockets[1].closed) - - def test_current_job_coverage_requires_an_observed_matching_tip(self) -> None: - server = coordinator(connection_limit=2) - client(server, 1, with_job=True) - - missing_tip = server.mining_delivery_snapshot(now=1.0) - self.assertEqual(missing_tip["clients_with_current_tip_jobs"], 0) - self.assertEqual(missing_tip["current_tip_job_coverage"], 0.0) - - server.current_tip_first_seen = ("aa" * 32, None) - matching_tip = server.mining_delivery_snapshot(now=2.0) - self.assertEqual(matching_tip["clients_with_current_tip_jobs"], 1) - self.assertEqual(matching_tip["current_tip_job_coverage"], 1.0) - - server.current_tip_first_seen = ("bb" * 32, time.monotonic()) - advanced_tip = server.mining_delivery_snapshot(now=3.0) - self.assertEqual(advanced_tip["clients_with_current_tip_jobs"], 0) - self.assertEqual(advanced_tip["current_tip_job_coverage"], 0.0) - - def test_long_authorized_previous_tip_work_gets_fresh_coverage_grace(self) -> None: - server = coordinator(connection_limit=2) - server.mining_health_startup_grace_seconds = 0.0 - server.stratum_initial_job_timeout_seconds = 5.0 - state = client(server, 1, with_job=True) - state.authorized_monotonic = 1.0 - state.tip_work_delivered = ("aa" * 32, 2.0) - server.current_tip_first_seen = ("bb" * 32, 100.0) - - snapshot = server.mining_delivery_snapshot(now=100.0) - - self.assertTrue(snapshot["mining_ready"]) - self.assertFalse(snapshot["initial_delivery_stalled"]) - self.assertEqual(snapshot["current_tip_coverage_gap_age_seconds"], 0.0) - self.assertEqual( - snapshot["oldest_genuinely_pending_initial_job_age_seconds"], - 0.0, - ) - self.assertEqual(snapshot["oldest_initial_job_pending_seconds"], 0.0) - - def test_sustained_tip_coverage_gap_fails_closed_and_delivery_resets_it(self) -> None: - server = coordinator(connection_limit=2) - server.mining_health_startup_grace_seconds = 0.0 - server.stratum_initial_job_timeout_seconds = 5.0 - state = client(server, 1, with_job=True) - state.authorized_monotonic = 1.0 - state.tip_work_delivered = ("aa" * 32, 2.0) - server.current_tip_first_seen = ("bb" * 32, 100.0) - - self.assertTrue(server.mining_delivery_snapshot(now=100.0)["mining_ready"]) - grace = server.mining_delivery_snapshot(now=104.999) - self.assertTrue(grace["mining_ready"]) - self.assertEqual(grace["current_tip_coverage_gap_age_seconds"], 4.999) - - failed = server.mining_delivery_snapshot(now=105.0) - self.assertFalse(failed["mining_ready"]) - self.assertIn("initial-delivery-stalled", failed["unhealthy_reasons"]) - - state.active_job = SimpleNamespace( - template={"previousblockhash": "bb" * 32}, - payout_state_generation=0, - ) - server.note_tip_work_delivered(state, "bb" * 32) - recovered = server.mining_delivery_snapshot(now=106.0) - self.assertTrue(recovered["mining_ready"]) - self.assertEqual(recovered["current_tip_coverage_gap_age_seconds"], 0.0) - - server.current_tip_first_seen = ("cc" * 32, 107.0) - next_gap = server.mining_delivery_snapshot(now=107.0) - self.assertTrue(next_gap["mining_ready"]) - self.assertEqual(next_gap["current_tip_coverage_gap_age_seconds"], 0.0) - - def test_never_delivered_initial_job_uses_authorization_deadline(self) -> None: - server = coordinator(connection_limit=2) - server.mining_health_startup_grace_seconds = 0.0 - server.stratum_initial_job_timeout_seconds = 5.0 - state = client(server, 1) - state.authorized_monotonic = 90.0 - server.current_tip_first_seen = ("aa" * 32, None) - - grace = server.mining_delivery_snapshot(now=94.999) - self.assertTrue(grace["mining_ready"]) - self.assertEqual( - grace["oldest_genuinely_pending_initial_job_age_seconds"], - 4.999, - ) - - failed = server.mining_delivery_snapshot(now=95.0) - self.assertFalse(failed["mining_ready"]) - self.assertTrue(failed["initial_delivery_stalled"]) - self.assertIn("initial-delivery-stalled", failed["unhealthy_reasons"]) - self.assertEqual(failed["oldest_initial_job_pending_seconds"], 5.0) - - def test_tip_churn_with_successful_deliveries_does_not_inherit_old_gap(self) -> None: - server = coordinator(connection_limit=2) - server.mining_health_startup_grace_seconds = 0.0 - server.stratum_initial_job_timeout_seconds = 5.0 - state = client(server, 1, with_job=True) - state.authorized_monotonic = 1.0 - state.tip_work_delivered = ("aa" * 32, 2.0) - - server.current_tip_first_seen = ("bb" * 32, 100.0) - self.assertTrue(server.mining_delivery_snapshot(now=100.0)["mining_ready"]) - - state.active_job = SimpleNamespace( - template={"previousblockhash": "bb" * 32}, - payout_state_generation=0, - ) - server.note_tip_work_delivered(state, "bb" * 32) - # The health refresh owns the fleet coverage census; delivery no - # longer repeats it per job (#159). The refresh that follows the - # delivery observes restored coverage and closes the gap, so the next - # tip's gap starts fresh instead of inheriting this one. - restored = server.mining_delivery_snapshot(now=101.0) - self.assertTrue(restored["mining_ready"]) - self.assertEqual(restored["current_tip_coverage_gap_age_seconds"], 0.0) - - server.current_tip_first_seen = ("cc" * 32, 106.0) - second_gap = server.mining_delivery_snapshot(now=106.0) - self.assertTrue(second_gap["mining_ready"]) - self.assertEqual(second_gap["current_tip_coverage_gap_age_seconds"], 0.0) - - state.active_job = SimpleNamespace( - template={"previousblockhash": "cc" * 32}, - payout_state_generation=0, - ) - server.note_tip_work_delivered(state, "cc" * 32) - recovered = server.mining_delivery_snapshot(now=107.0) - self.assertTrue(recovered["mining_ready"]) - self.assertEqual(recovered["current_tip_coverage_gap_age_seconds"], 0.0) - - server.current_tip_first_seen = ("dd" * 32, 112.0) - third_gap = server.mining_delivery_snapshot(now=112.0) - self.assertTrue(third_gap["mining_ready"]) - self.assertEqual(third_gap["current_tip_coverage_gap_age_seconds"], 0.0) - - def test_pending_initial_jobs_are_bounded_and_duplicate_requests_coalesce(self) -> None: - server = coordinator(connection_limit=5, pending_limit=2) - server.initial_job_max_workers = 1 - started = threading.Event() - release = threading.Event() - - def blocked(request: PendingInitialJob) -> bool: - started.set() - release.wait(5) - return not server._initial_request_cancelled(request) - - server._run_initial_job = blocked # type: ignore[method-assign] - first, second, excess = (client(server, index) for index in (1, 2, 3)) - - self.assertTrue(server.schedule_initial_job(first)) - self.assertTrue(started.wait(5)) - self.assertTrue(server.schedule_initial_job(first)) - self.assertTrue(server.schedule_initial_job(second)) - self.assertFalse(server.schedule_initial_job(excess)) - - executor = server.initial_job_executor() - self.assertEqual(executor.max_queue_size, 2) - self.assertEqual(executor.stats(), (1, 1)) - self.assertEqual(len(server.pending_initial_jobs), 2) - self.assertEqual(server.initial_job_coalesced_count, 1) - self.assertEqual(server.initial_job_queue_rejection_count, 1) - self.assertNotIn(excess, server.clients) - self.assertTrue(excess.sock.closed) - release.set() - server.shutdown_tip_refresh_executor() - - def test_cancelled_queued_requests_immediately_reclaim_admission(self) -> None: - server = coordinator(connection_limit=6, pending_limit=2) - server.initial_job_max_workers = 1 - started = threading.Event() - release = threading.Event() - - def blocked(_request: PendingInitialJob) -> bool: - started.set() - release.wait(5) - return False - - server._run_initial_job = blocked # type: ignore[method-assign] - running = client(server, 1) - cancelled = [client(server, index) for index in (2, 3)] - replacement = client(server, 4) - - try: - self.assertTrue(server.schedule_initial_job(running)) - self.assertTrue(started.wait(5)) - for state in cancelled: - self.assertTrue(server.schedule_initial_job(state)) - self.assertEqual(server.initial_job_executor().stats(), (1, 1)) - server.cancel_initial_job_delivery(state) - server.cancel_initial_job_delivery(running) - - executor = server.initial_job_executor() - self.assertEqual(executor.stats(), (0, 1)) - health = server.mining_delivery_snapshot() - self.assertEqual(health["pending_initial_jobs"], 0) - self.assertFalse(health["pending_initial_jobs_saturated"]) - self.assertEqual(server.initial_job_queue_capacity_reclaimed_count, 2) - metrics = "\n".join(server.initial_delivery_metrics_lines()) - self.assertIn("qbit_prism_initial_job_delivery_queue_depth 0", metrics) - self.assertIn( - "qbit_prism_initial_job_queue_capacity_reclaimed_total 2", - metrics, - ) - - # Before cancellation-aware removal, the two tombstones filled the - # physical queue and this valid replacement was rejected. - self.assertTrue(server.schedule_initial_job(replacement)) - self.assertEqual(executor.stats(), (1, 1)) - self.assertIn(replacement, server.clients) - self.assertFalse(replacement.sock.closed) - finally: - release.set() - server.shutdown_tip_refresh_executor() - - def test_disconnect_and_timeout_reclaim_queued_admission(self) -> None: - server = coordinator(connection_limit=6, pending_limit=2) - server.initial_job_max_workers = 1 - started = threading.Event() - release = threading.Event() - - def blocked(_request: PendingInitialJob) -> bool: - started.set() - release.wait(5) - return False - - server._run_initial_job = blocked # type: ignore[method-assign] - running = client(server, 1) - disconnected = client(server, 2) - timed_out = client(server, 3) - replacement = client(server, 4) - - try: - self.assertTrue(server.schedule_initial_job(running)) - self.assertTrue(started.wait(5)) - server.pending_initial_jobs[running].deadline_monotonic = None - - self.assertTrue(server.schedule_initial_job(disconnected)) - server.disconnect_client(disconnected) - self.assertEqual(server.initial_job_executor().stats(), (0, 1)) - self.assertTrue(disconnected.sock.closed) - - self.assertTrue(server.schedule_initial_job(timed_out)) - request = server.pending_initial_jobs[timed_out] - assert request.deadline_monotonic is not None - self.assertEqual( - server.sweep_initial_job_timeouts( - now=request.deadline_monotonic, - ), - 1, - ) - self.assertEqual(server.initial_job_executor().stats(), (0, 1)) - self.assertTrue(timed_out.sock.closed) - self.assertEqual(server.initial_job_timeout_count, 1) - - self.assertTrue(server.schedule_initial_job(replacement)) - self.assertEqual(server.initial_job_executor().stats(), (1, 1)) - self.assertEqual(server.initial_job_queue_capacity_reclaimed_count, 2) - finally: - release.set() - server.shutdown_tip_refresh_executor() - - def test_saturated_queue_reauthorization_preserves_working_session(self) -> None: - server = coordinator(connection_limit=4, pending_limit=1) - server.initial_job_max_workers = 1 - server.current_tip_first_seen = ("aa" * 32, time.monotonic()) - started = threading.Event() - release = threading.Event() - - def blocked(_request: PendingInitialJob) -> bool: - started.set() - release.wait(5) - return False - - server._run_initial_job = blocked # type: ignore[method-assign] - waiting = client(server, 1) - live = client(server, 2, with_job=True) - original_worker = live.worker - server.resolve_worker = lambda username: worker(username) # type: ignore[method-assign] - - try: - self.assertTrue(server.schedule_initial_job(waiting)) - self.assertTrue(started.wait(5)) - with self.assertRaises(StratumError) as raised: - server.handle_request( - live, - { - "id": 7, - "method": "mining.authorize", - "params": ["replacement-worker", "x"], - }, - ) - - self.assertFalse(raised.exception.disconnect) - self.assertEqual( - raised.exception.message, - "initial job delivery capacity unavailable", - ) - self.assertEqual(live.authorization_generation, 1) - self.assertIs(live.worker, original_worker) - self.assertIn(live, server.clients) - self.assertFalse(live.sock.closed) - self.assertIsNotNone(live.active_job) - self.assertIs(server.pending_initial_jobs[waiting].client, waiting) - self.assertEqual(server.initial_job_queue_rejection_count, 0) - finally: - release.set() - server.shutdown_tip_refresh_executor() - - def test_executor_cancel_racing_dequeue_does_not_leak_or_double_release(self) -> None: - for _ in range(10): - executor = _BoundedPriorityExecutor(max_workers=1, max_queue_size=1) - blocker_started = threading.Event() - release_blocker = threading.Event() - raced_started = threading.Event() - release_raced = threading.Event() - cancel_now = threading.Event() - cancel_results: list[bool] = [] - - def blocker() -> None: - blocker_started.set() - release_blocker.wait(5) - - def raced() -> None: - raced_started.set() - release_raced.wait(5) - - first = executor.submit(blocker) - self.assertTrue(blocker_started.wait(5)) - second = executor.submit(raced) - - def cancel_raced() -> None: - cancel_now.wait(5) - cancel_results.append(executor.cancel(second)) - - cancel_thread = threading.Thread(target=cancel_raced) - cancel_thread.start() - cancel_now.set() - release_blocker.set() - release_raced.set() - cancel_thread.join(5) - self.assertFalse(cancel_thread.is_alive()) - first.result(5) - if not second.cancelled(): - second.result(5) - - # A second cancellation cannot release the same queue entry, and - # fresh bounded work remains admissible whichever side won. - self.assertFalse(executor.cancel(second)) - probe = executor.submit(lambda: "admitted") - self.assertEqual(probe.result(5), "admitted") - executor._queue.join() - self.assertEqual(executor.stats(), (0, 0)) - self.assertEqual(executor._queue.unfinished_tasks, 0) - self.assertEqual(len(cancel_results), 1) - executor.shutdown(wait=True, cancel_futures=True) - - def test_initial_job_deadline_starts_only_after_protocol_handshake(self) -> None: - server = coordinator(connection_limit=3, pending_limit=2) - release = threading.Event() - server._run_initial_job = lambda _request: release.wait(5) # type: ignore[method-assign] - state = client(server, 1) - state.subscribed = False - - self.assertTrue(server.schedule_initial_job(state)) - self.assertNotIn(state, server.pending_initial_jobs) - - state.subscribed = True - self.assertTrue(server.schedule_initial_job(state)) - self.assertIn(state, server.pending_initial_jobs) - release.set() - server.shutdown_tip_refresh_executor() - - def test_expired_initial_request_is_fenced_before_preparation(self) -> None: - server = coordinator(connection_limit=2, pending_limit=1) - state = client(server, 1) - preparation_started = threading.Event() - request = PendingInitialJob( - client=state, - connection_id=state.connection_id, - authorization_generation=state.authorization_generation, - difficulty_generation=state.difficulty_generation, - worker=state.worker, - requested_monotonic=time.monotonic() - 2, - deadline_monotonic=time.monotonic() - 1, - ) - server.pending_initial_jobs = {state: request} - server.ensure_reorg_reconciled_for_current_tip = ( # type: ignore[method-assign] - lambda: preparation_started.set() or True - ) - - self.assertFalse(server._run_initial_job(request)) - self.assertFalse(preparation_started.is_set()) - - def test_failed_initial_job_releases_capacity_and_disconnects_client(self) -> None: - server = coordinator(connection_limit=3, pending_limit=1) - server._run_initial_job = lambda _request: False # type: ignore[method-assign] - state = client(server, 1) - - self.assertTrue(server.schedule_initial_job(state)) - deadline = time.monotonic() + 5 - while state in server.clients and time.monotonic() < deadline: - time.sleep(0.01) - - self.assertNotIn(state, server.clients) - self.assertNotIn(state, server.pending_initial_jobs) - self.assertTrue(state.sock.closed) - server.shutdown_tip_refresh_executor() - - def test_reauthorize_schedules_current_work_after_stale_direct_delivery(self) -> None: - server = coordinator(connection_limit=3, pending_limit=2) - server.current_tip_first_seen = ("bb" * 32, time.monotonic()) - state = client(server, 1, with_job=True) - old_request = PendingInitialJob( - client=state, - authorization_generation=state.authorization_generation, - worker=state.worker, - requested_monotonic=0.0, - deadline_monotonic=30.0, - ) - server.pending_initial_jobs = {state: old_request} - release = threading.Event() - server._run_initial_job = lambda _request: release.wait(5) # type: ignore[method-assign] - server.resolve_worker = lambda username: worker(username) # type: ignore[method-assign] - - def reserve(current: ClientState, identity: WorkerIdentity) -> bool: - current.worker = identity - current.username = identity.username - return True - - server.reserve_client_username = reserve # type: ignore[method-assign] - server.apply_client_difficulty_requests = lambda _client: Decimal("2") # type: ignore[method-assign] - server.advertise_client_difficulty = lambda _client, _target: True # type: ignore[method-assign] - server.send_result = lambda *_args: None # type: ignore[method-assign] - - server.handle_request( - state, - { - "id": 7, - "method": "mining.authorize", - "params": ["miner-new", "d=2"], - }, - ) - - replacement = server.pending_initial_jobs[state] - self.assertIsNot(replacement, old_request) - self.assertTrue(old_request.cancelled.is_set()) - self.assertEqual(replacement.authorization_generation, 2) - release.set() - server.shutdown_tip_refresh_executor() - - def test_difficulty_change_replaces_pending_first_job_generation(self) -> None: - server = coordinator(connection_limit=3, pending_limit=2) - server.current_tip_first_seen = ("aa" * 32, time.monotonic()) - state = client(server, 1) - started = threading.Event() - release = threading.Event() - - def deliver(request: PendingInitialJob) -> bool: - if request.difficulty_generation == 0: - started.set() - release.wait(5) - if server._initial_request_cancelled(request): - return False - request.client.active_job = SimpleNamespace( - template={"previousblockhash": "aa" * 32}, - payout_state_generation=0, - connection_id=request.connection_id, - authorization_generation=request.authorization_generation, - difficulty_generation=request.difficulty_generation, - ) - server.note_initial_job_delivered(request.client) - return True - - server._run_initial_job = deliver # type: ignore[method-assign] - self.assertTrue(server.request_initial_job_delivery(state)) - self.assertTrue(started.wait(5)) - original = server.pending_initial_jobs[state] - - self.assertFalse( - server.advertise_client_difficulty(state, Decimal("4")) - ) - - replacement = server.pending_initial_jobs[state] - self.assertIsNot(replacement, original) - self.assertTrue(original.cancelled.is_set()) - self.assertEqual(replacement.difficulty_generation, 1) - self.assertEqual(state.difficulty_generation, 1) - self.assertEqual(state.share_difficulty, Decimal("4")) - self.assertIsNone(state.pending_share_difficulty) - - release.set() - deadline = time.monotonic() + 5 - while server.pending_initial_jobs and time.monotonic() < deadline: - time.sleep(0.01) - server.shutdown_tip_refresh_executor() - - self.assertEqual(server.pending_initial_jobs, {}) - self.assertIn(state, server.clients) - self.assertFalse(state.sock.closed) - self.assertIsNotNone(state.active_job) - self.assertEqual(state.active_job.difficulty_generation, 1) - - def test_timeout_cleans_state_and_prevents_late_delivery(self) -> None: - server = coordinator(connection_limit=3, pending_limit=2) - release = threading.Event() - sent: list[int] = [] - - def blocked(request: PendingInitialJob) -> bool: - release.wait(5) - if not server._initial_request_cancelled(request): - sent.append(request.client.connection_id) - return False - - server._run_initial_job = blocked # type: ignore[method-assign] - state = client(server, 1) - self.assertTrue(server.schedule_initial_job(state)) - request = server.pending_initial_jobs[state] - - expired = server.sweep_initial_job_timeouts( - now=request.requested_monotonic + 31, - ) - release.set() - server.shutdown_tip_refresh_executor() - - self.assertEqual(expired, 1) - self.assertEqual(server.initial_job_timeout_count, 1) - self.assertNotIn(state, server.clients) - self.assertNotIn(state, server.pending_initial_jobs) - self.assertIsNone(state.worker) - self.assertEqual(state.active_job_ids, set()) - self.assertTrue(state.sock.closed) - self.assertEqual(sent, []) - - def test_timeout_commits_closing_before_disconnect_handoff(self) -> None: - server = coordinator(connection_limit=3, pending_limit=2) - state = client(server, 1) - request = PendingInitialJob( - client=state, - authorization_generation=state.authorization_generation, - worker=state.worker, - requested_monotonic=0.0, - deadline_monotonic=1.0, - ) - server.pending_initial_jobs = {state: request} - disconnect_client = server.disconnect_client - rescheduled: list[bool] = [] - - def observe_handoff(current: ClientState) -> None: - self.assertTrue(current.closing) - rescheduled.append(server.schedule_initial_job(current)) - self.assertNotIn(current, server.pending_initial_jobs) - disconnect_client(current) - - server.disconnect_client = observe_handoff # type: ignore[method-assign] - - self.assertEqual(server.sweep_initial_job_timeouts(now=2.0), 1) - - self.assertEqual(rescheduled, [True]) - self.assertNotIn(state, server.clients) - self.assertTrue(state.sock.closed) - - def test_overload_rejection_never_evicts_current_work(self) -> None: - server = coordinator(connection_limit=4, pending_limit=1) - healthy = client(server, 1, with_job=True) - waiting = client(server, 2) - excess = client(server, 3) - request = PendingInitialJob( - client=waiting, - authorization_generation=1, - worker=waiting.worker, - requested_monotonic=10.0, - deadline_monotonic=40.0, - ) - server.pending_initial_jobs = {waiting: request} - - self.assertFalse(server.schedule_initial_job(excess)) - - self.assertIn(healthy, server.clients) - self.assertIsNotNone(healthy.active_job) - self.assertIn(waiting, server.clients) - self.assertNotIn(excess, server.clients) - - def test_recovery_releases_pending_capacity_without_restart(self) -> None: - server = coordinator(connection_limit=4, pending_limit=2) - server.current_tip_first_seen = ("aa" * 32, None) - bundle_ready = threading.Event() - - def deliver(request: PendingInitialJob) -> bool: - bundle_ready.wait(5) - if server._initial_request_cancelled(request): - return False - request.client.active_job = SimpleNamespace( - template={"previousblockhash": "aa" * 32}, - payout_state_generation=0, - ) - server.note_initial_job_delivered(request.client) - return True - - server._run_initial_job = deliver # type: ignore[method-assign] - first, second = client(server, 1), client(server, 2) - self.assertTrue(server.schedule_initial_job(first)) - self.assertTrue(server.schedule_initial_job(second)) - self.assertEqual(len(server.pending_initial_jobs), 2) - - bundle_ready.set() - deadline = time.monotonic() + 5 - while server.pending_initial_jobs and time.monotonic() < deadline: - time.sleep(0.01) - - self.assertEqual(server.pending_initial_jobs, {}) - self.assertIsNotNone(first.active_job) - self.assertIsNotNone(second.active_job) - newcomer = client(server, 3) - self.assertTrue(server.schedule_initial_job(newcomer)) - server.shutdown_tip_refresh_executor() - - def test_delivery_executor_prioritizes_initial_then_new_tip_then_same_tip(self) -> None: - executor = _BoundedPriorityExecutor(max_workers=1, max_queue_size=4) - blocker_started = threading.Event() - release = threading.Event() - order: list[str] = [] - - def blocker() -> None: - blocker_started.set() - release.wait(5) - - executor.submit(blocker, priority=PRISM_DELIVERY_PRIORITY_SAME_TIP) - self.assertTrue(blocker_started.wait(5)) - routine = executor.submit( - lambda: order.append("routine"), - priority=PRISM_DELIVERY_PRIORITY_SAME_TIP, - ) - initial = executor.submit( - lambda: order.append("initial"), - priority=PRISM_DELIVERY_PRIORITY_INITIAL, - ) - new_tip = executor.submit( - lambda: order.append("new-tip"), - priority=PRISM_DELIVERY_PRIORITY_NEW_TIP, - ) - release.set() - new_tip.result(5) - initial.result(5) - routine.result(5) - executor.shutdown(wait=True, cancel_futures=True) - - self.assertEqual(order, ["initial", "new-tip", "routine"]) - - def test_blocked_initial_workers_do_not_starve_new_tip_delivery(self) -> None: - server = coordinator(connection_limit=8, pending_limit=8) - server.initial_job_max_workers = 4 - started = 0 - started_lock = threading.Lock() - all_started = threading.Event() - - def blocked_retry(request: PendingInitialJob) -> bool: - nonlocal started - with started_lock: - started += 1 - if started == server.initial_job_max_workers: - all_started.set() - while not request.cancelled.wait(0.01): - pass - return False - - server._run_initial_job = blocked_retry # type: ignore[method-assign] - clients = [client(server, index) for index in range(1, 9)] - for state in clients: - self.assertTrue(server.schedule_initial_job(state)) - - try: - self.assertTrue(all_started.wait(5)) - tip_started = threading.Event() - - def publish_new_tip() -> str: - tip_started.set() - return "published" - - future = server._submit_delivery_task( - server.tip_refresh_executor(), - publish_new_tip, - priority=PRISM_DELIVERY_PRIORITY_NEW_TIP, - ) - self.assertTrue(tip_started.wait(0.5)) - self.assertEqual(future.result(1), "published") - self.assertEqual(len(server.pending_initial_jobs), 8) - self.assertTrue( - server.mining_delivery_snapshot()["pending_initial_jobs_saturated"] - ) - self.assertEqual(server.initial_job_executor().stats(), (4, 4)) - metrics = "\n".join(server.initial_delivery_metrics_lines()) - self.assertIn("qbit_prism_initial_job_delivery_queue_depth 4", metrics) - self.assertIn("qbit_prism_initial_job_delivery_active_workers 4", metrics) - self.assertIn("qbit_prism_initial_job_delivery_configured_workers 4", metrics) - finally: - server.shutdown_tip_refresh_executor() - - def test_640_client_storm_cannot_displace_latest_tip_builder(self) -> None: - server = coordinator(connection_limit=800, pending_limit=128) - server.initial_job_max_workers = 4 - workers_ready = 0 - workers_contending = 0 - workers_passed_priority = 0 - worker_lock = threading.Lock() - all_workers_ready = threading.Event() - all_workers_contending = threading.Event() - any_worker_passed_priority = threading.Event() - contend = threading.Event() - release_workers = threading.Event() - - def build_request( - key: str, - *, - publication_critical: bool, - source: str, - requested_monotonic: float | None = None, - ) -> SimpleNamespace: - return SimpleNamespace( - idle_retarget=False, - mode="ready", - equivalence_key=(key,), - cache_key=(key,), - cancellation=_JobBuildCancellation(timeout_seconds=30.0), - promise=Future(), - publication_critical=publication_critical, - request_source=source, - requested_monotonic=( - time.monotonic() - if requested_monotonic is None - else requested_monotonic - ), - priority_admission_recorded=False, - ) - - def contending_initial(_request: PendingInitialJob) -> bool: - nonlocal workers_ready, workers_contending, workers_passed_priority - with worker_lock: - workers_ready += 1 - if workers_ready == server.initial_job_max_workers: - all_workers_ready.set() - if not contend.wait(5): - raise AssertionError("test did not release initial builders") - with worker_lock: - workers_contending += 1 - if workers_contending == server.initial_job_max_workers: - all_workers_contending.set() - preparation_token, _preparation_cancellation, _subscribed = ( - server._begin_routine_job_build_preparation( - request_source="initial", - cancelled=_request.cancelled.is_set, - ) - ) - try: - with worker_lock: - workers_passed_priority += 1 - any_worker_passed_priority.set() - release_workers.wait(5) - finally: - server._finish_routine_job_build_preparation( - preparation_token - ) - return False - - server._run_initial_job = contending_initial # type: ignore[method-assign] - clients = [client(server, index) for index in range(1, 641)] - admitted = 0 - for state in clients: - admitted += int(server.schedule_initial_job(state)) - - self.assertTrue(all_workers_ready.wait(5)) - self.assertEqual(admitted, 128) - self.assertEqual(len(server.pending_initial_jobs), 128) - self.assertEqual(server.initial_job_executor().stats(), (124, 4)) - - def start_without_execution(request: object) -> SimpleNamespace: - server.job_build_scheduler_counts["starts"] += 1 - server._record_priority_admission_locked( # type: ignore[arg-type] - request, - "started", - ) - return SimpleNamespace(request=request, future=Future()) - - server._start_job_build_locked = start_without_execution # type: ignore[method-assign] - server._arm_job_build_locked = lambda _flight: None # type: ignore[method-assign] - admission_started = time.monotonic() - priority_token, priority_requested = ( - server._begin_job_build_priority_preparation() - ) - latest = build_request( - "latest-tip", - publication_critical=True, - source="tip_refresh", - requested_monotonic=priority_requested, - ) - try: - latest_promise = server._request_job_build(latest) # type: ignore[arg-type] - finally: - server._finish_job_build_priority_preparation(priority_token) - admission_elapsed = time.monotonic() - admission_started - contend.set() - - try: - self.assertTrue(all_workers_contending.wait(5)) - self.assertFalse(any_worker_passed_priority.wait(0.1)) - self.assertLess(admission_elapsed, 0.1) - self.assertIs(latest_promise, latest.promise) - self.assertFalse(latest.cancellation.is_set()) - assert server._job_build_active is not None - self.assertIs(server._job_build_active.request, latest) - self.assertEqual( - server.job_build_priority_counts["routine_deferred"], - 4, - ) - self.assertEqual( - server.initial_job_prepared_work_counts["deferred"], - 4, - ) - self.assertEqual( - server.job_build_priority_admission_seconds["count"], - 1, - ) - finally: - with server._job_build_scheduler_lock: - server._job_build_active = None - server._job_build_priority_changed.set() - release_workers.set() - server.shutdown_initial_job_executor() - self.assertEqual(workers_passed_priority, 4) - - def test_shutdown_cancels_queued_initial_work_and_joins_both_executors(self) -> None: - server = coordinator(connection_limit=4, pending_limit=3) - server.initial_job_max_workers = 1 - initial_started = threading.Event() - initial_stopped = threading.Event() - initial_runs: list[int] = [] - queued_tip_ran = threading.Event() - - def blocked_initial(request: PendingInitialJob) -> bool: - initial_runs.append(request.client.connection_id) - initial_started.set() - request.cancelled.wait(5) - initial_stopped.set() - return False - - server._run_initial_job = blocked_initial # type: ignore[method-assign] - first, second = (client(server, index) for index in (1, 2)) - self.assertTrue(server.schedule_initial_job(first)) - self.assertTrue(initial_started.wait(5)) - self.assertTrue(server.schedule_initial_job(second)) - second_request = server.pending_initial_jobs[second] - assert second_request.future is not None - second_future = second_request.future - - tip_started = threading.Event() - release_tip = threading.Event() - - def blocked_tip() -> None: - tip_started.set() - release_tip.wait(5) - - tip_executor = server.tip_refresh_executor() - tip_executor.submit( - blocked_tip, - priority=PRISM_DELIVERY_PRIORITY_NEW_TIP, - ) - self.assertTrue(tip_started.wait(5)) - queued_tip = tip_executor.submit( - queued_tip_ran.set, - priority=PRISM_DELIVERY_PRIORITY_SAME_TIP, - ) - initial_executor = server.initial_job_executor() - initial_threads = tuple(initial_executor._threads) - tip_threads = tuple(tip_executor._threads) - shutdown_complete = threading.Event() - - def shutdown_executors() -> None: - server.shutdown_tip_refresh_executor() - shutdown_complete.set() - - shutdown_thread = threading.Thread(target=shutdown_executors) - shutdown_thread.start() - try: - self.assertTrue(initial_stopped.wait(1)) - self.assertFalse(shutdown_complete.wait(0.05)) - self.assertTrue(second_future.cancelled()) - self.assertTrue(queued_tip.cancelled()) - self.assertFalse(queued_tip_ran.is_set()) - finally: - release_tip.set() - shutdown_thread.join(5) - - self.assertFalse(shutdown_thread.is_alive()) - self.assertTrue(shutdown_complete.is_set()) - self.assertEqual(server.pending_initial_jobs, {}) - self.assertEqual(initial_runs, [first.connection_id]) - self.assertEqual(initial_executor.stats(), (0, 0)) - self.assertEqual(server.initial_job_queue_capacity_reclaimed_count, 1) - self.assertTrue(all(not thread.is_alive() for thread in initial_threads)) - self.assertTrue(all(not thread.is_alive() for thread in tip_threads)) - with self.assertRaisesRegex(RuntimeError, "initial job executor is shut down"): - server.initial_job_executor() - with self.assertRaisesRegex(RuntimeError, "tip refresh executor is shut down"): - server.tip_refresh_executor() - - def test_health_grace_failure_and_recovery_follow_job_coverage(self) -> None: - server = coordinator(connection_limit=2, pending_limit=2) - server.started_monotonic = 0.0 - first, second = client(server, 1), client(server, 2) - server.pending_initial_jobs = { - state: PendingInitialJob( - client=state, - authorization_generation=1, - worker=state.worker, - requested_monotonic=0.0, - deadline_monotonic=30.0, - ) - for state in (first, second) - } - - startup = server.mining_delivery_snapshot(now=10.0) - failed = server.mining_delivery_snapshot(now=31.0) - self.assertTrue(startup["mining_ready"]) - self.assertFalse(failed["mining_ready"]) - self.assertIn("initial-delivery-stalled", failed["unhealthy_reasons"]) - - server.current_tip_first_seen = ("aa" * 32, None) - for state in (first, second): - state.active_job = SimpleNamespace( - template={"previousblockhash": "aa" * 32}, - payout_state_generation=0, - ) - server.note_initial_job_delivered(state) - recovered = server.mining_delivery_snapshot(now=32.0) - - self.assertTrue(recovered["mining_ready"]) - self.assertEqual(recovered["current_tip_job_coverage"], 1.0) - self.assertTrue(recovered["connection_capacity_saturated"]) - - -if __name__ == "__main__": - unittest.main() diff --git a/tests/test_prism_recovery_evidence.py b/tests/test_prism_recovery_evidence.py new file mode 100644 index 000000000..0e3d04ef0 --- /dev/null +++ b/tests/test_prism_recovery_evidence.py @@ -0,0 +1,317 @@ +"""Fail closed on incomplete or inconsistent accounting reconciliation exports.""" +import hashlib +import importlib.util +import json +from pathlib import Path +import unittest + + +SCRIPT = Path(__file__).resolve().parents[1] / "scripts/prism-recovery-evidence.py" +spec = importlib.util.spec_from_file_location("recovery_evidence", SCRIPT) +module = importlib.util.module_from_spec(spec) +spec.loader.exec_module(module) + + +def record(kind, row): + return json.dumps({"kind": kind, "row": row}) + "\n" + + +def closing(**overrides): + report = {"mismatch_count": 0, "current_drift_count": 0, "checked_active_rows": 0} + report.update(overrides) + return [record("integrity", report), record("complete", True)] + + +def carry(seq, height, label, program, *, gross, prior, onchain=0): + """An active carry row as the export writes it, consistent in itself.""" + candidate = prior + gross + return { + "carry_forward_seq": seq, "block_hash": f"b{height}", "block_height": height, + "recipient_id": label, "order_key": label, "p2mr_program_hex": program, + "gross_amount_sats": gross, "prior_balance_sats": str(prior), + "candidate_balance_sats": str(candidate), "onchain_amount_sats": onchain, + "settlement_fee_sats": 0, "carry_forward_balance_sats": str(candidate - onchain), + "action": "onchain" if onchain else "accrued", "maturity_state": "mature", + } + + +class RecoveryEvidenceTests(unittest.TestCase): + def test_empty_history_has_legacy_zero_head(self): + report = module.summarize(iter(closing())) + self.assertEqual(report["audit_head_sha256"], "00" * 32) + self.assertEqual(report["accepted_shares"], 0) + + def test_legacy_head_keeps_ascii_escaping_for_unicode_recipients(self): + row = carry(1, 1, "Zoë🔑", "aa", gross=123, prior=0) + report = module.summarize(iter([record("active_carry", row)] + closing(checked_active_rows=1))) + # The legacy escaped bytes, with the UTF-16 surrogate pair for the key + # emoji; emitting raw UTF-8 produces a different head. + legacy = (b'{"action":"accrued","block_hash":"b1","block_height":1,' + b'"candidate_balance_sats":"123","carry_forward_balance_sats":"123",' + b'"carry_forward_seq":1,"gross_amount_sats":123,"maturity_state":"mature",' + b'"onchain_amount_sats":0,"order_key":"Zo\\u00eb\\ud83d\\udd11",' + b'"p2mr_program_hex":"aa","prior_balance_sats":"0",' + b'"recipient_id":"Zo\\u00eb\\ud83d\\udd11","settlement_fee_sats":0}') + self.assertEqual(report["audit_head_sha256"], + hashlib.sha256(bytes(32) + legacy).hexdigest()) + raw = legacy.replace(b"Zo\\u00eb\\ud83d\\udd11", "Zoë🔑".encode()) + self.assertNotEqual(report["audit_head_sha256"], + hashlib.sha256(bytes(32) + raw).hexdigest()) + + def test_legacy_chain_runs_per_payout_program_across_case_labels(self): + # Union's shape (#708): one program, an uppercase label carrying the + # balance a lowercase label accrued, interleaved by height, one row + # per program and block. + upper, lower = "QB1ZABC", "qb1zabc" + rows = [ + carry(1, 10, lower, "aa", gross=100, prior=0, onchain=0), + carry(2, 11, upper, "aa", gross=50, prior=100, onchain=120), + carry(3, 12, lower, "aa", gross=7, prior=30, onchain=0), + carry(4, 13, upper, "aa", gross=0, prior=37, onchain=0), + carry(5, 13, "other", "bb", gross=9, prior=0, onchain=0), + ] + lines = [record("active_carry", row) for row in rows] + # A 2.x validator partitions by label, so it lists the three rows + # whose label partition sum differs; 025's program rule lists none. + label_rule = [{"carry_forward_seq": seq, "block_hash": f"b{height}"} + for seq, height in ((2, 11), (3, 12), (4, 13))] + notes = [] + source = module.summarize(iter(lines + closing( + checked_active_rows=5, mismatch_count=3, mismatches=label_rule)), notes) + migrated = module.summarize(iter(lines + closing(checked_active_rows=5))) + self.assertEqual(source, migrated) + self.assertEqual(source["carry_forward_integrity"]["mismatch_count"], 0) + self.assertEqual(source["carry_forward_integrity"]["mismatches"], []) + self.assertIn("#708", notes[0]) + # A legacy finding the per-label rule does not make there is not + # cleared: the first row starts both chains at zero. + for unexplained in ({"carry_forward_seq": 1, "block_hash": "b10"}, + {"carry_forward_seq": 5, "block_hash": "b13"}): + with self.subTest(finding=unexplained), self.assertRaisesRegex(ValueError, "no chain rule"): + module.summarize(iter(lines + closing( + checked_active_rows=5, mismatch_count=4, mismatches=label_rule + [unexplained]))) + + def test_a_clean_report_is_summarized_as_the_database_wrote_it(self): + report = {"schema": "qbit.prism.carry-forward-integrity.v1", "checked_active_rows": 1, + "mismatch_count": 0, "current_drift_count": 0, "current_drift": [], "mismatches": []} + row = record("active_carry", carry(1, 10, "alice", "aa", gross=1, prior=0)) + summary = module.summarize(iter([row, record("integrity", report), record("complete", True)])) + self.assertEqual(summary["carry_forward_integrity"], report) + + def test_malformed_reports_and_amounts_fail_closed(self): + row = carry(1, 10, "alice", "aa", gross=1, prior=0) + for lines in ( + [record("integrity", [0]), record("complete", True)], + [record("active_carry", row)] + closing(checked_active_rows=1, mismatch_count=1, mismatches=[1]), + [record("active_carry", row | {"prior_balance_sats": " 0"})] + closing(checked_active_rows=1), + [record("active_carry", row | {"carry_forward_balance_sats": "0_001"})] + + closing(checked_active_rows=1), + ): + with self.subTest(lines=lines), self.assertRaises(ValueError): + module.summarize(iter(lines)) + + def test_a_real_legacy_chain_break_fails_in_any_label(self): + for field, value in (("prior_balance_sats", "99"), ("candidate_balance_sats", "8"), + ("carry_forward_balance_sats", "8")): + rows = [carry(1, 10, "qb1zabc", "aa", gross=100, prior=0), + carry(2, 11, "QB1ZABC", "aa", gross=0, prior=100) | {field: value}] + with self.subTest(field=field), self.assertRaisesRegex(ValueError, "payout program"): + module.summarize(iter([record("active_carry", row) for row in rows] + + closing(checked_active_rows=2))) + + def test_as_issued_findings_still_fail(self): + marked = record("blocks", {"block_hash": "b11", "as_issued_audit_sha256": "ab" * 32}) + row = record("active_carry", carry(1, 11, "alice", "aa", gross=5, prior=0)) + for finding in ({"carry_forward_seq": 1, "block_hash": "b11"}, + {"carry_forward_seq": None, "block_hash": "b11"}, + {"carry_forward_seq": None, "block_hash": "unmarked"}): + with self.subTest(finding=finding), self.assertRaisesRegex(ValueError, "as-issued"): + module.summarize(iter([marked, row] + closing( + checked_active_rows=1, mismatch_count=1, mismatches=[finding]))) + + def test_marked_rows_answer_to_their_manifest_not_the_running_chain(self): + # A divergent landing (#478) carries its manifest's prior, whatever + # the canonical balance was: the database's manifest rule judges it. + marked = record("blocks", {"block_hash": "b11", "as_issued_audit_sha256": "ab" * 32}) + rows = [carry(1, 10, "alice", "aa", gross=100, prior=0), + carry(2, 11, "alice", "aa", gross=10, prior=40)] + lines = [marked] + [record("active_carry", row) for row in rows] + report = module.summarize(iter(lines + closing(checked_active_rows=2))) + self.assertEqual(report["records"]["active_carry"]["count"], 2) + + def test_an_incomplete_mismatch_list_and_disordered_records_fail(self): + row = record("active_carry", carry(1, 10, "alice", "aa", gross=1, prior=0)) + later = record("active_carry", carry(2, 11, "alice", "aa", gross=1, prior=1)) + for lines in ( + [row] + closing(checked_active_rows=1, mismatch_count=2, + mismatches=[{"carry_forward_seq": 1, "block_hash": "b10"}]), + [later, row] + closing(checked_active_rows=2), + [row, record("blocks", {"block_hash": "b9"})] + closing(checked_active_rows=1), + ): + with self.subTest(lines=lines), self.assertRaises(ValueError): + module.summarize(iter(lines)) + + def test_interrupted_exports_and_trailing_records_fail(self): + for lines in ([], closing()[:-1], [record("complete", True)], closing() * 2): + with self.subTest(lines=lines), self.assertRaises(ValueError): + module.summarize(iter(lines)) + + def test_integrity_mismatch_and_missing_counts_fail(self): + for report in ( + {"mismatch_count": 1}, {"current_drift_count": 1}, + {"checked_active_rows": 1}, {"mismatch_count": None}, + ): + with self.subTest(report=report), self.assertRaises(ValueError): + module.summarize(iter(closing(**report))) + + def test_share_order_is_authoritative_and_gaps_are_valid(self): + shares = [record("shares", {"share_seq": seq, "accepted": True}) for seq in (1, 5, 8)] + report = module.summarize(iter(shares + closing())) + self.assertEqual(report["accepted_shares"], 3) + self.assertEqual(report["last_share_seq"], 8) + for invalid in (list(reversed(shares)), shares + shares[-1:]): + with self.subTest(invalid=invalid), self.assertRaises(ValueError): + module.summarize(iter(invalid + closing())) + changed = shares[:2] + [record("shares", {"share_seq": 8, "accepted": False})] + other = module.summarize(iter(changed + closing())) + self.assertEqual(other["accepted_shares"], 2) + self.assertNotEqual(report["records"]["shares"]["sha256"], other["records"]["shares"]["sha256"]) + + def test_unknown_records_fail_instead_of_skipping_accounting(self): + with self.assertRaises(ValueError): + module.summarize(iter([record("future_format", {})] + closing())) + + def test_a_pending_share_hash_backfill_marks_share_hashes_deferred_with_its_cursor(self): + # Fence 2 (`migrate --defer-share-hashes`): no share_hashes rows, and + # the cursor in their place, which only that kind's summary carries. + cursor = {"start_seq": 1, "next_seq": 1, "end_seq": 4129153, + "recent_min_height": 2000, "recent_start_seq": 30001} + shares = [record("shares", {"share_seq": seq, "accepted": True}) for seq in (1, 2)] + baseline = module.summarize(iter(shares + closing())) + self.assertNotIn("deferred", baseline["records"]["share_hashes"]) + notes = [] + # The export reads the cursor last, after the integrity report. + deferred = module.summarize(iter(shares + closing()[:1] + [record("share_hashes_deferred", cursor)] + + closing()[1:]), notes) + self.assertEqual(deferred["records"]["share_hashes"], { + "count": 0, "sha256": hashlib.sha256().hexdigest(), "deferred": cursor}) + self.assertEqual(deferred | {"records": baseline["records"]}, baseline) + for kind in baseline["records"]: + if kind != "share_hashes": + self.assertEqual(deferred["records"][kind], baseline["records"][kind]) + self.assertEqual(len(notes), 1) + self.assertIn("share_hashes deferred", notes[0]) + self.assertIn("from share_seq 1 up to 4129153", notes[0]) + # A cursor further on is different evidence. + moved = module.summarize(iter(shares + [record("share_hashes_deferred", cursor | {"next_seq": 9})] + + closing())) + self.assertNotEqual(moved, deferred) + # Before the integrity report it summarizes alike. + self.assertEqual(module.summarize(iter(shares + [record("share_hashes_deferred", cursor)] + closing())), + deferred) + + def test_a_share_hash_deferral_is_refused_beside_rows_twice_or_after_completion(self): + cursor = {"start_seq": 1, "next_seq": 1, "end_seq": 3} + row = record("share_hashes", {"header_hash": "ab", "share_id": "first"}) + deferral = record("share_hashes_deferred", cursor) + for lines, message in ( + ([row, deferral] + closing(), "beside their deferral"), + ([deferral, row] + closing(), "beside their deferral"), + ([deferral, deferral] + closing(), "duplicate share_hashes deferral"), + ([record("share_hashes_deferred", [1, 3])] + closing(), "not an object"), + (closing() + [deferral], "follow completion marker"), + ): + with self.subTest(lines=lines), self.assertRaisesRegex(ValueError, message): + module.summarize(iter(lines)) + + def test_offered_or_unknown_candidates_cannot_look_drained(self): + for state in ("offer_reserved", "offered", "reconciliation", "future_state"): + with self.subTest(state=state): + report = module.summarize(iter([record("candidates", {"state": state})] + closing())) + self.assertEqual(report["pending_candidates"], 0) + self.assertEqual(report["unfinished_candidates"], 1) + terminal = [record("candidates", {"state": state}) for state in ("submitted", "abandoned")] + self.assertEqual(module.summarize(iter(terminal + closing()))["unfinished_candidates"], 0) + + def test_recovery_obligations_change_summary_without_share_or_ctv_state_changes(self): + baseline = module.summarize(iter(closing())) + for kind, row, change in ( + ("share_sequence", {"last_value": 40, "is_called": True}, + {"last_value": 2}), + ("share_sequence", {"last_value": 40, "is_called": True}, + {"is_called": False}), + ("sequences", {"sequence": "qbit_pool_payout_entries_payout_entry_seq_seq", + "last_value": 40, "is_called": True}, + {"last_value": 2}), + ("sequences", {"sequence": "qbit_audit_publication_sequence_seq", + "last_value": 40, "is_called": True}, + {"is_called": False}), + ("share_hashes", {"header_hash": "ab", "share_id": "first"}, + {"share_id": "later"}), + ("candidate_balances", {"prior_balances_digest": "ab", "balances_sha256": "cd"}, + {"balances_sha256": "ef"}), + ("audit_bodies", {"block_hash": "ab", "audit_bundle": {"schema": "native"}}, + {"audit_bundle": {"schema": "corrupted"}}), + ("audit_snapshots", {"snapshot_sha256": "cd", "share_count": 3}, + {"share_count": 2}), + ("ctv_checkpoints", {"fanout_txid": "ab", "confirmed_depth": 999}, + {"confirmed_depth": 1000}), + ("ctv_retry_progress", {"fanout_txid": "ab", "broadcast_attempt_count": 40}, + {"next_broadcast_attempt_at": "2026-09-14T21:00:00"}), + ("cpfp_packages", {"fanout_txid": "ab", "signed_child_hex": None}, + {"signed_child_hex": "deadbeef"}), + ("cpfp_retired_funding", {"funding_txid": "cd", "wallet_lock_released": False}, + {"wallet_lock_released": True}), + ("ctv_broadcast_attempts", {"attempt_seq": 1, "submit_result": None}, + {"submit_result": {"accepted": True}}), + ("deferred_shares", {"block_hash": "ef", "share": {"miner_id": "alice"}}, + {"share": {"miner_id": "bob"}}), + ): + with self.subTest(kind=kind): + self.assertEqual(baseline["records"][kind]["count"], 0) + added = module.summarize(iter([record(kind, row)] + closing())) + changed = module.summarize(iter([record(kind, row | change)] + closing())) + self.assertEqual(added["records"][kind]["count"], 1) + self.assertEqual(changed["records"][kind]["count"], 1) + self.assertNotEqual(baseline["records"][kind]["sha256"], + added["records"][kind]["sha256"]) + self.assertNotEqual(added["records"][kind]["sha256"], + changed["records"][kind]["sha256"]) + for other in baseline["records"]: + if other != kind: + self.assertEqual(baseline["records"][other], changed["records"][other]) + + def test_fatal_state_and_recovery_history_change_summary_independently(self): + baseline = module.summarize(iter(closing())) + fatal = {"fatal_error": "fanout disconnected", "fatal_error_set_at": None} + checkpoint = {"best_chainwork": "2", "best_tip_hash": "aa" * 32, "best_tip_height": 7} + config = {"config_fingerprint": "fingerprint-one"} + revision = {"payout_revision": 1} + clock = {"ledger_clock_ms": 1758000000000} + event = { + "event_id": 1, "cleared_at": "2026-09-14T21:00:00+00:00", + "operator_identity": "operator", "database_role": "prism", + "reason": "reconciled", **fatal, + "instances": [], "reconciliation": {"blocks_checked": 1}, + } + for kind, row in (("fatal_state", fatal), ("fatal_state_events", event), + ("chain_checkpoint", checkpoint), ("cluster_config", config), + ("payout_revision", revision), ("ledger_clock", clock)): + with self.subTest(kind=kind): + self.assertEqual(baseline["records"][kind]["count"], 0) + added = module.summarize(iter([record(kind, row)] + closing())) + self.assertEqual(added["records"][kind]["count"], 1) + self.assertNotEqual(added, baseline) + for field in row: + with self.subTest(field=field): + changed = module.summarize(iter([ + record(kind, row | {field: "changed"})] + closing())) + self.assertNotEqual(added["records"][kind]["sha256"], + changed["records"][kind]["sha256"]) + for other in baseline["records"]: + if other != kind: + self.assertEqual(baseline["records"][other], added["records"][other]) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_prism_recovery_evidence_jobs.py b/tests/test_prism_recovery_evidence_jobs.py new file mode 100644 index 000000000..34c231cc1 --- /dev/null +++ b/tests/test_prism_recovery_evidence_jobs.py @@ -0,0 +1,389 @@ +"""The parallel summary (--jobs N) is the serial summary (--jobs 1), byte for byte. + +Each case writes an export and runs the summarizer serially and in parallel, +with chunk sizes small enough to put a range boundary at every line, and +compares the exit code, stdout and stderr exactly: the summary, its notes and +every refusal. +""" +import importlib.util +import json +from pathlib import Path +import re +import subprocess +import sys +import tempfile +import unittest + + +SCRIPT = Path(__file__).resolve().parents[1] / "scripts/prism-recovery-evidence.py" +spec = importlib.util.spec_from_file_location("recovery_evidence_jobs", SCRIPT) +module = importlib.util.module_from_spec(spec) +# Registered, so the forked workers find `_parse_range` by name, as they do +# for the script run as __main__. +sys.modules[spec.name] = module +spec.loader.exec_module(module) + +# (jobs, chunk bytes): a boundary after every line, after a few lines, and +# one range for the whole file. +PARALLEL = ((2, 1), (3, 37), (4, 300), (3, 1 << 20)) + + +def record(kind, row, *, ascii_only=True): + # psql writes jsonb as raw UTF-8; ascii_only=False keeps multibyte + # characters in the file, so ranges must split between them correctly. + return json.dumps({"kind": kind, "row": row}, ensure_ascii=ascii_only) + "\n" + + +def carry(seq, height, label, program, *, gross, prior, onchain=0): + candidate = prior + gross + return { + "carry_forward_seq": seq, "block_hash": f"b{height}", "block_height": height, + "recipient_id": label, "order_key": label, "p2mr_program_hex": program, + "gross_amount_sats": gross, "prior_balance_sats": str(prior), + "candidate_balance_sats": str(candidate), "onchain_amount_sats": onchain, + "settlement_fee_sats": 0, "carry_forward_balance_sats": str(candidate - onchain), + "action": "onchain" if onchain else "accrued", "maturity_state": "mature", + } + + +def closing(**overrides): + report = {"schema": "qbit.prism.carry-forward-integrity.v1", "mismatch_count": 0, + "current_drift_count": 0, "checked_active_rows": 0, "current_drift": [], "mismatches": []} + report.update(overrides) + return [record("integrity", report), record("complete", True)] + + +def full_export(): + """Every kind the export writes, in its order, with the #708 shape so a + note is printed, and non-ASCII text in several kinds.""" + lines = [record("shares", {"share_seq": seq, "share_id": f"s{seq}", "accepted": seq % 3 != 0, + "miner_id": "Zoë🔑" if seq % 5 == 0 else f"m{seq}"}, + ascii_only=seq % 2 == 0) + for seq in (1, 2, 4, 7, 8, 9, 15, 16, 23, 42, 43, 44, 100)] + lines.append(record("shares", {"share_seq": 101, "accepted": 1})) + lines.append(record("share_sequence", {"last_value": 101, "is_called": True})) + lines += [record("sequences", {"sequence": name, "last_value": 7, "is_called": True}) + for name in ("qbit_pool_payout_entries_payout_entry_seq_seq", "qbit_audit_publication_sequence_seq")] + lines += [record("share_hashes", {"header_hash": f"{index:064x}", "share_id": f"s{index}"}) + for index in range(40)] + lines.append(record("blocks", {"block_hash": "b10", "block_height": 10})) + lines.append(record("blocks", {"block_hash": "b11", "block_height": 11, "as_issued_audit_sha256": "ab" * 32})) + lines.append(record("audits", {"block_hash": "b10", "audit_bundle_sha256": "cd" * 32})) + lines.append(record("audit_bodies", {"block_hash": "b10", "audit_bundle": { + "schema": "native", "shares": [{"miner": "Ünïcødé", "weight": index} for index in range(30)]}}, + ascii_only=False)) + lines.append(record("audit_snapshots", {"snapshot_sha256": "ef" * 32, "share_count": 3})) + lines.append(record("carry", {"carry_forward_seq": 1, "label": "qb1zabc"})) + lines.append(record("payouts", {"payout_entry_seq": 1, "amount_sats": "12"})) + lines += [record("candidates", {"block_hash": f"c{index}", "state": state}) + for index, state in enumerate(("submitted", "pending", "abandoned", "orphaned", "reconciliation"))] + lines.append(record("candidate_balances", {"prior_balances_digest": "ab", "balances_sha256": "cd"})) + for kind in ("ctv_sets", "ctv_artifacts", "ctv_checkpoints", "ctv_retry_progress", "ctv_broadcast_attempts", + "cpfp_packages", "cpfp_retired_funding", "deferred_shares", "fatal_state", "fatal_state_events", + "policy_transitions", "chain_checkpoint", "cluster_config", "payout_revision", "ledger_clock"): + lines.append(record(kind, {"kind_field": kind, "note": "žluťoučký"}, ascii_only=False)) + upper, lower = "QB1ZABC", "qb1zabc" + rows = [ + carry(1, 10, lower, "aa", gross=100, prior=0), + carry(2, 11, upper, "aa", gross=50, prior=100, onchain=120), + carry(3, 12, lower, "aa", gross=7, prior=30), + carry(4, 13, upper, "aa", gross=0, prior=37), + carry(5, 13, "Zoë🔑", "bb", gross=9, prior=0), + ] + lines += [record("active_carry", row, ascii_only=index % 2 == 0) for index, row in enumerate(rows)] + label_rule = [{"carry_forward_seq": 3, "block_hash": "b12"}, {"carry_forward_seq": 4, "block_hash": "b13"}] + return lines + closing(checked_active_rows=5, mismatch_count=2, mismatches=label_rule) + + +class ParallelSummaryTests(unittest.TestCase): + def setUp(self): + self.directory = tempfile.TemporaryDirectory() + self.root = Path(self.directory.name) + self.count = 0 + + def tearDown(self): + self.directory.cleanup() + + def write(self, content): + self.count += 1 + path = self.root / f"export-{self.count}.jsonl" + path.write_bytes(content if isinstance(content, bytes) else "".join(content).encode("utf-8")) + return path + + def run_summary(self, path, *args): + result = subprocess.run([sys.executable, str(SCRIPT), *args, str(path)], capture_output=True, timeout=120) + return result.returncode, result.stdout, result.stderr + + def assert_same(self, content): + """Summarize `content` serially and in parallel; return the serial result.""" + path = self.write(content) + serial = self.run_summary(path, "--jobs", "1") + for jobs, chunk in PARALLEL: + with self.subTest(jobs=jobs, chunk=chunk): + self.assertEqual(self.run_summary(path, "--jobs", str(jobs), "--chunk-bytes", str(chunk)), serial) + return serial + + def test_a_complete_export_summarizes_identically_with_its_note(self): + code, stdout, stderr = self.assert_same(full_export()) + self.assertEqual(code, 0, stderr) + report = json.loads(stdout) + self.assertEqual(report["records"]["shares"]["count"], 14) + self.assertEqual(report["last_share_seq"], 101) + self.assertEqual(report["records"]["share_hashes"]["count"], 40) + self.assertEqual(report["records"]["active_carry"]["count"], 5) + self.assertEqual(report["pending_candidates"], 1) + self.assertEqual(report["unfinished_candidates"], 2) + self.assertIn(b"note: the database's per-label legacy carry rule reported 2 finding(s)", stderr) + + def test_a_deferred_export_summarizes_identically_and_its_refusals_are_the_serial_ones(self): + # While the share-hash backfill is pending at fence 2 the export has + # the cursor in place of every share_hashes row. + cursor = record("share_hashes_deferred", {"start_seq": 1, "next_seq": 40, "end_seq": 102, + "recent_min_height": None, "recent_start_seq": None}) + lines = [line for line in full_export() if '"kind": "share_hashes"' not in line] + # The export reads the cursor last, between the integrity report and + # the completion marker. + deferred = lines[:-1] + [cursor] + lines[-1:] + code, stdout, stderr = self.assert_same(deferred) + self.assertEqual(code, 0, stderr) + share_hashes = json.loads(stdout)["records"]["share_hashes"] + self.assertEqual(share_hashes["count"], 0) + self.assertEqual(share_hashes["deferred"]["next_seq"], 40) + self.assertIn(b"note: share_hashes deferred", stderr) + for bad in (deferred[:3] + [cursor] + deferred[3:], + deferred[:3] + [record("share_hashes", {"header_hash": "ab", "share_id": "s1"})] + deferred[3:]): + with self.subTest(bad=bad[3]): + code, _, stderr = self.assert_same(bad) + self.assertEqual(code, 1, stderr) + + def test_line_endings_and_a_missing_final_newline_are_read_as_text_mode_reads_them(self): + lines = full_export() + text = "".join(lines) + for content in (text.replace("\n", "\r\n"), text.replace("\n", "\r"), + "".join(line.replace("\n", "\r\n") if index % 3 else line + for index, line in enumerate(lines)), + text[:-1], "" + text): + with self.subTest(content=content[:40]): + self.assert_same(content) + + def test_share_order_refusals_at_and_inside_range_boundaries(self): + shares = [record("shares", {"share_seq": seq, "accepted": True}) for seq in (1, 2, 3, 9, 10)] + for bad in ( + shares + [record("shares", {"share_seq": 10, "accepted": True})], + shares[:2] + [record("shares", {"share_seq": 1, "accepted": False})] + shares[2:], + [shares[0], record("shares", {"share_seq": "7", "accepted": True})] + shares[1:], + shares[:3] + [record("shares", {"share_seq": 11, "accepted": "x"})], + shares[:3] + [record("shares", {"share_seq": 11.5, "accepted": "1"}), record("shares", {"share_seq": 11, "accepted": True})], + shares[:3] + [record("shares", {"accepted": True})], + shares[:3] + [record("shares", [11, True])], + shares[:3] + [record("shares", "eleven")], + ): + with self.subTest(bad=bad[-1]): + code, _, stderr = self.assert_same(bad + closing()) + self.assertEqual(code, 1, stderr) + # Mixed flag types that int() reads alike stay accepted. + code, stdout, _ = self.assert_same(shares[:3] + [ + record("shares", {"share_seq": 12, "accepted": "1"}), record("shares", {"share_seq": 13.0, "accepted": 0}), + record("shares", {"share_seq": 14, "accepted": False})] + closing()) + self.assertEqual(code, 0) + self.assertEqual(json.loads(stdout)["last_share_seq"], 14) + + def test_malformed_lines_refuse_at_the_same_line_with_the_same_message(self): + good = full_export() + middle = len(good) // 2 + for bad, expected in ( + (good[:middle] + ['{"kind": "shares", "row": {"share_seq": 1\n'] + good[middle:], b"Expecting"), + (good[:middle] + ["\n"] + good[middle:], b"Expecting value"), + (good[:middle] + ['{"row": {}}\n'] + good[middle:], b"'kind'"), + (good[:middle] + ['{"kind": "shares"}\n'] + good[middle:], b"'row'"), + (good[:middle] + ['"just a string"\n'] + good[middle:], b"string indices"), + (good[:middle] + ['{"kind": ["shares"], "row": {}}\n'] + good[middle:], b"unhashable"), + (good[:middle] + [record("future_format", {})] + good[middle:], b"unknown evidence kind"), + (good[:middle] + [record("audit_bodies", {"x": 1})[:-5]], b"Expecting"), + (good + ['{"kind": "shares", "row":\n'], b"records follow completion marker"), + (good + [record("shares", {"share_seq": 1000, "accepted": True})], b"records follow completion marker"), + (good[:-1], b"incomplete export"), + (good[:-2] + [record("complete", True)], b"invalid completion marker"), + (good[:-1] + [good[-2]] + good[-1:], b"duplicate integrity report"), + (good[:-1] + [record("payouts", {"payout_entry_seq": 9})] + good[-1:], b"accounting records follow"), + ([record("active_carry", carry(1, 10, "a", "aa", gross=1, prior=0)), + record("blocks", {"block_hash": "b9"})] + closing(checked_active_rows=1), b"block records follow"), + ): + with self.subTest(expected=expected): + code, stdout, stderr = self.assert_same(bad) + self.assertEqual((code, stdout), (1, b"")) + self.assertIn(expected, stderr) + + def test_carry_and_integrity_refusals_are_the_serial_ones(self): + row = carry(1, 10, "alice", "aa", gross=1, prior=0) + later = carry(2, 11, "alice", "aa", gross=1, prior=1) + marked = record("blocks", {"block_hash": "b11", "as_issued_audit_sha256": "ab" * 32}) + for bad in ( + [record("active_carry", later), record("active_carry", row)] + closing(checked_active_rows=2), + [record("active_carry", row), record("active_carry", later | {"prior_balance_sats": "99"})] + + closing(checked_active_rows=2), + [record("active_carry", row | {"carry_forward_balance_sats": "0_001"})] + closing(checked_active_rows=1), + [marked, record("active_carry", later)] + closing( + checked_active_rows=1, mismatch_count=1, mismatches=[{"carry_forward_seq": 2, "block_hash": "b11"}]), + [record("active_carry", row)] + closing( + checked_active_rows=1, mismatch_count=1, mismatches=[{"carry_forward_seq": 1, "block_hash": "b10"}]), + [record("active_carry", row)] + closing(checked_active_rows=2), + closing(current_drift_count=1), + closing(mismatch_count=1), + ): + with self.subTest(bad=bad[0][:60]): + code, _, stderr = self.assert_same(bad) + self.assertEqual(code, 1, stderr) + self.assertIn(b"recovery evidence failed:", stderr) + + def test_invalid_utf8_is_summarized_again_serially_with_the_serial_refusal(self): + good = "".join(full_export()).encode("utf-8") + middle = len(good) // 2 + for bad in (good[:middle] + b"\xff\xfe" + good[middle:], b"\xc3" + good, good + b"\x80\n"): + with self.subTest(position=bad.find(b"\xff")): + code, _, stderr = self.assert_same(bad) + self.assertEqual(code, 1) + self.assertIn(b"codec can't decode", stderr) + + def test_paths_that_are_not_regular_files_fail_as_before(self): + for path in (self.root / "missing.jsonl", self.root): + serial = self.run_summary(path, "--jobs", "1") + self.assertEqual(self.run_summary(path, "--jobs", "3", "--chunk-bytes", "7"), serial) + self.assertEqual(serial[0], 1) + + def test_the_parallel_path_parses_in_workers_and_reads_no_line_serially(self): + path = self.write(full_export()) + with path.open(encoding="utf-8") as source: + expected = module.summarize(source) + line = module._Summary.line + + def refuse(self, text): + raise AssertionError("the parallel path read a line serially") + + module._Summary.line = refuse + try: + notes = [] + report = module.summarize_file(path, notes, jobs=3, chunk_bytes=50) + finally: + module._Summary.line = line + self.assertEqual(report, expected) + self.assertEqual(len(notes), 1) + + def test_from_an_unreadable_range_on_the_file_is_read_serially_once(self): + lines = full_export() + content = "".join(lines).encode("utf-8") + cut = content.index(b'"kind": "blocks"') + path = self.write(content[:cut] + b"\xff" + content[cut:]) + read = [] + line = module._Summary.line + + def counted(self, text): + read.append(text) + return line(self, text) + + module._Summary.line = counted + try: + with self.assertRaises(UnicodeDecodeError): + module.summarize_file(path, [], jobs=3, chunk_bytes=64) + finally: + module._Summary.line = line + # The lines the workers' ranges checked are read again but not + # checked again: only lines from the unreadable range on are. + self.assertLess(len(read), len(lines) // 2) + + def test_a_copy_the_workers_cannot_find_by_name_summarizes_serially(self): + copy_spec = importlib.util.spec_from_file_location("recovery_evidence_unregistered", SCRIPT) + copy = importlib.util.module_from_spec(copy_spec) + copy_spec.loader.exec_module(copy) + path = self.write(full_export()) + with path.open(encoding="utf-8") as source: + expected = copy.summarize(source) + self.assertFalse(copy._workers_available()) + self.assertEqual(copy.summarize_file(path, [], jobs=3, chunk_bytes=50), expected) + + def test_one_job_is_the_serial_summarizer(self): + path = self.write(full_export()) + serial = module.summarize + calls = [] + + def counted(lines, notes=None, summary=None): + calls.append(summary) + return serial(lines, notes, summary) + + module.summarize = counted + try: + module.summarize_file(path, [], jobs=1) + finally: + module.summarize = serial + self.assertEqual(calls, [None]) + + def test_a_host_that_cannot_start_workers_summarizes_serially(self): + path = self.write(full_export()) + with path.open(encoding="utf-8") as source: + expected = module.summarize(source) + + class NoFork(module.ProcessPoolExecutor): + def submit(self, *args, **kwargs): + raise BlockingIOError(11, "Resource temporarily unavailable") + + class NoPool: + def __init__(self, *args, **kwargs): + raise OSError(38, "Function not implemented") + + pool = module.ProcessPoolExecutor + for broken in (NoFork, NoPool): + with self.subTest(broken=broken.__name__): + module.ProcessPoolExecutor = broken + try: + self.assertEqual(module.summarize_file(path, [], jobs=3, chunk_bytes=50), expected) + finally: + module.ProcessPoolExecutor = pool + + def test_ranges_end_after_newlines_and_the_last_reads_to_the_end(self): + for content in (b"", b"a\n", b"a\nbb\nccc\n", b"a\nbb\nccc", b"\n\n\n", b"x" * 10): + path = self.write(content) + for chunk in (1, 2, 3, 5, 100): + with self.subTest(content=content, chunk=chunk), path.open("rb") as source: + ranges = list(module._ranges(source, chunk)) + self.assertEqual(ranges[-1][1], None) + self.assertEqual(ranges[0][0], 0) + for (start, end), (following, _) in zip(ranges, ranges[1:]): + self.assertEqual(end, following) + self.assertGreater(end, start) + self.assertEqual(content[end - 1:end], b"\n") + # A line still being written when the ranges are cut is never split: + # the open last range is read wherever the file ends by then. + path = self.write(b"a\nbb\n" + b"c" * 50) + with path.open("rb") as source: + ranges = list(module._ranges(source, 1)) + with path.open("ab") as source: + source.write(b"c\n") + lines = [] + for start, end in ranges: + _, count, _items = module._parse_range(str(path), start, end) + lines.append(count) + self.assertEqual(sum(lines), 3) + + def test_an_uncaught_error_prints_the_serial_traceback(self): + # Python 3.14's stack-overflow message reports the C stack used, which + # depends on the process's argv and environment size: two serial runs + # with different arguments differ there. Every frame must not. + def masked(result): + code, stdout, stderr = result + return code, stdout, re.sub(rb"used \d+ kB", b"used N kB", stderr) + + deep = '{"kind": "audit_bodies", "row": ' + "[" * 200000 + "]" * 200000 + "}\n" + lines = full_export() + path = self.write(lines[:20] + [deep] + lines[20:]) + serial = masked(self.run_summary(path, "--jobs", "1")) + self.assertNotEqual(serial[0], 0) + self.assertIn(b"RecursionError", serial[2]) + self.assertIn(b"in summarize", serial[2]) + for jobs, chunk in PARALLEL: + with self.subTest(jobs=jobs, chunk=chunk): + self.assertEqual( + masked(self.run_summary(path, "--jobs", str(jobs), "--chunk-bytes", str(chunk))), serial) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_prism_recovery_evidence_parallel.py b/tests/test_prism_recovery_evidence_parallel.py new file mode 100644 index 000000000..a1dcc3731 --- /dev/null +++ b/tests/test_prism_recovery_evidence_parallel.py @@ -0,0 +1,247 @@ +"""Split the recovery-evidence export in order, and fail it closed (#712). + +The database-gated migration_rollback test compares the parallel export +with the serial one byte for byte; these check the cut and the order. +""" +import importlib.util +import json +import os +from pathlib import Path +import re +import subprocess +import sys +import tempfile +import unittest + + +DRIVER = Path(__file__).resolve().parents[1] / "scripts/prism-recovery-evidence-parallel.py" +spec = importlib.util.spec_from_file_location("recovery_evidence_parallel", DRIVER) +module = importlib.util.module_from_spec(spec) +spec.loader.exec_module(module) +SQL = module.SCRIPT.read_text(encoding="utf-8") + +# Stands in for psql: answers the coordinator, prints one JSON line per +# export statement it runs (taking every \if branch), and fails with exit 3 +# on a line containing FAKE_PSQL_FAIL. It refuses to prompt for a password, +# and to print rows anywhere but a pipe the driver reads, as the driver +# must ask of psql. +FAKE_PSQL = r'''#!{python} +import json, os, random, re, stat, sys, time +if "-XqAtw" not in sys.argv or not stat.S_ISFIFO(os.fstat(1).st_mode): + sys.exit(2) +fail = os.environ.get("FAKE_PSQL_FAIL") +source = open(sys.argv[sys.argv.index("--file") + 1]) if "--file" in sys.argv else sys.stdin +skipped = [] +statement = None +for line in source: + if fail and fail in line: + sys.exit(3) + stripped = line.strip() + if stripped.startswith("\\if"): + skipped.append(False) + elif stripped.startswith("\\else"): + skipped[-1] = True + elif stripped.startswith("\\endif"): + skipped.pop() + elif any(skipped): + pass + elif "pg_export_snapshot()" in line: + print("00000003-0000000F-1|t|" + ("t" if os.environ.get("FAKE_PSQL_DEFERRED") else "f"), flush=True) + elif "percentile_disc(" in line: + print("{{10,20,30}}" if "ORDER BY share_seq" in line else "{{4,8,c}}", flush=True) + elif stripped == "SELECT NULL;": + print("", flush=True) + elif statement is not None or stripped.startswith("SELECT jsonb_build_object('kind', '"): + statement = (statement or "") + line + if stripped.endswith(";"): + kind = re.match(r"SELECT jsonb_build_object\('kind', '(\w+)'", statement).group(1) + where = re.findall(r" WHERE (.*?) ORDER BY", statement) + print(json.dumps({{"kind": kind, "where": where}}), flush=True) + statement = None +time.sleep(random.random() / 20) +''' + + +def collapse(kinds): + """The kinds in order, a run of parts of one kind counted once.""" + return [kind for index, kind in enumerate(kinds) if index == 0 or kinds[index - 1] != kind] + + +class ScriptTests(unittest.TestCase): + def test_the_cut_loses_nothing_and_keeps_one_transaction(self): + script = module.Script(SQL) + self.assertEqual(script.preamble + module.SHARES + script.middle + script.hashes + script.tail, SQL) + self.assertEqual(script.preamble.count(module.BEGIN), 1) + # Every flag a later part branches on is set before the first export. + for flag in ("has_native_share_hashes", "share_hashes_deferred", "has_audit_snapshots", + "has_policy_transitions"): + self.assertIn(f"AS {flag}", script.preamble) + # A pending backfill's cursor is exported once, last, by the part that + # commits the closing kinds: no other session reads the cursor table, + # and none of the share-hash parts exports a row meanwhile. + deferred = script.tail.index("\\if :share_hashes_deferred\n") + self.assertGreater(deferred, script.tail.index("'kind', 'integrity'")) + self.assertLess(deferred, script.tail.index("'kind', 'complete'")) + self.assertIn("SELECT jsonb_build_object('kind', 'share_hashes_deferred'", script.tail[deferred:]) + cursor = re.compile(r"\bFROM qbit_prism_share_hash_backfill\b") + self.assertIsNone(cursor.search(script.preamble + script.middle + script.hashes)) + self.assertIn("WHERE NOT :'share_hashes_deferred'::boolean\n", script.hashes) + self.assertTrue(script.tail.endswith(module.COMMIT)) + self.assertNotIn(module.COMMIT, script.preamble + script.middle + script.hashes) + + def test_a_session_imports_the_snapshot_before_any_query(self): + session = module.Script(SQL).session("00000003-0000000F-1").splitlines() + begin = session.index(module.BEGIN.strip()) + self.assertEqual(session[begin + 1], "SET TRANSACTION SNAPSHOT '00000003-0000000F-1';") + + def test_parts_follow_the_serial_order_and_only_ranges_sort_with_more_memory(self): + script = module.Script(SQL) + parts = script.parts("00000003-0000000F-1", [10, 20], ["4", "c"], "64MB") + self.assertEqual([name.split(" (")[0] for name, _ in parts], [ + "shares part 1 of 3", "shares part 2 of 3", "shares part 3 of 3", + "the kinds between shares and share_hashes", + "share_hashes part 1 of 3", "share_hashes part 2 of 3", "share_hashes part 3 of 3", + "the kinds after share_hashes", + ]) + session = script.session("00000003-0000000F-1") + for name, sql in parts: + with self.subTest(part=name): + self.assertTrue(sql.startswith(session)) + self.assertTrue(sql.endswith(module.COMMIT)) + self.assertEqual(sql.count(module.COMMIT), 1) + self.assertEqual("SET LOCAL work_mem = '64MB';" in sql, " part " in name) + self.assertIn("WHERE share_seq >= 10 AND share_seq < 20 ORDER BY share_seq;", parts[1][1]) + hashes = parts[5][1] + # The native and the 2.x mapping are both restricted to the range, + # and 2.x shares outside it are never matched against the pattern. + self.assertEqual(hashes.count( + """WHERE header_hash COLLATE "C" >= '4' AND header_hash COLLATE "C" < 'c' ORDER BY"""), 2) + self.assertIn( + """WHERE accepted AND CASE WHEN lower(right(share_id,64)) COLLATE "C" >= '4' AND """ + """lower(right(share_id,64)) COLLATE "C" < 'c' THEN share_id ~ '[0-9a-fA-F]{64}$' END\n""", + hashes) + + def test_ranges_cover_every_key_in_order(self): + self.assertEqual(module.ranges([]), [(None, None)]) + self.assertEqual(module.ranges(["5", "9"]), [(None, "5"), ("5", "9"), ("9", None)]) + self.assertEqual(module.predicate("k", None, None), "true") + self.assertEqual(module.predicate("k", None, "5"), "k < 5") + self.assertEqual(module.predicate("k", "5", "9"), "k >= 5 AND k < 9") + # NULL sorts last, so the last range holds it. + self.assertEqual(module.predicate("k", "9", None), "(k >= 9 OR k IS NULL)") + + def test_sampled_bounds_are_only_well_formed_keys(self): + self.assertEqual(module.bounds("", module.SHARE_BOUND), []) + self.assertEqual(module.bounds("{}", module.SHARE_BOUND), []) + self.assertEqual(module.bounds("{1,5,8}", module.SHARE_BOUND), ["1", "5", "8"]) + self.assertEqual(module.bounds('{NULL,ab,"a b",AB}', module.HASH_BOUND), ["ab"]) + with self.assertRaises(module.ExportError): + module.bounds("ERROR", module.SHARE_BOUND) + + def test_an_edited_script_is_refused_rather_than_cut_elsewhere(self): + hashes_order = module.HASHES_ORDER + for edited in ( + SQL.replace(module.SHARES, ""), + SQL.replace(module.SHARES, module.SHARES * 2), + SQL.replace(module.SHARES, module.SHARES.replace("ORDER BY share_seq", "ORDER BY share_id")), + SQL.replace(hashes_order, hashes_order.replace("COLLATE \"C\"", ""), 1), + SQL.replace(module.HASHES, module.HASHES + "\\if :has_cluster\n\\endif\n"), + SQL.replace(module.LEGACY_MATCH, "share_id ~ '[0-9a-f]{64}$'"), + SQL.replace(f"{module.LEGACY_HEADER} AS header_hash", "lower(right(share_id, 64)) AS header_hash"), + SQL.replace(module.BEGIN, ""), + SQL[:-len(module.COMMIT)], + ): + with self.assertRaises(module.ExportError): + module.Script(edited) + + +class ExportTests(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls.directory = tempfile.TemporaryDirectory() + cls.fake = Path(cls.directory.name) / "psql" + cls.fake.write_text(FAKE_PSQL.format(python=sys.executable), encoding="utf-8") + cls.fake.chmod(0o755) + serial = subprocess.run([str(cls.fake), "-XqAtw"], input=SQL, capture_output=True, text=True, + check=True) + cls.serial = [json.loads(line)["kind"] for line in serial.stdout.splitlines()] + + @classmethod + def tearDownClass(cls): + cls.directory.cleanup() + + def export(self, *args, fail="", stdout=subprocess.PIPE, deferred=False): + with tempfile.TemporaryDirectory() as work: + env = dict(os.environ, FAKE_PSQL_FAIL=fail) + env.pop("FAKE_PSQL_DEFERRED", None) + if deferred: + env["FAKE_PSQL_DEFERRED"] = "1" + result = subprocess.run( + [sys.executable, str(DRIVER), "--psql", str(self.fake), "--work-dir", work, *args], + stdout=stdout, stderr=subprocess.PIPE, text=True, timeout=120, env=env) + return result, os.listdir(work) + + def test_parts_are_printed_in_the_serial_order_whenever_they_finish(self): + self.assertEqual(self.serial[0], "shares") + self.assertEqual(self.serial[-3:], ["integrity", "share_hashes_deferred", "complete"]) + share_ranges = [["share_seq < 10"], ["share_seq >= 10 AND share_seq < 20"], + ["share_seq >= 20 AND share_seq < 30"], ["(share_seq >= 30 OR share_seq IS NULL)"]] + key = 'header_hash COLLATE "C"' + for jobs, hash_ranges in ( + ("1", [["true"]]), + ("3", [[f"{key} < '4'"], [f"{key} >= '4' AND {key} < '8'"], + [f"{key} >= '8' AND {key} < 'c'"], [f"({key} >= 'c' OR {key} IS NULL)"]]), + ): + with self.subTest(jobs=jobs): + for _ in range(3): + result, leftovers = self.export("--jobs", jobs) + self.assertEqual(result.returncode, 0, result.stderr) + self.assertEqual(leftovers, []) + rows = [json.loads(line) for line in result.stdout.splitlines()] + self.assertEqual(collapse([row["kind"] for row in rows]), self.serial) + self.assertEqual([row["where"] for row in rows if row["kind"] == "shares"], share_ranges) + self.assertEqual([row["where"] for row in rows if row["kind"] == "share_hashes"], + hash_ranges) + + def test_a_pending_share_hash_backfill_exports_its_cursor_once_and_the_hashes_in_one_part(self): + self.assertEqual(self.serial.count("share_hashes_deferred"), 1) + self.assertGreater(self.serial.index("share_hashes_deferred"), self.serial.index("integrity")) + for jobs in ("1", "3"): + with self.subTest(jobs=jobs): + result, leftovers = self.export("--jobs", jobs, deferred=True) + self.assertEqual(result.returncode, 0, result.stderr) + self.assertEqual(leftovers, []) + self.assertIn("share_hashes in 1 (deferred: the share-hash backfill is pending)", result.stderr) + rows = [json.loads(line) for line in result.stdout.splitlines()] + self.assertEqual(collapse([row["kind"] for row in rows]), self.serial) + self.assertEqual([row["where"] for row in rows if row["kind"] == "share_hashes"], [["true"]]) + + def test_a_failed_session_fails_the_export_without_a_completion_marker(self): + for fail, reason in ( + ("pg_export_snapshot()", "the export transaction failed (psql exited 3)"), + ("share_seq >= 20 AND", "shares part 3 of 4"), + ("qbit_carry_forward_integrity_report()", "the kinds after share_hashes failed"), + ): + with self.subTest(fail=fail): + result, leftovers = self.export("--jobs", "3", fail=fail) + self.assertEqual(result.returncode, 1, result.stderr) + self.assertIn(reason, result.stderr) + self.assertEqual(leftovers, []) + self.assertNotIn('"complete"', result.stdout) + + @unittest.skipUnless(os.path.exists("/dev/full"), "needs /dev/full") + def test_an_output_that_cannot_be_written_fails_the_export(self): + with open("/dev/full", "wb") as full: + result, leftovers = self.export("--jobs", "2", stdout=full) + self.assertEqual(result.returncode, 1, result.stderr) + self.assertIn("No space left on device", result.stderr) + self.assertEqual(leftovers, []) + + def test_an_unusable_work_mem_is_refused_before_connecting(self): + result, _ = self.export("--work-mem", "1GB'; DROP TABLE x; --") + self.assertEqual(result.returncode, 2) + self.assertIn("is not a work_mem", result.stderr) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_prism_refresh_retry_pacing.py b/tests/test_prism_refresh_retry_pacing.py deleted file mode 100644 index 383a881a7..000000000 --- a/tests/test_prism_refresh_retry_pacing.py +++ /dev/null @@ -1,281 +0,0 @@ -#!/usr/bin/env python3 -"""Failed-refresh spacing and same-tip template reuse (retry-storm pacing). - -A blocked refresh used to re-attempt within one 0.25s trigger slice, and every -attempt re-issued a full getblocktemplate before rediscovering the blockage. -These tests pin the two throttles: consecutive failed passes are spaced by -PRISM_TIP_REFRESH_FAILURE_HOLDOFF_SECONDS (new tips and successes stay -immediate), and same-tip passes inside the PRISM_TEMPLATE_CACHE_SECONDS window -reuse the cached template instead of refetching it. -""" - -from __future__ import annotations - -import time -import unittest -from unittest.mock import patch - -from lab.prism.prism_coordinator import PrismCoordinator, TemplateRefreshBlocked -from tests.prism_coordinator_test_support import ( - base_template, - client, - coordinator, - install_fake_bundle_builder, -) - -NEW_TIP = "22" * 32 - - -def _block_refreshes(server: PrismCoordinator) -> None: - """Force every subsequent refresh pass to fail after snapshot acquisition.""" - server.ensure_reorg_reconciled_for_tip = ( # type: ignore[method-assign] - lambda _tip: False - ) - - -class FailedTipRefreshSpacingTests(unittest.TestCase): - def _fail_one_poll(self, server: PrismCoordinator) -> None: - with self.assertRaisesRegex(TemplateRefreshBlocked, "untrusted"): - server.poll_qbit_tip_template_once() - - def test_failed_pass_arms_holdoff_and_gates_the_trigger(self) -> None: - server, _rpc = coordinator() - install_fake_bundle_builder(server) - server.clients = set() - server.blockpoll_seconds = 30.0 - server.tip_refresh_failure_holdoff_seconds = 0.3 - - self.assertEqual(server.poll_qbit_tip_template_once(), 0) - self.assertEqual(server._tip_refresh_failure_holdoff_remaining(), 0.0) - - _block_refreshes(server) - with patch("lab.prism.prism_coordinator.random.uniform", return_value=0.0): - self._fail_one_poll(server) - deadline = server._tip_refresh_failure_holdoff_until - self.assertIsNotNone(deadline) - assert deadline is not None - self.assertGreater(server._tip_refresh_failure_holdoff_remaining(), 0.0) - - server._schedule_tip_refresh_retry() - self.assertTrue(server._wait_for_blockpoll_trigger()) - # The trigger only released after the spacing window, never before. - self.assertGreaterEqual(time.monotonic(), deadline) - self.assertFalse(server._tip_refresh_retry.is_set()) - - def test_holdoff_wait_keeps_blockpoll_heartbeat_alive(self) -> None: - server, _rpc = coordinator() - install_fake_bundle_builder(server) - server.clients = set() - server.blockpoll_seconds = 30.0 - server.tip_refresh_failure_holdoff_seconds = 0.4 - - self.assertEqual(server.poll_qbit_tip_template_once(), 0) - _block_refreshes(server) - with patch("lab.prism.prism_coordinator.random.uniform", return_value=0.0): - self._fail_one_poll(server) - - server._schedule_tip_refresh_retry() - started = time.monotonic() - self.assertTrue(server._wait_for_blockpoll_trigger()) - self.assertGreaterEqual(time.monotonic() - started, 0.3) - # The deliberately held poller must keep beating so a holdoff longer - # than the watchdog timeout cannot read as a hung loop. - beat = server._heartbeats.get("qbit_blockpoll") - self.assertIsNotNone(beat) - assert beat is not None - self.assertGreaterEqual(beat, started) - - def test_trigger_without_prior_failure_stays_immediate(self) -> None: - server, _rpc = coordinator() - server.blockpoll_seconds = 30.0 - - server._schedule_tip_refresh_retry() - started = time.monotonic() - self.assertTrue(server._wait_for_blockpoll_trigger()) - self.assertLess(time.monotonic() - started, 0.2) - - def test_new_tip_observation_releases_the_holdoff(self) -> None: - server, _rpc = coordinator() - install_fake_bundle_builder(server) - server.clients = set() - server.blockpoll_seconds = 30.0 - server.tip_refresh_failure_holdoff_seconds = 30.0 - - self.assertEqual(server.poll_qbit_tip_template_once(), 0) - _block_refreshes(server) - self._fail_one_poll(server) - self.assertGreater(server._tip_refresh_failure_holdoff_remaining(), 0.0) - - # Blockwait's push path: record the detection, then arm the poller - # trigger. The armed holdoff must not delay that wake even though the - # newer tip is not yet published as share-validation authority. - self.assertTrue(server.observe_tip_for_refresh(NEW_TIP)) - self.assertEqual(server._tip_refresh_failure_holdoff_remaining(), 0.0) - server._schedule_tip_refresh_retry() - started = time.monotonic() - self.assertTrue(server._wait_for_blockpoll_trigger()) - self.assertLess(time.monotonic() - started, 0.2) - - def test_mid_fetch_tip_discovery_releases_the_holdoff(self) -> None: - server, rpc = coordinator() - install_fake_bundle_builder(server) - server.clients = set() - server.blockpoll_seconds = 30.0 - server.tip_refresh_failure_holdoff_seconds = 30.0 - server.template_cache_seconds = 0.0 - - self.assertEqual(server.poll_qbit_tip_template_once(), 0) - - original_call = rpc.call - - def advance_tip_after_template( - method: str, - params: list[object] | None = None, - ) -> object: - result = original_call(method, params) - if method == "getblocktemplate": - # The served template still extends the old tip; the fetch's - # consistency probe is the only place the new tip surfaces - # (the blockwait-disabled configuration). - rpc.tip = NEW_TIP - return result - - rpc.call = advance_tip_after_template # type: ignore[method-assign] - with self.assertRaisesRegex( - TemplateRefreshBlocked, "tip changed while fetching" - ): - server.poll_qbit_tip_template_once() - - # The failed pass recorded its discovery, so the newer tip's retry - # is not held for the spacing window. - self.assertEqual(server._tip_refresh_failure_holdoff_remaining(), 0.0) - - def test_successful_pass_clears_the_holdoff(self) -> None: - server, _rpc = coordinator() - install_fake_bundle_builder(server) - server.clients = set() - server.tip_refresh_failure_holdoff_seconds = 30.0 - original_reconcile = server.ensure_reorg_reconciled_for_tip - - self.assertEqual(server.poll_qbit_tip_template_once(), 0) - _block_refreshes(server) - self._fail_one_poll(server) - self.assertGreater(server._tip_refresh_failure_holdoff_remaining(), 0.0) - - server.ensure_reorg_reconciled_for_tip = original_reconcile # type: ignore[method-assign] - self.assertEqual(server.poll_qbit_tip_template_once(), 0) - self.assertIsNone(server._tip_refresh_failure_holdoff_until) - self.assertEqual(server._tip_refresh_failure_holdoff_remaining(), 0.0) - - def test_zero_holdoff_restores_unspaced_retries(self) -> None: - server, _rpc = coordinator() - install_fake_bundle_builder(server) - server.clients = set() - server.tip_refresh_failure_holdoff_seconds = 0.0 - - self.assertEqual(server.poll_qbit_tip_template_once(), 0) - _block_refreshes(server) - self._fail_one_poll(server) - - self.assertIsNone(server._tip_refresh_failure_holdoff_until) - self.assertEqual(server._tip_refresh_failure_holdoff_remaining(), 0.0) - - def test_holdoff_includes_bounded_jitter(self) -> None: - server, _rpc = coordinator() - server.tip_refresh_failure_holdoff_seconds = 1.0 - - before = time.monotonic() - server._note_tip_refresh_attempt_failed() - deadline = server._tip_refresh_failure_holdoff_until - - self.assertIsNotNone(deadline) - assert deadline is not None - delay = deadline - before - self.assertGreaterEqual(delay, 1.0) - self.assertLessEqual(delay, 1.3) - - -class SameTipTemplateReuseTests(unittest.TestCase): - def test_same_tip_poll_within_window_skips_getblocktemplate(self) -> None: - server, rpc = coordinator() - install_fake_bundle_builder(server) - state = client(1) - sent: list[dict[str, object]] = [] - state.send = sent.append # type: ignore[method-assign] - server.clients = {state} - - try: - self.assertEqual(server.poll_qbit_tip_template_once(), 1) - self.assertEqual(rpc.count("getblocktemplate"), 1) - self.assertEqual(server.poll_qbit_tip_template_once(), 0) - finally: - server.shutdown_tip_refresh_executor() - - self.assertEqual(rpc.count("getblocktemplate"), 1) - self.assertEqual(server.job_cache_hit_counts["template"], 1) - # The reused pass rebuilt nothing: the client's job is still the one - # delivered from the originally fetched template. - self.assertEqual( - [payload["method"] for payload in sent], - ["mining.set_difficulty", "mining.notify"], - ) - - def test_blocked_passes_cost_one_template_per_tip(self) -> None: - server, rpc = coordinator() - install_fake_bundle_builder(server) - server.clients = set() - - self.assertEqual(server.poll_qbit_tip_template_once(), 0) - self.assertEqual(rpc.count("getblocktemplate"), 1) - - _block_refreshes(server) - for _attempt in range(5): - with self.assertRaisesRegex(TemplateRefreshBlocked, "untrusted"): - server.poll_qbit_tip_template_once() - - self.assertEqual(rpc.count("getblocktemplate"), 1) - self.assertEqual(server.job_cache_hit_counts["template"], 5) - - def test_tip_change_bypasses_template_reuse(self) -> None: - server, rpc = coordinator() - install_fake_bundle_builder(server) - server.clients = set() - - self.assertEqual(server.poll_qbit_tip_template_once(), 0) - self.assertEqual(rpc.count("getblocktemplate"), 1) - - rpc.tip = NEW_TIP - rpc.template = base_template(height=11, prevhash=NEW_TIP) - self.assertEqual(server.poll_qbit_tip_template_once(), 0) - - self.assertEqual(rpc.count("getblocktemplate"), 2) - assert server.tip_template_snapshot is not None - self.assertEqual(server.tip_template_snapshot.bestblockhash, NEW_TIP) - - def test_zero_window_disables_template_reuse(self) -> None: - server, rpc = coordinator() - install_fake_bundle_builder(server) - server.clients = set() - server.template_cache_seconds = 0.0 - - self.assertEqual(server.poll_qbit_tip_template_once(), 0) - self.assertEqual(server.poll_qbit_tip_template_once(), 0) - - self.assertEqual(rpc.count("getblocktemplate"), 2) - self.assertEqual(server.job_cache_hit_counts["template"], 0) - - def test_expired_window_refetches_the_template(self) -> None: - server, rpc = coordinator() - install_fake_bundle_builder(server) - server.clients = set() - server.template_cache_seconds = 0.05 - - self.assertEqual(server.poll_qbit_tip_template_once(), 0) - time.sleep(0.06) - self.assertEqual(server.poll_qbit_tip_template_once(), 0) - - self.assertEqual(rpc.count("getblocktemplate"), 2) - - -if __name__ == "__main__": - unittest.main() diff --git a/tests/test_prism_reorg_reconciler.py b/tests/test_prism_reorg_reconciler.py deleted file mode 100644 index 78a14c34d..000000000 --- a/tests/test_prism_reorg_reconciler.py +++ /dev/null @@ -1,633 +0,0 @@ -from __future__ import annotations - -import inspect -import threading -import time -import unittest -from unittest import mock -from contextlib import contextmanager, nullcontext -from dataclasses import replace as dataclass_replace -from types import SimpleNamespace - -from lab.prism.prism_coordinator import PrismCoordinator -from lab.prism.reorg_reconciler import ( - STRANDED_PREPARED_REJECT_MIN_DEPTH, - STRANDED_PREPARED_SWEEP_LIMIT, - ReorgPorts, - ReorgReconcilerService, - qbit_chain_view_untrusted, -) -from tests import prism_vardiff_test_support as support - - -class FakeLedger: - def __init__(self) -> None: - self.inactivated: list[str] = [] - self.reactivated: list[str] = [] - - def reorg_watch_blocks(self, *, active_tip_height: int) -> list[dict[str, object]]: - assert active_tip_height == 10 - return [ - {"block_height": 8, "block_hash": "aa", "chain_state": "confirmed"}, - {"block_height": 9, "block_hash": "bb", "chain_state": "inactive"}, - ] - - def mark_pool_block_inactive(self, **kwargs: object) -> dict[str, int]: - self.inactivated.append(str(kwargs["block_hash"])) - return {"inactive_count": 1} - - def reactivate_pool_block(self, **kwargs: object) -> dict[str, int]: - self.reactivated.append(str(kwargs["block_hash"])) - return {"reactivated_count": 1} - - def mark_mature_pool_payouts(self, **_kwargs: object) -> dict[str, int]: - return {"matured_count": 2} - - -def make_ports( - *, - ledger: object | None = None, - publish: object = 7, - chain_untrusted: bool = False, - ensure_tip: object = None, - tip_height: int = 10, - block_hashes: dict[int, str] | None = None, -) -> tuple[ReorgPorts, list[tuple[str, object]]]: - events: list[tuple[str, object]] = [] - active_ledger = FakeLedger() if ledger is None else ledger - state_lock = threading.RLock() - active_chain = {8: "not-aa", 9: "bb"} if block_hashes is None else block_hashes - - def rpc_call(method: str, params: object = None) -> object: - if method == "getbestblockhash": - return "tip" - if method == "getblockcount": - return tip_height - if method == "getblockhash": - height = int(params[0]) # type: ignore[index] - events.append(("getblockhash", height)) - return active_chain[height] - raise AssertionError(method) - - @contextmanager - def publication_guard(): - events.append(("guard", "enter")) - try: - yield - finally: - events.append(("guard", "exit")) - - def prepared_candidate(captured, **kwargs): - events.append(("prepare", kwargs)) - return SimpleNamespace(captured=captured) - - return ( - ReorgPorts( - rpc_call=rpc_call, - ledger=lambda: active_ledger, - ensure_job_cache_state=lambda: events.append(("ensure", None)), - state_lock=lambda: state_lock, - source_tip=lambda: "tip", - reserve_external_tip=lambda tip: events.append(("reserve", tip)), - max_supersession_retries=lambda: 1, - prepare_lock=nullcontext, - capture_source=lambda: (0, 0, "tip", "test", 0.0), - prepared_candidate=prepared_candidate, - captured_publication_required=lambda _captured: False, - block_publication=lambda **kwargs: events.append(("block", kwargs)), - publication_guard=publication_guard, - publish_candidate=lambda _candidate: publish, # type: ignore[return-value] - observe_preparation=lambda elapsed: events.append(("observe", elapsed)), - chain_view_untrusted=lambda: chain_untrusted, - reorg_proof_snapshot=lambda: ("tip", 0), - flight_wait_seconds=lambda: 30.0, - prefetch_join_timeout_seconds=lambda: 20.0, - reconcile_with_admission=lambda **kwargs: { - "tip": kwargs.get("tip_hash"), - "untrusted": chain_untrusted, - }, - reconcile_serialized=lambda **kwargs: { - "serialized": kwargs, - "untrusted": False, - "superseded": False, - }, - ensure_tip=( - ensure_tip - if ensure_tip is not None - else lambda tip: events.append(("ensure_tip", tip)) or True - ), - ), - events, - ) - - -class ReorgReconcilerServiceTests(unittest.TestCase): - def test_fresh_memo_still_checks_live_chain_trust(self) -> None: - ports, events = make_ports() - service = ReorgReconcilerService(ports, cache_seconds=5.0) - service._reorg_reconcile_trusted_memo["tip"] = time.monotonic() - self.assertTrue(service.ensure_current()) - # The memo satisfied the caller; no serialized pass ran. - self.assertNotIn(("ensure_tip", "tip"), events) - self.assertEqual( - service.reorg_reconcile_lookup_snapshot()[("job_build", "memo_hit")], - 1, - ) - - untrusted_ports, untrusted_events = make_ports(chain_untrusted=True) - untrusted = ReorgReconcilerService(untrusted_ports, cache_seconds=5.0) - untrusted._reorg_reconcile_trusted_memo["tip"] = time.monotonic() - # An arriving reorg (headers ahead of the validated tip) must force - # the full pass immediately, memo freshness notwithstanding. - untrusted.ensure_current() - self.assertIn(("ensure_tip", "tip"), untrusted_events) - self.assertEqual( - untrusted.reorg_reconcile_lookup_snapshot()[("job_build", "serial")], - 1, - ) - - def test_reconcile_owns_chain_mutations_counts_and_publication(self) -> None: - ledger = FakeLedger() - ports, events = make_ports(ledger=ledger) - service = ReorgReconcilerService(ports) - - summary = service.reconcile(tip_hash="tip", force_publish=True) - - self.assertEqual(ledger.inactivated, ["aa"]) - self.assertEqual(ledger.reactivated, ["bb"]) - self.assertEqual(summary["published_generation"], 7) - self.assertEqual(summary["inactive_blocks"], 1) - self.assertEqual(summary["reactivated_blocks"], 1) - self.assertEqual(summary["matured_payouts"], 2) - self.assertIn(("block", {"force": True}), events) - # PR 75 publication-order guard wraps reactivation. - guard_enter = events.index(("guard", "enter")) - guard_exit = events.index(("guard", "exit")) - self.assertLess(guard_enter, guard_exit) - self.assertEqual(ledger.reactivated, ["bb"]) - # A confirmed mutation moves payout balances but never the share - # ledger, so preparation asks for a live prior-balances reread over - # the still-exact window instead of the #112 full rescan (#224). - self.assertIn( - ( - "prepare", - { - "force_full_window_rescan": False, - "force_prior_balances_read": True, - }, - ), - events, - ) - state = service.snapshot() - self.assertEqual(state.inactive_block_count, 1) - self.assertEqual(state.reactivated_block_count, 1) - self.assertEqual(state.matured_payout_count, 2) - self.assertTrue(state.last_trusted) - # A trusted pass for the latest detected tip arms the per-tip memo. - self.assertIn("tip", service._reorg_reconcile_trusted_memo) - - def test_reconcile_stamps_progress_through_its_unbounded_chain_walk(self) -> None: - """One pass is many statements, so one bracket around it is not enough. - - An accepted-block landing runs ``ensure_tip`` inline on the - watchdog-monitored block-work thread. The pass walks every watched - pool block, spending a getblockhash and up to two ledger statements - per row, so its heartbeat-silent span grows with the reorg window. - Bracketing the call from the outside would leave that growth - unreported; the stamps have to follow the work (issue #125). - """ - ledger = FakeLedger() - ports, _events = make_ports(ledger=ledger) - phases: list[str] = [] - service = ReorgReconcilerService( - dataclass_replace(ports, record_progress=phases.append) - ) - - service.reconcile(tip_hash="tip", force_publish=True) - - # FakeLedger watches two blocks; each row gets its own stamp. - self.assertEqual(phases.count("reorg-reconcile:watch-block"), 2) - self.assertEqual( - phases[:3], - [ - "reorg-reconcile:tip-height", - "reorg-reconcile:watch-blocks", - "reorg-reconcile:watch-block", - ], - ) - for phase in ( - "reorg-reconcile:mature-payouts", - "reorg-reconcile:prepare-candidate", - "reorg-reconcile:publish", - ): - self.assertIn(phase, phases) - - def test_reconcile_ports_without_a_recorder_stamp_nothing(self) -> None: - """Background passes and focused tests keep their existing ports.""" - ledger = FakeLedger() - ports, _events = make_ports(ledger=ledger) - summary = ReorgReconcilerService(ports).reconcile( - tip_hash="tip", - force_publish=True, - ) - self.assertEqual(summary["watched_blocks"], 2) - - def test_untrusted_pass_records_skip_and_does_not_arm_memo(self) -> None: - ports, _events = make_ports(chain_untrusted=True) - service = ReorgReconcilerService(ports) - summary = service.reconcile(tip_hash="tip") - self.assertTrue(summary["untrusted"]) - self.assertEqual(service.snapshot().reconcile_skip_count, 1) - self.assertNotIn("tip", service._reorg_reconcile_trusted_memo) - self.assertFalse(service.snapshot().last_trusted) - - def test_stale_proof_epoch_refuses_to_arm_memo(self) -> None: - ports, _events = make_ports() - service = ReorgReconcilerService(ports) - # The pass started in epoch 3; detection has since moved to epoch 0 - # (make_ports pins reorg_proof_snapshot at ("tip", 0)). - service.note_outcome("tip", trusted=True, proof_epoch=3) - self.assertNotIn("tip", service._reorg_reconcile_trusted_memo) - service.note_outcome("tip", trusted=True, proof_epoch=0) - self.assertIn("tip", service._reorg_reconcile_trusted_memo) - - def test_mutating_pass_evicts_other_tips_and_error_clears_memo(self) -> None: - ports, _events = make_ports() - service = ReorgReconcilerService(ports) - service._reorg_reconcile_trusted_memo["other"] = time.monotonic() - service.note_outcome("tip", trusted=True, evict_others=True, proof_epoch=0) - self.assertNotIn("other", service._reorg_reconcile_trusted_memo) - self.assertIn("tip", service._reorg_reconcile_trusted_memo) - service.note_outcome("tip", trusted=False, clear_memo=True) - self.assertEqual(dict(service._reorg_reconcile_trusted_memo), {}) - - def test_flight_admission_runs_before_the_serialized_adapter(self) -> None: - ports, _events = make_ports() - service = ReorgReconcilerService(ports) - summary = service.reconcile_with_flights(tip_hash="tip") - self.assertEqual( - summary["serialized"], - {"tip_hash": "tip"}, - ) - # Forced/source-reserved callers bypass flight registration entirely. - forced = service.reconcile_with_flights(tip_hash="tip", _force_publish=True) - self.assertEqual( - forced["serialized"], - {"tip_hash": "tip", "_force_publish": True, "_source_reserved": False}, - ) - self.assertEqual(service._reconcile_flights, {}) - - def test_proving_prefetch_satisfies_both_memo_pass_neither(self) -> None: - release = threading.Event() - - def blocking_ensure_tip(_tip: str) -> bool: - release.wait(5.0) - return True - - ports, _events = make_ports(ensure_tip=blocking_ensure_tip) - service = ReorgReconcilerService(ports) - try: - memo_pass = service.submit_prefetch("aa" * 32) - self.assertIsNotNone(memo_pass) - # A memo-honoring pending pass may not satisfy a proving caller; - # the prove request replaces it like a tip change. - proving = service.submit_prefetch("aa" * 32, prove=True) - self.assertIsNotNone(proving) - self.assertIsNot(proving, memo_pass) - # A proving pending pass satisfies either kind of caller. - reused = service.submit_prefetch("aa" * 32) - self.assertIs(reused, proving) - reused_proving = service.submit_prefetch("aa" * 32, prove=True) - self.assertIs(reused_proving, proving) - finally: - release.set() - service.shutdown_prefetch_executor() - self.assertIsNone(service.submit_prefetch("bb" * 32)) - - def test_chain_view_validation_fails_closed(self) -> None: - self.assertTrue( - qbit_chain_view_untrusted( - lambda _method: { - "initialblockdownload": False, - "blocks": 9, - "headers": 10, - }, - "main", - ) - ) - with self.assertRaisesRegex(RuntimeError, "non-object"): - qbit_chain_view_untrusted(lambda _method: [], "regtest") - - -class CoordinatorReorgIntegrationTests(unittest.TestCase): - def test_compat_fields_adopt_pre_service_writes_and_route_after(self) -> None: - server = support.coordinator() - # The bare fixture wrote the legacy names before any service exists; - # the descriptors parked them in backing slots. - self.assertIn("_reorg_compat_enabled", server.__dict__) - self.assertNotIn("_reorg_reconciler_service", server.__dict__) - server.reorg_inactive_block_count = 5 - - service = server._ensure_reorg_reconciler_service() - - # Adoption seeds the service and removes every backing key. - self.assertFalse(service.enabled) - self.assertEqual(service.inactive_block_count, 5) - for key in list(server.__dict__): - self.assertFalse(key.startswith("_reorg_compat_"), key) - # Post-service reads and writes route to the single owner. - server.reorg_reconcile_skip_count = 9 - self.assertEqual(service.reconcile_skip_count, 9) - service.matured_payout_count = 4 - self.assertEqual(server.matured_payout_count, 4) - - def test_lookup_counts_property_returns_a_copy(self) -> None: - server = support.coordinator() - server._record_reorg_reconcile_lookup("job_build", "serial") - snapshot = server.reorg_reconcile_lookup_counts - self.assertEqual(snapshot[("job_build", "serial")], 1) - snapshot[("job_build", "serial")] = 99 - self.assertEqual( - server.reorg_reconcile_lookup_counts[("job_build", "serial")], - 1, - ) - - def test_serialized_adapter_keeps_writer_payout_preview_order(self) -> None: - source = inspect.getsource( - PrismCoordinator._reconcile_prism_pool_blocks_serialized - ) - # Writer admission decorates the adapter; the direct payout-balance - # mutation lock and the landed-preview fail-closed check run inside - # it, before the service core. The old _payout_balance_mutation() - # wrapper must not reappear. - self.assertIn("_payout_balance_mutation_lock", source) - self.assertIn("_accepted_block_payout_preview_condition", source) - self.assertIn("PayoutStatePublicationBlocked", source) - self.assertIn("_ensure_reorg_reconciler_service().reconcile(", source) - self.assertNotIn("with self._payout_balance_mutation()", source) - wrappers = getattr( - PrismCoordinator._reconcile_prism_pool_blocks_serialized, - "__wrapped__", - None, - ) - self.assertIsNotNone(wrappers, "writer-admission decorator missing") - - def test_service_wiring_uses_heartbeat_aware_preparation_lock(self) -> None: - source = inspect.getsource( - PrismCoordinator._ensure_reorg_reconciler_service - ) - self.assertIn("_block_submitter_lock", source) - self.assertIn('"payout-state-prepare"', source) - # Live config-override reads stay callable ports. - self.assertIn("reconcile_flight_wait_seconds", source) - self.assertIn("reconcile_prefetch_join_timeout_seconds", source) - # The atomic tip/epoch proof comes from the tip-refresh owner. - self.assertIn("reorg_proof_snapshot", source) - - -if __name__ == "__main__": - unittest.main() - - -class StrandedPreparedLedger: - """A ledger whose only outstanding reorg work is a stranded prepared row. - - ``reorg_watch_blocks`` returns nothing because it structurally cannot see - prepared rows -- that is exactly why the sweep exists. The depth filter - lives here, in the read, so the tests exercise the real constant the - reconciler passes rather than a value they choose themselves. - """ - - def __init__(self, rows: list[dict[str, object]]) -> None: - self.rows = rows - self.requests: list[dict[str, int]] = [] - self.rejected: list[str] = [] - - def reorg_watch_blocks(self, *, active_tip_height: int) -> list[dict[str, object]]: - return [] - - def stranded_prepared_blocks( - self, - *, - active_tip_height: int, - min_depth: int, - limit: int, - ) -> list[dict[str, object]]: - self.requests.append( - { - "active_tip_height": int(active_tip_height), - "min_depth": int(min_depth), - "limit": int(limit), - } - ) - return [ - row - for row in self.rows - if int(row["block_height"]) <= int(active_tip_height) - int(min_depth) - ][:limit] - - def reject_prepared_block( - self, - *, - block_hash: str, - active_tip_height: int, - ) -> dict[str, int]: - self.rejected.append(str(block_hash)) - return {"rejected_count": 1} - - -class StrandedPreparedSweepTests(unittest.TestCase): - """The reconciler's heal for prepared rows that lost their owner. - - A prepared pool block is resolved by the live submit/replay path that - holds its outbox row. When that row is gone -- quarantined, or completed - by a process that died before confirming -- nothing re-examines the - ledger row again: reorg_watch_blocks selects confirmed/inactive only. - Testnet4 carried one such row (height 30822) for six weeks with its - payout entries, carry forward, and fanout artifacts pinned immature. - """ - - def orphan_row(self, block_height: int = 30822) -> dict[str, object]: - return { - "block_hash": "19" * 32, - "block_height": block_height, - "parent_hash": "20" * 32, - } - - def test_orphaned_stranded_row_is_rejected_and_republished(self) -> None: - row = self.orphan_row() - ledger = StrandedPreparedLedger([row]) - ports, events = make_ports( - ledger=ledger, - tip_height=31000, - # The canonical block at that height is a different hash: the - # pool block was orphaned and never rejected. - block_hashes={30822: "aa" * 32}, - ) - service = ReorgReconcilerService(ports) - - with mock.patch("builtins.print"): - summary = service.reconcile(tip_hash="tip") - - self.assertEqual(ledger.rejected, ["19" * 32]) - self.assertEqual(summary["stranded_prepared_rejected"], 1) - self.assertEqual(summary["stranded_prepared_canonical"], 0) - # The rejection cascades payout entries, carry forward, and fanout - # artifacts, so the pass must republish rather than serve balances - # computed before it -- rereading balances over the exact window - # (#224), since the cascade never touches the share ledger. - self.assertIn( - ( - "prepare", - { - "force_full_window_rescan": False, - "force_prior_balances_read": True, - }, - ), - events, - ) - self.assertEqual(summary["published_generation"], 7) - self.assertIn(("block", {"force": True}), events) - # A row mutation invalidates proofs memoized for every other tip. - service._reorg_reconcile_trusted_memo["other"] = time.monotonic() - self.assertEqual( - ledger.requests, - [ - { - "active_tip_height": 31000, - "min_depth": STRANDED_PREPARED_REJECT_MIN_DEPTH, - "limit": STRANDED_PREPARED_SWEEP_LIMIT, - } - ], - ) - - def test_canonical_stranded_row_is_counted_but_never_mutated(self) -> None: - """The confirm path owns audit publication sequencing, not the sweep.""" - row = self.orphan_row() - ledger = StrandedPreparedLedger([row]) - ports, events = make_ports( - ledger=ledger, - tip_height=31000, - # The pool block *is* the canonical block at its height; it was - # simply never confirmed. - block_hashes={30822: "19" * 32}, - ) - service = ReorgReconcilerService(ports) - - printed: list[str] = [] - with mock.patch( - "builtins.print", - side_effect=lambda *args, **_kwargs: printed.append( - " ".join(str(value) for value in args) - ), - ): - summary = service.reconcile(tip_hash="tip") - - self.assertEqual(ledger.rejected, []) - self.assertEqual(summary["stranded_prepared_canonical"], 1) - self.assertEqual(summary["stranded_prepared_rejected"], 0) - # Nothing changed, so nothing is prepared or published. - self.assertNotIn(("block", {"force": True}), events) - self.assertIsNone(summary["published_generation"]) - self.assertTrue( - any( - "canonical and needs operator review" in message - and "19" * 32 in message - for message in printed - ), - printed, - ) - - def test_rows_inside_the_depth_floor_are_never_examined(self) -> None: - """A finalization still in flight owns its own row; depth is the proxy.""" - shallow = self.orphan_row(block_height=31000 - 1) - ledger = StrandedPreparedLedger([shallow]) - ports, events = make_ports( - ledger=ledger, - tip_height=31000, - block_hashes={}, - ) - service = ReorgReconcilerService(ports) - - summary = service.reconcile(tip_hash="tip") - - self.assertEqual(ledger.rejected, []) - self.assertEqual(summary["stranded_prepared_rejected"], 0) - self.assertEqual(summary["stranded_prepared_canonical"], 0) - # Not even a getblockhash: the row never left the ledger read. - self.assertNotIn( - "getblockhash", - [name for name, _payload in events], - ) - self.assertGreaterEqual(STRANDED_PREPARED_REJECT_MIN_DEPTH, 100) - - def test_sweep_stamps_progress_for_every_row_it_walks(self) -> None: - """Each row costs a getblockhash plus a fenced mutation, like watch rows.""" - rows = [ - self.orphan_row(block_height=30820), - self.orphan_row(block_height=30822), - ] - rows[0]["block_hash"] = "18" * 32 - ledger = StrandedPreparedLedger(rows) - ports, _events = make_ports( - ledger=ledger, - tip_height=31000, - block_hashes={30820: "aa" * 32, 30822: "aa" * 32}, - ) - phases: list[str] = [] - service = ReorgReconcilerService( - dataclass_replace(ports, record_progress=phases.append) - ) - - with mock.patch("builtins.print"): - summary = service.reconcile(tip_hash="tip") - - self.assertEqual(summary["stranded_prepared_rejected"], 2) - self.assertEqual(phases.count("reorg-reconcile:stranded-prepared"), 2) - self.assertIn("reorg-reconcile:stranded-prepared-blocks", phases) - - def test_partial_sweep_that_raises_still_accounts_its_mutations(self) -> None: - """A pass that dies mid-sweep must not lose the rows it already moved.""" - rows = [ - self.orphan_row(block_height=30820), - self.orphan_row(block_height=30822), - ] - rows[0]["block_hash"] = "18" * 32 - ledger = StrandedPreparedLedger(rows) - original_reject = ledger.reject_prepared_block - - def failing_reject(**kwargs: object) -> dict[str, int]: - if kwargs["block_hash"] == "19" * 32: - raise RuntimeError("postgres unavailable") - return original_reject(**kwargs) # type: ignore[arg-type] - - ledger.reject_prepared_block = failing_reject # type: ignore[method-assign] - ports, events = make_ports( - ledger=ledger, - tip_height=31000, - block_hashes={30820: "aa" * 32, 30822: "aa" * 32}, - ) - service = ReorgReconcilerService(ports) - - with mock.patch("builtins.print"), self.assertRaisesRegex( - RuntimeError, "postgres unavailable" - ): - service.reconcile(tip_hash="tip") - - # The first rejection committed; the error path republishes against a - # forced full rescan rather than leaving stale balances served. - self.assertEqual(ledger.rejected, ["18" * 32]) - self.assertIn(("prepare", {"force_full_window_rescan": True}), events) - self.assertEqual(service.snapshot().reconcile_error_count, 1) - self.assertEqual(dict(service._reorg_reconcile_trusted_memo), {}) - - def test_ledger_without_the_sweep_read_is_left_alone(self) -> None: - """Feature-detected like every other reconcile step.""" - ledger = FakeLedger() - ports, _events = make_ports(ledger=ledger) - service = ReorgReconcilerService(ports) - - summary = service.reconcile(tip_hash="tip", force_publish=True) - - self.assertEqual(summary["stranded_prepared_rejected"], 0) - self.assertEqual(summary["stranded_prepared_canonical"], 0) diff --git a/tests/test_prism_replay_enumeration_gate.py b/tests/test_prism_replay_enumeration_gate.py deleted file mode 100644 index c18676da1..000000000 --- a/tests/test_prism_replay_enumeration_gate.py +++ /dev/null @@ -1,526 +0,0 @@ -#!/usr/bin/env python3 -"""Replay enumeration must not queue behind the coordinator's writer convoy. - -Issue #211. ``PsqlShareLedger.pending_block_candidate_rows()`` is one -read-only, single-snapshot statement, but it used to take -``_operation_gate(self._lock, "writer lock")`` before sending it. During -accepted-block accounting that put a *read* in the same coordinator-local -convoy as the accounting write, and on ``union-mainnet`` the read spent most -of its bounded budget there: the outer call reported -``replay-outbox-query exceeded 5s`` while the inner PostgreSQL deadline still -had ~0.4-1.3s left, PostgreSQL showed zero blocked backends, and the host was -~96% idle. Accepted candidates still converged, but in 79-186s, and -``qbit_prism_accepted_parent_unresolved_oldest_seconds`` reached ~101s. - -These scenarios drive that interleaving deterministically. The writer gate is -genuinely held by a fenced write that has sent its statement and not yet -committed -- the shape a long accounting write has -- and the enumeration runs -against it. The evidence is not merely "the read returned": the same held gate, -under the same budget, is shown to *fail* writer admission on another actor, so -a run cannot pass by accident of a gate that happened to be free. - -The harness is the #128 deterministic one, extended here with the pending-page -statement (``LandingOp.OUTBOX_PENDING_PAGE``); the code under test is the -shipped ``PsqlShareLedger``, reached through its real ports. -""" - -from __future__ import annotations - -import sys -import unittest -from pathlib import Path -from typing import Any - -sys.path.insert(0, str(Path(__file__).resolve().parents[1])) - -from lab.prism.share_ledger import LedgerOperationTimeout # noqa: E402 -from tests.prism_concurrency_harness import ( # noqa: E402 - LandingOp, - LeaseHarness, - assert_deterministic, -) -from tests.prism_landing_harness import PARENT_HASH, LandingHarness # noqa: E402 - -BLOCK_A = "aa" * 32 -BLOCK_B = "bb" * 32 - -# The production fast-call budget the incident ran with -# (PRISM_BLOCK_SUBMIT_DB_TIMEOUT_SECONDS=5). Every enumeration below is armed -# with exactly this, and every writer hold outlasts it. -FAST_CALL_BUDGET_SECONDS = 5.0 - -# How long the fenced write stays parked mid-statement. Deliberately longer -# than the fast-call budget: that gap is the whole defect. -WRITER_HOLD_SECONDS = 12.0 - -# Statement checkpoints FakePostgres offers, tagged with the coordinator name. -FENCED_WRITE_PRECOMMIT = "writer.precommit" -PAGE_STATEMENT_BEGIN = "writer.begin:outbox_header_page" - - -def _intent(block_hash: str) -> dict[str, Any]: - return { - "schema": "qbit.prism.block-candidate-intent.v1", - "block_hash_hex": block_hash, - "block_hex": "00", - } - - -def _booted(harness: LeaseHarness) -> Any: - """Boot one coordinator and make its ledger gates schedulable. - - The gates are the subject, so they have to be visible to the scheduler: - a real ``threading.Lock`` would block an actor thread outside the baton - and read as a harness stall rather than as the contention it is. These - are the same substitutions the landing harness makes, and they change how - a waiter blocks, never what the ledger decides. - """ - coordinator = harness.coordinator("writer") - coordinator.start() - harness.run_until(coordinator.actor, "done:startup") - ledger = coordinator.ledger - ledger._lock = harness.lock("ledger-writer") - ledger._read_semaphore = harness.semaphore("ledger-read-slot", 4) - return ledger - - -def _park_a_fenced_write(harness: LeaseHarness, ledger: Any, block_hash: str) -> Any: - """Leave a fenced outbox write in flight, holding the writer gate. - - ``persist_block_candidate_intent`` goes through ``_run_fenced_json``, so - the writer gate is held for the whole statement. Arming a deadline is what - makes the backend run BEGIN / statement / COMMIT rather than autocommit, - which gives the scenario a checkpoint *inside* the held gate -- the - accounting write's own shape, and the only place a read can observe the - convoy. - """ - writer = harness.scheduler.actor("long-write") - - def run() -> Any: - with ledger.operation_timeout(WRITER_HOLD_SECONDS * 4): - return ledger.persist_block_candidate_intent(_intent(block_hash)) - - call = writer.submit(run, label=f"fenced-write:{block_hash[:4]}") - # Issue #255: the intent's body is staged and sealed first, in - # statements that hold no writer gate at all; only the final outbox - # insert is the fenced write this scenario parks. Run through the - # staging checkpoints until the parked statement is that insert. - for _ in range(64): - harness.run_until(writer, FENCED_WRITE_PRECOMMIT) - if harness.server.statements[-1].kind is LandingOp.OUTBOX_RECORD: - break - else: - raise AssertionError("the fenced outbox insert never reached its precommit") - return writer, call - - -def _enumerate(ledger: Any) -> list[dict[str, Any]]: - """One replay enumeration under the incident's own fast-call budget.""" - with ledger.operation_timeout(FAST_CALL_BUDGET_SECONDS): - return list(ledger.pending_block_candidate_headers(limit=32).rows) - - -def _take_writer_gate(ledger: Any) -> None: - """Ask for writer admission on the same budget: the control.""" - with ledger.operation_timeout(FAST_CALL_BUDGET_SECONDS): - ledger._acquire_operation_gate(ledger._lock, "writer lock") - ledger._lock.release() - - -def _step_to_the_stale_page_window( - harness: LandingHarness, - ledger: Any, - found: Any, -) -> None: - """Step the landing to the window a stale page is acted on in. - - Three conditions, deliberately named instead of a checkpoint: the - accepted-block write has committed so a durable ``qbit_pool_blocks`` row - exists, the candidate's outbox row is still pending so the terminal - write's ordinary state predicate cannot be what refuses it, and the writer - gate is free so that write can run at all. - - A checkpoint name would have been the shorter spelling and the wrong one. - This scenario originally stopped at ``ledger.begin:prior_balances_as_of``, - and issue #210 then routed fresh confirmations through the - current-balance summary instead of the height-bounded as-of read -- which - deleted that checkpoint from the landing while leaving the window itself - exactly where it was. Asking for the state the proof depends on survives - that kind of change; asking for the statement that happened to sit next to - it did not. - """ - block_hash = found.block_hash - for _ in range(200): - if not harness.accounting.runnable(): - raise AssertionError( - "the landing stalled before reaching the stale-page window " - f"(blocked at {harness.accounting.block_reason!r})" - ) - label = harness.step(harness.accounting) - if ( - harness.pool_block(block_hash) is not None - and harness.outbox_state(found) == "pending" - and not ledger._lock.locked() - ): - return - if label.startswith("done:"): - break - raise AssertionError( - "the landing never offered a landed-but-pending window with the " - "writer gate free" - ) - - -class EnumerationOutsideTheWriterConvoyTests(unittest.TestCase): - def _scenario(self, harness: LeaseHarness) -> dict[str, Any]: - """Enumerate a durable backlog while a fenced write holds the gate.""" - ledger = _booted(harness) - - # A durable pending candidate exists before the convoy forms, so the - # enumeration has something real to return. - setup = harness.scheduler.actor("setup") - setup.submit( - lambda: ledger.persist_block_candidate_intent(_intent(BLOCK_A)), - label="seed", - ) - harness.run_until(setup, "done:seed") - - # A second candidate's fenced write is now in flight and parked: its - # statement has been sent, its COMMIT has not, and the writer gate is - # held for the duration. - long_write, long_call = _park_a_fenced_write(harness, ledger, BLOCK_B) - gate_held_during_read = ledger._lock.locked() - - # The replay enumeration, under the incident's own budget. - replay = harness.scheduler.actor("replay") - page_call = replay.submit( - lambda: _enumerate(ledger), - label="replay-outbox-query", - ) - - # A second reader asks for *writer* admission under the same budget - # against the same held gate. It is the control: it proves the hold is - # real and outlasts the budget, so the enumeration's success below - # cannot be a gate that happened to be free. - contender = harness.scheduler.actor("writer-admission") - admission_call = contender.submit( - lambda: _take_writer_gate(ledger), - label="writer-admission", - ) - harness.run_until_blocked(contender, "lock:ledger-writer") - - # The enumeration reaches PostgreSQL with the gate still held... - harness.run_until(replay, PAGE_STATEMENT_BEGIN) - gate_held_at_statement = ledger._lock.locked() - # ...and spends measurable time in the server rather than in the gate. - harness.advance(0.75) - harness.run_until(replay, "done:replay-outbox-query") - - # Only now does the hold outlast the fast-call budget. The contender - # gives up; the fenced write is still parked. - harness.advance(WRITER_HOLD_SECONDS) - harness.drain([contender]) - - harness.drain([long_write, contender, replay]) - return { - # Random durable body IDs are irrelevant to this scheduler proof. - "page": [{key: value for key, value in row.items() if key != "body"} - for row in page_call.value()], - "gate_held_during_read": gate_held_during_read, - "gate_held_at_statement": gate_held_at_statement, - "admission_error": type(admission_call.error).__name__, - "admission_message": str(admission_call.error), - "long_write_result": ( - type(long_call.error).__name__ - if long_call.error is not None - else str(long_call.value()) - ), - "stats": ledger.ledger_read_gate_stats()[ - "pending_block_candidate_headers" - ], - } - - def test_enumeration_completes_through_the_read_path(self) -> None: - with LeaseHarness() as harness: - result = self._scenario(harness) - - # The gate really was held, both when the read started and when its - # statement reached the server. - self.assertTrue(result["gate_held_during_read"]) - self.assertTrue(result["gate_held_at_statement"]) - # The control proves the hold outlasted the identical budget. - self.assertEqual(result["admission_error"], "LedgerOperationTimeout") - self.assertIn("writer lock", result["admission_message"]) - # And the enumeration returned the durable backlog anyway. - # - # Exactly the durable backlog: BLOCK_B's row is staged inside the - # parked write's uncommitted transaction, and the page -- on its own - # pooled session, at its own snapshot -- does not see it. That is the - # commit-before-wakeup ordering the outbox relies on, stated from the - # reader's side: enumeration never surfaces a candidate before the row - # that makes it durable exists. - self.assertEqual( - [row["block_hash"] for row in result["page"]], - [BLOCK_A], - ) - self.assertEqual( - [row["pool_block_exists"] for row in result["page"]], - [False], - ) - # The long write is untouched by any of it: it commits when it resumes. - self.assertEqual( - result["long_write_result"], - "BlockCandidateIntentPersistResult(inserted=True, state='pending')", - ) - - def test_no_time_is_charged_to_local_admission(self) -> None: - """The attribution says database, because the database is where it went. - - The incident was unattributable from ``/metrics``: one duration - covered both admission and execution, so "exceeded 5s" could not be - told apart from a slow query. These two fields are that distinction. - """ - with LeaseHarness() as harness: - stats = self._scenario(harness)["stats"] - - self.assertEqual(stats["calls_total"], 1) - self.assertEqual(stats["gate_wait_seconds_max"], 0.0) - self.assertEqual(stats["gate_timeouts_total"], 0) - self.assertEqual(stats["execute_timeouts_total"], 0) - # The 0.75s advanced while the statement was in flight lands on the - # execution series, not the gate series. - self.assertAlmostEqual(stats["execute_seconds_max"], 0.75) - self.assertAlmostEqual(stats["execute_seconds_total"], 0.75) - - def test_the_interleaving_is_deterministic(self) -> None: - assert_deterministic(self, self._scenario) - - def test_enumeration_still_queues_on_the_bounded_read_slot(self) -> None: - """Bounded, not unbounded: the read slot is still the admission limit. - - Removing the writer gate must not mean removing every gate. A read - semaphore of one admits one enumeration; the second waits for a slot - rather than opening a connection or starting a thread of its own, and - it names the read slot when its own budget expires. - """ - with LeaseHarness() as harness: - ledger = _booted(harness) - ledger._read_semaphore = harness.semaphore("ledger-read-slot", 1) - - setup = harness.scheduler.actor("setup") - setup.submit( - lambda: ledger.persist_block_candidate_intent(_intent(BLOCK_A)), - label="seed", - ) - harness.run_until(setup, "done:seed") - - first = harness.scheduler.actor("replay-a") - second = harness.scheduler.actor("replay-b") - first_call = first.submit( - lambda: _enumerate(ledger), - label="page-a", - ) - second_call = second.submit( - lambda: _enumerate(ledger), - label="page-b", - ) - - # The first holds the only slot, parked inside its statement. - harness.run_until(first, PAGE_STATEMENT_BEGIN) - harness.run_until_blocked(second, "semaphore:ledger-read-slot") - - # The second's own budget expires while it waits for admission. - harness.advance(FAST_CALL_BUDGET_SECONDS * 2) - harness.drain([second]) - harness.drain([first, second]) - - self.assertIsNone(first_call.error) - self.assertEqual( - [row["block_hash"] for row in first_call.value()], - [BLOCK_A], - ) - self.assertIsInstance(second_call.error, LedgerOperationTimeout) - self.assertIn("read slot", str(second_call.error)) - - # An admission expiry is counted as one, and never as a statement - # that PostgreSQL was slow to answer. - stats = ledger.ledger_read_gate_stats()[ - "pending_block_candidate_headers" - ] - self.assertEqual(stats["calls_total"], 2) - self.assertEqual(stats["gate_timeouts_total"], 1) - self.assertEqual(stats["execute_timeouts_total"], 0) - self.assertGreaterEqual( - stats["gate_wait_seconds_max"], - FAST_CALL_BUDGET_SECONDS, - ) - - def test_no_extra_connection_or_thread_is_taken(self) -> None: - """The page read borrows a pooled connection, as it always did. - - The fix must not buy latency with a connection per call: the pool is - sized ``read_concurrency + 1`` precisely so the read slots and the - single writer cannot together outrun it. - """ - with LeaseHarness() as harness: - ledger = _booted(harness) - backend = harness.server - pool = ledger._native - - setup = harness.scheduler.actor("setup") - setup.submit( - lambda: ledger.persist_block_candidate_intent(_intent(BLOCK_A)), - label="seed", - ) - harness.run_until(setup, "done:seed") - sessions_before = len(backend.live_backends()) - - replay = harness.scheduler.actor("replay") - replay.submit(lambda: _enumerate(ledger), label="page") - harness.run_until(replay, "done:page") - - self.assertEqual(len(backend.live_backends()), sessions_before) - self.assertEqual( - pool.pool_size, - ledger._read_semaphore.value + len(pool._in_use) + 1, - ) - - -class StalePageAgainstALandingWriteTests(unittest.TestCase): - """The production landing topology, with the enumeration reading across it. - - Class one proves the gate is no longer taken. This one proves that not - taking it is *safe*: the page a caller now gets can be stale about a - candidate that lands while the read is in flight, and every terminal - decision built on that page must still refuse the landed row. The refusal - is the production ``mark_block_candidates_abandoned`` statement, whose - pool-block re-check sits inside the same fenced UPDATE that transitions - the rows. - """ - - def _scenario(self, harness: LandingHarness) -> dict[str, Any]: - ledger = harness.boot() - block_a = harness.found_block(BLOCK_A, height=10, parent_hash=PARENT_HASH) - block_b = harness.found_block(BLOCK_B, height=11, parent_hash=BLOCK_A) - - # Both candidates are durable before either lands: closely spaced - # found blocks, which is the population #211 measured converging in - # 79-186s. - harness.appender.submit( - lambda: harness._persist_intent(block_b), - label="seed-b", - ) - harness.run_until(harness.appender, "done:seed-b") - - # A's accepted-block accounting write is in flight: the statement has - # been sent, its COMMIT has not, and the writer gate is held for the - # whole of it. - harness.land_on_accounting_tail(block_a) - harness.run_until(harness.accounting, "ledger.done:persist_block") - harness.step(harness.accounting) - gate_held_during_read = ledger._lock.locked() - - # Writer admission under the fast-call budget cannot get through that - # hold -- the control for everything below. - contender = harness.scheduler.actor("writer-admission") - admission_call = contender.submit( - lambda: _take_writer_gate(ledger), - label="writer-admission", - ) - harness.run_until_blocked(contender, "lock:ledger-writer") - - # The enumeration does, on the same budget. - replay = harness.scheduler.actor("replay") - page_call = replay.submit( - lambda: _enumerate(ledger), - label="replay-outbox-query", - ) - harness.run_until(replay, "done:replay-outbox-query") - page = page_call.value() - - harness.advance(FAST_CALL_BUDGET_SECONDS * 2) - harness.drain([contender]) - - # A's write commits. Its pool-block row is now durable while its - # outbox row is still pending, and the writer gate is free again -- - # the exact window in which a caller holding the stale page acts. - _step_to_the_stale_page_window(harness, ledger, block_a) - landed = harness.pool_block(BLOCK_A) is not None - still_pending = harness.outbox_state(block_a) - - terminal = harness.scheduler.actor("collapse") - abandon_call = terminal.submit( - lambda: ledger.mark_block_candidates_abandoned( - block_hashes=[str(row["block_hash"]) for row in page], - error="superseded by a decided height", - ), - label="batch-abandon", - ) - harness.run_until(terminal, "done:batch-abandon") - - harness.drain([harness.accounting, harness.appender, terminal]) - return { - "gate_held_during_read": gate_held_during_read, - "pending_at_the_terminal_write": still_pending, - "admission_error": type(admission_call.error).__name__, - "page": [ - (str(row["block_hash"]), bool(row["pool_block_exists"])) - for row in page - ], - "landed_before_the_terminal_write": landed, - "abandoned": list(abandon_call.value()), - "outbox_a": harness.outbox_state(block_a), - "outbox_b": harness.outbox_state(block_b), - } - - def test_enumeration_crosses_a_landing_write_and_stays_advisory(self) -> None: - with LandingHarness() as harness: - result = self._scenario(harness) - - # The gate was genuinely held, and genuinely impassable on the budget. - self.assertTrue(result["gate_held_during_read"]) - self.assertEqual(result["admission_error"], "LedgerOperationTimeout") - - # The page came back, and it is stale about A: A's pool-block row - # exists only inside the uncommitted landing transaction, so this - # snapshot cannot see it. That staleness is allowed -- it is what - # "advisory" means -- and it is exactly the input the next assertion - # feeds to a terminal write. - # - # The order is A then B even though B was persisted first: both rows - # carry the same creation stamp here, and the total order's block-hash - # tiebreak decides. That tiebreak is the reason a cursor cannot replay - # or skip a colliding group. - self.assertEqual( - result["page"], - [(BLOCK_A, False), (BLOCK_B, False)], - ) - - def test_a_landed_candidate_is_refused_by_the_fence_not_the_page(self) -> None: - with LandingHarness() as harness: - result = self._scenario(harness) - - # By the time the terminal write runs, A has landed and is still - # pending -- both halves matter, or the row would be excluded by the - # ordinary state predicate and prove nothing about the re-check. - self.assertTrue(result["landed_before_the_terminal_write"]) - self.assertEqual(result["pending_at_the_terminal_write"], "pending") - - # The caller asked to abandon both hashes on the strength of a page - # that said neither had landed. The fenced UPDATE re-asked - # qbit_pool_blocks and returned only the row that had not. - self.assertEqual(result["abandoned"], [BLOCK_B]) - self.assertEqual(result["outbox_b"], "abandoned") - # A survived the stale page and finished its landing normally. - self.assertEqual(result["outbox_a"], "submitted") - - def test_the_interleaving_is_deterministic(self) -> None: - assert_deterministic( - self, - self._scenario, - harness_factory=LandingHarness, - ) - - -if __name__ == "__main__": # pragma: no cover - direct invocation - unittest.main() diff --git a/tests/test_prism_representative_independence.py b/tests/test_prism_representative_independence.py deleted file mode 100644 index 58e583339..000000000 --- a/tests/test_prism_representative_independence.py +++ /dev/null @@ -1,445 +0,0 @@ -#!/usr/bin/env python3 -"""Deterministic disconnect races for representative-free PRISM refreshes.""" - -from __future__ import annotations - -import threading -import time -import unittest - -from lab.prism.job_bundle import CachedJobBundle, CollectionIdentityUnavailable -from lab.prism.prism_coordinator import ( - CachedTemplateArtifacts, - StratumError, - TemplateRefreshBlocked, - WorkerIdentity, -) -from tests.prism_coordinator_test_support import ( - FakeLedger, - base_template, - client, - coordinator, - install_fake_bundle_builder, - worker, -) - - -class RepresentativeIndependentRefreshTests(unittest.TestCase): - def _run_ready_disconnect_race(self, stage: str) -> None: - server, rpc = coordinator() - recorded = install_fake_bundle_builder(server) - disconnected = client(1) - survivor = client(2) - server.clients = {disconnected, survivor} - survivor_payloads: list[dict[str, object]] = [] - survivor.send = survivor_payloads.append # type: ignore[method-assign] - disconnected.send = lambda _payload: self.fail( # type: ignore[method-assign] - "disconnected target received prepared work" - ) - - def remove_target() -> None: - with server.lock: - server.clients.discard(disconnected) - - if stage == "before_build": - original_shared_job_bundle = server.shared_job_bundle - - def disconnect_before_build( - artifacts: CachedTemplateArtifacts, - identity: WorkerIdentity | None = None, - **kwargs: object, - ) -> CachedJobBundle: - remove_target() - return original_shared_job_bundle(artifacts, identity, **kwargs) - - server.shared_job_bundle = disconnect_before_build # type: ignore[method-assign] - elif stage == "during_build": - original_build_audit_bundle = server.build_audit_bundle - - def disconnect_during_build(**kwargs: object) -> dict[str, object]: - remove_target() - return original_build_audit_bundle(**kwargs) - - server.build_audit_bundle = disconnect_during_build # type: ignore[method-assign] - elif stage == "before_fanout": - original_fanout = server._fanout_prepared_tip_refresh - - def disconnect_before_fanout(*args: object, **kwargs: object) -> object: - remove_target() - return original_fanout(*args, **kwargs) - - server._fanout_prepared_tip_refresh = disconnect_before_fanout # type: ignore[method-assign] - else: # pragma: no cover - test helper guard - raise AssertionError(f"unknown race stage {stage}") - - try: - refreshed = server.poll_qbit_tip_template_once() - finally: - server.shutdown_tip_refresh_executor() - - self.assertEqual(refreshed, 1) - self.assertEqual(recorded["calls"], 1) - self.assertEqual(rpc.count("getblocktemplate"), 1) - self.assertEqual(server.ledger.snapshot_calls, 1) - self.assertEqual( - [payload["method"] for payload in survivor_payloads], - ["mining.set_difficulty", "mining.notify"], - ) - self.assertEqual(server.tip_refresh_client_counts["disconnected"], 1) - self.assertIsNotNone(survivor.active_job) - assert survivor.active_job is not None - self.assertFalse(survivor.active_job.collection_only) - - def test_target_disconnect_immediately_before_ready_bundle_construction(self) -> None: - self._run_ready_disconnect_race("before_build") - - def test_target_disconnect_during_ready_bundle_construction(self) -> None: - self._run_ready_disconnect_race("during_build") - - def test_target_disconnect_after_ready_construction_before_fanout(self) -> None: - self._run_ready_disconnect_race("before_fanout") - - def test_reselected_poll_start_target_keeps_expected_job_snapshot(self) -> None: - server, _rpc = coordinator() - install_fake_bundle_builder(server) - stable = client(1) - reselected = client(2) - server.clients = {stable, reselected} - original_fetch = server.fetch_qbit_tip_template_snapshot - - def make_target_temporarily_ineligible() -> object: - snapshot = original_fetch() - reselected.authorized = False - return snapshot - - def restore_target(_tip_hash: str) -> bool: - reselected.authorized = True - return True - - captured_expected: dict[object, object] = {} - - def capture_fanout( - clients: list[object], - *_args: object, - expected_active_jobs: dict[object, object], - **_kwargs: object, - ) -> tuple[int, float, float, int]: - captured_expected.update(expected_active_jobs) - delivered = time.monotonic() - return len(clients), delivered, delivered, 0 - - server.fetch_qbit_tip_template_snapshot = make_target_temporarily_ineligible # type: ignore[method-assign] - server.ensure_reorg_reconciled_for_tip = restore_target # type: ignore[method-assign] - server._fanout_prepared_tip_refresh = capture_fanout # type: ignore[method-assign] - try: - refreshed = server.poll_qbit_tip_template_once() - finally: - server.shutdown_tip_refresh_executor() - - self.assertEqual(refreshed, 2) - self.assertIn(stable, captured_expected) - self.assertIn(reselected, captured_expected) - - def test_ready_bundle_builds_without_clients_or_worker_identity(self) -> None: - server, _rpc = coordinator() - recorded = install_fake_bundle_builder(server) - snapshot = server.fetch_qbit_tip_template_snapshot() - server.clients = set() - - bundle = server.prepare_tip_refresh_bundle(snapshot) - cached = server.prepare_tip_refresh_bundle(snapshot) - - self.assertFalse(bundle.collection_only) - self.assertIsNone(bundle.collection_identity) - self.assertIs(cached, bundle) - self.assertEqual(recorded["calls"], 1) - self.assertEqual(server.ledger.snapshot_calls, 1) - - def test_collection_ineligible_connected_target_is_skipped(self) -> None: - server, _rpc = coordinator(ledger=FakeLedger(miners=["solo"])) - state = client(1) - server.clients = {state} - original_observe_tip = server.observe_tip_first_seen - - def make_target_ineligible(*args: object, **kwargs: object) -> bool: - observed = original_observe_tip(*args, **kwargs) - state.authorized = False - return observed - - server.observe_tip_first_seen = make_target_ineligible # type: ignore[method-assign] - server.maybe_send_job = lambda *_args, **_kwargs: self.fail( # type: ignore[method-assign] - "ineligible collection target received work" - ) - - try: - refreshed = server.poll_qbit_tip_template_once() - finally: - server.shutdown_tip_refresh_executor() - - self.assertEqual(refreshed, 0) - self.assertEqual(server.tip_refresh_client_counts["skipped"], 1) - self.assertEqual(server.tip_refresh_client_counts["disconnected"], 0) - - def test_collection_identity_absence_has_a_distinct_temporary_result(self) -> None: - server, _rpc = coordinator(ledger=FakeLedger(miners=["solo"])) - artifacts = server.current_template_artifacts() - - with self.assertRaisesRegex( - CollectionIdentityUnavailable, - "temporarily unavailable", - ): - server.shared_job_bundle(artifacts, mode="collection") - - def test_all_collection_identities_disappear_then_authorization_reuses_artifacts( - self, - ) -> None: - server, rpc = coordinator(ledger=FakeLedger(miners=["solo"])) - recorded = install_fake_bundle_builder(server) - server.template_cache_seconds = 0.0 - original = client(1) - original.send = lambda _payload: None # type: ignore[method-assign] - original.close = lambda: None # type: ignore[method-assign] - server.clients = {original} - - self.assertEqual(server.poll_qbit_tip_template_once(), 1) - self.assertIsNone(server._retained_collection_refresh) - server.disconnect_client(original) - retained = server._retained_collection_refresh - self.assertIsNotNone(retained) - self.assertEqual(rpc.count("getblocktemplate"), 1) - self.assertEqual(recorded["calls"], 1) - - authorized = client(3, original.worker) - authorized.authorized = False - sent: list[dict[str, object]] = [] - authorized.send = sent.append # type: ignore[method-assign] - server.clients.add(authorized) - authorized.authorized = True - server._note_collection_identity_available(authorized) - - self.assertTrue(server.tip_refresh_is_pending()) - self.assertTrue(server._tip_refresh_retry.is_set()) - self.assertTrue(server.maybe_send_job(authorized, clean_jobs=True)) - self.assertEqual(rpc.count("getblocktemplate"), 1) - self.assertEqual(recorded["calls"], 1) - self.assertEqual( - [payload["method"] for payload in sent], - ["mining.set_difficulty", "mining.notify"], - ) - self.assertIsNotNone(authorized.active_job) - assert authorized.active_job is not None and retained is not None - self.assertTrue(authorized.active_job.collection_only) - self.assertIs( - authorized.active_job.template, - retained.snapshot.template_artifacts.template, - ) - self.assertIsNone(server._retained_collection_refresh) - - pending_token = server._tip_refresh_pending_token - if pending_token is not None: - server._clear_tip_refresh_pending(pending_token) - server._tip_refresh_retry.clear() - server._note_collection_identity_available(authorized) - - self.assertFalse(server.tip_refresh_is_pending()) - self.assertFalse(server._tip_refresh_retry.is_set()) - - def test_authorization_during_same_tip_publication_keeps_retained_wake( - self, - ) -> None: - server, rpc = coordinator(ledger=FakeLedger(miners=["solo"])) - recorded = install_fake_bundle_builder(server) - server.template_cache_seconds = 0.0 - server.clients = set() - - self.assertEqual(server.poll_qbit_tip_template_once(), 0) - retained = server._retained_collection_refresh - self.assertIsNotNone(retained) - self.assertEqual(rpc.count("getblocktemplate"), 1) - server._tip_refresh_retry.clear() - - publication_window = threading.Event() - release_publication = threading.Event() - original_observe_tip = server.observe_tip_first_seen - - def pause_after_same_tip_observation( - *args: object, - **kwargs: object, - ) -> bool: - observed = original_observe_tip(*args, **kwargs) - if kwargs.get("publish_refresh_observation"): - publication_window.set() - self.assertTrue(release_publication.wait(5.0)) - return observed - - server.observe_tip_first_seen = pause_after_same_tip_observation # type: ignore[method-assign] - poll_results: list[int] = [] - poll_errors: list[BaseException] = [] - - def poll_same_tip() -> None: - try: - poll_results.append(server.poll_qbit_tip_template_once()) - except BaseException as exc: # pragma: no cover - asserted below - poll_errors.append(exc) - - try: - poll_thread = threading.Thread(target=poll_same_tip) - poll_thread.start() - authorized = client(2) - sent: list[dict[str, object]] = [] - authorized.send = sent.append # type: ignore[method-assign] - try: - self.assertTrue(publication_window.wait(5.0)) - with server.lock: - server.clients.add(authorized) - server._note_collection_identity_available(authorized) - - self.assertTrue(server.tip_refresh_is_pending()) - self.assertTrue(server._tip_refresh_retry.is_set()) - self.assertTrue(server.maybe_send_job(authorized, clean_jobs=True)) - self.assertEqual(rpc.count("getblocktemplate"), 2) - self.assertEqual(recorded["calls"], 1) - self.assertIsNotNone(authorized.active_job) - with server.lock: - published_snapshot = server.tip_template_snapshot - assert ( - authorized.active_job is not None - and published_snapshot is not None - and published_snapshot.template_artifacts is not None - ) - self.assertIs( - authorized.active_job.template, - published_snapshot.template_artifacts.template, - ) - finally: - release_publication.set() - poll_thread.join(5.0) - - self.assertFalse(poll_thread.is_alive()) - self.assertEqual(poll_results, []) - self.assertEqual(len(poll_errors), 1) - self.assertIsInstance(poll_errors[0], TemplateRefreshBlocked) - # Authorization minted a newer wake token while this poll owned - # the previous state. The first poll must not clear that work. - self.assertTrue(server.tip_refresh_is_pending()) - self.assertTrue(server._tip_refresh_retry.is_set()) - self.assertEqual(server.poll_qbit_tip_template_once(), 0) - self.assertFalse(server.tip_refresh_is_pending()) - self.assertIsNone(server._retained_collection_refresh) - self.assertEqual( - [payload["method"] for payload in sent], - ["mining.set_difficulty", "mining.notify"], - ) - self.assertIsNotNone(authorized.active_job) - finally: - release_publication.set() - server.shutdown_tip_refresh_executor() - - def test_ready_latch_discards_retained_collection_marker(self) -> None: - ledger = FakeLedger(miners=["solo"]) - server, _rpc = coordinator(ledger=ledger) - server.clients = set() - - self.assertEqual(server.poll_qbit_tip_template_once(), 0) - self.assertIsNotNone(server._retained_collection_refresh) - ledger.miners = ["miner-a", "miner-b", "miner-c"] - - self.assertTrue(server.pool_readiness_latched()) - self.assertIsNone(server._retained_collection_refresh) - - def test_collection_reauthorization_reselects_identity_without_template_refetch( - self, - ) -> None: - server, rpc = coordinator(ledger=FakeLedger(miners=["solo"])) - recorded = install_fake_bundle_builder(server) - worker_a = worker(payout="tq1worker-a", username="worker-a") - worker_b = worker(payout="tq1worker-b", username="worker-b") - state = client(1, worker_a) - artifacts = server.current_template_artifacts() - original_shared_job_bundle = server.shared_job_bundle - calls = 0 - - def reauthorize_after_first_bundle( - build_artifacts: CachedTemplateArtifacts, - identity: WorkerIdentity | None = None, - **kwargs: object, - ) -> CachedJobBundle: - nonlocal calls - calls += 1 - bundle = original_shared_job_bundle( - build_artifacts, - identity, - **kwargs, - ) - if calls == 1: - state.worker = worker_b - state.username = worker_b.username - return bundle - - server.shared_job_bundle = reauthorize_after_first_bundle # type: ignore[method-assign] - - context = server.build_job_for_client_from_artifacts( - state, - artifacts, - clean_jobs=True, - ) - - self.assertEqual(calls, 2) - self.assertEqual(recorded["calls"], 2) - self.assertEqual(rpc.count("getblocktemplate"), 1) - self.assertIs(context.worker, worker_b) - self.assertEqual( - context.found_block["coinbase_value_sats"], - base_template()["coinbasevalue"], - ) - self.assertEqual( - recorded["last_kwargs"]["shares"][0]["miner_id"], - worker_b.payout_address, - ) - - def test_collection_bundle_cannot_be_stamped_across_worker_identities(self) -> None: - server, _rpc = coordinator(ledger=FakeLedger(miners=["solo"])) - install_fake_bundle_builder(server) - worker_a = worker(payout="tq1worker-a") - worker_b = worker(payout="tq1worker-b") - state = client(1, worker_a) - artifacts = server.current_template_artifacts() - bundle_a = server.shared_job_bundle(artifacts, worker_a) - - state.worker = worker_b - state.username = worker_b.username - - with self.assertRaisesRegex(StratumError, "no longer matches"): - server.stamp_job_for_client(state, bundle_a, clean_jobs=True) - - def test_new_tip_supersedes_retained_collection_preparation(self) -> None: - old_tip = "11" * 32 - new_tip = "22" * 32 - server, rpc = coordinator( - ledger=FakeLedger(miners=["solo"]), - template=base_template(height=10, prevhash=old_tip), - ) - install_fake_bundle_builder(server) - server.clients = set() - - self.assertEqual(server.poll_qbit_tip_template_once(), 0) - old_retained = server._retained_collection_refresh - self.assertIsNotNone(old_retained) - - rpc.tip = new_tip - rpc.template = base_template(height=11, prevhash=new_tip) - self.assertTrue(server.observe_tip_first_seen(new_tip)) - self.assertIsNone(server._retained_collection_refresh) - - self.assertEqual(server.poll_qbit_tip_template_once(), 0) - current = server._retained_collection_refresh - self.assertIsNotNone(current) - assert current is not None and old_retained is not None - self.assertIsNot(current.snapshot, old_retained.snapshot) - self.assertEqual(current.snapshot.bestblockhash, new_tip) - self.assertEqual(current.snapshot.previousblockhash, new_tip) - self.assertEqual(rpc.count("getblocktemplate"), 2) - - -if __name__ == "__main__": - unittest.main() diff --git a/tests/test_prism_retained_jobs.py b/tests/test_prism_retained_jobs.py deleted file mode 100644 index 7b0a216e4..000000000 --- a/tests/test_prism_retained_jobs.py +++ /dev/null @@ -1,1248 +0,0 @@ -#!/usr/bin/env python3 -"""Focused PRISM coordinator retained jobs tests.""" -# ruff: noqa: F403, F405 - -from __future__ import annotations - -import unittest -from tests.prism_vardiff_test_support import * - - -class PrismCoordinatorVardiffTests(unittest.TestCase): - def test_clean_job_prunes_previous_active_prism_job(self) -> None: - server = coordinator() - state = client() - state.worker = WorkerIdentity( - username="miner-a", - payout_address="miner-a", - worker_name=None, - script_pubkey_hex="5220" + "11" * 32, - p2mr_program_hex="11" * 32, - ) - server.jobs = {} - counter = {"value": 0} - - def build_context(client: ClientState, *, clean_jobs: bool) -> object: - counter["value"] += 1 - return SimpleNamespace( - job=SimpleNamespace(job_id=f"job-{counter['value']}", share_difficulty=Decimal("1")), - template={"previousblockhash": "00" * 32}, - collection_only=False, - ) - - server.build_job_for_client = build_context # type: ignore[method-assign] - server.send_difficulty = lambda *args, **kwargs: None - server.send_job = lambda *args, **kwargs: None - server.apply_job_difficulty = lambda *args, **kwargs: None - - server.maybe_send_job(state, clean_jobs=True) - first_job_id = next(iter(state.active_job_ids)) - server.maybe_send_job(state, clean_jobs=True) - second_job_id = next(iter(state.active_job_ids)) - - self.assertNotEqual(first_job_id, second_job_id) - self.assertNotIn(first_job_id, server.jobs) - self.assertIn(second_job_id, server.jobs) - - state.sock = SimpleNamespace(shutdown=lambda *_args: None, close=lambda: None) - server.disconnect_client(state) - self.assertNotIn(second_job_id, server.jobs) - self.assertEqual(state.active_job_ids, set()) - - def test_non_clean_job_retention_caps_previous_active_prism_jobs(self) -> None: - server = coordinator() - state = client() - state.worker = WorkerIdentity( - username="miner-a", - payout_address="miner-a", - worker_name=None, - script_pubkey_hex="5220" + "11" * 32, - p2mr_program_hex="11" * 32, - ) - server.jobs = {} - counter = {"value": 0} - - def build_context(client: ClientState, *, clean_jobs: bool) -> object: - counter["value"] += 1 - return SimpleNamespace( - job=SimpleNamespace(job_id=f"job-{counter['value']}", share_difficulty=Decimal("1")), - template={"previousblockhash": "00" * 32}, - collection_only=False, - ) - - server.build_job_for_client = build_context # type: ignore[method-assign] - server.send_difficulty = lambda *args, **kwargs: None - server.send_job = lambda *args, **kwargs: None - server.apply_job_difficulty = lambda *args, **kwargs: None - - total_jobs = MAX_ACTIVE_PRISM_JOBS_PER_CLIENT + 3 - for _ in range(total_jobs): - server.maybe_send_job(state, clean_jobs=False) - - retained_ids = { - f"job-{index}" - for index in range(4, total_jobs + 1) - } - self.assertEqual(state.active_job_ids, retained_ids) - self.assertEqual(set(server.jobs), retained_ids) - self.assertNotIn("job-1", server.jobs) - self.assertEqual(state.active_job.job.job_id, f"job-{total_jobs}") - - def test_normal_accepted_share_does_not_close_client(self) -> None: - server, state, ledger = submit_coordinator() - submission = SimpleNamespace( - header_hex="aa" * 80, - block_hash_hex="bb" * 32, - share_pass=True, - block_pass=False, - ) - - with patch( - "lab.prism.prism_coordinator.direct_stratum.assemble_submission", - return_value=submission, - ): - should_close = server.handle_submit( - state, - ["miner-a", "job-1", "00" * 8, "00000001", "00000002"], - ) - - self.assertFalse(should_close) - self.assertEqual(len(ledger.pending), 1) - self.assertEqual(ledger.pending[0].share_id, "miner-a:" + "bb" * 32) - - def test_prior_tip_share_inside_grace_is_credited_without_submitblock(self) -> None: - old_tip = "00" * 32 - new_tip = "11" * 32 - server, state, ledger = submit_coordinator(tip=old_tip) - rpc = ParentTipRpc(tip=new_tip, parent=old_tip) - server.rpc = rpc - server.current_tip_first_seen = (new_tip, time.monotonic()) - server.stale_grace_seconds = 3 - submission = SimpleNamespace( - header_hex="aa" * 80, - block_hash_hex="cc" * 32, - share_pass=True, - block_pass=True, - ) - - with ( - patch("builtins.print") as emitted, - patch( - "lab.prism.prism_coordinator.direct_stratum.assemble_submission", - return_value=submission, - ), - ): - should_close = server.handle_submit( - state, - ["miner-a", "job-1", "00" * 8, "00000001", "00000002"], - ) - - self.assertFalse(should_close) - self.assertEqual(rpc.submitblock_calls, 0) - self.assertEqual(len(ledger.pending), 1) - self.assertEqual(ledger.pending[0].credit_policy, PRISM_CREDIT_POLICY_STALE_GRACE) - self.assertEqual(server.grace_credited_share_count, 1) - self.assertEqual(server.worker_share_counts["miner-a"]["grace"], 1) - self.assertEqual(server.block_solves_dropped_counts, {"stale_grace": 1}) - self.assertIn( - 'qbit_prism_block_solves_dropped_total{reason="stale_grace"} 1', - server.metrics_payload(), - ) - self.assertEqual(emitted.call_count, 1) - log_line = " ".join(str(value) for value in emitted.call_args.args) - self.assertIn(submission.block_hash_hex, log_line) - self.assertIn(old_tip, log_line) - - def test_evicted_prior_tip_share_inside_grace_is_credited(self) -> None: - old_tip = "00" * 32 - new_tip = "11" * 32 - server, state, ledger = submit_coordinator(tip=old_tip) - rpc = ParentTipRpc(tip=new_tip, parent=old_tip) - server.rpc = rpc - server.current_tip_first_seen = (new_tip, time.monotonic()) - server.bury_evicted_job(state, "job-1") - server.jobs.pop("job-1") - state.active_job_ids.clear() - submission = SimpleNamespace( - header_hex="ad" * 80, - block_hash_hex="cd" * 32, - share_pass=True, - block_pass=False, - ) - - with patch( - "lab.prism.prism_coordinator.direct_stratum.assemble_submission", - return_value=submission, - ): - should_close = server.handle_submit( - state, - ["miner-a", "job-1", "00" * 8, "00000001", "00000002"], - ) - - self.assertFalse(should_close) - self.assertEqual(len(ledger.pending), 1) - self.assertEqual(ledger.pending[0].credit_policy, PRISM_CREDIT_POLICY_STALE_GRACE) - - def test_evicted_same_tip_share_is_credited_without_stale_grace_policy(self) -> None: - tip = "00" * 32 - server, state, ledger = submit_coordinator(tip=tip) - server.bury_evicted_job(state, "job-1") - server.jobs.pop("job-1") - state.active_job_ids.clear() - submission = SimpleNamespace( - header_hex="af" * 80, - block_hash_hex="cf" * 32, - share_pass=True, - block_pass=False, - ) - - with patch( - "lab.prism.prism_coordinator.direct_stratum.assemble_submission", - return_value=submission, - ): - should_close = server.handle_submit( - state, - ["miner-a", "job-1", "00" * 8, "00000001", "00000002"], - ) - - self.assertFalse(should_close) - self.assertEqual(len(ledger.pending), 1) - self.assertIsNone(ledger.pending[0].credit_policy) - - def test_retained_share_dedup_uses_original_worker_after_reauthorization(self) -> None: - tip = "00" * 32 - server, state, ledger = submit_coordinator(tip=tip) - server.bury_evicted_job(state, "job-1") - server.jobs.pop("job-1") - state.active_job_ids.clear() - submission = SimpleNamespace( - header_hex="af" * 80, - block_hash_hex="cf" * 32, - share_pass=True, - block_pass=False, - ) - - with patch( - "lab.prism.prism_coordinator.direct_stratum.assemble_submission", - return_value=submission, - ): - self.assertFalse( - server.handle_submit( - state, - ["miner-a", "job-1", "00" * 8, "00000001", "00000002"], - ) - ) - state.username = "miner-b" - state.worker = worker_identity("miner-b") - with self.assertRaises(StratumError) as raised: - server.handle_submit( - state, - ["miner-b", "job-1", "00" * 8, "00000001", "00000002"], - ) - - self.assertEqual(raised.exception.reason, PRISM_REJECTION_DUPLICATE_SHARE) - self.assertEqual(len(ledger.pending), 1) - self.assertEqual(ledger.pending[0].share_id, "miner-a:" + "cf" * 32) - self.assertEqual(server.worker_share_counts["miner-a"]["accepted"], 1) - self.assertEqual(server.worker_share_counts["miner-b"]["accepted"], 0) - - def test_evicted_same_tip_share_survives_beyond_legacy_one_second_floor(self) -> None: - tip = "00" * 32 - server, state, ledger = submit_coordinator(tip=tip) - server.current_tip_first_seen = (tip, None) - server.same_tip_job_retention_seconds = 30 - server.bury_evicted_job(state, "job-1", now=100.0) - server.jobs.pop("job-1") - state.active_job_ids.clear() - submission = SimpleNamespace( - header_hex="a1" * 80, - block_hash_hex="c1" * 32, - share_pass=True, - block_pass=False, - ) - - with patch( - "lab.prism.prism_coordinator.time.monotonic", - return_value=102.0, - ), patch( - "lab.prism.prism_coordinator.direct_stratum.assemble_submission", - return_value=submission, - ): - self.assertFalse( - server.handle_submit( - state, - ["miner-a", "job-1", "00" * 8, "00000001", "00000002"], - ) - ) - - self.assertEqual(len(ledger.pending), 1) - self.assertIsNone(ledger.pending[0].credit_policy) - self.assertIn("job-1", server.evicted_job_graveyard) - self.assertEqual(server.evicted_job_submit_counts["accepted_same_tip"], 1) - - def test_evicted_same_tip_submit_uses_original_job_difficulty(self) -> None: - tip = "00" * 32 - server, state, ledger = submit_coordinator(tip=tip) - server.current_tip_first_seen = (tip, None) - original_context = server.jobs["job-1"] - original_context.job.share_difficulty = Decimal("2") - state.share_difficulty = Decimal("32") - server.bury_evicted_job(state, "job-1") - server.jobs.pop("job-1") - state.active_job_ids.clear() - submission = SimpleNamespace( - header_hex="a2" * 80, - block_hash_hex="c2" * 32, - share_pass=True, - block_pass=False, - ) - - def assemble(job: object, **_kwargs: object) -> object: - self.assertIs(job, original_context.job) - self.assertEqual(job.share_difficulty, Decimal("2")) - return submission - - with patch( - "lab.prism.prism_coordinator.direct_stratum.assemble_submission", - side_effect=assemble, - ): - self.assertFalse( - server.handle_submit( - state, - ["miner-a", "job-1", "00" * 8, "00000001", "00000002"], - ) - ) - - self.assertEqual(len(ledger.pending), 1) - self.assertIsNone(ledger.pending[0].credit_policy) - - def test_same_tip_retention_ttl_and_capacity_are_bounded(self) -> None: - tip = "00" * 32 - server, state, _ledger = submit_coordinator(tip=tip) - server.current_tip_first_seen = (tip, None) - server.same_tip_job_retention_seconds = 30 - server.same_tip_job_retention_per_connection = 2 - identity = state.worker - for index in range(3): - job_id = f"job-{index + 1}" - server.jobs[job_id] = prism_context(job_id, tip, worker=identity) - server.bury_evicted_job(state, job_id, now=100.0 + index) - - self.assertNotIn("job-1", server.evicted_job_graveyard) - self.assertEqual( - list(server.evicted_job_graveyard), - ["job-2", "job-3"], - ) - self.assertEqual(server.evicted_job_capacity_eviction_counts["connection"], 1) - - server.prune_evicted_job_graveyard(now=133.1) - self.assertEqual(server.evicted_job_graveyard, {}) - self.assertEqual(server.evicted_job_expiration_counts["same_tip"], 2) - - def test_evicted_job_hit_is_constant_work_in_large_graveyard(self) -> None: - tip = "00" * 32 - server, state, _ledger = submit_coordinator(tip=tip) - server.current_tip_first_seen = (tip, None) - server.same_tip_job_retention_seconds = 30 - server.same_tip_job_retention_per_connection = 4_096 - for index in range(4_096): - job_id = f"retained-{index}" - server.jobs[job_id] = prism_context(job_id, tip, worker=state.worker) - server.bury_evicted_job(state, job_id, now=100.0, prune=False) - - self.assertEqual(len(server.evicted_job_graveyard), 4_096) - self.assertEqual(len(server.evicted_same_tip_job_ids), 4_096) - classify_calls = 0 - original_classify = server._evicted_job_class_locked - - def counted_classify(entry: object) -> str: - nonlocal classify_calls - classify_calls += 1 - return original_classify(entry) - - server._evicted_job_class_locked = counted_classify # type: ignore[method-assign] - with patch("lab.prism.prism_coordinator.time.monotonic", return_value=101.0): - for _ in range(100): - self.assertIsNotNone( - server.evicted_job_entry(state, "retained-2048") - ) - - self.assertEqual(classify_calls, 100) - - def test_pool_width_does_not_evict_other_connections_retained_jobs(self) -> None: - tip = "00" * 32 - server, _state, _ledger = submit_coordinator(tip=tip) - server.current_tip_first_seen = (tip, None) - server.same_tip_job_retention_seconds = 30 - server.same_tip_job_retention_per_connection = 1 - clients: list[ClientState] = [] - for index in range(4_097): - state = client() - state.connection_id = index + 1 - state.worker = worker_identity(f"miner-{index}") - clients.append(state) - job_id = f"wide-{index}" - server.jobs[job_id] = prism_context(job_id, tip, worker=state.worker) - server.bury_evicted_job(state, job_id, now=100.0, prune=False) - - self.assertEqual(len(server.evicted_job_graveyard), 4_097) - self.assertIn("wide-0", server.evicted_job_graveyard) - self.assertIn("wide-4096", server.evicted_job_graveyard) - - replacement_id = "wide-0-replacement" - server.jobs[replacement_id] = prism_context( - replacement_id, - tip, - worker=clients[0].worker, - ) - server.bury_evicted_job( - clients[0], - replacement_id, - now=101.0, - prune=False, - ) - - self.assertNotIn("wide-0", server.evicted_job_graveyard) - self.assertIn("wide-1", server.evicted_job_graveyard) - self.assertIn(replacement_id, server.evicted_job_graveyard) - self.assertEqual(len(server.evicted_job_graveyard), 4_097) - self.assertEqual(server.evicted_job_capacity_eviction_counts["connection"], 1) - - def test_tip_flip_reanchors_retained_job_grace_to_client_delivery(self) -> None: - old_tip = "00" * 32 - new_tip = "11" * 32 - server, state, _ledger = submit_coordinator(tip=old_tip) - server.clients = {state} - server.current_tip_first_seen = (old_tip, None) - server.stale_grace_seconds = 3 - server.bury_evicted_job(state, "job-1", now=100.0) - - with patch("lab.prism.prism_coordinator.time.monotonic", return_value=120.0): - server.observe_tip_first_seen(new_tip) - - # Burial predates the flip by twenty seconds, but grace does not begin - # until this connection actually receives replacement work. - server.prune_evicted_job_graveyard(now=130.0) - self.assertIn("job-1", server.evicted_job_graveyard) - - state.tip_work_delivered = (new_tip, 130.0) - server.prune_evicted_job_graveyard(now=132.9) - self.assertIn("job-1", server.evicted_job_graveyard) - server.prune_evicted_job_graveyard(now=133.1) - self.assertNotIn("job-1", server.evicted_job_graveyard) - self.assertEqual(server.evicted_job_expiration_counts["stale_grace"], 1) - - def test_tip_flip_prunes_by_chain_parent_when_poller_skips_observed_tip(self) -> None: - observed_tip = "00" * 32 - intermediate_tip = "11" * 32 - current_tip = "22" * 32 - server, state, _ledger = submit_coordinator(tip=intermediate_tip) - server.clients = {state} - server.current_tip_first_seen = (observed_tip, None) - server.stale_grace_seconds = 3 - server.jobs["older-job"] = prism_context( - "older-job", - observed_tip, - worker=state.worker, - ) - server.bury_evicted_job(state, "older-job", now=100.0, prune=False) - server.bury_evicted_job(state, "job-1", now=110.0, prune=False) - server.rpc = ParentTipRpc(tip=current_tip, parent=intermediate_tip) - - with patch("lab.prism.prism_coordinator.time.monotonic", return_value=120.0): - server.observe_tip_first_seen(current_tip) - - # The poller's previous observation is not authoritative. Tip - # observation proactively loads the actual parent, drops older work, - # and preserves the intermediate-tip context that submit can credit. - self.assertNotIn("older-job", server.evicted_job_graveyard) - self.assertIn("job-1", server.evicted_job_graveyard) - self.assertEqual( - server.current_tip_parent_hash(current_tip), - intermediate_tip, - ) - entry = server.evicted_job_entry(state, "job-1") - self.assertIsNotNone(entry) - assert entry is not None - self.assertEqual( - server.evicted_submit_context(state, entry, current_tip), - (entry.context, PRISM_CREDIT_POLICY_STALE_GRACE), - ) - - def test_slow_parent_lookup_cannot_overwrite_newer_tip_parent_cache(self) -> None: - old_tip = "00" * 32 - old_parent = "ff" * 32 - new_tip = "11" * 32 - new_parent = old_tip - server = coordinator() - server.current_tip_first_seen = (old_tip, None) - server.current_tip_observation_sequence = 1 - server.current_tip_parent = None - - def overtake_parent_lookup(tip_hash: str) -> str: - self.assertEqual(tip_hash, old_tip) - with server.lock: - server.current_tip_first_seen = (new_tip, 100.0) - server.current_tip_observation_sequence = 2 - server.current_tip_parent = (new_tip, new_parent) - return old_parent - - server._fetch_tip_parent_hash = overtake_parent_lookup # type: ignore[method-assign] - - self.assertEqual(server.current_tip_parent_hash(old_tip), old_parent) - self.assertEqual(server.current_tip_parent, (new_tip, new_parent)) - - def test_retained_same_tip_duplicate_remains_duplicate_share(self) -> None: - tip = "00" * 32 - server, state, ledger = submit_coordinator(tip=tip) - server.current_tip_first_seen = (tip, None) - server.bury_evicted_job(state, "job-1") - server.jobs.pop("job-1") - state.active_job_ids.clear() - submission = SimpleNamespace( - header_hex="a3" * 80, - block_hash_hex="c3" * 32, - share_pass=True, - block_pass=False, - ) - params = ["miner-a", "job-1", "00" * 8, "00000001", "00000002"] - - with patch( - "lab.prism.prism_coordinator.direct_stratum.assemble_submission", - return_value=submission, - ): - self.assertFalse(server.handle_submit(state, params)) - with self.assertRaises(StratumError) as raised: - server.handle_submit(state, params) - - self.assertEqual(raised.exception.reason, PRISM_REJECTION_DUPLICATE_SHARE) - self.assertEqual(len(ledger.pending), 1) - self.assertIn("job-1", server.evicted_job_graveyard) - metrics = server.metrics_payload() - self.assertIn('qbit_prism_evicted_job_contexts{class="same_tip"} 1', metrics) - self.assertIn( - 'qbit_prism_evicted_job_submits_total{outcome="accepted_same_tip"} 1', - metrics, - ) - - def test_pool_closed_submit_rejects_before_any_share_accounting(self) -> None: - # Post-close submits must not inflate submitted totals (the - # stale-percent denominator), per-worker submitted counters, or the - # vardiff window; only the pool-closed rejection itself is recorded. - server, state, ledger = submit_coordinator() - server.accepted_block_count = 1 - server.max_blocks = 1 - state.vardiff_config = SimpleNamespace(enabled=True) - submitted_before = server.submitted_share_count - - with self.assertRaises(StratumError) as raised: - server.handle_submit( - state, - ["miner-a", "job-1", "00" * 8, "00000001", "00000002"], - ) - - self.assertEqual(raised.exception.reason, PRISM_REJECTION_POOL_CLOSED) - self.assertEqual(server.submitted_share_count, submitted_before) - # The rejection itself may admit the label, but no submission counted. - self.assertEqual(server.worker_share_counts["miner-a"]["submitted"], 0) - self.assertEqual(state.vardiff_window_submitted, 0) - self.assertEqual(len(ledger.pending), 0) - self.assertEqual( - server.worker_rejection_counts[("miner-a", PRISM_REJECTION_POOL_CLOSED)], 1 - ) - - def test_malformed_submit_does_not_diverge_worker_and_aggregate_submitted(self) -> None: - # A malformed-ntime submit must count identically in the per-worker and - # aggregate submitted counters (i.e. not at all) so the two never drift. - server, state, _ledger = submit_coordinator() - submitted_before = server.submitted_share_count - - with self.assertRaises(StratumError) as raised: - server.handle_submit( - state, - ["miner-a", "job-1", "00" * 8, "bad-ntime", "00000002"], - ) - - self.assertEqual(raised.exception.reason, PRISM_REJECTION_INVALID_NTIME_OR_NONCE) - self.assertEqual(server.submitted_share_count, submitted_before) - self.assertEqual(server.worker_share_counts["miner-a"]["submitted"], 0) - self.assertEqual( - server.worker_rejection_counts[("miner-a", PRISM_REJECTION_INVALID_NTIME_OR_NONCE)], - 1, - ) - - def test_stale_grace_closed_when_refresh_path_has_not_observed_tip(self) -> None: - # Only blockpoll/blockwait may open the grace window. If the refresh path - # has not anchored the new tip (current_tip_first_seen is None) and only - # this submit's getbestblockhash sees it, the prior-tip share must reject - # as stale-job -- not get credited from a submit-anchored window. - old_tip = "00" * 32 - new_tip = "11" * 32 - server, state, ledger = submit_coordinator(tip=old_tip) - rpc = ParentTipRpc(tip=new_tip, parent=old_tip) - server.rpc = rpc - server.current_tip_first_seen = None - server.stale_grace_seconds = 3 - submission = SimpleNamespace( - header_hex="aa" * 80, - block_hash_hex="cc" * 32, - share_pass=True, - block_pass=False, - ) - - with patch( - "lab.prism.prism_coordinator.direct_stratum.assemble_submission", - return_value=submission, - ): - with self.assertRaises(StratumError) as raised: - server.handle_submit( - state, - ["miner-a", "job-1", "00" * 8, "00000001", "00000002"], - ) - - self.assertEqual(raised.exception.reason, PRISM_REJECTION_STALE_JOB) - self.assertEqual(len(ledger.pending), 0) - self.assertEqual(rpc.submitblock_calls, 0) - # The submit must not have anchored the window either. - self.assertIsNone(server.current_tip_first_seen) - - def test_stale_grace_rejected_after_window_expires(self) -> None: - # This connection received current-tip work well outside the grace - # window; a prior-tip share arriving now must reject rather than be - # credited late. - old_tip = "00" * 32 - new_tip = "11" * 32 - server, state, ledger = submit_coordinator(tip=old_tip) - rpc = ParentTipRpc(tip=new_tip, parent=old_tip) - server.rpc = rpc - server.stale_grace_seconds = 3 - server.current_tip_first_seen = (new_tip, time.monotonic() - 10) - state.tip_work_delivered = (new_tip, time.monotonic() - 10) - submission = SimpleNamespace( - header_hex="ab" * 80, - block_hash_hex="ce" * 32, - share_pass=True, - block_pass=False, - ) - - with patch( - "lab.prism.prism_coordinator.direct_stratum.assemble_submission", - return_value=submission, - ): - with self.assertRaises(StratumError) as raised: - server.handle_submit( - state, - ["miner-a", "job-1", "00" * 8, "00000001", "00000002"], - ) - - self.assertEqual(raised.exception.reason, PRISM_REJECTION_STALE_JOB) - self.assertEqual(len(ledger.pending), 0) - self.assertEqual(rpc.submitblock_calls, 0) - - def test_stale_grace_open_until_connection_receives_new_tip_work(self) -> None: - # The refresh pass may be slow or aborted (reorg reconcile failure, - # transient build errors). Until THIS connection is sent current-tip - # work, its prior-tip shares are still in flight and must stay - # creditable even after the global first-seen stamp ages past the - # grace window. - old_tip = "00" * 32 - new_tip = "11" * 32 - server, state, ledger = submit_coordinator(tip=old_tip) - rpc = ParentTipRpc(tip=new_tip, parent=old_tip) - server.rpc = rpc - server.stale_grace_seconds = 3 - server.current_tip_first_seen = (new_tip, time.monotonic() - 10) - state.tip_work_delivered = (old_tip, time.monotonic() - 60) - submission = SimpleNamespace( - header_hex="ac" * 80, - block_hash_hex="cd" * 32, - share_pass=True, - block_pass=False, - ) - - with patch( - "lab.prism.prism_coordinator.direct_stratum.assemble_submission", - return_value=submission, - ): - should_close = server.handle_submit( - state, - ["miner-a", "job-1", "00" * 8, "00000001", "00000002"], - ) - - self.assertFalse(should_close) - self.assertEqual(rpc.submitblock_calls, 0) - self.assertEqual(len(ledger.pending), 1) - self.assertEqual(ledger.pending[0].credit_policy, PRISM_CREDIT_POLICY_STALE_GRACE) - - def test_stale_grace_window_runs_from_per_connection_delivery_not_first_seen(self) -> None: - # A slow refresh pass can deliver current-tip work to a connection - # after the global first-seen stamp has already aged past the grace - # window. The window for that connection runs from ITS delivery. - old_tip = "00" * 32 - new_tip = "11" * 32 - server, state, ledger = submit_coordinator(tip=old_tip) - rpc = ParentTipRpc(tip=new_tip, parent=old_tip) - server.rpc = rpc - server.stale_grace_seconds = 3 - server.current_tip_first_seen = (new_tip, time.monotonic() - 10) - state.tip_work_delivered = (new_tip, time.monotonic() - 1) - submission = SimpleNamespace( - header_hex="ae" * 80, - block_hash_hex="cb" * 32, - share_pass=True, - block_pass=False, - ) - - with patch( - "lab.prism.prism_coordinator.direct_stratum.assemble_submission", - return_value=submission, - ): - should_close = server.handle_submit( - state, - ["miner-a", "job-1", "00" * 8, "00000001", "00000002"], - ) - - self.assertFalse(should_close) - self.assertEqual(len(ledger.pending), 1) - self.assertEqual(ledger.pending[0].credit_policy, PRISM_CREDIT_POLICY_STALE_GRACE) - - def test_startup_baseline_tip_does_not_open_stale_grace_window(self) -> None: - # The first tip observed after process start is a baseline, not a tip - # flip: it must not open the grace window. A later real flip must. - old_tip = "00" * 32 - new_tip = "11" * 32 - server, state, _ledger = submit_coordinator(tip=old_tip) - server.stale_grace_seconds = 3 - server.current_tip_first_seen = None - - server.observe_tip_first_seen(new_tip) - self.assertEqual(server.current_tip_first_seen, (new_tip, None)) - self.assertFalse(server.stale_grace_deadline_open(state, new_tip)) - - # A change away from the observed baseline is a real flip and opens - # the window for connections that have not yet received the new work. - flip_tip = "22" * 32 - server.observe_tip_first_seen(flip_tip) - self.assertIsNotNone(server.current_tip_first_seen[1]) - self.assertTrue(server.stale_grace_deadline_open(state, flip_tip)) - - def test_note_tip_work_delivered_keeps_first_delivery_per_tip(self) -> None: - # Same-tip template refreshes must not slide the grace anchor forward. - server, state, _ledger = submit_coordinator() - tip = "11" * 32 - - server.note_tip_work_delivered(state, tip) - first = state.tip_work_delivered - self.assertEqual(first[0], tip) - server.note_tip_work_delivered(state, tip) - self.assertEqual(state.tip_work_delivered, first) - - # A new tip re-anchors. - server.note_tip_work_delivered(state, "22" * 32) - self.assertEqual(state.tip_work_delivered[0], "22" * 32) - self.assertGreaterEqual(state.tip_work_delivered[1], first[1]) - - def test_evicted_graveyard_keeps_unexpired_entries_above_previous_cap_for_grace_credit(self) -> None: - old_tip = "00" * 32 - new_tip = "11" * 32 - server, state, ledger = submit_coordinator(tip=old_tip) - rpc = ParentTipRpc(tip=new_tip, parent=old_tip) - server.rpc = rpc - server.current_tip_first_seen = (new_tip, time.monotonic()) - context = server.jobs["job-1"] - evicted_at = time.monotonic() - server.evicted_job_graveyard = { - "job-1": (context, state.connection_id, evicted_at), - } - previous_hard_cap = 512 - for index in range(previous_hard_cap): - server.evicted_job_graveyard[f"filler-{index}"] = ( - context, - state.connection_id, - evicted_at + 0.001 + (index / 1_000_000), - ) - server.prune_evicted_job_graveyard(now=evicted_at + 0.5) - self.assertIn("job-1", server.evicted_job_graveyard) - server.jobs.pop("job-1") - state.active_job_ids.clear() - submission = SimpleNamespace( - header_hex="ae" * 80, - block_hash_hex="ce" * 32, - share_pass=True, - block_pass=False, - ) - - with patch( - "lab.prism.prism_coordinator.direct_stratum.assemble_submission", - return_value=submission, - ): - should_close = server.handle_submit( - state, - ["miner-a", "job-1", "00" * 8, "00000001", "00000002"], - ) - - self.assertFalse(should_close) - self.assertEqual(len(ledger.pending), 1) - self.assertEqual(ledger.pending[0].credit_policy, PRISM_CREDIT_POLICY_STALE_GRACE) - - def test_stale_grace_parent_rpc_failure_rejects_as_backend_unavailable(self) -> None: - old_tip = "00" * 32 - new_tip = "11" * 32 - server, state, _ledger = submit_coordinator(tip=old_tip) - server.rpc = TipRpc(new_tip) - server.current_tip_first_seen = (new_tip, time.monotonic()) - - with self.assertRaises(StratumError) as raised: - server.handle_submit( - state, - ["miner-a", "job-1", "00" * 8, "00000001", "00000002"], - ) - - self.assertEqual(raised.exception.reason, PRISM_REJECTION_BACKEND_RPC_UNAVAILABLE) - - def test_unknown_job_rejects_before_getbestblockhash_rpc(self) -> None: - class CountingTipRpc(TipRpc): - def __init__(self, tip: str) -> None: - super().__init__(tip) - self.getbest_calls = 0 - - def call(self, method: str, params: list[object] | None = None) -> object: - if method == "getbestblockhash": - self.getbest_calls += 1 - return super().call(method, params) - - server, state, _ledger = submit_coordinator() - rpc = CountingTipRpc("00" * 32) - server.rpc = rpc - - with self.assertRaises(StratumError) as raised: - server.handle_submit( - state, - ["miner-a", "garbage-job", "00" * 8, "00000001", "00000002"], - ) - - self.assertEqual(raised.exception.reason, PRISM_REJECTION_UNKNOWN_JOB) - self.assertEqual(rpc.getbest_calls, 0) - - def test_submit_passes_negotiated_version_bits_and_mask_to_stratum_assembly(self) -> None: - server, state, _ledger = submit_coordinator() - state.version_mask = 0x1FFFE000 - submission = SimpleNamespace( - header_hex="ac" * 80, - block_hash_hex="ba" * 32, - share_pass=True, - block_pass=False, - ) - - with patch( - "lab.prism.prism_coordinator.direct_stratum.assemble_submission", - return_value=submission, - ) as assemble_submission: - server.handle_submit( - state, - ["miner-a", "job-1", "00" * 8, "00000001", "00000002", "00002000"], - ) - - self.assertEqual(assemble_submission.call_args.kwargs["version_bits_hex"], "00002000") - self.assertEqual(assemble_submission.call_args.kwargs["version_mask"], 0x1FFFE000) - - def test_address_worker_submit_accrues_to_base_payout_address(self) -> None: - server, state, ledger = submit_coordinator() - username = f"{PAYOUT_ADDRESS}.rig-a" - worker = WorkerIdentity( - username=username, - payout_address=PAYOUT_ADDRESS, - worker_name="rig-a", - script_pubkey_hex="5220" + "44" * 32, - p2mr_program_hex="44" * 32, - ) - state.username = username - state.worker = worker - server.jobs["job-1"].worker = worker - server.share_weights_by_username = {PAYOUT_ADDRESS: 9} - submission = SimpleNamespace( - header_hex="aa" * 80, - block_hash_hex="bc" * 32, - share_pass=True, - block_pass=False, - ) - - with patch( - "lab.prism.prism_coordinator.direct_stratum.assemble_submission", - return_value=submission, - ): - should_close = server.handle_submit( - state, - [username, "job-1", "00" * 8, "00000001", "00000002"], - ) - - self.assertFalse(should_close) - self.assertEqual(len(ledger.pending), 1) - self.assertEqual(ledger.pending[0].share_id, username + ":" + "bc" * 32) - self.assertEqual(ledger.pending[0].miner_id, PAYOUT_ADDRESS) - self.assertEqual(ledger.pending[0].order_key, PAYOUT_ADDRESS) - self.assertEqual(ledger.pending[0].share_difficulty, 9) - - def test_address_worker_submit_still_requires_authorized_full_username(self) -> None: - server, state, ledger = submit_coordinator() - username = f"{PAYOUT_ADDRESS}.rig-a" - worker = WorkerIdentity( - username=username, - payout_address=PAYOUT_ADDRESS, - worker_name="rig-a", - script_pubkey_hex="5220" + "44" * 32, - p2mr_program_hex="44" * 32, - ) - state.username = username - state.worker = worker - server.jobs["job-1"].worker = worker - - with self.assertRaises(StratumError) as raised: - server.handle_submit( - state, - [PAYOUT_ADDRESS, "job-1", "00" * 8, "00000001", "00000002"], - ) - - self.assertEqual(raised.exception.code, 20) - self.assertEqual(len(ledger.pending), 0) - - def test_stale_tip_rejects_without_appending_share(self) -> None: - old_tip = "00" * 32 - new_tip = "11" * 32 - server, state, ledger = submit_coordinator(tip=old_tip) - server.rpc = ParentTipRpc(tip=new_tip, parent="22" * 32) - - with self.assertRaises(StratumError) as raised: - server.handle_submit( - state, - ["miner-a", "job-1", "00" * 8, "00000001", "00000002"], - ) - - self.assertEqual(raised.exception.code, 21) - self.assertEqual(raised.exception.reason, PRISM_REJECTION_STALE_JOB) - self.assertEqual(len(ledger.pending), 0) - self.assertEqual(server.stale_share_count, 1) - self.assertEqual(server.rejection_counts_by_reason[PRISM_REJECTION_STALE_JOB], 1) - - def _pending_append(self, tag: str, accepted_at_ms: int = 2) -> PendingShareAppend: - from lab.prism.share_ledger import PendingShare - - return PendingShareAppend( - pending_share=PendingShare( - share_id=f"miner-a:{tag}", - miner_id="miner-a", - order_key="miner-a", - p2mr_program_hex="11" * 32, - share_difficulty=1, - network_difficulty=1, - template_height=10, - job_id="job-1", - job_issued_at_ms=1, - accepted_at_ms=accepted_at_ms, - ntime=1_700_000_000, - ), - username="miner-a", - job_id="job-1", - block_hash_hex=tag * 32, - collection_only=False, - credit_policy=None, - ) - - def test_reconnected_same_username_submits_against_disconnected_job(self) -> None: - # A proxy flap leaves devices mining the dead connection's jobs; - # their shares arrive on the replacement connection and must credit - # against the retained context instead of rejecting unknown-job. - tip = "00" * 32 - server, state, ledger = submit_coordinator(tip=tip) - server.clients = {state} - state.close = lambda: None # type: ignore[method-assign] - server.disconnect_client(state) - retained = server.evicted_job_graveyard.get("job-1") - self.assertIsNotNone(retained) - assert retained is not None - self.assertIsNone(retained.client) - - reconnected = ClientState( - sock=object(), - address=("127.0.0.1", 2), - connection_id=2, - extranonce1_hex="00000002", - ) - reconnected.subscribed = True - reconnected.authorized = True - reconnected.username = "miner-a" - reconnected.worker = worker_identity("miner-a") - submission = SimpleNamespace( - header_hex="af" * 80, - block_hash_hex="cf" * 32, - share_pass=True, - block_pass=False, - ) - - with patch( - "lab.prism.prism_coordinator.direct_stratum.assemble_submission", - return_value=submission, - ): - should_close = server.handle_submit( - reconnected, - ["miner-a", "job-1", "00" * 8, "00000001", "00000002"], - ) - - self.assertFalse(should_close) - self.assertEqual(len(ledger.pending), 1) - self.assertIsNone(ledger.pending[0].credit_policy) - # Credit follows the retained context's original worker identity. - self.assertEqual(ledger.pending[0].share_id, "miner-a:" + "cf" * 32) - self.assertEqual( - server.evicted_job_submit_counts[ - "accepted_same_tip_cross_connection" - ], - 1, - ) - - def test_cross_connection_submit_requires_same_username(self) -> None: - tip = "00" * 32 - server, state, _ledger = submit_coordinator(tip=tip) - server.clients = {state} - state.close = lambda: None # type: ignore[method-assign] - server.disconnect_client(state) - self.assertIn("job-1", server.evicted_job_graveyard) - - intruder = ClientState( - sock=object(), - address=("127.0.0.1", 3), - connection_id=3, - extranonce1_hex="00000003", - ) - intruder.subscribed = True - intruder.authorized = True - intruder.username = "miner-b" - intruder.worker = worker_identity("miner-b") - submission = SimpleNamespace( - header_hex="af" * 80, - block_hash_hex="cf" * 32, - share_pass=True, - block_pass=False, - ) - - with patch( - "lab.prism.prism_coordinator.direct_stratum.assemble_submission", - return_value=submission, - ): - with self.assertRaises(StratumError) as raised: - server.handle_submit( - intruder, - ["miner-b", "job-1", "00" * 8, "00000001", "00000002"], - ) - - self.assertEqual(raised.exception.reason, PRISM_REJECTION_UNKNOWN_JOB) - - def test_cross_connection_submit_uses_retained_version_mask(self) -> None: - # In-flight work from the dead connection was version-rolled under - # the mask negotiated there; the replacement's own mask (0 before - # mining.configure) must not judge those version bits. - tip = "00" * 32 - server, state, _ledger = submit_coordinator(tip=tip) - server.jobs["job-1"].version_mask = 0x1FFFE000 - server.clients = {state} - state.close = lambda: None # type: ignore[method-assign] - server.disconnect_client(state) - - reconnected = ClientState( - sock=object(), - address=("127.0.0.1", 2), - connection_id=2, - extranonce1_hex="00000002", - ) - reconnected.subscribed = True - reconnected.authorized = True - reconnected.username = "miner-a" - reconnected.worker = worker_identity("miner-a") - self.assertEqual(reconnected.version_mask, 0) - submission = SimpleNamespace( - header_hex="af" * 80, - block_hash_hex="cf" * 32, - share_pass=True, - block_pass=False, - ) - - with patch( - "lab.prism.prism_coordinator.direct_stratum.assemble_submission", - return_value=submission, - ) as assemble: - self.assertFalse( - server.handle_submit( - reconnected, - [ - "miner-a", - "job-1", - "00" * 8, - "00000001", - "00000002", - "1fffe000", - ], - ) - ) - - self.assertEqual( - assemble.call_args.kwargs["version_mask"], - 0x1FFFE000, - ) - - def test_cross_connection_block_candidate_keeps_original_extranonce(self) -> None: - # The mined coinbase embeds the extranonce1 the retained job was - # stamped with; a candidate recording the replacement connection's - # extranonce would fail the audit fence after submitblock. - tip = "00" * 32 - server, state, _ledger = submit_coordinator(tip=tip) - server.stop_after_block = False - server.max_blocks = 10 - server.jobs["job-1"].job.extranonce1_hex = state.extranonce1_hex - server.clients = {state} - state.close = lambda: None # type: ignore[method-assign] - server.disconnect_client(state) - - reconnected = ClientState( - sock=object(), - address=("127.0.0.1", 2), - connection_id=2, - extranonce1_hex="00000002", - ) - reconnected.subscribed = True - reconnected.authorized = True - reconnected.username = "miner-a" - reconnected.worker = worker_identity("miner-a") - submission = SimpleNamespace( - header_hex="af" * 80, - block_hash_hex="cf" * 32, - share_pass=True, - block_pass=True, - coinbase_tx_hex="c0ffee", - block_hex="00", - ) - - with patch( - "lab.prism.prism_coordinator.direct_stratum.assemble_submission", - return_value=submission, - ): - self.assertFalse( - server.handle_submit( - reconnected, - ["miner-a", "job-1", "00" * 8, "00000001", "00000002"], - ) - ) - - self.assertEqual(server.block_candidate_queue.qsize(), 1) - candidate = server.block_candidate_queue.get_nowait() - self.assertEqual(candidate.extranonce1_hex, "00000001") - self.assertIs(candidate.client, reconnected) - - def test_detached_entry_never_falls_back_to_stale_grace(self) -> None: - # A tip moving between the graveyard lookup and submit - # classification must not let a detached cross-connection entry - # borrow the reconnect's stale-grace window: cross-connection - # resumes are same-tip only. - tip_old = "00" * 32 - tip_new = "11" * 32 - server, state, _ledger = submit_coordinator(tip=tip_old) - server.clients = {state} - state.close = lambda: None # type: ignore[method-assign] - server.disconnect_client(state) - detached = server.evicted_job_graveyard.get("job-1") - self.assertIsNotNone(detached) - assert detached is not None - self.assertIsNone(detached.client) - - reconnected = ClientState( - sock=object(), - address=("127.0.0.1", 2), - connection_id=2, - extranonce1_hex="00000002", - ) - reconnected.subscribed = True - reconnected.authorized = True - reconnected.username = "miner-a" - reconnected.worker = worker_identity("miner-a") - eligibility_calls: list[str] = [] - - def recording_eligibility( - _client: ClientState, - _context: object, - current_tip: str, - ) -> bool: - eligibility_calls.append(current_tip) - return True - - server.context_eligible_for_stale_grace = ( # type: ignore[method-assign] - recording_eligibility - ) - - self.assertIsNone( - server.evicted_submit_context(reconnected, detached, tip_new) - ) - self.assertEqual(eligibility_calls, []) - - # Same-connection entries keep today's stale-grace classification. - live_state = client() - server.jobs["job-2"] = prism_context( - "job-2", - tip_old, - worker=live_state.worker, - ) - server.bury_evicted_job(live_state, "job-2") - live_entry = server.evicted_job_graveyard["job-2"] - result = server.evicted_submit_context(live_state, live_entry, tip_new) - self.assertIsNotNone(result) - assert result is not None - self.assertEqual(result[1], PRISM_CREDIT_POLICY_STALE_GRACE) - self.assertEqual(eligibility_calls, [tip_new]) - - def test_disconnected_retention_capacity_is_bounded(self) -> None: - tip = "00" * 32 - server, state, _ledger = submit_coordinator(tip=tip) - server.disconnected_job_retention = 1 - server.jobs["job-2"] = prism_context("job-2", tip, worker=state.worker) - state.active_job_ids = {"job-1", "job-2"} - server.clients = {state} - state.close = lambda: None # type: ignore[method-assign] - - server.disconnect_client(state) - - self.assertEqual(len(server._disconnected_evicted_job_ids), 1) - self.assertEqual(len(server.evicted_job_graveyard), 1) - self.assertEqual( - server.evicted_job_capacity_eviction_counts["disconnected"], - 1, - ) - - def test_tip_change_prunes_and_disconnect_detaches_retained_contexts(self) -> None: - old_tip = "00" * 32 - new_tip = "11" * 32 - server, state, _ledger = submit_coordinator(tip=old_tip) - server.current_tip_first_seen = (old_tip, None) - server.stale_grace_seconds = 0 - server.bury_evicted_job(state, "job-1") - self.assertIn("job-1", server.evicted_job_graveyard) - - server.observe_tip_first_seen(new_tip) - self.assertNotIn("job-1", server.evicted_job_graveyard) - - server.current_tip_first_seen = (new_tip, None) - server.jobs["job-2"] = prism_context("job-2", new_tip, worker=state.worker) - server.bury_evicted_job(state, "job-2") - server.clients = {state} - state.close = lambda: None # type: ignore[method-assign] - server.disconnect_client(state) - # Same-tip work survives the disconnect for a same-username - # reconnect; the dead connection's entry is detached and ages out on - # the normal same-tip TTL instead of vanishing with the socket. - retained = server.evicted_job_graveyard.get("job-2") - self.assertIsNotNone(retained) - assert retained is not None - self.assertIsNone(retained.client) - server.prune_evicted_job_graveyard( - now=time.monotonic() - + float(server.same_tip_job_retention_seconds) - + 1.0 - ) - self.assertEqual(server.evicted_job_graveyard, {}) diff --git a/tests/test_prism_reward_window_invariant.py b/tests/test_prism_reward_window_invariant.py deleted file mode 100644 index 6afa2025b..000000000 --- a/tests/test_prism_reward_window_invariant.py +++ /dev/null @@ -1,87 +0,0 @@ -#!/usr/bin/env python3 - -from __future__ import annotations - -import re -import unittest -from pathlib import Path - - -ROOT = Path(__file__).resolve().parents[1] -COORDINATOR = ROOT / "lab" / "prism" / "prism_coordinator.py" -SHARE_LEDGER = ROOT / "lab" / "prism" / "share_ledger.py" -RUST_LIBRARY = ROOT / "crates" / "qbit-prism" / "src" / "lib.rs" -LEDGER_SCHEMA = ROOT / "crates" / "qbit-prism" / "sql" / "001_share_ledger.sql" - - -class PrismRewardWindowInvariantTests(unittest.TestCase): - def test_reward_window_multiplier_is_synchronized(self) -> None: - coordinator = COORDINATOR.read_text(encoding="utf-8") - share_ledger = SHARE_LEDGER.read_text(encoding="utf-8") - rust_library = RUST_LIBRARY.read_text(encoding="utf-8") - ledger_schema = LEDGER_SCHEMA.read_text(encoding="utf-8") - - python_multiplier = self._one( - r"^PRISM_REWARD_WINDOW_MULTIPLIER\s*=\s*(\d+)\s*$", - coordinator, - "Python coordinator multiplier", - ) - rust_multiplier = self._one( - r"^pub const PRISM_WINDOW_MULTIPLIER:\s*u128\s*=\s*(\d+);\s*$", - rust_library, - "Rust multiplier", - ) - dashboard_multiplier = self._one( - r"return difficulty \* Decimal\((\d+)\)", - share_ledger, - "Python dashboard multiplier", - ) - dashboard_sql_multiplier = self._one( - r"audit_network_difficulty::numeric \* (\d+)::numeric", - share_ledger, - "Python dashboard SQL multiplier", - ) - - audit_function = self._one_text( - r"CREATE OR REPLACE FUNCTION qbit_audit_share_window\(.*?\n\$\$;", - ledger_schema, - "SQL audit-window function", - ) - sql_multipliers = [ - self._one(r"SELECT\s+(\d+)::numeric AS window_multiplier", audit_function, "SQL metadata multiplier"), - self._one( - r"network_difficulty \* (\d+)::numeric AS requested_window_weight", - audit_function, - "SQL requested-weight multiplier", - ), - self._one( - r"qbit_prism_window\(anchor_job_issued_at, network_difficulty \* (\d+)::numeric\)", - audit_function, - "SQL selection multiplier", - ), - ] - - self.assertEqual(python_multiplier, 8) - self.assertEqual( - { - rust_multiplier, - dashboard_multiplier, - dashboard_sql_multiplier, - *sql_multipliers, - }, - {python_multiplier}, - ) - - def _one(self, pattern: str, source: str, label: str) -> int: - matches = re.findall(pattern, source, flags=re.MULTILINE | re.DOTALL) - self.assertEqual(len(matches), 1, f"expected exactly one {label}, found {matches!r}") - return int(matches[0]) - - def _one_text(self, pattern: str, source: str, label: str) -> str: - matches = re.findall(pattern, source, flags=re.MULTILINE | re.DOTALL) - self.assertEqual(len(matches), 1, f"expected exactly one {label}, found {len(matches)}") - return matches[0] - - -if __name__ == "__main__": - unittest.main() diff --git a/tests/test_prism_self_check.py b/tests/test_prism_self_check.py deleted file mode 100644 index bbb05b9a9..000000000 --- a/tests/test_prism_self_check.py +++ /dev/null @@ -1,1202 +0,0 @@ -#!/usr/bin/env python3 - -from __future__ import annotations - -import importlib.util -import json -import os -import socket -import sys -import tempfile -import threading -import time -import unittest -from pathlib import Path -from types import ModuleType -from unittest.mock import patch - - -def load_self_check_module() -> ModuleType: - path = Path(__file__).resolve().parents[1] / "scripts" / "prism-self-check.py" - spec = importlib.util.spec_from_file_location("prism_self_check", path) - assert spec is not None - module = importlib.util.module_from_spec(spec) - assert spec.loader is not None - sys.modules[spec.name] = module - spec.loader.exec_module(module) - return module - - -def authorize_mainnet_prelaunch(env: dict[str, str]) -> dict[str, str]: - env.update( - { - "QBIT_CHAIN": "mainnet", - "QBIT_CHAIN_FLAG": "-chain=main", - "QBIT_EXPECTED_GENESIS_HASH": "11" * 32, - "QBIT_PRODUCTION": "1", - "QBIT_TOOLS_PRODUCTION": "1", - "CKPOOL_NON_TEST_READINESS_GATE": "0", - "QBIT_MAINNET_LAUNCH_READINESS_CHECKS_ENABLED": "0", - "PRISM_POSTGRES_PASSWORD": "not-default", - "PRISM_STRATUM_STALE_GRACE_SECONDS": "3", - } - ) - return env - - -class PrismSelfCheckTests(unittest.TestCase): - @classmethod - def setUpClass(cls) -> None: - cls.self_check = load_self_check_module() - - def valid_env(self) -> dict[str, str]: - return { - "QBIT_CHAIN": "signet", - "QBIT_CHAIN_FLAG": "-signet", - "QBIT_RPC_USER": "qbitrpc", - "QBIT_RPC_PASSWORD": "not-default", - "PRISM_MANIFEST_SIGNING_SEED_HEX": "11" * 32, - "PRISM_LEDGER_ATTESTATION_SIGNING_SEED_HEX": "22" * 32, - "PRISM_LEDGER_WRITER_PUBLIC_KEY_HEX": "33" * 32, - "PRISM_LEDGER_WRITER_ID": "prism-coordinator", - "PRISM_LEDGER_WRITER_EPOCH": "1", - "PRISM_DATABASE_URL": "postgresql://qbit:secret@prism-postgres:5432/qbit", - "PRISM_AUDIT_DIR": "/var/lib/qbit-prism/audit", - "PRISM_EVIDENCE_PATH": "/var/lib/qbit-prism/audit/prism-live-evidence.json", - "PRISM_BLOCKWAIT_ENABLED": "1", - "PRISM_BLOCKWAIT_TIMEOUT_SECONDS": "5", - "PRISM_ALLOW_MEMORY_LEDGER": "0", - "PRISM_ALLOW_TEST_SIGNING_SEEDS": "0", - "PRISM_ALLOW_BUNDLE_EMBEDDED_LEDGER_KEY": "0", - "PRISM_ALLOW_FIXED_LEDGER_SESSION_TOKEN": "0", - "PRISM_MIN_READY_MINERS": "3", - "PRISM_STRATUM_SHARE_DIFF": "0.000000001", - "PRISM_STRATUM_VARDIFF": "1", - "PRISM_STRATUM_VARDIFF_MIN_DIFF": "0.000000001", - "PRISM_STRATUM_VARDIFF_MAX_DIFF": "1024", - "PRISM_STRATUM_VARDIFF_START_DIFF": "0.000000001", - "PRISM_STRATUM_VARDIFF_IDLE_SWEEP_SECONDS": "15", - "PRISM_STRATUM_STALE_GRACE_SECONDS": "3", - "PRISM_WORKER_METRICS_LIMIT": "100", - "PRISM_POOL_FEE_ENABLED": "0", - } - - @staticmethod - def public_blockchain_info(chain: str) -> dict[str, object]: - return { - "chain": chain, - "initialblockdownload": False, - "blocks": 100, - "headers": 100, - } - - @staticmethod - def fresh_template() -> dict[str, object]: - return {"previousblockhash": "11" * 32, "curtime": int(time.time())} - - def mainnet_live_reporter( - self, - env: dict[str, str], - *, - actual_chain: str = "main", - initial_block_download: bool = False, - peers: int = 2, - ) -> object: - reporter = self.self_check.Reporter() - - def fake_qbit_rpc_call( - rpc_env: dict[str, str], - method: str, - params: list[object] | None = None, - ) -> object: - if method == "getblockchaininfo": - info = self.public_blockchain_info(actual_chain) - info["initialblockdownload"] = initial_block_download - return info - if method == "getblocktemplate": - return self.fresh_template() - if method == "getblockhash": - return rpc_env["QBIT_EXPECTED_GENESIS_HASH"] - if method == "getnetworkinfo": - return {"connections": peers} - raise AssertionError(method) - - with patch.object(self.self_check, "qbit_rpc_call", fake_qbit_rpc_call): - self.self_check.qbit_live_checks(env, reporter) - return reporter - - def test_parse_host_port_accepts_port_only_and_host_port(self) -> None: - self.assertEqual( - self.self_check.parse_host_port("3340", default_host="127.0.0.1", default_port=3340), - ("127.0.0.1", 3340), - ) - self.assertEqual( - self.self_check.parse_host_port("0.0.0.0:43340", default_host="127.0.0.1", default_port=3340), - ("0.0.0.0", 43340), - ) - - def test_parse_decimal_rejects_non_finite_values(self) -> None: - for value in ("NaN", "Infinity", "-Infinity"): - with self.subTest(value=value): - with self.assertRaisesRegex(ValueError, "finite decimal number"): - self.self_check.parse_decimal(value) - - def test_compose_commands_include_prism_profile(self) -> None: - command = self.self_check.compose_base_command() - - self.assertIn("--profile", command) - self.assertEqual(command[command.index("--profile") + 1], "prism") - - def test_deployment_env_replaces_repository_env_in_compose_commands(self) -> None: - with tempfile.TemporaryDirectory() as temp_dir: - root = Path(temp_dir) - (root / "config").mkdir() - upstream_env = root / "config" / "upstream.env" - upstream_env.write_text("UPSTREAM=1\n", encoding="utf-8") - local_env = root / ".env" - local_env.write_text("LOCAL=1\n", encoding="utf-8") - deploy_env = root / "mainnet.env" - deploy_env.write_text("DEPLOYMENT=1\n", encoding="utf-8") - - with ( - patch.object(self.self_check, "ROOT_DIR", root), - patch.dict(os.environ, {"DEPLOY_ENV_FILE": str(deploy_env)}, clear=False), - ): - deploy_args = self.self_check.env_file_args() - with ( - patch.object(self.self_check, "ROOT_DIR", root), - patch.dict(os.environ, {"DEPLOY_ENV_FILE": ""}, clear=False), - ): - local_args = self.self_check.env_file_args() - - self.assertEqual(deploy_args, ["--env-file", str(upstream_env), "--env-file", str(deploy_env)]) - self.assertNotIn(str(local_env), deploy_args) - self.assertEqual(local_args, ["--env-file", str(upstream_env), "--env-file", str(local_env)]) - - def test_env_value_prefers_compose_resolved_value_over_host_env(self) -> None: - with patch.dict(os.environ, {"QBIT_CHAIN": "mainnet"}): - self.assertEqual( - self.self_check.env_value({"QBIT_CHAIN": "signet"}, "QBIT_CHAIN", "regtest"), - "signet", - ) - self.assertEqual( - self.self_check.env_value({}, "QBIT_CHAIN", "regtest"), - "regtest", - ) - - def test_missing_launch_readiness_flag_defaults_to_strict(self) -> None: - self.assertTrue(self.self_check.launch_readiness_checks_enabled({"QBIT_CHAIN": "mainnet"})) - - def test_static_checks_fail_malformed_launch_readiness_flag(self) -> None: - env = self.valid_env() - env["QBIT_CHAIN"] = "mainnet" - env["QBIT_CHAIN_FLAG"] = "" - env["QBIT_MAINNET_LAUNCH_READINESS_CHECKS_ENABLED"] = "prelaunch" - reporter = self.self_check.Reporter() - - self.self_check.static_checks(env, reporter) - - self.assertIn( - ("FAIL", "env.QBIT_MAINNET_LAUNCH_READINESS_CHECKS_ENABLED"), - {(row.status, row.name) for row in reporter.rows}, - ) - - def test_static_checks_require_full_prelaunch_authorization(self) -> None: - cases = ( - ("QBIT_PRODUCTION", "0"), - ("QBIT_TOOLS_PRODUCTION", "0"), - ("CKPOOL_NON_TEST_READINESS_GATE", "1"), - ) - for name, value in cases: - with self.subTest(name=name): - env = authorize_mainnet_prelaunch(self.valid_env()) - env[name] = value - reporter = self.self_check.Reporter() - - self.self_check.static_checks(env, reporter) - - self.assertIn( - ("FAIL", "launch.readiness"), - {(row.status, row.name) for row in reporter.rows}, - ) - launch_failure = next( - row for row in reporter.rows if row.name == "launch.readiness" - ) - self.assertIn("requires QBIT_CHAIN=mainnet", launch_failure.detail) - - def test_whitespace_around_launch_flag_preserves_prelaunch_authorization(self) -> None: - env = authorize_mainnet_prelaunch(self.valid_env()) - env["QBIT_MAINNET_LAUNCH_READINESS_CHECKS_ENABLED"] = " 0\t" - reporter = self.self_check.Reporter() - - self.self_check.static_checks(env, reporter) - - launch_rows = [row for row in reporter.rows if row.name == "launch.readiness"] - self.assertEqual([row.status for row in launch_rows], ["WARN"]) - - def test_static_checks_validate_prelaunch_tip_age(self) -> None: - env = authorize_mainnet_prelaunch(self.valid_env()) - env["QBIT_MAINNET_PRELAUNCH_MAX_TIP_AGE_SECONDS"] = "000456789" - reporter = self.self_check.Reporter() - - self.self_check.static_checks(env, reporter) - - rows = [ - row - for row in reporter.rows - if row.name == "env.QBIT_MAINNET_PRELAUNCH_MAX_TIP_AGE_SECONDS" - ] - self.assertEqual([row.status for row in rows], ["PASS"]) - self.assertIn("456789 seconds", rows[0].detail) - - def test_static_checks_reject_invalid_prelaunch_tip_age(self) -> None: - invalid_values = ("", "0", "-1", "1;echo injected", "9223372036854775808") - for value in invalid_values: - with self.subTest(value=value): - env = authorize_mainnet_prelaunch(self.valid_env()) - env["QBIT_MAINNET_PRELAUNCH_MAX_TIP_AGE_SECONDS"] = value - reporter = self.self_check.Reporter() - - self.self_check.static_checks(env, reporter) - - self.assertIn( - ("FAIL", "env.QBIT_MAINNET_PRELAUNCH_MAX_TIP_AGE_SECONDS"), - {(row.status, row.name) for row in reporter.rows}, - ) - - def test_static_checks_reject_tip_age_for_incompatible_mode(self) -> None: - cases = ( - {"QBIT_PRODUCTION": "0", "QBIT_CHAIN": "mainnet", "QBIT_CHAIN_FLAG": ""}, - {"QBIT_PRODUCTION": "1", "QBIT_CHAIN": "signet", "QBIT_CHAIN_FLAG": "-signet"}, - ) - for overrides in cases: - with self.subTest(overrides=overrides): - env = self.valid_env() - env.update(overrides) - env["QBIT_MAINNET_PRELAUNCH_MAX_TIP_AGE_SECONDS"] = "456789" - reporter = self.self_check.Reporter() - - self.self_check.static_checks(env, reporter) - - self.assertIn( - ("FAIL", "env.QBIT_MAINNET_PRELAUNCH_MAX_TIP_AGE_SECONDS"), - {(row.status, row.name) for row in reporter.rows}, - ) - - def test_static_checks_accept_valid_prism_operator_env(self) -> None: - reporter = self.self_check.Reporter() - - self.self_check.static_checks(self.valid_env(), reporter) - - self.assertFalse(reporter.failed) - - def test_static_checks_fail_testnet_chain_flag_mismatch(self) -> None: - env = self.valid_env() - env["QBIT_CHAIN"] = "testnet" - env["QBIT_CHAIN_FLAG"] = "-regtest" - reporter = self.self_check.Reporter() - - self.self_check.static_checks(env, reporter) - - self.assertTrue(reporter.failed) - self.assertIn( - ("FAIL", "qbit.chain_flag"), - {(row.status, row.name) for row in reporter.rows}, - ) - - def test_static_checks_fail_bitcoin_chain_flag_mismatch(self) -> None: - env = self.valid_env() - env.update({"BITCOIN_CHAIN": "mainnet", "BITCOIN_CHAIN_FLAG": "-server=1"}) - reporter = self.self_check.Reporter() - - self.self_check.static_checks(env, reporter) - - self.assertIn( - ("FAIL", "bitcoin.chain_flag"), - {(row.status, row.name) for row in reporter.rows}, - ) - - def test_static_checks_accept_canonical_bitcoin_chain_flags(self) -> None: - cases = { - "regtest": "-regtest", - "testnet": "-testnet", - "testnet3": "-testnet", - "testnet4": "-testnet4", - "signet": "-signet", - "mainnet": "-chain=main", - } - for bitcoin_chain, chain_flag in cases.items(): - with self.subTest(bitcoin_chain=bitcoin_chain): - env = self.valid_env() - env.update({"BITCOIN_CHAIN": bitcoin_chain, "BITCOIN_CHAIN_FLAG": chain_flag}) - reporter = self.self_check.Reporter() - - self.self_check.static_checks(env, reporter) - - self.assertIn( - ("PASS", "bitcoin.chain_flag"), - {(row.status, row.name) for row in reporter.rows}, - ) - - def test_static_checks_fail_production_test_bypass(self) -> None: - env = self.valid_env() - env["QBIT_PRODUCTION"] = "1" - env["PRISM_ALLOW_TEST_SIGNING_SEEDS"] = "1" - reporter = self.self_check.Reporter() - - self.self_check.static_checks(env, reporter) - - self.assertTrue(reporter.failed) - self.assertIn( - ("FAIL", "env.PRISM_ALLOW_TEST_SIGNING_SEEDS"), - {(row.status, row.name) for row in reporter.rows}, - ) - - def test_mainnet_implies_production_safeguards(self) -> None: - env = self.valid_env() - env.update( - { - "QBIT_CHAIN": "mainnet", - "QBIT_CHAIN_FLAG": "-chain=main", - "QBIT_EXPECTED_GENESIS_HASH": "11" * 32, - "QBIT_GIT_COMMIT": "41" * 20, - "PRISM_STRATUM_STALE_GRACE_SECONDS": "3", - "PRISM_ALLOW_TEST_SIGNING_SEEDS": "1", - } - ) - reporter = self.self_check.Reporter() - - self.self_check.static_checks(env, reporter) - - self.assertTrue(reporter.failed) - self.assertIn( - ("FAIL", "env.PRISM_ALLOW_TEST_SIGNING_SEEDS"), - {(row.status, row.name) for row in reporter.rows}, - ) - - def test_static_checks_require_mainnet_chain_selector_and_genesis(self) -> None: - for chain_flag in ("", "-server=1", "-regtest", "-testnet4"): - with self.subTest(chain_flag=chain_flag): - env = self.valid_env() - env.update({"QBIT_CHAIN": "mainnet", "QBIT_CHAIN_FLAG": chain_flag}) - reporter = self.self_check.Reporter() - - self.self_check.static_checks(env, reporter) - - failures = {(row.status, row.name) for row in reporter.rows} - self.assertIn(("FAIL", "qbit.chain_flag"), failures) - self.assertIn(("FAIL", "qbit.genesis_config"), failures) - - def test_static_checks_accept_mainnet_chain_selector_and_genesis(self) -> None: - env = self.valid_env() - env.update( - { - "QBIT_CHAIN": "mainnet", - "QBIT_CHAIN_FLAG": "-chain=main", - "QBIT_EXPECTED_GENESIS_HASH": "AB" * 32, - "QBIT_GIT_COMMIT": "41" * 20, - "PRISM_STRATUM_STALE_GRACE_SECONDS": "3", - } - ) - reporter = self.self_check.Reporter() - - self.self_check.static_checks(env, reporter) - - self.assertNotIn( - ("FAIL", "qbit.chain_flag"), - {(row.status, row.name) for row in reporter.rows}, - ) - self.assertIn( - ("PASS", "qbit.genesis_config"), - {(row.status, row.name) for row in reporter.rows}, - ) - - def test_static_checks_require_immutable_production_git_commit(self) -> None: - for commit in ("", "main", "11" * 19, "zz" * 20): - with self.subTest(commit=commit): - env = self.valid_env() - env.update( - { - "QBIT_PRODUCTION": "1", - "QBIT_GIT_COMMIT": commit, - "PRISM_STRATUM_STALE_GRACE_SECONDS": "3", - } - ) - reporter = self.self_check.Reporter() - - self.self_check.static_checks(env, reporter) - - self.assertIn( - ("FAIL", "qbit.source_pin"), - {(row.status, row.name) for row in reporter.rows}, - ) - - def test_static_checks_accept_immutable_production_git_commit(self) -> None: - env = self.valid_env() - env.update( - { - "QBIT_PRODUCTION": "1", - "QBIT_GIT_COMMIT": "AB" * 20, - "PRISM_STRATUM_STALE_GRACE_SECONDS": "3", - } - ) - reporter = self.self_check.Reporter() - - self.self_check.static_checks(env, reporter) - - self.assertIn( - ("PASS", "qbit.source_pin"), - {(row.status, row.name) for row in reporter.rows}, - ) - self.assertNotIn( - ("FAIL", "mining.stale_grace"), - {(row.status, row.name) for row in reporter.rows}, - ) - - def test_static_checks_do_not_include_capacity_evidence(self) -> None: - env = self.valid_env() - env.update( - { - "QBIT_PRODUCTION": "1", - "QBIT_GIT_COMMIT": "41" * 20, - "PRISM_POSTGRES_PASSWORD": "not-default", - "PRISM_STRATUM_STALE_GRACE_SECONDS": "3", - "PRISM_STRATUM_SHARE_DIFF": "1024", - "PRISM_STRATUM_VARDIFF_MIN_DIFF": "1024", - "PRISM_STRATUM_VARDIFF_START_DIFF": "4096", - "PRISM_STRATUM_VARDIFF_MAX_DIFF": "65536", - } - ) - reporter = self.self_check.Reporter() - - self.self_check.static_checks(env, reporter) - - rows = {(row.status, row.name) for row in reporter.rows} - self.assertIn(("PASS", "mining.production_difficulty"), rows) - self.assertFalse(any(name == "capacity.evidence" for _status, name in rows)) - - def test_static_checks_mainnet_does_not_require_capacity_evidence(self) -> None: - env = self.valid_env() - env.update( - { - "QBIT_CHAIN": "mainnet", - "QBIT_CHAIN_FLAG": "-chain=main", - "QBIT_EXPECTED_GENESIS_HASH": "AB" * 32, - "QBIT_GIT_COMMIT": "41" * 20, - "QBIT_REQUIRE_RELEASE_PROVENANCE": "0", - "PRISM_STRATUM_STALE_GRACE_SECONDS": "3", - "PRISM_STRATUM_SHARE_DIFF": "1024", - "PRISM_STRATUM_VARDIFF_MIN_DIFF": "1024", - "PRISM_STRATUM_VARDIFF_START_DIFF": "4096", - "PRISM_STRATUM_VARDIFF_MAX_DIFF": "65536", - } - ) - reporter = self.self_check.Reporter() - - self.self_check.static_checks(env, reporter) - - self.assertFalse(any(row.name == "capacity.evidence" for row in reporter.rows)) - - def test_static_checks_reject_lab_production_difficulty(self) -> None: - env = self.valid_env() - env.update( - { - "QBIT_PRODUCTION": "1", - "QBIT_GIT_COMMIT": "41" * 20, - "PRISM_STRATUM_STALE_GRACE_SECONDS": "3", - } - ) - reporter = self.self_check.Reporter() - - self.self_check.static_checks(env, reporter) - - self.assertIn( - ("FAIL", "mining.production_difficulty"), - {(row.status, row.name) for row in reporter.rows}, - ) - - def test_static_checks_accept_bounded_mainnet_stale_grace(self) -> None: - # Mainnet follows the same bounded-grace rule as every other chain: - # zero grace rejects every in-flight prior-tip share at each block, - # which miners read as pool failure. - env = self.valid_env() - env.update( - { - "QBIT_CHAIN": "mainnet", - "QBIT_CHAIN_FLAG": "-chain=main", - "QBIT_EXPECTED_GENESIS_HASH": "AB" * 32, - "QBIT_GIT_COMMIT": "41" * 20, - "PRISM_STRATUM_STALE_GRACE_SECONDS": "3", - } - ) - reporter = self.self_check.Reporter() - - self.self_check.static_checks(env, reporter) - - rows = {(row.status, row.name) for row in reporter.rows} - self.assertIn(("PASS", "mining.stale_grace"), rows) - self.assertNotIn(("FAIL", "mining.stale_grace"), rows) - - def test_static_checks_accept_bounded_stale_grace_off_mainnet(self) -> None: - env = self.valid_env() - env.update( - { - "QBIT_PRODUCTION": "1", - "QBIT_GIT_COMMIT": "41" * 20, - "PRISM_STRATUM_STALE_GRACE_SECONDS": "2", - } - ) - reporter = self.self_check.Reporter() - - self.self_check.static_checks(env, reporter) - - rows = {(row.status, row.name) for row in reporter.rows} - self.assertIn(("PASS", "mining.stale_grace"), rows) - self.assertNotIn(("FAIL", "mining.stale_grace"), rows) - - def test_static_checks_parse_stale_grace_with_runtime_float_semantics(self) -> None: - # The coordinator parses the window with env_nonnegative_float(), so a - # fractional window is valid while negative or non-finite values are not. - for grace, expected in ( - ("0.5", "PASS"), - ("0", "PASS"), - ("1e-1", "PASS"), - (".5", "PASS"), - ("3.", "PASS"), - ("-1", "FAIL"), - ("nan", "FAIL"), - ("inf", "FAIL"), - ("1" + "0" * 400, "FAIL"), - ("abc", "FAIL"), - ): - with self.subTest(grace=grace): - env = self.valid_env() - env["PRISM_STRATUM_STALE_GRACE_SECONDS"] = grace - reporter = self.self_check.Reporter() - - self.self_check.static_checks(env, reporter) - - rows = {(row.status, row.name) for row in reporter.rows} - self.assertIn((expected, "mining.stale_grace"), rows) - - def test_static_checks_require_explicit_production_auxpow_payouts(self) -> None: - env = self.valid_env() - env.update( - { - "QBIT_PRODUCTION": "1", - "QBIT_GIT_COMMIT": "41" * 20, - "PRISM_STRATUM_STALE_GRACE_SECONDS": "3", - "BITCOIN_CHAIN": "mainnet", - "BITCOIN_CHAIN_FLAG": "-chain=main", - "QBIT_MINER_ADDRESS": "auto", - "BITCOIN_MINER_ADDRESS": "auto", - } - ) - reporter = self.self_check.Reporter() - - self.self_check.static_checks(env, reporter) - - failures = {(row.status, row.name) for row in reporter.rows} - self.assertIn(("FAIL", "auxpow.qbit_payout"), failures) - self.assertIn(("FAIL", "auxpow.bitcoin_payout"), failures) - - def test_static_checks_accept_explicit_production_auxpow_payouts(self) -> None: - env = self.valid_env() - env.update( - { - "QBIT_PRODUCTION": "1", - "QBIT_GIT_COMMIT": "41" * 20, - "PRISM_STRATUM_STALE_GRACE_SECONDS": "3", - "BITCOIN_CHAIN": "mainnet", - "BITCOIN_CHAIN_FLAG": "-chain=main", - "QBIT_MINER_ADDRESS": "qb1explicit", - "BITCOIN_MINER_ADDRESS": "bc1explicit", - } - ) - reporter = self.self_check.Reporter() - - self.self_check.static_checks(env, reporter) - - statuses = {(row.status, row.name) for row in reporter.rows} - self.assertIn(("PASS", "auxpow.qbit_payout"), statuses) - self.assertIn(("PASS", "auxpow.bitcoin_payout"), statuses) - - def test_static_checks_accept_positive_explicit_ctv_fee_rate(self) -> None: - env = self.valid_env() - env.update( - { - "PRISM_CTV_SETTLEMENT_ENABLED": "1", - "PRISM_CTV_FANOUT_FEE_MARKET_RATE_BITS_PER_1000_WEIGHT": "1000", - "PRISM_CTV_FANOUT_FEE_PREMIUM_BPS": "12000", - } - ) - reporter = self.self_check.Reporter() - - self.self_check.static_checks(env, reporter) - - fee_rows = [row for row in reporter.rows if row.name == "ctv.fee_source"] - self.assertEqual([row.status for row in fee_rows], ["PASS"]) - self.assertIn("explicit rate=1000", fee_rows[0].detail) - - def test_static_checks_reject_invalid_explicit_ctv_fee_rate(self) -> None: - for fee_rate in ("0", "-1", "1.5", "invalid"): - with self.subTest(fee_rate=fee_rate): - env = self.valid_env() - env.update( - { - "PRISM_CTV_SETTLEMENT_ENABLED": "1", - "PRISM_CTV_FANOUT_FEE_MARKET_RATE_BITS_PER_1000_WEIGHT": fee_rate, - } - ) - reporter = self.self_check.Reporter() - - self.self_check.static_checks(env, reporter) - - self.assertIn( - ("FAIL", "ctv.fee_source"), - {(row.status, row.name) for row in reporter.rows}, - ) - - def test_static_checks_warn_when_ctv_requires_live_estimator(self) -> None: - env = self.valid_env() - env["PRISM_CTV_SETTLEMENT_ENABLED"] = "1" - reporter = self.self_check.Reporter() - - self.self_check.static_checks(env, reporter) - - fee_rows = [row for row in reporter.rows if row.name == "ctv.fee_source"] - self.assertEqual([row.status for row in fee_rows], ["WARN"]) - self.assertIn("live preflight required", fee_rows[0].detail) - self.assertIn("Fresh chains", fee_rows[0].hint or "") - - def test_static_checks_default_coinbase_output_policy_is_canonical(self) -> None: - reporter = self.self_check.Reporter() - - self.self_check.static_checks(self.valid_env(), reporter) - - policy_rows = [row for row in reporter.rows if row.name == "coinbase.output_policy"] - self.assertEqual([row.status for row in policy_rows], ["PASS"]) - self.assertEqual(policy_rows[0].detail, "canonical") - - def test_static_checks_accept_pool_fee_first_with_enabled_pool_fee(self) -> None: - env = self.valid_env() - env.update( - { - "PRISM_COINBASE_OUTPUT_POLICY": "pool-fee-first", - "PRISM_POOL_FEE_ENABLED": "1", - "PRISM_POOL_FEE_BPS": "125", - "PRISM_POOL_FEE_ADDRESS": "tq1fee", - } - ) - reporter = self.self_check.Reporter() - - self.self_check.static_checks(env, reporter) - - policy_rows = [row for row in reporter.rows if row.name == "coinbase.output_policy"] - self.assertEqual([row.status for row in policy_rows], ["PASS"]) - self.assertEqual(policy_rows[0].detail, "pool-fee-first") - - def test_static_checks_reject_pool_fee_first_without_pool_fee(self) -> None: - env = self.valid_env() - env["PRISM_COINBASE_OUTPUT_POLICY"] = "pool-fee-first" - reporter = self.self_check.Reporter() - - self.self_check.static_checks(env, reporter) - - self.assertIn( - ("FAIL", "coinbase.output_policy"), - {(row.status, row.name) for row in reporter.rows}, - ) - - def test_static_checks_reject_unknown_coinbase_output_policy(self) -> None: - for invalid in ("fee-first", "POOL-FEE-FIRST"): - with self.subTest(invalid=invalid): - env = self.valid_env() - env["PRISM_COINBASE_OUTPUT_POLICY"] = invalid - reporter = self.self_check.Reporter() - - self.self_check.static_checks(env, reporter) - - self.assertIn( - ("FAIL", "coinbase.output_policy"), - {(row.status, row.name) for row in reporter.rows}, - ) - - def test_highdiff_probe_target_uses_published_host_port_only(self) -> None: - # The published host mapping is the only valid probe target: falling - # back to the container listen port could pass while miners cannot - # reach the listener. - self.assertEqual( - self.self_check.highdiff_probe_target({"PRISM_STRATUM_HIGHDIFF_PORT_HOST": "4334"}), - ("127.0.0.1", 4334), - ) - self.assertEqual( - self.self_check.highdiff_probe_target( - {"PRISM_STRATUM_HIGHDIFF_PORT_HOST": "0.0.0.0:14334"} - ), - ("0.0.0.0", 14334), - ) - # Unset, empty, and the disabled-default ephemeral loopback mapping all - # mean "not published". - self.assertIsNone(self.self_check.highdiff_probe_target({})) - self.assertIsNone( - self.self_check.highdiff_probe_target({"PRISM_STRATUM_HIGHDIFF_PORT_HOST": ""}) - ) - self.assertIsNone( - self.self_check.highdiff_probe_target( - {"PRISM_STRATUM_HIGHDIFF_PORT_HOST": "127.0.0.1:0"} - ) - ) - - def test_static_checks_accept_highdiff_with_empty_share_diff(self) -> None: - # Compose resolves the fixed difficulty default to an empty string; the - # coordinator treats that as "track the start difficulty" and the - # self-check must agree instead of failing mining.highdiff. - env = self.valid_env() - env["PRISM_STRATUM_HIGHDIFF_PORT"] = "4334" - env["PRISM_STRATUM_HIGHDIFF_SHARE_DIFF"] = "" - reporter = self.self_check.Reporter() - - self.self_check.static_checks(env, reporter) - - self.assertFalse(reporter.failed) - self.assertIn( - "PASS", - {row.status for row in reporter.rows if row.name == "mining.highdiff"}, - ) - - def test_static_checks_fail_highdiff_share_diff_below_floor(self) -> None: - env = self.valid_env() - env["PRISM_STRATUM_HIGHDIFF_PORT"] = "4334" - env["PRISM_STRATUM_HIGHDIFF_SHARE_DIFF"] = "1000" - reporter = self.self_check.Reporter() - - self.self_check.static_checks(env, reporter) - - self.assertTrue(reporter.failed) - self.assertIn( - ("FAIL", "mining.highdiff"), - {(row.status, row.name) for row in reporter.rows}, - ) - - def test_ready_miner_threshold_fails_when_below_minimum(self) -> None: - env = self.valid_env() - env["PRISM_MIN_READY_MINERS"] = "3" - reporter = self.self_check.Reporter() - - self.self_check.check_ready_miner_threshold({"ready_miner_count": 2}, env, reporter) - - self.assertTrue(reporter.failed) - self.assertIn( - ("FAIL", "coordinator.ready_miners"), - {(row.status, row.name) for row in reporter.rows}, - ) - - def test_ready_miner_threshold_is_nonfatal_during_mainnet_prelaunch(self) -> None: - env = authorize_mainnet_prelaunch(self.valid_env()) - env["PRISM_MIN_READY_MINERS"] = "1" - reporter = self.self_check.Reporter() - - self.self_check.check_ready_miner_threshold({"ready_miner_count": 0}, env, reporter) - - self.assertFalse(reporter.failed) - rows = [row for row in reporter.rows if row.name == "coordinator.ready_miners"] - self.assertEqual([row.status for row in rows], ["WARN"]) - self.assertIn("tolerated only", rows[0].detail) - - def test_ready_miner_threshold_stays_fatal_without_full_prelaunch_authorization(self) -> None: - env = self.valid_env() - env.update( - { - "QBIT_CHAIN": "mainnet", - "PRISM_MIN_READY_MINERS": "1", - "QBIT_MAINNET_LAUNCH_READINESS_CHECKS_ENABLED": "0", - } - ) - reporter = self.self_check.Reporter() - - self.self_check.check_ready_miner_threshold({"ready_miner_count": 0}, env, reporter) - - self.assertIn( - ("FAIL", "coordinator.ready_miners"), - {(row.status, row.name) for row in reporter.rows}, - ) - - def test_ready_miner_threshold_is_fatal_after_mainnet_launch(self) -> None: - env = self.valid_env() - env.update( - { - "QBIT_CHAIN": "mainnet", - "PRISM_MIN_READY_MINERS": "1", - "QBIT_MAINNET_LAUNCH_READINESS_CHECKS_ENABLED": "1", - } - ) - reporter = self.self_check.Reporter() - - self.self_check.check_ready_miner_threshold({"ready_miner_count": 0}, env, reporter) - - self.assertTrue(reporter.failed) - self.assertIn( - ("FAIL", "coordinator.ready_miners"), - {(row.status, row.name) for row in reporter.rows}, - ) - - def test_ready_miner_threshold_passes_when_minimum_met(self) -> None: - env = self.valid_env() - env["PRISM_MIN_READY_MINERS"] = "3" - reporter = self.self_check.Reporter() - - self.self_check.check_ready_miner_threshold({"ready_miner_count": 3}, env, reporter) - - self.assertFalse(reporter.failed) - self.assertIn( - ("PASS", "coordinator.ready_miners"), - {(row.status, row.name) for row in reporter.rows}, - ) - - def test_public_chain_peer_rpc_error_is_hard_failure(self) -> None: - reporter = self.self_check.Reporter() - - def fake_qbit_rpc_call( - env: dict[str, str], - method: str, - params: list[object] | None = None, - ) -> object: - if method == "getblockchaininfo": - return self.public_blockchain_info("signet") - if method == "getblocktemplate": - return self.fresh_template() - if method == "getnetworkinfo": - raise RuntimeError("rpc unavailable") - raise AssertionError(method) - - with patch.object(self.self_check, "qbit_rpc_call", fake_qbit_rpc_call): - self.self_check.qbit_live_checks({"QBIT_CHAIN": "signet"}, reporter) - - self.assertTrue(reporter.failed) - self.assertIn( - ("FAIL", "qbit.peers"), - {(row.status, row.name) for row in reporter.rows}, - ) - - def test_live_checks_normalize_mainnet_rpc_chain_and_verify_genesis(self) -> None: - expected_genesis = "11" * 32 - calls: list[tuple[str, list[object] | None]] = [] - reporter = self.self_check.Reporter() - - def fake_qbit_rpc_call( - env: dict[str, str], - method: str, - params: list[object] | None = None, - ) -> object: - calls.append((method, params)) - if method == "getblockchaininfo": - return self.public_blockchain_info("main") - if method == "getblocktemplate": - return self.fresh_template() - if method == "getblockhash": - return expected_genesis - if method == "getnetworkinfo": - return {"connections": 2} - raise AssertionError(method) - - env = { - "QBIT_CHAIN": "mainnet", - "QBIT_EXPECTED_GENESIS_HASH": expected_genesis.upper(), - } - with patch.object(self.self_check, "qbit_rpc_call", fake_qbit_rpc_call): - self.self_check.qbit_live_checks(env, reporter) - - self.assertFalse(reporter.failed) - self.assertIn(("getblockhash", [0]), calls) - self.assertIn( - ("PASS", "qbit.rpc_chain"), - {(row.status, row.name) for row in reporter.rows}, - ) - self.assertIn( - ("PASS", "qbit.genesis"), - {(row.status, row.name) for row in reporter.rows}, - ) - - def test_live_checks_fail_genesis_mismatch(self) -> None: - reporter = self.self_check.Reporter() - - def fake_qbit_rpc_call( - env: dict[str, str], - method: str, - params: list[object] | None = None, - ) -> object: - if method == "getblockchaininfo": - return self.public_blockchain_info("main") - if method == "getblocktemplate": - return self.fresh_template() - if method == "getblockhash": - return "22" * 32 - if method == "getnetworkinfo": - return {"connections": 2} - raise AssertionError(method) - - env = {"QBIT_CHAIN": "mainnet", "QBIT_EXPECTED_GENESIS_HASH": "11" * 32} - with patch.object(self.self_check, "qbit_rpc_call", fake_qbit_rpc_call): - self.self_check.qbit_live_checks(env, reporter) - - self.assertIn( - ("FAIL", "qbit.genesis"), - {(row.status, row.name) for row in reporter.rows}, - ) - - def test_live_checks_accept_preflightable_ctv_fee_estimator(self) -> None: - reporter = self.self_check.Reporter() - - def fake_qbit_rpc_call( - env: dict[str, str], - method: str, - params: list[object] | None = None, - ) -> object: - if method == "getblockchaininfo": - return self.public_blockchain_info("signet") - if method == "getblocktemplate": - return self.fresh_template() - if method == "estimatesmartfee": - self.assertEqual(params, [2]) - return {"feerate": 0.00001000, "blocks": 2} - if method == "getnetworkinfo": - return {"connections": 2} - raise AssertionError(method) - - env = {"QBIT_CHAIN": "signet", "PRISM_CTV_SETTLEMENT_ENABLED": "1"} - with patch.object(self.self_check, "qbit_rpc_call", fake_qbit_rpc_call): - self.self_check.qbit_live_checks(env, reporter) - - self.assertFalse(reporter.failed) - self.assertIn( - ("PASS", "ctv.fee_estimator"), - {(row.status, row.name) for row in reporter.rows}, - ) - - def test_live_checks_fail_ctv_estimator_without_fee_history(self) -> None: - reporter = self.self_check.Reporter() - - def fake_qbit_rpc_call( - env: dict[str, str], - method: str, - params: list[object] | None = None, - ) -> object: - if method == "getblockchaininfo": - return self.public_blockchain_info("signet") - if method == "getblocktemplate": - return self.fresh_template() - if method == "estimatesmartfee": - return {"errors": ["Insufficient data or no feerate found"]} - if method == "getnetworkinfo": - return {"connections": 2} - raise AssertionError(method) - - env = {"QBIT_CHAIN": "signet", "PRISM_CTV_SETTLEMENT_ENABLED": "1"} - with patch.object(self.self_check, "qbit_rpc_call", fake_qbit_rpc_call): - self.self_check.qbit_live_checks(env, reporter) - - fee_rows = [row for row in reporter.rows if row.name == "ctv.fee_estimator"] - self.assertEqual([row.status for row in fee_rows], ["FAIL"]) - self.assertIn("Fresh chains", fee_rows[0].hint or "") - - def test_mainnet_rpc_chain_mismatch_remains_fatal(self) -> None: - reporter = self.mainnet_live_reporter( - authorize_mainnet_prelaunch({}), - actual_chain="testnet4", - ) - - self.assertTrue(reporter.failed) - self.assertIn( - ("FAIL", "qbit.rpc_chain"), - {(row.status, row.name) for row in reporter.rows}, - ) - - def test_ibd_is_nonfatal_during_mainnet_prelaunch(self) -> None: - reporter = self.mainnet_live_reporter( - authorize_mainnet_prelaunch({}), - initial_block_download=True, - ) - - self.assertFalse(reporter.failed) - self.assertIn( - ("WARN", "qbit.ibd"), - {(row.status, row.name) for row in reporter.rows}, - ) - - def test_ibd_is_fatal_after_mainnet_launch(self) -> None: - reporter = self.mainnet_live_reporter( - { - "QBIT_CHAIN": "mainnet", - "QBIT_MAINNET_LAUNCH_READINESS_CHECKS_ENABLED": "1", - }, - initial_block_download=True, - ) - - self.assertTrue(reporter.failed) - self.assertIn( - ("FAIL", "qbit.ibd"), - {(row.status, row.name) for row in reporter.rows}, - ) - - def test_missing_launch_flag_keeps_ibd_fatal(self) -> None: - reporter = self.mainnet_live_reporter( - {"QBIT_CHAIN": "mainnet"}, - initial_block_download=True, - ) - - self.assertTrue(reporter.failed) - self.assertIn( - ("FAIL", "qbit.ibd"), - {(row.status, row.name) for row in reporter.rows}, - ) - - def test_unrelated_peer_failure_remains_fatal_during_mainnet_prelaunch(self) -> None: - reporter = self.mainnet_live_reporter( - authorize_mainnet_prelaunch({}), - initial_block_download=True, - peers=0, - ) - - self.assertTrue(reporter.failed) - statuses = {(row.status, row.name) for row in reporter.rows} - self.assertIn(("WARN", "qbit.ibd"), statuses) - self.assertIn(("FAIL", "qbit.peers"), statuses) - - -class HighdiffFloorProbeTests(unittest.TestCase): - """The live high-diff check must judge the first advertised difficulty, - exactly like a marketplace verification probe, not mere reachability.""" - - @classmethod - def setUpClass(cls) -> None: - cls.self_check = load_self_check_module() - - def fake_stratum_server( - self, - notifications: list[dict[str, object]], - *, - reject_authorize: bool = False, - ) -> int: - """One-shot stratum server: answers subscribe/authorize, then emits - the scripted notifications and closes. Returns the listening port.""" - server_sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) - server_sock.bind(("127.0.0.1", 0)) - server_sock.listen(1) - server_sock.settimeout(5) - port = server_sock.getsockname()[1] - - def serve() -> None: - try: - conn, _ = server_sock.accept() - except OSError: - return - with conn: - conn.settimeout(5) - reader = conn.makefile("rb") - for _ in range(2): # subscribe + authorize requests - reader.readline() - responses: list[dict[str, object]] = [ - {"id": 1, "result": [[], "00000001", 8], "error": None} - ] - if reject_authorize: - responses.append({"id": 2, "result": None, "error": [20, "unauthorized", None]}) - else: - responses.append({"id": 2, "result": True, "error": None}) - responses.extend(notifications) - for message in responses: - conn.sendall((json.dumps(message) + "\n").encode()) - - thread = threading.Thread(target=serve, daemon=True) - thread.start() - self.addCleanup(server_sock.close) - self.addCleanup(lambda: thread.join(timeout=5)) - return port - - def probe_env(self) -> dict[str, str]: - return {"PRISM_STRATUM_HIGHDIFF_MIN_DIFF": "500000"} - - def floor_rows(self, reporter: object) -> list[object]: - return [row for row in reporter.rows if row.name == "stratum.highdiff_floor"] - - def test_first_difficulty_at_floor_passes(self) -> None: - port = self.fake_stratum_server( - [{"id": None, "method": "mining.set_difficulty", "params": [500000.0]}] - ) - reporter = self.self_check.Reporter() - - self.self_check.check_highdiff_advertised_floor(self.probe_env(), "127.0.0.1", port, reporter) - - rows = self.floor_rows(reporter) - self.assertEqual([row.status for row in rows], ["PASS"]) - - def test_first_difficulty_below_floor_fails(self) -> None: - # The regression this guards: a young chain dragging the first - # advertised difficulty below the floor. A later compliant value must - # not rescue the check -- marketplaces judge the first one. - port = self.fake_stratum_server( - [ - {"id": None, "method": "mining.set_difficulty", "params": [4.6565423739069247e-10]}, - {"id": None, "method": "mining.set_difficulty", "params": [500000.0]}, - ] - ) - reporter = self.self_check.Reporter() - - self.self_check.check_highdiff_advertised_floor(self.probe_env(), "127.0.0.1", port, reporter) - - rows = self.floor_rows(reporter) - self.assertEqual([row.status for row in rows], ["FAIL"]) - self.assertIn("below", rows[0].detail) - - def test_rejected_authorize_fails_with_handshake_detail(self) -> None: - port = self.fake_stratum_server([], reject_authorize=True) - reporter = self.self_check.Reporter() - - self.self_check.check_highdiff_advertised_floor(self.probe_env(), "127.0.0.1", port, reporter) - - rows = self.floor_rows(reporter) - self.assertEqual([row.status for row in rows], ["FAIL"]) - self.assertIn("authorize rejected", rows[0].detail) - - def test_connection_closed_without_difficulty_fails(self) -> None: - port = self.fake_stratum_server([{"id": None, "method": "mining.notify", "params": []}]) - reporter = self.self_check.Reporter() - - self.self_check.check_highdiff_advertised_floor(self.probe_env(), "127.0.0.1", port, reporter) - - rows = self.floor_rows(reporter) - self.assertEqual([row.status for row in rows], ["FAIL"]) - self.assertIn("mining.set_difficulty", rows[0].detail) - - def test_missing_difficulty_is_nonfatal_during_mainnet_prelaunch(self) -> None: - port = self.fake_stratum_server([{"id": None, "method": "mining.notify", "params": []}]) - env = authorize_mainnet_prelaunch(self.probe_env()) - reporter = self.self_check.Reporter() - - self.self_check.check_highdiff_advertised_floor(env, "127.0.0.1", port, reporter) - - rows = self.floor_rows(reporter) - self.assertFalse(reporter.failed) - self.assertEqual([row.status for row in rows], ["WARN"]) - self.assertIn("tolerated only", rows[0].detail) - - def test_missing_difficulty_is_fatal_after_mainnet_launch(self) -> None: - port = self.fake_stratum_server([{"id": None, "method": "mining.notify", "params": []}]) - env = self.probe_env() - env.update( - { - "QBIT_CHAIN": "mainnet", - "QBIT_MAINNET_LAUNCH_READINESS_CHECKS_ENABLED": "1", - } - ) - reporter = self.self_check.Reporter() - - self.self_check.check_highdiff_advertised_floor(env, "127.0.0.1", port, reporter) - - rows = self.floor_rows(reporter) - self.assertTrue(reporter.failed) - self.assertEqual([row.status for row in rows], ["FAIL"]) - - -if __name__ == "__main__": - unittest.main() diff --git a/tests/test_prism_settlement_balance_comparison.py b/tests/test_prism_settlement_balance_comparison.py deleted file mode 100644 index fb767c187..000000000 --- a/tests/test_prism_settlement_balance_comparison.py +++ /dev/null @@ -1,139 +0,0 @@ -#!/usr/bin/env python3 -"""Settlement-equality comparison for payout previews and ledger snapshots. - -A miner may re-authorize the same payout address under any valid bech32 -spelling (all-lowercase and all-uppercase encode the same program), which -changes the recipient/order-key identity strings without changing the -committed P2MR payout program. Ledger-confirmation gates must therefore -judge settlement equality on (program, amount) alone: a pure relabel is -equal, while any amount or membership divergence still fails closed. -""" - -from __future__ import annotations - -import unittest -from types import SimpleNamespace - -from lab.prism.prism_coordinator import PrismCoordinator - -PROGRAM_HEX = "cb" * 32 -OTHER_PROGRAM_HEX = "ad" * 32 -ADDRESS_LOWER = "tq1z70ukpvs96kye6jmgvl3nttevtkrq8uu89snkpm6m8gwqukw8u5dsz32kwa" -ADDRESS_UPPER = ADDRESS_LOWER.upper() - - -def balance_row( - *, - recipient_id: str, - order_key: str | None = None, - program_hex: str = PROGRAM_HEX, - balance_sats: int, -) -> dict[str, object]: - return { - "recipient_id": recipient_id, - "order_key": order_key if order_key is not None else recipient_id, - "p2mr_program_hex": program_hex, - "balance_sats": balance_sats, - } - - -class SettlementBalancesByProgramTest(unittest.TestCase): - def test_case_variant_identity_is_settlement_equal(self) -> None: - published = [balance_row(recipient_id=ADDRESS_LOWER, balance_sats=1234)] - durable = [balance_row(recipient_id=ADDRESS_UPPER, balance_sats=1234)] - self.assertNotEqual(published, durable) - self.assertEqual( - PrismCoordinator.settlement_balances_by_program(published), - PrismCoordinator.settlement_balances_by_program(durable), - ) - - def test_program_hex_case_is_normalized(self) -> None: - lower = [balance_row(recipient_id=ADDRESS_LOWER, balance_sats=5)] - upper = [ - balance_row( - recipient_id=ADDRESS_LOWER, - program_hex=PROGRAM_HEX.upper(), - balance_sats=5, - ) - ] - self.assertEqual( - PrismCoordinator.settlement_balances_by_program(lower), - PrismCoordinator.settlement_balances_by_program(upper), - ) - - def test_amount_divergence_still_detected(self) -> None: - published = [balance_row(recipient_id=ADDRESS_LOWER, balance_sats=1234)] - durable = [balance_row(recipient_id=ADDRESS_LOWER, balance_sats=1235)] - self.assertNotEqual( - PrismCoordinator.settlement_balances_by_program(published), - PrismCoordinator.settlement_balances_by_program(durable), - ) - - def test_membership_divergence_still_detected(self) -> None: - published = [balance_row(recipient_id=ADDRESS_LOWER, balance_sats=1234)] - durable = published + [ - balance_row( - recipient_id="qb1zother", - program_hex=OTHER_PROGRAM_HEX, - balance_sats=7, - ) - ] - self.assertNotEqual( - PrismCoordinator.settlement_balances_by_program(published), - PrismCoordinator.settlement_balances_by_program(durable), - ) - - def test_zero_totals_are_dropped(self) -> None: - with_zero = [ - balance_row(recipient_id=ADDRESS_LOWER, balance_sats=1234), - balance_row( - recipient_id="qb1zother", - program_hex=OTHER_PROGRAM_HEX, - balance_sats=0, - ), - ] - without_zero = [balance_row(recipient_id=ADDRESS_LOWER, balance_sats=1234)] - self.assertEqual( - PrismCoordinator.settlement_balances_by_program(with_zero), - PrismCoordinator.settlement_balances_by_program(without_zero), - ) - - def test_split_identities_aggregate_per_program(self) -> None: - split = [ - balance_row(recipient_id=ADDRESS_UPPER, balance_sats=1000), - balance_row(recipient_id=ADDRESS_LOWER, balance_sats=234), - ] - merged = [balance_row(recipient_id=ADDRESS_LOWER, balance_sats=1234)] - self.assertEqual( - PrismCoordinator.settlement_balances_by_program(split), - PrismCoordinator.settlement_balances_by_program(merged), - ) - - -class PriorBalancesMatchCurrentTest(unittest.TestCase): - def match_current( - self, - prior_balances: list[dict[str, object]], - current: list[dict[str, object]], - ) -> bool: - coordinator = SimpleNamespace( - settlement_balances_by_program=PrismCoordinator.settlement_balances_by_program, - ledger=SimpleNamespace(current_prior_balances=lambda: current), - ) - return PrismCoordinator.prior_balances_match_current( - coordinator, prior_balances - ) - - def test_relabeled_identity_matches_current(self) -> None: - prior = [balance_row(recipient_id=ADDRESS_UPPER, balance_sats=1234)] - current = [balance_row(recipient_id=ADDRESS_LOWER, balance_sats=1234)] - self.assertTrue(self.match_current(prior, current)) - - def test_changed_amount_does_not_match_current(self) -> None: - prior = [balance_row(recipient_id=ADDRESS_LOWER, balance_sats=1234)] - current = [balance_row(recipient_id=ADDRESS_LOWER, balance_sats=4321)] - self.assertFalse(self.match_current(prior, current)) - - -if __name__ == "__main__": - unittest.main() diff --git a/tests/test_prism_share_ledger.py b/tests/test_prism_share_ledger.py deleted file mode 100644 index 4c5a9faf0..000000000 --- a/tests/test_prism_share_ledger.py +++ /dev/null @@ -1,9669 +0,0 @@ -#!/usr/bin/env python3 - -from __future__ import annotations - -import contextlib -import gc -import gzip -import hashlib -import io -import json -import os -import subprocess -import sys -import tempfile -import threading -import time -import tracemalloc -import types -import unittest -import unittest.mock - -from datetime import datetime, timedelta, timezone -from decimal import Decimal -from pathlib import Path -from typing import Any - -from lab.prism.backfill_ctv_fanouts import backfill_input_from_path, backfill_input_from_payload, infer_block_hash_from_path -from lab.prism import public_api -from lab.prism import share_ledger as share_ledger_module -from lab.prism.audit_artifacts import ( - AuditArtifactConfig, - AuditArtifactStore, - CanonicalAuditBundleCorrupt, -) -from lab.prism.share_ledger import ( - database_url_from_psql_command, - parse_single_json_value, - AUDIT_BODY_REF_SCHEMA, - AUDIT_BUNDLE_V2_SCHEMA, - AcceptedShareRecord, - PendingShare, - PsqlShareLedger, - ShareReplayConflict, - ShareReplayResult, - LedgerOperationTimeout, - PAYOUT_WINDOW_ROW_BATCH_SIZE, - AUDIT_WINDOW_COMPLETENESS_PROOF_SCHEMA, - DEFAULT_WRITER_LEASE_ADOPTION_SILENCE_SECONDS, - SingleWriterShareLedger, - WRITER_LEASE_ACQUIRE_RETRY_ATTEMPTS, - WRITER_LEASE_ACQUIRE_RETRY_KEY, - WRITER_LEASE_ACQUIRE_RETRY_SUBJECT, - WRITER_LEASE_HEARTBEAT_SESSION_PREFIX, - _NativePostgresClient, - _NativePostgresLeaseGuard, - _prism_window_shares, - _writer_lease_advisory_lock_key, - canonical_json_text, - sha256_json_hex, -) - - -def pending_share( - index: int, - *, - share_difficulty: int | None = None, - job_issued_at_ms: int | None = None, - accepted_at_ms: int | None = None, - credit_policy: str | None = None, -) -> PendingShare: - return PendingShare( - share_id=f"share-{index}", - miner_id=f"miner-{index % 3}", - order_key=f"{index:04d}", - p2mr_program_hex=f"{index % 256:02x}" * 32, - share_difficulty=share_difficulty if share_difficulty is not None else 100 + index, - network_difficulty=1_000, - template_height=99, - job_id=f"job-{index}", - job_issued_at_ms=job_issued_at_ms if job_issued_at_ms is not None else 1_000 + index, - accepted_at_ms=accepted_at_ms if accepted_at_ms is not None else 2_000 + index, - ntime=1_700_000_000 + index, - credit_policy=credit_policy, - ) - - -def sample_ctv_manifest_set() -> dict[str, object]: - parent_coinbase_txid = "11" * 32 - fanout_txid = "22" * 32 - precommitment = { - "chunk_index": 0, - "chunk_count": 1, - "block_height": 123450, - "settlement_mode": "ctv_fanout", - "fanout_tx_template_hex": "0300000001", - "fanout_output_sum_sats": 25_000, - "anchor_vout": 1, - "ctv_hash_hex": "33" * 32, - } - manifest = { - "schema": "qbit.prism.ctv-fanout-manifest.v1", - "precommitment": precommitment, - "precommitment_sha256_hex": "44" * 32, - "commitment_witness_leaf_hex": "55" * 32, - "parent_coinbase_txid": parent_coinbase_txid, - "parent_coinbase_tx_hex": "0200000001", - "parent_coinbase_vout": 2, - "covenant_output_value_sats": 25_000, - "fanout_tx_hex": "0300000002", - "fanout_txid": fanout_txid, - } - return { - "schema": "qbit.prism.ctv-fanout-manifest-set.v1", - "block_height": 123450, - "settlement_mode": "ctv_fanout", - "parent_coinbase_txid": parent_coinbase_txid, - "fanout_count": 1, - "fanout_output_sum_sats": 25_000, - "covenant_output_value_sats": 25_000, - "manifests": [manifest], - } - - -def sample_no_anchor_fee_ctv_manifest_set() -> dict[str, object]: - manifest_set = sample_ctv_manifest_set() - manifest = manifest_set["manifests"][0] # type: ignore[index] - precommitment = manifest["precommitment"] # type: ignore[index] - precommitment["fanout_fee_sats"] = 100 # type: ignore[index] - precommitment["fanout_output_sum_sats"] = 24_900 # type: ignore[index] - precommitment.pop("anchor_vout", None) # type: ignore[union-attr] - manifest_set["fanout_output_sum_sats"] = 24_900 - return manifest_set - - -def fake_audit_bundle_bytes(payload: dict[str, Any]) -> bytes: - return json.dumps(payload, separators=(",", ":")).encode() - - -def fake_audit_bundle_sha256(payload: dict[str, Any]) -> str: - return hashlib.sha256(fake_audit_bundle_bytes(payload)).hexdigest() - - -def acquired_lease( - *, - writer_id: str = "writer-a", - writer_epoch: int = 1, - session: str = "new-session", -) -> dict[str, object]: - return { - "acquired": True, - "writer_id": writer_id, - "writer_epoch": writer_epoch, - "writer_session_token": session, - } - - -def held_lease( - *, - writer_id: str = "writer-a", - writer_epoch: int = 1, - session: str = "old-session", - wait_seconds: float = 5.0, - updated_at: str | None = None, - age_seconds: float | None = None, -) -> dict[str, object]: - result: dict[str, object] = { - "acquired": False, - "writer_id": writer_id, - "writer_epoch": writer_epoch, - "writer_session_token": session, - "lease_expires_at": "2026-06-26 19:50:22.233718+00", - "lease_wait_seconds": wait_seconds, - } - if updated_at is not None: - result["lease_updated_at"] = updated_at - if age_seconds is not None: - result["lease_age_seconds"] = age_seconds - return result - - -def snapshot_retry_lease() -> dict[str, object]: - """The acquisition statement's third arm, as PostgreSQL renders it. - - Built from the same constants the statement interpolates, so a change to - either the key or the subject shows up as a failure in the retry tests - rather than as a sentinel the ledger silently stops recognising. - """ - return { - "acquired": False, - WRITER_LEASE_ACQUIRE_RETRY_KEY: True, - "lease": WRITER_LEASE_ACQUIRE_RETRY_SUBJECT, - "retry_reason": ( - f"the {WRITER_LEASE_ACQUIRE_RETRY_SUBJECT} row was committed by a " - "concurrent first acquisition after this statement snapshot" - ), - } - - -class FakeMonotonicClock: - """Deterministic stand-in for time.monotonic in lease-adoption tests. - - Guard-acquisition silence is measured against the monotonic clock, so - these tests advance it exactly by each simulated retry sleep instead of - depending on wall time. - """ - - def __init__(self, start: float = 1000.0) -> None: - self.now = start - - def monotonic(self) -> float: - return self.now - - def sleep(self, seconds: float) -> None: - self.now += seconds - - -# Fixed hashes for the bulk terminal-update cases, named for the role each -# row plays in a page rather than for its bytes. -BULK_PENDING_A = "1a" * 32 -BULK_PENDING_B = "1b" * 32 -BULK_SUBMITTED = "1c" * 32 -BULK_MISSING = "99" * 32 - - -class CountingLock: - """Context-manager lock proxy that counts how often it is entered.""" - - def __init__(self, inner: Any) -> None: - self._inner = inner - self.acquisitions = 0 - - def __enter__(self) -> Any: - self.acquisitions += 1 - return self._inner.__enter__() - - def __exit__(self, *exc_info: Any) -> Any: - return self._inner.__exit__(*exc_info) - - -class FakeLeasePsqlShareLedger(PsqlShareLedger): - def __init__(self, lease_results: list[dict[str, object]], **kwargs: Any): - self.lease_results = list(lease_results) - self.lease_queries: list[str] = [] - self.sleeps: list[float] = [] - kwargs.setdefault("native_client_mode", "0") - retry_sleep = kwargs.pop("lease_retry_sleep", self.sleeps.append) - super().__init__( - psql_command="psql postgresql://example.invalid/qbit", - lease_retry_sleep=retry_sleep, - # One retry sleep must be able to cover a whole adoption silence, - # or a scenario that waits out the silence is split into several - # capped sleeps and consumes lease observations the fixture never - # scripted. Derived from the policy so retuning the silence does - # not silently change what these scenarios exercise. - lease_retry_max_sleep_seconds=max( - 1.0, - DEFAULT_WRITER_LEASE_ADOPTION_SILENCE_SECONDS, - ), - **kwargs, - ) - - def _make_writer_lease_guard(self, _database_url: str | None) -> Any: - class FakeGuard: - held = True - - def try_acquire(self) -> bool: - return True - - def close(self) -> None: - self.held = False - - guard = FakeGuard() - self.fake_writer_lease_guard = guard - return guard - - def _run_json(self, sql: str) -> Any: - self.lease_queries.append(sql) - if not self.lease_results: - raise AssertionError("unexpected extra lease query") - return self.lease_results.pop(0) - - -class QueryCapturePsqlShareLedger(PsqlShareLedger): - def __init__(self) -> None: - self._lock = threading.Lock() - self._read_semaphore = threading.BoundedSemaphore(1) - self.queries: list[str] = [] - - def _run_json(self, sql: str) -> Any: - self.queries.append(sql) - return [] - - -class GateStatementRecordingPsqlShareLedger(PsqlShareLedger): - """Ledger whose gate-held statements and progress stamps interleave. - - Built for the multi-statement gate bodies (confirm/reactivate/carry-forward - integrity), where the ordering of the stamp relative to each statement is - the property under test, not merely that a stamp happened. - """ - - def __init__(self, results: list[object]) -> None: - self._lock = threading.Lock() - self._read_semaphore = threading.BoundedSemaphore(1) - self._operation_timeout_local = threading.local() - self._statement_timeout_local = threading.local() - self._operation_progress_local = threading.local() - self._writer_id = "writer-a" - self._writer_epoch = 1 - self._writer_session_token = "session-a" - self._lease_interval_sql = "make_interval(secs => 30)" - self._pending_results = list(results) - self.events: list[str] = [] - self.statements = 0 - - def note_progress(self) -> None: - self.events.append("progress") - - def _run_json(self, sql: str) -> Any: - self.statements += 1 - self.events.append(f"statement-{self.statements}") - if not self._pending_results: - raise AssertionError(f"unexpected extra statement: {sql[:80]}") - return self._pending_results.pop(0) - - -class BlockingReadPsqlShareLedger(PsqlShareLedger): - def __init__(self, *, read_concurrency: int) -> None: - self._lock = threading.Lock() - self._read_semaphore = threading.BoundedSemaphore(read_concurrency) - self._condition = threading.Condition() - self._release = threading.Event() - self.active_reads = 0 - self.max_active_reads = 0 - self.started_reads = 0 - - def _run_json(self, sql: str) -> Any: - with self._condition: - self.active_reads += 1 - self.started_reads += 1 - self.max_active_reads = max(self.max_active_reads, self.active_reads) - self._condition.notify_all() - self._release.wait(timeout=5) - with self._condition: - self.active_reads -= 1 - self._condition.notify_all() - return None - - def wait_for_started_reads(self, count: int) -> None: - deadline = time.monotonic() + 5 - with self._condition: - while self.started_reads < count and time.monotonic() < deadline: - self._condition.wait(timeout=0.05) - if self.started_reads < count: - raise AssertionError(f"only {self.started_reads} reads started") - - def release_reads(self) -> None: - self._release.set() - - -class PrismShareLedgerTests(unittest.TestCase): - def test_psql_private_audit_delegates_and_subclass_override_route_through_a1(self) -> None: - ledger = PsqlShareLedger.__new__(PsqlShareLedger) - store = unittest.mock.Mock() - ledger._audit_artifact_store = store - ledger._audit_bundle_canonicalizer = lambda _bundle: b"legacy" - bundle = {"schema": "test"} - body_path = Path("body.json") - part = {"kind": "segment"} - recovery = {"load_missing_range": ledger._load_audit_share_ledger_range} - cases = ( - ( - "_externalize_audit_body", - ("aa" * 32, "bb" * 32, bundle), - {}, - "externalize_audit_body", - {}, - ), - ( - "_canonical_audit_body_bytes_for_sha", - (bundle, "bb" * 32), - {}, - "canonical_audit_body_bytes_for_sha", - {}, - ), - ("_audit_body_ref", (), {"payload": bundle}, "audit_body_ref", {}), - ("_audit_bundle_v2", (), {"payload": bundle}, "audit_bundle_v2", recovery), - ("_audit_share_parts", ([{"share_seq": 1}],), {}, "audit_share_parts", {}), - ( - "_audit_share_range_parts", - ([{"share_seq": 1}],), - {}, - "audit_share_range_parts", - recovery, - ), - ( - "_audit_share_segment_payload", - (), - {"shares": [{"share_seq": 1}]}, - "audit_share_segment_payload", - {}, - ), - ( - "_write_audit_share_segment", - (), - {"shares": [{"share_seq": 1}]}, - "write_audit_share_segment", - {}, - ), - ( - "_write_audit_share_segment_range", - (), - {"shares": [{"share_seq": 1}]}, - "write_audit_share_segment_range", - recovery, - ), - ( - "_merge_audit_share_ranges", - ([{"share_seq": 1}], [{"share_seq": 2}]), - {"segment_path": body_path}, - "merge_audit_share_ranges", - recovery, - ), - ( - "_audit_shares_by_seq", - ([{"share_seq": 1}],), - {"segment_path": body_path}, - "audit_shares_by_seq", - {"require_contiguous": True}, - ), - ("_storage_json_bytes", (bundle,), {}, "storage_json_bytes", {}), - ( - "_canonical_audit_bundle_bytes", - (bundle,), - {}, - "canonical_audit_bundle_bytes", - {}, - ), - ( - "_audit_body_path", - ("aa" * 32, "bb" * 32), - {}, - "body_path", - {}, - ), - ("_resolve_audit_body_path", (body_path,), {}, "resolve_owned_path", {}), - ( - "_audit_body_byte_len", - (body_path, bundle, body_path), - {}, - "audit_body_byte_len", - {}, - ), - ( - "_read_external_body", - (body_path,), - {"expected_sha256": "bb" * 32}, - "read_external_body", - {}, - ), - ( - "_external_body_matches_sha", - (body_path, "bb" * 32), - {}, - "external_body_matches_sha", - {}, - ), - ( - "_external_body_available_for_sha", - (body_path, "bb" * 32), - {}, - "external_body_available_for_sha", - {}, - ), - ( - "_resolve_audit_body_ref", - (bundle,), - {"expected_sha256": "bb" * 32, "body_uri": body_path}, - "resolve_audit_body_ref", - {}, - ), - ( - "_resolve_audit_bundle_v2", - (bundle,), - {"expected_sha256": "bb" * 32, "body_uri": body_path}, - "resolve_audit_bundle_v2", - {}, - ), - ( - "_read_audit_share_segment", - (part,), - {"parent_body_uri": body_path}, - "read_audit_share_segment", - {}, - ), - ( - "_select_audit_share_segment_range", - ([{"share_seq": 1}],), - { - "first_share_seq": 1, - "last_share_seq": 1, - "parent_body_uri": body_path, - "body_uri": body_path, - }, - "select_audit_share_segment_range", - {}, - ), - ) - token = object() - for wrapper_name, args, kwargs, target_name, extra_kwargs in cases: - with self.subTest(wrapper=wrapper_name): - target = getattr(store, target_name) - target.reset_mock() - target.return_value = token - self.assertIs(getattr(ledger, wrapper_name)(*args, **kwargs), token) - target.assert_called_once_with(*args, **{**kwargs, **extra_kwargs}) - self.assertIs(ledger._audit_store(), store) - - store.read_audit_share_segment.reset_mock() - self.assertTrue( - ledger._audit_share_segment_available( - part, - parent_body_uri=body_path, - ) - ) - store.read_audit_share_segment.assert_called_once_with( - part, - parent_body_uri=body_path, - ) - - override_calls: list[tuple[object, object]] = [] - ledger._read_external_body = ( # type: ignore[method-assign] - lambda body_uri, *, expected_sha256=None: ( - override_calls.append((body_uri, expected_sha256)) - or {"resolved": True} - ) - ) - self.assertEqual( - ledger._resolve_audit_bundle_row( - { - "audit_bundle": None, - "body_uri": "owned-body", - "audit_bundle_sha256": "bb" * 32, - } - ), - { - "audit_bundle": {"resolved": True}, - "audit_bundle_sha256": "bb" * 32, - }, - ) - self.assertEqual(override_calls, [("owned-body", "bb" * 32)]) - - def test_memory_recovery_append_distinguishes_insert_exact_and_conflict(self) -> None: - ledger = SingleWriterShareLedger() - share = pending_share(1) - - inserted = ledger.append_recovered_share(share) - exact = ledger.append_recovered_share(share) - - self.assertIsInstance(inserted, ShareReplayResult) - self.assertEqual(inserted.disposition, "inserted") - self.assertEqual(exact.disposition, "exact_existing") - with self.assertRaises(ShareReplayConflict): - ledger.append_recovered_share( - PendingShare(**{**share.__dict__, "ntime": share.ntime + 1}) - ) - - def test_writer_lease_advisory_guard_is_stable_and_session_scoped(self) -> None: - statements: list[str] = [] - connect_kwargs: dict[str, object] = {} - - class FakeConnection: - closed = False - - def execute(self, sql: str) -> FakeConnection: - statements.append(sql) - self.row = ( - True - if sql.startswith("SELECT pg_try_advisory_lock") - else {"renewed_count": 1} - ) - return self - - def fetchone(self) -> tuple[object]: - return (self.row,) - - def close(self) -> None: - self.closed = True - - connection = FakeConnection() - - class FakePsycopg: - @staticmethod - def connect(_conninfo: str, **kwargs: object) -> FakeConnection: - connect_kwargs.update(kwargs) - return connection - - key = _writer_lease_advisory_lock_key("writer-a", 7) - self.assertEqual(key, _writer_lease_advisory_lock_key("writer-a", 7)) - self.assertNotEqual(key, _writer_lease_advisory_lock_key("writer-a", 8)) - - with unittest.mock.patch.dict(sys.modules, {"psycopg": FakePsycopg}): - guard = _NativePostgresLeaseGuard("postgresql://example/qbit", key) - - self.assertTrue(guard.try_acquire()) - self.assertTrue(guard.held) - self.assertEqual( - guard.run_json("SELECT json_build_object('renewed_count', 1)"), - {"renewed_count": 1}, - ) - self.assertEqual(connect_kwargs["connect_timeout"], 2) - self.assertIn("statement_timeout=500", str(connect_kwargs["options"])) - self.assertIn(str(key), statements[0]) - - guard.close() - self.assertFalse(guard.held) - with self.assertRaisesRegex(RuntimeError, "guard is not held"): - guard.run_json("SELECT 1") - - def test_single_writer_assigns_contiguous_sequence_numbers(self) -> None: - ledger = SingleWriterShareLedger() - - records = [ledger.append(pending_share(index)) for index in range(1, 6)] - - self.assertEqual([record.share_seq for record in records], [1, 2, 3, 4, 5]) - self.assertEqual(len(ledger), 5) - self.assertEqual([record["share_seq"] for record in (item.to_prism_json() for item in records)], [1, 2, 3, 4, 5]) - - def test_credit_policy_round_trips_only_when_present(self) -> None: - ledger = SingleWriterShareLedger() - - normal = ledger.append(pending_share(1)) - grace = ledger.append(pending_share(2, credit_policy="stale-grace")) - - self.assertNotIn("credit_policy", normal.to_prism_json()) - self.assertEqual(grace.credit_policy, "stale-grace") - self.assertEqual(grace.to_prism_json()["credit_policy"], "stale-grace") - - def test_invalid_credit_policy_is_rejected_before_storage(self) -> None: - ledger = SingleWriterShareLedger() - - for policy in ("", "bogus"): - with self.subTest(policy=policy), self.assertRaisesRegex( - ValueError, - "unsupported credit_policy", - ): - ledger.append(pending_share(1, credit_policy=policy)) - - def test_job_issue_snapshot_excludes_later_job_shares(self) -> None: - ledger = SingleWriterShareLedger() - ledger.append(pending_share(1, job_issued_at_ms=1_000, accepted_at_ms=1_001)) - ledger.append(pending_share(2, job_issued_at_ms=1_005, accepted_at_ms=1_005)) - snapshot = ledger.snapshot_at_job_issue(1_005) - ledger.append(pending_share(3, job_issued_at_ms=1_006, accepted_at_ms=1_006)) - - self.assertEqual([share.share_id for share in snapshot], ["share-1", "share-2"]) - self.assertEqual( - [share.share_id for share in ledger.snapshot_at_job_issue(1_005)], - ["share-1", "share-2"], - ) - - def test_job_issue_snapshot_accepts_window_weight_hint(self) -> None: - # The in-memory backend follows the same exact crossing-row bound as - # Postgres so synchronous and incremental artifact hashes cannot - # diverge in local/embedded deployments. - ledger = SingleWriterShareLedger() - for index in range(1, 5): - ledger.append( - pending_share( - index, - share_difficulty=3, - job_issued_at_ms=1_000, - accepted_at_ms=1_000 + index, - ) - ) - - unbounded = [s.share_id for s in ledger.snapshot_at_job_issue(1_005)] - hinted = [ - s.share_id - for s in ledger.snapshot_at_job_issue(1_005, window_weight=5) - ] - - self.assertEqual( - unbounded, - ["share-1", "share-2", "share-3", "share-4"], - ) - self.assertEqual(hinted, ["share-3", "share-4"]) - - def test_job_issue_snapshot_excludes_old_job_shares_accepted_after_anchor(self) -> None: - ledger = SingleWriterShareLedger() - ledger.append(pending_share(1, job_issued_at_ms=1_000, accepted_at_ms=1_001)) - ledger.append(pending_share(2, job_issued_at_ms=1_000, accepted_at_ms=1_006)) - - self.assertEqual( - [share.share_id for share in ledger.snapshot_at_job_issue(1_005)], - ["share-1"], - ) - - def test_job_issue_delta_returns_each_newly_eligible_share_once(self) -> None: - ledger = SingleWriterShareLedger() - ledger.append( - pending_share(1, job_issued_at_ms=1_000, accepted_at_ms=1_001) - ) - ledger.append( - pending_share(2, job_issued_at_ms=1_000, accepted_at_ms=1_006) - ) - ledger.append( - pending_share(3, job_issued_at_ms=1_006, accepted_at_ms=1_004) - ) - ledger.append( - pending_share(4, job_issued_at_ms=1_007, accepted_at_ms=1_007) - ) - - delta = ledger.snapshot_between_job_issues(1_005, 1_006) - - self.assertEqual([share.share_id for share in delta], ["share-2", "share-3"]) - - def test_rejects_zero_difficulty_share(self) -> None: - ledger = SingleWriterShareLedger() - share = pending_share(1) - - with self.assertRaisesRegex(ValueError, "share_difficulty"): - ledger.append(share.__class__(**{**share.__dict__, "share_difficulty": 0})) - with self.assertRaisesRegex(ValueError, "network_difficulty"): - ledger.append(share.__class__(**{**share.__dict__, "network_difficulty": 0})) - - def test_exact_duplicate_share_is_idempotent_but_mutation_is_rejected(self) -> None: - ledger = SingleWriterShareLedger() - first = pending_share(1) - duplicate = pending_share(2).__class__(**{**pending_share(2).__dict__, "share_id": first.share_id}) - later_stamp = first.__class__( - **{**first.__dict__, "accepted_at_ms": first.accepted_at_ms + 42} - ) - - self.assertEqual(ledger.append(first).share_seq, 1) - replay = ledger.append(later_stamp) - self.assertEqual(replay.share_seq, 1) - self.assertFalse(replay.newly_inserted) - self.assertEqual(replay.accepted_at_ms, first.accepted_at_ms) - with self.assertRaisesRegex(ValueError, "payload mismatch"): - ledger.append(duplicate) - - self.assertEqual(ledger.append(pending_share(3)).share_seq, 2) - - def test_share_and_block_candidate_intent_commit_atomically(self) -> None: - ledger = SingleWriterShareLedger() - share = pending_share(1) - intent = { - "schema": "qbit.prism.block-candidate-intent.v1", - "block_hash_hex": "ab" * 32, - "block_hex": "00", - } - - records = ledger.append_batch([(share, intent)]) - - self.assertEqual(records[0].share_seq, 1) - self.assertTrue(records[0].newly_inserted) - self.assertEqual(records[0].candidate_outbox_state, "pending") - self.assertEqual(ledger.pending_block_candidates(), [intent]) - rows = ledger.pending_block_candidate_rows() - self.assertEqual( - [ - {key: value for key, value in row.items() if key != "cursor"} - for row in rows - ], - [ - { - "block_hash": "ab" * 32, - "candidate": intent, - "pool_block_exists": False, - } - ], - ) - # The pagination cursor is opaque to callers but always keyed on the - # row it came from. - self.assertEqual(rows[0]["cursor"][1], "ab" * 32) - # Exact replay returns the original row and does not duplicate outbox - # work. A changed intent with the same hash is rejected as corruption. - replay = ledger.append_batch([(share, intent)])[0] - self.assertEqual(replay.share_seq, 1) - self.assertFalse(replay.newly_inserted) - self.assertEqual(replay.candidate_outbox_state, "pending") - with self.assertRaisesRegex(ValueError, "candidate payload mismatch"): - ledger.append_batch([(share, {**intent, "block_hex": "01"})]) - self.assertEqual(len(ledger), 1) - self.assertTrue(ledger.mark_block_candidate_submitted(block_hash="ab" * 32)) - self.assertEqual(ledger.pending_block_candidates(), []) - later_share = share.__class__( - **{**share.__dict__, "accepted_at_ms": share.accepted_at_ms + 42} - ) - terminal_replay = ledger.append_batch([(later_share, intent)])[0] - self.assertFalse(terminal_replay.newly_inserted) - self.assertEqual(terminal_replay.candidate_outbox_state, "submitted") - self.assertFalse( - ledger.mark_block_candidate_abandoned( - block_hash="ab" * 32, - error="must not invert terminal state", - ) - ) - - def test_pending_candidate_age_distinguishes_first_attempt(self) -> None: - ledger = SingleWriterShareLedger() - block_hash = "ac" * 32 - intent = { - "schema": "qbit.prism.block-candidate-intent.v1", - "block_hash_hex": block_hash, - "block_hex": "00", - } - ledger.persist_block_candidate_intent(intent) - - pending = ledger.block_candidate_pending_metrics() - self.assertEqual(pending["pending_count"], 1) - self.assertGreaterEqual(pending["oldest_pending_age_seconds"], 0) - self.assertGreaterEqual(pending["oldest_unattempted_age_seconds"], 0) - - self.assertTrue(ledger.mark_block_candidate_attempted(block_hash=block_hash)) - attempted = ledger.block_candidate_pending_metrics() - self.assertEqual(attempted["oldest_unattempted_age_seconds"], 0) - self.assertEqual( - ledger._block_candidate_outbox[block_hash]["attempt_count"], - 1, - ) - - def _seeded_bulk_abandon_ledger(self) -> SingleWriterShareLedger: - """One pending pair, one already-terminal row, one never persisted.""" - ledger = SingleWriterShareLedger() - for block_hash in (BULK_PENDING_A, BULK_PENDING_B, BULK_SUBMITTED): - ledger.persist_block_candidate_intent( - { - "schema": "qbit.prism.block-candidate-intent.v1", - "block_hash_hex": block_hash, - "block_hex": "00", - } - ) - self.assertTrue( - ledger.mark_block_candidate_submitted(block_hash=BULK_SUBMITTED) - ) - return ledger - - def test_bulk_candidate_abandon_reports_only_the_rows_it_transitioned( - self, - ) -> None: - """The report is the caller's cleanup authority, not an echo. - - A page handed to the bulk update is a candidate set, not a decided - set: rows can already be terminal, or absent entirely. Returning the - rows this call actually moved -- rather than a count or the request - -- is what lets the caller confine its follow-up cleanup to rows it - won, so it is asserted against a page deliberately containing both - kinds of non-transition. - """ - ledger = self._seeded_bulk_abandon_ledger() - - abandoned = ledger.mark_block_candidates_abandoned( - block_hashes=[ - BULK_PENDING_B, - BULK_PENDING_A.upper(), - BULK_PENDING_A, - BULK_SUBMITTED, - BULK_MISSING, - ], - error="superseded by decided height", - ) - - # Duplicates and case collapse; ordering is the normalized set's, not - # the caller's, so one input set always reports one identical result. - self.assertEqual(abandoned, (BULK_PENDING_A, BULK_PENDING_B)) - for block_hash in (BULK_PENDING_A, BULK_PENDING_B): - row = ledger._block_candidate_outbox[block_hash] - self.assertEqual(row["state"], "abandoned") - self.assertEqual(row["last_error"], "superseded by decided height") - self.assertIsNone(row["candidate"]) - # An already-terminal row is neither reported nor rewritten, so its - # own terminal reason survives a page that happens to name it. - submitted = ledger._block_candidate_outbox[BULK_SUBMITTED] - self.assertEqual(submitted["state"], "submitted") - self.assertIsNone(submitted["last_error"]) - self.assertNotIn(BULK_MISSING, ledger._block_candidate_outbox) - self.assertEqual(ledger.pending_block_candidates(), []) - - # Re-running the same page wins nothing: the rows are terminal now. - self.assertEqual( - ledger.mark_block_candidates_abandoned( - block_hashes=[BULK_PENDING_A, BULK_PENDING_B], - error="second attempt", - ), - (), - ) - self.assertEqual( - ledger._block_candidate_outbox[BULK_PENDING_A]["last_error"], - "superseded by decided height", - ) - - def test_bulk_candidate_abandon_on_an_empty_page_touches_nothing(self) -> None: - ledger = self._seeded_bulk_abandon_ledger() - - self.assertEqual( - ledger.mark_block_candidates_abandoned(block_hashes=[], error="unused"), - (), - ) - self.assertEqual( - ledger._block_candidate_outbox[BULK_PENDING_A]["state"], - "pending", - ) - self.assertEqual(len(ledger.pending_block_candidates()), 2) - - def test_pending_candidate_page_reports_pool_block_existence(self) -> None: - """The page read answers the landed-block question, not a per-row call. - - Both backends carry the fact on the row itself so a caller reading a - page never has to ask once per hash. The memory mirror reads the same - durable evidence the Postgres ``EXISTS`` does: a pool-block row exists - only for a candidate that reached ``persist_accepted_block``. - """ - ledger = self._seeded_bulk_abandon_ledger() - ledger._memory_pool_blocks[BULK_PENDING_B] = (10, "prepared", "00" * 32) - - rows = ledger.pending_block_candidate_rows(limit=32) - - self.assertEqual( - {row["block_hash"]: row["pool_block_exists"] for row in rows}, - {BULK_PENDING_A: False, BULK_PENDING_B: True}, - ) - - def test_bulk_candidate_abandon_spares_a_landed_candidate(self) -> None: - """A pool-block row vetoes the transition even inside the page. - - The caller's page could have been read before the candidate landed, - so the veto has to live in the write itself. A row that acquired one - is simply absent from the report, which is exactly how the caller - learns not to clean it up. - """ - ledger = self._seeded_bulk_abandon_ledger() - ledger._memory_pool_blocks[BULK_PENDING_B] = (10, "prepared", "00" * 32) - - abandoned = ledger.mark_block_candidates_abandoned( - block_hashes=[BULK_PENDING_A, BULK_PENDING_B], - error="superseded by decided height", - ) - - self.assertEqual(abandoned, (BULK_PENDING_A,)) - landed = ledger._block_candidate_outbox[BULK_PENDING_B] - self.assertEqual(landed["state"], "pending") - self.assertIsNone(landed["last_error"]) - self.assertIsNotNone(landed["candidate"]) - - def test_bulk_candidate_abandon_decides_a_page_under_one_lock_hold(self) -> None: - """One page is one atomic decision, matching the Postgres statement. - - The Postgres backend resolves a whole page inside a single fenced - statement, so a memory backend that released and re-took the lock per - hash would let a concurrent writer split the reported set across two - views of the outbox -- a parity gap the returned set would then hide. - """ - ledger = self._seeded_bulk_abandon_ledger() - counting = CountingLock(ledger._lock) - ledger._lock = counting - - abandoned = ledger.mark_block_candidates_abandoned( - block_hashes=[BULK_PENDING_A, BULK_PENDING_B, BULK_MISSING], - error="superseded by decided height", - ) - - self.assertEqual(abandoned, (BULK_PENDING_A, BULK_PENDING_B)) - self.assertEqual(counting.acquisitions, 1) - - def test_pending_candidate_cursor_pages_walk_a_stable_total_order(self) -> None: - """Startup enumeration must complete for a backlog of any size. - - The doubling window it replaces fails closed once one query would - have to hold the whole backlog, which blocks every job build until - the outbox drains on its own. Pagination needs three properties for - that gate to be safe: one stable total order, a cursor that resumes - strictly after its own row, and a short page that proves nothing - followed it. - """ - ledger = SingleWriterShareLedger() - hashes = [f"{index:02x}" * 32 for index in (0xA1, 0xA2, 0xB1, 0xB2, 0xC1)] - # Inserted out of order so the assertions below prove the ordering - # comes from the row keys and not from the outbox's insertion order. - for index in (2, 4, 0, 3, 1): - ledger.persist_block_candidate_intent( - { - "schema": "qbit.prism.block-candidate-intent.v1", - "block_hash_hex": hashes[index], - "block_hex": "00", - } - ) - # Colliding creation stamps are the interesting case: time.monotonic - # (like one transaction's clock_timestamp) repeats, so ordering on - # the stamp alone would let a cursor replay or skip a whole group. - stamps = { - hashes[0]: 100.0, - hashes[1]: 100.0, - hashes[2]: 100.0, - hashes[3]: 200.0, - hashes[4]: 300.0, - } - for block_hash, stamp in stamps.items(): - ledger._block_candidate_outbox[block_hash]["created_monotonic"] = stamp - - walked: list[str] = [] - pages: list[int] = [] - after_cursor: object | None = None - while True: - page = ledger.pending_block_candidate_rows( - limit=2, - after_cursor=after_cursor, - ) - pages.append(len(page)) - walked.extend(str(row["block_hash"]) for row in page) - if len(page) < 2: - break - after_cursor = page[-1]["cursor"] - - # Total order: creation stamp first, block hash as the tiebreak. - self.assertEqual(walked, sorted(hashes, key=lambda h: (stamps[h], h))) - # Full pages until one short page proves the walk reached the end. - self.assertEqual(pages, [2, 2, 1]) - # No row was served twice, and the walk saw every pending row. - self.assertEqual(len(set(walked)), len(hashes)) - - # A cursor resumes strictly after its own row, including when its - # stamp collides with the next row's. - tied_cursor = [100.0, hashes[0]] - self.assertEqual( - [ - str(row["block_hash"]) - for row in ledger.pending_block_candidate_rows( - limit=32, - after_cursor=tied_cursor, - ) - ], - [hashes[1], hashes[2], hashes[3], hashes[4]], - ) - # The cursor round-trips: handing back a returned cursor verbatim is - # equivalent to reconstructing it from the row's own ordering parts. - last_first_page = ledger.pending_block_candidate_rows(limit=1)[0] - self.assertEqual(last_first_page["cursor"], [100.0, hashes[0]]) - # A cursor past every row proves completion with an empty page. - self.assertEqual( - ledger.pending_block_candidate_rows( - limit=32, - after_cursor=[300.0, hashes[4]], - ), - [], - ) - # Terminal rows leave the pending order entirely. - self.assertTrue( - ledger.mark_block_candidate_submitted(block_hash=hashes[1]) - ) - self.assertNotIn( - hashes[1], - [ - str(row["block_hash"]) - for row in ledger.pending_block_candidate_rows(limit=32) - ], - ) - - def test_pending_candidate_cursor_rejects_unusable_shapes(self) -> None: - """A cursor crosses process and JSON boundaries, so it is validated.""" - ledger = SingleWriterShareLedger() - for cursor in ( - [], - [1.0], - [1.0, "ab" * 32, "extra"], - [1.0, ""], - ["not-a-stamp", "ab" * 32], - "1.0,ab", - ): - with self.assertRaises(ValueError): - ledger.pending_block_candidate_rows(after_cursor=cursor) - - def test_stranded_prepared_blocks_filter_by_depth_and_state(self) -> None: - """The sweep's read only surfaces rows an operator-free heal may touch. - - reorg_watch_blocks selects confirmed/inactive rows only, so a - prepared row whose outbox entry is gone has nothing left to - re-examine it. The depth floor is what separates that from a - finalization still legitimately in flight. - """ - ledger = SingleWriterShareLedger() - heights = { - "deep": ("a1" * 32, 100), - "at_floor": ("a2" * 32, 900), - "shallow": ("a3" * 32, 901), - "confirmed": ("a4" * 32, 200), - } - for block_hash, block_height in heights.values(): - ledger.persist_accepted_block( - block_hash=block_hash, - block_height=block_height, - parent_hash="bb" * 32, - final_bundle={}, - audit_report={}, - ) - confirmed_hash, confirmed_height = heights["confirmed"] - self.assertEqual( - ledger.confirm_accepted_block( - block_hash=confirmed_hash, - active_tip_height=confirmed_height, - )["confirmed_count"], - 1, - ) - - stranded = ledger.stranded_prepared_blocks( - active_tip_height=1000, - min_depth=100, - ) - - self.assertEqual( - [(row["block_hash"], row["block_height"]) for row in stranded], - [ - (heights["deep"][0], heights["deep"][1]), - (heights["at_floor"][0], heights["at_floor"][1]), - ], - ) - self.assertEqual(stranded[0]["parent_hash"], "bb" * 32) - # A confirmed row belongs to the watch loop, not this sweep, and a - # row shallower than the floor may still be finalizing. - returned = {str(row["block_hash"]) for row in stranded} - self.assertNotIn(heights["confirmed"][0], returned) - self.assertNotIn(heights["shallow"][0], returned) - # The limit bounds one pass; the remainder waits for the next one. - self.assertEqual( - [ - row["block_hash"] - for row in ledger.stranded_prepared_blocks( - active_tip_height=1000, - min_depth=100, - limit=1, - ) - ], - [heights["deep"][0]], - ) - # Rejecting through the fenced method retires the row from the sweep. - self.assertEqual( - ledger.reject_prepared_block( - block_hash=heights["deep"][0], - active_tip_height=1000, - )["rejected_count"], - 1, - ) - self.assertEqual( - [ - row["block_hash"] - for row in ledger.stranded_prepared_blocks( - active_tip_height=1000, - min_depth=100, - ) - ], - [heights["at_floor"][0]], - ) - - def test_batch_validation_is_all_or_nothing(self) -> None: - ledger = SingleWriterShareLedger() - first = pending_share(1) - bad_duplicate = pending_share(2).__class__( - **{**pending_share(2).__dict__, "share_id": first.share_id} - ) - ledger.append(first) - - with self.assertRaisesRegex(ValueError, "payload mismatch"): - ledger.append_batch([(pending_share(3), None), (bad_duplicate, None)]) - - self.assertEqual(len(ledger), 1) - - def test_candidate_only_intent_links_to_share_after_block_lands(self) -> None: - ledger = SingleWriterShareLedger() - share = pending_share(1) - intent = { - "schema": "qbit.prism.block-candidate-intent.v1", - "block_hash_hex": "cd" * 32, - "block_hex": "00", - "credit_share_on_accept": True, - } - - self.assertTrue(ledger.persist_block_candidate_intent(intent)) - self.assertEqual(ledger.pending_block_candidates(), [intent]) - self.assertEqual(ledger.append_batch([(share, intent)])[0].share_seq, 1) - self.assertTrue(ledger.mark_block_candidate_submitted(block_hash="cd" * 32)) - self.assertEqual(ledger.pending_block_candidates(), []) - - def test_candidate_retry_with_new_acknowledgment_stamp_is_idempotent(self) -> None: - # A miner can resubmit the same solved block after a transient submit - # outcome, and the rebuilt intent differs from the persisted one only - # in pending_share.accepted_at_ms. The outbox must treat that as the - # same work and keep the first payload authoritative, while any other - # divergence stays a hard payload mismatch. - ledger = SingleWriterShareLedger() - share = pending_share(1, accepted_at_ms=2_000) - intent = { - "schema": "qbit.prism.block-candidate-intent.v1", - "block_hash_hex": "ef" * 32, - "block_hex": "00", - "pending_share": dict(share.__dict__), - "credit_share_on_accept": True, - } - retry_share = pending_share(1, accepted_at_ms=2_042) - retry_intent = {**intent, "pending_share": dict(retry_share.__dict__)} - - first_persist = ledger.persist_block_candidate_intent(intent) - retry_persist = ledger.persist_block_candidate_intent(retry_intent) - self.assertTrue(first_persist) - self.assertEqual(first_persist.state, "pending") - self.assertFalse(retry_persist) - self.assertEqual(retry_persist.state, "pending") - self.assertEqual(ledger.pending_block_candidates(), [intent]) - with self.assertRaisesRegex(ValueError, "candidate payload mismatch"): - ledger.persist_block_candidate_intent({**retry_intent, "block_hex": "01"}) - - # A retry that lands links its share to the first persisted payload. - self.assertEqual( - ledger.append_batch([(retry_share, retry_intent)])[0].share_seq, 1 - ) - self.assertEqual( - [ - {key: value for key, value in row.items() if key != "cursor"} - for row in ledger.pending_block_candidate_rows() - ], - [ - { - "block_hash": "ef" * 32, - "candidate": intent, - "pool_block_exists": False, - } - ], - ) - self.assertTrue(ledger.mark_block_candidate_submitted(block_hash="ef" * 32)) - terminal_persist = ledger.persist_block_candidate_intent(retry_intent) - self.assertFalse(terminal_persist) - self.assertEqual(terminal_persist.state, "submitted") - - def test_concurrent_append_still_has_one_canonical_sequence(self) -> None: - ledger = SingleWriterShareLedger() - - def append_range(start: int) -> None: - for index in range(start, start + 10): - ledger.append(pending_share(index)) - - threads = [threading.Thread(target=append_range, args=(start,)) for start in (1, 100, 200)] - for thread in threads: - thread.start() - for thread in threads: - thread.join() - - records = ledger.all_shares() - self.assertEqual(len(records), 30) - self.assertEqual(sorted(record.share_seq for record in records), list(range(1, 31))) - - def test_memory_ledger_persists_ctv_fanout_recovery_artifact(self) -> None: - ledger = SingleWriterShareLedger() - block_hash = "aa" * 32 - manifest_set = sample_ctv_manifest_set() - manifest_set_sha256 = "66" * 32 - - first = ledger.persist_ctv_fanout_manifest_set( - block_hash=block_hash, - manifest_set=manifest_set, - manifest_set_sha256=manifest_set_sha256, - ) - second = ledger.persist_ctv_fanout_manifest_set( - block_hash=block_hash, - manifest_set=manifest_set, - manifest_set_sha256=manifest_set_sha256, - ) - recovery = ledger.audit_ctv_fanout_manifest_set(block_hash=block_hash) - rows = ledger.audit_ctv_fanouts(block_hash=block_hash) - - self.assertEqual(first["fanout_artifact_count"], 1) - self.assertEqual(second["fanout_artifact_count"], 1) - self.assertIsNotNone(recovery) - self.assertEqual(recovery["manifest_set_sha256"], manifest_set_sha256) # type: ignore[index] - self.assertEqual(recovery["block_height"], 123450) # type: ignore[index] - self.assertIn("manifest_set_json", recovery) # type: ignore[operator] - self.assertEqual(rows[0]["fanout_txid"], "22" * 32) - self.assertEqual(rows[0]["block_height"], 123450) - self.assertEqual(rows[0]["settlement_status"], "awaiting_maturity") - self.assertEqual( - ledger.ctv_fanout_status(fanout_txid="22" * 32)["block_height"], # type: ignore[index, union-attr] - 123450, - ) - - def test_memory_ledger_persists_no_anchor_fee_fanout_recovery_artifact(self) -> None: - ledger = SingleWriterShareLedger() - block_hash = "ab" * 32 - manifest_set = sample_no_anchor_fee_ctv_manifest_set() - - ledger.persist_ctv_fanout_manifest_set( - block_hash=block_hash, - manifest_set=manifest_set, - manifest_set_sha256="67" * 32, - ) - rows = ledger.audit_ctv_fanouts(block_hash=block_hash) - status = ledger.ctv_fanout_status(fanout_txid="22" * 32) - - self.assertIsNone(rows[0]["anchor_vout"]) - self.assertIsNone(status["anchor_vout"]) # type: ignore[index] - self.assertEqual(rows[0]["covenant_output_value_sats"], 25_000) - self.assertEqual(rows[0]["fanout_output_sum_sats"], 24_900) - - def test_memory_ledger_rejects_built_in_fee_fanout_with_anchor(self) -> None: - ledger = SingleWriterShareLedger() - manifest_set = sample_no_anchor_fee_ctv_manifest_set() - precommitment = manifest_set["manifests"][0]["precommitment"] # type: ignore[index] - precommitment["anchor_vout"] = 1 # type: ignore[index] - - with self.assertRaisesRegex(ValueError, "must not include a CPFP anchor"): - ledger.persist_ctv_fanout_manifest_set( - block_hash="ac" * 32, - manifest_set=manifest_set, - manifest_set_sha256="68" * 32, - ) - - def test_ctv_backfill_input_extracts_manifest_set_from_audit_bundle(self) -> None: - manifest_set = sample_no_anchor_fee_ctv_manifest_set() - - item = backfill_input_from_payload( - {"ctv_fanout_manifest_set": manifest_set}, - source="audit.json", - block_hash="aa" * 32, - ) - - self.assertEqual(item.source, "audit.json") - self.assertEqual(item.block_hash, "aa" * 32) - self.assertEqual(item.manifest_set, manifest_set) - self.assertEqual(item.manifest_set_sha256, sha256_json_hex(manifest_set)) - - def test_ctv_backfill_input_accepts_audit_api_wrapper(self) -> None: - manifest_set = sample_no_anchor_fee_ctv_manifest_set() - - item = backfill_input_from_payload( - { - "block_hash": "bb" * 32, - "manifest_set": manifest_set, - "manifest_set_sha256": "cc" * 32, - }, - source="api", - ) - - self.assertEqual(item.block_hash, "bb" * 32) - self.assertEqual(item.manifest_set_sha256, "cc" * 32) - - def test_ctv_backfill_infers_block_hash_from_live_bundle_filename(self) -> None: - self.assertEqual( - infer_block_hash_from_path(Path(f"prism-live-audit-bundle-21886-{'dd' * 32}.json")), - "dd" * 32, - ) - - def test_ctv_backfill_path_follows_live_envelope_to_compact_body_ref(self) -> None: - manifest_set = sample_no_anchor_fee_ctv_manifest_set() - with tempfile.TemporaryDirectory() as tmp: - root = Path(tmp) - body_ref = { - "schema": AUDIT_BODY_REF_SCHEMA, - "bundle_without_shares": { - "schema": "qbit.prism.audit-bundle.v1", - "found_block": {"block_hash": "aa" * 32}, - "ctv_fanout_manifest_set": manifest_set, - }, - "share_parts": [], - "share_count": 0, - } - body_path = root / f"prism-audit-bundle-body-{'aa' * 32}-{'bb' * 32}.json" - body_path.write_text(json.dumps(body_ref), encoding="utf-8") - envelope_path = root / f"prism-live-audit-bundle-10-{'aa' * 32}.json" - envelope_path.write_text( - json.dumps( - { - "schema": "qbit.prism.live-audit-bundle-envelope.v1", - "block_hash": "aa" * 32, - "body_uri": str(body_path), - } - ), - encoding="utf-8", - ) - - item = backfill_input_from_path(envelope_path) - - self.assertEqual(item.block_hash, "aa" * 32) - self.assertEqual(item.manifest_set, manifest_set) - - def test_ctv_backfill_requires_block_hash(self) -> None: - with self.assertRaisesRegex(ValueError, "block hash"): - backfill_input_from_payload( - {"ctv_fanout_manifest_set": sample_no_anchor_fee_ctv_manifest_set()}, - source="audit.json", - ) - - def test_memory_ledger_public_artifact_returns_ctv_audit_bundle(self) -> None: - ledger = SingleWriterShareLedger() - block_hash = "aa" * 32 - audit_bundle_sha256 = "77" * 32 - manifest_set = { - **sample_ctv_manifest_set(), - "audit_bundle_sha256": audit_bundle_sha256, - "audit_bundle": {"schema": "qbit.prism.audit-bundle.v1"}, - } - - ledger.persist_ctv_fanout_manifest_set( - block_hash=block_hash, - manifest_set=manifest_set, - manifest_set_sha256="66" * 32, - ) - - self.assertEqual( - ledger.dashboard_public_artifact(sha256=audit_bundle_sha256), - {"schema": "qbit.prism.audit-bundle.v1"}, - ) - status = ledger.ctv_fanout_status(fanout_txid="22" * 32) - pending = ledger.pending_ctv_fanout_statuses() - - self.assertEqual(status["audit_bundle_sha256"], audit_bundle_sha256) # type: ignore[index] - self.assertEqual(pending[0]["audit_bundle_sha256"], audit_bundle_sha256) - - def test_memory_ledger_public_artifact_omits_missing_ctv_audit_bundle_body(self) -> None: - ledger = SingleWriterShareLedger() - block_hash = "aa" * 32 - audit_bundle_sha256 = "77" * 32 - manifest_set = { - **sample_ctv_manifest_set(), - "audit_bundle_sha256": audit_bundle_sha256, - } - - ledger.persist_ctv_fanout_manifest_set( - block_hash=block_hash, - manifest_set=manifest_set, - manifest_set_sha256="66" * 32, - ) - - self.assertIsNone(ledger.dashboard_public_artifact(sha256=audit_bundle_sha256)) - - def test_memory_ledger_public_artifact_document_returns_hashed_text(self) -> None: - # The canonical text is the exact byte sequence the advertised sha256 - # was computed over; serving anything else breaks external - # verification of content-addressed artifacts. - ledger = SingleWriterShareLedger() - block_hash = "aa" * 32 - audit_bundle_sha256 = "77" * 32 - manifest_set = { - **sample_ctv_manifest_set(), - "audit_bundle_sha256": audit_bundle_sha256, - "audit_bundle": {"schema": "qbit.prism.audit-bundle.v1"}, - } - manifest = manifest_set["manifests"][0] # type: ignore[index] - manifest_set_sha256 = sha256_json_hex(manifest_set) - - ledger.persist_ctv_fanout_manifest_set( - block_hash=block_hash, - manifest_set=manifest_set, - manifest_set_sha256=manifest_set_sha256, - ) - - set_document = ledger.dashboard_public_artifact_document(sha256=manifest_set_sha256) - manifest_document = ledger.dashboard_public_artifact_document(sha256=sha256_json_hex(manifest)) - bundle_document = ledger.dashboard_public_artifact_document(sha256=audit_bundle_sha256) - - self.assertEqual(set_document["canonical_json"], canonical_json_text(manifest_set)) # type: ignore[index] - self.assertEqual(set_document["payload"], manifest_set) # type: ignore[index] - self.assertEqual(manifest_document["canonical_json"], canonical_json_text(manifest)) # type: ignore[index] - self.assertEqual( - hashlib.sha256(str(set_document["canonical_json"]).encode()).hexdigest(), # type: ignore[index] - manifest_set_sha256, - ) - self.assertEqual(bundle_document["payload"], {"schema": "qbit.prism.audit-bundle.v1"}) # type: ignore[index] - self.assertIsNone(bundle_document["canonical_json"]) # type: ignore[index] - self.assertIsNone(ledger.dashboard_public_artifact_document(sha256="99" * 32)) - - def test_reorg_watch_blocks_keeps_height_mature_immature_rows(self) -> None: - ledger = QueryCapturePsqlShareLedger() - - self.assertEqual(ledger.reorg_watch_blocks(active_tip_height=10_000), []) - - self.assertIn("chain_state IN ('confirmed', 'inactive')", ledger.queries[0]) - self.assertIn("maturity_state = 'immature'", ledger.queries[0]) - self.assertNotIn("block_height + 1000", ledger.queries[0]) - - def test_pool_block_state_wraps_nullable_row_in_json(self) -> None: - ledger = CannedQueryPsqlShareLedger( - [ - acquired_lease_result(), - {"state": None}, - { - "state": { - "block_hash": "aa" * 32, - "block_height": 10, - "parent_hash": "bb" * 32, - "chain_state": "confirmed", - "maturity_state": "immature", - "audit_publication_sequence": None, - } - }, - ] - ) - - self.assertIsNone(ledger.pool_block_state(block_hash="aa" * 32)) - self.assertEqual( - ledger.pool_block_state(block_hash="aa" * 32), - { - "block_hash": "aa" * 32, - "block_height": 10, - "parent_hash": "bb" * 32, - "chain_state": "confirmed", - "maturity_state": "immature", - "audit_publication_sequence": None, - }, - ) - self.assertIn("SELECT json_build_object(\n 'state'", ledger.queries[1]) - - def test_postgres_publication_floor_reads_max_durable_row_ordinal(self) -> None: - ledger = CannedQueryPsqlShareLedger( - [ - acquired_lease_result(), - {"audit_publication_sequence_floor": 7}, - ] - ) - - self.assertEqual(ledger.audit_publication_sequence_floor(), 7) - query = ledger.queries[1] - self.assertIn("MAX(audit_publication_sequence)", query) - self.assertIn("COALESCE", query) - self.assertIn("FROM qbit_pool_blocks", query) - self.assertNotIn("last_value", query) - - for invalid in (None, True, "7", -1): - with self.subTest(invalid=invalid): - invalid_ledger = CannedQueryPsqlShareLedger( - [ - acquired_lease_result(), - {"audit_publication_sequence_floor": invalid}, - ] - ) - with self.assertRaisesRegex(RuntimeError, "floor is invalid"): - invalid_ledger.audit_publication_sequence_floor() - - def test_prior_balances_after_pool_block_is_height_bounded(self) -> None: - balance = { - "recipient_id": "miner-a", - "order_key": "miner-a", - "p2mr_program_hex": "11" * 32, - "balance_sats": "25", - } - ledger = CannedQueryPsqlShareLedger( - [acquired_lease_result(), [balance], []] - ) - - self.assertEqual( - ledger.prior_balances_after_pool_block(block_hash="aa" * 32), - [{**balance, "balance_sats": 25}], - ) - query = ledger.queries[-1] - self.assertIn("block.block_height <= target.block_height", query) - self.assertIn("block.chain_state = 'confirmed'", query) - self.assertIn("carry.maturity_state <> 'reversed'", query) - self.assertIn("ORDER BY payout_order_key, miner_id", query) - self.assertEqual( - ledger.prior_balances_after_pool_block(block_hash="bb" * 32), - [], - ) - - def test_memory_ledger_rejects_mutated_ctv_fanout_artifact(self) -> None: - ledger = SingleWriterShareLedger() - block_hash = "aa" * 32 - manifest_set = sample_ctv_manifest_set() - ledger.persist_ctv_fanout_manifest_set( - block_hash=block_hash, - manifest_set=manifest_set, - manifest_set_sha256="66" * 32, - ) - - with self.assertRaisesRegex(RuntimeError, "existing CTV fanout manifest set does not match payload"): - ledger.persist_ctv_fanout_manifest_set( - block_hash=block_hash, - manifest_set=manifest_set, - manifest_set_sha256="67" * 32, - ) - - def test_memory_ledger_preserves_ctv_fanout_broadcast_attempts(self) -> None: - ledger = SingleWriterShareLedger() - block_hash = "aa" * 32 - fanout_txid = "22" * 32 - ledger.persist_ctv_fanout_manifest_set( - block_hash=block_hash, - manifest_set=sample_ctv_manifest_set(), - manifest_set_sha256="66" * 32, - ) - - ledger.record_ctv_fanout_broadcast_attempt( - fanout_txid=fanout_txid, - attempt_status="rejected", - package_tx_hexes=["parent", "child"], - package_txids=[fanout_txid, "77" * 32], - submit_result={"tx-results": {"child": {"error": "insufficient fee"}}}, - error="insufficient fee", - ) - status = ledger.ctv_fanout_status(fanout_txid=fanout_txid) - pending = ledger.pending_ctv_fanout_statuses() - - self.assertIsNotNone(status) - self.assertEqual(status["settlement_status"], "failed") # type: ignore[index] - self.assertEqual(status["broadcast_attempt_count"], 1) # type: ignore[index] - self.assertEqual(status["broadcast_attempt_detail_count"], 1) # type: ignore[index] - self.assertEqual(status["last_broadcast_attempt_status"], "rejected") # type: ignore[index] - self.assertEqual(status["last_broadcast_error"], "insufficient fee") # type: ignore[index] - self.assertEqual(status["broadcast_attempt_summary"]["attempt_count"], 1) # type: ignore[index] - self.assertEqual(status["broadcast_attempts"][0]["attempt_status"], "rejected") # type: ignore[index] - self.assertEqual(status["broadcast_attempts"][0]["package_tx_hexes"], ["parent", "child"]) # type: ignore[index] - self.assertEqual(pending, []) - - ledger.update_ctv_fanout_status(fanout_txid=fanout_txid, settlement_status="confirmed") - self.assertEqual(ledger.pending_ctv_fanout_statuses(), []) - - def test_memory_ledger_caps_ctv_fanout_broadcast_attempt_details(self) -> None: - ledger = SingleWriterShareLedger(ctv_broadcast_attempt_detail_limit=2) - fanout_txid = "22" * 32 - ledger.persist_ctv_fanout_manifest_set( - block_hash="aa" * 32, - manifest_set=sample_ctv_manifest_set(), - manifest_set_sha256="66" * 32, - ) - - for index in range(4): - ledger.record_ctv_fanout_broadcast_attempt( - fanout_txid=fanout_txid, - attempt_status="planned", - package_txids=[fanout_txid], - submit_result={"package_msg": "error", "submitted": False}, - error=f"transient-{index}", - ) - - status = ledger.ctv_fanout_status(fanout_txid=fanout_txid) - assert status is not None - self.assertEqual(status["broadcast_attempt_count"], 4) - self.assertEqual(status["broadcast_attempt_detail_count"], 2) - self.assertEqual(status["last_broadcast_error"], "transient-3") - self.assertEqual([row["attempt_seq"] for row in status["broadcast_attempts"]], [3, 4]) - self.assertEqual(status["broadcast_attempt_summary"]["status_counts"]["planned"], 4) # type: ignore[index] - - def test_memory_ledger_ctv_broadcast_pending_respects_retry_backoff(self) -> None: - ledger = SingleWriterShareLedger(ctv_broadcast_retry_backoff_seconds=300) - fanout_txid = "22" * 32 - ledger.persist_ctv_fanout_manifest_set( - block_hash="aa" * 32, - manifest_set=sample_ctv_manifest_set(), - manifest_set_sha256="66" * 32, - ) - - ledger.record_ctv_fanout_broadcast_attempt( - fanout_txid=fanout_txid, - attempt_status="planned", - package_txids=[fanout_txid], - submit_result={"package_msg": "error", "submitted": False}, - error="transient", - ) - self.assertEqual(ledger.pending_ctv_fanout_statuses(), []) - self.assertEqual(ledger.dashboard_pending_fanout_rows(page=1, limit=10)["rows"], []) - - ledger._ctv_fanout_statuses[fanout_txid]["next_broadcast_attempt_at"] = ( # type: ignore[attr-defined] - datetime.now(timezone.utc) - timedelta(seconds=1) - ) - self.assertEqual(ledger.pending_ctv_fanout_statuses()[0]["fanout_txid"], fanout_txid) - self.assertEqual( - ledger.dashboard_pending_fanout_rows(page=1, limit=10)["rows"][0]["fanout_txid"], - fanout_txid, - ) - - def test_postgres_miner_worker_query_treats_percent_and_underscore_literally(self) -> None: - ledger = FakeLeasePsqlShareLedger( - [ - acquired_lease(), - {"total_count": 0, "active_count": 0, "rows": []}, - ] - ) - - payload = ledger.dashboard_miner_worker_rows( - recipient_id="miner_%", - page=1, - limit=15, - search="rig_%", - hide_inactive=False, - ) - query = ledger.lease_queries[-1] - - self.assertEqual(payload["pagination"], {"page": 1, "limit": 15, "total_count": 0, "total_pages": 0}) - self.assertIn("strpos(lower(worker_name), 'rig_%') > 0", query) - self.assertIn("left(username, 8) = 'miner_%.'", query) - self.assertNotIn("lower(worker_name) LIKE", query) - self.assertNotIn("username LIKE", query) - - def test_postgres_miner_worker_query_bounds_scan_to_largest_reported_window(self) -> None: - ledger = FakeLeasePsqlShareLedger( - [ - acquired_lease(), - {"total_count": 0, "active_count": 0, "rows": []}, - ] - ) - - ledger.dashboard_miner_worker_rows( - recipient_id="miner-a", - page=1, - limit=15, - search=None, - hide_inactive=False, - ) - query = ledger.lease_queries[-1] - - # The share scan must carry the 3-hour lower bound (the largest rollup - # window the endpoint reports) inside the ledger subquery itself, not - # only in the per-window FILTER clauses that follow it. - bound = "accepted_at >= (SELECT now_at FROM bounds) - interval '3 hours'" - self.assertIn(bound, query) - self.assertLess(query.index("FROM qbit_share_ledger"), query.index(bound)) - self.assertLess(query.index(bound), query.index(") shares")) - - def test_writer_lease_ttl_is_configurable_in_acquire_sql(self) -> None: - ledger = FakeLeasePsqlShareLedger([acquired_lease()], lease_ttl_seconds=42) - - self.assertEqual(ledger._lease_interval_sql, "make_interval(secs => 42.0)") - self.assertIn("make_interval(secs => 42.0)", ledger.lease_queries[0]) - self.assertNotIn("interval '5 minutes'", ledger.lease_queries[0]) - - def test_postgres_batch_sql_fences_share_and_candidate_in_one_statement(self) -> None: - share = pending_share(1) - record = { - "share_seq": 7, - "share_id": share.share_id, - "miner_id": share.miner_id, - "order_key": share.order_key, - "p2mr_program_hex": share.p2mr_program_hex, - "share_difficulty": str(share.share_difficulty), - "network_difficulty": str(share.network_difficulty), - "template_height": share.template_height, - "job_id": share.job_id, - "job_issued_at_ms": share.job_issued_at_ms, - "accepted_at_ms": share.accepted_at_ms, - "ntime": share.ntime, - "credit_policy": share.credit_policy, - } - ledger = FakeLeasePsqlShareLedger( - [acquired_lease(), {"records": [record]}] - ) - # This pins the legacy whole-jsonb statement (storage version 1); - # the chunked-body statement is covered by the #255 suites. - ledger._candidate_storage_version = 1 - intent = { - "schema": "qbit.prism.block-candidate-intent.v1", - "block_hash_hex": "ab" * 32, - "block_hex": "00", - } - - self.assertEqual(ledger.append_batch([(share, intent)])[0].share_seq, 7) - query = ledger.lease_queries[-1] - self.assertIn("inserted_shares AS", query) - self.assertIn("inserted_candidates AS", query) - self.assertIn("qbit_block_candidate_outbox", query) - self.assertIn("duplicate share_id payload mismatch", query) - self.assertIn("'candidate_outbox_state'", query) - self.assertNotIn("ledger.accepted_at IS DISTINCT", query) - self.assertEqual(query.count("SELECT CASE"), 1) - - def test_postgres_recovery_append_returns_exact_existing_from_batch_comparator(self) -> None: - share = pending_share(2) - record = { - "share_seq": 8, - "share_id": share.share_id, - "miner_id": share.miner_id, - "order_key": share.order_key, - "p2mr_program_hex": share.p2mr_program_hex, - "share_difficulty": str(share.share_difficulty), - "network_difficulty": str(share.network_difficulty), - "template_height": share.template_height, - "job_id": share.job_id, - "job_issued_at_ms": share.job_issued_at_ms, - "accepted_at_ms": share.accepted_at_ms, - "ntime": share.ntime, - "credit_policy": share.credit_policy, - "newly_inserted": False, - } - ledger = FakeLeasePsqlShareLedger( - [acquired_lease(), {"records": [record]}] - ) - - outcome = ledger.append_recovered_share(share) - - self.assertEqual(outcome.disposition, "exact_existing") - self.assertEqual(outcome.record.share_seq, 8) - self.assertIn("share_mismatch AS", ledger.lease_queries[-1]) - - def test_postgres_recovery_append_raises_typed_payload_conflict(self) -> None: - ledger = FakeLeasePsqlShareLedger( - [ - acquired_lease(), - { - "error": "duplicate share_id payload mismatch", - "error_kind": "share_replay_conflict", - }, - ] - ) - - with self.assertRaises(ShareReplayConflict): - ledger.append_recovered_share(pending_share(3)) - - def test_postgres_candidate_only_intent_forces_durable_fenced_commit(self) -> None: - ledger = FakeLeasePsqlShareLedger( - [acquired_lease(), {"inserted": 1}] - ) - # Legacy whole-jsonb statement (storage version 1); see #255 suites - # for the chunked-body route. - ledger._candidate_storage_version = 1 - intent = { - "schema": "qbit.prism.block-candidate-intent.v1", - "block_hash_hex": "cd" * 32, - "block_hex": "00", - "credit_share_on_accept": True, - } - - self.assertTrue(ledger.persist_block_candidate_intent(intent)) - query = ledger.lease_queries[-1] - self.assertIn("set_config('synchronous_commit', 'on', true)", query) - self.assertIn("qbit_ledger_writer_lease", query) - self.assertIn("qbit_block_candidate_outbox", query) - self.assertIn("SELECT candidate_sha256, state", query) - - def test_postgres_pending_candidate_rows_keep_authoritative_outbox_key(self) -> None: - intent = { - "schema": "unsupported", - "block_hex": "00", - } - cursor = ["2026-07-08T21:02:03.123456Z", "ef" * 32] - ledger = FakeLeasePsqlShareLedger( - [ - acquired_lease(), - [ - { - "block_hash": "ef" * 32, - "candidate": intent, - "pool_block_exists": False, - "cursor": cursor, - } - ], - ] - ) - - self.assertEqual( - ledger.pending_block_candidate_rows(), - [ - { - "block_hash": "ef" * 32, - "candidate": intent, - "pool_block_exists": False, - "cursor": cursor, - } - ], - ) - query = ledger.lease_queries[-1] - self.assertIn("'block_hash', pending.block_hash,", query) - self.assertIn("'candidate', pending.candidate,", query) - # The cursor is the two ordering columns, and its stamp keeps the - # microsecond precision the column stores. - self.assertIn( - "'cursor', json_build_array(\n pending.cursor_created_at,\n pending.block_hash\n )", - query, - ) - self.assertIn( - "'YYYY-MM-DD\"T\"HH24:MI:SS.US\"Z\"'", - query, - ) - self.assertIn("ORDER BY created_at, block_hash", query) - # A first page carries no keyset predicate at all. - self.assertNotIn("(created_at, block_hash) >", query) - - def test_postgres_pending_candidate_page_uses_an_indexable_keyset(self) -> None: - """Each page must stay one bounded range scan of the partial index. - - qbit_block_candidate_outbox_pending_idx is (created_at, block_hash) - WHERE state = 'pending'. A row-comparison predicate on exactly those - columns is what keeps page N of a 5,000-row backlog as cheap as page - one; an OFFSET walk, or a predicate on either column alone, would - not be. - """ - ledger = FakeLeasePsqlShareLedger([acquired_lease(), []]) - - self.assertEqual( - ledger.pending_block_candidate_rows( - limit=1024, - after_cursor=["2026-07-08T21:02:03.123456Z", "ef" * 32], - ), - [], - ) - - query = ledger.lease_queries[-1] - self.assertIn( - "AND (created_at, block_hash) > " - "('2026-07-08T21:02:03.123456Z'::timestamptz, " - f"'{'ef' * 32}')", - query, - ) - self.assertIn("WHERE state = 'pending'", query) - self.assertIn("ORDER BY created_at, block_hash", query) - self.assertIn("LIMIT 1024", query) - # A hostile cursor is a literal like any other and is escaped, not - # interpolated raw. - ledger.lease_results.append([]) - ledger.pending_block_candidate_rows( - after_cursor=["2026-07-08T21:02:03.123456Z", "ef' OR true --"], - ) - self.assertIn("'ef'' OR true --'", ledger.lease_queries[-1]) - - def test_postgres_stranded_prepared_blocks_bound_depth_and_page(self) -> None: - """The sweep read is depth-floored, state-filtered, and bounded.""" - ledger = FakeLeasePsqlShareLedger( - [ - acquired_lease(), - [ - { - "block_hash": "19" * 32, - "block_height": "30822", - "parent_hash": "20" * 32, - } - ], - ] - ) - - self.assertEqual( - ledger.stranded_prepared_blocks( - active_tip_height=31000, - min_depth=100, - limit=64, - ), - [ - { - "block_hash": "19" * 32, - "block_height": 30822, - "parent_hash": "20" * 32, - } - ], - ) - - query = ledger.lease_queries[-1] - self.assertIn("FROM qbit_pool_blocks", query) - # Leads with the qbit_pool_blocks_maturity_idx columns. - self.assertIn("WHERE maturity_state = 'immature'", query) - self.assertIn("AND block_height <= 30900", query) - self.assertIn("AND chain_state = 'prepared'", query) - self.assertIn("ORDER BY block_height ASC, block_hash ASC", query) - self.assertIn("LIMIT 64", query) - # Nothing is fetched at all when the caller allows no rows. - self.assertEqual( - ledger.stranded_prepared_blocks( - active_tip_height=31000, - min_depth=100, - limit=0, - ), - [], - ) - self.assertIs(ledger.lease_queries[-1], query) - - def test_postgres_pending_candidate_metrics_are_aggregate_and_indexable(self) -> None: - ledger = FakeLeasePsqlShareLedger( - [ - acquired_lease(), - { - "pending_count": 2, - "oldest_pending_age_seconds": "7.5", - "oldest_unattempted_age_seconds": "3.25", - }, - ] - ) - - self.assertEqual( - ledger.block_candidate_pending_metrics(), - { - "pending_count": 2, - "oldest_pending_age_seconds": 7.5, - "oldest_unattempted_age_seconds": 3.25, - }, - ) - query = ledger.lease_queries[-1] - self.assertIn("min(created_at)", query) - self.assertIn("FILTER (WHERE attempt_count = 0)", query) - self.assertIn("WHERE state = 'pending'", query) - - def test_postgres_candidate_attempt_is_writer_fenced(self) -> None: - ledger = FakeLeasePsqlShareLedger( - [acquired_lease(), {"updated": 1}], - writer_id="writer-a", - writer_epoch=7, - ) - - self.assertTrue( - ledger.mark_block_candidate_attempted(block_hash="ad" * 32) - ) - query = ledger.lease_queries[-1] - self.assertIn("qbit_ledger_writer_lease", query) - self.assertIn("writer_id = 'writer-a'", query) - self.assertIn("writer_epoch = 7", query) - self.assertIn("attempt_count = attempt_count + 1", query) - - def _bulk_abandon_query( - self, - *, - block_hashes: list[str], - error: str = "superseded by decided height", - abandoned: list[str] | None = None, - ) -> tuple[FakeLeasePsqlShareLedger, tuple[str, ...], str]: - ledger = FakeLeasePsqlShareLedger( - [acquired_lease(), {"abandoned": abandoned or []}], - writer_id="writer-a", - writer_epoch=7, - ) - result = ledger.mark_block_candidates_abandoned( - block_hashes=block_hashes, - error=error, - ) - return ledger, result, ledger.lease_queries[-1] - - def test_postgres_bulk_candidate_abandon_is_one_fenced_set_statement( - self, - ) -> None: - """One statement per page, carrying the same fence as the per-row path. - - The point of the batch form is that a storm-sized page costs one - fenced write rather than one per hash, so the shape assertions are - the behaviour: exactly one outbox ``UPDATE``, and a set predicate - rather than a disjunction of equalities. - """ - _, _, query = self._bulk_abandon_query( - block_hashes=[BULK_PENDING_A, BULK_PENDING_B], - ) - - self.assertEqual(query.count("UPDATE qbit_block_candidate_outbox"), 1) - self.assertIn("qbit_ledger_writer_lease", query) - self.assertIn("writer_id = 'writer-a'", query) - self.assertIn("writer_epoch = 7", query) - self.assertIn("writer_session_token = ", query) - self.assertIn( - f"block_hash = ANY(ARRAY['{BULK_PENDING_A}', '{BULK_PENDING_B}']::text[])", - query, - ) - self.assertIn("AND state = 'pending'", query) - self.assertIn("SET state = 'abandoned'", query) - self.assertIn("last_error = 'superseded by decided height'", query) - self.assertIn("updated_at = clock_timestamp()", query) - self.assertIn("completed_at = clock_timestamp()", query) - self.assertIn("candidate = NULL", query) - self.assertIn("RETURNING block_hash", query) - self.assertIn("'writer lease is not active'", query) - - def test_postgres_bulk_candidate_abandon_re_checks_pool_blocks(self) -> None: - """The landed-block fact is re-asked inside the write, not trusted. - - ``pool_block_exists`` from a page read is a fact about the moment of - that read. A candidate can land between the read and the write, and - the pool-block row is what records that it did, so the terminal - statement carries its own ``NOT EXISTS`` arm under the writer fence. - """ - _, _, query = self._bulk_abandon_query(block_hashes=[BULK_PENDING_A]) - - self.assertIn("AND NOT EXISTS (", query) - self.assertIn("FROM qbit_pool_blocks pool", query) - self.assertIn( - "WHERE pool.block_hash = qbit_block_candidate_outbox.block_hash", - query, - ) - - def test_postgres_bulk_candidate_abandon_returns_the_rows_it_won(self) -> None: - """``RETURNING`` decides the report, so the request cannot inflate it.""" - _, abandoned, _ = self._bulk_abandon_query( - block_hashes=[BULK_PENDING_A, BULK_PENDING_B, BULK_SUBMITTED], - abandoned=[BULK_PENDING_A], - ) - - self.assertEqual(abandoned, (BULK_PENDING_A,)) - - def test_postgres_bulk_candidate_abandon_reports_an_empty_win(self) -> None: - _, abandoned, query = self._bulk_abandon_query( - block_hashes=[BULK_PENDING_A], - abandoned=[], - ) - - self.assertEqual(abandoned, ()) - # The empty case is the aggregate's, not a missing key: an outer - # json_agg over zero rows is NULL and would decode as None. - self.assertIn("'[]'::json", query) - - def test_postgres_bulk_candidate_abandon_normalizes_the_target_set( - self, - ) -> None: - """Mixed case and duplicates resolve to one deterministic target set. - - Rows are keyed on the lowercase hash, so a page naming the same row - twice in two cases must ask for it once. Sorting is what makes the - generated statement a function of the set rather than of the caller's - iteration order. - """ - _, _, query = self._bulk_abandon_query( - block_hashes=[ - BULK_PENDING_B, - BULK_PENDING_A.upper(), - BULK_PENDING_A, - BULK_PENDING_B.upper(), - ], - ) - - self.assertIn( - f"ANY(ARRAY['{BULK_PENDING_A}', '{BULK_PENDING_B}']::text[])", - query, - ) - self.assertEqual(query.count(BULK_PENDING_A), 1) - self.assertNotIn(BULK_PENDING_A.upper(), query) - - def test_postgres_bulk_candidate_abandon_quotes_the_reason(self) -> None: - _, _, query = self._bulk_abandon_query( - block_hashes=[BULK_PENDING_A], - error="height 'H' decided by another block", - ) - - self.assertIn( - "last_error = 'height ''H'' decided by another block'", - query, - ) - - def test_postgres_bulk_candidate_abandon_on_an_empty_page_issues_no_query( - self, - ) -> None: - """An empty page must not reach the server at all. - - ``ANY(ARRAY[])`` has no inferable element type and would fail to - parse, so the guard is not an optimization: the alternative is a - malformed statement. - """ - ledger = FakeLeasePsqlShareLedger([acquired_lease()]) - queries_before = len(ledger.lease_queries) - - self.assertEqual( - ledger.mark_block_candidates_abandoned(block_hashes=[], error="unused"), - (), - ) - - self.assertEqual(len(ledger.lease_queries), queries_before) - - def test_postgres_bulk_candidate_abandon_fails_on_inactive_writer_lease( - self, - ) -> None: - ledger = FakeLeasePsqlShareLedger( - [acquired_lease(), {"error": "writer lease is not active"}] - ) - - with self.assertRaisesRegex(RuntimeError, "writer lease is not active"): - ledger.mark_block_candidates_abandoned( - block_hashes=[BULK_PENDING_A, BULK_PENDING_B], - error="superseded by decided height", - ) - - def test_postgres_pending_candidate_page_carries_pool_block_existence( - self, - ) -> None: - """One page read answers the landed-block question for every row. - - Asking per row is one round trip per row, and the page read exists - precisely for backlogs where that is the whole cost. The probe sits - outside the limited subquery so it runs at most ``limit`` times - rather than once per pending row in the backlog. - """ - ledger = FakeLeasePsqlShareLedger( - [ - acquired_lease(), - [ - { - "block_hash": BULK_PENDING_A, - "candidate": {"block_hash_hex": BULK_PENDING_A}, - "pool_block_exists": False, - "cursor": ["2026-08-20T00:00:00.000001Z", BULK_PENDING_A], - }, - { - "block_hash": BULK_PENDING_B, - "candidate": {"block_hash_hex": BULK_PENDING_B}, - "pool_block_exists": True, - "cursor": ["2026-08-20T00:00:00.000002Z", BULK_PENDING_B], - }, - ], - ] - ) - - rows = ledger.pending_block_candidate_rows(limit=32) - - self.assertEqual( - [row["pool_block_exists"] for row in rows], - [False, True], - ) - query = ledger.lease_queries[-1] - self.assertIn("'pool_block_exists', EXISTS (", query) - self.assertIn("FROM qbit_pool_blocks pool", query) - self.assertIn("WHERE pool.block_hash = pending.block_hash", query) - # Outside the LIMITed subquery: the probe is bounded by the page, not - # by the backlog behind it. - self.assertLess( - query.index("'pool_block_exists', EXISTS ("), - query.index("FROM (\n SELECT"), - ) - self.assertIn("LIMIT 32", query) - - def test_postgres_pending_candidate_page_without_existence_fails_closed( - self, - ) -> None: - """A page that cannot report the fact raises instead of implying false.""" - ledger = FakeLeasePsqlShareLedger( - [ - acquired_lease(), - [{"block_hash": BULK_PENDING_A, "candidate": {}, "cursor": [1, "a"]}], - ] - ) - - with self.assertRaisesRegex(RuntimeError, "pool block existence"): - ledger.pending_block_candidate_rows(limit=32) - - def test_postgres_pending_candidate_page_propagates_read_failure(self) -> None: - """A failed page read must not read as an empty backlog.""" - - class FailingPageLedger(FakeLeasePsqlShareLedger): - def _run_retry_safe_read_json( - self, - sql: str, - *, - on_statement_start: Any = None, - ) -> Any: - if on_statement_start is not None: - on_statement_start() - raise RuntimeError("connection reset by peer") - - ledger = FailingPageLedger([acquired_lease()]) - - with self.assertRaisesRegex(RuntimeError, "connection reset by peer"): - ledger.pending_block_candidate_rows(limit=32) - - def test_postgres_pending_candidate_page_never_touches_the_writer_gate( - self, - ) -> None: - """The page is a read, so it takes the read slot and nothing else. - - Issue #211: waiting for writer admission bought this statement - nothing and cost it its whole fast-call budget whenever accepted-block - accounting held the gate. A gate that refuses every acquisition is the - strongest available statement of "this path does not go there" -- - stronger than timing a wait, because it fails whether the gate is - contended or free. - """ - ledger = FakeLeasePsqlShareLedger( - [ - acquired_lease(), - [ - { - "block_hash": BULK_PENDING_A, - "candidate": {"block_hash_hex": BULK_PENDING_A}, - "pool_block_exists": False, - "cursor": ["2026-08-31T00:00:00.000001Z", BULK_PENDING_A], - } - ], - ] - ) - ledger._lock = share_ledger_module._RefusingWriterGate() - - rows = ledger.pending_block_candidate_rows(limit=32) - - self.assertEqual([row["block_hash"] for row in rows], [BULK_PENDING_A]) - # The read slot is still taken and still handed back: bounded, not - # ungated. - self.assertEqual(ledger._read_semaphore._value, 4) - - def test_postgres_pending_candidate_page_splits_gate_wait_from_execution( - self, - ) -> None: - """Attribution, not a single opaque duration. - - The incident reported ``exceeded 5s`` for a call whose statement - deadline was barely touched, and nothing exported said where the - budget went. These two fields are that answer, so a scenario drives - the clock across each half separately and reads them back. - """ - ledger = FakeLeasePsqlShareLedger([acquired_lease(), []]) - # Installed after construction so the script covers exactly this - # call's four reads -- admission start, admission end, statement - # start, statement end -- and an unexpected fifth fails loudly rather - # than silently re-using a stamp. - clock = iter([100.0, 100.25, 100.25, 101.75]) - ledger._monotonic = lambda: next(clock) - - ledger.pending_block_candidate_rows(limit=32) - - self.assertEqual(list(clock), []) - - stats = ledger.ledger_read_gate_stats()["pending_block_candidate_rows"] - self.assertEqual(stats["calls_total"], 1) - self.assertAlmostEqual(stats["gate_wait_seconds_total"], 0.25) - self.assertAlmostEqual(stats["gate_wait_seconds_max"], 0.25) - self.assertAlmostEqual(stats["execute_seconds_total"], 1.5) - self.assertAlmostEqual(stats["execute_seconds_max"], 1.5) - self.assertEqual(stats["gate_timeouts_total"], 0) - self.assertEqual(stats["execute_timeouts_total"], 0) - - def test_postgres_pending_candidate_page_attributes_a_statement_deadline( - self, - ) -> None: - """A deadline that expires inside PostgreSQL is PostgreSQL's. - - Including the tail a cancelled statement spends returning: cancel lag - is server time, and charging it to local admission would point the - next investigation at the wrong half of the system. - """ - - class TimingOutPageLedger(FakeLeasePsqlShareLedger): - def _run_retry_safe_read_json( - self, - sql: str, - *, - on_statement_start: Any = None, - ) -> Any: - if on_statement_start is not None: - on_statement_start() - raise LedgerOperationTimeout("postgres operation exceeded 5s") - - ledger = TimingOutPageLedger([acquired_lease()]) - - with self.assertRaises(LedgerOperationTimeout): - ledger.pending_block_candidate_rows(limit=32) - - stats = ledger.ledger_read_gate_stats()["pending_block_candidate_rows"] - self.assertEqual(stats["calls_total"], 1) - self.assertEqual(stats["execute_timeouts_total"], 1) - self.assertEqual(stats["gate_timeouts_total"], 0) - # The failed read still handed its slot back, or the next enumeration - # would queue behind a slot nobody holds. - self.assertEqual(ledger._read_semaphore._value, 4) - - def test_postgres_pending_candidate_page_attributes_pre_send_expiry_locally( - self, - ) -> None: - """A deadline exhausted after admission is still not server time.""" - - class NeverRunNative: - def run_json(self, sql: str, **kwargs: Any) -> Any: - raise AssertionError("deadline-expired read must not reach PostgreSQL") - - ledger = PsqlShareLedger.__new__(PsqlShareLedger) - ledger._read_semaphore = threading.BoundedSemaphore(1) - ledger._native = NeverRunNative() - deadline_local = threading.local() - deadline_local.deadline = 1.0 - ledger._operation_timeout_local = deadline_local - # admission start, deadline check while acquiring, admission end, - # deadline check immediately before the native statement - clock = iter([0.0, 0.5, 0.75, 1.01]) - ledger._monotonic = lambda: next(clock) - - with self.assertRaisesRegex(LedgerOperationTimeout, "deadline expired"): - ledger._run_attributed_read_json( - "SELECT json_build_object('ok', true);", - operation="pending_block_candidate_rows", - gate=ledger._read_semaphore, - gate_name="read slot", - ) - - self.assertEqual(list(clock), []) - stats = ledger.ledger_read_gate_stats()["pending_block_candidate_rows"] - self.assertEqual(stats["calls_total"], 1) - self.assertEqual(stats["gate_timeouts_total"], 1) - self.assertEqual(stats["execute_timeouts_total"], 0) - self.assertEqual(stats["execute_seconds_total"], 0.0) - self.assertEqual(ledger._read_semaphore._value, 1) - - def test_psql_pending_candidate_page_attributes_pre_spawn_expiry_locally( - self, - ) -> None: - """The subprocess backend validates its deadline before execution.""" - - ledger = PsqlShareLedger.__new__(PsqlShareLedger) - ledger._read_semaphore = threading.BoundedSemaphore(1) - ledger._native = None - ledger._command = ["psql"] - ledger._session_guards = None - deadline_local = threading.local() - deadline_local.deadline = 1.0 - ledger._operation_timeout_local = deadline_local - # admission start, deadline check while acquiring, admission end, - # deadline check in _run_sql immediately before subprocess setup - clock = iter([0.0, 0.5, 0.75, 1.01]) - ledger._monotonic = lambda: next(clock) - - with unittest.mock.patch.object( - share_ledger_module.subprocess, - "run", - side_effect=AssertionError("deadline-expired read must not spawn psql"), - ) as run: - with self.assertRaisesRegex(LedgerOperationTimeout, "deadline expired"): - ledger._run_attributed_read_json( - "SELECT json_build_object('ok', true);", - operation="pending_block_candidate_rows", - gate=ledger._read_semaphore, - gate_name="read slot", - ) - - run.assert_not_called() - self.assertEqual(list(clock), []) - stats = ledger.ledger_read_gate_stats()["pending_block_candidate_rows"] - self.assertEqual(stats["calls_total"], 1) - self.assertEqual(stats["gate_timeouts_total"], 1) - self.assertEqual(stats["execute_timeouts_total"], 0) - self.assertEqual(stats["execute_seconds_total"], 0.0) - self.assertEqual(ledger._read_semaphore._value, 1) - - def test_run_json_preserves_legacy_run_sql_override_signature(self) -> None: - """Private one-argument subprocess seams remain supported.""" - - calls: list[str] = [] - starts: list[str] = [] - - class LegacyRunSqlLedger(PsqlShareLedger): - def _run_sql(self, sql: str) -> str: - calls.append(sql) - return '{"ok": true}' - - ledger = LegacyRunSqlLedger.__new__(LegacyRunSqlLedger) - ledger._native = None - - result = ledger._run_json( - "SELECT json_build_object('ok', true);", - on_statement_start=lambda: starts.append("started"), - ) - - self.assertEqual(result, {"ok": True}) - self.assertEqual(calls, ["SELECT json_build_object('ok', true);"]) - self.assertEqual(starts, ["started"]) - - def test_postgres_pending_candidate_page_attributes_an_admission_expiry( - self, - ) -> None: - """An expiry before admission is a gate timeout with no SQL sample. - - No statement was sent, so recording an execution duration for it - would invent server time that never happened. - """ - ledger = FakeLeasePsqlShareLedger([acquired_lease()]) - ledger._read_semaphore = threading.BoundedSemaphore(1) - self.assertTrue(ledger._read_semaphore.acquire()) - - with self.assertRaisesRegex(LedgerOperationTimeout, "read slot"): - with ledger.operation_timeout(0.01): - ledger.pending_block_candidate_rows(limit=32) - - stats = ledger.ledger_read_gate_stats()["pending_block_candidate_rows"] - self.assertEqual(stats["calls_total"], 1) - self.assertEqual(stats["gate_timeouts_total"], 1) - self.assertEqual(stats["execute_timeouts_total"], 0) - self.assertEqual(stats["execute_seconds_total"], 0.0) - self.assertGreater(stats["gate_wait_seconds_total"], 0.0) - - def test_ledger_read_gate_stats_start_empty_and_are_per_instance(self) -> None: - """No operation, no series: an unused ledger exports nothing. - - The lazy bootstrap exists for ledgers built through ``__new__``; it - must not hand two of them the same dict, or one instance's admission - wait would appear on another's series. - """ - first = PsqlShareLedger.__new__(PsqlShareLedger) - second = PsqlShareLedger.__new__(PsqlShareLedger) - - self.assertEqual(first.ledger_read_gate_stats(), {}) - first._note_ledger_read_timing( - "pending_block_candidate_rows", - gate_wait_seconds=0.5, - execute_seconds=1.0, - timed_out=False, - ) - - self.assertEqual(second.ledger_read_gate_stats(), {}) - self.assertEqual( - first.ledger_read_gate_stats()["pending_block_candidate_rows"][ - "calls_total" - ], - 1, - ) - # The accessor hands back a copy, so a caller cannot edit the record. - snapshot = first.ledger_read_gate_stats() - snapshot["pending_block_candidate_rows"]["calls_total"] = 99 - self.assertEqual( - first.ledger_read_gate_stats()["pending_block_candidate_rows"][ - "calls_total" - ], - 1, - ) - - def test_writer_lease_ttl_defaults_to_sixty_seconds(self) -> None: - ledger = FakeLeasePsqlShareLedger([acquired_lease()]) - - self.assertEqual(ledger._lease_interval_sql, "make_interval(secs => 60.0)") - - def test_writer_lease_ttl_must_be_finite_positive(self) -> None: - for value in (0, float("nan"), float("inf")): - with self.subTest(value=value), self.assertRaisesRegex(ValueError, "lease_ttl_seconds"): - PsqlShareLedger(psql_command="psql postgresql://example.invalid/qbit", lease_ttl_seconds=value) - - def test_writer_lease_adoption_silence_must_be_finite_positive(self) -> None: - for value in (0, float("nan"), float("inf")): - with self.subTest(value=value), self.assertRaisesRegex( - ValueError, - "lease_adoption_silence_seconds", - ): - PsqlShareLedger( - psql_command="psql postgresql://example.invalid/qbit", - lease_adoption_silence_seconds=value, - ) - - def test_release_writer_lease_expires_only_held_identity(self) -> None: - ledger = FakeLeasePsqlShareLedger( - [acquired_lease(), {"released": 1}], - writer_id="writer-a", - writer_epoch=7, - ) - - self.assertTrue(ledger.release_writer_lease()) - - query = ledger.lease_queries[-1] - self.assertIn("UPDATE qbit_ledger_writer_lease", query) - self.assertIn("lease_expires_at = clock_timestamp() - interval '1 second'", query) - self.assertIn("qbit_ledger_writer_lease.singleton", query) - self.assertIn("writer_session_token = data->>'writer_session_token'", query) - self.assertIn("writer-a", query) - - def test_release_writer_lease_returns_false_when_not_held(self) -> None: - ledger = FakeLeasePsqlShareLedger([acquired_lease(), {"released": 0}]) - - self.assertFalse(ledger.release_writer_lease()) - - def test_watchdog_release_uses_fresh_psql_connection_path(self) -> None: - ledger = PsqlShareLedger.__new__(PsqlShareLedger) - ledger._writer_id = "writer-a" - ledger._writer_epoch = 7 - ledger._writer_session_token = "session-a" - - with unittest.mock.patch.object( - ledger, - "_run_fenced_json", - side_effect=AssertionError("shared fenced path must not run"), - ) as fenced, unittest.mock.patch.object( - ledger, - "_run_sql", - return_value='{"released": 1}\n', - ) as fresh: - self.assertTrue(ledger.release_writer_lease_fresh_connection()) - - fenced.assert_not_called() - fresh.assert_called_once() - self.assertIn("UPDATE qbit_ledger_writer_lease", fresh.call_args.args[0]) - - def test_renew_writer_lease_refreshes_only_held_identity(self) -> None: - ledger = FakeLeasePsqlShareLedger( - [acquired_lease(), {"backend": "postgres-psql", "renewed_count": 1}], - writer_id="writer-a", - writer_epoch=7, - ) - - self.assertEqual( - ledger.renew_writer_lease(), - {"backend": "postgres-psql", "renewed_count": 1}, - ) - - query = ledger.lease_queries[-1] - self.assertIn("UPDATE qbit_ledger_writer_lease", query) - self.assertIn( - "lease_expires_at = clock_timestamp() + make_interval(secs => 60.0)", - query, - ) - self.assertIn("qbit_ledger_writer_lease.singleton", query) - self.assertIn("writer_session_token = data->>'writer_session_token'", query) - self.assertIn("writer-a", query) - self.assertNotIn("qbit_ctv_fanout_artifacts", query) - - def test_renew_writer_lease_raises_when_fenced_out(self) -> None: - ledger = FakeLeasePsqlShareLedger( - [acquired_lease(), {"error": "writer lease is not active"}] - ) - - with self.assertRaisesRegex(RuntimeError, "writer lease is not active"): - ledger.renew_writer_lease() - - @staticmethod - def guarded_verification_ledger( - guard: Any, - *, - session: str = f"{WRITER_LEASE_HEARTBEAT_SESSION_PREFIX}session-a", - ) -> PsqlShareLedger: - ledger = PsqlShareLedger.__new__(PsqlShareLedger) - ledger._writer_id = "writer-a" - ledger._writer_epoch = 7 - ledger._writer_session_token = session - ledger._pool_application_name = "qbit-prism-writer-test-pool" - ledger._lease_interval_sql = "make_interval(secs => 60.0)" - ledger._lease_authority_margin_sql = "make_interval(secs => 30.0)" - ledger._writer_lease_guard = guard - return ledger - - class InSlotFakeGuard: - """Mimic the native guard's run_json slot semantics for fakes. - - One serialized-slot acquisition per run_json call, with followup - statements executed inside that same acquisition — the property - the attribution recheck relies on so its extra statement can never - queue behind other guard callers. Subclasses supply the per- - statement result via result_for(statement_index). - """ - - held = True - - def __init__(self) -> None: - self.slot_acquisitions = 0 - self.query_starts = 0 - self.statements: list[str] = [] - - def result_for(self, index: int) -> dict[str, object]: - raise NotImplementedError - - def run_json( - self, - sql: str, - *, - on_query_start: Any = None, - followup: Any = None, - ) -> dict[str, object]: - self.slot_acquisitions += 1 - if on_query_start is not None: - self.query_starts += 1 - on_query_start() - self.statements.append(sql) - result = self.result_for(len(self.statements) - 1) - while followup is not None: - next_sql = followup(result) - if next_sql is None: - break - self.statements.append(next_sql) - result = self.result_for(len(self.statements) - 1) - return result - - def test_guard_session_verification_is_non_blocking_and_exact_session(self) -> None: - class FakeGuard: - held = True - - def __init__(self) -> None: - self.statements: list[str] = [] - - def run_json( - self, - sql: str, - *, - on_query_start: Any = None, - followup: Any = None, - ) -> dict[str, object]: - self.statements.append(sql) - return { - "backend": "postgres-psql", - "guard_advisory_lock_held": True, - "writer_session_token_current": True, - "lease_renewed_count": 1, - "lease_expired": False, - } - - guard = FakeGuard() - ledger = self.guarded_verification_ledger(guard) - - with unittest.mock.patch.object( - ledger, - "_run_fenced_json", - side_effect=AssertionError("shared fenced path must not run"), - ) as fenced: - self.assertEqual( - ledger.verify_writer_lease_guard_session(), - { - "backend": "postgres-psql", - "verified_count": 1, - "renewed_count": 1, - "renewal_deferred_to_own_write": False, - }, - ) - - fenced.assert_not_called() - self.assertEqual(len(guard.statements), 1) - statement = guard.statements[0] - # The whole point: liveness must never wait on the lease tuple's row - # lock, which fenced writes hold for entire transactions. The only - # tuple-lock acquisition is the opportunistic TTL renewal, and it must - # skip an already-locked row instead of queueing behind it. - self.assertIn("FOR NO KEY UPDATE SKIP LOCKED", statement) - self.assertNotIn("FOR UPDATE", statement) - self.assertIn("pg_locks", statement) - self.assertIn("pg_backend_pid()", statement) - self.assertIn("qbit_ledger_writer_lease", statement) - self.assertIn(f"{WRITER_LEASE_HEARTBEAT_SESSION_PREFIX}session-a", statement) - # The statement must also report committed-row expiry so a skipped - # renewal over an expired lease can fail closed. - self.assertIn("lease_expires_at <= clock_timestamp()", statement) - # And the remaining-TTL authority margin, so an own-write skip over - # a nearly-lapsed row defers external side effects before its - # authority degenerates into a rollback-dependent argument. - self.assertIn( - "<= clock_timestamp() + make_interval(secs => 30.0)", - statement, - ) - # And it must attribute the lease tuple's locker, so this process's - # own fenced write outlasting the TTL is not mistaken for a - # competing expiry claim: the committed row's xmax is the locker's - # transaction id, and a pg_stat_activity backend running it under - # the pool's unique application_name is our own write. - self.assertIn("pg_stat_activity", statement) - self.assertIn("backend_xid = qbit_ledger_writer_lease.xmax", statement) - self.assertIn("qbit-prism-writer-test-pool", statement) - - def test_guard_verification_renews_idle_lease_ttl_for_exact_identity(self) -> None: - """An idle coordinator's heartbeat must keep lease_expires_at ahead. - - With the CTV broadcaster disabled and no fenced writes for a full - lease TTL, this heartbeat is the only writer-side refresh left. If it - only read, the singleton row would expire while the coordinator is - alive and any different-identity claimant could seize it through the - expiry CAS — that identity's advisory-lock key differs, so the live - guard would not block it. - """ - - class FakeGuard: - held = True - - def __init__(self) -> None: - self.statements: list[str] = [] - - def run_json( - self, - sql: str, - *, - on_query_start: Any = None, - followup: Any = None, - ) -> dict[str, object]: - self.statements.append(sql) - return { - "backend": "postgres-psql", - "guard_advisory_lock_held": True, - "writer_session_token_current": True, - "lease_renewed_count": 1, - } - - guard = FakeGuard() - ledger = self.guarded_verification_ledger(guard) - - result = ledger.verify_writer_lease_guard_session() - - self.assertEqual(result["renewed_count"], 1) - statement = guard.statements[0] - self.assertIn( - "lease_expires_at = clock_timestamp() + make_interval(secs => 60.0)", - statement, - ) - self.assertIn("updated_at = clock_timestamp()", statement) - # Renewal is fenced to this exact identity; it can never extend a - # lease row another writer already took over. - self.assertIn("writer_id = data->>'writer_id'", statement) - self.assertIn("writer_epoch = (data->>'writer_epoch')::bigint", statement) - self.assertIn( - "writer_session_token = data->>'writer_session_token'", - statement, - ) - - def test_guard_session_verification_fails_closed_on_lost_lock_or_token(self) -> None: - for missing_field, message in ( - ("guard_advisory_lock_held", "advisory lock is no longer held"), - ("writer_session_token_current", "writer lease is not active"), - ): - with self.subTest(missing_field=missing_field): - class FakeGuard: - held = True - - def run_json( - self, - sql: str, - *, - on_query_start: Any = None, - followup: Any = None, - ) -> dict[str, object]: - return { - "backend": "postgres-psql", - "guard_advisory_lock_held": True, - "writer_session_token_current": True, - missing_field: False, - } - - ledger = self.guarded_verification_ledger(FakeGuard()) - with self.assertRaisesRegex(RuntimeError, message): - ledger.verify_writer_lease_guard_session() - - def test_guard_session_verification_requires_live_guard(self) -> None: - class ClosedGuard: - held = False - - def run_json(self, sql: str) -> dict[str, object]: - raise AssertionError("closed guard must not be queried") - - ledger = self.guarded_verification_ledger(ClosedGuard()) - with self.assertRaisesRegex(RuntimeError, "not heartbeat-capable"): - ledger.verify_writer_lease_guard_session() - - def test_guard_verification_survives_lease_row_lock_held_by_fenced_write(self) -> None: - """A fenced write holds the lease tuple past the guard statement timeout. - - Simulates the production PostgreSQL behavior behind block 39416: any - guarded statement that waits on the qbit_ledger_writer_lease tuple - lock while persist_accepted_block's transaction holds it dies with - SQLSTATE 57014. The old heartbeat renewal did exactly that; the - SKIP LOCKED verification must keep succeeding for the whole - transaction, skipping the TTL renewal instead of queueing for it. - """ - accepted_block_row_lock = threading.Lock() - - class RowLockEnforcingGuard: - held = True - - def __init__(self) -> None: - self.statements: list[str] = [] - - def run_json( - self, - sql: str, - *, - on_query_start: Any = None, - followup: Any = None, - ) -> dict[str, object]: - if on_query_start is not None: - on_query_start() - self.statements.append(sql) - takes_lease_tuple_lock = "qbit_ledger_writer_lease" in sql and ( - "UPDATE" in sql or "DELETE" in sql or "FOR UPDATE" in sql - ) - lease_tuple_locked = accepted_block_row_lock.locked() - if takes_lease_tuple_lock and lease_tuple_locked: - if "SKIP LOCKED" not in sql: - raise RuntimeError( - "canceling statement due to statement timeout\n" - 'CONTEXT: while updating tuple (0,1) in relation ' - '"qbit_ledger_writer_lease"' - ) - # SKIP LOCKED sees the held tuple lock and moves on - # without waiting: the renewal simply does not happen. - renewed = 0 - else: - renewed = 1 if takes_lease_tuple_lock else 0 - return { - "backend": "postgres-psql", - "guard_advisory_lock_held": True, - "writer_session_token_current": True, - "lease_renewed_count": renewed, - # A healthy heartbeat kept the TTL ~a full lease ahead - # before the fenced write took the tuple lock. - "lease_expired": False, - # pg_stat_activity attributes the held tuple lock to - # this process's own pooled backend. - "lease_locked_by_this_process": lease_tuple_locked, - } - - guard = RowLockEnforcingGuard() - ledger = self.guarded_verification_ledger(guard) - - with accepted_block_row_lock: - # Control: the pre-fix lease-row renewal dies on the tuple lock. - with self.assertRaisesRegex(RuntimeError, "statement timeout"): - ledger._renew_writer_lease_with(guard.run_json) - # The corrected heartbeat keeps proving liveness throughout, - # skipping the TTL renewal while the fenced write holds the row. - for _ in range(3): - result = ledger.verify_writer_lease_guard_session() - self.assertEqual(result["verified_count"], 1) - self.assertEqual(result["renewed_count"], 0) - - # Once the fenced transaction commits (and refreshes the TTL itself), - # the very next heartbeat resumes renewing on the guard session. - result = ledger.verify_writer_lease_guard_session() - self.assertEqual(result["verified_count"], 1) - self.assertEqual(result["renewed_count"], 1) - - def test_guard_verification_fails_closed_on_expired_lease_with_blocked_renewal( - self, - ) -> None: - """An expired row plus a lock-blocked renewal is not proof of liveness. - - The tuple lock that made SKIP LOCKED skip may belong to a - different-identity _try_acquire_writer_lease taking the expired row - through its expiry CAS; the committed snapshot still names this - session until that claim commits (reachable after host suspend, - where the monotonic freshness gate cannot be trusted either). The - queueing renewal used to detect this by re-evaluating after the - claimant committed; the non-blocking spelling must fail closed - instead of trusting the stale token read. The fake omits - lease_locked_by_this_process to prove an absent locker attribution - also defaults closed. - """ - - class ExpiredContendedGuard(self.InSlotFakeGuard): - def result_for(self, index: int) -> dict[str, object]: - return { - "backend": "postgres-psql", - "guard_advisory_lock_held": True, - "writer_session_token_current": True, - "lease_renewed_count": 0, - "lease_expired": True, - } - - guard = ExpiredContendedGuard() - ledger = self.guarded_verification_ledger(guard) - with self.assertRaisesRegex( - RuntimeError, - "expired and its renewal was lock-blocked", - ): - ledger.verify_writer_lease_guard_session() - # The ambiguous shape earns exactly one in-slot attribution recheck - # before the fail-closed stands. - self.assertEqual(len(guard.statements), 2) - self.assertEqual(guard.slot_acquisitions, 1) - - def test_guard_verification_survives_own_fenced_write_outlasting_ttl(self) -> None: - """A fenced write outlasting the TTL must not hard-exit the coordinator. - - persist_accepted_block holds the lease tuple lock for its whole - autocommit statement, so the heartbeat cannot renew the TTL while it - runs; a statement outlasting the remaining TTL leaves the committed - row expired with the renewal lock-blocked. When pg_stat_activity - attributes that tuple lock to one of this process's own pooled - backends, failing closed would roll back the valid write and - restart-loop on every similarly slow block — and it is unnecessary: - the exclusive tuple lock means no expiry claim can be in flight, and - the write's own commit refreshes lease_expires_at before any queued - claimant re-evaluates its expiry CAS. - - Liveness is all the exemption grants. The survival argument assumes - the write commits — a rollback hands the expired row to a queued - claimant — so the result must flag the deferral for external - side-effect fences to withhold guarded RPCs until a renewal lands. - """ - - class OwnLongFencedWriteGuard: - held = True - - def run_json( - self, - sql: str, - *, - on_query_start: Any = None, - followup: Any = None, - ) -> dict[str, object]: - return { - "backend": "postgres-psql", - "guard_advisory_lock_held": True, - "writer_session_token_current": True, - "lease_renewed_count": 0, - "lease_expired": True, - "lease_locked_by_this_process": True, - } - - ledger = self.guarded_verification_ledger(OwnLongFencedWriteGuard()) - result = ledger.verify_writer_lease_guard_session() - self.assertEqual(result["verified_count"], 1) - self.assertEqual(result["renewed_count"], 0) - self.assertIs(result["renewal_deferred_to_own_write"], True) - - def test_guard_verification_defers_own_write_skip_inside_authority_margin( - self, - ) -> None: - """An own-write skip with a thin remaining TTL defers before expiry. - - The TTL erodes exactly while a long own fenced write withholds - renewals, so an external side effect authorized on a nearly-lapsed - committed row can outlive it mid-RPC — from expiry onward its - authority is the same rollback-dependent argument the expired-case - deferral rejects. The margin probe engages the deferral once less - than half the lease TTL remains, while the row is still unexpired - and the heartbeat keeps treating the session as live. - """ - - class ErodedMarginGuard: - held = True - - def run_json( - self, - sql: str, - *, - on_query_start: Any = None, - followup: Any = None, - ) -> dict[str, object]: - return { - "backend": "postgres-psql", - "guard_advisory_lock_held": True, - "writer_session_token_current": True, - "lease_renewed_count": 0, - "lease_expired": False, - "lease_expiring_within_authority_margin": True, - "lease_locked_by_this_process": True, - } - - ledger = self.guarded_verification_ledger(ErodedMarginGuard()) - result = ledger.verify_writer_lease_guard_session() - self.assertEqual(result["verified_count"], 1) - self.assertEqual(result["renewed_count"], 0) - self.assertIs(result["renewal_deferred_to_own_write"], True) - - def test_guard_verification_authorizes_own_write_skip_with_healthy_margin( - self, - ) -> None: - """A fresh-TTL own-write skip keeps authorizing external effects. - - Every fenced commit refreshes the TTL, so under steady append - traffic the lease tuple is frequently locked by this process while - the committed row still has most of a lease ahead. Deferring those - skips would withhold submitblock and broadcasts behind saturated - append traffic; the committed row's own unexpired validity — with - at least half the TTL of runway — is standalone authority that - needs no assumption about the in-flight write's fate. - """ - - class FreshTtlOwnWriteGuard: - held = True - - def run_json( - self, - sql: str, - *, - on_query_start: Any = None, - followup: Any = None, - ) -> dict[str, object]: - return { - "backend": "postgres-psql", - "guard_advisory_lock_held": True, - "writer_session_token_current": True, - "lease_renewed_count": 0, - "lease_expired": False, - "lease_expiring_within_authority_margin": False, - "lease_locked_by_this_process": True, - } - - ledger = self.guarded_verification_ledger(FreshTtlOwnWriteGuard()) - progress_marks: list[int] = [] - result = ledger.verify_writer_lease_guard_session( - on_statement_progress=lambda: progress_marks.append(1), - ) - self.assertEqual(result["verified_count"], 1) - self.assertEqual(result["renewed_count"], 0) - self.assertIs(result["renewal_deferred_to_own_write"], False) - # A single-statement verification never reports statement progress. - self.assertEqual(progress_marks, []) - - def test_lease_authority_margin_covers_guarded_rpc_deadlines(self) -> None: - """The deferral margin floors at TTL/2 and rises with RPC deadlines. - - The margin must cover the longest RPC the lease fence can - authorize, or an effect authorized just above the margin outlives - its runway and degenerates into rollback-dependent authority. The - floor keeps the deferral engaged through the eroded tail of a long - own write even when configured deadlines are short. - """ - resolve = PsqlShareLedger._resolve_lease_authority_margin_seconds - self.assertEqual(resolve(60.0, None), 30.0) - self.assertEqual(resolve(60.0, 20.0), 30.0) - self.assertEqual(resolve(60.0, 45.0), 45.0) - with self.assertRaises(ValueError): - resolve(60.0, float("nan")) - with self.assertRaises(ValueError): - resolve(60.0, -1.0) - - def test_lease_authority_margin_reaching_the_ttl_is_rejected(self) -> None: - """A margin at or above the TTL is a startup error, not a policy. - - The deferral only gates renewal skips; a verification over an - uncontended row renews and authorizes unconditionally, and a - landed renewal's runway is exactly one TTL. When the guarded - effect's deadline can reach the TTL, even a freshly renewed lease - cannot outlast the effect, so defer-every-skip would narrow but - never close the authorize-then-expire window. Construction must - refuse the configuration instead of running with a silent - split-brain hazard. - """ - resolve = PsqlShareLedger._resolve_lease_authority_margin_seconds - for margin in (60.0, 90.0): - with self.subTest(margin=margin): - with self.assertRaisesRegex( - ValueError, - "must stay below lease_ttl_seconds", - ): - resolve(60.0, margin) - - def test_guard_verification_rechecks_when_own_commit_clears_locker_attribution( - self, - ) -> None: - """An own-write commit racing the locker probe must not hard-exit. - - The verification statement's lease-row reads share one MVCC - snapshot while pg_stat_activity reports live backend state. When - the writer's own fenced write commits after the SKIP LOCKED renewal - skipped its locked row but before the probe runs, backend_xid has - already cleared and the snapshot still shows the old expired tuple: - renewal skipped, row expired, no attributable locker — the exact - shape of a competing claim. That commit refreshed the TTL, so a - fresh statement's snapshot renews normally; hard-exiting here would - restart-loop the coordinator on the very writes the own-lock - exemption exists to survive. - """ - - class CommitRacedGuard(self.InSlotFakeGuard): - def result_for(self, index: int) -> dict[str, object]: - if index == 0: - # First statement: snapshot taken before the own write's - # commit, probe run after it — locker unattributable. - return { - "backend": "postgres-psql", - "guard_advisory_lock_held": True, - "writer_session_token_current": True, - "lease_renewed_count": 0, - "lease_expired": True, - "lease_locked_by_this_process": False, - } - # Recheck: the fresh snapshot sees the committed refresh and - # the uncontended row renews. - return { - "backend": "postgres-psql", - "guard_advisory_lock_held": True, - "writer_session_token_current": True, - "lease_renewed_count": 1, - "lease_expired": False, - } - - guard = CommitRacedGuard() - ledger = self.guarded_verification_ledger(guard) - progress_marks: list[int] = [] - result = ledger.verify_writer_lease_guard_session( - on_query_start=lambda: None, - on_statement_progress=lambda: progress_marks.append(1), - ) - self.assertEqual(result["verified_count"], 1) - self.assertEqual(result["renewed_count"], 1) - self.assertIs(result["renewal_deferred_to_own_write"], False) - self.assertEqual(len(guard.statements), 2) - # The recheck reported the completed first round trip, so liveness - # monitors sized for one statement can count it as progress. - self.assertEqual(progress_marks, [1]) - # The recheck runs inside the same serialized slot acquisition: it - # can never queue behind other guard callers, so the only cost - # charged to the caller's execution budget is one more statement, - # and callers budgeting queue wait via on_query_start see it fire - # exactly once. - self.assertEqual(guard.slot_acquisitions, 1) - self.assertEqual(guard.query_starts, 1) - - def test_guard_verification_recheck_still_fails_closed_on_real_contention( - self, - ) -> None: - """The recheck is bounded and never converts contention to liveness. - - A different-identity expiry claim still in flight shows the same - expired-locked-unattributable shape on every fresh snapshot; the - second identical read must raise, not loop. - """ - - class ContendedGuard(self.InSlotFakeGuard): - def result_for(self, index: int) -> dict[str, object]: - return { - "backend": "postgres-psql", - "guard_advisory_lock_held": True, - "writer_session_token_current": True, - "lease_renewed_count": 0, - "lease_expired": True, - "lease_locked_by_this_process": False, - } - - guard = ContendedGuard() - ledger = self.guarded_verification_ledger(guard) - with self.assertRaisesRegex( - RuntimeError, - "expired and its renewal was lock-blocked", - ): - ledger.verify_writer_lease_guard_session() - self.assertEqual(len(guard.statements), 2) - self.assertEqual(guard.slot_acquisitions, 1) - - def test_guard_verification_recheck_detects_completed_takeover(self) -> None: - """A takeover committing mid-verification fails closed on identity. - - When the ambiguous first read was a competing claim that then - committed, the recheck's fresh snapshot no longer matches this - exact session and must raise the fenced-out error rather than - re-reporting the stale lock-blocked one. - """ - - class TakeoverGuard(self.InSlotFakeGuard): - def result_for(self, index: int) -> dict[str, object]: - if index == 0: - return { - "backend": "postgres-psql", - "guard_advisory_lock_held": True, - "writer_session_token_current": True, - "lease_renewed_count": 0, - "lease_expired": True, - "lease_locked_by_this_process": False, - } - return { - "backend": "postgres-psql", - "guard_advisory_lock_held": True, - "writer_session_token_current": False, - } - - guard = TakeoverGuard() - ledger = self.guarded_verification_ledger(guard) - with self.assertRaisesRegex(RuntimeError, "writer lease is not active"): - ledger.verify_writer_lease_guard_session() - self.assertEqual(len(guard.statements), 2) - self.assertEqual(guard.slot_acquisitions, 1) - - def test_guard_verification_reports_no_deferral_outside_own_lock_expiry( - self, - ) -> None: - """The deferral flag is scoped to exactly the own-lock expired skip. - - A landed renewal (idle recovery included) restores full authority: - the TTL is a lease ahead again, so external side effects need no - deferral. A healthy skipped renewal over an unexpired row keeps the - pre-existing contract as well. - """ - for renewed, expired in ((1, True), (1, False), (0, False)): - with self.subTest(renewed=renewed, expired=expired): - - class Guard: - held = True - - def run_json( - self, - sql: str, - *, - on_query_start: Any = None, - followup: Any = None, - ) -> dict[str, object]: - return { - "backend": "postgres-psql", - "guard_advisory_lock_held": True, - "writer_session_token_current": True, - "lease_renewed_count": renewed, - "lease_expired": expired, - "lease_locked_by_this_process": True, - } - - ledger = self.guarded_verification_ledger(Guard()) - result = ledger.verify_writer_lease_guard_session() - self.assertIs(result["renewal_deferred_to_own_write"], False) - - def test_guard_verification_recovers_expired_lease_when_renewal_lands(self) -> None: - """Renewing an expired-but-uncontended row is the idle-recovery path. - - Taking the tuple lock for the renewal proves no expiry claim was in - flight, and any claimant arriving afterwards re-evaluates against the - refreshed row and loses its CAS. Only a renewal that could not land - makes an expired row disqualifying. - """ - - class ExpiredUncontendedGuard: - held = True - - def run_json( - self, - sql: str, - *, - on_query_start: Any = None, - followup: Any = None, - ) -> dict[str, object]: - return { - "backend": "postgres-psql", - "guard_advisory_lock_held": True, - "writer_session_token_current": True, - "lease_renewed_count": 1, - "lease_expired": True, - } - - ledger = self.guarded_verification_ledger(ExpiredUncontendedGuard()) - result = ledger.verify_writer_lease_guard_session() - self.assertEqual(result["verified_count"], 1) - self.assertEqual(result["renewed_count"], 1) - - def test_block_state_functions_refresh_configured_lease_after_sql_function(self) -> None: - cases = ( - ( - "confirm_accepted_block", - {"confirmed_count": 1}, - "qbit_confirm_pool_block", - "confirmed_count", - ), - ( - "reject_prepared_block", - {"rejected_count": 1}, - "qbit_reject_prepared_pool_block", - "rejected_count", - ), - ( - "reverse_immature_block", - {"reversed_count": 1}, - "qbit_reverse_immature_pool_block", - "reversed_count", - ), - ( - "mark_pool_block_inactive", - {"inactive_count": 1}, - "qbit_mark_pool_block_inactive", - "inactive_count", - ), - ( - "reactivate_pool_block", - {"reactivated_count": 1}, - "qbit_reactivate_pool_block", - "reactivated_count", - ), - ) - for method_name, result, function_name, count_key in cases: - with self.subTest(method_name=method_name): - canned = [acquired_lease(), {"backend": "postgres-psql", **result}] - if method_name in { - "confirm_accepted_block", - "reactivate_pool_block", - }: - canned.append({"audit_publication_sequence": 7}) - ledger = FakeLeasePsqlShareLedger( - canned, - lease_ttl_seconds=42, - ) - - payload = getattr(ledger, method_name)(block_hash="aa" * 32, active_tip_height=10) - query = next( - query - for query in ledger.lease_queries - if function_name in query - ) - - self.assertEqual(payload[count_key], 1) - if method_name in { - "confirm_accepted_block", - "reactivate_pool_block", - }: - self.assertEqual(payload["audit_publication_sequence"], 7) - self.assertIn(function_name, query) - self.assertNotIn("lease_refresh AS", query) - self.assertIn("make_interval(secs => 42.0)", query) - - def test_postgres_read_concurrency_bounds_public_reads_without_writer_lock(self) -> None: - ledger = BlockingReadPsqlShareLedger(read_concurrency=2) - errors: list[BaseException] = [] - - def read_artifact() -> None: - try: - ledger.dashboard_public_artifact(sha256="a" * 64) - except BaseException as exc: # pragma: no cover - surfaced below - errors.append(exc) - - threads = [threading.Thread(target=read_artifact) for _ in range(4)] - for thread in threads: - thread.start() - - ledger.wait_for_started_reads(2) - with ledger._condition: - self.assertEqual(ledger.active_reads, 2) - self.assertEqual(ledger.started_reads, 2) - - ledger.release_reads() - for thread in threads: - thread.join(timeout=5) - - self.assertFalse(errors) - self.assertEqual(ledger.started_reads, 4) - self.assertLessEqual(ledger.max_active_reads, 2) - - def test_job_snapshot_does_not_wait_for_writer_connection_lock(self) -> None: - ledger = QueryCapturePsqlShareLedger() - snapshots: list[list[object]] = [] - errors: list[BaseException] = [] - - def snapshot() -> None: - try: - snapshots.append(ledger.snapshot_at_job_issue(1_000, window_weight=8)) - except BaseException as exc: # pragma: no cover - surfaced below - errors.append(exc) - - thread = threading.Thread(target=snapshot) - with ledger._lock: - thread.start() - thread.join(timeout=1) - completed_while_writer_locked = not thread.is_alive() - thread.join(timeout=5) - - self.assertTrue(completed_while_writer_locked) - self.assertFalse(thread.is_alive()) - self.assertEqual(errors, []) - self.assertEqual(snapshots, [[]]) - self.assertIn("WITH RECURSIVE pages", ledger.queries[0]) - self.assertIn( - "share_seq >= (SELECT min_share_seq FROM page_cutoff)", - ledger.queries[0], - ) - self.assertNotIn("CROSS JOIN page_cutoff", ledger.queries[0]) - - def test_postgres_job_snapshot_delta_uses_disjoint_timestamp_ranges(self) -> None: - ledger = QueryCapturePsqlShareLedger() - - self.assertEqual(ledger.snapshot_between_job_issues(1_000, 2_000), []) - - sql = ledger.queries[0] - self.assertIn("ledger.accepted_at >", sql) - self.assertIn("ledger.job_issued_at >", sql) - self.assertIn("UNION ALL", sql) - self.assertIn("ORDER BY share_seq ASC", sql) - - def test_postgres_read_concurrency_must_be_positive(self) -> None: - with self.assertRaisesRegex(ValueError, "read_concurrency"): - PsqlShareLedger(psql_command="psql postgresql://example.invalid/qbit", read_concurrency=0) - - def test_schema_defines_public_dashboard_indexes_and_recursive_window(self) -> None: - schema_path = Path(__file__).resolve().parents[1] / "crates/qbit-prism/sql/001_share_ledger.sql" - schema = schema_path.read_text(encoding="utf-8") - - for name in ( - "qbit_pool_blocks_public_recent_idx", - "qbit_pool_payout_entries_miner_public_history_idx", - "qbit_share_ledger_accepted_recent_idx", - "qbit_share_ledger_accepted_miner_recent_idx", - "qbit_share_ledger_accepted_seq_window_idx", - "qbit_share_ledger_accepted_block_suffix_idx", - "qbit_payout_carry_forward_miner_public_history_idx", - "qbit_payout_carry_forward_block_amount_idx", - ): - self.assertIn(name, schema) - - self.assertIn("WITH RECURSIVE pages AS", schema) - self.assertIn("AND ledger.share_seq < pages.min_share_seq", schema) - self.assertIn( - "ledger.share_seq >= (SELECT min_share_seq FROM page_cutoff)", - schema, - ) - self.assertNotIn("CROSS JOIN page_cutoff", schema) - # Exactly one windowed difficulty sum may exist in the schema: the - # cutoff-bounded ranked pass of qbit_prism_window. Any other schema - # function adopting a windowed sum must justify its scan bound here. - self.assertEqual(schema.count("sum(ledger.share_difficulty) OVER"), 1) - self.assertIn("ON qbit_share_ledger ((lower(right(share_id, 64))), accepted_at DESC, share_seq DESC)", schema) - self.assertIn("ALTER COLUMN anchor_vout DROP NOT NULL", schema) - self.assertIn("CHECK (credit_policy IS NULL OR credit_policy IN ('stale-grace'))", schema) - self.assertIn("NOT VALID", schema) - self.assertNotIn("DROP CONSTRAINT IF EXISTS qbit_share_ledger_credit_policy_check", schema) - self.assertLess(schema.index("writer_epoch bigint"), schema.index("credit_policy text")) - self.assertLess(schema.index("credit_policy text"), schema.index("CHECK (accepted OR reject_reason IS NOT NULL)")) - - def test_memory_pool_snapshot_reward_window_uses_anchor_eligible_shares(self) -> None: - ledger = SingleWriterShareLedger() - now_ms = int(datetime.now(timezone.utc).timestamp() * 1000) - old_eligible = pending_share(1, share_difficulty=5, job_issued_at_ms=now_ms - 10_000, accepted_at_ms=now_ms - 9_000) - new_eligible = pending_share(2, share_difficulty=5, job_issued_at_ms=now_ms - 8_000, accepted_at_ms=now_ms - 7_000) - future_share = pending_share(3, share_difficulty=5, job_issued_at_ms=now_ms + 60_000, accepted_at_ms=now_ms + 60_000) - ledger.append(old_eligible) - ledger.append(new_eligible) - ledger.append(future_share) - - snapshot = ledger.dashboard_pool_snapshot(current_network_difficulty="1.2", generated_at=public_api.utc_now_iso()) - - self.assertEqual(snapshot["reward_window"]["requested_window_weight"], "9.6") - self.assertEqual(snapshot["reward_window"]["included_share_count"], 2) - self.assertEqual( - snapshot["reward_window"]["oldest_share_accepted_at"], - public_api.iso_datetime(datetime.fromtimestamp(old_eligible.accepted_at_ms / 1000, timezone.utc)), - ) - self.assertEqual( - snapshot["reward_window"]["newest_share_accepted_at"], - public_api.iso_datetime(datetime.fromtimestamp(new_eligible.accepted_at_ms / 1000, timezone.utc)), - ) - - def test_memory_pool_snapshot_reward_window_allows_zero_difficulty(self) -> None: - ledger = SingleWriterShareLedger() - now_ms = int(datetime.now(timezone.utc).timestamp() * 1000) - ledger.append(pending_share(1, share_difficulty=5, job_issued_at_ms=now_ms - 10_000, accepted_at_ms=now_ms - 9_000)) - - snapshot = ledger.dashboard_pool_snapshot(current_network_difficulty="0", generated_at=public_api.utc_now_iso()) - - self.assertEqual(snapshot["reward_window"]["requested_window_weight"], "0") - self.assertEqual(snapshot["reward_window"]["included_share_count"], 0) - - def test_memory_miner_reward_window_uses_prism_window_not_three_hour_rollup(self) -> None: - ledger = SingleWriterShareLedger() - now_ms = int(datetime.now(timezone.utc).timestamp() * 1000) - ledger.append(pending_share(1, share_difficulty=8, job_issued_at_ms=now_ms - 120_000, accepted_at_ms=now_ms - 119_000)) - ledger.append(pending_share(2, share_difficulty=2, job_issued_at_ms=now_ms - 2_000, accepted_at_ms=now_ms - 1_900)) - - payload = ledger.dashboard_miner_reward_window(recipient_id="miner-2", current_network_difficulty="0.25") - - self.assertEqual(payload["accepted_difficulty"], "2") - self.assertEqual(payload["pool_accepted_difficulty"], "2") - self.assertEqual(payload["share_percent"], "100") - - def test_memory_leaderboard_hash_percent_uses_hashrate_share(self) -> None: - ledger = SingleWriterShareLedger() - now_ms = int(datetime.now(timezone.utc).timestamp() * 1000) - ledger.append(pending_share(1, share_difficulty=1, job_issued_at_ms=now_ms - 2_000, accepted_at_ms=now_ms - 1_900)) - ledger.append(pending_share(2, share_difficulty=2, job_issued_at_ms=now_ms - 1_000, accepted_at_ms=now_ms - 900)) - - payload = ledger.dashboard_leaderboard(page=1, limit=15) - pool_hashrate = Decimal(payload["totals"]["pool_hashrate_ths"]) # type: ignore[index] - - for row in payload["rows"]: # type: ignore[index] - expected = public_api.decimal_string(Decimal(row["hashrate_ths_3h"]) * Decimal(100) / pool_hashrate) - self.assertEqual(row["hash_percent"], expected) - - def test_memory_dashboard_windows_exclude_future_accepted_shares(self) -> None: - ledger = SingleWriterShareLedger() - now_ms = int(datetime.now(timezone.utc).timestamp() * 1000) - ledger.append(pending_share(1, share_difficulty=1, job_issued_at_ms=now_ms - 2_000, accepted_at_ms=now_ms - 1_900)) - ledger.append(pending_share(2, share_difficulty=999, job_issued_at_ms=now_ms + 60_000, accepted_at_ms=now_ms + 60_000)) - - snapshot = ledger.dashboard_pool_snapshot(current_network_difficulty="1", generated_at=public_api.utc_now_iso()) - leaderboard = ledger.dashboard_leaderboard(page=1, limit=15) - - self.assertEqual(snapshot["participants_3h"], 1) - self.assertEqual(snapshot["hashrate_ths"]["h3"], public_api.hashrate_ths_from_difficulty(1, 3 * 60 * 60)) - self.assertEqual(leaderboard["totals"]["pool_accepted_share_difficulty"], "1") - self.assertEqual([row["recipient_id"] for row in leaderboard["rows"]], ["miner-1"]) - - def test_memory_prism_window_counts_partial_boundary_share(self) -> None: - ledger = SingleWriterShareLedger() - first = ledger.append(pending_share(1, share_difficulty=5, job_issued_at_ms=1_000, accepted_at_ms=1_000)) - second = ledger.append(pending_share(2, share_difficulty=7, job_issued_at_ms=2_000, accepted_at_ms=2_000)) - - window_rows = _prism_window_shares( - [first, second], - anchor_job_issued_at_ms=2_000, - requested_window_weight=Decimal("9.5"), - ) - - self.assertEqual([row.share.share_seq for row in window_rows], [2, 1]) - self.assertEqual([row.counted_difficulty for row in window_rows], [Decimal(7), Decimal("2.5")]) - - def test_memory_reward_leaderboard_counts_partial_boundary_and_preserves_global_rank(self) -> None: - ledger = SingleWriterShareLedger() - now_ms = int(datetime.now(timezone.utc).timestamp() * 1000) - ledger.append( - pending_share( - 1, - share_difficulty=5, - job_issued_at_ms=now_ms - 10_000, - accepted_at_ms=now_ms - 9_000, - ) - ) - ledger.append( - pending_share( - 2, - share_difficulty=6, - job_issued_at_ms=now_ms - 5_000, - accepted_at_ms=now_ms - 4_000, - ) - ) - - payload = ledger.dashboard_reward_leaderboard( - page=1, - limit=15, - current_network_difficulty="1", - recipient_id="miner-1", - ) - - self.assertEqual(payload["window"]["requested_window_weight"], "8") - self.assertEqual(payload["window"]["counted_window_weight"], "8") - self.assertEqual(payload["window"]["included_share_count"], 2) - self.assertTrue(payload["window"]["is_complete"]) - self.assertEqual(payload["totals"]["pool_counted_share_difficulty"], "8") - self.assertEqual(payload["totals"]["participant_count"], 2) - self.assertEqual(payload["pagination"]["total_count"], 1) - self.assertEqual(payload["rows"][0]["rank"], 2) - self.assertEqual(payload["rows"][0]["recipient_id"], "miner-1") - self.assertEqual(payload["rows"][0]["counted_share_difficulty"], "2") - self.assertEqual(payload["rows"][0]["share_percent"], "25") - - searched = ledger.dashboard_reward_leaderboard( - page=1, - limit=15, - current_network_difficulty="1", - search="MINER-1", - ) - self.assertEqual(searched["rows"][0]["rank"], 2) - self.assertEqual(searched["totals"]["participant_count"], 2) - self.assertEqual(searched["pagination"]["total_count"], 1) - - def test_memory_reward_leaderboard_zero_weight_is_incomplete_and_has_null_rates(self) -> None: - ledger = SingleWriterShareLedger() - - payload = ledger.dashboard_reward_leaderboard( - page=1, - limit=15, - current_network_difficulty="0", - ) - - self.assertEqual(payload["window"]["requested_window_weight"], "0") - self.assertEqual(payload["window"]["counted_window_weight"], "0") - self.assertFalse(payload["window"]["is_complete"]) - self.assertIsNone(payload["window"]["observed_span_seconds"]) - self.assertIsNone(payload["totals"]["pool_hashrate_ths"]) - self.assertIsNone(payload["totals"]["expected_time_to_block_seconds"]) - self.assertEqual(payload["rows"], []) - - def test_memory_reward_leaderboard_underfilled_zero_span_has_null_rates(self) -> None: - ledger = SingleWriterShareLedger() - now_ms = int(datetime.now(timezone.utc).timestamp() * 1000) - ledger.append( - pending_share( - 1, - share_difficulty=5, - job_issued_at_ms=now_ms, - accepted_at_ms=now_ms, - ) - ) - - payload = ledger.dashboard_reward_leaderboard( - page=1, - limit=15, - current_network_difficulty="2", - ) - - self.assertEqual(payload["window"]["requested_window_weight"], "16") - self.assertEqual(payload["window"]["counted_window_weight"], "5") - self.assertFalse(payload["window"]["is_complete"]) - self.assertEqual(payload["window"]["observed_span_seconds"], 0) - self.assertIsNone(payload["totals"]["pool_hashrate_ths"]) - self.assertIsNone(payload["totals"]["expected_time_to_block_seconds"]) - self.assertIsNone(payload["rows"][0]["hashrate_ths"]) - self.assertEqual(payload["rows"][0]["share_percent"], "100") - - def test_memory_reward_leaderboard_rejects_ambiguous_filters(self) -> None: - ledger = SingleWriterShareLedger() - - with self.assertRaisesRegex(ValueError, "search and recipient_id are mutually exclusive"): - ledger.dashboard_reward_leaderboard( - page=1, - limit=15, - current_network_difficulty="1", - search="miner", - recipient_id="miner-1", - ) - - def test_postgres_pool_snapshot_reward_window_timestamps_come_from_window_rows(self) -> None: - ledger = FakeLeasePsqlShareLedger( - [ - acquired_lease(), - { - "h1_difficulty": "0", - "h3_difficulty": "0", - "h24_difficulty": "0", - "participants_3h": 0, - "blocks_found_total": 0, - "prism_blocks_total": 0, - "blocks_reversed_total": 0, - "blocks_inactive_total": 0, - "total_mined_bits": 0, - "latest_block": None, - "oldest_share_accepted_at": None, - "newest_share_accepted_at": None, - "included_share_count": 0, - }, - ] - ) - - ledger.dashboard_pool_snapshot(current_network_difficulty="1.2", generated_at=public_api.utc_now_iso()) - query = ledger.lease_queries[-1] - - self.assertIn("window_summary AS", query) - self.assertIn("qbit_prism_window(bounds.ended_at, 9.6::numeric)", query) - self.assertIn("FROM window_rows", query) - self.assertIn("accepted_at >= bounds.ended_at - interval '24 hours'", query) - self.assertIn("accepted_at <= bounds.ended_at", query) - self.assertIn("'oldest_share_accepted_at', (SELECT oldest_share_accepted_at FROM window_summary)", query) - self.assertIn("'included_share_count', (SELECT included_share_count FROM window_summary)", query) - - def test_postgres_reward_leaderboard_uses_one_window_and_filters_after_global_rank(self) -> None: - ledger = FakeLeasePsqlShareLedger( - [ - acquired_lease(), - { - "ended_at": "2026-06-26T20:45:00Z", - "oldest_share_accepted_at": "2026-06-26T20:44:50Z", - "observed_span_seconds": 10, - "counted_window_weight": "8", - "included_share_count": 2, - "participant_count": 2, - "total_count": 1, - "rows": [ - { - "rank": 2, - "recipient_id": "miner-a", - "display_name": None, - "included_share_count": 1, - "counted_share_difficulty": "2", - "share_percent": "25", - "blocks_found_total": 4, - "last_share_at": "2026-06-26T20:44:50Z", - } - ], - }, - ] - ) - - payload = ledger.dashboard_reward_leaderboard( - page=1, - limit=15, - current_network_difficulty="1", - recipient_id="miner-a", - ) - query = ledger.lease_queries[-1] - - self.assertEqual(query.count("qbit_prism_window("), 1) - self.assertIn("window_rows AS MATERIALIZED", query) - self.assertIn("ranked AS (", query) - self.assertIn("WHERE ranked.miner_id = 'miner-a'", query) - self.assertLess(query.index("ranked AS ("), query.index("WHERE ranked.miner_id = 'miner-a'")) - self.assertEqual(payload["rows"][0]["rank"], 2) - self.assertEqual(payload["rows"][0]["counted_share_difficulty"], "2") - self.assertEqual(payload["totals"]["participant_count"], 2) - self.assertEqual(payload["pagination"]["total_count"], 1) - self.assertEqual(payload["window"]["counted_window_weight"], "8") - self.assertTrue(payload["window"]["is_complete"]) - - def test_postgres_dashboard_pending_fanout_rows_include_broadcast_attempts(self) -> None: - ledger = FakeLeasePsqlShareLedger( - [ - acquired_lease(), - {"total_count": 0, "rows": []}, - ] - ) - - payload = ledger.dashboard_pending_fanout_rows(page=1, limit=15) - query = ledger.lease_queries[-1] - - self.assertEqual(payload["pagination"], {"page": 1, "limit": 15, "total_count": 0, "total_pages": 0}) - self.assertIn("'broadcast_attempts'", query) - self.assertIn("'broadcast_attempt_summary'", query) - self.assertIn("broadcast_attempt_count", query) - self.assertIn("qbit_ctv_fanout_broadcast_attempts", query) - # Unqualified, the outer fanout_txid resolves to attempt itself and every - # row would carry every attempt in the table. - self.assertIn("WHERE attempt.fanout_txid = page_rows.fanout_txid", query) - self.assertNotIn("WHERE attempt.fanout_txid = fanout_txid", query) - self.assertIn("settlement_status NOT IN ('confirmed', 'reorged', 'failed')", query) - self.assertIn("artifact.next_broadcast_attempt_at IS NULL", query) - self.assertIn("artifact.next_broadcast_attempt_at <= clock_timestamp()", query) - - def test_postgres_ctv_broadcast_pending_respects_retry_backoff(self) -> None: - ledger = FakeLeasePsqlShareLedger([acquired_lease(), []]) - - self.assertEqual(ledger.pending_ctv_fanout_statuses(), []) - query = ledger.lease_queries[-1] - - self.assertIn("artifact.next_broadcast_attempt_at IS NULL", query) - self.assertIn("artifact.next_broadcast_attempt_at <= clock_timestamp()", query) - self.assertIn("settlement_status NOT IN ('confirmed', 'reorged', 'failed')", query) - - def test_postgres_records_ctv_broadcast_summary_and_caps_detail_rows(self) -> None: - ledger = FakeLeasePsqlShareLedger( - [ - acquired_lease(), - { - "backend": "postgres-psql", - "attempt_count": 1, - "updated_count": 1, - "broadcast_attempt_count": 5, - "broadcast_attempt_detail_count": 2, - }, - ], - ctv_broadcast_attempt_detail_limit=2, - ctv_broadcast_retry_backoff_seconds=17, - ) - - payload = ledger.record_ctv_fanout_broadcast_attempt( - fanout_txid="22" * 32, - attempt_status="planned", - package_txids=["22" * 32], - submit_result={"package_msg": "error", "submitted": False}, - error="rpc unavailable", - ) - query = ledger.lease_queries[-1] - - self.assertEqual(payload["attempt_count"], 1) - self.assertIn('"attempt_detail_limit":2', query) - self.assertIn('"retry_backoff_seconds":17', query) - self.assertIn("last_broadcast_attempt_status = data->>'attempt_status'", query) - self.assertIn("last_broadcast_error = data->>'error'", query) - self.assertIn("broadcast_attempt_status_counts = jsonb_set", query) - self.assertIn("OFFSET GREATEST((data->>'attempt_detail_limit')::integer - 1, 0)", query) - self.assertIn("next_broadcast_attempt_at = CASE", query) - self.assertIn("unknown CTV fanout txid", query) - - def test_postgres_ctv_broadcast_attempt_rejects_unknown_fanout(self) -> None: - ledger = FakeLeasePsqlShareLedger( - [ - acquired_lease(), - {"error": "unknown CTV fanout txid"}, - ] - ) - - with self.assertRaisesRegex(RuntimeError, "unknown CTV fanout txid"): - ledger.record_ctv_fanout_broadcast_attempt( - fanout_txid="22" * 32, - attempt_status="planned", - ) - - def test_postgres_miner_earnings_block_gross_keeps_reversed_rows_in_denominator(self) -> None: - ledger = FakeLeasePsqlShareLedger( - [ - acquired_lease(), - {"total_count": 0, "rows": []}, - ] - ) - - payload = ledger.dashboard_miner_earning_rows(recipient_id="miner-a", page=1, limit=15) - query = ledger.lease_queries[-1] - - self.assertEqual(payload["pagination"], {"page": 1, "limit": 15, "total_count": 0, "total_pages": 0}) - block_totals = query.split("),\npage_rows AS", 1)[0].split("block_totals AS (", 1)[1] - self.assertIn("FROM qbit_payout_carry_forward", block_totals) - self.assertIn("WHERE block_hash IN (SELECT block_hash FROM page_base)", block_totals) - self.assertNotIn("maturity_state <> 'reversed'", block_totals) - - def test_postgres_miner_pending_maturity_sums_net_immature_onchain_outputs(self) -> None: - ledger = FakeLeasePsqlShareLedger( - [ - acquired_lease(), - {"pending_maturity_bits": 37}, - ] - ) - - self.assertEqual(ledger.dashboard_miner_pending_maturity_bits(recipient_id="miner-a"), 37) - query = ledger.lease_queries[-1] - self.assertIn("FROM qbit_payout_carry_forward carry", query) - self.assertIn( - "sum(GREATEST(carry.onchain_amount_sats - carry.settlement_fee_sats, 0))", - query, - ) - self.assertIn("carry.miner_id = 'miner-a'", query) - self.assertIn("carry.action = 'onchain'", query) - self.assertIn("carry.maturity_state = 'immature'", query) - self.assertIn("block.chain_state = 'confirmed'", query) - self.assertIn("block.maturity_state = 'immature'", query) - - with self.assertRaisesRegex(ValueError, "recipient_id is required"): - ledger.dashboard_miner_pending_maturity_bits(recipient_id="") - - def test_postgres_miner_payout_rows_resolve_ctv_fanout_outputs(self) -> None: - fanout_txid = "c5" * 32 - ledger = FakeLeasePsqlShareLedger( - [ - acquired_lease(), - { - "total_count": 2, - "rows": [ - { - "block_hash": "a1" * 32, - "block_height": 19, - "coinbase_txid": "b2" * 32, - "payout_manifest_sha256": "d4" * 32, - "recipient_id": "miner-a", - "order_key": "miner-a", - "p2mr_program_hex": "e5" * 32, - "onchain_amount_sats": 7_767_471, - "carry_forward_balance_sats": "0", - "action": "onchain", - "maturity_state": "immature", - "created_at": "2026-07-15 21:50:03+00", - "fanout_txid": fanout_txid, - "fanout_vout": 1, - "fanout_amount_sats": 7_767_319, - "fanout_fee_sats": 152, - "fanout_gross_amount_sats": 7_767_471, - "fanout_status": "awaiting_maturity", - }, - { - "block_hash": "a2" * 32, - "block_height": 17, - "coinbase_txid": "b3" * 32, - "payout_manifest_sha256": "d5" * 32, - "recipient_id": "miner-a", - "order_key": "miner-a", - "p2mr_program_hex": "e5" * 32, - "onchain_amount_sats": 11_991_078, - "carry_forward_balance_sats": "0", - "action": "onchain", - "maturity_state": "immature", - "created_at": "2026-07-15 20:50:03+00", - "fanout_txid": None, - "fanout_vout": None, - "fanout_amount_sats": None, - "fanout_fee_sats": None, - "fanout_gross_amount_sats": None, - "fanout_status": None, - }, - ], - }, - ] - ) - - payload = ledger.dashboard_miner_payout_rows(recipient_id="miner-a", page=1, limit=15) - query = ledger.lease_queries[-1] - - self.assertIn("LEFT JOIN LATERAL", query) - self.assertIn("qbit_ctv_fanout_artifacts", query) - self.assertIn("manifest->'precommitment'->'outputs'", query) - self.assertIn("output.value->>'recipient_id' = page_rows.miner_id", query) - self.assertIn("output.value->>'order_key' = page_rows.payout_order_key", query) - self.assertIn("output.value->>'p2mr_program_hex' = encode(page_rows.p2mr_program, 'hex')", query) - self.assertIn("page_rows.action = 'onchain'", query) - - ctv_row, direct_row = payload["rows"] - self.assertEqual(ctv_row["transaction_kind"], "ctv_fanout") - self.assertEqual(ctv_row["transaction_id"], fanout_txid) - self.assertEqual(ctv_row["onchain_amount_bits"], 7_767_319) - self.assertEqual(direct_row["transaction_kind"], "coinbase") - self.assertEqual(direct_row["transaction_id"], "b3" * 32) - self.assertEqual(direct_row["onchain_amount_bits"], 11_991_078) - - def test_postgres_pool_snapshot_reward_window_allows_zero_difficulty(self) -> None: - ledger = FakeLeasePsqlShareLedger( - [ - acquired_lease(), - { - "h1_difficulty": "0", - "h3_difficulty": "0", - "h24_difficulty": "0", - "participants_3h": 0, - "blocks_found_total": 0, - "prism_blocks_total": 0, - "blocks_reversed_total": 0, - "blocks_inactive_total": 0, - "total_mined_bits": 0, - "latest_block": None, - "oldest_share_accepted_at": None, - "newest_share_accepted_at": None, - "included_share_count": 0, - }, - ] - ) - - ledger.dashboard_pool_snapshot(current_network_difficulty="0", generated_at=public_api.utc_now_iso()) - query = ledger.lease_queries[-1] - - self.assertIn("qbit_prism_window(bounds.ended_at, 0::numeric)", query) - - def test_postgres_miner_reward_window_uses_prism_window(self) -> None: - ledger = FakeLeasePsqlShareLedger( - [ - acquired_lease(), - {"pool_counted_difficulty": "4", "miner_counted_difficulty": {"miner-a": "1", "miner-b": "3"}}, - ] - ) - - payload = ledger.dashboard_miner_reward_window(recipient_id="miner-a", current_network_difficulty="1.2") - query = ledger.lease_queries[-1] - - self.assertEqual(payload["accepted_difficulty"], "1") - self.assertEqual(payload["pool_accepted_difficulty"], "4") - self.assertEqual(payload["share_percent"], "25") - self.assertIn("qbit_prism_window(bounds.ended_at, 9.6::numeric)", query) - self.assertIn("json_object_agg(miner_id, counted_difficulty)", query) - self.assertIn("GROUP BY miner_id", query) - - def test_postgres_miner_reward_window_shares_one_cached_pool_aggregate(self) -> None: - # FakeLeasePsqlShareLedger raises on any query beyond the canned - # results, so the second and third requests below pass only if they - # are served from the shared cached aggregate. - ledger = FakeLeasePsqlShareLedger( - [ - acquired_lease(), - {"pool_counted_difficulty": "4", "miner_counted_difficulty": {"miner-a": "1", "miner-b": "3"}}, - ] - ) - - first = ledger.dashboard_miner_reward_window(recipient_id="miner-a", current_network_difficulty="1.2") - queries_after_first = len(ledger.lease_queries) - second = ledger.dashboard_miner_reward_window(recipient_id="miner-b", current_network_difficulty="1.2") - absent = ledger.dashboard_miner_reward_window(recipient_id="miner-absent", current_network_difficulty="1.2") - - self.assertEqual(len(ledger.lease_queries), queries_after_first) - self.assertEqual(first["share_percent"], "25") - self.assertEqual(second["share_percent"], "75") - self.assertEqual(absent["accepted_difficulty"], "0") - self.assertEqual(absent["pool_accepted_difficulty"], "4") - self.assertEqual(absent["share_percent"], "0") - - def test_postgres_miner_reward_window_recomputes_when_window_weight_changes(self) -> None: - ledger = FakeLeasePsqlShareLedger( - [ - acquired_lease(), - {"pool_counted_difficulty": "4", "miner_counted_difficulty": {"miner-a": "1"}}, - {"pool_counted_difficulty": "16", "miner_counted_difficulty": {"miner-a": "8"}}, - ] - ) - - first = ledger.dashboard_miner_reward_window(recipient_id="miner-a", current_network_difficulty="1.2") - second = ledger.dashboard_miner_reward_window(recipient_id="miner-a", current_network_difficulty="2") - - self.assertEqual(first["share_percent"], "25") - self.assertEqual(second["share_percent"], "50") - self.assertIn("qbit_prism_window(bounds.ended_at, 16::numeric)", ledger.lease_queries[-1]) - - def test_postgres_miner_reward_window_cache_expires_after_ttl(self) -> None: - ledger = FakeLeasePsqlShareLedger( - [ - acquired_lease(), - {"pool_counted_difficulty": "4", "miner_counted_difficulty": {"miner-a": "1"}}, - {"pool_counted_difficulty": "8", "miner_counted_difficulty": {"miner-a": "2"}}, - ], - reward_window_cache_seconds=30.0, - ) - clock = {"now": 100.0} - with unittest.mock.patch( - "lab.prism.share_ledger.time.monotonic", side_effect=lambda: clock["now"] - ): - first = ledger.dashboard_miner_reward_window(recipient_id="miner-a", current_network_difficulty="1.2") - clock["now"] = 129.0 - cached = ledger.dashboard_miner_reward_window(recipient_id="miner-a", current_network_difficulty="1.2") - clock["now"] = 131.0 - refreshed = ledger.dashboard_miner_reward_window(recipient_id="miner-a", current_network_difficulty="1.2") - - self.assertEqual(first["pool_accepted_difficulty"], "4") - self.assertEqual(cached["pool_accepted_difficulty"], "4") - self.assertEqual(refreshed["pool_accepted_difficulty"], "8") - - def test_postgres_miner_reward_window_cache_can_be_disabled(self) -> None: - ledger = FakeLeasePsqlShareLedger( - [ - acquired_lease(), - {"pool_counted_difficulty": "4", "miner_counted_difficulty": {"miner-a": "1"}}, - {"pool_counted_difficulty": "8", "miner_counted_difficulty": {"miner-a": "2"}}, - ], - reward_window_cache_seconds=0, - ) - - first = ledger.dashboard_miner_reward_window(recipient_id="miner-a", current_network_difficulty="1.2") - second = ledger.dashboard_miner_reward_window(recipient_id="miner-a", current_network_difficulty="1.2") - - self.assertEqual(first["pool_accepted_difficulty"], "4") - self.assertEqual(second["pool_accepted_difficulty"], "8") - - def test_postgres_miner_share_summary_zero_fills_empty_payload(self) -> None: - ledger = FakeLeasePsqlShareLedger( - [ - acquired_lease(), - { - "accepted_3h": None, - "m1_difficulty": None, - "m5_difficulty": None, - "m10_difficulty": None, - "h3_difficulty": None, - "h24_difficulty": None, - "pool_h3_difficulty": None, - "last_share_at": None, - }, - ] - ) - - payload = ledger.dashboard_miner_share_summary(recipient_id="missing-miner") - - self.assertEqual(payload["accepted_3h"], 0) - self.assertEqual(payload["accepted_difficulty_3h"], "0") - self.assertIsNone(payload["last_share_at"]) - self.assertIsNone(payload["share_percent"]) - self.assertEqual(payload["hashrate_ths"], {"m1": "0", "m5": "0", "m10": "0", "h3": "0", "h24": "0"}) - - def test_postgres_miner_share_summary_accepts_decimal_numeric_text(self) -> None: - ledger = FakeLeasePsqlShareLedger( - [ - acquired_lease(), - { - "accepted_3h": 7, - "m1_difficulty": "0E-9", - "m5_difficulty": "0.500000000", - "m10_difficulty": "1.000000000", - "h3_difficulty": "1.500000000", - "h24_difficulty": "2.000000000", - "pool_h3_difficulty": "3.000000000", - "last_share_at": "2026-06-26T20:44:53Z", - }, - ] - ) - - payload = ledger.dashboard_miner_share_summary(recipient_id="miner-a") - - self.assertEqual(payload["accepted_3h"], 7) - self.assertEqual(payload["accepted_difficulty_3h"], "1.500000000") - self.assertEqual(payload["share_percent"], "50") - self.assertEqual(payload["hashrate_ths"]["m1"], "0") - - def test_postgres_leaderboard_hash_percent_uses_hashrate_share(self) -> None: - ledger = FakeLeasePsqlShareLedger( - [ - acquired_lease(), - { - "started_at": "2026-06-26T17:45:00Z", - "ended_at": "2026-06-26T20:45:00Z", - "total_difficulty": "3", - "participant_count": 2, - "rows": [ - { - "rank": 1, - "recipient_id": "miner-b", - "display_name": None, - "accepted_share_difficulty": "2", - "share_percent": "66.66666666666666666666666666666666666667", - "blocks_found": 0, - "last_share_at": "2026-06-26T20:44:53Z", - }, - { - "rank": 2, - "recipient_id": "miner-a", - "display_name": None, - "accepted_share_difficulty": "1", - "share_percent": "33.33333333333333333333333333333333333333", - "blocks_found": 0, - "last_share_at": "2026-06-26T20:44:52Z", - }, - ], - }, - ] - ) - - payload = ledger.dashboard_leaderboard(page=1, limit=15) - query = ledger.lease_queries[-1] - pool_hashrate = Decimal(payload["totals"]["pool_hashrate_ths"]) # type: ignore[index] - - self.assertIn("ledger.accepted_at <= bounds.ended_at", query) - self.assertIn("ORDER BY accepted_share_difficulty DESC, filtered.miner_id ASC", query) - for row in payload["rows"]: # type: ignore[index] - expected = public_api.decimal_string(Decimal(row["hashrate_ths_3h"]) * Decimal(100) / pool_hashrate) - self.assertEqual(row["hash_percent"], expected) - - def test_postgres_public_block_solver_queries_use_suffix_index_expression(self) -> None: - ledger = FakeLeasePsqlShareLedger( - [ - acquired_lease(), - { - "h1_difficulty": "0", - "h3_difficulty": "0", - "h24_difficulty": "0", - "participants_3h": 0, - "blocks_found_total": 0, - "prism_blocks_total": 0, - "blocks_reversed_total": 0, - "blocks_inactive_total": 0, - "total_mined_bits": 0, - "latest_block": None, - "oldest_share_accepted_at": None, - "newest_share_accepted_at": None, - "included_share_count": 0, - }, - {"total_count": 0, "rows": []}, - { - "started_at": "2026-06-26T17:45:00Z", - "ended_at": "2026-06-26T20:45:00Z", - "total_difficulty": "0", - "participant_count": 0, - "rows": [], - }, - ] - ) - - ledger.dashboard_pool_snapshot(current_network_difficulty="1", generated_at=public_api.utc_now_iso()) - ledger.dashboard_blocks(page=1, limit=15) - ledger.dashboard_leaderboard(page=1, limit=15) - queries = "\n".join(ledger.lease_queries[1:]) - - self.assertNotIn("LIKE '%:' || block.block_hash", queries) - self.assertEqual(queries.count("lower(right(share.share_id, 64)) = block.block_hash"), 3) - self.assertEqual(queries.count("length(share.share_id) >= 65"), 3) - # solver_worker_name is derived from the solving share's share_id in - # the two block-facing queries (pool-snapshot latest_block and the blocks - # table) and is no longer hardcoded null. - self.assertNotIn("'solver_worker_name', null", queries) - self.assertIn("regexp_replace(rows.solver_share_id, ':[^:]*$', '')", queries) - self.assertIn("regexp_replace(latest_block.solver_share_id, ':[^:]*$', '')", queries) - - def test_postgres_dashboard_blocks_reads_bits_from_promoted_column_then_bundle(self) -> None: - ledger = FakeLeasePsqlShareLedger( - [ - acquired_lease(), - {"total_count": 0, "rows": []}, - ] - ) - - ledger.dashboard_blocks(page=1, limit=15) - query = ledger.lease_queries[-1] - - # Reads the promoted column first, falling back to the inline JSONB for - # legacy (pre-externalization) rows. - self.assertIn( - "COALESCE(bundle.found_block_bits, bundle.audit_bundle#>>'{found_block,bits}') AS audit_bits", - query, - ) - self.assertIn("'bits', COALESCE(rows.audit_bits, '00000000')", query) - self.assertNotIn("rows.audit_bundle#>>", query) - - def test_postgres_dashboard_blocks_chain_state_filters_partition_count_and_rows(self) -> None: - ledger = FakeLeasePsqlShareLedger( - [ - acquired_lease(), - {"total_count": 0, "rows": []}, - {"total_count": 0, "rows": []}, - {"total_count": 0, "rows": []}, - ] - ) - - ledger.dashboard_blocks(page=1, limit=15) - default_query = ledger.lease_queries[-1] - ledger.dashboard_blocks(page=1, limit=15, chain_state="all") - all_query = ledger.lease_queries[-1] - ledger.dashboard_blocks(page=1, limit=15, chain_state="reversed") - reversed_query = ledger.lease_queries[-1] - - # The default emits exactly the pre-filter SQL: reversed rows hidden - # from both the page and the total, and no state fields in the rows. - self.assertIn("WHERE chain_state <> 'reversed'", default_query) - self.assertIn("WHERE block.chain_state <> 'reversed'", default_query) - self.assertNotIn("'chain_state'", default_query) - self.assertNotIn("disconnected_at", default_query) - - # chain_state=all drops the filter from the page and the pagination - # count alike, and surfaces the public state fields. - self.assertIn("WHERE true", all_query) - self.assertNotIn("WHERE chain_state", all_query) - self.assertNotIn("WHERE block.chain_state", all_query) - self.assertIn("'chain_state', rows.chain_state", all_query) - # disconnected_at is a reorg disconnect time: rejected rows carry the - # column too (their maturity flip requires it), so the public row - # masks it to reversed chain states. - self.assertIn( - "'disconnected_at', CASE WHEN rows.chain_state = 'reversed'" - " THEN to_char(rows.disconnected_at AT TIME ZONE 'UTC'", - all_query, - ) - - # chain_state=reversed inverts the default filter in both places. - self.assertIn("WHERE chain_state = 'reversed'", reversed_query) - self.assertIn("WHERE block.chain_state = 'reversed'", reversed_query) - self.assertIn("'chain_state', rows.chain_state", reversed_query) - - with self.assertRaisesRegex(ValueError, "chain_state"): - ledger.dashboard_blocks(page=1, limit=15, chain_state="orphaned") - - def test_postgres_pool_snapshot_reports_reorg_state_counts(self) -> None: - ledger = FakeLeasePsqlShareLedger( - [ - acquired_lease(), - { - "h1_difficulty": "0", - "h3_difficulty": "0", - "h24_difficulty": "0", - "participants_3h": 0, - "blocks_found_total": 7, - "prism_blocks_total": 7, - "blocks_reversed_total": 4, - "blocks_inactive_total": 2, - "total_mined_bits": 0, - "latest_block": None, - "oldest_share_accepted_at": None, - "newest_share_accepted_at": None, - "included_share_count": 0, - }, - ] - ) - - snapshot = ledger.dashboard_pool_snapshot( - current_network_difficulty="1", generated_at=public_api.utc_now_iso() - ) - query = ledger.lease_queries[-1] - - # The public reorg counters reuse the audit summary's per-state count - # shape, and blocks_found_total keeps excluding only reversed rows. - self.assertIn( - "'blocks_reversed_total', (SELECT count(*) FROM qbit_pool_blocks WHERE chain_state = 'reversed')", - query, - ) - self.assertIn( - "'blocks_inactive_total', (SELECT count(*) FROM qbit_pool_blocks WHERE chain_state = 'inactive')", - query, - ) - self.assertIn( - "'blocks_found_total', (SELECT count(*) FROM qbit_pool_blocks WHERE chain_state <> 'reversed')", - query, - ) - self.assertEqual(snapshot["blocks_reversed_total"], 4) - self.assertEqual(snapshot["blocks_inactive_total"], 2) - self.assertEqual(snapshot["blocks_found_total"], 7) - - def test_psql_externalizes_audit_body_and_resolves_round_trip(self) -> None: - with tempfile.TemporaryDirectory() as tmp: - ledger = FakeLeasePsqlShareLedger( - [acquired_lease(), None], - audit_body_dir=tmp, - audit_bundle_canonicalizer=fake_audit_bundle_bytes, - ) - bundle = { - "schema": "qbit.prism.audit-bundle.v1", - "shares": [{"share_seq": 1}, {"share_seq": 2}], - "found_block": {"bits": "207fffff"}, - } - body_sha = fake_audit_bundle_sha256(bundle) - body_uri = ledger._externalize_audit_body("aa" * 32, body_sha, bundle) - self.assertIsNotNone(body_uri) - self.assertTrue(Path(str(body_uri)).is_file()) - self.assertIn(body_sha, Path(str(body_uri)).name) - self.assertEqual(json.loads(Path(str(body_uri)).read_text(encoding="utf-8")), bundle) - self.assertEqual(ledger._externalize_audit_body("aa" * 32, body_sha, bundle), body_uri) - with self.assertRaisesRegex(RuntimeError, "sha256 mismatch"): - ledger._externalize_audit_body("aa" * 32, body_sha, {**bundle, "shares": []}) - with self.assertRaisesRegex(RuntimeError, "sha256 mismatch"): - ledger._externalize_audit_body("bb" * 32, "00" * 32, bundle) - # A row with a NULL inline body resolves the body from the file and - # presents the same shape as an inline row (no body_uri leaks out). - resolved = ledger._resolve_audit_bundle_row( - { - "block_hash": "aa" * 32, - "audit_bundle_sha256": body_sha, - "coinbase_tx_hex": "00", - "audit_bundle": None, - "body_uri": body_uri, - } - ) - assert resolved is not None - self.assertEqual(resolved["audit_bundle"], bundle) - self.assertNotIn("body_uri", resolved) - - def test_psql_reuses_complete_10k_share_slot_without_parse_merge_or_memory_amplification(self) -> None: - with tempfile.TemporaryDirectory() as tmp: - ledger = FakeLeasePsqlShareLedger( - [acquired_lease()], - audit_body_dir=tmp, - audit_share_segment_size=10_000, - ) - shares = [ - { - "share_seq": share_seq, - "share_id": f"worker-{share_seq % 128}:{share_seq:016x}", - "miner_id": f"miner-{share_seq % 128}", - "order_key": f"{share_seq:020d}", - "p2mr_program_hex": f"{share_seq % 256:02x}" * 32, - "share_difficulty": 100_000 + share_seq, - "network_difficulty": 1_000_000, - "template_height": 123_456, - "job_id": f"job-{share_seq // 64}", - "job_issued_at_ms": 1_700_000_000_000 + share_seq, - "accepted_at_ms": 1_700_000_001_000 + share_seq, - "ntime": 1_700_000_000 + share_seq, - } - for share_seq in range(1, 10_001) - ] - segment_uri, expected_range_sha256 = ledger._write_audit_share_segment_range( - segment_first_share_seq=1, - segment_last_share_seq=10_000, - first_share_seq=1, - last_share_seq=10_000, - shares=shares, - ) - segment_path = Path(segment_uri) - expected_bytes = segment_path.read_bytes() - expected_file_sha256 = hashlib.sha256(expected_bytes).hexdigest() - expected_mtime_ns = segment_path.stat().st_mtime_ns - store = ledger._audit_artifact_store - assert store is not None - - gc.collect() - tracemalloc.start() - try: - with ( - unittest.mock.patch( - "lab.prism.audit_artifacts.json.loads", - side_effect=AssertionError("complete share slot must not be parsed"), - ), - unittest.mock.patch.object( - store, - "merge_audit_share_ranges", - side_effect=AssertionError("complete share slot must not be merged"), - ), - ): - reused_uri, reused_range_sha256 = ledger._write_audit_share_segment_range( - segment_first_share_seq=1, - segment_last_share_seq=10_000, - first_share_seq=1, - last_share_seq=10_000, - shares=shares, - ) - _current_bytes, peak_bytes = tracemalloc.get_traced_memory() - finally: - tracemalloc.stop() - - self.assertEqual(reused_uri, segment_uri) - self.assertEqual(reused_range_sha256, expected_range_sha256) - self.assertEqual(segment_path.read_bytes(), expected_bytes) - self.assertEqual(hashlib.sha256(segment_path.read_bytes()).hexdigest(), expected_file_sha256) - self.assertEqual(segment_path.stat().st_mtime_ns, expected_mtime_ns) - self.assertLess( - peak_bytes, - 4 * len(expected_bytes), - f"complete-slot reuse peaked at {peak_bytes} bytes for a {len(expected_bytes)}-byte slot", - ) - - def test_psql_segment_slow_path_rewrites_semantically_equal_byte_difference(self) -> None: - with tempfile.TemporaryDirectory() as tmp: - ledger = FakeLeasePsqlShareLedger( - [acquired_lease()], - audit_body_dir=tmp, - audit_share_segment_size=2, - ) - existing_shares = [ - {"share_seq": 1, "share_id": "s1"}, - {"share_seq": 2, "share_id": "s2"}, - ] - segment_uri, old_range_sha256 = ledger._write_audit_share_segment_range( - segment_first_share_seq=1, - segment_last_share_seq=2, - first_share_seq=1, - last_share_seq=2, - shares=existing_shares, - ) - segment_path = Path(segment_uri) - old_bytes = segment_path.read_bytes() - # Dict equality ignores insertion order, while the canonical range - # hash binds exact JSON bytes. The slow path must rewrite these. - incoming_shares = [ - {"share_id": "s1", "share_seq": 1}, - {"share_id": "s2", "share_seq": 2}, - ] - expected_bytes = ledger._storage_json_bytes( - ledger._audit_share_segment_payload( - first_share_seq=1, - last_share_seq=2, - shares=incoming_shares, - ) - ) - - reused_uri, new_range_sha256 = ledger._write_audit_share_segment_range( - segment_first_share_seq=1, - segment_last_share_seq=2, - first_share_seq=1, - last_share_seq=2, - shares=incoming_shares, - ) - - self.assertEqual(reused_uri, segment_uri) - self.assertNotEqual(old_bytes, expected_bytes) - self.assertNotEqual(old_range_sha256, new_range_sha256) - self.assertEqual(segment_path.read_bytes(), expected_bytes) - self.assertEqual(new_range_sha256, hashlib.sha256(expected_bytes).hexdigest()) - - def test_psql_segment_gap_backfills_missing_shares_from_ledger(self) -> None: - with tempfile.TemporaryDirectory() as tmp: - source = SingleWriterShareLedger() - shares = [ - source.append(pending_share(index)).to_prism_json() - for index in range(1, 5) - ] - backfill_rows = [ - { - **share, - "share_difficulty": str(share["share_difficulty"]), - "network_difficulty": str(share["network_difficulty"]), - "credit_policy": None, - } - for share in shares[1:3] - ] - ledger = FakeLeasePsqlShareLedger( - [acquired_lease(), backfill_rows], - audit_body_dir=tmp, - audit_share_segment_size=10, - ) - segment_uri, _old_range_sha256 = ledger._write_audit_share_segment_range( - segment_first_share_seq=1, - segment_last_share_seq=10, - first_share_seq=1, - last_share_seq=1, - shares=[shares[0]], - ) - segment_path = Path(segment_uri) - incoming_bytes = ledger._storage_json_bytes( - ledger._audit_share_segment_payload( - first_share_seq=4, - last_share_seq=4, - shares=[shares[3]], - ) - ) - - reused_uri, incoming_range_sha256 = ledger._write_audit_share_segment_range( - segment_first_share_seq=1, - segment_last_share_seq=10, - first_share_seq=4, - last_share_seq=4, - shares=[shares[3]], - ) - - merged = json.loads(segment_path.read_text(encoding="utf-8")) - self.assertEqual(reused_uri, segment_uri) - self.assertEqual(merged["first_share_seq"], 1) - self.assertEqual(merged["last_share_seq"], 4) - self.assertEqual(merged["share_count"], 4) - self.assertEqual(merged["shares"], shares) - self.assertEqual(incoming_range_sha256, hashlib.sha256(incoming_bytes).hexdigest()) - self.assertEqual(len(ledger.lease_queries), 2) - self.assertIn("FROM qbit_share_ledger", ledger.lease_queries[-1]) - self.assertIn("WHERE accepted", ledger.lease_queries[-1]) - self.assertIn("share_seq BETWEEN 2 AND 3", ledger.lease_queries[-1]) - self.assertEqual( - list(Path(tmp).glob(f"{segment_path.name}.conflict-*")), - [], - ) - - def test_psql_segment_gap_without_ledger_rows_quarantines_existing_slot(self) -> None: - with tempfile.TemporaryDirectory() as tmp: - source = SingleWriterShareLedger() - shares = [ - source.append(pending_share(index)).to_prism_json() - for index in range(1, 5) - ] - ledger = FakeLeasePsqlShareLedger( - [acquired_lease(), [], []], - audit_body_dir=tmp, - audit_share_segment_size=10, - ) - segment_uri, old_range_sha256 = ledger._write_audit_share_segment_range( - segment_first_share_seq=1, - segment_last_share_seq=10, - first_share_seq=1, - last_share_seq=1, - shares=[shares[0]], - ) - segment_path = Path(segment_uri) - old_bytes = segment_path.read_bytes() - old_stat = segment_path.stat() - incoming_bytes = ledger._storage_json_bytes( - ledger._audit_share_segment_payload( - first_share_seq=4, - last_share_seq=4, - shares=[shares[3]], - ) - ) - - fresh_uri, incoming_range_sha256 = ledger._write_audit_share_segment_range( - segment_first_share_seq=1, - segment_last_share_seq=10, - first_share_seq=4, - last_share_seq=4, - shares=[shares[3]], - ) - - quarantined = list(Path(tmp).glob(f"{segment_path.name}.conflict-*")) - fresh_path = Path(fresh_uri) - self.assertNotEqual(fresh_uri, segment_uri) - self.assertEqual(segment_path.read_bytes(), old_bytes) - self.assertEqual(segment_path.stat().st_ino, old_stat.st_ino) - self.assertEqual(segment_path.stat().st_mtime_ns, old_stat.st_mtime_ns) - self.assertEqual(fresh_path.read_bytes(), incoming_bytes) - self.assertEqual(incoming_range_sha256, hashlib.sha256(incoming_bytes).hexdigest()) - self.assertEqual(len(quarantined), 1) - self.assertEqual(quarantined[0].read_bytes(), old_bytes) - self.assertEqual(len(ledger.lease_queries), 2) - self.assertIn("FROM qbit_share_ledger", ledger.lease_queries[-1]) - self.assertIn("share_seq BETWEEN 2 AND 3", ledger.lease_queries[-1]) - old_part = { - "kind": "segment_range", - "first_share_seq": 1, - "last_share_seq": 1, - "share_count": 1, - "range_sha256": old_range_sha256, - "body_uri": segment_uri, - } - incoming_part = { - "kind": "segment_range", - "first_share_seq": 4, - "last_share_seq": 4, - "share_count": 1, - "range_sha256": incoming_range_sha256, - "body_uri": fresh_uri, - } - self.assertEqual( - ledger._read_audit_share_segment(old_part, parent_body_uri="old-body"), - [shares[0]], - ) - self.assertEqual( - ledger._read_audit_share_segment(incoming_part, parent_body_uri="new-body"), - [shares[3]], - ) - - retried_uri, retried_range_sha256 = ledger._write_audit_share_segment_range( - segment_first_share_seq=1, - segment_last_share_seq=10, - first_share_seq=4, - last_share_seq=4, - shares=[shares[3]], - ) - - self.assertEqual(retried_uri, fresh_uri) - self.assertEqual(retried_range_sha256, incoming_range_sha256) - self.assertEqual(segment_path.read_bytes(), old_bytes) - self.assertEqual(segment_path.stat().st_ino, old_stat.st_ino) - self.assertEqual(segment_path.stat().st_mtime_ns, old_stat.st_mtime_ns) - self.assertEqual(fresh_path.read_bytes(), incoming_bytes) - self.assertEqual( - len(list(Path(tmp).glob(f"{segment_path.name}.conflict-*"))), - 1, - ) - self.assertEqual(len(ledger.lease_queries), 3) - - def test_psql_segment_conflicting_duplicate_still_raises(self) -> None: - with tempfile.TemporaryDirectory() as tmp: - source = SingleWriterShareLedger() - share = source.append(pending_share(1)).to_prism_json() - ledger = FakeLeasePsqlShareLedger( - [acquired_lease()], - audit_body_dir=tmp, - audit_share_segment_size=10, - ) - segment_uri, _range_sha256 = ledger._write_audit_share_segment_range( - segment_first_share_seq=1, - segment_last_share_seq=10, - first_share_seq=1, - last_share_seq=1, - shares=[share], - ) - segment_path = Path(segment_uri) - old_bytes = segment_path.read_bytes() - - with self.assertRaisesRegex(RuntimeError, "conflicts at share_seq 1"): - ledger._write_audit_share_segment_range( - segment_first_share_seq=1, - segment_last_share_seq=10, - first_share_seq=1, - last_share_seq=1, - shares=[{**share, "share_id": "different"}], - ) - - self.assertEqual(segment_path.read_bytes(), old_bytes) - self.assertEqual(len(ledger.lease_queries), 1) - self.assertEqual( - list(Path(tmp).glob(f"{segment_path.name}.conflict-*")), - [], - ) - - def test_psql_canonical_bundle_path_skips_canonicalizer_and_is_retry_safe(self) -> None: - with tempfile.TemporaryDirectory() as tmp: - root = Path(tmp) - body_dir = root / "body-store" - body_dir.mkdir() - candidate_path = body_dir / "canonical-candidate.json" - bundle = { - "schema": "qbit.prism.audit-bundle.v1", - "shares": [{"share_seq": 1, "share_id": "s1"}], - "found_block": {"bits": "207fffff"}, - } - canonical_bytes = fake_audit_bundle_bytes(bundle) - candidate_path.write_bytes(canonical_bytes) - body_sha = hashlib.sha256(canonical_bytes).hexdigest() - canonicalizer = unittest.mock.Mock( - side_effect=AssertionError("canonical candidate must not be canonicalized again") - ) - ledger = FakeLeasePsqlShareLedger( - [ - acquired_lease(), - {"existing_block": False, "existing_body_uri": None}, - {"existing_block": False, "existing_body_uri": None}, - ], - audit_body_dir=body_dir, - audit_bundle_canonicalizer=canonicalizer, - audit_share_segment_size=0, - ) - payload = { - "block_hash": "aa" * 32, - "audit_bundle_sha256": body_sha, - "coinbase_tx_hex": "00", - "coinbase_txid": "11" * 32, - "payout_manifest_sha256": "22" * 32, - "block_height": 10, - "parent_hash": "bb" * 32, - "writer_id": ledger._writer_id, - "writer_epoch": ledger._writer_epoch, - "writer_session_token": ledger._writer_session_token, - } - - first_uri = ledger._prepare_external_audit_body( - payload, - bundle, - canonical_bundle_path=candidate_path, - ) - assert first_uri is not None - body_path = Path(first_uri) - first_stat = body_path.stat() - self.assertEqual(body_path.read_bytes(), canonical_bytes) - - # This models a retry after the atomic body rename succeeded but - # before the corresponding database row was committed. - second_uri = ledger._prepare_external_audit_body( - payload, - bundle, - canonical_bundle_path=candidate_path, - ) - - self.assertEqual(second_uri, first_uri) - self.assertEqual(body_path.read_bytes(), canonical_bytes) - self.assertEqual(hashlib.sha256(body_path.read_bytes()).hexdigest(), body_sha) - self.assertEqual(body_path.stat().st_ino, first_stat.st_ino) - self.assertEqual(body_path.stat().st_mtime_ns, first_stat.st_mtime_ns) - canonicalizer.assert_not_called() - self.assertEqual(list(body_dir.glob(".*.tmp")), []) - - def test_psql_compact_v2_retry_reconstructs_without_recanonicalizing(self) -> None: - with tempfile.TemporaryDirectory() as tmp: - root = Path(tmp) - body_dir = root / "body-store" - body_dir.mkdir() - candidate_path = body_dir / "canonical-candidate.json" - bundle = { - "schema": "qbit.prism.audit-bundle.v1", - "shares": [ - {"share_seq": 1, "share_id": "s1"}, - {"share_seq": 2, "share_id": "s2"}, - ], - "found_block": {"bits": "207fffff"}, - } - canonical_bytes = fake_audit_bundle_bytes(bundle) - candidate_path.write_bytes(canonical_bytes) - body_sha = hashlib.sha256(canonical_bytes).hexdigest() - canonicalizer = unittest.mock.Mock( - side_effect=AssertionError("compact retry must not canonicalize a full bundle") - ) - ledger = FakeLeasePsqlShareLedger( - [ - acquired_lease(), - {"existing_block": False, "existing_body_uri": None}, - {"existing_block": False, "existing_body_uri": None}, - ], - audit_body_dir=body_dir, - audit_bundle_canonicalizer=canonicalizer, - audit_share_segment_size=1, - ) - payload = { - "block_hash": "aa" * 32, - "audit_bundle_sha256": body_sha, - "coinbase_tx_hex": "00", - "coinbase_txid": "11" * 32, - "payout_manifest_sha256": "22" * 32, - "block_height": 10, - "parent_hash": "bb" * 32, - "writer_id": ledger._writer_id, - "writer_epoch": ledger._writer_epoch, - "writer_session_token": ledger._writer_session_token, - } - - first_uri = ledger._prepare_external_audit_body( - payload, - bundle, - canonical_bundle_path=candidate_path, - ) - assert first_uri is not None - body_path = Path(first_uri) - body_stat = body_path.stat() - segment_paths = sorted(body_dir.glob("prism-audit-share-segment-slot-*.json")) - segment_stats = {path: path.stat() for path in segment_paths} - self.assertEqual( - json.loads(body_path.read_text(encoding="utf-8"))["schema"], - "qbit.prism.audit-bundle.v2", - ) - store = ledger._audit_artifact_store - assert store is not None - - with ( - unittest.mock.patch.object( - store, - "external_body_matches_sha", - side_effect=AssertionError("same-version compact retry must compare bounded storage"), - ), - # The retry verifies the existing compact body by streaming - # its segments against the logical window (issue #255); it - # never reconstructs a whole-window bundle. - unittest.mock.patch.object( - store, - "resolve_audit_bundle_v2", - side_effect=AssertionError("compact retry must not reconstruct the whole window"), - ), - unittest.mock.patch.object( - store, - "_compact_body_reconstructs_to", - wraps=store._compact_body_reconstructs_to, - ) as verified, - ): - second_uri = ledger._prepare_external_audit_body( - payload, - bundle, - canonical_bundle_path=candidate_path, - ) - - self.assertEqual(second_uri, first_uri) - self.assertEqual(body_path.stat().st_ino, body_stat.st_ino) - self.assertEqual(body_path.stat().st_mtime_ns, body_stat.st_mtime_ns) - self.assertEqual(segment_paths, sorted(body_dir.glob("prism-audit-share-segment-slot-*.json"))) - for path, first_stat in segment_stats.items(): - self.assertEqual(path.stat().st_ino, first_stat.st_ino) - self.assertEqual(path.stat().st_mtime_ns, first_stat.st_mtime_ns) - canonicalizer.assert_not_called() - verified.assert_called_once() - self.assertEqual(verified.call_args.kwargs["expected"], body_sha) - self.assertEqual(list(body_dir.glob(".*.tmp")), []) - - def test_psql_compact_body_rejects_canonical_and_logical_bundle_mismatch(self) -> None: - with tempfile.TemporaryDirectory() as tmp: - root = Path(tmp) - body_dir = root / "body-store" - body_dir.mkdir() - candidate_path = body_dir / "canonical-candidate.json" - canonical_bundle = { - "schema": "qbit.prism.audit-bundle.v1", - "shares": [ - {"share_seq": 1, "share_id": "canonical-a"}, - {"share_seq": 2, "share_id": "canonical-a-2"}, - ], - } - logical_bundle = { - "schema": "qbit.prism.audit-bundle.v1", - "shares": [ - {"share_seq": 1, "share_id": "logical-b"}, - {"share_seq": 2, "share_id": "logical-b-2"}, - ], - } - canonical_bytes = fake_audit_bundle_bytes(canonical_bundle) - candidate_path.write_bytes(canonical_bytes) - body_sha = hashlib.sha256(canonical_bytes).hexdigest() - ledger = FakeLeasePsqlShareLedger( - [acquired_lease()], - audit_body_dir=body_dir, - audit_bundle_canonicalizer=unittest.mock.Mock( - side_effect=AssertionError("literal candidate should be authoritative") - ), - audit_share_segment_size=1, - ) - payload = { - "block_hash": "aa" * 32, - "audit_bundle_sha256": body_sha, - "body_uri": str( - body_dir - / f"prism-audit-bundle-body-{'aa' * 32}-{body_sha}.json" - ), - } - - with self.assertRaisesRegex(RuntimeError, "does not match logical bundle"): - ledger._prepare_external_audit_body( - payload, - logical_bundle, - canonical_bundle_path=candidate_path, - ) - - self.assertEqual( - [ - path - for path in body_dir.iterdir() - if path.name != ".prism-audit-publication.lock" - ], - [candidate_path], - ) - - def test_psql_canonical_bundle_path_hash_mismatch_publishes_nothing(self) -> None: - with tempfile.TemporaryDirectory() as tmp: - root = Path(tmp) - body_dir = root / "body-store" - body_dir.mkdir() - candidate_path = body_dir / "canonical-candidate.json" - bundle = {"schema": "qbit.prism.audit-bundle.v1", "shares": []} - candidate_path.write_bytes(fake_audit_bundle_bytes(bundle)) - canonicalizer = unittest.mock.Mock( - side_effect=AssertionError("mismatched candidate must not be canonicalized") - ) - ledger = FakeLeasePsqlShareLedger( - [ - acquired_lease(), - {"existing_block": False, "existing_body_uri": None}, - ], - audit_body_dir=body_dir, - audit_bundle_canonicalizer=canonicalizer, - audit_share_segment_size=0, - ) - payload = { - "block_hash": "aa" * 32, - "audit_bundle_sha256": "00" * 32, - "coinbase_tx_hex": "00", - "coinbase_txid": "11" * 32, - "payout_manifest_sha256": "22" * 32, - "block_height": 10, - "parent_hash": "bb" * 32, - "writer_id": ledger._writer_id, - "writer_epoch": ledger._writer_epoch, - "writer_session_token": ledger._writer_session_token, - } - - with self.assertRaisesRegex(RuntimeError, "audit bundle sha256 mismatch"): - ledger._prepare_external_audit_body( - payload, - bundle, - canonical_bundle_path=candidate_path, - ) - - canonicalizer.assert_not_called() - self.assertEqual(len(ledger.lease_results), 1) - self.assertEqual( - [ - path - for path in body_dir.iterdir() - if path.name != ".prism-audit-publication.lock" - ], - [candidate_path], - ) - - def test_psql_compact_audit_body_writes_v2_range_proof_and_resolves_v1_bundle(self) -> None: - with tempfile.TemporaryDirectory() as tmp: - bundle = { - "schema": "qbit.prism.audit-bundle.v1", - "shares": [ - {"share_seq": 1, "share_id": "s1"}, - {"share_seq": 2, "share_id": "s2"}, - {"share_seq": 3, "share_id": "s3"}, - {"share_seq": 4, "share_id": "s4"}, - {"share_seq": 5, "share_id": "s5"}, - ], - "found_block": {"bits": "207fffff"}, - "settlement_mode_decision": {"mode": "direct_coinbase"}, - } - body_sha = fake_audit_bundle_sha256(bundle) - writer = FakeLeasePsqlShareLedger( - [acquired_lease(), {"existing_block": False, "existing_body_uri": None}], - audit_body_dir=tmp, - audit_bundle_canonicalizer=fake_audit_bundle_bytes, - audit_share_segment_size=2, - ) - body_uri = writer._prepare_external_audit_body( - { - "block_hash": "aa" * 32, - "audit_bundle_sha256": body_sha, - "coinbase_tx_hex": "00", - "coinbase_txid": "11" * 32, - "payout_manifest_sha256": "22" * 32, - "block_height": 10, - "parent_hash": "bb" * 32, - "writer_id": writer._writer_id, - "writer_epoch": writer._writer_epoch, - "writer_session_token": writer._writer_session_token, - }, - bundle, - ) - assert body_uri is not None - body_path = Path(body_uri) - artifact = json.loads(body_path.read_text(encoding="utf-8")) - self.assertEqual(artifact["schema"], AUDIT_BUNDLE_V2_SCHEMA) - self.assertEqual(artifact["share_count"], 5) - self.assertNotIn("shares", artifact["bundle_without_shares"]) - proof = artifact["share_window_proof"] - self.assertEqual(proof["schema"], AUDIT_WINDOW_COMPLETENESS_PROOF_SCHEMA) - self.assertEqual([part["kind"] for part in proof["share_parts"]], ["segment_range", "segment_range", "segment_range"]) - segment_files = sorted(Path(tmp).glob("prism-audit-share-segment-slot-*.json")) - self.assertEqual(len(segment_files), 3) - self.assertNotIn('"kind":"inline"', body_path.read_text(encoding="utf-8")) - - reader = FakeLeasePsqlShareLedger( - [acquired_lease()], - audit_body_dir=tmp, - audit_bundle_canonicalizer=fake_audit_bundle_bytes, - ) - resolved = reader._read_external_body(body_uri, expected_sha256=body_sha) - self.assertEqual(resolved, bundle) - - def test_psql_v1_and_v2_body_tamper_fail_read_and_availability(self) -> None: - def assert_tamper_rejected( - ledger: PsqlShareLedger, - body_path: Path, - digest: str, - original: dict[str, Any], - mutate: Any, - ) -> None: - tampered = json.loads(json.dumps(original)) - mutate(tampered) - body_path.write_text( - json.dumps(tampered, separators=(",", ":")), - encoding="utf-8", - ) - self.assertFalse( - ledger._external_body_available_for_sha(str(body_path), digest) - ) - with self.assertRaises(RuntimeError): - ledger._read_external_body( - str(body_path), - expected_sha256=digest, - ) - - with tempfile.TemporaryDirectory() as tmp: - root = Path(tmp) - ledger = FakeLeasePsqlShareLedger( - [acquired_lease()], - audit_body_dir=root, - audit_bundle_canonicalizer=fake_audit_bundle_bytes, - ) - v1_bundle = { - "schema": "qbit.prism.audit-bundle.v1", - "shares": [ - {"share_seq": 1, "share_id": "s1"}, - {"share_seq": 2, "share_id": "s2"}, - ], - "found_block": {"bits": "207fffff"}, - } - v1_digest = fake_audit_bundle_sha256(v1_bundle) - v1_body = { - "schema": AUDIT_BODY_REF_SCHEMA, - "block_hash": "aa" * 32, - "audit_bundle_sha256": v1_digest, - "bundle_without_shares": { - "schema": "qbit.prism.audit-bundle.v1", - "found_block": {"bits": "207fffff"}, - }, - "share_count": 2, - "shares_key_index": 1, - "share_parts": [ - { - "kind": "inline", - "first_share_seq": 1, - "last_share_seq": 1, - "share_count": 1, - "shares": [{"share_seq": 1, "share_id": "s1"}], - }, - { - "kind": "inline", - "first_share_seq": 2, - "last_share_seq": 2, - "share_count": 1, - "shares": [{"share_seq": 2, "share_id": "s2"}], - }, - ], - } - v1_path = root / ( - f"prism-audit-bundle-body-{'aa' * 32}-{v1_digest}.json" - ) - v1_path.write_text( - json.dumps(v1_body, separators=(",", ":")), - encoding="utf-8", - ) - self.assertEqual( - ledger._read_external_body( - str(v1_path), - expected_sha256=v1_digest, - ), - v1_bundle, - ) - v1_mutations = ( - lambda body: body.__setitem__("block_hash", "bb" * 32), - lambda body: body["bundle_without_shares"]["found_block"].__setitem__( - "bits", - "1d00ffff", - ), - lambda body: body.__setitem__("shares_key_index", 0), - lambda body: body.__setitem__( - "share_parts", - list(reversed(body["share_parts"])), - ), - lambda body: body["share_parts"][0].__setitem__( - "first_share_seq", - 2, - ), - ) - for index, mutate in enumerate(v1_mutations): - with self.subTest(schema="v1", mutation=index): - assert_tamper_rejected( - ledger, - v1_path, - v1_digest, - v1_body, - mutate, - ) - - with tempfile.TemporaryDirectory() as tmp: - root = Path(tmp) - v2_bundle = { - "schema": "qbit.prism.audit-bundle.v1", - "shares": [ - {"share_seq": 1, "share_id": "s1"}, - {"share_seq": 2, "share_id": "s2"}, - ], - "reward_manifest": { - "anchor_job_issued_at_ms": 100, - "anchor_share_seq": 1, - "newest_share_seq": 2, - "oldest_share_seq": 1, - "included_share_count": 2, - "requested_window_weight": 20, - "counted_window_weight": 20, - "share_slice_digest_hex": "44" * 32, - }, - } - v2_digest = fake_audit_bundle_sha256(v2_bundle) - writer = FakeLeasePsqlShareLedger( - [ - acquired_lease(), - {"existing_block": False, "existing_body_uri": None}, - ], - audit_body_dir=root, - audit_bundle_canonicalizer=fake_audit_bundle_bytes, - audit_share_segment_size=1, - ) - body_uri = writer._prepare_external_audit_body( - { - "block_hash": "cc" * 32, - "audit_bundle_sha256": v2_digest, - "coinbase_tx_hex": "00", - "coinbase_txid": "11" * 32, - "payout_manifest_sha256": "22" * 32, - "block_height": 10, - "parent_hash": "bb" * 32, - "writer_id": writer._writer_id, - "writer_epoch": writer._writer_epoch, - "writer_session_token": writer._writer_session_token, - }, - v2_bundle, - ) - assert body_uri is not None - v2_path = Path(body_uri) - v2_body = json.loads(v2_path.read_text(encoding="utf-8")) - self.assertEqual(v2_body["schema"], AUDIT_BUNDLE_V2_SCHEMA) - self.assertEqual( - writer._read_external_body( - body_uri, - expected_sha256=v2_digest, - ), - v2_bundle, - ) - v2_mutations = ( - lambda body: body.__setitem__("block_hash", "dd" * 32), - lambda body: body["bundle_without_shares"]["reward_manifest"].__setitem__( - "included_share_count", - 3, - ), - lambda body: body.__setitem__("shares_key_index", 0), - lambda body: body["share_window_proof"].__setitem__( - "share_parts", - list(reversed(body["share_window_proof"]["share_parts"])), - ), - lambda body: body["share_window_proof"].__setitem__( - "included_share_count", - 3, - ), - ) - for index, mutate in enumerate(v2_mutations): - with self.subTest(schema="v2", mutation=index): - assert_tamper_rejected( - writer, - v2_path, - v2_digest, - v2_body, - mutate, - ) - - def test_psql_v2_range_segments_grow_without_breaking_old_refs(self) -> None: - with tempfile.TemporaryDirectory() as tmp: - first_bundle = { - "schema": "qbit.prism.audit-bundle.v1", - "shares": [ - {"share_seq": 1, "share_id": "s1"}, - {"share_seq": 2, "share_id": "s2"}, - {"share_seq": 3, "share_id": "s3"}, - ], - } - second_bundle = { - "schema": "qbit.prism.audit-bundle.v1", - "shares": [ - {"share_seq": 2, "share_id": "s2"}, - {"share_seq": 3, "share_id": "s3"}, - {"share_seq": 4, "share_id": "s4"}, - ], - } - first_sha = fake_audit_bundle_sha256(first_bundle) - second_sha = fake_audit_bundle_sha256(second_bundle) - writer = FakeLeasePsqlShareLedger( - [ - acquired_lease(), - {"existing_block": False, "existing_body_uri": None}, - {"existing_block": False, "existing_body_uri": None}, - ], - audit_body_dir=tmp, - audit_bundle_canonicalizer=fake_audit_bundle_bytes, - audit_share_segment_size=2, - ) - first_uri = writer._prepare_external_audit_body( - { - "block_hash": "aa" * 32, - "audit_bundle_sha256": first_sha, - "coinbase_tx_hex": "00", - "coinbase_txid": "11" * 32, - "payout_manifest_sha256": "22" * 32, - "block_height": 10, - "parent_hash": "bb" * 32, - "writer_id": writer._writer_id, - "writer_epoch": writer._writer_epoch, - "writer_session_token": writer._writer_session_token, - }, - first_bundle, - ) - second_uri = writer._prepare_external_audit_body( - { - "block_hash": "cc" * 32, - "audit_bundle_sha256": second_sha, - "coinbase_tx_hex": "00", - "coinbase_txid": "33" * 32, - "payout_manifest_sha256": "44" * 32, - "block_height": 11, - "parent_hash": "aa" * 32, - "writer_id": writer._writer_id, - "writer_epoch": writer._writer_epoch, - "writer_session_token": writer._writer_session_token, - }, - second_bundle, - ) - assert first_uri is not None - assert second_uri is not None - segment_files = sorted(Path(tmp).glob("prism-audit-share-segment-slot-*.json")) - self.assertEqual([path.name for path in segment_files], [ - "prism-audit-share-segment-slot-1-2.json", - "prism-audit-share-segment-slot-3-4.json", - ]) - slot_3_4 = json.loads((Path(tmp) / "prism-audit-share-segment-slot-3-4.json").read_text(encoding="utf-8")) - self.assertEqual([share["share_seq"] for share in slot_3_4["shares"]], [3, 4]) - - reader = FakeLeasePsqlShareLedger( - [acquired_lease()], - audit_body_dir=tmp, - audit_bundle_canonicalizer=fake_audit_bundle_bytes, - ) - self.assertEqual(reader._read_external_body(first_uri, expected_sha256=first_sha), first_bundle) - self.assertEqual(reader._read_external_body(second_uri, expected_sha256=second_sha), second_bundle) - - def test_psql_public_artifact_resolves_external_audit_bundle(self) -> None: - with tempfile.TemporaryDirectory() as tmp: - writer = FakeLeasePsqlShareLedger( - [acquired_lease()], - audit_body_dir=tmp, - audit_bundle_canonicalizer=fake_audit_bundle_bytes, - ) - bundle = { - "schema": "qbit.prism.audit-bundle.v1", - "shares": [{"share_seq": 1}], - } - body_sha = fake_audit_bundle_sha256(bundle) - body_uri = writer._externalize_audit_body("aa" * 32, body_sha, bundle) - ledger = FakeLeasePsqlShareLedger( - [ - acquired_lease(), - { - "audit_bundle": None, - "audit_bundle_sha256": body_sha, - "body_uri": body_uri, - "has_audit_row": True, - "fallback": None, - }, - ], - audit_body_dir=tmp, - audit_bundle_canonicalizer=fake_audit_bundle_bytes, - ) - - self.assertEqual(ledger.dashboard_public_artifact(sha256=body_sha), bundle) - query = ledger.lease_queries[-1] - self.assertIn("SELECT block_hash, audit_bundle, audit_bundle_sha256, body_uri", query) - - def test_psql_public_artifact_exists_uses_metadata_only(self) -> None: - ledger = FakeLeasePsqlShareLedger( - [ - acquired_lease(), - { - "has_audit_row": True, - "audit_bundle_inline": True, - "body_uri": None, - "fallback_exists": False, - }, - ] - ) - - self.assertTrue(ledger.dashboard_public_artifact_exists(sha256="aa" * 32)) - query = ledger.lease_queries[-1] - self.assertIn("FROM qbit_pool_audit_bundles", query) - self.assertIn("body_uri", query) - - def test_psql_public_artifact_document_reads_persisted_text_in_one_query(self) -> None: - manifest_set = sample_ctv_manifest_set() - manifest_set_json = canonical_json_text(manifest_set) - ledger = FakeLeasePsqlShareLedger( - [ - acquired_lease(), - { - "has_audit_row": False, - "audit_bundle": None, - "audit_bundle_sha256": None, - "body_uri": None, - "fallback": manifest_set, - "fallback_canonical": manifest_set_json, - }, - ] - ) - - document = ledger.dashboard_public_artifact_document(sha256=sha256_json_hex(manifest_set)) - - self.assertEqual(document, {"payload": manifest_set, "canonical_json": manifest_set_json}) - # The payload and its canonical text come from the same statement so a - # request touches each artifact table at most once. - query = ledger.lease_queries[-1] - self.assertIn("SELECT block_hash, audit_bundle, audit_bundle_sha256, body_uri", query) - self.assertIn("SELECT manifest_set, manifest_set_json", query) - self.assertIn("FROM qbit_ctv_fanout_sets", query) - self.assertIn("SELECT manifest, manifest_json", query) - self.assertIn("FROM qbit_ctv_fanout_artifacts", query) - self.assertEqual(query.count("FROM qbit_ctv_fanout_sets"), 1) - self.assertEqual(query.count("FROM qbit_ctv_fanout_artifacts"), 1) - - def test_psql_public_artifact_document_prefers_stored_canonical_bundle(self) -> None: - canonical_bytes = b'{"schema":"qbit.prism.audit-bundle.v1","accepted_shares":[]}' - digest = hashlib.sha256(canonical_bytes).hexdigest() - block_hash = "ab" * 32 - calls: list[tuple[str, str]] = [] - - def read_canonical(requested_block: str, requested_digest: str) -> bytes: - calls.append((requested_block, requested_digest)) - return canonical_bytes - - ledger = FakeLeasePsqlShareLedger( - [ - acquired_lease(), - { - "has_audit_row": True, - "block_hash": block_hash, - "audit_bundle": {"schema": "legacy-reconstructed"}, - "audit_bundle_sha256": digest, - "body_uri": None, - "fallback": None, - }, - ], - audit_artifact_store=types.SimpleNamespace( - read_canonical_audit_bundle=read_canonical, - ), - ) - - document = ledger.dashboard_public_artifact_document(sha256=digest) - - self.assertEqual( - document, - { - "payload": json.loads(canonical_bytes), - "canonical_json": canonical_bytes.decode(), - }, - ) - self.assertEqual(calls, [(block_hash, digest)]) - - def test_psql_public_artifact_serves_real_stored_bytes_without_reserialization(self) -> None: - canonical_bytes = b'{"schema":"qbit.prism.audit-bundle.v1","z":0,"a":1}' - digest = hashlib.sha256(canonical_bytes).hexdigest() - block_hash = "ab" * 32 - with tempfile.TemporaryDirectory() as tmp: - root = Path(tmp) - store = AuditArtifactStore( - AuditArtifactConfig( - root=root, - evidence_path=root / "evidence.json", - ) - ) - try: - store.write_canonical_audit_bundle( - block_hash, - digest, - canonical_bytes, - ) - ledger = FakeLeasePsqlShareLedger( - [ - acquired_lease(), - { - "has_audit_row": True, - "block_hash": block_hash, - "audit_bundle": {"schema": "legacy-reconstructed"}, - "audit_bundle_sha256": digest, - "body_uri": None, - "fallback": None, - }, - ], - audit_artifact_store=store, - ) - - response = public_api.artifact( - types.SimpleNamespace(ledger=ledger), - sha256=digest, - ) - - self.assertIsInstance(response, public_api.RawJsonBody) - self.assertEqual(response.body, canonical_bytes) # type: ignore[union-attr] - self.assertEqual(hashlib.sha256(response.body).hexdigest(), digest) # type: ignore[union-attr] - finally: - store.close() - - def test_psql_public_artifact_visible_row_missing_file_uses_legacy_body(self) -> None: - legacy = {"schema": "qbit.prism.audit-bundle.v1", "legacy": True} - ledger = FakeLeasePsqlShareLedger( - [ - acquired_lease(), - { - "has_audit_row": True, - "block_hash": "ab" * 32, - "audit_bundle": legacy, - "audit_bundle_sha256": "cd" * 32, - "body_uri": None, - "fallback": None, - }, - ], - audit_artifact_store=types.SimpleNamespace( - read_canonical_audit_bundle=lambda _block, _digest: None, - ), - ) - - with self.assertLogs("lab.prism.share_ledger", level="WARNING") as logs: - document = ledger.dashboard_public_artifact_document(sha256="cd" * 32) - - self.assertEqual( - document, - { - "payload": legacy, - "canonical_json": None, - "canonical_fallback_reason": "missing", - }, - ) - self.assertIn("reason=missing", "\n".join(logs.output)) - - def test_psql_public_artifact_corrupt_file_fails_closed(self) -> None: - canonical_bytes = b'{"schema":"qbit.prism.audit-bundle.v1"}' - digest = hashlib.sha256(canonical_bytes).hexdigest() - block_hash = "ab" * 32 - with tempfile.TemporaryDirectory() as tmp: - root = Path(tmp) - store = AuditArtifactStore( - AuditArtifactConfig( - root=root, - evidence_path=root / "evidence.json", - ) - ) - try: - path = store.write_canonical_audit_bundle( - block_hash, - digest, - canonical_bytes, - ) - path.chmod(0o600) - path.write_bytes(b"not a gzip stream") - ledger = FakeLeasePsqlShareLedger( - [ - acquired_lease(), - { - "has_audit_row": True, - "block_hash": block_hash, - "audit_bundle": {"schema": "legacy-reconstructed"}, - "audit_bundle_sha256": digest, - "body_uri": None, - "fallback": None, - }, - ], - audit_artifact_store=store, - ) - - with self.assertLogs( - "lab.prism.share_ledger", - level="ERROR", - ) as logs: - with self.assertRaises(CanonicalAuditBundleCorrupt): - ledger.dashboard_public_artifact_document(sha256=digest) - - output = "\n".join(logs.output) - self.assertIn("reason=corrupt", output) - self.assertIn(block_hash, output) - self.assertIn(digest, output) - finally: - store.close() - - def test_psql_public_artifact_file_before_replica_row_is_not_visible(self) -> None: - def unexpected_read(_block: str, _digest: str) -> bytes: - self.fail("canonical storage must not bypass replica row visibility") - - ledger = FakeLeasePsqlShareLedger( - [ - acquired_lease(), - { - "has_audit_row": False, - "block_hash": None, - "audit_bundle": None, - "audit_bundle_sha256": None, - "body_uri": None, - "fallback": None, - "fallback_canonical": None, - }, - ], - audit_artifact_store=types.SimpleNamespace( - read_canonical_audit_bundle=unexpected_read, - ), - ) - - self.assertIsNone(ledger.dashboard_public_artifact_document(sha256="ef" * 32)) - - def test_psql_public_artifact_exists_accepts_verified_canonical_file(self) -> None: - canonical_bytes = b'{"schema":"qbit.prism.audit-bundle.v1"}' - digest = hashlib.sha256(canonical_bytes).hexdigest() - block_hash = "ab" * 32 - ledger = FakeLeasePsqlShareLedger( - [ - acquired_lease(), - { - "has_audit_row": True, - "block_hash": block_hash, - "audit_bundle_sha256": digest, - "audit_bundle_inline": False, - "body_uri": "/missing/legacy-body.json", - "fallback_exists": False, - }, - ], - audit_artifact_store=types.SimpleNamespace( - read_canonical_audit_bundle=lambda _block, _digest: canonical_bytes, - ), - ) - - self.assertTrue(ledger.dashboard_public_artifact_exists(sha256=digest)) - - def test_psql_public_artifact_document_returns_none_when_unmatched(self) -> None: - ledger = FakeLeasePsqlShareLedger( - [ - acquired_lease(), - { - "has_audit_row": False, - "audit_bundle": None, - "audit_bundle_sha256": None, - "body_uri": None, - "fallback": None, - "fallback_canonical": None, - }, - ] - ) - - self.assertIsNone(ledger.dashboard_public_artifact_document(sha256="aa" * 32)) - - def test_psql_public_artifact_exists_rejects_missing_external_body(self) -> None: - with tempfile.TemporaryDirectory() as tmp: - ledger = FakeLeasePsqlShareLedger( - [ - acquired_lease(), - { - "has_audit_row": True, - "audit_bundle_inline": False, - "body_uri": str(Path(tmp) / "missing.json"), - "fallback_exists": False, - }, - ], - audit_body_dir=tmp, - ) - - self.assertFalse(ledger.dashboard_public_artifact_exists(sha256="aa" * 32)) - - def test_psql_public_artifact_exists_validates_compact_body_segments(self) -> None: - with tempfile.TemporaryDirectory() as tmp: - root = Path(tmp) - logical_bundle = { - "schema": "qbit.prism.audit-bundle.v1", - "shares": [{"share_seq": 1, "share_id": "s1"}], - } - body_sha = fake_audit_bundle_sha256(logical_bundle) - segment = { - "schema": "qbit.prism.audit-share-segment.v1", - "first_share_seq": 1, - "last_share_seq": 1, - "share_count": 1, - "shares": [{"share_seq": 1, "share_id": "s1"}], - } - segment_bytes = json.dumps(segment, separators=(",", ":")).encode("utf-8") - segment_sha256 = hashlib.sha256(segment_bytes).hexdigest() - segment_path = root / f"prism-audit-share-segment-1-1-{segment_sha256}.json" - segment_path.write_bytes(segment_bytes) - body_ref = { - "schema": AUDIT_BODY_REF_SCHEMA, - "block_hash": "bb" * 32, - "audit_bundle_sha256": body_sha, - "bundle_without_shares": {"schema": "qbit.prism.audit-bundle.v1"}, - "share_count": 1, - "shares_key_index": 1, - "share_parts": [ - { - "kind": "segment", - "first_share_seq": 1, - "last_share_seq": 1, - "share_count": 1, - "sha256": segment_sha256, - "body_uri": str(segment_path), - } - ], - } - body_path = root / f"prism-audit-bundle-body-{'bb' * 32}-{body_sha}.json" - body_path.write_text(json.dumps(body_ref, separators=(",", ":")), encoding="utf-8") - ledger = FakeLeasePsqlShareLedger( - [ - acquired_lease(), - { - "has_audit_row": True, - "audit_bundle_inline": False, - "body_uri": str(body_path), - "fallback_exists": False, - }, - ], - audit_body_dir=tmp, - audit_bundle_canonicalizer=fake_audit_bundle_bytes, - ) - - self.assertTrue(ledger.dashboard_public_artifact_exists(sha256=body_sha)) - segment_path.unlink() - ledger = FakeLeasePsqlShareLedger( - [ - acquired_lease(), - { - "has_audit_row": True, - "audit_bundle_inline": False, - "body_uri": str(body_path), - "fallback_exists": False, - }, - ], - audit_body_dir=tmp, - audit_bundle_canonicalizer=fake_audit_bundle_bytes, - ) - self.assertFalse(ledger.dashboard_public_artifact_exists(sha256=body_sha)) - - def test_psql_public_artifact_exists_rejects_overstated_inline_share_count(self) -> None: - body_ref = { - "schema": AUDIT_BODY_REF_SCHEMA, - "audit_bundle_sha256": "aa" * 32, - "bundle_without_shares": {"schema": "qbit.prism.audit-bundle.v1"}, - "share_count": 2, - "share_parts": [ - { - "kind": "inline", - "first_share_seq": 1, - "last_share_seq": 2, - "share_count": 2, - "shares": [{"share_seq": 1, "share_id": "s1"}], - } - ], - } - with tempfile.TemporaryDirectory() as tmp: - body_path = Path(tmp) / ( - f"prism-audit-bundle-body-{'11' * 32}-{'aa' * 32}.json" - ) - body_path.write_text(json.dumps(body_ref, separators=(",", ":")), encoding="utf-8") - ledger = FakeLeasePsqlShareLedger([acquired_lease()], audit_body_dir=tmp) - - self.assertFalse(ledger._external_body_available_for_sha(str(body_path), "aa" * 32)) - - def test_psql_body_ref_respects_zero_shares_key_index(self) -> None: - bundle = { - "shares": [{"share_seq": 1, "share_id": "s1"}], - "schema": "qbit.prism.audit-bundle.v1", - "found_block": {"bits": "207fffff"}, - } - body_sha = fake_audit_bundle_sha256(bundle) - body_ref = { - "schema": AUDIT_BODY_REF_SCHEMA, - "block_hash": "11" * 32, - "audit_bundle_sha256": body_sha, - "share_count": 1, - "shares_key_index": 0, - "bundle_without_shares": { - "schema": "qbit.prism.audit-bundle.v1", - "found_block": {"bits": "207fffff"}, - }, - "share_parts": [ - { - "kind": "inline", - "first_share_seq": 1, - "last_share_seq": 1, - "share_count": 1, - "shares": [{"share_seq": 1, "share_id": "s1"}], - } - ], - } - with tempfile.TemporaryDirectory() as tmp: - body_path = Path(tmp) / ( - f"prism-audit-bundle-body-{'11' * 32}-{body_sha}.json" - ) - body_path.write_text(json.dumps(body_ref, separators=(",", ":")), encoding="utf-8") - ledger = FakeLeasePsqlShareLedger( - [acquired_lease()], - audit_body_dir=tmp, - audit_bundle_canonicalizer=fake_audit_bundle_bytes, - ) - - self.assertEqual(ledger._read_external_body(str(body_path), expected_sha256=body_sha), bundle) - - def test_psql_external_body_hash_mismatch_fails_readers(self) -> None: - with tempfile.TemporaryDirectory() as tmp: - writer = FakeLeasePsqlShareLedger( - [acquired_lease()], - audit_body_dir=tmp, - audit_bundle_canonicalizer=fake_audit_bundle_bytes, - ) - bundle = { - "schema": "qbit.prism.audit-bundle.v1", - "shares": [{"share_seq": 1}], - } - body_sha = fake_audit_bundle_sha256(bundle) - body_uri = writer._externalize_audit_body("aa" * 32, body_sha, bundle) - Path(str(body_uri)).write_text(json.dumps({"schema": "corrupt"}), encoding="utf-8") - audit_row = { - "block_hash": "aa" * 32, - "audit_bundle_sha256": body_sha, - "coinbase_tx_hex": "00", - "audit_bundle": None, - "body_uri": body_uri, - } - - ledger = FakeLeasePsqlShareLedger( - [ - acquired_lease(), - audit_row, - {**audit_row, "audit_commitment_leaf_hex": "ab" * 32}, - ], - audit_body_dir=tmp, - audit_bundle_canonicalizer=fake_audit_bundle_bytes, - ) - with self.assertRaisesRegex(RuntimeError, "hash mismatch"): - ledger.audit_bundle(block_hash="aa" * 32) - with self.assertRaisesRegex(RuntimeError, "hash mismatch"): - ledger.audit_bundle_by_commitment(commitment_leaf_hex="ab" * 32) - - public_ledger = FakeLeasePsqlShareLedger( - [ - acquired_lease(), - { - "audit_bundle": None, - "audit_bundle_sha256": body_sha, - "body_uri": body_uri, - "has_audit_row": True, - "fallback": None, - }, - ], - audit_body_dir=tmp, - audit_bundle_canonicalizer=fake_audit_bundle_bytes, - ) - with self.assertRaisesRegex(RuntimeError, "hash mismatch"): - public_ledger.dashboard_public_artifact(sha256=body_sha) - - def test_psql_resolves_inline_body_and_flags_missing_external_body(self) -> None: - ledger = FakeLeasePsqlShareLedger([acquired_lease()]) # no body store configured - self.assertIsNone(ledger._externalize_audit_body("aa" * 32, "bb" * 32, {"x": 1})) - inline = {"schema": "qbit.prism.audit-bundle.v1"} - resolved = ledger._resolve_audit_bundle_row( - {"block_hash": "aa" * 32, "audit_bundle": inline, "body_uri": None} - ) - assert resolved is not None - self.assertEqual(resolved["audit_bundle"], inline) - self.assertNotIn("body_uri", resolved) - self.assertIsNone(ledger._resolve_audit_bundle_row(None)) - with self.assertRaisesRegex(RuntimeError, "not retrievable"): - ledger._resolve_audit_bundle_row( - { - "audit_bundle_sha256": "bb" * 32, - "audit_bundle": None, - "body_uri": "/nonexistent/prism-audit-bundle-body-zz.json", - } - ) - - def test_psql_persist_externalizes_bundle_body(self) -> None: - with tempfile.TemporaryDirectory() as tmp: - ledger = FakeLeasePsqlShareLedger( - [ - acquired_lease(), - {"existing_block": False, "existing_body_uri": None}, - { - "backend": "postgres-psql", - "share_count": 0, - "block_count": 1, - "bundle_count": 1, - "payout_entry_count": 1, - "carry_forward_count": 1, - "onchain_output_count": 0, - }, - ], - audit_body_dir=tmp, - audit_bundle_canonicalizer=fake_audit_bundle_bytes, - ) - bundle = { - "schema": "qbit.prism.audit-bundle.v1", - "signed_coinbase_manifest": {"manifest": {"payout_count": 1}}, - "found_block": {"network_difficulty": 1000, "bits": "207fffff", "coinbase_value_sats": 600}, - "audit_commitment_leaves_hex": ["ab" * 32], - "witness_merkle_leaves_hex": ["cd" * 32], - "payout_policy_manifest": { - "accounts": [ - { - "recipient_id": "miner-a", - "order_key": "a", - "p2mr_program_hex": "aa" * 32, - "gross_amount_sats": 1000, - "prior_balance_sats": 0, - "candidate_balance_sats": 1000, - "onchain_amount_sats": 0, - "carry_forward_balance_sats": 1000, - "action": "accrued", - } - ] - }, - } - report = { - "coinbase_txid": "ee" * 32, - "coinbase_manifest_sha256_hex": "11" * 32, - "audit_bundle_sha256_hex": fake_audit_bundle_sha256(bundle), - "coinbase_tx_hex": "00", - } - ledger.persist_accepted_block( - block_hash="aa" * 32, - block_height=10, - parent_hash="bb" * 32, - final_bundle=bundle, - audit_report=report, - ) - query = ledger.lease_queries[-1] - # New columns are written, and the inline JSONB body is NULL (externalized). - self.assertIn("body_uri", query) - self.assertIn("found_block_network_difficulty", query) - self.assertIn("audit_commitment_leaves_hex", query) - self.assertIn('"audit_bundle":null', query) - # The body lives in exactly one external file that round-trips. - body_files = sorted(Path(tmp).glob("prism-audit-bundle-body-*.json")) - self.assertEqual(len(body_files), 1) - self.assertIn(report["audit_bundle_sha256_hex"], body_files[0].name) - self.assertEqual(json.loads(body_files[0].read_text(encoding="utf-8")), bundle) - - def test_psql_persist_rejects_report_digest_mismatch_before_body_write(self) -> None: - with tempfile.TemporaryDirectory() as tmp: - ledger = FakeLeasePsqlShareLedger( - [acquired_lease(), {"existing_block": False, "existing_body_uri": None}], - audit_body_dir=tmp, - audit_bundle_canonicalizer=fake_audit_bundle_bytes, - ) - bundle = { - "schema": "qbit.prism.audit-bundle.v1", - "signed_coinbase_manifest": {"manifest": {"payout_count": 0}}, - "payout_policy_manifest": {"accounts": []}, - } - report = { - "coinbase_txid": "ee" * 32, - "coinbase_manifest_sha256_hex": "11" * 32, - "audit_bundle_sha256_hex": "22" * 32, - "coinbase_tx_hex": "00", - } - - with self.assertRaisesRegex(RuntimeError, "sha256 mismatch"): - ledger.persist_accepted_block( - block_hash="aa" * 32, - block_height=10, - parent_hash="bb" * 32, - final_bundle=bundle, - audit_report=report, - ) - self.assertEqual(list(Path(tmp).glob("prism-audit-bundle-body-*.json")), []) - self.assertEqual( - list(Path(tmp).glob("prism-audit-bundle-canonical-*.json.gz")), - [], - ) - - def test_psql_persist_requires_lease_preflight_before_external_body_write(self) -> None: - with tempfile.TemporaryDirectory() as tmp: - ledger = FakeLeasePsqlShareLedger( - [acquired_lease(), {"error": "writer lease is not active"}], - audit_body_dir=tmp, - audit_bundle_canonicalizer=fake_audit_bundle_bytes, - audit_share_segment_size=1, - ) - bundle = { - "schema": "qbit.prism.audit-bundle.v1", - "shares": [{"share_seq": 1, "share_id": "s1"}], - "signed_coinbase_manifest": {"manifest": {"payout_count": 0}}, - "payout_policy_manifest": {"accounts": []}, - } - report = { - "coinbase_txid": "ee" * 32, - "coinbase_manifest_sha256_hex": "11" * 32, - "audit_bundle_sha256_hex": fake_audit_bundle_sha256(bundle), - "coinbase_tx_hex": "00", - } - - with self.assertRaisesRegex(RuntimeError, "writer lease is not active"): - ledger.persist_accepted_block( - block_hash="aa" * 32, - block_height=10, - parent_hash="bb" * 32, - final_bundle=bundle, - audit_report=report, - ) - self.assertEqual(list(Path(tmp).glob("prism-audit-bundle-body-*.json")), []) - self.assertEqual(list(Path(tmp).glob("prism-audit-share-segment-*.json")), []) - # The canonical copy is a filesystem side effect like any other and - # must stay behind the same lease fence. - self.assertEqual( - list(Path(tmp).glob("prism-audit-bundle-canonical-*.json.gz")), - [], - ) - - def test_psql_persist_publishes_the_canonical_bundle_bytes(self) -> None: - with tempfile.TemporaryDirectory() as tmp: - bundle = { - "schema": "qbit.prism.audit-bundle.v1", - "shares": [ - {"share_seq": 1, "share_id": "s1"}, - {"share_seq": 2, "share_id": "s2"}, - ], - "signed_coinbase_manifest": {"manifest": {"payout_count": 0}}, - "payout_policy_manifest": {"accounts": []}, - } - digest = fake_audit_bundle_sha256(bundle) - report = { - "coinbase_txid": "ee" * 32, - "coinbase_manifest_sha256_hex": "11" * 32, - "audit_bundle_sha256_hex": digest, - "coinbase_tx_hex": "00", - } - insert_result = { - "backend": "postgres-psql", - "share_count": 0, - "block_count": 1, - "bundle_count": 1, - "payout_entry_count": 0, - "carry_forward_count": 0, - "onchain_output_count": 0, - } - ledger = FakeLeasePsqlShareLedger( - [ - acquired_lease(), - {"existing_block": False, "existing_body_uri": None}, - insert_result, - ], - audit_body_dir=tmp, - audit_bundle_canonicalizer=fake_audit_bundle_bytes, - audit_share_segment_size=2, - ) - ledger.persist_accepted_block( - block_hash="aa" * 32, - block_height=10, - parent_hash="bb" * 32, - final_bundle=bundle, - audit_report=report, - ) - canonical_files = sorted( - Path(tmp).glob("prism-audit-bundle-canonical-*.json.gz") - ) - self.assertEqual(len(canonical_files), 1) - self.assertEqual( - canonical_files[0].name, - f"prism-audit-bundle-canonical-{'aa' * 32}-{digest}.json.gz", - ) - # The compact body stays the pointer the row references; the - # canonical copy carries the exact bytes the digest is taken over. - compressed = canonical_files[0].read_bytes() - self.assertEqual(gzip.decompress(compressed), fake_audit_bundle_bytes(bundle)) - body_files = sorted(Path(tmp).glob("prism-audit-bundle-body-*.json")) - self.assertEqual(len(body_files), 1) - self.assertNotEqual(body_files[0].read_bytes(), fake_audit_bundle_bytes(bundle)) - self.assertEqual( - ledger._audit_store().read_canonical_audit_bundle("aa" * 32, digest), - fake_audit_bundle_bytes(bundle), - ) - - # Re-persisting the same block republishes byte-for-byte. - ledger.lease_results.extend( - [{"existing_block": True, "existing_body_uri": str(body_files[0])}, insert_result] - ) - ledger.persist_accepted_block( - block_hash="aa" * 32, - block_height=10, - parent_hash="bb" * 32, - final_bundle=bundle, - audit_report=report, - ) - self.assertEqual(canonical_files[0].read_bytes(), compressed) - self.assertEqual( - sorted(Path(tmp).glob("prism-audit-bundle-canonical-*.json.gz")), - canonical_files, - ) - - def test_backfill_seams_bind_the_ledger_loader_and_the_owned_store(self) -> None: - with tempfile.TemporaryDirectory() as tmp: - bundle = { - "schema": "qbit.prism.audit-bundle.v1", - "shares": [ - {"share_seq": 1, "share_id": "s1"}, - {"share_seq": 2, "share_id": "s2"}, - ], - "signed_coinbase_manifest": {"manifest": {"payout_count": 0}}, - "payout_policy_manifest": {"accounts": []}, - } - digest = fake_audit_bundle_sha256(bundle) - ledger = FakeLeasePsqlShareLedger( - [acquired_lease(), {"existing_block": False, "existing_body_uri": None}], - audit_body_dir=tmp, - audit_bundle_canonicalizer=fake_audit_bundle_bytes, - audit_share_segment_size=2, - ) - body_uri = ledger._prepare_external_audit_body( - {"block_hash": "aa" * 32, "audit_bundle_sha256": digest}, - bundle, - ) - assert body_uri is not None - canonical_path = Path( - ledger.publish_canonical_bundle_bytes( - block_hash="aa" * 32, - audit_bundle_sha256=digest, - canonical_bytes=fake_audit_bundle_bytes(bundle), - ) - ) - self.assertTrue(canonical_path.exists()) - - # Drop the segments; the seam must repair them from the ledger. - for segment in Path(tmp).glob("prism-audit-share-segment-*"): - segment.unlink() - loaded: list[tuple[int, int]] = [] - - def load_range(*, first_share_seq: int, last_share_seq: int) -> list[Any]: - loaded.append((first_share_seq, last_share_seq)) - return [ - {"share_seq": share_seq, "share_id": f"s{share_seq}"} - for share_seq in range(first_share_seq, last_share_seq + 1) - ] - - with unittest.mock.patch.object( - ledger, - "_load_audit_share_ledger_range", - side_effect=load_range, - ): - recovered = ledger.canonical_bundle_bytes_for_backfill( - block_hash="aa" * 32, - audit_bundle_sha256=digest, - body_uri=body_uri, - ) - self.assertEqual(recovered, fake_audit_bundle_bytes(bundle)) - self.assertEqual(loaded, [(1, 2)]) - - def test_preflight_canonical_bundle_publication_fences_each_backfill_write(self) -> None: - ledger = FakeLeasePsqlShareLedger( - [ - acquired_lease(), - { - "block_hash": "aa" * 32, - "audit_bundle_sha256": "22" * 32, - "body_uri": "prism-audit-bundle-body-x.json", - "has_inline_bundle": False, - }, - ] - ) - result = ledger.preflight_canonical_bundle_publication( - block_hash="aa" * 32, - audit_bundle_sha256="22" * 32, - ) - self.assertEqual( - result, - { - "block_hash": "aa" * 32, - "audit_bundle_sha256": "22" * 32, - "body_uri": "prism-audit-bundle-body-x.json", - "has_inline_bundle": False, - }, - ) - query = ledger.lease_queries[-1] - # Refreshes the lease this writer already holds, and reads the audit - # row it is about to answer for. No schema change, no row rewrite. - self.assertIn("UPDATE qbit_ledger_writer_lease", query) - self.assertIn("lease_expires_at = clock_timestamp() + make_interval", query) - self.assertIn("qbit_ledger_writer_lease.writer_session_token = data->>", query) - self.assertIn("FROM qbit_pool_audit_bundles", query) - self.assertIn("'writer lease is not active'", query) - self.assertIn("'audit bundle digest does not match the stored row'", query) - for forbidden in ( - "ALTER TABLE", - "CREATE ", - "INSERT INTO", - "DELETE FROM", - "UPDATE qbit_pool_audit_bundles", - ): - with self.subTest(forbidden=forbidden): - self.assertNotIn(forbidden, query) - - def test_preflight_canonical_bundle_publication_fails_closed(self) -> None: - for failure, expected in ( - ({"error": "writer lease is not active"}, "writer lease is not active"), - ({"error": "audit bundle row does not exist"}, "row does not exist"), - ( - {"error": "audit bundle digest does not match the stored row"}, - "digest does not match", - ), - (None, "returned no row"), - ): - with self.subTest(failure=failure): - ledger = FakeLeasePsqlShareLedger([acquired_lease(), failure]) - with self.assertRaisesRegex(RuntimeError, expected): - ledger.preflight_canonical_bundle_publication( - block_hash="aa" * 32, - audit_bundle_sha256="22" * 32, - ) - ledger = FakeLeasePsqlShareLedger([acquired_lease()]) - for block_hash, digest in (("aa" * 31, "22" * 32), ("aa" * 32, "zz" * 32)): - with self.subTest(block_hash=block_hash, digest=digest): - with self.assertRaises(ValueError): - ledger.preflight_canonical_bundle_publication( - block_hash=block_hash, - audit_bundle_sha256=digest, - ) - - def test_psql_external_body_path_must_stay_under_store(self) -> None: - with tempfile.TemporaryDirectory() as tmp: - ledger = FakeLeasePsqlShareLedger([acquired_lease()], audit_body_dir=tmp) - with self.assertRaisesRegex(RuntimeError, "escapes audit body store"): - ledger._read_external_body("/tmp/prism-audit-bundle-body-aa.json") - - def test_psql_audit_bundle_readers_select_external_body_pointer(self) -> None: - ledger = FakeLeasePsqlShareLedger( - [ - acquired_lease(), - { - "block_hash": "aa" * 32, - "audit_bundle_sha256": "22" * 32, - "coinbase_tx_hex": "00", - "audit_bundle": {"schema": "qbit.prism.audit-bundle.v1"}, - "body_uri": None, - }, - { - "block_hash": "aa" * 32, - "audit_commitment_leaf_hex": "ab" * 32, - "audit_bundle_sha256": "22" * 32, - "coinbase_tx_hex": "00", - "audit_bundle": {"schema": "qbit.prism.audit-bundle.v1"}, - "body_uri": None, - }, - ] - ) - - ledger.audit_bundle(block_hash="aa" * 32) - by_hash_query = ledger.lease_queries[-1] - self.assertIn("'body_uri', bundle.body_uri", by_hash_query) - self.assertIn("'block_height', block.block_height", by_hash_query) - self.assertIn("'payout_manifest_sha256', block.payout_manifest_sha256", by_hash_query) - self.assertIn("JOIN qbit_pool_blocks block", by_hash_query) - - ledger.audit_bundle_by_commitment(commitment_leaf_hex="ab" * 32) - by_commitment_query = ledger.lease_queries[-1] - self.assertIn("'body_uri', bundle.body_uri", by_commitment_query) - self.assertIn("'block_height', block.block_height", by_commitment_query) - self.assertIn("'payout_manifest_sha256', block.payout_manifest_sha256", by_commitment_query) - # Queries the promoted leaf columns (new rows) plus the inline JSONB - # (legacy rows), and orders by chain height rather than row creation time. - self.assertIn("bundle.audit_commitment_leaves_hex ?", by_commitment_query) - self.assertIn("bundle.audit_bundle->'audit_commitment_leaves_hex' ?", by_commitment_query) - self.assertIn("ORDER BY block.block_height DESC", by_commitment_query) - - def test_postgres_startup_waits_for_same_writer_predecessor_lease(self) -> None: - stdout = io.StringIO() - - with contextlib.redirect_stdout(stdout): - ledger = FakeLeasePsqlShareLedger( - [ - held_lease(wait_seconds=2.5), - held_lease(wait_seconds=0.1), - acquired_lease(session="replacement-session"), - ], - writer_id="writer-a", - writer_epoch=1, - ) - - # The first wait is clamped by the fixture's retry ceiling (the - # adoption silence), the second by the retry floor. - retry_ceiling = max(1.0, DEFAULT_WRITER_LEASE_ADOPTION_SILENCE_SECONDS) - self.assertEqual(ledger.sleeps, [retry_ceiling, 0.25]) - self.assertEqual(len(ledger.lease_queries), 3) - self.assertIn( - "prism ledger writer lease held until 2026-06-26 19:50:22.233718+00; " - f"waiting {retry_ceiling:.3g}s before retry", - stdout.getvalue(), - ) - self.assertIn("holder writer=writer-a epoch=1 session=old-session", stdout.getvalue()) - - def test_heartbeat_session_acquires_advisory_guard_before_lease_query(self) -> None: - attempts: list[object] = [] - - class FakeGuard: - def __init__(self, acquired: bool) -> None: - self.acquired = acquired - self.held = False - self.closed = False - - def try_acquire(self) -> bool: - attempts.append(self) - self.held = self.acquired - return self.acquired - - def close(self) -> None: - self.closed = True - self.held = False - - guards = [FakeGuard(False), FakeGuard(True)] - - class GuardedLedger(FakeLeasePsqlShareLedger): - def _make_writer_lease_guard(self, _database_url: str | None) -> Any: - return guards.pop(0) - - session = f"{WRITER_LEASE_HEARTBEAT_SESSION_PREFIX}guarded" - ledger = GuardedLedger( - [acquired_lease(session=session)], - writer_session_token=session, - ) - - self.assertEqual(len(attempts), 2) - self.assertTrue(attempts[0].closed) # type: ignore[attr-defined] - self.assertTrue(ledger.writer_lease_fast_adoption_capable) - self.assertEqual(ledger.sleeps, [0.25]) - self.assertEqual(len(ledger.lease_queries), 1) - - def test_psql_only_session_downgrades_fast_adoption_capability(self) -> None: - class NoGuardLedger(FakeLeasePsqlShareLedger): - def _make_writer_lease_guard(self, _database_url: str | None) -> None: - return None - - requested_session = f"{WRITER_LEASE_HEARTBEAT_SESSION_PREFIX}requested" - ledger = NoGuardLedger( - [acquired_lease()], - writer_session_token=requested_session, - ) - - self.assertFalse(ledger.writer_lease_fast_adoption_capable) - self.assertFalse( - ledger._writer_session_token.startswith( - WRITER_LEASE_HEARTBEAT_SESSION_PREFIX - ) - ) - - def test_postgres_startup_adopts_after_one_guard_acquisition_silence(self) -> None: - updated_at = "2026-06-26 19:49:22.233718+00" - old_session = f"{WRITER_LEASE_HEARTBEAT_SESSION_PREFIX}old-session" - new_session = f"{WRITER_LEASE_HEARTBEAT_SESSION_PREFIX}new-session" - stdout = io.StringIO() - clock = FakeMonotonicClock() - sleeps: list[float] = [] - - def sleep_and_advance(seconds: float) -> None: - sleeps.append(seconds) - clock.sleep(seconds) - - with contextlib.redirect_stdout(stdout), unittest.mock.patch.object( - share_ledger_module.time, - "monotonic", - clock.monotonic, - ): - ledger = FakeLeasePsqlShareLedger( - [ - held_lease( - session=old_session, - updated_at=updated_at, - age_seconds=DEFAULT_WRITER_LEASE_ADOPTION_SILENCE_SECONDS, - ), - held_lease( - session=old_session, - updated_at=updated_at, - age_seconds=( - DEFAULT_WRITER_LEASE_ADOPTION_SILENCE_SECONDS * 2 - ), - ), - acquired_lease(session=new_session) | {"adopted": True}, - ], - writer_id="writer-a", - writer_epoch=1, - writer_session_token=new_session, - lease_retry_sleep=sleep_and_advance, - ) - - # The row was already silent for a full interval, but adoption still - # waits out one interval measured from this process's own guard - # acquisition before the CAS. - self.assertEqual(sleeps, [DEFAULT_WRITER_LEASE_ADOPTION_SILENCE_SECONDS]) - self.assertEqual(len(ledger.lease_queries), 3) - self.assertNotIn("observed_writer_session_token", ledger.lease_queries[0]) - self.assertNotIn("observed_writer_session_token", ledger.lease_queries[1]) - adoption_query = ledger.lease_queries[2] - self.assertIn("observed_writer_session_token", adoption_query) - self.assertIn("observed_lease_updated_at", adoption_query) - self.assertIn("qbit_ledger_writer_lease.updated_at =", adoption_query) - self.assertIn(old_session, adoption_query) - self.assertIn(updated_at, adoption_query) - self.assertIn("adopted from same-identity predecessor", stdout.getvalue()) - - def test_successor_waits_full_guard_silence_despite_minutes_stale_lease_row(self) -> None: - """Regression: block-39416 restart loop. - - A predecessor dying inside a long persist_accepted_block transaction - leaves updated_at minutes old the moment its guard session drops. - The replacement must still grant it a full adoption-silence interval - measured from advisory-guard acquisition so it can self-fence, no - matter how stale the lease row already looks. - """ - updated_at = "2026-06-26 19:44:22.233718+00" - old_session = f"{WRITER_LEASE_HEARTBEAT_SESSION_PREFIX}old-session" - new_session = f"{WRITER_LEASE_HEARTBEAT_SESSION_PREFIX}new-session" - clock = FakeMonotonicClock() - sleeps: list[float] = [] - - def sleep_and_advance(seconds: float) -> None: - sleeps.append(seconds) - clock.sleep(seconds) - - with contextlib.redirect_stdout(io.StringIO()), unittest.mock.patch.object( - share_ledger_module.time, - "monotonic", - clock.monotonic, - ): - ledger = FakeLeasePsqlShareLedger( - [ - held_lease( - session=old_session, - updated_at=updated_at, - age_seconds=300.0, - ), - held_lease( - session=old_session, - updated_at=updated_at, - age_seconds=( - 300.0 + DEFAULT_WRITER_LEASE_ADOPTION_SILENCE_SECONDS - ), - ), - acquired_lease(session=new_session) | {"adopted": True}, - ], - writer_id="writer-a", - writer_epoch=1, - writer_session_token=new_session, - lease_retry_sleep=sleep_and_advance, - ) - - self.assertEqual(sleeps, [DEFAULT_WRITER_LEASE_ADOPTION_SILENCE_SECONDS]) - self.assertEqual(len(ledger.lease_queries), 3) - # No CAS before the guard-acquisition silence elapsed. - self.assertNotIn("observed_writer_session_token", ledger.lease_queries[0]) - self.assertNotIn("observed_writer_session_token", ledger.lease_queries[1]) - adoption_query = ledger.lease_queries[2] - # The CAS still targets the exact predecessor session and row state. - self.assertIn(old_session, adoption_query) - self.assertIn(updated_at, adoption_query) - - def test_postgres_startup_keeps_ttl_fallback_for_legacy_session(self) -> None: - class StopAfterFirstSleep(RuntimeError): - pass - - def stop_after_first_sleep(_seconds: float) -> None: - raise StopAfterFirstSleep - - ledger = FakeLeasePsqlShareLedger.__new__(FakeLeasePsqlShareLedger) - with self.assertRaises(StopAfterFirstSleep): - ledger.__init__( - [ - held_lease( - session="legacy-session", - updated_at="2026-06-26 19:49:22.233718+00", - age_seconds=5.0, - ) - ], - writer_id="writer-a", - writer_epoch=1, - writer_session_token=( - f"{WRITER_LEASE_HEARTBEAT_SESSION_PREFIX}new-session" - ), - lease_retry_sleep=stop_after_first_sleep, - ) - - self.assertEqual(len(ledger.lease_queries), 1) - self.assertNotIn("observed_writer_session_token", ledger.lease_queries[0]) - - def test_postgres_startup_refuses_concurrently_renewing_same_identity_session(self) -> None: - first_updated_at = "2026-06-26 19:49:22.233718+00" - renewed_updated_at = "2026-06-26 19:49:23.233718+00" - old_session = f"{WRITER_LEASE_HEARTBEAT_SESSION_PREFIX}old-session" - new_session = f"{WRITER_LEASE_HEARTBEAT_SESSION_PREFIX}new-session" - clock = FakeMonotonicClock() - sleeps: list[float] = [] - - class StopAfterRenewalObserved(RuntimeError): - pass - - def stop_after_second_sleep(seconds: float) -> None: - sleeps.append(seconds) - clock.sleep(seconds) - if len(sleeps) == 2: - raise StopAfterRenewalObserved - - ledger = FakeLeasePsqlShareLedger.__new__(FakeLeasePsqlShareLedger) - with self.assertRaises(StopAfterRenewalObserved), unittest.mock.patch.object( - share_ledger_module.time, - "monotonic", - clock.monotonic, - ): - ledger.__init__( - [ - held_lease( - session=old_session, - updated_at=first_updated_at, - age_seconds=0.1, - ), - held_lease( - session=old_session, - updated_at=renewed_updated_at, - age_seconds=0.1, - ), - ], - writer_id="writer-a", - writer_epoch=1, - writer_session_token=new_session, - lease_retry_sleep=stop_after_second_sleep, - ) - - # First wait is floored by the guard-acquisition silence; once that - # elapsed, the renewing twin's fresh updated_at keeps gating the CAS - # through the row-silence edge (the silence less the row's 0.1s age). - self.assertEqual( - sleeps, - [ - DEFAULT_WRITER_LEASE_ADOPTION_SILENCE_SECONDS, - round(DEFAULT_WRITER_LEASE_ADOPTION_SILENCE_SECONDS - 0.1, 6), - ], - ) - self.assertEqual(len(ledger.lease_queries), 2) - self.assertTrue( - all("observed_writer_session_token" not in query for query in ledger.lease_queries) - ) - - def test_postgres_startup_reobserves_after_losing_adoption_cas(self) -> None: - first_updated_at = "2026-06-26 19:49:22.233718+00" - renewed_updated_at = "2026-06-26 19:49:23.233718+00" - old_session = f"{WRITER_LEASE_HEARTBEAT_SESSION_PREFIX}old-session" - new_session = f"{WRITER_LEASE_HEARTBEAT_SESSION_PREFIX}new-session" - clock = FakeMonotonicClock() - sleeps: list[float] = [] - - def sleep_and_advance(seconds: float) -> None: - sleeps.append(seconds) - clock.sleep(seconds) - - with contextlib.redirect_stdout(io.StringIO()), unittest.mock.patch.object( - share_ledger_module.time, - "monotonic", - clock.monotonic, - ): - ledger = FakeLeasePsqlShareLedger( - [ - held_lease( - session=old_session, - updated_at=first_updated_at, - age_seconds=( - DEFAULT_WRITER_LEASE_ADOPTION_SILENCE_SECONDS + 0.1 - ), - ), - held_lease( - session=old_session, - updated_at=first_updated_at, - age_seconds=( - 2 * DEFAULT_WRITER_LEASE_ADOPTION_SILENCE_SECONDS - + 0.1 - ), - ), - held_lease( - session=old_session, - updated_at=renewed_updated_at, - age_seconds=0.0, - ), - held_lease( - session=old_session, - updated_at=renewed_updated_at, - age_seconds=( - DEFAULT_WRITER_LEASE_ADOPTION_SILENCE_SECONDS + 0.1 - ), - ), - acquired_lease(session=new_session) | {"adopted": True}, - ], - writer_id="writer-a", - writer_epoch=1, - writer_session_token=new_session, - lease_retry_sleep=sleep_and_advance, - ) - - # Guard-acquisition silence first, then a lost CAS re-observes the - # renewed row and requires a fresh full row-silence interval before - # the second CAS. - self.assertEqual( - sleeps, - [DEFAULT_WRITER_LEASE_ADOPTION_SILENCE_SECONDS] * 2, - ) - self.assertEqual(len(ledger.lease_queries), 5) - self.assertIn("observed_writer_session_token", ledger.lease_queries[2]) - self.assertIn("observed_writer_session_token", ledger.lease_queries[4]) - self.assertNotIn("observed_writer_session_token", ledger.lease_queries[3]) - - def test_postgres_startup_never_adopts_different_writer_or_epoch(self) -> None: - old_session = f"{WRITER_LEASE_HEARTBEAT_SESSION_PREFIX}old-session" - new_session = f"{WRITER_LEASE_HEARTBEAT_SESSION_PREFIX}new-session" - for holder in ( - held_lease( - writer_id="writer-b", - session=old_session, - updated_at="2026-06-26 19:49:22+00", - age_seconds=2.0, - ), - held_lease( - writer_epoch=2, - session=old_session, - updated_at="2026-06-26 19:49:22+00", - age_seconds=2.0, - ), - ): - with self.subTest(holder=holder): - ledger = FakeLeasePsqlShareLedger.__new__(FakeLeasePsqlShareLedger) - with self.assertRaisesRegex(RuntimeError, "qbit ledger writer lease is held by"): - ledger.__init__( - [holder], - writer_id="writer-a", - writer_epoch=1, - writer_session_token=new_session, - ) - self.assertEqual(len(ledger.lease_queries), 1) - self.assertNotIn("observed_lease_updated_at", ledger.lease_queries[0]) - self.assertFalse(ledger.fake_writer_lease_guard.held) - - def test_postgres_startup_refuses_another_active_writer_lease(self) -> None: - stdout = io.StringIO() - - with contextlib.redirect_stdout(stdout): - with self.assertRaisesRegex(RuntimeError, "qbit ledger writer lease is held by writer-b epoch=1"): - FakeLeasePsqlShareLedger( - [held_lease(writer_id="writer-b", wait_seconds=10.0)], - writer_id="writer-a", - writer_epoch=1, - ) - - self.assertNotIn("waiting", stdout.getvalue()) - - def test_postgres_startup_acquires_expired_lease_immediately(self) -> None: - stdout = io.StringIO() - - with contextlib.redirect_stdout(stdout): - ledger = FakeLeasePsqlShareLedger( - [acquired_lease()], - writer_id="writer-a", - writer_epoch=1, - ) - - self.assertEqual(ledger.sleeps, []) - self.assertEqual(len(ledger.lease_queries), 1) - self.assertEqual(stdout.getvalue(), "") - - -class FirstAcquireWriterLeaseRaceTests(unittest.TestCase): - """The first-ever concurrent acquisition, from the loser's side. - - Under READ COMMITTED the loser's ``ON CONFLICT DO UPDATE`` re-reads only - the conflicting tuple after its lock wait, so the upsert affects zero rows - while the holder ``SELECT`` still reads the pre-wait snapshot and finds - nothing. Both arms empty used to make the whole statement SQL NULL and kill - startup with a driver-level ``postgres query returned no JSON``. These - tests drive that interleaving through canned statement results, so they - pin the loser's experience without a live server. - """ - - def test_statement_carries_a_named_retry_arm_for_the_empty_snapshot(self) -> None: - ledger = FakeLeasePsqlShareLedger( - [acquired_lease()], - writer_id="writer-a", - writer_epoch=1, - ) - - acquire_sql = ledger.lease_queries[0] - # A third COALESCE arm makes the statement total: it is a constant, so - # the expression cannot evaluate to SQL NULL however the first two arms - # resolve. - self.assertIn(f"'{WRITER_LEASE_ACQUIRE_RETRY_KEY}', true", acquire_sql) - self.assertIn(f"'lease', '{WRITER_LEASE_ACQUIRE_RETRY_SUBJECT}'", acquire_sql) - self.assertIn("'acquired', false", acquire_sql) - - def test_first_acquire_loser_retries_into_the_named_lease_refusal(self) -> None: - stdout = io.StringIO() - - with contextlib.redirect_stdout(stdout): - with self.assertRaises(RuntimeError) as caught: - FakeLeasePsqlShareLedger( - [ - snapshot_retry_lease(), - held_lease(writer_id="writer-b", wait_seconds=10.0), - ], - writer_id="writer-a", - writer_epoch=1, - ) - - message = str(caught.exception) - # The operator-facing difference this change exists for: the loser now - # names the lease and its holder instead of a driver-level parse error. - self.assertIn( - "qbit ledger writer lease is held by writer-b epoch=1", - message, - ) - self.assertNotIn("postgres query returned no JSON", message) - - def test_first_acquire_loser_can_retry_into_an_ordinary_acquisition(self) -> None: - # The winner's row is visible on the fresh snapshot and has already - # expired, so the second statement takes the lease through the ordinary - # expiry CAS. Startup completes without operator action. - stdout = io.StringIO() - - with contextlib.redirect_stdout(stdout): - ledger = FakeLeasePsqlShareLedger( - [snapshot_retry_lease(), acquired_lease()], - writer_id="writer-a", - writer_epoch=1, - ) - - self.assertEqual(len(ledger.lease_queries), 2) - self.assertEqual(ledger.sleeps, []) - self.assertIn("fresh statement snapshot", stdout.getvalue()) - - def test_retry_result_never_reaches_the_wait_or_adopt_decisions(self) -> None: - # The sentinel carries no holder identity, so letting it out of the - # acquisition path would be read as a holder of None. It is consumed - # where the remedy is -- a fresh statement snapshot -- and the caller - # only ever sees an ordinary arm. - ledger = PsqlShareLedger.__new__(PsqlShareLedger) - ledger._writer_id = "writer-a" - ledger._writer_epoch = 1 - ledger._writer_session_token = ( - f"{WRITER_LEASE_HEARTBEAT_SESSION_PREFIX}session-a" - ) - retry = snapshot_retry_lease() - - self.assertFalse(PsqlShareLedger._can_wait_for_writer_lease(ledger, retry)) - self.assertIsNone( - PsqlShareLedger._writer_lease_adoption_wait_seconds(ledger, retry) - ) - self.assertFalse(retry["acquired"]) - - def test_unconverging_snapshot_retry_fails_naming_the_lease(self) -> None: - # A caller whose transaction pinned one snapshot for its whole life - # cannot advance it by re-running. The bound turns that into a visible - # failure that still names the lease, never a raw parser error and - # never an unbounded loop. - stdout = io.StringIO() - - with contextlib.redirect_stdout(stdout): - with self.assertRaises(RuntimeError) as caught: - FakeLeasePsqlShareLedger( - [snapshot_retry_lease()] * WRITER_LEASE_ACQUIRE_RETRY_ATTEMPTS, - writer_id="writer-a", - writer_epoch=1, - ) - - message = str(caught.exception) - self.assertIn(WRITER_LEASE_ACQUIRE_RETRY_SUBJECT, message) - self.assertNotIn("postgres query returned no JSON", message) - - def test_unrelated_empty_json_statement_still_raises_the_parser_error(self) -> None: - # The lease statement is what became total; the parser is unchanged, so - # any other statement returning SQL NULL still fails loudly rather than - # being reinterpreted as a retry. - with self.assertRaisesRegex( - RuntimeError, - "postgres query returned no JSON", - ): - parse_single_json_value(None) - - class FakeCursor: - @staticmethod - def fetchone() -> tuple[object]: - return (None,) - - class FakeConnection: - closed = False - - @staticmethod - def execute(sql: str) -> FakeCursor: - return FakeCursor() - - def close(self) -> None: - pass - - class FakeOperationalError(Exception): - pass - - fake_psycopg = types.ModuleType("psycopg") - fake_psycopg.connect = lambda conninfo, **kwargs: FakeConnection() # type: ignore[attr-defined] - fake_psycopg.conninfo = types.SimpleNamespace( # type: ignore[attr-defined] - conninfo_to_dict=lambda conninfo: {"dbname": "qbit"} - ) - fake_psycopg.OperationalError = FakeOperationalError # type: ignore[attr-defined] - - with unittest.mock.patch.dict(sys.modules, {"psycopg": fake_psycopg}): - client = _NativePostgresClient( - "postgresql://example.invalid/qbit", - pool_size=1, - ) - with self.assertRaisesRegex( - RuntimeError, - "postgres query returned no JSON", - ): - client.run_json("SELECT to_json(NULL);") - - -class CannedQueryPsqlShareLedger(PsqlShareLedger): - """Subprocess-mode ledger whose _run_json pops canned results. - - Mirrors FakeLeasePsqlShareLedger but is used for the hot-path stats and - native-backend tests, which also need the recorded SQL for assertions. - """ - - def __init__(self, results: list[object], **kwargs: Any): - self.canned_results = list(results) - self.queries: list[str] = [] - kwargs.setdefault("native_client_mode", "0") - super().__init__( - psql_command="psql postgresql://example.invalid/qbit", - lease_retry_sleep=lambda _seconds: None, - **kwargs, - ) - - def _run_json(self, sql: str) -> Any: - self.queries.append(sql) - if not self.canned_results: - raise AssertionError(f"unexpected extra query: {sql[:120]}") - result = self.canned_results.pop(0) - if isinstance(result, Exception): - raise result - return result - - -def acquired_lease_result() -> dict[str, object]: - return acquired_lease(writer_id="prism-coordinator", writer_epoch=1) - - -def record_payload( - index: int, - share_seq: int, - *, - new_miner: bool = False, -) -> dict[str, object]: - pending = pending_share(index) - return { - "share_seq": share_seq, - "share_id": pending.share_id, - "miner_id": pending.miner_id, - "order_key": pending.order_key, - "p2mr_program_hex": pending.p2mr_program_hex, - "share_difficulty": str(pending.share_difficulty), - "network_difficulty": str(pending.network_difficulty), - "template_height": pending.template_height, - "job_id": pending.job_id, - "job_issued_at_ms": pending.job_issued_at_ms, - "accepted_at_ms": pending.accepted_at_ms, - "ntime": pending.ntime, - "credit_policy": None, - "new_miner": new_miner, - } - - -def stats_payload( - count: int, - distinct_miner_count: int, - max_share_seq: int, -) -> dict[str, object]: - return { - "accepted_share_count": count, - "distinct_miner_count": distinct_miner_count, - "max_share_seq": max_share_seq, - } - - -class NativeClientSelectionTests(unittest.TestCase): - def test_operation_timeout_bounds_local_writer_lock_admission(self) -> None: - ledger = PsqlShareLedger.__new__(PsqlShareLedger) - ledger._operation_timeout_local = threading.local() - gate = threading.Lock() - gate.acquire() - started = time.monotonic() - try: - with ledger.operation_timeout(0.02): - with self.assertRaisesRegex( - LedgerOperationTimeout, - "writer lock", - ): - with ledger._operation_gate(gate, "writer lock"): - self.fail("contended writer lock unexpectedly acquired") - finally: - gate.release() - self.assertLess(time.monotonic() - started, 0.5) - - def test_operation_gate_stamps_progress_while_admission_is_blocked(self) -> None: - """A blocked admission wait must report liveness to its caller. - - Landing-class callers run this wait on a watchdog-monitored thread. - No statement has been sent yet, so nothing on the server can bound or - report the wait; without the progress hook the whole admission budget - is silence and the watchdog kills a coordinator that is merely queued - behind another writer. Slicing must not extend the wait either: the - caller's deadline still ends it with the same error. - """ - ledger = PsqlShareLedger.__new__(PsqlShareLedger) - ledger._operation_timeout_local = threading.local() - ledger._statement_timeout_local = threading.local() - gate = threading.Lock() - gate.acquire() - stamps: list[float] = [] - started = time.monotonic() - try: - with ledger.statement_timeout(0.4): - with ledger.operation_progress( - lambda: stamps.append(time.monotonic()), - slice_seconds=0.05, - ): - with self.assertRaisesRegex( - LedgerOperationTimeout, - "writer lock", - ): - with ledger._operation_gate(gate, "writer lock"): - self.fail("contended writer lock unexpectedly acquired") - finally: - gate.release() - elapsed = time.monotonic() - started - self.assertGreaterEqual(len(stamps), 2) - # No per-gap upper bound: a loaded CI host can stall a 0.05s slice - # wakeup well past any tight cutoff, and cadence is already proven - # deterministically under a virtual clock elsewhere in this module. - self.assertGreaterEqual(elapsed, 0.4) - self.assertLess(elapsed, 3.0) - - def test_operation_gate_slicing_times_out_on_the_virtual_clock(self) -> None: - """The sliced admission wait is driven by the injected clock. - - With a virtual clock installed and a gate that consumes each slice - without yielding, the deadline must expire exactly when the virtual - clock crosses the admission budget — never on wall time — so the - slicing loop can be exercised deterministically. - """ - clock = FakeMonotonicClock() - started = clock.now - - class SliceConsumingGate: - def __init__(self) -> None: - self.timeouts: list[float] = [] - - def acquire(self, timeout: float = -1) -> bool: - self.timeouts.append(timeout) - if len(self.timeouts) > 50: - raise AssertionError( - "admission wait ignored the virtual-clock deadline" - ) - clock.sleep(timeout) - return False - - def release(self) -> None: - raise AssertionError("gate released without being acquired") - - ledger = PsqlShareLedger.__new__(PsqlShareLedger) - ledger._operation_timeout_local = threading.local() - ledger._statement_timeout_local = threading.local() - ledger._monotonic = clock.monotonic - gate = SliceConsumingGate() - stamps: list[float] = [] - with ledger.statement_timeout(1.0): - with ledger.operation_progress( - lambda: stamps.append(clock.now), - slice_seconds=0.25, - ): - with self.assertRaisesRegex(LedgerOperationTimeout, "writer lock"): - ledger._acquire_operation_gate(gate, "writer lock") - self.assertEqual(clock.now, started + 1.0) - self.assertEqual(gate.timeouts, [0.25, 0.25, 0.25, 0.25]) - self.assertEqual( - stamps, - [started + 0.25, started + 0.5, started + 0.75], - ) - - def test_operation_gate_without_progress_hook_waits_once(self) -> None: - """The unhooked path is still a single blocking acquire. - - Ordinary ledger callers have no liveness monitor to satisfy and must - not start paying for wakeups they cannot use. - """ - acquires: list[float] = [] - - class RecordingGate: - def acquire(self, timeout: float = -1) -> bool: - acquires.append(timeout) - return False - - def release(self) -> None: - raise AssertionError("gate released without being acquired") - - ledger = PsqlShareLedger.__new__(PsqlShareLedger) - ledger._operation_timeout_local = threading.local() - ledger._statement_timeout_local = threading.local() - with ledger.statement_timeout(0.02): - with self.assertRaisesRegex(LedgerOperationTimeout, "writer lock"): - with ledger._operation_gate(RecordingGate(), "writer lock"): - self.fail("contended writer lock unexpectedly acquired") - self.assertEqual(acquires, [0.02]) - - def test_operation_progress_validates_and_restores_its_slice(self) -> None: - """The hook scope matches its sibling timeout scopes exactly.""" - ledger = PsqlShareLedger.__new__(PsqlShareLedger) - for slice_seconds in (0.0, -1.0, float("inf"), float("nan")): - with self.subTest(slice_seconds=slice_seconds): - with self.assertRaisesRegex(ValueError, "finite and positive"): - with ledger.operation_progress( - lambda: None, - slice_seconds=slice_seconds, - ): - self.fail("invalid progress slice unexpectedly accepted") - self.assertIsNone(ledger._operation_progress_hook()) - def outer() -> None: - return None - - def inner() -> None: - return None - - with ledger.operation_progress(outer, slice_seconds=1.0): - self.assertEqual(ledger._operation_progress_hook(), (outer, 1.0)) - with ledger.operation_progress(inner, slice_seconds=0.25): - self.assertEqual(ledger._operation_progress_hook(), (inner, 0.25)) - self.assertEqual(ledger._operation_progress_hook(), (outer, 1.0)) - self.assertIsNone(ledger._operation_progress_hook()) - - def test_operation_progress_local_is_created_before_any_scope(self) -> None: - """The production path must never lazily create the thread-local. - - ``operation_progress``'s ``getattr``-then-assign is not atomic, so a - ledger that reaches its first scope without the attribute already in - place can lose one caller's hook to a concurrent installer. - """ - with contextlib.redirect_stdout(io.StringIO()): - ledger = FakeLeasePsqlShareLedger( - [acquired_lease()], - writer_id="writer-a", - writer_epoch=1, - ) - self.assertIn("_operation_progress_local", vars(ledger)) - self.assertIsInstance( - vars(ledger)["_operation_progress_local"], - threading.local, - ) - - def test_concurrent_first_scopes_share_one_progress_local(self) -> None: - """Two first-ever scopes at once must both keep their own hook. - - The lazy check-then-set lets both callers build a ``local()`` and one - assignment win; the loser's hook then lives on an object the ledger - no longer references, so its admission wait silently reverts to the - heartbeat-silent path -- the exact failure this hook exists to - remove. The patched ``local`` below makes that interleaving - deterministic rather than a sleep race: it parks every constructing - thread on a two-party barrier, so neither can assign before the other - has decided to construct. A ledger that built the thread-local in - ``__init__`` never constructs one here at all, and the barrier is - simply never reached. - """ - with contextlib.redirect_stdout(io.StringIO()): - ledger = FakeLeasePsqlShareLedger( - [acquired_lease()], - writer_id="writer-a", - writer_epoch=1, - ) - constructing = threading.Barrier(2, timeout=10) - installed = threading.Barrier(2, timeout=10) - - def racing_local() -> threading.local: - # Neither thread may leave the constructor until both have - # decided to build one, so the check-then-set cannot serialize - # itself by luck. - constructing.wait() - return threading.local() - - def first_stamp() -> None: - return None - - def second_stamp() -> None: - return None - - observed: dict[str, object] = {} - errors: list[BaseException] = [] - - def install(name: str, hook: Any) -> None: - try: - with ledger.operation_progress(hook, slice_seconds=0.5): - # Both hooks are installed before either is read, so a - # lost assignment cannot hide behind thread ordering. - installed.wait() - observed[name] = ledger._operation_progress_hook() - except BaseException as exc: # pragma: no cover - asserted below - errors.append(exc) - - with unittest.mock.patch.object( - share_ledger_module, - "local", - racing_local, - ): - threads = [ - threading.Thread(target=install, args=("first", first_stamp)), - threading.Thread(target=install, args=("second", second_stamp)), - ] - for thread in threads: - thread.start() - for thread in threads: - thread.join(10) - self.assertEqual(errors, []) - self.assertEqual(observed.get("first"), (first_stamp, 0.5)) - self.assertEqual(observed.get("second"), (second_stamp, 0.5)) - - def test_operation_progress_nesting_keeps_the_tighter_slice(self) -> None: - """Nesting tightens the stamp cadence, exactly like its siblings. - - ``operation_timeout`` and ``statement_timeout`` merge by minimum, so - a nested scope can only narrow what the enclosing caller allowed. A - progress scope that replaced the slice outright would let an inner - caller widen the heartbeat gaps an outer caller had already sized - against its own monitor. - """ - ledger = PsqlShareLedger.__new__(PsqlShareLedger) - - def outer() -> None: - return None - - def inner() -> None: - return None - - with ledger.operation_progress(outer, slice_seconds=0.25): - self.assertEqual(ledger._operation_progress_hook(), (outer, 0.25)) - # A larger inner slice does not widen the cadence; the callback - # is still the inner caller's, because it is the one whose - # liveness the wait now belongs to. - with ledger.operation_progress(inner, slice_seconds=4.0): - self.assertEqual( - ledger._operation_progress_hook(), - (inner, 0.25), - ) - self.assertEqual(ledger._operation_progress_hook(), (outer, 0.25)) - self.assertIsNone(ledger._operation_progress_hook()) - - def test_operation_gate_propagates_progress_hook_failures(self) -> None: - """A liveness stamp that cannot be taken is a failure, not noise. - - Swallowing it would leave the caller blocked inside a lock wait it - believes is being reported, which is the exact silence this hook - exists to remove. - """ - - class Stamped(RuntimeError): - pass - - def on_progress() -> None: - raise Stamped("heartbeat unavailable") - - ledger = PsqlShareLedger.__new__(PsqlShareLedger) - ledger._operation_timeout_local = threading.local() - ledger._statement_timeout_local = threading.local() - gate = threading.Lock() - gate.acquire() - try: - with ledger.statement_timeout(5.0): - with ledger.operation_progress(on_progress, slice_seconds=0.01): - with self.assertRaises(Stamped): - with ledger._operation_gate(gate, "writer lock"): - self.fail("contended writer lock unexpectedly acquired") - finally: - gate.release() - - def test_note_operation_progress_reports_only_when_a_hook_is_installed(self) -> None: - """The between-statements reporter is opt-in, exactly like the wait. - - Ordinary ledger callers have no liveness monitor, so an operation - that reports unconditionally would be paying for stamps nobody - reads. The hook is installed only by callers whose thread is - watchdog-monitored. - """ - ledger = PsqlShareLedger.__new__(PsqlShareLedger) - ledger._operation_timeout_local = threading.local() - ledger._statement_timeout_local = threading.local() - # No hook installed: reporting must be a silent no-op, not an error. - ledger._note_operation_progress() - stamps: list[int] = [] - with ledger.operation_progress( - lambda: stamps.append(len(stamps)), - slice_seconds=0.05, - ): - ledger._note_operation_progress() - ledger._note_operation_progress() - self.assertEqual(stamps, [0, 1]) - # The scope restored the previous (absent) hook, so this is silent. - ledger._note_operation_progress() - self.assertEqual(stamps, [0, 1]) - - def test_note_operation_progress_propagates_hook_failures(self) -> None: - """A stamp that cannot be taken is a failure, as inside the gate wait.""" - - class Stamped(RuntimeError): - pass - - def on_progress() -> None: - raise Stamped("heartbeat unavailable") - - ledger = PsqlShareLedger.__new__(PsqlShareLedger) - ledger._operation_timeout_local = threading.local() - ledger._statement_timeout_local = threading.local() - with ledger.operation_progress(on_progress, slice_seconds=0.05): - with self.assertRaises(Stamped): - ledger._note_operation_progress() - - def test_confirm_accepted_block_reports_between_its_two_statements(self) -> None: - """The publication-ordinal read must not extend one silent span. - - confirm_accepted_block holds the writer lock across a mutating - statement and the follow-up read its command snapshot could not see. - No admission slice runs between them, so without a report the caller's - monitor is asked to sit through two statement budgets in a row -- at - the 120s default that is the whole watchdog tolerance with no margin - at all (issue #125). - """ - ledger = GateStatementRecordingPsqlShareLedger( - [ - {"backend": "postgres-psql", "confirmed_count": 1}, - {"audit_publication_sequence": 7}, - ] - ) - with ledger.operation_progress( - ledger.note_progress, - slice_seconds=0.05, - ): - response = ledger.confirm_accepted_block( - block_hash="ab" * 32, - active_tip_height=10, - ) - self.assertEqual(response["audit_publication_sequence"], 7) - self.assertEqual( - ledger.events, - ["statement-1", "progress", "statement-2"], - ) - - def test_confirmed_count_zero_stays_a_single_silent_statement(self) -> None: - """A one-statement gate body must not add stamps it does not owe.""" - ledger = GateStatementRecordingPsqlShareLedger( - [{"backend": "postgres-psql", "confirmed_count": 0}] - ) - with ledger.operation_progress( - ledger.note_progress, - slice_seconds=0.05, - ): - ledger.confirm_accepted_block( - block_hash="ab" * 32, - active_tip_height=10, - ) - self.assertEqual(ledger.events, ["statement-1"]) - - def test_reactivate_pool_block_reports_between_its_two_statements(self) -> None: - """The reorg-walk twin of confirm_accepted_block's ordinal read.""" - ledger = GateStatementRecordingPsqlShareLedger( - [ - {"backend": "postgres-psql", "reactivated_count": 1}, - {"audit_publication_sequence": 4}, - ] - ) - with ledger.operation_progress( - ledger.note_progress, - slice_seconds=0.05, - ): - response = ledger.reactivate_pool_block( - block_hash="cd" * 32, - active_tip_height=10, - ) - self.assertEqual(response["audit_publication_sequence"], 4) - self.assertEqual( - ledger.events, - ["statement-1", "progress", "statement-2"], - ) - - def test_carry_forward_integrity_report_reports_between_its_reads(self) -> None: - """Two reads that must see one another cannot release the lock. - - The report and its audit head are consistent only because they run - under one held writer lock, so the report between them is the only - liveness a caller's monitor can get. - """ - ledger = GateStatementRecordingPsqlShareLedger( - [ - {"checked_active_rows": 0, "mismatch_count": 0}, - [], - ] - ) - with ledger.operation_progress( - ledger.note_progress, - slice_seconds=0.05, - ): - report = ledger.carry_forward_integrity_report() - self.assertEqual(report["audit_row_count"], 0) - self.assertEqual( - ledger.events, - ["statement-1", "progress", "statement-2"], - ) - - def test_statement_timeout_refreshes_for_each_database_step(self) -> None: - ledger = PsqlShareLedger.__new__(PsqlShareLedger) - ledger._operation_timeout_local = threading.local() - ledger._statement_timeout_local = threading.local() - clock = {"now": 10.0} - - with unittest.mock.patch( - "lab.prism.share_ledger.time.monotonic", - side_effect=lambda: clock["now"], - ): - with ledger.statement_timeout(0.5): - self.assertEqual(ledger._remaining_operation_timeout(), 0.5) - clock["now"] += 60.0 - self.assertEqual(ledger._remaining_operation_timeout(), 0.5) - - def test_subprocess_operation_timeout_sets_client_and_server_deadlines(self) -> None: - ledger = PsqlShareLedger.__new__(PsqlShareLedger) - ledger._command = ["psql"] - ledger._native = None - ledger._operation_timeout_local = threading.local() - completed = unittest.mock.Mock(returncode=0, stdout="{}\n", stderr="") - - with unittest.mock.patch( - "lab.prism.share_ledger.subprocess.run", - return_value=completed, - ) as run: - with ledger.operation_timeout(0.5): - self.assertEqual(ledger._run_sql("SELECT '{}'::json;"), "{}\n") - - kwargs = run.call_args.kwargs - self.assertGreater(float(kwargs["timeout"]), 0.0) - self.assertLessEqual(float(kwargs["timeout"]), 0.5) - self.assertEqual(kwargs["env"]["PGCONNECT_TIMEOUT"], "1") - self.assertIn("statement_timeout=", kwargs["env"]["PGOPTIONS"]) - self.assertIn("lock_timeout=", kwargs["env"]["PGOPTIONS"]) - self.assertIn("VERBOSITY=verbose", run.call_args.args[0]) - - def test_subprocess_server_deadlines_raise_operation_timeout(self) -> None: - ledger = PsqlShareLedger.__new__(PsqlShareLedger) - ledger._command = ["psql"] - ledger._native = None - ledger._operation_timeout_local = threading.local() - deadline_errors = ( - "ERROR: 57014: canceling statement due to statement timeout", - "FEHLER: 57014: Anweisung wegen Zeitüberschreitung abgebrochen", - "ERROR: 55P03: canceling statement due to lock timeout", - "FEHLER: 55P03: Anweisung wegen Zeitüberschreitung abgebrochen", - "psql: error: connection to server failed: timeout expired", - ) - - for stderr in deadline_errors: - with self.subTest(stderr=stderr), unittest.mock.patch( - "lab.prism.share_ledger.subprocess.run", - return_value=unittest.mock.Mock( - returncode=3, - stdout="", - stderr=stderr, - ), - ): - with ledger.operation_timeout(0.5): - with self.assertRaises(LedgerOperationTimeout): - ledger._run_sql("SELECT '{}'::json;") - - def test_subprocess_hard_error_remains_runtime_error(self) -> None: - ledger = PsqlShareLedger.__new__(PsqlShareLedger) - ledger._command = ["psql"] - ledger._native = None - ledger._operation_timeout_local = threading.local() - completed = unittest.mock.Mock( - returncode=3, - stdout="", - stderr="ERROR: permission denied for table qbit_block_candidate_outbox", - ) - - with unittest.mock.patch( - "lab.prism.share_ledger.subprocess.run", - return_value=completed, - ), ledger.operation_timeout(0.5): - with self.assertRaisesRegex(RuntimeError, "permission denied"): - ledger._run_sql("SELECT '{}'::json;") - - def test_native_operation_timeout_is_transaction_local(self) -> None: - class OperationalError(Exception): - pass - - class FakePsycopg: - pass - - FakePsycopg.OperationalError = OperationalError # type: ignore[attr-defined] - executions: list[str] = [] - borrowed_with: list[float | None] = [] - - class FakeConnection: - @contextlib.contextmanager - def transaction(self) -> Any: - yield - - def execute(self, sql: str) -> FakeConnection: - executions.append(sql) - return self - - def fetchone(self) -> tuple[object]: - return ({"ok": True},) - - client = _NativePostgresClient.__new__(_NativePostgresClient) - client._psycopg = FakePsycopg - - @contextlib.contextmanager - def connection(*, timeout_seconds: float | None = None) -> Any: - borrowed_with.append(timeout_seconds) - yield FakeConnection() - - client.connection = connection # type: ignore[method-assign] - - self.assertEqual( - client.run_json("SELECT json_build_object('ok', true)", timeout_seconds=0.5), - {"ok": True}, - ) - self.assertEqual(len(borrowed_with), 1) - self.assertIsNotNone(borrowed_with[0]) - self.assertGreater(float(borrowed_with[0]), 0.0) - self.assertLessEqual(float(borrowed_with[0]), 0.5) - self.assertRegex(executions[0], r"^SET LOCAL statement_timeout = '\d+ms'$") - self.assertRegex(executions[1], r"^SET LOCAL lock_timeout = '\d+ms'$") - self.assertEqual(executions[2], "SELECT json_build_object('ok', true)") - - def test_native_connection_setup_expiry_does_not_start_statement_timing( - self, - ) -> None: - """A budget exhausted while connecting never becomes server time.""" - - class OperationalError(Exception): - pass - - class FakePsycopg: - pass - - FakePsycopg.OperationalError = OperationalError # type: ignore[attr-defined] - - client = _NativePostgresClient.__new__(_NativePostgresClient) - client._psycopg = FakePsycopg - starts: list[str] = [] - - @contextlib.contextmanager - def connection(*, timeout_seconds: float | None = None) -> Any: - self.assertGreater(float(timeout_seconds), 0.0) - yield unittest.mock.Mock() - - client.connection = connection # type: ignore[method-assign] - - # Establish deadline, calculate the connection budget, then discover - # that connection setup consumed it before SET LOCAL or the query. - with unittest.mock.patch.object( - share_ledger_module.time, - "monotonic", - side_effect=[0.0, 0.1, 0.6], - ): - with self.assertRaisesRegex( - LedgerOperationTimeout, - "statement deadline expired", - ): - client.run_json( - "SELECT json_build_object('ok', true)", - timeout_seconds=0.5, - on_statement_start=lambda: starts.append("started"), - ) - - self.assertEqual(starts, []) - - def test_native_server_deadline_raises_operation_timeout(self) -> None: - class OperationalError(Exception): - pass - - class FakePsycopg: - pass - - FakePsycopg.OperationalError = OperationalError # type: ignore[attr-defined] - - class FakeConnection: - def __init__(self, error: OperationalError): - self.error = error - - @contextlib.contextmanager - def transaction(self) -> Any: - yield - - def execute(self, sql: str) -> FakeConnection: - if sql.startswith("SET LOCAL"): - return self - raise self.error - - client = _NativePostgresClient.__new__(_NativePostgresClient) - client._psycopg = FakePsycopg - - deadline_errors = ( - ("57014", "canceling statement due to statement timeout"), - ("55P03", "Anweisung wegen Zeitüberschreitung abgebrochen"), - ) - for sqlstate, message in deadline_errors: - with self.subTest(sqlstate=sqlstate): - error = OperationalError(message) - error.sqlstate = sqlstate # type: ignore[attr-defined] - - @contextlib.contextmanager - def connection(*, timeout_seconds: float | None = None) -> Any: - yield FakeConnection(error) - - client.connection = connection # type: ignore[method-assign] - with self.assertRaises(LedgerOperationTimeout): - client.run_json("SELECT '{}'::json", timeout_seconds=0.5) - - def test_database_url_extraction_variants(self) -> None: - self.assertEqual( - database_url_from_psql_command(["psql", "postgres://u:p@h:5432/db"]), - "postgres://u:p@h:5432/db", - ) - self.assertEqual( - database_url_from_psql_command(["psql", "-d", "postgresql://h/db"]), - "postgresql://h/db", - ) - self.assertEqual( - database_url_from_psql_command(["psql", "--dbname=postgresql://h/db"]), - "postgresql://h/db", - ) - self.assertIsNone(database_url_from_psql_command(["psql", "-h", "host", "-U", "user"])) - self.assertIsNone(database_url_from_psql_command(["./fake-psql.sh"])) - - def test_mode_off_forces_subprocess_backend(self) -> None: - ledger = CannedQueryPsqlShareLedger([acquired_lease_result()], native_client_mode="0") - self.assertIsNone(ledger._native) - self.assertEqual(ledger.execution_backend, "psql-subprocess") - - def test_auto_mode_without_dsn_stays_on_subprocess(self) -> None: - class NoDsnLedger(CannedQueryPsqlShareLedger): - def __init__(self) -> None: - self.canned_results = [acquired_lease_result()] - self.queries = [] - PsqlShareLedger.__init__( - self, - psql_command="./fake-psql.sh --flag", - native_client_mode="auto", - lease_retry_sleep=lambda _seconds: None, - ) - - ledger = NoDsnLedger() - self.assertIsNone(ledger._native) - - def test_forced_native_without_dsn_raises(self) -> None: - # The DSN-free command means _make_native_client must raise before - # any lease query regardless of whether psycopg is installed. - with self.assertRaises(ValueError): - PsqlShareLedger(psql_command="./fake-psql.sh", native_client_mode="1") - - def test_forced_native_without_psycopg_raises(self) -> None: - with unittest.mock.patch.dict(sys.modules, {"psycopg": None}): - with self.assertRaises(ValueError): - PsqlShareLedger( - psql_command="psql postgres://example.invalid/qbit", - native_client_mode="1", - ) - - def test_auto_native_psycopg_import_fallback_warns(self) -> None: - stdout = io.StringIO() - with unittest.mock.patch.dict(sys.modules, {"psycopg": None}): - with contextlib.redirect_stdout(stdout): - ledger = CannedQueryPsqlShareLedger( - [acquired_lease_result()], - native_client_mode="auto", - ) - self.assertIsNone(ledger._native) - self.assertEqual(ledger.execution_backend, "psql-subprocess") - self.assertIn( - "psycopg import failed; falling back to the psql subprocess backend", - stdout.getvalue(), - ) - - def test_psql_run_sql_uses_single_transaction(self) -> None: - recorded: dict[str, object] = {} - - def fake_run(cmd, **kwargs): # type: ignore[no-untyped-def] - recorded["cmd"] = list(cmd) - return types.SimpleNamespace(returncode=0, stdout="{}\n", stderr="") - - ledger = CannedQueryPsqlShareLedger([acquired_lease_result()]) - with unittest.mock.patch.object( - share_ledger_module.subprocess, "run", side_effect=fake_run - ): - output = ledger._run_sql("SELECT '{}'::json;") - self.assertEqual(output, "{}\n") - cmd = recorded["cmd"] - self.assertIn("--single-transaction", cmd) - self.assertIn("--no-psqlrc", cmd) - self.assertIn("ON_ERROR_STOP=1", cmd) - - def test_parse_single_json_value_contract(self) -> None: - self.assertEqual(parse_single_json_value('{"a": 1}'), {"a": 1}) - self.assertEqual(parse_single_json_value({"a": 1}), {"a": 1}) - with self.assertRaises(RuntimeError): - parse_single_json_value(None) - - def test_run_json_routes_through_native_client(self) -> None: - class FakeNative: - def __init__(self) -> None: - self.statements: list[str] = [] - - def run_json(self, sql: str, *, retry_safe: bool = False) -> Any: - self.statements.append(sql) - self.retry_safe = retry_safe - return stats_payload(5, 1, 8) - - def close(self) -> None: - return None - - ledger = PsqlShareLedger.__new__(PsqlShareLedger) - ledger._lock = threading.Lock() - ledger._read_semaphore = threading.BoundedSemaphore(1) - ledger._stats_lock = threading.Lock() - ledger._stats_refresh_lock = threading.Lock() - ledger._stats_counts = None - ledger._stats_max_share_seq = 0 - ledger._stats_note_buffer = None - ledger._stats_refreshed_monotonic = None - ledger._accepted_stats_cache_seconds = 60.0 - native = FakeNative() - ledger._native = native - - stats = PsqlShareLedger.accepted_share_stats(ledger) - self.assertEqual( - stats, - {"accepted_share_count": 5, "distinct_miner_count": 1}, - ) - self.assertEqual(len(native.statements), 1) - self.assertIn("qbit_share_ledger", native.statements[0]) - self.assertTrue(native.retry_safe) - - def test_native_mutation_operational_error_is_not_retried(self) -> None: - class OperationalError(Exception): - pass - - class FakePsycopg: - pass - - FakePsycopg.OperationalError = OperationalError # type: ignore[attr-defined] - executions: list[str] = [] - outcomes: list[object] = [OperationalError("response lost"), {"ok": True}] - - class FakeConnection: - def execute(self, sql: str) -> FakeConnection: - executions.append(sql) - outcome = outcomes.pop(0) - if isinstance(outcome, Exception): - raise outcome - self.row = outcome - return self - - def fetchone(self) -> tuple[object]: - return (self.row,) - - client = _NativePostgresClient.__new__(_NativePostgresClient) - client._psycopg = FakePsycopg - - @contextlib.contextmanager - def connection() -> Any: - yield FakeConnection() - - client.connection = connection # type: ignore[method-assign] - - with self.assertRaisesRegex(RuntimeError, "postgres query failed"): - client.run_json("UPDATE counter SET value = value + 1 RETURNING value") - - self.assertEqual(executions, ["UPDATE counter SET value = value + 1 RETURNING value"]) - self.assertEqual(len(outcomes), 1) - - def test_native_retry_safe_read_retries_once(self) -> None: - class OperationalError(Exception): - pass - - class FakePsycopg: - pass - - FakePsycopg.OperationalError = OperationalError # type: ignore[attr-defined] - executions: list[str] = [] - outcomes: list[object] = [OperationalError("stale connection"), {"count": 11}] - - class FakeConnection: - def execute(self, sql: str) -> FakeConnection: - executions.append(sql) - outcome = outcomes.pop(0) - if isinstance(outcome, Exception): - raise outcome - self.row = outcome - return self - - def fetchone(self) -> tuple[object]: - return (self.row,) - - client = _NativePostgresClient.__new__(_NativePostgresClient) - client._psycopg = FakePsycopg - - @contextlib.contextmanager - def connection() -> Any: - yield FakeConnection() - - client.connection = connection # type: ignore[method-assign] - - result = client.run_json("SELECT count(*)", retry_safe=True) - - self.assertEqual(result, {"count": 11}) - self.assertEqual(executions, ["SELECT count(*)", "SELECT count(*)"]) - - def test_ctv_attempt_journal_uses_non_retrying_native_execution(self) -> None: - class FakeNative: - def __init__(self) -> None: - self.calls: list[tuple[str, bool]] = [] - - def run_json(self, sql: str, *, retry_safe: bool = False) -> Any: - self.calls.append((sql, retry_safe)) - raise RuntimeError("ambiguous connection loss") - - ledger = PsqlShareLedger.__new__(PsqlShareLedger) - ledger._lock = threading.Lock() - ledger._ctv_broadcast_attempt_detail_limit = 20 - ledger._ctv_broadcast_retry_backoff_seconds = 300 - ledger._writer_id = "writer-a" - ledger._writer_epoch = 1 - ledger._writer_session_token = "session-a" - ledger._lease_interval_sql = "make_interval(secs => 30)" - native = FakeNative() - ledger._native = native - - with self.assertRaisesRegex(RuntimeError, "ambiguous connection loss"): - ledger.record_ctv_fanout_broadcast_attempt( - fanout_txid="22" * 32, - attempt_status="submitted", - ) - - self.assertEqual(len(native.calls), 1) - sql, retry_safe = native.calls[0] - self.assertFalse(retry_safe) - self.assertIn("INSERT INTO qbit_ctv_fanout_broadcast_attempts", sql) - self.assertIn("broadcast_attempt_count = artifact.broadcast_attempt_count + 1", sql) - - -def make_session_guards(**overrides: Any) -> Any: - """Build a PostgresSessionGuards, resolving the class at call time. - - Resolved through the module object so a tree without the session guards - fails each guard test individually with a clear AttributeError instead - of breaking the whole module import. - """ - kwargs: dict[str, Any] = { - "idle_in_transaction_timeout_seconds": 15.0, - "tcp_keepalives_idle_seconds": 30, - "tcp_keepalives_interval_seconds": 10, - "tcp_keepalives_count": 3, - } - kwargs.update(overrides) - return share_ledger_module.PostgresSessionGuards(**kwargs) - - -class PostgresSessionGuardTests(unittest.TestCase): - def test_native_pool_connect_carries_session_guard_options(self) -> None: - connect_kwargs: list[dict[str, Any]] = [] - - class FakeConninfo: - @staticmethod - def conninfo_to_dict(conninfo: str) -> dict[str, Any]: - return {"dbname": "qbit"} - - class FakePsycopg: - conninfo = FakeConninfo - - @staticmethod - def connect(conninfo: str, **kwargs: Any) -> object: - connect_kwargs.append(kwargs) - return object() - - client = _NativePostgresClient.__new__(_NativePostgresClient) - client._psycopg = FakePsycopg - client._conninfo = "postgresql://example.invalid/qbit" - client._application_name = "qbit-prism-writer-test" - client._session_guards = make_session_guards( - idle_in_transaction_timeout_seconds=2.5, - tcp_keepalives_idle_seconds=31, - tcp_keepalives_interval_seconds=7, - tcp_keepalives_count=4, - ) - - client._connect() - - self.assertEqual(len(connect_kwargs), 1) - options = connect_kwargs[0]["options"] - self.assertIn("-c idle_in_transaction_session_timeout=2500ms", options) - self.assertIn("-c tcp_keepalives_idle=31", options) - self.assertIn("-c tcp_keepalives_interval=7", options) - self.assertIn("-c tcp_keepalives_count=4", options) - - def test_native_pool_connect_preserves_operator_conninfo_options(self) -> None: - connect_kwargs: list[dict[str, Any]] = [] - operator_options = "-c geqo=off -c idle_in_transaction_session_timeout=0" - - class FakeConninfo: - @staticmethod - def conninfo_to_dict(conninfo: str) -> dict[str, Any]: - return {"dbname": "qbit", "options": operator_options} - - class FakePsycopg: - conninfo = FakeConninfo - - @staticmethod - def connect(conninfo: str, **kwargs: Any) -> object: - connect_kwargs.append(kwargs) - return object() - - client = _NativePostgresClient.__new__(_NativePostgresClient) - client._psycopg = FakePsycopg - client._conninfo = ( - "postgresql://example.invalid/qbit?options=-c%20geqo%3Doff" - ) - client._application_name = None - guards = make_session_guards() - client._session_guards = guards - - client._connect() - - options = connect_kwargs[0]["options"] - # Both the operator's DSN-level options and the guards are present, - # and the guard fragment comes last so its -c duplicates win. - self.assertIn("-c geqo=off", options) - self.assertTrue(options.endswith(guards.options_fragment())) - self.assertLess( - options.index("geqo"), - options.index("tcp_keepalives_idle"), - ) - - def test_lease_guard_keeps_statement_timeout_and_gains_session_guards(self) -> None: - connect_calls: list[dict[str, Any]] = [] - - class FakeGuardConnection: - closed = False - - def close(self) -> None: - pass - - def fake_connect(conninfo: str, **kwargs: Any) -> FakeGuardConnection: - connect_calls.append(kwargs) - return FakeGuardConnection() - - fake_psycopg = types.ModuleType("psycopg") - fake_psycopg.connect = fake_connect # type: ignore[attr-defined] - fake_psycopg.conninfo = types.SimpleNamespace( # type: ignore[attr-defined] - conninfo_to_dict=lambda conninfo: {"dbname": "qbit"} - ) - - with unittest.mock.patch.dict(sys.modules, {"psycopg": fake_psycopg}): - _NativePostgresLeaseGuard( - "postgresql://example.invalid/qbit", - 1234, - session_guards=make_session_guards(), - ) - - self.assertEqual(len(connect_calls), 1) - options = connect_calls[0]["options"] - self.assertIn("-c statement_timeout=500", options) - self.assertIn("-c idle_in_transaction_session_timeout=15000ms", options) - self.assertIn("-c tcp_keepalives_idle=30", options) - self.assertIn("-c tcp_keepalives_interval=10", options) - self.assertIn("-c tcp_keepalives_count=3", options) - - def test_subprocess_backend_sets_session_guards_without_a_deadline(self) -> None: - ledger = PsqlShareLedger.__new__(PsqlShareLedger) - ledger._command = ["psql"] - ledger._native = None - ledger._operation_timeout_local = threading.local() - ledger._statement_timeout_local = threading.local() - ledger._session_guards = make_session_guards() - completed = unittest.mock.Mock(returncode=0, stdout="{}\n", stderr="") - - with unittest.mock.patch.dict( - "os.environ", - {"PGOPTIONS": "-c geqo=off"}, - ), unittest.mock.patch( - "lab.prism.share_ledger.subprocess.run", - return_value=completed, - ) as run: - self.assertIsNone(ledger._remaining_operation_timeout()) - self.assertEqual(ledger._run_sql("SELECT '{}'::json;"), "{}\n") - - kwargs = run.call_args.kwargs - pgoptions = kwargs["env"]["PGOPTIONS"] - self.assertIn("-c idle_in_transaction_session_timeout=15000ms", pgoptions) - self.assertIn("-c tcp_keepalives_idle=30", pgoptions) - self.assertIn("-c tcp_keepalives_interval=10", pgoptions) - self.assertIn("-c tcp_keepalives_count=3", pgoptions) - # Operator-supplied PGOPTIONS survive, ahead of the guard fragment. - self.assertTrue(pgoptions.startswith("-c geqo=off")) - # No armed deadline: no per-statement bounds, no subprocess timeout. - self.assertNotIn("statement_timeout", pgoptions) - self.assertNotIn("lock_timeout", pgoptions) - self.assertNotIn("timeout", kwargs) - - -class ReadOnlySessionEnforcementTests(unittest.TestCase): - """``read_only=True`` must be a promise PostgreSQL keeps, not a gate. - - The refusing writer gate only covers paths that take the gate. Any - statement issued outside it — the lease upsert, a one-shot psql release — - reaches SQL exactly as it would on a read-write ledger, and until #157 - pointed the public tier at a hot standby, nothing but the deployment - topology stopped it. These tests pin the connect-time GUC that makes the - server refuse instead, on every connection the instance can create. - """ - - READ_ONLY_GUC = "-c default_transaction_read_only=on" - - def test_read_only_ledger_builds_read_only_session_guards(self) -> None: - ledger = PsqlShareLedger( - psql_command="psql postgresql://example.invalid/qbit", - native_client_mode="0", - read_only=True, - ) - self.addCleanup(ledger.close) - - self.assertTrue(ledger._session_guards.read_only) - fragment = ledger._session_guards.options_fragment() - self.assertIn(self.READ_ONLY_GUC, fragment) - # The orphan-reaping guards #137 established are preserved, not - # displaced: one options carrier, both guarantees. - self.assertIn("-c idle_in_transaction_session_timeout=15000ms", fragment) - self.assertIn("-c tcp_keepalives_idle=30", fragment) - - def test_writable_ledger_keeps_a_writable_session(self) -> None: - # The coordinator's own ledger must be untouched by this: it commits - # every share, block and settlement the pool has. - ledger = FakeLeasePsqlShareLedger( - [acquired_lease()], - writer_id="writer-a", - writer_epoch=1, - ) - self.addCleanup(ledger.close) - - self.assertFalse(ledger._session_guards.read_only) - self.assertNotIn( - "default_transaction_read_only", - ledger._session_guards.options_fragment(), - ) - - def test_read_only_native_pool_connections_are_read_only(self) -> None: - connect_kwargs: list[dict[str, Any]] = [] - - class FakeConninfo: - @staticmethod - def conninfo_to_dict(conninfo: str) -> dict[str, Any]: - return {"dbname": "qbit"} - - class FakePsycopg: - conninfo = FakeConninfo - - @staticmethod - def connect(conninfo: str, **kwargs: Any) -> object: - connect_kwargs.append(kwargs) - return object() - - client = _NativePostgresClient.__new__(_NativePostgresClient) - client._psycopg = FakePsycopg - client._conninfo = "postgresql://example.invalid/qbit" - client._application_name = "qbit-prism-public-read" - client._session_guards = make_session_guards(read_only=True) - - client._connect() - - options = connect_kwargs[0]["options"] - self.assertIn(self.READ_ONLY_GUC, options) - self.assertIn("-c idle_in_transaction_session_timeout=15000ms", options) - - def test_read_only_native_options_preserve_operator_conninfo_options(self) -> None: - connect_kwargs: list[dict[str, Any]] = [] - operator_options = "-c geqo=off -c default_transaction_read_only=off" - - class FakeConninfo: - @staticmethod - def conninfo_to_dict(conninfo: str) -> dict[str, Any]: - return {"dbname": "qbit", "options": operator_options} - - class FakePsycopg: - conninfo = FakeConninfo - - @staticmethod - def connect(conninfo: str, **kwargs: Any) -> object: - connect_kwargs.append(kwargs) - return object() - - client = _NativePostgresClient.__new__(_NativePostgresClient) - client._psycopg = FakePsycopg - client._conninfo = "postgresql://example.invalid/qbit" - client._application_name = None - client._session_guards = make_session_guards(read_only=True) - - client._connect() - - options = connect_kwargs[0]["options"] - # The operator's own options survive, but libpq applies -c settings - # left to right with the last duplicate winning, so a DSN that tries - # to turn the session writable again cannot. - self.assertIn("-c geqo=off", options) - self.assertLess( - options.index("-c default_transaction_read_only=off"), - options.index(self.READ_ONLY_GUC), - ) - self.assertTrue(options.endswith(self.READ_ONLY_GUC)) - - def test_read_only_lease_guard_session_is_read_only(self) -> None: - connect_calls: list[dict[str, Any]] = [] - - class FakeGuardConnection: - closed = False - - def close(self) -> None: - pass - - fake_psycopg = types.ModuleType("psycopg") - fake_psycopg.connect = lambda conninfo, **kwargs: ( # type: ignore[attr-defined] - connect_calls.append(kwargs) or FakeGuardConnection() - ) - fake_psycopg.conninfo = types.SimpleNamespace( # type: ignore[attr-defined] - conninfo_to_dict=lambda conninfo: {"dbname": "qbit"} - ) - - with unittest.mock.patch.dict(sys.modules, {"psycopg": fake_psycopg}): - _NativePostgresLeaseGuard( - "postgresql://example.invalid/qbit", - 1234, - session_guards=make_session_guards(read_only=True), - ) - - options = connect_calls[0]["options"] - # A read-only ledger never builds a lease guard, but the guarantee is - # a property of the carrier rather than of which paths happen to use - # it, so the dedicated session carries it too. - self.assertIn(self.READ_ONLY_GUC, options) - self.assertIn("-c statement_timeout=500", options) - - def test_read_only_subprocess_backend_is_read_only(self) -> None: - ledger = PsqlShareLedger.__new__(PsqlShareLedger) - ledger._command = ["psql"] - ledger._native = None - ledger._operation_timeout_local = threading.local() - ledger._statement_timeout_local = threading.local() - ledger._session_guards = make_session_guards(read_only=True) - completed = unittest.mock.Mock(returncode=0, stdout="{}\n", stderr="") - - with unittest.mock.patch.dict( - "os.environ", - {"PGOPTIONS": "-c geqo=off"}, - ), unittest.mock.patch( - "lab.prism.share_ledger.subprocess.run", - return_value=completed, - ) as run: - ledger._run_sql("SELECT '{}'::json;") - - pgoptions = run.call_args.kwargs["env"]["PGOPTIONS"] - self.assertIn(self.READ_ONLY_GUC, pgoptions) - self.assertTrue(pgoptions.startswith("-c geqo=off")) - - def test_read_only_covers_a_write_method_that_never_takes_the_gate(self) -> None: - # release_writer_lease_fresh_connection issues an UPDATE through a - # one-shot psql connection, deliberately touching neither the writer - # gate nor the pool. It is the shape #163 is about: the gate cannot - # refuse it, so the session it opens must be the thing that does. - ledger = PsqlShareLedger( - psql_command="psql postgresql://example.invalid/qbit", - native_client_mode="0", - read_only=True, - ) - self.addCleanup(ledger.close) - completed = unittest.mock.Mock(returncode=0, stdout='{"released": 0}\n', stderr="") - - with unittest.mock.patch( - "lab.prism.share_ledger.subprocess.run", - return_value=completed, - ) as run: - ledger.release_writer_lease_fresh_connection() - - self.assertIn("UPDATE qbit_ledger_writer_lease", run.call_args.kwargs["input"]) - self.assertIn( - self.READ_ONLY_GUC, - run.call_args.kwargs["env"]["PGOPTIONS"], - ) - - def test_read_only_ledger_still_refuses_the_writer_gate(self) -> None: - # The server-side refusal is added to the in-process gate, not - # substituted for it. - ledger = PsqlShareLedger( - psql_command="psql postgresql://example.invalid/qbit", - native_client_mode="0", - read_only=True, - ) - self.addCleanup(ledger.close) - - with self.assertRaises(share_ledger_module.ReadOnlyLedgerError): - ledger.append(pending_share(0)) - - -class BoundedLeaseAcquisitionTests(unittest.TestCase): - class DeadlineRecordingLeaseLedger(FakeLeasePsqlShareLedger): - def __init__(self, lease_results: list[dict[str, object]], **kwargs: Any): - self.recorded_deadlines: list[float | None] = [] - super().__init__(lease_results, **kwargs) - - def _run_json(self, sql: str) -> Any: - self.recorded_deadlines.append(self._remaining_operation_timeout()) - return super()._run_json(sql) - - def test_startup_lease_upsert_runs_under_bounded_deadline(self) -> None: - ledger = self.DeadlineRecordingLeaseLedger( - [acquired_lease()], - writer_id="writer-a", - writer_epoch=1, - ) - - self.assertEqual(len(ledger.recorded_deadlines), 1) - # None here is precisely the issue #123 defect: the startup upsert - # queueing on the lease row lock with no deadline at all. - self.assertIsNotNone(ledger.recorded_deadlines[0]) - # DEFAULT_LEASE_ACQUIRE_LOCK_TIMEOUT_SECONDS bounds the statement. - self.assertGreater(ledger.recorded_deadlines[0], 0.0) - self.assertLessEqual(ledger.recorded_deadlines[0], 5.0) - - configured = self.DeadlineRecordingLeaseLedger( - [acquired_lease()], - writer_id="writer-a", - writer_epoch=1, - lease_acquire_lock_timeout_seconds=2.5, - ) - self.assertLessEqual(configured.recorded_deadlines[0], 2.5) - - def test_lock_blocked_acquisition_fails_visibly_after_bounded_retries(self) -> None: - attempts: list[str] = [] - sleeps: list[float] = [] - - class LockedLeaseLedger(PsqlShareLedger): - def _run_json(self, sql: str) -> Any: - attempts.append(sql) - raise LedgerOperationTimeout( - "canceling statement due to lock timeout" - ) - - stdout = io.StringIO() - with contextlib.redirect_stdout(stdout): - with self.assertRaises(RuntimeError) as caught: - LockedLeaseLedger( - psql_command="psql postgresql://example.invalid/qbit", - native_client_mode="0", - lease_retry_sleep=sleeps.append, - ) - - # Exactly DEFAULT_LEASE_ACQUIRE_ATTEMPTS bounded attempts, with a - # retry sleep between consecutive attempts, then a visible failure — - # never LedgerOperationTimeout leaking out, never an unbounded wait. - self.assertEqual(len(attempts), 5) - self.assertEqual(len(sleeps), 4) - self.assertIn("attempt 5/5", stdout.getvalue()) - # The diagnostic names the lock conflict as the likely cause without - # asserting it — an unreachable or overloaded server expires the same - # deadline — and quotes the underlying error on every line. - message = str(caught.exception) - self.assertIn("did not complete within its 5s deadline", message) - self.assertIn("commonly", message) - self.assertIn("lock-blocked", message) - self.assertIn("canceling statement due to lock timeout", message) - self.assertIn("lock-blocked", stdout.getvalue()) - self.assertIn("canceling statement due to lock timeout", stdout.getvalue()) - # The originating timeout is chained, not discarded: it is the only - # record of which of the several causes actually fired. - self.assertIsInstance(caught.exception.__cause__, LedgerOperationTimeout) - - def test_acquisition_recovers_once_the_orphaned_lock_is_reaped(self) -> None: - outcomes: list[Any] = [ - LedgerOperationTimeout("canceling statement due to lock timeout"), - LedgerOperationTimeout("canceling statement due to lock timeout"), - acquired_lease(), - ] - - class EventuallyReapedLeaseLedger(PsqlShareLedger): - def _run_json(self, sql: str) -> Any: - outcome = outcomes.pop(0) - if isinstance(outcome, Exception): - raise outcome - return outcome - - stdout = io.StringIO() - with contextlib.redirect_stdout(stdout): - ledger = EventuallyReapedLeaseLedger( - psql_command="psql postgresql://example.invalid/qbit", - native_client_mode="0", - lease_retry_sleep=lambda _seconds: None, - ) - - # The retry budget outlasting idle_in_transaction_session_timeout is - # what makes the ordinary orphaned-lock case self-heal: the server - # reaps the orphan mid-budget and a later attempt lands the lease. - self.assertEqual(outcomes, []) - self.assertIn("attempt 2/5", stdout.getvalue()) - ledger.close() - - @staticmethod - def _adoptable_predecessor_lease_results( - *, - old_session: str, - new_session: str, - updated_at: str, - ) -> list[dict[str, object]]: - """Two observations of a silent predecessor, then a won CAS. - - Mirrors test_postgres_startup_adopts_after_one_guard_acquisition_silence: - the row is already silent for a full interval, so the second - observation lands once the guard-acquisition edge has elapsed too and - _ensure_writer_lease proceeds to the adoption CAS. - """ - return [ - held_lease( - session=old_session, - updated_at=updated_at, - age_seconds=DEFAULT_WRITER_LEASE_ADOPTION_SILENCE_SECONDS, - ), - held_lease( - session=old_session, - updated_at=updated_at, - age_seconds=DEFAULT_WRITER_LEASE_ADOPTION_SILENCE_SECONDS * 2, - ), - acquired_lease(session=new_session) | {"adopted": True}, - ] - - def test_adoption_cas_runs_under_bounded_deadline(self) -> None: - """The adoption CAS row-locks the lease too, so it must be bounded. - - Reached through the real _ensure_writer_lease adoption path rather - than by calling the CAS helper directly: an unbounded - _try_adopt_writer_lease reintroduces the issue #123 startup hang on - exactly this branch, where the predecessor left the row locked. - """ - updated_at = "2026-06-26 19:49:22.233718+00" - old_session = f"{WRITER_LEASE_HEARTBEAT_SESSION_PREFIX}old-session" - new_session = f"{WRITER_LEASE_HEARTBEAT_SESSION_PREFIX}new-session" - clock = FakeMonotonicClock() - - with contextlib.redirect_stdout(io.StringIO()), unittest.mock.patch.object( - share_ledger_module.time, - "monotonic", - clock.monotonic, - ): - ledger = self.DeadlineRecordingLeaseLedger( - self._adoptable_predecessor_lease_results( - old_session=old_session, - new_session=new_session, - updated_at=updated_at, - ), - writer_id="writer-a", - writer_epoch=1, - writer_session_token=new_session, - lease_acquire_lock_timeout_seconds=2.5, - lease_retry_sleep=clock.sleep, - ) - - self.assertEqual(len(ledger.lease_queries), 3) - # The third query is the CAS, not another observation. - self.assertIn("observed_writer_session_token", ledger.lease_queries[2]) - adoption_deadline = ledger.recorded_deadlines[2] - # None here is the issue #123 defect on the adoption branch: the CAS - # queueing on the lease row lock with no deadline at all. - self.assertIsNotNone(adoption_deadline) - self.assertGreater(adoption_deadline, 0.0) - self.assertLessEqual(adoption_deadline, 2.5) - - def test_lock_blocked_adoption_fails_visibly_after_bounded_retries(self) -> None: - updated_at = "2026-06-26 19:49:22.233718+00" - old_session = f"{WRITER_LEASE_HEARTBEAT_SESSION_PREFIX}old-session" - new_session = f"{WRITER_LEASE_HEARTBEAT_SESSION_PREFIX}new-session" - clock = FakeMonotonicClock() - adoption_deadlines: list[float | None] = [] - - class BlockedAdoptionLeaseLedger(FakeLeasePsqlShareLedger): - def _run_json(self, sql: str) -> Any: - if "observed_writer_session_token" not in sql: - return super()._run_json(sql) - adoption_deadlines.append(self._remaining_operation_timeout()) - raise LedgerOperationTimeout( - "canceling statement due to lock timeout" - ) - - stdout = io.StringIO() - with contextlib.redirect_stdout(stdout), unittest.mock.patch.object( - share_ledger_module.time, - "monotonic", - clock.monotonic, - ): - with self.assertRaises(RuntimeError) as caught: - BlockedAdoptionLeaseLedger( - self._adoptable_predecessor_lease_results( - old_session=old_session, - new_session=new_session, - updated_at=updated_at, - ), - writer_id="writer-a", - writer_epoch=1, - writer_session_token=new_session, - lease_retry_sleep=clock.sleep, - ) - - # Every CAS attempt ran under the bound, and the budget stopped at - # DEFAULT_LEASE_ACQUIRE_ATTEMPTS instead of retrying forever. - self.assertEqual(len(adoption_deadlines), 5) - for deadline in adoption_deadlines: - self.assertIsNotNone(deadline) - self.assertLessEqual(deadline, 5.0) - message = str(caught.exception) - self.assertIn("writer lease adoption", message) - self.assertIn("did not complete within its 5s deadline", message) - self.assertIsInstance(caught.exception.__cause__, LedgerOperationTimeout) - self.assertIn("adoption: attempt 5/5", stdout.getvalue()) - - def test_non_timeout_lease_failure_propagates_without_retry(self) -> None: - """Only deadline expiries are retried; everything else propagates. - - Swallowing a broader exception class into the retry loop would spend - the whole budget on an error no retry can fix and then relabel it as - a possible lock conflict, hiding the real fault from the operator. - """ - - class FakeIdleInTransactionSessionTimeout(Exception): - """Shaped like psycopg.errors.IdleInTransactionSessionTimeout. - - sqlstate 25P03 is class 25 (invalid transaction state), which - psycopg raises as an InternalError, not an OperationalError, so - _NativePostgresClient.run_json's handler never wraps it and it - reaches the lease helper raw. The stand-in keeps this case - meaningful on interpreters without psycopg installed. - """ - - sqlstate = "25P03" - - errors: list[Exception] = [ - # What run_json raises for a wrapped, non-deadline OperationalError. - RuntimeError("postgres query failed: server closed the connection"), - FakeIdleInTransactionSessionTimeout( - "terminating connection due to idle-in-transaction timeout" - ), - ] - try: - import psycopg - except ImportError: - psycopg = None # type: ignore[assignment] - if psycopg is not None: - real_idle_timeout = psycopg.errors.IdleInTransactionSessionTimeout - self.assertFalse(issubclass(real_idle_timeout, psycopg.OperationalError)) - errors.append( - real_idle_timeout( - "terminating connection due to idle-in-transaction timeout" - ) - ) - - for error in errors: - with self.subTest(error=type(error).__name__): - attempts: list[str] = [] - sleeps: list[float] = [] - - class FailingLeaseLedger(PsqlShareLedger): - def _run_json(self, sql: str) -> Any: - attempts.append(sql) - raise error - - stdout = io.StringIO() - with contextlib.redirect_stdout(stdout): - with self.assertRaises(type(error)) as caught: - FailingLeaseLedger( - psql_command="psql postgresql://example.invalid/qbit", - native_client_mode="0", - lease_retry_sleep=sleeps.append, - ) - - # Unchanged, on the first attempt, with no retry and none of - # the deadline wording. - self.assertIs(caught.exception, error) - self.assertEqual(len(attempts), 1) - self.assertEqual(sleeps, []) - self.assertEqual(stdout.getvalue(), "") - - def test_init_rejects_disarmed_session_guard_and_lease_bounds(self) -> None: - cases = ( - {"lease_acquire_lock_timeout_seconds": 0}, - {"lease_acquire_lock_timeout_seconds": -1}, - {"lease_acquire_attempts": 0}, - {"postgres_idle_in_transaction_timeout_seconds": 0}, - {"postgres_tcp_keepalives_idle_seconds": 0}, - {"postgres_tcp_keepalives_interval_seconds": -1}, - {"postgres_tcp_keepalives_count": 0}, - ) - for overrides in cases: - with self.subTest(**overrides), self.assertRaises(ValueError): - FakeLeasePsqlShareLedger([acquired_lease()], **overrides) - - -class AcceptedStatsCacheTests(unittest.TestCase): - def test_stats_cached_within_ttl_and_incremented_by_appends(self) -> None: - ledger = CannedQueryPsqlShareLedger( - [ - acquired_lease_result(), - stats_payload(10, 2, 10), - record_payload(3, share_seq=11, new_miner=True), - {"records": [record_payload(6, share_seq=12)]}, - { - "records": [ - {**record_payload(6, share_seq=12), "newly_inserted": False} - ] - }, - ], - accepted_stats_cache_seconds=60.0, - ) - first = ledger.accepted_share_stats() - self.assertEqual( - first, - {"accepted_share_count": 10, "distinct_miner_count": 2}, - ) - queries_after_seed = len(ledger.queries) - - # Repeated reads inside the TTL never touch the database. - for _ in range(5): - self.assertEqual(ledger.accepted_share_stats(), first) - self.assertEqual(len(ledger.queries), queries_after_seed) - - # A committed append advances the counters without a query; - # pending_share(3) mines as miner-0, a brand new miner id. - ledger.append(pending_share(3)) - self.assertEqual( - ledger.accepted_share_stats(), - {"accepted_share_count": 11, "distinct_miner_count": 3}, - ) - # The writer-thread batch path advances the counters the same way; - # pending_share(6) is miner-0 again so only the share count moves. - ledger.append_batch([(pending_share(6), None)]) - self.assertEqual( - ledger.accepted_share_stats(), - {"accepted_share_count": 12, "distinct_miner_count": 3}, - ) - # Replaying the same committed batch returns its existing record but - # does not increment the cache again. - ledger.append_batch([(pending_share(6), None)]) - self.assertEqual( - ledger.accepted_share_stats(), - {"accepted_share_count": 12, "distinct_miner_count": 3}, - ) - - def test_refresh_does_not_hold_writer_lock_during_aggregate(self) -> None: - aggregate_started = threading.Event() - aggregate_can_return = threading.Event() - - class SlowAggregateLedger(CannedQueryPsqlShareLedger): - def __init__(self) -> None: - super().__init__( - [acquired_lease_result()], - accepted_stats_cache_seconds=60.0, - ) - - def _run_json(self, sql: str) -> Any: - self.queries.append(sql) - if "'max_share_seq'" in sql: - aggregate_started.set() - if not aggregate_can_return.wait(5): - raise AssertionError("aggregate query was not released") - return stats_payload(10, 2, 10) - if "INSERT INTO qbit_share_ledger" in sql: - return record_payload(3, share_seq=11, new_miner=True) - return super()._run_json(sql) - - ledger = SlowAggregateLedger() - stats: list[dict[str, int]] = [] - errors: list[BaseException] = [] - - def refresh_stats() -> None: - try: - stats.append(ledger.accepted_share_stats()) - except BaseException as exc: # noqa: BLE001 - surfaced below - errors.append(exc) - - stats_thread = threading.Thread(target=refresh_stats, name="stats-refresh") - stats_thread.start() - try: - self.assertTrue(aggregate_started.wait(5)) - # The full-history aggregate is still blocked, but the serialized - # writer must be able to commit and publish its note immediately. - appended = ledger.append(pending_share(3)) - self.assertEqual(appended.share_seq, 11) - finally: - aggregate_can_return.set() - stats_thread.join(5) - - self.assertFalse(stats_thread.is_alive()) - self.assertEqual(errors, []) - self.assertEqual( - stats, - [{"accepted_share_count": 11, "distinct_miner_count": 3}], - ) - - def test_refresh_does_not_replay_note_already_in_aggregate_snapshot(self) -> None: - append_committed = threading.Event() - append_can_publish_note = threading.Event() - aggregate_started = threading.Event() - aggregate_can_return = threading.Event() - - class CommitBeforeNoteLedger(CannedQueryPsqlShareLedger): - def __init__(self) -> None: - super().__init__( - [acquired_lease_result()], - accepted_stats_cache_seconds=60.0, - ) - - def _run_json(self, sql: str) -> Any: - self.queries.append(sql) - if "'max_share_seq'" in sql: - aggregate_started.set() - if not aggregate_can_return.wait(5): - raise AssertionError("aggregate query was not released") - return stats_payload(11, 2, 11) - if "INSERT INTO qbit_share_ledger" in sql: - return record_payload(1, share_seq=11, new_miner=True) - return super()._run_json(sql) - - def _record_from_json(self, payload: dict[str, Any]) -> Any: - record = PsqlShareLedger._record_from_json(payload) - if record.share_id == "share-1": - append_committed.set() - if not append_can_publish_note.wait(5): - raise AssertionError("append note was not released") - return record - - ledger = CommitBeforeNoteLedger() - appended: list[object] = [] - stats: list[dict[str, int]] = [] - errors: list[BaseException] = [] - - def append_share() -> None: - try: - appended.append(ledger.append(pending_share(1))) - except BaseException as exc: # noqa: BLE001 - surfaced below - errors.append(exc) - - def refresh_stats() -> None: - try: - stats.append(ledger.accepted_share_stats()) - except BaseException as exc: # noqa: BLE001 - surfaced below - errors.append(exc) - - append_thread = threading.Thread(target=append_share, name="share-append") - stats_thread = threading.Thread(target=refresh_stats, name="stats-refresh") - append_thread.start() - try: - self.assertTrue(append_committed.wait(5)) - stats_thread.start() - self.assertTrue(aggregate_started.wait(5)) - # Publish a snapshot that already contains the committed share, - # then release its delayed cache note. The published watermark - # must suppress that late note rather than count share 11 twice. - aggregate_can_return.set() - stats_thread.join(5) - self.assertFalse(stats_thread.is_alive()) - finally: - append_can_publish_note.set() - aggregate_can_return.set() - append_thread.join(5) - if stats_thread.ident is not None: - stats_thread.join(5) - - self.assertFalse(append_thread.is_alive()) - self.assertFalse(stats_thread.is_alive()) - self.assertEqual(errors, []) - self.assertEqual(len(appended), 1) - self.assertEqual( - stats, - [{"accepted_share_count": 11, "distinct_miner_count": 2}], - ) - self.assertEqual(ledger.accepted_share_stats(), stats[0]) - - def test_stats_requery_after_ttl_reconciles(self) -> None: - ledger = CannedQueryPsqlShareLedger( - [ - acquired_lease_result(), - stats_payload(10, 1, 10), - stats_payload(4, 2, 12), - ], - accepted_stats_cache_seconds=0.05, - ) - self.assertEqual( - ledger.accepted_share_stats(), - {"accepted_share_count": 10, "distinct_miner_count": 1}, - ) - time.sleep(0.06) - # The expired read stays non-blocking: it serves the maintained - # counters and arms the background reconcile that publishes the - # out-of-band correction. - self.assertEqual( - ledger.accepted_share_stats(), - {"accepted_share_count": 10, "distinct_miner_count": 1}, - ) - reconcile_thread = ledger._stats_background_refresh_thread - self.assertIsNotNone(reconcile_thread) - reconcile_thread.join(5) - self.assertEqual( - ledger.accepted_share_stats(), - {"accepted_share_count": 4, "distinct_miner_count": 2}, - ) - - def test_stats_query_disabled_cache_runs_every_time(self) -> None: - ledger = CannedQueryPsqlShareLedger( - [ - acquired_lease_result(), - stats_payload(1, 1, 1), - stats_payload(2, 1, 2), - ], - accepted_stats_cache_seconds=0.0, - ) - self.assertEqual(ledger.accepted_share_stats()["accepted_share_count"], 1) - self.assertEqual(ledger.accepted_share_stats()["accepted_share_count"], 2) - - def test_metrics_reports_shares_without_share_ledger_scan(self) -> None: - metrics_payload = { - "blocks": 2, - "confirmed_blocks": 1, - "inactive_blocks": 0, - "rejected_blocks": 0, - "reversed_blocks": 1, - "payout_entries": 5, - "owed_accounts": 3, - "ctv_fanouts_failed": 0, - } - ledger = CannedQueryPsqlShareLedger( - [ - acquired_lease_result(), - stats_payload(7, 1, 7), - metrics_payload, - ], - accepted_stats_cache_seconds=60.0, - ) - metrics = ledger.metrics() - self.assertEqual(metrics["shares"], 7) - self.assertEqual(metrics["blocks"], 2) - metrics_sql = ledger.queries[-1] - self.assertNotIn("qbit_share_ledger", metrics_sql) - stats_sql = ledger.queries[-2] - self.assertIn("count(DISTINCT miner_id)", stats_sql) - self.assertNotIn("json_agg(DISTINCT miner_id)", stats_sql) - - -class AcceptedStatsBackgroundReconcileTests(unittest.TestCase): - def _expire_stats(self, ledger: CannedQueryPsqlShareLedger) -> None: - with ledger._stats_lock: - ledger._stats_refreshed_monotonic = ( - time.monotonic() - ledger._accepted_stats_cache_seconds - 1.0 - ) - - def test_expired_read_serves_counters_and_reconciles_in_background(self) -> None: - aggregate_started = threading.Event() - aggregate_can_return = threading.Event() - - class SlowReconcileLedger(CannedQueryPsqlShareLedger): - block_aggregate = False - - def _run_json(self, sql: str) -> Any: - if self.block_aggregate and "'max_share_seq'" in sql: - self.queries.append(sql) - aggregate_started.set() - if not aggregate_can_return.wait(5): - raise AssertionError("reconcile aggregate was not released") - return stats_payload(40, 4, 12) - return super()._run_json(sql) - - ledger = SlowReconcileLedger( - [ - acquired_lease_result(), - stats_payload(10, 2, 10), - record_payload(3, share_seq=11, new_miner=True), - record_payload(4, share_seq=12), - record_payload(5, share_seq=13), - ], - accepted_stats_cache_seconds=60.0, - ) - seeded = ledger.accepted_share_stats() - self.assertEqual( - seeded, {"accepted_share_count": 10, "distinct_miner_count": 2} - ) - self.assertIsNone(ledger._stats_background_refresh_thread) - - self._expire_stats(ledger) - ledger.block_aggregate = True - - # The expired read returns a private copy of the maintained - # counters without waiting for the aggregate it armed. - stale = ledger.accepted_share_stats() - self.assertEqual(stale, seeded) - stale["accepted_share_count"] = -1 - self.assertEqual( - ledger.accepted_share_stats()["accepted_share_count"], 10 - ) - self.assertTrue(aggregate_started.wait(5)) - reconcile_thread = ledger._stats_background_refresh_thread - self.assertIsNotNone(reconcile_thread) - self.assertTrue(reconcile_thread.daemon) - try: - # While the aggregate is blocked, the serialized writer commits - # and readers observe its notes immediately -- and repeated - # expired reads reuse the single in-flight reconcile. The three - # appends straddle the reconcile snapshot's watermark of 12: - # sequences 11 and 12 are inside the snapshot, 13 is newer. - ledger.append(pending_share(3)) - ledger.append(pending_share(4)) - ledger.append(pending_share(5)) - self.assertEqual( - ledger.accepted_share_stats(), - {"accepted_share_count": 13, "distinct_miner_count": 3}, - ) - self.assertIs(ledger._stats_background_refresh_thread, reconcile_thread) - finally: - aggregate_can_return.set() - reconcile_thread.join(5) - self.assertFalse(reconcile_thread.is_alive()) - - # The reconcile publishes aggregate plus replayed notes: sequences - # 11 and 12 sit at or below the snapshot watermark (12 exactly at - # the boundary guards the <= comparison) and are suppressed, while - # sequence 13 is replayed on top of the aggregate. - self.assertEqual( - ledger.accepted_share_stats(), - {"accepted_share_count": 41, "distinct_miner_count": 4}, - ) - aggregate_queries = [ - sql for sql in ledger.queries if "'max_share_seq'" in sql - ] - self.assertEqual(len(aggregate_queries), 2) - - def test_failed_background_reconcile_keeps_serving_and_rearms(self) -> None: - class FlakyReconcileLedger(CannedQueryPsqlShareLedger): - fail_aggregate = False - - def _run_json(self, sql: str) -> Any: - if self.fail_aggregate and "'max_share_seq'" in sql: - self.queries.append(sql) - raise RuntimeError("reconcile aggregate failed") - return super()._run_json(sql) - - ledger = FlakyReconcileLedger( - [ - acquired_lease_result(), - stats_payload(10, 2, 10), - stats_payload(20, 3, 15), - ], - accepted_stats_cache_seconds=60.0, - ) - seeded = ledger.accepted_share_stats() - self.assertEqual( - seeded, {"accepted_share_count": 10, "distinct_miner_count": 2} - ) - status = ledger.accepted_stats_reconcile_status() - self.assertEqual(status["failures"], 0) - self.assertIsNotNone(status["age_seconds"]) - self.assertLess(status["age_seconds"], 5.0) - - self._expire_stats(ledger) - # Reconcile age keeps growing while no pass publishes, so alerting - # can see a stale (or wedged) reconcile that callers no longer feel. - self.assertGreater( - ledger.accepted_stats_reconcile_status()["age_seconds"], 60.0 - ) - ledger.fail_aggregate = True - with unittest.mock.patch( - "lab.prism.share_ledger.traceback.print_exc" - ) as print_exc: - self.assertEqual(ledger.accepted_share_stats(), seeded) - failed_thread = ledger._stats_background_refresh_thread - self.assertIsNotNone(failed_thread) - failed_thread.join(5) - self.assertFalse(failed_thread.is_alive()) - print_exc.assert_called_once() - self.assertEqual( - ledger.accepted_stats_reconcile_status()["failures"], 1 - ) - - # The failure left the counters stale, so the next read arms a new - # reconcile; a successful pass then publishes the fresh aggregate. - ledger.fail_aggregate = False - self.assertEqual(ledger.accepted_share_stats(), seeded) - retry_thread = ledger._stats_background_refresh_thread - self.assertIsNot(retry_thread, failed_thread) - retry_thread.join(5) - self.assertEqual( - ledger.accepted_share_stats(), - {"accepted_share_count": 20, "distinct_miner_count": 3}, - ) - final_status = ledger.accepted_stats_reconcile_status() - self.assertEqual(final_status["failures"], 1) - self.assertLess(final_status["age_seconds"], 5.0) - - def test_cold_seed_still_runs_exact_aggregate_synchronously(self) -> None: - ledger = CannedQueryPsqlShareLedger( - [acquired_lease_result(), stats_payload(5, 1, 4)], - accepted_stats_cache_seconds=60.0, - ) - self.assertIsNone( - ledger.accepted_stats_reconcile_status()["age_seconds"] - ) - self.assertEqual( - ledger.accepted_share_stats(), - {"accepted_share_count": 5, "distinct_miner_count": 1}, - ) - self.assertIsNone(ledger._stats_background_refresh_thread) - - -# A stand-in psql for the subprocess backend's row-result path (#236). It is -# a real child process -- spawned, fed the statement on stdin, killed on a -# deadline and reaped -- so the tests below exercise the shipped spool, -# timeout and exit translation rather than a mock of subprocess. The mode -# argument selects its behaviour; the ledger's own flags follow it and are -# echoed back by the ``env`` mode so the invocation can be asserted on. -FAKE_PSQL_SCRIPT = """ -import json -import os -import sys -import time - -mode = sys.argv[1] -sys.stdin.read() - - -def row(index): - return json.dumps({ - "share_seq": index, - "share_id": f"share-{index}", - "miner_id": f"miner-{index % 3}", - "order_key": f"{index % 3:04d}", - "p2mr_program_hex": "11" * 32, - "share_difficulty": "4096", - "network_difficulty": "1048576", - "template_height": 800000 + index, - "job_id": f"job-{index}", - "job_issued_at_ms": 1000 + index, - "accepted_at_ms": 1001 + index, - "ntime": 1700000000 + index, - "credit_policy": None, - }) - - -if mode == "rows": - count = int(os.environ.get("FAKE_PSQL_ROWS", "0")) - for index in range(1, count + 1): - sys.stdout.write(row(index) + "\\n") - sys.stdout.write("\\n") -elif mode == "empty": - pass -elif mode == "fail": - sys.stderr.write('ERROR: 42P01: relation "missing" does not exist\\n') - sys.exit(3) -elif mode == "partial-then-fail": - for index in range(1, 4): - sys.stdout.write(row(index) + "\\n") - sys.stdout.flush() - sys.stderr.write("ERROR: XX000: server closed the connection unexpectedly\\n") - sys.exit(2) -elif mode == "deadline": - sys.stderr.write("ERROR: 57014: canceling statement due to statement timeout\\n") - sys.exit(3) -elif mode == "hang": - time.sleep(30) -elif mode == "malformed": - sys.stdout.write(row(1) + "\\n") - sys.stdout.write("{not json\\n") -elif mode == "env": - sys.stdout.write(json.dumps({ - "pgoptions": os.environ.get("PGOPTIONS", ""), - "pgconnect_timeout": os.environ.get("PGCONNECT_TIMEOUT", ""), - "argv": sys.argv[2:], - }) + "\\n") -else: - sys.stderr.write(f"unknown mode {mode}\\n") - sys.exit(4) -""" - - -def row_payload(index: int) -> dict[str, Any]: - """One accepted-share payload in the shape the per-row projection returns.""" - return { - "share_seq": index, - "share_id": f"share-{index}", - "miner_id": f"miner-{index % 3}", - "order_key": f"{index % 3:04d}", - "p2mr_program_hex": "11" * 32, - "share_difficulty": "4096", - "network_difficulty": "1048576", - "template_height": 800_000 + index, - "job_id": f"job-{index}", - "job_issued_at_ms": 1_000 + index, - "accepted_at_ms": 1_001 + index, - "ntime": 1_700_000_000 + index, - "credit_policy": None, - } - - -class PayoutWindowRowDecodingTests(unittest.TestCase): - """Issue #236: the payout-window reads decode one row at a time. - - The snapshot and delta used to return one ``json_agg`` value the size of - the whole window, decoded in one C call that never released the GIL. - Both production backends now project one JSON object per row and - convert the rows in bounded batches, rechecking the caller's deadline - and stamping liveness between batches. These tests pin that contract on - the psql subprocess backend (through a real child process), on the - native client (through a fake driver), on the ledger plumbing that - joins them, and on the supported subclass seams. - """ - - @classmethod - def setUpClass(cls) -> None: - cls._script_dir = tempfile.TemporaryDirectory() - cls.script_path = str(Path(cls._script_dir.name) / "fake_psql.py") - Path(cls.script_path).write_text(FAKE_PSQL_SCRIPT, encoding="utf-8") - - @classmethod - def tearDownClass(cls) -> None: - cls._script_dir.cleanup() - - # -- fixtures ----------------------------------------------------------- - - def psql_ledger( - self, - mode: str, - *, - batch_size: int | None = None, - session_guards: Any = None, - ) -> PsqlShareLedger: - ledger = PsqlShareLedger.__new__(PsqlShareLedger) - ledger._native = None - ledger._command = [sys.executable, self.script_path, mode] - ledger._session_guards = session_guards - ledger._lock = threading.Lock() - ledger._read_semaphore = threading.BoundedSemaphore(1) - if batch_size is not None: - ledger._json_row_batch_size = batch_size - return ledger - - @staticmethod - def arm_deadline(ledger: PsqlShareLedger, deadline: float) -> None: - deadline_local = threading.local() - deadline_local.deadline = deadline - ledger._operation_timeout_local = deadline_local - - # -- psql subprocess backend -------------------------------------------- - - def test_psql_snapshot_decodes_one_json_object_per_line(self) -> None: - ledger = self.psql_ledger("rows", batch_size=512) - - with unittest.mock.patch.dict(os.environ, {"FAKE_PSQL_ROWS": "1200"}): - records = ledger.snapshot_at_job_issue(1_005, window_weight=64) - - self.assertEqual(len(records), 1200) - self.assertEqual([type(record) for record in records[:2]], [AcceptedShareRecord] * 2) - self.assertEqual([record.share_seq for record in records[:3]], [1, 2, 3]) - self.assertEqual(records[-1].share_id, "share-1200") - self.assertEqual(records[0].share_difficulty, 4096) - self.assertIsNone(records[0].credit_policy) - stats = ledger.ledger_read_gate_stats()["payout_window_snapshot"] - self.assertEqual(stats["calls_total"], 1) - self.assertEqual(stats["execute_timeouts_total"], 0) - self.assertEqual(ledger._read_semaphore._value, 1) - - def test_psql_empty_window_is_an_empty_list(self) -> None: - """psql prints nothing for zero rows; that is a window, not an error.""" - ledger = self.psql_ledger("empty") - - self.assertEqual(ledger.snapshot_at_job_issue(1_005, window_weight=64), []) - self.assertEqual(ledger.snapshot_between_job_issues(1_005, 1_006), []) - - def test_psql_failure_after_partial_output_publishes_nothing(self) -> None: - ledger = self.psql_ledger("partial-then-fail", batch_size=1) - stamped: list[str] = [] - - with ( - ledger.operation_progress(lambda: stamped.append("x"), slice_seconds=1.0), - self.assertRaisesRegex( - RuntimeError, - r"psql command failed \(exit 2\): ERROR: XX000", - ), - ): - ledger.snapshot_at_job_issue(1_005, window_weight=64) - - # The spool was never read: exit status is validated first, so not - # a single row of the partial output was decoded or stamped. - self.assertEqual(stamped, []) - self.assertEqual(ledger._read_semaphore._value, 1) - stats = ledger.ledger_read_gate_stats()["payout_window_snapshot"] - self.assertEqual(stats["calls_total"], 1) - self.assertEqual(stats["execute_timeouts_total"], 0) - - def test_psql_generic_failure_keeps_its_stderr(self) -> None: - ledger = self.psql_ledger("fail") - - with self.assertRaisesRegex(RuntimeError, r"\(exit 3\): ERROR: 42P01"): - ledger.snapshot_between_job_issues(1_005, 1_006) - - def test_psql_deadline_cancellation_is_a_timeout(self) -> None: - ledger = self.psql_ledger("deadline") - - with ledger.operation_timeout(5.0): - with self.assertRaisesRegex(LedgerOperationTimeout, "psql operation exceeded"): - ledger.snapshot_at_job_issue(1_005, window_weight=64) - - stats = ledger.ledger_read_gate_stats()["payout_window_snapshot"] - self.assertEqual(stats["execute_timeouts_total"], 1) - self.assertEqual(ledger._read_semaphore._value, 1) - - def test_psql_hang_is_killed_and_reaped_at_the_deadline(self) -> None: - ledger = self.psql_ledger("hang") - spawned: list[subprocess.Popen[bytes]] = [] - real_popen = share_ledger_module.subprocess.Popen - - def spy(*args: Any, **kwargs: Any) -> subprocess.Popen[bytes]: - process = real_popen(*args, **kwargs) - spawned.append(process) - return process - - started = time.monotonic() - with unittest.mock.patch.object(share_ledger_module.subprocess, "Popen", spy): - with ledger.operation_timeout(0.5): - with self.assertRaisesRegex(LedgerOperationTimeout, "psql operation exceeded"): - ledger.snapshot_at_job_issue(1_005, window_weight=64) - elapsed = time.monotonic() - started - - self.assertLess(elapsed, 10.0) - self.assertEqual(len(spawned), 1) - # Reaped, and by our signal rather than its own exit. - self.assertIsNotNone(spawned[0].returncode) - self.assertLess(spawned[0].returncode, 0) - - def test_psql_malformed_line_is_an_error_not_a_partial_snapshot(self) -> None: - ledger = self.psql_ledger("malformed") - - with self.assertRaisesRegex(RuntimeError, "malformed JSON on output line 2"): - ledger.snapshot_at_job_issue(1_005, window_weight=64) - - def test_psql_row_invocation_carries_guards_flags_and_deadline(self) -> None: - """The row path and the single-value path share one invocation.""" - guards = make_session_guards(idle_in_transaction_timeout_seconds=2.5) - ledger = self.psql_ledger("env", session_guards=guards) - - with ledger.operation_timeout(2.0): - rows = ledger._run_retry_safe_read_json_rows("SELECT 1;") - - self.assertEqual(len(rows), 1) - echoed = rows[0] - self.assertIn("-c idle_in_transaction_session_timeout=2500ms", echoed["pgoptions"]) - self.assertRegex(echoed["pgoptions"], r"-c statement_timeout=\d+ms -c lock_timeout=\d+ms") - self.assertEqual(echoed["pgconnect_timeout"], "2") - for flag in ( - "--no-psqlrc", - "--single-transaction", - "--tuples-only", - "--no-align", - "--quiet", - "ON_ERROR_STOP=1", - ): - self.assertIn(flag, echoed["argv"]) - - def test_psql_decode_deadline_expires_between_batches(self) -> None: - """A budget that runs out mid-decode fails at the next batch boundary.""" - clock = FakeMonotonicClock(start=0.0) - ledger = self.psql_ledger("rows", batch_size=500) - ledger._monotonic = clock.monotonic - self.arm_deadline(ledger, deadline=10.0) - stamps: list[float] = [] - - def on_progress() -> None: - stamps.append(clock.now) - clock.sleep(6.0) - - with unittest.mock.patch.dict(os.environ, {"FAKE_PSQL_ROWS": "2000"}): - with ledger.operation_progress(on_progress, slice_seconds=1.0): - with self.assertRaisesRegex(LedgerOperationTimeout, "deadline expired"): - ledger.snapshot_at_job_issue(1_005, window_weight=64) - - # Batch 1 decoded at t=0 and stamped; batch 2 decoded at t=6 (still - # inside the 10s budget) and stamped; the check before batch 3 sees - # t=12 and fails. Nothing was returned for the 1000 rows decoded. - self.assertEqual(stamps, [0.0, 6.0]) - self.assertEqual(ledger._read_semaphore._value, 1) - stats = ledger.ledger_read_gate_stats()["payout_window_snapshot"] - self.assertEqual(stats["execute_timeouts_total"], 1) - - def test_psql_progress_is_stamped_between_batches(self) -> None: - ledger = self.psql_ledger("rows", batch_size=500) - stamped: list[str] = [] - - with unittest.mock.patch.dict(os.environ, {"FAKE_PSQL_ROWS": "2000"}): - with ledger.operation_progress(lambda: stamped.append("x"), slice_seconds=1.0): - records = ledger.snapshot_at_job_issue(1_005, window_weight=64) - - self.assertEqual(len(records), 2000) - self.assertEqual(len(stamped), 4) - - def test_psql_deadline_expiring_in_the_final_partial_batch_publishes_nothing(self) -> None: - """A short trailing batch is still deadline-checked before the result is returned.""" - clock = FakeMonotonicClock(start=0.0) - ledger = self.psql_ledger("rows", batch_size=500) - ledger._monotonic = clock.monotonic - self.arm_deadline(ledger, deadline=10.0) - stamps: list[float] = [] - - def on_progress() -> None: - stamps.append(clock.now) - clock.sleep(12.0) - - with unittest.mock.patch.dict(os.environ, {"FAKE_PSQL_ROWS": "700"}): - with ledger.operation_progress(on_progress, slice_seconds=1.0): - with self.assertRaisesRegex(LedgerOperationTimeout, "deadline expired"): - ledger.snapshot_at_job_issue(1_005, window_weight=64) - - # One full batch stamped at t=0; the 200-row tail converted at t=12 - # and the final check refused to publish it. - self.assertEqual(stamps, [0.0]) - self.assertEqual(ledger._read_semaphore._value, 1) - stats = ledger.ledger_read_gate_stats()["payout_window_snapshot"] - self.assertEqual(stats["execute_timeouts_total"], 1) - - def test_psql_deadline_expiring_before_decoding_publishes_nothing(self) -> None: - """A result smaller than one batch is deadline-checked before any row is decoded. - - The clock advances 3s on every read, so the budget that was still - open when psql was dispatched (a 13s deadline read at t=9) has run - out by the time the child has exited and decoding would start. - Whichever of the two deadlines the pre-decode check trips first, - the read must fail as a timeout: the fixture's second line is not - JSON, so a missing pre-check would surface as a parse error instead. - """ - ledger = self.psql_ledger("malformed", batch_size=512) - now = 0.0 - - def stepping() -> float: - nonlocal now - value = now - now += 3.0 - return value - - ledger._monotonic = stepping - self.arm_deadline(ledger, deadline=13.0) - - with self.assertRaisesRegex(LedgerOperationTimeout, "deadline expired"): - ledger.snapshot_at_job_issue(1_005, window_weight=64) - - self.assertEqual(ledger._read_semaphore._value, 1) - - def test_legacy_run_sql_seam_checks_the_deadline_before_decoding(self) -> None: - clock = FakeMonotonicClock(start=0.0) - - class SlowTextLedger(PsqlShareLedger): - def __init__(self) -> None: - self._native = None - self._lock = threading.Lock() - self._read_semaphore = threading.BoundedSemaphore(1) - self._monotonic = clock.monotonic - - def _run_sql(self, sql: str) -> str: - clock.sleep(11.0) - # Not a share row: converting it would raise KeyError. - return json.dumps({"share_seq": 1}) - - ledger = SlowTextLedger() - self.arm_deadline(ledger, deadline=10.0) - - with self.assertRaisesRegex(LedgerOperationTimeout, "statement deadline expired"): - ledger.snapshot_between_job_issues(1_005, 1_006) - - def test_legacy_run_json_seam_checks_the_deadline_around_the_final_batch(self) -> None: - clock = FakeMonotonicClock(start=0.0) - stamps: list[float] = [] - - class LegacyJsonLedger(PsqlShareLedger): - def __init__(self, rows: list[Any], *, slow: bool) -> None: - self._native = None - self._lock = threading.Lock() - self._read_semaphore = threading.BoundedSemaphore(1) - self._json_row_batch_size = 2 - self._monotonic = clock.monotonic - self.rows = rows - self.slow = slow - - def _run_json(self, sql: str) -> Any: - if self.slow: - clock.sleep(11.0) - return self.rows - - def on_progress() -> None: - stamps.append(clock.now) - clock.sleep(12.0) - - # Final partial batch: two rows convert and stamp at t=0, the third - # converts after the stamp moved the clock to t=12, and the final - # check refuses to publish. - clock.now = 0.0 - ledger = LegacyJsonLedger([row_payload(1), row_payload(2), row_payload(3)], slow=False) - self.arm_deadline(ledger, deadline=10.0) - with ledger.operation_progress(on_progress, slice_seconds=1.0): - with self.assertRaisesRegex(LedgerOperationTimeout, "deadline expired"): - ledger.snapshot_at_job_issue(1_005, window_weight=64) - self.assertEqual(stamps, [0.0]) - - # Before decoding: the seam answered after the budget, with a row - # that would raise KeyError if it were converted. - clock.now = 0.0 - ledger = LegacyJsonLedger([{"share_seq": 1}], slow=True) - self.arm_deadline(ledger, deadline=10.0) - with self.assertRaisesRegex(LedgerOperationTimeout, "statement deadline expired"): - ledger.snapshot_at_job_issue(1_005, window_weight=64) - self.assertEqual(ledger._read_semaphore._value, 1) - - def test_native_rows_deadline_expiring_in_the_final_partial_batch(self) -> None: - clock = FakeMonotonicClock(start=0.0) - rows = [row_payload(index) for index in range(1, 6)] - client, events, cursors = self.native_client(results=[{"rows": rows}], clock=clock) - - with self.assertRaisesRegex(LedgerOperationTimeout, "expired while decoding rows"): - client.run_json_rows( - "SELECT rows", - retry_safe=True, - timeout_seconds=9.0, - batch_size=2, - on_batch=lambda: clock.sleep(4.0), - ) - - # Checks at t=0, t=4 and t=8 admit three fetches (2, 2 and the - # trailing 1); the check before the terminating fetch, at t=12, - # refuses to publish the converted rows. - self.assertEqual(events.count("fetch:2"), 2) - self.assertEqual(events.count("fetch:1"), 1) - self.assertNotIn("fetch:0", events) - self.assertTrue(cursors[0].closed) - - def test_native_rows_deadline_expiring_during_execute_decodes_nothing(self) -> None: - clock = FakeMonotonicClock(start=0.0) - events: list[str] = [] - - class OperationalError(Exception): - pass - - class FakePsycopg: - pass - - FakePsycopg.OperationalError = OperationalError # type: ignore[attr-defined] - cursor = self.FakeCursor([], events) - - class SlowConnection: - def execute(self, sql: str) -> Any: - events.append(f"execute:{sql[:24]}") - if not sql.startswith("SET LOCAL"): - clock.sleep(6.0) - return cursor - return None - - @contextlib.contextmanager - def transaction(self) -> Any: - yield - - client = _NativePostgresClient.__new__(_NativePostgresClient) - client._psycopg = FakePsycopg - client._monotonic = clock.monotonic - - @contextlib.contextmanager - def connection(*, timeout_seconds: float | None = None) -> Any: - yield SlowConnection() - - client.connection = connection # type: ignore[method-assign] - - with self.assertRaisesRegex(LedgerOperationTimeout, "expired while decoding rows"): - client.run_json_rows("SELECT rows", timeout_seconds=5.0, batch_size=2) - - # An empty result is not published either: no fetch happened. - self.assertNotIn("fetch:0", events) - self.assertTrue(cursor.closed) - - # -- supported subclass seams ----------------------------------------------- - - def test_legacy_run_json_override_answers_with_the_row_list(self) -> None: - """A ``_run_json`` fake keeps working: it returns what json_agg decoded to.""" - stamped: list[str] = [] - - class LegacyJsonLedger(PsqlShareLedger): - def __init__(self) -> None: - self._native = None - self._lock = threading.Lock() - self._read_semaphore = threading.BoundedSemaphore(1) - self._json_row_batch_size = 2 - self.queries: list[str] = [] - - def _run_json(self, sql: str) -> Any: - self.queries.append(sql) - return [row_payload(1), row_payload(2), row_payload(3)] - - ledger = LegacyJsonLedger() - with ledger.operation_progress(lambda: stamped.append("x"), slice_seconds=1.0): - records = ledger.snapshot_at_job_issue(1_005, window_weight=64) - - self.assertEqual([record.share_seq for record in records], [1, 2, 3]) - self.assertEqual(len(ledger.queries), 1) - self.assertNotIn("json_agg", ledger.queries[0]) - # One boundary between the two batches of two and one. - self.assertEqual(stamped, ["x"]) - - def test_legacy_run_json_override_returning_a_scalar_is_rejected(self) -> None: - class ScalarLedger(PsqlShareLedger): - def __init__(self) -> None: - self._native = None - self._lock = threading.Lock() - self._read_semaphore = threading.BoundedSemaphore(1) - - def _run_json(self, sql: str) -> Any: - return {"count": 3} - - with self.assertRaisesRegex(RuntimeError, "non-list value: dict"): - ScalarLedger().snapshot_at_job_issue(1_005, window_weight=64) - - def test_legacy_run_sql_override_is_parsed_line_by_line(self) -> None: - """The A1 gate's text seam: one JSON value per psql output line.""" - calls: list[str] = [] - - class LegacyRunSqlLedger(PsqlShareLedger): - def __init__(self) -> None: - self._native = None - self._lock = threading.Lock() - self._read_semaphore = threading.BoundedSemaphore(1) - - def _run_sql(self, sql: str) -> str: - calls.append(sql) - return "\n".join( - [json.dumps(row_payload(1)), "", json.dumps(row_payload(2)), ""] - ) - - records = LegacyRunSqlLedger().snapshot_between_job_issues(1_005, 1_006) - - self.assertEqual([record.share_seq for record in records], [1, 2]) - self.assertEqual(len(calls), 1) - self.assertIn("UNION ALL", calls[0]) - - # -- SQL shape ----------------------------------------------------------------- - - def test_payout_window_reads_project_one_json_object_per_row(self) -> None: - """Neither production statement builds a whole-window aggregate.""" - ledger = QueryCapturePsqlShareLedger() - - ledger.snapshot_at_job_issue(1_000, window_weight=8) - ledger.snapshot_at_job_issue(1_000) - ledger.snapshot_between_job_issues(1_000, 2_000) - - self.assertEqual(len(ledger.queries), 3) - for sql in ledger.queries: - self.assertNotIn("json_agg", sql) - self.assertIn("json_build_object(", sql) - self.assertTrue(sql.rstrip().endswith("FROM rows\nORDER BY share_seq ASC;"), sql[-120:]) - bounded, unbounded, delta = ledger.queries - self.assertIn("WITH RECURSIVE pages AS", bounded) - self.assertIn("cumulative_difficulty - share_difficulty < 8::numeric", bounded) - self.assertNotIn("RECURSIVE", unbounded) - self.assertIn("UNION ALL", delta) - - # -- native client ----------------------------------------------------------- - - class FakeCursor: - def __init__(self, rows: list[Any], events: list[str], fail_at_batch: int | None = None, - error: Exception | None = None) -> None: - self._rows = rows - self._pos = 0 - self.events = events - self.fail_at_batch = fail_at_batch - self.error = error - self.batches = 0 - self.closed = False - - def fetchmany(self, size: int) -> list[tuple[Any]]: - self.batches += 1 - if self.fail_at_batch is not None and self.batches == self.fail_at_batch: - assert self.error is not None - raise self.error - batch = self._rows[self._pos : self._pos + size] - self._pos += size - self.events.append(f"fetch:{len(batch)}") - return [(row,) for row in batch] - - def close(self) -> None: - self.closed = True - self.events.append("close") - - def native_client( - self, - *, - results: list[Any], - clock: FakeMonotonicClock | None = None, - ) -> tuple[_NativePostgresClient, list[str], list[Any]]: - """A client over a fake driver; ``results`` are per-execute outcomes. - - The client's ``outcomes`` attribute is the live per-execute queue, so - a test can seed it with instances of the client's own - ``OperationalError`` after construction. - """ - events: list[str] = [] - cursors: list[Any] = [] - outcomes = list(results) - - class OperationalError(Exception): - def __init__(self, message: str, sqlstate: str | None = None) -> None: - super().__init__(message) - self.sqlstate = sqlstate - - class FakePsycopg: - pass - - FakePsycopg.OperationalError = OperationalError # type: ignore[attr-defined] - - tests = self - - class FakeConnection: - def execute(self, sql: str) -> Any: - events.append(f"execute:{sql[:24]}") - if sql.startswith("SET LOCAL"): - return None - outcome = outcomes.pop(0) - if isinstance(outcome, Exception): - raise outcome - cursor = tests.FakeCursor( - outcome["rows"], - events, - fail_at_batch=outcome.get("fail_at_batch"), - error=outcome.get("error"), - ) - cursors.append(cursor) - return cursor - - @contextlib.contextmanager - def transaction(self) -> Any: - events.append("begin") - try: - yield - finally: - events.append("commit") - - client = _NativePostgresClient.__new__(_NativePostgresClient) - client._psycopg = FakePsycopg - if clock is not None: - client._monotonic = clock.monotonic - - @contextlib.contextmanager - def connection(*, timeout_seconds: float | None = None) -> Any: - events.append(f"borrow:{timeout_seconds is not None}") - try: - yield FakeConnection() - except BaseException: - events.append("discard") - raise - else: - events.append("return") - - client.connection = connection # type: ignore[method-assign] - client.OperationalError = OperationalError # type: ignore[attr-defined] - client.outcomes = outcomes # type: ignore[attr-defined] - return client, events, cursors - - def test_native_rows_are_fetched_in_batches_after_the_deadline_commit(self) -> None: - rows = [row_payload(index) for index in range(1, 1201)] - client, events, cursors = self.native_client(results=[{"rows": rows}]) - batches: list[int] = [] - - records = client.run_json_rows( - "SELECT json_build_object(...)", - retry_safe=True, - timeout_seconds=5.0, - row_converter=PsqlShareLedger._record_from_json, - batch_size=512, - on_batch=lambda: batches.append(len(events)), - ) - - self.assertEqual([record.share_seq for record in records[:2]], [1, 2]) - self.assertEqual(len(records), 1200) - self.assertEqual(type(records[0]), AcceptedShareRecord) - # One SELECT inside the SET LOCAL transaction, committed before the - # first fetch; then three bounded fetches, the empty terminator, and - # the cursor closed before the connection goes back to the pool. - self.assertEqual( - events, - [ - "borrow:True", - "begin", - "execute:SET LOCAL statement_time", - "execute:SET LOCAL lock_timeout =", - "execute:SELECT json_build_object", - "commit", - "fetch:512", - "fetch:512", - "fetch:176", - "fetch:0", - "close", - "return", - ], - ) - self.assertEqual(len(batches), 3) - self.assertTrue(cursors[0].closed) - - def test_native_rows_without_a_deadline_run_in_autocommit(self) -> None: - client, events, _cursors = self.native_client( - results=[{"rows": [row_payload(1)]}] - ) - - rows = client.run_json_rows("SELECT 1", batch_size=4) - - self.assertEqual(rows, [row_payload(1)]) - self.assertEqual( - events, - ["borrow:False", "execute:SELECT 1", "fetch:1", "fetch:0", "close", "return"], - ) - - def test_native_retry_safe_rows_restart_the_whole_statement(self) -> None: - """A connection loss mid-fetch discards the partial rows and re-runs.""" - rows = [row_payload(index) for index in range(1, 7)] - client, events, cursors = self.native_client( - results=[ - { - "rows": rows, - "fail_at_batch": 2, - "error": None, - }, - {"rows": rows}, - ] - ) - # The first cursor's second fetch raises the driver's OperationalError. - first_error = client.OperationalError("SSL connection has been closed unexpectedly") - cursors_seen: list[Any] = [] - original_fetchmany = self.FakeCursor.fetchmany - - def fetchmany(cursor: Any, size: int) -> Any: - if cursor not in cursors_seen: - cursors_seen.append(cursor) - if len(cursors_seen) == 1 and cursor.batches == 1: - cursor.batches += 1 - raise first_error - return original_fetchmany(cursor, size) - - with unittest.mock.patch.object(self.FakeCursor, "fetchmany", fetchmany): - records = client.run_json_rows( - "SELECT rows", - retry_safe=True, - row_converter=PsqlShareLedger._record_from_json, - batch_size=2, - ) - - self.assertEqual([record.share_seq for record in records], [1, 2, 3, 4, 5, 6]) - self.assertEqual(events.count("execute:SELECT rows"), 2) - # First attempt: borrowed, one batch fetched, the failure discarded - # the connection and closed the cursor; second attempt complete. - self.assertEqual(events.index("discard"), events.index("close") + 1) - self.assertTrue(all(cursor.closed for cursor in cursors)) - - def test_native_rows_without_retry_fail_after_the_first_attempt(self) -> None: - client, events, _cursors = self.native_client(results=[]) - client.outcomes[:] = [client.OperationalError("connection reset"), {"rows": []}] - - with self.assertRaisesRegex(RuntimeError, "postgres query failed: connection reset"): - client.run_json_rows("SELECT rows", retry_safe=False) - - self.assertEqual(events.count("execute:SELECT rows"), 1) - - def test_native_rows_deadline_expires_between_batches(self) -> None: - clock = FakeMonotonicClock(start=0.0) - rows = [row_payload(index) for index in range(1, 7)] - client, events, cursors = self.native_client(results=[{"rows": rows}], clock=clock) - - def on_batch() -> None: - clock.sleep(4.0) - - with self.assertRaisesRegex(LedgerOperationTimeout, "expired while decoding rows"): - client.run_json_rows( - "SELECT rows", - retry_safe=True, - timeout_seconds=5.0, - batch_size=2, - on_batch=on_batch, - ) - - # Two batches fit inside the budget (checks at t=0 and t=4); the - # check before the third, at t=8, fails. No retry: the deadline is - # not an OperationalError, and the cursor is closed on the way out. - self.assertEqual(events.count("fetch:2"), 2) - self.assertEqual(events.count("execute:SELECT rows"), 1) - self.assertTrue(cursors[0].closed) - self.assertIn("discard", events) - - def test_native_rows_deadline_cancellation_is_not_retried(self) -> None: - client, events, _cursors = self.native_client(results=[]) - client.outcomes[:] = [ - client.OperationalError("canceling statement due to statement timeout", "57014"), - {"rows": []}, - ] - - with self.assertRaisesRegex(LedgerOperationTimeout, "postgres operation exceeded 2s"): - client.run_json_rows("SELECT rows", retry_safe=True, timeout_seconds=2.0) - - self.assertEqual(events.count("execute:SELECT rows"), 1) - - def test_native_null_row_is_rejected(self) -> None: - client, _events, _cursors = self.native_client(results=[{"rows": [None]}]) - - with self.assertRaisesRegex(RuntimeError, "returned no JSON"): - client.run_json_rows("SELECT NULL") - - def test_native_text_rows_are_decoded(self) -> None: - client, _events, _cursors = self.native_client( - results=[{"rows": [json.dumps(row_payload(7))]}] - ) - - self.assertEqual(client.run_json_rows("SELECT ...::text"), [row_payload(7)]) - - # -- ledger plumbing over the native port ------------------------------- - - def test_ledger_hands_the_native_port_the_full_rows_contract(self) -> None: - calls: list[dict[str, Any]] = [] - - class FakeNative: - def run_json_rows(self, sql: str, **kwargs: Any) -> list[Any]: - calls.append({"sql": sql, **kwargs}) - if kwargs.get("on_statement_start") is not None: - kwargs["on_statement_start"]() - converter = kwargs["row_converter"] - return [converter(row_payload(1)), converter(row_payload(2))] - - def close(self) -> None: - return None - - ledger = PsqlShareLedger.__new__(PsqlShareLedger) - ledger._native = FakeNative() - ledger._lock = threading.Lock() - ledger._read_semaphore = threading.BoundedSemaphore(2) - - with ledger.operation_timeout(3.0): - records = ledger.snapshot_at_job_issue(1_005, window_weight=64) - - self.assertEqual([record.share_seq for record in records], [1, 2]) - self.assertEqual(len(calls), 1) - call = calls[0] - self.assertTrue(call["retry_safe"]) - self.assertEqual(call["batch_size"], PAYOUT_WINDOW_ROW_BATCH_SIZE) - self.assertEqual(call["on_batch"], ledger._note_json_row_batch) - self.assertEqual(call["row_converter"], ledger._record_from_json) - self.assertGreater(call["timeout_seconds"], 0.0) - self.assertLessEqual(call["timeout_seconds"], 3.0) - self.assertNotIn("json_agg", call["sql"]) - stats = ledger.ledger_read_gate_stats()["payout_window_snapshot"] - self.assertEqual(stats["calls_total"], 1) - self.assertGreaterEqual(stats["execute_seconds_total"], 0.0) - self.assertEqual(ledger._read_semaphore._value, 2) - - def test_ledger_batch_hook_enforces_the_operation_deadline(self) -> None: - """The ledger's own absolute deadline is rechecked between batches.""" - clock = FakeMonotonicClock(start=0.0) - stamped: list[float] = [] - - class FakeNative: - def run_json_rows(self, sql: str, **kwargs: Any) -> list[Any]: - kwargs["on_statement_start"]() - converter = kwargs["row_converter"] - results = [] - for index in range(1, 4): - results.append(converter(row_payload(index))) - clock.sleep(4.0) - kwargs["on_batch"]() - return results - - def close(self) -> None: - return None - - ledger = PsqlShareLedger.__new__(PsqlShareLedger) - ledger._native = FakeNative() - ledger._monotonic = clock.monotonic - ledger._lock = threading.Lock() - ledger._read_semaphore = threading.BoundedSemaphore(1) - self.arm_deadline(ledger, deadline=10.0) - - with ledger.operation_progress(lambda: stamped.append(clock.now), slice_seconds=1.0): - with self.assertRaisesRegex(LedgerOperationTimeout, "operation deadline expired"): - ledger.snapshot_between_job_issues(1_005, 1_006) - - # Stamped at t=4 and t=8 (both inside the budget); the hook at t=12 - # raised before stamping, so the decode never completed late. - self.assertEqual(stamped, [4.0, 8.0]) - self.assertEqual(ledger._read_semaphore._value, 1) - stats = ledger.ledger_read_gate_stats()["payout_window_delta"] - self.assertEqual(stats["execute_timeouts_total"], 1) - - def test_native_client_satisfies_the_row_result_port(self) -> None: - client = _NativePostgresClient.__new__(_NativePostgresClient) - self.assertIsInstance(client, share_ledger_module.LedgerSqlPort) - self.assertTrue(callable(getattr(client, "run_json_rows", None))) - - -class BlockCandidateIdentityDigestTests(unittest.TestCase): - """The candidate identity digest streams its share window (#236).""" - - def _candidate(self, share_count: int, accepted_at_ms: int) -> dict[str, Any]: - shares = [ - { - "share_seq": seq, - "share_id": f"miner-\u00e9.rig{seq}:\U0001F600,}}{{", - "miner_id": f"miner-{seq % 7}", - "order_key": f"{seq % 7:02d}:miner-{seq % 7}", - "p2mr_program_hex": "ab" * 32, - "share_difficulty": (1 << 130) + seq, - "network_difficulty": 5, - "template_height": 9, - "job_id": f"job-{seq}", - "job_issued_at_ms": 1_000 + seq, - "accepted_at_ms": 1_100 + seq, - "ntime": 1_700_000_000 + seq, - } - for seq in range(1, share_count + 1) - ] - return { - "schema": "qbit.prism.block-candidate-intent.v1", - "block_hash_hex": "ab" * 32, - "pending_share": {"share_id": "s", "accepted_at_ms": accepted_at_ms}, - "shares_json": shares, - "prior_balances": [["m", "o", "p", 5]], - "found_block": {"block_height": 9, "z": None}, - } - - def test_streamed_identity_digest_matches_whole_document_digest(self) -> None: - for count in (0, 1, 1_300): - with self.subTest(count=count): - candidate = self._candidate(count, 5) - self.assertEqual( - share_ledger_module.block_candidate_identity_sha256(candidate), - share_ledger_module.sha256_json_hex( - share_ledger_module.block_candidate_identity(candidate) - ), - ) - # The acknowledgment stamp stays neutralized: two resubmissions of - # the same solved block hash identically. - self.assertEqual( - share_ledger_module.block_candidate_identity_sha256(self._candidate(9, 5)), - share_ledger_module.block_candidate_identity_sha256(self._candidate(9, 6)), - ) - self.assertNotEqual( - share_ledger_module.block_candidate_identity_sha256(self._candidate(9, 5)), - share_ledger_module.block_candidate_identity_sha256(self._candidate(8, 5)), - ) - - -if __name__ == "__main__": - unittest.main() diff --git a/tests/test_prism_share_submission.py b/tests/test_prism_share_submission.py deleted file mode 100644 index 755143714..000000000 --- a/tests/test_prism_share_submission.py +++ /dev/null @@ -1,374 +0,0 @@ -#!/usr/bin/env python3 -"""Direct tests for pure PRISM share-submission decisions.""" - -from __future__ import annotations - -from dataclasses import FrozenInstanceError -from types import SimpleNamespace -import threading -import unittest -from unittest.mock import patch - -from lab.prism.job_delivery import PRISM_CREDIT_POLICY_STALE_GRACE -from lab.prism.prism_coordinator import PrismCoordinator -from lab.prism import prism_coordinator -from lab.prism.share_submission import ( - PRISM_REJECTION_INVALID_EXTRANONCE, - PRISM_REJECTION_LOW_DIFFICULTY, - PRISM_REJECTION_MALFORMED_SUBMIT, - PRISM_REJECTION_STALE_JOB, - PRISM_REJECTION_UNAUTHORIZED_WORKER, - PRISM_REJECTION_UNKNOWN_JOB, - PRISM_SHARE_ACK_RESULTS, - RecentShareIndex, - SubmitContextInput, - SubmitRejected, - classify_submit_context, - classify_submit_work, - parse_submit_request, - validate_submit_request, -) - - -def context(*, parent: str = "tip", username: str = "miner-a") -> object: - return SimpleNamespace( - template={"previousblockhash": parent}, - worker=SimpleNamespace(username=username), - ) - - -class SubmitRequestTests(unittest.TestCase): - def test_parse_returns_immutable_wire_values(self) -> None: - request = parse_submit_request( - ["miner-a", "job-1", "00" * 8, "01020304", "05060708", "1fffe000"] - ) - - self.assertEqual(request.job_id, "job-1") - self.assertEqual(request.version_bits_hex, "1fffe000") - with self.assertRaises(FrozenInstanceError): - request.job_id = "job-2" # type: ignore[misc] - - def test_parse_rejects_incomplete_params(self) -> None: - with self.assertRaises(SubmitRejected) as raised: - parse_submit_request(["miner-a", "job-1"]) - - self.assertEqual(raised.exception.reason, PRISM_REJECTION_MALFORMED_SUBMIT) - - def test_validation_preserves_protocol_order(self) -> None: - request = parse_submit_request( - ["miner-b", "job-1", "00", "01020304", "05060708"] - ) - with self.assertRaises(SubmitRejected) as unauthorized: - validate_submit_request( - request, - authorized_username="miner-a", - pool_open=True, - extranonce2_size=8, - ) - self.assertEqual( - unauthorized.exception.reason, - PRISM_REJECTION_UNAUTHORIZED_WORKER, - ) - - request = parse_submit_request( - ["miner-a", "job-1", "00", "01020304", "05060708"] - ) - with self.assertRaises(SubmitRejected) as invalid_width: - validate_submit_request( - request, - authorized_username="miner-a", - pool_open=True, - extranonce2_size=8, - ) - self.assertEqual( - invalid_width.exception.reason, - PRISM_REJECTION_INVALID_EXTRANONCE, - ) - - -class RecentShareIndexTests(unittest.TestCase): - def test_coordinator_initialization_is_single_flight(self) -> None: - server = PrismCoordinator.__new__(PrismCoordinator) - server.extranonce2_size = 8 - seed = {("miner-a", "header-a")} - server.recent_share_keys = seed - constructor_started = threading.Event() - release_constructor = threading.Event() - constructor_calls = 0 - results: list[object] = [] - errors: list[BaseException] = [] - real_constructor = prism_coordinator.ShareSubmissionService - - def delayed_constructor(*args: object, **kwargs: object) -> object: - nonlocal constructor_calls - constructor_calls += 1 - constructor_started.set() - release_constructor.wait(2) - return real_constructor(*args, **kwargs) # type: ignore[arg-type] - - def initialize() -> None: - try: - results.append(server._ensure_share_submission_service()) - except BaseException as exc: # noqa: BLE001 - surfaced below - errors.append(exc) - - with patch.object( - prism_coordinator, - "ShareSubmissionService", - side_effect=delayed_constructor, - ): - first = threading.Thread(target=initialize) - second = threading.Thread(target=initialize) - first.start() - self.assertTrue(constructor_started.wait(1)) - second.start() - release_constructor.set() - first.join(2) - second.join(2) - - self.assertFalse(first.is_alive()) - self.assertFalse(second.is_alive()) - self.assertEqual(errors, []) - self.assertEqual(constructor_calls, 1) - self.assertEqual(len(results), 2) - self.assertIs(results[0], results[1]) - self.assertEqual(results[0].recent_shares.snapshot(), tuple(seed)) - - def test_capacity_evicts_only_the_oldest_insertion(self) -> None: - index = RecentShareIndex(capacity=2) - first = ("miner", "header-a") - second = ("miner", "header-b") - third = ("miner", "header-c") - - self.assertTrue(index.reserve(first)) - self.assertTrue(index.reserve(second)) - self.assertFalse(index.reserve(first)) - self.assertTrue(index.reserve(third)) - - self.assertEqual(index.snapshot(), (second, third)) - self.assertFalse(index.reserve(second)) - self.assertTrue(index.reserve(first)) - self.assertEqual(index.snapshot(), (third, first)) - - def test_release_allows_exact_retry_without_disturbing_order(self) -> None: - index = RecentShareIndex(capacity=2) - first = ("miner", "header-a") - second = ("miner", "header-b") - index.reserve(first) - index.reserve(second) - - index.release(first) - - self.assertTrue(index.reserve(first)) - self.assertEqual(index.snapshot(), (second, first)) - - -class SubmitClassificationTests(unittest.TestCase): - def test_active_and_retained_current_tip_are_normal_credit(self) -> None: - active = context() - active_decision = classify_submit_context( - SubmitContextInput(active, None, "tip", False) # type: ignore[arg-type] - ) - self.assertEqual(active_decision.source, "active") - self.assertIsNone(active_decision.credit_policy) - - retained = SimpleNamespace(context=context()) - retained_decision = classify_submit_context( - SubmitContextInput(None, retained, "tip", False) # type: ignore[arg-type] - ) - self.assertEqual(retained_decision.source, "retained") - self.assertIsNone(retained_decision.credit_policy) - - def test_missing_and_stale_contexts_have_distinct_rejections(self) -> None: - with self.assertRaises(SubmitRejected) as missing: - classify_submit_context(SubmitContextInput(None, None, "tip", False)) - self.assertEqual(missing.exception.reason, PRISM_REJECTION_UNKNOWN_JOB) - - with self.assertRaises(SubmitRejected) as stale: - classify_submit_context( - SubmitContextInput(context(parent="old"), None, "tip", False) # type: ignore[arg-type] - ) - self.assertEqual(stale.exception.reason, PRISM_REJECTION_STALE_JOB) - - def test_cross_connection_retained_context_is_same_tip_only(self) -> None: - # A detached cross-connection entry must never borrow the - # reconnect's stale-grace window, even when a caller (wrongly) - # classified the window as open for it. - retained = SimpleNamespace(context=context(parent="old")) - with self.assertRaises(SubmitRejected) as stale: - classify_submit_context( - SubmitContextInput( - None, - retained, # type: ignore[arg-type] - "tip", - True, - cross_connection=True, - ) - ) - self.assertEqual(stale.exception.reason, PRISM_REJECTION_STALE_JOB) - - # A same-tip cross-connection resume stays normal credit and keeps - # its cross-connection tag for mask/outcome handling. - same_tip = SimpleNamespace(context=context()) - decision = classify_submit_context( - SubmitContextInput( - None, - same_tip, # type: ignore[arg-type] - "tip", - False, - cross_connection=True, - ) - ) - self.assertEqual(decision.source, "retained") - self.assertIsNone(decision.credit_policy) - self.assertTrue(decision.cross_connection) - - def test_stale_grace_suppresses_block_route(self) -> None: - decision = classify_submit_context( - SubmitContextInput(context(parent="old"), None, "tip", True) # type: ignore[arg-type] - ) - self.assertEqual(decision.credit_policy, PRISM_CREDIT_POLICY_STALE_GRACE) - - submission = SimpleNamespace( - share_pass=True, - block_pass=True, - header_hex="aa", - ) - work = classify_submit_work( - decision.context, - submission, - credit_policy=decision.credit_policy, - ) - self.assertEqual(work.route, "share") - self.assertFalse(work.block_worthy) - - def test_work_routes_cover_share_and_both_block_paths(self) -> None: - value = context() - ordinary = classify_submit_work( - value, # type: ignore[arg-type] - SimpleNamespace(share_pass=True, block_pass=False, header_hex="a"), - credit_policy=None, - ) - asynchronous = classify_submit_work( - value, # type: ignore[arg-type] - SimpleNamespace(share_pass=True, block_pass=True, header_hex="b"), - credit_policy=None, - ) - synchronous = classify_submit_work( - value, # type: ignore[arg-type] - SimpleNamespace(share_pass=False, block_pass=True, header_hex="c"), - credit_policy=None, - ) - - self.assertEqual(ordinary.route, "share") - self.assertEqual(asynchronous.route, "async_block") - self.assertEqual(synchronous.route, "synchronous_block") - self.assertTrue(synchronous.credit_share_on_accept) - - with self.assertRaises(SubmitRejected) as low: - classify_submit_work( - value, # type: ignore[arg-type] - SimpleNamespace(share_pass=False, block_pass=False, header_hex="d"), - credit_policy=None, - ) - self.assertEqual(low.exception.reason, PRISM_REJECTION_LOW_DIFFICULTY) - - def test_every_block_route_requires_block_pass(self) -> None: - """The precondition ``assemble_submission`` gates serialization on. - - ``block_hex`` is materialized only when ``block_pass`` is set, on the - strength of this classifier: no route that builds a block candidate - can be reached without it. If a future route reaches the candidate - path on ``share_pass`` alone, that submission would carry no block - bytes -- so pin the coupling here, next to the routing it constrains. - """ - value = context() - - for share_pass in (False, True): - for block_pass in (False, True): - for credit_policy in (None, PRISM_CREDIT_POLICY_STALE_GRACE): - with self.subTest( - share_pass=share_pass, - block_pass=block_pass, - credit_policy=credit_policy, - ): - try: - work = classify_submit_work( - value, # type: ignore[arg-type] - SimpleNamespace( - share_pass=share_pass, - block_pass=block_pass, - header_hex="e", - ), - credit_policy=credit_policy, - ) - except SubmitRejected: - continue - if work.route != "share": - self.assertTrue(block_pass) - self.assertEqual(work.block_worthy, work.route != "share") - - -class SubmissionMetricsOwnerTests(unittest.TestCase): - def make_server(self) -> PrismCoordinator: - server = PrismCoordinator.__new__(PrismCoordinator) - server.extranonce2_size = 8 - return server - - def test_share_ack_observation_lives_with_the_owner(self) -> None: - server = self.make_server() - server._observe_share_ack_seconds("accepted", 0.02) - server._observe_share_ack_seconds("rejected", 100.0) - - snapshot = server._ensure_share_submission_service().share_ack_snapshot() - self.assertEqual(tuple(snapshot), PRISM_SHARE_ACK_RESULTS) - self.assertEqual(snapshot["accepted"]["count"], 1) - self.assertEqual(snapshot["accepted"]["buckets"][0.025], 1) - self.assertEqual(snapshot["accepted"]["buckets"][0.01], 0) - self.assertEqual(snapshot["rejected"]["count"], 1) - # Above the last finite bucket: only sum/count observe it. - self.assertEqual(snapshot["rejected"]["buckets"][30.0], 0) - self.assertEqual(snapshot["rejected"]["sum"], 100.0) - metrics = "\n".join(server.share_ack_metrics_lines()) - self.assertIn( - 'qbit_prism_share_ack_seconds_count{result="accepted"} 1', - metrics, - ) - - with self.assertRaises(ValueError): - server._observe_share_ack_seconds("weird", 0.1) - - def test_share_ack_metrics_render_zeroed_without_the_owner(self) -> None: - server = PrismCoordinator.__new__(PrismCoordinator) - metrics = "\n".join(server.share_ack_metrics_lines()) - self.assertIn( - 'qbit_prism_share_ack_seconds_count{result="accepted"} 0', - metrics, - ) - self.assertIn( - 'qbit_prism_share_ack_seconds_count{result="rejected"} 0', - metrics, - ) - self.assertNotIn("_share_submission_service", server.__dict__) - - def test_dropped_solve_counts_route_to_the_owner_without_duplicates( - self, - ) -> None: - server = self.make_server() - server.block_solves_dropped_counts = {"stale_grace": 3} - - service = server._ensure_share_submission_service() - # Adopted at construction; the instance dict no longer holds a - # second copy of the mutable mapping. - self.assertNotIn("block_solves_dropped_counts", server.__dict__) - self.assertEqual(server.block_solves_dropped_counts, {"stale_grace": 3}) - - service._count_dropped_block_solve("stale_grace") - self.assertEqual(server.block_solves_dropped_counts, {"stale_grace": 4}) - - server.block_solves_dropped_counts = {"stale_grace": 0} - self.assertEqual(service.dropped_solves_snapshot(), {"stale_grace": 0}) - - -if __name__ == "__main__": - unittest.main() diff --git a/tests/test_prism_share_writer.py b/tests/test_prism_share_writer.py deleted file mode 100644 index 65720708a..000000000 --- a/tests/test_prism_share_writer.py +++ /dev/null @@ -1,839 +0,0 @@ -#!/usr/bin/env python3 -"""Focused PRISM coordinator share writer tests.""" -# ruff: noqa: F403, F405 - -from __future__ import annotations - -import unittest -from tests.prism_vardiff_test_support import * - - -class PrismCoordinatorVardiffTests(unittest.TestCase): - def test_share_ack_and_counters_wait_for_group_commit(self) -> None: - server, state, ledger = submit_coordinator() - server.share_writer_active = True - server.share_commit_timeout_seconds = 2.0 - server.share_commit_linger_seconds = 0.0 - commit_started = threading.Event() - release_commit = threading.Event() - - class BlockingBatchLedger(type(ledger)): - def append_batch(self, entries: object) -> list[object]: - commit_started.set() - release_commit.wait(timeout=2) - return [self.append(pending) for pending, _candidate in entries] - - ledger = BlockingBatchLedger() - server.ledger = ledger - submission = SimpleNamespace( - header_hex="aa" * 80, - block_hash_hex="cc" * 32, - share_pass=True, - block_pass=False, - ) - - writer = threading.Thread(target=server.share_append_loop, daemon=True) - writer.start() - outcome: list[object] = [] - - def submit() -> None: - with patch( - "lab.prism.prism_coordinator.direct_stratum.assemble_submission", - return_value=submission, - ): - outcome.append( - server.handle_submit( - state, - ["miner-a", "job-1", "00" * 8, "00000001", "00000002"], - ) - ) - - submitter = threading.Thread(target=submit) - submitter.start() - self.assertTrue(commit_started.wait(timeout=1)) - self.assertTrue(submitter.is_alive()) - self.assertEqual(len(ledger.pending), 0) - self.assertEqual(server.worker_share_counts["miner-a"]["accepted"], 0) - - release_commit.set() - submitter.join(timeout=2) - self.assertFalse(submitter.is_alive()) - self.assertEqual(outcome, [False]) - self.assertEqual(len(ledger.pending), 1) - self.assertEqual(ledger.pending[0].share_id, "miner-a:" + "cc" * 32) - self.assertEqual(server.worker_share_counts["miner-a"]["accepted"], 1) - server.request_shutdown() - writer.join(timeout=2) - - def test_job_snapshot_anchor_precedes_stamped_uncommitted_share(self) -> None: - # A share is stamped accepted_at_ms at validation time, before its - # group commit. Anchors chosen while that commit is pending must - # predate the stamp, or the issued window would omit a share that a - # later re-derivation at the same anchor includes. - server, state, ledger = submit_coordinator() - server.share_writer_active = True - server.share_commit_timeout_seconds = 2.0 - server.share_commit_linger_seconds = 0.0 - commit_started = threading.Event() - release_commit = threading.Event() - - class BlockingBatchLedger(type(ledger)): - def append_batch(self, entries: object) -> list[object]: - commit_started.set() - release_commit.wait(timeout=2) - return [self.append(pending) for pending, _candidate in entries] - - ledger = BlockingBatchLedger() - server.ledger = ledger - submission = SimpleNamespace( - header_hex="aa" * 80, - block_hash_hex="cc" * 32, - share_pass=True, - block_pass=False, - ) - - writer = threading.Thread(target=server.share_append_loop, daemon=True) - writer.start() - - def submit() -> None: - with patch( - "lab.prism.prism_coordinator.direct_stratum.assemble_submission", - return_value=submission, - ): - server.handle_submit( - state, - ["miner-a", "job-1", "00" * 8, "00000001", "00000002"], - ) - - submitter = threading.Thread(target=submit) - submitter.start() - self.assertTrue(commit_started.wait(timeout=1)) - - floor = dict(server._pending_share_commit_floor) - self.assertEqual(len(floor), 1) - (floor_entry,) = floor.values() - stamped_ms = int(floor_entry[0].accepted_at_ms) - self.assertEqual( - server._job_snapshot_anchor_ms(stamped_ms + 60_000), - stamped_ms - 1, - ) - - release_commit.set() - submitter.join(timeout=2) - self.assertFalse(submitter.is_alive()) - self.assertEqual(server._pending_share_commit_floor, {}) - # Drained floor: the anchor still sits strictly below the clamp - # instant so a same-millisecond stamp can never tie it. - self.assertEqual( - server._job_snapshot_anchor_ms(stamped_ms + 60_000), - stamped_ms + 60_000 - 1, - ) - server.request_shutdown() - writer.join(timeout=2) - - def test_share_batch_failure_still_restores_snapshot_anchor(self) -> None: - server, _state, _ledger = submit_coordinator() - entry = self._pending_append("aa", accepted_at_ms=2) - server._ensure_pending_share_commit_state() - with server._pending_share_commit_lock: - server._pending_share_commit_floor[id(entry.pending_share)] = [ - entry.pending_share, - time.monotonic(), - False, - ] - self.assertEqual(server._job_snapshot_anchor_ms(10_000), 1) - - class FailingLedger(FakeLedger): - def append(self, pending: object) -> object: - raise RuntimeError("ledger unavailable") - - server.ledger = FailingLedger() - self.assertFalse(server._append_share_batch([entry])) - self.assertTrue(entry.committed.is_set()) - self.assertEqual(server._pending_share_commit_floor, {}) - self.assertEqual(server._job_snapshot_anchor_ms(10_000), 9_999) - - def test_failed_commit_releases_duplicate_key_for_exact_retry(self) -> None: - server, state, healthy = submit_coordinator() - server.share_writer_active = True - server.share_commit_linger_seconds = 0.0 - server.share_commit_timeout_seconds = 1.0 - - class FailedLedger: - def append_batch(self, _entries: object) -> list[object]: - raise RuntimeError("postgres unavailable") - - server.ledger = FailedLedger() - writer = threading.Thread(target=server.share_append_loop, daemon=True) - writer.start() - submission = SimpleNamespace( - header_hex="aa" * 80, - block_hash_hex="dd" * 32, - share_pass=True, - block_pass=False, - ) - with patch( - "lab.prism.prism_coordinator.direct_stratum.assemble_submission", - return_value=submission, - ): - with self.assertRaisesRegex(StratumError, "commit failed"): - server.handle_submit( - state, - ["miner-a", "job-1", "00" * 8, "00000001", "00000002"], - ) - - server.share_writer_active = False - server.ledger = healthy - with patch( - "lab.prism.prism_coordinator.direct_stratum.assemble_submission", - return_value=submission, - ): - self.assertFalse( - server.handle_submit( - state, - ["miner-a", "job-1", "00" * 8, "00000001", "00000002"], - ) - ) - self.assertEqual(len(healthy.pending), 1) - self.assertEqual(server.worker_share_counts["miner-a"]["accepted"], 1) - server.request_shutdown() - writer.join(timeout=2) - - def test_share_writer_retries_failed_append_until_it_lands(self) -> None: - server, state, ledger = submit_coordinator() - - class FlakyLedger(type(ledger)): - def __init__(self) -> None: - super().__init__() - self.failures_remaining = 2 - - def append(self, pending: object) -> object: - if self.failures_remaining > 0: - self.failures_remaining -= 1 - raise RuntimeError("ledger briefly unavailable") - return super().append(pending) - - flaky = FlakyLedger() - server.ledger = flaky - entry = PendingShareAppend( - pending_share=SimpleNamespace( - share_id="miner-a:" + "ee" * 32, - job_issued_at_ms=0, - accepted_at_ms=0, - ), - username="miner-a", - job_id="job-1", - block_hash_hex="ee" * 32, - collection_only=False, - credit_policy=None, - ) - - with patch.object(server.stop_event, "wait", return_value=False) as waited: - server._append_share_entry(entry, retry_until_stopped=True) - - self.assertEqual(len(flaky.pending), 1) - self.assertEqual(server.share_append_failure_count, 2) - self.assertIn( - "qbit_prism_share_append_failures_total 2", - server.metrics_payload(), - ) - self.assertEqual(waited.call_count, 2) - - def _pending_append(self, tag: str, accepted_at_ms: int = 2) -> PendingShareAppend: - from lab.prism.share_ledger import PendingShare - - return PendingShareAppend( - pending_share=PendingShare( - share_id=f"miner-a:{tag}", - miner_id="miner-a", - order_key="miner-a", - p2mr_program_hex="11" * 32, - share_difficulty=1, - network_difficulty=1, - template_height=10, - job_id="job-1", - job_issued_at_ms=1, - accepted_at_ms=accepted_at_ms, - ntime=1_700_000_000, - ), - username="miner-a", - job_id="job-1", - block_hash_hex=tag * 32, - collection_only=False, - credit_policy=None, - ) - - def test_share_append_backlog_overflow_never_reports_success(self) -> None: - server, _state, _ledger = submit_coordinator() - server.share_append_queue = queue.Queue(maxsize=2) - with tempfile.TemporaryDirectory() as tempdir: - server.share_recovery_path = Path(tempdir) / "recovery.jsonl" - - server.enqueue_share_append(self._pending_append("aa")) - server.enqueue_share_append(self._pending_append("bb")) - with self.assertRaisesRegex(StratumError, "queue is full"): - server.enqueue_share_append(self._pending_append("cc")) - - self.assertEqual(server.shares_recovered_to_disk, 0) - remaining = [ - server.share_append_queue.get_nowait().pending_share.share_id - for _ in range(2) - ] - self.assertEqual(remaining, ["miner-a:aa", "miner-a:bb"]) - self.assertFalse(server.share_recovery_path.exists()) - - def test_writer_recovers_acked_share_on_shutdown_during_outage(self) -> None: - server, _state, ledger = submit_coordinator() - with tempfile.TemporaryDirectory() as tempdir: - server.share_recovery_path = Path(tempdir) / "recovery.jsonl" - - class DownLedger(type(ledger)): - def append(self, pending: object) -> object: - raise RuntimeError("postgres unavailable") - - server.ledger = DownLedger() - entry = self._pending_append("ee") - # First backoff wait returns True (stop requested): the share must - # be recovered, not silently dropped. - with patch.object(server.stop_event, "wait", return_value=True): - server._append_share_entry(entry, retry_until_stopped=True) - - self.assertEqual(server.shares_recovered_to_disk, 1) - recovered = json.loads(server.share_recovery_path.read_text().strip()) - self.assertEqual(recovered["share_id"], "miner-a:ee") - - def test_replay_recovered_shares_appends_and_clears_file(self) -> None: - server, _state, ledger = submit_coordinator() - with tempfile.TemporaryDirectory() as tempdir: - server.share_recovery_path = Path(tempdir) / "recovery.jsonl" - server._recover_share_to_disk(self._pending_append("f1"), "test") - server._recover_share_to_disk(self._pending_append("f2"), "test") - - replayed = server.replay_recovered_shares() - - self.assertEqual(replayed, 2) - self.assertEqual(server.shares_replayed, 2) - self.assertEqual( - [p.share_id for p in ledger.pending], ["miner-a:f1", "miner-a:f2"] - ) - # File is cleared after a clean replay so shares are not re-added. - self.assertFalse(server.share_recovery_path.exists()) - self.assertEqual(server.replay_recovered_shares(), 0) - - def test_replay_recovered_shares_orders_by_accepted_at(self) -> None: - # A share can be recovered out of FIFO order (overflow of the newest, or - # a ledger flap during the shutdown drain). Replay must reorder by - # accepted_at_ms so share_seq reflects acceptance order, keeping the - # reward window correctly ordered. - server, _state, ledger = submit_coordinator() - with tempfile.TemporaryDirectory() as tempdir: - server.share_recovery_path = Path(tempdir) / "recovery.jsonl" - server._recover_share_to_disk(self._pending_append("late", accepted_at_ms=300), "test") - server._recover_share_to_disk(self._pending_append("early", accepted_at_ms=100), "test") - server._recover_share_to_disk(self._pending_append("mid", accepted_at_ms=200), "test") - - replayed = server.replay_recovered_shares() - - self.assertEqual(replayed, 3) - self.assertEqual( - [p.share_id for p in ledger.pending], - ["miner-a:early", "miner-a:mid", "miner-a:late"], - ) - - def test_replay_skips_torn_line_and_keeps_file(self) -> None: - # A crash mid-append can leave the last line torn. That one line must - # not block the intact shares before it, and the file is kept so the - # torn line is preserved rather than silently discarded. - server, _state, ledger = submit_coordinator() - with tempfile.TemporaryDirectory() as tempdir: - server.share_recovery_path = Path(tempdir) / "recovery.jsonl" - server._recover_share_to_disk(self._pending_append("g1", accepted_at_ms=100), "test") - server._recover_share_to_disk(self._pending_append("g2", accepted_at_ms=200), "test") - with open(server.share_recovery_path, "a", encoding="utf-8") as handle: - handle.write('{"share_id": "miner-a:torn", "miner_i') # truncated, no newline - - replayed = server.replay_recovered_shares() - - self.assertEqual(replayed, 2) - self.assertEqual( - [p.share_id for p in ledger.pending], ["miner-a:g1", "miner-a:g2"] - ) - # File kept because a line could not be parsed. - self.assertTrue(server.share_recovery_path.exists()) - # Re-running classifies the intact rows as exact-existing. They are - # not inserted or counted again, while the torn line keeps the - # journal for inspection. - self.assertEqual(server.replay_recovered_shares(), 0) - self.assertEqual( - [p.share_id for p in ledger.pending], ["miner-a:g1", "miner-a:g2"] - ) - self.assertTrue(server.share_recovery_path.exists()) - - def test_replay_is_idempotent_across_partial_replay(self) -> None: - # Finding: a partial replay (A commits, B fails transiently) kept the - # whole file; on retry, replay hit A's duplicate and stopped, stranding - # B forever. Replay must skip the already-committed A and reach B. - server, _state, _ledger = submit_coordinator() - with tempfile.TemporaryDirectory() as tempdir: - server.share_recovery_path = Path(tempdir) / "recovery.jsonl" - server._recover_share_to_disk(self._pending_append("A", accepted_at_ms=100), "test") - server._recover_share_to_disk(self._pending_append("B", accepted_at_ms=200), "test") - - class DedupLedger: - def __init__(self) -> None: - self.ids: list[str] = [] - self.pending_by_id: dict[str, object] = {} - self.fail_b_once = True - - def append_recovered_share(self, pending: object) -> object: - if pending.share_id == "miner-a:B" and self.fail_b_once: - self.fail_b_once = False - raise RuntimeError("postgres unavailable") - if pending.share_id in self.ids: - if self.pending_by_id[pending.share_id] != pending: - raise ShareReplayConflict(pending.share_id) - return ShareReplayResult( - "exact_existing", - SimpleNamespace( - share_seq=self.ids.index(pending.share_id) + 1 - ), - ) - self.ids.append(pending.share_id) - self.pending_by_id[pending.share_id] = pending - return ShareReplayResult( - "inserted", - SimpleNamespace(share_seq=len(self.ids)), - ) - - server.ledger = DedupLedger() - - # Pass 1: A commits, B raises a transient (non-duplicate) error, so - # the pass stops and keeps the file. - self.assertEqual(server.replay_recovered_shares(), 1) - self.assertTrue(server.share_recovery_path.exists()) - self.assertEqual(server.ledger.ids, ["miner-a:A"]) - - # Pass 2: A is now a duplicate (skipped, not fatal); B commits and - # the file is cleared. - self.assertEqual(server.replay_recovered_shares(), 1) - self.assertEqual(server.ledger.ids, ["miner-a:A", "miner-a:B"]) - self.assertFalse(server.share_recovery_path.exists()) - - def test_replay_quarantines_conflicting_row_and_replays_the_rest(self) -> None: - # A ShareReplayConflict is keyed to one share_id; it must not strand - # the journal rows behind it, and the conflict must be visible as a - # counter, not only in stdout. - server, _state, _ledger = submit_coordinator() - with tempfile.TemporaryDirectory() as tempdir: - server.share_recovery_path = Path(tempdir) / "recovery.jsonl" - server._recover_share_to_disk(self._pending_append("A", accepted_at_ms=100), "test") - server._recover_share_to_disk(self._pending_append("B", accepted_at_ms=200), "test") - server._recover_share_to_disk(self._pending_append("C", accepted_at_ms=300), "test") - - class ConflictingLedger(RecordingLedger): - def __init__(self) -> None: - super().__init__() - self.ids: list[str] = [] - - def append_recovered_share(self, pending: object) -> object: - if pending.share_id == "miner-a:A": - raise ShareReplayConflict(pending.share_id) - if pending.share_id in self.ids: - return ShareReplayResult( - "exact_existing", - SimpleNamespace( - share_seq=self.ids.index(pending.share_id) + 1 - ), - ) - self.ids.append(pending.share_id) - return ShareReplayResult( - "inserted", - SimpleNamespace(share_seq=len(self.ids)), - ) - - server.ledger = ConflictingLedger() - - self.assertEqual(server.replay_recovered_shares(), 2) - self.assertTrue(server.share_recovery_path.exists()) - self.assertEqual(server.ledger.ids, ["miner-a:B", "miner-a:C"]) - self.assertEqual(server.share_replay_conflicts, 1) - self.assertIn( - "qbit_prism_share_replay_conflicts_total 1", - server.metrics_payload(), - ) - - # Re-running the retained journal credits nothing twice: B and C - # are exact duplicates, A conflicts again, the file stays. - self.assertEqual(server.replay_recovered_shares(), 0) - self.assertTrue(server.share_recovery_path.exists()) - self.assertEqual(server.ledger.ids, ["miner-a:B", "miner-a:C"]) - self.assertEqual(server.share_replay_conflicts, 2) - - def test_append_share_entry_reports_persisted_vs_recovered(self) -> None: - server, _state, ledger = submit_coordinator() - with tempfile.TemporaryDirectory() as tempdir: - server.share_recovery_path = Path(tempdir) / "recovery.jsonl" - # Healthy ledger: reports persisted. - self.assertTrue( - server._append_share_entry(self._pending_append("ok"), retry_until_stopped=True) - ) - - class DownLedger(type(ledger)): - def append(self, pending: object) -> object: - raise RuntimeError("postgres unavailable") - - server.ledger = DownLedger() - with patch.object(server.stop_event, "wait", return_value=True): - # Shutdown mid-outage: reports recovered, not persisted. - self.assertFalse( - server._append_share_entry(self._pending_append("down"), retry_until_stopped=True) - ) - - def test_group_commit_failure_releases_all_waiters_without_recovery_file(self) -> None: - server, _state, ledger = submit_coordinator() - with tempfile.TemporaryDirectory() as tempdir: - server.share_recovery_path = Path(tempdir) / "recovery.jsonl" - server.share_append_queue = queue.Queue(maxsize=MAX_PENDING_SHARE_APPENDS) - - append_calls: list[str] = [] - - class DownLedger(type(ledger)): - def append(self, pending: object) -> object: - append_calls.append(pending.share_id) - raise RuntimeError("postgres unavailable") - - server.ledger = DownLedger() - entries = [] - for tag in ("s1", "s2", "s3"): - entry = self._pending_append(tag) - entries.append(entry) - server.enqueue_share_append(entry) - - server.request_shutdown() - server.share_append_loop() - - # The compatibility ledger fails on the first row; the whole batch - # is reported failed and no uncommitted share is called durable. - self.assertEqual(append_calls, ["miner-a:s1"]) - self.assertEqual(ledger.pending, []) - self.assertTrue(all(entry.committed.is_set() for entry in entries)) - self.assertTrue(all(entry.error is not None for entry in entries)) - self.assertFalse(server.share_recovery_path.exists()) - - def test_predating_append_commits_while_fast_lane_offer_is_in_flight( - self, - ) -> None: - # The other side of the fence boundary flipped with the node fast - # lane: submitblock no longer holds the landing fence, so a - # predating append's durable commit does NOT wait for the RPC to - # return. The row may become durable mid-offer with its epoch bump - # landing under the fence; the landing's post-offer accounting - # fences (not the offer itself) are what observe the invalidation. - # Here the node rejects the block outright, so the rejection -- not - # the epoch race -- terminally abandons the candidate. - server, state, _recording = submit_coordinator() - ledger = SingleWriterShareLedger() - server.ledger = ledger - context = server.jobs["job-1"] - context.found_block = { - "network_difficulty": 1, - "anchor_job_issued_at_ms": 12000, - } - # Keep an anchor the late row predates exposed for the whole test, - # independent of the landing's own declared-anchor lifetime. - anchor_token = server._expose_inflight_scan_anchor(12000) - self.addCleanup(server._retire_inflight_scan_anchor, anchor_token) - - late_entry = self._pending_append("mid-rpc-append") - durable_mid_rpc: list[bool] = [] - append_threads: list[threading.Thread] = [] - - class MidRpcAppendRpc(TipRpc): - def call( - rpc_self, - method: str, - params: list[object] | None = None, - ) -> object: - if method == "getblockcount": - return 9 - if method == "submitblock": - thread = threading.Thread( - target=server._append_share_entry, - args=(late_entry,), - daemon=True, - ) - thread.start() - append_threads.append(thread) - deadline = time.monotonic() + 0.25 - became_durable = False - while time.monotonic() < deadline: - if late_entry.pending_share.share_id in ledger._share_ids: - became_durable = True - break - time.sleep(0.005) - durable_mid_rpc.append(became_durable) - return "rejected-by-test" - return super().call(method, params) - - server.rpc = MidRpcAppendRpc("00" * 32) - submission = SimpleNamespace( - coinbase_tx_hex="c0ffee", - block_hash_hex="dd" * 32, - block_hex="00", - ) - pending = self._pending_append("landing-holds-fence").pending_share - candidate = block_candidate( - server, - state, - submission, - pending_share=pending, - ) - ledger.append_batch( - [(pending, server.block_candidate_intent(candidate))] - ) - server.enqueue_block_candidate(candidate) - - self.assertTrue(server.submit_next_block_candidate()) - - # The fast-lane RPC holds no fence, so the predating row became - # durable while the offer was still in flight. - self.assertEqual(durable_mid_rpc, [True]) - self.assertTrue(append_threads) - append_threads[0].join(timeout=10.0) - self.assertFalse(append_threads[0].is_alive()) - self.assertIn(late_entry.pending_share.share_id, ledger._share_ids) - with server._job_cache_lock: - self.assertEqual(server._payout_ledger_append_invalidation_epoch, 1) - self.assertEqual( - server.block_candidate_abandoned_counts[ - PRISM_REJECTION_SUBMITBLOCK_REJECTED - ], - 1, - ) - - def test_completed_append_predating_seedless_published_window_fences( - self, - ) -> None: - # A build that seeds no artifact (the documented - # PRISM_PAYOUT_ARTIFACT_REUSE=0 rollback mode) retires its walk - # exposure at the publication fence, yet jobs stamped from it keep - # serving that window until they retire. A replay-shaped append - # that started AND finished in that gap used to see no live - # anchor: no epoch bump, and its registry entry was popped on - # completion -- so a later landing drained nothing, its - # nonnegative context epoch skipped durable revalidation, and - # submitblock accepted a coinbase omitting the durable share. - # Publication now hands the declared anchor to the - # published-window watermark, so the append commits under the - # landing fence with its epoch bump -- and the landing, whose - # offer already landed, keeps the as-issued snapshot for - # accounting. - server, state, _recording = submit_coordinator() - ledger = SingleWriterShareLedger() - server.ledger = ledger - bundle_builds: list[dict[str, object]] = [] - - def recording_build_audit_bundle(**kwargs: object) -> dict[str, object]: - bundle_builds.append(dict(kwargs)) - return verified_block_bundle() - - server.build_audit_bundle = recording_build_audit_bundle # type: ignore[method-assign] - server.verify_bundle = lambda *_args, **_kwargs: verified_audit_report() # type: ignore[method-assign] - tail_dir = tempfile.TemporaryDirectory() - self.addCleanup(tail_dir.cleanup) - server.audit_dir = Path(tail_dir.name) - server.evidence_path = Path(tail_dir.name) / "evidence.json" - server.ledger_writer_public_key_hex = "aa" * 32 - context = server.jobs["job-1"] - context.found_block = { - "network_difficulty": 1, - "anchor_job_issued_at_ms": 12000, - } - submission = SimpleNamespace( - coinbase_tx_hex="c0ffee", - block_hash_hex="dd" * 32, - block_hex="00", - ) - pending = self._pending_append("seedless-window").pending_share - candidate = block_candidate( - server, - state, - submission, - pending_share=pending, - ) - ledger.append_batch( - [(pending, server.block_candidate_intent(candidate))] - ) - server.enqueue_block_candidate(candidate) - - # The publication fence of a seedless build hands the bundle's - # declared anchor to the watermark when it retires the walk - # exposure. - server._ensure_job_cache_state() - with server._job_cache_lock: - server._publish_seedless_job_window_anchor_locked(12000) - - # The ordinary share hot path stays off the fence: a row stamped - # above the watermark commits unfenced and bumps nothing. - ordinary_entry = self._pending_append( - "ordinary-above-watermark", accepted_at_ms=13000 - ) - server._append_share_entry(ordinary_entry) - with server._job_cache_lock: - self.assertEqual(server._payout_ledger_append_invalidation_epoch, 0) - self.assertEqual( - server._payout_unfenced_append_inflight_stamps, {} - ) - - # The replay-shaped append starts and finishes entirely before the - # landing begins: nothing is in flight for the landing to drain. - late_entry = self._pending_append("completed-before-landing") - server._append_share_entry(late_entry) - self.assertIn(late_entry.pending_share.share_id, ledger._share_ids) - with server._job_cache_lock: - self.assertEqual(server._payout_ledger_append_invalidation_epoch, 1) - self.assertEqual( - server._payout_unfenced_append_inflight_stamps, {} - ) - - submitblock_calls: list[object] = [] - - class RecordingSubmitRpc(TipRpc): - def call( - rpc_self, - method: str, - params: list[object] | None = None, - ) -> object: - if method == "getblockcount": - return 9 - if method == "submitblock": - submitblock_calls.append(params) - return None - if method == "getblockhash": - return "dd" * 32 - return super().call(method, params) - - server.rpc = RecordingSubmitRpc("00" * 32) - - self.assertTrue(server.submit_next_block_candidate()) - - # The bump already happened at commit time; the landing's epoch - # fence observes it and, with the node offer already accepted, - # accounts the as-issued coinbase (the durable predating share - # rides the next window). - self.assertEqual(submitblock_calls, [["00"]]) - # The landing observed the bump but the node already accepted - # the block: accounting keeps the as-issued snapshot instead of - # terminally abandoning a live block's payouts. - self.assertEqual(len(bundle_builds), 1) - self.assertNotIn( - PRISM_REJECTION_STALE_JOB, - server.block_candidate_abandoned_counts, - ) - self.assertEqual( - getattr(server, "stale_job_abandon_counts", {}).get( - "append_epoch_stale", 0 - ), - 0, - ) - outbox_row = ledger._block_candidate_outbox[submission.block_hash_hex] - self.assertEqual(outbox_row["state"], "submitted") - self.assertIsNone(outbox_row["last_error"]) - self.assertEqual(server.accepted_block_count, 1) - - def test_replay_recovered_shares_commit_under_the_landing_fence(self) -> None: - # Recovered rows reconstruct pre-crash timestamps, so they predate - # live anchors. Each replay append must hold the landing fence across - # the ledger commit and its epoch bump -- never bump after the row is - # already durable, where a landing could verify the pre-bump epoch - # and submit a coinbase omitting the replayed share. - server, _state, ledger = submit_coordinator() - anchor_token = server._expose_inflight_scan_anchor(12000) - self.addCleanup(server._retire_inflight_scan_anchor, anchor_token) - with tempfile.TemporaryDirectory() as tempdir: - server.share_recovery_path = Path(tempdir) / "recovery.jsonl" - server._recover_share_to_disk(self._pending_append("h1"), "test") - - fence_held_during_commit: list[bool] = [] - original_append = ledger.append - - def recording_append(pending: object) -> object: - fence_held_during_commit.append( - server._payout_append_landing_fence_lock.locked() - ) - return original_append(pending) - - ledger.append = recording_append # type: ignore[method-assign] - replayed = server.replay_recovered_shares() - - self.assertEqual(replayed, 1) - self.assertEqual(fence_held_during_commit, [True]) - with server._job_cache_lock: - self.assertEqual(server._payout_ledger_append_invalidation_epoch, 1) - - def test_sync_append_commits_predating_row_under_the_landing_fence( - self, - ) -> None: - # The synchronous (no-writer) append path takes the same fence - # boundary as the group-commit writer for a row that predates a - # live anchor. - server, _state, ledger = submit_coordinator() - anchor_token = server._expose_inflight_scan_anchor(12000) - self.addCleanup(server._retire_inflight_scan_anchor, anchor_token) - - fence_held_during_commit: list[bool] = [] - original_append = ledger.append - - def recording_append(pending: object) -> object: - fence_held_during_commit.append( - server._payout_append_landing_fence_lock.locked() - ) - return original_append(pending) - - ledger.append = recording_append # type: ignore[method-assign] - self.assertTrue(server._append_share_entry(self._pending_append("h2"))) - self.assertEqual(fence_held_during_commit, [True]) - with server._job_cache_lock: - self.assertEqual(server._payout_ledger_append_invalidation_epoch, 1) - - -class PrismCoordinatorReliabilityTests(unittest.TestCase): - def _bare_coordinator(self) -> PrismCoordinator: - server = PrismCoordinator.__new__(PrismCoordinator) - server.lock = threading.RLock() - server.stop_event = threading.Event() - server._heartbeats = {} - server._watchdog_pauses = {} - server._heartbeats_lock = threading.Lock() - server.watchdog_timeout_seconds = 120.0 - server.watchdog_interval_seconds = 15.0 - return server - - def test_release_ledger_lease_is_noop_without_lease_support(self) -> None: - server = self._bare_coordinator() - server.ledger = SimpleNamespace() - - # In-memory/regtest ledgers have no release_writer_lease; must not raise. - server.release_ledger_lease() - - def test_release_ledger_lease_invokes_ledger_release(self) -> None: - server = self._bare_coordinator() - calls: list[bool] = [] - server.ledger = SimpleNamespace( - release_writer_lease=lambda: (calls.append(True), True)[1] - ) - - server.release_ledger_lease() - - self.assertEqual(calls, [True]) - - def test_release_ledger_lease_swallows_release_errors(self) -> None: - server = self._bare_coordinator() - - def _boom() -> bool: - raise RuntimeError("db unreachable during shutdown") - - server.ledger = SimpleNamespace(release_writer_lease=_boom) - - # Shutdown must not raise even if the lease release fails. - server.release_ledger_lease() diff --git a/tests/test_prism_share_writer_service.py b/tests/test_prism_share_writer_service.py deleted file mode 100644 index 7b6c7b79e..000000000 --- a/tests/test_prism_share_writer_service.py +++ /dev/null @@ -1,654 +0,0 @@ -#!/usr/bin/env python3 -"""Direct tests for the coordinator-free S3 share writer service. - -The service resolves the coordinator through a call-time ``ShareWriterRuntime`` -typed port, so these tests drive it with a small recording runtime plus the -real shutdown controller and (mostly) the real in-memory ledger. -""" - -from __future__ import annotations - -from contextlib import contextmanager -from pathlib import Path -import queue -import tempfile -import threading -import time -from types import SimpleNamespace -import unittest -from unittest.mock import patch - -from lab.prism.coordinator_shutdown import ( - CoordinatorShutdownController, - ShutdownInProgress, -) -from lab.prism.share_ledger import ( - PendingShare, - ShareReplayResult, - SingleWriterShareLedger, -) -from lab.prism.share_writer import ( - PENDING_SHARE_COMMIT_WARN_SECONDS, - PendingShareAppend, - ShareWriter, -) -from lab.prism.stratum_session import StratumError - - -class RecordingRuntime: - """Minimal call-time runtime port mirroring the coordinator facades.""" - - def __init__( - self, - *, - ledger: object, - controller: CoordinatorShutdownController, - stop_event: threading.Event, - ) -> None: - self.ledger = ledger - self.hot_path_log_enabled = False - self.stop_event = stop_event - self._controller = controller - self._share_accounting_lock = threading.Lock() - self.writer: ShareWriter | None = None - self.worker_notes: list[tuple[str, str | None]] = [] - self.vardiff_notes: list[tuple[object, object]] = [] - self.retired: list[tuple[PendingShare, int]] = [] - - def _ensure_shutdown_controller(self) -> CoordinatorShutdownController: - return self._controller - - def _record_heartbeat(self, _name: str, *, phase: str | None = None) -> None: - return None - - def _ensure_share_hot_path_state(self) -> None: - return None - - def _finish_pending_share_commit(self, pending: PendingShare) -> None: - assert self.writer is not None - self.writer.finish_pending_share(pending) - - @contextmanager - def _landing_fence_for_predating_append(self, _pendings: list[PendingShare]): - yield False - - def _record_late_visible_payout_append( - self, - _pending: PendingShare, - *, - landing_fence_owned: bool, - ) -> int | None: - return None - - def _retire_payout_windows_for_late_append( - self, pending: PendingShare, epoch: int - ) -> None: - self.retired.append((pending, epoch)) - - def _recover_share_to_disk(self, entry: PendingShareAppend, reason: str) -> None: - assert self.writer is not None - self.writer.recover_share_to_disk(entry, reason) - - def accepted_share_difficulty(self, _context: object) -> int: - return 1 - - def enqueue_share_append( - self, entry: PendingShareAppend, *, wait: bool = False - ) -> None: - assert self.writer is not None - self.writer.enqueue_share_append(entry, wait=wait) - - def _append_share_entry(self, entry: PendingShareAppend) -> bool: - assert self.writer is not None - return self.writer.append_share_entry(entry) - - def note_worker_accepted_share( - self, username: str, credit_policy: str | None - ) -> None: - self.worker_notes.append((username, credit_policy)) - - def note_vardiff_accepted_share(self, client: object, job: object) -> None: - self.vardiff_notes.append((client, job)) - - -class ShareWriterServiceTests(unittest.TestCase): - def _service( - self, - *, - ledger: object | None = None, - wall_times: list[int] | None = None, - controller: CoordinatorShutdownController | None = None, - ) -> tuple[ShareWriter, object, CoordinatorShutdownController, RecordingRuntime]: - ledger = ledger if ledger is not None else SingleWriterShareLedger() - controller = controller or CoordinatorShutdownController(1.0) - stop = threading.Event() - times = list(wall_times or [100]) - last = times[-1] - - def wall_time_ms() -> int: - return times.pop(0) if times else last - - runtime = RecordingRuntime( - ledger=ledger, - controller=controller, - stop_event=stop, - ) - service = ShareWriter( - runtime, - internal_error_reason="internal_error", - now_ms=wall_time_ms, - ) - runtime.writer = service - return service, ledger, controller, runtime - - @staticmethod - def _context(username: str = "miner-a") -> SimpleNamespace: - return SimpleNamespace( - worker=SimpleNamespace( - username=username, - payout_address="miner-a", - p2mr_program_hex="11" * 32, - ), - found_block={"network_difficulty": 1}, - template={"height": 10}, - job=SimpleNamespace(job_id="job-1"), - issued_at_ms=1, - collection_only=False, - ) - - @staticmethod - def _submission(block_hash_hex: str = "ab" * 32) -> SimpleNamespace: - return SimpleNamespace(block_hash_hex=block_hash_hex) - - def _stamp(self, service: ShareWriter, block_hash_hex: str = "ab" * 32) -> PendingShare: - return service.pending_share_from_submission( - context=self._context(), - submission=self._submission(block_hash_hex), - ntime_hex="65000000", - ) - - @staticmethod - def _entry(pending: PendingShare) -> PendingShareAppend: - return PendingShareAppend( - pending_share=pending, - username="miner-a", - job_id="job-1", - block_hash_hex=pending.share_id.rsplit(":", 1)[-1], - collection_only=False, - credit_policy=None, - ) - - @staticmethod - def _pending(share_id: str, *, accepted_at_ms: int = 100, ntime: int = 1_700_000_000) -> PendingShare: - return PendingShare( - share_id=share_id, - miner_id="miner-a", - order_key="miner-a", - p2mr_program_hex="11" * 32, - share_difficulty=1, - network_difficulty=1, - template_height=9, - job_id="job-1", - job_issued_at_ms=1, - accepted_at_ms=accepted_at_ms, - ntime=ntime, - ) - - # -- compatibility identity and descriptor routing --------------------- - - def test_coordinator_reexports_exact_pending_append_identity(self) -> None: - from lab.prism.prism_coordinator import PendingShareAppend as compatibility - - self.assertIs(compatibility, PendingShareAppend) - - def test_legacy_coordinator_fields_route_to_the_service_owner(self) -> None: - from lab.prism.prism_coordinator import PrismCoordinator - - server = PrismCoordinator.__new__(PrismCoordinator) - service = server._ensure_share_writer_service() - - self.assertIs(server.share_append_queue, service.share_append_queue) - self.assertIs( - server._pending_share_commit_floor, - service._pending_share_commit_floor, - ) - - server.share_commit_batch_size = 7 - self.assertEqual(service.share_commit_batch_size, 7) - - adopted: queue.Queue[PendingShareAppend] = queue.Queue(maxsize=3) - server.share_append_queue = adopted - self.assertIs(service.share_append_queue, adopted) - - server.share_writer_active = True - self.assertTrue(service.share_writer_active) - - # -- pending-commit floor ---------------------------------------------- - - def test_submission_stamp_registers_floor_and_anchor_clamps_below(self) -> None: - service, _ledger, _controller, _runtime = self._service(wall_times=[100]) - - pending = self._stamp(service) - - self.assertEqual(pending.accepted_at_ms, 100) - entry = service._pending_share_commit_floor[id(pending)] - self.assertIs(entry[0], pending) - self.assertEqual(len(entry), 3) - self.assertFalse(entry[2]) - self.assertEqual(service.snapshot_anchor_ms(1_000), 99) - - service.finish_pending_share(pending) - self.assertEqual(service._pending_share_commit_floor, {}) - self.assertEqual(service.snapshot_anchor_ms(1_000), 999) - - def test_floor_release_is_idempotent_and_keyed_by_object_identity(self) -> None: - # Layer note: the current tree keeps the id-keyed holder (the old - # stack's durable share-id lease was deliberately not built), so a - # reconstructed object with the same durable identity does not - # release the original stamp's floor entry. - service, _ledger, _controller, _runtime = self._service(wall_times=[100]) - pending = self._stamp(service) - - reconstructed = PendingShare(**pending.__dict__) - service.finish_pending_share(reconstructed) - self.assertIn(id(pending), service._pending_share_commit_floor) - self.assertEqual(service.snapshot_anchor_ms(1_000), 99) - - service.finish_pending_share(pending) - self.assertEqual(service._pending_share_commit_floor, {}) - service.finish_pending_share(pending) - self.assertEqual(service._pending_share_commit_floor, {}) - - def test_attempt_and_candidate_release_share_one_current_holder(self) -> None: - # finish_pending_attempt / finish_pending_candidate are the staged - # future split seams; at this layer both release the single holder. - service, _ledger, _controller, _runtime = self._service(wall_times=[100, 200]) - - attempt = self._stamp(service) - service.finish_pending_attempt(attempt) - self.assertEqual(service._pending_share_commit_floor, {}) - - candidate = self._stamp(service, "cd" * 32) - service.finish_pending_candidate(candidate) - self.assertEqual(service._pending_share_commit_floor, {}) - - def test_stale_floor_entry_warns_once_and_keeps_holding_the_anchor(self) -> None: - service, _ledger, _controller, _runtime = self._service() - pending = self._pending("miner-a:stale", accepted_at_ms=100) - service._pending_share_commit_floor[id(pending)] = [ - pending, - time.monotonic() - PENDING_SHARE_COMMIT_WARN_SECONDS - 1.0, - False, - ] - - with patch("builtins.print") as first_pass: - self.assertEqual(service.snapshot_anchor_ms(1_000), 99) - warned = [ - call - for call in first_pass.call_args_list - if "holding the job snapshot anchor floor" in str(call) - ] - self.assertEqual(len(warned), 1) - self.assertIn("miner-a:stale", str(warned[0])) - self.assertTrue(service._pending_share_commit_floor[id(pending)][2]) - - with patch("builtins.print") as second_pass: - self.assertEqual(service.snapshot_anchor_ms(1_000), 99) - self.assertEqual(second_pass.call_args_list, []) - - # -- append admission and floor release -------------------------------- - - def test_full_queue_admission_refusal_releases_token_and_floor(self) -> None: - service, _ledger, controller, _runtime = self._service() - service.share_writer_active = True - service.share_commit_timeout_seconds = 0.05 - service.share_append_queue = queue.Queue(maxsize=1) - blocker = self._entry(self._pending("miner-a:blocker")) - blocker.writer_token = controller.reserve_writer("share_persistence") - service.share_append_queue.put_nowait(blocker) - pending = self._stamp(service) - - with self.assertRaisesRegex(StratumError, "commit queue is full"): - service.append_accepted_share( - object(), - self._context(), - self._submission(), - pending, - ) - - self.assertEqual(service._pending_share_commit_floor, {}) - self.assertEqual( - controller.snapshot()["active_writers"], - {"share_persistence": 1}, - ) - blocker.writer_token.finish() - self.assertEqual(controller.snapshot()["active_writers"], {}) - - def test_closed_writer_admission_refusal_releases_unhanded_attempt(self) -> None: - controller = CoordinatorShutdownController(1.0) - service, ledger, _controller, runtime = self._service(controller=controller) - service.share_writer_active = True - pending = self._stamp(service) - controller.begin_shutdown("test shutdown") - - with self.assertRaises(ShutdownInProgress): - service.append_accepted_share( - object(), - self._context(), - self._submission(), - pending, - ) - - self.assertEqual(service._pending_share_commit_floor, {}) - self.assertEqual(len(ledger), 0) - self.assertEqual(runtime.worker_notes, []) - self.assertEqual(controller.snapshot()["active_writers"], {}) - - def test_nested_append_inherits_outer_admission_after_shutdown_closes(self) -> None: - controller = CoordinatorShutdownController(1.0) - service, ledger, _controller, _runtime = self._service(controller=controller) - service.share_writer_active = True - outer = controller.enter_writer("share_submission") - try: - pending = self._stamp(service) - controller.begin_shutdown("test shutdown") - entry = self._entry(pending) - service.enqueue_share_append(entry) - finally: - controller.exit_writer(outer) - - self.assertIsNotNone(entry.writer_token) - queued = service.share_append_queue.get_nowait() - self.assertIs(queued, entry) - self.assertTrue(service.append_share_batch([queued])) - self.assertEqual(len(ledger), 1) - self.assertTrue(entry.committed.is_set()) - self.assertIsNone(entry.writer_token) - self.assertEqual(service._pending_share_commit_floor, {}) - self.assertEqual(controller.snapshot()["active_writers"], {}) - - def test_synchronous_append_releases_floor_and_never_recounts_replays(self) -> None: - service, ledger, _controller, runtime = self._service() - service.share_writer_active = False - pending = self._stamp(service) - - service.append_accepted_share( - object(), - self._context(), - self._submission(), - pending, - ) - - self.assertEqual(len(ledger), 1) - self.assertEqual(service._pending_share_commit_floor, {}) - self.assertEqual(runtime.worker_notes, [("miner-a", None)]) - self.assertEqual(len(runtime.vardiff_notes), 1) - - # An exact ledger replay returns the original record; process-local - # worker/vardiff counters must not increment again. - service.append_accepted_share( - object(), - self._context(), - self._submission(), - pending, - ) - self.assertEqual(len(ledger), 1) - self.assertEqual(runtime.worker_notes, [("miner-a", None)]) - self.assertEqual(len(runtime.vardiff_notes), 1) - - def test_batch_success_sets_records_and_releases_every_waiter(self) -> None: - service, ledger, controller, _runtime = self._service(wall_times=[100, 200]) - first = self._entry(self._stamp(service, "aa" * 32)) - second = self._entry(self._stamp(service, "bb" * 32)) - for entry in (first, second): - entry.writer_token = controller.reserve_writer("share_persistence") - - self.assertTrue(service.append_share_batch([first, second])) - - self.assertEqual(len(ledger), 2) - for entry in (first, second): - self.assertTrue(entry.committed.is_set()) - self.assertIsNone(entry.error) - self.assertIsNotNone(entry.record) - self.assertIsNone(entry.writer_token) - self.assertEqual(service._pending_share_commit_floor, {}) - self.assertEqual(controller.snapshot()["active_writers"], {}) - - def test_batch_failure_marks_entries_and_still_releases_everything(self) -> None: - class FailingLedger: - def append_batch(self, _entries: object) -> list[object]: - raise RuntimeError("ledger unavailable") - - service, _ledger, controller, _runtime = self._service( - ledger=FailingLedger(), - wall_times=[100, 200], - ) - first = self._entry(self._stamp(service, "aa" * 32)) - second = self._entry(self._stamp(service, "bb" * 32)) - for entry in (first, second): - entry.writer_token = controller.reserve_writer("share_persistence") - - with patch("builtins.print"), patch("traceback.print_exc"): - self.assertFalse(service.append_share_batch([first, second])) - - self.assertEqual(service.share_append_failure_count, 2) - for entry in (first, second): - self.assertTrue(entry.committed.is_set()) - self.assertIsInstance(entry.error, RuntimeError) - self.assertIsNone(entry.writer_token) - self.assertEqual(service._pending_share_commit_floor, {}) - self.assertEqual(controller.snapshot()["active_writers"], {}) - - def test_append_loop_exits_only_when_admission_closed_and_writers_drained( - self, - ) -> None: - service, _ledger, controller, runtime = self._service() - runtime.stop_event.set() - exited = threading.Event() - - def run_loop() -> None: - service.share_append_loop() - exited.set() - - loop = threading.Thread(target=run_loop, daemon=True) - loop.start() - # Admission still open: the stopping loop keeps polling for late - # writer handoffs instead of abandoning queued shares. - self.assertFalse(exited.wait(0.4)) - - token = controller.reserve_writer("share_submission") - controller.begin_shutdown("test shutdown") - # Admission is closed but a submitting writer is still active. - self.assertFalse(exited.wait(0.4)) - - token.finish() - self.assertTrue(exited.wait(2.0)) - loop.join(timeout=2.0) - self.assertFalse(loop.is_alive()) - - # -- recovery journal and typed replay --------------------------------- - - def test_recovered_share_replays_inserted_and_unlinks_clean_journal(self) -> None: - with tempfile.TemporaryDirectory() as tempdir: - path = Path(tempdir) / "recovery.jsonl" - service, ledger, _controller, _runtime = self._service() - service.share_recovery_path = path - pending = self._stamp(service) - with patch("builtins.print"): - service.recover_share_to_disk(self._entry(pending), "test") - self.assertEqual(service.shares_recovered_to_disk, 1) - - with patch("builtins.print"): - self.assertEqual(service.replay_recovered_shares(), 1) - - self.assertFalse(path.exists()) - self.assertEqual(service.shares_replayed, 1) - self.assertEqual(len(ledger), 1) - self.assertEqual( - [record.share_id for record in ledger.all_shares()], - [pending.share_id], - ) - - def test_recovery_exact_existing_is_typed_skip_and_clears_clean_journal( - self, - ) -> None: - with tempfile.TemporaryDirectory() as tempdir: - path = Path(tempdir) / "recovery.jsonl" - service, ledger, _controller, _runtime = self._service() - service.share_recovery_path = path - pending = self._stamp(service) - with patch("builtins.print"): - service.recover_share_to_disk(self._entry(pending), "test") - replay = ledger.append_recovered_share(pending) - self.assertEqual(replay.disposition, "inserted") - - with patch("builtins.print"): - self.assertEqual(service.replay_recovered_shares(), 0) - - self.assertFalse(path.exists()) - self.assertEqual(len(ledger), 1) - - def test_recovery_payload_conflict_is_typed_and_retains_journal(self) -> None: - with tempfile.TemporaryDirectory() as tempdir: - path = Path(tempdir) / "recovery.jsonl" - service, ledger, _controller, _runtime = self._service() - service.share_recovery_path = path - pending = self._stamp(service) - with patch("builtins.print"): - service.recover_share_to_disk(self._entry(pending), "test") - ledger.append_recovered_share( - PendingShare(**{**pending.__dict__, "ntime": pending.ntime + 1}) - ) - - with patch("builtins.print"): - self.assertEqual(service.replay_recovered_shares(), 0) - - self.assertTrue(path.exists()) - self.assertEqual(service.shares_replayed, 0) - self.assertEqual(service.share_replay_conflicts, 1) - - def test_conflicting_row_is_quarantined_and_rest_of_journal_replays(self) -> None: - with tempfile.TemporaryDirectory() as tempdir: - path = Path(tempdir) / "recovery.jsonl" - service, ledger, _controller, _runtime = self._service() - service.share_recovery_path = path - conflicting = self._stamp(service, "ab" * 32) - later_a = self._stamp(service, "ac" * 32) - later_b = self._stamp(service, "ad" * 32) - with patch("builtins.print"): - service.recover_share_to_disk(self._entry(conflicting), "test") - service.recover_share_to_disk(self._entry(later_a), "test") - service.recover_share_to_disk(self._entry(later_b), "test") - # The durable copy of the first row disagrees with the journal - # payload, so its replay raises ShareReplayConflict. - ledger.append_recovered_share( - PendingShare( - **{**conflicting.__dict__, "ntime": conflicting.ntime + 1} - ) - ) - - with patch("builtins.print"): - self.assertEqual(service.replay_recovered_shares(), 2) - - # The rows behind the conflict are credited; the conflicting row - # is quarantined -- counted and preserved in the retained file -- - # instead of stranding the rest of the journal. - self.assertTrue(path.exists()) - self.assertEqual(service.shares_replayed, 2) - self.assertEqual(service.share_replay_conflicts, 1) - self.assertEqual(len(ledger), 3) - replayed_ids = {record.share_id for record in ledger.all_shares()} - self.assertIn(later_a.share_id, replayed_ids) - self.assertIn(later_b.share_id, replayed_ids) - - # Re-running the retained journal double-credits nothing: the - # replayed rows are exact duplicates now and the conflicting row - # conflicts again. - with patch("builtins.print"): - self.assertEqual(service.replay_recovered_shares(), 0) - - self.assertTrue(path.exists()) - self.assertEqual(service.shares_replayed, 2) - self.assertEqual(service.share_replay_conflicts, 2) - self.assertEqual(len(ledger), 3) - - def test_unknown_recovery_disposition_is_conservative_and_retains_journal( - self, - ) -> None: - class FutureLedger: - def append_recovered_share(self, pending: PendingShare) -> ShareReplayResult: - return ShareReplayResult( - "future-disposition", - SimpleNamespace(share_seq=1, share_id=pending.share_id), - ) - - with tempfile.TemporaryDirectory() as tempdir: - path = Path(tempdir) / "recovery.jsonl" - service, _ledger, _controller, _runtime = self._service( - ledger=FutureLedger(), - ) - service.share_recovery_path = path - pending = self._stamp(service) - with patch("builtins.print"): - service.recover_share_to_disk(self._entry(pending), "test") - - with patch("builtins.print"): - self.assertEqual(service.replay_recovered_shares(), 0) - - self.assertTrue(path.exists()) - - def test_untyped_append_only_ledger_keeps_legacy_duplicate_fallback(self) -> None: - class LegacyLedger: - def __init__(self) -> None: - self.appended: list[PendingShare] = [] - - def append(self, pending: PendingShare) -> SimpleNamespace: - for existing in self.appended: - if existing.share_id == pending.share_id: - raise RuntimeError( - f"duplicate share_id: {pending.share_id}" - ) - self.appended.append(pending) - return SimpleNamespace(share_seq=len(self.appended)) - - with tempfile.TemporaryDirectory() as tempdir: - path = Path(tempdir) / "recovery.jsonl" - ledger = LegacyLedger() - service, _ledger, _controller, _runtime = self._service(ledger=ledger) - service.share_recovery_path = path - first = self._stamp(service, "aa" * 32) - duplicate = self._stamp(service, "aa" * 32) - second = self._stamp(service, "bb" * 32) - with patch("builtins.print"): - for pending in (first, duplicate, second): - service.recover_share_to_disk(self._entry(pending), "test") - - with patch("builtins.print"): - self.assertEqual(service.replay_recovered_shares(), 2) - - self.assertFalse(path.exists()) - self.assertEqual( - [pending.share_id for pending in ledger.appended], - [first.share_id, second.share_id], - ) - - def test_torn_journal_line_is_preserved_while_intact_lines_replay(self) -> None: - with tempfile.TemporaryDirectory() as tempdir: - path = Path(tempdir) / "recovery.jsonl" - service, ledger, _controller, _runtime = self._service() - service.share_recovery_path = path - pending = self._stamp(service) - with patch("builtins.print"): - service.recover_share_to_disk(self._entry(pending), "test") - with open(path, "a", encoding="utf-8") as handle: - handle.write('{"share_id": "miner-a:torn", "miner_') - - with patch("builtins.print"), patch("traceback.print_exc"): - self.assertEqual(service.replay_recovered_shares(), 1) - - self.assertTrue(path.exists()) - self.assertEqual(len(ledger), 1) - - -if __name__ == "__main__": - unittest.main() diff --git a/tests/test_prism_shipped_image_lane.py b/tests/test_prism_shipped_image_lane.py new file mode 100644 index 000000000..62c5a3273 --- /dev/null +++ b/tests/test_prism_shipped_image_lane.py @@ -0,0 +1,455 @@ +"""Tests for scripts/prism_shipped_image_lane.py (#544, #487 L6).""" + +from __future__ import annotations + +import hashlib +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer +import io +import json +from pathlib import Path +import shutil +import sys +import tempfile +import threading +import time +import unittest +from unittest import mock + +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT / "scripts")) + +import prism_shipped_image_lane as lane # noqa: E402 + +# Bitcoin's genesis block: one transaction, so its merkle root is that txid. +GENESIS_COINBASE = ( + "01000000010000000000000000000000000000000000000000000000000000000000000000ffffffff4d04ffff001d" + "0104455468652054696d65732030332f4a616e2f32303039204368616e63656c6c6f72206f6e206272696e6b206f" + "66207365636f6e64206261696c6f757420666f722062616e6b73ffffffff0100f2052a01000000434104678afdb0fe" + "5548271967f1a67130b7105cd6a828e03909a67962e0ea1f61deb649f6bc3f4cef38c4f35504e51ec112de5c384df7" + "ba0b8d578a4c702b6bf11d5fac00000000" +) +GENESIS_HASH = "000000000019d6689c085ae165831e934ff763ae46a2a6c172b3f1b60a8ce26f" + + +def genesis_job(**overrides: object) -> dict: + job = { + "job_id": "1", + "prevhash": "00" * 32, + "coinb1": GENESIS_COINBASE[:20], + "coinb2": GENESIS_COINBASE[20:], + "branch": [], + "version": "00000001", + "nbits": "1d00ffff", + "ntime": "495fab29", + "clean": True, + } + job.update(overrides) + return job + + +class Header(unittest.TestCase): + def test_it_rebuilds_the_genesis_header_from_a_notify(self) -> None: + header = lane.block_header(genesis_job(), b"", b"", 0x7C2BAC1D) + self.assertEqual(len(header), 80) + self.assertEqual(lane.double_sha256(header)[::-1].hex(), GENESIS_HASH) + + def test_extranonces_sit_between_the_two_coinbase_halves(self) -> None: + split = genesis_job(coinb1=GENESIS_COINBASE[:16], coinb2=GENESIS_COINBASE[24:]) + extranonce1 = bytes.fromhex(GENESIS_COINBASE[16:20]) + extranonce2 = bytes.fromhex(GENESIS_COINBASE[20:24]) + header = lane.block_header(split, extranonce1, extranonce2, 0x7C2BAC1D) + self.assertEqual(lane.double_sha256(header)[::-1].hex(), GENESIS_HASH) + + def test_prevhash_words_are_byte_swapped_as_qbit_prism_miner_does(self) -> None: + internal = bytes(range(32)) + stratum = b"".join(internal[i : i + 4][::-1] for i in range(0, 32, 4)).hex() + header = lane.block_header(genesis_job(prevhash=stratum), b"", b"", 0) + self.assertEqual(header[4:36], internal) + + def test_the_merkle_branch_folds_on_the_right(self) -> None: + sibling = "11" * 32 + header = lane.block_header(genesis_job(branch=[sibling]), b"", b"", 0) + txid = lane.double_sha256(bytes.fromhex(GENESIS_COINBASE)) + self.assertEqual(header[36:68], lane.double_sha256(txid + bytes.fromhex(sibling))) + + +class Solving(unittest.TestCase): + def test_difficulty_one_is_the_diff1_target(self) -> None: + self.assertEqual(lane.difficulty_target(1.0), 0xFFFF << 208) + self.assertEqual(lane.difficulty_target(1e-30), (1 << 256) - 1) + + def test_a_solved_share_meets_half_the_share_target(self) -> None: + job = genesis_job(nbits="207fffff") + extranonce2, nonce = lane.solve_share(job, b"\x00\x00\x00\x01", 8, 1e-9) + header = lane.block_header(job, b"\x00\x00\x00\x01", bytes.fromhex(extranonce2), int(nonce, 16)) + self.assertEqual(len(bytes.fromhex(extranonce2)), 8) + self.assertLessEqual(int.from_bytes(lane.double_sha256(header), "little"), + lane.difficulty_target(1e-9) // 2) + + def test_notify_params_map_to_named_fields(self) -> None: + message = {"method": "mining.notify", + "params": ["j", "p", "c1", "c2", ["b"], "v", "n", "t", True]} + self.assertEqual(lane.notify_job(message), { + "job_id": "j", "prevhash": "p", "coinb1": "c1", "coinb2": "c2", "branch": ["b"], + "version": "v", "nbits": "n", "ntime": "t", "clean": True, + }) + + +class Refusals(unittest.TestCase): + def test_the_reason_id_names_a_refusal(self) -> None: + # As the lane's first run recorded them from the shipped images. + stale = {"id": 3, "result": None, "error": [21, "stale job", {"reason_id": "unknown-job"}]} + duplicate = {"id": 4, "result": None, "error": [22, "duplicate share", {"reason_id": "duplicate-share"}]} + self.assertEqual(lane.refusal_reason(stale), "unknown-job") + self.assertEqual(lane.refusal_reason(duplicate), "duplicate-share") + + def test_an_accepted_or_silent_answer_is_not_a_refusal(self) -> None: + self.assertIsNone(lane.refusal_reason({"id": 3, "result": True, "error": None})) + self.assertIsNone(lane.refusal_reason({"id": 3, "result": None, "error": None})) + + def test_a_reason_without_details_falls_back_to_the_message(self) -> None: + self.assertEqual(lane.refusal_reason({"result": False, "error": [23, "low difficulty"]}), "low difficulty") + + +class ClientOutput(unittest.TestCase): + def test_cpuminer_totals_come_from_its_last_result_line(self) -> None: + output = "\n".join([ + "[2026-09-29 10:00:00] 1 Submitted Diff 1e-09, Block 2, Job 1", + "[2026-09-29 10:00:00] 1 A1 S0 R0 BLOCK SOLVED 1, 0.001 sec (2ms)", + "[2026-09-29 10:00:01] 2 Submitted Diff 1e-09, Block 3, Job 2", + "[2026-09-29 10:00:01] 2 Accepted 2 S0 R0 B1, 0.900 sec (3ms)", + "[2026-09-29 10:00:02] 3 Submitted Diff 1e-09, Block 3, Job 2", + "[2026-09-29 10:00:02] 3 A2 Stale 1 R0 B1, 0.500 sec (3ms)", + "[2026-09-29 10:00:03] 4 Submitted Diff 1e-09, Block 4, Job 3", + "[2026-09-29 10:00:03] 4 A2 S1 Rejected 1 B1, 0.500 sec (3ms)", + "[2026-09-29 10:00:04] 5 Submitted Diff 1e-09, Block 4, Job 3", + ]) + tally = lane.parse_cpuminer(output) + self.assertEqual((tally.submitted, tally.accepted, tally.stale, tally.rejected, tally.blocks), + (5, 2, 1, 1, 1)) + self.assertEqual(tally.unanswered, 1) + + def test_cpuminer_with_no_results_has_nothing_accepted(self) -> None: + tally = lane.parse_cpuminer("[2026-09-29 10:00:00] Stratum connection failed\n") + self.assertEqual((tally.submitted, tally.accepted), (0, 0)) + + def test_prism_miner_totals_come_from_its_summary(self) -> None: + lines = [ + json.dumps({"event": "job", "job_id": "a"}), + json.dumps({"event": "submit", "block_target_met": True, "request_id": 10}), + json.dumps({"event": "share", "accepted": True}), + json.dumps({"event": "submit", "block_target_met": False, "request_id": 11}), + "2026-09-29T10:00:00Z INFO not json", + json.dumps({"event": "summary", "submitted": 3, "accepted": 1, "rejected": 1}), + ] + tally = lane.parse_prism_miner("\n".join(lines)) + self.assertEqual((tally.submitted, tally.accepted, tally.rejected, tally.blocks), (3, 1, 1, 1)) + self.assertEqual(tally.unanswered, 1) + + def test_prism_miner_without_a_summary_fails(self) -> None: + with self.assertRaises(lane.LaneFailure): + lane.parse_prism_miner(json.dumps({"event": "job"})) + + +class Reconciliation(unittest.TestCase): + def tally(self, submitted: int, accepted: int, rejected: int = 0) -> lane.ClientTally: + return lane.ClientTally(submitted=submitted, accepted=accepted, rejected=rejected) + + def test_every_acknowledged_share_in_the_ledger_reconciles(self) -> None: + self.assertEqual(lane.reconcile_client("c", self.tally(10, 9, 1), 9), []) + + def test_an_unanswered_submission_leaves_the_count_unprovable(self) -> None: + # One acknowledged share lost and one unanswered share committed + # would leave rows == acknowledged; counts cannot tell, so it fails. + problems = lane.reconcile_client("c", self.tally(12, 9, 1), 9) + self.assertEqual(len(problems), 1) + self.assertIn("2 submissions were unanswered", problems[0]) + + def test_a_missing_acknowledged_share_fails(self) -> None: + problems = lane.reconcile_client("c", self.tally(10, 9, 1), 8) + self.assertEqual(problems, ["c: 9 shares acknowledged but only 8 in the ledger"]) + + def test_rows_beyond_the_acknowledged_shares_fail(self) -> None: + problems = lane.reconcile_client("c", self.tally(10, 9, 1), 10) + self.assertEqual(problems, ["c: 10 ledger rows but only 9 shares acknowledged"]) + + def test_no_accepted_share_fails(self) -> None: + self.assertIn("c: no share was accepted", lane.reconcile_client("c", self.tally(0, 0), 0)) + + +class Wiring(unittest.TestCase): + def test_the_ha_overlay_is_applied_last_and_the_lane_env_file_wins(self) -> None: + command = lane.compose_command("p", Path("/w/lane.env"), "up", "-d") + self.assertEqual(command[:4], ["docker", "compose", "--project-name", "p"]) + files = [command[i + 1] for i, arg in enumerate(command) if arg == "-f"] + self.assertEqual(files, ["compose.yaml", "compose.prism-ha.yaml"]) + env_files = [command[i + 1] for i, arg in enumerate(command) if arg == "--env-file"] + self.assertEqual(env_files, [lane.UPSTREAM_ENV_FILE, ".env.example", "/w/lane.env"]) + self.assertEqual(command[-4:], ["--profile", "prism", "up", "-d"]) + + def test_the_upstream_pins_are_the_ones_prepare_qbit_source_reads(self) -> None: + # prepare-qbit-source.sh and the Makefile prefer config/upstream.env. + self.assertTrue((ROOT / "config/upstream.env").is_file()) + self.assertEqual(lane.UPSTREAM_ENV_FILE, "config/upstream.env") + prepare = (ROOT / "scripts/prepare-qbit-source.sh").read_text(encoding="utf-8") + self.assertIn('if [[ -f "${ROOT_DIR}/config/upstream.env" ]]; then', prepare) + + def test_every_compose_file_and_env_file_exists(self) -> None: + for name in (*lane.COMPOSE_FILES, *lane.ENV_FILES): + self.assertTrue((ROOT / name).is_file(), name) + + def test_the_lane_env_carries_the_seeds_and_instance_ids(self) -> None: + text = lane.lane_env(Path("/src/qbit"), "11" * 32, "22" * 32, "ab" * 32) + values = dict(line.split("=", 1) for line in text.splitlines() if not line.startswith("#")) + self.assertEqual(values["QBIT_SRC_DIR"], "/src/qbit") + self.assertEqual(values["PRISM_LEDGER_ATTESTATION_SIGNING_SEED_HEX"], "22" * 32) + self.assertEqual(values["PRISM_LEDGER_WRITER_PUBLIC_KEY_HEX"], "ab" * 32) + self.assertEqual(values["PRISM_PUBLIC_STRATUM_URL"], "stratum+tcp://127.0.0.1:3340") + self.assertEqual([values["PRISM_HA_INSTANCE_ID_1"], values["PRISM_HA_INSTANCE_ID_2"]], + list(lane.INSTANCE_IDS)) + + def test_the_lane_uses_the_overlays_default_host_ports(self) -> None: + overlay = (ROOT / "compose.prism-ha.yaml").read_text(encoding="utf-8") + self.assertIn("PRISM_HA_STRATUM_PORT_HOST_2:-3343}", overlay) + self.assertIn("PRISM_HA_HEALTH_PORT_HOST_1:-127.0.0.1:3341}", overlay) + self.assertIn("PRISM_HA_HEALTH_PORT_HOST_2:-127.0.0.1:3344}", overlay) + self.assertEqual((lane.STRATUM_PORTS, lane.HEALTH_PORTS), ((3340, 3343), (3341, 3344))) + + def test_the_checks_are_unique_kebab_names(self) -> None: + self.assertEqual(len(set(lane.CHECKS)), len(lane.CHECKS)) + for name in lane.CHECKS: + self.assertRegex(name, r"^[a-z0-9][a-z0-9-]*$") + + @unittest.skipUnless(shutil.which("openssl"), "openssl is not installed") + def test_the_writer_key_matches_qbit_pool_builder(self) -> None: + # `cargo run -p qbit-pool-builder -- --signing-key-seed-hex 22...22 + # --print-public-key-hex` prints this key. + self.assertEqual(lane.ed25519_public_key_hex("22" * 32), + "a09aa5f47a6759802ff955f8dc2d2a14a5c99d23be97f864127ff9383455a4f0") + + def test_a_short_seed_is_refused(self) -> None: + with self.assertRaises(ValueError): + lane.ed25519_public_key_hex("22" * 31) + + def test_the_address_guard_refuses_sql_metacharacters(self) -> None: + self.assertIsNotNone(lane.ADDRESS.match("qbrt1" + "q" * 58)) + self.assertIsNone(lane.ADDRESS.match("qbrt1q' OR '1'='1")) + + +class FakeQbitd: + """Just enough of qbitd's JSON-RPC for the lane's node phase (#637). + + createwallet is held until `release` is set: the node is still deriving + the wallet's keys. Holding it on an event, not a sleep, decides the race + with the client timeout. `on_listwallets` runs on each listwallets call + and may finish the wallet; listwallets shows a wallet only once it is + finished, as qbitd adds it to its list after creating and loading it. + """ + + ADDRESS = "qbrt1q" + "q" * 58 + + def __init__(self) -> None: + self.calls: list[str] = [] + self.wallets: list[str] = [] + self.release = threading.Event() + self.on_listwallets = lambda count: None + self.createwallet_error: dict | None = None + fake = self + + class Handler(BaseHTTPRequestHandler): + def log_message(self, *args) -> None: + pass + + def do_POST(self) -> None: + request = json.loads(self.rfile.read(int(self.headers["Content-Length"]))) + method, params = request["method"], request["params"] + fake.calls.append(method) + error = None + if method == "createwallet": + # Bounded so a test that never releases cannot hang. + fake.release.wait(timeout=60) + error = fake.createwallet_error + if error is None: + fake.wallets.append(params[0]) + result = None if error else {"name": params[0]} + elif method == "listwallets": + fake.on_listwallets(fake.calls.count("listwallets")) + result = list(fake.wallets) + elif method == "getnewaddress": + result = fake.ADDRESS + else: + result = {"method": method} + body = json.dumps({"result": result, "error": error, "id": request["id"]}).encode() + try: + self.send_response(500 if error else 200) + self.send_header("Content-Type", "application/json") + self.send_header("Content-Length", str(len(body))) + self.end_headers() + self.wfile.write(body) + except OSError: + pass # The client gave up on this call. + + self.server = ThreadingHTTPServer(("127.0.0.1", 0), Handler) + self.server.daemon_threads = True + self.thread = threading.Thread(target=self.server.serve_forever, daemon=True) + + def finish_wallet(self) -> None: + self.release.set() + # The held createwallet has added the wallet once it returns from the wait. + deadline = time.monotonic() + 10 + while not self.wallets and self.createwallet_error is None and time.monotonic() < deadline: + time.sleep(0.01) + + def __enter__(self) -> "FakeQbitd": + self.thread.start() + self.port_patch = mock.patch.object(lane, "QBIT_RPC_PORT", self.server.server_address[1]) + self.port_patch.start() + return self + + def __exit__(self, *exc) -> None: + self.release.set() + self.port_patch.stop() + self.server.shutdown() + self.server.server_close() + + +class WalletCreation(unittest.TestCase): + """#637: createwallet outlasting its client timeout must not fail the lane.""" + + def setUp(self) -> None: + # A short client timeout stands in for the lane's 30 s; create=True + # lets the same test run against the driver before the fix. + patch = mock.patch.object(lane, "RPC_TIMEOUT", 0.3, create=True) + patch.start() + self.addCleanup(patch.stop) + self.qbitd = FakeQbitd().__enter__() + self.addCleanup(self.qbitd.__exit__, None, None, None) + + def test_a_node_phase_whose_createwallet_outlasts_the_client_timeout_waits_for_the_wallet(self) -> None: + # The node finishes the wallet after the client has given up, as in + # run 36786483193: still loading at the first poll, loaded by the second. + self.qbitd.on_listwallets = lambda count: count == 2 and self.qbitd.finish_wallet() + with tempfile.TemporaryDirectory() as work: + (Path(work) / "lane.env").write_text("", encoding="utf-8") + driver = lane.Lane(work=Path(work), out=Path(work), project="p", load_seconds=1, + cpuminer_threads=1, cpuminer_diff_multiplier=1.0, + prism_miner_hashes_per_second=1) + with mock.patch.object(lane.Lane, "compose"): + addresses = driver.start() + self.assertEqual(addresses["bootstrap"], FakeQbitd.ADDRESS) + self.assertEqual(self.qbitd.calls.count("createwallet"), 1) + self.assertEqual(self.qbitd.calls.count("listwallets"), 2) + self.assertEqual(driver.report["wallet"]["outcome"], "loaded after createwallet timed out") + self.assertIn("node", driver.report["phases"]) + + def test_a_wallet_created_within_the_timeout_is_not_polled_for(self) -> None: + self.qbitd.release.set() + self.assertEqual(lane.create_wallet(lane.Rpc(), "l6", seconds=5)["outcome"], "created") + self.assertEqual(self.qbitd.calls, ["createwallet"]) + + def test_a_wallet_that_never_loads_fails_within_the_one_deadline(self) -> None: + started = time.monotonic() + with self.assertRaises(lane.LaneFailure) as caught: + lane.create_wallet(lane.Rpc(), "l6", seconds=1.5) + elapsed = time.monotonic() - started + self.assertIn("waiting for the node to load wallet 'l6' after createwallet timed out", str(caught.exception)) + self.assertIn("listwallets shows []", str(caught.exception)) + # The client timeout and the wait share the 1.5 s, plus at most one poll interval. + self.assertLess(elapsed, 3.0) + self.assertEqual(self.qbitd.calls.count("createwallet"), 1) + + def test_a_createwallet_error_fails_at_once_without_polling(self) -> None: + self.qbitd.createwallet_error = {"code": -4, "message": "Wallet file verification failed."} + self.qbitd.release.set() + with self.assertRaisesRegex(lane.LaneFailure, "qbit RPC createwallet failed: .*verification"): + lane.create_wallet(lane.Rpc(), "l6", seconds=5) + self.assertNotIn("listwallets", self.qbitd.calls) + + def test_an_rpc_timeout_names_the_method(self) -> None: + with self.assertRaisesRegex(lane.RpcTimeout, r"^qbit RPC createwallet timed out after 0\.3s$"): + lane.Rpc().call("createwallet", ["l6"]) + + +class Summary(unittest.TestCase): + def report(self, **fields) -> dict: + return {"checks": {}, "phases": {"total": 44.1}, **fields} + + def summary(self, report: dict) -> str: + with mock.patch("sys.stdout", io.StringIO()) as out, mock.patch.dict("os.environ", {}, clear=True): + lane.write_summary(report) + return out.getvalue() + + def test_an_error_names_the_phase_it_stopped(self) -> None: + text = self.summary(self.report(error="RpcTimeout: qbit RPC listwallets timed out after 30s", + failed_phase="node", failed_phase_seconds=36.2)) + self.assertIn("Error in the `node` phase after 36.2s: `RpcTimeout: qbit RPC listwallets timed out", text) + self.assertIn("| `ha-frontends-ready` | not reached |", text) + + def test_an_error_before_any_phase_has_no_phase(self) -> None: + self.assertIn("Error: `OSError: x`", self.summary(self.report(error="OSError: x", failed_phase=None))) + + def test_execute_records_the_phase_that_failed(self) -> None: + with tempfile.TemporaryDirectory() as work: + (Path(work) / "lane.env").write_text("", encoding="utf-8") + driver = lane.Lane(work=Path(work), out=Path(work), project="p", load_seconds=1, + cpuminer_threads=1, cpuminer_diff_multiplier=1.0, + prism_miner_hashes_per_second=1) + + def start() -> dict: + driver.begin("node") + raise lane.RpcTimeout("qbit RPC createwallet timed out after 30s") + + with mock.patch.object(driver, "start", start), mock.patch.object(driver, "collect"), \ + mock.patch.object(driver, "down"), mock.patch("sys.stdout", io.StringIO()), \ + mock.patch.dict("os.environ", {}, clear=True): + self.assertFalse(driver.execute()) + report = json.loads((Path(work) / "l6-report.json").read_text(encoding="utf-8")) + self.assertEqual(report["failed_phase"], "node") + self.assertEqual(report["error"], "RpcTimeout: qbit RPC createwallet timed out after 30s") + self.assertIn("failed_phase_seconds", report) + self.assertNotIn("node", report["phases"]) + + +class Triggers(unittest.TestCase): + def test_dockerfiles_compose_files_and_image_inputs_run_the_lane(self) -> None: + for path in ("docker/qbit/Dockerfile", "lab/miner-sim/Dockerfile", "Dockerfile", + "compose.yaml", "compose.prism-ha.yaml", ".env.example", + "config/prism-postgres/replica-entrypoint.sh", "docker/qbit/qbit-entrypoint.sh", + "lab/real-miner/real_miner.py", "docker/real-miner/cpuminer-regtest.patch", + "scripts/prism_shipped_image_lane.py", ".github/workflows/prism-load-nightly.yml", + "config/upstream.env", "config/upstream.env.example"): + with self.subTest(path=path): + self.assertTrue(lane.watched([path])) + + def test_other_changes_do_not(self) -> None: + for path in ("crates/qbit-prism-server/src/lib.rs", "docs/compose.yaml", "README.md", + "tests/test_prism_shipped_image_lane.py", "lab/prism-notes.md", ""): + with self.subTest(path=path): + self.assertFalse(lane.watched([path])) + + def test_one_watched_path_among_many_is_enough(self) -> None: + self.assertTrue(lane.watched(["README.md\n", "compose.production.yaml\n"])) + + def test_the_changed_command_prints_the_verdict(self) -> None: + import subprocess + script = ROOT / "scripts" / "prism_shipped_image_lane.py" + for stdin, verdict in (("README.md\nlab/prism/Dockerfile\n", "true"), ("README.md\n", "false")): + result = subprocess.run([sys.executable, str(script), "changed"], input=stdin, + text=True, capture_output=True, check=True) + self.assertEqual(result.stdout.strip(), verdict) + + def test_the_workflow_gates_the_lane_on_the_changes_job(self) -> None: + workflow = (ROOT / ".github/workflows/prism-load-nightly.yml").read_text(encoding="utf-8") + self.assertIn("python3 scripts/prism_shipped_image_lane.py changed", workflow) + self.assertIn("needs.changes.outputs.images == 'true'", workflow) + self.assertIn("github.base_ref == '3.x.x'", workflow) + # A change list at the API's cap may be truncated, so it runs the lane. + self.assertIn("cap=300", workflow) + self.assertIn("cap=3000", workflow) + self.assertIn("(( count >= cap ))", workflow) + # The load nightly's guard must not start the preset matrix on a push. + self.assertIn("github.event_name != 'push'", workflow) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_prism_singleflight_exception_retention.py b/tests/test_prism_singleflight_exception_retention.py deleted file mode 100644 index f64efb62e..000000000 --- a/tests/test_prism_singleflight_exception_retention.py +++ /dev/null @@ -1,571 +0,0 @@ -"""Failed singleflight owners must not pin their flight until cyclic GC (#251). - -``PublicResponseCache`` and ``P2mrAddressValidator`` record an owner's failure -on the in-flight entry so coalesced waiters can re-raise it. The recorded -exception carries a traceback, the traceback holds the owner's frame, and a -frame that still names the flight closes the cycle -``flight -> exception -> traceback -> frame -> flight``. Popping the flight -from its map removes the only external reference, but nothing in that cycle -is freed by reference counting: the failed origin/RPC frame and every local it -holds stay allocated until a cyclic collection happens to run. - -These tests exercise the real owner methods with automatic collection -disabled, so an object that is still alive when asserted is being held by a -cycle, not by a pending collection. Weak references are the only handles -kept: no ``f_locals`` or referrer inspection, which would itself retain the -frames under test. Threads are ordered with events and barriers, never -sleeps, and every wait is bounded. Test-only GC policy is restored on exit. -""" - -from __future__ import annotations - -import gc -import threading -import unittest -import weakref -from collections.abc import Callable, Iterator -from contextlib import contextmanager -from typing import Any - -from lab.prism.public_api import PublicResponseCache, _PublicInflight -from lab.prism.stratum_session import P2mrAddressValidator, StratumError - -CACHE_KEY: tuple[str, tuple[tuple[str, tuple[str, ...]], ...]] = ("/issue-251", ()) -ADDRESS = "qbit1issue251payoutaddress" -P2MR_SCRIPT = "5220" + "ab" * 32 -WAITER_COUNT = 3 -WAIT_SECONDS = 5.0 - - -class Payload: - """Weak-referenceable stand-in for a failed origin call's locals.""" - - -@contextmanager -def _without_cyclic_gc() -> Iterator[None]: - """Leave only reference counting to free objects inside the block. - - Assertions inside run before any ``gc.collect()``; whatever is still alive - there is held by a reference cycle. The prior policy is restored on exit, - after one full collection reclaims anything a failing case left behind. - """ - gc.collect() - was_enabled = gc.isenabled() - gc.disable() - try: - yield - finally: - gc.collect() - if was_enabled: - gc.enable() - - -def _traceback_names(error: BaseException) -> list[str]: - """Function names along ``error.__traceback__``, outermost first. - - Reads code objects only, never ``f_locals``, so checking that the - diagnostics survived does not itself keep the frames alive. - """ - names: list[str] = [] - traceback = error.__traceback__ - while traceback is not None: - names.append(traceback.tb_frame.f_code.co_name) - traceback = traceback.tb_next - return names - - -class _Origin: - """A ``compute()`` / ``rpc_call`` that creates a weak-referenceable local. - - With ``park=True`` the call blocks until ``release`` is set, so concurrent - requests can register as waiters while the owner is in flight. Failing - calls raise after creating the local; successful calls return ``success``. - """ - - def __init__(self, *, park: bool = False, success: object | None = None) -> None: - self.park = park - self.success = success - self.calls = 0 - self.entered = threading.Event() - self.release = threading.Event() - self.payloads: list[weakref.ReferenceType[Payload]] = [] - self._lock = threading.Lock() - - def __call__(self, *args: object) -> object: - payload = Payload() - with self._lock: - self.calls += 1 - self.payloads.append(weakref.ref(payload)) - self.entered.set() - if self.park and not self.release.wait(timeout=WAIT_SECONDS): - raise TimeoutError("test did not release the parked origin call") - if self.success is not None: - return self.success - raise RuntimeError("injected origin failure") - - -class _GatedEvent(threading.Event): - """Flight event whose waiters announce themselves and can be held. - - A waiter reaching the singleflight's ``event.wait()`` joins ``arrived`` - (the test thread is the extra party) and then blocks on ``proceed`` before - entering the real wait. The test therefore knows when every waiter is - inside the owner method and decides whether the owner finishes before or - after they observe the recorded result. - - The real wait is this class's own condition variable rather than the - inherited one, so the guarantee below depends on nothing private to the - standard library. Each waiter counts itself in ``all_inside`` while - holding the condition's lock and only releases that lock by entering - ``Condition.wait``; ``set()`` needs the same lock. Once ``all_inside`` is - set, every waiter is therefore blocked in the real wait before any - ``set()`` can complete. Waits are bounded: a flight that is never set - fails the test instead of hanging it. - """ - - def __init__(self, *, waiters: int) -> None: - super().__init__() - self.arrived = threading.Barrier(waiters + 1) - self.proceed = threading.Event() - self.all_inside = threading.Event() - self._expected_waiters = waiters - self._waiting = 0 - self._set = False - self._condition = threading.Condition() - - def is_set(self) -> bool: - with self._condition: - return self._set - - def set(self) -> None: - with self._condition: - self._set = True - self._condition.notify_all() - - def clear(self) -> None: - with self._condition: - self._set = False - - def wait(self, timeout: float | None = None) -> bool: - # The owner methods wait without a timeout; bound it here so a - # regression that forgets to wake waiters fails this call instead of - # leaving non-daemon test threads blocked after the join times out. - bound = WAIT_SECONDS if timeout is None else min(timeout, WAIT_SECONDS) - self.arrived.wait(timeout=WAIT_SECONDS) - if not self.proceed.wait(timeout=WAIT_SECONDS): - raise TimeoutError("test did not release the gated waiters") - with self._condition: - self._waiting += 1 - if self._waiting == self._expected_waiters: - self.all_inside.set() - if not self._set and not self._condition.wait(timeout=bound): - raise TimeoutError(f"the flight was not set within {bound}s") - return True - - -class _InflightMap(dict[Any, Any]): - """Inflight map that weakly records each registered flight. - - When ``event`` is given, the flight's event is swapped for it at - registration, which is the only point where the owner method exposes the - flight it just created. - """ - - def __init__(self, *, event: threading.Event | None = None) -> None: - super().__init__() - self.event = event - self.flights: list[weakref.ReferenceType[Any]] = [] - - def __setitem__(self, key: Any, flight: Any) -> None: - if self.event is not None: - flight.event = self.event - self.flights.append(weakref.ref(flight)) - super().__setitem__(key, flight) - - -class _ReleaseAssertions(unittest.TestCase): - def assert_released(self, refs: list[weakref.ReferenceType[Any]], what: str) -> None: - self.assertTrue(refs, f"no {what} was ever registered") - self.assertEqual( - [None] * len(refs), - [ref() for ref in refs], - f"{what} still alive without a cyclic collection", - ) - - def observe_failure(self, call: Callable[[], object]) -> list[str]: - """Run ``call``, require the injected failure, and return its frames. - - The exception is dropped before returning, exactly as a request - handler that logged and answered the error would drop it. - """ - try: - call() - except RuntimeError as error: - self.assertEqual(str(error), "injected origin failure") - return _traceback_names(error) - self.fail("expected the injected origin failure") - - def assert_shared_failure( - self, - outcomes: list[tuple[str, BaseException | None]], - *, - chained: bool, - ) -> None: - """Check every thread saw the injected failure with diagnostics intact. - - Runs in its own frame so no local keeps an exception, and with it the - failed origin frame, alive once the caller clears ``outcomes``. - """ - leader_errors = [error for name, error in outcomes if name == "owner"] - waiter_errors = [error for name, error in outcomes if name != "owner"] - self.assertEqual(len(leader_errors), 1) - self.assertEqual(len(waiter_errors), WAITER_COUNT) - leader_error = leader_errors[0] - self.assertIsInstance(leader_error, RuntimeError) - self.assertEqual(str(leader_error), "injected origin failure") - self.assertEqual(_traceback_names(leader_error)[-1], "__call__") - for waiter_error in waiter_errors: - self.assertIsInstance(waiter_error, RuntimeError) - self.assertEqual(str(waiter_error), "injected origin failure") - if chained: - # The documented wrapper: a fresh error chained to the leader's - # original, which still carries the RPC frame. - self.assertIs(waiter_error.__cause__, leader_error) - else: - # The owner's own exception object, re-raised by each waiter; - # its traceback grew by the waiter frames and kept the origin. - self.assertIs(waiter_error, leader_error) - - def run_concurrently( - self, - request: Callable[[], object], - *, - origin: _Origin, - gate: _GatedEvent, - owner_finishes_first: bool, - flight_released: Callable[[], None], - waiters_arrived: Callable[[], None] | None = None, - ) -> list[tuple[str, BaseException | None]]: - """Run one owner and WAITER_COUNT coalesced waiters through ``request``. - - Returns ``(thread name, error)`` per thread, ``None`` for a normal - return. The owner is parked inside the origin call until every waiter - has entered the flight's ``event.wait()``, when ``waiters_arrived`` - runs. With ``owner_finishes_first`` the owner completes, drops out of - the inflight map (checked by ``flight_released``), and unwinds - entirely before any waiter reads the recorded outcome. Otherwise the - owner is released only after every waiter is blocked in the real - wait, which the gate guarantees rather than merely makes likely. - """ - outcomes: list[tuple[str, BaseException | None]] = [] - outcomes_lock = threading.Lock() - - def run() -> None: - error: BaseException | None = None - try: - request() - except BaseException as exc: - error = exc - with outcomes_lock: - outcomes.append((threading.current_thread().name, error)) - del error - - owner = threading.Thread(target=run, name="owner") - waiters = [ - threading.Thread(target=run, name=f"waiter-{index}") for index in range(WAITER_COUNT) - ] - started: list[threading.Thread] = [] - try: - owner.start() - started.append(owner) - self.assertTrue(origin.entered.wait(timeout=WAIT_SECONDS)) - for waiter in waiters: - waiter.start() - started.append(waiter) - gate.arrived.wait(timeout=WAIT_SECONDS) - if waiters_arrived is not None: - waiters_arrived() - if owner_finishes_first: - origin.release.set() - owner.join(timeout=WAIT_SECONDS) - self.assertFalse(owner.is_alive()) - flight_released() - gate.proceed.set() - else: - gate.proceed.set() - self.assertTrue(gate.all_inside.wait(timeout=WAIT_SECONDS)) - origin.release.set() - finally: - # Unblock whatever is still parked, then join only the threads - # that were actually started so an earlier assertion failure is - # reported as itself rather than masked by joining unstarted ones. - origin.release.set() - gate.proceed.set() - for thread in started: - thread.join(timeout=WAIT_SECONDS) - self.assertEqual(len(started), 1 + WAITER_COUNT) - self.assertFalse(any(thread.is_alive() for thread in started)) - self.assertEqual(origin.calls, 1) - self.assertEqual(len(outcomes), 1 + WAITER_COUNT) - return outcomes - - -class PublicResponseCacheFailedFlightTests(_ReleaseAssertions): - """Foreground owners, background refreshes, and waiters release the flight.""" - - def lookup(self, cache: PublicResponseCache, compute: Callable[[], Any]) -> Any: - return cache.get_or_compute(key=CACHE_KEY, ttl_seconds=60, compute=compute) - - def test_foreground_owner_failure_releases_flight_and_compute_frame(self) -> None: - origin = _Origin() - with _without_cyclic_gc(): - cache = PublicResponseCache() - flights = _InflightMap() - cache._inflight = flights - - frames = self.observe_failure(lambda: self.lookup(cache, origin)) - - self.assertNotIn(CACHE_KEY, cache._inflight) - self.assertEqual(origin.calls, 1) - # The owner re-raised its own exception with the origin frame in - # the traceback; the diagnostics were not trimmed to break the cycle. - self.assertEqual(frames[-2:], ["get_or_compute", "__call__"]) - self.assert_released(flights.flights, "foreground flight") - self.assert_released(origin.payloads, "failed compute frame local") - - def test_background_refresh_failure_releases_flight_and_compute_frame(self) -> None: - origin = _Origin() - with _without_cyclic_gc(): - cache = PublicResponseCache() - flights = _InflightMap() - cache._inflight = flights - flight = _PublicInflight(event=threading.Event()) - cache._inflight[CACHE_KEY] = flight - - cache._refresh_entry(CACHE_KEY, 60, 30, origin, flight) - - self.assertNotIn(CACHE_KEY, cache._inflight) - self.assertTrue(flight.event.is_set()) - self.assertIsInstance(flight.exception, RuntimeError) - self.assertIsNone(flight.result) - # The swallowed error keeps its origin frame for whoever coalesced. - self.assertEqual( - _traceback_names(flight.exception)[-2:], ["_refresh_entry", "__call__"] - ) - del flight - self.assert_released(flights.flights, "background refresh flight") - self.assert_released(origin.payloads, "failed refresh compute frame local") - - def test_stale_hit_refresh_thread_failure_releases_flight_and_keeps_entry(self) -> None: - origin = _Origin() - with _without_cyclic_gc(): - cache = PublicResponseCache() - flights = _InflightMap() - cache._inflight = flights - status, payload, state, _age = cache.get_or_compute( - key=CACHE_KEY, ttl_seconds=60, compute=lambda: (200, {"v": "stale"}) - ) - self.assertEqual((200, {"v": "stale"}, "MISS"), (status, payload, state)) - entry = cache._entries[CACHE_KEY] - entry.expires_at = entry.stored_at - 1 # expired, inside the window below - entry.stale_until = entry.expires_at + 30 - del entry - - status, payload, state, _age = cache.get_or_compute( - key=CACHE_KEY, - ttl_seconds=60, - compute=origin, - stale_while_revalidate_seconds=30, - ) - - self.assertEqual((200, {"v": "stale"}, "STALE"), (status, payload, state)) - self.assertTrue(origin.entered.wait(timeout=WAIT_SECONDS)) - refreshers = [ - thread - for thread in threading.enumerate() - if thread.name == "prism-public-cache-refresh" - ] - for thread in refreshers: - thread.join(timeout=WAIT_SECONDS) - self.assertFalse(any(thread.is_alive() for thread in refreshers)) - self.assertNotIn(CACHE_KEY, cache._inflight) - self.assertIn(CACHE_KEY, cache._entries) # the failure deleted nothing - self.assertEqual(origin.calls, 1) - self.assert_released(flights.flights, "stale refresh flight") - self.assert_released(origin.payloads, "failed refresh compute frame local") - - def assert_waiters_release_failed_flight(self, *, owner_finishes_first: bool) -> None: - origin = _Origin(park=True) - gate = _GatedEvent(waiters=WAITER_COUNT) - with _without_cyclic_gc(): - cache = PublicResponseCache() - flights = _InflightMap(event=gate) - cache._inflight = flights - - outcomes = self.run_concurrently( - lambda: self.lookup(cache, origin), - origin=origin, - gate=gate, - owner_finishes_first=owner_finishes_first, - flight_released=lambda: self.assertNotIn(CACHE_KEY, cache._inflight), - ) - - self.assertNotIn(CACHE_KEY, cache._inflight) - self.assert_shared_failure(outcomes, chained=False) - outcomes.clear() - self.assert_released(flights.flights, "shared flight") - self.assert_released(origin.payloads, "failed compute frame local") - - def test_waiters_release_flight_when_owner_fails_while_they_wait(self) -> None: - self.assert_waiters_release_failed_flight(owner_finishes_first=False) - - def test_waiters_release_flight_when_owner_failed_before_they_observe(self) -> None: - self.assert_waiters_release_failed_flight(owner_finishes_first=True) - - def test_successful_flight_is_released_after_the_last_observer(self) -> None: - origin = _Origin(park=True, success=(200, {"v": "fresh"})) - gate = _GatedEvent(waiters=WAITER_COUNT) - with _without_cyclic_gc(): - cache = PublicResponseCache() - flights = _InflightMap(event=gate) - cache._inflight = flights - results: list[Any] = [] - results_lock = threading.Lock() - - def request() -> None: - result = self.lookup(cache, origin) - with results_lock: - results.append(result) - - outcomes = self.run_concurrently( - request, - origin=origin, - gate=gate, - owner_finishes_first=True, - flight_released=lambda: self.assertNotIn(CACHE_KEY, cache._inflight), - ) - - self.assertEqual([error for _name, error in outcomes], [None] * (1 + WAITER_COUNT)) - self.assertEqual(len(results), 1 + WAITER_COUNT) - self.assertEqual( - {(status, state) for status, _payload, state, _age in results}, {(200, "MISS")} - ) - self.assertEqual(cache._entries[CACHE_KEY].payload, {"v": "fresh"}) - self.assert_released(flights.flights, "successful flight") - - -class P2mrAddressValidatorFailedFlightTests(_ReleaseAssertions): - """The validation leader and its waiters release the pending flight.""" - - def validator(self, rpc_call: Callable[..., object], flights: _InflightMap) -> P2mrAddressValidator: - return P2mrAddressValidator( - rpc_call=rpc_call, - max_entries=lambda: 4, - ttl_seconds=lambda: 60.0, - inflight=flights, - ) - - def test_leader_rpc_failure_releases_flight_and_rpc_frame(self) -> None: - origin = _Origin() - with _without_cyclic_gc(): - flights = _InflightMap() - validator = self.validator(origin, flights) - - frames = self.observe_failure(lambda: validator.validate(ADDRESS, label="test")) - - self.assertEqual(validator.inflight, {}) - self.assertEqual(validator.cache, {}) - self.assertEqual(origin.calls, 1) - self.assertEqual(frames[-2:], ["validate", "__call__"]) - self.assert_released(flights.flights, "validation flight") - self.assert_released(origin.payloads, "failed RPC frame local") - - def test_leader_validation_failure_releases_flight(self) -> None: - with _without_cyclic_gc(): - flights = _InflightMap() - validator = self.validator(lambda *_args: {"isvalid": False}, flights) - - try: - validator.validate(ADDRESS, label="payout") - except StratumError as error: - self.assertEqual(error.code, 20) - self.assertIn("payout is not a valid qbit address", error.message) - else: - self.fail("expected the address to be rejected") - - self.assertEqual(validator.inflight, {}) - self.assert_released(flights.flights, "rejected validation flight") - - def assert_waiters_release_failed_flight(self, *, owner_finishes_first: bool) -> None: - origin = _Origin(park=True) - gate = _GatedEvent(waiters=WAITER_COUNT) - with _without_cyclic_gc(): - flights = _InflightMap(event=gate) - validator = self.validator(origin, flights) - - def flight_released() -> None: - with validator.cache_lock: - self.assertNotIn(ADDRESS, validator.inflight) - - def waiters_counted() -> None: - # The existing accounting contract: every coalesced request - # is counted on the leader's flight while it is in flight. - with validator.cache_lock: - self.assertEqual(validator.inflight[ADDRESS].waiters, WAITER_COUNT) - - outcomes = self.run_concurrently( - lambda: validator.validate(ADDRESS, label="test"), - origin=origin, - gate=gate, - owner_finishes_first=owner_finishes_first, - flight_released=flight_released, - waiters_arrived=waiters_counted, - ) - - self.assertEqual(validator.inflight, {}) - self.assertEqual(validator.cache, {}) - self.assert_shared_failure(outcomes, chained=True) - outcomes.clear() - self.assert_released(flights.flights, "shared validation flight") - self.assert_released(origin.payloads, "failed RPC frame local") - - def test_waiters_release_flight_when_leader_fails_while_they_wait(self) -> None: - self.assert_waiters_release_failed_flight(owner_finishes_first=False) - - def test_waiters_release_flight_when_leader_failed_before_they_observe(self) -> None: - self.assert_waiters_release_failed_flight(owner_finishes_first=True) - - def test_successful_flight_is_released_after_the_last_observer(self) -> None: - origin = _Origin(park=True, success={"isvalid": True, "scriptPubKey": P2MR_SCRIPT}) - gate = _GatedEvent(waiters=WAITER_COUNT) - with _without_cyclic_gc(): - flights = _InflightMap(event=gate) - validator = self.validator(origin, flights) - results: list[tuple[str, str]] = [] - results_lock = threading.Lock() - - def request() -> None: - result = validator.validate(ADDRESS, label="test") - with results_lock: - results.append(result) - - def flight_released() -> None: - with validator.cache_lock: - self.assertNotIn(ADDRESS, validator.inflight) - - outcomes = self.run_concurrently( - request, - origin=origin, - gate=gate, - owner_finishes_first=True, - flight_released=flight_released, - ) - - self.assertEqual([error for _name, error in outcomes], [None] * (1 + WAITER_COUNT)) - self.assertEqual(results, [(P2MR_SCRIPT, P2MR_SCRIPT[4:])] * (1 + WAITER_COUNT)) - self.assertIn(ADDRESS, validator.cache) - self.assert_released(flights.flights, "successful validation flight") - - -if __name__ == "__main__": - unittest.main() diff --git a/tests/test_prism_soak_capture.py b/tests/test_prism_soak_capture.py new file mode 100644 index 000000000..44f807714 --- /dev/null +++ b/tests/test_prism_soak_capture.py @@ -0,0 +1,992 @@ +#!/usr/bin/env python3 +"""Run the 24 h soak fences of ``docs/prism-capacity-readiness.md`` on a fake clock. + +The step-3 ``capture`` loop, the step-6 gate and the resident-set bound check +are extracted from the doc and run under ``sh`` and ``bash`` with shell +functions standing in for ``date``, ``sleep`` and ``docker``: ``sleep`` +advances a file-backed clock instead of waiting, so a whole 24 h soak is 289 +iterations, and ``docker`` answers ``inspect`` and ``exec`` from fixed bodies +with a fault injected at a chosen call. The contract under test is the one +#318's review asked for: ``capture`` ends a run itself after 86,400 s of +samples from its first, publishes ``soak-complete`` only after the final +sample, its post-read identity check and the check that its reads ended in +cadence with the previous sample's have passed, and does so by a rename +so that a failed or interrupted write leaves no marker; the gate judges a +directory only when it holds that marker and no ``soak-invalid``, so a run +cut short after the judge's 82,800 s floor cannot pass. The loop also takes +the hour-1, hour-12 and hour-24 snapshots itself, at the first sample due for +each counted from its first, so the baseline and intermediate snapshots +precede the end state. A snapshot that fails leaves ``soak-invalid`` instead +of a completion marker. +Each sample also requires the configured authorized-client population and a +valid accepted ACK count, with progress after the initial baseline sample. +""" + +from __future__ import annotations + +import os +import re +import signal +import subprocess +import tempfile +import unittest +from pathlib import Path + + +ROOT = Path(__file__).resolve().parents[1] +DOC = ROOT / "docs" / "prism-capacity-readiness.md" + +PLACEHOLDER = "c=<prism-coordinator-container>\n" +POPULATION_PLACEHOLDER = "expected_authorized_clients=<ordinary-load-authorized-client-count>\n" +RUN = "soak-20260914T000000Z" +RSS_KB = 102400 +SAMPLES_24H = 86400 // 300 + 1 +METRICS_LINES_PER_SAMPLE = 7 + +# `sleep` moves the clock; `date` reads it; `docker` counts its `inspect` calls +# so a test can change the identity, or run a snippet in the run directory, at +# a chosen call, and counts the snapshot functions' `curl -fsS` scrapes so a +# test can answer one with a cached body; every scrape's fixture header +# carries the clock it was read at. Anything the fences ask of a stub that it +# does not expect is appended to `$STUB/unexpected`, which every test requires +# to stay absent. +STUBS = r""" +stub_read() { read -r stub_value < "$STUB/$1"; } +stub_bump() { + stub_read "$1" + stub_value=$((stub_value + 1)) + echo "$stub_value" > "$STUB/$1" +} +date() { + case $* in + '+%s') stub_read clock; echo "$stub_value" ;; + '-u +%FT%TZ') stub_read clock; echo "T+$stub_value" ;; + '-u +%Y%m%dT%H%M%SZ') echo 20260914T000000Z ;; + *) echo "date $*" >> "$STUB/unexpected"; return 1 ;; + esac +} +sleep() { + stub_read clock + stub_value=$((stub_value + $1)) + echo "$stub_value" > "$STUB/clock" + if [ "$stub_value" = "$STUB_ROLLBACK_AT_SLEEP" ] && [ ! -e "$STUB/rolled-back" ]; then + echo "$STUB_ROLLBACK_TO" > "$STUB/clock" + : > "$STUB/rolled-back" + fi + if [ -n "$STUB_INTERRUPT_AT" ] && [ "$stub_value" -ge "$STUB_INTERRUPT_AT" ]; then + kill -s TERM $$ + fi +} +docker() { + case "$1 $3" in + 'inspect '*) + stub_bump inspects + if [ "$STUB_INSPECT_FAULT_AT" = all ] || [ "$stub_value" = "$STUB_INSPECT_FAULT_AT" ]; then + printf '%s' "$STUB_INSPECT_OUTPUT" + return "$STUB_INSPECT_STATUS" + fi + if [ "$stub_value" = "$STUB_SNIPPET_AT_INSPECT" ]; then + eval "$STUB_SNIPPET" + fi + if [ -n "$STUB_RESTART_AT_INSPECT" ] && [ "$stub_value" -ge "$STUB_RESTART_AT_INSPECT" ]; then + echo 'running 2026-09-14T00:00:00.000000000Z 1' + else + echo 'running 2026-09-13T00:00:00.000000000Z 0' + fi ;; + 'exec cat') + stub_read clock + printf 'Name:\tqbit-prism-server\nVmRSS:\t %s kB\nThreads:\t8\n' "$STUB_RSS_KB" + if [ "$stub_value" = "$STUB_RSS_FAULT_AT" ]; then return 1; fi ;; + 'exec curl') + stub_bump scrapes + stub_state=fresh + if [ "$4" = -fsS ]; then + stub_bump snapshots + if [ "$stub_value" = "$STUB_SNAPSHOT_STALE_AT" ]; then stub_state=stale; fi + stub_read samples + stub_accepted=$((STUB_ACCEPTED_BASE + stub_value - 1)) + else + stub_bump samples + stub_accepted=$((STUB_ACCEPTED_BASE + stub_value - 1)) + fi + stub_read clock + printf 'HTTP/1.1 200 OK\r\nx-prism-metrics-state: %s\r\ncontent-type: text/plain\r\nx-soak-fixture-clock: %s\r\n\r\n' "$stub_state" "$stub_value" + printf 'qbit_prism_collector_available{collector="process"} 1\n' + printf 'qbit_prism_process_resident_memory_bytes %s\n' "$((STUB_RSS_KB * 1024))" + printf 'qbit_prism_connections 3\n' + if [ "$STUB_AUTHORIZED_FAULT_AT" = all ] || [ "$stub_value" = "$STUB_AUTHORIZED_FAULT_AT" ]; then + printf '%s' "$STUB_AUTHORIZED_BODY" + else + printf 'qbit_prism_authorized_clients 3\n' + fi + if [ "$STUB_ACCEPTED_FAULT_AT" = all ] || [ "$stub_value" = "$STUB_ACCEPTED_FAULT_AT" ]; then + printf '%s' "$STUB_ACCEPTED_BODY" + else + printf 'qbit_prism_share_ack_seconds_count{result="accepted"} %s\n' "$stub_accepted" + fi + if [ -f "$STUB/ack-histogram" ]; then cat "$STUB/ack-histogram"; fi + printf 'qbit_prism_share_ack_seconds_count{result="rejected"} %s\n' "$((stub_value + 100))" + if [ "$stub_value" = "$STUB_METRICS_FAULT_AT" ]; then return 28; fi ;; + *) echo "docker $*" >> "$STUB/unexpected"; return 1 ;; + esac +} +if [ -n "$STUB_INTERRUPT_ON_MV" ]; then + mv() { + case $2 in */"$STUB_INTERRUPT_ON_MV") kill -s TERM $$ ;; esac + command mv "$@" + } +fi +""" + + +def shell_fences(text: str) -> list[str]: + """Every ```sh fence of the doc, with its list-item indentation removed.""" + fences = [] + lines = text.splitlines() + index = 0 + while index < len(lines): + opening = re.fullmatch(r"( *)```sh", lines[index]) + if opening: + indent = opening.group(1) + body = [] + index += 1 + while lines[index] != indent + "```": + line = lines[index] + if line: + assert line.startswith(indent), line + body.append(line[len(indent) :]) + index += 1 + fences.append("\n".join(body) + "\n") + index += 1 + return fences + + +def fence_containing(needle: str) -> str: + matches = [fence for fence in shell_fences(DOC.read_text()) if needle in fence] + assert len(matches) == 1, (needle, len(matches)) + return matches[0] + + +FENCES = shell_fences(DOC.read_text()) +CAPTURE = fence_containing("capture() {") +GATE = fence_containing("completed() {") +JUDGE = fence_containing("min_span=82800") +SHARE_ACK = fence_containing("share_ack_snapshot() (") +FULL_METRICS = fence_containing("full_metrics_snapshot() (") +START = "capture\n" +assert CAPTURE.count(PLACEHOLDER) == 1 +assert CAPTURE.count(POPULATION_PLACEHOLDER) == 1 +assert GATE.endswith("}\ncompleted\n") +# `capture` calls both snapshot functions, so the three fences only define +# functions and the run starts in the fence after them. +assert CAPTURE.endswith("\n}\n") and SHARE_ACK.endswith("\n)\n") and FULL_METRICS.endswith("\n)\n") +assert FENCES.count(START) == 1 +assert FENCES[FENCES.index(CAPTURE) : FENCES.index(START) + 1] == [CAPTURE, SHARE_ACK, FULL_METRICS, START] + +CAPTURE_SCRIPT = ( + STUBS + CAPTURE.replace(PLACEHOLDER, "c=stub-container\n") + .replace(POPULATION_PLACEHOLDER, 'expected_authorized_clients="$STUB_EXPECTED_CLIENTS"\n') + + SHARE_ACK + FULL_METRICS + START +) +GATE_SCRIPT = f"run={RUN}\n" + GATE +# What step 6 tells the operator to do: judge inside the directory, and only +# once the gate has accepted it. +GATE_AND_JUDGE_SCRIPT = f"run={RUN}\n" + GATE.rstrip("\n") + ' && (\n cd "$run" && ' + JUDGE + ")\n" + + +class SoakRun: + def __init__(self, shell: str, **env: str) -> None: + self.work = Path(tempfile.mkdtemp(prefix="soak-capture-")) + self.stub = self.work / "stub" + self.stub.mkdir() + (self.stub / "clock").write_text("0\n") + (self.stub / "inspects").write_text("0\n") + (self.stub / "snapshots").write_text("0\n") + (self.stub / "scrapes").write_text("0\n") + (self.stub / "samples").write_text("0\n") + self.run = self.work / RUN + self.shell = shell + self.env = { + **os.environ, "STUB": str(self.stub), "STUB_RSS_KB": str(RSS_KB), + "STUB_EXPECTED_CLIENTS": "3", "STUB_ACCEPTED_BASE": "0", **env, + } + + def capture(self, *, script: str = CAPTURE_SCRIPT) -> subprocess.CompletedProcess[str]: + path = self.work / "capture.sh" + path.write_text(script) + return self._run([self.shell, str(path)], self.work) + + def gate(self) -> subprocess.CompletedProcess[str]: + return self._run([self.shell, "-c", GATE_SCRIPT], self.work) + + def gate_and_judge(self) -> subprocess.CompletedProcess[str]: + return self._run([self.shell, "-c", GATE_AND_JUDGE_SCRIPT], self.work) + + def judge_alone(self) -> subprocess.CompletedProcess[str]: + return self._run([self.shell, "-c", JUDGE], self.run) + + def _run(self, argv: list[str], cwd: Path) -> subprocess.CompletedProcess[str]: + return subprocess.run(argv, cwd=cwd, env=self.env, capture_output=True, text=True, timeout=600) + + def unexpected(self) -> str: + path = self.stub / "unexpected" + return path.read_text() if path.exists() else "" + + def lines(self, name: str) -> list[str]: + return (self.run / name).read_text().splitlines() + + def entries(self) -> list[str]: + return sorted(entry.name for entry in self.run.iterdir()) + + def snapshot_clocks(self) -> dict[str, int]: + """The fake clock each published snapshot was scraped at.""" + clocks = {} + for name in SNAPSHOTS: + if (self.run / name).exists(): + (clock,) = re.findall( + r"^x-soak-fixture-clock: (\d+)$", (self.run / name).read_text(), re.M, + ) + clocks[name] = int(clock) + return clocks + + +COMPLETE_LINE = "T+86400: soak complete, the samples from 0 to 86400 span 86400 s" +RUN_FILES = ["soak-metrics.log", "soak-process.log", "soak-rss.csv"] +# In the order `capture` takes them, which is also the order of their scrapes. +SNAPSHOTS = ["share-ack-h01.txt", "metrics-h01.txt", "share-ack-h12.txt", "share-ack-h24.txt", "metrics-h24.txt"] + + +class ShareAckSnapshotTests(unittest.TestCase): + HISTOGRAM = ( + 'qbit_prism_share_ack_seconds_bucket{result="accepted",le="+Inf"} 5\n' + 'qbit_prism_share_ack_seconds_sum{result="accepted"} 0.1\n' + 'qbit_prism_share_ack_seconds_count{result="accepted"} 5\n' + ) + + def snapshot( + self, shell: str, state: str | None, *, status: int = 0, histogram: bool = True, body: str | None = None, + ) -> tuple[subprocess.CompletedProcess[str], str | None]: + with tempfile.TemporaryDirectory(prefix="share-ack-snapshot-") as directory: + work = Path(directory) + headers = "HTTP/1.1 200 OK\r\ncontent-type: text/plain\r\n" + if state is not None: + headers += f"x-prism-metrics-state: {state}\r\n" + if body is None: + body = "qbit_prism_connections 3\n" + (self.HISTOGRAM if histogram else "") + (work / "response").write_bytes((headers + "\r\n" + body).encode()) + # Like curl, the stub includes headers only when asked with -D -. + script = r''' +docker() { + case " $* " in + *" -D - "*) cat response ;; + *) sed '1,/^\r$/d' response ;; + esac + return "$SCRAPE_STATUS" +} +c=stub-container +run=. +''' + SHARE_ACK + 'share_ack_snapshot "$run/share-ack-h01.txt"\n' + result = subprocess.run( + [shell, "-c", script], cwd=work, capture_output=True, text=True, + env={**os.environ, "SCRAPE_STATUS": str(status)}, timeout=10, + ) + target = work / "share-ack-h01.txt" + return result, target.read_text() if target.exists() else None + + def test_fresh_snapshot_keeps_headers_and_histogram(self) -> None: + for shell in ("sh", "bash"): + with self.subTest(shell=shell): + result, snapshot = self.snapshot(shell, "fresh") + self.assertEqual((result.returncode, result.stderr), (0, "")) + self.assertEqual( + snapshot, + "HTTP/1.1 200 OK\ncontent-type: text/plain\n" + "x-prism-metrics-state: fresh\n\n" + self.HISTOGRAM, + ) + + def test_stale_unavailable_or_missing_header_publishes_no_snapshot(self) -> None: + for shell in ("sh", "bash"): + for state in ("stale", "unavailable", None): + with self.subTest(shell=shell, state=state): + result, snapshot = self.snapshot(shell, state) + self.assertNotEqual(result.returncode, 0, result) + self.assertIsNone(snapshot) + + def test_failed_scrape_or_missing_histogram_publishes_no_snapshot(self) -> None: + for shell in ("sh", "bash"): + for status, histogram in ((28, True), (22, False), (0, False)): + with self.subTest(shell=shell, status=status, histogram=histogram): + result, snapshot = self.snapshot(shell, "fresh", status=status, histogram=histogram) + self.assertNotEqual(result.returncode, 0, result) + self.assertIsNone(snapshot) + + def test_snapshot_requires_positive_unambiguous_accepted_count(self) -> None: + rejected = self.HISTOGRAM.replace('result="accepted"', 'result="rejected"') + prefix = 'qbit_prism_share_ack_seconds_count{result="accepted"}' + for shell in ("sh", "bash"): + for accepted in ("", *(f"{prefix} {value}\n" for value in ("0", "-1", "NaN", "1.5", "5 extra")), + f"{prefix} 5\n{prefix} 6\n"): + with self.subTest(shell=shell, accepted=accepted): + result, snapshot = self.snapshot(shell, "fresh", body=rejected + accepted) + self.assertNotEqual(result.returncode, 0, result) + self.assertIn("positive accepted ACK count required", result.stderr) + self.assertIsNone(snapshot) + + +class FullMetricsSnapshotTests(unittest.TestCase): + RESPONSE = ( + b"HTTP/1.1 200 OK\r\nx-prism-metrics-state: fresh\r\ncontent-type: text/plain\r\n\r\n" + b"# HELP qbit_prism_connections Active connections\nqbit_prism_connections 3\n" + b'qbit_prism_share_ack_seconds_count{result="accepted"} 5\n' + ) + + def snapshot(self, shell: str, response: bytes, *, status: int = 0, fault: str = "", existing: bool = False): + with tempfile.TemporaryDirectory(prefix="full-metrics-snapshot-") as directory: + work = Path(directory) + (work / "response").write_bytes(response) + target = work / "metrics-h01.txt" + temporary = work / "metrics-h01.txt.tmp" + if existing: + target.write_bytes(b"previous snapshot\n") + if fault == "write": + temporary.mkdir() + script = r''' +docker() { + if [ "$*" != "exec stub-container curl -fsS --max-time 5 -D - http://127.0.0.1:3341/metrics" ]; then + echo "unexpected docker invocation: $*" >&2 + return 99 + fi + cat response + return "$SCRAPE_STATUS" +} +case $PUBLISH_FAULT in + rename) mv() { return 1; } ;; + interrupt) mv() { kill -s TERM $$; } ;; +esac +c=stub-container +run=. +''' + FULL_METRICS + 'full_metrics_snapshot "$run/metrics-h01.txt"\n' + result = subprocess.run( + [shell, "-c", script], cwd=work, capture_output=True, text=True, + env={**os.environ, "SCRAPE_STATUS": str(status), "PUBLISH_FAULT": fault}, timeout=10, + ) + return result, target.read_bytes() if target.exists() else None, temporary.is_file() + + def test_fresh_snapshot_preserves_the_entire_response(self) -> None: + for shell in ("sh", "bash"): + for response in (self.RESPONSE, self.RESPONSE.replace(b"x-prism-metrics-state", b"X-Prism-Metrics-State")): + with self.subTest(shell=shell, response=response): + result, snapshot, temporary = self.snapshot(shell, response) + self.assertEqual((result.returncode, result.stderr), (0, "")) + self.assertEqual(snapshot, response) + self.assertFalse(temporary) + + def test_cached_or_missing_fresh_header_publishes_nothing(self) -> None: + for shell in ("sh", "bash"): + for state in (b"stale", b"unavailable", None): + response = (self.RESPONSE.replace(b"fresh", state) if state is not None + else self.RESPONSE.replace(b"x-prism-metrics-state: fresh\r\n", b"")) + # A fresh-looking line in the body cannot substitute for the header. + response += b"x-prism-metrics-state: fresh\n" + with self.subTest(shell=shell, state=state): + result, snapshot, _ = self.snapshot(shell, response) + self.assertNotEqual(result.returncode, 0) + self.assertIn("fresh state header required", result.stderr) + self.assertIsNone(snapshot) + + def test_http_and_partial_transport_failures_preserve_existing_evidence(self) -> None: + for shell in ("sh", "bash"): + for status, response in ((22, b"HTTP/1.1 500 Internal Server Error\r\n\r\nerror"), (28, self.RESPONSE[:-10])): + for existing in (False, True): + with self.subTest(shell=shell, status=status, existing=existing): + result, snapshot, _ = self.snapshot(shell, response, status=status, existing=existing) + self.assertNotEqual(result.returncode, 0) + self.assertIn("metrics scrape or temporary-file write failed", result.stderr) + self.assertEqual(snapshot, b"previous snapshot\n" if existing else None) + + def test_failed_or_interrupted_publication_leaves_no_final_snapshot(self) -> None: + for shell in ("sh", "bash"): + for fault in ("write", "rename", "interrupt"): + with self.subTest(shell=shell, fault=fault): + result, snapshot, temporary = self.snapshot(shell, self.RESPONSE, fault=fault) + self.assertNotEqual(result.returncode, 0) + self.assertIsNone(snapshot) + if fault != "write": + self.assertTrue(temporary) + + +class SoakCaptureTests(unittest.TestCase): + def assert_no_marker_at_all(self, soak: SoakRun) -> None: + self.assertFalse(os.path.lexists(soak.run / "soak-complete")) + + def assert_gate_refuses_for_want_of_marker(self, soak: SoakRun) -> None: + gate = soak.gate() + self.assertEqual(gate.returncode, 1, gate) + self.assertEqual(gate.stdout, "") + self.assertIn("no soak-complete", gate.stderr) + self.assertEqual(soak.gate_and_judge().returncode, 1) + + def test_fences_parse_under_sh_and_bash(self) -> None: + for shell in ("sh", "bash"): + for script in (CAPTURE_SCRIPT, GATE_SCRIPT, GATE_AND_JUDGE_SCRIPT): + check = subprocess.run([shell, "-n"], input=script, capture_output=True, text=True) + self.assertEqual(check.returncode, 0, (shell, check.stderr)) + + def test_accepted_progress_is_recorded_from_zero_without_extra_scrapes(self) -> None: + for shell in ("sh", "bash"): + with self.subTest(shell=shell): + soak = SoakRun(shell, STUB_INTERRUPT_AT="600") + result = soak.capture() + self.assertEqual((result.returncode, result.stderr, soak.unexpected()), (-signal.SIGTERM, "", "")) + metrics = soak.lines("soak-metrics.log") + self.assertEqual( + [line for line in metrics if 'share_ack_seconds_count{result="accepted"}' in line], + ['0 qbit_prism_share_ack_seconds_count{result="accepted"} 0', + '300 qbit_prism_share_ack_seconds_count{result="accepted"} 1'], + ) + self.assertEqual( + [line for line in metrics if "authorized_clients" in line], + ["0 qbit_prism_authorized_clients 3", "300 qbit_prism_authorized_clients 3"], + ) + self.assertEqual(len(metrics), 2 * METRICS_LINES_PER_SAMPLE) + self.assertEqual((soak.stub / "scrapes").read_text(), "2\n") + self.assert_gate_refuses_for_want_of_marker(soak) + + def test_cadence_histograms_preserve_slow_interval_after_fast_history(self) -> None: + # One million fast historical ACKs hide a final five-minute interval + # of slow accepted/rejected ACKs in the cumulative p99. Keep every + # bucket and both sums/counts so interval deltas expose it instead. + prefix = "qbit_prism_share_ack_seconds_" + + def histogram(result: str, fast: int, total: int) -> str: + return "".join( + f'{prefix}bucket{{result="{result}",le="{bound}"}} {count}\n' + for bound, count in (("1", fast), ("2", total), ("+Inf", total)) + ) + f'{prefix}sum{{result="{result}"}} {2 * total - fast}\n' + + initial = histogram("accepted", 1_000_000, 1_000_000) + histogram("rejected", 100, 100) + final = histogram("accepted", 1_000_000, 1_001_000) + histogram("rejected", 100, 400) + for shell in ("sh", "bash"): + with self.subTest(shell=shell): + soak = SoakRun( + shell, STUB_INTERRUPT_AT="600", STUB_ACCEPTED_BASE="1000000", + STUB_ACCEPTED_FAULT_AT="300", + STUB_ACCEPTED_BODY=f'{prefix}count{{result="accepted"}} 1001000\n', + STUB_SNIPPET_AT_INSPECT="4", + STUB_SNIPPET=f"printf '%s' '{final}' > \"$STUB/ack-histogram\"", + ) + (soak.stub / "ack-histogram").write_text(initial) + result = soak.capture() + self.assertEqual((result.returncode, result.stderr, soak.unexpected()), (-signal.SIGTERM, "", "")) + recorded = {} + for line in soak.lines("soak-metrics.log"): + timestamp, metric, value = line.split(maxsplit=2) + if metric.startswith(prefix): + recorded.setdefault(int(timestamp), {})[metric] = int(value) + for timestamp, body, accepted, rejected in ( + (0, initial, 1_000_000, 100), (300, final, 1_001_000, 400), + ): + expected = dict(line.split() for line in body.splitlines()) + expected[f'{prefix}count{{result="accepted"}}'] = str(accepted) + expected[f'{prefix}count{{result="rejected"}}'] = str(rejected) + self.assertEqual(recorded[timestamp], {key: int(value) for key, value in expected.items()}) + + def combined(timestamp: int, bound: str) -> int: + return sum(recorded[timestamp][f'{prefix}bucket{{result="{result}",le="{bound}"}}'] + for result in ("accepted", "rejected")) + + self.assertGreaterEqual(combined(300, "1") / combined(300, "+Inf"), 0.99) + observations = combined(300, "+Inf") - combined(0, "+Inf") + self.assertEqual(observations, 1300) + self.assertEqual(combined(300, "1") - combined(0, "1"), 0) + self.assertEqual(combined(300, "2") - combined(0, "2"), observations) + self.assertEqual((soak.stub / "scrapes").read_text(), "2\n") + + def test_frozen_or_decreasing_accepted_counts_invalidate_the_run(self) -> None: + prefix = 'qbit_prism_share_ack_seconds_count{result="accepted"}' + for shell in ("sh", "bash"): + for baseline, fault_at, count in (("0", "all", "42"), ("0", "all", "0"), ("10", "300", "9")): + with self.subTest(shell=shell, baseline=baseline, fault_at=fault_at, count=count): + soak = SoakRun( + shell, STUB_ACCEPTED_BASE=baseline, STUB_ACCEPTED_FAULT_AT=fault_at, + STUB_ACCEPTED_BODY=f"{prefix} {count}\n", + ) + result = soak.capture() + self.assertEqual((result.returncode, soak.unexpected()), (1, "")) + self.assertEqual(result.stderr, (soak.run / "soak-invalid").read_text()) + previous = count if fault_at == "all" else baseline + self.assertIn(f"accepted share ACK count did not increase from {previous} to {count}", result.stderr) + self.assertIn("no metrics sample at 300", result.stderr) + self.assertEqual(len(soak.lines("soak-metrics.log")), METRICS_LINES_PER_SAMPLE) + self.assertEqual((soak.stub / "scrapes").read_text(), "2\n") + self.assert_no_marker_at_all(soak) + self.assertEqual(soak.gate_and_judge().returncode, 1) + + def test_missing_malformed_or_duplicate_accepted_count_invalidates_the_run(self) -> None: + # Rejected counts keep growing in every stub response; they never + # supply the accepted progress required by the soak. + prefix = 'qbit_prism_share_ack_seconds_count{result="accepted"}' + for shell in ("sh", "bash"): + for body in ("", *(f"{prefix} {value}\n" for value in ("", "-1", "NaN", "1.5", "1e3", "5 extra")), + f"{prefix} 5\n{prefix} 6\n"): + with self.subTest(shell=shell, body=body): + soak = SoakRun(shell, STUB_ACCEPTED_FAULT_AT="all", STUB_ACCEPTED_BODY=body) + result = soak.capture() + self.assertEqual((result.returncode, soak.unexpected()), (1, "")) + self.assertIn(f"{prefix} must have one nonnegative integer sample", result.stderr) + self.assertFalse((soak.run / "soak-metrics.log").exists()) + self.assert_no_marker_at_all(soak) + self.assertEqual(soak.gate_and_judge().returncode, 1) + + def test_expected_population_must_be_positive_and_present_in_each_sample(self) -> None: + for shell in ("sh", "bash"): + for expected in ("", "0", "00", "-1", "1.5", "three"): + with self.subTest(shell=shell, expected=expected): + soak = SoakRun(shell, STUB_EXPECTED_CLIENTS=expected) + result = soak.capture() + self.assertEqual((result.returncode, soak.unexpected()), (1, "")) + self.assertIn("expected_authorized_clients must be a positive integer", result.stderr) + self.assertEqual((soak.stub / "scrapes").read_text(), "0\n") + self.assert_no_marker_at_all(soak) + for value in (None, "0", "2", "4", "-1", "NaN", "3.5", "3 extra"): + with self.subTest(shell=shell, actual=value): + body = "" if value is None else f"qbit_prism_authorized_clients {value}\n" + soak = SoakRun(shell, STUB_AUTHORIZED_FAULT_AT="300", STUB_AUTHORIZED_BODY=body) + result = soak.capture() + self.assertEqual((result.returncode, soak.unexpected()), (1, "")) + expected_error = (f"authorized client count {value} does not match expected 3" + if value in ("0", "2", "4") else + "qbit_prism_authorized_clients must have one nonnegative integer sample") + self.assertIn(expected_error, result.stderr) + self.assertEqual(len(soak.lines("soak-metrics.log")), METRICS_LINES_PER_SAMPLE) + self.assert_no_marker_at_all(soak) + self.assertEqual(soak.gate_and_judge().returncode, 1) + + def test_accepted_counter_comparison_preserves_large_integer_progress(self) -> None: + # Integer histogram counters must not round away a one-ACK increase + # above the exact-integer range of awk floating-point arithmetic. + for shell in ("sh", "bash"): + with self.subTest(shell=shell): + soak = SoakRun(shell, STUB_ACCEPTED_BASE="9007199254740992", STUB_INTERRUPT_AT="600") + result = soak.capture() + self.assertEqual((result.returncode, result.stderr, soak.unexpected()), (-signal.SIGTERM, "", "")) + self.assertIn( + '300 qbit_prism_share_ack_seconds_count{result="accepted"} 9007199254740993', + soak.lines("soak-metrics.log"), + ) + + def test_accepted_progress_stopping_on_final_sample_prevents_completion(self) -> None: + # Anchor the already-running soak at 0 while starting its last two + # samples at 86100, so the real completion branch is exercised + # without repeating 287 earlier samples for this fault case. + script = CAPTURE_SCRIPT.replace(" start=\n", " start=0\n") + self.assertNotEqual(script, CAPTURE_SCRIPT) + for shell in ("sh", "bash"): + with self.subTest(shell=shell): + soak = SoakRun( + shell, STUB_ACCEPTED_BASE="100", STUB_ACCEPTED_FAULT_AT="86400", + STUB_ACCEPTED_BODY='qbit_prism_share_ack_seconds_count{result="accepted"} 100\n', + ) + (soak.stub / "clock").write_text("86100\n") + result = soak.capture(script=script) + self.assertEqual((result.returncode, soak.unexpected()), (1, "")) + self.assertIn("no metrics sample at 86400", result.stderr) + self.assertIn("accepted share ACK count did not increase from 100 to 100", result.stderr) + self.assertEqual(len(soak.lines("soak-rss.csv")), 2) + self.assertFalse((soak.run / "share-ack-h24.txt").exists()) + self.assert_no_marker_at_all(soak) + self.assertEqual(soak.gate_and_judge().returncode, 1) + + def test_full_run_completes_after_24_h_and_is_judged(self) -> None: + for shell in ("sh", "bash"): + with self.subTest(shell=shell): + soak = SoakRun(shell) + result = soak.capture(script="set -e\n" + CAPTURE_SCRIPT) + self.assertEqual((result.returncode, result.stderr, soak.unexpected()), (0, "", "")) + self.assertEqual(soak.entries(), sorted([*SNAPSHOTS, "soak-complete", *RUN_FILES])) + # Each snapshot was scraped once, at the sample due for it, not at the end. + self.assertEqual( + soak.snapshot_clocks(), + { + "share-ack-h01.txt": 3600, "metrics-h01.txt": 3600, "share-ack-h12.txt": 43200, + "share-ack-h24.txt": 86400, "metrics-h24.txt": 86400, + }, + ) + self.assertEqual((soak.stub / "snapshots").read_text(), "5\n") + self.assertTrue((soak.run / "soak-complete").is_file()) + self.assertEqual(soak.lines("soak-complete"), [COMPLETE_LINE]) + rss = soak.lines("soak-rss.csv") + self.assertEqual(len(rss), SAMPLES_24H) + self.assertEqual((rss[0], rss[-1]), (f"0,{RSS_KB * 1024}", f"86400,{RSS_KB * 1024}")) + process = soak.lines("soak-process.log") + self.assertEqual(len(process), SAMPLES_24H) + self.assertEqual(set(line.split(" ", 1)[1] for line in process), {"running 2026-09-13T00:00:00.000000000Z 0"}) + self.assertEqual(len(soak.lines("soak-metrics.log")), SAMPLES_24H * METRICS_LINES_PER_SAMPLE) + gate = soak.gate() + self.assertEqual((gate.returncode, gate.stdout, gate.stderr), (0, COMPLETE_LINE + "\n", "")) + judged = soak.gate_and_judge() + self.assertEqual((judged.returncode, judged.stderr), (0, "")) + self.assertEqual( + judged.stdout.splitlines(), + [COMPLETE_LINE, "baseline=104857600 bound=209715200 peak=104857600 peak_at=3900 first_breach_at=none"], + ) + + def test_run_interrupted_after_23_h_is_not_judged(self) -> None: + # The review's case: a SIGTERM in the 24th hour runs no stop path and + # leaves a CSV the bound check's 82,800 s floor accepts on its own. + soak = SoakRun("sh", STUB_INTERRUPT_AT="83100") + result = soak.capture() + self.assertEqual((result.returncode, result.stderr, soak.unexpected()), (-signal.SIGTERM, "", "")) + self.assertEqual(soak.entries(), sorted([*SNAPSHOTS[:3], *RUN_FILES])) + rss = soak.lines("soak-rss.csv") + self.assertEqual((len(rss), rss[-1]), (82800 // 300 + 1, f"82800,{RSS_KB * 1024}")) + alone = soak.judge_alone() + self.assertEqual(alone.returncode, 0, alone) + self.assertIn("first_breach_at=none", alone.stdout) + self.assert_gate_refuses_for_want_of_marker(soak) + + def test_identity_change_on_the_final_sample_is_invalid(self) -> None: + # Inspect calls: one before the loop, then two per sample; the 289th + # sample's post-read check is call 579. + soak = SoakRun("sh", STUB_RESTART_AT_INSPECT=str(2 * SAMPLES_24H + 1)) + result = soak.capture() + self.assertEqual((result.returncode, soak.unexpected()), (1, "")) + self.assertEqual(soak.entries(), sorted([*SNAPSHOTS, "soak-invalid", *RUN_FILES])) + invalid = soak.lines("soak-invalid") + self.assertEqual(result.stderr, (soak.run / "soak-invalid").read_text()) + self.assertEqual( + invalid, + [ + "T+86400: soak invalid, the coordinator changed while the sample at 86400 was read", + " at start: running 2026-09-13T00:00:00.000000000Z 0", + " after read: running 2026-09-14T00:00:00.000000000Z 1", + ], + ) + self.assertEqual(len(soak.lines("soak-rss.csv")), SAMPLES_24H) + gate = soak.gate() + self.assertEqual((gate.returncode, gate.stdout), (1, "")) + self.assertIn("soak-invalid says why the run is invalid", gate.stderr) + self.assertEqual(soak.gate_and_judge().returncode, 1) + + def test_failed_or_empty_identity_probes_invalidate_the_run(self) -> None: + # A denied inspect must not pass by comparing empty identities, and + # matching output from a failed command is not a successful probe. + identity = "running 2026-09-13T00:00:00.000000000Z 0\n" + cases = [ + (shell, fault_at, probe, stage, status, output) + for shell in ("sh", "bash") + for fault_at, probe, stage in ( + ("all", 1, "at the start of the run"), + ("2", 2, "before the sample at 0"), + ("3", 3, "after the sample at 0"), + ) + for status, output in (("1", ""), ("0", ""), ("1", identity)) + ] + # The early cases cover both read boundaries under both shells. One + # full-duration post-read fault protects the last check before marker + # publication without repeating the entire matrix for 289 samples. + final_probe = 2 * SAMPLES_24H + 1 + cases.append(("sh", str(final_probe), final_probe, "after the sample at 86400", "1", identity)) + for shell, fault_at, probe, stage, status, output in cases: + with self.subTest(shell=shell, fault_at=fault_at, status=status, output=output): + soak = SoakRun( + shell, + STUB_INSPECT_FAULT_AT=fault_at, + STUB_INSPECT_STATUS=status, + STUB_INSPECT_OUTPUT=output, + ) + result = soak.capture() + self.assertEqual((result.returncode, soak.unexpected()), (1, "")) + self.assertEqual(result.stderr, (soak.run / "soak-invalid").read_text()) + self.assertIn(f"could not read the coordinator process identity {stage}", result.stderr) + self.assertEqual((soak.stub / "inspects").read_text(), f"{probe}\n") + rss = soak.run / "soak-rss.csv" + self.assertEqual(len(rss.read_text().splitlines()) if rss.exists() else 0, (probe - 1) // 2) + self.assert_no_marker_at_all(soak) + gate = soak.gate() + self.assertEqual((gate.returncode, gate.stdout), (1, "")) + self.assertIn("soak-invalid says why the run is invalid", gate.stderr) + self.assertEqual(soak.gate_and_judge().returncode, 1) + + def test_failed_rss_read_with_valid_output_invalidates_the_run(self) -> None: + for shell in ("sh", "bash"): + for now in (0, 86400): + with self.subTest(shell=shell, now=now): + soak = SoakRun(shell, STUB_RSS_FAULT_AT=str(now)) + result = soak.capture() + self.assertEqual((result.returncode, soak.unexpected()), (1, "")) + self.assertEqual(result.stderr, (soak.run / "soak-invalid").read_text()) + self.assertIn(f"no RSS sample at {now}", result.stderr) + self.assertIn("docker exec exited 1", result.stderr) + self.assertIn(f"VmRSS:\t {RSS_KB} kB", result.stderr) + for name, lines_per_sample in (("soak-rss.csv", 1), ("soak-metrics.log", METRICS_LINES_PER_SAMPLE)): + path = soak.run / name + lines = path.read_text().splitlines() if path.exists() else [] + self.assertEqual(len(lines), now // 300 * lines_per_sample) + self.assert_no_marker_at_all(soak) + gate = soak.gate() + self.assertEqual((gate.returncode, gate.stdout), (1, "")) + self.assertIn("soak-invalid says why the run is invalid", gate.stderr) + self.assertEqual(soak.gate_and_judge().returncode, 1) + + def test_failed_reads_record_invalid_marker_with_errexit(self) -> None: + # A failing assignment must reach capture's diagnostic path even + # when the operator's shell exits on unhandled command failures. + for shell in ("sh", "bash"): + for now in (0, 300): + for kind, fault, status, detail in ( + ("RSS", "STUB_RSS_FAULT_AT", 1, f"VmRSS:\t {RSS_KB} kB"), + ("metrics", "STUB_METRICS_FAULT_AT", 28, "x-prism-metrics-state: fresh"), + ): + with self.subTest(shell=shell, now=now, kind=kind): + soak = SoakRun(shell, **{fault: str(now)}) + result = soak.capture(script="set -e\n" + CAPTURE_SCRIPT) + self.assertTrue((soak.run / "soak-invalid").is_file(), result) + self.assertEqual((result.returncode, soak.unexpected()), (1, "")) + self.assertEqual(result.stderr, (soak.run / "soak-invalid").read_text()) + self.assertIn(f"no {kind} sample at {now}", result.stderr) + self.assertIn(f"docker exec exited {status}", result.stderr) + self.assertIn(detail, result.stderr) + self.assert_no_marker_at_all(soak) + self.assertEqual(soak.gate_and_judge().returncode, 1) + + def test_final_sample_whose_reads_stall_for_an_hour_is_invalid(self) -> None: + # Codex's case: the 289th sample's pre-read inspect (call 578) stalls + # for an hour while the identity and both bodies stay healthy. `now` + # was read before the stall, so the row is stamped 86400 for a process + # observed at 90000; the post-read identity check passes, and the + # completion check used to read that stale `now` and publish a marker + # spanning 0..86400, with no next iteration to see the gap. + for shell in ("sh", "bash"): + with self.subTest(shell=shell): + soak = SoakRun( + shell, + STUB_SNIPPET_AT_INSPECT=str(2 * SAMPLES_24H), + STUB_SNIPPET='echo 90000 > "$STUB/clock"', + ) + result = soak.capture() + self.assertEqual((result.returncode, soak.unexpected()), (1, "")) + self.assertEqual(soak.entries(), sorted([*SNAPSHOTS, "soak-invalid", *RUN_FILES])) + self.assertEqual(result.stderr, (soak.run / "soak-invalid").read_text()) + self.assertEqual( + soak.lines("soak-invalid"), + ["T+90000: soak invalid, the reads for the sample at 86400 ended at 90000, 3900 s after the previous sample's ended at 86100"], + ) + rss = soak.lines("soak-rss.csv") + self.assertEqual((len(rss), rss[-1]), (SAMPLES_24H, f"86400,{RSS_KB * 1024}")) + self.assertEqual(len(soak.lines("soak-process.log")), SAMPLES_24H) + alone = soak.judge_alone() + self.assertEqual(alone.returncode, 0, alone) + self.assertIn("first_breach_at=none", alone.stdout) + self.assert_no_marker_at_all(soak) + gate = soak.gate() + self.assertEqual((gate.returncode, gate.stdout), (1, "")) + self.assertIn("soak-invalid says why the run is invalid", gate.stderr) + self.assertEqual(soak.gate_and_judge().returncode, 1) + + def test_sample_reads_get_the_minute_the_gap_tolerance_leaves_them(self) -> None: + # From one sample's end to the next is the 300 s sleep plus the later + # sample's reads, so the 360 s tolerance leaves the reads 60 s. A + # non-final sample that overran it was already caught, by the next + # iteration's start-time check; the final sample is held to the same + # minute by the end-time check, where it used to complete with a + # marker for reads of any length. + final_pre_read = str(2 * SAMPLES_24H) + soak = SoakRun("sh", STUB_SNIPPET_AT_INSPECT=final_pre_read, STUB_SNIPPET='echo 86460 > "$STUB/clock"') + result = soak.capture() + self.assertEqual((result.returncode, result.stderr, soak.unexpected()), (0, "", "")) + self.assertEqual(soak.entries(), sorted([*SNAPSHOTS, "soak-complete", *RUN_FILES])) + late_marker = "T+86460: soak complete, the samples from 0 to 86400 span 86400 s" + self.assertEqual(soak.lines("soak-complete"), [late_marker]) + self.assertEqual(soak.lines("soak-rss.csv")[-1], f"86400,{RSS_KB * 1024}") + gate = soak.gate() + self.assertEqual((gate.returncode, gate.stdout, gate.stderr), (0, late_marker + "\n", "")) + + for sample, ended, snapshots, expected in ( + (SAMPLES_24H, 86461, SNAPSHOTS, "T+86461: soak invalid, the reads for the sample at 86400 ended at 86461, 361 s after the previous sample's ended at 86100"), + (2, 361, [], "T+361: soak invalid, the reads for the sample at 300 ended at 361, 361 s after the previous sample's ended at 0"), + ): + with self.subTest(sample=sample): + soak = SoakRun("sh", STUB_SNIPPET_AT_INSPECT=str(2 * sample), STUB_SNIPPET=f'echo {ended} > "$STUB/clock"') + result = soak.capture() + self.assertEqual((result.returncode, soak.unexpected()), (1, "")) + self.assertEqual(soak.entries(), sorted([*snapshots, "soak-invalid", *RUN_FILES])) + self.assertEqual(soak.lines("soak-invalid"), [expected]) + self.assertEqual(len(soak.lines("soak-rss.csv")), sample) + self.assert_no_marker_at_all(soak) + gate = soak.gate() + self.assertEqual((gate.returncode, gate.stdout), (1, "")) + self.assertIn("soak-invalid says why the run is invalid", gate.stderr) + self.assertEqual(soak.gate_and_judge().returncode, 1) + + def test_hour_snapshots_are_taken_at_the_first_sample_due_from_the_first(self) -> None: + # Codex's case: `capture` holds the shell for the whole soak, so + # snapshot calls typed after it all read the end state. The loop takes + # them itself. The run starts at 1234 and its first sample's reads end + # at 1279, so no later sample is a whole number of hours after the + # first or on the clock's own hour: each snapshot comes from the first + # sample at or past its hour, counted from the first sample. + for shell in ("sh", "bash"): + with self.subTest(shell=shell): + soak = SoakRun(shell, STUB_SNIPPET_AT_INSPECT="2", STUB_SNIPPET='echo 1279 > "$STUB/clock"') + (soak.stub / "clock").write_text("1234\n") + result = soak.capture() + self.assertEqual((result.returncode, result.stderr, soak.unexpected()), (0, "", "")) + self.assertEqual( + soak.snapshot_clocks(), + { + "share-ack-h01.txt": 4879, "metrics-h01.txt": 4879, "share-ack-h12.txt": 44479, + "share-ack-h24.txt": 87679, "metrics-h24.txt": 87679, + }, + ) + self.assertEqual((soak.stub / "snapshots").read_text(), "5\n") + self.assertEqual( + soak.lines("soak-complete"), ["T+87679: soak complete, the samples from 1234 to 87679 span 86445 s"], + ) + + def test_hour_snapshot_that_fails_invalidates_the_run(self) -> None: + # Snapshot scrapes 1 to 5 are SNAPSHOTS in order; the chosen one is + # answered with a cached body. The run stops at that sample, keeps the + # snapshots taken before it, publishes nothing under the failed name + # and, when the failure is an hour-24 snapshot, writes no marker. + share_ack = " share-ack snapshot failed: fresh state header and positive accepted ACK count required" + full_metrics = " full metrics snapshot failed: fresh state header required" + for shell in ("sh", "bash"): + for scrape, hour, kind, detail in ( + (1, 1, "share-ack", share_ack), + (2, 1, "full metrics", full_metrics), + (3, 12, "share-ack", share_ack), + (4, 24, "share-ack", share_ack), + (5, 24, "full metrics", full_metrics), + ): + with self.subTest(shell=shell, scrape=scrape): + now = hour * 3600 + soak = SoakRun(shell, STUB_SNAPSHOT_STALE_AT=str(scrape)) + result = soak.capture() + self.assertEqual((result.returncode, soak.unexpected()), (1, "")) + self.assertEqual(result.stderr, (soak.run / "soak-invalid").read_text()) + self.assertEqual( + soak.lines("soak-invalid"), [f"T+{now}: soak invalid, no hour-{hour} {kind} snapshot at {now}", detail], + ) + self.assertEqual((soak.stub / "snapshots").read_text(), f"{scrape}\n") + # full_metrics_snapshot leaves its unpublished temporary file. + temporary = [SNAPSHOTS[scrape - 1] + ".tmp"] if kind == "full metrics" else [] + self.assertEqual( + soak.entries(), sorted([*SNAPSHOTS[: scrape - 1], *temporary, "soak-invalid", *RUN_FILES]), + ) + self.assertEqual(len(soak.lines("soak-rss.csv")), now // 300 + 1) + self.assert_no_marker_at_all(soak) + gate = soak.gate() + self.assertEqual((gate.returncode, gate.stdout), (1, "")) + self.assertIn("soak-invalid says why the run is invalid", gate.stderr) + self.assertEqual(soak.gate_and_judge().returncode, 1) + + def test_backward_clock_step_between_samples_is_invalid_before_recording(self) -> None: + for shell in ("sh", "bash"): + for sleep_at, rollback_to, previous_end, samples, first_read_end in ( + (600, 0, 300, 2, 0), + # Still later than the previous start (0), but earlier than its end (30). + (330, 15, 30, 1, 30), + (86400, 3600, 86100, SAMPLES_24H - 1, 0), + ): + with self.subTest(shell=shell, sleep_at=sleep_at): + soak = SoakRun( + shell, STUB_ROLLBACK_AT_SLEEP=str(sleep_at), STUB_ROLLBACK_TO=str(rollback_to), + STUB_SNIPPET_AT_INSPECT="2", STUB_SNIPPET=f'echo {first_read_end} > "$STUB/clock"', + ) + result = soak.capture() + self.assertEqual((result.returncode, soak.unexpected()), (1, "")) + self.assertEqual(result.stderr, (soak.run / "soak-invalid").read_text()) + self.assertIn(f"clock moved backward from the previous sample's end at {previous_end} to {rollback_to}", result.stderr) + self.assertEqual(len(soak.lines("soak-rss.csv")), samples) + self.assertEqual(len(soak.lines("soak-process.log")), samples) + self.assert_no_marker_at_all(soak) + self.assertEqual(soak.gate_and_judge().returncode, 1) + + def test_backward_clock_step_during_sample_reads_is_invalid(self) -> None: + for shell in ("sh", "bash"): + for sample, end in ((1, -1), (2, 299), (14, 1800), (SAMPLES_24H, 86399)): + with self.subTest(shell=shell, sample=sample, end=end): + soak = SoakRun( + shell, STUB_SNIPPET_AT_INSPECT=str(2 * sample), + STUB_SNIPPET=f'echo {end} > "$STUB/clock"', + ) + result = soak.capture() + self.assertEqual((result.returncode, soak.unexpected()), (1, "")) + self.assertEqual(result.stderr, (soak.run / "soak-invalid").read_text()) + self.assertIn(f"clock moved backward during the sample from {(sample - 1) * 300} to {end}", result.stderr) + self.assert_no_marker_at_all(soak) + self.assertEqual(soak.gate_and_judge().returncode, 1) + + def test_equal_consecutive_clock_readings_remain_valid(self) -> None: + # Equality remains valid at both boundaries; only a negative interval fails. + soak = SoakRun("sh", STUB_ROLLBACK_AT_SLEEP="300", STUB_ROLLBACK_TO="0") + result = soak.capture() + self.assertEqual((result.returncode, result.stderr, soak.unexpected()), (0, "", "")) + self.assertEqual(soak.gate_and_judge().returncode, 0) + + @unittest.skipUnless(Path("/dev/full").exists(), "needs /dev/full to fail a write") + def test_marker_write_that_fails_publishes_no_marker_even_when_invalid_cannot_be_written(self) -> None: + # At the final post-read check the marker's temporary file already + # points at a full disk and `soak-invalid` at a directory that does + # not exist: the write fails after the file was created, and the + # invalid marker cannot be written either. Nothing is published. + soak = SoakRun( + "sh", + STUB_SNIPPET_AT_INSPECT=str(2 * SAMPLES_24H + 1), + STUB_SNIPPET='ln -s /dev/full "$run/soak-complete.tmp" && ln -s "$STUB/absent/soak-invalid" "$run/soak-invalid"', + ) + result = soak.capture() + self.assertEqual((result.returncode, soak.unexpected()), (1, "")) + self.assertIn("soak invalid, could not write", result.stderr) + self.assertIn("soak-complete after the sample at 86400", result.stderr) + self.assert_no_marker_at_all(soak) + self.assertTrue((soak.run / "soak-complete.tmp").is_symlink()) + self.assertTrue(os.path.lexists(soak.run / "soak-invalid")) + self.assertFalse((soak.run / "soak-invalid").exists()) + self.assertEqual(len(soak.lines("soak-rss.csv")), SAMPLES_24H) + self.assert_gate_refuses_for_want_of_marker(soak) + + def test_interruption_between_marker_write_and_rename_publishes_no_marker(self) -> None: + # Only the marker's rename is interrupted; the snapshots' renames run. + soak = SoakRun("sh", STUB_INTERRUPT_ON_MV="soak-complete") + result = soak.capture() + self.assertEqual((result.returncode, result.stderr, soak.unexpected()), (-signal.SIGTERM, "", "")) + self.assertEqual(soak.entries(), sorted([*SNAPSHOTS, "soak-complete.tmp", *RUN_FILES])) + self.assertEqual(soak.lines("soak-complete.tmp"), [COMPLETE_LINE]) + self.assert_no_marker_at_all(soak) + self.assert_gate_refuses_for_want_of_marker(soak) + + def test_existing_run_directory_is_never_written_into(self) -> None: + soak = SoakRun("sh") + soak.run.mkdir() + (soak.run / "soak-complete").write_text("stale marker from an earlier run\n") + result = soak.capture() + self.assertEqual((result.returncode, soak.unexpected()), (1, "")) + self.assertIn("File exists", result.stderr) + self.assertEqual(soak.entries(), ["soak-complete"]) + self.assertEqual(soak.lines("soak-complete"), ["stale marker from an earlier run"]) + self.assertEqual((soak.stub / "clock").read_text(), "0\n") + + def test_gate_reads_the_markers_before_anything_else(self) -> None: + both = SoakRun("sh") + both.run.mkdir() + (both.run / "soak-complete").write_text(COMPLETE_LINE + "\n") + (both.run / "soak-invalid").write_text("T+1: soak invalid, no RSS sample at 1\n") + gate = both.gate() + self.assertEqual((gate.returncode, gate.stdout), (1, "")) + self.assertIn("soak-invalid says why the run is invalid", gate.stderr) + + directory = SoakRun("sh") + directory.run.mkdir() + (directory.run / "soak-complete").mkdir() + self.assert_gate_refuses_for_want_of_marker(directory) + + only_temporary = SoakRun("sh") + only_temporary.run.mkdir() + (only_temporary.run / "soak-complete.tmp").write_text(COMPLETE_LINE + "\n") + self.assert_gate_refuses_for_want_of_marker(only_temporary) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_prism_statement_spool.py b/tests/test_prism_statement_spool.py deleted file mode 100644 index 12d433fc7..000000000 --- a/tests/test_prism_statement_spool.py +++ /dev/null @@ -1,106 +0,0 @@ -"""Native-ledger finalization transport: bytes, deadlines, and exact SQL.""" - -from __future__ import annotations - -import os -import subprocess -import threading -import time -import tracemalloc -import unittest -from unittest import mock -import uuid - -from lab.prism.share_ledger import LedgerOperationTimeout, PsqlShareLedger, WRITER_LEASE_HEARTBEAT_SESSION_PREFIX -from lab.prism.statement_spool import run_fenced_statement - - -@unittest.skipUnless(os.environ.get("PRISM_TEST_DATABASE_URL"), "requires disposable PostgreSQL") -class StatementSpoolPostgresTests(unittest.TestCase): - def setUp(self): - self.ledger = PsqlShareLedger( - database_url=os.environ["PRISM_TEST_DATABASE_URL"], psql_command="psql", - native_client_mode="native", writer_id="candidate-statement-tests", - writer_session_token=WRITER_LEASE_HEARTBEAT_SESSION_PREFIX + uuid.uuid4().hex, - ) - self.addCleanup(self.ledger.close) - self.addCleanup(self.ledger.release_writer_lease) - - def test_large_statement_never_enters_coordinator_native_buffer(self): - def pieces(): - yield "SELECT json_build_object('length', octet_length($payload$" - for _ in range(4096): - yield "x" * 4096 - yield "$payload$::text));" - - tracemalloc.start() - try: - with mock.patch.object(self.ledger._native, "run_json", side_effect=AssertionError("whole query")): - result = run_fenced_statement(self.ledger, pieces()) - _, peak = tracemalloc.get_traced_memory() - finally: - tracemalloc.stop() - self.assertEqual(result, {"length": 16 * 1024 * 1024}) - self.assertLess(peak, 3 * 1024 * 1024) - - def test_server_deadline_releases_gate_and_allows_retry(self): - with self.ledger.operation_timeout(0.15): - with self.assertRaises(LedgerOperationTimeout): - run_fenced_statement(self.ledger, ["SELECT pg_sleep(10);"]) - self.assertTrue(self.ledger._lock.acquire(blocking=False)) - self.ledger._lock.release() - self.assertEqual(run_fenced_statement(self.ledger, ["SELECT '{\"ok\":true}'::json;"]), {"ok": True}) - - def test_spooled_connection_matches_pool_options_and_keeps_password_out_of_argv(self): - from psycopg.conninfo import conninfo_to_dict - - sql = "SELECT json_build_object('schema', current_setting('search_path'), 'work_mem', current_setting('work_mem'), 'app', current_setting('application_name'));" - expected = self.ledger._run_json(sql) - with mock.patch.dict(os.environ, {"PGOPTIONS": "-c search_path=wrong_schema -c work_mem=1MB", "PGAPPNAME": "foreign"}), \ - mock.patch("lab.prism.statement_spool.subprocess.Popen", wraps=subprocess.Popen) as launch: - self.assertEqual(run_fenced_statement(self.ledger, [sql]), expected) - command = launch.call_args.args[0] - dsn = command[command.index("--dbname") + 1] - self.assertNotIn("password", conninfo_to_dict(dsn)) - - def test_expired_lease_attributes_spooled_write_to_its_owner(self): - self.ledger._run_json("UPDATE qbit_ledger_writer_lease SET lease_expires_at = clock_timestamp() - interval '1 second' RETURNING '{}'::json;") - results, errors = [], [] - - def write(): - try: - results.append(run_fenced_statement(self.ledger, [ - "BEGIN; UPDATE qbit_ledger_writer_lease SET lease_expires_at = clock_timestamp() + interval '60 seconds'; " - "SELECT json_build_object('ok', true) FROM pg_sleep(2); COMMIT;", - ])) - except Exception as exc: - errors.append(exc) - - thread = threading.Thread(target=write) - thread.start() - deadline = time.monotonic() + 5 - locked = False - try: - while thread.is_alive() and time.monotonic() < deadline: - locked = self.ledger._run_json(""" -SELECT json_build_object('locked', EXISTS ( - SELECT 1 FROM qbit_ledger_writer_lease lease, pg_stat_activity activity - WHERE activity.backend_xid = lease.xmax AND activity.backend_xid IS NOT NULL -)); -""")["locked"] - if locked: - break - time.sleep(0.01) - self.assertTrue(locked) - proof = self.ledger.verify_writer_lease_guard_session() - self.assertTrue(proof["renewal_deferred_to_own_write"]) - finally: - thread.join(10) - self.assertFalse(thread.is_alive()) - self.assertEqual(errors, []) - self.assertEqual(results, [{"ok": True}]) - self.assertFalse(self.ledger.verify_writer_lease_guard_session()["renewal_deferred_to_own_write"]) - - -if __name__ == "__main__": - unittest.main() diff --git a/tests/test_prism_stratum_restart_bind.py b/tests/test_prism_stratum_restart_bind.py deleted file mode 100644 index df9836853..000000000 --- a/tests/test_prism_stratum_restart_bind.py +++ /dev/null @@ -1,223 +0,0 @@ -#!/usr/bin/env python3 -"""Stratum listeners across a fast restart. - -The coordinator must bind its listen sockets before the slow parts of startup -(qbit readiness, policy validation, block-work recovery) so miners -reconnecting through a ~2s restart park in the kernel accept backlog instead -of getting connection refused, and must tolerate a predecessor process that -still holds the port while draining its shutdown. -""" - -from __future__ import annotations - -import socket -import threading -import time -import types -import unittest -from contextlib import ExitStack -from decimal import Decimal - -from lab.auxpow import vardiff -from lab.prism.prism_coordinator import ( - PrismCoordinator, - StratumListenerProfile, -) - - -def free_port() -> int: - with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as sock: - sock.bind(("127.0.0.1", 0)) - return sock.getsockname()[1] - - -def make_vardiff_config() -> vardiff.VardiffConfig: - return vardiff.VardiffConfig( - enabled=False, - target_share_interval_seconds=Decimal("15"), - min_difficulty=Decimal("0.000000001"), - max_difficulty=Decimal("1024"), - retarget_interval_seconds=Decimal("90"), - max_step_factor=Decimal("4"), - startup_difficulty=Decimal("1"), - max_step_down_factor=Decimal("4"), - ewma_alpha=Decimal("0.4"), - retarget_tolerance=Decimal("0.25"), - ) - - -def make_profile(port: int, name: str = "default") -> StratumListenerProfile: - return StratumListenerProfile( - name=name, - bind="127.0.0.1", - port=port, - share_difficulty=Decimal("1"), - vardiff_config=make_vardiff_config(), - heartbeat_name=f"stratum_accept_{name}", - ) - - -def make_coordinator( - profiles: list[StratumListenerProfile], - *, - bind_retry_seconds: float = 5.0, -) -> PrismCoordinator: - server = PrismCoordinator.__new__(PrismCoordinator) - server.listener_profiles = profiles - server.stratum_listen_backlog = 128 - server.stratum_bind_retry_seconds = bind_retry_seconds - return server - - -class OpenStratumListenersTest(unittest.TestCase): - def test_bound_listeners_hold_connections_before_accept(self) -> None: - """A connection completes its handshake in the kernel backlog even - though nothing has called accept() yet — the state miners sit in while - startup recovery runs.""" - ports = [free_port(), free_port()] - server = make_coordinator( - [make_profile(ports[0]), make_profile(ports[1], name="highdiff")] - ) - with ExitStack() as stack: - listeners = server.open_stratum_listeners(stack) - self.assertEqual(len(listeners), 2) - for port in ports: - with socket.create_connection(("127.0.0.1", port), timeout=2): - pass - - def test_bind_retries_until_predecessor_releases_port(self) -> None: - port = free_port() - predecessor = socket.socket(socket.AF_INET, socket.SOCK_STREAM) - predecessor.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) - predecessor.bind(("127.0.0.1", port)) - predecessor.listen(1) - self.addCleanup(predecessor.close) - releaser = threading.Timer(0.3, predecessor.close) - releaser.start() - self.addCleanup(releaser.cancel) - server = make_coordinator([make_profile(port)], bind_retry_seconds=5.0) - started = time.monotonic() - with ExitStack() as stack: - listeners = server.open_stratum_listeners(stack) - self.assertEqual(len(listeners), 1) - elapsed = time.monotonic() - started - self.assertLess(elapsed, 5.0) - - def test_bind_retry_aborts_on_shutdown_signal(self) -> None: - """A SIGTERM during the bind retry must stop the port contention - immediately and gracefully instead of running out the retry window.""" - port = free_port() - predecessor = socket.socket(socket.AF_INET, socket.SOCK_STREAM) - predecessor.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) - predecessor.bind(("127.0.0.1", port)) - predecessor.listen(1) - self.addCleanup(predecessor.close) - server = make_coordinator([make_profile(port)], bind_retry_seconds=30.0) - server.stop_event = threading.Event() - server.stop_event.set() - started = time.monotonic() - with ExitStack() as stack: - self.assertIsNone(server.open_stratum_listeners(stack)) - self.assertLess(time.monotonic() - started, 2.0) - - def test_zero_retry_window_fails_fast(self) -> None: - port = free_port() - predecessor = socket.socket(socket.AF_INET, socket.SOCK_STREAM) - predecessor.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) - predecessor.bind(("127.0.0.1", port)) - predecessor.listen(1) - self.addCleanup(predecessor.close) - server = make_coordinator([make_profile(port)], bind_retry_seconds=0.0) - started = time.monotonic() - with ExitStack() as stack: - with self.assertRaises(OSError): - server.open_stratum_listeners(stack) - self.assertLess(time.monotonic() - started, 1.0) - - -class ServeBindsBeforeRecoveryTest(unittest.TestCase): - def test_listeners_accept_tcp_connections_during_block_work_recovery(self) -> None: - """serve() must already be listening when block-work recovery runs, so - a restart's recovery window never bounces reconnecting miners.""" - port = free_port() - server = make_coordinator([make_profile(port)], bind_retry_seconds=0.0) - server.bind = "127.0.0.1" - server.port = port - server.share_difficulty = Decimal("1") - server.min_ready_miners = 1 - server.vardiff_config = make_vardiff_config() - server.max_blocks = 1 - server.blockpoll_seconds = 0.1 - server.version_mask = 0x1FFFE000 - server.version_mask_selection = types.SimpleNamespace( - source="test", detail="fixed" - ) - server.ledger = types.SimpleNamespace(backend_name="fake") - server.rpc = types.SimpleNamespace(call=lambda *args, **kwargs: 0) - server.audit_bind = None - server.audit_port = 0 - server.blockwait_enabled = False - server.vardiff_idle_sweep_seconds = 0.0 - server.stratum_initial_job_timeout_seconds = 0.0 - server.hot_path_log_enabled = False - server.tip_template_snapshot = None - server.ctv_broadcaster_enabled = False - server.watchdog_enabled = False - server.stop_event = threading.Event() - server.lock = threading.RLock() - server.clients = set() - server.validate_live_chain_identity = lambda: None # type: ignore[method-assign] - server.validate_live_template_and_fee_policy = lambda: None # type: ignore[method-assign] - server.prism_payout_policy = lambda: {} # type: ignore[method-assign] - server.prewarm_startup_jobs = lambda: None # type: ignore[method-assign] - server.replay_recovered_shares = lambda: 0 # type: ignore[method-assign] - server.shutdown = lambda *, reason="graceful": True # type: ignore[method-assign] - - observed: dict[str, bool] = {} - - def fake_replay_pending_block_candidates() -> int: - with socket.create_connection(("127.0.0.1", port), timeout=2): - observed["listening_during_recovery"] = True - server.stop_event.set() - return 0 - - server.replay_pending_block_candidates = ( # type: ignore[method-assign] - fake_replay_pending_block_candidates - ) - - server.serve() - - self.assertTrue(observed.get("listening_during_recovery")) - # The listen port must be free again once serve() returns, so a - # successor process can bind without waiting. - replacement = socket.socket(socket.AF_INET, socket.SOCK_STREAM) - self.addCleanup(replacement.close) - replacement.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) - replacement.bind(("127.0.0.1", port)) - - def test_shutdown_during_rpc_readiness_wait_releases_ports(self) -> None: - """A SIGTERM while serve() waits for qbit RPC must abort startup and - free the bound listen ports promptly, so a successor's bind retry - window is never starved by a dying predecessor.""" - port = free_port() - server = make_coordinator([make_profile(port)], bind_retry_seconds=0.0) - server.stop_event = threading.Event() - - def failing_rpc_call(*args: object, **kwargs: object) -> int: - raise ConnectionError("qbit not ready") - - server.rpc = types.SimpleNamespace(call=failing_rpc_call) - stopper = threading.Timer(0.2, server.stop_event.set) - stopper.start() - self.addCleanup(stopper.cancel) - started = time.monotonic() - server.serve() - self.assertLess(time.monotonic() - started, 5.0) - replacement = socket.socket(socket.AF_INET, socket.SOCK_STREAM) - self.addCleanup(replacement.close) - replacement.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) - replacement.bind(("127.0.0.1", port)) - - -if __name__ == "__main__": - unittest.main() diff --git a/tests/test_prism_stratum_session.py b/tests/test_prism_stratum_session.py deleted file mode 100644 index e575db550..000000000 --- a/tests/test_prism_stratum_session.py +++ /dev/null @@ -1,507 +0,0 @@ -#!/usr/bin/env python3 -"""Deterministic tests for the extracted PRISM Stratum session boundary. - -The current service resolves the coordinator through one call-time -``SessionRuntime`` typed port (not the old per-seam collaborator objects), so -these tests drive :class:`StratumSessionService` with a recording runtime and -exercise :class:`SessionRegistry` directly where the contract is registry-owned. -""" - -from __future__ import annotations - -import socket -import threading -import unittest -from decimal import Decimal -from types import SimpleNamespace -from unittest.mock import patch - -from lab.auxpow import vardiff -from lab.prism import stratum_session -from lab.prism.coordinator_config import StratumListenerProfile -from lab.prism.prism_coordinator import ( - ClientState as FacadeClientState, - PrismCoordinator, - StratumError as FacadeStratumError, - WorkerIdentity as FacadeWorkerIdentity, -) -from lab.prism.stratum_session import ( - ClientState, - SessionRegistry, - StratumError, - StratumSessionService, - WorkerIdentity, - client_can_receive_jobs, - error_payload, - parse_stratum_password_options, - result_payload, -) - - -def listener() -> StratumListenerProfile: - config = vardiff.VardiffConfig( - enabled=False, - target_share_interval_seconds=Decimal("15"), - min_difficulty=Decimal("1"), - max_difficulty=Decimal("1024"), - retarget_interval_seconds=Decimal("90"), - max_step_factor=Decimal("4"), - startup_difficulty=Decimal("1"), - max_step_down_factor=Decimal("4"), - ewma_alpha=Decimal("0.4"), - retarget_tolerance=Decimal("0.25"), - ) - return StratumListenerProfile( - name="default", - bind="127.0.0.1", - port=3340, - share_difficulty=Decimal("1"), - vardiff_config=config, - heartbeat_name="stratum_accept", - ) - - -def worker(username: str) -> WorkerIdentity: - return WorkerIdentity( - username=username, - payout_address=username, - worker_name=None, - script_pubkey_hex="5220" + "11" * 32, - p2mr_program_hex="11" * 32, - ) - - -class FakeSocket: - def __init__(self) -> None: - self.closed = threading.Event() - self.sent: list[bytes] = [] - - def settimeout(self, _timeout: object) -> None: - return - - def setsockopt(self, *_args: object) -> None: - return - - def shutdown(self, _how: object) -> None: - self.closed.set() - - def close(self) -> None: - self.closed.set() - - def sendall(self, data: bytes) -> None: - self.sent.append(data) - - -def client_state(connection_id: int) -> ClientState: - return ClientState( - FakeSocket(), # type: ignore[arg-type] - ("127.0.0.1", connection_id), - connection_id, - f"{connection_id:08x}", - ) - - -class FakeProgressService: - """G1 stand-in that records callback-time registry proof visibility.""" - - def __init__(self, registry: SessionRegistry) -> None: - self.registry = registry - self.proofs: list[object] = [] - self.registry_had_proof_at_callback: list[bool] = [] - - def record_delivery( - self, - proof: object, - ready_mode_required: bool, - *, - matches_latest_tip: bool, - ) -> bool: - snapshot = self.registry.eligible_snapshot() - delivered = snapshot.get(proof.connection_id) # type: ignore[union-attr] - self.registry_had_proof_at_callback.append( - delivered is not None and delivered.delivered is not None - ) - self.proofs.append(proof) - return True - - -class FakeSessionRuntime: - """Recording call-time SessionRuntime port for the session service.""" - - def __init__(self) -> None: - self.lock = threading.RLock() - self.stop_event = threading.Event() - self.stratum_max_connections = 8 - self.stratum_max_connections_per_username = 1 - self._pool_ready_latched = False - self.latest_detected_tip = None - self.rpc = SimpleNamespace( - call=lambda _method, _params: { - "isvalid": True, - "scriptPubKey": "5220" + "11" * 32, - } - ) - self.service: StratumSessionService | None = None - self.progress: FakeProgressService | None = None - self.cancelled: list[ClientState] = [] - self.cleaned: list[ClientState] = [] - self.retained = 0 - self.activity: list[float] = [] - self.reconciles: list[float] = [] - self.coverage: list[object] = [] - self.send_timeout_error: BaseException | None = None - self._controller = SimpleNamespace(phase="running") - - # -- lifecycle plumbing ------------------------------------------------- - - def _record_heartbeat(self, _name: str, *, phase: str | None = None) -> None: - return None - - def _ensure_shutdown_controller(self) -> object: - return self._controller - - def _ensure_initial_job_state(self) -> None: - return None - - def client_startup_difficulty(self, _profile: StratumListenerProfile) -> Decimal: - return Decimal("1") - - def apply_stratum_send_timeout(self, _sock: object) -> None: - if self.send_timeout_error is not None: - raise self.send_timeout_error - - def handle_client(self, _client: ClientState) -> None: - return None - - def disconnect_client(self, client: ClientState) -> None: - assert self.service is not None - self.service.disconnect_client(client) - - def _cancel_pending_initial_job_locked( - self, client: ClientState, *, count: bool - ) -> None: - self.cancelled.append(client) - - def _ensure_job_delivery_service(self) -> object: - return SimpleNamespace(cleanup_disconnected_client=self.cleaned.append) - - def _retain_current_collection_refresh_if_unrepresented(self) -> None: - self.retained += 1 - - def _record_stratum_resource_exhaustion(self, **_kwargs: object) -> None: - return None - - def _wait_after_stratum_resource_failure(self, _heartbeat_name: str) -> None: - return None - - # -- delivery/progress plumbing ---------------------------------------- - - def _ensure_job_cache_state(self) -> None: - return None - - def _ensure_progress_health_service(self) -> object: - assert self.progress is not None - return self.progress - - def _tip_refresh_epoch_coverage_reached_locked( - self, - _client: ClientState, - _context: object, - _delivered_monotonic: float, - ) -> list[tuple[str, float]]: - return [] - - def _progress_note_refresh_activity(self, delivered_monotonic: float) -> None: - self.activity.append(delivered_monotonic) - - def _progress_reconcile_pending(self, *, now: float) -> None: - self.reconciles.append(now) - - def _record_tip_refresh_epoch_coverage(self, coverage: object) -> None: - self.coverage.append(coverage) - - def _handle_request(self, _client: ClientState, _request: dict[str, object]) -> None: - raise _DispatchShutdown("writer admission closed") - - -class _DispatchShutdown(RuntimeError): - pass - - -def service_fixture() -> tuple[StratumSessionService, SessionRegistry, FakeSessionRuntime]: - runtime = FakeSessionRuntime() - service = StratumSessionService( - runtime, - shutdown_error=_DispatchShutdown, - pool_closed_reason="pool-closed", - ) - runtime.service = service - runtime.progress = FakeProgressService(service.registry) - return service, service.registry, runtime - - -class SessionRegistryTests(unittest.TestCase): - def test_registry_adopts_seeded_clients_and_derives_generation_and_counts( - self, - ) -> None: - early = client_state(3) - late = client_state(7) - late.handler_thread_registered = True - - registry = SessionRegistry(clients={early, late}) - - self.assertEqual(registry.connection_generation, 7) - self.assertEqual(registry.handler_thread_count, 1) - self.assertEqual(registry.peak_active_connections, 2) - self.assertEqual(registry.note_rejection_locked("global"), 1) - self.assertEqual(registry.note_rejection_locked("global"), 2) - self.assertEqual(registry.rejection_counts["global"], 2) - - def test_eligibility_snapshot_is_immutable_exact_and_uses_delivered_context( - self, - ) -> None: - current = client_state(1) - current.subscribed = current.authorized = True - current.worker = worker("miner") - current.username = "miner" - idle = client_state(2) - registry = SessionRegistry(clients=[current, idle]) - delivered = SimpleNamespace(name="delivered") - current.active_job = SimpleNamespace(name="registered-before-send") - registry.record_delivery_locked(current, delivered, 42.0) - current.active_job = SimpleNamespace(name="newer-unsent") - - snapshot = registry.eligible_snapshot() - - self.assertEqual(tuple(snapshot), (1,)) - self.assertIs(snapshot[1].delivered.context, delivered) # type: ignore[union-attr] - self.assertTrue(client_can_receive_jobs(current)) - with self.assertRaises(TypeError): - snapshot[3] = snapshot[1] # type: ignore[index] - - # Reauthorization does not erase a valid socket-delivery proof. - current.worker = worker("replacement") - self.assertIs( - registry.eligible_snapshot()[1].delivered.context, # type: ignore[union-attr] - delivered, - ) - registry.clients = set(registry.clients) - self.assertTrue(registry.begin_retirement_locked(current)) - self.assertFalse(registry.eligible_snapshot()) - - def test_retirement_claim_is_single_owner_and_clears_delivery_proof(self) -> None: - state = client_state(1) - registry = SessionRegistry(clients={state}) - registry.record_delivery_locked(state, SimpleNamespace(name="sent"), 1.0) - self.assertIn(1, registry._delivered_by_connection) - - self.assertTrue(registry.begin_retirement_locked(state)) - self.assertNotIn(state, registry.clients) - self.assertEqual(registry._delivered_by_connection, {}) - self.assertFalse(registry.begin_retirement_locked(state)) - - def test_ordered_membership_can_be_adopted_before_and_after_use(self) -> None: - first = client_state(1) - second = client_state(2) - registry = SessionRegistry(clients=[first, second]) - self.assertIsInstance(registry.clients, list) - self.assertEqual(registry.clients, [first, second]) - - replacement = [second, first] - registry.adopt_clients(replacement) - - self.assertIs(registry.clients, replacement) - self.assertEqual(registry.clients, [second, first]) - - def test_coordinator_registry_adopts_ordered_membership_replacement(self) -> None: - first = client_state(1) - second = client_state(2) - coordinator = PrismCoordinator.__new__(PrismCoordinator) - coordinator.lock = threading.RLock() - coordinator.connection_counter = 0 - coordinator.clients = [first, second] # type: ignore[assignment] - - registry = coordinator._ensure_stratum_session_service().registry - self.assertIs(registry.clients, coordinator.clients) - self.assertEqual(registry.connection_generation, 2) - first.handler_thread_registered = True - replacement = [second, first] - coordinator.clients = replacement # type: ignore[assignment] - - self.assertIs( - coordinator._ensure_stratum_session_service().registry, registry - ) - self.assertIs(registry.clients, replacement) - self.assertEqual(registry.clients, [second, first]) - self.assertEqual(registry.handler_thread_count, 1) - self.assertEqual(coordinator.connection_counter, 2) - self.assertEqual(coordinator.handler_thread_count, 1) - - -class SessionLifecycleTests(unittest.TestCase): - def test_username_reservation_enforces_capacity_and_preserves_identity( - self, - ) -> None: - service, registry, _runtime = service_fixture() - first = client_state(1) - occupant = client_state(2) - registry.clients.update({first, occupant}) - original = worker("original") - full = worker("full") - - self.assertTrue(service.reserve_client_username(first, original)) - self.assertTrue(service.reserve_client_username(occupant, full)) - - with patch("builtins.print"): - accepted = service.reserve_client_username(first, full) - - self.assertFalse(accepted) - self.assertIs(first.worker, original) - self.assertEqual(first.username, "original") - self.assertEqual(registry.rejection_counts["username"], 1) - - def test_handler_setup_failure_rolls_back_membership_and_socket(self) -> None: - service, registry, runtime = service_fixture() - runtime.send_timeout_error = RuntimeError("socket setup unavailable") - accepted = FakeSocket() - - class Listener: - calls = 0 - - def accept(self) -> tuple[FakeSocket, tuple[str, int]]: - self.calls += 1 - if self.calls == 1: - return accepted, ("127.0.0.1", 1) - runtime.stop_event.set() - raise socket.timeout - - with patch("builtins.print"): - service.accept_loop(Listener(), listener()) # type: ignore[arg-type] - - self.assertTrue(accepted.closed.is_set()) - self.assertFalse(registry.clients) - self.assertEqual(registry.handler_thread_count, 0) - self.assertEqual(service.connection_setup_failure_count, 1) - - def test_disconnect_closes_socket_before_waiting_for_job_update_lock(self) -> None: - service, registry, runtime = service_fixture() - state = client_state(1) - sock = state.sock - registry.clients.add(state) - state.job_update_lock.acquire() - finished = threading.Event() - - def disconnect() -> None: - service.disconnect_client(state) - finished.set() - - thread = threading.Thread(target=disconnect) - thread.start() - self.assertTrue(sock.closed.wait(1.0)) # type: ignore[union-attr] - self.assertFalse(finished.is_set()) - self.assertNotIn(state, registry.clients) - state.job_update_lock.release() - thread.join(1.0) - - self.assertTrue(finished.is_set()) - self.assertEqual(runtime.cancelled, [state]) - self.assertEqual(runtime.cleaned, [state]) - self.assertEqual(runtime.retained, 1) - # Only the first caller owns socket close and final cleanup. - service.disconnect_client(state) - self.assertEqual(runtime.cleaned, [state]) - - def test_successful_delivery_records_registry_proof_before_health_callback( - self, - ) -> None: - service, registry, runtime = service_fixture() - progress = runtime.progress - assert progress is not None - state = client_state(1) - state.subscribed = state.authorized = True - state.worker = worker("miner") - state.username = "miner" - registry.clients.add(state) - context = SimpleNamespace( - name="sent", - template={"previousblockhash": "00" * 32}, - template_fingerprint="fp-1", - template_generation=4, - payout_state_generation=2, - collection_only=False, - ) - - service.record_successful_delivery(state, context, 12.5) - - self.assertIs( - registry.eligible_snapshot()[1].delivered.context, # type: ignore[union-attr] - context, - ) - self.assertEqual(progress.registry_had_proof_at_callback, [True]) - self.assertEqual(len(progress.proofs), 1) - proof = progress.proofs[0] - self.assertEqual(proof.connection_id, 1) # type: ignore[union-attr] - self.assertEqual(proof.delivered_monotonic, 12.5) # type: ignore[union-attr] - # The staged-split compatibility mirrors advance in the same critical - # section as the registry proof. - self.assertIs(state._progress_delivered_context, context) - self.assertEqual(state._progress_delivered_template_generation, 4) - self.assertEqual(runtime.activity, [12.5]) - self.assertEqual(runtime.reconciles, [12.5]) - self.assertEqual(runtime.coverage, [[]]) - - def test_retired_client_leaves_no_eligible_snapshot_entry(self) -> None: - service, registry, _runtime = service_fixture() - state = client_state(1) - state.subscribed = state.authorized = True - state.worker = worker("miner") - registry.clients.add(state) - self.assertTrue(registry.begin_retirement_locked(state)) - - registry.record_delivery_locked( - state, SimpleNamespace(name="sent-before-retirement-won"), 12.5 - ) - - # A retired connection is invisible to fanout eligibility even if a - # racing delivery commits proof after retirement claimed the client. - self.assertFalse(registry.eligible_snapshot()) - - def test_shutdown_race_in_dispatch_maps_to_pool_closed_stratum_error(self) -> None: - service, _registry, _runtime = service_fixture() - state = client_state(1) - - with self.assertRaises(StratumError) as caught: - service.handle_request(state, {"id": 1, "method": "mining.submit"}) - - error = caught.exception - self.assertEqual(error.code, 20) - self.assertEqual(error.reason, "pool-closed") - self.assertTrue(error.disconnect) - self.assertIsInstance(error.__cause__, _DispatchShutdown) - - -class CompatibilityTests(unittest.TestCase): - def test_coordinator_reexports_exact_session_model_identities(self) -> None: - self.assertIs(FacadeClientState, ClientState) - self.assertIs(FacadeWorkerIdentity, WorkerIdentity) - self.assertIs(FacadeStratumError, StratumError) - self.assertFalse(hasattr(stratum_session, "PrismCoordinator")) - - def test_protocol_helpers_preserve_payload_and_password_behavior(self) -> None: - self.assertEqual(result_payload(1, True), {"id": 1, "result": True, "error": None}) - self.assertEqual( - error_payload(2, 21, "stale", reason="stale-job"), - { - "id": 2, - "result": None, - "error": [21, "stale", {"reason_id": "stale-job"}], - }, - ) - self.assertEqual( - parse_stratum_password_options("x,md=4,d=8,bad=1"), - (Decimal("8"), Decimal("4")), - ) - - -if __name__ == "__main__": - unittest.main() diff --git a/tests/test_prism_streamed_block_hex_consumers.py b/tests/test_prism_streamed_block_hex_consumers.py deleted file mode 100644 index 7624a4b13..000000000 --- a/tests/test_prism_streamed_block_hex_consumers.py +++ /dev/null @@ -1,120 +0,0 @@ -"""Streamed strings from a hydrated body survive every replay consumer (#255).""" - -from dataclasses import replace -import json -import threading -import unittest -from types import SimpleNamespace - -from lab.auxpow.stratum_codec import header_hash_hex -from lab.prism import candidate_codec as codec -from lab.prism import candidate_spool_view as views -from lab.prism import recover_pending_blocks as recovery -from lab.prism.block_candidates import block_candidate_from_intent, block_candidate_intent -from lab.prism.share_ledger import PsqlShareLedger, block_candidate_identity_sha256 -from tests.test_prism_candidate_spool_view import SpoolFixture, intent_for - -THRESHOLD = 32 * 1024 -HEADER = bytes([7]) * 80 -PENDING_SHARE = { - "share_id": "u:" + "ab" * 32, - "miner_id": "m", - "order_key": "m", - "p2mr_program_hex": "ab" * 32, - "share_difficulty": 100, - "network_difficulty": 1000, - "template_height": 10, - "job_id": "job", - "job_issued_at_ms": 100, - "accepted_at_ms": 123, - "ntime": 1_700_000_000, - "credit_policy": None, -} - - -def credited_fields() -> dict: - # Above the lowered threshold the hydrated block_hex is a streamed view, - # as a raw block of roughly 512 KiB is at the production threshold. - return intent_for( - 3, - block_hash_hex=header_hash_hex(HEADER), - block_hex=HEADER.hex() + "ab" * 40_000, - pending_share=dict(PENDING_SHARE), - credit_share_on_accept=True, - ) - - -class StreamedBlockHexConsumerTests(SpoolFixture): - def test_replayed_credit_intent_re_encodes_a_streamed_block(self): - fields = credited_fields() - with self.bounded(THRESHOLD): - hydrated, _body = self.hydrate(fields) - self.assertIsInstance(hydrated["block_hex"], views.SpoolStringView) - replayed = replace(block_candidate_from_intent(hydrated), durable_replay=True) - plain = replace(block_candidate_from_intent(fields), durable_replay=True) - self.assertEqual( - block_candidate_intent(replayed).candidate_sha256, - block_candidate_intent(plain).candidate_sha256, - ) - - def test_streamed_username_is_decoded_for_the_worker_identity(self): - fields = {**credited_fields(), "username": "q" * 40_000 + ".worker"} - with self.bounded(THRESHOLD): - hydrated, _body = self.hydrate(fields) - self.assertIsInstance(hydrated["username"], views.SpoolStringView) - replayed = replace(block_candidate_from_intent(hydrated), durable_replay=True) - self.assertEqual(replayed.context.worker.username, fields["username"]) - self.assertEqual(replayed.client.username, fields["username"]) - plain = replace(block_candidate_from_intent(fields), durable_replay=True) - self.assertEqual( - block_candidate_intent(replayed).candidate_sha256, - block_candidate_intent(plain).candidate_sha256, - ) - - def test_offline_recovery_reads_the_header_of_a_streamed_block(self): - fields = credited_fields() - block = recovery.RecoveryBlock(fields["block_hash_hex"], 10, fields["parent_hash"], "pending") - with self.bounded(THRESHOLD): - hydrated, _body = self.hydrate(fields) - coordinator = SimpleNamespace( - block_candidate_from_intent=block_candidate_from_intent, - ledger=SimpleNamespace(hydrate_block_candidate_intent=lambda _row, **_kwargs: hydrated), - stop_event=threading.Event(), - ) - row = { - "candidate": None, - "candidate_sha256": hydrated.candidate_sha256, - "storage_version": 2, - "body_id": "ab" * 16, - "replay_header": {}, - "byte_count": 0, - "chunk_count": 0, - "chunk_bytes": 0, - "share_count": 0, - "body_state": "sealed", - } - candidate = recovery.decode_candidate(coordinator, block, row) - self.assertEqual(candidate.submission.block_hash_hex, block.block_hash) - self.assertIsInstance(candidate.submission.block_hex, views.SpoolStringView) - - def test_legacy_document_omits_absent_shares_and_materializes_views(self): - candidate_only = intent_for(0) - del candidate_only["shares_json"] - prepared = codec.prepare_candidate_intent(candidate_only) - self.assertFalse(prepared.has_shares) - document = PsqlShareLedger._legacy_candidate_document(prepared) - self.assertNotIn("shares_json", document) - self.assertEqual(block_candidate_identity_sha256(document), prepared.candidate_sha256) - - fields = credited_fields() - with self.bounded(THRESHOLD): - hydrated, _body = self.hydrate(fields) - document = PsqlShareLedger._legacy_candidate_document(hydrated) - self.assertIsInstance(document["block_hex"], str) - self.assertEqual(document["block_hex"], fields["block_hex"]) - json.dumps(document) - self.assertEqual(block_candidate_identity_sha256(document), hydrated.candidate_sha256) - - -if __name__ == "__main__": - unittest.main() diff --git a/tests/test_prism_streamed_submitblock.py b/tests/test_prism_streamed_submitblock.py deleted file mode 100644 index fbfdfe059..000000000 --- a/tests/test_prism_streamed_submitblock.py +++ /dev/null @@ -1,85 +0,0 @@ -"""Real HTTP transport for a replayed block whose hex stays on disk.""" - -import hashlib -from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer -import json -import threading -import tracemalloc -import unittest - -from lab.prism.rpc import JsonRpc - - -class StreamedSubmitblockTests(unittest.TestCase): - def test_large_block_uses_content_length_and_bounded_stream(self): - class Block: - byte_length = 2 + 256 * 65536 - - def iter_byte_chunks(self): - yield b'"' - for _ in range(256): - yield b"ab" * 32768 - yield b'"' - - def __str__(self): - raise AssertionError("block was materialized") - - observed = [] - - class Handler(BaseHTTPRequestHandler): - def do_POST(self): - length = int(self.headers["Content-Length"]) - digest = hashlib.sha256() - remaining = length - while remaining: - chunk = self.rfile.read(min(65536, remaining)) - if not chunk: - break - digest.update(chunk) - remaining -= len(chunk) - observed.append((length, remaining, digest.hexdigest(), self.headers.get("Transfer-Encoding"))) - payload = b'{"result":null,"error":null,"id":"submitblock"}' - self.send_response(200) - self.send_header("Content-Length", str(len(payload))) - self.end_headers() - try: - self.wfile.write(payload) - except OSError: - pass # A deliberately interrupted upload closed its socket. - - def log_message(self, *args): - pass - - server = ThreadingHTTPServer(("127.0.0.1", 0), Handler) - thread = threading.Thread(target=server.serve_forever) - thread.start() - rpc = JsonRpc(host="127.0.0.1", port=server.server_port, user="u", password="p") - try: - tracemalloc.start() - try: - self.assertIsNone(rpc.call("submitblock", [Block()])) - _, peak = tracemalloc.get_traced_memory() - finally: - tracemalloc.stop() - expected = json.dumps({"jsonrpc": "1.0", "id": "submitblock", "method": "submitblock", - "params": ["ab" * (256 * 32768)]}, separators=(",", ":")).encode() - self.assertEqual(observed, [(len(expected), 0, hashlib.sha256(expected).hexdigest(), None)]) - self.assertLess(peak, 2 * 1024 * 1024) - - class FailedBlock(Block): - def iter_byte_chunks(self): - yield b'"ab' - raise ValueError("source closed") - - with self.assertRaisesRegex(ValueError, "source closed"): - rpc.call("submitblock", [FailedBlock()]) - self.assertIsNone(rpc._connections.conn) - finally: - rpc._drop_connection() - server.shutdown() - server.server_close() - thread.join() - - -if __name__ == "__main__": - unittest.main() diff --git a/tests/test_prism_tip_publication_boundary.py b/tests/test_prism_tip_publication_boundary.py deleted file mode 100644 index 3b7bcc8a6..000000000 --- a/tests/test_prism_tip_publication_boundary.py +++ /dev/null @@ -1,966 +0,0 @@ -#!/usr/bin/env python3 -"""Regressions for detected-tip versus published-tip refresh authority.""" - -from __future__ import annotations - -import dataclasses -import time -import types -import unittest - -from lab.prism.prism_coordinator import ( - TemplateRefreshBlocked, - _JobBuildCancellation, -) -from tests.prism_coordinator_test_support import ( - base_template, - client, - coordinator, - install_fake_bundle_builder, -) - - -class TipPublicationBoundaryTests(unittest.TestCase): - def test_final_validation_failure_keeps_previous_authority_and_snapshot(self) -> None: - server, rpc = coordinator() - builds = install_fake_bundle_builder(server) - state = client(1) - state.send = lambda _payload: None # type: ignore[method-assign] - server.clients = [state] # type: ignore[assignment] - server.ensure_reorg_reconciled_for_tip = lambda _tip: True # type: ignore[method-assign] - server.qbit_chain_view_untrusted = lambda: False # type: ignore[method-assign] - try: - self.assertEqual(server.poll_qbit_tip_template_once(), 1) - published_tip = server.current_tip_first_seen - published_snapshot = server.tip_template_snapshot - initial_builds = int(builds["calls"]) - - next_tip = "33" * 32 - rpc.tip = next_tip - rpc.template = base_template(height=11, prevhash=next_tip) - validation_reached = False - - def fail_final_validation(*_args: object, **_kwargs: object) -> object: - nonlocal validation_reached - validation_reached = True - raise TemplateRefreshBlocked("forced final validation failure") - - server._validate_prepared_tip_refresh = fail_final_validation # type: ignore[method-assign] - - with self.assertRaisesRegex( - TemplateRefreshBlocked, - "forced final validation failure", - ): - server.poll_qbit_tip_template_once() - - self.assertTrue(validation_reached) - self.assertGreater(int(builds["calls"]), initial_builds) - self.assertEqual(server.current_tip_first_seen, published_tip) - self.assertIs(server.tip_template_snapshot, published_snapshot) - self.assertEqual(server.latest_detected_tip[0], next_tip) - self.assertIsNone(server._active_tip_refresh) - finally: - server.shutdown_tip_refresh_executor() - - def test_newer_detection_before_atomic_activation_prevents_publication(self) -> None: - server, rpc = coordinator() - install_fake_bundle_builder(server) - state = client(1) - sent_tips: list[str] = [] - - def record_send(payload: dict[str, object]) -> None: - if payload["method"] == "mining.notify": - assert state.active_job is not None - sent_tips.append( - str(state.active_job.template["previousblockhash"]) - ) - - state.send = record_send # type: ignore[method-assign] - server.clients = [state] # type: ignore[assignment] - server.ensure_reorg_reconciled_for_tip = lambda _tip: True # type: ignore[method-assign] - server.qbit_chain_view_untrusted = lambda: False # type: ignore[method-assign] - try: - self.assertEqual(server.poll_qbit_tip_template_once(), 1) - published_tip = server.current_tip_first_seen - published_snapshot = server.tip_template_snapshot - self.assertEqual(sent_tips, [rpc.tip]) - - candidate_tip = "44" * 32 - rpc.tip = candidate_tip - rpc.template = base_template(height=11, prevhash=candidate_tip) - sequence = server._reserve_tip_observation_sequence() - self.assertTrue( - server.observe_tip_for_refresh( - candidate_tip, - observation_sequence=sequence, - mark_pending=False, - ) - ) - snapshot = server.fetch_qbit_tip_template_snapshot() - bundle = server.prepare_tip_refresh_bundle(snapshot) - token = server._validate_prepared_tip_refresh( - bundle, - snapshot, - sequence, - ) - - winning_tip = "55" * 32 - rpc.tip = winning_tip - rpc.template = base_template(height=12, prevhash=winning_tip) - self.assertTrue(server.observe_tip_for_refresh(winning_tip)) - - with self.assertRaisesRegex( - TemplateRefreshBlocked, - "superseded before atomic publication", - ): - server._publish_prepared_tip_refresh( - token, - bundle, - snapshot, - parent_hash=None, - ) - - self.assertEqual(server.current_tip_first_seen, published_tip) - self.assertIs(server.tip_template_snapshot, published_snapshot) - self.assertEqual(server.latest_detected_tip[0], winning_tip) - self.assertEqual(len(sent_tips), 1) - self.assertIsNone(server._active_tip_refresh) - finally: - server.shutdown_tip_refresh_executor() - - def test_executor_failure_happens_before_prepared_publication(self) -> None: - server, rpc = coordinator() - install_fake_bundle_builder(server) - state = client(1) - state.send = lambda _payload: None # type: ignore[method-assign] - server.clients = [state] # type: ignore[assignment] - server.ensure_reorg_reconciled_for_tip = lambda _tip: True # type: ignore[method-assign] - server.qbit_chain_view_untrusted = lambda: False # type: ignore[method-assign] - try: - self.assertEqual(server.poll_qbit_tip_template_once(), 1) - published_tip = server.current_tip_first_seen - published_snapshot = server.tip_template_snapshot - - next_tip = "66" * 32 - rpc.tip = next_tip - rpc.template = base_template(height=11, prevhash=next_tip) - - def fail_executor() -> object: - raise RuntimeError("executor unavailable") - - server.tip_refresh_executor = fail_executor # type: ignore[method-assign] - with self.assertRaisesRegex(RuntimeError, "executor unavailable"): - server.poll_qbit_tip_template_once() - - self.assertEqual(server.current_tip_first_seen, published_tip) - self.assertIs(server.tip_template_snapshot, published_snapshot) - self.assertEqual(server.latest_detected_tip[0], next_tip) - self.assertIsNone(server._active_tip_refresh) - finally: - server.shutdown_tip_refresh_executor() - - def test_payout_mutation_before_publication_keeps_previous_authority(self) -> None: - server, rpc = coordinator() - install_fake_bundle_builder(server) - state = client(1) - state.send = lambda _payload: None # type: ignore[method-assign] - server.clients = [state] # type: ignore[assignment] - server.ensure_reorg_reconciled_for_tip = lambda _tip: True # type: ignore[method-assign] - server.qbit_chain_view_untrusted = lambda: False # type: ignore[method-assign] - try: - self.assertEqual(server.poll_qbit_tip_template_once(), 1) - published_tip = server.current_tip_first_seen - published_snapshot = server.tip_template_snapshot - - candidate_tip = "77" * 32 - rpc.tip = candidate_tip - rpc.template = base_template(height=11, prevhash=candidate_tip) - sequence = server._reserve_tip_observation_sequence() - self.assertTrue( - server.observe_tip_for_refresh( - candidate_tip, - observation_sequence=sequence, - mark_pending=False, - ) - ) - snapshot = server.fetch_qbit_tip_template_snapshot() - bundle = server.prepare_tip_refresh_bundle(snapshot) - token = server._validate_prepared_tip_refresh( - bundle, - snapshot, - sequence, - ) - self.assertEqual(server._advance_payout_state_generation(), 1) - - with self.assertRaisesRegex( - TemplateRefreshBlocked, - "superseded before atomic publication", - ): - server._publish_prepared_tip_refresh( - token, - bundle, - snapshot, - parent_hash=None, - ) - - self.assertEqual(server.current_tip_first_seen, published_tip) - self.assertIs(server.tip_template_snapshot, published_snapshot) - self.assertIsNone(server._active_tip_refresh) - finally: - server.shutdown_tip_refresh_executor() - - def test_payout_publication_block_keeps_previous_authority(self) -> None: - server, rpc = coordinator() - install_fake_bundle_builder(server) - state = client(1) - state.send = lambda _payload: None # type: ignore[method-assign] - server.clients = [state] # type: ignore[assignment] - server.ensure_reorg_reconciled_for_tip = lambda _tip: True # type: ignore[method-assign] - server.qbit_chain_view_untrusted = lambda: False # type: ignore[method-assign] - try: - self.assertEqual(server.poll_qbit_tip_template_once(), 1) - published_tip = server.current_tip_first_seen - published_snapshot = server.tip_template_snapshot - - candidate_tip = "88" * 32 - rpc.tip = candidate_tip - rpc.template = base_template(height=11, prevhash=candidate_tip) - sequence = server._reserve_tip_observation_sequence() - self.assertTrue( - server.observe_tip_for_refresh( - candidate_tip, - observation_sequence=sequence, - mark_pending=False, - ) - ) - snapshot = server.fetch_qbit_tip_template_snapshot() - bundle = server.prepare_tip_refresh_bundle(snapshot) - token = server._validate_prepared_tip_refresh( - bundle, - snapshot, - sequence, - ) - - server._block_payout_state_publication() - self.assertTrue(server._payout_state_publication_blocked) - with self.assertRaisesRegex( - TemplateRefreshBlocked, - "superseded before atomic publication", - ): - server._publish_prepared_tip_refresh( - token, - bundle, - snapshot, - parent_hash=None, - ) - - self.assertEqual(server.current_tip_first_seen, published_tip) - self.assertIs(server.tip_template_snapshot, published_snapshot) - self.assertIsNone(server._active_tip_refresh) - finally: - server.shutdown_tip_refresh_executor() - - def test_tip_publication_preserves_first_payout_delivery_priority(self) -> None: - server, rpc = coordinator() - install_fake_bundle_builder(server) - state = client(1) - state.send = lambda _payload: None # type: ignore[method-assign] - server.clients = [state] # type: ignore[assignment] - server.ensure_reorg_reconciled_for_tip = lambda _tip: True # type: ignore[method-assign] - server.qbit_chain_view_untrusted = lambda: False # type: ignore[method-assign] - try: - self.assertEqual(server.poll_qbit_tip_template_once(), 1) - server._reserve_payout_state_source("payout_only") - self.assertEqual( - server._publish_payout_state_candidate( - server._current_payout_state_candidate() - ), - 1, - ) - self.assertEqual( - server._payout_state_delivery_gate._priority_generation, - 1, - ) - - sequence = server._reserve_tip_observation_sequence() - self.assertTrue( - server.observe_tip_for_refresh( - rpc.tip, - observation_sequence=sequence, - mark_pending=False, - ) - ) - snapshot = server.fetch_qbit_tip_template_snapshot() - bundle = server.prepare_tip_refresh_bundle(snapshot) - token = server._validate_prepared_tip_refresh( - bundle, - snapshot, - sequence, - ) - cancellation = server._publish_prepared_tip_refresh( - token, - bundle, - snapshot, - parent_hash=None, - ) - try: - self.assertEqual( - server._payout_state_delivery_gate._priority_generation, - 1, - ) - with server._payout_state_delivery_gate.delivery_cancelable( - lambda: False, - generation=1, - priority=False, - ) as routine_admission: - self.assertFalse(routine_admission) - finally: - server._clear_active_tip_refresh(token, cancellation) - finally: - server.shutdown_tip_refresh_executor() - - def test_same_tip_republication_preserves_parent_on_lookup_failure(self) -> None: - server, rpc = coordinator() - install_fake_bundle_builder(server) - state = client(1) - state.send = lambda _payload: None # type: ignore[method-assign] - server.clients = [state] # type: ignore[assignment] - server.ensure_reorg_reconciled_for_tip = lambda _tip: True # type: ignore[method-assign] - server.qbit_chain_view_untrusted = lambda: False # type: ignore[method-assign] - try: - self.assertEqual(server.poll_qbit_tip_template_once(), 1) - published_tip = server.current_tip_first_seen - assert published_tip is not None - cached_parent = (published_tip[0], "aa" * 32) - with server.lock: - server.current_tip_parent = cached_parent - - sequence = server._reserve_tip_observation_sequence() - snapshot = server.fetch_qbit_tip_template_snapshot() - bundle = server.prepare_tip_refresh_bundle(snapshot) - token = server._validate_prepared_tip_refresh(bundle, snapshot, sequence) - cancellation = server._publish_prepared_tip_refresh( - token, - bundle, - snapshot, - parent_hash=None, - ) - try: - # A transient parent lookup failure during a same-tip - # republication must not wipe the still-valid cached parent. - self.assertEqual(server.current_tip_parent, cached_parent) - self.assertEqual(server.current_tip_first_seen[0], published_tip[0]) - finally: - server._clear_active_tip_refresh(token, cancellation) - finally: - server.shutdown_tip_refresh_executor() - - def test_direct_issuance_pins_published_snapshot_during_unpublished_window(self) -> None: - server, rpc = coordinator() - install_fake_bundle_builder(server) - state = client(1) - state.send = lambda _payload: None # type: ignore[method-assign] - server.clients = [state] # type: ignore[assignment] - server.ensure_reorg_reconciled_for_tip = lambda _tip: True # type: ignore[method-assign] - server.qbit_chain_view_untrusted = lambda: False # type: ignore[method-assign] - try: - self.assertEqual(server.poll_qbit_tip_template_once(), 1) - published_tip = server.current_tip_first_seen - assert published_tip is not None - - # Detect a replacement tip and let its template fetch install the - # new global artifacts, exactly like an in-flight refresh does - # before publication. - next_tip = "77" * 32 - rpc.tip = next_tip - rpc.template = base_template(height=11, prevhash=next_tip) - sequence = server._reserve_tip_observation_sequence() - self.assertTrue( - server.observe_tip_for_refresh( - next_tip, - observation_sequence=sequence, - mark_pending=False, - ) - ) - server.fetch_qbit_tip_template_snapshot() - with server._job_cache_lock: - self.assertEqual( - server._template_artifacts.previousblockhash, - next_tip, - ) - - # Direct issuance stays pinned to the published snapshot while the - # published tip still owns share classification, so the issued job - # is immediately creditable. - artifacts = server.job_issuance_template_artifacts() - self.assertEqual(artifacts.previousblockhash, published_tip[0]) - context = server.build_job_for_client(state, clean_jobs=False) - self.assertEqual( - str(context.template["previousblockhash"]), - published_tip[0], - ) - - # Delivery-side currency accepts the pinned published snapshot - # even though the detected-tip globals have moved on, so initial - # delivery does not defer for the construction window. - self.assertFalse(server._template_artifacts_are_current(artifacts)) - self.assertTrue(server._issuance_artifacts_current(artifacts)) - - # A pruned bundle cache (payout mutation, LRU pressure) must not - # strand pinned issuance either: published-parent work stays - # buildable and cacheable while its authority holds. - with server._job_cache_lock: - server._job_bundle_cache.clear() - rebuilt = server.build_job_for_client(state, clean_jobs=False) - self.assertEqual( - str(rebuilt.template["previousblockhash"]), - published_tip[0], - ) - - # Once the published authority lapses, issuance falls through to - # the live template, mirroring the submit-path RPC fallback. - server.current_tip_observed_monotonic = ( - time.monotonic() - - float(getattr(server, "submit_tip_max_age_seconds", 10.0)) - - 1.0 - ) - server.template_refresh_failure_exit_seconds = 0.0 - lapsed = server.job_issuance_template_artifacts() - self.assertEqual(lapsed.previousblockhash, next_tip) - finally: - server.shutdown_tip_refresh_executor() - - def test_refresh_retry_reuses_cached_detected_tip_bundle(self) -> None: - server, rpc = coordinator() - builds = install_fake_bundle_builder(server) - state = client(1) - state.send = lambda _payload: None # type: ignore[method-assign] - server.clients = [state] # type: ignore[assignment] - server.ensure_reorg_reconciled_for_tip = lambda _tip: True # type: ignore[method-assign] - server.qbit_chain_view_untrusted = lambda: False # type: ignore[method-assign] - try: - self.assertEqual(server.poll_qbit_tip_template_once(), 1) - - next_tip = "88" * 32 - rpc.tip = next_tip - rpc.template = base_template(height=11, prevhash=next_tip) - sequence = server._reserve_tip_observation_sequence() - self.assertTrue( - server.observe_tip_for_refresh( - next_tip, - observation_sequence=sequence, - mark_pending=False, - ) - ) - snapshot = server.fetch_qbit_tip_template_snapshot() - first_bundle = server.prepare_tip_refresh_bundle(snapshot) - builds_after_first = int(builds["calls"]) - - # A transient final-validation or publication failure retries the - # same snapshot. The detected-tip bundle already in the cache must - # be reused while the previous tip is still published, not rebuilt - # from scratch for the rest of the divergence lease. - retry_bundle = server.prepare_tip_refresh_bundle(snapshot) - self.assertIs(retry_bundle, first_bundle) - self.assertEqual(int(builds["calls"]), builds_after_first) - finally: - server.shutdown_tip_refresh_executor() - - def test_pinned_published_rebuild_recaches_for_repeat_issuance(self) -> None: - server, rpc = coordinator() - builds = install_fake_bundle_builder(server) - state = client(1) - state.send = lambda _payload: None # type: ignore[method-assign] - server.clients = [state] # type: ignore[assignment] - server.ensure_reorg_reconciled_for_tip = lambda _tip: True # type: ignore[method-assign] - server.qbit_chain_view_untrusted = lambda: False # type: ignore[method-assign] - try: - self.assertEqual(server.poll_qbit_tip_template_once(), 1) - published_tip = server.current_tip_first_seen - assert published_tip is not None - - next_tip = "99" * 32 - rpc.tip = next_tip - rpc.template = base_template(height=11, prevhash=next_tip) - self.assertTrue(server.observe_tip_for_refresh(next_tip)) - server.fetch_qbit_tip_template_snapshot() - - # Simulate cache pressure dropping the retained published bundle, - # then issue twice: the first pinned rebuild re-enters the cache - # so the second issuance is a hit instead of another heavy build. - with server._job_cache_lock: - server._job_bundle_cache.clear() - first = server.build_job_for_client(state, clean_jobs=False) - builds_after_first = int(builds["calls"]) - second = server.build_job_for_client(state, clean_jobs=False) - self.assertEqual(int(builds["calls"]), builds_after_first) - self.assertEqual( - str(first.template["previousblockhash"]), - published_tip[0], - ) - self.assertEqual( - str(second.template["previousblockhash"]), - published_tip[0], - ) - finally: - server.shutdown_tip_refresh_executor() - - def test_direct_send_skips_pinned_work_published_over_mid_flight(self) -> None: - server, rpc = coordinator() - install_fake_bundle_builder(server) - state = client(1) - notifies: list[str] = [] - - def record_send(payload: dict[str, object]) -> None: - if payload["method"] == "mining.notify": - assert state.active_job is not None - notifies.append( - str(state.active_job.template["previousblockhash"]) - ) - - state.send = record_send # type: ignore[method-assign] - server.clients = [state] # type: ignore[assignment] - server.ensure_reorg_reconciled_for_tip = lambda _tip: True # type: ignore[method-assign] - server.ensure_reorg_reconciled_for_current_tip = ( # type: ignore[method-assign] - lambda **_kwargs: True - ) - server.qbit_chain_view_untrusted = lambda: False # type: ignore[method-assign] - try: - self.assertEqual(server.poll_qbit_tip_template_once(), 1) - published_tip = server.current_tip_first_seen - assert published_tip is not None - - next_tip = "aa" * 32 - rpc.tip = next_tip - rpc.template = base_template(height=11, prevhash=next_tip) - self.assertTrue(server.observe_tip_for_refresh(next_tip)) - server.fetch_qbit_tip_template_snapshot() - - # A Vardiff-style direct build selects pinned published work, then - # the prepared refresh publishes the replacement before the direct - # path reaches its install/send section. - original_build = server.build_job_for_client - - def build_then_publish_replacement( - target: object, - *, - clean_jobs: bool, - ) -> object: - context = original_build(target, clean_jobs=clean_jobs) # type: ignore[arg-type] - sequence = server._reserve_tip_observation_sequence() - snapshot = server.fetch_qbit_tip_template_snapshot() - bundle = server.prepare_tip_refresh_bundle(snapshot) - token = server._validate_prepared_tip_refresh( - bundle, - snapshot, - sequence, - ) - cancellation = server._publish_prepared_tip_refresh( - token, - bundle, - snapshot, - parent_hash=None, - ) - server._clear_active_tip_refresh(token, cancellation) - return context - - server.build_job_for_client = build_then_publish_replacement # type: ignore[method-assign] - sent_before = len(notifies) - - # The stale-on-arrival context must be skipped, not sent. - self.assertFalse(server.maybe_send_job(state, clean_jobs=False)) - self.assertEqual(len(notifies), sent_before) - self.assertEqual(server.current_tip_first_seen[0], next_tip) - finally: - server.shutdown_tip_refresh_executor() - - def test_direct_send_skips_pinned_work_after_lease_lapse(self) -> None: - server, rpc = coordinator() - install_fake_bundle_builder(server) - state = client(1) - notifies: list[str] = [] - - def record_send(payload: dict[str, object]) -> None: - if payload["method"] == "mining.notify": - assert state.active_job is not None - notifies.append( - str(state.active_job.template["previousblockhash"]) - ) - - state.send = record_send # type: ignore[method-assign] - server.clients = [state] # type: ignore[assignment] - server.ensure_reorg_reconciled_for_tip = lambda _tip: True # type: ignore[method-assign] - server.ensure_reorg_reconciled_for_current_tip = ( # type: ignore[method-assign] - lambda **_kwargs: True - ) - server.qbit_chain_view_untrusted = lambda: False # type: ignore[method-assign] - try: - self.assertEqual(server.poll_qbit_tip_template_once(), 1) - - next_tip = "dd" * 32 - rpc.tip = next_tip - rpc.template = base_template(height=11, prevhash=next_tip) - self.assertTrue(server.observe_tip_for_refresh(next_tip)) - server.fetch_qbit_tip_template_snapshot() - - # The pinned published-tip build starts while authority holds, - # then the divergence lease lapses during the payout-gate wait. - original_build = server.build_job_for_client - - def build_then_lapse( - target: object, - *, - clean_jobs: bool, - ) -> object: - context = original_build(target, clean_jobs=clean_jobs) # type: ignore[arg-type] - server.current_tip_observed_monotonic = ( - time.monotonic() - - float(getattr(server, "submit_tip_max_age_seconds", 10.0)) - - 1.0 - ) - server.template_refresh_failure_exit_seconds = 0.0 - return context - - server.build_job_for_client = build_then_lapse # type: ignore[method-assign] - sent_before = len(notifies) - - # Submit classification has fallen back to the live view; the - # pinned published-parent job would be stale on arrival. - self.assertFalse(server.maybe_send_job(state, clean_jobs=False)) - self.assertEqual(len(notifies), sent_before) - finally: - server.shutdown_tip_refresh_executor() - - def test_lapsed_direct_send_revalidates_against_live_tip(self) -> None: - server, rpc = coordinator() - install_fake_bundle_builder(server) - state = client(1) - notifies: list[str] = [] - - def record_send(payload: dict[str, object]) -> None: - if payload["method"] == "mining.notify": - assert state.active_job is not None - notifies.append( - str(state.active_job.template["previousblockhash"]) - ) - - state.send = record_send # type: ignore[method-assign] - server.clients = [state] # type: ignore[assignment] - server.ensure_reorg_reconciled_for_tip = lambda _tip: True # type: ignore[method-assign] - server.ensure_reorg_reconciled_for_current_tip = ( # type: ignore[method-assign] - lambda **_kwargs: True - ) - server.qbit_chain_view_untrusted = lambda: False # type: ignore[method-assign] - try: - self.assertEqual(server.poll_qbit_tip_template_once(), 1) - published_tip = server.current_tip_first_seen - assert published_tip is not None - - # The chain moves while the coordinator is blind (no blockpoll, - # no blockwait, template cache still fresh on the published tip), - # and the published authority lapses entirely. - next_tip = "ff" * 32 - rpc.tip = next_tip - server.current_tip_observed_monotonic = ( - time.monotonic() - - float(getattr(server, "submit_tip_max_age_seconds", 10.0)) - - 1.0 - ) - server.template_refresh_failure_exit_seconds = 0.0 - - state.active_job = None - state.active_job_ids = set() - sent_before = len(notifies) - # Submit classification is on the live RPC read now; advertising - # cached published-tip work would reject every share. The direct - # path revalidates against the live tip and records the detection. - self.assertFalse(server.maybe_send_job(state, clean_jobs=False)) - self.assertEqual(len(notifies), sent_before) - self.assertEqual(server.latest_detected_tip[0], next_tip) - finally: - server.shutdown_tip_refresh_executor() - - def test_mid_wait_lease_lapse_defers_unvalidated_direct_send(self) -> None: - server, rpc = coordinator() - install_fake_bundle_builder(server) - state = client(1) - notifies: list[str] = [] - - def record_send(payload: dict[str, object]) -> None: - if payload["method"] == "mining.notify": - assert state.active_job is not None - notifies.append( - str(state.active_job.template["previousblockhash"]) - ) - - state.send = record_send # type: ignore[method-assign] - server.clients = [state] # type: ignore[assignment] - server.ensure_reorg_reconciled_for_tip = lambda _tip: True # type: ignore[method-assign] - server.ensure_reorg_reconciled_for_current_tip = ( # type: ignore[method-assign] - lambda **_kwargs: True - ) - server.qbit_chain_view_untrusted = lambda: False # type: ignore[method-assign] - try: - self.assertEqual(server.poll_qbit_tip_template_once(), 1) - - # The chain moves while the coordinator is blind, and the lease - # lapses only while the send waits inside the payout gate — after - # the pre-wait authority check already passed. - next_tip = "ab" * 32 - rpc.tip = next_tip - original_observe_admission = server._observe_payout_gate_admission - - def lapse_during_gate(*args: object, **kwargs: object) -> None: - server.current_tip_observed_monotonic = ( - time.monotonic() - - float(getattr(server, "submit_tip_max_age_seconds", 10.0)) - - 1.0 - ) - server.template_refresh_failure_exit_seconds = 0.0 - original_observe_admission(*args, **kwargs) - - server._observe_payout_gate_admission = lapse_during_gate # type: ignore[method-assign] - - state.active_job = None - state.active_job_ids = set() - sent_before = len(notifies) - # The context never passed a live-tip revalidation, so the - # install boundary defers instead of trusting the blind cached - # observation; the fresh pass settles it against the live tip. - self.assertFalse(server.maybe_send_job(state, clean_jobs=False)) - self.assertEqual(len(notifies), sent_before) - finally: - server.shutdown_tip_refresh_executor() - - def test_fingerprint_cancel_keeps_pinned_published_build(self) -> None: - server, rpc = coordinator() - install_fake_bundle_builder(server) - state = client(1) - state.send = lambda _payload: None # type: ignore[method-assign] - server.clients = [state] # type: ignore[assignment] - server.ensure_reorg_reconciled_for_tip = lambda _tip: True # type: ignore[method-assign] - server.qbit_chain_view_untrusted = lambda: False # type: ignore[method-assign] - try: - self.assertEqual(server.poll_qbit_tip_template_once(), 1) - published_snapshot = server.tip_template_snapshot - assert published_snapshot is not None - pinned_artifacts = published_snapshot.template_artifacts - assert pinned_artifacts is not None - - next_tip = "ee" * 32 - rpc.tip = next_tip - rpc.template = base_template(height=11, prevhash=next_tip) - self.assertTrue(server.observe_tip_for_refresh(next_tip)) - - pinned_cancellation = _JobBuildCancellation(timeout_seconds=30.0) - pinned_flight = types.SimpleNamespace( - request=types.SimpleNamespace( - artifacts=pinned_artifacts, - cancellation=pinned_cancellation, - superseded_monotonic=None, - ) - ) - with server._job_build_scheduler_lock: - server._job_build_active = pinned_flight # type: ignore[assignment] - try: - # The detected-tip store prunes fingerprints and sweeps stale - # builds, but an in-flight build for exactly the published - # snapshot is still valid, creditable work and must survive. - server.fetch_qbit_tip_template_snapshot() - self.assertFalse(pinned_cancellation.is_set()) - finally: - with server._job_build_scheduler_lock: - server._job_build_active = None - finally: - server.shutdown_tip_refresh_executor() - - def test_issuance_fetch_racing_ahead_of_detection_repins_published(self) -> None: - server, rpc = coordinator() - install_fake_bundle_builder(server) - state = client(1) - state.send = lambda _payload: None # type: ignore[method-assign] - server.clients = [state] # type: ignore[assignment] - server.ensure_reorg_reconciled_for_tip = lambda _tip: True # type: ignore[method-assign] - server.qbit_chain_view_untrusted = lambda: False # type: ignore[method-assign] - try: - self.assertEqual(server.poll_qbit_tip_template_once(), 1) - published_tip = server.current_tip_first_seen - assert published_tip is not None - - # qbit advances, but neither blockpoll nor blockwait has observed - # it yet; a template-cache miss makes direct issuance the first - # reader to see the new parent. - next_tip = "cc" * 32 - rpc.tip = next_tip - rpc.template = base_template(height=11, prevhash=next_tip) - with server._job_cache_lock: - server._template_artifacts = None - - artifacts = server.job_issuance_template_artifacts() - # The fetch is recorded as a detection and issuance stays on the - # still-authoritative published snapshot instead of starving on - # work the buildability gates would reject. - self.assertEqual(artifacts.previousblockhash, published_tip[0]) - self.assertEqual(server.latest_detected_tip[0], next_tip) - context = server.build_job_for_client(state, clean_jobs=False) - self.assertEqual( - str(context.template["previousblockhash"]), - published_tip[0], - ) - finally: - server.shutdown_tip_refresh_executor() - - def test_pinned_delivery_keeps_priority_when_payout_sources_detected_tip(self) -> None: - server, rpc = coordinator() - install_fake_bundle_builder(server) - state = client(1) - notifies: list[str] = [] - - def record_send(payload: dict[str, object]) -> None: - if payload["method"] == "mining.notify": - assert state.active_job is not None - notifies.append( - str(state.active_job.template["previousblockhash"]) - ) - - state.send = record_send # type: ignore[method-assign] - server.clients = [state] # type: ignore[assignment] - server.ensure_reorg_reconciled_for_tip = lambda _tip: True # type: ignore[method-assign] - server.ensure_reorg_reconciled_for_current_tip = ( # type: ignore[method-assign] - lambda **_kwargs: True - ) - server.qbit_chain_view_untrusted = lambda: False # type: ignore[method-assign] - try: - self.assertEqual(server.poll_qbit_tip_template_once(), 1) - published_tip = server.current_tip_first_seen - assert published_tip is not None - - next_tip = "bb" * 32 - rpc.tip = next_tip - rpc.template = base_template(height=11, prevhash=next_tip) - self.assertTrue(server.observe_tip_for_refresh(next_tip)) - server.fetch_qbit_tip_template_snapshot() - # Reconciliation may source payout metadata at the detected tip - # before submit authority flips; pinned published-tip delivery - # must keep the priority lane through the payout gate. Arm the - # gate's first-delivery reservation exactly as a fresh payout - # publication does, so a non-priority same-generation delivery - # would be rejected rather than admitted. - with server._job_cache_lock: - server._published_payout_state = dataclasses.replace( - server._published_payout_state, - source_tip_hash=next_tip, - ) - gate = server._payout_state_delivery_gate - with gate._condition: - gate._priority_generation = int(server._payout_state_generation) - - state.active_job = None - state.active_job_ids = set() - sent_before = len(notifies) - self.assertTrue(server.maybe_send_job(state, clean_jobs=False)) - self.assertEqual(len(notifies), sent_before + 1) - self.assertEqual(notifies[-1], published_tip[0]) - finally: - server.shutdown_tip_refresh_executor() - - def test_poll_start_only_targets_still_get_prepared_publication(self) -> None: - server, rpc = coordinator() - install_fake_bundle_builder(server) - state = client(1) - sent_notifies: list[str] = [] - - def record_send(payload: dict[str, object]) -> None: - if payload["method"] == "mining.notify": - assert state.active_job is not None - sent_notifies.append( - str(state.active_job.template["previousblockhash"]) - ) - - state.send = record_send # type: ignore[method-assign] - server.clients = [state] # type: ignore[assignment] - server.ensure_reorg_reconciled_for_tip = lambda _tip: True # type: ignore[method-assign] - server.qbit_chain_view_untrusted = lambda: False # type: ignore[method-assign] - try: - self.assertEqual(server.poll_qbit_tip_template_once(), 1) - published_tip = server.current_tip_first_seen - assert published_tip is not None - - # A same-tip pass whose initial selection sees no work, while the - # poll-start merge finds a target that does need a refresh, must - # still take the prepared path: authority may not republish - # through the sequential branch with no deliverable bundle. - original_needs = server.client_needs_tip_template_refresh - calls = {"count": 0} - - def needs_after_initial_selection( - target: object, - snapshot: object, - ) -> bool: - calls["count"] += 1 - if calls["count"] == 1: - return False - return original_needs(target, snapshot) # type: ignore[arg-type] - - server.client_needs_tip_template_refresh = ( # type: ignore[method-assign] - needs_after_initial_selection - ) - state.active_job = None - state.active_job_ids = set() - - self.assertEqual(server.poll_qbit_tip_template_once(), 1) - self.assertGreaterEqual(calls["count"], 2) - self.assertEqual(sent_notifies[-1], published_tip[0]) - self.assertEqual(server.current_tip_first_seen[0], published_tip[0]) - self.assertIsNone(server._active_tip_refresh) - finally: - server.shutdown_tip_refresh_executor() - - def test_tip_flip_publication_clears_mismatched_parent_on_lookup_failure(self) -> None: - server, rpc = coordinator() - install_fake_bundle_builder(server) - state = client(1) - state.send = lambda _payload: None # type: ignore[method-assign] - server.clients = [state] # type: ignore[assignment] - server.ensure_reorg_reconciled_for_tip = lambda _tip: True # type: ignore[method-assign] - server.qbit_chain_view_untrusted = lambda: False # type: ignore[method-assign] - try: - self.assertEqual(server.poll_qbit_tip_template_once(), 1) - published_tip = server.current_tip_first_seen - assert published_tip is not None - with server.lock: - server.current_tip_parent = (published_tip[0], "aa" * 32) - - next_tip = "66" * 32 - rpc.tip = next_tip - rpc.template = base_template(height=11, prevhash=next_tip) - sequence = server._reserve_tip_observation_sequence() - self.assertTrue( - server.observe_tip_for_refresh( - next_tip, - observation_sequence=sequence, - mark_pending=False, - ) - ) - snapshot = server.fetch_qbit_tip_template_snapshot() - bundle = server.prepare_tip_refresh_bundle(snapshot) - token = server._validate_prepared_tip_refresh(bundle, snapshot, sequence) - cancellation = server._publish_prepared_tip_refresh( - token, - bundle, - snapshot, - parent_hash=None, - ) - try: - # The old tip's parent must never survive a flip; submit - # classification would otherwise trust stale lineage. - self.assertIsNone(server.current_tip_parent) - self.assertEqual(server.current_tip_first_seen[0], next_tip) - finally: - server._clear_active_tip_refresh(token, cancellation) - finally: - server.shutdown_tip_refresh_executor() - - -if __name__ == "__main__": - unittest.main() diff --git a/tests/test_prism_tip_refresh.py b/tests/test_prism_tip_refresh.py deleted file mode 100644 index 9dec17020..000000000 --- a/tests/test_prism_tip_refresh.py +++ /dev/null @@ -1,185 +0,0 @@ -#!/usr/bin/env python3 -"""Owner-boundary tests for the extracted R1 tip-refresh and P1 payout owners. - -The current services resolve the coordinator through call-time Runtime -Protocols (the old stack's standalone ``*Ports``/``*Config`` fixtures were -deliberately not built), so this suite pins the boundary itself: exact symbol -identity, private-alias compatibility, descriptor-routed legacy fields, and -the self-contained fanout delivery gate. The scheduler/trigger/publication -behavior is asserted by the upstream suites in -``tests/test_prism_tip_refresh_validation.py``, -``tests/test_prism_tip_refresh_epoch.py``, -``tests/test_prism_tip_publication_boundary.py``, -``tests/test_prism_refresh_retry_pacing.py`` and -``tests/test_prism_payout_state.py``. -""" - -from __future__ import annotations - -import threading -import unittest - -from lab.prism import ( - payout_state, - prism_coordinator, - template_artifacts, - tip_refresh, -) -from lab.prism.prism_coordinator import PrismCoordinator -from lab.prism.tip_refresh import FanoutCancellation -from tests.prism_vardiff_test_support import coordinator - -TIP_A = "11" * 32 - - -class FanoutCancellationTests(unittest.TestCase): - def test_fanout_cancellation_closes_admission_then_drains(self) -> None: - cancellation = FanoutCancellation() - self.assertTrue(cancellation.begin_delivery()) - drained = threading.Event() - - def cancel_and_drain() -> None: - cancellation.set() - drained.set() - - thread = threading.Thread(target=cancel_and_drain) - thread.start() - self.assertFalse(drained.wait(0.05)) - self.assertFalse(cancellation.begin_delivery()) - cancellation.end_delivery() - self.assertTrue(drained.wait(1.0)) - thread.join(1.0) - self.assertFalse(thread.is_alive()) - - def test_delivery_gate_release_without_admission_fails_closed(self) -> None: - cancellation = FanoutCancellation() - - with self.assertRaisesRegex(RuntimeError, "without admission"): - cancellation.end_delivery() - - # cancel() closes admission without waiting so workers may call it - # while holding a client lock; is_set reports the closed gate. - cancellation.cancel() - self.assertTrue(cancellation.is_set()) - self.assertFalse(cancellation.begin_delivery()) - - -class R1FacadeIdentityTests(unittest.TestCase): - def test_facade_reexports_exact_r1_identities(self) -> None: - self.assertIs( - prism_coordinator.RetainedCollectionRefresh, - tip_refresh.RetainedCollectionRefresh, - ) - self.assertIs(prism_coordinator.RefreshResult, tip_refresh.RefreshResult) - self.assertIs( - prism_coordinator.TipRefreshValidationToken, - tip_refresh.TipRefreshValidationToken, - ) - self.assertIs( - prism_coordinator._FanoutCancellation, tip_refresh.FanoutCancellation - ) - self.assertIs( - prism_coordinator.PRISM_TIP_REFRESH_SECONDS_BUCKETS, - tip_refresh.PRISM_TIP_REFRESH_SECONDS_BUCKETS, - ) - self.assertIs( - prism_coordinator.PRISM_TIP_REFRESH_BUILD_PHASES, - tip_refresh.PRISM_TIP_REFRESH_BUILD_PHASES, - ) - # PR 80 removed the result/cancellation-stage re-exports. - self.assertFalse(hasattr(prism_coordinator, "PRISM_TIP_REFRESH_RESULTS")) - self.assertFalse( - hasattr(prism_coordinator, "PRISM_TIP_REFRESH_CANCELLATION_STAGES") - ) - # The owner is a leaf: it never imports the coordinator module. - self.assertFalse(hasattr(tip_refresh, "PrismCoordinator")) - - def test_p1_reexports_and_private_aliases_preserve_identity(self) -> None: - self.assertIs( - prism_coordinator.PayoutStateArtifact, payout_state.PayoutStateArtifact - ) - self.assertIs( - prism_coordinator.PayoutStateCandidate, payout_state.PayoutStateCandidate - ) - # PR 80 removed the PublishedPayoutState compatibility re-export. - self.assertFalse(hasattr(prism_coordinator, "PublishedPayoutState")) - self.assertIs( - prism_coordinator.PayoutLedgerArtifact, payout_state.PayoutLedgerArtifact - ) - self.assertIs( - prism_coordinator.TemplateRefreshBlocked, - payout_state.TemplateRefreshBlocked, - ) - self.assertIs( - prism_coordinator.TemplateRefreshSuperseded, - payout_state.TemplateRefreshSuperseded, - ) - # PR 80 removed the four underscore payout aliases; the coordinator - # keeps only the unaliased exception it still raises. - self.assertFalse(hasattr(prism_coordinator, "_AcceptedBlockPayoutTransition")) - self.assertFalse(hasattr(prism_coordinator, "_PayoutDeliveryAdmission")) - self.assertFalse(hasattr(prism_coordinator, "_PayoutStateDeliveryGate")) - self.assertFalse(hasattr(prism_coordinator, "_PayoutStatePublicationBlocked")) - self.assertIs( - prism_coordinator.PayoutStatePublicationBlocked, - payout_state.PayoutStatePublicationBlocked, - ) - # template_artifacts re-imports the exception trio from the final P1 - # owner so the interim import path stays identity-compatible. - self.assertIs( - template_artifacts.TemplateRefreshBlocked, - payout_state.TemplateRefreshBlocked, - ) - self.assertIs( - template_artifacts.TemplateRefreshSuperseded, - payout_state.TemplateRefreshSuperseded, - ) - - -class R1DescriptorRoutingTests(unittest.TestCase): - def test_tip_state_descriptors_route_to_the_r1_owner(self) -> None: - server = coordinator() - service = server._ensure_tip_refresh_service() - self.assertIs(server._ensure_tip_refresh_service(), service) - - server.tip_refresh_job_count = 7 - self.assertEqual(service.tip_refresh_job_count, 7) - self.assertEqual(server.tip_refresh_job_count, 7) - - server.current_tip_first_seen = (TIP_A, 5.0) - self.assertEqual(service.current_tip_first_seen, (TIP_A, 5.0)) - with server.lock: - self.assertEqual(server._current_published_tip_hash_locked(), TIP_A) - - # The refresh lock is owner state reached through the legacy name. - self.assertIs(server._tip_refresh_lock, service._tip_refresh_lock) - - def test_unset_tip_state_reads_raise_attribute_error_like_lazy_fields( - self, - ) -> None: - server = PrismCoordinator.__new__(PrismCoordinator) - server.lock = threading.RLock() - - with self.assertRaises(AttributeError): - _ = server.current_tip_first_seen - with self.assertRaises(AttributeError): - _ = server.tip_refresh_job_count - - server.tip_refresh_job_count = 3 - self.assertEqual(server.tip_refresh_job_count, 3) - - def test_payout_state_descriptors_route_to_the_p1_owner(self) -> None: - server = coordinator() - service = server._ensure_payout_state_service() - - server._payout_state_generation = 9 - self.assertEqual(server._payout_state_generation, 9) - current = server._published_payout_state - self.assertIs(current, server._published_payout_state) - # The generation adapter reads the same owner state. - self.assertEqual(server._current_payout_generation(), 9) - self.assertIsNotNone(service) - - -if __name__ == "__main__": - unittest.main() diff --git a/tests/test_prism_tip_refresh_delivery.py b/tests/test_prism_tip_refresh_delivery.py deleted file mode 100644 index d1e148158..000000000 --- a/tests/test_prism_tip_refresh_delivery.py +++ /dev/null @@ -1,3180 +0,0 @@ -#!/usr/bin/env python3 -"""Focused PRISM coordinator tip refresh delivery tests.""" -# ruff: noqa: F821 - -from __future__ import annotations - -from pathlib import Path -import tempfile -import threading -import unittest -from lab.prism.job_delivery import JobBuildFailed -from lab.prism.payout_state import AcceptedParentPayoutPreviewPending -from tests import prism_coordinator_test_support as _job_support -from tests import prism_vardiff_test_support as _vardiff_support - - -class _VardiffSupportTestCase(unittest.TestCase): - def setUp(self) -> None: - globals().update( - {name: getattr(_vardiff_support, name) for name in _vardiff_support.__all__} - ) - - -class _JobSupportTestCase(unittest.TestCase): - def setUp(self) -> None: - globals().update( - {name: getattr(_job_support, name) for name in _job_support.__all__} - ) - - -class PrismCoordinatorVardiffTests(_VardiffSupportTestCase): - def test_tip_change_refreshes_clean_job_and_old_job_becomes_stale_without_reconnect(self) -> None: - old_tip = "00" * 32 - new_tip = "11" * 32 - server = coordinator() - server.accepted_block_count = 0 - server.max_blocks = 1 - server.stop_after_block = True - server.stale_grace_seconds = 0 - server.jobs = {} - server.recent_share_keys = set() - server.share_weights_by_username = {} - ledger = RecordingLedger() - server.ledger = ledger - worker = worker_identity() - state = client() - state.username = worker.username - state.worker = worker - state.share_difficulty = Decimal("1") - sent: list[dict[str, object]] = [] - state.send = lambda payload: sent.append(payload) # type: ignore[method-assign] - server.clients = {state} - - old_context = prism_context("old-job", old_tip, worker=worker) - state.active_job = old_context - state.active_job_ids = {"old-job"} - server.jobs["old-job"] = old_context - server.tip_template_snapshot = QbitTipTemplateSnapshot( - bestblockhash=old_tip, - previousblockhash=old_tip, - template_fingerprint=qbit_template_fingerprint(old_context.template), - ) - server.rpc = TipTemplateRpc(tip=new_tip, template=gbt_template(new_tip, height=11)) - - def build_fresh_job(client: ClientState, *, clean_jobs: bool) -> object: - self.assertIs(client, state) - return prism_context( - "fresh-job", - new_tip, - worker=worker, - difficulty=client.pending_share_difficulty or client.share_difficulty, - clean_jobs=clean_jobs, - ) - - server.build_job_for_client = build_fresh_job # type: ignore[method-assign] - - refreshed = server.poll_qbit_tip_template_once() - - self.assertEqual(refreshed, 1) - self.assertEqual(server.tip_refresh_job_count, 1) - self.assertIn(state, server.clients) - self.assertNotIn("old-job", server.jobs) - self.assertEqual(state.active_job_ids, {"fresh-job"}) - self.assertIn("fresh-job", server.jobs) - self.assertEqual([payload["method"] for payload in sent], ["mining.set_difficulty", "mining.notify"]) - self.assertEqual(sent[1]["params"][0], "fresh-job") - self.assertTrue(sent[1]["params"][8]) - - with self.assertRaises(StratumError) as raised: - server.handle_submit( - state, - ["miner-a", "old-job", "00" * 8, "00000001", "00000002"], - ) - self.assertEqual(raised.exception.code, 21) - self.assertEqual(raised.exception.reason, PRISM_REJECTION_UNKNOWN_JOB) - self.assertEqual(server.stale_share_count, 1) - self.assertEqual(server.rejection_counts_by_reason[PRISM_REJECTION_UNKNOWN_JOB], 1) - self.assertEqual(len(ledger.pending), 0) - - submission = SimpleNamespace( - header_hex="aa" * 80, - block_hash_hex="bb" * 32, - share_pass=True, - block_pass=False, - ) - with patch( - "lab.prism.prism_coordinator.direct_stratum.assemble_submission", - return_value=submission, - ): - should_close = server.handle_submit( - state, - ["miner-a", "fresh-job", "00" * 8, "00000001", "00000002"], - ) - - self.assertFalse(should_close) - self.assertEqual(len(ledger.pending), 1) - self.assertEqual(ledger.pending[0].job_id, "fresh-job") - self.assertIn(state, server.clients) - - def test_tip_refresh_rpc_race_blocks_mismatched_tip_template_snapshot(self) -> None: - old_tip = "00" * 32 - new_tip = "11" * 32 - server = coordinator() - server.jobs = {} - worker = worker_identity() - state = client() - state.username = worker.username - state.worker = worker - sent: list[dict[str, object]] = [] - state.send = lambda payload: sent.append(payload) # type: ignore[method-assign] - server.clients = {state} - - old_context = prism_context("old-job", old_tip, worker=worker) - state.active_job = old_context - state.active_job_ids = {"old-job"} - server.jobs["old-job"] = old_context - server.tip_template_snapshot = QbitTipTemplateSnapshot( - bestblockhash=old_tip, - previousblockhash=old_tip, - template_fingerprint=qbit_template_fingerprint(old_context.template), - ) - server.rpc = TipTemplateRpc(tip=old_tip, template=gbt_template(new_tip, height=11)) - - with self.assertRaisesRegex( - TemplateRefreshBlocked, - "tip changed while fetching block template", - ): - server.poll_qbit_tip_template_once() - - self.assertIs(state.active_job, old_context) - self.assertEqual(server.jobs, {"old-job": old_context}) - self.assertEqual(state.active_job_ids, {"old-job"}) - self.assertEqual(sent, []) - # Tip observation is recorded as a detection before expensive template - # construction so obsolete builders can be cancelled immediately, but - # submit authority is published only alongside a coherent snapshot. - # The incoherent template never enters the artifact cache, client job - # maps, or the published tip state. - self.assertEqual(server.latest_detected_tip[0], old_tip) - self.assertIsNone(server.current_tip_first_seen) - self.assertIsNone(server._template_artifacts) - - def test_slow_tip_poll_cannot_regress_newer_blockwait_observation(self) -> None: - old_tip = "00" * 32 - new_tip = "11" * 32 - server = coordinator() - old_snapshot = QbitTipTemplateSnapshot( - bestblockhash=old_tip, - previousblockhash=old_tip, - template_fingerprint="22" * 32, - ) - server.rpc = TipRpc(old_tip) - self.assertTrue(server.observe_tip_first_seen(old_tip)) - - def overtake_poll() -> QbitTipTemplateSnapshot: - self.assertTrue(server.observe_tip_for_refresh(new_tip)) - return old_snapshot - - server.fetch_qbit_tip_template_snapshot = overtake_poll # type: ignore[method-assign] - - with self.assertRaisesRegex( - TemplateRefreshSuperseded, - "tip/template poll was superseded during template fetch", - ): - server.poll_qbit_tip_template_once() - - self.assertEqual(server.latest_detected_tip[0], new_tip) - self.assertEqual(server.current_tip_first_seen[0], old_tip) - self.assertIsNone(server.tip_template_snapshot) - self.assertIsNone( - getattr(server, "template_refresh_failure_started_monotonic", None) - ) - self.assertTrue(server._consume_tip_refresh_retry()) - self.assertFalse(server._consume_tip_refresh_retry()) - - def test_same_tip_template_refresh_sends_non_clean_job_and_keeps_old_job_submittable(self) -> None: - tip = "00" * 32 - server = coordinator() - server.accepted_block_count = 0 - server.max_blocks = 1 - server.stop_after_block = True - server.jobs = {} - server.recent_share_keys = set() - server.share_weights_by_username = {} - ledger = RecordingLedger() - server.ledger = ledger - worker = worker_identity() - state = client() - state.username = worker.username - state.worker = worker - state.share_difficulty = Decimal("1") - sent: list[dict[str, object]] = [] - state.send = lambda payload: sent.append(payload) # type: ignore[method-assign] - server.clients = {state} - - old_context = prism_context("old-job", tip, worker=worker) - state.active_job = old_context - state.active_job_ids = {"old-job"} - server.jobs["old-job"] = old_context - server.tip_template_snapshot = QbitTipTemplateSnapshot( - bestblockhash=tip, - previousblockhash=tip, - template_fingerprint=qbit_template_fingerprint(old_context.template), - ) - refreshed_template = gbt_template(tip, height=10, coinbasevalue=50_00000001) - server.rpc = TipTemplateRpc(tip=tip, template=refreshed_template) - - def build_fresh_job(client: ClientState, *, clean_jobs: bool) -> object: - self.assertIs(client, state) - self.assertFalse(clean_jobs) - fresh_context = prism_context( - "fresh-job", - tip, - worker=worker, - difficulty=client.pending_share_difficulty or client.share_difficulty, - clean_jobs=clean_jobs, - ) - fresh_context.template["coinbasevalue"] = refreshed_template["coinbasevalue"] - fresh_context.template_fingerprint = qbit_template_fingerprint(fresh_context.template) - return fresh_context - - server.build_job_for_client = build_fresh_job # type: ignore[method-assign] - - refreshed = server.poll_qbit_tip_template_once() - - self.assertEqual(refreshed, 1) - self.assertEqual(server.tip_refresh_job_count, 1) - self.assertIn(state, server.clients) - self.assertIn("old-job", server.jobs) - self.assertIn("fresh-job", server.jobs) - self.assertEqual(state.active_job_ids, {"old-job", "fresh-job"}) - self.assertEqual([payload["method"] for payload in sent], ["mining.set_difficulty", "mining.notify"]) - self.assertEqual(sent[1]["params"][0], "fresh-job") - self.assertFalse(sent[1]["params"][8]) - self.assertIn("qbit_prism_active_job_contexts 2", server.metrics_payload()) - - submission = SimpleNamespace( - header_hex="aa" * 80, - block_hash_hex="bb" * 32, - share_pass=True, - block_pass=False, - ) - with patch( - "lab.prism.prism_coordinator.direct_stratum.assemble_submission", - return_value=submission, - ): - should_close = server.handle_submit( - state, - ["miner-a", "old-job", "00" * 8, "00000001", "00000002"], - ) - - self.assertFalse(should_close) - self.assertEqual(len(ledger.pending), 1) - self.assertEqual(ledger.pending[0].job_id, "old-job") - self.assertEqual(server.stale_share_count, 0) - self.assertEqual(server.rejection_counts_by_reason[PRISM_REJECTION_UNKNOWN_JOB], 0) - self.assertEqual(server.rejection_counts_by_reason[PRISM_REJECTION_STALE_JOB], 0) - - def test_tip_refresh_uses_pending_vardiff_difficulty_for_consistent_pair(self) -> None: - old_tip = "00" * 32 - new_tip = "22" * 32 - server = coordinator() - server.jobs = {} - worker = worker_identity() - state = client() - state.username = worker.username - state.worker = worker - state.share_difficulty = Decimal("1") - state.pending_share_difficulty = Decimal("8") - sent: list[dict[str, object]] = [] - state.send = lambda payload: sent.append(payload) # type: ignore[method-assign] - server.clients = {state} - old_context = prism_context("old-job", old_tip, worker=worker, difficulty=Decimal("1")) - state.active_job = old_context - state.active_job_ids = {"old-job"} - server.jobs["old-job"] = old_context - server.tip_template_snapshot = QbitTipTemplateSnapshot( - bestblockhash=old_tip, - previousblockhash=old_tip, - template_fingerprint=qbit_template_fingerprint(old_context.template), - ) - server.rpc = TipTemplateRpc(tip=new_tip, template=gbt_template(new_tip, height=11)) - - def build_fresh_job(client: ClientState, *, clean_jobs: bool) -> object: - return prism_context( - "fresh-vardiff-job", - new_tip, - worker=worker, - difficulty=server.desired_client_share_difficulty(client), - clean_jobs=clean_jobs, - ) - - server.build_job_for_client = build_fresh_job # type: ignore[method-assign] - - refreshed = server.poll_qbit_tip_template_once() - - self.assertEqual(refreshed, 1) - self.assertEqual(sent[0]["method"], "mining.set_difficulty") - self.assertEqual(sent[0]["params"], [8.0]) - self.assertEqual(sent[1]["method"], "mining.notify") - self.assertEqual(sent[1]["params"][0], "fresh-vardiff-job") - self.assertTrue(sent[1]["params"][8]) - self.assertEqual(state.share_difficulty, Decimal("8")) - self.assertIsNone(state.pending_share_difficulty) - self.assertEqual(server.jobs["fresh-vardiff-job"].job.share_difficulty, Decimal("8")) - - def test_tip_refresh_build_failure_keeps_client_connected_and_old_job_registered(self) -> None: - old_tip = "00" * 32 - new_tip = "33" * 32 - server = coordinator() - server.jobs = {} - worker = worker_identity() - state = client() - state.username = worker.username - state.worker = worker - sent: list[dict[str, object]] = [] - state.send = lambda payload: sent.append(payload) # type: ignore[method-assign] - server.clients = {state} - old_context = prism_context("old-job", old_tip, worker=worker) - state.active_job = old_context - state.active_job_ids = {"old-job"} - server.jobs["old-job"] = old_context - server.tip_template_snapshot = QbitTipTemplateSnapshot( - bestblockhash=old_tip, - previousblockhash=old_tip, - template_fingerprint=qbit_template_fingerprint(old_context.template), - ) - server.rpc = TipTemplateRpc(tip=new_tip, template=gbt_template(new_tip, height=11)) - - def failing_build(client: ClientState, *, clean_jobs: bool) -> object: - raise RuntimeError("transient getblocktemplate failure") - - server.build_job_for_client = failing_build # type: ignore[method-assign] - - with self.assertRaisesRegex(TemplateRefreshBlocked, "no refreshed work was issued"): - server.poll_qbit_tip_template_once() - - self.assertEqual(server.job_build_failure_count, 1) - self.assertEqual(server.tip_refresh_job_count, 0) - self.assertIn(state, server.clients) - self.assertEqual(state.active_job_ids, {"old-job"}) - self.assertIn("old-job", server.jobs) - self.assertEqual(sent, []) - - def test_tip_refresh_build_failure_is_not_masked_by_disconnected_client(self) -> None: - old_tip = "00" * 32 - new_tip = "33" * 32 - server = coordinator() - server.jobs = {} - - build_failed = client() - build_failed.worker = worker_identity("miner-build-failed") - build_failed.username = build_failed.worker.username - build_failed.active_job = prism_context( - "old-build-failed-job", old_tip, worker=build_failed.worker - ) - build_failed.active_job_ids = {"old-build-failed-job"} - - disconnected = client() - disconnected.connection_id = 2 - disconnected.worker = worker_identity("miner-disconnected") - disconnected.username = disconnected.worker.username - disconnected.active_job = prism_context( - "old-disconnected-job", old_tip, worker=disconnected.worker - ) - disconnected.active_job_ids = {"old-disconnected-job"} - - def disconnect_on_send(_payload: object) -> None: - raise OSError("socket closed") - - disconnected.send = disconnect_on_send # type: ignore[method-assign] - server.clients = {build_failed, disconnected} - server.jobs = { - "old-build-failed-job": build_failed.active_job, - "old-disconnected-job": disconnected.active_job, - } - server.tip_template_snapshot = QbitTipTemplateSnapshot( - bestblockhash=old_tip, - previousblockhash=old_tip, - template_fingerprint=qbit_template_fingerprint(build_failed.active_job.template), - ) - server.rpc = TipTemplateRpc(tip=new_tip, template=gbt_template(new_tip, height=11)) - - def mixed_build(state: ClientState, *, clean_jobs: bool) -> object: - if state is build_failed: - raise RuntimeError("template build unavailable") - return prism_context( - "disconnected-fresh-job", - new_tip, - worker=state.worker, - clean_jobs=clean_jobs, - ) - - disconnected_clients: list[ClientState] = [] - server.build_job_for_client = mixed_build # type: ignore[method-assign] - server.disconnect_client = disconnected_clients.append # type: ignore[method-assign] - - with self.assertRaisesRegex(TemplateRefreshBlocked, "no refreshed work was issued"): - server.poll_qbit_tip_template_once() - - self.assertEqual(server.job_build_failure_count, 1) - self.assertEqual(disconnected_clients, [disconnected]) - self.assertIn(build_failed, server.clients) - - def test_tip_reconciliation_quarantines_disconnected_block_before_refresh_job(self) -> None: - old_tip = "00" * 32 - new_tip = "44" * 32 - pool_block_hash = "aa" * 32 - server = coordinator() - server.reorg_reconciler_enabled = True - server.jobs = {} - worker = worker_identity() - state = client() - state.username = worker.username - state.worker = worker - state.share_difficulty = Decimal("1") - sent: list[dict[str, object]] = [] - state.send = lambda payload: sent.append(payload) # type: ignore[method-assign] - server.clients = {state} - old_context = prism_context("old-job", old_tip, worker=worker) - state.active_job = old_context - state.active_job_ids = {"old-job"} - server.jobs["old-job"] = old_context - server.tip_template_snapshot = QbitTipTemplateSnapshot( - bestblockhash=old_tip, - previousblockhash=old_tip, - template_fingerprint=qbit_template_fingerprint(old_context.template), - ) - ledger = ReorgLedger( - [ - { - "block_hash": pool_block_hash, - "block_height": 10, - "chain_state": "confirmed", - "maturity_state": "immature", - } - ] - ) - server.ledger = ledger - server.rpc = ReorgRpc( - tip=new_tip, - template=gbt_template(new_tip, height=11), - height=10, - block_hashes={10: "bb" * 32}, - ) - - def build_fresh_job(client: ClientState, *, clean_jobs: bool) -> object: - self.assertIn(("inactive", pool_block_hash, 10), ledger.events) - ledger.events.append(("build", client.connection_id)) - return prism_context("fresh-job", new_tip, worker=worker, clean_jobs=clean_jobs) - - server.build_job_for_client = build_fresh_job # type: ignore[method-assign] - - refreshed = server.poll_qbit_tip_template_once() - - self.assertEqual(refreshed, 1) - self.assertLess( - ledger.events.index(("inactive", pool_block_hash, 10)), - ledger.events.index(("build", state.connection_id)), - ) - self.assertEqual(server.reorg_inactive_block_count, 1) - self.assertEqual(ledger.rows[0]["chain_state"], "inactive") - self.assertEqual(sent[1]["params"][0], "fresh-job") - - def test_reconciliation_quarantines_confirmed_block_above_shortened_tip(self) -> None: - pool_block_hash = "af" * 32 - server = coordinator() - server.reorg_reconciler_enabled = True - ledger = ReorgLedger( - [ - { - "block_hash": pool_block_hash, - "block_height": 12, - "chain_state": "confirmed", - "maturity_state": "immature", - } - ] - ) - server.ledger = ledger - server.rpc = ReorgRpc( - tip="77" * 32, - template=gbt_template("77" * 32, height=11), - height=10, - block_hashes={}, - ) - - summary = server.reconcile_prism_pool_blocks_once(tip_hash="77" * 32) - - self.assertEqual(summary["inactive_blocks"], 1) - self.assertEqual(server.reorg_inactive_block_count, 1) - self.assertEqual(ledger.rows[0]["chain_state"], "inactive") - - def test_tip_reconciliation_skips_jobs_when_qbit_chain_view_is_untrusted(self) -> None: - old_tip = "00" * 32 - new_tip = "55" * 32 - server = coordinator() - server.reorg_reconciler_enabled = True - server.jobs = {} - worker = worker_identity() - state = client() - state.username = worker.username - state.worker = worker - state.active_job = prism_context("old-job", old_tip, worker=worker) - state.active_job_ids = {"old-job"} - server.jobs["old-job"] = state.active_job - server.clients = {state} - server.tip_template_snapshot = QbitTipTemplateSnapshot( - bestblockhash=old_tip, - previousblockhash=old_tip, - template_fingerprint=qbit_template_fingerprint(state.active_job.template), - ) - ledger = ReorgLedger([]) - server.ledger = ledger - server.rpc = ReorgRpc( - tip=new_tip, - template=gbt_template(new_tip, height=11), - height=10, - block_hashes={10: new_tip}, - initialblockdownload=True, - ) - - def unexpected_build(client: ClientState, *, clean_jobs: bool) -> object: - raise AssertionError("job build should be skipped while qbitd is in IBD") - - server.build_job_for_client = unexpected_build # type: ignore[method-assign] - - with self.assertRaisesRegex(TemplateRefreshBlocked, "chain view remained untrusted"): - server.poll_qbit_tip_template_once() - - self.assertEqual(server.reorg_reconcile_skip_count, 1) - self.assertEqual(ledger.events, []) - self.assertEqual(state.active_job_ids, {"old-job"}) - self.assertEqual(server.tip_template_snapshot.bestblockhash, old_tip) - - def test_reconciliation_error_before_job_build_is_not_counted_as_build_failure(self) -> None: - tip = "59" * 32 - server = coordinator() - server.reorg_reconciler_enabled = True - state = client() - state.username = "miner-a" - state.worker = worker_identity() - state.share_difficulty = Decimal("1") - server.rpc = ReorgRpc( - tip=tip, - template=gbt_template(tip, height=11), - height=10, - block_hashes={10: tip}, - ) - - class FailingReorgLedger(FakeLedger): - def reorg_watch_blocks(self, *, active_tip_height: int) -> list[dict[str, object]]: - raise RuntimeError("ledger unavailable") - - server.ledger = FailingReorgLedger() - server.build_job_for_client = lambda _client, *, clean_jobs: (_ for _ in ()).throw( # type: ignore[method-assign] - AssertionError("job build should not run after reconcile failure") - ) - - sent_job = server.maybe_send_job(state, clean_jobs=True) - - self.assertFalse(sent_job) - self.assertEqual(server.reorg_reconcile_error_count, 1) - self.assertEqual(server.job_build_failure_count, 0) - - def test_reconciliation_runs_again_for_same_tip_hash(self) -> None: - tip = "5a" * 32 - server = coordinator() - server.reorg_reconciler_enabled = True - ledger = ReorgLedger([]) - server.ledger = ledger - server.rpc = ReorgRpc( - tip=tip, - template=gbt_template(tip, height=11), - height=10, - block_hashes={10: tip}, - ) - - self.assertTrue(server.ensure_reorg_reconciled_for_tip(tip)) - self.assertTrue(server.ensure_reorg_reconciled_for_tip(tip)) - - self.assertEqual(ledger.events, [("watch", 10), ("mature", 10), ("watch", 10), ("mature", 10)]) - self.assertEqual(server._payout_state_generation, 1) - self.assertEqual(server._payout_state_source[0], 1) - - def test_reconciliation_reactivates_inactive_block_that_returns_to_active_chain(self) -> None: - pool_block_hash = "cc" * 32 - server = coordinator() - server.reorg_reconciler_enabled = True - ledger = ReorgLedger( - [ - { - "block_hash": pool_block_hash, - "block_height": 12, - "chain_state": "inactive", - "maturity_state": "immature", - } - ] - ) - server.ledger = ledger - temporary_audit = tempfile.TemporaryDirectory() - self.addCleanup(temporary_audit.cleanup) - server.audit_dir = Path(temporary_audit.name) / "audit" - server.evidence_path = ( - Path(temporary_audit.name) / "state" / "evidence.json" - ) - audit_store = server._ensure_audit_artifact_store() - self.addCleanup(audit_store.close) - payout_service = server._ensure_payout_state_service() - real_reactivate = ledger.reactivate_pool_block - - def guarded_reactivate(**kwargs: object) -> dict[str, object]: - self.assertTrue( - payout_service._payout_balance_mutation_lock._is_owned(), # type: ignore[attr-defined] - "P1 balance lock must be outermost during reactivation", - ) - self.assertEqual( - audit_store._publication_guard_owner, - threading.get_ident(), - ) - return real_reactivate(**kwargs) - - ledger.reactivate_pool_block = guarded_reactivate # type: ignore[method-assign] - server.rpc = ReorgRpc( - tip=pool_block_hash, - template=gbt_template(pool_block_hash, height=13), - height=12, - block_hashes={12: pool_block_hash}, - ) - - summary = server.reconcile_prism_pool_blocks_once(tip_hash=pool_block_hash) - - self.assertEqual(summary["reactivated_blocks"], 1) - self.assertEqual(server.reorg_reactivated_block_count, 1) - self.assertEqual(ledger.rows[0]["chain_state"], "confirmed") - self.assertIn(("mature", 12), ledger.events) - - def test_maybe_send_job_reconciles_before_direct_job_build(self) -> None: - tip = "66" * 32 - pool_block_hash = "dd" * 32 - server = coordinator() - server.reorg_reconciler_enabled = True - server.jobs = {} - worker = worker_identity() - state = client() - state.username = worker.username - state.worker = worker - state.share_difficulty = Decimal("1") - sent: list[dict[str, object]] = [] - state.send = lambda payload: sent.append(payload) # type: ignore[method-assign] - ledger = ReorgLedger( - [ - { - "block_hash": pool_block_hash, - "block_height": 20, - "chain_state": "confirmed", - "maturity_state": "immature", - } - ] - ) - server.ledger = ledger - server.rpc = ReorgRpc( - tip=tip, - template=gbt_template(tip, height=21), - height=20, - block_hashes={20: "ee" * 32}, - ) - - def build_direct_job(client: ClientState, *, clean_jobs: bool) -> object: - self.assertIn(("inactive", pool_block_hash, 20), ledger.events) - ledger.events.append(("build", client.connection_id)) - return prism_context("direct-job", tip, worker=worker, clean_jobs=clean_jobs) - - server.build_job_for_client = build_direct_job # type: ignore[method-assign] - - sent_job = server.maybe_send_job(state, clean_jobs=True) - - self.assertTrue(sent_job) - self.assertLess( - ledger.events.index(("inactive", pool_block_hash, 20)), - ledger.events.index(("build", state.connection_id)), - ) - self.assertEqual(sent[1]["params"][0], "direct-job") - - def test_template_refresh_failure_budget_starts_at_first_failure(self) -> None: - server = PrismCoordinator.__new__(PrismCoordinator) - server.template_refresh_failure_exit_seconds = 120 - server.last_successful_template_refresh_monotonic = 100.0 - - server._record_template_refresh_failure(500.0) - self.assertFalse(server.template_refresh_failure_expired(500.0)) - self.assertEqual(server.template_refresh_failure_started_monotonic, 500.0) - self.assertFalse(server.template_refresh_failure_expired(619.999)) - self.assertTrue(server.template_refresh_failure_expired(620.0)) - - def test_disabled_template_refresh_failure_budget_does_not_start_or_expire(self) -> None: - server = PrismCoordinator.__new__(PrismCoordinator) - server.template_refresh_failure_exit_seconds = 0 - - server._record_template_refresh_failure(500.0) - self.assertFalse(server.template_refresh_failure_expired(500.0)) - self.assertFalse(hasattr(server, "template_refresh_failure_started_monotonic")) - - def test_coordination_blocked_refresh_does_not_start_failure_budget(self) -> None: - for blocked_error in ( - TemplateRefreshSuperseded("qbit tip changed during sequential refresh"), - _PayoutStatePublicationBlocked( - "accepted block payout confirmation is still pending" - ), - ): - with self.subTest(blocked=type(blocked_error).__name__): - server = coordinator() - server.template_refresh_failure_exit_seconds = 10.0 - server.coordination_blocked_exit_seconds = 30.0 - server._record_heartbeat = lambda _name: None # type: ignore[method-assign] - server.rpc = TipRpc("11" * 32) - - def raise_blocked(error: Exception = blocked_error) -> QbitTipTemplateSnapshot: - raise error - - server.fetch_qbit_tip_template_snapshot = raise_blocked # type: ignore[method-assign] - with ( - patch( - "lab.prism.prism_coordinator.time.monotonic", - return_value=100.0, - ), - self.assertRaises(type(blocked_error)), - ): - server.poll_qbit_tip_template_once() - - self.assertIsNone( - getattr(server, "template_refresh_failure_started_monotonic", None) - ) - self.assertFalse(server.template_refresh_failure_expired(10_000.0)) - self.assertAlmostEqual( - server.coordination_blocked_streak_age_seconds(109.999), - 9.999, - ) - self.assertFalse( - server.coordination_blocked_streak_expired(129.999) - ) - - def test_non_coordination_blocked_refresh_still_starts_failure_budget(self) -> None: - # Plain TemplateRefreshBlocked also wraps genuine failures (malformed - # template artifacts, job builds failing, untrusted chain views); only - # the TemplateRefreshSuperseded/payout-fence subclasses are exempt. - server = coordinator() - server.template_refresh_failure_exit_seconds = 10.0 - server.coordination_blocked_exit_seconds = 10.0 - server._record_coordination_blocked_refresh(90.0) - server._record_heartbeat = lambda _name: None # type: ignore[method-assign] - server.rpc = TipRpc("11" * 32) - - def raise_blocked() -> QbitTipTemplateSnapshot: - raise TemplateRefreshBlocked( - "unable to derive exact artifacts for observed qbit template" - ) - - server.fetch_qbit_tip_template_snapshot = raise_blocked # type: ignore[method-assign] - with ( - patch("lab.prism.prism_coordinator.time.monotonic", return_value=100.0), - self.assertRaises(TemplateRefreshBlocked), - ): - server.poll_qbit_tip_template_once() - - self.assertEqual(server.template_refresh_failure_started_monotonic, 100.0) - self.assertEqual(server.coordination_blocked_streak_age_seconds(110.0), 0.0) - self.assertTrue(server.template_refresh_failure_expired(110.0)) - - def test_armed_budget_is_not_fired_by_coordination_blocked_refresh(self) -> None: - # A transient budgeted failure armed the clock, qbitd recovered, and - # only coordination churn follows. Blocked attempts must not trip the - # armed budget in blockpoll_loop: the exit is reserved for the next - # budgeted failure, and the clock clears on the next completed refresh. - server = coordinator() - server.blockpoll_seconds = 0 - server.template_refresh_failure_exit_seconds = 10.0 - server.template_refresh_failure_started_monotonic = 100.0 - server._record_heartbeat = lambda _name: None # type: ignore[method-assign] - server.rpc = TipRpc("11" * 32) - clock = {"now": 100.0} - blocked_polls = 0 - - def blocked_fetch() -> QbitTipTemplateSnapshot: - nonlocal blocked_polls - blocked_polls += 1 - clock["now"] += 6.0 - if blocked_polls >= 4: - server.stop_event.set() - raise TemplateRefreshSuperseded( - "qbit tip changed during sequential refresh; immediate retry scheduled" - ) - - server.fetch_qbit_tip_template_snapshot = blocked_fetch # type: ignore[method-assign] - with ( - patch( - "lab.prism.prism_coordinator.time.monotonic", - side_effect=lambda: clock["now"], - ), - patch("lab.prism.prism_coordinator.traceback.print_exc"), - patch("lab.prism.prism_coordinator.os._exit", side_effect=SystemExit(1)) as exit_process, - ): - server.blockpoll_loop() - - self.assertEqual(blocked_polls, 4) - self.assertGreater( - clock["now"], - server.template_refresh_failure_started_monotonic - + server.template_refresh_failure_exit_seconds, - ) - exit_process.assert_not_called() - self.assertEqual(server.template_refresh_failure_started_monotonic, 100.0) - - def test_rpc_outage_arms_and_exhausts_failure_budget(self) -> None: - server = coordinator() - server.blockpoll_seconds = 0 - server.template_refresh_failure_exit_seconds = 10.0 - server._record_heartbeat = lambda _name: None # type: ignore[method-assign] - clock = {"now": 100.0} - - class OutageRpc: - def call(self, method: str, params: list[object] | None = None, **_kwargs: object) -> object: - clock["now"] += 6.0 - raise ConnectionError("qbitd unreachable") - - server.rpc = OutageRpc() - with ( - patch( - "lab.prism.prism_coordinator.time.monotonic", - side_effect=lambda: clock["now"], - ), - patch("lab.prism.prism_coordinator.traceback.print_exc"), - patch("lab.prism.prism_coordinator.os._exit", side_effect=SystemExit(1)) as exit_process, - self.assertRaises(SystemExit), - ): - server.blockpoll_loop() - - exit_process.assert_called_once_with(1) - self.assertIsNotNone(server.template_refresh_failure_started_monotonic) - - def test_persistent_blocked_template_derivation_arms_and_exhausts_failure_budget(self) -> None: - # Top-of-poll RPCs stay healthy, but every refresh fails with plain - # TemplateRefreshBlocked (e.g. malformed template artifacts, all job - # builds failing). A fresh process must still arm the budget from its - # first such failure and take the budgeted restart path. - server = coordinator() - server.blockpoll_seconds = 0 - server.template_refresh_failure_exit_seconds = 10.0 - server._record_heartbeat = lambda _name: None # type: ignore[method-assign] - server.rpc = TipRpc("11" * 32) - clock = {"now": 100.0} - - def blocked_fetch() -> QbitTipTemplateSnapshot: - clock["now"] += 6.0 - raise TemplateRefreshBlocked( - "unable to derive exact artifacts for observed qbit template" - ) - - server.fetch_qbit_tip_template_snapshot = blocked_fetch # type: ignore[method-assign] - with ( - patch( - "lab.prism.prism_coordinator.time.monotonic", - side_effect=lambda: clock["now"], - ), - patch("lab.prism.prism_coordinator.traceback.print_exc"), - patch("lab.prism.prism_coordinator.os._exit", side_effect=SystemExit(1)) as exit_process, - self.assertRaises(SystemExit), - ): - server.blockpoll_loop() - - exit_process.assert_called_once_with(1) - self.assertIsNotNone(server.template_refresh_failure_started_monotonic) - - def test_healthy_noop_template_poll_resets_refresh_failure_clock(self) -> None: - server = coordinator() - server.blockpoll_seconds = 0 - server.last_successful_template_refresh_monotonic = 100.0 - server.template_refresh_failure_started_monotonic = 190.0 - server.template_refresh_failure_exit_seconds = 10.0 - server.coordination_blocked_exit_seconds = 10.0 - server._record_coordination_blocked_refresh(190.0) - server._record_heartbeat = lambda _name: None # type: ignore[method-assign] - snapshot = QbitTipTemplateSnapshot( - bestblockhash="11" * 32, - previousblockhash="11" * 32, - template_fingerprint="22" * 32, - ) - server.tip_template_snapshot = snapshot - server.rpc = TipRpc(snapshot.bestblockhash) - server.fetch_qbit_tip_template_snapshot = lambda: snapshot # type: ignore[method-assign] - - def trusted_chain_view(_tip: str) -> bool: - server.stop_event.set() - return True - - server.ensure_reorg_reconciled_for_tip = trusted_chain_view # type: ignore[method-assign] - with patch("lab.prism.prism_coordinator.time.monotonic", return_value=200.0): - server.blockpoll_loop() - - self.assertEqual(server.last_successful_template_refresh_monotonic, 200.0) - self.assertIsNone(server.template_refresh_failure_started_monotonic) - self.assertEqual(server.coordination_blocked_streak_age_seconds(300.0), 0.0) - self.assertFalse(server.coordination_blocked_streak_expired(300.0)) - self.assertFalse(server.template_refresh_failure_expired(300.0)) - self.assertIsNone(server.template_refresh_failure_started_monotonic) - server._record_template_refresh_failure(300.0) - self.assertEqual(server.template_refresh_failure_started_monotonic, 300.0) - - def test_shared_template_poll_records_success_for_blockwait_callers(self) -> None: - server = coordinator() - server.last_successful_template_refresh_monotonic = 100.0 - snapshot = QbitTipTemplateSnapshot( - bestblockhash="11" * 32, - previousblockhash="11" * 32, - template_fingerprint="22" * 32, - ) - server.rpc = TipRpc(snapshot.bestblockhash) - server.fetch_qbit_tip_template_snapshot = lambda: snapshot # type: ignore[method-assign] - server.ensure_reorg_reconciled_for_tip = lambda _tip: True # type: ignore[method-assign] - - with patch("lab.prism.prism_coordinator.time.monotonic", return_value=200.0): - refreshed = server.poll_qbit_tip_template_once(heartbeat_name="qbit_blockwait") - - self.assertEqual(refreshed, 0) - self.assertEqual(server.last_successful_template_refresh_monotonic, 200.0) - - def test_untrusted_reconciliation_exhausts_template_refresh_failure_budget(self) -> None: - server = coordinator() - server.blockpoll_seconds = 0 - server.last_successful_template_refresh_monotonic = 100.0 - server.template_refresh_failure_started_monotonic = 100.0 - server.template_refresh_failure_exit_seconds = 10.0 - server._record_heartbeat = lambda _name: None # type: ignore[method-assign] - snapshot = QbitTipTemplateSnapshot( - bestblockhash="11" * 32, - previousblockhash="11" * 32, - template_fingerprint="22" * 32, - ) - server.rpc = TipRpc(snapshot.bestblockhash) - server.fetch_qbit_tip_template_snapshot = lambda: snapshot # type: ignore[method-assign] - server.ensure_reorg_reconciled_for_tip = lambda _tip: False # type: ignore[method-assign] - - with ( - patch("lab.prism.prism_coordinator.time.monotonic", return_value=110.0), - patch("lab.prism.prism_coordinator.traceback.print_exc"), - patch("lab.prism.prism_coordinator.os._exit", side_effect=SystemExit(1)) as exit_process, - self.assertRaises(SystemExit), - ): - server.blockpoll_loop() - - exit_process.assert_called_once_with(1) - self.assertEqual(server.last_successful_template_refresh_monotonic, 100.0) - - def test_all_refresh_job_builds_failing_exhausts_failure_budget(self) -> None: - old_tip = "00" * 32 - new_tip = "33" * 32 - server = coordinator() - server.blockpoll_seconds = 0 - server.last_successful_template_refresh_monotonic = 100.0 - server.template_refresh_failure_started_monotonic = 100.0 - server.template_refresh_failure_exit_seconds = 10.0 - server._record_heartbeat = lambda _name: None # type: ignore[method-assign] - state = client() - state.username = "miner-a" - state.worker = worker_identity() - state.active_job = prism_context("old-job", old_tip, worker=state.worker) - state.active_job_ids = {"old-job"} - server.clients = {state} - server.jobs = {"old-job": state.active_job} - server.tip_template_snapshot = QbitTipTemplateSnapshot( - bestblockhash=old_tip, - previousblockhash=old_tip, - template_fingerprint=qbit_template_fingerprint(state.active_job.template), - ) - snapshot = QbitTipTemplateSnapshot( - bestblockhash=new_tip, - previousblockhash=new_tip, - template_fingerprint="44" * 32, - ) - server.rpc = TipRpc(new_tip) - server.fetch_qbit_tip_template_snapshot = lambda: snapshot # type: ignore[method-assign] - server.build_job_for_client = lambda *_args, **_kwargs: (_ for _ in ()).throw( # type: ignore[method-assign] - RuntimeError("template build unavailable") - ) - - with ( - patch("lab.prism.prism_coordinator.time.monotonic", return_value=110.0), - patch("lab.prism.prism_coordinator.traceback.print_exc"), - patch("lab.prism.prism_coordinator.os._exit", side_effect=SystemExit(1)) as exit_process, - self.assertRaises(SystemExit), - ): - server.blockpoll_loop() - - exit_process.assert_called_once_with(1) - self.assertEqual(server.job_build_failure_count, 1) - self.assertEqual(server.last_successful_template_refresh_monotonic, 100.0) - - def test_guarded_sequential_build_supersession_does_not_arm_failure_budget(self) -> None: - # Non-ready/collection mode: the sequential loop's guarded client - # build detects a superseded snapshot inside maybe_send_job. That is - # coordination churn, not template unhealthiness -- the real raise - # site must carry TemplateRefreshSuperseded so sustained pre-ready - # churn neither arms nor fires the budget. - old_tip = "00" * 32 - new_tip = "33" * 32 - server = coordinator() - server.blockpoll_seconds = 0 - server.template_refresh_failure_exit_seconds = 10.0 - server._record_heartbeat = lambda _name: None # type: ignore[method-assign] - state = client() - state.username = "miner-a" - state.worker = worker_identity() - state.active_job = prism_context("old-job", old_tip, worker=state.worker) - state.active_job_ids = {"old-job"} - server.clients = {state} - server.jobs = {"old-job": state.active_job} - server.tip_template_snapshot = QbitTipTemplateSnapshot( - bestblockhash=old_tip, - previousblockhash=old_tip, - template_fingerprint=qbit_template_fingerprint(state.active_job.template), - ) - snapshot = QbitTipTemplateSnapshot( - bestblockhash=new_tip, - previousblockhash=new_tip, - template_fingerprint="44" * 32, - ) - server.rpc = TipRpc(new_tip) - clock = {"now": 100.0} - fetches = 0 - - def fetch_snapshot() -> QbitTipTemplateSnapshot: - nonlocal fetches - fetches += 1 - clock["now"] += 6.0 - if fetches >= 4: - server.stop_event.set() - return snapshot - - server.fetch_qbit_tip_template_snapshot = fetch_snapshot # type: ignore[method-assign] - # The guarded pre-build currency check loses the race on every pass. - server._tip_refresh_snapshot_current_locked = ( # type: ignore[method-assign] - lambda *_args, **_kwargs: False - ) - - with ( - patch( - "lab.prism.prism_coordinator.time.monotonic", - side_effect=lambda: clock["now"], - ), - patch("lab.prism.prism_coordinator.traceback.print_exc"), - patch("lab.prism.prism_coordinator.os._exit", side_effect=SystemExit(1)) as exit_process, - ): - server.blockpoll_loop() - - self.assertEqual(fetches, 4) - self.assertGreater(clock["now"] - 100.0, server.template_refresh_failure_exit_seconds) - exit_process.assert_not_called() - self.assertIsNone( - getattr(server, "template_refresh_failure_started_monotonic", None) - ) - - def test_transient_template_refresh_failure_recovers_on_healthy_noop(self) -> None: - server = coordinator() - server.blockpoll_seconds = 0 - server.last_successful_template_refresh_monotonic = 100.0 - server.template_refresh_failure_exit_seconds = 10.0 - server._record_heartbeat = lambda _name: None # type: ignore[method-assign] - poll_count = 0 - - def fail_then_noop() -> int: - nonlocal poll_count - poll_count += 1 - if poll_count == 1: - raise RuntimeError("transient RPC failure") - server.last_successful_template_refresh_monotonic = time.monotonic() - server.stop_event.set() - return 0 - - server.poll_qbit_tip_template_once = fail_then_noop # type: ignore[method-assign] - with ( - patch("lab.prism.prism_coordinator.time.monotonic", side_effect=[105.0, 106.0]), - patch("lab.prism.prism_coordinator.traceback.print_exc"), - ): - server.blockpoll_loop() - - self.assertEqual(poll_count, 2) - self.assertEqual(server.last_successful_template_refresh_monotonic, 106.0) - - def _pending_append(self, tag: str, accepted_at_ms: int = 2) -> PendingShareAppend: - from lab.prism.share_ledger import PendingShare - - return PendingShareAppend( - pending_share=PendingShare( - share_id=f"miner-a:{tag}", - miner_id="miner-a", - order_key="miner-a", - p2mr_program_hex="11" * 32, - share_difficulty=1, - network_difficulty=1, - template_height=10, - job_id="job-1", - job_issued_at_ms=1, - accepted_at_ms=accepted_at_ms, - ntime=1_700_000_000, - ), - username="miner-a", - job_id="job-1", - block_hash_hex=tag * 32, - collection_only=False, - credit_policy=None, - ) - - def test_lost_mutation_response_publishes_fresh_balances_with_forced_rescan( - self, - ) -> None: - tip = "5e" * 32 - pool_block_hash = "af" * 32 - - class LostResponseLedger(ReorgLedger): - def __init__(self) -> None: - super().__init__([]) - self.prior_balance_reads = 0 - - def current_prior_balances(self) -> list[dict[str, object]]: - self.prior_balance_reads += 1 - if any(row["chain_state"] == "confirmed" for row in self.rows): - return [ - { - "recipient_id": "stale-carry", - "order_key": "01:stale-carry", - "p2mr_program_hex": "44" * 32, - "balance_sats": 546, - } - ] - return [] - - def mark_pool_block_inactive( - self, - *, - block_hash: str, - active_tip_height: int, - ) -> dict[str, object]: - result = super().mark_pool_block_inactive( - block_hash=block_hash, - active_tip_height=active_tip_height, - ) - if int(result["inactive_count"]): - raise ConnectionError("mutation response lost") - return result - - ledger = LostResponseLedger() - server = coordinator() - server.reorg_reconciler_enabled = True - server.ledger = ledger - server.rpc = ReorgRpc( - tip=tip, - template=gbt_template(tip, height=11), - height=10, - block_hashes={10: tip}, - ) - forced_rescans: list[bool] = [] - original_prepare = server._prepared_payout_state_candidate - - def record_prepare( - captured: tuple[int, int, str | None, str, float], - *, - force_full_window_rescan: bool = False, - bypass_build_interval: bool = False, - ) -> object: - forced_rescans.append(force_full_window_rescan) - return original_prepare( - captured, - force_full_window_rescan=force_full_window_rescan, - bypass_build_interval=bypass_build_interval, - ) - - server._prepared_payout_state_candidate = record_prepare # type: ignore[method-assign] - - first = server.reconcile_prism_pool_blocks_once(tip_hash=tip) - self.assertEqual(first["published_generation"], 1) - with server._job_cache_lock: - initially_published = server._published_payout_state.artifact - assert initially_published is not None - self.assertEqual(len(initially_published.prior_balances()), 0) - - ledger.rows.append( - { - "block_hash": pool_block_hash, - "block_height": 12, - "chain_state": "confirmed", - "maturity_state": "immature", - } - ) - # Rebuild the current generation's immutable state so the test starts - # from balances that predate the lost-response mutation. - with server._job_cache_lock: - server._published_payout_state = dataclass_replace( - server._published_payout_state, - artifact=None, - ) - stale_published = server._current_payout_state_artifact() - self.assertEqual(len(stale_published.prior_balances()), 1) - reads_before_error = ledger.prior_balance_reads - forced_rescans.clear() - - with self.assertRaisesRegex(ConnectionError, "mutation response lost"): - server.reconcile_prism_pool_blocks_once(tip_hash=tip) - - with server._job_cache_lock: - healed_state = server._published_payout_state - assert healed_state.artifact is not None - self.assertEqual(healed_state.generation, 2) - self.assertEqual(healed_state.artifact.prior_balances(), []) - self.assertEqual(ledger.prior_balance_reads, reads_before_error + 1) - self.assertEqual(forced_rescans, [True]) - self.assertEqual(ledger.rows[0]["chain_state"], "inactive") - - def test_read_phase_reconcile_failure_does_not_force_balance_rescan( - self, - ) -> None: - tip = "5f" * 32 - - class PrefetchFailureLedger(ReorgLedger): - fail_prefetch = False - - def __init__(self) -> None: - super().__init__([]) - self.prior_balance_reads = 0 - - def current_prior_balances(self) -> list[dict[str, object]]: - self.prior_balance_reads += 1 - return [] - - def reorg_watch_blocks( - self, - *, - active_tip_height: int, - ) -> list[dict[str, object]]: - if self.fail_prefetch: - raise TimeoutError("reconcile prefetch join exceeded 20s") - return super().reorg_watch_blocks( - active_tip_height=active_tip_height - ) - - ledger = PrefetchFailureLedger() - server = coordinator() - server.reorg_reconciler_enabled = True - server.ledger = ledger - server.rpc = ReorgRpc( - tip=tip, - template=gbt_template(tip, height=11), - height=10, - block_hashes={10: tip}, - ) - first = server.reconcile_prism_pool_blocks_once(tip_hash=tip) - self.assertEqual(first["published_generation"], 1) - reads_before_failure = ledger.prior_balance_reads - with server._job_cache_lock: - generation_before_failure = server._published_payout_state.generation - - forced_rescans: list[bool] = [] - original_prepare = server._prepared_payout_state_candidate - - def record_prepare( - captured: tuple[int, int, str | None, str, float], - *, - force_full_window_rescan: bool = False, - bypass_build_interval: bool = False, - ) -> object: - forced_rescans.append(force_full_window_rescan) - return original_prepare( - captured, - force_full_window_rescan=force_full_window_rescan, - bypass_build_interval=bypass_build_interval, - ) - - server._prepared_payout_state_candidate = record_prepare # type: ignore[method-assign] - ledger.fail_prefetch = True - - with self.assertRaisesRegex( - TimeoutError, - "reconcile prefetch join exceeded 20s", - ): - server.reconcile_prism_pool_blocks_once(tip_hash=tip) - - with server._job_cache_lock: - generation_after_failure = server._published_payout_state.generation - self.assertEqual(forced_rescans, []) - self.assertEqual(ledger.prior_balance_reads, reads_before_failure) - self.assertEqual(generation_after_failure, generation_before_failure) - - def test_concurrent_same_tip_reconciles_share_one_pass(self) -> None: - tip = "5c" * 32 - entered = threading.Event() - release = threading.Event() - - class BlockingReorgLedger(ReorgLedger): - def reorg_watch_blocks( - self, - *, - active_tip_height: int, - ) -> list[dict[str, object]]: - entered.set() - if not release.wait(timeout=5.0): - raise AssertionError("reconcile release was never signaled") - return super().reorg_watch_blocks( - active_tip_height=active_tip_height - ) - - ledger = BlockingReorgLedger([]) - server = coordinator() - server.reorg_reconciler_enabled = True - server.ledger = ledger - server.rpc = ReorgRpc( - tip=tip, - template=gbt_template(tip, height=11), - height=10, - block_hashes={10: tip}, - ) - server._ensure_job_cache_state() - - results: list[dict[str, object] | None] = [None, None] - - def reconcile(slot: int) -> None: - results[slot] = server.reconcile_prism_pool_blocks_once(tip_hash=tip) - - leader = threading.Thread(target=reconcile, args=(0,)) - leader.start() - self.assertTrue(entered.wait(timeout=5.0)) - with server._ensure_reorg_reconciler_service()._reconcile_flight_lock: - flight = server._ensure_reorg_reconciler_service()._reconcile_flights[tip] - - follower_waiting = threading.Event() - - class SignalingEvent: - def __init__(self, inner: threading.Event) -> None: - self._inner = inner - - def wait(self, timeout: float | None = None) -> bool: - follower_waiting.set() - return self._inner.wait(timeout) - - def set(self) -> None: - self._inner.set() - - flight.event = SignalingEvent(flight.event) # type: ignore[assignment] - follower = threading.Thread(target=reconcile, args=(1,)) - follower.start() - self.assertTrue(follower_waiting.wait(timeout=5.0)) - - release.set() - leader.join(timeout=5.0) - follower.join(timeout=5.0) - self.assertFalse(leader.is_alive()) - self.assertFalse(follower.is_alive()) - - self.assertEqual( - ledger.events, - [("watch", 10), ("mature", 10)], - "the follower must reuse the leader's pass, not run its own", - ) - self.assertEqual(results[0], results[1]) - self.assertIsNotNone(results[0]) - self.assertIsNot(results[0], results[1]) - with server._ensure_reorg_reconciler_service()._reconcile_flight_lock: - self.assertEqual(server._ensure_reorg_reconciler_service()._reconcile_flights, {}) - - def test_lock_owner_reconcile_bypasses_same_tip_flight(self) -> None: - tip = "5f" * 32 - ledger = ReorgLedger([]) - server = coordinator() - server.reorg_reconciler_enabled = True - server.ledger = ledger - server.rpc = ReorgRpc( - tip=tip, - template=gbt_template(tip, height=11), - height=10, - block_hashes={10: tip}, - ) - server._ensure_job_cache_state() - - leader_lock_attempted = threading.Event() - flight_waited = threading.Event() - flight_completed = threading.Event() - underlying_lock = threading.RLock() - leader: threading.Thread | None = None - - class ObservedBalanceLock: - def __enter__(self) -> ObservedBalanceLock: - if threading.current_thread() is leader: - if underlying_lock.acquire(blocking=False): - underlying_lock.release() - raise AssertionError( - "reconcile leader unexpectedly acquired the owned lock" - ) - leader_lock_attempted.set() - underlying_lock.acquire() - else: - underlying_lock.acquire() - return self - - def __exit__(self, *_args: object) -> None: - underlying_lock.release() - - class WaitForbiddenEvent: - def __init__(self, inner: threading.Event) -> None: - self._inner = inner - - def wait(self, timeout: float | None = None) -> bool: - flight_waited.set() - raise AssertionError( - "lock-owning reconciliation joined the same-tip flight" - ) - - def set(self) -> None: - flight_completed.set() - self._inner.set() - - server._payout_balance_mutation_lock = ObservedBalanceLock() # type: ignore[assignment] - leader_results: list[dict[str, object]] = [] - leader_errors: list[BaseException] = [] - - def lead_reconcile() -> None: - try: - leader_results.append( - server.reconcile_prism_pool_blocks_once(tip_hash=tip) - ) - except BaseException as exc: # noqa: BLE001 - asserted below - leader_errors.append(exc) - - leader = threading.Thread(target=lead_reconcile) - try: - with server._payout_balance_mutation_lock: - leader.start() - self.assertTrue(leader_lock_attempted.wait(timeout=5.0)) - with server._ensure_reorg_reconciler_service()._reconcile_flight_lock: - flight = server._ensure_reorg_reconciler_service()._reconcile_flights[tip] - flight.event = WaitForbiddenEvent( # type: ignore[assignment] - flight.event - ) - - bypassed = server.ensure_reorg_reconciled_for_tip( - tip, - _coalesce_same_tip=False, - ) - - self.assertTrue(bypassed) - self.assertFalse(flight_waited.is_set()) - self.assertTrue(leader.is_alive()) - self.assertEqual( - ledger.events, - [("watch", 10), ("mature", 10)], - ) - finally: - leader.join(timeout=5.0) - - self.assertFalse(leader.is_alive()) - if leader_errors: - raise leader_errors[0] - self.assertEqual(len(leader_results), 1) - self.assertTrue(flight_completed.is_set()) - self.assertFalse(flight_waited.is_set()) - self.assertEqual( - ledger.events, - [ - ("watch", 10), - ("mature", 10), - ("watch", 10), - ("mature", 10), - ], - ) - with server._ensure_reorg_reconciler_service()._reconcile_flight_lock: - self.assertEqual(server._ensure_reorg_reconciler_service()._reconcile_flights, {}) - - def test_lock_owner_reconcile_leads_visible_flight_when_absent(self) -> None: - tip = "60" * 32 - entered = threading.Event() - release = threading.Event() - - class BlockingReorgLedger(ReorgLedger): - def reorg_watch_blocks( - self, - *, - active_tip_height: int, - ) -> list[dict[str, object]]: - entered.set() - if not release.wait(timeout=5.0): - raise AssertionError("reconcile release was never signaled") - return super().reorg_watch_blocks( - active_tip_height=active_tip_height - ) - - ledger = BlockingReorgLedger([]) - server = coordinator() - server.reorg_reconciler_enabled = True - server.ledger = ledger - server.rpc = ReorgRpc( - tip=tip, - template=gbt_template(tip, height=11), - height=10, - block_hashes={10: tip}, - ) - server._ensure_job_cache_state() - - lock_owner_result: list[bool] = [] - follower_result: list[dict[str, object]] = [] - errors: list[BaseException] = [] - follower_waiting = threading.Event() - - class SignalingEvent: - def __init__(self, inner: threading.Event) -> None: - self._inner = inner - - def wait(self, timeout: float | None = None) -> bool: - follower_waiting.set() - return self._inner.wait(timeout) - - def set(self) -> None: - self._inner.set() - - def reconcile_while_owning_lock() -> None: - try: - with server._payout_balance_mutation_lock: - lock_owner_result.append( - server.ensure_reorg_reconciled_for_tip( - tip, - _coalesce_same_tip=False, - ) - ) - except BaseException as exc: # noqa: BLE001 - asserted below - errors.append(exc) - - def follow_reconcile() -> None: - try: - follower_result.append( - server.reconcile_prism_pool_blocks_once(tip_hash=tip) - ) - except BaseException as exc: # noqa: BLE001 - asserted below - errors.append(exc) - - lock_owner = threading.Thread(target=reconcile_while_owning_lock) - follower = threading.Thread(target=follow_reconcile) - lock_owner.start() - try: - self.assertTrue(entered.wait(timeout=5.0)) - with server._ensure_reorg_reconciler_service()._reconcile_flight_lock: - flight = server._ensure_reorg_reconciler_service()._reconcile_flights[tip] - flight.event = SignalingEvent(flight.event) # type: ignore[assignment] - - follower.start() - self.assertTrue(follower_waiting.wait(timeout=5.0)) - self.assertTrue(lock_owner.is_alive()) - self.assertTrue(follower.is_alive()) - self.assertEqual(ledger.events, []) - finally: - release.set() - lock_owner.join(timeout=5.0) - if follower.ident is not None: - follower.join(timeout=5.0) - - self.assertFalse(lock_owner.is_alive()) - self.assertFalse(follower.is_alive()) - if errors: - raise errors[0] - self.assertEqual(lock_owner_result, [True]) - self.assertEqual(len(follower_result), 1) - self.assertEqual( - ledger.events, - [("watch", 10), ("mature", 10)], - "the ordinary caller must reuse the lock owner's visible pass", - ) - with server._ensure_reorg_reconciler_service()._reconcile_flight_lock: - self.assertEqual(server._ensure_reorg_reconciler_service()._reconcile_flights, {}) - - def test_forced_and_reserved_reconciles_bypass_flight_reuse(self) -> None: - tip = "5e" * 32 - ledger = ReorgLedger([]) - server = coordinator() - server.reorg_reconciler_enabled = True - server.ledger = ledger - server.rpc = ReorgRpc( - tip=tip, - template=gbt_template(tip, height=11), - height=10, - block_hashes={10: tip}, - ) - server._ensure_job_cache_state() - - sentinel_summary: dict[str, object] = {"sentinel": True} - flight = _ReconcileFlight() - flight.summary = sentinel_summary - flight.event.set() - with server._ensure_reorg_reconciler_service()._reconcile_flight_lock: - server._ensure_reorg_reconciler_service()._reconcile_flights[tip] = flight - - joined = server.reconcile_prism_pool_blocks_once(tip_hash=tip) - self.assertEqual(joined, sentinel_summary) - self.assertIsNot(joined, sentinel_summary) - self.assertEqual(ledger.events, []) - - forced = server.reconcile_prism_pool_blocks_once( - tip_hash=tip, - _force_publish=True, - ) - self.assertEqual(forced["published_generation"], 1) - self.assertEqual(ledger.events, [("watch", 10), ("mature", 10)]) - - reserved = server.reconcile_prism_pool_blocks_once( - tip_hash=tip, - _source_reserved=True, - ) - self.assertIsNone(reserved["published_generation"]) - self.assertEqual( - ledger.events, - [("watch", 10), ("mature", 10), ("watch", 10), ("mature", 10)], - ) - - with server._ensure_reorg_reconciler_service()._reconcile_flight_lock: - server._ensure_reorg_reconciler_service()._reconcile_flights.pop(tip, None) - - def test_pass_spanning_detection_cycle_does_not_arm_memo(self) -> None: - # A pass whose reads straddle a flip away and back finishes with the - # latest detected hash matching its tip again, but its proof belongs - # to the closed epoch: arming must be refused so every memo consumer - # (refresh joins, initial-job and vardiff-idle builds) re-proves. - tip = "5c" * 32 - server = coordinator() - server.reorg_reconciler_enabled = True - ledger = ReorgLedger([]) - - def epoch_bumping_watch(*, active_tip_height: int) -> list[dict[str, object]]: - with server.lock: - server.tip_detection_epoch = ( - int(getattr(server, "tip_detection_epoch", 0)) + 2 - ) - return [] - - ledger.reorg_watch_blocks = epoch_bumping_watch # type: ignore[method-assign] - server.ledger = ledger - server.rpc = ReorgRpc( - tip=tip, - template=gbt_template(tip, height=11), - height=10, - block_hashes={10: tip}, - ) - - self.assertTrue(server.ensure_reorg_reconciled_for_tip(tip)) - - with server.lock: - self.assertNotIn(tip, server._ensure_reorg_reconciler_service()._reorg_reconcile_trusted_memo) - - def test_row_mutating_pass_evicts_other_tip_memo_entries(self) -> None: - tip_a = "a1" * 32 - tip_b = "b2" * 32 - pool_block_hash = "cd" * 32 - server = coordinator() - server.reorg_reconciler_enabled = True - ledger = ReorgLedger([]) - server.ledger = ledger - server.rpc = ReorgRpc( - tip=tip_b, - template=gbt_template(tip_b, height=11), - height=10, - block_hashes={10: tip_b}, - ) - server._ensure_job_cache_state() - server._note_reorg_reconcile_outcome(tip_a, trusted=True) - - # A pass for another tip that mutates no rows leaves tip A's cached - # proof valid: rows and tip-A chain state are both unchanged. - first = server.reconcile_prism_pool_blocks_once(tip_hash=tip_b) - self.assertEqual(first["published_generation"], 1) - with server.lock: - self.assertIn(tip_a, server._ensure_reorg_reconciler_service()._reorg_reconcile_trusted_memo) - self.assertIn(tip_b, server._ensure_reorg_reconciler_service()._reorg_reconcile_trusted_memo) - - # A pass that applies orphan/maturity row mutations ends every other - # tip's memo epoch, even though tip B was never observed by - # observe_tip_for_refresh. - ledger.rows.append( - { - "block_hash": pool_block_hash, - "block_height": 12, - "chain_state": "confirmed", - "maturity_state": "immature", - } - ) - second = server.reconcile_prism_pool_blocks_once(tip_hash=tip_b) - self.assertEqual(second["inactive_blocks"], 1) - with server.lock: - self.assertNotIn(tip_a, server._ensure_reorg_reconciler_service()._reorg_reconcile_trusted_memo) - self.assertIn(tip_b, server._ensure_reorg_reconciler_service()._reorg_reconcile_trusted_memo) - - def test_coordination_blocked_default_budget_is_fifteen_minutes(self) -> None: - server = PrismCoordinator.__new__(PrismCoordinator) - server._record_coordination_blocked_refresh(100.0) - - self.assertFalse(server.coordination_blocked_streak_expired(999.999)) - self.assertTrue(server.coordination_blocked_streak_expired(1000.0)) - - def test_sustained_blocked_refresh_storm_exits_via_coordination_budget(self) -> None: - server = coordinator() - server.blockpoll_seconds = 0 - server.template_refresh_failure_exit_seconds = 10.0 - server.coordination_blocked_exit_seconds = 10.0 - server._record_heartbeat = lambda _name: None # type: ignore[method-assign] - server.rpc = TipRpc("11" * 32) - clock = {"now": 100.0} - blocked_polls = 0 - - def blocked_fetch() -> QbitTipTemplateSnapshot: - nonlocal blocked_polls - blocked_polls += 1 - clock["now"] += 6.0 - if blocked_polls >= 4: - server.stop_event.set() - if blocked_polls % 2: - raise TemplateRefreshSuperseded( - "qbit tip changed during sequential refresh; immediate retry scheduled" - ) - raise _PayoutStatePublicationBlocked( - "payout state invalidation is pending publication" - ) - - server.fetch_qbit_tip_template_snapshot = blocked_fetch # type: ignore[method-assign] - with ( - patch( - "lab.prism.prism_coordinator.time.monotonic", - side_effect=lambda: clock["now"], - ), - patch("lab.prism.prism_coordinator.traceback.print_exc"), - ): - server.blockpoll_loop() - - self.assertEqual(blocked_polls, 4) - self.assertTrue(server.coordination_blocked_streak_expired(clock["now"])) - self.assertIsNone( - getattr(server, "template_refresh_failure_started_monotonic", None) - ) - server.stop_event.clear() - server.watchdog_enabled = False - server.watchdog_interval_seconds = 0.001 - with ( - patch( - "lab.prism.prism_coordinator.time.monotonic", - return_value=clock["now"], - ), - patch( - "lab.prism.prism_coordinator.os._exit", - side_effect=SystemExit(1), - ) as exit_process, - patch("builtins.print"), - self.assertRaises(SystemExit), - ): - server.watchdog_loop() - - exit_process.assert_called_once_with(1) - - -class AcceptedParentPreviewDeliveryBoundaryTests(_VardiffSupportTestCase): - """#182: the three job_delivery TemplateRefreshBlocked boundaries. - - The typed preview-pending exception must coalesce onto the ordinary - coordination retry at each boundary while staying outside - ``job_build_failure_count`` and the ordinary template-refresh failure - budget. - """ - - def _delivery_client(self, server: PrismCoordinator) -> ClientState: - state = client() - state.username = "miner-a" - state.worker = worker_identity() - state.send = lambda _payload: None # type: ignore[method-assign] - server.clients = {state} - return state - - @staticmethod - def _preview_pending() -> AcceptedParentPayoutPreviewPending: - return AcceptedParentPayoutPreviewPending( - "accepted parent payout preview is not ready yet", - parent_hash="cc" * 32, - waited_seconds=0.25, - timeout_count=3, - ) - - def test_reorg_boundary_propagates_preview_pending_unrelabelled(self) -> None: - # Boundary 2: the pre-build reorg-reconciliation guard. The generic - # clause below it converts failures into a plain TemplateRefreshBlocked - # (which is budgeted); the typed backpressure must pass through it - # unchanged instead. - server = coordinator() - state = self._delivery_client(server) - server.job_build_failure_count = 0 - raised = self._preview_pending() - - def blocked_reconcile() -> bool: - raise raised - - server.ensure_reorg_reconciled_for_current_tip = blocked_reconcile # type: ignore[method-assign] - server.build_job_for_client = lambda *_a, **_k: ( # type: ignore[method-assign] - self.fail("build must not run once the reorg guard blocks") - ) - - with ( - patch("builtins.print"), - self.assertRaises(AcceptedParentPayoutPreviewPending) as caught, - ): - server.maybe_send_job(state, clean_jobs=True) - - self.assertIs(caught.exception, raised) - self.assertEqual(server.job_build_failure_count, 0) - self.assertIsNone( - getattr(server, "template_refresh_failure_started_monotonic", None) - ) - - def test_build_boundary_retries_preview_pending_without_counting_it( - self, - ) -> None: - # Boundary 3: the client job-build boundary. Non-raising callers skip - # the job and retry; raising callers get the typed exception rather - # than the generic JobBuildFailed, and neither arms the failure count. - server = coordinator() - state = self._delivery_client(server) - server.job_build_failure_count = 0 - retries: list[int] = [] - server._schedule_tip_refresh_retry = lambda: retries.append(1) # type: ignore[method-assign] - server.ensure_reorg_reconciled_for_current_tip = lambda: True # type: ignore[method-assign] - raised = self._preview_pending() - - def blocked_build(*_args: object, **_kwargs: object) -> object: - raise raised - - server.build_job_for_client = blocked_build # type: ignore[method-assign] - - with patch("builtins.print") as logged: - self.assertFalse(server.maybe_send_job(state, clean_jobs=True)) - - self.assertEqual(server.job_build_failure_count, 0) - self.assertEqual(len(retries), 1) - # An explicit retry reason is logged instead of a build-failure - # traceback. - self.assertTrue( - any( - "accepted_parent_preview_pending" in str(call.args[0]) - for call in logged.call_args_list - if call.args - ) - ) - - # raise_on_build_failure must surface the typed backpressure, not the - # generic JobBuildFailed wrapper reserved for real build failures. - with ( - patch("builtins.print"), - self.assertRaises(AcceptedParentPayoutPreviewPending) as caught, - ): - server.maybe_send_job( - state, - clean_jobs=True, - raise_on_build_failure=True, - ) - - self.assertIs(caught.exception, raised) - self.assertEqual(server.job_build_failure_count, 0) - self.assertEqual(len(retries), 2) - - def test_build_boundary_still_counts_plain_build_failures(self) -> None: - # Negative control for boundary 3: an ordinary build failure keeps the - # generic accounting and the JobBuildFailed wrapper. - server = coordinator() - state = self._delivery_client(server) - server.job_build_failure_count = 0 - server.ensure_reorg_reconciled_for_current_tip = lambda: True # type: ignore[method-assign] - - def failing_build(*_args: object, **_kwargs: object) -> object: - raise RuntimeError("template build unavailable") - - server.build_job_for_client = failing_build # type: ignore[method-assign] - - with ( - patch("builtins.print"), - patch("lab.prism.job_delivery.traceback.print_exc"), - ): - self.assertFalse(server.maybe_send_job(state, clean_jobs=True)) - self.assertEqual(server.job_build_failure_count, 1) - - with ( - patch("builtins.print"), - patch("lab.prism.job_delivery.traceback.print_exc"), - self.assertRaises(JobBuildFailed), - ): - server.maybe_send_job( - state, - clean_jobs=True, - raise_on_build_failure=True, - ) - self.assertEqual(server.job_build_failure_count, 2) - - def test_sustained_preview_pending_exits_via_coordination_budget(self) -> None: - # The escalation that keeps this from retrying silently forever: the - # ordinary template-refresh budget stays disarmed, while the separate - # coordination-blocked streak expires and the watchdog exits. - server = coordinator() - server.blockpoll_seconds = 0 - server.template_refresh_failure_exit_seconds = 10.0 - server.coordination_blocked_exit_seconds = 10.0 - server._record_heartbeat = lambda _name: None # type: ignore[method-assign] - server.rpc = TipRpc("11" * 32) - clock = {"now": 100.0} - blocked_polls = 0 - - def blocked_fetch() -> QbitTipTemplateSnapshot: - nonlocal blocked_polls - blocked_polls += 1 - clock["now"] += 6.0 - if blocked_polls >= 4: - server.stop_event.set() - raise AcceptedParentPayoutPreviewPending( - "accepted parent payout preview is not ready yet" - ) - - server.fetch_qbit_tip_template_snapshot = blocked_fetch # type: ignore[method-assign] - with ( - patch( - "lab.prism.prism_coordinator.time.monotonic", - side_effect=lambda: clock["now"], - ), - patch("lab.prism.prism_coordinator.traceback.print_exc"), - patch("builtins.print"), - ): - server.blockpoll_loop() - - self.assertEqual(blocked_polls, 4) - self.assertIsNone( - getattr(server, "template_refresh_failure_started_monotonic", None) - ) - self.assertTrue(server.coordination_blocked_streak_expired(clock["now"])) - - server.stop_event.clear() - server.watchdog_enabled = False - server.watchdog_interval_seconds = 0.001 - with ( - patch( - "lab.prism.prism_coordinator.time.monotonic", - return_value=clock["now"], - ), - patch( - "lab.prism.prism_coordinator.os._exit", - side_effect=SystemExit(1), - ) as exit_process, - patch("builtins.print"), - self.assertRaises(SystemExit), - ): - server.watchdog_loop() - - exit_process.assert_called_once_with(1) - - -class PrismCoordinatorReliabilityTests(_VardiffSupportTestCase): - def _bare_coordinator(self) -> PrismCoordinator: - server = PrismCoordinator.__new__(PrismCoordinator) - server.lock = threading.RLock() - server.stop_event = threading.Event() - server._heartbeats = {} - server._watchdog_pauses = {} - server._heartbeats_lock = threading.Lock() - server.watchdog_timeout_seconds = 120.0 - server.watchdog_interval_seconds = 15.0 - return server - - def test_blockwait_failure_loop_reuses_existing_stop_event(self) -> None: - server = self._bare_coordinator() - tip = "aa" * 32 - server.rpc = SimpleNamespace(call=lambda method: tip) - server.blockpoll_seconds = 0.0 - server.observe_tip_for_refresh = lambda _tip: None # type: ignore[method-assign] - attempts = 0 - - def fail_blockwait(_known_tip: str) -> str: - nonlocal attempts - attempts += 1 - if attempts == 3: - server.stop_event.set() - raise RuntimeError("transient waitfornewblock failure") - - server.blockwait_once = fail_blockwait # type: ignore[method-assign] - # Construct the R1 owner up front: its own event allocations are - # owner state, while the loop itself must reuse the existing - # stop_event rather than allocating a fallback. - server._ensure_tip_refresh_service() - - with ( - patch("builtins.print"), - patch("lab.prism.tip_refresh.traceback.print_exc"), - patch( - "lab.prism.tip_refresh.threading.Event", - side_effect=AssertionError("allocated fallback stop event"), - ), - patch( - "lab.prism.prism_coordinator.threading.Event", - side_effect=AssertionError("allocated fallback stop event"), - ), - ): - server.blockwait_loop() - - self.assertEqual(attempts, 3) - - def test_blockwait_parameter_mismatch_is_treated_as_unsupported(self) -> None: - self.assertTrue( - PrismCoordinator._blockwait_unsupported( - RuntimeError("RPC error -32602: invalid params: wrong number of parameters") - ) - ) - - def test_blockwait_advances_known_tip_before_notification_failure(self) -> None: - server = self._bare_coordinator() - tip_a = "aa" * 32 - tip_b = "bb" * 32 - server.rpc = SimpleNamespace(call=lambda method: tip_a) - server.blockpoll_seconds = 1.0 - known_tips: list[str] = [] - detected_tips: list[str] = [] - - def blockwait_once(known_tip: str) -> str: - known_tips.append(known_tip) - if len(known_tips) == 2: - server.stop_event.set() - return tip_b - - server.blockwait_once = blockwait_once # type: ignore[method-assign] - - def observe_tip_for_refresh(tip_hash: str, **_kwargs: object) -> bool: - detected_tips.append(tip_hash) - if tip_hash == tip_b: - raise TemplateRefreshBlocked("notification failed") - return True - - def reject_premature_publication(*_args: object, **_kwargs: object) -> bool: - raise AssertionError("blockwait must not publish submit authority") - - server.observe_tip_for_refresh = observe_tip_for_refresh # type: ignore[method-assign] - server.observe_tip_first_seen = reject_premature_publication # type: ignore[method-assign] - - with patch("builtins.print"), patch( - "lab.prism.prism_coordinator.traceback.print_exc" - ), patch.object( - server.stop_event, - "wait", - side_effect=lambda _timeout: server.stop_event.is_set(), - ): - server.blockwait_loop() - - self.assertEqual(known_tips, [tip_a, tip_b]) - self.assertEqual(detected_tips, [tip_a, tip_b]) - - def test_blockwait_unsupported_removes_watchdog_heartbeat(self) -> None: - server = coordinator() - server.rpc = UnsupportedBlockwaitRpc("00" * 32) - server._record_heartbeat("qbit_blockwait") - - server.blockwait_loop() - - self.assertNotIn("qbit_blockwait", server._heartbeats) - self.assertNotIn("qbit_blockwait", server._watchdog_pauses) - - -class PrismStampedJobFloorTests(_VardiffSupportTestCase): - """The listener floor must hold on the wire, not just in vardiff policy. - - Stamped jobs are the single choke point for every mining.set_difficulty - the coordinator sends, and marketplace verification judges the first one. - The regression here is a young chain: qbit network difficulty below the - high-diff floor used to drag the advertised difficulty down with it. - """ - - def stamp_coordinator(self) -> PrismCoordinator: - server = coordinator() - server.job_counter = 0 - server.share_weights_by_username = {} - server.default_share_weight = 1 - return server - - def cached_bundle(self) -> CachedJobBundle: - # bits 207fffff: regtest-grade network difficulty (~4.7e-10), far - # below the 500k marketplace floor. - qbit_target = target_from_compact("207fffff") - base_job = direct_stratum.DirectQbitStratumJob( - job_id="prism-template-base", - previousblockhash_display="00" * 32, - prevhash="00" * 32, - coinb1="", - coinb2="", - full_coinbase_prefix="", - full_coinbase_suffix="", - merkle_branch=(), - transaction_hexes=(), - version="20000000", - nbits="207fffff", - ntime="6553f100", - qbit_target=qbit_target, - share_target=qbit_target, - share_difficulty=Decimal("1"), - extranonce1_hex="ffffffff", - extranonce2_size=8, - clean_jobs=True, - ) - return CachedJobBundle( - key=("test",), - template=gbt_template("00" * 32), - template_fingerprint="fp", - coinbase_manifest={}, - shares_json=[], - prior_balances=[], - found_block={"network_difficulty": 1}, - collection_only=False, - issued_at_ms=12345, - base_job=base_job, - built_monotonic=time.monotonic(), - ) - - def highdiff_client(self) -> ClientState: - state = client() - state.worker = worker_identity() - state.listener_vardiff_config = highdiff_vardiff_config() - state.minimum_advertised_difficulty = Decimal("500000") - state.share_difficulty = Decimal("500000") - return state - - def test_ready_pool_refreshes_clients_left_on_collection_jobs(self) -> None: - # Once the pool crosses min_ready_miners, the poller must replace - # collection jobs with windowed work even when the template snapshot - # is otherwise unchanged -- readiness itself is invisible to the - # template fingerprint. - server, state, _ledger = submit_coordinator() - tip = "00" * 32 - snapshot = SimpleNamespace( - bestblockhash=tip, - previousblockhash=tip, - template_fingerprint="fp", - ) - context = SimpleNamespace( - template={"previousblockhash": tip}, - template_fingerprint="fp", - collection_only=True, - ) - state.active_job = context - - server.min_ready_miners = 1 - server.accepted_share_stats = lambda: (0, 0) # type: ignore[method-assign] - self.assertFalse(server.pool_readiness_latched()) - self.assertFalse(server.client_needs_tip_template_refresh(state, snapshot)) - - server.accepted_share_stats = lambda: (1, 1) # type: ignore[method-assign] - self.assertTrue(server.pool_readiness_latched()) - self.assertTrue(server.client_needs_tip_template_refresh(state, snapshot)) - - # Readiness is monotonic: once latched the ledger is never consulted - # again, and ready (non-collection) jobs still need no refresh. - server.accepted_share_stats = None # type: ignore[assignment] - self.assertTrue(server.pool_readiness_latched()) - context.collection_only = False - self.assertFalse(server.client_needs_tip_template_refresh(state, snapshot)) - - -class JobBundleCacheTests(_JobSupportTestCase): - @staticmethod - def _capture_error( - errors: list[BaseException], - operation: object, - ) -> None: - try: - operation() # type: ignore[operator] - except BaseException as exc: # noqa: BLE001 - test thread handoff - errors.append(exc) - - def test_ready_tip_refresh_builds_once_and_stamps_every_client(self) -> None: - server, _ = coordinator() - recorded = install_fake_bundle_builder(server) - clients = [client(1), client(2), client(3)] - clients[1].pending_share_difficulty = Decimal("8") - sent: dict[int, list[dict[str, object]]] = {state.connection_id: [] for state in clients} - for state in clients: - state.send = ( # type: ignore[method-assign] - lambda payload, connection_id=state.connection_id: sent[connection_id].append(payload) - ) - server.clients = set(clients) - - try: - refreshed = server.poll_qbit_tip_template_once() - finally: - server.shutdown_tip_refresh_executor() - - self.assertEqual(refreshed, 3) - self.assertEqual(recorded["calls"], 1) - contexts = [state.active_job for state in clients] - self.assertEqual(len({context.job.job_id for context in contexts}), 3) - self.assertEqual( - [context.job.extranonce1_hex for context in contexts], - [state.extranonce1_hex for state in clients], - ) - self.assertEqual(contexts[1].job.share_difficulty, Decimal("8")) - self.assertEqual( - [payload["method"] for payload in sent[2]], - ["mining.set_difficulty", "mining.notify"], - ) - metrics = server.metrics_payload() - self.assertIn('qbit_prism_tip_refresh_clients_total{result="sent"} 3', metrics) - self.assertIn("qbit_prism_tip_refresh_first_delivery_seconds_count 1", metrics) - self.assertIn("qbit_prism_tip_refresh_last_delivery_seconds_count 1", metrics) - - def test_ready_tip_refresh_shares_one_bundle_across_250_clients(self) -> None: - server, rpc = coordinator() - recorded = install_fake_bundle_builder(server) - server.reorg_reconciler_enabled = True - reconciled: list[str] = [] - trust_checks = 0 - - def reconcile_once(tip_hash: str) -> bool: - reconciled.append(tip_hash) - return True - - def chain_view_untrusted() -> bool: - nonlocal trust_checks - trust_checks += 1 - return False - - server.ensure_reorg_reconciled_for_tip = reconcile_once # type: ignore[method-assign] - server.qbit_chain_view_untrusted = chain_view_untrusted # type: ignore[method-assign] - server.ensure_reorg_reconciled_for_current_tip = ( # type: ignore[method-assign] - lambda **_kwargs: self.fail("fanout repeated current-tip validation") - ) - clients = [client(index + 1) for index in range(250)] - sent: dict[int, list[dict[str, object]]] = { - state.connection_id: [] for state in clients - } - for state in clients: - state.send = ( # type: ignore[method-assign] - lambda payload, connection_id=state.connection_id: sent[ - connection_id - ].append(payload) - ) - server.clients = set(clients) - - try: - refreshed = server.poll_qbit_tip_template_once() - finally: - server.shutdown_tip_refresh_executor() - - self.assertEqual(refreshed, 250) - self.assertEqual(recorded["calls"], 1) - cached = next(iter(server._job_bundle_cache.values())) - self.assertEqual( - len({id(state.active_job.shares_json) for state in clients}), - 1, - ) - self.assertTrue( - all(state.active_job.shares_json is cached.shares_json for state in clients) - ) - self.assertEqual(reconciled, [rpc.tip]) - self.assertEqual(trust_checks, 2) - # The early priority probe, snapshot coherence, pre-fanout validation, - # and post-fanout detection are each constant-cost regardless of - # client count. - self.assertEqual(rpc.count("getbestblockhash"), 4) - self.assertTrue(all(len(payloads) == 2 for payloads in sent.values())) - fingerprints = {state.active_job.template_fingerprint for state in clients} - self.assertEqual(len(fingerprints), 1) - - def test_supersession_retry_wakes_blockpoll_without_full_interval(self) -> None: - server, _ = coordinator() - server.blockpoll_seconds = 60.0 - server._ensure_tip_refresh_state() - poll_called = threading.Event() - - def poll_once() -> int: - poll_called.set() - server.stop_event.set() - return 0 - - server.poll_qbit_tip_template_once = poll_once # type: ignore[method-assign] - thread = threading.Thread(target=server.blockpoll_loop) - thread.start() - try: - server._schedule_tip_refresh_retry() - self.assertTrue(poll_called.wait(1)) - finally: - server.stop_event.set() - server._schedule_tip_refresh_retry() - thread.join(1) - - self.assertFalse(thread.is_alive()) - - def test_ready_tip_refresh_respects_executor_bound(self) -> None: - server, _ = coordinator() - install_fake_bundle_builder(server) - server.tip_refresh_max_workers = 2 - clients = [client(index + 1) for index in range(6)] - server.clients = set(clients) - release = threading.Event() - two_started = threading.Event() - counter_lock = threading.Lock() - active = 0 - maximum = 0 - - def send(payload: dict[str, object]) -> None: - nonlocal active, maximum - if payload["method"] != "mining.notify": - return - with counter_lock: - active += 1 - maximum = max(maximum, active) - if active == 2: - two_started.set() - try: - self.assertTrue(release.wait(5)) - finally: - with counter_lock: - active -= 1 - - for state in clients: - state.send = send # type: ignore[method-assign] - result: list[int] = [] - thread = threading.Thread(target=lambda: result.append(server.poll_qbit_tip_template_once())) - thread.start() - try: - self.assertTrue(two_started.wait(5)) - self.assertLessEqual(maximum, 2) - finally: - release.set() - thread.join(5) - server.shutdown_tip_refresh_executor() - - self.assertFalse(thread.is_alive()) - self.assertEqual(result, [6]) - self.assertEqual(maximum, 2) - - def test_blocked_socket_does_not_delay_another_client(self) -> None: - server, _ = coordinator() - install_fake_bundle_builder(server) - server.tip_refresh_max_workers = 2 - blocked = client(1) - healthy = client(2) - server.clients = {blocked, healthy} - blocked_started = threading.Event() - healthy_delivered = threading.Event() - release = threading.Event() - - def blocked_send(payload: dict[str, object]) -> None: - if payload["method"] == "mining.notify": - blocked_started.set() - self.assertTrue(release.wait(5)) - - def healthy_send(payload: dict[str, object]) -> None: - if payload["method"] == "mining.notify": - healthy_delivered.set() - - blocked.send = blocked_send # type: ignore[method-assign] - healthy.send = healthy_send # type: ignore[method-assign] - result: list[int] = [] - thread = threading.Thread(target=lambda: result.append(server.poll_qbit_tip_template_once())) - thread.start() - try: - self.assertTrue(blocked_started.wait(5)) - self.assertTrue(healthy_delivered.wait(5)) - finally: - release.set() - thread.join(5) - server.shutdown_tip_refresh_executor() - - self.assertEqual(result, [2]) - - def test_shutdown_drains_inflight_tip_refresh_worker(self) -> None: - server, _ = coordinator() - install_fake_bundle_builder(server) - server.tip_refresh_max_workers = 1 - state = client(1) - server.clients = {state} - worker_started = threading.Event() - worker_send_finished = threading.Event() - release_worker = threading.Event() - shutdown_complete = threading.Event() - poll_errors: list[BaseException] = [] - - def blocked_send(payload: dict[str, object]) -> None: - if payload["method"] != "mining.notify": - return - worker_started.set() - try: - self.assertTrue(release_worker.wait(5)) - finally: - worker_send_finished.set() - - def poll() -> None: - try: - server.poll_qbit_tip_template_once() - except BaseException as exc: # noqa: BLE001 - surface to the test - poll_errors.append(exc) - - def shutdown() -> None: - server.shutdown_tip_refresh_executor() - shutdown_complete.set() - - state.send = blocked_send # type: ignore[method-assign] - poll_thread = threading.Thread(target=poll) - shutdown_thread = threading.Thread(target=shutdown) - poll_thread.start() - try: - self.assertTrue(worker_started.wait(5)) - server.stop_event.set() - shutdown_thread.start() - self.assertFalse(shutdown_complete.wait(0.05)) - self.assertFalse(worker_send_finished.is_set()) - finally: - release_worker.set() - shutdown_thread.join(5) - poll_thread.join(5) - - self.assertFalse(shutdown_thread.is_alive()) - self.assertFalse(poll_thread.is_alive()) - self.assertTrue(shutdown_complete.is_set()) - self.assertTrue(worker_send_finished.is_set()) - self.assertEqual(poll_errors, []) - self.assertEqual(server.tip_refresh_inflight, 0) - with self.assertRaisesRegex(RuntimeError, "executor is shut down"): - server.tip_refresh_executor() - - def test_queued_fanout_stops_when_chain_view_becomes_untrusted(self) -> None: - server, _ = coordinator() - install_fake_bundle_builder(server) - server.reorg_reconciler_enabled = True - server.ensure_reorg_reconciled_for_tip = lambda _tip: True # type: ignore[method-assign] - trust_checks = 0 - - def chain_view_untrusted() -> bool: - nonlocal trust_checks - trust_checks += 1 - return True - - server.qbit_chain_view_untrusted = chain_view_untrusted # type: ignore[method-assign] - first = client(1) - second = client(2) - server.clients = [first, second] # type: ignore[assignment] - sent: list[dict[str, object]] = [] - first.send = sent.append # type: ignore[method-assign] - second.send = sent.append # type: ignore[method-assign] - try: - with self.assertRaisesRegex(TemplateRefreshBlocked, "became untrusted"): - server.poll_qbit_tip_template_once() - finally: - server.shutdown_tip_refresh_executor() - - self.assertEqual(trust_checks, 1) - self.assertEqual(sent, []) - self.assertIsNone(first.active_job) - self.assertIsNone(second.active_job) - - def test_queued_fanout_stops_when_live_tip_changes(self) -> None: - server, rpc = coordinator() - install_fake_bundle_builder(server) - server.tip_refresh_max_workers = 1 - first = client(1) - second = client(2) - server.clients = [first, second] # type: ignore[assignment] - first_blocked = threading.Event() - release_first = threading.Event() - second_sent: list[dict[str, object]] = [] - - def first_send(payload: dict[str, object]) -> None: - if payload["method"] == "mining.notify": - first_blocked.set() - self.assertTrue(release_first.wait(5)) - - first.send = first_send # type: ignore[method-assign] - second.send = second_sent.append # type: ignore[method-assign] - refreshed: list[int] = [] - errors: list[BaseException] = [] - - def poll() -> None: - try: - refreshed.append(server.poll_qbit_tip_template_once()) - except BaseException as exc: # noqa: BLE001 - surface to the test - errors.append(exc) - - thread = threading.Thread(target=poll) - thread.start() - try: - self.assertTrue(first_blocked.wait(5)) - rpc.tip = "33" * 32 - finally: - release_first.set() - thread.join(5) - server.shutdown_tip_refresh_executor() - - self.assertFalse(thread.is_alive()) - self.assertEqual(refreshed, []) - self.assertEqual(len(errors), 1) - self.assertIsInstance(errors[0], TemplateRefreshBlocked) - self.assertIn("immediate retry scheduled", str(errors[0])) - self.assertTrue(server._tip_refresh_retry.is_set()) - self.assertIsNotNone(first.active_job) - self.assertIsNotNone(second.active_job) - self.assertEqual( - [payload["method"] for payload in second_sent], - ["mining.set_difficulty", "mining.notify"], - ) - - def test_multiworker_cancel_releases_client_lock_while_draining_peer(self) -> None: - server, _ = coordinator() - install_fake_bundle_builder(server) - server.tip_refresh_max_workers = 1 - admitted = client(1) - queued = client(2) - server.clients = [admitted, queued] # type: ignore[assignment] - admitted_send_started = threading.Event() - release_admitted_send = threading.Event() - queued_sent: list[dict[str, object]] = [] - - def admitted_send(payload: dict[str, object]) -> None: - if payload["method"] == "mining.notify": - admitted_send_started.set() - self.assertTrue(release_admitted_send.wait(5)) - - admitted.send = admitted_send # type: ignore[method-assign] - queued.send = queued_sent.append # type: ignore[method-assign] - refreshed: list[int] = [] - errors: list[BaseException] = [] - - def poll() -> None: - try: - refreshed.append(server.poll_qbit_tip_template_once()) - except BaseException as exc: # noqa: BLE001 - surface to the test - errors.append(exc) - - thread = threading.Thread(target=poll) - thread.start() - try: - self.assertTrue(admitted_send_started.wait(5)) - server.observe_tip_for_refresh("33" * 32) - lock_acquired = queued.job_update_lock.acquire(timeout=0.1) - self.assertTrue(lock_acquired) - if lock_acquired: - queued.job_update_lock.release() - # The coordinator still waits for the admitted peer delivery, but - # queued workers observe cancellation without taking client state. - self.assertTrue(thread.is_alive()) - self.assertIsNone(queued.active_job) - finally: - release_admitted_send.set() - thread.join(5) - server.shutdown_tip_refresh_executor() - - self.assertFalse(thread.is_alive()) - self.assertEqual(refreshed, []) - self.assertEqual(len(errors), 1) - self.assertIsInstance(errors[0], TemplateRefreshBlocked) - self.assertIsNotNone(admitted.active_job) - self.assertIsNone(queued.active_job) - self.assertEqual(queued_sent, []) - - def test_same_tip_cache_refresh_during_fanout_does_not_abort(self) -> None: - server, rpc = coordinator() - install_fake_bundle_builder(server) - server.tip_refresh_max_workers = 1 - first = client(1) - second = client(2) - # Preserve task order so the cache replacement happens after one - # delivery and before the next worker task starts. - server.clients = [first, second] # type: ignore[assignment] - first_blocked = threading.Event() - release_first = threading.Event() - second_sent: list[dict[str, object]] = [] - - def first_send(payload: dict[str, object]) -> None: - if payload["method"] == "mining.notify": - first_blocked.set() - self.assertTrue(release_first.wait(5)) - - first.send = first_send # type: ignore[method-assign] - second.send = second_sent.append # type: ignore[method-assign] - refreshed: list[int] = [] - errors: list[BaseException] = [] - - def poll() -> None: - try: - refreshed.append(server.poll_qbit_tip_template_once()) - except BaseException as exc: # noqa: BLE001 - surface to the test - errors.append(exc) - - thread = threading.Thread(target=poll) - thread.start() - try: - self.assertTrue(first_blocked.wait(5)) - replacement = dict(rpc.template) - replacement["coinbasevalue"] = int(replacement["coinbasevalue"]) + 1 - replacement_artifacts = server.store_template_artifacts(replacement) - self.assertIsNotNone(replacement_artifacts) - self.assertNotEqual( - replacement_artifacts.fingerprint, - first.active_job.template_fingerprint, - ) - finally: - release_first.set() - thread.join(5) - server.shutdown_tip_refresh_executor() - - self.assertFalse(thread.is_alive()) - self.assertEqual(errors, []) - self.assertEqual(refreshed, [2]) - self.assertIsNotNone(server.last_successful_template_refresh_monotonic) - self.assertEqual( - [payload["method"] for payload in second_sent], - ["mining.set_difficulty", "mining.notify"], - ) - - def test_queued_fanout_does_not_overwrite_intervening_job(self) -> None: - server, rpc = coordinator() - install_fake_bundle_builder(server) - server.tip_refresh_max_workers = 1 - first = client(1) - second = client(2) - server.clients = [first, second] # type: ignore[assignment] - first_blocked = threading.Event() - release_first = threading.Event() - second_sent: list[dict[str, object]] = [] - - def first_send(payload: dict[str, object]) -> None: - if payload["method"] == "mining.notify": - first_blocked.set() - self.assertTrue(release_first.wait(5)) - - first.send = first_send # type: ignore[method-assign] - second.send = second_sent.append # type: ignore[method-assign] - refreshed: list[int] = [] - errors: list[BaseException] = [] - - def poll() -> None: - try: - refreshed.append(server.poll_qbit_tip_template_once()) - except BaseException as exc: # noqa: BLE001 - surface to the test - errors.append(exc) - - thread = threading.Thread(target=poll) - thread.start() - try: - self.assertTrue(first_blocked.wait(5)) - replacement = dict(rpc.template) - replacement["coinbasevalue"] = int(replacement["coinbasevalue"]) + 1 - replacement_artifacts = server.store_template_artifacts(replacement) - self.assertIsNotNone(replacement_artifacts) - self.assertTrue(server.maybe_send_job(second, clean_jobs=False)) - intervening_job = second.active_job - self.assertEqual( - intervening_job.template_fingerprint, - replacement_artifacts.fingerprint, - ) - self.assertGreater( - intervening_job.template_generation, - first.active_job.template_generation, - ) - finally: - release_first.set() - thread.join(5) - server.shutdown_tip_refresh_executor() - - self.assertFalse(thread.is_alive()) - self.assertEqual(errors, []) - self.assertEqual(refreshed, [1]) - self.assertIs(second.active_job, intervening_job) - self.assertEqual( - [payload["method"] for payload in second_sent], - ["mining.set_difficulty", "mining.notify"], - ) - - def test_queued_fanout_replaces_stale_intervening_job(self) -> None: - server, rpc = coordinator() - install_fake_bundle_builder(server) - server.tip_refresh_max_workers = 1 - # The poll must observe the same-tip template rotation immediately for - # the queued-fanout replacement race below; disable the same-tip - # template reuse window so the rotation is fetched, not deferred. - server.template_cache_seconds = 0.0 - first = client(1) - second = client(2) - server.clients = [first, second] # type: ignore[assignment] - old_artifacts = server.store_template_artifacts(dict(rpc.template)) - self.assertIsNotNone(old_artifacts) - assert old_artifacts is not None - assert second.worker is not None - old_bundle = server.shared_job_bundle(old_artifacts, second.worker) - refreshed_template = dict(rpc.template) - refreshed_template["coinbasevalue"] = int( - refreshed_template["coinbasevalue"] - ) + 1 - rpc.template = refreshed_template - first_blocked = threading.Event() - release_first = threading.Event() - second_sent: list[dict[str, object]] = [] - - def first_send(payload: dict[str, object]) -> None: - if payload["method"] == "mining.notify": - first_blocked.set() - self.assertTrue(release_first.wait(5)) - - first.send = first_send # type: ignore[method-assign] - second.send = second_sent.append # type: ignore[method-assign] - refreshed: list[int] = [] - errors: list[BaseException] = [] - - def poll() -> None: - try: - refreshed.append(server.poll_qbit_tip_template_once()) - except BaseException as exc: # noqa: BLE001 - surface to the test - errors.append(exc) - - thread = threading.Thread(target=poll) - thread.start() - try: - self.assertTrue(first_blocked.wait(5)) - with second.job_update_lock, server.lock: - stale_intervening_job = server.stamp_job_for_client( - second, - old_bundle, - clean_jobs=False, - ) - second.active_job = stale_intervening_job - second.active_job_ids.add(stale_intervening_job.job.job_id) - server.jobs[stale_intervening_job.job.job_id] = stale_intervening_job - finally: - release_first.set() - thread.join(5) - server.shutdown_tip_refresh_executor() - - self.assertFalse(thread.is_alive()) - self.assertEqual(errors, []) - self.assertEqual(refreshed, [2]) - self.assertIsNot(second.active_job, stale_intervening_job) - self.assertEqual( - second.active_job.template_fingerprint, - qbit_template_fingerprint(refreshed_template), - ) - self.assertGreater( - second.active_job.template_generation, - stale_intervening_job.template_generation, - ) - self.assertEqual( - [payload["method"] for payload in second_sent], - ["mining.set_difficulty", "mining.notify"], - ) - - def test_newer_template_does_not_supersede_current_payout_refresh(self) -> None: - server, rpc = coordinator() - install_fake_bundle_builder(server) - state = client(1) - artifacts = server.store_template_artifacts(dict(rpc.template)) - self.assertIsNotNone(artifacts) - assert artifacts is not None and state.worker is not None - stale_bundle = server.shared_job_bundle(artifacts, state.worker) - snapshot = server.fetch_qbit_tip_template_snapshot() - stale_intervening_job = dataclass_replace( - server.stamp_job_for_client( - state, - stale_bundle, - clean_jobs=False, - ), - template_generation=snapshot.template_generation + 1, - ) - server._advance_payout_state_generation() - - self.assertFalse( - server.intervening_job_supersedes_snapshot( - stale_intervening_job, - None, - snapshot, - ) - ) - current_intervening_job = dataclass_replace( - stale_intervening_job, - payout_state_generation=server._payout_state_generation, - ) - self.assertTrue( - server.intervening_job_supersedes_snapshot( - current_intervening_job, - None, - snapshot, - ) - ) - - def test_higher_generation_old_tip_does_not_supersede_new_tip_snapshot(self) -> None: - server, rpc = coordinator() - install_fake_bundle_builder(server) - state = client(1) - old_artifacts = server.store_template_artifacts(dict(rpc.template)) - self.assertIsNotNone(old_artifacts) - assert old_artifacts is not None and state.worker is not None - old_bundle = server.shared_job_bundle(old_artifacts, state.worker) - - new_tip = "22" * 32 - rpc.tip = new_tip - rpc.template = base_template(height=11, prevhash=new_tip) - snapshot = server.fetch_qbit_tip_template_snapshot() - old_tip_job = dataclass_replace( - server.stamp_job_for_client( - state, - old_bundle, - clean_jobs=False, - ), - template_generation=snapshot.template_generation + 1, - ) - - self.assertNotEqual( - old_tip_job.template_fingerprint, - snapshot.template_fingerprint, - ) - self.assertFalse( - server.intervening_job_supersedes_snapshot( - old_tip_job, - None, - snapshot, - ) - ) - - def test_broken_socket_disconnects_only_that_client(self) -> None: - server, _ = coordinator() - install_fake_bundle_builder(server) - broken = client(1) - healthy = client(2) - server.clients = {broken, healthy} - healthy_sent: list[dict[str, object]] = [] - disconnected: list[ClientState] = [] - broken.send = lambda _payload: (_ for _ in ()).throw(OSError("closed")) # type: ignore[method-assign] - healthy.send = healthy_sent.append # type: ignore[method-assign] - server.disconnect_client = disconnected.append # type: ignore[method-assign] - - try: - refreshed = server.poll_qbit_tip_template_once() - finally: - server.shutdown_tip_refresh_executor() - - self.assertEqual(refreshed, 1) - self.assertEqual(disconnected, [broken]) - self.assertEqual( - [payload["method"] for payload in healthy_sent], - ["mining.set_difficulty", "mining.notify"], - ) - - def test_client_removed_before_pending_task_runs_is_skipped(self) -> None: - server, _ = coordinator() - install_fake_bundle_builder(server) - server.tip_refresh_max_workers = 1 - first = client(1) - removed = client(2) - clients = [first, removed] - server.clients = set(clients) - snapshot = server.fetch_qbit_tip_template_snapshot() - server.observe_tip_first_seen(snapshot.bestblockhash) - server.pool_readiness_latched() - server.tip_template_snapshot = snapshot - bundle = server.prepare_tip_refresh_bundle(snapshot) - blocked = threading.Event() - release = threading.Event() - - def first_send(payload: dict[str, object]) -> None: - if payload["method"] == "mining.notify": - blocked.set() - self.assertTrue(release.wait(5)) - - first.send = first_send # type: ignore[method-assign] - removed.send = lambda _payload: self.fail("removed client received a job") # type: ignore[method-assign] - result: list[tuple[int, float | None, float | None, int]] = [] - thread = threading.Thread( - target=lambda: result.append( - server._fanout_prepared_tip_refresh( - clients, - bundle, - snapshot, - heartbeat_name="qbit_blockpoll", - ) - ) - ) - thread.start() - try: - self.assertTrue(blocked.wait(5)) - with server.lock: - server.clients.remove(removed) - finally: - release.set() - thread.join(5) - server.shutdown_tip_refresh_executor() - - self.assertEqual(result[0][0], 1) - self.assertIsNone(removed.active_job) - self.assertEqual(removed.active_job_ids, set()) - - def test_template_fingerprint_race_uses_snapshot_owned_artifacts(self) -> None: - server, _ = coordinator() - install_fake_bundle_builder(server) - states = [client(1), client(2)] - sent: list[dict[str, object]] = [] - for state in states: - state.send = sent.append # type: ignore[method-assign] - server.clients = set(states) - original_shared_job_bundle = server.shared_job_bundle - race_calls = 0 - - def race_artifacts( - artifacts: object, - identity: WorkerIdentity | None = None, - **kwargs: object, - ) -> object: - nonlocal race_calls - race_calls += 1 - bundle = original_shared_job_bundle(artifacts, identity, **kwargs) - with server._job_cache_lock: - server._template_artifacts = dataclass_replace( - server._template_artifacts, - fingerprint="ff" * 32, - ) - return bundle - - server.shared_job_bundle = race_artifacts # type: ignore[method-assign] - - refreshed = server.poll_qbit_tip_template_once() - - self.assertEqual(race_calls, 1) - self.assertEqual(refreshed, 2) - self.assertEqual(len(sent), 4) - self.assertIsNotNone(server.tip_template_snapshot) - snapshot = server.tip_template_snapshot - assert snapshot is not None and snapshot.template_artifacts is not None - for state in states: - self.assertIs(state.active_job.template, snapshot.template_artifacts.template) - self.assertEqual( - state.active_job.template_fingerprint, - snapshot.template_fingerprint, - ) - - -class ClientCleanupTests(_JobSupportTestCase): - def test_disconnect_retires_and_closes_before_job_lock_cleanup(self) -> None: - server, _ = coordinator() - state = client(1) - server.clients = {state} - socket_closed = threading.Event() - state.close = socket_closed.set # type: ignore[method-assign] - state.job_update_lock.acquire() - disconnect = threading.Thread(target=server.disconnect_client, args=(state,)) - try: - disconnect.start() - self.assertTrue(socket_closed.wait(5)) - with server.lock: - self.assertNotIn(state, server.clients) - self.assertTrue(state.closing) - self.assertTrue(disconnect.is_alive()) - finally: - state.job_update_lock.release() - disconnect.join(5) - - self.assertFalse(disconnect.is_alive()) - - def test_disconnect_during_prepared_refresh_skips_without_job_state(self) -> None: - server, _ = coordinator() - install_fake_bundle_builder(server) - state = client(1) - observed_lock = ObservedRLock() - state.job_update_lock = observed_lock # type: ignore[assignment] - server.clients = {state} - snapshot = server.fetch_qbit_tip_template_snapshot() - server.observe_tip_first_seen(snapshot.bestblockhash) - server.pool_readiness_latched() - server.tip_template_snapshot = snapshot - bundle = server.prepare_tip_refresh_bundle(snapshot) - state.send = lambda _payload: self.fail( # type: ignore[method-assign] - "disconnected client received prepared work" - ) - socket_closed = threading.Event() - state.close = socket_closed.set # type: ignore[method-assign] - results: list[tuple[int, float | None, float | None, int]] = [] - errors: list[BaseException] = [] - - def refresh() -> None: - try: - results.append( - server._fanout_prepared_tip_refresh( - [state], - bundle, - snapshot, - heartbeat_name="qbit_blockpoll", - ) - ) - except BaseException as exc: # noqa: BLE001 - surface thread failures - errors.append(exc) - - observed_lock.acquire() - observed_lock.observe_acquires = True - refresh_thread = threading.Thread(target=refresh) - disconnect_thread = threading.Thread( - target=server.disconnect_client, - args=(state,), - ) - try: - refresh_thread.start() - self.assertTrue(observed_lock.acquire_attempted.wait(5)) - disconnect_thread.start() - self.assertTrue(socket_closed.wait(5)) - refresh_thread.join(5) - self.assertFalse(refresh_thread.is_alive()) - self.assertTrue(disconnect_thread.is_alive()) - finally: - observed_lock.release() - refresh_thread.join(5) - disconnect_thread.join(5) - server.shutdown_tip_refresh_executor() - - self.assertFalse(disconnect_thread.is_alive()) - self.assertEqual(errors, []) - self.assertEqual(results[0][0], 0) - self.assertIsNone(state.active_job) - self.assertEqual(state.active_job_ids, set()) - self.assertEqual(server.jobs, {}) - - def test_mass_disconnect_releases_active_connection_accounting(self) -> None: - server, _ = coordinator() - states = [client(index) for index in range(1, 129)] - for state in states: - state.close = lambda: None # type: ignore[method-assign] - server.clients = set(states) - - for state in states: - server.disconnect_client(state) - - with server.lock: - self.assertEqual(len(server.clients), 0) - self.assertTrue(all(state.closing for state in states)) - - def test_concurrent_disconnect_is_idempotent_and_deadlock_free(self) -> None: - server, _ = coordinator() - state = client(1) - server.clients = {state} - close_count = 0 - close_count_lock = threading.Lock() - caller_count = 16 - start = threading.Barrier(caller_count + 1) - errors: list[BaseException] = [] - - def close() -> None: - nonlocal close_count - with close_count_lock: - close_count += 1 - - def disconnect() -> None: - try: - start.wait() - server.disconnect_client(state) - except BaseException as exc: # noqa: BLE001 - surface thread failures - errors.append(exc) - - state.close = close # type: ignore[method-assign] - callers = [threading.Thread(target=disconnect) for _ in range(caller_count)] - for caller in callers: - caller.start() - start.wait() - for caller in callers: - caller.join(5) - - self.assertTrue(all(not caller.is_alive() for caller in callers)) - self.assertEqual(errors, []) - self.assertEqual(close_count, 1) - self.assertNotIn(state, server.clients) - - def test_disconnect_removes_active_and_evicted_job_contexts(self) -> None: - server, _ = coordinator() - install_fake_bundle_builder(server) - state = client(1) - active = server.build_job_for_client(state, clean_jobs=True) - evicted = server.build_job_for_client(state, clean_jobs=False) - active_id = active.job.job_id - evicted_id = evicted.job.job_id - state.active_job = active - state.active_job_ids = {active_id} - server.jobs = {active_id: active, evicted_id: evicted} - server.bury_evicted_job(state, evicted_id) - server.jobs.pop(evicted_id) - server.clients = {state} - state.close = lambda: None # type: ignore[method-assign] - - server.disconnect_client(state) - - self.assertIsNone(state.active_job) - self.assertEqual(state.active_job_ids, set()) - self.assertNotIn(active_id, server.jobs) - # Same-tip contexts survive the disconnect (detached from the dead - # connection object) so a same-username reconnect can still submit - # in-flight work; they age out on the same-tip retention TTL. - for job_id in (active_id, evicted_id): - entry = server.evicted_job_graveyard.get(job_id) - self.assertIsNotNone(entry) - assert entry is not None - self.assertIsNone(entry.client) - self.assertIn(job_id, server._disconnected_evicted_job_ids) - - def test_reconnect_storm_leaves_no_handler_threads_or_ghost_clients(self) -> None: - server, _ = coordinator() - connection_count = 32 - start = threading.Barrier(connection_count + 1) - peers: list[socket.socket] = [] - handlers: list[threading.Thread] = [] - - def handle(state: ClientState) -> None: - start.wait() - server.handle_client(state) - - for connection_id in range(1, connection_count + 1): - coordinator_socket, peer_socket = socket.socketpair() - state = client(connection_id) - state.sock = coordinator_socket - server.clients.add(state) - peers.append(peer_socket) - handler = threading.Thread( - target=handle, - args=(state,), - name=f"prism-test-handler-{connection_id}", - ) - handlers.append(handler) - handler.start() - - start.wait() - for peer in peers: - peer.close() - for handler in handlers: - handler.join(5) - - self.assertTrue(all(not handler.is_alive() for handler in handlers)) - with server.lock: - self.assertEqual(server.clients, set()) - - def test_disconnect_purges_evicted_contexts_when_retention_disabled( - self, - ) -> None: - server, _ = coordinator() - install_fake_bundle_builder(server) - server.disconnected_job_retention = 0 - state = client(1) - active = server.build_job_for_client(state, clean_jobs=True) - evicted = server.build_job_for_client(state, clean_jobs=False) - active_id = active.job.job_id - evicted_id = evicted.job.job_id - state.active_job = active - state.active_job_ids = {active_id} - server.jobs = {active_id: active, evicted_id: evicted} - server.bury_evicted_job(state, evicted_id) - server.jobs.pop(evicted_id) - server.clients = {state} - state.close = lambda: None # type: ignore[method-assign] - - server.disconnect_client(state) - - self.assertIsNone(state.active_job) - self.assertEqual(state.active_job_ids, set()) - self.assertNotIn(active_id, server.jobs) - self.assertNotIn(active_id, server.evicted_job_graveyard) - self.assertNotIn(evicted_id, server.evicted_job_graveyard) - self.assertNotIn(state.connection_id, server.evicted_jobs_by_connection) diff --git a/tests/test_prism_tip_refresh_epoch.py b/tests/test_prism_tip_refresh_epoch.py deleted file mode 100644 index d0e48b899..000000000 --- a/tests/test_prism_tip_refresh_epoch.py +++ /dev/null @@ -1,1915 +0,0 @@ -#!/usr/bin/env python3 -"""Deterministic epoch ordering and refresh-wave invariants.""" - -from __future__ import annotations - -import json -import threading -import time -import unittest -from concurrent.futures import ThreadPoolExecutor -from decimal import Decimal -from types import SimpleNamespace -from unittest.mock import patch - -from lab.prism.prism_coordinator import ( - PRISM_TIP_REFRESH_WAVE_PASS_BUDGET, - JobBuildSuperseded, - PendingInitialJob, - PrismCoordinator, - TipRefreshValidationToken, - _TipRefreshFanoutSuperseded, - _TipRefreshTrustBlocked, -) -from lab.prism.payout_state import ( - PayoutStatePublicationBlocked as _PayoutStatePublicationBlocked, -) -from tests.prism_coordinator_test_support import ( - base_template, - client, - coordinator, - install_fake_bundle_builder, -) -from tests.prism_vardiff_test_support import RecordingLedger - - -TIP_A = "11" * 32 -TIP_B = "22" * 32 -TIP_C = "33" * 32 -TIP_D = "44" * 32 - - -def validated_refresh( - server: PrismCoordinator, -) -> tuple[object, object, TipRefreshValidationToken]: - install_fake_bundle_builder(server) - snapshot = server.fetch_qbit_tip_template_snapshot() - sequence = server._reserve_tip_observation_sequence() - if not server.observe_tip_first_seen( - snapshot.bestblockhash, - observation_sequence=sequence, - publish_refresh_observation=True, - published_snapshot=snapshot, - ): - raise AssertionError("fixture refresh observation lost") - bundle = server.prepare_tip_refresh_bundle(snapshot) - token = server._validate_prepared_tip_refresh( - bundle, - snapshot, - sequence, - ) - return snapshot, bundle, token - - -def delivered_context( - *, - connection_id: int, - epoch_sequence: int, - tip_hash: str, - payout_generation: int = 0, -) -> SimpleNamespace: - return SimpleNamespace( - connection_id=connection_id, - tip_refresh_epoch_sequence=epoch_sequence, - template={"previousblockhash": tip_hash}, - payout_state_generation=payout_generation, - template_fingerprint=None, - template_generation=1, - collection_only=False, - ) - - -class TipRefreshEpochTests(unittest.TestCase): - def test_feature_fallback_keeps_epoch_ordering_disabled(self) -> None: - server, _rpc = coordinator() - server._ensure_tip_refresh_state() - state = client(1) - state.active_job = delivered_context( - connection_id=state.connection_id, - epoch_sequence=7, - tip_hash=TIP_A, - ) - - self.assertFalse(server.tip_refresh_epoch_fanout) - self.assertEqual( - server._mint_tip_refresh_epoch_locked( - tip_hash=TIP_A, - payout_state_generation=0, - started_monotonic=1.0, - ), - 0, - ) - self.assertEqual(server._tip_refresh_epoch_sequence, 0) - self.assertTrue( - server._admit_client_tip_refresh_epoch_locked(state, 1) - ) - self.assertEqual( - getattr(state, "_tip_refresh_admitted_epoch_sequence", 0), - 0, - ) - self.assertTrue(server._tip_refresh_epoch_fixpoint_reached()) - - def test_returning_tip_mints_a_new_epoch(self) -> None: - server, _rpc = coordinator() - server.tip_refresh_epoch_fanout = True - server._ensure_tip_refresh_state() - - observed = [] - for sequence, tip_hash in enumerate((TIP_A, TIP_B, TIP_A), start=1): - self.assertTrue( - server.observe_tip_for_refresh( - tip_hash, - observation_sequence=sequence, - mark_pending=False, - ) - ) - observed.append( - ( - server._tip_refresh_epoch_sequence, - server._tip_refresh_epoch_tip_hash, - ) - ) - - self.assertEqual( - observed, - [(1, TIP_A), (2, TIP_B), (3, TIP_A)], - ) - - def test_stamping_uses_published_token_not_pending_target(self) -> None: - server, _rpc = coordinator() - server.tip_refresh_epoch_fanout = True - server._ensure_tip_refresh_state() - state = client(1) - server.clients = [state] # type: ignore[assignment] - snapshot, bundle, token = validated_refresh(server) - with server.lock: - server._publish_tip_refresh_epoch_identity_locked(snapshot) - pending_epoch = server._mint_tip_refresh_epoch_locked( - tip_hash=TIP_B, - payout_state_generation=token.payout_state_generation, - started_monotonic=2.0, - ) - - context = server.stamp_job_for_client( - state, - bundle, - clean_jobs=True, - ) - - self.assertGreater(pending_epoch, token.epoch_sequence) - self.assertEqual( - context.tip_refresh_epoch_sequence, - token.epoch_sequence, - ) - - def test_returning_tip_invalidates_an_old_token(self) -> None: - server, _rpc = coordinator() - server.tip_refresh_epoch_fanout = True - server._ensure_tip_refresh_state() - snapshot, bundle, token = validated_refresh(server) - - self.assertTrue( - server.observe_tip_for_refresh( - TIP_B, - observation_sequence=token.observation_sequence + 1, - mark_pending=False, - ) - ) - self.assertTrue( - server.observe_tip_for_refresh( - TIP_A, - observation_sequence=token.observation_sequence + 2, - mark_pending=False, - ) - ) - - with server.lock: - self.assertFalse( - server._detected_tip_supersedes_locked( - token.tip_hash, - token.observation_sequence, - ) - ) - self.assertFalse( - server._tip_refresh_token_prepublication_current_locked( - token, - bundle, - snapshot, - ) - ) - - def test_admission_floor_orders_dequeued_work_across_tips(self) -> None: - server, _rpc = coordinator() - server.tip_refresh_epoch_fanout = True - server._ensure_tip_refresh_state() - state = client(1) - snapshot = SimpleNamespace( - bestblockhash=TIP_A, - previousblockhash=TIP_A, - template_generation=1, - ) - later = delivered_context( - connection_id=state.connection_id, - epoch_sequence=2, - tip_hash=TIP_B, - ) - later.template_generation = 9 - state.active_job = later - - self.assertFalse( - server.intervening_job_supersedes_snapshot( - later, - object(), # type: ignore[arg-type] - snapshot, # type: ignore[arg-type] - ) - ) - self.assertFalse( - server._admit_client_tip_refresh_epoch_locked(state, 1) - ) - - state.active_job = None - self.assertTrue( - server._admit_client_tip_refresh_epoch_locked(state, 3) - ) - self.assertFalse( - server._admit_client_tip_refresh_epoch_locked(state, 2) - ) - - def test_fence_blocks_stale_write_the_comparator_admits(self) -> None: - """Negative control: the fence is load-bearing on the wire. - - A queued delivery that already passed global validation carries an - older epoch than the connection has since advanced to. The - intervening-job comparator cannot express that ordering (an - old-tip snapshot must normally not be blocked by a newer-tip - active job), so with the fence disabled the stale write reaches - registration and the socket. Only the per-connection epoch fence - prevents the regression. - """ - server, rpc = coordinator() - server.tip_refresh_epoch_fanout = True - server._ensure_tip_refresh_state() - state = client(1) - server.clients = [state] # type: ignore[assignment] - wire: list[tuple[int, str]] = [] - - def sendall(data: bytes) -> None: - payloads = [ - json.loads(line) - for line in data.decode("utf-8").splitlines() - ] - self.assertEqual( - [payload["method"] for payload in payloads], - ["mining.set_difficulty", "mining.notify"], - ) - context = state.active_job - assert context is not None - wire.append( - ( - int(context.tip_refresh_epoch_sequence), - str(context.template["previousblockhash"]), - ) - ) - - state.sock = SimpleNamespace( - sendall=sendall, - shutdown=lambda *_args: None, - close=lambda: None, - ) - - snapshot_a, bundle_a, token_a = validated_refresh(server) - rpc.tip = TIP_B - rpc.template = base_template(height=11, prevhash=TIP_B) - snapshot_b, bundle_b, token_b = validated_refresh(server) - self.assertGreater(token_b.epoch_sequence, token_a.epoch_sequence) - - fresh = server.send_prepared_job( - state, - bundle_b, - snapshot_b, - token_b, - state.connection_id, - None, - ) - self.assertEqual(fresh.result, "sent") - current = state.active_job - assert current is not None - self.assertEqual( - int(current.tip_refresh_epoch_sequence), - token_b.epoch_sequence, - ) - - # Model the queued worker whose global validation already passed - # before the connection advanced; identical in both arms so only - # the fence distinguishes them. - server._prepared_tip_refresh_obsolete = ( # type: ignore[method-assign] - lambda *_args, **_kwargs: False - ) - server._tip_refresh_token_current_locked = ( # type: ignore[method-assign] - lambda *_args, **_kwargs: True - ) - - blocked = server.send_prepared_job( - state, - bundle_a, - snapshot_a, - token_a, - state.connection_id, - None, - ) - self.assertEqual(blocked.result, "skipped") - self.assertIs(state.active_job, current) - self.assertEqual(len(wire), 1) - - server._client_tip_refresh_epoch_blocked_locked = ( # type: ignore[method-assign] - lambda *_args, **_kwargs: False - ) - server._admit_client_tip_refresh_epoch_locked = ( # type: ignore[method-assign] - lambda *_args, **_kwargs: True - ) - - regressed = server.send_prepared_job( - state, - bundle_a, - snapshot_a, - token_a, - state.connection_id, - None, - ) - self.assertEqual(regressed.result, "sent") - active = state.active_job - assert active is not None - self.assertEqual( - str(active.template["previousblockhash"]), - TIP_A, - ) - self.assertEqual( - [epoch_sequence for epoch_sequence, _tip in wire], - [token_b.epoch_sequence, token_a.epoch_sequence], - ) - - def test_hashrate_coverage_records_fixed_thresholds_after_delivery(self) -> None: - server, _rpc = coordinator() - server.tip_refresh_epoch_fanout = True - server._ensure_tip_refresh_state() - clients = [client(index) for index in (1, 2, 3)] - for state, weight in zip( - clients, - (Decimal("60"), Decimal("36"), Decimal("4")), - ): - state.vardiff_difficulty_estimate = weight - server.clients = clients # type: ignore[assignment] - started = 10.0 - epoch = server._mint_tip_refresh_epoch_locked( - tip_hash=TIP_A, - payout_state_generation=0, - started_monotonic=started, - ) - - for state, delivered_at in zip(clients, (11.0, 12.0, 13.0)): - server._record_progress_delivery( - state, - delivered_context( - connection_id=state.connection_id, - epoch_sequence=epoch, - tip_hash=TIP_A, - ), - delivered_at, - ) - - self.assertEqual( - { - target: ( - histogram["count"], - histogram["sum"], - ) - for target, histogram in ( - server.tip_refresh_coverage_histograms.items() - ) - }, - { - "50": (1, 1.0), - "95": (1, 2.0), - "99": (1, 3.0), - }, - ) - metrics = "\n".join(server.tip_refresh_metrics_lines()) - coverage_series = [ - line - for line in metrics.splitlines() - if line.startswith( - "qbit_prism_tip_refresh_hashrate_coverage_seconds" - ) - ] - self.assertTrue(coverage_series) - self.assertTrue( - all("connection" not in line for line in coverage_series) - ) - self.assertTrue( - all("username" not in line for line in coverage_series) - ) - for target in ("50", "95", "99"): - self.assertIn( - "qbit_prism_tip_refresh_hashrate_coverage_seconds_count" - f'{{coverage="{target}"}} 1', - coverage_series, - ) - - def test_hashrate_coverage_keeps_epoch_cohort_fixed(self) -> None: - server, _rpc = coordinator() - server.tip_refresh_epoch_fanout = True - server._ensure_tip_refresh_state() - first, departed, remainder = ( - client(index) - for index in (1, 2, 3) - ) - for state, weight in ( - (first, Decimal("60")), - (departed, Decimal("36")), - (remainder, Decimal("4")), - ): - state.vardiff_difficulty_estimate = weight - server.clients = {first, departed, remainder} - epoch = server._mint_tip_refresh_epoch_locked( - tip_hash=TIP_A, - payout_state_generation=0, - started_monotonic=10.0, - ) - server._record_progress_delivery( - first, - delivered_context( - connection_id=first.connection_id, - epoch_sequence=epoch, - tip_hash=TIP_A, - ), - 11.0, - ) - with server.lock: - server.clients.remove(departed) - newcomer = client(4) - newcomer.vardiff_difficulty_estimate = Decimal("1000") - server.clients.add(newcomer) - server._record_progress_delivery( - newcomer, - delivered_context( - connection_id=newcomer.connection_id, - epoch_sequence=epoch, - tip_hash=TIP_A, - ), - 12.0, - ) - server._record_progress_delivery( - remainder, - delivered_context( - connection_id=remainder.connection_id, - epoch_sequence=epoch, - tip_hash=TIP_A, - ), - 13.0, - ) - - self.assertEqual( - { - target: histogram["count"] - for target, histogram in ( - server.tip_refresh_coverage_histograms.items() - ) - }, - {"50": 1, "95": 0, "99": 0}, - ) - - def test_fixpoint_requires_delivery_not_registration(self) -> None: - server, _rpc = coordinator() - server.tip_refresh_epoch_fanout = True - server._ensure_tip_refresh_state() - state = client(1) - server.clients = [state] # type: ignore[assignment] - snapshot = SimpleNamespace( - bestblockhash=TIP_A, - previousblockhash=TIP_A, - template_fingerprint="fixture-fingerprint", - ) - epoch = server._mint_tip_refresh_epoch_locked( - tip_hash=TIP_A, - payout_state_generation=0, - started_monotonic=1.0, - ) - context = delivered_context( - connection_id=state.connection_id, - epoch_sequence=epoch, - tip_hash=TIP_A, - ) - context.template_fingerprint = snapshot.template_fingerprint - state.active_job = context - state.active_job_ids.add("registered") - server.jobs["registered"] = context # type: ignore[assignment] - - self.assertTrue( - server.client_needs_tip_template_refresh( - state, - snapshot, # type: ignore[arg-type] - ) - ) - self.assertFalse(server._tip_refresh_epoch_fixpoint_reached()) - - state._progress_delivered_context = context - self.assertFalse( - server.client_needs_tip_template_refresh( - state, - snapshot, # type: ignore[arg-type] - ) - ) - self.assertTrue(server._tip_refresh_epoch_fixpoint_reached()) - - def test_ready_latch_reselects_delivered_collection_job(self) -> None: - """Reselection must cover every fixpoint-incomplete condition. - - A windowed registration can advance past a delivered collection - job when its send fails. The delivered collection job then keeps - the fixpoint open, so the client must stay selectable or the - owning wave spins forever with no candidate to serve. - """ - server, _rpc = coordinator() - server.tip_refresh_epoch_fanout = True - server._ensure_tip_refresh_state() - state = client(1) - server.clients = [state] # type: ignore[assignment] - snapshot = SimpleNamespace( - bestblockhash=TIP_A, - previousblockhash=TIP_A, - template_fingerprint="fixture-fingerprint", - ) - epoch = server._mint_tip_refresh_epoch_locked( - tip_hash=TIP_A, - payout_state_generation=0, - started_monotonic=1.0, - ) - delivered = delivered_context( - connection_id=state.connection_id, - epoch_sequence=epoch, - tip_hash=TIP_A, - ) - delivered.template_fingerprint = snapshot.template_fingerprint - delivered.collection_only = True - registered = delivered_context( - connection_id=state.connection_id, - epoch_sequence=epoch, - tip_hash=TIP_A, - ) - registered.template_fingerprint = snapshot.template_fingerprint - state.active_job = registered - state.active_job_ids.add("registered") - server.jobs["registered"] = registered # type: ignore[assignment] - state._progress_delivered_context = delivered - server._pool_ready_latched = True - - self.assertFalse(server._tip_refresh_epoch_fixpoint_reached()) - self.assertTrue( - server.client_needs_tip_template_refresh( - state, - snapshot, # type: ignore[arg-type] - ) - ) - - def test_wave_outcomes_are_recorded_once_per_owner(self) -> None: - cases = ( - ("completed", None), - ("fanout_superseded", _TipRefreshFanoutSuperseded("superseded")), - ("build_superseded", JobBuildSuperseded("superseded")), - ("payout_blocked", _PayoutStatePublicationBlocked("blocked")), - ("trust_blocked", _TipRefreshTrustBlocked("blocked")), - ("error", RuntimeError("failed")), - ) - - for expected, failure in cases: - feature_states = ( - (False,) - if expected in {"fanout_superseded", "build_superseded"} - else (False, True) - ) - for feature_enabled in feature_states: - with self.subTest( - expected=expected, - feature_enabled=feature_enabled, - ): - server, _rpc = coordinator() - server.tip_refresh_epoch_fanout = feature_enabled - server._ensure_tip_refresh_state() - - def pass_once(**_kwargs: object) -> int: - if failure is not None: - raise failure - return 0 - - server._poll_qbit_tip_template_pass_once = pass_once # type: ignore[method-assign] - if failure is None: - self.assertEqual( - server.poll_qbit_tip_template_once(), - 0, - ) - else: - with self.assertRaises(type(failure)): - server.poll_qbit_tip_template_once() - self.assertEqual( - server.tip_refresh_wave_outcome_counts[expected], - 1, - ) - self.assertEqual( - sum( - server.tip_refresh_wave_outcome_counts.values() - ), - 1, - ) - metrics = "\n".join( - server.tip_refresh_metrics_lines() - ) - for outcome, _failure in cases: - expected_count = 1 if outcome == expected else 0 - self.assertIn( - "qbit_prism_tip_refresh_wave_outcomes_total" - f'{{outcome="{outcome}"}} {expected_count}', - metrics, - ) - - def test_reentered_wave_keeps_one_supersession_outcome(self) -> None: - cases = ( - ( - "fanout_superseded", - _TipRefreshFanoutSuperseded("superseded"), - ), - ("build_superseded", JobBuildSuperseded("superseded")), - ) - - for expected, supersession in cases: - with self.subTest(expected=expected): - server, _rpc = coordinator() - server.tip_refresh_epoch_fanout = True - server._ensure_tip_refresh_state() - attempts = 0 - - def pass_once(**_kwargs: object) -> int: - nonlocal attempts - attempts += 1 - if attempts == 1: - raise supersession - return 0 - - server._poll_qbit_tip_template_pass_once = pass_once # type: ignore[method-assign] - server._tip_refresh_epoch_fixpoint_reached = lambda: True # type: ignore[method-assign] - - self.assertEqual(server.poll_qbit_tip_template_once(), 0) - self.assertEqual(attempts, 2) - self.assertEqual( - server.tip_refresh_wave_outcome_counts[expected], - 1, - ) - self.assertEqual( - sum(server.tip_refresh_wave_outcome_counts.values()), - 1, - ) - - def test_shutdown_before_fixpoint_records_shutdown_outcome(self) -> None: - server, _rpc = coordinator() - server.tip_refresh_epoch_fanout = True - server._ensure_tip_refresh_state() - - def pass_once(**_kwargs: object) -> int: - server.stop_event.set() - return 0 - - server._poll_qbit_tip_template_pass_once = pass_once # type: ignore[method-assign] - server._tip_refresh_epoch_fixpoint_reached = lambda: False # type: ignore[method-assign] - - self.assertEqual(server.poll_qbit_tip_template_once(), 0) - self.assertEqual( - server.tip_refresh_wave_outcome_counts["shutdown"], - 1, - ) - self.assertEqual( - sum(server.tip_refresh_wave_outcome_counts.values()), - 1, - ) - metrics = "\n".join(server.tip_refresh_metrics_lines()) - self.assertIn( - 'qbit_prism_tip_refresh_wave_outcomes_total{outcome="shutdown"} 1', - metrics, - ) - - def test_wave_pass_budget_falls_back_to_scheduled_retry(self) -> None: - server, _rpc = coordinator() - server.tip_refresh_epoch_fanout = True - server._ensure_tip_refresh_state() - passes = 0 - - def pass_once(**_kwargs: object) -> int: - nonlocal passes - passes += 1 - return 0 - - server._poll_qbit_tip_template_pass_once = pass_once # type: ignore[method-assign] - server._tip_refresh_epoch_fixpoint_reached = lambda: False # type: ignore[method-assign] - - with patch( - "lab.prism.prism_coordinator." - "PRISM_TIP_REFRESH_REENTRY_BACKOFF_SECONDS", - 0.0, - ): - self.assertEqual(server.poll_qbit_tip_template_once(), 0) - - self.assertEqual(passes, PRISM_TIP_REFRESH_WAVE_PASS_BUDGET) - self.assertEqual( - server.tip_refresh_wave_outcome_counts["fanout_superseded"], - 1, - ) - self.assertEqual( - sum(server.tip_refresh_wave_outcome_counts.values()), - 1, - ) - self.assertTrue(server._consume_tip_refresh_retry()) - - def test_shutdown_during_reentry_supersession_records_shutdown(self) -> None: - cases = ( - ("build", JobBuildSuperseded("superseded")), - ("fanout", _TipRefreshFanoutSuperseded("superseded")), - ) - for label, supersession in cases: - with self.subTest(supersession=label): - server, _rpc = coordinator() - server.tip_refresh_epoch_fanout = True - server._ensure_tip_refresh_state() - - def pass_once(**_kwargs: object) -> int: - server.stop_event.set() - raise supersession - - server._poll_qbit_tip_template_pass_once = pass_once # type: ignore[method-assign] - with self.assertRaises(type(supersession)): - server.poll_qbit_tip_template_once() - self.assertEqual( - server.tip_refresh_wave_outcome_counts["shutdown"], - 1, - ) - self.assertEqual( - sum(server.tip_refresh_wave_outcome_counts.values()), - 1, - ) - - def test_legacy_supersession_outcome_survives_shutdown(self) -> None: - server, _rpc = coordinator() - server._ensure_tip_refresh_state() - - def pass_once(**_kwargs: object) -> int: - server.stop_event.set() - raise _TipRefreshFanoutSuperseded("superseded") - - server._poll_qbit_tip_template_pass_once = pass_once # type: ignore[method-assign] - with self.assertRaises(_TipRefreshFanoutSuperseded): - server.poll_qbit_tip_template_once() - self.assertEqual( - server.tip_refresh_wave_outcome_counts["fanout_superseded"], - 1, - ) - self.assertEqual( - sum(server.tip_refresh_wave_outcome_counts.values()), - 1, - ) - - def test_converged_reentry_consumes_prewave_retry(self) -> None: - server, _rpc = coordinator() - server.tip_refresh_epoch_fanout = True - server._ensure_tip_refresh_state() - attempts = 0 - - def pass_once(**_kwargs: object) -> int: - nonlocal attempts - attempts += 1 - if attempts == 1: - server._schedule_tip_refresh_retry() - raise _TipRefreshFanoutSuperseded("superseded") - return 0 - - server._poll_qbit_tip_template_pass_once = pass_once # type: ignore[method-assign] - with patch( - "lab.prism.prism_coordinator." - "PRISM_TIP_REFRESH_REENTRY_BACKOFF_SECONDS", - 0.0, - ): - self.assertEqual(server.poll_qbit_tip_template_once(), 0) - - self.assertEqual(attempts, 2) - self.assertEqual( - server.tip_refresh_wave_outcome_counts["fanout_superseded"], - 1, - ) - self.assertFalse(server._tip_refresh_retry.is_set()) - self.assertFalse(server._consume_tip_refresh_retry()) - - def test_payout_epoch_tip_prefers_observed_refresh_target(self) -> None: - server, _rpc = coordinator() - server.tip_refresh_epoch_fanout = True - server._ensure_tip_refresh_state() - - self.assertIsNone(server._payout_epoch_tip_hash_locked()) - server.current_tip_first_seen = (TIP_C, 1.0) - self.assertEqual(server._payout_epoch_tip_hash_locked(), TIP_C) - server.latest_detected_tip = (TIP_B, 2.0) - self.assertEqual(server._payout_epoch_tip_hash_locked(), TIP_B) - server._tip_refresh_epoch_tip_hash = TIP_A - self.assertEqual(server._payout_epoch_tip_hash_locked(), TIP_A) - - def test_accepted_block_preview_keeps_epoch_on_observed_tip(self) -> None: - server, _rpc = coordinator() - server.tip_refresh_epoch_fanout = True - server._ensure_tip_refresh_state() - self.assertTrue( - server.observe_tip_for_refresh( - TIP_A, - observation_sequence=1, - mark_pending=False, - ) - ) - minted = server._tip_refresh_epoch_sequence - block_hash = "bb" * 32 - preview = [ - { - "recipient_id": "miner-a", - "order_key": "miner-a", - "p2mr_program_hex": "11" * 32, - "balance_sats": 25, - } - ] - server._begin_accepted_block_payout_preview(block_hash, block_height=10) - server._publish_accepted_block_payout_preview(block_hash, preview) - - # The accepted block hash is not a template parent; the payout - # mint must keep the epoch on the observed refresh target. - self.assertEqual(server._tip_refresh_epoch_tip_hash, TIP_A) - self.assertGreater(server._tip_refresh_epoch_sequence, minted) - self.assertEqual( - server._tip_refresh_epoch_payout_generation, - server._payout_state_generation, - ) - - def test_payout_mint_republishes_stamping_identity(self) -> None: - server, _rpc = coordinator() - server.tip_refresh_epoch_fanout = True - server._ensure_tip_refresh_state() - snapshot, bundle, token = validated_refresh(server) - with server.lock: - server._publish_tip_refresh_epoch_identity_locked(snapshot) - self.assertEqual( - server._tip_refresh_epoch_for_bundle_locked(bundle), - token.epoch_sequence, - ) - - block_hash = "bb" * 32 - preview = [ - { - "recipient_id": "miner-a", - "order_key": "miner-a", - "p2mr_program_hex": "11" * 32, - "balance_sats": 25, - } - ] - server._begin_accepted_block_payout_preview(block_hash, block_height=10) - server._publish_accepted_block_payout_preview(block_hash, preview) - - identity = server._published_tip_refresh_epoch_identity - assert identity is not None - self.assertEqual(identity[0], server._tip_refresh_epoch_sequence) - self.assertGreater(identity[0], token.epoch_sequence) - self.assertEqual(identity[2], server._payout_state_generation) - # Rebuilt bundles carry the published payout generation and must - # stamp the minted epoch, not fall to 0 behind a stale identity. - rebuilt = server.prepare_tip_refresh_bundle(snapshot) - self.assertEqual( - int(rebuilt.payout_state_generation), - int(server._payout_state_generation), - ) - self.assertEqual( - server._tip_refresh_epoch_for_bundle_locked(rebuilt), - server._tip_refresh_epoch_sequence, - ) - - def test_initial_fence_miss_completes_request_with_newer_work(self) -> None: - server, _rpc = coordinator() - install_fake_bundle_builder(server) - server.tip_refresh_epoch_fanout = True - server._ensure_tip_refresh_state() - server._ensure_initial_job_state() - bundle = server.prewarm_current_tip_ready_bundle() - assert bundle is not None - artifacts = server.current_template_artifacts() - state = client(1) - state.authorization_generation = 1 - state.difficulty_generation = 0 - state.authorized_monotonic = time.monotonic() - sent_payloads: list[dict[str, object]] = [] - state.send = sent_payloads.append # type: ignore[method-assign] - server.clients = {state} - snapshot = server.tip_template_snapshot - assert snapshot is not None - assert snapshot.template_artifacts is not None - newer = SimpleNamespace( - connection_id=state.connection_id, - tip_refresh_epoch_sequence=99, - template=snapshot.template_artifacts.template, - template_fingerprint=snapshot.template_fingerprint, - template_generation=snapshot.template_generation, - payout_state_generation=int( - getattr(server, "_payout_state_generation", 0) - ), - authorization_generation=1, - difficulty_generation=0, - collection_only=False, - job=SimpleNamespace(job_id="newer"), - ) - state.active_job = newer - state._tip_refresh_admitted_epoch_sequence = 99 - request = PendingInitialJob( - client=state, - authorization_generation=1, - worker=state.worker, - requested_monotonic=time.monotonic(), - deadline_monotonic=None, - connection_id=state.connection_id, - difficulty_generation=0, - ) - server.pending_initial_jobs = {state: request} - - self.assertIs( - server._deliver_initial_bundle(request, artifacts, bundle), - False, - ) - self.assertNotIn(state, server.pending_initial_jobs) - self.assertTrue(request.cancelled.is_set()) - self.assertEqual(server.initial_job_sent_count, 1) - self.assertIs(state.active_job, newer) - self.assertEqual(sent_payloads, []) - - def test_wave_reentry_gates_on_feature_shutdown_and_budget(self) -> None: - server, _rpc = coordinator() - server._ensure_tip_refresh_state() - - self.assertFalse(server._tip_refresh_wave_reenters(1)) - - server.tip_refresh_epoch_fanout = True - with patch( - "lab.prism.prism_coordinator." - "PRISM_TIP_REFRESH_REENTRY_BACKOFF_SECONDS", - 0.0, - ): - self.assertTrue(server._tip_refresh_wave_reenters(1)) - self.assertFalse(server._consume_tip_refresh_retry()) - self.assertFalse( - server._tip_refresh_wave_reenters( - PRISM_TIP_REFRESH_WAVE_PASS_BUDGET - ) - ) - self.assertTrue(server._consume_tip_refresh_retry()) - - server.stop_event.set() - self.assertFalse(server._tip_refresh_wave_reenters(1)) - - def test_reconciliation_failure_records_trust_blocked(self) -> None: - server, _rpc = coordinator() - install_fake_bundle_builder(server) - server.tip_refresh_epoch_fanout = True - server._ensure_tip_refresh_state() - - def fail_reconciliation(_tip_hash: str) -> bool: - raise RuntimeError("synthetic reconciliation failure") - - server.ensure_reorg_reconciled_for_tip = fail_reconciliation # type: ignore[method-assign] - - with self.assertRaises(_TipRefreshTrustBlocked): - server.poll_qbit_tip_template_once() - - self.assertEqual( - server.tip_refresh_wave_outcome_counts["trust_blocked"], - 1, - ) - self.assertEqual( - sum(server.tip_refresh_wave_outcome_counts.values()), - 1, - ) - - def _assert_feature_wave_converges_after_supersession( - self, - transitions: tuple[tuple[str, str, int], ...], - ) -> None: - server, rpc = coordinator() - install_fake_bundle_builder(server) - server.tip_refresh_epoch_fanout = True - server.tip_refresh_max_workers = 16 - server._ensure_tip_refresh_state() - clients = [client(index + 1) for index in range(200)] - for state in clients: - state.vardiff_difficulty_estimate = Decimal( - len(clients) + 1 - state.connection_id - ) - server.clients = clients # type: ignore[assignment] - superseded_tips = tuple( - old_tip - for old_tip, _new_tip, _height in transitions - ) - final_tip = transitions[-1][1] - tip_order = { - tip_hash: rank - for rank, tip_hash in enumerate( - ( - transitions[0][0], - *( - new_tip - for _old_tip, new_tip, _height in transitions - ), - ) - ) - } - release_sends = { - tip_hash: threading.Event() - for tip_hash in superseded_tips - } - sends_admitted = { - tip_hash: threading.Event() - for tip_hash in superseded_tips - } - send_lock = threading.Lock() - notify_starts: list[tuple[str, int, int]] = [] - wire_batches: dict[int, list[tuple[int, bytes]]] = { - state.connection_id: [] - for state in clients - } - active_sends = 0 - active_sends_by_tip = { - tip_hash: 0 - for tip_hash in superseded_tips - } - maximum_active_sends = 0 - - def client_sendall(state: object) -> object: - def sendall(data: bytes) -> None: - nonlocal active_sends, maximum_active_sends - payloads = [ - json.loads(line) - for line in data.decode("utf-8").splitlines() - ] - self.assertEqual( - [payload["method"] for payload in payloads], - ["mining.set_difficulty", "mining.notify"], - ) - context = state.active_job # type: ignore[attr-defined] - if context is None: - raise AssertionError("notify started without registered work") - self.assertEqual( - payloads, - [ - server.difficulty_payload( - context.job.share_difficulty - ), - server.job_payload(context.job), - ], - ) - tip_hash = str(context.template["previousblockhash"]) - epoch_sequence = int(context.tip_refresh_epoch_sequence) - with send_lock: - active_sends += 1 - if tip_hash in active_sends_by_tip: - active_sends_by_tip[tip_hash] += 1 - maximum_active_sends = max( - maximum_active_sends, - active_sends, - ) - notify_starts.append( - ( - tip_hash, - epoch_sequence, - int(state.connection_id), # type: ignore[attr-defined] - ) - ) - wire_batches[int(state.connection_id)].append( # type: ignore[attr-defined] - (epoch_sequence, data) - ) - if ( - tip_hash in active_sends_by_tip - and active_sends_by_tip[tip_hash] - == server.tip_refresh_max_workers - ): - sends_admitted[tip_hash].set() - try: - if ( - tip_hash in release_sends - and not release_sends[tip_hash].wait(10) - ): - raise AssertionError("test did not release admitted sends") - finally: - with send_lock: - active_sends -= 1 - if tip_hash in active_sends_by_tip: - active_sends_by_tip[tip_hash] -= 1 - - return sendall - - for state in clients: - state.sock = SimpleNamespace( - sendall=client_sendall(state), - shutdown=lambda *_args: None, - close=lambda: None, - ) - - results: list[int] = [] - errors: list[BaseException] = [] - - def poll() -> None: - try: - results.append(server.poll_qbit_tip_template_once()) - except BaseException as exc: - errors.append(exc) - - owner = threading.Thread(target=poll) - owner.start() - try: - for old_tip, new_tip, height in transitions: - self.assertTrue(sends_admitted[old_tip].wait(5)) - rpc.tip = new_tip - rpc.template = base_template( - height=height, - prevhash=new_tip, - ) - self.assertTrue(server.observe_tip_for_refresh(new_tip)) - with send_lock: - admitted_work = [ - (epoch_sequence, connection_id) - for ( - tip_hash, - epoch_sequence, - connection_id, - ) in notify_starts - if tip_hash == old_tip - ] - self.assertEqual( - len(admitted_work), - server.tip_refresh_max_workers, - ) - self.assertEqual( - { - connection_id - for _epoch_sequence, connection_id in admitted_work - }, - set( - range( - 1, - server.tip_refresh_max_workers + 1, - ) - ), - ) - clients_by_connection = { - state.connection_id: state - for state in clients - } - for epoch_sequence, connection_id in admitted_work: - active = clients_by_connection[ - connection_id - ].active_job - self.assertIsNotNone(active) - assert active is not None - self.assertEqual( - str(active.template["previousblockhash"]), - old_tip, - ) - self.assertEqual( - active.tip_refresh_epoch_sequence, - epoch_sequence, - ) - release_sends[old_tip].set() - owner.join(20) - - self.assertFalse(owner.is_alive()) - self.assertEqual(errors, []) - self.assertEqual(results, [len(clients)]) - final_epoch = server._tip_refresh_epoch_sequence - self.assertEqual(server.current_tip_first_seen[0], final_tip) - self.assertEqual(server._tip_refresh_epoch_tip_hash, final_tip) - self.assertTrue(server._tip_refresh_epoch_fixpoint_reached()) - self.assertFalse(server.tip_refresh_is_pending()) - for state in clients: - active = state.active_job - delivered = getattr(state, "_progress_delivered_context", None) - self.assertIsNotNone(active) - self.assertIsNotNone(delivered) - assert active is not None and delivered is not None - self.assertEqual( - str(active.template["previousblockhash"]), - final_tip, - ) - self.assertEqual( - str(delivered.template["previousblockhash"]), - final_tip, - ) - self.assertEqual( - active.tip_refresh_epoch_sequence, - final_epoch, - ) - self.assertEqual( - delivered.tip_refresh_epoch_sequence, - final_epoch, - ) - - with send_lock: - start_ranks = [ - tip_order[tip_hash] - for tip_hash, _epoch, _connection_id in notify_starts - ] - self.assertEqual(start_ranks, sorted(start_ranks)) - self.assertEqual(active_sends, 0) - self.assertEqual( - maximum_active_sends, - server.tip_refresh_max_workers, - ) - for state in clients: - batches = wire_batches[state.connection_id] - self.assertGreaterEqual(len(batches), 1) - epochs = [ - epoch_sequence - for epoch_sequence, _data in batches - ] - self.assertEqual(epochs, sorted(epochs)) - for _epoch_sequence, data in batches: - methods = [ - json.loads(line)["method"] - for line in data.decode("utf-8").splitlines() - ] - self.assertEqual( - methods, - ["mining.set_difficulty", "mining.notify"], - ) - self.assertEqual(server.tip_refresh_inflight, 0) - self.assertEqual(server.tip_refresh_build_inflight, 0) - self.assertEqual(server.tip_refresh_build_queue_depth, 0) - self.assertEqual(server._tip_refresh_executor.stats(), (0, 0)) - self.assertIsNone(server._active_tip_refresh) - self.assertEqual( - server.tip_refresh_wave_outcome_counts["fanout_superseded"], - 1, - ) - self.assertEqual( - sum(server.tip_refresh_wave_outcome_counts.values()), - 1, - ) - finally: - for release in release_sends.values(): - release.set() - owner.join(5) - server.shutdown_tip_refresh_executor() - - def test_feature_wave_converges_after_repeated_supersession(self) -> None: - cases = ( - ( - (TIP_A, TIP_B, 11), - (TIP_B, TIP_C, 12), - ), - ( - (TIP_A, TIP_B, 11), - (TIP_B, TIP_C, 12), - (TIP_C, TIP_D, 13), - ), - ) - for transitions in cases: - with self.subTest(supersessions=len(transitions)): - self._assert_feature_wave_converges_after_supersession( - transitions - ) - - def test_build_checkpoint_supersession_reenters_latest_epoch(self) -> None: - checkpoints = ( - "ledger_snapshot_complete", - "payout_derivation", - "ctv_manifest", - "signing_verification", - "bundle_assembly", - "serialization", - "bundle_publication", - ) - - for index, blocked_checkpoint in enumerate(checkpoints): - with self.subTest(checkpoint=blocked_checkpoint): - server, rpc = coordinator() - install_fake_bundle_builder(server) - server.tip_refresh_epoch_fanout = True - server._ensure_tip_refresh_state() - state = client(index + 1) - notifications: list[tuple[str, int]] = [] - - def send(payload: dict[str, object]) -> None: - if payload["method"] != "mining.notify": - return - context = state.active_job - if context is None: - raise AssertionError( - "notify started without registered work" - ) - notifications.append( - ( - str(context.template["previousblockhash"]), - int(context.tip_refresh_epoch_sequence), - ) - ) - - state.send = send # type: ignore[method-assign] - server.clients = [state] # type: ignore[assignment] - checkpoint_entered = threading.Event() - release_checkpoint = threading.Event() - checkpoint_hits = 0 - original_checkpoint = server._job_build_checkpoint - - def controlled_checkpoint( - phase: str, - cancellation: object, - ) -> None: - nonlocal checkpoint_hits - if phase == blocked_checkpoint: - checkpoint_hits += 1 - if checkpoint_hits == 1: - checkpoint_entered.set() - if not release_checkpoint.wait(10): - raise AssertionError( - "test did not release build checkpoint" - ) - original_checkpoint(phase, cancellation) # type: ignore[arg-type] - - server._job_build_checkpoint = controlled_checkpoint # type: ignore[method-assign] - next_tip = f"{64 + index:02x}" * 32 - results: list[int] = [] - errors: list[BaseException] = [] - - def poll() -> None: - try: - results.append( - server.poll_qbit_tip_template_once() - ) - except BaseException as exc: - errors.append(exc) - - owner = threading.Thread(target=poll) - owner.start() - try: - self.assertTrue(checkpoint_entered.wait(5)) - rpc.tip = next_tip - rpc.template = base_template( - height=20 + index, - prevhash=next_tip, - ) - self.assertTrue( - server.observe_tip_for_refresh(next_tip) - ) - release_checkpoint.set() - owner.join(20) - - self.assertFalse(owner.is_alive()) - self.assertEqual(errors, []) - self.assertEqual(results, [1]) - self.assertEqual(checkpoint_hits, 2) - final_epoch = server._tip_refresh_epoch_sequence - self.assertEqual( - notifications, - [(next_tip, final_epoch)], - ) - self.assertEqual( - server.current_tip_first_seen[0], - next_tip, - ) - self.assertIsNotNone(state.active_job) - delivered = getattr( - state, - "_progress_delivered_context", - None, - ) - self.assertIsNotNone(delivered) - assert state.active_job is not None and delivered is not None - self.assertEqual( - state.active_job.tip_refresh_epoch_sequence, - final_epoch, - ) - self.assertEqual( - delivered.tip_refresh_epoch_sequence, - final_epoch, - ) - self.assertTrue( - server._tip_refresh_epoch_fixpoint_reached() - ) - self.assertEqual(server.tip_refresh_inflight, 0) - self.assertEqual(server.tip_refresh_build_inflight, 0) - self.assertEqual( - server.tip_refresh_build_queue_depth, - 0, - ) - self.assertEqual( - server._tip_refresh_executor.stats(), - (0, 0), - ) - with server._job_build_scheduler_lock: - self.assertIsNone(server._job_build_active) - self.assertIsNone(server._job_build_retiring) - self.assertEqual( - server.tip_refresh_wave_outcome_counts[ - "build_superseded" - ], - 1, - ) - self.assertEqual( - sum( - server.tip_refresh_wave_outcome_counts.values() - ), - 1, - ) - finally: - release_checkpoint.set() - owner.join(5) - server.shutdown_tip_refresh_executor() - - def test_same_tip_payout_publications_recover_newest_epoch(self) -> None: - server, rpc = coordinator() - install_fake_bundle_builder(server) - server.tip_refresh_epoch_fanout = True - server._ensure_tip_refresh_state() - clients = [client(index + 1) for index in range(4)] - server.clients = clients # type: ignore[assignment] - initial_sends_started = threading.Event() - release_initial_sends = threading.Event() - mutation_started = threading.Event() - mutation_completed = threading.Event() - send_lock = threading.Lock() - sends_inflight = 0 - delivery_history: dict[int, list[tuple[int, int]]] = { - state.connection_id: [] - for state in clients - } - - def client_send(state: object) -> object: - def send(payload: dict[str, object]) -> None: - nonlocal sends_inflight - if payload["method"] != "mining.notify": - return - context = state.active_job # type: ignore[attr-defined] - if context is None: - raise AssertionError("notify started without registered work") - generation = int(context.payout_state_generation) - epoch_sequence = int(context.tip_refresh_epoch_sequence) - with send_lock: - sends_inflight += 1 - delivery_history[int(state.connection_id)].append( # type: ignore[attr-defined] - (generation, epoch_sequence) - ) - if generation == 0 and sends_inflight == len(clients): - initial_sends_started.set() - try: - if ( - generation == 0 - and not release_initial_sends.wait(10) - ): - raise AssertionError("test did not release initial sends") - finally: - with send_lock: - sends_inflight -= 1 - - return send - - for state in clients: - state.send = client_send(state) # type: ignore[method-assign,assignment] - - poll_results: list[int] = [] - poll_errors: list[BaseException] = [] - published_generations: list[int] = [] - published_epochs: list[int] = [] - - def poll() -> None: - try: - poll_results.append(server.poll_qbit_tip_template_once()) - except BaseException as exc: - poll_errors.append(exc) - - def mutate_payouts() -> None: - mutation_started.set() - try: - for _ in range(3): - generation = server._advance_payout_state_generation() - published_generations.append(generation) - published_epochs.append( - server._tip_refresh_epoch_sequence - ) - finally: - mutation_completed.set() - - owner = threading.Thread(target=poll) - mutator = threading.Thread(target=mutate_payouts) - owner.start() - try: - self.assertTrue(initial_sends_started.wait(5)) - first_epoch = server._tip_refresh_epoch_sequence - mutator.start() - self.assertTrue(mutation_started.wait(5)) - self.assertFalse(mutation_completed.wait(0.1)) - release_initial_sends.set() - mutator.join(20) - owner.join(20) - - self.assertFalse(mutator.is_alive()) - self.assertFalse(owner.is_alive()) - self.assertEqual(poll_errors, []) - self.assertEqual(len(poll_results), 1) - self.assertEqual(published_generations, [1, 2, 3]) - self.assertEqual( - published_epochs, - list( - range( - published_epochs[0], - published_epochs[0] + len(published_epochs), - ) - ), - ) - newest_payout_generation = published_generations[-1] - newest_epoch = server._tip_refresh_epoch_sequence - self.assertGreater(newest_epoch, first_epoch) - self.assertEqual(server._tip_refresh_epoch_tip_hash, rpc.tip) - self.assertTrue(server._tip_refresh_epoch_fixpoint_reached()) - for state in clients: - active = state.active_job - delivered = getattr( - state, - "_progress_delivered_context", - None, - ) - self.assertIsNotNone(active) - self.assertIsNotNone(delivered) - assert active is not None and delivered is not None - self.assertEqual( - active.payout_state_generation, - newest_payout_generation, - ) - self.assertEqual( - delivered.payout_state_generation, - newest_payout_generation, - ) - self.assertEqual( - active.tip_refresh_epoch_sequence, - newest_epoch, - ) - self.assertEqual( - delivered.tip_refresh_epoch_sequence, - newest_epoch, - ) - with send_lock: - client_deliveries = list( - delivery_history[state.connection_id] - ) - self.assertGreaterEqual(len(client_deliveries), 2) - self.assertEqual( - [generation for generation, _epoch in client_deliveries], - sorted( - generation - for generation, _epoch in client_deliveries - ), - ) - self.assertEqual( - [epoch for _generation, epoch in client_deliveries], - sorted( - epoch - for _generation, epoch in client_deliveries - ), - ) - self.assertEqual( - client_deliveries[-1], - (newest_payout_generation, newest_epoch), - ) - with send_lock: - self.assertEqual(sends_inflight, 0) - self.assertEqual(server.tip_refresh_inflight, 0) - self.assertEqual(server.tip_refresh_build_inflight, 0) - self.assertEqual(server.tip_refresh_build_queue_depth, 0) - self.assertEqual(server._tip_refresh_executor.stats(), (0, 0)) - self.assertEqual( - server.tip_refresh_wave_outcome_counts[ - "fanout_superseded" - ], - 1, - ) - self.assertEqual( - sum(server.tip_refresh_wave_outcome_counts.values()), - 1, - ) - finally: - release_initial_sends.set() - if mutator.ident is not None: - mutator.join(5) - owner.join(5) - server.shutdown_tip_refresh_executor() - - def test_broken_pipe_retires_only_failed_client(self) -> None: - server, _rpc = coordinator() - install_fake_bundle_builder(server) - server.tip_refresh_epoch_fanout = True - server._ensure_tip_refresh_state() - failed = client(1) - healthy = [client(2), client(3)] - healthy_notifications: dict[int, list[dict[str, object]]] = { - state.connection_id: [] - for state in healthy - } - - def fail_notify(payload: dict[str, object]) -> None: - if payload["method"] == "mining.notify": - raise BrokenPipeError("synthetic closed connection") - - failed.send = fail_notify # type: ignore[method-assign] - failed.close = lambda: None # type: ignore[method-assign] - for state in healthy: - state.send = healthy_notifications[state.connection_id].append # type: ignore[method-assign] - server.clients = {failed, *healthy} - - try: - self.assertEqual( - server.poll_qbit_tip_template_once(), - len(healthy), - ) - finally: - server.shutdown_tip_refresh_executor() - - self.assertNotIn(failed, server.clients) - self.assertTrue(failed.closing) - self.assertIsNone(failed.active_job) - self.assertEqual(failed.active_job_ids, set()) - self.assertIsNone(failed.worker) - final_epoch = server._tip_refresh_epoch_sequence - for state in healthy: - self.assertIn(state, server.clients) - self.assertIsNotNone(state.active_job) - assert state.active_job is not None - self.assertEqual( - state.active_job.tip_refresh_epoch_sequence, - final_epoch, - ) - self.assertEqual( - [ - payload["method"] - for payload in healthy_notifications[ - state.connection_id - ] - ], - ["mining.set_difficulty", "mining.notify"], - ) - self.assertTrue(server._tip_refresh_epoch_fixpoint_reached()) - self.assertEqual( - server.tip_refresh_client_counts["disconnected"], - 1, - ) - self.assertEqual( - server.tip_refresh_wave_outcome_counts["completed"], - 1, - ) - self.assertEqual( - sum(server.tip_refresh_wave_outcome_counts.values()), - 1, - ) - - def test_retained_same_tip_submits_survive_inflight_epoch_swap(self) -> None: - class ShareLedger(RecordingLedger): - def accepted_share_stats(self) -> dict[str, int]: - return { - "accepted_share_count": self.shares, - "distinct_miner_count": 0, - } - - def snapshot_at_job_issue( - self, - _anchor_job_issued_at_ms: int, - *, - window_weight: int | None = None, - ) -> list[object]: - del window_weight - return [] - - ledger = ShareLedger() - server, _rpc = coordinator(ledger=ledger) - server.vardiff_config = SimpleNamespace(enabled=False) - install_fake_bundle_builder(server) - server.tip_refresh_epoch_fanout = True - server._ensure_tip_refresh_state() - state = client(1) - state.send = lambda _payload: None # type: ignore[method-assign] - server.clients = {state} - self.assertEqual(server.poll_qbit_tip_template_once(), 1) - assert state.active_job is not None - retained_job_id = state.active_job.job.job_id - server.bury_evicted_job(state, retained_job_id) - server.jobs.pop(retained_job_id) - state.active_job_ids.clear() - - def submit(index: int) -> None: - submission = SimpleNamespace( - header_hex=f"{index:02x}" * 80, - block_hash_hex=f"{index + 16:02x}" * 32, - share_pass=True, - block_pass=False, - ) - with patch( - "lab.prism.prism_coordinator.direct_stratum.assemble_submission", - return_value=submission, - ): - self.assertFalse( - server.handle_submit( - state, - [ - state.username, - retained_job_id, - "00" * 8, - f"{index:08x}", - f"{index + 32:08x}", - ], - ) - ) - - submit(1) - server._advance_payout_state_generation() - send_started = threading.Event() - release_send = threading.Event() - poll_results: list[int] = [] - poll_errors: list[BaseException] = [] - - def block_notify(payload: dict[str, object]) -> None: - if payload["method"] == "mining.notify": - send_started.set() - if not release_send.wait(10): - raise AssertionError("test did not release epoch delivery") - - def poll() -> None: - try: - poll_results.append(server.poll_qbit_tip_template_once()) - except BaseException as exc: - poll_errors.append(exc) - - state.send = block_notify # type: ignore[method-assign] - owner = threading.Thread(target=poll) - owner.start() - try: - self.assertTrue(send_started.wait(5)) - submit(2) - release_send.set() - owner.join(10) - self.assertFalse(owner.is_alive()) - self.assertEqual(poll_errors, []) - self.assertEqual(poll_results, [1]) - submit(3) - finally: - release_send.set() - owner.join(5) - server.shutdown_tip_refresh_executor() - - self.assertEqual(len(ledger.pending), 3) - self.assertTrue( - all(share.credit_policy is None for share in ledger.pending) - ) - self.assertEqual( - server.evicted_job_submit_counts["accepted_same_tip"], - 3, - ) - self.assertIn(retained_job_id, server.evicted_job_graveyard) - self.assertTrue(server._tip_refresh_epoch_fixpoint_reached()) - - def test_shutdown_drains_admitted_send_and_drops_cleaned_queue(self) -> None: - server, _rpc = coordinator() - install_fake_bundle_builder(server) - server.tip_refresh_epoch_fanout = True - server.tip_refresh_max_workers = 2 - server._ensure_tip_refresh_state() - admitted, queued, untouched = ( - client(index) - for index in (1, 2, 3) - ) - admitted.share_difficulty = Decimal("3") - queued.share_difficulty = Decimal("2") - untouched.share_difficulty = Decimal("1") - server.clients = {admitted, queued, untouched} - admitted_send_started = threading.Event() - admitted_send_finished = threading.Event() - release_admitted_send = threading.Event() - queued_submitted = threading.Event() - queued_socket_called = threading.Event() - untouched_socket_called = threading.Event() - - def block_admitted_send(payload: dict[str, object]) -> None: - if payload["method"] != "mining.notify": - return - admitted_send_started.set() - try: - if not release_admitted_send.wait(10): - raise AssertionError("test did not release admitted send") - finally: - admitted_send_finished.set() - - admitted.send = block_admitted_send # type: ignore[method-assign] - queued.send = lambda _payload: queued_socket_called.set() # type: ignore[method-assign] - queued.close = lambda: None # type: ignore[method-assign] - untouched.send = lambda _payload: untouched_socket_called.set() # type: ignore[method-assign] - - class ObservedExecutor: - def __init__(self) -> None: - self.delegate = ThreadPoolExecutor(max_workers=1) - self.submission_count = 0 - - def submit(self, function: object, *args: object) -> object: - self.submission_count += 1 - future = self.delegate.submit(function, *args) # type: ignore[arg-type] - if self.submission_count == 2: - queued_submitted.set() - return future - - def shutdown(self, **kwargs: object) -> None: - self.delegate.shutdown(**kwargs) # type: ignore[arg-type] - - executor = ObservedExecutor() - server._tip_refresh_executor = executor # type: ignore[assignment] - poll_results: list[int] = [] - poll_errors: list[BaseException] = [] - shutdown_complete = threading.Event() - - def poll() -> None: - try: - poll_results.append(server.poll_qbit_tip_template_once()) - except BaseException as exc: - poll_errors.append(exc) - - def shutdown() -> None: - server.shutdown_tip_refresh_executor() - shutdown_complete.set() - - owner = threading.Thread(target=poll) - shutdown_thread = threading.Thread(target=shutdown) - owner.start() - try: - self.assertTrue(admitted_send_started.wait(5)) - self.assertTrue(queued_submitted.wait(5)) - server.disconnect_client(queued) - server.stop_event.set() - shutdown_thread.start() - self.assertFalse(shutdown_complete.wait(0.1)) - self.assertFalse(admitted_send_finished.is_set()) - self.assertFalse(queued_socket_called.is_set()) - self.assertFalse(untouched_socket_called.is_set()) - finally: - release_admitted_send.set() - if shutdown_thread.ident is not None: - shutdown_thread.join(10) - owner.join(10) - - self.assertFalse(shutdown_thread.is_alive()) - self.assertFalse(owner.is_alive()) - self.assertTrue(shutdown_complete.is_set()) - self.assertTrue(admitted_send_finished.is_set()) - self.assertEqual(poll_errors, []) - self.assertEqual(poll_results, [1]) - self.assertFalse(queued_socket_called.is_set()) - self.assertFalse(untouched_socket_called.is_set()) - self.assertNotIn(queued, server.clients) - self.assertTrue(queued.closing) - self.assertIsNone(queued.active_job) - self.assertEqual(queued.active_job_ids, set()) - self.assertEqual(server.tip_refresh_inflight, 0) - self.assertIsNone(server._active_tip_refresh) - # Shutdown interrupted convergence before the untouched client got - # newest delivered work; the wave must not report completed. - self.assertEqual( - server.tip_refresh_wave_outcome_counts["shutdown"], - 1, - ) - self.assertEqual( - sum(server.tip_refresh_wave_outcome_counts.values()), - 1, - ) - with self.assertRaisesRegex( - RuntimeError, - "tip refresh executor is shut down", - ): - server.tip_refresh_executor() - - def test_coalesced_observer_does_not_record_an_owner_outcome(self) -> None: - server, _rpc = coordinator() - server._ensure_tip_refresh_state() - - def unexpected_pass(**_kwargs: object) -> int: - raise AssertionError("coalesced observer entered owner pass") - - server._poll_qbit_tip_template_pass_once = unexpected_pass # type: ignore[method-assign] - server._tip_refresh_singleflight_lock.acquire() - try: - self.assertEqual(server.poll_qbit_tip_template_once(), 0) - finally: - server._tip_refresh_singleflight_lock.release() - - self.assertEqual( - sum(server.tip_refresh_wave_outcome_counts.values()), - 0, - ) - - -if __name__ == "__main__": - unittest.main() diff --git a/tests/test_prism_tip_refresh_validation.py b/tests/test_prism_tip_refresh_validation.py deleted file mode 100644 index 074b7f792..000000000 --- a/tests/test_prism_tip_refresh_validation.py +++ /dev/null @@ -1,3405 +0,0 @@ -#!/usr/bin/env python3 -"""Once-per-refresh chain validation and prepared-fanout cancellation tests.""" - -from __future__ import annotations - -import threading -import unittest -from concurrent.futures import Future, ThreadPoolExecutor -from contextlib import contextmanager -from dataclasses import FrozenInstanceError -from unittest.mock import patch - -from lab.prism.prism_coordinator import ( - ShutdownInProgress, - TemplateRefreshBlocked, - TemplateRefreshSuperseded, - TipRefreshValidationToken, - _FanoutCancellation, -) -from lab.prism.payout_state import ( - PayoutStateDeliveryGate as _PayoutStateDeliveryGate, -) -from tests.prism_coordinator_test_support import ( - FakeLedger, - FakeRpc, - base_template, - client, - coordinator, - install_fake_bundle_builder, -) - - -class _ControlledTipRefreshLock: - """Legacy refresh-lock fixture retained for non-discovered test helpers.""" - - def __init__(self) -> None: - self._lock = threading.Lock() - self._condition = threading.Condition() - self._controlled_threads: set[int] = set() - self._probe_gates: list[threading.Event] = [] - self.allow_waiters_to_acquire = threading.Event() - - def acquire(self, timeout: float = -1) -> bool: - if self._lock.acquire(blocking=False): - return True - thread_id = threading.get_ident() - with self._condition: - if thread_id not in self._controlled_threads: - self._controlled_threads.add(thread_id) - probe_gate = threading.Event() - self._probe_gates.append(probe_gate) - self._condition.notify_all() - else: - probe_gate = None - if probe_gate is not None: - if not probe_gate.wait(5): - raise AssertionError("test did not release contention probe") - return False - if not self.allow_waiters_to_acquire.wait(5): - raise AssertionError("test did not release refresh lock waiters") - return self._lock.acquire(timeout=timeout) - - def release(self) -> None: - self._lock.release() - - def next_probe_gate(self) -> threading.Event: - with self._condition: - if not self._condition.wait_for(lambda: bool(self._probe_gates), timeout=5): - raise AssertionError("poll did not contend for the refresh lock") - return self._probe_gates.pop(0) - - -def _advance_fake_tip(rpc: FakeRpc, tip_hash: str, height: int) -> None: - rpc.tip = tip_hash - rpc.template = base_template(height=height, prevhash=tip_hash) - - -class ObservedRLock: - def __init__(self) -> None: - self.lock = threading.RLock() - self.acquire_attempted = threading.Event() - - def acquire(self, *args: object, **kwargs: object) -> bool: - self.acquire_attempted.set() - return self.lock.acquire(*args, **kwargs) # type: ignore[arg-type] - - def release(self) -> None: - self.lock.release() - - def __enter__(self) -> ObservedRLock: - self.acquire() - return self - - def __exit__(self, *_args: object) -> None: - self.release() - - -class ObservedPayoutGate: - def __init__(self, delegate: object) -> None: - self.delegate = delegate - self.delivery_wait_started = threading.Event() - - def delivery(self) -> object: - self.delivery_wait_started.set() - return self.delegate.delivery() # type: ignore[attr-defined,no-any-return] - - @contextmanager - def delivery_cancelable( - self, - cancelled: object, - **kwargs: object, - ) -> object: - self.delivery_wait_started.set() - with self.delegate.delivery_cancelable(cancelled, **kwargs) as admitted: # type: ignore[attr-defined] - yield admitted - - def mutation(self) -> object: - return self.delegate.mutation() # type: ignore[attr-defined,no-any-return] - - -class TipRefreshValidationTests(unittest.TestCase): - def advance_tip( - self, - server: object, - rpc: object, - tip_hash: str, - *, - height: int, - ) -> None: - rpc.tip = tip_hash # type: ignore[attr-defined] - rpc.template = base_template(height=height, prevhash=tip_hash) # type: ignore[attr-defined] - self.assertTrue(server.observe_tip_for_refresh(tip_hash)) # type: ignore[attr-defined] - - def test_collection_refresh_reconciles_once_for_multiple_clients(self) -> None: - server, rpc = coordinator(ledger=FakeLedger(miners=["miner-a"])) - install_fake_bundle_builder(server) - clients = [client(index + 1) for index in range(5)] - notifications: set[int] = set() - for state in clients: - state.send = ( # type: ignore[method-assign] - lambda payload, connection_id=state.connection_id: ( - notifications.add(connection_id) - if payload["method"] == "mining.notify" - else None - ) - ) - server.clients = clients # type: ignore[assignment] - reconciliation_calls: list[str] = [] - server.ensure_reorg_reconciled_for_tip = ( # type: ignore[method-assign] - lambda tip_hash: reconciliation_calls.append(tip_hash) or True - ) - server.ensure_reorg_reconciled_for_current_tip = ( # type: ignore[method-assign] - lambda **_kwargs: self.fail("collection fanout repeated chain validation") - ) - - refreshed = server.poll_qbit_tip_template_once() - - self.assertEqual(refreshed, len(clients)) - self.assertEqual(reconciliation_calls, [rpc.tip]) - self.assertEqual(notifications, {state.connection_id for state in clients}) - self.assertTrue(all(state.active_job is not None for state in clients)) - self.assertTrue(all(state.active_job.collection_only for state in clients)) - self.assertLessEqual(rpc.count("getbestblockhash"), 3) - - def test_external_tip_preparation_does_not_hold_delivery_gate(self) -> None: - entered = threading.Event() - release = threading.Event() - - class BlockingExternalTipLedger(FakeLedger): - def reorg_watch_blocks( - self, - *, - active_tip_height: int, - ) -> list[dict[str, object]]: - return [ - { - "block_height": active_tip_height + 1, - "block_hash": "aa" * 32, - "chain_state": "confirmed", - } - ] - - def mark_pool_block_inactive( - self, - *, - block_hash: str, - active_tip_height: int, - ) -> dict[str, object]: - self.assert_not_atomic() - entered.set() - if not release.wait(5): - raise AssertionError("test did not release external-tip preparation") - return {"inactive_count": 1} - - def mark_mature_pool_payouts( - self, - *, - active_tip_height: int, - ) -> dict[str, object]: - return {"matured_count": 0} - - ledger = BlockingExternalTipLedger() - server, rpc = coordinator(ledger=ledger) - server.reorg_reconciler_enabled = True - server._ensure_job_cache_state() - ledger.assert_not_atomic = lambda: self.assertIsNone( # type: ignore[attr-defined] - server._payout_state_delivery_gate._mutation_owner - ) - results: list[dict[str, object]] = [] - errors: list[BaseException] = [] - - def reconcile() -> None: - try: - results.append( - server.reconcile_prism_pool_blocks_once(tip_hash=rpc.tip) - ) - except BaseException as exc: # pragma: no cover - asserted below - errors.append(exc) - - thread = threading.Thread(target=reconcile) - thread.start() - try: - self.assertTrue(entered.wait(5)) - with server._payout_state_delivery_gate.delivery_cancelable( - lambda: False, - generation=0, - ) as admission: - self.assertTrue(admission) - release.set() - with server._payout_state_delivery_gate._condition: - self.assertTrue( - server._payout_state_delivery_gate._condition.wait_for( - lambda: ( - server._payout_state_delivery_gate._publisher_waiting - ), - timeout=5, - ) - ) - self.assertTrue( - server._payout_state_prepare_lock.acquire(timeout=1) - ) - server._payout_state_prepare_lock.release() - finally: - release.set() - thread.join(5) - - self.assertFalse(thread.is_alive()) - self.assertEqual(errors, []) - self.assertEqual(results[0]["inactive_blocks"], 1) - self.assertEqual(server._payout_state_generation, 1) - - def test_payout_only_preparation_does_not_hold_delivery_gate(self) -> None: - entered = threading.Event() - release = threading.Event() - - class BlockingMaturityLedger(FakeLedger): - def reorg_watch_blocks( - self, - *, - active_tip_height: int, - ) -> list[dict[str, object]]: - return [] - - def mark_mature_pool_payouts( - self, - *, - active_tip_height: int, - ) -> dict[str, object]: - self.assert_not_atomic() - entered.set() - if not release.wait(5): - raise AssertionError("test did not release payout-only preparation") - return {"matured_count": 1} - - ledger = BlockingMaturityLedger() - server, rpc = coordinator(ledger=ledger) - server.reorg_reconciler_enabled = True - server._ensure_job_cache_state() - ledger.assert_not_atomic = lambda: self.assertIsNone( # type: ignore[attr-defined] - server._payout_state_delivery_gate._mutation_owner - ) - results: list[dict[str, object]] = [] - - thread = threading.Thread( - target=lambda: results.append( - server.reconcile_prism_pool_blocks_once(tip_hash=rpc.tip) - ) - ) - thread.start() - try: - self.assertTrue(entered.wait(5)) - with server._payout_state_delivery_gate.delivery_cancelable( - lambda: False, - generation=0, - ) as admission: - self.assertTrue(admission) - release.set() - with server._payout_state_delivery_gate._condition: - self.assertTrue( - server._payout_state_delivery_gate._condition.wait_for( - lambda: ( - server._payout_state_delivery_gate._publisher_waiting - ), - timeout=5, - ) - ) - self.assertTrue( - server._payout_state_prepare_lock.acquire(timeout=1) - ) - server._payout_state_prepare_lock.release() - finally: - release.set() - thread.join(5) - - self.assertFalse(thread.is_alive()) - self.assertEqual(results[0]["matured_payouts"], 1) - self.assertEqual(server._payout_state_generation, 1) - - def test_tip_poll_reuses_reconciled_external_source(self) -> None: - server, rpc = coordinator() - server.reorg_reconciler_enabled = True - initial_snapshot = server.fetch_qbit_tip_template_snapshot() - initial_sequence = server._reserve_tip_observation_sequence() - self.assertTrue( - server.observe_tip_first_seen( - initial_snapshot.bestblockhash, - observation_sequence=initial_sequence, - publish_refresh_observation=True, - ) - ) - with server.lock: - server.tip_template_snapshot = initial_snapshot - - next_tip = "76" * 32 - _advance_fake_tip(rpc, next_tip, 11) - - self.assertEqual(server.poll_qbit_tip_template_once(), 0) - self.assertEqual(server._payout_state_generation, 1) - self.assertEqual(server._payout_state_source[0], 1) - self.assertEqual(server._published_payout_state.source_generation, 1) - self.assertEqual(server._published_payout_state.source_tip_hash, next_tip) - self.assertFalse(server.tip_refresh_is_pending()) - - self.assertEqual(server.poll_qbit_tip_template_once(), 0) - self.assertEqual(server._payout_state_generation, 1) - self.assertEqual(server._payout_state_source[0], 1) - - def test_newer_tip_discards_in_progress_payout_candidate(self) -> None: - entered = threading.Event() - release = threading.Event() - - class SupersededLedger(FakeLedger): - def reorg_watch_blocks( - self, - *, - active_tip_height: int, - ) -> list[dict[str, object]]: - return [ - { - "block_height": active_tip_height + 1, - "block_hash": "bb" * 32, - "chain_state": "confirmed", - } - ] - - def mark_pool_block_inactive( - self, - *, - block_hash: str, - active_tip_height: int, - ) -> dict[str, object]: - entered.set() - if not release.wait(5): - raise AssertionError("test did not release superseded preparation") - return {"inactive_count": 1} - - def mark_mature_pool_payouts( - self, - *, - active_tip_height: int, - ) -> dict[str, object]: - return {"matured_count": 0} - - server, rpc = coordinator(ledger=SupersededLedger()) - server.reorg_reconciler_enabled = True - self.assertTrue(server.observe_tip_first_seen(rpc.tip)) - old_tip = rpc.tip - published_tip = server.current_tip_first_seen - published_sequence = server.current_tip_observation_sequence - errors: list[BaseException] = [] - - def reconcile() -> None: - try: - server.reconcile_prism_pool_blocks_once(tip_hash=old_tip) - except BaseException as exc: # pragma: no cover - asserted below - errors.append(exc) - - thread = threading.Thread(target=reconcile) - thread.start() - try: - self.assertTrue(entered.wait(5)) - new_tip = "77" * 32 - rpc.tip = new_tip - rpc.template = base_template(height=11, prevhash=new_tip) - self.assertTrue(server.observe_tip_for_refresh(new_tip)) - self.assertEqual(server.current_tip_first_seen, published_tip) - self.assertEqual( - server.current_tip_observation_sequence, - published_sequence, - ) - finally: - release.set() - thread.join(5) - - self.assertFalse(thread.is_alive()) - self.assertEqual(errors, []) - self.assertEqual(server._payout_state_generation, 1) - self.assertEqual(server.payout_state_candidates_discarded, 1) - self.assertEqual(server._published_payout_state.source_tip_hash, new_tip) - self.assertEqual(server.current_tip_first_seen, published_tip) - - def test_supersession_retry_preserves_durable_reorg_counts(self) -> None: - new_tip = "78" * 32 - - class SupersedingLedger(FakeLedger): - def __init__(self) -> None: - super().__init__() - self.server: object | None = None - self.rpc: FakeRpc | None = None - self.inactive_calls = 0 - - def reorg_watch_blocks( - self, - *, - active_tip_height: int, - ) -> list[dict[str, object]]: - return [ - { - "block_height": active_tip_height + 1, - "block_hash": "bc" * 32, - "chain_state": "confirmed", - } - ] - - def mark_pool_block_inactive( - self, - *, - block_hash: str, - active_tip_height: int, - ) -> dict[str, object]: - self.inactive_calls += 1 - if self.inactive_calls == 1: - assert self.server is not None - assert self.rpc is not None - _advance_fake_tip(self.rpc, new_tip, 11) - self.server._reserve_payout_state_source( # type: ignore[attr-defined] - "external_tip", - tip_hash=new_tip, - ) - return {"inactive_count": 1} - return {"inactive_count": 0} - - def mark_mature_pool_payouts( - self, - *, - active_tip_height: int, - ) -> dict[str, object]: - return {"matured_count": 0} - - ledger = SupersedingLedger() - server, rpc = coordinator(ledger=ledger) - ledger.server = server - ledger.rpc = rpc - server.reorg_reconciler_enabled = True - old_tip = rpc.tip - - summary = server.reconcile_prism_pool_blocks_once(tip_hash=old_tip) - - self.assertEqual(ledger.inactive_calls, 2) - self.assertEqual(summary["watched_blocks"], 1) - self.assertEqual(summary["inactive_blocks"], 1) - self.assertEqual(server.reorg_inactive_block_count, 1) - self.assertEqual(summary["published_generation"], 1) - self.assertEqual(server.payout_state_candidates_discarded, 1) - self.assertEqual(server._published_payout_state.source_tip_hash, new_tip) - - def test_failed_superseded_reconcile_does_not_publish_latest_source(self) -> None: - newer_tip = "79" * 32 - - class FailingSupersededLedger(FakeLedger): - def __init__(self) -> None: - super().__init__() - self.server: object | None = None - - def reorg_watch_blocks( - self, - *, - active_tip_height: int, - ) -> list[dict[str, object]]: - return [ - { - "block_height": active_tip_height + 1, - "block_hash": "bd" * 32, - "chain_state": "confirmed", - } - ] - - def mark_pool_block_inactive( - self, - *, - block_hash: str, - active_tip_height: int, - ) -> dict[str, object]: - assert self.server is not None - self.server._reserve_payout_state_source( # type: ignore[attr-defined] - "external_tip", - tip_hash=newer_tip, - ) - return {"inactive_count": 1} - - def mark_mature_pool_payouts( - self, - *, - active_tip_height: int, - ) -> dict[str, object]: - raise RuntimeError("maturity RPC failed") - - ledger = FailingSupersededLedger() - server, rpc = coordinator(ledger=ledger) - ledger.server = server - server.reorg_reconciler_enabled = True - - with self.assertRaisesRegex(RuntimeError, "maturity RPC failed"): - server.reconcile_prism_pool_blocks_once(tip_hash=rpc.tip) - - self.assertEqual(server._payout_state_generation, 0) - self.assertIsNone(server._published_payout_state.source_tip_hash) - self.assertTrue(server._payout_state_publication_blocked) - self.assertEqual(server.payout_state_candidates_discarded, 1) - self.assertEqual(server.reorg_inactive_block_count, 1) - self.assertTrue(server.tip_refresh_is_pending()) - - def test_tip_churn_bounds_reconcile_retries_and_fences_job_builds(self) -> None: - server, rpc = coordinator() - server.reorg_reconciler_enabled = True - server.payout_reconcile_supersession_retries = 2 - server.qbit_chain_view_untrusted = lambda: True # type: ignore[method-assign] - real_publish = server._publish_payout_state_candidate - publish_attempts = 0 - - def supersede_before_publish(candidate: object) -> int | None: - nonlocal publish_attempts - publish_attempts += 1 - newer_tip = f"{publish_attempts + 1:064x}" - server._reserve_payout_state_source( - "external_tip", - tip_hash=newer_tip, - ) - return real_publish(candidate) # type: ignore[arg-type] - - server._publish_payout_state_candidate = supersede_before_publish # type: ignore[method-assign] - - summary = server.reconcile_prism_pool_blocks_once( - tip_hash=rpc.tip, - _force_publish=True, - ) - - self.assertTrue(summary["superseded"]) - self.assertEqual(publish_attempts, 3) - self.assertEqual(server.reorg_reconcile_skip_count, 1) - self.assertEqual(server._payout_state_generation, 0) - self.assertEqual(server.payout_state_candidates_discarded, 3) - self.assertTrue(server._payout_state_publication_blocked) - state = client(1) - assert state.worker is not None - with self.assertRaisesRegex( - TemplateRefreshBlocked, - "pending publication", - ): - server.build_shared_job_bundle( - server.current_template_artifacts(), - state.worker, - ) - - def test_published_generation_prioritizes_current_tip_over_waiters(self) -> None: - gate = _PayoutStateDeliveryGate() - active_entered = threading.Event() - release_active = threading.Event() - publication_done = threading.Event() - stale_done = threading.Event() - admitted_order: list[str] = [] - - def active_delivery() -> None: - with gate.delivery_cancelable(lambda: False, generation=0) as admission: - self.assertTrue(admission) - active_entered.set() - self.assertTrue(release_active.wait(5)) - - def publish() -> None: - with gate.publication(): - gate.publish_generation(1, prioritize_delivery=True) - publication_done.set() - - def wait_for_delivery(name: str, *, priority: bool) -> None: - with gate.delivery_cancelable( - lambda: False, - generation=1, - priority=priority, - ) as admission: - if admission: - admitted_order.append(name) - admission.mark_delivered() - - def stale_waiter() -> None: - with gate.delivery_cancelable(lambda: False, generation=0) as admission: - self.assertFalse(admission) - stale_done.set() - - active_thread = threading.Thread(target=active_delivery) - publish_thread = threading.Thread(target=publish) - routine_thread = threading.Thread( - target=wait_for_delivery, - args=("routine",), - kwargs={"priority": False}, - ) - priority_thread = threading.Thread( - target=wait_for_delivery, - args=("priority",), - kwargs={"priority": True}, - ) - stale_thread = threading.Thread(target=stale_waiter) - active_thread.start() - self.assertTrue(active_entered.wait(5)) - publish_thread.start() - with gate._condition: - self.assertTrue( - gate._condition.wait_for(lambda: gate._publisher_waiting, timeout=5) - ) - routine_thread.start() - stale_thread.start() - priority_thread.start() - release_active.set() - for thread in ( - active_thread, - publish_thread, - routine_thread, - priority_thread, - stale_thread, - ): - thread.join(5) - - self.assertTrue(publication_done.is_set()) - self.assertTrue(stale_done.is_set()) - self.assertFalse(routine_thread.is_alive()) - self.assertEqual(admitted_order[:1], ["priority"]) - if admitted_order == ["priority"]: - with gate.delivery_cancelable( - lambda: False, - generation=1, - priority=False, - ) as admission: - self.assertTrue(admission) - admitted_order.append("routine") - self.assertEqual(admitted_order, ["priority", "routine"]) - - def test_priority_reservation_rejects_nonpriority_without_waiting(self) -> None: - gate = _PayoutStateDeliveryGate() - with gate.publication(): - gate.publish_generation(1, prioritize_delivery=True) - cancellation_checks = 0 - - def cancel_after_first_wait() -> bool: - nonlocal cancellation_checks - cancellation_checks += 1 - return cancellation_checks > 1 - - with gate.delivery_cancelable( - cancel_after_first_wait, - generation=1, - priority=False, - poll_seconds=0.0, - ) as admission: - self.assertFalse(admission) - - self.assertEqual(cancellation_checks, 1) - with gate.delivery_cancelable( - lambda: False, - generation=1, - priority=True, - ) as admission: - self.assertTrue(admission) - admission.mark_delivered() - - def test_publication_updates_gate_without_coordinator_locks(self) -> None: - server, _rpc = coordinator() - server._reserve_payout_state_source("payout_only") - original_publish_generation = ( - server._payout_state_delivery_gate.publish_generation - ) - lock_snapshots: list[tuple[bool, bool]] = [] - - def observe_locks( - generation: int, - *, - prioritize_delivery: bool, - ) -> None: - lock_snapshots.append( - ( - server._job_cache_lock.locked(), - server.lock._is_owned(), # type: ignore[attr-defined] - ) - ) - original_publish_generation( - generation, - prioritize_delivery=prioritize_delivery, - ) - - server._payout_state_delivery_gate.publish_generation = observe_locks # type: ignore[method-assign] - - self.assertEqual( - server._publish_payout_state_candidate( - server._current_payout_state_candidate() - ), - 1, - ) - self.assertEqual(lock_snapshots, [(False, False)]) - - def test_pending_clear_does_not_consume_first_delivery_priority(self) -> None: - server, _rpc = coordinator() - snapshot = server.fetch_qbit_tip_template_snapshot() - sequence = server._reserve_tip_observation_sequence() - self.assertTrue( - server.observe_tip_first_seen( - snapshot.bestblockhash, - observation_sequence=sequence, - publish_refresh_observation=True, - ) - ) - with server.lock: - server.tip_template_snapshot = snapshot - server._reserve_payout_state_source("payout_only") - self.assertEqual( - server._publish_payout_state_candidate( - server._current_payout_state_candidate() - ), - 1, - ) - pending_token = server._claim_tip_refresh_pending() - self.assertIsNotNone(pending_token) - - self.assertTrue( - server._clear_tip_refresh_pending_for_completed_refresh( - snapshot, - sequence, - 1, - pending_token, - ) - ) - self.assertEqual( - server._payout_state_delivery_gate._priority_generation, - 1, - ) - with server._payout_state_delivery_gate.delivery_cancelable( - lambda: False, - generation=1, - priority=False, - ) as routine_admission: - self.assertFalse(routine_admission) - with server._payout_state_delivery_gate.delivery_cancelable( - lambda: False, - generation=1, - priority=True, - ) as first_delivery: - self.assertTrue(first_delivery) - first_delivery.mark_delivered() - self.assertIsNone( - server._payout_state_delivery_gate._priority_generation - ) - - def test_collection_refresh_stops_when_chain_becomes_untrusted(self) -> None: - server, _rpc = coordinator(ledger=FakeLedger(miners=["miner-a"])) - install_fake_bundle_builder(server) - server.reorg_reconciler_enabled = True - first, second = client(1), client(2) - notifications: list[int] = [] - for state in (first, second): - state.send = ( # type: ignore[method-assign] - lambda payload, connection_id=state.connection_id: ( - notifications.append(connection_id) - if payload["method"] == "mining.notify" - else None - ) - ) - server.clients = [first, second] # type: ignore[assignment] - server.ensure_reorg_reconciled_for_tip = lambda _tip: True # type: ignore[method-assign] - chain_view_checks = 0 - - def chain_view_untrusted() -> bool: - nonlocal chain_view_checks - chain_view_checks += 1 - return chain_view_checks == 2 - - server.qbit_chain_view_untrusted = chain_view_untrusted # type: ignore[method-assign] - - with self.assertRaises(TemplateRefreshBlocked): - server.poll_qbit_tip_template_once() - - self.assertEqual(chain_view_checks, 2) - self.assertEqual(notifications, [first.connection_id]) - self.assertIsNotNone(first.active_job) - self.assertIsNone(second.active_job) - self.assertTrue(server._tip_refresh_retry.is_set()) - - def test_orphan_reconciliation_rebuilds_bundle_from_post_reorg_balances(self) -> None: - orphaned_balance = { - "recipient_id": "orphaned-miner", - "order_key": "orphaned-miner", - "p2mr_program_hex": "44" * 32, - "balance_sats": 12_345, - } - - class OrphanBalanceLedger(FakeLedger): - def __init__(self) -> None: - super().__init__() - self.prior_balances = [dict(orphaned_balance)] - self.inactive_calls = 0 - - def current_prior_balances(self) -> list[dict[str, object]]: - return [dict(balance) for balance in self.prior_balances] - - def reorg_watch_blocks( - self, - *, - active_tip_height: int, - ) -> list[dict[str, object]]: - self.assert_active_tip_height = active_tip_height - return [ - { - "block_height": 99, - "block_hash": "aa" * 32, - "chain_state": "confirmed", - } - ] - - def mark_pool_block_inactive( - self, - *, - block_hash: str, - active_tip_height: int, - ) -> dict[str, object]: - self.inactive_calls += 1 - self.prior_balances.clear() - events.append("reconcile") - return {"inactive_count": 1} - - def mark_mature_pool_payouts( - self, - *, - active_tip_height: int, - ) -> dict[str, object]: - return {"matured_count": 0} - - events: list[str] = [] - ledger = OrphanBalanceLedger() - server, rpc = coordinator(ledger=ledger) - recorded = install_fake_bundle_builder(server) - server.reorg_reconciler_enabled = True - state = client(1) - state.send = lambda _payload: None # type: ignore[method-assign] - server.clients = [state] # type: ignore[assignment] - original_rpc_call = rpc.call - - def reorg_rpc_call( - method: str, - params: list[object] | None = None, - ) -> object: - if method == "getblockhash" and params == [99]: - return "bb" * 32 - return original_rpc_call(method, params) - - rpc.call = reorg_rpc_call # type: ignore[method-assign] - original_build = server.build_audit_bundle - signed_balance_snapshots: list[list[dict[str, object]]] = [] - - def record_signed_balances(**kwargs: object) -> dict[str, object]: - balances = [ - dict(balance) - for balance in kwargs["prior_balances"] # type: ignore[union-attr] - ] - signed_balance_snapshots.append(balances) - events.append(f"bundle:{sum(int(row['balance_sats']) for row in balances)}") - bundle = original_build(**kwargs) - bundle["prior_balances"] = balances - return bundle - - server.build_audit_bundle = record_signed_balances # type: ignore[method-assign] - - artifacts = server.store_template_artifacts(dict(rpc.template)) - self.assertIsNotNone(artifacts) - stale_bundle = server.shared_job_bundle(artifacts, state.worker) # type: ignore[arg-type] - self.assertEqual(stale_bundle.prior_balances, [orphaned_balance]) - self.assertEqual(recorded["calls"], 1) - stale_context = server.stamp_job_for_client( - state, - stale_bundle, - clean_jobs=False, - ) - state.active_job = stale_context - state.active_job_ids.add(stale_context.job.job_id) - server.jobs[stale_context.job.job_id] = stale_context - events.clear() - clean_notifications: list[bool] = [] - - def record_send(payload: dict[str, object]) -> None: - if payload["method"] == "mining.notify": - events.append("fanout") - clean_notifications.append(bool(payload["params"][-1])) # type: ignore[index] - - state.send = record_send # type: ignore[method-assign] - - try: - self.assertEqual(server.poll_qbit_tip_template_once(), 1) - finally: - server.shutdown_tip_refresh_executor() - - self.assertEqual(events, ["reconcile", "bundle:0", "fanout"]) - self.assertEqual(ledger.inactive_calls, 1) - self.assertEqual(server.reorg_inactive_block_count, 1) - self.assertEqual( - signed_balance_snapshots, - [[orphaned_balance], []], - ) - self.assertEqual(recorded["calls"], 2) - self.assertIsNotNone(state.active_job) - self.assertEqual(state.active_job.prior_balances, []) - self.assertFalse(hasattr(state.active_job, "bundle")) - self.assertFalse(hasattr(stale_bundle, "bundle")) - refreshed_bundle = next(iter(server._job_bundle_cache.values())) - self.assertEqual(refreshed_bundle.prior_balances, []) - self.assertIsNot(refreshed_bundle.coinbase_manifest, stale_bundle.coinbase_manifest) - self.assertIn("coinbase_tx_hex", refreshed_bundle.coinbase_manifest) - self.assertEqual(stale_bundle.payout_state_generation, 0) - self.assertEqual(server._payout_state_generation, 1) - self.assertEqual(state.active_job.payout_state_generation, 1) - self.assertEqual(clean_notifications, [True]) - self.assertNotIn(stale_context.job.job_id, state.active_job_ids) - - def test_hundred_client_refresh_validates_chain_once(self) -> None: - server, rpc = coordinator() - install_fake_bundle_builder(server) - server.reorg_reconciler_enabled = True - server.tip_refresh_max_workers = 4 - clients = [client(index + 1) for index in range(100)] - notifications: set[int] = set() - notifications_lock = threading.Lock() - - for state in clients: - def record_send( - payload: dict[str, object], - *, - connection_id: int = state.connection_id, - ) -> None: - if payload["method"] == "mining.notify": - with notifications_lock: - notifications.add(connection_id) - - state.send = record_send # type: ignore[method-assign] - # A list makes task submission order deterministic while retaining the - # membership behavior the coordinator needs. - server.clients = clients # type: ignore[assignment] - - reconciliation_calls: list[str] = [] - chain_view_checks = 0 - forbidden_worker_checks = 0 - validation_tokens: list[TipRefreshValidationToken] = [] - - def reconcile_once(tip_hash: str) -> bool: - reconciliation_calls.append(tip_hash) - return True - - def chain_view_untrusted() -> bool: - nonlocal chain_view_checks - chain_view_checks += 1 - return False - - def reject_per_client_validation( - *, - expected_tip_hash: str | None = None, - ) -> bool: - nonlocal forbidden_worker_checks - forbidden_worker_checks += 1 - raise AssertionError( - "prepared fanout worker performed live chain validation " - f"for {expected_tip_hash}" - ) - - original_validate = server._validate_prepared_tip_refresh - - def record_validation( - bundle: object, - snapshot: object, - observation_sequence: int, - ) -> TipRefreshValidationToken: - token = original_validate(bundle, snapshot, observation_sequence) - validation_tokens.append(token) - return token - - server.ensure_reorg_reconciled_for_tip = reconcile_once # type: ignore[method-assign] - server.qbit_chain_view_untrusted = chain_view_untrusted # type: ignore[method-assign] - server.ensure_reorg_reconciled_for_current_tip = ( # type: ignore[method-assign] - reject_per_client_validation - ) - server._validate_prepared_tip_refresh = record_validation # type: ignore[method-assign] - - try: - refreshed = server.poll_qbit_tip_template_once() - finally: - server.shutdown_tip_refresh_executor() - - self.assertEqual(refreshed, 100) - self.assertEqual(notifications, {state.connection_id for state in clients}) - self.assertTrue(all(state.active_job is not None for state in clients)) - self.assertEqual(reconciliation_calls, [rpc.tip]) - self.assertEqual(chain_view_checks, 2) - self.assertEqual(forbidden_worker_checks, 0) - self.assertEqual(len(validation_tokens), 1) - - token = validation_tokens[0] - self.assertEqual(token.tip_hash, rpc.tip) - self.assertEqual( - token.template_fingerprint, - server.tip_template_snapshot.template_fingerprint, - ) - self.assertEqual( - token.template_generation, - server.tip_template_snapshot.template_generation, - ) - self.assertEqual( - token.payout_state_generation, - server._payout_state_generation, - ) - self.assertEqual( - token.observation_sequence, - server.current_tip_observation_sequence, - ) - with self.assertRaises(FrozenInstanceError): - token.tip_hash = "ff" * 32 # type: ignore[misc] - - # One fetch check, bounded pre-fanout checks, and one post-fanout check - # are acceptable. The count must not scale to the 100-client fanout. - self.assertLessEqual(rpc.count("getbestblockhash"), 4) - - def test_untrusted_prevalidation_sends_zero_jobs(self) -> None: - server, _rpc = coordinator() - install_fake_bundle_builder(server) - server.reorg_reconciler_enabled = True - clients = [client(1), client(2)] - sent: list[dict[str, object]] = [] - for state in clients: - state.send = sent.append # type: ignore[method-assign] - server.clients = clients # type: ignore[assignment] - reconciliation_calls: list[str] = [] - chain_view_checks = 0 - - def reconcile(tip_hash: str) -> bool: - reconciliation_calls.append(tip_hash) - return True - - def untrusted() -> bool: - nonlocal chain_view_checks - chain_view_checks += 1 - return True - - server.ensure_reorg_reconciled_for_tip = reconcile # type: ignore[method-assign] - server.qbit_chain_view_untrusted = untrusted # type: ignore[method-assign] - server.ensure_reorg_reconciled_for_current_tip = ( # type: ignore[method-assign] - lambda **_kwargs: self.fail("prepared worker performed chain validation") - ) - - try: - with self.assertRaises(TemplateRefreshBlocked): - server.poll_qbit_tip_template_once() - finally: - server.shutdown_tip_refresh_executor() - - self.assertEqual(reconciliation_calls, [server.rpc.tip]) - self.assertEqual(chain_view_checks, 1) - self.assertEqual(sent, []) - self.assertTrue(all(state.active_job is None for state in clients)) - self.assertTrue(server._tip_refresh_retry.is_set()) - - def test_untrusted_post_fanout_does_not_report_refresh_success(self) -> None: - server, _rpc = coordinator() - install_fake_bundle_builder(server) - server.reorg_reconciler_enabled = True - clients = [client(1), client(2)] - notifications: list[int] = [] - for state in clients: - state.send = ( # type: ignore[method-assign] - lambda payload, connection_id=state.connection_id: ( - notifications.append(connection_id) - if payload["method"] == "mining.notify" - else None - ) - ) - server.clients = clients # type: ignore[assignment] - server.ensure_reorg_reconciled_for_tip = lambda _tip: True # type: ignore[method-assign] - chain_view_checks = 0 - - def becomes_untrusted() -> bool: - nonlocal chain_view_checks - chain_view_checks += 1 - return chain_view_checks == 2 - - server.qbit_chain_view_untrusted = becomes_untrusted # type: ignore[method-assign] - - try: - with self.assertRaisesRegex( - TemplateRefreshBlocked, - "qbit chain view became untrusted during prepared fanout", - ): - server.poll_qbit_tip_template_once() - finally: - server.shutdown_tip_refresh_executor() - - self.assertEqual(chain_view_checks, 2) - self.assertEqual( - set(notifications), - {state.connection_id for state in clients}, - ) - self.assertTrue(server._tip_refresh_retry.is_set()) - self.assertIsNone( - getattr(server, "last_successful_template_refresh_monotonic", None) - ) - - def test_shutdown_during_live_fanout_trust_check_is_not_reclassified(self) -> None: - server, _rpc = coordinator() - install_fake_bundle_builder(server) - server.reorg_reconciler_enabled = True - state = client(1) - server.clients = [state] # type: ignore[assignment] - notification_started = threading.Event() - release_notification = threading.Event() - - def block_notification(payload: dict[str, object]) -> None: - if payload["method"] != "mining.notify": - return - notification_started.set() - self.assertTrue(release_notification.wait(5)) - - def shutdown_during_reconciliation(**_kwargs: object) -> bool: - server.request_shutdown() - raise ShutdownInProgress("PRISM coordinator is shutting down") - - state.send = block_notification # type: ignore[method-assign] - server.ensure_reorg_reconciled_for_tip = lambda _tip: True # type: ignore[method-assign] - server.qbit_chain_view_untrusted = lambda: False # type: ignore[method-assign] - server.ensure_reorg_reconciled_for_current_tip = ( # type: ignore[method-assign] - shutdown_during_reconciliation - ) - - try: - with self.assertRaises(ShutdownInProgress): - server.poll_qbit_tip_template_once() - self.assertTrue(notification_started.is_set()) - self.assertFalse(server._tip_refresh_retry.is_set()) - self.assertTrue( - server._tip_refresh_singleflight_lock.acquire(blocking=False) - ) - server._tip_refresh_singleflight_lock.release() - finally: - release_notification.set() - server.shutdown_tip_refresh_executor() - - def test_prevalidation_rpc_failure_schedules_immediate_retry(self) -> None: - server, rpc = coordinator() - install_fake_bundle_builder(server) - state = client(1) - server.clients = [state] # type: ignore[assignment] - snapshot = server.fetch_qbit_tip_template_snapshot() - bundle = server.prepare_tip_refresh_bundle(snapshot) - original_rpc_call = rpc.call - - def fail_tip_validation( - method: str, - params: list[object] | None = None, - ) -> object: - if method == "getbestblockhash": - raise RuntimeError("qbit rpc unavailable") - return original_rpc_call(method, params) - - rpc.call = fail_tip_validation # type: ignore[method-assign] - - with self.assertRaisesRegex( - TemplateRefreshBlocked, - "qbit tip validation failed before prepared fanout", - ): - server._validate_prepared_tip_refresh(bundle, snapshot, 1) - - self.assertTrue(server._tip_refresh_retry.is_set()) - - def test_prevalidation_trust_check_failure_schedules_immediate_retry(self) -> None: - server, _rpc = coordinator() - install_fake_bundle_builder(server) - server.reorg_reconciler_enabled = True - state = client(1) - server.clients = [state] # type: ignore[assignment] - snapshot = server.fetch_qbit_tip_template_snapshot() - bundle = server.prepare_tip_refresh_bundle(snapshot) - - def fail_chain_trust_check() -> bool: - raise RuntimeError("getblockchaininfo unavailable") - - server.qbit_chain_view_untrusted = fail_chain_trust_check # type: ignore[method-assign] - - with self.assertRaisesRegex( - TemplateRefreshBlocked, - "qbit chain trust check failed before prepared fanout", - ): - server._validate_prepared_tip_refresh(bundle, snapshot, 1) - - self.assertTrue(server._tip_refresh_retry.is_set()) - - def test_reconciliation_failure_before_fanout_sends_zero_jobs(self) -> None: - server, _rpc = coordinator() - recorded = install_fake_bundle_builder(server) - server.reorg_reconciler_enabled = True - clients = [client(1), client(2)] - sent: list[dict[str, object]] = [] - for state in clients: - state.send = sent.append # type: ignore[method-assign] - server.clients = clients # type: ignore[assignment] - reconciliation_calls: list[str] = [] - - def fail_reconciliation(tip_hash: str) -> bool: - reconciliation_calls.append(tip_hash) - raise RuntimeError("ledger reconciliation unavailable") - - server.ensure_reorg_reconciled_for_tip = fail_reconciliation # type: ignore[method-assign] - server.qbit_chain_view_untrusted = ( # type: ignore[method-assign] - lambda: self.fail("chain view checked after reconciliation failure") - ) - server.ensure_reorg_reconciled_for_current_tip = ( # type: ignore[method-assign] - lambda **_kwargs: self.fail("prepared worker performed chain validation") - ) - - try: - with self.assertRaises(TemplateRefreshBlocked): - server.poll_qbit_tip_template_once() - finally: - server.shutdown_tip_refresh_executor() - - self.assertEqual(reconciliation_calls, [server.rpc.tip]) - self.assertEqual(recorded["calls"], 0) - self.assertEqual(sent, []) - self.assertTrue(all(state.active_job is None for state in clients)) - - def test_newer_observation_cancels_pending_fanout_tasks(self) -> None: - server, rpc = coordinator() - install_fake_bundle_builder(server) - server.reorg_reconciler_enabled = True - server.tip_refresh_max_workers = 1 - first, second, third = client(1), client(2), client(3) - clients = [first, second, third] - server.clients = clients # type: ignore[assignment] - first_send_started = threading.Event() - release_first_send = threading.Event() - later_sends: list[dict[str, object]] = [] - - def first_send(payload: dict[str, object]) -> None: - if payload["method"] != "mining.notify": - return - first_send_started.set() - self.assertTrue(release_first_send.wait(5)) - - first.send = first_send # type: ignore[method-assign] - second.send = later_sends.append # type: ignore[method-assign] - third.send = later_sends.append # type: ignore[method-assign] - server.ensure_reorg_reconciled_for_tip = lambda _tip: True # type: ignore[method-assign] - server.qbit_chain_view_untrusted = lambda: False # type: ignore[method-assign] - server.ensure_reorg_reconciled_for_current_tip = ( # type: ignore[method-assign] - lambda **_kwargs: self.fail("prepared worker performed chain validation") - ) - refreshed: list[int] = [] - errors: list[BaseException] = [] - - def poll() -> None: - try: - refreshed.append(server.poll_qbit_tip_template_once()) - except BaseException as exc: # noqa: BLE001 - surface thread errors - errors.append(exc) - - poll_thread = threading.Thread(target=poll) - poll_thread.start() - try: - self.assertTrue(first_send_started.wait(5)) - old_sequence = server.current_tip_observation_sequence - old_tip = server.current_tip_first_seen - new_tip = "33" * 32 - rpc.tip = new_tip - self.assertTrue(server.observe_tip_for_refresh(new_tip)) - self.assertEqual(server.current_tip_first_seen, old_tip) - self.assertEqual(server.current_tip_observation_sequence, old_sequence) - self.assertEqual(server.latest_detected_tip[0], new_tip) - self.assertGreater(server.latest_detected_tip[1], old_sequence) - finally: - release_first_send.set() - poll_thread.join(5) - server.shutdown_tip_refresh_executor() - - self.assertFalse(poll_thread.is_alive()) - self.assertEqual(later_sends, []) - self.assertIsNotNone(first.active_job) - self.assertIsNone(second.active_job) - self.assertIsNone(third.active_job) - self.assertLessEqual(sum(refreshed), 1) - self.assertTrue( - all(isinstance(error, TemplateRefreshBlocked) for error in errors), - errors, - ) - - def test_waiting_poll_observes_new_tip_then_retries_after_owner_exits(self) -> None: - server, rpc = coordinator() - install_fake_bundle_builder(server) - server.reorg_reconciler_enabled = True - server.tip_refresh_max_workers = 1 - first, second = client(1), client(2) - server.clients = [first, second] # type: ignore[assignment] - server._ensure_tip_refresh_state() - server.ensure_reorg_reconciled_for_tip = lambda _tip: True # type: ignore[method-assign] - server.qbit_chain_view_untrusted = lambda: False # type: ignore[method-assign] - server.ensure_reorg_reconciled_for_current_tip = ( # type: ignore[method-assign] - lambda **_kwargs: True - ) - self.assertTrue(server.observe_tip_first_seen(rpc.tip)) - server._tip_refresh_retry.clear() - - tip_a = rpc.tip - tip_b = "33" * 32 - first_send_started = threading.Event() - release_first_send = threading.Event() - tip_b_observed = threading.Event() - sent_tips: dict[int, list[str]] = {1: [], 2: []} - first_notification_count = 0 - - def record_first_send(payload: dict[str, object]) -> None: - nonlocal first_notification_count - if payload["method"] != "mining.notify": - return - assert first.active_job is not None - sent_tips[1].append(str(first.active_job.template["previousblockhash"])) - first_notification_count += 1 - if first_notification_count == 1: - first_send_started.set() - self.assertTrue(release_first_send.wait(5)) - - def record_second_send(payload: dict[str, object]) -> None: - if payload["method"] != "mining.notify": - return - assert second.active_job is not None - sent_tips[2].append(str(second.active_job.template["previousblockhash"])) - - first.send = record_first_send # type: ignore[method-assign] - second.send = record_second_send # type: ignore[method-assign] - original_observe = server.observe_tip_for_refresh - - def record_observation(*args: object, **kwargs: object) -> bool: - observed = original_observe(*args, **kwargs) # type: ignore[arg-type] - if args and args[0] == tip_b and observed: - tip_b_observed.set() - return observed - - server.observe_tip_for_refresh = record_observation # type: ignore[method-assign] - # Park the replacement build so the detected-but-unpublished window - # can be asserted deterministically before tip B is published. - replacement_build_started = threading.Event() - release_replacement_build = threading.Event() - build_calls = 0 - original_build = server.build_shared_job_bundle - - def gated_build(*args: object, **kwargs: object) -> object: - nonlocal build_calls - build_calls += 1 - if build_calls >= 2: - replacement_build_started.set() - if not release_replacement_build.wait(5): - raise AssertionError("test did not release replacement build") - return original_build(*args, **kwargs) # type: ignore[arg-type] - - server.build_shared_job_bundle = gated_build # type: ignore[method-assign] - results: dict[str, list[int]] = {"old": [], "new": []} - errors: dict[str, list[BaseException]] = {"old": [], "new": []} - - def poll(label: str) -> None: - try: - results[label].append(server.poll_qbit_tip_template_once()) - except BaseException as exc: # noqa: BLE001 - surface thread errors - errors[label].append(exc) - - old_poll = threading.Thread(target=poll, args=("old",)) - new_poll = threading.Thread(target=poll, args=("new",)) - old_poll.start() - try: - self.assertTrue(first_send_started.wait(5)) - with server.lock: - active = server._active_tip_refresh - self.assertIsNotNone(active) - assert active is not None - - _advance_fake_tip(rpc, tip_b, 11) - new_poll.start() - - self.assertTrue(tip_b_observed.wait(5)) - self.assertTrue(active[1].is_set()) - self.assertFalse(replacement_build_started.wait(0.1)) - self.assertEqual(server.latest_detected_tip[0], tip_b) - self.assertEqual(server.current_tip_first_seen[0], tip_a) - pending_tip_b = server._tip_refresh_pending_token - self.assertIsNotNone(pending_tip_b) - finally: - release_first_send.set() - old_poll.join(5) - - self.assertFalse(old_poll.is_alive()) - self.assertEqual(results["old"], []) - self.assertEqual(len(errors["old"]), 1) - self.assertIsInstance(errors["old"][0], TemplateRefreshBlocked) - if server._tip_refresh_pending_token is not None: - self.assertEqual(server._tip_refresh_pending_token, pending_tip_b) - - new_poll.join(5) - self.assertFalse(new_poll.is_alive()) - self.assertEqual(errors["new"], []) - self.assertEqual(results["new"], [0]) - self.assertFalse(replacement_build_started.is_set()) - - release_replacement_build.set() - try: - self.assertEqual(server.poll_qbit_tip_template_once(), 2) - finally: - server.shutdown_tip_refresh_executor() - - self.assertTrue(replacement_build_started.is_set()) - self.assertEqual(sent_tips, {1: [tip_a, tip_b], 2: [tip_b]}) - self.assertEqual(server.current_tip_first_seen[0], tip_b) - self.assertFalse(server.tip_refresh_is_pending()) - - def test_replacement_build_waits_until_obsolete_owner_is_released(self) -> None: - server, rpc = coordinator() - install_fake_bundle_builder(server) - state = client(1) - server.clients = [state] # type: ignore[assignment] - server._ensure_tip_refresh_state() - server.ensure_reorg_reconciled_for_tip = lambda _tip: True # type: ignore[method-assign] - server.qbit_chain_view_untrusted = lambda: False # type: ignore[method-assign] - self.assertTrue(server.observe_tip_first_seen(rpc.tip)) - server._tip_refresh_retry.clear() - - bundle_started = threading.Event() - replacement_started = threading.Event() - release_bundle = threading.Event() - builder_lock = threading.Lock() - active_builders = 0 - max_active_builders = 0 - build_calls = 0 - original_build = server.build_shared_job_bundle - - def blocking_build(*args: object, **kwargs: object) -> object: - nonlocal active_builders, max_active_builders, build_calls - with builder_lock: - build_calls += 1 - active_builders += 1 - max_active_builders = max(max_active_builders, active_builders) - should_block = build_calls == 1 - try: - if should_block: - bundle_started.set() - if not release_bundle.wait(5): - raise AssertionError("test did not release bundle construction") - else: - replacement_started.set() - return original_build(*args, **kwargs) # type: ignore[arg-type] - finally: - with builder_lock: - active_builders -= 1 - - server.build_shared_job_bundle = blocking_build # type: ignore[method-assign] - sent_generations: list[int] = [] - - def record_send(payload: dict[str, object]) -> None: - if payload["method"] != "mining.notify": - return - assert state.active_job is not None - sent_generations.append(state.active_job.payout_state_generation) - - state.send = record_send # type: ignore[method-assign] - results: dict[str, list[int]] = {"old": [], "new": []} - errors: dict[str, list[BaseException]] = {"old": [], "new": []} - - def poll(label: str) -> None: - try: - results[label].append(server.poll_qbit_tip_template_once()) - except BaseException as exc: # noqa: BLE001 - surface thread errors - errors[label].append(exc) - - old_poll = threading.Thread(target=poll, args=("old",)) - new_poll = threading.Thread(target=poll, args=("new",)) - old_poll.start() - try: - self.assertTrue(bundle_started.wait(5)) - # submit_block_candidate emits this cancellation immediately once - # qbit confirms a direct PRISM block, before final audit/payout - # publication completes. - server._cancel_obsolete_job_builds("direct PRISM block accepted") - with server._job_build_scheduler_lock: - active = server._job_build_active - self.assertIsNotNone(active) - assert active is not None - self.assertTrue(active.request.cancellation.is_set()) - self.assertEqual(server._advance_payout_state_generation(), 1) - new_poll.start() - self.assertFalse(replacement_started.wait(0.1)) - new_poll.join(5) - self.assertFalse(new_poll.is_alive()) - self.assertEqual(errors["new"], []) - self.assertEqual(results["new"], [0]) - self.assertEqual(sent_generations, []) - finally: - release_bundle.set() - old_poll.join(5) - new_poll.join(5) - - self.assertFalse(old_poll.is_alive()) - server.shutdown_tip_refresh_executor() - self.assertEqual(errors["old"], []) - self.assertEqual(results["old"], [1]) - self.assertTrue(replacement_started.is_set()) - self.assertEqual(sent_generations, [1]) - self.assertEqual(max_active_builders, 1) - self.assertTrue( - all( - bundle.payout_state_generation == 1 - for bundle in server._job_bundle_cache.values() - ) - ) - self.assertFalse(server.tip_refresh_is_pending()) - - - def test_tip_change_cancels_each_expensive_build_phase(self) -> None: - phases = ( - "ledger_snapshot", - "payout_derivation", - "ctv_manifest", - "signing_verification", - "bundle_assembly", - "serialization", - ) - for index, blocked_phase in enumerate(phases, start=1): - with self.subTest(phase=blocked_phase): - server, rpc = coordinator() - install_fake_bundle_builder(server) - state = client(index) - notifications: list[dict[str, object]] = [] - state.send = notifications.append # type: ignore[method-assign] - server.clients = [state] # type: ignore[assignment] - server.ensure_reorg_reconciled_for_tip = lambda _tip: True # type: ignore[method-assign] - server.qbit_chain_view_untrusted = lambda: False # type: ignore[method-assign] - server._ensure_tip_refresh_state() - self.assertTrue(server.observe_tip_first_seen(rpc.tip)) - server._tip_refresh_retry.clear() - - phase_entered = threading.Event() - release_phase = threading.Event() - original_checkpoint = server._job_build_checkpoint - - def blocking_checkpoint( - phase: str, - cancellation: object, - ) -> None: - if phase == blocked_phase and not phase_entered.is_set(): - phase_entered.set() - if not release_phase.wait(5): - raise AssertionError( - f"test did not release {blocked_phase}" - ) - original_checkpoint(phase, cancellation) # type: ignore[arg-type] - - server._job_build_checkpoint = blocking_checkpoint # type: ignore[method-assign] - errors: list[BaseException] = [] - - def poll() -> None: - try: - server.poll_qbit_tip_template_once() - except BaseException as exc: # noqa: BLE001 - asserted below - errors.append(exc) - - poll_thread = threading.Thread(target=poll) - poll_thread.start() - try: - self.assertTrue(phase_entered.wait(5)) - next_tip = f"{index + 120:02x}" * 32 - _advance_fake_tip(rpc, next_tip, 10 + index) - self.assertTrue(server.observe_tip_first_seen(next_tip)) - finally: - release_phase.set() - poll_thread.join(5) - server.shutdown_tip_refresh_executor() - - self.assertFalse(poll_thread.is_alive()) - self.assertEqual(len(errors), 1) - self.assertIsInstance(errors[0], TemplateRefreshBlocked) - self.assertEqual(notifications, []) - self.assertIsNone(state.active_job) - self.assertTrue( - all( - entry.template["previousblockhash"] == next_tip - for entry in server._job_bundle_cache.values() - ) - ) - - def test_new_tip_during_ledger_snapshot_retries_and_publishes_latest(self) -> None: - server, rpc = coordinator() - install_fake_bundle_builder(server) - state = client(1) - server.clients = [state] # type: ignore[assignment] - server.ensure_reorg_reconciled_for_tip = lambda _tip: True # type: ignore[method-assign] - server.qbit_chain_view_untrusted = lambda: False # type: ignore[method-assign] - self.assertTrue(server.observe_tip_first_seen(rpc.tip)) - - snapshot_complete = threading.Event() - release_snapshot = threading.Event() - original_checkpoint = server._job_build_checkpoint - - def slow_snapshot_checkpoint( - phase: str, - cancellation: object, - ) -> None: - if phase == "ledger_snapshot_complete" and not snapshot_complete.is_set(): - snapshot_complete.set() - if not release_snapshot.wait(5): - raise AssertionError("test did not release ledger snapshot") - original_checkpoint(phase, cancellation) # type: ignore[arg-type] - - server._job_build_checkpoint = slow_snapshot_checkpoint # type: ignore[method-assign] - notifications: list[str] = [] - - def record_send(payload: dict[str, object]) -> None: - if payload["method"] == "mining.notify": - assert state.active_job is not None - notifications.append( - str(state.active_job.template["previousblockhash"]) - ) - - state.send = record_send # type: ignore[method-assign] - errors: list[BaseException] = [] - - def poll() -> None: - try: - server.poll_qbit_tip_template_once() - except BaseException as exc: # noqa: BLE001 - asserted below - errors.append(exc) - - owner = threading.Thread(target=poll) - owner.start() - try: - self.assertTrue(snapshot_complete.wait(5)) - latest_tip = "88" * 32 - _advance_fake_tip(rpc, latest_tip, 11) - self.assertTrue(server.observe_tip_for_refresh(latest_tip)) - finally: - release_snapshot.set() - owner.join(5) - - self.assertFalse(owner.is_alive()) - self.assertEqual(len(errors), 1) - self.assertIsInstance(errors[0], TemplateRefreshBlocked) - self.assertEqual(notifications, []) - self.assertTrue(server._consume_tip_refresh_retry()) - self.assertFalse(server._consume_tip_refresh_retry()) - - try: - self.assertEqual(server.poll_qbit_tip_template_once(), 1) - finally: - server.shutdown_tip_refresh_executor() - - self.assertEqual(notifications, ["88" * 32]) - self.assertEqual(server.current_tip_first_seen[0], "88" * 32) - self.assertFalse(server.tip_refresh_is_pending()) - - def test_refresh_exception_releases_owner_and_coalesces_one_retry(self) -> None: - server, _rpc = coordinator() - original_fetch = server.fetch_qbit_tip_template_snapshot - fetch_started = threading.Event() - release_fetch = threading.Event() - fetches = 0 - errors: list[BaseException] = [] - - def fail_once() -> object: - nonlocal fetches - fetches += 1 - if fetches == 1: - fetch_started.set() - if not release_fetch.wait(5): - raise AssertionError("test did not release failing fetch") - raise RuntimeError("transient template failure") - return original_fetch() - - server.fetch_qbit_tip_template_snapshot = fail_once # type: ignore[method-assign] - - def run_owner() -> None: - try: - server.poll_qbit_tip_template_once() - except BaseException as exc: - errors.append(exc) - - owner = threading.Thread(target=run_owner) - owner.start() - self.assertTrue(fetch_started.wait(5)) - try: - self.assertEqual(server.poll_qbit_tip_template_once(), 0) - finally: - release_fetch.set() - owner.join(5) - - self.assertFalse(owner.is_alive()) - self.assertEqual(len(errors), 1) - self.assertIsInstance(errors[0], RuntimeError) - self.assertEqual(str(errors[0]), "transient template failure") - self.assertTrue(server._tip_refresh_singleflight_lock.acquire(blocking=False)) - server._tip_refresh_singleflight_lock.release() - self.assertTrue(server._consume_tip_refresh_retry()) - self.assertFalse(server._consume_tip_refresh_retry()) - self.assertEqual(server.poll_qbit_tip_template_once(), 0) - self.assertEqual(fetches, 2) - self.assertIsNone(server.template_refresh_failure_started_monotonic) - - def test_refresh_exception_does_not_self_schedule_immediate_retry(self) -> None: - server, _rpc = coordinator() - - def fail_fetch() -> object: - raise RuntimeError("persistent template failure") - - server.fetch_qbit_tip_template_snapshot = fail_fetch # type: ignore[method-assign] - - with self.assertRaisesRegex(RuntimeError, "persistent template failure"): - server.poll_qbit_tip_template_once() - - self.assertTrue(server._tip_refresh_singleflight_lock.acquire(blocking=False)) - server._tip_refresh_singleflight_lock.release() - self.assertFalse(server._consume_tip_refresh_retry()) - self.assertIsNotNone(server.template_refresh_failure_started_monotonic) - - def test_post_accept_rpc_failure_still_wakes_refresh_driver(self) -> None: - server, rpc = coordinator() - original_call = rpc.call - - def fail_best_tip(method: str, *args: object) -> object: - if method == "getbestblockhash": - raise RuntimeError("temporary best-tip failure") - return original_call(method, *args) - - rpc.call = fail_best_tip # type: ignore[method-assign] - - with patch("lab.prism.prism_coordinator.traceback.print_exc"): - self.assertEqual( - server.refresh_jobs_after_accepted_block( - block_height=14, - block_hash="88" * 32, - heartbeat_name="block_submitter", - ), - 0, - ) - - self.assertEqual(server.post_accept_refresh_failure_count, 1) - self.assertTrue(server._consume_tip_refresh_retry()) - self.assertFalse(server._consume_tip_refresh_retry()) - - def test_post_accept_supersession_is_not_counted_as_failure(self) -> None: - server, _rpc = coordinator() - server.observe_tip_for_refresh = ( # type: ignore[method-assign] - lambda *_args, **_kwargs: False - ) - - with patch("lab.prism.prism_coordinator.traceback.print_exc") as print_exc: - self.assertEqual( - server.refresh_jobs_after_accepted_block( - block_height=15, - block_hash="99" * 32, - heartbeat_name="block_submitter", - ), - 0, - ) - - self.assertEqual(server.post_accept_refresh_failure_count, 0) - print_exc.assert_not_called() - self.assertTrue(server._consume_tip_refresh_retry()) - self.assertFalse(server._consume_tip_refresh_retry()) - - def test_tip_flicker_back_to_published_work_clears_watchdog(self) -> None: - server, rpc = coordinator() - install_fake_bundle_builder(server) - server.template_refresh_failure_exit_seconds = 10.0 - server.min_ready_miners = 10_000 - state = client(1) - state.send = lambda _payload: None # type: ignore[method-assign] - server.clients = [state] # type: ignore[assignment] - server.ensure_reorg_reconciled_for_tip = lambda _tip: True # type: ignore[method-assign] - server.qbit_chain_view_untrusted = lambda: False # type: ignore[method-assign] - original_tip = rpc.tip - original_template = rpc.template - - self.assertEqual(server.poll_qbit_tip_template_once(), 1) - published_job = state.active_job - self.assertIsNotNone(published_job) - - _advance_fake_tip(rpc, "aa" * 32, 11) - self.assertTrue(server.observe_tip_for_refresh(rpc.tip)) - rpc.tip = original_tip - rpc.template = original_template - self.assertTrue(server.observe_tip_for_refresh(original_tip)) - divergence_started = ( - server._progress_publication_divergence_since_monotonic - ) - self.assertIsNotNone(divergence_started) - assert divergence_started is not None - self.assertTrue( - server.publication_progress_failure_expired( - divergence_started + 10.001 - ) - ) - - # The coherent A poll has no replacement delivery to make because the - # existing A job is already current. Completion itself must close the - # A -> B -> A publication-divergence epoch. - self.assertEqual(server.poll_qbit_tip_template_once(), 0) - - self.assertIs(state.active_job, published_job) - self.assertIsNone( - server._progress_publication_divergence_since_monotonic - ) - self.assertFalse( - server.publication_progress_failure_expired(10_000_000_000.0) - ) - - def test_rapid_contention_coalesces_one_latest_tip_retry(self) -> None: - server, rpc = coordinator() - install_fake_bundle_builder(server) - state = client(1) - server.clients = [state] # type: ignore[assignment] - server.ensure_reorg_reconciled_for_tip = lambda _tip: True # type: ignore[method-assign] - server.qbit_chain_view_untrusted = lambda: False # type: ignore[method-assign] - self.assertTrue(server.observe_tip_first_seen(rpc.tip)) - - build_started = threading.Event() - release_build = threading.Event() - original_build = server.build_shared_job_bundle - build_lock = threading.Lock() - active_builders = 0 - maximum_builders = 0 - - def blocked_first_build(*args: object, **kwargs: object) -> object: - nonlocal active_builders, maximum_builders - with build_lock: - active_builders += 1 - maximum_builders = max(maximum_builders, active_builders) - try: - if not build_started.is_set(): - build_started.set() - if not release_build.wait(5): - raise AssertionError("test did not release obsolete build") - return original_build(*args, **kwargs) # type: ignore[arg-type] - finally: - with build_lock: - active_builders -= 1 - - server.build_shared_job_bundle = blocked_first_build # type: ignore[method-assign] - sent_tips: list[str] = [] - - def record_send(payload: dict[str, object]) -> None: - if payload["method"] == "mining.notify": - assert state.active_job is not None - sent_tips.append(str(state.active_job.template["previousblockhash"])) - - state.send = record_send # type: ignore[method-assign] - owner_errors: list[BaseException] = [] - - def own_refresh() -> None: - try: - server.poll_qbit_tip_template_once() - except BaseException as exc: # noqa: BLE001 - asserted below - owner_errors.append(exc) - - owner = threading.Thread(target=own_refresh) - owner.start() - try: - self.assertTrue(build_started.wait(5)) - # blockpoll observes B while the A owner is still building. - _advance_fake_tip(rpc, "55" * 32, 11) - self.assertEqual(server.poll_qbit_tip_template_once(), 0) - - # blockwait observes C and only wakes the driver; it never enters - # the heavy lane itself. - blockwait_calls = 0 - - def blockwait_once(known_tip: str) -> str: - nonlocal blockwait_calls - blockwait_calls += 1 - if blockwait_calls == 1: - _advance_fake_tip(rpc, "66" * 32, 12) - return rpc.tip - server.stop_event.set() - return known_tip - - server.blockwait_once = blockwait_once # type: ignore[method-assign] - blockwait = threading.Thread(target=server.blockwait_loop) - blockwait.start() - blockwait.join(5) - self.assertFalse(blockwait.is_alive()) - server.stop_event.clear() - - # Accepted-block finalization observes D and also only wakes the - # same driver. - _advance_fake_tip(rpc, "77" * 32, 13) - self.assertEqual( - server.refresh_jobs_after_accepted_block( - block_height=13, - block_hash=rpc.tip, - heartbeat_name="block_submitter", - ), - 0, - ) - - self.assertEqual(server.latest_detected_tip[0], "77" * 32) - self.assertEqual(maximum_builders, 1) - self.assertEqual(sent_tips, []) - finally: - release_build.set() - owner.join(5) - - self.assertFalse(owner.is_alive()) - self.assertEqual(len(owner_errors), 1) - self.assertIsInstance(owner_errors[0], TemplateRefreshBlocked) - self.assertTrue(server._consume_tip_refresh_retry()) - self.assertFalse(server._consume_tip_refresh_retry()) - - try: - self.assertEqual(server.poll_qbit_tip_template_once(), 1) - finally: - server.shutdown_tip_refresh_executor() - - self.assertEqual(maximum_builders, 1) - self.assertEqual(sent_tips, ["77" * 32]) - self.assertEqual(server.current_tip_first_seen[0], "77" * 32) - self.assertFalse(server.tip_refresh_is_pending()) - - def _legacy_rapid_contention_observations_are_latest_wins(self) -> None: - server, rpc = coordinator() - install_fake_bundle_builder(server) - state = client(1) - server.clients = [state] # type: ignore[assignment] - server._ensure_tip_refresh_state() - refresh_lock = _ControlledTipRefreshLock() - server._tip_refresh_lock = refresh_lock # type: ignore[assignment] - server.ensure_reorg_reconciled_for_tip = lambda _tip: True # type: ignore[method-assign] - server.qbit_chain_view_untrusted = lambda: False # type: ignore[method-assign] - self.assertTrue(server.observe_tip_first_seen(rpc.tip)) - server._tip_refresh_retry.clear() - - tip_a = rpc.tip - published_tip_a_sequence = server.current_tip_observation_sequence - tip_b = "55" * 32 - tip_c = "66" * 32 - bundle_started = threading.Event() - release_bundle = threading.Event() - original_shared_job_bundle = server.shared_job_bundle - build_calls = 0 - - def blocking_shared_job_bundle(*args: object, **kwargs: object) -> object: - nonlocal build_calls - build_calls += 1 - if build_calls == 1: - bundle_started.set() - if not release_bundle.wait(5): - raise AssertionError("test did not release bundle construction") - return original_shared_job_bundle(*args, **kwargs) # type: ignore[arg-type] - - server.shared_job_bundle = blocking_shared_job_bundle # type: ignore[method-assign] - stale_probe_started = threading.Event() - release_stale_probe = threading.Event() - tip_b_observed = threading.Event() - tip_c_observed = threading.Event() - stale_probe_finished = threading.Event() - stale_probe_results: list[bool] = [] - original_rpc_call = rpc.call - stale_probe: threading.Thread - tip_b_poll: threading.Thread - tip_c_poll: threading.Thread - stale_rpc_intercepted = False - - def ordered_rpc_call( - method: str, - params: list[object] | None = None, - ) -> object: - nonlocal stale_rpc_intercepted - if ( - method == "getbestblockhash" - and threading.current_thread() is stale_probe - and not stale_rpc_intercepted - ): - stale_rpc_intercepted = True - stale_probe_started.set() - if not release_stale_probe.wait(5): - raise AssertionError("test did not release stale contention probe") - return tip_b - return original_rpc_call(method, params) - - rpc.call = ordered_rpc_call # type: ignore[method-assign] - original_observe = server.observe_tip_for_refresh - - def record_observation(*args: object, **kwargs: object) -> bool: - observed = original_observe(*args, **kwargs) # type: ignore[arg-type] - if threading.current_thread() is stale_probe and args and args[0] == tip_b: - stale_probe_results.append(observed) - stale_probe_finished.set() - elif threading.current_thread() is tip_b_poll and args and args[0] == tip_b: - if observed: - tip_b_observed.set() - elif threading.current_thread() is tip_c_poll and args and args[0] == tip_c: - if observed: - tip_c_observed.set() - return observed - - server.observe_tip_for_refresh = record_observation # type: ignore[method-assign] - sent_tips: list[str] = [] - - def record_send(payload: dict[str, object]) -> None: - if payload["method"] != "mining.notify": - return - assert state.active_job is not None - sent_tips.append(str(state.active_job.template["previousblockhash"])) - - state.send = record_send # type: ignore[method-assign] - results: dict[str, list[int]] = { - "old": [], - "stale": [], - "b": [], - "c": [], - } - errors: dict[str, list[BaseException]] = { - "old": [], - "stale": [], - "b": [], - "c": [], - } - - def poll(label: str) -> None: - try: - results[label].append(server.poll_qbit_tip_template_once()) - except BaseException as exc: # noqa: BLE001 - surface thread errors - errors[label].append(exc) - - old_poll = threading.Thread(target=poll, args=("old",)) - stale_probe = threading.Thread(target=poll, args=("stale",)) - tip_b_poll = threading.Thread(target=poll, args=("b",)) - tip_c_poll = threading.Thread(target=poll, args=("c",)) - old_poll.start() - try: - self.assertTrue(bundle_started.wait(5)) - - stale_probe.start() - refresh_lock.next_probe_gate().set() - self.assertTrue(stale_probe_started.wait(5)) - - _advance_fake_tip(rpc, tip_b, 11) - tip_b_poll.start() - refresh_lock.next_probe_gate().set() - self.assertTrue(tip_b_observed.wait(5)) - - _advance_fake_tip(rpc, tip_c, 12) - tip_c_poll.start() - refresh_lock.next_probe_gate().set() - self.assertTrue(tip_c_observed.wait(5)) - pending_tip_c = server._tip_refresh_pending_token - detected_tip_c_sequence = server.latest_detected_tip[1] - self.assertEqual(server.latest_detected_tip[0], tip_c) - self.assertEqual(server.current_tip_first_seen[0], tip_a) - self.assertEqual( - server.current_tip_observation_sequence, - published_tip_a_sequence, - ) - - release_stale_probe.set() - self.assertTrue(stale_probe_finished.wait(5)) - self.assertEqual(stale_probe_results, [False]) - self.assertEqual(server.latest_detected_tip[0], tip_c) - self.assertEqual( - server.latest_detected_tip[1], - detected_tip_c_sequence, - ) - self.assertEqual(server.current_tip_first_seen[0], tip_a) - self.assertEqual( - server.current_tip_observation_sequence, - published_tip_a_sequence, - ) - self.assertEqual(server._tip_refresh_pending_token, pending_tip_c) - finally: - release_stale_probe.set() - release_bundle.set() - old_poll.join(5) - - self.assertFalse(old_poll.is_alive()) - self.assertEqual(results["old"], []) - self.assertEqual(len(errors["old"]), 1) - self.assertIsInstance(errors["old"][0], TemplateRefreshBlocked) - self.assertEqual(server._tip_refresh_pending_token, pending_tip_c) - self.assertEqual(sent_tips, []) - - refresh_lock.allow_waiters_to_acquire.set() - for poll_thread in (stale_probe, tip_b_poll, tip_c_poll): - poll_thread.join(5) - server.shutdown_tip_refresh_executor() - - self.assertTrue( - all(not poll_thread.is_alive() for poll_thread in (stale_probe, tip_b_poll, tip_c_poll)) - ) - self.assertEqual(errors["stale"] + errors["b"] + errors["c"], []) - self.assertEqual( - sorted(results["stale"] + results["b"] + results["c"]), - [0, 0, 1], - ) - self.assertEqual(sent_tips, [tip_c]) - self.assertEqual(server.current_tip_first_seen[0], tip_c) - self.assertFalse(server.tip_refresh_is_pending()) - - def test_client_lock_waiter_cancels_before_lock_owner_releases(self) -> None: - server, rpc = coordinator() - install_fake_bundle_builder(server) - state = client(1) - observed_lock = ObservedRLock() - state.job_update_lock = observed_lock # type: ignore[assignment] - observed_lock.acquire() - observed_lock.acquire_attempted.clear() - notifications: list[dict[str, object]] = [] - state.send = notifications.append # type: ignore[method-assign] - server.clients = [state] # type: ignore[assignment] - poll_done = threading.Event() - errors: list[BaseException] = [] - - def poll() -> None: - try: - server.poll_qbit_tip_template_once() - except BaseException as exc: # pragma: no cover - asserted below - errors.append(exc) - finally: - poll_done.set() - - thread = threading.Thread(target=poll) - thread.start() - try: - self.assertTrue(observed_lock.acquire_attempted.wait(5)) - self.advance_tip(server, rpc, "33" * 32, height=11) - self.assertTrue(poll_done.wait(5)) - self.assertEqual(notifications, []) - self.assertIsNone(state.active_job) - finally: - observed_lock.release() - thread.join(5) - - try: - self.assertEqual(len(errors), 1) - self.assertIsInstance(errors[0], TemplateRefreshBlocked) - self.assertEqual(server.poll_qbit_tip_template_once(), 1) - finally: - server.shutdown_tip_refresh_executor() - - self.assertEqual( - sum(payload["method"] == "mining.notify" for payload in notifications), - 1, - ) - self.assertEqual(state.active_job.template["previousblockhash"], rpc.tip) - self.assertGreaterEqual( - server.tip_refresh_cancellation_counts["client_lock"], - 1, - ) - - def test_publication_gate_waits_for_mutation_and_rechecks_supersession(self) -> None: - server, rpc = coordinator() - install_fake_bundle_builder(server) - state = client(1) - notifications: list[dict[str, object]] = [] - state.send = notifications.append # type: ignore[method-assign] - server.clients = [state] # type: ignore[assignment] - gate = ObservedPayoutGate(server._payout_state_delivery_gate) - server._payout_state_delivery_gate = gate # type: ignore[assignment] - poll_done = threading.Event() - errors: list[BaseException] = [] - - def poll() -> None: - try: - server.poll_qbit_tip_template_once() - except BaseException as exc: # pragma: no cover - asserted below - errors.append(exc) - finally: - poll_done.set() - - thread = threading.Thread(target=poll) - with gate.mutation(): # type: ignore[attr-defined] - thread.start() - self.assertTrue(gate.delivery_wait_started.wait(5)) - self.advance_tip(server, rpc, "44" * 32, height=11) - self.assertFalse(poll_done.wait(0.1)) - self.assertEqual(notifications, []) - self.assertIsNone(state.active_job) - thread.join(5) - - try: - self.assertFalse(thread.is_alive()) - self.assertTrue(poll_done.is_set()) - self.assertEqual(len(errors), 1) - self.assertIsInstance(errors[0], TemplateRefreshBlocked) - self.assertEqual(server.poll_qbit_tip_template_once(), 1) - finally: - server.shutdown_tip_refresh_executor() - - self.assertEqual( - sum(payload["method"] == "mining.notify" for payload in notifications), - 1, - ) - self.assertEqual(state.active_job.template["previousblockhash"], rpc.tip) - self.assertEqual(server.tip_refresh_cancellation_counts["payout_gate"], 0) - - def test_obsolete_backlog_never_starts_the_unsubmitted_fleet(self) -> None: - server, rpc = coordinator() - install_fake_bundle_builder(server) - server.tip_refresh_max_workers = 2 - clients = [client(index + 1) for index in range(8)] - observed_locks = [ObservedRLock(), ObservedRLock()] - for state, observed_lock in zip(clients, observed_locks): - state.job_update_lock = observed_lock # type: ignore[assignment] - observed_lock.acquire() - observed_lock.acquire_attempted.clear() - notifications: set[int] = set() - for state in clients: - state.send = ( # type: ignore[method-assign] - lambda payload, connection_id=state.connection_id: ( - notifications.add(connection_id) - if payload["method"] == "mining.notify" - else None - ) - ) - server.clients = clients # type: ignore[assignment] - started_clients: list[int] = [] - started_lock = threading.Lock() - original_send_prepared_job = server.send_prepared_job - - def tracked_send_prepared_job(state: object, *args: object) -> object: - with started_lock: - started_clients.append(state.connection_id) # type: ignore[attr-defined] - return original_send_prepared_job(state, *args) # type: ignore[arg-type] - - server.send_prepared_job = tracked_send_prepared_job # type: ignore[method-assign] - poll_done = threading.Event() - errors: list[BaseException] = [] - - def poll() -> None: - try: - server.poll_qbit_tip_template_once() - except BaseException as exc: # pragma: no cover - asserted below - errors.append(exc) - finally: - poll_done.set() - - thread = threading.Thread(target=poll) - thread.start() - try: - self.assertTrue(observed_locks[0].acquire_attempted.wait(5)) - self.assertTrue(observed_locks[1].acquire_attempted.wait(5)) - self.advance_tip(server, rpc, "55" * 32, height=11) - self.assertTrue(poll_done.wait(5)) - with started_lock: - obsolete_started = list(started_clients) - self.assertEqual(set(obsolete_started), {1, 2}) - self.assertEqual(notifications, set()) - finally: - for observed_lock in observed_locks: - observed_lock.release() - thread.join(5) - - try: - self.assertEqual(len(errors), 1) - self.assertIsInstance(errors[0], TemplateRefreshBlocked) - self.assertEqual(server.poll_qbit_tip_template_once(), len(clients)) - finally: - server.shutdown_tip_refresh_executor() - - self.assertEqual(notifications, {state.connection_id for state in clients}) - self.assertGreaterEqual( - server.tip_refresh_cancellation_counts["client_lock"], - 2, - ) - - def test_obsolete_executor_queue_entry_is_canceled_and_counted(self) -> None: - server, rpc = coordinator() - install_fake_bundle_builder(server) - server.tip_refresh_max_workers = 2 - server._ensure_tip_refresh_state() - - class ObservedExecutor: - def __init__(self) -> None: - self.delegate = ThreadPoolExecutor(max_workers=1) - self.submission_count = 0 - self.second_submitted = threading.Event() - - def submit(self, function: object, *args: object) -> Future[object]: - self.submission_count += 1 - future = self.delegate.submit(function, *args) # type: ignore[arg-type] - if self.submission_count == 2: - self.second_submitted.set() - return future - - def shutdown(self, **kwargs: object) -> None: - self.delegate.shutdown(**kwargs) # type: ignore[arg-type] - - observed_executor = ObservedExecutor() - server._tip_refresh_executor = observed_executor # type: ignore[assignment] - admitted = client(1) - queued = client(2) - server.clients = [admitted, queued] # type: ignore[assignment] - admitted_send_started = threading.Event() - release_admitted_send = threading.Event() - queued_canceled = threading.Event() - queued_notifications: list[dict[str, object]] = [] - - def block_notify(payload: dict[str, object]) -> None: - if payload["method"] == "mining.notify": - admitted_send_started.set() - self.assertTrue(release_admitted_send.wait(5)) - - admitted.send = block_notify # type: ignore[method-assign] - queued.send = queued_notifications.append # type: ignore[method-assign] - original_record_cancellation = server._record_tip_refresh_cancellation - - def record_cancellation(stage: str) -> None: - original_record_cancellation(stage) - if stage == "executor_queue": - queued_canceled.set() - - server._record_tip_refresh_cancellation = record_cancellation # type: ignore[method-assign] - errors: list[BaseException] = [] - - def poll() -> None: - try: - server.poll_qbit_tip_template_once() - except BaseException as exc: # pragma: no cover - asserted below - errors.append(exc) - - thread = threading.Thread(target=poll) - thread.start() - try: - self.assertTrue(admitted_send_started.wait(5)) - self.assertTrue(observed_executor.second_submitted.wait(5)) - self.advance_tip(server, rpc, "59" * 32, height=11) - self.assertTrue(queued_canceled.wait(5)) - self.assertEqual(queued_notifications, []) - self.assertIsNone(queued.active_job) - finally: - release_admitted_send.set() - thread.join(5) - server.shutdown_tip_refresh_executor() - - self.assertFalse(thread.is_alive()) - self.assertEqual(len(errors), 1) - self.assertIsInstance(errors[0], TemplateRefreshBlocked) - self.assertEqual( - server.tip_refresh_cancellation_counts["executor_queue"], - 1, - ) - - def test_latest_pending_tip_wins_across_a_b_c_observations(self) -> None: - server, rpc = coordinator() - install_fake_bundle_builder(server) - tip_a = rpc.tip - tip_b = "66" * 32 - tip_c = "77" * 32 - state = client(1) - observed_lock = ObservedRLock() - state.job_update_lock = observed_lock # type: ignore[assignment] - observed_lock.acquire() - observed_lock.acquire_attempted.clear() - notifications: list[dict[str, object]] = [] - state.send = notifications.append # type: ignore[method-assign] - server.clients = [state] # type: ignore[assignment] - poll_done = threading.Event() - errors: list[BaseException] = [] - - def poll() -> None: - try: - server.poll_qbit_tip_template_once() - except BaseException as exc: # pragma: no cover - asserted below - errors.append(exc) - finally: - poll_done.set() - - thread = threading.Thread(target=poll) - thread.start() - try: - self.assertTrue(observed_lock.acquire_attempted.wait(5)) - self.advance_tip(server, rpc, tip_b, height=11) - self.advance_tip(server, rpc, tip_c, height=12) - newest_pending_token = server._tip_refresh_pending_token - self.assertEqual(server.latest_detected_tip[0], tip_c) - self.assertEqual(server.current_tip_first_seen[0], tip_a) - self.assertTrue(poll_done.wait(5)) - self.assertTrue(server.tip_refresh_is_pending()) - self.assertEqual(server._tip_refresh_pending_token, newest_pending_token) - self.assertEqual(server.latest_detected_tip[0], tip_c) - self.assertEqual(server.current_tip_first_seen[0], tip_a) - self.assertEqual(notifications, []) - finally: - observed_lock.release() - thread.join(5) - - try: - self.assertEqual(len(errors), 1) - self.assertIsInstance(errors[0], TemplateRefreshBlocked) - self.assertEqual(server.poll_qbit_tip_template_once(), 1) - finally: - server.shutdown_tip_refresh_executor() - - self.assertFalse(server.tip_refresh_is_pending()) - self.assertEqual( - sum(payload["method"] == "mining.notify" for payload in notifications), - 1, - ) - self.assertEqual(server.current_tip_first_seen[0], tip_c) - self.assertEqual(state.active_job.template["previousblockhash"], tip_c) - - def test_admitted_old_send_finishes_before_new_tip_delivery(self) -> None: - server, rpc = coordinator() - install_fake_bundle_builder(server) - server.tip_refresh_max_workers = 2 - admitted = client(1) - waiting = client(2) - waiting_lock = ObservedRLock() - waiting.job_update_lock = waiting_lock # type: ignore[assignment] - waiting_lock.acquire() - waiting_lock.acquire_attempted.clear() - server.clients = [admitted, waiting] # type: ignore[assignment] - admitted_send_started = threading.Event() - release_admitted_send = threading.Event() - waiting_worker_finished = threading.Event() - notifications: dict[int, list[str]] = {1: [], 2: []} - - def block_old_notify(payload: dict[str, object]) -> None: - if payload["method"] == "mining.notify": - notifications[1].append("A") - admitted_send_started.set() - self.assertTrue(release_admitted_send.wait(5)) - - admitted.send = block_old_notify # type: ignore[method-assign] - waiting.send = ( # type: ignore[method-assign] - lambda payload: notifications[2].append("A") - if payload["method"] == "mining.notify" - else None - ) - original_send_prepared_job = server.send_prepared_job - - def tracked_send_prepared_job(state: object, *args: object) -> object: - try: - return original_send_prepared_job(state, *args) # type: ignore[arg-type] - finally: - if state is waiting: - waiting_worker_finished.set() - - server.send_prepared_job = tracked_send_prepared_job # type: ignore[method-assign] - poll_done = threading.Event() - errors: list[BaseException] = [] - - def poll() -> None: - try: - server.poll_qbit_tip_template_once() - except BaseException as exc: # pragma: no cover - asserted below - errors.append(exc) - finally: - poll_done.set() - - thread = threading.Thread(target=poll) - thread.start() - try: - self.assertTrue(admitted_send_started.wait(5)) - self.assertTrue(waiting_lock.acquire_attempted.wait(5)) - self.advance_tip(server, rpc, "88" * 32, height=11) - self.assertTrue(waiting_worker_finished.wait(5)) - self.assertFalse(poll_done.is_set()) - self.assertEqual(notifications[2], []) - finally: - release_admitted_send.set() - thread.join(5) - waiting_lock.release() - - admitted.send = ( # type: ignore[method-assign] - lambda payload: notifications[1].append("B") - if payload["method"] == "mining.notify" - else None - ) - waiting.send = ( # type: ignore[method-assign] - lambda payload: notifications[2].append("B") - if payload["method"] == "mining.notify" - else None - ) - try: - self.assertEqual(len(errors), 1) - self.assertIsInstance(errors[0], TemplateRefreshBlocked) - self.assertEqual(server.poll_qbit_tip_template_once(), 2) - finally: - server.shutdown_tip_refresh_executor() - - self.assertEqual(notifications[1], ["A", "B"]) - self.assertEqual(notifications[2], ["B"]) - - def test_prepared_wait_phases_account_for_total_elapsed(self) -> None: - server, _rpc = coordinator() - install_fake_bundle_builder(server) - state = client(1) - server.clients = [state] # type: ignore[assignment] - snapshot = server.fetch_qbit_tip_template_snapshot() - sequence = server._reserve_tip_observation_sequence() - self.assertTrue( - server.observe_tip_first_seen( - snapshot.bestblockhash, - observation_sequence=sequence, - publish_refresh_observation=True, - ) - ) - with server.lock: - server.tip_template_snapshot = snapshot - bundle = server.prepare_tip_refresh_bundle(snapshot) - token = server._validate_prepared_tip_refresh(bundle, snapshot, sequence) - - class ManualClock: - def __init__(self) -> None: - self.now = 10.0 - - def monotonic(self) -> float: - return self.now - - def advance(self, seconds: float) -> None: - self.now += seconds - - clock = ManualClock() - - class AdvancingLock: - def acquire(self, *, timeout: float) -> bool: - self.timeout = timeout - clock.advance(2.0) - return True - - def release(self) -> None: - return None - - class AdvancingGate: - class Admission: - def __bool__(self) -> bool: - return True - - @staticmethod - def mark_delivered() -> None: - pass - - @contextmanager - def delivery_cancelable( - self, - _cancelled: object, - **_kwargs: object, - ) -> object: - clock.advance(4.0) - yield self.Admission() - - state.job_update_lock = AdvancingLock() # type: ignore[assignment] - server._payout_state_delivery_gate = AdvancingGate() # type: ignore[assignment] - original_stamp = server.stamp_job_for_client - - def advancing_stamp(*args: object, **kwargs: object) -> object: - clock.advance(5.0) - return original_stamp(*args, **kwargs) # type: ignore[arg-type] - - server.stamp_job_for_client = advancing_stamp # type: ignore[method-assign] - state.send = lambda _payload: clock.advance(3.0) # type: ignore[method-assign] - - with patch( - "lab.prism.prism_coordinator.time.monotonic", - clock.monotonic, - ): - result = server.send_prepared_job( - state, - bundle, - snapshot, - token, - state.connection_id, - None, - _FanoutCancellation(), - submitted_monotonic=7.0, - ) - - self.assertEqual(result.result, "sent") - expected_phases = { - "executor_queue": 3.0, - "client_lock": 2.0, - "payout_gate": 4.0, - "stamp": 5.0, - "socket_send": 6.0, - } - for phase, expected in expected_phases.items(): - self.assertAlmostEqual( - server.job_build_phase_seconds[phase], - expected, - ) - self.assertAlmostEqual(server.job_build_seconds_sum, 20.0) - self.assertAlmostEqual(sum(expected_phases.values()), 20.0) - metrics = server.metrics_payload() - self.assertIn( - 'qbit_prism_tip_refresh_cancellations_total{stage="executor_queue"} 0', - metrics, - ) - self.assertIn( - 'qbit_prism_job_build_phase_seconds_total{phase="payout_gate"} 4.000000', - metrics, - ) - - def test_prepared_fanout_vardiff_wait_does_not_hold_coordinator_lock(self) -> None: - server, _rpc = coordinator() - install_fake_bundle_builder(server) - state = client(1) - state.send = lambda _payload: None # type: ignore[method-assign] - server.clients = [state] # type: ignore[assignment] - snapshot = server.fetch_qbit_tip_template_snapshot() - sequence = server._reserve_tip_observation_sequence() - self.assertTrue( - server.observe_tip_first_seen( - snapshot.bestblockhash, - observation_sequence=sequence, - publish_refresh_observation=True, - ) - ) - with server.lock: - server.tip_template_snapshot = snapshot - bundle = server.prepare_tip_refresh_bundle(snapshot) - token = server._validate_prepared_tip_refresh(bundle, snapshot, sequence) - vardiff_lock = ObservedRLock() - state.vardiff_lock = vardiff_lock # type: ignore[assignment] - results: list[object] = [] - - with vardiff_lock: - vardiff_lock.acquire_attempted.clear() - delivery = threading.Thread( - target=lambda: results.append( - server.send_prepared_job( - state, - bundle, - snapshot, - token, - state.connection_id, - None, - _FanoutCancellation(), - ) - ) - ) - delivery.start() - self.assertTrue(vardiff_lock.acquire_attempted.wait(2)) - acquired = server.lock.acquire(timeout=0.25) - self.assertTrue(acquired) - if acquired: - server.lock.release() - - delivery.join(2) - self.assertFalse(delivery.is_alive()) - self.assertEqual(len(results), 1) - self.assertEqual(results[0].result, "sent") - - def test_routine_same_tip_observation_keeps_active_fanout_valid(self) -> None: - server, rpc = coordinator() - install_fake_bundle_builder(server) - server.reorg_reconciler_enabled = True - server.tip_refresh_max_workers = 1 - first, second, third = client(1), client(2), client(3) - clients = [first, second, third] - server.clients = clients # type: ignore[assignment] - first_send_started = threading.Event() - release_first_send = threading.Event() - notifications: set[int] = set() - - def record_send( - payload: dict[str, object], - *, - connection_id: int, - block: bool = False, - ) -> None: - if payload["method"] != "mining.notify": - return - notifications.add(connection_id) - if block: - first_send_started.set() - self.assertTrue(release_first_send.wait(5)) - - first.send = lambda payload: record_send( # type: ignore[method-assign] - payload, - connection_id=first.connection_id, - block=True, - ) - second.send = lambda payload: record_send( # type: ignore[method-assign] - payload, - connection_id=second.connection_id, - ) - third.send = lambda payload: record_send( # type: ignore[method-assign] - payload, - connection_id=third.connection_id, - ) - server.ensure_reorg_reconciled_for_tip = lambda _tip: True # type: ignore[method-assign] - server.qbit_chain_view_untrusted = lambda: False # type: ignore[method-assign] - results: list[int] = [] - errors: list[BaseException] = [] - - def poll() -> None: - try: - results.append(server.poll_qbit_tip_template_once()) - except BaseException as exc: # pragma: no cover - asserted below - errors.append(exc) - - poll_thread = threading.Thread(target=poll) - poll_thread.start() - try: - self.assertTrue(first_send_started.wait(5)) - active_sequence = server.current_tip_observation_sequence - self.assertTrue(server.observe_tip_first_seen(rpc.tip)) - self.assertEqual( - server.current_tip_observation_sequence, - active_sequence, - ) - finally: - release_first_send.set() - poll_thread.join(5) - server.shutdown_tip_refresh_executor() - - self.assertFalse(poll_thread.is_alive()) - self.assertEqual(errors, []) - self.assertEqual(results, [3]) - self.assertEqual(notifications, {1, 2, 3}) - self.assertFalse(server._tip_refresh_retry.is_set()) - - def test_repeated_same_tip_contenders_do_not_supersede_active_fanout(self) -> None: - server, rpc = coordinator() - install_fake_bundle_builder(server) - server.tip_refresh_max_workers = 1 - first, second = client(1), client(2) - server.clients = [first, second] # type: ignore[assignment] - server._ensure_tip_refresh_state() - server.ensure_reorg_reconciled_for_tip = lambda _tip: True # type: ignore[method-assign] - server.qbit_chain_view_untrusted = lambda: False # type: ignore[method-assign] - server.ensure_reorg_reconciled_for_current_tip = ( # type: ignore[method-assign] - lambda **_kwargs: True - ) - self.assertTrue(server.observe_tip_first_seen(rpc.tip)) - server._tip_refresh_retry.clear() - - first_send_started = threading.Event() - release_first_send = threading.Event() - notifications: list[int] = [] - - def record_send( - payload: dict[str, object], - *, - connection_id: int, - block_first: bool = False, - ) -> None: - if payload["method"] != "mining.notify": - return - notifications.append(connection_id) - if block_first: - first_send_started.set() - self.assertTrue(release_first_send.wait(5)) - - first.send = lambda payload: record_send( # type: ignore[method-assign] - payload, - connection_id=1, - block_first=True, - ) - second.send = lambda payload: record_send( # type: ignore[method-assign] - payload, - connection_id=2, - ) - results: list[int] = [] - errors: list[BaseException] = [] - - def poll() -> None: - try: - results.append(server.poll_qbit_tip_template_once()) - except BaseException as exc: # noqa: BLE001 - surface thread errors - errors.append(exc) - - active_poll = threading.Thread(target=poll) - active_poll.start() - try: - self.assertTrue(first_send_started.wait(5)) - with server.lock: - active = server._active_tip_refresh - active_sequence = server.current_tip_observation_sequence - self.assertIsNotNone(active) - assert active is not None - - contender_results = [ - server.poll_qbit_tip_template_once() - for _attempt in range(10) - ] - self.assertEqual(contender_results, [0] * 10) - self.assertFalse(active[1].is_set()) - self.assertEqual( - server.current_tip_observation_sequence, - active_sequence, - ) - self.assertFalse(server.tip_refresh_is_pending()) - finally: - release_first_send.set() - active_poll.join(5) - - self.assertFalse(active_poll.is_alive()) - self.assertEqual(errors, []) - self.assertEqual(results, [2]) - self.assertEqual(notifications, [1, 2]) - self.assertFalse(server.tip_refresh_is_pending()) - self.assertTrue(server._consume_tip_refresh_retry()) - self.assertFalse(server._consume_tip_refresh_retry()) - try: - self.assertEqual(server.poll_qbit_tip_template_once(), 0) - finally: - server.shutdown_tip_refresh_executor() - self.assertEqual(notifications, [1, 2]) - - def _legacy_same_tip_contention_probe_keeps_active_fanout_valid(self) -> None: - server, rpc = coordinator() - install_fake_bundle_builder(server) - server.tip_refresh_max_workers = 1 - first, second = client(1), client(2) - server.clients = [first, second] # type: ignore[assignment] - server._ensure_tip_refresh_state() - refresh_lock = _ControlledTipRefreshLock() - server._tip_refresh_lock = refresh_lock # type: ignore[assignment] - server.ensure_reorg_reconciled_for_tip = lambda _tip: True # type: ignore[method-assign] - server.qbit_chain_view_untrusted = lambda: False # type: ignore[method-assign] - server.ensure_reorg_reconciled_for_current_tip = ( # type: ignore[method-assign] - lambda **_kwargs: True - ) - self.assertTrue(server.observe_tip_first_seen(rpc.tip)) - server._tip_refresh_retry.clear() - - first_send_started = threading.Event() - release_first_send = threading.Event() - contention_probe_finished = threading.Event() - notifications: list[int] = [] - - def record_send( - payload: dict[str, object], - *, - connection_id: int, - block_first: bool = False, - ) -> None: - if payload["method"] != "mining.notify": - return - notifications.append(connection_id) - if block_first: - first_send_started.set() - self.assertTrue(release_first_send.wait(5)) - - first.send = lambda payload: record_send( # type: ignore[method-assign] - payload, - connection_id=1, - block_first=True, - ) - second.send = lambda payload: record_send( # type: ignore[method-assign] - payload, - connection_id=2, - ) - original_contention_probe = server._probe_tip_while_refresh_waiting - - def record_contention_probe() -> None: - original_contention_probe() - contention_probe_finished.set() - - server._probe_tip_while_refresh_waiting = record_contention_probe # type: ignore[method-assign] - results: dict[str, list[int]] = {"active": [], "waiting": []} - errors: dict[str, list[BaseException]] = {"active": [], "waiting": []} - - def poll(label: str) -> None: - try: - results[label].append(server.poll_qbit_tip_template_once()) - except BaseException as exc: # noqa: BLE001 - surface thread errors - errors[label].append(exc) - - active_poll = threading.Thread(target=poll, args=("active",)) - waiting_poll = threading.Thread(target=poll, args=("waiting",)) - active_poll.start() - try: - self.assertTrue(first_send_started.wait(5)) - with server.lock: - active = server._active_tip_refresh - active_sequence = server.current_tip_observation_sequence - self.assertIsNotNone(active) - assert active is not None - - waiting_poll.start() - refresh_lock.next_probe_gate().set() - self.assertTrue(contention_probe_finished.wait(5)) - self.assertFalse(active[1].is_set()) - self.assertEqual( - server.current_tip_observation_sequence, - active_sequence, - ) - self.assertFalse(server.tip_refresh_is_pending()) - finally: - release_first_send.set() - active_poll.join(5) - - self.assertFalse(active_poll.is_alive()) - self.assertEqual(errors["active"], []) - self.assertEqual(results["active"], [2]) - - refresh_lock.allow_waiters_to_acquire.set() - waiting_poll.join(5) - server.shutdown_tip_refresh_executor() - - self.assertFalse(waiting_poll.is_alive()) - self.assertEqual(errors["waiting"], []) - self.assertEqual(results["waiting"], [0]) - self.assertEqual(notifications, [1, 2]) - self.assertFalse(server.tip_refresh_is_pending()) - - def test_prepublication_payout_change_uses_coordination_budget(self) -> None: - server, _rpc = coordinator() - install_fake_bundle_builder(server) - server._ensure_tip_refresh_state() - state = client(1) - state.send = lambda _payload: None # type: ignore[method-assign] - server.clients = {state} - server._pool_ready_latched = True - delegate = server._tip_refresh_lock - blocked_at = server.started_monotonic + 10.0 - - class AdvancePayoutBeforePublication: - def __init__(self) -> None: - self.advanced = False - - def acquire(self, timeout: float = -1) -> bool: - if not self.advanced: - self.advanced = True - server._advance_payout_state_generation() - return delegate.acquire(timeout=timeout) - - def release(self) -> None: - delegate.release() - - server._tip_refresh_lock = AdvancePayoutBeforePublication() # type: ignore[assignment] - - try: - with ( - patch( - "lab.prism.prism_coordinator.time.monotonic", - return_value=blocked_at, - ), - self.assertRaisesRegex( - TemplateRefreshSuperseded, - "payout state changed before refresh publication", - ), - ): - server.poll_qbit_tip_template_once() - finally: - server.shutdown_tip_refresh_executor() - - self.assertIsNone( - getattr(server, "template_refresh_failure_started_monotonic", None) - ) - # blocked_at is a real monotonic reading plus an offset, large enough - # on a long-uptime host that (blocked_at + 5.0) - blocked_at rounds - # away from exactly 5.0 — the streak age is plain float subtraction. - self.assertAlmostEqual( - server.coordination_blocked_streak_age_seconds(blocked_at + 5.0), - 5.0, - delta=1e-6, - ) - self.assertTrue(server._tip_refresh_retry.is_set()) - - def test_post_fanout_payout_change_schedules_immediate_retry(self) -> None: - server, rpc = coordinator() - install_fake_bundle_builder(server) - server.reorg_reconciler_enabled = True - state = client(1) - state.send = lambda _payload: None # type: ignore[method-assign] - server.clients = [state] # type: ignore[assignment] - server.ensure_reorg_reconciled_for_tip = lambda _tip: True # type: ignore[method-assign] - server.qbit_chain_view_untrusted = lambda: False # type: ignore[method-assign] - original_rpc_call = rpc.call - - def advance_during_post_fanout_tip_check( - method: str, - params: list[object] | None = None, - ) -> object: - result = original_rpc_call(method, params) - if method == "getbestblockhash" and rpc.count(method) == 4: - server._advance_payout_state_generation() - return result - - rpc.call = advance_during_post_fanout_tip_check # type: ignore[method-assign] - - try: - with self.assertRaises(TemplateRefreshBlocked): - server.poll_qbit_tip_template_once() - finally: - server.shutdown_tip_refresh_executor() - - self.assertIsNotNone(state.active_job) - self.assertEqual(state.active_job.payout_state_generation, 0) - self.assertEqual(server._payout_state_generation, 1) - self.assertTrue(server._tip_refresh_retry.is_set()) - - def test_payout_mutation_waits_for_prepared_network_delivery(self) -> None: - server, _rpc = coordinator() - install_fake_bundle_builder(server) - state = client(1) - server.clients = [state] # type: ignore[assignment] - send_started = threading.Event() - release_send = threading.Event() - mutation_started = threading.Event() - mutation_completed = threading.Event() - poll_results: list[int] = [] - poll_errors: list[BaseException] = [] - - def block_notify(payload: dict[str, object]) -> None: - if payload["method"] == "mining.notify": - send_started.set() - self.assertTrue(release_send.wait(5)) - - state.send = block_notify # type: ignore[method-assign] - - def poll() -> None: - try: - poll_results.append(server.poll_qbit_tip_template_once()) - except BaseException as exc: # noqa: BLE001 - asserted below - poll_errors.append(exc) - - def mutate() -> None: - mutation_started.set() - server._advance_payout_state_generation() - mutation_completed.set() - - poll_thread = threading.Thread(target=poll) - mutation_thread = threading.Thread(target=mutate) - poll_thread.start() - try: - self.assertTrue(send_started.wait(5)) - mutation_thread.start() - self.assertTrue(mutation_started.wait(5)) - self.assertFalse(mutation_completed.wait(0.1)) - self.assertEqual(server._payout_state_generation, 0) - with server._payout_state_delivery_gate._condition: - self.assertTrue( - server._payout_state_delivery_gate._publisher_waiting - ) - self.assertIsNone( - server._payout_state_delivery_gate._mutation_owner - ) - self.assertTrue( - server._payout_state_prepare_lock.acquire(timeout=1) - ) - server._payout_state_prepare_lock.release() - finally: - release_send.set() - mutation_thread.join(5) - poll_thread.join(5) - server.shutdown_tip_refresh_executor() - - self.assertFalse(mutation_thread.is_alive()) - self.assertFalse(poll_thread.is_alive()) - self.assertTrue(mutation_completed.is_set()) - self.assertEqual(server._payout_state_generation, 1) - self.assertEqual(len(poll_results) + len(poll_errors), 1) - if poll_errors: - self.assertIsInstance(poll_errors[0], TemplateRefreshBlocked) - else: - self.assertEqual(poll_results, [1]) - self.assertTrue(server._tip_refresh_retry.is_set()) - - def test_prepared_disconnection_after_admission_releases_cancellation_gate(self) -> None: - server, _rpc = coordinator() - install_fake_bundle_builder(server) - state = client(1) - server.clients = [state] # type: ignore[assignment] - snapshot = server.fetch_qbit_tip_template_snapshot() - sequence = server._reserve_tip_observation_sequence() - self.assertTrue( - server.observe_tip_first_seen( - snapshot.bestblockhash, - observation_sequence=sequence, - publish_refresh_observation=True, - ) - ) - with server.lock: - server.tip_template_snapshot = snapshot - bundle = server.prepare_tip_refresh_bundle(snapshot) - token = server._validate_prepared_tip_refresh( - bundle, - snapshot, - sequence, - ) - cancellation = _FanoutCancellation() - original_gate = server._payout_state_delivery_gate - - class RemoveClientAtAdmission: - @contextmanager - def delivery(self) -> object: - with original_gate.delivery(): - server.clients = [] # type: ignore[assignment] - yield - - @contextmanager - def delivery_cancelable( - self, - cancelled: object, - **kwargs: object, - ) -> object: - with original_gate.delivery_cancelable(cancelled, **kwargs) as admitted: - if admitted: - server.clients = [] # type: ignore[assignment] - yield admitted - - def mutation(self) -> object: - return original_gate.mutation() - - server._payout_state_delivery_gate = RemoveClientAtAdmission() - - result = server.send_prepared_job( - state, - bundle, - snapshot, - token, - state.connection_id, - None, - cancellation, - ) - - self.assertEqual(result.result, "disconnected") - self.assertEqual(cancellation._active_deliveries, 0) - cancellation.set() - - def test_publication_block_fences_escaped_prepared_bundle(self) -> None: - server, _rpc = coordinator() - install_fake_bundle_builder(server) - state = client(1) - sent: list[dict[str, object]] = [] - state.send = sent.append # type: ignore[method-assign] - server.clients = [state] # type: ignore[assignment] - snapshot = server.fetch_qbit_tip_template_snapshot() - sequence = server._reserve_tip_observation_sequence() - self.assertTrue( - server.observe_tip_first_seen( - snapshot.bestblockhash, - observation_sequence=sequence, - publish_refresh_observation=True, - ) - ) - with server.lock: - server.tip_template_snapshot = snapshot - bundle = server.prepare_tip_refresh_bundle(snapshot) - token = server._validate_prepared_tip_refresh( - bundle, - snapshot, - sequence, - ) - cancellation = _FanoutCancellation() - - server._reserve_payout_state_source("payout_only") - server._block_payout_state_publication() - - with server.lock: - self.assertTrue( - server._tip_refresh_token_current_locked(token, bundle, snapshot) - ) - self.assertFalse(cancellation.is_set()) - result = server.send_prepared_job( - state, - bundle, - snapshot, - token, - state.connection_id, - None, - cancellation, - ) - - self.assertEqual(result.result, "skipped") - self.assertEqual(sent, []) - self.assertIsNone(state.active_job) - self.assertTrue(server._payout_state_delivery_gate._delivery_blocked) - - published = server._publish_payout_state_candidate( - server._current_payout_state_candidate() - ) - self.assertEqual(published, 1) - with server._payout_state_delivery_gate.delivery_cancelable( - lambda: False, - generation=1, - priority=True, - ) as admission: - self.assertTrue(admission) - - def test_sequential_payout_change_schedules_immediate_retry(self) -> None: - server, _rpc = coordinator(ledger=FakeLedger(miners=["solo"])) - install_fake_bundle_builder(server) - first, second = client(1), client(2) - server.clients = [first, second] # type: ignore[assignment] - first_notifications = 0 - - def record_first_send(payload: dict[str, object]) -> None: - nonlocal first_notifications - if payload["method"] == "mining.notify": - first_notifications += 1 - - first.send = record_first_send # type: ignore[method-assign] - second.send = lambda _payload: None # type: ignore[method-assign] - original_maybe_send_job = server.maybe_send_job - send_calls = 0 - - def advance_between_clients(*args: object, **kwargs: object) -> bool: - nonlocal send_calls - sent = original_maybe_send_job(*args, **kwargs) # type: ignore[arg-type] - send_calls += 1 - if send_calls == 1: - server._advance_payout_state_generation() - return sent - - server.maybe_send_job = advance_between_clients # type: ignore[method-assign] - - with self.assertRaises(TemplateRefreshBlocked): - server.poll_qbit_tip_template_once() - - self.assertEqual(first_notifications, 1) - self.assertIsNotNone(first.active_job) - self.assertIsNotNone(second.active_job) - self.assertEqual(first.active_job.payout_state_generation, 0) - self.assertEqual(second.active_job.payout_state_generation, 1) - self.assertTrue(server._tip_refresh_retry.is_set()) - - def test_payout_change_after_client_selection_coalesces_full_same_tip_set( - self, - ) -> None: - server, _rpc = coordinator() - install_fake_bundle_builder(server) - first = client(1) - second = client(2) - first_sent: list[dict[str, object]] = [] - second_sent: list[dict[str, object]] = [] - first.send = first_sent.append # type: ignore[method-assign] - second.send = second_sent.append # type: ignore[method-assign] - server.clients = [second] # type: ignore[assignment] - - try: - self.assertEqual(server.poll_qbit_tip_template_once(), 1) - self.assertIsNotNone(second.active_job) - self.assertEqual(second.active_job.payout_state_generation, 0) - - server.clients = [first, second] # type: ignore[assignment] - original_prepare = server.prepare_tip_refresh_bundle - advanced = False - - def advance_before_prepare(*args: object, **kwargs: object) -> object: - nonlocal advanced - if not advanced: - advanced = True - server._advance_payout_state_generation() - return original_prepare(*args, **kwargs) # type: ignore[arg-type] - - server.prepare_tip_refresh_bundle = advance_before_prepare # type: ignore[method-assign] - - self.assertEqual(server.poll_qbit_tip_template_once(), 2) - finally: - server.shutdown_tip_refresh_executor() - - self.assertIsNotNone(first.active_job) - self.assertIsNotNone(second.active_job) - self.assertEqual(first.active_job.payout_state_generation, 1) - self.assertEqual(second.active_job.payout_state_generation, 1) - self.assertEqual( - sum(payload["method"] == "mining.notify" for payload in first_sent), - 1, - ) - self.assertEqual( - sum(payload["method"] == "mining.notify" for payload in second_sent), - 2, - ) - self.assertFalse(server.tip_refresh_is_pending()) - - def test_payout_change_during_build_coalesces_latest_generation(self) -> None: - server, _rpc = coordinator() - install_fake_bundle_builder(server) - state = client(1) - sent_generations: list[int] = [] - - def record_send(payload: dict[str, object]) -> None: - if payload["method"] == "mining.notify": - assert state.active_job is not None - sent_generations.append( - state.active_job.payout_state_generation - ) - - state.send = record_send # type: ignore[method-assign] - server.clients = [state] # type: ignore[assignment] - original_build = server.build_shared_job_bundle - build_calls = 0 - - def advance_first_build(*args: object, **kwargs: object) -> object: - nonlocal build_calls - build_calls += 1 - if build_calls == 1: - self.assertEqual(server._advance_payout_state_generation(), 1) - return original_build(*args, **kwargs) # type: ignore[arg-type] - - server.build_shared_job_bundle = advance_first_build # type: ignore[method-assign] - - try: - self.assertEqual(server.poll_qbit_tip_template_once(), 1) - finally: - server.shutdown_tip_refresh_executor() - - self.assertEqual(build_calls, 2) - self.assertEqual(sent_generations, [1]) - self.assertIsNotNone(state.active_job) - self.assertEqual(state.active_job.payout_state_generation, 1) - self.assertTrue( - all( - bundle.payout_state_generation == 1 - for bundle in server._job_bundle_cache.values() - ) - ) - self.assertFalse(server.tip_refresh_is_pending()) - - def test_executor_submission_failure_cancels_and_drains_started_work(self) -> None: - server, _rpc = coordinator() - install_fake_bundle_builder(server) - server.reorg_reconciler_enabled = True - first, second = client(1), client(2) - server.clients = [first, second] # type: ignore[assignment] - first_send_started = threading.Event() - release_first_send = threading.Event() - first_notifications: list[dict[str, object]] = [] - later_notifications: list[dict[str, object]] = [] - - def blocking_send(payload: dict[str, object]) -> None: - if payload["method"] != "mining.notify": - return - first_notifications.append(payload) - first_send_started.set() - self.assertTrue(release_first_send.wait(5)) - - first.send = blocking_send # type: ignore[method-assign] - second.send = later_notifications.append # type: ignore[method-assign] - server.ensure_reorg_reconciled_for_tip = lambda _tip: True # type: ignore[method-assign] - server.qbit_chain_view_untrusted = lambda: False # type: ignore[method-assign] - backing_executor = ThreadPoolExecutor(max_workers=1) - - class RejectSecondSubmission: - def __init__(self) -> None: - self.submission_count = 0 - - def submit(self, function: object, *args: object) -> Future[object]: - self.submission_count += 1 - if self.submission_count == 1: - future = backing_executor.submit(function, *args) # type: ignore[arg-type] - if not first_send_started.wait(5): - raise AssertionError("first fanout task did not start") - return future - raise RuntimeError("executor rejected queued fanout task") - - server.tip_refresh_executor = lambda: RejectSecondSubmission() # type: ignore[method-assign] - errors: list[BaseException] = [] - - def poll() -> None: - try: - server.poll_qbit_tip_template_once() - except BaseException as exc: # pragma: no cover - asserted below - errors.append(exc) - - poll_thread = threading.Thread(target=poll) - poll_thread.start() - self.assertTrue(first_send_started.wait(5)) - release_first_send.set() - poll_thread.join(5) - backing_executor.shutdown(wait=True, cancel_futures=True) - - self.assertFalse(poll_thread.is_alive()) - self.assertEqual(len(first_notifications), 1) - self.assertEqual(later_notifications, []) - self.assertEqual(len(errors), 1) - self.assertIsInstance(errors[0], RuntimeError) - self.assertIsNone(server._active_tip_refresh) - self.assertTrue(server._tip_refresh_retry.is_set()) - - -if __name__ == "__main__": - unittest.main() diff --git a/tests/test_prism_vardiff.py b/tests/test_prism_vardiff.py deleted file mode 100644 index eb4ab501a..000000000 --- a/tests/test_prism_vardiff.py +++ /dev/null @@ -1,1416 +0,0 @@ -#!/usr/bin/env python3 -"""Focused PRISM coordinator vardiff tests.""" -# ruff: noqa: F403, F405 - -from __future__ import annotations - -import unittest -from lab.prism.vardiff_service import VardiffService -from tests.prism_vardiff_test_support import * - - -class PrismCoordinatorVardiffTests(unittest.TestCase): - def test_vardiff_owner_initialization_is_single_flight(self) -> None: - server = coordinator() - server.__dict__.pop("_vardiff_service", None) - server.vardiff_idle_queue_depth = 7 - real_service_type = VardiffService - constructor_entered = threading.Event() - release_constructor = threading.Event() - calls = 0 - calls_lock = threading.Lock() - - def construct(runtime: object) -> VardiffService: - nonlocal calls - with calls_lock: - calls += 1 - constructor_entered.set() - self.assertTrue(release_constructor.wait(1.0)) - return real_service_type(runtime) # type: ignore[arg-type] - - services: list[VardiffService] = [] - descriptor_write_finished = threading.Event() - - def write_compatibility_field() -> None: - server.idle_retarget_count = 9 - descriptor_write_finished.set() - - with patch( - "lab.prism.prism_coordinator.VardiffService", - side_effect=construct, - ): - threads = [ - threading.Thread( - target=lambda: services.append( - server._ensure_vardiff_service() - ) - ) - for _ in range(2) - ] - threads[0].start() - self.assertTrue(constructor_entered.wait(1.0)) - threads[1].start() - descriptor_writer = threading.Thread(target=write_compatibility_field) - descriptor_writer.start() - time.sleep(0.05) - with calls_lock: - self.assertEqual(calls, 1) - self.assertFalse(descriptor_write_finished.is_set()) - release_constructor.set() - for thread in threads: - thread.join(1.0) - self.assertFalse(thread.is_alive()) - descriptor_writer.join(1.0) - self.assertFalse(descriptor_writer.is_alive()) - - self.assertEqual(len(services), 2) - self.assertIs(services[0], services[1]) - self.assertEqual(services[0].vardiff_idle_queue_depth, 7) - self.assertEqual(services[0].idle_retarget_count, 9) - self.assertNotIn("idle_retarget_count", server.__dict__) - - def test_load_prism_vardiff_config_defaults_to_small_miner_vardiff(self) -> None: - names = [name for name in os.environ if name.startswith("PRISM_STRATUM_VARDIFF")] - with patch.dict(os.environ, {}, clear=False): - for name in names: - os.environ.pop(name, None) - config = load_prism_vardiff_config(Decimal("0.000000001")) - - self.assertTrue(config.enabled) - self.assertEqual(config.target_share_interval_seconds, Decimal("15")) - self.assertEqual(config.min_difficulty, Decimal("1E-9")) - self.assertEqual(config.startup_difficulty, Decimal("1E-9")) - self.assertEqual(config.max_step_factor, Decimal("4")) - self.assertEqual(config.max_step_down_factor, Decimal("4")) - - def test_vardiff_retarget_build_failure_keeps_consistent_difficulty_and_job(self) -> None: - # If the job build is skipped during a retarget, the client must stay on its - # existing job at its existing difficulty -- never advertise a new difficulty - # for a job it never received. Otherwise its easier shares miss the old - # target, nothing is accepted, and (since retargets only fire on accepted - # shares) it cannot self-heal without reconnecting. - server = coordinator() - server.jobs = {"old-job": SimpleNamespace(job=SimpleNamespace(job_id="old-job"))} - server.extranonce2_size = 8 - state = client() - state.username = "miner-a" - state.worker = WorkerIdentity( - username="miner-a", - payout_address="miner-a", - worker_name=None, - script_pubkey_hex="5220" + "11" * 32, - p2mr_program_hex="11" * 32, - ) - state.share_difficulty = Decimal("1") - state.active_job_ids = {"old-job"} - state.vardiff_window_started_monotonic = time.monotonic() - 2 - advertised: list[object] = [] - state.send = lambda payload: advertised.append(payload) # type: ignore[method-assign] - - def failing_build(client: ClientState, *, clean_jobs: bool) -> object: - raise ValueError("transient getblocktemplate failure") - - server.build_job_for_client = failing_build # type: ignore[method-assign] - - server.note_vardiff_submitted_share(state) - server.note_vardiff_accepted_share(state, FakeJob(Decimal("1"))) # type: ignore[arg-type] - - self.assertEqual(server.job_build_failure_count, 1) - self.assertIsNone(state.pending_share_difficulty) # rolled back, not left at the new value - self.assertEqual(state.share_difficulty, Decimal("1")) # unchanged - self.assertEqual(state.active_job_ids, {"old-job"}) # old job retained, still submittable - self.assertEqual(set(server.jobs), {"old-job"}) - self.assertEqual(advertised, []) # no set_difficulty / notify advertised for the skipped build - - def test_idle_vardiff_success_sends_paired_job_and_resets_window(self) -> None: - server = coordinator() - state = client() - prepare_idle_client(server, state) - install_idle_job_cache(server) - sent: list[dict[str, object]] = [] - delivered = threading.Event() - window_started = state.vardiff_window_started_monotonic - - def record(payload: dict[str, object]) -> None: - sent.append(payload) - if payload.get("method") == "mining.notify": - delivered.set() - - state.send = record # type: ignore[method-assign] - - self.assertEqual(server.vardiff_idle_sweep_once(), 1) - self.assertTrue(delivered.wait(timeout=1)) - server.shutdown_vardiff_idle_executor() - - self.assertEqual(server.idle_retarget_count, 1) - self.assertEqual( - [payload["method"] for payload in sent], - ["mining.set_difficulty", "mining.notify"], - ) - self.assertEqual(sent[0]["params"], [4.0]) - self.assertTrue(sent[1]["params"][8]) - self.assertEqual(state.share_difficulty, Decimal("4")) - self.assertIsNone(state.pending_share_difficulty) - self.assertGreater(state.vardiff_window_started_monotonic, window_started) - self.assertEqual(state.vardiff_window_accepted, 0) - self.assertEqual(state.vardiff_window_submitted, 0) - self.assertEqual(state.vardiff_window_work, Decimal("0")) - - def test_idle_vardiff_shutdown_after_delivery_keeps_committed_window(self) -> None: - server = coordinator() - state = client() - prepare_idle_client(server, state) - install_idle_job_cache(server) - sent: list[dict[str, object]] = [] - delivered = threading.Event() - window_started = state.vardiff_window_started_monotonic - - def stop_after_delivery(payload: dict[str, object]) -> None: - sent.append(payload) - if payload.get("method") == "mining.notify": - server.stop_event.set() - delivered.set() - - state.send = stop_after_delivery # type: ignore[method-assign] - - self.assertEqual(server.vardiff_idle_sweep_once(), 1) - self.assertTrue(delivered.wait(timeout=1)) - server.shutdown_vardiff_idle_executor() - - self.assertEqual( - [payload["method"] for payload in sent], - ["mining.set_difficulty", "mining.notify"], - ) - self.assertEqual(server.idle_retarget_count, 1) - self.assertEqual(state.share_difficulty, Decimal("4")) - self.assertIsNone(state.pending_share_difficulty) - self.assertGreater(state.vardiff_window_started_monotonic, window_started) - self.assertEqual(state.vardiff_window_accepted, 0) - self.assertEqual(state.vardiff_window_submitted, 0) - self.assertEqual(state.vardiff_window_work, Decimal("0")) - - def test_idle_vardiff_sweep_skips_submitted_reject_storm_window(self) -> None: - server = coordinator() - state = client() - state.worker = worker_identity() - state.active_job = prism_context("job-1", "00" * 32, worker=state.worker) - state.share_difficulty = Decimal("16") - state.vardiff_window_started_monotonic = time.monotonic() - 2 - state.vardiff_window_submitted = 3 - server.clients = {state} - - def fail_send_job(client: ClientState, *, clean_jobs: bool) -> bool: - raise AssertionError("reject-storm windows must not idle-retarget") - - server.maybe_send_job = fail_send_job # type: ignore[method-assign] - - retargeted = server.vardiff_idle_sweep_once() - - self.assertEqual(retargeted, 0) - self.assertEqual(state.pending_share_difficulty, None) - self.assertEqual(state.vardiff_window_submitted, 3) - - def test_idle_vardiff_share_after_snapshot_is_not_stepped_down(self) -> None: - server = coordinator() - blockers = [client(), client()] - for connection_id, blocker in enumerate(blockers, start=1): - prepare_idle_client(server, blocker, connection_id=connection_id) - install_idle_job_cache(server) - workers_started = threading.Barrier(3) - release_workers = threading.Event() - - def block_delivery(payload: dict[str, object]) -> None: - if payload.get("method") == "mining.set_difficulty": - workers_started.wait(timeout=1) - release_workers.wait(timeout=1) - - for blocker in blockers: - blocker.send = block_delivery # type: ignore[method-assign] - - target = client() - target.connection_id = 3 - sent: list[dict[str, object]] = [] - target.send = sent.append # type: ignore[method-assign] - try: - self.assertEqual(server.vardiff_idle_sweep_once(), 2) - workers_started.wait(timeout=1) - prepare_idle_client(server, target, connection_id=3) - self.assertEqual(server.vardiff_idle_sweep_once(), 1) - - # The task is queued from an idle snapshot. A later submit changes - # that exact window before any worker can commit the step-down. - server.note_vardiff_submitted_share(target) - finally: - release_workers.set() - server.shutdown_vardiff_idle_executor() - - self.assertEqual(sent, []) - self.assertIsNone(target.pending_share_difficulty) - self.assertEqual(target.share_difficulty, Decimal("16")) - self.assertEqual(target.vardiff_window_submitted, 1) - self.assertGreaterEqual(server.vardiff_idle_skip_counts["not_idle"], 1) - - def test_idle_vardiff_failure_restores_pending_and_idle_window(self) -> None: - server = coordinator() - state = client() - prepare_idle_client(server, state) - install_idle_job_cache(server) - window_state = ( - state.vardiff_window_started_monotonic, - state.vardiff_window_accepted, - state.vardiff_window_submitted, - state.vardiff_window_work, - ) - disconnected: list[ClientState] = [] - failure_finished = threading.Event() - - def failing_send(_payload: dict[str, object]) -> None: - self.assertEqual(state.pending_share_difficulty, Decimal("4")) - raise OSError("socket send failed") - - def fake_disconnect(client: ClientState) -> None: - disconnected.append(client) - server.clients.discard(client) - failure_finished.set() - - state.send = failing_send # type: ignore[method-assign] - server.disconnect_client = fake_disconnect # type: ignore[method-assign] - - self.assertEqual(server.vardiff_idle_sweep_once(), 1) - self.assertTrue(failure_finished.wait(timeout=1)) - server.shutdown_vardiff_idle_executor() - - self.assertEqual(disconnected, [state]) - self.assertIsNone(state.pending_share_difficulty) - self.assertEqual(state.share_difficulty, Decimal("16")) - self.assertEqual( - ( - state.vardiff_window_started_monotonic, - state.vardiff_window_accepted, - state.vardiff_window_submitted, - state.vardiff_window_work, - ), - window_state, - ) - self.assertEqual(server.vardiff_idle_task_failures, 1) - - def test_idle_vardiff_stamp_failure_restores_speculative_state(self) -> None: - server = coordinator() - state = client() - prepare_idle_client(server, state) - install_idle_job_cache(server) - window_state = ( - state.vardiff_window_started_monotonic, - state.vardiff_window_accepted, - state.vardiff_window_submitted, - state.vardiff_window_work, - ) - - def fail_stamp(*_args: object, **_kwargs: object) -> None: - self.assertEqual(state.pending_share_difficulty, Decimal("4")) - raise RuntimeError("cached job stamping failed") - - state.send = lambda payload: self.fail( # type: ignore[method-assign] - f"unexpected delivery: {payload}" - ) - server.stamp_job_for_client = fail_stamp # type: ignore[method-assign] - - self.assertEqual(server.vardiff_idle_sweep_once(), 1) - server.shutdown_vardiff_idle_executor() - - self.assertIsNone(state.pending_share_difficulty) - self.assertEqual(state.share_difficulty, Decimal("16")) - self.assertEqual( - ( - state.vardiff_window_started_monotonic, - state.vardiff_window_accepted, - state.vardiff_window_submitted, - state.vardiff_window_work, - ), - window_state, - ) - self.assertEqual(server.job_build_failure_count, 1) - self.assertEqual(server.vardiff_idle_task_failures, 1) - - def test_idle_cached_bundle_requires_live_reorg_trust(self) -> None: - server = coordinator() - state = client() - prepare_idle_client(server, state) - install_idle_job_cache(server) - trust_checked = threading.Event() - sent: list[dict[str, object]] = [] - window_started = state.vardiff_window_started_monotonic - - def reject_untrusted_tip() -> bool: - trust_checked.set() - return False - - state.send = sent.append # type: ignore[method-assign] - server.ensure_reorg_reconciled_for_current_tip = ( # type: ignore[method-assign] - reject_untrusted_tip - ) - - self.assertEqual(server.vardiff_idle_sweep_once(), 1) - server.shutdown_vardiff_idle_executor() - - self.assertTrue(trust_checked.is_set()) - self.assertEqual(sent, []) - self.assertEqual(server.idle_retarget_count, 0) - self.assertEqual(state.share_difficulty, Decimal("16")) - self.assertIsNone(state.pending_share_difficulty) - self.assertEqual(state.vardiff_window_started_monotonic, window_started) - self.assertGreaterEqual(server.vardiff_idle_skip_counts["superseded"], 1) - - def test_idle_retarget_defers_detected_payout_source_during_tip_divergence( - self, - ) -> None: - old_tip = "00" * 32 - new_tip = "11" * 32 - server = coordinator() - state = client() - prepare_idle_client(server, state, tip=old_tip) - install_idle_job_cache(server, tip=old_tip) - with server._job_cache_lock: - published_artifacts = server._template_artifacts - assert published_artifacts is not None - detected_template = gbt_template(new_tip, height=11) - detected_artifacts = CachedTemplateArtifacts( - template=detected_template, - fingerprint=qbit_template_fingerprint(detected_template), - previousblockhash=new_tip, - transaction_hexes=(), - witness_merkle_leaves_hex=(), - network_difficulty=1, - fetched_monotonic=time.monotonic(), - generation=2, - ) - now = time.monotonic() - server.current_tip_first_seen = (old_tip, now) - server.current_tip_observed_monotonic = now - server.latest_detected_tip = (new_tip, 2) - server.tip_refresh_divergence_started_monotonic = now - server.tip_template_snapshot = QbitTipTemplateSnapshot( - bestblockhash=old_tip, - previousblockhash=old_tip, - template_fingerprint=published_artifacts.fingerprint, - template_generation=published_artifacts.generation, - template_artifacts=published_artifacts, - ) - with server._job_cache_lock: - server._template_artifacts = detected_artifacts - server._published_payout_state = dataclass_replace( - server._published_payout_state, - source_tip_hash=new_tip, - ) - window_state = ( - state.vardiff_window_started_monotonic, - state.vardiff_window_accepted, - state.vardiff_window_submitted, - state.vardiff_window_work, - ) - sent: list[dict[str, object]] = [] - - server._build_idle_job_bundle = lambda _request: self.fail( # type: ignore[method-assign] - "idle divergence entered the shared build scheduler" - ) - state.send = sent.append # type: ignore[method-assign] - - self.assertEqual(server.vardiff_idle_sweep_once(), 0) - server.shutdown_vardiff_idle_executor() - - self.assertEqual(sent, []) - self.assertEqual(state.share_difficulty, Decimal("16")) - self.assertIsNone(state.pending_share_difficulty) - self.assertEqual( - ( - state.vardiff_window_started_monotonic, - state.vardiff_window_accepted, - state.vardiff_window_submitted, - state.vardiff_window_work, - ), - window_state, - ) - self.assertEqual(server.vardiff_idle_skip_counts["superseded"], 1) - - def test_replacement_tip_build_survives_repeated_idle_sweeps(self) -> None: - old_tip = "00" * 32 - new_tip = "11" * 32 - server = coordinator() - state = client() - prepare_idle_client(server, state, tip=old_tip) - install_idle_job_cache(server, tip=old_tip) - with server._job_cache_lock: - published_artifacts = server._template_artifacts - assert published_artifacts is not None - detected_template = gbt_template(new_tip, height=11) - detected_artifacts = CachedTemplateArtifacts( - template=detected_template, - fingerprint=qbit_template_fingerprint(detected_template), - previousblockhash=new_tip, - transaction_hexes=(), - witness_merkle_leaves_hex=(), - network_difficulty=1, - fetched_monotonic=time.monotonic(), - generation=2, - ) - now = time.monotonic() - server.current_tip_first_seen = (old_tip, now) - server.current_tip_observed_monotonic = now - server.latest_detected_tip = (new_tip, 2) - server.tip_refresh_divergence_started_monotonic = now - server.tip_template_snapshot = QbitTipTemplateSnapshot( - bestblockhash=old_tip, - previousblockhash=old_tip, - template_fingerprint=published_artifacts.fingerprint, - template_generation=published_artifacts.generation, - template_artifacts=published_artifacts, - ) - with server._job_cache_lock: - server._template_artifacts = detected_artifacts - server._ensure_job_cache_state() - replacement_cancellation = _JobBuildCancellation(timeout_seconds=60.0) - replacement_request = SimpleNamespace( - artifacts=detected_artifacts, - cancellation=replacement_cancellation, - ) - replacement_flight = SimpleNamespace(request=replacement_request) - with server._job_build_scheduler_lock: - server._job_build_active = replacement_flight - build_called = threading.Event() - window_started = state.vardiff_window_started_monotonic - - def unexpected_idle_build(_request: object) -> CachedJobBundle: - build_called.set() - raise AssertionError("idle sweep displaced the replacement build") - - server._build_idle_job_bundle = unexpected_idle_build # type: ignore[method-assign] - - for _sweep in range(4): - self.assertEqual(server.vardiff_idle_sweep_once(), 0) - - # Close the race where a worker passed its first divergence check just - # before detection. Scheduler admission must reject that idle request - # without superseding the active replacement build. - racing_idle_cancellation = _JobBuildCancellation(timeout_seconds=60.0) - racing_idle_request = SimpleNamespace( - idle_retarget=True, - cancellation=racing_idle_cancellation, - promise=Future(), - ) - racing_idle_promise = server._request_job_build(racing_idle_request) # type: ignore[arg-type] - with self.assertRaises(JobBuildCancelled): - racing_idle_promise.result() - - with server._job_build_scheduler_lock: - self.assertIs(server._job_build_active, replacement_flight) - self.assertFalse(replacement_cancellation.is_set()) - self.assertTrue(racing_idle_cancellation.is_set()) - self.assertFalse(build_called.is_set()) - self.assertEqual(state.vardiff_window_started_monotonic, window_started) - self.assertEqual(server.vardiff_idle_skip_counts["superseded"], 4) - with server._job_build_scheduler_lock: - server._job_build_active = None - server.shutdown_vardiff_idle_executor() - - def test_idle_shared_build_does_not_retry_scheduler_divergence_race( - self, - ) -> None: - old_tip = "00" * 32 - new_tip = "11" * 32 - server = coordinator() - state = client() - prepare_idle_client(server, state, tip=old_tip) - install_idle_job_cache(server, tip=old_tip) - with server._job_cache_lock: - old_artifacts = server._template_artifacts - server._job_bundle_cache.clear() - assert old_artifacts is not None - now = time.monotonic() - server.current_tip_first_seen = (old_tip, now) - server.current_tip_observed_monotonic = now - server.latest_detected_tip = (old_tip, 1) - server.tip_template_snapshot = QbitTipTemplateSnapshot( - bestblockhash=old_tip, - previousblockhash=old_tip, - template_fingerprint=old_artifacts.fingerprint, - template_generation=old_artifacts.generation, - template_artifacts=old_artifacts, - ) - server._ensure_job_cache_state() - replacement_cancellation = _JobBuildCancellation(timeout_seconds=60.0) - replacement_flight = SimpleNamespace( - request=SimpleNamespace( - artifacts=SimpleNamespace(previousblockhash=new_tip), - cancellation=replacement_cancellation, - ) - ) - with server._job_build_scheduler_lock: - server._job_build_active = replacement_flight - request_builds = 0 - admission_attempts = 0 - idle_cancellation: _JobBuildCancellation | None = None - original_request_job_build = server._request_job_build - - def make_idle_request( - _artifacts: CachedTemplateArtifacts, - _worker: WorkerIdentity | None, - **kwargs: object, - ) -> object: - nonlocal request_builds, idle_cancellation - request_builds += 1 - self.assertTrue(kwargs["idle_retarget"]) - idle_cancellation = _JobBuildCancellation(timeout_seconds=60.0) - return SimpleNamespace( - idle_retarget=True, - cancellation=idle_cancellation, - promise=Future(), - ) - - def detect_before_admission(request: object) -> Future[CachedJobBundle]: - nonlocal admission_attempts - admission_attempts += 1 - with server.lock: - server.latest_detected_tip = (new_tip, 2) - server.tip_refresh_divergence_started_monotonic = time.monotonic() - return original_request_job_build(request) # type: ignore[arg-type] - - server._new_job_build_request = make_idle_request # type: ignore[method-assign] - server._request_job_build = detect_before_admission # type: ignore[method-assign] - server._schedule_tip_refresh_retry = lambda: None # type: ignore[method-assign] - assert state.worker is not None - - with self.assertRaises(JobBuildSuperseded): - server._build_idle_job_bundle(SimpleNamespace(worker=state.worker)) # type: ignore[arg-type] - - self.assertEqual(request_builds, 1) - self.assertEqual(admission_attempts, 1) - self.assertIs(server._template_artifacts, old_artifacts) - self.assertIsNotNone(idle_cancellation) - assert idle_cancellation is not None - self.assertTrue(idle_cancellation.is_set()) - self.assertFalse(replacement_cancellation.is_set()) - with server._job_build_scheduler_lock: - self.assertIs(server._job_build_active, replacement_flight) - server._job_build_active = None - server.shutdown_vardiff_idle_executor() - - def test_idle_cached_collection_bundle_refreshes_readiness(self) -> None: - server = coordinator() - state = client() - prepare_idle_client(server, state) - ready_bundle = install_idle_job_cache(server) - assert state.worker is not None - with server._job_cache_lock: - artifacts = server._template_artifacts - assert artifacts is not None - collection_key = server._job_bundle_key( - artifacts, - mode="collection", - payout_state_generation=server._payout_state_generation, - payout_artifact_generation=0, - worker=state.worker, - ) - collection_bundle = dataclass_replace( - ready_bundle, - key=collection_key, - collection_only=True, - collection_identity=( - state.worker.payout_address, - state.worker.p2mr_program_hex, - ), - ) - server._job_bundle_cache.clear() - server._job_bundle_cache[collection_key] = collection_bundle - server._pool_ready_latched = False - server.min_ready_miners = 3 - server.accepted_share_stats = lambda: (3, 3) # type: ignore[method-assign] - rebuilt = threading.Event() - sent: list[dict[str, object]] = [] - - def build_ready(_request: object) -> CachedJobBundle: - rebuilt.set() - with server._job_cache_lock: - server._job_bundle_cache[ready_bundle.key] = ready_bundle - return ready_bundle - - state.send = sent.append # type: ignore[method-assign] - server._build_idle_job_bundle = build_ready # type: ignore[method-assign] - - self.assertEqual(server.vardiff_idle_sweep_once(), 1) - server.shutdown_vardiff_idle_executor() - - self.assertTrue(server._pool_ready_latched) - self.assertTrue(rebuilt.is_set()) - self.assertEqual( - [payload["method"] for payload in sent], - ["mining.set_difficulty", "mining.notify"], - ) - self.assertIsNotNone(state.active_job) - self.assertFalse(state.active_job.collection_only) - - def test_idle_cached_ready_bundle_rebinds_same_tip_observation(self) -> None: - server = coordinator() - state = client() - prepare_idle_client(server, state) - cached = install_idle_job_cache(server) - updated_template = dict(cached.template) - updated_template["curtime"] = int(updated_template["curtime"]) + 30 - current = server.store_template_artifacts(updated_template, generation=2) - assert current is not None - delivered = threading.Event() - rebound = threading.Event() - sent: list[dict[str, object]] = [] - - def bind_current_observation( - bundle: CachedJobBundle, - artifacts: CachedTemplateArtifacts, - ) -> CachedJobBundle: - rebound.set() - return dataclass_replace( - bundle, - template=artifacts.template, - base_job=dataclass_replace( - bundle.base_job, - ntime=f'{artifacts.template["curtime"]:08x}', - ), - template_generation=artifacts.generation, - ) - - def record(payload: dict[str, object]) -> None: - sent.append(payload) - if payload.get("method") == "mining.notify": - delivered.set() - - state.send = record # type: ignore[method-assign] - server._bind_cached_bundle_to_artifacts = ( # type: ignore[method-assign] - bind_current_observation - ) - - self.assertEqual(server.vardiff_idle_sweep_once(), 1) - self.assertTrue(delivered.wait(timeout=1)) - server.shutdown_vardiff_idle_executor() - - self.assertIsNotNone(state.active_job) - self.assertTrue(rebound.is_set()) - self.assertIs(state.active_job.template, current.template) - self.assertEqual(state.active_job.template_generation, current.generation) - expected_ntime = f'{updated_template["curtime"]:08x}' - self.assertEqual(state.active_job.job.ntime, expected_ntime) - notify = next( - payload - for payload in sent - if payload.get("method") == "mining.notify" - ) - self.assertEqual(notify["params"][7], expected_ntime) - - def test_repeated_idle_sweeps_do_not_enqueue_duplicate_connection_work(self) -> None: - server = coordinator() - state = client() - prepare_idle_client(server, state) - install_idle_job_cache(server) - delivery_started = threading.Event() - release_delivery = threading.Event() - sent: list[dict[str, object]] = [] - - def blocking_send(payload: dict[str, object]) -> None: - if payload.get("method") == "mining.set_difficulty": - delivery_started.set() - release_delivery.wait(timeout=1) - sent.append(payload) - - state.send = blocking_send # type: ignore[method-assign] - try: - self.assertEqual(server.vardiff_idle_sweep_once(), 1) - self.assertTrue(delivery_started.wait(timeout=1)) - self.assertEqual(server.vardiff_idle_sweep_once(), 0) - with server._vardiff_idle_lock: - self.assertEqual(len(server._vardiff_idle_pending), 1) - finally: - release_delivery.set() - server.shutdown_vardiff_idle_executor() - - self.assertEqual( - [payload["method"] for payload in sent], - ["mining.set_difficulty", "mining.notify"], - ) - self.assertGreaterEqual( - server.vardiff_idle_skip_counts["superseded"] - + server.vardiff_idle_skip_counts["not_idle"], - 1, - ) - - def test_idle_sweep_skips_busy_client_lock_and_returns_promptly(self) -> None: - server = coordinator() - state = client() - prepare_idle_client(server, state) - install_idle_job_cache(server) - lock_held = threading.Event() - release_lock = threading.Event() - - def hold_client_lock() -> None: - state.job_update_lock.acquire() - try: - lock_held.set() - release_lock.wait(timeout=1) - finally: - state.job_update_lock.release() - - holder = threading.Thread(target=hold_client_lock) - holder.start() - self.assertTrue(lock_held.wait(timeout=1)) - started = time.monotonic() - try: - self.assertEqual(server.vardiff_idle_sweep_once(), 0) - finally: - elapsed = time.monotonic() - started - release_lock.set() - holder.join(timeout=1) - server.shutdown_vardiff_idle_executor() - - self.assertLess(elapsed, 0.25) - self.assertEqual(server.vardiff_idle_skip_counts["busy"], 1) - - def test_stuck_bundle_builder_does_not_stale_idle_sweep_heartbeat(self) -> None: - server = coordinator() - state = client() - prepare_idle_client(server, state) - bundle = install_idle_job_cache(server) - with server._job_cache_lock: - server._job_bundle_cache.clear() - state.send = lambda _payload: None # type: ignore[method-assign] - build_started = threading.Event() - release_build = threading.Event() - - def blocked_build( - *_args: object, - **_kwargs: object, - ) -> CachedJobBundle: - build_started.set() - if not release_build.wait(timeout=1): - raise AssertionError("idle retarget bundle build was not released") - return bundle - - server.build_shared_job_bundle = blocked_build # type: ignore[method-assign] - server._record_heartbeat("vardiff_idle_sweep") - heartbeat_before = server._heartbeats["vardiff_idle_sweep"] - started = time.monotonic() - try: - self.assertEqual(server.vardiff_idle_sweep_once(), 1) - elapsed = time.monotonic() - started - self.assertTrue(build_started.wait(timeout=0.25)) - self.assertLess(elapsed, 0.25) - self.assertGreater( - server._heartbeats["vardiff_idle_sweep"], - heartbeat_before, - ) - finally: - release_build.set() - server.shutdown_vardiff_idle_executor() - - self.assertEqual(server.vardiff_idle_skip_counts["cache_miss"], 1) - - def test_idle_sweep_cache_miss_builds_only_on_bounded_worker(self) -> None: - server = coordinator() - state = client() - prepare_idle_client(server, state) - bundle = install_idle_job_cache(server) - server.job_bundle_cache_seconds = 10.0 - expired_bundle = dataclass_replace( - bundle, - built_monotonic=time.monotonic() - 11.0, - ) - with server._job_cache_lock: - server._job_bundle_cache[bundle.key] = expired_bundle - build_started = threading.Event() - release_build = threading.Event() - build_thread_ids: list[int] = [] - sent: list[dict[str, object]] = [] - - def blocked_build( - *_args: object, - **_kwargs: object, - ) -> CachedJobBundle: - build_thread_ids.append(threading.get_ident()) - build_started.set() - if not release_build.wait(timeout=1): - raise AssertionError("idle retarget bundle build was not released") - return bundle - - state.send = sent.append # type: ignore[method-assign] - server.build_shared_job_bundle = blocked_build # type: ignore[method-assign] - sweep_thread_id = threading.get_ident() - started = time.monotonic() - try: - self.assertEqual(server.vardiff_idle_sweep_once(), 1) - elapsed = time.monotonic() - started - self.assertTrue(build_started.wait(timeout=0.25)) - self.assertLess(elapsed, 0.25) - self.assertEqual(server.vardiff_idle_sweep_once(), 0) - self.assertEqual(len(build_thread_ids), 1) - self.assertNotEqual(build_thread_ids[0], sweep_thread_id) - with server._vardiff_idle_lock: - self.assertEqual(len(server._vardiff_idle_pending), 1) - finally: - release_build.set() - server.shutdown_vardiff_idle_executor() - - self.assertEqual(server.vardiff_idle_skip_counts["cache_miss"], 1) - self.assertEqual( - [payload["method"] for payload in sent], - ["mining.set_difficulty", "mining.notify"], - ) - self.assertEqual(server.idle_retarget_count, 1) - self.assertIsNone(state.pending_share_difficulty) - self.assertEqual(state.share_difficulty, Decimal("4")) - self.assertEqual(state.vardiff_window_accepted, 0) - self.assertEqual(state.vardiff_window_submitted, 0) - self.assertEqual(state.vardiff_window_work, Decimal("0")) - - def test_idle_retarget_delivers_fresh_bundle_when_cache_is_disabled( - self, - ) -> None: - server = coordinator() - state = client() - prepare_idle_client(server, state) - bundle = install_idle_job_cache(server) - server.job_bundle_cache_seconds = 0.0 - with server._job_cache_lock: - server._job_bundle_cache.clear() - built = threading.Event() - sent: list[dict[str, object]] = [] - - def build_uncached(_request: object) -> CachedJobBundle: - built.set() - return bundle - - server._build_idle_job_bundle = build_uncached # type: ignore[method-assign] - state.send = sent.append # type: ignore[method-assign] - - self.assertEqual(server.vardiff_idle_sweep_once(), 1) - server.shutdown_vardiff_idle_executor() - - self.assertTrue(built.is_set()) - self.assertEqual(server.vardiff_idle_skip_counts["cache_miss"], 1) - self.assertEqual( - [payload["method"] for payload in sent], - ["mining.set_difficulty", "mining.notify"], - ) - self.assertEqual(server.idle_retarget_count, 1) - self.assertEqual(state.share_difficulty, Decimal("4")) - self.assertIsNone(state.pending_share_difficulty) - with server._job_cache_lock: - self.assertEqual(server._job_bundle_cache, {}) - - def test_idle_preparation_oserror_keeps_client_connected(self) -> None: - for failure_phase in ("bundle", "reorg"): - with self.subTest(failure_phase=failure_phase): - server = coordinator() - state = client() - prepare_idle_client(server, state) - install_idle_job_cache(server) - disconnected: list[ClientState] = [] - window_started = state.vardiff_window_started_monotonic - - def fail_bundle(_request: object) -> CachedJobBundle: - raise OSError("qbit RPC transport unavailable") - - def fail_reorg() -> bool: - raise OSError("qbit trust RPC transport unavailable") - - def unexpected_send(payload: dict[str, object]) -> None: - self.fail(f"unexpected idle delivery: {payload}") - - def record_disconnect(client_state: ClientState) -> None: - disconnected.append(client_state) - - if failure_phase == "bundle": - server._build_idle_job_bundle = ( # type: ignore[method-assign] - fail_bundle - ) - else: - server.ensure_reorg_reconciled_for_current_tip = ( # type: ignore[method-assign] - fail_reorg - ) - state.send = unexpected_send # type: ignore[method-assign] - server.disconnect_client = record_disconnect # type: ignore[method-assign] - - self.assertEqual(server.vardiff_idle_sweep_once(), 1) - server.shutdown_vardiff_idle_executor() - - self.assertEqual(disconnected, []) - self.assertIn(state, server.clients) - self.assertFalse(state.closing) - self.assertEqual(server.vardiff_idle_task_failures, 1) - self.assertIsNone(state.pending_share_difficulty) - self.assertEqual(state.share_difficulty, Decimal("16")) - self.assertEqual( - state.vardiff_window_started_monotonic, - window_started, - ) - - def test_disconnect_while_idle_retarget_pending_prevents_delivery(self) -> None: - server = coordinator() - blockers = [client(), client()] - for connection_id, blocker in enumerate(blockers, start=1): - prepare_idle_client(server, blocker, connection_id=connection_id) - install_idle_job_cache(server) - workers_started = threading.Barrier(3) - release_workers = threading.Event() - - def block_delivery(payload: dict[str, object]) -> None: - if payload.get("method") == "mining.set_difficulty": - workers_started.wait(timeout=1) - release_workers.wait(timeout=1) - - for blocker in blockers: - blocker.send = block_delivery # type: ignore[method-assign] - - target = client() - target_sent: list[dict[str, object]] = [] - target.send = target_sent.append # type: ignore[method-assign] - target.close = lambda: None # type: ignore[method-assign] - disconnected_skip = threading.Event() - record_skip = server._record_vardiff_idle_skip - - def record_and_signal(reason: str) -> None: - record_skip(reason) - if reason == "disconnected": - disconnected_skip.set() - - server._record_vardiff_idle_skip = record_and_signal # type: ignore[method-assign] - try: - self.assertEqual(server.vardiff_idle_sweep_once(), 2) - workers_started.wait(timeout=1) - prepare_idle_client(server, target, connection_id=3) - self.assertEqual(server.vardiff_idle_sweep_once(), 1) - server.disconnect_client(target) - finally: - release_workers.set() - try: - self.assertTrue(disconnected_skip.wait(timeout=1)) - finally: - server.shutdown_vardiff_idle_executor() - - self.assertEqual(target_sent, []) - self.assertTrue(target.closing) - self.assertNotIn(target, server.clients) - self.assertGreaterEqual(server.vardiff_idle_skip_counts["disconnected"], 1) - - def test_hundreds_of_busy_and_dead_clients_cannot_stall_idle_sweep(self) -> None: - server = coordinator() - - class BusyLock: - def acquire(self, blocking: bool = True) -> bool: - self.assert_nonblocking = blocking - return False - - def release(self) -> None: - raise AssertionError("unacquired busy lock released") - - for connection_id in range(1, 201): - state = client() - prepare_idle_client(server, state, connection_id=connection_id) - state.job_update_lock = BusyLock() # type: ignore[assignment] - for connection_id in range(201, 401): - state = client() - prepare_idle_client(server, state, connection_id=connection_id) - state.closing = True - - server.vardiff_idle_sweep_seconds = 0.5 - started = time.monotonic() - self.assertEqual(server.vardiff_idle_sweep_once(), 0) - elapsed = time.monotonic() - started - server.shutdown_vardiff_idle_executor() - - self.assertLess(elapsed, server.vardiff_idle_sweep_seconds) - self.assertEqual(server.vardiff_idle_clients_inspected, 400) - self.assertEqual(server.vardiff_idle_skip_counts["busy"], 200) - self.assertEqual(server.vardiff_idle_skip_counts["disconnected"], 200) - - def test_idle_retarget_queue_is_globally_bounded(self) -> None: - server = coordinator() - states = [client() for _ in range(9)] - for connection_id, state in enumerate(states, start=1): - prepare_idle_client(server, state, connection_id=connection_id) - install_idle_job_cache(server) - release_workers = threading.Event() - two_workers_started = threading.Event() - started_lock = threading.Lock() - started_count = 0 - - def block_delivery(payload: dict[str, object]) -> None: - nonlocal started_count - if payload.get("method") != "mining.set_difficulty": - return - with started_lock: - started_count += 1 - if started_count == 2: - two_workers_started.set() - release_workers.wait(timeout=1) - - for state in states: - state.send = block_delivery # type: ignore[method-assign] - - try: - self.assertEqual(server.vardiff_idle_sweep_once(), 8) - self.assertTrue(two_workers_started.wait(timeout=1)) - with server._vardiff_idle_lock: - self.assertEqual(len(server._vardiff_idle_pending), 8) - self.assertEqual(server.vardiff_idle_inflight, 2) - self.assertEqual(server.vardiff_idle_queue_depth, 6) - self.assertEqual(server.vardiff_idle_skip_counts["queue_full"], 1) - finally: - release_workers.set() - server.shutdown_vardiff_idle_executor() - - def test_maybe_send_job_isolates_build_failure_and_keeps_client_connected(self) -> None: - server = coordinator() - server.jobs = {} - server.extranonce2_size = 8 - state = client() - state.username = "miner-a" - state.worker = WorkerIdentity( - username="miner-a", - payout_address="miner-a", - worker_name=None, - script_pubkey_hex="5220" + "11" * 32, - p2mr_program_hex="11" * 32, - ) - sent: list[object] = [] - state.send = lambda payload: sent.append(payload) # type: ignore[method-assign] - - def boom(client: ClientState, *, clean_jobs: bool) -> None: - raise ValueError( - "full coinbase transaction does not end its coinbase scriptSig " - "with the extranonce placeholder" - ) - - server.build_job_for_client = boom # type: ignore[method-assign] - - # The bug: this used to propagate out of handle_client and drop the miner. - # It must now be swallowed so the client thread survives a single bad template. - server.maybe_send_job(state, clean_jobs=True) - - self.assertEqual(server.job_build_failure_count, 1) - self.assertEqual(state.active_job_ids, set()) - self.assertEqual(server.jobs, {}) - self.assertEqual(sent, []) # no difficulty / mining.notify pushed for the failed build - - # A subsequent good template still issues a job (skip, do not permanently break). - server.build_job_for_client = lambda client, *, clean_jobs: SimpleNamespace( # type: ignore[method-assign] - job=SimpleNamespace( - job_id="job-ok", - share_difficulty=Decimal("1"), - share_target=target_from_compact("207fffff"), - ), - template={"previousblockhash": "00" * 32}, - collection_only=False, - ) - server.send_difficulty = lambda client, job: None # type: ignore[method-assign] - server.send_job = lambda client, job: sent.append("notify") # type: ignore[method-assign] - server.apply_job_difficulty = lambda client, job: None # type: ignore[method-assign] - - server.maybe_send_job(state, clean_jobs=True) - - self.assertEqual(server.job_build_failure_count, 1) - self.assertEqual(state.active_job_ids, {"job-ok"}) - self.assertEqual(sent, ["notify"]) - - def test_maybe_send_job_does_not_swallow_send_failures_as_build_failures(self) -> None: - # Only the job build is isolated. A Stratum send failure (a dead socket) - # must propagate so handle_client disconnects and cleans up, rather than - # being miscounted as a build failure or leaving the client wedged. - server = coordinator() - server.jobs = {} - server.extranonce2_size = 8 - state = client() - state.username = "miner-a" - state.worker = WorkerIdentity( - username="miner-a", - payout_address="miner-a", - worker_name=None, - script_pubkey_hex="5220" + "11" * 32, - p2mr_program_hex="11" * 32, - ) - - server.build_job_for_client = lambda client, *, clean_jobs: SimpleNamespace( # type: ignore[method-assign] - job=SimpleNamespace( - job_id="job-dead", - share_difficulty=Decimal("1"), - share_target=target_from_compact("207fffff"), - ), - collection_only=False, - ) - server.send_difficulty = lambda client, job: None # type: ignore[method-assign] - - def dead_socket(client: ClientState, job: object) -> None: - raise OSError("broken pipe") - - server.send_job = dead_socket # type: ignore[method-assign] - - with self.assertRaises(OSError): - server.maybe_send_job(state, clean_jobs=True) - - # The send failure is not a build failure, and handle_client (not us) owns - # the disconnect/cleanup of the registered job for the dead connection. - self.assertEqual(server.job_build_failure_count, 0) - - def _pending_append(self, tag: str, accepted_at_ms: int = 2) -> PendingShareAppend: - from lab.prism.share_ledger import PendingShare - - return PendingShareAppend( - pending_share=PendingShare( - share_id=f"miner-a:{tag}", - miner_id="miner-a", - order_key="miner-a", - p2mr_program_hex="11" * 32, - share_difficulty=1, - network_difficulty=1, - template_height=10, - job_id="job-1", - job_issued_at_ms=1, - accepted_at_ms=accepted_at_ms, - ntime=1_700_000_000, - ), - username="miner-a", - job_id="job-1", - block_hash_hex=tag * 32, - collection_only=False, - credit_policy=None, - ) - - def test_vardiff_step_up_retarget_sends_new_difficulty_without_flush(self) -> None: - server = coordinator() - state = client() - state.share_difficulty = Decimal("1") - state.vardiff_window_started_monotonic = time.monotonic() - 2 - sent: dict[str, object] = {"jobs": 0} - - def fake_send_job(client: object, clean_jobs: bool) -> bool: - sent.update({"jobs": sent["jobs"] + 1, "clean": clean_jobs}) - return True - - server.maybe_send_job = fake_send_job # type: ignore[method-assign] - - server.note_vardiff_submitted_share(state) - server.note_vardiff_accepted_share(state, FakeJob(Decimal("1"))) # type: ignore[arg-type] - - # Difficulty is advertised by maybe_send_job alongside the job (gated on - # a successful build). A step-up must not flush the miner's in-flight - # work: old jobs validate at their own stamped share_target, so the - # retarget requests a single non-clean job. - self.assertEqual(state.pending_share_difficulty, Decimal("4")) - self.assertEqual(sent["jobs"], 1) - self.assertFalse(sent["clean"]) - - def test_vardiff_step_down_retarget_flushes_with_clean_job(self) -> None: - # Firmware that applies mining.set_difficulty retroactively would - # submit sub-target shares against the old job after a step-down, so - # the step-down keeps flushing in-flight work. - server = coordinator() - state = client() - state.share_difficulty = Decimal("16") - state.vardiff_window_started_monotonic = time.monotonic() - 65 - sent: dict[str, object] = {"jobs": 0} - - def fake_send_job(client: object, clean_jobs: bool) -> bool: - sent.update({"jobs": sent["jobs"] + 1, "clean": clean_jobs}) - return True - - server.maybe_send_job = fake_send_job # type: ignore[method-assign] - - server.note_vardiff_submitted_share(state) - server.note_vardiff_accepted_share(state, FakeJob(Decimal("16"))) # type: ignore[arg-type] - - self.assertEqual(state.pending_share_difficulty, Decimal("4")) - self.assertEqual(sent["jobs"], 1) - self.assertTrue(sent["clean"]) - - def test_step_up_retarget_keeps_old_job_submittable_at_old_difficulty(self) -> None: - tip = "00" * 32 - server = coordinator() - server.accepted_block_count = 0 - server.max_blocks = 1 - server.stop_after_block = True - server.jobs = {} - server.recent_share_keys = set() - server.share_weights_by_username = {} - ledger = RecordingLedger() - server.ledger = ledger - worker = worker_identity() - state = client() - state.username = worker.username - state.worker = worker - state.share_difficulty = Decimal("1") - state.vardiff_window_started_monotonic = time.monotonic() - 2 - sent: list[dict[str, object]] = [] - state.send = lambda payload: sent.append(payload) # type: ignore[method-assign] - server.clients = {state} - server.rpc = TipTemplateRpc(tip=tip, template=gbt_template(tip)) - - old_context = prism_context("old-job", tip, worker=worker, difficulty=Decimal("1")) - state.active_job = old_context - state.active_job_ids = {"old-job"} - server.jobs["old-job"] = old_context - - def build_fresh_job(client: ClientState, *, clean_jobs: bool) -> object: - return prism_context( - "fresh-job", - tip, - worker=worker, - difficulty=client.pending_share_difficulty or client.share_difficulty, - clean_jobs=clean_jobs, - ) - - server.build_job_for_client = build_fresh_job # type: ignore[method-assign] - - server.note_vardiff_submitted_share(state) - server.note_vardiff_accepted_share(state, FakeJob(Decimal("1"))) # type: ignore[arg-type] - - # The 1 -> 4 step-up pairs the new difficulty with a non-clean job and - # keeps the old job registered and submittable. - self.assertEqual(state.share_difficulty, Decimal("4")) - self.assertIsNone(state.pending_share_difficulty) - self.assertEqual( - [payload["method"] for payload in sent], - ["mining.set_difficulty", "mining.notify"], - ) - self.assertEqual(sent[0]["params"], [4.0]) - self.assertEqual(sent[1]["params"][0], "fresh-job") - self.assertFalse(sent[1]["params"][8]) - self.assertIn("old-job", server.jobs) - self.assertIn("fresh-job", server.jobs) - self.assertEqual(state.active_job_ids, {"old-job", "fresh-job"}) - - # In-flight work against the old job still lands, validated at the old - # job's own stamped difficulty: this share passes the diff-1 target it - # was mined against, not the diff-4 target advertised afterward. - def assemble_at_stamped_difficulty(job: object, **_kwargs: object) -> SimpleNamespace: - return SimpleNamespace( - header_hex="aa" * 80, - block_hash_hex="bb" * 32, - share_pass=job.share_difficulty <= Decimal("1"), - block_pass=False, - ) - - with patch( - "lab.prism.prism_coordinator.direct_stratum.assemble_submission", - side_effect=assemble_at_stamped_difficulty, - ): - should_close = server.handle_submit( - state, - ["miner-a", "old-job", "00" * 8, "00000001", "00000002"], - ) - - self.assertFalse(should_close) - self.assertEqual(len(ledger.pending), 1) - self.assertEqual(ledger.pending[0].job_id, "old-job") - self.assertEqual(server.low_difficulty_share_count, 0) - self.assertEqual(server.rejection_counts_by_reason[PRISM_REJECTION_LOW_DIFFICULTY], 0) - self.assertEqual(server.rejection_counts_by_reason[PRISM_REJECTION_UNKNOWN_JOB], 0) - self.assertEqual(server.rejection_counts_by_reason[PRISM_REJECTION_STALE_JOB], 0) - - -class PrismStampedJobFloorTests(unittest.TestCase): - """The listener floor must hold on the wire, not just in vardiff policy. - - Stamped jobs are the single choke point for every mining.set_difficulty - the coordinator sends, and marketplace verification judges the first one. - The regression here is a young chain: qbit network difficulty below the - high-diff floor used to drag the advertised difficulty down with it. - """ - - def stamp_coordinator(self) -> PrismCoordinator: - server = coordinator() - server.job_counter = 0 - server.share_weights_by_username = {} - server.default_share_weight = 1 - return server - - def cached_bundle(self) -> CachedJobBundle: - # bits 207fffff: regtest-grade network difficulty (~4.7e-10), far - # below the 500k marketplace floor. - qbit_target = target_from_compact("207fffff") - base_job = direct_stratum.DirectQbitStratumJob( - job_id="prism-template-base", - previousblockhash_display="00" * 32, - prevhash="00" * 32, - coinb1="", - coinb2="", - full_coinbase_prefix="", - full_coinbase_suffix="", - merkle_branch=(), - transaction_hexes=(), - version="20000000", - nbits="207fffff", - ntime="6553f100", - qbit_target=qbit_target, - share_target=qbit_target, - share_difficulty=Decimal("1"), - extranonce1_hex="ffffffff", - extranonce2_size=8, - clean_jobs=True, - ) - return CachedJobBundle( - key=("test",), - template=gbt_template("00" * 32), - template_fingerprint="fp", - coinbase_manifest={}, - shares_json=[], - prior_balances=[], - found_block={"network_difficulty": 1}, - collection_only=False, - issued_at_ms=12345, - base_job=base_job, - built_monotonic=time.monotonic(), - ) - - def highdiff_client(self) -> ClientState: - state = client() - state.worker = worker_identity() - state.listener_vardiff_config = highdiff_vardiff_config() - state.minimum_advertised_difficulty = Decimal("500000") - state.share_difficulty = Decimal("500000") - return state - - def test_stamped_job_enforces_floor_below_network_difficulty(self) -> None: - server = self.stamp_coordinator() - state = self.highdiff_client() - - context = server.stamp_job_for_client(state, self.cached_bundle(), clean_jobs=True) - - self.assertEqual( - context.job.share_target, - direct_stratum.difficulty_target(Decimal("500000")), - ) - # Decimal round-tripping can land within 1e-27 of the floor; the wire - # value is float(difficulty), which is what marketplaces judge. - self.assertGreaterEqual(float(context.job.share_difficulty), 500000.0) - - def test_stamped_job_keeps_network_cap_without_listener_floor(self) -> None: - server = self.stamp_coordinator() - state = client() - state.worker = worker_identity() - # Even an absurd desired difficulty stays capped at the network - # target on the default listener: shares are never required to be - # harder than blocks there. - state.share_difficulty = Decimal("500000") - - context = server.stamp_job_for_client(state, self.cached_bundle(), clean_jobs=True) - - self.assertEqual(context.job.share_target, target_from_compact("207fffff")) - self.assertLess(context.job.share_difficulty, Decimal("1")) - - def test_stamped_job_honors_md_raised_floor_on_highdiff_listener(self) -> None: - server = self.stamp_coordinator() - state = self.highdiff_client() - state.requested_min_difficulty = Decimal("2000000") - server.apply_client_difficulty_requests(state) - - context = server.stamp_job_for_client(state, self.cached_bundle(), clean_jobs=True) - - self.assertEqual( - context.job.share_target, - direct_stratum.difficulty_target(Decimal("2000000")), - ) - self.assertGreaterEqual(float(context.job.share_difficulty), 2000000.0) diff --git a/tests/test_prism_vardiff_durable_resume.py b/tests/test_prism_vardiff_durable_resume.py deleted file mode 100644 index 7f463df96..000000000 --- a/tests/test_prism_vardiff_durable_resume.py +++ /dev/null @@ -1,1045 +0,0 @@ -#!/usr/bin/env python3 -"""Durable worker-difficulty wiring at the vardiff ownership boundary.""" - -from __future__ import annotations - -from decimal import Decimal -import inspect -from itertools import product -from types import SimpleNamespace -import threading -import time -import unittest -from unittest import mock - -from lab.auxpow import vardiff -from lab.prism.prism_coordinator import PrismCoordinator -from lab.prism.stratum_session import ClientState -from lab.prism.vardiff_service import ( - MAX_PENDING_VARDIFF_DURABLE_WRITES, - PRISM_VARDIFF_DURABLE_PRUNE_BATCH, - PRISM_VARDIFF_DURABLE_PRUNE_MAX_BATCHES, - VardiffService, - _PendingDurableWrite, -) -from lab.prism.worker_difficulty_store import ( - MemoryWorkerDifficultyStore, - PostgresWorkerDifficultyStore, -) -from tests.prism_vardiff_test_support import worker_identity - - -def lane_config() -> vardiff.VardiffConfig: - return vardiff.VardiffConfig( - enabled=True, - target_share_interval_seconds=Decimal("15"), - min_difficulty=Decimal("1024"), - max_difficulty=Decimal("4294967296"), - retarget_interval_seconds=Decimal("300"), - max_step_factor=Decimal("4"), - startup_difficulty=Decimal("16384"), - max_step_down_factor=Decimal("4"), - ) - - -def runtime( - store: MemoryWorkerDifficultyStore, - *, - ttl_seconds: float = 900.0, - enabled: bool = True, - max_entries: int = 32, -) -> SimpleNamespace: - return SimpleNamespace( - worker_difficulty_store=store, - vardiff_resume_enabled=enabled, - vardiff_resume_ttl_seconds=ttl_seconds, - vardiff_resume_max_entries=max_entries, - vardiff_resume_max_start_factor=Decimal("1024"), - vardiff_config=lane_config(), - share_difficulty=Decimal("16384"), - lock=threading.RLock(), - clients=set(), - ) - - -def client( - *, - listener: str = "default", - username: str = "miner-a.rig-1", - difficulty: Decimal = Decimal("16384"), -) -> ClientState: - state = ClientState( - sock=object(), - address=("127.0.0.1", 1), - connection_id=1, - extranonce1_hex="00000001", - listener_name=listener, - listener_vardiff_config=lane_config(), - share_difficulty=difficulty, - ) - state.worker = worker_identity(username) - state.username = username - return state - - -class DurableVardiffResumeTests(unittest.TestCase): - def service(self, runtime_namespace: SimpleNamespace) -> VardiffService: - """A service whose persistence lane is closed when the test ends.""" - built = VardiffService(runtime_namespace) - self.addCleanup(built.shutdown_durable_executor) - return built - - def drained(self, built: VardiffService) -> VardiffService: - """Wait out the asynchronous persistence lane. - - Durable writes are deliberately fire-and-forget so mining never waits - on them; tests that assert on the store have to say so explicitly. - """ - self.assertTrue(built.flush_durable_writes(timeout=5.0)) - return built - - def test_durable_cache_is_loaded_before_any_listener_accepts(self) -> None: - source = inspect.getsource(PrismCoordinator._serve_with_listener_stack) - preload = source.index("self._ensure_vardiff_service()") - - self.assertLess(preload, source.index("self._start_secondary_accept_service")) - self.assertLess(preload, source.index("self.accept_loop")) - - def test_share_backed_retarget_survives_service_restart(self) -> None: - durable = MemoryWorkerDifficultyStore() - first = self.service(runtime(durable)) - state = client() - state.vardiff_last_accepted_difficulty = Decimal("16384") - state.vardiff_last_accepted_wall_ms = 1_000_000 - - with mock.patch("lab.prism.vardiff_service.time.time", return_value=1000.1): - first.record_session_difficulty( - state, - Decimal("500000"), - share_backed=True, - ) - self.drained(first) - - with ( - mock.patch("lab.prism.vardiff_service.time.time", return_value=1001.0), - mock.patch( - "lab.prism.vardiff_service.time.monotonic", - return_value=2000.0, - ), - ): - restarted = self.service(runtime(durable)) - retained, outcome = restarted.session_difficulty_store.lookup( - ("default", "miner-a.rig-1"), - now=2000.0, - ) - - self.assertEqual((retained, outcome), (Decimal("500000"), "hit")) - self.assertEqual(restarted.vardiff_durable_preload_records, 1) - self.assertEqual(first.vardiff_durable_write_outcome_counts["applied"], 1) - - def test_unproven_disconnect_step_up_is_not_persisted(self) -> None: - durable = MemoryWorkerDifficultyStore() - service = self.service(runtime(durable)) - state = client(difficulty=Decimal("500000")) - state.vardiff_last_accepted_difficulty = Decimal("16384") - state.vardiff_last_accepted_wall_ms = 1_000_000 - - self.drained(service).record_session_difficulty(state, share_backed=True) - self.drained(service) - - self.assertEqual(len(durable), 0) - self.assertEqual( - service.session_difficulty_store.lookup( - ("default", "miner-a.rig-1"), - now=state.vardiff_window_started_monotonic, - )[0], - Decimal("500000"), - ) - - def test_fast_initial_step_is_not_durable_until_proven(self) -> None: - durable = MemoryWorkerDifficultyStore() - service = self.service(runtime(durable)) - state = client() - state.vardiff_last_accepted_difficulty = Decimal("16384") - state.vardiff_last_accepted_wall_ms = 1_000_000 - - service.record_session_difficulty( - state, - Decimal("1048576"), - share_backed=True, - previous_difficulty=Decimal("16384"), - initial_convergence=True, - ) - self.drained(service) - self.assertEqual(len(durable), 0) - - state.share_difficulty = Decimal("1048576") - state.vardiff_last_accepted_difficulty = Decimal("1048576") - state.vardiff_last_accepted_wall_ms = 1_000_100 - service.record_session_difficulty(state, share_backed=True) - self.drained(service) - self.assertEqual( - durable.load_recent(evidence_after_ms=1_000_000, limit=1)[0].difficulty, - Decimal("1048576"), - ) - - def test_idle_step_down_replaces_higher_durable_row_across_restart(self) -> None: - durable = MemoryWorkerDifficultyStore() - service = self.service(runtime(durable)) - state = client(difficulty=Decimal("1048576")) - state.vardiff_last_accepted_difficulty = Decimal("1048576") - state.vardiff_last_accepted_wall_ms = 1_000_000 - - with mock.patch("lab.prism.vardiff_service.time.time", return_value=1000.1): - service.record_session_difficulty(state, share_backed=True) - with mock.patch("lab.prism.vardiff_service.time.time", return_value=1000.2): - service.record_session_difficulty( - state, - Decimal("65536"), - share_backed=False, - previous_difficulty=Decimal("1048576"), - ) - self.drained(service) - with ( - mock.patch("lab.prism.vardiff_service.time.time", return_value=1000.3), - mock.patch("lab.prism.vardiff_service.time.monotonic", return_value=2000.0), - ): - restarted = self.service(runtime(durable)) - - self.assertEqual( - restarted.session_difficulty_store.lookup( - ("default", "miner-a.rig-1"), - now=2000.0, - )[0], - Decimal("65536"), - ) - - def test_retention_kill_switch_prevents_durable_writes(self) -> None: - durable = MemoryWorkerDifficultyStore() - service = self.service(runtime(durable, enabled=False)) - state = client() - state.vardiff_last_accepted_difficulty = Decimal("16384") - state.vardiff_last_accepted_wall_ms = 1_000_000 - - service.record_session_difficulty( - state, - Decimal("500000"), - share_backed=True, - ) - self.drained(service) - - self.assertEqual(len(durable), 0) - self.assertEqual( - sum(service.vardiff_durable_write_outcome_counts.values()), - 0, - ) - - def test_startup_prune_removes_all_expired_rows_not_only_cache_limit(self) -> None: - durable = MemoryWorkerDifficultyStore() - for index in range(5): - durable.upsert( - listener="default", - worker_username=f"expired-{index}", - difficulty=Decimal("16384"), - evidence_at_ms=1_000, - now_ms=1_000, - ) - with mock.patch("lab.prism.vardiff_service.time.time", return_value=1000.0): - service = VardiffService( - runtime(durable, ttl_seconds=1.0, max_entries=2) - ) - - self.assertEqual(len(durable), 0) - self.assertEqual(service.vardiff_durable_pruned_records, 5) - - def test_exact_difficulty_share_evidence_persists_on_disconnect(self) -> None: - durable = MemoryWorkerDifficultyStore() - service = self.service(runtime(durable)) - state = client(difficulty=Decimal("500000")) - state.vardiff_last_accepted_difficulty = Decimal("500000") - state.vardiff_last_accepted_wall_ms = 1_000_000 - - with mock.patch("lab.prism.vardiff_service.time.time", return_value=1000.2): - service.record_session_difficulty(state, share_backed=True) - self.drained(service) - - recent = durable.load_recent(evidence_after_ms=999_999, limit=10) - self.assertEqual(len(recent), 1) - self.assertEqual(recent[0].difficulty, Decimal("500000")) - - def test_preload_preserves_remaining_ttl(self) -> None: - durable = MemoryWorkerDifficultyStore() - durable.upsert( - listener="default", - worker_username="miner-a.rig-1", - difficulty=Decimal("500000"), - evidence_at_ms=100_000, - now_ms=100_000, - ) - with ( - mock.patch("lab.prism.vardiff_service.time.time", return_value=109.0), - mock.patch( - "lab.prism.vardiff_service.time.monotonic", - return_value=1000.0, - ), - ): - service = self.service(runtime(durable, ttl_seconds=10.0)) - - self.assertEqual( - service.session_difficulty_store.lookup( - ("default", "miner-a.rig-1"), - now=1000.5, - )[0], - Decimal("500000"), - ) - self.assertEqual( - service.session_difficulty_store.lookup( - ("default", "miner-a.rig-1"), - now=1001.1, - ), - (None, "expired"), - ) - - def test_preload_is_lane_and_exact_username_scoped(self) -> None: - durable = MemoryWorkerDifficultyStore() - for listener, username, difficulty in ( - ("default", "miner-a.rig-1", Decimal("500000")), - ("highdiff", "miner-a.rig-1", Decimal("750000")), - ("default", "miner-a.RIG-1", Decimal("250000")), - ): - durable.upsert( - listener=listener, - worker_username=username, - difficulty=difficulty, - evidence_at_ms=1_000_000, - now_ms=1_000_000, - ) - with ( - mock.patch("lab.prism.vardiff_service.time.time", return_value=1000.1), - mock.patch( - "lab.prism.vardiff_service.time.monotonic", - return_value=2000.0, - ), - ): - service = self.service(runtime(durable)) - - for key, expected in ( - (("default", "miner-a.rig-1"), Decimal("500000")), - (("highdiff", "miner-a.rig-1"), Decimal("750000")), - (("default", "miner-a.RIG-1"), Decimal("250000")), - ): - self.assertEqual( - service.session_difficulty_store.lookup(key, now=2000.0)[0], - expected, - ) - - def test_store_failure_falls_back_without_breaking_retention(self) -> None: - class FailingStore(MemoryWorkerDifficultyStore): - def upsert(self, **kwargs: object): # type: ignore[no-untyped-def] - raise RuntimeError("database unavailable") - - durable = FailingStore() - service = self.service(runtime(durable)) - state = client() - state.vardiff_last_accepted_difficulty = Decimal("16384") - state.vardiff_last_accepted_wall_ms = 1_000_000 - - service.record_session_difficulty( - state, - Decimal("500000"), - share_backed=True, - ) - self.drained(service) - - self.assertEqual(service.vardiff_durable_write_outcome_counts["failed"], 1) - self.assertEqual( - service.session_difficulty_store.lookup( - ("default", "miner-a.rig-1"), - now=state.vardiff_window_started_monotonic, - )[0], - Decimal("500000"), - ) - - def test_durable_write_never_waits_on_the_client_job_lock(self) -> None: - # The payload is snapshotted by record_session_difficulty, so the lane - # has no reason to touch client state -- and a lane that waited on - # job_update_lock would let one client's slow job send stall every - # other client's write behind it on the single worker. - invoked = threading.Event() - - def run_json(_sql: str) -> dict[str, object]: - invoked.set() - return {"applied": True, "stored": None} - - service = self.service(runtime(MemoryWorkerDifficultyStore())) - service.worker_difficulty_store = PostgresWorkerDifficultyStore(run_json) - state = client() - state.vardiff_last_accepted_difficulty = Decimal("16384") - state.vardiff_last_accepted_wall_ms = 1_000_000 - - with state.job_update_lock: - service.record_session_difficulty( - state, - Decimal("500000"), - share_backed=True, - ) - self.assertTrue(invoked.wait(5.0)) - - self.assertEqual( - service.vardiff_durable_write_outcome_counts["applied"], - 1, - ) - - def test_one_clients_held_job_lock_cannot_stall_another_clients_write( - self, - ) -> None: - service = self.service(runtime(MemoryWorkerDifficultyStore())) - blocked = client(username="blocked.rig") - blocked.vardiff_last_accepted_difficulty = Decimal("16384") - blocked.vardiff_last_accepted_wall_ms = 1_000_000 - other = client(username="other.rig") - other.vardiff_last_accepted_difficulty = Decimal("16384") - other.vardiff_last_accepted_wall_ms = 1_000_000 - - with blocked.job_update_lock: - service.record_session_difficulty( - blocked, Decimal("500000"), share_backed=True - ) - service.record_session_difficulty( - other, Decimal("500000"), share_backed=True - ) - self.assertTrue(service.flush_durable_writes(timeout=5.0)) - - self.assertEqual( - service.vardiff_durable_write_outcome_counts["applied"], - 2, - ) - - def test_saturated_lane_evicts_resume_hints_before_step_downs(self) -> None: - durable = MemoryWorkerDifficultyStore() - service = self.service(runtime(durable, max_entries=4096)) - - # Park the lane so the queue shape is deterministic: no worker is - # concurrently draining entries out from under the assertions. - with mock.patch.object(service, "_ensure_durable_worker_locked"): - capacity = MAX_PENDING_VARDIFF_DURABLE_WRITES - for index in range(capacity): - service._enqueue_durable_write( - _PendingDurableWrite( - key=("default", f"hint-{index}"), - downward_only=False, - difficulty=Decimal("16384"), - evidence_at_ms=1_000_000, - now_ms=1_000_000, - ) - ) - self.assertEqual(service.durable_pending_depth(), capacity) - # Turn one hint into a composite whose correction matters if its - # evidence is stale. It must receive the same queue priority as a - # pure step-down even though its primary operation is an upsert. - service._enqueue_durable_write( - _PendingDurableWrite( - key=("default", "hint-0"), - downward_only=True, - difficulty=Decimal("1024"), - evidence_at_ms=None, - now_ms=1_000_100, - ) - ) - # New step-downs must still get in by evicting the remaining pure - # hints rather than any safety-critical correction. - for index in range(8): - service._enqueue_durable_write( - _PendingDurableWrite( - key=("default", f"stepdown-{index}"), - downward_only=True, - difficulty=Decimal("1024"), - evidence_at_ms=None, - now_ms=1_000_000, - ) - ) - self.assertEqual(service.durable_pending_depth(), capacity) - with service._vardiff_durable_lock: - queued = list(service._vardiff_durable_pending.values()) - - self.assertEqual(sum(1 for entry in queued if entry.downward_only), 8) - self.assertEqual( - sum(1 for entry in queued if entry.has_downward_correction), - 9, - ) - self.assertEqual(service.vardiff_durable_downward_dropped, 0) - self.assertEqual( - service.vardiff_durable_write_outcome_counts["dropped"], - 8, - ) - - def test_a_step_down_the_lane_cannot_keep_is_counted_and_logged(self) -> None: - service = self.service(runtime(MemoryWorkerDifficultyStore())) - - with mock.patch.object(service, "_ensure_durable_worker_locked"): - for index in range(MAX_PENDING_VARDIFF_DURABLE_WRITES + 4): - service._enqueue_durable_write( - _PendingDurableWrite( - key=("default", f"stepdown-{index}"), - downward_only=True, - difficulty=Decimal("1024"), - evidence_at_ms=None, - now_ms=1_000_000, - ) - ) - depth = service.durable_pending_depth() - - # Bounded, and the losses are loud rather than silent. - self.assertEqual(depth, MAX_PENDING_VARDIFF_DURABLE_WRITES) - self.assertGreaterEqual(service.vardiff_durable_downward_dropped, 4) - self.assertIn( - "qbit_prism_vardiff_durable_downward_dropped_total", - "\n".join(service.metrics_lines()), - ) - - def test_queued_writes_coalesce_by_worker_identity(self) -> None: - durable = MemoryWorkerDifficultyStore() - service = self.service(runtime(durable)) - - with mock.patch.object(service, "_ensure_durable_worker_locked"): - for difficulty in ("1048576", "262144", "65536"): - service._enqueue_durable_write( - _PendingDurableWrite( - key=("default", "miner-a.rig-1"), - downward_only=True, - difficulty=Decimal(difficulty), - evidence_at_ms=None, - now_ms=1_000_000, - ) - ) - self.assertEqual(service.durable_pending_depth(), 1) - with service._vardiff_durable_lock: - merged = service._vardiff_durable_pending[ - ("default", "miner-a.rig-1") - ] - - # Three step-downs collapse to the lowest, which is what applying all - # three in order would have left behind. - self.assertEqual(merged.difficulty, Decimal("65536")) - self.assertTrue(merged.downward_only) - self.assertEqual(service.vardiff_durable_coalesced, 2) - - def test_a_later_step_down_clamps_a_queued_evidence_write(self) -> None: - service = self.service(runtime(MemoryWorkerDifficultyStore())) - upsert = _PendingDurableWrite( - key=("default", "miner-a.rig-1"), - downward_only=False, - difficulty=Decimal("1048576"), - evidence_at_ms=1_000_000, - now_ms=1_000_000, - ) - lower = _PendingDurableWrite( - key=("default", "miner-a.rig-1"), - downward_only=True, - difficulty=Decimal("65536"), - evidence_at_ms=None, - now_ms=1_000_100, - ) - - merged = upsert.merged_with(lower) - - # upsert-then-lower must leave the minimum difficulty AND the upsert's - # evidence, so the merge cannot discard the share evidence. - self.assertFalse(merged.downward_only) - self.assertEqual(merged.difficulty, Decimal("65536")) - self.assertEqual(merged.evidence_at_ms, 1_000_000) - # A later evidence write supersedes the step-down if it applies, but - # must retain the correction for the branch where the store already - # has still-newer evidence and rejects the queued upsert. - lower_then_upsert = lower.merged_with(upsert) - self.assertEqual(lower_then_upsert.difficulty, upsert.difficulty) - self.assertEqual( - lower_then_upsert.stale_downward_difficulty, - lower.difficulty, - ) - - def test_coalescing_keeps_newer_evidence_when_it_was_queued_first(self) -> None: - newer = _PendingDurableWrite( - key=("default", "miner-a.rig-1"), - downward_only=False, - difficulty=Decimal("1048576"), - evidence_at_ms=1_000_100, - now_ms=1_000_100, - ) - older = _PendingDurableWrite( - key=newer.key, - downward_only=False, - difficulty=Decimal("65536"), - evidence_at_ms=1_000_000, - now_ms=1_000_200, - ) - - merged = newer.merged_with(older) - - # Applying newer then older leaves the newer row standing because the - # store rejects the second write as stale. Coalescing must preserve the - # same evidence and difficulty. - self.assertEqual(merged.evidence_at_ms, newer.evidence_at_ms) - self.assertEqual(merged.difficulty, newer.difficulty) - self.assertEqual(merged.now_ms, newer.now_ms) - - def test_coalesced_step_down_still_applies_when_upsert_is_stale(self) -> None: - durable = MemoryWorkerDifficultyStore() - service = self.service(runtime(durable)) - durable.upsert( - listener="default", - worker_username="miner-a.rig-1", - difficulty=Decimal("1048576"), - evidence_at_ms=1_000_200, - now_ms=1_000_200, - ) - stale_upsert = _PendingDurableWrite( - key=("default", "miner-a.rig-1"), - downward_only=False, - difficulty=Decimal("262144"), - evidence_at_ms=1_000_100, - now_ms=1_000_300, - ) - lower = _PendingDurableWrite( - key=stale_upsert.key, - downward_only=True, - difficulty=Decimal("65536"), - evidence_at_ms=None, - now_ms=1_000_400, - ) - - service._apply_durable_write(stale_upsert.merged_with(lower)) - - stored = durable._entries[stale_upsert.key] - self.assertEqual(stored.difficulty, Decimal("65536")) - self.assertEqual(stored.evidence_at_ms, 1_000_200) - self.assertEqual(stored.updated_at_ms, 1_000_400) - self.assertEqual(service.vardiff_durable_write_outcome_counts["stale"], 1) - self.assertEqual(service.vardiff_durable_write_outcome_counts["lowered"], 1) - - def test_coalescing_is_order_equivalent_for_upserts_and_step_downs(self) -> None: - key = ("default", "miner-a.rig-1") - operation_specs = ( - (False, Decimal("16"), 1_000_000), - (False, Decimal("64"), 1_000_100), - (False, Decimal("8"), 1_000_100), - (True, Decimal("8"), None), - (True, Decimal("32"), None), - (True, Decimal("128"), None), - ) - seeds = ( - None, - (999_900, Decimal("4")), - (999_900, Decimal("256")), - (1_000_050, Decimal("32")), - (1_000_200, Decimal("256")), - ) - - def apply(store, pending: _PendingDurableWrite) -> None: - if pending.downward_only: - store.apply_downward( - listener=key[0], - worker_username=key[1], - difficulty=pending.difficulty, - now_ms=pending.now_ms, - ) - return - result = store.upsert( - listener=key[0], - worker_username=key[1], - difficulty=pending.difficulty, - evidence_at_ms=int(pending.evidence_at_ms or 0), - now_ms=pending.now_ms, - ) - stale_downward = pending._stale_downward() - if not result.applied and stale_downward is not None: - store.apply_downward( - listener=key[0], - worker_username=key[1], - difficulty=stale_downward[0], - now_ms=stale_downward[1], - ) - - for length in range(1, 5): - for specs in product(operation_specs, repeat=length): - operations = [ - _PendingDurableWrite( - key=key, - downward_only=downward_only, - difficulty=difficulty, - evidence_at_ms=evidence_at_ms, - now_ms=1_001_000 + index, - ) - for index, (downward_only, difficulty, evidence_at_ms) in enumerate( - specs - ) - ] - merged = operations[0] - for operation in operations[1:]: - merged = merged.merged_with(operation) - for seed in seeds: - sequential = MemoryWorkerDifficultyStore() - coalesced = MemoryWorkerDifficultyStore() - if seed is not None: - for store in (sequential, coalesced): - store.upsert( - listener=key[0], - worker_username=key[1], - difficulty=seed[1], - evidence_at_ms=seed[0], - now_ms=999_000, - ) - for operation in operations: - apply(sequential, operation) - apply(coalesced, merged) - self.assertEqual( - coalesced._entries, - sequential._entries, - (specs, seed, merged), - ) - - def test_shutdown_drains_step_downs_instead_of_cancelling_them(self) -> None: - durable = MemoryWorkerDifficultyStore() - # Live evidence, so the service's startup prune leaves these standing. - live_ms = max(0, int(time.time() * 1000)) - for index in range(6): - durable.upsert( - listener="default", - worker_username=f"stepdown-{index}", - difficulty=Decimal("1048576"), - evidence_at_ms=live_ms, - now_ms=live_ms, - ) - service = self.service(runtime(durable)) - - # Queue with the lane parked, so every entry is still pending when - # shutdown begins -- the case where cancelling would lose them. - with mock.patch.object(service, "_ensure_durable_worker_locked"): - for index in range(6): - service._enqueue_durable_write( - _PendingDurableWrite( - key=("default", f"stepdown-{index}"), - downward_only=True, - difficulty=Decimal("65536"), - evidence_at_ms=None, - now_ms=1_000_100, - ) - ) - self.assertEqual(service.durable_pending_depth(), 6) - service.shutdown_durable_executor() - - self.assertEqual(service.durable_pending_depth(), 0) - self.assertEqual(service.vardiff_durable_downward_dropped, 0) - for index in range(6): - self.assertEqual( - durable._entries[("default", f"stepdown-{index}")].difficulty, - Decimal("65536"), - ) - - def test_shutdown_drain_stays_bounded_when_the_store_hangs(self) -> None: - service = self.service(runtime(MemoryWorkerDifficultyStore())) - service._vardiff_durable_operation_timeout_seconds = 0.3 - - with mock.patch.object(service, "_ensure_durable_worker_locked"): - for index in range(40): - service._enqueue_durable_write( - _PendingDurableWrite( - key=("default", f"stepdown-{index}"), - downward_only=True, - difficulty=Decimal("65536"), - evidence_at_ms=None, - now_ms=1_000_100, - ) - ) - # Each statement is bounded by the store's own operation deadline; the - # drain's job is to bound how many of them shutdown will wait for. - with mock.patch.object( - service, - "_apply_durable_write", - side_effect=lambda pending: time.sleep(0.02), - ): - started = time.monotonic() - service.shutdown_durable_executor() - elapsed = time.monotonic() - started - - # Bounded by the ledger operation budget, and whatever the budget - # could not cover is accounted for rather than vanishing. - self.assertLess(elapsed, 3.0) - self.assertEqual(service.durable_pending_depth(), 0) - self.assertGreater(service.vardiff_durable_downward_dropped, 0) - - def test_step_down_never_suppresses_later_share_backed_evidence(self) -> None: - durable = MemoryWorkerDifficultyStore() - service = self.service(runtime(durable)) - state = client(difficulty=Decimal("1048576")) - state.vardiff_last_accepted_difficulty = Decimal("1048576") - state.vardiff_last_accepted_wall_ms = 1_000_000 - - # A share-backed write, then an idle step-down whose wall clock is - # LATER than the share evidence that follows it. - with mock.patch("lab.prism.vardiff_service.time.time", return_value=1000.1): - service.record_session_difficulty(state, share_backed=True) - with mock.patch("lab.prism.vardiff_service.time.time", return_value=1200.0): - service.record_session_difficulty( - state, - Decimal("65536"), - share_backed=False, - previous_difficulty=Decimal("1048576"), - ) - self.drained(service) - stored = durable._entries[("default", "miner-a.rig-1")] - self.assertEqual(stored.difficulty, Decimal("65536")) - # The correction carries no evidence of its own, so evidence_at still - # points at the last accepted share. - self.assertEqual(stored.evidence_at_ms, 1_000_000) - - # Genuine share evidence recorded a moment after the last accepted - # share -- but before the step-down's wall clock -- must still apply. - state.share_difficulty = Decimal("500000") - state.vardiff_last_accepted_difficulty = Decimal("500000") - state.vardiff_last_accepted_wall_ms = 1_000_050 - with mock.patch("lab.prism.vardiff_service.time.time", return_value=1300.0): - service.record_session_difficulty(state, share_backed=True) - self.drained(service) - - self.assertEqual( - durable._entries[("default", "miner-a.rig-1")].difficulty, - Decimal("500000"), - ) - self.assertEqual( - service.vardiff_durable_write_outcome_counts["stale"], - 0, - ) - - def test_step_down_never_inserts_a_row_for_an_unknown_worker(self) -> None: - durable = MemoryWorkerDifficultyStore() - service = self.service(runtime(durable)) - state = client(difficulty=Decimal("1048576")) - - service.record_session_difficulty( - state, - Decimal("65536"), - share_backed=False, - previous_difficulty=Decimal("1048576"), - ) - self.drained(service) - - # Nothing stored means nothing a restart could resurrect. - self.assertEqual(len(durable), 0) - self.assertEqual( - service.vardiff_durable_write_outcome_counts["unchanged"], - 1, - ) - - def test_periodic_prune_runs_without_any_successful_write(self) -> None: - durable = MemoryWorkerDifficultyStore() - with mock.patch("lab.prism.vardiff_service.time.time", return_value=1000.0): - service = self.service(runtime(durable, ttl_seconds=1.0)) - self.assertEqual(service.vardiff_durable_pruned_records, 0) - - # Rows this process never wrote -- inherited from a previous run. - for index in range(3): - durable.upsert( - listener="default", - worker_username=f"expired-{index}", - difficulty=Decimal("16384"), - evidence_at_ms=1_000, - now_ms=1_000, - ) - - # Not yet due: the interval is what drives it, not write traffic. - self.assertFalse(service.request_durable_prune_if_due()) - self.assertEqual(len(durable), 3) - - service._vardiff_durable_next_prune_monotonic = time.monotonic() - 1.0 - with mock.patch("lab.prism.vardiff_service.time.time", return_value=2000.0): - self.assertTrue(service.request_durable_prune_if_due()) - self.assertTrue(service.flush_durable_writes(timeout=5.0)) - for _ in range(500): - if not durable._entries: - break - time.sleep(0.005) - - self.assertEqual(len(durable), 0) - self.assertEqual(service.vardiff_durable_pruned_records, 3) - - def test_periodic_prune_runs_when_idle_sweeps_are_disabled(self) -> None: - pruned = threading.Event() - - class RecordingStore(MemoryWorkerDifficultyStore): - def prune(self, **kwargs: object) -> int: # type: ignore[override] - deleted = super().prune(**kwargs) # type: ignore[arg-type] - pruned.set() - return deleted - - durable = RecordingStore() - service = self.service(runtime(durable, ttl_seconds=1.0)) - pruned.clear() # Ignore the service's empty startup-prune call. - durable.upsert( - listener="default", - worker_username="expired", - difficulty=Decimal("16384"), - evidence_at_ms=1_000, - now_ms=1_000, - ) - service._vardiff_durable_next_prune_monotonic = time.monotonic() - 1.0 - - # No idle-sweep runtime fields or thread are involved: the durable - # lane's scheduler must wake and prune on its own. - service.start_durable_prune_scheduler() - service._vardiff_durable_wake.set() - - self.assertTrue(pruned.wait(5.0)) - self.assertEqual(len(durable), 0) - - def test_serve_starts_durable_pruning_before_accepting(self) -> None: - source = inspect.getsource(PrismCoordinator._serve_with_listener_stack) - scheduler = source.index("start_durable_prune_scheduler") - - self.assertLess(scheduler, source.index("self.accept_loop")) - - def test_prune_metric_is_a_counter_covering_every_prune(self) -> None: - service = self.service(runtime(MemoryWorkerDifficultyStore())) - text = "\n".join(service.metrics_lines()) - - self.assertIn( - "# TYPE qbit_prism_vardiff_durable_pruned_total counter", - text, - ) - self.assertIn("qbit_prism_vardiff_durable_pruned_total 0", text) - self.assertNotIn("qbit_prism_vardiff_durable_pruned gauge", text) - self.assertIn( - "# TYPE qbit_prism_vardiff_durable_prune_failures_total counter", - text, - ) - - def test_step_down_after_shutdown_is_applied_inline_not_dropped(self) -> None: - durable = MemoryWorkerDifficultyStore() - live_ms = max(0, int(time.time() * 1000)) - durable.upsert( - listener="default", - worker_username="miner-a.rig-1", - difficulty=Decimal("1048576"), - evidence_at_ms=live_ms, - now_ms=live_ms, - ) - service = self.service(runtime(durable)) - service.shutdown_durable_executor() - - # A disconnect landing after the lane closed still has to correct the - # stored row; queueing it would strand it past the drain. - service._enqueue_durable_write( - _PendingDurableWrite( - key=("default", "miner-a.rig-1"), - downward_only=True, - difficulty=Decimal("65536"), - evidence_at_ms=None, - now_ms=live_ms + 100, - ) - ) - - self.assertEqual( - durable._entries[("default", "miner-a.rig-1")].difficulty, - Decimal("65536"), - ) - self.assertEqual(service.vardiff_durable_downward_dropped, 0) - self.assertEqual( - service.vardiff_durable_write_outcome_counts["lowered"], - 1, - ) - # An optional resume hint arriving after the drain is still dropped. - service._enqueue_durable_write( - _PendingDurableWrite( - key=("default", "other.rig"), - downward_only=False, - difficulty=Decimal("500000"), - evidence_at_ms=live_ms, - now_ms=live_ms, - ) - ) - self.assertEqual( - service.vardiff_durable_write_outcome_counts["dropped"], - 1, - ) - - def test_every_prune_statement_carries_a_finite_limit(self) -> None: - limits: list[object] = [] - - class RecordingStore(MemoryWorkerDifficultyStore): - def prune(self, **kwargs: object) -> int: # type: ignore[override] - limits.append(kwargs.get("limit")) - return super().prune(**kwargs) # type: ignore[arg-type] - - durable = RecordingStore() - with mock.patch("lab.prism.vardiff_service.time.time", return_value=1000.0): - service = self.service(runtime(durable, ttl_seconds=1.0)) - service._vardiff_durable_next_prune_monotonic = time.monotonic() - 1.0 - service.request_durable_prune_if_due() - service._prune_durable_worker_difficulties_if_requested() - - self.assertTrue(limits) - for limit in limits: - self.assertIsInstance(limit, int) - self.assertGreater(limit, 0) - - def test_prune_batches_until_the_backlog_drains(self) -> None: - durable = MemoryWorkerDifficultyStore() - with mock.patch("lab.prism.vardiff_service.time.time", return_value=1000.0): - service = self.service(runtime(durable, ttl_seconds=1.0)) - batches: list[int] = [] - - def batched(**kwargs: object) -> int: - batches.append(int(kwargs["limit"])) # type: ignore[arg-type] - # Two full batches, then a short one: the loop must stop there. - return ( - PRISM_VARDIFF_DURABLE_PRUNE_BATCH - if len(batches) <= 2 - else 7 - ) - - with mock.patch.object(durable, "prune", side_effect=batched): - pruned = service._prune_expired_rows(1_000) - - self.assertEqual(len(batches), 3) - self.assertEqual(pruned, 2 * PRISM_VARDIFF_DURABLE_PRUNE_BATCH + 7) - self.assertEqual( - batches, - [PRISM_VARDIFF_DURABLE_PRUNE_BATCH] * 3, - ) - - def test_prune_batching_is_capped_per_pass(self) -> None: - durable = MemoryWorkerDifficultyStore() - with mock.patch("lab.prism.vardiff_service.time.time", return_value=1000.0): - service = self.service(runtime(durable, ttl_seconds=1.0)) - - with mock.patch.object( - durable, - "prune", - side_effect=lambda **kwargs: PRISM_VARDIFF_DURABLE_PRUNE_BATCH, - ) as always_full: - service._prune_expired_rows(1_000) - - # An endless backlog must not hold the lane for an unbounded pass. - self.assertEqual( - always_full.call_count, - PRISM_VARDIFF_DURABLE_PRUNE_MAX_BATCHES, - ) - - def test_metrics_expose_only_bounded_durable_outcomes(self) -> None: - service = self.service(runtime(MemoryWorkerDifficultyStore())) - text = "\n".join(service.metrics_lines()) - - self.assertIn("qbit_prism_vardiff_durable_preloaded 0", text) - for outcome in ("applied", "stale", "failed", "dropped"): - self.assertIn( - f'qbit_prism_vardiff_durable_writes_total{{outcome="{outcome}"}} 0', - text, - ) - self.assertIn( - 'qbit_prism_vardiff_durable_backend{backend="memory"} 1', - text, - ) - - -if __name__ == "__main__": - unittest.main() diff --git a/tests/test_prism_vardiff_fast_arrival.py b/tests/test_prism_vardiff_fast_arrival.py deleted file mode 100644 index 46d576829..000000000 --- a/tests/test_prism_vardiff_fast_arrival.py +++ /dev/null @@ -1,451 +0,0 @@ -#!/usr/bin/env python3 -"""Fast-arrival initial vardiff convergence (#132). - -Credit is target-based: a share is worth the difficulty its job was stamped -at. A rental router that filters submissions at its own, much higher -difficulty therefore under-credits its miner for as long as the stamped -target lags the difficulty the rig is really working at. These tests pin the -incident shape -- a standard-lane session started at 16384 behind a router -filtering at 500000 -- and the guards that keep the fast path from touching -anyone else. -""" -# ruff: noqa: F403, F405 - -from __future__ import annotations - -import unittest - -from lab.prism.vardiff_service import ( - PRISM_VARDIFF_HIGH_DIFF_ARRIVAL_SECONDS_BUCKETS, - PRISM_VARDIFF_HIGH_DIFF_ARRIVAL_SHARES_BUCKETS, - PRISM_VARDIFF_INITIAL_RETARGET_OUTCOMES, -) -from tests.prism_vardiff_test_support import * - - -STANDARD_LANE_START = Decimal("16384") -ROUTER_FILTER_DIFFICULTY = Decimal("500000") -# 9.4 PH/s clearing a 500000 filter: 9.4e15 / (500000 * 2**32) accepted -# shares per second, so one every ~0.2285s. -ROUTER_SHARE_INTERVAL_SECONDS = 0.2285 - - -def standard_lane_config(**overrides: object) -> vardiff.VardiffConfig: - """The deployed standard lane: 16384 start, 4x steps, 300s retargets.""" - base = dict( - enabled=True, - target_share_interval_seconds=Decimal("15"), - min_difficulty=STANDARD_LANE_START, - max_difficulty=Decimal("4294967296"), - retarget_interval_seconds=Decimal("300"), - max_step_factor=Decimal("4"), - startup_difficulty=STANDARD_LANE_START, - max_step_down_factor=Decimal("4"), - ewma_alpha=Decimal("0.4"), - retarget_tolerance=Decimal("0.25"), - ) - base.update(overrides) - return vardiff.VardiffConfig(**base) # type: ignore[arg-type] - - -class FastArrivalHarness: - """A coordinator, a client, and a clock the test drives by hand.""" - - def __init__( - self, - config: vardiff.VardiffConfig, - *, - difficulty: Decimal = STANDARD_LANE_START, - ) -> None: - self.server = coordinator() - self.server.vardiff_config = config - self.client = client() - self.client.share_difficulty = difficulty - self.now = 1000.0 - self.client.vardiff_window_started_monotonic = self.now - self.client.vardiff_session_started_monotonic = self.now - self.sent: list[dict[str, object]] = [] - self.server.maybe_send_job = self._send # type: ignore[method-assign] - - def _send(self, client: object, clean_jobs: bool) -> bool: - self.sent.append({"clean": clean_jobs, "at": self.now}) - return True - - @property - def service(self) -> object: - return self.server._ensure_vardiff_service() - - @property - def difficulty(self) -> Decimal: - return self.client.pending_share_difficulty or self.client.share_difficulty - - def deliver(self, count: int, *, interval: float, difficulty: Decimal | None = None) -> None: - """Feed ``count`` accepted shares stamped at the client's target.""" - for _ in range(count): - self.now += interval - stamped = difficulty if difficulty is not None else self.difficulty - with patch( - "lab.prism.vardiff_service.time.monotonic", - side_effect=lambda: self.now, - ): - self.server.note_vardiff_submitted_share(self.client) - self.server.note_vardiff_accepted_share( - self.client, - FakeJob(stamped), # type: ignore[arg-type] - ) - - def metrics(self) -> dict[str, str]: - lines = self.server.vardiff_idle_metrics_lines() - return { - line.rsplit(" ", 1)[0]: line.rsplit(" ", 1)[1] - for line in lines - if not line.startswith("#") - } - - -class PrismVardiffFastArrivalTests(unittest.TestCase): - def test_router_filtered_session_reaches_router_difficulty_in_the_initial_burst( - self, - ) -> None: - # The incident: a 9.4 PH/s rental order behind a router that only - # forwards shares clearing 500000, into a standard lane that stamps - # 16384. Every share the pool sees is worth ~30x what it is credited, - # and the ordinary ladder needs three 300s retargets to close that. - harness = FastArrivalHarness(standard_lane_config()) - self.assertEqual(harness.difficulty, STANDARD_LANE_START) - - harness.deliver(8, interval=ROUTER_SHARE_INTERVAL_SECONDS) - - self.assertGreaterEqual(harness.difficulty, ROUTER_FILTER_DIFFICULTY) - self.assertEqual(harness.difficulty, STANDARD_LANE_START * 64) - # Arrival is bounded by the burst, not by the retarget interval. - elapsed = harness.now - 1000.0 - self.assertLess(elapsed, 2.0) - self.assertLess( - Decimal(str(elapsed)), - standard_lane_config().retarget_interval_seconds, - ) - # The step up keeps the miner's in-flight work: one non-clean job. - self.assertEqual(len(harness.sent), 1) - self.assertFalse(harness.sent[0]["clean"]) - - def test_ordinary_ladder_would_still_be_short_after_the_same_burst(self) -> None: - # Same burst with the fast path off: the ordinary 4x bound lands at - # 65536, still 7.6x below what the router is filtering at, and the - # next move cannot come until the 300s interval elapses. - harness = FastArrivalHarness( - standard_lane_config(initial_convergence_enabled=False) - ) - - harness.deliver(8, interval=ROUTER_SHARE_INTERVAL_SECONDS) - - self.assertEqual(harness.difficulty, STANDARD_LANE_START) - self.assertEqual(harness.sent, []) - - # Only a full ordinary window of that same cadence retargets, and the - # 4x bound lands at 65536 -- still 7.6x below what the router filters - # at, with another 300s before the next move. Three intervals of - # under-credited shares is what the fast path removes. - harness.deliver(1305, interval=ROUTER_SHARE_INTERVAL_SECONDS) - - self.assertGreaterEqual(harness.now - 1000.0, 300.0) - self.assertEqual(harness.difficulty, STANDARD_LANE_START * 4) - self.assertLess(harness.difficulty, ROUTER_FILTER_DIFFICULTY) - - def test_on_cadence_small_miner_is_not_driven_upward(self) -> None: - # One accepted share per target interval, for well past the point - # where the share threshold is met. The early path exists but never - # engages, and no difficulty change is advertised. - harness = FastArrivalHarness(standard_lane_config()) - - harness.deliver(20, interval=15.0) - - self.assertEqual(harness.difficulty, STANDARD_LANE_START) - self.assertEqual(harness.sent, []) - self.assertTrue(self.client_initial_pending(harness)) - metrics = harness.metrics() - self.assertEqual( - metrics["qbit_prism_vardiff_initial_retarget_attempts_total"], "0" - ) - - def test_burst_shorter_than_the_elapsed_floor_waits(self) -> None: - # Eight shares read off the socket back to back. The work is real but - # the window is not: dividing by ~0 would observe ~1e9. The elapsed - # floor holds the session at its lane start until a window long - # enough to mean something has passed. - harness = FastArrivalHarness(standard_lane_config()) - - harness.deliver(8, interval=0.001) - - self.assertEqual(harness.difficulty, STANDARD_LANE_START) - self.assertEqual(harness.sent, []) - - # One more share carries the window past the floor, and now the - # accumulated burst converges in a single bounded move. - harness.deliver(1, interval=1.0) - - self.assertEqual(harness.difficulty, STANDARD_LANE_START * 64) - - def test_initial_step_obeys_the_lane_ceiling(self) -> None: - harness = FastArrivalHarness( - standard_lane_config(max_difficulty=Decimal("262144")) - ) - - harness.deliver(8, interval=ROUTER_SHARE_INTERVAL_SECONDS) - - self.assertEqual(harness.difficulty, Decimal("262144")) - - def test_initial_step_stays_within_high_diff_lane_bounds(self) -> None: - # A rental-scale rig on the high-diff lane, whose floor is a wire - # guarantee. The initial move is bounded above by the lane ceiling - # and can never land below the floor. - high_diff = standard_lane_config( - min_difficulty=ROUTER_FILTER_DIFFICULTY, - startup_difficulty=ROUTER_FILTER_DIFFICULTY, - max_difficulty=Decimal("8000000"), - ) - harness = FastArrivalHarness(high_diff, difficulty=ROUTER_FILTER_DIFFICULTY) - harness.client.listener_name = "highdiff" - harness.client.minimum_advertised_difficulty = ROUTER_FILTER_DIFFICULTY - - harness.deliver(8, interval=ROUTER_SHARE_INTERVAL_SECONDS) - - self.assertEqual(harness.difficulty, Decimal("8000000")) - self.assertGreaterEqual(harness.difficulty, ROUTER_FILTER_DIFFICULTY) - self.assertEqual( - harness.server.client_minimum_advertised_difficulty(harness.client), - ROUTER_FILTER_DIFFICULTY, - ) - - def test_only_the_first_qualifying_phase_gets_the_larger_bound(self) -> None: - harness = FastArrivalHarness(standard_lane_config()) - - harness.deliver(8, interval=ROUTER_SHARE_INTERVAL_SECONDS) - converged = harness.difficulty - self.assertEqual(converged, STANDARD_LANE_START * 64) - self.assertFalse(self.client_initial_pending(harness)) - - # Commit the advertised difficulty as delivered, then present an - # equally decisive window: 600 accepted shares over a full 300s - # interval observes 30x the current difficulty. The relaxation is - # spent, so this move is bounded by the ordinary 4x, not by 30x. - harness.client.share_difficulty = converged - harness.client.pending_share_difficulty = None - harness.deliver(600, interval=0.5) - - self.assertEqual(harness.difficulty, converged * 4) - metrics = harness.metrics() - self.assertEqual( - metrics["qbit_prism_vardiff_initial_retarget_attempts_total"], "1" - ) - - def test_a_failed_send_does_not_spend_the_relaxation(self) -> None: - # The relaxation is spent at the paired-send commit point, so a build - # the coordinator skipped cannot consume it while leaving the miner - # on its old difficulty. - harness = FastArrivalHarness(standard_lane_config()) - harness.server.maybe_send_job = lambda client, clean_jobs: False # type: ignore[method-assign] - - harness.deliver(8, interval=ROUTER_SHARE_INTERVAL_SECONDS) - - self.assertEqual(harness.difficulty, STANDARD_LANE_START) - self.assertIsNone(harness.client.pending_share_difficulty) - self.assertTrue(self.client_initial_pending(harness)) - metrics = harness.metrics() - self.assertEqual( - metrics['qbit_prism_vardiff_initial_retargets_total{outcome="superseded"}'], - "1", - ) - self.assertEqual( - metrics['qbit_prism_vardiff_initial_retargets_total{outcome="applied"}'], - "0", - ) - - # Delivery recovers, and the session still converges in one move. - harness.server.maybe_send_job = harness._send # type: ignore[method-assign] - harness.deliver(8, interval=ROUTER_SHARE_INTERVAL_SECONDS) - - self.assertEqual(harness.difficulty, STANDARD_LANE_START * 64) - - def test_explicit_request_outranks_a_converged_initial_difficulty(self) -> None: - # The early path measures a session against the difficulty it is - # currently stamped at, so an explicit d= raises the baseline the - # evidence is judged against rather than being bypassed by it. And - # once vardiff has converged, a d= still resolves to exactly what - # was asked for: the request outranks the converged value. - harness = FastArrivalHarness(standard_lane_config()) - harness.deliver(8, interval=ROUTER_SHARE_INTERVAL_SECONDS) - self.assertEqual(harness.difficulty, STANDARD_LANE_START * 64) - - harness.client.requested_difficulty = Decimal("262144") - target = harness.server.apply_client_difficulty_requests(harness.client) - - self.assertEqual(target, Decimal("262144")) - self.assertEqual( - harness.client.vardiff_config.startup_difficulty, - Decimal("262144"), - ) - - def test_md_floor_is_never_undercut_by_an_initial_step(self) -> None: - # md= raises the client's own minimum. The initial step clamps to the - # specialized policy's bounds like any other step. - harness = FastArrivalHarness(standard_lane_config()) - harness.client.requested_min_difficulty = Decimal("262144") - harness.server.apply_client_difficulty_requests(harness.client) - self.assertEqual( - harness.client.vardiff_config.min_difficulty, - Decimal("262144"), - ) - - harness.deliver(8, interval=ROUTER_SHARE_INTERVAL_SECONDS) - - self.assertGreaterEqual(harness.difficulty, Decimal("262144")) - - def test_request_landing_mid_retarget_wins_over_the_initial_step(self) -> None: - # The race: the early retarget computed its step from 16384, but an - # explicit request moved the client before the paired send. The - # retarget yields rather than overriding the request. - harness = FastArrivalHarness(standard_lane_config()) - harness.client.pending_share_difficulty = ROUTER_FILTER_DIFFICULTY - - applied = harness.service.retarget_locked( - harness.client, - current_difficulty=STANDARD_LANE_START, - accepted_shares=8, - submitted_shares=8, - accepted_difficulty=STANDARD_LANE_START * 8, - elapsed_seconds=Decimal("1.828"), - initial_convergence=True, - ) - - self.assertFalse(applied) - self.assertEqual(harness.client.pending_share_difficulty, ROUTER_FILTER_DIFFICULTY) - self.assertEqual(harness.sent, []) - metrics = harness.metrics() - self.assertEqual( - metrics['qbit_prism_vardiff_initial_retargets_total{outcome="superseded"}'], - "1", - ) - - def test_idle_step_down_never_takes_the_initial_bound(self) -> None: - # An idle window carries no accepted shares, so it can never be the - # evidence the relaxation is granted for, even if a caller asks. - harness = FastArrivalHarness(standard_lane_config(), difficulty=Decimal("1048576")) - # The client is never admitted, so the idle preconditions reject this - # retarget outright. What matters is that require_idle stripped the - # relaxation before any of that: a zero-share window is not evidence. - applied = harness.service.retarget_locked( - harness.client, - current_difficulty=Decimal("1048576"), - accepted_shares=0, - submitted_shares=0, - accepted_difficulty=Decimal("0"), - elapsed_seconds=Decimal("300"), - initial_convergence=True, - require_idle=True, - prepared_bundle=object(), # type: ignore[arg-type] - ) - - self.assertFalse(applied) - metrics = harness.metrics() - self.assertEqual( - metrics["qbit_prism_vardiff_initial_retarget_attempts_total"], "0" - ) - - def test_arrival_metrics_are_bounded_and_observed_once(self) -> None: - harness = FastArrivalHarness(standard_lane_config()) - - harness.deliver(8, interval=ROUTER_SHARE_INTERVAL_SECONDS) - - metrics = harness.metrics() - self.assertEqual( - metrics["qbit_prism_vardiff_initial_retarget_attempts_total"], "1" - ) - self.assertEqual( - metrics['qbit_prism_vardiff_initial_retargets_total{outcome="applied"}'], - "1", - ) - self.assertEqual( - metrics["qbit_prism_vardiff_high_diff_arrival_seconds_count"], "1" - ) - self.assertEqual( - metrics["qbit_prism_vardiff_high_diff_arrival_shares_count"], "1" - ) - # Eight accepted shares over ~1.8s of connection life. - self.assertEqual( - metrics["qbit_prism_vardiff_high_diff_arrival_shares_sum"], "8.000000" - ) - self.assertEqual( - metrics['qbit_prism_vardiff_high_diff_arrival_shares_bucket{le="8"}'], "1" - ) - self.assertEqual( - metrics['qbit_prism_vardiff_high_diff_arrival_shares_bucket{le="4"}'], "0" - ) - self.assertEqual( - metrics['qbit_prism_vardiff_high_diff_arrival_seconds_bucket{le="2.5"}'], - "1", - ) - - # A later retarget across the same threshold does not re-observe. - harness.client.share_difficulty = harness.difficulty - harness.client.pending_share_difficulty = None - harness.deliver(8, interval=ROUTER_SHARE_INTERVAL_SECONDS) - - self.assertEqual( - harness.metrics()["qbit_prism_vardiff_high_diff_arrival_shares_count"], "1" - ) - - def test_session_starting_above_the_threshold_never_contributes(self) -> None: - high_diff = standard_lane_config( - min_difficulty=ROUTER_FILTER_DIFFICULTY, - startup_difficulty=ROUTER_FILTER_DIFFICULTY, - ) - harness = FastArrivalHarness(high_diff, difficulty=ROUTER_FILTER_DIFFICULTY) - - harness.deliver(8, interval=ROUTER_SHARE_INTERVAL_SECONDS) - - self.assertGreater(harness.difficulty, ROUTER_FILTER_DIFFICULTY) - self.assertEqual( - harness.metrics()["qbit_prism_vardiff_high_diff_arrival_shares_count"], "0" - ) - - def test_metric_label_sets_are_fixed(self) -> None: - harness = FastArrivalHarness(standard_lane_config()) - harness.deliver(8, interval=ROUTER_SHARE_INTERVAL_SECONDS) - - rendered = harness.metrics() - outcomes = [ - name - for name in rendered - if name.startswith("qbit_prism_vardiff_initial_retargets_total") - ] - self.assertEqual(len(outcomes), len(PRISM_VARDIFF_INITIAL_RETARGET_OUTCOMES)) - for outcome in PRISM_VARDIFF_INITIAL_RETARGET_OUTCOMES: - self.assertIn( - f'qbit_prism_vardiff_initial_retargets_total{{outcome="{outcome}"}}', - rendered, - ) - for metric, buckets in ( - ( - "qbit_prism_vardiff_high_diff_arrival_seconds", - PRISM_VARDIFF_HIGH_DIFF_ARRIVAL_SECONDS_BUCKETS, - ), - ( - "qbit_prism_vardiff_high_diff_arrival_shares", - PRISM_VARDIFF_HIGH_DIFF_ARRIVAL_SHARES_BUCKETS, - ), - ): - emitted = [ - name for name in rendered if name.startswith(f"{metric}_bucket") - ] - self.assertEqual(len(emitted), len(buckets) + 1) - # No worker or username ever reaches a label. - self.assertNotIn("miner", " ".join(rendered)) - - @staticmethod - def client_initial_pending(harness: FastArrivalHarness) -> bool: - return bool(harness.client.vardiff_initial_convergence_pending) - - -if __name__ == "__main__": - unittest.main() diff --git a/tests/test_prism_vardiff_resume.py b/tests/test_prism_vardiff_resume.py deleted file mode 100644 index a0aa86f1a..000000000 --- a/tests/test_prism_vardiff_resume.py +++ /dev/null @@ -1,889 +0,0 @@ -#!/usr/bin/env python3 -"""Reconnect difficulty resume: retention store, adoption, and observability. - -A reconnecting worker (same listener + exact username) resumes at its last -converged difficulty instead of the lane start, so a mass reconnect does not -re-flood the share path while every session re-climbs (the 2026-07-16 -amplifier). These tests pin the store semantics, the authorize-time adoption -ordering, the plausibility clamp, the kill switch, and the new metrics. -""" - -from __future__ import annotations - -from contextlib import contextmanager -from decimal import Decimal -from pathlib import Path -from types import SimpleNamespace -import tempfile -import threading -import time -import unittest -from typing import Iterator -from unittest import mock - -from lab.auxpow import vardiff -from lab.prism.coordinator_config import load_coordinator_config -from lab.prism.metrics import MetricsRenderer -from lab.prism.prism_coordinator import PrismCoordinator -from lab.prism.rpc import JsonRpc -from lab.prism.stratum_session import ClientState, StratumError -from lab.prism.vardiff_service import ( - PRISM_VARDIFF_RESUME_OUTCOMES, - SessionDifficultyStore, - VardiffService, - session_difficulty_key, -) -from tests.prism_vardiff_test_support import ( - PAYOUT_ADDRESS, - AddressValidationRpc, - coordinator, - highdiff_vardiff_config, - worker_identity, -) - - -STANDARD_LANE_START = Decimal("16384") - - -def standard_lane_config() -> vardiff.VardiffConfig: - """The deployed standard lane from issue #132's policy table.""" - return vardiff.VardiffConfig( - enabled=True, - target_share_interval_seconds=Decimal("15"), - min_difficulty=Decimal("1024"), - max_difficulty=Decimal("4294967296"), - retarget_interval_seconds=Decimal("300"), - max_step_factor=Decimal("4"), - startup_difficulty=STANDARD_LANE_START, - max_step_down_factor=Decimal("4"), - ) - - -class SessionDifficultyStoreTests(unittest.TestCase): - def test_lru_bound_evicts_oldest_and_counts(self) -> None: - store = SessionDifficultyStore(max_entries=2, ttl_seconds=900.0) - now = time.monotonic() - store.record(("default", "a"), Decimal("512"), now=now, share_backed=True) - store.record(("default", "b"), Decimal("256"), now=now, share_backed=True) - store.record(("default", "c"), Decimal("128"), now=now, share_backed=True) - - self.assertEqual(len(store), 2) - self.assertEqual(store.lookup(("default", "a"), now=now), (None, "miss")) - self.assertEqual( - store.lookup(("default", "c"), now=now), (Decimal("128"), "hit") - ) - self.assertEqual(store.snapshot()["evicted"], 1) - - def test_hit_refreshes_recency_but_not_ttl_and_keeps_the_entry(self) -> None: - store = SessionDifficultyStore(max_entries=2, ttl_seconds=10.0) - now = time.monotonic() - store.record(("default", "a"), Decimal("512"), now=now, share_backed=True) - store.record(("default", "b"), Decimal("1"), now=now + 1, share_backed=True) - - # A hit does not remove the entry (a miner may re-authorize) and it - # refreshes LRU recency: the next capacity eviction takes "b". - self.assertEqual( - store.lookup(("default", "a"), now=now + 2), (Decimal("512"), "hit") - ) - store.record(("default", "c"), Decimal("2"), now=now + 2, share_backed=True) - self.assertEqual(store.lookup(("default", "b"), now=now + 2), (None, "miss")) - self.assertEqual( - store.lookup(("default", "a"), now=now + 6), (Decimal("512"), "hit") - ) - # But the TTL still measures the age of the VALUE, not of the last - # access: the entry recorded at `now` is gone at now + 11. - self.assertEqual( - store.lookup(("default", "a"), now=now + 11), (None, "expired") - ) - self.assertEqual(store.snapshot()["expired"], 1) - - def test_record_silently_ignores_unusable_difficulties(self) -> None: - store = SessionDifficultyStore(max_entries=4, ttl_seconds=10.0) - now = time.monotonic() - store.record(("default", "a"), Decimal("0"), now=now, share_backed=True) - store.record(("default", "a"), Decimal("-4"), now=now, share_backed=True) - store.record(("default", "a"), Decimal("NaN"), now=now, share_backed=True) - store.record( - ("default", "a"), Decimal("Infinity"), now=now, share_backed=True - ) - self.assertEqual(len(store), 0) - - def test_non_positive_bounds_disable_retention_entirely(self) -> None: - now = time.monotonic() - for store in ( - SessionDifficultyStore(max_entries=0, ttl_seconds=10.0), - SessionDifficultyStore(max_entries=8, ttl_seconds=0.0), - ): - store.record(("default", "a"), Decimal("512"), now=now, share_backed=True) - self.assertEqual(len(store), 0) - self.assertEqual( - store.lookup(("default", "a"), now=now), (None, "disabled") - ) - - def test_prune_drops_only_expired_entries(self) -> None: - store = SessionDifficultyStore(max_entries=4, ttl_seconds=10.0) - now = time.monotonic() - store.record(("default", "old"), Decimal("512"), now=now, share_backed=True) - store.record( - ("default", "new"), Decimal("256"), now=now + 8, share_backed=True - ) - - self.assertEqual(store.prune(now=now + 11), 1) - self.assertEqual(len(store), 1) - self.assertEqual( - store.lookup(("default", "new"), now=now + 11), - (Decimal("256"), "hit"), - ) - - def test_unchanged_value_without_share_evidence_keeps_ageing(self) -> None: - # The disconnect seam records on EVERY disconnect. If that refreshed - # the timestamp, a session that resumes a retained value, submits - # nothing and disconnects would keep the value alive forever on - # reconnect cycles shorter than the TTL. - store = SessionDifficultyStore(max_entries=4, ttl_seconds=10.0) - now = time.monotonic() - store.record(("default", "a"), Decimal("512"), now=now, share_backed=True) - store.record( - ("default", "a"), Decimal("512"), now=now + 8, share_backed=False - ) - - self.assertEqual( - store.lookup(("default", "a"), now=now + 9), (Decimal("512"), "hit") - ) - self.assertEqual( - store.lookup(("default", "a"), now=now + 11), (None, "expired") - ) - - def test_ttl_refreshes_on_re_validation_only(self) -> None: - store = SessionDifficultyStore(max_entries=4, ttl_seconds=10.0) - now = time.monotonic() - # An unchanged value backed by accepted shares was re-validated. - store.record(("default", "a"), Decimal("512"), now=now, share_backed=True) - store.record( - ("default", "a"), Decimal("512"), now=now + 8, share_backed=True - ) - self.assertEqual( - store.lookup(("default", "a"), now=now + 17), (Decimal("512"), "hit") - ) - # A value that actually moved is new information, share-backed or not - # (an idle step-down carries no shares but does lower the value). - store.record(("default", "b"), Decimal("512"), now=now, share_backed=True) - store.record( - ("default", "b"), Decimal("128"), now=now + 8, share_backed=False - ) - self.assertEqual( - store.lookup(("default", "b"), now=now + 17), (Decimal("128"), "hit") - ) - - def test_non_refreshing_record_still_moves_lru_recency(self) -> None: - # Keeping the old timestamp must not turn record() into a no-op: - # recency and the TTL stay independent, so the LRU bound keeps - # evicting genuinely cold entries. - store = SessionDifficultyStore(max_entries=2, ttl_seconds=100.0) - now = time.monotonic() - store.record(("default", "a"), Decimal("512"), now=now, share_backed=True) - store.record(("default", "b"), Decimal("256"), now=now + 1, share_backed=True) - store.record(("default", "a"), Decimal("512"), now=now + 2, share_backed=False) - store.record(("default", "c"), Decimal("128"), now=now + 3, share_backed=True) - - self.assertEqual(store.lookup(("default", "b"), now=now + 3), (None, "miss")) - self.assertEqual( - store.lookup(("default", "a"), now=now + 3), (Decimal("512"), "hit") - ) - - def test_session_difficulty_key_requires_a_reserved_worker(self) -> None: - state = SimpleNamespace(listener_name="highdiff", worker=None) - self.assertIsNone(session_difficulty_key(state)) # type: ignore[arg-type] - state.worker = worker_identity("miner-a.rig-1") - self.assertEqual( - session_difficulty_key(state), # type: ignore[arg-type] - ("highdiff", "miner-a.rig-1"), - ) - - -def resume_coordinator(**attributes: object): - """A no-environment coordinator wired for authorize round trips. - - Resume attributes must land before the first vardiff-service touch, so - they are applied here, ahead of any authorize call. - """ - server = coordinator() - for name, value in attributes.items(): - setattr(server, name, value) - server.rpc = AddressValidationRpc() - server.username_fallback_address = None - server.maybe_send_job = lambda client, *, clean_jobs: True - server.jobs = {} - return server - - -def connect_client( - server, - connection_id: int, - *, - listener_name: str = "default", - lane_config: vardiff.VardiffConfig | None = None, -) -> ClientState: - """A freshly accepted connection sitting at its lane's start difficulty.""" - config = lane_config if lane_config is not None else standard_lane_config() - state = ClientState( - sock=object(), - address=("127.0.0.1", connection_id), - connection_id=connection_id, - extranonce1_hex=f"{connection_id:08x}", - listener_name=listener_name, - listener_vardiff_config=config, - share_difficulty=config.startup_difficulty, - ) - state.subscribed = True - state.send = lambda payload: None # type: ignore[method-assign] - state.close = lambda: None # type: ignore[method-assign] - return state - - -def authorize(server, state: ClientState, password: str = "x") -> None: - server.handle_request( - state, - { - "id": 1, - "method": "mining.authorize", - "params": [PAYOUT_ADDRESS, password], - }, - ) - - -def converge_and_disconnect(server, state: ClientState, difficulty: Decimal) -> None: - with server._client_vardiff_lock(state): - state.share_difficulty = difficulty - state.pending_share_difficulty = None - server.disconnect_client(state) - - -@contextmanager -def frozen_clock(now: float) -> Iterator[None]: - """Run a reconnect cycle at an exact monotonic instant. - - Retention timestamps come from time.monotonic() inside the vardiff - service, so the whole record/lookup path has to see the injected value. - """ - with mock.patch.object(time, "monotonic", return_value=now): - yield - - -def silent_reconnect_cycle(server, connection_id: int, now: float) -> ClientState: - """Connect, authorize, submit nothing, disconnect -- all at ``now``.""" - with frozen_clock(now): - state = connect_client(server, connection_id) - authorize(server, state) - server.disconnect_client(state) - return state - - -def retained_difficulty(server, now: float | None = None) -> Decimal | None: - store = server._ensure_vardiff_service().session_difficulty_store - retained, _outcome = store.lookup( - ("default", PAYOUT_ADDRESS), - now=time.monotonic() if now is None else now, - ) - return retained - - -def resume_outcomes(server) -> dict[str, int]: - outcomes = server.vardiff_convergence_snapshot()["resume_outcomes"] - assert isinstance(outcomes, dict) - return outcomes - - -class ReconnectResumeTests(unittest.TestCase): - def test_reconnect_resumes_last_converged_difficulty(self) -> None: - server = resume_coordinator() - first = connect_client(server, 1) - authorize(server, first) - self.assertEqual(first.share_difficulty, STANDARD_LANE_START) - converge_and_disconnect(server, first, Decimal("262144")) - - second = connect_client(server, 2) - authorize(server, second) - - self.assertEqual(second.share_difficulty, Decimal("262144")) - self.assertIsNone(second.pending_share_difficulty) - self.assertEqual(second.difficulty_generation, 1) - outcomes = resume_outcomes(server) - self.assertEqual(outcomes["resumed"], 1) - # The very first authorize of the worker found nothing retained. - self.assertEqual(outcomes["miss"], 1) - - def test_expired_retention_falls_back_to_lane_start(self) -> None: - server = resume_coordinator(vardiff_resume_ttl_seconds=60.0) - first = connect_client(server, 1) - authorize(server, first) - converge_and_disconnect(server, first, Decimal("262144")) - - second = connect_client(server, 2) - frozen_now = time.monotonic() + 61.0 - with mock.patch.object(time, "monotonic", return_value=frozen_now): - authorize(server, second) - - self.assertEqual(second.share_difficulty, STANDARD_LANE_START) - self.assertEqual(resume_outcomes(server)["expired"], 1) - - def test_implausible_retained_value_is_clamped_to_the_lane_ceiling(self) -> None: - server = resume_coordinator() - service = server._ensure_vardiff_service() - service.session_difficulty_store.record( - ("default", PAYOUT_ADDRESS), - Decimal("1e15"), - now=time.monotonic(), - share_backed=True, - ) - - state = connect_client(server, 1) - authorize(server, state) - - # min(lane max, lane start * default 1024x factor). - config = standard_lane_config() - expected = min( - config.max_difficulty, - config.startup_difficulty * Decimal("1024"), - ) - self.assertEqual(expected, Decimal("16777216")) - self.assertEqual(state.share_difficulty, expected) - self.assertNotEqual(state.share_difficulty, Decimal("1e15")) - self.assertEqual(resume_outcomes(server)["clamped"], 1) - - def test_retention_is_scoped_per_lane(self) -> None: - server = resume_coordinator() - first = connect_client( - server, - 1, - listener_name="highdiff", - lane_config=highdiff_vardiff_config(), - ) - authorize(server, first) - converge_and_disconnect(server, first, Decimal("2000000")) - - second = connect_client(server, 2, listener_name="default") - authorize(server, second) - - # A lane switch is a miss that behaves exactly like today. - self.assertEqual(second.share_difficulty, STANDARD_LANE_START) - outcomes = resume_outcomes(server) - self.assertEqual(outcomes["miss"], 2) - self.assertEqual(outcomes["resumed"], 0) - self.assertEqual(outcomes["clamped"], 0) - - def test_explicit_password_difficulty_outranks_the_resume(self) -> None: - server = resume_coordinator() - service = server._ensure_vardiff_service() - service.session_difficulty_store.record( - ("default", PAYOUT_ADDRESS), - Decimal("262144"), - now=time.monotonic(), - share_backed=True, - ) - - state = connect_client(server, 1) - authorize(server, state, password="d=32768") - - self.assertEqual(state.share_difficulty, Decimal("32768")) - self.assertIsNone(state.pending_share_difficulty) - - def test_md_only_password_clamps_the_resumed_value(self) -> None: - server = resume_coordinator() - service = server._ensure_vardiff_service() - service.session_difficulty_store.record( - ("default", PAYOUT_ADDRESS), - Decimal("262144"), - now=time.monotonic(), - share_backed=True, - ) - - state = connect_client(server, 1) - authorize(server, state, password="md=524288") - - # The md= floor applies to the just-resumed value, not the lane start. - self.assertEqual(state.share_difficulty, Decimal("524288")) - self.assertIsNone(state.pending_share_difficulty) - - def test_reauthorize_of_a_live_session_never_re_resumes(self) -> None: - server = resume_coordinator() - state = connect_client(server, 1) - authorize(server, state) - with server._client_vardiff_lock(state): - state.share_difficulty = Decimal("65536") - service = server._ensure_vardiff_service() - service.session_difficulty_store.record( - ("default", PAYOUT_ADDRESS), - Decimal("262144"), - now=time.monotonic(), - share_backed=True, - ) - - authorize(server, state) - - self.assertEqual(state.share_difficulty, Decimal("65536")) - self.assertEqual(resume_outcomes(server)["resumed"], 0) - - def test_silent_reconnect_cycles_never_extend_the_retention_ttl(self) -> None: - # The review repro: with the idle step-down sweep disabled (as - # deployed), a session resumed above its true hashrate produces no - # accepted shares and cannot step itself down. If each disconnect - # re-stamped the entry, cycles shorter than the TTL would keep that - # value adoptable forever and the short TTL would protect nothing. - server = resume_coordinator(vardiff_resume_ttl_seconds=900.0) - # The +900.0s cycle probes the TTL boundary exactly, where the store - # keeps the entry (expiry is strictly age > TTL). A real time. - # monotonic() reading makes that comparison depend on float - # rounding: (start + 900.0) - start exceeds 900.0 for roughly one in - # five thousand captured clock values, flipping the boundary cycle - # to "expired" and failing the test. Every record and lookup below - # runs under frozen_clock, so an exactly-representable constant - # makes the boundary arithmetic exact instead of lucky. - start = 1_000_000.0 - with frozen_clock(start): - first = connect_client(server, 1) - authorize(server, first) - converge_and_disconnect(server, first, Decimal("262144")) - - # Three reconnect cycles inside the TTL, each submitting nothing. - for index, offset in enumerate((300.0, 600.0, 900.0), start=2): - state = silent_reconnect_cycle(server, index, start + offset) - self.assertEqual( - state.share_difficulty, - Decimal("262144"), - f"cycle at +{offset}s should still resume", - ) - - # The entry ages from the original convergence at `start`, not from - # the last silent disconnect: it is gone one cycle later. - with frozen_clock(start + 1200.0): - last = connect_client(server, 5) - authorize(server, last) - - self.assertEqual(last.share_difficulty, STANDARD_LANE_START) - outcomes = resume_outcomes(server) - self.assertEqual(outcomes["resumed"], 3) - self.assertEqual(outcomes["expired"], 1) - self.assertIsNone(retained_difficulty(server, now=start + 1200.0)) - - def test_reconnect_with_an_accepted_share_refreshes_the_ttl(self) -> None: - # The counterpart: an accepted share re-validates the retained value, - # so its TTL restarts and the session keeps resuming. - server = resume_coordinator(vardiff_resume_ttl_seconds=900.0) - service = server._ensure_vardiff_service() - start = time.monotonic() - with frozen_clock(start): - first = connect_client(server, 1) - authorize(server, first) - converge_and_disconnect(server, first, Decimal("262144")) - - with frozen_clock(start + 800.0): - second = connect_client(server, 2) - authorize(server, second) - self.assertEqual(second.share_difficulty, Decimal("262144")) - # Open the vardiff window at the injected instant so the share - # below only marks the connection, leaving the retarget path - # (covered separately) out of this test. - with server._client_vardiff_lock(second): - second.vardiff_window_started_monotonic = start + 800.0 - service.note_accepted(second, Decimal("262144")) - self.assertTrue(second.vardiff_accepted_any) - self.assertIsNone(second.pending_share_difficulty) - server.disconnect_client(second) - - with frozen_clock(start + 1600.0): - third = connect_client(server, 3) - authorize(server, third) - - self.assertEqual(third.share_difficulty, Decimal("262144")) - outcomes = resume_outcomes(server) - self.assertEqual(outcomes["resumed"], 2) - self.assertEqual(outcomes["expired"], 0) - - def test_disconnect_retains_the_delivered_difficulty_not_a_pending_one( - self, - ) -> None: - # pending_share_difficulty is advertised with a FUTURE job, so a - # disconnect racing an in-flight retarget must not retain a value the - # miner was never given. A retarget that did land records itself at - # its own commit point. - server = resume_coordinator() - state = connect_client(server, 1) - authorize(server, state) - with server._client_vardiff_lock(state): - state.share_difficulty = Decimal("262144") - state.pending_share_difficulty = Decimal("1048576") - - server.disconnect_client(state) - - self.assertEqual(retained_difficulty(server), Decimal("262144")) - - def test_committed_retarget_retains_the_new_difficulty(self) -> None: - # The `if sent:` commit point inside retarget_locked: a delivered - # retarget retains its own value, with share evidence taken from the - # window that drove it. - server = resume_coordinator() - service = server._ensure_vardiff_service() - state = connect_client(server, 1) - authorize(server, state) - recorded: list[tuple[Decimal, bool]] = [] - store = service.session_difficulty_store - store_record = store.record - - def spy(key, difficulty, *, now, share_backed): # type: ignore[no-untyped-def] - recorded.append((difficulty, share_backed)) - store_record(key, difficulty, now=now, share_backed=share_backed) - - store.record = spy # type: ignore[method-assign] - - # A share-driven window at four times the target share rate. - server.retarget_client( - state, - current_difficulty=STANDARD_LANE_START, - accepted_shares=80, - submitted_shares=80, - accepted_difficulty=STANDARD_LANE_START * 80, - elapsed_seconds=Decimal("300"), - ) - - self.assertEqual(state.pending_share_difficulty, Decimal("65536")) - self.assertEqual(recorded, [(Decimal("65536"), True)]) - self.assertEqual(retained_difficulty(server), Decimal("65536")) - - # A zero-share window steps back down; it carries no share evidence, - # but it moves the value, so the store refreshes the TTL anyway. - server.retarget_client( - state, - current_difficulty=Decimal("65536"), - accepted_shares=0, - submitted_shares=0, - accepted_difficulty=Decimal("0"), - elapsed_seconds=Decimal("300"), - ) - - self.assertEqual(recorded[-1], (STANDARD_LANE_START, False)) - self.assertEqual(retained_difficulty(server), STANDARD_LANE_START) - - def test_explicit_password_difficulty_counts_an_overridden_resume(self) -> None: - server = resume_coordinator() - service = server._ensure_vardiff_service() - service.session_difficulty_store.record( - ("default", PAYOUT_ADDRESS), - Decimal("262144"), - now=time.monotonic(), - share_backed=True, - ) - - state = connect_client(server, 1) - authorize(server, state, password="d=32768") - - self.assertEqual(state.share_difficulty, Decimal("32768")) - outcomes = resume_outcomes(server) - # resumed stays an attempt counter; overridden subtracts the ones an - # explicit request replaced, so resumed + clamped - overridden is the - # number of resumes that actually stuck. - self.assertEqual(outcomes["resumed"], 1) - self.assertEqual(outcomes["overridden"], 1) - - def test_a_resume_that_sticks_is_not_counted_as_overridden(self) -> None: - server = resume_coordinator() - service = server._ensure_vardiff_service() - service.session_difficulty_store.record( - ("default", PAYOUT_ADDRESS), - Decimal("262144"), - now=time.monotonic(), - share_backed=True, - ) - - # No password request at all, then a request that resolves to exactly - # the resumed value: neither supersedes the resume. - first = connect_client(server, 1) - authorize(server, first) - second = connect_client(server, 2) - authorize(server, second, password="d=262144") - - self.assertEqual(first.share_difficulty, Decimal("262144")) - self.assertEqual(second.share_difficulty, Decimal("262144")) - outcomes = resume_outcomes(server) - self.assertEqual(outcomes["resumed"], 2) - self.assertEqual(outcomes["overridden"], 0) - - def test_failed_username_reservation_neither_resumes_nor_retains(self) -> None: - server = resume_coordinator() - service = server._ensure_vardiff_service() - service.session_difficulty_store.record( - ("default", PAYOUT_ADDRESS), - Decimal("262144"), - now=time.monotonic(), - share_backed=True, - ) - server.reserve_client_username = lambda client, worker: False - state = connect_client(server, 1) - - with self.assertRaises(StratumError): - authorize(server, state) - - self.assertFalse(state.authorized) - self.assertEqual(state.share_difficulty, STANDARD_LANE_START) - outcomes = resume_outcomes(server) - self.assertEqual(outcomes["resumed"], 0) - self.assertEqual(outcomes["clamped"], 0) - # No reserved worker means no retention identity, so the disconnect - # that follows the rejection writes nothing either. - server.disconnect_client(state) - self.assertEqual(service.session_difficulty_store.snapshot()["records"], 1) - self.assertEqual(retained_difficulty(server), Decimal("262144")) - - def test_kill_switch_restores_lane_start_reconnects(self) -> None: - server = resume_coordinator(vardiff_resume_enabled=False) - first = connect_client(server, 1) - authorize(server, first) - converge_and_disconnect(server, first, Decimal("262144")) - - second = connect_client(server, 2) - authorize(server, second) - - self.assertEqual(second.share_difficulty, STANDARD_LANE_START) - snapshot = server.vardiff_convergence_snapshot() - self.assertEqual(snapshot["retained_sessions"], 0) - outcomes = snapshot["resume_outcomes"] - assert isinstance(outcomes, dict) - self.assertEqual(outcomes["disabled"], 2) - self.assertEqual(outcomes["resumed"], 0) - - -class ResumeConfigTests(unittest.TestCase): - BASE_ENV = { - "QBIT_RPC_HOST": "127.0.0.1", - "QBIT_RPC_USER": "user", - "QBIT_RPC_PASSWORD": "password", - "PRISM_ALLOW_BUNDLE_EMBEDDED_LEDGER_KEY": "1", - "PRISM_ALLOW_TEST_SIGNING_SEEDS": "1", - } - - def test_defaults_and_overrides(self) -> None: - stratum = load_coordinator_config(self.BASE_ENV).stratum - self.assertTrue(stratum.vardiff_resume_enabled) - self.assertEqual(stratum.vardiff_resume_ttl_seconds, 900.0) - self.assertEqual(stratum.vardiff_resume_max_entries, 8192) - self.assertEqual(stratum.vardiff_resume_max_start_factor, Decimal("1024")) - - tuned = load_coordinator_config( - { - **self.BASE_ENV, - "PRISM_STRATUM_VARDIFF_RESUME": "0", - "PRISM_STRATUM_VARDIFF_RESUME_TTL_SECONDS": "120", - "PRISM_STRATUM_VARDIFF_RESUME_MAX_ENTRIES": "16", - "PRISM_STRATUM_VARDIFF_RESUME_MAX_START_FACTOR": "64", - } - ).stratum - self.assertFalse(tuned.vardiff_resume_enabled) - self.assertEqual(tuned.vardiff_resume_ttl_seconds, 120.0) - self.assertEqual(tuned.vardiff_resume_max_entries, 16) - self.assertEqual(tuned.vardiff_resume_max_start_factor, Decimal("64")) - - def test_env_sizes_the_coordinator_store_end_to_end(self) -> None: - # env -> StratumConfig -> PrismCoordinator attributes -> - # SessionDifficultyStore. The service reads those attributes through - # getattr defaults, so a rename anywhere in the chain would silently - # fall back to the defaults instead of failing; this asserts the - # configured values actually reach the store. - with tempfile.TemporaryDirectory() as temp_dir: - root = Path(temp_dir) - source = { - **self.BASE_ENV, - "PRISM_ALLOW_MEMORY_LEDGER": "1", - "PRISM_AUDIT_DIR": str(root), - "PRISM_EVIDENCE_PATH": str(root / "evidence.json"), - "PRISM_STRATUM_VARDIFF_RESUME_TTL_SECONDS": "123", - "PRISM_STRATUM_VARDIFF_RESUME_MAX_ENTRIES": "17", - } - config = load_coordinator_config(source) - with mock.patch.object( - JsonRpc, "call", side_effect=RuntimeError("offline") - ): - server = PrismCoordinator(config) - - self.assertEqual(server.vardiff_resume_ttl_seconds, 123.0) - self.assertEqual(server.vardiff_resume_max_entries, 17) - store = server._ensure_vardiff_service().session_difficulty_store - self.assertEqual(store.ttl_seconds, 123.0) - self.assertEqual(store.max_entries, 17) - self.assertTrue(store.enabled) - - def test_kill_switch_env_disables_the_coordinator_store(self) -> None: - with tempfile.TemporaryDirectory() as temp_dir: - root = Path(temp_dir) - source = { - **self.BASE_ENV, - "PRISM_ALLOW_MEMORY_LEDGER": "1", - "PRISM_AUDIT_DIR": str(root), - "PRISM_EVIDENCE_PATH": str(root / "evidence.json"), - "PRISM_STRATUM_VARDIFF_RESUME": "0", - } - config = load_coordinator_config(source) - with mock.patch.object( - JsonRpc, "call", side_effect=RuntimeError("offline") - ): - server = PrismCoordinator(config) - - self.assertFalse(server.vardiff_resume_enabled) - store = server._ensure_vardiff_service().session_difficulty_store - self.assertEqual(store.max_entries, 0) - self.assertFalse(store.enabled) - - def test_start_factor_below_one_fails_startup(self) -> None: - with self.assertRaises(SystemExit) as raised: - load_coordinator_config( - { - **self.BASE_ENV, - "PRISM_STRATUM_VARDIFF_RESUME_MAX_START_FACTOR": "0.5", - } - ) - self.assertEqual( - str(raised.exception), - "PRISM_STRATUM_VARDIFF_RESUME_MAX_START_FACTOR must be at least 1", - ) - - -class ServiceRuntime: - """Lightweight VardiffRuntime fake mirroring test_prism_vardiff_service.""" - - def __init__(self) -> None: - self.lock = threading.RLock() - # A list rather than a set: SimpleNamespace clients are unhashable, - # and convergence_snapshot only needs an iterable membership view. - self.clients: list[object] = [] - self.stop_event = threading.Event() - self.vardiff_config = standard_lane_config() - self.share_difficulty = STANDARD_LANE_START - self.vardiff_idle_sweep_seconds = 1.0 - - -def service_client(**overrides: object) -> SimpleNamespace: - values: dict[str, object] = dict( - vardiff_config=None, - listener_vardiff_config=None, - listener_name="default", - pending_share_difficulty=None, - share_difficulty=STANDARD_LANE_START, - vardiff_window_started_monotonic=time.monotonic(), - vardiff_window_accepted=0, - vardiff_window_submitted=0, - vardiff_window_work=Decimal("0"), - vardiff_difficulty_estimate=None, - ) - values.update(overrides) - return SimpleNamespace(**values) - - -class ConvergenceObservabilityTests(unittest.TestCase): - def test_snapshot_counts_lanes_ceiling_sessions_and_retention(self) -> None: - runtime = ServiceRuntime() - service = VardiffService(runtime) # type: ignore[arg-type] - at_ceiling = service_client( - share_difficulty=runtime.vardiff_config.max_difficulty, - listener_name="highdiff", - ) - converging = service_client() - runtime.clients.extend([at_ceiling, converging]) - - service.note_accepted(at_ceiling, Decimal("4")) # type: ignore[arg-type] - service.note_accepted(converging, Decimal("4")) # type: ignore[arg-type] - service.note_accepted(converging, Decimal("4")) # type: ignore[arg-type] - service.session_difficulty_store.record( - ("default", "miner-a"), - Decimal("262144"), - now=time.monotonic(), - share_backed=True, - ) - - snapshot = service.convergence_snapshot() - - self.assertEqual(snapshot["sessions_at_max_difficulty"], 1) - self.assertEqual( - snapshot["lane_accepted_shares"], {"highdiff": 1, "default": 2} - ) - self.assertEqual(snapshot["retained_sessions"], 1) - self.assertEqual( - snapshot["resume_outcomes"], - {outcome: 0 for outcome in PRISM_VARDIFF_RESUME_OUTCOMES}, - ) - - def test_lane_accepted_counts_where_vardiff_is_disabled(self) -> None: - runtime = ServiceRuntime() - runtime.vardiff_config = vardiff.VardiffConfig( - enabled=False, - target_share_interval_seconds=Decimal("15"), - min_difficulty=Decimal("1024"), - max_difficulty=Decimal("4294967296"), - retarget_interval_seconds=Decimal("300"), - max_step_factor=Decimal("4"), - startup_difficulty=STANDARD_LANE_START, - ) - service = VardiffService(runtime) # type: ignore[arg-type] - - service.note_accepted(service_client(), Decimal("4")) # type: ignore[arg-type] - - snapshot = service.convergence_snapshot() - self.assertEqual(snapshot["lane_accepted_shares"], {"default": 1}) - self.assertEqual(snapshot["sessions_at_max_difficulty"], 0) - - def test_renderer_emits_deterministic_lane_and_outcome_series(self) -> None: - started_monotonic = time.monotonic() - port = SimpleNamespace( - vardiff_convergence_snapshot=lambda: { - "sessions_at_max_difficulty": 2, - "lane_accepted_shares": {"default": 30, "extra": 5}, - "resume_outcomes": {"resumed": 4}, - "retained_sessions": 3, - }, - listener_profiles=[ - SimpleNamespace(name="default"), - SimpleNamespace(name="highdiff"), - ], - prometheus_label_value=lambda value: value, - started_monotonic=started_monotonic, - ) - renderer = MetricsRenderer(port) # type: ignore[arg-type] - - with mock.patch.object( - time, "monotonic", return_value=started_monotonic + 10.0 - ): - lines = renderer.vardiff_convergence_metrics_lines() - text = "\n".join(lines) - - self.assertIn("qbit_prism_vardiff_sessions_at_max_difficulty 2", text) - self.assertIn( - 'qbit_prism_vardiff_lane_accepted_shares_total{lane="default"} 30', - text, - ) - self.assertIn( - 'qbit_prism_vardiff_lane_accepted_shares_total{lane="highdiff"} 0', - text, - ) - self.assertIn( - 'qbit_prism_vardiff_lane_accepted_shares_total{lane="extra"} 5', - text, - ) - self.assertIn( - 'qbit_prism_vardiff_lane_accepted_shares_per_second{lane="default"} 3', - text, - ) - for outcome in PRISM_VARDIFF_RESUME_OUTCOMES: - self.assertIn( - f'qbit_prism_vardiff_resume_total{{outcome="{outcome}"}}', - text, - ) - self.assertIn('qbit_prism_vardiff_resume_total{outcome="resumed"} 4', text) - self.assertIn("qbit_prism_vardiff_resume_retained_sessions 3", text) - # Configured lanes lead in profile order; observed-only lanes follow. - default_index = text.index('shares_total{lane="default"}') - highdiff_index = text.index('shares_total{lane="highdiff"}') - extra_index = text.index('shares_total{lane="extra"}') - self.assertLess(default_index, highdiff_index) - self.assertLess(highdiff_index, extra_index) - - -if __name__ == "__main__": - unittest.main() diff --git a/tests/test_prism_vardiff_service.py b/tests/test_prism_vardiff_service.py deleted file mode 100644 index 38bc4688e..000000000 --- a/tests/test_prism_vardiff_service.py +++ /dev/null @@ -1,108 +0,0 @@ -#!/usr/bin/env python3 -"""Direct tests for the PRISM vardiff owner.""" - -from __future__ import annotations - -from decimal import Decimal -from types import SimpleNamespace -import threading -import time -import unittest - -from lab.auxpow import vardiff -from lab.prism.vardiff_service import VardiffService - - -def config(*, interval: str = "1") -> vardiff.VardiffConfig: - return vardiff.VardiffConfig( - enabled=True, - target_share_interval_seconds=Decimal("15"), - min_difficulty=Decimal("1"), - max_difficulty=Decimal("1024"), - retarget_interval_seconds=Decimal(interval), - max_step_factor=Decimal("4"), - startup_difficulty=Decimal("4"), - max_step_down_factor=Decimal("4"), - ewma_alpha=Decimal("0.4"), - retarget_tolerance=Decimal("0.25"), - ) - - -class Runtime: - def __init__(self) -> None: - self.lock = threading.RLock() - self.clients: set[object] = set() - self.stop_event = threading.Event() - self.vardiff_config = config() - self.share_difficulty = Decimal("2") - self.vardiff_idle_sweep_seconds = 1.0 - self.retargets: list[dict[str, object]] = [] - - def retarget_client(self, client: object, **kwargs: object) -> bool: - self.retargets.append({"client": client, **kwargs}) - return True - - -def client() -> SimpleNamespace: - return SimpleNamespace( - vardiff_config=None, - listener_vardiff_config=None, - minimum_advertised_difficulty=Decimal("0"), - pending_share_difficulty=None, - share_difficulty=Decimal("4"), - vardiff_window_started_monotonic=time.monotonic() - 2, - vardiff_window_accepted=0, - vardiff_window_submitted=1, - vardiff_window_work=Decimal("0"), - vardiff_difficulty_estimate=None, - ) - - -class VardiffServiceTests(unittest.TestCase): - def test_accepted_window_is_captured_and_reset_before_retarget(self) -> None: - runtime = Runtime() - service = VardiffService(runtime) # type: ignore[arg-type] - state = client() - - service.note_accepted(state, Decimal("3")) # type: ignore[arg-type] - - self.assertEqual(len(runtime.retargets), 1) - retarget = runtime.retargets[0] - self.assertEqual(retarget["accepted_shares"], 1) - self.assertEqual(retarget["submitted_shares"], 1) - self.assertEqual(retarget["accepted_difficulty"], Decimal("3")) - self.assertEqual(state.vardiff_window_accepted, 0) - self.assertEqual(state.vardiff_window_submitted, 0) - self.assertEqual(state.vardiff_window_work, Decimal("0")) - - def test_speculative_idle_rollback_requires_unchanged_reset_stamp(self) -> None: - state = client() - original = (10.0, 0, 0, Decimal("0")) - state.vardiff_window_started_monotonic = 20.0 - state.vardiff_window_submitted = 0 - - VardiffService.restore_idle_window_state(state, original, 20.0) # type: ignore[arg-type] - self.assertEqual(state.vardiff_window_started_monotonic, 10.0) - - state.vardiff_window_started_monotonic = 30.0 - state.vardiff_window_submitted = 1 - VardiffService.restore_idle_window_state(state, original, 30.0) # type: ignore[arg-type] - self.assertEqual(state.vardiff_window_started_monotonic, 30.0) - self.assertEqual(state.vardiff_window_submitted, 1) - - def test_idle_metrics_are_service_owned(self) -> None: - service = VardiffService(Runtime()) # type: ignore[arg-type] - service.record_idle_skip("busy") - service.observe_idle_seconds("sweep", 0.01) - - metrics = "\n".join(service.metrics_lines()) - - self.assertIn( - 'qbit_prism_vardiff_idle_skips_total{reason="busy"} 1', - metrics, - ) - self.assertIn("qbit_prism_vardiff_idle_sweep_seconds_count 1", metrics) - - -if __name__ == "__main__": - unittest.main() diff --git a/tests/test_prism_window_pipeline_rust.py b/tests/test_prism_window_pipeline_rust.py deleted file mode 100644 index a00366346..000000000 --- a/tests/test_prism_window_pipeline_rust.py +++ /dev/null @@ -1,1472 +0,0 @@ -#!/usr/bin/env python3 -"""Coordinator-side tests for the Rust payout-window pipeline slice. - -Why these tests exist: the byte-level equivalence of the Rust fold is proven -by the parity oracle (``tests.test_window_pipeline_parity`` with the -``rust-daemon`` adapter); what still needs proof is the coordinator's side of -the contract -- that the ``PRISM_WINDOW_PIPELINE_RUST`` switch defaults to -Python and fails closed, that the daemon path preserves the append-epoch and -scan-anchor fences, and that every non-success shape (daemon death, -``needs_full`` after a respawn, an ``advance`` invariant failure, a protocol -mismatch, a value outside the daemon's declared integer widths) degrades -exactly like its in-process counterpart instead of inventing a new failure -mode. The daemon here is the fake serve builder, which serves prepare_window -with the real Python fold so mirror digests verify byte-exactly without a -Rust build, and which declares the real daemon's integer widths. -""" -# ruff: noqa: F403, F405 - -from __future__ import annotations - -import dataclasses -import hashlib -import json -import os -import tempfile -import threading -import time -import unittest -from pathlib import Path - -from lab.prism import share_ledger as share_ledger_module -from lab.prism.coordinator_config import ( - env_window_pipeline_rust, - load_coordinator_config, -) -from lab.prism.share_ledger import ( - DaemonShareJsonSequence, - DaemonShareWindowMirror, - DaemonWindowMirrorDivergence, - IncrementalShareWindow, -) -from tests.prism_coordinator_test_support import * - - -def _canonical_items(window: IncrementalShareWindow) -> bytes: - return b",".join( - page.canonical_json_items - for page in window.pages - if page.canonical_json_items - ) - - -def _python_window_for( - ledger: IncrementalRecordingLedger, - *, - anchor_ms: int, - network_difficulty: int, -) -> IncrementalShareWindow: - """The in-process oracle over the same ledger rows, counters untouched.""" - records = list( - SingleWriterShareLedger.snapshot_at_job_issue( - ledger, - anchor_ms, - window_weight=None, - ) - ) - return IncrementalShareWindow.from_full_snapshot( - records, - anchor_job_issued_at_ms=anchor_ms, - window_weight=8 * 2 * int(network_difficulty), - ) - - -class WindowPipelineSwitchTests(unittest.TestCase): - """PRISM_WINDOW_PIPELINE_RUST: default Python, strict values, inert off.""" - - def test_switch_defaults_to_python(self) -> None: - self.assertFalse(env_window_pipeline_rust(environ={})) - - def test_switch_accepts_boolean_tokens(self) -> None: - for raw in ("1", "true", "YES", "On"): - with self.subTest(raw=raw): - self.assertTrue( - env_window_pipeline_rust( - environ={"PRISM_WINDOW_PIPELINE_RUST": raw} - ) - ) - for raw in ("0", "false", "NO", "Off"): - with self.subTest(raw=raw): - self.assertFalse( - env_window_pipeline_rust( - environ={"PRISM_WINDOW_PIPELINE_RUST": raw} - ) - ) - - def test_switch_fails_closed_on_non_boolean_values(self) -> None: - for raw in ("2", "banana", "tru"): - with self.subTest(raw=raw): - with self.assertRaisesRegex( - SystemExit, "PRISM_WINDOW_PIPELINE_RUST must be a boolean" - ): - env_window_pipeline_rust( - environ={"PRISM_WINDOW_PIPELINE_RUST": raw} - ) - - def test_coordinator_config_load_validates_the_switch_at_startup(self) -> None: - with tempfile.TemporaryDirectory() as tmp: - root = Path(tmp) - base = { - "QBIT_RPC_HOST": "qbit.example", - "QBIT_RPC_USER": "rpc-user", - "QBIT_RPC_PASSWORD": "rpc-password", - "PRISM_ALLOW_MEMORY_LEDGER": "1", - "PRISM_ALLOW_TEST_SIGNING_SEEDS": "1", - "PRISM_ALLOW_BUNDLE_EMBEDDED_LEDGER_KEY": "1", - "PRISM_AUDIT_DIR": str(root), - "PRISM_EVIDENCE_PATH": str(root / "evidence.json"), - } - config = load_coordinator_config(base) - self.assertFalse(config.jobs.window_pipeline_rust_enabled) - enabled = load_coordinator_config( - {**base, "PRISM_WINDOW_PIPELINE_RUST": "1"} - ) - self.assertTrue(enabled.jobs.window_pipeline_rust_enabled) - with self.assertRaisesRegex( - SystemExit, "PRISM_WINDOW_PIPELINE_RUST must be a boolean" - ): - load_coordinator_config( - {**base, "PRISM_WINDOW_PIPELINE_RUST": "maybe"} - ) - - -class DaemonShareWindowMirrorTests(unittest.TestCase): - """The digest-verified byte mirror and its lazy JSON sequence.""" - - def _window(self) -> IncrementalShareWindow: - records = [ - stamped_pending_share(999_900 + seq) for seq in range(3) - ] - ledger = SingleWriterShareLedger() - appended = [ledger.append(record) for record in records] - return IncrementalShareWindow.from_full_snapshot( - appended, - anchor_job_issued_at_ms=1_000_000, - window_weight=1_000, - ) - - def test_full_mirror_verifies_and_parses_lazily(self) -> None: - window = self._window() - items = _canonical_items(window) - digest = window.json_records().canonical_json_sha256() - mirror = DaemonShareWindowMirror.from_full_items( - anchor_job_issued_at_ms=1_000_000, - window_weight=1_000, - page_size=512, - record_count=window.record_count, - canonical_items=items, - share_snapshot_sha256=digest, - ) - sequence = mirror.json_records() - self.assertIsInstance(sequence, DaemonShareJsonSequence) - self.assertEqual(len(sequence), window.record_count) - self.assertIsNone(sequence._parsed) - self.assertEqual(sequence.canonical_json_sha256(), digest) - # The digest never forces a parse; iteration does, exactly once. - self.assertIsNone(sequence._parsed) - self.assertEqual(list(sequence), list(window.json_records())) - self.assertEqual(sequence[0], window.json_records()[0]) - - def _awkward_window(self, count: int = 40) -> IncrementalShareWindow: - """Records whose canonical bytes carry every escape the parser meets.""" - ledger = SingleWriterShareLedger() - appended = [] - for seq in range(count): - # Stamped well inside the anchor so every record is eligible - # whatever the count. - pending = stamped_pending_share(900_000 + seq) - pending = dataclasses.replace( - pending, - share_id=f"miner-\u00e9.rig{seq}:\U0001F600,}}{{\"\\", - miner_id=f"miner-{seq % 3}-\u2603", - share_difficulty=(1 << 130) + seq, - credit_policy='stale-grace' if seq % 2 else None, - ) - appended.append(ledger.append(pending)) - # A weight past the whole snapshot's total, so nothing expires. - return IncrementalShareWindow.from_full_snapshot( - appended, - anchor_job_issued_at_ms=1_000_000, - window_weight=(1 << 131) * count, - ) - - def test_lazy_parse_walks_records_across_decode_chunks(self) -> None: - window = self._awkward_window() - items = _canonical_items(window) - expected = json.loads(b"[" + items + b"]") - self.assertEqual(expected, list(window.json_records())) - # Chunk sizes that cut records, escapes and separators everywhere; - # the last is the shipped default. - for chunk_bytes in (1, 2, 3, 5, 64, 1_000, None): - with self.subTest(chunk_bytes=chunk_bytes), patch.object( - share_ledger_module, - "CANONICAL_ITEMS_DECODE_CHUNK_BYTES", - chunk_bytes or share_ledger_module.CANONICAL_ITEMS_DECODE_CHUNK_BYTES, - ): - sequence = DaemonShareJsonSequence(items, len(expected)) - self.assertIsNone(sequence._parsed) - self.assertEqual(list(sequence), expected) - self.assertEqual(sequence[0], expected[0]) - self.assertEqual(sequence[-1], expected[-1]) - self.assertEqual(sequence[3:7], tuple(expected[3:7])) - self.assertEqual(len(sequence), len(expected)) - # The parse ran exactly once and was published whole. - self.assertIs(sequence._records(), sequence._parsed) - # Keys are shared across records like a whole-array - # json.loads shares them: one string object per key. - key_ids = {id(key) for record in sequence for key in record} - self.assertLessEqual(len(key_ids), 13) - self.assertEqual( - share_ledger_module._canonical_items_layout(items), - (len(expected), False), - ) - self.assertEqual( - share_ledger_module._canonical_items_layout(items + b","), - (len(expected), True), - ) - - def test_lazy_parse_decodes_raw_utf8_split_across_chunks(self) -> None: - # Not canonical (the daemon escapes non-ASCII), but valid JSON: the - # incremental decoder must never split a multi-byte sequence and - # the digest check, not the parser, is what pins canonicality. - items = b'{"k":"\xc3\xa9\xf0\x9f\x98\x80"},{"k":"\xe2\x98\x83"}' - expected = json.loads(b"[" + items + b"]") - for chunk_bytes in (1, 2, 3, 4, 5, 7, 100): - with self.subTest(chunk_bytes=chunk_bytes), patch.object( - share_ledger_module, - "CANONICAL_ITEMS_DECODE_CHUNK_BYTES", - chunk_bytes, - ): - self.assertEqual( - list(DaemonShareJsonSequence(items, 2)), - expected, - ) - - def test_lazy_parse_rejects_malformed_streams_without_publishing(self) -> None: - window = self._awkward_window(6) - items = _canonical_items(window) - count = window.record_count - broken = { - "truncated record": items[:-1], - "truncated escape": items[: items.index(b"\\ud83d") + 3], - "trailing separator": items + b",", - "leading separator": b"," + items, - "doubled separator": items.replace(b"},{", b"},,{", 1), - "missing separator": items.replace(b"},{", b"}{", 1), - "junk between records": items.replace(b"},{", b"}x{", 1), - "whitespace between records": items.replace(b"},{", b"}, {", 1), - "trailing whitespace": items + b" ", - "non-object item": items.replace(b"},{", b"},1,{", 1), - "invalid utf-8 mid-stream": items.replace(b"},{", b"},\xff{", 1), - "invalid utf-8 in a string": items.replace(b"miner", b"mi\xc3ner", 1), - "truncated multi-byte tail": items + b",{\"k\":\"\xc3", - } - for label, stream in broken.items(): - for chunk_bytes in (1, 4, 64 * 1024): - with self.subTest(label=label, chunk_bytes=chunk_bytes), patch.object( - share_ledger_module, - "CANONICAL_ITEMS_DECODE_CHUNK_BYTES", - chunk_bytes, - ): - with self.assertRaises(DaemonWindowMirrorDivergence): - share_ledger_module._canonical_items_record_count(stream) - sequence = DaemonShareJsonSequence(stream, count) - with self.assertRaises(DaemonWindowMirrorDivergence): - list(sequence) - # Nothing partial is ever published. - self.assertIsNone(sequence._parsed) - # A count the bytes refute is refused after the whole walk, and - # the parse is not memoized either. - miscounted = DaemonShareJsonSequence(items, count + 1) - with self.assertRaisesRegex( - DaemonWindowMirrorDivergence, - f"parsed {count} records where {count + 1}", - ): - miscounted[0] - self.assertIsNone(miscounted._parsed) - self.assertEqual(list(DaemonShareJsonSequence(b"", 0)), []) - - def test_large_mirror_is_counted_and_parsed_chunk_by_chunk(self) -> None: - # Past the decode chunk (64 KiB) several times over, so record - # boundaries land inside chunk boundaries throughout the stream. - window = self._awkward_window(3_000) - items = _canonical_items(window) - self.assertGreater( - len(items), - 4 * share_ledger_module.CANONICAL_ITEMS_DECODE_CHUNK_BYTES, - ) - self.assertEqual(window.record_count, 3_000) - mirror = DaemonShareWindowMirror.from_full_items( - anchor_job_issued_at_ms=1_000_000, - window_weight=(1 << 131) * 3_000, - page_size=512, - record_count=3_000, - canonical_items=items, - share_snapshot_sha256=window.json_records().canonical_json_sha256(), - ) - self.assertEqual( - list(mirror.json_records()), - list(window.json_records()), - ) - - def test_full_mirror_rejects_bytes_that_do_not_hash_to_the_digest(self) -> None: - window = self._window() - digest = window.json_records().canonical_json_sha256() - with self.assertRaises(DaemonWindowMirrorDivergence): - DaemonShareWindowMirror.from_full_items( - anchor_job_issued_at_ms=1_000_000, - window_weight=1_000, - page_size=512, - record_count=window.record_count, - canonical_items=_canonical_items(window) + b" ", - share_snapshot_sha256=digest, - ) - - def test_full_mirror_rejects_a_count_the_bytes_refute(self) -> None: - window = self._window() - items = _canonical_items(window) - digest = window.json_records().canonical_json_sha256() - for declared in (window.record_count - 1, window.record_count + 1): - with self.subTest(declared=declared): - # Eager, beside the digest check: the bytes are right, only - # the count beside them is wrong, and that must not be left - # for whichever consumer first parses the stream. - with self.assertRaisesRegex( - DaemonWindowMirrorDivergence, - f"holds {window.record_count} records where {declared}", - ): - DaemonShareWindowMirror.from_full_items( - anchor_job_issued_at_ms=1_000_000, - window_weight=1_000, - page_size=512, - record_count=declared, - canonical_items=items, - share_snapshot_sha256=digest, - ) - - def test_full_mirror_rejects_a_stream_that_is_not_whole_records( - self, - ) -> None: - window = self._window() - items = _canonical_items(window) - for broken in (items + b",", b"," + items, items[:-1]): - with self.subTest(broken=broken[:8]): - digest = hashlib.sha256(b"[" + broken + b"]").hexdigest() - with self.assertRaises(DaemonWindowMirrorDivergence): - DaemonShareWindowMirror.from_full_items( - anchor_job_issued_at_ms=1_000_000, - window_weight=1_000, - page_size=512, - record_count=window.record_count, - canonical_items=broken, - share_snapshot_sha256=digest, - ) - - def test_advanced_applies_drop_and_append_surgery_exactly(self) -> None: - window = self._window() - mirror = DaemonShareWindowMirror.from_full_items( - anchor_job_issued_at_ms=1_000_000, - window_weight=1_000, - page_size=512, - record_count=window.record_count, - canonical_items=_canonical_items(window), - share_snapshot_sha256=window.json_records().canonical_json_sha256(), - ) - ledger = SingleWriterShareLedger(first_share_seq=10) - delta = [ledger.append(stamped_pending_share(1_000_010))] - advanced_window, _stats = window.advance( - delta, - anchor_job_issued_at_ms=1_000_020, - ) - old_items = _canonical_items(window) - new_items = _canonical_items(advanced_window) - self.assertTrue(new_items.startswith(old_items)) - appended = new_items[len(old_items):] - advanced = mirror.advanced( - anchor_job_issued_at_ms=1_000_020, - record_count=advanced_window.record_count, - retained_drop_bytes=0, - appended_items=appended, - share_snapshot_sha256=( - advanced_window.json_records().canonical_json_sha256() - ), - ) - self.assertEqual(advanced.canonical_items, new_items) - self.assertEqual(advanced.anchor_job_issued_at_ms, 1_000_020) - # The advance's count is counted from the surgery, not accepted. - with self.assertRaisesRegex( - DaemonWindowMirrorDivergence, - "records where", - ): - mirror.advanced( - anchor_job_issued_at_ms=1_000_020, - record_count=advanced_window.record_count + 1, - retained_drop_bytes=0, - appended_items=appended, - share_snapshot_sha256=( - advanced_window.json_records().canonical_json_sha256() - ), - ) - with self.assertRaises(DaemonWindowMirrorDivergence): - mirror.advanced( - anchor_job_issued_at_ms=1_000_020, - record_count=advanced_window.record_count, - retained_drop_bytes=1, - appended_items=appended, - share_snapshot_sha256=( - advanced_window.json_records().canonical_json_sha256() - ), - ) - - def test_anchor_only_advance_reuses_bytes_and_checks_the_digest(self) -> None: - window = self._window() - digest = window.json_records().canonical_json_sha256() - mirror = DaemonShareWindowMirror.from_full_items( - anchor_job_issued_at_ms=1_000_000, - window_weight=1_000, - page_size=512, - record_count=window.record_count, - canonical_items=_canonical_items(window), - share_snapshot_sha256=digest, - ) - advanced = mirror.advanced( - anchor_job_issued_at_ms=1_000_050, - record_count=window.record_count, - retained_drop_bytes=0, - appended_items=b"", - share_snapshot_sha256=digest, - ) - self.assertIs(advanced.canonical_items, mirror.canonical_items) - with self.assertRaisesRegex( - DaemonWindowMirrorDivergence, - "records where", - ): - mirror.advanced( - anchor_job_issued_at_ms=1_000_050, - record_count=window.record_count + 1, - retained_drop_bytes=0, - appended_items=b"", - share_snapshot_sha256=digest, - ) - with self.assertRaises(DaemonWindowMirrorDivergence): - mirror.advanced( - anchor_job_issued_at_ms=1_000_050, - record_count=window.record_count, - retained_drop_bytes=0, - appended_items=b"", - share_snapshot_sha256="0" * 64, - ) - - -class DaemonWindowPipelineTestCase(unittest.TestCase): - """Shared coordinator/fake-daemon fixture for the window-pipeline tests.""" - - def _server(self) -> tuple[PrismCoordinator, IncrementalRecordingLedger, object]: - ledger = IncrementalRecordingLedger() - append_incremental_share(ledger, share_seq=1, accepted_at_ms=999_900) - append_incremental_share(ledger, share_seq=2, accepted_at_ms=999_910) - append_incremental_share(ledger, share_seq=3, accepted_at_ms=999_920) - server, rpc = coordinator(ledger=ledger) - server.signing_seed_hex = "42" * 32 - server.ledger_attestation_signing_seed_hex = "43" * 32 - artifacts = server.store_template_artifacts(dict(rpc.template)) - assert artifacts is not None - server._pool_ready_latched = True - server.payout_artifact_min_build_interval_seconds = 60.0 - server.payout_artifact_full_rescan_seconds = 3_600.0 - self.addCleanup(server.shutdown_serve_builder) - return server, ledger, artifacts - - def _rust_env(self, mode: str = "ok", **extra: str): - values = { - "PRISM_WINDOW_PIPELINE_RUST": "1", - "FAKE_SERVE_BUILDER_MODE": mode, - **extra, - } - return patch.dict(os.environ, values) - - def _fake_daemon_command(self): - return patch( - "lab.prism.prism_coordinator.prism_tool_command", - return_value=list(FAKE_SERVE_BUILDER_COMMAND), - ) - - def _serve_counts(self, server: PrismCoordinator) -> dict[str, int]: - with server._serve_builder_metrics_lock: - return dict(server.serve_builder_counts) - - -class WindowPipelineRustMaterializationTests(DaemonWindowPipelineTestCase): - """The daemon-backed materialization path against the fake serve builder.""" - - def test_full_then_advance_materializes_daemon_windows_byte_exactly( - self, - ) -> None: - server, ledger, artifacts = self._server() - clock_ms = [1_000_000] - with self._rust_env(), self._fake_daemon_command(), patch( - "lab.prism.prism_coordinator.now_ms", - side_effect=lambda: clock_ms[0], - ): - initial = server._build_payout_ledger_artifact( - 0, 0, artifacts.network_difficulty - ) - assert initial is not None - self.assertEqual(initial.window_build_mode, "full_rescan") - cached = server._incremental_payout_artifact_window - assert cached is not None - self.assertIsInstance(cached.window, DaemonShareWindowMirror) - self.assertIsInstance(initial.shares_json, DaemonShareJsonSequence) - # The routine path must not have parsed the mirror's dicts. - self.assertIsNone(initial.shares_json._parsed) - oracle = _python_window_for( - ledger, - anchor_ms=int(initial.snapshot_anchor_ms or 0), - network_difficulty=int(artifacts.network_difficulty), - ) - self.assertEqual( - initial.share_snapshot_sha256, - oracle.json_records().canonical_json_sha256(), - ) - self.assertEqual(ledger.full_snapshot_calls, 1) - - append_incremental_share(ledger, share_seq=4, accepted_at_ms=1_000_010) - server.payout_artifact_min_build_interval_seconds = 0.0 - clock_ms[0] = 1_000_020 - advanced = server._build_payout_ledger_artifact( - 0, 0, artifacts.network_difficulty - ) - assert advanced is not None - self.assertEqual(advanced.window_build_mode, "incremental") - self.assertEqual(advanced.window_delta_rows, 1) - self.assertEqual(ledger.full_snapshot_calls, 1) - self.assertEqual(ledger.delta_snapshot_calls, 1) - advanced_oracle = _python_window_for( - ledger, - anchor_ms=int(advanced.snapshot_anchor_ms or 0), - network_difficulty=int(artifacts.network_difficulty), - ) - self.assertEqual( - advanced.share_snapshot_sha256, - advanced_oracle.json_records().canonical_json_sha256(), - ) - # Forcing the rare-path parse must reproduce the oracle's dicts. - self.assertEqual( - list(advanced.shares_json), - list(advanced_oracle.json_records()), - ) - counts = self._serve_counts(server) - self.assertEqual(counts["window_prepares"], 2) - self.assertEqual(counts["spawns"], 1) - self.assertEqual(counts["fallbacks"], 0) - - def test_prepared_window_serves_builds_without_upload_or_parse(self) -> None: - server, ledger, artifacts = self._server() - clock_ms = [1_000_000] - with self._rust_env(), self._fake_daemon_command(), patch( - "lab.prism.prism_coordinator.now_ms", - side_effect=lambda: clock_ms[0], - ): - artifact = server._build_payout_ledger_artifact( - 0, 0, artifacts.network_difficulty - ) - assert artifact is not None - serialization = server._share_window_serialization_for_artifact( - artifact, - artifact.shares_json, - ) - summary = server.build_audit_bundle( - shares=artifact.shares_json, - found_block={ - "block_height": 101, - "coinbase_value_sats": 50_00000000, - "network_difficulty": int(artifacts.network_difficulty), - "anchor_job_issued_at_ms": int( - artifact.snapshot_anchor_ms or 0 - ), - }, - prior_balances=[], - coinbase_script_sig_suffix_hex="00", - summary_only=True, - payout_policy={"policy": "day-one"}, - share_serialization=serialization, # type: ignore[arg-type] - append_invalidation_epoch=artifact.append_invalidation_epoch, - ) - self.assertEqual(summary["transport"], "serve") - # The prepared window is the build cache entry: no upload rode - # along, and the routine build never materialized the dicts. - self.assertFalse(summary["request_had_window"]) - self.assertIsNone(artifact.shares_json._parsed) - counts = self._serve_counts(server) - self.assertEqual(counts["window_uploads"], 0) - self.assertEqual(counts["requests"], 1) - - def test_epoch_bump_between_daemon_call_and_install_discards_artifact( - self, - ) -> None: - server, _ledger, artifacts = self._server() - clock_ms = [1_000_000] - with self._rust_env(), self._fake_daemon_command(), patch( - "lab.prism.prism_coordinator.now_ms", - side_effect=lambda: clock_ms[0], - ): - artifact = server._build_payout_ledger_artifact( - 0, 0, artifacts.network_difficulty - ) - assert artifact is not None - with server._job_cache_lock: - server._payout_ledger_append_invalidation_epoch += 1 - self.assertFalse(server._install_payout_ledger_artifact(artifact)) - with server._job_cache_lock: - self.assertIsNone(server._payout_ledger_artifact) - - def test_inflight_scan_anchor_spans_the_daemon_round_trip(self) -> None: - server, _ledger, artifacts = self._server() - observed_anchors: list[dict[int, int]] = [] - real_prepare = server.prepare_payout_window - - def observing_prepare(**kwargs: object) -> object: - with server._job_cache_lock: - observed_anchors.append( - dict(server._payout_window_inflight_scan_anchors) - ) - return real_prepare(**kwargs) - - clock_ms = [1_000_000] - with self._rust_env(), self._fake_daemon_command(), patch( - "lab.prism.prism_coordinator.now_ms", - side_effect=lambda: clock_ms[0], - ), patch.object(server, "prepare_payout_window", observing_prepare): - artifact = server._build_payout_ledger_artifact( - 0, 0, artifacts.network_difficulty - ) - assert artifact is not None - self.assertEqual(len(observed_anchors), 1) - self.assertIn( - int(artifact.snapshot_anchor_ms or 0), - observed_anchors[0].values(), - ) - with server._job_cache_lock: - self.assertEqual(server._payout_window_inflight_scan_anchors, {}) - - def test_daemon_death_mid_prepare_degrades_to_python_same_build(self) -> None: - server, ledger, artifacts = self._server() - clock_ms = [1_000_000] - with self._rust_env(mode="crash-during-prepare"), ( - self._fake_daemon_command() - ), patch( - "lab.prism.prism_coordinator.now_ms", - side_effect=lambda: clock_ms[0], - ): - artifact = server._build_payout_ledger_artifact( - 0, 0, artifacts.network_difficulty - ) - assert artifact is not None - self.assertEqual(artifact.window_build_mode, "full_rescan") - cached = server._incremental_payout_artifact_window - assert cached is not None - self.assertIsInstance(cached.window, IncrementalShareWindow) - oracle = _python_window_for( - ledger, - anchor_ms=int(artifact.snapshot_anchor_ms or 0), - network_difficulty=int(artifacts.network_difficulty), - ) - self.assertEqual( - artifact.share_snapshot_sha256, - oracle.json_records().canonical_json_sha256(), - ) - self.assertTrue(server._install_payout_ledger_artifact(artifact)) - counts = self._serve_counts(server) - self.assertEqual(counts["fallbacks"], 1) - self.assertEqual(counts["window_prepares"], 0) - - def test_needs_full_after_respawn_reprepares_identical_digest(self) -> None: - server, ledger, artifacts = self._server() - clock_ms = [1_000_000] - with self._rust_env(), self._fake_daemon_command(), patch( - "lab.prism.prism_coordinator.now_ms", - side_effect=lambda: clock_ms[0], - ): - initial = server._build_payout_ledger_artifact( - 0, 0, artifacts.network_difficulty - ) - assert initial is not None - # The daemon dies between generations; its prepared state dies - # with it, while the coordinator's mirror stays digest-verified. - client = server._serve_builder - assert client is not None - client.process.kill() - client.process.wait(timeout=5.0) - - append_incremental_share(ledger, share_seq=4, accepted_at_ms=1_000_010) - server.payout_artifact_min_build_interval_seconds = 0.0 - clock_ms[0] = 1_000_020 - rebuilt = server._build_payout_ledger_artifact( - 0, 0, artifacts.network_difficulty - ) - assert rebuilt is not None - self.assertEqual(rebuilt.window_build_mode, "full_rescan") - self.assertEqual( - rebuilt.window_full_rescan_reason, - "window_daemon_state_lost", - ) - oracle = _python_window_for( - ledger, - anchor_ms=int(rebuilt.snapshot_anchor_ms or 0), - network_difficulty=int(artifacts.network_difficulty), - ) - self.assertEqual( - rebuilt.share_snapshot_sha256, - oracle.json_records().canonical_json_sha256(), - ) - cached = server._incremental_payout_artifact_window - assert cached is not None - self.assertIsInstance(cached.window, DaemonShareWindowMirror) - counts = self._serve_counts(server) - self.assertEqual(counts["spawns"], 2) - self.assertEqual(ledger.full_snapshot_calls, 2) - - def test_advance_invariant_failure_full_rescans_without_retiring_daemon( - self, - ) -> None: - server, ledger, artifacts = self._server() - clock_ms = [1_000_000] - with self._rust_env(mode="prepare-fallback"), ( - self._fake_daemon_command() - ), patch( - "lab.prism.prism_coordinator.now_ms", - side_effect=lambda: clock_ms[0], - ): - initial = server._build_payout_ledger_artifact( - 0, 0, artifacts.network_difficulty - ) - assert initial is not None - append_incremental_share(ledger, share_seq=4, accepted_at_ms=1_000_010) - server.payout_artifact_min_build_interval_seconds = 0.0 - clock_ms[0] = 1_000_020 - rebuilt = server._build_payout_ledger_artifact( - 0, 0, artifacts.network_difficulty - ) - assert rebuilt is not None - self.assertEqual(rebuilt.window_build_mode, "full_rescan") - self.assertEqual( - rebuilt.window_full_rescan_reason, - "incremental_invariant_failed", - ) - client = server._serve_builder - assert client is not None - self.assertIsNone(client.process.poll()) - counts = self._serve_counts(server) - self.assertEqual(counts["fallbacks"], 0) - self.assertEqual(counts["spawns"], 1) - self.assertEqual(ledger.full_snapshot_calls, 2) - - def test_protocol_mismatch_retires_daemon_and_degrades_to_python(self) -> None: - server, _ledger, artifacts = self._server() - clock_ms = [1_000_000] - with self._rust_env(FAKE_SERVE_BUILDER_PROTOCOL="1"), ( - self._fake_daemon_command() - ), patch( - "lab.prism.prism_coordinator.now_ms", - side_effect=lambda: clock_ms[0], - ): - artifact = server._build_payout_ledger_artifact( - 0, 0, artifacts.network_difficulty - ) - assert artifact is not None - cached = server._incremental_payout_artifact_window - assert cached is not None - self.assertIsInstance(cached.window, IncrementalShareWindow) - self.assertIsNone(server._serve_builder) - counts = self._serve_counts(server) - self.assertEqual(counts["fallbacks"], 1) - self.assertEqual(counts["window_prepares"], 0) - - def test_switch_defaults_to_python_pipeline(self) -> None: - server, _ledger, artifacts = self._server() - clock_ms = [1_000_000] - with patch.dict(os.environ) as _env, self._fake_daemon_command(), patch( - "lab.prism.prism_coordinator.now_ms", - side_effect=lambda: clock_ms[0], - ): - os.environ.pop("PRISM_WINDOW_PIPELINE_RUST", None) - artifact = server._build_payout_ledger_artifact( - 0, 0, artifacts.network_difficulty - ) - assert artifact is not None - cached = server._incremental_payout_artifact_window - assert cached is not None - self.assertIsInstance(cached.window, IncrementalShareWindow) - counts = self._serve_counts(server) - self.assertEqual(counts["spawns"], 0) - self.assertEqual(counts["window_prepares"], 0) - - def test_switch_is_inert_without_the_daemon_transport(self) -> None: - server, _ledger, artifacts = self._server() - clock_ms = [1_000_000] - with self._rust_env(PRISM_BUILDER_SERVE="0"), ( - self._fake_daemon_command() - ), patch( - "lab.prism.prism_coordinator.now_ms", - side_effect=lambda: clock_ms[0], - ): - artifact = server._build_payout_ledger_artifact( - 0, 0, artifacts.network_difficulty - ) - assert artifact is not None - cached = server._incremental_payout_artifact_window - assert cached is not None - self.assertIsInstance(cached.window, IncrementalShareWindow) - counts = self._serve_counts(server) - self.assertEqual(counts["spawns"], 0) - self.assertEqual(counts["window_prepares"], 0) - - def test_flipping_the_switch_off_rebuilds_from_the_oracle(self) -> None: - server, ledger, artifacts = self._server() - clock_ms = [1_000_000] - with self._fake_daemon_command(), patch( - "lab.prism.prism_coordinator.now_ms", - side_effect=lambda: clock_ms[0], - ): - with self._rust_env(): - initial = server._build_payout_ledger_artifact( - 0, 0, artifacts.network_difficulty - ) - assert initial is not None - cached = server._incremental_payout_artifact_window - assert cached is not None - self.assertIsInstance(cached.window, DaemonShareWindowMirror) - server.payout_artifact_min_build_interval_seconds = 0.0 - clock_ms[0] = 1_000_020 - with patch.dict( - os.environ, {"PRISM_WINDOW_PIPELINE_RUST": "0"} - ): - rebuilt = server._build_payout_ledger_artifact( - 0, 0, artifacts.network_difficulty - ) - assert rebuilt is not None - self.assertEqual(rebuilt.window_build_mode, "full_rescan") - self.assertEqual( - rebuilt.window_full_rescan_reason, - "window_pipeline_mode_changed", - ) - cached = server._incremental_payout_artifact_window - assert cached is not None - self.assertIsInstance(cached.window, IncrementalShareWindow) - self.assertEqual(ledger.full_snapshot_calls, 2) - - -class DaemonWindowContentionTests(DaemonWindowPipelineTestCase): - """A daemon another build owns is degraded around, never retired. - - The prepare path used to stamp its deadline before an untimed blocking - acquire, so a preparation that queued behind a slow build arrived inside - the lock with its whole budget already spent: the first write read as a - daemon timeout, and the anomaly handler SIGKILLed a daemon that had done - nothing wrong. Waiting out a lock is not a daemon fault. - """ - - def _lock_holder(self, server: PrismCoordinator, hold_seconds: float): - """Own the serve lock from another thread for a bounded window.""" - taken = threading.Event() - release = threading.Event() - - def hold() -> None: - server._serve_builder_lock.acquire() - try: - taken.set() - release.wait(hold_seconds) - finally: - server._serve_builder_lock.release() - - thread = threading.Thread(target=hold, daemon=True) - thread.start() - self.assertTrue(taken.wait(5.0)) - self.addCleanup(thread.join, 10.0) - self.addCleanup(release.set) - return release - - def _worker_counts(self, server: PrismCoordinator) -> dict[str, int]: - with server._job_build_scheduler_lock: - return dict(server.job_build_worker_counts) - - def test_contended_daemon_degrades_without_being_retired(self) -> None: - server, ledger, artifacts = self._server() - clock_ms = [1_000_000] - with self._rust_env(), self._fake_daemon_command(), patch( - "lab.prism.prism_coordinator.now_ms", - side_effect=lambda: clock_ms[0], - ): - initial = server._build_payout_ledger_artifact( - 0, 0, artifacts.network_difficulty - ) - assert initial is not None - self.assertIsInstance( - server._incremental_payout_artifact_window.window, - DaemonShareWindowMirror, - ) - daemon = server._serve_builder - assert daemon is not None - - append_incremental_share(ledger, share_seq=4, accepted_at_ms=1_000_010) - server.payout_artifact_min_build_interval_seconds = 0.0 - server.bundle_build_timeout_seconds = 0.2 - clock_ms[0] = 1_000_020 - # Held past this preparation's whole budget: the pre-fix code - # blocked here, then read its own exhausted deadline as a sick - # daemon and killed it. - self._lock_holder(server, 3.0) - rebuilt = server._build_payout_ledger_artifact( - 0, 0, artifacts.network_difficulty - ) - assert rebuilt is not None - self.assertEqual(rebuilt.window_build_mode, "full_rescan") - self.assertEqual( - rebuilt.window_full_rescan_reason, - "window_daemon_busy", - ) - # Degraded to the in-process fold for this materialization, and - # still byte-exact against the oracle. - cached = server._incremental_payout_artifact_window - assert cached is not None - self.assertIsInstance(cached.window, IncrementalShareWindow) - oracle = _python_window_for( - ledger, - anchor_ms=int(rebuilt.snapshot_anchor_ms or 0), - network_difficulty=int(artifacts.network_difficulty), - ) - self.assertEqual( - rebuilt.share_snapshot_sha256, - oracle.json_records().canonical_json_sha256(), - ) - # The daemon is the same live process it was before the contest. - self.assertIs(server._serve_builder, daemon) - self.assertIsNone(daemon.process.poll()) - counts = self._serve_counts(server) - self.assertEqual(counts["fallbacks"], 0) - self.assertEqual(counts["spawns"], 1) - # One refusal for the advance, one for the full re-preparation. - self.assertEqual(counts["window_prepare_contended"], 2) - self.assertEqual(self._worker_counts(server)["terminations"], 0) - self.assertEqual(ledger.full_snapshot_calls, 2) - - def test_deadline_is_stamped_after_the_daemon_lock_is_taken(self) -> None: - server, ledger, artifacts = self._server() - clock_ms = [1_000_000] - server.bundle_build_timeout_seconds = 1.0 - with self._rust_env( - mode="prepare-slow", - FAKE_SERVE_BUILDER_PREPARE_DELAY_SECONDS="0.6", - ), self._fake_daemon_command(), patch( - "lab.prism.prism_coordinator.now_ms", - side_effect=lambda: clock_ms[0], - ): - initial = server._build_payout_ledger_artifact( - 0, 0, artifacts.network_difficulty - ) - assert initial is not None - self.assertIsInstance( - server._incremental_payout_artifact_window.window, - DaemonShareWindowMirror, - ) - - append_incremental_share(ledger, share_seq=4, accepted_at_ms=1_000_010) - server.payout_artifact_min_build_interval_seconds = 0.0 - clock_ms[0] = 1_000_020 - # Most of the budget is spent waiting; the round trip that - # follows needs more than what would be left of it. Charging the - # wait against the exchange is what used to retire the daemon. - self._lock_holder(server, 0.7) - advanced = server._build_payout_ledger_artifact( - 0, 0, artifacts.network_difficulty - ) - assert advanced is not None - self.assertEqual(advanced.window_build_mode, "incremental") - self.assertIsNone(advanced.window_full_rescan_reason) - self.assertIsInstance( - server._incremental_payout_artifact_window.window, - DaemonShareWindowMirror, - ) - oracle = _python_window_for( - ledger, - anchor_ms=int(advanced.snapshot_anchor_ms or 0), - network_difficulty=int(artifacts.network_difficulty), - ) - self.assertEqual( - advanced.share_snapshot_sha256, - oracle.json_records().canonical_json_sha256(), - ) - counts = self._serve_counts(server) - self.assertEqual(counts["fallbacks"], 0) - self.assertEqual(counts["spawns"], 1) - self.assertEqual(counts["window_prepares"], 2) - self.assertEqual(ledger.full_snapshot_calls, 1) - - def test_urgent_preview_materialization_never_queues_for_the_daemon( - self, - ) -> None: - server, _ledger, artifacts = self._server() - clock_ms = [1_000_000] - server.bundle_build_timeout_seconds = 30.0 - with self._rust_env(), self._fake_daemon_command(), patch( - "lab.prism.prism_coordinator.now_ms", - side_effect=lambda: clock_ms[0], - ): - initial = server._build_payout_ledger_artifact( - 0, 0, artifacts.network_difficulty - ) - assert initial is not None - clock_ms[0] = 1_000_020 - # An accepted-block preview publishes on the critical path. It - # takes _payout_state_prepare_lock non-blocking for exactly that - # reason, so its materialization must not then wait on the serve - # lock either. - self._lock_holder(server, 30.0) - started = time.monotonic() - preview = server._build_payout_ledger_artifact( - 0, - 0, - artifacts.network_difficulty, - bypass_build_interval=True, - ) - waited = time.monotonic() - started - assert preview is not None - self.assertLess(waited, 5.0) - self.assertEqual(preview.window_build_mode, "full_rescan") - self.assertEqual( - preview.window_full_rescan_reason, - "window_daemon_busy", - ) - counts = self._serve_counts(server) - self.assertEqual(counts["fallbacks"], 0) - self.assertEqual(counts["window_prepare_contended"], 2) - - -class DaemonWindowRecordCountTests(DaemonWindowPipelineTestCase): - """A declared record count is reconciled with the bytes, eagerly. - - The construction-time digest pins the bytes and says nothing about the - count beside them, so a miscount used to surface only at whichever - consumer first parsed the stream -- possibly inside a serve request that - had already written its prefix, leaving the daemon's stdin holding an - unterminated line for the next request to concatenate onto. - """ - - def test_miscounted_window_is_refused_at_materialization(self) -> None: - server, ledger, artifacts = self._server() - clock_ms = [1_000_000] - with self._rust_env(mode="prepare-miscount"), ( - self._fake_daemon_command() - ), patch( - "lab.prism.prism_coordinator.now_ms", - side_effect=lambda: clock_ms[0], - ): - artifact = server._build_payout_ledger_artifact( - 0, 0, artifacts.network_difficulty - ) - assert artifact is not None - # Refused where it was built: the mirror never reaches the cache, - # so no consumer can be the first to learn of the divergence. - cached = server._incremental_payout_artifact_window - assert cached is not None - self.assertIsInstance(cached.window, IncrementalShareWindow) - self.assertNotIsInstance( - artifact.shares_json, - DaemonShareJsonSequence, - ) - oracle = _python_window_for( - ledger, - anchor_ms=int(artifact.snapshot_anchor_ms or 0), - network_difficulty=int(artifacts.network_difficulty), - ) - self.assertEqual( - artifact.share_snapshot_sha256, - oracle.json_records().canonical_json_sha256(), - ) - # A wrong count is the daemon's answer being wrong, not the - # daemon being sick: it stays registered and alive. - client = server._serve_builder - assert client is not None - self.assertIsNone(client.process.poll()) - with server._payout_artifact_executor_lock: - events = dict(server.payout_artifact_event_counts) - self.assertEqual(events.get("window_mirror_divergence"), 1) - self.assertEqual(self._serve_counts(server)["fallbacks"], 0) - - def test_half_written_request_retires_the_poisoned_daemon(self) -> None: - server, _ledger, artifacts = self._server() - clock_ms = [1_000_000] - with self._rust_env(), self._fake_daemon_command(), patch( - "lab.prism.prism_coordinator.now_ms", - side_effect=lambda: clock_ms[0], - ): - artifact = server._build_payout_ledger_artifact( - 0, 0, artifacts.network_difficulty - ) - assert artifact is not None - assert isinstance(artifact.shares_json, DaemonShareJsonSequence) - # A sequence that only refuses itself once iterated, standing in - # for whatever could still reach a build holding the serve lock. - poisoned = DaemonShareJsonSequence( - canonical_items=artifact.shares_json.canonical_items, - record_count=len(artifact.shares_json) + 1, - ) - found_block = { - "block_height": 101, - "coinbase_value_sats": 50_00000000, - "network_difficulty": int(artifacts.network_difficulty), - "anchor_job_issued_at_ms": int(artifact.snapshot_anchor_ms or 0), - } - serialization = server._share_window_serialization_for_artifact( - artifact, - poisoned, - ) - daemon = server._serve_builder - assert daemon is not None - # Force the window to ride along: the divergence has to be - # raised after the request prefix is already on the daemon's - # stdin, which is the case the narrow catch used to miss. - daemon.uploaded_windows.clear() - with self.assertRaises(DaemonWindowMirrorDivergence): - server.build_audit_bundle( - shares=poisoned, - found_block=found_block, - prior_balances=[], - coinbase_script_sig_suffix_hex="00", - summary_only=True, - payout_policy={"policy": "day-one"}, - share_serialization=serialization, # type: ignore[arg-type] - append_invalidation_epoch=artifact.append_invalidation_epoch, - ) - # The request never reached its terminating newline, so the - # daemon went with it rather than answering "malformed serve - # request" to the next innocent build. - self.assertIsNone(server._serve_builder) - self.assertIsNotNone(daemon.process.poll()) - self.assertIsNone(server._incremental_payout_artifact_window) - - # The next honest build still gets the daemon transport. - rebuilt = server._build_payout_ledger_artifact( - 0, 0, artifacts.network_difficulty - ) - assert rebuilt is not None - summary = server.build_audit_bundle( - shares=rebuilt.shares_json, - found_block=found_block, - prior_balances=[], - coinbase_script_sig_suffix_hex="00", - summary_only=True, - payout_policy={"policy": "day-one"}, - share_serialization=( - server._share_window_serialization_for_artifact( - rebuilt, - rebuilt.shares_json, - ) - ), # type: ignore[arg-type] - append_invalidation_epoch=rebuilt.append_invalidation_epoch, - ) - self.assertEqual(summary["transport"], "serve") - with server._payout_artifact_executor_lock: - events = dict(server.payout_artifact_event_counts) - self.assertEqual(events.get("window_mirror_divergence"), 1) - - -class DaemonWindowIntegerDomainTests(DaemonWindowPipelineTestCase): - """A value outside the daemon's declared widths degrades to Python; the daemon lives. - - The daemon carries ``share_difficulty`` in u128 (and the other integers - in u64/u32/i64); Python carries them unbounded. A window holding a value - above those widths used to come back as a malformed-request error, which - the coordinator read as an anomaly: it SIGKILLed a healthy daemon, - degraded once, and respawned on the next materialization only to kill - it again -- a spawn-and-kill loop, once per materialization, for as long - as the window held the value. The daemon now declines such a request as - ``out_of_range`` and keeps its state; the coordinator folds the window - in-process and never touches the daemon. - """ - - def _append_wide_share( - self, - ledger: SingleWriterShareLedger, - *, - share_seq: int, - accepted_at_ms: int, - ) -> None: - """A share whose difficulty is one above u128: inside Python, outside the daemon.""" - ledger.append( - PendingShare( - share_id=f"wide-{share_seq}", - miner_id=f"miner-{share_seq % 3}", - order_key=f"miner-{share_seq % 3}", - p2mr_program_hex=f"{share_seq % 256:02x}" * 32, - share_difficulty=2**128 + 1, - network_difficulty=1, - template_height=9, - job_id=f"job-{share_seq}", - job_issued_at_ms=accepted_at_ms - 1, - accepted_at_ms=accepted_at_ms, - ntime=1_700_000_000 + share_seq, - ) - ) - - def _worker_counts(self, server: PrismCoordinator) -> dict[str, int]: - with server._job_build_scheduler_lock: - return dict(server.job_build_worker_counts) - - def test_out_of_range_snapshot_folds_in_python_and_keeps_the_daemon(self) -> None: - server, ledger, artifacts = self._server() - self._append_wide_share(ledger, share_seq=4, accepted_at_ms=999_930) - clock_ms = [1_000_000] - with self._rust_env(), self._fake_daemon_command(), patch( - "lab.prism.prism_coordinator.now_ms", - side_effect=lambda: clock_ms[0], - ): - artifact = server._build_payout_ledger_artifact( - 0, 0, artifacts.network_difficulty - ) - assert artifact is not None - self.assertEqual(artifact.window_build_mode, "full_rescan") - # Declined by the daemon, folded in-process: a Python window is - # cached, byte-exact against the oracle, with the wide digits. - cached = server._incremental_payout_artifact_window - assert cached is not None - self.assertIsInstance(cached.window, IncrementalShareWindow) - oracle = _python_window_for( - ledger, - anchor_ms=int(artifact.snapshot_anchor_ms or 0), - network_difficulty=int(artifacts.network_difficulty), - ) - self.assertEqual( - artifact.share_snapshot_sha256, - oracle.json_records().canonical_json_sha256(), - ) - self.assertTrue( - any( - int(record["share_difficulty"]) == 2**128 + 1 - for record in artifact.shares_json - ) - ) - self.assertTrue(server._install_payout_ledger_artifact(artifact)) - # The daemon was never retired: same live process, registered. - daemon = server._serve_builder - assert daemon is not None - self.assertIsNone(daemon.process.poll()) - - # The next materialization still holds the value. Before the fix - # this is where the loop turned: a fresh spawn, declined again, - # killed again. Now the degraded Python cache advances in - # process and the daemon is left alone. - append_incremental_share(ledger, share_seq=5, accepted_at_ms=1_000_010) - server.payout_artifact_min_build_interval_seconds = 0.0 - clock_ms[0] = 1_000_020 - advanced = server._build_payout_ledger_artifact( - 0, 0, artifacts.network_difficulty - ) - assert advanced is not None - self.assertEqual(advanced.window_build_mode, "incremental") - self.assertIs(server._serve_builder, daemon) - self.assertIsNone(daemon.process.poll()) - advanced_oracle = _python_window_for( - ledger, - anchor_ms=int(advanced.snapshot_anchor_ms or 0), - network_difficulty=int(artifacts.network_difficulty), - ) - self.assertEqual( - advanced.share_snapshot_sha256, - advanced_oracle.json_records().canonical_json_sha256(), - ) - counts = self._serve_counts(server) - self.assertEqual(counts["spawns"], 1) - self.assertEqual(counts["fallbacks"], 0) - # window_prepares counts completed round trips, declined ones - # included: one, and it was the decline. The Python-advancing - # second materialization never asked the daemon at all. - self.assertEqual(counts["window_prepares"], 1) - self.assertEqual(counts["window_prepare_out_of_range"], 1) - self.assertEqual(self._worker_counts(server)["terminations"], 0) - - def test_out_of_range_delta_rebuilds_in_python_and_keeps_the_daemon(self) -> None: - server, ledger, artifacts = self._server() - clock_ms = [1_000_000] - with self._rust_env(), self._fake_daemon_command(), patch( - "lab.prism.prism_coordinator.now_ms", - side_effect=lambda: clock_ms[0], - ): - initial = server._build_payout_ledger_artifact( - 0, 0, artifacts.network_difficulty - ) - assert initial is not None - cached = server._incremental_payout_artifact_window - assert cached is not None - self.assertIsInstance(cached.window, DaemonShareWindowMirror) - daemon = server._serve_builder - assert daemon is not None - - # The wide share arrives as a delta against a daemon-held mirror: - # the daemon declines the advance, the full re-preparation is - # declined the same way, and the in-process fold takes over. - self._append_wide_share(ledger, share_seq=4, accepted_at_ms=1_000_010) - server.payout_artifact_min_build_interval_seconds = 0.0 - clock_ms[0] = 1_000_020 - rebuilt = server._build_payout_ledger_artifact( - 0, 0, artifacts.network_difficulty - ) - assert rebuilt is not None - self.assertEqual(rebuilt.window_build_mode, "full_rescan") - self.assertEqual( - rebuilt.window_full_rescan_reason, - "window_value_out_of_range", - ) - cached = server._incremental_payout_artifact_window - assert cached is not None - self.assertIsInstance(cached.window, IncrementalShareWindow) - oracle = _python_window_for( - ledger, - anchor_ms=int(rebuilt.snapshot_anchor_ms or 0), - network_difficulty=int(artifacts.network_difficulty), - ) - self.assertEqual( - rebuilt.share_snapshot_sha256, - oracle.json_records().canonical_json_sha256(), - ) - self.assertIs(server._serve_builder, daemon) - self.assertIsNone(daemon.process.poll()) - counts = self._serve_counts(server) - self.assertEqual(counts["spawns"], 1) - self.assertEqual(counts["fallbacks"], 0) - # Three completed round trips: the initial full preparation - # succeeded, then the advance and the full re-preparation were - # both declined. - self.assertEqual(counts["window_prepares"], 3) - self.assertEqual(counts["window_prepare_out_of_range"], 2) - self.assertEqual(self._worker_counts(server)["terminations"], 0) - self.assertEqual(ledger.full_snapshot_calls, 2) - - def test_rejection_envelopes_carry_the_category_not_just_the_message(self) -> None: - # The fold_invalid/fallback outcomes name their condition in a stable - # category beside the diagnostic message, so the coordinator (and the - # parity adapter) never have to match on prose that may drift. - server, ledger, artifacts = self._server() - clock_ms = [1_000_000] - with self._rust_env(mode="prepare-fallback"), ( - self._fake_daemon_command() - ), patch( - "lab.prism.prism_coordinator.now_ms", - side_effect=lambda: clock_ms[0], - ): - initial = server._build_payout_ledger_artifact( - 0, 0, artifacts.network_difficulty - ) - assert initial is not None - outcome = server.prepare_payout_window( - mode="advance", - records_json=[], - anchor_job_issued_at_ms=1_000_010, - append_invalidation_epoch=0, - base_digest=initial.share_snapshot_sha256, - ) - assert outcome is not None - self.assertEqual(outcome.status, "fallback") - self.assertEqual(outcome.rejection, "delta_not_append") - duplicate = dict(initial.shares_json[0]) - declined = server.prepare_payout_window( - mode="full", - records_json=[duplicate, duplicate], - anchor_job_issued_at_ms=1_000_000, - append_invalidation_epoch=0, - window_weight=1_000, - page_size=512, - ) - assert declined is not None - self.assertEqual(declined.status, "fold_invalid") - self.assertEqual(declined.rejection, "duplicate_share_seq") - - -class DaemonWindowRetagTests(DaemonWindowPipelineTestCase): - """A retagged window advances in place, Rust switch on or off. - - _retire_payout_windows_for_late_append clears the cache only when the - late row predates the cached anchor, and otherwise retags it precisely so - the next delta build advances the window it already has. A daemon that - refused an older-tagged base turned every such retag into a full DB walk, - fold and upload -- the common case made worse, under exactly the replay - bursts that produce the bumps. - """ - - def _retag_after_late_append(self, server: PrismCoordinator) -> int: - cached = server._incremental_payout_artifact_window - assert cached is not None - anchor_ms = int(cached.window.anchor_job_issued_at_ms) - with server._job_cache_lock: - server._payout_ledger_append_invalidation_epoch += 1 - epoch = int(server._payout_ledger_append_invalidation_epoch) - # Stamped above the cached anchor: the armed artifact was affected, - # this incremental window was not, so the coordinator retags it. - server._retire_payout_windows_for_late_append( - stamped_pending_share(anchor_ms + 1_000), - epoch, - ) - retagged = server._incremental_payout_artifact_window - assert retagged is not None - self.assertEqual(retagged.append_invalidation_epoch, epoch) - return epoch - - def test_retagged_window_advances_incrementally_on_both_pipelines( - self, - ) -> None: - for rust in (False, True): - with self.subTest(rust=rust): - server, ledger, artifacts = self._server() - clock_ms = [1_000_000] - env = ( - self._rust_env() - if rust - else patch.dict( - os.environ, - {"PRISM_WINDOW_PIPELINE_RUST": "0"}, - ) - ) - with env, self._fake_daemon_command(), patch( - "lab.prism.prism_coordinator.now_ms", - side_effect=lambda: clock_ms[0], - ): - initial = server._build_payout_ledger_artifact( - 0, 0, artifacts.network_difficulty - ) - assert initial is not None - self.assertEqual(ledger.full_snapshot_calls, 1) - self._retag_after_late_append(server) - - append_incremental_share( - ledger, share_seq=4, accepted_at_ms=1_000_010 - ) - server.payout_artifact_min_build_interval_seconds = 0.0 - clock_ms[0] = 1_000_020 - advanced = server._build_payout_ledger_artifact( - 0, 0, artifacts.network_difficulty - ) - assert advanced is not None - self.assertEqual(advanced.window_build_mode, "incremental") - self.assertIsNone(advanced.window_full_rescan_reason) - self.assertEqual(advanced.window_delta_rows, 1) - # The retag survived the advance: no oracle was paid. - self.assertEqual(ledger.full_snapshot_calls, 1) - oracle = _python_window_for( - ledger, - anchor_ms=int(advanced.snapshot_anchor_ms or 0), - network_difficulty=int(artifacts.network_difficulty), - ) - self.assertEqual( - advanced.share_snapshot_sha256, - oracle.json_records().canonical_json_sha256(), - ) - cached = server._incremental_payout_artifact_window - assert cached is not None - self.assertIsInstance( - cached.window, - DaemonShareWindowMirror if rust else IncrementalShareWindow, - ) - - -if __name__ == "__main__": - unittest.main() diff --git a/tests/test_prism_worker_difficulty_store.py b/tests/test_prism_worker_difficulty_store.py deleted file mode 100644 index 091274139..000000000 --- a/tests/test_prism_worker_difficulty_store.py +++ /dev/null @@ -1,700 +0,0 @@ -#!/usr/bin/env python3 -"""Focused tests for the durable worker-difficulty storage seam. - -Everything here runs without a database: the in-memory back end is exercised -directly, the PostgreSQL back end through query capture and canned results, -and the schema through string contracts against the monolithic SQL file. -Real-PostgreSQL parity for the same contract lives in -``tests/prism_worker_difficulty_store_gate.py``. -""" - -from __future__ import annotations - -import unittest - -from decimal import Decimal -from pathlib import Path -from typing import Any - -from lab.prism.worker_difficulty_store import ( - MemoryWorkerDifficultyStore, - PostgresWorkerDifficultyStore, - WORKER_DIFFICULTY_SCHEMA_SQL, - WorkerDifficultyRecord, - WorkerDifficultyStorePort, - WorkerDifficultyUpsertResult, -) - - -ROOT = Path(__file__).resolve().parents[1] -LEDGER_SCHEMA = ROOT / "crates" / "qbit-prism" / "sql" / "001_share_ledger.sql" - -BASE_MS = 1_756_000_000_000 - - -def record( - *, - listener: str = "default", - worker_username: str = "miner.worker1", - difficulty: str = "8192", - evidence_at_ms: int = BASE_MS, - updated_at_ms: int = BASE_MS, -) -> WorkerDifficultyRecord: - return WorkerDifficultyRecord( - listener=listener, - worker_username=worker_username, - difficulty=Decimal(difficulty), - evidence_at_ms=evidence_at_ms, - updated_at_ms=updated_at_ms, - ) - - -class QueryCaptureStore(PostgresWorkerDifficultyStore): - """Postgres store whose runner records SQL and replays canned results.""" - - def __init__(self, results: list[Any] | None = None) -> None: - self.queries: list[str] = [] - self._results = list(results or []) - super().__init__(self._capture) - - def _capture(self, sql: str) -> Any: - self.queries.append(sql) - if self._results: - return self._results.pop(0) - return {"applied": True, "stored": None} - - -class SchemaContractTests(unittest.TestCase): - def test_module_schema_block_matches_monolithic_file_verbatim(self) -> None: - schema = LEDGER_SCHEMA.read_text(encoding="utf-8") - self.assertIn(WORKER_DIFFICULTY_SCHEMA_SQL.strip(), schema) - # Exactly once, inside the file's single BEGIN/COMMIT transaction. - self.assertEqual(schema.count("CREATE TABLE IF NOT EXISTS qbit_worker_difficulty"), 1) - self.assertLess( - schema.index("CREATE TABLE IF NOT EXISTS qbit_worker_difficulty"), - schema.rindex("COMMIT;"), - ) - - def test_schema_block_is_idempotent_and_scoped(self) -> None: - # Idempotent reapply: every statement in the block tolerates the - # object already existing. - for statement in WORKER_DIFFICULTY_SCHEMA_SQL.split(";"): - statement = statement.strip() - if not statement: - continue - self.assertIn("IF NOT EXISTS", statement) - # Primary key covers listener plus exact username; the preload/prune - # index leads on the evidence timestamp and matches the preload - # ORDER BY column-for-column. - self.assertIn("PRIMARY KEY (listener, worker_username)", WORKER_DIFFICULTY_SCHEMA_SQL) - self.assertIn( - "ON qbit_worker_difficulty (evidence_at DESC, listener, worker_username)", - WORKER_DIFFICULTY_SCHEMA_SQL, - ) - # Positive finite difficulty: NaN and Infinity both compare greater - # than zero in PostgreSQL numeric, so the finite guard is load-bearing. - self.assertIn("difficulty > 0 AND difficulty < 'Infinity'::numeric", WORKER_DIFFICULTY_SCHEMA_SQL) - # The decimal wire value must not be forced through an integer typmod. - self.assertIn("difficulty numeric NOT NULL", WORKER_DIFFICULTY_SCHEMA_SQL) - self.assertNotIn("numeric(", WORKER_DIFFICULTY_SCHEMA_SQL) - # Evidence and ordinary-update timestamps are separate columns. - self.assertIn("evidence_at timestamptz NOT NULL", WORKER_DIFFICULTY_SCHEMA_SQL) - self.assertIn("updated_at timestamptz NOT NULL", WORKER_DIFFICULTY_SCHEMA_SQL) - # Tie-breaks are byte order in every deployment and match the - # in-memory store's code-point ordering. - self.assertIn('listener text COLLATE "C" NOT NULL', WORKER_DIFFICULTY_SCHEMA_SQL) - self.assertIn('worker_username text COLLATE "C" NOT NULL', WORKER_DIFFICULTY_SCHEMA_SQL) - - def test_schema_touches_no_canonical_share_or_payout_tables(self) -> None: - for name in ( - "qbit_share_ledger", - "qbit_pool_blocks", - "qbit_pool_payout_entries", - "qbit_payout_carry_forward", - ): - self.assertNotIn(name, WORKER_DIFFICULTY_SCHEMA_SQL) - - -class StorePortTests(unittest.TestCase): - def test_port_exposes_no_per_key_read(self) -> None: - # Reconnect authorization must never block on a synchronous - # database read, so the contract is upsert + bounded batch preload + - # prune, and nothing key-addressed to read. - for store_type in (MemoryWorkerDifficultyStore, PostgresWorkerDifficultyStore): - self.assertTrue(issubclass(store_type, WorkerDifficultyStorePort)) - for forbidden in ("get", "lookup", "load_one", "fetch"): - self.assertFalse( - hasattr(store_type, forbidden), - f"{store_type.__name__} must not expose {forbidden}", - ) - - -class MemoryStoreTests(unittest.TestCase): - def setUp(self) -> None: - self.store = MemoryWorkerDifficultyStore() - - def test_insert_then_update(self) -> None: - first = self.store.upsert( - listener="default", - worker_username="miner.worker1", - difficulty=Decimal("4096"), - evidence_at_ms=BASE_MS, - now_ms=BASE_MS, - ) - self.assertTrue(first.applied) - self.assertEqual( - first.stored, - record(difficulty="4096"), - ) - second = self.store.upsert( - listener="default", - worker_username="miner.worker1", - difficulty=Decimal("8192"), - evidence_at_ms=BASE_MS + 1_000, - now_ms=BASE_MS + 1_500, - ) - self.assertTrue(second.applied) - self.assertEqual( - second.stored, - record( - difficulty="8192", - evidence_at_ms=BASE_MS + 1_000, - updated_at_ms=BASE_MS + 1_500, - ), - ) - self.assertEqual(len(self.store), 1) - - def test_older_evidence_cannot_replace_newer_evidence(self) -> None: - self.store.upsert( - listener="default", - worker_username="miner.worker1", - difficulty=Decimal("8192"), - evidence_at_ms=BASE_MS + 5_000, - now_ms=BASE_MS + 5_000, - ) - stale = self.store.upsert( - listener="default", - worker_username="miner.worker1", - difficulty=Decimal("1"), - evidence_at_ms=BASE_MS + 4_999, - now_ms=BASE_MS + 6_000, - ) - self.assertFalse(stale.applied) - self.assertEqual( - stale.stored, - record( - difficulty="8192", - evidence_at_ms=BASE_MS + 5_000, - updated_at_ms=BASE_MS + 5_000, - ), - ) - loaded = self.store.load_recent(evidence_after_ms=0, limit=10) - self.assertEqual(loaded[0].difficulty, Decimal("8192")) - self.assertEqual(loaded[0].updated_at_ms, BASE_MS + 5_000) - - def test_equal_evidence_is_deterministic_last_write_wins(self) -> None: - self.store.upsert( - listener="default", - worker_username="miner.worker1", - difficulty=Decimal("8192"), - evidence_at_ms=BASE_MS, - now_ms=BASE_MS, - ) - rewrite = self.store.upsert( - listener="default", - worker_username="miner.worker1", - difficulty=Decimal("2048"), - evidence_at_ms=BASE_MS, - now_ms=BASE_MS + 100, - ) - self.assertTrue(rewrite.applied) - self.assertEqual( - rewrite.stored, - record(difficulty="2048", updated_at_ms=BASE_MS + 100), - ) - - def test_lane_separation(self) -> None: - self.store.upsert( - listener="stratum-public", - worker_username="miner.worker1", - difficulty=Decimal("16"), - evidence_at_ms=BASE_MS + 1, - now_ms=BASE_MS + 1, - ) - self.store.upsert( - listener="stratum-highdiff", - worker_username="miner.worker1", - difficulty=Decimal("65536"), - evidence_at_ms=BASE_MS + 2, - now_ms=BASE_MS + 2, - ) - loaded = self.store.load_recent(evidence_after_ms=0, limit=10) - self.assertEqual(len(loaded), 2) - by_lane = {entry.listener: entry.difficulty for entry in loaded} - self.assertEqual(by_lane["stratum-public"], Decimal("16")) - self.assertEqual(by_lane["stratum-highdiff"], Decimal("65536")) - - def test_exact_username_separation(self) -> None: - # Case, surrounding whitespace and lookalikes are distinct identities: - # exact byte match only, no normalization. - usernames = ["miner.worker1", "Miner.worker1", " miner.worker1", "miner.worker1 "] - for index, username in enumerate(usernames): - outcome = self.store.upsert( - listener="default", - worker_username=username, - difficulty=Decimal(2 ** (index + 1)), - evidence_at_ms=BASE_MS + index, - now_ms=BASE_MS + index, - ) - self.assertTrue(outcome.applied) - loaded = self.store.load_recent(evidence_after_ms=0, limit=10) - self.assertEqual(len(loaded), len(usernames)) - self.assertEqual( - {entry.worker_username for entry in loaded}, - set(usernames), - ) - - def test_load_recent_is_bounded_cutoff_exclusive_and_ordered(self) -> None: - for index in range(5): - self.store.upsert( - listener="default", - worker_username=f"miner.worker{index}", - difficulty=Decimal("64"), - evidence_at_ms=BASE_MS + index * 1_000, - now_ms=BASE_MS + index * 1_000, - ) - # Tie on the evidence timestamp to pin the (listener, username) - # tie-break. - self.store.upsert( - listener="alpha-lane", - worker_username="miner.workerZ", - difficulty=Decimal("64"), - evidence_at_ms=BASE_MS + 4_000, - now_ms=BASE_MS + 4_000, - ) - loaded = self.store.load_recent(evidence_after_ms=BASE_MS + 1_000, limit=3) - self.assertEqual( - [ - (entry.evidence_at_ms, entry.listener, entry.worker_username) - for entry in loaded - ], - [ - (BASE_MS + 4_000, "alpha-lane", "miner.workerZ"), - (BASE_MS + 4_000, "default", "miner.worker4"), - (BASE_MS + 3_000, "default", "miner.worker3"), - ], - ) - # The cutoff is strict: an entry exactly at the cutoff is excluded. - cutoff_hits = self.store.load_recent(evidence_after_ms=BASE_MS + 4_000, limit=10) - self.assertEqual(cutoff_hits, []) - - def test_prune_is_cutoff_inclusive_and_complements_load(self) -> None: - for index in range(4): - self.store.upsert( - listener="default", - worker_username=f"miner.worker{index}", - difficulty=Decimal("64"), - evidence_at_ms=BASE_MS + index * 1_000, - now_ms=BASE_MS + index * 1_000, - ) - deleted = self.store.prune(evidence_cutoff_ms=BASE_MS + 1_000) - self.assertEqual(deleted, 2) - survivors = self.store.load_recent(evidence_after_ms=BASE_MS + 1_000, limit=10) - self.assertEqual(len(survivors), 2) - self.assertEqual(len(self.store), 2) - - def test_bounded_prune_removes_oldest_first_deterministically(self) -> None: - entries = [ - ("default", "miner.workerB", BASE_MS), - ("default", "miner.workerA", BASE_MS), - ("alpha-lane", "miner.workerA", BASE_MS), - ("default", "miner.workerC", BASE_MS + 1_000), - ] - for listener, username, evidence_ms in entries: - self.store.upsert( - listener=listener, - worker_username=username, - difficulty=Decimal("64"), - evidence_at_ms=evidence_ms, - now_ms=evidence_ms, - ) - deleted = self.store.prune(evidence_cutoff_ms=BASE_MS + 1_000, limit=2) - self.assertEqual(deleted, 2) - remaining = self.store.load_recent(evidence_after_ms=0, limit=10) - # Oldest evidence first, ties broken by (listener, username): the - # alpha-lane row and default/workerA go, workerB and workerC stay. - self.assertEqual( - {(entry.listener, entry.worker_username) for entry in remaining}, - {("default", "miner.workerB"), ("default", "miner.workerC")}, - ) - - def test_decimal_values_round_trip_exactly(self) -> None: - difficulties = [ - Decimal("0.000030517578125"), # 2**-15 - Decimal("123456789012345678901234567890.5"), - Decimal("1E-12"), - Decimal("8192.000"), - ] - for index, difficulty in enumerate(difficulties): - self.store.upsert( - listener="default", - worker_username=f"miner.worker{index}", - difficulty=difficulty, - evidence_at_ms=BASE_MS + index, - now_ms=BASE_MS + index, - ) - loaded = self.store.load_recent(evidence_after_ms=0, limit=10) - by_user = {entry.worker_username: entry.difficulty for entry in loaded} - for index, difficulty in enumerate(difficulties): - self.assertEqual(by_user[f"miner.worker{index}"], difficulty) - - def test_input_validation(self) -> None: - good = dict( - listener="default", - worker_username="miner.worker1", - difficulty=Decimal("64"), - evidence_at_ms=BASE_MS, - now_ms=BASE_MS, - ) - rejected = [ - {**good, "difficulty": Decimal("NaN")}, - {**good, "difficulty": Decimal("Infinity")}, - {**good, "difficulty": Decimal("0")}, - {**good, "difficulty": Decimal("-1")}, - {**good, "difficulty": 64}, - {**good, "difficulty": 64.0}, - {**good, "difficulty": Decimal("1E+1000")}, - {**good, "difficulty": Decimal("1E-1000")}, - {**good, "listener": ""}, - {**good, "listener": "bad\x00lane"}, - {**good, "worker_username": ""}, - {**good, "evidence_at_ms": -1}, - {**good, "evidence_at_ms": 1.5}, - {**good, "now_ms": True}, - ] - for kwargs in rejected: - with self.assertRaises(ValueError, msg=repr(kwargs)): - self.store.upsert(**kwargs) - with self.assertRaises(ValueError): - self.store.load_recent(evidence_after_ms=0, limit=0) - with self.assertRaises(ValueError): - self.store.prune(evidence_cutoff_ms=0, limit=-1) - self.assertEqual(len(self.store), 0) - - -class PostgresStoreSqlTests(unittest.TestCase): - def test_upsert_sql_is_one_guarded_atomic_statement(self) -> None: - store = QueryCaptureStore( - results=[ - { - "applied": True, - "stored": { - "listener": "default", - "worker_username": "miner.worker1", - "difficulty": "8192.5", - "evidence_at_ms": BASE_MS, - "updated_at_ms": BASE_MS + 1, - }, - } - ] - ) - outcome = store.upsert( - listener="default", - worker_username="o'brien.worker1", - difficulty=Decimal("8192.5"), - evidence_at_ms=BASE_MS, - now_ms=BASE_MS + 1, - ) - self.assertEqual(len(store.queries), 1) - sql = store.queries[0] - self.assertIn("INSERT INTO qbit_worker_difficulty", sql) - self.assertIn("ON CONFLICT (listener, worker_username) DO UPDATE", sql) - # The evidence guard is the only thing standing between older and - # newer evidence; pin it verbatim. - self.assertIn("WHERE EXCLUDED.evidence_at >= existing.evidence_at", sql) - # Values are inlined with quoting; the embedded quote must be doubled. - self.assertIn("'o''brien.worker1'", sql) - self.assertIn("'8192.5'::numeric", sql) - self.assertIn(f"to_timestamp(({BASE_MS}::double precision / 1000.0))", sql) - # Difficulty travels back as text, never as a JSON float. - self.assertIn("difficulty::text", sql) - self.assertEqual( - outcome, - WorkerDifficultyUpsertResult( - applied=True, - stored=record(difficulty="8192.5", updated_at_ms=BASE_MS + 1), - ), - ) - - def test_refused_upsert_parses_standing_row(self) -> None: - store = QueryCaptureStore( - results=[ - { - "applied": False, - "stored": { - "listener": "default", - "worker_username": "miner.worker1", - "difficulty": "8192", - "evidence_at_ms": BASE_MS + 5_000, - "updated_at_ms": BASE_MS + 5_000, - }, - } - ] - ) - outcome = store.upsert( - listener="default", - worker_username="miner.worker1", - difficulty=Decimal("1"), - evidence_at_ms=BASE_MS, - now_ms=BASE_MS + 6_000, - ) - self.assertFalse(outcome.applied) - assert outcome.stored is not None - self.assertEqual(outcome.stored.difficulty, Decimal("8192")) - self.assertEqual(outcome.stored.evidence_at_ms, BASE_MS + 5_000) - - def test_load_recent_sql_pins_bound_order_and_cutoff(self) -> None: - store = QueryCaptureStore(results=[[]]) - loaded = store.load_recent(evidence_after_ms=BASE_MS, limit=250) - self.assertEqual(loaded, []) - sql = store.queries[0] - self.assertIn( - f"WHERE evidence_at > to_timestamp(({BASE_MS}::double precision / 1000.0))", - sql, - ) - self.assertIn("ORDER BY evidence_at DESC, listener, worker_username", sql) - self.assertIn("LIMIT 250", sql) - self.assertIn( - "ORDER BY recent.evidence_at DESC, recent.listener, recent.worker_username", - sql, - ) - - def test_prune_sql_is_store_side_and_boundable(self) -> None: - store = QueryCaptureStore(results=[{"deleted": 3}, {"deleted": 2}]) - self.assertEqual(store.prune(evidence_cutoff_ms=BASE_MS), 3) - self.assertEqual(store.prune(evidence_cutoff_ms=BASE_MS, limit=2), 2) - unbounded, bounded = store.queries - self.assertIn("DELETE FROM qbit_worker_difficulty", unbounded) - self.assertIn( - f"WHERE evidence_at <= to_timestamp(({BASE_MS}::double precision / 1000.0))", - unbounded, - ) - self.assertIn("ORDER BY evidence_at ASC, listener, worker_username", bounded) - self.assertIn("LIMIT 2", bounded) - # The candidate scan and DELETE share one statement, but a concurrent - # upsert can refresh a selected row while DELETE waits on its lock. - # Recheck eligibility on the tuple DELETE actually reaches. - self.assertIn( - f"target.evidence_at <= to_timestamp(({BASE_MS}::double precision / 1000.0))", - bounded, - ) - - def test_statements_touch_only_the_worker_difficulty_table(self) -> None: - store = QueryCaptureStore( - results=[ - {"applied": True, "stored": None}, - {"applied": True, "stored": None}, - [], - {"deleted": 0}, - ] - ) - store.upsert( - listener="default", - worker_username="miner.worker1", - difficulty=Decimal("64"), - evidence_at_ms=BASE_MS, - now_ms=BASE_MS, - ) - store.apply_downward( - listener="default", - worker_username="miner.worker1", - difficulty=Decimal("32"), - now_ms=BASE_MS, - ) - store.load_recent(evidence_after_ms=0, limit=1) - store.prune(evidence_cutoff_ms=0) - for sql in store.queries: - for name in ( - "qbit_share_ledger", - "qbit_pool_blocks", - "qbit_pool_payout_entries", - "qbit_payout_carry_forward", - "qbit_ledger_writer_lease", - ): - self.assertNotIn(name, sql) - - -class LowerOnlyCorrectionTests(unittest.TestCase): - """The atomic downward correction, on the in-memory reference back end. - - The PostgreSQL twin is driven through the same sequence by - tests/prism_worker_difficulty_store_gate.py. - """ - - def setUp(self) -> None: - self.store = MemoryWorkerDifficultyStore() - self.key = {"listener": "default", "worker_username": "miner.worker1"} - - def seed(self, difficulty: str, *, evidence_at_ms: int = BASE_MS) -> None: - self.store.upsert( - **self.key, - difficulty=Decimal(difficulty), - evidence_at_ms=evidence_at_ms, - now_ms=evidence_at_ms, - ) - - def test_lowers_a_higher_row_and_preserves_its_evidence(self) -> None: - self.seed("1048576") - - result = self.store.apply_downward( - **self.key, - difficulty=Decimal("65536"), - now_ms=BASE_MS + 5_000, - ) - - self.assertTrue(result.applied) - assert result.stored is not None - self.assertEqual(result.stored.difficulty, Decimal("65536")) - # The correction is not evidence, so it must not re-stamp the clock - # the TTL and the write ordering both read. - self.assertEqual(result.stored.evidence_at_ms, BASE_MS) - self.assertEqual(result.stored.updated_at_ms, BASE_MS + 5_000) - - def test_never_raises_a_stored_difficulty(self) -> None: - self.seed("65536") - - result = self.store.apply_downward( - **self.key, - difficulty=Decimal("1048576"), - now_ms=BASE_MS + 5_000, - ) - - self.assertFalse(result.applied) - assert result.stored is not None - self.assertEqual(result.stored.difficulty, Decimal("65536")) - self.assertEqual(result.stored.updated_at_ms, BASE_MS) - - def test_equal_difficulty_is_a_no_op(self) -> None: - self.seed("65536") - - result = self.store.apply_downward( - **self.key, - difficulty=Decimal("65536.0"), - now_ms=BASE_MS + 5_000, - ) - - self.assertFalse(result.applied) - self.assertEqual(len(self.store), 1) - - def test_missing_row_is_a_no_op_and_never_an_insert(self) -> None: - result = self.store.apply_downward( - **self.key, - difficulty=Decimal("65536"), - now_ms=BASE_MS, - ) - - # Nothing stored means nothing a restart could resurrect, so there is - # no value to write and no evidence to invent. - self.assertFalse(result.applied) - self.assertIsNone(result.stored) - self.assertEqual(len(self.store), 0) - - def test_a_correction_cannot_suppress_later_share_backed_evidence(self) -> None: - self.seed("1048576", evidence_at_ms=BASE_MS) - self.store.apply_downward( - **self.key, - difficulty=Decimal("65536"), - now_ms=BASE_MS + 60_000, - ) - - # Share evidence recorded after the seed but BEFORE the correction's - # wall clock still has to win: the correction wrote no evidence. - result = self.store.upsert( - **self.key, - difficulty=Decimal("500000"), - evidence_at_ms=BASE_MS + 50, - now_ms=BASE_MS + 70_000, - ) - - self.assertTrue(result.applied) - assert result.stored is not None - self.assertEqual(result.stored.difficulty, Decimal("500000")) - - def test_correction_is_scoped_to_lane_and_exact_username(self) -> None: - self.seed("1048576") - self.store.upsert( - listener="highdiff", - worker_username="miner.worker1", - difficulty=Decimal("1048576"), - evidence_at_ms=BASE_MS, - now_ms=BASE_MS, - ) - self.store.upsert( - listener="default", - worker_username="miner.worker10", - difficulty=Decimal("1048576"), - evidence_at_ms=BASE_MS, - now_ms=BASE_MS, - ) - - self.store.apply_downward( - **self.key, - difficulty=Decimal("65536"), - now_ms=BASE_MS + 1, - ) - - untouched = { - (record.listener, record.worker_username): record.difficulty - for record in self.store.load_recent(evidence_after_ms=0, limit=10) - } - self.assertEqual(untouched[("default", "miner.worker1")], Decimal("65536")) - self.assertEqual( - untouched[("highdiff", "miner.worker1")], Decimal("1048576") - ) - self.assertEqual( - untouched[("default", "miner.worker10")], Decimal("1048576") - ) - - def test_input_validation_matches_the_upsert_path(self) -> None: - with self.assertRaises(ValueError): - self.store.apply_downward( - **self.key, difficulty=Decimal("0"), now_ms=BASE_MS - ) - with self.assertRaises(ValueError): - self.store.apply_downward( - **self.key, difficulty=Decimal("1"), now_ms=-1 - ) - with self.assertRaises(ValueError): - self.store.apply_downward( - listener="", - worker_username="miner.worker1", - difficulty=Decimal("1"), - now_ms=BASE_MS, - ) - - -class LowerOnlySqlTests(unittest.TestCase): - def test_correction_is_one_guarded_atomic_statement(self) -> None: - store = QueryCaptureStore(results=[{"applied": False, "stored": None}]) - - store.apply_downward( - listener="default", - worker_username="miner.worker1", - difficulty=Decimal("65536"), - now_ms=BASE_MS, - ) - - self.assertEqual(len(store.queries), 1) - sql = store.queries[0] - self.assertEqual(sql.count("UPDATE qbit_worker_difficulty"), 1) - # The guard is what makes the correction lower-only... - self.assertIn("AND difficulty > ", sql) - # ...and evidence_at is never on the SET list, which is what stops it - # suppressing later share-backed evidence. - self.assertNotIn("evidence_at =", sql) - self.assertIn("SET difficulty = ", sql) - self.assertIn("updated_at = ", sql) - self.assertNotIn("INSERT", sql) - -if __name__ == "__main__": - unittest.main() diff --git a/tests/test_prism_writer_lease_policy.py b/tests/test_prism_writer_lease_policy.py deleted file mode 100644 index ebdae4f8a..000000000 --- a/tests/test_prism_writer_lease_policy.py +++ /dev/null @@ -1,660 +0,0 @@ -#!/usr/bin/env python3 -"""The writer-lease heartbeat timing policy, as one inequality (issue #212). - -The five heartbeat timings were previously five constants derived from the -adoption silence by hand (``silence / 4``, ``silence * 0.75``) and checked at -startup by a print statement. That is how ``union-mainnet`` came to restart -69 times in one rapid-block burst while every component was healthy: the -derived envelope covered a heartbeat's *idle interval* but not the *tail* of -a lawful verification, and nothing in the code said so. - -These tests hold the policy to three separate obligations: - -``PolicyInequalityTests`` - Safety — the terms that make "this process is gone before a replacement - may adopt" true — and the refusal of combinations that break it. - -``ProductionEvidenceTests`` - The shipped defaults, checked against the numbers issue #212 actually - reported from production. The old policy is *shown* to reject those - observations and the new one to accept them, so the regression is - pinned to evidence rather than to a hand-picked constant. - -``VerificationAttributionTests`` - The phase accumulator: what an operator reads when a heartbeat does - exit. Driven by an injected clock, so every duration in the assertions - is exact rather than approximate. -""" - -from __future__ import annotations - -import unittest - -from lab.prism.coordinator_config import ( - DEFAULT_PRISM_LEDGER_LEASE_HEARTBEAT_EXIT_TIMEOUT_SECONDS, - DEFAULT_PRISM_LEDGER_LEASE_HEARTBEAT_FAILURE_SECONDS, - DEFAULT_PRISM_LEDGER_LEASE_HEARTBEAT_MONITOR_SECONDS, - DEFAULT_PRISM_LEDGER_LEASE_HEARTBEAT_SECONDS, -) -import lab.prism.writer_lease_timing as writer_lease_timing -from lab.prism.writer_lease_timing import ( - DEFAULT_WRITER_LEASE_ADOPTION_SILENCE_SECONDS, - DEFAULT_WRITER_LEASE_HEARTBEAT_POLICY, - LEASE_HEARTBEAT_MODE_PROOF, - LEASE_HEARTBEAT_OUTCOME_PROVEN, - LEASE_HEARTBEAT_PHASES, - LEASE_HEARTBEAT_POLICY_TERMS, - OBSERVED_MONITOR_WAKE_DELAY_SECONDS_227, - WRITER_LEASE_GUARD_STATEMENT_TIMEOUT_SECONDS, - WriterLeaseHeartbeatPolicy, - WriterLeaseHeartbeatPolicyError, - WriterLeaseVerificationAttempt, -) - -# The policy that shipped before this change, reconstructed from the -# constants it derived. Referenced rather than described so the regression -# assertions below are about a real prior configuration. -PRE_212_POLICY = WriterLeaseHeartbeatPolicy( - adoption_silence_seconds=1.0, - heartbeat_interval_seconds=0.25, - failure_budget_seconds=0.75, - monitor_interval_seconds=0.05, - exit_margin_seconds=0.1, -) - -# The staleness cap the pre-#212 code actually enforced, spelled out rather -# than recomputed: it reserved only `exit_margin + 2 * monitor_interval` -# inside the silence and made no room at all for the monitor thread's own -# lateness. Evaluating PRE_212_POLICY through today's corrected envelope -# would report a different (and safer) number than production ran, so the -# evidence assertions below use this constant. -PRE_212_SERVER_PROVEN_CAP_SECONDS = 0.8 - -# The activity budget the same code enforced. -PRE_212_FAILURE_BUDGET_SECONDS = 0.75 - -# The downstream operational mitigation issue #212 describes as temporary: -# it stopped the restart loop but added ~3s to genuine writer failover and -# explained nothing. -DOWNSTREAM_MITIGATION_POLICY = WriterLeaseHeartbeatPolicy( - adoption_silence_seconds=4.0, - heartbeat_interval_seconds=1.0, - failure_budget_seconds=3.0, - monitor_interval_seconds=0.05, - exit_margin_seconds=0.1, -) - -# Server-proven ages, in seconds, reported by representative protective -# exits during the union-mainnet burst on 2026-08-31 (bootstrap pin -# c27a1f5336e661360a86f1df70f2a769e7bcaa14). Every one of these came from a -# coordinator that still held its PostgreSQL advisory guard. -OBSERVED_HEALTHY_SERVER_PROVEN_AGES = (0.76, 0.81, 0.87, 0.91) - -# Activity ages from the same exits. -OBSERVED_HEALTHY_ACTIVITY_AGES = (0.54, 0.61, 0.70, 0.78) - - -class PolicyInequalityTests(unittest.TestCase): - def test_shipped_defaults_satisfy_every_term(self) -> None: - policy = DEFAULT_WRITER_LEASE_HEARTBEAT_POLICY - self.assertEqual(policy.violations(), ()) - self.assertEqual(policy.advisories(), ()) - # (1) exit before adoption, stated as the worst-case timeline - # rather than as a rearranged constant: the newest server-proven - # edge cannot postdate the session's death, the monitor observes - # the stale edge at most one poll plus its whole budgeted lateness - # after the cap elapses, and the hard exit then spends its margin. - # All of that must finish strictly before a successor — which - # cannot start its silence until the guard session is released — - # becomes eligible. - worst_case_exit = ( - policy.server_proven_cap_seconds - + policy.monitor_interval_seconds - + policy.scheduler_slack_seconds - + policy.exit_margin_seconds - ) - self.assertLess(worst_case_exit, policy.adoption_silence_seconds) - # The reserve that makes it strict is one monitor poll. - self.assertAlmostEqual( - policy.adoption_silence_seconds - worst_case_exit, - policy.monitor_interval_seconds, - ) - self.assertLessEqual( - policy.server_proven_cap_seconds + policy.exit_envelope_seconds, - policy.adoption_silence_seconds, - ) - # The startup check is stated on the failure budget, which must - # leave the same envelope strictly free. - self.assertLess( - policy.failure_budget_seconds + policy.exit_envelope_seconds, - policy.adoption_silence_seconds, - ) - # (2) one idle wait cannot exhaust the liveness budget. - self.assertLess( - policy.heartbeat_interval_seconds, - policy.failure_budget_seconds, - ) - # (3) the cap covers the largest gap a healthy coordinator produces. - self.assertGreaterEqual( - policy.server_proven_cap_seconds, - policy.max_healthy_server_gap_seconds, - ) - - def test_configuration_defaults_are_the_policy(self) -> None: - """The coordinator config must not carry a second copy of the numbers.""" - policy = DEFAULT_WRITER_LEASE_HEARTBEAT_POLICY - self.assertEqual( - ( - DEFAULT_PRISM_LEDGER_LEASE_HEARTBEAT_SECONDS, - DEFAULT_PRISM_LEDGER_LEASE_HEARTBEAT_FAILURE_SECONDS, - DEFAULT_PRISM_LEDGER_LEASE_HEARTBEAT_MONITOR_SECONDS, - DEFAULT_PRISM_LEDGER_LEASE_HEARTBEAT_EXIT_TIMEOUT_SECONDS, - DEFAULT_WRITER_LEASE_ADOPTION_SILENCE_SECONDS, - ), - ( - policy.heartbeat_interval_seconds, - policy.failure_budget_seconds, - policy.monitor_interval_seconds, - policy.exit_margin_seconds, - policy.adoption_silence_seconds, - ), - ) - - def test_exit_envelope_reserves_the_monitors_own_lateness(self) -> None: - """The monitor is a thread in the same process as the heartbeat. - - Regression for the review finding on the first cut of this policy: - the envelope reserved only poll granularity and the exit margin, so - a monitor delayed by exactly the slack the policy budgets for the - heartbeat observed the stale edge after the successor's adoption - edge — a stall the policy called acceptable produced two live - writers. - """ - policy = DEFAULT_WRITER_LEASE_HEARTBEAT_POLICY - self.assertAlmostEqual( - policy.exit_envelope_seconds, - policy.exit_margin_seconds - + 2.0 * policy.monitor_interval_seconds - + policy.scheduler_slack_seconds, - ) - # The unreserved spelling is unsafe at the shipped numbers, which - # is why it is not the spelling. - unreserved_envelope = ( - policy.exit_margin_seconds + 2.0 * policy.monitor_interval_seconds - ) - unreserved_cap = ( - policy.adoption_silence_seconds - unreserved_envelope - ) - self.assertGreater( - unreserved_cap - + policy.monitor_interval_seconds - + policy.scheduler_slack_seconds - + policy.exit_margin_seconds, - policy.adoption_silence_seconds, - ) - - def test_healthy_gap_is_derived_from_the_guard_statement_bound(self) -> None: - """The dominant term is the guard session's own statement timeout. - - The monitor can only measure from completed round trips, and - PostgreSQL will let one guarded statement run for the whole - statement_timeout. Any envelope shorter than that plus one idle - interval is a false-exit generator, whatever else is tuned. - """ - policy = DEFAULT_WRITER_LEASE_HEARTBEAT_POLICY - self.assertEqual( - policy.guard_statement_timeout_seconds, - WRITER_LEASE_GUARD_STATEMENT_TIMEOUT_SECONDS, - ) - self.assertEqual( - policy.max_healthy_server_gap_seconds, - policy.heartbeat_interval_seconds - + policy.guard_statement_timeout_seconds - + policy.scheduler_slack_seconds, - ) - - def test_silence_that_cannot_contain_the_exit_envelope_is_unsafe(self) -> None: - policy = WriterLeaseHeartbeatPolicy( - adoption_silence_seconds=1.0, - heartbeat_interval_seconds=0.25, - failure_budget_seconds=1.25, - monitor_interval_seconds=0.05, - exit_margin_seconds=0.1, - ) - violations = policy.violations() - self.assertEqual(len(violations), 1) - self.assertIn("adoption silence", violations[0]) - with self.assertRaises(WriterLeaseHeartbeatPolicyError): - policy.validate() - - def test_interval_at_or_above_the_budget_is_unsafe(self) -> None: - policy = WriterLeaseHeartbeatPolicy( - adoption_silence_seconds=10.0, - heartbeat_interval_seconds=1.0, - failure_budget_seconds=1.0, - monitor_interval_seconds=0.05, - exit_margin_seconds=0.1, - ) - self.assertIn( - "must exceed the heartbeat interval", - "".join(policy.violations()), - ) - - def test_non_finite_and_negative_terms_are_unsafe(self) -> None: - for term in ( - "adoption_silence_seconds", - "heartbeat_interval_seconds", - "failure_budget_seconds", - "monitor_interval_seconds", - "exit_margin_seconds", - ): - for bad in (float("nan"), float("inf"), -1.0): - with self.subTest(term=term, value=bad): - policy = WriterLeaseHeartbeatPolicy( - **{ - **{ - "adoption_silence_seconds": 2.0, - "heartbeat_interval_seconds": 0.25, - "failure_budget_seconds": 1.25, - "monitor_interval_seconds": 0.05, - "exit_margin_seconds": 0.1, - }, - term: bad, - } - ) - self.assertTrue(policy.violations()) - - def test_safe_but_tight_policy_advises_instead_of_refusing(self) -> None: - """Stability is a warning; safety is a refusal. - - A lab or test policy may legitimately run a silence far below the - guard's statement timeout. That cannot produce two writers — the - exit still precedes adoption — so refusing to start would turn a - deliberate trade into an outage. Its scheduler slack is scaled with - the rest of it: the slack is a term of the safety inequality, so a - scaled policy that left it at the production value would be - genuinely unsafe rather than merely tight. - """ - policy = WriterLeaseHeartbeatPolicy( - adoption_silence_seconds=0.5, - heartbeat_interval_seconds=0.01, - failure_budget_seconds=0.2, - monitor_interval_seconds=0.005, - exit_margin_seconds=0.01, - scheduler_slack_seconds=0.005, - ) - self.assertEqual(policy.violations(), ()) - self.assertEqual(len(policy.advisories()), 1) - self.assertIn("issue #212", policy.advisories()[0]) - self.assertLess(policy.stability_surplus_seconds, 0.0) - - def test_describe_names_every_term_of_the_inequality(self) -> None: - described = DEFAULT_WRITER_LEASE_HEARTBEAT_POLICY.describe() - for term in ( - "adoption_silence=", - "interval=", - "failure_budget=", - "monitor=", - "exit_margin=", - "guard_statement_timeout=", - "scheduler_slack=", - "server_proven_cap=", - "healthy_gap=", - "surplus=", - ): - self.assertIn(term, described) - - -class ProductionEvidenceTests(unittest.TestCase): - """The shipped policy, judged against issue #212's production numbers.""" - - def test_pre_212_policy_bounds_fall_inside_healthy_production_ranges( - self, - ) -> None: - """The old envelope hard-exits coordinators that were fine. - - This is the defect, stated as arithmetic. Both of the old bounds - sit *inside* the ranges the burst reported from coordinators that - still held their PostgreSQL advisory guard, so healthy operation - crosses them: a protective exit stops being evidence of a problem - and becomes a function of ordinary latency. The observations are - reported as ranges rather than paired samples, so the claim under - test is about the ranges reaching the bounds, not about which of - the two bounds any individual exit tripped. - """ - self.assertGreaterEqual( - max(OBSERVED_HEALTHY_SERVER_PROVEN_AGES), - PRE_212_SERVER_PROVEN_CAP_SECONDS, - ) - self.assertGreaterEqual( - max(OBSERVED_HEALTHY_ACTIVITY_AGES), - PRE_212_FAILURE_BUDGET_SECONDS, - ) - self.assertEqual( - PRE_212_POLICY.failure_budget_seconds, - PRE_212_FAILURE_BUDGET_SECONDS, - ) - # And the old policy could not have been fixed by tuning alone. It - # does not merely lack tail-latency headroom: judged by the - # corrected exit envelope it never satisfied exit-before-adoption - # at all, because a one-second silence cannot hold a 0.75s budget - # plus the monitor's own budgeted lateness. - self.assertTrue(PRE_212_POLICY.violations()) - self.assertLess(PRE_212_POLICY.stability_surplus_seconds, 0.0) - - def test_shipped_policy_accepts_every_healthy_observation(self) -> None: - policy = DEFAULT_WRITER_LEASE_HEARTBEAT_POLICY - self.assertGreater( - policy.server_proven_cap_seconds, - PRE_212_SERVER_PROVEN_CAP_SECONDS, - ) - for age in OBSERVED_HEALTHY_SERVER_PROVEN_AGES: - with self.subTest(server_proven_age=age): - self.assertLess(age, policy.server_proven_cap_seconds) - for age in OBSERVED_HEALTHY_ACTIVITY_AGES: - with self.subTest(activity_age=age): - self.assertLess(age, policy.failure_budget_seconds) - - def test_shipped_policy_fails_over_faster_than_the_mitigation(self) -> None: - """The point of the upstream fix over the 4-second pin. - - Both are safe; the shipped one costs half the failover latency and - keeps a heartbeat interval short enough to prove ownership several - times inside one silence window. - """ - policy = DEFAULT_WRITER_LEASE_HEARTBEAT_POLICY - self.assertEqual(DOWNSTREAM_MITIGATION_POLICY.violations(), ()) - self.assertLess( - policy.adoption_silence_seconds, - DOWNSTREAM_MITIGATION_POLICY.adoption_silence_seconds, - ) - self.assertGreaterEqual( - policy.adoption_silence_seconds / policy.heartbeat_interval_seconds, - 4.0, - ) - - def test_shipped_silence_is_the_smallest_round_safe_value(self) -> None: - """No unexplained slack: one step down breaks the inequality. - - Guards the derivation itself. The silence is not a comfort value — - 1.95s is the exact floor for these phase budgets, so anything at or - below it must be refused, and the shipped 2.0s is the next round - value above it. - """ - policy = DEFAULT_WRITER_LEASE_HEARTBEAT_POLICY - floor = policy.failure_budget_seconds + policy.exit_envelope_seconds - self.assertAlmostEqual(floor, 1.95) - at_the_floor = WriterLeaseHeartbeatPolicy( - adoption_silence_seconds=floor, - heartbeat_interval_seconds=policy.heartbeat_interval_seconds, - failure_budget_seconds=policy.failure_budget_seconds, - monitor_interval_seconds=policy.monitor_interval_seconds, - exit_margin_seconds=policy.exit_margin_seconds, - ) - self.assertTrue(at_the_floor.violations()) - - -class VerificationAttributionTests(unittest.TestCase): - """Phase attribution: the answer to "which phase consumed the envelope?".""" - - class Clock: - def __init__(self) -> None: - self.now = 1_000.0 - - def __call__(self) -> float: - return self.now - - def advance(self, seconds: float) -> float: - self.now += seconds - return self.now - - def test_phases_separate_queue_wait_sql_and_scheduler_delay(self) -> None: - clock = self.Clock() - attempt = WriterLeaseVerificationAttempt( - LEASE_HEARTBEAT_MODE_PROOF, - monotonic=clock, - ) - clock.advance(0.30) # queued behind another guard caller - attempt.slot_acquired() - clock.advance(0.40) # the statement itself - attempt.statement_completed() - clock.advance(0.05) # this process could not get scheduled - phases = attempt.finish(LEASE_HEARTBEAT_OUTCOME_PROVEN) - - self.assertAlmostEqual(phases.slot_wait_seconds, 0.30) - self.assertAlmostEqual(phases.guard_sql_seconds, 0.40) - self.assertAlmostEqual(phases.scheduler_delay_seconds, 0.05) - self.assertAlmostEqual(phases.total_seconds, 0.75) - self.assertEqual(phases.statement_count, 1) - # No server-reported time: the conservative reading, with the - # whole statement span in guard_sql and nothing in client resume. - self.assertFalse(phases.server_reported) - self.assertAlmostEqual(phases.client_resume_seconds, 0.0) - self.assertEqual( - set(phases.phase_seconds()), - { - "guard_slot_wait", - "guard_sql", - "guard_client_resume", - "scheduler_delay", - "total", - }, - ) - self.assertEqual(tuple(phases.phase_seconds()), LEASE_HEARTBEAT_PHASES) - - def test_summary_names_the_phase_that_consumed_the_time(self) -> None: - clock = self.Clock() - attempt = WriterLeaseVerificationAttempt( - LEASE_HEARTBEAT_MODE_PROOF, - monotonic=clock, - ) - attempt.slot_acquired() - clock.advance(0.45) - attempt.statement_completed() - summary = attempt.finish(LEASE_HEARTBEAT_OUTCOME_PROVEN).summary() - self.assertIn("mode=proof", summary) - self.assertIn("outcome=proven", summary) - self.assertIn("guard_sql=0.450s(1 stmt)", summary) - - def test_proven_edge_is_the_send_time_not_the_receipt_time(self) -> None: - """A response proves liveness no later than when the request left. - - Stamping receipt time would let scheduler delay between the answer - arriving and the stamp make the session look fresher than - PostgreSQL proved, spending adoption envelope the exit ordering - argument budgets for the monitor's own poll. - """ - clock = self.Clock() - attempt = WriterLeaseVerificationAttempt( - LEASE_HEARTBEAT_MODE_PROOF, - monotonic=clock, - ) - attempt.slot_acquired() - sent_at = clock.now - clock.advance(0.40) - proven = attempt.statement_completed() - self.assertEqual(proven, sent_at) - self.assertEqual(attempt.proven_edge_monotonic, sent_at) - - def test_second_statement_proves_from_the_first_response(self) -> None: - """The attribution recheck's send edge is the previous answer. - - A followup leaves for PostgreSQL as soon as the previous result is - in hand, so that instant — not the recheck's completion — is the - conservative edge it proves. - """ - clock = self.Clock() - attempt = WriterLeaseVerificationAttempt("renew", monotonic=clock) - attempt.slot_acquired() - clock.advance(0.20) - attempt.statement_completed() - first_response_at = clock.now - clock.advance(0.30) - self.assertEqual(attempt.statement_completed(), first_response_at) - phases = attempt.finish("renewed") - self.assertEqual(phases.statement_count, 2) - self.assertAlmostEqual(phases.guard_sql_seconds, 0.50) - self.assertAlmostEqual(phases.scheduler_delay_seconds, 0.0) - - def test_unreported_attempt_lands_entirely_in_scheduler_delay(self) -> None: - """A verifier with no callbacks still yields a truthful total. - - Nothing is invented: with no slot mark and no round trip, the whole - duration is unattributed to the database, which is itself the - signal that the attempt never demonstrably reached PostgreSQL. - """ - clock = self.Clock() - attempt = WriterLeaseVerificationAttempt("renew", monotonic=clock) - clock.advance(0.9) - phases = attempt.finish("renewed") - self.assertAlmostEqual(phases.slot_wait_seconds, 0.0) - self.assertAlmostEqual(phases.guard_sql_seconds, 0.0) - self.assertAlmostEqual(phases.scheduler_delay_seconds, 0.9) - self.assertEqual(phases.statement_count, 0) - - -class ProofDocstringTests(unittest.TestCase): - """The module docstring is the proof; it must not drift from the code. - - Issue #227 found the proof presenting ``1.95 < 2.00`` as though the - slack term bounded reality, three days after production had exceeded - it. These tests hold the docstring to the constants, hold the stated - monitor-lateness bound (inequality (4)) to the policy, and pin the - production observation the docstring now discusses to the number the - code carries. - """ - - DOCSTRING = writer_lease_timing.__doc__ or "" - - def _documented_number(self, term: str) -> float: - """The value a ``term = ... = <number>`` line in the defaults table states.""" - import re - - heading = "The shipped defaults\n" - self.assertIn(heading, self.DOCSTRING) - table = self.DOCSTRING.split(heading, 1)[1] - pattern = re.compile( - r"^\s+(?:=>\s+)?" + re.escape(term) + r"\s+=\s+(?P<expr>[^(\n]+)", - re.MULTILINE, - ) - match = pattern.search(table) - self.assertIsNotNone(match, f"defaults table has no line for {term}") - assert match is not None - numbers = re.findall(r"[0-9]+\.[0-9]+", match.group("expr")) - self.assertTrue(numbers, f"no number on the {term} line") - # A derived line spells the arithmetic and ends with its result. - return float(numbers[-1]) - - def test_documented_shipped_numbers_match_the_constants(self) -> None: - policy = DEFAULT_WRITER_LEASE_HEARTBEAT_POLICY - for term, actual in ( - ("guard_statement_timeout", policy.guard_statement_timeout_seconds), - ("heartbeat_interval", policy.heartbeat_interval_seconds), - ("scheduler_slack", policy.scheduler_slack_seconds), - ("max_healthy_server_gap", policy.max_healthy_server_gap_seconds), - ("failure_budget", policy.failure_budget_seconds), - ("monitor_interval", policy.monitor_interval_seconds), - ("exit_margin", policy.exit_margin_seconds), - ("exit_envelope", policy.exit_envelope_seconds), - ("adoption_silence", policy.adoption_silence_seconds), - ("server_proven_cap", policy.server_proven_cap_seconds), - ("stability surplus", policy.stability_surplus_seconds), - ( - "max_guaranteed_monitor_lateness", - policy.max_guaranteed_monitor_lateness_seconds, - ), - ): - with self.subTest(term=term): - self.assertAlmostEqual(self._documented_number(term), actual) - - def test_inequality_one_is_still_rejected_when_violated(self) -> None: - """Guarding the proof: an unsafe silence is refused, not advised.""" - policy = DEFAULT_WRITER_LEASE_HEARTBEAT_POLICY - floor = policy.failure_budget_seconds + policy.exit_envelope_seconds - for silence in (floor, floor - 0.01, 1.0): - with self.subTest(adoption_silence=silence): - unsafe = WriterLeaseHeartbeatPolicy( - adoption_silence_seconds=silence, - heartbeat_interval_seconds=policy.heartbeat_interval_seconds, - failure_budget_seconds=policy.failure_budget_seconds, - monitor_interval_seconds=policy.monitor_interval_seconds, - exit_margin_seconds=policy.exit_margin_seconds, - ) - self.assertTrue(unsafe.violations()) - with self.assertRaises(WriterLeaseHeartbeatPolicyError): - unsafe.validate() - - def test_monitor_lateness_bound_is_inequality_four(self) -> None: - """(4): exit precedes adoption iff L < slack + monitor_interval. - - Stated from the worst-case timeline rather than from the property: - the poll that first sees the stale edge is at most one interval - plus L after the cap elapsed, the exit spends its margin, and the - successor is eligible one silence after the proven edge. - """ - policy = DEFAULT_WRITER_LEASE_HEARTBEAT_POLICY - bound = policy.max_guaranteed_monitor_lateness_seconds - self.assertAlmostEqual( - bound, - policy.scheduler_slack_seconds + policy.monitor_interval_seconds, - ) - self.assertAlmostEqual(bound, 0.55) - for lateness, guaranteed in ( - (policy.scheduler_slack_seconds, True), - (bound - 1e-9, True), - (bound + 1e-9, False), - (OBSERVED_MONITOR_WAKE_DELAY_SECONDS_227, False), - ): - with self.subTest(lateness=lateness): - worst_case_exit = ( - policy.server_proven_cap_seconds - + policy.monitor_interval_seconds - + lateness - + policy.exit_margin_seconds - ) - self.assertEqual( - worst_case_exit < policy.adoption_silence_seconds, - guaranteed, - ) - self.assertEqual( - policy.exit_guarantee_overrun_seconds(lateness) == 0.0, - guaranteed, - ) - - def test_production_observation_exceeds_the_bound_and_is_documented( - self, - ) -> None: - """The #227 record breaks the assumption, and the docstring says so.""" - policy = DEFAULT_WRITER_LEASE_HEARTBEAT_POLICY - observed = OBSERVED_MONITOR_WAKE_DELAY_SECONDS_227 - self.assertGreater(observed, policy.scheduler_slack_seconds) - self.assertGreater(observed, policy.max_guaranteed_monitor_lateness_seconds) - self.assertAlmostEqual( - policy.exit_guarantee_overrun_seconds(observed), - observed - 0.55, - ) - # The sum the spec quotes, with the real lateness in place of slack. - self.assertAlmostEqual( - policy.server_proven_cap_seconds - + policy.monitor_interval_seconds - + observed - + policy.exit_margin_seconds, - 2.098, - ) - self.assertIn(f"{observed:.3f}s", self.DOCSTRING) - self.assertIn("2.098 > 2.00", self.DOCSTRING) - self.assertIn("issue #227", self.DOCSTRING) - # The decision is recorded, not just the observation. - self.assertIn("(b)", self.DOCSTRING) - self.assertIn("the shipped response is (b)", self.DOCSTRING) - - def test_policy_terms_and_describe_include_the_lateness_bound(self) -> None: - self.assertIn("max_guaranteed_monitor_lateness", LEASE_HEARTBEAT_POLICY_TERMS) - self.assertIn( - "max_monitor_lateness=0.55s", - DEFAULT_WRITER_LEASE_HEARTBEAT_POLICY.describe(), - ) - - -if __name__ == "__main__": - unittest.main() diff --git a/tests/test_public_dashboard_api_contract.py b/tests/test_public_dashboard_api_contract.py deleted file mode 100644 index 709798424..000000000 --- a/tests/test_public_dashboard_api_contract.py +++ /dev/null @@ -1,761 +0,0 @@ -#!/usr/bin/env python3 - -from __future__ import annotations - -import json -import math -import re -import unittest -from datetime import datetime -from decimal import Decimal -from pathlib import Path -from typing import Any - -from lab.prism import public_api - - -ROOT = Path(__file__).resolve().parents[1] -OPENAPI_PATH = ROOT / "docs" / "public-dashboard-api-v1.openapi.yaml" -CONTRACT_DIR = ROOT / "docs" / "public-dashboard-api" -FIXTURE_DIR = CONTRACT_DIR / "fixtures" - -EXPECTED_FIXTURES = { - "pool-summary.json": "prism.dashboard.pool-summary.v1", - "hashrate-series.json": "prism.dashboard.hashrate-series.v1", - "hashrate-series-dual-rate.json": "prism.dashboard.hashrate-series.v2", - "leaderboard.json": "prism.dashboard.leaderboard.v2", - "leaderboard-legacy.json": "prism.dashboard.leaderboard.v1", - "blocks.json": "prism.dashboard.blocks.v1", - "blocks-chain-states.json": "prism.dashboard.blocks.v2", - "block-markers.json": "prism.dashboard.block-markers.v1", - "settlement-artifacts.json": "prism.dashboard.settlement-artifacts.v1", - "settlement-artifacts-direct-coinbase.json": "prism.dashboard.settlement-artifacts.v1", - "pending-fanouts.json": "prism.dashboard.pending-fanouts.v1", - "fanout.json": "prism.dashboard.fanout.v1", - "mining-configuration.json": "prism.dashboard.mining-configuration.v1", - "miner.json": "prism.dashboard.miner.v1", - "miner-earnings.json": "prism.dashboard.miner-earnings.v1", - "miner-payouts.json": "prism.dashboard.miner-payouts.v1", - "miner-workers.json": "prism.dashboard.miner-workers.v1", - "error.json": "prism.dashboard.error.v1", -} - -EXPECTED_ROUTES = ( - "/pool-summary:", - "/hashrate-series:", - "/block-markers:", - "/leaderboard:", - "/blocks:", - "/blocks/{block_hash}/settlement-artifacts:", - "/fanouts/pending:", - "/fanouts/{fanout_txid}:", - "/artifacts/{sha256}:", - "/mining-configuration:", - "/miners/{recipient_id}:", - "/miners/{recipient_id}/earnings:", - "/miners/{recipient_id}/payouts:", - "/miners/{recipient_id}/workers:", -) - -DECIMAL_KEYS = { - "network_difficulty", - "requested_window_weight", - "accepted_share_difficulty", - "pool_accepted_share_difficulty", - "solver_share_difficulty", - "reward_window_weight", - "accepted_difficulty_3h", -} - -HASHRATE_ROLLUP_KEYS = {"h1", "h3", "h24", "m1", "m5", "m10"} -HEX_HASH_KEYS = {"hash", "tip_hash", "block_hash", "fanout_txid", "transaction_id", "parent_coinbase_txid", "ctv_hash", "sha256"} -HEX_STRING_KEYS = {"fanout_tx_hex"} - -DECIMAL_PATTERN = re.compile(r"^-?[0-9]+(?:\.[0-9]+)?$") -HEX_HASH_PATTERN = re.compile(r"^[0-9a-f]{64}$") -HEX_STRING_PATTERN = re.compile(r"^(?:[0-9a-f]{2})+$") - - -class PublicDashboardApiContractTests(unittest.TestCase): - def test_contract_files_exist(self) -> None: - self.assertTrue(OPENAPI_PATH.is_file()) - self.assertTrue((CONTRACT_DIR / "README.md").is_file()) - for fixture_name in EXPECTED_FIXTURES: - self.assertTrue((FIXTURE_DIR / fixture_name).is_file(), fixture_name) - - def test_openapi_declares_expected_routes_and_schema_tags(self) -> None: - text = OPENAPI_PATH.read_text(encoding="utf-8") - for route in EXPECTED_ROUTES: - self.assertIn(route, text) - for schema_tag in EXPECTED_FIXTURES.values(): - self.assertIn(schema_tag, text) - - def test_openapi_narrows_ambiguous_or_contextual_types(self) -> None: - text = OPENAPI_PATH.read_text(encoding="utf-8") - self.assertIn('pattern: "^(?:[0-9a-f]{2})+$"', text) - self.assertIn("1m = 30 days", text) - self.assertIn("5m = 5 minutes", text) - self.assertIn("PendingSettlementStatus:", text) - self.assertIn("$ref: \"#/components/schemas/PendingFanoutArtifact\"", text) - self.assertIn("HashrateSubject:", text) - self.assertIn("const: pool", text) - self.assertIn("type: \"null\"", text) - self.assertIn("const: miner", text) - self.assertIn("minLength: 1", text) - self.assertIn("Rejected for `window=3h`", text) - self.assertIn( - "pool_fee_bps:\n type: integer\n minimum: 0\n maximum: 10000", - text, - ) - - def test_public_contract_avoids_retired_nomenclature(self) -> None: - contract_files = [OPENAPI_PATH, CONTRACT_DIR / "README.md", *FIXTURE_DIR.glob("*.json")] - for path in contract_files: - text = path.read_text(encoding="utf-8").lower() - self.assertNotIn("tides", text, str(path)) - self.assertNotIn("qbit_tides", text, str(path)) - self.assertNotIn("qbit.tides", text, str(path)) - self.assertNotIn("_sats", text, str(path)) - self.assertNotIn("satoshis", text, str(path)) - - def test_fixtures_have_expected_schema_tags(self) -> None: - for fixture_name, schema_tag in EXPECTED_FIXTURES.items(): - payload = self.load_fixture(fixture_name) - self.assertEqual(payload["schema"], schema_tag) - - def test_fixtures_follow_common_field_conventions(self) -> None: - for fixture_name in EXPECTED_FIXTURES: - payload = self.load_fixture(fixture_name) - self.assert_conventions(payload, fixture_name) - - def test_pagination_fixtures_are_consistent(self) -> None: - for fixture_name in ( - "leaderboard.json", - "leaderboard-legacy.json", - "blocks.json", - "blocks-chain-states.json", - "pending-fanouts.json", - "miner-earnings.json", - "miner-payouts.json", - "miner-workers.json", - ): - fixture = self.load_fixture(fixture_name) - pagination = fixture["pagination"] - rows = fixture["rows"] - self.assertGreaterEqual(pagination["page"], 1) - self.assertGreaterEqual(pagination["limit"], 1) - self.assertGreaterEqual(pagination["total_count"], 0) - self.assertGreaterEqual(pagination["total_pages"], 0) - expected_total_pages = math.ceil(pagination["total_count"] / pagination["limit"]) if pagination["total_count"] else 0 - self.assertEqual(pagination["total_pages"], expected_total_pages) - self.assertLessEqual(len(rows), pagination["limit"]) - if pagination["total_count"]: - self.assertLessEqual(len(rows), pagination["total_count"]) - else: - self.assertEqual(rows, []) - - def test_miner_earnings_fixture_math_is_consistent(self) -> None: - blocks_by_hash = { - row["hash"]: row - for row in self.load_fixture("blocks.json")["rows"] - } - payouts_by_hash = { - row["block_hash"]: row - for row in self.load_fixture("miner-payouts.json")["rows"] - } - recent_payouts_by_hash = { - row["block_hash"]: row - for row in self.load_fixture("miner.json")["recent_payouts"] - } - for row in self.load_fixture("miner-earnings.json")["rows"]: - gross = row["gross_earning_bits"] - settlement_fee = row["settlement_fee_bits"] - net = row["net_earning_bits"] - self.assertGreaterEqual(settlement_fee, 0) - self.assertEqual(net, gross - settlement_fee) - - block = blocks_by_hash[row["block_hash"]] - expected_percent = (Decimal(gross) * Decimal("100") / Decimal(block["coinbase_value_bits"])).quantize( - Decimal("0.01") - ) - self.assertEqual(Decimal(row["reward_share_percent"]), expected_percent) - - self.assertIn(row["block_hash"], payouts_by_hash) - self.assertIn(row["block_hash"], recent_payouts_by_hash) - payout = payouts_by_hash[row["block_hash"]] - recent_payout = recent_payouts_by_hash[row["block_hash"]] - self.assertLessEqual(payout["onchain_amount_bits"], net) - self.assertEqual(recent_payout["onchain_amount_bits"], payout["onchain_amount_bits"]) - - def test_live_reward_share_is_consistent_and_distinct_from_historical_payout(self) -> None: - miner = self.load_fixture("miner.json") - earnings = self.load_fixture("miner-earnings.json") - leaderboard = self.load_fixture("leaderboard.json") - matching_rows = [ - row - for row in leaderboard["rows"] - if row["recipient_id"] == earnings["recipient_id"] - ] - - self.assertEqual(len(matching_rows), 1) - live_percent = Decimal(miner["reward_window_percent"]) - self.assertEqual(Decimal(miner["estimated_next_block"]["share_percent"]), live_percent) - self.assertEqual(Decimal(matching_rows[0]["share_percent"]), live_percent) - self.assertNotEqual(Decimal(earnings["rows"][0]["reward_share_percent"]), live_percent) - - latest_coinbase = self.load_fixture("blocks.json")["rows"][0]["coinbase_value_bits"] - pool_fee_bps = self.load_fixture("mining-configuration.json")["configurations"][0]["pool_fee_bps"] - self.assertEqual( - miner["estimated_next_block"]["estimated_reward_bits"], - public_api.estimated_next_block_reward_bits( - share_percent=str(live_percent), - expected_coinbase_bits=latest_coinbase, - pool_fee_bps=pool_fee_bps, - ), - ) - - def test_reward_leaderboard_share_percent_matches_counted_work(self) -> None: - leaderboard = self.load_fixture("leaderboard.json") - counted_window_weight = Decimal(leaderboard["window"]["counted_window_weight"]) - pool_hashrate = Decimal(leaderboard["totals"]["pool_hashrate_ths"]) - observed_span = leaderboard["window"]["observed_span_seconds"] - self.assertEqual( - counted_window_weight, - Decimal(leaderboard["totals"]["pool_counted_share_difficulty"]), - ) - self.assertEqual( - pool_hashrate, - Decimal(public_api.hashrate_ths_from_difficulty(counted_window_weight, observed_span)), - ) - for row in leaderboard["rows"]: - expected_percent = (Decimal(row["counted_share_difficulty"]) * Decimal("100") / counted_window_weight).quantize( - Decimal("0.01") - ) - self.assertEqual(Decimal(row["share_percent"]), expected_percent) - self.assertEqual( - Decimal(row["hashrate_ths"]), - Decimal(public_api.hashrate_ths_from_difficulty(row["counted_share_difficulty"], observed_span)), - ) - hashrate_percent = (Decimal(row["hashrate_ths"]) * Decimal("100") / pool_hashrate).quantize( - Decimal("0.01") - ) - self.assertEqual(Decimal(row["share_percent"]), hashrate_percent) - - def test_complete_reward_window_fixture_eta_matches_observed_span(self) -> None: - leaderboard = self.load_fixture("leaderboard.json") - - self.assertTrue(leaderboard["window"]["is_complete"]) - observed_span = leaderboard["window"]["observed_span_seconds"] - expected_eta = leaderboard["totals"]["expected_time_to_block_seconds"] - self.assertLessEqual(abs(expected_eta - observed_span / leaderboard["window"]["window_multiplier"]), 1) - - def test_legacy_leaderboard_fixture_retains_exact_v1_shape(self) -> None: - leaderboard = self.load_fixture("leaderboard-legacy.json") - - self.assertEqual(leaderboard["window"]["id"], "3h") - self.assertEqual( - set(leaderboard["totals"]), - {"pool_hashrate_ths", "pool_accepted_share_difficulty", "participant_count"}, - ) - self.assertEqual( - set(leaderboard["rows"][0]), - { - "rank", - "recipient_id", - "display_name", - "hashrate_ths_3h", - "share_percent", - "hash_percent", - "blocks_found", - "last_share_at", - }, - ) - - def test_hashrate_subject_fixture_matches_subject_type(self) -> None: - for fixture_name in ("hashrate-series.json", "hashrate-series-dual-rate.json"): - subject = self.load_fixture(fixture_name)["subject"] - if subject["type"] == "pool": - self.assertIsNone(subject["id"], fixture_name) - else: - self.assertEqual(subject["type"], "miner", fixture_name) - self.assertIsInstance(subject["id"], str, fixture_name) - self.assertGreater(len(subject["id"]), 0, fixture_name) - - def test_legacy_hashrate_series_fixture_retains_exact_v1_shape(self) -> None: - series = self.load_fixture("hashrate-series.json") - - self.assertEqual(set(series), {"schema", "generated_at", "subject", "range", "bucket", "unit", "points"}) - self.assertGreater(len(series["points"]), 0) - for point in series["points"]: - self.assertEqual( - set(point), - {"timestamp", "hashrate_ths", "accepted_share_count", "accepted_share_difficulty"}, - ) - - def test_dual_rate_hashrate_series_fixture_is_internally_consistent(self) -> None: - series = self.load_fixture("hashrate-series-dual-rate.json") - - self.assertEqual( - set(series), - { - "schema", - "generated_at", - "subject", - "range", - "bucket", - "bucket_seconds", - "unit", - "rate_basis", - "smoothing", - "points", - }, - ) - self.assertEqual(series["unit"], "ths") - self.assertEqual(series["rate_basis"], "accepted_share_difficulty") - bucket_seconds = series["bucket_seconds"] - self.assertEqual(bucket_seconds, public_api.HASHRATE_SERIES_BUCKET_SECONDS[series["bucket"]]) - smoothing = series["smoothing"] - self.assertEqual(set(smoothing), {"method", "window_seconds"}) - window_seconds = smoothing["window_seconds"] - self.assertIsInstance(window_seconds, int) - self.assertEqual(window_seconds % bucket_seconds, 0) - if smoothing["method"] == "none": - self.assertEqual(window_seconds, bucket_seconds) - else: - self.assertEqual(smoothing["method"], "trailing") - self.assertGreaterEqual(window_seconds // bucket_seconds, 2) - self.assertEqual( - smoothing, - public_api.hashrate_series_smoothing(bucket_seconds=bucket_seconds, window_seconds=window_seconds), - ) - - generated_at_epoch = self.epoch(series["generated_at"]) - points = series["points"] - self.assertGreater(len(points), 0) - epochs = [self.epoch(point["timestamp"]) for point in points] - self.assertEqual(epochs, sorted(set(epochs))) - for point, epoch in zip(points, epochs): - self.assertEqual( - set(point), - { - "timestamp", - "raw_hashrate_ths", - "smoothed_hashrate_ths", - "accepted_share_count", - "accepted_share_difficulty", - "complete", - }, - ) - self.assertEqual(epoch % bucket_seconds, 0, point["timestamp"]) - self.assertIsInstance(point["complete"], bool) - self.assertEqual(point["complete"], generated_at_epoch >= epoch + bucket_seconds, point["timestamp"]) - # Every raw rate is credit over the full bucket, so the trailing - # estimate is the window's raw rates summed and re-scaled from - # bucket duration to window duration, with missing buckets as zero. - window_raw_total = sum( - Decimal(other["raw_hashrate_ths"]) - for other, other_epoch in zip(points, epochs) - if epoch - window_seconds < other_epoch <= epoch - ) - # Each public rate is independently rounded by decimal_string, so - # their re-scaled totals may differ by the final decimal place. - smoothed_total = Decimal(point["smoothed_hashrate_ths"]) * Decimal( - window_seconds - ) - raw_total = window_raw_total * Decimal(bucket_seconds) - self.assertLessEqual( - abs(smoothed_total - raw_total), - max(abs(raw_total) * Decimal("1e-30"), Decimal("1e-30")), - point["timestamp"], - ) - if smoothing["method"] == "none": - self.assertEqual(Decimal(point["smoothed_hashrate_ths"]), Decimal(point["raw_hashrate_ths"])) - - def test_block_markers_fixture_is_internally_consistent(self) -> None: - markers = self.load_fixture("block-markers.json") - - self.assertEqual( - set(markers), - {"schema", "generated_at", "range", "bucket", "bucket_seconds", "total_blocks", "points"}, - ) - bucket_seconds = markers["bucket_seconds"] - self.assertEqual(bucket_seconds, public_api.HASHRATE_SERIES_BUCKET_SECONDS[markers["bucket"]]) - self.assertIn(markers["bucket"], public_api.HASHRATE_SERIES_ALLOWED_BUCKETS[markers["range"]]) - points = markers["points"] - self.assertGreater(len(points), 0) - epochs = [self.epoch(point["timestamp"]) for point in points] - self.assertEqual(epochs, sorted(set(epochs))) - # total_blocks counts every non-reversed block in range and every such - # block lands in some listed bucket, so the two totals must agree. - self.assertEqual( - markers["total_blocks"], - sum(point["block_count"] for point in points), - ) - for point, epoch in zip(points, epochs): - self.assertEqual(set(point), {"timestamp", "block_count", "blocks", "truncated"}) - # Marker buckets sit on the hashrate chart's floor-aligned grid. - self.assertEqual(epoch % bucket_seconds, 0, point["timestamp"]) - self.assertGreater(point["block_count"], 0) - blocks = point["blocks"] - self.assertEqual( - len(blocks), - min(point["block_count"], public_api.BLOCK_MARKERS_MAX_BLOCKS_PER_BUCKET), - ) - self.assertEqual( - point["truncated"], - point["block_count"] > public_api.BLOCK_MARKERS_MAX_BLOCKS_PER_BUCKET, - ) - found_epochs = [self.epoch(block["found_at"]) for block in blocks] - self.assertEqual(found_epochs, sorted(found_epochs, reverse=True)) - for block, found_epoch in zip(blocks, found_epochs): - self.assertEqual(set(block), {"height", "hash", "found_at"}) - self.assertEqual(found_epoch // bucket_seconds * bucket_seconds, epoch, block["hash"]) - - def test_block_markers_fixture_shares_the_blocks_fixture_pool(self) -> None: - # The newest marker is blocks.json's latest block, so the two fixtures - # describe one pool and a dashboard can render them together. - markers = self.load_fixture("block-markers.json") - latest = self.load_fixture("blocks.json")["rows"][0] - newest_marker = markers["points"][-1]["blocks"][0] - self.assertEqual(newest_marker["hash"], latest["hash"]) - self.assertEqual(newest_marker["height"], latest["height"]) - self.assertEqual(newest_marker["found_at"], latest["found_at"]) - - def test_openapi_block_markers_declares_marker_contract(self) -> None: - text = OPENAPI_PATH.read_text(encoding="utf-8") - self.assertIn( - " BlockMarkerPoint:\n" - " type: object\n" - " additionalProperties: false\n" - " required: [timestamp, block_count, blocks, truncated]\n", - text, - ) - self.assertIn( - " BlockMarkerBlock:\n" - " type: object\n" - " additionalProperties: false\n" - " required: [height, hash, found_at]\n", - text, - ) - self.assertIn("const: prism.dashboard.block-markers.v1", text) - self.assertIn("maxItems: 3", text) - - readme_text = (CONTRACT_DIR / "README.md").read_text(encoding="utf-8") - self.assertIn("/public/v1/block-markers", readme_text) - self.assertIn("block-markers.json", readme_text) - - def test_openapi_hashrate_series_declares_dual_rate_view(self) -> None: - text = OPENAPI_PATH.read_text(encoding="utf-8") - self.assertIn("- name: view\n in: query", text) - self.assertIn("enum: [both]", text) - self.assertIn('- $ref: "#/components/schemas/HashrateSeriesResponse"', text) - self.assertIn('- $ref: "#/components/schemas/HashrateSeriesDualRateResponse"', text) - self.assertIn("HashrateSmoothing:", text) - self.assertIn("enum: [trailing, none]", text) - self.assertIn("const: accepted_share_difficulty", text) - self.assertIn("credited hashrate", text) - # The documented required fields are exactly the fixture's keys. - self.assertIn( - " DualRateHashratePoint:\n" - " type: object\n" - " additionalProperties: false\n" - " required:\n" - " - timestamp\n" - " - raw_hashrate_ths\n" - " - smoothed_hashrate_ths\n" - " - accepted_share_count\n" - " - accepted_share_difficulty\n" - " - complete\n", - text, - ) - self.assertIn( - " HashrateSeriesDualRateResponse:\n" - " type: object\n" - " additionalProperties: false\n" - " required:\n" - " - schema\n" - " - generated_at\n" - " - subject\n" - " - range\n" - " - bucket\n" - " - bucket_seconds\n" - " - unit\n" - " - rate_basis\n" - " - smoothing\n" - " - points\n", - text, - ) - # The legacy point schema is untouched. - self.assertIn( - " HashratePoint:\n" - " type: object\n" - " additionalProperties: false\n" - " required:\n" - " - timestamp\n" - " - hashrate_ths\n" - " - accepted_share_count\n" - " - accepted_share_difficulty\n", - text, - ) - - readme_text = (CONTRACT_DIR / "README.md").read_text(encoding="utf-8") - self.assertIn("credited hashrate", readme_text) - self.assertIn("`view=both`", readme_text) - self.assertIn("prism.dashboard.hashrate-series.v2", readme_text) - self.assertIn("PRISM_PUBLIC_HASHRATE_SMOOTHING_SECONDS", readme_text) - self.assertIn("router telemetry", readme_text) - - def test_fanout_fixture_status_matches_tip_height(self) -> None: - tip_height = self.load_fixture("pool-summary.json")["network"]["height"] - fanout_rows = [ - self.load_fixture("fanout.json")["fanout"], - *self.load_fixture("pending-fanouts.json")["rows"], - *self.load_fixture("settlement-artifacts.json")["fanouts"], - ] - for row in fanout_rows: - broadcastable_at_height = row["broadcastable_at_height"] - if broadcastable_at_height is not None and tip_height < broadcastable_at_height: - self.assertEqual(row["status"], "awaiting_maturity") - self.assertFalse(row["cpfp_anchor_spendable"]) - - def test_reward_window_weight_fixtures_match_network_difficulty(self) -> None: - pool_summary = self.load_fixture("pool-summary.json") - window_multiplier = Decimal(pool_summary["pool"]["reward_window"]["window_multiplier"]) - expected_window_weight = Decimal(pool_summary["network"]["network_difficulty"]) * Decimal( - window_multiplier - ) - self.assertEqual(Decimal(pool_summary["pool"]["reward_window"]["requested_window_weight"]), expected_window_weight) - - for row in self.load_fixture("blocks.json")["rows"]: - if row["reward_window_weight"] is None: - continue - self.assertEqual(Decimal(row["reward_window_weight"]), Decimal(row["network_difficulty"]) * window_multiplier) - - def test_current_fixture_difficulty_is_derived_from_compact_bits(self) -> None: - pool_summary = self.load_fixture("pool-summary.json") - expected_difficulty = public_api.scaled_network_difficulty(pool_summary["network"]["bits"]) - - self.assertEqual( - Decimal(pool_summary["network"]["network_difficulty"]), - expected_difficulty, - ) - self.assertEqual( - Decimal(self.load_fixture("leaderboard.json")["window"]["network_difficulty"]), - expected_difficulty, - ) - self.assertEqual( - pool_summary["pool"]["expected_time_to_block_seconds"], - public_api.expected_time_to_block_seconds( - hashrate_ths=pool_summary["pool"]["hashrate_ths"]["h3"], - network_difficulty=str(expected_difficulty), - ), - ) - - def test_historical_block_fixture_difficulty_is_derived_from_compact_bits(self) -> None: - for fixture_name in ("blocks.json", "blocks-chain-states.json"): - for row in self.load_fixture(fixture_name)["rows"]: - self.assertEqual( - Decimal(row["network_difficulty"]), - public_api.scaled_network_difficulty(row["bits"]), - fixture_name, - ) - - BLOCK_ROW_V1_KEYS = { - "height", - "hash", - "found_at", - "network_difficulty", - "bits", - "solver_recipient_id", - "solver_worker_name", - "solver_share_difficulty", - "reward_window_weight", - "coinbase_value_bits", - "audit_bundle_sha256", - "payout_manifest_sha256", - "explorer_url", - } - - def test_default_blocks_fixture_keeps_exact_v1_row_shape(self) -> None: - # The chain_state filter must not leak into the default response: the - # v1 fixture stays byte-shape identical to the pre-filter contract. - for row in self.load_fixture("blocks.json")["rows"]: - self.assertEqual(set(row), self.BLOCK_ROW_V1_KEYS) - - def test_chain_state_blocks_fixture_adds_only_reorg_visibility_fields(self) -> None: - fixture = self.load_fixture("blocks-chain-states.json") - rows = fixture["rows"] - - self.assertGreater(len(rows), 0) - reversed_rows = [row for row in rows if row["chain_state"] == "reversed"] - self.assertGreater(len(reversed_rows), 0) - for row in rows: - self.assertEqual( - set(row), - self.BLOCK_ROW_V1_KEYS | {"chain_state", "disconnected_at"}, - ) - self.assertIn( - row["chain_state"], - {"prepared", "confirmed", "inactive", "rejected", "reversed"}, - ) - if row["chain_state"] == "reversed": - self.assertIsNotNone(row["disconnected_at"]) - else: - self.assertIsNone(row["disconnected_at"]) - - def test_pool_summary_fixture_reports_reorg_counts_and_network_hashrate(self) -> None: - pool_summary = self.load_fixture("pool-summary.json") - pool = pool_summary["pool"] - - for key in ("blocks_reversed_total", "blocks_inactive_total"): - self.assertIsInstance(pool[key], int) - self.assertGreaterEqual(pool[key], 0) - network_hashrate = pool_summary["network"]["hashrate_ths"] - self.assertIsInstance(network_hashrate, str) - self.assertRegex(network_hashrate, DECIMAL_PATTERN) - # The pool's 3h credited rate divided by the network estimate is the - # documented share-of-network ratio; the fixture keeps it plausible. - share = Decimal(pool["hashrate_ths"]["h3"]) / Decimal(network_hashrate) - self.assertGreater(share, 0) - self.assertLess(share, 1) - - def test_openapi_declares_blocks_chain_state_filter_and_summary_reorg_fields(self) -> None: - text = OPENAPI_PATH.read_text(encoding="utf-8") - self.assertIn("- name: chain_state\n in: query", text) - self.assertIn("enum: [active, all, reversed]", text) - self.assertIn('- $ref: "#/components/schemas/BlocksResponse"', text) - self.assertIn('- $ref: "#/components/schemas/BlocksChainStateResponse"', text) - self.assertIn("ChainStateBlockRow:", text) - self.assertIn("const: prism.dashboard.blocks.v2", text) - self.assertIn("ChainState:", text) - self.assertIn("- disconnected_at", text) - self.assertIn("- blocks_reversed_total", text) - self.assertIn("- blocks_inactive_total", text) - # network.hashrate_ths is required-but-nullable and documents the - # credited-work vs chain-estimate caveat. - self.assertIn("- hashrate_ths\n - initial_block_download", text) - self.assertIn("ratio is approximate", text) - - readme_text = (CONTRACT_DIR / "README.md").read_text(encoding="utf-8") - self.assertIn("`chain_state=all`", readme_text) - self.assertIn("`chain_state=reversed`", readme_text) - self.assertIn("prism.dashboard.blocks.v2", readme_text) - self.assertIn("blocks_reversed_total", readme_text) - self.assertIn("blocks_inactive_total", readme_text) - self.assertIn("network.hashrate_ths", readme_text) - - def test_miner_summary_embeds_only_bounded_previews(self) -> None: - miner = self.load_fixture("miner.json") - self.assertLessEqual(len(miner["workers"]), 5) - self.assertLessEqual(len(miner["recent_payouts"]), 5) - - text = OPENAPI_PATH.read_text(encoding="utf-8") - self.assertIn("workers:\n type: array\n maxItems: 5", text) - self.assertIn("recent_payouts:\n type: array\n maxItems: 5", text) - - def test_miner_fixture_uses_default_minimum_payout_floor(self) -> None: - miner = self.load_fixture("miner.json") - self.assertEqual(miner["minimum_payout_bits"], 10_485_760) - self.assertLess(miner["owed_balance_bits"], miner["minimum_payout_bits"]) - self.assertIsNone(miner["estimated_time_to_minimum_payout_seconds"]) - - def test_miner_contract_exposes_pending_maturity_total(self) -> None: - miner = self.load_fixture("miner.json") - earnings = self.load_fixture("miner-earnings.json")["rows"] - expected_pending = sum( - row["net_earning_bits"] - for row in earnings - if row["maturity_state"] == "immature" - ) - - self.assertEqual(miner["pending_maturity_bits"], expected_pending) - text = OPENAPI_PATH.read_text(encoding="utf-8") - self.assertIn("- pending_maturity_bits", text) - self.assertIn('pending_maturity_bits:\n $ref: "#/components/schemas/Bits"', text) - - def test_dashboard_api_is_public_read_model_not_internal_audit_api(self) -> None: - text = OPENAPI_PATH.read_text(encoding="utf-8") - self.assertNotIn("/audit/", text) - self.assertNotIn("postgres", text.lower()) - self.assertNotIn("rpc password", text.lower()) - - def test_public_api_contract_documents_cache_headers_and_knobs(self) -> None: - openapi_text = OPENAPI_PATH.read_text(encoding="utf-8") - readme_text = (CONTRACT_DIR / "README.md").read_text(encoding="utf-8") - - for header in ("Cache-Control", "CDN-Cache-Control", "Vercel-CDN-Cache-Control", "Age"): - self.assertIn(header, openapi_text) - self.assertIn("BrowserCacheControl:", openapi_text) - self.assertIn("SharedCacheControl:", openapi_text) - self.assertIn("Cache-Control: no-store", readme_text) - self.assertIn("PRISM_PUBLIC_CACHE_TTL_SECONDS", readme_text) - self.assertIn("PRISM_PUBLIC_AGGREGATE_CACHE_TTL_SECONDS", readme_text) - self.assertIn("PRISM_PUBLIC_AGGREGATE_CACHE_STALE_WHILE_REVALIDATE_SECONDS", readme_text) - self.assertIn("PRISM_PUBLIC_REWARD_WINDOW_CACHE_SECONDS", readme_text) - self.assertIn("PRISM_PUBLIC_CACHE_MAX_ENTRIES", readme_text) - self.assertIn("PRISM_PUBLIC_STRATUM_HIGHDIFF_URL", readme_text) - - def load_fixture(self, fixture_name: str) -> dict[str, Any]: - with (FIXTURE_DIR / fixture_name).open(encoding="utf-8") as fixture_file: - payload = json.load(fixture_file) - self.assertIsInstance(payload, dict) - return payload - - def assert_conventions(self, value: Any, source: str, path: tuple[str, ...] = ()) -> None: - if isinstance(value, dict): - for key, child in value.items(): - child_path = (*path, key) - if key == "schema": - self.assertIsInstance(child, str, ".".join(child_path)) - self.assertTrue(child.startswith("prism.dashboard."), ".".join(child_path)) - if key == "generated_at" or key.endswith("_at"): - if child is not None: - self.assert_iso_timestamp(child, source, child_path) - if key.endswith("_bits"): - if child is not None: - self.assertIsInstance(child, int, ".".join(child_path)) - self.assertGreaterEqual(child, 0, ".".join(child_path)) - if self.is_decimal_key(key, child_path) and not isinstance(child, (dict, list)): - if child is not None: - self.assertIsInstance(child, str, ".".join(child_path)) - self.assertRegex(child, DECIMAL_PATTERN, ".".join(child_path)) - if self.is_hex_hash_key(key) and not isinstance(child, (dict, list)): - if child is not None: - self.assertIsInstance(child, str, ".".join(child_path)) - self.assertRegex(child, HEX_HASH_PATTERN, ".".join(child_path)) - if key in HEX_STRING_KEYS and not isinstance(child, (dict, list)): - if child is not None: - self.assertIsInstance(child, str, ".".join(child_path)) - self.assertRegex(child, HEX_STRING_PATTERN, ".".join(child_path)) - self.assert_conventions(child, source, child_path) - elif isinstance(value, list): - for index, child in enumerate(value): - self.assert_conventions(child, source, (*path, str(index))) - - def is_decimal_key(self, key: str, path: tuple[str, ...] = ()) -> bool: - return ( - key in DECIMAL_KEYS - or key.startswith("hashrate_ths_") - or (len(path) >= 2 and path[-2] == "hashrate_ths" and key in HASHRATE_ROLLUP_KEYS) - or key.endswith("_ths") - or key.endswith("_percent") - or key.endswith("_difficulty") - or key.endswith("_weight") - ) - - def is_hex_hash_key(self, key: str) -> bool: - return key in HEX_HASH_KEYS or key.endswith("_sha256") or key.endswith("_txid") - - def epoch(self, timestamp: str) -> int: - self.assertTrue(timestamp.endswith("Z"), timestamp) - return int(datetime.fromisoformat(timestamp.removesuffix("Z") + "+00:00").timestamp()) - - def assert_iso_timestamp(self, value: object, source: str, path: tuple[str, ...]) -> None: - self.assertIsInstance(value, str, ".".join(path)) - text = str(value) - self.assertTrue(text.endswith("Z"), f"{source}:{'.'.join(path)} must be UTC/Z") - datetime.fromisoformat(text.removesuffix("Z") + "+00:00") - - -if __name__ == "__main__": - unittest.main() diff --git a/tests/test_qbit_ckpool_preflight.py b/tests/test_qbit_ckpool_preflight.py index b71c256f2..44aa37344 100644 --- a/tests/test_qbit_ckpool_preflight.py +++ b/tests/test_qbit_ckpool_preflight.py @@ -845,11 +845,14 @@ def call(self, method: str, params: list[Any] | None = None) -> Any: def test_supervisor_forwards_term_and_int_and_reaps_child(self) -> None: from tests.test_ckpool_startup import FakeRpcServer + # Publish readiness atomically so the parent cannot read an empty PID. child_code = ( "import os,signal,sys,time; ready,out=sys.argv[1:]; " + "from pathlib import Path; " "stop=lambda sig,frame:(open(out,'w').write(str(sig)),sys.exit(0)); " "signal.signal(signal.SIGTERM,stop); signal.signal(signal.SIGINT,stop); " - "open(ready,'w').write(str(os.getpid())); " + "pending=Path(ready+'.tmp'); " + "pending.write_text(str(os.getpid()),encoding='utf-8'); pending.replace(ready); " "exec('while True:\\n time.sleep(0.02)')" ) for signum in (signal.SIGTERM, signal.SIGINT): diff --git a/tests/test_qbit_image_runtime_smoke.py b/tests/test_qbit_image_runtime_smoke.py index 692bbb4e9..467fd6b9b 100644 --- a/tests/test_qbit_image_runtime_smoke.py +++ b/tests/test_qbit_image_runtime_smoke.py @@ -25,10 +25,6 @@ IMAGE = "qbit-runtime-smoke-fixture:test" GENESIS_HASH = "0f9188f13cb7b2c71f2a335e3a4fc328bf5beb436012afca590b1a11466e2206" -sys.path.insert(0, str(ROOT)) - -from lab.prism.block_candidates import _collapse_scaled_difficulty # noqa: E402 -from lab.prism.template_artifacts import scaled_network_difficulty # noqa: E402 # A stand-in for the docker CLI covering exactly the subcommands the smoke @@ -211,6 +207,13 @@ def main(argv): class QbitImageRuntimeSmokeTests(unittest.TestCase): + @classmethod + def setUpClass(cls) -> None: + subprocess.run( + ["cargo", "build", "--locked", "-p", "qbit-prism-server", "--bin", "qbit-prism-server"], + cwd=ROOT, check=True, capture_output=True, text=True, + ) + def setUp(self) -> None: self.temp_dir = tempfile.TemporaryDirectory() self.addCleanup(self.temp_dir.cleanup) @@ -278,8 +281,7 @@ def test_smoke_passes_and_reports_production_scaled_difficulty(self) -> None: self.assertEqual(result.returncode, 0, result.stdout + result.stderr) self.assertIn("qbit runtime image smoke: PASS", result.stdout) - expected = _collapse_scaled_difficulty("207fffff") - self.assertIsNotNone(expected) + expected = 1_000_000 self.assertIn( f"bits 207fffff scaled difficulty {expected}", result.stdout, @@ -313,9 +315,8 @@ def test_smoke_derives_mainnet_scale_difficulty_without_shell_overflow(self) -> result = self.run_smoke(bits="1b0404cb") self.assertEqual(result.returncode, 0, result.stdout + result.stderr) - expected = scaled_network_difficulty("1b0404cb") + expected = 35020450001492753689 self.assertGreater(expected, 2**63) - self.assertEqual(expected, _collapse_scaled_difficulty("1b0404cb")) self.assertIn(f"bits 1b0404cb scaled difficulty {expected}", result.stdout) def test_smoke_fails_when_header_bits_are_too_short(self) -> None: @@ -349,9 +350,8 @@ def test_smoke_fails_when_header_bits_carry_a_zero_target(self) -> None: # which is exactly the shape collapse fails its page closed on. result = self.run_smoke(bits="00000000") - self.assertIsNone(_collapse_scaled_difficulty("00000000")) self.assert_smoke_failed(result, "did not derive PRISM's scaled difficulty") - self.assertIn("collapse rejected the compact bits", result.stderr) + self.assertIn("negative or zero compact target", result.stderr) def test_smoke_fails_when_best_block_hash_is_not_a_block_hash(self) -> None: self.assert_smoke_failed( @@ -418,26 +418,15 @@ def test_regtest_container_is_asserted_against_the_header_contract(self) -> None self.script, ) - def test_scaled_difficulty_comes_from_the_production_helpers(self) -> None: - self.assertIn( - "from lab.prism.block_candidates import _collapse_scaled_difficulty", - self.script, - ) - self.assertIn( - "from lab.prism.template_artifacts import scaled_network_difficulty", - self.script, - ) - # The scale and the powLimit belong to the production helpers. A copy - # of either here could drift away from collapse with no test noticing. - for constant in ("1000000", "1_000_000", "207fffff"): - with self.subTest(constant=constant): - self.assertNotIn(constant, self.script) + def test_scaled_difficulty_comes_from_the_native_runtime(self) -> None: + self.assertIn('header-difficulty --bits "${bits}"', self.script) + self.assertNotIn("lab.prism", self.script) - def test_ci_runs_the_smoke_with_an_interpreter_for_the_helpers(self) -> None: + def test_ci_builds_the_native_header_probe(self) -> None: workflow = CI_WORKFLOW.read_text(encoding="utf-8") job = workflow.split("Build and smoke real qbitd image", 1)[1] self.assertIn("bash .github/scripts/test-qbit-image-runtime.sh", job) - self.assertIn("uses: actions/setup-python@v6", job) + self.assertIn("cargo build --locked -p qbit-prism-server --bin qbit-prism-server", job) if __name__ == "__main__": diff --git a/tests/test_run_python_test_shard.py b/tests/test_run_python_test_shard.py index 584f1154a..89e85fe89 100644 --- a/tests/test_run_python_test_shard.py +++ b/tests/test_run_python_test_shard.py @@ -3,7 +3,10 @@ import contextlib import io from pathlib import Path +import subprocess +import sys import tempfile +import textwrap import unittest from unittest import mock @@ -105,6 +108,113 @@ def runTest(self): 0 if passes else 1, ) + def run_parity_shard(self, sources, citations, *, index=0, count=1): + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + (root / "tests").mkdir() + for name, source in sources.items(): + (root / "tests" / name).write_text(textwrap.dedent(source), encoding="utf-8") + if citations is not None: + (root / "docs").mkdir() + (root / "docs/prism-deleted-test-map.md").write_text( + "\n".join(f"`tests/{name}::test_kept`" for name in citations), encoding="utf-8" + ) + return subprocess.run( + [sys.executable, "-c", + "from pathlib import Path; import sys; " + "from scripts import run_python_test_shard as runner; " + "runner.ROOT = Path(sys.argv[1]); raise SystemExit(runner.main(sys.argv[2:]))", + str(root), "--shard-index", str(index), "--shard-count", str(count)], + cwd=Path(__file__).resolve().parents[1], text=True, capture_output=True, check=False, + ) + + def test_cited_tests_must_pass_despite_unittest_treating_skips_as_success(self): + base = "import unittest\nclass Kept(unittest.TestCase):\n def test_kept(self):\n pass\n" + sources = { + "module fixture": base + '\ndef setUpModule():\n raise unittest.SkipTest("unavailable")\n', + "class fixture": base + '\n @classmethod\n def setUpClass(cls):\n raise unittest.SkipTest("unavailable")\n', + "case fixture": base + '\n def setUp(self):\n self.skipTest("unavailable")\n', + "async fixture": base.replace("unittest.TestCase", "unittest.IsolatedAsyncioTestCase") + + '\n async def asyncSetUp(self):\n self.skipTest("unavailable")\n', + "test body": base.replace(" pass", ' self.skipTest("unavailable")'), + "subtest": base.replace(" pass", ' with self.subTest():\n self.skipTest("unavailable")'), + "expected failure": base.replace(" def test_kept", " @unittest.expectedFailure\n def test_kept") + .replace(" pass", " self.fail('not implemented')"), + } + for label, source in sources.items(): + with self.subTest(label=label): + result = self.run_parity_shard({"test_parity_a.py": source}, ["test_parity_a.py"]) + self.assertEqual(result.returncode, 1, result.stderr) + self.assertIn("tests/test_parity_a.py::test_kept", result.stderr) + self.assertIn("no successful execution", result.stderr) + + def test_cited_success_runs_fixtures_and_body_once_and_allows_unrelated_skips(self): + result = self.run_parity_shard({"test_parity_a.py": ''' + import unittest + def setUpModule(): + print("MODULE READY") + def tearDownModule(): + print("MODULE CLEANED") + class Kept(unittest.TestCase): + @classmethod + def setUpClass(cls): + print("CLASS READY") + @classmethod + def tearDownClass(cls): + print("CLASS CLEANED") + def setUp(self): + print("CASE READY") + self.addCleanup(print, "CASE CLEANED") + def test_kept(self): + print("BODY RAN") + @unittest.skip("optional uncited test") + def test_optional(self): + self.fail("should not run") + '''}, ["test_parity_a.py"]) + self.assertEqual(result.returncode, 0, result.stderr) + for marker in ("MODULE READY", "MODULE CLEANED", "CLASS READY", "CLASS CLEANED", + "CASE READY", "CASE CLEANED", "BODY RAN"): + self.assertEqual(result.stdout.count(marker), 1, result.stdout) + + def test_live_inherited_implementation_satisfies_citation_despite_skipped_subclass(self): + result = self.run_parity_shard({"test_parity_a.py": ''' + import unittest + class Kept(unittest.TestCase): + def test_kept(self): + pass + class Skipped(Kept): + @classmethod + def setUpClass(cls): + raise unittest.SkipTest("optional subclass") + '''}, ["test_parity_a.py"]) + self.assertEqual(result.returncode, 0, result.stderr) + + def test_parity_check_requires_only_citations_assigned_to_this_shard(self): + source = "import unittest\nclass Kept(unittest.TestCase):\n def test_kept(self):\n pass\n" + sources = { + "test_parity_a.py": source + '\n def setUp(self):\n self.skipTest("unavailable")\n', + "test_parity_b.py": source, + } + for index, expected in ((0, 1), (1, 0)): + with self.subTest(index=index): + result = self.run_parity_shard(sources, list(sources), index=index, count=2) + self.assertEqual(result.returncode, expected, result.stderr) + self.assertNotIn("tests/test_parity_b.py::test_kept", result.stderr) + result = self.run_parity_shard(sources, list(sources)) + self.assertEqual(result.returncode, 1, result.stderr) + self.assertIn("tests/test_parity_a.py::test_kept", result.stderr) + self.assertNotIn("tests/test_parity_b.py::test_kept", result.stderr) + + def test_missing_parity_map_fails_closed(self): + result = self.run_parity_shard({"test_parity_a.py": ''' + import unittest + class Kept(unittest.TestCase): + def test_kept(self): + pass + '''}, None) + self.assertNotEqual(result.returncode, 0, result.stderr) + self.assertIn("prism-deleted-test-map.md", result.stderr) + if __name__ == "__main__": unittest.main() diff --git a/tests/test_run_rust_test_shard.py b/tests/test_run_rust_test_shard.py new file mode 100644 index 000000000..145b95d70 --- /dev/null +++ b/tests/test_run_rust_test_shard.py @@ -0,0 +1,173 @@ +from __future__ import annotations + +import contextlib +import io +import json +from pathlib import Path +import re +import shlex +import subprocess +import tempfile +import unittest +from unittest import mock + +from scripts.run_rust_test_shard import ( + COMPACT_SCALE, DURATION_SECONDS, IGNORED, main, shard_commands, typical_seconds, workspace_targets, +) + + +NATIVE_SCRIPT = Path(__file__).resolve().parents[1] / "test" / "prism-native-tests.sh" + + +def fixture_metadata(): + packages = [ + {"id": "server", "name": "qbit-prism-server", "targets": [ + {"kind": [kind], "name": name} + for kind, name in [ + ("lib", "qbit_prism_server"), ("bin", "qbit-prism-server"), + ("test", "stratum_admission_postgres"), + ("test", "observability_database"), ("test", "issued_job_dependency"), + ("test", "new_contract"), + ("test", "compact_runtime_scale"), + ("custom-build", "build-script-build"), + ] + ]}, + {"id": "core", "name": "core", "targets": [ + {"kind": [kind], "name": kind} for kind in ("lib", "bin", "test", "bench", "example") + ]}, + {"id": "dependency", "name": "dependency", "targets": [ + {"kind": ["lib"], "name": "dependency"} + ]}, + ] + return {"packages": packages, "workspace_members": ["server", "core"]} + + +class RustTestShardTests(unittest.TestCase): + def test_disjoint_shards_cover_all_targets_and_keep_ignored_runs_with_owner(self): + metadata = fixture_metadata() + expected = shard_commands(metadata, 0, 1) + shards = [shard_commands(metadata, i, 4) for i in range(4)] + self.assertCountEqual([cmd for shard in shards for cmd in shard], expected) + ordinary = [cmd for cmd in expected if "--ignored" not in cmd] + self.assertEqual(len(ordinary), 12) + self.assertEqual(len({tuple(cmd) for cmd in ordinary}), 12) + self.assertEqual(len(expected) - len(ordinary), len(IGNORED)) + for shard in shards: + for command in shard: + if "--ignored" in command: + # The optimized scale invocation belongs to the same target + # as its ordinary debug run; profile is not target identity. + prefix = [arg for arg in command[:command.index("--")] if arg != "--release"] + self.assertIn(prefix + ["--", "--nocapture"], shard) + scale = [cmd for cmd in expected if COMPACT_SCALE[2] in cmd] + self.assertEqual(len(scale), 2) + self.assertNotIn("--release", scale[0]) + self.assertNotIn("--ignored", scale[0]) + self.assertEqual(scale[1], scale[0][:-2] + [ + "--release", "--", "--ignored", "--nocapture", "--test-threads=1", + ]) + admission = next(cmd for cmd in expected if "--exact" in cmd) + self.assertEqual(admission[-1], next(iter(IGNORED.values()))[-1]) + metadata["packages"].reverse() + for package in metadata["packages"]: + package["targets"].reverse() + self.assertEqual(shards, [shard_commands(metadata, i, 4) for i in range(4)]) + + def test_heavy_targets_spread_across_shards_and_loads_stay_balanced(self): + # Round-robin over the sorted list put the server lib, + # compact_runtime_scale and live_regtest on one shard (#647). + metadata = fixture_metadata() + server = metadata["packages"][0]["targets"] + for package, kind, name in DURATION_SECONDS: + if {"kind": [kind], "name": name} not in server: + server.append({"kind": [kind], "name": name}) + for index in range(40): + server.append({"kind": ["test"], "name": f"light_{index:02}"}) + targets = workspace_targets(metadata) + count = 4 + owners = {} + for index in range(count): + for command in shard_commands(metadata, index, count): + if "--ignored" in command: + continue + package, kind = command[4], command[5][2:] + name = command[6] if kind != "lib" else next( + target[2] for target in targets if target[:2] == (package, "lib") + ) + owners[package, kind, name] = index + self.assertCountEqual(owners, targets) + heaviest = sorted(DURATION_SECONDS, key=typical_seconds, reverse=True)[:count] + self.assertEqual(len({owners[target] for target in heaviest}), count) + loads = [sum(typical_seconds(target) for target in targets if owners[target] == index) for index in range(count)] + self.assertLessEqual(max(loads), max(DURATION_SECONDS.values()) + max(typical_seconds(target) for target in targets if target not in DURATION_SECONDS)) + + def test_local_database_mode_selects_the_same_ignored_contracts(self): + script = NATIVE_SCRIPT.read_text(encoding="utf-8").replace("\\\n", " ") + database = script.split("\n database)\n", 1)[1].split("\n ;;", 1)[0] + local = [shlex.split(line)[1:] for line in database.splitlines() if "--ignored" in line] + ci = [cmd for cmd in shard_commands(fixture_metadata(), 0, 1) if "--ignored" in cmd] + self.assertEqual(len(ci), len(IGNORED)) + self.assertCountEqual(local, ci) + + def test_bad_coordinates_empty_and_unknown_targets_fail_closed(self): + for index, count in [(-1, 4), (4, 4), (0, 0), (0, -1), (12, 13)]: + with self.subTest(index=index, count=count), self.assertRaises(ValueError): + shard_commands(fixture_metadata(), index, count) + with self.assertRaises(ValueError): + shard_commands({"packages": [], "workspace_members": []}, 0, 1) + metadata = fixture_metadata() + metadata["packages"][0]["targets"][0]["kind"] = ["unknown"] + with self.assertRaises(ValueError): + workspace_targets(metadata) + + def test_dry_run_and_execution_use_identical_commands(self): + metadata = fixture_metadata() + for dry_run in [True, False]: + with ( + mock.patch("scripts.run_rust_test_shard.subprocess.check_output", return_value=json.dumps(metadata)), + mock.patch("scripts.run_rust_test_shard.subprocess.run") as run, + contextlib.redirect_stdout(io.StringIO()), + ): + args = ["--shard-index", "2", "--shard-count", "4"] + self.assertEqual(main(args + (["--dry-run"] if dry_run else [])), 0) + self.assertEqual( + [call.args[0] for call in run.call_args_list], + [] if dry_run else shard_commands(metadata, 2, 4), + ) + for call in run.call_args_list: + self.assertTrue(call.kwargs["check"]) + + def test_job_summary_records_each_command_duration_even_on_failure(self): + metadata = fixture_metadata() + commands = shard_commands(metadata, 2, 4) + with tempfile.TemporaryDirectory() as directory: + summary = Path(directory) / "summary.md" + with ( + mock.patch.dict("os.environ", {"GITHUB_STEP_SUMMARY": str(summary)}), + mock.patch("scripts.run_rust_test_shard.subprocess.check_output", return_value=json.dumps(metadata)), + mock.patch("scripts.run_rust_test_shard.subprocess.run") as run, + contextlib.redirect_stdout(io.StringIO()), + ): + run.side_effect = [None, subprocess.CalledProcessError(101, ["cargo"])] + self.assertEqual(main(["--shard-index", "2", "--shard-count", "4"]), 101) + self.assertEqual(main(["--shard-index", "2", "--shard-count", "4", "--dry-run"]), 0) + lines = summary.read_text(encoding="utf-8").splitlines() + self.assertEqual(lines[:2], ["| Command | Seconds |", "| --- | ---: |"]) + self.assertEqual(len(lines), 4) + for line, command in zip(lines[2:], commands): + self.assertRegex(line, rf"^\| `{re.escape(shlex.join(command))}` \| \d+ \|$") + + def test_cargo_failure_stops_shard_and_preserves_nonzero_status(self): + for phase in ["metadata", "tests"]: + with ( + mock.patch("scripts.run_rust_test_shard.subprocess.check_output", return_value=json.dumps(fixture_metadata())) as metadata, + mock.patch("scripts.run_rust_test_shard.subprocess.run") as run, + contextlib.redirect_stdout(io.StringIO()), + ): + (metadata if phase == "metadata" else run).side_effect = subprocess.CalledProcessError(101, ["cargo"]) + self.assertEqual(main(["--shard-index", "0", "--shard-count", "1"]), 101) + self.assertEqual(run.call_count, 0 if phase == "metadata" else 1) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_window_paging_allocation_harness.py b/tests/test_window_paging_allocation_harness.py deleted file mode 100644 index 15331016c..000000000 --- a/tests/test_window_paging_allocation_harness.py +++ /dev/null @@ -1,337 +0,0 @@ -#!/usr/bin/env python3 -"""Bounded safety tests for ``tests/perf/window_paging_allocation.py``. - -The measurement driver talks to a disposable ``--serve`` daemon over blocking -pipes. A stalled daemon must never hang a benchmark: every exchange, the -handshake included, runs under a wall-clock deadline that kills and reaps the -child and records the failure, and every exit path closes the pipes and the -stderr capture. These tests exercise that with a fake daemon that stalls in -three ways -- no handshake, after consuming the request, and without ever -reading it (so the request writer is blocked on a full pipe) -- and stay -well under a few seconds each. They also pin the provenance of the reported -peak resident set: every observation made before process exit, including -kernel ``VmHWM`` before a kill or normal close, is a lower bound rendered -with ``≥``; nothing observed is ``unavailable``. -""" - -from __future__ import annotations - -import os -import io -import json -import stat -import tempfile -import time -import unittest -from pathlib import Path -from contextlib import redirect_stdout -from unittest import mock - -from tests.perf import window_paging_allocation as harness - - -FAKE_DAEMON = """#!/usr/bin/env python3 -import json, os, sys, time -mode = os.environ.get("FAKE_DAEMON_MODE", "stall_after_handshake") -if mode != "no_handshake": - sys.stdout.write(json.dumps({ - "event": "handshake", - "tool": "qbit-prism-build-audit-bundle", - "protocol": %d, - }) + "\\n") - sys.stdout.flush() -if mode == "exit_after_handshake": - sys.exit(0) -if mode == "stall_after_handshake": - sys.stdin.readline() -time.sleep(120) -""" % harness.PRISM_SERVE_BUILDER_PROTOCOL_VERSION - - -def _assert_reaped(test: unittest.TestCase, pid: int) -> None: - try: - os.kill(pid, 0) - except ProcessLookupError: - return - except PermissionError: # pragma: no cover - pid reused by another user - return - test.fail(f"fake daemon pid {pid} is still present after close()") - - -class StalledDaemonTests(unittest.TestCase): - @classmethod - def setUpClass(cls) -> None: - cls.tmp = tempfile.TemporaryDirectory() - cls.binary = Path(cls.tmp.name) / "fake-daemon" - cls.binary.write_text(FAKE_DAEMON) - cls.binary.chmod(cls.binary.stat().st_mode | stat.S_IXUSR) - - @classmethod - def tearDownClass(cls) -> None: - cls.tmp.cleanup() - - def _daemon(self, mode: str, timeout: float = 0.5) -> harness.Daemon: - os.environ["FAKE_DAEMON_MODE"] = mode - try: - return harness.Daemon( - self.binary, - memory_limit_mb=None, - stderr_path=Path(self.tmp.name) / f"{mode}.stderr", - exchange_timeout=timeout, - shutdown_timeout=2.0, - ) - finally: - os.environ.pop("FAKE_DAEMON_MODE", None) - - def test_exchange_that_stalls_after_reading_times_out_and_is_reaped(self) -> None: - daemon = self._daemon("stall_after_handshake") - started = time.perf_counter() - exchange = daemon.exchange(b'{"request":"prepare_window"}\n', label="full") - elapsed = time.perf_counter() - started - self.assertLess(elapsed, 5.0) - self.assertIsNone(exchange.envelope) - self.assertIn("full exceeded 0.5s; daemon killed", exchange.error or "") - outcome = daemon.close() - self.assertEqual(outcome["timed_out"], "full") - self.assertEqual(outcome["signal"], 9) - # The watchdog sampled the kernel high-water mark before killing. - if Path("/proc/self/status").exists(): - self.assertEqual(outcome["peak_rss_source"], "vmhwm_before_kill") - self.assertIsInstance(outcome["peak_rss_mb"], float) - else: - self.assertEqual(outcome["peak_rss_source"], "unavailable") - self.assertIsNone(outcome["peak_rss_mb"]) - self.assertTrue(daemon.process.stdin.closed) - self.assertTrue(daemon.process.stdout.closed) - self.assertTrue(daemon._stderr.closed) - _assert_reaped(self, outcome["pid"]) - # Idempotent, and a later exchange never touches the dead child. - self.assertIs(daemon.close(), outcome) - self.assertEqual(daemon.exchange(b"{}\n").error, "daemon is not running") - - def test_stall_without_reading_unblocks_the_request_writer(self) -> None: - daemon = self._daemon("stall_without_reading") - request = b'{"records":"' + b"x" * (4 << 20) + b'"}\n' # far beyond one pipe capacity - started = time.perf_counter() - exchange = daemon.exchange(request, label="full") - elapsed = time.perf_counter() - started - self.assertLess(elapsed, 5.0) - self.assertIn("daemon killed", exchange.error or "") - self.assertIn("write:", exchange.error or "") - outcome = daemon.close() - self.assertEqual(outcome["signal"], 9) - _assert_reaped(self, outcome["pid"]) - - def test_missing_handshake_times_out_and_is_reaped(self) -> None: - started = time.perf_counter() - with self.assertRaises(harness.DaemonTimeout) as caught: - self._daemon("no_handshake") - self.assertLess(time.perf_counter() - started, 5.0) - outcome = caught.exception.daemon_outcome # type: ignore[attr-defined] - self.assertEqual(outcome["timed_out"], "handshake") - self.assertEqual(outcome["signal"], 9) - _assert_reaped(self, outcome["pid"]) - - def test_run_variant_records_the_timeout_and_reaps(self) -> None: - fixture = harness.build_fixture(64, miners=2, page_size=16, small=2, large=3) - os.environ["FAKE_DAEMON_MODE"] = "stall_after_handshake" - try: - started = time.perf_counter() - result = harness.run_variant( - "fake", - self.binary, - fixture, - oracle=None, - memory_limit_mb=None, - memory_margin_mb=0, - sample_interval=0.05, - exchange_timeout=0.5, - stderr_dir=Path(self.tmp.name), - log=lambda _text: None, - ) - finally: - os.environ.pop("FAKE_DAEMON_MODE", None) - self.assertLess(time.perf_counter() - started, 10.0) - self.assertIsNone(result.skipped) - self.assertEqual([phase.phase for phase in result.phases], ["full"]) - self.assertEqual(result.phases[0].status, "timeout") - self.assertIsNone(result.phases[0].residual_wait_seconds) - self.assertEqual(result.outcome["timed_out"], "full") - self.assertEqual(result.outcome["signal"], 9) - _assert_reaped(self, result.outcome["pid"]) - # The tables render a killed run rather than choking on it. - self.assertIn("killed on full timeout", harness.render_summary([result])) - self.assertIn("| timeout", harness.render_runs([result])) - - def test_main_skips_before_parent_allocations_when_memory_is_low(self) -> None: - fixture = harness.build_fixture(64, miners=2, page_size=16, small=2, large=3) - for available, stage, builds in ( - ([0], "fixture construction", 0), - ([10000, 0], "Python oracle", 1), - ): - with self.subTest(stage=stage), \ - mock.patch.object(harness, "read_meminfo_mb", side_effect=available), \ - mock.patch.object(harness, "machine_info", return_value={}), \ - mock.patch.object(harness, "git_describe", return_value="test"), \ - mock.patch.object(harness, "build_fixture", return_value=fixture) as build, \ - mock.patch.object(harness, "python_oracle") as oracle, \ - mock.patch.object(harness, "run_variant") as variant, \ - redirect_stdout(io.StringIO()): - output = Path(self.tmp.name) / "low-memory.json" - self.assertEqual(harness.main([ - "--daemon-binary", str(self.binary), "--sizes", "400000", - "--stderr-dir", self.tmp.name, "--json", str(output), "--quiet", - ]), 0) - result = json.loads(output.read_text()) - self.assertEqual(build.call_count, builds) - oracle.assert_not_called() - variant.assert_not_called() - self.assertEqual(result["oracles"], {}) - self.assertEqual(len(result["runs"]), 1) - self.assertEqual(result["runs"][0]["size"], 400000) - self.assertIn(f"before {stage}", result["runs"][0]["skipped"]) - - def test_handshake_timeout_preserves_outcome_in_result_and_summary(self) -> None: - fixture = harness.build_fixture(64, miners=2, page_size=16, small=2, large=3) - os.environ["FAKE_DAEMON_MODE"] = "no_handshake" - try: - result = harness.run_variant( - "fake", self.binary, fixture, oracle=None, - memory_limit_mb=None, memory_margin_mb=0, - sample_interval=0.01, exchange_timeout=0.5, - stderr_dir=Path(self.tmp.name), log=lambda _text: None, - ) - finally: - os.environ.pop("FAKE_DAEMON_MODE", None) - self.assertEqual(result.phases, []) - self.assertIn("daemon failed to start", result.error or "") - self.assertEqual(result.outcome["timed_out"], "handshake") - self.assertEqual(result.outcome["signal"], 9) - self.assertIn("stderr_tail", result.outcome) - self.assertIn("peak_rss_source", result.outcome) - _assert_reaped(self, result.outcome["pid"]) - summary = harness.render_summary([result]) - self.assertIn("signal 9, killed on handshake timeout", summary) - self.assertIn("failed: daemon failed to start", summary) - self.assertIn(harness.peak_source_label(result.outcome["peak_rss_source"]), summary) - - def test_close_on_a_live_daemon_reports_the_kernel_high_water_mark(self) -> None: - daemon = self._daemon("stall_after_handshake") - outcome = daemon.close() # alive at close: VmHWM read live, then killed - if Path("/proc/self/status").exists(): - self.assertEqual(outcome["peak_rss_source"], "vmhwm") - self.assertIsInstance(outcome["peak_rss_mb"], float) - else: - self.assertEqual(outcome["peak_rss_source"], "unavailable") - self.assertIsNone(outcome["peak_rss_mb"]) - self.assertEqual(outcome["signal"], 9) - _assert_reaped(self, outcome["pid"]) - - def test_self_exited_daemon_peak_is_a_lower_bound_or_unavailable(self) -> None: - daemon = self._daemon("exit_after_handshake") - daemon.process.wait(timeout=5.0) - outcome = daemon.close() - self.assertEqual(outcome["exit_code"], 0) - self.assertEqual(outcome["peak_rss_source"], "unavailable") - self.assertIsNone(outcome["peak_rss_mb"]) - _assert_reaped(self, outcome["pid"]) - - fixture = harness.build_fixture(64, miners=2, page_size=16, small=2, large=3) - os.environ["FAKE_DAEMON_MODE"] = "exit_after_handshake" - try: - result = harness.run_variant( - "fake", - self.binary, - fixture, - oracle=None, - memory_limit_mb=None, - memory_margin_mb=0, - sample_interval=0.01, - exchange_timeout=5.0, - stderr_dir=Path(self.tmp.name), - log=lambda _text: None, - ) - finally: - os.environ.pop("FAKE_DAEMON_MODE", None) - self.assertEqual(result.outcome["exit_code"], 0) - self.assertEqual(result.phases[0].status, "no_response") - # Whatever the sampler caught before the exit, the figure is never - # presented as a kernel high-water mark. - source = result.outcome["peak_rss_source"] - self.assertIn(source, ("lower_bound", "unavailable")) - summary = harness.render_summary([result]) - if source == "lower_bound": - self.assertIsInstance(result.outcome["peak_rss_mb"], float) - self.assertIn("| ≥ ", summary) - self.assertIn("lower bound", summary) - else: - self.assertIsNone(result.outcome["peak_rss_mb"]) - self.assertIn("| - | unavailable |", summary) - self.assertNotIn("kernel VmHWM", summary) - - def test_resolve_peak_rss_provenance(self) -> None: - exact = harness.resolve_peak_rss({"peak_rss_mb": 601.0, "peak_rss_source": "vmhwm"}, [900.0]) - self.assertEqual((exact["peak_rss_mb"], exact["peak_rss_source"]), (601.0, "vmhwm")) - before_kill = harness.resolve_peak_rss( - {"peak_rss_mb": 601.0, "peak_rss_source": "vmhwm_before_kill"}, [900.0] - ) - self.assertEqual(before_kill["peak_rss_source"], "vmhwm_before_kill") - legacy = harness.resolve_peak_rss({"peak_rss_mb": 601.0}, []) - self.assertEqual(legacy["peak_rss_source"], "vmhwm") - bound = harness.resolve_peak_rss({"peak_rss_mb": None}, [435.0, None, 0.0, 601.0, 12.5]) - self.assertEqual((bound["peak_rss_mb"], bound["peak_rss_source"]), (601.0, "lower_bound")) - nothing = harness.resolve_peak_rss({"peak_rss_mb": None}, [None, 0.0]) - self.assertEqual((nothing["peak_rss_mb"], nothing["peak_rss_source"]), (None, "unavailable")) - - def test_peak_rendering_marks_all_live_observations_as_lower_bounds(self) -> None: - self.assertEqual(harness.format_peak(601.4, "vmhwm"), "≥ 601") - self.assertEqual(harness.format_peak(601.4, "vmhwm_before_kill"), "≥ 601") - self.assertEqual(harness.format_peak(601.4, "lower_bound"), "≥ 601") - self.assertEqual(harness.format_peak(2.857, "lower_bound", 2), "≥ 2.86") - self.assertEqual(harness.format_peak(None, "unavailable"), "-") - - def run(source: str | None, peak: float | None) -> harness.RunResult: - return harness.RunResult( - "fixed", "bin", 210_000, 512, 16_000.0, None, 6144, - [ - harness.PhaseResult( - phase="full", records_sent=210_000, request_bytes=1, - request_write_seconds=0.15, response_wait_seconds=1.29, - response_read_seconds=0.12, daemon_metrics={"fold_seconds": 0.377}, - status="prepared", - ) - ], - {"exit_code": 0, "signal": None, "timed_out": None, "peak_rss_mb": peak, - "peak_rss_source": source, "peak_vsize_mb": 640.0}, - 0.0, [], - ) - - exact_row = harness.render_summary([run("vmhwm", 601.0)]).splitlines()[-1] - self.assertIn("| ≥ 601 | lower bound (kernel VmHWM at close) | ≥ 640 | ≥ 2.86 |", exact_row) - kill_row = harness.render_summary([run("vmhwm_before_kill", 601.0)]).splitlines()[-1] - self.assertIn("| ≥ 601 | lower bound (kernel VmHWM before watchdog kill) | ≥ 640 | ≥ 2.86 |", kill_row) - bound_row = harness.render_summary([run("lower_bound", 601.0)]).splitlines()[-1] - self.assertIn("| ≥ 601 | lower bound (highest earlier VmHWM/RSS observation) | ≥ 640 | ≥ 2.86 |", bound_row) - none_row = harness.render_summary([run("unavailable", None)]).splitlines()[-1] - self.assertIn("| - | unavailable | ≥ 640 | - |", none_row) - header = harness.render_summary([]).splitlines()[0] - self.assertIn("| peak RSS MiB | peak source |", header) - self.assertNotIn("VmHWM", header) - - def test_residual_wait_is_an_approximation_clamped_at_zero(self) -> None: - phase = harness.PhaseResult( - phase="full", - records_sent=1, - request_bytes=1, - request_write_seconds=0.0, - response_wait_seconds=1.0, - response_read_seconds=0.0, - daemon_metrics={"input_deserialization_seconds": 0.4, "fold_seconds": 0.7, "output_serialization_seconds": 0.2}, - ) - self.assertEqual(phase.residual_wait_seconds, 0.0) - phase.daemon_metrics = {} - self.assertIsNone(phase.residual_wait_seconds) - - -if __name__ == "__main__": - unittest.main() diff --git a/tests/test_window_pipeline_parity.py b/tests/test_window_pipeline_parity.py deleted file mode 100644 index 5ef8c3ac3..000000000 --- a/tests/test_window_pipeline_parity.py +++ /dev/null @@ -1,1155 +0,0 @@ -#!/usr/bin/env python3 -"""Hold the window pipeline to its frozen byte-exact reference. - -Why these tests exist: a second implementation of the payout-window pipeline -can only be adopted if it is byte-identical to the shipped Python one, and -that comparison is only trustworthy if (a) the shipped pipeline reproduces a -frozen reference committed at a known revision, (b) the corpus demonstrably -exercises the fold's edge cases rather than just its bulk behavior, and -(c) the harness has been *seen to fail* on known divergence classes. The -first two are the parity tests below; the third is the divergence-detection -suite, which drives deliberately perturbed backends -- wrong field order, -wrong digest framing, null-instead-of-absent, a dropped crossing row, spool -whitespace, narrow or saturating integers, a fold that never refuses or -misfiles its refusals, advance expiry with the wrong comparison or confined -to pre-existing pages, an equal-anchor rejection, a raw DEL or an escaped -solidus, advance stats derived from the wrong thing, and a backend that -declares Unsupported inside its own domain or returns wrong bytes outside it --- and asserts the harness catches each one exactly where it should and -nowhere else. Green alone is not evidence here: every class from the -integer-width one onward was green against the original fourteen-case -corpus, which could not see it, and the width class is re-checked against -the retuned ``wide-integers`` case so the declared-domain split could not -have quietly taken that protection away. -""" - -from __future__ import annotations - -import json -import os -import unittest -from dataclasses import replace -from unittest import mock - -from lab.prism.bundle_compiler import _compact_share_payload -from lab.prism.share_ledger import ( - DEFAULT_INCREMENTAL_SHARE_WINDOW_PAGE_SIZE, - IncrementalShareWindow, - IncrementalWindowFallback, -) -from tests import window_pipeline_parity as parity - - -class WindowPipelineAdapterRegistryTests(unittest.TestCase): - def test_selection_defaults_to_the_shipped_python_pipeline(self) -> None: - with mock.patch.dict(os.environ): - os.environ.pop(parity.ADAPTER_ENV_VAR, None) - adapter = parity.resolve_adapter() - self.assertEqual(adapter.name, "python") - self.assertIsInstance(adapter, parity.PythonWindowPipelineAdapter) - - def test_environment_variable_selects_a_registered_backend(self) -> None: - class _RegisteredBackend: - name = "registry-selection-probe" - - def run(self, case: parity.WindowPipelineCase) -> parity.WindowPipelineResult: - raise NotImplementedError - - parity.register_adapter("registry-selection-probe", _RegisteredBackend) - try: - with mock.patch.dict( - os.environ, - {parity.ADAPTER_ENV_VAR: "registry-selection-probe"}, - ): - self.assertIsInstance(parity.resolve_adapter(), _RegisteredBackend) - finally: - parity._ADAPTER_FACTORIES.pop("registry-selection-probe", None) - - def test_duplicate_registration_is_rejected(self) -> None: - with self.assertRaisesRegex(ValueError, "already registered"): - parity.register_adapter("python", parity.PythonWindowPipelineAdapter) - - def test_unknown_backend_selection_is_a_clear_error(self) -> None: - with self.assertRaisesRegex( - ValueError, "unknown window-pipeline parity adapter" - ): - parity.resolve_adapter("no-such-backend") - - def test_rejection_vocabulary_is_closed(self) -> None: - # A backend cannot invent a category, and the vocabulary is exactly - # the two entry points' condition lists: a rejection the shipped - # pipeline cannot express cannot be frozen as parity either. - with self.assertRaisesRegex(ValueError, "unknown window-pipeline rejection reason"): - parity.WindowPipelineRejection("ValueError") - self.assertEqual( - parity.REJECTION_REASONS, - parity.FULL_SNAPSHOT_REJECTIONS + parity.ADVANCE_REJECTIONS, - ) - self.assertEqual(len(set(parity.REJECTION_REASONS)), 11) - - -class WindowPipelineParityOracleTests(unittest.TestCase): - """The selected backend must reproduce every frozen corpus case.""" - - corpus: tuple[parity.WindowPipelineCase, ...] - cases: dict[str, parity.WindowPipelineCase] - reference_cases: dict[str, dict[str, object]] - results: dict[str, parity.WindowPipelineResult] - - @classmethod - def setUpClass(cls) -> None: - cls.corpus = parity.build_corpus() - cls.cases = {case.name: case for case in cls.corpus} - cls.reference_cases = parity.load_reference_document()["cases"] - cls.adapter = parity.resolve_adapter() - cls.results = {case.name: cls.adapter.run(case) for case in cls.corpus} - - def _outputs(self, name: str) -> parity.WindowPipelineOutputs: - result = self.results[name] - self.assertIsInstance(result, parity.WindowPipelineOutputs, name) - assert isinstance(result, parity.WindowPipelineOutputs) - return result - - def _declared_unsupported(self, name: str) -> bool: - """Whether the backend legitimately declined this case. - - True only for an Unsupported result on a case outside the backend's - declared domain; the main parity test fails any other Unsupported, so - the output-shape tests may skip exactly these cases and no others. - """ - result = self.results[name] - if not isinstance(result, parity.WindowPipelineUnsupported): - return False - self.assertIsNotNone( - parity.domain_exclusion(self.adapter, self.cases[name]), - f"{name}: Unsupported inside the declared domain", - ) - return True - - def test_backend_reproduces_every_frozen_case(self) -> None: - self.assertEqual(set(self.cases), set(self.reference_cases)) - for name, case in self.cases.items(): - with self.subTest(case=name): - entry = self.reference_cases[name] - self.assertEqual( - parity.case_input_sha256(case), - entry["input_sha256"], - "corpus inputs drifted from the ones the reference froze", - ) - pinned_input = (entry.get("pinned_literals") or {}).get("input") - if pinned_input is not None: - # The literal makes input drift legible in the fixture - # diff; here it also localizes which field moved. - self.assertEqual(parity.case_input_document(case), pinned_input) - self.assertEqual( - parity.diverging_outputs( - entry, - self.results[name], - domain_exclusion=parity.domain_exclusion(self.adapter, case), - ), - [], - ) - - def test_unsupported_is_declared_only_outside_the_declared_domain(self) -> None: - # Every Unsupported the selected backend returned names a case its - # declared domain excludes, the reference still holds Python's bytes - # for it, and the shipped pipeline itself declines nothing. - for name in self.cases: - result = self.results[name] - if not isinstance(result, parity.WindowPipelineUnsupported): - continue - with self.subTest(case=name): - self.assertTrue(result.reason) - self.assertIsNotNone(parity.domain_exclusion(self.adapter, self.cases[name])) - self.assertIn("record_count", self.reference_cases[name]) - if isinstance(self.adapter, parity.PythonWindowPipelineAdapter): - self.assertFalse( - any(isinstance(result, parity.WindowPipelineUnsupported) for result in self.results.values()) - ) - - def test_outputs_are_internally_consistent(self) -> None: - # The digest must hash exactly the framed canonical bytes, and the - # page-fragment concatenation must equal the flat per-record encoding - # -- the identity that makes page layout invisible on the wire. - for name, case in self.cases.items(): - if case.expected_rejection is not None or self._declared_unsupported(name): - continue - with self.subTest(case=name): - outputs = self._outputs(name) - self.assertEqual( - parity.sha256_hex(outputs.canonical_bytes), - outputs.canonical_digest, - ) - self.assertEqual( - b"[" + b",".join(outputs.record_jsons) + b"]", - outputs.canonical_bytes, - ) - self.assertEqual(len(outputs.advance_stats), len(case.advances)) - - def test_incremental_advances_land_on_full_rebuild_bytes(self) -> None: - # advance() folding deltas must produce the same byte outputs as - # from_full_snapshot over the union at the final anchor. The rebuild - # side always runs the shipped Python pipeline, so a selected - # non-default backend is compared against it here as well. Only - # accepted advances have a rebuild to land on: a rejected advance's - # rebuild is the coordinator's recovery, which is deliberately not - # pinned (see the module docstring). - python_adapter = parity.PythonWindowPipelineAdapter() - advancing = [ - case for case in self.corpus if case.advances and case.expected_rejection is None - ] - self.assertGreaterEqual(len(advancing), 8) - for case in advancing: - if self._declared_unsupported(case.name): - continue - with self.subTest(case=case.name): - rebuilt = python_adapter.run(parity.union_rebuild_case(case)) - self.assertEqual( - replace(self._outputs(case.name), advance_stats=()), - rebuilt, - ) - - def test_rejections_are_pinned_for_every_condition(self) -> None: - # Every category in the vocabulary is frozen by at least one case, - # each rejection case froze exactly the category it was built for - # (so it cannot drift into passing for a different reason), and no - # accepted case froze a rejection. - pinned: dict[str, list[str]] = {} - for name, case in self.cases.items(): - entry = self.reference_cases[name] - with self.subTest(case=name): - if case.expected_rejection is None: - self.assertNotIn("rejected", entry) - continue - self.assertEqual(entry.get("rejected"), case.expected_rejection) - self.assertNotIn("record_count", entry) - self.assertEqual(self.results[name], parity.WindowPipelineRejection(case.expected_rejection)) - pinned.setdefault(case.expected_rejection, []).append(name) - self.assertEqual(set(pinned), set(parity.REJECTION_REASONS)) - # Both eligibility exclusions are pinned separately, and the repeat - # case repeats a share still inside the window. - self.assertEqual(len(pinned["delta_ineligible_at_anchor"]), 2) - repeat = self.cases["reject-delta-repeats-retained-share"] - retained_seqs = { - record.share_seq - for record in parity.fold_case(replace(repeat, advances=())).records() - } - self.assertTrue( - all(record.share_seq in retained_seqs for record in repeat.advances[0].delta_records) - ) - - def test_frozen_reference_regeneration_is_a_no_op(self) -> None: - committed = parity.REFERENCE_FIXTURE_PATH.read_bytes() - regenerated = parity.encode_reference_document(parity.reference_document()) - if regenerated != committed: - self.fail( - "the frozen reference no longer matches the shipped pipeline" - " and corpus; if the byte contract changed deliberately," - f" regenerate it with: {parity.REGENERATE_COMMAND}" - ) - - def test_corpus_exercises_the_declared_edge_cases(self) -> None: - # An accidentally hollow corpus would prove nothing, so pin the - # structural properties each case exists for. - self.assertEqual(DEFAULT_INCREMENTAL_SHARE_WINDOW_PAGE_SIZE, 512) - reference = self.reference_cases - self.assertEqual(reference["empty-window"]["record_count"], 0) - self.assertEqual(reference["single-record"]["record_count"], 1) - for count in (511, 512, 513): - self.assertEqual(reference[f"page-boundary-{count}"]["record_count"], count) - self.assertEqual( - len(parity.fold_case(self.cases["page-boundary-512"]).pages), 1 - ) - self.assertEqual( - len(parity.fold_case(self.cases["page-boundary-513"]).pages), 2 - ) - - multi_page_case = self.cases["multi-page-interior-cutoff"] - multi_page = parity.fold_case(multi_page_case) - self.assertGreaterEqual(len(multi_page.pages), 3) - self.assertGreater(multi_page.total_difficulty, multi_page_case.window_weight) - - bulk = parity.fold_case(self.cases["bulk-seeded"]) - self.assertGreaterEqual(reference["bulk-seeded"]["record_count"], 2_500) - self.assertGreaterEqual(len(bulk.pages), 4) - self.assertEqual(len(self.cases["bulk-seeded"].advances), 1) - self.assertGreaterEqual(len(self.cases["incremental-two-advances"].advances), 2) - - crossing = parity.fold_case(self.cases["crossing-row-retained"]) - self.assertGreater( - crossing.total_difficulty, - self.cases["crossing-row-retained"].window_weight, - ) - exact_fit = parity.fold_case(self.cases["crossing-row-exact-fit"]) - self.assertEqual( - exact_fit.total_difficulty, - self.cases["crossing-row-exact-fit"].window_weight, - ) - - self.assertEqual(reference["eligibility-filtering"]["record_count"], 2) - - unsorted_case = self.cases["unsorted-input"] - input_seqs = [record.share_seq for record in unsorted_case.snapshot_records] - self.assertNotEqual(input_seqs, sorted(input_seqs)) - output_seqs = [ - record.share_seq for record in parity.fold_case(unsorted_case).records() - ] - self.assertEqual(output_seqs, sorted(input_seqs)) - - # The escape alphabet: everything Python's ensure_ascii encoder - # treats specially, plus the printable characters that must pass - # through raw (solidus, space, uppercase). DEL is the separating - # case -- escaped by Python, raw in most other encoders -- so it is - # asserted both escaped and never raw, in the spool identities too. - non_ascii = reference["non-ascii-strings"]["pinned_literals"]["canonical_bytes"] - for needle in ( - "\\ud83d\\ude80", "\\u00e9", "\\u0001", "\\t", "\\n", '\\"', "\\\\", - "\\u007f", "\\r", "\\b", "\\f", "\\u001f", "\\u2028", "\\u2029", - "MINER-A/worker 1", - ): - self.assertIn(needle, non_ascii) - self.assertNotIn("\\/", non_ascii) - self.assertNotIn("\x7f", non_ascii) - non_ascii_spool = reference["non-ascii-strings"]["pinned_literals"]["spool_tail"] - self.assertIn("MINER-A/worker 1\\u007f", non_ascii_spool) - self.assertNotIn("\x7f", non_ascii_spool) - - # Null-vs-absent: prism JSON omits an absent credit_policy while the - # compact share tuple carries an explicit null. - single = reference["single-record"]["pinned_literals"] - self.assertNotIn("credit_policy", single["canonical_bytes"]) - self.assertIn(",null]", single["spool_tail"]) - - reuse_tail = reference["credit-policy-identity-reuse"]["pinned_literals"]["spool_tail"] - spool = json.loads("{" + reuse_tail[1:]) - identities = spool["compact_share_identities"] - compact_shares = spool["compact_shares"] - self.assertLess(len(identities), len(compact_shares)) - self.assertEqual( - sorted({row[2] for row in compact_shares}), - list(range(len(identities))), - ) - - # Integer width: wide-integers cannot be quietly shrunk back under - # 32/53/64 bits, must stay inside every declared Rust width (so both - # backends are held to its bytes), and its retention cutoff genuinely - # depends on bits above 2^64 (the three newest rows fall short of the - # weight only because of its 2^127 term; truncated to 64 bits they - # would not). - wide_case = self.cases["wide-integers"] - wide_inputs = wide_case.snapshot_records - self.assertTrue(any(record.share_difficulty.bit_length() > 64 for record in wide_inputs)) - self.assertTrue(any(record.network_difficulty.bit_length() > 64 for record in wide_inputs)) - self.assertTrue(any(record.network_difficulty == 2**128 - 1 for record in wide_inputs)) - self.assertTrue(any(record.share_seq.bit_length() > 32 for record in wide_inputs)) - self.assertTrue(any(record.share_seq == 2**64 - 1 for record in wide_inputs)) - self.assertTrue(any(record.template_height.bit_length() > 32 for record in wide_inputs)) - # ntime: above i32 (top bit set) and at u32's maximum, never beyond. - self.assertTrue(any(record.ntime == 2**32 - 1 for record in wide_inputs)) - self.assertTrue(any(record.ntime.bit_length() == 32 for record in wide_inputs)) - self.assertTrue(any(record.template_height == 0 and record.ntime == 0 for record in wide_inputs)) - self.assertGreater(wide_case.anchor_job_issued_at_ms.bit_length(), 40) - self.assertGreater(wide_case.window_weight.bit_length(), 64) - self.assertIsNone(parity.RUST_INTEGER_DOMAIN.exclusion(wide_case)) - self.assertEqual(reference["wide-integers"]["record_count"], 4) - wide = parity.fold_case(wide_case) - self.assertGreater(wide.total_difficulty, wide_case.window_weight) - newest_three = sum(record.share_difficulty for record in wide.records()[1:]) - self.assertLess(newest_three, wide_case.window_weight) - self.assertGreaterEqual(newest_three % 2**64, wide_case.window_weight % 2**64) - wide_literal = reference["wide-integers"]["pinned_literals"]["canonical_bytes"] - for digits in (2**127 + 3, 2**128 - 1, 2**64 - 1, 2**62, 2**32 - 1, 2**53 + 1): - self.assertIn(str(digits), wide_literal) - - # The genuinely out-of-domain values, one case per width, each - # excluded from the Rust domain by exactly the field it exists for, - # with Python's bytes frozen. - beyond_case = self.cases["difficulty-beyond-u128"] - self.assertTrue( - any(record.share_difficulty.bit_length() > 128 for record in beyond_case.snapshot_records) - ) - self.assertGreater(beyond_case.window_weight.bit_length(), 128) - self.assertIn("u128", parity.RUST_INTEGER_DOMAIN.exclusion(beyond_case) or "") - self.assertEqual(reference["difficulty-beyond-u128"]["record_count"], 3) - beyond = parity.fold_case(beyond_case) - self.assertGreater(beyond.total_difficulty.bit_length(), 128) - self.assertEqual(beyond.records()[0].share_difficulty, 2**128 + 1) - beyond_literal = reference["difficulty-beyond-u128"]["pinned_literals"]["canonical_bytes"] - for digits in (2**128 + 1, 2**128, 2**128 - 1, 10**78 - 1): - self.assertIn(str(digits), beyond_literal) - - ntime_case = self.cases["ntime-beyond-u32"] - self.assertTrue(any(record.ntime.bit_length() > 32 for record in ntime_case.snapshot_records)) - self.assertTrue(any(record.ntime == 2**63 - 1 for record in ntime_case.snapshot_records)) - self.assertEqual( - parity.RUST_INTEGER_DOMAIN.exclusion(ntime_case), - "snapshot record[0].ntime 4294967296 is outside u32", - ) - self.assertIsNone( - parity.RUST_INTEGER_DOMAIN.exclusion( - replace( - ntime_case, - snapshot_records=tuple( - replace(record, ntime=2**32 - 1) for record in ntime_case.snapshot_records - ), - ) - ) - ) - self.assertEqual(reference["ntime-beyond-u32"]["record_count"], 3) - ntime_literal = reference["ntime-beyond-u32"]["pinned_literals"]["canonical_bytes"] - for digits in (2**32, 2**63 - 1): - self.assertIn(str(digits), ntime_literal) - - # Advance-path shapes and their stats, pinned at the values the - # shipped fold reports so touched_pages' subtle definition cannot be - # re-frozen to a plausible wrong one without a visible diff here. - exact_case = self.cases["advance-exact-fit-expiry"] - self.assertEqual(reference["advance-exact-fit-expiry"]["record_count"], 3) - self.assertEqual(parity.fold_case(exact_case).total_difficulty, exact_case.window_weight) - self.assertEqual( - reference["advance-exact-fit-expiry"]["advance_stats"], - [{"added_rows": 1, "expired_rows": 1, "touched_pages": 1}], - ) - exceeds_case = self.cases["advance-delta-exceeds-window"] - self.assertGreater( - sum(record.share_difficulty for record in exceeds_case.advances[0].delta_records), - exceeds_case.window_weight, - ) - self.assertEqual(reference["advance-delta-exceeds-window"]["record_count"], 2) - self.assertEqual( - reference["advance-delta-exceeds-window"]["advance_stats"], - [{"added_rows": 5, "expired_rows": 6, "touched_pages": 1}], - ) - self.assertEqual(reference["advance-partial-expiry-in-appended-page"]["record_count"], 1) - self.assertEqual( - reference["advance-partial-expiry-in-appended-page"]["advance_stats"], - [{"added_rows": 5, "expired_rows": 7, "touched_pages": 1}], - ) - self.assertEqual(self.cases["advance-from-empty-window"].snapshot_records, ()) - self.assertEqual(reference["advance-from-empty-window"]["record_count"], 1) - self.assertEqual( - reference["advance-from-empty-window"]["advance_stats"], - [{"added_rows": 1, "expired_rows": 0, "touched_pages": 0}], - ) - for name in ("advance-empty-delta", "advance-equal-anchor"): - case = self.cases[name] - self.assertEqual(case.advances[0].delta_records, ()) - self.assertEqual( - reference[name]["advance_stats"], - [{"added_rows": 0, "expired_rows": 0, "touched_pages": 0}], - ) - # Bytes identical to the snapshot alone: the advance is a no-op. - self.assertEqual( - reference[name]["canonical_bytes_sha256"], - parity.sha256_hex(parity.outputs_from_window(parity.fold_case(replace(case, advances=()))).canonical_bytes), - ) - equal_anchor = self.cases["advance-equal-anchor"] - self.assertEqual( - equal_anchor.advances[0].anchor_job_issued_at_ms, - equal_anchor.anchor_job_issued_at_ms, - ) - self.assertGreater( - self.cases["advance-empty-delta"].advances[0].anchor_job_issued_at_ms, - self.cases["advance-empty-delta"].anchor_job_issued_at_ms, - ) - - -# --- perturbed backends ----------------------------------------------------- - - -def _window_holding( - records: tuple[parity.AcceptedShareRecord, ...], - case: parity.WindowPipelineCase, - anchor_ms: int, -) -> IncrementalShareWindow: - """A window retaining exactly these records, with no cutoff applied.""" - window = IncrementalShareWindow.from_full_snapshot( - list(records), - anchor_job_issued_at_ms=anchor_ms, - window_weight=sum(int(record.share_difficulty) for record in records) + 1, - page_size=case.page_size, - ) - return replace(window, window_weight=case.window_weight) - - -_INTEGER_RECORD_FIELDS = ( - "share_seq", - "share_difficulty", - "network_difficulty", - "template_height", - "job_issued_at_ms", - "accepted_at_ms", - "ntime", -) - - -def _map_case_integers( - case: parity.WindowPipelineCase, - transform, - *, - record_fields: tuple[str, ...] = _INTEGER_RECORD_FIELDS, -) -> parity.WindowPipelineCase: - """The case as a backend carrying its integers in some narrower type sees it.""" - - def mapped(record: parity.AcceptedShareRecord) -> parity.AcceptedShareRecord: - return replace(record, **{name: transform(getattr(record, name)) for name in record_fields}) - - return replace( - case, - anchor_job_issued_at_ms=transform(case.anchor_job_issued_at_ms), - window_weight=transform(case.window_weight), - snapshot_records=tuple(mapped(record) for record in case.snapshot_records), - advances=tuple( - replace( - step, - anchor_job_issued_at_ms=transform(step.anchor_job_issued_at_ms), - delta_records=tuple(mapped(record) for record in step.delta_records), - ) - for step in case.advances - ), - ) - - -def _reencoded(result: parity.WindowPipelineResult, old: bytes, new: bytes) -> parity.WindowPipelineResult: - """The backend's native byte outputs re-encoded with one escape rule changed.""" - if isinstance(result, parity.WindowPipelineRejection): - return result - record_jsons = tuple(record.replace(old, new) for record in result.record_jsons) - canonical_bytes = result.canonical_bytes.replace(old, new) - return replace( - result, - record_jsons=record_jsons, - canonical_bytes=canonical_bytes, - canonical_digest=parity.sha256_hex(canonical_bytes), - ) - - -class _FieldOrderPerturbedAdapter: - """Encodes record keys in insertion order instead of sorted order.""" - - name = "perturbed-field-order" - - def run(self, case: parity.WindowPipelineCase) -> parity.WindowPipelineResult: - window = parity.fold_case(case) - base = parity.outputs_from_window(window) - record_jsons = tuple( - json.dumps(record, sort_keys=False, separators=(",", ":"), default=str).encode() - for record in window.json_records() - ) - canonical_bytes = b"[" + b",".join(record_jsons) + b"]" - return replace( - base, - record_jsons=record_jsons, - canonical_bytes=canonical_bytes, - canonical_digest=parity.sha256_hex(canonical_bytes), - ) - - -class _MissingPageSeparatorAdapter: - """Streams the digest without the comma between page fragments.""" - - name = "perturbed-missing-page-separator" - - def run(self, case: parity.WindowPipelineCase) -> parity.WindowPipelineResult: - window = parity.fold_case(case) - base = parity.outputs_from_window(window) - fragments = [ - page.canonical_json_items - for page in window.pages - if page.canonical_json_items - ] - return replace( - base, - canonical_digest=parity.sha256_hex(b"[" + b"".join(fragments) + b"]"), - ) - - -class _NullCreditPolicyAdapter: - """Emits credit_policy as an explicit null instead of omitting the key.""" - - name = "perturbed-null-credit-policy" - - def run(self, case: parity.WindowPipelineCase) -> parity.WindowPipelineResult: - window = parity.fold_case(case) - base = parity.outputs_from_window(window) - record_jsons = tuple( - parity.canonical_json_bytes({**record, "credit_policy": record.get("credit_policy")}) - for record in window.json_records() - ) - canonical_bytes = b"[" + b",".join(record_jsons) + b"]" - return replace( - base, - record_jsons=record_jsons, - canonical_bytes=canonical_bytes, - canonical_digest=parity.sha256_hex(canonical_bytes), - ) - - -class _CrossingRowDroppedAdapter: - """Applies a wrong strict cutoff: the crossing row is dropped.""" - - name = "perturbed-crossing-row-dropped" - - def run(self, case: parity.WindowPipelineCase) -> parity.WindowPipelineResult: - window = parity.fold_case(case) - records = window.records() - if records and window.total_difficulty > case.window_weight: - records = records[1:] - rebuilt = IncrementalShareWindow.from_full_snapshot( - list(records), - anchor_job_issued_at_ms=parity.final_anchor_job_issued_at_ms(case), - window_weight=case.window_weight, - page_size=case.page_size, - ) - return parity.outputs_from_window(rebuilt) - - -class _SpoolWhitespaceAdapter: - """Writes the spool fragments with json.dumps's default spaced separators.""" - - name = "perturbed-spool-whitespace" - - def run(self, case: parity.WindowPipelineCase) -> parity.WindowPipelineResult: - window = parity.fold_case(case) - base = parity.outputs_from_window(window) - identities, compact_shares = _compact_share_payload(list(window.json_records())) - spool_tail = ( - b',"compact_share_identities":' - + json.dumps(identities).encode() - + b',"compact_shares":' - + json.dumps(compact_shares).encode() - + b"}" - ) - return replace(base, spool_tail=spool_tail) - - -class _WrappingU64Adapter: - """Carries every integer in a wrapping unsigned 64-bit type.""" - - name = "perturbed-u64-wrap" - - def run(self, case: parity.WindowPipelineCase) -> parity.WindowPipelineResult: - return parity.run_python_pipeline(_map_case_integers(case, lambda value: value % 2**64)) - - -class _WrappingU32Adapter: - """Carries every integer in a wrapping unsigned 32-bit type.""" - - name = "perturbed-u32-wrap" - - def run(self, case: parity.WindowPipelineCase) -> parity.WindowPipelineResult: - return parity.run_python_pipeline(_map_case_integers(case, lambda value: value % 2**32)) - - -class _SaturatingU64AccumulatorAdapter: - """Clamps the difficulties and window_weight at u64::MAX: a saturating u64 accumulator.""" - - name = "perturbed-u64-saturating-accumulator" - - def run(self, case: parity.WindowPipelineCase) -> parity.WindowPipelineResult: - return parity.run_python_pipeline( - _map_case_integers( - case, - lambda value: min(value, 2**64 - 1), - record_fields=("share_difficulty",), - ) - ) - - -class _SaturatingU128DifficultyAdapter: - """Clamps share_difficulty and window_weight at u128::MAX: a saturating u128 accumulator. - - Declares the Rust domain, so where it answers the oracle must hold it to - the bytes and not excuse it: a declared domain permits Unsupported, never - wrong digits. - """ - - name = "perturbed-u128-saturating-difficulty" - integer_domain = parity.RUST_INTEGER_DOMAIN - - def run(self, case: parity.WindowPipelineCase) -> parity.WindowPipelineResult: - return parity.run_python_pipeline( - _map_case_integers( - case, - lambda value: min(value, 2**128 - 1), - record_fields=("share_difficulty",), - ) - ) - - -class _DeclaredDomainDecliningAdapter: - """Declares the Rust domain and declines exactly the cases outside it. - - The honest shape of a fixed-width backend: Unsupported with a declared - reason where the domain excludes the case, Python's bytes everywhere else. - """ - - name = "perturbed-declared-domain-declining" - integer_domain = parity.RUST_INTEGER_DOMAIN - - def run(self, case: parity.WindowPipelineCase) -> parity.WindowPipelineResult: - exclusion = self.integer_domain.exclusion(case) - if exclusion is not None: - return parity.WindowPipelineUnsupported(exclusion) - return parity.run_python_pipeline(case) - - -class _OverDecliningAdapter: - """Declares the Rust domain but declines anything above 64 bits. - - A backend whose real width is narrower than the one it declares: it - answers Unsupported for wide-integers, which its declared domain includes. - """ - - name = "perturbed-over-declining" - integer_domain = parity.RUST_INTEGER_DOMAIN - - def run(self, case: parity.WindowPipelineCase) -> parity.WindowPipelineResult: - records = case.snapshot_records + tuple( - record for step in case.advances for record in step.delta_records - ) - if any(record.share_difficulty.bit_length() > 64 for record in records): - return parity.WindowPipelineUnsupported("difficulty above u64") - return parity.run_python_pipeline(case) - - -class _UndeclaredDecliningAdapter: - """Declares no domain at all yet declines the beyond-u128 case.""" - - name = "perturbed-undeclared-declining" - - def run(self, case: parity.WindowPipelineCase) -> parity.WindowPipelineResult: - if case.name == "difficulty-beyond-u128": - return parity.WindowPipelineUnsupported("cannot carry 2^128+1") - return parity.run_python_pipeline(case) - - -class _NeverRefusingAdapter: - """Folds whatever it is given: where the shipped fold refuses, it recovers by itself. - - Models a backend with every check removed, or one that performs the - coordinator's full-rescan recovery internally and returns its bytes as - if nothing had happened -- the bytes a lenient fold or the recovery - would produce are exactly what the oracle must not accept as parity. - """ - - name = "perturbed-never-refusing" - - def run(self, case: parity.WindowPipelineCase) -> parity.WindowPipelineResult: - result = parity.run_python_pipeline(case) - if isinstance(result, parity.WindowPipelineOutputs): - return result - union = case.snapshot_records + tuple( - record for step in case.advances for record in step.delta_records - ) - seen_seqs: set[int] = set() - seen_ids: set[str] = set() - kept = [] - for record in union: - if ( - record.share_seq in seen_seqs - or record.share_id in seen_ids - or record.share_difficulty <= 0 - ): - continue - seen_seqs.add(record.share_seq) - seen_ids.add(record.share_id) - kept.append(record) - folded = IncrementalShareWindow.from_full_snapshot( - kept, - anchor_job_issued_at_ms=parity.final_anchor_job_issued_at_ms(case), - window_weight=max(case.window_weight, 1), - page_size=max(case.page_size, 1), - ) - return parity.outputs_from_window( - folded, - tuple( - parity.WindowPipelineAdvanceStats(len(step.delta_records), 0, 0) - for step in case.advances - ), - ) - - -class _MisfiledRejectionAdapter: - """Refuses where the shipped fold does, but files every refusal under one category.""" - - name = "perturbed-misfiled-rejection" - - def run(self, case: parity.WindowPipelineCase) -> parity.WindowPipelineResult: - result = parity.run_python_pipeline(case) - if isinstance(result, parity.WindowPipelineRejection): - return parity.WindowPipelineRejection("anchor_regression") - return result - - -class _StrictAdvanceExpiryAdapter: - """Expires on > instead of >= in the advance loops (modelled as weight+1 while advancing).""" - - name = "perturbed-strict-advance-expiry" - - def run(self, case: parity.WindowPipelineCase) -> parity.WindowPipelineResult: - try: - window = IncrementalShareWindow.from_full_snapshot( - list(case.snapshot_records), - anchor_job_issued_at_ms=case.anchor_job_issued_at_ms, - window_weight=case.window_weight, - page_size=case.page_size, - ) - stats = [] - for step in case.advances: - window, step_stats = replace(window, window_weight=case.window_weight + 1).advance( - list(step.delta_records), - anchor_job_issued_at_ms=step.anchor_job_issued_at_ms, - ) - stats.append(parity.WindowPipelineAdvanceStats.from_shipped(step_stats)) - except (ValueError, IncrementalWindowFallback): - return parity.run_python_pipeline(case) - return parity.outputs_from_window(replace(window, window_weight=case.window_weight), tuple(stats)) - - -class _OldPagesOnlyExpiryAdapter: - """Expires only among pre-existing rows; the appended delta is immune. - - The natural shape of a drop-prefix / append-suffix protocol: when the - shipped fold exhausts every old row and goes on into the delta, this - backend stops at the old rows and keeps the delta whole. - """ - - name = "perturbed-old-pages-only-expiry" - - def run(self, case: parity.WindowPipelineCase) -> parity.WindowPipelineResult: - try: - window = IncrementalShareWindow.from_full_snapshot( - list(case.snapshot_records), - anchor_job_issued_at_ms=case.anchor_job_issued_at_ms, - window_weight=case.window_weight, - page_size=case.page_size, - ) - stats = [] - for step in case.advances: - advanced, step_stats = window.advance( - list(step.delta_records), - anchor_job_issued_at_ms=step.anchor_job_issued_at_ms, - ) - retained_seqs = {record.share_seq for record in advanced.records()} - if all(record.share_seq in retained_seqs for record in step.delta_records): - window = advanced - else: - step_stats = replace(step_stats, expired_rows=len(window.records())) - window = _window_holding(step.delta_records, case, step.anchor_job_issued_at_ms) - stats.append(parity.WindowPipelineAdvanceStats.from_shipped(step_stats)) - except (ValueError, IncrementalWindowFallback): - return parity.run_python_pipeline(case) - return parity.outputs_from_window(window, tuple(stats)) - - -class _EqualAnchorRejectingAdapter: - """Treats an advance at the same anchor as a regression (<= instead of <).""" - - name = "perturbed-equal-anchor-rejecting" - - def run(self, case: parity.WindowPipelineCase) -> parity.WindowPipelineResult: - anchor_ms = case.anchor_job_issued_at_ms - for step in case.advances: - if step.anchor_job_issued_at_ms == anchor_ms: - return parity.WindowPipelineRejection("anchor_regression") - anchor_ms = step.anchor_job_issued_at_ms - return parity.run_python_pipeline(case) - - -class _RawDelAdapter: - """An encoder that escapes only below 0x20 plus quote and backslash: DEL stays raw.""" - - name = "perturbed-raw-del" - - def run(self, case: parity.WindowPipelineCase) -> parity.WindowPipelineResult: - return _reencoded(parity.run_python_pipeline(case), b"\\u007f", b"\x7f") - - -class _EscapedSolidusAdapter: - """An encoder that emits the optional JSON escape for solidus.""" - - name = "perturbed-escaped-solidus" - - def run(self, case: parity.WindowPipelineCase) -> parity.WindowPipelineResult: - return _reencoded(parity.run_python_pipeline(case), b"/", b"\\/") - - -class _NoAdvanceStatsAdapter: - """Produces the bytes but reports no per-advance stats.""" - - name = "perturbed-no-advance-stats" - - def run(self, case: parity.WindowPipelineCase) -> parity.WindowPipelineResult: - result = parity.run_python_pipeline(case) - if isinstance(result, parity.WindowPipelineRejection): - return result - return replace(result, advance_stats=()) - - -class _SurvivorTouchedPagesAdapter: - """Derives touched_pages from which pre-existing pages survive, not from the work done. - - Counts a pre-existing page when it is partially retained, and the - append-into page only if any of its rows survive -- so a page appended - into and then expired wholesale in the same advance is not counted, where - the shipped fold counts it because its records were rewritten. - """ - - name = "perturbed-survivor-touched-pages" - - def run(self, case: parity.WindowPipelineCase) -> parity.WindowPipelineResult: - try: - window = IncrementalShareWindow.from_full_snapshot( - list(case.snapshot_records), - anchor_job_issued_at_ms=case.anchor_job_issued_at_ms, - window_weight=case.window_weight, - page_size=case.page_size, - ) - stats = [] - for step in case.advances: - advanced, step_stats = window.advance( - list(step.delta_records), - anchor_job_issued_at_ms=step.anchor_job_issued_at_ms, - ) - retained_seqs = {record.share_seq for record in advanced.records()} - touched: set[int] = set() - last = window.pages[-1] if window.pages else None - if step.delta_records and last is not None and len(last.records) < case.page_size: - if any(record.share_seq in retained_seqs for record in last.records): - touched.add(len(window.pages) - 1) - for index, page in enumerate(window.pages): - surviving = sum(1 for record in page.records if record.share_seq in retained_seqs) - if 0 < surviving < len(page.records): - touched.add(index) - stats.append( - parity.WindowPipelineAdvanceStats( - added_rows=step_stats.added_rows, - expired_rows=step_stats.expired_rows, - touched_pages=len(touched), - ) - ) - window = advanced - except (ValueError, IncrementalWindowFallback): - return parity.run_python_pipeline(case) - return parity.outputs_from_window(window, tuple(stats)) - - -class WindowPipelineDivergenceDetectionTests(unittest.TestCase): - """Prove the harness fails on known divergence classes, and only there.""" - - cases: dict[str, parity.WindowPipelineCase] - reference_cases: dict[str, dict[str, object]] - - @classmethod - def setUpClass(cls) -> None: - cls.cases = {case.name: case for case in parity.build_corpus()} - cls.reference_cases = parity.load_reference_document()["cases"] - - def _diverging(self, adapter: parity.WindowPipelineAdapter, case_name: str) -> list[str]: - return parity.diverging_outputs( - self.reference_cases[case_name], - adapter.run(self.cases[case_name]), - ) - - def test_record_field_reordering_is_detected(self) -> None: - problems = self._diverging(_FieldOrderPerturbedAdapter(), "single-record") - self.assertTrue(problems) - self.assertTrue(any(problem.startswith("record_jsons[0]") for problem in problems)) - - def test_missing_page_separator_is_detected_only_on_multi_page_windows(self) -> None: - adapter = _MissingPageSeparatorAdapter() - problems = self._diverging(adapter, "page-boundary-513") - self.assertTrue(problems) - self.assertTrue(all(problem.startswith("canonical_digest") for problem in problems)) - # On a single page the missing separator never fires, so this framing - # bug is invisible there -- the reason multi-page corpora must exist. - self.assertEqual(self._diverging(adapter, "page-boundary-512"), []) - - def test_null_credit_policy_instead_of_absent_is_detected(self) -> None: - problems = self._diverging(_NullCreditPolicyAdapter(), "single-record") - self.assertTrue(problems) - self.assertTrue(any(problem.startswith("record_jsons[0]") for problem in problems)) - - def test_dropped_crossing_row_is_detected_only_where_one_exists(self) -> None: - adapter = _CrossingRowDroppedAdapter() - problems = self._diverging(adapter, "crossing-row-retained") - self.assertTrue(any(problem.startswith("record_count") for problem in problems)) - # The exact-fit sibling has no crossing row, so the wrong cutoff - # passes there -- the pair is what localizes the off-by-one. - self.assertEqual(self._diverging(adapter, "crossing-row-exact-fit"), []) - - def test_spool_separator_whitespace_is_detected(self) -> None: - problems = self._diverging(_SpoolWhitespaceAdapter(), "credit-policy-identity-reuse") - self.assertTrue(problems) - self.assertTrue(all(problem.startswith("spool_tail") for problem in problems)) - - def _diverging_declared(self, adapter: parity.WindowPipelineAdapter, case_name: str) -> list[str]: - """As _diverging, but telling the harness the backend's declared domain.""" - case = self.cases[case_name] - return parity.diverging_outputs( - self.reference_cases[case_name], - adapter.run(case), - domain_exclusion=parity.domain_exclusion(adapter, case), - ) - - def test_wrapping_64_bit_integers_are_detected_only_above_64_bits(self) -> None: - adapter = _WrappingU64Adapter() - # Truncating window_weight and the difficulties stops retention one - # row early, so the count itself diverges, not only the digits. - problems = self._diverging(adapter, "wide-integers") - self.assertTrue(any(problem.startswith("record_count") for problem in problems)) - self.assertTrue(self._diverging(adapter, "difficulty-beyond-u128")) - # Every other case lives below 2^31, where a 64-bit backend is exact: - # the width holes were invisible until the wide cases existed. - for name in ("single-record", "incremental-two-advances", "bulk-seeded"): - self.assertEqual(self._diverging(adapter, name), []) - - def test_retuned_wide_integers_still_refuses_every_narrow_backend(self) -> None: - # The protection wide-integers was built for, re-checked after the - # values were brought inside the declared Rust widths: a u32-wrapping, - # a u64-wrapping and a u64-saturating backend each still fail it (the - # count itself, not only the digits, for the two that touch the - # accumulator), while passing the sub-2^31 corpus exactly as before. - # Declaring the Rust domain could not excuse any of them, because the - # case lies inside that domain. - for adapter in (_WrappingU32Adapter(), _WrappingU64Adapter(), _SaturatingU64AccumulatorAdapter()): - with self.subTest(adapter=adapter.name): - problems = self._diverging(adapter, "wide-integers") - self.assertTrue(problems, adapter.name) - if adapter.name != "perturbed-u32-wrap": - self.assertTrue(any(problem.startswith("record_count") for problem in problems)) - declared = type(adapter.name, (object,), {"name": adapter.name, "integer_domain": parity.RUST_INTEGER_DOMAIN, "run": staticmethod(adapter.run)})() - self.assertEqual(self._diverging_declared(declared, "wide-integers"), problems) - for name in ("single-record", "incremental-two-advances", "crossing-row-retained"): - self.assertEqual(self._diverging(adapter, name), []) - self.assertIsNone(parity.RUST_INTEGER_DOMAIN.exclusion(self.cases["wide-integers"])) - - def test_saturating_u128_difficulty_is_refused_by_the_beyond_u128_case_alone(self) -> None: - # 128-bit difficulty accumulation passes wide-integers (inside the - # declared domain) and is refused where numeric(78,0) exceeds it, by - # the clamped crossing row's digits (retained count still agrees). - # The backend declares the Rust domain, and that excuses nothing: it - # produced bytes, so it is held to the frozen bytes. - adapter = _SaturatingU128DifficultyAdapter() - self.assertEqual(self._diverging_declared(adapter, "wide-integers"), []) - self.assertIsNotNone(parity.domain_exclusion(adapter, self.cases["difficulty-beyond-u128"])) - problems = self._diverging_declared(adapter, "difficulty-beyond-u128") - self.assertTrue(any(problem.startswith("record_jsons[0]") for problem in problems)) - self.assertFalse(any(problem.startswith("record_count") for problem in problems)) - - def test_unsupported_is_parity_outside_the_declared_domain_and_divergence_inside(self) -> None: - honest = _DeclaredDomainDecliningAdapter() - for name in ("difficulty-beyond-u128", "ntime-beyond-u32"): - with self.subTest(case=name): - # Outside the declared domain: Unsupported is parity, while - # the frozen reference keeps Python's bytes for the case. - self.assertEqual(self._diverging_declared(honest, name), []) - self.assertIn("record_count", self.reference_cases[name]) - result = honest.run(self.cases[name]) - self.assertIsInstance(result, parity.WindowPipelineUnsupported) - # Inside it, the same backend produces bytes and is held to them. - self.assertEqual(self._diverging_declared(honest, "wide-integers"), []) - self.assertIsInstance(honest.run(self.cases["wide-integers"]), parity.WindowPipelineOutputs) - - # A backend narrower than it declares: Unsupported inside the declared - # domain is a divergence, named as such. - over = _OverDecliningAdapter() - self.assertEqual( - self._diverging_declared(over, "wide-integers"), - ["unsupported: backend declared 'difficulty above u64' but the case is inside its declared integer domain"], - ) - self.assertEqual(self._diverging_declared(over, "difficulty-beyond-u128"), []) - - # A backend that declares nothing claims the whole domain, so any - # Unsupported it returns is a divergence -- even on the beyond case. - undeclared = _UndeclaredDecliningAdapter() - self.assertIs(parity.adapter_integer_domain(undeclared), parity.UNBOUNDED_INTEGER_DOMAIN) - self.assertEqual( - self._diverging_declared(undeclared, "difficulty-beyond-u128"), - ["unsupported: backend declared 'cannot carry 2^128+1' but the case is inside its declared integer domain"], - ) - # And the shipped pipeline never declines: the reference cannot even - # freeze an Unsupported. - with self.assertRaisesRegex(RuntimeError, "declared the case unsupported"): - parity.reference_case_entry( - self.cases["single-record"], - parity.WindowPipelineUnsupported("never"), - ) - - def test_a_backend_that_never_refuses_is_detected_on_every_rejection_case(self) -> None: - adapter = _NeverRefusingAdapter() - rejection_cases = [case for case in self.cases.values() if case.expected_rejection] - self.assertEqual(len(rejection_cases), 12) - for case in rejection_cases: - with self.subTest(case=case.name): - problems = self._diverging(adapter, case.name) - self.assertEqual(len(problems), 1) - self.assertTrue(problems[0].startswith("rejected: backend produced outputs")) - # Where the shipped fold accepts, the lenient backend is identical. - self.assertEqual(self._diverging(adapter, "incremental-two-advances"), []) - - def test_misfiled_rejection_category_is_detected(self) -> None: - adapter = _MisfiledRejectionAdapter() - self.assertEqual(self._diverging(adapter, "reject-anchor-regression"), []) - self.assertEqual( - self._diverging(adapter, "reject-delta-not-append"), - ["rejected: 'anchor_regression' != frozen 'delta_not_append'"], - ) - - def test_strict_advance_expiry_is_detected_only_at_exact_fit(self) -> None: - adapter = _StrictAdvanceExpiryAdapter() - problems = self._diverging(adapter, "advance-exact-fit-expiry") - self.assertTrue(any(problem.startswith("record_count") for problem in problems)) - # No other advance in the corpus lands an expiry exactly on the - # weight, and the snapshot-side twin is separate code. - for name in ("incremental-two-advances", "bulk-seeded", "advance-delta-exceeds-window", "crossing-row-exact-fit"): - self.assertEqual(self._diverging(adapter, name), []) - - def test_expiry_confined_to_old_pages_is_detected_only_when_the_delta_outweighs_the_window(self) -> None: - adapter = _OldPagesOnlyExpiryAdapter() - for name in ("advance-delta-exceeds-window", "advance-partial-expiry-in-appended-page"): - with self.subTest(case=name): - problems = self._diverging(adapter, name) - self.assertTrue(any(problem.startswith("record_count") for problem in problems)) - for name in ("incremental-two-advances", "bulk-seeded", "advance-exact-fit-expiry"): - self.assertEqual(self._diverging(adapter, name), []) - - def test_equal_anchor_rejection_is_detected(self) -> None: - adapter = _EqualAnchorRejectingAdapter() - self.assertEqual( - self._diverging(adapter, "advance-equal-anchor"), - ["rejected: backend rejected as 'anchor_regression' but the frozen reference has outputs"], - ) - self.assertEqual(self._diverging(adapter, "reject-anchor-regression"), []) - self.assertEqual(self._diverging(adapter, "advance-empty-delta"), []) - - def test_raw_del_and_escaped_solidus_are_detected(self) -> None: - for adapter in (_RawDelAdapter(), _EscapedSolidusAdapter()): - with self.subTest(adapter=adapter.name): - problems = self._diverging(adapter, "non-ascii-strings") - self.assertTrue(any(problem.startswith("record_jsons[3]") for problem in problems)) - self.assertEqual(self._diverging(adapter, "single-record"), []) - - def test_missing_advance_stats_are_detected_on_advancing_cases(self) -> None: - adapter = _NoAdvanceStatsAdapter() - problems = self._diverging(adapter, "advance-from-empty-window") - self.assertEqual(len(problems), 1) - self.assertTrue(problems[0].startswith("advance_stats")) - self.assertEqual(self._diverging(adapter, "single-record"), []) - - def test_touched_pages_from_survivors_is_detected_only_where_an_appended_page_expires(self) -> None: - adapter = _SurvivorTouchedPagesAdapter() - for name in ("advance-delta-exceeds-window", "advance-partial-expiry-in-appended-page"): - with self.subTest(case=name): - problems = self._diverging(adapter, name) - self.assertTrue(problems) - self.assertTrue(all(problem.startswith("advance_stats") for problem in problems)) - # Where the appended page survives (partially or whole), the two - # definitions agree -- which is why those cases alone proved nothing. - for name in ("advance-exact-fit-expiry", "incremental-two-advances", "advance-from-empty-window"): - self.assertEqual(self._diverging(adapter, name), []) - - -if __name__ == "__main__": - unittest.main() diff --git a/tests/test_window_pipeline_parity_gate.py b/tests/test_window_pipeline_parity_gate.py deleted file mode 100644 index af72800c6..000000000 --- a/tests/test_window_pipeline_parity_gate.py +++ /dev/null @@ -1,84 +0,0 @@ -#!/usr/bin/env python3 -"""Fail-closed contracts for the payout-window CI parity entry point.""" - -from __future__ import annotations - -import os -import sys -import tempfile -import unittest -from pathlib import Path -from unittest import mock - -from tests import window_pipeline_parity as parity -from tests import window_pipeline_parity_gate as gate - - -class WindowPipelineParityGateTests(unittest.TestCase): - def test_adapter_selection_must_be_explicit(self) -> None: - with mock.patch.dict(os.environ, {}, clear=True): - with self.assertRaisesRegex( - RuntimeError, - f"{parity.ADAPTER_ENV_VAR} must explicitly select", - ): - gate.resolve_gate_adapter() - - def test_python_selection_resolves_the_python_adapter(self) -> None: - with mock.patch.dict( - os.environ, - {parity.ADAPTER_ENV_VAR: "python"}, - clear=True, - ): - adapter, command = gate.resolve_gate_adapter() - self.assertIsInstance(adapter, parity.PythonWindowPipelineAdapter) - self.assertEqual(adapter.name, "python") - self.assertEqual(command, ()) - - def test_rust_selection_resolves_the_prebuilt_real_daemon(self) -> None: - with tempfile.TemporaryDirectory() as tmp: - binary = Path(tmp) / gate.RUST_DAEMON_BINARY - binary.touch(mode=0o755) - binary.chmod(0o755) - with mock.patch.dict( - os.environ, - { - parity.ADAPTER_ENV_VAR: "rust-daemon", - "PRISM_TOOL_BIN_DIR": tmp, - }, - clear=True, - ): - adapter, command = gate.resolve_gate_adapter() - self.assertIsInstance(adapter, parity.RustDaemonWindowPipelineAdapter) - self.assertEqual(adapter.name, "rust-daemon") - self.assertEqual(command, (str(binary),)) - - def test_rust_selection_refuses_a_missing_binary_and_cargo_fallback(self) -> None: - with tempfile.TemporaryDirectory() as tmp, mock.patch.dict( - os.environ, - { - parity.ADAPTER_ENV_VAR: "rust-daemon", - "PRISM_TOOL_BIN_DIR": tmp, - }, - clear=True, - ): - with self.assertRaisesRegex(RuntimeError, "refusing cargo fallback"): - gate.resolve_gate_adapter() - - def test_rust_adapter_refuses_a_protocol_mismatch(self) -> None: - fake_daemon = Path(__file__).parent / "fixtures" / "fake_serve_builder.py" - case = parity.build_corpus()[0] - with mock.patch.object( - parity, - "prism_tool_command", - return_value=[sys.executable, str(fake_daemon)], - ), mock.patch.dict( - os.environ, - {"FAKE_SERVE_BUILDER_PROTOCOL": "1"}, - clear=True, - ): - with self.assertRaisesRegex(RuntimeError, "daemon announced handshake"): - parity.RustDaemonWindowPipelineAdapter().run(case) - - -if __name__ == "__main__": - unittest.main() diff --git a/tests/window_pipeline_parity.py b/tests/window_pipeline_parity.py deleted file mode 100644 index 203fb762f..000000000 --- a/tests/window_pipeline_parity.py +++ /dev/null @@ -1,2147 +0,0 @@ -#!/usr/bin/env python3 -"""Differential parity oracle for the PRISM payout-window pipeline. - -Why this exists: the window pipeline -- accepted-share records in, folded -window plus canonical digest plus audit-builder spool fragments out -- is -slated to gain a second implementation that must be proven byte-identical to -the shipped Python one before it can take traffic (issue #131's method: run -both against the same inputs and assert byte-identical outputs). That proof -needs three things to exist *before* the second implementation does: a corpus -derived from the fold's own invariants, frozen byte-exact reference outputs -produced by the shipped pipeline at a known revision, and a runner that can -drive more than one backend over the same corpus. This module provides all -three, composing the two house precedents: the seeded-corpus discipline of -``tests.test_prism_incremental_payout_window`` and the frozen-reference -discipline of the metrics render-parity fixture. - -What is pinned per corpus case, all inside the migration's scope: - -1. ``record_jsons`` -- the folded window as the ordered - ``AcceptedShareRecord.to_prism_json()`` sequence, one canonical JSON - encoding per record (sorted keys, ``(",", ":")`` separators, ASCII - escapes), exactly the encoding each window page pre-computes. -2. ``canonical_bytes`` / ``canonical_digest`` -- the canonical JSON array - exactly as ``IncrementalShareJsonSequence.canonical_json_sha256`` streams - it (non-empty page fragments joined with ``,`` inside ``[``/``]``) and its - SHA-256 hex digest. The digest is *not* a hash of a monolithic re-encode; - an implementation with wrong framing produces a different digest, so the - framed bytes are pinned alongside the digest. -3. ``spool_tail`` -- the audit-builder spool payload tail written by - ``_ShareWindowSerialization.acquire_spooled_tail``, byte-for-byte, - including the ``,"compact_share_identities":`` / ``,"compact_shares":`` / - ``}`` framing, because that framing is the wire contract. -4. ``advance_stats`` -- one ``IncrementalWindowAdvanceStats`` triple - (``added_rows``, ``expired_rows``, ``touched_pages``) per advance step, in - order. These are not wire bytes, but the coordinator exports them to - metrics on every incremental materialization, so a backend that owns - ``advance()`` reports them and they must agree. ``touched_pages`` has a - definition (see :class:`WindowPipelineAdvanceStats`) that is easy to - re-derive wrongly from the retained result instead of from the work done. -5. A rejection, when the shipped pipeline refuses the case. ``from_full_ - snapshot`` refuses five shapes and ``advance()`` six; a backend that - accepts an input the shipped pipeline rejects, or rejects one it accepts, - ships a different payout window. A rejected case freezes only the - implementation-neutral reason category (``REJECTION_REASONS``), never a - Python exception class or message, and ``diverging_outputs`` treats - rejected-vs-outputs in either direction as a divergence and - rejected-vs-rejected with different categories as one too. - -The oracle pins the *pipeline's rejection*, not the coordinator's recovery -from it. In production ``payout_state`` catches the advance fallback, drops -its cached window and re-materializes from a full rescan; that recovery is -coordinator behaviour and is tested there (``tests.test_prism_payout_state``). -Pinning recovery bytes here would let a backend that silently recovers inside -itself -- returning the rescan's bytes where the shipped pipeline refuses -- -pass as parity, which is exactly the divergence class the rejection cases -exist to catch. - -Integer width. The ledger schema is wider than any fixed machine type: -``share_difficulty`` and ``network_difficulty`` are ``numeric(78,0)``, -``share_seq``, ``template_height`` and ``ntime`` are ``bigint``, and the -millisecond fields are epoch milliseconds above 2^40; Python carries all of -them exactly and renders every digit. The contract the corpus enforces is -narrower, and the decision is made here explicitly: the pipeline's declared -integer widths are the Rust widths -- - - share_seq u64 - share_difficulty u128 - network_difficulty u128 - template_height u64 - ntime u32 - job_issued_at_ms, accepted_at_ms i64 (and every anchor) - window_weight u128 - page_size u64 - --- with the fold's difficulty accumulators u128 as well, so a window whose -difficulty total leaves u128 is outside the domain too. Why u128 and not the -schema's 78 digits: ``share_difficulty`` is ``(pow_limit_target*1e6)//target``, -so at a Bitcoin-mainnet-scale target a share is ~2^98 and a 16x window totals -~2^102 -- a factor of roughly 6*10^7 short of 2^128, unreachable on any chain -this code will see. ``numeric(78,0)`` is a generic wide-numeric column -choice, not a considered statement that difficulties exceed 2^128; widening -the Rust fold to arbitrary precision would put bignum arithmetic on the exact -path the migration exists to make faster. - -A narrower domain is modelled as exactly that, not as a divergence. A -backend declares its domain (:class:`WindowPipelineIntegerDomain`: the -shipped Python pipeline declares ``UNBOUNDED_INTEGER_DOMAIN``, the daemon -``RUST_INTEGER_DOMAIN``, and a backend that declares nothing claims all of -it) and may answer a case outside that domain with the third adapter -outcome, :class:`WindowPipelineUnsupported`, carrying a declared reason. The -frozen reference keeps Python's bytes for such cases; the harness accepts -``Unsupported`` only where the case really lies outside the backend's -declared domain (:func:`diverging_outputs` is told so by -:func:`domain_exclusion`), still fails a backend that returns wrong *bytes* -there (a saturating u128 accumulator, say), and fails a backend that -declares ``Unsupported`` for a case inside its declared domain. This is what -the system really does: the Rust daemon answers ``out_of_range`` instead of -folding, the coordinator degrades to the in-process fold for that -materialization without retiring the daemon, and Python's unbounded integers -produce the right answer. - -The width cases are split accordingly. ``wide-integers`` carries every -integer field far above the 2^31/2^53/2^64 boundaries at which a wrapping or -saturating 32/53/64-bit backend breaks, yet inside every declared width, so -both backends produce bytes and full parity is enforced there: the -protection it was built for (nine narrow-integer backends passed the -original fourteen-case corpus) survives the domain restriction, and the -divergence suite re-checks it against u32-wrapping, u64-wrapping and -u64-saturating backends. The genuinely out-of-domain values live in -``difficulty-beyond-u128`` (``share_difficulty`` 2^128+1 as the retained -crossing row, ``network_difficulty`` at 2^128-1, 2^128 and numeric(78,0)'s -maximum, ``window_weight`` and the retained total above 2^128) and -``ntime-beyond-u32`` (``ntime`` at 2^32 and bigint's maximum): Python's -bytes are frozen, a u128/u32 backend answers ``Unsupported``, and one that -clamps or truncates renders different digits and fails. - -Adapter contract (the seam a future backend implements): inputs in, one -result out. Inputs are one :class:`WindowPipelineCase`, available as an -implementation-neutral JSON document via :func:`case_input_document` -- the -snapshot records with every durable field explicit (``credit_policy`` is -null when absent), the snapshot anchor, ``window_weight``, ``page_size``, and -zero or more append-only advance steps that the backend must fold through -its incremental path. The backend returns :class:`WindowPipelineOutputs`, -:class:`WindowPipelineRejection`, or -- only for a case outside its declared -integer domain -- :class:`WindowPipelineUnsupported`. Register it with -:func:`register_adapter` and select it by setting -``QBIT_WINDOW_PIPELINE_PARITY_ADAPTER``; selection defaults to the shipped -Python pipeline. There is deliberately no production feature switch here -- -the per-component switch belongs to the migration slice, not to the oracle. - -The corpus owns its record factory (:func:`corpus_record`) rather than -importing the sibling golden test's, so an edit there cannot move this -corpus; and literal-pinned cases freeze the input document itself alongside -the outputs, so when the fixture is regenerated the diff shows whether -inputs moved, outputs moved, or both. Without that, any input-only change -fails the suite with "regenerate" and the resulting diff cannot tell the -two apart. - -Regenerating the frozen reference is reproducible and a no-op on an -unchanged tree: - - python3 -m tests.window_pipeline_parity regenerate -""" - -from __future__ import annotations - -import hashlib -import io -import json -import os -import random -import subprocess -import sys -from dataclasses import dataclass, replace -from pathlib import Path -from typing import Callable, Protocol - -from lab.prism.bundle_compiler import ( - PRISM_SERVE_BUILDER_PROTOCOL_VERSION, - _ShareWindowSerialization, -) -from lab.prism.prism_tools import prism_tool_command -from lab.prism.share_ledger import ( - DEFAULT_INCREMENTAL_SHARE_WINDOW_PAGE_SIZE, - AcceptedShareRecord, - IncrementalShareWindow, - IncrementalWindowAdvanceStats, - IncrementalWindowFallback, -) - - -# Distinct from the sibling test's RANDOM_MASTER_SEED so the two corpora can -# never silently alias; derived per-case seeds use fixed documented offsets. -PARITY_RANDOM_MASTER_SEED = 0xB17E_5EED - -REFERENCE_SCHEMA = "qbit-prism-window-pipeline-parity-reference/v2" -CASE_INPUT_SCHEMA = "qbit-prism-window-pipeline-parity-input/v1" -REGENERATE_COMMAND = "python3 -m tests.window_pipeline_parity regenerate" -REFERENCE_FIXTURE_RELPATH = "tests/fixtures/window_pipeline_parity/reference.json" -REFERENCE_FIXTURE_PATH = ( - Path(__file__).resolve().parent / "fixtures" / "window_pipeline_parity" / "reference.json" -) - -# Cases whose canonical document fits under this bound additionally pin the -# literal bytes in the fixture, so a divergence shows *what* diverged, not -# just that something did; cases whose canonical *input* document fits under -# it pin that document too, so an input-only change is visible as such in the -# fixture diff. Larger cases are pinned by length + SHA-256, which is still -# byte-exact; every framing and encoding rule those cases exercise is also -# covered by a literal-pinned case. -LITERAL_PIN_MAX_CANONICAL_BYTES = 16_384 - -ADAPTER_ENV_VAR = "QBIT_WINDOW_PIPELINE_PARITY_ADAPTER" -DEFAULT_ADAPTER_NAME = "python" - -# The durable/public share identity, in declaration order. Deliberately -# excludes ``newly_inserted`` and ``candidate_outbox_state``: both are -# process-local append metadata (compare=False on the dataclass) and are -# outside the pipeline's byte contract. -_RECORD_INPUT_FIELDS = ( - "share_seq", - "share_id", - "miner_id", - "order_key", - "p2mr_program_hex", - "share_difficulty", - "network_difficulty", - "template_height", - "job_id", - "job_issued_at_ms", - "accepted_at_ms", - "ntime", - "credit_policy", -) - -# The implementation-neutral rejection vocabulary, one category per condition -# the shipped pipeline refuses. Categories name the *condition*, never the -# Python exception or its message: a backend in another language reports the -# same category from its own error type. The two groups mirror the two entry -# points so a reader can tell which phase refused without knowing the case. -FULL_SNAPSHOT_REJECTIONS = ( - "duplicate_share_seq", - "duplicate_share_id", - "non_positive_difficulty", - "non_positive_window_weight", - "non_positive_page_size", -) -ADVANCE_REJECTIONS = ( - "anchor_regression", - "delta_non_positive_difficulty", - "delta_ineligible_at_anchor", - "delta_repeats_eligible_share", - "delta_not_append", - "delta_order_not_increasing", -) -REJECTION_REASONS = FULL_SNAPSHOT_REJECTIONS + ADVANCE_REJECTIONS - -# How the shipped pipeline's rejections map onto the vocabulary. The shipped -# fold signals each condition only through an exception message, so the -# Python adapter classifies by exact message; a message not in this table is -# re-raised rather than guessed, so a new or reworded rejection path surfaces -# as a harness error instead of being quietly filed under a wrong category. -_REJECTION_MESSAGE_CATEGORIES = { - "full payout window contains duplicate share_seq": "duplicate_share_seq", - "full payout window contains duplicate share_id": "duplicate_share_id", - "full payout window contains non-positive difficulty": "non_positive_difficulty", - "window_weight must be positive": "non_positive_window_weight", - "page_size must be positive": "non_positive_page_size", - "snapshot anchor moved backwards": "anchor_regression", - "delta contains non-positive share difficulty": "delta_non_positive_difficulty", - "delta contains a share ineligible at the new anchor": "delta_ineligible_at_anchor", - "delta repeats a share eligible at the previous anchor": "delta_repeats_eligible_share", - "newly eligible share is not an append": "delta_not_append", - "delta share_seq order is not increasing": "delta_order_not_increasing", -} -assert set(_REJECTION_MESSAGE_CATEGORIES.values()) == set(REJECTION_REASONS) - - -def rejection_category_for_message(message: str) -> str | None: - """The vocabulary category for one shipped-pipeline rejection message. - - None for a message the table does not know, which callers must treat as - a harness fault rather than guess at. Shared with the fake serve builder - so its ``rejection`` field is classified exactly as the Python adapter - classifies. - """ - return _REJECTION_MESSAGE_CATEGORIES.get(message) - - -# --- declared integer domains ------------------------------------------------ -# -# The integer fields a record carries, in declaration order; the strings are -# outside the width contract. -_RECORD_INTEGER_FIELDS = ( - "share_seq", - "share_difficulty", - "network_difficulty", - "template_height", - "job_issued_at_ms", - "accepted_at_ms", - "ntime", -) - - -@dataclass(frozen=True) -class IntegerWidth: - """One declared integer width: an inclusive range, or unbounded.""" - - name: str - minimum: int | None - maximum: int | None - - def admits(self, value: int) -> bool: - return (self.minimum is None or value >= self.minimum) and ( - self.maximum is None or value <= self.maximum - ) - - -WIDTH_U32 = IntegerWidth("u32", 0, 2**32 - 1) -WIDTH_U64 = IntegerWidth("u64", 0, 2**64 - 1) -WIDTH_U128 = IntegerWidth("u128", 0, 2**128 - 1) -WIDTH_I64 = IntegerWidth("i64", -(2**63), 2**63 - 1) -WIDTH_UNBOUNDED = IntegerWidth("unbounded", None, None) - - -@dataclass(frozen=True) -class WindowPipelineIntegerDomain: - """The integer widths one backend declares for the pipeline's inputs. - - Every record integer, the anchors, ``window_weight`` and ``page_size`` - carry a declared width; ``difficulty_total`` bounds the fold's - accumulator -- the sum of ``share_difficulty`` over every record a case - feeds the backend (snapshot plus every delta) must fit it, which is a - conservative envelope of every partial sum the fold forms. A case with - any value outside its width is outside the domain, and only then may the - backend answer :class:`WindowPipelineUnsupported`. - """ - - share_seq: IntegerWidth - share_difficulty: IntegerWidth - network_difficulty: IntegerWidth - template_height: IntegerWidth - ntime: IntegerWidth - timestamps_ms: IntegerWidth - window_weight: IntegerWidth - page_size: IntegerWidth - difficulty_total: IntegerWidth - - def record_field_width(self, name: str) -> IntegerWidth: - if name in ("job_issued_at_ms", "accepted_at_ms"): - return self.timestamps_ms - if name not in _RECORD_INTEGER_FIELDS: - raise KeyError(f"{name!r} is not an integer record field") - return getattr(self, name) - - def exclusion(self, case: WindowPipelineCase) -> str | None: - """Why the case lies outside this domain, or None when it is inside. - - Walks the inputs in document order -- the top-level fields, the - snapshot records, then each advance's anchor and records -- and names - the first value outside its width, so the reason is stable and reads - as the field that caused it; the accumulator envelope is checked last. - """ - - def outside(label: str, value: int, width: IntegerWidth) -> str | None: - if width.admits(int(value)): - return None - return f"{label} {int(value)} is outside {width.name}" - - for label, value, width in ( - ("window_weight", case.window_weight, self.window_weight), - ("page_size", case.page_size, self.page_size), - ("snapshot anchor_job_issued_at_ms", case.anchor_job_issued_at_ms, self.timestamps_ms), - ): - problem = outside(label, value, width) - if problem is not None: - return problem - groups: list[tuple[str, int | None, tuple[AcceptedShareRecord, ...]]] = [ - ("snapshot", None, case.snapshot_records) - ] - groups.extend( - (f"advance[{index}]", step.anchor_job_issued_at_ms, step.delta_records) - for index, step in enumerate(case.advances) - ) - total = 0 - for label, anchor_ms, records in groups: - if anchor_ms is not None: - problem = outside(f"{label} anchor_job_issued_at_ms", anchor_ms, self.timestamps_ms) - if problem is not None: - return problem - for index, record in enumerate(records): - for name in _RECORD_INTEGER_FIELDS: - problem = outside( - f"{label} record[{index}].{name}", - getattr(record, name), - self.record_field_width(name), - ) - if problem is not None: - return problem - total += int(record.share_difficulty) - return outside("difficulty total over every record", total, self.difficulty_total) - - -# The shipped Python pipeline: arbitrary-precision integers throughout. -UNBOUNDED_INTEGER_DOMAIN = WindowPipelineIntegerDomain( - share_seq=WIDTH_UNBOUNDED, - share_difficulty=WIDTH_UNBOUNDED, - network_difficulty=WIDTH_UNBOUNDED, - template_height=WIDTH_UNBOUNDED, - ntime=WIDTH_UNBOUNDED, - timestamps_ms=WIDTH_UNBOUNDED, - window_weight=WIDTH_UNBOUNDED, - page_size=WIDTH_UNBOUNDED, - difficulty_total=WIDTH_UNBOUNDED, -) - -# The Rust window pipeline, mirroring the declared-width table in -# crates/qbit-prism/src/window.rs (the AcceptedShare field types plus the -# prepare_window request fields and the u128 accumulators). Changing either -# side without the other is a contract change and must show up here. -RUST_INTEGER_DOMAIN = WindowPipelineIntegerDomain( - share_seq=WIDTH_U64, - share_difficulty=WIDTH_U128, - network_difficulty=WIDTH_U128, - template_height=WIDTH_U64, - ntime=WIDTH_U32, - timestamps_ms=WIDTH_I64, - window_weight=WIDTH_U128, - page_size=WIDTH_U64, - difficulty_total=WIDTH_U128, -) - - -def corpus_record( - share_seq: int, - *, - share_difficulty: int, - job_issued_at_ms: int, - accepted_at_ms: int, - credit_policy: str | None = None, -) -> AcceptedShareRecord: - """The corpus's own deterministic record factory. - - Owned here rather than imported from the sibling golden test so that an - edit to that module's factory can never move this corpus: the frozen - reference is only as stable as the inputs that produced it, and the - fixture stores those inputs' hash (and, for small cases, the document). - The derivations deliberately match the sibling factory as of the first - freeze, so the reference needed no regeneration when ownership moved. - """ - return AcceptedShareRecord( - share_seq=share_seq, - share_id=f"share-{share_seq}", - miner_id=f"miner-{share_seq % 11}", - order_key=f"{share_seq % 11:02d}:miner-{share_seq % 11}", - p2mr_program_hex=f"{share_seq % 256:02x}" * 32, - share_difficulty=share_difficulty, - network_difficulty=1_000_000 + (share_seq % 17), - template_height=800_000 + (share_seq // 32), - job_id=f"job-{share_seq}", - job_issued_at_ms=job_issued_at_ms, - accepted_at_ms=accepted_at_ms, - ntime=1_700_000_000 + share_seq, - credit_policy=credit_policy, - ) - - -@dataclass(frozen=True) -class WindowPipelineAdvance: - """One append-only delta step folded through the incremental path.""" - - anchor_job_issued_at_ms: int - delta_records: tuple[AcceptedShareRecord, ...] - - -@dataclass(frozen=True) -class WindowPipelineCase: - """One corpus case: a full snapshot plus optional incremental advances. - - ``expected_rejection`` is the category the case was *built* to trigger - (None for a case the pipeline accepts). It is not input -- the reference - freezes what the shipped pipeline actually reports -- but the structural - test holds the two equal so a rejection case cannot quietly start passing - for a different reason, or none. - """ - - name: str - why: str - anchor_job_issued_at_ms: int - window_weight: int - page_size: int - snapshot_records: tuple[AcceptedShareRecord, ...] - advances: tuple[WindowPipelineAdvance, ...] = () - expected_rejection: str | None = None - - -@dataclass(frozen=True) -class WindowPipelineAdvanceStats: - """The bounded-work triple one advance step reports, seam-owned. - - Mirrors ``IncrementalWindowAdvanceStats`` without importing its type into - the contract. ``touched_pages`` counts pre-existing pages whose records - the advance inspected or rewrote: the last pre-existing page when the - delta was appended into it, and the pre-existing page that was partially - expired, deduplicated -- so a page both appended into and partially - expired counts once, a page appended into and then expired wholesale in - the same advance still counts (its records were rewritten), a freshly - allocated append page never counts even when partial expiry lands in it, - and pages expired wholesale without being appended into never count. - That is the shipped definition; it is derived from the work performed, - not from the retained result, which is why a re-implementation that - computes it from survivors gets the expired-append page wrong. - """ - - added_rows: int - expired_rows: int - touched_pages: int - - @classmethod - def from_shipped(cls, stats: IncrementalWindowAdvanceStats) -> WindowPipelineAdvanceStats: - return cls( - added_rows=int(stats.added_rows), - expired_rows=int(stats.expired_rows), - touched_pages=int(stats.touched_pages), - ) - - def as_document(self) -> dict[str, int]: - return { - "added_rows": self.added_rows, - "expired_rows": self.expired_rows, - "touched_pages": self.touched_pages, - } - - -@dataclass(frozen=True) -class WindowPipelineOutputs: - """The pipeline's outputs for one accepted corpus case. - - ``advance_stats`` carries one entry per advance step the case folded, in - order; it is empty for snapshot-only cases. - """ - - record_jsons: tuple[bytes, ...] - canonical_bytes: bytes - canonical_digest: str - spool_tail: bytes - advance_stats: tuple[WindowPipelineAdvanceStats, ...] = () - - -@dataclass(frozen=True) -class WindowPipelineRejection: - """The pipeline refused the case, for one of ``REJECTION_REASONS``.""" - - reason: str - - def __post_init__(self) -> None: - if self.reason not in REJECTION_REASONS: - raise ValueError( - f"unknown window-pipeline rejection reason {self.reason!r};" - f" the vocabulary is {list(REJECTION_REASONS)}" - ) - - -@dataclass(frozen=True) -class WindowPipelineUnsupported: - """The backend's declared integer domain does not include this input. - - Neither a rejection (the pipeline did not refuse the case; this backend - cannot represent it) nor a divergence (the frozen reference keeps the - shipped pipeline's result): it is the outcome the coordinator really - sees when the daemon answers ``out_of_range`` and the window is folded - in-process instead. ``reason`` is the backend's declared explanation, - carried for the report and never matched on -- whether the outcome is - legitimate is decided from the backend's declared domain alone, see - :func:`diverging_outputs`. - """ - - reason: str - - -WindowPipelineResult = ( - WindowPipelineOutputs | WindowPipelineRejection | WindowPipelineUnsupported -) - - -class WindowPipelineAdapter(Protocol): - """One backend driven over the corpus: inputs in, a result out. - - ``integer_domain`` is the domain the backend declares; a backend without - the attribute claims the unbounded domain, so any ``Unsupported`` it - returns is a divergence. - """ - - name: str - integer_domain: WindowPipelineIntegerDomain - - def run(self, case: WindowPipelineCase) -> WindowPipelineResult: ... - - -def adapter_integer_domain(adapter: object) -> WindowPipelineIntegerDomain: - """The domain a backend declares; one that declares nothing claims all of it.""" - domain = getattr(adapter, "integer_domain", UNBOUNDED_INTEGER_DOMAIN) - if not isinstance(domain, WindowPipelineIntegerDomain): - raise TypeError( - f"adapter {getattr(adapter, 'name', adapter)!r} declares an" - " integer_domain that is not a WindowPipelineIntegerDomain" - ) - return domain - - -def domain_exclusion(adapter: object, case: WindowPipelineCase) -> str | None: - """Why the case is outside the backend's declared domain, or None.""" - return adapter_integer_domain(adapter).exclusion(case) - - -def sha256_hex(data: bytes) -> str: - return hashlib.sha256(data).hexdigest() - - -def canonical_json_bytes(value: object) -> bytes: - """The pipeline's canonical encoder, byte-identical to the window pages.""" - return json.dumps(value, sort_keys=True, separators=(",", ":"), default=str).encode() - - -def record_input_json(record: AcceptedShareRecord) -> dict[str, object]: - """One record as implementation-neutral input data, every field explicit.""" - return {name: getattr(record, name) for name in _RECORD_INPUT_FIELDS} - - -def case_input_document(case: WindowPipelineCase) -> dict[str, object]: - """The full case as the JSON document a non-Python backend consumes. - - Record order is preserved verbatim: unsorted snapshot input is part of - what several cases exercise, so normalizing here would erase it. - """ - return { - "schema": CASE_INPUT_SCHEMA, - "name": case.name, - "window_weight": case.window_weight, - "page_size": case.page_size, - "snapshot": { - "anchor_job_issued_at_ms": case.anchor_job_issued_at_ms, - "records": [record_input_json(record) for record in case.snapshot_records], - }, - "advances": [ - { - "anchor_job_issued_at_ms": step.anchor_job_issued_at_ms, - "records": [record_input_json(record) for record in step.delta_records], - } - for step in case.advances - ], - } - - -def case_input_bytes(case: WindowPipelineCase) -> bytes: - return canonical_json_bytes(case_input_document(case)) - - -def case_input_sha256(case: WindowPipelineCase) -> str: - return sha256_hex(case_input_bytes(case)) - - -def final_anchor_job_issued_at_ms(case: WindowPipelineCase) -> int: - if case.advances: - return case.advances[-1].anchor_job_issued_at_ms - return case.anchor_job_issued_at_ms - - -def union_rebuild_case(case: WindowPipelineCase) -> WindowPipelineCase: - """The snapshot-only case whose bytes an incremental fold must land on.""" - union_records = case.snapshot_records + tuple( - record for step in case.advances for record in step.delta_records - ) - return WindowPipelineCase( - name=f"{case.name}-union-rebuild", - why=( - "from_full_snapshot over the union of the snapshot and every" - " advance delta, at the final anchor; the incremental path must" - " land on these exact bytes" - ), - anchor_job_issued_at_ms=final_anchor_job_issued_at_ms(case), - window_weight=case.window_weight, - page_size=case.page_size, - snapshot_records=union_records, - ) - - -def fold_case_with_stats( - case: WindowPipelineCase, -) -> tuple[IncrementalShareWindow, tuple[WindowPipelineAdvanceStats, ...]]: - """Drive the shipped fold: full snapshot, then each advance in order. - - Raises exactly what the shipped pipeline raises; :func:`run_python_pipeline` - is the seam-facing wrapper that classifies those into rejections. - """ - window = IncrementalShareWindow.from_full_snapshot( - list(case.snapshot_records), - anchor_job_issued_at_ms=case.anchor_job_issued_at_ms, - window_weight=case.window_weight, - page_size=case.page_size, - ) - stats: list[WindowPipelineAdvanceStats] = [] - for step in case.advances: - window, step_stats = window.advance( - list(step.delta_records), - anchor_job_issued_at_ms=step.anchor_job_issued_at_ms, - ) - stats.append(WindowPipelineAdvanceStats.from_shipped(step_stats)) - return window, tuple(stats) - - -def fold_case(case: WindowPipelineCase) -> IncrementalShareWindow: - """The folded window of a case the shipped pipeline accepts.""" - return fold_case_with_stats(case)[0] - - -def spool_tail_bytes(shares: list[dict[str, object]]) -> bytes: - """The exact spool payload tail the shipped serialization writes. - - Drives the real ``acquire_spooled_tail`` against an in-memory spool file - so the pinned bytes are the shipped write path's output, framing - included, not a reconstruction of it. - """ - serialization = _ShareWindowSerialization( - key=("window-pipeline-parity", len(shares), 0), - share_count=len(shares), - share_snapshot_sha256="window-pipeline-parity", - _spool_factory=io.BytesIO, - ) - leased = serialization.acquire_spooled_tail(shares) - if leased is None: - raise RuntimeError("in-memory spool lease unexpectedly unavailable") - spool, size = leased - spool.seek(0) - tail = spool.read() - if len(tail) != size: - raise RuntimeError(f"spool reported {size} bytes but holds {len(tail)}") - serialization.release_spooled_tail() - serialization.retire_spool() - return tail - - -def outputs_from_window( - window: IncrementalShareWindow, - advance_stats: tuple[WindowPipelineAdvanceStats, ...] = (), -) -> WindowPipelineOutputs: - """Capture the byte outputs from one folded window.""" - json_records = window.json_records() - record_jsons = tuple(canonical_json_bytes(record) for record in json_records) - fragments = [ - page.canonical_json_items - for page in json_records.pages - if page.canonical_json_items - ] - canonical_bytes = b"[" + b",".join(fragments) + b"]" - return WindowPipelineOutputs( - record_jsons=record_jsons, - canonical_bytes=canonical_bytes, - canonical_digest=json_records.canonical_json_sha256(), - spool_tail=spool_tail_bytes(list(json_records)), - advance_stats=advance_stats, - ) - - -def run_python_pipeline(case: WindowPipelineCase) -> WindowPipelineResult: - """The shipped pipeline over one case: outputs, or its rejection classified. - - Only the fold is guarded: a failure while capturing outputs (spool, JSON) - is a harness fault and propagates, never a rejection. - """ - try: - window, advance_stats = fold_case_with_stats(case) - except (ValueError, IncrementalWindowFallback) as exc: - category = rejection_category_for_message(str(exc)) - if category is None: - raise - return WindowPipelineRejection(category) - return outputs_from_window(window, advance_stats) - - -def record_stream_bytes(record_jsons: tuple[bytes, ...]) -> bytes: - # Injective: canonical encodings ASCII-escape every control character, so - # a raw newline can never occur inside one record's bytes. - return b"\n".join(record_jsons) - - -@dataclass(frozen=True) -class PythonWindowPipelineAdapter: - """The shipped Python pipeline, registered as the default backend.""" - - name: str = "python" - integer_domain: WindowPipelineIntegerDomain = UNBOUNDED_INTEGER_DOMAIN - - def run(self, case: WindowPipelineCase) -> WindowPipelineResult: - return run_python_pipeline(case) - - -def _split_record_fragments(items: bytes) -> tuple[bytes, ...]: - """Split one canonical items stream into its per-record byte spans. - - The spans come straight from the backend's bytes (``raw_decode`` only - reports where each JSON object ends), so a backend encoding divergence is - still visible in the returned fragments rather than being papered over by - a re-encode. - """ - if not items: - return () - text = items.decode("ascii") - decoder = json.JSONDecoder() - fragments: list[bytes] = [] - position = 0 - while position < len(text): - _, end = decoder.raw_decode(text, position) - fragments.append(text[position:end].encode("ascii")) - if end < len(text) and text[end] != ",": - raise ValueError( - f"canonical items stream has {text[end]!r} where a record" - " separator was expected" - ) - position = end + 1 - return tuple(fragments) - - -@dataclass(frozen=True) -class RustDaemonWindowPipelineAdapter: - """The Rust --serve daemon's prepare_window pipeline as a backend. - - Drives the real daemon binary over its JSONL protocol: one ``full`` - preparation for the snapshot, then one ``advance`` per delta step, holding - the canonical items stream exactly the way the coordinator's opaque - mirror does (drop the reported prefix, append the returned suffix) and - taking each advance's stats triple from its envelope. The three byte - outputs are taken from the daemon's bytes; only the spool tail runs the - shipped Python serialization over the lazily parsed records, which is - precisely the production fallback path under the Rust switch. - - Non-success envelopes are classified on their structure, never on the - diagnostic ``error`` text: ``fold_invalid`` and ``fallback`` carry the - daemon's ``rejection`` category, which is the oracle's own vocabulary and - is filed as :class:`WindowPipelineRejection` (an unknown category is a - harness error, not a guess); ``out_of_range`` -- an integer or difficulty - total outside the widths this adapter declares as ``RUST_INTEGER_DOMAIN`` - -- is :class:`WindowPipelineUnsupported`. Anything else is a harness - error, exactly as the coordinator treats it as a daemon anomaly. - - Selected with ``QBIT_WINDOW_PIPELINE_PARITY_ADAPTER=rust-daemon``; the - daemon binary resolves through ``prism_tool_command`` (``cargo run`` - unless ``PRISM_TOOL_BIN_DIR`` provides a prebuilt binary). - """ - - name: str = "rust-daemon" - integer_domain: WindowPipelineIntegerDomain = RUST_INTEGER_DOMAIN - - @staticmethod - def daemon_command() -> list[str]: - """Return the real deployment tool invocation used by this adapter.""" - return prism_tool_command("qbit-prism-build-audit-bundle") + [ - "--serve", - "--signing-key-seed-hex", - "42" * 32, - "--ledger-signing-key-seed-hex", - "43" * 32, - ] - - @staticmethod - def _wire_record(record: AcceptedShareRecord) -> dict[str, object]: - return record_input_json(record) - - @staticmethod - def _declined(envelope: dict[str, object]) -> WindowPipelineRejection | WindowPipelineUnsupported: - """Classify one non-``ok`` prepare_window envelope by its structure.""" - if envelope.get("request") != "prepare_window": - raise RuntimeError(f"prepare_window failed: {envelope}") - if envelope.get("out_of_range") is True: - return WindowPipelineUnsupported( - str(envelope.get("error") or "value outside the daemon's declared widths") - ) - if envelope.get("fold_invalid") is True or envelope.get("fallback") is True: - category = envelope.get("rejection") - if not isinstance(category, str): - raise RuntimeError( - f"daemon rejection carries no machine-readable category: {envelope}" - ) - # A category outside the vocabulary raises ValueError here: a - # harness error, never a quiet filing under the wrong reason. - return WindowPipelineRejection(category) - raise RuntimeError(f"prepare_window failed: {envelope}") - - def run(self, case: WindowPipelineCase) -> WindowPipelineResult: - process = subprocess.Popen( - self.daemon_command(), - stdin=subprocess.PIPE, - stdout=subprocess.PIPE, - ) - advance_stats: list[WindowPipelineAdvanceStats] = [] - try: - assert process.stdin is not None and process.stdout is not None - handshake = json.loads(process.stdout.readline()) - expected_handshake = { - "event": "handshake", - "tool": "qbit-prism-build-audit-bundle", - "protocol": PRISM_SERVE_BUILDER_PROTOCOL_VERSION, - } - announced_handshake = { - key: handshake.get(key) for key in expected_handshake - } - if announced_handshake != expected_handshake: - raise RuntimeError( - f"daemon announced handshake {announced_handshake!r}," - f" expected {expected_handshake!r}" - ) - - def exchange(request: dict[str, object]) -> tuple[dict[str, object], bytes | None]: - """One round trip; the payload is None when the daemon declined.""" - assert process.stdin is not None and process.stdout is not None - process.stdin.write( - json.dumps(request, separators=(",", ":")).encode("ascii") + b"\n" - ) - process.stdin.flush() - envelope = json.loads(process.stdout.readline()) - if envelope.get("ok") is not True: - return envelope, None - length_key = ( - "window_items_len" - if "window_items_len" in envelope - else "appended_items_len" - ) - payload = process.stdout.read(int(envelope[length_key])) - process.stdout.read(1) # the terminating newline - return envelope, payload - - envelope, items = exchange( - { - "request": "prepare_window", - "mode": "full", - "append_invalidation_epoch": 0, - "anchor_job_issued_at_ms": case.anchor_job_issued_at_ms, - "window_weight": case.window_weight, - "page_size": case.page_size, - "records": [ - self._wire_record(record) for record in case.snapshot_records - ], - } - ) - if items is None: - return self._declined(envelope) - for step in case.advances: - envelope, appended = exchange( - { - "request": "prepare_window", - "mode": "advance", - "append_invalidation_epoch": 0, - "anchor_job_issued_at_ms": step.anchor_job_issued_at_ms, - "base_digest": envelope["share_snapshot_sha256"], - "records": [ - self._wire_record(record) for record in step.delta_records - ], - } - ) - if appended is None: - return self._declined(envelope) - items = items[int(envelope["retained_drop_bytes"]) :] + appended - advance_stats.append( - WindowPipelineAdvanceStats( - added_rows=int(envelope["added_rows"]), - expired_rows=int(envelope["expired_rows"]), - touched_pages=int(envelope["touched_pages"]), - ) - ) - finally: - if process.stdin is not None: - process.stdin.close() - process.wait(timeout=30.0) - if process.stdout is not None: - process.stdout.close() - - record_jsons = _split_record_fragments(items) - if len(record_jsons) != int(envelope["record_count"]): - raise RuntimeError( - f"daemon reported {envelope['record_count']} records but the" - f" items stream holds {len(record_jsons)}" - ) - shares = [json.loads(fragment) for fragment in record_jsons] - return WindowPipelineOutputs( - record_jsons=record_jsons, - canonical_bytes=b"[" + items + b"]", - canonical_digest=str(envelope["share_snapshot_sha256"]), - spool_tail=spool_tail_bytes(shares), - advance_stats=tuple(advance_stats), - ) - - -_ADAPTER_FACTORIES: dict[str, Callable[[], WindowPipelineAdapter]] = { - DEFAULT_ADAPTER_NAME: PythonWindowPipelineAdapter, - "rust-daemon": RustDaemonWindowPipelineAdapter, -} - - -def register_adapter(name: str, factory: Callable[[], WindowPipelineAdapter]) -> None: - if name in _ADAPTER_FACTORIES: - raise ValueError(f"window-pipeline parity adapter {name!r} is already registered") - _ADAPTER_FACTORIES[name] = factory - - -def resolve_adapter(name: str | None = None) -> WindowPipelineAdapter: - """Resolve one backend; defaults to the shipped Python pipeline.""" - resolved = name or os.environ.get(ADAPTER_ENV_VAR) or DEFAULT_ADAPTER_NAME - factory = _ADAPTER_FACTORIES.get(resolved) - if factory is None: - raise ValueError( - f"unknown window-pipeline parity adapter {resolved!r};" - f" registered adapters: {sorted(_ADAPTER_FACTORIES)}" - ) - return factory() - - -# --- corpus ----------------------------------------------------------------- -# -# Every case is derived from an invariant of ``from_full_snapshot``/``advance`` -# or of the byte encodings, and says which. Seeded and deterministic: the -# builders below always produce the same records, so the frozen reference is -# reproducible from an unchanged tree. - - -def _empty_window_case() -> WindowPipelineCase: - return WindowPipelineCase( - name="empty-window", - why="zero eligible records: [] framing, digest of [], empty compact arrays", - anchor_job_issued_at_ms=1_000, - window_weight=10, - page_size=DEFAULT_INCREMENTAL_SHARE_WINDOW_PAGE_SIZE, - snapshot_records=(), - ) - - -def _single_record_case() -> WindowPipelineCase: - record = corpus_record( - 1, - share_difficulty=5, - job_issued_at_ms=1_999, - accepted_at_ms=2_000, - ) - return WindowPipelineCase( - name="single-record", - why=( - "one record with credit_policy None: to_prism_json omits the key" - " while the compact share tuple carries an explicit null -- the" - " null-vs-absent divergence surface" - ), - anchor_job_issued_at_ms=2_000, - window_weight=10, - page_size=DEFAULT_INCREMENTAL_SHARE_WINDOW_PAGE_SIZE, - snapshot_records=(record,), - ) - - -def _crossing_row_retained_case() -> WindowPipelineCase: - anchor_ms = 10_000 - records = ( - corpus_record(1, share_difficulty=2, job_issued_at_ms=anchor_ms - 4, accepted_at_ms=anchor_ms - 3), - corpus_record( - 2, - share_difficulty=3, - job_issued_at_ms=anchor_ms - 2, - accepted_at_ms=anchor_ms - 1, - credit_policy="stale-grace", - ), - corpus_record(3, share_difficulty=8, job_issued_at_ms=anchor_ms, accepted_at_ms=anchor_ms), - ) - return WindowPipelineCase( - name="crossing-row-retained", - why=( - "difficulties (2,3,8) at weight 10: the final whole share crossing" - " window_weight (seq 2) is deliberately retained, total 11 > 10;" - " an implementation that drops it diverges here" - ), - anchor_job_issued_at_ms=anchor_ms, - window_weight=10, - page_size=DEFAULT_INCREMENTAL_SHARE_WINDOW_PAGE_SIZE, - snapshot_records=records, - ) - - -def _crossing_row_exact_fit_case() -> WindowPipelineCase: - anchor_ms = 10_000 - records = ( - corpus_record(1, share_difficulty=2, job_issued_at_ms=anchor_ms - 4, accepted_at_ms=anchor_ms - 3), - corpus_record(2, share_difficulty=3, job_issued_at_ms=anchor_ms - 2, accepted_at_ms=anchor_ms - 1), - corpus_record(3, share_difficulty=7, job_issued_at_ms=anchor_ms, accepted_at_ms=anchor_ms), - ) - return WindowPipelineCase( - name="crossing-row-exact-fit", - why=( - "difficulties (2,3,7) at weight 10: cumulative weight lands" - " exactly on window_weight, so seq 1 is dropped and there is no" - " crossing row; the pair with crossing-row-retained separates the" - " >= cutoff from a wrong > cutoff" - ), - anchor_job_issued_at_ms=anchor_ms, - window_weight=10, - page_size=DEFAULT_INCREMENTAL_SHARE_WINDOW_PAGE_SIZE, - snapshot_records=records, - ) - - -def _eligibility_filtering_case() -> WindowPipelineCase: - anchor_ms = 30_000 - records = ( - corpus_record(1, share_difficulty=4, job_issued_at_ms=anchor_ms - 2, accepted_at_ms=anchor_ms - 1), - corpus_record(2, share_difficulty=5, job_issued_at_ms=anchor_ms + 1, accepted_at_ms=anchor_ms - 1), - corpus_record(3, share_difficulty=6, job_issued_at_ms=anchor_ms - 1, accepted_at_ms=anchor_ms + 1), - corpus_record(4, share_difficulty=7, job_issued_at_ms=anchor_ms + 2, accepted_at_ms=anchor_ms + 3), - corpus_record(5, share_difficulty=8, job_issued_at_ms=anchor_ms, accepted_at_ms=anchor_ms), - ) - return WindowPipelineCase( - name="eligibility-filtering", - why=( - "one record excluded by job_issued_at_ms alone, one by" - " accepted_at_ms alone, one by both; seq 5 sits exactly on the" - " anchor, pinning the inclusive <= comparisons" - ), - anchor_job_issued_at_ms=anchor_ms, - window_weight=100, - page_size=DEFAULT_INCREMENTAL_SHARE_WINDOW_PAGE_SIZE, - snapshot_records=records, - ) - - -def _unsorted_input_case() -> WindowPipelineCase: - rng = random.Random(PARITY_RANDOM_MASTER_SEED + 1) - anchor_ms = 40_000 - share_seqs = rng.sample(range(1, 100), 9) - records = [ - corpus_record( - share_seq, - share_difficulty=rng.choice((1, 2, 3, 7, 19)), - job_issued_at_ms=anchor_ms - rng.randint(0, 30), - accepted_at_ms=anchor_ms - rng.randint(0, 30), - credit_policy="stale-grace" if rng.randrange(4) == 0 else None, - ) - for share_seq in share_seqs - ] - return WindowPipelineCase( - name="unsorted-input", - why=( - "snapshot input arrives in non-ascending share_seq order with" - " non-contiguous seqs; the fold's sorted() is load-bearing and the" - " output ordering is pinned" - ), - anchor_job_issued_at_ms=anchor_ms, - window_weight=1_000_000, - page_size=DEFAULT_INCREMENTAL_SHARE_WINDOW_PAGE_SIZE, - snapshot_records=tuple(records), - ) - - -def _identity_reuse_case() -> WindowPipelineCase: - anchor_ms = 50_000 - base = [ - corpus_record( - share_seq, - share_difficulty=share_seq, - job_issued_at_ms=anchor_ms - share_seq, - accepted_at_ms=anchor_ms - share_seq, - credit_policy="stale-grace" if share_seq in (2, 5) else None, - ) - for share_seq in range(1, 7) - ] - # Force full (miner_id, order_key, p2mr_program_hex) identity collisions: - # the factory varies p2mr with share_seq, so collisions must be explicit. - base[3] = replace( - base[3], - miner_id=base[0].miner_id, - order_key=base[0].order_key, - p2mr_program_hex=base[0].p2mr_program_hex, - ) - base[5] = replace( - base[5], - miner_id=base[1].miner_id, - order_key=base[1].order_key, - p2mr_program_hex=base[1].p2mr_program_hex, - ) - return WindowPipelineCase( - name="credit-policy-identity-reuse", - why=( - "duplicate (miner_id, order_key, p2mr_program_hex) triples pin" - " identity deduplication and identity_index reuse in the compact" - " spool fragments, alongside mixed credit_policy values" - ), - anchor_job_issued_at_ms=anchor_ms, - window_weight=1_000_000, - page_size=DEFAULT_INCREMENTAL_SHARE_WINDOW_PAGE_SIZE, - snapshot_records=tuple(base), - ) - - -def _non_ascii_strings_case() -> WindowPipelineCase: - # share_id embeds the raw Stratum username, which survives payout-address - # fallback un-sanitized, and the ledger validates none of the string - # fields, so non-ASCII (including non-BMP), quotes, backslashes and - # control characters can all reach the durable record. json.dumps here - # runs with ensure_ascii=True, so every such character is \\u-escaped - # (non-BMP as surrogate pairs) -- a default a different JSON encoder is - # likely to get wrong. The fourth record widens the alphabet to the - # characters that separate Python's encoder from a "below 0x20 plus quote - # and backslash" re-encoder: DEL (0x7f, escaped by Python, raw in - # serde_json and most hand-rolled encoders), solidus (raw in Python, - # escaped by some encoders), the short escapes \\r \\b \\f, U+001F (the - # last short-less control), U+2028/U+2029 (escaped by Python, raw in - # encoders that only escape JavaScript-unsafe text), and space and - # uppercase, which must pass through untouched. They sit in the identity - # fields too so the spool's compact identities carry the same escapes. - anchor_ms = 60_000 - records = ( - replace( - corpus_record(1, share_difficulty=3, job_issued_at_ms=anchor_ms - 3, accepted_at_ms=anchor_ms - 2), - share_id="minér-é.workér:0a1b", - miner_id="minér-é", - order_key="00:minér-é", - ), - replace( - corpus_record( - 2, - share_difficulty=4, - job_issued_at_ms=anchor_ms - 2, - accepted_at_ms=anchor_ms - 1, - credit_policy="stale-grace", - ), - share_id='\U0001f680-share-"quoted"\\back\\slash', - job_id="job-\t\n\x01", - ), - corpus_record(3, share_difficulty=5, job_issued_at_ms=anchor_ms - 1, accepted_at_ms=anchor_ms), - replace( - corpus_record(4, share_difficulty=6, job_issued_at_ms=anchor_ms - 1, accepted_at_ms=anchor_ms), - share_id="MINER-A/worker 1:\x7f\r\b\x0c\x1f\u2028\u2029", - miner_id="MINER-A/worker 1\x7f", - order_key="04:MINER-A/worker 1\x7f", - ), - ) - return WindowPipelineCase( - name="non-ascii-strings", - why=( - "string fields carrying non-ASCII, a non-BMP code point, quotes," - " backslashes, control characters, DEL, solidus, CR/BS/FF, U+001F," - " U+2028/U+2029, space and uppercase: pins the ensure_ascii escape" - " rules -- surrogate pairs, \\u007f for DEL, raw solidus, short" - " escapes, and pass-through of printable ASCII" - ), - anchor_job_issued_at_ms=anchor_ms, - window_weight=1_000_000, - page_size=DEFAULT_INCREMENTAL_SHARE_WINDOW_PAGE_SIZE, - snapshot_records=records, - ) - - -def _page_boundary_case(retained_count: int, seed_offset: int) -> WindowPipelineCase: - rng = random.Random(PARITY_RANDOM_MASTER_SEED + seed_offset) - anchor_ms = 70_000 + retained_count - records = tuple( - corpus_record( - share_seq, - share_difficulty=rng.choice((1, 2, 3, 7, 19)), - job_issued_at_ms=anchor_ms - rng.randint(0, 30), - accepted_at_ms=anchor_ms - rng.randint(0, 30), - credit_policy="stale-grace" if rng.randrange(4) == 0 else None, - ) - for share_seq in range(1, retained_count + 1) - ) - return WindowPipelineCase( - name=f"page-boundary-{retained_count}", - why=( - f"exactly {retained_count} retained records around the default" - f" page size {DEFAULT_INCREMENTAL_SHARE_WINDOW_PAGE_SIZE}: pins" - " page splitting and the digest's page-fragment framing at the" - " boundary" - ), - anchor_job_issued_at_ms=anchor_ms, - window_weight=10**9, - page_size=DEFAULT_INCREMENTAL_SHARE_WINDOW_PAGE_SIZE, - snapshot_records=records, - ) - - -def _multi_page_case() -> WindowPipelineCase: - rng = random.Random(PARITY_RANDOM_MASTER_SEED + 11) - anchor_ms = 90_000 - record_count = 1_800 - records = [ - corpus_record( - share_seq, - share_difficulty=rng.choice((1, 2, 3, 7, 19)), - job_issued_at_ms=anchor_ms - rng.randint(0, 30), - accepted_at_ms=anchor_ms - rng.randint(0, 30), - credit_policy="stale-grace" if rng.randrange(4) == 0 else None, - ) - for share_seq in range(1, record_count + 1) - ] - # Weight chosen so the newest 1699 shares miss the weight by one and the - # 1700th (by seq: 101) is the retained crossing row, landing the cutoff in - # a page interior across four pages. - window_weight = sum(record.share_difficulty for record in records[101:]) + 1 - shuffled = list(records) - rng.shuffle(shuffled) - return WindowPipelineCase( - name="multi-page-interior-cutoff", - why=( - "1800 unsorted input records with the retention cutoff landing" - " inside a page interior: pins multi-page fragment concatenation" - " and a crossing row far from page edges" - ), - anchor_job_issued_at_ms=anchor_ms, - window_weight=window_weight, - page_size=DEFAULT_INCREMENTAL_SHARE_WINDOW_PAGE_SIZE, - snapshot_records=tuple(shuffled), - ) - - -def _incremental_small_case() -> WindowPipelineCase: - anchor_ms = 80_000 - difficulties = {1: 5, 2: 1, 3: 1, 4: 2, 5: 1, 6: 3, 7: 1} - snapshot = [ - corpus_record( - share_seq, - share_difficulty=difficulty, - job_issued_at_ms=anchor_ms - share_seq, - accepted_at_ms=anchor_ms - share_seq, - credit_policy="stale-grace" if share_seq == 6 else None, - ) - for share_seq, difficulty in difficulties.items() - ] - # Never eligible at any anchor this case reaches: the incremental path - # never sees it in a delta, and the union rebuild must exclude it too. - snapshot.append( - corpus_record(8, share_difficulty=100, job_issued_at_ms=10**9, accepted_at_ms=10**9) - ) - first_advance = WindowPipelineAdvance( - anchor_job_issued_at_ms=anchor_ms + 1, - delta_records=( - corpus_record(9, share_difficulty=2, job_issued_at_ms=anchor_ms + 1, accepted_at_ms=anchor_ms - 1), - corpus_record(10, share_difficulty=4, job_issued_at_ms=anchor_ms - 10, accepted_at_ms=anchor_ms + 1), - ), - ) - second_advance = WindowPipelineAdvance( - anchor_job_issued_at_ms=anchor_ms + 10, - delta_records=( - corpus_record(11, share_difficulty=1, job_issued_at_ms=anchor_ms + 5, accepted_at_ms=anchor_ms + 2), - corpus_record( - 12, - share_difficulty=1, - job_issued_at_ms=anchor_ms + 10, - accepted_at_ms=anchor_ms - 5, - credit_policy="stale-grace", - ), - corpus_record(13, share_difficulty=1, job_issued_at_ms=anchor_ms, accepted_at_ms=anchor_ms + 10), - corpus_record( - 14, - share_difficulty=1, - job_issued_at_ms=anchor_ms + 9, - accepted_at_ms=anchor_ms + 9, - credit_policy="stale-grace", - ), - ), - ) - return WindowPipelineCase( - name="incremental-two-advances", - why=( - "two advance() steps over a page_size-3 window: append into a" - " partially filled page, spill into new pages, expire a whole head" - " page and partially expire the next; the folded bytes must equal" - " a full rebuild over the union" - ), - anchor_job_issued_at_ms=anchor_ms, - window_weight=9, - page_size=3, - snapshot_records=tuple(snapshot), - advances=(first_advance, second_advance), - ) - - -def _bulk_seeded_case() -> WindowPipelineCase: - rng = random.Random(PARITY_RANDOM_MASTER_SEED + 12) - anchor_ms = 1_000_000 - advance_anchor_ms = anchor_ms + 10 - snapshot: list[AcceptedShareRecord] = [] - for share_seq in range(1, 4_001): - if rng.random() < 0.1: - # Ineligible at the snapshot anchor and at every later anchor in - # this case, since the incremental path never re-delivers it. - lateness = rng.choice(("job", "accepted", "both")) - job_ms = ( - anchor_ms + rng.randint(11, 500) - if lateness in ("job", "both") - else anchor_ms - rng.randint(0, 30) - ) - accepted_ms = ( - anchor_ms + rng.randint(11, 500) - if lateness in ("accepted", "both") - else anchor_ms - rng.randint(0, 30) - ) - else: - job_ms = anchor_ms - rng.randint(0, 30) - accepted_ms = anchor_ms - rng.randint(0, 30) - snapshot.append( - corpus_record( - share_seq, - share_difficulty=rng.choice((1, 2, 3, 7, 19, 101, 1_000)), - job_issued_at_ms=job_ms, - accepted_at_ms=accepted_ms, - credit_policy="stale-grace" if rng.randrange(5) == 0 else None, - ) - ) - rng.shuffle(snapshot) - delta: list[AcceptedShareRecord] = [] - for share_seq in range(4_001, 4_701): - if rng.randrange(2): - job_ms = rng.randint(anchor_ms + 1, advance_anchor_ms) - accepted_ms = advance_anchor_ms - rng.randint(0, 40) - else: - job_ms = advance_anchor_ms - rng.randint(0, 40) - accepted_ms = rng.randint(anchor_ms + 1, advance_anchor_ms) - delta.append( - corpus_record( - share_seq, - share_difficulty=rng.choice((1, 2, 3, 7, 19, 101, 1_000)), - job_issued_at_ms=job_ms, - accepted_at_ms=accepted_ms, - credit_policy="stale-grace" if rng.randrange(5) == 0 else None, - ) - ) - return WindowPipelineCase( - name="bulk-seeded", - why=( - "4000 seeded snapshot records (roughly a tenth ineligible) plus a" - " 700-record advance: representative page-fragment concatenation," - " identity deduplication at natural factory collisions, and head" - " expiry at scale" - ), - anchor_job_issued_at_ms=anchor_ms, - window_weight=450_000, - page_size=DEFAULT_INCREMENTAL_SHARE_WINDOW_PAGE_SIZE, - snapshot_records=tuple(snapshot), - advances=( - WindowPipelineAdvance( - anchor_job_issued_at_ms=advance_anchor_ms, - delta_records=tuple(delta), - ), - ), - ) - - -# A realistic production anchor: epoch milliseconds in 2025, above 2^40. The -# previous cases all sit below 2^31, which is exactly why a 32-bit backend -# could pass them; every width case below runs at this scale. -_REALISTIC_ANCHOR_MS = 1_755_000_000_000 - - -def _wide_integers_case() -> WindowPipelineCase: - anchor_ms = _REALISTIC_ANCHOR_MS - # Every value sits far above the 2^31 / 2^53 / 2^64 boundaries at which a - # wrapping or saturating 32/53/64-bit backend breaks, yet inside every - # declared Rust width (share_seq and template_height u64, difficulties - # and window_weight u128, ntime u32), so both backends must produce these - # exact bytes. Retention runs newest-first until the accumulated weight - # reaches window_weight. The three newest (2^53+1, 2^63, 2^64+1) sum to - # 2^64+2^63+2^53+2, below window_weight only because of its 2^127 term, - # so the oldest (2^127+3) is the retained crossing row and all four - # survive: a backend that wraps or saturates at 64 bits (or truncates - # window_weight to 64 bits) stops after three, and one that clamps the - # inputs renders different digits even where it keeps the count. The - # difficulty total over every record stays below 2^128, inside the u128 - # accumulator, so the case is inside the declared domain entirely. - records = ( - replace( - corpus_record(1, share_difficulty=2**127 + 3, job_issued_at_ms=anchor_ms - 3, accepted_at_ms=anchor_ms - 2), - network_difficulty=2**128 - 1, - template_height=2**62, - ntime=2**32 - 1, - ), - replace( - corpus_record(2**31, share_difficulty=2**64 + 1, job_issued_at_ms=anchor_ms - 2, accepted_at_ms=anchor_ms - 1), - network_difficulty=2**64 + 1, - template_height=2**31 + 1, - ntime=2**31, - ), - replace( - corpus_record(2**32 + 5, share_difficulty=2**63, job_issued_at_ms=anchor_ms - 1, accepted_at_ms=anchor_ms - 1), - network_difficulty=2**64 - 1, - template_height=0, - ntime=0, - ), - # share_seq at u64's maximum (the factory's derived template_height - # lands near 2^59); ntime must be set explicitly because the - # factory's default would add the seq to it. - replace( - corpus_record(2**64 - 1, share_difficulty=2**53 + 1, job_issued_at_ms=anchor_ms, accepted_at_ms=anchor_ms), - network_difficulty=2**53 + 1, - ntime=2**31 + 1, - ), - # Excluded by a one-millisecond margin at 2^40 scale: a backend - # comparing truncated or float-rounded timestamps includes it. - corpus_record(7, share_difficulty=1, job_issued_at_ms=anchor_ms + 1, accepted_at_ms=anchor_ms), - ) - return WindowPipelineCase( - name="wide-integers", - why=( - "every integer field above 2^32 at a realistic epoch-ms anchor," - " all inside the declared Rust widths: share_difficulty across" - " 2^53+1, 2^63, 2^64+1 and 2^127+3, network_difficulty up to" - " 2^128-1, template_height 2^31+1, 2^62 and near 2^59, ntime" - " 2^31 to 2^32-1, share_seq up to 2^64-1, a zero" - " template_height/ntime, and window_weight 2^127+2^64+2^63+4 so" - " the retention cutoff depends on exact arithmetic above 2^64;" - " pins full-digit rendering and non-wrapping, non-saturating," - " non-float accumulation, with full parity enforced on every" - " backend" - ), - anchor_job_issued_at_ms=anchor_ms, - window_weight=2**127 + 2**64 + 2**63 + 4, - page_size=DEFAULT_INCREMENTAL_SHARE_WINDOW_PAGE_SIZE, - snapshot_records=records, - ) - - -def _difficulty_beyond_u128_case() -> WindowPipelineCase: - anchor_ms = _REALISTIC_ANCHOR_MS - # Newest-first: 2^63, then 2^64-1, then 2^128+1 crosses window_weight and - # is retained; the oldest (2^127) is dropped. The retained total is above - # 2^128 and so is window_weight itself, so neither the inputs nor the - # accumulator fit a 128-bit integer: a backend that clamps at u128::MAX - # renders different digits for the crossing row and for the numeric(78,0) - # maximum network_difficulty even though its retained count agrees. - records = ( - replace( - corpus_record(1, share_difficulty=2**127, job_issued_at_ms=anchor_ms - 3, accepted_at_ms=anchor_ms - 3), - network_difficulty=2**128 - 1, - ), - replace( - corpus_record(2, share_difficulty=2**128 + 1, job_issued_at_ms=anchor_ms - 2, accepted_at_ms=anchor_ms - 2), - network_difficulty=10**78 - 1, - ), - replace( - corpus_record(3, share_difficulty=2**64 - 1, job_issued_at_ms=anchor_ms - 1, accepted_at_ms=anchor_ms - 1), - network_difficulty=2**128, - ), - replace( - corpus_record(4, share_difficulty=2**63, job_issued_at_ms=anchor_ms, accepted_at_ms=anchor_ms), - network_difficulty=2**128 - 1, - ), - ) - return WindowPipelineCase( - name="difficulty-beyond-u128", - why=( - "numeric(78,0) width taken literally: share_difficulty 2^128+1 as" - " the retained crossing row, network_difficulty at 2^128-1, 2^128" - " and 10^78-1, window_weight and the retained total above 2^128" - " -- outside the declared u128 widths, so a backend declaring" - " them answers Unsupported here while the frozen bytes stay" - " Python's; one that clamps at u128::MAX, saturating or not," - " renders different digits and fails -- see the module docstring" - ), - anchor_job_issued_at_ms=anchor_ms, - window_weight=2**128 + 2**64, - page_size=DEFAULT_INCREMENTAL_SHARE_WINDOW_PAGE_SIZE, - snapshot_records=records, - ) - - -def _ntime_beyond_u32_case() -> WindowPipelineCase: - anchor_ms = _REALISTIC_ANCHOR_MS - # ntime is bigint in the ledger but a 32-bit header field and u32 in the - # declared Rust domain: 2^32 (one above the width) and bigint's maximum - # both lie outside it while every other field is ordinary, so this is - # the one width the case exercises. - records = ( - replace( - corpus_record(1, share_difficulty=3, job_issued_at_ms=anchor_ms - 2, accepted_at_ms=anchor_ms - 1), - ntime=2**32, - ), - replace( - corpus_record( - 2, - share_difficulty=4, - job_issued_at_ms=anchor_ms - 1, - accepted_at_ms=anchor_ms - 1, - credit_policy="stale-grace", - ), - ntime=2**63 - 1, - ), - corpus_record(3, share_difficulty=5, job_issued_at_ms=anchor_ms, accepted_at_ms=anchor_ms), - ) - return WindowPipelineCase( - name="ntime-beyond-u32", - why=( - "ntime at 2^32 and 2^63-1 (bigint's maximum) on otherwise" - " ordinary records at a realistic anchor: outside the declared" - " u32 width alone, so a backend declaring it answers Unsupported" - " here while the frozen bytes stay Python's full-digit rendering;" - " one that truncates ntime to 32 bits renders different digits" - " and fails" - ), - anchor_job_issued_at_ms=anchor_ms, - window_weight=1_000, - page_size=DEFAULT_INCREMENTAL_SHARE_WINDOW_PAGE_SIZE, - snapshot_records=records, - ) - - -def _advance_exact_fit_expiry_case() -> WindowPipelineCase: - anchor_ms = 100_000 - snapshot = tuple( - corpus_record(share_seq, share_difficulty=4, job_issued_at_ms=anchor_ms - share_seq, accepted_at_ms=anchor_ms - share_seq) - for share_seq in (1, 2, 3) - ) - return WindowPipelineCase( - name="advance-exact-fit-expiry", - why=( - "three records of difficulty 4 at weight 10 (all retained, total" - " 12) plus one delta of difficulty 2: 14-4 = 10 >= 10 expires the" - " head row exactly at the weight, retaining seqs 2..4 at total 10;" - " an advance-side expiry using > instead of >= retains all four." - " The crossing-row-exact-fit sibling pins the snapshot-side twin" - " of this comparison; the advance loops are separate code in any" - " re-implementation" - ), - anchor_job_issued_at_ms=anchor_ms, - window_weight=10, - page_size=DEFAULT_INCREMENTAL_SHARE_WINDOW_PAGE_SIZE, - snapshot_records=snapshot, - advances=( - WindowPipelineAdvance( - anchor_job_issued_at_ms=anchor_ms + 1, - delta_records=( - corpus_record(4, share_difficulty=2, job_issued_at_ms=anchor_ms + 1, accepted_at_ms=anchor_ms + 1), - ), - ), - ), - ) - - -def _advance_delta_exceeds_window_case() -> WindowPipelineCase: - anchor_ms = 110_000 - snapshot = tuple( - corpus_record(share_seq, share_difficulty=3, job_issued_at_ms=anchor_ms - share_seq, accepted_at_ms=anchor_ms - share_seq) - for share_seq in (1, 2, 3) - ) - delta = tuple( - corpus_record(share_seq, share_difficulty=5, job_issued_at_ms=anchor_ms + 1, accepted_at_ms=anchor_ms + 1) - for share_seq in (4, 5, 6, 7, 8) - ) - return WindowPipelineCase( - name="advance-delta-exceeds-window", - why=( - "weight 8, page_size 2, three retained records of difficulty 3 in" - " pages [1,2],[3], then one advance of five records of difficulty" - " 5 (25 alone outweighs the window): the delta fills page [3,4]" - " and allocates [5,6],[7,8]; expiry must run across the appended" - " pages too, expiring every old page and three of the five new" - " rows to retain [7,8] at total 10. A backend that expires only" - " among pre-existing pages -- the natural shape of a drop-prefix /" - " append-suffix protocol -- retains four. touched_pages is 1: the" - " page appended into counts although this same advance then" - " expires it wholesale" - ), - anchor_job_issued_at_ms=anchor_ms, - window_weight=8, - page_size=2, - snapshot_records=snapshot, - advances=(WindowPipelineAdvance(anchor_job_issued_at_ms=anchor_ms + 1, delta_records=delta),), - ) - - -def _advance_partial_expiry_in_appended_page_case() -> WindowPipelineCase: - anchor_ms = 120_000 - snapshot = tuple( - corpus_record(share_seq, share_difficulty=3, job_issued_at_ms=anchor_ms - share_seq, accepted_at_ms=anchor_ms - share_seq) - for share_seq in (1, 2, 3) - ) - delta = tuple( - corpus_record(share_seq, share_difficulty=difficulty, job_issued_at_ms=anchor_ms + 1, accepted_at_ms=anchor_ms + 1) - for share_seq, difficulty in zip((4, 5, 6, 7, 8), (5, 5, 5, 2, 9)) - ) - return WindowPipelineCase( - name="advance-partial-expiry-in-appended-page", - why=( - "as advance-delta-exceeds-window but with delta difficulties" - " (5,5,5,2,9): whole-page expiry stops at the freshly allocated" - " page [7,8] (total 11) and the partial loop then expires seq 7" - " inside it, retaining [8] at total 9; touched_pages stays 1" - " because a freshly allocated append page never counts even when" - " partial expiry lands in it" - ), - anchor_job_issued_at_ms=anchor_ms, - window_weight=8, - page_size=2, - snapshot_records=snapshot, - advances=(WindowPipelineAdvance(anchor_job_issued_at_ms=anchor_ms + 1, delta_records=delta),), - ) - - -def _advance_from_empty_window_case() -> WindowPipelineCase: - anchor_ms = 130_000 - return WindowPipelineCase( - name="advance-from-empty-window", - why=( - "an empty window (no pages, no last page to append into) advanced" - " with a one-record delta: the record lands in a freshly allocated" - " page, nothing expires, touched_pages is 0; a backend that" - " unwraps the last page fails here" - ), - anchor_job_issued_at_ms=anchor_ms, - window_weight=10, - page_size=DEFAULT_INCREMENTAL_SHARE_WINDOW_PAGE_SIZE, - snapshot_records=(), - advances=( - WindowPipelineAdvance( - anchor_job_issued_at_ms=anchor_ms + 1, - delta_records=( - corpus_record(1, share_difficulty=5, job_issued_at_ms=anchor_ms + 1, accepted_at_ms=anchor_ms + 1), - ), - ), - ), - ) - - -def _advance_empty_delta_case() -> WindowPipelineCase: - anchor_ms = 140_000 - return WindowPipelineCase( - name="advance-empty-delta", - why=( - "one retained record advanced to a later anchor with an empty" - " delta: bytes identical to the snapshot, stats all zero; a backend" - " that allocates an empty page and frames a trailing separator, or" - " re-runs expiry with a stale total, diverges" - ), - anchor_job_issued_at_ms=anchor_ms, - window_weight=10, - page_size=DEFAULT_INCREMENTAL_SHARE_WINDOW_PAGE_SIZE, - snapshot_records=( - corpus_record(1, share_difficulty=5, job_issued_at_ms=anchor_ms, accepted_at_ms=anchor_ms), - ), - advances=(WindowPipelineAdvance(anchor_job_issued_at_ms=anchor_ms + 1, delta_records=()),), - ) - - -def _advance_equal_anchor_case() -> WindowPipelineCase: - anchor_ms = 150_000 - return WindowPipelineCase( - name="advance-equal-anchor", - why=( - "an advance at the same anchor as the snapshot: the regression" - " check is strictly <, so this is accepted (only an empty delta" - " can be, since no record is both newly eligible and eligible at" - " the same anchor); a backend rejecting on <= diverges" - ), - anchor_job_issued_at_ms=anchor_ms, - window_weight=10, - page_size=DEFAULT_INCREMENTAL_SHARE_WINDOW_PAGE_SIZE, - snapshot_records=( - corpus_record(1, share_difficulty=5, job_issued_at_ms=anchor_ms, accepted_at_ms=anchor_ms), - ), - advances=(WindowPipelineAdvance(anchor_job_issued_at_ms=anchor_ms, delta_records=()),), - ) - - -# --- rejection cases -------------------------------------------------------- -# -# One case per condition the shipped pipeline refuses, each built to trigger -# exactly that condition and nothing earlier in the check order. The three -# from_full_snapshot duplicate/non-positive shapes are unreachable from the -# database (PK, UNIQUE, CHECK), but the in-memory ledger and the daemon -# protocol are not the database, so they are pinned too. What is pinned is -# the rejection itself, not the coordinator's recovery from it. - -_REJECTION_ANCHOR_MS = 200_000 - - -def _rejection_case( - name: str, - category: str, - why: str, - *, - snapshot_records: tuple[AcceptedShareRecord, ...], - window_weight: int = 100, - page_size: int = DEFAULT_INCREMENTAL_SHARE_WINDOW_PAGE_SIZE, - advances: tuple[WindowPipelineAdvance, ...] = (), -) -> WindowPipelineCase: - return WindowPipelineCase( - name=name, - why=why, - anchor_job_issued_at_ms=_REJECTION_ANCHOR_MS, - window_weight=window_weight, - page_size=page_size, - snapshot_records=snapshot_records, - advances=advances, - expected_rejection=category, - ) - - -def _eligible(share_seq: int, *, share_difficulty: int = 1, share_id: str | None = None) -> AcceptedShareRecord: - """A record eligible at the rejection anchor, at one ms per seq of age.""" - record = corpus_record( - share_seq, - share_difficulty=share_difficulty, - job_issued_at_ms=_REJECTION_ANCHOR_MS - share_seq, - accepted_at_ms=_REJECTION_ANCHOR_MS - share_seq, - ) - return record if share_id is None else replace(record, share_id=share_id) - - -def _newly_eligible(share_seq: int, *, share_difficulty: int = 1, job_offset_ms: int = 1, accepted_offset_ms: int = 1) -> AcceptedShareRecord: - """A delta record first eligible at anchor+1 (both stamps above the old anchor).""" - return corpus_record( - share_seq, - share_difficulty=share_difficulty, - job_issued_at_ms=_REJECTION_ANCHOR_MS + job_offset_ms, - accepted_at_ms=_REJECTION_ANCHOR_MS + accepted_offset_ms, - ) - - -def _full_snapshot_rejection_cases() -> tuple[WindowPipelineCase, ...]: - return ( - _rejection_case( - "reject-duplicate-share-seq", - "duplicate_share_seq", - "two eligible records share share_seq 1 under distinct share_ids;" - " the sorted-order duplicate check fires before the share_id one", - snapshot_records=(_eligible(1), _eligible(1, share_difficulty=2, share_id="share-1-again")), - ), - _rejection_case( - "reject-duplicate-share-id", - "duplicate_share_id", - "seqs 1 and 2 carry the same share_id; distinct seqs so only the" - " share_id check can fire", - snapshot_records=(_eligible(1), _eligible(2, share_id="share-1")), - ), - _rejection_case( - "reject-non-positive-difficulty", - "non_positive_difficulty", - "one eligible record with share_difficulty 0 (the check runs on" - " eligible records only, so it must be eligible to be refused)", - snapshot_records=(_eligible(1, share_difficulty=0),), - ), - _rejection_case( - "reject-non-positive-window-weight", - "non_positive_window_weight", - "window_weight 0 with one ordinary eligible record; refused before" - " any record is examined", - snapshot_records=(_eligible(1),), - window_weight=0, - ), - _rejection_case( - "reject-non-positive-page-size", - "non_positive_page_size", - "page_size 0 with one ordinary eligible record; refused before any" - " record is examined", - snapshot_records=(_eligible(1),), - page_size=0, - ), - ) - - -def _advance_rejection_cases() -> tuple[WindowPipelineCase, ...]: - # Seqs 1 and 3 retained at the anchor; seq 2 is the gap a late-visible - # row would fill. - retained = (_eligible(1), _eligible(3)) - - def advance(*delta: AcceptedShareRecord, anchor_offset_ms: int = 1) -> tuple[WindowPipelineAdvance, ...]: - return ( - WindowPipelineAdvance( - anchor_job_issued_at_ms=_REJECTION_ANCHOR_MS + anchor_offset_ms, - delta_records=delta, - ), - ) - - return ( - _rejection_case( - "reject-anchor-regression", - "anchor_regression", - "an advance whose anchor is one ms earlier than the window's, with" - " an empty delta: the anchor check fires before any record check", - snapshot_records=retained, - advances=advance(anchor_offset_ms=-1), - ), - _rejection_case( - "reject-delta-non-positive-difficulty", - "delta_non_positive_difficulty", - "a newly eligible delta record of difficulty 0; the difficulty" - " check is the first per-record check", - snapshot_records=retained, - advances=advance(_newly_eligible(4, share_difficulty=0)), - ), - _rejection_case( - "reject-delta-ineligible-job-only", - "delta_ineligible_at_anchor", - "a delta record whose job_issued_at_ms is one ms past the new" - " anchor while accepted_at_ms is on it: ineligible by job alone", - snapshot_records=retained, - advances=advance(_newly_eligible(4, job_offset_ms=2, accepted_offset_ms=1)), - ), - _rejection_case( - "reject-delta-ineligible-accepted-only", - "delta_ineligible_at_anchor", - "a delta record whose accepted_at_ms is one ms past the new anchor" - " while job_issued_at_ms is on it: ineligible by accepted alone", - snapshot_records=retained, - advances=advance(_newly_eligible(4, job_offset_ms=1, accepted_offset_ms=2)), - ), - _rejection_case( - "reject-delta-repeats-retained-share", - "delta_repeats_eligible_share", - "the delta re-delivers seq 3, a share eligible at the previous" - " anchor and still inside the window: refused as a repeat before" - " the append-order check could see its seq", - snapshot_records=retained, - advances=advance(_eligible(3)), - ), - _rejection_case( - "reject-delta-not-append", - "delta_not_append", - "a newly eligible record with seq 2, below the window's last seq" - " 3: the late-visible row that cannot be folded as an append and" - " in production forces the full rescan", - snapshot_records=retained, - advances=advance(_newly_eligible(2)), - ), - _rejection_case( - "reject-delta-order-not-increasing", - "delta_order_not_increasing", - "two newly eligible records delivered as seqs 5 then 4, both" - " above the window's last seq so only the intra-delta order check" - " can fire", - snapshot_records=retained, - advances=advance(_newly_eligible(5), _newly_eligible(4)), - ), - ) - - -def build_corpus() -> tuple[WindowPipelineCase, ...]: - """Every corpus case, deterministic and in a stable order.""" - return ( - _empty_window_case(), - _single_record_case(), - _crossing_row_retained_case(), - _crossing_row_exact_fit_case(), - _eligibility_filtering_case(), - _unsorted_input_case(), - _identity_reuse_case(), - _non_ascii_strings_case(), - _page_boundary_case(DEFAULT_INCREMENTAL_SHARE_WINDOW_PAGE_SIZE - 1, seed_offset=8), - _page_boundary_case(DEFAULT_INCREMENTAL_SHARE_WINDOW_PAGE_SIZE, seed_offset=9), - _page_boundary_case(DEFAULT_INCREMENTAL_SHARE_WINDOW_PAGE_SIZE + 1, seed_offset=10), - _multi_page_case(), - _incremental_small_case(), - _bulk_seeded_case(), - _wide_integers_case(), - _difficulty_beyond_u128_case(), - _ntime_beyond_u32_case(), - _advance_exact_fit_expiry_case(), - _advance_delta_exceeds_window_case(), - _advance_partial_expiry_in_appended_page_case(), - _advance_from_empty_window_case(), - _advance_empty_delta_case(), - _advance_equal_anchor_case(), - *_full_snapshot_rejection_cases(), - *_advance_rejection_cases(), - ) - - -# --- frozen reference ------------------------------------------------------- - - -def reference_case_entry(case: WindowPipelineCase, result: WindowPipelineResult) -> dict[str, object]: - entry: dict[str, object] = { - "why": case.why, - "input_sha256": case_input_sha256(case), - } - literals: dict[str, object] = {} - if len(case_input_bytes(case)) <= LITERAL_PIN_MAX_CANONICAL_BYTES: - literals["input"] = case_input_document(case) - if isinstance(result, WindowPipelineUnsupported): - raise RuntimeError( - f"case {case.name}: the shipped pipeline declared the case" - f" unsupported ({result.reason}); the reference freezes only the" - " shipped pipeline's outputs or rejections" - ) - if isinstance(result, WindowPipelineRejection): - entry["rejected"] = result.reason - else: - outputs = result - stream = record_stream_bytes(outputs.record_jsons) - if outputs.canonical_digest != sha256_hex(outputs.canonical_bytes): - raise RuntimeError( - f"case {case.name}: canonical digest does not hash the framed" - " canonical bytes; the streaming model no longer matches the" - " shipped pipeline" - ) - entry.update( - { - "record_count": len(outputs.record_jsons), - "record_stream_len": len(stream), - "record_stream_sha256": sha256_hex(stream), - "canonical_bytes_len": len(outputs.canonical_bytes), - "canonical_bytes_sha256": sha256_hex(outputs.canonical_bytes), - "canonical_digest": outputs.canonical_digest, - "spool_tail_len": len(outputs.spool_tail), - "spool_tail_sha256": sha256_hex(outputs.spool_tail), - "advance_stats": [stats.as_document() for stats in outputs.advance_stats], - } - ) - if len(outputs.canonical_bytes) <= LITERAL_PIN_MAX_CANONICAL_BYTES: - literals.update( - { - "record_jsons": [record.decode("ascii") for record in outputs.record_jsons], - "canonical_bytes": outputs.canonical_bytes.decode("ascii"), - "spool_tail": outputs.spool_tail.decode("ascii"), - } - ) - if literals: - entry["pinned_literals"] = literals - return entry - - -def reference_document() -> dict[str, object]: - """The frozen-reference document, regenerated from the shipped pipeline.""" - adapter = PythonWindowPipelineAdapter() - return { - "schema": REFERENCE_SCHEMA, - "regenerate_with": REGENERATE_COMMAND, - "master_seed": f"0x{PARITY_RANDOM_MASTER_SEED:08x}", - "default_page_size": DEFAULT_INCREMENTAL_SHARE_WINDOW_PAGE_SIZE, - "cases": {case.name: reference_case_entry(case, adapter.run(case)) for case in build_corpus()}, - } - - -def encode_reference_document(document: dict[str, object]) -> bytes: - # sort_keys plus the default ensure_ascii keeps the fixture byte-stable, - # so regeneration on an unchanged tree is a no-op. - return json.dumps(document, indent=2, sort_keys=True).encode("ascii") + b"\n" - - -def load_reference_document() -> dict[str, object]: - try: - raw = REFERENCE_FIXTURE_PATH.read_bytes() - except FileNotFoundError: - raise AssertionError( - f"frozen reference {REFERENCE_FIXTURE_RELPATH} is missing;" - f" regenerate it with: {REGENERATE_COMMAND}" - ) from None - document = json.loads(raw) - if document.get("schema") != REFERENCE_SCHEMA: - raise AssertionError( - f"frozen reference {REFERENCE_FIXTURE_RELPATH} has schema" - f" {document.get('schema')!r}, expected {REFERENCE_SCHEMA!r};" - f" regenerate it with: {REGENERATE_COMMAND}" - ) - return document - - -def diverging_outputs( - reference_case: dict[str, object], - result: WindowPipelineResult, - *, - domain_exclusion: str | None = None, -) -> list[str]: - """Every way one backend's result diverges from one frozen case. - - An empty list is parity. A rejection matches only a frozen rejection of - the same category; outputs match only a frozen outputs entry, where - length and SHA-256 comparisons hold for every case, literal comparisons - additionally localize the first divergence when the case pins literals, - and the per-advance stats must agree exactly. - - ``domain_exclusion`` is why the case lies outside the backend's declared - integer domain (:func:`domain_exclusion`), or None when it lies inside. - An ``Unsupported`` result is parity exactly when the case is outside: - the frozen reference keeps the shipped pipeline's result and the backend - has declared, rather than mis-rendered, the input it cannot carry. A - backend that answers ``Unsupported`` for a case inside its declared - domain diverges, and a backend that produces bytes for a case outside - its domain is held to the frozen bytes like any other. - """ - expected_rejection = reference_case.get("rejected") - if isinstance(result, WindowPipelineUnsupported): - if domain_exclusion is None: - return [ - f"unsupported: backend declared {result.reason!r} but the" - " case is inside its declared integer domain" - ] - return [] - if isinstance(result, WindowPipelineRejection): - if expected_rejection is None: - return [ - f"rejected: backend rejected as {result.reason!r} but the" - " frozen reference has outputs" - ] - if result.reason != expected_rejection: - return [f"rejected: {result.reason!r} != frozen {expected_rejection!r}"] - return [] - outputs = result - if expected_rejection is not None: - return [ - f"rejected: backend produced outputs but the frozen reference is" - f" rejected as {expected_rejection!r}" - ] - - problems: list[str] = [] - - def check_stream(label: str, actual: bytes, literal: str | None) -> None: - expected_len = reference_case[f"{label}_len"] - expected_sha = reference_case[f"{label}_sha256"] - if len(actual) != expected_len: - problems.append(f"{label}: {len(actual)} bytes != frozen {expected_len}") - if sha256_hex(actual) != expected_sha: - problems.append(f"{label}: sha256 diverges from the frozen reference") - if literal is not None and literal.encode("ascii") != actual: - problems.append(f"{label}: bytes diverge from the frozen literal") - - literals = reference_case.get("pinned_literals") or {} - literal_records = literals.get("record_jsons") - if len(outputs.record_jsons) != reference_case["record_count"]: - problems.append( - f"record_count: {len(outputs.record_jsons)} != frozen {reference_case['record_count']}" - ) - check_stream( - "record_stream", - record_stream_bytes(outputs.record_jsons), - "\n".join(literal_records) if literal_records is not None else None, - ) - if literal_records is not None: - expected_records = [text.encode("ascii") for text in literal_records] - for index, (expected, actual) in enumerate(zip(expected_records, outputs.record_jsons)): - if expected != actual: - problems.append( - f"record_jsons[{index}]: {actual!r} != frozen {expected!r}" - ) - break - check_stream("canonical_bytes", outputs.canonical_bytes, literals.get("canonical_bytes")) - if outputs.canonical_digest != reference_case["canonical_digest"]: - problems.append( - f"canonical_digest: {outputs.canonical_digest} != frozen" - f" {reference_case['canonical_digest']}" - ) - check_stream("spool_tail", outputs.spool_tail, literals.get("spool_tail")) - actual_stats = [stats.as_document() for stats in outputs.advance_stats] - if actual_stats != reference_case["advance_stats"]: - problems.append( - f"advance_stats: {actual_stats} != frozen {reference_case['advance_stats']}" - ) - return problems - - -def _main(argv: list[str]) -> int: - if argv != ["regenerate"]: - print(f"usage: {REGENERATE_COMMAND}", file=sys.stderr) - return 2 - payload = encode_reference_document(reference_document()) - REFERENCE_FIXTURE_PATH.parent.mkdir(parents=True, exist_ok=True) - if REFERENCE_FIXTURE_PATH.exists() and REFERENCE_FIXTURE_PATH.read_bytes() == payload: - print(f"unchanged: {REFERENCE_FIXTURE_RELPATH}") - return 0 - REFERENCE_FIXTURE_PATH.write_bytes(payload) - print(f"wrote {len(payload)} bytes: {REFERENCE_FIXTURE_RELPATH}") - return 0 - - -if __name__ == "__main__": - sys.exit(_main(sys.argv[1:])) diff --git a/tests/window_pipeline_parity_gate.py b/tests/window_pipeline_parity_gate.py deleted file mode 100644 index 010212217..000000000 --- a/tests/window_pipeline_parity_gate.py +++ /dev/null @@ -1,79 +0,0 @@ -#!/usr/bin/env python3 -"""Strict CI entry point for the frozen payout-window parity corpus.""" - -from __future__ import annotations - -import os -import sys -import unittest -from pathlib import Path - -from lab.prism.prism_tools import prism_tool_command -from tests import window_pipeline_parity as parity - - -PARITY_TEST_TARGET = ( - "tests.test_window_pipeline_parity.WindowPipelineParityOracleTests" -) -RUST_DAEMON_BINARY = "qbit-prism-build-audit-bundle" - - -def resolve_gate_adapter() -> tuple[parity.WindowPipelineAdapter, tuple[str, ...]]: - """Resolve the explicitly selected adapter and its required prebuilt tool.""" - requested = os.environ.get(parity.ADAPTER_ENV_VAR) - if not requested: - raise RuntimeError( - f"{parity.ADAPTER_ENV_VAR} must explicitly select a parity adapter" - ) - adapter = parity.resolve_adapter(requested) - if adapter.name != requested: - raise RuntimeError( - f"resolved parity adapter {adapter.name!r}, expected {requested!r}" - ) - - if requested != "rust-daemon": - return adapter, () - - bin_dir = os.environ.get("PRISM_TOOL_BIN_DIR") - if not bin_dir: - raise RuntimeError( - "rust-daemon parity requires PRISM_TOOL_BIN_DIR to select the" - " prebuilt deployment binary" - ) - expected = Path(bin_dir) / RUST_DAEMON_BINARY - command = tuple(prism_tool_command(RUST_DAEMON_BINARY)) - if command != (str(expected),): - raise RuntimeError( - f"rust-daemon parity requires executable prebuilt binary {expected};" - " refusing cargo fallback" - ) - return adapter, command - - -def main() -> int: - try: - adapter, command = resolve_gate_adapter() - except (RuntimeError, ValueError) as error: - print(f"payout-window parity gate: {error}", file=sys.stderr) - return 2 - - corpus = parity.build_corpus() - advance_cases = sum(bool(case.advances) for case in corpus) - advance_steps = sum(len(case.advances) for case in corpus) - print(f"payout-window parity adapter: {adapter.name}", flush=True) - if command: - print(f"payout-window parity daemon: {command[0]} --serve", flush=True) - print( - "payout-window frozen corpus:" - f" {len(corpus)} full preparations," - f" {advance_steps} advance() calls across {advance_cases} cases", - flush=True, - ) - - suite = unittest.defaultTestLoader.loadTestsFromName(PARITY_TEST_TARGET) - result = unittest.TextTestRunner(verbosity=2).run(suite) - return 0 if result.wasSuccessful() else 1 - - -if __name__ == "__main__": - sys.exit(main()) From b9579c675fae38343f9bbceac86585de9a1df719 Mon Sep 17 00:00:00 2001 From: Alex <alex@swaplabs.xyz> Date: Mon, 26 Oct 2026 14:35:03 -0400 Subject: [PATCH 09/10] main: update tokio for RUSTSEC-2026-0386, and keep the block-proof acknowledgement bound from dropping a disposition poll's connection (#781) * main: update tokio to 1.53.3 for RUSTSEC-2026-0386 The Dependency audit job (`cargo deny --locked check`) fails on RUSTSEC-2026-0386: tokio 1.53.1's safe `AsyncFd` constructors violate I/O safety. Move tokio to its first fixed patch release, 1.53.3, in the root and fuzz lockfiles, as #777 did for the TLS advisories. Nothing in the workspace uses `AsyncFd`. Only tokio's version and checksum change. `cargo update -p tokio --precise 1.53.3` also moved the windows-sys edges of four unrelated crates (errno, quinn-udp, rustix, tempfile) from 0.61.2 to 0.52.0, so the edit keeps those as they were; both lockfiles pass `cargo metadata --locked`. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> * main: keep the block-proof acknowledgement bound from dropping a disposition poll's connection b574_ack_cap's a_captured_proof_is_answered_at_the_cap_and_credited_after_its_landing failed on #780 with `proxy connection task failed: Connection reset by peer (os error 104)`, which the ledger execution proxy reported when the fixture closed. `deferred_share_disposition` wrapped each poll's checkout and SQL in `timeout_at(bound, ...)`. When the bound fell inside the checkout, during the pool's `before_acquire` ping of a connection idle for 100 ms or a new connection's setup, SQLx dropped the raw connection with the reply unread, and the kernel answered with a TCP reset. The test proxy reports that as a failed connection; PostgreSQL logs it as a client reset, and the pool loses a healthy connection. Each poll now runs in its own task, as the duplicate probe and enqueue do (#574), and only the wait for it is bounded. Answers, phases and `block_proof_ack_capped_total` are unchanged. A poll still running at the bound, or when the submission is cancelled, finishes on its own and returns its connection, so `database_pool_acquire_seconds` records its checkout when it ends instead of as a cancelled failure. Two acquire tests and docs/prism-pool-acquire-metrics.md now say so. A bound already past on entry, as for a #657 capture that committed in the share grace, starts no poll: the old code's poll there could only be cut off. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> --------- Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com> --- Cargo.lock | 4 +- crates/qbit-prism-server/fuzz/Cargo.lock | 4 +- .../src/coordinator/miner_submit.rs | 38 ++++++++++++++----- .../coordinator/miner_submit_acquire_tests.rs | 18 ++++++--- docs/prism-pool-acquire-metrics.md | 22 ++++++----- scripts/prism_pool_acquires.json | 2 +- 6 files changed, 59 insertions(+), 29 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 5a2f431f2..335785cb2 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2340,9 +2340,9 @@ checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" [[package]] name = "tokio" -version = "1.53.1" +version = "1.53.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "202caea871b69668250d242070849eb495be178ed697a3e98aebce5bc81a0bed" +checksum = "ce3335fa71841cda333a58d7615b03901380ecf09d59b3296d21f8bbac0dde4e" dependencies = [ "bytes", "libc", diff --git a/crates/qbit-prism-server/fuzz/Cargo.lock b/crates/qbit-prism-server/fuzz/Cargo.lock index e1e9102fc..5c8e7d75c 100644 --- a/crates/qbit-prism-server/fuzz/Cargo.lock +++ b/crates/qbit-prism-server/fuzz/Cargo.lock @@ -2235,9 +2235,9 @@ checksum = "fd3ca314f692efd6c868f8408f53fe444634a845f96c028b97d35f6a1f79f0ee" [[package]] name = "tokio" -version = "1.53.1" +version = "1.53.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "202caea871b69668250d242070849eb495be178ed697a3e98aebce5bc81a0bed" +checksum = "ce3335fa71841cda333a58d7615b03901380ecf09d59b3296d21f8bbac0dde4e" dependencies = [ "bytes", "libc", diff --git a/crates/qbit-prism-server/src/coordinator/miner_submit.rs b/crates/qbit-prism-server/src/coordinator/miner_submit.rs index 4ab0a3ad1..fdc734469 100644 --- a/crates/qbit-prism-server/src/coordinator/miner_submit.rs +++ b/crates/qbit-prism-server/src/coordinator/miner_submit.rs @@ -1010,6 +1010,12 @@ impl Coordinator { (capped, log_path): (BlockAckPath, &'static str), ) -> SaveOutcome { loop { + // A bound already past, as for a capture whose append committed + // in the share grace (#657), starts no poll: its answer would + // arrive too late to use. + if tokio::time::Instant::now() >= bound { + break; + } // Observe credit and disposition in one MVCC snapshot so // finalization cannot fall between two separate reads. The credit // this polls for is written by the reconciliation of the block @@ -1017,16 +1023,30 @@ impl Coordinator { // leaves and carries the same probe floor as the probe above: // without it, every poll of this loop would descend one per-leaf // `share_id` index per attached partition. - let poll = tokio::time::timeout_at(bound, async { - sqlx::query_as::<_, (bool, Option<String>, Option<String>, Option<String>)>( - "SELECT EXISTS(SELECT 1 FROM qbit_share_ledger WHERE share_id=$1 AND share_seq>=qbit_prism_share_probe_floor()), (SELECT state FROM qbit_block_candidate_outbox WHERE block_hash=$2), (SELECT offer_outcome FROM qbit_block_candidate_outbox WHERE block_hash=$2), (SELECT last_error FROM qbit_block_candidate_outbox WHERE block_hash=$2)", - ) - .bind(share_id) - .bind(block_hash) - .fetch_one(&mut *self.ledger.acquire().await?) + // + // Each poll runs in its own task, as the probe and enqueue do, and + // only the wait for it is bounded. A bound that fell inside the + // poll's checkout, during the pool's probe of an idle connection + // or a new connection's setup, would drop that connection with + // its reply unread: the server sees a reset and the pool loses a + // healthy connection. Past the bound the poll finishes on its own + // and returns its connection. + let poll = tokio::spawn({ + let ledger = self.ledger.clone(); + let (share_id, block_hash) = (share_id.to_owned(), block_hash.to_owned()); + async move { + sqlx::query_as::<_, (bool, Option<String>, Option<String>, Option<String>)>( + "SELECT EXISTS(SELECT 1 FROM qbit_share_ledger WHERE share_id=$1 AND share_seq>=qbit_prism_share_probe_floor()), (SELECT state FROM qbit_block_candidate_outbox WHERE block_hash=$2), (SELECT offer_outcome FROM qbit_block_candidate_outbox WHERE block_hash=$2), (SELECT last_error FROM qbit_block_candidate_outbox WHERE block_hash=$2)", + ) + .bind(share_id) + .bind(block_hash) + .fetch_one(&mut *ledger.acquire().await?) + .await + } + }); + let poll = tokio::time::timeout_at(bound, poll) .await - }) - .await; + .map(|joined| anyhow::Ok(joined??)); match poll { Err(_) => break, Ok(Ok((true, _, _, _))) => return SaveOutcome::Accepted, diff --git a/crates/qbit-prism-server/src/coordinator/miner_submit_acquire_tests.rs b/crates/qbit-prism-server/src/coordinator/miner_submit_acquire_tests.rs index 9626b29c8..e2010eeef 100644 --- a/crates/qbit-prism-server/src/coordinator/miner_submit_acquire_tests.rs +++ b/crates/qbit-prism-server/src/coordinator/miner_submit_acquire_tests.rs @@ -535,7 +535,6 @@ async fn block_only_poll_checkout_cancel_and_deadline_stay_unknown() -> Result<( let mut persist = Box::pin(h.persist(start)); let held = first_poll(h, &mut persist).await?; let before = counts(h.metrics()); - let sum = sample(h.metrics(), "failure", "sum"); tokio::time::pause(); let resume = ResumeClock; tokio::time::advance(Duration::from_millis(50)).await; @@ -543,10 +542,13 @@ async fn block_only_poll_checkout_cancel_and_deadline_stay_unknown() -> Result<( assert_eq!(counts(h.metrics()), before); tokio::time::advance(Duration::from_millis(75)).await; drop(persist); - assert_eq!(counts(h.metrics()), (before.0, before.1 + 1.)); - assert!((sample(h.metrics(), "failure", "sum") - sum - 0.075).abs() < 0.000001); + // The poll runs in a task the submission does not own, so a + // cancelled submission leaves its checkout waiting, not failed, + // and the poll still completes. + assert_eq!(counts(h.metrics()), before); drop(resume); drop(held); + wait_counts(h.metrics(), (before.0 + 1., before.1)).await?; let deferred: Value = sqlx::query_scalar( "SELECT share FROM qbit_prism_deferred_shares WHERE block_hash=$1", ) @@ -584,9 +586,12 @@ async fn block_only_poll_checkout_cancel_and_deadline_stay_unknown() -> Result<( .. } )); - assert_eq!(counts(h.metrics()), (before.0, before.1 + 1.)); + // The bound ends only the wait: the poll's checkout is not + // cancelled with it, and completes once the slot is free. + assert_eq!(counts(h.metrics()), before); drop(resume); drop(held); + wait_counts(h.metrics(), (before.0 + 1., before.1)).await?; Ok(()) }) }) @@ -721,9 +726,12 @@ async fn block_only_poll_missing_and_sql_error_stay_unknown() -> Result<()> { panic!("nonterminal: {answer:?}") }; assert_eq!(phase, expected_phase); - assert_eq!(counts(h.metrics()), (before.0 + 1., before.1 + 1.)); + // The poll still waiting for the slot at the bound is not + // cancelled with the answer; it checks out once released. + assert_eq!(counts(h.metrics()), (before.0 + 1., before.1)); drop(resume); drop(held); + wait_counts(h.metrics(), (before.0 + 2., before.1)).await?; drop(tokio::time::timeout(WAIT, h.ledger().pool.acquire()).await??); Ok(()) }) diff --git a/docs/prism-pool-acquire-metrics.md b/docs/prism-pool-acquire-metrics.md index 57d15be3e..13ec4bbc7 100644 --- a/docs/prism-pool-acquire-metrics.md +++ b/docs/prism-pool-acquire-metrics.md @@ -116,16 +116,18 @@ unchanged. The coordinator database clock and chain-observation state each record one checkout. The latter retains its one-statement revision/epoch/tip snapshot. Each block-only duplicate probe and each credit/disposition poll records its -own checkout, inside the -original acknowledgement deadline. The two block-only probes are not issued-job -reads. Credit and disposition still share one SQL statement and MVCC snapshot. -Connections are released at the statement boundary, before enqueue or poll -sleep. Missing rows, duplicates and terminal dispositions retain successful -checkout observations; SQL errors or cancellation after acquisition do too. -Cancellation while acquiring records a failure. Before enqueue a read failure -is definite; after enqueue a failed or missing read does not prove whether the -durable candidate earned credit. Timing changes neither that distinction nor -the enqueue's existing handling of an unknown outcome. +own checkout. Each runs in a task the acknowledgement does not own (#574), so +neither the acknowledgement deadline nor a cancelled submission cancels its +checkout or SQL: a checkout still waiting then records its own outcome when it +ends, and a poll that answers after the deadline is discarded. The two +block-only probes are not issued-job reads. Credit and disposition still share +one SQL statement and MVCC snapshot. Connections are released at the statement +boundary, before enqueue or poll sleep. Missing rows, duplicates and terminal +dispositions retain successful checkout observations; SQL errors do too. +Before enqueue a read failure is definite; after enqueue a failed or missing +read does not prove whether the durable candidate earned credit. Timing changes +neither that distinction nor the enqueue's existing handling of an unknown +outcome. The server passes its existing metrics handle to the rollup loop. Each valid rollup attempt records one checkout before `BEGIN`; `SET LOCAL`, the single diff --git a/scripts/prism_pool_acquires.json b/scripts/prism_pool_acquires.json index ce610a734..b94a03d6b 100644 --- a/scripts/prism_pool_acquires.json +++ b/scripts/prism_pool_acquires.json @@ -142,7 +142,7 @@ "coordinator.rs::submit_loop_with#1::attempt .run(& self , lease)": [1,"non-database"], "coordinator.rs::with_tracked_heartbeat#1::self . ledger .acquire()": [2,"ledger"], "coordinator/issued_batcher.rs::with_dwell#1::run(ledger , receiver , shutdown . clone ( ) , dwell)": [1,"non-database"], - "coordinator/miner_submit.rs::deferred_share_disposition#1::self . ledger .acquire()": [1,"ledger"], + "coordinator/miner_submit.rs::deferred_share_disposition#1::ledger .acquire()": [1,"ledger"], "coordinator/miner_submit.rs::persist_block_only#1::ledger .acquire()": [1,"ledger"], "coordinator/refresh_window.rs::capture_refresh_window#1::prepare(std :: mem :: take ( & mut snapshot . snapshot . prior_balances ) , & admission ,)": [1,"non-database"], "coordinator/work_ledger.rs::clocked_payout_revision#1::self .acquire()": [1,"ledger"], From 6a09d3666d52f034a30dc264e48af614fb0d4506 Mon Sep 17 00:00:00 2001 From: Alexander Miller <alex@swaplabs.xyz> Date: Tue, 27 Oct 2026 09:57:56 -0400 Subject: [PATCH 10/10] v3.0.0 chore: sync released main into 3.0.x: current releasing guide Take docs/releasing.md as main has it after 3.1.0's promotion (#783), with the reconcile step saying what it merges into 3.x.x and 3.0.x, as the 3.x.x sync does, so that both branches carry the same guide. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> --- docs/releasing.md | 54 +++++++++++++++++++++++++++++++++-------------- 1 file changed, 38 insertions(+), 16 deletions(-) diff --git a/docs/releasing.md b/docs/releasing.md index 859185544..863a37654 100644 --- a/docs/releasing.md +++ b/docs/releasing.md @@ -1,11 +1,11 @@ # Publishing a release -`main` is the release promotion branch. `3.x.x` is the development branch for -the 3.x series, now carrying 3.1; `2.x.x` is the maintenance branch of the -2.x series. A release cut from an earlier point of a development branch gets -its own branch at that cutoff, as `3.0.x` does for 3.0.0. A version is -published only when its tag and GitHub release exist. A version bump or -merged preparation PR alone does not publish it. +`main` is the release promotion branch. `3.x.x` is the development branch and +production line for the 3.x series, whose current release is 3.1.0; `2.x.x` is +the maintenance branch of the 2.x series. A release cut from an earlier point +of a development branch gets its own branch at that cutoff, as `3.0.x` does +for 3.0.0. A version is published only when its tag and GitHub release exist. +A version bump or merged preparation PR alone does not publish it. ## Prepare one release candidate @@ -35,6 +35,18 @@ tag. The 3.0.0-rc.N pre-releases were tagged on `3.x.x`. 3.0.0-rc.7 was prepared (#772) but never tagged, and its changes ship in 3.0.0. The L3 matrix for the release is recorded in #779. +3.1.0 needed no release branch: `v3.1.0` was tagged and published on `3.x.x` +itself, at the squash of its version bump and notes (#782, `59194d03`), as +`v3.1.0-rc.1` was. The promotion into `main` gives `main` that tagged tree, +plus this guide, a README banner that names the version, and the released +3.0.0 notes in place of `3.x.x`'s pre-release copy; the links in the +development-line note, the migration guide and the ledger operations guide, +which pointed at the notes at the `v3.0.0` tag, point at that local copy. It +also keeps #781's disposition-poll change: 3.1.0 made #781's tokio update +itself (#782) but not the poll change, since it changed no runtime code after +its release candidate. It adds no tag. #779 records the 400k two-frontend L3 +failures on 3.0.0's tree; 3.1.0 doesn't fix their cause, #716. + Merge the intended development cutoff into the promotion branch and retain fixes already made during promotion review. The repository allows only squash and rebase merges, so the promotion lands on `main` as one squashed commit. Check package versions together, @@ -45,14 +57,16 @@ set the actual release date in a reviewed commit before the final merge. ## Publish the reviewed main commit Tag the promoted `main` commit unless the release was already tagged on its -release branch, as 3.0.0 was. After the promotion PR passes required checks -and is merged, identify the exact resulting commit on `main`. A squash or rebase merge produces a different -commit from the promotion branch head. Verify the resulting commit's checks, -version metadata and dated release notes before tagging it. +release or development branch, as 3.0.0 and 3.1.0 were. After the promotion PR +passes required checks and is merged, identify the exact resulting commit on +`main`. A squash or rebase merge produces a different commit from the +promotion branch head. Verify the resulting commit's checks, version metadata +and dated release notes before tagging it. After confirming that the tag doesn't already exist, use the reviewed main commit in place of `<release-main-sha>`, and the version in place of -`<version>` (as `v2.0.0` and `v3.0.0` were): +`<version>` (as `v2.0.0` was; `v3.0.0` and `v3.1.0` are signed annotated +tags on their branches' commits, with their own messages): ```sh git fetch origin main --tags @@ -77,11 +91,19 @@ release version. ## Reconcile the development branch After promotion, open a synchronization PR from a branch that merges the -released `main` commit into the latest development branch (`3.x.x` after -3.0.0, `2.x.x` after 2.0.0), and into a release branch such as `3.0.x` that -will take patch releases. Preserve subsequent development -and resolve metadata and notes deliberately; do not reset or force-push the -shared branch. Run its checks before merging. +released `main` commit into the latest development branch (`3.x.x` after a 3.x +release, `2.x.x` after 2.0.0), and into a release branch such as `3.0.x` that +will take patch releases. Preserve subsequent development and resolve metadata +and notes deliberately; do not reset or force-push the shared branch. Run its +checks before merging. + +For 3.1.0 and 3.0.0 this step merges the released `main` (`b9187551`, #783) +into `3.x.x`, and `main` at `b9579c67` into `3.0.x`. That commit is 3.0.0's +promotion (#778) plus #781, before 3.1.0's, because a 3.0 patch branch must +not take the 3.1 dual writer. Both branches gain this guide, the README banner +and #781's disposition-poll change. `3.0.x` also gains #781's tokio update, +and `3.x.x` the released 3.0.0 notes in place of its pre-release copy. The +next release from either branch must not drop them. For 2.0.0 this step is required: PR #246 contains promotion fixes and the consolidated metadata that were not yet on `2.x.x`. Until this synchronization